4070 files changed, 252483 insertions, 91121 deletions

diff --git a/qemu/CODING_STYLE b/qemu/CODING_STYLE
index d46cfa5f6..3c6978f83 100644
--- a/qemu/CODING_STYLE
+++ b/qemu/CODING_STYLE
@@ -87,10 +87,15 @@ Furthermore, it is the QEMU coding style.
 
 5. Declarations
 
-Mixed declarations (interleaving statements and declarations within blocks)
-are not allowed; declarations should be at the beginning of blocks.  In other
-words, the code should not generate warnings if using GCC's
--Wdeclaration-after-statement option.
+Mixed declarations (interleaving statements and declarations within
+blocks) are generally not allowed; declarations should be at the beginning
+of blocks.
+
+Every now and then, an exception is made for declarations inside a
+#ifdef or #ifndef block: if the code looks nicer, such declarations can
+be placed at the top of the block even if there are statements above.
+On the other hand, however, it's often best to move that #ifdef/#ifndef
+block to a separate function altogether.
 
 6. Conditional statements
 
diff --git a/qemu/HACKING b/qemu/HACKING
index 12fbc8afe..058aa8fd4 100644
--- a/qemu/HACKING
+++ b/qemu/HACKING
@@ -157,3 +157,58 @@ painful. These are:
  * you may assume that integers are 2s complement representation
  * you may assume that right shift of a signed integer duplicates
    the sign bit (ie it is an arithmetic shift, not a logical shift)
+
+7. Error handling and reporting
+
+7.1 Reporting errors to the human user
+
+Do not use printf(), fprintf() or monitor_printf().  Instead, use
+error_report() or error_vreport() from error-report.h.  This ensures the
+error is reported in the right place (current monitor or stderr), and in
+a uniform format.
+
+Use error_printf() & friends to print additional information.
+
+error_report() prints the current location.  In certain common cases
+like command line parsing, the current location is tracked
+automatically.  To manipulate it manually, use the loc_*() from
+error-report.h.
+
+7.2 Propagating errors
+
+An error can't always be reported to the user right where it's detected,
+but often needs to be propagated up the call chain to a place that can
+handle it.  This can be done in various ways.
+
+The most flexible one is Error objects.  See error.h for usage
+information.
+
+Use the simplest suitable method to communicate success / failure to
+callers.  Stick to common methods: non-negative on success / -1 on
+error, non-negative / -errno, non-null / null, or Error objects.
+
+Example: when a function returns a non-null pointer on success, and it
+can fail only in one way (as far as the caller is concerned), returning
+null on failure is just fine, and certainly simpler and a lot easier on
+the eyes than propagating an Error object through an Error ** parameter.
+
+Example: when a function's callers need to report details on failure
+only the function really knows, use Error **, and set suitable errors.
+
+Do not report an error to the user when you're also returning an error
+for somebody else to handle.  Leave the reporting to the place that
+consumes the error returned.
+
+7.3 Handling errors
+
+Calling exit() is fine when handling configuration errors during
+startup.  It's problematic during normal operation.  In particular,
+monitor commands should never exit().
+
+Do not call exit() or abort() to handle an error that can be triggered
+by the guest (e.g., some unimplemented corner case in guest code
+translation or device emulation).  Guests should not be able to
+terminate QEMU.
+
+Note that &error_fatal is just another way to exit(1), and &error_abort
+is just another way to abort().
diff --git a/qemu/MAINTAINERS b/qemu/MAINTAINERS
index 978b7174f..81e7fac2f 100644
--- a/qemu/MAINTAINERS
+++ b/qemu/MAINTAINERS
@@ -52,6 +52,11 @@ General Project Administration
 ------------------------------
 M: Peter Maydell <peter.maydell@linaro.org>
 
+All patches CC here
+L: qemu-devel@nongnu.org
+F: *
+F: */
+
 Responsible Disclosure, Reporting Security Issues
 ------------------------------
 W: http://wiki.qemu.org/SecurityProcess
@@ -62,14 +67,29 @@ Guest CPU cores (TCG):
 ----------------------
 Overall
 L: qemu-devel@nongnu.org
-S: Odd fixes
+M: Paolo Bonzini <pbonzini@redhat.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+M: Richard Henderson <rth@twiddle.net>
+S: Maintained
 F: cpu-exec.c
+F: cpu-exec-common.c
+F: cpus.c
 F: cputlb.c
+F: exec.c
 F: softmmu_template.h
-F: translate-all.c
-F: include/exec/cpu_ldst.h
-F: include/exec/cpu_ldst_template.h
+F: translate-all.*
+F: translate-common.c
+F: include/exec/cpu*.h
+F: include/exec/exec-all.h
 F: include/exec/helper*.h
+F: include/exec/tb-hash.h
+
+FPU emulation
+M: Aurelien Jarno <aurelien@aurel32.net>
+M: Peter Maydell <peter.maydell@linaro.org>
+S: Odd Fixes
+F: fpu/
+F: include/fpu/
 
 Alpha
 M: Richard Henderson <rth@twiddle.net>
@@ -77,13 +97,18 @@ S: Maintained
 F: target-alpha/
 F: hw/alpha/
 F: tests/tcg/alpha/
+F: disas/alpha.c
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/
 F: hw/arm/
 F: hw/cpu/a*mpcore.c
+F: disas/arm.c
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 CRIS
 M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -91,6 +116,7 @@ S: Maintained
 F: target-cris/
 F: hw/cris/
 F: tests/tcg/cris/
+F: disas/cris.c
 
 LM32
 M: Michael Walle <michael@walle.cc>
@@ -114,6 +140,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: target-microblaze/
 F: hw/microblaze/
+F: disas/microblaze.c
 
 MIPS
 M: Aurelien Jarno <aurelien@aurel32.net>
@@ -122,11 +149,13 @@ S: Maintained
 F: target-mips/
 F: hw/mips/
 F: tests/tcg/mips/
+F: disas/mips.c
 
 Moxie
 M: Anthony Green <green@moxielogic.com>
 S: Maintained
 F: target-moxie/
+F: disas/moxie.c
 
 OpenRISC
 M: Jia Liu <proljc@gmail.com>
@@ -141,6 +170,7 @@ L: qemu-ppc@nongnu.org
 S: Maintained
 F: target-ppc/
 F: hw/ppc/
+F: disas/ppc.c
 
 S390
 M: Richard Henderson <rth@twiddle.net>
@@ -148,12 +178,14 @@ M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/
 F: hw/s390x/
+F: disas/s390.c
 
 SH4
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Odd Fixes
 F: target-sh4/
 F: hw/sh4/
+F: disas/sh4.c
 
 SPARC
 M: Blue Swirl <blauwirbel@gmail.com>
@@ -162,6 +194,7 @@ S: Maintained
 F: target-sparc/
 F: hw/sparc/
 F: hw/sparc64/
+F: disas/sparc.c
 
 UniCore32
 M: Guan Xuetao <gxt@mprc.pku.edu.cn>
@@ -176,6 +209,7 @@ M: Eduardo Habkost <ehabkost@redhat.com>
 S: Maintained
 F: target-i386/
 F: hw/i386/
+F: disas/i386.c
 
 Xtensa
 M: Max Filippov <jcmvbkbc@gmail.com>
@@ -200,9 +234,11 @@ L: kvm@vger.kernel.org
 S: Supported
 F: kvm-*
 F: */kvm.*
+F: include/sysemu/kvm*.h
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/kvm.c
 
@@ -222,9 +258,14 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/kvm.c
+F: target-s390x/ioinst.[ch]
+F: target-s390x/machine.c
 F: hw/intc/s390_flic.c
 F: hw/intc/s390_flic_kvm.c
 F: include/hw/s390x/s390_flic.h
+F: gdb-xml/s390*.xml
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 X86
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -237,11 +278,21 @@ Guest CPU Cores (Xen):
 ----------------------
 
 X86
-M: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
+M: Stefano Stabellini <sstabellini@kernel.org>
+M: Anthony Perard <anthony.perard@citrix.com>
 L: xen-devel@lists.xensource.com
 S: Supported
 F: xen-*
 F: */xen*
+F: hw/char/xen_console.c
+F: hw/display/xenfb.c
+F: hw/net/xen_nic.c
+F: hw/block/xen_*
+F: hw/xen/
+F: hw/xenpv/
+F: hw/i386/xen/
+F: include/hw/xen/
+F: include/sysemu/xen-mapcache.h
 
 Hosts:
 ------
@@ -262,103 +313,163 @@ L: qemu-devel@nongnu.org
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: *win32*
+F: qemu.nsi
 
 ARM Machines
 ------------
 Allwinner-a10
-M: Li Guang <lig.fnst@cn.fujitsu.com>
+M: Beniamino Galvani <b.galvani@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
-F: hw/*/allwinner-a10*
-F: include/hw/*/allwinner-a10*
+F: hw/*/allwinner*
+F: include/hw/*/allwinner*
 F: hw/arm/cubieboard.c
 
+ARM PrimeCell
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/char/pl011.c
+F: hw/display/pl110*
+F: hw/dma/pl080.c
+F: hw/dma/pl330.c
+F: hw/gpio/pl061.c
+F: hw/input/pl050.c
+F: hw/intc/pl190.c
+F: hw/sd/pl181.c
+F: hw/timer/pl031.c
+F: include/hw/arm/primecell.h
+
+ARM cores
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/intc/arm*
+F: hw/intc/gic_internal.h
+F: hw/misc/a9scu.c
+F: hw/misc/arm11scu.c
+F: hw/timer/a9gtimer*
+F: hw/timer/arm_*
+F: include/hw/arm/arm.h
+F: include/hw/intc/arm*
+F: include/hw/misc/a9scu.h
+F: include/hw/misc/arm11scu.h
+F: include/hw/timer/a9gtimer.h
+F: include/hw/timer/arm_mptimer.h
+
 Exynos
-M: Evgeny Voevodin <e.voevodin@samsung.com>
-M: Maksim Kozlov <m.kozlov@samsung.com>
 M: Igor Mitsyanko <i.mitsyanko@gmail.com>
-M: Dmitry Solodkiy <d.solodkiy@samsung.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/exynos*
+F: include/hw/arm/exynos4210.h
 
 Calxeda Highbank
 M: Rob Herring <robh@kernel.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/highbank.c
 F: hw/net/xgmac.c
 
 Canon DIGIC
 M: Antony Pavlov <antonynpavlov@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: include/hw/arm/digic.h
 F: hw/*/digic*
 
 Gumstix
 L: qemu-devel@nongnu.org
+L: qemu-arm@nongnu.org
 S: Orphan
 F: hw/arm/gumstix.c
 
 i.MX31
 M: Peter Chubb <peter.chubb@nicta.com.au>
+L: qemu-arm@nongnu.org
 S: Odd fixes
 F: hw/*/imx*
 F: hw/arm/kzm.c
+F: include/hw/arm/fsl-imx31.h
 
 Integrator CP
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/integratorcp.c
 
-Mainstone
-L: qemu-devel@nongnu.org
-S: Orphan
-F: hw/arm/mainstone.c
-
 Musicpal
 M: Jan Kiszka <jan.kiszka@web.de>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/musicpal.c
 
 nSeries
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/nseries.c
 
 Palm
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/palm.c
 
 Real View
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/realview*
+F: hw/intc/realview_gic.c
+F: include/hw/intc/realview_gic.h
 
-Spitz
+PXA2XX
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
+F: hw/arm/mainstone.c
 F: hw/arm/spitz.c
+F: hw/arm/tosa.c
+F: hw/arm/z2.c
+F: hw/*/pxa2xx*
+F: include/hw/arm/pxa.h
 
 Stellaris
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/stellaris*
 
 Versatile PB
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/versatile*
 
 Xilinx Zynq
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/xilinx_zynq.c
 F: hw/misc/zynq_slcr.c
 F: hw/*/cadence_*
 F: hw/ssi/xilinx_spips.c
 
+Xilinx ZynqMP
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/arm/xlnx-zynqmp.c
+F: hw/arm/xlnx-ep108.c
+F: include/hw/arm/xlnx-zynqmp.h
+
 ARM ACPI Subsystem
 M: Shannon Zhao <zhaoshenglong@huawei.com>
 M: Shannon Zhao <shannon.zhao@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/virt-acpi-build.c
 F: include/hw/arm/virt-acpi-build.h
@@ -405,7 +516,7 @@ S: Maintained
 F: hw/microblaze/petalogix_s3adsp1800_mmu.c
 
 petalogix_ml605
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: hw/microblaze/petalogix_ml605_mmu.c
 
@@ -493,7 +604,7 @@ F: hw/ppc/prep.c
 F: hw/pci-host/prep.[hc]
 F: hw/isa/pc87312.[hc]
 
-sPAPR (pseries)
+sPAPR
 M: David Gibson <david@gibson.dropbear.id.au>
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
@@ -544,25 +655,18 @@ F: hw/*/grlib*
 
 S390 Machines
 -------------
-S390 Virtio
-M: Alexander Graf <agraf@suse.de>
-S: Maintained
-F: hw/s390x/s390-*.c
-X: hw/s390x/*pci*.[hc]
-
 S390 Virtio-ccw
 M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Supported
-F: hw/s390x/s390-virtio-ccw.c
-F: hw/s390x/css.[hc]
-F: hw/s390x/sclp*.[hc]
-F: hw/s390x/ipl*.[hc]
-F: hw/s390x/*pci*.[hc]
+F: hw/char/sclp*.[hc]
+F: hw/s390x/
 F: include/hw/s390x/
 F: pc-bios/s390-ccw/
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+F: hw/watchdog/wdt_diag288.c
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 UniCore32 Machines
 -------------
@@ -591,7 +695,32 @@ F: hw/acpi/piix4.c
 F: hw/acpi/ich9.c
 F: include/hw/acpi/ich9.h
 F: include/hw/acpi/piix.h
+F: hw/misc/sga.c
 
+PC Chipset
+M: Michael S. Tsirkin <mst@redhat.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Support
+F: hw/char/debugcon.c
+F: hw/char/parallel.c
+F: hw/char/serial*
+F: hw/dma/i8257*
+F: hw/i2c/pm_smbus.c
+F: hw/intc/apic*
+F: hw/intc/ioapic*
+F: hw/intc/i8259*
+F: hw/misc/debugexit.c
+F: hw/misc/pc-testdev.c
+F: hw/timer/hpet*
+F: hw/timer/i8254*
+F: hw/timer/mc146818rtc*
+
+Machine core
+M: Eduardo Habkost <ehabkost@redhat.com>
+M: Marcel Apfelbaum <marcel@redhat.com>
+S: Supported
+F: hw/core/machine.c
+F: include/hw/boards.h
 
 Xtensa Machines
 ---------------
@@ -624,6 +753,7 @@ F: hw/block/cdrom.c
 F: hw/block/hd-geometry.c
 F: tests/ide-test.c
 F: tests/ahci-test.c
+F: tests/libqos/ahci*
 T: git git://github.com/jnsnow/qemu.git ide
 
 Floppy
@@ -632,26 +762,38 @@ L: qemu-block@nongnu.org
 S: Supported
 F: hw/block/fdc.c
 F: include/hw/block/fdc.h
+F: tests/fdc-test.c
 T: git git://github.com/jnsnow/qemu.git ide
 
 OMAP
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/*/omap*
+F: include/hw/arm/omap.h
+
+IPack
+M: Alberto Garcia <berto@igalia.com>
+S: Odd Fixes
+F: hw/char/ipoctal232.c
+F: hw/ipack/
 
 PCI
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
 F: include/hw/pci/*
+F: hw/misc/pci-testdev.c
 F: hw/pci/*
+F: hw/pci-bridge/*
 
-ACPI
+ACPI/SMBIOS
 M: Michael S. Tsirkin <mst@redhat.com>
 M: Igor Mammedov <imammedo@redhat.com>
 S: Supported
 F: include/hw/acpi/*
+F: include/hw/smbios/*
 F: hw/mem/*
 F: hw/acpi/*
+F: hw/smbios/*
 F: hw/i386/acpi-build.[hc]
 F: hw/i386/*dsl
 F: hw/arm/virt-acpi-build.c
@@ -670,6 +812,19 @@ M: Scott Wood <scottwood@freescale.com>
 L: qemu-ppc@nongnu.org
 S: Supported
 F: hw/ppc/e500*
+F: hw/pci-host/ppce500.c
+F: hw/net/fsl_etsec/
+
+Character devices
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Odd Fixes
+F: hw/char/
+
+Network devices
+M: Jason Wang <jasowang@redhat.com>
+S: Odd Fixes
+F: hw/net/
+T: git git://github.com/jasowang/qemu.git net
 
 SCSI
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -683,16 +838,27 @@ S: Orphan
 F: hw/scsi/lsi53c895a.c
 
 SSI
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 S: Maintained
 F: hw/ssi/*
 F: hw/block/m25p80.c
+X: hw/ssi/xilinx_*
+
+Xilinx SPI
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+S: Maintained
+F: hw/ssi/xilinx_*
 
 USB
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: hw/usb/*
 F: tests/usb-*-test.c
+F: docs/usb2.txt
+F: docs/usb-storage.txt
+F: include/hw/usb.h
+F: include/hw/usb/
 
 USB (serial adapter)
 M: Gerd Hoffmann <kraxel@redhat.com>
@@ -704,6 +870,7 @@ VFIO
 M: Alex Williamson <alex.williamson@redhat.com>
 S: Supported
 F: hw/vfio/*
+F: include/hw/vfio/
 
 vhost
 M: Michael S. Tsirkin <mst@redhat.com>
@@ -715,9 +882,11 @@ M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
 F: hw/*/virtio*
 F: net/vhost-user.c
+F: include/hw/virtio/
 
 virtio-9p
 M: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+M: Greg Kurz <gkurz@linux.vnet.ibm.com>
 S: Supported
 F: hw/9pfs/
 F: fsdev/
@@ -738,7 +907,8 @@ M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 S: Supported
 F: hw/s390x/virtio-ccw.[hc]
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 virtio-input
 M: Gerd Hoffmann <kraxel@redhat.com>
@@ -758,6 +928,7 @@ M: Amit Shah <amit.shah@redhat.com>
 S: Supported
 F: hw/virtio/virtio-rng.c
 F: include/hw/virtio/virtio-rng.h
+F: include/sysemu/rng*.h
 F: backends/rng*.c
 
 nvme
@@ -775,8 +946,9 @@ F: hw/scsi/megasas.c
 F: hw/scsi/mfi.h
 
 Xilinx EDK
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
 M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 S: Maintained
 F: hw/*/xilinx_*
 F: include/hw/xilinx.h
@@ -793,6 +965,13 @@ M: Jiri Pirko <jiri@resnulli.us>
 S: Maintained
 F: hw/net/rocker/
 
+NVDIMM
+M: Xiao Guangrong <guangrong.xiao@linux.intel.com>
+S: Maintained
+F: hw/acpi/nvdimm.c
+F: hw/mem/nvdimm.c
+F: include/hw/mem/nvdimm.h
+
 Subsystems
 ----------
 Audio
@@ -806,6 +985,7 @@ F: tests/intel-hda-test.c
 
 Block layer core
 M: Kevin Wolf <kwolf@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: block*
@@ -819,6 +999,7 @@ T: git git://repo.or.cz/qemu/kevin.git block
 
 Block I/O path
 M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Fam Zheng <famz@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: async.c
@@ -835,7 +1016,7 @@ F: blockjob.c
 F: include/block/blockjob.h
 F: block/backup.c
 F: block/commit.c
-F: block/stream.h
+F: block/stream.c
 F: block/mirror.c
 T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
@@ -847,7 +1028,7 @@ F: block/qapi.c
 F: qapi/block*.json
 T: git git://repo.or.cz/qemu/armbru.git block-next
 
-Character Devices
+Character device backends
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: qemu-char.c
@@ -869,7 +1050,6 @@ M: Andreas Färber <afaerber@suse.de>
 S: Supported
 F: qom/cpu.c
 F: include/qom/cpu.h
-F: target-i386/cpu.c
 
 ICC Bus
 M: Igor Mammedov <imammedo@redhat.com>
@@ -878,10 +1058,19 @@ F: include/hw/cpu/icc_bus.h
 F: hw/cpu/icc_bus.c
 
 Device Tree
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
-F: device_tree.[ch]
+F: device_tree.c
+F: include/sysemu/device_tree.h
+
+Error reporting
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
+F: include/qapi/error.h
+F: include/qemu/error-report.h
+F: util/error.c
+F: util/qemu-error.c
 
 GDB stub
 L: qemu-devel@nongnu.org
@@ -895,6 +1084,7 @@ S: Supported
 F: include/exec/ioport.h
 F: ioport.c
 F: include/exec/memory.h
+F: include/exec/ram_addr.h
 F: memory.c
 F: include/exec/memory-internal.h
 F: exec.c
@@ -903,6 +1093,7 @@ SPICE
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Supported
 F: include/ui/qemu-spice.h
+F: include/ui/spice-display.h
 F: ui/spice-*.c
 F: audio/spiceaudio.c
 F: hw/display/qxl*
@@ -911,6 +1102,7 @@ Graphics
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Odd Fixes
 F: ui/
+F: include/ui/
 
 Cocoa graphics
 M: Andreas Färber <andreas.faerber@web.de>
@@ -934,12 +1126,12 @@ F: hmp.c
 F: hmp-commands.hx
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
-Network device layer
-M: Stefan Hajnoczi <stefanha@redhat.com>
+Network device backends
 M: Jason Wang <jasowang@redhat.com>
 S: Maintained
 F: net/
-T: git git://github.com/stefanha/qemu.git net
+F: include/net/
+T: git git://github.com/jasowang/qemu.git net
 
 Netmap network backend
 M: Luigi Rizzo <rizzo@iet.unipi.it>
@@ -952,8 +1144,9 @@ F: net/netmap.c
 Network Block Device (NBD)
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Odd Fixes
-F: block/nbd.c
-F: nbd.*
+F: block/nbd*
+F: nbd/
+F: include/block/nbd*
 F: qemu-nbd.c
 T: git git://github.com/bonzini/qemu.git nbd-next
 
@@ -962,8 +1155,6 @@ M: Eduardo Habkost <ehabkost@redhat.com>
 S: Maintained
 F: numa.c
 F: include/sysemu/numa.h
-K: numa|NUMA
-K: srat|SRAT
 T: git git://github.com/ehabkost/qemu.git numa
 
 QAPI
@@ -972,7 +1163,12 @@ M: Michael Roth <mdroth@linux.vnet.ibm.com>
 S: Supported
 F: qapi/
 X: qapi/*.json
+F: include/qapi/
+X: include/qapi/qmp/
+F: include/qapi/qmp/dispatch.h
 F: tests/qapi-schema/
+F: tests/test-*-visitor.c
+F: tests/test-qmp-*.c
 F: scripts/qapi*
 F: docs/qapi*
 T: git git://repo.or.cz/qemu/armbru.git qapi-next
@@ -989,6 +1185,14 @@ QObject
 M: Luiz Capitulino <lcapitulino@redhat.com>
 S: Maintained
 F: qobject/
+F: include/qapi/qmp/
+X: include/qapi/qmp/dispatch.h
+F: tests/check-qdict.c
+F: tests/check-qfloat.c
+F: tests/check-qint.c
+F: tests/check-qjson.c
+F: tests/check-qlist.c
+F: tests/check-qstring.c
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
 QEMU Guest Agent
@@ -1005,6 +1209,8 @@ F: include/qom/
 X: include/qom/cpu.h
 F: qom/
 X: qom/cpu.c
+F: tests/check-qom-interface.c
+F: tests/check-qom-proplist.c
 F: tests/qom-test.c
 
 QMP
@@ -1013,14 +1219,17 @@ S: Supported
 F: qmp.c
 F: monitor.c
 F: qmp-commands.hx
-F: docs/qmp/
+F: docs/*qmp-*
 F: scripts/qmp/
 T: git git://repo.or.cz/qemu/armbru.git qapi-next
 
 SLIRP
+M: Samuel Thibault <samuel.thibault@ens-lyon.org>
 M: Jan Kiszka <jan.kiszka@siemens.com>
 S: Maintained
 F: slirp/
+F: net/slirp.c
+F: include/net/slirp.h
 T: git git://git.kiszka.org/qemu.git queues/slirp
 
 Tracing
@@ -1045,6 +1254,7 @@ F: include/migration/
 F: migration/
 F: scripts/vmstate-static-checker.py
 F: tests/vmstate-static-checker-data/
+F: docs/migration.txt
 
 Seccomp
 M: Eduardo Otubo <eduardo.otubo@profitbricks.com>
@@ -1059,6 +1269,43 @@ F: crypto/
 F: include/crypto/
 F: tests/test-crypto-*
 
+Coroutines
+M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Kevin Wolf <kwolf@redhat.com>
+F: util/*coroutine*
+F: include/qemu/coroutine*
+F: tests/test-coroutine.c
+
+Buffers
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd fixes
+F: util/buffer.c
+F: include/qemu/buffer.h
+
+I/O Channels
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Maintained
+F: io/
+F: include/io/
+F: tests/test-io-*
+
+Sockets
+M: Daniel P. Berrange <berrange@redhat.com>
+M: Gerd Hoffmann <kraxel@redhat.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Maintained
+F: include/qemu/sockets.h
+F: util/qemu-sockets.c
+
+Throttling infrastructure
+M: Alberto Garcia <berto@igalia.com>
+S: Supported
+F: block/throttle-groups.c
+F: include/block/throttle-groups.h
+F: include/qemu/throttle.h
+F: util/throttle.c
+L: qemu-block@nongnu.org
+
 Usermode Emulation
 ------------------
 Overall
@@ -1088,54 +1335,61 @@ AArch64 target
 M: Claudio Fontana <claudio.fontana@huawei.com>
 M: Claudio Fontana <claudio.fontana@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/aarch64/
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 ARM target
 M: Andrzej Zaborowski <balrogg@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/arm/
+F: disas/arm.c
 
 i386 target
 L: qemu-devel@nongnu.org
 S: Maintained
 F: tcg/i386/
+F: disas/i386.c
 
 IA64 target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/ia64/
+F: disas/ia64.c
 
 MIPS target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/mips/
+F: disas/mips.c
 
 PPC
 M: Vassili Karpov (malc) <av1474@comtv.ru>
 S: Maintained
 F: tcg/ppc/
-
-PPC64 target
-M: Vassili Karpov (malc) <av1474@comtv.ru>
-S: Maintained
-F: tcg/ppc64/
+F: disas/ppc.c
 
 S390 target
 M: Alexander Graf <agraf@suse.de>
 M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: tcg/s390/
+F: disas/s390.c
 
 SPARC target
 M: Blue Swirl <blauwirbel@gmail.com>
 S: Maintained
 F: tcg/sparc/
+F: disas/sparc.c
 
 TCI target
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: tcg/tci/
 F: tci.c
+F: disas/tci.c
 
 Stable branches
 ---------------
@@ -1288,9 +1542,11 @@ F: block/dmg.c
 
 parallels
 M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Denis V. Lunev <den@openvz.org>
 L: qemu-block@nongnu.org
 S: Supported
 F: block/parallels.c
+F: docs/specs/parallels.txt
 
 qed
 M: Stefan Hajnoczi <stefanha@redhat.com>
@@ -1311,6 +1567,7 @@ F: block/win32-aio.c
 
 qcow2
 M: Kevin Wolf <kwolf@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: block/qcow2*
@@ -1323,6 +1580,7 @@ F: block/qcow.c
 
 blkdebug
 M: Kevin Wolf <kwolf@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: block/blkdebug.c
@@ -1344,3 +1602,17 @@ M: Stefan Hajnoczi <stefanha@redhat.com>
 L: qemu-block@nongnu.org
 S: Supported
 F: tests/image-fuzzer/
+
+Build and test automation
+-------------------------
+M: Alex Bennée <alex.bennee@linaro.org>
+L: qemu-devel@nongnu.org
+S: Supported
+F: .travis.yml
+
+Documentation
+-------------
+Build system architecture
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd Fixes
+F: docs/build-system.txt
diff --git a/qemu/Makefile b/qemu/Makefile
index 340d9c8fa..1d076a9d8 100644
--- a/qemu/Makefile
+++ b/qemu/Makefile
@@ -52,6 +52,8 @@ endif
 GENERATED_HEADERS = config-host.h qemu-options.def
 GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h qapi-event.h
 GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c qapi-event.c
+GENERATED_HEADERS += qmp-introspect.h
+GENERATED_SOURCES += qmp-introspect.c
 
 GENERATED_HEADERS += trace/generated-events.h
 GENERATED_SOURCES += trace/generated-events.c
@@ -88,7 +90,8 @@ LIBS+=-lz $(LIBS_TOOLS)
 HELPERS-$(CONFIG_LINUX) = qemu-bridge-helper$(EXESUF)
 
 ifdef BUILD_DOCS
-DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qmp-commands.txt
+DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qemu-ga.8
+DOCS+=qmp-commands.txt
 ifdef CONFIG_LINUX
 DOCS+=kvm_stat.1
 endif
@@ -148,18 +151,21 @@ dummy := $(call unnest-vars,, \
                 stub-obj-y \
                 util-obj-y \
                 qga-obj-y \
+                ivshmem-client-obj-y \
+                ivshmem-server-obj-y \
                 qga-vss-dll-obj-y \
                 block-obj-y \
                 block-obj-m \
+                crypto-obj-y \
+                crypto-aes-obj-y \
+                qom-obj-y \
+                io-obj-y \
                 common-obj-y \
                 common-obj-m)
 
 ifneq ($(wildcard config-host.mak),)
 include $(SRC_PATH)/tests/Makefile
 endif
-ifeq ($(CONFIG_SMARTCARD_NSS),y)
-include $(SRC_PATH)/libcacard/Makefile
-endif
 
 all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all modules
 
@@ -172,6 +178,8 @@ SUBDIR_RULES=$(patsubst %,subdir-%, $(TARGET_DIRS))
 SOFTMMU_SUBDIR_RULES=$(filter %-softmmu,$(SUBDIR_RULES))
 
 $(SOFTMMU_SUBDIR_RULES): $(block-obj-y)
+$(SOFTMMU_SUBDIR_RULES): $(crypto-obj-y)
+$(SOFTMMU_SUBDIR_RULES): $(io-obj-y)
 $(SOFTMMU_SUBDIR_RULES): config-all-devices.mak
 
 subdir-%:
@@ -196,7 +204,7 @@ subdir-dtc:dtc/libfdt dtc/tests
 dtc/%:
 	mkdir -p $@
 
-$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y)
+$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y) $(qom-obj-y) $(crypto-aes-obj-$(CONFIG_USER_ONLY))
 
 ROMSUBDIR_RULES=$(patsubst %,romsubdir-%, $(ROMS))
 romsubdir-%:
@@ -226,13 +234,13 @@ util/module.o-cflags = -D'CONFIG_BLOCK_MODULES=$(block-modules)'
 
 qemu-img.o: qemu-img-cmds.h
 
-qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-io$(EXESUF): qemu-io.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-img$(EXESUF): qemu-img.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-io$(EXESUF): qemu-io.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 
-qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
+qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o libqemuutil.a libqemustub.a
 
-fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o libqemuutil.a libqemustub.a
+fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/9p-marshal.o fsdev/9p-iov-marshal.o libqemuutil.a libqemustub.a
 fsdev/virtfs-proxy-helper$(EXESUF): LIBS += -lcap
 
 qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx
@@ -263,7 +271,9 @@ $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 
 qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json \
                $(SRC_PATH)/qapi/block.json $(SRC_PATH)/qapi/block-core.json \
-               $(SRC_PATH)/qapi/event.json
+               $(SRC_PATH)/qapi/event.json $(SRC_PATH)/qapi/introspect.json \
+               $(SRC_PATH)/qapi/crypto.json $(SRC_PATH)/qapi/rocker.json \
+               $(SRC_PATH)/qapi/trace.json
 
 qapi-types.c qapi-types.h :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
@@ -285,6 +295,11 @@ $(qapi-modules) $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
 		$(gen-out-type) -o "." -m $<, \
 		"  GEN   $@")
+qmp-introspect.h qmp-introspect.c :\
+$(qapi-modules) $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o "." $<, \
+		"  GEN   $@")
 
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
@@ -295,24 +310,30 @@ qemu-ga$(EXESUF): $(qga-obj-y) libqemuutil.a libqemustub.a
 ifdef QEMU_GA_MSI_ENABLED
 QEMU_GA_MSI=qemu-ga-$(ARCH).msi
 
-msi: ${QEMU_GA_MSI}
+msi: $(QEMU_GA_MSI)
 
-$(QEMU_GA_MSI): qemu-ga.exe
-
-ifdef QEMU_GA_MSI_WITH_VSS
-$(QEMU_GA_MSI): qga/vss-win32/qga-vss.dll
-endif
+$(QEMU_GA_MSI): qemu-ga.exe $(QGA_VSS_PROVIDER)
 
 $(QEMU_GA_MSI): config-host.mak
 
-$(QEMU_GA_MSI):  qga/installer/qemu-ga.wxs
-	$(call quiet-command,QEMU_GA_VERSION="$(QEMU_GA_VERSION)" QEMU_GA_MANUFACTURER="$(QEMU_GA_MANUFACTURER)" QEMU_GA_DISTRO="$(QEMU_GA_DISTRO)" \
+$(QEMU_GA_MSI):  $(SRC_PATH)/qga/installer/qemu-ga.wxs
+	$(call quiet-command,QEMU_GA_VERSION="$(QEMU_GA_VERSION)" QEMU_GA_MANUFACTURER="$(QEMU_GA_MANUFACTURER)" QEMU_GA_DISTRO="$(QEMU_GA_DISTRO)" BUILD_DIR="$(BUILD_DIR)" \
 	wixl -o $@ $(QEMU_GA_MSI_ARCH) $(QEMU_GA_MSI_WITH_VSS) $(QEMU_GA_MSI_MINGW_DLL_PATH) $<, "  WIXL  $@")
 else
 msi:
-	@echo MSI build not configured or dependency resolution failed (reconfigure with --enable-guest-agent-msi option)
+	@echo "MSI build not configured or dependency resolution failed (reconfigure with --enable-guest-agent-msi option)"
 endif
 
+ifneq ($(EXESUF),)
+.PHONY: qemu-ga
+qemu-ga: qemu-ga$(EXESUF) $(QGA_VSS_PROVIDER) $(QEMU_GA_MSI)
+endif
+
+ivshmem-client$(EXESUF): $(ivshmem-client-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
+ivshmem-server$(EXESUF): $(ivshmem-server-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
+
 clean:
 # avoid old build problems by removing potentially incorrect old files
 	rm -f config.mak op-i386.h opc-i386.h gen-op-i386.h op-arm.h opc-arm.h gen-op-arm.h
@@ -344,7 +365,7 @@ qemu-%.tar.bz2:
 	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.bz2,%,$@)"
 
 distclean: clean
-	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
+	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi qemu-monitor-info.texi
 	rm -f config-all-devices.mak config-all-disas.mak config.status
 	rm -f po/*.mo tests/qemu-iotests/common.env
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
@@ -370,7 +391,7 @@ bepo    cz
 ifdef INSTALL_BLOBS
 BLOBS=bios.bin bios-256k.bin sgabios.bin vgabios.bin vgabios-cirrus.bin \
 vgabios-stdvga.bin vgabios-vmware.bin vgabios-qxl.bin vgabios-virtio.bin \
-acpi-dsdt.aml q35-acpi-dsdt.aml \
+acpi-dsdt.aml \
 ppc_rom.bin openbios-sparc32 openbios-sparc64 openbios-ppc QEMU,tcx.bin QEMU,cgthree.bin \
 pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \
 pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \
@@ -379,7 +400,6 @@ efi-pcnet.rom efi-rtl8139.rom efi-virtio.rom \
 qemu-icon.bmp qemu_logo_no_text.svg \
 bamboo.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \
 multiboot.bin linuxboot.bin kvmvapic.bin \
-s390-zipl.rom \
 s390-ccw.img \
 spapr-rtas.bin slof.bin \
 palcode-clipper \
@@ -400,6 +420,9 @@ ifneq ($(TOOLS),)
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man8"
 	$(INSTALL_DATA) qemu-nbd.8 "$(DESTDIR)$(mandir)/man8"
 endif
+ifneq (,$(findstring qemu-ga,$(TOOLS)))
+	$(INSTALL_DATA) qemu-ga.8 "$(DESTDIR)$(mandir)/man8"
+endif
 endif
 ifdef CONFIG_VIRTFS
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man1"
@@ -420,7 +443,7 @@ endif
 install: all $(if $(BUILD_DOCS),install-doc) \
 install-datadir install-localstatedir
 ifneq ($(TOOLS),)
-	$(call install-prog,$(TOOLS),$(DESTDIR)$(bindir))
+	$(call install-prog,$(subst qemu-ga,qemu-ga$(EXESUF),$(TOOLS)),$(DESTDIR)$(bindir))
 endif
 ifneq ($(CONFIG_MODULES),)
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_moddir)"
@@ -508,13 +531,16 @@ qemu-options.texi: $(SRC_PATH)/qemu-options.hx
 qemu-monitor.texi: $(SRC_PATH)/hmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
+qemu-monitor-info.texi: $(SRC_PATH)/hmp-commands-info.hx
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
+
 qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -q < $< > $@,"  GEN   $@")
 
 qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi
+qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi qemu-monitor-info.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu.pod && \
 	  $(POD2MAN) --section=1 --center=" " --release=" " qemu.pod > $@, \
@@ -538,6 +564,12 @@ qemu-nbd.8: qemu-nbd.texi
 	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-nbd.pod > $@, \
 	  "  GEN   $@")
 
+qemu-ga.8: qemu-ga.texi
+	$(call quiet-command, \
+	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu-ga.pod && \
+	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-ga.pod > $@, \
+	  "  GEN   $@")
+
 kvm_stat.1: scripts/kvm/kvm_stat.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< kvm_stat.pod && \
@@ -551,7 +583,8 @@ pdf: qemu-doc.pdf qemu-tech.pdf
 
 qemu-doc.dvi qemu-doc.html qemu-doc.info qemu-doc.pdf: \
 	qemu-img.texi qemu-nbd.texi qemu-options.texi \
-	qemu-monitor.texi qemu-img-cmds.texi
+	qemu-monitor.texi qemu-img-cmds.texi qemu-ga.texi \
+	qemu-monitor-info.texi
 
 ifdef CONFIG_WIN32
 
@@ -601,6 +634,7 @@ endif # SIGNCODE
                 $(if $(DLL_PATH),-DDLLDIR="$(DLL_PATH)") \
                 -DSRCDIR="$(SRC_PATH)" \
                 -DOUTFILE="$(INSTALLER)" \
+                -DDISPLAYVERSION="$(VERSION)" \
                 $(SRC_PATH)/qemu.nsi
 	rm -r ${INSTDIR}
 ifdef SIGNCODE
diff --git a/qemu/Makefile.objs b/qemu/Makefile.objs
index f094eff15..8f705f620 100644
--- a/qemu/Makefile.objs
+++ b/qemu/Makefile.objs
@@ -1,38 +1,38 @@
 #######################################################################
 # Common libraries for tools and emulators
-stub-obj-y = stubs/
-util-obj-y = util/ qobject/ qapi/ qapi-types.o qapi-visit.o qapi-event.o
-util-obj-y += crypto/
+stub-obj-y = stubs/ crypto/
+util-obj-y = util/ qobject/ qapi/
+util-obj-y += qmp-introspect.o qapi-types.o qapi-visit.o qapi-event.o
 
 #######################################################################
 # block-obj-y is code used by both qemu system emulation and qemu-img
 
 block-obj-y = async.o thread-pool.o
-block-obj-y += nbd.o block.o blockjob.o
+block-obj-y += nbd/
+block-obj-y += block.o blockjob.o
 block-obj-y += main-loop.o iohandler.o qemu-timer.o
 block-obj-$(CONFIG_POSIX) += aio-posix.o
 block-obj-$(CONFIG_WIN32) += aio-win32.o
 block-obj-y += block/
 block-obj-y += qemu-io-cmds.o
 
-block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
-block-obj-y += qemu-coroutine-sleep.o
-block-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
-
 block-obj-m = block/
 
+#######################################################################
+# crypto-obj-y is code used by both qemu system emulation and qemu-img
 
-######################################################################
-# smartcard
+crypto-obj-y = crypto/
+crypto-aes-obj-y = crypto/
+
+#######################################################################
+# qom-obj-y is code used by both qemu system emulation and qemu-img
+
+qom-obj-y = qom/
 
-libcacard-y += libcacard/cac.o libcacard/event.o
-libcacard-y += libcacard/vcard.o libcacard/vreader.o
-libcacard-y += libcacard/vcard_emul_nss.o
-libcacard-y += libcacard/vcard_emul_type.o
-libcacard-y += libcacard/card_7816.o
-libcacard-y += libcacard/vcardt.o
-libcacard/vcard_emul_nss.o-cflags := $(NSS_CFLAGS)
-libcacard/vcard_emul_nss.o-libs := $(NSS_LIBS)
+#######################################################################
+# io-obj-y is code used by both qemu system emulation and qemu-img
+
+io-obj-y = io/
 
 ######################################################################
 # Target independent part of system emulation. The long term path is to
@@ -60,6 +60,8 @@ common-obj-y += audio/
 common-obj-y += hw/
 common-obj-y += accel.o
 
+common-obj-y += replay/
+
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 bt-host.o-cflags := $(BLUEZ_CFLAGS)
@@ -75,20 +77,18 @@ common-obj-y += backends/
 
 common-obj-$(CONFIG_SECCOMP) += qemu-seccomp.o
 
-common-obj-$(CONFIG_SMARTCARD_NSS) += $(libcacard-y)
-
 common-obj-$(CONFIG_FDT) += device_tree.o
 
 ######################################################################
 # qapi
 
 common-obj-y += qmp-marshal.o
+common-obj-y += qmp-introspect.o
 common-obj-y += qmp.o hmp.o
 endif
 
 #######################################################################
 # Target-independent parts used in system and user emulation
-common-obj-y += qemu-log.o
 common-obj-y += tcg-runtime.o
 common-obj-y += hw/
 common-obj-y += qom/
@@ -111,3 +111,8 @@ target-obj-y += trace/
 # by libqemuutil.a.  These should be moved to a separate .json schema.
 qga-obj-y = qga/
 qga-vss-dll-obj-y = qga/
+
+######################################################################
+# contrib
+ivshmem-client-obj-y = contrib/ivshmem-client/
+ivshmem-server-obj-y = contrib/ivshmem-server/
diff --git a/qemu/Makefile.target b/qemu/Makefile.target
index 3e7aafd72..34ddb7e76 100644
--- a/qemu/Makefile.target
+++ b/qemu/Makefile.target
@@ -7,7 +7,7 @@ include config-target.mak
 include config-devices.mak
 include $(SRC_PATH)/rules.mak
 
-$(call set-vpath, $(SRC_PATH))
+$(call set-vpath, $(SRC_PATH):$(BUILD_DIR))
 ifdef CONFIG_LINUX
 QEMU_CFLAGS += -I../linux-headers
 endif
@@ -85,8 +85,11 @@ all: $(PROGS) stap
 #########################################################
 # cpu emulator library
 obj-y = exec.o translate-all.o cpu-exec.o
+obj-y += translate-common.o
+obj-y += cpu-exec-common.o
 obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o
 obj-$(CONFIG_TCG_INTERPRETER) += tci.o
+obj-y += tcg/tcg-common.o
 obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
 obj-y += target-$(TARGET_BASE_ARCH)/
@@ -151,7 +154,7 @@ else
 obj-y += hw/$(TARGET_BASE_ARCH)/
 endif
 
-GENERATED_HEADERS += hmp-commands.h qmp-commands-old.h
+GENERATED_HEADERS += hmp-commands.h hmp-commands-info.h qmp-commands-old.h
 
 endif # CONFIG_SOFTMMU
 
@@ -170,12 +173,20 @@ target-obj-y-save := $(target-obj-y)
 dummy := $(call unnest-vars,.., \
                block-obj-y \
                block-obj-m \
+               crypto-obj-y \
+               crypto-aes-obj-y \
+               qom-obj-y \
+               io-obj-y \
                common-obj-y \
                common-obj-m)
 target-obj-y := $(target-obj-y-save)
 all-obj-y += $(common-obj-y)
 all-obj-y += $(target-obj-y)
+all-obj-y += $(qom-obj-y)
 all-obj-$(CONFIG_SOFTMMU) += $(block-obj-y)
+all-obj-$(CONFIG_USER_ONLY) += $(crypto-aes-obj-y)
+all-obj-$(CONFIG_SOFTMMU) += $(crypto-obj-y)
+all-obj-$(CONFIG_SOFTMMU) += $(io-obj-y)
 
 $(QEMU_PROG_BUILD): config-devices.mak
 
@@ -193,6 +204,9 @@ gdbstub-xml.c: $(TARGET_XML_FILES) $(SRC_PATH)/scripts/feature_to_c.sh
 hmp-commands.h: $(SRC_PATH)/hmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
+hmp-commands-info.h: $(SRC_PATH)/hmp-commands-info.hx
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
+
 qmp-commands-old.h: $(SRC_PATH)/qmp-commands.hx
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
diff --git a/qemu/README b/qemu/README
index c7c990d89..f38193fc6 100644
--- a/qemu/README
+++ b/qemu/README
@@ -1,3 +1,107 @@
-Read the documentation in qemu-doc.html or on http://wiki.qemu-project.org
+         QEMU README
+         ===========
 
-- QEMU team
+QEMU is a generic and open source machine & userspace emulator and
+virtualizer.
+
+QEMU is capable of emulating a complete machine in software without any
+need for hardware virtualization support. By using dynamic translation,
+it achieves very good performance. QEMU can also integrate with the Xen
+and KVM hypervisors to provide emulated hardware while allowing the
+hypervisor to manage the CPU. With hypervisor support, QEMU can achieve
+near native performance for CPUs. When QEMU emulates CPUs directly it is
+capable of running operating systems made for one machine (e.g. an ARMv7
+board) on a different machine (e.g. an x86_64 PC board).
+
+QEMU is also capable of providing userspace API virtualization for Linux
+and BSD kernel interfaces. This allows binaries compiled against one
+architecture ABI (e.g. the Linux PPC64 ABI) to be run on a host using a
+different architecture ABI (e.g. the Linux x86_64 ABI). This does not
+involve any hardware emulation, simply CPU and syscall emulation.
+
+QEMU aims to fit into a variety of use cases. It can be invoked directly
+by users wishing to have full control over its behaviour and settings.
+It also aims to facilitate integration into higher level management
+layers, by providing a stable command line interface and monitor API.
+It is commonly invoked indirectly via the libvirt library when using
+open source applications such as oVirt, OpenStack and virt-manager.
+
+QEMU as a whole is released under the GNU General Public License,
+version 2. For full licensing details, consult the LICENSE file.
+
+
+Building
+========
+
+QEMU is multi-platform software intended to be buildable on all modern
+Linux platforms, OS-X, Win32 (via the Mingw64 toolchain) and a variety
+of other UNIX targets. The simple steps to build QEMU are:
+
+  mkdir build
+  cd build
+  ../configure
+  make
+
+Complete details of the process for building and configuring QEMU for
+all supported host platforms can be found in the qemu-tech.html file.
+Additional information can also be found online via the QEMU website:
+
+  http://qemu-project.org/Hosts/Linux
+  http://qemu-project.org/Hosts/W32
+
+
+Submitting patches
+==================
+
+The QEMU source code is maintained under the GIT version control system.
+
+   git clone git://git.qemu-project.org/qemu.git
+
+When submitting patches, the preferred approach is to use 'git
+format-patch' and/or 'git send-email' to format & send the mail to the
+qemu-devel@nongnu.org mailing list. All patches submitted must contain
+a 'Signed-off-by' line from the author. Patches should follow the
+guidelines set out in the HACKING and CODING_STYLE files.
+
+Additional information on submitting patches can be found online via
+the QEMU website
+
+  http://qemu-project.org/Contribute/SubmitAPatch
+  http://qemu-project.org/Contribute/TrivialPatches
+
+
+Bug reporting
+=============
+
+The QEMU project uses Launchpad as its primary upstream bug tracker. Bugs
+found when running code built from QEMU git or upstream released sources
+should be reported via:
+
+  https://bugs.launchpad.net/qemu/
+
+If using QEMU via an operating system vendor pre-built binary package, it
+is preferable to report bugs to the vendor's own bug tracker first. If
+the bug is also known to affect latest upstream code, it can also be
+reported via launchpad.
+
+For additional information on bug reporting consult:
+
+  http://qemu-project.org/Contribute/ReportABug
+
+
+Contact
+=======
+
+The QEMU community can be contacted in a number of ways, with the two
+main methods being email and IRC
+
+ - qemu-devel@nongnu.org
+   http://lists.nongnu.org/mailman/listinfo/qemu-devel
+ - #qemu on irc.oftc.net
+
+Information on additional methods of contacting the community can be
+found online via the QEMU website:
+
+  http://qemu-project.org/Contribute/StartHere
+
+-- End
diff --git a/qemu/VERSION b/qemu/VERSION
index 81b7c5124..e70b4523a 100644
--- a/qemu/VERSION
+++ b/qemu/VERSION
@@ -1 +1 @@
-2.4.0.1
+2.6.0
diff --git a/qemu/accel.c b/qemu/accel.c
index 74e41daaa..0510b90f6 100644
--- a/qemu/accel.c
+++ b/qemu/accel.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/accel.h"
 #include "hw/boards.h"
 #include "qemu-common.h"
diff --git a/qemu/aio-posix.c b/qemu/aio-posix.c
index d4770336c..6006122e0 100644
--- a/qemu/aio-posix.c
+++ b/qemu/aio-posix.c
@@ -13,10 +13,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block.h"
 #include "qemu/queue.h"
 #include "qemu/sockets.h"
+#ifdef CONFIG_EPOLL_CREATE1
+#include <sys/epoll.h>
+#endif
 
 struct AioHandler
 {
@@ -25,9 +29,166 @@ struct AioHandler
     IOHandler *io_write;
     int deleted;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
+#ifdef CONFIG_EPOLL_CREATE1
+
+/* The fd number threashold to switch to epoll */
+#define EPOLL_ENABLE_THRESHOLD 64
+
+static void aio_epoll_disable(AioContext *ctx)
+{
+    ctx->epoll_available = false;
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    ctx->epoll_enabled = false;
+    close(ctx->epollfd);
+}
+
+static inline int epoll_events_from_pfd(int pfd_events)
+{
+    return (pfd_events & G_IO_IN ? EPOLLIN : 0) |
+           (pfd_events & G_IO_OUT ? EPOLLOUT : 0) |
+           (pfd_events & G_IO_HUP ? EPOLLHUP : 0) |
+           (pfd_events & G_IO_ERR ? EPOLLERR : 0);
+}
+
+static bool aio_epoll_try_enable(AioContext *ctx)
+{
+    AioHandler *node;
+    struct epoll_event event;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        int r;
+        if (node->deleted || !node->pfd.events) {
+            continue;
+        }
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        event.data.ptr = node;
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+        if (r) {
+            return false;
+        }
+    }
+    ctx->epoll_enabled = true;
+    return true;
+}
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+    struct epoll_event event;
+    int r;
+
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    if (!node->pfd.events) {
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_DEL, node->pfd.fd, &event);
+        if (r) {
+            aio_epoll_disable(ctx);
+        }
+    } else {
+        event.data.ptr = node;
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        if (is_new) {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        } else {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_MOD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        }
+    }
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    AioHandler *node;
+    int i, ret = 0;
+    struct epoll_event events[128];
+
+    assert(npfd == 1);
+    assert(pfds[0].fd == ctx->epollfd);
+    if (timeout > 0) {
+        ret = qemu_poll_ns(pfds, npfd, timeout);
+    }
+    if (timeout <= 0 || ret > 0) {
+        ret = epoll_wait(ctx->epollfd, events,
+                         sizeof(events) / sizeof(events[0]),
+                         timeout);
+        if (ret <= 0) {
+            goto out;
+        }
+        for (i = 0; i < ret; i++) {
+            int ev = events[i].events;
+            node = events[i].data.ptr;
+            node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) |
+                (ev & EPOLLOUT ? G_IO_OUT : 0) |
+                (ev & EPOLLHUP ? G_IO_HUP : 0) |
+                (ev & EPOLLERR ? G_IO_ERR : 0);
+        }
+    }
+out:
+    return ret;
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    /* Fall back to ppoll when external clients are disabled. */
+    return !aio_external_disabled(ctx) && ctx->epoll_enabled;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                                 unsigned npfd, int64_t timeout)
+{
+    if (!ctx->epoll_available) {
+        return false;
+    }
+    if (aio_epoll_enabled(ctx)) {
+        return true;
+    }
+    if (npfd >= EPOLL_ENABLE_THRESHOLD) {
+        if (aio_epoll_try_enable(ctx)) {
+            return true;
+        } else {
+            aio_epoll_disable(ctx);
+        }
+    }
+    return false;
+}
+
+#else
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    assert(false);
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    return false;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                          unsigned npfd, int64_t timeout)
+{
+    return false;
+}
+
+#endif
+
 static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 {
     AioHandler *node;
@@ -43,11 +204,14 @@ static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
 {
     AioHandler *node;
+    bool is_new = false;
+    bool deleted = false;
 
     node = find_aio_handler(ctx, fd);
 
@@ -66,7 +230,7 @@ void aio_set_fd_handler(AioContext *ctx,
                  * releasing the walking_handlers lock.
                  */
                 QLIST_REMOVE(node, node);
-                g_free(node);
+                deleted = true;
             }
         }
     } else {
@@ -77,25 +241,32 @@ void aio_set_fd_handler(AioContext *ctx,
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
+            is_new = true;
         }
         /* Update handler with latest information */
         node->io_read = io_read;
         node->io_write = io_write;
         node->opaque = opaque;
+        node->is_external = is_external;
 
         node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
         node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
     }
 
+    aio_epoll_update(ctx, node, is_new);
     aio_notify(ctx);
+    if (deleted) {
+        g_free(node);
+    }
 }
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read)
 {
     aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
-                       (IOHandler *)io_read, NULL, notifier);
+                       is_external, (IOHandler *)io_read, NULL, notifier);
 }
 
 bool aio_prepare(AioContext *ctx)
@@ -111,10 +282,12 @@ bool aio_pending(AioContext *ctx)
         int revents;
 
         revents = node->pfd.revents & node->pfd.events;
-        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read &&
+            aio_node_check(ctx, node->is_external)) {
             return true;
         }
-        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write &&
+            aio_node_check(ctx, node->is_external)) {
             return true;
         }
     }
@@ -152,6 +325,7 @@ bool aio_dispatch(AioContext *ctx)
 
         if (!node->deleted &&
             (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
+            aio_node_check(ctx, node->is_external) &&
             node->io_read) {
             node->io_read(node->opaque);
 
@@ -162,6 +336,7 @@ bool aio_dispatch(AioContext *ctx)
         }
         if (!node->deleted &&
             (revents & (G_IO_OUT | G_IO_ERR)) &&
+            aio_node_check(ctx, node->is_external) &&
             node->io_write) {
             node->io_write(node->opaque);
             progress = true;
@@ -257,7 +432,9 @@ bool aio_poll(AioContext *ctx, bool blocking)
 
     /* fill pollfds */
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (!node->deleted && node->pfd.events) {
+        if (!node->deleted && node->pfd.events
+            && !aio_epoll_enabled(ctx)
+            && aio_node_check(ctx, node->is_external)) {
             add_pollfd(node);
         }
     }
@@ -268,7 +445,17 @@ bool aio_poll(AioContext *ctx, bool blocking)
     if (timeout) {
         aio_context_release(ctx);
     }
-    ret = qemu_poll_ns((GPollFD *)pollfds, npfd, timeout);
+    if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) {
+        AioHandler epoll_handler;
+
+        epoll_handler.pfd.fd = ctx->epollfd;
+        epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
+        npfd = 0;
+        add_pollfd(&epoll_handler);
+        ret = aio_epoll(ctx, pollfds, npfd, timeout);
+    } else  {
+        ret = qemu_poll_ns(pollfds, npfd, timeout);
+    }
     if (blocking) {
         atomic_sub(&ctx->notify_me, 2);
     }
@@ -297,3 +484,16 @@ bool aio_poll(AioContext *ctx, bool blocking)
 
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx, Error **errp)
+{
+#ifdef CONFIG_EPOLL_CREATE1
+    assert(!ctx->epollfd);
+    ctx->epollfd = epoll_create1(EPOLL_CLOEXEC);
+    if (ctx->epollfd == -1) {
+        ctx->epoll_available = false;
+    } else {
+        ctx->epoll_available = true;
+    }
+#endif
+}
diff --git a/qemu/aio-win32.c b/qemu/aio-win32.c
index 50a686745..6aaa32a14 100644
--- a/qemu/aio-win32.c
+++ b/qemu/aio-win32.c
@@ -15,6 +15,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block.h"
 #include "qemu/queue.h"
@@ -28,11 +29,13 @@ struct AioHandler {
     GPollFD pfd;
     int deleted;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
@@ -86,6 +89,7 @@ void aio_set_fd_handler(AioContext *ctx,
         node->opaque = opaque;
         node->io_read = io_read;
         node->io_write = io_write;
+        node->is_external = is_external;
 
         event = event_notifier_get_handle(&ctx->notifier);
         WSAEventSelect(node->pfd.fd, event,
@@ -98,6 +102,7 @@ void aio_set_fd_handler(AioContext *ctx,
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *e,
+                            bool is_external,
                             EventNotifierHandler *io_notify)
 {
     AioHandler *node;
@@ -133,6 +138,7 @@ void aio_set_event_notifier(AioContext *ctx,
             node->e = e;
             node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
             node->pfd.events = G_IO_IN;
+            node->is_external = is_external;
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
@@ -304,7 +310,8 @@ bool aio_poll(AioContext *ctx, bool blocking)
     /* fill fd sets */
     count = 0;
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (!node->deleted && node->io_notify) {
+        if (!node->deleted && node->io_notify
+            && aio_node_check(ctx, node->is_external)) {
             events[count++] = event_notifier_get_handle(node->e);
         }
     }
@@ -363,3 +370,7 @@ bool aio_poll(AioContext *ctx, bool blocking)
     aio_context_release(ctx);
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx, Error **errp)
+{
+}
diff --git a/qemu/arch_init.c b/qemu/arch_init.c
index 725c638ec..e3bb1b3ac 100644
--- a/qemu/arch_init.c
+++ b/qemu/arch_init.c
@@ -21,16 +21,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/arch_init.h"
 #include "hw/pci/pci.h"
 #include "hw/audio/audio.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "qmp-commands.h"
 #include "hw/acpi/acpi.h"
+#include "qemu/help_option.h"
 
 #ifdef TARGET_SPARC
 int graphic_width = 1024;
@@ -258,9 +259,7 @@ void do_acpitable_option(const QemuOpts *opts)
 
     acpi_table_add(opts, &err);
     if (err) {
-        error_report("Wrong acpi table provided: %s",
-                     error_get_pretty(err));
-        error_free(err);
+        error_reportf_err(err, "Wrong acpi table provided: ");
         exit(1);
     }
 #endif
diff --git a/qemu/async.c b/qemu/async.c
index efce14b63..b4bf205a0 100644
--- a/qemu/async.c
+++ b/qemu/async.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "block/thread-pool.h"
@@ -59,6 +61,11 @@ QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
     return bh;
 }
 
+void aio_bh_call(QEMUBH *bh)
+{
+    bh->cb(bh->opaque);
+}
+
 /* Multiple occurrences of aio_bh_poll cannot be called concurrently */
 int aio_bh_poll(AioContext *ctx)
 {
@@ -84,7 +91,7 @@ int aio_bh_poll(AioContext *ctx)
                 ret = 1;
             }
             bh->idle = 0;
-            bh->cb(bh->opaque);
+            aio_bh_call(bh);
         }
     }
 
@@ -247,7 +254,7 @@ aio_ctx_finalize(GSource     *source)
     }
     qemu_mutex_unlock(&ctx->bh_lock);
 
-    aio_set_event_notifier(ctx, &ctx->notifier, NULL);
+    aio_set_event_notifier(ctx, &ctx->notifier, false, NULL);
     event_notifier_cleanup(&ctx->notifier);
     rfifolock_destroy(&ctx->lock);
     qemu_mutex_destroy(&ctx->bh_lock);
@@ -320,15 +327,22 @@ AioContext *aio_context_new(Error **errp)
 {
     int ret;
     AioContext *ctx;
+    Error *local_err = NULL;
+
     ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    aio_context_setup(ctx, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto fail;
+    }
     ret = event_notifier_init(&ctx->notifier, false);
     if (ret < 0) {
-        g_source_destroy(&ctx->source);
         error_setg_errno(errp, -ret, "Failed to initialize event notifier");
-        return NULL;
+        goto fail;
     }
     g_source_set_can_recurse(&ctx->source, true);
     aio_set_event_notifier(ctx, &ctx->notifier,
+                           false,
                            (EventNotifierHandler *)
                            event_notifier_dummy_cb);
     ctx->thread_pool = NULL;
@@ -339,6 +353,9 @@ AioContext *aio_context_new(Error **errp)
     ctx->notify_dummy_bh = aio_bh_new(ctx, notify_dummy_bh, NULL);
 
     return ctx;
+fail:
+    g_source_destroy(&ctx->source);
+    return NULL;
 }
 
 void aio_context_ref(AioContext *ctx)
diff --git a/qemu/audio/alsaaudio.c b/qemu/audio/alsaaudio.c
index 6315b2d74..3652a7b5f 100644
--- a/qemu/audio/alsaaudio.c
+++ b/qemu/audio/alsaaudio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <alsa/asoundlib.h>
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
diff --git a/qemu/audio/audio.c b/qemu/audio/audio.c
index 5be4b15fc..e60c124de 100644
--- a/qemu/audio/audio.c
+++ b/qemu/audio/audio.c
@@ -21,11 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio.h"
 #include "monitor/monitor.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
 #define AUDIO_CAP "audio"
 #include "audio_int.h"
@@ -1806,9 +1808,6 @@ static void audio_init (void)
     atexit (audio_atexit);
 
     s->ts = timer_new_ns(QEMU_CLOCK_VIRTUAL, audio_timer, s);
-    if (!s->ts) {
-        hw_error("Could not create audio timer\n");
-    }
 
     audio_process_options ("AUDIO", audio_options);
 
@@ -1859,12 +1858,8 @@ static void audio_init (void)
 
     if (!done) {
         done = !audio_driver_init (s, &no_audio_driver);
-        if (!done) {
-            hw_error("Could not initialize audio subsystem\n");
-        }
-        else {
-            dolog ("warning: Using timer based audio emulation\n");
-        }
+        assert(done);
+        dolog("warning: Using timer based audio emulation\n");
     }
 
     if (conf.period.hertz <= 0) {
@@ -1875,8 +1870,7 @@ static void audio_init (void)
         }
         conf.period.ticks = 1;
     } else {
-        conf.period.ticks =
-            muldiv64 (1, get_ticks_per_sec (), conf.period.hertz);
+        conf.period.ticks = NANOSECONDS_PER_SECOND / conf.period.hertz;
     }
 
     e = qemu_add_vm_change_state_handler (audio_vm_change_state_handler, s);
diff --git a/qemu/audio/audio.h b/qemu/audio/audio.h
index e7ea39777..b41a97053 100644
--- a/qemu/audio/audio.h
+++ b/qemu/audio/audio.h
@@ -24,7 +24,6 @@
 #ifndef QEMU_AUDIO_H
 #define QEMU_AUDIO_H
 
-#include "config-host.h"
 #include "qemu/queue.h"
 
 typedef void (*audio_callback_fn) (void *opaque, int avail);
diff --git a/qemu/audio/audio_pt_int.c b/qemu/audio/audio_pt_int.c
index 9a9c306a9..21ff9c580 100644
--- a/qemu/audio/audio_pt_int.c
+++ b/qemu/audio/audio_pt_int.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
diff --git a/qemu/audio/audio_win_int.c b/qemu/audio/audio_win_int.c
index e1324056a..6900008d0 100644
--- a/qemu/audio/audio_win_int.c
+++ b/qemu/audio/audio_win_int.c
@@ -1,5 +1,6 @@
 /* public domain */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 #define AUDIO_CAP "win-int"
diff --git a/qemu/audio/coreaudio.c b/qemu/audio/coreaudio.c
index 6dfd63eb4..d4ad22459 100644
--- a/qemu/audio/coreaudio.c
+++ b/qemu/audio/coreaudio.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <CoreAudio/CoreAudio.h>
-#include <string.h>             /* strerror */
 #include <pthread.h>            /* pthread_X */
 
 #include "qemu-common.h"
@@ -32,6 +32,10 @@
 #define AUDIO_CAP "coreaudio"
 #include "audio_int.h"
 
+#ifndef MAC_OS_X_VERSION_10_6
+#define MAC_OS_X_VERSION_10_6 1060
+#endif
+
 static int isAtexit;
 
 typedef struct {
@@ -45,11 +49,233 @@ typedef struct coreaudioVoiceOut {
     AudioDeviceID outputDeviceID;
     UInt32 audioDevicePropertyBufferFrameSize;
     AudioStreamBasicDescription outputStreamBasicDescription;
+    AudioDeviceIOProcID ioprocid;
     int live;
     int decr;
     int rpos;
 } coreaudioVoiceOut;
 
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
+/* The APIs used here only become available from 10.6 */
+
+static OSStatus coreaudio_get_voice(AudioDeviceID *id)
+{
+    UInt32 size = sizeof(*id);
+    AudioObjectPropertyAddress addr = {
+        kAudioHardwarePropertyDefaultOutputDevice,
+        kAudioObjectPropertyScopeGlobal,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(kAudioObjectSystemObject,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      id);
+}
+
+static OSStatus coreaudio_get_framesizerange(AudioDeviceID id,
+                                             AudioValueRange *framerange)
+{
+    UInt32 size = sizeof(*framerange);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSizeRange,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      framerange);
+}
+
+static OSStatus coreaudio_get_framesize(AudioDeviceID id, UInt32 *framesize)
+{
+    UInt32 size = sizeof(*framesize);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSize,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      framesize);
+}
+
+static OSStatus coreaudio_set_framesize(AudioDeviceID id, UInt32 *framesize)
+{
+    UInt32 size = sizeof(*framesize);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSize,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectSetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      size,
+                                      framesize);
+}
+
+static OSStatus coreaudio_get_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d)
+{
+    UInt32 size = sizeof(*d);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyStreamFormat,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      d);
+}
+
+static OSStatus coreaudio_set_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d)
+{
+    UInt32 size = sizeof(*d);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyStreamFormat,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectSetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      size,
+                                      d);
+}
+
+static OSStatus coreaudio_get_isrunning(AudioDeviceID id, UInt32 *result)
+{
+    UInt32 size = sizeof(*result);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyDeviceIsRunning,
+        kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      result);
+}
+#else
+/* Legacy versions of functions using deprecated APIs */
+
+static OSStatus coreaudio_get_voice(AudioDeviceID *id)
+{
+    UInt32 size = sizeof(*id);
+
+    return AudioHardwareGetProperty(
+        kAudioHardwarePropertyDefaultOutputDevice,
+        &size,
+        id);
+}
+
+static OSStatus coreaudio_get_framesizerange(AudioDeviceID id,
+                                             AudioValueRange *framerange)
+{
+    UInt32 size = sizeof(*framerange);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        0,
+        kAudioDevicePropertyBufferFrameSizeRange,
+        &size,
+        framerange);
+}
+
+static OSStatus coreaudio_get_framesize(AudioDeviceID id, UInt32 *framesize)
+{
+    UInt32 size = sizeof(*framesize);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        false,
+        kAudioDevicePropertyBufferFrameSize,
+        &size,
+        framesize);
+}
+
+static OSStatus coreaudio_set_framesize(AudioDeviceID id, UInt32 *framesize)
+{
+    UInt32 size = sizeof(*framesize);
+
+    return AudioDeviceSetProperty(
+        id,
+        NULL,
+        0,
+        false,
+        kAudioDevicePropertyBufferFrameSize,
+        size,
+        framesize);
+}
+
+static OSStatus coreaudio_get_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d)
+{
+    UInt32 size = sizeof(*d);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        false,
+        kAudioDevicePropertyStreamFormat,
+        &size,
+        d);
+}
+
+static OSStatus coreaudio_set_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d)
+{
+    UInt32 size = sizeof(*d);
+
+    return AudioDeviceSetProperty(
+        id,
+        0,
+        0,
+        0,
+        kAudioDevicePropertyStreamFormat,
+        size,
+        d);
+}
+
+static OSStatus coreaudio_get_isrunning(AudioDeviceID id, UInt32 *result)
+{
+    UInt32 size = sizeof(*result);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        0,
+        kAudioDevicePropertyDeviceIsRunning,
+        &size,
+        result);
+}
+#endif
+
 static void coreaudio_logstatus (OSStatus status)
 {
     const char *str = "BUG";
@@ -144,10 +370,7 @@ static inline UInt32 isPlaying (AudioDeviceID outputDeviceID)
 {
     OSStatus status;
     UInt32 result = 0;
-    UInt32 propertySize = sizeof(outputDeviceID);
-    status = AudioDeviceGetProperty(
-        outputDeviceID, 0, 0,
-        kAudioDevicePropertyDeviceIsRunning, &propertySize, &result);
+    status = coreaudio_get_isrunning(outputDeviceID, &result);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr(status,
                          "Could not determine whether Device is playing\n");
@@ -288,7 +511,6 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
 {
     OSStatus status;
     coreaudioVoiceOut *core = (coreaudioVoiceOut *) hw;
-    UInt32 propertySize;
     int err;
     const char *typ = "playback";
     AudioValueRange frameRange;
@@ -303,12 +525,7 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
 
     audio_pcm_init_info (&hw->info, as);
 
-    /* open default output device */
-    propertySize = sizeof(core->outputDeviceID);
-    status = AudioHardwareGetProperty(
-        kAudioHardwarePropertyDefaultOutputDevice,
-        &propertySize,
-        &core->outputDeviceID);
+    status = coreaudio_get_voice(&core->outputDeviceID);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get default output Device\n");
@@ -320,14 +537,8 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
     }
 
     /* get minimum and maximum buffer frame sizes */
-    propertySize = sizeof(frameRange);
-    status = AudioDeviceGetProperty(
-        core->outputDeviceID,
-        0,
-        0,
-        kAudioDevicePropertyBufferFrameSizeRange,
-        &propertySize,
-        &frameRange);
+    status = coreaudio_get_framesizerange(core->outputDeviceID,
+                                          &frameRange);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get device buffer frame range\n");
@@ -347,15 +558,8 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
     }
 
     /* set Buffer Frame Size */
-    propertySize = sizeof(core->audioDevicePropertyBufferFrameSize);
-    status = AudioDeviceSetProperty(
-        core->outputDeviceID,
-        NULL,
-        0,
-        false,
-        kAudioDevicePropertyBufferFrameSize,
-        propertySize,
-        &core->audioDevicePropertyBufferFrameSize);
+    status = coreaudio_set_framesize(core->outputDeviceID,
+                                     &core->audioDevicePropertyBufferFrameSize);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not set device buffer frame size %" PRIu32 "\n",
@@ -364,14 +568,8 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
     }
 
     /* get Buffer Frame Size */
-    propertySize = sizeof(core->audioDevicePropertyBufferFrameSize);
-    status = AudioDeviceGetProperty(
-        core->outputDeviceID,
-        0,
-        false,
-        kAudioDevicePropertyBufferFrameSize,
-        &propertySize,
-        &core->audioDevicePropertyBufferFrameSize);
+    status = coreaudio_get_framesize(core->outputDeviceID,
+                                     &core->audioDevicePropertyBufferFrameSize);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get device buffer frame size\n");
@@ -380,14 +578,8 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
     hw->samples = conf->nbuffers * core->audioDevicePropertyBufferFrameSize;
 
     /* get StreamFormat */
-    propertySize = sizeof(core->outputStreamBasicDescription);
-    status = AudioDeviceGetProperty(
-        core->outputDeviceID,
-        0,
-        false,
-        kAudioDevicePropertyStreamFormat,
-        &propertySize,
-        &core->outputStreamBasicDescription);
+    status = coreaudio_get_streamformat(core->outputDeviceID,
+                                        &core->outputStreamBasicDescription);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get Device Stream properties\n");
@@ -397,15 +589,8 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
 
     /* set Samplerate */
     core->outputStreamBasicDescription.mSampleRate = (Float64) as->freq;
-    propertySize = sizeof(core->outputStreamBasicDescription);
-    status = AudioDeviceSetProperty(
-        core->outputDeviceID,
-        0,
-        0,
-        0,
-        kAudioDevicePropertyStreamFormat,
-        propertySize,
-        &core->outputStreamBasicDescription);
+    status = coreaudio_set_streamformat(core->outputDeviceID,
+                                        &core->outputStreamBasicDescription);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ, "Could not set samplerate %d\n",
                            as->freq);
@@ -414,8 +599,12 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
     }
 
     /* set Callback */
-    status = AudioDeviceAddIOProc(core->outputDeviceID, audioDeviceIOProc, hw);
-    if (status != kAudioHardwareNoError) {
+    core->ioprocid = NULL;
+    status = AudioDeviceCreateIOProcID(core->outputDeviceID,
+                                       audioDeviceIOProc,
+                                       hw,
+                                       &core->ioprocid);
+    if (status != kAudioHardwareNoError || core->ioprocid == NULL) {
         coreaudio_logerr2 (status, typ, "Could not set IOProc\n");
         core->outputDeviceID = kAudioDeviceUnknown;
         return -1;
@@ -423,10 +612,10 @@ static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
 
     /* start Playback */
     if (!isPlaying(core->outputDeviceID)) {
-        status = AudioDeviceStart(core->outputDeviceID, audioDeviceIOProc);
+        status = AudioDeviceStart(core->outputDeviceID, core->ioprocid);
         if (status != kAudioHardwareNoError) {
             coreaudio_logerr2 (status, typ, "Could not start playback\n");
-            AudioDeviceRemoveIOProc(core->outputDeviceID, audioDeviceIOProc);
+            AudioDeviceDestroyIOProcID(core->outputDeviceID, core->ioprocid);
             core->outputDeviceID = kAudioDeviceUnknown;
             return -1;
         }
@@ -444,15 +633,15 @@ static void coreaudio_fini_out (HWVoiceOut *hw)
     if (!isAtexit) {
         /* stop playback */
         if (isPlaying(core->outputDeviceID)) {
-            status = AudioDeviceStop(core->outputDeviceID, audioDeviceIOProc);
+            status = AudioDeviceStop(core->outputDeviceID, core->ioprocid);
             if (status != kAudioHardwareNoError) {
                 coreaudio_logerr (status, "Could not stop playback\n");
             }
         }
 
         /* remove callback */
-        status = AudioDeviceRemoveIOProc(core->outputDeviceID,
-                                         audioDeviceIOProc);
+        status = AudioDeviceDestroyIOProcID(core->outputDeviceID,
+                                            core->ioprocid);
         if (status != kAudioHardwareNoError) {
             coreaudio_logerr (status, "Could not remove IOProc\n");
         }
@@ -475,7 +664,7 @@ static int coreaudio_ctl_out (HWVoiceOut *hw, int cmd, ...)
     case VOICE_ENABLE:
         /* start playback */
         if (!isPlaying(core->outputDeviceID)) {
-            status = AudioDeviceStart(core->outputDeviceID, audioDeviceIOProc);
+            status = AudioDeviceStart(core->outputDeviceID, core->ioprocid);
             if (status != kAudioHardwareNoError) {
                 coreaudio_logerr (status, "Could not resume playback\n");
             }
@@ -486,7 +675,8 @@ static int coreaudio_ctl_out (HWVoiceOut *hw, int cmd, ...)
         /* stop playback */
         if (!isAtexit) {
             if (isPlaying(core->outputDeviceID)) {
-                status = AudioDeviceStop(core->outputDeviceID, audioDeviceIOProc);
+                status = AudioDeviceStop(core->outputDeviceID,
+                                         core->ioprocid);
                 if (status != kAudioHardwareNoError) {
                     coreaudio_logerr (status, "Could not pause playback\n");
                 }
diff --git a/qemu/audio/dsoundaudio.c b/qemu/audio/dsoundaudio.c
index e9472c105..516846eb8 100644
--- a/qemu/audio/dsoundaudio.c
+++ b/qemu/audio/dsoundaudio.c
@@ -26,6 +26,7 @@
  * SEAL 1.07 by Carlos 'pel' Hasan was used as documentation
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
diff --git a/qemu/audio/mixeng.c b/qemu/audio/mixeng.c
index 0e4976f27..981b97a96 100644
--- a/qemu/audio/mixeng.c
+++ b/qemu/audio/mixeng.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
diff --git a/qemu/audio/noaudio.c b/qemu/audio/noaudio.c
index 50db1f344..b360c199a 100644
--- a/qemu/audio/noaudio.c
+++ b/qemu/audio/noaudio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 #include "qemu/timer.h"
@@ -48,8 +49,8 @@ static int no_run_out (HWVoiceOut *hw, int live)
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     ticks = now - no->old_ticks;
-    bytes = muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
-    bytes = audio_MIN (bytes, INT_MAX);
+    bytes = muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
+    bytes = audio_MIN(bytes, INT_MAX);
     samples = bytes >> hw->info.shift;
 
     no->old_ticks = now;
@@ -60,7 +61,7 @@ static int no_run_out (HWVoiceOut *hw, int live)
 
 static int no_write (SWVoiceOut *sw, void *buf, int len)
 {
-    return audio_pcm_sw_write (sw, buf, len);
+    return audio_pcm_sw_write(sw, buf, len);
 }
 
 static int no_init_out(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque)
@@ -105,7 +106,7 @@ static int no_run_in (HWVoiceIn *hw)
         int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         int64_t ticks = now - no->old_ticks;
         int64_t bytes =
-            muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
+            muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
 
         no->old_ticks = now;
         bytes = audio_MIN (bytes, INT_MAX);
diff --git a/qemu/audio/ossaudio.c b/qemu/audio/ossaudio.c
index 7dbe3332d..349e9dd53 100644
--- a/qemu/audio/ossaudio.c
+++ b/qemu/audio/ossaudio.c
@@ -21,9 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
-#include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/soundcard.h>
 #include "qemu-common.h"
diff --git a/qemu/audio/paaudio.c b/qemu/audio/paaudio.c
index fea607166..57678e72e 100644
--- a/qemu/audio/paaudio.c
+++ b/qemu/audio/paaudio.c
@@ -1,4 +1,5 @@
 /* public domain */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
diff --git a/qemu/audio/sdlaudio.c b/qemu/audio/sdlaudio.c
index 1140f2ea0..db69fe141 100644
--- a/qemu/audio/sdlaudio.c
+++ b/qemu/audio/sdlaudio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <SDL.h>
 #include <SDL_thread.h>
 #include "qemu-common.h"
diff --git a/qemu/audio/spiceaudio.c b/qemu/audio/spiceaudio.c
index 42ae4a45f..dea71d37a 100644
--- a/qemu/audio/spiceaudio.c
+++ b/qemu/audio/spiceaudio.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
@@ -103,11 +104,11 @@ static int rate_get_samples (struct audio_pcm_info *info, SpiceRateCtl *rate)
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     ticks = now - rate->start_ticks;
-    bytes = muldiv64 (ticks, info->bytes_per_second, get_ticks_per_sec ());
+    bytes = muldiv64(ticks, info->bytes_per_second, NANOSECONDS_PER_SECOND);
     samples = (bytes - rate->bytes_sent) >> info->shift;
     if (samples < 0 || samples > 65536) {
         error_report("Resetting rate control (%" PRId64 " samples)", samples);
-        rate_start (rate);
+        rate_start(rate);
         samples = 0;
     }
     rate->bytes_sent += samples << info->shift;
diff --git a/qemu/audio/wavaudio.c b/qemu/audio/wavaudio.c
index c586020c5..345952e51 100644
--- a/qemu/audio/wavaudio.c
+++ b/qemu/audio/wavaudio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "audio.h"
@@ -50,7 +51,7 @@ static int wav_run_out (HWVoiceOut *hw, int live)
     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     int64_t ticks = now - wav->old_ticks;
     int64_t bytes =
-        muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
+        muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
 
     if (bytes > INT_MAX) {
         samples = INT_MAX >> hw->info.shift;
diff --git a/qemu/audio/wavcapture.c b/qemu/audio/wavcapture.c
index 86e905627..8bfb9e765 100644
--- a/qemu/audio/wavcapture.c
+++ b/qemu/audio/wavcapture.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "monitor/monitor.h"
 #include "qemu/error-report.h"
diff --git a/qemu/backends/baum.c b/qemu/backends/baum.c
index a69aafff4..c537141b2 100644
--- a/qemu/backends/baum.c
+++ b/qemu/backends/baum.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "qemu/timer.h"
@@ -303,7 +305,7 @@ static int baum_eat_packet(BaumDriverState *baum, const uint8_t *buf, int len)
                 return 0;
             cur++;
         }
-        DPRINTF("Dropped %d bytes!\n", cur - buf);
+        DPRINTF("Dropped %td bytes!\n", cur - buf);
     }
 
 #define EAT(c) do {\
@@ -335,7 +337,7 @@ static int baum_eat_packet(BaumDriverState *baum, const uint8_t *buf, int len)
 
         /* Allow 100ms to complete the DisplayData packet */
         timer_mod(baum->cellCount_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                       get_ticks_per_sec() / 10);
+                       NANOSECONDS_PER_SECOND / 10);
         for (i = 0; i < baum->x * baum->y ; i++) {
             EAT(c);
             cells[i] = c;
@@ -561,8 +563,12 @@ static void baum_close(struct CharDriverState *chr)
     g_free(baum);
 }
 
-CharDriverState *chr_baum_init(void)
+static CharDriverState *chr_baum_init(const char *id,
+                                      ChardevBackend *backend,
+                                      ChardevReturn *ret,
+                                      Error **errp)
 {
+    ChardevCommon *common = backend->u.braille.data;
     BaumDriverState *baum;
     CharDriverState *chr;
     brlapi_handle_t *handle;
@@ -573,8 +579,12 @@ CharDriverState *chr_baum_init(void)
 #endif
     int tty;
 
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     baum = g_malloc0(sizeof(BaumDriverState));
-    baum->chr = chr = qemu_chr_alloc();
+    baum->chr = chr;
 
     chr->opaque = baum;
     chr->chr_write = baum_write;
@@ -586,14 +596,16 @@ CharDriverState *chr_baum_init(void)
 
     baum->brlapi_fd = brlapi__openConnection(handle, NULL, NULL);
     if (baum->brlapi_fd == -1) {
-        brlapi_perror("baum_init: brlapi_openConnection");
+        error_setg(errp, "brlapi__openConnection: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail_handle;
     }
 
     baum->cellCount_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, baum_cellCount_timer_cb, baum);
 
     if (brlapi__getDisplaySize(handle, &baum->x, &baum->y) == -1) {
-        brlapi_perror("baum_init: brlapi_getDisplaySize");
+        error_setg(errp, "brlapi__getDisplaySize: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -609,7 +621,8 @@ CharDriverState *chr_baum_init(void)
         tty = BRLAPI_TTY_DEFAULT;
 
     if (brlapi__enterTtyMode(handle, tty, NULL) == -1) {
-        brlapi_perror("baum_init: brlapi_enterTtyMode");
+        error_setg(errp, "brlapi__enterTtyMode: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -629,7 +642,8 @@ fail_handle:
 
 static void register_types(void)
 {
-    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL);
+    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL,
+                         chr_baum_init);
 }
 
 type_init(register_types);
diff --git a/qemu/backends/hostmem-file.c b/qemu/backends/hostmem-file.c
index 4b5536101..5c4b808c1 100644
--- a/qemu/backends/hostmem-file.c
+++ b/qemu/backends/hostmem-file.c
@@ -9,6 +9,8 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "sysemu/hostmem.h"
 #include "sysemu/sysemu.h"
@@ -50,11 +52,14 @@ file_backend_memory_alloc(HostMemoryBackend *backend, Error **errp)
     error_setg(errp, "-mem-path not supported on this host");
 #else
     if (!memory_region_size(&backend->mr)) {
+        gchar *path;
         backend->force_prealloc = mem_prealloc;
+        path = object_get_canonical_path(OBJECT(backend));
         memory_region_init_ram_from_file(&backend->mr, OBJECT(backend),
-                                 object_get_canonical_path(OBJECT(backend)),
+                                 path,
                                  backend->size, fb->share,
                                  fb->mem_path, errp);
+        g_free(path);
     }
 #endif
 }
@@ -83,9 +88,7 @@ static void set_mem_path(Object *o, const char *str, Error **errp)
         error_setg(errp, "cannot change property value");
         return;
     }
-    if (fb->mem_path) {
-        g_free(fb->mem_path);
-    }
+    g_free(fb->mem_path);
     fb->mem_path = g_strdup(str);
 }
 
@@ -118,11 +121,19 @@ file_backend_instance_init(Object *o)
                             set_mem_path, NULL);
 }
 
+static void file_backend_instance_finalize(Object *o)
+{
+    HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o);
+
+    g_free(fb->mem_path);
+}
+
 static const TypeInfo file_backend_info = {
     .name = TYPE_MEMORY_BACKEND_FILE,
     .parent = TYPE_MEMORY_BACKEND,
     .class_init = file_backend_class_init,
     .instance_init = file_backend_instance_init,
+    .instance_finalize = file_backend_instance_finalize,
     .instance_size = sizeof(HostMemoryBackendFile),
 };
 
diff --git a/qemu/backends/hostmem-ram.c b/qemu/backends/hostmem-ram.c
index a67a13452..04a7ac362 100644
--- a/qemu/backends/hostmem-ram.c
+++ b/qemu/backends/hostmem-ram.c
@@ -9,7 +9,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "sysemu/hostmem.h"
+#include "qapi/error.h"
 #include "qom/object_interfaces.h"
 
 #define TYPE_MEMORY_BACKEND_RAM "memory-backend-ram"
diff --git a/qemu/backends/hostmem.c b/qemu/backends/hostmem.c
index 41ba2af52..6e28be11e 100644
--- a/qemu/backends/hostmem.c
+++ b/qemu/backends/hostmem.c
@@ -9,8 +9,10 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "sysemu/hostmem.h"
 #include "hw/boards.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "qapi-types.h"
 #include "qapi-visit.h"
@@ -26,18 +28,18 @@ QEMU_BUILD_BUG_ON(HOST_MEM_POLICY_INTERLEAVE != MPOL_INTERLEAVE);
 #endif
 
 static void
-host_memory_backend_get_size(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+host_memory_backend_get_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint64_t value = backend->size;
 
-    visit_type_size(v, &value, name, errp);
+    visit_type_size(v, name, &value, errp);
 }
 
 static void
-host_memory_backend_set_size(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+host_memory_backend_set_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     Error *local_err = NULL;
@@ -48,7 +50,7 @@ host_memory_backend_set_size(Object *obj, Visitor *v, void *opaque,
         goto out;
     }
 
-    visit_type_size(v, &value, name, &local_err);
+    visit_type_size(v, name, &value, &local_err);
     if (local_err) {
         goto out;
     }
@@ -63,8 +65,8 @@ out:
 }
 
 static void
-host_memory_backend_get_host_nodes(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+host_memory_backend_get_host_nodes(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint16List *host_nodes = NULL;
@@ -91,18 +93,18 @@ host_memory_backend_get_host_nodes(Object *obj, Visitor *v, void *opaque,
         node = &(*node)->next;
     } while (true);
 
-    visit_type_uint16List(v, &host_nodes, name, errp);
+    visit_type_uint16List(v, name, &host_nodes, errp);
 }
 
 static void
-host_memory_backend_set_host_nodes(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+host_memory_backend_set_host_nodes(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
 #ifdef CONFIG_NUMA
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint16List *l = NULL;
 
-    visit_type_uint16List(v, &l, name, errp);
+    visit_type_uint16List(v, name, &l, errp);
 
     while (l) {
         bitmap_set(backend->host_nodes, l->value, 1);
@@ -313,9 +315,11 @@ host_memory_backend_memory_complete(UserCreatable *uc, Error **errp)
         assert(maxnode <= MAX_NODES);
         if (mbind(ptr, sz, backend->policy,
                   maxnode ? backend->host_nodes : NULL, maxnode + 1, flags)) {
-            error_setg_errno(errp, errno,
-                             "cannot bind memory to host NUMA nodes");
-            return;
+            if (backend->policy != MPOL_DEFAULT || errno != ENOSYS) {
+                error_setg_errno(errp, errno,
+                                 "cannot bind memory to host NUMA nodes");
+                return;
+            }
         }
 #endif
         /* Preallocate memory after the NUMA policy has been instantiated.
diff --git a/qemu/backends/msmouse.c b/qemu/backends/msmouse.c
index 0119110a4..8dea5a130 100644
--- a/qemu/backends/msmouse.c
+++ b/qemu/backends/msmouse.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "ui/console.h"
@@ -63,11 +63,18 @@ static void msmouse_chr_close (struct CharDriverState *chr)
     g_free (chr);
 }
 
-CharDriverState *qemu_chr_open_msmouse(void)
+static CharDriverState *qemu_chr_open_msmouse(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevCommon *common = backend->u.msmouse.data;
     CharDriverState *chr;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     chr->chr_write = msmouse_chr_write;
     chr->chr_close = msmouse_chr_close;
     chr->explicit_be_open = true;
@@ -79,7 +86,8 @@ CharDriverState *qemu_chr_open_msmouse(void)
 
 static void register_types(void)
 {
-    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL);
+    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL,
+                         qemu_chr_open_msmouse);
 }
 
 type_init(register_types);
diff --git a/qemu/backends/rng-egd.c b/qemu/backends/rng-egd.c
index 6c1340963..7a1b9242d 100644
--- a/qemu/backends/rng-egd.c
+++ b/qemu/backends/rng-egd.c
@@ -10,8 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng.h"
 #include "sysemu/char.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "hw/qdev.h" /* just for DEFINE_PROP_CHR */
 
@@ -24,33 +26,12 @@ typedef struct RngEgd
 
     CharDriverState *chr;
     char *chr_name;
-
-    GSList *requests;
 } RngEgd;
 
-typedef struct RngRequest
-{
-    EntropyReceiveFunc *receive_entropy;
-    uint8_t *data;
-    void *opaque;
-    size_t offset;
-    size_t size;
-} RngRequest;
-
-static void rng_egd_request_entropy(RngBackend *b, size_t size,
-                                    EntropyReceiveFunc *receive_entropy,
-                                    void *opaque)
+static void rng_egd_request_entropy(RngBackend *b, RngRequest *req)
 {
     RngEgd *s = RNG_EGD(b);
-    RngRequest *req;
-
-    req = g_malloc(sizeof(*req));
-
-    req->offset = 0;
-    req->size = size;
-    req->receive_entropy = receive_entropy;
-    req->opaque = opaque;
-    req->data = g_malloc(req->size);
+    size_t size = req->size;
 
     while (size > 0) {
         uint8_t header[2];
@@ -64,24 +45,15 @@ static void rng_egd_request_entropy(RngBackend *b, size_t size,
 
         size -= len;
     }
-
-    s->requests = g_slist_append(s->requests, req);
-}
-
-static void rng_egd_free_request(RngRequest *req)
-{
-    g_free(req->data);
-    g_free(req);
 }
 
 static int rng_egd_chr_can_read(void *opaque)
 {
     RngEgd *s = RNG_EGD(opaque);
-    GSList *i;
+    RngRequest *req;
     int size = 0;
 
-    for (i = s->requests; i; i = i->next) {
-        RngRequest *req = i->data;
+    QSIMPLEQ_FOREACH(req, &s->parent.requests, next) {
         size += req->size - req->offset;
     }
 
@@ -93,8 +65,8 @@ static void rng_egd_chr_read(void *opaque, const uint8_t *buf, int size)
     RngEgd *s = RNG_EGD(opaque);
     size_t buf_offset = 0;
 
-    while (size > 0 && s->requests) {
-        RngRequest *req = s->requests->data;
+    while (size > 0 && !QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        RngRequest *req = QSIMPLEQ_FIRST(&s->parent.requests);
         int len = MIN(size, req->size - req->offset);
 
         memcpy(req->data + req->offset, buf + buf_offset, len);
@@ -103,38 +75,13 @@ static void rng_egd_chr_read(void *opaque, const uint8_t *buf, int size)
         size -= len;
 
         if (req->offset == req->size) {
-            s->requests = g_slist_remove_link(s->requests, s->requests);
-
             req->receive_entropy(req->opaque, req->data, req->size);
 
-            rng_egd_free_request(req);
+            rng_backend_finalize_request(&s->parent, req);
         }
     }
 }
 
-static void rng_egd_free_requests(RngEgd *s)
-{
-    GSList *i;
-
-    for (i = s->requests; i; i = i->next) {
-        rng_egd_free_request(i->data);
-    }
-
-    g_slist_free(s->requests);
-    s->requests = NULL;
-}
-
-static void rng_egd_cancel_requests(RngBackend *b)
-{
-    RngEgd *s = RNG_EGD(b);
-
-    /* We simply delete the list of pending requests.  If there is data in the 
-     * queue waiting to be read, this is okay, because there will always be
-     * more data than we requested originally
-     */
-    rng_egd_free_requests(s);
-}
-
 static void rng_egd_opened(RngBackend *b, Error **errp)
 {
     RngEgd *s = RNG_EGD(b);
@@ -203,8 +150,6 @@ static void rng_egd_finalize(Object *obj)
     }
 
     g_free(s->chr_name);
-
-    rng_egd_free_requests(s);
 }
 
 static void rng_egd_class_init(ObjectClass *klass, void *data)
@@ -212,7 +157,6 @@ static void rng_egd_class_init(ObjectClass *klass, void *data)
     RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
 
     rbc->request_entropy = rng_egd_request_entropy;
-    rbc->cancel_requests = rng_egd_cancel_requests;
     rbc->opened = rng_egd_opened;
 }
 
diff --git a/qemu/backends/rng-random.c b/qemu/backends/rng-random.c
index 4e51f4609..2e44e2519 100644
--- a/qemu/backends/rng-random.c
+++ b/qemu/backends/rng-random.c
@@ -10,8 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng-random.h"
 #include "sysemu/rng.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/main-loop.h"
 
@@ -21,10 +23,6 @@ struct RndRandom
 
     int fd;
     char *filename;
-
-    EntropyReceiveFunc *receive_func;
-    void *opaque;
-    size_t size;
 };
 
 /**
@@ -37,36 +35,35 @@ struct RndRandom
 static void entropy_available(void *opaque)
 {
     RndRandom *s = RNG_RANDOM(opaque);
-    uint8_t buffer[s->size];
-    ssize_t len;
 
-    len = read(s->fd, buffer, s->size);
-    if (len < 0 && errno == EAGAIN) {
-        return;
-    }
-    g_assert(len != -1);
+    while (!QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        RngRequest *req = QSIMPLEQ_FIRST(&s->parent.requests);
+        ssize_t len;
+
+        len = read(s->fd, req->data, req->size);
+        if (len < 0 && errno == EAGAIN) {
+            return;
+        }
+        g_assert(len != -1);
 
-    s->receive_func(s->opaque, buffer, len);
-    s->receive_func = NULL;
+        req->receive_entropy(req->opaque, req->data, len);
 
+        rng_backend_finalize_request(&s->parent, req);
+    }
+
+    /* We've drained all requests, the fd handler can be reset. */
     qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
 }
 
-static void rng_random_request_entropy(RngBackend *b, size_t size,
-                                        EntropyReceiveFunc *receive_entropy,
-                                        void *opaque)
+static void rng_random_request_entropy(RngBackend *b, RngRequest *req)
 {
     RndRandom *s = RNG_RANDOM(b);
 
-    if (s->receive_func) {
-        s->receive_func(s->opaque, NULL, 0);
+    if (QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        /* If there are no pending requests yet, we need to
+         * install our fd handler. */
+        qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
     }
-
-    s->receive_func = receive_entropy;
-    s->opaque = opaque;
-    s->size = size;
-
-    qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
 }
 
 static void rng_random_opened(RngBackend *b, Error **errp)
diff --git a/qemu/backends/rng.c b/qemu/backends/rng.c
index 5065fdc15..398ebe4a7 100644
--- a/qemu/backends/rng.c
+++ b/qemu/backends/rng.c
@@ -10,7 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qom/object_interfaces.h"
 
@@ -19,18 +21,20 @@ void rng_backend_request_entropy(RngBackend *s, size_t size,
                                  void *opaque)
 {
     RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+    RngRequest *req;
 
     if (k->request_entropy) {
-        k->request_entropy(s, size, receive_entropy, opaque);
-    }
-}
+        req = g_malloc(sizeof(*req));
 
-void rng_backend_cancel_requests(RngBackend *s)
-{
-    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+        req->offset = 0;
+        req->size = size;
+        req->receive_entropy = receive_entropy;
+        req->opaque = opaque;
+        req->data = g_malloc(req->size);
 
-    if (k->cancel_requests) {
-        k->cancel_requests(s);
+        k->request_entropy(s, req);
+
+        QSIMPLEQ_INSERT_TAIL(&s->requests, req, next);
     }
 }
 
@@ -72,14 +76,48 @@ static void rng_backend_prop_set_opened(Object *obj, bool value, Error **errp)
     s->opened = true;
 }
 
+static void rng_backend_free_request(RngRequest *req)
+{
+    g_free(req->data);
+    g_free(req);
+}
+
+static void rng_backend_free_requests(RngBackend *s)
+{
+    RngRequest *req, *next;
+
+    QSIMPLEQ_FOREACH_SAFE(req, &s->requests, next, next) {
+        rng_backend_free_request(req);
+    }
+
+    QSIMPLEQ_INIT(&s->requests);
+}
+
+void rng_backend_finalize_request(RngBackend *s, RngRequest *req)
+{
+    QSIMPLEQ_REMOVE(&s->requests, req, RngRequest, next);
+    rng_backend_free_request(req);
+}
+
 static void rng_backend_init(Object *obj)
 {
+    RngBackend *s = RNG_BACKEND(obj);
+
+    QSIMPLEQ_INIT(&s->requests);
+
     object_property_add_bool(obj, "opened",
                              rng_backend_prop_get_opened,
                              rng_backend_prop_set_opened,
                              NULL);
 }
 
+static void rng_backend_finalize(Object *obj)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+
+    rng_backend_free_requests(s);
+}
+
 static void rng_backend_class_init(ObjectClass *oc, void *data)
 {
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
@@ -92,6 +130,7 @@ static const TypeInfo rng_backend_info = {
     .parent = TYPE_OBJECT,
     .instance_size = sizeof(RngBackend),
     .instance_init = rng_backend_init,
+    .instance_finalize = rng_backend_finalize,
     .class_size = sizeof(RngBackendClass),
     .class_init = rng_backend_class_init,
     .abstract = true,
diff --git a/qemu/backends/testdev.c b/qemu/backends/testdev.c
index eba396aeb..3ab1c90c1 100644
--- a/qemu/backends/testdev.c
+++ b/qemu/backends/testdev.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 
@@ -108,13 +109,16 @@ static void testdev_close(struct CharDriverState *chr)
     g_free(testdev);
 }
 
-CharDriverState *chr_testdev_init(void)
+static CharDriverState *chr_testdev_init(const char *id,
+                                         ChardevBackend *backend,
+                                         ChardevReturn *ret,
+                                         Error **errp)
 {
     TestdevCharState *testdev;
     CharDriverState *chr;
 
-    testdev = g_malloc0(sizeof(TestdevCharState));
-    testdev->chr = chr = g_malloc0(sizeof(CharDriverState));
+    testdev = g_new0(TestdevCharState, 1);
+    testdev->chr = chr = g_new0(CharDriverState, 1);
 
     chr->opaque = testdev;
     chr->chr_write = testdev_write;
@@ -125,7 +129,8 @@ CharDriverState *chr_testdev_init(void)
 
 static void register_types(void)
 {
-    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL);
+    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL,
+                         chr_testdev_init);
 }
 
 type_init(register_types);
diff --git a/qemu/backends/tpm.c b/qemu/backends/tpm.c
index a512693b1..536f262bb 100644
--- a/qemu/backends/tpm.c
+++ b/qemu/backends/tpm.c
@@ -12,7 +12,9 @@
  * Based on backends/rng.c by Anthony Liguori
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/tpm_backend.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/tpm.h"
 #include "qemu/thread.h"
diff --git a/qemu/balloon.c b/qemu/balloon.c
index 5d69e8a00..f2ef50cf7 100644
--- a/qemu/balloon.c
+++ b/qemu/balloon.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/cpu-common.h"
 #include "sysemu/kvm.h"
@@ -36,6 +37,17 @@
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
 static void *balloon_opaque;
+static bool balloon_inhibited;
+
+bool qemu_balloon_is_inhibited(void)
+{
+    return balloon_inhibited;
+}
+
+void qemu_balloon_inhibit(bool state)
+{
+    balloon_inhibited = state;
+}
 
 static bool have_balloon(Error **errp)
 {
diff --git a/qemu/block.c b/qemu/block.c
index d088ee02f..d4939b49b 100644
--- a/qemu/block.c
+++ b/qemu/block.c
@@ -21,28 +21,28 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qjson.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/qapi.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 #include "qapi-event.h"
 #include "block/throttle-groups.h"
+#include "qemu/cutils.h"
+#include "qemu/id.h"
 
 #ifdef CONFIG_BSD
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/queue.h>
 #ifndef __DragonFly__
@@ -54,44 +54,27 @@
 #include <windows.h>
 #endif
 
-/**
- * A BdrvDirtyBitmap can be in three possible states:
- * (1) successor is NULL and disabled is false: full r/w mode
- * (2) successor is NULL and disabled is true: read only mode ("disabled")
- * (3) successor is set: frozen mode.
- *     A frozen bitmap cannot be renamed, deleted, anonymized, cleared, set,
- *     or enabled. A frozen bitmap can only abdicate() or reclaim().
- */
-struct BdrvDirtyBitmap {
-    HBitmap *bitmap;            /* Dirty sector bitmap implementation */
-    BdrvDirtyBitmap *successor; /* Anonymous child; implies frozen status */
-    char *name;                 /* Optional non-empty unique ID */
-    int64_t size;               /* Size of the bitmap (Number of sectors) */
-    bool disabled;              /* Bitmap is read-only */
-    QLIST_ENTRY(BdrvDirtyBitmap) list;
-};
-
 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
 
-static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
-    QTAILQ_HEAD_INITIALIZER(bdrv_states);
-
 static QTAILQ_HEAD(, BlockDriverState) graph_bdrv_states =
     QTAILQ_HEAD_INITIALIZER(graph_bdrv_states);
 
+static QTAILQ_HEAD(, BlockDriverState) all_bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(all_bdrv_states);
+
 static QLIST_HEAD(, BlockDriver) bdrv_drivers =
     QLIST_HEAD_INITIALIZER(bdrv_drivers);
 
 static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
                              const char *reference, QDict *options, int flags,
                              BlockDriverState *parent,
-                             const BdrvChildRole *child_role,
-                             BlockDriver *drv, Error **errp);
+                             const BdrvChildRole *child_role, Error **errp);
 
-static void bdrv_dirty_bitmap_truncate(BlockDriverState *bs);
 /* If non-zero, use only whitelisted block drivers */
 static int use_bdrv_whitelist;
 
+static void bdrv_close(BlockDriverState *bs);
+
 #ifdef _WIN32
 static int is_windows_drive_prefix(const char *filename)
 {
@@ -242,10 +225,7 @@ void bdrv_register(BlockDriver *bdrv)
 
 BlockDriverState *bdrv_new_root(void)
 {
-    BlockDriverState *bs = bdrv_new();
-
-    QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
-    return bs;
+    return bdrv_new();
 }
 
 BlockDriverState *bdrv_new(void)
@@ -258,20 +238,15 @@ BlockDriverState *bdrv_new(void)
     for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
         QLIST_INIT(&bs->op_blockers[i]);
     }
-    bdrv_iostatus_disable(bs);
-    notifier_list_init(&bs->close_notifiers);
     notifier_with_return_list_init(&bs->before_write_notifiers);
     qemu_co_queue_init(&bs->throttled_reqs[0]);
     qemu_co_queue_init(&bs->throttled_reqs[1]);
     bs->refcnt = 1;
     bs->aio_context = qemu_get_aio_context();
 
-    return bs;
-}
+    QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
 
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
-{
-    notifier_list_add(&bs->close_notifiers, notify);
+    return bs;
 }
 
 BlockDriver *bdrv_find_format(const char *format_name)
@@ -314,11 +289,9 @@ static int bdrv_is_whitelisted(BlockDriver *drv, bool read_only)
     return 0;
 }
 
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool read_only)
+bool bdrv_uses_whitelist(void)
 {
-    BlockDriver *drv = bdrv_find_format(format_name);
-    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
+    return use_bdrv_whitelist;
 }
 
 typedef struct CreateCo {
@@ -634,6 +607,20 @@ static int refresh_total_sectors(BlockDriverState *bs, int64_t hint)
 }
 
 /**
+ * Combines a QDict of new block driver @options with any missing options taken
+ * from @old_options, so that leaving out an option defaults to its old value.
+ */
+static void bdrv_join_options(BlockDriverState *bs, QDict *options,
+                              QDict *old_options)
+{
+    if (bs->drv && bs->drv->bdrv_join_options) {
+        bs->drv->bdrv_join_options(options, old_options);
+    } else {
+        qdict_join(options, old_options, false);
+    }
+}
+
+/**
  * Set open flags for a given discard mode
  *
  * Return 0 on success, -1 if the discard mode was invalid.
@@ -658,21 +645,23 @@ int bdrv_parse_discard_flags(const char *mode, int *flags)
  *
  * Return 0 on success, -1 if the cache mode was invalid.
  */
-int bdrv_parse_cache_flags(const char *mode, int *flags)
+int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough)
 {
     *flags &= ~BDRV_O_CACHE_MASK;
 
     if (!strcmp(mode, "off") || !strcmp(mode, "none")) {
-        *flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+        *writethrough = false;
+        *flags |= BDRV_O_NOCACHE;
     } else if (!strcmp(mode, "directsync")) {
+        *writethrough = true;
         *flags |= BDRV_O_NOCACHE;
     } else if (!strcmp(mode, "writeback")) {
-        *flags |= BDRV_O_CACHE_WB;
+        *writethrough = false;
     } else if (!strcmp(mode, "unsafe")) {
-        *flags |= BDRV_O_CACHE_WB;
+        *writethrough = false;
         *flags |= BDRV_O_NO_FLUSH;
     } else if (!strcmp(mode, "writethrough")) {
-        /* this is the default */
+        *writethrough = true;
     } else {
         return -1;
     }
@@ -681,75 +670,101 @@ int bdrv_parse_cache_flags(const char *mode, int *flags)
 }
 
 /*
- * Returns the flags that a temporary snapshot should get, based on the
- * originally requested flags (the originally requested image will have flags
- * like a backing file)
+ * Returns the options and flags that a temporary snapshot should get, based on
+ * the originally requested flags (the originally requested image will have
+ * flags like a backing file)
  */
-static int bdrv_temp_snapshot_flags(int flags)
+static void bdrv_temp_snapshot_options(int *child_flags, QDict *child_options,
+                                       int parent_flags, QDict *parent_options)
 {
-    return (flags & ~BDRV_O_SNAPSHOT) | BDRV_O_TEMPORARY;
+    *child_flags = (parent_flags & ~BDRV_O_SNAPSHOT) | BDRV_O_TEMPORARY;
+
+    /* For temporary files, unconditional cache=unsafe is fine */
+    qdict_set_default_str(child_options, BDRV_OPT_CACHE_DIRECT, "off");
+    qdict_set_default_str(child_options, BDRV_OPT_CACHE_NO_FLUSH, "on");
 }
 
 /*
- * Returns the flags that bs->file should get if a protocol driver is expected,
- * based on the given flags for the parent BDS
+ * Returns the options and flags that bs->file should get if a protocol driver
+ * is expected, based on the given options and flags for the parent BDS
  */
-static int bdrv_inherited_flags(int flags)
+static void bdrv_inherited_options(int *child_flags, QDict *child_options,
+                                   int parent_flags, QDict *parent_options)
 {
+    int flags = parent_flags;
+
     /* Enable protocol handling, disable format probing for bs->file */
     flags |= BDRV_O_PROTOCOL;
 
+    /* If the cache mode isn't explicitly set, inherit direct and no-flush from
+     * the parent. */
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_DIRECT);
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_NO_FLUSH);
+
     /* Our block drivers take care to send flushes and respect unmap policy,
-     * so we can enable both unconditionally on lower layers. */
-    flags |= BDRV_O_CACHE_WB | BDRV_O_UNMAP;
+     * so we can default to enable both on lower layers regardless of the
+     * corresponding parent options. */
+    flags |= BDRV_O_UNMAP;
 
     /* Clear flags that only apply to the top layer */
-    flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ);
+    flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ |
+               BDRV_O_NO_IO);
 
-    return flags;
+    *child_flags = flags;
 }
 
 const BdrvChildRole child_file = {
-    .inherit_flags = bdrv_inherited_flags,
+    .inherit_options = bdrv_inherited_options,
 };
 
 /*
- * Returns the flags that bs->file should get if the use of formats (and not
- * only protocols) is permitted for it, based on the given flags for the parent
- * BDS
+ * Returns the options and flags that bs->file should get if the use of formats
+ * (and not only protocols) is permitted for it, based on the given options and
+ * flags for the parent BDS
  */
-static int bdrv_inherited_fmt_flags(int parent_flags)
+static void bdrv_inherited_fmt_options(int *child_flags, QDict *child_options,
+                                       int parent_flags, QDict *parent_options)
 {
-    int flags = child_file.inherit_flags(parent_flags);
-    return flags & ~BDRV_O_PROTOCOL;
+    child_file.inherit_options(child_flags, child_options,
+                               parent_flags, parent_options);
+
+    *child_flags &= ~(BDRV_O_PROTOCOL | BDRV_O_NO_IO);
 }
 
 const BdrvChildRole child_format = {
-    .inherit_flags = bdrv_inherited_fmt_flags,
+    .inherit_options = bdrv_inherited_fmt_options,
 };
 
 /*
- * Returns the flags that bs->backing_hd should get, based on the given flags
- * for the parent BDS
+ * Returns the options and flags that bs->backing should get, based on the
+ * given options and flags for the parent BDS
  */
-static int bdrv_backing_flags(int flags)
+static void bdrv_backing_options(int *child_flags, QDict *child_options,
+                                 int parent_flags, QDict *parent_options)
 {
+    int flags = parent_flags;
+
+    /* The cache mode is inherited unmodified for backing files; except WCE,
+     * which is only applied on the top level (BlockBackend) */
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_DIRECT);
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_NO_FLUSH);
+
     /* backing files always opened read-only */
     flags &= ~(BDRV_O_RDWR | BDRV_O_COPY_ON_READ);
 
     /* snapshot=on is handled on the top layer */
     flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_TEMPORARY);
 
-    return flags;
+    *child_flags = flags;
 }
 
 static const BdrvChildRole child_backing = {
-    .inherit_flags = bdrv_backing_flags,
+    .inherit_options = bdrv_backing_options,
 };
 
 static int bdrv_open_flags(BlockDriverState *bs, int flags)
 {
-    int open_flags = flags | BDRV_O_CACHE_WB;
+    int open_flags = flags;
 
     /*
      * Clear flags that are internal to the block layer before opening the
@@ -767,16 +782,46 @@ static int bdrv_open_flags(BlockDriverState *bs, int flags)
     return open_flags;
 }
 
+static void update_flags_from_options(int *flags, QemuOpts *opts)
+{
+    *flags &= ~BDRV_O_CACHE_MASK;
+
+    assert(qemu_opt_find(opts, BDRV_OPT_CACHE_NO_FLUSH));
+    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_NO_FLUSH, false)) {
+        *flags |= BDRV_O_NO_FLUSH;
+    }
+
+    assert(qemu_opt_find(opts, BDRV_OPT_CACHE_DIRECT));
+    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_DIRECT, false)) {
+        *flags |= BDRV_O_NOCACHE;
+    }
+}
+
+static void update_options_from_flags(QDict *options, int flags)
+{
+    if (!qdict_haskey(options, BDRV_OPT_CACHE_DIRECT)) {
+        qdict_put(options, BDRV_OPT_CACHE_DIRECT,
+                  qbool_from_bool(flags & BDRV_O_NOCACHE));
+    }
+    if (!qdict_haskey(options, BDRV_OPT_CACHE_NO_FLUSH)) {
+        qdict_put(options, BDRV_OPT_CACHE_NO_FLUSH,
+                  qbool_from_bool(flags & BDRV_O_NO_FLUSH));
+    }
+}
+
 static void bdrv_assign_node_name(BlockDriverState *bs,
                                   const char *node_name,
                                   Error **errp)
 {
-    if (!node_name) {
-        return;
-    }
+    char *gen_node_name = NULL;
 
-    /* Check for empty string or invalid characters */
-    if (!id_wellformed(node_name)) {
+    if (!node_name) {
+        node_name = gen_node_name = id_generate(ID_BLOCK);
+    } else if (!id_wellformed(node_name)) {
+        /*
+         * Check for empty string or invalid characters, but not if it is
+         * generated (generated names use characters not available to the user)
+         */
         error_setg(errp, "Invalid node name");
         return;
     }
@@ -785,18 +830,20 @@ static void bdrv_assign_node_name(BlockDriverState *bs,
     if (blk_by_name(node_name)) {
         error_setg(errp, "node-name=%s is conflicting with a device id",
                    node_name);
-        return;
+        goto out;
     }
 
     /* takes care of avoiding duplicates node names */
     if (bdrv_find_node(node_name)) {
         error_setg(errp, "Duplicate node name");
-        return;
+        goto out;
     }
 
     /* copy node name into the bs and insert it into the graph list */
     pstrcpy(bs->node_name, sizeof(bs->node_name), node_name);
     QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs, node_list);
+out:
+    g_free(gen_node_name);
 }
 
 static QemuOptsList bdrv_runtime_opts = {
@@ -808,6 +855,21 @@ static QemuOptsList bdrv_runtime_opts = {
             .type = QEMU_OPT_STRING,
             .help = "Node name of the block device node",
         },
+        {
+            .name = "driver",
+            .type = QEMU_OPT_STRING,
+            .help = "Block driver to use for the node",
+        },
+        {
+            .name = BDRV_OPT_CACHE_DIRECT,
+            .type = QEMU_OPT_BOOL,
+            .help = "Bypass software writeback cache on the host",
+        },
+        {
+            .name = BDRV_OPT_CACHE_NO_FLUSH,
+            .type = QEMU_OPT_BOOL,
+            .help = "Ignore flush requests",
+        },
         { /* end of list */ }
     },
 };
@@ -817,21 +879,34 @@ static QemuOptsList bdrv_runtime_opts = {
  *
  * Removes all processed options from *options.
  */
-static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
-    QDict *options, int flags, BlockDriver *drv, Error **errp)
+static int bdrv_open_common(BlockDriverState *bs, BdrvChild *file,
+                            QDict *options, Error **errp)
 {
     int ret, open_flags;
     const char *filename;
+    const char *driver_name = NULL;
     const char *node_name = NULL;
     QemuOpts *opts;
+    BlockDriver *drv;
     Error *local_err = NULL;
 
-    assert(drv != NULL);
     assert(bs->file == NULL);
     assert(options != NULL && bs->options != options);
 
+    opts = qemu_opts_create(&bdrv_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail_opts;
+    }
+
+    driver_name = qemu_opt_get(opts, "driver");
+    drv = bdrv_find_format(driver_name);
+    assert(drv != NULL);
+
     if (file != NULL) {
-        filename = file->filename;
+        filename = file->bs->filename;
     } else {
         filename = qdict_get_try_str(options, "filename");
     }
@@ -839,19 +914,13 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     if (drv->bdrv_needs_filename && !filename) {
         error_setg(errp, "The '%s' block driver requires a file name",
                    drv->format_name);
-        return -EINVAL;
-    }
-
-    trace_bdrv_open_common(bs, filename ?: "", flags, drv->format_name);
-
-    opts = qemu_opts_create(&bdrv_runtime_opts, NULL, 0, &error_abort);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto fail_opts;
     }
 
+    trace_bdrv_open_common(bs, filename ?: "", bs->open_flags,
+                           drv->format_name);
+
     node_name = qemu_opt_get(opts, "node-name");
     bdrv_assign_node_name(bs, node_name, &local_err);
     if (local_err) {
@@ -860,11 +929,9 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
         goto fail_opts;
     }
 
-    bs->guest_block_size = 512;
     bs->request_alignment = 512;
     bs->zero_beyond_eof = true;
-    open_flags = bdrv_open_flags(bs, flags);
-    bs->read_only = !(open_flags & BDRV_O_RDWR);
+    bs->read_only = !(bs->open_flags & BDRV_O_RDWR);
 
     if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
         error_setg(errp,
@@ -877,7 +944,7 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     }
 
     assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
-    if (flags & BDRV_O_COPY_ON_READ) {
+    if (bs->open_flags & BDRV_O_COPY_ON_READ) {
         if (!bs->read_only) {
             bdrv_enable_copy_on_read(bs);
         } else {
@@ -897,9 +964,11 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
     bs->drv = drv;
     bs->opaque = g_malloc0(drv->instance_size);
 
-    bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
+    /* Apply cache mode options */
+    update_flags_from_options(&bs->open_flags, opts);
 
     /* Open the image, either directly or using a protocol */
+    open_flags = bdrv_open_flags(bs, bs->open_flags);
     if (drv->bdrv_file_open) {
         assert(file == NULL);
         assert(!drv->bdrv_needs_filename || filename != NULL);
@@ -926,13 +995,6 @@ static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
         goto free_and_fail;
     }
 
-    if (bs->encrypted) {
-        error_report("Encrypted images are deprecated");
-        error_printf("Support for them will be removed in a future release.\n"
-                     "You can use 'qemu-img convert' to convert your image"
-                     " to an unencrypted one.\n");
-    }
-
     ret = refresh_total_sectors(bs, bs->total_sectors);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not refresh total sector count");
@@ -990,47 +1052,55 @@ static QDict *parse_json_filename(const char *filename, Error **errp)
     return options;
 }
 
+static void parse_json_protocol(QDict *options, const char **pfilename,
+                                Error **errp)
+{
+    QDict *json_options;
+    Error *local_err = NULL;
+
+    /* Parse json: pseudo-protocol */
+    if (!*pfilename || !g_str_has_prefix(*pfilename, "json:")) {
+        return;
+    }
+
+    json_options = parse_json_filename(*pfilename, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Options given in the filename have lower priority than options
+     * specified directly */
+    qdict_join(options, json_options, false);
+    QDECREF(json_options);
+    *pfilename = NULL;
+}
+
 /*
  * Fills in default options for opening images and converts the legacy
  * filename/flags pair to option QDict entries.
  * The BDRV_O_PROTOCOL flag in *flags will be set or cleared accordingly if a
  * block driver has been specified explicitly.
  */
-static int bdrv_fill_options(QDict **options, const char **pfilename,
-                             int *flags, BlockDriver *drv, Error **errp)
+static int bdrv_fill_options(QDict **options, const char *filename,
+                             int *flags, Error **errp)
 {
-    const char *filename = *pfilename;
     const char *drvname;
     bool protocol = *flags & BDRV_O_PROTOCOL;
     bool parse_filename = false;
-    BlockDriver *tmp_drv;
+    BlockDriver *drv = NULL;
     Error *local_err = NULL;
 
-    /* Parse json: pseudo-protocol */
-    if (filename && g_str_has_prefix(filename, "json:")) {
-        QDict *json_options = parse_json_filename(filename, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return -EINVAL;
-        }
-
-        /* Options given in the filename have lower priority than options
-         * specified directly */
-        qdict_join(*options, json_options, false);
-        QDECREF(json_options);
-        *pfilename = filename = NULL;
-    }
-
     drvname = qdict_get_try_str(*options, "driver");
-
-    /* If the user has explicitly specified the driver, this choice should
-     * override the BDRV_O_PROTOCOL flag */
-    tmp_drv = drv;
-    if (!tmp_drv && drvname) {
-        tmp_drv = bdrv_find_format(drvname);
-    }
-    if (tmp_drv) {
-        protocol = tmp_drv->bdrv_file_open;
+    if (drvname) {
+        drv = bdrv_find_format(drvname);
+        if (!drv) {
+            error_setg(errp, "Unknown driver '%s'", drvname);
+            return -ENOENT;
+        }
+        /* If the user has explicitly specified the driver, this choice should
+         * override the BDRV_O_PROTOCOL flag */
+        protocol = drv->bdrv_file_open;
     }
 
     if (protocol) {
@@ -1039,6 +1109,9 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
         *flags &= ~BDRV_O_PROTOCOL;
     }
 
+    /* Translate cache options from flags into options */
+    update_options_from_flags(*options, *flags);
+
     /* Fetch the file name from the options QDict if necessary */
     if (protocol && filename) {
         if (!qdict_haskey(*options, "filename")) {
@@ -1054,33 +1127,18 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
     /* Find the right block driver */
     filename = qdict_get_try_str(*options, "filename");
 
-    if (drv) {
-        if (drvname) {
-            error_setg(errp, "Driver specified twice");
-            return -EINVAL;
-        }
-        drvname = drv->format_name;
-        qdict_put(*options, "driver", qstring_from_str(drvname));
-    } else {
-        if (!drvname && protocol) {
-            if (filename) {
-                drv = bdrv_find_protocol(filename, parse_filename, errp);
-                if (!drv) {
-                    return -EINVAL;
-                }
-
-                drvname = drv->format_name;
-                qdict_put(*options, "driver", qstring_from_str(drvname));
-            } else {
-                error_setg(errp, "Must specify either driver or file");
-                return -EINVAL;
-            }
-        } else if (drvname) {
-            drv = bdrv_find_format(drvname);
+    if (!drvname && protocol) {
+        if (filename) {
+            drv = bdrv_find_protocol(filename, parse_filename, errp);
             if (!drv) {
-                error_setg(errp, "Unknown driver '%s'", drvname);
-                return -ENOENT;
+                return -EINVAL;
             }
+
+            drvname = drv->format_name;
+            qdict_put(*options, "driver", qstring_from_str(drvname));
+        } else {
+            error_setg(errp, "Must specify either driver or file");
+            return -EINVAL;
         }
     }
 
@@ -1102,68 +1160,100 @@ static int bdrv_fill_options(QDict **options, const char **pfilename,
     return 0;
 }
 
-static BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
-                                    BlockDriverState *child_bs,
-                                    const BdrvChildRole *child_role)
+BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs,
+                                  const char *child_name,
+                                  const BdrvChildRole *child_role)
 {
     BdrvChild *child = g_new(BdrvChild, 1);
     *child = (BdrvChild) {
         .bs     = child_bs,
+        .name   = g_strdup(child_name),
         .role   = child_role,
     };
 
-    QLIST_INSERT_HEAD(&parent_bs->children, child, next);
+    QLIST_INSERT_HEAD(&child_bs->parents, child, next_parent);
+
+    return child;
+}
 
+static BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
+                                    BlockDriverState *child_bs,
+                                    const char *child_name,
+                                    const BdrvChildRole *child_role)
+{
+    BdrvChild *child = bdrv_root_attach_child(child_bs, child_name, child_role);
+    QLIST_INSERT_HEAD(&parent_bs->children, child, next);
     return child;
 }
 
 static void bdrv_detach_child(BdrvChild *child)
 {
-    QLIST_REMOVE(child, next);
+    if (child->next.le_prev) {
+        QLIST_REMOVE(child, next);
+        child->next.le_prev = NULL;
+    }
+    QLIST_REMOVE(child, next_parent);
+    g_free(child->name);
     g_free(child);
 }
 
+void bdrv_root_unref_child(BdrvChild *child)
+{
+    BlockDriverState *child_bs;
+
+    child_bs = child->bs;
+    bdrv_detach_child(child);
+    bdrv_unref(child_bs);
+}
+
 void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child)
 {
-    BlockDriverState *child_bs = child->bs;
+    if (child == NULL) {
+        return;
+    }
 
     if (child->bs->inherits_from == parent) {
         child->bs->inherits_from = NULL;
     }
 
-    bdrv_detach_child(child);
-    bdrv_unref(child_bs);
+    bdrv_root_unref_child(child);
 }
 
+/*
+ * Sets the backing file link of a BDS. A new reference is created; callers
+ * which don't need their own reference any more must call bdrv_unref().
+ */
 void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
 {
+    if (backing_hd) {
+        bdrv_ref(backing_hd);
+    }
 
-    if (bs->backing_hd) {
+    if (bs->backing) {
         assert(bs->backing_blocker);
-        bdrv_op_unblock_all(bs->backing_hd, bs->backing_blocker);
-        bdrv_detach_child(bs->backing_child);
+        bdrv_op_unblock_all(bs->backing->bs, bs->backing_blocker);
+        bdrv_unref_child(bs, bs->backing);
     } else if (backing_hd) {
         error_setg(&bs->backing_blocker,
                    "node is used as backing hd of '%s'",
                    bdrv_get_device_or_node_name(bs));
     }
 
-    bs->backing_hd = backing_hd;
     if (!backing_hd) {
         error_free(bs->backing_blocker);
         bs->backing_blocker = NULL;
-        bs->backing_child = NULL;
+        bs->backing = NULL;
         goto out;
     }
-    bs->backing_child = bdrv_attach_child(bs, backing_hd, &child_backing);
+    bs->backing = bdrv_attach_child(bs, backing_hd, "backing", &child_backing);
     bs->open_flags &= ~BDRV_O_NO_BACKING;
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_hd->filename);
     pstrcpy(bs->backing_format, sizeof(bs->backing_format),
             backing_hd->drv ? backing_hd->drv->format_name : "");
 
-    bdrv_op_block_all(bs->backing_hd, bs->backing_blocker);
+    bdrv_op_block_all(backing_hd, bs->backing_blocker);
     /* Otherwise we won't be able to commit due to check in bdrv_commit */
-    bdrv_op_unblock(bs->backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET,
+    bdrv_op_unblock(backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET,
                     bs->backing_blocker);
 out:
     bdrv_refresh_limits(bs, NULL);
@@ -1172,30 +1262,43 @@ out:
 /*
  * Opens the backing file for a BlockDriverState if not yet open
  *
- * options is a QDict of options to pass to the block drivers, or NULL for an
- * empty set of options. The reference to the QDict is transferred to this
- * function (even on failure), so if the caller intends to reuse the dictionary,
- * it needs to use QINCREF() before calling bdrv_file_open.
+ * bdref_key specifies the key for the image's BlockdevRef in the options QDict.
+ * That QDict has to be flattened; therefore, if the BlockdevRef is a QDict
+ * itself, all options starting with "${bdref_key}." are considered part of the
+ * BlockdevRef.
+ *
+ * TODO Can this be unified with bdrv_open_image()?
  */
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options,
+                           const char *bdref_key, Error **errp)
 {
     char *backing_filename = g_malloc0(PATH_MAX);
+    char *bdref_key_dot;
+    const char *reference = NULL;
     int ret = 0;
     BlockDriverState *backing_hd;
+    QDict *options;
+    QDict *tmp_parent_options = NULL;
     Error *local_err = NULL;
 
-    if (bs->backing_hd != NULL) {
-        QDECREF(options);
+    if (bs->backing != NULL) {
         goto free_exit;
     }
 
     /* NULL means an empty set of options */
-    if (options == NULL) {
-        options = qdict_new();
+    if (parent_options == NULL) {
+        tmp_parent_options = qdict_new();
+        parent_options = tmp_parent_options;
     }
 
     bs->open_flags &= ~BDRV_O_NO_BACKING;
-    if (qdict_haskey(options, "file.filename")) {
+
+    bdref_key_dot = g_strdup_printf("%s.", bdref_key);
+    qdict_extract_subqdict(parent_options, &options, bdref_key_dot);
+    g_free(bdref_key_dot);
+
+    reference = qdict_get_try_str(parent_options, bdref_key);
+    if (reference || qdict_haskey(options, "file.filename")) {
         backing_filename[0] = '\0';
     } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
         QDECREF(options);
@@ -1218,31 +1321,31 @@ int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
         goto free_exit;
     }
 
-    backing_hd = bdrv_new();
-
     if (bs->backing_format[0] != '\0' && !qdict_haskey(options, "driver")) {
         qdict_put(options, "driver", qstring_from_str(bs->backing_format));
     }
 
-    assert(bs->backing_hd == NULL);
+    backing_hd = NULL;
     ret = bdrv_open_inherit(&backing_hd,
                             *backing_filename ? backing_filename : NULL,
-                            NULL, options, 0, bs, &child_backing,
-                            NULL, &local_err);
+                            reference, options, 0, bs, &child_backing,
+                            errp);
     if (ret < 0) {
-        bdrv_unref(backing_hd);
-        backing_hd = NULL;
         bs->open_flags |= BDRV_O_NO_BACKING;
-        error_setg(errp, "Could not open backing file: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_prepend(errp, "Could not open backing file: ");
         goto free_exit;
     }
 
+    /* Hook up the backing file link; drop our reference, bs owns the
+     * backing_hd reference now */
     bdrv_set_backing_hd(bs, backing_hd);
+    bdrv_unref(backing_hd);
+
+    qdict_del(parent_options, bdref_key);
 
 free_exit:
     g_free(backing_filename);
+    QDECREF(tmp_parent_options);
     return ret;
 }
 
@@ -1291,59 +1394,25 @@ BdrvChild *bdrv_open_child(const char *filename,
 
     bs = NULL;
     ret = bdrv_open_inherit(&bs, filename, reference, image_options, 0,
-                            parent, child_role, NULL, errp);
+                            parent, child_role, errp);
     if (ret < 0) {
         goto done;
     }
 
-    c = bdrv_attach_child(parent, bs, child_role);
+    c = bdrv_attach_child(parent, bs, bdref_key, child_role);
 
 done:
     qdict_del(options, bdref_key);
     return c;
 }
 
-/*
- * This is a version of bdrv_open_child() that returns 0/-EINVAL instead of
- * a BdrvChild object.
- *
- * If allow_none is true, no image will be opened if filename is false and no
- * BlockdevRef is given. *pbs will remain unchanged and 0 will be returned.
- *
- * To conform with the behavior of bdrv_open(), *pbs has to be NULL.
- */
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key,
-                    BlockDriverState* parent, const BdrvChildRole *child_role,
-                    bool allow_none, Error **errp)
-{
-    Error *local_err = NULL;
-    BdrvChild *c;
-
-    assert(pbs);
-    assert(*pbs == NULL);
-
-    c = bdrv_open_child(filename, options, bdref_key, parent, child_role,
-                        allow_none, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return -EINVAL;
-    }
-
-    if (c != NULL) {
-        *pbs = c->bs;
-    }
-
-    return 0;
-}
-
-int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
+static int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags,
+                                     QDict *snapshot_options, Error **errp)
 {
     /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
     char *tmp_filename = g_malloc0(PATH_MAX + 1);
     int64_t total_size;
     QemuOpts *opts = NULL;
-    QDict *snapshot_options;
     BlockDriverState *bs_snapshot;
     Error *local_err = NULL;
     int ret;
@@ -1369,27 +1438,27 @@ int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
     opts = qemu_opts_create(bdrv_qcow2.create_opts, NULL, 0,
                             &error_abort);
     qemu_opt_set_number(opts, BLOCK_OPT_SIZE, total_size, &error_abort);
-    ret = bdrv_create(&bdrv_qcow2, tmp_filename, opts, &local_err);
+    ret = bdrv_create(&bdrv_qcow2, tmp_filename, opts, errp);
     qemu_opts_del(opts);
     if (ret < 0) {
-        error_setg_errno(errp, -ret, "Could not create temporary overlay "
-                         "'%s': %s", tmp_filename,
-                         error_get_pretty(local_err));
-        error_free(local_err);
+        error_prepend(errp, "Could not create temporary overlay '%s': ",
+                      tmp_filename);
         goto out;
     }
 
-    /* Prepare a new options QDict for the temporary file */
-    snapshot_options = qdict_new();
+    /* Prepare options QDict for the temporary file */
     qdict_put(snapshot_options, "file.driver",
               qstring_from_str("file"));
     qdict_put(snapshot_options, "file.filename",
               qstring_from_str(tmp_filename));
+    qdict_put(snapshot_options, "driver",
+              qstring_from_str("qcow2"));
 
     bs_snapshot = bdrv_new();
 
     ret = bdrv_open(&bs_snapshot, NULL, NULL, snapshot_options,
-                    flags, &bdrv_qcow2, &local_err);
+                    flags, &local_err);
+    snapshot_options = NULL;
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -1398,6 +1467,7 @@ int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
     bdrv_append(bs_snapshot, bs);
 
 out:
+    QDECREF(snapshot_options);
     g_free(tmp_filename);
     return ret;
 }
@@ -1420,13 +1490,16 @@ out:
 static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
                              const char *reference, QDict *options, int flags,
                              BlockDriverState *parent,
-                             const BdrvChildRole *child_role,
-                             BlockDriver *drv, Error **errp)
+                             const BdrvChildRole *child_role, Error **errp)
 {
     int ret;
-    BlockDriverState *file = NULL, *bs;
+    BdrvChild *file = NULL;
+    BlockDriverState *bs;
+    BlockDriver *drv = NULL;
     const char *drvname;
+    const char *backing;
     Error *local_err = NULL;
+    QDict *snapshot_options = NULL;
     int snapshot_flags = 0;
 
     assert(pbs);
@@ -1453,6 +1526,13 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
         if (!bs) {
             return -ENODEV;
         }
+
+        if (bs->throttle_state) {
+            error_setg(errp, "Cannot reference an existing block device for "
+                       "which I/O throttling is enabled");
+            return -EINVAL;
+        }
+
         bdrv_ref(bs);
         *pbs = bs;
         return 0;
@@ -1469,22 +1549,34 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
         options = qdict_new();
     }
 
+    /* json: syntax counts as explicit options, as if in the QDict */
+    parse_json_protocol(options, &filename, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    bs->explicit_options = qdict_clone_shallow(options);
+
     if (child_role) {
         bs->inherits_from = parent;
-        flags = child_role->inherit_flags(parent->open_flags);
+        child_role->inherit_options(&flags, options,
+                                    parent->open_flags, parent->options);
     }
 
-    ret = bdrv_fill_options(&options, &filename, &flags, drv, &local_err);
+    ret = bdrv_fill_options(&options, filename, &flags, &local_err);
     if (local_err) {
         goto fail;
     }
 
+    bs->open_flags = flags;
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
     /* Find the right image format driver */
-    drv = NULL;
     drvname = qdict_get_try_str(options, "driver");
     if (drvname) {
         drv = bdrv_find_format(drvname);
-        qdict_del(options, "driver");
         if (!drv) {
             error_setg(errp, "Unknown driver: '%s'", drvname);
             ret = -EINVAL;
@@ -1494,9 +1586,11 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     assert(drvname || !(flags & BDRV_O_PROTOCOL));
 
-    bs->open_flags = flags;
-    bs->options = options;
-    options = qdict_clone_shallow(options);
+    backing = qdict_get_try_str(options, "backing");
+    if (backing && *backing == '\0') {
+        flags |= BDRV_O_NO_BACKING;
+        qdict_del(options, "backing");
+    }
 
     /* Open image file without format layer */
     if ((flags & BDRV_O_PROTOCOL) == 0) {
@@ -1504,15 +1598,18 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
             flags |= BDRV_O_ALLOW_RDWR;
         }
         if (flags & BDRV_O_SNAPSHOT) {
-            snapshot_flags = bdrv_temp_snapshot_flags(flags);
-            flags = bdrv_backing_flags(flags);
+            snapshot_options = qdict_new();
+            bdrv_temp_snapshot_options(&snapshot_flags, snapshot_options,
+                                       flags, options);
+            bdrv_backing_options(&flags, options, flags, options);
         }
 
-        assert(file == NULL);
         bs->open_flags = flags;
-        ret = bdrv_open_image(&file, filename, options, "file",
-                              bs, &child_file, true, &local_err);
-        if (ret < 0) {
+
+        file = bdrv_open_child(filename, options, "file", bs,
+                               &child_file, true, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
             goto fail;
         }
     }
@@ -1520,10 +1617,23 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
     /* Image format probing */
     bs->probed = !drv;
     if (!drv && file) {
-        ret = find_image_format(file, filename, &drv, &local_err);
+        ret = find_image_format(file->bs, filename, &drv, &local_err);
         if (ret < 0) {
             goto fail;
         }
+        /*
+         * This option update would logically belong in bdrv_fill_options(),
+         * but we first need to open bs->file for the probing to work, while
+         * opening bs->file already requires the (mostly) final set of options
+         * so that cache mode etc. can be inherited.
+         *
+         * Adding the driver later is somewhat ugly, but it's not an option
+         * that would ever be inherited, so it's correct. We just need to make
+         * sure to update both bs->options (which has the full effective
+         * options for bs) and options (which has file.* already removed).
+         */
+        qdict_put(bs->options, "driver", qstring_from_str(drv->format_name));
+        qdict_put(options, "driver", qstring_from_str(drv->format_name));
     } else if (!drv) {
         error_setg(errp, "Must specify either driver or file");
         ret = -EINVAL;
@@ -1537,22 +1647,19 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
     assert(!(flags & BDRV_O_PROTOCOL) || !file);
 
     /* Open the image */
-    ret = bdrv_open_common(bs, file, options, flags, drv, &local_err);
+    ret = bdrv_open_common(bs, file, options, &local_err);
     if (ret < 0) {
         goto fail;
     }
 
     if (file && (bs->file != file)) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
         file = NULL;
     }
 
     /* If there is a backing file, use it */
     if ((flags & BDRV_O_NO_BACKING) == 0) {
-        QDict *backing_options;
-
-        qdict_extract_subqdict(options, &backing_options, "backing.");
-        ret = bdrv_open_backing_file(bs, backing_options, &local_err);
+        ret = bdrv_open_backing_file(bs, options, "backing", &local_err);
         if (ret < 0) {
             goto close_and_fail;
         }
@@ -1560,15 +1667,6 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     bdrv_refresh_filename(bs);
 
-    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
-     * temporary snapshot afterwards. */
-    if (snapshot_flags) {
-        ret = bdrv_append_temp_snapshot(bs, snapshot_flags, &local_err);
-        if (local_err) {
-            goto close_and_fail;
-        }
-    }
-
     /* Check if any unknown options were used */
     if (options && (qdict_size(options) != 0)) {
         const QDictEntry *entry = qdict_first(options);
@@ -1576,9 +1674,9 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
             error_setg(errp, "Block protocol '%s' doesn't support the option "
                        "'%s'", drv->format_name, entry->key);
         } else {
-            error_setg(errp, "Block format '%s' used by device '%s' doesn't "
-                       "support the option '%s'", drv->format_name,
-                       bdrv_get_device_name(bs), entry->key);
+            error_setg(errp,
+                       "Block format '%s' does not support the option '%s'",
+                       drv->format_name, entry->key);
         }
 
         ret = -EINVAL;
@@ -1600,12 +1698,26 @@ static int bdrv_open_inherit(BlockDriverState **pbs, const char *filename,
 
     QDECREF(options);
     *pbs = bs;
+
+    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
+     * temporary snapshot afterwards. */
+    if (snapshot_flags) {
+        ret = bdrv_append_temp_snapshot(bs, snapshot_flags, snapshot_options,
+                                        &local_err);
+        snapshot_options = NULL;
+        if (local_err) {
+            goto close_and_fail;
+        }
+    }
+
     return 0;
 
 fail:
     if (file != NULL) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
     }
+    QDECREF(snapshot_options);
+    QDECREF(bs->explicit_options);
     QDECREF(bs->options);
     QDECREF(options);
     bs->options = NULL;
@@ -1627,6 +1739,7 @@ close_and_fail:
     } else {
         bdrv_unref(bs);
     }
+    QDECREF(snapshot_options);
     QDECREF(options);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -1635,11 +1748,10 @@ close_and_fail:
 }
 
 int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp)
+              const char *reference, QDict *options, int flags, Error **errp)
 {
     return bdrv_open_inherit(pbs, filename, reference, options, flags, NULL,
-                             NULL, drv, errp);
+                             NULL, errp);
 }
 
 typedef struct BlockReopenQueueEntry {
@@ -1660,48 +1772,116 @@ typedef struct BlockReopenQueueEntry {
  *
  * bs is the BlockDriverState to add to the reopen queue.
  *
+ * options contains the changed options for the associated bs
+ * (the BlockReopenQueue takes ownership)
+ *
  * flags contains the open flags for the associated bs
  *
  * returns a pointer to bs_queue, which is either the newly allocated
  * bs_queue, or the existing bs_queue being used.
  *
  */
-BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags)
+static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
+                                                 BlockDriverState *bs,
+                                                 QDict *options,
+                                                 int flags,
+                                                 const BdrvChildRole *role,
+                                                 QDict *parent_options,
+                                                 int parent_flags)
 {
     assert(bs != NULL);
 
     BlockReopenQueueEntry *bs_entry;
     BdrvChild *child;
+    QDict *old_options, *explicit_options;
 
     if (bs_queue == NULL) {
         bs_queue = g_new0(BlockReopenQueue, 1);
         QSIMPLEQ_INIT(bs_queue);
     }
 
+    if (!options) {
+        options = qdict_new();
+    }
+
+    /*
+     * Precedence of options:
+     * 1. Explicitly passed in options (highest)
+     * 2. Set in flags (only for top level)
+     * 3. Retained from explicitly set options of bs
+     * 4. Inherited from parent node
+     * 5. Retained from effective options of bs
+     */
+
+    if (!parent_options) {
+        /*
+         * Any setting represented by flags is always updated. If the
+         * corresponding QDict option is set, it takes precedence. Otherwise
+         * the flag is translated into a QDict option. The old setting of bs is
+         * not considered.
+         */
+        update_options_from_flags(options, flags);
+    }
+
+    /* Old explicitly set values (don't overwrite by inherited value) */
+    old_options = qdict_clone_shallow(bs->explicit_options);
+    bdrv_join_options(bs, options, old_options);
+    QDECREF(old_options);
+
+    explicit_options = qdict_clone_shallow(options);
+
+    /* Inherit from parent node */
+    if (parent_options) {
+        assert(!flags);
+        role->inherit_options(&flags, options, parent_flags, parent_options);
+    }
+
+    /* Old values are used for options that aren't set yet */
+    old_options = qdict_clone_shallow(bs->options);
+    bdrv_join_options(bs, options, old_options);
+    QDECREF(old_options);
+
     /* bdrv_open() masks this flag out */
     flags &= ~BDRV_O_PROTOCOL;
 
     QLIST_FOREACH(child, &bs->children, next) {
-        int child_flags;
+        QDict *new_child_options;
+        char *child_key_dot;
 
+        /* reopen can only change the options of block devices that were
+         * implicitly created and inherited options. For other (referenced)
+         * block devices, a syntax like "backing.foo" results in an error. */
         if (child->bs->inherits_from != bs) {
             continue;
         }
 
-        child_flags = child->role->inherit_flags(flags);
-        bdrv_reopen_queue(bs_queue, child->bs, child_flags);
+        child_key_dot = g_strdup_printf("%s.", child->name);
+        qdict_extract_subqdict(options, &new_child_options, child_key_dot);
+        g_free(child_key_dot);
+
+        bdrv_reopen_queue_child(bs_queue, child->bs, new_child_options, 0,
+                                child->role, options, flags);
     }
 
     bs_entry = g_new0(BlockReopenQueueEntry, 1);
     QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 
     bs_entry->state.bs = bs;
+    bs_entry->state.options = options;
+    bs_entry->state.explicit_options = explicit_options;
     bs_entry->state.flags = flags;
 
     return bs_queue;
 }
 
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags)
+{
+    return bdrv_reopen_queue_child(bs_queue, bs, options, flags,
+                                   NULL, NULL, 0);
+}
+
 /*
  * Reopen multiple BlockDriverStates atomically & transactionally.
  *
@@ -1748,7 +1928,10 @@ cleanup:
     QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
         if (ret && bs_entry->prepared) {
             bdrv_reopen_abort(&bs_entry->state);
+        } else if (ret) {
+            QDECREF(bs_entry->state.explicit_options);
         }
+        QDECREF(bs_entry->state.options);
         g_free(bs_entry);
     }
     g_free(bs_queue);
@@ -1761,7 +1944,7 @@ int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
 {
     int ret = -1;
     Error *local_err = NULL;
-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
 
     ret = bdrv_reopen_multiple(queue, &local_err);
     if (local_err != NULL) {
@@ -1794,11 +1977,36 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
     int ret = -1;
     Error *local_err = NULL;
     BlockDriver *drv;
+    QemuOpts *opts;
+    const char *value;
 
     assert(reopen_state != NULL);
     assert(reopen_state->bs->drv != NULL);
     drv = reopen_state->bs->drv;
 
+    /* Process generic block layer options */
+    opts = qemu_opts_create(&bdrv_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, reopen_state->options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto error;
+    }
+
+    update_flags_from_options(&reopen_state->flags, opts);
+
+    /* node-name and driver must be unchanged. Put them back into the QDict, so
+     * that they are checked at the end of this function. */
+    value = qemu_opt_get(opts, "node-name");
+    if (value) {
+        qdict_put(reopen_state->options, "node-name", qstring_from_str(value));
+    }
+
+    value = qemu_opt_get(opts, "driver");
+    if (value) {
+        qdict_put(reopen_state->options, "driver", qstring_from_str(value));
+    }
+
     /* if we are to stay read-only, do not allow permission change
      * to r/w */
     if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
@@ -1811,8 +2019,7 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
 
     ret = bdrv_flush(reopen_state->bs);
     if (ret) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
-                  strerror(-ret));
+        error_setg_errno(errp, -ret, "Error flushing drive");
         goto error;
     }
 
@@ -1837,9 +2044,30 @@ int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
         goto error;
     }
 
+    /* Options that are not handled are only okay if they are unchanged
+     * compared to the old state. It is expected that some options are only
+     * used for the initial open, but not reopen (e.g. filename) */
+    if (qdict_size(reopen_state->options)) {
+        const QDictEntry *entry = qdict_first(reopen_state->options);
+
+        do {
+            QString *new_obj = qobject_to_qstring(entry->value);
+            const char *new = qstring_get_str(new_obj);
+            const char *old = qdict_get_try_str(reopen_state->bs->options,
+                                                entry->key);
+
+            if (!old || strcmp(new, old)) {
+                error_setg(errp, "Cannot change the option '%s'", entry->key);
+                ret = -EINVAL;
+                goto error;
+            }
+        } while ((entry = qdict_next(reopen_state->options, entry)));
+    }
+
     ret = 0;
 
 error:
+    qemu_opts_del(opts);
     return ret;
 }
 
@@ -1862,9 +2090,10 @@ void bdrv_reopen_commit(BDRVReopenState *reopen_state)
     }
 
     /* set BDS specific flags now */
+    QDECREF(reopen_state->bs->explicit_options);
+
+    reopen_state->bs->explicit_options   = reopen_state->explicit_options;
     reopen_state->bs->open_flags         = reopen_state->flags;
-    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
-                                              BDRV_O_CACHE_WB);
     reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
 
     bdrv_refresh_limits(reopen_state->bs, NULL);
@@ -1885,30 +2114,44 @@ void bdrv_reopen_abort(BDRVReopenState *reopen_state)
     if (drv->bdrv_reopen_abort) {
         drv->bdrv_reopen_abort(reopen_state);
     }
+
+    QDECREF(reopen_state->explicit_options);
 }
 
 
-void bdrv_close(BlockDriverState *bs)
+static void bdrv_close(BlockDriverState *bs)
 {
     BdrvAioNotifier *ban, *ban_next;
 
-    if (bs->job) {
-        block_job_cancel_sync(bs->job);
+    assert(!bs->job);
+
+    /* Disable I/O limits and drain all pending throttled requests */
+    if (bs->throttle_state) {
+        bdrv_io_limits_disable(bs);
     }
-    bdrv_drain(bs); /* complete I/O */
+
+    bdrv_drained_begin(bs); /* complete I/O */
     bdrv_flush(bs);
     bdrv_drain(bs); /* in case flush left pending I/O */
-    notifier_list_notify(&bs->close_notifiers, bs);
+
+    bdrv_release_named_dirty_bitmaps(bs);
+    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
+
+    if (bs->blk) {
+        blk_dev_change_media_cb(bs->blk, false);
+    }
 
     if (bs->drv) {
         BdrvChild *child, *next;
 
         bs->drv->bdrv_close(bs);
+        bs->drv = NULL;
+
+        bdrv_set_backing_hd(bs, NULL);
 
-        if (bs->backing_hd) {
-            BlockDriverState *backing_hd = bs->backing_hd;
-            bdrv_set_backing_hd(bs, NULL);
-            bdrv_unref(backing_hd);
+        if (bs->file != NULL) {
+            bdrv_unref_child(bs, bs->file);
+            bs->file = NULL;
         }
 
         QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
@@ -1922,7 +2165,6 @@ void bdrv_close(BlockDriverState *bs)
 
         g_free(bs->opaque);
         bs->opaque = NULL;
-        bs->drv = NULL;
         bs->copy_on_read = 0;
         bs->backing_file[0] = '\0';
         bs->backing_format[0] = '\0';
@@ -1932,227 +2174,97 @@ void bdrv_close(BlockDriverState *bs)
         bs->sg = 0;
         bs->zero_beyond_eof = false;
         QDECREF(bs->options);
+        QDECREF(bs->explicit_options);
         bs->options = NULL;
         QDECREF(bs->full_open_options);
         bs->full_open_options = NULL;
-
-        if (bs->file != NULL) {
-            bdrv_unref(bs->file);
-            bs->file = NULL;
-        }
-    }
-
-    if (bs->blk) {
-        blk_dev_change_media_cb(bs->blk, false);
-    }
-
-    /*throttling disk I/O limits*/
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_disable(bs);
     }
 
     QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
         g_free(ban);
     }
     QLIST_INIT(&bs->aio_notifiers);
+    bdrv_drained_end(bs);
 }
 
 void bdrv_close_all(void)
 {
     BlockDriverState *bs;
+    AioContext *aio_context;
 
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
+    /* Drop references from requests still in flight, such as canceled block
+     * jobs whose AIO context has not been polled yet */
+    bdrv_drain_all();
 
-        aio_context_acquire(aio_context);
-        bdrv_close(bs);
-        aio_context_release(aio_context);
-    }
-}
+    blk_remove_all_bs();
+    blockdev_close_all_bdrv_states();
 
-/* make a BlockDriverState anonymous by removing from bdrv_state and
- * graph_bdrv_state list.
-   Also, NULL terminate the device_name to prevent double remove */
-void bdrv_make_anon(BlockDriverState *bs)
-{
-    /*
-     * Take care to remove bs from bdrv_states only when it's actually
-     * in it.  Note that bs->device_list.tqe_prev is initially null,
-     * and gets set to non-null by QTAILQ_INSERT_TAIL().  Establish
-     * the useful invariant "bs in bdrv_states iff bs->tqe_prev" by
-     * resetting it to null on remove.
-     */
-    if (bs->device_list.tqe_prev) {
-        QTAILQ_REMOVE(&bdrv_states, bs, device_list);
-        bs->device_list.tqe_prev = NULL;
-    }
-    if (bs->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
-    }
-    bs->node_name[0] = '\0';
-}
+    /* Cancel all block jobs */
+    while (!QTAILQ_EMPTY(&all_bdrv_states)) {
+        QTAILQ_FOREACH(bs, &all_bdrv_states, bs_list) {
+            aio_context = bdrv_get_aio_context(bs);
 
-static void bdrv_rebind(BlockDriverState *bs)
-{
-    if (bs->drv && bs->drv->bdrv_rebind) {
-        bs->drv->bdrv_rebind(bs);
+            aio_context_acquire(aio_context);
+            if (bs->job) {
+                block_job_cancel_sync(bs->job);
+                aio_context_release(aio_context);
+                break;
+            }
+            aio_context_release(aio_context);
+        }
+
+        /* All the remaining BlockDriverStates are referenced directly or
+         * indirectly from block jobs, so there needs to be at least one BDS
+         * directly used by a block job */
+        assert(bs);
     }
 }
 
+/* Fields that need to stay with the top-level BDS */
 static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
                                      BlockDriverState *bs_src)
 {
     /* move some fields that need to stay attached to the device */
-
-    /* dev info */
-    bs_dest->guest_block_size   = bs_src->guest_block_size;
-    bs_dest->copy_on_read       = bs_src->copy_on_read;
-
-    bs_dest->enable_write_cache = bs_src->enable_write_cache;
-
-    /* i/o throttled req */
-    bs_dest->throttle_state     = bs_src->throttle_state,
-    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
-    bs_dest->pending_reqs[0]    = bs_src->pending_reqs[0];
-    bs_dest->pending_reqs[1]    = bs_src->pending_reqs[1];
-    bs_dest->throttled_reqs[0]  = bs_src->throttled_reqs[0];
-    bs_dest->throttled_reqs[1]  = bs_src->throttled_reqs[1];
-    memcpy(&bs_dest->round_robin,
-           &bs_src->round_robin,
-           sizeof(bs_dest->round_robin));
-    memcpy(&bs_dest->throttle_timers,
-           &bs_src->throttle_timers,
-           sizeof(ThrottleTimers));
-
-    /* r/w error */
-    bs_dest->on_read_error      = bs_src->on_read_error;
-    bs_dest->on_write_error     = bs_src->on_write_error;
-
-    /* i/o status */
-    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
-    bs_dest->iostatus           = bs_src->iostatus;
-
-    /* dirty bitmap */
-    bs_dest->dirty_bitmaps      = bs_src->dirty_bitmaps;
-
-    /* reference count */
-    bs_dest->refcnt             = bs_src->refcnt;
-
-    /* job */
-    bs_dest->job                = bs_src->job;
-
-    /* keep the same entry in bdrv_states */
-    bs_dest->device_list = bs_src->device_list;
-    bs_dest->blk = bs_src->blk;
-
-    memcpy(bs_dest->op_blockers, bs_src->op_blockers,
-           sizeof(bs_dest->op_blockers));
 }
 
-/*
- * Swap bs contents for two image chains while they are live,
- * while keeping required fields on the BlockDriverState that is
- * actually attached to a device.
- *
- * This will modify the BlockDriverState fields, and swap contents
- * between bs_new and bs_old. Both bs_new and bs_old are modified.
- *
- * bs_new must not be attached to a BlockBackend.
- *
- * This function does not create any image files.
- */
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
+static void change_parent_backing_link(BlockDriverState *from,
+                                       BlockDriverState *to)
 {
-    BlockDriverState tmp;
-    BdrvChild *child;
-
-    bdrv_drain(bs_new);
-    bdrv_drain(bs_old);
+    BdrvChild *c, *next;
 
-    /* The code needs to swap the node_name but simply swapping node_list won't
-     * work so first remove the nodes from the graph list, do the swap then
-     * insert them back if needed.
-     */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_old, node_list);
+    if (from->blk) {
+        /* FIXME We bypass blk_set_bs(), so we need to make these updates
+         * manually. The root problem is not in this change function, but the
+         * existence of BlockDriverState.blk. */
+        to->blk = from->blk;
+        from->blk = NULL;
     }
 
-    /* If the BlockDriverState is part of a throttling group acquire
-     * its lock since we're going to mess with the protected fields.
-     * Otherwise there's no need to worry since no one else can touch
-     * them. */
-    if (bs_old->throttle_state) {
-        throttle_group_lock(bs_old);
+    QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) {
+        assert(c->role != &child_backing);
+        c->bs = to;
+        QLIST_REMOVE(c, next_parent);
+        QLIST_INSERT_HEAD(&to->parents, c, next_parent);
+        bdrv_ref(to);
+        bdrv_unref(from);
     }
+}
 
-    /* bs_new must be unattached and shouldn't have anything fancy enabled */
-    assert(!bs_new->blk);
-    assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->throttle_state == NULL);
-    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
-
-    tmp = *bs_new;
-    *bs_new = *bs_old;
-    *bs_old = tmp;
+static void swap_feature_fields(BlockDriverState *bs_top,
+                                BlockDriverState *bs_new)
+{
+    BlockDriverState tmp;
 
-    /* there are some fields that should not be swapped, move them back */
-    bdrv_move_feature_fields(&tmp, bs_old);
-    bdrv_move_feature_fields(bs_old, bs_new);
+    bdrv_move_feature_fields(&tmp, bs_top);
+    bdrv_move_feature_fields(bs_top, bs_new);
     bdrv_move_feature_fields(bs_new, &tmp);
 
-    /* bs_new must remain unattached */
-    assert(!bs_new->blk);
-
-    /* Check a few fields that should remain attached to the device */
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(bs_new->throttle_state == NULL);
-    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
-
-    /* Release the ThrottleGroup lock */
-    if (bs_old->throttle_state) {
-        throttle_group_unlock(bs_old);
-    }
-
-    /* insert the nodes back into the graph node list if needed */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_old, node_list);
-    }
-
-    /*
-     * Update lh_first.le_prev for non-empty lists.
-     *
-     * The head of the op blocker list doesn't change because it is moved back
-     * in bdrv_move_feature_fields().
-     */
-    assert(QLIST_EMPTY(&bs_old->tracked_requests));
-    assert(QLIST_EMPTY(&bs_new->tracked_requests));
-
-    QLIST_FIX_HEAD_PTR(&bs_new->children, next);
-    QLIST_FIX_HEAD_PTR(&bs_old->children, next);
-
-    /* Update references in bs->opaque and children */
-    QLIST_FOREACH(child, &bs_old->children, next) {
-        if (child->bs->inherits_from == bs_new) {
-            child->bs->inherits_from = bs_old;
-        }
+    assert(!bs_new->throttle_state);
+    if (bs_top->throttle_state) {
+        assert(bs_top->io_limits_enabled);
+        bdrv_io_limits_enable(bs_new, throttle_group_get_name(bs_top));
+        bdrv_io_limits_disable(bs_top);
     }
-    QLIST_FOREACH(child, &bs_new->children, next) {
-        if (child->bs->inherits_from == bs_old) {
-            child->bs->inherits_from = bs_new;
-        }
-    }
-
-    bdrv_rebind(bs_new);
-    bdrv_rebind(bs_old);
 }
 
 /*
@@ -2165,14 +2277,59 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
  * bs_new must not be attached to a BlockBackend.
  *
  * This function does not create any image files.
+ *
+ * bdrv_append() takes ownership of a bs_new reference and unrefs it because
+ * that's what the callers commonly need. bs_new will be referenced by the old
+ * parents of bs_top after bdrv_append() returns. If the caller needs to keep a
+ * reference of its own, it must call bdrv_ref().
  */
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
 {
-    bdrv_swap(bs_new, bs_top);
+    assert(!bdrv_requests_pending(bs_top));
+    assert(!bdrv_requests_pending(bs_new));
+
+    bdrv_ref(bs_top);
+    change_parent_backing_link(bs_top, bs_new);
+
+    /* Some fields always stay on top of the backing file chain */
+    swap_feature_fields(bs_top, bs_new);
 
-    /* The contents of 'tmp' will become bs_top, as we are
-     * swapping bs_new and bs_top contents. */
-    bdrv_set_backing_hd(bs_top, bs_new);
+    bdrv_set_backing_hd(bs_new, bs_top);
+    bdrv_unref(bs_top);
+
+    /* bs_new is now referenced by its new parents, we don't need the
+     * additional reference any more. */
+    bdrv_unref(bs_new);
+}
+
+void bdrv_replace_in_backing_chain(BlockDriverState *old, BlockDriverState *new)
+{
+    assert(!bdrv_requests_pending(old));
+    assert(!bdrv_requests_pending(new));
+
+    bdrv_ref(old);
+
+    if (old->blk) {
+        /* As long as these fields aren't in BlockBackend, but in the top-level
+         * BlockDriverState, it's not possible for a BDS to have two BBs.
+         *
+         * We really want to copy the fields from old to new, but we go for a
+         * swap instead so that pointers aren't duplicated and cause trouble.
+         * (Also, bdrv_swap() used to do the same.) */
+        assert(!new->blk);
+        swap_feature_fields(old, new);
+    }
+    change_parent_backing_link(old, new);
+
+    /* Change backing files if a previously independent node is added to the
+     * chain. For active commit, we replace top by its own (indirect) backing
+     * file and don't do anything here so we don't build a loop. */
+    if (new->backing == NULL && !bdrv_chain_contains(backing_bs(old), new)) {
+        bdrv_set_backing_hd(new, backing_bs(old));
+        bdrv_set_backing_hd(old, NULL);
+    }
+
+    bdrv_unref(old);
 }
 
 static void bdrv_delete(BlockDriverState *bs)
@@ -2180,12 +2337,14 @@ static void bdrv_delete(BlockDriverState *bs)
     assert(!bs->job);
     assert(bdrv_op_blocker_is_empty(bs));
     assert(!bs->refcnt);
-    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
 
     bdrv_close(bs);
 
     /* remove from list, if necessary */
-    bdrv_make_anon(bs);
+    if (bs->node_name[0] != '\0') {
+        QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
+    }
+    QTAILQ_REMOVE(&all_bdrv_states, bs, bs_list);
 
     g_free(bs);
 }
@@ -2224,20 +2383,20 @@ int bdrv_commit(BlockDriverState *bs)
     if (!drv)
         return -ENOMEDIUM;
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         return -ENOTSUP;
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, NULL) ||
-        bdrv_op_is_blocked(bs->backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET, NULL)) {
+        bdrv_op_is_blocked(bs->backing->bs, BLOCK_OP_TYPE_COMMIT_TARGET, NULL)) {
         return -EBUSY;
     }
 
-    ro = bs->backing_hd->read_only;
-    open_flags =  bs->backing_hd->open_flags;
+    ro = bs->backing->bs->read_only;
+    open_flags =  bs->backing->bs->open_flags;
 
     if (ro) {
-        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+        if (bdrv_reopen(bs->backing->bs, open_flags | BDRV_O_RDWR, NULL)) {
             return -EACCES;
         }
     }
@@ -2248,7 +2407,7 @@ int bdrv_commit(BlockDriverState *bs)
         goto ro_cleanup;
     }
 
-    backing_length = bdrv_getlength(bs->backing_hd);
+    backing_length = bdrv_getlength(bs->backing->bs);
     if (backing_length < 0) {
         ret = backing_length;
         goto ro_cleanup;
@@ -2258,7 +2417,7 @@ int bdrv_commit(BlockDriverState *bs)
      * grow the backing file image if possible.  If not possible,
      * we must return an error */
     if (length > backing_length) {
-        ret = bdrv_truncate(bs->backing_hd, length);
+        ret = bdrv_truncate(bs->backing->bs, length);
         if (ret < 0) {
             goto ro_cleanup;
         }
@@ -2267,7 +2426,7 @@ int bdrv_commit(BlockDriverState *bs)
     total_sectors = length >> BDRV_SECTOR_BITS;
 
     /* qemu_try_blockalign() for bs will choose an alignment that works for
-     * bs->backing_hd as well, so no need to compare the alignment manually. */
+     * bs->backing->bs as well, so no need to compare the alignment manually. */
     buf = qemu_try_blockalign(bs, COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
     if (buf == NULL) {
         ret = -ENOMEM;
@@ -2285,7 +2444,7 @@ int bdrv_commit(BlockDriverState *bs)
                 goto ro_cleanup;
             }
 
-            ret = bdrv_write(bs->backing_hd, sector, buf, n);
+            ret = bdrv_write(bs->backing->bs, sector, buf, n);
             if (ret < 0) {
                 goto ro_cleanup;
             }
@@ -2304,8 +2463,8 @@ int bdrv_commit(BlockDriverState *bs)
      * Make sure all data we wrote to the backing device is actually
      * stable on disk.
      */
-    if (bs->backing_hd) {
-        bdrv_flush(bs->backing_hd);
+    if (bs->backing) {
+        bdrv_flush(bs->backing->bs);
     }
 
     ret = 0;
@@ -2314,32 +2473,12 @@ ro_cleanup:
 
     if (ro) {
         /* ignoring error return here */
-        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
+        bdrv_reopen(bs->backing->bs, open_flags & ~BDRV_O_RDWR, NULL);
     }
 
     return ret;
 }
 
-int bdrv_commit_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        aio_context_acquire(aio_context);
-        if (bs->drv && bs->backing_hd) {
-            int ret = bdrv_commit(bs);
-            if (ret < 0) {
-                aio_context_release(aio_context);
-                return ret;
-            }
-        }
-        aio_context_release(aio_context);
-    }
-    return 0;
-}
-
 /*
  * Return values:
  * 0        - success
@@ -2385,8 +2524,8 @@ int bdrv_change_backing_file(BlockDriverState *bs,
 BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
                                     BlockDriverState *bs)
 {
-    while (active && bs != active->backing_hd) {
-        active = active->backing_hd;
+    while (active && bs != backing_bs(active)) {
+        active = backing_bs(active);
     }
 
     return active;
@@ -2398,12 +2537,6 @@ BlockDriverState *bdrv_find_base(BlockDriverState *bs)
     return bdrv_find_overlay(bs, NULL);
 }
 
-typedef struct BlkIntermediateStates {
-    BlockDriverState *bs;
-    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
-} BlkIntermediateStates;
-
-
 /*
  * Drops images above 'base' up to and including 'top', and sets the image
  * above 'top' to have base as its backing file.
@@ -2436,15 +2569,9 @@ typedef struct BlkIntermediateStates {
 int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
                            BlockDriverState *base, const char *backing_file_str)
 {
-    BlockDriverState *intermediate;
-    BlockDriverState *base_bs = NULL;
     BlockDriverState *new_top_bs = NULL;
-    BlkIntermediateStates *intermediate_state, *next;
     int ret = -EIO;
 
-    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
-    QSIMPLEQ_INIT(&states_to_delete);
-
     if (!top->drv || !base->drv) {
         goto exit;
     }
@@ -2456,55 +2583,29 @@ int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
         goto exit;
     }
 
-    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+    /* special case of new_top_bs->backing->bs already pointing to base - nothing
      * to do, no intermediate images */
-    if (new_top_bs->backing_hd == base) {
+    if (backing_bs(new_top_bs) == base) {
         ret = 0;
         goto exit;
     }
 
-    intermediate = top;
-
-    /* now we will go down through the list, and add each BDS we find
-     * into our deletion queue, until we hit the 'base'
-     */
-    while (intermediate) {
-        intermediate_state = g_new0(BlkIntermediateStates, 1);
-        intermediate_state->bs = intermediate;
-        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
-
-        if (intermediate->backing_hd == base) {
-            base_bs = intermediate->backing_hd;
-            break;
-        }
-        intermediate = intermediate->backing_hd;
-    }
-    if (base_bs == NULL) {
-        /* something went wrong, we did not end at the base. safely
-         * unravel everything, and exit with error */
+    /* Make sure that base is in the backing chain of top */
+    if (!bdrv_chain_contains(top, base)) {
         goto exit;
     }
 
     /* success - we can delete the intermediate states, and link top->base */
-    backing_file_str = backing_file_str ? backing_file_str : base_bs->filename;
+    backing_file_str = backing_file_str ? backing_file_str : base->filename;
     ret = bdrv_change_backing_file(new_top_bs, backing_file_str,
-                                   base_bs->drv ? base_bs->drv->format_name : "");
+                                   base->drv ? base->drv->format_name : "");
     if (ret) {
         goto exit;
     }
-    bdrv_set_backing_hd(new_top_bs, base_bs);
+    bdrv_set_backing_hd(new_top_bs, base);
 
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        /* so that bdrv_close() does not recursively close the chain */
-        bdrv_set_backing_hd(intermediate_state->bs, NULL);
-        bdrv_unref(intermediate_state->bs);
-    }
     ret = 0;
-
 exit:
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        g_free(intermediate_state);
-    }
     return ret;
 }
 
@@ -2547,7 +2648,7 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
         return drv->bdrv_get_allocated_file_size(bs);
     }
     if (bs->file) {
-        return bdrv_get_allocated_file_size(bs->file);
+        return bdrv_get_allocated_file_size(bs->file->bs);
     }
     return -ENOTSUP;
 }
@@ -2591,82 +2692,6 @@ void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr)
     *nb_sectors_ptr = nb_sectors < 0 ? 0 : nb_sectors;
 }
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error)
-{
-    bs->on_read_error = on_read_error;
-    bs->on_write_error = on_write_error;
-}
-
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
-{
-    return is_read ? bs->on_read_error : bs->on_write_error;
-}
-
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
-{
-    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
-
-    switch (on_err) {
-    case BLOCKDEV_ON_ERROR_ENOSPC:
-        return (error == ENOSPC) ?
-               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_STOP:
-        return BLOCK_ERROR_ACTION_STOP;
-    case BLOCKDEV_ON_ERROR_REPORT:
-        return BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_IGNORE:
-        return BLOCK_ERROR_ACTION_IGNORE;
-    default:
-        abort();
-    }
-}
-
-static void send_qmp_error_event(BlockDriverState *bs,
-                                 BlockErrorAction action,
-                                 bool is_read, int error)
-{
-    IoOperationType optype;
-
-    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
-    qapi_event_send_block_io_error(bdrv_get_device_name(bs), optype, action,
-                                   bdrv_iostatus_is_enabled(bs),
-                                   error == ENOSPC, strerror(error),
-                                   &error_abort);
-}
-
-/* This is done by device models because, while the block layer knows
- * about the error, it does not know whether an operation comes from
- * the device or the block layer (from a job, for example).
- */
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error)
-{
-    assert(error >= 0);
-
-    if (action == BLOCK_ERROR_ACTION_STOP) {
-        /* First set the iostatus, so that "info block" returns an iostatus
-         * that matches the events raised so far (an additional error iostatus
-         * is fine, but not a lost one).
-         */
-        bdrv_iostatus_set_err(bs, error);
-
-        /* Then raise the request to stop the VM and the event.
-         * qemu_system_vmstop_request_prepare has two effects.  First,
-         * it ensures that the STOP event always comes after the
-         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
-         * can observe the STOP event and do a "cont" before the STOP
-         * event is issued, the VM will not stop.  In this case, vm_start()
-         * also ensures that the STOP/RESUME pair of events is emitted.
-         */
-        qemu_system_vmstop_request_prepare();
-        send_qmp_error_event(bs, action, is_read, error);
-        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
-    } else {
-        send_qmp_error_event(bs, action, is_read, error);
-    }
-}
-
 int bdrv_is_read_only(BlockDriverState *bs)
 {
     return bs->read_only;
@@ -2677,44 +2702,29 @@ int bdrv_is_sg(BlockDriverState *bs)
     return bs->sg;
 }
 
-int bdrv_enable_write_cache(BlockDriverState *bs)
-{
-    return bs->enable_write_cache;
-}
-
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
-{
-    bs->enable_write_cache = wce;
-
-    /* so a reopen() will preserve wce */
-    if (wce) {
-        bs->open_flags |= BDRV_O_CACHE_WB;
-    } else {
-        bs->open_flags &= ~BDRV_O_CACHE_WB;
-    }
-}
-
 int bdrv_is_encrypted(BlockDriverState *bs)
 {
-    if (bs->backing_hd && bs->backing_hd->encrypted)
+    if (bs->backing && bs->backing->bs->encrypted) {
         return 1;
+    }
     return bs->encrypted;
 }
 
 int bdrv_key_required(BlockDriverState *bs)
 {
-    BlockDriverState *backing_hd = bs->backing_hd;
+    BdrvChild *backing = bs->backing;
 
-    if (backing_hd && backing_hd->encrypted && !backing_hd->valid_key)
+    if (backing && backing->bs->encrypted && !backing->bs->valid_key) {
         return 1;
+    }
     return (bs->encrypted && !bs->valid_key);
 }
 
 int bdrv_set_key(BlockDriverState *bs, const char *key)
 {
     int ret;
-    if (bs->backing_hd && bs->backing_hd->encrypted) {
-        ret = bdrv_set_key(bs->backing_hd, key);
+    if (bs->backing && bs->backing->bs->encrypted) {
+        ret = bdrv_set_key(bs->backing->bs, key);
         if (ret < 0)
             return ret;
         if (!bs->encrypted)
@@ -2833,7 +2843,7 @@ BlockDeviceInfoList *bdrv_named_nodes_list(Error **errp)
 
     list = NULL;
     QTAILQ_FOREACH(bs, &graph_bdrv_states, node_list) {
-        BlockDeviceInfo *info = bdrv_block_device_info(bs, errp);
+        BlockDeviceInfo *info = bdrv_block_device_info(NULL, bs, errp);
         if (!info) {
             qapi_free_BlockDeviceInfoList(list);
             return NULL;
@@ -2858,7 +2868,12 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
         blk = blk_by_name(device);
 
         if (blk) {
-            return blk_bs(blk);
+            bs = blk_bs(blk);
+            if (!bs) {
+                error_setg(errp, "Device '%s' has no medium", device);
+            }
+
+            return bs;
         }
     }
 
@@ -2881,7 +2896,7 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base)
 {
     while (top && top != base) {
-        top = top->backing_hd;
+        top = backing_bs(top);
     }
 
     return top != NULL;
@@ -2895,12 +2910,23 @@ BlockDriverState *bdrv_next_node(BlockDriverState *bs)
     return QTAILQ_NEXT(bs, node_list);
 }
 
+/* Iterates over all top-level BlockDriverStates, i.e. BDSs that are owned by
+ * the monitor or attached to a BlockBackend */
 BlockDriverState *bdrv_next(BlockDriverState *bs)
 {
-    if (!bs) {
-        return QTAILQ_FIRST(&bdrv_states);
+    if (!bs || bs->blk) {
+        bs = blk_next_root_bs(bs);
+        if (bs) {
+            return bs;
+        }
     }
-    return QTAILQ_NEXT(bs, device_list);
+
+    /* Ignore all BDSs that are attached to a BlockBackend here; they have been
+     * handled by the above block already */
+    do {
+        bs = bdrv_next_monitor_owned(bs);
+    } while (bs && bs->blk);
+    return bs;
 }
 
 const char *bdrv_get_node_name(const BlockDriverState *bs)
@@ -2939,7 +2965,7 @@ int bdrv_has_zero_init(BlockDriverState *bs)
 
     /* If BS is a copy on write image, it is initialized to
        the contents of the base image, which may not be zeroes.  */
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return 0;
     }
     if (bs->drv->bdrv_has_zero_init) {
@@ -2954,7 +2980,7 @@ bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs)
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return false;
     }
 
@@ -2969,7 +2995,7 @@ bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs)
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd || !(bs->open_flags & BDRV_O_UNMAP)) {
+    if (bs->backing || !(bs->open_flags & BDRV_O_UNMAP)) {
         return false;
     }
 
@@ -2982,7 +3008,7 @@ bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs)
 
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
 {
-    if (bs->backing_hd && bs->backing_hd->encrypted)
+    if (bs->backing && bs->backing->bs->encrypted)
         return bs->backing_file;
     else if (bs->encrypted)
         return bs->filename;
@@ -3016,7 +3042,7 @@ ImageInfoSpecific *bdrv_get_specific_info(BlockDriverState *bs)
     return NULL;
 }
 
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
+void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event)
 {
     if (!bs || !bs->drv || !bs->drv->bdrv_debug_event) {
         return;
@@ -3029,7 +3055,7 @@ int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
                           const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
@@ -3042,7 +3068,7 @@ int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
 int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_remove_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_remove_breakpoint) {
@@ -3055,7 +3081,7 @@ int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
 {
     while (bs && (!bs->drv || !bs->drv->bdrv_debug_resume)) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
@@ -3068,7 +3094,7 @@ int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
 bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
@@ -3107,13 +3133,13 @@ BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
 
     is_protocol = path_has_protocol(backing_file);
 
-    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+    for (curr_bs = bs; curr_bs->backing; curr_bs = curr_bs->backing->bs) {
 
         /* If either of the filename paths is actually a protocol, then
          * compare unmodified paths; otherwise make paths relative */
         if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
             if (strcmp(backing_file, curr_bs->backing_file) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         } else {
@@ -3137,7 +3163,7 @@ BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
             }
 
             if (strcmp(backing_file_full, filename_full) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         }
@@ -3155,11 +3181,11 @@ int bdrv_get_backing_file_depth(BlockDriverState *bs)
         return 0;
     }
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         return 0;
     }
 
-    return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
+    return 1 + bdrv_get_backing_file_depth(bs->backing->bs);
 }
 
 void bdrv_init(void)
@@ -3182,23 +3208,25 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
         return;
     }
 
-    if (!(bs->open_flags & BDRV_O_INCOMING)) {
+    if (!(bs->open_flags & BDRV_O_INACTIVE)) {
         return;
     }
-    bs->open_flags &= ~BDRV_O_INCOMING;
+    bs->open_flags &= ~BDRV_O_INACTIVE;
 
     if (bs->drv->bdrv_invalidate_cache) {
         bs->drv->bdrv_invalidate_cache(bs, &local_err);
     } else if (bs->file) {
-        bdrv_invalidate_cache(bs->file, &local_err);
+        bdrv_invalidate_cache(bs->file->bs, &local_err);
     }
     if (local_err) {
+        bs->open_flags |= BDRV_O_INACTIVE;
         error_propagate(errp, local_err);
         return;
     }
 
     ret = refresh_total_sectors(bs, bs->total_sectors);
     if (ret < 0) {
+        bs->open_flags |= BDRV_O_INACTIVE;
         error_setg_errno(errp, -ret, "Could not refresh total sector count");
         return;
     }
@@ -3206,10 +3234,10 @@ void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
 
 void bdrv_invalidate_cache_all(Error **errp)
 {
-    BlockDriverState *bs;
+    BlockDriverState *bs = NULL;
     Error *local_err = NULL;
 
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
+    while ((bs = bdrv_next(bs)) != NULL) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
         aio_context_acquire(aio_context);
@@ -3222,21 +3250,63 @@ void bdrv_invalidate_cache_all(Error **errp)
     }
 }
 
+static int bdrv_inactivate(BlockDriverState *bs)
+{
+    int ret;
+
+    if (bs->drv->bdrv_inactivate) {
+        ret = bs->drv->bdrv_inactivate(bs);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    bs->open_flags |= BDRV_O_INACTIVE;
+    return 0;
+}
+
+int bdrv_inactivate_all(void)
+{
+    BlockDriverState *bs = NULL;
+    int ret;
+
+    while ((bs = bdrv_next(bs)) != NULL) {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(aio_context);
+        ret = bdrv_inactivate(bs);
+        aio_context_release(aio_context);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
 /**************************************************************/
 /* removable device support */
 
 /**
  * Return TRUE if the media is present
  */
-int bdrv_is_inserted(BlockDriverState *bs)
+bool bdrv_is_inserted(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
+    BdrvChild *child;
 
-    if (!drv)
-        return 0;
-    if (!drv->bdrv_is_inserted)
-        return 1;
-    return drv->bdrv_is_inserted(bs);
+    if (!drv) {
+        return false;
+    }
+    if (drv->bdrv_is_inserted) {
+        return drv->bdrv_is_inserted(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (!bdrv_is_inserted(child->bs)) {
+            return false;
+        }
+    }
+    return true;
 }
 
 /**
@@ -3287,317 +3357,6 @@ void bdrv_lock_medium(BlockDriverState *bs, bool locked)
     }
 }
 
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align)
-{
-    bs->guest_block_size = align;
-}
-
-BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs, const char *name)
-{
-    BdrvDirtyBitmap *bm;
-
-    assert(name);
-    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
-        if (bm->name && !strcmp(name, bm->name)) {
-            return bm;
-        }
-    }
-    return NULL;
-}
-
-void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap)
-{
-    assert(!bdrv_dirty_bitmap_frozen(bitmap));
-    g_free(bitmap->name);
-    bitmap->name = NULL;
-}
-
-BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
-                                          uint32_t granularity,
-                                          const char *name,
-                                          Error **errp)
-{
-    int64_t bitmap_size;
-    BdrvDirtyBitmap *bitmap;
-    uint32_t sector_granularity;
-
-    assert((granularity & (granularity - 1)) == 0);
-
-    if (name && bdrv_find_dirty_bitmap(bs, name)) {
-        error_setg(errp, "Bitmap already exists: %s", name);
-        return NULL;
-    }
-    sector_granularity = granularity >> BDRV_SECTOR_BITS;
-    assert(sector_granularity);
-    bitmap_size = bdrv_nb_sectors(bs);
-    if (bitmap_size < 0) {
-        error_setg_errno(errp, -bitmap_size, "could not get length of device");
-        errno = -bitmap_size;
-        return NULL;
-    }
-    bitmap = g_new0(BdrvDirtyBitmap, 1);
-    bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(sector_granularity));
-    bitmap->size = bitmap_size;
-    bitmap->name = g_strdup(name);
-    bitmap->disabled = false;
-    QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
-    return bitmap;
-}
-
-bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap)
-{
-    return bitmap->successor;
-}
-
-bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)
-{
-    return !(bitmap->disabled || bitmap->successor);
-}
-
-DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap)
-{
-    if (bdrv_dirty_bitmap_frozen(bitmap)) {
-        return DIRTY_BITMAP_STATUS_FROZEN;
-    } else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
-        return DIRTY_BITMAP_STATUS_DISABLED;
-    } else {
-        return DIRTY_BITMAP_STATUS_ACTIVE;
-    }
-}
-
-/**
- * Create a successor bitmap destined to replace this bitmap after an operation.
- * Requires that the bitmap is not frozen and has no successor.
- */
-int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
-                                       BdrvDirtyBitmap *bitmap, Error **errp)
-{
-    uint64_t granularity;
-    BdrvDirtyBitmap *child;
-
-    if (bdrv_dirty_bitmap_frozen(bitmap)) {
-        error_setg(errp, "Cannot create a successor for a bitmap that is "
-                   "currently frozen");
-        return -1;
-    }
-    assert(!bitmap->successor);
-
-    /* Create an anonymous successor */
-    granularity = bdrv_dirty_bitmap_granularity(bitmap);
-    child = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
-    if (!child) {
-        return -1;
-    }
-
-    /* Successor will be on or off based on our current state. */
-    child->disabled = bitmap->disabled;
-
-    /* Install the successor and freeze the parent */
-    bitmap->successor = child;
-    return 0;
-}
-
-/**
- * For a bitmap with a successor, yield our name to the successor,
- * delete the old bitmap, and return a handle to the new bitmap.
- */
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
-                                            BdrvDirtyBitmap *bitmap,
-                                            Error **errp)
-{
-    char *name;
-    BdrvDirtyBitmap *successor = bitmap->successor;
-
-    if (successor == NULL) {
-        error_setg(errp, "Cannot relinquish control if "
-                   "there's no successor present");
-        return NULL;
-    }
-
-    name = bitmap->name;
-    bitmap->name = NULL;
-    successor->name = name;
-    bitmap->successor = NULL;
-    bdrv_release_dirty_bitmap(bs, bitmap);
-
-    return successor;
-}
-
-/**
- * In cases of failure where we can no longer safely delete the parent,
- * we may wish to re-join the parent and child/successor.
- * The merged parent will be un-frozen, but not explicitly re-enabled.
- */
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
-                                           BdrvDirtyBitmap *parent,
-                                           Error **errp)
-{
-    BdrvDirtyBitmap *successor = parent->successor;
-
-    if (!successor) {
-        error_setg(errp, "Cannot reclaim a successor when none is present");
-        return NULL;
-    }
-
-    if (!hbitmap_merge(parent->bitmap, successor->bitmap)) {
-        error_setg(errp, "Merging of parent and successor bitmap failed");
-        return NULL;
-    }
-    bdrv_release_dirty_bitmap(bs, successor);
-    parent->successor = NULL;
-
-    return parent;
-}
-
-/**
- * Truncates _all_ bitmaps attached to a BDS.
- */
-static void bdrv_dirty_bitmap_truncate(BlockDriverState *bs)
-{
-    BdrvDirtyBitmap *bitmap;
-    uint64_t size = bdrv_nb_sectors(bs);
-
-    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
-        assert(!bdrv_dirty_bitmap_frozen(bitmap));
-        hbitmap_truncate(bitmap->bitmap, size);
-        bitmap->size = size;
-    }
-}
-
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
-{
-    BdrvDirtyBitmap *bm, *next;
-    QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
-        if (bm == bitmap) {
-            assert(!bdrv_dirty_bitmap_frozen(bm));
-            QLIST_REMOVE(bitmap, list);
-            hbitmap_free(bitmap->bitmap);
-            g_free(bitmap->name);
-            g_free(bitmap);
-            return;
-        }
-    }
-}
-
-void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
-{
-    assert(!bdrv_dirty_bitmap_frozen(bitmap));
-    bitmap->disabled = true;
-}
-
-void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
-{
-    assert(!bdrv_dirty_bitmap_frozen(bitmap));
-    bitmap->disabled = false;
-}
-
-BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
-{
-    BdrvDirtyBitmap *bm;
-    BlockDirtyInfoList *list = NULL;
-    BlockDirtyInfoList **plist = &list;
-
-    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
-        BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
-        BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
-        info->count = bdrv_get_dirty_count(bm);
-        info->granularity = bdrv_dirty_bitmap_granularity(bm);
-        info->has_name = !!bm->name;
-        info->name = g_strdup(bm->name);
-        info->status = bdrv_dirty_bitmap_status(bm);
-        entry->value = info;
-        *plist = entry;
-        plist = &entry->next;
-    }
-
-    return list;
-}
-
-int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, int64_t sector)
-{
-    if (bitmap) {
-        return hbitmap_get(bitmap->bitmap, sector);
-    } else {
-        return 0;
-    }
-}
-
-/**
- * Chooses a default granularity based on the existing cluster size,
- * but clamped between [4K, 64K]. Defaults to 64K in the case that there
- * is no cluster size information available.
- */
-uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs)
-{
-    BlockDriverInfo bdi;
-    uint32_t granularity;
-
-    if (bdrv_get_info(bs, &bdi) >= 0 && bdi.cluster_size > 0) {
-        granularity = MAX(4096, bdi.cluster_size);
-        granularity = MIN(65536, granularity);
-    } else {
-        granularity = 65536;
-    }
-
-    return granularity;
-}
-
-uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap)
-{
-    return BDRV_SECTOR_SIZE << hbitmap_granularity(bitmap->bitmap);
-}
-
-void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, HBitmapIter *hbi)
-{
-    hbitmap_iter_init(hbi, bitmap->bitmap, 0);
-}
-
-void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-                           int64_t cur_sector, int nr_sectors)
-{
-    assert(bdrv_dirty_bitmap_enabled(bitmap));
-    hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
-}
-
-void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-                             int64_t cur_sector, int nr_sectors)
-{
-    assert(bdrv_dirty_bitmap_enabled(bitmap));
-    hbitmap_reset(bitmap->bitmap, cur_sector, nr_sectors);
-}
-
-void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap)
-{
-    assert(bdrv_dirty_bitmap_enabled(bitmap));
-    hbitmap_reset_all(bitmap->bitmap);
-}
-
-void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
-                    int nr_sectors)
-{
-    BdrvDirtyBitmap *bitmap;
-    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
-        if (!bdrv_dirty_bitmap_enabled(bitmap)) {
-            continue;
-        }
-        hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
-    }
-}
-
-/**
- * Advance an HBitmapIter to an arbitrary offset.
- */
-void bdrv_set_dirty_iter(HBitmapIter *hbi, int64_t offset)
-{
-    assert(hbi->hb);
-    hbitmap_iter_init(hbi, hbi->hb, offset);
-}
-
-int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
-{
-    return hbitmap_count(bitmap->bitmap);
-}
-
 /* Get a reference to bs */
 void bdrv_ref(BlockDriverState *bs)
 {
@@ -3630,9 +3389,9 @@ bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp)
     if (!QLIST_EMPTY(&bs->op_blockers[op])) {
         blocker = QLIST_FIRST(&bs->op_blockers[op]);
         if (errp) {
-            error_setg(errp, "Node '%s' is busy: %s",
-                       bdrv_get_device_or_node_name(bs),
-                       error_get_pretty(blocker->reason));
+            *errp = error_copy(blocker->reason);
+            error_prepend(errp, "Node '%s' is busy: ",
+                          bdrv_get_device_or_node_name(bs));
         }
         return true;
     }
@@ -3689,46 +3448,6 @@ bool bdrv_op_blocker_is_empty(BlockDriverState *bs)
     return true;
 }
 
-void bdrv_iostatus_enable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = true;
-    bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-}
-
-/* The I/O status is only enabled if the drive explicitly
- * enables it _and_ the VM is configured to stop on errors */
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
-{
-    return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
-            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
-            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
-}
-
-void bdrv_iostatus_disable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = false;
-}
-
-void bdrv_iostatus_reset(BlockDriverState *bs)
-{
-    if (bdrv_iostatus_is_enabled(bs)) {
-        bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-        if (bs->job) {
-            block_job_iostatus_reset(bs->job);
-        }
-    }
-}
-
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
-{
-    assert(bdrv_iostatus_is_enabled(bs));
-    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
-                                         BLOCK_DEVICE_IO_STATUS_FAILED;
-    }
-}
-
 void bdrv_img_create(const char *filename, const char *fmt,
                      const char *base_filename, const char *base_fmt,
                      char *options, uint64_t img_size, int flags,
@@ -3739,7 +3458,6 @@ void bdrv_img_create(const char *filename, const char *fmt,
     const char *backing_fmt, *backing_file;
     int64_t size;
     BlockDriver *drv, *proto_drv;
-    BlockDriver *backing_drv = NULL;
     Error *local_err = NULL;
     int ret = 0;
 
@@ -3813,14 +3531,6 @@ void bdrv_img_create(const char *filename, const char *fmt,
     }
 
     backing_fmt = qemu_opt_get(opts, BLOCK_OPT_BACKING_FMT);
-    if (backing_fmt) {
-        backing_drv = bdrv_find_format(backing_fmt);
-        if (!backing_drv) {
-            error_setg(errp, "Unknown backing file format '%s'",
-                       backing_fmt);
-            goto out;
-        }
-    }
 
     // The size for the image must always be specified, with one exception:
     // If we are using a backing file, we can obtain the size from there
@@ -3831,6 +3541,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
             char *full_backing = g_new0(char, PATH_MAX);
             int64_t size;
             int back_flags;
+            QDict *backing_options = NULL;
 
             bdrv_get_full_backing_filename_from_filename(filename, backing_file,
                                                          full_backing, PATH_MAX,
@@ -3841,12 +3552,18 @@ void bdrv_img_create(const char *filename, const char *fmt,
             }
 
             /* backing files always opened read-only */
-            back_flags =
-                flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+            back_flags = flags;
+            back_flags &= ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+
+            if (backing_fmt) {
+                backing_options = qdict_new();
+                qdict_put(backing_options, "driver",
+                          qstring_from_str(backing_fmt));
+            }
 
             bs = NULL;
-            ret = bdrv_open(&bs, full_backing, NULL, NULL, back_flags,
-                            backing_drv, &local_err);
+            ret = bdrv_open(&bs, full_backing, NULL, backing_options,
+                            back_flags, &local_err);
             g_free(full_backing);
             if (ret < 0) {
                 goto out;
@@ -3869,7 +3586,7 @@ void bdrv_img_create(const char *filename, const char *fmt,
     }
 
     if (!quiet) {
-        printf("Formatting '%s', fmt=%s", filename, fmt);
+        printf("Formatting '%s', fmt=%s ", filename, fmt);
         qemu_opts_print(opts, " ");
         puts("");
     }
@@ -3915,17 +3632,17 @@ void bdrv_detach_aio_context(BlockDriverState *bs)
         baf->detach_aio_context(baf->opaque);
     }
 
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         throttle_timers_detach_aio_context(&bs->throttle_timers);
     }
     if (bs->drv->bdrv_detach_aio_context) {
         bs->drv->bdrv_detach_aio_context(bs);
     }
     if (bs->file) {
-        bdrv_detach_aio_context(bs->file);
+        bdrv_detach_aio_context(bs->file->bs);
     }
-    if (bs->backing_hd) {
-        bdrv_detach_aio_context(bs->backing_hd);
+    if (bs->backing) {
+        bdrv_detach_aio_context(bs->backing->bs);
     }
 
     bs->aio_context = NULL;
@@ -3942,16 +3659,16 @@ void bdrv_attach_aio_context(BlockDriverState *bs,
 
     bs->aio_context = new_context;
 
-    if (bs->backing_hd) {
-        bdrv_attach_aio_context(bs->backing_hd, new_context);
+    if (bs->backing) {
+        bdrv_attach_aio_context(bs->backing->bs, new_context);
     }
     if (bs->file) {
-        bdrv_attach_aio_context(bs->file, new_context);
+        bdrv_attach_aio_context(bs->file->bs, new_context);
     }
     if (bs->drv->bdrv_attach_aio_context) {
         bs->drv->bdrv_attach_aio_context(bs, new_context);
     }
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         throttle_timers_attach_aio_context(&bs->throttle_timers, new_context);
     }
 
@@ -4012,12 +3729,12 @@ void bdrv_remove_aio_context_notifier(BlockDriverState *bs,
 }
 
 int bdrv_amend_options(BlockDriverState *bs, QemuOpts *opts,
-                       BlockDriverAmendStatusCB *status_cb)
+                       BlockDriverAmendStatusCB *status_cb, void *cb_opaque)
 {
     if (!bs->drv->bdrv_amend_options) {
         return -ENOTSUP;
     }
-    return bs->drv->bdrv_amend_options(bs, opts, status_cb);
+    return bs->drv->bdrv_amend_options(bs, opts, status_cb, cb_opaque);
 }
 
 /* This function will be called by the bdrv_recurse_is_first_non_filter method
@@ -4059,10 +3776,10 @@ bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
  */
 bool bdrv_is_first_non_filter(BlockDriverState *candidate)
 {
-    BlockDriverState *bs;
+    BlockDriverState *bs = NULL;
 
     /* walk down the bs forest recursively */
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
+    while ((bs = bdrv_next(bs)) != NULL) {
         bool perm;
 
         /* try to recurse in this top level bs */
@@ -4077,7 +3794,8 @@ bool bdrv_is_first_non_filter(BlockDriverState *candidate)
     return false;
 }
 
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp)
 {
     BlockDriverState *to_replace_bs = bdrv_find_node(node_name);
     AioContext *aio_context;
@@ -4100,7 +3818,7 @@ BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
      * Another benefit is that this tests exclude backing files which are
      * blocked by the backing blockers.
      */
-    if (!bdrv_is_first_non_filter(to_replace_bs)) {
+    if (!bdrv_recurse_is_first_non_filter(parent_bs, to_replace_bs)) {
         error_setg(errp, "Only top most non filter can be replaced");
         to_replace_bs = NULL;
         goto out;
@@ -4114,20 +3832,39 @@ out:
 static bool append_open_options(QDict *d, BlockDriverState *bs)
 {
     const QDictEntry *entry;
+    QemuOptDesc *desc;
+    BdrvChild *child;
     bool found_any = false;
+    const char *p;
 
     for (entry = qdict_first(bs->options); entry;
          entry = qdict_next(bs->options, entry))
     {
-        /* Only take options for this level and exclude all non-driver-specific
-         * options */
-        if (!strchr(qdict_entry_key(entry), '.') &&
-            strcmp(qdict_entry_key(entry), "node-name"))
-        {
-            qobject_incref(qdict_entry_value(entry));
-            qdict_put_obj(d, qdict_entry_key(entry), qdict_entry_value(entry));
-            found_any = true;
+        /* Exclude options for children */
+        QLIST_FOREACH(child, &bs->children, next) {
+            if (strstart(qdict_entry_key(entry), child->name, &p)
+                && (!*p || *p == '.'))
+            {
+                break;
+            }
         }
+        if (child) {
+            continue;
+        }
+
+        /* And exclude all non-driver-specific options */
+        for (desc = bdrv_runtime_opts.desc; desc->name; desc++) {
+            if (!strcmp(qdict_entry_key(entry), desc->name)) {
+                break;
+            }
+        }
+        if (desc->name) {
+            continue;
+        }
+
+        qobject_incref(qdict_entry_value(entry));
+        qdict_put_obj(d, qdict_entry_key(entry), qdict_entry_value(entry));
+        found_any = true;
     }
 
     return found_any;
@@ -4157,7 +3894,7 @@ void bdrv_refresh_filename(BlockDriverState *bs)
     /* This BDS's file name will most probably depend on its file's name, so
      * refresh that first */
     if (bs->file) {
-        bdrv_refresh_filename(bs->file);
+        bdrv_refresh_filename(bs->file->bs);
     }
 
     if (drv->bdrv_refresh_filename) {
@@ -4169,7 +3906,10 @@ void bdrv_refresh_filename(BlockDriverState *bs)
             bs->full_open_options = NULL;
         }
 
-        drv->bdrv_refresh_filename(bs);
+        opts = qdict_new();
+        append_open_options(opts, bs);
+        drv->bdrv_refresh_filename(bs, opts);
+        QDECREF(opts);
     } else if (bs->file) {
         /* Try to reconstruct valid information from the underlying file */
         bool has_open_options;
@@ -4185,19 +3925,20 @@ void bdrv_refresh_filename(BlockDriverState *bs)
 
         /* If no specific options have been given for this BDS, the filename of
          * the underlying file should suffice for this one as well */
-        if (bs->file->exact_filename[0] && !has_open_options) {
-            strcpy(bs->exact_filename, bs->file->exact_filename);
+        if (bs->file->bs->exact_filename[0] && !has_open_options) {
+            strcpy(bs->exact_filename, bs->file->bs->exact_filename);
         }
         /* Reconstructing the full options QDict is simple for most format block
          * drivers, as long as the full options are known for the underlying
          * file BDS. The full options QDict of that file BDS should somehow
          * contain a representation of the filename, therefore the following
          * suffices without querying the (exact_)filename of this BDS. */
-        if (bs->file->full_open_options) {
+        if (bs->file->bs->full_open_options) {
             qdict_put_obj(opts, "driver",
                           QOBJECT(qstring_from_str(drv->format_name)));
-            QINCREF(bs->file->full_open_options);
-            qdict_put_obj(opts, "file", QOBJECT(bs->file->full_open_options));
+            QINCREF(bs->file->bs->full_open_options);
+            qdict_put_obj(opts, "file",
+                          QOBJECT(bs->file->bs->full_open_options));
 
             bs->full_open_options = opts;
         } else {
@@ -4240,14 +3981,3 @@ void bdrv_refresh_filename(BlockDriverState *bs)
         QDECREF(json);
     }
 }
-
-/* This accessor function purpose is to allow the device models to access the
- * BlockAcctStats structure embedded inside a BlockDriverState without being
- * aware of the BlockDriverState structure layout.
- * It will go away when the BlockAcctStats structure will be moved inside
- * the device models.
- */
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs)
-{
-    return &bs->stats;
-}
diff --git a/qemu/block/Makefile.objs b/qemu/block/Makefile.objs
index 58ef2ef3f..44a541622 100644
--- a/qemu/block/Makefile.objs
+++ b/qemu/block/Makefile.objs
@@ -4,7 +4,7 @@ block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
 block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o vhdx-log.o
 block-obj-y += quorum.o
-block-obj-y += parallels.o blkdebug.o blkverify.o
+block-obj-y += parallels.o blkdebug.o blkverify.o blkreplay.o
 block-obj-y += block-backend.o snapshot.o qapi.o
 block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
 block-obj-$(CONFIG_POSIX) += raw-posix.o
@@ -20,9 +20,11 @@ block-obj-$(CONFIG_RBD) += rbd.o
 block-obj-$(CONFIG_GLUSTERFS) += gluster.o
 block-obj-$(CONFIG_ARCHIPELAGO) += archipelago.o
 block-obj-$(CONFIG_LIBSSH2) += ssh.o
-block-obj-y += accounting.o
+block-obj-y += accounting.o dirty-bitmap.o
 block-obj-y += write-threshold.o
 
+block-obj-y += crypto.o
+
 common-obj-y += stream.o
 common-obj-y += commit.o
 common-obj-y += backup.o
diff --git a/qemu/block/accounting.c b/qemu/block/accounting.c
index 01d594ffd..3f457c4e7 100644
--- a/qemu/block/accounting.c
+++ b/qemu/block/accounting.c
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,9 +23,58 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/accounting.h"
 #include "block/block_int.h"
 #include "qemu/timer.h"
+#include "sysemu/qtest.h"
+
+static QEMUClockType clock_type = QEMU_CLOCK_REALTIME;
+static const int qtest_latency_ns = NANOSECONDS_PER_SECOND / 1000;
+
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed)
+{
+    stats->account_invalid = account_invalid;
+    stats->account_failed = account_failed;
+
+    if (qtest_enabled()) {
+        clock_type = QEMU_CLOCK_VIRTUAL;
+    }
+}
+
+void block_acct_cleanup(BlockAcctStats *stats)
+{
+    BlockAcctTimedStats *s, *next;
+    QSLIST_FOREACH_SAFE(s, &stats->intervals, entries, next) {
+        g_free(s);
+    }
+}
+
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length)
+{
+    BlockAcctTimedStats *s;
+    unsigned i;
+
+    s = g_new0(BlockAcctTimedStats, 1);
+    s->interval_length = interval_length;
+    QSLIST_INSERT_HEAD(&stats->intervals, s, entries);
+
+    for (i = 0; i < BLOCK_MAX_IOTYPE; i++) {
+        timed_average_init(&s->latency[i], clock_type,
+                           (uint64_t) interval_length * NANOSECONDS_PER_SECOND);
+    }
+}
+
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s)
+{
+    if (s == NULL) {
+        return QSLIST_FIRST(&stats->intervals);
+    } else {
+        return QSLIST_NEXT(s, entries);
+    }
+}
 
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type)
@@ -32,26 +82,69 @@ void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
     assert(type < BLOCK_MAX_IOTYPE);
 
     cookie->bytes = bytes;
-    cookie->start_time_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    cookie->start_time_ns = qemu_clock_get_ns(clock_type);
     cookie->type = type;
 }
 
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie)
 {
+    BlockAcctTimedStats *s;
+    int64_t time_ns = qemu_clock_get_ns(clock_type);
+    int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+    if (qtest_enabled()) {
+        latency_ns = qtest_latency_ns;
+    }
+
     assert(cookie->type < BLOCK_MAX_IOTYPE);
 
     stats->nr_bytes[cookie->type] += cookie->bytes;
     stats->nr_ops[cookie->type]++;
-    stats->total_time_ns[cookie->type] +=
-        qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - cookie->start_time_ns;
+    stats->total_time_ns[cookie->type] += latency_ns;
+    stats->last_access_time_ns = time_ns;
+
+    QSLIST_FOREACH(s, &stats->intervals, entries) {
+        timed_average_account(&s->latency[cookie->type], latency_ns);
+    }
 }
 
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie)
+{
+    assert(cookie->type < BLOCK_MAX_IOTYPE);
+
+    stats->failed_ops[cookie->type]++;
+
+    if (stats->account_failed) {
+        BlockAcctTimedStats *s;
+        int64_t time_ns = qemu_clock_get_ns(clock_type);
+        int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+        if (qtest_enabled()) {
+            latency_ns = qtest_latency_ns;
+        }
 
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors)
+        stats->total_time_ns[cookie->type] += latency_ns;
+        stats->last_access_time_ns = time_ns;
+
+        QSLIST_FOREACH(s, &stats->intervals, entries) {
+            timed_average_account(&s->latency[cookie->type], latency_ns);
+        }
+    }
+}
+
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type)
 {
-    if (stats->wr_highest_sector < sector_num + nb_sectors - 1) {
-        stats->wr_highest_sector = sector_num + nb_sectors - 1;
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    /* block_acct_done() and block_acct_failed() update
+     * total_time_ns[], but this one does not. The reason is that
+     * invalid requests are accounted during their submission,
+     * therefore there's no actual I/O involved. */
+
+    stats->invalid_ops[type]++;
+
+    if (stats->account_invalid) {
+        stats->last_access_time_ns = qemu_clock_get_ns(clock_type);
     }
 }
 
@@ -61,3 +154,20 @@ void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
     assert(type < BLOCK_MAX_IOTYPE);
     stats->merged[type] += num_requests;
 }
+
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats)
+{
+    return qemu_clock_get_ns(clock_type) - stats->last_access_time_ns;
+}
+
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type)
+{
+    uint64_t sum, elapsed;
+
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    sum = timed_average_sum(&stats->latency[type], &elapsed);
+
+    return (double) sum / elapsed;
+}
diff --git a/qemu/block/archipelago.c b/qemu/block/archipelago.c
index 855655c6b..b9f5e69d4 100644
--- a/qemu/block/archipelago.c
+++ b/qemu/block/archipelago.c
@@ -50,7 +50,8 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "block/block_int.h"
 #include "qemu/error-report.h"
 #include "qemu/thread.h"
@@ -59,7 +60,6 @@
 #include "qapi/qmp/qjson.h"
 #include "qemu/atomic.h"
 
-#include <inttypes.h>
 #include <xseg/xseg.h>
 #include <xseg/protocol.h>
 
diff --git a/qemu/block/backup.c b/qemu/block/backup.c
index 965654d52..491fd1406 100644
--- a/qemu/block/backup.c
+++ b/qemu/block/backup.c
@@ -11,21 +11,20 @@
  *
  */
 
-#include <stdio.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "trace.h"
 #include "block/block.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "qemu/cutils.h"
+#include "sysemu/block-backend.h"
+#include "qemu/bitmap.h"
 
-#define BACKUP_CLUSTER_BITS 16
-#define BACKUP_CLUSTER_SIZE (1 << BACKUP_CLUSTER_BITS)
-#define BACKUP_SECTORS_PER_CLUSTER (BACKUP_CLUSTER_SIZE / BDRV_SECTOR_SIZE)
-
+#define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)
 #define SLICE_TIME 100000000ULL /* ns */
 
 typedef struct CowRequest {
@@ -46,10 +45,17 @@ typedef struct BackupBlockJob {
     BlockdevOnError on_target_error;
     CoRwlock flush_rwlock;
     uint64_t sectors_read;
-    HBitmap *bitmap;
+    unsigned long *done_bitmap;
+    int64_t cluster_size;
     QLIST_HEAD(, CowRequest) inflight_reqs;
 } BackupBlockJob;
 
+/* Size of a cluster in sectors, instead of bytes. */
+static inline int64_t cluster_size_sectors(BackupBlockJob *job)
+{
+  return job->cluster_size / BDRV_SECTOR_SIZE;
+}
+
 /* See if in-flight requests overlap and wait for them to complete */
 static void coroutine_fn wait_for_overlapping_requests(BackupBlockJob *job,
                                                        int64_t start,
@@ -89,7 +95,8 @@ static void cow_request_end(CowRequest *req)
 
 static int coroutine_fn backup_do_cow(BlockDriverState *bs,
                                       int64_t sector_num, int nb_sectors,
-                                      bool *error_is_read)
+                                      bool *error_is_read,
+                                      bool is_write_notifier)
 {
     BackupBlockJob *job = (BackupBlockJob *)bs->job;
     CowRequest cow_request;
@@ -97,13 +104,14 @@ static int coroutine_fn backup_do_cow(BlockDriverState *bs,
     QEMUIOVector bounce_qiov;
     void *bounce_buffer = NULL;
     int ret = 0;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
     int64_t start, end;
     int n;
 
     qemu_co_rwlock_rdlock(&job->flush_rwlock);
 
-    start = sector_num / BACKUP_SECTORS_PER_CLUSTER;
-    end = DIV_ROUND_UP(sector_num + nb_sectors, BACKUP_SECTORS_PER_CLUSTER);
+    start = sector_num / sectors_per_cluster;
+    end = DIV_ROUND_UP(sector_num + nb_sectors, sectors_per_cluster);
 
     trace_backup_do_cow_enter(job, start, sector_num, nb_sectors);
 
@@ -111,26 +119,32 @@ static int coroutine_fn backup_do_cow(BlockDriverState *bs,
     cow_request_begin(&cow_request, job, start, end);
 
     for (; start < end; start++) {
-        if (hbitmap_get(job->bitmap, start)) {
+        if (test_bit(start, job->done_bitmap)) {
             trace_backup_do_cow_skip(job, start);
             continue; /* already copied */
         }
 
         trace_backup_do_cow_process(job, start);
 
-        n = MIN(BACKUP_SECTORS_PER_CLUSTER,
+        n = MIN(sectors_per_cluster,
                 job->common.len / BDRV_SECTOR_SIZE -
-                start * BACKUP_SECTORS_PER_CLUSTER);
+                start * sectors_per_cluster);
 
         if (!bounce_buffer) {
-            bounce_buffer = qemu_blockalign(bs, BACKUP_CLUSTER_SIZE);
+            bounce_buffer = qemu_blockalign(bs, job->cluster_size);
         }
         iov.iov_base = bounce_buffer;
         iov.iov_len = n * BDRV_SECTOR_SIZE;
         qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 
-        ret = bdrv_co_readv(bs, start * BACKUP_SECTORS_PER_CLUSTER, n,
-                            &bounce_qiov);
+        if (is_write_notifier) {
+            ret = bdrv_co_readv_no_serialising(bs,
+                                           start * sectors_per_cluster,
+                                           n, &bounce_qiov);
+        } else {
+            ret = bdrv_co_readv(bs, start * sectors_per_cluster, n,
+                                &bounce_qiov);
+        }
         if (ret < 0) {
             trace_backup_do_cow_read_fail(job, start, ret);
             if (error_is_read) {
@@ -141,11 +155,11 @@ static int coroutine_fn backup_do_cow(BlockDriverState *bs,
 
         if (buffer_is_zero(iov.iov_base, iov.iov_len)) {
             ret = bdrv_co_write_zeroes(job->target,
-                                       start * BACKUP_SECTORS_PER_CLUSTER,
+                                       start * sectors_per_cluster,
                                        n, BDRV_REQ_MAY_UNMAP);
         } else {
             ret = bdrv_co_writev(job->target,
-                                 start * BACKUP_SECTORS_PER_CLUSTER, n,
+                                 start * sectors_per_cluster, n,
                                  &bounce_qiov);
         }
         if (ret < 0) {
@@ -156,7 +170,7 @@ static int coroutine_fn backup_do_cow(BlockDriverState *bs,
             goto out;
         }
 
-        hbitmap_set(job->bitmap, start, 1);
+        set_bit(start, job->done_bitmap);
 
         /* Publish progress, guest I/O counts as progress too.  Note that the
          * offset field is an opaque progress value, it is not a disk offset.
@@ -190,7 +204,7 @@ static int coroutine_fn backup_before_write_notify(
     assert((req->offset & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert((req->bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 
-    return backup_do_cow(req->bs, sector_num, nb_sectors, NULL);
+    return backup_do_cow(req->bs, sector_num, nb_sectors, NULL, true);
 }
 
 static void backup_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -208,7 +222,41 @@ static void backup_iostatus_reset(BlockJob *job)
 {
     BackupBlockJob *s = container_of(job, BackupBlockJob, common);
 
-    bdrv_iostatus_reset(s->target);
+    if (s->target->blk) {
+        blk_iostatus_reset(s->target->blk);
+    }
+}
+
+static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
+{
+    BdrvDirtyBitmap *bm;
+    BlockDriverState *bs = job->common.bs;
+
+    if (ret < 0 || block_job_is_cancelled(&job->common)) {
+        /* Merge the successor back into the parent, delete nothing. */
+        bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    } else {
+        /* Everything is fine, delete this bitmap and install the backup. */
+        bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    }
+}
+
+static void backup_commit(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, 0);
+    }
+}
+
+static void backup_abort(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, -1);
+    }
 }
 
 static const BlockJobDriver backup_job_driver = {
@@ -216,6 +264,8 @@ static const BlockJobDriver backup_job_driver = {
     .job_type       = BLOCK_JOB_TYPE_BACKUP,
     .set_speed      = backup_set_speed,
     .iostatus_reset = backup_iostatus_reset,
+    .commit         = backup_commit,
+    .abort          = backup_abort,
 };
 
 static BlockErrorAction backup_error_action(BackupBlockJob *job,
@@ -280,21 +330,22 @@ static int coroutine_fn backup_run_incremental(BackupBlockJob *job)
     int64_t cluster;
     int64_t end;
     int64_t last_cluster = -1;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
     BlockDriverState *bs = job->common.bs;
     HBitmapIter hbi;
 
     granularity = bdrv_dirty_bitmap_granularity(job->sync_bitmap);
-    clusters_per_iter = MAX((granularity / BACKUP_CLUSTER_SIZE), 1);
+    clusters_per_iter = MAX((granularity / job->cluster_size), 1);
     bdrv_dirty_iter_init(job->sync_bitmap, &hbi);
 
     /* Find the next dirty sector(s) */
     while ((sector = hbitmap_iter_next(&hbi)) != -1) {
-        cluster = sector / BACKUP_SECTORS_PER_CLUSTER;
+        cluster = sector / sectors_per_cluster;
 
         /* Fake progress updates for any clusters we skipped */
         if (cluster != last_cluster + 1) {
             job->common.offset += ((cluster - last_cluster - 1) *
-                                   BACKUP_CLUSTER_SIZE);
+                                   job->cluster_size);
         }
 
         for (end = cluster + clusters_per_iter; cluster < end; cluster++) {
@@ -302,8 +353,9 @@ static int coroutine_fn backup_run_incremental(BackupBlockJob *job)
                 if (yield_and_check(job)) {
                     return ret;
                 }
-                ret = backup_do_cow(bs, cluster * BACKUP_SECTORS_PER_CLUSTER,
-                                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read);
+                ret = backup_do_cow(bs, cluster * sectors_per_cluster,
+                                    sectors_per_cluster, &error_is_read,
+                                    false);
                 if ((ret < 0) &&
                     backup_error_action(job, error_is_read, -ret) ==
                     BLOCK_ERROR_ACTION_REPORT) {
@@ -314,17 +366,17 @@ static int coroutine_fn backup_run_incremental(BackupBlockJob *job)
 
         /* If the bitmap granularity is smaller than the backup granularity,
          * we need to advance the iterator pointer to the next cluster. */
-        if (granularity < BACKUP_CLUSTER_SIZE) {
-            bdrv_set_dirty_iter(&hbi, cluster * BACKUP_SECTORS_PER_CLUSTER);
+        if (granularity < job->cluster_size) {
+            bdrv_set_dirty_iter(&hbi, cluster * sectors_per_cluster);
         }
 
         last_cluster = cluster - 1;
     }
 
     /* Play some final catchup with the progress meter */
-    end = DIV_ROUND_UP(job->common.len, BACKUP_CLUSTER_SIZE);
+    end = DIV_ROUND_UP(job->common.len, job->cluster_size);
     if (last_cluster + 1 < end) {
-        job->common.offset += ((end - last_cluster - 1) * BACKUP_CLUSTER_SIZE);
+        job->common.offset += ((end - last_cluster - 1) * job->cluster_size);
     }
 
     return ret;
@@ -341,19 +393,21 @@ static void coroutine_fn backup_run(void *opaque)
         .notify = backup_before_write_notify,
     };
     int64_t start, end;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
     int ret = 0;
 
     QLIST_INIT(&job->inflight_reqs);
     qemu_co_rwlock_init(&job->flush_rwlock);
 
     start = 0;
-    end = DIV_ROUND_UP(job->common.len, BACKUP_CLUSTER_SIZE);
+    end = DIV_ROUND_UP(job->common.len, job->cluster_size);
 
-    job->bitmap = hbitmap_alloc(end, 0);
+    job->done_bitmap = bitmap_new(end);
 
-    bdrv_set_enable_write_cache(target, true);
-    bdrv_set_on_error(target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(target);
+    if (target->blk) {
+        blk_set_on_error(target->blk, on_target_error, on_target_error);
+        blk_iostatus_enable(target->blk);
+    }
 
     bdrv_add_before_write_notifier(bs, &before_write);
 
@@ -382,7 +436,7 @@ static void coroutine_fn backup_run(void *opaque)
                 /* Check to see if these blocks are already in the
                  * backing file. */
 
-                for (i = 0; i < BACKUP_SECTORS_PER_CLUSTER;) {
+                for (i = 0; i < sectors_per_cluster;) {
                     /* bdrv_is_allocated() only returns true/false based
                      * on the first set of sectors it comes across that
                      * are are all in the same state.
@@ -391,8 +445,8 @@ static void coroutine_fn backup_run(void *opaque)
                      * needed but at some point that is always the case. */
                     alloced =
                         bdrv_is_allocated(bs,
-                                start * BACKUP_SECTORS_PER_CLUSTER + i,
-                                BACKUP_SECTORS_PER_CLUSTER - i, &n);
+                                start * sectors_per_cluster + i,
+                                sectors_per_cluster - i, &n);
                     i += n;
 
                     if (alloced == 1 || n == 0) {
@@ -407,8 +461,8 @@ static void coroutine_fn backup_run(void *opaque)
                 }
             }
             /* FULL sync mode we copy the whole drive. */
-            ret = backup_do_cow(bs, start * BACKUP_SECTORS_PER_CLUSTER,
-                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read);
+            ret = backup_do_cow(bs, start * sectors_per_cluster,
+                                sectors_per_cluster, &error_is_read, false);
             if (ret < 0) {
                 /* Depending on error action, fail now or retry cluster */
                 BlockErrorAction action =
@@ -428,22 +482,11 @@ static void coroutine_fn backup_run(void *opaque)
     /* wait until pending backup_do_cow() calls have completed */
     qemu_co_rwlock_wrlock(&job->flush_rwlock);
     qemu_co_rwlock_unlock(&job->flush_rwlock);
+    g_free(job->done_bitmap);
 
-    if (job->sync_bitmap) {
-        BdrvDirtyBitmap *bm;
-        if (ret < 0 || block_job_is_cancelled(&job->common)) {
-            /* Merge the successor back into the parent, delete nothing. */
-            bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
-            assert(bm);
-        } else {
-            /* Everything is fine, delete this bitmap and install the backup. */
-            bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
-            assert(bm);
-        }
+    if (target->blk) {
+        blk_iostatus_disable(target->blk);
     }
-    hbitmap_free(job->bitmap);
-
-    bdrv_iostatus_disable(target);
     bdrv_op_unblock_all(target, job->common.blocker);
 
     data = g_malloc(sizeof(*data));
@@ -457,9 +500,11 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp)
+                  BlockJobTxn *txn, Error **errp)
 {
     int64_t len;
+    BlockDriverInfo bdi;
+    int ret;
 
     assert(bs);
     assert(target);
@@ -472,7 +517,7 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
 
     if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
          on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error");
         return;
     }
@@ -529,16 +574,35 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
         goto error;
     }
 
-    bdrv_op_block_all(target, job->common.blocker);
-
     job->on_source_error = on_source_error;
     job->on_target_error = on_target_error;
     job->target = target;
     job->sync_mode = sync_mode;
     job->sync_bitmap = sync_mode == MIRROR_SYNC_MODE_INCREMENTAL ?
                        sync_bitmap : NULL;
+
+    /* If there is no backing file on the target, we cannot rely on COW if our
+     * backup cluster size is smaller than the target cluster size. Even for
+     * targets with a backing file, try to avoid COW if possible. */
+    ret = bdrv_get_info(job->target, &bdi);
+    if (ret < 0 && !target->backing) {
+        error_setg_errno(errp, -ret,
+            "Couldn't determine the cluster size of the target image, "
+            "which has no backing file");
+        error_append_hint(errp,
+            "Aborting, since this may create an unusable destination image\n");
+        goto error;
+    } else if (ret < 0 && target->backing) {
+        /* Not fatal; just trudge on ahead. */
+        job->cluster_size = BACKUP_CLUSTER_SIZE_DEFAULT;
+    } else {
+        job->cluster_size = MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster_size);
+    }
+
+    bdrv_op_block_all(target, job->common.blocker);
     job->common.len = len;
     job->common.co = qemu_coroutine_create(backup_run);
+    block_job_txn_add_job(txn, &job->common);
     qemu_coroutine_enter(job->common.co, job);
     return;
 
diff --git a/qemu/block/blkdebug.c b/qemu/block/blkdebug.c
index bc247f46f..20d25bda6 100644
--- a/qemu/block/blkdebug.c
+++ b/qemu/block/blkdebug.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/config-file.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
@@ -30,12 +32,13 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qstring.h"
+#include "sysemu/qtest.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
     int new_state;
 
-    QLIST_HEAD(, BlkdebugRule) rules[BLKDBG_EVENT_MAX];
+    QLIST_HEAD(, BlkdebugRule) rules[BLKDBG__MAX];
     QSIMPLEQ_HEAD(, BlkdebugRule) active_rules;
     QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
 } BDRVBlkdebugState;
@@ -63,7 +66,7 @@ enum {
 };
 
 typedef struct BlkdebugRule {
-    BlkDebugEvent event;
+    BlkdebugEvent event;
     int action;
     int state;
     union {
@@ -142,69 +145,12 @@ static QemuOptsList *config_groups[] = {
     NULL
 };
 
-static const char *event_names[BLKDBG_EVENT_MAX] = {
-    [BLKDBG_L1_UPDATE]                      = "l1_update",
-    [BLKDBG_L1_GROW_ALLOC_TABLE]            = "l1_grow.alloc_table",
-    [BLKDBG_L1_GROW_WRITE_TABLE]            = "l1_grow.write_table",
-    [BLKDBG_L1_GROW_ACTIVATE_TABLE]         = "l1_grow.activate_table",
-
-    [BLKDBG_L2_LOAD]                        = "l2_load",
-    [BLKDBG_L2_UPDATE]                      = "l2_update",
-    [BLKDBG_L2_UPDATE_COMPRESSED]           = "l2_update_compressed",
-    [BLKDBG_L2_ALLOC_COW_READ]              = "l2_alloc.cow_read",
-    [BLKDBG_L2_ALLOC_WRITE]                 = "l2_alloc.write",
-
-    [BLKDBG_READ_AIO]                       = "read_aio",
-    [BLKDBG_READ_BACKING_AIO]               = "read_backing_aio",
-    [BLKDBG_READ_COMPRESSED]                = "read_compressed",
-
-    [BLKDBG_WRITE_AIO]                      = "write_aio",
-    [BLKDBG_WRITE_COMPRESSED]               = "write_compressed",
-
-    [BLKDBG_VMSTATE_LOAD]                   = "vmstate_load",
-    [BLKDBG_VMSTATE_SAVE]                   = "vmstate_save",
-
-    [BLKDBG_COW_READ]                       = "cow_read",
-    [BLKDBG_COW_WRITE]                      = "cow_write",
-
-    [BLKDBG_REFTABLE_LOAD]                  = "reftable_load",
-    [BLKDBG_REFTABLE_GROW]                  = "reftable_grow",
-    [BLKDBG_REFTABLE_UPDATE]                = "reftable_update",
-
-    [BLKDBG_REFBLOCK_LOAD]                  = "refblock_load",
-    [BLKDBG_REFBLOCK_UPDATE]                = "refblock_update",
-    [BLKDBG_REFBLOCK_UPDATE_PART]           = "refblock_update_part",
-    [BLKDBG_REFBLOCK_ALLOC]                 = "refblock_alloc",
-    [BLKDBG_REFBLOCK_ALLOC_HOOKUP]          = "refblock_alloc.hookup",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE]           = "refblock_alloc.write",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS]    = "refblock_alloc.write_blocks",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE]     = "refblock_alloc.write_table",
-    [BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE]    = "refblock_alloc.switch_table",
-
-    [BLKDBG_CLUSTER_ALLOC]                  = "cluster_alloc",
-    [BLKDBG_CLUSTER_ALLOC_BYTES]            = "cluster_alloc_bytes",
-    [BLKDBG_CLUSTER_FREE]                   = "cluster_free",
-
-    [BLKDBG_FLUSH_TO_OS]                    = "flush_to_os",
-    [BLKDBG_FLUSH_TO_DISK]                  = "flush_to_disk",
-
-    [BLKDBG_PWRITEV_RMW_HEAD]               = "pwritev_rmw.head",
-    [BLKDBG_PWRITEV_RMW_AFTER_HEAD]         = "pwritev_rmw.after_head",
-    [BLKDBG_PWRITEV_RMW_TAIL]               = "pwritev_rmw.tail",
-    [BLKDBG_PWRITEV_RMW_AFTER_TAIL]         = "pwritev_rmw.after_tail",
-    [BLKDBG_PWRITEV]                        = "pwritev",
-    [BLKDBG_PWRITEV_ZERO]                   = "pwritev_zero",
-    [BLKDBG_PWRITEV_DONE]                   = "pwritev_done",
-
-    [BLKDBG_EMPTY_IMAGE_PREPARE]            = "empty_image_prepare",
-};
-
-static int get_event_by_name(const char *name, BlkDebugEvent *event)
+static int get_event_by_name(const char *name, BlkdebugEvent *event)
 {
     int i;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
-        if (!strcmp(event_names[i], name)) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
+        if (!strcmp(BlkdebugEvent_lookup[i], name)) {
             *event = i;
             return 0;
         }
@@ -223,7 +169,7 @@ static int add_rule(void *opaque, QemuOpts *opts, Error **errp)
     struct add_rule_data *d = opaque;
     BDRVBlkdebugState *s = d->s;
     const char* event_name;
-    BlkDebugEvent event;
+    BlkdebugEvent event;
     struct BlkdebugRule *rule;
 
     /* Find the right event for the rule */
@@ -426,11 +372,11 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
     /* Set initial state */
     s->state = 1;
 
-    /* Open the backing file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-image"), options, "image",
-                          bs, &child_file, false, &local_err);
-    if (ret < 0) {
+    /* Open the image file */
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options, "image",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto out;
     }
@@ -449,7 +395,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,
     goto out;
 
 fail_unref:
-    bdrv_unref(bs->file);
+    bdrv_unref_child(bs, bs->file);
 out:
     qemu_opts_del(opts);
     return ret;
@@ -510,7 +456,8 @@ static BlockAIOCB *blkdebug_aio_readv(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_readv(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_readv(bs->file->bs, sector_num, qiov, nb_sectors,
+                          cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
@@ -532,7 +479,8 @@ static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_writev(bs->file->bs, sector_num, qiov, nb_sectors,
+                           cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
@@ -551,7 +499,7 @@ static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_flush(bs->file, cb, opaque);
+    return bdrv_aio_flush(bs->file->bs, cb, opaque);
 }
 
 
@@ -561,7 +509,7 @@ static void blkdebug_close(BlockDriverState *bs)
     BlkdebugRule *rule, *next;
     int i;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
             remove_rule(rule);
         }
@@ -581,9 +529,13 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)
     remove_rule(rule);
     QLIST_INSERT_HEAD(&s->suspended_reqs, &r, next);
 
-    printf("blkdebug: Suspended request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Suspended request '%s'\n", r.tag);
+    }
     qemu_coroutine_yield();
-    printf("blkdebug: Resuming request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Resuming request '%s'\n", r.tag);
+    }
 
     QLIST_REMOVE(&r, next);
     g_free(r.tag);
@@ -620,13 +572,13 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
     return injected;
 }
 
-static void blkdebug_debug_event(BlockDriverState *bs, BlkDebugEvent event)
+static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event)
 {
     BDRVBlkdebugState *s = bs->opaque;
     struct BlkdebugRule *rule, *next;
     bool injected;
 
-    assert((int)event >= 0 && event < BLKDBG_EVENT_MAX);
+    assert((int)event >= 0 && event < BLKDBG__MAX);
 
     injected = false;
     s->new_state = s->state;
@@ -641,7 +593,7 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,
 {
     BDRVBlkdebugState *s = bs->opaque;
     struct BlkdebugRule *rule;
-    BlkDebugEvent blkdebug_event;
+    BlkdebugEvent blkdebug_event;
 
     if (get_event_by_name(event, &blkdebug_event) < 0) {
         return -ENOENT;
@@ -683,7 +635,7 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,
     BlkdebugRule *rule, *next;
     int i, ret = -ENOENT;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
             if (rule->action == ACTION_SUSPEND &&
                 !strcmp(rule->options.suspend.tag, tag)) {
@@ -716,55 +668,50 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)
 
 static int64_t blkdebug_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
 static int blkdebug_truncate(BlockDriverState *bs, int64_t offset)
 {
-    return bdrv_truncate(bs->file, offset);
+    return bdrv_truncate(bs->file->bs, offset);
 }
 
-static void blkdebug_refresh_filename(BlockDriverState *bs)
+static void blkdebug_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     QDict *opts;
     const QDictEntry *e;
     bool force_json = false;
 
-    for (e = qdict_first(bs->options); e; e = qdict_next(bs->options, e)) {
+    for (e = qdict_first(options); e; e = qdict_next(options, e)) {
         if (strcmp(qdict_entry_key(e), "config") &&
-            strcmp(qdict_entry_key(e), "x-image") &&
-            strcmp(qdict_entry_key(e), "image") &&
-            strncmp(qdict_entry_key(e), "image.", strlen("image.")))
+            strcmp(qdict_entry_key(e), "x-image"))
         {
             force_json = true;
             break;
         }
     }
 
-    if (force_json && !bs->file->full_open_options) {
+    if (force_json && !bs->file->bs->full_open_options) {
         /* The config file cannot be recreated, so creating a plain filename
          * is impossible */
         return;
     }
 
-    if (!force_json && bs->file->exact_filename[0]) {
+    if (!force_json && bs->file->bs->exact_filename[0]) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
                  "blkdebug:%s:%s",
-                 qdict_get_try_str(bs->options, "config") ?: "",
-                 bs->file->exact_filename);
+                 qdict_get_try_str(options, "config") ?: "",
+                 bs->file->bs->exact_filename);
     }
 
     opts = qdict_new();
     qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkdebug")));
 
-    QINCREF(bs->file->full_open_options);
-    qdict_put_obj(opts, "image", QOBJECT(bs->file->full_open_options));
+    QINCREF(bs->file->bs->full_open_options);
+    qdict_put_obj(opts, "image", QOBJECT(bs->file->bs->full_open_options));
 
-    for (e = qdict_first(bs->options); e; e = qdict_next(bs->options, e)) {
-        if (strcmp(qdict_entry_key(e), "x-image") &&
-            strcmp(qdict_entry_key(e), "image") &&
-            strncmp(qdict_entry_key(e), "image.", strlen("image.")))
-        {
+    for (e = qdict_first(options); e; e = qdict_next(options, e)) {
+        if (strcmp(qdict_entry_key(e), "x-image")) {
             qobject_incref(qdict_entry_value(e));
             qdict_put_obj(opts, qdict_entry_key(e), qdict_entry_value(e));
         }
@@ -773,6 +720,12 @@ static void blkdebug_refresh_filename(BlockDriverState *bs)
     bs->full_open_options = opts;
 }
 
+static int blkdebug_reopen_prepare(BDRVReopenState *reopen_state,
+                                   BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static BlockDriver bdrv_blkdebug = {
     .format_name            = "blkdebug",
     .protocol_name          = "blkdebug",
@@ -781,6 +734,7 @@ static BlockDriver bdrv_blkdebug = {
     .bdrv_parse_filename    = blkdebug_parse_filename,
     .bdrv_file_open         = blkdebug_open,
     .bdrv_close             = blkdebug_close,
+    .bdrv_reopen_prepare    = blkdebug_reopen_prepare,
     .bdrv_getlength         = blkdebug_getlength,
     .bdrv_truncate          = blkdebug_truncate,
     .bdrv_refresh_filename  = blkdebug_refresh_filename,
diff --git a/qemu/block/blkreplay.c b/qemu/block/blkreplay.c
new file mode 100755
index 000000000..42f1813af
--- /dev/null
+++ b/qemu/block/blkreplay.c
@@ -0,0 +1,160 @@
+/*
+ * Block protocol for record/replay
+ *
+ * Copyright (c) 2010-2016 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "sysemu/replay.h"
+#include "qapi/error.h"
+
+typedef struct Request {
+    Coroutine *co;
+    QEMUBH *bh;
+} Request;
+
+/* Next request id.
+   This counter is global, because requests from different
+   block devices should not get overlapping ids. */
+static uint64_t request_id;
+
+static int blkreplay_open(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp)
+{
+    Error *local_err = NULL;
+    int ret;
+
+    /* Open the image file */
+    bs->file = bdrv_open_child(NULL, options, "image",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    ret = 0;
+fail:
+    if (ret < 0) {
+        bdrv_unref_child(bs, bs->file);
+    }
+    return ret;
+}
+
+static void blkreplay_close(BlockDriverState *bs)
+{
+}
+
+static int64_t blkreplay_getlength(BlockDriverState *bs)
+{
+    return bdrv_getlength(bs->file->bs);
+}
+
+/* This bh is used for synchronization of return from coroutines.
+   It continues yielded coroutine which then finishes its execution.
+   BH is called adjusted to some replay checkpoint, therefore
+   record and replay will always finish coroutines deterministically.
+*/
+static void blkreplay_bh_cb(void *opaque)
+{
+    Request *req = opaque;
+    qemu_coroutine_enter(req->co, NULL);
+    qemu_bh_delete(req->bh);
+    g_free(req);
+}
+
+static void block_request_create(uint64_t reqid, BlockDriverState *bs,
+                                 Coroutine *co)
+{
+    Request *req = g_new(Request, 1);
+    *req = (Request) {
+        .co = co,
+        .bh = aio_bh_new(bdrv_get_aio_context(bs), blkreplay_bh_cb, req),
+    };
+    replay_block_event(req->bh, reqid);
+}
+
+static int coroutine_fn blkreplay_co_readv(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_readv(bs->file->bs, sector_num, nb_sectors, qiov);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_writev(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_writev(bs->file->bs, sector_num, nb_sectors, qiov);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_write_zeroes(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_write_zeroes(bs->file->bs, sector_num, nb_sectors, flags);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_discard(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_discard(bs->file->bs, sector_num, nb_sectors);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_flush(BlockDriverState *bs)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_flush(bs->file->bs);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static BlockDriver bdrv_blkreplay = {
+    .format_name            = "blkreplay",
+    .protocol_name          = "blkreplay",
+    .instance_size          = 0,
+
+    .bdrv_file_open         = blkreplay_open,
+    .bdrv_close             = blkreplay_close,
+    .bdrv_getlength         = blkreplay_getlength,
+
+    .bdrv_co_readv          = blkreplay_co_readv,
+    .bdrv_co_writev         = blkreplay_co_writev,
+
+    .bdrv_co_write_zeroes   = blkreplay_co_write_zeroes,
+    .bdrv_co_discard        = blkreplay_co_discard,
+    .bdrv_co_flush          = blkreplay_co_flush,
+};
+
+static void bdrv_blkreplay_init(void)
+{
+    bdrv_register(&bdrv_blkreplay);
+}
+
+block_init(bdrv_blkreplay_init);
diff --git a/qemu/block/blkverify.c b/qemu/block/blkverify.c
index d277e6322..9414b7a84 100644
--- a/qemu/block/blkverify.c
+++ b/qemu/block/blkverify.c
@@ -7,14 +7,16 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdarg.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "block/block_int.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
+#include "qemu/cutils.h"
 
 typedef struct {
-    BlockDriverState *test_file;
+    BdrvChild *test_file;
 } BDRVBlkverifyState;
 
 typedef struct BlkverifyAIOCB BlkverifyAIOCB;
@@ -123,26 +125,29 @@ static int blkverify_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Open the raw file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-raw"), options,
-                          "raw", bs, &child_file, false, &local_err);
-    if (ret < 0) {
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-raw"), options, "raw",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto fail;
     }
 
     /* Open the test file */
-    assert(s->test_file == NULL);
-    ret = bdrv_open_image(&s->test_file, qemu_opt_get(opts, "x-image"), options,
-                          "test", bs, &child_format, false, &local_err);
-    if (ret < 0) {
+    s->test_file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options,
+                                   "test", bs, &child_format, false,
+                                   &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
-        s->test_file = NULL;
         goto fail;
     }
 
     ret = 0;
 fail:
+    if (ret < 0) {
+        bdrv_unref_child(bs, bs->file);
+    }
     qemu_opts_del(opts);
     return ret;
 }
@@ -151,7 +156,7 @@ static void blkverify_close(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_unref(s->test_file);
+    bdrv_unref_child(bs, s->test_file);
     s->test_file = NULL;
 }
 
@@ -159,7 +164,7 @@ static int64_t blkverify_getlength(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    return bdrv_getlength(s->test_file);
+    return bdrv_getlength(s->test_file->bs);
 }
 
 static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
@@ -238,13 +243,13 @@ static BlockAIOCB *blkverify_aio_readv(BlockDriverState *bs,
                                             nb_sectors, cb, opaque);
 
     acb->verify = blkverify_verify_readv;
-    acb->buf = qemu_blockalign(bs->file, qiov->size);
+    acb->buf = qemu_blockalign(bs->file->bs, qiov->size);
     qemu_iovec_init(&acb->raw_qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->raw_qiov, qiov, acb->buf);
 
-    bdrv_aio_readv(s->test_file, sector_num, qiov, nb_sectors,
+    bdrv_aio_readv(s->test_file->bs, sector_num, qiov, nb_sectors,
                    blkverify_aio_cb, acb);
-    bdrv_aio_readv(bs->file, sector_num, &acb->raw_qiov, nb_sectors,
+    bdrv_aio_readv(bs->file->bs, sector_num, &acb->raw_qiov, nb_sectors,
                    blkverify_aio_cb, acb);
     return &acb->common;
 }
@@ -257,9 +262,9 @@ static BlockAIOCB *blkverify_aio_writev(BlockDriverState *bs,
     BlkverifyAIOCB *acb = blkverify_aio_get(bs, true, sector_num, qiov,
                                             nb_sectors, cb, opaque);
 
-    bdrv_aio_writev(s->test_file, sector_num, qiov, nb_sectors,
+    bdrv_aio_writev(s->test_file->bs, sector_num, qiov, nb_sectors,
                     blkverify_aio_cb, acb);
-    bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors,
+    bdrv_aio_writev(bs->file->bs, sector_num, qiov, nb_sectors,
                     blkverify_aio_cb, acb);
     return &acb->common;
 }
@@ -271,7 +276,7 @@ static BlockAIOCB *blkverify_aio_flush(BlockDriverState *bs,
     BDRVBlkverifyState *s = bs->opaque;
 
     /* Only flush test file, the raw file is not important */
-    return bdrv_aio_flush(s->test_file, cb, opaque);
+    return bdrv_aio_flush(s->test_file->bs, cb, opaque);
 }
 
 static bool blkverify_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -279,13 +284,13 @@ static bool blkverify_recurse_is_first_non_filter(BlockDriverState *bs,
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bool perm = bdrv_recurse_is_first_non_filter(bs->file, candidate);
+    bool perm = bdrv_recurse_is_first_non_filter(bs->file->bs, candidate);
 
     if (perm) {
         return true;
     }
 
-    return bdrv_recurse_is_first_non_filter(s->test_file, candidate);
+    return bdrv_recurse_is_first_non_filter(s->test_file->bs, candidate);
 }
 
 /* Propagate AioContext changes to ->test_file */
@@ -293,7 +298,7 @@ static void blkverify_detach_aio_context(BlockDriverState *bs)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_detach_aio_context(s->test_file);
+    bdrv_detach_aio_context(s->test_file->bs);
 }
 
 static void blkverify_attach_aio_context(BlockDriverState *bs,
@@ -301,32 +306,38 @@ static void blkverify_attach_aio_context(BlockDriverState *bs,
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_attach_aio_context(s->test_file, new_context);
+    bdrv_attach_aio_context(s->test_file->bs, new_context);
 }
 
-static void blkverify_refresh_filename(BlockDriverState *bs)
+static void blkverify_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    /* bs->file has already been refreshed */
-    bdrv_refresh_filename(s->test_file);
+    /* bs->file->bs has already been refreshed */
+    bdrv_refresh_filename(s->test_file->bs);
 
-    if (bs->file->full_open_options && s->test_file->full_open_options) {
+    if (bs->file->bs->full_open_options
+        && s->test_file->bs->full_open_options)
+    {
         QDict *opts = qdict_new();
         qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkverify")));
 
-        QINCREF(bs->file->full_open_options);
-        qdict_put_obj(opts, "raw", QOBJECT(bs->file->full_open_options));
-        QINCREF(s->test_file->full_open_options);
-        qdict_put_obj(opts, "test", QOBJECT(s->test_file->full_open_options));
+        QINCREF(bs->file->bs->full_open_options);
+        qdict_put_obj(opts, "raw", QOBJECT(bs->file->bs->full_open_options));
+        QINCREF(s->test_file->bs->full_open_options);
+        qdict_put_obj(opts, "test",
+                      QOBJECT(s->test_file->bs->full_open_options));
 
         bs->full_open_options = opts;
     }
 
-    if (bs->file->exact_filename[0] && s->test_file->exact_filename[0]) {
+    if (bs->file->bs->exact_filename[0]
+        && s->test_file->bs->exact_filename[0])
+    {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
                  "blkverify:%s:%s",
-                 bs->file->exact_filename, s->test_file->exact_filename);
+                 bs->file->bs->exact_filename,
+                 s->test_file->bs->exact_filename);
     }
 }
 
diff --git a/qemu/block/block-backend.c b/qemu/block/block-backend.c
index aee8a1202..16c9d5e0f 100644
--- a/qemu/block/block-backend.c
+++ b/qemu/block/block-backend.c
@@ -10,74 +10,105 @@
  * or later.  See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
+#include "block/blockjob.h"
+#include "block/throttle-groups.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
 #include "qapi-event.h"
+#include "qemu/id.h"
 
 /* Number of coroutines to reserve per attached device model */
 #define COROUTINE_POOL_RESERVATION 64
 
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb);
+
 struct BlockBackend {
     char *name;
     int refcnt;
-    BlockDriverState *bs;
+    BdrvChild *root;
     DriveInfo *legacy_dinfo;    /* null unless created by drive_new() */
-    QTAILQ_ENTRY(BlockBackend) link; /* for blk_backends */
+    QTAILQ_ENTRY(BlockBackend) link;         /* for block_backends */
+    QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */
 
     void *dev;                  /* attached device model, if any */
     /* TODO change to DeviceState when all users are qdevified */
     const BlockDevOps *dev_ops;
     void *dev_opaque;
+
+    /* the block size for which the guest device expects atomicity */
+    int guest_block_size;
+
+    /* If the BDS tree is removed, some of its options are stored here (which
+     * can be used to restore those options in the new BDS on insert) */
+    BlockBackendRootState root_state;
+
+    bool enable_write_cache;
+
+    /* I/O stats (display with "info blockstats"). */
+    BlockAcctStats stats;
+
+    BlockdevOnError on_read_error, on_write_error;
+    bool iostatus_enabled;
+    BlockDeviceIoStatus iostatus;
+
+    bool allow_write_beyond_eof;
+
+    NotifierList remove_bs_notifiers, insert_bs_notifiers;
 };
 
 typedef struct BlockBackendAIOCB {
     BlockAIOCB common;
     QEMUBH *bh;
+    BlockBackend *blk;
     int ret;
 } BlockBackendAIOCB;
 
 static const AIOCBInfo block_backend_aiocb_info = {
+    .get_aio_context = blk_aiocb_get_aio_context,
     .aiocb_size = sizeof(BlockBackendAIOCB),
 };
 
 static void drive_info_del(DriveInfo *dinfo);
 
-/* All the BlockBackends (except for hidden ones) */
-static QTAILQ_HEAD(, BlockBackend) blk_backends =
-    QTAILQ_HEAD_INITIALIZER(blk_backends);
+/* All BlockBackends */
+static QTAILQ_HEAD(, BlockBackend) block_backends =
+    QTAILQ_HEAD_INITIALIZER(block_backends);
+
+/* All BlockBackends referenced by the monitor and which are iterated through by
+ * blk_next() */
+static QTAILQ_HEAD(, BlockBackend) monitor_block_backends =
+    QTAILQ_HEAD_INITIALIZER(monitor_block_backends);
+
+static void blk_root_inherit_options(int *child_flags, QDict *child_options,
+                                     int parent_flags, QDict *parent_options)
+{
+    /* We're not supposed to call this function for root nodes */
+    abort();
+}
+
+static const BdrvChildRole child_root = {
+    .inherit_options = blk_root_inherit_options,
+};
 
 /*
- * Create a new BlockBackend with @name, with a reference count of one.
- * @name must not be null or empty.
- * Fail if a BlockBackend with this name already exists.
+ * Create a new BlockBackend with a reference count of one.
  * Store an error through @errp on failure, unless it's null.
  * Return the new BlockBackend on success, null on failure.
  */
-BlockBackend *blk_new(const char *name, Error **errp)
+BlockBackend *blk_new(Error **errp)
 {
     BlockBackend *blk;
 
-    assert(name && name[0]);
-    if (!id_wellformed(name)) {
-        error_setg(errp, "Invalid device name");
-        return NULL;
-    }
-    if (blk_by_name(name)) {
-        error_setg(errp, "Device with id '%s' already exists", name);
-        return NULL;
-    }
-    if (bdrv_find_node(name)) {
-        error_setg(errp,
-                   "Device name '%s' conflicts with an existing node name",
-                   name);
-        return NULL;
-    }
-
     blk = g_new0(BlockBackend, 1);
-    blk->name = g_strdup(name);
     blk->refcnt = 1;
-    QTAILQ_INSERT_TAIL(&blk_backends, blk, link);
+    notifier_list_init(&blk->remove_bs_notifiers);
+    notifier_list_init(&blk->insert_bs_notifiers);
+    QTAILQ_INSERT_TAIL(&block_backends, blk, link);
     return blk;
 }
 
@@ -85,18 +116,18 @@ BlockBackend *blk_new(const char *name, Error **errp)
  * Create a new BlockBackend with a new BlockDriverState attached.
  * Otherwise just like blk_new(), which see.
  */
-BlockBackend *blk_new_with_bs(const char *name, Error **errp)
+BlockBackend *blk_new_with_bs(Error **errp)
 {
     BlockBackend *blk;
     BlockDriverState *bs;
 
-    blk = blk_new(name, errp);
+    blk = blk_new(errp);
     if (!blk) {
         return NULL;
     }
 
     bs = bdrv_new_root();
-    blk->bs = bs;
+    blk->root = bdrv_root_attach_child(bs, "root", &child_root);
     bs->blk = blk;
     return blk;
 }
@@ -113,44 +144,46 @@ BlockBackend *blk_new_with_bs(const char *name, Error **errp)
  * though, so callers of this function have to be able to specify @filename and
  * @flags.
  */
-BlockBackend *blk_new_open(const char *name, const char *filename,
-                           const char *reference, QDict *options, int flags,
-                           Error **errp)
+BlockBackend *blk_new_open(const char *filename, const char *reference,
+                           QDict *options, int flags, Error **errp)
 {
     BlockBackend *blk;
     int ret;
 
-    blk = blk_new_with_bs(name, errp);
+    blk = blk_new_with_bs(errp);
     if (!blk) {
         QDECREF(options);
         return NULL;
     }
 
-    ret = bdrv_open(&blk->bs, filename, reference, options, flags, NULL, errp);
+    ret = bdrv_open(&blk->root->bs, filename, reference, options, flags, errp);
     if (ret < 0) {
         blk_unref(blk);
         return NULL;
     }
 
+    blk_set_enable_write_cache(blk, true);
+
     return blk;
 }
 
 static void blk_delete(BlockBackend *blk)
 {
     assert(!blk->refcnt);
+    assert(!blk->name);
     assert(!blk->dev);
-    if (blk->bs) {
-        assert(blk->bs->blk == blk);
-        blk->bs->blk = NULL;
-        bdrv_unref(blk->bs);
-        blk->bs = NULL;
+    if (blk->root) {
+        blk_remove_bs(blk);
     }
-    /* Avoid double-remove after blk_hide_on_behalf_of_hmp_drive_del() */
-    if (blk->name[0]) {
-        QTAILQ_REMOVE(&blk_backends, blk, link);
+    assert(QLIST_EMPTY(&blk->remove_bs_notifiers.notifiers));
+    assert(QLIST_EMPTY(&blk->insert_bs_notifiers.notifiers));
+    if (blk->root_state.throttle_state) {
+        g_free(blk->root_state.throttle_group);
+        throttle_group_unref(blk->root_state.throttle_state);
     }
-    g_free(blk->name);
+    QTAILQ_REMOVE(&block_backends, blk, link);
     drive_info_del(blk->legacy_dinfo);
+    block_acct_cleanup(&blk->stats);
     g_free(blk);
 }
 
@@ -164,6 +197,11 @@ static void drive_info_del(DriveInfo *dinfo)
     g_free(dinfo);
 }
 
+int blk_get_refcnt(BlockBackend *blk)
+{
+    return blk ? blk->refcnt : 0;
+}
+
 /*
  * Increment @blk's reference count.
  * @blk must not be null.
@@ -189,7 +227,32 @@ void blk_unref(BlockBackend *blk)
 }
 
 /*
- * Return the BlockBackend after @blk.
+ * Behaves similarly to blk_next() but iterates over all BlockBackends, even the
+ * ones which are hidden (i.e. are not referenced by the monitor).
+ */
+static BlockBackend *blk_all_next(BlockBackend *blk)
+{
+    return blk ? QTAILQ_NEXT(blk, link)
+               : QTAILQ_FIRST(&block_backends);
+}
+
+void blk_remove_all_bs(void)
+{
+    BlockBackend *blk = NULL;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *ctx = blk_get_aio_context(blk);
+
+        aio_context_acquire(ctx);
+        if (blk->root) {
+            blk_remove_bs(blk);
+        }
+        aio_context_release(ctx);
+    }
+}
+
+/*
+ * Return the monitor-owned BlockBackend after @blk.
  * If @blk is null, return the first one.
  * Else, return @blk's next sibling, which may be null.
  *
@@ -200,17 +263,91 @@ void blk_unref(BlockBackend *blk)
  */
 BlockBackend *blk_next(BlockBackend *blk)
 {
-    return blk ? QTAILQ_NEXT(blk, link) : QTAILQ_FIRST(&blk_backends);
+    return blk ? QTAILQ_NEXT(blk, monitor_link)
+               : QTAILQ_FIRST(&monitor_block_backends);
+}
+
+/*
+ * Iterates over all BlockDriverStates which are attached to a BlockBackend.
+ * This function is for use by bdrv_next().
+ *
+ * @bs must be NULL or a BDS that is attached to a BB.
+ */
+BlockDriverState *blk_next_root_bs(BlockDriverState *bs)
+{
+    BlockBackend *blk;
+
+    if (bs) {
+        assert(bs->blk);
+        blk = bs->blk;
+    } else {
+        blk = NULL;
+    }
+
+    do {
+        blk = blk_all_next(blk);
+    } while (blk && !blk->root);
+
+    return blk ? blk->root->bs : NULL;
+}
+
+/*
+ * Add a BlockBackend into the list of backends referenced by the monitor, with
+ * the given @name acting as the handle for the monitor.
+ * Strictly for use by blockdev.c.
+ *
+ * @name must not be null or empty.
+ *
+ * Returns true on success and false on failure. In the latter case, an Error
+ * object is returned through @errp.
+ */
+bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp)
+{
+    assert(!blk->name);
+    assert(name && name[0]);
+
+    if (!id_wellformed(name)) {
+        error_setg(errp, "Invalid device name");
+        return false;
+    }
+    if (blk_by_name(name)) {
+        error_setg(errp, "Device with id '%s' already exists", name);
+        return false;
+    }
+    if (bdrv_find_node(name)) {
+        error_setg(errp,
+                   "Device name '%s' conflicts with an existing node name",
+                   name);
+        return false;
+    }
+
+    blk->name = g_strdup(name);
+    QTAILQ_INSERT_TAIL(&monitor_block_backends, blk, monitor_link);
+    return true;
+}
+
+/*
+ * Remove a BlockBackend from the list of backends referenced by the monitor.
+ * Strictly for use by blockdev.c.
+ */
+void monitor_remove_blk(BlockBackend *blk)
+{
+    if (!blk->name) {
+        return;
+    }
+
+    QTAILQ_REMOVE(&monitor_block_backends, blk, monitor_link);
+    g_free(blk->name);
+    blk->name = NULL;
 }
 
 /*
  * Return @blk's name, a non-null string.
- * Wart: the name is empty iff @blk has been hidden with
- * blk_hide_on_behalf_of_hmp_drive_del().
+ * Returns an empty string iff @blk is not referenced by the monitor.
  */
 const char *blk_name(BlockBackend *blk)
 {
-    return blk->name;
+    return blk->name ?: "";
 }
 
 /*
@@ -219,10 +356,10 @@ const char *blk_name(BlockBackend *blk)
  */
 BlockBackend *blk_by_name(const char *name)
 {
-    BlockBackend *blk;
+    BlockBackend *blk = NULL;
 
     assert(name);
-    QTAILQ_FOREACH(blk, &blk_backends, link) {
+    while ((blk = blk_next(blk)) != NULL) {
         if (!strcmp(name, blk->name)) {
             return blk;
         }
@@ -235,7 +372,7 @@ BlockBackend *blk_by_name(const char *name)
  */
 BlockDriverState *blk_bs(BlockBackend *blk)
 {
-    return blk->bs;
+    return blk->root ? blk->root->bs : NULL;
 }
 
 /*
@@ -263,9 +400,9 @@ DriveInfo *blk_set_legacy_dinfo(BlockBackend *blk, DriveInfo *dinfo)
  */
 BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
 {
-    BlockBackend *blk;
+    BlockBackend *blk = NULL;
 
-    QTAILQ_FOREACH(blk, &blk_backends, link) {
+    while ((blk = blk_next(blk)) != NULL) {
         if (blk->legacy_dinfo == dinfo) {
             return blk;
         }
@@ -274,21 +411,32 @@ BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
 }
 
 /*
- * Hide @blk.
- * @blk must not have been hidden already.
- * Make attached BlockDriverState, if any, anonymous.
- * Once hidden, @blk is invisible to all functions that don't receive
- * it as argument.  For example, blk_by_name() won't return it.
- * Strictly for use by do_drive_del().
- * TODO get rid of it!
+ * Disassociates the currently associated BlockDriverState from @blk.
  */
-void blk_hide_on_behalf_of_hmp_drive_del(BlockBackend *blk)
+void blk_remove_bs(BlockBackend *blk)
 {
-    QTAILQ_REMOVE(&blk_backends, blk, link);
-    blk->name[0] = 0;
-    if (blk->bs) {
-        bdrv_make_anon(blk->bs);
-    }
+    assert(blk->root->bs->blk == blk);
+
+    notifier_list_notify(&blk->remove_bs_notifiers, blk);
+
+    blk_update_root_state(blk);
+
+    blk->root->bs->blk = NULL;
+    bdrv_root_unref_child(blk->root);
+    blk->root = NULL;
+}
+
+/*
+ * Associates a new BlockDriverState with @blk.
+ */
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs)
+{
+    assert(!blk->root && !bs->blk);
+    bdrv_ref(bs);
+    blk->root = bdrv_root_attach_child(bs, "root", &child_root);
+    bs->blk = blk;
+
+    notifier_list_notify(&blk->insert_bs_notifiers, blk);
 }
 
 /*
@@ -303,7 +451,7 @@ int blk_attach_dev(BlockBackend *blk, void *dev)
     }
     blk_ref(blk);
     blk->dev = dev;
-    bdrv_iostatus_reset(blk->bs);
+    blk_iostatus_reset(blk);
     return 0;
 }
 
@@ -330,7 +478,7 @@ void blk_detach_dev(BlockBackend *blk, void *dev)
     blk->dev = NULL;
     blk->dev_ops = NULL;
     blk->dev_opaque = NULL;
-    bdrv_set_guest_block_size(blk->bs, 512);
+    blk->guest_block_size = 512;
     blk_unref(blk);
 }
 
@@ -364,18 +512,15 @@ void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops,
 void blk_dev_change_media_cb(BlockBackend *blk, bool load)
 {
     if (blk->dev_ops && blk->dev_ops->change_media_cb) {
-        bool tray_was_closed = !blk_dev_is_tray_open(blk);
+        bool tray_was_open, tray_is_open;
 
+        tray_was_open = blk_dev_is_tray_open(blk);
         blk->dev_ops->change_media_cb(blk->dev_opaque, load);
-        if (tray_was_closed) {
-            /* tray open */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              true, &error_abort);
-        }
-        if (load) {
-            /* tray close */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              false, &error_abort);
+        tray_is_open = blk_dev_is_tray_open(blk);
+
+        if (tray_was_open != tray_is_open) {
+            qapi_event_send_device_tray_moved(blk_name(blk), tray_is_open,
+                                              &error_abort);
         }
     }
 }
@@ -390,6 +535,14 @@ bool blk_dev_has_removable_media(BlockBackend *blk)
 }
 
 /*
+ * Does @blk's attached device model have a tray?
+ */
+bool blk_dev_has_tray(BlockBackend *blk)
+{
+    return blk->dev_ops && blk->dev_ops->is_tray_open;
+}
+
+/*
  * Notify @blk's attached device model of a media eject request.
  * If @force is true, the medium is about to be yanked out forcefully.
  */
@@ -405,7 +558,7 @@ void blk_dev_eject_request(BlockBackend *blk, bool force)
  */
 bool blk_dev_is_tray_open(BlockBackend *blk)
 {
-    if (blk->dev_ops && blk->dev_ops->is_tray_open) {
+    if (blk_dev_has_tray(blk)) {
         return blk->dev_ops->is_tray_open(blk->dev_opaque);
     }
     return false;
@@ -435,7 +588,53 @@ void blk_dev_resize_cb(BlockBackend *blk)
 
 void blk_iostatus_enable(BlockBackend *blk)
 {
-    bdrv_iostatus_enable(blk->bs);
+    blk->iostatus_enabled = true;
+    blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+}
+
+/* The I/O status is only enabled if the drive explicitly
+ * enables it _and_ the VM is configured to stop on errors */
+bool blk_iostatus_is_enabled(const BlockBackend *blk)
+{
+    return (blk->iostatus_enabled &&
+           (blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            blk->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
+}
+
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
+{
+    return blk->iostatus;
+}
+
+void blk_iostatus_disable(BlockBackend *blk)
+{
+    blk->iostatus_enabled = false;
+}
+
+void blk_iostatus_reset(BlockBackend *blk)
+{
+    if (blk_iostatus_is_enabled(blk)) {
+        BlockDriverState *bs = blk_bs(blk);
+        blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (bs && bs->job) {
+            block_job_iostatus_reset(bs->job);
+        }
+    }
+}
+
+void blk_iostatus_set_err(BlockBackend *blk, int error)
+{
+    assert(blk_iostatus_is_enabled(blk));
+    if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                          BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
+}
+
+void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow)
+{
+    blk->allow_write_beyond_eof = allow;
 }
 
 static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
@@ -447,21 +646,23 @@ static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
         return -EIO;
     }
 
-    if (!blk_is_inserted(blk)) {
+    if (!blk_is_available(blk)) {
         return -ENOMEDIUM;
     }
 
-    len = blk_getlength(blk);
-    if (len < 0) {
-        return len;
-    }
-
     if (offset < 0) {
         return -EIO;
     }
 
-    if (offset > len || len - offset < size) {
-        return -EIO;
+    if (!blk->allow_write_beyond_eof) {
+        len = blk_getlength(blk);
+        if (len < 0) {
+            return len;
+        }
+
+        if (offset > len || len - offset < size) {
+            return -EIO;
+        }
     }
 
     return 0;
@@ -482,48 +683,144 @@ static int blk_check_request(BlockBackend *blk, int64_t sector_num,
                                   nb_sectors * BDRV_SECTOR_SIZE);
 }
 
-int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-             int nb_sectors)
+static int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
+                                      unsigned int bytes, QEMUIOVector *qiov,
+                                      BdrvRequestFlags flags)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
+    int ret = blk_check_byte_request(blk, offset, bytes);
     if (ret < 0) {
         return ret;
     }
 
-    return bdrv_read(blk->bs, sector_num, buf, nb_sectors);
+    return bdrv_co_do_preadv(blk_bs(blk), offset, bytes, qiov, flags);
 }
 
-int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-                         int nb_sectors)
+static int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
+                                      unsigned int bytes, QEMUIOVector *qiov,
+                                      BdrvRequestFlags flags)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
+    int ret;
+
+    ret = blk_check_byte_request(blk, offset, bytes);
     if (ret < 0) {
         return ret;
     }
 
-    return bdrv_read_unthrottled(blk->bs, sector_num, buf, nb_sectors);
+    if (!blk->enable_write_cache) {
+        flags |= BDRV_REQ_FUA;
+    }
+
+    return bdrv_co_do_pwritev(blk_bs(blk), offset, bytes, qiov, flags);
 }
 
-int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
-              int nb_sectors)
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    QEMUIOVector *qiov;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
-    if (ret < 0) {
-        return ret;
+    BlkRwCo *rwco = opaque;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, rwco->qiov->size,
+                              rwco->qiov, rwco->flags);
+}
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, rwco->qiov->size,
+                               rwco->qiov, rwco->flags);
+}
+
+static int blk_prw(BlockBackend *blk, int64_t offset, uint8_t *buf,
+                   int64_t bytes, CoroutineEntry co_entry,
+                   BdrvRequestFlags flags)
+{
+    AioContext *aio_context;
+    QEMUIOVector qiov;
+    struct iovec iov;
+    Coroutine *co;
+    BlkRwCo rwco;
+
+    iov = (struct iovec) {
+        .iov_base = buf,
+        .iov_len = bytes,
+    };
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    rwco = (BlkRwCo) {
+        .blk    = blk,
+        .offset = offset,
+        .qiov   = &qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    co = qemu_coroutine_create(co_entry);
+    qemu_coroutine_enter(co, &rwco);
+
+    aio_context = blk_get_aio_context(blk);
+    while (rwco.ret == NOT_DONE) {
+        aio_poll(aio_context, true);
     }
 
-    return bdrv_write(blk->bs, sector_num, buf, nb_sectors);
+    return rwco.ret;
 }
 
-int blk_write_zeroes(BlockBackend *blk, int64_t sector_num,
-                     int nb_sectors, BdrvRequestFlags flags)
+static int blk_rw(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+                  int nb_sectors, CoroutineEntry co_entry,
+                  BdrvRequestFlags flags)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EINVAL;
+    }
+
+    return blk_prw(blk, sector_num << BDRV_SECTOR_BITS, buf,
+                   nb_sectors << BDRV_SECTOR_BITS, co_entry, flags);
+}
+
+int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+             int nb_sectors)
+{
+    return blk_rw(blk, sector_num, buf, nb_sectors, blk_read_entry, 0);
+}
+
+int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
+                         int nb_sectors)
+{
+    BlockDriverState *bs = blk_bs(blk);
+    bool enabled;
+    int ret;
+
+    ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
         return ret;
     }
 
-    return bdrv_write_zeroes(blk->bs, sector_num, nb_sectors, flags);
+    enabled = bs->io_limits_enabled;
+    bs->io_limits_enabled = false;
+    ret = blk_read(blk, sector_num, buf, nb_sectors);
+    bs->io_limits_enabled = enabled;
+    return ret;
+}
+
+int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
+              int nb_sectors)
+{
+    return blk_rw(blk, sector_num, (uint8_t*) buf, nb_sectors,
+                  blk_write_entry, 0);
+}
+
+int blk_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                     int nb_sectors, BdrvRequestFlags flags)
+{
+    return blk_rw(blk, sector_num, NULL, nb_sectors, blk_write_entry,
+                  flags | BDRV_REQ_ZERO_WRITE);
 }
 
 static void error_callback_bh(void *opaque)
@@ -534,13 +831,15 @@ static void error_callback_bh(void *opaque)
     qemu_aio_unref(acb);
 }
 
-static BlockAIOCB *abort_aio_request(BlockBackend *blk, BlockCompletionFunc *cb,
-                                     void *opaque, int ret)
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret)
 {
     struct BlockBackendAIOCB *acb;
     QEMUBH *bh;
 
     acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
+    acb->blk = blk;
     acb->ret = ret;
 
     bh = aio_bh_new(blk_get_aio_context(blk), error_callback_bh, acb);
@@ -550,82 +849,182 @@ static BlockAIOCB *abort_aio_request(BlockBackend *blk, BlockCompletionFunc *cb,
     return &acb->common;
 }
 
+typedef struct BlkAioEmAIOCB {
+    BlockAIOCB common;
+    BlkRwCo rwco;
+    int bytes;
+    bool has_returned;
+    QEMUBH* bh;
+} BlkAioEmAIOCB;
+
+static const AIOCBInfo blk_aio_em_aiocb_info = {
+    .aiocb_size         = sizeof(BlkAioEmAIOCB),
+};
+
+static void blk_aio_complete(BlkAioEmAIOCB *acb)
+{
+    if (acb->bh) {
+        assert(acb->has_returned);
+        qemu_bh_delete(acb->bh);
+    }
+    if (acb->has_returned) {
+        acb->common.cb(acb->common.opaque, acb->rwco.ret);
+        qemu_aio_unref(acb);
+    }
+}
+
+static void blk_aio_complete_bh(void *opaque)
+{
+    blk_aio_complete(opaque);
+}
+
+static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, int bytes,
+                                QEMUIOVector *qiov, CoroutineEntry co_entry,
+                                BdrvRequestFlags flags,
+                                BlockCompletionFunc *cb, void *opaque)
+{
+    BlkAioEmAIOCB *acb;
+    Coroutine *co;
+
+    acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
+    acb->rwco = (BlkRwCo) {
+        .blk    = blk,
+        .offset = offset,
+        .qiov   = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+    acb->bytes = bytes;
+    acb->bh = NULL;
+    acb->has_returned = false;
+
+    co = qemu_coroutine_create(co_entry);
+    qemu_coroutine_enter(co, acb);
+
+    acb->has_returned = true;
+    if (acb->rwco.ret != NOT_DONE) {
+        acb->bh = aio_bh_new(blk_get_aio_context(blk), blk_aio_complete_bh, acb);
+        qemu_bh_schedule(acb->bh);
+    }
+
+    return &acb->common;
+}
+
+static void blk_aio_read_entry(void *opaque)
+{
+    BlkAioEmAIOCB *acb = opaque;
+    BlkRwCo *rwco = &acb->rwco;
+
+    assert(rwco->qiov->size == acb->bytes);
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, acb->bytes,
+                              rwco->qiov, rwco->flags);
+    blk_aio_complete(acb);
+}
+
+static void blk_aio_write_entry(void *opaque)
+{
+    BlkAioEmAIOCB *acb = opaque;
+    BlkRwCo *rwco = &acb->rwco;
+
+    assert(!rwco->qiov || rwco->qiov->size == acb->bytes);
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, acb->bytes,
+                               rwco->qiov, rwco->flags);
+    blk_aio_complete(acb);
+}
+
 BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
                                  int nb_sectors, BdrvRequestFlags flags,
                                  BlockCompletionFunc *cb, void *opaque)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
-    if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return blk_abort_aio_request(blk, cb, opaque, -EINVAL);
     }
 
-    return bdrv_aio_write_zeroes(blk->bs, sector_num, nb_sectors, flags,
-                                 cb, opaque);
+    return blk_aio_prwv(blk, sector_num << BDRV_SECTOR_BITS,
+                        nb_sectors << BDRV_SECTOR_BITS, NULL,
+                        blk_aio_write_entry, flags | BDRV_REQ_ZERO_WRITE,
+                        cb, opaque);
 }
 
 int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count)
 {
-    int ret = blk_check_byte_request(blk, offset, count);
+    int ret = blk_prw(blk, offset, buf, count, blk_read_entry, 0);
     if (ret < 0) {
         return ret;
     }
-
-    return bdrv_pread(blk->bs, offset, buf, count);
+    return count;
 }
 
 int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count)
 {
-    int ret = blk_check_byte_request(blk, offset, count);
+    int ret = blk_prw(blk, offset, (void*) buf, count, blk_write_entry, 0);
     if (ret < 0) {
         return ret;
     }
-
-    return bdrv_pwrite(blk->bs, offset, buf, count);
+    return count;
 }
 
 int64_t blk_getlength(BlockBackend *blk)
 {
-    return bdrv_getlength(blk->bs);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_getlength(blk_bs(blk));
 }
 
 void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr)
 {
-    bdrv_get_geometry(blk->bs, nb_sectors_ptr);
+    if (!blk_bs(blk)) {
+        *nb_sectors_ptr = 0;
+    } else {
+        bdrv_get_geometry(blk_bs(blk), nb_sectors_ptr);
+    }
 }
 
 int64_t blk_nb_sectors(BlockBackend *blk)
 {
-    return bdrv_nb_sectors(blk->bs);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_nb_sectors(blk_bs(blk));
 }
 
 BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
                           QEMUIOVector *iov, int nb_sectors,
                           BlockCompletionFunc *cb, void *opaque)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
-    if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return blk_abort_aio_request(blk, cb, opaque, -EINVAL);
     }
 
-    return bdrv_aio_readv(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+    assert(nb_sectors << BDRV_SECTOR_BITS == iov->size);
+    return blk_aio_prwv(blk, sector_num << BDRV_SECTOR_BITS, iov->size, iov,
+                        blk_aio_read_entry, 0, cb, opaque);
 }
 
 BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
                            QEMUIOVector *iov, int nb_sectors,
                            BlockCompletionFunc *cb, void *opaque)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
-    if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return blk_abort_aio_request(blk, cb, opaque, -EINVAL);
     }
 
-    return bdrv_aio_writev(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+    assert(nb_sectors << BDRV_SECTOR_BITS == iov->size);
+    return blk_aio_prwv(blk, sector_num << BDRV_SECTOR_BITS, iov->size, iov,
+                        blk_aio_write_entry, 0, cb, opaque);
 }
 
 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
                           BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_flush(blk->bs, cb, opaque);
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
+    return bdrv_aio_flush(blk_bs(blk), cb, opaque);
 }
 
 BlockAIOCB *blk_aio_discard(BlockBackend *blk,
@@ -634,10 +1033,10 @@ BlockAIOCB *blk_aio_discard(BlockBackend *blk,
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
-        return abort_aio_request(blk, cb, opaque, ret);
+        return blk_abort_aio_request(blk, cb, opaque, ret);
     }
 
-    return bdrv_aio_discard(blk->bs, sector_num, nb_sectors, cb, opaque);
+    return bdrv_aio_discard(blk_bs(blk), sector_num, nb_sectors, cb, opaque);
 }
 
 void blk_aio_cancel(BlockAIOCB *acb)
@@ -661,18 +1060,26 @@ int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs)
         }
     }
 
-    return bdrv_aio_multiwrite(blk->bs, reqs, num_reqs);
+    return bdrv_aio_multiwrite(blk_bs(blk), reqs, num_reqs);
 }
 
 int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
 {
-    return bdrv_ioctl(blk->bs, req, buf);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_ioctl(blk_bs(blk), req, buf);
 }
 
 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
                           BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_ioctl(blk->bs, req, buf, cb, opaque);
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
+    return bdrv_aio_ioctl(blk_bs(blk), req, buf, cb, opaque);
 }
 
 int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
@@ -682,27 +1089,32 @@ int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
         return ret;
     }
 
-    return bdrv_co_discard(blk->bs, sector_num, nb_sectors);
+    return bdrv_co_discard(blk_bs(blk), sector_num, nb_sectors);
 }
 
 int blk_co_flush(BlockBackend *blk)
 {
-    return bdrv_co_flush(blk->bs);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_co_flush(blk_bs(blk));
 }
 
 int blk_flush(BlockBackend *blk)
 {
-    return bdrv_flush(blk->bs);
-}
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
 
-int blk_flush_all(void)
-{
-    return bdrv_flush_all();
+    return bdrv_flush(blk_bs(blk));
 }
 
 void blk_drain(BlockBackend *blk)
 {
-    bdrv_drain(blk->bs);
+    if (blk_bs(blk)) {
+        bdrv_drain(blk_bs(blk));
+    }
 }
 
 void blk_drain_all(void)
@@ -710,119 +1122,273 @@ void blk_drain_all(void)
     bdrv_drain_all();
 }
 
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error)
+{
+    blk->on_read_error = on_read_error;
+    blk->on_write_error = on_write_error;
+}
+
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
 {
-    return bdrv_get_on_error(blk->bs, is_read);
+    return is_read ? blk->on_read_error : blk->on_write_error;
 }
 
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error)
 {
-    return bdrv_get_error_action(blk->bs, is_read, error);
+    BlockdevOnError on_err = blk_get_on_error(blk, is_read);
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ?
+               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BLOCK_ERROR_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BLOCK_ERROR_ACTION_IGNORE;
+    default:
+        abort();
+    }
+}
+
+static void send_qmp_error_event(BlockBackend *blk,
+                                 BlockErrorAction action,
+                                 bool is_read, int error)
+{
+    IoOperationType optype;
+
+    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
+    qapi_event_send_block_io_error(blk_name(blk), optype, action,
+                                   blk_iostatus_is_enabled(blk),
+                                   error == ENOSPC, strerror(error),
+                                   &error_abort);
 }
 
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
 void blk_error_action(BlockBackend *blk, BlockErrorAction action,
                       bool is_read, int error)
 {
-    bdrv_error_action(blk->bs, action, is_read, error);
+    assert(error >= 0);
+
+    if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* First set the iostatus, so that "info block" returns an iostatus
+         * that matches the events raised so far (an additional error iostatus
+         * is fine, but not a lost one).
+         */
+        blk_iostatus_set_err(blk, error);
+
+        /* Then raise the request to stop the VM and the event.
+         * qemu_system_vmstop_request_prepare has two effects.  First,
+         * it ensures that the STOP event always comes after the
+         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
+         * can observe the STOP event and do a "cont" before the STOP
+         * event is issued, the VM will not stop.  In this case, vm_start()
+         * also ensures that the STOP/RESUME pair of events is emitted.
+         */
+        qemu_system_vmstop_request_prepare();
+        send_qmp_error_event(blk, action, is_read, error);
+        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
+    } else {
+        send_qmp_error_event(blk, action, is_read, error);
+    }
 }
 
 int blk_is_read_only(BlockBackend *blk)
 {
-    return bdrv_is_read_only(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_is_read_only(bs);
+    } else {
+        return blk->root_state.read_only;
+    }
 }
 
 int blk_is_sg(BlockBackend *blk)
 {
-    return bdrv_is_sg(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        return 0;
+    }
+
+    return bdrv_is_sg(bs);
 }
 
 int blk_enable_write_cache(BlockBackend *blk)
 {
-    return bdrv_enable_write_cache(blk->bs);
+    return blk->enable_write_cache;
 }
 
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
 {
-    bdrv_set_enable_write_cache(blk->bs, wce);
+    blk->enable_write_cache = wce;
 }
 
 void blk_invalidate_cache(BlockBackend *blk, Error **errp)
 {
-    bdrv_invalidate_cache(blk->bs, errp);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no medium", blk->name);
+        return;
+    }
+
+    bdrv_invalidate_cache(bs, errp);
+}
+
+bool blk_is_inserted(BlockBackend *blk)
+{
+    BlockDriverState *bs = blk_bs(blk);
+
+    return bs && bdrv_is_inserted(bs);
 }
 
-int blk_is_inserted(BlockBackend *blk)
+bool blk_is_available(BlockBackend *blk)
 {
-    return bdrv_is_inserted(blk->bs);
+    return blk_is_inserted(blk) && !blk_dev_is_tray_open(blk);
 }
 
 void blk_lock_medium(BlockBackend *blk, bool locked)
 {
-    bdrv_lock_medium(blk->bs, locked);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_lock_medium(bs, locked);
+    }
 }
 
 void blk_eject(BlockBackend *blk, bool eject_flag)
 {
-    bdrv_eject(blk->bs, eject_flag);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_eject(bs, eject_flag);
+    }
 }
 
 int blk_get_flags(BlockBackend *blk)
 {
-    return bdrv_get_flags(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_get_flags(bs);
+    } else {
+        return blk->root_state.open_flags;
+    }
 }
 
 int blk_get_max_transfer_length(BlockBackend *blk)
 {
-    return blk->bs->bl.max_transfer_length;
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bs->bl.max_transfer_length;
+    } else {
+        return 0;
+    }
+}
+
+int blk_get_max_iov(BlockBackend *blk)
+{
+    return blk->root->bs->bl.max_iov;
 }
 
 void blk_set_guest_block_size(BlockBackend *blk, int align)
 {
-    bdrv_set_guest_block_size(blk->bs, align);
+    blk->guest_block_size = align;
+}
+
+void *blk_try_blockalign(BlockBackend *blk, size_t size)
+{
+    return qemu_try_blockalign(blk ? blk_bs(blk) : NULL, size);
 }
 
 void *blk_blockalign(BlockBackend *blk, size_t size)
 {
-    return qemu_blockalign(blk ? blk->bs : NULL, size);
+    return qemu_blockalign(blk ? blk_bs(blk) : NULL, size);
 }
 
 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
 {
-    return bdrv_op_is_blocked(blk->bs, op, errp);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        return false;
+    }
+
+    return bdrv_op_is_blocked(bs, op, errp);
 }
 
 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
 {
-    bdrv_op_unblock(blk->bs, op, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_unblock(bs, op, reason);
+    }
 }
 
 void blk_op_block_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_block_all(blk->bs, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_block_all(bs, reason);
+    }
 }
 
 void blk_op_unblock_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_unblock_all(blk->bs, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_unblock_all(bs, reason);
+    }
 }
 
 AioContext *blk_get_aio_context(BlockBackend *blk)
 {
-    return bdrv_get_aio_context(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_get_aio_context(bs);
+    } else {
+        return qemu_get_aio_context();
+    }
+}
+
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb)
+{
+    BlockBackendAIOCB *blk_acb = DO_UPCAST(BlockBackendAIOCB, common, acb);
+    return blk_get_aio_context(blk_acb->blk);
 }
 
 void blk_set_aio_context(BlockBackend *blk, AioContext *new_context)
 {
-    bdrv_set_aio_context(blk->bs, new_context);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_set_aio_context(bs, new_context);
+    }
 }
 
 void blk_add_aio_context_notifier(BlockBackend *blk,
         void (*attached_aio_context)(AioContext *new_context, void *opaque),
         void (*detach_aio_context)(void *opaque), void *opaque)
 {
-    bdrv_add_aio_context_notifier(blk->bs, attached_aio_context,
-                                  detach_aio_context, opaque);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_add_aio_context_notifier(bs, attached_aio_context,
+                                      detach_aio_context, opaque);
+    }
 }
 
 void blk_remove_aio_context_notifier(BlockBackend *blk,
@@ -831,28 +1397,45 @@ void blk_remove_aio_context_notifier(BlockBackend *blk,
                                      void (*detach_aio_context)(void *),
                                      void *opaque)
 {
-    bdrv_remove_aio_context_notifier(blk->bs, attached_aio_context,
-                                     detach_aio_context, opaque);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_remove_aio_context_notifier(bs, attached_aio_context,
+                                         detach_aio_context, opaque);
+    }
 }
 
-void blk_add_close_notifier(BlockBackend *blk, Notifier *notify)
+void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify)
 {
-    bdrv_add_close_notifier(blk->bs, notify);
+    notifier_list_add(&blk->remove_bs_notifiers, notify);
+}
+
+void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify)
+{
+    notifier_list_add(&blk->insert_bs_notifiers, notify);
 }
 
 void blk_io_plug(BlockBackend *blk)
 {
-    bdrv_io_plug(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_io_plug(bs);
+    }
 }
 
 void blk_io_unplug(BlockBackend *blk)
 {
-    bdrv_io_unplug(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_io_unplug(bs);
+    }
 }
 
 BlockAcctStats *blk_get_stats(BlockBackend *blk)
 {
-    return bdrv_get_stats(blk->bs);
+    return &blk->stats;
 }
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
@@ -864,12 +1447,13 @@ void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
 int coroutine_fn blk_co_write_zeroes(BlockBackend *blk, int64_t sector_num,
                                      int nb_sectors, BdrvRequestFlags flags)
 {
-    int ret = blk_check_request(blk, sector_num, nb_sectors);
-    if (ret < 0) {
-        return ret;
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EINVAL;
     }
 
-    return bdrv_co_write_zeroes(blk->bs, sector_num, nb_sectors, flags);
+    return blk_co_pwritev(blk, sector_num << BDRV_SECTOR_BITS,
+                          nb_sectors << BDRV_SECTOR_BITS, NULL,
+                          flags | BDRV_REQ_ZERO_WRITE);
 }
 
 int blk_write_compressed(BlockBackend *blk, int64_t sector_num,
@@ -880,12 +1464,16 @@ int blk_write_compressed(BlockBackend *blk, int64_t sector_num,
         return ret;
     }
 
-    return bdrv_write_compressed(blk->bs, sector_num, buf, nb_sectors);
+    return bdrv_write_compressed(blk_bs(blk), sector_num, buf, nb_sectors);
 }
 
 int blk_truncate(BlockBackend *blk, int64_t offset)
 {
-    return bdrv_truncate(blk->bs, offset);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_truncate(blk_bs(blk), offset);
 }
 
 int blk_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
@@ -895,26 +1483,153 @@ int blk_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
         return ret;
     }
 
-    return bdrv_discard(blk->bs, sector_num, nb_sectors);
+    return bdrv_discard(blk_bs(blk), sector_num, nb_sectors);
 }
 
 int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
                      int64_t pos, int size)
 {
-    return bdrv_save_vmstate(blk->bs, buf, pos, size);
+    int ret;
+
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    ret = bdrv_save_vmstate(blk_bs(blk), buf, pos, size);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret == size && !blk->enable_write_cache) {
+        ret = bdrv_flush(blk_bs(blk));
+    }
+
+    return ret < 0 ? ret : size;
 }
 
 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size)
 {
-    return bdrv_load_vmstate(blk->bs, buf, pos, size);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_load_vmstate(blk_bs(blk), buf, pos, size);
 }
 
 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz)
 {
-    return bdrv_probe_blocksizes(blk->bs, bsz);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_probe_blocksizes(blk_bs(blk), bsz);
 }
 
 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo)
 {
-    return bdrv_probe_geometry(blk->bs, geo);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_probe_geometry(blk_bs(blk), geo);
+}
+
+/*
+ * Updates the BlockBackendRootState object with data from the currently
+ * attached BlockDriverState.
+ */
+void blk_update_root_state(BlockBackend *blk)
+{
+    assert(blk->root);
+
+    blk->root_state.open_flags    = blk->root->bs->open_flags;
+    blk->root_state.read_only     = blk->root->bs->read_only;
+    blk->root_state.detect_zeroes = blk->root->bs->detect_zeroes;
+
+    if (blk->root_state.throttle_group) {
+        g_free(blk->root_state.throttle_group);
+        throttle_group_unref(blk->root_state.throttle_state);
+    }
+    if (blk->root->bs->throttle_state) {
+        const char *name = throttle_group_get_name(blk->root->bs);
+        blk->root_state.throttle_group = g_strdup(name);
+        blk->root_state.throttle_state = throttle_group_incref(name);
+    } else {
+        blk->root_state.throttle_group = NULL;
+        blk->root_state.throttle_state = NULL;
+    }
+}
+
+/*
+ * Applies the information in the root state to the given BlockDriverState. This
+ * does not include the flags which have to be specified for bdrv_open(), use
+ * blk_get_open_flags_from_root_state() to inquire them.
+ */
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs)
+{
+    bs->detect_zeroes = blk->root_state.detect_zeroes;
+    if (blk->root_state.throttle_group) {
+        bdrv_io_limits_enable(bs, blk->root_state.throttle_group);
+    }
+}
+
+/*
+ * Returns the flags to be used for bdrv_open() of a BlockDriverState which is
+ * supposed to inherit the root state.
+ */
+int blk_get_open_flags_from_root_state(BlockBackend *blk)
+{
+    int bs_flags;
+
+    bs_flags = blk->root_state.read_only ? 0 : BDRV_O_RDWR;
+    bs_flags |= blk->root_state.open_flags & ~BDRV_O_RDWR;
+
+    return bs_flags;
+}
+
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk)
+{
+    return &blk->root_state;
+}
+
+int blk_commit_all(void)
+{
+    BlockBackend *blk = NULL;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *aio_context = blk_get_aio_context(blk);
+
+        aio_context_acquire(aio_context);
+        if (blk_is_inserted(blk) && blk->root->bs->backing) {
+            int ret = bdrv_commit(blk->root->bs);
+            if (ret < 0) {
+                aio_context_release(aio_context);
+                return ret;
+            }
+        }
+        aio_context_release(aio_context);
+    }
+    return 0;
+}
+
+int blk_flush_all(void)
+{
+    BlockBackend *blk = NULL;
+    int result = 0;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *aio_context = blk_get_aio_context(blk);
+        int ret;
+
+        aio_context_acquire(aio_context);
+        if (blk_is_inserted(blk)) {
+            ret = blk_flush(blk);
+            if (ret < 0 && !result) {
+                result = ret;
+            }
+        }
+        aio_context_release(aio_context);
+    }
+
+    return result;
 }
diff --git a/qemu/block/bochs.c b/qemu/block/bochs.c
index 199ac2b9a..af8b7abdf 100644
--- a/qemu/block/bochs.c
+++ b/qemu/block/bochs.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
@@ -103,7 +105,7 @@ static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
 
     bs->read_only = 1; // no write support yet
 
-    ret = bdrv_pread(bs->file, 0, &bochs, sizeof(bochs));
+    ret = bdrv_pread(bs->file->bs, 0, &bochs, sizeof(bochs));
     if (ret < 0) {
         return ret;
     }
@@ -137,7 +139,7 @@ static int bochs_open(BlockDriverState *bs, QDict *options, int flags,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, le32_to_cpu(bochs.header), s->catalog_bitmap,
+    ret = bdrv_pread(bs->file->bs, le32_to_cpu(bochs.header), s->catalog_bitmap,
                      s->catalog_size * 4);
     if (ret < 0) {
         goto fail;
@@ -206,7 +208,7 @@ static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
         (s->extent_blocks + s->bitmap_blocks));
 
     /* read in bitmap for current extent */
-    ret = bdrv_pread(bs->file, bitmap_offset + (extent_offset / 8),
+    ret = bdrv_pread(bs->file->bs, bitmap_offset + (extent_offset / 8),
                      &bitmap_entry, 1);
     if (ret < 0) {
         return ret;
@@ -229,7 +231,7 @@ static int bochs_read(BlockDriverState *bs, int64_t sector_num,
         if (block_offset < 0) {
             return block_offset;
         } else if (block_offset > 0) {
-            ret = bdrv_pread(bs->file, block_offset, buf, 512);
+            ret = bdrv_pread(bs->file->bs, block_offset, buf, 512);
             if (ret < 0) {
                 return ret;
             }
diff --git a/qemu/block/cloop.c b/qemu/block/cloop.c
index f328be06f..a84f14019 100644
--- a/qemu/block/cloop.c
+++ b/qemu/block/cloop.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
@@ -66,7 +68,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
     bs->read_only = 1;
 
     /* read header */
-    ret = bdrv_pread(bs->file, 128, &s->block_size, 4);
+    ret = bdrv_pread(bs->file->bs, 128, &s->block_size, 4);
     if (ret < 0) {
         return ret;
     }
@@ -92,7 +94,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
         return -EINVAL;
     }
 
-    ret = bdrv_pread(bs->file, 128 + 4, &s->n_blocks, 4);
+    ret = bdrv_pread(bs->file->bs, 128 + 4, &s->n_blocks, 4);
     if (ret < 0) {
         return ret;
     }
@@ -123,7 +125,7 @@ static int cloop_open(BlockDriverState *bs, QDict *options, int flags,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, 128 + 4 + 4, s->offsets, offsets_size);
+    ret = bdrv_pread(bs->file->bs, 128 + 4 + 4, s->offsets, offsets_size);
     if (ret < 0) {
         goto fail;
     }
@@ -203,8 +205,8 @@ static inline int cloop_read_block(BlockDriverState *bs, int block_num)
         int ret;
         uint32_t bytes = s->offsets[block_num + 1] - s->offsets[block_num];
 
-        ret = bdrv_pread(bs->file, s->offsets[block_num], s->compressed_block,
-                         bytes);
+        ret = bdrv_pread(bs->file->bs, s->offsets[block_num],
+                         s->compressed_block, bytes);
         if (ret != bytes) {
             return -1;
         }
diff --git a/qemu/block/commit.c b/qemu/block/commit.c
index 7312a5bdc..cba0e8c1e 100644
--- a/qemu/block/commit.c
+++ b/qemu/block/commit.c
@@ -12,11 +12,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -213,7 +216,7 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
 
     if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
          on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, "Invalid parameter combination");
         return;
     }
@@ -235,14 +238,14 @@ void commit_start(BlockDriverState *bs, BlockDriverState *base,
     orig_overlay_flags = bdrv_get_flags(overlay_bs);
 
     /* convert base & overlay_bs to r/w, if necessary */
-    if (!(orig_base_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, base,
-                                         orig_base_flags | BDRV_O_RDWR);
-    }
     if (!(orig_overlay_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs,
+        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs, NULL,
                                          orig_overlay_flags | BDRV_O_RDWR);
     }
+    if (!(orig_base_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, base, NULL,
+                                         orig_base_flags | BDRV_O_RDWR);
+    }
     if (reopen_queue) {
         bdrv_reopen_multiple(reopen_queue, &local_err);
         if (local_err != NULL) {
diff --git a/qemu/block/crypto.c b/qemu/block/crypto.c
new file mode 100644
index 000000000..1903e84fb
--- /dev/null
+++ b/qemu/block/crypto.c
@@ -0,0 +1,586 @@
+/*
+ * QEMU block full disk encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "crypto/block.h"
+#include "qapi/opts-visitor.h"
+#include "qapi-visit.h"
+#include "qapi/error.h"
+
+#define BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET "key-secret"
+#define BLOCK_CRYPTO_OPT_LUKS_CIPHER_ALG "cipher-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_CIPHER_MODE "cipher-mode"
+#define BLOCK_CRYPTO_OPT_LUKS_IVGEN_ALG "ivgen-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_IVGEN_HASH_ALG "ivgen-hash-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_HASH_ALG "hash-alg"
+
+typedef struct BlockCrypto BlockCrypto;
+
+struct BlockCrypto {
+    QCryptoBlock *block;
+};
+
+
+static int block_crypto_probe_generic(QCryptoBlockFormat format,
+                                      const uint8_t *buf,
+                                      int buf_size,
+                                      const char *filename)
+{
+    if (qcrypto_block_has_format(format, buf, buf_size)) {
+        return 100;
+    } else {
+        return 0;
+    }
+}
+
+
+static ssize_t block_crypto_read_func(QCryptoBlock *block,
+                                      size_t offset,
+                                      uint8_t *buf,
+                                      size_t buflen,
+                                      Error **errp,
+                                      void *opaque)
+{
+    BlockDriverState *bs = opaque;
+    ssize_t ret;
+
+    ret = bdrv_pread(bs->file->bs, offset, buf, buflen);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not read encryption header");
+        return ret;
+    }
+    return ret;
+}
+
+
+struct BlockCryptoCreateData {
+    const char *filename;
+    QemuOpts *opts;
+    BlockBackend *blk;
+    uint64_t size;
+};
+
+
+static ssize_t block_crypto_write_func(QCryptoBlock *block,
+                                       size_t offset,
+                                       const uint8_t *buf,
+                                       size_t buflen,
+                                       Error **errp,
+                                       void *opaque)
+{
+    struct BlockCryptoCreateData *data = opaque;
+    ssize_t ret;
+
+    ret = blk_pwrite(data->blk, offset, buf, buflen);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not write encryption header");
+        return ret;
+    }
+    return ret;
+}
+
+
+static ssize_t block_crypto_init_func(QCryptoBlock *block,
+                                      size_t headerlen,
+                                      Error **errp,
+                                      void *opaque)
+{
+    struct BlockCryptoCreateData *data = opaque;
+    int ret;
+
+    /* User provided size should reflect amount of space made
+     * available to the guest, so we must take account of that
+     * which will be used by the crypto header
+     */
+    data->size += headerlen;
+
+    qemu_opt_set_number(data->opts, BLOCK_OPT_SIZE, data->size, &error_abort);
+    ret = bdrv_create_file(data->filename, data->opts, errp);
+    if (ret < 0) {
+        return -1;
+    }
+
+    data->blk = blk_new_open(data->filename, NULL, NULL,
+                             BDRV_O_RDWR | BDRV_O_PROTOCOL, errp);
+    if (!data->blk) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static QemuOptsList block_crypto_runtime_opts_luks = {
+    .name = "crypto",
+    .head = QTAILQ_HEAD_INITIALIZER(block_crypto_runtime_opts_luks.head),
+    .desc = {
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret that provides the encryption key",
+        },
+        { /* end of list */ }
+    },
+};
+
+
+static QemuOptsList block_crypto_create_opts_luks = {
+    .name = "crypto",
+    .head = QTAILQ_HEAD_INITIALIZER(block_crypto_create_opts_luks.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size"
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret that provides the encryption key",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_CIPHER_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption cipher algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_CIPHER_MODE,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption cipher mode",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_IVGEN_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of IV generator algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_IVGEN_HASH_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of IV generator hash algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_HASH_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption hash algorithm",
+        },
+        { /* end of list */ }
+    },
+};
+
+
+static QCryptoBlockOpenOptions *
+block_crypto_open_opts_init(QCryptoBlockFormat format,
+                            QemuOpts *opts,
+                            Error **errp)
+{
+    OptsVisitor *ov;
+    QCryptoBlockOpenOptions *ret = NULL;
+    Error *local_err = NULL;
+    Error *end_err = NULL;
+
+    ret = g_new0(QCryptoBlockOpenOptions, 1);
+    ret->format = format;
+
+    ov = opts_visitor_new(opts);
+
+    visit_start_struct(opts_get_visitor(ov),
+                       NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    switch (format) {
+    case Q_CRYPTO_BLOCK_FORMAT_LUKS:
+        visit_type_QCryptoBlockOptionsLUKS_members(
+            opts_get_visitor(ov), &ret->u.luks, &local_err);
+        break;
+
+    default:
+        error_setg(&local_err, "Unsupported block format %d", format);
+        break;
+    }
+
+    visit_end_struct(opts_get_visitor(ov), &end_err);
+    error_propagate(&local_err, end_err);
+
+ out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        qapi_free_QCryptoBlockOpenOptions(ret);
+        ret = NULL;
+    }
+    opts_visitor_cleanup(ov);
+    return ret;
+}
+
+
+static QCryptoBlockCreateOptions *
+block_crypto_create_opts_init(QCryptoBlockFormat format,
+                              QemuOpts *opts,
+                              Error **errp)
+{
+    OptsVisitor *ov;
+    QCryptoBlockCreateOptions *ret = NULL;
+    Error *local_err = NULL;
+    Error *end_err = NULL;
+
+    ret = g_new0(QCryptoBlockCreateOptions, 1);
+    ret->format = format;
+
+    ov = opts_visitor_new(opts);
+
+    visit_start_struct(opts_get_visitor(ov),
+                       NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    switch (format) {
+    case Q_CRYPTO_BLOCK_FORMAT_LUKS:
+        visit_type_QCryptoBlockCreateOptionsLUKS_members(
+            opts_get_visitor(ov), &ret->u.luks, &local_err);
+        break;
+
+    default:
+        error_setg(&local_err, "Unsupported block format %d", format);
+        break;
+    }
+
+    visit_end_struct(opts_get_visitor(ov), &end_err);
+    error_propagate(&local_err, end_err);
+
+ out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        qapi_free_QCryptoBlockCreateOptions(ret);
+        ret = NULL;
+    }
+    opts_visitor_cleanup(ov);
+    return ret;
+}
+
+
+static int block_crypto_open_generic(QCryptoBlockFormat format,
+                                     QemuOptsList *opts_spec,
+                                     BlockDriverState *bs,
+                                     QDict *options,
+                                     int flags,
+                                     Error **errp)
+{
+    BlockCrypto *crypto = bs->opaque;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
+    int ret = -EINVAL;
+    QCryptoBlockOpenOptions *open_opts = NULL;
+    unsigned int cflags = 0;
+
+    opts = qemu_opts_create(opts_spec, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto cleanup;
+    }
+
+    open_opts = block_crypto_open_opts_init(format, opts, errp);
+    if (!open_opts) {
+        goto cleanup;
+    }
+
+    if (flags & BDRV_O_NO_IO) {
+        cflags |= QCRYPTO_BLOCK_OPEN_NO_IO;
+    }
+    crypto->block = qcrypto_block_open(open_opts,
+                                       block_crypto_read_func,
+                                       bs,
+                                       cflags,
+                                       errp);
+
+    if (!crypto->block) {
+        ret = -EIO;
+        goto cleanup;
+    }
+
+    bs->encrypted = 1;
+    bs->valid_key = 1;
+
+    ret = 0;
+ cleanup:
+    qapi_free_QCryptoBlockOpenOptions(open_opts);
+    return ret;
+}
+
+
+static int block_crypto_create_generic(QCryptoBlockFormat format,
+                                       const char *filename,
+                                       QemuOpts *opts,
+                                       Error **errp)
+{
+    int ret = -EINVAL;
+    QCryptoBlockCreateOptions *create_opts = NULL;
+    QCryptoBlock *crypto = NULL;
+    struct BlockCryptoCreateData data = {
+        .size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                         BDRV_SECTOR_SIZE),
+        .opts = opts,
+        .filename = filename,
+    };
+
+    create_opts = block_crypto_create_opts_init(format, opts, errp);
+    if (!create_opts) {
+        return -1;
+    }
+
+    crypto = qcrypto_block_create(create_opts,
+                                  block_crypto_init_func,
+                                  block_crypto_write_func,
+                                  &data,
+                                  errp);
+
+    if (!crypto) {
+        ret = -EIO;
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    qcrypto_block_free(crypto);
+    blk_unref(data.blk);
+    qapi_free_QCryptoBlockCreateOptions(create_opts);
+    return ret;
+}
+
+static int block_crypto_truncate(BlockDriverState *bs, int64_t offset)
+{
+    BlockCrypto *crypto = bs->opaque;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block);
+
+    offset += payload_offset;
+
+    return bdrv_truncate(bs->file->bs, offset);
+}
+
+static void block_crypto_close(BlockDriverState *bs)
+{
+    BlockCrypto *crypto = bs->opaque;
+    qcrypto_block_free(crypto->block);
+}
+
+
+#define BLOCK_CRYPTO_MAX_SECTORS 32
+
+static coroutine_fn int
+block_crypto_co_readv(BlockDriverState *bs, int64_t sector_num,
+                      int remaining_sectors, QEMUIOVector *qiov)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t bytes_done = 0;
+    uint8_t *cipher_data = NULL;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block) / 512;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    /* Bounce buffer so we have a linear mem region for
+     * entire sector. XXX optimize so we avoid bounce
+     * buffer in case that qiov->niov == 1
+     */
+    cipher_data =
+        qemu_try_blockalign(bs->file->bs, MIN(BLOCK_CRYPTO_MAX_SECTORS * 512,
+                                              qiov->size));
+    if (cipher_data == NULL) {
+        ret = -ENOMEM;
+        goto cleanup;
+    }
+
+    while (remaining_sectors) {
+        cur_nr_sectors = remaining_sectors;
+
+        if (cur_nr_sectors > BLOCK_CRYPTO_MAX_SECTORS) {
+            cur_nr_sectors = BLOCK_CRYPTO_MAX_SECTORS;
+        }
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_add(&hd_qiov, cipher_data, cur_nr_sectors * 512);
+
+        ret = bdrv_co_readv(bs->file->bs,
+                            payload_offset + sector_num,
+                            cur_nr_sectors, &hd_qiov);
+        if (ret < 0) {
+            goto cleanup;
+        }
+
+        if (qcrypto_block_decrypt(crypto->block,
+                                  sector_num,
+                                  cipher_data, cur_nr_sectors * 512,
+                                  NULL) < 0) {
+            ret = -EIO;
+            goto cleanup;
+        }
+
+        qemu_iovec_from_buf(qiov, bytes_done,
+                            cipher_data, cur_nr_sectors * 512);
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+    }
+
+ cleanup:
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cipher_data);
+
+    return ret;
+}
+
+
+static coroutine_fn int
+block_crypto_co_writev(BlockDriverState *bs, int64_t sector_num,
+                       int remaining_sectors, QEMUIOVector *qiov)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t bytes_done = 0;
+    uint8_t *cipher_data = NULL;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block) / 512;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    /* Bounce buffer so we have a linear mem region for
+     * entire sector. XXX optimize so we avoid bounce
+     * buffer in case that qiov->niov == 1
+     */
+    cipher_data =
+        qemu_try_blockalign(bs->file->bs, MIN(BLOCK_CRYPTO_MAX_SECTORS * 512,
+                                              qiov->size));
+    if (cipher_data == NULL) {
+        ret = -ENOMEM;
+        goto cleanup;
+    }
+
+    while (remaining_sectors) {
+        cur_nr_sectors = remaining_sectors;
+
+        if (cur_nr_sectors > BLOCK_CRYPTO_MAX_SECTORS) {
+            cur_nr_sectors = BLOCK_CRYPTO_MAX_SECTORS;
+        }
+
+        qemu_iovec_to_buf(qiov, bytes_done,
+                          cipher_data, cur_nr_sectors * 512);
+
+        if (qcrypto_block_encrypt(crypto->block,
+                                  sector_num,
+                                  cipher_data, cur_nr_sectors * 512,
+                                  NULL) < 0) {
+            ret = -EIO;
+            goto cleanup;
+        }
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_add(&hd_qiov, cipher_data, cur_nr_sectors * 512);
+
+        ret = bdrv_co_writev(bs->file->bs,
+                             payload_offset + sector_num,
+                             cur_nr_sectors, &hd_qiov);
+        if (ret < 0) {
+            goto cleanup;
+        }
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+    }
+
+ cleanup:
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cipher_data);
+
+    return ret;
+}
+
+
+static int64_t block_crypto_getlength(BlockDriverState *bs)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int64_t len = bdrv_getlength(bs->file->bs);
+
+    ssize_t offset = qcrypto_block_get_payload_offset(crypto->block);
+
+    len -= offset;
+
+    return len;
+}
+
+
+static int block_crypto_probe_luks(const uint8_t *buf,
+                                   int buf_size,
+                                   const char *filename) {
+    return block_crypto_probe_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                      buf, buf_size, filename);
+}
+
+static int block_crypto_open_luks(BlockDriverState *bs,
+                                  QDict *options,
+                                  int flags,
+                                  Error **errp)
+{
+    return block_crypto_open_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                     &block_crypto_runtime_opts_luks,
+                                     bs, options, flags, errp);
+}
+
+static int block_crypto_create_luks(const char *filename,
+                                    QemuOpts *opts,
+                                    Error **errp)
+{
+    return block_crypto_create_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                       filename, opts, errp);
+}
+
+BlockDriver bdrv_crypto_luks = {
+    .format_name        = "luks",
+    .instance_size      = sizeof(BlockCrypto),
+    .bdrv_probe         = block_crypto_probe_luks,
+    .bdrv_open          = block_crypto_open_luks,
+    .bdrv_close         = block_crypto_close,
+    .bdrv_create        = block_crypto_create_luks,
+    .bdrv_truncate      = block_crypto_truncate,
+    .create_opts        = &block_crypto_create_opts_luks,
+
+    .bdrv_co_readv      = block_crypto_co_readv,
+    .bdrv_co_writev     = block_crypto_co_writev,
+    .bdrv_getlength     = block_crypto_getlength,
+};
+
+static void block_crypto_init(void)
+{
+    bdrv_register(&bdrv_crypto_luks);
+}
+
+block_init(block_crypto_init);
diff --git a/qemu/block/curl.c b/qemu/block/curl.c
index 032cc8ae2..5a8f8b623 100644
--- a/qemu/block/curl.c
+++ b/qemu/block/curl.c
@@ -21,12 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "block/block_int.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qstring.h"
+#include "crypto/secret.h"
 #include <curl/curl.h>
+#include "qemu/cutils.h"
 
 // #define DEBUG_CURL
 // #define DEBUG_VERBOSE
@@ -77,6 +81,10 @@ static CURLMcode __curl_multi_socket_action(CURLM *multi_handle,
 #define CURL_BLOCK_OPT_SSLVERIFY "sslverify"
 #define CURL_BLOCK_OPT_TIMEOUT "timeout"
 #define CURL_BLOCK_OPT_COOKIE    "cookie"
+#define CURL_BLOCK_OPT_USERNAME "username"
+#define CURL_BLOCK_OPT_PASSWORD_SECRET "password-secret"
+#define CURL_BLOCK_OPT_PROXY_USERNAME "proxy-username"
+#define CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET "proxy-password-secret"
 
 struct BDRVCURLState;
 
@@ -119,6 +127,10 @@ typedef struct BDRVCURLState {
     char *cookie;
     bool accept_range;
     AioContext *aio_context;
+    char *username;
+    char *password;
+    char *proxyusername;
+    char *proxypassword;
 } BDRVCURLState;
 
 static void curl_clean_state(CURLState *s);
@@ -154,18 +166,20 @@ static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
     switch (action) {
         case CURL_POLL_IN:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               NULL, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, NULL, state);
             break;
         case CURL_POLL_OUT:
-            aio_set_fd_handler(s->aio_context, fd, NULL, curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, curl_multi_do, state);
             break;
         case CURL_POLL_INOUT:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, curl_multi_do, state);
             break;
         case CURL_POLL_REMOVE:
-            aio_set_fd_handler(s->aio_context, fd, NULL, NULL, NULL);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, NULL, NULL);
             break;
     }
 
@@ -416,6 +430,21 @@ static CURLState *curl_init_state(BlockDriverState *bs, BDRVCURLState *s)
         curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
         curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 
+        if (s->username) {
+            curl_easy_setopt(state->curl, CURLOPT_USERNAME, s->username);
+        }
+        if (s->password) {
+            curl_easy_setopt(state->curl, CURLOPT_PASSWORD, s->password);
+        }
+        if (s->proxyusername) {
+            curl_easy_setopt(state->curl,
+                             CURLOPT_PROXYUSERNAME, s->proxyusername);
+        }
+        if (s->proxypassword) {
+            curl_easy_setopt(state->curl,
+                             CURLOPT_PROXYPASSWORD, s->proxypassword);
+        }
+
         /* Restrict supported protocols to avoid security issues in the more
          * obscure protocols.  For example, do not allow POP3/SMTP/IMAP see
          * CVE-2013-0249.
@@ -522,10 +551,31 @@ static QemuOptsList runtime_opts = {
             .type = QEMU_OPT_STRING,
             .help = "Pass the cookie or list of cookies with each request"
         },
+        {
+            .name = CURL_BLOCK_OPT_USERNAME,
+            .type = QEMU_OPT_STRING,
+            .help = "Username for HTTP auth"
+        },
+        {
+            .name = CURL_BLOCK_OPT_PASSWORD_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret used as password for HTTP auth",
+        },
+        {
+            .name = CURL_BLOCK_OPT_PROXY_USERNAME,
+            .type = QEMU_OPT_STRING,
+            .help = "Username for HTTP proxy auth"
+        },
+        {
+            .name = CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret used as password for HTTP proxy auth",
+        },
         { /* end of list */ }
     },
 };
 
+
 static int curl_open(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp)
 {
@@ -536,6 +586,7 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
     const char *file;
     const char *cookie;
     double d;
+    const char *secretid;
 
     static int inited = 0;
 
@@ -577,6 +628,26 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
         goto out_noclean;
     }
 
+    s->username = g_strdup(qemu_opt_get(opts, CURL_BLOCK_OPT_USERNAME));
+    secretid = qemu_opt_get(opts, CURL_BLOCK_OPT_PASSWORD_SECRET);
+
+    if (secretid) {
+        s->password = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!s->password) {
+            goto out_noclean;
+        }
+    }
+
+    s->proxyusername = g_strdup(
+        qemu_opt_get(opts, CURL_BLOCK_OPT_PROXY_USERNAME));
+    secretid = qemu_opt_get(opts, CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET);
+    if (secretid) {
+        s->proxypassword = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!s->proxypassword) {
+            goto out_noclean;
+        }
+    }
+
     if (!inited) {
         curl_global_init(CURL_GLOBAL_ALL);
         inited = 1;
diff --git a/qemu/block/dirty-bitmap.c b/qemu/block/dirty-bitmap.c
new file mode 100644
index 000000000..4902ca557
--- /dev/null
+++ b/qemu/block/dirty-bitmap.c
@@ -0,0 +1,387 @@
+/*
+ * Block Dirty Bitmap
+ *
+ * Copyright (c) 2016 Red Hat. Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+
+/**
+ * A BdrvDirtyBitmap can be in three possible states:
+ * (1) successor is NULL and disabled is false: full r/w mode
+ * (2) successor is NULL and disabled is true: read only mode ("disabled")
+ * (3) successor is set: frozen mode.
+ *     A frozen bitmap cannot be renamed, deleted, anonymized, cleared, set,
+ *     or enabled. A frozen bitmap can only abdicate() or reclaim().
+ */
+struct BdrvDirtyBitmap {
+    HBitmap *bitmap;            /* Dirty sector bitmap implementation */
+    BdrvDirtyBitmap *successor; /* Anonymous child; implies frozen status */
+    char *name;                 /* Optional non-empty unique ID */
+    int64_t size;               /* Size of the bitmap (Number of sectors) */
+    bool disabled;              /* Bitmap is read-only */
+    QLIST_ENTRY(BdrvDirtyBitmap) list;
+};
+
+BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs, const char *name)
+{
+    BdrvDirtyBitmap *bm;
+
+    assert(name);
+    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
+        if (bm->name && !strcmp(name, bm->name)) {
+            return bm;
+        }
+    }
+    return NULL;
+}
+
+void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    g_free(bitmap->name);
+    bitmap->name = NULL;
+}
+
+BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
+                                          uint32_t granularity,
+                                          const char *name,
+                                          Error **errp)
+{
+    int64_t bitmap_size;
+    BdrvDirtyBitmap *bitmap;
+    uint32_t sector_granularity;
+
+    assert((granularity & (granularity - 1)) == 0);
+
+    if (name && bdrv_find_dirty_bitmap(bs, name)) {
+        error_setg(errp, "Bitmap already exists: %s", name);
+        return NULL;
+    }
+    sector_granularity = granularity >> BDRV_SECTOR_BITS;
+    assert(sector_granularity);
+    bitmap_size = bdrv_nb_sectors(bs);
+    if (bitmap_size < 0) {
+        error_setg_errno(errp, -bitmap_size, "could not get length of device");
+        errno = -bitmap_size;
+        return NULL;
+    }
+    bitmap = g_new0(BdrvDirtyBitmap, 1);
+    bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(sector_granularity));
+    bitmap->size = bitmap_size;
+    bitmap->name = g_strdup(name);
+    bitmap->disabled = false;
+    QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
+    return bitmap;
+}
+
+bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap)
+{
+    return bitmap->successor;
+}
+
+bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)
+{
+    return !(bitmap->disabled || bitmap->successor);
+}
+
+DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap)
+{
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        return DIRTY_BITMAP_STATUS_FROZEN;
+    } else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
+        return DIRTY_BITMAP_STATUS_DISABLED;
+    } else {
+        return DIRTY_BITMAP_STATUS_ACTIVE;
+    }
+}
+
+/**
+ * Create a successor bitmap destined to replace this bitmap after an operation.
+ * Requires that the bitmap is not frozen and has no successor.
+ */
+int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
+                                       BdrvDirtyBitmap *bitmap, Error **errp)
+{
+    uint64_t granularity;
+    BdrvDirtyBitmap *child;
+
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        error_setg(errp, "Cannot create a successor for a bitmap that is "
+                   "currently frozen");
+        return -1;
+    }
+    assert(!bitmap->successor);
+
+    /* Create an anonymous successor */
+    granularity = bdrv_dirty_bitmap_granularity(bitmap);
+    child = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
+    if (!child) {
+        return -1;
+    }
+
+    /* Successor will be on or off based on our current state. */
+    child->disabled = bitmap->disabled;
+
+    /* Install the successor and freeze the parent */
+    bitmap->successor = child;
+    return 0;
+}
+
+/**
+ * For a bitmap with a successor, yield our name to the successor,
+ * delete the old bitmap, and return a handle to the new bitmap.
+ */
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
+                                            BdrvDirtyBitmap *bitmap,
+                                            Error **errp)
+{
+    char *name;
+    BdrvDirtyBitmap *successor = bitmap->successor;
+
+    if (successor == NULL) {
+        error_setg(errp, "Cannot relinquish control if "
+                   "there's no successor present");
+        return NULL;
+    }
+
+    name = bitmap->name;
+    bitmap->name = NULL;
+    successor->name = name;
+    bitmap->successor = NULL;
+    bdrv_release_dirty_bitmap(bs, bitmap);
+
+    return successor;
+}
+
+/**
+ * In cases of failure where we can no longer safely delete the parent,
+ * we may wish to re-join the parent and child/successor.
+ * The merged parent will be un-frozen, but not explicitly re-enabled.
+ */
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
+                                           BdrvDirtyBitmap *parent,
+                                           Error **errp)
+{
+    BdrvDirtyBitmap *successor = parent->successor;
+
+    if (!successor) {
+        error_setg(errp, "Cannot reclaim a successor when none is present");
+        return NULL;
+    }
+
+    if (!hbitmap_merge(parent->bitmap, successor->bitmap)) {
+        error_setg(errp, "Merging of parent and successor bitmap failed");
+        return NULL;
+    }
+    bdrv_release_dirty_bitmap(bs, successor);
+    parent->successor = NULL;
+
+    return parent;
+}
+
+/**
+ * Truncates _all_ bitmaps attached to a BDS.
+ */
+void bdrv_dirty_bitmap_truncate(BlockDriverState *bs)
+{
+    BdrvDirtyBitmap *bitmap;
+    uint64_t size = bdrv_nb_sectors(bs);
+
+    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
+        assert(!bdrv_dirty_bitmap_frozen(bitmap));
+        hbitmap_truncate(bitmap->bitmap, size);
+        bitmap->size = size;
+    }
+}
+
+static void bdrv_do_release_matching_dirty_bitmap(BlockDriverState *bs,
+                                                  BdrvDirtyBitmap *bitmap,
+                                                  bool only_named)
+{
+    BdrvDirtyBitmap *bm, *next;
+    QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
+        if ((!bitmap || bm == bitmap) && (!only_named || bm->name)) {
+            assert(!bdrv_dirty_bitmap_frozen(bm));
+            QLIST_REMOVE(bm, list);
+            hbitmap_free(bm->bitmap);
+            g_free(bm->name);
+            g_free(bm);
+
+            if (bitmap) {
+                return;
+            }
+        }
+    }
+}
+
+void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
+{
+    bdrv_do_release_matching_dirty_bitmap(bs, bitmap, false);
+}
+
+/**
+ * Release all named dirty bitmaps attached to a BDS (for use in bdrv_close()).
+ * There must not be any frozen bitmaps attached.
+ */
+void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs)
+{
+    bdrv_do_release_matching_dirty_bitmap(bs, NULL, true);
+}
+
+void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    bitmap->disabled = true;
+}
+
+void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    bitmap->disabled = false;
+}
+
+BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
+{
+    BdrvDirtyBitmap *bm;
+    BlockDirtyInfoList *list = NULL;
+    BlockDirtyInfoList **plist = &list;
+
+    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
+        BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
+        BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
+        info->count = bdrv_get_dirty_count(bm);
+        info->granularity = bdrv_dirty_bitmap_granularity(bm);
+        info->has_name = !!bm->name;
+        info->name = g_strdup(bm->name);
+        info->status = bdrv_dirty_bitmap_status(bm);
+        entry->value = info;
+        *plist = entry;
+        plist = &entry->next;
+    }
+
+    return list;
+}
+
+int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
+                   int64_t sector)
+{
+    if (bitmap) {
+        return hbitmap_get(bitmap->bitmap, sector);
+    } else {
+        return 0;
+    }
+}
+
+/**
+ * Chooses a default granularity based on the existing cluster size,
+ * but clamped between [4K, 64K]. Defaults to 64K in the case that there
+ * is no cluster size information available.
+ */
+uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs)
+{
+    BlockDriverInfo bdi;
+    uint32_t granularity;
+
+    if (bdrv_get_info(bs, &bdi) >= 0 && bdi.cluster_size > 0) {
+        granularity = MAX(4096, bdi.cluster_size);
+        granularity = MIN(65536, granularity);
+    } else {
+        granularity = 65536;
+    }
+
+    return granularity;
+}
+
+uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap)
+{
+    return BDRV_SECTOR_SIZE << hbitmap_granularity(bitmap->bitmap);
+}
+
+void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, HBitmapIter *hbi)
+{
+    hbitmap_iter_init(hbi, bitmap->bitmap, 0);
+}
+
+void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                           int64_t cur_sector, int nr_sectors)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
+}
+
+void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                             int64_t cur_sector, int nr_sectors)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    hbitmap_reset(bitmap->bitmap, cur_sector, nr_sectors);
+}
+
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    if (!out) {
+        hbitmap_reset_all(bitmap->bitmap);
+    } else {
+        HBitmap *backup = bitmap->bitmap;
+        bitmap->bitmap = hbitmap_alloc(bitmap->size,
+                                       hbitmap_granularity(backup));
+        *out = backup;
+    }
+}
+
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in)
+{
+    HBitmap *tmp = bitmap->bitmap;
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    bitmap->bitmap = in;
+    hbitmap_free(tmp);
+}
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
+                    int nr_sectors)
+{
+    BdrvDirtyBitmap *bitmap;
+    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
+        if (!bdrv_dirty_bitmap_enabled(bitmap)) {
+            continue;
+        }
+        hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
+    }
+}
+
+/**
+ * Advance an HBitmapIter to an arbitrary offset.
+ */
+void bdrv_set_dirty_iter(HBitmapIter *hbi, int64_t offset)
+{
+    assert(hbi->hb);
+    hbitmap_iter_init(hbi, hbi->hb, offset);
+}
+
+int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
+{
+    return hbitmap_count(bitmap->bitmap);
+}
diff --git a/qemu/block/dmg.c b/qemu/block/dmg.c
index 9f2528169..a496eb7c9 100644
--- a/qemu/block/dmg.c
+++ b/qemu/block/dmg.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/bswap.h"
@@ -85,7 +87,7 @@ static int read_uint64(BlockDriverState *bs, int64_t offset, uint64_t *result)
     uint64_t buffer;
     int ret;
 
-    ret = bdrv_pread(bs->file, offset, &buffer, 8);
+    ret = bdrv_pread(bs->file->bs, offset, &buffer, 8);
     if (ret < 0) {
         return ret;
     }
@@ -99,7 +101,7 @@ static int read_uint32(BlockDriverState *bs, int64_t offset, uint32_t *result)
     uint32_t buffer;
     int ret;
 
-    ret = bdrv_pread(bs->file, offset, &buffer, 4);
+    ret = bdrv_pread(bs->file->bs, offset, &buffer, 4);
     if (ret < 0) {
         return ret;
     }
@@ -354,7 +356,7 @@ static int dmg_read_resource_fork(BlockDriverState *bs, DmgHeaderState *ds,
         offset += 4;
 
         buffer = g_realloc(buffer, count);
-        ret = bdrv_pread(bs->file, offset, buffer, count);
+        ret = bdrv_pread(bs->file->bs, offset, buffer, count);
         if (ret < 0) {
             goto fail;
         }
@@ -391,7 +393,7 @@ static int dmg_read_plist_xml(BlockDriverState *bs, DmgHeaderState *ds,
 
     buffer = g_malloc(info_length + 1);
     buffer[info_length] = '\0';
-    ret = bdrv_pread(bs->file, info_begin, buffer, info_length);
+    ret = bdrv_pread(bs->file->bs, info_begin, buffer, info_length);
     if (ret != info_length) {
         ret = -EINVAL;
         goto fail;
@@ -446,7 +448,7 @@ static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
     ds.max_sectors_per_chunk = 1;
 
     /* locate the UDIF trailer */
-    offset = dmg_find_koly_offset(bs->file, errp);
+    offset = dmg_find_koly_offset(bs->file->bs, errp);
     if (offset < 0) {
         ret = offset;
         goto fail;
@@ -514,9 +516,9 @@ static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* initialize zlib engine */
-    s->compressed_chunk = qemu_try_blockalign(bs->file,
+    s->compressed_chunk = qemu_try_blockalign(bs->file->bs,
                                               ds.max_compressed_size + 1);
-    s->uncompressed_chunk = qemu_try_blockalign(bs->file,
+    s->uncompressed_chunk = qemu_try_blockalign(bs->file->bs,
                                                 512 * ds.max_sectors_per_chunk);
     if (s->compressed_chunk == NULL || s->uncompressed_chunk == NULL) {
         ret = -ENOMEM;
@@ -592,7 +594,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
         case 0x80000005: { /* zlib compressed */
             /* we need to buffer, because only the chunk as whole can be
              * inflated. */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->compressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
@@ -616,7 +618,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
         case 0x80000006: /* bzip2 compressed */
             /* we need to buffer, because only the chunk as whole can be
              * inflated. */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->compressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
@@ -641,7 +643,7 @@ static inline int dmg_read_chunk(BlockDriverState *bs, uint64_t sector_num)
             break;
 #endif /* CONFIG_BZIP2 */
         case 1: /* copy */
-            ret = bdrv_pread(bs->file, s->offsets[chunk],
+            ret = bdrv_pread(bs->file->bs, s->offsets[chunk],
                              s->uncompressed_chunk, s->lengths[chunk]);
             if (ret != s->lengths[chunk]) {
                 return -1;
diff --git a/qemu/block/gluster.c b/qemu/block/gluster.c
index 1eb3a8c39..a8aaacf64 100644
--- a/qemu/block/gluster.c
+++ b/qemu/block/gluster.c
@@ -7,8 +7,10 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include <glusterfs/api/glfs.h>
 #include "block/block_int.h"
+#include "qapi/error.h"
 #include "qemu/uri.h"
 
 typedef struct GlusterAIOCB {
@@ -245,7 +247,7 @@ static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
     if (!ret || ret == acb->size) {
         acb->ret = 0; /* Success */
     } else if (ret < 0) {
-        acb->ret = ret; /* Read/Write failed */
+        acb->ret = -errno; /* Read/Write failed */
     } else {
         acb->ret = -EIO; /* Partial read/write - fail it */
     }
@@ -312,6 +314,23 @@ static int qemu_gluster_open(BlockDriverState *bs,  QDict *options,
         goto out;
     }
 
+#ifdef CONFIG_GLUSTERFS_XLATOR_OPT
+    /* Without this, if fsync fails for a recoverable reason (for instance,
+     * ENOSPC), gluster will dump its cache, preventing retries.  This means
+     * almost certain data loss.  Not all gluster versions support the
+     * 'resync-failed-syncs-after-fsync' key value, but there is no way to
+     * discover during runtime if it is supported (this api returns success for
+     * unknown key/value pairs) */
+    ret = glfs_set_xlator_option(s->glfs, "*-write-behind",
+                                          "resync-failed-syncs-after-fsync",
+                                          "on");
+    if (ret < 0) {
+        error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
+        ret = -errno;
+        goto out;
+    }
+#endif
+
     qemu_gluster_parse_flags(bdrv_flags, &open_flags);
 
     s->fd = glfs_open(s->glfs, gconf->image, open_flags);
@@ -364,6 +383,16 @@ static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
         goto exit;
     }
 
+#ifdef CONFIG_GLUSTERFS_XLATOR_OPT
+    ret = glfs_set_xlator_option(reop_s->glfs, "*-write-behind",
+                                 "resync-failed-syncs-after-fsync", "on");
+    if (ret < 0) {
+        error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
+        ret = -errno;
+        goto exit;
+    }
+#endif
+
     reop_s->fd = glfs_open(reop_s->glfs, gconf->image, open_flags);
     if (reop_s->fd == NULL) {
         /* reops->glfs will be cleaned up in _abort */
@@ -429,28 +458,23 @@ static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     off_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_zerofill_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_zerofill_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static inline bool gluster_supports_zerofill(void)
@@ -541,35 +565,30 @@ static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     size_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
     if (write) {
         ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+            gluster_finish_aiocb, &acb);
     } else {
         ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+            gluster_finish_aiocb, &acb);
     }
 
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
@@ -597,28 +616,58 @@ static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
     return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 1);
 }
 
+static void qemu_gluster_close(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+
+    if (s->fd) {
+        glfs_close(s->fd);
+        s->fd = NULL;
+    }
+    glfs_fini(s->glfs);
+}
+
 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
+    ret = glfs_fsync_async(s->fd, gluster_finish_aiocb, &acb);
     if (ret < 0) {
         ret = -errno;
-        goto out;
+        goto error;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
+    if (acb.ret < 0) {
+        ret = acb.ret;
+        goto error;
+    }
+
+    return acb.ret;
+
+error:
+    /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
+     * after a fsync failure, so we have no way of allowing the guest to safely
+     * continue.  Gluster versions prior to 3.5.6 don't retain the cache
+     * either, but will invalidate the fd on error, so this is again our only
+     * option.
+     *
+     * The 'resync-failed-syncs-after-fsync' xlator option for the
+     * write-behind cache will cause later gluster versions to retain its
+     * cache after error, so long as the fd remains open.  However, we
+     * currently have no way of knowing if this option is supported.
+     *
+     * TODO: Once gluster provides a way for us to determine if the option
+     * is supported, bypass the closure and setting drv to NULL.  */
+    qemu_gluster_close(bs);
+    bs->drv = NULL;
     return ret;
 }
 
@@ -627,28 +676,23 @@ static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     size_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_discard_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_discard_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 #endif
 
@@ -679,17 +723,6 @@ static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
     }
 }
 
-static void qemu_gluster_close(BlockDriverState *bs)
-{
-    BDRVGlusterState *s = bs->opaque;
-
-    if (s->fd) {
-        glfs_close(s->fd);
-        s->fd = NULL;
-    }
-    glfs_fini(s->glfs);
-}
-
 static int qemu_gluster_has_zero_init(BlockDriverState *bs)
 {
     /* GlusterFS volume could be backed by a block device */
diff --git a/qemu/block/io.c b/qemu/block/io.c
index d4bc83b33..a7dbf85b1 100644
--- a/qemu/block/io.c
+++ b/qemu/block/io.c
@@ -22,10 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
+#include "sysemu/block-backend.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
 #include "block/throttle-groups.h"
+#include "qemu/cutils.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 
 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
@@ -42,12 +46,6 @@ static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
 static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
                                          int64_t sector_num, int nb_sectors,
                                          QEMUIOVector *iov);
-static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
-static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
                                          int64_t sector_num,
                                          QEMUIOVector *qiov,
@@ -156,38 +154,45 @@ void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
 
     /* Take some limits from the children as a default */
     if (bs->file) {
-        bdrv_refresh_limits(bs->file, &local_err);
+        bdrv_refresh_limits(bs->file->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
-        bs->bl.opt_transfer_length = bs->file->bl.opt_transfer_length;
-        bs->bl.max_transfer_length = bs->file->bl.max_transfer_length;
-        bs->bl.min_mem_alignment = bs->file->bl.min_mem_alignment;
-        bs->bl.opt_mem_alignment = bs->file->bl.opt_mem_alignment;
+        bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length;
+        bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length;
+        bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment;
+        bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment;
+        bs->bl.max_iov = bs->file->bs->bl.max_iov;
     } else {
         bs->bl.min_mem_alignment = 512;
         bs->bl.opt_mem_alignment = getpagesize();
+
+        /* Safe default since most protocols use readv()/writev()/etc */
+        bs->bl.max_iov = IOV_MAX;
     }
 
-    if (bs->backing_hd) {
-        bdrv_refresh_limits(bs->backing_hd, &local_err);
+    if (bs->backing) {
+        bdrv_refresh_limits(bs->backing->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
         bs->bl.opt_transfer_length =
             MAX(bs->bl.opt_transfer_length,
-                bs->backing_hd->bl.opt_transfer_length);
+                bs->backing->bs->bl.opt_transfer_length);
         bs->bl.max_transfer_length =
             MIN_NON_ZERO(bs->bl.max_transfer_length,
-                         bs->backing_hd->bl.max_transfer_length);
+                         bs->backing->bs->bl.max_transfer_length);
         bs->bl.opt_mem_alignment =
             MAX(bs->bl.opt_mem_alignment,
-                bs->backing_hd->bl.opt_mem_alignment);
+                bs->backing->bs->bl.opt_mem_alignment);
         bs->bl.min_mem_alignment =
             MAX(bs->bl.min_mem_alignment,
-                bs->backing_hd->bl.min_mem_alignment);
+                bs->backing->bs->bl.min_mem_alignment);
+        bs->bl.max_iov =
+            MIN(bs->bl.max_iov,
+                bs->backing->bs->bl.max_iov);
     }
 
     /* Then let the driver override it */
@@ -213,8 +218,10 @@ void bdrv_disable_copy_on_read(BlockDriverState *bs)
 }
 
 /* Check if any requests are in-flight (including throttled requests) */
-static bool bdrv_requests_pending(BlockDriverState *bs)
+bool bdrv_requests_pending(BlockDriverState *bs)
 {
+    BdrvChild *child;
+
     if (!QLIST_EMPTY(&bs->tracked_requests)) {
         return true;
     }
@@ -224,17 +231,72 @@ static bool bdrv_requests_pending(BlockDriverState *bs)
     if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) {
         return true;
     }
-    if (bs->file && bdrv_requests_pending(bs->file)) {
-        return true;
-    }
-    if (bs->backing_hd && bdrv_requests_pending(bs->backing_hd)) {
-        return true;
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (bdrv_requests_pending(child->bs)) {
+            return true;
+        }
     }
+
     return false;
 }
 
+static void bdrv_drain_recurse(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    if (bs->drv && bs->drv->bdrv_drain) {
+        bs->drv->bdrv_drain(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_drain_recurse(child->bs);
+    }
+}
+
+typedef struct {
+    Coroutine *co;
+    BlockDriverState *bs;
+    QEMUBH *bh;
+    bool done;
+} BdrvCoDrainData;
+
+static void bdrv_co_drain_bh_cb(void *opaque)
+{
+    BdrvCoDrainData *data = opaque;
+    Coroutine *co = data->co;
+
+    qemu_bh_delete(data->bh);
+    bdrv_drain(data->bs);
+    data->done = true;
+    qemu_coroutine_enter(co, NULL);
+}
+
+void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
+{
+    BdrvCoDrainData data;
+
+    /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
+     * other coroutines run if they were queued from
+     * qemu_co_queue_run_restart(). */
+
+    assert(qemu_in_coroutine());
+    data = (BdrvCoDrainData) {
+        .co = qemu_coroutine_self(),
+        .bs = bs,
+        .done = false,
+        .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
+    };
+    qemu_bh_schedule(data.bh);
+
+    qemu_coroutine_yield();
+    /* If we are resumed from some other event (such as an aio completion or a
+     * timer callback), it is a bug in the caller that should be fixed. */
+    assert(data.done);
+}
+
 /*
- * Wait for pending requests to complete on a single BlockDriverState subtree
+ * Wait for pending requests to complete on a single BlockDriverState subtree,
+ * and suspend block driver's internal I/O until next request arrives.
  *
  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
  * AioContext.
@@ -247,6 +309,11 @@ void bdrv_drain(BlockDriverState *bs)
 {
     bool busy = true;
 
+    bdrv_drain_recurse(bs);
+    if (qemu_in_coroutine()) {
+        bdrv_co_drain(bs);
+        return;
+    }
     while (busy) {
         /* Keep iterating */
          bdrv_flush_io_queue(bs);
@@ -275,6 +342,7 @@ void bdrv_drain_all(void)
         if (bs->job) {
             block_job_pause(bs->job);
         }
+        bdrv_drain_recurse(bs);
         aio_context_release(aio_context);
 
         if (!g_slist_find(aio_ctxs, aio_context)) {
@@ -344,13 +412,14 @@ static void tracked_request_end(BdrvTrackedRequest *req)
 static void tracked_request_begin(BdrvTrackedRequest *req,
                                   BlockDriverState *bs,
                                   int64_t offset,
-                                  unsigned int bytes, bool is_write)
+                                  unsigned int bytes,
+                                  enum BdrvTrackedRequestType type)
 {
     *req = (BdrvTrackedRequest){
         .bs = bs,
         .offset         = offset,
         .bytes          = bytes,
-        .is_write       = is_write,
+        .type           = type,
         .co             = qemu_coroutine_self(),
         .serialising    = false,
         .overlap_offset = offset,
@@ -593,20 +662,6 @@ int bdrv_read(BlockDriverState *bs, int64_t sector_num,
     return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
 }
 
-/* Just like bdrv_read(), but with I/O throttling temporarily disabled */
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors)
-{
-    bool enabled;
-    int ret;
-
-    enabled = bs->io_limits_enabled;
-    bs->io_limits_enabled = false;
-    ret = bdrv_read(bs, sector_num, buf, nb_sectors);
-    bs->io_limits_enabled = enabled;
-    return ret;
-}
-
 /* Return < 0 if error. Important errors are:
   -EIO         generic I/O error (may happen for all errors)
   -ENOMEDIUM   No media inserted.
@@ -637,6 +692,7 @@ int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
 {
     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
+    BlockDriverState *file;
     int n;
 
     target_sectors = bdrv_nb_sectors(bs);
@@ -649,7 +705,7 @@ int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
         if (nb_sectors <= 0) {
             return 0;
         }
-        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n);
+        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
         if (ret < 0) {
             error_report("error getting block status at sector %" PRId64 ": %s",
                          sector_num, strerror(-ret));
@@ -736,9 +792,9 @@ int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
         return ret;
     }
 
-    /* No flush needed for cache modes that already do it */
-    if (bs->enable_write_cache) {
-        bdrv_flush(bs);
+    ret = bdrv_flush(bs);
+    if (ret < 0) {
+        return ret;
     }
 
     return 0;
@@ -833,6 +889,7 @@ static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert(!qiov || bytes == qiov->size);
+    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
 
     /* Handle Copy on Read and associated serialisation */
     if (flags & BDRV_REQ_COPY_ON_READ) {
@@ -844,7 +901,9 @@ static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
         mark_request_serialising(req, bdrv_get_cluster_size(bs));
     }
 
-    wait_serialising_requests(req);
+    if (!(flags & BDRV_REQ_NO_SERIALISING)) {
+        wait_serialising_requests(req);
+    }
 
     if (flags & BDRV_REQ_COPY_ON_READ) {
         int pnum;
@@ -908,7 +967,7 @@ out:
 /*
  * Handle a read request in coroutine context
  */
-static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
+int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
     BdrvRequestFlags flags)
 {
@@ -932,7 +991,8 @@ static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
         return ret;
     }
 
-    if (bs->copy_on_read) {
+    /* Don't do copy-on-read if we read data before write operation */
+    if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
         flags |= BDRV_REQ_COPY_ON_READ;
     }
 
@@ -966,7 +1026,7 @@ static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
         bytes = ROUND_UP(bytes, align);
     }
 
-    tracked_request_begin(&req, bs, offset, bytes, false);
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
                               use_local_qiov ? &local_qiov : qiov,
                               flags);
@@ -1001,6 +1061,15 @@ int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
     return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
 }
 
+int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
+                            BDRV_REQ_NO_SERIALISING);
+}
+
 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
 {
@@ -1107,6 +1176,7 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert(!qiov || bytes == qiov->size);
+    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
 
     waited = wait_serialising_requests(req);
     assert(!waited || !req->serialising);
@@ -1127,21 +1197,30 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
     if (ret < 0) {
         /* Do nothing, write notifier decided to fail this request */
     } else if (flags & BDRV_REQ_ZERO_WRITE) {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_ZERO);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
         ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
+    } else if (drv->bdrv_co_writev_flags) {
+        bdrv_debug_event(bs, BLKDBG_PWRITEV);
+        ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
+                                        flags);
     } else {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV);
+        assert(drv->supported_write_flags == 0);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV);
         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
     }
-    BLKDBG_EVENT(bs, BLKDBG_PWRITEV_DONE);
+    bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
 
-    if (ret == 0 && !bs->enable_write_cache) {
+    if (ret == 0 && (flags & BDRV_REQ_FUA) &&
+        !(drv->supported_write_flags & BDRV_REQ_FUA))
+    {
         ret = bdrv_co_flush(bs);
     }
 
     bdrv_set_dirty(bs, sector_num, nb_sectors);
 
-    block_acct_highest_sector(&bs->stats, sector_num, nb_sectors);
+    if (bs->wr_highest_offset < offset + bytes) {
+        bs->wr_highest_offset = offset + bytes;
+    }
 
     if (ret >= 0) {
         bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
@@ -1182,13 +1261,13 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
         /* RMW the unaligned part before head. */
         mark_request_serialising(req, align);
         wait_serialising_requests(req);
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
                                   align, &local_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
 
         memset(buf + head_padding_bytes, 0, zero_bytes);
         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
@@ -1220,13 +1299,13 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
         /* RMW the unaligned part after tail. */
         mark_request_serialising(req, align);
         wait_serialising_requests(req);
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
         ret = bdrv_aligned_preadv(bs, req, offset, align,
                                   align, &local_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
 
         memset(buf, 0, bytes);
         ret = bdrv_aligned_pwritev(bs, req, offset, align,
@@ -1241,7 +1320,7 @@ fail:
 /*
  * Handle a write request in coroutine context
  */
-static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
+int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
     BdrvRequestFlags flags)
 {
@@ -1260,6 +1339,7 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
     if (bs->read_only) {
         return -EPERM;
     }
+    assert(!(bs->open_flags & BDRV_O_INACTIVE));
 
     ret = bdrv_check_byte_request(bs, offset, bytes);
     if (ret < 0) {
@@ -1276,7 +1356,7 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
      * Pad qiov with the read parts and be sure to have a tracked request not
      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
      */
-    tracked_request_begin(&req, bs, offset, bytes, true);
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
 
     if (!qiov) {
         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
@@ -1297,13 +1377,13 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
         };
         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
 
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
                                   align, &head_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
 
         qemu_iovec_init(&local_qiov, qiov->niov + 2);
         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
@@ -1331,13 +1411,13 @@ static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
         };
         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
 
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
                                   align, &tail_qiov, 0);
         if (ret < 0) {
             goto fail;
         }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
 
         if (!use_local_qiov) {
             qemu_iovec_init(&local_qiov, qiov->niov + 1);
@@ -1401,29 +1481,10 @@ int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
                              BDRV_REQ_ZERO_WRITE | flags);
 }
 
-int bdrv_flush_all(void)
-{
-    BlockDriverState *bs = NULL;
-    int result = 0;
-
-    while ((bs = bdrv_next(bs))) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-        int ret;
-
-        aio_context_acquire(aio_context);
-        ret = bdrv_flush(bs);
-        if (ret < 0 && !result) {
-            result = ret;
-        }
-        aio_context_release(aio_context);
-    }
-
-    return result;
-}
-
 typedef struct BdrvCoGetBlockStatusData {
     BlockDriverState *bs;
     BlockDriverState *base;
+    BlockDriverState **file;
     int64_t sector_num;
     int nb_sectors;
     int *pnum;
@@ -1445,10 +1506,14 @@ typedef struct BdrvCoGetBlockStatusData {
  *
  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
  * beyond the end of the disk image it will be clamped.
+ *
+ * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
+ * points to the BDS which the sector range is allocated in.
  */
 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
                                                      int64_t sector_num,
-                                                     int nb_sectors, int *pnum)
+                                                     int nb_sectors, int *pnum,
+                                                     BlockDriverState **file)
 {
     int64_t total_sectors;
     int64_t n;
@@ -1478,7 +1543,9 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
         return ret;
     }
 
-    ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum);
+    *file = NULL;
+    ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
+                                            file);
     if (ret < 0) {
         *pnum = 0;
         return ret;
@@ -1486,8 +1553,8 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
 
     if (ret & BDRV_BLOCK_RAW) {
         assert(ret & BDRV_BLOCK_OFFSET_VALID);
-        return bdrv_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
-                                     *pnum, pnum);
+        return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
+                                     *pnum, pnum, file);
     }
 
     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
@@ -1495,8 +1562,8 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
     } else {
         if (bdrv_unallocated_blocks_are_zero(bs)) {
             ret |= BDRV_BLOCK_ZERO;
-        } else if (bs->backing_hd) {
-            BlockDriverState *bs2 = bs->backing_hd;
+        } else if (bs->backing) {
+            BlockDriverState *bs2 = bs->backing->bs;
             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
                 ret |= BDRV_BLOCK_ZERO;
@@ -1504,13 +1571,14 @@ static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
         }
     }
 
-    if (bs->file &&
+    if (*file && *file != bs &&
         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
         (ret & BDRV_BLOCK_OFFSET_VALID)) {
+        BlockDriverState *file2;
         int file_pnum;
 
-        ret2 = bdrv_co_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
-                                        *pnum, &file_pnum);
+        ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
+                                        *pnum, &file_pnum, &file2);
         if (ret2 >= 0) {
             /* Ignore errors.  This is just providing extra information, it
              * is useful but not necessary.
@@ -1535,14 +1603,15 @@ static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
         BlockDriverState *base,
         int64_t sector_num,
         int nb_sectors,
-        int *pnum)
+        int *pnum,
+        BlockDriverState **file)
 {
     BlockDriverState *p;
     int64_t ret = 0;
 
     assert(bs != base);
-    for (p = bs; p != base; p = p->backing_hd) {
-        ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum);
+    for (p = bs; p != base; p = backing_bs(p)) {
+        ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
             break;
         }
@@ -1561,7 +1630,8 @@ static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
                                                data->sector_num,
                                                data->nb_sectors,
-                                               data->pnum);
+                                               data->pnum,
+                                               data->file);
     data->done = true;
 }
 
@@ -1573,12 +1643,14 @@ static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
                                     BlockDriverState *base,
                                     int64_t sector_num,
-                                    int nb_sectors, int *pnum)
+                                    int nb_sectors, int *pnum,
+                                    BlockDriverState **file)
 {
     Coroutine *co;
     BdrvCoGetBlockStatusData data = {
         .bs = bs,
         .base = base,
+        .file = file,
         .sector_num = sector_num,
         .nb_sectors = nb_sectors,
         .pnum = pnum,
@@ -1602,16 +1674,19 @@ int64_t bdrv_get_block_status_above(BlockDriverState *bs,
 
 int64_t bdrv_get_block_status(BlockDriverState *bs,
                               int64_t sector_num,
-                              int nb_sectors, int *pnum)
+                              int nb_sectors, int *pnum,
+                              BlockDriverState **file)
 {
-    return bdrv_get_block_status_above(bs, bs->backing_hd,
-                                       sector_num, nb_sectors, pnum);
+    return bdrv_get_block_status_above(bs, backing_bs(bs),
+                                       sector_num, nb_sectors, pnum, file);
 }
 
 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
                                    int nb_sectors, int *pnum)
 {
-    int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum);
+    BlockDriverState *file;
+    int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
+                                        &file);
     if (ret < 0) {
         return ret;
     }
@@ -1662,7 +1737,7 @@ int bdrv_is_allocated_above(BlockDriverState *top,
             n = pnum_inter;
         }
 
-        intermediate = intermediate->backing_hd;
+        intermediate = backing_bs(intermediate);
     }
 
     *pnum = n;
@@ -1713,7 +1788,7 @@ int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
     } else if (drv->bdrv_save_vmstate) {
         return drv->bdrv_save_vmstate(bs, qiov, pos);
     } else if (bs->file) {
-        return bdrv_writev_vmstate(bs->file, qiov, pos);
+        return bdrv_writev_vmstate(bs->file->bs, qiov, pos);
     }
 
     return -ENOTSUP;
@@ -1728,7 +1803,7 @@ int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
     if (drv->bdrv_load_vmstate)
         return drv->bdrv_load_vmstate(bs, buf, pos, size);
     if (bs->file)
-        return bdrv_load_vmstate(bs->file, buf, pos, size);
+        return bdrv_load_vmstate(bs->file->bs, buf, pos, size);
     return -ENOTSUP;
 }
 
@@ -1849,7 +1924,8 @@ static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
             merge = 1;
         }
 
-        if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > IOV_MAX) {
+        if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 >
+            bs->bl.max_iov) {
             merge = 0;
         }
 
@@ -1893,7 +1969,10 @@ static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
         }
     }
 
-    block_acct_merge_done(&bs->stats, BLOCK_ACCT_WRITE, num_reqs - outidx - 1);
+    if (bs->blk) {
+        block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE,
+                              num_reqs - outidx - 1);
+    }
 
     return outidx + 1;
 }
@@ -2208,7 +2287,7 @@ void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
 {
     BlockAIOCB *acb;
 
-    acb = g_slice_alloc(aiocb_info->aiocb_size);
+    acb = g_malloc(aiocb_info->aiocb_size);
     acb->aiocb_info = aiocb_info;
     acb->bs = bs;
     acb->cb = cb;
@@ -2228,7 +2307,7 @@ void qemu_aio_unref(void *p)
     BlockAIOCB *acb = p;
     assert(acb->refcnt > 0);
     if (--acb->refcnt == 0) {
-        g_slice_free1(acb->aiocb_info->aiocb_size, acb);
+        g_free(acb);
     }
 }
 
@@ -2298,18 +2377,27 @@ static void coroutine_fn bdrv_flush_co_entry(void *opaque)
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
 {
     int ret;
+    BdrvTrackedRequest req;
 
     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
         bdrv_is_sg(bs)) {
         return 0;
     }
 
+    tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
+
+    /* Write back all layers by calling one driver function */
+    if (bs->drv->bdrv_co_flush) {
+        ret = bs->drv->bdrv_co_flush(bs);
+        goto out;
+    }
+
     /* Write back cached data to the OS even with cache=unsafe */
     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
     if (bs->drv->bdrv_co_flush_to_os) {
         ret = bs->drv->bdrv_co_flush_to_os(bs);
         if (ret < 0) {
-            return ret;
+            goto out;
         }
     }
 
@@ -2349,14 +2437,17 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
         ret = 0;
     }
     if (ret < 0) {
-        return ret;
+        goto out;
     }
 
     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
      * in the case of cache=unsafe, so there are no useless flushes.
      */
 flush_parent:
-    return bdrv_co_flush(bs->file);
+    ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
+out:
+    tracked_request_end(&req);
+    return ret;
 }
 
 int bdrv_flush(BlockDriverState *bs)
@@ -2399,6 +2490,7 @@ static void coroutine_fn bdrv_discard_co_entry(void *opaque)
 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
                                  int nb_sectors)
 {
+    BdrvTrackedRequest req;
     int max_discard, ret;
 
     if (!bs->drv) {
@@ -2411,6 +2503,7 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
     } else if (bs->read_only) {
         return -EPERM;
     }
+    assert(!(bs->open_flags & BDRV_O_INACTIVE));
 
     /* Do nothing if disabled.  */
     if (!(bs->open_flags & BDRV_O_UNMAP)) {
@@ -2421,6 +2514,8 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
         return 0;
     }
 
+    tracked_request_begin(&req, bs, sector_num, nb_sectors,
+                          BDRV_TRACKED_DISCARD);
     bdrv_set_dirty(bs, sector_num, nb_sectors);
 
     max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS);
@@ -2454,20 +2549,24 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
             acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
                                             bdrv_co_io_em_complete, &co);
             if (acb == NULL) {
-                return -EIO;
+                ret = -EIO;
+                goto out;
             } else {
                 qemu_coroutine_yield();
                 ret = co.ret;
             }
         }
         if (ret && ret != -ENOTSUP) {
-            return ret;
+            goto out;
         }
 
         sector_num += num;
         nb_sectors -= num;
     }
-    return 0;
+    ret = 0;
+out:
+    tracked_request_end(&req);
+    return ret;
 }
 
 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
@@ -2496,26 +2595,110 @@ int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
     return rwco.ret;
 }
 
-/* needed for generic scsi interface */
+typedef struct {
+    CoroutineIOCompletion *co;
+    QEMUBH *bh;
+} BdrvIoctlCompletionData;
 
-int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+static void bdrv_ioctl_bh_cb(void *opaque)
+{
+    BdrvIoctlCompletionData *data = opaque;
+
+    bdrv_co_io_em_complete(data->co, -ENOTSUP);
+    qemu_bh_delete(data->bh);
+}
+
+static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
 {
     BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest tracked_req;
+    CoroutineIOCompletion co = {
+        .coroutine = qemu_coroutine_self(),
+    };
+    BlockAIOCB *acb;
 
-    if (drv && drv->bdrv_ioctl)
-        return drv->bdrv_ioctl(bs, req, buf);
-    return -ENOTSUP;
+    tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
+    if (!drv || !drv->bdrv_aio_ioctl) {
+        co.ret = -ENOTSUP;
+        goto out;
+    }
+
+    acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
+    if (!acb) {
+        BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1);
+        data->bh = aio_bh_new(bdrv_get_aio_context(bs),
+                                bdrv_ioctl_bh_cb, data);
+        data->co = &co;
+        qemu_bh_schedule(data->bh);
+    }
+    qemu_coroutine_yield();
+out:
+    tracked_request_end(&tracked_req);
+    return co.ret;
+}
+
+typedef struct {
+    BlockDriverState *bs;
+    int req;
+    void *buf;
+    int ret;
+} BdrvIoctlCoData;
+
+static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
+{
+    BdrvIoctlCoData *data = opaque;
+    data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
+}
+
+/* needed for generic scsi interface */
+int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+{
+    BdrvIoctlCoData data = {
+        .bs = bs,
+        .req = req,
+        .buf = buf,
+        .ret = -EINPROGRESS,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_co_ioctl_entry(&data);
+    } else {
+        Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry);
+
+        qemu_coroutine_enter(co, &data);
+        while (data.ret == -EINPROGRESS) {
+            aio_poll(bdrv_get_aio_context(bs), true);
+        }
+    }
+    return data.ret;
+}
+
+static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+    acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
+                                      acb->req.req, acb->req.buf);
+    bdrv_co_complete(acb);
 }
 
 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque)
 {
-    BlockDriver *drv = bs->drv;
+    BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
+                                            bs, cb, opaque);
+    Coroutine *co;
+
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+    acb->req.req = req;
+    acb->req.buf = buf;
+    co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry);
+    qemu_coroutine_enter(co, acb);
 
-    if (drv && drv->bdrv_aio_ioctl)
-        return drv->bdrv_aio_ioctl(bs, req, buf, cb, opaque);
-    return NULL;
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
 }
 
 void *qemu_blockalign(BlockDriverState *bs, size_t size)
@@ -2584,7 +2767,7 @@ void bdrv_io_plug(BlockDriverState *bs)
     if (drv && drv->bdrv_io_plug) {
         drv->bdrv_io_plug(bs);
     } else if (bs->file) {
-        bdrv_io_plug(bs->file);
+        bdrv_io_plug(bs->file->bs);
     }
 }
 
@@ -2594,7 +2777,7 @@ void bdrv_io_unplug(BlockDriverState *bs)
     if (drv && drv->bdrv_io_unplug) {
         drv->bdrv_io_unplug(bs);
     } else if (bs->file) {
-        bdrv_io_unplug(bs->file);
+        bdrv_io_unplug(bs->file->bs);
     }
 }
 
@@ -2604,7 +2787,24 @@ void bdrv_flush_io_queue(BlockDriverState *bs)
     if (drv && drv->bdrv_flush_io_queue) {
         drv->bdrv_flush_io_queue(bs);
     } else if (bs->file) {
-        bdrv_flush_io_queue(bs->file);
+        bdrv_flush_io_queue(bs->file->bs);
     }
     bdrv_start_throttled_reqs(bs);
 }
+
+void bdrv_drained_begin(BlockDriverState *bs)
+{
+    if (!bs->quiesce_counter++) {
+        aio_disable_external(bdrv_get_aio_context(bs));
+    }
+    bdrv_drain(bs);
+}
+
+void bdrv_drained_end(BlockDriverState *bs)
+{
+    assert(bs->quiesce_counter > 0);
+    if (--bs->quiesce_counter > 0) {
+        return;
+    }
+    aio_enable_external(bdrv_get_aio_context(bs));
+}
diff --git a/qemu/block/iscsi.c b/qemu/block/iscsi.c
index 50029168e..302baf84c 100644
--- a/qemu/block/iscsi.c
+++ b/qemu/block/iscsi.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <poll.h>
 #include <math.h>
@@ -39,6 +39,7 @@
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
 #include "qapi/qmp/qstring.h"
+#include "crypto/secret.h"
 
 #include <iscsi/iscsi.h>
 #include <iscsi/scsi-lowlevel.h>
@@ -69,7 +70,6 @@ typedef struct IscsiLun {
     bool lbprz;
     bool dpofua;
     bool has_write_same;
-    bool force_next_flush;
     bool request_timed_out;
 } IscsiLun;
 
@@ -83,7 +83,7 @@ typedef struct IscsiTask {
     QEMUBH *bh;
     IscsiLun *iscsilun;
     QEMUTimer retry_timer;
-    bool force_next_flush;
+    int err_code;
 } IscsiTask;
 
 typedef struct IscsiAIOCB {
@@ -96,6 +96,7 @@ typedef struct IscsiAIOCB {
     int status;
     int64_t sector_num;
     int nb_sectors;
+    int ret;
 #ifdef __linux__
     sg_io_hdr_t *ioh;
 #endif
@@ -169,19 +170,70 @@ static inline unsigned exp_random(double mean)
     return -mean * log((double)rand() / RAND_MAX);
 }
 
-/* SCSI_STATUS_TASK_SET_FULL and SCSI_STATUS_TIMEOUT were introduced
- * in libiscsi 1.10.0 as part of an enum. The LIBISCSI_API_VERSION
- * macro was introduced in 1.11.0. So use the API_VERSION macro as
- * a hint that the macros are defined and define them ourselves
- * otherwise to keep the required libiscsi version at 1.9.0 */
-#if !defined(LIBISCSI_API_VERSION)
-#define QEMU_SCSI_STATUS_TASK_SET_FULL  0x28
-#define QEMU_SCSI_STATUS_TIMEOUT        0x0f000002
-#else
-#define QEMU_SCSI_STATUS_TASK_SET_FULL  SCSI_STATUS_TASK_SET_FULL
-#define QEMU_SCSI_STATUS_TIMEOUT        SCSI_STATUS_TIMEOUT
+/* SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST was introduced in
+ * libiscsi 1.10.0, together with other constants we need.  Use it as
+ * a hint that we have to define them ourselves if needed, to keep the
+ * minimum required libiscsi version at 1.9.0.  We use an ASCQ macro for
+ * the test because SCSI_STATUS_* is an enum.
+ *
+ * To guard against future changes where SCSI_SENSE_ASCQ_* also becomes
+ * an enum, check against the LIBISCSI_API_VERSION macro, which was
+ * introduced in 1.11.0.  If it is present, there is no need to define
+ * anything.
+ */
+#if !defined(SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST) && \
+    !defined(LIBISCSI_API_VERSION)
+#define SCSI_STATUS_TASK_SET_FULL                          0x28
+#define SCSI_STATUS_TIMEOUT                                0x0f000002
+#define SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST    0x2600
+#define SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR        0x1a00
 #endif
 
+static int iscsi_translate_sense(struct scsi_sense *sense)
+{
+    int ret;
+
+    switch (sense->key) {
+    case SCSI_SENSE_NOT_READY:
+        return -EBUSY;
+    case SCSI_SENSE_DATA_PROTECTION:
+        return -EACCES;
+    case SCSI_SENSE_COMMAND_ABORTED:
+        return -ECANCELED;
+    case SCSI_SENSE_ILLEGAL_REQUEST:
+        /* Parse ASCQ */
+        break;
+    default:
+        return -EIO;
+    }
+    switch (sense->ascq) {
+    case SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR:
+    case SCSI_SENSE_ASCQ_INVALID_OPERATION_CODE:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_CDB:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST:
+        ret = -EINVAL;
+        break;
+    case SCSI_SENSE_ASCQ_LBA_OUT_OF_RANGE:
+        ret = -ENOSPC;
+        break;
+    case SCSI_SENSE_ASCQ_LOGICAL_UNIT_NOT_SUPPORTED:
+        ret = -ENOTSUP;
+        break;
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_CLOSED:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_OPEN:
+        ret = -ENOMEDIUM;
+        break;
+    case SCSI_SENSE_ASCQ_WRITE_PROTECTED:
+        ret = -EACCES;
+        break;
+    default:
+        ret = -EIO;
+        break;
+    }
+    return ret;
+}
+
 static void
 iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                         void *command_data, void *opaque)
@@ -203,11 +255,11 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                 goto out;
             }
             if (status == SCSI_STATUS_BUSY ||
-                status == QEMU_SCSI_STATUS_TIMEOUT ||
-                status == QEMU_SCSI_STATUS_TASK_SET_FULL) {
+                status == SCSI_STATUS_TIMEOUT ||
+                status == SCSI_STATUS_TASK_SET_FULL) {
                 unsigned retry_time =
                     exp_random(iscsi_retry_times[iTask->retries - 1]);
-                if (status == QEMU_SCSI_STATUS_TIMEOUT) {
+                if (status == SCSI_STATUS_TIMEOUT) {
                     /* make sure the request is rescheduled AFTER the
                      * reconnect is initiated */
                     retry_time = EVENT_INTERVAL * 2;
@@ -226,9 +278,8 @@ iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                 return;
             }
         }
+        iTask->err_code = iscsi_translate_sense(&task->sense);
         error_report("iSCSI Failure: %s", iscsi_get_error(iscsi));
-    } else {
-        iTask->iscsilun->force_next_flush |= iTask->force_next_flush;
     }
 
 out:
@@ -291,8 +342,8 @@ iscsi_set_events(IscsiLun *iscsilun)
     int ev = iscsi_which_events(iscsi);
 
     if (ev != iscsilun->events) {
-        aio_set_fd_handler(iscsilun->aio_context,
-                           iscsi_get_fd(iscsi),
+        aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsi),
+                           false,
                            (ev & POLLIN) ? iscsi_process_read : NULL,
                            (ev & POLLOUT) ? iscsi_process_write : NULL,
                            iscsilun);
@@ -397,15 +448,15 @@ static void iscsi_allocationmap_clear(IscsiLun *iscsilun, int64_t sector_num,
     }
 }
 
-static int coroutine_fn iscsi_co_writev(BlockDriverState *bs,
-                                        int64_t sector_num, int nb_sectors,
-                                        QEMUIOVector *iov)
+static int coroutine_fn
+iscsi_co_writev_flags(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                      QEMUIOVector *iov, int flags)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
     uint64_t lba;
     uint32_t num_sectors;
-    int fua;
+    bool fua;
 
     if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
         return -EINVAL;
@@ -421,8 +472,7 @@ static int coroutine_fn iscsi_co_writev(BlockDriverState *bs,
     num_sectors = sector_qemu2lun(nb_sectors, iscsilun);
     iscsi_co_init_iscsitask(iscsilun, &iTask);
 retry:
-    fua = iscsilun->dpofua && !bs->enable_write_cache;
-    iTask.force_next_flush = !fua;
+    fua = iscsilun->dpofua && (flags & BDRV_REQ_FUA);
     if (iscsilun->use_16_for_rw) {
         iTask.task = iscsi_write16_task(iscsilun->iscsi, iscsilun->lun, lba,
                                         NULL, num_sectors * iscsilun->block_size,
@@ -455,7 +505,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     iscsi_allocationmap_set(iscsilun, sector_num, nb_sectors);
@@ -463,6 +513,13 @@ retry:
     return 0;
 }
 
+static int coroutine_fn
+iscsi_co_writev(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                QEMUIOVector *iov)
+{
+    return iscsi_co_writev_flags(bs, sector_num, nb_sectors, iov, 0);
+}
+
 
 static bool iscsi_allocationmap_is_allocated(IscsiLun *iscsilun,
                                              int64_t sector_num, int nb_sectors)
@@ -478,7 +535,8 @@ static bool iscsi_allocationmap_is_allocated(IscsiLun *iscsilun,
 
 static int64_t coroutine_fn iscsi_co_get_block_status(BlockDriverState *bs,
                                                   int64_t sector_num,
-                                                  int nb_sectors, int *pnum)
+                                                  int nb_sectors, int *pnum,
+                                                  BlockDriverState **file)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct scsi_get_lba_status *lbas = NULL;
@@ -570,6 +628,9 @@ out:
     if (iTask.task != NULL) {
         scsi_free_scsi_task(iTask.task);
     }
+    if (ret > 0 && ret & BDRV_BLOCK_OFFSET_VALID) {
+        *file = bs;
+    }
     return ret;
 }
 
@@ -596,7 +657,8 @@ static int coroutine_fn iscsi_co_readv(BlockDriverState *bs,
         !iscsi_allocationmap_is_allocated(iscsilun, sector_num, nb_sectors)) {
         int64_t ret;
         int pnum;
-        ret = iscsi_co_get_block_status(bs, sector_num, INT_MAX, &pnum);
+        BlockDriverState *file;
+        ret = iscsi_co_get_block_status(bs, sector_num, INT_MAX, &pnum, &file);
         if (ret < 0) {
             return ret;
         }
@@ -644,7 +706,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -655,11 +717,6 @@ static int coroutine_fn iscsi_co_flush(BlockDriverState *bs)
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
 
-    if (!iscsilun->force_next_flush) {
-        return 0;
-    }
-    iscsilun->force_next_flush = false;
-
     iscsi_co_init_iscsitask(iscsilun, &iTask);
 retry:
     if (iscsi_synchronizecache10_task(iscsilun->iscsi, iscsilun->lun, 0, 0, 0,
@@ -683,7 +740,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -703,7 +760,7 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     if (status < 0) {
         error_report("Failed to ioctl(SG_IO) to iSCSI lun. %s",
                      iscsi_get_error(iscsi));
-        acb->status = -EIO;
+        acb->status = iscsi_translate_sense(&acb->task->sense);
     }
 
     acb->ioh->driver_status = 0;
@@ -726,6 +783,38 @@ iscsi_aio_ioctl_cb(struct iscsi_context *iscsi, int status,
     iscsi_schedule_bh(acb);
 }
 
+static void iscsi_ioctl_bh_completion(void *opaque)
+{
+    IscsiAIOCB *acb = opaque;
+
+    qemu_bh_delete(acb->bh);
+    acb->common.cb(acb->common.opaque, acb->ret);
+    qemu_aio_unref(acb);
+}
+
+static void iscsi_ioctl_handle_emulated(IscsiAIOCB *acb, int req, void *buf)
+{
+    BlockDriverState *bs = acb->common.bs;
+    IscsiLun *iscsilun = bs->opaque;
+    int ret = 0;
+
+    switch (req) {
+    case SG_GET_VERSION_NUM:
+        *(int *)buf = 30000;
+        break;
+    case SG_GET_SCSI_ID:
+        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
+        break;
+    default:
+        ret = -EINVAL;
+    }
+    assert(!acb->bh);
+    acb->bh = aio_bh_new(bdrv_get_aio_context(bs),
+                         iscsi_ioctl_bh_completion, acb);
+    acb->ret = ret;
+    qemu_bh_schedule(acb->bh);
+}
+
 static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque)
@@ -735,8 +824,6 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     struct iscsi_data data;
     IscsiAIOCB *acb;
 
-    assert(req == SG_IO);
-
     acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
@@ -745,6 +832,11 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     acb->buf         = NULL;
     acb->ioh         = buf;
 
+    if (req != SG_IO) {
+        iscsi_ioctl_handle_emulated(acb, req, buf);
+        return &acb->common;
+    }
+
     acb->task = malloc(sizeof(struct scsi_task));
     if (acb->task == NULL) {
         error_report("iSCSI: Failed to allocate task for scsi command. %s",
@@ -809,38 +901,6 @@ static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     return &acb->common;
 }
 
-static void ioctl_cb(void *opaque, int status)
-{
-    int *p_status = opaque;
-    *p_status = status;
-}
-
-static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    IscsiLun *iscsilun = bs->opaque;
-    int status;
-
-    switch (req) {
-    case SG_GET_VERSION_NUM:
-        *(int *)buf = 30000;
-        break;
-    case SG_GET_SCSI_ID:
-        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
-        break;
-    case SG_IO:
-        status = -EINPROGRESS;
-        iscsi_aio_ioctl(bs, req, buf, ioctl_cb, &status);
-
-        while (status == -EINPROGRESS) {
-            aio_poll(iscsilun->aio_context, true);
-        }
-
-        return 0;
-    default:
-        return -1;
-    }
-    return 0;
-}
 #endif
 
 static int64_t
@@ -905,7 +965,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     iscsi_allocationmap_clear(iscsilun, sector_num, nb_sectors);
@@ -956,7 +1016,6 @@ coroutine_fn iscsi_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
     }
 
     iscsi_co_init_iscsitask(iscsilun, &iTask);
-    iTask.force_next_flush = true;
 retry:
     if (use_16_for_ws) {
         iTask.task = iscsi_writesame16_task(iscsilun->iscsi, iscsilun->lun, lba,
@@ -999,7 +1058,7 @@ retry:
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     if (flags & BDRV_REQ_MAY_UNMAP) {
@@ -1018,6 +1077,8 @@ static void parse_chap(struct iscsi_context *iscsi, const char *target,
     QemuOpts *opts;
     const char *user = NULL;
     const char *password = NULL;
+    const char *secretid;
+    char *secret = NULL;
 
     list = qemu_find_opts("iscsi");
     if (!list) {
@@ -1037,8 +1098,20 @@ static void parse_chap(struct iscsi_context *iscsi, const char *target,
         return;
     }
 
+    secretid = qemu_opt_get(opts, "password-secret");
     password = qemu_opt_get(opts, "password");
-    if (!password) {
+    if (secretid && password) {
+        error_setg(errp, "'password' and 'password-secret' properties are "
+                   "mutually exclusive");
+        return;
+    }
+    if (secretid) {
+        secret = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!secret) {
+            return;
+        }
+        password = secret;
+    } else if (!password) {
         error_setg(errp, "CHAP username specified but no password was given");
         return;
     }
@@ -1046,6 +1119,8 @@ static void parse_chap(struct iscsi_context *iscsi, const char *target,
     if (iscsi_set_initiator_username_pwd(iscsi, user, password)) {
         error_setg(errp, "Failed to set initiator username and password");
     }
+
+    g_free(secret);
 }
 
 static void parse_header_digest(struct iscsi_context *iscsi, const char *target,
@@ -1186,8 +1261,13 @@ static void iscsi_readcapacity_sync(IscsiLun *iscsilun, Error **errp)
                     iscsilun->lbprz = !!rc16->lbprz;
                     iscsilun->use_16_for_rw = (rc16->returned_lba > 0xffffffff);
                 }
+                break;
             }
-            break;
+            if (task != NULL && task->status == SCSI_STATUS_CHECK_CONDITION
+                && task->sense.key == SCSI_SENSE_UNIT_ATTENTION) {
+                break;
+            }
+            /* Fall through and try READ CAPACITY(10) instead.  */
         case TYPE_ROM:
             task = iscsi_readcapacity10_sync(iscsilun->iscsi, iscsilun->lun, 0, 0);
             if (task != NULL && task->status == SCSI_STATUS_GOOD) {
@@ -1213,7 +1293,11 @@ static void iscsi_readcapacity_sync(IscsiLun *iscsilun, Error **errp)
              && retries-- > 0);
 
     if (task == NULL || task->status != SCSI_STATUS_GOOD) {
-        error_setg(errp, "iSCSI: failed to send readcapacity10 command.");
+        error_setg(errp, "iSCSI: failed to send readcapacity10/16 command");
+    } else if (!iscsilun->block_size ||
+               iscsilun->block_size % BDRV_SECTOR_SIZE) {
+        error_setg(errp, "iSCSI: the target returned an invalid "
+                   "block size of %d.", iscsilun->block_size);
     }
     if (task) {
         scsi_free_scsi_task(task);
@@ -1276,9 +1360,8 @@ static void iscsi_detach_aio_context(BlockDriverState *bs)
 {
     IscsiLun *iscsilun = bs->opaque;
 
-    aio_set_fd_handler(iscsilun->aio_context,
-                       iscsi_get_fd(iscsilun->iscsi),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsilun->iscsi),
+                       false, NULL, NULL, NULL);
     iscsilun->events = 0;
 
     if (iscsilun->nop_timer) {
@@ -1765,10 +1848,11 @@ static BlockDriver bdrv_iscsi = {
     .bdrv_co_write_zeroes = iscsi_co_write_zeroes,
     .bdrv_co_readv         = iscsi_co_readv,
     .bdrv_co_writev        = iscsi_co_writev,
+    .bdrv_co_writev_flags  = iscsi_co_writev_flags,
+    .supported_write_flags = BDRV_REQ_FUA,
     .bdrv_co_flush_to_disk = iscsi_co_flush,
 
 #ifdef __linux__
-    .bdrv_ioctl       = iscsi_ioctl,
     .bdrv_aio_ioctl   = iscsi_aio_ioctl,
 #endif
 
@@ -1789,6 +1873,11 @@ static QemuOptsList qemu_iscsi_opts = {
             .type = QEMU_OPT_STRING,
             .help = "password for CHAP authentication to target",
         },{
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret providing password for CHAP "
+                    "authentication to target",
+        },{
             .name = "header-digest",
             .type = QEMU_OPT_STRING,
             .help = "HeaderDigest setting. "
diff --git a/qemu/block/linux-aio.c b/qemu/block/linux-aio.c
index c991443c5..805757e02 100644
--- a/qemu/block/linux-aio.c
+++ b/qemu/block/linux-aio.c
@@ -7,6 +7,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "qemu/queue.h"
@@ -287,7 +288,7 @@ void laio_detach_aio_context(void *s_, AioContext *old_context)
 {
     struct qemu_laio_state *s = s_;
 
-    aio_set_event_notifier(old_context, &s->e, NULL);
+    aio_set_event_notifier(old_context, &s->e, false, NULL);
     qemu_bh_delete(s->completion_bh);
 }
 
@@ -296,7 +297,8 @@ void laio_attach_aio_context(void *s_, AioContext *new_context)
     struct qemu_laio_state *s = s_;
 
     s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
-    aio_set_event_notifier(new_context, &s->e, qemu_laio_completion_cb);
+    aio_set_event_notifier(new_context, &s->e, false,
+                           qemu_laio_completion_cb);
 }
 
 void *laio_init(void)
diff --git a/qemu/block/mirror.c b/qemu/block/mirror.c
index fc4d8f561..039f48125 100644
--- a/qemu/block/mirror.c
+++ b/qemu/block/mirror.c
@@ -11,12 +11,16 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
 #include "qemu/bitmap.h"
+#include "qemu/error-report.h"
 
 #define SLICE_TIME    100000000ULL /* ns */
 #define MAX_IN_FLIGHT 16
@@ -44,7 +48,6 @@ typedef struct MirrorBlockJob {
     BlockdevOnError on_source_error, on_target_error;
     bool synced;
     bool should_complete;
-    int64_t sector_num;
     int64_t granularity;
     size_t buf_size;
     int64_t bdev_length;
@@ -60,6 +63,9 @@ typedef struct MirrorBlockJob {
     int sectors_in_flight;
     int ret;
     bool unmap;
+    bool waiting_for_io;
+    int target_cluster_sectors;
+    int max_iov;
 } MirrorBlockJob;
 
 typedef struct MirrorOp {
@@ -102,7 +108,7 @@ static void mirror_iteration_done(MirrorOp *op, int ret)
 
     sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
     chunk_num = op->sector_num / sectors_per_chunk;
-    nb_chunks = op->nb_sectors / sectors_per_chunk;
+    nb_chunks = DIV_ROUND_UP(op->nb_sectors, sectors_per_chunk);
     bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
     if (ret >= 0) {
         if (s->cow_bitmap) {
@@ -112,13 +118,9 @@ static void mirror_iteration_done(MirrorOp *op, int ret)
     }
 
     qemu_iovec_destroy(&op->qiov);
-    g_slice_free(MirrorOp, op);
+    g_free(op);
 
-    /* Enter coroutine when it is not sleeping.  The coroutine sleeps to
-     * rate-limit itself.  The coroutine will eventually resume since there is
-     * a sleep timeout so don't wake it early.
-     */
-    if (s->common.busy) {
+    if (s->waiting_for_io) {
         qemu_coroutine_enter(s->common.co, NULL);
     }
 }
@@ -159,107 +161,97 @@ static void mirror_read_complete(void *opaque, int ret)
                     mirror_write_complete, op);
 }
 
-static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
+static inline void mirror_clip_sectors(MirrorBlockJob *s,
+                                       int64_t sector_num,
+                                       int *nb_sectors)
 {
-    BlockDriverState *source = s->common.bs;
-    int nb_sectors, sectors_per_chunk, nb_chunks;
-    int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
-    uint64_t delay_ns = 0;
-    MirrorOp *op;
-    int pnum;
-    int64_t ret;
-
-    s->sector_num = hbitmap_iter_next(&s->hbi);
-    if (s->sector_num < 0) {
-        bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi);
-        s->sector_num = hbitmap_iter_next(&s->hbi);
-        trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
-        assert(s->sector_num >= 0);
-    }
-
-    hbitmap_next_sector = s->sector_num;
-    sector_num = s->sector_num;
-    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
-    end = s->bdev_length / BDRV_SECTOR_SIZE;
-
-    /* Extend the QEMUIOVector to include all adjacent blocks that will
-     * be copied in this operation.
-     *
-     * We have to do this if we have no backing file yet in the destination,
-     * and the cluster size is very large.  Then we need to do COW ourselves.
-     * The first time a cluster is copied, copy it entirely.  Note that,
-     * because both the granularity and the cluster size are powers of two,
-     * the number of sectors to copy cannot exceed one cluster.
-     *
-     * We also want to extend the QEMUIOVector to include more adjacent
-     * dirty blocks if possible, to limit the number of I/O operations and
-     * run efficiently even with a small granularity.
-     */
-    nb_chunks = 0;
-    nb_sectors = 0;
-    next_sector = sector_num;
-    next_chunk = sector_num / sectors_per_chunk;
+    *nb_sectors = MIN(*nb_sectors,
+                      s->bdev_length / BDRV_SECTOR_SIZE - sector_num);
+}
 
-    /* Wait for I/O to this cluster (from a previous iteration) to be done.  */
-    while (test_bit(next_chunk, s->in_flight_bitmap)) {
-        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
-        qemu_coroutine_yield();
+/* Round sector_num and/or nb_sectors to target cluster if COW is needed, and
+ * return the offset of the adjusted tail sector against original. */
+static int mirror_cow_align(MirrorBlockJob *s,
+                            int64_t *sector_num,
+                            int *nb_sectors)
+{
+    bool need_cow;
+    int ret = 0;
+    int chunk_sectors = s->granularity >> BDRV_SECTOR_BITS;
+    int64_t align_sector_num = *sector_num;
+    int align_nb_sectors = *nb_sectors;
+    int max_sectors = chunk_sectors * s->max_iov;
+
+    need_cow = !test_bit(*sector_num / chunk_sectors, s->cow_bitmap);
+    need_cow |= !test_bit((*sector_num + *nb_sectors - 1) / chunk_sectors,
+                          s->cow_bitmap);
+    if (need_cow) {
+        bdrv_round_to_clusters(s->target, *sector_num, *nb_sectors,
+                               &align_sector_num, &align_nb_sectors);
+    }
+
+    if (align_nb_sectors > max_sectors) {
+        align_nb_sectors = max_sectors;
+        if (need_cow) {
+            align_nb_sectors = QEMU_ALIGN_DOWN(align_nb_sectors,
+                                               s->target_cluster_sectors);
+        }
     }
+    /* Clipping may result in align_nb_sectors unaligned to chunk boundary, but
+     * that doesn't matter because it's already the end of source image. */
+    mirror_clip_sectors(s, align_sector_num, &align_nb_sectors);
 
-    do {
-        int added_sectors, added_chunks;
-
-        if (!bdrv_get_dirty(source, s->dirty_bitmap, next_sector) ||
-            test_bit(next_chunk, s->in_flight_bitmap)) {
-            assert(nb_sectors > 0);
-            break;
-        }
+    ret = align_sector_num + align_nb_sectors - (*sector_num + *nb_sectors);
+    *sector_num = align_sector_num;
+    *nb_sectors = align_nb_sectors;
+    assert(ret >= 0);
+    return ret;
+}
 
-        added_sectors = sectors_per_chunk;
-        if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) {
-            bdrv_round_to_clusters(s->target,
-                                   next_sector, added_sectors,
-                                   &next_sector, &added_sectors);
+static inline void mirror_wait_for_io(MirrorBlockJob *s)
+{
+    assert(!s->waiting_for_io);
+    s->waiting_for_io = true;
+    qemu_coroutine_yield();
+    s->waiting_for_io = false;
+}
 
-            /* On the first iteration, the rounding may make us copy
-             * sectors before the first dirty one.
-             */
-            if (next_sector < sector_num) {
-                assert(nb_sectors == 0);
-                sector_num = next_sector;
-                next_chunk = next_sector / sectors_per_chunk;
-            }
-        }
+/* Submit async read while handling COW.
+ * Returns: nb_sectors if no alignment is necessary, or
+ *          (new_end - sector_num) if tail is rounded up or down due to
+ *          alignment or buffer limit.
+ */
+static int mirror_do_read(MirrorBlockJob *s, int64_t sector_num,
+                          int nb_sectors)
+{
+    BlockDriverState *source = s->common.bs;
+    int sectors_per_chunk, nb_chunks;
+    int ret = nb_sectors;
+    MirrorOp *op;
 
-        added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors));
-        added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk;
+    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
 
-        /* When doing COW, it may happen that there is not enough space for
-         * a full cluster.  Wait if that is the case.
-         */
-        while (nb_chunks == 0 && s->buf_free_count < added_chunks) {
-            trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight);
-            qemu_coroutine_yield();
-        }
-        if (s->buf_free_count < nb_chunks + added_chunks) {
-            trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
-            break;
-        }
+    /* We can only handle as much as buf_size at a time. */
+    nb_sectors = MIN(s->buf_size >> BDRV_SECTOR_BITS, nb_sectors);
+    assert(nb_sectors);
 
-        /* We have enough free space to copy these sectors.  */
-        bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);
+    if (s->cow_bitmap) {
+        ret += mirror_cow_align(s, &sector_num, &nb_sectors);
+    }
+    assert(nb_sectors << BDRV_SECTOR_BITS <= s->buf_size);
+    /* The sector range must meet granularity because:
+     * 1) Caller passes in aligned values;
+     * 2) mirror_cow_align is used only when target cluster is larger. */
+    assert(!(sector_num % sectors_per_chunk));
+    nb_chunks = DIV_ROUND_UP(nb_sectors, sectors_per_chunk);
 
-        nb_sectors += added_sectors;
-        nb_chunks += added_chunks;
-        next_sector += added_sectors;
-        next_chunk += added_chunks;
-        if (!s->synced && s->common.speed) {
-            delay_ns = ratelimit_calculate_delay(&s->limit, added_sectors);
-        }
-    } while (delay_ns == 0 && next_sector < end);
+    while (s->buf_free_count < nb_chunks) {
+        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+        mirror_wait_for_io(s);
+    }
 
     /* Allocate a MirrorOp that is used as an AIO callback.  */
-    op = g_slice_new(MirrorOp);
+    op = g_new(MirrorOp, 1);
     op->s = s;
     op->sector_num = sector_num;
     op->nb_sectors = nb_sectors;
@@ -268,47 +260,158 @@ static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
      * from s->buf_free.
      */
     qemu_iovec_init(&op->qiov, nb_chunks);
-    next_sector = sector_num;
     while (nb_chunks-- > 0) {
         MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
-        size_t remaining = (nb_sectors * BDRV_SECTOR_SIZE) - op->qiov.size;
+        size_t remaining = nb_sectors * BDRV_SECTOR_SIZE - op->qiov.size;
 
         QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
         s->buf_free_count--;
         qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
-
-        /* Advance the HBitmapIter in parallel, so that we do not examine
-         * the same sector twice.
-         */
-        if (next_sector > hbitmap_next_sector
-            && bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) {
-            hbitmap_next_sector = hbitmap_iter_next(&s->hbi);
-        }
-
-        next_sector += sectors_per_chunk;
     }
 
-    bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num, nb_sectors);
-
     /* Copy the dirty cluster.  */
     s->in_flight++;
     s->sectors_in_flight += nb_sectors;
     trace_mirror_one_iteration(s, sector_num, nb_sectors);
 
-    ret = bdrv_get_block_status_above(source, NULL, sector_num,
-                                      nb_sectors, &pnum);
-    if (ret < 0 || pnum < nb_sectors ||
-            (ret & BDRV_BLOCK_DATA && !(ret & BDRV_BLOCK_ZERO))) {
-        bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
-                       mirror_read_complete, op);
-    } else if (ret & BDRV_BLOCK_ZERO) {
+    bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
+                   mirror_read_complete, op);
+    return ret;
+}
+
+static void mirror_do_zero_or_discard(MirrorBlockJob *s,
+                                      int64_t sector_num,
+                                      int nb_sectors,
+                                      bool is_discard)
+{
+    MirrorOp *op;
+
+    /* Allocate a MirrorOp that is used as an AIO callback. The qiov is zeroed
+     * so the freeing in mirror_iteration_done is nop. */
+    op = g_new0(MirrorOp, 1);
+    op->s = s;
+    op->sector_num = sector_num;
+    op->nb_sectors = nb_sectors;
+
+    s->in_flight++;
+    s->sectors_in_flight += nb_sectors;
+    if (is_discard) {
+        bdrv_aio_discard(s->target, sector_num, op->nb_sectors,
+                         mirror_write_complete, op);
+    } else {
         bdrv_aio_write_zeroes(s->target, sector_num, op->nb_sectors,
                               s->unmap ? BDRV_REQ_MAY_UNMAP : 0,
                               mirror_write_complete, op);
-    } else {
-        assert(!(ret & BDRV_BLOCK_DATA));
-        bdrv_aio_discard(s->target, sector_num, op->nb_sectors,
-                         mirror_write_complete, op);
+    }
+}
+
+static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
+{
+    BlockDriverState *source = s->common.bs;
+    int64_t sector_num, first_chunk;
+    uint64_t delay_ns = 0;
+    /* At least the first dirty chunk is mirrored in one iteration. */
+    int nb_chunks = 1;
+    int64_t end = s->bdev_length / BDRV_SECTOR_SIZE;
+    int sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+
+    sector_num = hbitmap_iter_next(&s->hbi);
+    if (sector_num < 0) {
+        bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi);
+        sector_num = hbitmap_iter_next(&s->hbi);
+        trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
+        assert(sector_num >= 0);
+    }
+
+    first_chunk = sector_num / sectors_per_chunk;
+    while (test_bit(first_chunk, s->in_flight_bitmap)) {
+        trace_mirror_yield_in_flight(s, first_chunk, s->in_flight);
+        mirror_wait_for_io(s);
+    }
+
+    /* Find the number of consective dirty chunks following the first dirty
+     * one, and wait for in flight requests in them. */
+    while (nb_chunks * sectors_per_chunk < (s->buf_size >> BDRV_SECTOR_BITS)) {
+        int64_t hbitmap_next;
+        int64_t next_sector = sector_num + nb_chunks * sectors_per_chunk;
+        int64_t next_chunk = next_sector / sectors_per_chunk;
+        if (next_sector >= end ||
+            !bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) {
+            break;
+        }
+        if (test_bit(next_chunk, s->in_flight_bitmap)) {
+            break;
+        }
+
+        hbitmap_next = hbitmap_iter_next(&s->hbi);
+        if (hbitmap_next > next_sector || hbitmap_next < 0) {
+            /* The bitmap iterator's cache is stale, refresh it */
+            bdrv_set_dirty_iter(&s->hbi, next_sector);
+            hbitmap_next = hbitmap_iter_next(&s->hbi);
+        }
+        assert(hbitmap_next == next_sector);
+        nb_chunks++;
+    }
+
+    /* Clear dirty bits before querying the block status, because
+     * calling bdrv_get_block_status_above could yield - if some blocks are
+     * marked dirty in this window, we need to know.
+     */
+    bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num,
+                            nb_chunks * sectors_per_chunk);
+    bitmap_set(s->in_flight_bitmap, sector_num / sectors_per_chunk, nb_chunks);
+    while (nb_chunks > 0 && sector_num < end) {
+        int ret;
+        int io_sectors;
+        BlockDriverState *file;
+        enum MirrorMethod {
+            MIRROR_METHOD_COPY,
+            MIRROR_METHOD_ZERO,
+            MIRROR_METHOD_DISCARD
+        } mirror_method = MIRROR_METHOD_COPY;
+
+        assert(!(sector_num % sectors_per_chunk));
+        ret = bdrv_get_block_status_above(source, NULL, sector_num,
+                                          nb_chunks * sectors_per_chunk,
+                                          &io_sectors, &file);
+        if (ret < 0) {
+            io_sectors = nb_chunks * sectors_per_chunk;
+        }
+
+        io_sectors -= io_sectors % sectors_per_chunk;
+        if (io_sectors < sectors_per_chunk) {
+            io_sectors = sectors_per_chunk;
+        } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
+            int64_t target_sector_num;
+            int target_nb_sectors;
+            bdrv_round_to_clusters(s->target, sector_num, io_sectors,
+                                   &target_sector_num, &target_nb_sectors);
+            if (target_sector_num == sector_num &&
+                target_nb_sectors == io_sectors) {
+                mirror_method = ret & BDRV_BLOCK_ZERO ?
+                                    MIRROR_METHOD_ZERO :
+                                    MIRROR_METHOD_DISCARD;
+            }
+        }
+
+        mirror_clip_sectors(s, sector_num, &io_sectors);
+        switch (mirror_method) {
+        case MIRROR_METHOD_COPY:
+            io_sectors = mirror_do_read(s, sector_num, io_sectors);
+            break;
+        case MIRROR_METHOD_ZERO:
+            mirror_do_zero_or_discard(s, sector_num, io_sectors, false);
+            break;
+        case MIRROR_METHOD_DISCARD:
+            mirror_do_zero_or_discard(s, sector_num, io_sectors, true);
+            break;
+        default:
+            abort();
+        }
+        assert(io_sectors);
+        sector_num += io_sectors;
+        nb_chunks -= DIV_ROUND_UP(io_sectors, sectors_per_chunk);
+        delay_ns += ratelimit_calculate_delay(&s->limit, io_sectors);
     }
     return delay_ns;
 }
@@ -333,7 +436,7 @@ static void mirror_free_init(MirrorBlockJob *s)
 static void mirror_drain(MirrorBlockJob *s)
 {
     while (s->in_flight > 0) {
-        qemu_coroutine_yield();
+        mirror_wait_for_io(s);
     }
 }
 
@@ -346,6 +449,11 @@ static void mirror_exit(BlockJob *job, void *opaque)
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
     MirrorExitData *data = opaque;
     AioContext *replace_aio_context = NULL;
+    BlockDriverState *src = s->common.bs;
+
+    /* Make sure that the source BDS doesn't go away before we called
+     * block_job_completed(). */
+    bdrv_ref(src);
 
     if (s->to_replace) {
         replace_aio_context = bdrv_get_aio_context(s->to_replace);
@@ -357,18 +465,22 @@ static void mirror_exit(BlockJob *job, void *opaque)
         if (s->to_replace) {
             to_replace = s->to_replace;
         }
+
+        /* This was checked in mirror_start_job(), but meanwhile one of the
+         * nodes could have been newly attached to a BlockBackend. */
+        if (to_replace->blk && s->target->blk) {
+            error_report("block job: Can't create node with two BlockBackends");
+            data->ret = -EINVAL;
+            goto out;
+        }
+
         if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) {
             bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL);
         }
-        bdrv_swap(s->target, to_replace);
-        if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) {
-            /* drop the bs loop chain formed by the swap: break the loop then
-             * trigger the unref from the top one */
-            BlockDriverState *p = s->base->backing_hd;
-            bdrv_set_backing_hd(s->base, NULL);
-            bdrv_unref(p);
-        }
+        bdrv_replace_in_backing_chain(to_replace, s->target);
     }
+
+out:
     if (s->to_replace) {
         bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
         error_free(s->replace_blocker);
@@ -378,9 +490,15 @@ static void mirror_exit(BlockJob *job, void *opaque)
         aio_context_release(replace_aio_context);
     }
     g_free(s->replaces);
+    bdrv_op_unblock_all(s->target, s->common.blocker);
     bdrv_unref(s->target);
     block_job_completed(&s->common, data->ret);
     g_free(data);
+    bdrv_drained_end(src);
+    if (qemu_get_aio_context() == bdrv_get_aio_context(src)) {
+        aio_enable_external(iohandler_get_aio_context());
+    }
+    bdrv_unref(src);
 }
 
 static void coroutine_fn mirror_run(void *opaque)
@@ -395,6 +513,7 @@ static void coroutine_fn mirror_run(void *opaque)
                                  checking for a NULL string */
     int ret = 0;
     int n;
+    int target_cluster_size = BDRV_SECTOR_SIZE;
 
     if (block_job_is_cancelled(&s->common)) {
         goto immediate_exit;
@@ -424,16 +543,16 @@ static void coroutine_fn mirror_run(void *opaque)
      */
     bdrv_get_backing_filename(s->target, backing_filename,
                               sizeof(backing_filename));
-    if (backing_filename[0] && !s->target->backing_hd) {
-        ret = bdrv_get_info(s->target, &bdi);
-        if (ret < 0) {
-            goto immediate_exit;
-        }
-        if (s->granularity < bdi.cluster_size) {
-            s->buf_size = MAX(s->buf_size, bdi.cluster_size);
-            s->cow_bitmap = bitmap_new(length);
-        }
+    if (!bdrv_get_info(s->target, &bdi) && bdi.cluster_size) {
+        target_cluster_size = bdi.cluster_size;
     }
+    if (backing_filename[0] && !s->target->backing
+        && s->granularity < target_cluster_size) {
+        s->buf_size = MAX(s->buf_size, target_cluster_size);
+        s->cow_bitmap = bitmap_new(length);
+    }
+    s->target_cluster_sectors = target_cluster_size >> BDRV_SECTOR_BITS;
+    s->max_iov = MIN(s->common.bs->bl.max_iov, s->target->bl.max_iov);
 
     end = s->bdev_length / BDRV_SECTOR_SIZE;
     s->buf = qemu_try_blockalign(bs, s->buf_size);
@@ -448,6 +567,8 @@ static void coroutine_fn mirror_run(void *opaque)
     if (!s->is_none_mode) {
         /* First part, loop on the sectors and initialize the dirty bitmap.  */
         BlockDriverState *base = s->base;
+        bool mark_all_dirty = s->base == NULL && !bdrv_has_zero_init(s->target);
+
         for (sector_num = 0; sector_num < end; ) {
             /* Just to make sure we are not exceeding int limit. */
             int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS,
@@ -470,7 +591,7 @@ static void coroutine_fn mirror_run(void *opaque)
             }
 
             assert(n > 0);
-            if (ret == 1) {
+            if (ret == 1 || mark_all_dirty) {
                 bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n);
             }
             sector_num += n;
@@ -506,7 +627,7 @@ static void coroutine_fn mirror_run(void *opaque)
             if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 ||
                 (cnt == 0 && s->in_flight > 0)) {
                 trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt);
-                qemu_coroutine_yield();
+                mirror_wait_for_io(s);
                 continue;
             } else if (cnt != 0) {
                 delay_ns = mirror_iteration(s);
@@ -549,7 +670,7 @@ static void coroutine_fn mirror_run(void *opaque)
              * mirror_populate runs.
              */
             trace_mirror_before_drain(s, cnt);
-            bdrv_drain(bs);
+            bdrv_co_drain(bs);
             cnt = bdrv_get_dirty_count(s->dirty_bitmap);
         }
 
@@ -589,10 +710,21 @@ immediate_exit:
     g_free(s->cow_bitmap);
     g_free(s->in_flight_bitmap);
     bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
-    bdrv_iostatus_disable(s->target);
+    if (s->target->blk) {
+        blk_iostatus_disable(s->target->blk);
+    }
 
     data = g_malloc(sizeof(*data));
     data->ret = ret;
+    /* Before we switch to target in mirror_exit, make sure data doesn't
+     * change. */
+    bdrv_drained_begin(s->common.bs);
+    if (qemu_get_aio_context() == bdrv_get_aio_context(bs)) {
+        /* FIXME: virtio host notifiers run on iohandler_ctx, therefore the
+         * above bdrv_drained_end isn't enough to quiesce it. This is ugly, we
+         * need a block layer API change to achieve this. */
+        aio_disable_external(iohandler_get_aio_context());
+    }
     block_job_defer_to_main_loop(&s->common, mirror_exit, data);
 }
 
@@ -611,7 +743,9 @@ static void mirror_iostatus_reset(BlockJob *job)
 {
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
 
-    bdrv_iostatus_reset(s->target);
+    if (s->target->blk) {
+        blk_iostatus_reset(s->target->blk);
+    }
 }
 
 static void mirror_complete(BlockJob *job, Error **errp)
@@ -620,14 +754,13 @@ static void mirror_complete(BlockJob *job, Error **errp)
     Error *local_err = NULL;
     int ret;
 
-    ret = bdrv_open_backing_file(s->target, NULL, &local_err);
+    ret = bdrv_open_backing_file(s->target, NULL, "backing", &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         return;
     }
     if (!s->synced) {
-        error_setg(errp, QERR_BLOCK_JOB_NOT_READY,
-                   bdrv_get_device_name(job->bs));
+        error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id);
         return;
     }
 
@@ -635,9 +768,9 @@ static void mirror_complete(BlockJob *job, Error **errp)
     if (s->replaces) {
         AioContext *replace_aio_context;
 
-        s->to_replace = check_to_replace_node(s->replaces, &local_err);
+        s->to_replace = bdrv_find_node(s->replaces);
         if (!s->to_replace) {
-            error_propagate(errp, local_err);
+            error_setg(errp, "Node name '%s' not found", s->replaces);
             return;
         }
 
@@ -686,6 +819,7 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
                              bool is_none_mode, BlockDriverState *base)
 {
     MirrorBlockJob *s;
+    BlockDriverState *replaced_bs;
 
     if (granularity == 0) {
         granularity = bdrv_get_default_bitmap_granularity(target);
@@ -695,7 +829,7 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
 
     if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
          on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-source-error");
         return;
     }
@@ -709,6 +843,21 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
         buf_size = DEFAULT_MIRROR_BUF_SIZE;
     }
 
+    /* We can't support this case as long as the block layer can't handle
+     * multiple BlockBackends per BlockDriverState. */
+    if (replaces) {
+        replaced_bs = bdrv_lookup_bs(replaces, replaces, errp);
+        if (replaced_bs == NULL) {
+            return;
+        }
+    } else {
+        replaced_bs = bs;
+    }
+    if (replaced_bs->blk && target->blk) {
+        error_setg(errp, "Can't create node with two BlockBackends");
+        return;
+    }
+
     s = block_job_create(driver, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
@@ -727,12 +876,16 @@ static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
     s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
     if (!s->dirty_bitmap) {
         g_free(s->replaces);
-        block_job_release(bs);
+        block_job_unref(&s->common);
         return;
     }
-    bdrv_set_enable_write_cache(s->target, true);
-    bdrv_set_on_error(s->target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(s->target);
+
+    bdrv_op_block_all(s->target, s->common.blocker);
+
+    if (s->target->blk) {
+        blk_set_on_error(s->target->blk, on_target_error, on_target_error);
+        blk_iostatus_enable(s->target->blk);
+    }
     s->common.co = qemu_coroutine_create(mirror_run);
     trace_mirror_start(bs, s, s->common.co, opaque);
     qemu_coroutine_enter(s->common.co, s);
@@ -755,7 +908,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
         return;
     }
     is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
-    base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL;
+    base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
     mirror_start_job(bs, target, replaces,
                      speed, granularity, buf_size,
                      on_source_error, on_target_error, unmap, cb, opaque, errp,
diff --git a/qemu/block/nbd-client.c b/qemu/block/nbd-client.c
index e1bb9198c..878e879ac 100644
--- a/qemu/block/nbd-client.c
+++ b/qemu/block/nbd-client.c
@@ -26,8 +26,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "nbd-client.h"
-#include "qemu/sockets.h"
 
 #define HANDLE_TO_INDEX(bs, handle) ((handle) ^ ((uint64_t)(intptr_t)bs))
 #define INDEX_TO_HANDLE(bs, index)  ((index)  ^ ((uint64_t)(intptr_t)bs))
@@ -47,13 +47,21 @@ static void nbd_teardown_connection(BlockDriverState *bs)
 {
     NbdClientSession *client = nbd_get_client_session(bs);
 
+    if (!client->ioc) { /* Already closed */
+        return;
+    }
+
     /* finish any pending coroutines */
-    shutdown(client->sock, 2);
+    qio_channel_shutdown(client->ioc,
+                         QIO_CHANNEL_SHUTDOWN_BOTH,
+                         NULL);
     nbd_recv_coroutines_enter_all(client);
 
     nbd_client_detach_aio_context(bs);
-    closesocket(client->sock);
-    client->sock = -1;
+    object_unref(OBJECT(client->sioc));
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->ioc = NULL;
 }
 
 static void nbd_reply_ready(void *opaque)
@@ -63,12 +71,16 @@ static void nbd_reply_ready(void *opaque)
     uint64_t i;
     int ret;
 
+    if (!s->ioc) { /* Already closed */
+        return;
+    }
+
     if (s->reply.handle == 0) {
         /* No reply already in flight.  Fetch a header.  It is possible
          * that another thread has done the same thing in parallel, so
          * the socket is not readable anymore.
          */
-        ret = nbd_receive_reply(s->sock, &s->reply);
+        ret = nbd_receive_reply(s->ioc, &s->reply);
         if (ret == -EAGAIN) {
             return;
         }
@@ -119,32 +131,36 @@ static int nbd_co_send_request(BlockDriverState *bs,
         }
     }
 
+    g_assert(qemu_in_coroutine());
     assert(i < MAX_NBD_REQUESTS);
     request->handle = INDEX_TO_HANDLE(s, i);
+
+    if (!s->ioc) {
+        qemu_co_mutex_unlock(&s->send_mutex);
+        return -EPIPE;
+    }
+
     s->send_coroutine = qemu_coroutine_self();
     aio_context = bdrv_get_aio_context(bs);
 
-    aio_set_fd_handler(aio_context, s->sock,
+    aio_set_fd_handler(aio_context, s->sioc->fd, false,
                        nbd_reply_ready, nbd_restart_write, bs);
     if (qiov) {
-        if (!s->is_unix) {
-            socket_set_cork(s->sock, 1);
-        }
-        rc = nbd_send_request(s->sock, request);
+        qio_channel_set_cork(s->ioc, true);
+        rc = nbd_send_request(s->ioc, request);
         if (rc >= 0) {
-            ret = qemu_co_sendv(s->sock, qiov->iov, qiov->niov,
-                                offset, request->len);
+            ret = nbd_wr_syncv(s->ioc, qiov->iov, qiov->niov,
+                               offset, request->len, 0);
             if (ret != request->len) {
                 rc = -EIO;
             }
         }
-        if (!s->is_unix) {
-            socket_set_cork(s->sock, 0);
-        }
+        qio_channel_set_cork(s->ioc, false);
     } else {
-        rc = nbd_send_request(s->sock, request);
+        rc = nbd_send_request(s->ioc, request);
     }
-    aio_set_fd_handler(aio_context, s->sock, nbd_reply_ready, NULL, bs);
+    aio_set_fd_handler(aio_context, s->sioc->fd, false,
+                       nbd_reply_ready, NULL, bs);
     s->send_coroutine = NULL;
     qemu_co_mutex_unlock(&s->send_mutex);
     return rc;
@@ -160,12 +176,13 @@ static void nbd_co_receive_reply(NbdClientSession *s,
      * peek at the next reply and avoid yielding if it's ours?  */
     qemu_coroutine_yield();
     *reply = s->reply;
-    if (reply->handle != request->handle) {
+    if (reply->handle != request->handle ||
+        !s->ioc) {
         reply->error = EIO;
     } else {
         if (qiov && reply->error == 0) {
-            ret = qemu_co_recvv(s->sock, qiov->iov, qiov->niov,
-                                offset, request->len);
+            ret = nbd_wr_syncv(s->ioc, qiov->iov, qiov->niov,
+                               offset, request->len, 1);
             if (ret != request->len) {
                 reply->error = EIO;
             }
@@ -226,15 +243,15 @@ static int nbd_co_readv_1(BlockDriverState *bs, int64_t sector_num,
 
 static int nbd_co_writev_1(BlockDriverState *bs, int64_t sector_num,
                            int nb_sectors, QEMUIOVector *qiov,
-                           int offset)
+                           int offset, int *flags)
 {
     NbdClientSession *client = nbd_get_client_session(bs);
     struct nbd_request request = { .type = NBD_CMD_WRITE };
     struct nbd_reply reply;
     ssize_t ret;
 
-    if (!bdrv_enable_write_cache(bs) &&
-        (client->nbdflags & NBD_FLAG_SEND_FUA)) {
+    if ((*flags & BDRV_REQ_FUA) && (client->nbdflags & NBD_FLAG_SEND_FUA)) {
+        *flags &= ~BDRV_REQ_FUA;
         request.type |= NBD_CMD_FLAG_FUA;
     }
 
@@ -274,12 +291,13 @@ int nbd_client_co_readv(BlockDriverState *bs, int64_t sector_num,
 }
 
 int nbd_client_co_writev(BlockDriverState *bs, int64_t sector_num,
-                         int nb_sectors, QEMUIOVector *qiov)
+                         int nb_sectors, QEMUIOVector *qiov, int *flags)
 {
     int offset = 0;
     int ret;
     while (nb_sectors > NBD_MAX_SECTORS) {
-        ret = nbd_co_writev_1(bs, sector_num, NBD_MAX_SECTORS, qiov, offset);
+        ret = nbd_co_writev_1(bs, sector_num, NBD_MAX_SECTORS, qiov, offset,
+                              flags);
         if (ret < 0) {
             return ret;
         }
@@ -287,7 +305,7 @@ int nbd_client_co_writev(BlockDriverState *bs, int64_t sector_num,
         sector_num += NBD_MAX_SECTORS;
         nb_sectors -= NBD_MAX_SECTORS;
     }
-    return nbd_co_writev_1(bs, sector_num, nb_sectors, qiov, offset);
+    return nbd_co_writev_1(bs, sector_num, nb_sectors, qiov, offset, flags);
 }
 
 int nbd_client_co_flush(BlockDriverState *bs)
@@ -301,10 +319,6 @@ int nbd_client_co_flush(BlockDriverState *bs)
         return 0;
     }
 
-    if (client->nbdflags & NBD_FLAG_SEND_FUA) {
-        request.type |= NBD_CMD_FLAG_FUA;
-    }
-
     request.from = 0;
     request.len = 0;
 
@@ -348,14 +362,15 @@ int nbd_client_co_discard(BlockDriverState *bs, int64_t sector_num,
 void nbd_client_detach_aio_context(BlockDriverState *bs)
 {
     aio_set_fd_handler(bdrv_get_aio_context(bs),
-                       nbd_get_client_session(bs)->sock, NULL, NULL, NULL);
+                       nbd_get_client_session(bs)->sioc->fd,
+                       false, NULL, NULL, NULL);
 }
 
 void nbd_client_attach_aio_context(BlockDriverState *bs,
                                    AioContext *new_context)
 {
-    aio_set_fd_handler(new_context, nbd_get_client_session(bs)->sock,
-                       nbd_reply_ready, NULL, bs);
+    aio_set_fd_handler(new_context, nbd_get_client_session(bs)->sioc->fd,
+                       false, nbd_reply_ready, NULL, bs);
 }
 
 void nbd_client_close(BlockDriverState *bs)
@@ -367,16 +382,20 @@ void nbd_client_close(BlockDriverState *bs)
         .len = 0
     };
 
-    if (client->sock == -1) {
+    if (client->ioc == NULL) {
         return;
     }
 
-    nbd_send_request(client->sock, &request);
+    nbd_send_request(client->ioc, &request);
 
     nbd_teardown_connection(bs);
 }
 
-int nbd_client_init(BlockDriverState *bs, int sock, const char *export,
+int nbd_client_init(BlockDriverState *bs,
+                    QIOChannelSocket *sioc,
+                    const char *export,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *hostname,
                     Error **errp)
 {
     NbdClientSession *client = nbd_get_client_session(bs);
@@ -384,22 +403,32 @@ int nbd_client_init(BlockDriverState *bs, int sock, const char *export,
 
     /* NBD handshake */
     logout("session init %s\n", export);
-    qemu_set_block(sock);
-    ret = nbd_receive_negotiate(sock, export,
-                                &client->nbdflags, &client->size, errp);
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL);
+
+    ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), export,
+                                &client->nbdflags,
+                                tlscreds, hostname,
+                                &client->ioc,
+                                &client->size, errp);
     if (ret < 0) {
         logout("Failed to negotiate with the NBD server\n");
-        closesocket(sock);
         return ret;
     }
 
     qemu_co_mutex_init(&client->send_mutex);
     qemu_co_mutex_init(&client->free_sema);
-    client->sock = sock;
+    client->sioc = sioc;
+    object_ref(OBJECT(client->sioc));
+
+    if (!client->ioc) {
+        client->ioc = QIO_CHANNEL(sioc);
+        object_ref(OBJECT(client->ioc));
+    }
 
     /* Now that we're connected, set the socket to be non-blocking and
      * kick the reply mechanism.  */
-    qemu_set_nonblock(sock);
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL);
+
     nbd_client_attach_aio_context(bs, bdrv_get_aio_context(bs));
 
     logout("Established connection with NBD server\n");
diff --git a/qemu/block/nbd-client.h b/qemu/block/nbd-client.h
index e8413408b..bc7aec079 100644
--- a/qemu/block/nbd-client.h
+++ b/qemu/block/nbd-client.h
@@ -4,6 +4,7 @@
 #include "qemu-common.h"
 #include "block/nbd.h"
 #include "block/block_int.h"
+#include "io/channel-socket.h"
 
 /* #define DEBUG_NBD */
 
@@ -17,7 +18,8 @@
 #define MAX_NBD_REQUESTS    16
 
 typedef struct NbdClientSession {
-    int sock;
+    QIOChannelSocket *sioc; /* The master data channel */
+    QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
     uint32_t nbdflags;
     off_t size;
 
@@ -34,7 +36,11 @@ typedef struct NbdClientSession {
 
 NbdClientSession *nbd_get_client_session(BlockDriverState *bs);
 
-int nbd_client_init(BlockDriverState *bs, int sock, const char *export_name,
+int nbd_client_init(BlockDriverState *bs,
+                    QIOChannelSocket *sock,
+                    const char *export_name,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *hostname,
                     Error **errp);
 void nbd_client_close(BlockDriverState *bs);
 
@@ -42,7 +48,7 @@ int nbd_client_co_discard(BlockDriverState *bs, int64_t sector_num,
                           int nb_sectors);
 int nbd_client_co_flush(BlockDriverState *bs);
 int nbd_client_co_writev(BlockDriverState *bs, int64_t sector_num,
-                         int nb_sectors, QEMUIOVector *qiov);
+                         int nb_sectors, QEMUIOVector *qiov, int *flags);
 int nbd_client_co_readv(BlockDriverState *bs, int64_t sector_num,
                         int nb_sectors, QEMUIOVector *qiov);
 
diff --git a/qemu/block/nbd.c b/qemu/block/nbd.c
index 217618612..f7ea3b360 100644
--- a/qemu/block/nbd.c
+++ b/qemu/block/nbd.c
@@ -26,24 +26,22 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/nbd-client.h"
+#include "qapi/error.h"
 #include "qemu/uri.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
-#include "qemu/sockets.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qstring.h"
-
-#include <sys/types.h>
-#include <unistd.h>
+#include "qemu/cutils.h"
 
 #define EN_OPTSTR ":exportname="
 
 typedef struct BDRVNBDState {
     NbdClientSession client;
-    QemuOpts *socket_opts;
 } BDRVNBDState;
 
 static int nbd_parse_uri(const char *filename, QDict *options)
@@ -190,10 +188,10 @@ out:
     g_free(file);
 }
 
-static void nbd_config(BDRVNBDState *s, QDict *options, char **export,
-                       Error **errp)
+static SocketAddress *nbd_config(BDRVNBDState *s, QDict *options, char **export,
+                                 Error **errp)
 {
-    Error *local_err = NULL;
+    SocketAddress *saddr;
 
     if (qdict_haskey(options, "path") == qdict_haskey(options, "host")) {
         if (qdict_haskey(options, "path")) {
@@ -201,28 +199,39 @@ static void nbd_config(BDRVNBDState *s, QDict *options, char **export,
         } else {
             error_setg(errp, "one of path and host must be specified.");
         }
-        return;
+        return NULL;
     }
 
-    s->client.is_unix = qdict_haskey(options, "path");
-    s->socket_opts = qemu_opts_create(&socket_optslist, NULL, 0,
-                                      &error_abort);
+    saddr = g_new0(SocketAddress, 1);
 
-    qemu_opts_absorb_qdict(s->socket_opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
+    if (qdict_haskey(options, "path")) {
+        UnixSocketAddress *q_unix;
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        q_unix = saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        q_unix->path = g_strdup(qdict_get_str(options, "path"));
+        qdict_del(options, "path");
+    } else {
+        InetSocketAddress *inet;
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+        inet->host = g_strdup(qdict_get_str(options, "host"));
+        if (!qdict_get_try_str(options, "port")) {
+            inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
+        } else {
+            inet->port = g_strdup(qdict_get_str(options, "port"));
+        }
+        qdict_del(options, "host");
+        qdict_del(options, "port");
     }
 
-    if (!qemu_opt_get(s->socket_opts, "port")) {
-        qemu_opt_set_number(s->socket_opts, "port", NBD_DEFAULT_PORT,
-                            &error_abort);
-    }
+    s->client.is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
 
     *export = g_strdup(qdict_get_try_str(options, "export"));
     if (*export) {
         qdict_del(options, "export");
     }
+
+    return saddr;
 }
 
 NbdClientSession *nbd_get_client_session(BlockDriverState *bs)
@@ -231,57 +240,113 @@ NbdClientSession *nbd_get_client_session(BlockDriverState *bs)
     return &s->client;
 }
 
-static int nbd_establish_connection(BlockDriverState *bs, Error **errp)
+static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr,
+                                                  Error **errp)
 {
-    BDRVNBDState *s = bs->opaque;
-    int sock;
+    QIOChannelSocket *sioc;
+    Error *local_err = NULL;
 
-    if (s->client.is_unix) {
-        sock = unix_connect_opts(s->socket_opts, errp, NULL, NULL);
-    } else {
-        sock = inet_connect_opts(s->socket_opts, errp, NULL, NULL);
-        if (sock >= 0) {
-            socket_set_nodelay(sock);
-        }
+    sioc = qio_channel_socket_new();
+
+    qio_channel_socket_connect_sync(sioc,
+                                    saddr,
+                                    &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return NULL;
     }
 
-    /* Failed to establish connection */
-    if (sock < 0) {
-        logout("Failed to establish connection to NBD server\n");
-        return -EIO;
+    qio_channel_set_delay(QIO_CHANNEL(sioc), false);
+
+    return sioc;
+}
+
+
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
     }
 
-    return sock;
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a client endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
 }
 
+
 static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
     BDRVNBDState *s = bs->opaque;
     char *export = NULL;
-    int result, sock;
-    Error *local_err = NULL;
+    QIOChannelSocket *sioc = NULL;
+    SocketAddress *saddr;
+    const char *tlscredsid;
+    QCryptoTLSCreds *tlscreds = NULL;
+    const char *hostname = NULL;
+    int ret = -EINVAL;
 
     /* Pop the config into our state object. Exit if invalid. */
-    nbd_config(s, options, &export, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return -EINVAL;
+    saddr = nbd_config(s, options, &export, errp);
+    if (!saddr) {
+        goto error;
+    }
+
+    tlscredsid = g_strdup(qdict_get_try_str(options, "tls-creds"));
+    if (tlscredsid) {
+        qdict_del(options, "tls-creds");
+        tlscreds = nbd_get_tls_creds(tlscredsid, errp);
+        if (!tlscreds) {
+            goto error;
+        }
+
+        if (saddr->type != SOCKET_ADDRESS_KIND_INET) {
+            error_setg(errp, "TLS only supported over IP sockets");
+            goto error;
+        }
+        hostname = saddr->u.inet.data->host;
     }
 
     /* establish TCP connection, return error if it fails
      * TODO: Configurable retry-until-timeout behaviour.
      */
-    sock = nbd_establish_connection(bs, errp);
-    if (sock < 0) {
-        g_free(export);
-        return sock;
+    sioc = nbd_establish_connection(saddr, errp);
+    if (!sioc) {
+        ret = -ECONNREFUSED;
+        goto error;
     }
 
     /* NBD handshake */
-    result = nbd_client_init(bs, sock, export, errp);
+    ret = nbd_client_init(bs, sioc, export,
+                          tlscreds, hostname, errp);
+ error:
+    if (sioc) {
+        object_unref(OBJECT(sioc));
+    }
+    if (tlscreds) {
+        object_unref(OBJECT(tlscreds));
+    }
+    qapi_free_SocketAddress(saddr);
     g_free(export);
-    return result;
+    return ret;
 }
 
 static int nbd_co_readv(BlockDriverState *bs, int64_t sector_num,
@@ -290,10 +355,29 @@ static int nbd_co_readv(BlockDriverState *bs, int64_t sector_num,
     return nbd_client_co_readv(bs, sector_num, nb_sectors, qiov);
 }
 
+static int nbd_co_writev_flags(BlockDriverState *bs, int64_t sector_num,
+                               int nb_sectors, QEMUIOVector *qiov, int flags)
+{
+    int ret;
+
+    ret = nbd_client_co_writev(bs, sector_num, nb_sectors, qiov, &flags);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* The flag wasn't sent to the server, so we need to emulate it with an
+     * explicit flush */
+    if (flags & BDRV_REQ_FUA) {
+        ret = nbd_client_co_flush(bs);
+    }
+
+    return ret;
+}
+
 static int nbd_co_writev(BlockDriverState *bs, int64_t sector_num,
                          int nb_sectors, QEMUIOVector *qiov)
 {
-    return nbd_client_co_writev(bs, sector_num, nb_sectors, qiov);
+    return nbd_co_writev_flags(bs, sector_num, nb_sectors, qiov, 0);
 }
 
 static int nbd_co_flush(BlockDriverState *bs)
@@ -315,9 +399,6 @@ static int nbd_co_discard(BlockDriverState *bs, int64_t sector_num,
 
 static void nbd_close(BlockDriverState *bs)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    qemu_opts_del(s->socket_opts);
     nbd_client_close(bs);
 }
 
@@ -339,13 +420,14 @@ static void nbd_attach_aio_context(BlockDriverState *bs,
     nbd_client_attach_aio_context(bs, new_context);
 }
 
-static void nbd_refresh_filename(BlockDriverState *bs)
+static void nbd_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     QDict *opts = qdict_new();
-    const char *path   = qdict_get_try_str(bs->options, "path");
-    const char *host   = qdict_get_try_str(bs->options, "host");
-    const char *port   = qdict_get_try_str(bs->options, "port");
-    const char *export = qdict_get_try_str(bs->options, "export");
+    const char *path   = qdict_get_try_str(options, "path");
+    const char *host   = qdict_get_try_str(options, "host");
+    const char *port   = qdict_get_try_str(options, "port");
+    const char *export = qdict_get_try_str(options, "export");
+    const char *tlscreds = qdict_get_try_str(options, "tls-creds");
 
     qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("nbd")));
 
@@ -380,6 +462,9 @@ static void nbd_refresh_filename(BlockDriverState *bs)
     if (export) {
         qdict_put_obj(opts, "export", QOBJECT(qstring_from_str(export)));
     }
+    if (tlscreds) {
+        qdict_put_obj(opts, "tls-creds", QOBJECT(qstring_from_str(tlscreds)));
+    }
 
     bs->full_open_options = opts;
 }
@@ -392,6 +477,8 @@ static BlockDriver bdrv_nbd = {
     .bdrv_file_open             = nbd_open,
     .bdrv_co_readv              = nbd_co_readv,
     .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_writev_flags       = nbd_co_writev_flags,
+    .supported_write_flags      = BDRV_REQ_FUA,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
     .bdrv_co_discard            = nbd_co_discard,
@@ -410,6 +497,8 @@ static BlockDriver bdrv_nbd_tcp = {
     .bdrv_file_open             = nbd_open,
     .bdrv_co_readv              = nbd_co_readv,
     .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_writev_flags       = nbd_co_writev_flags,
+    .supported_write_flags      = BDRV_REQ_FUA,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
     .bdrv_co_discard            = nbd_co_discard,
@@ -428,6 +517,8 @@ static BlockDriver bdrv_nbd_unix = {
     .bdrv_file_open             = nbd_open,
     .bdrv_co_readv              = nbd_co_readv,
     .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_writev_flags       = nbd_co_writev_flags,
+    .supported_write_flags      = BDRV_REQ_FUA,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
     .bdrv_co_discard            = nbd_co_discard,
diff --git a/qemu/block/nfs.c b/qemu/block/nfs.c
index c026ff688..9f51cc3f1 100644
--- a/qemu/block/nfs.c
+++ b/qemu/block/nfs.c
@@ -22,20 +22,23 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <poll.h>
 #include "qemu-common.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "trace.h"
 #include "qemu/iov.h"
 #include "qemu/uri.h"
+#include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
 #include <nfsc/libnfs.h>
 
 #define QEMU_NFS_MAX_READAHEAD_SIZE 1048576
+#define QEMU_NFS_MAX_DEBUG_LEVEL 2
 
 typedef struct NFSClient {
     struct nfs_context *context;
@@ -43,6 +46,7 @@ typedef struct NFSClient {
     int events;
     bool has_zero_init;
     AioContext *aio_context;
+    blkcnt_t st_blocks;
 } NFSClient;
 
 typedef struct NFSRPC {
@@ -62,11 +66,10 @@ static void nfs_set_events(NFSClient *client)
 {
     int ev = nfs_which_events(client->context);
     if (ev != client->events) {
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false,
                            (ev & POLLIN) ? nfs_process_read : NULL,
-                           (ev & POLLOUT) ? nfs_process_write : NULL,
-                           client);
+                           (ev & POLLOUT) ? nfs_process_write : NULL, client);
 
     }
     client->events = ev;
@@ -241,9 +244,8 @@ static void nfs_detach_aio_context(BlockDriverState *bs)
 {
     NFSClient *client = bs->opaque;
 
-    aio_set_fd_handler(client->aio_context,
-                       nfs_get_fd(client->context),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                       false, NULL, NULL, NULL);
     client->events = 0;
 }
 
@@ -262,9 +264,8 @@ static void nfs_client_close(NFSClient *client)
         if (client->fh) {
             nfs_close(client->context, client->fh);
         }
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
-                           NULL, NULL, NULL);
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false, NULL, NULL, NULL);
         nfs_destroy_context(client->context);
     }
     memset(client, 0, sizeof(NFSClient));
@@ -336,6 +337,17 @@ static int64_t nfs_client_open(NFSClient *client, const char *filename,
             }
             nfs_set_readahead(client->context, val);
 #endif
+#ifdef LIBNFS_FEATURE_DEBUG
+        } else if (!strcmp(qp->p[i].name, "debug")) {
+            /* limit the maximum debug level to avoid potential flooding
+             * of our log files. */
+            if (val > QEMU_NFS_MAX_DEBUG_LEVEL) {
+                error_report("NFS Warning: Limiting NFS debug level"
+                             " to %d", QEMU_NFS_MAX_DEBUG_LEVEL);
+                val = QEMU_NFS_MAX_DEBUG_LEVEL;
+            }
+            nfs_set_debug(client->context, val);
+#endif
         } else {
             error_setg(errp, "Unknown NFS parameter name: %s",
                        qp->p[i].name);
@@ -374,6 +386,7 @@ static int64_t nfs_client_open(NFSClient *client, const char *filename,
     }
 
     ret = DIV_ROUND_UP(st.st_size, BDRV_SECTOR_SIZE);
+    client->st_blocks = st.st_blocks;
     client->has_zero_init = S_ISREG(st.st_mode);
     goto out;
 fail:
@@ -464,6 +477,11 @@ static int64_t nfs_get_allocated_file_size(BlockDriverState *bs)
     NFSRPC task = {0};
     struct stat st;
 
+    if (bdrv_is_read_only(bs) &&
+        !(bs->open_flags & BDRV_O_NOCACHE)) {
+        return client->st_blocks * 512;
+    }
+
     task.st = &st;
     if (nfs_fstat_async(client->context, client->fh, nfs_co_generic_cb,
                         &task) != 0) {
@@ -475,7 +493,7 @@ static int64_t nfs_get_allocated_file_size(BlockDriverState *bs)
         aio_poll(client->aio_context, true);
     }
 
-    return (task.ret < 0 ? task.ret : st.st_blocks * st.st_blksize);
+    return (task.ret < 0 ? task.ret : st.st_blocks * 512);
 }
 
 static int nfs_file_truncate(BlockDriverState *bs, int64_t offset)
@@ -484,6 +502,34 @@ static int nfs_file_truncate(BlockDriverState *bs, int64_t offset)
     return nfs_ftruncate(client->context, client->fh, offset);
 }
 
+/* Note that this will not re-establish a connection with the NFS server
+ * - it is effectively a NOP.  */
+static int nfs_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    NFSClient *client = state->bs->opaque;
+    struct stat st;
+    int ret = 0;
+
+    if (state->flags & BDRV_O_RDWR && bdrv_is_read_only(state->bs)) {
+        error_setg(errp, "Cannot open a read-only mount as read-write");
+        return -EACCES;
+    }
+
+    /* Update cache for read-only reopens */
+    if (!(state->flags & BDRV_O_RDWR)) {
+        ret = nfs_fstat(client->context, client->fh, &st);
+        if (ret < 0) {
+            error_setg(errp, "Failed to fstat file: %s",
+                       nfs_get_error(client->context));
+            return ret;
+        }
+        client->st_blocks = st.st_blocks;
+    }
+
+    return 0;
+}
+
 static BlockDriver bdrv_nfs = {
     .format_name                    = "nfs",
     .protocol_name                  = "nfs",
@@ -499,6 +545,7 @@ static BlockDriver bdrv_nfs = {
     .bdrv_file_open                 = nfs_file_open,
     .bdrv_close                     = nfs_file_close,
     .bdrv_create                    = nfs_file_create,
+    .bdrv_reopen_prepare            = nfs_reopen_prepare,
 
     .bdrv_co_readv                  = nfs_co_readv,
     .bdrv_co_writev                 = nfs_co_writev,
diff --git a/qemu/block/null.c b/qemu/block/null.c
index 7d083233f..396500bab 100644
--- a/qemu/block/null.c
+++ b/qemu/block/null.c
@@ -10,13 +10,17 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 
 #define NULL_OPT_LATENCY "latency-ns"
+#define NULL_OPT_ZEROES  "read-zeroes"
 
 typedef struct {
     int64_t length;
     int64_t latency_ns;
+    bool read_zeroes;
 } BDRVNullState;
 
 static QemuOptsList runtime_opts = {
@@ -39,6 +43,11 @@ static QemuOptsList runtime_opts = {
             .help = "nanoseconds (approximated) to wait "
                     "before completing request",
         },
+        {
+            .name = NULL_OPT_ZEROES,
+            .type = QEMU_OPT_BOOL,
+            .help = "return zeroes when read",
+        },
         { /* end of list */ }
     },
 };
@@ -60,6 +69,7 @@ static int null_file_open(BlockDriverState *bs, QDict *options, int flags,
         error_setg(errp, "latency-ns is invalid");
         ret = -EINVAL;
     }
+    s->read_zeroes = qemu_opt_get_bool(opts, NULL_OPT_ZEROES, false);
     qemu_opts_del(opts);
     return ret;
 }
@@ -89,6 +99,12 @@ static coroutine_fn int null_co_readv(BlockDriverState *bs,
                                       int64_t sector_num, int nb_sectors,
                                       QEMUIOVector *qiov)
 {
+    BDRVNullState *s = bs->opaque;
+
+    if (s->read_zeroes) {
+        qemu_iovec_memset(qiov, 0, 0, nb_sectors * BDRV_SECTOR_SIZE);
+    }
+
     return null_co_common(bs);
 }
 
@@ -158,6 +174,12 @@ static BlockAIOCB *null_aio_readv(BlockDriverState *bs,
                                   BlockCompletionFunc *cb,
                                   void *opaque)
 {
+    BDRVNullState *s = bs->opaque;
+
+    if (s->read_zeroes) {
+        qemu_iovec_memset(qiov, 0, 0, nb_sectors * BDRV_SECTOR_SIZE);
+    }
+
     return null_aio_common(bs, cb, opaque);
 }
 
@@ -183,6 +205,24 @@ static int null_reopen_prepare(BDRVReopenState *reopen_state,
     return 0;
 }
 
+static int64_t coroutine_fn null_co_get_block_status(BlockDriverState *bs,
+                                                     int64_t sector_num,
+                                                     int nb_sectors, int *pnum,
+                                                     BlockDriverState **file)
+{
+    BDRVNullState *s = bs->opaque;
+    off_t start = sector_num * BDRV_SECTOR_SIZE;
+
+    *pnum = nb_sectors;
+    *file = bs;
+
+    if (s->read_zeroes) {
+        return BDRV_BLOCK_OFFSET_VALID | start | BDRV_BLOCK_ZERO;
+    } else {
+        return BDRV_BLOCK_OFFSET_VALID | start;
+    }
+}
+
 static BlockDriver bdrv_null_co = {
     .format_name            = "null-co",
     .protocol_name          = "null-co",
@@ -196,6 +236,8 @@ static BlockDriver bdrv_null_co = {
     .bdrv_co_writev         = null_co_writev,
     .bdrv_co_flush_to_disk  = null_co_flush,
     .bdrv_reopen_prepare    = null_reopen_prepare,
+
+    .bdrv_co_get_block_status   = null_co_get_block_status,
 };
 
 static BlockDriver bdrv_null_aio = {
@@ -211,6 +253,8 @@ static BlockDriver bdrv_null_aio = {
     .bdrv_aio_writev        = null_aio_writev,
     .bdrv_aio_flush         = null_aio_flush,
     .bdrv_reopen_prepare    = null_reopen_prepare,
+
+    .bdrv_co_get_block_status   = null_co_get_block_status,
 };
 
 static void bdrv_null_init(void)
diff --git a/qemu/block/parallels.c b/qemu/block/parallels.c
index 046b56844..324ed43ac 100644
--- a/qemu/block/parallels.c
+++ b/qemu/block/parallels.c
@@ -27,8 +27,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "qemu/bitmap.h"
 #include "qapi/util.h"
@@ -61,7 +64,7 @@ typedef struct ParallelsHeader {
 typedef enum ParallelsPreallocMode {
     PRL_PREALLOC_MODE_FALLOCATE = 0,
     PRL_PREALLOC_MODE_TRUNCATE = 1,
-    PRL_PREALLOC_MODE_MAX = 2,
+    PRL_PREALLOC_MODE__MAX = 2,
 } ParallelsPreallocMode;
 
 static const char *prealloc_mode_lookup[] = {
@@ -202,13 +205,13 @@ static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
 
     to_allocate = (sector_num + *pnum + s->tracks - 1) / s->tracks - idx;
     space = to_allocate * s->tracks;
-    if (s->data_end + space > bdrv_getlength(bs->file) >> BDRV_SECTOR_BITS) {
+    if (s->data_end + space > bdrv_getlength(bs->file->bs) >> BDRV_SECTOR_BITS) {
         int ret;
         space += s->prealloc_size;
         if (s->prealloc_mode == PRL_PREALLOC_MODE_FALLOCATE) {
-            ret = bdrv_write_zeroes(bs->file, s->data_end, space, 0);
+            ret = bdrv_write_zeroes(bs->file->bs, s->data_end, space, 0);
         } else {
-            ret = bdrv_truncate(bs->file,
+            ret = bdrv_truncate(bs->file->bs,
                                 (s->data_end + space) << BDRV_SECTOR_BITS);
         }
         if (ret < 0) {
@@ -220,7 +223,7 @@ static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
         s->bat_bitmap[idx + i] = cpu_to_le32(s->data_end / s->off_multiplier);
         s->data_end += s->tracks;
         bitmap_set(s->bat_dirty_bmap,
-                   bat_entry_off(idx) / s->bat_dirty_block, 1);
+                   bat_entry_off(idx + i) / s->bat_dirty_block, 1);
     }
 
     return bat2sect(s, idx) + sector_num % s->tracks;
@@ -244,7 +247,8 @@ static coroutine_fn int parallels_co_flush_to_os(BlockDriverState *bs)
         if (off + to_write > s->header_size) {
             to_write = s->header_size - off;
         }
-        ret = bdrv_pwrite(bs->file, off, (uint8_t *)s->header + off, to_write);
+        ret = bdrv_pwrite(bs->file->bs, off, (uint8_t *)s->header + off,
+                          to_write);
         if (ret < 0) {
             qemu_co_mutex_unlock(&s->lock);
             return ret;
@@ -259,7 +263,7 @@ static coroutine_fn int parallels_co_flush_to_os(BlockDriverState *bs)
 
 
 static int64_t coroutine_fn parallels_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVParallelsState *s = bs->opaque;
     int64_t offset;
@@ -272,6 +276,7 @@ static int64_t coroutine_fn parallels_co_get_block_status(BlockDriverState *bs,
         return 0;
     }
 
+    *file = bs->file->bs;
     return (offset << BDRV_SECTOR_BITS) |
         BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
 }
@@ -303,7 +308,7 @@ static coroutine_fn int parallels_co_writev(BlockDriverState *bs,
         qemu_iovec_reset(&hd_qiov);
         qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
 
-        ret = bdrv_co_writev(bs->file, position, n, &hd_qiov);
+        ret = bdrv_co_writev(bs->file->bs, position, n, &hd_qiov);
         if (ret < 0) {
             break;
         }
@@ -343,7 +348,7 @@ static coroutine_fn int parallels_co_readv(BlockDriverState *bs,
             qemu_iovec_reset(&hd_qiov);
             qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
 
-            ret = bdrv_co_readv(bs->file, position, n, &hd_qiov);
+            ret = bdrv_co_readv(bs->file->bs, position, n, &hd_qiov);
             if (ret < 0) {
                 break;
             }
@@ -369,7 +374,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
     bool flush_bat = false;
     int cluster_size = s->tracks << BDRV_SECTOR_BITS;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     if (size < 0) {
         res->check_errors++;
         return size;
@@ -424,7 +429,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
     }
 
     if (flush_bat) {
-        ret = bdrv_pwrite_sync(bs->file, 0, s->header, s->header_size);
+        ret = bdrv_pwrite_sync(bs->file->bs, 0, s->header, s->header_size);
         if (ret < 0) {
             res->check_errors++;
             return ret;
@@ -440,7 +445,7 @@ static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
                 size - res->image_end_offset);
         res->leaks += count;
         if (fix & BDRV_FIX_LEAKS) {
-            ret = bdrv_truncate(bs->file, res->image_end_offset);
+            ret = bdrv_truncate(bs->file->bs, res->image_end_offset);
             if (ret < 0) {
                 res->check_errors++;
                 return ret;
@@ -458,7 +463,7 @@ static int parallels_create(const char *filename, QemuOpts *opts, Error **errp)
     int64_t total_size, cl_size;
     uint8_t tmp[BDRV_SECTOR_SIZE];
     Error *local_err = NULL;
-    BlockDriverState *file;
+    BlockBackend *file;
     uint32_t bat_entries, bat_sectors;
     ParallelsHeader header;
     int ret;
@@ -474,14 +479,16 @@ static int parallels_create(const char *filename, QemuOpts *opts, Error **errp)
         return ret;
     }
 
-    file = NULL;
-    ret = bdrv_open(&file, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
+    file = blk_new_open(filename, NULL, NULL,
+                        BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (file == NULL) {
         error_propagate(errp, local_err);
-        return ret;
+        return -EIO;
     }
-    ret = bdrv_truncate(file, 0);
+
+    blk_set_allow_write_beyond_eof(file, true);
+
+    ret = blk_truncate(file, 0);
     if (ret < 0) {
         goto exit;
     }
@@ -505,18 +512,18 @@ static int parallels_create(const char *filename, QemuOpts *opts, Error **errp)
     memset(tmp, 0, sizeof(tmp));
     memcpy(tmp, &header, sizeof(header));
 
-    ret = bdrv_pwrite(file, 0, tmp, BDRV_SECTOR_SIZE);
+    ret = blk_pwrite(file, 0, tmp, BDRV_SECTOR_SIZE);
     if (ret < 0) {
         goto exit;
     }
-    ret = bdrv_write_zeroes(file, 1, bat_sectors - 1, 0);
+    ret = blk_write_zeroes(file, 1, bat_sectors - 1, 0);
     if (ret < 0) {
         goto exit;
     }
     ret = 0;
 
 done:
-    bdrv_unref(file);
+    blk_unref(file);
     return ret;
 
 exit:
@@ -546,12 +553,13 @@ static int parallels_probe(const uint8_t *buf, int buf_size,
 static int parallels_update_header(BlockDriverState *bs)
 {
     BDRVParallelsState *s = bs->opaque;
-    unsigned size = MAX(bdrv_opt_mem_align(bs->file), sizeof(ParallelsHeader));
+    unsigned size = MAX(bdrv_opt_mem_align(bs->file->bs),
+                        sizeof(ParallelsHeader));
 
     if (size > s->header_size) {
         size = s->header_size;
     }
-    return bdrv_pwrite_sync(bs->file, 0, s->header, size);
+    return bdrv_pwrite_sync(bs->file->bs, 0, s->header, size);
 }
 
 static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
@@ -564,7 +572,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     Error *local_err = NULL;
     char *buf;
 
-    ret = bdrv_pread(bs->file, 0, &ph, sizeof(ph));
+    ret = bdrv_pread(bs->file->bs, 0, &ph, sizeof(ph));
     if (ret < 0) {
         goto fail;
     }
@@ -603,8 +611,8 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     size = bat_entry_off(s->bat_size);
-    s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file));
-    s->header = qemu_try_blockalign(bs->file, s->header_size);
+    s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file->bs));
+    s->header = qemu_try_blockalign(bs->file->bs, s->header_size);
     if (s->header == NULL) {
         ret = -ENOMEM;
         goto fail;
@@ -619,7 +627,7 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
         s->header_size = size;
     }
 
-    ret = bdrv_pread(bs->file, 0, s->header, s->header_size);
+    ret = bdrv_pread(bs->file->bs, 0, s->header, s->header_size);
     if (ret < 0) {
         goto fail;
     }
@@ -658,13 +666,13 @@ static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
     s->prealloc_size = MAX(s->tracks, s->prealloc_size >> BDRV_SECTOR_BITS);
     buf = qemu_opt_get_del(opts, PARALLELS_OPT_PREALLOC_MODE);
     s->prealloc_mode = qapi_enum_parse(prealloc_mode_lookup, buf,
-            PRL_PREALLOC_MODE_MAX, PRL_PREALLOC_MODE_FALLOCATE, &local_err);
+            PRL_PREALLOC_MODE__MAX, PRL_PREALLOC_MODE_FALLOCATE, &local_err);
     g_free(buf);
     if (local_err != NULL) {
         goto fail_options;
     }
-    if (!bdrv_has_zero_init(bs->file) ||
-            bdrv_truncate(bs->file, bdrv_getlength(bs->file)) != 0) {
+    if (!bdrv_has_zero_init(bs->file->bs) ||
+            bdrv_truncate(bs->file->bs, bdrv_getlength(bs->file->bs)) != 0) {
         s->prealloc_mode = PRL_PREALLOC_MODE_FALLOCATE;
     }
 
@@ -707,7 +715,7 @@ static void parallels_close(BlockDriverState *bs)
     }
 
     if (bs->open_flags & BDRV_O_RDWR) {
-        bdrv_truncate(bs->file, s->data_end << BDRV_SECTOR_BITS);
+        bdrv_truncate(bs->file->bs, s->data_end << BDRV_SECTOR_BITS);
     }
 
     g_free(s->bat_dirty_bmap);
diff --git a/qemu/block/qapi.c b/qemu/block/qapi.c
index 2ce509711..c5f6ba643 100644
--- a/qemu/block/qapi.c
+++ b/qemu/block/qapi.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/qapi.h"
 #include "block/block_int.h"
 #include "block/throttle-groups.h"
@@ -31,8 +32,10 @@
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp/types.h"
 #include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
-BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
+BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
+                                        BlockDriverState *bs, Error **errp)
 {
     ImageInfo **p_image_info;
     BlockDriverState *bs0;
@@ -46,7 +49,7 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
 
     info->cache = g_new(BlockdevCacheInfo, 1);
     *info->cache = (BlockdevCacheInfo) {
-        .writeback      = bdrv_enable_write_cache(bs),
+        .writeback      = blk ? blk_enable_write_cache(blk) : true,
         .direct         = !!(bs->open_flags & BDRV_O_NOCACHE),
         .no_flush       = !!(bs->open_flags & BDRV_O_NO_FLUSH),
     };
@@ -64,7 +67,7 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
     info->backing_file_depth = bdrv_get_backing_file_depth(bs);
     info->detect_zeroes = bs->detect_zeroes;
 
-    if (bs->io_limits_enabled) {
+    if (bs->throttle_state) {
         ThrottleConfig cfg;
 
         throttle_group_get_config(bs, &cfg);
@@ -91,6 +94,26 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
         info->has_iops_wr_max = cfg.buckets[THROTTLE_OPS_WRITE].max;
         info->iops_wr_max     = cfg.buckets[THROTTLE_OPS_WRITE].max;
 
+        info->has_bps_max_length     = info->has_bps_max;
+        info->bps_max_length         =
+            cfg.buckets[THROTTLE_BPS_TOTAL].burst_length;
+        info->has_bps_rd_max_length  = info->has_bps_rd_max;
+        info->bps_rd_max_length      =
+            cfg.buckets[THROTTLE_BPS_READ].burst_length;
+        info->has_bps_wr_max_length  = info->has_bps_wr_max;
+        info->bps_wr_max_length      =
+            cfg.buckets[THROTTLE_BPS_WRITE].burst_length;
+
+        info->has_iops_max_length    = info->has_iops_max;
+        info->iops_max_length        =
+            cfg.buckets[THROTTLE_OPS_TOTAL].burst_length;
+        info->has_iops_rd_max_length = info->has_iops_rd_max;
+        info->iops_rd_max_length     =
+            cfg.buckets[THROTTLE_OPS_READ].burst_length;
+        info->has_iops_wr_max_length = info->has_iops_wr_max;
+        info->iops_wr_max_length     =
+            cfg.buckets[THROTTLE_OPS_WRITE].burst_length;
+
         info->has_iops_size = cfg.op_size;
         info->iops_size = cfg.op_size;
 
@@ -110,8 +133,8 @@ BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp)
             qapi_free_BlockDeviceInfo(info);
             return NULL;
         }
-        if (bs0->drv && bs0->backing_hd) {
-            bs0 = bs0->backing_hd;
+        if (bs0->drv && bs0->backing) {
+            bs0 = bs0->backing->bs;
             (*p_image_info)->has_backing_image = true;
             p_image_info = &((*p_image_info)->backing_image);
         } else {
@@ -210,11 +233,13 @@ void bdrv_query_image_info(BlockDriverState *bs,
     Error *err = NULL;
     ImageInfo *info;
 
+    aio_context_acquire(bdrv_get_aio_context(bs));
+
     size = bdrv_getlength(bs);
     if (size < 0) {
         error_setg_errno(errp, -size, "Can't get size of device '%s'",
                          bdrv_get_device_name(bs));
-        return;
+        goto out;
     }
 
     info = g_new0(ImageInfo, 1);
@@ -245,15 +270,18 @@ void bdrv_query_image_info(BlockDriverState *bs,
         info->has_backing_filename = true;
         bdrv_get_full_backing_filename(bs, backing_filename2, PATH_MAX, &err);
         if (err) {
-            error_propagate(errp, err);
-            qapi_free_ImageInfo(info);
+            /* Can't reconstruct the full backing filename, so we must omit
+             * this field and apply a Best Effort to this query. */
             g_free(backing_filename2);
-            return;
+            backing_filename2 = NULL;
+            error_free(err);
+            err = NULL;
         }
 
-        if (strcmp(backing_filename, backing_filename2) != 0) {
-            info->full_backing_filename =
-                        g_strdup(backing_filename2);
+        /* Always report the full_backing_filename if present, even if it's the
+         * same as backing_filename. That they are same is useful info. */
+        if (backing_filename2) {
+            info->full_backing_filename = g_strdup(backing_filename2);
             info->has_full_backing_filename = true;
         }
 
@@ -279,10 +307,13 @@ void bdrv_query_image_info(BlockDriverState *bs,
     default:
         error_propagate(errp, err);
         qapi_free_ImageInfo(info);
-        return;
+        goto out;
     }
 
     *p_info = info;
+
+out:
+    aio_context_release(bdrv_get_aio_context(bs));
 }
 
 /* @p_info will be set only on success. */
@@ -296,24 +327,24 @@ static void bdrv_query_info(BlockBackend *blk, BlockInfo **p_info,
     info->locked = blk_dev_is_medium_locked(blk);
     info->removable = blk_dev_has_removable_media(blk);
 
-    if (blk_dev_has_removable_media(blk)) {
+    if (blk_dev_has_tray(blk)) {
         info->has_tray_open = true;
         info->tray_open = blk_dev_is_tray_open(blk);
     }
 
-    if (bdrv_iostatus_is_enabled(bs)) {
+    if (blk_iostatus_is_enabled(blk)) {
         info->has_io_status = true;
-        info->io_status = bs->iostatus;
+        info->io_status = blk_iostatus(blk);
     }
 
-    if (!QLIST_EMPTY(&bs->dirty_bitmaps)) {
+    if (bs && !QLIST_EMPTY(&bs->dirty_bitmaps)) {
         info->has_dirty_bitmaps = true;
         info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
     }
 
-    if (bs->drv) {
+    if (bs && bs->drv) {
         info->has_inserted = true;
-        info->inserted = bdrv_block_device_info(bs, errp);
+        info->inserted = bdrv_block_device_info(blk, bs, errp);
         if (info->inserted == NULL) {
             goto err;
         }
@@ -326,45 +357,115 @@ static void bdrv_query_info(BlockBackend *blk, BlockInfo **p_info,
     qapi_free_BlockInfo(info);
 }
 
-static BlockStats *bdrv_query_stats(const BlockDriverState *bs,
-                                    bool query_backing)
+static BlockStats *bdrv_query_stats(BlockBackend *blk,
+                                    const BlockDriverState *bs,
+                                    bool query_backing);
+
+static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk)
 {
-    BlockStats *s;
+    BlockAcctStats *stats = blk_get_stats(blk);
+    BlockAcctTimedStats *ts = NULL;
 
-    s = g_malloc0(sizeof(*s));
+    ds->rd_bytes = stats->nr_bytes[BLOCK_ACCT_READ];
+    ds->wr_bytes = stats->nr_bytes[BLOCK_ACCT_WRITE];
+    ds->rd_operations = stats->nr_ops[BLOCK_ACCT_READ];
+    ds->wr_operations = stats->nr_ops[BLOCK_ACCT_WRITE];
 
-    if (bdrv_get_device_name(bs)[0]) {
-        s->has_device = true;
-        s->device = g_strdup(bdrv_get_device_name(bs));
+    ds->failed_rd_operations = stats->failed_ops[BLOCK_ACCT_READ];
+    ds->failed_wr_operations = stats->failed_ops[BLOCK_ACCT_WRITE];
+    ds->failed_flush_operations = stats->failed_ops[BLOCK_ACCT_FLUSH];
+
+    ds->invalid_rd_operations = stats->invalid_ops[BLOCK_ACCT_READ];
+    ds->invalid_wr_operations = stats->invalid_ops[BLOCK_ACCT_WRITE];
+    ds->invalid_flush_operations =
+        stats->invalid_ops[BLOCK_ACCT_FLUSH];
+
+    ds->rd_merged = stats->merged[BLOCK_ACCT_READ];
+    ds->wr_merged = stats->merged[BLOCK_ACCT_WRITE];
+    ds->flush_operations = stats->nr_ops[BLOCK_ACCT_FLUSH];
+    ds->wr_total_time_ns = stats->total_time_ns[BLOCK_ACCT_WRITE];
+    ds->rd_total_time_ns = stats->total_time_ns[BLOCK_ACCT_READ];
+    ds->flush_total_time_ns = stats->total_time_ns[BLOCK_ACCT_FLUSH];
+
+    ds->has_idle_time_ns = stats->last_access_time_ns > 0;
+    if (ds->has_idle_time_ns) {
+        ds->idle_time_ns = block_acct_idle_time_ns(stats);
     }
 
+    ds->account_invalid = stats->account_invalid;
+    ds->account_failed = stats->account_failed;
+
+    while ((ts = block_acct_interval_next(stats, ts))) {
+        BlockDeviceTimedStatsList *timed_stats =
+            g_malloc0(sizeof(*timed_stats));
+        BlockDeviceTimedStats *dev_stats = g_malloc0(sizeof(*dev_stats));
+        timed_stats->next = ds->timed_stats;
+        timed_stats->value = dev_stats;
+        ds->timed_stats = timed_stats;
+
+        TimedAverage *rd = &ts->latency[BLOCK_ACCT_READ];
+        TimedAverage *wr = &ts->latency[BLOCK_ACCT_WRITE];
+        TimedAverage *fl = &ts->latency[BLOCK_ACCT_FLUSH];
+
+        dev_stats->interval_length = ts->interval_length;
+
+        dev_stats->min_rd_latency_ns = timed_average_min(rd);
+        dev_stats->max_rd_latency_ns = timed_average_max(rd);
+        dev_stats->avg_rd_latency_ns = timed_average_avg(rd);
+
+        dev_stats->min_wr_latency_ns = timed_average_min(wr);
+        dev_stats->max_wr_latency_ns = timed_average_max(wr);
+        dev_stats->avg_wr_latency_ns = timed_average_avg(wr);
+
+        dev_stats->min_flush_latency_ns = timed_average_min(fl);
+        dev_stats->max_flush_latency_ns = timed_average_max(fl);
+        dev_stats->avg_flush_latency_ns = timed_average_avg(fl);
+
+        dev_stats->avg_rd_queue_depth =
+            block_acct_queue_depth(ts, BLOCK_ACCT_READ);
+        dev_stats->avg_wr_queue_depth =
+            block_acct_queue_depth(ts, BLOCK_ACCT_WRITE);
+    }
+}
+
+static void bdrv_query_bds_stats(BlockStats *s, const BlockDriverState *bs,
+                                 bool query_backing)
+{
     if (bdrv_get_node_name(bs)[0]) {
         s->has_node_name = true;
         s->node_name = g_strdup(bdrv_get_node_name(bs));
     }
 
-    s->stats = g_malloc0(sizeof(*s->stats));
-    s->stats->rd_bytes = bs->stats.nr_bytes[BLOCK_ACCT_READ];
-    s->stats->wr_bytes = bs->stats.nr_bytes[BLOCK_ACCT_WRITE];
-    s->stats->rd_operations = bs->stats.nr_ops[BLOCK_ACCT_READ];
-    s->stats->wr_operations = bs->stats.nr_ops[BLOCK_ACCT_WRITE];
-    s->stats->rd_merged = bs->stats.merged[BLOCK_ACCT_READ];
-    s->stats->wr_merged = bs->stats.merged[BLOCK_ACCT_WRITE];
-    s->stats->wr_highest_offset =
-        bs->stats.wr_highest_sector * BDRV_SECTOR_SIZE;
-    s->stats->flush_operations = bs->stats.nr_ops[BLOCK_ACCT_FLUSH];
-    s->stats->wr_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_WRITE];
-    s->stats->rd_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_READ];
-    s->stats->flush_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_FLUSH];
+    s->stats->wr_highest_offset = bs->wr_highest_offset;
 
     if (bs->file) {
         s->has_parent = true;
-        s->parent = bdrv_query_stats(bs->file, query_backing);
+        s->parent = bdrv_query_stats(NULL, bs->file->bs, query_backing);
     }
 
-    if (query_backing && bs->backing_hd) {
+    if (query_backing && bs->backing) {
         s->has_backing = true;
-        s->backing = bdrv_query_stats(bs->backing_hd, query_backing);
+        s->backing = bdrv_query_stats(NULL, bs->backing->bs, query_backing);
+    }
+
+}
+
+static BlockStats *bdrv_query_stats(BlockBackend *blk,
+                                    const BlockDriverState *bs,
+                                    bool query_backing)
+{
+    BlockStats *s;
+
+    s = g_malloc0(sizeof(*s));
+    s->stats = g_malloc0(sizeof(*s->stats));
+
+    if (blk) {
+        s->has_device = true;
+        s->device = g_strdup(blk_name(blk));
+        bdrv_query_blk_stats(s->stats, blk);
+    }
+    if (bs) {
+        bdrv_query_bds_stats(s, bs, query_backing);
     }
 
     return s;
@@ -381,7 +482,9 @@ BlockInfoList *qmp_query_block(Error **errp)
         bdrv_query_info(blk, &info->value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
-            goto err;
+            g_free(info);
+            qapi_free_BlockInfoList(head);
+            return NULL;
         }
 
         *p_next = info;
@@ -389,10 +492,20 @@ BlockInfoList *qmp_query_block(Error **errp)
     }
 
     return head;
+}
 
- err:
-    qapi_free_BlockInfoList(head);
-    return NULL;
+static bool next_query_bds(BlockBackend **blk, BlockDriverState **bs,
+                           bool query_nodes)
+{
+    if (query_nodes) {
+        *bs = bdrv_next_node(*bs);
+        return !!*bs;
+    }
+
+    *blk = blk_next(*blk);
+    *bs = *blk ? blk_bs(*blk) : NULL;
+
+    return !!*blk;
 }
 
 BlockStatsList *qmp_query_blockstats(bool has_query_nodes,
@@ -400,17 +513,19 @@ BlockStatsList *qmp_query_blockstats(bool has_query_nodes,
                                      Error **errp)
 {
     BlockStatsList *head = NULL, **p_next = &head;
+    BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
 
     /* Just to be safe if query_nodes is not always initialized */
     query_nodes = has_query_nodes && query_nodes;
 
-    while ((bs = query_nodes ? bdrv_next_node(bs) : bdrv_next(bs))) {
+    while (next_query_bds(&blk, &bs, query_nodes)) {
         BlockStatsList *info = g_malloc0(sizeof(*info));
-        AioContext *ctx = bdrv_get_aio_context(bs);
+        AioContext *ctx = blk ? blk_get_aio_context(blk)
+                              : bdrv_get_aio_context(bs);
 
         aio_context_acquire(ctx);
-        info->value = bdrv_query_stats(bs, !query_nodes);
+        info->value = bdrv_query_stats(blk, bs, !query_nodes);
         aio_context_release(ctx);
 
         *p_next = info;
@@ -535,11 +650,10 @@ static void dump_qlist(fprintf_function func_fprintf, void *f, int indentation,
     int i = 0;
 
     for (entry = qlist_first(list); entry; entry = qlist_next(entry), i++) {
-        qtype_code type = qobject_type(entry->value);
+        QType type = qobject_type(entry->value);
         bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
-        const char *format = composite ? "%*s[%i]:\n" : "%*s[%i]: ";
-
-        func_fprintf(f, format, indentation * 4, "", i);
+        func_fprintf(f, "%*s[%i]:%c", indentation * 4, "", i,
+                     composite ? '\n' : ' ');
         dump_qobject(func_fprintf, f, indentation + 1, entry->value);
         if (!composite) {
             func_fprintf(f, "\n");
@@ -553,10 +667,9 @@ static void dump_qdict(fprintf_function func_fprintf, void *f, int indentation,
     const QDictEntry *entry;
 
     for (entry = qdict_first(dict); entry; entry = qdict_next(dict, entry)) {
-        qtype_code type = qobject_type(entry->value);
+        QType type = qobject_type(entry->value);
         bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
-        const char *format = composite ? "%*s%s:\n" : "%*s%s: ";
-        char key[strlen(entry->key) + 1];
+        char *key = g_malloc(strlen(entry->key) + 1);
         int i;
 
         /* replace dashes with spaces in key (variable) names */
@@ -564,12 +677,13 @@ static void dump_qdict(fprintf_function func_fprintf, void *f, int indentation,
             key[i] = entry->key[i] == '-' ? ' ' : entry->key[i];
         }
         key[i] = 0;
-
-        func_fprintf(f, format, indentation * 4, "", key);
+        func_fprintf(f, "%*s%s:%c", indentation * 4, "", key,
+                     composite ? '\n' : ' ');
         dump_qobject(func_fprintf, f, indentation + 1, entry->value);
         if (!composite) {
             func_fprintf(f, "\n");
         }
+        g_free(key);
     }
 }
 
@@ -579,7 +693,7 @@ void bdrv_image_info_specific_dump(fprintf_function func_fprintf, void *f,
     QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj, *data;
 
-    visit_type_ImageInfoSpecific(qmp_output_get_visitor(ov), &info_spec, NULL,
+    visit_type_ImageInfoSpecific(qmp_output_get_visitor(ov), NULL, &info_spec,
                                  &error_abort);
     obj = qmp_output_get_qobject(ov);
     assert(qobject_type(obj) == QTYPE_QDICT);
@@ -623,7 +737,10 @@ void bdrv_image_info_dump(fprintf_function func_fprintf, void *f,
 
     if (info->has_backing_filename) {
         func_fprintf(f, "backing file: %s", info->backing_filename);
-        if (info->has_full_backing_filename) {
+        if (!info->has_full_backing_filename) {
+            func_fprintf(f, " (cannot determine actual path)");
+        } else if (strcmp(info->backing_filename,
+                          info->full_backing_filename) != 0) {
             func_fprintf(f, " (actual path: %s)", info->full_backing_filename);
         }
         func_fprintf(f, "\n");
diff --git a/qemu/block/qcow.c b/qemu/block/qcow.c
index 01fba54ce..60ddb12ec 100644
--- a/qemu/block/qcow.c
+++ b/qemu/block/qcow.c
@@ -21,8 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include <zlib.h>
 #include "qapi/qmp/qerror.h"
@@ -100,7 +104,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     int ret;
     QCowHeader header;
 
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    ret = bdrv_pread(bs->file->bs, 0, &header, sizeof(header));
     if (ret < 0) {
         goto fail;
     }
@@ -119,11 +123,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
         goto fail;
     }
     if (header.version != QCOW_VERSION) {
-        char version[64];
-        snprintf(version, sizeof(version), "QCOW version %" PRIu32,
-                 header.version);
-        error_setg(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                   bdrv_get_device_or_node_name(bs), "qcow", version);
+        error_setg(errp, "Unsupported qcow version %" PRIu32, header.version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -159,6 +159,14 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
     }
     s->crypt_method_header = header.crypt_method;
     if (s->crypt_method_header) {
+        if (bdrv_uses_whitelist() &&
+            s->crypt_method_header == QCOW_CRYPT_AES) {
+            error_report("qcow built-in AES encryption is deprecated");
+            error_printf("Support for it will be removed in a future release.\n"
+                         "You can use 'qemu-img convert' to switch to an\n"
+                         "unencrypted qcow image, or a LUKS raw image.\n");
+        }
+
         bs->encrypted = 1;
     }
     s->cluster_bits = header.cluster_bits;
@@ -193,7 +201,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+    ret = bdrv_pread(bs->file->bs, s->l1_table_offset, s->l1_table,
                s->l1_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
@@ -205,7 +213,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
 
     /* alloc L2 cache (max. 64k * 16 * 8 = 8 MB) */
     s->l2_cache =
-        qemu_try_blockalign(bs->file,
+        qemu_try_blockalign(bs->file->bs,
                             s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
     if (s->l2_cache == NULL) {
         error_setg(errp, "Could not allocate L2 table cache");
@@ -224,7 +232,7 @@ static int qcow_open(BlockDriverState *bs, QDict *options, int flags,
             ret = -EINVAL;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
+        ret = bdrv_pread(bs->file->bs, header.backing_file_offset,
                    bs->backing_file, len);
         if (ret < 0) {
             goto fail;
@@ -369,13 +377,13 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
         if (!allocate)
             return 0;
         /* allocate a new l2 entry */
-        l2_offset = bdrv_getlength(bs->file);
+        l2_offset = bdrv_getlength(bs->file->bs);
         /* round to cluster size */
         l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1);
         /* update the L1 entry */
         s->l1_table[l1_index] = l2_offset;
         tmp = cpu_to_be64(l2_offset);
-        if (bdrv_pwrite_sync(bs->file,
+        if (bdrv_pwrite_sync(bs->file->bs,
                 s->l1_table_offset + l1_index * sizeof(tmp),
                 &tmp, sizeof(tmp)) < 0)
             return 0;
@@ -405,11 +413,12 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
     l2_table = s->l2_cache + (min_index << s->l2_bits);
     if (new_l2_table) {
         memset(l2_table, 0, s->l2_size * sizeof(uint64_t));
-        if (bdrv_pwrite_sync(bs->file, l2_offset, l2_table,
+        if (bdrv_pwrite_sync(bs->file->bs, l2_offset, l2_table,
                 s->l2_size * sizeof(uint64_t)) < 0)
             return 0;
     } else {
-        if (bdrv_pread(bs->file, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) !=
+        if (bdrv_pread(bs->file->bs, l2_offset, l2_table,
+                       s->l2_size * sizeof(uint64_t)) !=
             s->l2_size * sizeof(uint64_t))
             return 0;
     }
@@ -430,20 +439,21 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
                overwritten */
             if (decompress_cluster(bs, cluster_offset) < 0)
                 return 0;
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             cluster_offset = (cluster_offset + s->cluster_size - 1) &
                 ~(s->cluster_size - 1);
             /* write the cluster content */
-            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache, s->cluster_size) !=
+            if (bdrv_pwrite(bs->file->bs, cluster_offset, s->cluster_cache,
+                            s->cluster_size) !=
                 s->cluster_size)
                 return -1;
         } else {
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             if (allocate == 1) {
                 /* round to cluster size */
                 cluster_offset = (cluster_offset + s->cluster_size - 1) &
                     ~(s->cluster_size - 1);
-                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);
+                bdrv_truncate(bs->file->bs, cluster_offset + s->cluster_size);
                 /* if encrypted, we must initialize the cluster
                    content which won't be written */
                 if (bs->encrypted &&
@@ -463,7 +473,8 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
                                 errno = EIO;
                                 return -1;
                             }
-                            if (bdrv_pwrite(bs->file, cluster_offset + i * 512,
+                            if (bdrv_pwrite(bs->file->bs,
+                                            cluster_offset + i * 512,
                                             s->cluster_data, 512) != 512)
                                 return -1;
                         }
@@ -477,7 +488,7 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
         /* update L2 table */
         tmp = cpu_to_be64(cluster_offset);
         l2_table[l2_index] = tmp;
-        if (bdrv_pwrite_sync(bs->file, l2_offset + l2_index * sizeof(tmp),
+        if (bdrv_pwrite_sync(bs->file->bs, l2_offset + l2_index * sizeof(tmp),
                 &tmp, sizeof(tmp)) < 0)
             return 0;
     }
@@ -485,7 +496,7 @@ static uint64_t get_cluster_offset(BlockDriverState *bs,
 }
 
 static int64_t coroutine_fn qcow_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVQcowState *s = bs->opaque;
     int index_in_cluster, n;
@@ -506,6 +517,7 @@ static int64_t coroutine_fn qcow_co_get_block_status(BlockDriverState *bs,
         return BDRV_BLOCK_DATA;
     }
     cluster_offset |= (index_in_cluster << BDRV_SECTOR_BITS);
+    *file = bs->file->bs;
     return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | cluster_offset;
 }
 
@@ -546,7 +558,7 @@ static int decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
     if (s->cluster_cache_offset != coffset) {
         csize = cluster_offset >> (63 - s->cluster_bits);
         csize &= (s->cluster_size - 1);
-        ret = bdrv_pread(bs->file, coffset, s->cluster_data, csize);
+        ret = bdrv_pread(bs->file->bs, coffset, s->cluster_data, csize);
         if (ret != csize)
             return -1;
         if (decompress_buffer(s->cluster_cache, s->cluster_size,
@@ -594,13 +606,13 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
         }
 
         if (!cluster_offset) {
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
                 hd_iov.iov_base = (void *)buf;
                 hd_iov.iov_len = n * 512;
                 qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                ret = bdrv_co_readv(bs->backing->bs, sector_num,
                                     n, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
                 if (ret < 0) {
@@ -625,7 +637,7 @@ static coroutine_fn int qcow_co_readv(BlockDriverState *bs, int64_t sector_num,
             hd_iov.iov_len = n * 512;
             qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
             qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
+            ret = bdrv_co_readv(bs->file->bs,
                                 (cluster_offset >> 9) + index_in_cluster,
                                 n, &hd_qiov);
             qemu_co_mutex_lock(&s->lock);
@@ -727,7 +739,7 @@ static coroutine_fn int qcow_co_writev(BlockDriverState *bs, int64_t sector_num,
         hd_iov.iov_len = n * 512;
         qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
         qemu_co_mutex_unlock(&s->lock);
-        ret = bdrv_co_writev(bs->file,
+        ret = bdrv_co_writev(bs->file->bs,
                              (cluster_offset >> 9) + index_in_cluster,
                              n, &hd_qiov);
         qemu_co_mutex_lock(&s->lock);
@@ -775,7 +787,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     int flags = 0;
     Error *local_err = NULL;
     int ret;
-    BlockDriverState *qcow_bs;
+    BlockBackend *qcow_blk;
 
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
@@ -791,15 +803,17 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
         goto cleanup;
     }
 
-    qcow_bs = NULL;
-    ret = bdrv_open(&qcow_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
+    qcow_blk = blk_new_open(filename, NULL, NULL,
+                            BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (qcow_blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto cleanup;
     }
 
-    ret = bdrv_truncate(qcow_bs, 0);
+    blk_set_allow_write_beyond_eof(qcow_blk, true);
+
+    ret = blk_truncate(qcow_blk, 0);
     if (ret < 0) {
         goto exit;
     }
@@ -839,13 +853,13 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
     /* write all the data */
-    ret = bdrv_pwrite(qcow_bs, 0, &header, sizeof(header));
+    ret = blk_pwrite(qcow_blk, 0, &header, sizeof(header));
     if (ret != sizeof(header)) {
         goto exit;
     }
 
     if (backing_file) {
-        ret = bdrv_pwrite(qcow_bs, sizeof(header),
+        ret = blk_pwrite(qcow_blk, sizeof(header),
             backing_file, backing_filename_len);
         if (ret != backing_filename_len) {
             goto exit;
@@ -855,7 +869,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     tmp = g_malloc0(BDRV_SECTOR_SIZE);
     for (i = 0; i < ((sizeof(uint64_t)*l1_size + BDRV_SECTOR_SIZE - 1)/
         BDRV_SECTOR_SIZE); i++) {
-        ret = bdrv_pwrite(qcow_bs, header_size +
+        ret = blk_pwrite(qcow_blk, header_size +
             BDRV_SECTOR_SIZE*i, tmp, BDRV_SECTOR_SIZE);
         if (ret != BDRV_SECTOR_SIZE) {
             g_free(tmp);
@@ -866,7 +880,7 @@ static int qcow_create(const char *filename, QemuOpts *opts, Error **errp)
     g_free(tmp);
     ret = 0;
 exit:
-    bdrv_unref(qcow_bs);
+    blk_unref(qcow_blk);
 cleanup:
     g_free(backing_file);
     return ret;
@@ -879,10 +893,10 @@ static int qcow_make_empty(BlockDriverState *bs)
     int ret;
 
     memset(s->l1_table, 0, l1_length);
-    if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
+    if (bdrv_pwrite_sync(bs->file->bs, s->l1_table_offset, s->l1_table,
             l1_length) < 0)
         return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
+    ret = bdrv_truncate(bs->file->bs, s->l1_table_offset + l1_length);
     if (ret < 0)
         return ret;
 
@@ -962,7 +976,7 @@ static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num,
         }
 
         cluster_offset &= s->cluster_offset_mask;
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        ret = bdrv_pwrite(bs->file->bs, cluster_offset, out_buf, out_len);
         if (ret < 0) {
             goto fail;
         }
diff --git a/qemu/block/qcow2-cache.c b/qemu/block/qcow2-cache.c
index 53b8afc3d..0fe8edae4 100644
--- a/qemu/block/qcow2-cache.c
+++ b/qemu/block/qcow2-cache.c
@@ -22,6 +22,13 @@
  * THE SOFTWARE.
  */
 
+/* Needed for CONFIG_MADVISE */
+#include "qemu/osdep.h"
+
+#if defined(CONFIG_MADVISE) || defined(CONFIG_POSIX_MADVISE)
+#include <sys/mman.h>
+#endif
+
 #include "block/block_int.h"
 #include "qemu-common.h"
 #include "qcow2.h"
@@ -29,9 +36,9 @@
 
 typedef struct Qcow2CachedTable {
     int64_t  offset;
-    bool     dirty;
     uint64_t lru_counter;
     int      ref;
+    bool     dirty;
 } Qcow2CachedTable;
 
 struct Qcow2Cache {
@@ -41,34 +48,85 @@ struct Qcow2Cache {
     bool                    depends_on_flush;
     void                   *table_array;
     uint64_t                lru_counter;
+    uint64_t                cache_clean_lru_counter;
 };
 
 static inline void *qcow2_cache_get_table_addr(BlockDriverState *bs,
                     Qcow2Cache *c, int table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     return (uint8_t *) c->table_array + (size_t) table * s->cluster_size;
 }
 
 static inline int qcow2_cache_get_table_idx(BlockDriverState *bs,
                   Qcow2Cache *c, void *table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     ptrdiff_t table_offset = (uint8_t *) table - (uint8_t *) c->table_array;
     int idx = table_offset / s->cluster_size;
     assert(idx >= 0 && idx < c->size && table_offset % s->cluster_size == 0);
     return idx;
 }
 
+static void qcow2_cache_table_release(BlockDriverState *bs, Qcow2Cache *c,
+                                      int i, int num_tables)
+{
+#if QEMU_MADV_DONTNEED != QEMU_MADV_INVALID
+    BDRVQcow2State *s = bs->opaque;
+    void *t = qcow2_cache_get_table_addr(bs, c, i);
+    int align = getpagesize();
+    size_t mem_size = (size_t) s->cluster_size * num_tables;
+    size_t offset = QEMU_ALIGN_UP((uintptr_t) t, align) - (uintptr_t) t;
+    size_t length = QEMU_ALIGN_DOWN(mem_size - offset, align);
+    if (length > 0) {
+        qemu_madvise((uint8_t *) t + offset, length, QEMU_MADV_DONTNEED);
+    }
+#endif
+}
+
+static inline bool can_clean_entry(Qcow2Cache *c, int i)
+{
+    Qcow2CachedTable *t = &c->entries[i];
+    return t->ref == 0 && !t->dirty && t->offset != 0 &&
+        t->lru_counter <= c->cache_clean_lru_counter;
+}
+
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int i = 0;
+    while (i < c->size) {
+        int to_clean = 0;
+
+        /* Skip the entries that we don't need to clean */
+        while (i < c->size && !can_clean_entry(c, i)) {
+            i++;
+        }
+
+        /* And count how many we can clean in a row */
+        while (i < c->size && can_clean_entry(c, i)) {
+            c->entries[i].offset = 0;
+            c->entries[i].lru_counter = 0;
+            i++;
+            to_clean++;
+        }
+
+        if (to_clean > 0) {
+            qcow2_cache_table_release(bs, c, i - to_clean, to_clean);
+        }
+    }
+
+    c->cache_clean_lru_counter = c->lru_counter;
+}
+
 Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2Cache *c;
 
     c = g_new0(Qcow2Cache, 1);
     c->size = num_tables;
     c->entries = g_try_new0(Qcow2CachedTable, num_tables);
-    c->table_array = qemu_try_blockalign(bs->file,
+    c->table_array = qemu_try_blockalign(bs->file->bs,
                                          (size_t) num_tables * s->cluster_size);
 
     if (!c->entries || !c->table_array) {
@@ -113,7 +171,7 @@ static int qcow2_cache_flush_dependency(BlockDriverState *bs, Qcow2Cache *c)
 
 static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret = 0;
 
     if (!c->entries[i].dirty || !c->entries[i].offset) {
@@ -126,7 +184,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
     if (c->depends) {
         ret = qcow2_cache_flush_dependency(bs, c);
     } else if (c->depends_on_flush) {
-        ret = bdrv_flush(bs->file);
+        ret = bdrv_flush(bs->file->bs);
         if (ret >= 0) {
             c->depends_on_flush = false;
         }
@@ -157,7 +215,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
         BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);
     }
 
-    ret = bdrv_pwrite(bs->file, c->entries[i].offset,
+    ret = bdrv_pwrite(bs->file->bs, c->entries[i].offset,
                       qcow2_cache_get_table_addr(bs, c, i), s->cluster_size);
     if (ret < 0) {
         return ret;
@@ -170,7 +228,7 @@ static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
 
 int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int result = 0;
     int ret;
     int i;
@@ -185,7 +243,7 @@ int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
     }
 
     if (result == 0) {
-        ret = bdrv_flush(bs->file);
+        ret = bdrv_flush(bs->file->bs);
         if (ret < 0) {
             result = ret;
         }
@@ -237,6 +295,8 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c)
         c->entries[i].lru_counter = 0;
     }
 
+    qcow2_cache_table_release(bs, c, 0, c->size);
+
     c->lru_counter = 0;
 
     return 0;
@@ -245,7 +305,7 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c)
 static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
     uint64_t offset, void **table, bool read_from_disk)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
     int ret;
     int lookup_index;
@@ -295,7 +355,8 @@ static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
             BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);
         }
 
-        ret = bdrv_pread(bs->file, offset, qcow2_cache_get_table_addr(bs, c, i),
+        ret = bdrv_pread(bs->file->bs, offset,
+                         qcow2_cache_get_table_addr(bs, c, i),
                          s->cluster_size);
         if (ret < 0) {
             return ret;
diff --git a/qemu/block/qcow2-cluster.c b/qemu/block/qcow2-cluster.c
index b43f186eb..31ecc1030 100644
--- a/qemu/block/qcow2-cluster.c
+++ b/qemu/block/qcow2-cluster.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h>
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
@@ -32,7 +34,7 @@
 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         bool exact_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int new_l1_size2, ret, i;
     uint64_t *new_l1_table;
     int64_t old_l1_table_offset, old_l1_size;
@@ -72,7 +74,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 #endif
 
     new_l1_size2 = sizeof(uint64_t) * new_l1_size;
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_size2, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
@@ -105,7 +107,8 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
     for(i = 0; i < s->l1_size; i++)
         new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
-    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_table, new_l1_size2);
+    ret = bdrv_pwrite_sync(bs->file->bs, new_l1_table_offset,
+                           new_l1_table, new_l1_size2);
     if (ret < 0)
         goto fail;
     for(i = 0; i < s->l1_size; i++)
@@ -115,7 +118,8 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
     cpu_to_be32w((uint32_t*)data, new_l1_size);
     stq_be_p(data + 4, new_l1_table_offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size), data,sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, l1_size),
+                           data, sizeof(data));
     if (ret < 0) {
         goto fail;
     }
@@ -148,7 +152,7 @@ int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
 static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
     uint64_t **l2_table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset, (void**) l2_table);
@@ -163,7 +167,7 @@ static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
 #define L1_ENTRIES_PER_SECTOR (512 / 8)
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t buf[L1_ENTRIES_PER_SECTOR] = { 0 };
     int l1_start_index;
     int i, ret;
@@ -182,8 +186,9 @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset + 8 * l1_start_index,
-        buf, sizeof(buf));
+    ret = bdrv_pwrite_sync(bs->file->bs,
+                           s->l1_table_offset + 8 * l1_start_index,
+                           buf, sizeof(buf));
     if (ret < 0) {
         return ret;
     }
@@ -203,7 +208,7 @@ int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 
 static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t old_l2_offset;
     uint64_t *l2_table = NULL;
     int64_t l2_offset;
@@ -298,7 +303,7 @@ fail:
  * as contiguous. (This allows it, for example, to stop at the first compressed
  * cluster which may require a different handling)
  */
-static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
+static int count_contiguous_clusters(int nb_clusters, int cluster_size,
         uint64_t *l2_table, uint64_t stop_flags)
 {
     int i;
@@ -309,7 +314,7 @@ static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
     if (!offset)
         return 0;
 
-    assert(qcow2_get_cluster_type(first_entry) != QCOW2_CLUSTER_COMPRESSED);
+    assert(qcow2_get_cluster_type(first_entry) == QCOW2_CLUSTER_NORMAL);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
@@ -321,14 +326,16 @@ static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
 	return i;
 }
 
-static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_table)
+static int count_contiguous_clusters_by_type(int nb_clusters,
+                                             uint64_t *l2_table,
+                                             int wanted_type)
 {
     int i;
 
     for (i = 0; i < nb_clusters; i++) {
         int type = qcow2_get_cluster_type(be64_to_cpu(l2_table[i]));
 
-        if (type != QCOW2_CLUSTER_UNALLOCATED) {
+        if (type != wanted_type) {
             break;
         }
     }
@@ -339,7 +346,7 @@ static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_tab
 /* The crypt function is compatible with the linux cryptoloop
    algorithm for < 4 GB images. NOTE: out_buf == in_buf is
    supported */
-int qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
                           uint8_t *out_buf, const uint8_t *in_buf,
                           int nb_sectors, bool enc,
                           Error **errp)
@@ -387,7 +394,7 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
                                      uint64_t cluster_offset,
                                      int n_start, int n_end)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUIOVector qiov;
     struct iovec iov;
     int n, ret;
@@ -440,7 +447,8 @@ static int coroutine_fn copy_sectors(BlockDriverState *bs,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
-    ret = bdrv_co_writev(bs->file, (cluster_offset >> 9) + n_start, n, &qiov);
+    ret = bdrv_co_writev(bs->file->bs, (cluster_offset >> 9) + n_start, n,
+                         &qiov);
     if (ret < 0) {
         goto out;
     }
@@ -469,7 +477,7 @@ out:
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset, *l2_table;
     int l1_bits, c;
@@ -495,10 +503,11 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
     if (nb_needed > nb_available) {
         nb_needed = nb_available;
     }
+    assert(nb_needed <= INT_MAX);
 
     *cluster_offset = 0;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> l1_bits;
     if (l1_index >= s->l1_size) {
@@ -530,6 +539,8 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
 
     l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
     *cluster_offset = be64_to_cpu(l2_table[l2_index]);
+
+    /* nb_needed <= INT_MAX, thus nb_clusters <= INT_MAX, too */
     nb_clusters = size_to_clusters(s, nb_needed << 9);
 
     ret = qcow2_get_cluster_type(*cluster_offset);
@@ -547,13 +558,14 @@ int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
             ret = -EIO;
             goto fail;
         }
-        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], QCOW_OFLAG_ZERO);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_ZERO);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
         /* how many empty clusters ? */
-        c = count_contiguous_free_clusters(nb_clusters, &l2_table[l2_index]);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_UNALLOCATED);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_NORMAL:
@@ -606,13 +618,13 @@ static int get_cluster_table(BlockDriverState *bs, uint64_t offset,
                              uint64_t **new_l2_table,
                              int *new_l2_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset;
     uint64_t *l2_table = NULL;
     int ret;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> (s->l2_bits + s->cluster_bits);
     if (l1_index >= s->l1_size) {
@@ -680,7 +692,7 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
                                                uint64_t offset,
                                                int compressed_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index, ret;
     uint64_t *l2_table;
     int64_t cluster_offset;
@@ -725,7 +737,7 @@ uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
 
 static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     if (r->nb_sectors == 0) {
@@ -754,7 +766,7 @@ static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 
 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i, j = 0, l2_index, ret;
     uint64_t *old_cluster, *l2_table;
     uint64_t cluster_offset = m->alloc_offset;
@@ -814,7 +826,6 @@ int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
 
     /*
      * If this was a COW, we need to decrease the refcount of the old cluster.
-     * Also flush bs->file to get the right order for L2 and refcount update.
      *
      * Don't discard clusters that reach a refcount of 0 (e.g. compressed
      * clusters), the next write will reuse them anyway.
@@ -837,7 +848,7 @@ err:
  * write, but require COW to be performed (this includes yet unallocated space,
  * which must copy from the backing file)
  */
-static int count_cow_clusters(BDRVQcowState *s, int nb_clusters,
+static int count_cow_clusters(BDRVQcow2State *s, int nb_clusters,
     uint64_t *l2_table, int l2_index)
 {
     int i;
@@ -883,7 +894,7 @@ out:
 static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *cur_bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowL2Meta *old_alloc;
     uint64_t bytes = *cur_bytes;
 
@@ -956,11 +967,11 @@ static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
 static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t cluster_offset;
     uint64_t *l2_table;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     unsigned int keep_clusters;
     int ret;
 
@@ -979,6 +990,7 @@ static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
 
     l2_index = offset_to_l2_index(s, guest_offset);
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     /* Find L2 entry for the first involved cluster */
     ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
@@ -1061,9 +1073,9 @@ out:
  * restarted, but the whole request should not be failed.
  */
 static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, unsigned int *nb_clusters)
+                                   uint64_t *host_offset, uint64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
                                          *host_offset, *nb_clusters);
@@ -1079,7 +1091,7 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
         *host_offset = cluster_offset;
         return 0;
     } else {
-        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
+        int64_t ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
         if (ret < 0) {
             return ret;
         }
@@ -1111,11 +1123,11 @@ static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
 static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t *l2_table;
     uint64_t entry;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     int ret;
 
     uint64_t alloc_cluster_offset;
@@ -1133,6 +1145,7 @@ static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
 
     l2_index = offset_to_l2_index(s, guest_offset);
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     /* Find L2 entry for the first involved cluster */
     ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
@@ -1263,7 +1276,7 @@ fail:
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
     int *num, uint64_t *host_offset, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start, remaining;
     uint64_t cluster_offset;
     uint64_t cur_bytes;
@@ -1397,7 +1410,7 @@ static int decompress_buffer(uint8_t *out_buf, int out_buf_size,
 
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, csize, nb_csectors, sector_offset;
     uint64_t coffset;
 
@@ -1407,7 +1420,8 @@ int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
         sector_offset = coffset & 511;
         csize = nb_csectors * 512 - sector_offset;
         BLKDBG_EVENT(bs->file, BLKDBG_READ_COMPRESSED);
-        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data, nb_csectors);
+        ret = bdrv_read(bs->file->bs, coffset >> 9, s->cluster_data,
+                        nb_csectors);
         if (ret < 0) {
             return ret;
         }
@@ -1426,9 +1440,10 @@ int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
  * clusters.
  */
 static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters, enum qcow2_discard_type type, bool full_discard)
+                             uint64_t nb_clusters, enum qcow2_discard_type type,
+                             bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1441,6 +1456,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 
     /* Limit nb_clusters to one L2 table */
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t old_l2_entry;
@@ -1462,7 +1478,7 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
          */
         switch (qcow2_get_cluster_type(old_l2_entry)) {
             case QCOW2_CLUSTER_UNALLOCATED:
-                if (full_discard || !bs->backing_hd) {
+                if (full_discard || !bs->backing) {
                     continue;
                 }
                 break;
@@ -1501,9 +1517,9 @@ static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
     int nb_sectors, enum qcow2_discard_type type, bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t end_offset;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     int ret;
 
     end_offset = offset + (nb_sectors << BDRV_SECTOR_BITS);
@@ -1545,9 +1561,9 @@ fail:
  * clusters.
  */
 static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters)
+                          uint64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1560,6 +1576,7 @@ static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
 
     /* Limit nb_clusters to one L2 table */
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t old_offset;
@@ -1583,8 +1600,8 @@ static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
 
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
-    unsigned int nb_clusters;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t nb_clusters;
     int ret;
 
     /* The zero flag is only supported by version 3 and newer */
@@ -1626,9 +1643,10 @@ fail:
 static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                                       int l1_size, int64_t *visited_l1_entries,
                                       int64_t l1_entries,
-                                      BlockDriverAmendStatusCB *status_cb)
+                                      BlockDriverAmendStatusCB *status_cb,
+                                      void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bool is_active_l1 = (l1_table == s->l1_table);
     uint64_t *l2_table = NULL;
     int ret;
@@ -1637,7 +1655,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
     if (!is_active_l1) {
         /* inactive L2 tables require a buffer to be stored in when loading
          * them from disk */
-        l2_table = qemu_try_blockalign(bs->file, s->cluster_size);
+        l2_table = qemu_try_blockalign(bs->file->bs, s->cluster_size);
         if (l2_table == NULL) {
             return -ENOMEM;
         }
@@ -1652,7 +1670,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
             /* unallocated */
             (*visited_l1_entries)++;
             if (status_cb) {
-                status_cb(bs, *visited_l1_entries, l1_entries);
+                status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
             }
             continue;
         }
@@ -1671,8 +1689,8 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                     (void **)&l2_table);
         } else {
             /* load inactive L2 tables from disk */
-            ret = bdrv_read(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                    (void *)l2_table, s->cluster_sectors);
+            ret = bdrv_read(bs->file->bs, l2_offset / BDRV_SECTOR_SIZE,
+                            (void *)l2_table, s->cluster_sectors);
         }
         if (ret < 0) {
             goto fail;
@@ -1695,7 +1713,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
             }
 
             if (!preallocated) {
-                if (!bs->backing_hd) {
+                if (!bs->backing) {
                     /* not backed; therefore we can simply deallocate the
                      * cluster */
                     l2_table[j] = 0;
@@ -1746,7 +1764,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                 goto fail;
             }
 
-            ret = bdrv_write_zeroes(bs->file, offset / BDRV_SECTOR_SIZE,
+            ret = bdrv_write_zeroes(bs->file->bs, offset / BDRV_SECTOR_SIZE,
                                     s->cluster_sectors, 0);
             if (ret < 0) {
                 if (!preallocated) {
@@ -1779,8 +1797,8 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                     goto fail;
                 }
 
-                ret = bdrv_write(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                        (void *)l2_table, s->cluster_sectors);
+                ret = bdrv_write(bs->file->bs, l2_offset / BDRV_SECTOR_SIZE,
+                                 (void *)l2_table, s->cluster_sectors);
                 if (ret < 0) {
                     goto fail;
                 }
@@ -1789,7 +1807,7 @@ static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
 
         (*visited_l1_entries)++;
         if (status_cb) {
-            status_cb(bs, *visited_l1_entries, l1_entries);
+            status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
         }
     }
 
@@ -1813,9 +1831,10 @@ fail:
  * qcow2 version which doesn't yet support metadata zero clusters.
  */
 int qcow2_expand_zero_clusters(BlockDriverState *bs,
-                               BlockDriverAmendStatusCB *status_cb)
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL;
     int64_t l1_entries = 0, visited_l1_entries = 0;
     int ret;
@@ -1830,7 +1849,7 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs,
 
     ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
                                      &visited_l1_entries, l1_entries,
-                                     status_cb);
+                                     status_cb, cb_opaque);
     if (ret < 0) {
         goto fail;
     }
@@ -1853,8 +1872,9 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs,
 
         l1_table = g_realloc(l1_table, l1_sectors * BDRV_SECTOR_SIZE);
 
-        ret = bdrv_read(bs->file, s->snapshots[i].l1_table_offset /
-                BDRV_SECTOR_SIZE, (void *)l1_table, l1_sectors);
+        ret = bdrv_read(bs->file->bs,
+                        s->snapshots[i].l1_table_offset / BDRV_SECTOR_SIZE,
+                        (void *)l1_table, l1_sectors);
         if (ret < 0) {
             goto fail;
         }
@@ -1865,7 +1885,7 @@ int qcow2_expand_zero_clusters(BlockDriverState *bs,
 
         ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
                                          &visited_l1_entries, l1_entries,
-                                         status_cb);
+                                         status_cb, cb_opaque);
         if (ret < 0) {
             goto fail;
         }
diff --git a/qemu/block/qcow2-refcount.c b/qemu/block/qcow2-refcount.c
index b0ee42d81..ca6094ff5 100644
--- a/qemu/block/qcow2-refcount.c
+++ b/qemu/block/qcow2-refcount.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
@@ -82,7 +84,7 @@ static Qcow2SetRefcountFunc *const set_refcount_funcs[] = {
 
 int qcow2_refcount_init(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_size2, i;
     int ret;
 
@@ -101,7 +103,7 @@ int qcow2_refcount_init(BlockDriverState *bs)
             goto fail;
         }
         BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_LOAD);
-        ret = bdrv_pread(bs->file, s->refcount_table_offset,
+        ret = bdrv_pread(bs->file->bs, s->refcount_table_offset,
                          s->refcount_table, refcount_table_size2);
         if (ret < 0) {
             goto fail;
@@ -116,7 +118,7 @@ int qcow2_refcount_init(BlockDriverState *bs)
 
 void qcow2_refcount_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     g_free(s->refcount_table);
 }
 
@@ -214,7 +216,7 @@ static int load_refcount_block(BlockDriverState *bs,
                                int64_t refcount_block_offset,
                                void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_LOAD);
@@ -231,7 +233,7 @@ static int load_refcount_block(BlockDriverState *bs,
 int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
                        uint64_t *refcount)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t refcount_table_index, block_index;
     int64_t refcount_block_offset;
     int ret;
@@ -274,7 +276,7 @@ int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
  * Rounds the refcount table size up to avoid growing the table for each single
  * refcount block that is allocated.
  */
-static unsigned int next_refcount_table_size(BDRVQcowState *s,
+static unsigned int next_refcount_table_size(BDRVQcow2State *s,
     unsigned int min_size)
 {
     unsigned int min_clusters = (min_size >> (s->cluster_bits - 3)) + 1;
@@ -290,7 +292,7 @@ static unsigned int next_refcount_table_size(BDRVQcowState *s,
 
 
 /* Checks if two offsets are described by the same refcount block */
-static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
+static int in_same_refcount_block(BDRVQcow2State *s, uint64_t offset_a,
     uint64_t offset_b)
 {
     uint64_t block_a = offset_a >> (s->cluster_bits + s->refcount_block_bits);
@@ -308,7 +310,7 @@ static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
 static int alloc_refcount_block(BlockDriverState *bs,
                                 int64_t cluster_index, void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_index;
     int ret;
 
@@ -431,7 +433,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     if (refcount_table_index < s->refcount_table_size) {
         uint64_t data64 = cpu_to_be64(new_block);
         BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_HOOKUP);
-        ret = bdrv_pwrite_sync(bs->file,
+        ret = bdrv_pwrite_sync(bs->file->bs,
             s->refcount_table_offset + refcount_table_index * sizeof(uint64_t),
             &data64, sizeof(data64));
         if (ret < 0) {
@@ -535,7 +537,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
 
     /* Write refcount blocks to disk */
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS);
-    ret = bdrv_pwrite_sync(bs->file, meta_offset, new_blocks,
+    ret = bdrv_pwrite_sync(bs->file->bs, meta_offset, new_blocks,
         blocks_clusters * s->cluster_size);
     g_free(new_blocks);
     new_blocks = NULL;
@@ -549,7 +551,7 @@ static int alloc_refcount_block(BlockDriverState *bs,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, table_offset, new_table,
+    ret = bdrv_pwrite_sync(bs->file->bs, table_offset, new_table,
         table_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail_table;
@@ -560,12 +562,16 @@ static int alloc_refcount_block(BlockDriverState *bs,
     }
 
     /* Hook up the new refcount table in the qcow2 header */
-    uint8_t data[12];
-    cpu_to_be64w((uint64_t*)data, table_offset);
-    cpu_to_be32w((uint32_t*)(data + 8), table_clusters);
+    struct QEMU_PACKED {
+        uint64_t d64;
+        uint32_t d32;
+    } data;
+    cpu_to_be64w(&data.d64, table_offset);
+    cpu_to_be32w(&data.d32, table_clusters);
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset),
-        data, sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file->bs,
+                           offsetof(QCowHeader, refcount_table_offset),
+                           &data, sizeof(data));
     if (ret < 0) {
         goto fail_table;
     }
@@ -605,7 +611,7 @@ fail_block:
 
 void qcow2_process_discards(BlockDriverState *bs, int ret)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *next;
 
     QTAILQ_FOREACH_SAFE(d, &s->discards, next, next) {
@@ -613,7 +619,7 @@ void qcow2_process_discards(BlockDriverState *bs, int ret)
 
         /* Discard is optional, ignore the return value */
         if (ret >= 0) {
-            bdrv_discard(bs->file,
+            bdrv_discard(bs->file->bs,
                          d->offset >> BDRV_SECTOR_BITS,
                          d->bytes >> BDRV_SECTOR_BITS);
         }
@@ -625,7 +631,7 @@ void qcow2_process_discards(BlockDriverState *bs, int ret)
 static void update_refcount_discard(BlockDriverState *bs,
                                     uint64_t offset, uint64_t length)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *p, *next;
 
     QTAILQ_FOREACH(d, &s->discards, next) {
@@ -682,7 +688,7 @@ static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
                                                    bool decrease,
                                                    enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t start, last, cluster_offset;
     void *refcount_block = NULL;
     int64_t old_table_index = -1;
@@ -793,7 +799,7 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs,
                                   uint64_t addend, bool decrease,
                                   enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     ret = update_refcount(bs, cluster_index << s->cluster_bits, 1, addend,
@@ -815,7 +821,7 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs,
 /* return < 0 if error */
 static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t i, nb_clusters, refcount;
     int ret;
 
@@ -875,10 +881,10 @@ int64_t qcow2_alloc_clusters(BlockDriverState *bs, uint64_t size)
     return offset;
 }
 
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters)
+int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+                                int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t cluster_index, refcount;
     uint64_t i;
     int ret;
@@ -916,7 +922,7 @@ int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
    contiguous sectors. size must be <= cluster_size */
 int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t offset;
     size_t free_in_cluster;
     int ret;
@@ -949,11 +955,17 @@ int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
 
             if (!offset || ROUND_UP(offset, s->cluster_size) != new_cluster) {
                 offset = new_cluster;
+                free_in_cluster = s->cluster_size;
+            } else {
+                free_in_cluster += s->cluster_size;
             }
         }
 
         assert(offset);
         ret = update_refcount(bs, offset, size, 1, false, QCOW2_DISCARD_NEVER);
+        if (ret < 0) {
+            offset = 0;
+        }
     } while (ret == -EAGAIN);
     if (ret < 0) {
         return ret;
@@ -992,7 +1004,7 @@ void qcow2_free_clusters(BlockDriverState *bs,
 void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
                              int nb_clusters, enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     switch (qcow2_get_cluster_type(l2_entry)) {
     case QCOW2_CLUSTER_COMPRESSED:
@@ -1036,7 +1048,7 @@ void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
 int qcow2_update_snapshot_refcount(BlockDriverState *bs,
     int64_t l1_table_offset, int l1_size, int addend)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, refcount;
     bool l1_allocated = false;
     int64_t old_offset, old_l2_offset;
@@ -1062,7 +1074,7 @@ int qcow2_update_snapshot_refcount(BlockDriverState *bs,
         }
         l1_allocated = true;
 
-        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pread(bs->file->bs, l1_table_offset, l1_table, l1_size2);
         if (ret < 0) {
             goto fail;
         }
@@ -1215,7 +1227,8 @@ fail:
             cpu_to_be64s(&l1_table[i]);
         }
 
-        ret = bdrv_pwrite_sync(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pwrite_sync(bs->file->bs, l1_table_offset,
+                               l1_table, l1_size2);
 
         for (i = 0; i < l1_size; i++) {
             be64_to_cpus(&l1_table[i]);
@@ -1233,7 +1246,7 @@ fail:
 /* refcount checking functions */
 
 
-static size_t refcount_array_byte_size(BDRVQcowState *s, uint64_t entries)
+static uint64_t refcount_array_byte_size(BDRVQcow2State *s, uint64_t entries)
 {
     /* This assertion holds because there is no way we can address more than
      * 2^(64 - 9) clusters at once (with cluster size 512 = 2^9, and because
@@ -1256,10 +1269,10 @@ static size_t refcount_array_byte_size(BDRVQcowState *s, uint64_t entries)
  * refcount array buffer will be aligned to a cluster boundary, and the newly
  * allocated area will be zeroed.
  */
-static int realloc_refcount_array(BDRVQcowState *s, void **array,
+static int realloc_refcount_array(BDRVQcow2State *s, void **array,
                                   int64_t *size, int64_t new_size)
 {
-    size_t old_byte_size, new_byte_size;
+    int64_t old_byte_size, new_byte_size;
     void *new_ptr;
 
     /* Round to clusters so the array can be directly written to disk */
@@ -1275,13 +1288,17 @@ static int realloc_refcount_array(BDRVQcowState *s, void **array,
 
     assert(new_byte_size > 0);
 
+    if (new_byte_size > SIZE_MAX) {
+        return -ENOMEM;
+    }
+
     new_ptr = g_try_realloc(*array, new_byte_size);
     if (!new_ptr) {
         return -ENOMEM;
     }
 
     if (new_byte_size > old_byte_size) {
-        memset((void *)((uintptr_t)new_ptr + old_byte_size), 0,
+        memset((char *)new_ptr + old_byte_size, 0,
                new_byte_size - old_byte_size);
     }
 
@@ -1294,7 +1311,7 @@ static int realloc_refcount_array(BDRVQcowState *s, void **array,
 /*
  * Increases the refcount for a range of clusters in a given refcount table.
  * This is used to construct a temporary refcount table out of L1 and L2 tables
- * which can be compared the the refcount table saved in the image.
+ * which can be compared to the refcount table saved in the image.
  *
  * Modifies the number of errors in res.
  */
@@ -1304,7 +1321,7 @@ static int inc_refcounts(BlockDriverState *bs,
                          int64_t *refcount_table_size,
                          int64_t offset, int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start, last, cluster_offset, k, refcount;
     int ret;
 
@@ -1330,6 +1347,9 @@ static int inc_refcounts(BlockDriverState *bs,
         if (refcount == s->refcount_max) {
             fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
                     "\n", cluster_offset);
+            fprintf(stderr, "Use qemu-img amend to increase the refcount entry "
+                    "width or qemu-img convert to create a clean copy if the "
+                    "image cannot be opened for writing\n");
             res->corruptions++;
             continue;
         }
@@ -1357,7 +1377,7 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
                               int64_t *refcount_table_size, int64_t l2_offset,
                               int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table, l2_entry;
     uint64_t next_contiguous_offset = 0;
     int i, l2_size, nb_csectors, ret;
@@ -1366,7 +1386,7 @@ static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
     l2_size = s->l2_size * sizeof(uint64_t);
     l2_table = g_malloc(l2_size);
 
-    ret = bdrv_pread(bs->file, l2_offset, l2_table, l2_size);
+    ret = bdrv_pread(bs->file->bs, l2_offset, l2_table, l2_size);
     if (ret < 0) {
         fprintf(stderr, "ERROR: I/O error in check_refcounts_l2\n");
         res->check_errors++;
@@ -1477,7 +1497,7 @@ static int check_refcounts_l1(BlockDriverState *bs,
                               int64_t l1_table_offset, int l1_size,
                               int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL, l2_offset, l1_size2;
     int i, ret;
 
@@ -1498,7 +1518,7 @@ static int check_refcounts_l1(BlockDriverState *bs,
             res->check_errors++;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pread(bs->file->bs, l1_table_offset, l1_table, l1_size2);
         if (ret < 0) {
             fprintf(stderr, "ERROR: I/O error in check_refcounts_l1\n");
             res->check_errors++;
@@ -1554,7 +1574,7 @@ fail:
 static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
                               BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table = qemu_blockalign(bs, s->cluster_size);
     int ret;
     uint64_t refcount;
@@ -1596,7 +1616,7 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
             }
         }
 
-        ret = bdrv_pread(bs->file, l2_offset, l2_table,
+        ret = bdrv_pread(bs->file->bs, l2_offset, l2_table,
                          s->l2_size * sizeof(uint64_t));
         if (ret < 0) {
             fprintf(stderr, "ERROR: Could not read L2 table: %s\n",
@@ -1648,7 +1668,8 @@ static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
                 goto fail;
             }
 
-            ret = bdrv_pwrite(bs->file, l2_offset, l2_table, s->cluster_size);
+            ret = bdrv_pwrite(bs->file->bs, l2_offset, l2_table,
+                              s->cluster_size);
             if (ret < 0) {
                 fprintf(stderr, "ERROR: Could not write L2 table: %s\n",
                         strerror(-ret));
@@ -1673,7 +1694,7 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                            BdrvCheckMode fix, bool *rebuild,
                            void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i, size;
     int ret;
 
@@ -1703,11 +1724,11 @@ static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                     goto resize_fail;
                 }
 
-                ret = bdrv_truncate(bs->file, offset + s->cluster_size);
+                ret = bdrv_truncate(bs->file->bs, offset + s->cluster_size);
                 if (ret < 0) {
                     goto resize_fail;
                 }
-                size = bdrv_getlength(bs->file);
+                size = bdrv_getlength(bs->file->bs);
                 if (size < 0) {
                     ret = size;
                     goto resize_fail;
@@ -1776,7 +1797,7 @@ static int calculate_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                                BdrvCheckMode fix, bool *rebuild,
                                void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
     QCowSnapshot *sn;
     int ret;
@@ -1840,7 +1861,7 @@ static void compare_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                               int64_t *highest_cluster,
                               void *refcount_table, int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
     uint64_t refcount1, refcount2;
     int ret;
@@ -1917,7 +1938,7 @@ static int64_t alloc_clusters_imrt(BlockDriverState *bs,
                                    int64_t *imrt_nb_clusters,
                                    int64_t *first_free_cluster)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t cluster = *first_free_cluster, i;
     bool first_gap = true;
     int contiguous_free_clusters;
@@ -1987,7 +2008,7 @@ static int rebuild_refcount_structure(BlockDriverState *bs,
                                       void **refcount_table,
                                       int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t first_free_cluster = 0, reftable_offset = -1, cluster = 0;
     int64_t refblock_offset, refblock_start, refblock_index;
     uint32_t reftable_size = 0;
@@ -2081,7 +2102,7 @@ write_refblocks:
         on_disk_refblock = (void *)((char *) *refcount_table +
                                     refblock_index * s->cluster_size);
 
-        ret = bdrv_write(bs->file, refblock_offset / BDRV_SECTOR_SIZE,
+        ret = bdrv_write(bs->file->bs, refblock_offset / BDRV_SECTOR_SIZE,
                          on_disk_refblock, s->cluster_sectors);
         if (ret < 0) {
             fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret));
@@ -2130,7 +2151,7 @@ write_refblocks:
     }
 
     assert(reftable_size < INT_MAX / sizeof(uint64_t));
-    ret = bdrv_pwrite(bs->file, reftable_offset, on_disk_reftable,
+    ret = bdrv_pwrite(bs->file->bs, reftable_offset, on_disk_reftable,
                       reftable_size * sizeof(uint64_t));
     if (ret < 0) {
         fprintf(stderr, "ERROR writing reftable: %s\n", strerror(-ret));
@@ -2142,8 +2163,8 @@ write_refblocks:
                  reftable_offset);
     cpu_to_be32w(&reftable_offset_and_clusters.reftable_clusters,
                  size_to_clusters(s, reftable_size * sizeof(uint64_t)));
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader,
-                                              refcount_table_offset),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader,
+                                                  refcount_table_offset),
                            &reftable_offset_and_clusters,
                            sizeof(reftable_offset_and_clusters));
     if (ret < 0) {
@@ -2174,14 +2195,14 @@ fail:
 int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                           BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     BdrvCheckResult pre_compare_res;
     int64_t size, highest_cluster, nb_clusters;
     void *refcount_table = NULL;
     bool rebuild = false;
     int ret;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     if (size < 0) {
         res->check_errors++;
         return size;
@@ -2311,7 +2332,7 @@ fail:
 int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int chk = s->overlap_check & ~ign;
     int i, j;
 
@@ -2390,7 +2411,7 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                 return -ENOMEM;
             }
 
-            ret = bdrv_pread(bs->file, l1_ofs, l1, l1_sz2);
+            ret = bdrv_pread(bs->file->bs, l1_ofs, l1, l1_sz2);
             if (ret < 0) {
                 g_free(l1);
                 return ret;
@@ -2451,3 +2472,450 @@ int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
 
     return 0;
 }
+
+/* A pointer to a function of this type is given to walk_over_reftable(). That
+ * function will create refblocks and pass them to a RefblockFinishOp once they
+ * are completed (@refblock). @refblock_empty is set if the refblock is
+ * completely empty.
+ *
+ * Along with the refblock, a corresponding reftable entry is passed, in the
+ * reftable @reftable (which may be reallocated) at @reftable_index.
+ *
+ * @allocated should be set to true if a new cluster has been allocated.
+ */
+typedef int (RefblockFinishOp)(BlockDriverState *bs, uint64_t **reftable,
+                               uint64_t reftable_index, uint64_t *reftable_size,
+                               void *refblock, bool refblock_empty,
+                               bool *allocated, Error **errp);
+
+/**
+ * This "operation" for walk_over_reftable() allocates the refblock on disk (if
+ * it is not empty) and inserts its offset into the new reftable. The size of
+ * this new reftable is increased as required.
+ */
+static int alloc_refblock(BlockDriverState *bs, uint64_t **reftable,
+                          uint64_t reftable_index, uint64_t *reftable_size,
+                          void *refblock, bool refblock_empty, bool *allocated,
+                          Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int64_t offset;
+
+    if (!refblock_empty && reftable_index >= *reftable_size) {
+        uint64_t *new_reftable;
+        uint64_t new_reftable_size;
+
+        new_reftable_size = ROUND_UP(reftable_index + 1,
+                                     s->cluster_size / sizeof(uint64_t));
+        if (new_reftable_size > QCOW_MAX_REFTABLE_SIZE / sizeof(uint64_t)) {
+            error_setg(errp,
+                       "This operation would make the refcount table grow "
+                       "beyond the maximum size supported by QEMU, aborting");
+            return -ENOTSUP;
+        }
+
+        new_reftable = g_try_realloc(*reftable, new_reftable_size *
+                                                sizeof(uint64_t));
+        if (!new_reftable) {
+            error_setg(errp, "Failed to increase reftable buffer size");
+            return -ENOMEM;
+        }
+
+        memset(new_reftable + *reftable_size, 0,
+               (new_reftable_size - *reftable_size) * sizeof(uint64_t));
+
+        *reftable      = new_reftable;
+        *reftable_size = new_reftable_size;
+    }
+
+    if (!refblock_empty && !(*reftable)[reftable_index]) {
+        offset = qcow2_alloc_clusters(bs, s->cluster_size);
+        if (offset < 0) {
+            error_setg_errno(errp, -offset, "Failed to allocate refblock");
+            return offset;
+        }
+        (*reftable)[reftable_index] = offset;
+        *allocated = true;
+    }
+
+    return 0;
+}
+
+/**
+ * This "operation" for walk_over_reftable() writes the refblock to disk at the
+ * offset specified by the new reftable's entry. It does not modify the new
+ * reftable or change any refcounts.
+ */
+static int flush_refblock(BlockDriverState *bs, uint64_t **reftable,
+                          uint64_t reftable_index, uint64_t *reftable_size,
+                          void *refblock, bool refblock_empty, bool *allocated,
+                          Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int64_t offset;
+    int ret;
+
+    if (reftable_index < *reftable_size && (*reftable)[reftable_index]) {
+        offset = (*reftable)[reftable_index];
+
+        ret = qcow2_pre_write_overlap_check(bs, 0, offset, s->cluster_size);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Overlap check failed");
+            return ret;
+        }
+
+        ret = bdrv_pwrite(bs->file->bs, offset, refblock, s->cluster_size);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to write refblock");
+            return ret;
+        }
+    } else {
+        assert(refblock_empty);
+    }
+
+    return 0;
+}
+
+/**
+ * This function walks over the existing reftable and every referenced refblock;
+ * if @new_set_refcount is non-NULL, it is called for every refcount entry to
+ * create an equal new entry in the passed @new_refblock. Once that
+ * @new_refblock is completely filled, @operation will be called.
+ *
+ * @status_cb and @cb_opaque are used for the amend operation's status callback.
+ * @index is the index of the walk_over_reftable() calls and @total is the total
+ * number of walk_over_reftable() calls per amend operation. Both are used for
+ * calculating the parameters for the status callback.
+ *
+ * @allocated is set to true if a new cluster has been allocated.
+ */
+static int walk_over_reftable(BlockDriverState *bs, uint64_t **new_reftable,
+                              uint64_t *new_reftable_index,
+                              uint64_t *new_reftable_size,
+                              void *new_refblock, int new_refblock_size,
+                              int new_refcount_bits,
+                              RefblockFinishOp *operation, bool *allocated,
+                              Qcow2SetRefcountFunc *new_set_refcount,
+                              BlockDriverAmendStatusCB *status_cb,
+                              void *cb_opaque, int index, int total,
+                              Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t reftable_index;
+    bool new_refblock_empty = true;
+    int refblock_index;
+    int new_refblock_index = 0;
+    int ret;
+
+    for (reftable_index = 0; reftable_index < s->refcount_table_size;
+         reftable_index++)
+    {
+        uint64_t refblock_offset = s->refcount_table[reftable_index]
+                                 & REFT_OFFSET_MASK;
+
+        status_cb(bs, (uint64_t)index * s->refcount_table_size + reftable_index,
+                  (uint64_t)total * s->refcount_table_size, cb_opaque);
+
+        if (refblock_offset) {
+            void *refblock;
+
+            if (offset_into_cluster(s, refblock_offset)) {
+                qcow2_signal_corruption(bs, true, -1, -1, "Refblock offset %#"
+                                        PRIx64 " unaligned (reftable index: %#"
+                                        PRIx64 ")", refblock_offset,
+                                        reftable_index);
+                error_setg(errp,
+                           "Image is corrupt (unaligned refblock offset)");
+                return -EIO;
+            }
+
+            ret = qcow2_cache_get(bs, s->refcount_block_cache, refblock_offset,
+                                  &refblock);
+            if (ret < 0) {
+                error_setg_errno(errp, -ret, "Failed to retrieve refblock");
+                return ret;
+            }
+
+            for (refblock_index = 0; refblock_index < s->refcount_block_size;
+                 refblock_index++)
+            {
+                uint64_t refcount;
+
+                if (new_refblock_index >= new_refblock_size) {
+                    /* new_refblock is now complete */
+                    ret = operation(bs, new_reftable, *new_reftable_index,
+                                    new_reftable_size, new_refblock,
+                                    new_refblock_empty, allocated, errp);
+                    if (ret < 0) {
+                        qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+                        return ret;
+                    }
+
+                    (*new_reftable_index)++;
+                    new_refblock_index = 0;
+                    new_refblock_empty = true;
+                }
+
+                refcount = s->get_refcount(refblock, refblock_index);
+                if (new_refcount_bits < 64 && refcount >> new_refcount_bits) {
+                    uint64_t offset;
+
+                    qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+
+                    offset = ((reftable_index << s->refcount_block_bits)
+                              + refblock_index) << s->cluster_bits;
+
+                    error_setg(errp, "Cannot decrease refcount entry width to "
+                               "%i bits: Cluster at offset %#" PRIx64 " has a "
+                               "refcount of %" PRIu64, new_refcount_bits,
+                               offset, refcount);
+                    return -EINVAL;
+                }
+
+                if (new_set_refcount) {
+                    new_set_refcount(new_refblock, new_refblock_index++,
+                                     refcount);
+                } else {
+                    new_refblock_index++;
+                }
+                new_refblock_empty = new_refblock_empty && refcount == 0;
+            }
+
+            qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+        } else {
+            /* No refblock means every refcount is 0 */
+            for (refblock_index = 0; refblock_index < s->refcount_block_size;
+                 refblock_index++)
+            {
+                if (new_refblock_index >= new_refblock_size) {
+                    /* new_refblock is now complete */
+                    ret = operation(bs, new_reftable, *new_reftable_index,
+                                    new_reftable_size, new_refblock,
+                                    new_refblock_empty, allocated, errp);
+                    if (ret < 0) {
+                        return ret;
+                    }
+
+                    (*new_reftable_index)++;
+                    new_refblock_index = 0;
+                    new_refblock_empty = true;
+                }
+
+                if (new_set_refcount) {
+                    new_set_refcount(new_refblock, new_refblock_index++, 0);
+                } else {
+                    new_refblock_index++;
+                }
+            }
+        }
+    }
+
+    if (new_refblock_index > 0) {
+        /* Complete the potentially existing partially filled final refblock */
+        if (new_set_refcount) {
+            for (; new_refblock_index < new_refblock_size;
+                 new_refblock_index++)
+            {
+                new_set_refcount(new_refblock, new_refblock_index, 0);
+            }
+        }
+
+        ret = operation(bs, new_reftable, *new_reftable_index,
+                        new_reftable_size, new_refblock, new_refblock_empty,
+                        allocated, errp);
+        if (ret < 0) {
+            return ret;
+        }
+
+        (*new_reftable_index)++;
+    }
+
+    status_cb(bs, (uint64_t)(index + 1) * s->refcount_table_size,
+              (uint64_t)total * s->refcount_table_size, cb_opaque);
+
+    return 0;
+}
+
+int qcow2_change_refcount_order(BlockDriverState *bs, int refcount_order,
+                                BlockDriverAmendStatusCB *status_cb,
+                                void *cb_opaque, Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    Qcow2GetRefcountFunc *new_get_refcount;
+    Qcow2SetRefcountFunc *new_set_refcount;
+    void *new_refblock = qemu_blockalign(bs->file->bs, s->cluster_size);
+    uint64_t *new_reftable = NULL, new_reftable_size = 0;
+    uint64_t *old_reftable, old_reftable_size, old_reftable_offset;
+    uint64_t new_reftable_index = 0;
+    uint64_t i;
+    int64_t new_reftable_offset = 0, allocated_reftable_size = 0;
+    int new_refblock_size, new_refcount_bits = 1 << refcount_order;
+    int old_refcount_order;
+    int walk_index = 0;
+    int ret;
+    bool new_allocation;
+
+    assert(s->qcow_version >= 3);
+    assert(refcount_order >= 0 && refcount_order <= 6);
+
+    /* see qcow2_open() */
+    new_refblock_size = 1 << (s->cluster_bits - (refcount_order - 3));
+
+    new_get_refcount = get_refcount_funcs[refcount_order];
+    new_set_refcount = set_refcount_funcs[refcount_order];
+
+
+    do {
+        int total_walks;
+
+        new_allocation = false;
+
+        /* At least we have to do this walk and the one which writes the
+         * refblocks; also, at least we have to do this loop here at least
+         * twice (normally), first to do the allocations, and second to
+         * determine that everything is correctly allocated, this then makes
+         * three walks in total */
+        total_walks = MAX(walk_index + 2, 3);
+
+        /* First, allocate the structures so they are present in the refcount
+         * structures */
+        ret = walk_over_reftable(bs, &new_reftable, &new_reftable_index,
+                                 &new_reftable_size, NULL, new_refblock_size,
+                                 new_refcount_bits, &alloc_refblock,
+                                 &new_allocation, NULL, status_cb, cb_opaque,
+                                 walk_index++, total_walks, errp);
+        if (ret < 0) {
+            goto done;
+        }
+
+        new_reftable_index = 0;
+
+        if (new_allocation) {
+            if (new_reftable_offset) {
+                qcow2_free_clusters(bs, new_reftable_offset,
+                                    allocated_reftable_size * sizeof(uint64_t),
+                                    QCOW2_DISCARD_NEVER);
+            }
+
+            new_reftable_offset = qcow2_alloc_clusters(bs, new_reftable_size *
+                                                           sizeof(uint64_t));
+            if (new_reftable_offset < 0) {
+                error_setg_errno(errp, -new_reftable_offset,
+                                 "Failed to allocate the new reftable");
+                ret = new_reftable_offset;
+                goto done;
+            }
+            allocated_reftable_size = new_reftable_size;
+        }
+    } while (new_allocation);
+
+    /* Second, write the new refblocks */
+    ret = walk_over_reftable(bs, &new_reftable, &new_reftable_index,
+                             &new_reftable_size, new_refblock,
+                             new_refblock_size, new_refcount_bits,
+                             &flush_refblock, &new_allocation, new_set_refcount,
+                             status_cb, cb_opaque, walk_index, walk_index + 1,
+                             errp);
+    if (ret < 0) {
+        goto done;
+    }
+    assert(!new_allocation);
+
+
+    /* Write the new reftable */
+    ret = qcow2_pre_write_overlap_check(bs, 0, new_reftable_offset,
+                                        new_reftable_size * sizeof(uint64_t));
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Overlap check failed");
+        goto done;
+    }
+
+    for (i = 0; i < new_reftable_size; i++) {
+        cpu_to_be64s(&new_reftable[i]);
+    }
+
+    ret = bdrv_pwrite(bs->file->bs, new_reftable_offset, new_reftable,
+                      new_reftable_size * sizeof(uint64_t));
+
+    for (i = 0; i < new_reftable_size; i++) {
+        be64_to_cpus(&new_reftable[i]);
+    }
+
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Failed to write the new reftable");
+        goto done;
+    }
+
+
+    /* Empty the refcount cache */
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Failed to flush the refblock cache");
+        goto done;
+    }
+
+    /* Update the image header to point to the new reftable; this only updates
+     * the fields which are relevant to qcow2_update_header(); other fields
+     * such as s->refcount_table or s->refcount_bits stay stale for now
+     * (because we have to restore everything if qcow2_update_header() fails) */
+    old_refcount_order  = s->refcount_order;
+    old_reftable_size   = s->refcount_table_size;
+    old_reftable_offset = s->refcount_table_offset;
+
+    s->refcount_order        = refcount_order;
+    s->refcount_table_size   = new_reftable_size;
+    s->refcount_table_offset = new_reftable_offset;
+
+    ret = qcow2_update_header(bs);
+    if (ret < 0) {
+        s->refcount_order        = old_refcount_order;
+        s->refcount_table_size   = old_reftable_size;
+        s->refcount_table_offset = old_reftable_offset;
+        error_setg_errno(errp, -ret, "Failed to update the qcow2 header");
+        goto done;
+    }
+
+    /* Now update the rest of the in-memory information */
+    old_reftable = s->refcount_table;
+    s->refcount_table = new_reftable;
+
+    s->refcount_bits = 1 << refcount_order;
+    s->refcount_max = UINT64_C(1) << (s->refcount_bits - 1);
+    s->refcount_max += s->refcount_max - 1;
+
+    s->refcount_block_bits = s->cluster_bits - (refcount_order - 3);
+    s->refcount_block_size = 1 << s->refcount_block_bits;
+
+    s->get_refcount = new_get_refcount;
+    s->set_refcount = new_set_refcount;
+
+    /* For cleaning up all old refblocks and the old reftable below the "done"
+     * label */
+    new_reftable        = old_reftable;
+    new_reftable_size   = old_reftable_size;
+    new_reftable_offset = old_reftable_offset;
+
+done:
+    if (new_reftable) {
+        /* On success, new_reftable actually points to the old reftable (and
+         * new_reftable_size is the old reftable's size); but that is just
+         * fine */
+        for (i = 0; i < new_reftable_size; i++) {
+            uint64_t offset = new_reftable[i] & REFT_OFFSET_MASK;
+            if (offset) {
+                qcow2_free_clusters(bs, offset, s->cluster_size,
+                                    QCOW2_DISCARD_OTHER);
+            }
+        }
+        g_free(new_reftable);
+
+        if (new_reftable_offset > 0) {
+            qcow2_free_clusters(bs, new_reftable_offset,
+                                new_reftable_size * sizeof(uint64_t),
+                                QCOW2_DISCARD_OTHER);
+        }
+    }
+
+    qemu_vfree(new_refblock);
+    return ret;
+}
diff --git a/qemu/block/qcow2-snapshot.c b/qemu/block/qcow2-snapshot.c
index b6f58c13e..5f4a17e47 100644
--- a/qemu/block/qcow2-snapshot.c
+++ b/qemu/block/qcow2-snapshot.c
@@ -22,14 +22,16 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
 #include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 void qcow2_free_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     for(i = 0; i < s->nb_snapshots; i++) {
@@ -43,7 +45,7 @@ void qcow2_free_snapshots(BlockDriverState *bs)
 
 int qcow2_read_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
     QCowSnapshot *sn;
@@ -64,7 +66,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
     for(i = 0; i < s->nb_snapshots; i++) {
         /* Read statically sized part of the snapshot header */
         offset = align_offset(offset, 8);
-        ret = bdrv_pread(bs->file, offset, &h, sizeof(h));
+        ret = bdrv_pread(bs->file->bs, offset, &h, sizeof(h));
         if (ret < 0) {
             goto fail;
         }
@@ -83,7 +85,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
         name_size = be16_to_cpu(h.name_size);
 
         /* Read extra data */
-        ret = bdrv_pread(bs->file, offset, &extra,
+        ret = bdrv_pread(bs->file->bs, offset, &extra,
                          MIN(sizeof(extra), extra_data_size));
         if (ret < 0) {
             goto fail;
@@ -102,7 +104,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
 
         /* Read snapshot ID */
         sn->id_str = g_malloc(id_str_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->id_str, id_str_size);
+        ret = bdrv_pread(bs->file->bs, offset, sn->id_str, id_str_size);
         if (ret < 0) {
             goto fail;
         }
@@ -111,7 +113,7 @@ int qcow2_read_snapshots(BlockDriverState *bs)
 
         /* Read snapshot name */
         sn->name = g_malloc(name_size + 1);
-        ret = bdrv_pread(bs->file, offset, sn->name, name_size);
+        ret = bdrv_pread(bs->file->bs, offset, sn->name, name_size);
         if (ret < 0) {
             goto fail;
         }
@@ -136,7 +138,7 @@ fail:
 /* add at the end of the file a new list of snapshots */
 static int qcow2_write_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
@@ -214,25 +216,25 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
         h.name_size = cpu_to_be16(name_size);
         offset = align_offset(offset, 8);
 
-        ret = bdrv_pwrite(bs->file, offset, &h, sizeof(h));
+        ret = bdrv_pwrite(bs->file->bs, offset, &h, sizeof(h));
         if (ret < 0) {
             goto fail;
         }
         offset += sizeof(h);
 
-        ret = bdrv_pwrite(bs->file, offset, &extra, sizeof(extra));
+        ret = bdrv_pwrite(bs->file->bs, offset, &extra, sizeof(extra));
         if (ret < 0) {
             goto fail;
         }
         offset += sizeof(extra);
 
-        ret = bdrv_pwrite(bs->file, offset, sn->id_str, id_str_size);
+        ret = bdrv_pwrite(bs->file->bs, offset, sn->id_str, id_str_size);
         if (ret < 0) {
             goto fail;
         }
         offset += id_str_size;
 
-        ret = bdrv_pwrite(bs->file, offset, sn->name, name_size);
+        ret = bdrv_pwrite(bs->file->bs, offset, sn->name, name_size);
         if (ret < 0) {
             goto fail;
         }
@@ -254,7 +256,7 @@ static int qcow2_write_snapshots(BlockDriverState *bs)
     header_data.nb_snapshots        = cpu_to_be32(s->nb_snapshots);
     header_data.snapshots_offset    = cpu_to_be64(snapshots_offset);
 
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, nb_snapshots),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, nb_snapshots),
                            &header_data, sizeof(header_data));
     if (ret < 0) {
         goto fail;
@@ -278,7 +280,7 @@ fail:
 static void find_new_snapshot_id(BlockDriverState *bs,
                                  char *id_str, int id_str_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i;
     unsigned long id, id_max = 0;
@@ -296,7 +298,7 @@ static int find_snapshot_by_id_and_name(BlockDriverState *bs,
                                         const char *id,
                                         const char *name)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     if (id && name) {
@@ -338,7 +340,7 @@ static int find_snapshot_by_id_or_name(BlockDriverState *bs,
 /* if no id is provided, a new one is constructed */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *new_snapshot_list = NULL;
     QCowSnapshot *old_snapshot_list = NULL;
     QCowSnapshot sn1, *sn = &sn1;
@@ -396,7 +398,7 @@ int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
         goto fail;
     }
 
-    ret = bdrv_pwrite(bs->file, sn->l1_table_offset, l1_table,
+    ret = bdrv_pwrite(bs->file->bs, sn->l1_table_offset, l1_table,
                       s->l1_size * sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
@@ -461,7 +463,7 @@ fail:
 /* copy the snapshot 'snapshot_name' into the current disk image */
 int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i, snapshot_index;
     int cur_l1_bytes, sn_l1_bytes;
@@ -509,7 +511,8 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
+    ret = bdrv_pread(bs->file->bs, sn->l1_table_offset,
+                     sn_l1_table, sn_l1_bytes);
     if (ret < 0) {
         goto fail;
     }
@@ -526,7 +529,7 @@ int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto fail;
     }
 
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset, sn_l1_table,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->l1_table_offset, sn_l1_table,
                            cur_l1_bytes);
     if (ret < 0) {
         goto fail;
@@ -587,7 +590,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
                           const char *name,
                           Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot sn;
     int snapshot_index, ret;
 
@@ -650,7 +653,7 @@ int qcow2_snapshot_delete(BlockDriverState *bs,
 
 int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUSnapshotInfo *sn_tab, *sn_info;
     QCowSnapshot *sn;
     int i;
@@ -683,7 +686,7 @@ int qcow2_snapshot_load_tmp(BlockDriverState *bs,
                             Error **errp)
 {
     int i, snapshot_index;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     uint64_t *new_l1_table;
     int new_l1_bytes;
@@ -706,13 +709,14 @@ int qcow2_snapshot_load_tmp(BlockDriverState *bs,
         return -EFBIG;
     }
     new_l1_bytes = sn->l1_size * sizeof(uint64_t);
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_bytes, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
+    ret = bdrv_pread(bs->file->bs, sn->l1_table_offset,
+                     new_l1_table, new_l1_bytes);
     if (ret < 0) {
         error_setg(errp, "Failed to read l1 table for snapshot");
         qemu_vfree(new_l1_table);
diff --git a/qemu/block/qcow2.c b/qemu/block/qcow2.c
index 76c331b38..470734be9 100644
--- a/qemu/block/qcow2.c
+++ b/qemu/block/qcow2.c
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include <zlib.h>
 #include "block/qcow2.h"
@@ -34,6 +35,7 @@
 #include "qapi-event.h"
 #include "trace.h"
 #include "qemu/option_int.h"
+#include "qemu/cutils.h"
 
 /*
   Differences with QCOW:
@@ -85,7 +87,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                                  uint64_t end_offset, void **p_feature_table,
                                  Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowExtension ext;
     uint64_t offset;
     int ret;
@@ -104,7 +106,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         printf("attempting to read extended header in offset %lu\n", offset);
 #endif
 
-        ret = bdrv_pread(bs->file, offset, &ext, sizeof(ext));
+        ret = bdrv_pread(bs->file->bs, offset, &ext, sizeof(ext));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "qcow2_read_extension: ERROR: "
                              "pread fail from offset %" PRIu64, offset);
@@ -132,7 +134,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                            sizeof(bs->backing_format));
                 return 2;
             }
-            ret = bdrv_pread(bs->file, offset, bs->backing_format, ext.len);
+            ret = bdrv_pread(bs->file->bs, offset, bs->backing_format, ext.len);
             if (ret < 0) {
                 error_setg_errno(errp, -ret, "ERROR: ext_backing_format: "
                                  "Could not read format name");
@@ -148,7 +150,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
         case QCOW2_EXT_MAGIC_FEATURE_TABLE:
             if (p_feature_table != NULL) {
                 void* feature_table = g_malloc0(ext.len + 2 * sizeof(Qcow2Feature));
-                ret = bdrv_pread(bs->file, offset , feature_table, ext.len);
+                ret = bdrv_pread(bs->file->bs, offset , feature_table, ext.len);
                 if (ret < 0) {
                     error_setg_errno(errp, -ret, "ERROR: ext_feature_table: "
                                      "Could not read table");
@@ -169,7 +171,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
                 uext->len = ext.len;
                 QLIST_INSERT_HEAD(&s->unknown_header_ext, uext, next);
 
-                ret = bdrv_pread(bs->file, offset , uext->data, uext->len);
+                ret = bdrv_pread(bs->file->bs, offset , uext->data, uext->len);
                 if (ret < 0) {
                     error_setg_errno(errp, -ret, "ERROR: unknown extension: "
                                      "Could not read data");
@@ -187,7 +189,7 @@ static int qcow2_read_extensions(BlockDriverState *bs, uint64_t start_offset,
 
 static void cleanup_unknown_header_ext(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2UnknownHeaderExtension *uext, *next;
 
     QLIST_FOREACH_SAFE(uext, &s->unknown_header_ext, next, next) {
@@ -196,22 +198,8 @@ static void cleanup_unknown_header_ext(BlockDriverState *bs)
     }
 }
 
-static void GCC_FMT_ATTR(3, 4) report_unsupported(BlockDriverState *bs,
-    Error **errp, const char *fmt, ...)
-{
-    char msg[64];
-    va_list ap;
-
-    va_start(ap, fmt);
-    vsnprintf(msg, sizeof(msg), fmt, ap);
-    va_end(ap);
-
-    error_setg(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-               bdrv_get_device_or_node_name(bs), "qcow2", msg);
-}
-
-static void report_unsupported_feature(BlockDriverState *bs,
-    Error **errp, Qcow2Feature *table, uint64_t mask)
+static void report_unsupported_feature(Error **errp, Qcow2Feature *table,
+                                       uint64_t mask)
 {
     char *features = g_strdup("");
     char *old;
@@ -236,7 +224,7 @@ static void report_unsupported_feature(BlockDriverState *bs,
         g_free(old);
     }
 
-    report_unsupported(bs, errp, "%s", features);
+    error_setg(errp, "Unsupported qcow2 feature(s): %s", features);
     g_free(features);
 }
 
@@ -249,7 +237,7 @@ static void report_unsupported_feature(BlockDriverState *bs,
  */
 int qcow2_mark_dirty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t val;
     int ret;
 
@@ -260,12 +248,12 @@ int qcow2_mark_dirty(BlockDriverState *bs)
     }
 
     val = cpu_to_be64(s->incompatible_features | QCOW2_INCOMPAT_DIRTY);
-    ret = bdrv_pwrite(bs->file, offsetof(QCowHeader, incompatible_features),
+    ret = bdrv_pwrite(bs->file->bs, offsetof(QCowHeader, incompatible_features),
                       &val, sizeof(val));
     if (ret < 0) {
         return ret;
     }
-    ret = bdrv_flush(bs->file);
+    ret = bdrv_flush(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -282,7 +270,7 @@ int qcow2_mark_dirty(BlockDriverState *bs)
  */
 static int qcow2_mark_clean(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
         int ret;
@@ -304,7 +292,7 @@ static int qcow2_mark_clean(BlockDriverState *bs)
  */
 int qcow2_mark_corrupt(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     s->incompatible_features |= QCOW2_INCOMPAT_CORRUPT;
     return qcow2_update_header(bs);
@@ -316,7 +304,7 @@ int qcow2_mark_corrupt(BlockDriverState *bs)
  */
 int qcow2_mark_consistent(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_CORRUPT) {
         int ret = bdrv_flush(bs);
@@ -351,7 +339,7 @@ static int qcow2_check(BlockDriverState *bs, BdrvCheckResult *result,
 static int validate_table_offset(BlockDriverState *bs, uint64_t offset,
                                  uint64_t entries, size_t entry_len)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t size;
 
     /* Use signed INT64_MAX as the maximum even for uint64_t header fields,
@@ -467,6 +455,11 @@ static QemuOptsList qcow2_runtime_opts = {
             .type = QEMU_OPT_SIZE,
             .help = "Maximum refcount block cache size",
         },
+        {
+            .name = QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Clean unused cache entries after this time (in seconds)",
+        },
         { /* end of list */ }
     },
 };
@@ -482,11 +475,54 @@ static const char *overlap_bool_option_names[QCOW2_OL_MAX_BITNR] = {
     [QCOW2_OL_INACTIVE_L2_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L2,
 };
 
+static void cache_clean_timer_cb(void *opaque)
+{
+    BlockDriverState *bs = opaque;
+    BDRVQcow2State *s = bs->opaque;
+    qcow2_cache_clean_unused(bs, s->l2_table_cache);
+    qcow2_cache_clean_unused(bs, s->refcount_block_cache);
+    timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              (int64_t) s->cache_clean_interval * 1000);
+}
+
+static void cache_clean_timer_init(BlockDriverState *bs, AioContext *context)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_interval > 0) {
+        s->cache_clean_timer = aio_timer_new(context, QEMU_CLOCK_VIRTUAL,
+                                             SCALE_MS, cache_clean_timer_cb,
+                                             bs);
+        timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                  (int64_t) s->cache_clean_interval * 1000);
+    }
+}
+
+static void cache_clean_timer_del(BlockDriverState *bs)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_timer) {
+        timer_del(s->cache_clean_timer);
+        timer_free(s->cache_clean_timer);
+        s->cache_clean_timer = NULL;
+    }
+}
+
+static void qcow2_detach_aio_context(BlockDriverState *bs)
+{
+    cache_clean_timer_del(bs);
+}
+
+static void qcow2_attach_aio_context(BlockDriverState *bs,
+                                     AioContext *new_context)
+{
+    cache_clean_timer_init(bs, new_context);
+}
+
 static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts,
                              uint64_t *l2_cache_size,
                              uint64_t *refcount_cache_size, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t combined_cache_size;
     bool l2_cache_size_set, refcount_cache_size_set, combined_cache_size_set;
 
@@ -541,22 +577,246 @@ static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts,
     }
 }
 
+typedef struct Qcow2ReopenState {
+    Qcow2Cache *l2_table_cache;
+    Qcow2Cache *refcount_block_cache;
+    bool use_lazy_refcounts;
+    int overlap_check;
+    bool discard_passthrough[QCOW2_DISCARD_MAX];
+    uint64_t cache_clean_interval;
+} Qcow2ReopenState;
+
+static int qcow2_update_options_prepare(BlockDriverState *bs,
+                                        Qcow2ReopenState *r,
+                                        QDict *options, int flags,
+                                        Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    QemuOpts *opts = NULL;
+    const char *opt_overlap_check, *opt_overlap_check_template;
+    int overlap_check_template = 0;
+    uint64_t l2_cache_size, refcount_cache_size;
+    int i;
+    Error *local_err = NULL;
+    int ret;
+
+    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* get L2 table/refcount block cache size from command line options */
+    read_cache_sizes(bs, opts, &l2_cache_size, &refcount_cache_size,
+                     &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    l2_cache_size /= s->cluster_size;
+    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
+        l2_cache_size = MIN_L2_CACHE_SIZE;
+    }
+    if (l2_cache_size > INT_MAX) {
+        error_setg(errp, "L2 cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    refcount_cache_size /= s->cluster_size;
+    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
+        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
+    }
+    if (refcount_cache_size > INT_MAX) {
+        error_setg(errp, "Refcount cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* alloc new L2 table/refcount block cache, flush old one */
+    if (s->l2_table_cache) {
+        ret = qcow2_cache_flush(bs, s->l2_table_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret, "Failed to flush the L2 table cache");
+            goto fail;
+        }
+    }
+
+    if (s->refcount_block_cache) {
+        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret,
+                             "Failed to flush the refcount block cache");
+            goto fail;
+        }
+    }
+
+    r->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
+    r->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
+    if (r->l2_table_cache == NULL || r->refcount_block_cache == NULL) {
+        error_setg(errp, "Could not allocate metadata caches");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    /* New interval for cache cleanup timer */
+    r->cache_clean_interval =
+        qemu_opt_get_number(opts, QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+                            s->cache_clean_interval);
+    if (r->cache_clean_interval > UINT_MAX) {
+        error_setg(errp, "Cache clean interval too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* lazy-refcounts; flush if going from enabled to disabled */
+    r->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
+        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
+    if (r->use_lazy_refcounts && s->qcow_version < 3) {
+        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
+                   "qemu 1.1 compatibility level");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    if (s->use_lazy_refcounts && !r->use_lazy_refcounts) {
+        ret = qcow2_mark_clean(bs);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to disable lazy refcounts");
+            goto fail;
+        }
+    }
+
+    /* Overlap check options */
+    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
+    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
+    if (opt_overlap_check_template && opt_overlap_check &&
+        strcmp(opt_overlap_check_template, opt_overlap_check))
+    {
+        error_setg(errp, "Conflicting values for qcow2 options '"
+                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
+                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (!opt_overlap_check) {
+        opt_overlap_check = opt_overlap_check_template ?: "cached";
+    }
+
+    if (!strcmp(opt_overlap_check, "none")) {
+        overlap_check_template = 0;
+    } else if (!strcmp(opt_overlap_check, "constant")) {
+        overlap_check_template = QCOW2_OL_CONSTANT;
+    } else if (!strcmp(opt_overlap_check, "cached")) {
+        overlap_check_template = QCOW2_OL_CACHED;
+    } else if (!strcmp(opt_overlap_check, "all")) {
+        overlap_check_template = QCOW2_OL_ALL;
+    } else {
+        error_setg(errp, "Unsupported value '%s' for qcow2 option "
+                   "'overlap-check'. Allowed are any of the following: "
+                   "none, constant, cached, all", opt_overlap_check);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    r->overlap_check = 0;
+    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
+        /* overlap-check defines a template bitmask, but every flag may be
+         * overwritten through the associated boolean option */
+        r->overlap_check |=
+            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
+                              overlap_check_template & (1 << i)) << i;
+    }
+
+    r->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
+    r->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
+    r->discard_passthrough[QCOW2_DISCARD_REQUEST] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
+                          flags & BDRV_O_UNMAP);
+    r->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
+    r->discard_passthrough[QCOW2_DISCARD_OTHER] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
+
+    ret = 0;
+fail:
+    qemu_opts_del(opts);
+    opts = NULL;
+    return ret;
+}
+
+static void qcow2_update_options_commit(BlockDriverState *bs,
+                                        Qcow2ReopenState *r)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int i;
+
+    if (s->l2_table_cache) {
+        qcow2_cache_destroy(bs, s->l2_table_cache);
+    }
+    if (s->refcount_block_cache) {
+        qcow2_cache_destroy(bs, s->refcount_block_cache);
+    }
+    s->l2_table_cache = r->l2_table_cache;
+    s->refcount_block_cache = r->refcount_block_cache;
+
+    s->overlap_check = r->overlap_check;
+    s->use_lazy_refcounts = r->use_lazy_refcounts;
+
+    for (i = 0; i < QCOW2_DISCARD_MAX; i++) {
+        s->discard_passthrough[i] = r->discard_passthrough[i];
+    }
+
+    if (s->cache_clean_interval != r->cache_clean_interval) {
+        cache_clean_timer_del(bs);
+        s->cache_clean_interval = r->cache_clean_interval;
+        cache_clean_timer_init(bs, bdrv_get_aio_context(bs));
+    }
+}
+
+static void qcow2_update_options_abort(BlockDriverState *bs,
+                                       Qcow2ReopenState *r)
+{
+    if (r->l2_table_cache) {
+        qcow2_cache_destroy(bs, r->l2_table_cache);
+    }
+    if (r->refcount_block_cache) {
+        qcow2_cache_destroy(bs, r->refcount_block_cache);
+    }
+}
+
+static int qcow2_update_options(BlockDriverState *bs, QDict *options,
+                                int flags, Error **errp)
+{
+    Qcow2ReopenState r = {};
+    int ret;
+
+    ret = qcow2_update_options_prepare(bs, &r, options, flags, errp);
+    if (ret >= 0) {
+        qcow2_update_options_commit(bs, &r);
+    } else {
+        qcow2_update_options_abort(bs, &r);
+    }
+
+    return ret;
+}
+
 static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
                       Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int len, i;
     int ret = 0;
     QCowHeader header;
-    QemuOpts *opts = NULL;
     Error *local_err = NULL;
     uint64_t ext_end;
     uint64_t l1_vm_state_index;
-    const char *opt_overlap_check, *opt_overlap_check_template;
-    int overlap_check_template = 0;
-    uint64_t l2_cache_size, refcount_cache_size;
 
-    ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
+    ret = bdrv_pread(bs->file->bs, 0, &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not read qcow2 header");
         goto fail;
@@ -581,7 +841,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         goto fail;
     }
     if (header.version < 2 || header.version > 3) {
-        report_unsupported(bs, errp, "QCOW version %" PRIu32, header.version);
+        error_setg(errp, "Unsupported qcow2 version %" PRIu32, header.version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -631,7 +891,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     if (header.header_length > sizeof(header)) {
         s->unknown_header_fields_size = header.header_length - sizeof(header);
         s->unknown_header_fields = g_malloc(s->unknown_header_fields_size);
-        ret = bdrv_pread(bs->file, sizeof(header), s->unknown_header_fields,
+        ret = bdrv_pread(bs->file->bs, sizeof(header), s->unknown_header_fields,
                          s->unknown_header_fields_size);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read unknown qcow2 header "
@@ -661,7 +921,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         void *feature_table = NULL;
         qcow2_read_extensions(bs, header.header_length, ext_end,
                               &feature_table, NULL);
-        report_unsupported_feature(bs, errp, feature_table,
+        report_unsupported_feature(errp, feature_table,
                                    s->incompatible_features &
                                    ~QCOW2_INCOMPAT_MASK);
         ret = -ENOTSUP;
@@ -705,6 +965,14 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     }
     s->crypt_method_header = header.crypt_method;
     if (s->crypt_method_header) {
+        if (bdrv_uses_whitelist() &&
+            s->crypt_method_header == QCOW_CRYPT_AES) {
+            error_report("qcow2 built-in AES encryption is deprecated");
+            error_printf("Support for it will be removed in a future release.\n"
+                         "You can use 'qemu-img convert' to switch to an\n"
+                         "unencrypted qcow2 image, or a LUKS raw image.\n");
+        }
+
         bs->encrypted = 1;
     }
 
@@ -784,14 +1052,14 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
 
     if (s->l1_size > 0) {
-        s->l1_table = qemu_try_blockalign(bs->file,
+        s->l1_table = qemu_try_blockalign(bs->file->bs,
             align_offset(s->l1_size * sizeof(uint64_t), 512));
         if (s->l1_table == NULL) {
             error_setg(errp, "Could not allocate L1 table");
             ret = -ENOMEM;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, s->l1_table_offset, s->l1_table,
+        ret = bdrv_pread(bs->file->bs, s->l1_table_offset, s->l1_table,
                          s->l1_size * sizeof(uint64_t));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read L1 table");
@@ -802,55 +1070,15 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* get L2 table/refcount block cache size from command line options */
-    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    read_cache_sizes(bs, opts, &l2_cache_size, &refcount_cache_size,
-                     &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    l2_cache_size /= s->cluster_size;
-    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
-        l2_cache_size = MIN_L2_CACHE_SIZE;
-    }
-    if (l2_cache_size > INT_MAX) {
-        error_setg(errp, "L2 cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    refcount_cache_size /= s->cluster_size;
-    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
-        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
-    }
-    if (refcount_cache_size > INT_MAX) {
-        error_setg(errp, "Refcount cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    /* alloc L2 table/refcount block cache */
-    s->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
-    s->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
-    if (s->l2_table_cache == NULL || s->refcount_block_cache == NULL) {
-        error_setg(errp, "Could not allocate metadata caches");
-        ret = -ENOMEM;
+    /* Parse driver-specific options */
+    ret = qcow2_update_options(bs, options, flags, errp);
+    if (ret < 0) {
         goto fail;
     }
 
     s->cluster_cache = g_malloc(s->cluster_size);
     /* one more sector for decompressed data alignment */
-    s->cluster_data = qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
+    s->cluster_data = qemu_try_blockalign(bs->file->bs, QCOW_MAX_CRYPT_CLUSTERS
                                                     * s->cluster_size + 512);
     if (s->cluster_data == NULL) {
         error_setg(errp, "Could not allocate temporary cluster buffer");
@@ -887,7 +1115,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
             ret = -EINVAL;
             goto fail;
         }
-        ret = bdrv_pread(bs->file, header.backing_file_offset,
+        ret = bdrv_pread(bs->file->bs, header.backing_file_offset,
                          bs->backing_file, len);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not read backing file name");
@@ -908,7 +1136,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Clear unknown autoclear feature bits */
-    if (!bs->read_only && !(flags & BDRV_O_INCOMING) && s->autoclear_features) {
+    if (!bs->read_only && !(flags & BDRV_O_INACTIVE) && s->autoclear_features) {
         s->autoclear_features = 0;
         ret = qcow2_update_header(bs);
         if (ret < 0) {
@@ -921,7 +1149,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     qemu_co_mutex_init(&s->lock);
 
     /* Repair image if dirty */
-    if (!(flags & (BDRV_O_CHECK | BDRV_O_INCOMING)) && !bs->read_only &&
+    if (!(flags & (BDRV_O_CHECK | BDRV_O_INACTIVE)) && !bs->read_only &&
         (s->incompatible_features & QCOW2_INCOMPAT_DIRTY)) {
         BdrvCheckResult result = {0};
 
@@ -932,70 +1160,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* Enable lazy_refcounts according to image and command line options */
-    s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
-        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
-
-    s->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
-    s->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
-    s->discard_passthrough[QCOW2_DISCARD_REQUEST] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
-                          flags & BDRV_O_UNMAP);
-    s->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
-    s->discard_passthrough[QCOW2_DISCARD_OTHER] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
-
-    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
-    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
-    if (opt_overlap_check_template && opt_overlap_check &&
-        strcmp(opt_overlap_check_template, opt_overlap_check))
-    {
-        error_setg(errp, "Conflicting values for qcow2 options '"
-                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
-                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
-        ret = -EINVAL;
-        goto fail;
-    }
-    if (!opt_overlap_check) {
-        opt_overlap_check = opt_overlap_check_template ?: "cached";
-    }
-
-    if (!strcmp(opt_overlap_check, "none")) {
-        overlap_check_template = 0;
-    } else if (!strcmp(opt_overlap_check, "constant")) {
-        overlap_check_template = QCOW2_OL_CONSTANT;
-    } else if (!strcmp(opt_overlap_check, "cached")) {
-        overlap_check_template = QCOW2_OL_CACHED;
-    } else if (!strcmp(opt_overlap_check, "all")) {
-        overlap_check_template = QCOW2_OL_ALL;
-    } else {
-        error_setg(errp, "Unsupported value '%s' for qcow2 option "
-                   "'overlap-check'. Allowed are either of the following: "
-                   "none, constant, cached, all", opt_overlap_check);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    s->overlap_check = 0;
-    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
-        /* overlap-check defines a template bitmask, but every flag may be
-         * overwritten through the associated boolean option */
-        s->overlap_check |=
-            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
-                              overlap_check_template & (1 << i)) << i;
-    }
-
-    qemu_opts_del(opts);
-    opts = NULL;
-
-    if (s->use_lazy_refcounts && s->qcow_version < 3) {
-        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
-                   "qemu 1.1 compatibility level");
-        ret = -EINVAL;
-        goto fail;
-    }
-
 #ifdef DEBUG_ALLOC
     {
         BdrvCheckResult result = {0};
@@ -1005,7 +1169,6 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     return ret;
 
  fail:
-    qemu_opts_del(opts);
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
     qcow2_free_snapshots(bs);
@@ -1013,6 +1176,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
+    cache_clean_timer_del(bs);
     if (s->l2_table_cache) {
         qcow2_cache_destroy(bs, s->l2_table_cache);
     }
@@ -1026,14 +1190,14 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
 
 static void qcow2_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     bs->bl.write_zeroes_alignment = s->cluster_sectors;
 }
 
 static int qcow2_set_key(BlockDriverState *bs, const char *key)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint8_t keybuf[16];
     int len, i;
     Error *err = NULL;
@@ -1066,32 +1230,104 @@ static int qcow2_set_key(BlockDriverState *bs, const char *key)
     return 0;
 }
 
-/* We have no actual commit/abort logic for qcow2, but we need to write out any
- * unwritten data if we reopen read-only. */
 static int qcow2_reopen_prepare(BDRVReopenState *state,
                                 BlockReopenQueue *queue, Error **errp)
 {
+    Qcow2ReopenState *r;
     int ret;
 
+    r = g_new0(Qcow2ReopenState, 1);
+    state->opaque = r;
+
+    ret = qcow2_update_options_prepare(state->bs, r, state->options,
+                                       state->flags, errp);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* We need to write out any unwritten data if we reopen read-only. */
     if ((state->flags & BDRV_O_RDWR) == 0) {
         ret = bdrv_flush(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
 
         ret = qcow2_mark_clean(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
     }
 
     return 0;
+
+fail:
+    qcow2_update_options_abort(state->bs, r);
+    g_free(r);
+    return ret;
+}
+
+static void qcow2_reopen_commit(BDRVReopenState *state)
+{
+    qcow2_update_options_commit(state->bs, state->opaque);
+    g_free(state->opaque);
+}
+
+static void qcow2_reopen_abort(BDRVReopenState *state)
+{
+    qcow2_update_options_abort(state->bs, state->opaque);
+    g_free(state->opaque);
+}
+
+static void qcow2_join_options(QDict *options, QDict *old_options)
+{
+    bool has_new_overlap_template =
+        qdict_haskey(options, QCOW2_OPT_OVERLAP) ||
+        qdict_haskey(options, QCOW2_OPT_OVERLAP_TEMPLATE);
+    bool has_new_total_cache_size =
+        qdict_haskey(options, QCOW2_OPT_CACHE_SIZE);
+    bool has_all_cache_options;
+
+    /* New overlap template overrides all old overlap options */
+    if (has_new_overlap_template) {
+        qdict_del(old_options, QCOW2_OPT_OVERLAP);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_TEMPLATE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_MAIN_HEADER);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_ACTIVE_L1);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_ACTIVE_L2);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_REFCOUNT_TABLE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_REFCOUNT_BLOCK);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_INACTIVE_L1);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_INACTIVE_L2);
+    }
+
+    /* New total cache size overrides all old options */
+    if (qdict_haskey(options, QCOW2_OPT_CACHE_SIZE)) {
+        qdict_del(old_options, QCOW2_OPT_L2_CACHE_SIZE);
+        qdict_del(old_options, QCOW2_OPT_REFCOUNT_CACHE_SIZE);
+    }
+
+    qdict_join(options, old_options, false);
+
+    /*
+     * If after merging all cache size options are set, an old total size is
+     * overwritten. Do keep all options, however, if all three are new. The
+     * resulting error message is what we want to happen.
+     */
+    has_all_cache_options =
+        qdict_haskey(options, QCOW2_OPT_CACHE_SIZE) ||
+        qdict_haskey(options, QCOW2_OPT_L2_CACHE_SIZE) ||
+        qdict_haskey(options, QCOW2_OPT_REFCOUNT_CACHE_SIZE);
+
+    if (has_all_cache_options && !has_new_total_cache_size) {
+        qdict_del(options, QCOW2_OPT_CACHE_SIZE);
+    }
 }
 
 static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t cluster_offset;
     int index_in_cluster, ret;
     int64_t status = 0;
@@ -1108,6 +1344,7 @@ static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
         !s->cipher) {
         index_in_cluster = sector_num & (s->cluster_sectors - 1);
         cluster_offset |= (index_in_cluster << BDRV_SECTOR_BITS);
+        *file = bs->file->bs;
         status |= BDRV_BLOCK_OFFSET_VALID | cluster_offset;
     }
     if (ret == QCOW2_CLUSTER_ZERO) {
@@ -1138,7 +1375,7 @@ int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
 static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
                           int remaining_sectors, QEMUIOVector *qiov)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int index_in_cluster, n1;
     int ret;
     int cur_nr_sectors; /* number of sectors in current iteration */
@@ -1175,9 +1412,9 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
         switch (ret) {
         case QCOW2_CLUSTER_UNALLOCATED:
 
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
-                n1 = qcow2_backing_read1(bs->backing_hd, &hd_qiov,
+                n1 = qcow2_backing_read1(bs->backing->bs, &hd_qiov,
                     sector_num, cur_nr_sectors);
                 if (n1 > 0) {
                     QEMUIOVector local_qiov;
@@ -1188,7 +1425,7 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
 
                     BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
                     qemu_co_mutex_unlock(&s->lock);
-                    ret = bdrv_co_readv(bs->backing_hd, sector_num,
+                    ret = bdrv_co_readv(bs->backing->bs, sector_num,
                                         n1, &local_qiov);
                     qemu_co_mutex_lock(&s->lock);
 
@@ -1235,8 +1472,9 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
                  */
                 if (!cluster_data) {
                     cluster_data =
-                        qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
-                                                      * s->cluster_size);
+                        qemu_try_blockalign(bs->file->bs,
+                                            QCOW_MAX_CRYPT_CLUSTERS
+                                            * s->cluster_size);
                     if (cluster_data == NULL) {
                         ret = -ENOMEM;
                         goto fail;
@@ -1252,7 +1490,7 @@ static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
 
             BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
             qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
+            ret = bdrv_co_readv(bs->file->bs,
                                 (cluster_offset >> 9) + index_in_cluster,
                                 cur_nr_sectors, &hd_qiov);
             qemu_co_mutex_lock(&s->lock);
@@ -1300,7 +1538,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
                            int remaining_sectors,
                            QEMUIOVector *qiov)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int index_in_cluster;
     int ret;
     int cur_nr_sectors; /* number of sectors in current iteration */
@@ -1349,7 +1587,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
             Error *err = NULL;
             assert(s->cipher);
             if (!cluster_data) {
-                cluster_data = qemu_try_blockalign(bs->file,
+                cluster_data = qemu_try_blockalign(bs->file->bs,
                                                    QCOW_MAX_CRYPT_CLUSTERS
                                                    * s->cluster_size);
                 if (cluster_data == NULL) {
@@ -1386,7 +1624,7 @@ static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         trace_qcow2_writev_data(qemu_coroutine_self(),
                                 (cluster_offset >> 9) + index_in_cluster);
-        ret = bdrv_co_writev(bs->file,
+        ret = bdrv_co_writev(bs->file->bs,
                              (cluster_offset >> 9) + index_in_cluster,
                              cur_nr_sectors, &hd_qiov);
         qemu_co_mutex_lock(&s->lock);
@@ -1444,33 +1682,44 @@ fail:
     return ret;
 }
 
+static int qcow2_inactivate(BlockDriverState *bs)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int ret, result = 0;
+
+    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    if (ret) {
+        result = ret;
+        error_report("Failed to flush the L2 table cache: %s",
+                     strerror(-ret));
+    }
+
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret) {
+        result = ret;
+        error_report("Failed to flush the refcount block cache: %s",
+                     strerror(-ret));
+    }
+
+    if (result == 0) {
+        qcow2_mark_clean(bs);
+    }
+
+    return result;
+}
+
 static void qcow2_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
 
-    if (!(bs->open_flags & BDRV_O_INCOMING)) {
-        int ret1, ret2;
-
-        ret1 = qcow2_cache_flush(bs, s->l2_table_cache);
-        ret2 = qcow2_cache_flush(bs, s->refcount_block_cache);
-
-        if (ret1) {
-            error_report("Failed to flush the L2 table cache: %s",
-                         strerror(-ret1));
-        }
-        if (ret2) {
-            error_report("Failed to flush the refcount block cache: %s",
-                         strerror(-ret2));
-        }
-
-        if (!ret1 && !ret2) {
-            qcow2_mark_clean(bs);
-        }
+    if (!(s->flags & BDRV_O_INACTIVE)) {
+        qcow2_inactivate(bs);
     }
 
+    cache_clean_timer_del(bs);
     qcow2_cache_destroy(bs, s->l2_table_cache);
     qcow2_cache_destroy(bs, s->refcount_block_cache);
 
@@ -1491,7 +1740,7 @@ static void qcow2_close(BlockDriverState *bs)
 
 static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int flags = s->flags;
     QCryptoCipher *cipher = NULL;
     QDict *options;
@@ -1508,24 +1757,27 @@ static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
 
     qcow2_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
+    bdrv_invalidate_cache(bs->file->bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
+        bs->drv = NULL;
         return;
     }
 
-    memset(s, 0, sizeof(BDRVQcowState));
+    memset(s, 0, sizeof(BDRVQcow2State));
     options = qdict_clone_shallow(bs->options);
 
+    flags &= ~BDRV_O_INACTIVE;
     ret = qcow2_open(bs, options, flags, &local_err);
     QDECREF(options);
     if (local_err) {
-        error_setg(errp, "Could not reopen qcow2 layer: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_propagate(errp, local_err);
+        error_prepend(errp, "Could not reopen qcow2 layer: ");
+        bs->drv = NULL;
         return;
     } else if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not reopen qcow2 layer");
+        bs->drv = NULL;
         return;
     }
 
@@ -1561,7 +1813,7 @@ static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
  */
 int qcow2_update_header(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowHeader *header;
     char *buf;
     size_t buflen = s->cluster_size;
@@ -1653,31 +1905,33 @@ int qcow2_update_header(BlockDriverState *bs)
     }
 
     /* Feature table */
-    Qcow2Feature features[] = {
-        {
-            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
-            .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
-            .name = "dirty bit",
-        },
-        {
-            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
-            .bit  = QCOW2_INCOMPAT_CORRUPT_BITNR,
-            .name = "corrupt bit",
-        },
-        {
-            .type = QCOW2_FEAT_TYPE_COMPATIBLE,
-            .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
-            .name = "lazy refcounts",
-        },
-    };
+    if (s->qcow_version >= 3) {
+        Qcow2Feature features[] = {
+            {
+                .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
+                .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
+                .name = "dirty bit",
+            },
+            {
+                .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
+                .bit  = QCOW2_INCOMPAT_CORRUPT_BITNR,
+                .name = "corrupt bit",
+            },
+            {
+                .type = QCOW2_FEAT_TYPE_COMPATIBLE,
+                .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
+                .name = "lazy refcounts",
+            },
+        };
 
-    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
-                         features, sizeof(features), buflen);
-    if (ret < 0) {
-        goto fail;
+        ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
+                             features, sizeof(features), buflen);
+        if (ret < 0) {
+            goto fail;
+        }
+        buf += ret;
+        buflen -= ret;
     }
-    buf += ret;
-    buflen -= ret;
 
     /* Keep unknown header extensions */
     QLIST_FOREACH(uext, &s->unknown_header_ext, next) {
@@ -1716,7 +1970,7 @@ int qcow2_update_header(BlockDriverState *bs)
     }
 
     /* Write the new header */
-    ret = bdrv_pwrite(bs->file, 0, header, s->cluster_size);
+    ret = bdrv_pwrite(bs->file->bs, 0, header, s->cluster_size);
     if (ret < 0) {
         goto fail;
     }
@@ -1730,7 +1984,11 @@ fail:
 static int qcow2_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
+
+    if (backing_file && strlen(backing_file) > 1023) {
+        return -EINVAL;
+    }
 
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
     pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
@@ -1796,7 +2054,8 @@ static int preallocate(BlockDriverState *bs)
     if (host_offset != 0) {
         uint8_t buf[BDRV_SECTOR_SIZE];
         memset(buf, 0, BDRV_SECTOR_SIZE);
-        ret = bdrv_write(bs->file, (host_offset >> BDRV_SECTOR_BITS) + num - 1,
+        ret = bdrv_write(bs->file->bs,
+                         (host_offset >> BDRV_SECTOR_BITS) + num - 1,
                          buf, 1);
         if (ret < 0) {
             return ret;
@@ -1812,8 +2071,10 @@ static int qcow2_create2(const char *filename, int64_t total_size,
                          QemuOpts *opts, int version, int refcount_order,
                          Error **errp)
 {
-    /* Calculate cluster_bits */
     int cluster_bits;
+    QDict *options;
+
+    /* Calculate cluster_bits */
     cluster_bits = ctz32(cluster_size);
     if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
         (1 << cluster_bits) != cluster_size)
@@ -1835,7 +2096,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
      * 2 GB for 64k clusters, and we don't want to have a 2 GB initial file
      * size for any qcow2 image.
      */
-    BlockDriverState* bs;
+    BlockBackend *blk;
     QCowHeader *header;
     uint64_t* refcount_table;
     Error *local_err = NULL;
@@ -1910,14 +2171,15 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         return ret;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
-        return ret;
+        return -EIO;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* Write the header */
     QEMU_BUILD_BUG_ON((1 << MIN_CLUSTER_BITS) < sizeof(*header));
     header = g_malloc0(cluster_size);
@@ -1945,7 +2207,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
             cpu_to_be64(QCOW2_COMPAT_LAZY_REFCOUNTS);
     }
 
-    ret = bdrv_pwrite(bs, 0, header, cluster_size);
+    ret = blk_pwrite(blk, 0, header, cluster_size);
     g_free(header);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not write qcow2 header");
@@ -1955,7 +2217,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     /* Write a refcount table with one refcount block */
     refcount_table = g_malloc0(2 * cluster_size);
     refcount_table[0] = cpu_to_be64(2 * cluster_size);
-    ret = bdrv_pwrite(bs, cluster_size, refcount_table, 2 * cluster_size);
+    ret = blk_pwrite(blk, cluster_size, refcount_table, 2 * cluster_size);
     g_free(refcount_table);
 
     if (ret < 0) {
@@ -1963,23 +2225,25 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         goto out;
     }
 
-    bdrv_unref(bs);
-    bs = NULL;
+    blk_unref(blk);
+    blk = NULL;
 
     /*
      * And now open the image and make it consistent first (i.e. increase the
      * refcount of the cluster that is occupied by the header and the refcount
      * table)
      */
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
-                    &bdrv_qcow2, &local_err);
-    if (ret < 0) {
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    blk = blk_new_open(filename, NULL, options,
+                       BDRV_O_RDWR | BDRV_O_NO_FLUSH, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
-    ret = qcow2_alloc_clusters(bs, 3 * cluster_size);
+    ret = qcow2_alloc_clusters(blk_bs(blk), 3 * cluster_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not allocate clusters for qcow2 "
                          "header and refcount table");
@@ -1990,8 +2254,15 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         abort();
     }
 
+    /* Create a full header (including things like feature table) */
+    ret = qcow2_update_header(blk_bs(blk));
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not update qcow2 header");
+        goto out;
+    }
+
     /* Okay, now that we have a valid image, let's give it the right size */
-    ret = bdrv_truncate(bs, total_size);
+    ret = blk_truncate(blk, total_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not resize image");
         goto out;
@@ -1999,7 +2270,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
 
     /* Want a backing file? There you go.*/
     if (backing_file) {
-        ret = bdrv_change_backing_file(bs, backing_file, backing_format);
+        ret = bdrv_change_backing_file(blk_bs(blk), backing_file, backing_format);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not assign backing file '%s' "
                              "with format '%s'", backing_file, backing_format);
@@ -2009,9 +2280,9 @@ static int qcow2_create2(const char *filename, int64_t total_size,
 
     /* And if we're supposed to preallocate metadata, do that now */
     if (prealloc != PREALLOC_MODE_OFF) {
-        BDRVQcowState *s = bs->opaque;
+        BDRVQcow2State *s = blk_bs(blk)->opaque;
         qemu_co_mutex_lock(&s->lock);
-        ret = preallocate(bs);
+        ret = preallocate(blk_bs(blk));
         qemu_co_mutex_unlock(&s->lock);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not preallocate metadata");
@@ -2019,22 +2290,24 @@ static int qcow2_create2(const char *filename, int64_t total_size,
         }
     }
 
-    bdrv_unref(bs);
-    bs = NULL;
+    blk_unref(blk);
+    blk = NULL;
 
     /* Reopen the image without BDRV_O_NO_FLUSH to flush it before returning */
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_BACKING,
-                    &bdrv_qcow2, &local_err);
-    if (local_err) {
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    blk = blk_new_open(filename, NULL, options,
+                       BDRV_O_RDWR | BDRV_O_NO_BACKING, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
     ret = 0;
 out:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     return ret;
 }
@@ -2066,7 +2339,7 @@ static int qcow2_create(const char *filename, QemuOpts *opts, Error **errp)
                                          DEFAULT_CLUSTER_SIZE);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
-                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               PREALLOC_MODE__MAX, PREALLOC_MODE_OFF,
                                &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -2142,7 +2415,7 @@ static coroutine_fn int qcow2_co_write_zeroes(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     /* Emulate misaligned zero writes */
     if (sector_num % s->cluster_sectors || nb_sectors % s->cluster_sectors) {
@@ -2162,7 +2435,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     qemu_co_mutex_lock(&s->lock);
     ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
@@ -2173,7 +2446,7 @@ static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
 
 static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t new_l1_size;
     int ret;
 
@@ -2202,7 +2475,7 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 
     /* write updated header.size */
     offset = cpu_to_be64(offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, size),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, size),
                            &offset, sizeof(uint64_t));
     if (ret < 0) {
         return ret;
@@ -2217,7 +2490,7 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
                                   const uint8_t *buf, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     z_stream strm;
     int ret, out_len;
     uint8_t *out_buf;
@@ -2226,8 +2499,8 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
     if (nb_sectors == 0) {
         /* align end of file to a sector boundary to ease reading with
            sector based I/Os */
-        cluster_offset = bdrv_getlength(bs->file);
-        return bdrv_truncate(bs->file, cluster_offset);
+        cluster_offset = bdrv_getlength(bs->file->bs);
+        return bdrv_truncate(bs->file->bs, cluster_offset);
     }
 
     if (nb_sectors != s->cluster_sectors) {
@@ -2294,7 +2567,7 @@ static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
         }
 
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_COMPRESSED);
-        ret = bdrv_pwrite(bs->file, cluster_offset, out_buf, out_len);
+        ret = bdrv_pwrite(bs->file->bs, cluster_offset, out_buf, out_len);
         if (ret < 0) {
             goto fail;
         }
@@ -2308,7 +2581,7 @@ fail:
 
 static int make_completely_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, l1_clusters;
     int64_t offset;
     uint64_t *new_reftable = NULL;
@@ -2343,7 +2616,7 @@ static int make_completely_empty(BlockDriverState *bs)
     /* After this call, neither the in-memory nor the on-disk refcount
      * information accurately describe the actual references */
 
-    ret = bdrv_write_zeroes(bs->file, s->l1_table_offset / BDRV_SECTOR_SIZE,
+    ret = bdrv_write_zeroes(bs->file->bs, s->l1_table_offset / BDRV_SECTOR_SIZE,
                             l1_clusters * s->cluster_sectors, 0);
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2357,7 +2630,7 @@ static int make_completely_empty(BlockDriverState *bs)
      * overwrite parts of the existing refcount and L1 table, which is not
      * an issue because the dirty flag is set, complete data loss is in fact
      * desired and partial data loss is consequently fine as well */
-    ret = bdrv_write_zeroes(bs->file, s->cluster_size / BDRV_SECTOR_SIZE,
+    ret = bdrv_write_zeroes(bs->file->bs, s->cluster_size / BDRV_SECTOR_SIZE,
                             (2 + l1_clusters) * s->cluster_size /
                             BDRV_SECTOR_SIZE, 0);
     /* This call (even if it failed overall) may have overwritten on-disk
@@ -2377,7 +2650,7 @@ static int make_completely_empty(BlockDriverState *bs)
     cpu_to_be64w(&l1_ofs_rt_ofs_cls.l1_offset, 3 * s->cluster_size);
     cpu_to_be64w(&l1_ofs_rt_ofs_cls.reftable_offset, s->cluster_size);
     cpu_to_be32w(&l1_ofs_rt_ofs_cls.reftable_clusters, 1);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_table_offset),
+    ret = bdrv_pwrite_sync(bs->file->bs, offsetof(QCowHeader, l1_table_offset),
                            &l1_ofs_rt_ofs_cls, sizeof(l1_ofs_rt_ofs_cls));
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2408,7 +2681,7 @@ static int make_completely_empty(BlockDriverState *bs)
 
     /* Enter the first refblock into the reftable */
     rt_entry = cpu_to_be64(2 * s->cluster_size);
-    ret = bdrv_pwrite_sync(bs->file, s->cluster_size,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->cluster_size,
                            &rt_entry, sizeof(rt_entry));
     if (ret < 0) {
         goto fail_broken_refcounts;
@@ -2433,7 +2706,7 @@ static int make_completely_empty(BlockDriverState *bs)
         goto fail;
     }
 
-    ret = bdrv_truncate(bs->file, (3 + l1_clusters) * s->cluster_size);
+    ret = bdrv_truncate(bs->file->bs, (3 + l1_clusters) * s->cluster_size);
     if (ret < 0) {
         goto fail;
     }
@@ -2456,7 +2729,7 @@ fail:
 
 static int qcow2_make_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start_sector;
     int sector_step = INT_MAX / BDRV_SECTOR_SIZE;
     int l1_clusters, ret = 0;
@@ -2497,7 +2770,7 @@ static int qcow2_make_empty(BlockDriverState *bs)
 
 static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     qemu_co_mutex_lock(&s->lock);
@@ -2521,7 +2794,7 @@ static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 
 static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bdi->unallocated_blocks_are_zero = true;
     bdi->can_write_zeroes_with_unmap = (s->qcow_version >= 3);
     bdi->cluster_size = s->cluster_size;
@@ -2531,22 +2804,20 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 
 static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
 
     *spec_info = (ImageInfoSpecific){
-        .kind  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
-        {
-            .qcow2 = g_new(ImageInfoSpecificQCow2, 1),
-        },
+        .type  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
+        .u.qcow2.data = g_new(ImageInfoSpecificQCow2, 1),
     };
     if (s->qcow_version == 2) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+        *spec_info->u.qcow2.data = (ImageInfoSpecificQCow2){
             .compat             = g_strdup("0.10"),
             .refcount_bits      = s->refcount_bits,
         };
     } else if (s->qcow_version == 3) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+        *spec_info->u.qcow2.data = (ImageInfoSpecificQCow2){
             .compat             = g_strdup("1.1"),
             .lazy_refcounts     = s->compatible_features &
                                   QCOW2_COMPAT_LAZY_REFCOUNTS,
@@ -2556,6 +2827,10 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
             .has_corrupt        = true,
             .refcount_bits      = s->refcount_bits,
         };
+    } else {
+        /* if this assertion fails, this probably means a new version was
+         * added without having it covered here */
+        assert(false);
     }
 
     return spec_info;
@@ -2564,11 +2839,11 @@ static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
 #if 0
 static void dump_refcounts(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t nb_clusters, k, k1, size;
     int refcount;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     nb_clusters = size_to_clusters(s, size);
     for(k = 0; k < nb_clusters;) {
         k1 = k;
@@ -2585,7 +2860,7 @@ static void dump_refcounts(BlockDriverState *bs)
 static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
                               int64_t pos)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t total_sectors = bs->total_sectors;
     bool zero_beyond_eof = bs->zero_beyond_eof;
     int ret;
@@ -2606,7 +2881,7 @@ static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
 static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
                               int64_t pos, int size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bool zero_beyond_eof = bs->zero_beyond_eof;
     int ret;
 
@@ -2623,9 +2898,9 @@ static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
  * have to be removed.
  */
 static int qcow2_downgrade(BlockDriverState *bs, int target_version,
-                           BlockDriverAmendStatusCB *status_cb)
+                           BlockDriverAmendStatusCB *status_cb, void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int current_version = s->qcow_version;
     int ret;
 
@@ -2638,13 +2913,7 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
     }
 
     if (s->refcount_order != 4) {
-        /* we would have to convert the image to a refcount_order == 4 image
-         * here; however, since qemu (at the time of writing this) does not
-         * support anything different than 4 anyway, there is no point in doing
-         * so right now; however, we should error out (if qemu supports this in
-         * the future and this code has not been adapted) */
-        error_report("qcow2_downgrade: Image refcount orders other than 4 are "
-                     "currently not supported.");
+        error_report("compat=0.10 requires refcount_bits=16");
         return -ENOTSUP;
     }
 
@@ -2672,7 +2941,7 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
     /* clearing autoclear features is trivial */
     s->autoclear_features = 0;
 
-    ret = qcow2_expand_zero_clusters(bs, status_cb);
+    ret = qcow2_expand_zero_clusters(bs, status_cb, cb_opaque);
     if (ret < 0) {
         return ret;
     }
@@ -2686,10 +2955,81 @@ static int qcow2_downgrade(BlockDriverState *bs, int target_version,
     return 0;
 }
 
+typedef enum Qcow2AmendOperation {
+    /* This is the value Qcow2AmendHelperCBInfo::last_operation will be
+     * statically initialized to so that the helper CB can discern the first
+     * invocation from an operation change */
+    QCOW2_NO_OPERATION = 0,
+
+    QCOW2_CHANGING_REFCOUNT_ORDER,
+    QCOW2_DOWNGRADING,
+} Qcow2AmendOperation;
+
+typedef struct Qcow2AmendHelperCBInfo {
+    /* The code coordinating the amend operations should only modify
+     * these four fields; the rest will be managed by the CB */
+    BlockDriverAmendStatusCB *original_status_cb;
+    void *original_cb_opaque;
+
+    Qcow2AmendOperation current_operation;
+
+    /* Total number of operations to perform (only set once) */
+    int total_operations;
+
+    /* The following fields are managed by the CB */
+
+    /* Number of operations completed */
+    int operations_completed;
+
+    /* Cumulative offset of all completed operations */
+    int64_t offset_completed;
+
+    Qcow2AmendOperation last_operation;
+    int64_t last_work_size;
+} Qcow2AmendHelperCBInfo;
+
+static void qcow2_amend_helper_cb(BlockDriverState *bs,
+                                  int64_t operation_offset,
+                                  int64_t operation_work_size, void *opaque)
+{
+    Qcow2AmendHelperCBInfo *info = opaque;
+    int64_t current_work_size;
+    int64_t projected_work_size;
+
+    if (info->current_operation != info->last_operation) {
+        if (info->last_operation != QCOW2_NO_OPERATION) {
+            info->offset_completed += info->last_work_size;
+            info->operations_completed++;
+        }
+
+        info->last_operation = info->current_operation;
+    }
+
+    assert(info->total_operations > 0);
+    assert(info->operations_completed < info->total_operations);
+
+    info->last_work_size = operation_work_size;
+
+    current_work_size = info->offset_completed + operation_work_size;
+
+    /* current_work_size is the total work size for (operations_completed + 1)
+     * operations (which includes this one), so multiply it by the number of
+     * operations not covered and divide it by the number of operations
+     * covered to get a projection for the operations not covered */
+    projected_work_size = current_work_size * (info->total_operations -
+                                               info->operations_completed - 1)
+                                            / (info->operations_completed + 1);
+
+    info->original_status_cb(bs, info->offset_completed + operation_offset,
+                             current_work_size + projected_work_size,
+                             info->original_cb_opaque);
+}
+
 static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
-                               BlockDriverAmendStatusCB *status_cb)
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int old_version = s->qcow_version, new_version = old_version;
     uint64_t new_size = 0;
     const char *backing_file = NULL, *backing_format = NULL;
@@ -2697,8 +3037,10 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
     const char *compat = NULL;
     uint64_t cluster_size = s->cluster_size;
     bool encrypt;
+    int refcount_bits = s->refcount_bits;
     int ret;
     QemuOptDesc *desc = opts->list->desc;
+    Qcow2AmendHelperCBInfo helper_cb_info;
 
     while (desc && desc->name) {
         if (!qemu_opt_find(opts, desc->name)) {
@@ -2716,11 +3058,11 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
             } else if (!strcmp(compat, "1.1")) {
                 new_version = 3;
             } else {
-                fprintf(stderr, "Unknown compatibility level %s.\n", compat);
+                error_report("Unknown compatibility level %s", compat);
                 return -EINVAL;
             }
         } else if (!strcmp(desc->name, BLOCK_OPT_PREALLOC)) {
-            fprintf(stderr, "Cannot change preallocation mode.\n");
+            error_report("Cannot change preallocation mode");
             return -ENOTSUP;
         } else if (!strcmp(desc->name, BLOCK_OPT_SIZE)) {
             new_size = qemu_opt_get_size(opts, BLOCK_OPT_SIZE, 0);
@@ -2733,47 +3075,74 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
                                         !!s->cipher);
 
             if (encrypt != !!s->cipher) {
-                fprintf(stderr, "Changing the encryption flag is not "
-                        "supported.\n");
+                error_report("Changing the encryption flag is not supported");
                 return -ENOTSUP;
             }
         } else if (!strcmp(desc->name, BLOCK_OPT_CLUSTER_SIZE)) {
             cluster_size = qemu_opt_get_size(opts, BLOCK_OPT_CLUSTER_SIZE,
                                              cluster_size);
             if (cluster_size != s->cluster_size) {
-                fprintf(stderr, "Changing the cluster size is not "
-                        "supported.\n");
+                error_report("Changing the cluster size is not supported");
                 return -ENOTSUP;
             }
         } else if (!strcmp(desc->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
             lazy_refcounts = qemu_opt_get_bool(opts, BLOCK_OPT_LAZY_REFCOUNTS,
                                                lazy_refcounts);
         } else if (!strcmp(desc->name, BLOCK_OPT_REFCOUNT_BITS)) {
-            error_report("Cannot change refcount entry width");
-            return -ENOTSUP;
+            refcount_bits = qemu_opt_get_number(opts, BLOCK_OPT_REFCOUNT_BITS,
+                                                refcount_bits);
+
+            if (refcount_bits <= 0 || refcount_bits > 64 ||
+                !is_power_of_2(refcount_bits))
+            {
+                error_report("Refcount width must be a power of two and may "
+                             "not exceed 64 bits");
+                return -EINVAL;
+            }
         } else {
-            /* if this assertion fails, this probably means a new option was
+            /* if this point is reached, this probably means a new option was
              * added without having it covered here */
-            assert(false);
+            abort();
         }
 
         desc++;
     }
 
-    if (new_version != old_version) {
-        if (new_version > old_version) {
-            /* Upgrade */
-            s->qcow_version = new_version;
-            ret = qcow2_update_header(bs);
-            if (ret < 0) {
-                s->qcow_version = old_version;
-                return ret;
-            }
-        } else {
-            ret = qcow2_downgrade(bs, new_version, status_cb);
-            if (ret < 0) {
-                return ret;
-            }
+    helper_cb_info = (Qcow2AmendHelperCBInfo){
+        .original_status_cb = status_cb,
+        .original_cb_opaque = cb_opaque,
+        .total_operations = (new_version < old_version)
+                          + (s->refcount_bits != refcount_bits)
+    };
+
+    /* Upgrade first (some features may require compat=1.1) */
+    if (new_version > old_version) {
+        s->qcow_version = new_version;
+        ret = qcow2_update_header(bs);
+        if (ret < 0) {
+            s->qcow_version = old_version;
+            return ret;
+        }
+    }
+
+    if (s->refcount_bits != refcount_bits) {
+        int refcount_order = ctz32(refcount_bits);
+        Error *local_error = NULL;
+
+        if (new_version < 3 && refcount_bits != 16) {
+            error_report("Different refcount widths than 16 bits require "
+                         "compatibility level 1.1 or above (use compat=1.1 or "
+                         "greater)");
+            return -EINVAL;
+        }
+
+        helper_cb_info.current_operation = QCOW2_CHANGING_REFCOUNT_ORDER;
+        ret = qcow2_change_refcount_order(bs, refcount_order,
+                                          &qcow2_amend_helper_cb,
+                                          &helper_cb_info, &local_error);
+        if (ret < 0) {
+            error_report_err(local_error);
+            return ret;
         }
     }
 
@@ -2788,9 +3157,9 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
 
     if (s->use_lazy_refcounts != lazy_refcounts) {
         if (lazy_refcounts) {
-            if (s->qcow_version < 3) {
-                fprintf(stderr, "Lazy refcounts only supported with compatibility "
-                        "level 1.1 and above (use compat=1.1 or greater)\n");
+            if (new_version < 3) {
+                error_report("Lazy refcounts only supported with compatibility "
+                             "level 1.1 and above (use compat=1.1 or greater)");
                 return -EINVAL;
             }
             s->compatible_features |= QCOW2_COMPAT_LAZY_REFCOUNTS;
@@ -2824,6 +3193,16 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
         }
     }
 
+    /* Downgrade last (so unsupported features can be removed before) */
+    if (new_version < old_version) {
+        helper_cb_info.current_operation = QCOW2_DOWNGRADING;
+        ret = qcow2_downgrade(bs, new_version, &qcow2_amend_helper_cb,
+                              &helper_cb_info);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
     return 0;
 }
 
@@ -2836,7 +3215,7 @@ static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
 void qcow2_signal_corruption(BlockDriverState *bs, bool fatal, int64_t offset,
                              int64_t size, const char *message_format, ...)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     const char *node_name;
     char *message;
     va_list ap;
@@ -2937,11 +3316,14 @@ static QemuOptsList qcow2_create_opts = {
 
 BlockDriver bdrv_qcow2 = {
     .format_name        = "qcow2",
-    .instance_size      = sizeof(BDRVQcowState),
+    .instance_size      = sizeof(BDRVQcow2State),
     .bdrv_probe         = qcow2_probe,
     .bdrv_open          = qcow2_open,
     .bdrv_close         = qcow2_close,
     .bdrv_reopen_prepare  = qcow2_reopen_prepare,
+    .bdrv_reopen_commit   = qcow2_reopen_commit,
+    .bdrv_reopen_abort    = qcow2_reopen_abort,
+    .bdrv_join_options    = qcow2_join_options,
     .bdrv_create        = qcow2_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
     .bdrv_co_get_block_status = qcow2_co_get_block_status,
@@ -2973,10 +3355,14 @@ BlockDriver bdrv_qcow2 = {
 
     .bdrv_refresh_limits        = qcow2_refresh_limits,
     .bdrv_invalidate_cache      = qcow2_invalidate_cache,
+    .bdrv_inactivate            = qcow2_inactivate,
 
     .create_opts         = &qcow2_create_opts,
     .bdrv_check          = qcow2_check,
     .bdrv_amend_options  = qcow2_amend_options,
+
+    .bdrv_detach_aio_context  = qcow2_detach_aio_context,
+    .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 };
 
 static void bdrv_qcow2_init(void)
diff --git a/qemu/block/qcow2.h b/qemu/block/qcow2.h
index 72e132838..a063a3c1a 100644
--- a/qemu/block/qcow2.h
+++ b/qemu/block/qcow2.h
@@ -26,7 +26,7 @@
 #define BLOCK_QCOW2_H
 
 #include "crypto/cipher.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 //#define DEBUG_ALLOC
 //#define DEBUG_ALLOC2
@@ -96,6 +96,7 @@
 #define QCOW2_OPT_CACHE_SIZE "cache-size"
 #define QCOW2_OPT_L2_CACHE_SIZE "l2-cache-size"
 #define QCOW2_OPT_REFCOUNT_CACHE_SIZE "refcount-cache-size"
+#define QCOW2_OPT_CACHE_CLEAN_INTERVAL "cache-clean-interval"
 
 typedef struct QCowHeader {
     uint32_t magic;
@@ -221,7 +222,7 @@ typedef uint64_t Qcow2GetRefcountFunc(const void *refcount_array,
 typedef void Qcow2SetRefcountFunc(void *refcount_array,
                                   uint64_t index, uint64_t value);
 
-typedef struct BDRVQcowState {
+typedef struct BDRVQcow2State {
     int cluster_bits;
     int cluster_size;
     int cluster_sectors;
@@ -239,6 +240,8 @@ typedef struct BDRVQcowState {
 
     Qcow2Cache* l2_table_cache;
     Qcow2Cache* refcount_block_cache;
+    QEMUTimer *cache_clean_timer;
+    unsigned cache_clean_interval;
 
     uint8_t *cluster_cache;
     uint8_t *cluster_data;
@@ -290,9 +293,7 @@ typedef struct BDRVQcowState {
      * override) */
     char *image_backing_file;
     char *image_backing_format;
-} BDRVQcowState;
-
-struct QCowAIOCB;
+} BDRVQcow2State;
 
 typedef struct Qcow2COWRegion {
     /**
@@ -402,28 +403,28 @@ typedef enum QCow2MetadataOverlap {
 
 #define REFT_OFFSET_MASK 0xfffffffffffffe00ULL
 
-static inline int64_t start_of_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t start_of_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & ~(s->cluster_size - 1);
 }
 
-static inline int64_t offset_into_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t offset_into_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & (s->cluster_size - 1);
 }
 
-static inline int size_to_clusters(BDRVQcowState *s, int64_t size)
+static inline uint64_t size_to_clusters(BDRVQcow2State *s, uint64_t size)
 {
     return (size + (s->cluster_size - 1)) >> s->cluster_bits;
 }
 
-static inline int64_t size_to_l1(BDRVQcowState *s, int64_t size)
+static inline int64_t size_to_l1(BDRVQcow2State *s, int64_t size)
 {
     int shift = s->cluster_bits + s->l2_bits;
     return (size + (1ULL << shift) - 1) >> shift;
 }
 
-static inline int offset_to_l2_index(BDRVQcowState *s, int64_t offset)
+static inline int offset_to_l2_index(BDRVQcow2State *s, int64_t offset)
 {
     return (offset >> s->cluster_bits) & (s->l2_size - 1);
 }
@@ -434,12 +435,12 @@ static inline int64_t align_offset(int64_t offset, int n)
     return offset;
 }
 
-static inline int64_t qcow2_vm_state_offset(BDRVQcowState *s)
+static inline int64_t qcow2_vm_state_offset(BDRVQcow2State *s)
 {
     return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
 }
 
-static inline uint64_t qcow2_max_refcount_clusters(BDRVQcowState *s)
+static inline uint64_t qcow2_max_refcount_clusters(BDRVQcow2State *s)
 {
     return QCOW_MAX_REFTABLE_SIZE >> s->cluster_bits;
 }
@@ -458,7 +459,7 @@ static inline int qcow2_get_cluster_type(uint64_t l2_entry)
 }
 
 /* Check whether refcounts are eager or lazy */
-static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
+static inline bool qcow2_need_accurate_refcounts(BDRVQcow2State *s)
 {
     return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY);
 }
@@ -506,8 +507,8 @@ int qcow2_update_cluster_refcount(BlockDriverState *bs, int64_t cluster_index,
                                   enum qcow2_discard_type type);
 
 int64_t qcow2_alloc_clusters(BlockDriverState *bs, uint64_t size);
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters);
+int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+                                int64_t nb_clusters);
 int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size);
 void qcow2_free_clusters(BlockDriverState *bs,
                           int64_t offset, int64_t size,
@@ -528,13 +529,17 @@ int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
 int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
                                   int64_t size);
 
+int qcow2_change_refcount_order(BlockDriverState *bs, int refcount_order,
+                                BlockDriverAmendStatusCB *status_cb,
+                                void *cb_opaque, Error **errp);
+
 /* qcow2-cluster.c functions */
 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         bool exact_size);
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index);
 void qcow2_l2_cache_reset(BlockDriverState *bs);
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
-int qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
                           uint8_t *out_buf, const uint8_t *in_buf,
                           int nb_sectors, bool enc, Error **errp);
 
@@ -552,7 +557,8 @@ int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
 
 int qcow2_expand_zero_clusters(BlockDriverState *bs,
-                               BlockDriverAmendStatusCB *status_cb);
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque);
 
 /* qcow2-snapshot.c functions */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
@@ -581,6 +587,7 @@ int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
     Qcow2Cache *dependency);
 void qcow2_cache_depends_on_flush(Qcow2Cache *c);
 
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c);
 int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c);
 
 int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
diff --git a/qemu/block/qed-check.c b/qemu/block/qed-check.c
index 36ecd290d..622f30897 100644
--- a/qemu/block/qed-check.c
+++ b/qemu/block/qed-check.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 typedef struct {
diff --git a/qemu/block/qed-cluster.c b/qemu/block/qed-cluster.c
index f64b2af8f..c24e75616 100644
--- a/qemu/block/qed-cluster.c
+++ b/qemu/block/qed-cluster.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 /**
diff --git a/qemu/block/qed-gencb.c b/qemu/block/qed-gencb.c
index b817a8bf5..faf8ecc84 100644
--- a/qemu/block/qed-gencb.c
+++ b/qemu/block/qed-gencb.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 void *gencb_alloc(size_t len, BlockCompletionFunc *cb, void *opaque)
diff --git a/qemu/block/qed-l2-cache.c b/qemu/block/qed-l2-cache.c
index e9b2aae44..5cba79465 100644
--- a/qemu/block/qed-l2-cache.c
+++ b/qemu/block/qed-l2-cache.c
@@ -50,6 +50,7 @@
  * table will be deleted in favor of the existing cache entry.
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "qed.h"
 
diff --git a/qemu/block/qed-table.c b/qemu/block/qed-table.c
index 513aa872c..802945f5e 100644
--- a/qemu/block/qed-table.c
+++ b/qemu/block/qed-table.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "qed.h"
@@ -63,7 +64,7 @@ static void qed_read_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
     read_table_cb->iov.iov_len = s->header.cluster_size * s->header.table_size,
 
     qemu_iovec_init_external(qiov, &read_table_cb->iov, 1);
-    bdrv_aio_readv(s->bs->file, offset / BDRV_SECTOR_SIZE, qiov,
+    bdrv_aio_readv(s->bs->file->bs, offset / BDRV_SECTOR_SIZE, qiov,
                    qiov->size / BDRV_SECTOR_SIZE,
                    qed_read_table_cb, read_table_cb);
 }
@@ -152,7 +153,7 @@ static void qed_write_table(BDRVQEDState *s, uint64_t offset, QEDTable *table,
     /* Adjust for offset into table */
     offset += start * sizeof(uint64_t);
 
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, offset / BDRV_SECTOR_SIZE,
                     &write_table_cb->qiov,
                     write_table_cb->qiov.size / BDRV_SECTOR_SIZE,
                     qed_write_table_cb, write_table_cb);
diff --git a/qemu/block/qed.c b/qemu/block/qed.c
index 954ed007c..0af52741d 100644
--- a/qemu/block/qed.c
+++ b/qemu/block/qed.c
@@ -12,11 +12,14 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "trace.h"
 #include "qed.h"
 #include "qapi/qmp/qerror.h"
 #include "migration/migration.h"
+#include "sysemu/block-backend.h"
 
 static const AIOCBInfo qed_aiocb_info = {
     .aiocb_size         = sizeof(QEDAIOCB),
@@ -82,7 +85,7 @@ int qed_write_header_sync(BDRVQEDState *s)
     int ret;
 
     qed_header_cpu_to_le(&s->header, &le);
-    ret = bdrv_pwrite(s->bs->file, 0, &le, sizeof(le));
+    ret = bdrv_pwrite(s->bs->file->bs, 0, &le, sizeof(le));
     if (ret != sizeof(le)) {
         return ret;
     }
@@ -119,7 +122,7 @@ static void qed_write_header_read_cb(void *opaque, int ret)
     /* Update header */
     qed_header_cpu_to_le(&s->header, (QEDHeader *)write_header_cb->buf);
 
-    bdrv_aio_writev(s->bs->file, 0, &write_header_cb->qiov,
+    bdrv_aio_writev(s->bs->file->bs, 0, &write_header_cb->qiov,
                     write_header_cb->nsectors, qed_write_header_cb,
                     write_header_cb);
 }
@@ -152,7 +155,7 @@ static void qed_write_header(BDRVQEDState *s, BlockCompletionFunc cb,
     write_header_cb->iov.iov_len = len;
     qemu_iovec_init_external(&write_header_cb->qiov, &write_header_cb->iov, 1);
 
-    bdrv_aio_readv(s->bs->file, 0, &write_header_cb->qiov, nsectors,
+    bdrv_aio_readv(s->bs->file->bs, 0, &write_header_cb->qiov, nsectors,
                    qed_write_header_read_cb, write_header_cb);
 }
 
@@ -344,7 +347,7 @@ static void qed_start_need_check_timer(BDRVQEDState *s)
      * migration.
      */
     timer_mod(s->need_check_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() * QED_NEED_CHECK_TIMEOUT);
+                   NANOSECONDS_PER_SECOND * QED_NEED_CHECK_TIMEOUT);
 }
 
 /* It's okay to call this multiple times or when no timer is started */
@@ -354,12 +357,6 @@ static void qed_cancel_need_check_timer(BDRVQEDState *s)
     timer_del(s->need_check_timer);
 }
 
-static void bdrv_qed_rebind(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static void bdrv_qed_detach_aio_context(BlockDriverState *bs)
 {
     BDRVQEDState *s = bs->opaque;
@@ -392,7 +389,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
     s->bs = bs;
     QSIMPLEQ_INIT(&s->allocating_write_reqs);
 
-    ret = bdrv_pread(bs->file, 0, &le_header, sizeof(le_header));
+    ret = bdrv_pread(bs->file->bs, 0, &le_header, sizeof(le_header));
     if (ret < 0) {
         return ret;
     }
@@ -404,11 +401,8 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
     }
     if (s->header.features & ~QED_FEATURE_MASK) {
         /* image uses unsupported feature bits */
-        char buf[64];
-        snprintf(buf, sizeof(buf), "%" PRIx64,
-            s->header.features & ~QED_FEATURE_MASK);
-        error_setg(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                   bdrv_get_device_or_node_name(bs), "QED", buf);
+        error_setg(errp, "Unsupported QED features: %" PRIx64,
+                   s->header.features & ~QED_FEATURE_MASK);
         return -ENOTSUP;
     }
     if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
@@ -416,7 +410,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* Round down file size to the last cluster */
-    file_size = bdrv_getlength(bs->file);
+    file_size = bdrv_getlength(bs->file->bs);
     if (file_size < 0) {
         return file_size;
     }
@@ -452,7 +446,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
             return -EINVAL;
         }
 
-        ret = qed_read_string(bs->file, s->header.backing_filename_offset,
+        ret = qed_read_string(bs->file->bs, s->header.backing_filename_offset,
                               s->header.backing_filename_size, bs->backing_file,
                               sizeof(bs->backing_file));
         if (ret < 0) {
@@ -471,7 +465,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
      * feature is no longer valid.
      */
     if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
-        !bdrv_is_read_only(bs->file) && !(flags & BDRV_O_INCOMING)) {
+        !bdrv_is_read_only(bs->file->bs) && !(flags & BDRV_O_INACTIVE)) {
         s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
 
         ret = qed_write_header_sync(s);
@@ -480,7 +474,7 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
         }
 
         /* From here on only known autoclear feature bits are valid */
-        bdrv_flush(bs->file);
+        bdrv_flush(bs->file->bs);
     }
 
     s->l1_table = qed_alloc_table(s);
@@ -498,8 +492,8 @@ static int bdrv_qed_open(BlockDriverState *bs, QDict *options, int flags,
          * potentially inconsistent images to be opened read-only.  This can
          * aid data recovery from an otherwise inconsistent image.
          */
-        if (!bdrv_is_read_only(bs->file) &&
-            !(flags & BDRV_O_INCOMING)) {
+        if (!bdrv_is_read_only(bs->file->bs) &&
+            !(flags & BDRV_O_INACTIVE)) {
             BdrvCheckResult result = {0};
 
             ret = qed_check(s, &result, true);
@@ -541,7 +535,7 @@ static void bdrv_qed_close(BlockDriverState *bs)
     bdrv_qed_detach_aio_context(bs);
 
     /* Ensure writes reach stable storage */
-    bdrv_flush(bs->file);
+    bdrv_flush(bs->file->bs);
 
     /* Clean shutdown, no check required on next open */
     if (s->header.features & QED_F_NEED_CHECK) {
@@ -573,7 +567,7 @@ static int qed_create(const char *filename, uint32_t cluster_size,
     size_t l1_size = header.cluster_size * header.table_size;
     Error *local_err = NULL;
     int ret = 0;
-    BlockDriverState *bs;
+    BlockBackend *blk;
 
     ret = bdrv_create_file(filename, opts, &local_err);
     if (ret < 0) {
@@ -581,17 +575,17 @@ static int qed_create(const char *filename, uint32_t cluster_size,
         return ret;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_PROTOCOL, NULL,
-                    &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
-        return ret;
+        return -EIO;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* File must start empty and grow, check truncate is supported */
-    ret = bdrv_truncate(bs, 0);
+    ret = blk_truncate(blk, 0);
     if (ret < 0) {
         goto out;
     }
@@ -607,18 +601,18 @@ static int qed_create(const char *filename, uint32_t cluster_size,
     }
 
     qed_header_cpu_to_le(&header, &le_header);
-    ret = bdrv_pwrite(bs, 0, &le_header, sizeof(le_header));
+    ret = blk_pwrite(blk, 0, &le_header, sizeof(le_header));
     if (ret < 0) {
         goto out;
     }
-    ret = bdrv_pwrite(bs, sizeof(le_header), backing_file,
-                      header.backing_filename_size);
+    ret = blk_pwrite(blk, sizeof(le_header), backing_file,
+                     header.backing_filename_size);
     if (ret < 0) {
         goto out;
     }
 
     l1_table = g_malloc0(l1_size);
-    ret = bdrv_pwrite(bs, header.l1_table_offset, l1_table, l1_size);
+    ret = blk_pwrite(blk, header.l1_table_offset, l1_table, l1_size);
     if (ret < 0) {
         goto out;
     }
@@ -626,7 +620,7 @@ static int qed_create(const char *filename, uint32_t cluster_size,
     ret = 0; /* success */
 out:
     g_free(l1_table);
-    bdrv_unref(bs);
+    blk_unref(blk);
     return ret;
 }
 
@@ -686,6 +680,7 @@ typedef struct {
     uint64_t pos;
     int64_t status;
     int *pnum;
+    BlockDriverState **file;
 } QEDIsAllocatedCB;
 
 static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t len)
@@ -697,6 +692,7 @@ static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t l
     case QED_CLUSTER_FOUND:
         offset |= qed_offset_into_cluster(s, cb->pos);
         cb->status = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | offset;
+        *cb->file = cb->bs->file->bs;
         break;
     case QED_CLUSTER_ZERO:
         cb->status = BDRV_BLOCK_ZERO;
@@ -718,7 +714,8 @@ static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t l
 
 static int64_t coroutine_fn bdrv_qed_co_get_block_status(BlockDriverState *bs,
                                                  int64_t sector_num,
-                                                 int nb_sectors, int *pnum)
+                                                 int nb_sectors, int *pnum,
+                                                 BlockDriverState **file)
 {
     BDRVQEDState *s = bs->opaque;
     size_t len = (size_t)nb_sectors * BDRV_SECTOR_SIZE;
@@ -727,6 +724,7 @@ static int64_t coroutine_fn bdrv_qed_co_get_block_status(BlockDriverState *bs,
         .pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE,
         .status = BDRV_BLOCK_OFFSET_MASK,
         .pnum = pnum,
+        .file = file,
     };
     QEDRequest request = { .l2_table = NULL };
 
@@ -772,8 +770,8 @@ static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
     /* If there is a backing file, get its length.  Treat the absence of a
      * backing file like a zero length backing file.
      */
-    if (s->bs->backing_hd) {
-        int64_t l = bdrv_getlength(s->bs->backing_hd);
+    if (s->bs->backing) {
+        int64_t l = bdrv_getlength(s->bs->backing->bs);
         if (l < 0) {
             cb(opaque, l);
             return;
@@ -802,7 +800,7 @@ static void qed_read_backing_file(BDRVQEDState *s, uint64_t pos,
     qemu_iovec_concat(*backing_qiov, qiov, 0, size);
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
-    bdrv_aio_readv(s->bs->backing_hd, pos / BDRV_SECTOR_SIZE,
+    bdrv_aio_readv(s->bs->backing->bs, pos / BDRV_SECTOR_SIZE,
                    *backing_qiov, size / BDRV_SECTOR_SIZE, cb, opaque);
 }
 
@@ -839,7 +837,7 @@ static void qed_copy_from_backing_file_write(void *opaque, int ret)
     }
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_COW_WRITE);
-    bdrv_aio_writev(s->bs->file, copy_cb->offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, copy_cb->offset / BDRV_SECTOR_SIZE,
                     &copy_cb->qiov, copy_cb->qiov.size / BDRV_SECTOR_SIZE,
                     qed_copy_from_backing_file_cb, copy_cb);
 }
@@ -1055,7 +1053,7 @@ static void qed_aio_write_flush_before_l2_update(void *opaque, int ret)
     QEDAIOCB *acb = opaque;
     BDRVQEDState *s = acb_to_s(acb);
 
-    if (!bdrv_aio_flush(s->bs->file, qed_aio_write_l2_update_cb, opaque)) {
+    if (!bdrv_aio_flush(s->bs->file->bs, qed_aio_write_l2_update_cb, opaque)) {
         qed_aio_complete(acb, -EIO);
     }
 }
@@ -1081,7 +1079,7 @@ static void qed_aio_write_main(void *opaque, int ret)
     if (acb->find_cluster_ret == QED_CLUSTER_FOUND) {
         next_fn = qed_aio_next_io;
     } else {
-        if (s->bs->backing_hd) {
+        if (s->bs->backing) {
             next_fn = qed_aio_write_flush_before_l2_update;
         } else {
             next_fn = qed_aio_write_l2_update_cb;
@@ -1089,7 +1087,7 @@ static void qed_aio_write_main(void *opaque, int ret)
     }
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_WRITE_AIO);
-    bdrv_aio_writev(s->bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_writev(s->bs->file->bs, offset / BDRV_SECTOR_SIZE,
                     &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
                     next_fn, acb);
 }
@@ -1139,7 +1137,7 @@ static void qed_aio_write_prefill(void *opaque, int ret)
 static bool qed_should_set_need_check(BDRVQEDState *s)
 {
     /* The flush before L2 update path ensures consistency */
-    if (s->bs->backing_hd) {
+    if (s->bs->backing) {
         return false;
     }
 
@@ -1321,7 +1319,7 @@ static void qed_aio_read_data(void *opaque, int ret,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    bdrv_aio_readv(bs->file, offset / BDRV_SECTOR_SIZE,
+    bdrv_aio_readv(bs->file->bs, offset / BDRV_SECTOR_SIZE,
                    &acb->cur_qiov, acb->cur_qiov.size / BDRV_SECTOR_SIZE,
                    qed_aio_next_io, acb);
     return;
@@ -1443,7 +1441,7 @@ static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
     struct iovec iov;
 
     /* Refuse if there are untouched backing file sectors */
-    if (bs->backing_hd) {
+    if (bs->backing) {
         if (qed_offset_into_cluster(s, sector_num * BDRV_SECTOR_SIZE) != 0) {
             return -ENOTSUP;
         }
@@ -1580,7 +1578,7 @@ static int bdrv_qed_change_backing_file(BlockDriverState *bs,
     }
 
     /* Write new header */
-    ret = bdrv_pwrite_sync(bs->file, 0, buffer, buffer_len);
+    ret = bdrv_pwrite_sync(bs->file->bs, 0, buffer, buffer_len);
     g_free(buffer);
     if (ret == 0) {
         memcpy(&s->header, &new_header, sizeof(new_header));
@@ -1596,7 +1594,7 @@ static void bdrv_qed_invalidate_cache(BlockDriverState *bs, Error **errp)
 
     bdrv_qed_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
+    bdrv_invalidate_cache(bs->file->bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1605,9 +1603,8 @@ static void bdrv_qed_invalidate_cache(BlockDriverState *bs, Error **errp)
     memset(s, 0, sizeof(BDRVQEDState));
     ret = bdrv_qed_open(bs, NULL, bs->open_flags, &local_err);
     if (local_err) {
-        error_setg(errp, "Could not reopen qed layer: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_propagate(errp, local_err);
+        error_prepend(errp, "Could not reopen qed layer: ");
         return;
     } else if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not reopen qed layer");
@@ -1664,7 +1661,6 @@ static BlockDriver bdrv_qed = {
     .supports_backing         = true,
 
     .bdrv_probe               = bdrv_qed_probe,
-    .bdrv_rebind              = bdrv_qed_rebind,
     .bdrv_open                = bdrv_qed_open,
     .bdrv_close               = bdrv_qed_close,
     .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
diff --git a/qemu/block/qed.h b/qemu/block/qed.h
index 615e676fc..22b319875 100644
--- a/qemu/block/qed.h
+++ b/qemu/block/qed.h
@@ -16,6 +16,7 @@
 #define BLOCK_QED_H
 
 #include "block/block_int.h"
+#include "qemu/cutils.h"
 
 /* The layout of a QED file is as follows:
  *
diff --git a/qemu/block/quorum.c b/qemu/block/quorum.c
index 2f6c45f76..da15465a9 100644
--- a/qemu/block/quorum.c
+++ b/qemu/block/quorum.c
@@ -13,6 +13,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "block/block_int.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qdict.h"
@@ -64,7 +65,7 @@ typedef struct QuorumVotes {
 
 /* the following structure holds the state of one quorum instance */
 typedef struct BDRVQuorumState {
-    BlockDriverState **bs; /* children BlockDriverStates */
+    BdrvChild **children;  /* children BlockDriverStates */
     int num_children;      /* children count */
     int threshold;         /* if less than threshold children reads gave the
                             * same result a quorum error occurs.
@@ -214,14 +215,16 @@ static QuorumAIOCB *quorum_aio_get(BDRVQuorumState *s,
     return acb;
 }
 
-static void quorum_report_bad(QuorumAIOCB *acb, char *node_name, int ret)
+static void quorum_report_bad(QuorumOpType type, uint64_t sector_num,
+                              int nb_sectors, char *node_name, int ret)
 {
     const char *msg = NULL;
     if (ret < 0) {
         msg = strerror(-ret);
     }
-    qapi_event_send_quorum_report_bad(!!msg, msg, node_name,
-                                      acb->sector_num, acb->nb_sectors, &error_abort);
+
+    qapi_event_send_quorum_report_bad(type, !!msg, msg, node_name,
+                                      sector_num, nb_sectors, &error_abort);
 }
 
 static void quorum_report_failure(QuorumAIOCB *acb)
@@ -283,9 +286,19 @@ static void quorum_aio_cb(void *opaque, int ret)
     BDRVQuorumState *s = acb->common.bs->opaque;
     bool rewrite = false;
 
+    if (ret == 0) {
+        acb->success_count++;
+    } else {
+        QuorumOpType type;
+        type = acb->is_read ? QUORUM_OP_TYPE_READ : QUORUM_OP_TYPE_WRITE;
+        quorum_report_bad(type, acb->sector_num, acb->nb_sectors,
+                          sacb->aiocb->bs->node_name, ret);
+    }
+
     if (acb->is_read && s->read_pattern == QUORUM_READ_PATTERN_FIFO) {
         /* We try to read next child in FIFO order if we fail to read */
-        if (ret < 0 && ++acb->child_iter < s->num_children) {
+        if (ret < 0 && (acb->child_iter + 1) < s->num_children) {
+            acb->child_iter++;
             read_fifo_child(acb);
             return;
         }
@@ -300,11 +313,6 @@ static void quorum_aio_cb(void *opaque, int ret)
 
     sacb->ret = ret;
     acb->count++;
-    if (ret == 0) {
-        acb->success_count++;
-    } else {
-        quorum_report_bad(acb, sacb->aiocb->bs->node_name, ret);
-    }
     assert(acb->count <= s->num_children);
     assert(acb->success_count <= s->num_children);
     if (acb->count < s->num_children) {
@@ -336,7 +344,9 @@ static void quorum_report_bad_versions(BDRVQuorumState *s,
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            quorum_report_bad(acb, s->bs[item->index]->node_name, 0);
+            quorum_report_bad(QUORUM_OP_TYPE_READ, acb->sector_num,
+                              acb->nb_sectors,
+                              s->children[item->index]->bs->node_name, 0);
         }
     }
 }
@@ -369,8 +379,9 @@ static bool quorum_rewrite_bad_versions(BDRVQuorumState *s, QuorumAIOCB *acb,
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            bdrv_aio_writev(s->bs[item->index], acb->sector_num, acb->qiov,
-                            acb->nb_sectors, quorum_rewrite_aio_cb, acb);
+            bdrv_aio_writev(s->children[item->index]->bs, acb->sector_num,
+                            acb->qiov, acb->nb_sectors, quorum_rewrite_aio_cb,
+                            acb);
         }
     }
 
@@ -639,14 +650,15 @@ static BlockAIOCB *read_quorum_children(QuorumAIOCB *acb)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].buf = qemu_blockalign(s->bs[i], acb->qiov->size);
+        acb->qcrs[i].buf = qemu_blockalign(s->children[i]->bs, acb->qiov->size);
         qemu_iovec_init(&acb->qcrs[i].qiov, acb->qiov->niov);
         qemu_iovec_clone(&acb->qcrs[i].qiov, acb->qiov, acb->qcrs[i].buf);
     }
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_aio_readv(s->bs[i], acb->sector_num, &acb->qcrs[i].qiov,
-                       acb->nb_sectors, quorum_aio_cb, &acb->qcrs[i]);
+        acb->qcrs[i].aiocb = bdrv_aio_readv(s->children[i]->bs, acb->sector_num,
+                                            &acb->qcrs[i].qiov, acb->nb_sectors,
+                                            quorum_aio_cb, &acb->qcrs[i]);
     }
 
     return &acb->common;
@@ -656,14 +668,15 @@ static BlockAIOCB *read_fifo_child(QuorumAIOCB *acb)
 {
     BDRVQuorumState *s = acb->common.bs->opaque;
 
-    acb->qcrs[acb->child_iter].buf = qemu_blockalign(s->bs[acb->child_iter],
-                                                     acb->qiov->size);
+    acb->qcrs[acb->child_iter].buf =
+        qemu_blockalign(s->children[acb->child_iter]->bs, acb->qiov->size);
     qemu_iovec_init(&acb->qcrs[acb->child_iter].qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->qcrs[acb->child_iter].qiov, acb->qiov,
                      acb->qcrs[acb->child_iter].buf);
-    bdrv_aio_readv(s->bs[acb->child_iter], acb->sector_num,
-                   &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
-                   quorum_aio_cb, &acb->qcrs[acb->child_iter]);
+    acb->qcrs[acb->child_iter].aiocb =
+        bdrv_aio_readv(s->children[acb->child_iter]->bs, acb->sector_num,
+                       &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
+                       quorum_aio_cb, &acb->qcrs[acb->child_iter]);
 
     return &acb->common;
 }
@@ -702,8 +715,8 @@ static BlockAIOCB *quorum_aio_writev(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].aiocb = bdrv_aio_writev(s->bs[i], sector_num, qiov,
-                                             nb_sectors, &quorum_aio_cb,
+        acb->qcrs[i].aiocb = bdrv_aio_writev(s->children[i]->bs, sector_num,
+                                             qiov, nb_sectors, &quorum_aio_cb,
                                              &acb->qcrs[i]);
     }
 
@@ -717,12 +730,12 @@ static int64_t quorum_getlength(BlockDriverState *bs)
     int i;
 
     /* check that all file have the same length */
-    result = bdrv_getlength(s->bs[0]);
+    result = bdrv_getlength(s->children[0]->bs);
     if (result < 0) {
         return result;
     }
     for (i = 1; i < s->num_children; i++) {
-        int64_t value = bdrv_getlength(s->bs[i]);
+        int64_t value = bdrv_getlength(s->children[i]->bs);
         if (value < 0) {
             return value;
         }
@@ -741,7 +754,7 @@ static void quorum_invalidate_cache(BlockDriverState *bs, Error **errp)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_invalidate_cache(s->bs[i], &local_err);
+        bdrv_invalidate_cache(s->children[i]->bs, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
@@ -757,19 +770,30 @@ static coroutine_fn int quorum_co_flush(BlockDriverState *bs)
     QuorumVoteValue result_value;
     int i;
     int result = 0;
+    int success_count = 0;
 
     QLIST_INIT(&error_votes.vote_list);
     error_votes.compare = quorum_64bits_compare;
 
     for (i = 0; i < s->num_children; i++) {
-        result = bdrv_co_flush(s->bs[i]);
-        result_value.l = result;
-        quorum_count_vote(&error_votes, &result_value, i);
+        result = bdrv_co_flush(s->children[i]->bs);
+        if (result) {
+            quorum_report_bad(QUORUM_OP_TYPE_FLUSH, 0,
+                              bdrv_nb_sectors(s->children[i]->bs),
+                              s->children[i]->bs->node_name, result);
+            result_value.l = result;
+            quorum_count_vote(&error_votes, &result_value, i);
+        } else {
+            success_count++;
+        }
     }
 
-    winner = quorum_get_vote_winner(&error_votes);
-    result = winner->value.l;
-
+    if (success_count >= s->threshold) {
+        result = 0;
+    } else {
+        winner = quorum_get_vote_winner(&error_votes);
+        result = winner->value.l;
+    }
     quorum_free_vote_list(&error_votes);
 
     return result;
@@ -782,7 +806,7 @@ static bool quorum_recurse_is_first_non_filter(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bool perm = bdrv_recurse_is_first_non_filter(s->bs[i],
+        bool perm = bdrv_recurse_is_first_non_filter(s->children[i]->bs,
                                                      candidate);
         if (perm) {
             return true;
@@ -846,7 +870,7 @@ static int parse_read_pattern(const char *opt)
         return QUORUM_READ_PATTERN_QUORUM;
     }
 
-    for (i = 0; i < QUORUM_READ_PATTERN_MAX; i++) {
+    for (i = 0; i < QUORUM_READ_PATTERN__MAX; i++) {
         if (!strcmp(opt, QuorumReadPattern_lookup[i])) {
             return i;
         }
@@ -889,6 +913,12 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     s->threshold = qemu_opt_get_number(opts, QUORUM_OPT_VOTE_THRESHOLD, 0);
+    /* and validate it against s->num_children */
+    ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
+    if (ret < 0) {
+        goto exit;
+    }
+
     ret = parse_read_pattern(qemu_opt_get(opts, QUORUM_OPT_READ_PATTERN));
     if (ret < 0) {
         error_setg(&local_err, "Please set read-pattern as fifo or quorum");
@@ -897,12 +927,6 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
     s->read_pattern = ret;
 
     if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) {
-        /* and validate it against s->num_children */
-        ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
-        if (ret < 0) {
-            goto exit;
-        }
-
         /* is the driver in blkverify mode */
         if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) &&
             s->num_children == 2 && s->threshold == 2) {
@@ -922,8 +946,8 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
-    /* allocate the children BlockDriverState array */
-    s->bs = g_new0(BlockDriverState *, s->num_children);
+    /* allocate the children array */
+    s->children = g_new0(BdrvChild *, s->num_children);
     opened = g_new0(bool, s->num_children);
 
     for (i = 0; i < s->num_children; i++) {
@@ -931,9 +955,10 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags,
         ret = snprintf(indexstr, 32, "children.%d", i);
         assert(ret < 32);
 
-        ret = bdrv_open_image(&s->bs[i], NULL, options, indexstr, bs,
-                              &child_format, false, &local_err);
-        if (ret < 0) {
+        s->children[i] = bdrv_open_child(NULL, options, indexstr, bs,
+                                         &child_format, false, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
             goto close_exit;
         }
 
@@ -949,9 +974,9 @@ close_exit:
         if (!opened[i]) {
             continue;
         }
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
-    g_free(s->bs);
+    g_free(s->children);
     g_free(opened);
 exit:
     qemu_opts_del(opts);
@@ -968,10 +993,10 @@ static void quorum_close(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
 
-    g_free(s->bs);
+    g_free(s->children);
 }
 
 static void quorum_detach_aio_context(BlockDriverState *bs)
@@ -980,7 +1005,7 @@ static void quorum_detach_aio_context(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_detach_aio_context(s->bs[i]);
+        bdrv_detach_aio_context(s->children[i]->bs);
     }
 }
 
@@ -991,11 +1016,11 @@ static void quorum_attach_aio_context(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_attach_aio_context(s->bs[i], new_context);
+        bdrv_attach_aio_context(s->children[i]->bs, new_context);
     }
 }
 
-static void quorum_refresh_filename(BlockDriverState *bs)
+static void quorum_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     BDRVQuorumState *s = bs->opaque;
     QDict *opts;
@@ -1003,16 +1028,17 @@ static void quorum_refresh_filename(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_refresh_filename(s->bs[i]);
-        if (!s->bs[i]->full_open_options) {
+        bdrv_refresh_filename(s->children[i]->bs);
+        if (!s->children[i]->bs->full_open_options) {
             return;
         }
     }
 
     children = qlist_new();
     for (i = 0; i < s->num_children; i++) {
-        QINCREF(s->bs[i]->full_open_options);
-        qlist_append_obj(children, QOBJECT(s->bs[i]->full_open_options));
+        QINCREF(s->children[i]->bs->full_open_options);
+        qlist_append_obj(children,
+                         QOBJECT(s->children[i]->bs->full_open_options));
     }
 
     opts = qdict_new();
diff --git a/qemu/block/raw-aio.h b/qemu/block/raw-aio.h
index 31d791fe6..811e37501 100644
--- a/qemu/block/raw-aio.h
+++ b/qemu/block/raw-aio.h
@@ -15,6 +15,8 @@
 #ifndef QEMU_RAW_AIO_H
 #define QEMU_RAW_AIO_H
 
+#include "qemu/iov.h"
+
 /* AIO request types */
 #define QEMU_AIO_READ         0x0001
 #define QEMU_AIO_WRITE        0x0002
diff --git a/qemu/block/raw-posix.c b/qemu/block/raw-posix.c
index 855febed5..906d5c941 100644
--- a/qemu/block/raw-posix.c
+++ b/qemu/block/raw-posix.c
@@ -21,7 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "qemu/log.h"
@@ -43,6 +45,7 @@
 #include <IOKit/storage/IOMedia.h>
 #include <IOKit/storage/IOCDMedia.h>
 //#include <IOKit/storage/IOCDTypes.h>
+#include <IOKit/storage/IODVDMedia.h>
 #include <CoreFoundation/CoreFoundation.h>
 #endif
 
@@ -51,8 +54,6 @@
 #include <sys/dkio.h>
 #endif
 #ifdef __linux__
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/param.h>
 #include <linux/cdrom.h>
@@ -127,11 +128,6 @@ do { \
 
 #define FTYPE_FILE   0
 #define FTYPE_CD     1
-#define FTYPE_FD     2
-
-/* if the FD is not accessed during that time (in ns), we try to
-   reopen it to see if the disk has been changed */
-#define FD_OPEN_TIMEOUT (1000000000)
 
 #define MAX_BLOCKSIZE	4096
 
@@ -141,13 +137,6 @@ typedef struct BDRVRawState {
     int open_flags;
     size_t buf_align;
 
-#if defined(__linux__)
-    /* linux floppy specific */
-    int64_t fd_open_time;
-    int64_t fd_error_time;
-    int fd_got_error;
-    int fd_media_changed;
-#endif
 #ifdef CONFIG_LINUX_AIO
     int use_aio;
     void *aio_ctx;
@@ -512,14 +501,19 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
         goto fail;
     }
     if (!s->use_aio && (bdrv_flags & BDRV_O_NATIVE_AIO)) {
-        error_printf("WARNING: aio=native was specified for '%s', but "
-                     "it requires cache.direct=on, which was not "
-                     "specified. Falling back to aio=threads.\n"
-                     "         This will become an error condition in "
-                     "future QEMU versions.\n",
-                     bs->filename);
+        error_setg(errp, "aio=native was specified, but it requires "
+                         "cache.direct=on, which was not specified.");
+        ret = -EINVAL;
+        goto fail;
     }
-#endif
+#else
+    if (bdrv_flags & BDRV_O_NATIVE_AIO) {
+        error_setg(errp, "aio=native was specified, but is not supported "
+                         "in this build.");
+        ret = -EINVAL;
+        goto fail;
+    }
+#endif /* !defined(CONFIG_LINUX_AIO) */
 
     s->has_discard = true;
     s->has_write_zeroes = true;
@@ -626,7 +620,7 @@ static int raw_reopen_prepare(BDRVReopenState *state,
     }
 #endif
 
-    if (s->type == FTYPE_FD || s->type == FTYPE_CD) {
+    if (s->type == FTYPE_CD) {
         raw_s->open_flags |= O_NONBLOCK;
     }
 
@@ -670,11 +664,17 @@ static int raw_reopen_prepare(BDRVReopenState *state,
 
     /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
     if (raw_s->fd == -1) {
-        assert(!(raw_s->open_flags & O_CREAT));
-        raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
-        if (raw_s->fd == -1) {
-            error_setg_errno(errp, errno, "Could not reopen file");
-            ret = -1;
+        const char *normalized_filename = state->bs->filename;
+        ret = raw_normalize_devicepath(&normalized_filename);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not normalize device path");
+        } else {
+            assert(!(raw_s->open_flags & O_CREAT));
+            raw_s->fd = qemu_open(normalized_filename, raw_s->open_flags);
+            if (raw_s->fd == -1) {
+                error_setg_errno(errp, errno, "Could not reopen file");
+                ret = -1;
+            }
         }
     }
 
@@ -780,7 +780,6 @@ static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
 {
     BDRVRawState *s = bs->opaque;
     struct hd_geometry ioctl_geo = {0};
-    uint32_t blksize;
 
     /* If DASD, get its geometry */
     if (check_for_dasd(s->fd) < 0) {
@@ -800,12 +799,6 @@ static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
     }
     geo->heads = ioctl_geo.heads;
     geo->sectors = ioctl_geo.sectors;
-    if (!probe_physical_blocksize(s->fd, &blksize)) {
-        /* overwrite cyls: HDIO_GETGEO result is incorrect for big drives */
-        geo->cylinders = bdrv_nb_sectors(bs) / (blksize / BDRV_SECTOR_SIZE)
-                                             / (geo->heads * geo->sectors);
-        return 0;
-    }
     geo->cylinders = ioctl_geo.cylinders;
 
     return 0;
@@ -1253,7 +1246,7 @@ static int aio_worker(void *arg)
         break;
     }
 
-    g_slice_free(RawPosixAIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
@@ -1261,7 +1254,7 @@ static int paio_submit_co(BlockDriverState *bs, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
@@ -1286,7 +1279,7 @@ static BlockAIOCB *paio_submit(BlockDriverState *bs, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
@@ -1633,7 +1626,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
     nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
-                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               PREALLOC_MODE__MAX, PREALLOC_MODE_OFF,
                                &local_err);
     g_free(buf);
     if (local_err) {
@@ -1642,7 +1635,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
         goto out;
     }
 
-    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
+    fd = qemu_open(filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
     if (fd < 0) {
         result = -errno;
@@ -1827,7 +1820,8 @@ static int find_allocation(BlockDriverState *bs, off_t start,
  */
 static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                                     int64_t sector_num,
-                                                    int nb_sectors, int *pnum)
+                                                    int nb_sectors, int *pnum,
+                                                    BlockDriverState **file)
 {
     off_t start, data = 0, hole = 0;
     int64_t total_size;
@@ -1869,6 +1863,7 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
         *pnum = MIN(nb_sectors, (data - start) / BDRV_SECTOR_SIZE);
         ret = BDRV_BLOCK_ZERO;
     }
+    *file = bs;
     return ret | BDRV_BLOCK_OFFSET_VALID | start;
 }
 
@@ -1972,36 +1967,51 @@ BlockDriver bdrv_file = {
 /* host device */
 
 #if defined(__APPLE__) && defined(__MACH__)
-static kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator );
-static kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize );
-
-kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator )
+static kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                                CFIndex maxPathSize, int flags);
+static char *FindEjectableOpticalMedia(io_iterator_t *mediaIterator)
 {
-    kern_return_t       kernResult;
+    kern_return_t kernResult = KERN_FAILURE;
     mach_port_t     masterPort;
     CFMutableDictionaryRef  classesToMatch;
+    const char *matching_array[] = {kIODVDMediaClass, kIOCDMediaClass};
+    char *mediaType = NULL;
 
     kernResult = IOMasterPort( MACH_PORT_NULL, &masterPort );
     if ( KERN_SUCCESS != kernResult ) {
         printf( "IOMasterPort returned %d\n", kernResult );
     }
 
-    classesToMatch = IOServiceMatching( kIOCDMediaClass );
-    if ( classesToMatch == NULL ) {
-        printf( "IOServiceMatching returned a NULL dictionary.\n" );
-    } else {
-    CFDictionarySetValue( classesToMatch, CFSTR( kIOMediaEjectableKey ), kCFBooleanTrue );
-    }
-    kernResult = IOServiceGetMatchingServices( masterPort, classesToMatch, mediaIterator );
-    if ( KERN_SUCCESS != kernResult )
-    {
-        printf( "IOServiceGetMatchingServices returned %d\n", kernResult );
-    }
+    int index;
+    for (index = 0; index < ARRAY_SIZE(matching_array); index++) {
+        classesToMatch = IOServiceMatching(matching_array[index]);
+        if (classesToMatch == NULL) {
+            error_report("IOServiceMatching returned NULL for %s",
+                         matching_array[index]);
+            continue;
+        }
+        CFDictionarySetValue(classesToMatch, CFSTR(kIOMediaEjectableKey),
+                             kCFBooleanTrue);
+        kernResult = IOServiceGetMatchingServices(masterPort, classesToMatch,
+                                                  mediaIterator);
+        if (kernResult != KERN_SUCCESS) {
+            error_report("Note: IOServiceGetMatchingServices returned %d",
+                         kernResult);
+            continue;
+        }
 
-    return kernResult;
+        /* If a match was found, leave the loop */
+        if (*mediaIterator != 0) {
+            DPRINTF("Matching using %s\n", matching_array[index]);
+            mediaType = g_strdup(matching_array[index]);
+            break;
+        }
+    }
+    return mediaType;
 }
 
-kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize )
+kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                         CFIndex maxPathSize, int flags)
 {
     io_object_t     nextMedia;
     kern_return_t   kernResult = KERN_FAILURE;
@@ -2014,7 +2024,9 @@ kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex ma
         if ( bsdPathAsCFString ) {
             size_t devPathLength;
             strcpy( bsdPath, _PATH_DEV );
-            strcat( bsdPath, "r" );
+            if (flags & BDRV_O_NOCACHE) {
+                strcat(bsdPath, "r");
+            }
             devPathLength = strlen( bsdPath );
             if ( CFStringGetCString( bsdPathAsCFString, bsdPath + devPathLength, maxPathSize - devPathLength, kCFStringEncodingASCII ) ) {
                 kernResult = KERN_SUCCESS;
@@ -2027,7 +2039,46 @@ kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex ma
     return kernResult;
 }
 
-#endif
+/* Sets up a real cdrom for use in QEMU */
+static bool setup_cdrom(char *bsd_path, Error **errp)
+{
+    int index, num_of_test_partitions = 2, fd;
+    char test_partition[MAXPATHLEN];
+    bool partition_found = false;
+
+    /* look for a working partition */
+    for (index = 0; index < num_of_test_partitions; index++) {
+        snprintf(test_partition, sizeof(test_partition), "%ss%d", bsd_path,
+                 index);
+        fd = qemu_open(test_partition, O_RDONLY | O_BINARY | O_LARGEFILE);
+        if (fd >= 0) {
+            partition_found = true;
+            qemu_close(fd);
+            break;
+        }
+    }
+
+    /* if a working partition on the device was not found */
+    if (partition_found == false) {
+        error_setg(errp, "Failed to find a working partition on disc");
+    } else {
+        DPRINTF("Using %s as optical disc\n", test_partition);
+        pstrcpy(bsd_path, MAXPATHLEN, test_partition);
+    }
+    return partition_found;
+}
+
+/* Prints directions on mounting and unmounting a device */
+static void print_unmounting_directions(const char *file_name)
+{
+    error_report("If device %s is mounted on the desktop, unmount"
+                 " it first before using it in QEMU", file_name);
+    error_report("Command to unmount device: diskutil unmountDisk %s",
+                 file_name);
+    error_report("Command to mount device: diskutil mountDisk %s", file_name);
+}
+
+#endif /* defined(__APPLE__) && defined(__MACH__) */
 
 static int hdev_probe_device(const char *filename)
 {
@@ -2118,33 +2169,57 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
 
 #if defined(__APPLE__) && defined(__MACH__)
     const char *filename = qdict_get_str(options, "filename");
+    char bsd_path[MAXPATHLEN] = "";
+    bool error_occurred = false;
+
+    /* If using a real cdrom */
+    if (strcmp(filename, "/dev/cdrom") == 0) {
+        char *mediaType = NULL;
+        kern_return_t ret_val;
+        io_iterator_t mediaIterator = 0;
+
+        mediaType = FindEjectableOpticalMedia(&mediaIterator);
+        if (mediaType == NULL) {
+            error_setg(errp, "Please make sure your CD/DVD is in the optical"
+                       " drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
 
-    if (strstart(filename, "/dev/cdrom", NULL)) {
-        kern_return_t kernResult;
-        io_iterator_t mediaIterator;
-        char bsdPath[ MAXPATHLEN ];
-        int fd;
-
-        kernResult = FindEjectableCDMedia( &mediaIterator );
-        kernResult = GetBSDPath( mediaIterator, bsdPath, sizeof( bsdPath ) );
-
-        if ( bsdPath[ 0 ] != '\0' ) {
-            strcat(bsdPath,"s0");
-            /* some CDs don't have a partition 0 */
-            fd = qemu_open(bsdPath, O_RDONLY | O_BINARY | O_LARGEFILE);
-            if (fd < 0) {
-                bsdPath[strlen(bsdPath)-1] = '1';
-            } else {
-                qemu_close(fd);
-            }
-            filename = bsdPath;
-            qdict_put(options, "filename", qstring_from_str(filename));
+        ret_val = GetBSDPath(mediaIterator, bsd_path, sizeof(bsd_path), flags);
+        if (ret_val != KERN_SUCCESS) {
+            error_setg(errp, "Could not get BSD path for optical drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
+
+        /* If a real optical drive was not found */
+        if (bsd_path[0] == '\0') {
+            error_setg(errp, "Failed to obtain bsd path for optical drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
+
+        /* If using a cdrom disc and finding a partition on the disc failed */
+        if (strncmp(mediaType, kIOCDMediaClass, 9) == 0 &&
+            setup_cdrom(bsd_path, errp) == false) {
+            print_unmounting_directions(bsd_path);
+            error_occurred = true;
+            goto hdev_open_Mac_error;
         }
 
-        if ( mediaIterator )
-            IOObjectRelease( mediaIterator );
+        qdict_put(options, "filename", qstring_from_str(bsd_path));
+
+hdev_open_Mac_error:
+        g_free(mediaType);
+        if (mediaIterator) {
+            IOObjectRelease(mediaIterator);
+        }
+        if (error_occurred) {
+            return -ENOENT;
+        }
     }
-#endif
+#endif /* defined(__APPLE__) && defined(__MACH__) */
 
     s->type = FTYPE_FILE;
 
@@ -2153,6 +2228,15 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
         if (local_err) {
             error_propagate(errp, local_err);
         }
+#if defined(__APPLE__) && defined(__MACH__)
+        if (*bsd_path) {
+            filename = bsd_path;
+        }
+        /* if a physical device experienced an error while being opened */
+        if (strncmp(filename, "/dev/", 5) == 0) {
+            print_unmounting_directions(filename);
+        }
+#endif /* defined(__APPLE__) && defined(__MACH__) */
         return ret;
     }
 
@@ -2172,53 +2256,6 @@ static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
 }
 
 #if defined(__linux__)
-/* Note: we do not have a reliable method to detect if the floppy is
-   present. The current method is to try to open the floppy at every
-   I/O and to keep it opened during a few hundreds of ms. */
-static int fd_open(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int last_media_present;
-
-    if (s->type != FTYPE_FD)
-        return 0;
-    last_media_present = (s->fd >= 0);
-    if (s->fd >= 0 &&
-        (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->fd_open_time) >= FD_OPEN_TIMEOUT) {
-        qemu_close(s->fd);
-        s->fd = -1;
-        DPRINTF("Floppy closed\n");
-    }
-    if (s->fd < 0) {
-        if (s->fd_got_error &&
-            (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->fd_error_time) < FD_OPEN_TIMEOUT) {
-            DPRINTF("No floppy (open delayed)\n");
-            return -EIO;
-        }
-        s->fd = qemu_open(bs->filename, s->open_flags & ~O_NONBLOCK);
-        if (s->fd < 0) {
-            s->fd_error_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-            s->fd_got_error = 1;
-            if (last_media_present)
-                s->fd_media_changed = 1;
-            DPRINTF("No floppy\n");
-            return -EIO;
-        }
-        DPRINTF("Floppy opened\n");
-    }
-    if (!last_media_present)
-        s->fd_media_changed = 1;
-    s->fd_open_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-    s->fd_got_error = 0;
-    return 0;
-}
-
-static int hdev_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    BDRVRawState *s = bs->opaque;
-
-    return ioctl(s->fd, req, buf);
-}
 
 static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
@@ -2231,7 +2268,7 @@ static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
     if (fd_open(bs) < 0)
         return NULL;
 
-    acb = g_slice_new(RawPosixAIOData);
+    acb = g_new(RawPosixAIOData, 1);
     acb->bs = bs;
     acb->aio_type = QEMU_AIO_IOCTL;
     acb->aio_fildes = s->fd;
@@ -2241,8 +2278,8 @@ static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
+#endif /* linux */
 
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static int fd_open(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
@@ -2252,14 +2289,6 @@ static int fd_open(BlockDriverState *bs)
         return 0;
     return -EIO;
 }
-#else /* !linux && !FreeBSD */
-
-static int fd_open(BlockDriverState *bs)
-{
-    return 0;
-}
-
-#endif /* !linux && !FreeBSD */
 
 static coroutine_fn BlockAIOCB *hdev_aio_discard(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors,
@@ -2303,17 +2332,22 @@ static int hdev_create(const char *filename, QemuOpts *opts,
     int64_t total_size = 0;
     bool has_prefix;
 
-    /* This function is used by all three protocol block drivers and therefore
-     * any of these three prefixes may be given.
+    /* This function is used by both protocol block drivers and therefore either
+     * of these prefixes may be given.
      * The return value has to be stored somewhere, otherwise this is an error
      * due to -Werror=unused-value. */
     has_prefix =
         strstart(filename, "host_device:", &filename) ||
-        strstart(filename, "host_cdrom:" , &filename) ||
-        strstart(filename, "host_floppy:", &filename);
+        strstart(filename, "host_cdrom:" , &filename);
 
     (void)has_prefix;
 
+    ret = raw_normalize_devicepath(&filename);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not normalize device path");
+        return ret;
+    }
+
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
                           BDRV_SECTOR_SIZE);
@@ -2379,160 +2413,10 @@ static BlockDriver bdrv_host_device = {
 
     /* generic scsi device */
 #ifdef __linux__
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 #endif
 };
 
-#ifdef __linux__
-static void floppy_parse_filename(const char *filename, QDict *options,
-                                  Error **errp)
-{
-    /* The prefix is optional, just as for "file". */
-    strstart(filename, "host_floppy:", &filename);
-
-    qdict_put_obj(options, "filename", QOBJECT(qstring_from_str(filename)));
-}
-
-static int floppy_open(BlockDriverState *bs, QDict *options, int flags,
-                       Error **errp)
-{
-    BDRVRawState *s = bs->opaque;
-    Error *local_err = NULL;
-    int ret;
-
-    s->type = FTYPE_FD;
-
-    /* open will not fail even if no floppy is inserted, so add O_NONBLOCK */
-    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
-    if (ret) {
-        if (local_err) {
-            error_propagate(errp, local_err);
-        }
-        return ret;
-    }
-
-    /* close fd so that we can reopen it as needed */
-    qemu_close(s->fd);
-    s->fd = -1;
-    s->fd_media_changed = 1;
-
-    error_report("Host floppy pass-through is deprecated");
-    error_printf("Support for it will be removed in a future release.\n");
-    return 0;
-}
-
-static int floppy_probe_device(const char *filename)
-{
-    int fd, ret;
-    int prio = 0;
-    struct floppy_struct fdparam;
-    struct stat st;
-
-    if (strstart(filename, "/dev/fd", NULL) &&
-        !strstart(filename, "/dev/fdset/", NULL) &&
-        !strstart(filename, "/dev/fd/", NULL)) {
-        prio = 50;
-    }
-
-    fd = qemu_open(filename, O_RDONLY | O_NONBLOCK);
-    if (fd < 0) {
-        goto out;
-    }
-    ret = fstat(fd, &st);
-    if (ret == -1 || !S_ISBLK(st.st_mode)) {
-        goto outc;
-    }
-
-    /* Attempt to detect via a floppy specific ioctl */
-    ret = ioctl(fd, FDGETPRM, &fdparam);
-    if (ret >= 0)
-        prio = 100;
-
-outc:
-    qemu_close(fd);
-out:
-    return prio;
-}
-
-
-static int floppy_is_inserted(BlockDriverState *bs)
-{
-    return fd_open(bs) >= 0;
-}
-
-static int floppy_media_changed(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int ret;
-
-    /*
-     * XXX: we do not have a true media changed indication.
-     * It does not work if the floppy is changed without trying to read it.
-     */
-    fd_open(bs);
-    ret = s->fd_media_changed;
-    s->fd_media_changed = 0;
-    DPRINTF("Floppy changed=%d\n", ret);
-    return ret;
-}
-
-static void floppy_eject(BlockDriverState *bs, bool eject_flag)
-{
-    BDRVRawState *s = bs->opaque;
-    int fd;
-
-    if (s->fd >= 0) {
-        qemu_close(s->fd);
-        s->fd = -1;
-    }
-    fd = qemu_open(bs->filename, s->open_flags | O_NONBLOCK);
-    if (fd >= 0) {
-        if (ioctl(fd, FDEJECT, 0) < 0)
-            perror("FDEJECT");
-        qemu_close(fd);
-    }
-}
-
-static BlockDriver bdrv_host_floppy = {
-    .format_name        = "host_floppy",
-    .protocol_name      = "host_floppy",
-    .instance_size      = sizeof(BDRVRawState),
-    .bdrv_needs_filename = true,
-    .bdrv_probe_device	= floppy_probe_device,
-    .bdrv_parse_filename = floppy_parse_filename,
-    .bdrv_file_open     = floppy_open,
-    .bdrv_close         = raw_close,
-    .bdrv_reopen_prepare = raw_reopen_prepare,
-    .bdrv_reopen_commit  = raw_reopen_commit,
-    .bdrv_reopen_abort   = raw_reopen_abort,
-    .bdrv_create         = hdev_create,
-    .create_opts         = &raw_create_opts,
-
-    .bdrv_aio_readv     = raw_aio_readv,
-    .bdrv_aio_writev    = raw_aio_writev,
-    .bdrv_aio_flush	= raw_aio_flush,
-    .bdrv_refresh_limits = raw_refresh_limits,
-    .bdrv_io_plug = raw_aio_plug,
-    .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
-
-    .bdrv_truncate      = raw_truncate,
-    .bdrv_getlength      = raw_getlength,
-    .has_variable_length = true,
-    .bdrv_get_allocated_file_size
-                        = raw_get_allocated_file_size,
-
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
-    /* removable device support */
-    .bdrv_is_inserted   = floppy_is_inserted,
-    .bdrv_media_changed = floppy_media_changed,
-    .bdrv_eject         = floppy_eject,
-};
-#endif
-
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static void cdrom_parse_filename(const char *filename, QDict *options,
                                  Error **errp)
@@ -2588,15 +2472,13 @@ out:
     return prio;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
     int ret;
 
     ret = ioctl(s->fd, CDROM_DRIVE_STATUS, CDSL_CURRENT);
-    if (ret == CDS_DISC_OK)
-        return 1;
-    return 0;
+    return ret == CDS_DISC_OK;
 }
 
 static void cdrom_eject(BlockDriverState *bs, bool eject_flag)
@@ -2663,7 +2545,6 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_lock_medium   = cdrom_lock_medium,
 
     /* generic scsi device */
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 };
 #endif /* __linux__ */
@@ -2722,7 +2603,7 @@ static int cdrom_reopen(BlockDriverState *bs)
     return 0;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     return raw_getlength(bs) > 0;
 }
@@ -2810,7 +2691,6 @@ static void bdrv_file_init(void)
     bdrv_register(&bdrv_file);
     bdrv_register(&bdrv_host_device);
 #ifdef __linux__
-    bdrv_register(&bdrv_host_floppy);
     bdrv_register(&bdrv_host_cdrom);
 #endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
diff --git a/qemu/block/raw-win32.c b/qemu/block/raw-win32.c
index 68f2338ac..fd2389153 100644
--- a/qemu/block/raw-win32.c
+++ b/qemu/block/raw-win32.c
@@ -21,7 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
@@ -119,9 +121,9 @@ static int aio_worker(void *arg)
     case QEMU_AIO_WRITE:
         count = handle_aiocb_rw(aiocb);
         if (count == aiocb->aio_nbytes) {
-            count = 0;
+            ret = 0;
         } else {
-            count = -EINVAL;
+            ret = -EINVAL;
         }
         break;
     case QEMU_AIO_FLUSH:
@@ -135,7 +137,7 @@ static int aio_worker(void *arg)
         break;
     }
 
-    g_slice_free(RawWin32AIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
@@ -143,7 +145,7 @@ static BlockAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
+    RawWin32AIOData *acb = g_new(RawWin32AIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
diff --git a/qemu/block/raw_bsd.c b/qemu/block/raw_bsd.c
index e3d2d0468..a6cc7e991 100644
--- a/qemu/block/raw_bsd.c
+++ b/qemu/block/raw_bsd.c
@@ -26,7 +26,9 @@
  * IN THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/block_int.h"
+#include "qapi/error.h"
 #include "qemu/option.h"
 
 static QemuOptsList raw_create_opts = {
@@ -52,11 +54,12 @@ static int coroutine_fn raw_co_readv(BlockDriverState *bs, int64_t sector_num,
                                      int nb_sectors, QEMUIOVector *qiov)
 {
     BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    return bdrv_co_readv(bs->file, sector_num, nb_sectors, qiov);
+    return bdrv_co_readv(bs->file->bs, sector_num, nb_sectors, qiov);
 }
 
-static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors, QEMUIOVector *qiov)
+static int coroutine_fn
+raw_co_writev_flags(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                    QEMUIOVector *qiov, int flags)
 {
     void *buf = NULL;
     BlockDriver *drv;
@@ -75,7 +78,7 @@ static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
             return 0;
         }
 
-        buf = qemu_try_blockalign(bs->file, 512);
+        buf = qemu_try_blockalign(bs->file->bs, 512);
         if (!buf) {
             ret = -ENOMEM;
             goto fail;
@@ -102,7 +105,8 @@ static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
-    ret = bdrv_co_writev(bs->file, sector_num, nb_sectors, qiov);
+    ret = bdrv_co_do_pwritev(bs->file->bs, sector_num * BDRV_SECTOR_SIZE,
+                             nb_sectors * BDRV_SECTOR_SIZE, qiov, flags);
 
 fail:
     if (qiov == &local_qiov) {
@@ -112,11 +116,20 @@ fail:
     return ret;
 }
 
+static int coroutine_fn
+raw_co_writev(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+              QEMUIOVector *qiov)
+{
+    return raw_co_writev_flags(bs, sector_num, nb_sectors, qiov, 0);
+}
+
 static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                             int64_t sector_num,
-                                            int nb_sectors, int *pnum)
+                                            int nb_sectors, int *pnum,
+                                            BlockDriverState **file)
 {
     *pnum = nb_sectors;
+    *file = bs->file->bs;
     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_DATA |
            (sector_num << BDRV_SECTOR_BITS);
 }
@@ -125,58 +138,48 @@ static int coroutine_fn raw_co_write_zeroes(BlockDriverState *bs,
                                             int64_t sector_num, int nb_sectors,
                                             BdrvRequestFlags flags)
 {
-    return bdrv_co_write_zeroes(bs->file, sector_num, nb_sectors, flags);
+    return bdrv_co_write_zeroes(bs->file->bs, sector_num, nb_sectors, flags);
 }
 
 static int coroutine_fn raw_co_discard(BlockDriverState *bs,
                                        int64_t sector_num, int nb_sectors)
 {
-    return bdrv_co_discard(bs->file, sector_num, nb_sectors);
+    return bdrv_co_discard(bs->file->bs, sector_num, nb_sectors);
 }
 
 static int64_t raw_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
 static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    return bdrv_get_info(bs->file, bdi);
+    return bdrv_get_info(bs->file->bs, bdi);
 }
 
 static void raw_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    bs->bl = bs->file->bl;
+    bs->bl = bs->file->bs->bl;
 }
 
 static int raw_truncate(BlockDriverState *bs, int64_t offset)
 {
-    return bdrv_truncate(bs->file, offset);
-}
-
-static int raw_is_inserted(BlockDriverState *bs)
-{
-    return bdrv_is_inserted(bs->file);
+    return bdrv_truncate(bs->file->bs, offset);
 }
 
 static int raw_media_changed(BlockDriverState *bs)
 {
-    return bdrv_media_changed(bs->file);
+    return bdrv_media_changed(bs->file->bs);
 }
 
 static void raw_eject(BlockDriverState *bs, bool eject_flag)
 {
-    bdrv_eject(bs->file, eject_flag);
+    bdrv_eject(bs->file->bs, eject_flag);
 }
 
 static void raw_lock_medium(BlockDriverState *bs, bool locked)
 {
-    bdrv_lock_medium(bs->file, locked);
-}
-
-static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    return bdrv_ioctl(bs->file, req, buf);
+    bdrv_lock_medium(bs->file->bs, locked);
 }
 
 static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
@@ -184,12 +187,12 @@ static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
                                  BlockCompletionFunc *cb,
                                  void *opaque)
 {
-    return bdrv_aio_ioctl(bs->file, req, buf, cb, opaque);
+    return bdrv_aio_ioctl(bs->file->bs, req, buf, cb, opaque);
 }
 
 static int raw_has_zero_init(BlockDriverState *bs)
 {
-    return bdrv_has_zero_init(bs->file);
+    return bdrv_has_zero_init(bs->file->bs);
 }
 
 static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
@@ -207,7 +210,7 @@ static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
 static int raw_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
-    bs->sg = bs->file->sg;
+    bs->sg = bs->file->bs->sg;
 
     if (bs->probed && !bdrv_is_read_only(bs)) {
         fprintf(stderr,
@@ -217,7 +220,7 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags,
                 "raw images, write operations on block 0 will be restricted.\n"
                 "         Specify the 'raw' format explicitly to remove the "
                 "restrictions.\n",
-                bs->file->filename);
+                bs->file->bs->filename);
     }
 
     return 0;
@@ -237,12 +240,12 @@ static int raw_probe(const uint8_t *buf, int buf_size, const char *filename)
 
 static int raw_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 {
-    return bdrv_probe_blocksizes(bs->file, bsz);
+    return bdrv_probe_blocksizes(bs->file->bs, bsz);
 }
 
 static int raw_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
 {
-    return bdrv_probe_geometry(bs->file, geo);
+    return bdrv_probe_geometry(bs->file->bs, geo);
 }
 
 BlockDriver bdrv_raw = {
@@ -254,6 +257,8 @@ BlockDriver bdrv_raw = {
     .bdrv_create          = &raw_create,
     .bdrv_co_readv        = &raw_co_readv,
     .bdrv_co_writev       = &raw_co_writev,
+    .bdrv_co_writev_flags = &raw_co_writev_flags,
+    .supported_write_flags = BDRV_REQ_FUA,
     .bdrv_co_write_zeroes = &raw_co_write_zeroes,
     .bdrv_co_discard      = &raw_co_discard,
     .bdrv_co_get_block_status = &raw_co_get_block_status,
@@ -264,11 +269,9 @@ BlockDriver bdrv_raw = {
     .bdrv_refresh_limits  = &raw_refresh_limits,
     .bdrv_probe_blocksizes = &raw_probe_blocksizes,
     .bdrv_probe_geometry  = &raw_probe_geometry,
-    .bdrv_is_inserted     = &raw_is_inserted,
     .bdrv_media_changed   = &raw_media_changed,
     .bdrv_eject           = &raw_eject,
     .bdrv_lock_medium     = &raw_lock_medium,
-    .bdrv_ioctl           = &raw_ioctl,
     .bdrv_aio_ioctl       = &raw_aio_ioctl,
     .create_opts          = &raw_create_opts,
     .bdrv_has_zero_init   = &raw_has_zero_init
diff --git a/qemu/block/rbd.c b/qemu/block/rbd.c
index a60a19d58..5bc5b3253 100644
--- a/qemu/block/rbd.c
+++ b/qemu/block/rbd.c
@@ -11,11 +11,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "block/block_int.h"
+#include "crypto/secret.h"
+#include "qemu/cutils.h"
 
 #include <rbd/librbd.h>
 
@@ -228,6 +230,27 @@ static char *qemu_rbd_parse_clientname(const char *conf, char *clientname)
     return NULL;
 }
 
+
+static int qemu_rbd_set_auth(rados_t cluster, const char *secretid,
+                             Error **errp)
+{
+    if (secretid == 0) {
+        return 0;
+    }
+
+    gchar *secret = qcrypto_secret_lookup_as_base64(secretid,
+                                                    errp);
+    if (!secret) {
+        return -1;
+    }
+
+    rados_conf_set(cluster, "key", secret);
+    g_free(secret);
+
+    return 0;
+}
+
+
 static int qemu_rbd_set_conf(rados_t cluster, const char *conf,
                              bool only_read_conf_file,
                              Error **errp)
@@ -299,10 +322,13 @@ static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
     char conf[RBD_MAX_CONF_SIZE];
     char clientname_buf[RBD_MAX_CONF_SIZE];
     char *clientname;
+    const char *secretid;
     rados_t cluster;
     rados_ioctx_t io_ctx;
     int ret;
 
+    secretid = qemu_opt_get(opts, "password-secret");
+
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
                            name, sizeof(name),
@@ -350,6 +376,11 @@ static int qemu_rbd_create(const char *filename, QemuOpts *opts, Error **errp)
         return -EIO;
     }
 
+    if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
+        rados_shutdown(cluster);
+        return -EIO;
+    }
+
     if (rados_connect(cluster) < 0) {
         error_setg(errp, "error connecting");
         rados_shutdown(cluster);
@@ -423,6 +454,11 @@ static QemuOptsList runtime_opts = {
             .type = QEMU_OPT_STRING,
             .help = "Specification of the rbd image",
         },
+        {
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret providing the password",
+        },
         { /* end of list */ }
     },
 };
@@ -436,6 +472,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
     char conf[RBD_MAX_CONF_SIZE];
     char clientname_buf[RBD_MAX_CONF_SIZE];
     char *clientname;
+    const char *secretid;
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *filename;
@@ -450,6 +487,7 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     filename = qemu_opt_get(opts, "filename");
+    secretid = qemu_opt_get(opts, "password-secret");
 
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
@@ -488,6 +526,11 @@ static int qemu_rbd_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }
 
+    if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
+        r = -EIO;
+        goto failed_shutdown;
+    }
+
     /*
      * Fallback to more conservative semantics if setting cache
      * options fails. Ignore errors from setting rbd_cache because the
@@ -919,6 +962,11 @@ static QemuOptsList qemu_rbd_create_opts = {
             .type = QEMU_OPT_SIZE,
             .help = "RBD object size"
         },
+        {
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret providing the password",
+        },
         { /* end of list */ }
     }
 };
diff --git a/qemu/block/sheepdog.c b/qemu/block/sheepdog.c
index 9585beb73..33e0a3382 100644
--- a/qemu/block/sheepdog.c
+++ b/qemu/block/sheepdog.c
@@ -12,12 +12,15 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/bitops.h"
+#include "qemu/cutils.h"
 
 #define SD_PROTO_VER 0x01
 
@@ -28,7 +31,6 @@
 #define SD_OP_READ_OBJ       0x02
 #define SD_OP_WRITE_OBJ      0x03
 /* 0x04 is used internally by Sheepdog */
-#define SD_OP_DISCARD_OBJ    0x05
 
 #define SD_OP_NEW_VDI        0x11
 #define SD_OP_LOCK_VDI       0x12
@@ -284,6 +286,12 @@ static inline bool is_snapshot(struct SheepdogInode *inode)
     return !!inode->snap_ctime;
 }
 
+static inline size_t count_data_objs(const struct SheepdogInode *inode)
+{
+    return DIV_ROUND_UP(inode->vdi_size,
+                        (1UL << inode->block_size_shift));
+}
+
 #undef DPRINTF
 #ifdef DEBUG_SDOG
 #define DPRINTF(fmt, args...)                                       \
@@ -318,7 +326,7 @@ enum AIOCBState {
     AIOCB_DISCARD_OBJ,
 };
 
-#define AIOCBOverwrapping(x, y)                                 \
+#define AIOCBOverlapping(x, y)                                 \
     (!(x->max_affect_data_idx < y->min_affect_data_idx          \
        || y->max_affect_data_idx < x->min_affect_data_idx))
 
@@ -342,6 +350,15 @@ struct SheepdogAIOCB {
     uint32_t min_affect_data_idx;
     uint32_t max_affect_data_idx;
 
+    /*
+     * The difference between affect_data_idx and dirty_data_idx:
+     * affect_data_idx represents range of index of all request types.
+     * dirty_data_idx represents range of index updated by COW requests.
+     * dirty_data_idx is used for updating an inode object.
+     */
+    uint32_t min_dirty_data_idx;
+    uint32_t max_dirty_data_idx;
+
     QLIST_ENTRY(SheepdogAIOCB) aiocb_siblings;
 };
 
@@ -351,9 +368,6 @@ typedef struct BDRVSheepdogState {
 
     SheepdogInode inode;
 
-    uint32_t min_dirty_data_idx;
-    uint32_t max_dirty_data_idx;
-
     char name[SD_MAX_VDI_LEN];
     bool is_snapshot;
     uint32_t cache_flags;
@@ -373,10 +387,15 @@ typedef struct BDRVSheepdogState {
     QLIST_HEAD(inflight_aio_head, AIOReq) inflight_aio_head;
     QLIST_HEAD(failed_aio_head, AIOReq) failed_aio_head;
 
-    CoQueue overwrapping_queue;
+    CoQueue overlapping_queue;
     QLIST_HEAD(inflight_aiocb_head, SheepdogAIOCB) inflight_aiocb_head;
 } BDRVSheepdogState;
 
+typedef struct BDRVSheepdogReopenState {
+    int fd;
+    int cache_flags;
+} BDRVSheepdogReopenState;
+
 static const char * sd_strerror(int err)
 {
     int i;
@@ -556,6 +575,9 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
     acb->max_affect_data_idx = (acb->sector_num * BDRV_SECTOR_SIZE +
                               acb->nb_sectors * BDRV_SECTOR_SIZE) / object_size;
 
+    acb->min_dirty_data_idx = UINT32_MAX;
+    acb->max_dirty_data_idx = 0;
+
     return acb;
 }
 
@@ -595,14 +617,13 @@ static coroutine_fn int send_co_req(int sockfd, SheepdogReq *hdr, void *data,
     ret = qemu_co_send(sockfd, hdr, sizeof(*hdr));
     if (ret != sizeof(*hdr)) {
         error_report("failed to send a req, %s", strerror(errno));
-        ret = -socket_error();
-        return ret;
+        return -errno;
     }
 
     ret = qemu_co_send(sockfd, data, *wlen);
     if (ret != *wlen) {
-        ret = -socket_error();
         error_report("failed to send a req, %s", strerror(errno));
+        return -errno;
     }
 
     return ret;
@@ -638,14 +659,16 @@ static coroutine_fn void do_co_req(void *opaque)
     unsigned int *rlen = srco->rlen;
 
     co = qemu_coroutine_self();
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, restart_co_req, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, restart_co_req, co);
 
     ret = send_co_req(sockfd, hdr, data, wlen);
     if (ret < 0) {
         goto out;
     }
 
-    aio_set_fd_handler(srco->aio_context, sockfd, restart_co_req, NULL, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       restart_co_req, NULL, co);
 
     ret = qemu_co_recv(sockfd, hdr, sizeof(*hdr));
     if (ret != sizeof(*hdr)) {
@@ -670,7 +693,8 @@ static coroutine_fn void do_co_req(void *opaque)
 out:
     /* there is at most one request for this sockfd, so it is safe to
      * set each handler to NULL. */
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, NULL, NULL);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, NULL, NULL);
 
     srco->ret = ret;
     srco->finished = true;
@@ -722,7 +746,8 @@ static coroutine_fn void reconnect_to_sdog(void *opaque)
     BDRVSheepdogState *s = opaque;
     AIOReq *aio_req, *next;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
     close(s->fd);
     s->fd = -1;
 
@@ -819,8 +844,8 @@ static void coroutine_fn aio_read_response(void *opaque)
              */
             if (rsp.result == SD_RES_SUCCESS) {
                 s->inode.data_vdi_id[idx] = s->inode.vdi_id;
-                s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
-                s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
+                acb->max_dirty_data_idx = MAX(idx, acb->max_dirty_data_idx);
+                acb->min_dirty_data_idx = MIN(idx, acb->min_dirty_data_idx);
             }
         }
         break;
@@ -847,10 +872,6 @@ static void coroutine_fn aio_read_response(void *opaque)
             rsp.result = SD_RES_SUCCESS;
             s->discard_supported = false;
             break;
-        case SD_RES_SUCCESS:
-            idx = data_oid_to_idx(aio_req->oid);
-            s->inode.data_vdi_id[idx] = 0;
-            break;
         default:
             break;
         }
@@ -929,7 +950,8 @@ static int get_sheep_fd(BDRVSheepdogState *s, Error **errp)
         return fd;
     }
 
-    aio_set_fd_handler(s->aio_context, fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, fd, false,
+                       co_read_response, NULL, s);
     return fd;
 }
 
@@ -1165,7 +1187,13 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
         hdr.flags = SD_FLAG_CMD_WRITE | flags;
         break;
     case AIOCB_DISCARD_OBJ:
-        hdr.opcode = SD_OP_DISCARD_OBJ;
+        hdr.opcode = SD_OP_WRITE_OBJ;
+        hdr.flags = SD_FLAG_CMD_WRITE | flags;
+        s->inode.data_vdi_id[data_oid_to_idx(oid)] = 0;
+        offset = offsetof(SheepdogInode,
+                          data_vdi_id[data_oid_to_idx(oid)]);
+        oid = vid_to_vdi_oid(s->inode.vdi_id);
+        wlen = datalen = sizeof(uint32_t);
         break;
     }
 
@@ -1184,7 +1212,7 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
     qemu_co_mutex_lock(&s->lock);
     s->co_send = qemu_coroutine_self();
-    aio_set_fd_handler(s->aio_context, s->fd,
+    aio_set_fd_handler(s->aio_context, s->fd, false,
                        co_read_response, co_write_request, s);
     socket_set_cork(s->fd, 1);
 
@@ -1203,7 +1231,8 @@ static void coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
     }
 out:
     socket_set_cork(s->fd, 0);
-    aio_set_fd_handler(s->aio_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, s->fd, false,
+                       co_read_response, NULL, s);
     s->co_send = NULL;
     qemu_co_mutex_unlock(&s->lock);
 }
@@ -1353,7 +1382,8 @@ static void sd_detach_aio_context(BlockDriverState *bs)
 {
     BDRVSheepdogState *s = bs->opaque;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
 }
 
 static void sd_attach_aio_context(BlockDriverState *bs,
@@ -1362,7 +1392,8 @@ static void sd_attach_aio_context(BlockDriverState *bs,
     BDRVSheepdogState *s = bs->opaque;
 
     s->aio_context = new_context;
-    aio_set_fd_handler(new_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(new_context, s->fd, false,
+                       co_read_response, NULL, s);
 }
 
 /* TODO Convert to fine grained options */
@@ -1466,18 +1497,17 @@ static int sd_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     memcpy(&s->inode, buf, sizeof(s->inode));
-    s->min_dirty_data_idx = UINT32_MAX;
-    s->max_dirty_data_idx = 0;
 
     bs->total_sectors = s->inode.vdi_size / BDRV_SECTOR_SIZE;
     pstrcpy(s->name, sizeof(s->name), vdi);
     qemu_co_mutex_init(&s->lock);
-    qemu_co_queue_init(&s->overwrapping_queue);
+    qemu_co_queue_init(&s->overlapping_queue);
     qemu_opts_del(opts);
     g_free(buf);
     return 0;
 out:
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     if (s->fd >= 0) {
         closesocket(s->fd);
     }
@@ -1486,6 +1516,70 @@ out:
     return ret;
 }
 
+static int sd_reopen_prepare(BDRVReopenState *state, BlockReopenQueue *queue,
+                             Error **errp)
+{
+    BDRVSheepdogState *s = state->bs->opaque;
+    BDRVSheepdogReopenState *re_s;
+    int ret = 0;
+
+    re_s = state->opaque = g_new0(BDRVSheepdogReopenState, 1);
+
+    re_s->cache_flags = SD_FLAG_CMD_CACHE;
+    if (state->flags & BDRV_O_NOCACHE) {
+        re_s->cache_flags = SD_FLAG_CMD_DIRECT;
+    }
+
+    re_s->fd = get_sheep_fd(s, errp);
+    if (re_s->fd < 0) {
+        ret = re_s->fd;
+        return ret;
+    }
+
+    return ret;
+}
+
+static void sd_reopen_commit(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (s->fd) {
+        aio_set_fd_handler(s->aio_context, s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(s->fd);
+    }
+
+    s->fd = re_s->fd;
+    s->cache_flags = re_s->cache_flags;
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
+static void sd_reopen_abort(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (re_s == NULL) {
+        return;
+    }
+
+    if (re_s->fd) {
+        aio_set_fd_handler(s->aio_context, re_s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(re_s->fd);
+    }
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
 static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot,
                         Error **errp)
 {
@@ -1544,7 +1638,7 @@ static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot,
 
 static int sd_prealloc(const char *filename, Error **errp)
 {
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
     BDRVSheepdogState *base = NULL;
     unsigned long buf_size;
     uint32_t idx, max_idx;
@@ -1553,19 +1647,22 @@ static int sd_prealloc(const char *filename, Error **errp)
     void *buf = NULL;
     int ret;
 
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, errp);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, errp);
+    if (blk == NULL) {
+        ret = -EIO;
         goto out_with_err_set;
     }
 
-    vdi_size = bdrv_getlength(bs);
+    blk_set_allow_write_beyond_eof(blk, true);
+
+    vdi_size = blk_getlength(blk);
     if (vdi_size < 0) {
         ret = vdi_size;
         goto out;
     }
 
-    base = bs->opaque;
+    base = blk_bs(blk)->opaque;
     object_size = (UINT32_C(1) << base->inode.block_size_shift);
     buf_size = MIN(object_size, SD_DATA_OBJ_SIZE);
     buf = g_malloc0(buf_size);
@@ -1577,23 +1674,24 @@ static int sd_prealloc(const char *filename, Error **errp)
          * The created image can be a cloned image, so we need to read
          * a data from the source image.
          */
-        ret = bdrv_pread(bs, idx * buf_size, buf, buf_size);
+        ret = blk_pread(blk, idx * buf_size, buf, buf_size);
         if (ret < 0) {
             goto out;
         }
-        ret = bdrv_pwrite(bs, idx * buf_size, buf, buf_size);
+        ret = blk_pwrite(blk, idx * buf_size, buf, buf_size);
         if (ret < 0) {
             goto out;
         }
     }
 
+    ret = 0;
 out:
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Can't pre-allocate");
     }
 out_with_err_set:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     g_free(buf);
 
@@ -1733,7 +1831,7 @@ static int sd_create(const char *filename, QemuOpts *opts,
     }
 
     if (backing_file) {
-        BlockDriverState *bs;
+        BlockBackend *blk;
         BDRVSheepdogState *base;
         BlockDriver *drv;
 
@@ -1745,23 +1843,23 @@ static int sd_create(const char *filename, QemuOpts *opts,
             goto out;
         }
 
-        bs = NULL;
-        ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_PROTOCOL, NULL,
-                        errp);
-        if (ret < 0) {
+        blk = blk_new_open(backing_file, NULL, NULL,
+                           BDRV_O_PROTOCOL, errp);
+        if (blk == NULL) {
+            ret = -EIO;
             goto out;
         }
 
-        base = bs->opaque;
+        base = blk_bs(blk)->opaque;
 
         if (!is_snapshot(&base->inode)) {
             error_setg(errp, "cannot clone from a non snapshot vdi");
-            bdrv_unref(bs);
+            blk_unref(blk);
             ret = -EINVAL;
             goto out;
         }
         s->inode.vdi_id = base->inode.vdi_id;
-        bdrv_unref(bs);
+        blk_unref(blk);
     }
 
     s->aio_context = qemu_get_aio_context();
@@ -1776,8 +1874,7 @@ static int sd_create(const char *filename, QemuOpts *opts,
 
         fd = connect_to_sdog(s, &local_err);
         if (fd < 0) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
             ret = -EIO;
             goto out;
         }
@@ -1861,7 +1958,8 @@ static void sd_close(BlockDriverState *bs)
         error_report("%s, %s", sd_strerror(rsp->result), s->name);
     }
 
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     closesocket(s->fd);
     g_free(s->host_spec);
 }
@@ -1923,16 +2021,16 @@ static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
     AIOReq *aio_req;
     uint32_t offset, data_len, mn, mx;
 
-    mn = s->min_dirty_data_idx;
-    mx = s->max_dirty_data_idx;
+    mn = acb->min_dirty_data_idx;
+    mx = acb->max_dirty_data_idx;
     if (mn <= mx) {
         /* we need to update the vdi object. */
         offset = sizeof(s->inode) - sizeof(s->inode.data_vdi_id) +
             mn * sizeof(s->inode.data_vdi_id[0]);
         data_len = (mx - mn + 1) * sizeof(s->inode.data_vdi_id[0]);
 
-        s->min_dirty_data_idx = UINT32_MAX;
-        s->max_dirty_data_idx = 0;
+        acb->min_dirty_data_idx = UINT32_MAX;
+        acb->max_dirty_data_idx = 0;
 
         iov.iov_base = &s->inode;
         iov.iov_len = sizeof(s->inode);
@@ -2141,7 +2239,9 @@ static int coroutine_fn sd_co_rw_vector(void *p)
         }
 
         aio_req = alloc_aio_req(s, acb, oid, len, offset, flags, create,
-                                old_oid, done);
+                                old_oid,
+                                acb->aiocb_type == AIOCB_DISCARD_OBJ ?
+                                0 : done);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
 
         add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
@@ -2158,12 +2258,12 @@ out:
     return 1;
 }
 
-static bool check_overwrapping_aiocb(BDRVSheepdogState *s, SheepdogAIOCB *aiocb)
+static bool check_overlapping_aiocb(BDRVSheepdogState *s, SheepdogAIOCB *aiocb)
 {
     SheepdogAIOCB *cb;
 
     QLIST_FOREACH(cb, &s->inflight_aiocb_head, aiocb_siblings) {
-        if (AIOCBOverwrapping(aiocb, cb)) {
+        if (AIOCBOverlapping(aiocb, cb)) {
             return true;
         }
     }
@@ -2192,15 +2292,15 @@ static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
     acb->aiocb_type = AIOCB_WRITE_UDATA;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2208,7 +2308,7 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
 
     return acb->ret;
 }
@@ -2225,15 +2325,15 @@ static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
     acb->aio_done_func = sd_finish_aiocb;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2241,7 +2341,7 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
     return acb->ret;
 }
 
@@ -2318,9 +2418,8 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 
     ret = do_sd_create(s, &new_vid, 1, &local_err);
     if (ret < 0) {
-        error_report("failed to create inode for snapshot: %s",
-                     error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err,
+                          "failed to create inode for snapshot: ");
         goto cleanup;
     }
 
@@ -2391,13 +2490,131 @@ out:
     return ret;
 }
 
+#define NR_BATCHED_DISCARD 128
+
+static bool remove_objects(BDRVSheepdogState *s)
+{
+    int fd, i = 0, nr_objs = 0;
+    Error *local_err = NULL;
+    int ret = 0;
+    bool result = true;
+    SheepdogInode *inode = &s->inode;
+
+    fd = connect_to_sdog(s, &local_err);
+    if (fd < 0) {
+        error_report_err(local_err);
+        return false;
+    }
+
+    nr_objs = count_data_objs(inode);
+    while (i < nr_objs) {
+        int start_idx, nr_filled_idx;
+
+        while (i < nr_objs && !inode->data_vdi_id[i]) {
+            i++;
+        }
+        start_idx = i;
+
+        nr_filled_idx = 0;
+        while (i < nr_objs && nr_filled_idx < NR_BATCHED_DISCARD) {
+            if (inode->data_vdi_id[i]) {
+                inode->data_vdi_id[i] = 0;
+                nr_filled_idx++;
+            }
+
+            i++;
+        }
+
+        ret = write_object(fd, s->aio_context,
+                           (char *)&inode->data_vdi_id[start_idx],
+                           vid_to_vdi_oid(s->inode.vdi_id), inode->nr_copies,
+                           (i - start_idx) * sizeof(uint32_t),
+                           offsetof(struct SheepdogInode,
+                                    data_vdi_id[start_idx]),
+                           false, s->cache_flags);
+        if (ret < 0) {
+            error_report("failed to discard snapshot inode.");
+            result = false;
+            goto out;
+        }
+    }
+
+out:
+    closesocket(fd);
+    return result;
+}
+
 static int sd_snapshot_delete(BlockDriverState *bs,
                               const char *snapshot_id,
                               const char *name,
                               Error **errp)
 {
-    /* FIXME: Delete specified snapshot id.  */
-    return 0;
+    unsigned long snap_id = 0;
+    char snap_tag[SD_MAX_VDI_TAG_LEN];
+    Error *local_err = NULL;
+    int fd, ret;
+    char buf[SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN];
+    BDRVSheepdogState *s = bs->opaque;
+    unsigned int wlen = SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN, rlen = 0;
+    uint32_t vid;
+    SheepdogVdiReq hdr = {
+        .opcode = SD_OP_DEL_VDI,
+        .data_length = wlen,
+        .flags = SD_FLAG_CMD_WRITE,
+    };
+    SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
+
+    if (!remove_objects(s)) {
+        return -1;
+    }
+
+    memset(buf, 0, sizeof(buf));
+    memset(snap_tag, 0, sizeof(snap_tag));
+    pstrcpy(buf, SD_MAX_VDI_LEN, s->name);
+    ret = qemu_strtoul(snapshot_id, NULL, 10, &snap_id);
+    if (ret || snap_id > UINT32_MAX) {
+        error_setg(errp, "Invalid snapshot ID: %s",
+                         snapshot_id ? snapshot_id : "<null>");
+        return -EINVAL;
+    }
+
+    if (snap_id) {
+        hdr.snapid = (uint32_t) snap_id;
+    } else {
+        pstrcpy(snap_tag, sizeof(snap_tag), snapshot_id);
+        pstrcpy(buf + SD_MAX_VDI_LEN, SD_MAX_VDI_TAG_LEN, snap_tag);
+    }
+
+    ret = find_vdi_name(s, s->name, snap_id, snap_tag, &vid, true,
+                        &local_err);
+    if (ret) {
+        return ret;
+    }
+
+    fd = connect_to_sdog(s, &local_err);
+    if (fd < 0) {
+        error_report_err(local_err);
+        return -1;
+    }
+
+    ret = do_req(fd, s->aio_context, (SheepdogReq *)&hdr,
+                 buf, &wlen, &rlen);
+    closesocket(fd);
+    if (ret) {
+        return ret;
+    }
+
+    switch (rsp->result) {
+    case SD_RES_NO_VDI:
+        error_report("%s was already deleted", s->name);
+    case SD_RES_SUCCESS:
+        break;
+    default:
+        error_report("%s, %s", sd_strerror(rsp->result), s->name);
+        return -1;
+    }
+
+    return ret;
 }
 
 static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
@@ -2577,28 +2794,36 @@ static coroutine_fn int sd_co_discard(BlockDriverState *bs, int64_t sector_num,
                                       int nb_sectors)
 {
     SheepdogAIOCB *acb;
-    QEMUIOVector dummy;
     BDRVSheepdogState *s = bs->opaque;
     int ret;
+    QEMUIOVector discard_iov;
+    struct iovec iov;
+    uint32_t zero = 0;
 
     if (!s->discard_supported) {
             return 0;
     }
 
-    acb = sd_aio_setup(bs, &dummy, sector_num, nb_sectors);
+    memset(&discard_iov, 0, sizeof(discard_iov));
+    memset(&iov, 0, sizeof(iov));
+    iov.iov_base = &zero;
+    iov.iov_len = sizeof(zero);
+    discard_iov.iov = &iov;
+    discard_iov.niov = 1;
+    acb = sd_aio_setup(bs, &discard_iov, sector_num, nb_sectors);
     acb->aiocb_type = AIOCB_DISCARD_OBJ;
     acb->aio_done_func = sd_finish_aiocb;
 
 retry:
-    if (check_overwrapping_aiocb(s, acb)) {
-        qemu_co_queue_wait(&s->overwrapping_queue);
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
         goto retry;
     }
 
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
         QLIST_REMOVE(acb, aiocb_siblings);
-        qemu_co_queue_restart_all(&s->overwrapping_queue);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
@@ -2606,14 +2831,14 @@ retry:
     qemu_coroutine_yield();
 
     QLIST_REMOVE(acb, aiocb_siblings);
-    qemu_co_queue_restart_all(&s->overwrapping_queue);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
 
     return acb->ret;
 }
 
 static coroutine_fn int64_t
 sd_co_get_block_status(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-                       int *pnum)
+                       int *pnum, BlockDriverState **file)
 {
     BDRVSheepdogState *s = bs->opaque;
     SheepdogInode *inode = &s->inode;
@@ -2644,6 +2869,9 @@ sd_co_get_block_status(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
     if (*pnum > nb_sectors) {
         *pnum = nb_sectors;
     }
+    if (ret > 0 && ret & BDRV_BLOCK_OFFSET_VALID) {
+        *file = bs;
+    }
     return ret;
 }
 
@@ -2703,6 +2931,9 @@ static BlockDriver bdrv_sheepdog = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2736,6 +2967,9 @@ static BlockDriver bdrv_sheepdog_tcp = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2769,6 +3003,9 @@ static BlockDriver bdrv_sheepdog_unix = {
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
diff --git a/qemu/block/snapshot.c b/qemu/block/snapshot.c
index 49e143e99..e9d721df6 100644
--- a/qemu/block/snapshot.c
+++ b/qemu/block/snapshot.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/snapshot.h"
 #include "block/block_int.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 
 QemuOptsList internal_snapshot_opts = {
@@ -149,7 +151,7 @@ int bdrv_can_snapshot(BlockDriverState *bs)
 
     if (!drv->bdrv_snapshot_create) {
         if (bs->file != NULL) {
-            return bdrv_can_snapshot(bs->file);
+            return bdrv_can_snapshot(bs->file->bs);
         }
         return 0;
     }
@@ -168,7 +170,7 @@ int bdrv_snapshot_create(BlockDriverState *bs,
         return drv->bdrv_snapshot_create(bs, sn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_create(bs->file, sn_info);
+        return bdrv_snapshot_create(bs->file->bs, sn_info);
     }
     return -ENOTSUP;
 }
@@ -188,10 +190,10 @@ int bdrv_snapshot_goto(BlockDriverState *bs,
 
     if (bs->file) {
         drv->bdrv_close(bs);
-        ret = bdrv_snapshot_goto(bs->file, snapshot_id);
+        ret = bdrv_snapshot_goto(bs->file->bs, snapshot_id);
         open_ret = drv->bdrv_open(bs, NULL, bs->open_flags, NULL);
         if (open_ret < 0) {
-            bdrv_unref(bs->file);
+            bdrv_unref(bs->file->bs);
             bs->drv = NULL;
             return open_ret;
         }
@@ -229,6 +231,8 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
                          Error **errp)
 {
     BlockDriver *drv = bs->drv;
+    int ret;
+
     if (!drv) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
         return -ENOMEDIUM;
@@ -239,23 +243,26 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
     }
 
     /* drain all pending i/o before deleting snapshot */
-    bdrv_drain(bs);
+    bdrv_drained_begin(bs);
 
     if (drv->bdrv_snapshot_delete) {
-        return drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
-    }
-    if (bs->file) {
-        return bdrv_snapshot_delete(bs->file, snapshot_id, name, errp);
+        ret = drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
+    } else if (bs->file) {
+        ret = bdrv_snapshot_delete(bs->file->bs, snapshot_id, name, errp);
+    } else {
+        error_setg(errp, "Block format '%s' used by device '%s' "
+                   "does not support internal snapshot deletion",
+                   drv->format_name, bdrv_get_device_name(bs));
+        ret = -ENOTSUP;
     }
-    error_setg(errp, "Block format '%s' used by device '%s' "
-               "does not support internal snapshot deletion",
-               drv->format_name, bdrv_get_device_name(bs));
-    return -ENOTSUP;
+
+    bdrv_drained_end(bs);
+    return ret;
 }
 
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp)
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp)
 {
     int ret;
     Error *local_err = NULL;
@@ -270,6 +277,7 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
     if (ret < 0) {
         error_propagate(errp, local_err);
     }
+    return ret;
 }
 
 int bdrv_snapshot_list(BlockDriverState *bs,
@@ -283,7 +291,7 @@ int bdrv_snapshot_list(BlockDriverState *bs,
         return drv->bdrv_snapshot_list(bs, psn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_list(bs->file, psn_info);
+        return bdrv_snapshot_list(bs->file->bs, psn_info);
     }
     return -ENOTSUP;
 }
@@ -356,3 +364,130 @@ int bdrv_snapshot_load_tmp_by_id_or_name(BlockDriverState *bs,
 
     return ret;
 }
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers) */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs)
+{
+    bool ok = true;
+    BlockDriverState *bs = NULL;
+
+    while (ok && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_is_inserted(bs) && !bdrv_is_read_only(bs)) {
+            ok = bdrv_can_snapshot(bs);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return ok;
+}
+
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bad_bs,
+                             Error **err)
+{
+    int ret = 0;
+    BlockDriverState *bs = NULL;
+    QEMUSnapshotInfo sn1, *snapshot = &sn1;
+
+    while (ret == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs) &&
+                bdrv_snapshot_find(bs, snapshot, name) >= 0) {
+            ret = bdrv_snapshot_delete_by_id_or_name(bs, name, err);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return ret;
+}
+
+
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_goto(bs, name);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    QEMUSnapshotInfo sn;
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_find(bs, &sn, name);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs = NULL;
+
+    while (err == 0 && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bs == vm_state_bs) {
+            sn->vm_state_size = vm_state_size;
+            err = bdrv_snapshot_create(bs, sn);
+        } else if (bdrv_can_snapshot(bs)) {
+            sn->vm_state_size = 0;
+            err = bdrv_snapshot_create(bs, sn);
+        }
+        aio_context_release(ctx);
+    }
+
+    *first_bad_bs = bs;
+    return err;
+}
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void)
+{
+    bool not_found = true;
+    BlockDriverState *bs = NULL;
+
+    while (not_found && (bs = bdrv_next(bs))) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        not_found = !bdrv_can_snapshot(bs);
+        aio_context_release(ctx);
+    }
+    return bs;
+}
diff --git a/qemu/block/ssh.c b/qemu/block/ssh.c
index 8d0673903..06928ed93 100644
--- a/qemu/block/ssh.c
+++ b/qemu/block/ssh.c
@@ -22,14 +22,13 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include <libssh2.h>
 #include <libssh2_sftp.h>
 
 #include "block/block_int.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "qemu/uri.h"
@@ -193,7 +192,7 @@ sftp_error_report(BDRVSSHState *s, const char *fs, ...)
 static int parse_uri(const char *filename, QDict *options, Error **errp)
 {
     URI *uri = NULL;
-    QueryParams *qp = NULL;
+    QueryParams *qp;
     int i;
 
     uri = uri_parse(filename);
@@ -249,9 +248,6 @@ static int parse_uri(const char *filename, QDict *options, Error **errp)
     return 0;
 
  err:
-    if (qp) {
-      query_params_free(qp);
-    }
     if (uri) {
       uri_free(uri);
     }
@@ -803,14 +799,15 @@ static coroutine_fn void set_fd_handler(BDRVSSHState *s, BlockDriverState *bs)
             rd_handler, wr_handler);
 
     aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
-                       rd_handler, wr_handler, co);
+                       false, rd_handler, wr_handler, co);
 }
 
 static coroutine_fn void clear_fd_handler(BDRVSSHState *s,
                                           BlockDriverState *bs)
 {
     DPRINTF("s->sock=%d", s->sock);
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
+                       false, NULL, NULL, NULL);
 }
 
 /* A non-blocking call returned EAGAIN, so yield, ensuring the
diff --git a/qemu/block/stream.c b/qemu/block/stream.c
index ab0bd057f..332b9a183 100644
--- a/qemu/block/stream.c
+++ b/qemu/block/stream.c
@@ -11,11 +11,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -52,34 +55,6 @@ static int coroutine_fn stream_populate(BlockDriverState *bs,
     return bdrv_co_copy_on_readv(bs, sector_num, nb_sectors, &qiov);
 }
 
-static void close_unused_images(BlockDriverState *top, BlockDriverState *base,
-                                const char *base_id)
-{
-    BlockDriverState *intermediate;
-    intermediate = top->backing_hd;
-
-    /* Must assign before bdrv_delete() to prevent traversing dangling pointer
-     * while we delete backing image instances.
-     */
-    bdrv_set_backing_hd(top, base);
-
-    while (intermediate) {
-        BlockDriverState *unused;
-
-        /* reached base */
-        if (intermediate == base) {
-            break;
-        }
-
-        unused = intermediate;
-        intermediate = intermediate->backing_hd;
-        bdrv_set_backing_hd(unused, NULL);
-        bdrv_unref(unused);
-    }
-
-    bdrv_refresh_limits(top, NULL);
-}
-
 typedef struct {
     int ret;
     bool reached_end;
@@ -101,7 +76,7 @@ static void stream_complete(BlockJob *job, void *opaque)
             }
         }
         data->ret = bdrv_change_backing_file(job->bs, base_id, base_fmt);
-        close_unused_images(job->bs, base, base_id);
+        bdrv_set_backing_hd(job->bs, base);
     }
 
     g_free(s->backing_file_str);
@@ -115,21 +90,21 @@ static void coroutine_fn stream_run(void *opaque)
     StreamCompleteData *data;
     BlockDriverState *bs = s->common.bs;
     BlockDriverState *base = s->base;
-    int64_t sector_num, end;
+    int64_t sector_num = 0;
+    int64_t end = -1;
     int error = 0;
     int ret = 0;
     int n = 0;
     void *buf;
 
-    if (!bs->backing_hd) {
-        block_job_completed(&s->common, 0);
-        return;
+    if (!bs->backing) {
+        goto out;
     }
 
     s->common.len = bdrv_getlength(bs);
     if (s->common.len < 0) {
-        block_job_completed(&s->common, s->common.len);
-        return;
+        ret = s->common.len;
+        goto out;
     }
 
     end = s->common.len >> BDRV_SECTOR_BITS;
@@ -166,7 +141,7 @@ wait:
         } else if (ret >= 0) {
             /* Copy if allocated in the intermediate images.  Limit to the
              * known-unallocated area [sector_num, sector_num+n).  */
-            ret = bdrv_is_allocated_above(bs->backing_hd, base,
+            ret = bdrv_is_allocated_above(backing_bs(bs), base,
                                           sector_num, n, &n);
 
             /* Finish early if end of backing file has been reached */
@@ -216,6 +191,7 @@ wait:
 
     qemu_vfree(buf);
 
+out:
     /* Modify backing chain and close BDSes in main loop */
     data = g_malloc(sizeof(*data));
     data->ret = ret;
@@ -250,7 +226,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
 
     if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
          on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
+        (!bs->blk || !blk_iostatus_is_enabled(bs->blk))) {
         error_setg(errp, QERR_INVALID_PARAMETER, "on-error");
         return;
     }
diff --git a/qemu/block/throttle-groups.c b/qemu/block/throttle-groups.c
index 1abc6fcae..4920e0949 100644
--- a/qemu/block/throttle-groups.c
+++ b/qemu/block/throttle-groups.c
@@ -22,6 +22,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "block/throttle-groups.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
@@ -33,8 +34,7 @@
  * its own locking.
  *
  * This locking is however handled internally in this file, so it's
- * mostly transparent to outside users (but see the documentation in
- * throttle_groups_lock()).
+ * transparent to outside users.
  *
  * The whole ThrottleGroup structure is private and invisible to
  * outside users, that only use it through its ThrottleState.
@@ -76,9 +76,9 @@ static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
  * created.
  *
  * @name: the name of the ThrottleGroup
- * @ret:  the ThrottleGroup
+ * @ret:  the ThrottleState member of the ThrottleGroup
  */
-static ThrottleGroup *throttle_group_incref(const char *name)
+ThrottleState *throttle_group_incref(const char *name)
 {
     ThrottleGroup *tg = NULL;
     ThrottleGroup *iter;
@@ -108,7 +108,7 @@ static ThrottleGroup *throttle_group_incref(const char *name)
 
     qemu_mutex_unlock(&throttle_groups_lock);
 
-    return tg;
+    return &tg->ts;
 }
 
 /* Decrease the reference count of a ThrottleGroup.
@@ -116,10 +116,12 @@ static ThrottleGroup *throttle_group_incref(const char *name)
  * When the reference count reaches zero the ThrottleGroup is
  * destroyed.
  *
- * @tg:  The ThrottleGroup to unref
+ * @ts:  The ThrottleGroup to unref, given by its ThrottleState member
  */
-static void throttle_group_unref(ThrottleGroup *tg)
+void throttle_group_unref(ThrottleState *ts)
 {
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+
     qemu_mutex_lock(&throttle_groups_lock);
     if (--tg->refcount == 0) {
         QTAILQ_REMOVE(&throttle_groups, tg, list);
@@ -401,7 +403,8 @@ static void write_timer_cb(void *opaque)
 void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
 {
     int i;
-    ThrottleGroup *tg = throttle_group_incref(groupname);
+    ThrottleState *ts = throttle_group_incref(groupname);
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
     int clock_type = QEMU_CLOCK_REALTIME;
 
     if (qtest_enabled()) {
@@ -409,7 +412,7 @@ void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
         clock_type = QEMU_CLOCK_VIRTUAL;
     }
 
-    bs->throttle_state = &tg->ts;
+    bs->throttle_state = ts;
 
     qemu_mutex_lock(&tg->lock);
     /* If the ThrottleGroup is new set this BlockDriverState as the token */
@@ -435,6 +438,9 @@ void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
  * list, destroying the timers and setting the throttle_state pointer
  * to NULL.
  *
+ * The BlockDriverState must not have pending throttled requests, so
+ * the caller has to drain them first.
+ *
  * The group will be destroyed if it's empty after this operation.
  *
  * @bs: the BlockDriverState to remove
@@ -444,6 +450,10 @@ void throttle_group_unregister_bs(BlockDriverState *bs)
     ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
     int i;
 
+    assert(bs->pending_reqs[0] == 0 && bs->pending_reqs[1] == 0);
+    assert(qemu_co_queue_empty(&bs->throttled_reqs[0]));
+    assert(qemu_co_queue_empty(&bs->throttled_reqs[1]));
+
     qemu_mutex_lock(&tg->lock);
     for (i = 0; i < 2; i++) {
         if (tg->tokens[i] == bs) {
@@ -461,38 +471,10 @@ void throttle_group_unregister_bs(BlockDriverState *bs)
     throttle_timers_destroy(&bs->throttle_timers);
     qemu_mutex_unlock(&tg->lock);
 
-    throttle_group_unref(tg);
+    throttle_group_unref(&tg->ts);
     bs->throttle_state = NULL;
 }
 
-/* Acquire the lock of this throttling group.
- *
- * You won't normally need to use this. None of the functions from the
- * ThrottleGroup API require you to acquire the lock since all of them
- * deal with it internally.
- *
- * This should only be used in exceptional cases when you want to
- * access the protected fields of a BlockDriverState directly
- * (e.g. bdrv_swap()).
- *
- * @bs: a BlockDriverState that is member of the group
- */
-void throttle_group_lock(BlockDriverState *bs)
-{
-    ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
-    qemu_mutex_lock(&tg->lock);
-}
-
-/* Release the lock of this throttling group.
- *
- * See the comments in throttle_group_lock().
- */
-void throttle_group_unlock(BlockDriverState *bs)
-{
-    ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
-    qemu_mutex_unlock(&tg->lock);
-}
-
 static void throttle_groups_init(void)
 {
     qemu_mutex_init(&throttle_groups_lock);
diff --git a/qemu/block/vdi.c b/qemu/block/vdi.c
index 7642ef359..75d4819ed 100644
--- a/qemu/block/vdi.c
+++ b/qemu/block/vdi.c
@@ -49,11 +49,14 @@
  * so this seems to be reasonable.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
+#include "qemu/cutils.h"
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
@@ -399,7 +402,7 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
 
     logout("\n");
 
-    ret = bdrv_read(bs->file, 0, (uint8_t *)&header, 1);
+    ret = bdrv_read(bs->file->bs, 0, (uint8_t *)&header, 1);
     if (ret < 0) {
         goto fail;
     }
@@ -490,13 +493,14 @@ static int vdi_open(BlockDriverState *bs, QDict *options, int flags,
 
     bmap_size = header.blocks_in_image * sizeof(uint32_t);
     bmap_size = DIV_ROUND_UP(bmap_size, SECTOR_SIZE);
-    s->bmap = qemu_try_blockalign(bs->file, bmap_size * SECTOR_SIZE);
+    s->bmap = qemu_try_blockalign(bs->file->bs, bmap_size * SECTOR_SIZE);
     if (s->bmap == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
 
-    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
+    ret = bdrv_read(bs->file->bs, s->bmap_sector, (uint8_t *)s->bmap,
+                    bmap_size);
     if (ret < 0) {
         goto fail_free_bmap;
     }
@@ -525,7 +529,7 @@ static int vdi_reopen_prepare(BDRVReopenState *state,
 }
 
 static int64_t coroutine_fn vdi_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     /* TODO: Check for too large sector_num (in bdrv_is_allocated or here). */
     BDRVVdiState *s = (BDRVVdiState *)bs->opaque;
@@ -549,6 +553,7 @@ static int64_t coroutine_fn vdi_co_get_block_status(BlockDriverState *bs,
     offset = s->header.offset_data +
                               (uint64_t)bmap_entry * s->block_size +
                               sector_in_block * SECTOR_SIZE;
+    *file = bs->file->bs;
     return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | offset;
 }
 
@@ -585,7 +590,7 @@ static int vdi_co_read(BlockDriverState *bs,
             uint64_t offset = s->header.offset_data / SECTOR_SIZE +
                               (uint64_t)bmap_entry * s->block_sectors +
                               sector_in_block;
-            ret = bdrv_read(bs->file, offset, buf, n_sectors);
+            ret = bdrv_read(bs->file->bs, offset, buf, n_sectors);
         }
         logout("%u sectors read\n", n_sectors);
 
@@ -653,7 +658,7 @@ static int vdi_co_write(BlockDriverState *bs,
              * acquire the lock and thus the padded cluster is written before
              * the other coroutines can write to the affected area. */
             qemu_co_mutex_lock(&s->write_lock);
-            ret = bdrv_write(bs->file, offset, block, s->block_sectors);
+            ret = bdrv_write(bs->file->bs, offset, block, s->block_sectors);
             qemu_co_mutex_unlock(&s->write_lock);
         } else {
             uint64_t offset = s->header.offset_data / SECTOR_SIZE +
@@ -669,7 +674,7 @@ static int vdi_co_write(BlockDriverState *bs,
              * that that write operation has returned (there may be other writes
              * in flight, but they do not concern this very operation). */
             qemu_co_mutex_unlock(&s->write_lock);
-            ret = bdrv_write(bs->file, offset, buf, n_sectors);
+            ret = bdrv_write(bs->file->bs, offset, buf, n_sectors);
         }
 
         nb_sectors -= n_sectors;
@@ -694,7 +699,7 @@ static int vdi_co_write(BlockDriverState *bs,
         assert(VDI_IS_ALLOCATED(bmap_first));
         *header = s->header;
         vdi_header_to_le(header);
-        ret = bdrv_write(bs->file, 0, block, 1);
+        ret = bdrv_write(bs->file->bs, 0, block, 1);
         g_free(block);
         block = NULL;
 
@@ -712,7 +717,7 @@ static int vdi_co_write(BlockDriverState *bs,
         base = ((uint8_t *)&s->bmap[0]) + bmap_first * SECTOR_SIZE;
         logout("will write %u block map sectors starting from entry %u\n",
                n_sectors, bmap_first);
-        ret = bdrv_write(bs->file, offset, base, n_sectors);
+        ret = bdrv_write(bs->file->bs, offset, base, n_sectors);
     }
 
     return ret;
@@ -730,7 +735,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     size_t bmap_size;
     int64_t offset = 0;
     Error *local_err = NULL;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
     uint32_t *bmap = NULL;
 
     logout("\n");
@@ -763,13 +768,17 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
         error_propagate(errp, local_err);
         goto exit;
     }
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* We need enough blocks to store the given disk size,
        so always round up. */
     blocks = DIV_ROUND_UP(bytes, block_size);
@@ -799,7 +808,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     vdi_header_print(&header);
 #endif
     vdi_header_to_le(&header);
-    ret = bdrv_pwrite_sync(bs, offset, &header, sizeof(header));
+    ret = blk_pwrite(blk, offset, &header, sizeof(header));
     if (ret < 0) {
         error_setg(errp, "Error writing header to %s", filename);
         goto exit;
@@ -820,7 +829,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
                 bmap[i] = VDI_UNALLOCATED;
             }
         }
-        ret = bdrv_pwrite_sync(bs, offset, bmap, bmap_size);
+        ret = blk_pwrite(blk, offset, bmap, bmap_size);
         if (ret < 0) {
             error_setg(errp, "Error writing bmap to %s", filename);
             goto exit;
@@ -829,7 +838,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
     if (image_type == VDI_TYPE_STATIC) {
-        ret = bdrv_truncate(bs, offset + blocks * block_size);
+        ret = blk_truncate(blk, offset + blocks * block_size);
         if (ret < 0) {
             error_setg(errp, "Failed to statically allocate %s", filename);
             goto exit;
@@ -837,7 +846,7 @@ static int vdi_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
 exit:
-    bdrv_unref(bs);
+    blk_unref(blk);
     g_free(bmap);
     return ret;
 }
diff --git a/qemu/block/vhdx-endian.c b/qemu/block/vhdx-endian.c
index 0640d3f4a..da33cd38e 100644
--- a/qemu/block/vhdx-endian.c
+++ b/qemu/block/vhdx-endian.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "block/vhdx.h"
diff --git a/qemu/block/vhdx-log.c b/qemu/block/vhdx-log.c
index 47fec63c6..7ea7187fc 100644
--- a/qemu/block/vhdx-log.c
+++ b/qemu/block/vhdx-log.c
@@ -17,6 +17,8 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/error-report.h"
@@ -81,7 +83,7 @@ static int vhdx_log_peek_hdr(BlockDriverState *bs, VHDXLogEntries *log,
 
     offset = log->offset + read;
 
-    ret = bdrv_pread(bs->file, offset, hdr, sizeof(VHDXLogEntryHeader));
+    ret = bdrv_pread(bs->file->bs, offset, hdr, sizeof(VHDXLogEntryHeader));
     if (ret < 0) {
         goto exit;
     }
@@ -141,7 +143,7 @@ static int vhdx_log_read_sectors(BlockDriverState *bs, VHDXLogEntries *log,
         }
         offset = log->offset + read;
 
-        ret = bdrv_pread(bs->file, offset, buffer, VHDX_LOG_SECTOR_SIZE);
+        ret = bdrv_pread(bs->file->bs, offset, buffer, VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -191,7 +193,8 @@ static int vhdx_log_write_sectors(BlockDriverState *bs, VHDXLogEntries *log,
             /* full */
             break;
         }
-        ret = bdrv_pwrite(bs->file, offset, buffer_tmp, VHDX_LOG_SECTOR_SIZE);
+        ret = bdrv_pwrite(bs->file->bs, offset, buffer_tmp,
+                          VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -353,7 +356,7 @@ static int vhdx_log_read_desc(BlockDriverState *bs, BDRVVHDXState *s,
     }
 
     desc_sectors = vhdx_compute_desc_sectors(hdr.descriptor_count);
-    desc_entries = qemu_try_blockalign(bs->file,
+    desc_entries = qemu_try_blockalign(bs->file->bs,
                                        desc_sectors * VHDX_LOG_SECTOR_SIZE);
     if (desc_entries == NULL) {
         ret = -ENOMEM;
@@ -462,7 +465,7 @@ static int vhdx_log_flush_desc(BlockDriverState *bs, VHDXLogDescriptor *desc,
 
     /* count is only > 1 if we are writing zeroes */
     for (i = 0; i < count; i++) {
-        ret = bdrv_pwrite_sync(bs->file, file_offset, buffer,
+        ret = bdrv_pwrite_sync(bs->file->bs, file_offset, buffer,
                                VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
@@ -509,7 +512,7 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
         /* if the log shows a FlushedFileOffset larger than our current file
          * size, then that means the file has been truncated / corrupted, and
          * we must refused to open it / use it */
-        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file)) {
+        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file->bs)) {
             ret = -EINVAL;
             goto exit;
         }
@@ -539,12 +542,12 @@ static int vhdx_log_flush(BlockDriverState *bs, BDRVVHDXState *s,
                 goto exit;
             }
         }
-        if (bdrv_getlength(bs->file) < desc_entries->hdr.last_file_offset) {
+        if (bdrv_getlength(bs->file->bs) < desc_entries->hdr.last_file_offset) {
             new_file_size = desc_entries->hdr.last_file_offset;
             if (new_file_size % (1024*1024)) {
                 /* round up to nearest 1MB boundary */
                 new_file_size = ((new_file_size >> 20) + 1) << 20;
-                bdrv_truncate(bs->file, new_file_size);
+                bdrv_truncate(bs->file->bs, new_file_size);
             }
         }
         qemu_vfree(desc_entries);
@@ -783,12 +786,13 @@ int vhdx_parse_log(BlockDriverState *bs, BDRVVHDXState *s, bool *flushed,
     if (logs.valid) {
         if (bs->read_only) {
             ret = -EPERM;
-            error_setg_errno(errp, EPERM,
-                             "VHDX image file '%s' opened read-only, but "
-                             "contains a log that needs to be replayed.  To "
-                             "replay the log, execute:\n qemu-img check -r "
-                             "all '%s'",
-                             bs->filename, bs->filename);
+            error_setg(errp,
+                       "VHDX image file '%s' opened read-only, but "
+                       "contains a log that needs to be replayed",
+                       bs->filename);
+            error_append_hint(errp,  "To replay the log, run:\n"
+                              "qemu-img check -r all '%s'\n",
+                              bs->filename);
             goto exit;
         }
         /* now flush the log */
@@ -908,8 +912,8 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
                 .sequence_number     = s->log.sequence,
                 .descriptor_count    = sectors,
                 .reserved            = 0,
-                .flushed_file_offset = bdrv_getlength(bs->file),
-                .last_file_offset    = bdrv_getlength(bs->file),
+                .flushed_file_offset = bdrv_getlength(bs->file->bs),
+                .last_file_offset    = bdrv_getlength(bs->file->bs),
               };
 
     new_hdr.log_guid = header->log_guid;
@@ -940,7 +944,7 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
 
         if (i == 0 && leading_length) {
             /* partial sector at the front of the buffer */
-            ret = bdrv_pread(bs->file, file_offset, merged_sector,
+            ret = bdrv_pread(bs->file->bs, file_offset, merged_sector,
                              VHDX_LOG_SECTOR_SIZE);
             if (ret < 0) {
                 goto exit;
@@ -950,7 +954,7 @@ static int vhdx_log_write(BlockDriverState *bs, BDRVVHDXState *s,
             sector_write = merged_sector;
         } else if (i == sectors - 1 && trailing_length) {
             /* partial sector at the end of the buffer */
-            ret = bdrv_pread(bs->file,
+            ret = bdrv_pread(bs->file->bs,
                             file_offset,
                             merged_sector + trailing_length,
                             VHDX_LOG_SECTOR_SIZE - trailing_length);
diff --git a/qemu/block/vhdx.c b/qemu/block/vhdx.c
index 0776de717..2b7b33240 100644
--- a/qemu/block/vhdx.c
+++ b/qemu/block/vhdx.c
@@ -15,8 +15,11 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "qemu/crc32c.h"
 #include "block/vhdx.h"
@@ -263,10 +266,10 @@ static void vhdx_region_unregister_all(BDRVVHDXState *s)
 
 static void vhdx_set_shift_bits(BDRVVHDXState *s)
 {
-    s->logical_sector_size_bits = 31 - clz32(s->logical_sector_size);
-    s->sectors_per_block_bits =   31 - clz32(s->sectors_per_block);
-    s->chunk_ratio_bits =         63 - clz64(s->chunk_ratio);
-    s->block_size_bits =          31 - clz32(s->block_size);
+    s->logical_sector_size_bits = ctz32(s->logical_sector_size);
+    s->sectors_per_block_bits =   ctz32(s->sectors_per_block);
+    s->chunk_ratio_bits =         ctz64(s->chunk_ratio);
+    s->block_size_bits =          ctz32(s->block_size);
 }
 
 /*
@@ -375,7 +378,7 @@ static int vhdx_update_header(BlockDriverState *bs, BDRVVHDXState *s,
         inactive_header->log_guid = *log_guid;
     }
 
-    ret = vhdx_write_header(bs->file, inactive_header, header_offset, true);
+    ret = vhdx_write_header(bs->file->bs, inactive_header, header_offset, true);
     if (ret < 0) {
         goto exit;
     }
@@ -427,7 +430,8 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
     /* We have to read the whole VHDX_HEADER_SIZE instead of
      * sizeof(VHDXHeader), because the checksum is over the whole
      * region */
-    ret = bdrv_pread(bs->file, VHDX_HEADER1_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file->bs, VHDX_HEADER1_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -443,7 +447,8 @@ static void vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s,
         }
     }
 
-    ret = bdrv_pread(bs->file, VHDX_HEADER2_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file->bs, VHDX_HEADER2_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -516,7 +521,7 @@ static int vhdx_open_region_tables(BlockDriverState *bs, BDRVVHDXState *s)
      * whole block */
     buffer = qemu_blockalign(bs, VHDX_HEADER_BLOCK_SIZE);
 
-    ret = bdrv_pread(bs->file, VHDX_REGION_TABLE_OFFSET, buffer,
+    ret = bdrv_pread(bs->file->bs, VHDX_REGION_TABLE_OFFSET, buffer,
                      VHDX_HEADER_BLOCK_SIZE);
     if (ret < 0) {
         goto fail;
@@ -629,7 +634,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
 
     buffer = qemu_blockalign(bs, VHDX_METADATA_TABLE_MAX_SIZE);
 
-    ret = bdrv_pread(bs->file, s->metadata_rt.file_offset, buffer,
+    ret = bdrv_pread(bs->file->bs, s->metadata_rt.file_offset, buffer,
                      VHDX_METADATA_TABLE_MAX_SIZE);
     if (ret < 0) {
         goto exit;
@@ -732,7 +737,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
         goto exit;
     }
 
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.file_parameters_entry.offset
                                          + s->metadata_rt.file_offset,
                      &s->params,
@@ -767,7 +772,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     /* determine virtual disk size, logical sector size,
      * and phys sector size */
 
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.virtual_disk_size_entry.offset
                                            + s->metadata_rt.file_offset,
                      &s->virtual_disk_size,
@@ -775,7 +780,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     if (ret < 0) {
         goto exit;
     }
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.logical_sector_size_entry.offset
                                              + s->metadata_rt.file_offset,
                      &s->logical_sector_size,
@@ -783,7 +788,7 @@ static int vhdx_parse_metadata(BlockDriverState *bs, BDRVVHDXState *s)
     if (ret < 0) {
         goto exit;
     }
-    ret = bdrv_pread(bs->file,
+    ret = bdrv_pread(bs->file->bs,
                      s->metadata_entries.phys_sector_size_entry.offset
                                           + s->metadata_rt.file_offset,
                      &s->physical_sector_size,
@@ -854,14 +859,8 @@ static void vhdx_calc_bat_entries(BDRVVHDXState *s)
 {
     uint32_t data_blocks_cnt, bitmap_blocks_cnt;
 
-    data_blocks_cnt = s->virtual_disk_size >> s->block_size_bits;
-    if (s->virtual_disk_size - (data_blocks_cnt << s->block_size_bits)) {
-        data_blocks_cnt++;
-    }
-    bitmap_blocks_cnt = data_blocks_cnt >> s->chunk_ratio_bits;
-    if (data_blocks_cnt - (bitmap_blocks_cnt << s->chunk_ratio_bits)) {
-        bitmap_blocks_cnt++;
-    }
+    data_blocks_cnt = DIV_ROUND_UP(s->virtual_disk_size, s->block_size);
+    bitmap_blocks_cnt = DIV_ROUND_UP(data_blocks_cnt, s->chunk_ratio);
 
     if (s->parent_entries) {
         s->bat_entries = bitmap_blocks_cnt * (s->chunk_ratio + 1);
@@ -906,7 +905,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     QLIST_INIT(&s->regions);
 
     /* validate the file signature */
-    ret = bdrv_pread(bs->file, 0, &signature, sizeof(uint64_t));
+    ret = bdrv_pread(bs->file->bs, 0, &signature, sizeof(uint64_t));
     if (ret < 0) {
         goto fail;
     }
@@ -959,13 +958,13 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     /* s->bat is freed in vhdx_close() */
-    s->bat = qemu_try_blockalign(bs->file, s->bat_rt.length);
+    s->bat = qemu_try_blockalign(bs->file->bs, s->bat_rt.length);
     if (s->bat == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, s->bat_offset, s->bat, s->bat_rt.length);
+    ret = bdrv_pread(bs->file->bs, s->bat_offset, s->bat, s->bat_rt.length);
     if (ret < 0) {
         goto fail;
     }
@@ -1118,7 +1117,7 @@ static coroutine_fn int vhdx_co_readv(BlockDriverState *bs, int64_t sector_num,
                 break;
             case PAYLOAD_BLOCK_FULLY_PRESENT:
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->file,
+                ret = bdrv_co_readv(bs->file->bs,
                                     sinfo.file_offset >> BDRV_SECTOR_BITS,
                                     sinfo.sectors_avail, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
@@ -1156,12 +1155,12 @@ exit:
 static int vhdx_allocate_block(BlockDriverState *bs, BDRVVHDXState *s,
                                     uint64_t *new_offset)
 {
-    *new_offset = bdrv_getlength(bs->file);
+    *new_offset = bdrv_getlength(bs->file->bs);
 
     /* per the spec, the address for a block is in units of 1MB */
     *new_offset = ROUND_UP(*new_offset, 1024 * 1024);
 
-    return bdrv_truncate(bs->file, *new_offset + s->block_size);
+    return bdrv_truncate(bs->file->bs, *new_offset + s->block_size);
 }
 
 /*
@@ -1260,7 +1259,7 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,
                 /* Queue another write of zero buffers if the underlying file
                  * does not zero-fill on file extension */
 
-                if (bdrv_has_zero_init(bs->file) == 0) {
+                if (bdrv_has_zero_init(bs->file->bs) == 0) {
                     use_zero_buffers = true;
 
                     /* zero fill the front, if any */
@@ -1327,7 +1326,7 @@ static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,
                 }
                 /* block exists, so we can just overwrite it */
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_writev(bs->file,
+                ret = bdrv_co_writev(bs->file->bs,
                                     sinfo.file_offset >> BDRV_SECTOR_BITS,
                                     sectors_to_write, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
@@ -1454,7 +1453,7 @@ static int vhdx_create_new_metadata(BlockDriverState *bs,
     uint32_t offset = 0;
     void *buffer = NULL;
     void *entry_buffer;
-    VHDXMetadataTableHeader *md_table;;
+    VHDXMetadataTableHeader *md_table;
     VHDXMetadataTableEntry  *md_table_entry;
 
     /* Metadata entries */
@@ -1775,7 +1774,7 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
 
     gunichar2 *creator = NULL;
     glong creator_items;
-    BlockDriverState *bs;
+    BlockBackend *blk;
     char *type = NULL;
     VHDXImageType image_type;
     Error *local_err = NULL;
@@ -1840,14 +1839,16 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
         goto exit;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* Create (A) */
 
     /* The creator field is optional, but may be useful for
@@ -1855,13 +1856,13 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
     creator = g_utf8_to_utf16("QEMU v" QEMU_VERSION, -1, NULL,
                               &creator_items, NULL);
     signature = cpu_to_le64(VHDX_FILE_SIGNATURE);
-    ret = bdrv_pwrite(bs, VHDX_FILE_ID_OFFSET, &signature, sizeof(signature));
+    ret = blk_pwrite(blk, VHDX_FILE_ID_OFFSET, &signature, sizeof(signature));
     if (ret < 0) {
         goto delete_and_exit;
     }
     if (creator) {
-        ret = bdrv_pwrite(bs, VHDX_FILE_ID_OFFSET + sizeof(signature),
-                          creator, creator_items * sizeof(gunichar2));
+        ret = blk_pwrite(blk, VHDX_FILE_ID_OFFSET + sizeof(signature),
+                         creator, creator_items * sizeof(gunichar2));
         if (ret < 0) {
             goto delete_and_exit;
         }
@@ -1869,13 +1870,13 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
 
 
     /* Creates (B),(C) */
-    ret = vhdx_create_new_headers(bs, image_size, log_size);
+    ret = vhdx_create_new_headers(blk_bs(blk), image_size, log_size);
     if (ret < 0) {
         goto delete_and_exit;
     }
 
     /* Creates (D),(E),(G) explicitly. (F) created as by-product */
-    ret = vhdx_create_new_region_table(bs, image_size, block_size, 512,
+    ret = vhdx_create_new_region_table(blk_bs(blk), image_size, block_size, 512,
                                        log_size, use_zero_blocks, image_type,
                                        &metadata_offset);
     if (ret < 0) {
@@ -1883,7 +1884,7 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
     }
 
     /* Creates (H) */
-    ret = vhdx_create_new_metadata(bs, image_size, block_size, 512,
+    ret = vhdx_create_new_metadata(blk_bs(blk), image_size, block_size, 512,
                                    metadata_offset, image_type);
     if (ret < 0) {
         goto delete_and_exit;
@@ -1891,7 +1892,7 @@ static int vhdx_create(const char *filename, QemuOpts *opts, Error **errp)
 
 
 delete_and_exit:
-    bdrv_unref(bs);
+    blk_unref(blk);
 exit:
     g_free(type);
     g_free(creator);
diff --git a/qemu/block/vmdk.c b/qemu/block/vmdk.c
index fbaab67c8..45f9d3c5b 100644
--- a/qemu/block/vmdk.c
+++ b/qemu/block/vmdk.c
@@ -23,12 +23,15 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
+#include "qemu/cutils.h"
 #include <zlib.h>
 #include <glib.h>
 
@@ -87,7 +90,7 @@ typedef struct {
 #define L2_CACHE_SIZE 16
 
 typedef struct VmdkExtent {
-    BlockDriverState *file;
+    BdrvChild *file;
     bool flat;
     bool compressed;
     bool has_marker;
@@ -222,7 +225,7 @@ static void vmdk_free_extents(BlockDriverState *bs)
         g_free(e->l1_backup_table);
         g_free(e->type);
         if (e->file != bs->file) {
-            bdrv_unref(e->file);
+            bdrv_unref_child(bs, e->file);
         }
     }
     g_free(s->extents);
@@ -241,15 +244,17 @@ static void vmdk_free_last_extent(BlockDriverState *bs)
 
 static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent)
 {
-    char desc[DESC_SIZE];
+    char *desc;
     uint32_t cid = 0xffffffff;
     const char *p_name, *cid_str;
     size_t cid_str_size;
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    desc = g_malloc0(DESC_SIZE);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
+        g_free(desc);
         return 0;
     }
 
@@ -268,50 +273,55 @@ static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent)
         sscanf(p_name, "%" SCNx32, &cid);
     }
 
+    g_free(desc);
     return cid;
 }
 
 static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
 {
-    char desc[DESC_SIZE], tmp_desc[DESC_SIZE];
+    char *desc, *tmp_desc;
     char *p_name, *tmp_str;
     BDRVVmdkState *s = bs->opaque;
-    int ret;
+    int ret = 0;
 
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    desc = g_malloc0(DESC_SIZE);
+    tmp_desc = g_malloc0(DESC_SIZE);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
 
     desc[DESC_SIZE - 1] = '\0';
     tmp_str = strstr(desc, "parentCID");
     if (tmp_str == NULL) {
-        return -EINVAL;
+        ret = -EINVAL;
+        goto out;
     }
 
-    pstrcpy(tmp_desc, sizeof(tmp_desc), tmp_str);
+    pstrcpy(tmp_desc, DESC_SIZE, tmp_str);
     p_name = strstr(desc, "CID");
     if (p_name != NULL) {
         p_name += sizeof("CID");
-        snprintf(p_name, sizeof(desc) - (p_name - desc), "%" PRIx32 "\n", cid);
-        pstrcat(desc, sizeof(desc), tmp_desc);
+        snprintf(p_name, DESC_SIZE - (p_name - desc), "%" PRIx32 "\n", cid);
+        pstrcat(desc, DESC_SIZE, tmp_desc);
     }
 
-    ret = bdrv_pwrite_sync(bs->file, s->desc_offset, desc, DESC_SIZE);
-    if (ret < 0) {
-        return ret;
-    }
+    ret = bdrv_pwrite_sync(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
 
-    return 0;
+out:
+    g_free(desc);
+    g_free(tmp_desc);
+    return ret;
 }
 
 static int vmdk_is_cid_valid(BlockDriverState *bs)
 {
     BDRVVmdkState *s = bs->opaque;
-    BlockDriverState *p_bs = bs->backing_hd;
     uint32_t cur_pcid;
 
-    if (!s->cid_checked && p_bs) {
+    if (!s->cid_checked && bs->backing) {
+        BlockDriverState *p_bs = bs->backing->bs;
+
         cur_pcid = vmdk_read_cid(p_bs, 0);
         if (s->parent_cid != cur_pcid) {
             /* CID not valid */
@@ -335,15 +345,16 @@ static int vmdk_reopen_prepare(BDRVReopenState *state,
 static int vmdk_parent_open(BlockDriverState *bs)
 {
     char *p_name;
-    char desc[DESC_SIZE + 1];
+    char *desc;
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
-    desc[DESC_SIZE] = '\0';
-    ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
+    desc = g_malloc0(DESC_SIZE + 1);
+    ret = bdrv_pread(bs->file->bs, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
+    ret = 0;
 
     p_name = strstr(desc, "parentFileNameHint");
     if (p_name != NULL) {
@@ -352,22 +363,26 @@ static int vmdk_parent_open(BlockDriverState *bs)
         p_name += sizeof("parentFileNameHint") + 1;
         end_name = strchr(p_name, '\"');
         if (end_name == NULL) {
-            return -EINVAL;
+            ret = -EINVAL;
+            goto out;
         }
         if ((end_name - p_name) > sizeof(bs->backing_file) - 1) {
-            return -EINVAL;
+            ret = -EINVAL;
+            goto out;
         }
 
         pstrcpy(bs->backing_file, end_name - p_name + 1, p_name);
     }
 
-    return 0;
+out:
+    g_free(desc);
+    return ret;
 }
 
 /* Create and append extent to the extent array. Return the added VmdkExtent
  * address. return NULL if allocation failed. */
 static int vmdk_add_extent(BlockDriverState *bs,
-                           BlockDriverState *file, bool flat, int64_t sectors,
+                           BdrvChild *file, bool flat, int64_t sectors,
                            int64_t l1_offset, int64_t l1_backup_offset,
                            uint32_t l1_size,
                            int l2_size, uint64_t cluster_sectors,
@@ -392,7 +407,7 @@ static int vmdk_add_extent(BlockDriverState *bs,
         return -EFBIG;
     }
 
-    nb_sectors = bdrv_nb_sectors(file);
+    nb_sectors = bdrv_nb_sectors(file->bs);
     if (nb_sectors < 0) {
         return nb_sectors;
     }
@@ -439,14 +454,14 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(extent->file,
+    ret = bdrv_pread(extent->file->bs,
                      extent->l1_table_offset,
                      extent->l1_table,
                      l1_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read l1 table from extent '%s'",
-                         extent->file->filename);
+                         extent->file->bs->filename);
         goto fail_l1;
     }
     for (i = 0; i < extent->l1_size; i++) {
@@ -459,14 +474,14 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
             ret = -ENOMEM;
             goto fail_l1;
         }
-        ret = bdrv_pread(extent->file,
+        ret = bdrv_pread(extent->file->bs,
                          extent->l1_backup_table_offset,
                          extent->l1_backup_table,
                          l1_size);
         if (ret < 0) {
             error_setg_errno(errp, -ret,
                              "Could not read l1 backup table from extent '%s'",
-                             extent->file->filename);
+                             extent->file->bs->filename);
             goto fail_l1b;
         }
         for (i = 0; i < extent->l1_size; i++) {
@@ -485,7 +500,7 @@ static int vmdk_init_tables(BlockDriverState *bs, VmdkExtent *extent,
 }
 
 static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
-                                 BlockDriverState *file,
+                                 BdrvChild *file,
                                  int flags, Error **errp)
 {
     int ret;
@@ -493,11 +508,11 @@ static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
     VMDK3Header header;
     VmdkExtent *extent;
 
-    ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
+    ret = bdrv_pread(file->bs, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return ret;
     }
     ret = vmdk_add_extent(bs, file, false,
@@ -559,7 +574,7 @@ static char *vmdk_read_desc(BlockDriverState *file, uint64_t desc_offset,
 }
 
 static int vmdk_open_vmdk4(BlockDriverState *bs,
-                           BlockDriverState *file,
+                           BdrvChild *file,
                            int flags, QDict *options, Error **errp)
 {
     int ret;
@@ -569,18 +584,19 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     VmdkExtent *extent;
     BDRVVmdkState *s = bs->opaque;
     int64_t l1_backup_offset = 0;
+    bool compressed;
 
-    ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
+    ret = bdrv_pread(file->bs, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return -EINVAL;
     }
     if (header.capacity == 0) {
         uint64_t desc_offset = le64_to_cpu(header.desc_offset);
         if (desc_offset) {
-            char *buf = vmdk_read_desc(file, desc_offset << 9, errp);
+            char *buf = vmdk_read_desc(file->bs, desc_offset << 9, errp);
             if (!buf) {
                 return -EINVAL;
             }
@@ -620,8 +636,8 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
             } QEMU_PACKED eos_marker;
         } QEMU_PACKED footer;
 
-        ret = bdrv_pread(file,
-            bs->file->total_sectors * 512 - 1536,
+        ret = bdrv_pread(file->bs,
+            bs->file->bs->total_sectors * 512 - 1536,
             &footer, sizeof(footer));
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Failed to read footer");
@@ -643,14 +659,14 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
         header = footer.header;
     }
 
+    compressed =
+        le16_to_cpu(header.compressAlgorithm) == VMDK4_COMPRESSION_DEFLATE;
     if (le32_to_cpu(header.version) > 3) {
-        char buf[64];
-        snprintf(buf, sizeof(buf), "VMDK version %" PRId32,
-                 le32_to_cpu(header.version));
-        error_setg(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                   bdrv_get_device_or_node_name(bs), "vmdk", buf);
+        error_setg(errp, "Unsupported VMDK version %" PRIu32,
+                   le32_to_cpu(header.version));
         return -ENOTSUP;
-    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR)) {
+    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR) &&
+               !compressed) {
         /* VMware KB 2064959 explains that version 3 added support for
          * persistent changed block tracking (CBT), and backup software can
          * read it as version=1 if it doesn't care about the changed area
@@ -675,7 +691,7 @@ static int vmdk_open_vmdk4(BlockDriverState *bs,
     if (le32_to_cpu(header.flags) & VMDK4_FLAG_RGD) {
         l1_backup_offset = le64_to_cpu(header.rgd_offset) << 9;
     }
-    if (bdrv_nb_sectors(file) < le64_to_cpu(header.grain_offset)) {
+    if (bdrv_nb_sectors(file->bs) < le64_to_cpu(header.grain_offset)) {
         error_setg(errp, "File truncated, expecting at least %" PRId64 " bytes",
                    (int64_t)(le64_to_cpu(header.grain_offset)
                              * BDRV_SECTOR_SIZE));
@@ -739,8 +755,7 @@ static int vmdk_parse_description(const char *desc, const char *opt_name,
 }
 
 /* Open an extent file and append to bs array */
-static int vmdk_open_sparse(BlockDriverState *bs,
-                            BlockDriverState *file, int flags,
+static int vmdk_open_sparse(BlockDriverState *bs, BdrvChild *file, int flags,
                             char *buf, QDict *options, Error **errp)
 {
     uint32_t magic;
@@ -760,6 +775,17 @@ static int vmdk_open_sparse(BlockDriverState *bs,
     }
 }
 
+static const char *next_line(const char *s)
+{
+    while (*s) {
+        if (*s == '\n') {
+            return s + 1;
+        }
+        s++;
+    }
+    return s;
+}
+
 static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
                               const char *desc_file_path, QDict *options,
                               Error **errp)
@@ -769,16 +795,17 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
     char access[11];
     char type[11];
     char fname[512];
-    const char *p = desc;
+    const char *p, *np;
     int64_t sectors = 0;
     int64_t flat_offset;
     char *extent_path;
-    BlockDriverState *extent_file;
+    BdrvChild *extent_file;
     BDRVVmdkState *s = bs->opaque;
     VmdkExtent *extent;
     char extent_opt_prefix[32];
+    Error *local_err = NULL;
 
-    while (*p) {
+    for (p = desc; *p; p = next_line(p)) {
         /* parse extent line in one of below formats:
          *
          * RW [size in sectors] FLAT "file-name.vmdk" OFFSET
@@ -790,51 +817,48 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
         matches = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64,
                          access, &sectors, type, fname, &flat_offset);
         if (matches < 4 || strcmp(access, "RW")) {
-            goto next_line;
+            continue;
         } else if (!strcmp(type, "FLAT")) {
             if (matches != 5 || flat_offset < 0) {
-                error_setg(errp, "Invalid extent lines: \n%s", p);
-                return -EINVAL;
+                goto invalid;
             }
         } else if (!strcmp(type, "VMFS")) {
             if (matches == 4) {
                 flat_offset = 0;
             } else {
-                error_setg(errp, "Invalid extent lines:\n%s", p);
-                return -EINVAL;
+                goto invalid;
             }
         } else if (matches != 4) {
-            error_setg(errp, "Invalid extent lines:\n%s", p);
-            return -EINVAL;
+            goto invalid;
         }
 
         if (sectors <= 0 ||
             (strcmp(type, "FLAT") && strcmp(type, "SPARSE") &&
              strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE")) ||
             (strcmp(access, "RW"))) {
-            goto next_line;
+            continue;
         }
 
         if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
             !desc_file_path[0])
         {
             error_setg(errp, "Cannot use relative extent paths with VMDK "
-                       "descriptor file '%s'", bs->file->filename);
+                       "descriptor file '%s'", bs->file->bs->filename);
             return -EINVAL;
         }
 
         extent_path = g_malloc0(PATH_MAX);
         path_combine(extent_path, PATH_MAX, desc_file_path, fname);
-        extent_file = NULL;
 
         ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
         assert(ret < 32);
 
-        ret = bdrv_open_image(&extent_file, extent_path, options,
-                              extent_opt_prefix, bs, &child_file, false, errp);
+        extent_file = bdrv_open_child(extent_path, options, extent_opt_prefix,
+                                      bs, &child_file, false, &local_err);
         g_free(extent_path);
-        if (ret) {
-            return ret;
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return -EINVAL;
         }
 
         /* save to extents array */
@@ -844,13 +868,13 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             ret = vmdk_add_extent(bs, extent_file, true, sectors,
                             0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent->flat_start_offset = flat_offset << 9;
         } else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
             /* SPARSE extent and VMFSSPARSE extent are both "COWD" sparse file*/
-            char *buf = vmdk_read_desc(extent_file, 0, errp);
+            char *buf = vmdk_read_desc(extent_file->bs, 0, errp);
             if (!buf) {
                 ret = -EINVAL;
             } else {
@@ -859,27 +883,27 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
             }
             g_free(buf);
             if (ret) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent = &s->extents[s->num_extents - 1];
         } else {
             error_setg(errp, "Unsupported extent type '%s'", type);
-            bdrv_unref(extent_file);
+            bdrv_unref_child(bs, extent_file);
             return -ENOTSUP;
         }
         extent->type = g_strdup(type);
-next_line:
-        /* move to next line */
-        while (*p) {
-            if (*p == '\n') {
-                p++;
-                break;
-            }
-            p++;
-        }
     }
     return 0;
+
+invalid:
+    np = next_line(p);
+    assert(np != p);
+    if (np[-1] == '\n') {
+        np--;
+    }
+    error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
+    return -EINVAL;
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
@@ -905,7 +929,8 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
     }
     s->create_type = g_strdup(ct);
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->exact_filename, options, errp);
+    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
+                             errp);
 exit:
     return ret;
 }
@@ -918,7 +943,7 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
     BDRVVmdkState *s = bs->opaque;
     uint32_t magic;
 
-    buf = vmdk_read_desc(bs->file, 0, errp);
+    buf = vmdk_read_desc(bs->file->bs, 0, errp);
     if (!buf) {
         return -EINVAL;
     }
@@ -927,7 +952,8 @@ static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
     switch (magic) {
         case VMDK3_MAGIC:
         case VMDK4_MAGIC:
-            ret = vmdk_open_sparse(bs, bs->file, flags, buf, options, errp);
+            ret = vmdk_open_sparse(bs, bs->file, flags, buf, options,
+                                   errp);
             s->desc_offset = 0x200;
             break;
         default:
@@ -1004,7 +1030,7 @@ static int get_whole_cluster(BlockDriverState *bs,
     cluster_bytes = extent->cluster_sectors << BDRV_SECTOR_BITS;
     whole_grain = qemu_blockalign(bs, cluster_bytes);
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         memset(whole_grain, 0,  skip_start_sector << BDRV_SECTOR_BITS);
         memset(whole_grain + (skip_end_sector << BDRV_SECTOR_BITS), 0,
                cluster_bytes - (skip_end_sector << BDRV_SECTOR_BITS));
@@ -1013,22 +1039,22 @@ static int get_whole_cluster(BlockDriverState *bs,
     assert(skip_end_sector <= extent->cluster_sectors);
     /* we will be here if it's first write on non-exist grain(cluster).
      * try to read from parent image, if exist */
-    if (bs->backing_hd && !vmdk_is_cid_valid(bs)) {
+    if (bs->backing && !vmdk_is_cid_valid(bs)) {
         ret = VMDK_ERROR;
         goto exit;
     }
 
     /* Read backing data before skip range */
     if (skip_start_sector > 0) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num,
+        if (bs->backing) {
+            ret = bdrv_read(bs->backing->bs, sector_num,
                             whole_grain, skip_start_sector);
             if (ret < 0) {
                 ret = VMDK_ERROR;
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num, whole_grain,
+        ret = bdrv_write(extent->file->bs, cluster_sector_num, whole_grain,
                          skip_start_sector);
         if (ret < 0) {
             ret = VMDK_ERROR;
@@ -1037,8 +1063,8 @@ static int get_whole_cluster(BlockDriverState *bs,
     }
     /* Read backing data after skip range */
     if (skip_end_sector < extent->cluster_sectors) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num + skip_end_sector,
+        if (bs->backing) {
+            ret = bdrv_read(bs->backing->bs, sector_num + skip_end_sector,
                             whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
                             extent->cluster_sectors - skip_end_sector);
             if (ret < 0) {
@@ -1046,7 +1072,7 @@ static int get_whole_cluster(BlockDriverState *bs,
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num + skip_end_sector,
+        ret = bdrv_write(extent->file->bs, cluster_sector_num + skip_end_sector,
                          whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
                          extent->cluster_sectors - skip_end_sector);
         if (ret < 0) {
@@ -1066,7 +1092,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
     offset = cpu_to_le32(offset);
     /* update L2 table */
     if (bdrv_pwrite_sync(
-                extent->file,
+                extent->file->bs,
                 ((int64_t)m_data->l2_offset * 512)
                     + (m_data->l2_index * sizeof(offset)),
                 &offset, sizeof(offset)) < 0) {
@@ -1076,7 +1102,7 @@ static int vmdk_L2update(VmdkExtent *extent, VmdkMetaData *m_data,
     if (extent->l1_backup_table_offset != 0) {
         m_data->l2_offset = extent->l1_backup_table[m_data->l1_index];
         if (bdrv_pwrite_sync(
-                    extent->file,
+                    extent->file->bs,
                     ((int64_t)m_data->l2_offset * 512)
                         + (m_data->l2_index * sizeof(offset)),
                     &offset, sizeof(offset)) < 0) {
@@ -1166,7 +1192,7 @@ static int get_cluster_offset(BlockDriverState *bs,
     }
     l2_table = extent->l2_cache + (min_index * extent->l2_size);
     if (bdrv_pread(
-                extent->file,
+                extent->file->bs,
                 (int64_t)l2_offset * 512,
                 l2_table,
                 extent->l2_size * sizeof(uint32_t)
@@ -1245,7 +1271,7 @@ static inline uint64_t vmdk_find_index_in_cluster(VmdkExtent *extent,
 }
 
 static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVVmdkState *s = bs->opaque;
     int64_t index_in_cluster, n, ret;
@@ -1262,6 +1288,7 @@ static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
                              0, 0);
     qemu_co_mutex_unlock(&s->lock);
 
+    index_in_cluster = vmdk_find_index_in_cluster(extent, sector_num);
     switch (ret) {
     case VMDK_ERROR:
         ret = -EIO;
@@ -1274,14 +1301,15 @@ static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
         break;
     case VMDK_OK:
         ret = BDRV_BLOCK_DATA;
-        if (extent->file == bs->file && !extent->compressed) {
-            ret |= BDRV_BLOCK_OFFSET_VALID | offset;
+        if (!extent->compressed) {
+            ret |= BDRV_BLOCK_OFFSET_VALID;
+            ret |= (offset + (index_in_cluster << BDRV_SECTOR_BITS))
+                    & BDRV_BLOCK_OFFSET_MASK;
         }
-
+        *file = extent->file->bs;
         break;
     }
 
-    index_in_cluster = vmdk_find_index_in_cluster(extent, sector_num);
     n = extent->cluster_sectors - index_in_cluster;
     if (n > nb_sectors) {
         n = nb_sectors;
@@ -1320,12 +1348,16 @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
         write_len = buf_len + sizeof(VmdkGrainMarker);
     }
     write_offset = cluster_offset + offset_in_cluster,
-    ret = bdrv_pwrite(extent->file, write_offset, write_buf, write_len);
+    ret = bdrv_pwrite(extent->file->bs, write_offset, write_buf, write_len);
 
     write_end_sector = DIV_ROUND_UP(write_offset + write_len, BDRV_SECTOR_SIZE);
 
-    extent->next_cluster_sector = MAX(extent->next_cluster_sector,
-                                      write_end_sector);
+    if (extent->compressed) {
+        extent->next_cluster_sector = write_end_sector;
+    } else {
+        extent->next_cluster_sector = MAX(extent->next_cluster_sector,
+                                          write_end_sector);
+    }
 
     if (ret != write_len) {
         ret = ret < 0 ? ret : -EIO;
@@ -1351,7 +1383,7 @@ static int vmdk_read_extent(VmdkExtent *extent, int64_t cluster_offset,
 
 
     if (!extent->compressed) {
-        ret = bdrv_pread(extent->file,
+        ret = bdrv_pread(extent->file->bs,
                           cluster_offset + offset_in_cluster,
                           buf, nb_sectors * 512);
         if (ret == nb_sectors * 512) {
@@ -1365,7 +1397,7 @@ static int vmdk_read_extent(VmdkExtent *extent, int64_t cluster_offset,
     buf_bytes = cluster_bytes * 2;
     cluster_buf = g_malloc(buf_bytes);
     uncomp_buf = g_malloc(cluster_bytes);
-    ret = bdrv_pread(extent->file,
+    ret = bdrv_pread(extent->file->bs,
                 cluster_offset,
                 cluster_buf, buf_bytes);
     if (ret < 0) {
@@ -1427,11 +1459,11 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
         }
         if (ret != VMDK_OK) {
             /* if not allocated, try to read from parent image, if exist */
-            if (bs->backing_hd && ret != VMDK_ZEROED) {
+            if (bs->backing && ret != VMDK_ZEROED) {
                 if (!vmdk_is_cid_valid(bs)) {
                     return -EINVAL;
                 }
-                ret = bdrv_read(bs->backing_hd, sector_num, buf, n);
+                ret = bdrv_read(bs->backing->bs, sector_num, buf, n);
                 if (ret < 0) {
                     return ret;
                 }
@@ -1487,8 +1519,8 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
 
     if (sector_num > bs->total_sectors) {
         error_report("Wrong offset: sector_num=0x%" PRIx64
-                " total_sectors=0x%" PRIx64 "\n",
-                sector_num, bs->total_sectors);
+                     " total_sectors=0x%" PRIx64,
+                     sector_num, bs->total_sectors);
         return -EIO;
     }
 
@@ -1617,7 +1649,7 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
                               QemuOpts *opts, Error **errp)
 {
     int ret, i;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
     VMDK4Header header;
     Error *local_err = NULL;
     uint32_t tmp, magic, grains, gd_sectors, gt_size, gt_count;
@@ -1630,16 +1662,18 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
         goto exit;
     }
 
-    assert(bs == NULL);
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     if (flat) {
-        ret = bdrv_truncate(bs, filesize);
+        ret = blk_truncate(blk, filesize);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not truncate file");
         }
@@ -1647,7 +1681,13 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
     }
     magic = cpu_to_be32(VMDK4_MAGIC);
     memset(&header, 0, sizeof(header));
-    header.version = zeroed_grain ? 2 : 1;
+    if (compress) {
+        header.version = 3;
+    } else if (zeroed_grain) {
+        header.version = 2;
+    } else {
+        header.version = 1;
+    }
     header.flags = VMDK4_FLAG_RGD | VMDK4_FLAG_NL_DETECT
                    | (compress ? VMDK4_FLAG_COMPRESS | VMDK4_FLAG_MARKER : 0)
                    | (zeroed_grain ? VMDK4_FLAG_ZERO_GRAIN : 0);
@@ -1688,18 +1728,18 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
     header.check_bytes[3] = 0xa;
 
     /* write all the data */
-    ret = bdrv_pwrite(bs, 0, &magic, sizeof(magic));
+    ret = blk_pwrite(blk, 0, &magic, sizeof(magic));
     if (ret < 0) {
         error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
-    ret = bdrv_pwrite(bs, sizeof(magic), &header, sizeof(header));
+    ret = blk_pwrite(blk, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
 
-    ret = bdrv_truncate(bs, le64_to_cpu(header.grain_offset) << 9);
+    ret = blk_truncate(blk, le64_to_cpu(header.grain_offset) << 9);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not truncate file");
         goto exit;
@@ -1712,8 +1752,8 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
          i < gt_count; i++, tmp += gt_size) {
         gd_buf[i] = cpu_to_le32(tmp);
     }
-    ret = bdrv_pwrite(bs, le64_to_cpu(header.rgd_offset) * BDRV_SECTOR_SIZE,
-                      gd_buf, gd_buf_size);
+    ret = blk_pwrite(blk, le64_to_cpu(header.rgd_offset) * BDRV_SECTOR_SIZE,
+                     gd_buf, gd_buf_size);
     if (ret < 0) {
         error_setg(errp, QERR_IO_ERROR);
         goto exit;
@@ -1724,8 +1764,8 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
          i < gt_count; i++, tmp += gt_size) {
         gd_buf[i] = cpu_to_le32(tmp);
     }
-    ret = bdrv_pwrite(bs, le64_to_cpu(header.gd_offset) * BDRV_SECTOR_SIZE,
-                      gd_buf, gd_buf_size);
+    ret = blk_pwrite(blk, le64_to_cpu(header.gd_offset) * BDRV_SECTOR_SIZE,
+                     gd_buf, gd_buf_size);
     if (ret < 0) {
         error_setg(errp, QERR_IO_ERROR);
         goto exit;
@@ -1733,8 +1773,8 @@ static int vmdk_create_extent(const char *filename, int64_t filesize,
 
     ret = 0;
 exit:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     g_free(gd_buf);
     return ret;
@@ -1783,7 +1823,7 @@ static int filename_decompose(const char *filename, char *path, char *prefix,
 static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
 {
     int idx = 0;
-    BlockDriverState *new_bs = NULL;
+    BlockBackend *new_blk = NULL;
     Error *local_err = NULL;
     char *desc = NULL;
     int64_t total_size = 0, filesize;
@@ -1894,7 +1934,7 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
         goto exit;
     }
     if (backing_file) {
-        BlockDriverState *bs = NULL;
+        BlockBackend *blk;
         char *full_backing = g_new0(char, PATH_MAX);
         bdrv_get_full_backing_filename_from_filename(filename, backing_file,
                                                      full_backing, PATH_MAX,
@@ -1905,19 +1945,21 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
             ret = -ENOENT;
             goto exit;
         }
-        ret = bdrv_open(&bs, full_backing, NULL, NULL, BDRV_O_NO_BACKING, NULL,
-                        errp);
+
+        blk = blk_new_open(full_backing, NULL, NULL,
+                           BDRV_O_NO_BACKING, errp);
         g_free(full_backing);
-        if (ret != 0) {
+        if (blk == NULL) {
+            ret = -EIO;
             goto exit;
         }
-        if (strcmp(bs->drv->format_name, "vmdk")) {
-            bdrv_unref(bs);
+        if (strcmp(blk_bs(blk)->drv->format_name, "vmdk")) {
+            blk_unref(blk);
             ret = -EINVAL;
             goto exit;
         }
-        parent_cid = vmdk_read_cid(bs, 0);
-        bdrv_unref(bs);
+        parent_cid = vmdk_read_cid(blk_bs(blk), 0);
+        blk_unref(blk);
         snprintf(parent_desc_line, BUF_SIZE,
                 "parentFileNameHint=\"%s\"", backing_file);
     }
@@ -1975,14 +2017,18 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
             goto exit;
         }
     }
-    assert(new_bs == NULL);
-    ret = bdrv_open(&new_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
+
+    new_blk = blk_new_open(filename, NULL, NULL,
+                           BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (new_blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
-    ret = bdrv_pwrite(new_bs, desc_offset, desc, desc_len);
+
+    blk_set_allow_write_beyond_eof(new_blk, true);
+
+    ret = blk_pwrite(new_blk, desc_offset, desc, desc_len);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not write description");
         goto exit;
@@ -1990,14 +2036,14 @@ static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
     /* bdrv_pwrite write padding zeros to align to sector, we don't need that
      * for description file */
     if (desc_offset == 0) {
-        ret = bdrv_truncate(new_bs, desc_len);
+        ret = blk_truncate(new_blk, desc_len);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not truncate file");
         }
     }
 exit:
-    if (new_bs) {
-        bdrv_unref(new_bs);
+    if (new_blk) {
+        blk_unref(new_blk);
     }
     g_free(adapter_type);
     g_free(backing_file);
@@ -2032,7 +2078,7 @@ static coroutine_fn int vmdk_co_flush(BlockDriverState *bs)
     int ret = 0;
 
     for (i = 0; i < s->num_extents; i++) {
-        err = bdrv_co_flush(s->extents[i].file);
+        err = bdrv_co_flush(s->extents[i].file->bs);
         if (err < 0) {
             ret = err;
         }
@@ -2047,7 +2093,7 @@ static int64_t vmdk_get_allocated_file_size(BlockDriverState *bs)
     int64_t r;
     BDRVVmdkState *s = bs->opaque;
 
-    ret = bdrv_get_allocated_file_size(bs->file);
+    ret = bdrv_get_allocated_file_size(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -2055,7 +2101,7 @@ static int64_t vmdk_get_allocated_file_size(BlockDriverState *bs)
         if (s->extents[i].file == bs->file) {
             continue;
         }
-        r = bdrv_get_allocated_file_size(s->extents[i].file);
+        r = bdrv_get_allocated_file_size(s->extents[i].file->bs);
         if (r < 0) {
             return r;
         }
@@ -2073,7 +2119,7 @@ static int vmdk_has_zero_init(BlockDriverState *bs)
      * return 0. */
     for (i = 0; i < s->num_extents; i++) {
         if (s->extents[i].flat) {
-            if (!bdrv_has_zero_init(s->extents[i].file)) {
+            if (!bdrv_has_zero_init(s->extents[i].file->bs)) {
                 return 0;
             }
         }
@@ -2086,7 +2132,7 @@ static ImageInfo *vmdk_get_extent_info(VmdkExtent *extent)
     ImageInfo *info = g_new0(ImageInfo, 1);
 
     *info = (ImageInfo){
-        .filename         = g_strdup(extent->file->filename),
+        .filename         = g_strdup(extent->file->bs->filename),
         .format           = g_strdup(extent->type),
         .virtual_size     = extent->sectors * BDRV_SECTOR_SIZE,
         .compressed       = extent->compressed,
@@ -2132,7 +2178,9 @@ static int vmdk_check(BlockDriverState *bs, BdrvCheckResult *result,
                     PRId64 "\n", sector_num);
             break;
         }
-        if (ret == VMDK_OK && cluster_offset >= bdrv_getlength(extent->file)) {
+        if (ret == VMDK_OK &&
+            cluster_offset >= bdrv_getlength(extent->file->bs))
+        {
             fprintf(stderr,
                     "ERROR: cluster offset for sector %"
                     PRId64 " points after EOF\n", sector_num);
@@ -2153,19 +2201,19 @@ static ImageInfoSpecific *vmdk_get_specific_info(BlockDriverState *bs)
     ImageInfoList **next;
 
     *spec_info = (ImageInfoSpecific){
-        .kind = IMAGE_INFO_SPECIFIC_KIND_VMDK,
-        {
-            .vmdk = g_new0(ImageInfoSpecificVmdk, 1),
+        .type = IMAGE_INFO_SPECIFIC_KIND_VMDK,
+        .u = {
+            .vmdk.data = g_new0(ImageInfoSpecificVmdk, 1),
         },
     };
 
-    *spec_info->vmdk = (ImageInfoSpecificVmdk) {
+    *spec_info->u.vmdk.data = (ImageInfoSpecificVmdk) {
         .create_type = g_strdup(s->create_type),
         .cid = s->cid,
         .parent_cid = s->parent_cid,
     };
 
-    next = &spec_info->vmdk->extents;
+    next = &spec_info->u.vmdk.data->extents;
     for (i = 0; i < s->num_extents; i++) {
         *next = g_new0(ImageInfoList, 1);
         (*next)->value = vmdk_get_extent_info(&s->extents[i]);
@@ -2208,7 +2256,7 @@ static void vmdk_detach_aio_context(BlockDriverState *bs)
     int i;
 
     for (i = 0; i < s->num_extents; i++) {
-        bdrv_detach_aio_context(s->extents[i].file);
+        bdrv_detach_aio_context(s->extents[i].file->bs);
     }
 }
 
@@ -2219,7 +2267,7 @@ static void vmdk_attach_aio_context(BlockDriverState *bs,
     int i;
 
     for (i = 0; i < s->num_extents; i++) {
-        bdrv_attach_aio_context(s->extents[i].file, new_context);
+        bdrv_attach_aio_context(s->extents[i].file->bs, new_context);
     }
 }
 
diff --git a/qemu/block/vpc.c b/qemu/block/vpc.c
index 3e385d9fb..3e2ea698d 100644
--- a/qemu/block/vpc.c
+++ b/qemu/block/vpc.c
@@ -22,8 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
 #if defined(CONFIG_UUID)
@@ -42,28 +45,34 @@ enum vhd_type {
     VHD_DIFFERENCING    = 4,
 };
 
-// Seconds since Jan 1, 2000 0:00:00 (UTC)
+/* Seconds since Jan 1, 2000 0:00:00 (UTC) */
 #define VHD_TIMESTAMP_BASE 946684800
 
-#define VHD_MAX_SECTORS       (65535LL * 255 * 255)
-#define VHD_MAX_GEOMETRY      (65535LL *  16 * 255)
+#define VHD_CHS_MAX_C   65535LL
+#define VHD_CHS_MAX_H   16
+#define VHD_CHS_MAX_S   255
 
-// always big-endian
+#define VHD_MAX_SECTORS       0xff000000    /* 2040 GiB max image size */
+#define VHD_MAX_GEOMETRY      (VHD_CHS_MAX_C * VHD_CHS_MAX_H * VHD_CHS_MAX_S)
+
+#define VPC_OPT_FORCE_SIZE "force_size"
+
+/* always big-endian */
 typedef struct vhd_footer {
-    char        creator[8]; // "conectix"
+    char        creator[8]; /* "conectix" */
     uint32_t    features;
     uint32_t    version;
 
-    // Offset of next header structure, 0xFFFFFFFF if none
+    /* Offset of next header structure, 0xFFFFFFFF if none */
     uint64_t    data_offset;
 
-    // Seconds since Jan 1, 2000 0:00:00 (UTC)
+    /* Seconds since Jan 1, 2000 0:00:00 (UTC) */
     uint32_t    timestamp;
 
-    char        creator_app[4]; // "vpc "
+    char        creator_app[4]; /*  e.g., "vpc " */
     uint16_t    major;
     uint16_t    minor;
-    char        creator_os[4]; // "Wi2k"
+    char        creator_os[4]; /* "Wi2k" */
 
     uint64_t    orig_size;
     uint64_t    current_size;
@@ -74,29 +83,29 @@ typedef struct vhd_footer {
 
     uint32_t    type;
 
-    // Checksum of the Hard Disk Footer ("one's complement of the sum of all
-    // the bytes in the footer without the checksum field")
+    /* Checksum of the Hard Disk Footer ("one's complement of the sum of all
+       the bytes in the footer without the checksum field") */
     uint32_t    checksum;
 
-    // UUID used to identify a parent hard disk (backing file)
+    /* UUID used to identify a parent hard disk (backing file) */
     uint8_t     uuid[16];
 
     uint8_t     in_saved_state;
 } QEMU_PACKED VHDFooter;
 
 typedef struct vhd_dyndisk_header {
-    char        magic[8]; // "cxsparse"
+    char        magic[8]; /* "cxsparse" */
 
-    // Offset of next header structure, 0xFFFFFFFF if none
+    /* Offset of next header structure, 0xFFFFFFFF if none */
     uint64_t    data_offset;
 
-    // Offset of the Block Allocation Table (BAT)
+    /* Offset of the Block Allocation Table (BAT) */
     uint64_t    table_offset;
 
     uint32_t    version;
-    uint32_t    max_table_entries; // 32bit/entry
+    uint32_t    max_table_entries; /* 32bit/entry */
 
-    // 2 MB by default, must be a power of two
+    /* 2 MB by default, must be a power of two */
     uint32_t    block_size;
 
     uint32_t    checksum;
@@ -104,7 +113,7 @@ typedef struct vhd_dyndisk_header {
     uint32_t    parent_timestamp;
     uint32_t    reserved;
 
-    // Backing file name (in UTF-16)
+    /* Backing file name (in UTF-16) */
     uint8_t     parent_name[512];
 
     struct {
@@ -127,6 +136,8 @@ typedef struct BDRVVPCState {
 
     uint32_t block_size;
     uint32_t bitmap_size;
+    bool force_use_chs;
+    bool force_use_sz;
 
 #ifdef CACHE
     uint8_t *pageentry_u8;
@@ -139,6 +150,22 @@ typedef struct BDRVVPCState {
     Error *migration_blocker;
 } BDRVVPCState;
 
+#define VPC_OPT_SIZE_CALC "force_size_calc"
+static QemuOptsList vpc_runtime_opts = {
+    .name = "vpc-runtime-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(vpc_runtime_opts.head),
+    .desc = {
+        {
+            .name = VPC_OPT_SIZE_CALC,
+            .type = QEMU_OPT_STRING,
+            .help = "Force disk size calculation to use either CHS geometry, "
+                    "or use the disk current_size specified in the VHD footer. "
+                    "{chs, current_size}"
+        },
+        { /* end of list */ }
+    }
+};
+
 static uint32_t vpc_checksum(uint8_t* buf, size_t size)
 {
     uint32_t res = 0;
@@ -158,6 +185,25 @@ static int vpc_probe(const uint8_t *buf, int buf_size, const char *filename)
     return 0;
 }
 
+static void vpc_parse_options(BlockDriverState *bs, QemuOpts *opts,
+                              Error **errp)
+{
+    BDRVVPCState *s = bs->opaque;
+    const char *size_calc;
+
+    size_calc = qemu_opt_get(opts, VPC_OPT_SIZE_CALC);
+
+    if (!size_calc) {
+       /* no override, use autodetect only */
+    } else if (!strcmp(size_calc, "current_size")) {
+        s->force_use_sz = true;
+    } else if (!strcmp(size_calc, "chs")) {
+        s->force_use_chs = true;
+    } else {
+        error_setg(errp, "Invalid size calculation mode: '%s'", size_calc);
+    }
+}
+
 static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
@@ -165,6 +211,9 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     int i;
     VHDFooter *footer;
     VHDDynDiskHeader *dyndisk_header;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
+    bool use_chs;
     uint8_t buf[HEADER_SIZE];
     uint32_t checksum;
     uint64_t computed_size;
@@ -172,24 +221,42 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     int disk_type = VHD_DYNAMIC;
     int ret;
 
-    ret = bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE);
+    opts = qemu_opts_create(&vpc_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    vpc_parse_options(bs, opts, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    ret = bdrv_pread(bs->file->bs, 0, s->footer_buf, HEADER_SIZE);
     if (ret < 0) {
+        error_setg(errp, "Unable to read VHD header");
         goto fail;
     }
 
     footer = (VHDFooter *) s->footer_buf;
     if (strncmp(footer->creator, "conectix", 8)) {
-        int64_t offset = bdrv_getlength(bs->file);
+        int64_t offset = bdrv_getlength(bs->file->bs);
         if (offset < 0) {
             ret = offset;
+            error_setg(errp, "Invalid file size");
             goto fail;
         } else if (offset < HEADER_SIZE) {
             ret = -EINVAL;
+            error_setg(errp, "File too small for a VHD header");
             goto fail;
         }
 
         /* If a fixed disk, the footer is found only at the end of the file */
-        ret = bdrv_pread(bs->file, offset-HEADER_SIZE, s->footer_buf,
+        ret = bdrv_pread(bs->file->bs, offset-HEADER_SIZE, s->footer_buf,
                          HEADER_SIZE);
         if (ret < 0) {
             goto fail;
@@ -211,36 +278,66 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     /* Write 'checksum' back to footer, or else will leave it with zero. */
     footer->checksum = cpu_to_be32(checksum);
 
-    // The visible size of a image in Virtual PC depends on the geometry
-    // rather than on the size stored in the footer (the size in the footer
-    // is too large usually)
+    /* The visible size of a image in Virtual PC depends on the geometry
+       rather than on the size stored in the footer (the size in the footer
+       is too large usually) */
     bs->total_sectors = (int64_t)
         be16_to_cpu(footer->cyls) * footer->heads * footer->secs_per_cyl;
 
-    /* Images that have exactly the maximum geometry are probably bigger and
-     * would be truncated if we adhered to the geometry for them. Rely on
-     * footer->current_size for them. */
-    if (bs->total_sectors == VHD_MAX_GEOMETRY) {
+    /* Microsoft Virtual PC and Microsoft Hyper-V produce and read
+     * VHD image sizes differently.  VPC will rely on CHS geometry,
+     * while Hyper-V and disk2vhd use the size specified in the footer.
+     *
+     * We use a couple of approaches to try and determine the correct method:
+     * look at the Creator App field, and look for images that have CHS
+     * geometry that is the maximum value.
+     *
+     * If the CHS geometry is the maximum CHS geometry, then we assume that
+     * the size is the footer->current_size to avoid truncation.  Otherwise,
+     * we follow the table based on footer->creator_app:
+     *
+     *  Known creator apps:
+     *      'vpc '  :  CHS              Virtual PC (uses disk geometry)
+     *      'qemu'  :  CHS              QEMU (uses disk geometry)
+     *      'qem2'  :  current_size     QEMU (uses current_size)
+     *      'win '  :  current_size     Hyper-V
+     *      'd2v '  :  current_size     Disk2vhd
+     *      'tap\0' :  current_size     XenServer
+     *      'CTXS'  :  current_size     XenConverter
+     *
+     *  The user can override the table values via drive options, however
+     *  even with an override we will still use current_size for images
+     *  that have CHS geometry of the maximum size.
+     */
+    use_chs = (!!strncmp(footer->creator_app, "win ", 4) &&
+               !!strncmp(footer->creator_app, "qem2", 4) &&
+               !!strncmp(footer->creator_app, "d2v ", 4) &&
+               !!strncmp(footer->creator_app, "CTXS", 4) &&
+               !!memcmp(footer->creator_app, "tap", 4)) || s->force_use_chs;
+
+    if (!use_chs || bs->total_sectors == VHD_MAX_GEOMETRY || s->force_use_sz) {
         bs->total_sectors = be64_to_cpu(footer->current_size) /
-                            BDRV_SECTOR_SIZE;
+                                        BDRV_SECTOR_SIZE;
     }
 
-    /* Allow a maximum disk size of approximately 2 TB */
-    if (bs->total_sectors >= VHD_MAX_SECTORS) {
+    /* Allow a maximum disk size of 2040 GiB */
+    if (bs->total_sectors > VHD_MAX_SECTORS) {
         ret = -EFBIG;
         goto fail;
     }
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
+        ret = bdrv_pread(bs->file->bs, be64_to_cpu(footer->data_offset), buf,
                          HEADER_SIZE);
         if (ret < 0) {
+            error_setg(errp, "Error reading dynamic VHD header");
             goto fail;
         }
 
         dyndisk_header = (VHDDynDiskHeader *) buf;
 
         if (strncmp(dyndisk_header->magic, "cxsparse", 8)) {
+            error_setg(errp, "Invalid header magic");
             ret = -EINVAL;
             goto fail;
         }
@@ -256,16 +353,14 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
         s->max_table_entries = be32_to_cpu(dyndisk_header->max_table_entries);
 
         if ((bs->total_sectors * 512) / s->block_size > 0xffffffffU) {
-            ret = -EINVAL;
-            goto fail;
-        }
-        if (s->max_table_entries > (VHD_MAX_SECTORS * 512) / s->block_size) {
+            error_setg(errp, "Too many blocks");
             ret = -EINVAL;
             goto fail;
         }
 
         computed_size = (uint64_t) s->max_table_entries * s->block_size;
         if (computed_size < bs->total_sectors * 512) {
+            error_setg(errp, "Page table too small");
             ret = -EINVAL;
             goto fail;
         }
@@ -280,16 +375,19 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
 
         pagetable_size = (uint64_t) s->max_table_entries * 4;
 
-        s->pagetable = qemu_try_blockalign(bs->file, pagetable_size);
+        s->pagetable = qemu_try_blockalign(bs->file->bs, pagetable_size);
         if (s->pagetable == NULL) {
+            error_setg(errp, "Unable to allocate memory for page table");
             ret = -ENOMEM;
             goto fail;
         }
 
         s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
 
-        ret = bdrv_pread(bs->file, s->bat_offset, s->pagetable, pagetable_size);
+        ret = bdrv_pread(bs->file->bs, s->bat_offset, s->pagetable,
+                         pagetable_size);
         if (ret < 0) {
+            error_setg(errp, "Error reading pagetable");
             goto fail;
         }
 
@@ -308,7 +406,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
             }
         }
 
-        if (s->free_data_block_offset > bdrv_getlength(bs->file)) {
+        if (s->free_data_block_offset > bdrv_getlength(bs->file->bs)) {
             error_setg(errp, "block-vpc: free_data_block_offset points after "
                              "the end of file. The image has been truncated.");
             ret = -EINVAL;
@@ -368,22 +466,22 @@ static inline int64_t get_sector_offset(BlockDriverState *bs,
     pageentry_index = (offset % s->block_size) / 512;
 
     if (pagetable_index >= s->max_table_entries || s->pagetable[pagetable_index] == 0xffffffff)
-        return -1; // not allocated
+        return -1; /* not allocated */
 
     bitmap_offset = 512 * (uint64_t) s->pagetable[pagetable_index];
     block_offset = bitmap_offset + s->bitmap_size + (512 * pageentry_index);
 
-    // We must ensure that we don't write to any sectors which are marked as
-    // unused in the bitmap. We get away with setting all bits in the block
-    // bitmap each time we write to a new block. This might cause Virtual PC to
-    // miss sparse read optimization, but it's not a problem in terms of
-    // correctness.
+    /* We must ensure that we don't write to any sectors which are marked as
+       unused in the bitmap. We get away with setting all bits in the block
+       bitmap each time we write to a new block. This might cause Virtual PC to
+       miss sparse read optimization, but it's not a problem in terms of
+       correctness. */
     if (write && (s->last_bitmap_offset != bitmap_offset)) {
         uint8_t bitmap[s->bitmap_size];
 
         s->last_bitmap_offset = bitmap_offset;
         memset(bitmap, 0xff, s->bitmap_size);
-        bdrv_pwrite_sync(bs->file, bitmap_offset, bitmap, s->bitmap_size);
+        bdrv_pwrite_sync(bs->file->bs, bitmap_offset, bitmap, s->bitmap_size);
     }
 
     return block_offset;
@@ -401,7 +499,7 @@ static int rewrite_footer(BlockDriverState* bs)
     BDRVVPCState *s = bs->opaque;
     int64_t offset = s->free_data_block_offset;
 
-    ret = bdrv_pwrite_sync(bs->file, offset, s->footer_buf, HEADER_SIZE);
+    ret = bdrv_pwrite_sync(bs->file->bs, offset, s->footer_buf, HEADER_SIZE);
     if (ret < 0)
         return ret;
 
@@ -423,35 +521,35 @@ static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
     int ret;
     uint8_t bitmap[s->bitmap_size];
 
-    // Check if sector_num is valid
+    /* Check if sector_num is valid */
     if ((sector_num < 0) || (sector_num > bs->total_sectors))
         return -1;
 
-    // Write entry into in-memory BAT
+    /* Write entry into in-memory BAT */
     index = (sector_num * 512) / s->block_size;
     if (s->pagetable[index] != 0xFFFFFFFF)
         return -1;
 
     s->pagetable[index] = s->free_data_block_offset / 512;
 
-    // Initialize the block's bitmap
+    /* Initialize the block's bitmap */
     memset(bitmap, 0xff, s->bitmap_size);
-    ret = bdrv_pwrite_sync(bs->file, s->free_data_block_offset, bitmap,
+    ret = bdrv_pwrite_sync(bs->file->bs, s->free_data_block_offset, bitmap,
         s->bitmap_size);
     if (ret < 0) {
         return ret;
     }
 
-    // Write new footer (the old one will be overwritten)
+    /* Write new footer (the old one will be overwritten) */
     s->free_data_block_offset += s->block_size + s->bitmap_size;
     ret = rewrite_footer(bs);
     if (ret < 0)
         goto fail;
 
-    // Write BAT entry to disk
+    /* Write BAT entry to disk */
     bat_offset = s->bat_offset + (4 * index);
     bat_value = cpu_to_be32(s->pagetable[index]);
-    ret = bdrv_pwrite_sync(bs->file, bat_offset, &bat_value, 4);
+    ret = bdrv_pwrite_sync(bs->file->bs, bat_offset, &bat_value, 4);
     if (ret < 0)
         goto fail;
 
@@ -485,7 +583,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
     VHDFooter *footer = (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_read(bs->file, sector_num, buf, nb_sectors);
+        return bdrv_read(bs->file->bs, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
         offset = get_sector_offset(bs, sector_num, 0);
@@ -499,7 +597,7 @@ static int vpc_read(BlockDriverState *bs, int64_t sector_num,
         if (offset == -1) {
             memset(buf, 0, sectors * BDRV_SECTOR_SIZE);
         } else {
-            ret = bdrv_pread(bs->file, offset, buf,
+            ret = bdrv_pread(bs->file->bs, offset, buf,
                 sectors * BDRV_SECTOR_SIZE);
             if (ret != sectors * BDRV_SECTOR_SIZE) {
                 return -1;
@@ -534,7 +632,7 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_write(bs->file, sector_num, buf, nb_sectors);
+        return bdrv_write(bs->file->bs, sector_num, buf, nb_sectors);
     }
     while (nb_sectors > 0) {
         offset = get_sector_offset(bs, sector_num, 1);
@@ -551,7 +649,8 @@ static int vpc_write(BlockDriverState *bs, int64_t sector_num,
                 return -1;
         }
 
-        ret = bdrv_pwrite(bs->file, offset, buf, sectors * BDRV_SECTOR_SIZE);
+        ret = bdrv_pwrite(bs->file->bs, offset, buf,
+                          sectors * BDRV_SECTOR_SIZE);
         if (ret != sectors * BDRV_SECTOR_SIZE) {
             return -1;
         }
@@ -576,7 +675,7 @@ static coroutine_fn int vpc_co_write(BlockDriverState *bs, int64_t sector_num,
 }
 
 static int64_t coroutine_fn vpc_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVVPCState *s = bs->opaque;
     VHDFooter *footer = (VHDFooter*) s->footer_buf;
@@ -586,6 +685,7 @@ static int64_t coroutine_fn vpc_co_get_block_status(BlockDriverState *bs,
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
         *pnum = nb_sectors;
+        *file = bs->file->bs;
         return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_DATA |
                (sector_num << BDRV_SECTOR_BITS);
     }
@@ -607,6 +707,7 @@ static int64_t coroutine_fn vpc_co_get_block_status(BlockDriverState *bs,
         /* *pnum can't be greater than one block for allocated
          * sectors since there is always a bitmap in between. */
         if (allocated) {
+            *file = bs->file->bs;
             return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | start;
         }
         if (nb_sectors == 0) {
@@ -626,7 +727,7 @@ static int64_t coroutine_fn vpc_co_get_block_status(BlockDriverState *bs,
  * Note that the geometry doesn't always exactly match total_sectors but
  * may round it down.
  *
- * Returns 0 on success, -EFBIG if the size is larger than ~2 TB. Override
+ * Returns 0 on success, -EFBIG if the size is larger than 2040 GiB. Override
  * the hardware EIDE and ATA-2 limit of 16 heads (max disk size of 127 GB)
  * and instead allow up to 255 heads.
  */
@@ -668,7 +769,7 @@ static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
     return 0;
 }
 
-static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
+static int create_dynamic_disk(BlockBackend *blk, uint8_t *buf,
                                int64_t total_sectors)
 {
     VHDDynDiskHeader *dyndisk_header =
@@ -678,34 +779,34 @@ static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
     int ret;
     int64_t offset = 0;
 
-    // Write the footer (twice: at the beginning and at the end)
+    /* Write the footer (twice: at the beginning and at the end) */
     block_size = 0x200000;
     num_bat_entries = (total_sectors + block_size / 512) / (block_size / 512);
 
-    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
-    if (ret) {
+    ret = blk_pwrite(blk, offset, buf, HEADER_SIZE);
+    if (ret < 0) {
         goto fail;
     }
 
     offset = 1536 + ((num_bat_entries * 4 + 511) & ~511);
-    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
+    ret = blk_pwrite(blk, offset, buf, HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
 
-    // Write the initial BAT
+    /* Write the initial BAT */
     offset = 3 * 512;
 
     memset(buf, 0xFF, 512);
     for (i = 0; i < (num_bat_entries * 4 + 511) / 512; i++) {
-        ret = bdrv_pwrite_sync(bs, offset, buf, 512);
+        ret = blk_pwrite(blk, offset, buf, 512);
         if (ret < 0) {
             goto fail;
         }
         offset += 512;
     }
 
-    // Prepare the Dynamic Disk Header
+    /* Prepare the Dynamic Disk Header */
     memset(buf, 0, 1024);
 
     memcpy(dyndisk_header->magic, "cxsparse", 8);
@@ -722,10 +823,10 @@ static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
 
     dyndisk_header->checksum = cpu_to_be32(vpc_checksum(buf, 1024));
 
-    // Write the header
+    /* Write the header */
     offset = 512;
 
-    ret = bdrv_pwrite_sync(bs, offset, buf, 1024);
+    ret = blk_pwrite(blk, offset, buf, 1024);
     if (ret < 0) {
         goto fail;
     }
@@ -734,7 +835,7 @@ static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
     return ret;
 }
 
-static int create_fixed_disk(BlockDriverState *bs, uint8_t *buf,
+static int create_fixed_disk(BlockBackend *blk, uint8_t *buf,
                              int64_t total_size)
 {
     int ret;
@@ -742,12 +843,12 @@ static int create_fixed_disk(BlockDriverState *bs, uint8_t *buf,
     /* Add footer to total size */
     total_size += HEADER_SIZE;
 
-    ret = bdrv_truncate(bs, total_size);
+    ret = blk_truncate(blk, total_size);
     if (ret < 0) {
         return ret;
     }
 
-    ret = bdrv_pwrite_sync(bs, total_size - HEADER_SIZE, buf, HEADER_SIZE);
+    ret = blk_pwrite(blk, total_size - HEADER_SIZE, buf, HEADER_SIZE);
     if (ret < 0) {
         return ret;
     }
@@ -768,8 +869,9 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     int64_t total_size;
     int disk_type;
     int ret = -EIO;
+    bool force_size;
     Error *local_err = NULL;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
 
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
@@ -781,6 +883,7 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
         } else if (!strcmp(disk_type_param, "fixed")) {
             disk_type = VHD_FIXED;
         } else {
+            error_setg(errp, "Invalid disk type, %s", disk_type_param);
             ret = -EINVAL;
             goto out;
         }
@@ -788,36 +891,50 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
         disk_type = VHD_DYNAMIC;
     }
 
+    force_size = qemu_opt_get_bool_del(opts, VPC_OPT_FORCE_SIZE, false);
+
     ret = bdrv_create_file(filename, opts, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
     }
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /*
      * Calculate matching total_size and geometry. Increase the number of
      * sectors requested until we get enough (or fail). This ensures that
      * qemu-img convert doesn't truncate images, but rather rounds up.
      *
-     * If the image size can't be represented by a spec conform CHS geometry,
+     * If the image size can't be represented by a spec conformant CHS geometry,
      * we set the geometry to 65535 x 16 x 255 (CxHxS) sectors and use
      * the image size from the VHD footer to calculate total_sectors.
      */
-    total_sectors = MIN(VHD_MAX_GEOMETRY, total_size / BDRV_SECTOR_SIZE);
-    for (i = 0; total_sectors > (int64_t)cyls * heads * secs_per_cyl; i++) {
-        calculate_geometry(total_sectors + i, &cyls, &heads, &secs_per_cyl);
+    if (force_size) {
+        /* This will force the use of total_size for sector count, below */
+        cyls         = VHD_CHS_MAX_C;
+        heads        = VHD_CHS_MAX_H;
+        secs_per_cyl = VHD_CHS_MAX_S;
+    } else {
+        total_sectors = MIN(VHD_MAX_GEOMETRY, total_size / BDRV_SECTOR_SIZE);
+        for (i = 0; total_sectors > (int64_t)cyls * heads * secs_per_cyl; i++) {
+            calculate_geometry(total_sectors + i, &cyls, &heads, &secs_per_cyl);
+        }
     }
 
     if ((int64_t)cyls * heads * secs_per_cyl == VHD_MAX_GEOMETRY) {
         total_sectors = total_size / BDRV_SECTOR_SIZE;
-        /* Allow a maximum disk size of approximately 2 TB */
+        /* Allow a maximum disk size of 2040 GiB */
         if (total_sectors > VHD_MAX_SECTORS) {
+            error_setg(errp, "Disk size is too large, max size is 2040 GiB");
             ret = -EFBIG;
             goto out;
         }
@@ -830,8 +947,11 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     memset(buf, 0, 1024);
 
     memcpy(footer->creator, "conectix", 8);
-    /* TODO Check if "qemu" creator_app is ok for VPC */
-    memcpy(footer->creator_app, "qemu", 4);
+    if (force_size) {
+        memcpy(footer->creator_app, "qem2", 4);
+    } else {
+        memcpy(footer->creator_app, "qemu", 4);
+    }
     memcpy(footer->creator_os, "Wi2k", 4);
 
     footer->features = cpu_to_be32(0x02);
@@ -861,13 +981,16 @@ static int vpc_create(const char *filename, QemuOpts *opts, Error **errp)
     footer->checksum = cpu_to_be32(vpc_checksum(buf, HEADER_SIZE));
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = create_dynamic_disk(bs, buf, total_sectors);
+        ret = create_dynamic_disk(blk, buf, total_sectors);
     } else {
-        ret = create_fixed_disk(bs, buf, total_size);
+        ret = create_fixed_disk(blk, buf, total_size);
+    }
+    if (ret < 0) {
+        error_setg(errp, "Unable to create or write VHD header");
     }
 
 out:
-    bdrv_unref(bs);
+    blk_unref(blk);
     g_free(disk_type_param);
     return ret;
 }
@@ -878,7 +1001,7 @@ static int vpc_has_zero_init(BlockDriverState *bs)
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_has_zero_init(bs->file);
+        return bdrv_has_zero_init(bs->file->bs);
     } else {
         return 1;
     }
@@ -912,6 +1035,13 @@ static QemuOptsList vpc_create_opts = {
                 "Type of virtual hard disk format. Supported formats are "
                 "{dynamic (default) | fixed} "
         },
+        {
+            .name = VPC_OPT_FORCE_SIZE,
+            .type = QEMU_OPT_BOOL,
+            .help = "Force disk size calculation to use the actual size "
+                    "specified, rather than using the nearest CHS-based "
+                    "calculation"
+        },
         { /* end of list */ }
     }
 };
diff --git a/qemu/block/vvfat.c b/qemu/block/vvfat.c
index 206869712..183fc4f04 100644
--- a/qemu/block/vvfat.c
+++ b/qemu/block/vvfat.c
@@ -22,15 +22,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <sys/stat.h>
+#include "qemu/osdep.h"
 #include <dirent.h>
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qstring.h"
+#include "qemu/cutils.h"
 
 #ifndef S_IWGRP
 #define S_IWGRP 0
@@ -985,12 +986,6 @@ static BDRVVVFATState *vvv = NULL;
 static int enable_write_target(BDRVVVFATState *s, Error **errp);
 static int is_consistent(BDRVVVFATState *s);
 
-static void vvfat_rebind(BlockDriverState *bs)
-{
-    BDRVVVFATState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static QemuOptsList runtime_opts = {
     .name = "vvfat",
     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
@@ -1114,6 +1109,8 @@ static int vvfat_open(BlockDriverState *bs, QDict *options, int flags,
             goto fail;
         }
         memcpy(s->volume_label, label, label_length);
+    } else {
+        memcpy(s->volume_label, "QEMU VVFAT", 10);
     }
 
     if (floppy) {
@@ -2288,12 +2285,17 @@ DLOG(fprintf(stderr, "commit_direntries for %s, parent_mapping_index %d\n", mapp
 		factor * (old_cluster_count - new_cluster_count));
 
     for (c = first_cluster; !fat_eof(s, c); c = modified_fat_get(s, c)) {
+        direntry_t *first_direntry;
 	void* direntry = array_get(&(s->directory), current_dir_index);
 	int ret = vvfat_read(s->bs, cluster2sector(s, c), direntry,
 		s->sectors_per_cluster);
 	if (ret)
 	    return ret;
-	assert(!strncmp(s->directory.pointer, "QEMU", 4));
+
+        /* The first directory entry on the filesystem is the volume name */
+        first_direntry = (direntry_t*) s->directory.pointer;
+        assert(!memcmp(first_direntry->name, s->volume_label, 11));
+
 	current_dir_index += factor;
     }
 
@@ -2890,7 +2892,7 @@ static coroutine_fn int vvfat_co_write(BlockDriverState *bs, int64_t sector_num,
 }
 
 static int64_t coroutine_fn vvfat_co_get_block_status(BlockDriverState *bs,
-	int64_t sector_num, int nb_sectors, int* n)
+	int64_t sector_num, int nb_sectors, int *n, BlockDriverState **file)
 {
     BDRVVVFATState* s = bs->opaque;
     *n = s->sector_count - sector_num;
@@ -2923,9 +2925,12 @@ static BlockDriver vvfat_write_target = {
 static int enable_write_target(BDRVVVFATState *s, Error **errp)
 {
     BlockDriver *bdrv_qcow = NULL;
+    BlockDriverState *backing;
     QemuOpts *opts = NULL;
     int ret;
     int size = sector2cluster(s, s->sector_count);
+    QDict *options;
+
     s->used_clusters = calloc(size, 1);
 
     array_init(&(s->commits), sizeof(commit_t));
@@ -2956,9 +2961,10 @@ static int enable_write_target(BDRVVVFATState *s, Error **errp)
     }
 
     s->qcow = NULL;
-    ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
-                    bdrv_qcow, errp);
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow"));
+    ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, options,
+                    BDRV_O_RDWR | BDRV_O_NO_FLUSH, errp);
     if (ret < 0) {
         goto err;
     }
@@ -2967,10 +2973,13 @@ static int enable_write_target(BDRVVVFATState *s, Error **errp)
     unlink(s->qcow_filename);
 #endif
 
-    bdrv_set_backing_hd(s->bs, bdrv_new());
-    s->bs->backing_hd->drv = &vvfat_write_target;
-    s->bs->backing_hd->opaque = g_new(void *, 1);
-    *(void**)s->bs->backing_hd->opaque = s;
+    backing = bdrv_new();
+    bdrv_set_backing_hd(s->bs, backing);
+    bdrv_unref(backing);
+
+    s->bs->backing->bs->drv = &vvfat_write_target;
+    s->bs->backing->bs->opaque = g_new(void *, 1);
+    *(void**)s->bs->backing->bs->opaque = s;
 
     return 0;
 
@@ -3004,7 +3013,6 @@ static BlockDriver bdrv_vvfat = {
     .bdrv_parse_filename    = vvfat_parse_filename,
     .bdrv_file_open         = vvfat_open,
     .bdrv_close             = vvfat_close,
-    .bdrv_rebind            = vvfat_rebind,
 
     .bdrv_read              = vvfat_co_read,
     .bdrv_write             = vvfat_co_write,
diff --git a/qemu/block/win32-aio.c b/qemu/block/win32-aio.c
index 64e86827b..2d509a9a7 100644
--- a/qemu/block/win32-aio.c
+++ b/qemu/block/win32-aio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "block/block_int.h"
@@ -174,7 +175,7 @@ int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile)
 void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *old_context)
 {
-    aio_set_event_notifier(old_context, &aio->e, NULL);
+    aio_set_event_notifier(old_context, &aio->e, false, NULL);
     aio->is_aio_context_attached = false;
 }
 
@@ -182,7 +183,8 @@ void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *new_context)
 {
     aio->is_aio_context_attached = true;
-    aio_set_event_notifier(new_context, &aio->e, win32_aio_completion_cb);
+    aio_set_event_notifier(new_context, &aio->e, false,
+                           win32_aio_completion_cb);
 }
 
 QEMUWin32AIOState *win32_aio_init(void)
diff --git a/qemu/block/write-threshold.c b/qemu/block/write-threshold.c
index a53c1f5e6..cc2ca7183 100644
--- a/qemu/block/write-threshold.c
+++ b/qemu/block/write-threshold.c
@@ -10,8 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "block/block_int.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/write-threshold.h"
 #include "qemu/notify.h"
 #include "qapi-event.h"
diff --git a/qemu/blockdev-nbd.c b/qemu/blockdev-nbd.c
index bcdd18b3f..12cae0ea7 100644
--- a/qemu/blockdev-nbd.c
+++ b/qemu/blockdev-nbd.c
@@ -9,6 +9,7 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
 #include "hw/block/block.h"
@@ -17,58 +18,128 @@
 #include "qmp-commands.h"
 #include "trace.h"
 #include "block/nbd.h"
-#include "qemu/sockets.h"
+#include "io/channel-socket.h"
 
-static int server_fd = -1;
+typedef struct NBDServerData {
+    QIOChannelSocket *listen_ioc;
+    int watch;
+    QCryptoTLSCreds *tlscreds;
+} NBDServerData;
 
-static void nbd_accept(void *opaque)
+static NBDServerData *nbd_server;
+
+
+static gboolean nbd_accept(QIOChannel *ioc, GIOCondition condition,
+                           gpointer opaque)
 {
-    struct sockaddr_in addr;
-    socklen_t addr_len = sizeof(addr);
+    QIOChannelSocket *cioc;
+
+    if (!nbd_server) {
+        return FALSE;
+    }
 
-    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    if (fd >= 0 && !nbd_client_new(NULL, fd, nbd_client_put)) {
-        shutdown(fd, 2);
-        close(fd);
+    cioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc),
+                                     NULL);
+    if (!cioc) {
+        return TRUE;
     }
+
+    nbd_client_new(NULL, cioc,
+                   nbd_server->tlscreds, NULL,
+                   nbd_client_put);
+    object_unref(OBJECT(cioc));
+    return TRUE;
 }
 
-void qmp_nbd_server_start(SocketAddress *addr, Error **errp)
+
+static void nbd_server_free(NBDServerData *server)
 {
-    if (server_fd != -1) {
-        error_setg(errp, "NBD server already running");
+    if (!server) {
         return;
     }
 
-    server_fd = socket_listen(addr, errp);
-    if (server_fd != -1) {
-        qemu_set_fd_handler(server_fd, nbd_accept, NULL, NULL);
+    if (server->watch != -1) {
+        g_source_remove(server->watch);
+    }
+    object_unref(OBJECT(server->listen_ioc));
+    if (server->tlscreds) {
+        object_unref(OBJECT(server->tlscreds));
     }
+
+    g_free(server);
 }
 
-/*
- * Hook into the BlockBackend notifiers to close the export when the
- * backend is closed.
- */
-typedef struct NBDCloseNotifier {
-    Notifier n;
-    NBDExport *exp;
-    QTAILQ_ENTRY(NBDCloseNotifier) next;
-} NBDCloseNotifier;
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
+    }
 
-static QTAILQ_HEAD(, NBDCloseNotifier) close_notifiers =
-    QTAILQ_HEAD_INITIALIZER(close_notifiers);
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a server endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
+}
 
-static void nbd_close_notifier(Notifier *n, void *data)
+
+void qmp_nbd_server_start(SocketAddress *addr,
+                          bool has_tls_creds, const char *tls_creds,
+                          Error **errp)
 {
-    NBDCloseNotifier *cn = DO_UPCAST(NBDCloseNotifier, n, n);
+    if (nbd_server) {
+        error_setg(errp, "NBD server already running");
+        return;
+    }
+
+    nbd_server = g_new0(NBDServerData, 1);
+    nbd_server->watch = -1;
+    nbd_server->listen_ioc = qio_channel_socket_new();
+    if (qio_channel_socket_listen_sync(
+            nbd_server->listen_ioc, addr, errp) < 0) {
+        goto error;
+    }
+
+    if (has_tls_creds) {
+        nbd_server->tlscreds = nbd_get_tls_creds(tls_creds, errp);
+        if (!nbd_server->tlscreds) {
+            goto error;
+        }
+
+        if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+            error_setg(errp, "TLS is only supported with IPv4/IPv6");
+            goto error;
+        }
+    }
 
-    notifier_remove(&cn->n);
-    QTAILQ_REMOVE(&close_notifiers, cn, next);
+    nbd_server->watch = qio_channel_add_watch(
+        QIO_CHANNEL(nbd_server->listen_ioc),
+        G_IO_IN,
+        nbd_accept,
+        NULL,
+        NULL);
 
-    nbd_export_close(cn->exp);
-    nbd_export_put(cn->exp);
-    g_free(cn);
+    return;
+
+ error:
+    nbd_server_free(nbd_server);
+    nbd_server = NULL;
 }
 
 void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
@@ -76,9 +147,8 @@ void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
 {
     BlockBackend *blk;
     NBDExport *exp;
-    NBDCloseNotifier *n;
 
-    if (server_fd == -1) {
+    if (!nbd_server) {
         error_setg(errp, "NBD server not running");
         return;
     }
@@ -114,23 +184,16 @@ void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
 
     nbd_export_set_name(exp, device);
 
-    n = g_new0(NBDCloseNotifier, 1);
-    n->n.notify = nbd_close_notifier;
-    n->exp = exp;
-    blk_add_close_notifier(blk, &n->n);
-    QTAILQ_INSERT_TAIL(&close_notifiers, n, next);
+    /* The list of named exports has a strong reference to this export now and
+     * our only way of accessing it is through nbd_export_find(), so we can drop
+     * the strong reference that is @exp. */
+    nbd_export_put(exp);
 }
 
 void qmp_nbd_server_stop(Error **errp)
 {
-    while (!QTAILQ_EMPTY(&close_notifiers)) {
-        NBDCloseNotifier *cn = QTAILQ_FIRST(&close_notifiers);
-        nbd_close_notifier(&cn->n, nbd_export_get_blockdev(cn->exp));
-    }
+    nbd_export_close_all();
 
-    if (server_fd != -1) {
-        qemu_set_fd_handler(server_fd, NULL, NULL, NULL);
-        close(server_fd);
-        server_fd = -1;
-    }
+    nbd_server_free(nbd_server);
+    nbd_server = NULL;
 }
diff --git a/qemu/blockdev.c b/qemu/blockdev.c
index 62a4586cd..f1f520a26 100644
--- a/qemu/blockdev.c
+++ b/qemu/blockdev.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
@@ -49,6 +50,11 @@
 #include "qmp-commands.h"
 #include "trace.h"
 #include "sysemu/arch_init.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
+
+static QTAILQ_HEAD(, BlockDriverState) monitor_bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(monitor_bdrv_states);
 
 static const char *const if_name[IF_COUNT] = {
     [IF_NONE] = "none",
@@ -124,14 +130,16 @@ void blockdev_mark_auto_del(BlockBackend *blk)
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (bs) {
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
 
-    if (bs->job) {
-        block_job_cancel(bs->job);
-    }
+        if (bs->job) {
+            block_job_cancel(bs->job);
+        }
 
-    aio_context_release(aio_context);
+        aio_context_release(aio_context);
+    }
 
     dinfo->auto_del = 1;
 }
@@ -141,6 +149,7 @@ void blockdev_auto_del(BlockBackend *blk)
     DriveInfo *dinfo = blk_legacy_dinfo(blk);
 
     if (dinfo && dinfo->auto_del) {
+        monitor_remove_blk(blk);
         blk_unref(blk);
     }
 }
@@ -229,8 +238,8 @@ bool drive_check_orphaned(void)
             dinfo->type != IF_NONE) {
             fprintf(stderr, "Warning: Orphaned drive without device: "
                     "id=%s,file=%s,if=%s,bus=%d,unit=%d\n",
-                    blk_name(blk), blk_bs(blk)->filename, if_name[dinfo->type],
-                    dinfo->bus, dinfo->unit);
+                    blk_name(blk), blk_bs(blk) ? blk_bs(blk)->filename : "",
+                    if_name[dinfo->type], dinfo->bus, dinfo->unit);
             rs = true;
         }
     }
@@ -281,32 +290,6 @@ typedef struct {
     BlockDriverState *bs;
 } BDRVPutRefBH;
 
-static void bdrv_put_ref_bh(void *opaque)
-{
-    BDRVPutRefBH *s = opaque;
-
-    bdrv_unref(s->bs);
-    qemu_bh_delete(s->bh);
-    g_free(s);
-}
-
-/*
- * Release a BDS reference in a BH
- *
- * It is not safe to use bdrv_unref() from a callback function when the callers
- * still need the BlockDriverState.  In such cases we schedule a BH to release
- * the reference.
- */
-static void bdrv_put_ref_bh_schedule(BlockDriverState *bs)
-{
-    BDRVPutRefBH *s;
-
-    s = g_new(BDRVPutRefBH, 1);
-    s->bh = qemu_bh_new(bdrv_put_ref_bh, s);
-    s->bs = bs;
-    qemu_bh_schedule(s->bh);
-}
-
 static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
 {
     if (!strcmp(buf, "ignore")) {
@@ -324,43 +307,181 @@ static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
     }
 }
 
-static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
+static bool parse_stats_intervals(BlockAcctStats *stats, QList *intervals,
+                                  Error **errp)
 {
-    if (throttle_conflicting(cfg)) {
-        error_setg(errp, "bps/iops/max total values and read/write values"
-                         " cannot be used at the same time");
-        return false;
-    }
+    const QListEntry *entry;
+    for (entry = qlist_first(intervals); entry; entry = qlist_next(entry)) {
+        switch (qobject_type(entry->value)) {
+
+        case QTYPE_QSTRING: {
+            unsigned long long length;
+            const char *str = qstring_get_str(qobject_to_qstring(entry->value));
+            if (parse_uint_full(str, &length, 10) == 0 &&
+                length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %s", str);
+                return false;
+            }
+            break;
+        }
 
-    if (!throttle_is_valid(cfg)) {
-        error_setg(errp, "bps/iops/maxs values must be 0 or greater");
-        return false;
-    }
+        case QTYPE_QINT: {
+            int64_t length = qint_get_int(qobject_to_qint(entry->value));
+            if (length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %" PRId64, length);
+                return false;
+            }
+            break;
+        }
 
+        default:
+            error_setg(errp, "The specification of stats-intervals is invalid");
+            return false;
+        }
+    }
     return true;
 }
 
 typedef enum { MEDIA_DISK, MEDIA_CDROM } DriveMediaType;
 
+/* All parameters but @opts are optional and may be set to NULL. */
+static void extract_common_blockdev_options(QemuOpts *opts, int *bdrv_flags,
+    const char **throttling_group, ThrottleConfig *throttle_cfg,
+    BlockdevDetectZeroesOptions *detect_zeroes, Error **errp)
+{
+    const char *discard;
+    Error *local_error = NULL;
+    const char *aio;
+
+    if (bdrv_flags) {
+        if (!qemu_opt_get_bool(opts, "read-only", false)) {
+            *bdrv_flags |= BDRV_O_RDWR;
+        }
+        if (qemu_opt_get_bool(opts, "copy-on-read", false)) {
+            *bdrv_flags |= BDRV_O_COPY_ON_READ;
+        }
+
+        if ((discard = qemu_opt_get(opts, "discard")) != NULL) {
+            if (bdrv_parse_discard_flags(discard, bdrv_flags) != 0) {
+                error_setg(errp, "Invalid discard option");
+                return;
+            }
+        }
+
+        if ((aio = qemu_opt_get(opts, "aio")) != NULL) {
+            if (!strcmp(aio, "native")) {
+                *bdrv_flags |= BDRV_O_NATIVE_AIO;
+            } else if (!strcmp(aio, "threads")) {
+                /* this is the default */
+            } else {
+               error_setg(errp, "invalid aio option");
+               return;
+            }
+        }
+    }
+
+    /* disk I/O throttling */
+    if (throttling_group) {
+        *throttling_group = qemu_opt_get(opts, "throttling.group");
+    }
+
+    if (throttle_cfg) {
+        throttle_config_init(throttle_cfg);
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.bps-total", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].avg  =
+            qemu_opt_get_number(opts, "throttling.bps-read", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.bps-write", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.iops-total", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].avg =
+            qemu_opt_get_number(opts, "throttling.iops-read", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.iops-write", 0);
+
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].max  =
+            qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].max =
+            qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
+
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].burst_length =
+            qemu_opt_get_number(opts, "throttling.bps-total-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].burst_length  =
+            qemu_opt_get_number(opts, "throttling.bps-read-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].burst_length =
+            qemu_opt_get_number(opts, "throttling.bps-write-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-total-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-read-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-write-max-length", 1);
+
+        throttle_cfg->op_size =
+            qemu_opt_get_number(opts, "throttling.iops-size", 0);
+
+        if (!throttle_is_valid(throttle_cfg, errp)) {
+            return;
+        }
+    }
+
+    if (detect_zeroes) {
+        *detect_zeroes =
+            qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                            qemu_opt_get(opts, "detect-zeroes"),
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS__MAX,
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                            &local_error);
+        if (local_error) {
+            error_propagate(errp, local_error);
+            return;
+        }
+
+        if (bdrv_flags &&
+            *detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+            !(*bdrv_flags & BDRV_O_UNMAP))
+        {
+            error_setg(errp, "setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+            return;
+        }
+    }
+}
+
 /* Takes the ownership of bs_opts */
 static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
                                    Error **errp)
 {
     const char *buf;
-    int ro = 0;
     int bdrv_flags = 0;
     int on_read_error, on_write_error;
+    bool account_invalid, account_failed;
+    bool writethrough;
     BlockBackend *blk;
     BlockDriverState *bs;
     ThrottleConfig cfg;
     int snapshot = 0;
-    bool copy_on_read;
     Error *error = NULL;
     QemuOpts *opts;
+    QDict *interval_dict = NULL;
+    QList *interval_list = NULL;
     const char *id;
-    bool has_driver_specific_opts;
-    BlockdevDetectZeroesOptions detect_zeroes;
-    const char *throttling_group;
+    BlockdevDetectZeroesOptions detect_zeroes =
+        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
+    const char *throttling_group = NULL;
 
     /* Check common options by copying from bs_opts to opts, all other options
      * stay in bs_opts for processing by bdrv_open(). */
@@ -381,42 +502,29 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
         qdict_del(bs_opts, "id");
     }
 
-    has_driver_specific_opts = !!qdict_size(bs_opts);
-
     /* extract parameters */
     snapshot = qemu_opt_get_bool(opts, "snapshot", 0);
-    ro = qemu_opt_get_bool(opts, "read-only", 0);
-    copy_on_read = qemu_opt_get_bool(opts, "copy-on-read", false);
 
-    if ((buf = qemu_opt_get(opts, "discard")) != NULL) {
-        if (bdrv_parse_discard_flags(buf, &bdrv_flags) != 0) {
-            error_setg(errp, "invalid discard option");
-            goto early_err;
-        }
-    }
+    account_invalid = qemu_opt_get_bool(opts, "stats-account-invalid", true);
+    account_failed = qemu_opt_get_bool(opts, "stats-account-failed", true);
 
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_WB, true)) {
-        bdrv_flags |= BDRV_O_CACHE_WB;
-    }
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_DIRECT, false)) {
-        bdrv_flags |= BDRV_O_NOCACHE;
-    }
-    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_NO_FLUSH, false)) {
-        bdrv_flags |= BDRV_O_NO_FLUSH;
+    writethrough = !qemu_opt_get_bool(opts, BDRV_OPT_CACHE_WB, true);
+
+    qdict_extract_subqdict(bs_opts, &interval_dict, "stats-intervals.");
+    qdict_array_split(interval_dict, &interval_list);
+
+    if (qdict_size(interval_dict) != 0) {
+        error_setg(errp, "Invalid option stats-intervals.%s",
+                   qdict_first(interval_dict)->key);
+        goto early_err;
     }
 
-#ifdef CONFIG_LINUX_AIO
-    if ((buf = qemu_opt_get(opts, "aio")) != NULL) {
-        if (!strcmp(buf, "native")) {
-            bdrv_flags |= BDRV_O_NATIVE_AIO;
-        } else if (!strcmp(buf, "threads")) {
-            /* this is the default */
-        } else {
-           error_setg(errp, "invalid aio option");
-           goto early_err;
-        }
+    extract_common_blockdev_options(opts, &bdrv_flags, &throttling_group, &cfg,
+                                    &detect_zeroes, &error);
+    if (error) {
+        error_propagate(errp, error);
+        goto early_err;
     }
-#endif
 
     if ((buf = qemu_opt_get(opts, "format")) != NULL) {
         if (is_help_option(buf)) {
@@ -433,43 +541,6 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
         qdict_put(bs_opts, "driver", qstring_from_str(buf));
     }
 
-    /* disk I/O throttling */
-    memset(&cfg, 0, sizeof(cfg));
-    cfg.buckets[THROTTLE_BPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.bps-total", 0);
-    cfg.buckets[THROTTLE_BPS_READ].avg  =
-        qemu_opt_get_number(opts, "throttling.bps-read", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.bps-write", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.iops-total", 0);
-    cfg.buckets[THROTTLE_OPS_READ].avg =
-        qemu_opt_get_number(opts, "throttling.iops-read", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.iops-write", 0);
-
-    cfg.buckets[THROTTLE_BPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
-    cfg.buckets[THROTTLE_BPS_READ].max  =
-        qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
-    cfg.buckets[THROTTLE_OPS_READ].max =
-        qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
-
-    cfg.op_size = qemu_opt_get_number(opts, "throttling.iops-size", 0);
-
-    throttling_group = qemu_opt_get(opts, "throttling.group");
-
-    if (!check_throttle_config(&cfg, &error)) {
-        error_propagate(errp, error);
-        goto early_err;
-    }
-
     on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
     if ((buf = qemu_opt_get(opts, "werror")) != NULL) {
         on_write_error = parse_block_error_action(buf, 0, &error);
@@ -488,34 +559,32 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
         }
     }
 
-    detect_zeroes =
-        qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
-                        qemu_opt_get(opts, "detect-zeroes"),
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
-                        &error);
-    if (error) {
-        error_propagate(errp, error);
-        goto early_err;
-    }
-
-    if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
-        !(bdrv_flags & BDRV_O_UNMAP)) {
-        error_setg(errp, "setting detect-zeroes to unmap is not allowed "
-                         "without setting discard operation to unmap");
-        goto early_err;
+    if (snapshot) {
+        bdrv_flags |= BDRV_O_SNAPSHOT;
     }
 
     /* init */
-    if ((!file || !*file) && !has_driver_specific_opts) {
-        blk = blk_new_with_bs(qemu_opts_id(opts), errp);
+    if ((!file || !*file) && !qdict_size(bs_opts)) {
+        BlockBackendRootState *blk_rs;
+
+        blk = blk_new(errp);
         if (!blk) {
             goto early_err;
         }
 
-        bs = blk_bs(blk);
-        bs->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
-        bs->read_only = ro;
+        blk_rs = blk_get_root_state(blk);
+        blk_rs->open_flags    = bdrv_flags;
+        blk_rs->read_only     = !(bdrv_flags & BDRV_O_RDWR);
+        blk_rs->detect_zeroes = detect_zeroes;
+
+        if (throttle_enabled(&cfg)) {
+            if (!throttling_group) {
+                throttling_group = blk_name(blk);
+            }
+            blk_rs->throttle_group = g_strdup(throttling_group);
+            blk_rs->throttle_state = throttle_group_incref(throttling_group);
+            blk_rs->throttle_state->cfg = cfg;
+        }
 
         QDECREF(bs_opts);
     } else {
@@ -523,58 +592,149 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
             file = NULL;
         }
 
-        if (snapshot) {
-            /* always use cache=unsafe with snapshot */
-            bdrv_flags &= ~BDRV_O_CACHE_MASK;
-            bdrv_flags |= (BDRV_O_SNAPSHOT|BDRV_O_CACHE_WB|BDRV_O_NO_FLUSH);
-        }
-
-        if (copy_on_read) {
-            bdrv_flags |= BDRV_O_COPY_ON_READ;
-        }
+        /* bdrv_open() defaults to the values in bdrv_flags (for compatibility
+         * with other callers) rather than what we want as the real defaults.
+         * Apply the defaults here instead. */
+        qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_DIRECT, "off");
+        qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_NO_FLUSH, "off");
+        assert((bdrv_flags & BDRV_O_CACHE_MASK) == 0);
 
         if (runstate_check(RUN_STATE_INMIGRATE)) {
-            bdrv_flags |= BDRV_O_INCOMING;
+            bdrv_flags |= BDRV_O_INACTIVE;
         }
 
-        bdrv_flags |= ro ? 0 : BDRV_O_RDWR;
-
-        blk = blk_new_open(qemu_opts_id(opts), file, NULL, bs_opts, bdrv_flags,
-                           errp);
+        blk = blk_new_open(file, NULL, bs_opts, bdrv_flags, errp);
         if (!blk) {
             goto err_no_bs_opts;
         }
         bs = blk_bs(blk);
-    }
 
-    bs->detect_zeroes = detect_zeroes;
+        bs->detect_zeroes = detect_zeroes;
 
-    bdrv_set_on_error(bs, on_read_error, on_write_error);
+        /* disk I/O throttling */
+        if (throttle_enabled(&cfg)) {
+            if (!throttling_group) {
+                throttling_group = blk_name(blk);
+            }
+            bdrv_io_limits_enable(bs, throttling_group);
+            bdrv_set_io_limits(bs, &cfg);
+        }
 
-    /* disk I/O throttling */
-    if (throttle_enabled(&cfg)) {
-        if (!throttling_group) {
-            throttling_group = blk_name(blk);
+        if (bdrv_key_required(bs)) {
+            autostart = 0;
+        }
+
+        block_acct_init(blk_get_stats(blk), account_invalid, account_failed);
+
+        if (!parse_stats_intervals(blk_get_stats(blk), interval_list, errp)) {
+            blk_unref(blk);
+            blk = NULL;
+            goto err_no_bs_opts;
         }
-        bdrv_io_limits_enable(bs, throttling_group);
-        bdrv_set_io_limits(bs, &cfg);
     }
 
-    if (bdrv_key_required(bs)) {
-        autostart = 0;
+    blk_set_enable_write_cache(blk, !writethrough);
+    blk_set_on_error(blk, on_read_error, on_write_error);
+
+    if (!monitor_add_blk(blk, qemu_opts_id(opts), errp)) {
+        blk_unref(blk);
+        blk = NULL;
+        goto err_no_bs_opts;
     }
 
 err_no_bs_opts:
     qemu_opts_del(opts);
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
     return blk;
 
 early_err:
     qemu_opts_del(opts);
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
 err_no_opts:
     QDECREF(bs_opts);
     return NULL;
 }
 
+static QemuOptsList qemu_root_bds_opts;
+
+/* Takes the ownership of bs_opts */
+static BlockDriverState *bds_tree_init(QDict *bs_opts, Error **errp)
+{
+    BlockDriverState *bs;
+    QemuOpts *opts;
+    Error *local_error = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes;
+    int ret;
+    int bdrv_flags = 0;
+
+    opts = qemu_opts_create(&qemu_root_bds_opts, NULL, 1, errp);
+    if (!opts) {
+        goto fail;
+    }
+
+    qemu_opts_absorb_qdict(opts, bs_opts, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    extract_common_blockdev_options(opts, &bdrv_flags, NULL, NULL,
+                                    &detect_zeroes, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    /* bdrv_open() defaults to the values in bdrv_flags (for compatibility
+     * with other callers) rather than what we want as the real defaults.
+     * Apply the defaults here instead. */
+    qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_DIRECT, "off");
+    qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_NO_FLUSH, "off");
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        bdrv_flags |= BDRV_O_INACTIVE;
+    }
+
+    bs = NULL;
+    ret = bdrv_open(&bs, NULL, NULL, bs_opts, bdrv_flags, errp);
+    if (ret < 0) {
+        goto fail_no_bs_opts;
+    }
+
+    bs->detect_zeroes = detect_zeroes;
+
+fail_no_bs_opts:
+    qemu_opts_del(opts);
+    return bs;
+
+fail:
+    qemu_opts_del(opts);
+    QDECREF(bs_opts);
+    return NULL;
+}
+
+void blockdev_close_all_bdrv_states(void)
+{
+    BlockDriverState *bs, *next_bs;
+
+    QTAILQ_FOREACH_SAFE(bs, &monitor_bdrv_states, monitor_list, next_bs) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        bdrv_unref(bs);
+        aio_context_release(ctx);
+    }
+}
+
+/* Iterates over the list of monitor-owned BlockDriverStates */
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs)
+{
+    return bs ? QTAILQ_NEXT(bs, monitor_list)
+              : QTAILQ_FIRST(&monitor_bdrv_states);
+}
+
 static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to,
                             Error **errp)
 {
@@ -737,8 +897,9 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
     value = qemu_opt_get(all_opts, "cache");
     if (value) {
         int flags = 0;
+        bool writethrough;
 
-        if (bdrv_parse_cache_flags(value, &flags) != 0) {
+        if (bdrv_parse_cache_mode(value, &flags, &writethrough) != 0) {
             error_report("invalid cache option");
             return NULL;
         }
@@ -746,7 +907,7 @@ DriveInfo *drive_new(QemuOpts *all_opts, BlockInterfaceType block_default_type)
         /* Specific options take precedence */
         if (!qemu_opt_get(all_opts, BDRV_OPT_CACHE_WB)) {
             qemu_opt_set_bool(all_opts, BDRV_OPT_CACHE_WB,
-                              !!(flags & BDRV_O_CACHE_WB), &error_abort);
+                              !writethrough, &error_abort);
         }
         if (!qemu_opt_get(all_opts, BDRV_OPT_CACHE_DIRECT)) {
             qemu_opt_set_bool(all_opts, BDRV_OPT_CACHE_DIRECT,
@@ -1031,14 +1192,28 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
     int ret;
 
     if (!strcmp(device, "all")) {
-        ret = bdrv_commit_all();
+        ret = blk_commit_all();
     } else {
+        BlockDriverState *bs;
+        AioContext *aio_context;
+
         blk = blk_by_name(device);
         if (!blk) {
             monitor_printf(mon, "Device '%s' not found\n", device);
             return;
         }
-        ret = bdrv_commit(blk_bs(blk));
+        if (!blk_is_available(blk)) {
+            monitor_printf(mon, "Device '%s' has no medium\n", device);
+            return;
+        }
+
+        bs = blk_bs(blk);
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
+
+        ret = bdrv_commit(bs);
+
+        aio_context_release(aio_context);
     }
     if (ret < 0) {
         monitor_printf(mon, "'commit' error for '%s': %s\n", device,
@@ -1046,16 +1221,13 @@ void hmp_commit(Monitor *mon, const QDict *qdict)
     }
 }
 
-static void blockdev_do_action(int kind, void *data, Error **errp)
+static void blockdev_do_action(TransactionAction *action, Error **errp)
 {
-    TransactionAction action;
     TransactionActionList list;
 
-    action.kind = kind;
-    action.data = data;
-    list.value = &action;
+    list.value = action;
     list.next = NULL;
-    qmp_transaction(&list, errp);
+    qmp_transaction(&list, false, NULL, errp);
 }
 
 void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
@@ -1066,7 +1238,7 @@ void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
                                 bool has_format, const char *format,
                                 bool has_mode, NewImageMode mode, Error **errp)
 {
-    BlockdevSnapshot snapshot = {
+    BlockdevSnapshotSync snapshot = {
         .has_device = has_device,
         .device = (char *) device,
         .has_node_name = has_node_name,
@@ -1079,8 +1251,25 @@ void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
         .has_mode = has_mode,
         .mode = mode,
     };
-    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC,
-                       &snapshot, errp);
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC,
+        .u.blockdev_snapshot_sync.data = &snapshot,
+    };
+    blockdev_do_action(&action, errp);
+}
+
+void qmp_blockdev_snapshot(const char *node, const char *overlay,
+                           Error **errp)
+{
+    BlockdevSnapshot snapshot_data = {
+        .node = (char *) node,
+        .overlay = (char *) overlay
+    };
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT,
+        .u.blockdev_snapshot.data = &snapshot_data,
+    };
+    blockdev_do_action(&action, errp);
 }
 
 void qmp_blockdev_snapshot_internal_sync(const char *device,
@@ -1091,9 +1280,11 @@ void qmp_blockdev_snapshot_internal_sync(const char *device,
         .device = (char *) device,
         .name = (char *) name
     };
-
-    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC,
-                       &snapshot, errp);
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC,
+        .u.blockdev_snapshot_internal_sync.data = &snapshot,
+    };
+    blockdev_do_action(&action, errp);
 }
 
 SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
@@ -1117,7 +1308,9 @@ SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
                   "Device '%s' not found", device);
         return NULL;
     }
-    bs = blk_bs(blk);
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
 
     if (!has_id) {
         id = NULL;
@@ -1129,11 +1322,14 @@ SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
 
     if (!id && !name) {
         error_setg(errp, "Name or id must be provided");
-        return NULL;
+        goto out_aio_context;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out_aio_context;
+    }
+    bs = blk_bs(blk);
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE, errp)) {
         goto out_aio_context;
@@ -1240,43 +1436,75 @@ static BdrvDirtyBitmap *block_dirty_bitmap_lookup(const char *node,
 
 /* New and old BlockDriverState structs for atomic group operations */
 
-typedef struct BlkTransactionState BlkTransactionState;
+typedef struct BlkActionState BlkActionState;
 
-/* Only prepare() may fail. In a single transaction, only one of commit() or
-   abort() will be called, clean() will always be called if it present. */
-typedef struct BdrvActionOps {
-    /* Size of state struct, in bytes. */
+/**
+ * BlkActionOps:
+ * Table of operations that define an Action.
+ *
+ * @instance_size: Size of state struct, in bytes.
+ * @prepare: Prepare the work, must NOT be NULL.
+ * @commit: Commit the changes, can be NULL.
+ * @abort: Abort the changes on fail, can be NULL.
+ * @clean: Clean up resources after all transaction actions have called
+ *         commit() or abort(). Can be NULL.
+ *
+ * Only prepare() may fail. In a single transaction, only one of commit() or
+ * abort() will be called. clean() will always be called if it is present.
+ */
+typedef struct BlkActionOps {
     size_t instance_size;
-    /* Prepare the work, must NOT be NULL. */
-    void (*prepare)(BlkTransactionState *common, Error **errp);
-    /* Commit the changes, can be NULL. */
-    void (*commit)(BlkTransactionState *common);
-    /* Abort the changes on fail, can be NULL. */
-    void (*abort)(BlkTransactionState *common);
-    /* Clean up resource in the end, can be NULL. */
-    void (*clean)(BlkTransactionState *common);
-} BdrvActionOps;
+    void (*prepare)(BlkActionState *common, Error **errp);
+    void (*commit)(BlkActionState *common);
+    void (*abort)(BlkActionState *common);
+    void (*clean)(BlkActionState *common);
+} BlkActionOps;
 
-/*
- * This structure must be arranged as first member in child type, assuming
- * that compiler will also arrange it to the same address with parent instance.
- * Later it will be used in free().
+/**
+ * BlkActionState:
+ * Describes one Action's state within a Transaction.
+ *
+ * @action: QAPI-defined enum identifying which Action to perform.
+ * @ops: Table of ActionOps this Action can perform.
+ * @block_job_txn: Transaction which this action belongs to.
+ * @entry: List membership for all Actions in this Transaction.
+ *
+ * This structure must be arranged as first member in a subclassed type,
+ * assuming that the compiler will also arrange it to the same offsets as the
+ * base class.
  */
-struct BlkTransactionState {
+struct BlkActionState {
     TransactionAction *action;
-    const BdrvActionOps *ops;
-    QSIMPLEQ_ENTRY(BlkTransactionState) entry;
+    const BlkActionOps *ops;
+    BlockJobTxn *block_job_txn;
+    TransactionProperties *txn_props;
+    QSIMPLEQ_ENTRY(BlkActionState) entry;
 };
 
 /* internal snapshot private data */
 typedef struct InternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     AioContext *aio_context;
     QEMUSnapshotInfo sn;
+    bool created;
 } InternalSnapshotState;
 
-static void internal_snapshot_prepare(BlkTransactionState *common,
+
+static int action_check_completion_mode(BlkActionState *s, Error **errp)
+{
+    if (s->txn_props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        error_setg(errp,
+                   "Action '%s' does not support Transaction property "
+                   "completion-mode = %s",
+                   TransactionActionKind_lookup[s->action->type],
+                   ActionCompletionMode_lookup[s->txn_props->completion_mode]);
+        return -1;
+    }
+    return 0;
+}
+
+static void internal_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
     Error *local_err = NULL;
@@ -1291,9 +1519,9 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     InternalSnapshotState *state;
     int ret1;
 
-    g_assert(common->action->kind ==
+    g_assert(common->action->type ==
              TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC);
-    internal = common->action->blockdev_snapshot_internal_sync;
+    internal = common->action->u.blockdev_snapshot_internal_sync.data;
     state = DO_UPCAST(InternalSnapshotState, common, common);
 
     /* 1. parse input */
@@ -1301,22 +1529,29 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     name = internal->name;
 
     /* 2. check for validation */
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
     blk = blk_by_name(device);
     if (!blk) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
+    state->aio_context = blk_get_aio_context(blk);
     aio_context_acquire(state->aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         return;
     }
+    bs = blk_bs(blk);
+
+    state->bs = bs;
+    bdrv_drained_begin(bs);
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) {
         return;
@@ -1369,10 +1604,10 @@ static void internal_snapshot_prepare(BlkTransactionState *common,
     }
 
     /* 4. succeed, mark a snapshot is created */
-    state->bs = bs;
+    state->created = true;
 }
 
-static void internal_snapshot_abort(BlkTransactionState *common)
+static void internal_snapshot_abort(BlkActionState *common)
 {
     InternalSnapshotState *state =
                              DO_UPCAST(InternalSnapshotState, common, common);
@@ -1380,106 +1615,96 @@ static void internal_snapshot_abort(BlkTransactionState *common)
     QEMUSnapshotInfo *sn = &state->sn;
     Error *local_error = NULL;
 
-    if (!bs) {
+    if (!state->created) {
         return;
     }
 
     if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) {
-        error_report("Failed to delete snapshot with id '%s' and name '%s' on "
-                     "device '%s' in abort: %s",
-                     sn->id_str,
-                     sn->name,
-                     bdrv_get_device_name(bs),
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error,
+                          "Failed to delete snapshot with id '%s' and "
+                          "name '%s' on device '%s' in abort: ",
+                          sn->id_str, sn->name,
+                          bdrv_get_device_name(bs));
     }
 }
 
-static void internal_snapshot_clean(BlkTransactionState *common)
+static void internal_snapshot_clean(BlkActionState *common)
 {
     InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState,
                                              common, common);
 
     if (state->aio_context) {
+        if (state->bs) {
+            bdrv_drained_end(state->bs);
+        }
         aio_context_release(state->aio_context);
     }
 }
 
 /* external snapshot private data */
 typedef struct ExternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *old_bs;
     BlockDriverState *new_bs;
     AioContext *aio_context;
 } ExternalSnapshotState;
 
-static void external_snapshot_prepare(BlkTransactionState *common,
+static void external_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
-    BlockDriver *drv;
-    int flags, ret;
+    int flags = 0, ret;
     QDict *options = NULL;
     Error *local_err = NULL;
-    bool has_device = false;
+    /* Device and node name of the image to generate the snapshot from */
     const char *device;
-    bool has_node_name = false;
     const char *node_name;
-    bool has_snapshot_node_name = false;
-    const char *snapshot_node_name;
+    /* Reference to the new image (for 'blockdev-snapshot') */
+    const char *snapshot_ref;
+    /* File name of the new image (for 'blockdev-snapshot-sync') */
     const char *new_image_file;
-    const char *format = "qcow2";
-    enum NewImageMode mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     TransactionAction *action = common->action;
 
-    /* get parameters */
-    g_assert(action->kind == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC);
-
-    has_device = action->blockdev_snapshot_sync->has_device;
-    device = action->blockdev_snapshot_sync->device;
-    has_node_name = action->blockdev_snapshot_sync->has_node_name;
-    node_name = action->blockdev_snapshot_sync->node_name;
-    has_snapshot_node_name =
-        action->blockdev_snapshot_sync->has_snapshot_node_name;
-    snapshot_node_name = action->blockdev_snapshot_sync->snapshot_node_name;
-
-    new_image_file = action->blockdev_snapshot_sync->snapshot_file;
-    if (action->blockdev_snapshot_sync->has_format) {
-        format = action->blockdev_snapshot_sync->format;
-    }
-    if (action->blockdev_snapshot_sync->has_mode) {
-        mode = action->blockdev_snapshot_sync->mode;
+    /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar
+     * purpose but a different set of parameters */
+    switch (action->type) {
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT:
+        {
+            BlockdevSnapshot *s = action->u.blockdev_snapshot.data;
+            device = s->node;
+            node_name = s->node;
+            new_image_file = NULL;
+            snapshot_ref = s->overlay;
+        }
+        break;
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC:
+        {
+            BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync.data;
+            device = s->has_device ? s->device : NULL;
+            node_name = s->has_node_name ? s->node_name : NULL;
+            new_image_file = s->snapshot_file;
+            snapshot_ref = NULL;
+        }
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     /* start processing */
-    drv = bdrv_find_format(format);
-    if (!drv) {
-        error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
+    if (action_check_completion_mode(common, errp) < 0) {
         return;
     }
 
-    state->old_bs = bdrv_lookup_bs(has_device ? device : NULL,
-                                   has_node_name ? node_name : NULL,
-                                   &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    if (has_node_name && !has_snapshot_node_name) {
-        error_setg(errp, "New snapshot node name missing");
-        return;
-    }
-
-    if (has_snapshot_node_name && bdrv_find_node(snapshot_node_name)) {
-        error_setg(errp, "New snapshot node name already existing");
+    state->old_bs = bdrv_lookup_bs(device, node_name, errp);
+    if (!state->old_bs) {
         return;
     }
 
     /* Acquire AioContext now so any threads operating on old_bs stop */
     state->aio_context = bdrv_get_aio_context(state->old_bs);
     aio_context_acquire(state->aio_context);
+    bdrv_drained_begin(state->old_bs);
 
     if (!bdrv_is_inserted(state->old_bs)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
@@ -1503,38 +1728,85 @@ static void external_snapshot_prepare(BlkTransactionState *common,
         return;
     }
 
-    flags = state->old_bs->open_flags;
+    if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) {
+        BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync.data;
+        const char *format = s->has_format ? s->format : "qcow2";
+        enum NewImageMode mode;
+        const char *snapshot_node_name =
+            s->has_snapshot_node_name ? s->snapshot_node_name : NULL;
 
-    /* create new image w/backing file */
-    if (mode != NEW_IMAGE_MODE_EXISTING) {
-        bdrv_img_create(new_image_file, format,
-                        state->old_bs->filename,
-                        state->old_bs->drv->format_name,
-                        NULL, -1, flags, &local_err, false);
-        if (local_err) {
-            error_propagate(errp, local_err);
+        if (node_name && !snapshot_node_name) {
+            error_setg(errp, "New snapshot node name missing");
             return;
         }
-    }
 
-    if (has_snapshot_node_name) {
+        if (snapshot_node_name &&
+            bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {
+            error_setg(errp, "New snapshot node name already in use");
+            return;
+        }
+
+        flags = state->old_bs->open_flags;
+        flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ);
+
+        /* create new image w/backing file */
+        mode = s->has_mode ? s->mode : NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+        if (mode != NEW_IMAGE_MODE_EXISTING) {
+            int64_t size = bdrv_getlength(state->old_bs);
+            if (size < 0) {
+                error_setg_errno(errp, -size, "bdrv_getlength failed");
+                return;
+            }
+            bdrv_img_create(new_image_file, format,
+                            state->old_bs->filename,
+                            state->old_bs->drv->format_name,
+                            NULL, size, flags, &local_err, false);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+
         options = qdict_new();
-        qdict_put(options, "node-name",
-                  qstring_from_str(snapshot_node_name));
+        if (s->has_snapshot_node_name) {
+            qdict_put(options, "node-name",
+                      qstring_from_str(snapshot_node_name));
+        }
+        qdict_put(options, "driver", qstring_from_str(format));
+
+        flags |= BDRV_O_NO_BACKING;
     }
 
-    /* TODO Inherit bs->options or only take explicit options with an
-     * extended QMP command? */
     assert(state->new_bs == NULL);
-    ret = bdrv_open(&state->new_bs, new_image_file, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
+    ret = bdrv_open(&state->new_bs, new_image_file, snapshot_ref, options,
+                    flags, errp);
     /* We will manually add the backing_hd field to the bs later */
     if (ret != 0) {
-        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (state->new_bs->blk != NULL) {
+        error_setg(errp, "The snapshot is already in use by %s",
+                   blk_name(state->new_bs->blk));
+        return;
+    }
+
+    if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
+                           errp)) {
+        return;
+    }
+
+    if (state->new_bs->backing != NULL) {
+        error_setg(errp, "The snapshot already has a backing image");
+        return;
+    }
+
+    if (!state->new_bs->drv->supports_backing) {
+        error_setg(errp, "The snapshot does not support backing images");
     }
 }
 
-static void external_snapshot_commit(BlkTransactionState *common)
+static void external_snapshot_commit(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
@@ -1546,41 +1818,59 @@ static void external_snapshot_commit(BlkTransactionState *common)
     /* We don't need (or want) to use the transactional
      * bdrv_reopen_multiple() across all the entries at once, because we
      * don't want to abort all of them if one of them fails the reopen */
-    bdrv_reopen(state->new_bs, state->new_bs->open_flags & ~BDRV_O_RDWR,
-                NULL);
-
-    aio_context_release(state->aio_context);
+    if (!state->old_bs->copy_on_read) {
+        bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR,
+                    NULL);
+    }
 }
 
-static void external_snapshot_abort(BlkTransactionState *common)
+static void external_snapshot_abort(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->new_bs) {
         bdrv_unref(state->new_bs);
     }
+}
+
+static void external_snapshot_clean(BlkActionState *common)
+{
+    ExternalSnapshotState *state =
+                             DO_UPCAST(ExternalSnapshotState, common, common);
     if (state->aio_context) {
+        bdrv_drained_end(state->old_bs);
         aio_context_release(state->aio_context);
     }
 }
 
 typedef struct DriveBackupState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     AioContext *aio_context;
     BlockJob *job;
 } DriveBackupState;
 
-static void drive_backup_prepare(BlkTransactionState *common, Error **errp)
+static void do_drive_backup(const char *device, const char *target,
+                            bool has_format, const char *format,
+                            enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp);
+
+static void drive_backup_prepare(BlkActionState *common, Error **errp)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
-    BlockDriverState *bs;
     BlockBackend *blk;
     DriveBackup *backup;
     Error *local_err = NULL;
 
-    assert(common->action->kind == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
-    backup = common->action->drive_backup;
+    assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
+    backup = common->action->u.drive_backup.data;
 
     blk = blk_by_name(backup->device);
     if (!blk) {
@@ -1588,31 +1878,36 @@ static void drive_backup_prepare(BlkTransactionState *common, Error **errp)
                   "Device '%s' not found", backup->device);
         return;
     }
-    bs = blk_bs(blk);
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
+    state->aio_context = blk_get_aio_context(blk);
     aio_context_acquire(state->aio_context);
-
-    qmp_drive_backup(backup->device, backup->target,
-                     backup->has_format, backup->format,
-                     backup->sync,
-                     backup->has_mode, backup->mode,
-                     backup->has_speed, backup->speed,
-                     backup->has_bitmap, backup->bitmap,
-                     backup->has_on_source_error, backup->on_source_error,
-                     backup->has_on_target_error, backup->on_target_error,
-                     &local_err);
+    bdrv_drained_begin(blk_bs(blk));
+    state->bs = blk_bs(blk);
+
+    do_drive_backup(backup->device, backup->target,
+                    backup->has_format, backup->format,
+                    backup->sync,
+                    backup->has_mode, backup->mode,
+                    backup->has_speed, backup->speed,
+                    backup->has_bitmap, backup->bitmap,
+                    backup->has_on_source_error, backup->on_source_error,
+                    backup->has_on_target_error, backup->on_target_error,
+                    common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
 
-    state->bs = bs;
     state->job = state->bs->job;
 }
 
-static void drive_backup_abort(BlkTransactionState *common)
+static void drive_backup_abort(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
     BlockDriverState *bs = state->bs;
@@ -1623,72 +1918,85 @@ static void drive_backup_abort(BlkTransactionState *common)
     }
 }
 
-static void drive_backup_clean(BlkTransactionState *common)
+static void drive_backup_clean(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
 
     if (state->aio_context) {
+        bdrv_drained_end(state->bs);
         aio_context_release(state->aio_context);
     }
 }
 
 typedef struct BlockdevBackupState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
     BlockJob *job;
     AioContext *aio_context;
 } BlockdevBackupState;
 
-static void blockdev_backup_prepare(BlkTransactionState *common, Error **errp)
+static void do_blockdev_backup(const char *device, const char *target,
+                               enum MirrorSyncMode sync,
+                               bool has_speed, int64_t speed,
+                               bool has_on_source_error,
+                               BlockdevOnError on_source_error,
+                               bool has_on_target_error,
+                               BlockdevOnError on_target_error,
+                               BlockJobTxn *txn, Error **errp);
+
+static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
     BlockdevBackup *backup;
-    BlockDriverState *bs, *target;
-    BlockBackend *blk;
+    BlockBackend *blk, *target;
     Error *local_err = NULL;
 
-    assert(common->action->kind == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
-    backup = common->action->blockdev_backup;
+    assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
+    backup = common->action->u.blockdev_backup.data;
 
     blk = blk_by_name(backup->device);
     if (!blk) {
         error_setg(errp, "Device '%s' not found", backup->device);
         return;
     }
-    bs = blk_bs(blk);
 
-    blk = blk_by_name(backup->target);
-    if (!blk) {
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
+
+    target = blk_by_name(backup->target);
+    if (!target) {
         error_setg(errp, "Device '%s' not found", backup->target);
         return;
     }
-    target = blk_bs(blk);
 
     /* AioContext is released in .clean() */
-    state->aio_context = bdrv_get_aio_context(bs);
-    if (state->aio_context != bdrv_get_aio_context(target)) {
+    state->aio_context = blk_get_aio_context(blk);
+    if (state->aio_context != blk_get_aio_context(target)) {
         state->aio_context = NULL;
         error_setg(errp, "Backup between two IO threads is not implemented");
         return;
     }
     aio_context_acquire(state->aio_context);
-
-    qmp_blockdev_backup(backup->device, backup->target,
-                        backup->sync,
-                        backup->has_speed, backup->speed,
-                        backup->has_on_source_error, backup->on_source_error,
-                        backup->has_on_target_error, backup->on_target_error,
-                        &local_err);
+    state->bs = blk_bs(blk);
+    bdrv_drained_begin(state->bs);
+
+    do_blockdev_backup(backup->device, backup->target,
+                       backup->sync,
+                       backup->has_speed, backup->speed,
+                       backup->has_on_source_error, backup->on_source_error,
+                       backup->has_on_target_error, backup->on_target_error,
+                       common->block_job_txn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
 
-    state->bs = bs;
     state->job = state->bs->job;
 }
 
-static void blockdev_backup_abort(BlkTransactionState *common)
+static void blockdev_backup_abort(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
     BlockDriverState *bs = state->bs;
@@ -1699,31 +2007,148 @@ static void blockdev_backup_abort(BlkTransactionState *common)
     }
 }
 
-static void blockdev_backup_clean(BlkTransactionState *common)
+static void blockdev_backup_clean(BlkActionState *common)
 {
     BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
 
     if (state->aio_context) {
+        bdrv_drained_end(state->bs);
         aio_context_release(state->aio_context);
     }
 }
 
-static void abort_prepare(BlkTransactionState *common, Error **errp)
+typedef struct BlockDirtyBitmapState {
+    BlkActionState common;
+    BdrvDirtyBitmap *bitmap;
+    BlockDriverState *bs;
+    AioContext *aio_context;
+    HBitmap *backup;
+    bool prepared;
+} BlockDirtyBitmapState;
+
+static void block_dirty_bitmap_add_prepare(BlkActionState *common,
+                                           Error **errp)
+{
+    Error *local_err = NULL;
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_add.data;
+    /* AIO context taken and released within qmp_block_dirty_bitmap_add */
+    qmp_block_dirty_bitmap_add(action->node, action->name,
+                               action->has_granularity, action->granularity,
+                               &local_err);
+
+    if (!local_err) {
+        state->prepared = true;
+    } else {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void block_dirty_bitmap_add_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    action = common->action->u.block_dirty_bitmap_add.data;
+    /* Should not be able to fail: IF the bitmap was added via .prepare(),
+     * then the node reference and bitmap name must have been valid.
+     */
+    if (state->prepared) {
+        qmp_block_dirty_bitmap_remove(action->node, action->name, &error_abort);
+    }
+}
+
+static void block_dirty_bitmap_clear_prepare(BlkActionState *common,
+                                             Error **errp)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+    BlockDirtyBitmap *action;
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_clear.data;
+    state->bitmap = block_dirty_bitmap_lookup(action->node,
+                                              action->name,
+                                              &state->bs,
+                                              &state->aio_context,
+                                              errp);
+    if (!state->bitmap) {
+        return;
+    }
+
+    if (bdrv_dirty_bitmap_frozen(state->bitmap)) {
+        error_setg(errp, "Cannot modify a frozen bitmap");
+        return;
+    } else if (!bdrv_dirty_bitmap_enabled(state->bitmap)) {
+        error_setg(errp, "Cannot clear a disabled bitmap");
+        return;
+    }
+
+    bdrv_clear_dirty_bitmap(state->bitmap, &state->backup);
+    /* AioContext is released in .clean() */
+}
+
+static void block_dirty_bitmap_clear_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    bdrv_undo_clear_dirty_bitmap(state->bitmap, state->backup);
+}
+
+static void block_dirty_bitmap_clear_commit(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    hbitmap_free(state->backup);
+}
+
+static void block_dirty_bitmap_clear_clean(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (state->aio_context) {
+        aio_context_release(state->aio_context);
+    }
+}
+
+static void abort_prepare(BlkActionState *common, Error **errp)
 {
     error_setg(errp, "Transaction aborted using Abort action");
 }
 
-static void abort_commit(BlkTransactionState *common)
+static void abort_commit(BlkActionState *common)
 {
     g_assert_not_reached(); /* this action never succeeds */
 }
 
-static const BdrvActionOps actions[] = {
+static const BlkActionOps actions[] = {
+    [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT] = {
+        .instance_size = sizeof(ExternalSnapshotState),
+        .prepare  = external_snapshot_prepare,
+        .commit   = external_snapshot_commit,
+        .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
+    },
     [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC] = {
         .instance_size = sizeof(ExternalSnapshotState),
         .prepare  = external_snapshot_prepare,
         .commit   = external_snapshot_commit,
         .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
     },
     [TRANSACTION_ACTION_KIND_DRIVE_BACKUP] = {
         .instance_size = sizeof(DriveBackupState),
@@ -1738,7 +2163,7 @@ static const BdrvActionOps actions[] = {
         .clean = blockdev_backup_clean,
     },
     [TRANSACTION_ACTION_KIND_ABORT] = {
-        .instance_size = sizeof(BlkTransactionState),
+        .instance_size = sizeof(BlkActionState),
         .prepare = abort_prepare,
         .commit = abort_commit,
     },
@@ -1748,40 +2173,85 @@ static const BdrvActionOps actions[] = {
         .abort = internal_snapshot_abort,
         .clean = internal_snapshot_clean,
     },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_ADD] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_add_prepare,
+        .abort = block_dirty_bitmap_add_abort,
+    },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_CLEAR] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_clear_prepare,
+        .commit = block_dirty_bitmap_clear_commit,
+        .abort = block_dirty_bitmap_clear_abort,
+        .clean = block_dirty_bitmap_clear_clean,
+    }
 };
 
+/**
+ * Allocate a TransactionProperties structure if necessary, and fill
+ * that structure with desired defaults if they are unset.
+ */
+static TransactionProperties *get_transaction_properties(
+    TransactionProperties *props)
+{
+    if (!props) {
+        props = g_new0(TransactionProperties, 1);
+    }
+
+    if (!props->has_completion_mode) {
+        props->has_completion_mode = true;
+        props->completion_mode = ACTION_COMPLETION_MODE_INDIVIDUAL;
+    }
+
+    return props;
+}
+
 /*
  * 'Atomic' group operations.  The operations are performed as a set, and if
  * any fail then we roll back all operations in the group.
  */
-void qmp_transaction(TransactionActionList *dev_list, Error **errp)
+void qmp_transaction(TransactionActionList *dev_list,
+                     bool has_props,
+                     struct TransactionProperties *props,
+                     Error **errp)
 {
     TransactionActionList *dev_entry = dev_list;
-    BlkTransactionState *state, *next;
+    BlockJobTxn *block_job_txn = NULL;
+    BlkActionState *state, *next;
     Error *local_err = NULL;
 
-    QSIMPLEQ_HEAD(snap_bdrv_states, BlkTransactionState) snap_bdrv_states;
+    QSIMPLEQ_HEAD(snap_bdrv_states, BlkActionState) snap_bdrv_states;
     QSIMPLEQ_INIT(&snap_bdrv_states);
 
+    /* Does this transaction get canceled as a group on failure?
+     * If not, we don't really need to make a BlockJobTxn.
+     */
+    props = get_transaction_properties(props);
+    if (props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        block_job_txn = block_job_txn_new();
+    }
+
     /* drain all i/o before any operations */
     bdrv_drain_all();
 
     /* We don't do anything in this loop that commits us to the operations */
     while (NULL != dev_entry) {
         TransactionAction *dev_info = NULL;
-        const BdrvActionOps *ops;
+        const BlkActionOps *ops;
 
         dev_info = dev_entry->value;
         dev_entry = dev_entry->next;
 
-        assert(dev_info->kind < ARRAY_SIZE(actions));
+        assert(dev_info->type < ARRAY_SIZE(actions));
 
-        ops = &actions[dev_info->kind];
+        ops = &actions[dev_info->type];
         assert(ops->instance_size > 0);
 
         state = g_malloc0(ops->instance_size);
         state->ops = ops;
         state->action = dev_info;
+        state->block_job_txn = block_job_txn;
+        state->txn_props = props;
         QSIMPLEQ_INSERT_TAIL(&snap_bdrv_states, state, entry);
 
         state->ops->prepare(state, &local_err);
@@ -1814,42 +2284,91 @@ exit:
         }
         g_free(state);
     }
+    if (!has_props) {
+        qapi_free_TransactionProperties(props);
+    }
+    block_job_txn_unref(block_job_txn);
 }
 
+void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
+{
+    Error *local_err = NULL;
+
+    qmp_blockdev_open_tray(device, has_force, force, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    qmp_x_blockdev_remove_medium(device, errp);
+}
 
-static void eject_device(BlockBackend *blk, int force, Error **errp)
+void qmp_block_passwd(bool has_device, const char *device,
+                      bool has_node_name, const char *node_name,
+                      const char *password, Error **errp)
 {
-    BlockDriverState *bs = blk_bs(blk);
+    Error *local_err = NULL;
+    BlockDriverState *bs;
     AioContext *aio_context;
 
+    bs = bdrv_lookup_bs(has_device ? device : NULL,
+                        has_node_name ? node_name : NULL,
+                        &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     aio_context = bdrv_get_aio_context(bs);
     aio_context_acquire(aio_context);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
-        goto out;
+    bdrv_add_key(bs, password, errp);
+
+    aio_context_release(aio_context);
+}
+
+void qmp_blockdev_open_tray(const char *device, bool has_force, bool force,
+                            Error **errp)
+{
+    BlockBackend *blk;
+    bool locked;
+
+    if (!has_force) {
+        force = false;
     }
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
     if (!blk_dev_has_removable_media(blk)) {
-        error_setg(errp, "Device '%s' is not removable",
-                   bdrv_get_device_name(bs));
-        goto out;
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
     }
 
-    if (blk_dev_is_medium_locked(blk) && !blk_dev_is_tray_open(blk)) {
-        blk_dev_eject_request(blk, force);
-        if (!force) {
-            error_setg(errp, "Device '%s' is locked",
-                       bdrv_get_device_name(bs));
-            goto out;
-        }
+    if (!blk_dev_has_tray(blk)) {
+        /* Ignore this command on tray-less devices */
+        return;
     }
 
-    bdrv_close(bs);
+    if (blk_dev_is_tray_open(blk)) {
+        return;
+    }
 
-out:
-    aio_context_release(aio_context);
+    locked = blk_dev_is_medium_locked(blk);
+    if (locked) {
+        blk_dev_eject_request(blk, force);
+    }
+
+    if (!locked || force) {
+        blk_dev_change_media_cb(blk, false);
+    }
 }
 
-void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
+void qmp_blockdev_close_tray(const char *device, Error **errp)
 {
     BlockBackend *blk;
 
@@ -1860,92 +2379,230 @@ void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
         return;
     }
 
-    eject_device(blk, force, errp);
+    if (!blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (!blk_dev_has_tray(blk)) {
+        /* Ignore this command on tray-less devices */
+        return;
+    }
+
+    if (!blk_dev_is_tray_open(blk)) {
+        return;
+    }
+
+    blk_dev_change_media_cb(blk, true);
 }
 
-void qmp_block_passwd(bool has_device, const char *device,
-                      bool has_node_name, const char *node_name,
-                      const char *password, Error **errp)
+void qmp_x_blockdev_remove_medium(const char *device, Error **errp)
 {
-    Error *local_err = NULL;
+    BlockBackend *blk;
     BlockDriverState *bs;
     AioContext *aio_context;
+    bool has_device;
 
-    bs = bdrv_lookup_bs(has_device ? device : NULL,
-                        has_node_name ? node_name : NULL,
-                        &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    bs = blk_bs(blk);
+    if (!bs) {
         return;
     }
 
     aio_context = bdrv_get_aio_context(bs);
     aio_context_acquire(aio_context);
 
-    bdrv_add_key(bs, password, errp);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
+        goto out;
+    }
+
+    blk_remove_bs(blk);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-open-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be ejected here.
+         * Do it after blk_remove_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. false). */
+        blk_dev_change_media_cb(blk, false);
+    }
 
+out:
     aio_context_release(aio_context);
 }
 
-/* Assumes AioContext is held */
-static void qmp_bdrv_open_encrypted(BlockDriverState *bs, const char *filename,
-                                    int bdrv_flags, BlockDriver *drv,
-                                    const char *password, Error **errp)
+static void qmp_blockdev_insert_anon_medium(const char *device,
+                                            BlockDriverState *bs, Error **errp)
 {
-    Error *local_err = NULL;
-    int ret;
+    BlockBackend *blk;
+    bool has_device;
 
-    ret = bdrv_open(&bs, filename, NULL, NULL, bdrv_flags, drv, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    bdrv_add_key(bs, password, errp);
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    if (blk_bs(blk)) {
+        error_setg(errp, "There already is a medium in device '%s'", device);
+        return;
+    }
+
+    blk_insert_bs(blk, bs);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-close-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be pushed into the
+         * slot here.
+         * Do it after blk_insert_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. true). */
+        blk_dev_change_media_cb(blk, true);
+    }
 }
 
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp)
+void qmp_x_blockdev_insert_medium(const char *device, const char *node_name,
+                                  Error **errp)
 {
-    BlockBackend *blk;
     BlockDriverState *bs;
-    AioContext *aio_context;
-    BlockDriver *drv = NULL;
-    int bdrv_flags;
+
+    bs = bdrv_find_node(node_name);
+    if (!bs) {
+        error_setg(errp, "Node '%s' not found", node_name);
+        return;
+    }
+
+    if (bs->blk) {
+        error_setg(errp, "Node '%s' is already in use by '%s'", node_name,
+                   blk_name(bs->blk));
+        return;
+    }
+
+    qmp_blockdev_insert_anon_medium(device, bs, errp);
+}
+
+void qmp_blockdev_change_medium(const char *device, const char *filename,
+                                bool has_format, const char *format,
+                                bool has_read_only,
+                                BlockdevChangeReadOnlyMode read_only,
+                                Error **errp)
+{
+    BlockBackend *blk;
+    BlockDriverState *medium_bs = NULL;
+    int bdrv_flags, ret;
+    QDict *options = NULL;
     Error *err = NULL;
 
     blk = blk_by_name(device);
     if (!blk) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", device);
-        return;
+        goto fail;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (blk_bs(blk)) {
+        blk_update_root_state(blk);
+    }
 
-    if (format) {
-        drv = bdrv_find_whitelisted_format(format, bs->read_only);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
+    bdrv_flags = blk_get_open_flags_from_root_state(blk);
+    bdrv_flags &= ~(BDRV_O_TEMPORARY | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING |
+        BDRV_O_PROTOCOL);
+
+    if (!has_read_only) {
+        read_only = BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN;
     }
 
-    eject_device(blk, 0, &err);
+    switch (read_only) {
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN:
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_ONLY:
+        bdrv_flags &= ~BDRV_O_RDWR;
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_WRITE:
+        bdrv_flags |= BDRV_O_RDWR;
+        break;
+
+    default:
+        abort();
+    }
+
+    if (has_format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
+    assert(!medium_bs);
+    ret = bdrv_open(&medium_bs, filename, NULL, options, bdrv_flags, errp);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    blk_apply_root_state(blk, medium_bs);
+
+    bdrv_add_key(medium_bs, NULL, &err);
     if (err) {
         error_propagate(errp, err);
-        goto out;
+        goto fail;
     }
 
-    bdrv_flags = bdrv_is_read_only(bs) ? 0 : BDRV_O_RDWR;
-    bdrv_flags |= bdrv_is_snapshot(bs) ? BDRV_O_SNAPSHOT : 0;
+    qmp_blockdev_open_tray(device, false, false, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
 
-    qmp_bdrv_open_encrypted(bs, filename, bdrv_flags, drv, NULL, errp);
+    qmp_x_blockdev_remove_medium(device, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
 
-out:
-    aio_context_release(aio_context);
+    qmp_blockdev_insert_anon_medium(device, medium_bs, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+
+    qmp_blockdev_close_tray(device, errp);
+
+fail:
+    /* If the medium has been inserted, the device has its own reference, so
+     * ours must be relinquished; and if it has not been inserted successfully,
+     * the reference must be relinquished anyway */
+    bdrv_unref(medium_bs);
 }
 
 /* throttling disk I/O limits */
@@ -1966,6 +2623,18 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
                                int64_t iops_rd_max,
                                bool has_iops_wr_max,
                                int64_t iops_wr_max,
+                               bool has_bps_max_length,
+                               int64_t bps_max_length,
+                               bool has_bps_rd_max_length,
+                               int64_t bps_rd_max_length,
+                               bool has_bps_wr_max_length,
+                               int64_t bps_wr_max_length,
+                               bool has_iops_max_length,
+                               int64_t iops_max_length,
+                               bool has_iops_rd_max_length,
+                               int64_t iops_rd_max_length,
+                               bool has_iops_wr_max_length,
+                               int64_t iops_wr_max_length,
                                bool has_iops_size,
                                int64_t iops_size,
                                bool has_group,
@@ -1982,9 +2651,24 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
                   "Device '%s' not found", device);
         return;
     }
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
     bs = blk_bs(blk);
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
 
-    memset(&cfg, 0, sizeof(cfg));
+    /* The BlockBackend must be the only parent */
+    assert(QLIST_FIRST(&bs->parents));
+    if (QLIST_NEXT(QLIST_FIRST(&bs->parents), next_parent)) {
+        error_setg(errp, "Cannot throttle device with multiple parents");
+        goto out;
+    }
+
+    throttle_config_init(&cfg);
     cfg.buckets[THROTTLE_BPS_TOTAL].avg = bps;
     cfg.buckets[THROTTLE_BPS_READ].avg  = bps_rd;
     cfg.buckets[THROTTLE_BPS_WRITE].avg = bps_wr;
@@ -2012,32 +2696,49 @@ void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
         cfg.buckets[THROTTLE_OPS_WRITE].max = iops_wr_max;
     }
 
+    if (has_bps_max_length) {
+        cfg.buckets[THROTTLE_BPS_TOTAL].burst_length = bps_max_length;
+    }
+    if (has_bps_rd_max_length) {
+        cfg.buckets[THROTTLE_BPS_READ].burst_length = bps_rd_max_length;
+    }
+    if (has_bps_wr_max_length) {
+        cfg.buckets[THROTTLE_BPS_WRITE].burst_length = bps_wr_max_length;
+    }
+    if (has_iops_max_length) {
+        cfg.buckets[THROTTLE_OPS_TOTAL].burst_length = iops_max_length;
+    }
+    if (has_iops_rd_max_length) {
+        cfg.buckets[THROTTLE_OPS_READ].burst_length = iops_rd_max_length;
+    }
+    if (has_iops_wr_max_length) {
+        cfg.buckets[THROTTLE_OPS_WRITE].burst_length = iops_wr_max_length;
+    }
+
     if (has_iops_size) {
         cfg.op_size = iops_size;
     }
 
-    if (!check_throttle_config(&cfg, errp)) {
-        return;
+    if (!throttle_is_valid(&cfg, errp)) {
+        goto out;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
-
     if (throttle_enabled(&cfg)) {
         /* Enable I/O limits if they're not enabled yet, otherwise
          * just update the throttling group. */
-        if (!bs->io_limits_enabled) {
+        if (!bs->throttle_state) {
             bdrv_io_limits_enable(bs, has_group ? group : device);
         } else if (has_group) {
             bdrv_io_limits_update_group(bs, group);
         }
         /* Set the new throttling configuration */
         bdrv_set_io_limits(bs, &cfg);
-    } else if (bs->io_limits_enabled) {
+    } else if (bs->throttle_state) {
         /* If all throttling settings are set to 0, disable I/O limits */
         bdrv_io_limits_disable(bs);
     }
 
+out:
     aio_context_release(aio_context);
 }
 
@@ -2131,7 +2832,7 @@ void qmp_block_dirty_bitmap_clear(const char *node, const char *name,
         goto out;
     }
 
-    bdrv_clear_dirty_bitmap(bitmap);
+    bdrv_clear_dirty_bitmap(bitmap, NULL);
 
  out:
     aio_context_release(aio_context);
@@ -2145,12 +2846,20 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict)
     AioContext *aio_context;
     Error *local_err = NULL;
 
+    bs = bdrv_find_node(id);
+    if (bs) {
+        qmp_x_blockdev_del(false, NULL, true, id, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+        }
+        return;
+    }
+
     blk = blk_by_name(id);
     if (!blk) {
         error_report("Device '%s' not found", id);
         return;
     }
-    bs = blk_bs(blk);
 
     if (!blk_legacy_dinfo(blk)) {
         error_report("Deleting device added with blockdev-add"
@@ -2158,27 +2867,30 @@ void hmp_drive_del(Monitor *mon, const QDict *qdict)
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
-        error_report_err(local_err);
-        aio_context_release(aio_context);
-        return;
+    bs = blk_bs(blk);
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
+            error_report_err(local_err);
+            aio_context_release(aio_context);
+            return;
+        }
+
+        blk_remove_bs(blk);
     }
 
-    bdrv_close(bs);
+    /* Make the BlockBackend and the attached BlockDriverState anonymous */
+    monitor_remove_blk(blk);
 
-    /* if we have a device attached to this BlockDriverState
-     * then we need to make the drive anonymous until the device
-     * can be removed.  If this is a drive with no device backing
-     * then we can just get rid of the block driver state right here.
+    /* If this BlockBackend has a device attached to it, its refcount will be
+     * decremented when the device is removed; otherwise we have to do so here.
      */
     if (blk_get_attached_dev(blk)) {
-        blk_hide_on_behalf_of_hmp_drive_del(blk);
         /* Further I/O must not pause the guest */
-        bdrv_set_on_error(bs, BLOCKDEV_ON_ERROR_REPORT,
-                          BLOCKDEV_ON_ERROR_REPORT);
+        blk_set_on_error(blk, BLOCKDEV_ON_ERROR_REPORT,
+                         BLOCKDEV_ON_ERROR_REPORT);
     } else {
         blk_unref(blk);
     }
@@ -2272,8 +2984,6 @@ static void block_job_cb(void *opaque, int ret)
     } else {
         block_job_event_completed(bs->job, msg);
     }
-
-    bdrv_put_ref_bh_schedule(bs);
 }
 
 void qmp_block_stream(const char *device,
@@ -2300,11 +3010,16 @@ void qmp_block_stream(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_STREAM, errp)) {
         goto out;
     }
@@ -2375,11 +3090,16 @@ void qmp_block_commit(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, errp)) {
         goto out;
     }
@@ -2444,15 +3164,17 @@ out:
     aio_context_release(aio_context);
 }
 
-void qmp_drive_backup(const char *device, const char *target,
-                      bool has_format, const char *format,
-                      enum MirrorSyncMode sync,
-                      bool has_mode, enum NewImageMode mode,
-                      bool has_speed, int64_t speed,
-                      bool has_bitmap, const char *bitmap,
-                      bool has_on_source_error, BlockdevOnError on_source_error,
-                      bool has_on_target_error, BlockdevOnError on_target_error,
-                      Error **errp)
+static void do_drive_backup(const char *device, const char *target,
+                            bool has_format, const char *format,
+                            enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp)
 {
     BlockBackend *blk;
     BlockDriverState *bs;
@@ -2460,7 +3182,7 @@ void qmp_drive_backup(const char *device, const char *target,
     BlockDriverState *source = NULL;
     BdrvDirtyBitmap *bmap = NULL;
     AioContext *aio_context;
-    BlockDriver *drv = NULL;
+    QDict *options = NULL;
     Error *local_err = NULL;
     int flags;
     int64_t size;
@@ -2485,28 +3207,21 @@ void qmp_drive_backup(const char *device, const char *target,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
     /* Although backup_run has this check too, we need to use bs->drv below, so
      * do an early check redundantly. */
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         goto out;
     }
+    bs = blk_bs(blk);
 
     if (!has_format) {
         format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
     }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
-    }
 
     /* Early check to avoid creating target */
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
@@ -2518,7 +3233,7 @@ void qmp_drive_backup(const char *device, const char *target,
     /* See if we have a backing HD we can use to create our new image
      * on top of. */
     if (sync == MIRROR_SYNC_MODE_TOP) {
-        source = bs->backing_hd;
+        source = backing_bs(bs);
         if (!source) {
             sync = MIRROR_SYNC_MODE_FULL;
         }
@@ -2534,7 +3249,7 @@ void qmp_drive_backup(const char *device, const char *target,
     }
 
     if (mode != NEW_IMAGE_MODE_EXISTING) {
-        assert(format && drv);
+        assert(format);
         if (source) {
             bdrv_img_create(target, format, source->filename,
                             source->drv->format_name, NULL,
@@ -2550,8 +3265,13 @@ void qmp_drive_backup(const char *device, const char *target,
         goto out;
     }
 
+    if (format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
     target_bs = NULL;
-    ret = bdrv_open(&target_bs, target, NULL, NULL, flags, drv, &local_err);
+    ret = bdrv_open(&target_bs, target, NULL, options, flags, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -2563,13 +3283,14 @@ void qmp_drive_backup(const char *device, const char *target,
         bmap = bdrv_find_dirty_bitmap(bs, bitmap);
         if (!bmap) {
             error_setg(errp, "Bitmap '%s' could not be found", bitmap);
+            bdrv_unref(target_bs);
             goto out;
         }
     }
 
     backup_start(bs, target_bs, speed, sync, bmap,
                  on_source_error, on_target_error,
-                 block_job_cb, bs, &local_err);
+                 block_job_cb, bs, txn, &local_err);
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
@@ -2580,21 +3301,39 @@ out:
     aio_context_release(aio_context);
 }
 
+void qmp_drive_backup(const char *device, const char *target,
+                      bool has_format, const char *format,
+                      enum MirrorSyncMode sync,
+                      bool has_mode, enum NewImageMode mode,
+                      bool has_speed, int64_t speed,
+                      bool has_bitmap, const char *bitmap,
+                      bool has_on_source_error, BlockdevOnError on_source_error,
+                      bool has_on_target_error, BlockdevOnError on_target_error,
+                      Error **errp)
+{
+    return do_drive_backup(device, target, has_format, format, sync,
+                           has_mode, mode, has_speed, speed,
+                           has_bitmap, bitmap,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           NULL, errp);
+}
+
 BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)
 {
     return bdrv_named_nodes_list(errp);
 }
 
-void qmp_blockdev_backup(const char *device, const char *target,
+void do_blockdev_backup(const char *device, const char *target,
                          enum MirrorSyncMode sync,
                          bool has_speed, int64_t speed,
                          bool has_on_source_error,
                          BlockdevOnError on_source_error,
                          bool has_on_target_error,
                          BlockdevOnError on_target_error,
-                         Error **errp)
+                         BlockJobTxn *txn, Error **errp)
 {
-    BlockBackend *blk;
+    BlockBackend *blk, *target_blk;
     BlockDriverState *bs;
     BlockDriverState *target_bs;
     Error *local_err = NULL;
@@ -2615,22 +3354,32 @@ void qmp_blockdev_backup(const char *device, const char *target,
         error_setg(errp, "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    blk = blk_by_name(target);
-    if (!blk) {
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
+    target_blk = blk_by_name(target);
+    if (!target_blk) {
         error_setg(errp, "Device '%s' not found", target);
         goto out;
     }
-    target_bs = blk_bs(blk);
+
+    if (!blk_is_available(target_blk)) {
+        error_setg(errp, "Device '%s' has no medium", target);
+        goto out;
+    }
+    target_bs = blk_bs(target_blk);
 
     bdrv_ref(target_bs);
     bdrv_set_aio_context(target_bs, aio_context);
     backup_start(bs, target_bs, speed, sync, NULL, on_source_error,
-                 on_target_error, block_job_cb, bs, &local_err);
+                 on_target_error, block_job_cb, bs, txn, &local_err);
     if (local_err != NULL) {
         bdrv_unref(target_bs);
         error_propagate(errp, local_err);
@@ -2639,30 +3388,38 @@ out:
     aio_context_release(aio_context);
 }
 
-void qmp_drive_mirror(const char *device, const char *target,
-                      bool has_format, const char *format,
-                      bool has_node_name, const char *node_name,
-                      bool has_replaces, const char *replaces,
-                      enum MirrorSyncMode sync,
-                      bool has_mode, enum NewImageMode mode,
-                      bool has_speed, int64_t speed,
-                      bool has_granularity, uint32_t granularity,
-                      bool has_buf_size, int64_t buf_size,
-                      bool has_on_source_error, BlockdevOnError on_source_error,
-                      bool has_on_target_error, BlockdevOnError on_target_error,
-                      bool has_unmap, bool unmap,
-                      Error **errp)
+void qmp_blockdev_backup(const char *device, const char *target,
+                         enum MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         Error **errp)
+{
+    do_blockdev_backup(device, target, sync, has_speed, speed,
+                       has_on_source_error, on_source_error,
+                       has_on_target_error, on_target_error,
+                       NULL, errp);
+}
+
+/* Parameter check and block job starting for drive mirroring.
+ * Caller should hold @device and @target's aio context (must be the same).
+ **/
+static void blockdev_mirror_common(BlockDriverState *bs,
+                                   BlockDriverState *target,
+                                   bool has_replaces, const char *replaces,
+                                   enum MirrorSyncMode sync,
+                                   bool has_speed, int64_t speed,
+                                   bool has_granularity, uint32_t granularity,
+                                   bool has_buf_size, int64_t buf_size,
+                                   bool has_on_source_error,
+                                   BlockdevOnError on_source_error,
+                                   bool has_on_target_error,
+                                   BlockdevOnError on_target_error,
+                                   bool has_unmap, bool unmap,
+                                   Error **errp)
 {
-    BlockBackend *blk;
-    BlockDriverState *bs;
-    BlockDriverState *source, *target_bs;
-    AioContext *aio_context;
-    BlockDriver *drv = NULL;
-    Error *local_err = NULL;
-    QDict *options = NULL;
-    int flags;
-    int64_t size;
-    int ret;
 
     if (!has_speed) {
         speed = 0;
@@ -2673,9 +3430,6 @@ void qmp_drive_mirror(const char *device, const char *target,
     if (!has_on_target_error) {
         on_target_error = BLOCKDEV_ON_ERROR_REPORT;
     }
-    if (!has_mode) {
-        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
-    }
     if (!has_granularity) {
         granularity = 0;
     }
@@ -2697,39 +3451,80 @@ void qmp_drive_mirror(const char *device, const char *target,
         return;
     }
 
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR_SOURCE, errp)) {
+        return;
+    }
+    if (bdrv_op_is_blocked(target, BLOCK_OP_TYPE_MIRROR_TARGET, errp)) {
+        return;
+    }
+    if (target->blk) {
+        error_setg(errp, "Cannot mirror to an attached block device");
+        return;
+    }
+
+    if (!bs->backing && sync == MIRROR_SYNC_MODE_TOP) {
+        sync = MIRROR_SYNC_MODE_FULL;
+    }
+
+    /* pass the node name to replace to mirror start since it's loose coupling
+     * and will allow to check whether the node still exist at mirror completion
+     */
+    mirror_start(bs, target,
+                 has_replaces ? replaces : NULL,
+                 speed, granularity, buf_size, sync,
+                 on_source_error, on_target_error, unmap,
+                 block_job_cb, bs, errp);
+}
+
+void qmp_drive_mirror(const char *device, const char *target,
+                      bool has_format, const char *format,
+                      bool has_node_name, const char *node_name,
+                      bool has_replaces, const char *replaces,
+                      enum MirrorSyncMode sync,
+                      bool has_mode, enum NewImageMode mode,
+                      bool has_speed, int64_t speed,
+                      bool has_granularity, uint32_t granularity,
+                      bool has_buf_size, int64_t buf_size,
+                      bool has_on_source_error, BlockdevOnError on_source_error,
+                      bool has_on_target_error, BlockdevOnError on_target_error,
+                      bool has_unmap, bool unmap,
+                      Error **errp)
+{
+    BlockDriverState *bs;
+    BlockBackend *blk;
+    BlockDriverState *source, *target_bs;
+    AioContext *aio_context;
+    Error *local_err = NULL;
+    QDict *options = NULL;
+    int flags;
+    int64_t size;
+    int ret;
+
     blk = blk_by_name(device);
     if (!blk) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
+    if (!blk_is_available(blk)) {
         error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         goto out;
     }
+    bs = blk_bs(blk);
+    if (!has_mode) {
+        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+    }
 
     if (!has_format) {
         format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
     }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_setg(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
-    }
-
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
-        goto out;
-    }
 
     flags = bs->open_flags | BDRV_O_RDWR;
-    source = bs->backing_hd;
+    source = backing_bs(bs);
     if (!source && sync == MIRROR_SYNC_MODE_TOP) {
         sync = MIRROR_SYNC_MODE_FULL;
     }
@@ -2754,7 +3549,7 @@ void qmp_drive_mirror(const char *device, const char *target,
             goto out;
         }
 
-        to_replace_bs = check_to_replace_node(replaces, &local_err);
+        to_replace_bs = check_to_replace_node(bs, replaces, &local_err);
 
         if (!to_replace_bs) {
             error_propagate(errp, local_err);
@@ -2777,7 +3572,7 @@ void qmp_drive_mirror(const char *device, const char *target,
         && mode != NEW_IMAGE_MODE_EXISTING)
     {
         /* create new image w/o backing file */
-        assert(format && drv);
+        assert(format);
         bdrv_img_create(target, format,
                         NULL, NULL, NULL, size, flags, &local_err, false);
     } else {
@@ -2801,17 +3596,20 @@ void qmp_drive_mirror(const char *device, const char *target,
         goto out;
     }
 
+    options = qdict_new();
     if (has_node_name) {
-        options = qdict_new();
         qdict_put(options, "node-name", qstring_from_str(node_name));
     }
+    if (format) {
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
 
     /* Mirroring takes care of copy-on-write using the source's backing
      * file.
      */
     target_bs = NULL;
     ret = bdrv_open(&target_bs, target, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
+                    flags | BDRV_O_NO_BACKING, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
@@ -2819,22 +3617,78 @@ void qmp_drive_mirror(const char *device, const char *target,
 
     bdrv_set_aio_context(target_bs, aio_context);
 
-    /* pass the node name to replace to mirror start since it's loose coupling
-     * and will allow to check whether the node still exist at mirror completion
-     */
-    mirror_start(bs, target_bs,
-                 has_replaces ? replaces : NULL,
-                 speed, granularity, buf_size, sync,
-                 on_source_error, on_target_error,
-                 unmap,
-                 block_job_cb, bs, &local_err);
-    if (local_err != NULL) {
+    blockdev_mirror_common(bs, target_bs,
+                           has_replaces, replaces, sync,
+                           has_speed, speed,
+                           has_granularity, granularity,
+                           has_buf_size, buf_size,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           has_unmap, unmap,
+                           &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         bdrv_unref(target_bs);
+    }
+out:
+    aio_context_release(aio_context);
+}
+
+void qmp_blockdev_mirror(const char *device, const char *target,
+                         bool has_replaces, const char *replaces,
+                         MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_granularity, uint32_t granularity,
+                         bool has_buf_size, int64_t buf_size,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         Error **errp)
+{
+    BlockDriverState *bs;
+    BlockBackend *blk;
+    BlockDriverState *target_bs;
+    AioContext *aio_context;
+    Error *local_err = NULL;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_setg(errp, "Device '%s' not found", device);
+        return;
+    }
+    bs = blk_bs(blk);
+
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no media", device);
+        return;
+    }
+
+    target_bs = bdrv_lookup_bs(target, target, errp);
+    if (!target_bs) {
+        return;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_ref(target_bs);
+    bdrv_set_aio_context(target_bs, aio_context);
+
+    blockdev_mirror_common(bs, target_bs,
+                           has_replaces, replaces, sync,
+                           has_speed, speed,
+                           has_granularity, granularity,
+                           has_buf_size, buf_size,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           true, true,
+                           &local_err);
+    if (local_err) {
         error_propagate(errp, local_err);
-        goto out;
+        bdrv_unref(target_bs);
     }
 
-out:
     aio_context_release(aio_context);
 }
 
@@ -2845,17 +3699,22 @@ static BlockJob *find_block_job(const char *device, AioContext **aio_context,
     BlockBackend *blk;
     BlockDriverState *bs;
 
+    *aio_context = NULL;
+
     blk = blk_by_name(device);
     if (!blk) {
         goto notfound;
     }
-    bs = blk_bs(blk);
 
-    *aio_context = bdrv_get_aio_context(bs);
+    *aio_context = blk_get_aio_context(blk);
     aio_context_acquire(*aio_context);
 
+    if (!blk_is_available(blk)) {
+        goto notfound;
+    }
+    bs = blk_bs(blk);
+
     if (!bs->job) {
-        aio_context_release(*aio_context);
         goto notfound;
     }
 
@@ -2864,7 +3723,10 @@ static BlockJob *find_block_job(const char *device, AioContext **aio_context,
 notfound:
     error_set(errp, ERROR_CLASS_DEVICE_NOT_ACTIVE,
               "No active block job on device '%s'", device);
-    *aio_context = NULL;
+    if (*aio_context) {
+        aio_context_release(*aio_context);
+        *aio_context = NULL;
+    }
     return NULL;
 }
 
@@ -2971,11 +3833,16 @@ void qmp_change_backing_file(const char *device,
                   "Device '%s' not found", device);
         return;
     }
-    bs = blk_bs(blk);
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     image_bs = bdrv_lookup_bs(NULL, image_node_name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -3039,20 +3906,46 @@ out:
     aio_context_release(aio_context);
 }
 
+void hmp_drive_add_node(Monitor *mon, const char *optstr)
+{
+    QemuOpts *opts;
+    QDict *qdict;
+    Error *local_err = NULL;
+
+    opts = qemu_opts_parse_noisily(&qemu_drive_opts, optstr, false);
+    if (!opts) {
+        return;
+    }
+
+    qdict = qemu_opts_to_qdict(opts, NULL);
+
+    if (!qdict_get_try_str(qdict, "node-name")) {
+        QDECREF(qdict);
+        error_report("'node-name' needs to be specified");
+        goto out;
+    }
+
+    BlockDriverState *bs = bds_tree_init(qdict, &local_err);
+    if (!bs) {
+        error_report_err(local_err);
+        goto out;
+    }
+
+    QTAILQ_INSERT_TAIL(&monitor_bdrv_states, bs, monitor_list);
+
+out:
+    qemu_opts_del(opts);
+}
+
 void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 {
     QmpOutputVisitor *ov = qmp_output_visitor_new();
-    BlockBackend *blk;
+    BlockDriverState *bs;
+    BlockBackend *blk = NULL;
     QObject *obj;
     QDict *qdict;
     Error *local_err = NULL;
 
-    /* Require an ID in the top level */
-    if (!options->has_id) {
-        error_setg(errp, "Block device needs an ID");
-        goto fail;
-    }
-
     /* TODO Sort it out in raw-posix and drive_new(): Reject aio=native with
      * cache.direct=false instead of silently switching to aio=threads, except
      * when called from drive_new().
@@ -3068,8 +3961,8 @@ void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
         }
     }
 
-    visit_type_BlockdevOptions(qmp_output_get_visitor(ov),
-                               &options, NULL, &local_err);
+    visit_type_BlockdevOptions(qmp_output_get_visitor(ov), NULL, &options,
+                               &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto fail;
@@ -3080,14 +3973,37 @@ void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 
     qdict_flatten(qdict);
 
-    blk = blockdev_init(NULL, qdict, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        goto fail;
+    if (options->has_id) {
+        blk = blockdev_init(NULL, qdict, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+
+        bs = blk_bs(blk);
+    } else {
+        if (!qdict_get_try_str(qdict, "node-name")) {
+            error_setg(errp, "'id' and/or 'node-name' need to be specified for "
+                       "the root node");
+            goto fail;
+        }
+
+        bs = bds_tree_init(qdict, errp);
+        if (!bs) {
+            goto fail;
+        }
+
+        QTAILQ_INSERT_TAIL(&monitor_bdrv_states, bs, monitor_list);
     }
 
-    if (bdrv_key_required(blk_bs(blk))) {
-        blk_unref(blk);
+    if (bs && bdrv_key_required(bs)) {
+        if (blk) {
+            monitor_remove_blk(blk);
+            blk_unref(blk);
+        } else {
+            QTAILQ_REMOVE(&monitor_bdrv_states, bs, monitor_list);
+            bdrv_unref(bs);
+        }
         error_setg(errp, "blockdev-add doesn't support encrypted devices");
         goto fail;
     }
@@ -3096,6 +4012,86 @@ fail:
     qmp_output_visitor_cleanup(ov);
 }
 
+void qmp_x_blockdev_del(bool has_id, const char *id,
+                        bool has_node_name, const char *node_name, Error **errp)
+{
+    AioContext *aio_context;
+    BlockBackend *blk;
+    BlockDriverState *bs;
+
+    if (has_id && has_node_name) {
+        error_setg(errp, "Only one of id and node-name must be specified");
+        return;
+    } else if (!has_id && !has_node_name) {
+        error_setg(errp, "No block device specified");
+        return;
+    }
+
+    if (has_id) {
+        /* blk_by_name() never returns a BB that is not owned by the monitor */
+        blk = blk_by_name(id);
+        if (!blk) {
+            error_setg(errp, "Cannot find block backend %s", id);
+            return;
+        }
+        if (blk_legacy_dinfo(blk)) {
+            error_setg(errp, "Deleting block backend added with drive-add"
+                       " is not supported");
+            return;
+        }
+        if (blk_get_refcnt(blk) > 1) {
+            error_setg(errp, "Block backend %s is in use", id);
+            return;
+        }
+        bs = blk_bs(blk);
+        aio_context = blk_get_aio_context(blk);
+    } else {
+        bs = bdrv_find_node(node_name);
+        if (!bs) {
+            error_setg(errp, "Cannot find node %s", node_name);
+            return;
+        }
+        blk = bs->blk;
+        if (blk) {
+            error_setg(errp, "Node %s is in use by %s",
+                       node_name, blk_name(blk));
+            return;
+        }
+        aio_context = bdrv_get_aio_context(bs);
+    }
+
+    aio_context_acquire(aio_context);
+
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, errp)) {
+            goto out;
+        }
+
+        if (!blk && !bs->monitor_list.tqe_prev) {
+            error_setg(errp, "Node %s is not owned by the monitor",
+                       bs->node_name);
+            goto out;
+        }
+
+        if (bs->refcnt > 1) {
+            error_setg(errp, "Block device %s is in use",
+                       bdrv_get_device_or_node_name(bs));
+            goto out;
+        }
+    }
+
+    if (blk) {
+        monitor_remove_blk(blk);
+        blk_unref(blk);
+    } else {
+        QTAILQ_REMOVE(&monitor_bdrv_states, bs, monitor_list);
+        bdrv_unref(bs);
+    }
+
+out:
+    aio_context_release(aio_context);
+}
+
 BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 {
     BlockJobInfoList *head = NULL, **p_next = &head;
@@ -3132,22 +4128,14 @@ QemuOptsList qemu_common_drive_opts = {
             .type = QEMU_OPT_STRING,
             .help = "discard operation (ignore/off, unmap/on)",
         },{
-            .name = BDRV_OPT_CACHE_WB,
-            .type = QEMU_OPT_BOOL,
-            .help = "enables writeback mode for any caches",
-        },{
-            .name = BDRV_OPT_CACHE_DIRECT,
-            .type = QEMU_OPT_BOOL,
-            .help = "enables use of O_DIRECT (bypass the host page cache)",
-        },{
-            .name = BDRV_OPT_CACHE_NO_FLUSH,
-            .type = QEMU_OPT_BOOL,
-            .help = "ignore any flush requests for the device",
-        },{
             .name = "aio",
             .type = QEMU_OPT_STRING,
             .help = "host AIO implementation (threads, native)",
         },{
+            .name = BDRV_OPT_CACHE_WB,
+            .type = QEMU_OPT_BOOL,
+            .help = "Enable writeback mode",
+        },{
             .name = "format",
             .type = QEMU_OPT_STRING,
             .help = "disk format (raw, qcow2, ...)",
@@ -3212,6 +4200,30 @@ QemuOptsList qemu_common_drive_opts = {
             .type = QEMU_OPT_NUMBER,
             .help = "total bytes write burst",
         },{
+            .name = "throttling.iops-total-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-total-max burst period, in seconds",
+        },{
+            .name = "throttling.iops-read-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-read-max burst period, in seconds",
+        },{
+            .name = "throttling.iops-write-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-write-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-total-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-total-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-read-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-read-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-write-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-write-max burst period, in seconds",
+        },{
             .name = "throttling.iops-size",
             .type = QEMU_OPT_NUMBER,
             .help = "when limiting by iops max size of an I/O in bytes",
@@ -3227,6 +4239,45 @@ QemuOptsList qemu_common_drive_opts = {
             .name = "detect-zeroes",
             .type = QEMU_OPT_STRING,
             .help = "try to optimize zero writes (off, on, unmap)",
+        },{
+            .name = "stats-account-invalid",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for invalid I/O operations "
+                    "in the statistics",
+        },{
+            .name = "stats-account-failed",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for failed I/O operations "
+                    "in the statistics",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_root_bds_opts = {
+    .name = "root-bds",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_root_bds_opts.head),
+    .desc = {
+        {
+            .name = "discard",
+            .type = QEMU_OPT_STRING,
+            .help = "discard operation (ignore/off, unmap/on)",
+        },{
+            .name = "aio",
+            .type = QEMU_OPT_STRING,
+            .help = "host AIO implementation (threads, native)",
+        },{
+            .name = "read-only",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },{
+            .name = "detect-zeroes",
+            .type = QEMU_OPT_STRING,
+            .help = "try to optimize zero writes (off, on, unmap)",
         },
         { /* end of list */ }
     },
diff --git a/qemu/blockjob.c b/qemu/blockjob.c
index 62bb90663..9fc37ca96 100644
--- a/qemu/blockjob.c
+++ b/qemu/blockjob.c
@@ -23,19 +23,33 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "trace.h"
 #include "block/block.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 #include "qapi-event.h"
 
+/* Transactional group of block jobs */
+struct BlockJobTxn {
+
+    /* Is this txn being cancelled? */
+    bool aborting;
+
+    /* List of jobs */
+    QLIST_HEAD(, BlockJob) jobs;
+
+    /* Reference count */
+    int refcnt;
+};
+
 void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
                        int64_t speed, BlockCompletionFunc *cb,
                        void *opaque, Error **errp)
@@ -54,10 +68,12 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
     bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
 
     job->driver        = driver;
+    job->id            = g_strdup(bdrv_get_device_name(bs));
     job->bs            = bs;
     job->cb            = cb;
     job->opaque        = opaque;
     job->busy          = true;
+    job->refcnt        = 1;
     bs->job = job;
 
     /* Only set speed when necessary to avoid NotSupported error */
@@ -66,7 +82,7 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
 
         block_job_set_speed(job, speed, &local_err);
         if (local_err) {
-            block_job_release(bs);
+            block_job_unref(job);
             error_propagate(errp, local_err);
             return NULL;
         }
@@ -74,14 +90,101 @@ void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
     return job;
 }
 
-void block_job_release(BlockDriverState *bs)
+void block_job_ref(BlockJob *job)
+{
+    ++job->refcnt;
+}
+
+void block_job_unref(BlockJob *job)
 {
-    BlockJob *job = bs->job;
+    if (--job->refcnt == 0) {
+        job->bs->job = NULL;
+        bdrv_op_unblock_all(job->bs, job->blocker);
+        bdrv_unref(job->bs);
+        error_free(job->blocker);
+        g_free(job->id);
+        g_free(job);
+    }
+}
 
-    bs->job = NULL;
-    bdrv_op_unblock_all(bs, job->blocker);
-    error_free(job->blocker);
-    g_free(job);
+static void block_job_completed_single(BlockJob *job)
+{
+    if (!job->ret) {
+        if (job->driver->commit) {
+            job->driver->commit(job);
+        }
+    } else {
+        if (job->driver->abort) {
+            job->driver->abort(job);
+        }
+    }
+    job->cb(job->opaque, job->ret);
+    if (job->txn) {
+        block_job_txn_unref(job->txn);
+    }
+    block_job_unref(job);
+}
+
+static void block_job_completed_txn_abort(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+
+    if (txn->aborting) {
+        /*
+         * We are cancelled by another job, which will handle everything.
+         */
+        return;
+    }
+    txn->aborting = true;
+    /* We are the first failed job. Cancel other jobs. */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        aio_context_acquire(ctx);
+    }
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (other_job == job || other_job->completed) {
+            /* Other jobs are "effectively" cancelled by us, set the status for
+             * them; this job, however, may or may not be cancelled, depending
+             * on the caller, so leave it. */
+            if (other_job != job) {
+                other_job->cancelled = true;
+            }
+            continue;
+        }
+        block_job_cancel_sync(other_job);
+        assert(other_job->completed);
+    }
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
+}
+
+static void block_job_completed_txn_success(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+    /*
+     * Successful completion, see if there are other running jobs in this
+     * txn.
+     */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (!other_job->completed) {
+            return;
+        }
+    }
+    /* We are the last completed job, commit the transaction. */
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = bdrv_get_aio_context(other_job->bs);
+        aio_context_acquire(ctx);
+        assert(other_job->ret == 0);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
 }
 
 void block_job_completed(BlockJob *job, int ret)
@@ -89,8 +192,16 @@ void block_job_completed(BlockJob *job, int ret)
     BlockDriverState *bs = job->bs;
 
     assert(bs->job == job);
-    job->cb(job->opaque, ret);
-    block_job_release(bs);
+    assert(!job->completed);
+    job->completed = true;
+    job->ret = ret;
+    if (!job->txn) {
+        block_job_completed_single(job);
+    } else if (ret < 0 || block_job_is_cancelled(job)) {
+        block_job_completed_txn_abort(job);
+    } else {
+        block_job_completed_txn_success(job);
+    }
 }
 
 void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -113,8 +224,7 @@ void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
 void block_job_complete(BlockJob *job, Error **errp)
 {
     if (job->pause_count || job->cancelled || !job->driver->complete) {
-        error_setg(errp, QERR_BLOCK_JOB_NOT_READY,
-                   bdrv_get_device_name(job->bs));
+        error_setg(errp, QERR_BLOCK_JOB_NOT_READY, job->id);
         return;
     }
 
@@ -168,51 +278,31 @@ void block_job_iostatus_reset(BlockJob *job)
     }
 }
 
-struct BlockFinishData {
-    BlockJob *job;
-    BlockCompletionFunc *cb;
-    void *opaque;
-    bool cancelled;
-    int ret;
-};
-
-static void block_job_finish_cb(void *opaque, int ret)
-{
-    struct BlockFinishData *data = opaque;
-
-    data->cancelled = block_job_is_cancelled(data->job);
-    data->ret = ret;
-    data->cb(data->opaque, ret);
-}
-
 static int block_job_finish_sync(BlockJob *job,
                                  void (*finish)(BlockJob *, Error **errp),
                                  Error **errp)
 {
-    struct BlockFinishData data;
     BlockDriverState *bs = job->bs;
     Error *local_err = NULL;
+    int ret;
 
     assert(bs->job == job);
 
-    /* Set up our own callback to store the result and chain to
-     * the original callback.
-     */
-    data.job = job;
-    data.cb = job->cb;
-    data.opaque = job->opaque;
-    data.ret = -EINPROGRESS;
-    job->cb = block_job_finish_cb;
-    job->opaque = &data;
+    block_job_ref(job);
     finish(job, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
+        block_job_unref(job);
         return -EBUSY;
     }
-    while (data.ret == -EINPROGRESS) {
-        aio_poll(bdrv_get_aio_context(bs), true);
+    while (!job->completed) {
+        aio_poll(job->deferred_to_main_loop ? qemu_get_aio_context() :
+                                              bdrv_get_aio_context(bs),
+                 true);
     }
-    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
+    ret = (job->cancelled && job->ret == 0) ? -ECANCELED : job->ret;
+    block_job_unref(job);
+    return ret;
 }
 
 /* A wrapper around block_job_cancel() taking an Error ** parameter so it may be
@@ -269,7 +359,7 @@ BlockJobInfo *block_job_query(BlockJob *job)
 {
     BlockJobInfo *info = g_new0(BlockJobInfo, 1);
     info->type      = g_strdup(BlockJobType_lookup[job->driver->job_type]);
-    info->device    = g_strdup(bdrv_get_device_name(job->bs));
+    info->device    = g_strdup(job->id);
     info->len       = job->len;
     info->busy      = job->busy;
     info->paused    = job->pause_count > 0;
@@ -291,7 +381,7 @@ static void block_job_iostatus_set_err(BlockJob *job, int error)
 void block_job_event_cancelled(BlockJob *job)
 {
     qapi_event_send_block_job_cancelled(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -301,7 +391,7 @@ void block_job_event_cancelled(BlockJob *job)
 void block_job_event_completed(BlockJob *job, const char *msg)
 {
     qapi_event_send_block_job_completed(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -315,7 +405,7 @@ void block_job_event_ready(BlockJob *job)
     job->ready = true;
 
     qapi_event_send_block_job_ready(job->driver->job_type,
-                                    bdrv_get_device_name(job->bs),
+                                    job->id,
                                     job->len,
                                     job->offset,
                                     job->speed, &error_abort);
@@ -344,7 +434,7 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
     default:
         abort();
     }
-    qapi_event_send_block_job_error(bdrv_get_device_name(job->bs),
+    qapi_event_send_block_job_error(job->id,
                                     is_read ? IO_OPERATION_TYPE_READ :
                                     IO_OPERATION_TYPE_WRITE,
                                     action, &error_abort);
@@ -353,8 +443,8 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
         job->user_paused = true;
         block_job_pause(job);
         block_job_iostatus_set_err(job, error);
-        if (bs != job->bs) {
-            bdrv_iostatus_set_err(bs, error);
+        if (bs->blk && bs != job->bs) {
+            blk_iostatus_set_err(bs->blk, error);
         }
     }
     return action;
@@ -382,6 +472,7 @@ static void block_job_defer_to_main_loop_bh(void *opaque)
     aio_context = bdrv_get_aio_context(data->job->bs);
     aio_context_acquire(aio_context);
 
+    data->job->deferred_to_main_loop = false;
     data->fn(data->job, data->opaque);
 
     aio_context_release(aio_context);
@@ -401,6 +492,40 @@ void block_job_defer_to_main_loop(BlockJob *job,
     data->aio_context = bdrv_get_aio_context(job->bs);
     data->fn = fn;
     data->opaque = opaque;
+    job->deferred_to_main_loop = true;
 
     qemu_bh_schedule(data->bh);
 }
+
+BlockJobTxn *block_job_txn_new(void)
+{
+    BlockJobTxn *txn = g_new0(BlockJobTxn, 1);
+    QLIST_INIT(&txn->jobs);
+    txn->refcnt = 1;
+    return txn;
+}
+
+static void block_job_txn_ref(BlockJobTxn *txn)
+{
+    txn->refcnt++;
+}
+
+void block_job_txn_unref(BlockJobTxn *txn)
+{
+    if (txn && --txn->refcnt == 0) {
+        g_free(txn);
+    }
+}
+
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job)
+{
+    if (!txn) {
+        return;
+    }
+
+    assert(!job->txn);
+    job->txn = txn;
+
+    QLIST_INSERT_HEAD(&txn->jobs, job, txn_list);
+    block_job_txn_ref(txn);
+}
diff --git a/qemu/bootdevice.c b/qemu/bootdevice.c
index 3cdc0d7b2..2e83ff05e 100644
--- a/qemu/bootdevice.c
+++ b/qemu/bootdevice.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "qapi/visitor.h"
 #include "qemu/error-report.h"
@@ -270,21 +272,21 @@ typedef struct {
     DeviceState *dev;
 } BootIndexProperty;
 
-static void device_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void device_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     BootIndexProperty *prop = opaque;
-    visit_type_int32(v, prop->bootindex, name, errp);
+    visit_type_int32(v, name, prop->bootindex, errp);
 }
 
-static void device_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void device_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     BootIndexProperty *prop = opaque;
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
diff --git a/qemu/bsd-user/bsdload.c b/qemu/bsd-user/bsdload.c
index 2abc7136e..94eec363b 100644
--- a/qemu/bsd-user/bsdload.c
+++ b/qemu/bsd-user/bsdload.c
@@ -1,12 +1,6 @@
 /* Code for loading BSD executables.  Mostly linux kernel code.  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
diff --git a/qemu/bsd-user/elfload.c b/qemu/bsd-user/elfload.c
index 2bf57eb1f..898ee0547 100644
--- a/qemu/bsd-user/elfload.c
+++ b/qemu/bsd-user/elfload.c
@@ -1,16 +1,11 @@
 /* This is the Linux kernel elf-loading code, ported into user space */
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
-#include <stdlib.h>
-#include <string.h>
 
 #include "qemu.h"
 #include "disas/disas.h"
+#include "qemu/path.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
@@ -740,8 +735,7 @@ static void padzero(abi_ulong elf_bss, abi_ulong last_bss)
            size must be known */
         if (qemu_real_host_page_size < qemu_host_page_size) {
             abi_ulong end_addr, end_addr1;
-            end_addr1 = (elf_bss + qemu_real_host_page_size - 1) &
-                ~(qemu_real_host_page_size - 1);
+            end_addr1 = REAL_HOST_PAGE_ALIGN(elf_bss);
             end_addr = HOST_PAGE_ALIGN(elf_bss);
             if (end_addr1 < end_addr) {
                 mmap((void *)g2h(end_addr1), end_addr - end_addr1,
@@ -1355,9 +1349,7 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
             }
         }
         if (!bprm->p) {
-            if (elf_interpreter) {
-                free(elf_interpreter);
-            }
+            free(elf_interpreter);
             free (elf_phdata);
             close(bprm->fd);
             return -E2BIG;
@@ -1371,7 +1363,6 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
     info->mmap = 0;
     elf_entry = (abi_ulong) elf_ex.e_entry;
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * In case where user has not explicitly set the guest_base, we
      * probe here that should we set it automatically.
@@ -1392,7 +1383,6 @@ int load_elf_binary(struct linux_binprm * bprm, struct target_pt_regs * regs,
             }
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /* Do this so that we can load the interpreter, if need be.  We will
        change some of these later */
diff --git a/qemu/bsd-user/i386/syscall.h b/qemu/bsd-user/i386/target_syscall.h
index 9b34c61bb..82d1c58ca 100644
--- a/qemu/bsd-user/i386/syscall.h
+++ b/qemu/bsd-user/i386/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 /* default linux values for the selectors */
 #define __USER_CS	(0x23)
 #define __USER_DS	(0x2B)
@@ -159,3 +162,4 @@ struct target_vm86plus_struct {
 
 #define UNAME_MACHINE "i386"
 
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/bsd-user/main.c b/qemu/bsd-user/main.c
index ee68daa39..27854c1f9 100644
--- a/qemu/bsd-user/main.c
+++ b/qemu/bsd-user/main.c
@@ -16,31 +16,25 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 #include <machine/trap.h>
-#include <sys/types.h>
 #include <sys/mman.h>
 
 #include "qemu.h"
-#include "qemu-common.h"
+#include "qemu/path.h"
+#include "qemu/help_option.h"
 /* For tb_lock */
 #include "cpu.h"
 #include "tcg.h"
 #include "qemu/timer.h"
 #include "qemu/envlist.h"
+#include "exec/log.h"
 
 int singlestep;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 int have_guest_base;
 unsigned long reserved_va;
-#endif
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -110,7 +104,7 @@ void cpu_list_unlock(void)
 
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -682,9 +676,7 @@ static void usage(void)
            "-drop-ld-preload  drop LD_PRELOAD for target process\n"
            "-E var=value      sets/modifies targets environment variable(s)\n"
            "-U var            unsets targets environment variable(s)\n"
-#if defined(CONFIG_USE_GUEST_BASE)
            "-B address        set guest_base address to address\n"
-#endif
            "-bsd type         select emulated BSD type FreeBSD/NetBSD/OpenBSD (default)\n"
            "\n"
            "Debug options:\n"
@@ -830,11 +822,9 @@ int main(int argc, char **argv)
 #endif
                 exit(1);
             }
-#if defined(CONFIG_USE_GUEST_BASE)
         } else if (!strcmp(r, "B")) {
            guest_base = strtol(argv[optind++], NULL, 0);
            have_guest_base = 1;
-#endif
         } else if (!strcmp(r, "drop-ld-preload")) {
             (void) envlist_unsetenv(envlist, "LD_PRELOAD");
         } else if (!strcmp(r, "bsd")) {
@@ -923,7 +913,6 @@ int main(int argc, char **argv)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -945,12 +934,11 @@ int main(int argc, char **argv)
             unsigned long tmp;
             if (fscanf(fp, "%lu", &tmp) == 1) {
                 mmap_min_addr = tmp;
-                qemu_log("host mmap_min_addr=0x%lx\n", mmap_min_addr);
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n", mmap_min_addr);
             }
             fclose(fp);
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     if (loader_exec(filename, argv+optind, target_environ, regs, info) != 0) {
         printf("Error loading %s\n", filename);
@@ -963,10 +951,8 @@ int main(int argc, char **argv)
 
     free(target_environ);
 
-    if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
+    if (qemu_loglevel_mask(CPU_LOG_PAGE)) {
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -986,12 +972,10 @@ int main(int argc, char **argv)
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
     /* build Task State */
     memset(ts, 0, sizeof(TaskState));
diff --git a/qemu/bsd-user/mmap.c b/qemu/bsd-user/mmap.c
index 092bf7f89..6ab533470 100644
--- a/qemu/bsd-user/mmap.c
+++ b/qemu/bsd-user/mmap.c
@@ -16,12 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 
 #include "qemu.h"
diff --git a/qemu/bsd-user/qemu.h b/qemu/bsd-user/qemu.h
index 5362297fe..03b502ad3 100644
--- a/qemu/bsd-user/qemu.h
+++ b/qemu/bsd-user/qemu.h
@@ -17,15 +17,12 @@
 #ifndef QEMU_H
 #define QEMU_H
 
-#include <signal.h>
-#include <string.h>
 
 #include "cpu.h"
 #include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
-#include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
 #include "exec/user/abitypes.h"
@@ -38,7 +35,7 @@ enum BSDType {
 extern enum BSDType bsd_type;
 
 #include "syscall_defs.h"
-#include "syscall.h"
+#include "target_syscall.h"
 #include "target_signal.h"
 #include "exec/gdbstub.h"
 
@@ -101,9 +98,7 @@ typedef struct TaskState {
 
 void init_task_state(TaskState *ts);
 extern const char *qemu_uname_release;
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long mmap_min_addr;
-#endif
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
 /*
@@ -213,8 +208,6 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                        abi_ulong new_addr);
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
-void mmap_lock(void);
-void mmap_unlock(void);
 void cpu_list_lock(void);
 void cpu_list_unlock(void);
 #if defined(CONFIG_USE_NPTL)
diff --git a/qemu/bsd-user/signal.c b/qemu/bsd-user/signal.c
index 445f69e83..f6f7aa242 100644
--- a/qemu/bsd-user/signal.c
+++ b/qemu/bsd-user/signal.c
@@ -16,19 +16,11 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <signal.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "target_signal.h"
 
-//#define DEBUG_SIGNAL
-
 void signal_init(void)
 {
 }
diff --git a/qemu/bsd-user/sparc/syscall.h b/qemu/bsd-user/sparc/target_syscall.h
index 5a9bb7e54..c7eec6ba6 100644
--- a/qemu/bsd-user/sparc/syscall.h
+++ b/qemu/bsd-user/sparc/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
 	abi_ulong psr;
 	abi_ulong pc;
@@ -7,3 +10,5 @@ struct target_pt_regs {
 };
 
 #define UNAME_MACHINE "sun4"
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/bsd-user/sparc64/syscall.h b/qemu/bsd-user/sparc64/target_syscall.h
index 81a816de9..2f06100ae 100644
--- a/qemu/bsd-user/sparc64/syscall.h
+++ b/qemu/bsd-user/sparc64/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
 	abi_ulong u_regs[16];
 	abi_ulong tstate;
@@ -8,3 +11,5 @@ struct target_pt_regs {
 };
 
 #define UNAME_MACHINE "sun4u"
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/bsd-user/strace.c b/qemu/bsd-user/strace.c
index e33dd4d48..fa66fe1ee 100644
--- a/qemu/bsd-user/strace.c
+++ b/qemu/bsd-user/strace.c
@@ -16,14 +16,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/select.h>
-#include <sys/types.h>
-#include <unistd.h>
 #include <sys/syscall.h>
 #include <sys/ioccom.h>
-#include <ctype.h>
 
 #include "qemu.h"
 
diff --git a/qemu/bsd-user/syscall.c b/qemu/bsd-user/syscall.c
index a4d1583fe..47cf865a3 100644
--- a/qemu/bsd-user/syscall.c
+++ b/qemu/bsd-user/syscall.c
@@ -16,17 +16,9 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <limits.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <sys/param.h>
diff --git a/qemu/bsd-user/uaccess.c b/qemu/bsd-user/uaccess.c
index 677f19c26..91e206793 100644
--- a/qemu/bsd-user/uaccess.c
+++ b/qemu/bsd-user/uaccess.c
@@ -1,6 +1,6 @@
 /* User memory access */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "qemu.h"
 
@@ -51,7 +51,7 @@ abi_long target_strlen(abi_ulong guest_addr1)
         ptr = lock_user(VERIFY_READ, guest_addr, max_len, 1);
         if (!ptr)
             return -TARGET_EFAULT;
-        len = qemu_strnlen((char *)ptr, max_len);
+        len = qemu_strnlen((const char *)ptr, max_len);
         unlock_user(ptr, guest_addr, 0);
         guest_addr += len;
         /* we don't allow wrapping or integer overflow */
diff --git a/qemu/bsd-user/x86_64/syscall.h b/qemu/bsd-user/x86_64/target_syscall.h
index 630514a93..85a976697 100644
--- a/qemu/bsd-user/x86_64/syscall.h
+++ b/qemu/bsd-user/x86_64/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 #define __USER_CS	(0x33)
 #define __USER_DS	(0x2B)
 
@@ -114,3 +117,5 @@ struct target_msqid64_ds {
 #define TARGET_ARCH_SET_FS 0x1002
 #define TARGET_ARCH_GET_FS 0x1003
 #define TARGET_ARCH_GET_GS 0x1004
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/bt-host.c b/qemu/bt-host.c
index 49205bf28..2f8f631c2 100644
--- a/qemu/bt-host.c
+++ b/qemu/bt-host.c
@@ -17,12 +17,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "qemu/main-loop.h"
 
 #ifndef _WIN32
-# include <errno.h>
 # include <sys/ioctl.h>
 # include <sys/uio.h>
 # ifdef CONFIG_BLUEZ
diff --git a/qemu/bt-vhci.c b/qemu/bt-vhci.c
index e267c8ad1..9d277c32b 100644
--- a/qemu/bt-vhci.c
+++ b/qemu/bt-vhci.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
diff --git a/qemu/configure b/qemu/configure
index cd219d8e3..c37fc5fe7 100755
--- a/qemu/configure
+++ b/qemu/configure
@@ -8,6 +8,9 @@
 CLICOLOR_FORCE= GREP_OPTIONS=
 unset CLICOLOR_FORCE GREP_OPTIONS
 
+# Don't allow CCACHE, if present, to use cached results of compile tests!
+export CCACHE_RECACHE=yes
+
 # Temporary directory used for files created while
 # configure runs. Since it is in the build directory
 # we can safely blow away any previous version of it
@@ -113,38 +116,6 @@ compile_prog() {
   do_cc $QEMU_CFLAGS $local_cflags -o $TMPE $TMPC $LDFLAGS $local_ldflags
 }
 
-do_libtool() {
-    local mode=$1
-    shift
-    # Run the compiler, capturing its output to the log.
-    echo $libtool $mode --tag=CC $cc "$@" >> config.log
-    $libtool $mode --tag=CC $cc "$@" >> config.log 2>&1 || return $?
-    # Test passed. If this is an --enable-werror build, rerun
-    # the test with -Werror and bail out if it fails. This
-    # makes warning-generating-errors in configure test code
-    # obvious to developers.
-    if test "$werror" != "yes"; then
-        return 0
-    fi
-    # Don't bother rerunning the compile if we were already using -Werror
-    case "$*" in
-        *-Werror*)
-           return 0
-        ;;
-    esac
-    echo $libtool $mode --tag=CC $cc -Werror "$@" >> config.log
-    $libtool $mode --tag=CC $cc -Werror "$@" >> config.log 2>&1 && return $?
-    error_exit "configure test passed without -Werror but failed with -Werror." \
-        "This is probably a bug in the configure script. The failing command" \
-        "will be at the bottom of config.log." \
-        "You can run configure with --disable-werror to bypass this check."
-}
-
-libtool_prog() {
-    do_libtool --mode=compile $QEMU_CFLAGS -c -fPIE -DPIE -o $TMPO $TMPC || return $?
-    do_libtool --mode=link $LDFLAGS -o $TMPA $TMPL -rpath /usr/local/lib
-}
-
 # symbolically link $1 to $2.  Portable version of "ln -sf".
 symlink() {
   rm -rf "$2"
@@ -242,12 +213,12 @@ vnc="yes"
 sparse="no"
 uuid=""
 vde=""
-vnc_tls=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
 xen=""
 xen_ctrl_version=""
+xen_pv_domain_build="no"
 xen_pci_passthrough=""
 linux_aio=""
 cap_ng=""
@@ -262,6 +233,7 @@ rdma=""
 gprof="no"
 debug_tcg="no"
 debug="no"
+fortify_source=""
 strip_opt="yes"
 tcg_interpreter="no"
 bigendian="no"
@@ -293,21 +265,22 @@ cocoa="no"
 softmmu="yes"
 linux_user="no"
 bsd_user="no"
-guest_base="yes"
 aix="no"
 blobs="yes"
 pkgversion=""
 pie=""
 zero_malloc=""
 qom_cast_debug="yes"
-trace_backends="nop"
+trace_backends="log"
 trace_file="trace"
 spice=""
 rbd=""
-smartcard_nss=""
+smartcard=""
 libusb=""
 usb_redir=""
 opengl=""
+opengl_dmabuf="no"
+avx2_opt="no"
 zlib="yes"
 lzo=""
 snappy=""
@@ -325,19 +298,28 @@ coroutine=""
 coroutine_pool=""
 seccomp=""
 glusterfs=""
+glusterfs_xlator_opt="no"
 glusterfs_discard="no"
 glusterfs_zerofill="no"
 archipelago="no"
 gtk=""
 gtkabi=""
+gtk_gl="no"
 gnutls=""
 gnutls_hash=""
+gnutls_rnd=""
+nettle=""
+nettle_kdf="no"
+gcrypt=""
+gcrypt_kdf="no"
 vte=""
+virglrenderer=""
 tpm="yes"
 libssh2=""
 vhdx=""
 numa=""
 tcmalloc="no"
+jemalloc="no"
 
 # parse CC options first
 for opt do
@@ -390,7 +372,6 @@ as="${AS-${cross_prefix}as}"
 cpp="${CPP-$cc -E}"
 objcopy="${OBJCOPY-${cross_prefix}objcopy}"
 ld="${LD-${cross_prefix}ld}"
-libtool="${LIBTOOL-${cross_prefix}libtool}"
 nm="${NM-${cross_prefix}nm}"
 strip="${STRIP-${cross_prefix}strip}"
 windres="${WINDRES-${cross_prefix}windres}"
@@ -720,6 +701,8 @@ if test "$mingw32" = "yes" ; then
   QEMU_CFLAGS="-DWIN32_LEAN_AND_MEAN -DWINVER=0x501 $QEMU_CFLAGS"
   # enable C99/POSIX format strings (needs mingw32-runtime 3.15 or later)
   QEMU_CFLAGS="-D__USE_MINGW_ANSI_STDIO=1 $QEMU_CFLAGS"
+  # MinGW needs -mthreads for TLS and macro _MT.
+  QEMU_CFLAGS="-mthreads $QEMU_CFLAGS"
   LIBS="-lwinmm -lws2_32 -liphlpapi $LIBS"
   write_c_skeleton;
   if compile_prog "" "-liberty" ; then
@@ -733,7 +716,7 @@ if test "$mingw32" = "yes" ; then
   sysconfdir="\${prefix}"
   local_statedir=
   confsuffix=""
-  libs_qga="-lws2_32 -lwinmm -lpowrprof -liphlpapi $libs_qga"
+  libs_qga="-lws2_32 -lwinmm -lpowrprof -liphlpapi -lnetapi32 $libs_qga"
 fi
 
 werror=""
@@ -784,6 +767,9 @@ for opt do
   --enable-modules)
       modules="yes"
   ;;
+  --disable-modules)
+      modules="no"
+  ;;
   --cpu=*)
   ;;
   --target-list=*) target_list="$optarg"
@@ -873,6 +859,7 @@ for opt do
       debug_tcg="yes"
       debug="yes"
       strip_opt="no"
+      fortify_source="no"
   ;;
   --enable-sparse) sparse="yes"
   ;;
@@ -880,10 +867,6 @@ for opt do
   ;;
   --disable-strip) strip_opt="no"
   ;;
-  --disable-vnc-tls) vnc_tls="no"
-  ;;
-  --enable-vnc-tls) vnc_tls="yes"
-  ;;
   --disable-vnc-sasl) vnc_sasl="no"
   ;;
   --enable-vnc-sasl) vnc_sasl="yes"
@@ -918,6 +901,10 @@ for opt do
   ;;
   --enable-xen-pci-passthrough) xen_pci_passthrough="yes"
   ;;
+  --disable-xen-pv-domain-build) xen_pv_domain_build="no"
+  ;;
+  --enable-xen-pv-domain-build) xen_pv_domain_build="yes"
+  ;;
   --disable-brlapi) brlapi="no"
   ;;
   --enable-brlapi) brlapi="yes"
@@ -956,7 +943,6 @@ for opt do
   ;;
   --enable-cocoa)
       cocoa="yes" ;
-      sdl="no" ;
       audio_drv_list="coreaudio `echo $audio_drv_list | sed s,coreaudio,,g`"
   ;;
   --disable-system) softmmu="no"
@@ -976,10 +962,6 @@ for opt do
   ;;
   --enable-bsd-user) bsd_user="yes"
   ;;
-  --enable-guest-base) guest_base="yes"
-  ;;
-  --disable-guest-base) guest_base="no"
-  ;;
   --enable-pie) pie="yes"
   ;;
   --disable-pie) pie="no"
@@ -1046,9 +1028,9 @@ for opt do
   ;;
   --enable-xfsctl) xfs="yes"
   ;;
-  --disable-smartcard-nss) smartcard_nss="no"
+  --disable-smartcard) smartcard="no"
   ;;
-  --enable-smartcard-nss) smartcard_nss="yes"
+  --enable-smartcard) smartcard="yes"
   ;;
   --disable-libusb) libusb="no"
   ;;
@@ -1119,6 +1101,14 @@ for opt do
   ;;
   --enable-gnutls) gnutls="yes"
   ;;
+  --disable-nettle) nettle="no"
+  ;;
+  --enable-nettle) nettle="yes"
+  ;;
+  --disable-gcrypt) gcrypt="no"
+  ;;
+  --enable-gcrypt) gcrypt="yes"
+  ;;
   --enable-rdma) rdma="yes"
   ;;
   --disable-rdma) rdma="no"
@@ -1129,6 +1119,10 @@ for opt do
   ;;
   --enable-vte) vte="yes"
   ;;
+  --disable-virglrenderer) virglrenderer="no"
+  ;;
+  --enable-virglrenderer) virglrenderer="yes"
+  ;;
   --disable-tpm) tpm="no"
   ;;
   --enable-tpm) tpm="yes"
@@ -1149,6 +1143,10 @@ for opt do
   ;;
   --enable-tcmalloc) tcmalloc="yes"
   ;;
+  --disable-jemalloc) jemalloc="no"
+  ;;
+  --enable-jemalloc) jemalloc="yes"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"
@@ -1163,18 +1161,14 @@ fi
 
 # Note that if the Python conditional here evaluates True we will exit
 # with status 1 which is a shell 'false' value.
-if ! $python -c 'import sys; sys.exit(sys.version_info < (2,4) or sys.version_info >= (3,))'; then
-  error_exit "Cannot use '$python', Python 2.4 or later is required." \
+if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
+  error_exit "Cannot use '$python', Python 2.6 or later is required." \
       "Note that Python 3 or later is not yet supported." \
       "Use --python=/path/to/python to specify a supported Python."
 fi
 
-# The -B switch was added in Python 2.6.
-# If it is supplied, compiled files are not written.
-# Use it for Python versions which support it.
-if $python -B -c 'import sys; sys.exit(0)' 2>/dev/null; then
-  python="$python -B"
-fi
+# Suppress writing compiled files
+python="$python -B"
 
 case "$cpu" in
     ppc)
@@ -1315,7 +1309,6 @@ disabled with --disable-FEATURE, default is enabled if available:
   user            supported user emulation targets
   linux-user      all linux usermode emulation targets
   bsd-user        all BSD usermode emulation targets
-  guest-base      GUEST_BASE support for usermode emulation targets
   docs            build documentation
   guest-agent     build the QEMU Guest Agent
   guest-agent-msi build guest agent Windows MSI installation package
@@ -1326,6 +1319,8 @@ disabled with --disable-FEATURE, default is enabled if available:
   sparse          sparse checker
 
   gnutls          GNUTLS cryptography support
+  nettle          nettle cryptography support
+  gcrypt          libgcrypt cryptography support
   sdl             SDL UI
   --with-sdlabi     select preferred SDL ABI 1.2 or 2.0
   gtk             gtk UI
@@ -1333,7 +1328,6 @@ disabled with --disable-FEATURE, default is enabled if available:
   vte             vte support for the gtk UI
   curses          curses UI
   vnc             VNC UI support
-  vnc-tls         TLS encryption for VNC server
   vnc-sasl        SASL encryption for VNC server
   vnc-jpeg        JPEG lossy compression for VNC server
   vnc-png         PNG compression for VNC server
@@ -1358,7 +1352,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   rbd             rados block device (rbd)
   libiscsi        iscsi support
   libnfs          nfs support
-  smartcard-nss   smartcard nss support
+  smartcard       smartcard support (libcacard)
   libusb          libusb (for usb passthrough)
   usb-redir       usb network redirection support
   lzo             support of lzo compression library
@@ -1374,6 +1368,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   vhdx            support for the Microsoft VHDX image format
   numa            libnuma support
   tcmalloc        tcmalloc support
+  jemalloc        jemalloc support
 
 NOTE: The object files are built at the place where configure is launched
 EOF
@@ -1411,6 +1406,9 @@ if compile_object ; then
 else
     error_exit "\"$cc\" either does not exist or does not work"
 fi
+if ! compile_prog ; then
+    error_exit "\"$cc\" cannot build an executable (is your linker broken?)"
+fi
 
 # Check that the C++ compiler exists and works with the C compiler
 if has $cxx; then
@@ -1471,6 +1469,16 @@ for flag in $gcc_flags; do
 done
 
 if test "$stack_protector" != "no"; then
+  cat > $TMPC << EOF
+int main(int argc, char *argv[])
+{
+    char arr[64], *p = arr, *c = argv[0];
+    while (*c) {
+        *p++ = *c++;
+    }
+    return 0;
+}
+EOF
   gcc_flags="-fstack-protector-strong -fstack-protector-all"
   sp_on=0
   for flag in $gcc_flags; do
@@ -1479,7 +1487,6 @@ if test "$stack_protector" != "no"; then
     if do_cc $QEMU_CFLAGS -Werror $flag -c -o $TMPO $TMPC &&
        compile_prog "-Werror $flag" ""; then
       QEMU_CFLAGS="$QEMU_CFLAGS $flag"
-      LIBTOOLFLAGS="$LIBTOOLFLAGS -Wc,$flag"
       sp_on=1
       break
     fi
@@ -1574,32 +1581,6 @@ EOF
   fi
 fi
 
-# check for broken gcc and libtool in RHEL5
-if test -n "$libtool" -a "$pie" != "no" ; then
-  cat > $TMPC <<EOF
-
-void *f(unsigned char *buf, int len);
-void *g(unsigned char *buf, int len);
-
-void *
-f(unsigned char *buf, int len)
-{
-    return (void*)0L;
-}
-
-void *
-g(unsigned char *buf, int len)
-{
-    return f(buf, len);
-}
-
-EOF
-  if ! libtool_prog; then
-    echo "Disabling libtool due to broken toolchain support"
-    libtool=
-  fi
-fi
-
 ##########################################
 # __sync_fetch_and_and requires at least -march=i486. Many toolchains
 # use i686 as default anyway, but for those that don't, an explicit
@@ -1711,6 +1692,21 @@ else
 fi
 
 ##########################################
+# cocoa implies not SDL or GTK
+# (the cocoa UI code currently assumes it is always the active UI
+# and doesn't interact well with other UI frontend code)
+if test "$cocoa" = "yes"; then
+    if test "$sdl" = "yes"; then
+        error_exit "Cocoa and SDL UIs cannot both be enabled at once"
+    fi
+    if test "$gtk" = "yes"; then
+        error_exit "Cocoa and GTK UIs cannot both be enabled at once"
+    fi
+    gtk=no
+    sdl=no
+fi
+
+##########################################
 # L2TPV3 probe
 
 cat > $TMPC <<EOF
@@ -1725,6 +1721,37 @@ else
 fi
 
 ##########################################
+# MinGW / Mingw-w64 localtime_r/gmtime_r check
+
+if test "$mingw32" = "yes"; then
+    # Some versions of MinGW / Mingw-w64 lack localtime_r
+    # and gmtime_r entirely.
+    #
+    # Some versions of Mingw-w64 define a macro for
+    # localtime_r/gmtime_r.
+    #
+    # Some versions of Mingw-w64 will define functions
+    # for localtime_r/gmtime_r, but only if you have
+    # _POSIX_THREAD_SAFE_FUNCTIONS defined. For fun
+    # though, unistd.h and pthread.h both define
+    # that for you.
+    #
+    # So this #undef localtime_r and #include <unistd.h>
+    # are not in fact redundant.
+cat > $TMPC << EOF
+#include <unistd.h>
+#include <time.h>
+#undef localtime_r
+int main(void) { localtime_r(NULL, NULL); return 0; }
+EOF
+    if compile_prog "" "" ; then
+        localtime_r="yes"
+    else
+        localtime_r="no"
+    fi
+fi
+
+##########################################
 # pkg-config probe
 
 if ! has "$pkg_config_exe"; then
@@ -1751,6 +1778,21 @@ EOF
 fi
 
 ##########################################
+# avx2 optimization requirement check
+
+cat > $TMPC << EOF
+static void bar(void) {}
+static void *bar_ifunc(void) {return (void*) bar;}
+static void foo(void) __attribute__((ifunc("bar_ifunc")));
+int main(void) { foo(); return 0; }
+EOF
+if compile_prog "-mavx2" "" ; then
+    if readelf --syms $TMPE |grep "IFUNC.*foo" >/dev/null 2>&1; then
+        avx2_opt="yes"
+    fi
+fi
+
+#########################################
 # zlib check
 
 if test "$zlib" != "no" ; then
@@ -1827,16 +1869,37 @@ fi
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if test "$cpu" = "i386" || test "$cpu" = "x86_64" &&
-        $pkg_config --atleast-version=2.1.1 libseccomp; then
+    case "$cpu" in
+    i386|x86_64)
+        libseccomp_minver="2.1.0"
+        ;;
+    mips)
+        libseccomp_minver="2.2.0"
+        ;;
+    arm|aarch64)
+        libseccomp_minver="2.2.3"
+        ;;
+    *)
+        libseccomp_minver=""
+        ;;
+    esac
+
+    if test "$libseccomp_minver" != "" &&
+       $pkg_config --atleast-version=$libseccomp_minver libseccomp ; then
         libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
         QEMU_CFLAGS="$QEMU_CFLAGS `$pkg_config --cflags libseccomp`"
-	seccomp="yes"
+        seccomp="yes"
     else
-	if test "$seccomp" = "yes"; then
-            feature_not_found "libseccomp" "Install libseccomp devel >= 2.1.1"
-	fi
-	seccomp="no"
+        if test "$seccomp" = "yes" ; then
+            if test "$libseccomp_minver" != "" ; then
+                feature_not_found "libseccomp" \
+                    "Install libseccomp devel >= $libseccomp_minver"
+            else
+                feature_not_found "libseccomp" \
+                    "libseccomp is not supported for host cpu $cpu"
+            fi
+        fi
+        seccomp="no"
     fi
 fi
 ##########################################
@@ -1844,6 +1907,7 @@ fi
 
 if test "$xen" != "no" ; then
   xen_libs="-lxenstore -lxenctrl -lxenguest"
+  xen_stable_libs="-lxenforeignmemory -lxengnttab -lxenevtchn"
 
   # First we test whether Xen headers and libraries are available.
   # If no, we are done and there is no Xen support.
@@ -1866,30 +1930,72 @@ EOF
   # Xen unstable
   elif
       cat > $TMPC <<EOF &&
+/*
+ * If we have stable libs the we don't want the libxc compat
+ * layers, regardless of what CFLAGS we may have been given.
+ */
+#undef XC_WANT_COMPAT_EVTCHN_API
+#undef XC_WANT_COMPAT_GNTTAB_API
+#undef XC_WANT_COMPAT_MAP_FOREIGN_API
 #include <xenctrl.h>
 #include <xenstore.h>
+#include <xenevtchn.h>
+#include <xengnttab.h>
+#include <xenforeignmemory.h>
 #include <stdint.h>
 #include <xen/hvm/hvm_info_table.h>
 #if !defined(HVM_MAX_VCPUS)
 # error HVM_MAX_VCPUS not defined
 #endif
 int main(void) {
-  xc_interface *xc;
+  xc_interface *xc = NULL;
+  xenforeignmemory_handle *xfmem;
+  xenevtchn_handle *xe;
+  xengnttab_handle *xg;
+  xen_domain_handle_t handle;
+
   xs_daemon_open();
+
   xc = xc_interface_open(0, 0, 0);
   xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_gnttab_open(NULL, 0);
   xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
   xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
-  xc_hvm_create_ioreq_server(xc, 0, 0, NULL);
+  xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL);
+  xc_domain_create(xc, 0, handle, 0, NULL, NULL);
+
+  xfmem = xenforeignmemory_open(0, 0);
+  xenforeignmemory_map(xfmem, 0, 0, 0, 0, 0);
+
+  xe = xenevtchn_open(0, 0);
+  xenevtchn_fd(xe);
+
+  xg = xengnttab_open(0, 0);
+  xengnttab_map_grant_ref(xg, 0, 0, 0);
+
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs $xen_stable_libs"
+    then
+    xen_ctrl_version=471
+    xen=yes
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
+#include <stdint.h>
+int main(void) {
+  xc_interface *xc = NULL;
+  xen_domain_handle_t handle;
+  xc_domain_create(xc, 0, handle, 0, NULL, NULL);
   return 0;
 }
 EOF
       compile_prog "" "$xen_libs"
     then
-    xen_ctrl_version=450
+    xen_ctrl_version=470
     xen=yes
 
+  # Xen 4.6
   elif
       cat > $TMPC <<EOF &&
 #include <xenctrl.h>
@@ -1907,127 +2013,89 @@ int main(void) {
   xc_gnttab_open(NULL, 0);
   xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
   xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL);
+  xc_reserved_device_memory_map(xc, 0, 0, 0, 0, NULL, 0);
   return 0;
 }
 EOF
       compile_prog "" "$xen_libs"
     then
-    xen_ctrl_version=420
+    xen_ctrl_version=460
     xen=yes
 
+  # Xen 4.5
   elif
       cat > $TMPC <<EOF &&
 #include <xenctrl.h>
-#include <xs.h>
+#include <xenstore.h>
 #include <stdint.h>
 #include <xen/hvm/hvm_info_table.h>
 #if !defined(HVM_MAX_VCPUS)
 # error HVM_MAX_VCPUS not defined
 #endif
 int main(void) {
+  xc_interface *xc;
   xs_daemon_open();
-  xc_interface_open(0, 0, 0);
+  xc = xc_interface_open(0, 0, 0);
   xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
   xc_gnttab_open(NULL, 0);
   xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, 0, NULL);
   return 0;
 }
 EOF
       compile_prog "" "$xen_libs"
     then
-    xen_ctrl_version=410
+    xen_ctrl_version=450
     xen=yes
 
-  # Xen 4.0.0
   elif
       cat > $TMPC <<EOF &&
 #include <xenctrl.h>
-#include <xs.h>
+#include <xenstore.h>
 #include <stdint.h>
 #include <xen/hvm/hvm_info_table.h>
 #if !defined(HVM_MAX_VCPUS)
 # error HVM_MAX_VCPUS not defined
 #endif
 int main(void) {
-  struct xen_add_to_physmap xatp = {
-    .domid = 0, .space = XENMAPSPACE_gmfn, .idx = 0, .gpfn = 0,
-  };
-  xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_memory_op(0, XENMEM_add_to_physmap, &xatp);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=400
-    xen=yes
-
-  # Xen 3.4.0
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-int main(void) {
-  struct xen_add_to_physmap xatp = {
-    .domid = 0, .space = XENMAPSPACE_gmfn, .idx = 0, .gpfn = 0,
-  };
-  xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_memory_op(0, XENMEM_add_to_physmap, &xatp);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=340
-    xen=yes
-
-  # Xen 3.3.0
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-int main(void) {
+  xc_interface *xc;
   xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
+  xc = xc_interface_open(0, 0, 0);
   xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
+  xc_gnttab_open(NULL, 0);
+  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
   return 0;
 }
 EOF
       compile_prog "" "$xen_libs"
     then
-    xen_ctrl_version=330
+    xen_ctrl_version=420
     xen=yes
 
-  # Xen version unsupported
   else
     if test "$xen" = "yes" ; then
-      feature_not_found "xen (unsupported version)" "Install supported xen (e.g. 4.0, 3.4, 3.3)"
+      feature_not_found "xen (unsupported version)" \
+                        "Install a supported xen (xen 4.2 or newer)"
     fi
     xen=no
   fi
 
   if test "$xen" = yes; then
+    if test $xen_ctrl_version -ge 471  ; then
+      libs_softmmu="$xen_stable_libs $libs_softmmu"
+    fi
     libs_softmmu="$xen_libs $libs_softmmu"
   fi
 fi
 
 if test "$xen_pci_passthrough" != "no"; then
-  if test "$xen" = "yes" && test "$linux" = "yes" &&
-    test "$xen_ctrl_version" -ge 340; then
+  if test "$xen" = "yes" && test "$linux" = "yes"; then
     xen_pci_passthrough=yes
   else
     if test "$xen_pci_passthrough" = "yes"; then
-      if test "$xen_ctrl_version" -lt 340; then
-        error_exit "User requested feature Xen PCI Passthrough" \
-            "This feature does not work with Xen 3.3"
-      fi
       error_exit "User requested feature Xen PCI Passthrough" \
           " but this feature requires /sys from Linux"
     fi
@@ -2035,19 +2103,10 @@ if test "$xen_pci_passthrough" != "no"; then
   fi
 fi
 
-##########################################
-# libtool probe
-
-if ! has $libtool; then
-    libtool=
-fi
-
-# MacOSX ships with a libtool which isn't the GNU one; weed this
-# out by checking whether libtool supports the --version switch
-if test -n "$libtool"; then
-  if ! "$libtool" --version >/dev/null 2>&1; then
-    libtool=
-  fi
+if test "$xen_pv_domain_build" = "yes" &&
+   test "$xen" != "yes"; then
+    error_exit "User requested Xen PV domain builder support" \
+	       "which requires Xen support."
 fi
 
 ##########################################
@@ -2149,6 +2208,13 @@ if test "$gnutls" != "no"; then
 	    gnutls_hash="no"
 	fi
 
+	# gnutls_rnd requires >= 2.11.0
+	if $pkg_config --exists "gnutls >= 2.11.0"; then
+	    gnutls_rnd="yes"
+	else
+	    gnutls_rnd="no"
+	fi
+
 	if $pkg_config --exists 'gnutls >= 3.0'; then
 	    gnutls_gcrypt=no
 	    gnutls_nettle=yes
@@ -2176,25 +2242,96 @@ if test "$gnutls" != "no"; then
     else
         gnutls="no"
         gnutls_hash="no"
+        gnutls_rnd="no"
     fi
 else
     gnutls_hash="no"
+    gnutls_rnd="no"
+fi
+
+
+# If user didn't give a --disable/enable-gcrypt flag,
+# then mark as disabled if user requested nettle
+# explicitly, or if gnutls links to nettle
+if test -z "$gcrypt"
+then
+    if test "$nettle" = "yes" || test "$gnutls_nettle" = "yes"
+    then
+        gcrypt="no"
+    fi
+fi
+
+# If user didn't give a --disable/enable-nettle flag,
+# then mark as disabled if user requested gcrypt
+# explicitly, or if gnutls links to gcrypt
+if test -z "$nettle"
+then
+    if test "$gcrypt" = "yes" || test "$gnutls_gcrypt" = "yes"
+    then
+        nettle="no"
+    fi
 fi
 
-if test "$gnutls_gcrypt" != "no"; then
-    if has "libgcrypt-config"; then
+has_libgcrypt_config() {
+    if ! has "libgcrypt-config"
+    then
+	return 1
+    fi
+
+    if test -n "$cross_prefix"
+    then
+	host=`libgcrypt-config --host`
+	if test "$host-" != $cross_prefix
+	then
+	    return 1
+	fi
+    fi
+
+    return 0
+}
+
+if test "$gcrypt" != "no"; then
+    if has_libgcrypt_config; then
         gcrypt_cflags=`libgcrypt-config --cflags`
         gcrypt_libs=`libgcrypt-config --libs`
+        # Debian has remove -lgpg-error from libgcrypt-config
+        # as it "spreads unnecessary dependencies" which in
+        # turn breaks static builds...
+        if test "$static" = "yes"
+        then
+            gcrypt_libs="$gcrypt_libs -lgpg-error"
+        fi
         libs_softmmu="$gcrypt_libs $libs_softmmu"
         libs_tools="$gcrypt_libs $libs_tools"
         QEMU_CFLAGS="$QEMU_CFLAGS $gcrypt_cflags"
+        gcrypt="yes"
+        if test -z "$nettle"; then
+           nettle="no"
+        fi
+
+        cat > $TMPC << EOF
+#include <gcrypt.h>
+int main(void) {
+  gcry_kdf_derive(NULL, 0, GCRY_KDF_PBKDF2,
+                  GCRY_MD_SHA256,
+                  NULL, 0, 0, 0, NULL);
+ return 0;
+}
+EOF
+        if compile_prog "$gcrypt_cflags" "$gcrypt_libs" ; then
+            gcrypt_kdf=yes
+        fi
     else
-        feature_not_found "gcrypt" "Install gcrypt devel"
+        if test "$gcrypt" = "yes"; then
+            feature_not_found "gcrypt" "Install gcrypt devel"
+        else
+            gcrypt="no"
+        fi
     fi
 fi
 
 
-if test "$gnutls_nettle" != "no"; then
+if test "$nettle" != "no"; then
     if $pkg_config --exists "nettle"; then
         nettle_cflags=`$pkg_config --cflags nettle`
         nettle_libs=`$pkg_config --libs nettle`
@@ -2202,11 +2339,53 @@ if test "$gnutls_nettle" != "no"; then
         libs_softmmu="$nettle_libs $libs_softmmu"
         libs_tools="$nettle_libs $libs_tools"
         QEMU_CFLAGS="$QEMU_CFLAGS $nettle_cflags"
+        nettle="yes"
+
+        cat > $TMPC << EOF
+#include <nettle/pbkdf2.h>
+int main(void) {
+     pbkdf2_hmac_sha256(8, NULL, 1000, 8, NULL, 8, NULL);
+     return 0;
+}
+EOF
+        if compile_prog "$nettle_cflags" "$nettle_libs" ; then
+            nettle_kdf=yes
+        fi
     else
-        feature_not_found "nettle" "Install nettle devel"
+        if test "$nettle" = "yes"; then
+            feature_not_found "nettle" "Install nettle devel"
+        else
+            nettle="no"
+        fi
     fi
 fi
 
+if test "$gcrypt" = "yes" && test "$nettle" = "yes"
+then
+    error_exit "Only one of gcrypt & nettle can be enabled"
+fi
+
+##########################################
+# libtasn1 - only for the TLS creds/session test suite
+
+tasn1=yes
+tasn1_cflags=""
+tasn1_libs=""
+if $pkg_config --exists "libtasn1"; then
+    tasn1_cflags=`$pkg_config --cflags libtasn1`
+    tasn1_libs=`$pkg_config --libs libtasn1`
+else
+    tasn1=no
+fi
+
+
+##########################################
+# getifaddrs (for tests/test-io-channel-socket )
+
+have_ifaddrs_h=yes
+if ! check_include "ifaddrs.h" ; then
+  have_ifaddrs_h=no
+fi
 
 ##########################################
 # VTE probe
@@ -2287,9 +2466,7 @@ EOF
     if test "$_sdlversion" -lt 121 ; then
       sdl_too_old=yes
     else
-      if test "$cocoa" = "no" ; then
-        sdl=yes
-      fi
+      sdl=yes
     fi
 
     # static link with sdl ? (note: sdl.pc's --static --libs is broken)
@@ -2353,28 +2530,6 @@ EOF
   fi
 fi
 
-##########################################
-# VNC TLS/WS detection
-if test "$vnc" = "yes" -a "$vnc_tls" != "no" ; then
-  cat > $TMPC <<EOF
-#include <gnutls/gnutls.h>
-int main(void) { gnutls_session_t s; gnutls_init(&s, GNUTLS_SERVER); return 0; }
-EOF
-  vnc_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
-  vnc_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
-  if compile_prog "$vnc_tls_cflags" "$vnc_tls_libs" ; then
-    if test "$vnc_tls" != "no" ; then
-      vnc_tls=yes
-    fi
-    libs_softmmu="$vnc_tls_libs $libs_softmmu"
-    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_tls_cflags"
-  else
-    if test "$vnc_tls" = "yes" ; then
-      feature_not_found "vnc-tls" "Install gnutls devel"
-    fi
-    vnc_tls=no
-  fi
-fi
 
 ##########################################
 # VNC SASL detection
@@ -2697,7 +2852,7 @@ fi
 # curses probe
 if test "$curses" != "no" ; then
   if test "$mingw32" = "yes" ; then
-    curses_list="-lpdcurses"
+    curses_list="$($pkg_config --libs ncurses 2>/dev/null):-lpdcurses"
   else
     curses_list="$($pkg_config --libs ncurses 2>/dev/null):-lncurses:-lcurses"
   fi
@@ -2795,6 +2950,30 @@ for i in $glib_modules; do
     fi
 done
 
+# Sanity check that the current size_t matches the
+# size that glib thinks it should be. This catches
+# problems on multi-arch where people try to build
+# 32-bit QEMU while pointing at 64-bit glib headers
+cat > $TMPC <<EOF
+#include <glib.h>
+#include <unistd.h>
+
+#define QEMU_BUILD_BUG_ON(x) \
+  typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused));
+
+int main(void) {
+   QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T);
+   return 0;
+}
+EOF
+
+if ! compile_prog "-Werror $CFLAGS" "$LIBS" ; then
+    error_exit "sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T."\
+               "You probably need to set PKG_CONFIG_LIBDIR"\
+	       "to point to the right pkg-config files for your"\
+	       "build target"
+fi
+
 # g_test_trap_subprocess added in 2.38. Used by some tests.
 glib_subprocess=yes
 if ! $pkg_config --atleast-version=2.38 glib-2.0; then
@@ -3149,28 +3328,17 @@ fi
 libs_softmmu="$libs_softmmu $fdt_libs"
 
 ##########################################
-# opengl probe (for sdl2, milkymist-tmu2)
-
-# GLX probe, used by milkymist-tmu2
-# this is temporary, code will be switched to egl mid-term.
-cat > $TMPC << EOF
-#include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
-int main(void) { glBegin(0); glXQueryVersion(0,0,0); return 0; }
-EOF
-if compile_prog "" "-lGL -lX11" ; then
-  have_glx=yes
-else
-  have_glx=no
-fi
+# opengl probe (for sdl2, gtk, milkymist-tmu2)
 
 if test "$opengl" != "no" ; then
-  opengl_pkgs="gl glesv2 epoxy egl"
-  if $pkg_config $opengl_pkgs x11 && test "$have_glx" = "yes"; then
+  opengl_pkgs="epoxy libdrm gbm"
+  if $pkg_config $opengl_pkgs x11; then
     opengl_cflags="$($pkg_config --cflags $opengl_pkgs) $x11_cflags"
     opengl_libs="$($pkg_config --libs $opengl_pkgs) $x11_libs"
     opengl=yes
+    if test "$gtk" = "yes" && $pkg_config --exists "$gtkpackage >= 3.16"; then
+        gtk_gl="yes"
+    fi
   else
     if test "$opengl" = "yes" ; then
       feature_not_found "opengl" "Please install opengl (mesa) devel pkgs: $opengl_pkgs"
@@ -3181,6 +3349,18 @@ if test "$opengl" != "no" ; then
   fi
 fi
 
+if test "$opengl" = "yes"; then
+  cat > $TMPC << EOF
+#include <epoxy/egl.h>
+#ifndef EGL_MESA_image_dma_buf_export
+# error mesa/epoxy lacks support for dmabufs (mesa 10.6+)
+#endif
+int main(void) { return 0; }
+EOF
+  if compile_prog "" "" ; then
+    opengl_dmabuf=yes
+  fi
+fi
 
 ##########################################
 # archipelago probe
@@ -3221,6 +3401,9 @@ if test "$glusterfs" != "no" ; then
     glusterfs="yes"
     glusterfs_cflags=`$pkg_config --cflags glusterfs-api`
     glusterfs_libs=`$pkg_config --libs glusterfs-api`
+    if $pkg_config --atleast-version=4 glusterfs-api; then
+      glusterfs_xlator_opt="yes"
+    fi
     if $pkg_config --atleast-version=5 glusterfs-api; then
       glusterfs_discard="yes"
     fi
@@ -3361,6 +3544,11 @@ EOF
   fi
 fi
 
+if test "$tcmalloc" = "yes" && test "$jemalloc" = "yes" ; then
+    echo "ERROR: tcmalloc && jemalloc can't be used at the same time"
+    exit 1
+fi
+
 ##########################################
 # tcmalloc probe
 
@@ -3378,6 +3566,22 @@ EOF
 fi
 
 ##########################################
+# jemalloc probe
+
+if test "$jemalloc" = "yes" ; then
+  cat > $TMPC << EOF
+#include <stdlib.h>
+int main(void) { malloc(1); return 0; }
+EOF
+
+  if compile_prog "" "-ljemalloc" ; then
+    LIBS="-ljemalloc $LIBS"
+  else
+    feature_not_found "jemalloc" "install jemalloc devel"
+  fi
+fi
+
+##########################################
 # signalfd probe
 signalfd="no"
 cat > $TMPC << EOF
@@ -3405,6 +3609,22 @@ if compile_prog "" "" ; then
   eventfd=yes
 fi
 
+# check if memfd is supported
+memfd=no
+cat > $TMPC << EOF
+#include <sys/memfd.h>
+
+int main(void)
+{
+    return memfd_create("foo", MFD_ALLOW_SEALING);
+}
+EOF
+if compile_prog "" "" ; then
+  memfd=yes
+fi
+
+
+
 # check for fallocate
 fallocate=no
 cat > $TMPC << EOF
@@ -3760,34 +3980,20 @@ EOF
   fi
 fi
 
-# check for libcacard for smartcard support
+# check for smartcard support
 smartcard_cflags=""
-# TODO - what's the minimal nss version we support?
-if test "$smartcard_nss" != "no"; then
-  cat > $TMPC << EOF
-#include <pk11pub.h>
-int main(void) { PK11_FreeSlot(0); return 0; }
-EOF
-    # FIXME: do not include $glib_* in here
-    nss_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
-    nss_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
-    test_cflags="$nss_cflags"
-    # The header files in nss < 3.13.3 have a bug which causes them to
-    # emit a warning. If we're going to compile QEMU with -Werror, then
-    # test that the headers don't have this bug. Otherwise we would pass
-    # the configure test but fail to compile QEMU later.
-    if test "$werror" = "yes"; then
-        test_cflags="-Werror $test_cflags"
-    fi
-    if test -n "$libtool" &&
-       $pkg_config --atleast-version=3.12.8 nss && \
-      compile_prog "$test_cflags" "$nss_libs"; then
-        smartcard_nss="yes"
+if test "$smartcard" != "no"; then
+    if $pkg_config libcacard; then
+        libcacard_cflags=$($pkg_config --cflags libcacard)
+        libcacard_libs=$($pkg_config --libs libcacard)
+        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
+        libs_softmmu="$libs_softmmu $libcacard_libs"
+        smartcard="yes"
     else
-        if test "$smartcard_nss" = "yes"; then
-            feature_not_found "nss" "Install nss devel >= 3.12.8"
+        if test "$smartcard" = "yes"; then
+            feature_not_found "smartcard" "Install libcacard devel"
         fi
-        smartcard_nss="no"
+        smartcard="no"
     fi
 fi
 
@@ -3845,6 +4051,7 @@ EOF
     guest_agent_with_vss="yes"
     QEMU_CFLAGS="$QEMU_CFLAGS $vss_win32_include"
     libs_qga="-lole32 -loleaut32 -lshlwapi -luuid -lstdc++ -Wl,--enable-stdcall-fixup $libs_qga"
+    qga_vss_provider="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb"
   else
     if test "$vss_win32_sdk" != "" ; then
       echo "ERROR: Please download and install Microsoft VSS SDK:"
@@ -3899,55 +4106,24 @@ EOF
 fi
 
 ##########################################
-# Guest agent Window MSI  package
+# virgl renderer probe
 
-if test "$guest_agent" != yes; then
-  if test "$guest_agent_msi" = yes; then
-    error_exit "MSI guest agent package requires guest agent enabled"
-  fi
-  guest_agent_msi=no
-elif test "$mingw32" != "yes"; then
-  if test "$guest_agent_msi" = "yes"; then
-    error_exit "MSI guest agent package is available only for MinGW Windows cross-compilation"
-  fi
-  guest_agent_msi=no
-elif ! has wixl; then
-  if test "$guest_agent_msi" = "yes"; then
-    error_exit "MSI guest agent package requires wixl tool installed ( usually from msitools package )"
+if test "$virglrenderer" != "no" ; then
+  cat > $TMPC << EOF
+#include <virglrenderer.h>
+int main(void) { virgl_renderer_poll(); return 0; }
+EOF
+  virgl_cflags=$($pkg_config --cflags virglrenderer 2>/dev/null)
+  virgl_libs=$($pkg_config --libs virglrenderer 2>/dev/null)
+  if $pkg_config virglrenderer >/dev/null 2>&1 && \
+     compile_prog "$virgl_cflags" "$virgl_libs" ; then
+    virglrenderer="yes"
+  else
+    if test "$virglrenderer" = "yes" ; then
+      feature_not_found "virglrenderer"
+    fi
+    virglrenderer="no"
   fi
-  guest_agent_msi=no
-fi
-
-if test "$guest_agent_msi" != "no"; then
-  if test "$guest_agent_with_vss" = "yes"; then
-    QEMU_GA_MSI_WITH_VSS="-D InstallVss"
-  fi
-
-  if test "$QEMU_GA_MANUFACTURER" = ""; then
-    QEMU_GA_MANUFACTURER=QEMU
-  fi
-
-  if test "$QEMU_GA_DISTRO" = ""; then
-    QEMU_GA_DISTRO=Linux
-  fi
-
-  if test "$QEMU_GA_VERSION" = ""; then
-      QEMU_GA_VERSION=`cat $source_path/VERSION`
-  fi
-
-  QEMU_GA_MSI_MINGW_DLL_PATH="-D Mingw_dlls=`$pkg_config --variable=prefix glib-2.0`/bin"
-  
-  case "$cpu" in
-  x86_64)
-    QEMU_GA_MSI_ARCH="-a x64 -D Arch=64"
-    ;;
-  i386)
-    QEMU_GA_MSI_ARCH="-D Arch=32"
-    ;;
-  *)
-    error_exit "CPU $cpu not supported for building installation package"
-    ;;
-  esac
 fi
 
 ##########################################
@@ -4279,6 +4455,7 @@ fi
 # check if ccache is interfering with
 # semantic analysis of macros
 
+unset CCACHE_CPP2
 ccache_cpp2=no
 cat > $TMPC << EOF
 static const int Z = 1;
@@ -4302,6 +4479,35 @@ if ! compile_object "-Werror"; then
     ccache_cpp2=yes
 fi
 
+#################################################
+# clang does not support glibc + FORTIFY_SOURCE.
+
+if test "$fortify_source" != "no"; then
+  if echo | $cc -dM -E - | grep __clang__ > /dev/null 2>&1 ; then
+    fortify_source="no";
+  elif test -n "$cxx" &&
+       echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then
+    fortify_source="no";
+  else
+    fortify_source="yes"
+  fi
+fi
+
+##########################################
+# check if struct fsxattr is available via linux/fs.h
+
+have_fsxattr=no
+cat > $TMPC << EOF
+#include <linux/fs.h>
+struct fsxattr foo;
+int main(void) {
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    have_fsxattr=yes
+fi
+
 ##########################################
 # End of CC checks
 # After here, no more $cc or $ld runs
@@ -4309,8 +4515,10 @@ fi
 if test "$gcov" = "yes" ; then
   CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
   LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
-elif test "$debug" = "no" ; then
+elif test "$fortify_source" = "yes" ; then
   CFLAGS="-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $CFLAGS"
+elif test "$debug" = "no"; then
+  CFLAGS="-O2 $CFLAGS"
 fi
 
 ##########################################
@@ -4375,6 +4583,7 @@ if test "$want_tools" = "yes" ; then
   tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) $tools"
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
     tools="qemu-nbd\$(EXESUF) $tools"
+    tools="ivshmem-client\$(EXESUF) ivshmem-server\$(EXESUF) $tools"
   fi
 fi
 if test "$softmmu" = yes ; then
@@ -4390,12 +4599,12 @@ if test "$softmmu" = yes ; then
     fi
   fi
 fi
+
+# Probe for guest agent support/options
+
 if [ "$guest_agent" != "no" ]; then
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" -o "$mingw32" = "yes" ] ; then
-      tools="qemu-ga\$(EXESUF) $tools"
-      if [ "$mingw32" = "yes" -a "$guest_agent_with_vss" = "yes" ]; then
-        tools="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb $tools"
-      fi
+      tools="qemu-ga $tools"
       guest_agent=yes
   elif [ "$guest_agent" != yes ]; then
       guest_agent=no
@@ -4404,6 +4613,63 @@ if [ "$guest_agent" != "no" ]; then
   fi
 fi
 
+# Guest agent Window MSI  package
+
+if test "$guest_agent" != yes; then
+  if test "$guest_agent_msi" = yes; then
+    error_exit "MSI guest agent package requires guest agent enabled"
+  fi
+  guest_agent_msi=no
+elif test "$mingw32" != "yes"; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package is available only for MinGW Windows cross-compilation"
+  fi
+  guest_agent_msi=no
+elif ! has wixl; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package requires wixl tool installed ( usually from msitools package )"
+  fi
+  guest_agent_msi=no
+else
+  # we support qemu-ga, mingw32, and wixl: default to MSI enabled if it wasn't
+  # disabled explicitly
+  if test "$guest_agent_msi" != "no"; then
+    guest_agent_msi=yes
+  fi
+fi
+
+if test "$guest_agent_msi" = "yes"; then
+  if test "$guest_agent_with_vss" = "yes"; then
+    QEMU_GA_MSI_WITH_VSS="-D InstallVss"
+  fi
+
+  if test "$QEMU_GA_MANUFACTURER" = ""; then
+    QEMU_GA_MANUFACTURER=QEMU
+  fi
+
+  if test "$QEMU_GA_DISTRO" = ""; then
+    QEMU_GA_DISTRO=Linux
+  fi
+
+  if test "$QEMU_GA_VERSION" = ""; then
+      QEMU_GA_VERSION=`cat $source_path/VERSION`
+  fi
+
+  QEMU_GA_MSI_MINGW_DLL_PATH="-D Mingw_dlls=`$pkg_config --variable=prefix glib-2.0`/bin"
+
+  case "$cpu" in
+  x86_64)
+    QEMU_GA_MSI_ARCH="-a x64 -D Arch=64"
+    ;;
+  i386)
+    QEMU_GA_MSI_ARCH="-D Arch=32"
+    ;;
+  *)
+    error_exit "CPU $cpu not supported for building installation package"
+    ;;
+  esac
+fi
+
 # Mac OS X ships with a broken assembler
 roms=
 if test \( "$cpu" = "i386" -o "$cpu" = "x86_64" \) -a \
@@ -4503,12 +4769,22 @@ fi
 echo "pixman            $pixman"
 echo "SDL support       $sdl"
 echo "GTK support       $gtk"
+echo "GTK GL support    $gtk_gl"
 echo "GNUTLS support    $gnutls"
 echo "GNUTLS hash       $gnutls_hash"
-echo "GNUTLS gcrypt     $gnutls_gcrypt"
-echo "GNUTLS nettle     $gnutls_nettle ${gnutls_nettle+($nettle_version)}"
+echo "GNUTLS rnd        $gnutls_rnd"
+echo "libgcrypt         $gcrypt"
+echo "libgcrypt kdf     $gcrypt_kdf"
+if test "$nettle" = "yes"; then
+    echo "nettle            $nettle ($nettle_version)"
+else
+    echo "nettle            $nettle"
+fi
+echo "nettle kdf        $nettle_kdf"
+echo "libtasn1          $tasn1"
 echo "VTE support       $vte"
 echo "curses support    $curses"
+echo "virgl support     $virglrenderer"
 echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
@@ -4517,7 +4793,6 @@ echo "Block whitelist (ro) $block_drv_ro_whitelist"
 echo "VirtFS support    $virtfs"
 echo "VNC support       $vnc"
 if test "$vnc" = "yes" ; then
-    echo "VNC TLS support   $vnc_tls"
     echo "VNC SASL support  $vnc_sasl"
     echo "VNC JPEG support  $vnc_jpeg"
     echo "VNC PNG support   $vnc_png"
@@ -4528,11 +4803,11 @@ fi
 echo "xen support       $xen"
 if test "$xen" = "yes" ; then
   echo "xen ctrl version  $xen_ctrl_version"
+  echo "pv dom build      $xen_pv_domain_build"
 fi
 echo "brlapi support    $brlapi"
 echo "bluez  support    $bluez"
 echo "Documentation     $docs"
-echo "GUEST_BASE        $guest_base"
 echo "PIE               $pie"
 echo "vde support       $vde"
 echo "netmap support    $netmap"
@@ -4553,7 +4828,7 @@ echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "vhost-scsi support $vhost_scsi"
 echo "Trace backends    $trace_backends"
-if test "$trace_backend" = "simple"; then
+if have_backend "simple"; then
 echo "Trace output file $trace_file-<pid>"
 fi
 if test "$spice" = "yes"; then
@@ -4563,15 +4838,17 @@ echo "spice support     $spice"
 fi
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
-echo "nss used          $smartcard_nss"
+echo "smartcard support $smartcard"
 echo "libusb            $libusb"
 echo "usb net redir     $usb_redir"
 echo "OpenGL support    $opengl"
+echo "OpenGL dmabufs    $opengl_dmabuf"
 echo "libiscsi support  $libiscsi"
 echo "libnfs support    $libnfs"
 echo "build guest agent $guest_agent"
 echo "QGA VSS support   $guest_agent_with_vss"
 echo "QGA w32 disk info $guest_agent_ntddscsi"
+echo "QGA MSI support   $guest_agent_msi"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine"
 echo "coroutine pool    $coroutine_pool"
@@ -4589,6 +4866,8 @@ echo "snappy support    $snappy"
 echo "bzip2 support     $bzip2"
 echo "NUMA host support $numa"
 echo "tcmalloc support  $tcmalloc"
+echo "jemalloc support  $jemalloc"
+echo "avx2 optimization $avx2_opt"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -4646,12 +4925,13 @@ if test "$mingw32" = "yes" ; then
   echo "CONFIG_PRODUCTVERSION=$version_major,$version_minor,$version_subminor,$version_micro" >> $config_host_mak
   if test "$guest_agent_with_vss" = "yes" ; then
     echo "CONFIG_QGA_VSS=y" >> $config_host_mak
+    echo "QGA_VSS_PROVIDER=$qga_vss_provider" >> $config_host_mak
     echo "WIN_SDK=\"$win_sdk\"" >> $config_host_mak
   fi
   if test "$guest_agent_ntddscsi" = "yes" ; then
     echo "CONFIG_QGA_NTDDDISK=y" >> $config_host_mak
   fi
-  if test "$guest_agent_msi" != "no"; then
+  if test "$guest_agent_msi" = "yes"; then
     echo "QEMU_GA_MSI_ENABLED=yes" >> $config_host_mak  
     echo "QEMU_GA_MSI_MINGW_DLL_PATH=${QEMU_GA_MSI_MINGW_DLL_PATH}" >> $config_host_mak
     echo "QEMU_GA_MSI_WITH_VSS=${QEMU_GA_MSI_WITH_VSS}" >> $config_host_mak
@@ -4724,9 +5004,6 @@ echo "CONFIG_BDRV_RO_WHITELIST=$block_drv_ro_whitelist" >> $config_host_mak
 if test "$vnc" = "yes" ; then
   echo "CONFIG_VNC=y" >> $config_host_mak
 fi
-if test "$vnc_tls" = "yes" ; then
-  echo "CONFIG_VNC_TLS=y" >> $config_host_mak
-fi
 if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
 fi
@@ -4785,6 +5062,9 @@ fi
 if test "$eventfd" = "yes" ; then
   echo "CONFIG_EVENTFD=y" >> $config_host_mak
 fi
+if test "$memfd" = "yes" ; then
+  echo "CONFIG_MEMFD=y" >> $config_host_mak
+fi
 if test "$fallocate" = "yes" ; then
   echo "CONFIG_FALLOCATE=y" >> $config_host_mak
 fi
@@ -4862,6 +5142,10 @@ if test "$gtk" = "yes" ; then
   echo "CONFIG_GTK=y" >> $config_host_mak
   echo "CONFIG_GTKABI=$gtkabi" >> $config_host_mak
   echo "GTK_CFLAGS=$gtk_cflags" >> $config_host_mak
+  echo "GTK_LIBS=$gtk_libs" >> $config_host_mak
+  if test "$gtk_gl" = "yes" ; then
+    echo "CONFIG_GTK_GL=y" >> $config_host_mak
+  fi
 fi
 if test "$gnutls" = "yes" ; then
   echo "CONFIG_GNUTLS=y" >> $config_host_mak
@@ -4869,20 +5153,51 @@ fi
 if test "$gnutls_hash" = "yes" ; then
   echo "CONFIG_GNUTLS_HASH=y" >> $config_host_mak
 fi
-if test "$gnutls_gcrypt" = "yes" ; then
-  echo "CONFIG_GNUTLS_GCRYPT=y" >> $config_host_mak
+if test "$gnutls_rnd" = "yes" ; then
+  echo "CONFIG_GNUTLS_RND=y" >> $config_host_mak
+fi
+if test "$gcrypt" = "yes" ; then
+  echo "CONFIG_GCRYPT=y" >> $config_host_mak
+  if test "$gcrypt_kdf" = "yes" ; then
+    echo "CONFIG_GCRYPT_KDF=y" >> $config_host_mak
+  fi
 fi
-if test "$gnutls_nettle" = "yes" ; then
-  echo "CONFIG_GNUTLS_NETTLE=y" >> $config_host_mak
+if test "$nettle" = "yes" ; then
+  echo "CONFIG_NETTLE=y" >> $config_host_mak
   echo "CONFIG_NETTLE_VERSION_MAJOR=${nettle_version%%.*}" >> $config_host_mak
+  if test "$nettle_kdf" = "yes" ; then
+    echo "CONFIG_NETTLE_KDF=y" >> $config_host_mak
+  fi
+fi
+if test "$tasn1" = "yes" ; then
+  echo "CONFIG_TASN1=y" >> $config_host_mak
+fi
+if test "$have_ifaddrs_h" = "yes" ; then
+    echo "HAVE_IFADDRS_H=y" >> $config_host_mak
+fi
+
+# Work around a system header bug with some kernel/XFS header
+# versions where they both try to define 'struct fsxattr':
+# xfs headers will not try to redefine structs from linux headers
+# if this macro is set.
+if test "$have_fsxattr" = "yes" ; then
+    echo "HAVE_FSXATTR=y" >> $config_host_mak
 fi
 if test "$vte" = "yes" ; then
   echo "CONFIG_VTE=y" >> $config_host_mak
   echo "VTE_CFLAGS=$vte_cflags" >> $config_host_mak
 fi
+if test "$virglrenderer" = "yes" ; then
+  echo "CONFIG_VIRGL=y" >> $config_host_mak
+  echo "VIRGL_CFLAGS=$virgl_cflags" >> $config_host_mak
+  echo "VIRGL_LIBS=$virgl_libs" >> $config_host_mak
+fi
 if test "$xen" = "yes" ; then
   echo "CONFIG_XEN_BACKEND=y" >> $config_host_mak
   echo "CONFIG_XEN_CTRL_INTERFACE_VERSION=$xen_ctrl_version" >> $config_host_mak
+  if test "$xen_pv_domain_build" = "yes" ; then
+    echo "CONFIG_XEN_PV_DOMAIN_BUILD=y" >> $config_host_mak
+  fi
 fi
 if test "$linux_aio" = "yes" ; then
   echo "CONFIG_LINUX_AIO=y" >> $config_host_mak
@@ -4937,10 +5252,8 @@ if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
 fi
 
-if test "$smartcard_nss" = "yes" ; then
-  echo "CONFIG_SMARTCARD_NSS=y" >> $config_host_mak
-  echo "NSS_LIBS=$nss_libs" >> $config_host_mak
-  echo "NSS_CFLAGS=$nss_cflags" >> $config_host_mak
+if test "$smartcard" = "yes" ; then
+  echo "CONFIG_SMARTCARD=y" >> $config_host_mak
 fi
 
 if test "$libusb" = "yes" ; then
@@ -4955,6 +5268,13 @@ if test "$opengl" = "yes" ; then
   echo "CONFIG_OPENGL=y" >> $config_host_mak
   echo "OPENGL_CFLAGS=$opengl_cflags" >> $config_host_mak
   echo "OPENGL_LIBS=$opengl_libs" >> $config_host_mak
+  if test "$opengl_dmabuf" = "yes" ; then
+    echo "CONFIG_OPENGL_DMABUF=y" >> $config_host_mak
+  fi
+fi
+
+if test "$avx2_opt" = "yes" ; then
+  echo "CONFIG_AVX2_OPT=y" >> $config_host_mak
 fi
 
 if test "$lzo" = "yes" ; then
@@ -4992,6 +5312,9 @@ fi
 if test "$zero_malloc" = "yes" ; then
   echo "CONFIG_ZERO_MALLOC=y" >> $config_host_mak
 fi
+if test "$localtime_r" = "yes" ; then
+  echo "CONFIG_LOCALTIME_R=y" >> $config_host_mak
+fi
 if test "$qom_cast_debug" = "yes" ; then
   echo "CONFIG_QOM_CAST_DEBUG=y" >> $config_host_mak
 fi
@@ -5046,6 +5369,10 @@ if test "$glusterfs" = "yes" ; then
   echo "GLUSTERFS_LIBS=$glusterfs_libs" >> $config_host_mak
 fi
 
+if test "$glusterfs_xlator_opt" = "yes" ; then
+  echo "CONFIG_GLUSTERFS_XLATOR_OPT=y" >> $config_host_mak
+fi
+
 if test "$glusterfs_discard" = "yes" ; then
   echo "CONFIG_GLUSTERFS_DISCARD=y" >> $config_host_mak
 fi
@@ -5093,8 +5420,8 @@ if have_backend "simple"; then
   # Set the appropriate trace file.
   trace_file="\"$trace_file-\" FMT_pid"
 fi
-if have_backend "stderr"; then
-  echo "CONFIG_TRACE_STDERR=y" >> $config_host_mak
+if have_backend "log"; then
+  echo "CONFIG_TRACE_LOG=y" >> $config_host_mak
 fi
 if have_backend "ust"; then
   echo "CONFIG_TRACE_UST=y" >> $config_host_mak
@@ -5149,13 +5476,8 @@ echo "MAKE=$make" >> $config_host_mak
 echo "INSTALL=$install" >> $config_host_mak
 echo "INSTALL_DIR=$install -d -m 0755" >> $config_host_mak
 echo "INSTALL_DATA=$install -c -m 0644" >> $config_host_mak
-if test -n "$libtool"; then
-  echo "INSTALL_PROG=\$(LIBTOOL) --mode=install $install -c -m 0755" >> $config_host_mak
-  echo "INSTALL_LIB=\$(LIBTOOL) --mode=install $install -c -m 0644" >> $config_host_mak
-else
-  echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
-  echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
-fi
+echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
 echo "PYTHON=$python" >> $config_host_mak
 echo "CC=$cc" >> $config_host_mak
 if $iasl -h > /dev/null 2>&1; then
@@ -5173,7 +5495,6 @@ echo "OBJCOPY=$objcopy" >> $config_host_mak
 echo "LD=$ld" >> $config_host_mak
 echo "NM=$nm" >> $config_host_mak
 echo "WINDRES=$windres" >> $config_host_mak
-echo "LIBTOOL=$libtool" >> $config_host_mak
 echo "CFLAGS=$CFLAGS" >> $config_host_mak
 echo "CFLAGS_NOPIE=$CFLAGS_NOPIE" >> $config_host_mak
 echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak
@@ -5192,13 +5513,14 @@ else
 fi
 echo "LDFLAGS=$LDFLAGS" >> $config_host_mak
 echo "LDFLAGS_NOPIE=$LDFLAGS_NOPIE" >> $config_host_mak
-echo "LIBTOOLFLAGS=$LIBTOOLFLAGS" >> $config_host_mak
 echo "LIBS+=$LIBS" >> $config_host_mak
 echo "LIBS_TOOLS+=$libs_tools" >> $config_host_mak
 echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "DSOSUF=$DSOSUF" >> $config_host_mak
 echo "LDFLAGS_SHARED=$LDFLAGS_SHARED" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
+echo "TASN1_LIBS=$tasn1_libs" >> $config_host_mak
+echo "TASN1_CFLAGS=$tasn1_cflags" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
 if test "$gcov" = "yes" ; then
@@ -5348,20 +5670,20 @@ case "$target_name" in
   ppc64)
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   ppc64le)
     TARGET_ARCH=ppc64
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   ppc64abi32)
     TARGET_ARCH=ppc64
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     echo "TARGET_ABI32=y" >> $config_target_mak
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   sh4|sh4eb)
     TARGET_ARCH=sh4
@@ -5379,7 +5701,9 @@ case "$target_name" in
     echo "TARGET_ABI32=y" >> $config_target_mak
   ;;
   s390x)
-    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml"
+    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml s390-cr.xml s390-virt.xml"
+  ;;
+  tilegx)
   ;;
   tricore)
   ;;
@@ -5442,6 +5766,7 @@ case "$target_name" in
       echo "CONFIG_KVM=y" >> $config_target_mak
       if test "$vhost_net" = "yes" ; then
         echo "CONFIG_VHOST_NET=y" >> $config_target_mak
+        echo "CONFIG_VHOST_NET_TEST_$target_name=y" >> $config_host_mak
       fi
     fi
 esac
@@ -5469,9 +5794,6 @@ fi
 if test "$target_user_only" = "yes" -a "$bflt" = "yes"; then
   echo "TARGET_HAS_BFLT=y" >> $config_target_mak
 fi
-if test "$target_user_only" = "yes" -a "$guest_base" = "yes"; then
-  echo "CONFIG_USE_GUEST_BASE=y" >> $config_target_mak
-fi
 if test "$target_bsd_user" = "yes" ; then
   echo "CONFIG_BSD_USER=y" >> $config_target_mak
 fi
@@ -5481,91 +5803,76 @@ fi
 cflags=""
 ldflags=""
 
+disas_config() {
+  echo "CONFIG_${1}_DIS=y" >> $config_target_mak
+  echo "CONFIG_${1}_DIS=y" >> config-all-disas.mak
+}
+
 for i in $ARCH $TARGET_BASE_ARCH ; do
   case "$i" in
   alpha)
-    echo "CONFIG_ALPHA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ALPHA_DIS=y"  >> config-all-disas.mak
+    disas_config "ALPHA"
   ;;
   aarch64)
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   arm)
-    echo "CONFIG_ARM_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ARM_DIS=y"  >> config-all-disas.mak
+    disas_config "ARM"
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   cris)
-    echo "CONFIG_CRIS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_CRIS_DIS=y"  >> config-all-disas.mak
+    disas_config "CRIS"
   ;;
   hppa)
-    echo "CONFIG_HPPA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_HPPA_DIS=y"  >> config-all-disas.mak
+    disas_config "HPPA"
   ;;
   i386|x86_64|x32)
-    echo "CONFIG_I386_DIS=y"  >> $config_target_mak
-    echo "CONFIG_I386_DIS=y"  >> config-all-disas.mak
+    disas_config "I386"
   ;;
   ia64*)
-    echo "CONFIG_IA64_DIS=y"  >> $config_target_mak
-    echo "CONFIG_IA64_DIS=y"  >> config-all-disas.mak
+    disas_config "IA64"
   ;;
   lm32)
-    echo "CONFIG_LM32_DIS=y"  >> $config_target_mak
-    echo "CONFIG_LM32_DIS=y"  >> config-all-disas.mak
+    disas_config "LM32"
   ;;
   m68k)
-    echo "CONFIG_M68K_DIS=y"  >> $config_target_mak
-    echo "CONFIG_M68K_DIS=y"  >> config-all-disas.mak
+    disas_config "M68K"
   ;;
   microblaze*)
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> config-all-disas.mak
+    disas_config "MICROBLAZE"
   ;;
   mips*)
-    echo "CONFIG_MIPS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MIPS_DIS=y"  >> config-all-disas.mak
+    disas_config "MIPS"
   ;;
   moxie*)
-    echo "CONFIG_MOXIE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MOXIE_DIS=y"  >> config-all-disas.mak
+    disas_config "MOXIE"
   ;;
   or32)
-    echo "CONFIG_OPENRISC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_OPENRISC_DIS=y"  >> config-all-disas.mak
+    disas_config "OPENRISC"
   ;;
   ppc*)
-    echo "CONFIG_PPC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_PPC_DIS=y"  >> config-all-disas.mak
+    disas_config "PPC"
   ;;
   s390*)
-    echo "CONFIG_S390_DIS=y"  >> $config_target_mak
-    echo "CONFIG_S390_DIS=y"  >> config-all-disas.mak
+    disas_config "S390"
   ;;
   sh4)
-    echo "CONFIG_SH4_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SH4_DIS=y"  >> config-all-disas.mak
+    disas_config "SH4"
   ;;
   sparc*)
-    echo "CONFIG_SPARC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SPARC_DIS=y"  >> config-all-disas.mak
+    disas_config "SPARC"
   ;;
   xtensa*)
-    echo "CONFIG_XTENSA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_XTENSA_DIS=y"  >> config-all-disas.mak
+    disas_config "XTENSA"
   ;;
   esac
 done
 if test "$tcg_interpreter" = "yes" ; then
-  echo "CONFIG_TCI_DIS=y"  >> $config_target_mak
-  echo "CONFIG_TCI_DIS=y"  >> config-all-disas.mak
+  disas_config "TCI"
 fi
 
 case "$ARCH" in
@@ -5600,10 +5907,6 @@ if [ "$pixman" = "internal" ]; then
   echo "config-host.h: subdir-pixman" >> $config_host_mak
 fi
 
-if test "$rdma" = "yes" ; then
-echo "CONFIG_RDMA=y" >> $config_host_mak
-fi
-
 if [ "$dtc_internal" = "yes" ]; then
   echo "config-host.h: subdir-dtc" >> $config_host_mak
 fi
@@ -5689,7 +5992,7 @@ cat <<EOD >config.status
 EOD
 printf "exec" >>config.status
 printf " '%s'" "$0" "$@" >>config.status
-echo >>config.status
+echo ' "$@"' >>config.status
 chmod +x config.status
 
 rm -r "$TMPDIR1"
diff --git a/qemu/contrib/ivshmem-client/Makefile.objs b/qemu/contrib/ivshmem-client/Makefile.objs
new file mode 100644
index 000000000..bfab2d20d
--- /dev/null
+++ b/qemu/contrib/ivshmem-client/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-client-obj-y = ivshmem-client.o main.o
diff --git a/qemu/contrib/ivshmem-client/ivshmem-client.c b/qemu/contrib/ivshmem-client/ivshmem-client.c
new file mode 100644
index 000000000..44ae3646e
--- /dev/null
+++ b/qemu/contrib/ivshmem-client/ivshmem-client.c
@@ -0,0 +1,446 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+
+#include "ivshmem-client.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_CLIENT_DEBUG(client, fmt, ...) do { \
+        if ((client)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/* read message from the unix socket */
+static int
+ivshmem_client_read_one_msg(IvshmemClient *client, int64_t *index, int *fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    iov[0].iov_base = index;
+    iov[0].iov_len = sizeof(*index);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = &msg_control;
+    msg.msg_controllen = sizeof(msg_control);
+
+    ret = recvmsg(client->sock_fd, &msg, 0);
+    if (ret < sizeof(*index)) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read message: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+    if (ret == 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "lost connection to server\n");
+        return -1;
+    }
+
+    *index = GINT64_FROM_LE(*index);
+    *fd = -1;
+
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+
+        if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+
+        memcpy(fd, CMSG_DATA(cmsg), sizeof(*fd));
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * client is freed */
+static void
+ivshmem_client_free_peer(IvshmemClient *client, IvshmemClientPeer *peer)
+{
+    unsigned vector;
+
+    QTAILQ_REMOVE(&client->peer_list, peer, next);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        close(peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* handle message coming from server (new peer, new vectors) */
+static int
+ivshmem_client_handle_server_msg(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int64_t peer_id;
+    int ret, fd;
+
+    ret = ivshmem_client_read_one_msg(client, &peer_id, &fd);
+    if (ret < 0) {
+        return -1;
+    }
+
+    /* can return a peer or the local client */
+    peer = ivshmem_client_search_peer(client, peer_id);
+
+    /* delete peer */
+    if (fd == -1) {
+
+        if (peer == NULL || peer == &client->local) {
+            IVSHMEM_CLIENT_DEBUG(client, "receive delete for invalid "
+                                 "peer %" PRId64 "\n", peer_id);
+            return -1;
+        }
+
+        IVSHMEM_CLIENT_DEBUG(client, "delete peer id = %" PRId64 "\n", peer_id);
+        ivshmem_client_free_peer(client, peer);
+        return 0;
+    }
+
+    /* new peer */
+    if (peer == NULL) {
+        peer = g_malloc0(sizeof(*peer));
+        peer->id = peer_id;
+        peer->vectors_count = 0;
+        QTAILQ_INSERT_TAIL(&client->peer_list, peer, next);
+        IVSHMEM_CLIENT_DEBUG(client, "new peer id = %" PRId64 "\n", peer_id);
+    }
+
+    /* new vector */
+    IVSHMEM_CLIENT_DEBUG(client, "  new vector %d (fd=%d) for peer id %"
+                         PRId64 "\n", peer->vectors_count, fd, peer->id);
+    if (peer->vectors_count >= G_N_ELEMENTS(peer->vectors)) {
+        IVSHMEM_CLIENT_DEBUG(client, "Too many vectors received, failing");
+        return -1;
+    }
+
+    peer->vectors[peer->vectors_count] = fd;
+    peer->vectors_count++;
+
+    return 0;
+}
+
+/* init a new ivshmem client */
+int
+ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                    IvshmemClientNotifCb notif_cb, void *notif_arg,
+                    bool verbose)
+{
+    int ret;
+    unsigned i;
+
+    memset(client, 0, sizeof(*client));
+
+    ret = snprintf(client->unix_sock_path, sizeof(client->unix_sock_path),
+                   "%s", unix_sock_path);
+
+    if (ret < 0 || ret >= sizeof(client->unix_sock_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        return -1;
+    }
+
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        client->local.vectors[i] = -1;
+    }
+
+    QTAILQ_INIT(&client->peer_list);
+    client->local.id = -1;
+
+    client->notif_cb = notif_cb;
+    client->notif_arg = notif_arg;
+    client->verbose = verbose;
+    client->shm_fd = -1;
+    client->sock_fd = -1;
+
+    return 0;
+}
+
+/* create and connect to the unix socket */
+int
+ivshmem_client_connect(IvshmemClient *client)
+{
+    struct sockaddr_un sun;
+    int fd, ret;
+    int64_t tmp;
+
+    IVSHMEM_CLIENT_DEBUG(client, "connect to client %s\n",
+                         client->unix_sock_path);
+
+    client->sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (client->sock_fd < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot create socket: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   client->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        goto err_close;
+    }
+
+    if (connect(client->sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close;
+    }
+
+    /* first, we expect a protocol version */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        (tmp != IVSHMEM_PROTOCOL_VERSION) || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server\n");
+        goto err_close;
+    }
+
+    /* then, we expect our index + a fd == -1 */
+    if (ivshmem_client_read_one_msg(client, &client->local.id, &fd) < 0 ||
+        client->local.id < 0 || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (2)\n");
+        goto err_close;
+    }
+    IVSHMEM_CLIENT_DEBUG(client, "our_id=%" PRId64 "\n", client->local.id);
+
+    /* now, we expect shared mem fd + a -1 index, note that shm fd
+     * is not used */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        tmp != -1 || fd < 0) {
+        if (fd >= 0) {
+            close(fd);
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (3)\n");
+        goto err_close;
+    }
+    client->shm_fd = fd;
+    IVSHMEM_CLIENT_DEBUG(client, "shm_fd=%d\n", fd);
+
+    return 0;
+
+err_close:
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    return -1;
+}
+
+/* close connection to the server, and free all peer structures */
+void
+ivshmem_client_close(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    unsigned i;
+
+    IVSHMEM_CLIENT_DEBUG(client, "close client\n");
+
+    while ((peer = QTAILQ_FIRST(&client->peer_list)) != NULL) {
+        ivshmem_client_free_peer(client, peer);
+    }
+
+    close(client->shm_fd);
+    client->shm_fd = -1;
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    client->local.id = -1;
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        close(client->local.vectors[i]);
+        client->local.vectors[i] = -1;
+    }
+    client->local.vectors_count = 0;
+}
+
+/* get the fd_set according to the unix socket and peer list */
+void
+ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds, int *maxfd)
+{
+    int fd;
+    unsigned vector;
+
+    FD_SET(client->sock_fd, fds);
+    if (client->sock_fd >= *maxfd) {
+        *maxfd = client->sock_fd + 1;
+    }
+
+    for (vector = 0; vector < client->local.vectors_count; vector++) {
+        fd = client->local.vectors[vector];
+        FD_SET(fd, fds);
+        if (fd >= *maxfd) {
+            *maxfd = fd + 1;
+        }
+    }
+}
+
+/* handle events from eventfd: just print a message on notification */
+static int
+ivshmem_client_handle_event(IvshmemClient *client, const fd_set *cur, int maxfd)
+{
+    IvshmemClientPeer *peer;
+    uint64_t kick;
+    unsigned i;
+    int ret;
+
+    peer = &client->local;
+
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (peer->vectors[i] >= maxfd || !FD_ISSET(peer->vectors[i], cur)) {
+            continue;
+        }
+
+        ret = read(peer->vectors[i], &kick, sizeof(kick));
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret != sizeof(kick)) {
+            IVSHMEM_CLIENT_DEBUG(client, "invalid read size = %d\n", ret);
+            errno = EINVAL;
+            return -1;
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "received event on fd %d vector %d: %"
+                             PRIu64 "\n", peer->vectors[i], i, kick);
+        if (client->notif_cb != NULL) {
+            client->notif_cb(client, peer, i, client->notif_arg);
+        }
+    }
+
+    return 0;
+}
+
+/* read and handle new messages on the given fd_set */
+int
+ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd)
+{
+    if (client->sock_fd < maxfd && FD_ISSET(client->sock_fd, fds) &&
+        ivshmem_client_handle_server_msg(client) < 0 && errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_server_msg() "
+                             "failed\n");
+        return -1;
+    } else if (ivshmem_client_handle_event(client, fds, maxfd) < 0 &&
+               errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_event() failed\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+/* send a notification on a vector of a peer */
+int
+ivshmem_client_notify(const IvshmemClient *client,
+                      const IvshmemClientPeer *peer, unsigned vector)
+{
+    uint64_t kick;
+    int fd;
+
+    if (vector >= peer->vectors_count) {
+        IVSHMEM_CLIENT_DEBUG(client, "invalid vector %u on peer %" PRId64 "\n",
+                             vector, peer->id);
+        return -1;
+    }
+    fd = peer->vectors[vector];
+    IVSHMEM_CLIENT_DEBUG(client, "notify peer %" PRId64
+                         " on vector %d, fd %d\n", peer->id, vector, fd);
+
+    kick = 1;
+    if (write(fd, &kick, sizeof(kick)) != sizeof(kick)) {
+        fprintf(stderr, "could not write to %d: %s\n", peer->vectors[vector],
+                strerror(errno));
+        return -1;
+    }
+    return 0;
+}
+
+/* send a notification to all vectors of a peer */
+int
+ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                const IvshmemClientPeer *peer)
+{
+    unsigned vector;
+    int ret = 0;
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        if (ivshmem_client_notify(client, peer, vector) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* send a notification to all peers */
+int
+ivshmem_client_notify_broadcast(const IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int ret = 0;
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (ivshmem_client_notify_all_vects(client, peer) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* lookup peer from its id */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id)
+{
+    IvshmemClientPeer *peer;
+
+    if (peer_id == client->local.id) {
+        return &client->local;
+    }
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_client_dump(const IvshmemClient *client)
+{
+    const IvshmemClientPeer *peer;
+    unsigned vector;
+
+    /* dump local infos */
+    peer = &client->local;
+    printf("our_id = %" PRId64 "\n", peer->id);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        printf("  vector %d is enabled (fd=%d)\n", vector,
+               peer->vectors[vector]);
+    }
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   peer->vectors[vector]);
+        }
+    }
+}
diff --git a/qemu/contrib/ivshmem-client/ivshmem-client.h b/qemu/contrib/ivshmem-client/ivshmem-client.h
new file mode 100644
index 000000000..54cde17d9
--- /dev/null
+++ b/qemu/contrib/ivshmem-client/ivshmem-client.h
@@ -0,0 +1,212 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _IVSHMEM_CLIENT_H_
+#define _IVSHMEM_CLIENT_H_
+
+/**
+ * This file provides helper to implement an ivshmem client. It is used
+ * on the host to ask QEMU to send an interrupt to an ivshmem PCI device in a
+ * guest. QEMU also implements an ivshmem client similar to this one, they both
+ * connect to an ivshmem server.
+ *
+ * A standalone ivshmem client based on this file is provided for debug/test
+ * purposes.
+ */
+
+#include <sys/select.h>
+
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the client
+ */
+#define IVSHMEM_CLIENT_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, it is advertised to
+ * all connected clients through the unix socket. When our ivshmem
+ * client receives a notification, it creates a IvshmemClientPeer
+ * structure to store the infos of this peer.
+ *
+ * This structure is also used to store the information of our own
+ * client in (IvshmemClient)->local.
+ */
+typedef struct IvshmemClientPeer {
+    QTAILQ_ENTRY(IvshmemClientPeer) next;    /**< next in list*/
+    int64_t id;                              /**< the id of the peer */
+    int vectors[IVSHMEM_CLIENT_MAX_VECTORS]; /**< one fd per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemClientPeer;
+QTAILQ_HEAD(IvshmemClientPeerList, IvshmemClientPeer);
+
+typedef struct IvshmemClientPeerList IvshmemClientPeerList;
+typedef struct IvshmemClient IvshmemClient;
+
+/**
+ * Typedef of callback function used when our IvshmemClient receives a
+ * notification from a peer.
+ */
+typedef void (*IvshmemClientNotifCb)(
+    const IvshmemClient *client,
+    const IvshmemClientPeer *peer,
+    unsigned vect, void *arg);
+
+/**
+ * Structure describing an ivshmem client
+ *
+ * This structure stores all information related to our client: the name
+ * of the server unix socket, the list of peers advertised by the
+ * server, our own client information, and a pointer the notification
+ * callback function used when we receive a notification from a peer.
+ */
+struct IvshmemClient {
+    char unix_sock_path[PATH_MAX];      /**< path to unix sock */
+    int sock_fd;                        /**< unix sock filedesc */
+    int shm_fd;                         /**< shm file descriptor */
+
+    IvshmemClientPeerList peer_list;    /**< list of peers */
+    IvshmemClientPeer local;            /**< our own infos */
+
+    IvshmemClientNotifCb notif_cb;      /**< notification callback */
+    void *notif_arg;                    /**< notification argument */
+
+    bool verbose;                       /**< true to enable debug */
+};
+
+/**
+ * Initialize an ivshmem client
+ *
+ * @client:         A pointer to an uninitialized IvshmemClient structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @notif_cb:       If not NULL, the pointer to the function to be called when
+ *                  our IvshmemClient receives a notification from a peer
+ * @notif_arg:      Opaque pointer given as-is to the notification callback
+ *                  function
+ * @verbose:        True to enable debug
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                        IvshmemClientNotifCb notif_cb, void *notif_arg,
+                        bool verbose);
+
+/**
+ * Connect to the server
+ *
+ * Connect to the server unix socket, and read the first initial
+ * messages sent by the server, giving the ID of the client and the file
+ * descriptor of the shared memory.
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_connect(IvshmemClient *client);
+
+/**
+ * Close connection to the server and free all peer structures
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_close(IvshmemClient *client);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors
+ * that must be polled (unix server socket and peers eventfd). The
+ * function will not initialize the fd_set, it is up to the caller
+ * to do this.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds,
+                            int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set filled by select(), handle incoming messages from
+ * server or peers.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set containing the file descriptors to be checked. Note
+ *          that file descriptors that are not related to our client are
+ *          ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd);
+
+/**
+ * Send a notification to a vector of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ * @vector: The number of the vector
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_notify(const IvshmemClient *client,
+                          const IvshmemClientPeer *peer, unsigned vector);
+
+/**
+ * Send a notification to all vectors of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                    const IvshmemClientPeer *peer);
+
+/**
+ * Broadcat a notification to all vectors of all peers
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_broadcast(const IvshmemClient *client);
+
+/**
+ * Search a peer from its identifier
+ *
+ * Return the peer structure from its peer_id. If the given peer_id is
+ * the local id, the function returns the local peer structure.
+ *
+ * @client:  The ivshmem client
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem client on stdout
+ *
+ * Dump the id and the vectors of the given ivshmem client and the list
+ * of its peers and their vectors on stdout.
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_dump(const IvshmemClient *client);
+
+#endif /* _IVSHMEM_CLIENT_H_ */
diff --git a/qemu/contrib/ivshmem-client/main.c b/qemu/contrib/ivshmem-client/main.c
new file mode 100644
index 000000000..33ae1daa1
--- /dev/null
+++ b/qemu/contrib/ivshmem-client/main.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "ivshmem-client.h"
+
+#define IVSHMEM_CLIENT_DEFAULT_VERBOSE        0
+#define IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+
+typedef struct IvshmemClientArgs {
+    bool verbose;
+    const char *unix_sock_path;
+} IvshmemClientArgs;
+
+/* show ivshmem_client_usage and exit with given error code */
+static void
+ivshmem_client_usage(const char *name, int code)
+{
+    fprintf(stderr, "%s [opts]\n", name);
+    fprintf(stderr, "  -h: show this help\n");
+    fprintf(stderr, "  -v: verbose mode\n");
+    fprintf(stderr, "  -S <unix_sock_path>: path to the unix socket\n"
+                    "     to connect to.\n"
+                    "     default=%s\n", IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH);
+    exit(code);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_client_parse_args(IvshmemClientArgs *args, int argc, char *argv[])
+{
+    int c;
+
+    while ((c = getopt(argc, argv,
+                       "h"  /* help */
+                       "v"  /* verbose */
+                       "S:" /* unix_sock_path */
+                      )) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_client_usage(argv[0], 0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'S': /* unix_sock_path */
+            args->unix_sock_path = optarg;
+            break;
+
+        default:
+            ivshmem_client_usage(argv[0], 1);
+            break;
+        }
+    }
+}
+
+/* show command line help */
+static void
+ivshmem_client_cmdline_help(void)
+{
+    printf("dump: dump peers (including us)\n"
+           "int <peer> <vector>: notify one vector on a peer\n"
+           "int <peer> all: notify all vectors of a peer\n"
+           "int all: notify all vectors of all peers (excepting us)\n");
+}
+
+/* read stdin and handle commands */
+static int
+ivshmem_client_handle_stdin_command(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    char buf[128];
+    char *s, *token;
+    int ret;
+    int peer_id, vector;
+
+    memset(buf, 0, sizeof(buf));
+    ret = read(0, buf, sizeof(buf) - 1);
+    if (ret < 0) {
+        return -1;
+    }
+
+    s = buf;
+    while ((token = strsep(&s, "\n\r;")) != NULL) {
+        if (!strcmp(token, "")) {
+            continue;
+        }
+        if (!strcmp(token, "?")) {
+            ivshmem_client_cmdline_help();
+        }
+        if (!strcmp(token, "help")) {
+            ivshmem_client_cmdline_help();
+        } else if (!strcmp(token, "dump")) {
+            ivshmem_client_dump(client);
+        } else if (!strcmp(token, "int all")) {
+            ivshmem_client_notify_broadcast(client);
+        } else if (sscanf(token, "int %d %d", &peer_id, &vector) == 2) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify(client, peer, vector);
+        } else if (sscanf(token, "int %d all", &peer_id) == 1) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify_all_vects(client, peer);
+        } else {
+            printf("invalid command, type help\n");
+        }
+    }
+
+    printf("cmd> ");
+    fflush(stdout);
+    return 0;
+}
+
+/* listen on stdin (command line), on unix socket (notifications of new
+ * and dead peers), and on eventfd (IRQ request) */
+static int
+ivshmem_client_poll_events(IvshmemClient *client)
+{
+    fd_set fds;
+    int ret, maxfd;
+
+    while (1) {
+
+        FD_ZERO(&fds);
+        FD_SET(0, &fds); /* add stdin in fd_set */
+        maxfd = 1;
+
+        ivshmem_client_get_fds(client, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(0, &fds) &&
+            ivshmem_client_handle_stdin_command(client) < 0 && errno != EINTR) {
+            fprintf(stderr, "ivshmem_client_handle_stdin_command() failed\n");
+            break;
+        }
+
+        if (ivshmem_client_handle_fds(client, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_client_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+/* callback when we receive a notification (just display it) */
+static void
+ivshmem_client_notification_cb(const IvshmemClient *client,
+                               const IvshmemClientPeer *peer,
+                               unsigned vect, void *arg)
+{
+    (void)client;
+    (void)arg;
+    printf("receive notification from peer_id=%" PRId64 " vector=%u\n",
+           peer->id, vect);
+}
+
+int
+main(int argc, char *argv[])
+{
+    struct sigaction sa;
+    IvshmemClient client;
+    IvshmemClientArgs args = {
+        .verbose = IVSHMEM_CLIENT_DEFAULT_VERBOSE,
+        .unix_sock_path = IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH,
+    };
+
+    /* parse arguments, will exit on error */
+    ivshmem_client_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        return 1;
+    }
+
+    ivshmem_client_cmdline_help();
+    printf("cmd> ");
+    fflush(stdout);
+
+    if (ivshmem_client_init(&client, args.unix_sock_path,
+                            ivshmem_client_notification_cb, NULL,
+                            args.verbose) < 0) {
+        fprintf(stderr, "cannot init client\n");
+        return 1;
+    }
+
+    while (1) {
+        if (ivshmem_client_connect(&client) < 0) {
+            fprintf(stderr, "cannot connect to server, retry in 1 second\n");
+            sleep(1);
+            continue;
+        }
+
+        fprintf(stdout, "listen on server socket %d\n", client.sock_fd);
+
+        if (ivshmem_client_poll_events(&client) == 0) {
+            continue;
+        }
+
+        /* disconnected from server, reset all peers */
+        fprintf(stdout, "disconnected from server\n");
+
+        ivshmem_client_close(&client);
+    }
+
+    return 0;
+}
diff --git a/qemu/contrib/ivshmem-server/Makefile.objs b/qemu/contrib/ivshmem-server/Makefile.objs
new file mode 100644
index 000000000..c060dd369
--- /dev/null
+++ b/qemu/contrib/ivshmem-server/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-server-obj-y = ivshmem-server.o main.o
diff --git a/qemu/contrib/ivshmem-server/ivshmem-server.c b/qemu/contrib/ivshmem-server/ivshmem-server.c
new file mode 100644
index 000000000..172db78b3
--- /dev/null
+++ b/qemu/contrib/ivshmem-server/ivshmem-server.c
@@ -0,0 +1,457 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "ivshmem-server.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_SERVER_DEBUG(server, fmt, ...) do { \
+        if ((server)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/** maximum size of a huge page, used by ivshmem_server_ftruncate() */
+#define IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE (1024 * 1024 * 1024)
+
+/** default listen backlog (number of sockets not accepted) */
+#define IVSHMEM_SERVER_LISTEN_BACKLOG 10
+
+/* send message to a client unix socket */
+static int
+ivshmem_server_send_one_msg(int sock_fd, int64_t peer_id, int fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    peer_id = GINT64_TO_LE(peer_id);
+    iov[0].iov_base = &peer_id;
+    iov[0].iov_len = sizeof(peer_id);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+
+    /* if fd is specified, add it in a cmsg */
+    if (fd >= 0) {
+        memset(&msg_control, 0, sizeof(msg_control));
+        msg.msg_control = &msg_control;
+        msg.msg_controllen = sizeof(msg_control);
+        cmsg = CMSG_FIRSTHDR(&msg);
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type = SCM_RIGHTS;
+        cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+        memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd));
+    }
+
+    ret = sendmsg(sock_fd, &msg, 0);
+    if (ret <= 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * server is freed */
+static void
+ivshmem_server_free_peer(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    unsigned vector;
+    IvshmemServerPeer *other_peer;
+
+    IVSHMEM_SERVER_DEBUG(server, "free peer %" PRId64 "\n", peer->id);
+    close(peer->sock_fd);
+    QTAILQ_REMOVE(&server->peer_list, peer, next);
+
+    /* advertise the deletion to other peers */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id, -1);
+    }
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        event_notifier_cleanup(&peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* send the peer id and the shm_fd just after a new client connection */
+static int
+ivshmem_server_send_initial_info(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    int ret;
+
+    /* send our protocol version first */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, IVSHMEM_PROTOCOL_VERSION,
+                                      -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send version: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the peer id to the client */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, peer->id, -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send peer id: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the shm_fd */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, -1, server->shm_fd);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send shm fd: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/* handle message on listening unix socket (new client connection) */
+static int
+ivshmem_server_handle_new_conn(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *other_peer;
+    struct sockaddr_un unaddr;
+    socklen_t unaddr_len;
+    int newfd;
+    unsigned i;
+
+    /* accept the incoming connection */
+    unaddr_len = sizeof(unaddr);
+    newfd = qemu_accept(server->sock_fd,
+                        (struct sockaddr *)&unaddr, &unaddr_len);
+
+    if (newfd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot accept() %s\n", strerror(errno));
+        return -1;
+    }
+
+    qemu_set_nonblock(newfd);
+    IVSHMEM_SERVER_DEBUG(server, "accept()=%d\n", newfd);
+
+    /* allocate new structure for this peer */
+    peer = g_malloc0(sizeof(*peer));
+    peer->sock_fd = newfd;
+
+    /* get an unused peer id */
+    /* XXX: this could use id allocation such as Linux IDA, or simply
+     * a free-list */
+    for (i = 0; i < G_MAXUINT16; i++) {
+        if (ivshmem_server_search_peer(server, server->cur_id) == NULL) {
+            break;
+        }
+        server->cur_id++;
+    }
+    if (i == G_MAXUINT16) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot allocate new client id\n");
+        close(newfd);
+        g_free(peer);
+        return -1;
+    }
+    peer->id = server->cur_id++;
+
+    /* create eventfd, one per vector */
+    peer->vectors_count = server->n_vectors;
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (event_notifier_init(&peer->vectors[i], FALSE) < 0) {
+            IVSHMEM_SERVER_DEBUG(server, "cannot create eventfd\n");
+            goto fail;
+        }
+    }
+
+    /* send peer id and shm fd */
+    if (ivshmem_server_send_initial_info(server, peer) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send initial info\n");
+        goto fail;
+    }
+
+    /* advertise the new peer to others */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id,
+                                        peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the other peers to the new one */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(peer->sock_fd, other_peer->id,
+                                        other_peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the new peer to itself */
+    for (i = 0; i < peer->vectors_count; i++) {
+        ivshmem_server_send_one_msg(peer->sock_fd, peer->id,
+                                    event_notifier_get_fd(&peer->vectors[i]));
+    }
+
+    QTAILQ_INSERT_TAIL(&server->peer_list, peer, next);
+    IVSHMEM_SERVER_DEBUG(server, "new peer id = %" PRId64 "\n",
+                         peer->id);
+    return 0;
+
+fail:
+    while (i--) {
+        event_notifier_cleanup(&peer->vectors[i]);
+    }
+    close(newfd);
+    g_free(peer);
+    return -1;
+}
+
+/* Try to ftruncate a file to next power of 2 of shmsize.
+ * If it fails; all power of 2 above shmsize are tested until
+ * we reach the maximum huge page size. This is useful
+ * if the shm file is in a hugetlbfs that cannot be truncated to the
+ * shm_size value. */
+static int
+ivshmem_server_ftruncate(int fd, unsigned shmsize)
+{
+    int ret;
+    struct stat mapstat;
+
+    /* align shmsize to next power of 2 */
+    shmsize = pow2ceil(shmsize);
+
+    if (fstat(fd, &mapstat) != -1 && mapstat.st_size == shmsize) {
+        return 0;
+    }
+
+    while (shmsize <= IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE) {
+        ret = ftruncate(fd, shmsize);
+        if (ret == 0) {
+            return ret;
+        }
+        shmsize *= 2;
+    }
+
+    return -1;
+}
+
+/* Init a new ivshmem server */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, bool use_shm_open,
+                    size_t shm_size, unsigned n_vectors,
+                    bool verbose)
+{
+    int ret;
+
+    memset(server, 0, sizeof(*server));
+    server->verbose = verbose;
+
+    ret = snprintf(server->unix_sock_path, sizeof(server->unix_sock_path),
+                   "%s", unix_sock_path);
+    if (ret < 0 || ret >= sizeof(server->unix_sock_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        return -1;
+    }
+    ret = snprintf(server->shm_path, sizeof(server->shm_path),
+                   "%s", shm_path);
+    if (ret < 0 || ret >= sizeof(server->shm_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy shm path\n");
+        return -1;
+    }
+
+    server->use_shm_open = use_shm_open;
+    server->shm_size = shm_size;
+    server->n_vectors = n_vectors;
+
+    QTAILQ_INIT(&server->peer_list);
+
+    return 0;
+}
+
+/* open shm, create and bind to the unix socket */
+int
+ivshmem_server_start(IvshmemServer *server)
+{
+    struct sockaddr_un sun;
+    int shm_fd, sock_fd, ret;
+
+    /* open shm file */
+    if (server->use_shm_open) {
+        IVSHMEM_SERVER_DEBUG(server, "Using POSIX shared memory: %s\n",
+                             server->shm_path);
+        shm_fd = shm_open(server->shm_path, O_CREAT | O_RDWR, S_IRWXU);
+    } else {
+        gchar *filename = g_strdup_printf("%s/ivshmem.XXXXXX", server->shm_path);
+        IVSHMEM_SERVER_DEBUG(server, "Using file-backed shared memory: %s\n",
+                             server->shm_path);
+        shm_fd = mkstemp(filename);
+        unlink(filename);
+        g_free(filename);
+    }
+
+    if (shm_fd < 0) {
+        fprintf(stderr, "cannot open shm file %s: %s\n", server->shm_path,
+                strerror(errno));
+        return -1;
+    }
+    if (ivshmem_server_ftruncate(shm_fd, server->shm_size) < 0) {
+        fprintf(stderr, "ftruncate(%s) failed: %s\n", server->shm_path,
+                strerror(errno));
+        goto err_close_shm;
+    }
+
+    IVSHMEM_SERVER_DEBUG(server, "create & bind socket %s\n",
+                         server->unix_sock_path);
+
+    /* create the unix listening socket */
+    sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (sock_fd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot create socket: %s\n",
+                             strerror(errno));
+        goto err_close_shm;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   server->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        goto err_close_sock;
+    }
+    if (bind(sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close_sock;
+    }
+
+    if (listen(sock_fd, IVSHMEM_SERVER_LISTEN_BACKLOG) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "listen() failed: %s\n", strerror(errno));
+        goto err_close_sock;
+    }
+
+    server->sock_fd = sock_fd;
+    server->shm_fd = shm_fd;
+
+    return 0;
+
+err_close_sock:
+    close(sock_fd);
+err_close_shm:
+    close(shm_fd);
+    return -1;
+}
+
+/* close connections to clients, the unix socket and the shm fd */
+void
+ivshmem_server_close(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *npeer;
+
+    IVSHMEM_SERVER_DEBUG(server, "close server\n");
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, npeer) {
+        ivshmem_server_free_peer(server, peer);
+    }
+
+    unlink(server->unix_sock_path);
+    close(server->sock_fd);
+    close(server->shm_fd);
+    server->sock_fd = -1;
+    server->shm_fd = -1;
+}
+
+/* get the fd_set according to the unix socket and the peer list */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd)
+{
+    IvshmemServerPeer *peer;
+
+    if (server->sock_fd == -1) {
+        return;
+    }
+
+    FD_SET(server->sock_fd, fds);
+    if (server->sock_fd >= *maxfd) {
+        *maxfd = server->sock_fd + 1;
+    }
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        FD_SET(peer->sock_fd, fds);
+        if (peer->sock_fd >= *maxfd) {
+            *maxfd = peer->sock_fd + 1;
+        }
+    }
+}
+
+/* process incoming messages on the sockets in fd_set */
+int
+ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd)
+{
+    IvshmemServerPeer *peer, *peer_next;
+
+    if (server->sock_fd < maxfd && FD_ISSET(server->sock_fd, fds) &&
+        ivshmem_server_handle_new_conn(server) < 0 && errno != EINTR) {
+        IVSHMEM_SERVER_DEBUG(server, "ivshmem_server_handle_new_conn() "
+                             "failed\n");
+        return -1;
+    }
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, peer_next) {
+        /* any message from a peer socket result in a close() */
+        IVSHMEM_SERVER_DEBUG(server, "peer->sock_fd=%d\n", peer->sock_fd);
+        if (peer->sock_fd < maxfd && FD_ISSET(peer->sock_fd, fds)) {
+            ivshmem_server_free_peer(server, peer);
+        }
+    }
+
+    return 0;
+}
+
+/* lookup peer from its id */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id)
+{
+    IvshmemServerPeer *peer;
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_server_dump(const IvshmemServer *server)
+{
+    const IvshmemServerPeer *peer;
+    unsigned vector;
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   event_notifier_get_fd(&peer->vectors[vector]));
+        }
+    }
+}
diff --git a/qemu/contrib/ivshmem-server/ivshmem-server.h b/qemu/contrib/ivshmem-server/ivshmem-server.h
new file mode 100644
index 000000000..385163961
--- /dev/null
+++ b/qemu/contrib/ivshmem-server/ivshmem-server.h
@@ -0,0 +1,166 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _IVSHMEM_SERVER_H_
+#define _IVSHMEM_SERVER_H_
+
+/**
+ * The ivshmem server is a daemon that creates a unix socket in listen
+ * mode. The ivshmem clients (qemu or ivshmem-client) connect to this
+ * unix socket. For each client, the server will create some eventfd
+ * (see EVENTFD(2)), one per vector. These fd are transmitted to all
+ * clients using the SCM_RIGHTS cmsg message. Therefore, each client is
+ * able to send a notification to another client without beeing
+ * "profixied" by the server.
+ *
+ * We use this mechanism to send interruptions between guests.
+ * qemu is able to transform an event on a eventfd into a PCI MSI-x
+ * interruption in the guest.
+ *
+ * The ivshmem server is also able to share the file descriptor
+ * associated to the ivshmem shared memory.
+ */
+
+#include <sys/select.h>
+
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the server
+ */
+#define IVSHMEM_SERVER_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, a new
+ * IvshmemServerPeer structure is created. This peer and all its
+ * vectors are advertised to all connected clients through the connected
+ * unix sockets.
+ */
+typedef struct IvshmemServerPeer {
+    QTAILQ_ENTRY(IvshmemServerPeer) next;    /**< next in list*/
+    int sock_fd;                             /**< connected unix sock */
+    int64_t id;                              /**< the id of the peer */
+    EventNotifier vectors[IVSHMEM_SERVER_MAX_VECTORS]; /**< one per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemServerPeer;
+QTAILQ_HEAD(IvshmemServerPeerList, IvshmemServerPeer);
+
+typedef struct IvshmemServerPeerList IvshmemServerPeerList;
+
+/**
+ * Structure describing an ivshmem server
+ *
+ * This structure stores all information related to our server: the name
+ * of the server unix socket and the list of connected peers.
+ */
+typedef struct IvshmemServer {
+    char unix_sock_path[PATH_MAX];   /**< path to unix socket */
+    int sock_fd;                     /**< unix sock file descriptor */
+    char shm_path[PATH_MAX];         /**< path to shm */
+    bool use_shm_open;
+    size_t shm_size;                 /**< size of shm */
+    int shm_fd;                      /**< shm file descriptor */
+    unsigned n_vectors;              /**< number of vectors */
+    uint16_t cur_id;                 /**< id to be given to next client */
+    bool verbose;                    /**< true in verbose mode */
+    IvshmemServerPeerList peer_list; /**< list of peers */
+} IvshmemServer;
+
+/**
+ * Initialize an ivshmem server
+ *
+ * @server:         A pointer to an uninitialized IvshmemServer structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @shm_path:       Path to the shared memory. The path corresponds to a POSIX
+ *                  shm name or a hugetlbfs mount point.
+ * @shm_size:       Size of shared memory
+ * @n_vectors:      Number of interrupt vectors per client
+ * @verbose:        True to enable verbose mode
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, bool use_shm_open,
+                    size_t shm_size, unsigned n_vectors,
+                    bool verbose);
+
+/**
+ * Open the shm, then create and bind to the unix socket
+ *
+ * @server: The pointer to the initialized IvshmemServer structure
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_start(IvshmemServer *server);
+
+/**
+ * Close the server
+ *
+ * Close connections to all clients, close the unix socket and the
+ * shared memory file descriptor. The structure remains initialized, so
+ * it is possible to call ivshmem_server_start() again after a call to
+ * ivshmem_server_close().
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_close(IvshmemServer *server);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors that must
+ * be polled (unix server socket and peers unix socket). The function
+ * will not initialize the fd_set, it is up to the caller to do it.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set (for instance filled by a call to select()), handle
+ * incoming messages from peers.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set containing the file descriptors to be checked. Note that
+ *          file descriptors that are not related to our server are ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd);
+
+/**
+ * Search a peer from its identifier
+ *
+ * @server:  The ivshmem server
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem server and its peers on stdout
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_dump(const IvshmemServer *server);
+
+#endif /* _IVSHMEM_SERVER_H_ */
diff --git a/qemu/contrib/ivshmem-server/main.c b/qemu/contrib/ivshmem-server/main.c
new file mode 100644
index 000000000..45776d8af
--- /dev/null
+++ b/qemu/contrib/ivshmem-server/main.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+
+#include "ivshmem-server.h"
+
+#define IVSHMEM_SERVER_DEFAULT_VERBOSE        0
+#define IVSHMEM_SERVER_DEFAULT_FOREGROUND     0
+#define IVSHMEM_SERVER_DEFAULT_PID_FILE       "/var/run/ivshmem-server.pid"
+#define IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+#define IVSHMEM_SERVER_DEFAULT_SHM_PATH       "ivshmem"
+#define IVSHMEM_SERVER_DEFAULT_SHM_SIZE       (4*1024*1024)
+#define IVSHMEM_SERVER_DEFAULT_N_VECTORS      1
+
+/* used to quit on signal SIGTERM */
+static int ivshmem_server_quit;
+
+/* arguments given by the user */
+typedef struct IvshmemServerArgs {
+    bool verbose;
+    bool foreground;
+    const char *pid_file;
+    const char *unix_socket_path;
+    const char *shm_path;
+    bool use_shm_open;
+    uint64_t shm_size;
+    unsigned n_vectors;
+} IvshmemServerArgs;
+
+static void
+ivshmem_server_usage(const char *progname)
+{
+    printf("Usage: %s [OPTION]...\n"
+           "  -h: show this help\n"
+           "  -v: verbose mode\n"
+           "  -F: foreground mode (default is to daemonize)\n"
+           "  -p <pid-file>: path to the PID file (used in daemon mode only)\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_PID_FILE "\n"
+           "  -S <unix-socket-path>: path to the unix socket to listen to\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH "\n"
+           "  -M <shm-name>: POSIX shared memory object to use\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_SHM_PATH "\n"
+           "  -m <dir-name>: where to create shared memory\n"
+           "  -l <size>: size of shared memory in bytes\n"
+           "     suffixes K, M and G can be used, e.g. 1K means 1024\n"
+           "     default %u\n"
+           "  -n <nvectors>: number of vectors\n"
+           "     default %u\n",
+           progname, IVSHMEM_SERVER_DEFAULT_SHM_SIZE,
+           IVSHMEM_SERVER_DEFAULT_N_VECTORS);
+}
+
+static void
+ivshmem_server_help(const char *progname)
+{
+    fprintf(stderr, "Try '%s -h' for more information.\n", progname);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_server_parse_args(IvshmemServerArgs *args, int argc, char *argv[])
+{
+    int c;
+    unsigned long long v;
+    Error *err = NULL;
+
+    while ((c = getopt(argc, argv, "hvFp:S:m:M:l:n:")) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_server_usage(argv[0]);
+            exit(0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'F': /* foreground */
+            args->foreground = 1;
+            break;
+
+        case 'p': /* pid file */
+            args->pid_file = optarg;
+            break;
+
+        case 'S': /* unix socket path */
+            args->unix_socket_path = optarg;
+            break;
+
+        case 'M': /* shm name */
+        case 'm': /* dir name */
+            args->shm_path = optarg;
+            args->use_shm_open = c == 'M';
+            break;
+
+        case 'l': /* shm size */
+            parse_option_size("shm_size", optarg, &args->shm_size, &err);
+            if (err) {
+                error_report_err(err);
+                ivshmem_server_help(argv[0]);
+                exit(1);
+            }
+            break;
+
+        case 'n': /* number of vectors */
+            if (parse_uint_full(optarg, &v, 0) < 0) {
+                fprintf(stderr, "cannot parse n_vectors\n");
+                ivshmem_server_help(argv[0]);
+                exit(1);
+            }
+            args->n_vectors = v;
+            break;
+
+        default:
+            ivshmem_server_usage(argv[0]);
+            exit(1);
+            break;
+        }
+    }
+
+    if (args->n_vectors > IVSHMEM_SERVER_MAX_VECTORS) {
+        fprintf(stderr, "too many requested vectors (max is %d)\n",
+                IVSHMEM_SERVER_MAX_VECTORS);
+        ivshmem_server_help(argv[0]);
+        exit(1);
+    }
+
+    if (args->verbose == 1 && args->foreground == 0) {
+        fprintf(stderr, "cannot use verbose in daemon mode\n");
+        ivshmem_server_help(argv[0]);
+        exit(1);
+    }
+}
+
+/* wait for events on listening server unix socket and connected client
+ * sockets */
+static int
+ivshmem_server_poll_events(IvshmemServer *server)
+{
+    fd_set fds;
+    int ret = 0, maxfd;
+
+    while (!ivshmem_server_quit) {
+
+        FD_ZERO(&fds);
+        maxfd = 0;
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+static void
+ivshmem_server_quit_cb(int signum)
+{
+    ivshmem_server_quit = 1;
+}
+
+int
+main(int argc, char *argv[])
+{
+    IvshmemServer server;
+    struct sigaction sa, sa_quit;
+    IvshmemServerArgs args = {
+        .verbose = IVSHMEM_SERVER_DEFAULT_VERBOSE,
+        .foreground = IVSHMEM_SERVER_DEFAULT_FOREGROUND,
+        .pid_file = IVSHMEM_SERVER_DEFAULT_PID_FILE,
+        .unix_socket_path = IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH,
+        .shm_path = IVSHMEM_SERVER_DEFAULT_SHM_PATH,
+        .use_shm_open = true,
+        .shm_size = IVSHMEM_SERVER_DEFAULT_SHM_SIZE,
+        .n_vectors = IVSHMEM_SERVER_DEFAULT_N_VECTORS,
+    };
+    int ret = 1;
+
+    /*
+     * Do not remove this notice without adding proper error handling!
+     * Start with handling ivshmem_server_send_one_msg() failure.
+     */
+    printf("*** Example code, do not use in production ***\n");
+
+    /* parse arguments, will exit on error */
+    ivshmem_server_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        goto err;
+    }
+
+    sa_quit.sa_handler = ivshmem_server_quit_cb;
+    sa_quit.sa_flags = 0;
+    if (sigemptyset(&sa_quit.sa_mask) == -1 ||
+        sigaction(SIGTERM, &sa_quit, 0) == -1) {
+        perror("failed to add SIGTERM handler; sigaction");
+        goto err;
+    }
+
+    /* init the ivshms structure */
+    if (ivshmem_server_init(&server, args.unix_socket_path,
+                            args.shm_path, args.use_shm_open,
+                            args.shm_size, args.n_vectors, args.verbose) < 0) {
+        fprintf(stderr, "cannot init server\n");
+        goto err;
+    }
+
+    /* start the ivshmem server (open shm & unix socket) */
+    if (ivshmem_server_start(&server) < 0) {
+        fprintf(stderr, "cannot bind\n");
+        goto err;
+    }
+
+    /* daemonize if asked to */
+    if (!args.foreground) {
+        FILE *fp;
+
+        if (qemu_daemon(1, 1) < 0) {
+            fprintf(stderr, "cannot daemonize: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        /* write pid file */
+        fp = fopen(args.pid_file, "w");
+        if (fp == NULL) {
+            fprintf(stderr, "cannot write pid file: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        fprintf(fp, "%d\n", (int) getpid());
+        fclose(fp);
+    }
+
+    ivshmem_server_poll_events(&server);
+    fprintf(stdout, "server disconnected\n");
+    ret = 0;
+
+err_close:
+    ivshmem_server_close(&server);
+err:
+    return ret;
+}
diff --git a/qemu/cpu-exec-common.c b/qemu/cpu-exec-common.c
new file mode 100644
index 000000000..1b1731cd8
--- /dev/null
+++ b/qemu/cpu-exec-common.c
@@ -0,0 +1,82 @@
+/*
+ *  emulator main execution loop
+ *
+ *  Copyright (c) 2003-2005 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "sysemu/cpus.h"
+#include "exec/memory-internal.h"
+
+bool exit_request;
+CPUState *tcg_current_cpu;
+
+/* exit the current TB from a signal handler. The host registers are
+   restored in a state compatible with the CPU emulator
+ */
+#if defined(CONFIG_SOFTMMU)
+void cpu_resume_from_signal(CPUState *cpu, void *puc)
+{
+    /* XXX: restore cpu registers saved in host registers */
+
+    cpu->exception_index = -1;
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+void cpu_reloading_memory_map(void)
+{
+    if (qemu_in_vcpu_thread()) {
+        /* The guest can in theory prolong the RCU critical section as long
+         * as it feels like. The major problem with this is that because it
+         * can do multiple reconfigurations of the memory map within the
+         * critical section, we could potentially accumulate an unbounded
+         * collection of memory data structures awaiting reclamation.
+         *
+         * Because the only thing we're currently protecting with RCU is the
+         * memory data structures, it's sufficient to break the critical section
+         * in this callback, which we know will get called every time the
+         * memory map is rearranged.
+         *
+         * (If we add anything else in the system that uses RCU to protect
+         * its data structures, we will need to implement some other mechanism
+         * to force TCG CPUs to exit the critical section, at which point this
+         * part of this callback might become unnecessary.)
+         *
+         * This pair matches cpu_exec's rcu_read_lock()/rcu_read_unlock(), which
+         * only protects cpu->as->dispatch. Since we know our caller is about
+         * to reload it, it's safe to split the critical section.
+         */
+        rcu_read_unlock();
+        rcu_read_lock();
+    }
+}
+#endif
+
+void cpu_loop_exit(CPUState *cpu)
+{
+    cpu->current_tb = NULL;
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc)
+{
+    if (pc) {
+        cpu_restore_state(cpu, pc);
+    }
+    cpu->current_tb = NULL;
+    siglongjmp(cpu->jmp_env, 1);
+}
diff --git a/qemu/cpu-exec.c b/qemu/cpu-exec.c
index 75694f3bb..bbfcbfb54 100644
--- a/qemu/cpu-exec.c
+++ b/qemu/cpu-exec.c
@@ -16,7 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "disas/disas.h"
@@ -25,9 +25,13 @@
 #include "sysemu/qtest.h"
 #include "qemu/timer.h"
 #include "exec/address-spaces.h"
-#include "exec/memory-internal.h"
 #include "qemu/rcu.h"
 #include "exec/tb-hash.h"
+#include "exec/log.h"
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+#include "hw/i386/apic.h"
+#endif
+#include "sysemu/replay.h"
 
 /* -icount align implementation. */
 
@@ -128,57 +132,16 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
 }
 #endif /* CONFIG USER ONLY */
 
-void cpu_loop_exit(CPUState *cpu)
-{
-    cpu->current_tb = NULL;
-    siglongjmp(cpu->jmp_env, 1);
-}
-
-/* exit the current TB from a signal handler. The host registers are
-   restored in a state compatible with the CPU emulator
- */
-#if defined(CONFIG_SOFTMMU)
-void cpu_resume_from_signal(CPUState *cpu, void *puc)
-{
-    /* XXX: restore cpu registers saved in host registers */
-
-    cpu->exception_index = -1;
-    siglongjmp(cpu->jmp_env, 1);
-}
-
-void cpu_reload_memory_map(CPUState *cpu)
-{
-    AddressSpaceDispatch *d;
-
-    if (qemu_in_vcpu_thread()) {
-        /* Do not let the guest prolong the critical section as much as it
-         * as it desires.
-         *
-         * Currently, this is prevented by the I/O thread's periodinc kicking
-         * of the VCPU thread (iothread_requesting_mutex, qemu_cpu_kick_thread)
-         * but this will go away once TCG's execution moves out of the global
-         * mutex.
-         *
-         * This pair matches cpu_exec's rcu_read_lock()/rcu_read_unlock(), which
-         * only protects cpu->as->dispatch.  Since we reload it below, we can
-         * split the critical section.
-         */
-        rcu_read_unlock();
-        rcu_read_lock();
-    }
-
-    /* The CPU and TLB are protected by the iothread lock.  */
-    d = atomic_rcu_read(&cpu->as->dispatch);
-    cpu->memory_dispatch = d;
-    tlb_flush(cpu, 1);
-}
-#endif
-
 /* Execute a TB, and fix up the CPU state afterwards if necessary */
-static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
+static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
 {
     CPUArchState *env = cpu->env_ptr;
     uintptr_t next_tb;
+    uint8_t *tb_ptr = itb->tc_ptr;
+
+    qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
+                           "Trace %p [" TARGET_FMT_lx "] %s\n",
+                           itb->tc_ptr, itb->pc, lookup_symbol(itb->pc));
 
 #if defined(DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
@@ -196,7 +159,7 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
     }
 #endif /* DEBUG_DISAS */
 
-    cpu->can_do_io = 0;
+    cpu->can_do_io = !use_icount;
     next_tb = tcg_qemu_tb_exec(env, tb_ptr);
     cpu->can_do_io = 1;
     trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
@@ -209,6 +172,10 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
          */
         CPUClass *cc = CPU_GET_CLASS(cpu);
         TranslationBlock *tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
+        qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
+                               "Stopped execution of TB chain before %p ["
+                               TARGET_FMT_lx "] %s\n",
+                               itb->tc_ptr, itb->pc, lookup_symbol(itb->pc));
         if (cc->synchronize_from_tb) {
             cc->synchronize_from_tb(cpu, tb);
         } else {
@@ -228,35 +195,32 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
 /* Execute the code without caching the generated code. An interpreter
    could be used if available. */
 static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
-                             TranslationBlock *orig_tb)
+                             TranslationBlock *orig_tb, bool ignore_icount)
 {
     TranslationBlock *tb;
-    target_ulong pc = orig_tb->pc;
-    target_ulong cs_base = orig_tb->cs_base;
-    uint64_t flags = orig_tb->flags;
 
     /* Should never happen.
        We only end up here when an existing TB is too long.  */
     if (max_cycles > CF_COUNT_MASK)
         max_cycles = CF_COUNT_MASK;
 
-    /* tb_gen_code can flush our orig_tb, invalidate it now */
-    tb_phys_invalidate(orig_tb, -1);
-    tb = tb_gen_code(cpu, pc, cs_base, flags,
-                     max_cycles | CF_NOCACHE);
+    tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags,
+                     max_cycles | CF_NOCACHE
+                         | (ignore_icount ? CF_IGNORE_ICOUNT : 0));
+    tb->orig_tb = tcg_ctx.tb_ctx.tb_invalidated_flag ? NULL : orig_tb;
     cpu->current_tb = tb;
     /* execute the generated code */
     trace_exec_tb_nocache(tb, tb->pc);
-    cpu_tb_exec(cpu, tb->tc_ptr);
+    cpu_tb_exec(cpu, tb);
     cpu->current_tb = NULL;
     tb_phys_invalidate(tb, -1);
     tb_free(tb);
 }
 
-static TranslationBlock *tb_find_slow(CPUState *cpu,
-                                      target_ulong pc,
-                                      target_ulong cs_base,
-                                      uint64_t flags)
+static TranslationBlock *tb_find_physical(CPUState *cpu,
+                                          target_ulong pc,
+                                          target_ulong cs_base,
+                                          uint64_t flags)
 {
     CPUArchState *env = (CPUArchState *)cpu->env_ptr;
     TranslationBlock *tb, **ptb1;
@@ -273,8 +237,9 @@ static TranslationBlock *tb_find_slow(CPUState *cpu,
     ptb1 = &tcg_ctx.tb_ctx.tb_phys_hash[h];
     for(;;) {
         tb = *ptb1;
-        if (!tb)
-            goto not_found;
+        if (!tb) {
+            return NULL;
+        }
         if (tb->pc == pc &&
             tb->page_addr[0] == phys_page1 &&
             tb->cs_base == cs_base &&
@@ -286,25 +251,59 @@ static TranslationBlock *tb_find_slow(CPUState *cpu,
                 virt_page2 = (pc & TARGET_PAGE_MASK) +
                     TARGET_PAGE_SIZE;
                 phys_page2 = get_page_addr_code(env, virt_page2);
-                if (tb->page_addr[1] == phys_page2)
-                    goto found;
+                if (tb->page_addr[1] == phys_page2) {
+                    break;
+                }
             } else {
-                goto found;
+                break;
             }
         }
         ptb1 = &tb->phys_hash_next;
     }
- not_found:
-   /* if no translated code available, then translate it now */
-    tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
 
- found:
-    /* Move the last found TB to the head of the list */
-    if (likely(*ptb1)) {
-        *ptb1 = tb->phys_hash_next;
-        tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
-        tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
+    /* Move the TB to the head of the list */
+    *ptb1 = tb->phys_hash_next;
+    tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
+    tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
+    return tb;
+}
+
+static TranslationBlock *tb_find_slow(CPUState *cpu,
+                                      target_ulong pc,
+                                      target_ulong cs_base,
+                                      uint64_t flags)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        goto found;
+    }
+
+#ifdef CONFIG_USER_ONLY
+    /* mmap_lock is needed by tb_gen_code, and mmap_lock must be
+     * taken outside tb_lock.  Since we're momentarily dropping
+     * tb_lock, there's a chance that our desired tb has been
+     * translated.
+     */
+    tb_unlock();
+    mmap_lock();
+    tb_lock();
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        mmap_unlock();
+        goto found;
     }
+#endif
+
+    /* if no translated code available, then translate it now */
+    tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
+
+#ifdef CONFIG_USER_ONLY
+    mmap_unlock();
+#endif
+
+found:
     /* we add the TB in the virtual pc hash table */
     cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)] = tb;
     return tb;
@@ -345,8 +344,6 @@ static void cpu_handle_debug_exception(CPUState *cpu)
 
 /* main execution loop */
 
-volatile sig_atomic_t exit_request;
-
 int cpu_exec(CPUState *cpu)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
@@ -356,34 +353,32 @@ int cpu_exec(CPUState *cpu)
 #endif
     int ret, interrupt_request;
     TranslationBlock *tb;
-    uint8_t *tc_ptr;
     uintptr_t next_tb;
     SyncClocks sc;
 
-    /* This must be volatile so it is not trashed by longjmp() */
-    volatile bool have_tb_lock = false;
+    /* replay_interrupt may need current_cpu */
+    current_cpu = cpu;
 
     if (cpu->halted) {
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+        if ((cpu->interrupt_request & CPU_INTERRUPT_POLL)
+            && replay_interrupt()) {
+            apic_poll_irq(x86_cpu->apic_state);
+            cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
+        }
+#endif
         if (!cpu_has_work(cpu)) {
+            current_cpu = NULL;
             return EXCP_HALTED;
         }
 
         cpu->halted = 0;
     }
 
-    current_cpu = cpu;
-
-    /* As long as current_cpu is null, up to the assignment just above,
-     * requests by other threads to exit the execution loop are expected to
-     * be issued using the exit_request global. We must make sure that our
-     * evaluation of the global value is performed past the current_cpu
-     * value transition point, which requires a memory barrier as well as
-     * an instruction scheduling constraint on modern architectures.  */
-    smp_mb();
-
+    atomic_mb_set(&tcg_current_cpu, cpu);
     rcu_read_lock();
 
-    if (unlikely(exit_request)) {
+    if (unlikely(atomic_mb_read(&exit_request))) {
         cpu->exit_request = 1;
     }
 
@@ -421,10 +416,22 @@ int cpu_exec(CPUState *cpu)
                     cpu->exception_index = -1;
                     break;
 #else
-                    cc->do_interrupt(cpu);
-                    cpu->exception_index = -1;
+                    if (replay_exception()) {
+                        cc->do_interrupt(cpu);
+                        cpu->exception_index = -1;
+                    } else if (!replay_has_interrupt()) {
+                        /* give a chance to iothread in replay mode */
+                        ret = EXCP_INTERRUPT;
+                        break;
+                    }
 #endif
                 }
+            } else if (replay_has_exception()
+                       && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
+                /* try to cause an exception pending in the log */
+                cpu_exec_nocache(cpu, 1, tb_find_fast(cpu), true);
+                ret = -1;
+                break;
             }
 
             next_tb = 0; /* force lookup of first TB */
@@ -440,30 +447,40 @@ int cpu_exec(CPUState *cpu)
                         cpu->exception_index = EXCP_DEBUG;
                         cpu_loop_exit(cpu);
                     }
-                    if (interrupt_request & CPU_INTERRUPT_HALT) {
+                    if (replay_mode == REPLAY_MODE_PLAY
+                        && !replay_has_interrupt()) {
+                        /* Do nothing */
+                    } else if (interrupt_request & CPU_INTERRUPT_HALT) {
+                        replay_interrupt();
                         cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
                         cpu->halted = 1;
                         cpu->exception_index = EXCP_HLT;
                         cpu_loop_exit(cpu);
                     }
 #if defined(TARGET_I386)
-                    if (interrupt_request & CPU_INTERRUPT_INIT) {
+                    else if (interrupt_request & CPU_INTERRUPT_INIT) {
+                        replay_interrupt();
                         cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
                         do_cpu_init(x86_cpu);
                         cpu->exception_index = EXCP_HALTED;
                         cpu_loop_exit(cpu);
                     }
 #else
-                    if (interrupt_request & CPU_INTERRUPT_RESET) {
+                    else if (interrupt_request & CPU_INTERRUPT_RESET) {
+                        replay_interrupt();
                         cpu_reset(cpu);
+                        cpu_loop_exit(cpu);
                     }
 #endif
                     /* The target hook has 3 exit conditions:
                        False when the interrupt isn't processed,
                        True when it is, and we should restart on a new TB,
                        and via longjmp via cpu_loop_exit.  */
-                    if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
-                        next_tb = 0;
+                    else {
+                        replay_interrupt();
+                        if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
+                            next_tb = 0;
+                        }
                     }
                     /* Don't use the cached interrupt_request value,
                        do_interrupt may have updated the EXITTB flag. */
@@ -474,13 +491,13 @@ int cpu_exec(CPUState *cpu)
                         next_tb = 0;
                     }
                 }
-                if (unlikely(cpu->exit_request)) {
+                if (unlikely(cpu->exit_request
+                             || replay_has_interrupt())) {
                     cpu->exit_request = 0;
                     cpu->exception_index = EXCP_INTERRUPT;
                     cpu_loop_exit(cpu);
                 }
-                spin_lock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = true;
+                tb_lock();
                 tb = tb_find_fast(cpu);
                 /* Note: we do it here to avoid a gcc bug on Mac OS X when
                    doing it in tb_find_slow */
@@ -491,31 +508,21 @@ int cpu_exec(CPUState *cpu)
                     next_tb = 0;
                     tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
                 }
-                if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
-                    qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n",
-                             tb->tc_ptr, tb->pc, lookup_symbol(tb->pc));
-                }
                 /* see if we can patch the calling TB. When the TB
                    spans two pages, we cannot safely do a direct
                    jump. */
-                if (next_tb != 0 && tb->page_addr[1] == -1) {
+                if (next_tb != 0 && tb->page_addr[1] == -1
+                    && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
                     tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK),
                                 next_tb & TB_EXIT_MASK, tb);
                 }
-                have_tb_lock = false;
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-
-                /* cpu_interrupt might be called while translating the
-                   TB, but before it is linked into a potentially
-                   infinite loop and becomes env->current_tb. Avoid
-                   starting execution if there is a pending interrupt. */
-                cpu->current_tb = tb;
-                barrier();
+                tb_unlock();
                 if (likely(!cpu->exit_request)) {
                     trace_exec_tb(tb, tb->pc);
-                    tc_ptr = tb->tc_ptr;
                     /* execute the generated code */
-                    next_tb = cpu_tb_exec(cpu, tc_ptr);
+                    cpu->current_tb = tb;
+                    next_tb = cpu_tb_exec(cpu, tb);
+                    cpu->current_tb = NULL;
                     switch (next_tb & TB_EXIT_MASK) {
                     case TB_EXIT_REQUESTED:
                         /* Something asked us to stop executing
@@ -523,8 +530,12 @@ int cpu_exec(CPUState *cpu)
                          * loop. Whatever requested the exit will also
                          * have set something else (eg exit_request or
                          * interrupt_request) which we will handle
-                         * next time around the loop.
+                         * next time around the loop.  But we need to
+                         * ensure the tcg_exit_req read in generated code
+                         * comes before the next read of cpu->exit_request
+                         * or cpu->interrupt_request.
                          */
+                        smp_rmb();
                         next_tb = 0;
                         break;
                     case TB_EXIT_ICOUNT_EXPIRED:
@@ -541,7 +552,7 @@ int cpu_exec(CPUState *cpu)
                             if (insns_left > 0) {
                                 /* Execute remaining instructions.  */
                                 tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
-                                cpu_exec_nocache(cpu, insns_left, tb);
+                                cpu_exec_nocache(cpu, insns_left, tb, false);
                                 align_clocks(&sc, cpu);
                             }
                             cpu->exception_index = EXCP_INTERRUPT;
@@ -554,7 +565,6 @@ int cpu_exec(CPUState *cpu)
                         break;
                     }
                 }
-                cpu->current_tb = NULL;
                 /* Try to align the host and virtual clocks
                    if the guest is in advance */
                 align_clocks(&sc, cpu);
@@ -562,19 +572,28 @@ int cpu_exec(CPUState *cpu)
                    only be set by a memory fault) */
             } /* for(;;) */
         } else {
-            /* Reload env after longjmp - the compiler may have smashed all
-             * local variables as longjmp is marked 'noreturn'. */
+#if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6)
+            /* Some compilers wrongly smash all local variables after
+             * siglongjmp. There were bug reports for gcc 4.5.0 and clang.
+             * Reload essential local variables here for those compilers.
+             * Newer versions of gcc would complain about this code (-Wclobbered). */
             cpu = current_cpu;
             cc = CPU_GET_CLASS(cpu);
-            cpu->can_do_io = 1;
 #ifdef TARGET_I386
             x86_cpu = X86_CPU(cpu);
             env = &x86_cpu->env;
 #endif
-            if (have_tb_lock) {
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = false;
-            }
+#else /* buggy compiler */
+            /* Assert that the compiler does not smash local variables. */
+            g_assert(cpu == current_cpu);
+            g_assert(cc == CPU_GET_CLASS(cpu));
+#ifdef TARGET_I386
+            g_assert(x86_cpu == X86_CPU(cpu));
+            g_assert(env == &x86_cpu->env);
+#endif
+#endif /* buggy compiler */
+            cpu->can_do_io = 1;
+            tb_lock_reset();
         }
     } /* for(;;) */
 
@@ -583,5 +602,8 @@ int cpu_exec(CPUState *cpu)
 
     /* fail safe : never use current_cpu outside cpu_exec() */
     current_cpu = NULL;
+
+    /* Does not need atomic_mb_set because a spurious wakeup is okay.  */
+    atomic_set(&tcg_current_cpu, NULL);
     return ret;
 }
diff --git a/qemu/cpus.c b/qemu/cpus.c
index a822ce3d8..cbeb1f613 100644
--- a/qemu/cpus.c
+++ b/qemu/cpus.c
@@ -23,12 +23,13 @@
  */
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "monitor/monitor.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "exec/gdbstub.h"
 #include "sysemu/dma.h"
 #include "sysemu/kvm.h"
@@ -42,6 +43,7 @@
 #include "qemu/seqlock.h"
 #include "qapi-event.h"
 #include "hw/nmi.h"
+#include "sysemu/replay.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -69,6 +71,14 @@ static CPUState *next_cpu;
 int64_t max_delay;
 int64_t max_advance;
 
+/* vcpu throttling controls */
+static QEMUTimer *throttle_timer;
+static unsigned int throttle_percentage;
+
+#define CPU_THROTTLE_PCT_MIN 1
+#define CPU_THROTTLE_PCT_MAX 99
+#define CPU_THROTTLE_TIMESLICE_NS 10000000
+
 bool cpu_is_stopped(CPUState *cpu)
 {
     return cpu->stopped || !runstate_is_running();
@@ -145,7 +155,7 @@ int64_t cpu_get_icount_raw(void)
 
     icount = timers_state.qemu_icount;
     if (cpu) {
-        if (!cpu_can_do_io(cpu)) {
+        if (!cpu->can_do_io) {
             fprintf(stderr, "Bad icount read\n");
             exit(1);
         }
@@ -191,7 +201,7 @@ int64_t cpu_get_ticks(void)
 
     ticks = timers_state.cpu_ticks_offset;
     if (timers_state.cpu_ticks_enabled) {
-        ticks += cpu_get_real_ticks();
+        ticks += cpu_get_host_ticks();
     }
 
     if (timers_state.cpu_ticks_prev > ticks) {
@@ -239,7 +249,7 @@ void cpu_enable_ticks(void)
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (!timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset -= cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset -= cpu_get_host_ticks();
         timers_state.cpu_clock_offset -= get_clock();
         timers_state.cpu_ticks_enabled = 1;
     }
@@ -255,7 +265,7 @@ void cpu_disable_ticks(void)
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset += cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset += cpu_get_host_ticks();
         timers_state.cpu_clock_offset = cpu_get_clock_locked();
         timers_state.cpu_ticks_enabled = 0;
     }
@@ -266,7 +276,7 @@ void cpu_disable_ticks(void)
    fairly approximate, so ignore small variation.
    When the guest is idle real and virtual time will be aligned in
    the IO wait loop.  */
-#define ICOUNT_WOBBLE (get_ticks_per_sec() / 10)
+#define ICOUNT_WOBBLE (NANOSECONDS_PER_SECOND / 10)
 
 static void icount_adjust(void)
 {
@@ -317,7 +327,7 @@ static void icount_adjust_vm(void *opaque)
 {
     timer_mod(icount_vm_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 10);
+                   NANOSECONDS_PER_SECOND / 10);
     icount_adjust();
 }
 
@@ -326,18 +336,27 @@ static int64_t qemu_icount_round(int64_t count)
     return (count + (1 << icount_time_shift) - 1) >> icount_time_shift;
 }
 
-static void icount_warp_rt(void *opaque)
+static void icount_warp_rt(void)
 {
+    unsigned seq;
+    int64_t warp_start;
+
     /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
      * changes from -1 to another value, so the race here is okay.
      */
-    if (atomic_read(&vm_clock_warp_start) == -1) {
+    do {
+        seq = seqlock_read_begin(&timers_state.vm_clock_seqlock);
+        warp_start = vm_clock_warp_start;
+    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, seq));
+
+    if (warp_start == -1) {
         return;
     }
 
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (runstate_is_running()) {
-        int64_t clock = cpu_get_clock_locked();
+        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
+                                     cpu_get_clock_locked());
         int64_t warp_delta;
 
         warp_delta = clock - vm_clock_warp_start;
@@ -360,6 +379,14 @@ static void icount_warp_rt(void *opaque)
     }
 }
 
+static void icount_timer_cb(void *opaque)
+{
+    /* No need for a checkpoint because the timer already synchronizes
+     * with CHECKPOINT_CLOCK_VIRTUAL_RT.
+     */
+    icount_warp_rt();
+}
+
 void qtest_clock_warp(int64_t dest)
 {
     int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
@@ -381,39 +408,34 @@ void qtest_clock_warp(int64_t dest)
     qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 }
 
-void qemu_clock_warp(QEMUClockType type)
+void qemu_start_warp_timer(void)
 {
     int64_t clock;
     int64_t deadline;
 
-    /*
-     * There are too many global variables to make the "warp" behavior
-     * applicable to other clocks.  But a clock argument removes the
-     * need for if statements all over the place.
+    if (!use_icount) {
+        return;
+    }
+
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
      */
-    if (type != QEMU_CLOCK_VIRTUAL || !use_icount) {
+    if (!runstate_is_running()) {
         return;
     }
 
-    if (icount_sleep) {
-        /*
-         * If the CPUs have been sleeping, advance QEMU_CLOCK_VIRTUAL timer now.
-         * This ensures that the deadline for the timer is computed correctly
-         * below.
-         * This also makes sure that the insn counter is synchronized before
-         * the CPU starts running, in case the CPU is woken by an event other
-         * than the earliest QEMU_CLOCK_VIRTUAL timer.
-         */
-        icount_warp_rt(NULL);
-        timer_del(icount_warp_timer);
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_START)) {
+        return;
     }
+
     if (!all_cpu_threads_idle()) {
         return;
     }
 
     if (qtest_enabled()) {
         /* When testing, qtest commands advance icount.  */
-	return;
+        return;
     }
 
     /* We want to use the earliest deadline from ALL vm_clocks */
@@ -469,6 +491,28 @@ void qemu_clock_warp(QEMUClockType type)
     }
 }
 
+static void qemu_account_warp_timer(void)
+{
+    if (!use_icount || !icount_sleep) {
+        return;
+    }
+
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
+     */
+    if (!runstate_is_running()) {
+        return;
+    }
+
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_ACCOUNT)) {
+        return;
+    }
+
+    timer_del(icount_warp_timer);
+    icount_warp_rt();
+}
+
 static bool icount_state_needed(void *opaque)
 {
     return use_icount;
@@ -505,10 +549,80 @@ static const VMStateDescription vmstate_timers = {
     }
 };
 
+static void cpu_throttle_thread(void *opaque)
+{
+    CPUState *cpu = opaque;
+    double pct;
+    double throttle_ratio;
+    long sleeptime_ns;
+
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    throttle_ratio = pct / (1 - pct);
+    sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);
+
+    qemu_mutex_unlock_iothread();
+    atomic_set(&cpu->throttle_thread_scheduled, 0);
+    g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
+    qemu_mutex_lock_iothread();
+}
+
+static void cpu_throttle_timer_tick(void *opaque)
+{
+    CPUState *cpu;
+    double pct;
+
+    /* Stop the timer if needed */
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+    CPU_FOREACH(cpu) {
+        if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
+            async_run_on_cpu(cpu, cpu_throttle_thread, cpu);
+        }
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                   CPU_THROTTLE_TIMESLICE_NS / (1-pct));
+}
+
+void cpu_throttle_set(int new_throttle_pct)
+{
+    /* Ensure throttle percentage is within valid range */
+    new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
+    new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);
+
+    atomic_set(&throttle_percentage, new_throttle_pct);
+
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                       CPU_THROTTLE_TIMESLICE_NS);
+}
+
+void cpu_throttle_stop(void)
+{
+    atomic_set(&throttle_percentage, 0);
+}
+
+bool cpu_throttle_active(void)
+{
+    return (cpu_throttle_get_percentage() != 0);
+}
+
+int cpu_throttle_get_percentage(void)
+{
+    return atomic_read(&throttle_percentage);
+}
+
 void cpu_ticks_init(void)
 {
     seqlock_init(&timers_state.vm_clock_seqlock, NULL);
     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
+    throttle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
+                                           cpu_throttle_timer_tick, NULL);
 }
 
 void configure_icount(QemuOpts *opts, Error **errp)
@@ -527,13 +641,13 @@ void configure_icount(QemuOpts *opts, Error **errp)
     icount_sleep = qemu_opt_get_bool(opts, "sleep", true);
     if (icount_sleep) {
         icount_warp_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
-                                         icount_warp_rt, NULL);
+                                         icount_timer_cb, NULL);
     }
 
     icount_align_option = qemu_opt_get_bool(opts, "align", false);
 
     if (icount_align_option && !icount_sleep) {
-        error_setg(errp, "align=on and sleep=no are incompatible");
+        error_setg(errp, "align=on and sleep=off are incompatible");
     }
     if (strcmp(option, "auto") != 0) {
         errno = 0;
@@ -546,7 +660,7 @@ void configure_icount(QemuOpts *opts, Error **errp)
     } else if (icount_align_option) {
         error_setg(errp, "shift=auto and align=on are incompatible");
     } else if (!icount_sleep) {
-        error_setg(errp, "shift=auto and sleep=no are incompatible");
+        error_setg(errp, "shift=auto and sleep=off are incompatible");
     }
 
     use_icount = 2;
@@ -568,7 +682,7 @@ void configure_icount(QemuOpts *opts, Error **errp)
                                         icount_adjust_vm, NULL);
     timer_mod(icount_vm_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 10);
+                   NANOSECONDS_PER_SECOND / 10);
 }
 
 /***********************************************************/
@@ -616,15 +730,6 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
-void cpu_clean_all_dirty(void)
-{
-    CPUState *cpu;
-
-    CPU_FOREACH(cpu) {
-        cpu_clean_state(cpu);
-    }
-}
-
 static int do_vm_stop(RunState state)
 {
     int ret = 0;
@@ -638,7 +743,7 @@ static int do_vm_stop(RunState state)
     }
 
     bdrv_drain_all();
-    ret = bdrv_flush_all();
+    ret = blk_flush_all();
 
     return ret;
 }
@@ -661,14 +766,6 @@ static void cpu_handle_guest_debug(CPUState *cpu)
     cpu->stopped = true;
 }
 
-static void cpu_signal(int sig)
-{
-    if (current_cpu) {
-        cpu_exit(current_cpu);
-    }
-    exit_request = 1;
-}
-
 #ifdef CONFIG_LINUX
 static void sigbus_reraise(void)
 {
@@ -781,29 +878,11 @@ static void qemu_kvm_init_cpu_signals(CPUState *cpu)
     }
 }
 
-static void qemu_tcg_init_cpu_signals(void)
-{
-    sigset_t set;
-    struct sigaction sigact;
-
-    memset(&sigact, 0, sizeof(sigact));
-    sigact.sa_handler = cpu_signal;
-    sigaction(SIG_IPI, &sigact, NULL);
-
-    sigemptyset(&set);
-    sigaddset(&set, SIG_IPI);
-    pthread_sigmask(SIG_UNBLOCK, &set, NULL);
-}
-
 #else /* _WIN32 */
 static void qemu_kvm_init_cpu_signals(CPUState *cpu)
 {
     abort();
 }
-
-static void qemu_tcg_init_cpu_signals(void)
-{
-}
 #endif /* _WIN32 */
 
 static QemuMutex qemu_global_mutex;
@@ -812,9 +891,6 @@ static unsigned iothread_requesting_mutex;
 
 static QemuThread io_thread;
 
-static QemuThread *tcg_cpu_thread;
-static QemuCond *tcg_halt_cond;
-
 /* cpu creation */
 static QemuCond qemu_cpu_cond;
 /* system init */
@@ -845,6 +921,8 @@ void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     wi.func = func;
     wi.data = data;
     wi.free = false;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = &wi;
     } else {
@@ -853,9 +931,10 @@ void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     cpu->queued_work_last = &wi;
     wi.next = NULL;
     wi.done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
-    while (!wi.done) {
+    while (!atomic_mb_read(&wi.done)) {
         CPUState *self_cpu = current_cpu;
 
         qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
@@ -876,6 +955,8 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     wi->func = func;
     wi->data = data;
     wi->free = true;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = wi;
     } else {
@@ -884,6 +965,7 @@ void async_run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
     cpu->queued_work_last = wi;
     wi->next = NULL;
     wi->done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
 }
@@ -896,15 +978,23 @@ static void flush_queued_work(CPUState *cpu)
         return;
     }
 
-    while ((wi = cpu->queued_work_first)) {
+    qemu_mutex_lock(&cpu->work_mutex);
+    while (cpu->queued_work_first != NULL) {
+        wi = cpu->queued_work_first;
         cpu->queued_work_first = wi->next;
+        if (!cpu->queued_work_first) {
+            cpu->queued_work_last = NULL;
+        }
+        qemu_mutex_unlock(&cpu->work_mutex);
         wi->func(wi->data);
-        wi->done = true;
+        qemu_mutex_lock(&cpu->work_mutex);
         if (wi->free) {
             g_free(wi);
+        } else {
+            atomic_mb_set(&wi->done, true);
         }
     }
-    cpu->queued_work_last = NULL;
+    qemu_mutex_unlock(&cpu->work_mutex);
     qemu_cond_broadcast(&qemu_work_cond);
 }
 
@@ -913,21 +1003,16 @@ static void qemu_wait_io_event_common(CPUState *cpu)
     if (cpu->stop) {
         cpu->stop = false;
         cpu->stopped = true;
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
     flush_queued_work(cpu);
     cpu->thread_kicked = false;
 }
 
-static void qemu_tcg_wait_io_event(void)
+static void qemu_tcg_wait_io_event(CPUState *cpu)
 {
-    CPUState *cpu;
-
     while (all_cpu_threads_idle()) {
-       /* Start accounting real time to the virtual clock if the CPUs
-          are idle.  */
-        qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
-        qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
     while (iothread_requesting_mutex) {
@@ -1041,7 +1126,6 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     rcu_register_thread();
 
     qemu_mutex_lock_iothread();
-    qemu_tcg_init_cpu_signals();
     qemu_thread_get_self(cpu->thread);
 
     CPU_FOREACH(cpu) {
@@ -1053,7 +1137,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
 
     /* wait for initial kick-off after machine start */
     while (first_cpu->stopped) {
-        qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(first_cpu->halt_cond, &qemu_global_mutex);
 
         /* process any pending work */
         CPU_FOREACH(cpu) {
@@ -1062,7 +1146,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     }
 
     /* process any pending work */
-    exit_request = 1;
+    atomic_mb_set(&exit_request, 1);
 
     while (1) {
         tcg_exec_all();
@@ -1074,7 +1158,7 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
                 qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
             }
         }
-        qemu_tcg_wait_io_event();
+        qemu_tcg_wait_io_event(QTAILQ_FIRST(&cpus));
     }
 
     return NULL;
@@ -1085,61 +1169,47 @@ static void qemu_cpu_kick_thread(CPUState *cpu)
 #ifndef _WIN32
     int err;
 
+    if (cpu->thread_kicked) {
+        return;
+    }
+    cpu->thread_kicked = true;
     err = pthread_kill(cpu->thread->thread, SIG_IPI);
     if (err) {
         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
         exit(1);
     }
 #else /* _WIN32 */
-    if (!qemu_cpu_is_self(cpu)) {
-        CONTEXT tcgContext;
-
-        if (SuspendThread(cpu->hThread) == (DWORD)-1) {
-            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
-                    GetLastError());
-            exit(1);
-        }
-
-        /* On multi-core systems, we are not sure that the thread is actually
-         * suspended until we can get the context.
-         */
-        tcgContext.ContextFlags = CONTEXT_CONTROL;
-        while (GetThreadContext(cpu->hThread, &tcgContext) != 0) {
-            continue;
-        }
-
-        cpu_signal(0);
+    abort();
+#endif
+}
 
-        if (ResumeThread(cpu->hThread) == (DWORD)-1) {
-            fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
-                    GetLastError());
-            exit(1);
-        }
+static void qemu_cpu_kick_no_halt(void)
+{
+    CPUState *cpu;
+    /* Ensure whatever caused the exit has reached the CPU threads before
+     * writing exit_request.
+     */
+    atomic_mb_set(&exit_request, 1);
+    cpu = atomic_mb_read(&tcg_current_cpu);
+    if (cpu) {
+        cpu_exit(cpu);
     }
-#endif
 }
 
 void qemu_cpu_kick(CPUState *cpu)
 {
     qemu_cond_broadcast(cpu->halt_cond);
-    if (!tcg_enabled() && !cpu->thread_kicked) {
+    if (tcg_enabled()) {
+        qemu_cpu_kick_no_halt();
+    } else {
         qemu_cpu_kick_thread(cpu);
-        cpu->thread_kicked = true;
     }
 }
 
 void qemu_cpu_kick_self(void)
 {
-#ifndef _WIN32
     assert(current_cpu);
-
-    if (!current_cpu->thread_kicked) {
-        qemu_cpu_kick_thread(current_cpu);
-        current_cpu->thread_kicked = true;
-    }
-#else
-    abort();
-#endif
+    qemu_cpu_kick_thread(current_cpu);
 }
 
 bool qemu_cpu_is_self(CPUState *cpu)
@@ -1166,12 +1236,12 @@ void qemu_mutex_lock_iothread(void)
      * TCG code execution.
      */
     if (!tcg_enabled() || qemu_in_vcpu_thread() ||
-        !first_cpu || !first_cpu->thread) {
+        !first_cpu || !first_cpu->created) {
         qemu_mutex_lock(&qemu_global_mutex);
         atomic_dec(&iothread_requesting_mutex);
     } else {
         if (qemu_mutex_trylock(&qemu_global_mutex)) {
-            qemu_cpu_kick_thread(first_cpu);
+            qemu_cpu_kick_no_halt();
             qemu_mutex_lock(&qemu_global_mutex);
         }
         atomic_dec(&iothread_requesting_mutex);
@@ -1251,8 +1321,8 @@ void resume_all_vcpus(void)
 static void qemu_tcg_init_vcpu(CPUState *cpu)
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
-
-    tcg_cpu_address_space_init(cpu, cpu->as);
+    static QemuCond *tcg_halt_cond;
+    static QemuThread *tcg_cpu_thread;
 
     /* share a single thread for all cpus with TCG */
     if (!tcg_cpu_thread) {
@@ -1314,6 +1384,17 @@ void qemu_init_vcpu(CPUState *cpu)
     cpu->nr_cores = smp_cores;
     cpu->nr_threads = smp_threads;
     cpu->stopped = true;
+
+    if (!cpu->as) {
+        /* If the target cpu hasn't set up any address spaces itself,
+         * give it the default one.
+         */
+        AddressSpace *as = address_space_init_shareable(cpu->memory,
+                                                        "cpu-memory");
+        cpu->num_ases = 1;
+        cpu_address_space_init(cpu, as, 0);
+    }
+
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(cpu);
     } else if (tcg_enabled()) {
@@ -1329,7 +1410,7 @@ void cpu_stop_current(void)
         current_cpu->stop = false;
         current_cpu->stopped = true;
         cpu_exit(current_cpu);
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
 }
 
@@ -1357,9 +1438,33 @@ int vm_stop_force_state(RunState state)
         return vm_stop(state);
     } else {
         runstate_set(state);
+
+        bdrv_drain_all();
         /* Make sure to return an error if the flush in a previous vm_stop()
          * failed. */
-        return bdrv_flush_all();
+        return blk_flush_all();
+    }
+}
+
+static int64_t tcg_get_icount_limit(void)
+{
+    int64_t deadline;
+
+    if (replay_mode != REPLAY_MODE_PLAY) {
+        deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
+
+        /* Maintain prior (possibly buggy) behaviour where if no deadline
+         * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
+         * INT32_MAX nanoseconds ahead, we still use INT32_MAX
+         * nanoseconds.
+         */
+        if ((deadline < 0) || (deadline > INT32_MAX)) {
+            deadline = INT32_MAX;
+        }
+
+        return qemu_icount_round(deadline);
+    } else {
+        return replay_get_instructions();
     }
 }
 
@@ -1375,24 +1480,12 @@ static int tcg_cpu_exec(CPUState *cpu)
 #endif
     if (use_icount) {
         int64_t count;
-        int64_t deadline;
         int decr;
         timers_state.qemu_icount -= (cpu->icount_decr.u16.low
                                     + cpu->icount_extra);
         cpu->icount_decr.u16.low = 0;
         cpu->icount_extra = 0;
-        deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
-
-        /* Maintain prior (possibly buggy) behaviour where if no deadline
-         * was set (as there is no QEMU_CLOCK_VIRTUAL timer) or it is more than
-         * INT32_MAX nanoseconds ahead, we still use INT32_MAX
-         * nanoseconds.
-         */
-        if ((deadline < 0) || (deadline > INT32_MAX)) {
-            deadline = INT32_MAX;
-        }
-
-        count = qemu_icount_round(deadline);
+        count = tcg_get_icount_limit();
         timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
         count -= decr;
@@ -1410,6 +1503,7 @@ static int tcg_cpu_exec(CPUState *cpu)
                         + cpu->icount_extra);
         cpu->icount_decr.u32 = 0;
         cpu->icount_extra = 0;
+        replay_account_executed_instructions();
     }
     return ret;
 }
@@ -1419,7 +1513,7 @@ static void tcg_exec_all(void)
     int r;
 
     /* Account partial waits to QEMU_CLOCK_VIRTUAL.  */
-    qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
+    qemu_account_warp_timer();
 
     if (next_cpu == NULL) {
         next_cpu = first_cpu;
@@ -1440,7 +1534,9 @@ static void tcg_exec_all(void)
             break;
         }
     }
-    exit_request = 0;
+
+    /* Pairs with smp_wmb in qemu_cpu_kick.  */
+    atomic_mb_set(&exit_request, 0);
 }
 
 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
@@ -1485,22 +1581,23 @@ CpuInfoList *qmp_query_cpus(Error **errp)
         info->value->qom_path = object_get_canonical_path(OBJECT(cpu));
         info->value->thread_id = cpu->thread_id;
 #if defined(TARGET_I386)
-        info->value->has_pc = true;
-        info->value->pc = env->eip + env->segs[R_CS].base;
+        info->value->arch = CPU_INFO_ARCH_X86;
+        info->value->u.x86.pc = env->eip + env->segs[R_CS].base;
 #elif defined(TARGET_PPC)
-        info->value->has_nip = true;
-        info->value->nip = env->nip;
+        info->value->arch = CPU_INFO_ARCH_PPC;
+        info->value->u.ppc.nip = env->nip;
 #elif defined(TARGET_SPARC)
-        info->value->has_pc = true;
-        info->value->pc = env->pc;
-        info->value->has_npc = true;
-        info->value->npc = env->npc;
+        info->value->arch = CPU_INFO_ARCH_SPARC;
+        info->value->u.q_sparc.pc = env->pc;
+        info->value->u.q_sparc.npc = env->npc;
 #elif defined(TARGET_MIPS)
-        info->value->has_PC = true;
-        info->value->PC = env->active_tc.PC;
+        info->value->arch = CPU_INFO_ARCH_MIPS;
+        info->value->u.q_mips.PC = env->active_tc.PC;
 #elif defined(TARGET_TRICORE)
-        info->value->has_PC = true;
-        info->value->PC = env->PC;
+        info->value->arch = CPU_INFO_ARCH_TRICORE;
+        info->value->u.tricore.PC = env->PC;
+#else
+        info->value->arch = CPU_INFO_ARCH_OTHER;
 #endif
 
         /* XXX: waiting for the qapi to support GSList */
diff --git a/qemu/cputlb.c b/qemu/cputlb.c
index a50608676..466663b56 100644
--- a/qemu/cputlb.c
+++ b/qemu/cputlb.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/memory.h"
@@ -30,8 +30,30 @@
 #include "exec/ram_addr.h"
 #include "tcg/tcg.h"
 
-//#define DEBUG_TLB
-//#define DEBUG_TLB_CHECK
+/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
+/* #define DEBUG_TLB */
+/* #define DEBUG_TLB_LOG */
+
+#ifdef DEBUG_TLB
+# define DEBUG_TLB_GATE 1
+# ifdef DEBUG_TLB_LOG
+#  define DEBUG_TLB_LOG_GATE 1
+# else
+#  define DEBUG_TLB_LOG_GATE 0
+# endif
+#else
+# define DEBUG_TLB_GATE 0
+# define DEBUG_TLB_LOG_GATE 0
+#endif
+
+#define tlb_debug(fmt, ...) do { \
+    if (DEBUG_TLB_LOG_GATE) { \
+        qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
+                      ## __VA_ARGS__); \
+    } else if (DEBUG_TLB_GATE) { \
+        fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
+    } \
+} while (0)
 
 /* statistics */
 int tlb_flush_count;
@@ -52,9 +74,8 @@ void tlb_flush(CPUState *cpu, int flush_global)
 {
     CPUArchState *env = cpu->env_ptr;
 
-#if defined(DEBUG_TLB)
-    printf("tlb_flush:\n");
-#endif
+    tlb_debug("(%d)\n", flush_global);
+
     /* must reset current TB so that interrupts cannot modify the
        links while we are modifying them */
     cpu->current_tb = NULL;
@@ -69,6 +90,39 @@ void tlb_flush(CPUState *cpu, int flush_global)
     tlb_flush_count++;
 }
 
+static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
+{
+    CPUArchState *env = cpu->env_ptr;
+
+    tlb_debug("start\n");
+    /* must reset current TB so that interrupts cannot modify the
+       links while we are modifying them */
+    cpu->current_tb = NULL;
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+        tlb_debug("%d\n", mmu_idx);
+
+        memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
+        memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
+    }
+
+    memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+}
+
+void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+    va_list argp;
+    va_start(argp, cpu);
+    v_tlb_flush_by_mmuidx(cpu, argp);
+    va_end(argp);
+}
+
 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 {
     if (addr == (tlb_entry->addr_read &
@@ -87,16 +141,14 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
     int i;
     int mmu_idx;
 
-#if defined(DEBUG_TLB)
-    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
-#endif
+    tlb_debug("page :" TARGET_FMT_lx "\n", addr);
+
     /* Check if we need to flush due to large pages.  */
     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
-#if defined(DEBUG_TLB)
-        printf("tlb_flush_page: forced full flush ("
-               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
-               env->tlb_flush_addr, env->tlb_flush_mask);
-#endif
+        tlb_debug("forcing full flush ("
+                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+                  env->tlb_flush_addr, env->tlb_flush_mask);
+
         tlb_flush(cpu, 1);
         return;
     }
@@ -121,6 +173,54 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
     tb_flush_jmp_cache(cpu, addr);
 }
 
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
+{
+    CPUArchState *env = cpu->env_ptr;
+    int i, k;
+    va_list argp;
+
+    va_start(argp, addr);
+
+    tlb_debug("addr "TARGET_FMT_lx"\n", addr);
+
+    /* Check if we need to flush due to large pages.  */
+    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
+        tlb_debug("forced full flush ("
+                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+                  env->tlb_flush_addr, env->tlb_flush_mask);
+
+        v_tlb_flush_by_mmuidx(cpu, argp);
+        va_end(argp);
+        return;
+    }
+    /* must reset current TB so that interrupts cannot modify the
+       links while we are modifying them */
+    cpu->current_tb = NULL;
+
+    addr &= TARGET_PAGE_MASK;
+    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+        tlb_debug("idx %d\n", mmu_idx);
+
+        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
+
+        /* check whether there are vltb entries that need to be flushed */
+        for (k = 0; k < CPU_VTLB_SIZE; k++) {
+            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
+        }
+    }
+    va_end(argp);
+
+    tb_flush_jmp_cache(cpu, addr);
+}
+
 /* update the TLBs so that writes to code in the virtual page 'addr'
    can be detected */
 void tlb_protect_code(ram_addr_t ram_addr)
@@ -165,27 +265,24 @@ static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
     return ram_addr;
 }
 
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 {
-    CPUState *cpu;
     CPUArchState *env;
 
-    CPU_FOREACH(cpu) {
-        int mmu_idx;
+    int mmu_idx;
 
-        env = cpu->env_ptr;
-        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-            unsigned int i;
+    env = cpu->env_ptr;
+    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+        unsigned int i;
 
-            for (i = 0; i < CPU_TLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_TLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
+                                  start1, length);
+        }
 
-            for (i = 0; i < CPU_VTLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_VTLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
+                                  start1, length);
         }
     }
 }
@@ -199,8 +296,9 @@ static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 
 /* update the TLB corresponding to virtual page vaddr
    so that it is no longer dirty */
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr)
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 {
+    CPUArchState *env = cpu->env_ptr;
     int i;
     int mmu_idx;
 
@@ -261,6 +359,7 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
     CPUTLBEntry *te;
     hwaddr iotlb, xlat, sz;
     unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
 
     assert(size >= TARGET_PAGE_SIZE);
     if (size != TARGET_PAGE_SIZE) {
@@ -268,15 +367,12 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
     }
 
     sz = size;
-    section = address_space_translate_for_iotlb(cpu, paddr, &xlat, &sz);
+    section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
     assert(sz >= TARGET_PAGE_SIZE);
 
-#if defined(DEBUG_TLB)
-    qemu_log_mask(CPU_LOG_MMU,
-           "tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
-           " prot=%x idx=%d\n",
-           vaddr, paddr, prot, mmu_idx);
-#endif
+    tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
+              " prot=%x idx=%d\n",
+              vaddr, paddr, prot, mmu_idx);
 
     address = vaddr;
     if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
@@ -320,8 +416,8 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
             /* Write access calls the I/O callback.  */
             te->addr_write = address | TLB_MMIO;
         } else if (memory_region_is_ram(section->mr)
-                   && cpu_physical_memory_is_clean(section->mr->ram_addr
-                                                   + xlat)) {
+                   && cpu_physical_memory_is_clean(
+                        memory_region_get_ram_addr(section->mr) + xlat)) {
             te->addr_write = address | TLB_NOTDIRTY;
         } else {
             te->addr_write = address;
@@ -353,15 +449,17 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
     void *p;
     MemoryRegion *mr;
     CPUState *cpu = ENV_GET_CPU(env1);
+    CPUIOTLBEntry *iotlbentry;
 
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    mmu_idx = cpu_mmu_index(env1);
+    mmu_idx = cpu_mmu_index(env1, true);
     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                  (addr & TARGET_PAGE_MASK))) {
         cpu_ldub_code(env1, addr);
     }
-    pd = env1->iotlb[mmu_idx][page_index].addr & ~TARGET_PAGE_MASK;
-    mr = iotlb_to_region(cpu, pd);
+    iotlbentry = &env1->iotlb[mmu_idx][page_index];
+    pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
+    mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
     if (memory_region_is_unassigned(mr)) {
         CPUClass *cc = CPU_GET_CLASS(cpu);
 
diff --git a/qemu/crypto/Makefile.objs b/qemu/crypto/Makefile.objs
index b05013831..0737f4811 100644
--- a/qemu/crypto/Makefile.objs
+++ b/qemu/crypto/Makefile.objs
@@ -1,5 +1,30 @@
-util-obj-y += init.o
-util-obj-y += hash.o
-util-obj-y += aes.o
-util-obj-y += desrfb.o
-util-obj-y += cipher.o
+crypto-obj-y = init.o
+crypto-obj-y += hash.o
+crypto-obj-y += aes.o
+crypto-obj-y += desrfb.o
+crypto-obj-y += cipher.o
+crypto-obj-y += tlscreds.o
+crypto-obj-y += tlscredsanon.o
+crypto-obj-y += tlscredsx509.o
+crypto-obj-y += tlssession.o
+crypto-obj-y += secret.o
+crypto-obj-$(CONFIG_GCRYPT) += random-gcrypt.o
+crypto-obj-$(if $(CONFIG_GCRYPT),n,$(CONFIG_GNUTLS_RND)) += random-gnutls.o
+crypto-obj-y += pbkdf.o
+crypto-obj-$(CONFIG_NETTLE_KDF) += pbkdf-nettle.o
+crypto-obj-$(if $(CONFIG_NETTLE_KDF),n,$(CONFIG_GCRYPT_KDF)) += pbkdf-gcrypt.o
+crypto-obj-y += ivgen.o
+crypto-obj-y += ivgen-essiv.o
+crypto-obj-y += ivgen-plain.o
+crypto-obj-y += ivgen-plain64.o
+crypto-obj-y += afsplit.o
+crypto-obj-y += xts.o
+crypto-obj-y += block.o
+crypto-obj-y += block-qcow.o
+crypto-obj-y += block-luks.o
+
+# Let the userspace emulators avoid linking gnutls/etc
+crypto-aes-obj-y = aes.o
+
+stub-obj-y += random-stub.o
+stub-obj-y += pbkdf-stub.o
diff --git a/qemu/crypto/aes.c b/qemu/crypto/aes.c
index 244a388eb..3456eacd0 100644
--- a/qemu/crypto/aes.c
+++ b/qemu/crypto/aes.c
@@ -27,6 +27,7 @@
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "crypto/aes.h"
 
diff --git a/qemu/crypto/afsplit.c b/qemu/crypto/afsplit.c
new file mode 100644
index 000000000..8074913cd
--- /dev/null
+++ b/qemu/crypto/afsplit.c
@@ -0,0 +1,158 @@
+/*
+ * QEMU Crypto anti forensic information splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * Derived from cryptsetup package lib/luks1/af.c
+ *
+ * Copyright (C) 2004, Clemens Fruhwirth <clemens@endorphin.org>
+ * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/afsplit.h"
+#include "crypto/random.h"
+
+
+static void qcrypto_afsplit_xor(size_t blocklen,
+                                const uint8_t *in1,
+                                const uint8_t *in2,
+                                uint8_t *out)
+{
+    size_t i;
+    for (i = 0; i < blocklen; i++) {
+        out[i] = in1[i] ^ in2[i];
+    }
+}
+
+
+static int qcrypto_afsplit_hash(QCryptoHashAlgorithm hash,
+                                size_t blocklen,
+                                uint8_t *block,
+                                Error **errp)
+{
+    size_t digestlen = qcrypto_hash_digest_len(hash);
+
+    size_t hashcount = blocklen / digestlen;
+    size_t finallen = blocklen % digestlen;
+    uint32_t i;
+
+    if (finallen) {
+        hashcount++;
+    } else {
+        finallen = digestlen;
+    }
+
+    for (i = 0; i < hashcount; i++) {
+        uint8_t *out = NULL;
+        size_t outlen = 0;
+        uint32_t iv = cpu_to_be32(i);
+        struct iovec in[] = {
+            { .iov_base = &iv,
+              .iov_len = sizeof(iv) },
+            { .iov_base = block + (i * digestlen),
+              .iov_len = (i == (hashcount - 1)) ? finallen : digestlen },
+        };
+
+        if (qcrypto_hash_bytesv(hash,
+                                in,
+                                G_N_ELEMENTS(in),
+                                &out, &outlen,
+                                errp) < 0) {
+            return -1;
+        }
+
+        assert(outlen == digestlen);
+        memcpy(block + (i * digestlen), out,
+               (i == (hashcount - 1)) ? finallen : digestlen);
+        g_free(out);
+    }
+
+    return 0;
+}
+
+
+int qcrypto_afsplit_encode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp)
+{
+    uint8_t *block = g_new0(uint8_t, blocklen);
+    size_t i;
+    int ret = -1;
+
+    for (i = 0; i < (stripes - 1); i++) {
+        if (qcrypto_random_bytes(out + (i * blocklen), blocklen, errp) < 0) {
+            goto cleanup;
+        }
+
+        qcrypto_afsplit_xor(blocklen,
+                            out + (i * blocklen),
+                            block,
+                            block);
+        if (qcrypto_afsplit_hash(hash, blocklen, block,
+                                 errp) < 0) {
+            goto cleanup;
+        }
+    }
+    qcrypto_afsplit_xor(blocklen,
+                        in,
+                        block,
+                        out + (i * blocklen));
+    ret = 0;
+
+ cleanup:
+    g_free(block);
+    return ret;
+}
+
+
+int qcrypto_afsplit_decode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp)
+{
+    uint8_t *block = g_new0(uint8_t, blocklen);
+    size_t i;
+    int ret = -1;
+
+    for (i = 0; i < (stripes - 1); i++) {
+        qcrypto_afsplit_xor(blocklen,
+                            in + (i * blocklen),
+                            block,
+                            block);
+        if (qcrypto_afsplit_hash(hash, blocklen, block,
+                                 errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    qcrypto_afsplit_xor(blocklen,
+                        in + (i * blocklen),
+                        block,
+                        out);
+
+    ret = 0;
+
+ cleanup:
+    g_free(block);
+    return ret;
+}
diff --git a/qemu/crypto/block-luks.c b/qemu/crypto/block-luks.c
new file mode 100644
index 000000000..439f89230
--- /dev/null
+++ b/qemu/crypto/block-luks.c
@@ -0,0 +1,1329 @@
+/*
+ * QEMU Crypto block device encryption LUKS format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+
+#include "crypto/block-luks.h"
+
+#include "crypto/hash.h"
+#include "crypto/afsplit.h"
+#include "crypto/pbkdf.h"
+#include "crypto/secret.h"
+#include "crypto/random.h"
+
+#ifdef CONFIG_UUID
+#include <uuid/uuid.h>
+#endif
+
+#include "qemu/coroutine.h"
+
+/*
+ * Reference for the LUKS format implemented here is
+ *
+ *   docs/on-disk-format.pdf
+ *
+ * in 'cryptsetup' package source code
+ *
+ * This file implements the 1.2.1 specification, dated
+ * Oct 16, 2011.
+ */
+
+typedef struct QCryptoBlockLUKS QCryptoBlockLUKS;
+typedef struct QCryptoBlockLUKSHeader QCryptoBlockLUKSHeader;
+typedef struct QCryptoBlockLUKSKeySlot QCryptoBlockLUKSKeySlot;
+
+
+/* The following constants are all defined by the LUKS spec */
+#define QCRYPTO_BLOCK_LUKS_VERSION 1
+
+#define QCRYPTO_BLOCK_LUKS_MAGIC_LEN 6
+#define QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN 32
+#define QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN 32
+#define QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN 32
+#define QCRYPTO_BLOCK_LUKS_DIGEST_LEN 20
+#define QCRYPTO_BLOCK_LUKS_SALT_LEN 32
+#define QCRYPTO_BLOCK_LUKS_UUID_LEN 40
+#define QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS 8
+#define QCRYPTO_BLOCK_LUKS_STRIPES 4000
+#define QCRYPTO_BLOCK_LUKS_MIN_SLOT_KEY_ITERS 1000
+#define QCRYPTO_BLOCK_LUKS_MIN_MASTER_KEY_ITERS 1000
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET 4096
+
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_DISABLED 0x0000DEAD
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED 0x00AC71F3
+
+#define QCRYPTO_BLOCK_LUKS_SECTOR_SIZE 512LL
+
+static const char qcrypto_block_luks_magic[QCRYPTO_BLOCK_LUKS_MAGIC_LEN] = {
+    'L', 'U', 'K', 'S', 0xBA, 0xBE
+};
+
+typedef struct QCryptoBlockLUKSNameMap QCryptoBlockLUKSNameMap;
+struct QCryptoBlockLUKSNameMap {
+    const char *name;
+    int id;
+};
+
+typedef struct QCryptoBlockLUKSCipherSizeMap QCryptoBlockLUKSCipherSizeMap;
+struct QCryptoBlockLUKSCipherSizeMap {
+    uint32_t key_bytes;
+    int id;
+};
+typedef struct QCryptoBlockLUKSCipherNameMap QCryptoBlockLUKSCipherNameMap;
+struct QCryptoBlockLUKSCipherNameMap {
+    const char *name;
+    const QCryptoBlockLUKSCipherSizeMap *sizes;
+};
+
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_aes[] = {
+    { 16, QCRYPTO_CIPHER_ALG_AES_128 },
+    { 24, QCRYPTO_CIPHER_ALG_AES_192 },
+    { 32, QCRYPTO_CIPHER_ALG_AES_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_cast5[] = {
+    { 16, QCRYPTO_CIPHER_ALG_CAST5_128 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_serpent[] = {
+    { 16, QCRYPTO_CIPHER_ALG_SERPENT_128 },
+    { 24, QCRYPTO_CIPHER_ALG_SERPENT_192 },
+    { 32, QCRYPTO_CIPHER_ALG_SERPENT_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_twofish[] = {
+    { 16, QCRYPTO_CIPHER_ALG_TWOFISH_128 },
+    { 24, QCRYPTO_CIPHER_ALG_TWOFISH_192 },
+    { 32, QCRYPTO_CIPHER_ALG_TWOFISH_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherNameMap
+qcrypto_block_luks_cipher_name_map[] = {
+    { "aes", qcrypto_block_luks_cipher_size_map_aes },
+    { "cast5", qcrypto_block_luks_cipher_size_map_cast5 },
+    { "serpent", qcrypto_block_luks_cipher_size_map_serpent },
+    { "twofish", qcrypto_block_luks_cipher_size_map_twofish },
+};
+
+
+/*
+ * This struct is written to disk in big-endian format,
+ * but operated upon in native-endian format.
+ */
+struct QCryptoBlockLUKSKeySlot {
+    /* state of keyslot, enabled/disable */
+    uint32_t active;
+    /* iterations for PBKDF2 */
+    uint32_t iterations;
+    /* salt for PBKDF2 */
+    uint8_t salt[QCRYPTO_BLOCK_LUKS_SALT_LEN];
+    /* start sector of key material */
+    uint32_t key_offset;
+    /* number of anti-forensic stripes */
+    uint32_t stripes;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct QCryptoBlockLUKSKeySlot) != 48);
+
+
+/*
+ * This struct is written to disk in big-endian format,
+ * but operated upon in native-endian format.
+ */
+struct QCryptoBlockLUKSHeader {
+    /* 'L', 'U', 'K', 'S', '0xBA', '0xBE' */
+    char magic[QCRYPTO_BLOCK_LUKS_MAGIC_LEN];
+
+    /* LUKS version, currently 1 */
+    uint16_t version;
+
+    /* cipher name specification (aes, etc) */
+    char cipher_name[QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN];
+
+    /* cipher mode specification (cbc-plain, xts-essiv:sha256, etc) */
+    char cipher_mode[QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN];
+
+    /* hash specification (sha256, etc) */
+    char hash_spec[QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN];
+
+    /* start offset of the volume data (in 512 byte sectors) */
+    uint32_t payload_offset;
+
+    /* Number of key bytes */
+    uint32_t key_bytes;
+
+    /* master key checksum after PBKDF2 */
+    uint8_t master_key_digest[QCRYPTO_BLOCK_LUKS_DIGEST_LEN];
+
+    /* salt for master key PBKDF2 */
+    uint8_t master_key_salt[QCRYPTO_BLOCK_LUKS_SALT_LEN];
+
+    /* iterations for master key PBKDF2 */
+    uint32_t master_key_iterations;
+
+    /* UUID of the partition in standard ASCII representation */
+    uint8_t uuid[QCRYPTO_BLOCK_LUKS_UUID_LEN];
+
+    /* key slots */
+    QCryptoBlockLUKSKeySlot key_slots[QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct QCryptoBlockLUKSHeader) != 592);
+
+
+struct QCryptoBlockLUKS {
+    QCryptoBlockLUKSHeader header;
+};
+
+
+static int qcrypto_block_luks_cipher_name_lookup(const char *name,
+                                                 QCryptoCipherMode mode,
+                                                 uint32_t key_bytes,
+                                                 Error **errp)
+{
+    const QCryptoBlockLUKSCipherNameMap *map =
+        qcrypto_block_luks_cipher_name_map;
+    size_t maplen = G_N_ELEMENTS(qcrypto_block_luks_cipher_name_map);
+    size_t i, j;
+
+    if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+        key_bytes /= 2;
+    }
+
+    for (i = 0; i < maplen; i++) {
+        if (!g_str_equal(map[i].name, name)) {
+            continue;
+        }
+        for (j = 0; j < map[i].sizes[j].key_bytes; j++) {
+            if (map[i].sizes[j].key_bytes == key_bytes) {
+                return map[i].sizes[j].id;
+            }
+        }
+    }
+
+    error_setg(errp, "Algorithm %s with key size %d bytes not supported",
+               name, key_bytes);
+    return 0;
+}
+
+static const char *
+qcrypto_block_luks_cipher_alg_lookup(QCryptoCipherAlgorithm alg,
+                                     Error **errp)
+{
+    const QCryptoBlockLUKSCipherNameMap *map =
+        qcrypto_block_luks_cipher_name_map;
+    size_t maplen = G_N_ELEMENTS(qcrypto_block_luks_cipher_name_map);
+    size_t i, j;
+    for (i = 0; i < maplen; i++) {
+        for (j = 0; j < map[i].sizes[j].key_bytes; j++) {
+            if (map[i].sizes[j].id == alg) {
+                return map[i].name;
+            }
+        }
+    }
+
+    error_setg(errp, "Algorithm '%s' not supported",
+               QCryptoCipherAlgorithm_lookup[alg]);
+    return NULL;
+}
+
+/* XXX replace with qapi_enum_parse() in future, when we can
+ * make that function emit a more friendly error message */
+static int qcrypto_block_luks_name_lookup(const char *name,
+                                          const char *const *map,
+                                          size_t maplen,
+                                          const char *type,
+                                          Error **errp)
+{
+    size_t i;
+    for (i = 0; i < maplen; i++) {
+        if (g_str_equal(map[i], name)) {
+            return i;
+        }
+    }
+
+    error_setg(errp, "%s %s not supported", type, name);
+    return 0;
+}
+
+#define qcrypto_block_luks_cipher_mode_lookup(name, errp)               \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoCipherMode_lookup,            \
+                                   QCRYPTO_CIPHER_MODE__MAX,            \
+                                   "Cipher mode",                       \
+                                   errp)
+
+#define qcrypto_block_luks_hash_name_lookup(name, errp)                 \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoHashAlgorithm_lookup,         \
+                                   QCRYPTO_HASH_ALG__MAX,               \
+                                   "Hash algorithm",                    \
+                                   errp)
+
+#define qcrypto_block_luks_ivgen_name_lookup(name, errp)                \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoIVGenAlgorithm_lookup,        \
+                                   QCRYPTO_IVGEN_ALG__MAX,              \
+                                   "IV generator",                      \
+                                   errp)
+
+
+static bool
+qcrypto_block_luks_has_format(const uint8_t *buf,
+                              size_t buf_size)
+{
+    const QCryptoBlockLUKSHeader *luks_header = (const void *)buf;
+
+    if (buf_size >= offsetof(QCryptoBlockLUKSHeader, cipher_name) &&
+        memcmp(luks_header->magic, qcrypto_block_luks_magic,
+               QCRYPTO_BLOCK_LUKS_MAGIC_LEN) == 0 &&
+        be16_to_cpu(luks_header->version) == QCRYPTO_BLOCK_LUKS_VERSION) {
+        return true;
+    } else {
+        return false;
+    }
+}
+
+
+/**
+ * Deal with a quirk of dm-crypt usage of ESSIV.
+ *
+ * When calculating ESSIV IVs, the cipher length used by ESSIV
+ * may be different from the cipher length used for the block
+ * encryption, becauses dm-crypt uses the hash digest length
+ * as the key size. ie, if you have AES 128 as the block cipher
+ * and SHA 256 as ESSIV hash, then ESSIV will use AES 256 as
+ * the cipher since that gets a key length matching the digest
+ * size, not AES 128 with truncated digest as might be imagined
+ */
+static QCryptoCipherAlgorithm
+qcrypto_block_luks_essiv_cipher(QCryptoCipherAlgorithm cipher,
+                                QCryptoHashAlgorithm hash,
+                                Error **errp)
+{
+    size_t digestlen = qcrypto_hash_digest_len(hash);
+    size_t keylen = qcrypto_cipher_get_key_len(cipher);
+    if (digestlen == keylen) {
+        return cipher;
+    }
+
+    switch (cipher) {
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_AES_128)) {
+            return QCRYPTO_CIPHER_ALG_AES_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_AES_192)) {
+            return QCRYPTO_CIPHER_ALG_AES_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_AES_256)) {
+            return QCRYPTO_CIPHER_ALG_AES_256;
+        } else {
+            error_setg(errp, "No AES cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_SERPENT_128)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_SERPENT_192)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_SERPENT_256)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_256;
+        } else {
+            error_setg(errp, "No Serpent cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_TWOFISH_128)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_TWOFISH_192)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_TWOFISH_256)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_256;
+        } else {
+            error_setg(errp, "No Twofish cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    default:
+        error_setg(errp, "Cipher %s not supported with essiv",
+                   QCryptoCipherAlgorithm_lookup[cipher]);
+        return 0;
+    }
+}
+
+/*
+ * Given a key slot, and user password, this will attempt to unlock
+ * the master encryption key from the key slot.
+ *
+ * Returns:
+ *    0 if the key slot is disabled, or key could not be decrypted
+ *      with the provided password
+ *    1 if the key slot is enabled, and key decrypted successfully
+ *      with the provided password
+ *   -1 if a fatal error occurred loading the key
+ */
+static int
+qcrypto_block_luks_load_key(QCryptoBlock *block,
+                            QCryptoBlockLUKSKeySlot *slot,
+                            const char *password,
+                            QCryptoCipherAlgorithm cipheralg,
+                            QCryptoCipherMode ciphermode,
+                            QCryptoHashAlgorithm hash,
+                            QCryptoIVGenAlgorithm ivalg,
+                            QCryptoCipherAlgorithm ivcipheralg,
+                            QCryptoHashAlgorithm ivhash,
+                            uint8_t *masterkey,
+                            size_t masterkeylen,
+                            QCryptoBlockReadFunc readfunc,
+                            void *opaque,
+                            Error **errp)
+{
+    QCryptoBlockLUKS *luks = block->opaque;
+    uint8_t *splitkey;
+    size_t splitkeylen;
+    uint8_t *possiblekey;
+    int ret = -1;
+    ssize_t rv;
+    QCryptoCipher *cipher = NULL;
+    uint8_t keydigest[QCRYPTO_BLOCK_LUKS_DIGEST_LEN];
+    QCryptoIVGen *ivgen = NULL;
+    size_t niv;
+
+    if (slot->active != QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED) {
+        return 0;
+    }
+
+    splitkeylen = masterkeylen * slot->stripes;
+    splitkey = g_new0(uint8_t, splitkeylen);
+    possiblekey = g_new0(uint8_t, masterkeylen);
+
+    /*
+     * The user password is used to generate a (possible)
+     * decryption key. This may or may not successfully
+     * decrypt the master key - we just blindly assume
+     * the key is correct and validate the results of
+     * decryption later.
+     */
+    if (qcrypto_pbkdf2(hash,
+                       (const uint8_t *)password, strlen(password),
+                       slot->salt, QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       slot->iterations,
+                       possiblekey, masterkeylen,
+                       errp) < 0) {
+        goto cleanup;
+    }
+
+    /*
+     * We need to read the master key material from the
+     * LUKS key material header. What we're reading is
+     * not the raw master key, but rather the data after
+     * it has been passed through AFSplit and the result
+     * then encrypted.
+     */
+    rv = readfunc(block,
+                  slot->key_offset * QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                  splitkey, splitkeylen,
+                  errp,
+                  opaque);
+    if (rv < 0) {
+        goto cleanup;
+    }
+
+
+    /* Setup the cipher/ivgen that we'll use to try to decrypt
+     * the split master key material */
+    cipher = qcrypto_cipher_new(cipheralg, ciphermode,
+                                possiblekey, masterkeylen,
+                                errp);
+    if (!cipher) {
+        goto cleanup;
+    }
+
+    niv = qcrypto_cipher_get_iv_len(cipheralg,
+                                    ciphermode);
+    ivgen = qcrypto_ivgen_new(ivalg,
+                              ivcipheralg,
+                              ivhash,
+                              possiblekey, masterkeylen,
+                              errp);
+    if (!ivgen) {
+        goto cleanup;
+    }
+
+
+    /*
+     * The master key needs to be decrypted in the same
+     * way that the block device payload will be decrypted
+     * later. In particular we'll be using the IV generator
+     * to reset the encryption cipher every time the master
+     * key crosses a sector boundary.
+     */
+    if (qcrypto_block_decrypt_helper(cipher,
+                                     niv,
+                                     ivgen,
+                                     QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                     0,
+                                     splitkey,
+                                     splitkeylen,
+                                     errp) < 0) {
+        goto cleanup;
+    }
+
+    /*
+     * Now we've decrypted the split master key, join
+     * it back together to get the actual master key.
+     */
+    if (qcrypto_afsplit_decode(hash,
+                               masterkeylen,
+                               slot->stripes,
+                               splitkey,
+                               masterkey,
+                               errp) < 0) {
+        goto cleanup;
+    }
+
+
+    /*
+     * We still don't know that the masterkey we got is valid,
+     * because we just blindly assumed the user's password
+     * was correct. This is where we now verify it. We are
+     * creating a hash of the master key using PBKDF and
+     * then comparing that to the hash stored in the key slot
+     * header
+     */
+    if (qcrypto_pbkdf2(hash,
+                       masterkey, masterkeylen,
+                       luks->header.master_key_salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.master_key_iterations,
+                       keydigest, G_N_ELEMENTS(keydigest),
+                       errp) < 0) {
+        goto cleanup;
+    }
+
+    if (memcmp(keydigest, luks->header.master_key_digest,
+               QCRYPTO_BLOCK_LUKS_DIGEST_LEN) == 0) {
+        /* Success, we got the right master key */
+        ret = 1;
+        goto cleanup;
+    }
+
+    /* Fail, user's password was not valid for this key slot,
+     * tell caller to try another slot */
+    ret = 0;
+
+ cleanup:
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+    g_free(splitkey);
+    g_free(possiblekey);
+    return ret;
+}
+
+
+/*
+ * Given a user password, this will iterate over all key
+ * slots and try to unlock each active key slot using the
+ * password until it successfully obtains a master key.
+ *
+ * Returns 0 if a key was loaded, -1 if no keys could be loaded
+ */
+static int
+qcrypto_block_luks_find_key(QCryptoBlock *block,
+                            const char *password,
+                            QCryptoCipherAlgorithm cipheralg,
+                            QCryptoCipherMode ciphermode,
+                            QCryptoHashAlgorithm hash,
+                            QCryptoIVGenAlgorithm ivalg,
+                            QCryptoCipherAlgorithm ivcipheralg,
+                            QCryptoHashAlgorithm ivhash,
+                            uint8_t **masterkey,
+                            size_t *masterkeylen,
+                            QCryptoBlockReadFunc readfunc,
+                            void *opaque,
+                            Error **errp)
+{
+    QCryptoBlockLUKS *luks = block->opaque;
+    size_t i;
+    int rv;
+
+    *masterkey = g_new0(uint8_t, luks->header.key_bytes);
+    *masterkeylen = luks->header.key_bytes;
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        rv = qcrypto_block_luks_load_key(block,
+                                         &luks->header.key_slots[i],
+                                         password,
+                                         cipheralg,
+                                         ciphermode,
+                                         hash,
+                                         ivalg,
+                                         ivcipheralg,
+                                         ivhash,
+                                         *masterkey,
+                                         *masterkeylen,
+                                         readfunc,
+                                         opaque,
+                                         errp);
+        if (rv < 0) {
+            goto error;
+        }
+        if (rv == 1) {
+            return 0;
+        }
+    }
+
+    error_setg(errp, "Invalid password, cannot unlock any keyslot");
+
+ error:
+    g_free(*masterkey);
+    *masterkey = NULL;
+    *masterkeylen = 0;
+    return -1;
+}
+
+
+static int
+qcrypto_block_luks_open(QCryptoBlock *block,
+                        QCryptoBlockOpenOptions *options,
+                        QCryptoBlockReadFunc readfunc,
+                        void *opaque,
+                        unsigned int flags,
+                        Error **errp)
+{
+    QCryptoBlockLUKS *luks;
+    Error *local_err = NULL;
+    int ret = 0;
+    size_t i;
+    ssize_t rv;
+    uint8_t *masterkey = NULL;
+    size_t masterkeylen;
+    char *ivgen_name, *ivhash_name;
+    QCryptoCipherMode ciphermode;
+    QCryptoCipherAlgorithm cipheralg;
+    QCryptoIVGenAlgorithm ivalg;
+    QCryptoCipherAlgorithm ivcipheralg;
+    QCryptoHashAlgorithm hash;
+    QCryptoHashAlgorithm ivhash;
+    char *password = NULL;
+
+    if (!(flags & QCRYPTO_BLOCK_OPEN_NO_IO)) {
+        if (!options->u.luks.key_secret) {
+            error_setg(errp, "Parameter 'key-secret' is required for cipher");
+            return -1;
+        }
+        password = qcrypto_secret_lookup_as_utf8(
+            options->u.luks.key_secret, errp);
+        if (!password) {
+            return -1;
+        }
+    }
+
+    luks = g_new0(QCryptoBlockLUKS, 1);
+    block->opaque = luks;
+
+    /* Read the entire LUKS header, minus the key material from
+     * the underlying device */
+    rv = readfunc(block, 0,
+                  (uint8_t *)&luks->header,
+                  sizeof(luks->header),
+                  errp,
+                  opaque);
+    if (rv < 0) {
+        ret = rv;
+        goto fail;
+    }
+
+    /* The header is always stored in big-endian format, so
+     * convert everything to native */
+    be16_to_cpus(&luks->header.version);
+    be32_to_cpus(&luks->header.payload_offset);
+    be32_to_cpus(&luks->header.key_bytes);
+    be32_to_cpus(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        be32_to_cpus(&luks->header.key_slots[i].active);
+        be32_to_cpus(&luks->header.key_slots[i].iterations);
+        be32_to_cpus(&luks->header.key_slots[i].key_offset);
+        be32_to_cpus(&luks->header.key_slots[i].stripes);
+    }
+
+    if (memcmp(luks->header.magic, qcrypto_block_luks_magic,
+               QCRYPTO_BLOCK_LUKS_MAGIC_LEN) != 0) {
+        error_setg(errp, "Volume is not in LUKS format");
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (luks->header.version != QCRYPTO_BLOCK_LUKS_VERSION) {
+        error_setg(errp, "LUKS version %" PRIu32 " is not supported",
+                   luks->header.version);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    /*
+     * The cipher_mode header contains a string that we have
+     * to further parse, of the format
+     *
+     *    <cipher-mode>-<iv-generator>[:<iv-hash>]
+     *
+     * eg  cbc-essiv:sha256, cbc-plain64
+     */
+    ivgen_name = strchr(luks->header.cipher_mode, '-');
+    if (!ivgen_name) {
+        ret = -EINVAL;
+        error_setg(errp, "Unexpected cipher mode string format %s",
+                   luks->header.cipher_mode);
+        goto fail;
+    }
+    *ivgen_name = '\0';
+    ivgen_name++;
+
+    ivhash_name = strchr(ivgen_name, ':');
+    if (!ivhash_name) {
+        ivhash = 0;
+    } else {
+        *ivhash_name = '\0';
+        ivhash_name++;
+
+        ivhash = qcrypto_block_luks_hash_name_lookup(ivhash_name,
+                                                     &local_err);
+        if (local_err) {
+            ret = -ENOTSUP;
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+    }
+
+    ciphermode = qcrypto_block_luks_cipher_mode_lookup(luks->header.cipher_mode,
+                                                       &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    cipheralg = qcrypto_block_luks_cipher_name_lookup(luks->header.cipher_name,
+                                                      ciphermode,
+                                                      luks->header.key_bytes,
+                                                      &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    hash = qcrypto_block_luks_hash_name_lookup(luks->header.hash_spec,
+                                               &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    ivalg = qcrypto_block_luks_ivgen_name_lookup(ivgen_name,
+                                                 &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    if (ivalg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        ivcipheralg = qcrypto_block_luks_essiv_cipher(cipheralg,
+                                                      ivhash,
+                                                      &local_err);
+        if (local_err) {
+            ret = -ENOTSUP;
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+    } else {
+        ivcipheralg = cipheralg;
+    }
+
+    if (!(flags & QCRYPTO_BLOCK_OPEN_NO_IO)) {
+        /* Try to find which key slot our password is valid for
+         * and unlock the master key from that slot.
+         */
+        if (qcrypto_block_luks_find_key(block,
+                                        password,
+                                        cipheralg, ciphermode,
+                                        hash,
+                                        ivalg,
+                                        ivcipheralg,
+                                        ivhash,
+                                        &masterkey, &masterkeylen,
+                                        readfunc, opaque,
+                                        errp) < 0) {
+            ret = -EACCES;
+            goto fail;
+        }
+
+        /* We have a valid master key now, so can setup the
+         * block device payload decryption objects
+         */
+        block->kdfhash = hash;
+        block->niv = qcrypto_cipher_get_iv_len(cipheralg,
+                                               ciphermode);
+        block->ivgen = qcrypto_ivgen_new(ivalg,
+                                         ivcipheralg,
+                                         ivhash,
+                                         masterkey, masterkeylen,
+                                         errp);
+        if (!block->ivgen) {
+            ret = -ENOTSUP;
+            goto fail;
+        }
+
+        block->cipher = qcrypto_cipher_new(cipheralg,
+                                           ciphermode,
+                                           masterkey, masterkeylen,
+                                           errp);
+        if (!block->cipher) {
+            ret = -ENOTSUP;
+            goto fail;
+        }
+    }
+
+    block->payload_offset = luks->header.payload_offset *
+        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
+
+    g_free(masterkey);
+    g_free(password);
+
+    return 0;
+
+ fail:
+    g_free(masterkey);
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    g_free(luks);
+    g_free(password);
+    return ret;
+}
+
+
+static int
+qcrypto_block_luks_uuid_gen(uint8_t *uuidstr, Error **errp)
+{
+#ifdef CONFIG_UUID
+    uuid_t uuid;
+    uuid_generate(uuid);
+    uuid_unparse(uuid, (char *)uuidstr);
+    return 0;
+#else
+    error_setg(errp, "Unable to generate uuids on this platform");
+    return -1;
+#endif
+}
+
+static int
+qcrypto_block_luks_create(QCryptoBlock *block,
+                          QCryptoBlockCreateOptions *options,
+                          QCryptoBlockInitFunc initfunc,
+                          QCryptoBlockWriteFunc writefunc,
+                          void *opaque,
+                          Error **errp)
+{
+    QCryptoBlockLUKS *luks;
+    QCryptoBlockCreateOptionsLUKS luks_opts;
+    Error *local_err = NULL;
+    uint8_t *masterkey = NULL;
+    uint8_t *slotkey = NULL;
+    uint8_t *splitkey = NULL;
+    size_t splitkeylen = 0;
+    size_t i;
+    QCryptoCipher *cipher = NULL;
+    QCryptoIVGen *ivgen = NULL;
+    char *password;
+    const char *cipher_alg;
+    const char *cipher_mode;
+    const char *ivgen_alg;
+    const char *ivgen_hash_alg = NULL;
+    const char *hash_alg;
+    char *cipher_mode_spec = NULL;
+    QCryptoCipherAlgorithm ivcipheralg = 0;
+
+    memcpy(&luks_opts, &options->u.luks, sizeof(luks_opts));
+    if (!luks_opts.has_cipher_alg) {
+        luks_opts.cipher_alg = QCRYPTO_CIPHER_ALG_AES_256;
+    }
+    if (!luks_opts.has_cipher_mode) {
+        luks_opts.cipher_mode = QCRYPTO_CIPHER_MODE_XTS;
+    }
+    if (!luks_opts.has_ivgen_alg) {
+        luks_opts.ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64;
+    }
+    if (!luks_opts.has_hash_alg) {
+        luks_opts.hash_alg = QCRYPTO_HASH_ALG_SHA256;
+    }
+
+    if (!options->u.luks.key_secret) {
+        error_setg(errp, "Parameter 'key-secret' is required for cipher");
+        return -1;
+    }
+    password = qcrypto_secret_lookup_as_utf8(luks_opts.key_secret, errp);
+    if (!password) {
+        return -1;
+    }
+
+    luks = g_new0(QCryptoBlockLUKS, 1);
+    block->opaque = luks;
+
+    memcpy(luks->header.magic, qcrypto_block_luks_magic,
+           QCRYPTO_BLOCK_LUKS_MAGIC_LEN);
+
+    /* We populate the header in native endianness initially and
+     * then convert everything to big endian just before writing
+     * it out to disk
+     */
+    luks->header.version = QCRYPTO_BLOCK_LUKS_VERSION;
+    if (qcrypto_block_luks_uuid_gen(luks->header.uuid,
+                                    errp) < 0) {
+        goto error;
+    }
+
+    cipher_alg = qcrypto_block_luks_cipher_alg_lookup(luks_opts.cipher_alg,
+                                                      errp);
+    if (!cipher_alg) {
+        goto error;
+    }
+
+    cipher_mode = QCryptoCipherMode_lookup[luks_opts.cipher_mode];
+    ivgen_alg = QCryptoIVGenAlgorithm_lookup[luks_opts.ivgen_alg];
+    if (luks_opts.has_ivgen_hash_alg) {
+        ivgen_hash_alg = QCryptoHashAlgorithm_lookup[luks_opts.ivgen_hash_alg];
+        cipher_mode_spec = g_strdup_printf("%s-%s:%s", cipher_mode, ivgen_alg,
+                                           ivgen_hash_alg);
+    } else {
+        cipher_mode_spec = g_strdup_printf("%s-%s", cipher_mode, ivgen_alg);
+    }
+    hash_alg = QCryptoHashAlgorithm_lookup[luks_opts.hash_alg];
+
+
+    if (strlen(cipher_alg) >= QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN) {
+        error_setg(errp, "Cipher name '%s' is too long for LUKS header",
+                   cipher_alg);
+        goto error;
+    }
+    if (strlen(cipher_mode_spec) >= QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN) {
+        error_setg(errp, "Cipher mode '%s' is too long for LUKS header",
+                   cipher_mode_spec);
+        goto error;
+    }
+    if (strlen(hash_alg) >= QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN) {
+        error_setg(errp, "Hash name '%s' is too long for LUKS header",
+                   hash_alg);
+        goto error;
+    }
+
+    if (luks_opts.ivgen_alg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        ivcipheralg = qcrypto_block_luks_essiv_cipher(luks_opts.cipher_alg,
+                                                      luks_opts.ivgen_hash_alg,
+                                                      &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            goto error;
+        }
+    } else {
+        ivcipheralg = luks_opts.cipher_alg;
+    }
+
+    strcpy(luks->header.cipher_name, cipher_alg);
+    strcpy(luks->header.cipher_mode, cipher_mode_spec);
+    strcpy(luks->header.hash_spec, hash_alg);
+
+    luks->header.key_bytes = qcrypto_cipher_get_key_len(luks_opts.cipher_alg);
+    if (luks_opts.cipher_mode == QCRYPTO_CIPHER_MODE_XTS) {
+        luks->header.key_bytes *= 2;
+    }
+
+    /* Generate the salt used for hashing the master key
+     * with PBKDF later
+     */
+    if (qcrypto_random_bytes(luks->header.master_key_salt,
+                             QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                             errp) < 0) {
+        goto error;
+    }
+
+    /* Generate random master key */
+    masterkey = g_new0(uint8_t, luks->header.key_bytes);
+    if (qcrypto_random_bytes(masterkey,
+                             luks->header.key_bytes, errp) < 0) {
+        goto error;
+    }
+
+
+    /* Setup the block device payload encryption objects */
+    block->cipher = qcrypto_cipher_new(luks_opts.cipher_alg,
+                                       luks_opts.cipher_mode,
+                                       masterkey, luks->header.key_bytes,
+                                       errp);
+    if (!block->cipher) {
+        goto error;
+    }
+
+    block->kdfhash = luks_opts.hash_alg;
+    block->niv = qcrypto_cipher_get_iv_len(luks_opts.cipher_alg,
+                                           luks_opts.cipher_mode);
+    block->ivgen = qcrypto_ivgen_new(luks_opts.ivgen_alg,
+                                     ivcipheralg,
+                                     luks_opts.ivgen_hash_alg,
+                                     masterkey, luks->header.key_bytes,
+                                     errp);
+
+    if (!block->ivgen) {
+        goto error;
+    }
+
+
+    /* Determine how many iterations we need to hash the master
+     * key, in order to have 1 second of compute time used
+     */
+    luks->header.master_key_iterations =
+        qcrypto_pbkdf2_count_iters(luks_opts.hash_alg,
+                                   masterkey, luks->header.key_bytes,
+                                   luks->header.master_key_salt,
+                                   QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                                   &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Why /= 8 ?  That matches cryptsetup, but there's no
+     * explanation why they chose /= 8... Probably so that
+     * if all 8 keyslots are active we only spend 1 second
+     * in total time to check all keys */
+    luks->header.master_key_iterations /= 8;
+    luks->header.master_key_iterations = MAX(
+        luks->header.master_key_iterations,
+        QCRYPTO_BLOCK_LUKS_MIN_MASTER_KEY_ITERS);
+
+
+    /* Hash the master key, saving the result in the LUKS
+     * header. This hash is used when opening the encrypted
+     * device to verify that the user password unlocked a
+     * valid master key
+     */
+    if (qcrypto_pbkdf2(luks_opts.hash_alg,
+                       masterkey, luks->header.key_bytes,
+                       luks->header.master_key_salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.master_key_iterations,
+                       luks->header.master_key_digest,
+                       QCRYPTO_BLOCK_LUKS_DIGEST_LEN,
+                       errp) < 0) {
+        goto error;
+    }
+
+
+    /* Although LUKS has multiple key slots, we're just going
+     * to use the first key slot */
+    splitkeylen = luks->header.key_bytes * QCRYPTO_BLOCK_LUKS_STRIPES;
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        luks->header.key_slots[i].active = i == 0 ?
+            QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED :
+            QCRYPTO_BLOCK_LUKS_KEY_SLOT_DISABLED;
+        luks->header.key_slots[i].stripes = QCRYPTO_BLOCK_LUKS_STRIPES;
+
+        /* This calculation doesn't match that shown in the spec,
+         * but instead follows the cryptsetup implementation.
+         */
+        luks->header.key_slots[i].key_offset =
+            (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+             QCRYPTO_BLOCK_LUKS_SECTOR_SIZE) +
+            (ROUND_UP(((splitkeylen + (QCRYPTO_BLOCK_LUKS_SECTOR_SIZE - 1)) /
+                       QCRYPTO_BLOCK_LUKS_SECTOR_SIZE),
+                      (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+                       QCRYPTO_BLOCK_LUKS_SECTOR_SIZE)) * i);
+    }
+
+    if (qcrypto_random_bytes(luks->header.key_slots[0].salt,
+                             QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                             errp) < 0) {
+        goto error;
+    }
+
+    /* Again we determine how many iterations are required to
+     * hash the user password while consuming 1 second of compute
+     * time */
+    luks->header.key_slots[0].iterations =
+        qcrypto_pbkdf2_count_iters(luks_opts.hash_alg,
+                                   (uint8_t *)password, strlen(password),
+                                   luks->header.key_slots[0].salt,
+                                   QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                                   &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+    /* Why /= 2 ?  That matches cryptsetup, but there's no
+     * explanation why they chose /= 2... */
+    luks->header.key_slots[0].iterations /= 2;
+    luks->header.key_slots[0].iterations = MAX(
+        luks->header.key_slots[0].iterations,
+        QCRYPTO_BLOCK_LUKS_MIN_SLOT_KEY_ITERS);
+
+
+    /* Generate a key that we'll use to encrypt the master
+     * key, from the user's password
+     */
+    slotkey = g_new0(uint8_t, luks->header.key_bytes);
+    if (qcrypto_pbkdf2(luks_opts.hash_alg,
+                       (uint8_t *)password, strlen(password),
+                       luks->header.key_slots[0].salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.key_slots[0].iterations,
+                       slotkey, luks->header.key_bytes,
+                       errp) < 0) {
+        goto error;
+    }
+
+
+    /* Setup the encryption objects needed to encrypt the
+     * master key material
+     */
+    cipher = qcrypto_cipher_new(luks_opts.cipher_alg,
+                                luks_opts.cipher_mode,
+                                slotkey, luks->header.key_bytes,
+                                errp);
+    if (!cipher) {
+        goto error;
+    }
+
+    ivgen = qcrypto_ivgen_new(luks_opts.ivgen_alg,
+                              ivcipheralg,
+                              luks_opts.ivgen_hash_alg,
+                              slotkey, luks->header.key_bytes,
+                              errp);
+    if (!ivgen) {
+        goto error;
+    }
+
+    /* Before storing the master key, we need to vastly
+     * increase its size, as protection against forensic
+     * disk data recovery */
+    splitkey = g_new0(uint8_t, splitkeylen);
+
+    if (qcrypto_afsplit_encode(luks_opts.hash_alg,
+                               luks->header.key_bytes,
+                               luks->header.key_slots[0].stripes,
+                               masterkey,
+                               splitkey,
+                               errp) < 0) {
+        goto error;
+    }
+
+    /* Now we encrypt the split master key with the key generated
+     * from the user's password, before storing it */
+    if (qcrypto_block_encrypt_helper(cipher, block->niv, ivgen,
+                                     QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                     0,
+                                     splitkey,
+                                     splitkeylen,
+                                     errp) < 0) {
+        goto error;
+    }
+
+
+    /* The total size of the LUKS headers is the partition header + key
+     * slot headers, rounded up to the nearest sector, combined with
+     * the size of each master key material region, also rounded up
+     * to the nearest sector */
+    luks->header.payload_offset =
+        (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+         QCRYPTO_BLOCK_LUKS_SECTOR_SIZE) +
+        (ROUND_UP(((splitkeylen + (QCRYPTO_BLOCK_LUKS_SECTOR_SIZE - 1)) /
+                   QCRYPTO_BLOCK_LUKS_SECTOR_SIZE),
+                  (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+                   QCRYPTO_BLOCK_LUKS_SECTOR_SIZE)) *
+         QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS);
+
+    block->payload_offset = luks->header.payload_offset *
+        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
+
+    /* Reserve header space to match payload offset */
+    initfunc(block, block->payload_offset, &local_err, opaque);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Everything on disk uses Big Endian, so flip header fields
+     * before writing them */
+    cpu_to_be16s(&luks->header.version);
+    cpu_to_be32s(&luks->header.payload_offset);
+    cpu_to_be32s(&luks->header.key_bytes);
+    cpu_to_be32s(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        cpu_to_be32s(&luks->header.key_slots[i].active);
+        cpu_to_be32s(&luks->header.key_slots[i].iterations);
+        cpu_to_be32s(&luks->header.key_slots[i].key_offset);
+        cpu_to_be32s(&luks->header.key_slots[i].stripes);
+    }
+
+
+    /* Write out the partition header and key slot headers */
+    writefunc(block, 0,
+              (const uint8_t *)&luks->header,
+              sizeof(luks->header),
+              &local_err,
+              opaque);
+
+    /* Delay checking local_err until we've byte-swapped */
+
+    /* Byte swap the header back to native, in case we need
+     * to read it again later */
+    be16_to_cpus(&luks->header.version);
+    be32_to_cpus(&luks->header.payload_offset);
+    be32_to_cpus(&luks->header.key_bytes);
+    be32_to_cpus(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        be32_to_cpus(&luks->header.key_slots[i].active);
+        be32_to_cpus(&luks->header.key_slots[i].iterations);
+        be32_to_cpus(&luks->header.key_slots[i].key_offset);
+        be32_to_cpus(&luks->header.key_slots[i].stripes);
+    }
+
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Write out the master key material, starting at the
+     * sector immediately following the partition header. */
+    if (writefunc(block,
+                  luks->header.key_slots[0].key_offset *
+                  QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                  splitkey, splitkeylen,
+                  errp,
+                  opaque) != splitkeylen) {
+        goto error;
+    }
+
+    memset(masterkey, 0, luks->header.key_bytes);
+    g_free(masterkey);
+    memset(slotkey, 0, luks->header.key_bytes);
+    g_free(slotkey);
+    g_free(splitkey);
+    g_free(password);
+    g_free(cipher_mode_spec);
+
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+
+    return 0;
+
+ error:
+    if (masterkey) {
+        memset(masterkey, 0, luks->header.key_bytes);
+    }
+    g_free(masterkey);
+    if (slotkey) {
+        memset(slotkey, 0, luks->header.key_bytes);
+    }
+    g_free(slotkey);
+    g_free(splitkey);
+    g_free(password);
+    g_free(cipher_mode_spec);
+
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+
+    g_free(luks);
+    return -1;
+}
+
+
+static void qcrypto_block_luks_cleanup(QCryptoBlock *block)
+{
+    g_free(block->opaque);
+}
+
+
+static int
+qcrypto_block_luks_decrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_decrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+static int
+qcrypto_block_luks_encrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_encrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+const QCryptoBlockDriver qcrypto_block_driver_luks = {
+    .open = qcrypto_block_luks_open,
+    .create = qcrypto_block_luks_create,
+    .cleanup = qcrypto_block_luks_cleanup,
+    .decrypt = qcrypto_block_luks_decrypt,
+    .encrypt = qcrypto_block_luks_encrypt,
+    .has_format = qcrypto_block_luks_has_format,
+};
diff --git a/qemu/crypto/block-luks.h b/qemu/crypto/block-luks.h
new file mode 100644
index 000000000..0934138aa
--- /dev/null
+++ b/qemu/crypto/block-luks.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block device encryption LUKS format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_LUKS_H__
+#define QCRYPTO_BLOCK_LUKS_H__
+
+#include "crypto/blockpriv.h"
+
+extern const QCryptoBlockDriver qcrypto_block_driver_luks;
+
+#endif /* QCRYPTO_BLOCK_LUKS_H__ */
diff --git a/qemu/crypto/block-qcow.c b/qemu/crypto/block-qcow.c
new file mode 100644
index 000000000..be88c6f0e
--- /dev/null
+++ b/qemu/crypto/block-qcow.c
@@ -0,0 +1,174 @@
+/*
+ * QEMU Crypto block device encryption QCow/QCow2 AES-CBC format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * Note that the block encryption implemented in this file is broken
+ * by design. This exists only to allow data to be liberated from
+ * existing qcow[2] images and should not be used in any new areas.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+
+#include "crypto/block-qcow.h"
+#include "crypto/secret.h"
+
+#define QCRYPTO_BLOCK_QCOW_SECTOR_SIZE 512
+
+
+static bool
+qcrypto_block_qcow_has_format(const uint8_t *buf G_GNUC_UNUSED,
+                              size_t buf_size G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+static int
+qcrypto_block_qcow_init(QCryptoBlock *block,
+                        const char *keysecret,
+                        Error **errp)
+{
+    char *password;
+    int ret;
+    uint8_t keybuf[16];
+    int len;
+
+    memset(keybuf, 0, 16);
+
+    password = qcrypto_secret_lookup_as_utf8(keysecret, errp);
+    if (!password) {
+        return -1;
+    }
+
+    len = strlen(password);
+    memcpy(keybuf, password, MIN(len, sizeof(keybuf)));
+    g_free(password);
+
+    block->niv = qcrypto_cipher_get_iv_len(QCRYPTO_CIPHER_ALG_AES_128,
+                                           QCRYPTO_CIPHER_MODE_CBC);
+    block->ivgen = qcrypto_ivgen_new(QCRYPTO_IVGEN_ALG_PLAIN64,
+                                     0, 0, NULL, 0, errp);
+    if (!block->ivgen) {
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    block->cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+                                       QCRYPTO_CIPHER_MODE_CBC,
+                                       keybuf, G_N_ELEMENTS(keybuf),
+                                       errp);
+    if (!block->cipher) {
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    block->payload_offset = 0;
+
+    return 0;
+
+ fail:
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    return ret;
+}
+
+
+static int
+qcrypto_block_qcow_open(QCryptoBlock *block,
+                        QCryptoBlockOpenOptions *options,
+                        QCryptoBlockReadFunc readfunc G_GNUC_UNUSED,
+                        void *opaque G_GNUC_UNUSED,
+                        unsigned int flags,
+                        Error **errp)
+{
+    if (flags & QCRYPTO_BLOCK_OPEN_NO_IO) {
+        return 0;
+    } else {
+        if (!options->u.qcow.key_secret) {
+            error_setg(errp,
+                       "Parameter 'key-secret' is required for cipher");
+            return -1;
+        }
+        return qcrypto_block_qcow_init(block,
+                                       options->u.qcow.key_secret, errp);
+    }
+}
+
+
+static int
+qcrypto_block_qcow_create(QCryptoBlock *block,
+                          QCryptoBlockCreateOptions *options,
+                          QCryptoBlockInitFunc initfunc G_GNUC_UNUSED,
+                          QCryptoBlockWriteFunc writefunc G_GNUC_UNUSED,
+                          void *opaque G_GNUC_UNUSED,
+                          Error **errp)
+{
+    if (!options->u.qcow.key_secret) {
+        error_setg(errp, "Parameter 'key-secret' is required for cipher");
+        return -1;
+    }
+    /* QCow2 has no special header, since everything is hardwired */
+    return qcrypto_block_qcow_init(block, options->u.qcow.key_secret, errp);
+}
+
+
+static void
+qcrypto_block_qcow_cleanup(QCryptoBlock *block)
+{
+}
+
+
+static int
+qcrypto_block_qcow_decrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_decrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_QCOW_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+static int
+qcrypto_block_qcow_encrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_encrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_QCOW_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+const QCryptoBlockDriver qcrypto_block_driver_qcow = {
+    .open = qcrypto_block_qcow_open,
+    .create = qcrypto_block_qcow_create,
+    .cleanup = qcrypto_block_qcow_cleanup,
+    .decrypt = qcrypto_block_qcow_decrypt,
+    .encrypt = qcrypto_block_qcow_encrypt,
+    .has_format = qcrypto_block_qcow_has_format,
+};
diff --git a/qemu/crypto/block-qcow.h b/qemu/crypto/block-qcow.h
new file mode 100644
index 000000000..569f83610
--- /dev/null
+++ b/qemu/crypto/block-qcow.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block device encryption QCow/QCow2 AES-CBC format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_QCOW_H__
+#define QCRYPTO_BLOCK_QCOW_H__
+
+#include "crypto/blockpriv.h"
+
+extern const QCryptoBlockDriver qcrypto_block_driver_qcow;
+
+#endif /* QCRYPTO_BLOCK_QCOW_H__ */
diff --git a/qemu/crypto/block.c b/qemu/crypto/block.c
new file mode 100644
index 000000000..da60eba85
--- /dev/null
+++ b/qemu/crypto/block.c
@@ -0,0 +1,261 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/blockpriv.h"
+#include "crypto/block-qcow.h"
+#include "crypto/block-luks.h"
+
+static const QCryptoBlockDriver *qcrypto_block_drivers[] = {
+    [Q_CRYPTO_BLOCK_FORMAT_QCOW] = &qcrypto_block_driver_qcow,
+    [Q_CRYPTO_BLOCK_FORMAT_LUKS] = &qcrypto_block_driver_luks,
+};
+
+
+bool qcrypto_block_has_format(QCryptoBlockFormat format,
+                              const uint8_t *buf,
+                              size_t len)
+{
+    const QCryptoBlockDriver *driver;
+
+    if (format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[format]) {
+        return false;
+    }
+
+    driver = qcrypto_block_drivers[format];
+
+    return driver->has_format(buf, len);
+}
+
+
+QCryptoBlock *qcrypto_block_open(QCryptoBlockOpenOptions *options,
+                                 QCryptoBlockReadFunc readfunc,
+                                 void *opaque,
+                                 unsigned int flags,
+                                 Error **errp)
+{
+    QCryptoBlock *block = g_new0(QCryptoBlock, 1);
+
+    block->format = options->format;
+
+    if (options->format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[options->format]) {
+        error_setg(errp, "Unsupported block driver %d", options->format);
+        g_free(block);
+        return NULL;
+    }
+
+    block->driver = qcrypto_block_drivers[options->format];
+
+    if (block->driver->open(block, options,
+                            readfunc, opaque, flags, errp) < 0) {
+        g_free(block);
+        return NULL;
+    }
+
+    return block;
+}
+
+
+QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
+                                   QCryptoBlockInitFunc initfunc,
+                                   QCryptoBlockWriteFunc writefunc,
+                                   void *opaque,
+                                   Error **errp)
+{
+    QCryptoBlock *block = g_new0(QCryptoBlock, 1);
+
+    block->format = options->format;
+
+    if (options->format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[options->format]) {
+        error_setg(errp, "Unsupported block driver %d", options->format);
+        g_free(block);
+        return NULL;
+    }
+
+    block->driver = qcrypto_block_drivers[options->format];
+
+    if (block->driver->create(block, options, initfunc,
+                              writefunc, opaque, errp) < 0) {
+        g_free(block);
+        return NULL;
+    }
+
+    return block;
+}
+
+
+int qcrypto_block_decrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp)
+{
+    return block->driver->decrypt(block, startsector, buf, len, errp);
+}
+
+
+int qcrypto_block_encrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp)
+{
+    return block->driver->encrypt(block, startsector, buf, len, errp);
+}
+
+
+QCryptoCipher *qcrypto_block_get_cipher(QCryptoBlock *block)
+{
+    return block->cipher;
+}
+
+
+QCryptoIVGen *qcrypto_block_get_ivgen(QCryptoBlock *block)
+{
+    return block->ivgen;
+}
+
+
+QCryptoHashAlgorithm qcrypto_block_get_kdf_hash(QCryptoBlock *block)
+{
+    return block->kdfhash;
+}
+
+
+uint64_t qcrypto_block_get_payload_offset(QCryptoBlock *block)
+{
+    return block->payload_offset;
+}
+
+
+void qcrypto_block_free(QCryptoBlock *block)
+{
+    if (!block) {
+        return;
+    }
+
+    block->driver->cleanup(block);
+
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    g_free(block);
+}
+
+
+int qcrypto_block_decrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp)
+{
+    uint8_t *iv;
+    int ret = -1;
+
+    iv = niv ? g_new0(uint8_t, niv) : NULL;
+
+    while (len > 0) {
+        size_t nbytes;
+        if (niv) {
+            if (qcrypto_ivgen_calculate(ivgen,
+                                        startsector,
+                                        iv, niv,
+                                        errp) < 0) {
+                goto cleanup;
+            }
+
+            if (qcrypto_cipher_setiv(cipher,
+                                     iv, niv,
+                                     errp) < 0) {
+                goto cleanup;
+            }
+        }
+
+        nbytes = len > sectorsize ? sectorsize : len;
+        if (qcrypto_cipher_decrypt(cipher, buf, buf,
+                                   nbytes, errp) < 0) {
+            goto cleanup;
+        }
+
+        startsector++;
+        buf += nbytes;
+        len -= nbytes;
+    }
+
+    ret = 0;
+ cleanup:
+    g_free(iv);
+    return ret;
+}
+
+
+int qcrypto_block_encrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp)
+{
+    uint8_t *iv;
+    int ret = -1;
+
+    iv = niv ? g_new0(uint8_t, niv) : NULL;
+
+    while (len > 0) {
+        size_t nbytes;
+        if (niv) {
+            if (qcrypto_ivgen_calculate(ivgen,
+                                        startsector,
+                                        iv, niv,
+                                        errp) < 0) {
+                goto cleanup;
+            }
+
+            if (qcrypto_cipher_setiv(cipher,
+                                     iv, niv,
+                                     errp) < 0) {
+                goto cleanup;
+            }
+        }
+
+        nbytes = len > sectorsize ? sectorsize : len;
+        if (qcrypto_cipher_encrypt(cipher, buf, buf,
+                                   nbytes, errp) < 0) {
+            goto cleanup;
+        }
+
+        startsector++;
+        buf += nbytes;
+        len -= nbytes;
+    }
+
+    ret = 0;
+ cleanup:
+    g_free(iv);
+    return ret;
+}
diff --git a/qemu/crypto/blockpriv.h b/qemu/crypto/blockpriv.h
new file mode 100644
index 000000000..62970859d
--- /dev/null
+++ b/qemu/crypto/blockpriv.h
@@ -0,0 +1,92 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_PRIV_H__
+#define QCRYPTO_BLOCK_PRIV_H__
+
+#include "crypto/block.h"
+
+typedef struct QCryptoBlockDriver QCryptoBlockDriver;
+
+struct QCryptoBlock {
+    QCryptoBlockFormat format;
+
+    const QCryptoBlockDriver *driver;
+    void *opaque;
+
+    QCryptoCipher *cipher;
+    QCryptoIVGen *ivgen;
+    QCryptoHashAlgorithm kdfhash;
+    size_t niv;
+    uint64_t payload_offset; /* In bytes */
+};
+
+struct QCryptoBlockDriver {
+    int (*open)(QCryptoBlock *block,
+                QCryptoBlockOpenOptions *options,
+                QCryptoBlockReadFunc readfunc,
+                void *opaque,
+                unsigned int flags,
+                Error **errp);
+
+    int (*create)(QCryptoBlock *block,
+                  QCryptoBlockCreateOptions *options,
+                  QCryptoBlockInitFunc initfunc,
+                  QCryptoBlockWriteFunc writefunc,
+                  void *opaque,
+                  Error **errp);
+
+    void (*cleanup)(QCryptoBlock *block);
+
+    int (*encrypt)(QCryptoBlock *block,
+                   uint64_t startsector,
+                   uint8_t *buf,
+                   size_t len,
+                   Error **errp);
+    int (*decrypt)(QCryptoBlock *block,
+                   uint64_t startsector,
+                   uint8_t *buf,
+                   size_t len,
+                   Error **errp);
+
+    bool (*has_format)(const uint8_t *buf,
+                       size_t buflen);
+};
+
+
+int qcrypto_block_decrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp);
+
+int qcrypto_block_encrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp);
+
+#endif /* QCRYPTO_BLOCK_PRIV_H__ */
diff --git a/qemu/crypto/cipher-builtin.c b/qemu/crypto/cipher-builtin.c
index 30f4853c8..88963f65c 100644
--- a/qemu/crypto/cipher-builtin.c
+++ b/qemu/crypto/cipher-builtin.c
@@ -18,15 +18,21 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "crypto/aes.h"
 #include "crypto/desrfb.h"
+#include "crypto/xts.h"
 
+typedef struct QCryptoCipherBuiltinAESContext QCryptoCipherBuiltinAESContext;
+struct QCryptoCipherBuiltinAESContext {
+    AES_KEY enc;
+    AES_KEY dec;
+};
 typedef struct QCryptoCipherBuiltinAES QCryptoCipherBuiltinAES;
 struct QCryptoCipherBuiltinAES {
-    AES_KEY encrypt_key;
-    AES_KEY decrypt_key;
-    uint8_t *iv;
-    size_t niv;
+    QCryptoCipherBuiltinAESContext key;
+    QCryptoCipherBuiltinAESContext key_tweak;
+    uint8_t iv[AES_BLOCK_SIZE];
 };
 typedef struct QCryptoCipherBuiltinDESRFB QCryptoCipherBuiltinDESRFB;
 struct QCryptoCipherBuiltinDESRFB {
@@ -40,6 +46,7 @@ struct QCryptoCipherBuiltin {
         QCryptoCipherBuiltinAES aes;
         QCryptoCipherBuiltinDESRFB desrfb;
     } state;
+    size_t blocksize;
     void (*free)(QCryptoCipher *cipher);
     int (*setiv)(QCryptoCipher *cipher,
                  const uint8_t *iv, size_t niv,
@@ -61,12 +68,87 @@ static void qcrypto_cipher_free_aes(QCryptoCipher *cipher)
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    g_free(ctxt->state.aes.iv);
     g_free(ctxt);
     cipher->opaque = NULL;
 }
 
 
+static void qcrypto_cipher_aes_ecb_encrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_encrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_encrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
+static void qcrypto_cipher_aes_ecb_decrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_decrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_decrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
+static void qcrypto_cipher_aes_xts_encrypt(const void *ctx,
+                                           size_t length,
+                                           uint8_t *dst,
+                                           const uint8_t *src)
+{
+    const QCryptoCipherBuiltinAESContext *aesctx = ctx;
+
+    qcrypto_cipher_aes_ecb_encrypt((AES_KEY *)&aesctx->enc,
+                                   src, dst, length);
+}
+
+
+static void qcrypto_cipher_aes_xts_decrypt(const void *ctx,
+                                           size_t length,
+                                           uint8_t *dst,
+                                           const uint8_t *src)
+{
+    const QCryptoCipherBuiltinAESContext *aesctx = ctx;
+
+    qcrypto_cipher_aes_ecb_decrypt((AES_KEY *)&aesctx->dec,
+                                   src, dst, length);
+}
+
+
 static int qcrypto_cipher_encrypt_aes(QCryptoCipher *cipher,
                                       const void *in,
                                       void *out,
@@ -75,29 +157,26 @@ static int qcrypto_cipher_encrypt_aes(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    if (cipher->mode == QCRYPTO_CIPHER_MODE_ECB) {
-        const uint8_t *inptr = in;
-        uint8_t *outptr = out;
-        while (len) {
-            if (len > AES_BLOCK_SIZE) {
-                AES_encrypt(inptr, outptr, &ctxt->state.aes.encrypt_key);
-                inptr += AES_BLOCK_SIZE;
-                outptr += AES_BLOCK_SIZE;
-                len -= AES_BLOCK_SIZE;
-            } else {
-                uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
-                memcpy(tmp1, inptr, len);
-                /* Fill with 0 to avoid valgrind uninitialized reads */
-                memset(tmp1 + len, 0, sizeof(tmp1) - len);
-                AES_encrypt(tmp1, tmp2, &ctxt->state.aes.encrypt_key);
-                memcpy(outptr, tmp2, len);
-                len = 0;
-            }
-        }
-    } else {
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_encrypt(&ctxt->state.aes.key.enc,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
         AES_cbc_encrypt(in, out, len,
-                        &ctxt->state.aes.encrypt_key,
+                        &ctxt->state.aes.key.enc,
                         ctxt->state.aes.iv, 1);
+        break;
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_encrypt(&ctxt->state.aes.key,
+                    &ctxt->state.aes.key_tweak,
+                    qcrypto_cipher_aes_xts_encrypt,
+                    qcrypto_cipher_aes_xts_decrypt,
+                    ctxt->state.aes.iv,
+                    len, out, in);
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     return 0;
@@ -112,29 +191,26 @@ static int qcrypto_cipher_decrypt_aes(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
-    if (cipher->mode == QCRYPTO_CIPHER_MODE_ECB) {
-        const uint8_t *inptr = in;
-        uint8_t *outptr = out;
-        while (len) {
-            if (len > AES_BLOCK_SIZE) {
-                AES_decrypt(inptr, outptr, &ctxt->state.aes.decrypt_key);
-                inptr += AES_BLOCK_SIZE;
-                outptr += AES_BLOCK_SIZE;
-                len -= AES_BLOCK_SIZE;
-            } else {
-                uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
-                memcpy(tmp1, inptr, len);
-                /* Fill with 0 to avoid valgrind uninitialized reads */
-                memset(tmp1 + len, 0, sizeof(tmp1) - len);
-                AES_decrypt(tmp1, tmp2, &ctxt->state.aes.decrypt_key);
-                memcpy(outptr, tmp2, len);
-                len = 0;
-            }
-        }
-    } else {
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_decrypt(&ctxt->state.aes.key.dec,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
         AES_cbc_encrypt(in, out, len,
-                        &ctxt->state.aes.decrypt_key,
+                        &ctxt->state.aes.key.dec,
                         ctxt->state.aes.iv, 0);
+        break;
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_decrypt(&ctxt->state.aes.key,
+                    &ctxt->state.aes.key_tweak,
+                    qcrypto_cipher_aes_xts_encrypt,
+                    qcrypto_cipher_aes_xts_decrypt,
+                    ctxt->state.aes.iv,
+                    len, out, in);
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     return 0;
@@ -145,15 +221,13 @@ static int qcrypto_cipher_setiv_aes(QCryptoCipher *cipher,
                                      Error **errp)
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
-    if (niv != 16) {
-        error_setg(errp, "IV must be 16 bytes not %zu", niv);
+    if (niv != AES_BLOCK_SIZE) {
+        error_setg(errp, "IV must be %d bytes not %zu",
+                   AES_BLOCK_SIZE, niv);
         return -1;
     }
 
-    g_free(ctxt->state.aes.iv);
-    ctxt->state.aes.iv = g_new0(uint8_t, niv);
-    memcpy(ctxt->state.aes.iv, iv, niv);
-    ctxt->state.aes.niv = niv;
+    memcpy(ctxt->state.aes.iv, iv, AES_BLOCK_SIZE);
 
     return 0;
 }
@@ -168,23 +242,49 @@ static int qcrypto_cipher_init_aes(QCryptoCipher *cipher,
     QCryptoCipherBuiltin *ctxt;
 
     if (cipher->mode != QCRYPTO_CIPHER_MODE_CBC &&
-        cipher->mode != QCRYPTO_CIPHER_MODE_ECB) {
+        cipher->mode != QCRYPTO_CIPHER_MODE_ECB &&
+        cipher->mode != QCRYPTO_CIPHER_MODE_XTS) {
         error_setg(errp, "Unsupported cipher mode %d", cipher->mode);
         return -1;
     }
 
     ctxt = g_new0(QCryptoCipherBuiltin, 1);
 
-    if (AES_set_encrypt_key(key, nkey * 8, &ctxt->state.aes.encrypt_key) != 0) {
-        error_setg(errp, "Failed to set encryption key");
-        goto error;
-    }
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        if (AES_set_encrypt_key(key, nkey * 4, &ctxt->state.aes.key.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
 
-    if (AES_set_decrypt_key(key, nkey * 8, &ctxt->state.aes.decrypt_key) != 0) {
-        error_setg(errp, "Failed to set decryption key");
-        goto error;
+        if (AES_set_decrypt_key(key, nkey * 4, &ctxt->state.aes.key.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
+
+        if (AES_set_encrypt_key(key + (nkey / 2), nkey * 4,
+                                &ctxt->state.aes.key_tweak.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
+
+        if (AES_set_decrypt_key(key + (nkey / 2), nkey * 4,
+                                &ctxt->state.aes.key_tweak.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
+    } else {
+        if (AES_set_encrypt_key(key, nkey * 8, &ctxt->state.aes.key.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
+
+        if (AES_set_decrypt_key(key, nkey * 8, &ctxt->state.aes.key.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
     }
 
+    ctxt->blocksize = AES_BLOCK_SIZE;
     ctxt->free = qcrypto_cipher_free_aes;
     ctxt->setiv = qcrypto_cipher_setiv_aes;
     ctxt->encrypt = qcrypto_cipher_encrypt_aes;
@@ -286,6 +386,7 @@ static int qcrypto_cipher_init_des_rfb(QCryptoCipher *cipher,
     memcpy(ctxt->state.desrfb.key, key, nkey);
     ctxt->state.desrfb.nkey = nkey;
 
+    ctxt->blocksize = 8;
     ctxt->free = qcrypto_cipher_free_des_rfb;
     ctxt->setiv = qcrypto_cipher_setiv_des_rfb;
     ctxt->encrypt = qcrypto_cipher_encrypt_des_rfb;
@@ -322,7 +423,7 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     cipher->alg = alg;
     cipher->mode = mode;
 
-    if (!qcrypto_cipher_validate_key_length(alg, nkey, errp)) {
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
         goto error;
     }
 
@@ -374,6 +475,12 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
     return ctxt->encrypt(cipher, in, out, len, errp);
 }
 
@@ -386,6 +493,12 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherBuiltin *ctxt = cipher->opaque;
 
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
     return ctxt->decrypt(cipher, in, out, len, errp);
 }
 
diff --git a/qemu/crypto/cipher-gcrypt.c b/qemu/crypto/cipher-gcrypt.c
index 8cfc56250..ede2f70df 100644
--- a/qemu/crypto/cipher-gcrypt.c
+++ b/qemu/crypto/cipher-gcrypt.c
@@ -18,6 +18,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
 #include <gcrypt.h>
 
 
@@ -28,12 +31,25 @@ bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
     case QCRYPTO_CIPHER_ALG_AES_128:
     case QCRYPTO_CIPHER_ALG_AES_192:
     case QCRYPTO_CIPHER_ALG_AES_256:
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
         return true;
     default:
         return false;
     }
 }
 
+typedef struct QCryptoCipherGcrypt QCryptoCipherGcrypt;
+struct QCryptoCipherGcrypt {
+    gcry_cipher_hd_t handle;
+    gcry_cipher_hd_t tweakhandle;
+    size_t blocksize;
+    uint8_t *iv;
+};
 
 QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
                                   QCryptoCipherMode mode,
@@ -41,12 +57,13 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
                                   Error **errp)
 {
     QCryptoCipher *cipher;
-    gcry_cipher_hd_t handle;
+    QCryptoCipherGcrypt *ctx;
     gcry_error_t err;
     int gcryalg, gcrymode;
 
     switch (mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
+    case QCRYPTO_CIPHER_MODE_XTS:
         gcrymode = GCRY_CIPHER_MODE_ECB;
         break;
     case QCRYPTO_CIPHER_MODE_CBC:
@@ -57,7 +74,7 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         return NULL;
     }
 
-    if (!qcrypto_cipher_validate_key_length(alg, nkey, errp)) {
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
         return NULL;
     }
 
@@ -78,6 +95,30 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
         gcryalg = GCRY_CIPHER_AES256;
         break;
 
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+        gcryalg = GCRY_CIPHER_CAST5;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+        gcryalg = GCRY_CIPHER_SERPENT128;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+        gcryalg = GCRY_CIPHER_SERPENT192;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        gcryalg = GCRY_CIPHER_SERPENT256;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+        gcryalg = GCRY_CIPHER_TWOFISH128;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        gcryalg = GCRY_CIPHER_TWOFISH;
+        break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d", alg);
         return NULL;
@@ -87,12 +128,22 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     cipher->alg = alg;
     cipher->mode = mode;
 
-    err = gcry_cipher_open(&handle, gcryalg, gcrymode, 0);
+    ctx = g_new0(QCryptoCipherGcrypt, 1);
+
+    err = gcry_cipher_open(&ctx->handle, gcryalg, gcrymode, 0);
     if (err != 0) {
         error_setg(errp, "Cannot initialize cipher: %s",
                    gcry_strerror(err));
         goto error;
     }
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        err = gcry_cipher_open(&ctx->tweakhandle, gcryalg, gcrymode, 0);
+        if (err != 0) {
+            error_setg(errp, "Cannot initialize cipher: %s",
+                       gcry_strerror(err));
+            goto error;
+        }
+    }
 
     if (cipher->alg == QCRYPTO_CIPHER_ALG_DES_RFB) {
         /* We're using standard DES cipher from gcrypt, so we need
@@ -100,22 +151,59 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
          * bizarre RFB variant of DES :-)
          */
         uint8_t *rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
-        err = gcry_cipher_setkey(handle, rfbkey, nkey);
+        err = gcry_cipher_setkey(ctx->handle, rfbkey, nkey);
         g_free(rfbkey);
+        ctx->blocksize = 8;
     } else {
-        err = gcry_cipher_setkey(handle, key, nkey);
+        if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+            nkey /= 2;
+            err = gcry_cipher_setkey(ctx->handle, key, nkey);
+            if (err != 0) {
+                error_setg(errp, "Cannot set key: %s",
+                           gcry_strerror(err));
+                goto error;
+            }
+            err = gcry_cipher_setkey(ctx->tweakhandle, key + nkey, nkey);
+        } else {
+            err = gcry_cipher_setkey(ctx->handle, key, nkey);
+        }
+        if (err != 0) {
+            error_setg(errp, "Cannot set key: %s",
+                       gcry_strerror(err));
+            goto error;
+        }
+        switch (cipher->alg) {
+        case QCRYPTO_CIPHER_ALG_AES_128:
+        case QCRYPTO_CIPHER_ALG_AES_192:
+        case QCRYPTO_CIPHER_ALG_AES_256:
+        case QCRYPTO_CIPHER_ALG_SERPENT_128:
+        case QCRYPTO_CIPHER_ALG_SERPENT_192:
+        case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+        case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+            ctx->blocksize = 16;
+            break;
+        case QCRYPTO_CIPHER_ALG_CAST5_128:
+            ctx->blocksize = 8;
+            break;
+        default:
+            g_assert_not_reached();
+        }
     }
-    if (err != 0) {
-        error_setg(errp, "Cannot set key: %s",
-                   gcry_strerror(err));
-        goto error;
+
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        ctx->iv = g_new0(uint8_t, ctx->blocksize);
     }
 
-    cipher->opaque = handle;
+    cipher->opaque = ctx;
     return cipher;
 
  error:
-    gcry_cipher_close(handle);
+    gcry_cipher_close(ctx->handle);
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        gcry_cipher_close(ctx->tweakhandle);
+    }
+    g_free(ctx);
     g_free(cipher);
     return NULL;
 }
@@ -123,34 +211,72 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
 
 void qcrypto_cipher_free(QCryptoCipher *cipher)
 {
-    gcry_cipher_hd_t handle;
+    QCryptoCipherGcrypt *ctx;
     if (!cipher) {
         return;
     }
-    handle = cipher->opaque;
-    gcry_cipher_close(handle);
+    ctx = cipher->opaque;
+    gcry_cipher_close(ctx->handle);
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        gcry_cipher_close(ctx->tweakhandle);
+    }
+    g_free(ctx->iv);
+    g_free(ctx);
     g_free(cipher);
 }
 
 
+static void qcrypto_gcrypt_xts_encrypt(const void *ctx,
+                                       size_t length,
+                                       uint8_t *dst,
+                                       const uint8_t *src)
+{
+    gcry_error_t err;
+    err = gcry_cipher_encrypt((gcry_cipher_hd_t)ctx, dst, length, src, length);
+    g_assert(err == 0);
+}
+
+static void qcrypto_gcrypt_xts_decrypt(const void *ctx,
+                                       size_t length,
+                                       uint8_t *dst,
+                                       const uint8_t *src)
+{
+    gcry_error_t err;
+    err = gcry_cipher_decrypt((gcry_cipher_hd_t)ctx, dst, length, src, length);
+    g_assert(err == 0);
+}
+
 int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
                            const void *in,
                            void *out,
                            size_t len,
                            Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    err = gcry_cipher_encrypt(handle,
-                              out, len,
-                              in, len);
-    if (err != 0) {
-        error_setg(errp, "Cannot encrypt data: %s",
-                   gcry_strerror(err));
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
         return -1;
     }
 
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        xts_encrypt(ctx->handle, ctx->tweakhandle,
+                    qcrypto_gcrypt_xts_encrypt,
+                    qcrypto_gcrypt_xts_decrypt,
+                    ctx->iv, len, out, in);
+    } else {
+        err = gcry_cipher_encrypt(ctx->handle,
+                                  out, len,
+                                  in, len);
+        if (err != 0) {
+            error_setg(errp, "Cannot encrypt data: %s",
+                       gcry_strerror(err));
+            return -1;
+        }
+    }
+
     return 0;
 }
 
@@ -161,18 +287,31 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
                            size_t len,
                            Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    err = gcry_cipher_decrypt(handle,
-                              out, len,
-                              in, len);
-    if (err != 0) {
-        error_setg(errp, "Cannot decrypt data: %s",
-                   gcry_strerror(err));
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
         return -1;
     }
 
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        xts_decrypt(ctx->handle, ctx->tweakhandle,
+                    qcrypto_gcrypt_xts_encrypt,
+                    qcrypto_gcrypt_xts_decrypt,
+                    ctx->iv, len, out, in);
+    } else {
+        err = gcry_cipher_decrypt(ctx->handle,
+                                  out, len,
+                                  in, len);
+        if (err != 0) {
+            error_setg(errp, "Cannot decrypt data: %s",
+                       gcry_strerror(err));
+            return -1;
+        }
+    }
+
     return 0;
 }
 
@@ -180,16 +319,26 @@ int qcrypto_cipher_setiv(QCryptoCipher *cipher,
                          const uint8_t *iv, size_t niv,
                          Error **errp)
 {
-    gcry_cipher_hd_t handle = cipher->opaque;
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
     gcry_error_t err;
 
-    gcry_cipher_reset(handle);
-    err = gcry_cipher_setiv(handle, iv, niv);
-    if (err != 0) {
-        error_setg(errp, "Cannot set IV: %s",
-                   gcry_strerror(err));
+    if (niv != ctx->blocksize) {
+        error_setg(errp, "Expected IV size %zu not %zu",
+                   ctx->blocksize, niv);
         return -1;
     }
 
+    if (ctx->iv) {
+        memcpy(ctx->iv, iv, niv);
+    } else {
+        gcry_cipher_reset(ctx->handle);
+        err = gcry_cipher_setiv(ctx->handle, iv, niv);
+        if (err != 0) {
+            error_setg(errp, "Cannot set IV: %s",
+                   gcry_strerror(err));
+            return -1;
+        }
+    }
+
     return 0;
 }
diff --git a/qemu/crypto/cipher-nettle.c b/qemu/crypto/cipher-nettle.c
index b01cb1c85..70909fb7f 100644
--- a/qemu/crypto/cipher-nettle.c
+++ b/qemu/crypto/cipher-nettle.c
@@ -18,58 +18,177 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
 #include <nettle/nettle-types.h>
 #include <nettle/aes.h>
 #include <nettle/des.h>
 #include <nettle/cbc.h>
+#include <nettle/cast128.h>
+#include <nettle/serpent.h>
+#include <nettle/twofish.h>
 
-#if CONFIG_NETTLE_VERSION_MAJOR < 3
-typedef nettle_crypt_func nettle_cipher_func;
+typedef void (*QCryptoCipherNettleFuncWrapper)(const void *ctx,
+                                               size_t length,
+                                               uint8_t *dst,
+                                               const uint8_t *src);
 
+#if CONFIG_NETTLE_VERSION_MAJOR < 3
+typedef nettle_crypt_func * QCryptoCipherNettleFuncNative;
 typedef void *       cipher_ctx_t;
 typedef unsigned     cipher_length_t;
+
+#define cast5_set_key cast128_set_key
 #else
+typedef nettle_cipher_func * QCryptoCipherNettleFuncNative;
 typedef const void * cipher_ctx_t;
 typedef size_t       cipher_length_t;
 #endif
 
-static nettle_cipher_func aes_encrypt_wrapper;
-static nettle_cipher_func aes_decrypt_wrapper;
-static nettle_cipher_func des_encrypt_wrapper;
-static nettle_cipher_func des_decrypt_wrapper;
+typedef struct QCryptoNettleAES {
+    struct aes_ctx enc;
+    struct aes_ctx dec;
+} QCryptoNettleAES;
+
+static void aes_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_encrypt(&aesctx->enc, length, dst, src);
+}
+
+static void aes_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_decrypt(&aesctx->dec, length, dst, src);
+}
+
+static void des_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    des_encrypt(ctx, length, dst, src);
+}
+
+static void des_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    des_decrypt(ctx, length, dst, src);
+}
+
+static void cast128_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    cast128_encrypt(ctx, length, dst, src);
+}
+
+static void cast128_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    cast128_decrypt(ctx, length, dst, src);
+}
+
+static void serpent_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    serpent_encrypt(ctx, length, dst, src);
+}
 
-static void aes_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+static void serpent_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    serpent_decrypt(ctx, length, dst, src);
+}
+
+static void twofish_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    twofish_encrypt(ctx, length, dst, src);
+}
+
+static void twofish_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    twofish_decrypt(ctx, length, dst, src);
+}
+
+static void aes_encrypt_wrapper(const void *ctx, size_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
-    aes_encrypt(ctx, length, dst, src);
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_encrypt(&aesctx->enc, length, dst, src);
 }
 
-static void aes_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+static void aes_decrypt_wrapper(const void *ctx, size_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
-    aes_decrypt(ctx, length, dst, src);
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_decrypt(&aesctx->dec, length, dst, src);
 }
 
-static void des_encrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+static void des_encrypt_wrapper(const void *ctx, size_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
     des_encrypt(ctx, length, dst, src);
 }
 
-static void des_decrypt_wrapper(cipher_ctx_t ctx, cipher_length_t length,
+static void des_decrypt_wrapper(const void *ctx, size_t length,
                                 uint8_t *dst, const uint8_t *src)
 {
     des_decrypt(ctx, length, dst, src);
 }
 
+static void cast128_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    cast128_encrypt(ctx, length, dst, src);
+}
+
+static void cast128_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    cast128_decrypt(ctx, length, dst, src);
+}
+
+static void serpent_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    serpent_encrypt(ctx, length, dst, src);
+}
+
+static void serpent_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    serpent_decrypt(ctx, length, dst, src);
+}
+
+static void twofish_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    twofish_encrypt(ctx, length, dst, src);
+}
+
+static void twofish_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    twofish_decrypt(ctx, length, dst, src);
+}
+
 typedef struct QCryptoCipherNettle QCryptoCipherNettle;
 struct QCryptoCipherNettle {
-    void *ctx_encrypt;
-    void *ctx_decrypt;
-    nettle_cipher_func *alg_encrypt;
-    nettle_cipher_func *alg_decrypt;
+    /* Primary cipher context for all modes */
+    void *ctx;
+    /* Second cipher context for XTS mode only */
+    void *ctx_tweak;
+    /* Cipher callbacks for both contexts */
+    QCryptoCipherNettleFuncNative alg_encrypt_native;
+    QCryptoCipherNettleFuncNative alg_decrypt_native;
+    QCryptoCipherNettleFuncWrapper alg_encrypt_wrapper;
+    QCryptoCipherNettleFuncWrapper alg_decrypt_wrapper;
+
     uint8_t *iv;
-    size_t niv;
+    size_t blocksize;
 };
 
 bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
@@ -79,6 +198,13 @@ bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
     case QCRYPTO_CIPHER_ALG_AES_128:
     case QCRYPTO_CIPHER_ALG_AES_192:
     case QCRYPTO_CIPHER_ALG_AES_256:
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
         return true;
     default:
         return false;
@@ -98,13 +224,14 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
     switch (mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
     case QCRYPTO_CIPHER_MODE_CBC:
+    case QCRYPTO_CIPHER_MODE_XTS:
         break;
     default:
         error_setg(errp, "Unsupported cipher mode %d", mode);
         return NULL;
     }
 
-    if (!qcrypto_cipher_validate_key_length(alg, nkey, errp)) {
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
         return NULL;
     }
 
@@ -116,38 +243,125 @@ QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
 
     switch (alg) {
     case QCRYPTO_CIPHER_ALG_DES_RFB:
-        ctx->ctx_encrypt = g_new0(struct des_ctx, 1);
-        ctx->ctx_decrypt = NULL; /* 1 ctx can do both */
+        ctx->ctx = g_new0(struct des_ctx, 1);
         rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
-        des_set_key(ctx->ctx_encrypt, rfbkey);
+        des_set_key(ctx->ctx, rfbkey);
         g_free(rfbkey);
 
-        ctx->alg_encrypt = des_encrypt_wrapper;
-        ctx->alg_decrypt = des_decrypt_wrapper;
+        ctx->alg_encrypt_native = des_encrypt_native;
+        ctx->alg_decrypt_native = des_decrypt_native;
+        ctx->alg_encrypt_wrapper = des_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = des_decrypt_wrapper;
 
-        ctx->niv = DES_BLOCK_SIZE;
+        ctx->blocksize = DES_BLOCK_SIZE;
         break;
 
     case QCRYPTO_CIPHER_ALG_AES_128:
     case QCRYPTO_CIPHER_ALG_AES_192:
     case QCRYPTO_CIPHER_ALG_AES_256:
-        ctx->ctx_encrypt = g_new0(struct aes_ctx, 1);
-        ctx->ctx_decrypt = g_new0(struct aes_ctx, 1);
+        ctx->ctx = g_new0(QCryptoNettleAES, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(QCryptoNettleAES, 1);
+
+            nkey /= 2;
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx)->enc,
+                                nkey, key);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx)->dec,
+                                nkey, key);
+
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx_tweak)->enc,
+                                nkey, key + nkey);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx_tweak)->dec,
+                                nkey, key + nkey);
+        } else {
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx)->enc,
+                                nkey, key);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx)->dec,
+                                nkey, key);
+        }
+
+        ctx->alg_encrypt_native = aes_encrypt_native;
+        ctx->alg_decrypt_native = aes_decrypt_native;
+        ctx->alg_encrypt_wrapper = aes_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = aes_decrypt_wrapper;
+
+        ctx->blocksize = AES_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+        ctx->ctx = g_new0(struct cast128_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct cast128_ctx, 1);
 
-        aes_set_encrypt_key(ctx->ctx_encrypt, nkey, key);
-        aes_set_decrypt_key(ctx->ctx_decrypt, nkey, key);
+            nkey /= 2;
+            cast5_set_key(ctx->ctx, nkey, key);
+            cast5_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            cast5_set_key(ctx->ctx, nkey, key);
+        }
 
-        ctx->alg_encrypt = aes_encrypt_wrapper;
-        ctx->alg_decrypt = aes_decrypt_wrapper;
+        ctx->alg_encrypt_native = cast128_encrypt_native;
+        ctx->alg_decrypt_native = cast128_decrypt_native;
+        ctx->alg_encrypt_wrapper = cast128_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = cast128_decrypt_wrapper;
 
-        ctx->niv = AES_BLOCK_SIZE;
+        ctx->blocksize = CAST128_BLOCK_SIZE;
         break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        ctx->ctx = g_new0(struct serpent_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct serpent_ctx, 1);
+
+            nkey /= 2;
+            serpent_set_key(ctx->ctx, nkey, key);
+            serpent_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            serpent_set_key(ctx->ctx, nkey, key);
+        }
+
+        ctx->alg_encrypt_native = serpent_encrypt_native;
+        ctx->alg_decrypt_native = serpent_decrypt_native;
+        ctx->alg_encrypt_wrapper = serpent_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = serpent_decrypt_wrapper;
+
+        ctx->blocksize = SERPENT_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        ctx->ctx = g_new0(struct twofish_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct twofish_ctx, 1);
+
+            nkey /= 2;
+            twofish_set_key(ctx->ctx, nkey, key);
+            twofish_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            twofish_set_key(ctx->ctx, nkey, key);
+        }
+
+        ctx->alg_encrypt_native = twofish_encrypt_native;
+        ctx->alg_decrypt_native = twofish_decrypt_native;
+        ctx->alg_encrypt_wrapper = twofish_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = twofish_decrypt_wrapper;
+
+        ctx->blocksize = TWOFISH_BLOCK_SIZE;
+        break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d", alg);
         goto error;
     }
 
-    ctx->iv = g_new0(uint8_t, ctx->niv);
+    ctx->iv = g_new0(uint8_t, ctx->blocksize);
     cipher->opaque = ctx;
 
     return cipher;
@@ -169,8 +383,8 @@ void qcrypto_cipher_free(QCryptoCipher *cipher)
 
     ctx = cipher->opaque;
     g_free(ctx->iv);
-    g_free(ctx->ctx_encrypt);
-    g_free(ctx->ctx_decrypt);
+    g_free(ctx->ctx);
+    g_free(ctx->ctx_tweak);
     g_free(ctx);
     g_free(cipher);
 }
@@ -184,16 +398,29 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
 
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
-        ctx->alg_encrypt(ctx->ctx_encrypt, len, out, in);
+        ctx->alg_encrypt_wrapper(ctx->ctx, len, out, in);
         break;
 
     case QCRYPTO_CIPHER_MODE_CBC:
-        cbc_encrypt(ctx->ctx_encrypt, ctx->alg_encrypt,
-                    ctx->niv, ctx->iv,
+        cbc_encrypt(ctx->ctx, ctx->alg_encrypt_native,
+                    ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
+
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_encrypt(ctx->ctx, ctx->ctx_tweak,
+                    ctx->alg_encrypt_wrapper, ctx->alg_encrypt_wrapper,
+                    ctx->iv, len, out, in);
+        break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d",
                    cipher->alg);
@@ -211,17 +438,34 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
 
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
     switch (cipher->mode) {
     case QCRYPTO_CIPHER_MODE_ECB:
-        ctx->alg_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
-                         len, out, in);
+        ctx->alg_decrypt_wrapper(ctx->ctx, len, out, in);
         break;
 
     case QCRYPTO_CIPHER_MODE_CBC:
-        cbc_decrypt(ctx->ctx_decrypt ? ctx->ctx_decrypt : ctx->ctx_encrypt,
-                    ctx->alg_decrypt, ctx->niv, ctx->iv,
+        cbc_decrypt(ctx->ctx, ctx->alg_decrypt_native,
+                    ctx->blocksize, ctx->iv,
                     len, out, in);
         break;
+
+    case QCRYPTO_CIPHER_MODE_XTS:
+        if (ctx->blocksize != XTS_BLOCK_SIZE) {
+            error_setg(errp, "Block size must be %d not %zu",
+                       XTS_BLOCK_SIZE, ctx->blocksize);
+            return -1;
+        }
+        xts_decrypt(ctx->ctx, ctx->ctx_tweak,
+                    ctx->alg_encrypt_wrapper, ctx->alg_decrypt_wrapper,
+                    ctx->iv, len, out, in);
+        break;
+
     default:
         error_setg(errp, "Unsupported cipher algorithm %d",
                    cipher->alg);
@@ -235,9 +479,9 @@ int qcrypto_cipher_setiv(QCryptoCipher *cipher,
                          Error **errp)
 {
     QCryptoCipherNettle *ctx = cipher->opaque;
-    if (niv != ctx->niv) {
+    if (niv != ctx->blocksize) {
         error_setg(errp, "Expected IV size %zu not %zu",
-                   ctx->niv, niv);
+                   ctx->blocksize, niv);
         return -1;
     }
     memcpy(ctx->iv, iv, niv);
diff --git a/qemu/crypto/cipher.c b/qemu/crypto/cipher.c
index 024a00cb5..cafb45436 100644
--- a/qemu/crypto/cipher.c
+++ b/qemu/crypto/cipher.c
@@ -18,36 +18,119 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "crypto/cipher.h"
 
 
-static size_t alg_key_len[QCRYPTO_CIPHER_ALG_LAST] = {
+static size_t alg_key_len[QCRYPTO_CIPHER_ALG__MAX] = {
     [QCRYPTO_CIPHER_ALG_AES_128] = 16,
     [QCRYPTO_CIPHER_ALG_AES_192] = 24,
     [QCRYPTO_CIPHER_ALG_AES_256] = 32,
     [QCRYPTO_CIPHER_ALG_DES_RFB] = 8,
+    [QCRYPTO_CIPHER_ALG_CAST5_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_192] = 24,
+    [QCRYPTO_CIPHER_ALG_SERPENT_256] = 32,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_128] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_192] = 24,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_256] = 32,
 };
 
+static size_t alg_block_len[QCRYPTO_CIPHER_ALG__MAX] = {
+    [QCRYPTO_CIPHER_ALG_AES_128] = 16,
+    [QCRYPTO_CIPHER_ALG_AES_192] = 16,
+    [QCRYPTO_CIPHER_ALG_AES_256] = 16,
+    [QCRYPTO_CIPHER_ALG_DES_RFB] = 8,
+    [QCRYPTO_CIPHER_ALG_CAST5_128] = 8,
+    [QCRYPTO_CIPHER_ALG_SERPENT_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_192] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_256] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_128] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_192] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_256] = 16,
+};
+
+static bool mode_need_iv[QCRYPTO_CIPHER_MODE__MAX] = {
+    [QCRYPTO_CIPHER_MODE_ECB] = false,
+    [QCRYPTO_CIPHER_MODE_CBC] = true,
+    [QCRYPTO_CIPHER_MODE_XTS] = true,
+};
+
+
+size_t qcrypto_cipher_get_block_len(QCryptoCipherAlgorithm alg)
+{
+    if (alg >= G_N_ELEMENTS(alg_key_len)) {
+        return 0;
+    }
+    return alg_block_len[alg];
+}
+
+
+size_t qcrypto_cipher_get_key_len(QCryptoCipherAlgorithm alg)
+{
+    if (alg >= G_N_ELEMENTS(alg_key_len)) {
+        return 0;
+    }
+    return alg_key_len[alg];
+}
+
+
+size_t qcrypto_cipher_get_iv_len(QCryptoCipherAlgorithm alg,
+                                 QCryptoCipherMode mode)
+{
+    if (alg >= G_N_ELEMENTS(alg_block_len)) {
+        return 0;
+    }
+    if (mode >= G_N_ELEMENTS(mode_need_iv)) {
+        return 0;
+    }
+
+    if (mode_need_iv[mode]) {
+        return alg_block_len[alg];
+    }
+    return 0;
+}
+
+
 static bool
 qcrypto_cipher_validate_key_length(QCryptoCipherAlgorithm alg,
+                                   QCryptoCipherMode mode,
                                    size_t nkey,
                                    Error **errp)
 {
-    if ((unsigned)alg >= QCRYPTO_CIPHER_ALG_LAST) {
+    if ((unsigned)alg >= QCRYPTO_CIPHER_ALG__MAX) {
         error_setg(errp, "Cipher algorithm %d out of range",
                    alg);
         return false;
     }
 
-    if (alg_key_len[alg] != nkey) {
-        error_setg(errp, "Cipher key length %zu should be %zu",
-                   alg_key_len[alg], nkey);
-        return false;
+    if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+        if (alg == QCRYPTO_CIPHER_ALG_DES_RFB) {
+            error_setg(errp, "XTS mode not compatible with DES-RFB");
+            return false;
+        }
+        if (nkey % 2) {
+            error_setg(errp, "XTS cipher key length should be a multiple of 2");
+            return false;
+        }
+
+        if (alg_key_len[alg] != (nkey / 2)) {
+            error_setg(errp, "Cipher key length %zu should be %zu",
+                       nkey, alg_key_len[alg] * 2);
+            return false;
+        }
+    } else {
+        if (alg_key_len[alg] != nkey) {
+            error_setg(errp, "Cipher key length %zu should be %zu",
+                       nkey, alg_key_len[alg]);
+            return false;
+        }
     }
     return true;
 }
 
-#if defined(CONFIG_GNUTLS_GCRYPT) || defined(CONFIG_GNUTLS_NETTLE)
+#if defined(CONFIG_GCRYPT) || defined(CONFIG_NETTLE)
 static uint8_t *
 qcrypto_cipher_munge_des_rfb_key(const uint8_t *key,
                                  size_t nkey)
@@ -63,11 +146,11 @@ qcrypto_cipher_munge_des_rfb_key(const uint8_t *key,
     }
     return ret;
 }
-#endif /* CONFIG_GNUTLS_GCRYPT || CONFIG_GNUTLS_NETTLE */
+#endif /* CONFIG_GCRYPT || CONFIG_NETTLE */
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
 #include "crypto/cipher-gcrypt.c"
-#elif defined CONFIG_GNUTLS_NETTLE
+#elif defined CONFIG_NETTLE
 #include "crypto/cipher-nettle.c"
 #else
 #include "crypto/cipher-builtin.c"
diff --git a/qemu/crypto/desrfb.c b/qemu/crypto/desrfb.c
index fc20a30df..ec47dea3b 100644
--- a/qemu/crypto/desrfb.c
+++ b/qemu/crypto/desrfb.c
@@ -26,6 +26,7 @@
  * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
  */
 
+#include "qemu/osdep.h"
 #include "crypto/desrfb.h"
 
 static void scrunch(unsigned char *, unsigned long *);
diff --git a/qemu/crypto/hash.c b/qemu/crypto/hash.c
index 81e74de86..b90af3495 100644
--- a/qemu/crypto/hash.c
+++ b/qemu/crypto/hash.c
@@ -18,13 +18,33 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "crypto/hash.h"
 
 #ifdef CONFIG_GNUTLS_HASH
 #include <gnutls/gnutls.h>
 #include <gnutls/crypto.h>
+#endif
 
-static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG_LAST] = {
+
+static size_t qcrypto_hash_alg_size[QCRYPTO_HASH_ALG__MAX] = {
+    [QCRYPTO_HASH_ALG_MD5] = 16,
+    [QCRYPTO_HASH_ALG_SHA1] = 20,
+    [QCRYPTO_HASH_ALG_SHA256] = 32,
+};
+
+size_t qcrypto_hash_digest_len(QCryptoHashAlgorithm alg)
+{
+    if (alg >= G_N_ELEMENTS(qcrypto_hash_alg_size)) {
+        return 0;
+    }
+    return qcrypto_hash_alg_size[alg];
+}
+
+
+#ifdef CONFIG_GNUTLS_HASH
+static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG__MAX] = {
     [QCRYPTO_HASH_ALG_MD5] = GNUTLS_DIG_MD5,
     [QCRYPTO_HASH_ALG_SHA1] = GNUTLS_DIG_SHA1,
     [QCRYPTO_HASH_ALG_SHA256] = GNUTLS_DIG_SHA256,
@@ -38,6 +58,7 @@ gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
     return false;
 }
 
+
 int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
                         const struct iovec *iov,
                         size_t niov,
diff --git a/qemu/crypto/init.c b/qemu/crypto/init.c
index 7447882c7..1e564d949 100644
--- a/qemu/crypto/init.c
+++ b/qemu/crypto/init.c
@@ -18,14 +18,17 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "crypto/init.h"
+#include "qapi/error.h"
 #include "qemu/thread.h"
 
 #ifdef CONFIG_GNUTLS
 #include <gnutls/gnutls.h>
 #include <gnutls/crypto.h>
+#endif
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
 #include <gcrypt.h>
 #endif
 
@@ -37,6 +40,7 @@
  *  - When GNUTLS >= 2.12, we must not initialize gcrypt threading
  *    because GNUTLS will do that itself
  *  - When GNUTLS < 2.12 we must always initialize gcrypt threading
+ *  - When GNUTLS is disabled we must always initialize gcrypt threading
  *
  * But....
  *
@@ -47,12 +51,15 @@
  *
  *   - gcrypt < 1.6.0
  * AND
- *   - gnutls < 2.12
+ *      - gnutls < 2.12
+ *   OR
+ *      - gnutls is disabled
  *
  */
 
-#if (defined(CONFIG_GNUTLS_GCRYPT) &&           \
-     (!defined(GNUTLS_VERSION_NUMBER) ||        \
+#if (defined(CONFIG_GCRYPT) &&                  \
+     (!defined(CONFIG_GNUTLS) ||                \
+      !defined(GNUTLS_VERSION_NUMBER) ||       \
       (GNUTLS_VERSION_NUMBER < 0x020c00)) &&    \
      (!defined(GCRYPT_VERSION_NUMBER) ||        \
       (GCRYPT_VERSION_NUMBER < 0x010600)))
@@ -113,6 +120,7 @@ static struct gcry_thread_cbs qcrypto_gcrypt_thread_impl = {
 
 int qcrypto_init(Error **errp)
 {
+#ifdef CONFIG_GNUTLS
     int ret;
     ret = gnutls_global_init();
     if (ret < 0) {
@@ -125,8 +133,9 @@ int qcrypto_init(Error **errp)
     gnutls_global_set_log_level(10);
     gnutls_global_set_log_function(qcrypto_gnutls_log);
 #endif
+#endif
 
-#ifdef CONFIG_GNUTLS_GCRYPT
+#ifdef CONFIG_GCRYPT
     if (!gcry_check_version(GCRYPT_VERSION)) {
         error_setg(errp, "Unable to initialize gcrypt");
         return -1;
@@ -139,12 +148,3 @@ int qcrypto_init(Error **errp)
 
     return 0;
 }
-
-#else /* ! CONFIG_GNUTLS */
-
-int qcrypto_init(Error **errp G_GNUC_UNUSED)
-{
-    return 0;
-}
-
-#endif /* ! CONFIG_GNUTLS */
diff --git a/qemu/crypto/ivgen-essiv.c b/qemu/crypto/ivgen-essiv.c
new file mode 100644
index 000000000..634de6333
--- /dev/null
+++ b/qemu/crypto/ivgen-essiv.c
@@ -0,0 +1,120 @@
+/*
+ * QEMU Crypto block IV generator - essiv
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-essiv.h"
+
+typedef struct QCryptoIVGenESSIV QCryptoIVGenESSIV;
+struct QCryptoIVGenESSIV {
+    QCryptoCipher *cipher;
+};
+
+static int qcrypto_ivgen_essiv_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    uint8_t *salt;
+    size_t nhash;
+    size_t nsalt;
+    QCryptoIVGenESSIV *essiv = g_new0(QCryptoIVGenESSIV, 1);
+
+    /* Not necessarily the same as nkey */
+    nsalt = qcrypto_cipher_get_key_len(ivgen->cipher);
+
+    nhash = qcrypto_hash_digest_len(ivgen->hash);
+    /* Salt must be larger of hash size or key size */
+    salt = g_new0(uint8_t, MAX(nhash, nsalt));
+
+    if (qcrypto_hash_bytes(ivgen->hash, (const gchar *)key, nkey,
+                           &salt, &nhash,
+                           errp) < 0) {
+        g_free(essiv);
+        return -1;
+    }
+
+    /* Now potentially truncate salt to match cipher key len */
+    essiv->cipher = qcrypto_cipher_new(ivgen->cipher,
+                                       QCRYPTO_CIPHER_MODE_ECB,
+                                       salt, MIN(nhash, nsalt),
+                                       errp);
+    if (!essiv->cipher) {
+        g_free(essiv);
+        g_free(salt);
+        return -1;
+    }
+
+    g_free(salt);
+    ivgen->private = essiv;
+
+    return 0;
+}
+
+static int qcrypto_ivgen_essiv_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    QCryptoIVGenESSIV *essiv = ivgen->private;
+    size_t ndata = qcrypto_cipher_get_block_len(ivgen->cipher);
+    uint8_t *data = g_new(uint8_t, ndata);
+
+    sector = cpu_to_le64(sector);
+    memcpy(data, (uint8_t *)&sector, ndata);
+    if (sizeof(sector) < ndata) {
+        memset(data + sizeof(sector), 0, ndata - sizeof(sector));
+    }
+
+    if (qcrypto_cipher_encrypt(essiv->cipher,
+                               data,
+                               data,
+                               ndata,
+                               errp) < 0) {
+        g_free(data);
+        return -1;
+    }
+
+    if (ndata > niv) {
+        ndata = niv;
+    }
+    memcpy(iv, data, ndata);
+    if (ndata < niv) {
+        memset(iv + ndata, 0, niv - ndata);
+    }
+    g_free(data);
+    return 0;
+}
+
+static void qcrypto_ivgen_essiv_cleanup(QCryptoIVGen *ivgen)
+{
+    QCryptoIVGenESSIV *essiv = ivgen->private;
+
+    qcrypto_cipher_free(essiv->cipher);
+    g_free(essiv);
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_essiv = {
+    .init = qcrypto_ivgen_essiv_init,
+    .calculate = qcrypto_ivgen_essiv_calculate,
+    .cleanup = qcrypto_ivgen_essiv_cleanup,
+};
+
diff --git a/qemu/crypto/ivgen-essiv.h b/qemu/crypto/ivgen-essiv.h
new file mode 100644
index 000000000..4a00af849
--- /dev/null
+++ b/qemu/crypto/ivgen-essiv.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - essiv
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_ESSIV_H__
+#define QCRYPTO_IVGEN_ESSIV_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_essiv;
+
+#endif /* QCRYPTO_IVGEN_ESSIV_H__ */
diff --git a/qemu/crypto/ivgen-plain.c b/qemu/crypto/ivgen-plain.c
new file mode 100644
index 000000000..9b9b4ad0b
--- /dev/null
+++ b/qemu/crypto/ivgen-plain.c
@@ -0,0 +1,61 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-plain.h"
+
+static int qcrypto_ivgen_plain_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    return 0;
+}
+
+static int qcrypto_ivgen_plain_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    size_t ivprefix;
+    uint32_t shortsector = cpu_to_le32((sector & 0xffffffff));
+    ivprefix = sizeof(shortsector);
+    if (ivprefix > niv) {
+        ivprefix = niv;
+    }
+    memcpy(iv, &shortsector, ivprefix);
+    if (ivprefix < niv) {
+        memset(iv + ivprefix, 0, niv - ivprefix);
+    }
+    return 0;
+}
+
+static void qcrypto_ivgen_plain_cleanup(QCryptoIVGen *ivgen)
+{
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_plain = {
+    .init = qcrypto_ivgen_plain_init,
+    .calculate = qcrypto_ivgen_plain_calculate,
+    .cleanup = qcrypto_ivgen_plain_cleanup,
+};
+
diff --git a/qemu/crypto/ivgen-plain.h b/qemu/crypto/ivgen-plain.h
new file mode 100644
index 000000000..0fe8835c3
--- /dev/null
+++ b/qemu/crypto/ivgen-plain.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_PLAIN_H__
+#define QCRYPTO_IVGEN_PLAIN_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_plain;
+
+#endif /* QCRYPTO_IVGEN_PLAIN_H__ */
diff --git a/qemu/crypto/ivgen-plain64.c b/qemu/crypto/ivgen-plain64.c
new file mode 100644
index 000000000..6c6b1b44c
--- /dev/null
+++ b/qemu/crypto/ivgen-plain64.c
@@ -0,0 +1,61 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-plain.h"
+
+static int qcrypto_ivgen_plain_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    return 0;
+}
+
+static int qcrypto_ivgen_plain_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    size_t ivprefix;
+    ivprefix = sizeof(sector);
+    sector = cpu_to_le64(sector);
+    if (ivprefix > niv) {
+        ivprefix = niv;
+    }
+    memcpy(iv, &sector, ivprefix);
+    if (ivprefix < niv) {
+        memset(iv + ivprefix, 0, niv - ivprefix);
+    }
+    return 0;
+}
+
+static void qcrypto_ivgen_plain_cleanup(QCryptoIVGen *ivgen)
+{
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_plain64 = {
+    .init = qcrypto_ivgen_plain_init,
+    .calculate = qcrypto_ivgen_plain_calculate,
+    .cleanup = qcrypto_ivgen_plain_cleanup,
+};
+
diff --git a/qemu/crypto/ivgen-plain64.h b/qemu/crypto/ivgen-plain64.h
new file mode 100644
index 000000000..c4104459b
--- /dev/null
+++ b/qemu/crypto/ivgen-plain64.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - plain64
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_PLAIN64_H__
+#define QCRYPTO_IVGEN_PLAIN64_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_plain64;
+
+#endif /* QCRYPTO_IVGEN_PLAIN64_H__ */
diff --git a/qemu/crypto/ivgen.c b/qemu/crypto/ivgen.c
new file mode 100644
index 000000000..f66435112
--- /dev/null
+++ b/qemu/crypto/ivgen.c
@@ -0,0 +1,101 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+
+#include "crypto/ivgenpriv.h"
+#include "crypto/ivgen-plain.h"
+#include "crypto/ivgen-plain64.h"
+#include "crypto/ivgen-essiv.h"
+
+
+QCryptoIVGen *qcrypto_ivgen_new(QCryptoIVGenAlgorithm alg,
+                                QCryptoCipherAlgorithm cipheralg,
+                                QCryptoHashAlgorithm hash,
+                                const uint8_t *key, size_t nkey,
+                                Error **errp)
+{
+    QCryptoIVGen *ivgen = g_new0(QCryptoIVGen, 1);
+
+    ivgen->algorithm = alg;
+    ivgen->cipher = cipheralg;
+    ivgen->hash = hash;
+
+    switch (alg) {
+    case QCRYPTO_IVGEN_ALG_PLAIN:
+        ivgen->driver = &qcrypto_ivgen_plain;
+        break;
+    case QCRYPTO_IVGEN_ALG_PLAIN64:
+        ivgen->driver = &qcrypto_ivgen_plain64;
+        break;
+    case QCRYPTO_IVGEN_ALG_ESSIV:
+        ivgen->driver = &qcrypto_ivgen_essiv;
+        break;
+    default:
+        error_setg(errp, "Unknown block IV generator algorithm %d", alg);
+        g_free(ivgen);
+        return NULL;
+    }
+
+    if (ivgen->driver->init(ivgen, key, nkey, errp) < 0) {
+        g_free(ivgen);
+        return NULL;
+    }
+
+    return ivgen;
+}
+
+
+int qcrypto_ivgen_calculate(QCryptoIVGen *ivgen,
+                            uint64_t sector,
+                            uint8_t *iv, size_t niv,
+                            Error **errp)
+{
+    return ivgen->driver->calculate(ivgen, sector, iv, niv, errp);
+}
+
+
+QCryptoIVGenAlgorithm qcrypto_ivgen_get_algorithm(QCryptoIVGen *ivgen)
+{
+    return ivgen->algorithm;
+}
+
+
+QCryptoCipherAlgorithm qcrypto_ivgen_get_cipher(QCryptoIVGen *ivgen)
+{
+    return ivgen->cipher;
+}
+
+
+QCryptoHashAlgorithm qcrypto_ivgen_get_hash(QCryptoIVGen *ivgen)
+{
+    return ivgen->hash;
+}
+
+
+void qcrypto_ivgen_free(QCryptoIVGen *ivgen)
+{
+    if (!ivgen) {
+        return;
+    }
+    ivgen->driver->cleanup(ivgen);
+    g_free(ivgen);
+}
diff --git a/qemu/crypto/ivgenpriv.h b/qemu/crypto/ivgenpriv.h
new file mode 100644
index 000000000..7b87e02ea
--- /dev/null
+++ b/qemu/crypto/ivgenpriv.h
@@ -0,0 +1,49 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_IVGEN_PRIV_H__
+#define QCRYPTO_IVGEN_PRIV_H__
+
+#include "crypto/ivgen.h"
+
+typedef struct QCryptoIVGenDriver QCryptoIVGenDriver;
+
+struct QCryptoIVGenDriver {
+    int (*init)(QCryptoIVGen *ivgen,
+                const uint8_t *key, size_t nkey,
+                Error **errp);
+    int (*calculate)(QCryptoIVGen *ivgen,
+                     uint64_t sector,
+                     uint8_t *iv, size_t niv,
+                     Error **errp);
+    void (*cleanup)(QCryptoIVGen *ivgen);
+};
+
+struct QCryptoIVGen {
+    QCryptoIVGenDriver *driver;
+    void *private;
+
+    QCryptoIVGenAlgorithm algorithm;
+    QCryptoCipherAlgorithm cipher;
+    QCryptoHashAlgorithm hash;
+};
+
+
+#endif /* QCRYPTO_IVGEN_PRIV_H__ */
diff --git a/qemu/crypto/pbkdf-gcrypt.c b/qemu/crypto/pbkdf-gcrypt.c
new file mode 100644
index 000000000..997b311d8
--- /dev/null
+++ b/qemu/crypto/pbkdf-gcrypt.c
@@ -0,0 +1,69 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+#include "gcrypt.h"
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_MD5:
+    case QCRYPTO_HASH_ALG_SHA1:
+    case QCRYPTO_HASH_ALG_SHA256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp)
+{
+    static const int hash_map[QCRYPTO_HASH_ALG__MAX] = {
+        [QCRYPTO_HASH_ALG_MD5] = GCRY_MD_MD5,
+        [QCRYPTO_HASH_ALG_SHA1] = GCRY_MD_SHA1,
+        [QCRYPTO_HASH_ALG_SHA256] = GCRY_MD_SHA256,
+    };
+    int ret;
+
+    if (hash >= G_N_ELEMENTS(hash_map) ||
+        hash_map[hash] == GCRY_MD_NONE) {
+        error_setg(errp, "Unexpected hash algorithm %d", hash);
+        return -1;
+    }
+
+    ret = gcry_kdf_derive(key, nkey, GCRY_KDF_PBKDF2,
+                          hash_map[hash],
+                          salt, nsalt, iterations,
+                          nout, out);
+    if (ret != 0) {
+        error_setg(errp, "Cannot derive password: %s",
+                   gcry_strerror(ret));
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/qemu/crypto/pbkdf-nettle.c b/qemu/crypto/pbkdf-nettle.c
new file mode 100644
index 000000000..db9fc1578
--- /dev/null
+++ b/qemu/crypto/pbkdf-nettle.c
@@ -0,0 +1,66 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+#include "nettle/pbkdf2.h"
+
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_SHA1:
+    case QCRYPTO_HASH_ALG_SHA256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_SHA1:
+        pbkdf2_hmac_sha1(nkey, key,
+                         iterations,
+                         nsalt, salt,
+                         nout, out);
+        break;
+
+    case QCRYPTO_HASH_ALG_SHA256:
+        pbkdf2_hmac_sha256(nkey, key,
+                           iterations,
+                           nsalt, salt,
+                           nout, out);
+        break;
+
+    default:
+        error_setg_errno(errp, ENOSYS,
+                         "PBKDF does not support hash algorithm %d", hash);
+        return -1;
+    }
+    return 0;
+}
diff --git a/qemu/crypto/pbkdf-stub.c b/qemu/crypto/pbkdf-stub.c
new file mode 100644
index 000000000..266a5051b
--- /dev/null
+++ b/qemu/crypto/pbkdf-stub.c
@@ -0,0 +1,43 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash G_GNUC_UNUSED)
+{
+    return false;
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash G_GNUC_UNUSED,
+                   const uint8_t *key G_GNUC_UNUSED,
+                   size_t nkey G_GNUC_UNUSED,
+                   const uint8_t *salt G_GNUC_UNUSED,
+                   size_t nsalt G_GNUC_UNUSED,
+                   unsigned int iterations G_GNUC_UNUSED,
+                   uint8_t *out G_GNUC_UNUSED,
+                   size_t nout G_GNUC_UNUSED,
+                   Error **errp)
+{
+    error_setg_errno(errp, ENOSYS,
+                     "No crypto library supporting PBKDF in this build");
+    return -1;
+}
diff --git a/qemu/crypto/pbkdf.c b/qemu/crypto/pbkdf.c
new file mode 100644
index 000000000..695cc35df
--- /dev/null
+++ b/qemu/crypto/pbkdf.c
@@ -0,0 +1,110 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+
+static int qcrypto_pbkdf2_get_thread_cpu(unsigned long long *val_ms,
+                                         Error **errp)
+{
+#ifdef _WIN32
+    FILETIME creation_time, exit_time, kernel_time, user_time;
+    ULARGE_INTEGER thread_time;
+
+    if (!GetThreadTimes(GetCurrentThread(), &creation_time, &exit_time,
+                        &kernel_time, &user_time)) {
+        error_setg(errp, "Unable to get thread CPU usage");
+        return -1;
+    }
+
+    thread_time.LowPart = user_time.dwLowDateTime;
+    thread_time.HighPart = user_time.dwHighDateTime;
+
+    /* QuadPart is units of 100ns and we want ms as unit */
+    *val_ms = thread_time.QuadPart / 10000ll;
+    return 0;
+#elif defined(RUSAGE_THREAD)
+    struct rusage ru;
+    if (getrusage(RUSAGE_THREAD, &ru) < 0) {
+        error_setg_errno(errp, errno, "Unable to get thread CPU usage");
+        return -1;
+    }
+
+    *val_ms = ((ru.ru_utime.tv_sec * 1000ll) +
+               (ru.ru_utime.tv_usec / 1000));
+    return 0;
+#else
+    *val_ms = 0;
+    error_setg(errp, "Unable to calculate thread CPU usage on this platform");
+    return -1;
+#endif
+}
+
+int qcrypto_pbkdf2_count_iters(QCryptoHashAlgorithm hash,
+                               const uint8_t *key, size_t nkey,
+                               const uint8_t *salt, size_t nsalt,
+                               Error **errp)
+{
+    uint8_t out[32];
+    long long int iterations = (1 << 15);
+    unsigned long long delta_ms, start_ms, end_ms;
+
+    while (1) {
+        if (qcrypto_pbkdf2_get_thread_cpu(&start_ms, errp) < 0) {
+            return -1;
+        }
+        if (qcrypto_pbkdf2(hash,
+                           key, nkey,
+                           salt, nsalt,
+                           iterations,
+                           out, sizeof(out),
+                           errp) < 0) {
+            return -1;
+        }
+        if (qcrypto_pbkdf2_get_thread_cpu(&end_ms, errp) < 0) {
+            return -1;
+        }
+
+        delta_ms = end_ms - start_ms;
+
+        if (delta_ms > 500) {
+            break;
+        } else if (delta_ms < 100) {
+            iterations = iterations * 10;
+        } else {
+            iterations = (iterations * 1000 / delta_ms);
+        }
+    }
+
+    iterations = iterations * 1000 / delta_ms;
+
+    if (iterations > INT32_MAX) {
+        error_setg(errp, "Iterations %lld too large for a 32-bit int",
+                   iterations);
+        return -1;
+    }
+
+    return iterations;
+}
diff --git a/qemu/crypto/random-gcrypt.c b/qemu/crypto/random-gcrypt.c
new file mode 100644
index 000000000..0de9a096d
--- /dev/null
+++ b/qemu/crypto/random-gcrypt.c
@@ -0,0 +1,33 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+#include <gcrypt.h>
+
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp G_GNUC_UNUSED)
+{
+    gcry_randomize(buf, buflen, GCRY_STRONG_RANDOM);
+    return 0;
+}
diff --git a/qemu/crypto/random-gnutls.c b/qemu/crypto/random-gnutls.c
new file mode 100644
index 000000000..04b45a8f8
--- /dev/null
+++ b/qemu/crypto/random-gnutls.c
@@ -0,0 +1,43 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+#include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
+
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp)
+{
+    int ret;
+
+    ret = gnutls_rnd(GNUTLS_RND_RANDOM, buf, buflen);
+
+    if (ret < 0) {
+        error_setg(errp, "Cannot get random bytes: %s",
+                   gnutls_strerror(ret));
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/qemu/crypto/random-stub.c b/qemu/crypto/random-stub.c
new file mode 100644
index 000000000..63bbf4147
--- /dev/null
+++ b/qemu/crypto/random-stub.c
@@ -0,0 +1,31 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+int qcrypto_random_bytes(uint8_t *buf G_GNUC_UNUSED,
+                         size_t buflen G_GNUC_UNUSED,
+                         Error **errp)
+{
+    error_setg(errp, "No random byte source provided in this build");
+    return -1;
+}
diff --git a/qemu/crypto/secret.c b/qemu/crypto/secret.c
new file mode 100644
index 000000000..285ab7a63
--- /dev/null
+++ b/qemu/crypto/secret.c
@@ -0,0 +1,509 @@
+/*
+ * QEMU crypto secret support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/secret.h"
+#include "crypto/cipher.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "qemu/base64.h"
+#include "trace.h"
+
+
+static void
+qcrypto_secret_load_data(QCryptoSecret *secret,
+                         uint8_t **output,
+                         size_t *outputlen,
+                         Error **errp)
+{
+    char *data = NULL;
+    size_t length = 0;
+    GError *gerr = NULL;
+
+    *output = NULL;
+    *outputlen = 0;
+
+    if (secret->file) {
+        if (secret->data) {
+            error_setg(errp,
+                       "'file' and 'data' are mutually exclusive");
+            return;
+        }
+        if (!g_file_get_contents(secret->file, &data, &length, &gerr)) {
+            error_setg(errp,
+                       "Unable to read %s: %s",
+                       secret->file, gerr->message);
+            g_error_free(gerr);
+            return;
+        }
+        *output = (uint8_t *)data;
+        *outputlen = length;
+    } else if (secret->data) {
+        *outputlen = strlen(secret->data);
+        *output = (uint8_t *)g_strdup(secret->data);
+    } else {
+        error_setg(errp, "Either 'file' or 'data' must be provided");
+    }
+}
+
+
+static void qcrypto_secret_decrypt(QCryptoSecret *secret,
+                                   const uint8_t *input,
+                                   size_t inputlen,
+                                   uint8_t **output,
+                                   size_t *outputlen,
+                                   Error **errp)
+{
+    uint8_t *key = NULL, *ciphertext = NULL, *iv = NULL;
+    size_t keylen, ciphertextlen, ivlen;
+    QCryptoCipher *aes = NULL;
+    uint8_t *plaintext = NULL;
+
+    *output = NULL;
+    *outputlen = 0;
+
+    if (qcrypto_secret_lookup(secret->keyid,
+                              &key, &keylen,
+                              errp) < 0) {
+        goto cleanup;
+    }
+
+    if (keylen != 32) {
+        error_setg(errp, "Key should be 32 bytes in length");
+        goto cleanup;
+    }
+
+    if (!secret->iv) {
+        error_setg(errp, "IV is required to decrypt secret");
+        goto cleanup;
+    }
+
+    iv = qbase64_decode(secret->iv, -1, &ivlen, errp);
+    if (!iv) {
+        goto cleanup;
+    }
+    if (ivlen != 16) {
+        error_setg(errp, "IV should be 16 bytes in length not %zu",
+                   ivlen);
+        goto cleanup;
+    }
+
+    aes = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_256,
+                             QCRYPTO_CIPHER_MODE_CBC,
+                             key, keylen,
+                             errp);
+    if (!aes) {
+        goto cleanup;
+    }
+
+    if (qcrypto_cipher_setiv(aes, iv, ivlen, errp) < 0) {
+        goto cleanup;
+    }
+
+    if (secret->format == QCRYPTO_SECRET_FORMAT_BASE64) {
+        ciphertext = qbase64_decode((const gchar*)input,
+                                    inputlen,
+                                    &ciphertextlen,
+                                    errp);
+        if (!ciphertext) {
+            goto cleanup;
+        }
+        plaintext = g_new0(uint8_t, ciphertextlen + 1);
+    } else {
+        ciphertextlen = inputlen;
+        plaintext = g_new0(uint8_t, inputlen + 1);
+    }
+    if (qcrypto_cipher_decrypt(aes,
+                               ciphertext ? ciphertext : input,
+                               plaintext,
+                               ciphertextlen,
+                               errp) < 0) {
+        plaintext = NULL;
+        goto cleanup;
+    }
+
+    if (plaintext[ciphertextlen - 1] > 16 ||
+        plaintext[ciphertextlen - 1] > ciphertextlen) {
+        error_setg(errp, "Incorrect number of padding bytes (%d) "
+                   "found on decrypted data",
+                   (int)plaintext[ciphertextlen - 1]);
+        g_free(plaintext);
+        plaintext = NULL;
+        goto cleanup;
+    }
+
+    /* Even though plaintext may contain arbitrary NUL
+     * ensure it is explicitly NUL terminated.
+     */
+    ciphertextlen -= plaintext[ciphertextlen - 1];
+    plaintext[ciphertextlen] = '\0';
+
+    *output = plaintext;
+    *outputlen = ciphertextlen;
+
+ cleanup:
+    g_free(ciphertext);
+    g_free(iv);
+    g_free(key);
+    qcrypto_cipher_free(aes);
+}
+
+
+static void qcrypto_secret_decode(const uint8_t *input,
+                                  size_t inputlen,
+                                  uint8_t **output,
+                                  size_t *outputlen,
+                                  Error **errp)
+{
+    *output = qbase64_decode((const gchar*)input,
+                             inputlen,
+                             outputlen,
+                             errp);
+}
+
+
+static void
+qcrypto_secret_prop_set_loaded(Object *obj,
+                               bool value,
+                               Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    if (value) {
+        Error *local_err = NULL;
+        uint8_t *input = NULL;
+        size_t inputlen = 0;
+        uint8_t *output = NULL;
+        size_t outputlen = 0;
+
+        qcrypto_secret_load_data(secret, &input, &inputlen, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        if (secret->keyid) {
+            qcrypto_secret_decrypt(secret, input, inputlen,
+                                   &output, &outputlen, &local_err);
+            g_free(input);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+            input = output;
+            inputlen = outputlen;
+        } else {
+            if (secret->format != QCRYPTO_SECRET_FORMAT_RAW) {
+                qcrypto_secret_decode(input, inputlen,
+                                      &output, &outputlen, &local_err);
+                g_free(input);
+                if (local_err) {
+                    error_propagate(errp, local_err);
+                    return;
+                }
+                input = output;
+                inputlen = outputlen;
+            }
+        }
+
+        secret->rawdata = input;
+        secret->rawlen = inputlen;
+    } else {
+        g_free(secret->rawdata);
+        secret->rawlen = 0;
+    }
+}
+
+
+static bool
+qcrypto_secret_prop_get_loaded(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return secret->data != NULL;
+}
+
+
+static void
+qcrypto_secret_prop_set_format(Object *obj,
+                               int value,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *creds = QCRYPTO_SECRET(obj);
+
+    creds->format = value;
+}
+
+
+static int
+qcrypto_secret_prop_get_format(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *creds = QCRYPTO_SECRET(obj);
+
+    return creds->format;
+}
+
+
+static void
+qcrypto_secret_prop_set_data(Object *obj,
+                             const char *value,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->data);
+    secret->data = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_data(Object *obj,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->data);
+}
+
+
+static void
+qcrypto_secret_prop_set_file(Object *obj,
+                             const char *value,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->file);
+    secret->file = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_file(Object *obj,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->file);
+}
+
+
+static void
+qcrypto_secret_prop_set_iv(Object *obj,
+                           const char *value,
+                           Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->iv);
+    secret->iv = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_iv(Object *obj,
+                           Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->iv);
+}
+
+
+static void
+qcrypto_secret_prop_set_keyid(Object *obj,
+                              const char *value,
+                              Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->keyid);
+    secret->keyid = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_keyid(Object *obj,
+                              Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->keyid);
+}
+
+
+static void
+qcrypto_secret_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_secret_finalize(Object *obj)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->iv);
+    g_free(secret->file);
+    g_free(secret->keyid);
+    g_free(secret->rawdata);
+    g_free(secret->data);
+}
+
+static void
+qcrypto_secret_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_secret_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_secret_prop_get_loaded,
+                                   qcrypto_secret_prop_set_loaded,
+                                   NULL);
+    object_class_property_add_enum(oc, "format",
+                                   "QCryptoSecretFormat",
+                                   QCryptoSecretFormat_lookup,
+                                   qcrypto_secret_prop_get_format,
+                                   qcrypto_secret_prop_set_format,
+                                   NULL);
+    object_class_property_add_str(oc, "data",
+                                  qcrypto_secret_prop_get_data,
+                                  qcrypto_secret_prop_set_data,
+                                  NULL);
+    object_class_property_add_str(oc, "file",
+                                  qcrypto_secret_prop_get_file,
+                                  qcrypto_secret_prop_set_file,
+                                  NULL);
+    object_class_property_add_str(oc, "keyid",
+                                  qcrypto_secret_prop_get_keyid,
+                                  qcrypto_secret_prop_set_keyid,
+                                  NULL);
+    object_class_property_add_str(oc, "iv",
+                                  qcrypto_secret_prop_get_iv,
+                                  qcrypto_secret_prop_set_iv,
+                                  NULL);
+}
+
+
+int qcrypto_secret_lookup(const char *secretid,
+                          uint8_t **data,
+                          size_t *datalen,
+                          Error **errp)
+{
+    Object *obj;
+    QCryptoSecret *secret;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), secretid);
+    if (!obj) {
+        error_setg(errp, "No secret with id '%s'", secretid);
+        return -1;
+    }
+
+    secret = (QCryptoSecret *)
+        object_dynamic_cast(obj,
+                            TYPE_QCRYPTO_SECRET);
+    if (!secret) {
+        error_setg(errp, "Object with id '%s' is not a secret",
+                   secretid);
+        return -1;
+    }
+
+    if (!secret->rawdata) {
+        error_setg(errp, "Secret with id '%s' has no data",
+                   secretid);
+        return -1;
+    }
+
+    *data = g_new0(uint8_t, secret->rawlen + 1);
+    memcpy(*data, secret->rawdata, secret->rawlen);
+    (*data)[secret->rawlen] = '\0';
+    *datalen = secret->rawlen;
+
+    return 0;
+}
+
+
+char *qcrypto_secret_lookup_as_utf8(const char *secretid,
+                                    Error **errp)
+{
+    uint8_t *data;
+    size_t datalen;
+
+    if (qcrypto_secret_lookup(secretid,
+                              &data,
+                              &datalen,
+                              errp) < 0) {
+        return NULL;
+    }
+
+    if (!g_utf8_validate((const gchar*)data, datalen, NULL)) {
+        error_setg(errp,
+                   "Data from secret %s is not valid UTF-8",
+                   secretid);
+        g_free(data);
+        return NULL;
+    }
+
+    return (char *)data;
+}
+
+
+char *qcrypto_secret_lookup_as_base64(const char *secretid,
+                                      Error **errp)
+{
+    uint8_t *data;
+    size_t datalen;
+    char *ret;
+
+    if (qcrypto_secret_lookup(secretid,
+                              &data,
+                              &datalen,
+                              errp) < 0) {
+        return NULL;
+    }
+
+    ret = g_base64_encode(data, datalen);
+    g_free(data);
+    return ret;
+}
+
+
+static const TypeInfo qcrypto_secret_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QCRYPTO_SECRET,
+    .instance_size = sizeof(QCryptoSecret),
+    .instance_finalize = qcrypto_secret_finalize,
+    .class_size = sizeof(QCryptoSecretClass),
+    .class_init = qcrypto_secret_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_secret_register_types(void)
+{
+    type_register_static(&qcrypto_secret_info);
+}
+
+
+type_init(qcrypto_secret_register_types);
diff --git a/qemu/crypto/tlscreds.c b/qemu/crypto/tlscreds.c
new file mode 100644
index 000000000..1620e126a
--- /dev/null
+++ b/qemu/crypto/tlscreds.c
@@ -0,0 +1,259 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/tlscredspriv.h"
+#include "trace.h"
+
+#define DH_BITS 2048
+
+#ifdef CONFIG_GNUTLS
+int
+qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                     const char *filename,
+                                     gnutls_dh_params_t *dh_params,
+                                     Error **errp)
+{
+    int ret;
+
+    trace_qcrypto_tls_creds_load_dh(creds, filename ? filename : "<generated>");
+
+    if (filename == NULL) {
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_generate2(*dh_params, DH_BITS);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to generate DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+    } else {
+        GError *gerr = NULL;
+        gchar *contents;
+        gsize len;
+        gnutls_datum_t data;
+        if (!g_file_get_contents(filename,
+                                 &contents,
+                                 &len,
+                                 &gerr)) {
+
+            error_setg(errp, "%s", gerr->message);
+            g_error_free(gerr);
+            return -1;
+        }
+        data.data = (unsigned char *)contents;
+        data.size = len;
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            g_free(contents);
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_import_pkcs3(*dh_params,
+                                            &data,
+                                            GNUTLS_X509_FMT_PEM);
+        g_free(contents);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to load DH parameters from %s: %s",
+                       filename, gnutls_strerror(ret));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+int
+qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                           const char *filename,
+                           bool required,
+                           char **cred,
+                           Error **errp)
+{
+    struct stat sb;
+    int ret = -1;
+
+    if (!creds->dir) {
+        if (required) {
+            error_setg(errp, "Missing 'dir' property value");
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    *cred = g_strdup_printf("%s/%s", creds->dir, filename);
+
+    if (stat(*cred, &sb) < 0) {
+        if (errno == ENOENT && !required) {
+            ret = 0;
+        } else {
+            error_setg_errno(errp, errno,
+                             "Unable to access credentials %s",
+                             *cred);
+        }
+        g_free(*cred);
+        *cred = NULL;
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    trace_qcrypto_tls_creds_get_path(creds, filename,
+                                     *cred ? *cred : "<none>");
+    return ret;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_prop_set_verify(Object *obj,
+                                  bool value,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = value;
+}
+
+
+static bool
+qcrypto_tls_creds_prop_get_verify(Object *obj,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->verifyPeer;
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_dir(Object *obj,
+                               const char *value,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->dir = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_prop_get_dir(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return g_strdup(creds->dir);
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_endpoint(Object *obj,
+                                    int value,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->endpoint = value;
+}
+
+
+static int
+qcrypto_tls_creds_prop_get_endpoint(Object *obj,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->endpoint;
+}
+
+
+static void
+qcrypto_tls_creds_class_init(ObjectClass *oc, void *data)
+{
+    object_class_property_add_bool(oc, "verify-peer",
+                                   qcrypto_tls_creds_prop_get_verify,
+                                   qcrypto_tls_creds_prop_set_verify,
+                                   NULL);
+    object_class_property_add_str(oc, "dir",
+                                  qcrypto_tls_creds_prop_get_dir,
+                                  qcrypto_tls_creds_prop_set_dir,
+                                  NULL);
+    object_class_property_add_enum(oc, "endpoint",
+                                   "QCryptoTLSCredsEndpoint",
+                                   QCryptoTLSCredsEndpoint_lookup,
+                                   qcrypto_tls_creds_prop_get_endpoint,
+                                   qcrypto_tls_creds_prop_set_endpoint,
+                                   NULL);
+}
+
+
+static void
+qcrypto_tls_creds_init(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = true;
+}
+
+
+static void
+qcrypto_tls_creds_finalize(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    g_free(creds->dir);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QCRYPTO_TLS_CREDS,
+    .instance_size = sizeof(QCryptoTLSCreds),
+    .instance_init = qcrypto_tls_creds_init,
+    .instance_finalize = qcrypto_tls_creds_finalize,
+    .class_init = qcrypto_tls_creds_class_init,
+    .class_size = sizeof(QCryptoTLSCredsClass),
+    .abstract = true,
+};
+
+
+static void
+qcrypto_tls_creds_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_info);
+}
+
+
+type_init(qcrypto_tls_creds_register_types);
diff --git a/qemu/crypto/tlscredsanon.c b/qemu/crypto/tlscredsanon.c
new file mode 100644
index 000000000..146422008
--- /dev/null
+++ b/qemu/crypto/tlscredsanon.c
@@ -0,0 +1,219 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredspriv.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static int
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds,
+                            Error **errp)
+{
+    char *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_anon_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+
+        ret = gnutls_anon_allocate_server_credentials(&creds->data.server);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+
+        gnutls_anon_set_server_dh_params(creds->data.server,
+                                         creds->parent_obj.dh_params);
+    } else {
+        ret = gnutls_anon_allocate_client_credentials(&creds->data.client);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds)
+{
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+        if (creds->data.client) {
+            gnutls_anon_free_client_credentials(creds->data.client);
+            creds->data.client = NULL;
+        }
+    } else {
+        if (creds->data.server) {
+            gnutls_anon_free_server_credentials(creds->data.server);
+            creds->data.server = NULL;
+        }
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (value) {
+        qcrypto_tls_creds_anon_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_anon_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        return creds->data.server != NULL;
+    } else {
+        return creds->data.client != NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_anon_finalize(Object *obj)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    qcrypto_tls_creds_anon_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_anon_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_anon_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_tls_creds_anon_prop_get_loaded,
+                                   qcrypto_tls_creds_anon_prop_set_loaded,
+                                   NULL);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_anon_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_ANON,
+    .instance_size = sizeof(QCryptoTLSCredsAnon),
+    .instance_finalize = qcrypto_tls_creds_anon_finalize,
+    .class_size = sizeof(QCryptoTLSCredsAnonClass),
+    .class_init = qcrypto_tls_creds_anon_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_anon_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_anon_info);
+}
+
+
+type_init(qcrypto_tls_creds_anon_register_types);
diff --git a/qemu/crypto/tlscredspriv.h b/qemu/crypto/tlscredspriv.h
new file mode 100644
index 000000000..9222be4a9
--- /dev/null
+++ b/qemu/crypto/tlscredspriv.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU crypto TLS credential support private helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_PRIV_H__
+#define QCRYPTO_TLSCRED_PRIV_H__
+
+#include "crypto/tlscreds.h"
+
+#ifdef CONFIG_GNUTLS
+
+int qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                               const char *filename,
+                               bool required,
+                               char **cred,
+                               Error **errp);
+
+int qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                         const char *filename,
+                                         gnutls_dh_params_t *dh_params,
+                                         Error **errp);
+
+#endif
+
+#endif /* QCRYPTO_TLSCRED_PRIV_H__ */
+
diff --git a/qemu/crypto/tlscredsx509.c b/qemu/crypto/tlscredsx509.c
new file mode 100644
index 000000000..6a0179c2e
--- /dev/null
+++ b/qemu/crypto/tlscredsx509.c
@@ -0,0 +1,865 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlscredspriv.h"
+#include "crypto/secret.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+#include <gnutls/x509.h>
+
+
+static int
+qcrypto_tls_creds_check_cert_times(gnutls_x509_crt_t cert,
+                                   const char *certFile,
+                                   bool isServer,
+                                   bool isCA,
+                                   Error **errp)
+{
+    time_t now = time(NULL);
+
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "cannot get current time");
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s has expired" :
+                    (isServer ?
+                     "The server certificate %s has expired" :
+                     "The client certificate %s has expired")),
+                   certFile);
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_activation_time(cert) > now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s is not yet active" :
+                    (isServer ?
+                     "The server certificate %s is not yet active" :
+                     "The client certificate %s is not yet active")),
+                   certFile);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+/*
+ * The gnutls_x509_crt_get_basic_constraints function isn't
+ * available in GNUTLS 1.0.x branches. This isn't critical
+ * though, since gnutls_certificate_verify_peers2 will do
+ * pretty much the same check at runtime, so we can just
+ * disable this code
+ */
+static int
+qcrypto_tls_creds_check_cert_basic_constraints(QCryptoTLSCredsX509 *creds,
+                                               gnutls_x509_crt_t cert,
+                                               const char *certFile,
+                                               bool isServer,
+                                               bool isCA,
+                                               Error **errp)
+{
+    int status;
+
+    status = gnutls_x509_crt_get_basic_constraints(cert, NULL, NULL, NULL);
+    trace_qcrypto_tls_creds_x509_check_basic_constraints(
+        creds, certFile, status);
+
+    if (status > 0) { /* It is a CA cert */
+        if (!isCA) {
+            error_setg(errp, isServer ?
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a server" :
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a client",
+                       certFile);
+            return -1;
+        }
+    } else if (status == 0) { /* It is not a CA cert */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s basic constraints do not "
+                       "show a CA",
+                       certFile);
+            return -1;
+        }
+    } else if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+        /* Missing basicConstraints */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s is missing basic constraints "
+                       "for a CA",
+                       certFile);
+            return -1;
+        }
+    } else { /* General error */
+        error_setg(errp,
+                   "Unable to query certificate %s basic constraints: %s",
+                   certFile, gnutls_strerror(status));
+        return -1;
+    }
+
+    return 0;
+}
+#endif
+
+
+static int
+qcrypto_tls_creds_check_cert_key_usage(QCryptoTLSCredsX509 *creds,
+                                       gnutls_x509_crt_t cert,
+                                       const char *certFile,
+                                       bool isCA,
+                                       Error **errp)
+{
+    int status;
+    unsigned int usage = 0;
+    unsigned int critical = 0;
+
+    status = gnutls_x509_crt_get_key_usage(cert, &usage, &critical);
+    trace_qcrypto_tls_creds_x509_check_key_usage(
+        creds, certFile, status, usage, critical);
+
+    if (status < 0) {
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            usage = isCA ? GNUTLS_KEY_KEY_CERT_SIGN :
+                GNUTLS_KEY_DIGITAL_SIGNATURE|GNUTLS_KEY_KEY_ENCIPHERMENT;
+        } else {
+            error_setg(errp,
+                       "Unable to query certificate %s key usage: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+    }
+
+    if (isCA) {
+        if (!(usage & GNUTLS_KEY_KEY_CERT_SIGN)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit "
+                           "certificate signing", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!(usage & GNUTLS_KEY_DIGITAL_SIGNATURE)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit digital "
+                           "signature", certFile);
+                return -1;
+            }
+        }
+        if (!(usage & GNUTLS_KEY_KEY_ENCIPHERMENT)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit key "
+                           "encipherment", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_key_purpose(QCryptoTLSCredsX509 *creds,
+                                         gnutls_x509_crt_t cert,
+                                         const char *certFile,
+                                         bool isServer,
+                                         Error **errp)
+{
+    int status;
+    size_t i;
+    unsigned int purposeCritical;
+    unsigned int critical;
+    char *buffer = NULL;
+    size_t size;
+    bool allowClient = false, allowServer = false;
+
+    critical = 0;
+    for (i = 0; ; i++) {
+        size = 0;
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, NULL);
+
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+
+            /* If there is no data at all, then we must allow
+               client/server to pass */
+            if (i == 0) {
+                allowServer = allowClient = true;
+            }
+            break;
+        }
+        if (status != GNUTLS_E_SHORT_MEMORY_BUFFER) {
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+
+        buffer = g_new0(char, size);
+
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, &purposeCritical);
+
+        if (status < 0) {
+            trace_qcrypto_tls_creds_x509_check_key_purpose(
+                creds, certFile, status, "<none>", purposeCritical);
+            g_free(buffer);
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+        trace_qcrypto_tls_creds_x509_check_key_purpose(
+            creds, certFile, status, buffer, purposeCritical);
+        if (purposeCritical) {
+            critical = true;
+        }
+
+        if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_SERVER)) {
+            allowServer = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_CLIENT)) {
+            allowClient = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_ANY)) {
+            allowServer = allowClient = true;
+        }
+
+        g_free(buffer);
+        buffer = NULL;
+    }
+
+    if (isServer) {
+        if (!allowServer) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow "
+                           "use with a TLS server", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!allowClient) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow use "
+                           "with a TLS client", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert(QCryptoTLSCredsX509 *creds,
+                             gnutls_x509_crt_t cert,
+                             const char *certFile,
+                             bool isServer,
+                             bool isCA,
+                             Error **errp)
+{
+    if (qcrypto_tls_creds_check_cert_times(cert, certFile,
+                                           isServer, isCA,
+                                           errp) < 0) {
+        return -1;
+    }
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+    if (qcrypto_tls_creds_check_cert_basic_constraints(creds,
+                                                       cert, certFile,
+                                                       isServer, isCA,
+                                                       errp) < 0) {
+        return -1;
+    }
+#endif
+
+    if (qcrypto_tls_creds_check_cert_key_usage(creds,
+                                               cert, certFile,
+                                               isCA, errp) < 0) {
+        return -1;
+    }
+
+    if (!isCA &&
+        qcrypto_tls_creds_check_cert_key_purpose(creds,
+                                                 cert, certFile,
+                                                 isServer, errp) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_pair(gnutls_x509_crt_t cert,
+                                  const char *certFile,
+                                  gnutls_x509_crt_t *cacerts,
+                                  size_t ncacerts,
+                                  const char *cacertFile,
+                                  bool isServer,
+                                  Error **errp)
+{
+    unsigned int status;
+
+    if (gnutls_x509_crt_list_verify(&cert, 1,
+                                    cacerts, ncacerts,
+                                    NULL, 0,
+                                    0, &status) < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to verify server certificate %s against "
+                   "CA certificate %s" :
+                   "Unable to verify client certificate %s against "
+                   "CA certificate %s",
+                   certFile, cacertFile);
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+#ifndef GNUTLS_1_0_COMPAT
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+#endif
+
+        error_setg(errp,
+                   "Our own certificate %s failed validation against %s: %s",
+                   certFile, cacertFile, reason);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static gnutls_x509_crt_t
+qcrypto_tls_creds_load_cert(QCryptoTLSCredsX509 *creds,
+                            const char *certFile,
+                            bool isServer,
+                            Error **errp)
+{
+    gnutls_datum_t data;
+    gnutls_x509_crt_t cert = NULL;
+    char *buf = NULL;
+    gsize buflen;
+    GError *gerr;
+    int ret = -1;
+
+    trace_qcrypto_tls_creds_x509_load_cert(creds, isServer, certFile);
+
+    if (gnutls_x509_crt_init(&cert) < 0) {
+        error_setg(errp, "Unable to initialize certificate");
+        goto cleanup;
+    }
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    if (gnutls_x509_crt_import(cert, &data, GNUTLS_X509_FMT_PEM) < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to import server certificate %s" :
+                   "Unable to import client certificate %s",
+                   certFile);
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (ret != 0) {
+        gnutls_x509_crt_deinit(cert);
+        cert = NULL;
+    }
+    g_free(buf);
+    return cert;
+}
+
+
+static int
+qcrypto_tls_creds_load_ca_cert_list(QCryptoTLSCredsX509 *creds,
+                                    const char *certFile,
+                                    gnutls_x509_crt_t *certs,
+                                    unsigned int certMax,
+                                    size_t *ncerts,
+                                    Error **errp)
+{
+    gnutls_datum_t data;
+    char *buf = NULL;
+    gsize buflen;
+    int ret = -1;
+    GError *gerr = NULL;
+
+    *ncerts = 0;
+    trace_qcrypto_tls_creds_x509_load_cert_list(creds, certFile);
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    if (gnutls_x509_crt_list_import(certs, &certMax, &data,
+                                    GNUTLS_X509_FMT_PEM, 0) < 0) {
+        error_setg(errp,
+                   "Unable to import CA certificate list %s",
+                   certFile);
+        goto cleanup;
+    }
+    *ncerts = certMax;
+
+    ret = 0;
+
+ cleanup:
+    g_free(buf);
+    return ret;
+}
+
+
+#define MAX_CERTS 16
+static int
+qcrypto_tls_creds_x509_sanity_check(QCryptoTLSCredsX509 *creds,
+                                    bool isServer,
+                                    const char *cacertFile,
+                                    const char *certFile,
+                                    Error **errp)
+{
+    gnutls_x509_crt_t cert = NULL;
+    gnutls_x509_crt_t cacerts[MAX_CERTS];
+    size_t ncacerts = 0;
+    size_t i;
+    int ret = -1;
+
+    memset(cacerts, 0, sizeof(cacerts));
+    if (certFile &&
+        access(certFile, R_OK) == 0) {
+        cert = qcrypto_tls_creds_load_cert(creds,
+                                           certFile, isServer,
+                                           errp);
+        if (!cert) {
+            goto cleanup;
+        }
+    }
+    if (access(cacertFile, R_OK) == 0) {
+        if (qcrypto_tls_creds_load_ca_cert_list(creds,
+                                                cacertFile, cacerts,
+                                                MAX_CERTS, &ncacerts,
+                                                errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert &&
+        qcrypto_tls_creds_check_cert(creds,
+                                     cert, certFile, isServer,
+                                     false, errp) < 0) {
+        goto cleanup;
+    }
+
+    for (i = 0; i < ncacerts; i++) {
+        if (qcrypto_tls_creds_check_cert(creds,
+                                         cacerts[i], cacertFile,
+                                         isServer, true, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert && ncacerts &&
+        qcrypto_tls_creds_check_cert_pair(cert, certFile, cacerts,
+                                          ncacerts, cacertFile,
+                                          isServer, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (cert) {
+        gnutls_x509_crt_deinit(cert);
+    }
+    for (i = 0; i < ncacerts; i++) {
+        gnutls_x509_crt_deinit(cacerts[i]);
+    }
+    return ret;
+}
+
+
+static int
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds,
+                            Error **errp)
+{
+    char *cacert = NULL, *cacrl = NULL, *cert = NULL,
+        *key = NULL, *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_x509_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CRL,
+                                       false, &cacrl, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_CERT,
+                                       true, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_KEY,
+                                       true, &key, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+    } else {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_CERT,
+                                       false, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_KEY,
+                                       false, &key, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (creds->sanityCheck &&
+        qcrypto_tls_creds_x509_sanity_check(creds,
+            creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+            cacert, cert, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_allocate_credentials(&creds->data);
+    if (ret < 0) {
+        error_setg(errp, "Cannot allocate credentials: '%s'",
+                   gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_set_x509_trust_file(creds->data,
+                                                 cacert,
+                                                 GNUTLS_X509_FMT_PEM);
+    if (ret < 0) {
+        error_setg(errp, "Cannot load CA certificate '%s': %s",
+                   cacert, gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    if (cert != NULL && key != NULL) {
+#if GNUTLS_VERSION_NUMBER >= 0x030111
+        char *password = NULL;
+        if (creds->passwordid) {
+            password = qcrypto_secret_lookup_as_utf8(creds->passwordid,
+                                                     errp);
+            if (!password) {
+                goto cleanup;
+            }
+        }
+        ret = gnutls_certificate_set_x509_key_file2(creds->data,
+                                                    cert, key,
+                                                    GNUTLS_X509_FMT_PEM,
+                                                    password,
+                                                    0);
+        g_free(password);
+#else /* GNUTLS_VERSION_NUMBER < 0x030111 */
+        if (creds->passwordid) {
+            error_setg(errp, "PKCS8 decryption requires GNUTLS >= 3.1.11");
+            goto cleanup;
+        }
+        ret = gnutls_certificate_set_x509_key_file(creds->data,
+                                                   cert, key,
+                                                   GNUTLS_X509_FMT_PEM);
+#endif /* GNUTLS_VERSION_NUMBER < 0x030111 */
+        if (ret < 0) {
+            error_setg(errp, "Cannot load certificate '%s' & key '%s': %s",
+                       cert, key, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (cacrl != NULL) {
+        ret = gnutls_certificate_set_x509_crl_file(creds->data,
+                                                   cacrl,
+                                                   GNUTLS_X509_FMT_PEM);
+        if (ret < 0) {
+            error_setg(errp, "Cannot load CRL '%s': %s",
+                       cacrl, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+        gnutls_certificate_set_dh_params(creds->data,
+                                         creds->parent_obj.dh_params);
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(cacert);
+    g_free(cacrl);
+    g_free(cert);
+    g_free(key);
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds)
+{
+    if (creds->data) {
+        gnutls_certificate_free_credentials(creds->data);
+        creds->data = NULL;
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    if (value) {
+        qcrypto_tls_creds_x509_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_x509_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->data != NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_sanity(Object *obj,
+                                       bool value,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = value;
+}
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_passwordid(Object *obj,
+                                           const char *value,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->passwordid = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_x509_prop_get_passwordid(Object *obj,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return g_strdup(creds->passwordid);
+}
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_sanity(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->sanityCheck;
+}
+
+
+static void
+qcrypto_tls_creds_x509_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_x509_init(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = true;
+}
+
+
+static void
+qcrypto_tls_creds_x509_finalize(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    g_free(creds->passwordid);
+    qcrypto_tls_creds_x509_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_x509_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_x509_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_tls_creds_x509_prop_get_loaded,
+                                   qcrypto_tls_creds_x509_prop_set_loaded,
+                                   NULL);
+    object_class_property_add_bool(oc, "sanity-check",
+                                   qcrypto_tls_creds_x509_prop_get_sanity,
+                                   qcrypto_tls_creds_x509_prop_set_sanity,
+                                   NULL);
+    object_class_property_add_str(oc, "passwordid",
+                                  qcrypto_tls_creds_x509_prop_get_passwordid,
+                                  qcrypto_tls_creds_x509_prop_set_passwordid,
+                                  NULL);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_x509_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_X509,
+    .instance_size = sizeof(QCryptoTLSCredsX509),
+    .instance_init = qcrypto_tls_creds_x509_init,
+    .instance_finalize = qcrypto_tls_creds_x509_finalize,
+    .class_size = sizeof(QCryptoTLSCredsX509Class),
+    .class_init = qcrypto_tls_creds_x509_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_x509_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_x509_info);
+}
+
+
+type_init(qcrypto_tls_creds_x509_register_types);
diff --git a/qemu/crypto/tlssession.c b/qemu/crypto/tlssession.c
new file mode 100644
index 000000000..a543e5a57
--- /dev/null
+++ b/qemu/crypto/tlssession.c
@@ -0,0 +1,576 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlssession.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qapi/error.h"
+#include "qemu/acl.h"
+#include "trace.h"
+
+#ifdef CONFIG_GNUTLS
+
+
+#include <gnutls/x509.h>
+
+
+struct QCryptoTLSSession {
+    QCryptoTLSCreds *creds;
+    gnutls_session_t handle;
+    char *hostname;
+    char *aclname;
+    bool handshakeComplete;
+    QCryptoTLSSessionWriteFunc writeFunc;
+    QCryptoTLSSessionReadFunc readFunc;
+    void *opaque;
+    char *peername;
+};
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *session)
+{
+    if (!session) {
+        return;
+    }
+
+    gnutls_deinit(session->handle);
+    g_free(session->hostname);
+    g_free(session->peername);
+    g_free(session->aclname);
+    object_unref(OBJECT(session->creds));
+    g_free(session);
+}
+
+
+static ssize_t
+qcrypto_tls_session_push(void *opaque, const void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->writeFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->writeFunc(buf, len, session->opaque);
+}
+
+
+static ssize_t
+qcrypto_tls_session_pull(void *opaque, void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->readFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->readFunc(buf, len, session->opaque);
+}
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                        const char *hostname,
+                        const char *aclname,
+                        QCryptoTLSCredsEndpoint endpoint,
+                        Error **errp)
+{
+    QCryptoTLSSession *session;
+    int ret;
+
+    session = g_new0(QCryptoTLSSession, 1);
+    trace_qcrypto_tls_session_new(
+        session, creds, hostname ? hostname : "<none>",
+        aclname ? aclname : "<none>", endpoint);
+
+    if (hostname) {
+        session->hostname = g_strdup(hostname);
+    }
+    if (aclname) {
+        session->aclname = g_strdup(aclname);
+    }
+    session->creds = creds;
+    object_ref(OBJECT(creds));
+
+    if (creds->endpoint != endpoint) {
+        error_setg(errp, "Credentials endpoint doesn't match session");
+        goto error;
+    }
+
+    if (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        ret = gnutls_init(&session->handle, GNUTLS_SERVER);
+    } else {
+        ret = gnutls_init(&session->handle, GNUTLS_CLIENT);
+    }
+    if (ret < 0) {
+        error_setg(errp, "Cannot initialize TLS session: %s",
+                   gnutls_strerror(ret));
+        goto error;
+    }
+
+    if (object_dynamic_cast(OBJECT(creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        QCryptoTLSCredsAnon *acreds = QCRYPTO_TLS_CREDS_ANON(creds);
+
+        ret = gnutls_priority_set_direct(session->handle,
+                                         "NORMAL:+ANON-DH", NULL);
+        if (ret < 0) {
+            error_setg(errp, "Unable to set TLS session priority: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.server);
+        } else {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.client);
+        }
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+    } else if (object_dynamic_cast(OBJECT(creds),
+                                   TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        QCryptoTLSCredsX509 *tcreds = QCRYPTO_TLS_CREDS_X509(creds);
+
+        ret = gnutls_set_default_priority(session->handle);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set default TLS session priority: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+        ret = gnutls_credentials_set(session->handle,
+                                     GNUTLS_CRD_CERTIFICATE,
+                                     tcreds->data);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            /* This requests, but does not enforce a client cert.
+             * The cert checking code later does enforcement */
+            gnutls_certificate_server_set_request(session->handle,
+                                                  GNUTLS_CERT_REQUEST);
+        }
+    } else {
+        error_setg(errp, "Unsupported TLS credentials type %s",
+                   object_get_typename(OBJECT(creds)));
+        goto error;
+    }
+
+    gnutls_transport_set_ptr(session->handle, session);
+    gnutls_transport_set_push_function(session->handle,
+                                       qcrypto_tls_session_push);
+    gnutls_transport_set_pull_function(session->handle,
+                                       qcrypto_tls_session_pull);
+
+    return session;
+
+ error:
+    qcrypto_tls_session_free(session);
+    return NULL;
+}
+
+static int
+qcrypto_tls_session_check_certificate(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    int ret;
+    unsigned int status;
+    const gnutls_datum_t *certs;
+    unsigned int nCerts, i;
+    time_t now;
+    gnutls_x509_crt_t cert = NULL;
+
+    now = time(NULL);
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "Cannot get current time");
+        return -1;
+    }
+
+    ret = gnutls_certificate_verify_peers2(session->handle, &status);
+    if (ret < 0) {
+        error_setg(errp, "Verify failed: %s", gnutls_strerror(ret));
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+
+        error_setg(errp, "%s", reason);
+        return -1;
+    }
+
+    certs = gnutls_certificate_get_peers(session->handle, &nCerts);
+    if (!certs) {
+        error_setg(errp, "No certificate peers");
+        return -1;
+    }
+
+    for (i = 0; i < nCerts; i++) {
+        ret = gnutls_x509_crt_init(&cert);
+        if (ret < 0) {
+            error_setg(errp, "Cannot initialize certificate: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+
+        ret = gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER);
+        if (ret < 0) {
+            error_setg(errp, "Cannot import certificate: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+            error_setg(errp, "The certificate has expired");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (i == 0) {
+            size_t dnameSize = 1024;
+            session->peername = g_malloc(dnameSize);
+        requery:
+            ret = gnutls_x509_crt_get_dn(cert, session->peername, &dnameSize);
+            if (ret < 0) {
+                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                    session->peername = g_realloc(session->peername,
+                                                  dnameSize);
+                    goto requery;
+                }
+                error_setg(errp, "Cannot get client distinguished name: %s",
+                           gnutls_strerror(ret));
+                goto error;
+            }
+            if (session->aclname) {
+                qemu_acl *acl = qemu_acl_find(session->aclname);
+                int allow;
+                if (!acl) {
+                    error_setg(errp, "Cannot find ACL %s",
+                               session->aclname);
+                    goto error;
+                }
+
+                allow = qemu_acl_party_is_allowed(acl, session->peername);
+
+                if (!allow) {
+                    error_setg(errp, "TLS x509 ACL check for %s is denied",
+                               session->peername);
+                    goto error;
+                }
+            }
+            if (session->hostname) {
+                if (!gnutls_x509_crt_check_hostname(cert, session->hostname)) {
+                    error_setg(errp,
+                               "Certificate does not match the hostname %s",
+                               session->hostname);
+                    goto error;
+                }
+            }
+        }
+
+        gnutls_x509_crt_deinit(cert);
+    }
+
+    return 0;
+
+ error:
+    gnutls_x509_crt_deinit(cert);
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        return 0;
+    } else if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        if (session->creds->verifyPeer) {
+            return qcrypto_tls_session_check_certificate(session,
+                                                         errp);
+        } else {
+            return 0;
+        }
+    } else {
+        error_setg(errp, "Unexpected credential type %s",
+                   object_get_typename(OBJECT(session->creds)));
+        return -1;
+    }
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(QCryptoTLSSession *session,
+                                  QCryptoTLSSessionWriteFunc writeFunc,
+                                  QCryptoTLSSessionReadFunc readFunc,
+                                  void *opaque)
+{
+    session->writeFunc = writeFunc;
+    session->readFunc = readFunc;
+    session->opaque = opaque;
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *session,
+                          const char *buf,
+                          size_t len)
+{
+    ssize_t ret = gnutls_record_send(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *session,
+                         char *buf,
+                         size_t len)
+{
+    ssize_t ret = gnutls_record_recv(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *session,
+                              Error **errp)
+{
+    int ret = gnutls_handshake(session->handle);
+    if (ret == 0) {
+        session->handshakeComplete = true;
+    } else {
+        if (ret == GNUTLS_E_INTERRUPTED ||
+            ret == GNUTLS_E_AGAIN) {
+            ret = 1;
+        } else {
+            error_setg(errp, "TLS handshake failed: %s",
+                       gnutls_strerror(ret));
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *session)
+{
+    if (session->handshakeComplete) {
+        return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+    } else if (gnutls_record_get_direction(session->handle) == 0) {
+        return QCRYPTO_TLS_HANDSHAKE_RECVING;
+    } else {
+        return QCRYPTO_TLS_HANDSHAKE_SENDING;
+    }
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *session,
+                                 Error **errp)
+{
+    gnutls_cipher_algorithm_t cipher;
+    int ssf;
+
+    cipher = gnutls_cipher_get(session->handle);
+    ssf = gnutls_cipher_get_key_size(cipher);
+    if (!ssf) {
+        error_setg(errp, "Cannot get TLS cipher key size");
+        return -1;
+    }
+    return ssf;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *session)
+{
+    if (session->peername) {
+        return g_strdup(session->peername);
+    }
+    return NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds G_GNUC_UNUSED,
+                        const char *hostname G_GNUC_UNUSED,
+                        const char *aclname G_GNUC_UNUSED,
+                        QCryptoTLSCredsEndpoint endpoint G_GNUC_UNUSED,
+                        Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return NULL;
+}
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *sess G_GNUC_UNUSED)
+{
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess G_GNUC_UNUSED,
+                                      Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(
+    QCryptoTLSSession *sess G_GNUC_UNUSED,
+    QCryptoTLSSessionWriteFunc writeFunc G_GNUC_UNUSED,
+    QCryptoTLSSessionReadFunc readFunc G_GNUC_UNUSED,
+    void *opaque G_GNUC_UNUSED)
+{
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                          const char *buf,
+                          size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                         char *buf,
+                         size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                              Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess)
+{
+    return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                 Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess)
+{
+    return NULL;
+}
+
+#endif
diff --git a/qemu/crypto/xts.c b/qemu/crypto/xts.c
new file mode 100644
index 000000000..95212341f
--- /dev/null
+++ b/qemu/crypto/xts.c
@@ -0,0 +1,230 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
+static void xts_mult_x(uint8_t *I)
+{
+    int x;
+    uint8_t t, tt;
+
+    for (x = t = 0; x < 16; x++) {
+        tt = I[x] >> 7;
+        I[x] = ((I[x] << 1) | t) & 0xFF;
+        t = tt;
+    }
+    if (tt) {
+        I[0] ^= 0x87;
+    }
+}
+
+
+/**
+ * xts_tweak_uncrypt:
+ * @param ctxt: the cipher context
+ * @param func: the cipher function
+ * @src: buffer providing the cipher text of XTS_BLOCK_SIZE bytes
+ * @dst: buffer to output the plain text of XTS_BLOCK_SIZE bytes
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ *
+ * Decrypt data with a tweak
+ */
+static void xts_tweak_decrypt(const void *ctx,
+                              xts_cipher_func *func,
+                              const uint8_t *src,
+                              uint8_t *dst,
+                              uint8_t *iv)
+{
+    unsigned long x;
+
+    /* tweak encrypt block i */
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = src[x] ^ iv[x];
+    }
+
+    func(ctx, XTS_BLOCK_SIZE, dst, dst);
+
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = dst[x] ^ iv[x];
+    }
+
+    /* LFSR the tweak */
+    xts_mult_x(iv);
+}
+
+
+void xts_decrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src)
+{
+    uint8_t PP[XTS_BLOCK_SIZE], CC[XTS_BLOCK_SIZE], T[XTS_BLOCK_SIZE];
+    unsigned long i, m, mo, lim;
+
+    /* get number of blocks */
+    m = length >> 4;
+    mo = length & 15;
+
+    /* must have at least one full block */
+    g_assert(m != 0);
+
+    if (mo == 0) {
+        lim = m;
+    } else {
+        lim = m - 1;
+    }
+
+    /* encrypt the iv */
+    encfunc(tweakctx, XTS_BLOCK_SIZE, T, iv);
+
+    for (i = 0; i < lim; i++) {
+        xts_tweak_decrypt(datactx, decfunc, src, dst, T);
+
+        src += XTS_BLOCK_SIZE;
+        dst += XTS_BLOCK_SIZE;
+    }
+
+    /* if length is not a multiple of XTS_BLOCK_SIZE then */
+    if (mo > 0) {
+        memcpy(CC, T, XTS_BLOCK_SIZE);
+        xts_mult_x(CC);
+
+        /* PP = tweak decrypt block m-1 */
+        xts_tweak_decrypt(datactx, decfunc, src, PP, CC);
+
+        /* Pm = first length % XTS_BLOCK_SIZE bytes of PP */
+        for (i = 0; i < mo; i++) {
+            CC[i] = src[XTS_BLOCK_SIZE + i];
+            dst[XTS_BLOCK_SIZE + i] = PP[i];
+        }
+        for (; i < XTS_BLOCK_SIZE; i++) {
+            CC[i] = PP[i];
+        }
+
+        /* Pm-1 = Tweak uncrypt CC */
+        xts_tweak_decrypt(datactx, decfunc, CC, dst, T);
+    }
+
+    /* Decrypt the iv back */
+    decfunc(tweakctx, XTS_BLOCK_SIZE, iv, T);
+}
+
+
+/**
+ * xts_tweak_crypt:
+ * @param ctxt: the cipher context
+ * @param func: the cipher function
+ * @src: buffer providing the plain text of XTS_BLOCK_SIZE bytes
+ * @dst: buffer to output the cipher text of XTS_BLOCK_SIZE bytes
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ *
+ * Encrypt data with a tweak
+ */
+static void xts_tweak_encrypt(const void *ctx,
+                              xts_cipher_func *func,
+                              const uint8_t *src,
+                              uint8_t *dst,
+                              uint8_t *iv)
+{
+    unsigned long x;
+
+    /* tweak encrypt block i */
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = src[x] ^ iv[x];
+    }
+
+    func(ctx, XTS_BLOCK_SIZE, dst, dst);
+
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = dst[x] ^ iv[x];
+    }
+
+    /* LFSR the tweak */
+    xts_mult_x(iv);
+}
+
+
+void xts_encrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src)
+{
+    uint8_t PP[XTS_BLOCK_SIZE], CC[XTS_BLOCK_SIZE], T[XTS_BLOCK_SIZE];
+    unsigned long i, m, mo, lim;
+
+    /* get number of blocks */
+    m = length >> 4;
+    mo = length & 15;
+
+    /* must have at least one full block */
+    g_assert(m != 0);
+
+    if (mo == 0) {
+        lim = m;
+    } else {
+        lim = m - 1;
+    }
+
+    /* encrypt the iv */
+    encfunc(tweakctx, XTS_BLOCK_SIZE, T, iv);
+
+    for (i = 0; i < lim; i++) {
+        xts_tweak_encrypt(datactx, encfunc, src, dst, T);
+
+        dst += XTS_BLOCK_SIZE;
+        src += XTS_BLOCK_SIZE;
+    }
+
+    /* if length is not a multiple of XTS_BLOCK_SIZE then */
+    if (mo > 0) {
+        /* CC = tweak encrypt block m-1 */
+        xts_tweak_encrypt(datactx, encfunc, src, CC, T);
+
+        /* Cm = first length % XTS_BLOCK_SIZE bytes of CC */
+        for (i = 0; i < mo; i++) {
+            PP[i] = src[XTS_BLOCK_SIZE + i];
+            dst[XTS_BLOCK_SIZE + i] = CC[i];
+        }
+
+        for (; i < XTS_BLOCK_SIZE; i++) {
+            PP[i] = CC[i];
+        }
+
+        /* Cm-1 = Tweak encrypt PP */
+        xts_tweak_encrypt(datactx, encfunc, PP, dst, T);
+    }
+
+    /* Decrypt the iv back */
+    decfunc(tweakctx, XTS_BLOCK_SIZE, iv, T);
+}
diff --git a/qemu/default-configs/aarch64-linux-user.mak b/qemu/default-configs/aarch64-linux-user.mak
index 3df7de5b8..0a5b08a00 100644
--- a/qemu/default-configs/aarch64-linux-user.mak
+++ b/qemu/default-configs/aarch64-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for aarch64-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/qemu/default-configs/arm-softmmu.mak b/qemu/default-configs/arm-softmmu.mak
index 74f1db3a0..c63cdd073 100644
--- a/qemu/default-configs/arm-softmmu.mak
+++ b/qemu/default-configs/arm-softmmu.mak
@@ -28,6 +28,7 @@ CONFIG_SSI_M25P80=y
 CONFIG_LAN9118=y
 CONFIG_SMC91C111=y
 CONFIG_ALLWINNER_EMAC=y
+CONFIG_IMX_FEC=y
 CONFIG_DS1338=y
 CONFIG_PFLASH_CFI01=y
 CONFIG_PFLASH_CFI02=y
@@ -78,6 +79,7 @@ CONFIG_TUSB6010=y
 CONFIG_IMX=y
 CONFIG_MAINSTONE=y
 CONFIG_NSERIES=y
+CONFIG_RASPI=y
 CONFIG_REALVIEW=y
 CONFIG_ZAURUS=y
 CONFIG_ZYNQ=y
@@ -98,7 +100,15 @@ CONFIG_ALLWINNER_A10_PIT=y
 CONFIG_ALLWINNER_A10_PIC=y
 CONFIG_ALLWINNER_A10=y
 
+CONFIG_FSL_IMX31=y
+CONFIG_FSL_IMX25=y
+
+CONFIG_IMX_I2C=y
+
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
 CONFIG_ACPI=y
+CONFIG_SMBIOS=y
+CONFIG_ASPEED_SOC=y
+CONFIG_GPIO_KEY=y
diff --git a/qemu/default-configs/i386-softmmu.mak b/qemu/default-configs/i386-softmmu.mak
index 48b57623f..b177e5210 100644
--- a/qemu/default-configs/i386-softmmu.mak
+++ b/qemu/default-configs/i386-softmmu.mak
@@ -7,7 +7,13 @@ CONFIG_QXL=$(CONFIG_SPICE)
 CONFIG_VGA_ISA=y
 CONFIG_VGA_CIRRUS=y
 CONFIG_VMWARE_VGA=y
+CONFIG_VIRTIO_VGA=y
 CONFIG_VMMOUSE=y
+CONFIG_IPMI=y
+CONFIG_IPMI_LOCAL=y
+CONFIG_IPMI_EXTERN=y
+CONFIG_ISA_IPMI_KCS=y
+CONFIG_ISA_IPMI_BT=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -43,9 +49,12 @@ CONFIG_LPC_ICH9=y
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
+CONFIG_NVDIMM=y
+CONFIG_ACPI_NVDIMM=y
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
+CONFIG_HYPERV_TESTDEV=$(CONFIG_KVM)
diff --git a/qemu/default-configs/mips-softmmu-common.mak b/qemu/default-configs/mips-softmmu-common.mak
new file mode 100644
index 000000000..0394514b9
--- /dev/null
+++ b/qemu/default-configs/mips-softmmu-common.mak
@@ -0,0 +1,34 @@
+# Common mips*-softmmu CONFIG defines
+
+include pci.mak
+include sound.mak
+include usb.mak
+CONFIG_ESP=y
+CONFIG_VGA_ISA=y
+CONFIG_VGA_ISA_MM=y
+CONFIG_VGA_CIRRUS=y
+CONFIG_VMWARE_VGA=y
+CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
+CONFIG_I8254=y
+CONFIG_PCSPK=y
+CONFIG_PCKBD=y
+CONFIG_FDC=y
+CONFIG_ACPI=y
+CONFIG_ACPI_X86=y
+CONFIG_ACPI_MEMORY_HOTPLUG=y
+CONFIG_ACPI_CPU_HOTPLUG=y
+CONFIG_APM=y
+CONFIG_I8257=y
+CONFIG_PIIX4=y
+CONFIG_IDE_ISA=y
+CONFIG_IDE_PIIX=y
+CONFIG_NE2000_ISA=y
+CONFIG_MIPSNET=y
+CONFIG_PFLASH_CFI01=y
+CONFIG_I8259=y
+CONFIG_MC146818RTC=y
+CONFIG_ISA_TESTDEV=y
+CONFIG_EMPTY_SLOT=y
+CONFIG_MIPS_CPS=y
+CONFIG_MIPS_ITU=y
diff --git a/qemu/default-configs/mips-softmmu.mak b/qemu/default-configs/mips-softmmu.mak
index 44467c37c..9fede6e00 100644
--- a/qemu/default-configs/mips-softmmu.mak
+++ b/qemu/default-configs/mips-softmmu.mak
@@ -1,32 +1,3 @@
 # Default configuration for mips-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_ACPI_X86=y
-CONFIG_ACPI_MEMORY_HOTPLUG=y
-CONFIG_ACPI_CPU_HOTPLUG=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
-CONFIG_I8259=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
+include mips-softmmu-common.mak
diff --git a/qemu/default-configs/mips64-softmmu.mak b/qemu/default-configs/mips64-softmmu.mak
index 66ed5f94c..bad749667 100644
--- a/qemu/default-configs/mips64-softmmu.mak
+++ b/qemu/default-configs/mips64-softmmu.mak
@@ -1,38 +1,9 @@
 # Default configuration for mips64-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_ACPI_X86=y
-CONFIG_ACPI_MEMORY_HOTPLUG=y
-CONFIG_ACPI_CPU_HOTPLUG=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
+include mips-softmmu-common.mak
 CONFIG_RC4030=y
 CONFIG_DP8393X=y
 CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
 CONFIG_JAZZ=y
 CONFIG_G364FB=y
-CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
diff --git a/qemu/default-configs/mips64el-softmmu.mak b/qemu/default-configs/mips64el-softmmu.mak
index bfca2b2b7..485e218cf 100644
--- a/qemu/default-configs/mips64el-softmmu.mak
+++ b/qemu/default-configs/mips64el-softmmu.mak
@@ -1,41 +1,12 @@
 # Default configuration for mips64el-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_ACPI_X86=y
-CONFIG_ACPI_MEMORY_HOTPLUG=y
-CONFIG_ACPI_CPU_HOTPLUG=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
+include mips-softmmu-common.mak
 CONFIG_IDE_VIA=y
-CONFIG_NE2000_ISA=y
 CONFIG_RC4030=y
 CONFIG_DP8393X=y
 CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
 CONFIG_FULONG=y
 CONFIG_JAZZ=y
 CONFIG_G364FB=y
-CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
 CONFIG_VT82C686=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
diff --git a/qemu/default-configs/mipsel-softmmu.mak b/qemu/default-configs/mipsel-softmmu.mak
index 0162ef024..a7f605948 100644
--- a/qemu/default-configs/mipsel-softmmu.mak
+++ b/qemu/default-configs/mipsel-softmmu.mak
@@ -1,32 +1,3 @@
 # Default configuration for mipsel-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_ACPI_X86=y
-CONFIG_ACPI_MEMORY_HOTPLUG=y
-CONFIG_ACPI_CPU_HOTPLUG=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
-CONFIG_I8259=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
+include mips-softmmu-common.mak
diff --git a/qemu/default-configs/pci.mak b/qemu/default-configs/pci.mak
index 7e10903ba..9c8bc68c4 100644
--- a/qemu/default-configs/pci.mak
+++ b/qemu/default-configs/pci.mak
@@ -15,6 +15,7 @@ CONFIG_ES1370=y
 CONFIG_LSI_SCSI_PCI=y
 CONFIG_VMW_PVSCSI_SCSI_PCI=y
 CONFIG_MEGASAS_SCSI_PCI=y
+CONFIG_MPTSAS_SCSI_PCI=y
 CONFIG_RTL8139_PCI=y
 CONFIG_E1000_PCI=y
 CONFIG_VMXNET3_PCI=y
@@ -35,5 +36,5 @@ CONFIG_SDHCI=y
 CONFIG_EDU=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
-CONFIG_IVSHMEM=$(CONFIG_KVM)
+CONFIG_IVSHMEM=$(CONFIG_EVENTFD)
 CONFIG_ROCKER=y
diff --git a/qemu/default-configs/ppc64-softmmu.mak b/qemu/default-configs/ppc64-softmmu.mak
index ab62cc7ff..bb71b23ee 100644
--- a/qemu/default-configs/ppc64-softmmu.mak
+++ b/qemu/default-configs/ppc64-softmmu.mak
@@ -3,6 +3,7 @@
 include pci.mak
 include sound.mak
 include usb.mak
+CONFIG_VIRTIO_VGA=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
@@ -52,3 +53,4 @@ CONFIG_XICS_KVM=$(and $(CONFIG_PSERIES),$(CONFIG_KVM))
 # For PReP
 CONFIG_MC146818RTC=y
 CONFIG_ISA_TESTDEV=y
+CONFIG_MEM_HOTPLUG=y
diff --git a/qemu/default-configs/tilegx-linux-user.mak b/qemu/default-configs/tilegx-linux-user.mak
new file mode 100644
index 000000000..3e47493af
--- /dev/null
+++ b/qemu/default-configs/tilegx-linux-user.mak
@@ -0,0 +1 @@
+# Default configuration for tilegx-linux-user
diff --git a/qemu/default-configs/x86_64-softmmu.mak b/qemu/default-configs/x86_64-softmmu.mak
index 4962ed70a..6e3b312c5 100644
--- a/qemu/default-configs/x86_64-softmmu.mak
+++ b/qemu/default-configs/x86_64-softmmu.mak
@@ -9,6 +9,11 @@ CONFIG_VGA_CIRRUS=y
 CONFIG_VMWARE_VGA=y
 CONFIG_VIRTIO_VGA=y
 CONFIG_VMMOUSE=y
+CONFIG_IPMI=y
+CONFIG_IPMI_LOCAL=y
+CONFIG_IPMI_EXTERN=y
+CONFIG_ISA_IPMI_KCS=y
+CONFIG_ISA_IPMI_BT=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -44,9 +49,12 @@ CONFIG_LPC_ICH9=y
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
+CONFIG_NVDIMM=y
+CONFIG_ACPI_NVDIMM=y
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
+CONFIG_HYPERV_TESTDEV=$(CONFIG_KVM)
diff --git a/qemu/device-hotplug.c b/qemu/device-hotplug.c
index 68b94967c..126f73c67 100644
--- a/qemu/device-hotplug.c
+++ b/qemu/device-hotplug.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
@@ -29,6 +30,7 @@
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "block/block_int.h"
 
 static DriveInfo *add_init_drive(const char *optstr)
 {
@@ -54,6 +56,12 @@ void hmp_drive_add(Monitor *mon, const QDict *qdict)
 {
     DriveInfo *dinfo = NULL;
     const char *opts = qdict_get_str(qdict, "opts");
+    bool node = qdict_get_try_bool(qdict, "node", false);
+
+    if (node) {
+        hmp_drive_add_node(mon, opts);
+        return;
+    }
 
     dinfo = add_init_drive(opts);
     if (!dinfo) {
@@ -76,6 +84,8 @@ void hmp_drive_add(Monitor *mon, const QDict *qdict)
 
 err:
     if (dinfo) {
-        blk_unref(blk_by_legacy_dinfo(dinfo));
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+        monitor_remove_blk(blk);
+        blk_unref(blk);
     }
 }
diff --git a/qemu/device_tree.c b/qemu/device_tree.c
index a9f5f8e59..ccba1fd4a 100644
--- a/qemu/device_tree.c
+++ b/qemu/device_tree.c
@@ -11,13 +11,13 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
+#ifdef CONFIG_LINUX
+#include <dirent.h>
+#endif
+
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "sysemu/device_tree.h"
@@ -117,6 +117,102 @@ fail:
     return NULL;
 }
 
+#ifdef CONFIG_LINUX
+
+#define SYSFS_DT_BASEDIR "/proc/device-tree"
+
+/**
+ * read_fstree: this function is inspired from dtc read_fstree
+ * @fdt: preallocated fdt blob buffer, to be populated
+ * @dirname: directory to scan under SYSFS_DT_BASEDIR
+ * the search is recursive and the tree is searched down to the
+ * leaves (property files).
+ *
+ * the function asserts in case of error
+ */
+static void read_fstree(void *fdt, const char *dirname)
+{
+    DIR *d;
+    struct dirent *de;
+    struct stat st;
+    const char *root_dir = SYSFS_DT_BASEDIR;
+    const char *parent_node;
+
+    if (strstr(dirname, root_dir) != dirname) {
+        error_setg(&error_fatal, "%s: %s must be searched within %s",
+                   __func__, dirname, root_dir);
+    }
+    parent_node = &dirname[strlen(SYSFS_DT_BASEDIR)];
+
+    d = opendir(dirname);
+    if (!d) {
+        error_setg(&error_fatal, "%s cannot open %s", __func__, dirname);
+    }
+
+    while ((de = readdir(d)) != NULL) {
+        char *tmpnam;
+
+        if (!g_strcmp0(de->d_name, ".")
+            || !g_strcmp0(de->d_name, "..")) {
+            continue;
+        }
+
+        tmpnam = g_strdup_printf("%s/%s", dirname, de->d_name);
+
+        if (lstat(tmpnam, &st) < 0) {
+            error_setg(&error_fatal, "%s cannot lstat %s", __func__, tmpnam);
+        }
+
+        if (S_ISREG(st.st_mode)) {
+            gchar *val;
+            gsize len;
+
+            if (!g_file_get_contents(tmpnam, &val, &len, NULL)) {
+                error_setg(&error_fatal, "%s not able to extract info from %s",
+                           __func__, tmpnam);
+            }
+
+            if (strlen(parent_node) > 0) {
+                qemu_fdt_setprop(fdt, parent_node,
+                                 de->d_name, val, len);
+            } else {
+                qemu_fdt_setprop(fdt, "/", de->d_name, val, len);
+            }
+            g_free(val);
+        } else if (S_ISDIR(st.st_mode)) {
+            char *node_name;
+
+            node_name = g_strdup_printf("%s/%s",
+                                        parent_node, de->d_name);
+            qemu_fdt_add_subnode(fdt, node_name);
+            g_free(node_name);
+            read_fstree(fdt, tmpnam);
+        }
+
+        g_free(tmpnam);
+    }
+
+    closedir(d);
+}
+
+/* load_device_tree_from_sysfs: extract the dt blob from host sysfs */
+void *load_device_tree_from_sysfs(void)
+{
+    void *host_fdt;
+    int host_fdt_size;
+
+    host_fdt = create_device_tree(&host_fdt_size);
+    read_fstree(host_fdt, SYSFS_DT_BASEDIR);
+    if (fdt_check_header(host_fdt)) {
+        error_setg(&error_fatal,
+                   "%s host device tree extracted into memory is invalid",
+                   __func__);
+    }
+    return host_fdt;
+}
+
+#endif /* CONFIG_LINUX */
+
 static int findnode_nofail(void *fdt, const char *node_path)
 {
     int offset;
@@ -131,6 +227,60 @@ static int findnode_nofail(void *fdt, const char *node_path)
     return offset;
 }
 
+char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
+                          Error **errp)
+{
+    int offset, len, ret;
+    const char *iter_name;
+    unsigned int path_len = 16, n = 0;
+    GSList *path_list = NULL, *iter;
+    char **path_array;
+
+    offset = fdt_node_offset_by_compatible(fdt, -1, compat);
+
+    while (offset >= 0) {
+        iter_name = fdt_get_name(fdt, offset, &len);
+        if (!iter_name) {
+            offset = len;
+            break;
+        }
+        if (!strcmp(iter_name, name)) {
+            char *path;
+
+            path = g_malloc(path_len);
+            while ((ret = fdt_get_path(fdt, offset, path, path_len))
+                  == -FDT_ERR_NOSPACE) {
+                path_len += 16;
+                path = g_realloc(path, path_len);
+            }
+            path_list = g_slist_prepend(path_list, path);
+            n++;
+        }
+        offset = fdt_node_offset_by_compatible(fdt, offset, compat);
+    }
+
+    if (offset < 0 && offset != -FDT_ERR_NOTFOUND) {
+        error_setg(errp, "%s: abort parsing dt for %s/%s: %s",
+                   __func__, name, compat, fdt_strerror(offset));
+        for (iter = path_list; iter; iter = iter->next) {
+            g_free(iter->data);
+        }
+        g_slist_free(path_list);
+        return NULL;
+    }
+
+    path_array = g_new(char *, n + 1);
+    path_array[n--] = NULL;
+
+    for (iter = path_list; iter; iter = iter->next) {
+        path_array[n--] = iter->data;
+    }
+
+    g_slist_free(path_list);
+
+    return path_array;
+}
+
 int qemu_fdt_setprop(void *fdt, const char *node_path,
                      const char *property, const void *val, int size)
 {
@@ -184,31 +334,39 @@ int qemu_fdt_setprop_string(void *fdt, const char *node_path,
 }
 
 const void *qemu_fdt_getprop(void *fdt, const char *node_path,
-                             const char *property, int *lenp)
+                             const char *property, int *lenp, Error **errp)
 {
     int len;
     const void *r;
+
     if (!lenp) {
         lenp = &len;
     }
     r = fdt_getprop(fdt, findnode_nofail(fdt, node_path), property, lenp);
     if (!r) {
-        error_report("%s: Couldn't get %s/%s: %s", __func__,
-                     node_path, property, fdt_strerror(*lenp));
-        exit(1);
+        error_setg(errp, "%s: Couldn't get %s/%s: %s", __func__,
+                  node_path, property, fdt_strerror(*lenp));
     }
     return r;
 }
 
 uint32_t qemu_fdt_getprop_cell(void *fdt, const char *node_path,
-                               const char *property)
+                               const char *property, int *lenp, Error **errp)
 {
     int len;
-    const uint32_t *p = qemu_fdt_getprop(fdt, node_path, property, &len);
-    if (len != 4) {
-        error_report("%s: %s/%s not 4 bytes long (not a cell?)",
-                     __func__, node_path, property);
-        exit(1);
+    const uint32_t *p;
+
+    if (!lenp) {
+        lenp = &len;
+    }
+    p = qemu_fdt_getprop(fdt, node_path, property, lenp, errp);
+    if (!p) {
+        return 0;
+    } else if (*lenp != 4) {
+        error_setg(errp, "%s: %s/%s not 4 bytes long (not a cell?)",
+                   __func__, node_path, property);
+        *lenp = -EINVAL;
+        return 0;
     }
     return be32_to_cpu(*p);
 }
diff --git a/qemu/disas.c b/qemu/disas.c
index 69a606691..05a7a1260 100644
--- a/qemu/disas.c
+++ b/qemu/disas.c
@@ -1,9 +1,8 @@
 /* General "disassemble this chunk" code.  Used for debugging. */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 #include "elf.h"
-#include <errno.h>
 
 #include "cpu.h"
 #include "disas/disas.h"
@@ -72,14 +71,6 @@ generic_print_address (bfd_vma addr, struct disassemble_info *info)
     (*info->fprintf_func) (info->stream, "0x%" PRIx64, addr);
 }
 
-/* Print address in hex, truncated to the width of a target virtual address. */
-static void
-generic_print_target_address(bfd_vma addr, struct disassemble_info *info)
-{
-    uint64_t mask = ~0ULL >> (64 - TARGET_VIRT_ADDR_SPACE_BITS);
-    generic_print_address(addr & mask, info);
-}
-
 /* Print address in hex, truncated to the width of a host virtual address. */
 static void
 generic_print_host_address(bfd_vma addr, struct disassemble_info *info)
@@ -201,7 +192,7 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
     s.info.read_memory_func = target_read_memory;
     s.info.buffer_vma = code;
     s.info.buffer_length = size;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
 #ifdef TARGET_WORDS_BIGENDIAN
     s.info.endian = BFD_ENDIAN_BIG;
@@ -222,11 +213,6 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
         s.info.mach = bfd_mach_i386_i386;
     }
     s.info.print_insn = print_insn_i386;
-#elif defined(TARGET_SPARC)
-    s.info.print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
 #elif defined(TARGET_PPC)
     if ((flags >> 16) & 1) {
         s.info.endian = BFD_ENDIAN_LITTLE;
@@ -243,29 +229,6 @@ void target_disas(FILE *out, CPUState *cpu, target_ulong code,
     }
     s.info.disassembler_options = (char *)"any";
     s.info.print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    s.info.print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    s.info.print_insn = print_insn_big_mips;
-#else
-    s.info.print_insn = print_insn_little_mips;
-#endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    s.info.print_insn = print_insn_sh;
-#elif defined(TARGET_ALPHA)
-    s.info.mach = bfd_mach_alpha_ev6;
-    s.info.print_insn = print_insn_alpha;
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    s.info.print_insn = print_insn_s390;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    s.info.print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    s.info.print_insn = print_insn_lm32;
 #endif
     if (s.info.print_insn == NULL) {
         s.info.print_insn = print_insn_od_target;
@@ -400,16 +363,6 @@ monitor_read_memory (bfd_vma memaddr, bfd_byte *myaddr, int length,
     return 0;
 }
 
-static int GCC_FMT_ATTR(2, 3)
-monitor_fprintf(FILE *stream, const char *fmt, ...)
-{
-    va_list ap;
-    va_start(ap, fmt);
-    monitor_vprintf((Monitor *)stream, fmt, ap);
-    va_end(ap);
-    return 0;
-}
-
 /* Disassembler for the monitor.
    See target_disas for a description of flags. */
 void monitor_disas(Monitor *mon, CPUState *cpu,
@@ -424,7 +377,7 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
     s.cpu = cpu;
     monitor_disas_is_physical = is_physical;
     s.info.read_memory_func = monitor_read_memory;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
     s.info.buffer_vma = pc;
 
@@ -447,13 +400,6 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
         s.info.mach = bfd_mach_i386_i386;
     }
     s.info.print_insn = print_insn_i386;
-#elif defined(TARGET_ALPHA)
-    s.info.print_insn = print_insn_alpha;
-#elif defined(TARGET_SPARC)
-    s.info.print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
 #elif defined(TARGET_PPC)
     if (flags & 0xFFFF) {
         /* If we have a precise definition of the instruction set, use it. */
@@ -469,26 +415,6 @@ void monitor_disas(Monitor *mon, CPUState *cpu,
         s.info.endian = BFD_ENDIAN_LITTLE;
     }
     s.info.print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    s.info.print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    s.info.print_insn = print_insn_big_mips;
-#else
-    s.info.print_insn = print_insn_little_mips;
-#endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    s.info.print_insn = print_insn_sh;
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    s.info.print_insn = print_insn_s390;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    s.info.print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    s.info.print_insn = print_insn_lm32;
 #endif
     if (!s.info.print_insn) {
         monitor_printf(mon, "0x" TARGET_FMT_lx
diff --git a/qemu/disas/Makefile.objs b/qemu/disas/Makefile.objs
index 8dae4daec..abeba8466 100644
--- a/qemu/disas/Makefile.objs
+++ b/qemu/disas/Makefile.objs
@@ -4,7 +4,10 @@ common-obj-$(CONFIG_ARM_DIS) += arm.o
 common-obj-$(CONFIG_ARM_A64_DIS) += arm-a64.o
 common-obj-$(CONFIG_ARM_A64_DIS) += libvixl/
 libvixldir = $(SRC_PATH)/disas/libvixl
-arm-a64.o-cflags := -I$(libvixldir)
+# The -Wno-sign-compare is needed only for gcc 4.6, which complains about
+# some signed-unsigned equality comparisons in libvixl which later gcc
+# versions do not.
+arm-a64.o-cflags := -I$(libvixldir) -Wno-sign-compare
 common-obj-$(CONFIG_CRIS_DIS) += cris.o
 common-obj-$(CONFIG_HPPA_DIS) += hppa.o
 common-obj-$(CONFIG_I386_DIS) += i386.o
diff --git a/qemu/disas/alpha.c b/qemu/disas/alpha.c
index a950b9cee..44d00a363 100644
--- a/qemu/disas/alpha.c
+++ b/qemu/disas/alpha.c
@@ -19,7 +19,7 @@ You should have received a copy of the GNU General Public License
 along with this file; see the file COPYING.  If not, see
 <http://www.gnu.org/licenses/>. */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* MAX is redefined below, so remove any previous definition. */
diff --git a/qemu/disas/arm-a64.cc b/qemu/disas/arm-a64.cc
index b57256b26..9280950ce 100644
--- a/qemu/disas/arm-a64.cc
+++ b/qemu/disas/arm-a64.cc
@@ -17,12 +17,13 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "a64/disasm-a64.h"
-
 extern "C" {
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 }
 
+#include "vixl/a64/disasm-a64.h"
+
 using namespace vixl;
 
 static Decoder *vixl_decoder = NULL;
diff --git a/qemu/disas/arm.c b/qemu/disas/arm.c
index 616524653..70da5298a 100644
--- a/qemu/disas/arm.c
+++ b/qemu/disas/arm.c
@@ -22,6 +22,7 @@
 /* Start of qemu specific additions.  Mostly this is stub definitions
    for things we don't care about.  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 #define ATTRIBUTE_UNUSED __attribute__((unused))
 #define ISSPACE(x) ((x) == ' ' || (x) == '\t' || (x) == '\n')
@@ -1779,7 +1780,7 @@ print_insn_coprocessor (bfd_vma pc, struct disassemble_info *info, long given,
 
 			/* Is ``imm'' a negative number?  */
 			if (imm & 0x40)
-			  imm |= (-1 << 7);
+			  imm |= (~0u << 7);
 
 			func (stream, "%d", imm);
 		      }
diff --git a/qemu/disas/cris.c b/qemu/disas/cris.c
index 1b76a09db..7f35bc0c6 100644
--- a/qemu/disas/cris.c
+++ b/qemu/disas/cris.c
@@ -18,6 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 //#include "sysdep.h"
@@ -2492,7 +2493,7 @@ print_with_operands (const struct cris_opcode *opcodep,
 	  = spec_reg_info ((insn >> 12) & 15, disdata->distype);
 
 	if (sregp->name == NULL)
-	  /* Should have been caught as a non-match eariler.  */
+	  /* Should have been caught as a non-match earlier.  */
 	  *tp++ = '?';
 	else
 	  {
diff --git a/qemu/disas/hppa.c b/qemu/disas/hppa.c
index c7c8be66a..43facdc47 100644
--- a/qemu/disas/hppa.c
+++ b/qemu/disas/hppa.c
@@ -18,6 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* HP PA-RISC SOM object file format:  definitions internal to BFD.
diff --git a/qemu/disas/i386.c b/qemu/disas/i386.c
index 00ceca9c5..c0e717abe 100644
--- a/qemu/disas/i386.c
+++ b/qemu/disas/i386.c
@@ -31,8 +31,10 @@
    and the small letter tells about the operand size.  Refer to
    the Intel manual for details.  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
+#include "qemu/cutils.h"
+
 /* include/opcode/i386.h r1.78 */
 
 /* opcode/i386.h -- Intel 80386 opcode macros
@@ -153,8 +155,6 @@
 /* opcodes/i386-dis.c r1.126 */
 #include "qemu-common.h"
 
-#include <setjmp.h>
-
 static int fetch_data2(struct disassemble_info *, bfd_byte *);
 static int fetch_data(struct disassemble_info *, bfd_byte *);
 static void ckprefix (void);
@@ -357,7 +357,7 @@ fetch_data(struct disassemble_info *info, bfd_byte *addr)
 #define Rd { OP_R, d_mode }
 #define Rm { OP_R, m_mode }
 #define Ib { OP_I, b_mode }
-#define sIb { OP_sI, b_mode }	/* sign extened byte */
+#define sIb { OP_sI, b_mode }	/* sign extended byte */
 #define Iv { OP_I, v_mode }
 #define Iq { OP_I, q_mode }
 #define Iv64 { OP_I64, v_mode }
diff --git a/qemu/disas/ia64.c b/qemu/disas/ia64.c
index a8fe26c41..140754c94 100644
--- a/qemu/disas/ia64.c
+++ b/qemu/disas/ia64.c
@@ -18,8 +18,7 @@
    along with this file; see the file COPYING.  If not, see
    <http://www.gnu.org/licenses/>. */
 
-#include <assert.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "disas/bfd.h"
 
@@ -28,7 +27,6 @@
    Free Software Foundation, Inc.
    Contributed by David Mosberger-Tang <davidm@hpl.hp.com> */
 
-#include <sys/types.h>
 
 typedef uint64_t ia64_insn;
 
diff --git a/qemu/disas/libvixl/Makefile.objs b/qemu/disas/libvixl/Makefile.objs
index 17e6565d1..bbe7695fd 100644
--- a/qemu/disas/libvixl/Makefile.objs
+++ b/qemu/disas/libvixl/Makefile.objs
@@ -1,8 +1,11 @@
-libvixl_OBJS = utils.o \
-               a64/instructions-a64.o \
-               a64/decoder-a64.o \
-               a64/disasm-a64.o
+libvixl_OBJS = vixl/utils.o \
+               vixl/compiler-intrinsics.o \
+               vixl/a64/instructions-a64.o \
+               vixl/a64/decoder-a64.o \
+               vixl/a64/disasm-a64.o
 
-$(addprefix $(obj)/,$(libvixl_OBJS)): QEMU_CFLAGS := -I$(SRC_PATH)/disas/libvixl $(QEMU_CFLAGS)
+# The -Wno-sign-compare is needed only for gcc 4.6, which complains about
+# some signed-unsigned equality comparisons which later gcc versions do not.
+$(addprefix $(obj)/,$(libvixl_OBJS)): QEMU_CFLAGS := -I$(SRC_PATH)/disas/libvixl $(QEMU_CFLAGS) -Wno-sign-compare
 
 common-obj-$(CONFIG_ARM_A64_DIS) += $(libvixl_OBJS)
diff --git a/qemu/disas/libvixl/README b/qemu/disas/libvixl/README
index 58db41c67..932a41adf 100644
--- a/qemu/disas/libvixl/README
+++ b/qemu/disas/libvixl/README
@@ -2,11 +2,10 @@
 The code in this directory is a subset of libvixl:
  https://github.com/armvixl/vixl
 (specifically, it is the set of files needed for disassembly only,
-taken from libvixl 1.7).
+taken from libvixl 1.12).
 Bugfixes should preferably be sent upstream initially.
 
 The disassembler does not currently support the entire A64 instruction
 set. Notably:
- * No Advanced SIMD support.
  * Limited support for system instructions.
  * A few miscellaneous integer and floating point instructions are missing.
diff --git a/qemu/disas/libvixl/a64/assembler-a64.h b/qemu/disas/libvixl/a64/assembler-a64.h
deleted file mode 100644
index 35aaf20f7..000000000
--- a/qemu/disas/libvixl/a64/assembler-a64.h
+++ /dev/null
@@ -1,2353 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_ASSEMBLER_A64_H_
-#define VIXL_A64_ASSEMBLER_A64_H_
-
-#include <list>
-#include <stack>
-
-#include "globals.h"
-#include "utils.h"
-#include "code-buffer.h"
-#include "a64/instructions-a64.h"
-
-namespace vixl {
-
-typedef uint64_t RegList;
-static const int kRegListSizeInBits = sizeof(RegList) * 8;
-
-
-// Registers.
-
-// Some CPURegister methods can return Register and FPRegister types, so we
-// need to declare them in advance.
-class Register;
-class FPRegister;
-
-
-class CPURegister {
- public:
-  enum RegisterType {
-    // The kInvalid value is used to detect uninitialized static instances,
-    // which are always zero-initialized before any constructors are called.
-    kInvalid = 0,
-    kRegister,
-    kFPRegister,
-    kNoRegister
-  };
-
-  CPURegister() : code_(0), size_(0), type_(kNoRegister) {
-    VIXL_ASSERT(!IsValid());
-    VIXL_ASSERT(IsNone());
-  }
-
-  CPURegister(unsigned code, unsigned size, RegisterType type)
-      : code_(code), size_(size), type_(type) {
-    VIXL_ASSERT(IsValidOrNone());
-  }
-
-  unsigned code() const {
-    VIXL_ASSERT(IsValid());
-    return code_;
-  }
-
-  RegisterType type() const {
-    VIXL_ASSERT(IsValidOrNone());
-    return type_;
-  }
-
-  RegList Bit() const {
-    VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
-    return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
-  }
-
-  unsigned size() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  int SizeInBytes() const {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(size() % 8 == 0);
-    return size_ / 8;
-  }
-
-  int SizeInBits() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  bool Is32Bits() const {
-    VIXL_ASSERT(IsValid());
-    return size_ == 32;
-  }
-
-  bool Is64Bits() const {
-    VIXL_ASSERT(IsValid());
-    return size_ == 64;
-  }
-
-  bool IsValid() const {
-    if (IsValidRegister() || IsValidFPRegister()) {
-      VIXL_ASSERT(!IsNone());
-      return true;
-    } else {
-      VIXL_ASSERT(IsNone());
-      return false;
-    }
-  }
-
-  bool IsValidRegister() const {
-    return IsRegister() &&
-           ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
-           ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
-  }
-
-  bool IsValidFPRegister() const {
-    return IsFPRegister() &&
-           ((size_ == kSRegSize) || (size_ == kDRegSize)) &&
-           (code_ < kNumberOfFPRegisters);
-  }
-
-  bool IsNone() const {
-    // kNoRegister types should always have size 0 and code 0.
-    VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
-    VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
-
-    return type_ == kNoRegister;
-  }
-
-  bool Aliases(const CPURegister& other) const {
-    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
-    return (code_ == other.code_) && (type_ == other.type_);
-  }
-
-  bool Is(const CPURegister& other) const {
-    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
-    return Aliases(other) && (size_ == other.size_);
-  }
-
-  bool IsZero() const {
-    VIXL_ASSERT(IsValid());
-    return IsRegister() && (code_ == kZeroRegCode);
-  }
-
-  bool IsSP() const {
-    VIXL_ASSERT(IsValid());
-    return IsRegister() && (code_ == kSPRegInternalCode);
-  }
-
-  bool IsRegister() const {
-    return type_ == kRegister;
-  }
-
-  bool IsFPRegister() const {
-    return type_ == kFPRegister;
-  }
-
-  bool IsW() const { return IsValidRegister() && Is32Bits(); }
-  bool IsX() const { return IsValidRegister() && Is64Bits(); }
-  bool IsS() const { return IsValidFPRegister() && Is32Bits(); }
-  bool IsD() const { return IsValidFPRegister() && Is64Bits(); }
-
-  const Register& W() const;
-  const Register& X() const;
-  const FPRegister& S() const;
-  const FPRegister& D() const;
-
-  bool IsSameSizeAndType(const CPURegister& other) const {
-    return (size_ == other.size_) && (type_ == other.type_);
-  }
-
- protected:
-  unsigned code_;
-  unsigned size_;
-  RegisterType type_;
-
- private:
-  bool IsValidOrNone() const {
-    return IsValid() || IsNone();
-  }
-};
-
-
-class Register : public CPURegister {
- public:
-  Register() : CPURegister() {}
-  explicit Register(const CPURegister& other)
-      : CPURegister(other.code(), other.size(), other.type()) {
-    VIXL_ASSERT(IsValidRegister());
-  }
-  Register(unsigned code, unsigned size)
-      : CPURegister(code, size, kRegister) {}
-
-  bool IsValid() const {
-    VIXL_ASSERT(IsRegister() || IsNone());
-    return IsValidRegister();
-  }
-
-  static const Register& WRegFromCode(unsigned code);
-  static const Register& XRegFromCode(unsigned code);
-
- private:
-  static const Register wregisters[];
-  static const Register xregisters[];
-};
-
-
-class FPRegister : public CPURegister {
- public:
-  FPRegister() : CPURegister() {}
-  explicit FPRegister(const CPURegister& other)
-      : CPURegister(other.code(), other.size(), other.type()) {
-    VIXL_ASSERT(IsValidFPRegister());
-  }
-  FPRegister(unsigned code, unsigned size)
-      : CPURegister(code, size, kFPRegister) {}
-
-  bool IsValid() const {
-    VIXL_ASSERT(IsFPRegister() || IsNone());
-    return IsValidFPRegister();
-  }
-
-  static const FPRegister& SRegFromCode(unsigned code);
-  static const FPRegister& DRegFromCode(unsigned code);
-
- private:
-  static const FPRegister sregisters[];
-  static const FPRegister dregisters[];
-};
-
-
-// No*Reg is used to indicate an unused argument, or an error case. Note that
-// these all compare equal (using the Is() method). The Register and FPRegister
-// variants are provided for convenience.
-const Register NoReg;
-const FPRegister NoFPReg;
-const CPURegister NoCPUReg;
-
-
-#define DEFINE_REGISTERS(N)  \
-const Register w##N(N, kWRegSize);  \
-const Register x##N(N, kXRegSize);
-REGISTER_CODE_LIST(DEFINE_REGISTERS)
-#undef DEFINE_REGISTERS
-const Register wsp(kSPRegInternalCode, kWRegSize);
-const Register sp(kSPRegInternalCode, kXRegSize);
-
-
-#define DEFINE_FPREGISTERS(N)  \
-const FPRegister s##N(N, kSRegSize);  \
-const FPRegister d##N(N, kDRegSize);
-REGISTER_CODE_LIST(DEFINE_FPREGISTERS)
-#undef DEFINE_FPREGISTERS
-
-
-// Registers aliases.
-const Register ip0 = x16;
-const Register ip1 = x17;
-const Register lr = x30;
-const Register xzr = x31;
-const Register wzr = w31;
-
-
-// AreAliased returns true if any of the named registers overlap. Arguments
-// set to NoReg are ignored. The system stack pointer may be specified.
-bool AreAliased(const CPURegister& reg1,
-                const CPURegister& reg2,
-                const CPURegister& reg3 = NoReg,
-                const CPURegister& reg4 = NoReg,
-                const CPURegister& reg5 = NoReg,
-                const CPURegister& reg6 = NoReg,
-                const CPURegister& reg7 = NoReg,
-                const CPURegister& reg8 = NoReg);
-
-
-// AreSameSizeAndType returns true if all of the specified registers have the
-// same size, and are of the same type. The system stack pointer may be
-// specified. Arguments set to NoReg are ignored, as are any subsequent
-// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
-bool AreSameSizeAndType(const CPURegister& reg1,
-                        const CPURegister& reg2,
-                        const CPURegister& reg3 = NoCPUReg,
-                        const CPURegister& reg4 = NoCPUReg,
-                        const CPURegister& reg5 = NoCPUReg,
-                        const CPURegister& reg6 = NoCPUReg,
-                        const CPURegister& reg7 = NoCPUReg,
-                        const CPURegister& reg8 = NoCPUReg);
-
-
-// Lists of registers.
-class CPURegList {
- public:
-  explicit CPURegList(CPURegister reg1,
-                      CPURegister reg2 = NoCPUReg,
-                      CPURegister reg3 = NoCPUReg,
-                      CPURegister reg4 = NoCPUReg)
-      : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
-        size_(reg1.size()), type_(reg1.type()) {
-    VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
-    VIXL_ASSERT(IsValid());
-  }
-
-  CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
-      : list_(list), size_(size), type_(type) {
-    VIXL_ASSERT(IsValid());
-  }
-
-  CPURegList(CPURegister::RegisterType type, unsigned size,
-             unsigned first_reg, unsigned last_reg)
-      : size_(size), type_(type) {
-    VIXL_ASSERT(((type == CPURegister::kRegister) &&
-                 (last_reg < kNumberOfRegisters)) ||
-                ((type == CPURegister::kFPRegister) &&
-                 (last_reg < kNumberOfFPRegisters)));
-    VIXL_ASSERT(last_reg >= first_reg);
-    list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
-    list_ &= ~((UINT64_C(1) << first_reg) - 1);
-    VIXL_ASSERT(IsValid());
-  }
-
-  CPURegister::RegisterType type() const {
-    VIXL_ASSERT(IsValid());
-    return type_;
-  }
-
-  // Combine another CPURegList into this one. Registers that already exist in
-  // this list are left unchanged. The type and size of the registers in the
-  // 'other' list must match those in this list.
-  void Combine(const CPURegList& other) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
-    list_ |= other.list();
-  }
-
-  // Remove every register in the other CPURegList from this one. Registers that
-  // do not exist in this list are ignored. The type and size of the registers
-  // in the 'other' list must match those in this list.
-  void Remove(const CPURegList& other) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
-    list_ &= ~other.list();
-  }
-
-  // Variants of Combine and Remove which take a single register.
-  void Combine(const CPURegister& other) {
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.size() == size_);
-    Combine(other.code());
-  }
-
-  void Remove(const CPURegister& other) {
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.size() == size_);
-    Remove(other.code());
-  }
-
-  // Variants of Combine and Remove which take a single register by its code;
-  // the type and size of the register is inferred from this list.
-  void Combine(int code) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
-    list_ |= (UINT64_C(1) << code);
-  }
-
-  void Remove(int code) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
-    list_ &= ~(UINT64_C(1) << code);
-  }
-
-  static CPURegList Union(const CPURegList& list_1, const CPURegList& list_2) {
-    VIXL_ASSERT(list_1.type_ == list_2.type_);
-    VIXL_ASSERT(list_1.size_ == list_2.size_);
-    return CPURegList(list_1.type_, list_1.size_, list_1.list_ | list_2.list_);
-  }
-  static CPURegList Union(const CPURegList& list_1,
-                          const CPURegList& list_2,
-                          const CPURegList& list_3);
-  static CPURegList Union(const CPURegList& list_1,
-                          const CPURegList& list_2,
-                          const CPURegList& list_3,
-                          const CPURegList& list_4);
-
-  static CPURegList Intersection(const CPURegList& list_1,
-                                 const CPURegList& list_2) {
-    VIXL_ASSERT(list_1.type_ == list_2.type_);
-    VIXL_ASSERT(list_1.size_ == list_2.size_);
-    return CPURegList(list_1.type_, list_1.size_, list_1.list_ & list_2.list_);
-  }
-  static CPURegList Intersection(const CPURegList& list_1,
-                                 const CPURegList& list_2,
-                                 const CPURegList& list_3);
-  static CPURegList Intersection(const CPURegList& list_1,
-                                 const CPURegList& list_2,
-                                 const CPURegList& list_3,
-                                 const CPURegList& list_4);
-
-  RegList list() const {
-    VIXL_ASSERT(IsValid());
-    return list_;
-  }
-
-  void set_list(RegList new_list) {
-    VIXL_ASSERT(IsValid());
-    list_ = new_list;
-  }
-
-  // Remove all callee-saved registers from the list. This can be useful when
-  // preparing registers for an AAPCS64 function call, for example.
-  void RemoveCalleeSaved();
-
-  CPURegister PopLowestIndex();
-  CPURegister PopHighestIndex();
-
-  // AAPCS64 callee-saved registers.
-  static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
-  static CPURegList GetCalleeSavedFP(unsigned size = kDRegSize);
-
-  // AAPCS64 caller-saved registers. Note that this includes lr.
-  static CPURegList GetCallerSaved(unsigned size = kXRegSize);
-  static CPURegList GetCallerSavedFP(unsigned size = kDRegSize);
-
-  bool IsEmpty() const {
-    VIXL_ASSERT(IsValid());
-    return list_ == 0;
-  }
-
-  bool IncludesAliasOf(const CPURegister& other) const {
-    VIXL_ASSERT(IsValid());
-    return (type_ == other.type()) && ((other.Bit() & list_) != 0);
-  }
-
-  bool IncludesAliasOf(int code) const {
-    VIXL_ASSERT(IsValid());
-    return ((code & list_) != 0);
-  }
-
-  int Count() const {
-    VIXL_ASSERT(IsValid());
-    return CountSetBits(list_, kRegListSizeInBits);
-  }
-
-  unsigned RegisterSizeInBits() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  unsigned RegisterSizeInBytes() const {
-    int size_in_bits = RegisterSizeInBits();
-    VIXL_ASSERT((size_in_bits % 8) == 0);
-    return size_in_bits / 8;
-  }
-
-  unsigned TotalSizeInBytes() const {
-    VIXL_ASSERT(IsValid());
-    return RegisterSizeInBytes() * Count();
-  }
-
- private:
-  RegList list_;
-  unsigned size_;
-  CPURegister::RegisterType type_;
-
-  bool IsValid() const;
-};
-
-
-// AAPCS64 callee-saved registers.
-extern const CPURegList kCalleeSaved;
-extern const CPURegList kCalleeSavedFP;
-
-
-// AAPCS64 caller-saved registers. Note that this includes lr.
-extern const CPURegList kCallerSaved;
-extern const CPURegList kCallerSavedFP;
-
-
-// Operand.
-class Operand {
- public:
-  // #<immediate>
-  // where <immediate> is int64_t.
-  // This is allowed to be an implicit constructor because Operand is
-  // a wrapper class that doesn't normally perform any type conversion.
-  Operand(int64_t immediate);           // NOLINT(runtime/explicit)
-
-  // rm, {<shift> #<shift_amount>}
-  // where <shift> is one of {LSL, LSR, ASR, ROR}.
-  //       <shift_amount> is uint6_t.
-  // This is allowed to be an implicit constructor because Operand is
-  // a wrapper class that doesn't normally perform any type conversion.
-  Operand(Register reg,
-          Shift shift = LSL,
-          unsigned shift_amount = 0);   // NOLINT(runtime/explicit)
-
-  // rm, {<extend> {#<shift_amount>}}
-  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
-  //       <shift_amount> is uint2_t.
-  explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
-
-  bool IsImmediate() const;
-  bool IsShiftedRegister() const;
-  bool IsExtendedRegister() const;
-  bool IsZero() const;
-
-  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
-  // which helps in the encoding of instructions that use the stack pointer.
-  Operand ToExtendedRegister() const;
-
-  int64_t immediate() const {
-    VIXL_ASSERT(IsImmediate());
-    return immediate_;
-  }
-
-  Register reg() const {
-    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
-    return reg_;
-  }
-
-  Shift shift() const {
-    VIXL_ASSERT(IsShiftedRegister());
-    return shift_;
-  }
-
-  Extend extend() const {
-    VIXL_ASSERT(IsExtendedRegister());
-    return extend_;
-  }
-
-  unsigned shift_amount() const {
-    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
-    return shift_amount_;
-  }
-
- private:
-  int64_t immediate_;
-  Register reg_;
-  Shift shift_;
-  Extend extend_;
-  unsigned shift_amount_;
-};
-
-
-// MemOperand represents the addressing mode of a load or store instruction.
-class MemOperand {
- public:
-  explicit MemOperand(Register base,
-                      int64_t offset = 0,
-                      AddrMode addrmode = Offset);
-  explicit MemOperand(Register base,
-                      Register regoffset,
-                      Shift shift = LSL,
-                      unsigned shift_amount = 0);
-  explicit MemOperand(Register base,
-                      Register regoffset,
-                      Extend extend,
-                      unsigned shift_amount = 0);
-  explicit MemOperand(Register base,
-                      const Operand& offset,
-                      AddrMode addrmode = Offset);
-
-  const Register& base() const { return base_; }
-  const Register& regoffset() const { return regoffset_; }
-  int64_t offset() const { return offset_; }
-  AddrMode addrmode() const { return addrmode_; }
-  Shift shift() const { return shift_; }
-  Extend extend() const { return extend_; }
-  unsigned shift_amount() const { return shift_amount_; }
-  bool IsImmediateOffset() const;
-  bool IsRegisterOffset() const;
-  bool IsPreIndex() const;
-  bool IsPostIndex() const;
-
- private:
-  Register base_;
-  Register regoffset_;
-  int64_t offset_;
-  AddrMode addrmode_;
-  Shift shift_;
-  Extend extend_;
-  unsigned shift_amount_;
-};
-
-
-class Label {
- public:
-  Label() : location_(kLocationUnbound) {}
-  ~Label() {
-    // If the label has been linked to, it needs to be bound to a target.
-    VIXL_ASSERT(!IsLinked() || IsBound());
-  }
-
-  bool IsBound() const { return location_ >= 0; }
-  bool IsLinked() const { return !links_.empty(); }
-
-  ptrdiff_t location() const { return location_; }
-
- private:
-  // The list of linked instructions is stored in a stack-like structure. We
-  // don't use std::stack directly because it's slow for the common case where
-  // only one or two instructions refer to a label, and labels themselves are
-  // short-lived. This class behaves like std::stack, but the first few links
-  // are preallocated (configured by kPreallocatedLinks).
-  //
-  // If more than N links are required, this falls back to std::stack.
-  class LinksStack {
-   public:
-    LinksStack() : size_(0), links_extended_(NULL) {}
-    ~LinksStack() {
-      delete links_extended_;
-    }
-
-    size_t size() const {
-      return size_;
-    }
-
-    bool empty() const {
-      return size_ == 0;
-    }
-
-    void push(ptrdiff_t value) {
-      if (size_ < kPreallocatedLinks) {
-        links_[size_] = value;
-      } else {
-        if (links_extended_ == NULL) {
-          links_extended_ = new std::stack<ptrdiff_t>();
-        }
-        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
-        links_extended_->push(value);
-      }
-      size_++;
-    }
-
-    ptrdiff_t top() const {
-      return (size_ <= kPreallocatedLinks) ? links_[size_ - 1]
-                                           : links_extended_->top();
-    }
-
-    void pop() {
-      size_--;
-      if (size_ >= kPreallocatedLinks) {
-        links_extended_->pop();
-        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
-      }
-    }
-
-   private:
-    static const size_t kPreallocatedLinks = 4;
-
-    size_t size_;
-    ptrdiff_t links_[kPreallocatedLinks];
-    std::stack<ptrdiff_t> * links_extended_;
-  };
-
-  void Bind(ptrdiff_t location) {
-    // Labels can only be bound once.
-    VIXL_ASSERT(!IsBound());
-    location_ = location;
-  }
-
-  void AddLink(ptrdiff_t instruction) {
-    // If a label is bound, the assembler already has the information it needs
-    // to write the instruction, so there is no need to add it to links_.
-    VIXL_ASSERT(!IsBound());
-    links_.push(instruction);
-  }
-
-  ptrdiff_t GetAndRemoveNextLink() {
-    VIXL_ASSERT(IsLinked());
-    ptrdiff_t link = links_.top();
-    links_.pop();
-    return link;
-  }
-
-  // The offsets of the instructions that have linked to this label.
-  LinksStack links_;
-  // The label location.
-  ptrdiff_t location_;
-
-  static const ptrdiff_t kLocationUnbound = -1;
-
-  // It is not safe to copy labels, so disable the copy constructor by declaring
-  // it private (without an implementation).
-  Label(const Label&);
-
-  // The Assembler class is responsible for binding and linking labels, since
-  // the stored offsets need to be consistent with the Assembler's buffer.
-  friend class Assembler;
-};
-
-
-// A literal is a 32-bit or 64-bit piece of data stored in the instruction
-// stream and loaded through a pc relative load. The same literal can be
-// referred to by multiple instructions but a literal can only reside at one
-// place in memory. A literal can be used by a load before or after being
-// placed in memory.
-//
-// Internally an offset of 0 is associated with a literal which has been
-// neither used nor placed. Then two possibilities arise:
-//  1) the label is placed, the offset (stored as offset + 1) is used to
-//     resolve any subsequent load using the label.
-//  2) the label is not placed and offset is the offset of the last load using
-//     the literal (stored as -offset -1). If multiple loads refer to this
-//     literal then the last load holds the offset of the preceding load and
-//     all loads form a chain. Once the offset is placed all the loads in the
-//     chain are resolved and future loads fall back to possibility 1.
-class RawLiteral {
- public:
-  RawLiteral() : size_(0), offset_(0), raw_value_(0) {}
-
-  size_t size() {
-    VIXL_STATIC_ASSERT(kDRegSizeInBytes == kXRegSizeInBytes);
-    VIXL_STATIC_ASSERT(kSRegSizeInBytes == kWRegSizeInBytes);
-    VIXL_ASSERT((size_ == kXRegSizeInBytes) || (size_ == kWRegSizeInBytes));
-    return size_;
-  }
-  uint64_t raw_value64() {
-    VIXL_ASSERT(size_ == kXRegSizeInBytes);
-    return raw_value_;
-  }
-  uint32_t raw_value32() {
-    VIXL_ASSERT(size_ == kWRegSizeInBytes);
-    VIXL_ASSERT(is_uint32(raw_value_) || is_int32(raw_value_));
-    return static_cast<uint32_t>(raw_value_);
-  }
-  bool IsUsed() { return offset_ < 0; }
-  bool IsPlaced() { return offset_ > 0; }
-
- protected:
-  ptrdiff_t offset() {
-    VIXL_ASSERT(IsPlaced());
-    return offset_ - 1;
-  }
-  void set_offset(ptrdiff_t offset) {
-    VIXL_ASSERT(offset >= 0);
-    VIXL_ASSERT(IsWordAligned(offset));
-    VIXL_ASSERT(!IsPlaced());
-    offset_ = offset + 1;
-  }
-  ptrdiff_t last_use() {
-    VIXL_ASSERT(IsUsed());
-    return -offset_ - 1;
-  }
-  void set_last_use(ptrdiff_t offset) {
-    VIXL_ASSERT(offset >= 0);
-    VIXL_ASSERT(IsWordAligned(offset));
-    VIXL_ASSERT(!IsPlaced());
-    offset_ = -offset - 1;
-  }
-
-  size_t size_;
-  ptrdiff_t offset_;
-  uint64_t raw_value_;
-
-  friend class Assembler;
-};
-
-
-template <typename T>
-class Literal : public RawLiteral {
- public:
-  explicit Literal(T value) {
-    size_ = sizeof(value);
-    memcpy(&raw_value_, &value, sizeof(value));
-  }
-};
-
-
-// Control whether or not position-independent code should be emitted.
-enum PositionIndependentCodeOption {
-  // All code generated will be position-independent; all branches and
-  // references to labels generated with the Label class will use PC-relative
-  // addressing.
-  PositionIndependentCode,
-
-  // Allow VIXL to generate code that refers to absolute addresses. With this
-  // option, it will not be possible to copy the code buffer and run it from a
-  // different address; code must be generated in its final location.
-  PositionDependentCode,
-
-  // Allow VIXL to assume that the bottom 12 bits of the address will be
-  // constant, but that the top 48 bits may change. This allows `adrp` to
-  // function in systems which copy code between pages, but otherwise maintain
-  // 4KB page alignment.
-  PageOffsetDependentCode
-};
-
-
-// Control how scaled- and unscaled-offset loads and stores are generated.
-enum LoadStoreScalingOption {
-  // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
-  // register-offset, pre-index or post-index instructions if necessary.
-  PreferScaledOffset,
-
-  // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
-  // register-offset, pre-index or post-index instructions if necessary.
-  PreferUnscaledOffset,
-
-  // Require scaled-immediate-offset instructions.
-  RequireScaledOffset,
-
-  // Require unscaled-immediate-offset instructions.
-  RequireUnscaledOffset
-};
-
-
-// Assembler.
-class Assembler {
- public:
-  Assembler(size_t capacity,
-            PositionIndependentCodeOption pic = PositionIndependentCode);
-  Assembler(byte* buffer, size_t capacity,
-            PositionIndependentCodeOption pic = PositionIndependentCode);
-
-  // The destructor asserts that one of the following is true:
-  //  * The Assembler object has not been used.
-  //  * Nothing has been emitted since the last Reset() call.
-  //  * Nothing has been emitted since the last FinalizeCode() call.
-  ~Assembler();
-
-  // System functions.
-
-  // Start generating code from the beginning of the buffer, discarding any code
-  // and data that has already been emitted into the buffer.
-  void Reset();
-
-  // Finalize a code buffer of generated instructions. This function must be
-  // called before executing or copying code from the buffer.
-  void FinalizeCode();
-
-  // Label.
-  // Bind a label to the current PC.
-  void bind(Label* label);
-
-  // Bind a label to a specified offset from the start of the buffer.
-  void BindToOffset(Label* label, ptrdiff_t offset);
-
-  // Place a literal at the current PC.
-  void place(RawLiteral* literal);
-
-  ptrdiff_t CursorOffset() const {
-    return buffer_->CursorOffset();
-  }
-
-  ptrdiff_t BufferEndOffset() const {
-    return static_cast<ptrdiff_t>(buffer_->capacity());
-  }
-
-  // Return the address of an offset in the buffer.
-  template <typename T>
-  T GetOffsetAddress(ptrdiff_t offset) {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return buffer_->GetOffsetAddress<T>(offset);
-  }
-
-  // Return the address of a bound label.
-  template <typename T>
-  T GetLabelAddress(const Label * label) {
-    VIXL_ASSERT(label->IsBound());
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(label->location());
-  }
-
-  // Return the address of the cursor.
-  template <typename T>
-  T GetCursorAddress() {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(CursorOffset());
-  }
-
-  // Return the address of the start of the buffer.
-  template <typename T>
-  T GetStartAddress() {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(0);
-  }
-
-  // Instruction set functions.
-
-  // Branch / Jump instructions.
-  // Branch to register.
-  void br(const Register& xn);
-
-  // Branch with link to register.
-  void blr(const Register& xn);
-
-  // Branch to register with return hint.
-  void ret(const Register& xn = lr);
-
-  // Unconditional branch to label.
-  void b(Label* label);
-
-  // Conditional branch to label.
-  void b(Label* label, Condition cond);
-
-  // Unconditional branch to PC offset.
-  void b(int imm26);
-
-  // Conditional branch to PC offset.
-  void b(int imm19, Condition cond);
-
-  // Branch with link to label.
-  void bl(Label* label);
-
-  // Branch with link to PC offset.
-  void bl(int imm26);
-
-  // Compare and branch to label if zero.
-  void cbz(const Register& rt, Label* label);
-
-  // Compare and branch to PC offset if zero.
-  void cbz(const Register& rt, int imm19);
-
-  // Compare and branch to label if not zero.
-  void cbnz(const Register& rt, Label* label);
-
-  // Compare and branch to PC offset if not zero.
-  void cbnz(const Register& rt, int imm19);
-
-  // Test bit and branch to label if zero.
-  void tbz(const Register& rt, unsigned bit_pos, Label* label);
-
-  // Test bit and branch to PC offset if zero.
-  void tbz(const Register& rt, unsigned bit_pos, int imm14);
-
-  // Test bit and branch to label if not zero.
-  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
-
-  // Test bit and branch to PC offset if not zero.
-  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
-
-  // Address calculation instructions.
-  // Calculate a PC-relative address. Unlike for branches the offset in adr is
-  // unscaled (i.e. the result can be unaligned).
-
-  // Calculate the address of a label.
-  void adr(const Register& rd, Label* label);
-
-  // Calculate the address of a PC offset.
-  void adr(const Register& rd, int imm21);
-
-  // Calculate the page address of a label.
-  void adrp(const Register& rd, Label* label);
-
-  // Calculate the page address of a PC offset.
-  void adrp(const Register& rd, int imm21);
-
-  // Data Processing instructions.
-  // Add.
-  void add(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Add and update status flags.
-  void adds(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Compare negative.
-  void cmn(const Register& rn, const Operand& operand);
-
-  // Subtract.
-  void sub(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Subtract and update status flags.
-  void subs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Compare.
-  void cmp(const Register& rn, const Operand& operand);
-
-  // Negate.
-  void neg(const Register& rd,
-           const Operand& operand);
-
-  // Negate and update status flags.
-  void negs(const Register& rd,
-            const Operand& operand);
-
-  // Add with carry bit.
-  void adc(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Add with carry bit and update status flags.
-  void adcs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Subtract with carry bit.
-  void sbc(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Subtract with carry bit and update status flags.
-  void sbcs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Negate with carry bit.
-  void ngc(const Register& rd,
-           const Operand& operand);
-
-  // Negate with carry bit and update status flags.
-  void ngcs(const Register& rd,
-            const Operand& operand);
-
-  // Logical instructions.
-  // Bitwise and (A & B).
-  void and_(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bitwise and (A & B) and update status flags.
-  void ands(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bit test and set flags.
-  void tst(const Register& rn, const Operand& operand);
-
-  // Bit clear (A & ~B).
-  void bic(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Bit clear (A & ~B) and update status flags.
-  void bics(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bitwise or (A | B).
-  void orr(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise nor (A | ~B).
-  void orn(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise eor/xor (A ^ B).
-  void eor(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise enor/xnor (A ^ ~B).
-  void eon(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Logical shift left by variable.
-  void lslv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Logical shift right by variable.
-  void lsrv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Arithmetic shift right by variable.
-  void asrv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Rotate right by variable.
-  void rorv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Bitfield instructions.
-  // Bitfield move.
-  void bfm(const Register& rd,
-           const Register& rn,
-           unsigned immr,
-           unsigned imms);
-
-  // Signed bitfield move.
-  void sbfm(const Register& rd,
-            const Register& rn,
-            unsigned immr,
-            unsigned imms);
-
-  // Unsigned bitfield move.
-  void ubfm(const Register& rd,
-            const Register& rn,
-            unsigned immr,
-            unsigned imms);
-
-  // Bfm aliases.
-  // Bitfield insert.
-  void bfi(const Register& rd,
-           const Register& rn,
-           unsigned lsb,
-           unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Bitfield extract and insert low.
-  void bfxil(const Register& rd,
-             const Register& rn,
-             unsigned lsb,
-             unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    bfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Sbfm aliases.
-  // Arithmetic shift right.
-  void asr(const Register& rd, const Register& rn, unsigned shift) {
-    VIXL_ASSERT(shift < rd.size());
-    sbfm(rd, rn, shift, rd.size() - 1);
-  }
-
-  // Signed bitfield insert with zero at right.
-  void sbfiz(const Register& rd,
-             const Register& rn,
-             unsigned lsb,
-             unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Signed bitfield extract.
-  void sbfx(const Register& rd,
-            const Register& rn,
-            unsigned lsb,
-            unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    sbfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Signed extend byte.
-  void sxtb(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 7);
-  }
-
-  // Signed extend halfword.
-  void sxth(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 15);
-  }
-
-  // Signed extend word.
-  void sxtw(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 31);
-  }
-
-  // Ubfm aliases.
-  // Logical shift left.
-  void lsl(const Register& rd, const Register& rn, unsigned shift) {
-    unsigned reg_size = rd.size();
-    VIXL_ASSERT(shift < reg_size);
-    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
-  }
-
-  // Logical shift right.
-  void lsr(const Register& rd, const Register& rn, unsigned shift) {
-    VIXL_ASSERT(shift < rd.size());
-    ubfm(rd, rn, shift, rd.size() - 1);
-  }
-
-  // Unsigned bitfield insert with zero at right.
-  void ubfiz(const Register& rd,
-             const Register& rn,
-             unsigned lsb,
-             unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Unsigned bitfield extract.
-  void ubfx(const Register& rd,
-            const Register& rn,
-            unsigned lsb,
-            unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    ubfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Unsigned extend byte.
-  void uxtb(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 7);
-  }
-
-  // Unsigned extend halfword.
-  void uxth(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 15);
-  }
-
-  // Unsigned extend word.
-  void uxtw(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 31);
-  }
-
-  // Extract.
-  void extr(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            unsigned lsb);
-
-  // Conditional select: rd = cond ? rn : rm.
-  void csel(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            Condition cond);
-
-  // Conditional select increment: rd = cond ? rn : rm + 1.
-  void csinc(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional select inversion: rd = cond ? rn : ~rm.
-  void csinv(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional select negation: rd = cond ? rn : -rm.
-  void csneg(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional set: rd = cond ? 1 : 0.
-  void cset(const Register& rd, Condition cond);
-
-  // Conditional set mask: rd = cond ? -1 : 0.
-  void csetm(const Register& rd, Condition cond);
-
-  // Conditional increment: rd = cond ? rn + 1 : rn.
-  void cinc(const Register& rd, const Register& rn, Condition cond);
-
-  // Conditional invert: rd = cond ? ~rn : rn.
-  void cinv(const Register& rd, const Register& rn, Condition cond);
-
-  // Conditional negate: rd = cond ? -rn : rn.
-  void cneg(const Register& rd, const Register& rn, Condition cond);
-
-  // Rotate right.
-  void ror(const Register& rd, const Register& rs, unsigned shift) {
-    extr(rd, rs, rs, shift);
-  }
-
-  // Conditional comparison.
-  // Conditional compare negative.
-  void ccmn(const Register& rn,
-            const Operand& operand,
-            StatusFlags nzcv,
-            Condition cond);
-
-  // Conditional compare.
-  void ccmp(const Register& rn,
-            const Operand& operand,
-            StatusFlags nzcv,
-            Condition cond);
-
-  // Multiply.
-  void mul(const Register& rd, const Register& rn, const Register& rm);
-
-  // Negated multiply.
-  void mneg(const Register& rd, const Register& rn, const Register& rm);
-
-  // Signed long multiply: 32 x 32 -> 64-bit.
-  void smull(const Register& rd, const Register& rn, const Register& rm);
-
-  // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
-  void smulh(const Register& xd, const Register& xn, const Register& xm);
-
-  // Multiply and accumulate.
-  void madd(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            const Register& ra);
-
-  // Multiply and subtract.
-  void msub(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            const Register& ra);
-
-  // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
-  void smaddl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
-  void umaddl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
-  void smsubl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
-  void umsubl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Signed integer divide.
-  void sdiv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Unsigned integer divide.
-  void udiv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Bit reverse.
-  void rbit(const Register& rd, const Register& rn);
-
-  // Reverse bytes in 16-bit half words.
-  void rev16(const Register& rd, const Register& rn);
-
-  // Reverse bytes in 32-bit words.
-  void rev32(const Register& rd, const Register& rn);
-
-  // Reverse bytes.
-  void rev(const Register& rd, const Register& rn);
-
-  // Count leading zeroes.
-  void clz(const Register& rd, const Register& rn);
-
-  // Count leading sign bits.
-  void cls(const Register& rd, const Register& rn);
-
-  // Memory instructions.
-  // Load integer or FP register.
-  void ldr(const CPURegister& rt, const MemOperand& src,
-           LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store integer or FP register.
-  void str(const CPURegister& rt, const MemOperand& dst,
-           LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load word with sign extension.
-  void ldrsw(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load byte.
-  void ldrb(const Register& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store byte.
-  void strb(const Register& rt, const MemOperand& dst,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load byte with sign extension.
-  void ldrsb(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load half-word.
-  void ldrh(const Register& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store half-word.
-  void strh(const Register& rt, const MemOperand& dst,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load half-word with sign extension.
-  void ldrsh(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load integer or FP register (with unscaled offset).
-  void ldur(const CPURegister& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store integer or FP register (with unscaled offset).
-  void stur(const CPURegister& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load word with sign extension.
-  void ldursw(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load byte (with unscaled offset).
-  void ldurb(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store byte (with unscaled offset).
-  void sturb(const Register& rt, const MemOperand& dst,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load byte with sign extension (and unscaled offset).
-  void ldursb(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load half-word (with unscaled offset).
-  void ldurh(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store half-word (with unscaled offset).
-  void sturh(const Register& rt, const MemOperand& dst,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load half-word with sign extension (and unscaled offset).
-  void ldursh(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load integer or FP register pair.
-  void ldp(const CPURegister& rt, const CPURegister& rt2,
-           const MemOperand& src);
-
-  // Store integer or FP register pair.
-  void stp(const CPURegister& rt, const CPURegister& rt2,
-           const MemOperand& dst);
-
-  // Load word pair with sign extension.
-  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Load integer or FP register pair, non-temporal.
-  void ldnp(const CPURegister& rt, const CPURegister& rt2,
-            const MemOperand& src);
-
-  // Store integer or FP register pair, non-temporal.
-  void stnp(const CPURegister& rt, const CPURegister& rt2,
-            const MemOperand& dst);
-
-  // Load integer or FP register from literal pool.
-  void ldr(const CPURegister& rt, RawLiteral* literal);
-
-  // Load word with sign extension from literal pool.
-  void ldrsw(const Register& rt, RawLiteral* literal);
-
-  // Load integer or FP register from pc + imm19 << 2.
-  void ldr(const CPURegister& rt, int imm19);
-
-  // Load word with sign extension from pc + imm19 << 2.
-  void ldrsw(const Register& rt, int imm19);
-
-  // Store exclusive byte.
-  void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store exclusive half-word.
-  void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store exclusive register.
-  void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Load exclusive byte.
-  void ldxrb(const Register& rt, const MemOperand& src);
-
-  // Load exclusive half-word.
-  void ldxrh(const Register& rt, const MemOperand& src);
-
-  // Load exclusive register.
-  void ldxr(const Register& rt, const MemOperand& src);
-
-  // Store exclusive register pair.
-  void stxp(const Register& rs,
-            const Register& rt,
-            const Register& rt2,
-            const MemOperand& dst);
-
-  // Load exclusive register pair.
-  void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Store-release exclusive byte.
-  void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store-release exclusive half-word.
-  void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store-release exclusive register.
-  void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Load-acquire exclusive byte.
-  void ldaxrb(const Register& rt, const MemOperand& src);
-
-  // Load-acquire exclusive half-word.
-  void ldaxrh(const Register& rt, const MemOperand& src);
-
-  // Load-acquire exclusive register.
-  void ldaxr(const Register& rt, const MemOperand& src);
-
-  // Store-release exclusive register pair.
-  void stlxp(const Register& rs,
-             const Register& rt,
-             const Register& rt2,
-             const MemOperand& dst);
-
-  // Load-acquire exclusive register pair.
-  void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Store-release byte.
-  void stlrb(const Register& rt, const MemOperand& dst);
-
-  // Store-release half-word.
-  void stlrh(const Register& rt, const MemOperand& dst);
-
-  // Store-release register.
-  void stlr(const Register& rt, const MemOperand& dst);
-
-  // Load-acquire byte.
-  void ldarb(const Register& rt, const MemOperand& src);
-
-  // Load-acquire half-word.
-  void ldarh(const Register& rt, const MemOperand& src);
-
-  // Load-acquire register.
-  void ldar(const Register& rt, const MemOperand& src);
-
-  // Prefetch memory.
-  void prfm(PrefetchOperation op, const MemOperand& addr,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Prefetch memory (with unscaled offset).
-  void prfum(PrefetchOperation op, const MemOperand& addr,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Prefetch memory in the literal pool.
-  void prfm(PrefetchOperation op, RawLiteral* literal);
-
-  // Prefetch from pc + imm19 << 2.
-  void prfm(PrefetchOperation op, int imm19);
-
-  // Move instructions. The default shift of -1 indicates that the move
-  // instruction will calculate an appropriate 16-bit immediate and left shift
-  // that is equal to the 64-bit immediate argument. If an explicit left shift
-  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
-  //
-  // For movk, an explicit shift can be used to indicate which half word should
-  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
-  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
-  // most-significant.
-
-  // Move immediate and keep.
-  void movk(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVK);
-  }
-
-  // Move inverted immediate.
-  void movn(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVN);
-  }
-
-  // Move immediate.
-  void movz(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVZ);
-  }
-
-  // Misc instructions.
-  // Monitor debug-mode breakpoint.
-  void brk(int code);
-
-  // Halting debug-mode breakpoint.
-  void hlt(int code);
-
-  // Move register to register.
-  void mov(const Register& rd, const Register& rn);
-
-  // Move inverted operand to register.
-  void mvn(const Register& rd, const Operand& operand);
-
-  // System instructions.
-  // Move to register from system register.
-  void mrs(const Register& rt, SystemRegister sysreg);
-
-  // Move from register to system register.
-  void msr(SystemRegister sysreg, const Register& rt);
-
-  // System hint.
-  void hint(SystemHint code);
-
-  // Clear exclusive monitor.
-  void clrex(int imm4 = 0xf);
-
-  // Data memory barrier.
-  void dmb(BarrierDomain domain, BarrierType type);
-
-  // Data synchronization barrier.
-  void dsb(BarrierDomain domain, BarrierType type);
-
-  // Instruction synchronization barrier.
-  void isb();
-
-  // Alias for system instructions.
-  // No-op.
-  void nop() {
-    hint(NOP);
-  }
-
-  // FP instructions.
-  // Move double precision immediate to FP register.
-  void fmov(const FPRegister& fd, double imm);
-
-  // Move single precision immediate to FP register.
-  void fmov(const FPRegister& fd, float imm);
-
-  // Move FP register to register.
-  void fmov(const Register& rd, const FPRegister& fn);
-
-  // Move register to FP register.
-  void fmov(const FPRegister& fd, const Register& rn);
-
-  // Move FP register to FP register.
-  void fmov(const FPRegister& fd, const FPRegister& fn);
-
-  // FP add.
-  void fadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP subtract.
-  void fsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP multiply.
-  void fmul(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP fused multiply and add.
-  void fmadd(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             const FPRegister& fa);
-
-  // FP fused multiply and subtract.
-  void fmsub(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             const FPRegister& fa);
-
-  // FP fused multiply, add and negate.
-  void fnmadd(const FPRegister& fd,
-              const FPRegister& fn,
-              const FPRegister& fm,
-              const FPRegister& fa);
-
-  // FP fused multiply, subtract and negate.
-  void fnmsub(const FPRegister& fd,
-              const FPRegister& fn,
-              const FPRegister& fm,
-              const FPRegister& fa);
-
-  // FP divide.
-  void fdiv(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP maximum.
-  void fmax(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP minimum.
-  void fmin(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP maximum number.
-  void fmaxnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP minimum number.
-  void fminnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP absolute.
-  void fabs(const FPRegister& fd, const FPRegister& fn);
-
-  // FP negate.
-  void fneg(const FPRegister& fd, const FPRegister& fn);
-
-  // FP square root.
-  void fsqrt(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (nearest with ties to away).
-  void frinta(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (implicit rounding).
-  void frinti(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (toward minus infinity).
-  void frintm(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (nearest with ties to even).
-  void frintn(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (toward plus infinity).
-  void frintp(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (exact, implicit rounding).
-  void frintx(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (towards zero).
-  void frintz(const FPRegister& fd, const FPRegister& fn);
-
-  // FP compare registers.
-  void fcmp(const FPRegister& fn, const FPRegister& fm);
-
-  // FP compare immediate.
-  void fcmp(const FPRegister& fn, double value);
-
-  // FP conditional compare.
-  void fccmp(const FPRegister& fn,
-             const FPRegister& fm,
-             StatusFlags nzcv,
-             Condition cond);
-
-  // FP conditional select.
-  void fcsel(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             Condition cond);
-
-  // Common FP Convert function.
-  void FPConvertToInt(const Register& rd,
-                      const FPRegister& fn,
-                      FPIntegerConvertOp op);
-
-  // FP convert between single and double precision.
-  void fcvt(const FPRegister& fd, const FPRegister& fn);
-
-  // Convert FP to signed integer (nearest with ties to away).
-  void fcvtas(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (nearest with ties to away).
-  void fcvtau(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (round towards -infinity).
-  void fcvtms(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (round towards -infinity).
-  void fcvtmu(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (nearest with ties to even).
-  void fcvtns(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (nearest with ties to even).
-  void fcvtnu(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (round towards zero).
-  void fcvtzs(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (round towards zero).
-  void fcvtzu(const Register& rd, const FPRegister& fn);
-
-  // Convert signed integer or fixed point to FP.
-  void scvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
-
-  // Convert unsigned integer or fixed point to FP.
-  void ucvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
-
-  // Emit generic instructions.
-  // Emit raw instructions into the instruction stream.
-  void dci(Instr raw_inst) { Emit(raw_inst); }
-
-  // Emit 32 bits of data into the instruction stream.
-  void dc32(uint32_t data) {
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit32(data);
-  }
-
-  // Emit 64 bits of data into the instruction stream.
-  void dc64(uint64_t data) {
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit64(data);
-  }
-
-  // Copy a string into the instruction stream, including the terminating NULL
-  // character. The instruction pointer is then aligned correctly for
-  // subsequent instructions.
-  void EmitString(const char * string) {
-    VIXL_ASSERT(string != NULL);
-    VIXL_ASSERT(buffer_monitor_ > 0);
-
-    buffer_->EmitString(string);
-    buffer_->Align();
-  }
-
-  // Code generation helpers.
-
-  // Register encoding.
-  static Instr Rd(CPURegister rd) {
-    VIXL_ASSERT(rd.code() != kSPRegInternalCode);
-    return rd.code() << Rd_offset;
-  }
-
-  static Instr Rn(CPURegister rn) {
-    VIXL_ASSERT(rn.code() != kSPRegInternalCode);
-    return rn.code() << Rn_offset;
-  }
-
-  static Instr Rm(CPURegister rm) {
-    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
-    return rm.code() << Rm_offset;
-  }
-
-  static Instr Ra(CPURegister ra) {
-    VIXL_ASSERT(ra.code() != kSPRegInternalCode);
-    return ra.code() << Ra_offset;
-  }
-
-  static Instr Rt(CPURegister rt) {
-    VIXL_ASSERT(rt.code() != kSPRegInternalCode);
-    return rt.code() << Rt_offset;
-  }
-
-  static Instr Rt2(CPURegister rt2) {
-    VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
-    return rt2.code() << Rt2_offset;
-  }
-
-  static Instr Rs(CPURegister rs) {
-    VIXL_ASSERT(rs.code() != kSPRegInternalCode);
-    return rs.code() << Rs_offset;
-  }
-
-  // These encoding functions allow the stack pointer to be encoded, and
-  // disallow the zero register.
-  static Instr RdSP(Register rd) {
-    VIXL_ASSERT(!rd.IsZero());
-    return (rd.code() & kRegCodeMask) << Rd_offset;
-  }
-
-  static Instr RnSP(Register rn) {
-    VIXL_ASSERT(!rn.IsZero());
-    return (rn.code() & kRegCodeMask) << Rn_offset;
-  }
-
-  // Flags encoding.
-  static Instr Flags(FlagsUpdate S) {
-    if (S == SetFlags) {
-      return 1 << FlagsUpdate_offset;
-    } else if (S == LeaveFlags) {
-      return 0 << FlagsUpdate_offset;
-    }
-    VIXL_UNREACHABLE();
-    return 0;
-  }
-
-  static Instr Cond(Condition cond) {
-    return cond << Condition_offset;
-  }
-
-  // PC-relative address encoding.
-  static Instr ImmPCRelAddress(int imm21) {
-    VIXL_ASSERT(is_int21(imm21));
-    Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
-    Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
-    Instr immlo = imm << ImmPCRelLo_offset;
-    return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
-  }
-
-  // Branch encoding.
-  static Instr ImmUncondBranch(int imm26) {
-    VIXL_ASSERT(is_int26(imm26));
-    return truncate_to_int26(imm26) << ImmUncondBranch_offset;
-  }
-
-  static Instr ImmCondBranch(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmCondBranch_offset;
-  }
-
-  static Instr ImmCmpBranch(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmCmpBranch_offset;
-  }
-
-  static Instr ImmTestBranch(int imm14) {
-    VIXL_ASSERT(is_int14(imm14));
-    return truncate_to_int14(imm14) << ImmTestBranch_offset;
-  }
-
-  static Instr ImmTestBranchBit(unsigned bit_pos) {
-    VIXL_ASSERT(is_uint6(bit_pos));
-    // Subtract five from the shift offset, as we need bit 5 from bit_pos.
-    unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
-    unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
-    b5 &= ImmTestBranchBit5_mask;
-    b40 &= ImmTestBranchBit40_mask;
-    return b5 | b40;
-  }
-
-  // Data Processing encoding.
-  static Instr SF(Register rd) {
-      return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
-  }
-
-  static Instr ImmAddSub(int64_t imm) {
-    VIXL_ASSERT(IsImmAddSub(imm));
-    if (is_uint12(imm)) {  // No shift required.
-      return imm << ImmAddSub_offset;
-    } else {
-      return ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
-    }
-  }
-
-  static Instr ImmS(unsigned imms, unsigned reg_size) {
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(imms)) ||
-           ((reg_size == kWRegSize) && is_uint5(imms)));
-    USE(reg_size);
-    return imms << ImmS_offset;
-  }
-
-  static Instr ImmR(unsigned immr, unsigned reg_size) {
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
-           ((reg_size == kWRegSize) && is_uint5(immr)));
-    USE(reg_size);
-    VIXL_ASSERT(is_uint6(immr));
-    return immr << ImmR_offset;
-  }
-
-  static Instr ImmSetBits(unsigned imms, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT(is_uint6(imms));
-    VIXL_ASSERT((reg_size == kXRegSize) || is_uint6(imms + 3));
-    USE(reg_size);
-    return imms << ImmSetBits_offset;
-  }
-
-  static Instr ImmRotate(unsigned immr, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
-           ((reg_size == kWRegSize) && is_uint5(immr)));
-    USE(reg_size);
-    return immr << ImmRotate_offset;
-  }
-
-  static Instr ImmLLiteral(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmLLiteral_offset;
-  }
-
-  static Instr BitN(unsigned bitn, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
-    USE(reg_size);
-    return bitn << BitN_offset;
-  }
-
-  static Instr ShiftDP(Shift shift) {
-    VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
-    return shift << ShiftDP_offset;
-  }
-
-  static Instr ImmDPShift(unsigned amount) {
-    VIXL_ASSERT(is_uint6(amount));
-    return amount << ImmDPShift_offset;
-  }
-
-  static Instr ExtendMode(Extend extend) {
-    return extend << ExtendMode_offset;
-  }
-
-  static Instr ImmExtendShift(unsigned left_shift) {
-    VIXL_ASSERT(left_shift <= 4);
-    return left_shift << ImmExtendShift_offset;
-  }
-
-  static Instr ImmCondCmp(unsigned imm) {
-    VIXL_ASSERT(is_uint5(imm));
-    return imm << ImmCondCmp_offset;
-  }
-
-  static Instr Nzcv(StatusFlags nzcv) {
-    return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
-  }
-
-  // MemOperand offset encoding.
-  static Instr ImmLSUnsigned(int imm12) {
-    VIXL_ASSERT(is_uint12(imm12));
-    return imm12 << ImmLSUnsigned_offset;
-  }
-
-  static Instr ImmLS(int imm9) {
-    VIXL_ASSERT(is_int9(imm9));
-    return truncate_to_int9(imm9) << ImmLS_offset;
-  }
-
-  static Instr ImmLSPair(int imm7, LSDataSize size) {
-    VIXL_ASSERT(((imm7 >> size) << size) == imm7);
-    int scaled_imm7 = imm7 >> size;
-    VIXL_ASSERT(is_int7(scaled_imm7));
-    return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
-  }
-
-  static Instr ImmShiftLS(unsigned shift_amount) {
-    VIXL_ASSERT(is_uint1(shift_amount));
-    return shift_amount << ImmShiftLS_offset;
-  }
-
-  static Instr ImmPrefetchOperation(int imm5) {
-    VIXL_ASSERT(is_uint5(imm5));
-    return imm5 << ImmPrefetchOperation_offset;
-  }
-
-  static Instr ImmException(int imm16) {
-    VIXL_ASSERT(is_uint16(imm16));
-    return imm16 << ImmException_offset;
-  }
-
-  static Instr ImmSystemRegister(int imm15) {
-    VIXL_ASSERT(is_uint15(imm15));
-    return imm15 << ImmSystemRegister_offset;
-  }
-
-  static Instr ImmHint(int imm7) {
-    VIXL_ASSERT(is_uint7(imm7));
-    return imm7 << ImmHint_offset;
-  }
-
-  static Instr CRm(int imm4) {
-    VIXL_ASSERT(is_uint4(imm4));
-    return imm4 << CRm_offset;
-  }
-
-  static Instr ImmBarrierDomain(int imm2) {
-    VIXL_ASSERT(is_uint2(imm2));
-    return imm2 << ImmBarrierDomain_offset;
-  }
-
-  static Instr ImmBarrierType(int imm2) {
-    VIXL_ASSERT(is_uint2(imm2));
-    return imm2 << ImmBarrierType_offset;
-  }
-
-  static LSDataSize CalcLSDataSize(LoadStoreOp op) {
-    VIXL_ASSERT((SizeLS_offset + SizeLS_width) == (kInstructionSize * 8));
-    return static_cast<LSDataSize>(op >> SizeLS_offset);
-  }
-
-  // Move immediates encoding.
-  static Instr ImmMoveWide(uint64_t imm) {
-    VIXL_ASSERT(is_uint16(imm));
-    return imm << ImmMoveWide_offset;
-  }
-
-  static Instr ShiftMoveWide(int64_t shift) {
-    VIXL_ASSERT(is_uint2(shift));
-    return shift << ShiftMoveWide_offset;
-  }
-
-  // FP Immediates.
-  static Instr ImmFP32(float imm);
-  static Instr ImmFP64(double imm);
-
-  // FP register type.
-  static Instr FPType(FPRegister fd) {
-    return fd.Is64Bits() ? FP64 : FP32;
-  }
-
-  static Instr FPScale(unsigned scale) {
-    VIXL_ASSERT(is_uint6(scale));
-    return scale << FPScale_offset;
-  }
-
-  // Immediate field checking helpers.
-  static bool IsImmAddSub(int64_t immediate);
-  static bool IsImmConditionalCompare(int64_t immediate);
-  static bool IsImmFP32(float imm);
-  static bool IsImmFP64(double imm);
-  static bool IsImmLogical(uint64_t value,
-                           unsigned width,
-                           unsigned* n = NULL,
-                           unsigned* imm_s = NULL,
-                           unsigned* imm_r = NULL);
-  static bool IsImmLSPair(int64_t offset, LSDataSize size);
-  static bool IsImmLSScaled(int64_t offset, LSDataSize size);
-  static bool IsImmLSUnscaled(int64_t offset);
-  static bool IsImmMovn(uint64_t imm, unsigned reg_size);
-  static bool IsImmMovz(uint64_t imm, unsigned reg_size);
-
-  // Size of the code generated since label to the current position.
-  size_t SizeOfCodeGeneratedSince(Label* label) const {
-    VIXL_ASSERT(label->IsBound());
-    return buffer_->OffsetFrom(label->location());
-  }
-
-  size_t SizeOfCodeGenerated() const {
-    return buffer_->CursorOffset();
-  }
-
-  size_t BufferCapacity() const { return buffer_->capacity(); }
-
-  size_t RemainingBufferSpace() const { return buffer_->RemainingBytes(); }
-
-  void EnsureSpaceFor(size_t amount) {
-    if (buffer_->RemainingBytes() < amount) {
-      size_t capacity = buffer_->capacity();
-      size_t size = buffer_->CursorOffset();
-      do {
-        // TODO(all): refine.
-        capacity *= 2;
-      } while ((capacity - size) <  amount);
-      buffer_->Grow(capacity);
-    }
-  }
-
-#ifdef VIXL_DEBUG
-  void AcquireBuffer() {
-    VIXL_ASSERT(buffer_monitor_ >= 0);
-    buffer_monitor_++;
-  }
-
-  void ReleaseBuffer() {
-    buffer_monitor_--;
-    VIXL_ASSERT(buffer_monitor_ >= 0);
-  }
-#endif
-
-  PositionIndependentCodeOption pic() const {
-    return pic_;
-  }
-
-  bool AllowPageOffsetDependentCode() const {
-    return (pic() == PageOffsetDependentCode) ||
-           (pic() == PositionDependentCode);
-  }
-
-  static const Register& AppropriateZeroRegFor(const CPURegister& reg) {
-    return reg.Is64Bits() ? xzr : wzr;
-  }
-
-
- protected:
-  void LoadStore(const CPURegister& rt,
-                 const MemOperand& addr,
-                 LoadStoreOp op,
-                 LoadStoreScalingOption option = PreferScaledOffset);
-
-  void LoadStorePair(const CPURegister& rt,
-                     const CPURegister& rt2,
-                     const MemOperand& addr,
-                     LoadStorePairOp op);
-
-  void Prefetch(PrefetchOperation op,
-                const MemOperand& addr,
-                LoadStoreScalingOption option = PreferScaledOffset);
-
-  // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
-  // reports a bogus uninitialised warning then.
-  void Logical(const Register& rd,
-               const Register& rn,
-               const Operand operand,
-               LogicalOp op);
-  void LogicalImmediate(const Register& rd,
-                        const Register& rn,
-                        unsigned n,
-                        unsigned imm_s,
-                        unsigned imm_r,
-                        LogicalOp op);
-
-  void ConditionalCompare(const Register& rn,
-                          const Operand& operand,
-                          StatusFlags nzcv,
-                          Condition cond,
-                          ConditionalCompareOp op);
-
-  void AddSubWithCarry(const Register& rd,
-                       const Register& rn,
-                       const Operand& operand,
-                       FlagsUpdate S,
-                       AddSubWithCarryOp op);
-
-
-  // Functions for emulating operands not directly supported by the instruction
-  // set.
-  void EmitShift(const Register& rd,
-                 const Register& rn,
-                 Shift shift,
-                 unsigned amount);
-  void EmitExtendShift(const Register& rd,
-                       const Register& rn,
-                       Extend extend,
-                       unsigned left_shift);
-
-  void AddSub(const Register& rd,
-              const Register& rn,
-              const Operand& operand,
-              FlagsUpdate S,
-              AddSubOp op);
-
-  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
-  // registers. Only simple loads are supported; sign- and zero-extension (such
-  // as in LDPSW_x or LDRB_w) are not supported.
-  static LoadStoreOp LoadOpFor(const CPURegister& rt);
-  static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
-                                       const CPURegister& rt2);
-  static LoadStoreOp StoreOpFor(const CPURegister& rt);
-  static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
-                                        const CPURegister& rt2);
-  static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
-    const CPURegister& rt, const CPURegister& rt2);
-  static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
-    const CPURegister& rt, const CPURegister& rt2);
-  static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
-
-
- private:
-  // Instruction helpers.
-  void MoveWide(const Register& rd,
-                uint64_t imm,
-                int shift,
-                MoveWideImmediateOp mov_op);
-  void DataProcShiftedRegister(const Register& rd,
-                               const Register& rn,
-                               const Operand& operand,
-                               FlagsUpdate S,
-                               Instr op);
-  void DataProcExtendedRegister(const Register& rd,
-                                const Register& rn,
-                                const Operand& operand,
-                                FlagsUpdate S,
-                                Instr op);
-  void LoadStorePairNonTemporal(const CPURegister& rt,
-                                const CPURegister& rt2,
-                                const MemOperand& addr,
-                                LoadStorePairNonTemporalOp op);
-  void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
-  void ConditionalSelect(const Register& rd,
-                         const Register& rn,
-                         const Register& rm,
-                         Condition cond,
-                         ConditionalSelectOp op);
-  void DataProcessing1Source(const Register& rd,
-                             const Register& rn,
-                             DataProcessing1SourceOp op);
-  void DataProcessing3Source(const Register& rd,
-                             const Register& rn,
-                             const Register& rm,
-                             const Register& ra,
-                             DataProcessing3SourceOp op);
-  void FPDataProcessing1Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               FPDataProcessing1SourceOp op);
-  void FPDataProcessing2Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               const FPRegister& fm,
-                               FPDataProcessing2SourceOp op);
-  void FPDataProcessing3Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               const FPRegister& fm,
-                               const FPRegister& fa,
-                               FPDataProcessing3SourceOp op);
-
-  // Encode the specified MemOperand for the specified access size and scaling
-  // preference.
-  Instr LoadStoreMemOperand(const MemOperand& addr,
-                            LSDataSize size,
-                            LoadStoreScalingOption option);
-
-  // Link the current (not-yet-emitted) instruction to the specified label, then
-  // return an offset to be encoded in the instruction. If the label is not yet
-  // bound, an offset of 0 is returned.
-  ptrdiff_t LinkAndGetByteOffsetTo(Label * label);
-  ptrdiff_t LinkAndGetInstructionOffsetTo(Label * label);
-  ptrdiff_t LinkAndGetPageOffsetTo(Label * label);
-
-  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
-  template <int element_shift>
-  ptrdiff_t LinkAndGetOffsetTo(Label* label);
-
-  // Literal load offset are in words (32-bit).
-  ptrdiff_t LinkAndGetWordOffsetTo(RawLiteral* literal);
-
-  // Emit the instruction in buffer_.
-  void Emit(Instr instruction) {
-    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit32(instruction);
-  }
-
-  // Buffer where the code is emitted.
-  CodeBuffer* buffer_;
-  PositionIndependentCodeOption pic_;
-
-#ifdef VIXL_DEBUG
-  int64_t buffer_monitor_;
-#endif
-};
-
-
-// All Assembler emits MUST acquire/release the underlying code buffer. The
-// helper scope below will do so and optionally ensure the buffer is big enough
-// to receive the emit. It is possible to request the scope not to perform any
-// checks (kNoCheck) if for example it is known in advance the buffer size is
-// adequate or there is some other size checking mechanism in place.
-class CodeBufferCheckScope {
- public:
-  // Tell whether or not the scope needs to ensure the associated CodeBuffer
-  // has enough space for the requested size.
-  enum CheckPolicy {
-    kNoCheck,
-    kCheck
-  };
-
-  // Tell whether or not the scope should assert the amount of code emitted
-  // within the scope is consistent with the requested amount.
-  enum AssertPolicy {
-    kNoAssert,    // No assert required.
-    kExactSize,   // The code emitted must be exactly size bytes.
-    kMaximumSize  // The code emitted must be at most size bytes.
-  };
-
-  CodeBufferCheckScope(Assembler* assm,
-                       size_t size,
-                       CheckPolicy check_policy = kCheck,
-                       AssertPolicy assert_policy = kMaximumSize)
-      : assm_(assm) {
-    if (check_policy == kCheck) assm->EnsureSpaceFor(size);
-#ifdef VIXL_DEBUG
-    assm->bind(&start_);
-    size_ = size;
-    assert_policy_ = assert_policy;
-    assm->AcquireBuffer();
-#else
-    USE(assert_policy);
-#endif
-  }
-
-  // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
-  explicit CodeBufferCheckScope(Assembler* assm) : assm_(assm) {
-#ifdef VIXL_DEBUG
-    size_ = 0;
-    assert_policy_ = kNoAssert;
-    assm->AcquireBuffer();
-#endif
-  }
-
-  ~CodeBufferCheckScope() {
-#ifdef VIXL_DEBUG
-    assm_->ReleaseBuffer();
-    switch (assert_policy_) {
-      case kNoAssert: break;
-      case kExactSize:
-        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) == size_);
-        break;
-      case kMaximumSize:
-        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) <= size_);
-        break;
-      default:
-        VIXL_UNREACHABLE();
-    }
-#endif
-  }
-
- protected:
-  Assembler* assm_;
-#ifdef VIXL_DEBUG
-  Label start_;
-  size_t size_;
-  AssertPolicy assert_policy_;
-#endif
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/qemu/disas/libvixl/a64/disasm-a64.cc b/qemu/disas/libvixl/a64/disasm-a64.cc
deleted file mode 100644
index f7bc2468b..000000000
--- a/qemu/disas/libvixl/a64/disasm-a64.cc
+++ /dev/null
@@ -1,1954 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cstdlib>
-#include "a64/disasm-a64.h"
-
-namespace vixl {
-
-Disassembler::Disassembler() {
-  buffer_size_ = 256;
-  buffer_ = reinterpret_cast<char*>(malloc(buffer_size_));
-  buffer_pos_ = 0;
-  own_buffer_ = true;
-  code_address_offset_ = 0;
-}
-
-
-Disassembler::Disassembler(char* text_buffer, int buffer_size) {
-  buffer_size_ = buffer_size;
-  buffer_ = text_buffer;
-  buffer_pos_ = 0;
-  own_buffer_ = false;
-  code_address_offset_ = 0;
-}
-
-
-Disassembler::~Disassembler() {
-  if (own_buffer_) {
-    free(buffer_);
-  }
-}
-
-
-char* Disassembler::GetOutput() {
-  return buffer_;
-}
-
-
-void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
-                  (instr->ImmAddSub() == 0) ? true : false;
-  const char *mnemonic = "";
-  const char *form = "'Rds, 'Rns, 'IAddSub";
-  const char *form_cmp = "'Rns, 'IAddSub";
-  const char *form_mov = "'Rds, 'Rns";
-
-  switch (instr->Mask(AddSubImmediateMask)) {
-    case ADD_w_imm:
-    case ADD_x_imm: {
-      mnemonic = "add";
-      if (stack_op) {
-        mnemonic = "mov";
-        form = form_mov;
-      }
-      break;
-    }
-    case ADDS_w_imm:
-    case ADDS_x_imm: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_imm:
-    case SUB_x_imm: mnemonic = "sub"; break;
-    case SUBS_w_imm:
-    case SUBS_x_imm: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubShifted(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm'HDP";
-  const char *form_cmp = "'Rn, 'Rm'HDP";
-  const char *form_neg = "'Rd, 'Rm'HDP";
-
-  switch (instr->Mask(AddSubShiftedMask)) {
-    case ADD_w_shift:
-    case ADD_x_shift: mnemonic = "add"; break;
-    case ADDS_w_shift:
-    case ADDS_x_shift: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_shift:
-    case SUB_x_shift: {
-      mnemonic = "sub";
-      if (rn_is_zr) {
-        mnemonic = "neg";
-        form = form_neg;
-      }
-      break;
-    }
-    case SUBS_w_shift:
-    case SUBS_x_shift: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      } else if (rn_is_zr) {
-        mnemonic = "negs";
-        form = form_neg;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubExtended(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  const char *mnemonic = "";
-  Extend mode = static_cast<Extend>(instr->ExtendMode());
-  const char *form = ((mode == UXTX) || (mode == SXTX)) ?
-                     "'Rds, 'Rns, 'Xm'Ext" : "'Rds, 'Rns, 'Wm'Ext";
-  const char *form_cmp = ((mode == UXTX) || (mode == SXTX)) ?
-                         "'Rns, 'Xm'Ext" : "'Rns, 'Wm'Ext";
-
-  switch (instr->Mask(AddSubExtendedMask)) {
-    case ADD_w_ext:
-    case ADD_x_ext: mnemonic = "add"; break;
-    case ADDS_w_ext:
-    case ADDS_x_ext: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_ext:
-    case SUB_x_ext: mnemonic = "sub"; break;
-    case SUBS_w_ext:
-    case SUBS_x_ext: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm";
-  const char *form_neg = "'Rd, 'Rm";
-
-  switch (instr->Mask(AddSubWithCarryMask)) {
-    case ADC_w:
-    case ADC_x: mnemonic = "adc"; break;
-    case ADCS_w:
-    case ADCS_x: mnemonic = "adcs"; break;
-    case SBC_w:
-    case SBC_x: {
-      mnemonic = "sbc";
-      if (rn_is_zr) {
-        mnemonic = "ngc";
-        form = form_neg;
-      }
-      break;
-    }
-    case SBCS_w:
-    case SBCS_x: {
-      mnemonic = "sbcs";
-      if (rn_is_zr) {
-        mnemonic = "ngcs";
-        form = form_neg;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rds, 'Rn, 'ITri";
-
-  if (instr->ImmLogical() == 0) {
-    // The immediate encoded in the instruction is not in the expected format.
-    Format(instr, "unallocated", "(LogicalImmediate)");
-    return;
-  }
-
-  switch (instr->Mask(LogicalImmediateMask)) {
-    case AND_w_imm:
-    case AND_x_imm: mnemonic = "and"; break;
-    case ORR_w_imm:
-    case ORR_x_imm: {
-      mnemonic = "orr";
-      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
-                                                        : kWRegSize;
-      if (rn_is_zr && !IsMovzMovnImm(reg_size, instr->ImmLogical())) {
-        mnemonic = "mov";
-        form = "'Rds, 'ITri";
-      }
-      break;
-    }
-    case EOR_w_imm:
-    case EOR_x_imm: mnemonic = "eor"; break;
-    case ANDS_w_imm:
-    case ANDS_x_imm: {
-      mnemonic = "ands";
-      if (rd_is_zr) {
-        mnemonic = "tst";
-        form = "'Rn, 'ITri";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
-  VIXL_ASSERT((reg_size == kXRegSize) ||
-              ((reg_size == kWRegSize) && (value <= 0xffffffff)));
-
-  // Test for movz: 16 bits set at positions 0, 16, 32 or 48.
-  if (((value & UINT64_C(0xffffffffffff0000)) == 0) ||
-      ((value & UINT64_C(0xffffffff0000ffff)) == 0) ||
-      ((value & UINT64_C(0xffff0000ffffffff)) == 0) ||
-      ((value & UINT64_C(0x0000ffffffffffff)) == 0)) {
-    return true;
-  }
-
-  // Test for movn: NOT(16 bits set at positions 0, 16, 32 or 48).
-  if ((reg_size == kXRegSize) &&
-      (((~value & UINT64_C(0xffffffffffff0000)) == 0) ||
-       ((~value & UINT64_C(0xffffffff0000ffff)) == 0) ||
-       ((~value & UINT64_C(0xffff0000ffffffff)) == 0) ||
-       ((~value & UINT64_C(0x0000ffffffffffff)) == 0))) {
-    return true;
-  }
-  if ((reg_size == kWRegSize) &&
-      (((value & 0xffff0000) == 0xffff0000) ||
-       ((value & 0x0000ffff) == 0x0000ffff))) {
-    return true;
-  }
-  return false;
-}
-
-
-void Disassembler::VisitLogicalShifted(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm'HLo";
-
-  switch (instr->Mask(LogicalShiftedMask)) {
-    case AND_w:
-    case AND_x: mnemonic = "and"; break;
-    case BIC_w:
-    case BIC_x: mnemonic = "bic"; break;
-    case EOR_w:
-    case EOR_x: mnemonic = "eor"; break;
-    case EON_w:
-    case EON_x: mnemonic = "eon"; break;
-    case BICS_w:
-    case BICS_x: mnemonic = "bics"; break;
-    case ANDS_w:
-    case ANDS_x: {
-      mnemonic = "ands";
-      if (rd_is_zr) {
-        mnemonic = "tst";
-        form = "'Rn, 'Rm'HLo";
-      }
-      break;
-    }
-    case ORR_w:
-    case ORR_x: {
-      mnemonic = "orr";
-      if (rn_is_zr && (instr->ImmDPShift() == 0) && (instr->ShiftDP() == LSL)) {
-        mnemonic = "mov";
-        form = "'Rd, 'Rm";
-      }
-      break;
-    }
-    case ORN_w:
-    case ORN_x: {
-      mnemonic = "orn";
-      if (rn_is_zr) {
-        mnemonic = "mvn";
-        form = "'Rd, 'Rm'HLo";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
-
-  switch (instr->Mask(ConditionalCompareRegisterMask)) {
-    case CCMN_w:
-    case CCMN_x: mnemonic = "ccmn"; break;
-    case CCMP_w:
-    case CCMP_x: mnemonic = "ccmp"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
-
-  switch (instr->Mask(ConditionalCompareImmediateMask)) {
-    case CCMN_w_imm:
-    case CCMN_x_imm: mnemonic = "ccmn"; break;
-    case CCMP_w_imm:
-    case CCMP_x_imm: mnemonic = "ccmp"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalSelect(const Instruction* instr) {
-  bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
-  bool rn_is_rm = (instr->Rn() == instr->Rm());
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm, 'Cond";
-  const char *form_test = "'Rd, 'CInv";
-  const char *form_update = "'Rd, 'Rn, 'CInv";
-
-  Condition cond = static_cast<Condition>(instr->Condition());
-  bool invertible_cond = (cond != al) && (cond != nv);
-
-  switch (instr->Mask(ConditionalSelectMask)) {
-    case CSEL_w:
-    case CSEL_x: mnemonic = "csel"; break;
-    case CSINC_w:
-    case CSINC_x: {
-      mnemonic = "csinc";
-      if (rnm_is_zr && invertible_cond) {
-        mnemonic = "cset";
-        form = form_test;
-      } else if (rn_is_rm && invertible_cond) {
-        mnemonic = "cinc";
-        form = form_update;
-      }
-      break;
-    }
-    case CSINV_w:
-    case CSINV_x: {
-      mnemonic = "csinv";
-      if (rnm_is_zr && invertible_cond) {
-        mnemonic = "csetm";
-        form = form_test;
-      } else if (rn_is_rm && invertible_cond) {
-        mnemonic = "cinv";
-        form = form_update;
-      }
-      break;
-    }
-    case CSNEG_w:
-    case CSNEG_x: {
-      mnemonic = "csneg";
-      if (rn_is_rm && invertible_cond) {
-        mnemonic = "cneg";
-        form = form_update;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitBitfield(const Instruction* instr) {
-  unsigned s = instr->ImmS();
-  unsigned r = instr->ImmR();
-  unsigned rd_size_minus_1 =
-    ((instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize) - 1;
-  const char *mnemonic = "";
-  const char *form = "";
-  const char *form_shift_right = "'Rd, 'Rn, 'IBr";
-  const char *form_extend = "'Rd, 'Wn";
-  const char *form_bfiz = "'Rd, 'Rn, 'IBZ-r, 'IBs+1";
-  const char *form_bfx = "'Rd, 'Rn, 'IBr, 'IBs-r+1";
-  const char *form_lsl = "'Rd, 'Rn, 'IBZ-r";
-
-  switch (instr->Mask(BitfieldMask)) {
-    case SBFM_w:
-    case SBFM_x: {
-      mnemonic = "sbfx";
-      form = form_bfx;
-      if (r == 0) {
-        form = form_extend;
-        if (s == 7) {
-          mnemonic = "sxtb";
-        } else if (s == 15) {
-          mnemonic = "sxth";
-        } else if ((s == 31) && (instr->SixtyFourBits() == 1)) {
-          mnemonic = "sxtw";
-        } else {
-          form = form_bfx;
-        }
-      } else if (s == rd_size_minus_1) {
-        mnemonic = "asr";
-        form = form_shift_right;
-      } else if (s < r) {
-        mnemonic = "sbfiz";
-        form = form_bfiz;
-      }
-      break;
-    }
-    case UBFM_w:
-    case UBFM_x: {
-      mnemonic = "ubfx";
-      form = form_bfx;
-      if (r == 0) {
-        form = form_extend;
-        if (s == 7) {
-          mnemonic = "uxtb";
-        } else if (s == 15) {
-          mnemonic = "uxth";
-        } else {
-          form = form_bfx;
-        }
-      }
-      if (s == rd_size_minus_1) {
-        mnemonic = "lsr";
-        form = form_shift_right;
-      } else if (r == s + 1) {
-        mnemonic = "lsl";
-        form = form_lsl;
-      } else if (s < r) {
-        mnemonic = "ubfiz";
-        form = form_bfiz;
-      }
-      break;
-    }
-    case BFM_w:
-    case BFM_x: {
-      mnemonic = "bfxil";
-      form = form_bfx;
-      if (s < r) {
-        mnemonic = "bfi";
-        form = form_bfiz;
-      }
-    }
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitExtract(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
-
-  switch (instr->Mask(ExtractMask)) {
-    case EXTR_w:
-    case EXTR_x: {
-      if (instr->Rn() == instr->Rm()) {
-        mnemonic = "ror";
-        form = "'Rd, 'Rn, 'IExtract";
-      } else {
-        mnemonic = "extr";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
-  switch (instr->Mask(PCRelAddressingMask)) {
-    case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
-    case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
-    default: Format(instr, "unimplemented", "(PCRelAddressing)");
-  }
-}
-
-
-void Disassembler::VisitConditionalBranch(const Instruction* instr) {
-  switch (instr->Mask(ConditionalBranchMask)) {
-    case B_cond: Format(instr, "b.'CBrn", "'BImmCond"); break;
-    default: VIXL_UNREACHABLE();
-  }
-}
-
-
-void Disassembler::VisitUnconditionalBranchToRegister(
-    const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Xn";
-
-  switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
-    case BR: mnemonic = "br"; break;
-    case BLR: mnemonic = "blr"; break;
-    case RET: {
-      mnemonic = "ret";
-      if (instr->Rn() == kLinkRegCode) {
-        form = NULL;
-      }
-      break;
-    }
-    default: form = "(UnconditionalBranchToRegister)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'BImmUncn";
-
-  switch (instr->Mask(UnconditionalBranchMask)) {
-    case B: mnemonic = "b"; break;
-    case BL: mnemonic = "bl"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn";
-
-  switch (instr->Mask(DataProcessing1SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_w:           \
-    case A##_x: mnemonic = B; break;
-    FORMAT(RBIT, "rbit");
-    FORMAT(REV16, "rev16");
-    FORMAT(REV, "rev");
-    FORMAT(CLZ, "clz");
-    FORMAT(CLS, "cls");
-    #undef FORMAT
-    case REV32_x: mnemonic = "rev32"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Rd, 'Rn, 'Rm";
-
-  switch (instr->Mask(DataProcessing2SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_w:           \
-    case A##_x: mnemonic = B; break;
-    FORMAT(UDIV, "udiv");
-    FORMAT(SDIV, "sdiv");
-    FORMAT(LSLV, "lsl");
-    FORMAT(LSRV, "lsr");
-    FORMAT(ASRV, "asr");
-    FORMAT(RORV, "ror");
-    #undef FORMAT
-    default: form = "(DataProcessing2Source)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
-  bool ra_is_zr = RaIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
-  const char *form_rrr = "'Rd, 'Rn, 'Rm";
-  const char *form_rrrr = "'Rd, 'Rn, 'Rm, 'Ra";
-  const char *form_xww = "'Xd, 'Wn, 'Wm";
-  const char *form_xxx = "'Xd, 'Xn, 'Xm";
-
-  switch (instr->Mask(DataProcessing3SourceMask)) {
-    case MADD_w:
-    case MADD_x: {
-      mnemonic = "madd";
-      form = form_rrrr;
-      if (ra_is_zr) {
-        mnemonic = "mul";
-        form = form_rrr;
-      }
-      break;
-    }
-    case MSUB_w:
-    case MSUB_x: {
-      mnemonic = "msub";
-      form = form_rrrr;
-      if (ra_is_zr) {
-        mnemonic = "mneg";
-        form = form_rrr;
-      }
-      break;
-    }
-    case SMADDL_x: {
-      mnemonic = "smaddl";
-      if (ra_is_zr) {
-        mnemonic = "smull";
-        form = form_xww;
-      }
-      break;
-    }
-    case SMSUBL_x: {
-      mnemonic = "smsubl";
-      if (ra_is_zr) {
-        mnemonic = "smnegl";
-        form = form_xww;
-      }
-      break;
-    }
-    case UMADDL_x: {
-      mnemonic = "umaddl";
-      if (ra_is_zr) {
-        mnemonic = "umull";
-        form = form_xww;
-      }
-      break;
-    }
-    case UMSUBL_x: {
-      mnemonic = "umsubl";
-      if (ra_is_zr) {
-        mnemonic = "umnegl";
-        form = form_xww;
-      }
-      break;
-    }
-    case SMULH_x: {
-      mnemonic = "smulh";
-      form = form_xxx;
-      break;
-    }
-    case UMULH_x: {
-      mnemonic = "umulh";
-      form = form_xxx;
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitCompareBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rt, 'BImmCmpa";
-
-  switch (instr->Mask(CompareBranchMask)) {
-    case CBZ_w:
-    case CBZ_x: mnemonic = "cbz"; break;
-    case CBNZ_w:
-    case CBNZ_x: mnemonic = "cbnz"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitTestBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  // If the top bit of the immediate is clear, the tested register is
-  // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
-  // encoded in bit 31 of the instruction, we can reuse the Rt form, which
-  // uses bit 31 (normally "sf") to choose the register size.
-  const char *form = "'Rt, 'IS, 'BImmTest";
-
-  switch (instr->Mask(TestBranchMask)) {
-    case TBZ: mnemonic = "tbz"; break;
-    case TBNZ: mnemonic = "tbnz"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'IMoveImm";
-
-  // Print the shift separately for movk, to make it clear which half word will
-  // be overwritten. Movn and movz print the computed immediate, which includes
-  // shift calculation.
-  switch (instr->Mask(MoveWideImmediateMask)) {
-    case MOVN_w:
-    case MOVN_x:
-      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0)) {
-        if ((instr->SixtyFourBits() == 0) && (instr->ImmMoveWide() == 0xffff)) {
-          mnemonic = "movn";
-        } else {
-          mnemonic = "mov";
-          form = "'Rd, 'IMoveNeg";
-        }
-      } else {
-        mnemonic = "movn";
-      }
-      break;
-    case MOVZ_w:
-    case MOVZ_x:
-      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0))
-        mnemonic = "mov";
-      else
-        mnemonic = "movz";
-      break;
-    case MOVK_w:
-    case MOVK_x: mnemonic = "movk"; form = "'Rd, 'IMoveLSL"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-#define LOAD_STORE_LIST(V)    \
-  V(STRB_w, "strb", "'Wt")    \
-  V(STRH_w, "strh", "'Wt")    \
-  V(STR_w, "str", "'Wt")      \
-  V(STR_x, "str", "'Xt")      \
-  V(LDRB_w, "ldrb", "'Wt")    \
-  V(LDRH_w, "ldrh", "'Wt")    \
-  V(LDR_w, "ldr", "'Wt")      \
-  V(LDR_x, "ldr", "'Xt")      \
-  V(LDRSB_x, "ldrsb", "'Xt")  \
-  V(LDRSH_x, "ldrsh", "'Xt")  \
-  V(LDRSW_x, "ldrsw", "'Xt")  \
-  V(LDRSB_w, "ldrsb", "'Wt")  \
-  V(LDRSH_w, "ldrsh", "'Wt")  \
-  V(STR_s, "str", "'St")      \
-  V(STR_d, "str", "'Dt")      \
-  V(LDR_s, "ldr", "'St")      \
-  V(LDR_d, "ldr", "'Dt")
-
-void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePreIndex)";
-
-  switch (instr->Mask(LoadStorePreIndexMask)) {
-    #define LS_PREINDEX(A, B, C) \
-    case A##_pre: mnemonic = B; form = C ", ['Xns'ILS]!"; break;
-    LOAD_STORE_LIST(LS_PREINDEX)
-    #undef LS_PREINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePostIndex)";
-
-  switch (instr->Mask(LoadStorePostIndexMask)) {
-    #define LS_POSTINDEX(A, B, C) \
-    case A##_post: mnemonic = B; form = C ", ['Xns]'ILS"; break;
-    LOAD_STORE_LIST(LS_POSTINDEX)
-    #undef LS_POSTINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStoreUnsignedOffset)";
-
-  switch (instr->Mask(LoadStoreUnsignedOffsetMask)) {
-    #define LS_UNSIGNEDOFFSET(A, B, C) \
-    case A##_unsigned: mnemonic = B; form = C ", ['Xns'ILU]"; break;
-    LOAD_STORE_LIST(LS_UNSIGNEDOFFSET)
-    #undef LS_UNSIGNEDOFFSET
-    case PRFM_unsigned: mnemonic = "prfm"; form = "'PrefOp, ['Xns'ILU]";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStoreRegisterOffset)";
-
-  switch (instr->Mask(LoadStoreRegisterOffsetMask)) {
-    #define LS_REGISTEROFFSET(A, B, C) \
-    case A##_reg: mnemonic = B; form = C ", ['Xns, 'Offsetreg]"; break;
-    LOAD_STORE_LIST(LS_REGISTEROFFSET)
-    #undef LS_REGISTEROFFSET
-    case PRFM_reg: mnemonic = "prfm"; form = "'PrefOp, ['Xns, 'Offsetreg]";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Wt, ['Xns'ILS]";
-  const char *form_x = "'Xt, ['Xns'ILS]";
-  const char *form_s = "'St, ['Xns'ILS]";
-  const char *form_d = "'Dt, ['Xns'ILS]";
-  const char *form_prefetch = "'PrefOp, ['Xns'ILS]";
-
-  switch (instr->Mask(LoadStoreUnscaledOffsetMask)) {
-    case STURB_w:  mnemonic = "sturb"; break;
-    case STURH_w:  mnemonic = "sturh"; break;
-    case STUR_w:   mnemonic = "stur"; break;
-    case STUR_x:   mnemonic = "stur"; form = form_x; break;
-    case STUR_s:   mnemonic = "stur"; form = form_s; break;
-    case STUR_d:   mnemonic = "stur"; form = form_d; break;
-    case LDURB_w:  mnemonic = "ldurb"; break;
-    case LDURH_w:  mnemonic = "ldurh"; break;
-    case LDUR_w:   mnemonic = "ldur"; break;
-    case LDUR_x:   mnemonic = "ldur"; form = form_x; break;
-    case LDUR_s:   mnemonic = "ldur"; form = form_s; break;
-    case LDUR_d:   mnemonic = "ldur"; form = form_d; break;
-    case LDURSB_x: form = form_x;  // Fall through.
-    case LDURSB_w: mnemonic = "ldursb"; break;
-    case LDURSH_x: form = form_x;  // Fall through.
-    case LDURSH_w: mnemonic = "ldursh"; break;
-    case LDURSW_x: mnemonic = "ldursw"; form = form_x; break;
-    case PRFUM:    mnemonic = "prfum"; form = form_prefetch; break;
-    default: form = "(LoadStoreUnscaledOffset)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadLiteral(const Instruction* instr) {
-  const char *mnemonic = "ldr";
-  const char *form = "(LoadLiteral)";
-
-  switch (instr->Mask(LoadLiteralMask)) {
-    case LDR_w_lit: form = "'Wt, 'ILLiteral 'LValue"; break;
-    case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
-    case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
-    case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
-    case LDRSW_x_lit: {
-      mnemonic = "ldrsw";
-      form = "'Xt, 'ILLiteral 'LValue";
-      break;
-    }
-    case PRFM_lit: {
-      mnemonic = "prfm";
-      form = "'PrefOp, 'ILLiteral 'LValue";
-      break;
-    }
-    default: mnemonic = "unimplemented";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-#define LOAD_STORE_PAIR_LIST(V)         \
-  V(STP_w, "stp", "'Wt, 'Wt2", "4")     \
-  V(LDP_w, "ldp", "'Wt, 'Wt2", "4")     \
-  V(LDPSW_x, "ldpsw", "'Xt, 'Xt2", "4") \
-  V(STP_x, "stp", "'Xt, 'Xt2", "8")     \
-  V(LDP_x, "ldp", "'Xt, 'Xt2", "8")     \
-  V(STP_s, "stp", "'St, 'St2", "4")     \
-  V(LDP_s, "ldp", "'St, 'St2", "4")     \
-  V(STP_d, "stp", "'Dt, 'Dt2", "8")     \
-  V(LDP_d, "ldp", "'Dt, 'Dt2", "8")
-
-void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairPostIndex)";
-
-  switch (instr->Mask(LoadStorePairPostIndexMask)) {
-    #define LSP_POSTINDEX(A, B, C, D) \
-    case A##_post: mnemonic = B; form = C ", ['Xns]'ILP" D; break;
-    LOAD_STORE_PAIR_LIST(LSP_POSTINDEX)
-    #undef LSP_POSTINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairPreIndex)";
-
-  switch (instr->Mask(LoadStorePairPreIndexMask)) {
-    #define LSP_PREINDEX(A, B, C, D) \
-    case A##_pre: mnemonic = B; form = C ", ['Xns'ILP" D "]!"; break;
-    LOAD_STORE_PAIR_LIST(LSP_PREINDEX)
-    #undef LSP_PREINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairOffset)";
-
-  switch (instr->Mask(LoadStorePairOffsetMask)) {
-    #define LSP_OFFSET(A, B, C, D) \
-    case A##_off: mnemonic = B; form = C ", ['Xns'ILP" D "]"; break;
-    LOAD_STORE_PAIR_LIST(LSP_OFFSET)
-    #undef LSP_OFFSET
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form;
-
-  switch (instr->Mask(LoadStorePairNonTemporalMask)) {
-    case STNP_w: mnemonic = "stnp"; form = "'Wt, 'Wt2, ['Xns'ILP4]"; break;
-    case LDNP_w: mnemonic = "ldnp"; form = "'Wt, 'Wt2, ['Xns'ILP4]"; break;
-    case STNP_x: mnemonic = "stnp"; form = "'Xt, 'Xt2, ['Xns'ILP8]"; break;
-    case LDNP_x: mnemonic = "ldnp"; form = "'Xt, 'Xt2, ['Xns'ILP8]"; break;
-    case STNP_s: mnemonic = "stnp"; form = "'St, 'St2, ['Xns'ILP4]"; break;
-    case LDNP_s: mnemonic = "ldnp"; form = "'St, 'St2, ['Xns'ILP4]"; break;
-    case STNP_d: mnemonic = "stnp"; form = "'Dt, 'Dt2, ['Xns'ILP8]"; break;
-    case LDNP_d: mnemonic = "ldnp"; form = "'Dt, 'Dt2, ['Xns'ILP8]"; break;
-    default: form = "(LoadStorePairNonTemporal)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form;
-
-  switch (instr->Mask(LoadStoreExclusiveMask)) {
-    case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
-    case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
-    case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
-    case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
-    case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
-    case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
-    case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
-    case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
-    case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
-    case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
-    case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
-    case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
-    case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
-    case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
-    case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
-    case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
-    case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
-    case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
-    case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
-    case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
-    case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
-    case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
-    case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
-    case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
-    case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
-    case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
-    default: form = "(LoadStoreExclusive)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPCompare(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Fn, 'Fm";
-  const char *form_zero = "'Fn, #0.0";
-
-  switch (instr->Mask(FPCompareMask)) {
-    case FCMP_s_zero:
-    case FCMP_d_zero: form = form_zero;  // Fall through.
-    case FCMP_s:
-    case FCMP_d: mnemonic = "fcmp"; break;
-    default: form = "(FPCompare)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
-  const char *mnemonic = "unmplemented";
-  const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
-
-  switch (instr->Mask(FPConditionalCompareMask)) {
-    case FCCMP_s:
-    case FCCMP_d: mnemonic = "fccmp"; break;
-    case FCCMPE_s:
-    case FCCMPE_d: mnemonic = "fccmpe"; break;
-    default: form = "(FPConditionalCompare)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
-
-  switch (instr->Mask(FPConditionalSelectMask)) {
-    case FCSEL_s:
-    case FCSEL_d: mnemonic = "fcsel"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Fd, 'Fn";
-
-  switch (instr->Mask(FPDataProcessing1SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMOV, "fmov");
-    FORMAT(FABS, "fabs");
-    FORMAT(FNEG, "fneg");
-    FORMAT(FSQRT, "fsqrt");
-    FORMAT(FRINTN, "frintn");
-    FORMAT(FRINTP, "frintp");
-    FORMAT(FRINTM, "frintm");
-    FORMAT(FRINTZ, "frintz");
-    FORMAT(FRINTA, "frinta");
-    FORMAT(FRINTX, "frintx");
-    FORMAT(FRINTI, "frinti");
-    #undef FORMAT
-    case FCVT_ds: mnemonic = "fcvt"; form = "'Dd, 'Sn"; break;
-    case FCVT_sd: mnemonic = "fcvt"; form = "'Sd, 'Dn"; break;
-    default: form = "(FPDataProcessing1Source)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm";
-
-  switch (instr->Mask(FPDataProcessing2SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMUL, "fmul");
-    FORMAT(FDIV, "fdiv");
-    FORMAT(FADD, "fadd");
-    FORMAT(FSUB, "fsub");
-    FORMAT(FMAX, "fmax");
-    FORMAT(FMIN, "fmin");
-    FORMAT(FMAXNM, "fmaxnm");
-    FORMAT(FMINNM, "fminnm");
-    FORMAT(FNMUL, "fnmul");
-    #undef FORMAT
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
-
-  switch (instr->Mask(FPDataProcessing3SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMADD, "fmadd");
-    FORMAT(FMSUB, "fmsub");
-    FORMAT(FNMADD, "fnmadd");
-    FORMAT(FNMSUB, "fnmsub");
-    #undef FORMAT
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "(FPImmediate)";
-
-  switch (instr->Mask(FPImmediateMask)) {
-    case FMOV_s_imm: mnemonic = "fmov"; form = "'Sd, 'IFPSingle"; break;
-    case FMOV_d_imm: mnemonic = "fmov"; form = "'Dd, 'IFPDouble"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(FPIntegerConvert)";
-  const char *form_rf = "'Rd, 'Fn";
-  const char *form_fr = "'Fd, 'Rn";
-
-  switch (instr->Mask(FPIntegerConvertMask)) {
-    case FMOV_ws:
-    case FMOV_xd: mnemonic = "fmov"; form = form_rf; break;
-    case FMOV_sw:
-    case FMOV_dx: mnemonic = "fmov"; form = form_fr; break;
-    case FCVTAS_ws:
-    case FCVTAS_xs:
-    case FCVTAS_wd:
-    case FCVTAS_xd: mnemonic = "fcvtas"; form = form_rf; break;
-    case FCVTAU_ws:
-    case FCVTAU_xs:
-    case FCVTAU_wd:
-    case FCVTAU_xd: mnemonic = "fcvtau"; form = form_rf; break;
-    case FCVTMS_ws:
-    case FCVTMS_xs:
-    case FCVTMS_wd:
-    case FCVTMS_xd: mnemonic = "fcvtms"; form = form_rf; break;
-    case FCVTMU_ws:
-    case FCVTMU_xs:
-    case FCVTMU_wd:
-    case FCVTMU_xd: mnemonic = "fcvtmu"; form = form_rf; break;
-    case FCVTNS_ws:
-    case FCVTNS_xs:
-    case FCVTNS_wd:
-    case FCVTNS_xd: mnemonic = "fcvtns"; form = form_rf; break;
-    case FCVTNU_ws:
-    case FCVTNU_xs:
-    case FCVTNU_wd:
-    case FCVTNU_xd: mnemonic = "fcvtnu"; form = form_rf; break;
-    case FCVTZU_xd:
-    case FCVTZU_ws:
-    case FCVTZU_wd:
-    case FCVTZU_xs: mnemonic = "fcvtzu"; form = form_rf; break;
-    case FCVTZS_xd:
-    case FCVTZS_wd:
-    case FCVTZS_xs:
-    case FCVTZS_ws: mnemonic = "fcvtzs"; form = form_rf; break;
-    case SCVTF_sw:
-    case SCVTF_sx:
-    case SCVTF_dw:
-    case SCVTF_dx: mnemonic = "scvtf"; form = form_fr; break;
-    case UCVTF_sw:
-    case UCVTF_sx:
-    case UCVTF_dw:
-    case UCVTF_dx: mnemonic = "ucvtf"; form = form_fr; break;
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Fn, 'IFPFBits";
-  const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
-
-  switch (instr->Mask(FPFixedPointConvertMask)) {
-    case FCVTZS_ws_fixed:
-    case FCVTZS_xs_fixed:
-    case FCVTZS_wd_fixed:
-    case FCVTZS_xd_fixed: mnemonic = "fcvtzs"; break;
-    case FCVTZU_ws_fixed:
-    case FCVTZU_xs_fixed:
-    case FCVTZU_wd_fixed:
-    case FCVTZU_xd_fixed: mnemonic = "fcvtzu"; break;
-    case SCVTF_sw_fixed:
-    case SCVTF_sx_fixed:
-    case SCVTF_dw_fixed:
-    case SCVTF_dx_fixed: mnemonic = "scvtf"; form = form_fr; break;
-    case UCVTF_sw_fixed:
-    case UCVTF_sx_fixed:
-    case UCVTF_dw_fixed:
-    case UCVTF_dx_fixed: mnemonic = "ucvtf"; form = form_fr; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitSystem(const Instruction* instr) {
-  // Some system instructions hijack their Op and Cp fields to represent a
-  // range of immediates instead of indicating a different instruction. This
-  // makes the decoding tricky.
-  const char *mnemonic = "unimplemented";
-  const char *form = "(System)";
-
-  if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
-    switch (instr->Mask(SystemExclusiveMonitorMask)) {
-      case CLREX: {
-        mnemonic = "clrex";
-        form = (instr->CRm() == 0xf) ? NULL : "'IX";
-        break;
-      }
-    }
-  } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
-    switch (instr->Mask(SystemSysRegMask)) {
-      case MRS: {
-        mnemonic = "mrs";
-        switch (instr->ImmSystemRegister()) {
-          case NZCV: form = "'Xt, nzcv"; break;
-          case FPCR: form = "'Xt, fpcr"; break;
-          default: form = "'Xt, (unknown)"; break;
-        }
-        break;
-      }
-      case MSR: {
-        mnemonic = "msr";
-        switch (instr->ImmSystemRegister()) {
-          case NZCV: form = "nzcv, 'Xt"; break;
-          case FPCR: form = "fpcr, 'Xt"; break;
-          default: form = "(unknown), 'Xt"; break;
-        }
-        break;
-      }
-    }
-  } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
-    switch (instr->ImmHint()) {
-      case NOP: {
-        mnemonic = "nop";
-        form = NULL;
-        break;
-      }
-    }
-  } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
-    switch (instr->Mask(MemBarrierMask)) {
-      case DMB: {
-        mnemonic = "dmb";
-        form = "'M";
-        break;
-      }
-      case DSB: {
-        mnemonic = "dsb";
-        form = "'M";
-        break;
-      }
-      case ISB: {
-        mnemonic = "isb";
-        form = NULL;
-        break;
-      }
-    }
-  }
-
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitException(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'IDebug";
-
-  switch (instr->Mask(ExceptionMask)) {
-    case HLT: mnemonic = "hlt"; break;
-    case BRK: mnemonic = "brk"; break;
-    case SVC: mnemonic = "svc"; break;
-    case HVC: mnemonic = "hvc"; break;
-    case SMC: mnemonic = "smc"; break;
-    case DCPS1: mnemonic = "dcps1"; form = "{'IDebug}"; break;
-    case DCPS2: mnemonic = "dcps2"; form = "{'IDebug}"; break;
-    case DCPS3: mnemonic = "dcps3"; form = "{'IDebug}"; break;
-    default: form = "(Exception)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitUnimplemented(const Instruction* instr) {
-  Format(instr, "unimplemented", "(Unimplemented)");
-}
-
-
-void Disassembler::VisitUnallocated(const Instruction* instr) {
-  Format(instr, "unallocated", "(Unallocated)");
-}
-
-
-void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
-  // The base disasm does nothing more than disassembling into a buffer.
-}
-
-
-void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
-                                              const CPURegister& reg) {
-  USE(instr);
-  VIXL_ASSERT(reg.IsValid());
-  char reg_char;
-
-  if (reg.IsRegister()) {
-    reg_char = reg.Is64Bits() ? 'x' : 'w';
-  } else {
-    VIXL_ASSERT(reg.IsFPRegister());
-    reg_char = reg.Is64Bits() ? 'd' : 's';
-  }
-
-  if (reg.IsFPRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
-    // A normal register: w0 - w30, x0 - x30, s0 - s31, d0 - d31.
-    AppendToOutput("%c%d", reg_char, reg.code());
-  } else if (reg.Aliases(sp)) {
-    // Disassemble w31/x31 as stack pointer wsp/sp.
-    AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
-  } else {
-    // Disassemble w31/x31 as zero register wzr/xzr.
-    AppendToOutput("%czr", reg_char);
-  }
-}
-
-
-void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
-                                                  int64_t offset) {
-  USE(instr);
-  char sign = (offset < 0) ? '-' : '+';
-  AppendToOutput("#%c0x%" PRIx64, sign, std::abs(offset));
-}
-
-
-void Disassembler::AppendAddressToOutput(const Instruction* instr,
-                                         const void* addr) {
-  USE(instr);
-  AppendToOutput("(addr 0x%" PRIxPTR ")", reinterpret_cast<uintptr_t>(addr));
-}
-
-
-void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
-                                             const void* addr) {
-  AppendAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
-                                             const void* addr) {
-  AppendAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::AppendCodeRelativeAddressToOutput(const Instruction* instr,
-                                                     const void* addr) {
-  USE(instr);
-  int64_t rel_addr = CodeRelativeAddress(addr);
-  if (rel_addr >= 0) {
-    AppendToOutput("(addr 0x%" PRIx64 ")", rel_addr);
-  } else {
-    AppendToOutput("(addr -0x%" PRIx64 ")", -rel_addr);
-  }
-}
-
-
-void Disassembler::AppendCodeRelativeCodeAddressToOutput(
-    const Instruction* instr, const void* addr) {
-  AppendCodeRelativeAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::AppendCodeRelativeDataAddressToOutput(
-    const Instruction* instr, const void* addr) {
-  AppendCodeRelativeAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::MapCodeAddress(int64_t base_address,
-                                  const Instruction* instr_address) {
-  set_code_address_offset(
-      base_address - reinterpret_cast<intptr_t>(instr_address));
-}
-int64_t Disassembler::CodeRelativeAddress(const void* addr) {
-  return reinterpret_cast<intptr_t>(addr) + code_address_offset();
-}
-
-
-void Disassembler::Format(const Instruction* instr, const char* mnemonic,
-                          const char* format) {
-  VIXL_ASSERT(mnemonic != NULL);
-  ResetOutput();
-  Substitute(instr, mnemonic);
-  if (format != NULL) {
-    buffer_[buffer_pos_++] = ' ';
-    Substitute(instr, format);
-  }
-  buffer_[buffer_pos_] = 0;
-  ProcessOutput(instr);
-}
-
-
-void Disassembler::Substitute(const Instruction* instr, const char* string) {
-  char chr = *string++;
-  while (chr != '\0') {
-    if (chr == '\'') {
-      string += SubstituteField(instr, string);
-    } else {
-      buffer_[buffer_pos_++] = chr;
-    }
-    chr = *string++;
-  }
-}
-
-
-int Disassembler::SubstituteField(const Instruction* instr,
-                                  const char* format) {
-  switch (format[0]) {
-    case 'R':  // Register. X or W, selected by sf bit.
-    case 'F':  // FP Register. S or D, selected by type field.
-    case 'W':
-    case 'X':
-    case 'S':
-    case 'D': return SubstituteRegisterField(instr, format);
-    case 'I': return SubstituteImmediateField(instr, format);
-    case 'L': return SubstituteLiteralField(instr, format);
-    case 'H': return SubstituteShiftField(instr, format);
-    case 'P': return SubstitutePrefetchField(instr, format);
-    case 'C': return SubstituteConditionField(instr, format);
-    case 'E': return SubstituteExtendField(instr, format);
-    case 'A': return SubstitutePCRelAddressField(instr, format);
-    case 'B': return SubstituteBranchTargetField(instr, format);
-    case 'O': return SubstituteLSRegOffsetField(instr, format);
-    case 'M': return SubstituteBarrierField(instr, format);
-    default: {
-      VIXL_UNREACHABLE();
-      return 1;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteRegisterField(const Instruction* instr,
-                                          const char* format) {
-  unsigned reg_num = 0;
-  unsigned field_len = 2;
-  switch (format[1]) {
-    case 'd': reg_num = instr->Rd(); break;
-    case 'n': reg_num = instr->Rn(); break;
-    case 'm': reg_num = instr->Rm(); break;
-    case 'a': reg_num = instr->Ra(); break;
-    case 's': reg_num = instr->Rs(); break;
-    case 't': {
-      if (format[2] == '2') {
-        reg_num = instr->Rt2();
-        field_len = 3;
-      } else {
-        reg_num = instr->Rt();
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-
-  // Increase field length for registers tagged as stack.
-  if (format[2] == 's') {
-    field_len = 3;
-  }
-
-  CPURegister::RegisterType reg_type;
-  unsigned reg_size;
-
-  if (format[0] == 'R') {
-    // Register type is R: use sf bit to choose X and W.
-    reg_type = CPURegister::kRegister;
-    reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
-  } else if (format[0] == 'F') {
-    // Floating-point register: use type field to choose S or D.
-    reg_type = CPURegister::kFPRegister;
-    reg_size = ((instr->FPType() & 1) == 0) ? kSRegSize : kDRegSize;
-  } else {
-    // The register type is specified.
-    switch (format[0]) {
-      case 'W':
-        reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
-      case 'X':
-        reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
-      case 'S':
-        reg_type = CPURegister::kFPRegister; reg_size = kSRegSize; break;
-      case 'D':
-        reg_type = CPURegister::kFPRegister; reg_size = kDRegSize; break;
-      default:
-        VIXL_UNREACHABLE();
-        reg_type = CPURegister::kRegister;
-        reg_size = kXRegSize;
-    }
-  }
-
-  if ((reg_type == CPURegister::kRegister) &&
-      (reg_num == kZeroRegCode) && (format[2] == 's')) {
-    reg_num = kSPRegInternalCode;
-  }
-
-  AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
-
-  return field_len;
-}
-
-
-int Disassembler::SubstituteImmediateField(const Instruction* instr,
-                                           const char* format) {
-  VIXL_ASSERT(format[0] == 'I');
-
-  switch (format[1]) {
-    case 'M': {  // IMoveImm, IMoveNeg or IMoveLSL.
-      if (format[5] == 'L') {
-        AppendToOutput("#0x%" PRIx64, instr->ImmMoveWide());
-        if (instr->ShiftMoveWide() > 0) {
-          AppendToOutput(", lsl #%" PRId64, 16 * instr->ShiftMoveWide());
-        }
-      } else {
-        VIXL_ASSERT((format[5] == 'I') || (format[5] == 'N'));
-        uint64_t imm = instr->ImmMoveWide() << (16 * instr->ShiftMoveWide());
-        if (format[5] == 'N')
-          imm = ~imm;
-        if (!instr->SixtyFourBits())
-          imm &= UINT64_C(0xffffffff);
-        AppendToOutput("#0x%" PRIx64, imm);
-      }
-      return 8;
-    }
-    case 'L': {
-      switch (format[2]) {
-        case 'L': {  // ILLiteral - Immediate Load Literal.
-          AppendToOutput("pc%+" PRId64,
-                         instr->ImmLLiteral() << kLiteralEntrySizeLog2);
-          return 9;
-        }
-        case 'S': {  // ILS - Immediate Load/Store.
-          if (instr->ImmLS() != 0) {
-            AppendToOutput(", #%" PRId64, instr->ImmLS());
-          }
-          return 3;
-        }
-        case 'P': {  // ILPx - Immediate Load/Store Pair, x = access size.
-          if (instr->ImmLSPair() != 0) {
-            // format[3] is the scale value. Convert to a number.
-            int scale = format[3] - 0x30;
-            AppendToOutput(", #%" PRId64, instr->ImmLSPair() * scale);
-          }
-          return 4;
-        }
-        case 'U': {  // ILU - Immediate Load/Store Unsigned.
-          if (instr->ImmLSUnsigned() != 0) {
-            AppendToOutput(", #%" PRIu64,
-                           instr->ImmLSUnsigned() << instr->SizeLS());
-          }
-          return 3;
-        }
-      }
-    }
-    case 'C': {  // ICondB - Immediate Conditional Branch.
-      int64_t offset = instr->ImmCondBranch() << 2;
-      AppendPCRelativeOffsetToOutput(instr, offset);
-      return 6;
-    }
-    case 'A': {  // IAddSub.
-      VIXL_ASSERT(instr->ShiftAddSub() <= 1);
-      int64_t imm = instr->ImmAddSub() << (12 * instr->ShiftAddSub());
-      AppendToOutput("#0x%" PRIx64 " (%" PRId64 ")", imm, imm);
-      return 7;
-    }
-    case 'F': {  // IFPSingle, IFPDouble or IFPFBits.
-      if (format[3] == 'F') {  // IFPFbits.
-        AppendToOutput("#%" PRId64, 64 - instr->FPScale());
-        return 8;
-      } else {
-        AppendToOutput("#0x%" PRIx64 " (%.4f)", instr->ImmFP(),
-                       format[3] == 'S' ? instr->ImmFP32() : instr->ImmFP64());
-        return 9;
-      }
-    }
-    case 'T': {  // ITri - Immediate Triangular Encoded.
-      AppendToOutput("#0x%" PRIx64, instr->ImmLogical());
-      return 4;
-    }
-    case 'N': {  // INzcv.
-      int nzcv = (instr->Nzcv() << Flags_offset);
-      AppendToOutput("#%c%c%c%c", ((nzcv & NFlag) == 0) ? 'n' : 'N',
-                                  ((nzcv & ZFlag) == 0) ? 'z' : 'Z',
-                                  ((nzcv & CFlag) == 0) ? 'c' : 'C',
-                                  ((nzcv & VFlag) == 0) ? 'v' : 'V');
-      return 5;
-    }
-    case 'P': {  // IP - Conditional compare.
-      AppendToOutput("#%" PRId64, instr->ImmCondCmp());
-      return 2;
-    }
-    case 'B': {  // Bitfields.
-      return SubstituteBitfieldImmediateField(instr, format);
-    }
-    case 'E': {  // IExtract.
-      AppendToOutput("#%" PRId64, instr->ImmS());
-      return 8;
-    }
-    case 'S': {  // IS - Test and branch bit.
-      AppendToOutput("#%" PRId64, (instr->ImmTestBranchBit5() << 5) |
-                                  instr->ImmTestBranchBit40());
-      return 2;
-    }
-    case 'D': {  // IDebug - HLT and BRK instructions.
-      AppendToOutput("#0x%" PRIx64, instr->ImmException());
-      return 6;
-    }
-    case 'X': {  // IX - CLREX instruction.
-      AppendToOutput("#0x%" PRIx64, instr->CRm());
-      return 2;
-    }
-    default: {
-      VIXL_UNIMPLEMENTED();
-      return 0;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
-                                                   const char* format) {
-  VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
-  unsigned r = instr->ImmR();
-  unsigned s = instr->ImmS();
-
-  switch (format[2]) {
-    case 'r': {  // IBr.
-      AppendToOutput("#%d", r);
-      return 3;
-    }
-    case 's': {  // IBs+1 or IBs-r+1.
-      if (format[3] == '+') {
-        AppendToOutput("#%d", s + 1);
-        return 5;
-      } else {
-        VIXL_ASSERT(format[3] == '-');
-        AppendToOutput("#%d", s - r + 1);
-        return 7;
-      }
-    }
-    case 'Z': {  // IBZ-r.
-      VIXL_ASSERT((format[3] == '-') && (format[4] == 'r'));
-      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize;
-      AppendToOutput("#%d", reg_size - r);
-      return 5;
-    }
-    default: {
-      VIXL_UNREACHABLE();
-      return 0;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteLiteralField(const Instruction* instr,
-                                         const char* format) {
-  VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
-  USE(format);
-
-  const void * address = instr->LiteralAddress<const void *>();
-  switch (instr->Mask(LoadLiteralMask)) {
-    case LDR_w_lit:
-    case LDR_x_lit:
-    case LDRSW_x_lit:
-    case LDR_s_lit:
-    case LDR_d_lit:
-      AppendCodeRelativeDataAddressToOutput(instr, address);
-      break;
-    case PRFM_lit: {
-      // Use the prefetch hint to decide how to print the address.
-      switch (instr->PrefetchHint()) {
-        case 0x0:     // PLD: prefetch for load.
-        case 0x2:     // PST: prepare for store.
-          AppendCodeRelativeDataAddressToOutput(instr, address);
-          break;
-        case 0x1:     // PLI: preload instructions.
-          AppendCodeRelativeCodeAddressToOutput(instr, address);
-          break;
-        case 0x3:     // Unallocated hint.
-          AppendCodeRelativeAddressToOutput(instr, address);
-          break;
-      }
-      break;
-    }
-    default:
-      VIXL_UNREACHABLE();
-  }
-
-  return 6;
-}
-
-
-int Disassembler::SubstituteShiftField(const Instruction* instr,
-                                       const char* format) {
-  VIXL_ASSERT(format[0] == 'H');
-  VIXL_ASSERT(instr->ShiftDP() <= 0x3);
-
-  switch (format[1]) {
-    case 'D': {  // HDP.
-      VIXL_ASSERT(instr->ShiftDP() != ROR);
-    }  // Fall through.
-    case 'L': {  // HLo.
-      if (instr->ImmDPShift() != 0) {
-        const char* shift_type[] = {"lsl", "lsr", "asr", "ror"};
-        AppendToOutput(", %s #%" PRId64, shift_type[instr->ShiftDP()],
-                       instr->ImmDPShift());
-      }
-      return 3;
-    }
-    default:
-      VIXL_UNIMPLEMENTED();
-      return 0;
-  }
-}
-
-
-int Disassembler::SubstituteConditionField(const Instruction* instr,
-                                           const char* format) {
-  VIXL_ASSERT(format[0] == 'C');
-  const char* condition_code[] = { "eq", "ne", "hs", "lo",
-                                   "mi", "pl", "vs", "vc",
-                                   "hi", "ls", "ge", "lt",
-                                   "gt", "le", "al", "nv" };
-  int cond;
-  switch (format[1]) {
-    case 'B': cond = instr->ConditionBranch(); break;
-    case 'I': {
-      cond = InvertCondition(static_cast<Condition>(instr->Condition()));
-      break;
-    }
-    default: cond = instr->Condition();
-  }
-  AppendToOutput("%s", condition_code[cond]);
-  return 4;
-}
-
-
-int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
-                                              const char* format) {
-  VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) ||   // Used by `adr`.
-              (strcmp(format, "AddrPCRelPage") == 0));    // Used by `adrp`.
-
-  int64_t offset = instr->ImmPCRel();
-
-  // Compute the target address based on the effective address (after applying
-  // code_address_offset). This is required for correct behaviour of adrp.
-  const Instruction* base = instr + code_address_offset();
-  if (format[9] == 'P') {
-    offset *= kPageSize;
-    base = AlignDown(base, kPageSize);
-  }
-  // Strip code_address_offset before printing, so we can use the
-  // semantically-correct AppendCodeRelativeAddressToOutput.
-  const void* target =
-      reinterpret_cast<const void*>(base + offset - code_address_offset());
-
-  AppendPCRelativeOffsetToOutput(instr, offset);
-  AppendToOutput(" ");
-  AppendCodeRelativeAddressToOutput(instr, target);
-  return 13;
-}
-
-
-int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
-                                              const char* format) {
-  VIXL_ASSERT(strncmp(format, "BImm", 4) == 0);
-
-  int64_t offset = 0;
-  switch (format[5]) {
-    // BImmUncn - unconditional branch immediate.
-    case 'n': offset = instr->ImmUncondBranch(); break;
-    // BImmCond - conditional branch immediate.
-    case 'o': offset = instr->ImmCondBranch(); break;
-    // BImmCmpa - compare and branch immediate.
-    case 'm': offset = instr->ImmCmpBranch(); break;
-    // BImmTest - test and branch immediate.
-    case 'e': offset = instr->ImmTestBranch(); break;
-    default: VIXL_UNIMPLEMENTED();
-  }
-  offset <<= kInstructionSizeLog2;
-  const void* target_address = reinterpret_cast<const void*>(instr + offset);
-  VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
-
-  AppendPCRelativeOffsetToOutput(instr, offset);
-  AppendToOutput(" ");
-  AppendCodeRelativeCodeAddressToOutput(instr, target_address);
-
-  return 8;
-}
-
-
-int Disassembler::SubstituteExtendField(const Instruction* instr,
-                                        const char* format) {
-  VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
-  VIXL_ASSERT(instr->ExtendMode() <= 7);
-  USE(format);
-
-  const char* extend_mode[] = { "uxtb", "uxth", "uxtw", "uxtx",
-                                "sxtb", "sxth", "sxtw", "sxtx" };
-
-  // If rd or rn is SP, uxtw on 32-bit registers and uxtx on 64-bit
-  // registers becomes lsl.
-  if (((instr->Rd() == kZeroRegCode) || (instr->Rn() == kZeroRegCode)) &&
-      (((instr->ExtendMode() == UXTW) && (instr->SixtyFourBits() == 0)) ||
-       (instr->ExtendMode() == UXTX))) {
-    if (instr->ImmExtendShift() > 0) {
-      AppendToOutput(", lsl #%" PRId64, instr->ImmExtendShift());
-    }
-  } else {
-    AppendToOutput(", %s", extend_mode[instr->ExtendMode()]);
-    if (instr->ImmExtendShift() > 0) {
-      AppendToOutput(" #%" PRId64, instr->ImmExtendShift());
-    }
-  }
-  return 3;
-}
-
-
-int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
-                                             const char* format) {
-  VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
-  const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
-                                "undefined", "undefined", "sxtw", "sxtx" };
-  USE(format);
-
-  unsigned shift = instr->ImmShiftLS();
-  Extend ext = static_cast<Extend>(instr->ExtendMode());
-  char reg_type = ((ext == UXTW) || (ext == SXTW)) ? 'w' : 'x';
-
-  unsigned rm = instr->Rm();
-  if (rm == kZeroRegCode) {
-    AppendToOutput("%czr", reg_type);
-  } else {
-    AppendToOutput("%c%d", reg_type, rm);
-  }
-
-  // Extend mode UXTX is an alias for shift mode LSL here.
-  if (!((ext == UXTX) && (shift == 0))) {
-    AppendToOutput(", %s", extend_mode[ext]);
-    if (shift != 0) {
-      AppendToOutput(" #%" PRId64, instr->SizeLS());
-    }
-  }
-  return 9;
-}
-
-
-int Disassembler::SubstitutePrefetchField(const Instruction* instr,
-                                          const char* format) {
-  VIXL_ASSERT(format[0] == 'P');
-  USE(format);
-
-  static const char* hints[] = {"ld", "li", "st"};
-  static const char* stream_options[] = {"keep", "strm"};
-
-  unsigned hint = instr->PrefetchHint();
-  unsigned target = instr->PrefetchTarget() + 1;
-  unsigned stream = instr->PrefetchStream();
-
-  if ((hint >= (sizeof(hints) / sizeof(hints[0]))) || (target > 3)) {
-    // Unallocated prefetch operations.
-    int prefetch_mode = instr->ImmPrefetchOperation();
-    AppendToOutput("#0b%c%c%c%c%c",
-                   (prefetch_mode & (1 << 4)) ? '1' : '0',
-                   (prefetch_mode & (1 << 3)) ? '1' : '0',
-                   (prefetch_mode & (1 << 2)) ? '1' : '0',
-                   (prefetch_mode & (1 << 1)) ? '1' : '0',
-                   (prefetch_mode & (1 << 0)) ? '1' : '0');
-  } else {
-    VIXL_ASSERT(stream < (sizeof(stream_options) / sizeof(stream_options[0])));
-    AppendToOutput("p%sl%d%s", hints[hint], target, stream_options[stream]);
-  }
-  return 6;
-}
-
-int Disassembler::SubstituteBarrierField(const Instruction* instr,
-                                         const char* format) {
-  VIXL_ASSERT(format[0] == 'M');
-  USE(format);
-
-  static const char* options[4][4] = {
-    { "sy (0b0000)", "oshld", "oshst", "osh" },
-    { "sy (0b0100)", "nshld", "nshst", "nsh" },
-    { "sy (0b1000)", "ishld", "ishst", "ish" },
-    { "sy (0b1100)", "ld", "st", "sy" }
-  };
-  int domain = instr->ImmBarrierDomain();
-  int type = instr->ImmBarrierType();
-
-  AppendToOutput("%s", options[domain][type]);
-  return 1;
-}
-
-void Disassembler::ResetOutput() {
-  buffer_pos_ = 0;
-  buffer_[buffer_pos_] = 0;
-}
-
-
-void Disassembler::AppendToOutput(const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  buffer_pos_ += vsnprintf(&buffer_[buffer_pos_], buffer_size_, format, args);
-  va_end(args);
-}
-
-
-void PrintDisassembler::ProcessOutput(const Instruction* instr) {
-  fprintf(stream_, "0x%016" PRIx64 "  %08" PRIx32 "\t\t%s\n",
-          reinterpret_cast<uint64_t>(instr),
-          instr->InstructionBits(),
-          GetOutput());
-}
-}  // namespace vixl
diff --git a/qemu/disas/libvixl/a64/instructions-a64.cc b/qemu/disas/libvixl/a64/instructions-a64.cc
deleted file mode 100644
index b09188683..000000000
--- a/qemu/disas/libvixl/a64/instructions-a64.cc
+++ /dev/null
@@ -1,314 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "a64/instructions-a64.h"
-#include "a64/assembler-a64.h"
-
-namespace vixl {
-
-
-// Floating-point infinity values.
-const float kFP32PositiveInfinity = rawbits_to_float(0x7f800000);
-const float kFP32NegativeInfinity = rawbits_to_float(0xff800000);
-const double kFP64PositiveInfinity =
-    rawbits_to_double(UINT64_C(0x7ff0000000000000));
-const double kFP64NegativeInfinity =
-    rawbits_to_double(UINT64_C(0xfff0000000000000));
-
-
-// The default NaN values (for FPCR.DN=1).
-const double kFP64DefaultNaN = rawbits_to_double(UINT64_C(0x7ff8000000000000));
-const float kFP32DefaultNaN = rawbits_to_float(0x7fc00000);
-
-
-static uint64_t RotateRight(uint64_t value,
-                            unsigned int rotate,
-                            unsigned int width) {
-  VIXL_ASSERT(width <= 64);
-  rotate &= 63;
-  return ((value & ((UINT64_C(1) << rotate) - 1)) <<
-          (width - rotate)) | (value >> rotate);
-}
-
-
-static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
-                                    uint64_t value,
-                                    unsigned width) {
-  VIXL_ASSERT((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
-              (width == 32));
-  VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-  uint64_t result = value & ((UINT64_C(1) << width) - 1);
-  for (unsigned i = width; i < reg_size; i *= 2) {
-    result |= (result << i);
-  }
-  return result;
-}
-
-
-bool Instruction::IsLoad() const {
-  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
-    return false;
-  }
-
-  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
-    return Mask(LoadStorePairLBit) != 0;
-  } else {
-    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreOpMask));
-    switch (op) {
-      case LDRB_w:
-      case LDRH_w:
-      case LDR_w:
-      case LDR_x:
-      case LDRSB_w:
-      case LDRSB_x:
-      case LDRSH_w:
-      case LDRSH_x:
-      case LDRSW_x:
-      case LDR_s:
-      case LDR_d: return true;
-      default: return false;
-    }
-  }
-}
-
-
-bool Instruction::IsStore() const {
-  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
-    return false;
-  }
-
-  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
-    return Mask(LoadStorePairLBit) == 0;
-  } else {
-    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreOpMask));
-    switch (op) {
-      case STRB_w:
-      case STRH_w:
-      case STR_w:
-      case STR_x:
-      case STR_s:
-      case STR_d: return true;
-      default: return false;
-    }
-  }
-}
-
-
-// Logical immediates can't encode zero, so a return value of zero is used to
-// indicate a failure case. Specifically, where the constraints on imm_s are
-// not met.
-uint64_t Instruction::ImmLogical() const {
-  unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
-  int64_t n = BitN();
-  int64_t imm_s = ImmSetBits();
-  int64_t imm_r = ImmRotate();
-
-  // An integer is constructed from the n, imm_s and imm_r bits according to
-  // the following table:
-  //
-  //  N   imms    immr    size        S             R
-  //  1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
-  //  0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
-  //  0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
-  //  0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
-  //  0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
-  //  0  11110s  xxxxxr     2    UInt(s)       UInt(r)
-  // (s bits must not be all set)
-  //
-  // A pattern is constructed of size bits, where the least significant S+1
-  // bits are set. The pattern is rotated right by R, and repeated across a
-  // 32 or 64-bit value, depending on destination register width.
-  //
-
-  if (n == 1) {
-    if (imm_s == 0x3F) {
-      return 0;
-    }
-    uint64_t bits = (UINT64_C(1) << (imm_s + 1)) - 1;
-    return RotateRight(bits, imm_r, 64);
-  } else {
-    if ((imm_s >> 1) == 0x1F) {
-      return 0;
-    }
-    for (int width = 0x20; width >= 0x2; width >>= 1) {
-      if ((imm_s & width) == 0) {
-        int mask = width - 1;
-        if ((imm_s & mask) == mask) {
-          return 0;
-        }
-        uint64_t bits = (UINT64_C(1) << ((imm_s & mask) + 1)) - 1;
-        return RepeatBitsAcrossReg(reg_size,
-                                   RotateRight(bits, imm_r & mask, width),
-                                   width);
-      }
-    }
-  }
-  VIXL_UNREACHABLE();
-  return 0;
-}
-
-
-float Instruction::ImmFP32() const {
-  //  ImmFP: abcdefgh (8 bits)
-  // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
-  // where B is b ^ 1
-  uint32_t bits = ImmFP();
-  uint32_t bit7 = (bits >> 7) & 0x1;
-  uint32_t bit6 = (bits >> 6) & 0x1;
-  uint32_t bit5_to_0 = bits & 0x3f;
-  uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
-
-  return rawbits_to_float(result);
-}
-
-
-double Instruction::ImmFP64() const {
-  //  ImmFP: abcdefgh (8 bits)
-  // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
-  //         0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
-  // where B is b ^ 1
-  uint32_t bits = ImmFP();
-  uint64_t bit7 = (bits >> 7) & 0x1;
-  uint64_t bit6 = (bits >> 6) & 0x1;
-  uint64_t bit5_to_0 = bits & 0x3f;
-  uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
-
-  return rawbits_to_double(result);
-}
-
-
-LSDataSize CalcLSPairDataSize(LoadStorePairOp op) {
-  switch (op) {
-    case STP_x:
-    case LDP_x:
-    case STP_d:
-    case LDP_d: return LSDoubleWord;
-    default: return LSWord;
-  }
-}
-
-
-const Instruction* Instruction::ImmPCOffsetTarget() const {
-  const Instruction * base = this;
-  ptrdiff_t offset;
-  if (IsPCRelAddressing()) {
-    // ADR and ADRP.
-    offset = ImmPCRel();
-    if (Mask(PCRelAddressingMask) == ADRP) {
-      base = AlignDown(base, kPageSize);
-      offset *= kPageSize;
-    } else {
-      VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
-    }
-  } else {
-    // All PC-relative branches.
-    VIXL_ASSERT(BranchType() != UnknownBranchType);
-    // Relative branch offsets are instruction-size-aligned.
-    offset = ImmBranch() << kInstructionSizeLog2;
-  }
-  return base + offset;
-}
-
-
-inline int Instruction::ImmBranch() const {
-  switch (BranchType()) {
-    case CondBranchType: return ImmCondBranch();
-    case UncondBranchType: return ImmUncondBranch();
-    case CompareBranchType: return ImmCmpBranch();
-    case TestBranchType: return ImmTestBranch();
-    default: VIXL_UNREACHABLE();
-  }
-  return 0;
-}
-
-
-void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
-  if (IsPCRelAddressing()) {
-    SetPCRelImmTarget(target);
-  } else {
-    SetBranchImmTarget(target);
-  }
-}
-
-
-void Instruction::SetPCRelImmTarget(const Instruction* target) {
-  int32_t imm21;
-  if ((Mask(PCRelAddressingMask) == ADR)) {
-    imm21 = target - this;
-  } else {
-    VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
-    uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
-    uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
-    imm21 = target_page - this_page;
-  }
-  Instr imm = Assembler::ImmPCRelAddress(imm21);
-
-  SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
-}
-
-
-void Instruction::SetBranchImmTarget(const Instruction* target) {
-  VIXL_ASSERT(((target - this) & 3) == 0);
-  Instr branch_imm = 0;
-  uint32_t imm_mask = 0;
-  int offset = (target - this) >> kInstructionSizeLog2;
-  switch (BranchType()) {
-    case CondBranchType: {
-      branch_imm = Assembler::ImmCondBranch(offset);
-      imm_mask = ImmCondBranch_mask;
-      break;
-    }
-    case UncondBranchType: {
-      branch_imm = Assembler::ImmUncondBranch(offset);
-      imm_mask = ImmUncondBranch_mask;
-      break;
-    }
-    case CompareBranchType: {
-      branch_imm = Assembler::ImmCmpBranch(offset);
-      imm_mask = ImmCmpBranch_mask;
-      break;
-    }
-    case TestBranchType: {
-      branch_imm = Assembler::ImmTestBranch(offset);
-      imm_mask = ImmTestBranch_mask;
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  SetInstructionBits(Mask(~imm_mask) | branch_imm);
-}
-
-
-void Instruction::SetImmLLiteral(const Instruction* source) {
-  VIXL_ASSERT(IsWordAligned(source));
-  ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
-  Instr imm = Assembler::ImmLLiteral(offset);
-  Instr mask = ImmLLiteral_mask;
-
-  SetInstructionBits(Mask(~mask) | imm);
-}
-}  // namespace vixl
-
diff --git a/qemu/disas/libvixl/a64/instructions-a64.h b/qemu/disas/libvixl/a64/instructions-a64.h
deleted file mode 100644
index f1d883ccc..000000000
--- a/qemu/disas/libvixl/a64/instructions-a64.h
+++ /dev/null
@@ -1,384 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_INSTRUCTIONS_A64_H_
-#define VIXL_A64_INSTRUCTIONS_A64_H_
-
-#include "globals.h"
-#include "utils.h"
-#include "a64/constants-a64.h"
-
-namespace vixl {
-// ISA constants. --------------------------------------------------------------
-
-typedef uint32_t Instr;
-const unsigned kInstructionSize = 4;
-const unsigned kInstructionSizeLog2 = 2;
-const unsigned kLiteralEntrySize = 4;
-const unsigned kLiteralEntrySizeLog2 = 2;
-const unsigned kMaxLoadLiteralRange = 1 * MBytes;
-
-// This is the nominal page size (as used by the adrp instruction); the actual
-// size of the memory pages allocated by the kernel is likely to differ.
-const unsigned kPageSize = 4 * KBytes;
-const unsigned kPageSizeLog2 = 12;
-
-const unsigned kWRegSize = 32;
-const unsigned kWRegSizeLog2 = 5;
-const unsigned kWRegSizeInBytes = kWRegSize / 8;
-const unsigned kWRegSizeInBytesLog2 = kWRegSizeLog2 - 3;
-const unsigned kXRegSize = 64;
-const unsigned kXRegSizeLog2 = 6;
-const unsigned kXRegSizeInBytes = kXRegSize / 8;
-const unsigned kXRegSizeInBytesLog2 = kXRegSizeLog2 - 3;
-const unsigned kSRegSize = 32;
-const unsigned kSRegSizeLog2 = 5;
-const unsigned kSRegSizeInBytes = kSRegSize / 8;
-const unsigned kSRegSizeInBytesLog2 = kSRegSizeLog2 - 3;
-const unsigned kDRegSize = 64;
-const unsigned kDRegSizeLog2 = 6;
-const unsigned kDRegSizeInBytes = kDRegSize / 8;
-const unsigned kDRegSizeInBytesLog2 = kDRegSizeLog2 - 3;
-const uint64_t kWRegMask = UINT64_C(0xffffffff);
-const uint64_t kXRegMask = UINT64_C(0xffffffffffffffff);
-const uint64_t kSRegMask = UINT64_C(0xffffffff);
-const uint64_t kDRegMask = UINT64_C(0xffffffffffffffff);
-const uint64_t kSSignMask = UINT64_C(0x80000000);
-const uint64_t kDSignMask = UINT64_C(0x8000000000000000);
-const uint64_t kWSignMask = UINT64_C(0x80000000);
-const uint64_t kXSignMask = UINT64_C(0x8000000000000000);
-const uint64_t kByteMask = UINT64_C(0xff);
-const uint64_t kHalfWordMask = UINT64_C(0xffff);
-const uint64_t kWordMask = UINT64_C(0xffffffff);
-const uint64_t kXMaxUInt = UINT64_C(0xffffffffffffffff);
-const uint64_t kWMaxUInt = UINT64_C(0xffffffff);
-const int64_t kXMaxInt = INT64_C(0x7fffffffffffffff);
-const int64_t kXMinInt = INT64_C(0x8000000000000000);
-const int32_t kWMaxInt = INT32_C(0x7fffffff);
-const int32_t kWMinInt = INT32_C(0x80000000);
-const unsigned kLinkRegCode = 30;
-const unsigned kZeroRegCode = 31;
-const unsigned kSPRegInternalCode = 63;
-const unsigned kRegCodeMask = 0x1f;
-
-const unsigned kAddressTagOffset = 56;
-const unsigned kAddressTagWidth = 8;
-const uint64_t kAddressTagMask =
-    ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
-VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
-
-// AArch64 floating-point specifics. These match IEEE-754.
-const unsigned kDoubleMantissaBits = 52;
-const unsigned kDoubleExponentBits = 11;
-const unsigned kFloatMantissaBits = 23;
-const unsigned kFloatExponentBits = 8;
-
-// Floating-point infinity values.
-extern const float kFP32PositiveInfinity;
-extern const float kFP32NegativeInfinity;
-extern const double kFP64PositiveInfinity;
-extern const double kFP64NegativeInfinity;
-
-// The default NaN values (for FPCR.DN=1).
-extern const double kFP64DefaultNaN;
-extern const float kFP32DefaultNaN;
-
-
-enum LSDataSize {
-  LSByte        = 0,
-  LSHalfword    = 1,
-  LSWord        = 2,
-  LSDoubleWord  = 3
-};
-
-LSDataSize CalcLSPairDataSize(LoadStorePairOp op);
-
-enum ImmBranchType {
-  UnknownBranchType = 0,
-  CondBranchType    = 1,
-  UncondBranchType  = 2,
-  CompareBranchType = 3,
-  TestBranchType    = 4
-};
-
-enum AddrMode {
-  Offset,
-  PreIndex,
-  PostIndex
-};
-
-enum FPRounding {
-  // The first four values are encodable directly by FPCR<RMode>.
-  FPTieEven = 0x0,
-  FPPositiveInfinity = 0x1,
-  FPNegativeInfinity = 0x2,
-  FPZero = 0x3,
-
-  // The final rounding mode is only available when explicitly specified by the
-  // instruction (such as with fcvta). It cannot be set in FPCR.
-  FPTieAway
-};
-
-enum Reg31Mode {
-  Reg31IsStackPointer,
-  Reg31IsZeroRegister
-};
-
-// Instructions. ---------------------------------------------------------------
-
-class Instruction {
- public:
-  Instr InstructionBits() const {
-    return *(reinterpret_cast<const Instr*>(this));
-  }
-
-  void SetInstructionBits(Instr new_instr) {
-    *(reinterpret_cast<Instr*>(this)) = new_instr;
-  }
-
-  int Bit(int pos) const {
-    return (InstructionBits() >> pos) & 1;
-  }
-
-  uint32_t Bits(int msb, int lsb) const {
-    return unsigned_bitextract_32(msb, lsb, InstructionBits());
-  }
-
-  int32_t SignedBits(int msb, int lsb) const {
-    int32_t bits = *(reinterpret_cast<const int32_t*>(this));
-    return signed_bitextract_32(msb, lsb, bits);
-  }
-
-  Instr Mask(uint32_t mask) const {
-    return InstructionBits() & mask;
-  }
-
-  #define DEFINE_GETTER(Name, HighBit, LowBit, Func)             \
-  int64_t Name() const { return Func(HighBit, LowBit); }
-  INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
-  #undef DEFINE_GETTER
-
-  // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
-  // formed from ImmPCRelLo and ImmPCRelHi.
-  int ImmPCRel() const {
-    int const offset = ((ImmPCRelHi() << ImmPCRelLo_width) | ImmPCRelLo());
-    int const width = ImmPCRelLo_width + ImmPCRelHi_width;
-    return signed_bitextract_32(width-1, 0, offset);
-  }
-
-  uint64_t ImmLogical() const;
-  float ImmFP32() const;
-  double ImmFP64() const;
-
-  LSDataSize SizeLSPair() const {
-    return CalcLSPairDataSize(
-             static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
-  }
-
-  // Helpers.
-  bool IsCondBranchImm() const {
-    return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
-  }
-
-  bool IsUncondBranchImm() const {
-    return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
-  }
-
-  bool IsCompareBranch() const {
-    return Mask(CompareBranchFMask) == CompareBranchFixed;
-  }
-
-  bool IsTestBranch() const {
-    return Mask(TestBranchFMask) == TestBranchFixed;
-  }
-
-  bool IsPCRelAddressing() const {
-    return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
-  }
-
-  bool IsLogicalImmediate() const {
-    return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
-  }
-
-  bool IsAddSubImmediate() const {
-    return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
-  }
-
-  bool IsAddSubExtended() const {
-    return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
-  }
-
-  bool IsLoadOrStore() const {
-    return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
-  }
-
-  bool IsLoad() const;
-  bool IsStore() const;
-
-  bool IsLoadLiteral() const {
-    // This includes PRFM_lit.
-    return Mask(LoadLiteralFMask) == LoadLiteralFixed;
-  }
-
-  bool IsMovn() const {
-    return (Mask(MoveWideImmediateMask) == MOVN_x) ||
-           (Mask(MoveWideImmediateMask) == MOVN_w);
-  }
-
-  // Indicate whether Rd can be the stack pointer or the zero register. This
-  // does not check that the instruction actually has an Rd field.
-  Reg31Mode RdMode() const {
-    // The following instructions use sp or wsp as Rd:
-    //  Add/sub (immediate) when not setting the flags.
-    //  Add/sub (extended) when not setting the flags.
-    //  Logical (immediate) when not setting the flags.
-    // Otherwise, r31 is the zero register.
-    if (IsAddSubImmediate() || IsAddSubExtended()) {
-      if (Mask(AddSubSetFlagsBit)) {
-        return Reg31IsZeroRegister;
-      } else {
-        return Reg31IsStackPointer;
-      }
-    }
-    if (IsLogicalImmediate()) {
-      // Of the logical (immediate) instructions, only ANDS (and its aliases)
-      // can set the flags. The others can all write into sp.
-      // Note that some logical operations are not available to
-      // immediate-operand instructions, so we have to combine two masks here.
-      if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
-        return Reg31IsZeroRegister;
-      } else {
-        return Reg31IsStackPointer;
-      }
-    }
-    return Reg31IsZeroRegister;
-  }
-
-  // Indicate whether Rn can be the stack pointer or the zero register. This
-  // does not check that the instruction actually has an Rn field.
-  Reg31Mode RnMode() const {
-    // The following instructions use sp or wsp as Rn:
-    //  All loads and stores.
-    //  Add/sub (immediate).
-    //  Add/sub (extended).
-    // Otherwise, r31 is the zero register.
-    if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
-      return Reg31IsStackPointer;
-    }
-    return Reg31IsZeroRegister;
-  }
-
-  ImmBranchType BranchType() const {
-    if (IsCondBranchImm()) {
-      return CondBranchType;
-    } else if (IsUncondBranchImm()) {
-      return UncondBranchType;
-    } else if (IsCompareBranch()) {
-      return CompareBranchType;
-    } else if (IsTestBranch()) {
-      return TestBranchType;
-    } else {
-      return UnknownBranchType;
-    }
-  }
-
-  // Find the target of this instruction. 'this' may be a branch or a
-  // PC-relative addressing instruction.
-  const Instruction* ImmPCOffsetTarget() const;
-
-  // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
-  // a PC-relative addressing instruction.
-  void SetImmPCOffsetTarget(const Instruction* target);
-  // Patch a literal load instruction to load from 'source'.
-  void SetImmLLiteral(const Instruction* source);
-
-  // Calculate the address of a literal referred to by a load-literal
-  // instruction, and return it as the specified type.
-  //
-  // The literal itself is safely mutable only if the backing buffer is safely
-  // mutable.
-  template <typename T>
-  T LiteralAddress() const {
-    uint64_t base_raw = reinterpret_cast<uintptr_t>(this);
-    ptrdiff_t offset = ImmLLiteral() << kLiteralEntrySizeLog2;
-    uint64_t address_raw = base_raw + offset;
-
-    // Cast the address using a C-style cast. A reinterpret_cast would be
-    // appropriate, but it can't cast one integral type to another.
-    T address = (T)(address_raw);
-
-    // Assert that the address can be represented by the specified type.
-    VIXL_ASSERT((uint64_t)(address) == address_raw);
-
-    return address;
-  }
-
-  uint32_t Literal32() const {
-    uint32_t literal;
-    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
-    return literal;
-  }
-
-  uint64_t Literal64() const {
-    uint64_t literal;
-    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
-    return literal;
-  }
-
-  float LiteralFP32() const {
-    return rawbits_to_float(Literal32());
-  }
-
-  double LiteralFP64() const {
-    return rawbits_to_double(Literal64());
-  }
-
-  const Instruction* NextInstruction() const {
-    return this + kInstructionSize;
-  }
-
-  const Instruction* InstructionAtOffset(int64_t offset) const {
-    VIXL_ASSERT(IsWordAligned(this + offset));
-    return this + offset;
-  }
-
-  template<typename T> static Instruction* Cast(T src) {
-    return reinterpret_cast<Instruction*>(src);
-  }
-
-  template<typename T> static const Instruction* CastConst(T src) {
-    return reinterpret_cast<const Instruction*>(src);
-  }
-
- private:
-  int ImmBranch() const;
-
-  void SetPCRelImmTarget(const Instruction* target);
-  void SetBranchImmTarget(const Instruction* target);
-};
-}  // namespace vixl
-
-#endif  // VIXL_A64_INSTRUCTIONS_A64_H_
diff --git a/qemu/disas/libvixl/vixl/a64/assembler-a64.h b/qemu/disas/libvixl/vixl/a64/assembler-a64.h
new file mode 100644
index 000000000..fda5ccc6c
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/a64/assembler-a64.h
@@ -0,0 +1,4624 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_ASSEMBLER_A64_H_
+#define VIXL_A64_ASSEMBLER_A64_H_
+
+
+#include "vixl/globals.h"
+#include "vixl/invalset.h"
+#include "vixl/utils.h"
+#include "vixl/code-buffer.h"
+#include "vixl/a64/instructions-a64.h"
+
+namespace vixl {
+
+typedef uint64_t RegList;
+static const int kRegListSizeInBits = sizeof(RegList) * 8;
+
+
+// Registers.
+
+// Some CPURegister methods can return Register or VRegister types, so we need
+// to declare them in advance.
+class Register;
+class VRegister;
+
+class CPURegister {
+ public:
+  enum RegisterType {
+    // The kInvalid value is used to detect uninitialized static instances,
+    // which are always zero-initialized before any constructors are called.
+    kInvalid = 0,
+    kRegister,
+    kVRegister,
+    kFPRegister = kVRegister,
+    kNoRegister
+  };
+
+  CPURegister() : code_(0), size_(0), type_(kNoRegister) {
+    VIXL_ASSERT(!IsValid());
+    VIXL_ASSERT(IsNone());
+  }
+
+  CPURegister(unsigned code, unsigned size, RegisterType type)
+      : code_(code), size_(size), type_(type) {
+    VIXL_ASSERT(IsValidOrNone());
+  }
+
+  unsigned code() const {
+    VIXL_ASSERT(IsValid());
+    return code_;
+  }
+
+  RegisterType type() const {
+    VIXL_ASSERT(IsValidOrNone());
+    return type_;
+  }
+
+  RegList Bit() const {
+    VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
+    return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
+  }
+
+  unsigned size() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  int SizeInBytes() const {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(size() % 8 == 0);
+    return size_ / 8;
+  }
+
+  int SizeInBits() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  bool Is8Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 8;
+  }
+
+  bool Is16Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 16;
+  }
+
+  bool Is32Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 32;
+  }
+
+  bool Is64Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 64;
+  }
+
+  bool Is128Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 128;
+  }
+
+  bool IsValid() const {
+    if (IsValidRegister() || IsValidVRegister()) {
+      VIXL_ASSERT(!IsNone());
+      return true;
+    } else {
+      // This assert is hit when the register has not been properly initialized.
+      // One cause for this can be an initialisation order fiasco. See
+      // https://isocpp.org/wiki/faq/ctors#static-init-order for some details.
+      VIXL_ASSERT(IsNone());
+      return false;
+    }
+  }
+
+  bool IsValidRegister() const {
+    return IsRegister() &&
+           ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
+           ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
+  }
+
+  bool IsValidVRegister() const {
+    return IsVRegister() &&
+           ((size_ == kBRegSize) || (size_ == kHRegSize) ||
+            (size_ == kSRegSize) || (size_ == kDRegSize) ||
+            (size_ == kQRegSize)) &&
+           (code_ < kNumberOfVRegisters);
+  }
+
+  bool IsValidFPRegister() const {
+    return IsFPRegister() && (code_ < kNumberOfVRegisters);
+  }
+
+  bool IsNone() const {
+    // kNoRegister types should always have size 0 and code 0.
+    VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
+    VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
+
+    return type_ == kNoRegister;
+  }
+
+  bool Aliases(const CPURegister& other) const {
+    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+    return (code_ == other.code_) && (type_ == other.type_);
+  }
+
+  bool Is(const CPURegister& other) const {
+    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+    return Aliases(other) && (size_ == other.size_);
+  }
+
+  bool IsZero() const {
+    VIXL_ASSERT(IsValid());
+    return IsRegister() && (code_ == kZeroRegCode);
+  }
+
+  bool IsSP() const {
+    VIXL_ASSERT(IsValid());
+    return IsRegister() && (code_ == kSPRegInternalCode);
+  }
+
+  bool IsRegister() const {
+    return type_ == kRegister;
+  }
+
+  bool IsVRegister() const {
+    return type_ == kVRegister;
+  }
+
+  bool IsFPRegister() const {
+    return IsS() || IsD();
+  }
+
+  bool IsW() const { return IsValidRegister() && Is32Bits(); }
+  bool IsX() const { return IsValidRegister() && Is64Bits(); }
+
+  // These assertions ensure that the size and type of the register are as
+  // described. They do not consider the number of lanes that make up a vector.
+  // So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
+  // does not imply Is1D() or Is8B().
+  // Check the number of lanes, ie. the format of the vector, using methods such
+  // as Is8B(), Is1D(), etc. in the VRegister class.
+  bool IsV() const { return IsVRegister(); }
+  bool IsB() const { return IsV() && Is8Bits(); }
+  bool IsH() const { return IsV() && Is16Bits(); }
+  bool IsS() const { return IsV() && Is32Bits(); }
+  bool IsD() const { return IsV() && Is64Bits(); }
+  bool IsQ() const { return IsV() && Is128Bits(); }
+
+  const Register& W() const;
+  const Register& X() const;
+  const VRegister& V() const;
+  const VRegister& B() const;
+  const VRegister& H() const;
+  const VRegister& S() const;
+  const VRegister& D() const;
+  const VRegister& Q() const;
+
+  bool IsSameSizeAndType(const CPURegister& other) const {
+    return (size_ == other.size_) && (type_ == other.type_);
+  }
+
+ protected:
+  unsigned code_;
+  unsigned size_;
+  RegisterType type_;
+
+ private:
+  bool IsValidOrNone() const {
+    return IsValid() || IsNone();
+  }
+};
+
+
+class Register : public CPURegister {
+ public:
+  Register() : CPURegister() {}
+  explicit Register(const CPURegister& other)
+      : CPURegister(other.code(), other.size(), other.type()) {
+    VIXL_ASSERT(IsValidRegister());
+  }
+  Register(unsigned code, unsigned size)
+      : CPURegister(code, size, kRegister) {}
+
+  bool IsValid() const {
+    VIXL_ASSERT(IsRegister() || IsNone());
+    return IsValidRegister();
+  }
+
+  static const Register& WRegFromCode(unsigned code);
+  static const Register& XRegFromCode(unsigned code);
+
+ private:
+  static const Register wregisters[];
+  static const Register xregisters[];
+};
+
+
+class VRegister : public CPURegister {
+ public:
+  VRegister() : CPURegister(), lanes_(1) {}
+  explicit VRegister(const CPURegister& other)
+      : CPURegister(other.code(), other.size(), other.type()), lanes_(1) {
+    VIXL_ASSERT(IsValidVRegister());
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+  VRegister(unsigned code, unsigned size, unsigned lanes = 1)
+      : CPURegister(code, size, kVRegister), lanes_(lanes) {
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+  VRegister(unsigned code, VectorFormat format)
+      : CPURegister(code, RegisterSizeInBitsFromFormat(format), kVRegister),
+        lanes_(IsVectorFormat(format) ? LaneCountFromFormat(format) : 1) {
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+
+  bool IsValid() const {
+    VIXL_ASSERT(IsVRegister() || IsNone());
+    return IsValidVRegister();
+  }
+
+  static const VRegister& BRegFromCode(unsigned code);
+  static const VRegister& HRegFromCode(unsigned code);
+  static const VRegister& SRegFromCode(unsigned code);
+  static const VRegister& DRegFromCode(unsigned code);
+  static const VRegister& QRegFromCode(unsigned code);
+  static const VRegister& VRegFromCode(unsigned code);
+
+  VRegister V8B() const { return VRegister(code_, kDRegSize, 8); }
+  VRegister V16B() const { return VRegister(code_, kQRegSize, 16); }
+  VRegister V4H() const { return VRegister(code_, kDRegSize, 4); }
+  VRegister V8H() const { return VRegister(code_, kQRegSize, 8); }
+  VRegister V2S() const { return VRegister(code_, kDRegSize, 2); }
+  VRegister V4S() const { return VRegister(code_, kQRegSize, 4); }
+  VRegister V2D() const { return VRegister(code_, kQRegSize, 2); }
+  VRegister V1D() const { return VRegister(code_, kDRegSize, 1); }
+
+  bool Is8B() const { return (Is64Bits() && (lanes_ == 8)); }
+  bool Is16B() const { return (Is128Bits() && (lanes_ == 16)); }
+  bool Is4H() const { return (Is64Bits() && (lanes_ == 4)); }
+  bool Is8H() const { return (Is128Bits() && (lanes_ == 8)); }
+  bool Is2S() const { return (Is64Bits() && (lanes_ == 2)); }
+  bool Is4S() const { return (Is128Bits() && (lanes_ == 4)); }
+  bool Is1D() const { return (Is64Bits() && (lanes_ == 1)); }
+  bool Is2D() const { return (Is128Bits() && (lanes_ == 2)); }
+
+  // For consistency, we assert the number of lanes of these scalar registers,
+  // even though there are no vectors of equivalent total size with which they
+  // could alias.
+  bool Is1B() const {
+    VIXL_ASSERT(!(Is8Bits() && IsVector()));
+    return Is8Bits();
+  }
+  bool Is1H() const {
+    VIXL_ASSERT(!(Is16Bits() && IsVector()));
+    return Is16Bits();
+  }
+  bool Is1S() const {
+    VIXL_ASSERT(!(Is32Bits() && IsVector()));
+    return Is32Bits();
+  }
+
+  bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSize; }
+  bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSize; }
+  bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSize; }
+  bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSize; }
+
+  int lanes() const {
+    return lanes_;
+  }
+
+  bool IsScalar() const {
+    return lanes_ == 1;
+  }
+
+  bool IsVector() const {
+    return lanes_ > 1;
+  }
+
+  bool IsSameFormat(const VRegister& other) const {
+    return (size_ == other.size_) && (lanes_ == other.lanes_);
+  }
+
+  unsigned LaneSizeInBytes() const {
+    return SizeInBytes() / lanes_;
+  }
+
+  unsigned LaneSizeInBits() const {
+    return LaneSizeInBytes() * 8;
+  }
+
+ private:
+  static const VRegister bregisters[];
+  static const VRegister hregisters[];
+  static const VRegister sregisters[];
+  static const VRegister dregisters[];
+  static const VRegister qregisters[];
+  static const VRegister vregisters[];
+  int lanes_;
+};
+
+
+// Backward compatibility for FPRegisters.
+typedef VRegister FPRegister;
+
+// No*Reg is used to indicate an unused argument, or an error case. Note that
+// these all compare equal (using the Is() method). The Register and VRegister
+// variants are provided for convenience.
+const Register NoReg;
+const VRegister NoVReg;
+const FPRegister NoFPReg;  // For backward compatibility.
+const CPURegister NoCPUReg;
+
+
+#define DEFINE_REGISTERS(N)  \
+const Register w##N(N, kWRegSize);  \
+const Register x##N(N, kXRegSize);
+REGISTER_CODE_LIST(DEFINE_REGISTERS)
+#undef DEFINE_REGISTERS
+const Register wsp(kSPRegInternalCode, kWRegSize);
+const Register sp(kSPRegInternalCode, kXRegSize);
+
+
+#define DEFINE_VREGISTERS(N)  \
+const VRegister b##N(N, kBRegSize);  \
+const VRegister h##N(N, kHRegSize);  \
+const VRegister s##N(N, kSRegSize);  \
+const VRegister d##N(N, kDRegSize);  \
+const VRegister q##N(N, kQRegSize);  \
+const VRegister v##N(N, kQRegSize);
+REGISTER_CODE_LIST(DEFINE_VREGISTERS)
+#undef DEFINE_VREGISTERS
+
+
+// Registers aliases.
+const Register ip0 = x16;
+const Register ip1 = x17;
+const Register lr = x30;
+const Register xzr = x31;
+const Register wzr = w31;
+
+
+// AreAliased returns true if any of the named registers overlap. Arguments
+// set to NoReg are ignored. The system stack pointer may be specified.
+bool AreAliased(const CPURegister& reg1,
+                const CPURegister& reg2,
+                const CPURegister& reg3 = NoReg,
+                const CPURegister& reg4 = NoReg,
+                const CPURegister& reg5 = NoReg,
+                const CPURegister& reg6 = NoReg,
+                const CPURegister& reg7 = NoReg,
+                const CPURegister& reg8 = NoReg);
+
+
+// AreSameSizeAndType returns true if all of the specified registers have the
+// same size, and are of the same type. The system stack pointer may be
+// specified. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
+bool AreSameSizeAndType(const CPURegister& reg1,
+                        const CPURegister& reg2,
+                        const CPURegister& reg3 = NoCPUReg,
+                        const CPURegister& reg4 = NoCPUReg,
+                        const CPURegister& reg5 = NoCPUReg,
+                        const CPURegister& reg6 = NoCPUReg,
+                        const CPURegister& reg7 = NoCPUReg,
+                        const CPURegister& reg8 = NoCPUReg);
+
+
+// AreSameFormat returns true if all of the specified VRegisters have the same
+// vector format. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoVReg).
+bool AreSameFormat(const VRegister& reg1,
+                   const VRegister& reg2,
+                   const VRegister& reg3 = NoVReg,
+                   const VRegister& reg4 = NoVReg);
+
+
+// AreConsecutive returns true if all of the specified VRegisters are
+// consecutive in the register file. Arguments set to NoReg are ignored, as are
+// any subsequent arguments. At least one argument (reg1) must be valid
+// (not NoVReg).
+bool AreConsecutive(const VRegister& reg1,
+                    const VRegister& reg2,
+                    const VRegister& reg3 = NoVReg,
+                    const VRegister& reg4 = NoVReg);
+
+
+// Lists of registers.
+class CPURegList {
+ public:
+  explicit CPURegList(CPURegister reg1,
+                      CPURegister reg2 = NoCPUReg,
+                      CPURegister reg3 = NoCPUReg,
+                      CPURegister reg4 = NoCPUReg)
+      : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
+        size_(reg1.size()), type_(reg1.type()) {
+    VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
+      : list_(list), size_(size), type_(type) {
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegList(CPURegister::RegisterType type, unsigned size,
+             unsigned first_reg, unsigned last_reg)
+      : size_(size), type_(type) {
+    VIXL_ASSERT(((type == CPURegister::kRegister) &&
+                 (last_reg < kNumberOfRegisters)) ||
+                ((type == CPURegister::kVRegister) &&
+                 (last_reg < kNumberOfVRegisters)));
+    VIXL_ASSERT(last_reg >= first_reg);
+    list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
+    list_ &= ~((UINT64_C(1) << first_reg) - 1);
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegister::RegisterType type() const {
+    VIXL_ASSERT(IsValid());
+    return type_;
+  }
+
+  // Combine another CPURegList into this one. Registers that already exist in
+  // this list are left unchanged. The type and size of the registers in the
+  // 'other' list must match those in this list.
+  void Combine(const CPURegList& other) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+    list_ |= other.list();
+  }
+
+  // Remove every register in the other CPURegList from this one. Registers that
+  // do not exist in this list are ignored. The type and size of the registers
+  // in the 'other' list must match those in this list.
+  void Remove(const CPURegList& other) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+    list_ &= ~other.list();
+  }
+
+  // Variants of Combine and Remove which take a single register.
+  void Combine(const CPURegister& other) {
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.size() == size_);
+    Combine(other.code());
+  }
+
+  void Remove(const CPURegister& other) {
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.size() == size_);
+    Remove(other.code());
+  }
+
+  // Variants of Combine and Remove which take a single register by its code;
+  // the type and size of the register is inferred from this list.
+  void Combine(int code) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+    list_ |= (UINT64_C(1) << code);
+  }
+
+  void Remove(int code) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+    list_ &= ~(UINT64_C(1) << code);
+  }
+
+  static CPURegList Union(const CPURegList& list_1, const CPURegList& list_2) {
+    VIXL_ASSERT(list_1.type_ == list_2.type_);
+    VIXL_ASSERT(list_1.size_ == list_2.size_);
+    return CPURegList(list_1.type_, list_1.size_, list_1.list_ | list_2.list_);
+  }
+  static CPURegList Union(const CPURegList& list_1,
+                          const CPURegList& list_2,
+                          const CPURegList& list_3);
+  static CPURegList Union(const CPURegList& list_1,
+                          const CPURegList& list_2,
+                          const CPURegList& list_3,
+                          const CPURegList& list_4);
+
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2) {
+    VIXL_ASSERT(list_1.type_ == list_2.type_);
+    VIXL_ASSERT(list_1.size_ == list_2.size_);
+    return CPURegList(list_1.type_, list_1.size_, list_1.list_ & list_2.list_);
+  }
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2,
+                                 const CPURegList& list_3);
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2,
+                                 const CPURegList& list_3,
+                                 const CPURegList& list_4);
+
+  bool Overlaps(const CPURegList& other) const {
+    return (type_ == other.type_) && ((list_ & other.list_) != 0);
+  }
+
+  RegList list() const {
+    VIXL_ASSERT(IsValid());
+    return list_;
+  }
+
+  void set_list(RegList new_list) {
+    VIXL_ASSERT(IsValid());
+    list_ = new_list;
+  }
+
+  // Remove all callee-saved registers from the list. This can be useful when
+  // preparing registers for an AAPCS64 function call, for example.
+  void RemoveCalleeSaved();
+
+  CPURegister PopLowestIndex();
+  CPURegister PopHighestIndex();
+
+  // AAPCS64 callee-saved registers.
+  static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
+  static CPURegList GetCalleeSavedV(unsigned size = kDRegSize);
+
+  // AAPCS64 caller-saved registers. Note that this includes lr.
+  // TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
+  // 64-bits being caller-saved.
+  static CPURegList GetCallerSaved(unsigned size = kXRegSize);
+  static CPURegList GetCallerSavedV(unsigned size = kDRegSize);
+
+  bool IsEmpty() const {
+    VIXL_ASSERT(IsValid());
+    return list_ == 0;
+  }
+
+  bool IncludesAliasOf(const CPURegister& other) const {
+    VIXL_ASSERT(IsValid());
+    return (type_ == other.type()) && ((other.Bit() & list_) != 0);
+  }
+
+  bool IncludesAliasOf(int code) const {
+    VIXL_ASSERT(IsValid());
+    return ((code & list_) != 0);
+  }
+
+  int Count() const {
+    VIXL_ASSERT(IsValid());
+    return CountSetBits(list_);
+  }
+
+  unsigned RegisterSizeInBits() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  unsigned RegisterSizeInBytes() const {
+    int size_in_bits = RegisterSizeInBits();
+    VIXL_ASSERT((size_in_bits % 8) == 0);
+    return size_in_bits / 8;
+  }
+
+  unsigned TotalSizeInBytes() const {
+    VIXL_ASSERT(IsValid());
+    return RegisterSizeInBytes() * Count();
+  }
+
+ private:
+  RegList list_;
+  unsigned size_;
+  CPURegister::RegisterType type_;
+
+  bool IsValid() const;
+};
+
+
+// AAPCS64 callee-saved registers.
+extern const CPURegList kCalleeSaved;
+extern const CPURegList kCalleeSavedV;
+
+
+// AAPCS64 caller-saved registers. Note that this includes lr.
+extern const CPURegList kCallerSaved;
+extern const CPURegList kCallerSavedV;
+
+
+// Operand.
+class Operand {
+ public:
+  // #<immediate>
+  // where <immediate> is int64_t.
+  // This is allowed to be an implicit constructor because Operand is
+  // a wrapper class that doesn't normally perform any type conversion.
+  Operand(int64_t immediate = 0);           // NOLINT(runtime/explicit)
+
+  // rm, {<shift> #<shift_amount>}
+  // where <shift> is one of {LSL, LSR, ASR, ROR}.
+  //       <shift_amount> is uint6_t.
+  // This is allowed to be an implicit constructor because Operand is
+  // a wrapper class that doesn't normally perform any type conversion.
+  Operand(Register reg,
+          Shift shift = LSL,
+          unsigned shift_amount = 0);   // NOLINT(runtime/explicit)
+
+  // rm, {<extend> {#<shift_amount>}}
+  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
+  //       <shift_amount> is uint2_t.
+  explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
+
+  bool IsImmediate() const;
+  bool IsShiftedRegister() const;
+  bool IsExtendedRegister() const;
+  bool IsZero() const;
+
+  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
+  // which helps in the encoding of instructions that use the stack pointer.
+  Operand ToExtendedRegister() const;
+
+  int64_t immediate() const {
+    VIXL_ASSERT(IsImmediate());
+    return immediate_;
+  }
+
+  Register reg() const {
+    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+    return reg_;
+  }
+
+  Shift shift() const {
+    VIXL_ASSERT(IsShiftedRegister());
+    return shift_;
+  }
+
+  Extend extend() const {
+    VIXL_ASSERT(IsExtendedRegister());
+    return extend_;
+  }
+
+  unsigned shift_amount() const {
+    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+    return shift_amount_;
+  }
+
+ private:
+  int64_t immediate_;
+  Register reg_;
+  Shift shift_;
+  Extend extend_;
+  unsigned shift_amount_;
+};
+
+
+// MemOperand represents the addressing mode of a load or store instruction.
+class MemOperand {
+ public:
+  explicit MemOperand(Register base,
+                      int64_t offset = 0,
+                      AddrMode addrmode = Offset);
+  MemOperand(Register base,
+             Register regoffset,
+             Shift shift = LSL,
+             unsigned shift_amount = 0);
+  MemOperand(Register base,
+             Register regoffset,
+             Extend extend,
+             unsigned shift_amount = 0);
+  MemOperand(Register base,
+             const Operand& offset,
+             AddrMode addrmode = Offset);
+
+  const Register& base() const { return base_; }
+  const Register& regoffset() const { return regoffset_; }
+  int64_t offset() const { return offset_; }
+  AddrMode addrmode() const { return addrmode_; }
+  Shift shift() const { return shift_; }
+  Extend extend() const { return extend_; }
+  unsigned shift_amount() const { return shift_amount_; }
+  bool IsImmediateOffset() const;
+  bool IsRegisterOffset() const;
+  bool IsPreIndex() const;
+  bool IsPostIndex() const;
+
+  void AddOffset(int64_t offset);
+
+ private:
+  Register base_;
+  Register regoffset_;
+  int64_t offset_;
+  AddrMode addrmode_;
+  Shift shift_;
+  Extend extend_;
+  unsigned shift_amount_;
+};
+
+
+class LabelTestHelper;  // Forward declaration.
+
+
+class Label {
+ public:
+  Label() : location_(kLocationUnbound) {}
+  ~Label() {
+    // If the label has been linked to, it needs to be bound to a target.
+    VIXL_ASSERT(!IsLinked() || IsBound());
+  }
+
+  bool IsBound() const { return location_ >= 0; }
+  bool IsLinked() const { return !links_.empty(); }
+
+  ptrdiff_t location() const { return location_; }
+
+  static const int kNPreallocatedLinks = 4;
+  static const ptrdiff_t kInvalidLinkKey = PTRDIFF_MAX;
+  static const size_t kReclaimFrom = 512;
+  static const size_t kReclaimFactor = 2;
+
+  typedef InvalSet<ptrdiff_t,
+                   kNPreallocatedLinks,
+                   ptrdiff_t,
+                   kInvalidLinkKey,
+                   kReclaimFrom,
+                   kReclaimFactor> LinksSetBase;
+  typedef InvalSetIterator<LinksSetBase> LabelLinksIteratorBase;
+
+ private:
+  class LinksSet : public LinksSetBase {
+   public:
+    LinksSet() : LinksSetBase() {}
+  };
+
+  // Allows iterating over the links of a label. The behaviour is undefined if
+  // the list of links is modified in any way while iterating.
+  class LabelLinksIterator : public LabelLinksIteratorBase {
+   public:
+    explicit LabelLinksIterator(Label* label)
+        : LabelLinksIteratorBase(&label->links_) {}
+  };
+
+  void Bind(ptrdiff_t location) {
+    // Labels can only be bound once.
+    VIXL_ASSERT(!IsBound());
+    location_ = location;
+  }
+
+  void AddLink(ptrdiff_t instruction) {
+    // If a label is bound, the assembler already has the information it needs
+    // to write the instruction, so there is no need to add it to links_.
+    VIXL_ASSERT(!IsBound());
+    links_.insert(instruction);
+  }
+
+  void DeleteLink(ptrdiff_t instruction) {
+    links_.erase(instruction);
+  }
+
+  void ClearAllLinks() {
+    links_.clear();
+  }
+
+  // TODO: The comment below considers average case complexity for our
+  // usual use-cases. The elements of interest are:
+  // - Branches to a label are emitted in order: branch instructions to a label
+  // are generated at an offset in the code generation buffer greater than any
+  // other branch to that same label already generated. As an example, this can
+  // be broken when an instruction is patched to become a branch. Note that the
+  // code will still work, but the complexity considerations below may locally
+  // not apply any more.
+  // - Veneers are generated in order: for multiple branches of the same type
+  // branching to the same unbound label going out of range, veneers are
+  // generated in growing order of the branch instruction offset from the start
+  // of the buffer.
+  //
+  // When creating a veneer for a branch going out of range, the link for this
+  // branch needs to be removed from this `links_`. Since all branches are
+  // tracked in one underlying InvalSet, the complexity for this deletion is the
+  // same as for finding the element, ie. O(n), where n is the number of links
+  // in the set.
+  // This could be reduced to O(1) by using the same trick as used when tracking
+  // branch information for veneers: split the container to use one set per type
+  // of branch. With that setup, when a veneer is created and the link needs to
+  // be deleted, if the two points above hold, it must be the minimum element of
+  // the set for its type of branch, and that minimum element will be accessible
+  // in O(1).
+
+  // The offsets of the instructions that have linked to this label.
+  LinksSet links_;
+  // The label location.
+  ptrdiff_t location_;
+
+  static const ptrdiff_t kLocationUnbound = -1;
+
+  // It is not safe to copy labels, so disable the copy constructor and operator
+  // by declaring them private (without an implementation).
+  Label(const Label&);
+  void operator=(const Label&);
+
+  // The Assembler class is responsible for binding and linking labels, since
+  // the stored offsets need to be consistent with the Assembler's buffer.
+  friend class Assembler;
+  // The MacroAssembler and VeneerPool handle resolution of branches to distant
+  // targets.
+  friend class MacroAssembler;
+  friend class VeneerPool;
+};
+
+
+// Required InvalSet template specialisations.
+#define INVAL_SET_TEMPLATE_PARAMETERS \
+    ptrdiff_t,                        \
+    Label::kNPreallocatedLinks,       \
+    ptrdiff_t,                        \
+    Label::kInvalidLinkKey,           \
+    Label::kReclaimFrom,              \
+    Label::kReclaimFactor
+template<>
+inline ptrdiff_t InvalSet<INVAL_SET_TEMPLATE_PARAMETERS>::Key(
+    const ptrdiff_t& element) {
+  return element;
+}
+template<>
+inline void InvalSet<INVAL_SET_TEMPLATE_PARAMETERS>::SetKey(
+              ptrdiff_t* element, ptrdiff_t key) {
+  *element = key;
+}
+#undef INVAL_SET_TEMPLATE_PARAMETERS
+
+
+class Assembler;
+class LiteralPool;
+
+// A literal is a 32-bit or 64-bit piece of data stored in the instruction
+// stream and loaded through a pc relative load. The same literal can be
+// referred to by multiple instructions but a literal can only reside at one
+// place in memory. A literal can be used by a load before or after being
+// placed in memory.
+//
+// Internally an offset of 0 is associated with a literal which has been
+// neither used nor placed. Then two possibilities arise:
+//  1) the label is placed, the offset (stored as offset + 1) is used to
+//     resolve any subsequent load using the label.
+//  2) the label is not placed and offset is the offset of the last load using
+//     the literal (stored as -offset -1). If multiple loads refer to this
+//     literal then the last load holds the offset of the preceding load and
+//     all loads form a chain. Once the offset is placed all the loads in the
+//     chain are resolved and future loads fall back to possibility 1.
+class RawLiteral {
+ public:
+  enum DeletionPolicy {
+    kDeletedOnPlacementByPool,
+    kDeletedOnPoolDestruction,
+    kManuallyDeleted
+  };
+
+  RawLiteral(size_t size,
+             LiteralPool* literal_pool,
+             DeletionPolicy deletion_policy = kManuallyDeleted);
+
+  // The literal pool only sees and deletes `RawLiteral*` pointers, but they are
+  // actually pointing to `Literal<T>` objects.
+  virtual ~RawLiteral() {}
+
+  size_t size() {
+    VIXL_STATIC_ASSERT(kDRegSizeInBytes == kXRegSizeInBytes);
+    VIXL_STATIC_ASSERT(kSRegSizeInBytes == kWRegSizeInBytes);
+    VIXL_ASSERT((size_ == kXRegSizeInBytes) ||
+                (size_ == kWRegSizeInBytes) ||
+                (size_ == kQRegSizeInBytes));
+    return size_;
+  }
+  uint64_t raw_value128_low64() {
+    VIXL_ASSERT(size_ == kQRegSizeInBytes);
+    return low64_;
+  }
+  uint64_t raw_value128_high64() {
+    VIXL_ASSERT(size_ == kQRegSizeInBytes);
+    return high64_;
+  }
+  uint64_t raw_value64() {
+    VIXL_ASSERT(size_ == kXRegSizeInBytes);
+    VIXL_ASSERT(high64_ == 0);
+    return low64_;
+  }
+  uint32_t raw_value32() {
+    VIXL_ASSERT(size_ == kWRegSizeInBytes);
+    VIXL_ASSERT(high64_ == 0);
+    VIXL_ASSERT(is_uint32(low64_) || is_int32(low64_));
+    return static_cast<uint32_t>(low64_);
+  }
+  bool IsUsed() { return offset_ < 0; }
+  bool IsPlaced() { return offset_ > 0; }
+
+  LiteralPool* GetLiteralPool() const {
+    return literal_pool_;
+  }
+
+  ptrdiff_t offset() {
+    VIXL_ASSERT(IsPlaced());
+    return offset_ - 1;
+  }
+
+ protected:
+  void set_offset(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = offset + 1;
+  }
+  ptrdiff_t last_use() {
+    VIXL_ASSERT(IsUsed());
+    return -offset_ - 1;
+  }
+  void set_last_use(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = -offset - 1;
+  }
+
+  size_t size_;
+  ptrdiff_t offset_;
+  uint64_t low64_;
+  uint64_t high64_;
+
+ private:
+  LiteralPool* literal_pool_;
+  DeletionPolicy deletion_policy_;
+
+  friend class Assembler;
+  friend class LiteralPool;
+};
+
+
+template <typename T>
+class Literal : public RawLiteral {
+ public:
+  explicit Literal(T value,
+                   LiteralPool* literal_pool = NULL,
+                   RawLiteral::DeletionPolicy ownership = kManuallyDeleted)
+      : RawLiteral(sizeof(value), literal_pool, ownership) {
+    VIXL_STATIC_ASSERT(sizeof(value) <= kXRegSizeInBytes);
+    UpdateValue(value);
+  }
+
+  Literal(T high64, T low64,
+          LiteralPool* literal_pool = NULL,
+          RawLiteral::DeletionPolicy ownership = kManuallyDeleted)
+      : RawLiteral(kQRegSizeInBytes, literal_pool, ownership) {
+    VIXL_STATIC_ASSERT(sizeof(low64) == (kQRegSizeInBytes / 2));
+    UpdateValue(high64, low64);
+  }
+
+  virtual ~Literal() {}
+
+  // Update the value of this literal, if necessary by rewriting the value in
+  // the pool.
+  // If the literal has already been placed in a literal pool, the address of
+  // the start of the code buffer must be provided, as the literal only knows it
+  // offset from there.  This also allows patching the value after the code has
+  // been moved in memory.
+  void UpdateValue(T new_value, uint8_t* code_buffer = NULL) {
+    VIXL_ASSERT(sizeof(new_value) == size_);
+    memcpy(&low64_, &new_value, sizeof(new_value));
+    if (IsPlaced()) {
+      VIXL_ASSERT(code_buffer != NULL);
+      RewriteValueInCode(code_buffer);
+    }
+  }
+
+  void UpdateValue(T high64, T low64, uint8_t* code_buffer = NULL) {
+    VIXL_ASSERT(sizeof(low64) == size_ / 2);
+    memcpy(&low64_, &low64, sizeof(low64));
+    memcpy(&high64_, &high64, sizeof(high64));
+    if (IsPlaced()) {
+      VIXL_ASSERT(code_buffer != NULL);
+      RewriteValueInCode(code_buffer);
+    }
+  }
+
+  void UpdateValue(T new_value, const Assembler* assembler);
+  void UpdateValue(T high64, T low64, const Assembler* assembler);
+
+ private:
+  void RewriteValueInCode(uint8_t* code_buffer) {
+    VIXL_ASSERT(IsPlaced());
+    VIXL_STATIC_ASSERT(sizeof(T) <= kXRegSizeInBytes);
+    switch (size()) {
+      case kSRegSizeInBytes:
+        *reinterpret_cast<uint32_t*>(code_buffer + offset()) = raw_value32();
+        break;
+      case kDRegSizeInBytes:
+        *reinterpret_cast<uint64_t*>(code_buffer + offset()) = raw_value64();
+        break;
+      default:
+        VIXL_ASSERT(size() == kQRegSizeInBytes);
+        uint64_t* base_address =
+            reinterpret_cast<uint64_t*>(code_buffer + offset());
+        *base_address = raw_value128_low64();
+        *(base_address + 1) = raw_value128_high64();
+    }
+  }
+};
+
+
+// Control whether or not position-independent code should be emitted.
+enum PositionIndependentCodeOption {
+  // All code generated will be position-independent; all branches and
+  // references to labels generated with the Label class will use PC-relative
+  // addressing.
+  PositionIndependentCode,
+
+  // Allow VIXL to generate code that refers to absolute addresses. With this
+  // option, it will not be possible to copy the code buffer and run it from a
+  // different address; code must be generated in its final location.
+  PositionDependentCode,
+
+  // Allow VIXL to assume that the bottom 12 bits of the address will be
+  // constant, but that the top 48 bits may change. This allows `adrp` to
+  // function in systems which copy code between pages, but otherwise maintain
+  // 4KB page alignment.
+  PageOffsetDependentCode
+};
+
+
+// Control how scaled- and unscaled-offset loads and stores are generated.
+enum LoadStoreScalingOption {
+  // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferScaledOffset,
+
+  // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferUnscaledOffset,
+
+  // Require scaled-immediate-offset instructions.
+  RequireScaledOffset,
+
+  // Require unscaled-immediate-offset instructions.
+  RequireUnscaledOffset
+};
+
+
+// Assembler.
+class Assembler {
+ public:
+  Assembler(size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
+  Assembler(byte* buffer, size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
+
+  // The destructor asserts that one of the following is true:
+  //  * The Assembler object has not been used.
+  //  * Nothing has been emitted since the last Reset() call.
+  //  * Nothing has been emitted since the last FinalizeCode() call.
+  ~Assembler();
+
+  // System functions.
+
+  // Start generating code from the beginning of the buffer, discarding any code
+  // and data that has already been emitted into the buffer.
+  void Reset();
+
+  // Finalize a code buffer of generated instructions. This function must be
+  // called before executing or copying code from the buffer.
+  void FinalizeCode();
+
+  // Label.
+  // Bind a label to the current PC.
+  void bind(Label* label);
+
+  // Bind a label to a specified offset from the start of the buffer.
+  void BindToOffset(Label* label, ptrdiff_t offset);
+
+  // Place a literal at the current PC.
+  void place(RawLiteral* literal);
+
+  ptrdiff_t CursorOffset() const {
+    return buffer_->CursorOffset();
+  }
+
+  ptrdiff_t BufferEndOffset() const {
+    return static_cast<ptrdiff_t>(buffer_->capacity());
+  }
+
+  // Return the address of an offset in the buffer.
+  template <typename T>
+  T GetOffsetAddress(ptrdiff_t offset) const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return buffer_->GetOffsetAddress<T>(offset);
+  }
+
+  // Return the address of a bound label.
+  template <typename T>
+  T GetLabelAddress(const Label * label) const {
+    VIXL_ASSERT(label->IsBound());
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(label->location());
+  }
+
+  // Return the address of the cursor.
+  template <typename T>
+  T GetCursorAddress() const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(CursorOffset());
+  }
+
+  // Return the address of the start of the buffer.
+  template <typename T>
+  T GetStartAddress() const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(0);
+  }
+
+  Instruction* InstructionAt(ptrdiff_t instruction_offset) {
+    return GetOffsetAddress<Instruction*>(instruction_offset);
+  }
+
+  ptrdiff_t InstructionOffset(Instruction* instruction) {
+    VIXL_STATIC_ASSERT(sizeof(*instruction) == 1);
+    ptrdiff_t offset = instruction - GetStartAddress<Instruction*>();
+    VIXL_ASSERT((0 <= offset) &&
+                (offset < static_cast<ptrdiff_t>(BufferCapacity())));
+    return offset;
+  }
+
+  // Instruction set functions.
+
+  // Branch / Jump instructions.
+  // Branch to register.
+  void br(const Register& xn);
+
+  // Branch with link to register.
+  void blr(const Register& xn);
+
+  // Branch to register with return hint.
+  void ret(const Register& xn = lr);
+
+  // Unconditional branch to label.
+  void b(Label* label);
+
+  // Conditional branch to label.
+  void b(Label* label, Condition cond);
+
+  // Unconditional branch to PC offset.
+  void b(int imm26);
+
+  // Conditional branch to PC offset.
+  void b(int imm19, Condition cond);
+
+  // Branch with link to label.
+  void bl(Label* label);
+
+  // Branch with link to PC offset.
+  void bl(int imm26);
+
+  // Compare and branch to label if zero.
+  void cbz(const Register& rt, Label* label);
+
+  // Compare and branch to PC offset if zero.
+  void cbz(const Register& rt, int imm19);
+
+  // Compare and branch to label if not zero.
+  void cbnz(const Register& rt, Label* label);
+
+  // Compare and branch to PC offset if not zero.
+  void cbnz(const Register& rt, int imm19);
+
+  // Table lookup from one register.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Table lookup from two registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vm);
+
+  // Table lookup from three registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vm);
+
+  // Table lookup from four registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vn4,
+           const VRegister& vm);
+
+  // Table lookup extension from one register.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Table lookup extension from two registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vm);
+
+  // Table lookup extension from three registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vm);
+
+  // Table lookup extension from four registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vn4,
+           const VRegister& vm);
+
+  // Test bit and branch to label if zero.
+  void tbz(const Register& rt, unsigned bit_pos, Label* label);
+
+  // Test bit and branch to PC offset if zero.
+  void tbz(const Register& rt, unsigned bit_pos, int imm14);
+
+  // Test bit and branch to label if not zero.
+  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
+
+  // Test bit and branch to PC offset if not zero.
+  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
+
+  // Address calculation instructions.
+  // Calculate a PC-relative address. Unlike for branches the offset in adr is
+  // unscaled (i.e. the result can be unaligned).
+
+  // Calculate the address of a label.
+  void adr(const Register& rd, Label* label);
+
+  // Calculate the address of a PC offset.
+  void adr(const Register& rd, int imm21);
+
+  // Calculate the page address of a label.
+  void adrp(const Register& rd, Label* label);
+
+  // Calculate the page address of a PC offset.
+  void adrp(const Register& rd, int imm21);
+
+  // Data Processing instructions.
+  // Add.
+  void add(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Add and update status flags.
+  void adds(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Compare negative.
+  void cmn(const Register& rn, const Operand& operand);
+
+  // Subtract.
+  void sub(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Subtract and update status flags.
+  void subs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Compare.
+  void cmp(const Register& rn, const Operand& operand);
+
+  // Negate.
+  void neg(const Register& rd,
+           const Operand& operand);
+
+  // Negate and update status flags.
+  void negs(const Register& rd,
+            const Operand& operand);
+
+  // Add with carry bit.
+  void adc(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Add with carry bit and update status flags.
+  void adcs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Subtract with carry bit.
+  void sbc(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Subtract with carry bit and update status flags.
+  void sbcs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Negate with carry bit.
+  void ngc(const Register& rd,
+           const Operand& operand);
+
+  // Negate with carry bit and update status flags.
+  void ngcs(const Register& rd,
+            const Operand& operand);
+
+  // Logical instructions.
+  // Bitwise and (A & B).
+  void and_(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bitwise and (A & B) and update status flags.
+  void ands(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bit test and set flags.
+  void tst(const Register& rn, const Operand& operand);
+
+  // Bit clear (A & ~B).
+  void bic(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Bit clear (A & ~B) and update status flags.
+  void bics(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bitwise or (A | B).
+  void orr(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise nor (A | ~B).
+  void orn(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise eor/xor (A ^ B).
+  void eor(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise enor/xnor (A ^ ~B).
+  void eon(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Logical shift left by variable.
+  void lslv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Logical shift right by variable.
+  void lsrv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Arithmetic shift right by variable.
+  void asrv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Rotate right by variable.
+  void rorv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Bitfield instructions.
+  // Bitfield move.
+  void bfm(const Register& rd,
+           const Register& rn,
+           unsigned immr,
+           unsigned imms);
+
+  // Signed bitfield move.
+  void sbfm(const Register& rd,
+            const Register& rn,
+            unsigned immr,
+            unsigned imms);
+
+  // Unsigned bitfield move.
+  void ubfm(const Register& rd,
+            const Register& rn,
+            unsigned immr,
+            unsigned imms);
+
+  // Bfm aliases.
+  // Bitfield insert.
+  void bfi(const Register& rd,
+           const Register& rn,
+           unsigned lsb,
+           unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Bitfield extract and insert low.
+  void bfxil(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    bfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Sbfm aliases.
+  // Arithmetic shift right.
+  void asr(const Register& rd, const Register& rn, unsigned shift) {
+    VIXL_ASSERT(shift < rd.size());
+    sbfm(rd, rn, shift, rd.size() - 1);
+  }
+
+  // Signed bitfield insert with zero at right.
+  void sbfiz(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Signed bitfield extract.
+  void sbfx(const Register& rd,
+            const Register& rn,
+            unsigned lsb,
+            unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    sbfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Signed extend byte.
+  void sxtb(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 7);
+  }
+
+  // Signed extend halfword.
+  void sxth(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 15);
+  }
+
+  // Signed extend word.
+  void sxtw(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 31);
+  }
+
+  // Ubfm aliases.
+  // Logical shift left.
+  void lsl(const Register& rd, const Register& rn, unsigned shift) {
+    unsigned reg_size = rd.size();
+    VIXL_ASSERT(shift < reg_size);
+    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
+  }
+
+  // Logical shift right.
+  void lsr(const Register& rd, const Register& rn, unsigned shift) {
+    VIXL_ASSERT(shift < rd.size());
+    ubfm(rd, rn, shift, rd.size() - 1);
+  }
+
+  // Unsigned bitfield insert with zero at right.
+  void ubfiz(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Unsigned bitfield extract.
+  void ubfx(const Register& rd,
+            const Register& rn,
+            unsigned lsb,
+            unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    ubfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Unsigned extend byte.
+  void uxtb(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 7);
+  }
+
+  // Unsigned extend halfword.
+  void uxth(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 15);
+  }
+
+  // Unsigned extend word.
+  void uxtw(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 31);
+  }
+
+  // Extract.
+  void extr(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            unsigned lsb);
+
+  // Conditional select: rd = cond ? rn : rm.
+  void csel(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            Condition cond);
+
+  // Conditional select increment: rd = cond ? rn : rm + 1.
+  void csinc(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional select inversion: rd = cond ? rn : ~rm.
+  void csinv(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional select negation: rd = cond ? rn : -rm.
+  void csneg(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional set: rd = cond ? 1 : 0.
+  void cset(const Register& rd, Condition cond);
+
+  // Conditional set mask: rd = cond ? -1 : 0.
+  void csetm(const Register& rd, Condition cond);
+
+  // Conditional increment: rd = cond ? rn + 1 : rn.
+  void cinc(const Register& rd, const Register& rn, Condition cond);
+
+  // Conditional invert: rd = cond ? ~rn : rn.
+  void cinv(const Register& rd, const Register& rn, Condition cond);
+
+  // Conditional negate: rd = cond ? -rn : rn.
+  void cneg(const Register& rd, const Register& rn, Condition cond);
+
+  // Rotate right.
+  void ror(const Register& rd, const Register& rs, unsigned shift) {
+    extr(rd, rs, rs, shift);
+  }
+
+  // Conditional comparison.
+  // Conditional compare negative.
+  void ccmn(const Register& rn,
+            const Operand& operand,
+            StatusFlags nzcv,
+            Condition cond);
+
+  // Conditional compare.
+  void ccmp(const Register& rn,
+            const Operand& operand,
+            StatusFlags nzcv,
+            Condition cond);
+
+  // CRC-32 checksum from byte.
+  void crc32b(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from half-word.
+  void crc32h(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from word.
+  void crc32w(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from double word.
+  void crc32x(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 C checksum from byte.
+  void crc32cb(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32 C checksum from half-word.
+  void crc32ch(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32 C checksum from word.
+  void crc32cw(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32C checksum from double word.
+  void crc32cx(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // Multiply.
+  void mul(const Register& rd, const Register& rn, const Register& rm);
+
+  // Negated multiply.
+  void mneg(const Register& rd, const Register& rn, const Register& rm);
+
+  // Signed long multiply: 32 x 32 -> 64-bit.
+  void smull(const Register& rd, const Register& rn, const Register& rm);
+
+  // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
+  void smulh(const Register& xd, const Register& xn, const Register& xm);
+
+  // Multiply and accumulate.
+  void madd(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            const Register& ra);
+
+  // Multiply and subtract.
+  void msub(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            const Register& ra);
+
+  // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+  void smaddl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+  void umaddl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply: 32 x 32 -> 64-bit.
+  void umull(const Register& rd,
+             const Register& rn,
+             const Register& rm) {
+    umaddl(rd, rn, rm, xzr);
+  }
+
+  // Unsigned multiply high: 64 x 64 -> 64-bit <127:64>.
+  void umulh(const Register& xd,
+             const Register& xn,
+             const Register& xm);
+
+  // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+  void smsubl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+  void umsubl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Signed integer divide.
+  void sdiv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Unsigned integer divide.
+  void udiv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Bit reverse.
+  void rbit(const Register& rd, const Register& rn);
+
+  // Reverse bytes in 16-bit half words.
+  void rev16(const Register& rd, const Register& rn);
+
+  // Reverse bytes in 32-bit words.
+  void rev32(const Register& rd, const Register& rn);
+
+  // Reverse bytes.
+  void rev(const Register& rd, const Register& rn);
+
+  // Count leading zeroes.
+  void clz(const Register& rd, const Register& rn);
+
+  // Count leading sign bits.
+  void cls(const Register& rd, const Register& rn);
+
+  // Memory instructions.
+  // Load integer or FP register.
+  void ldr(const CPURegister& rt, const MemOperand& src,
+           LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store integer or FP register.
+  void str(const CPURegister& rt, const MemOperand& dst,
+           LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load word with sign extension.
+  void ldrsw(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load byte.
+  void ldrb(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store byte.
+  void strb(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load byte with sign extension.
+  void ldrsb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load half-word.
+  void ldrh(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store half-word.
+  void strh(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load half-word with sign extension.
+  void ldrsh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load integer or FP register (with unscaled offset).
+  void ldur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store integer or FP register (with unscaled offset).
+  void stur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load word with sign extension.
+  void ldursw(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte (with unscaled offset).
+  void ldurb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store byte (with unscaled offset).
+  void sturb(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte with sign extension (and unscaled offset).
+  void ldursb(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word (with unscaled offset).
+  void ldurh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store half-word (with unscaled offset).
+  void sturh(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word with sign extension (and unscaled offset).
+  void ldursh(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load integer or FP register pair.
+  void ldp(const CPURegister& rt, const CPURegister& rt2,
+           const MemOperand& src);
+
+  // Store integer or FP register pair.
+  void stp(const CPURegister& rt, const CPURegister& rt2,
+           const MemOperand& dst);
+
+  // Load word pair with sign extension.
+  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Load integer or FP register pair, non-temporal.
+  void ldnp(const CPURegister& rt, const CPURegister& rt2,
+            const MemOperand& src);
+
+  // Store integer or FP register pair, non-temporal.
+  void stnp(const CPURegister& rt, const CPURegister& rt2,
+            const MemOperand& dst);
+
+  // Load integer or FP register from literal pool.
+  void ldr(const CPURegister& rt, RawLiteral* literal);
+
+  // Load word with sign extension from literal pool.
+  void ldrsw(const Register& rt, RawLiteral* literal);
+
+  // Load integer or FP register from pc + imm19 << 2.
+  void ldr(const CPURegister& rt, int imm19);
+
+  // Load word with sign extension from pc + imm19 << 2.
+  void ldrsw(const Register& rt, int imm19);
+
+  // Store exclusive byte.
+  void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive half-word.
+  void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive register.
+  void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load exclusive byte.
+  void ldxrb(const Register& rt, const MemOperand& src);
+
+  // Load exclusive half-word.
+  void ldxrh(const Register& rt, const MemOperand& src);
+
+  // Load exclusive register.
+  void ldxr(const Register& rt, const MemOperand& src);
+
+  // Store exclusive register pair.
+  void stxp(const Register& rs,
+            const Register& rt,
+            const Register& rt2,
+            const MemOperand& dst);
+
+  // Load exclusive register pair.
+  void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release exclusive byte.
+  void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive half-word.
+  void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive register.
+  void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load-acquire exclusive byte.
+  void ldaxrb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive half-word.
+  void ldaxrh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive register.
+  void ldaxr(const Register& rt, const MemOperand& src);
+
+  // Store-release exclusive register pair.
+  void stlxp(const Register& rs,
+             const Register& rt,
+             const Register& rt2,
+             const MemOperand& dst);
+
+  // Load-acquire exclusive register pair.
+  void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release byte.
+  void stlrb(const Register& rt, const MemOperand& dst);
+
+  // Store-release half-word.
+  void stlrh(const Register& rt, const MemOperand& dst);
+
+  // Store-release register.
+  void stlr(const Register& rt, const MemOperand& dst);
+
+  // Load-acquire byte.
+  void ldarb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire half-word.
+  void ldarh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire register.
+  void ldar(const Register& rt, const MemOperand& src);
+
+  // Prefetch memory.
+  void prfm(PrefetchOperation op, const MemOperand& addr,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Prefetch memory (with unscaled offset).
+  void prfum(PrefetchOperation op, const MemOperand& addr,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Prefetch memory in the literal pool.
+  void prfm(PrefetchOperation op, RawLiteral* literal);
+
+  // Prefetch from pc + imm19 << 2.
+  void prfm(PrefetchOperation op, int imm19);
+
+  // Move instructions. The default shift of -1 indicates that the move
+  // instruction will calculate an appropriate 16-bit immediate and left shift
+  // that is equal to the 64-bit immediate argument. If an explicit left shift
+  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
+  //
+  // For movk, an explicit shift can be used to indicate which half word should
+  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
+  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
+  // most-significant.
+
+  // Move immediate and keep.
+  void movk(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVK);
+  }
+
+  // Move inverted immediate.
+  void movn(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVN);
+  }
+
+  // Move immediate.
+  void movz(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVZ);
+  }
+
+  // Misc instructions.
+  // Monitor debug-mode breakpoint.
+  void brk(int code);
+
+  // Halting debug-mode breakpoint.
+  void hlt(int code);
+
+  // Generate exception targeting EL1.
+  void svc(int code);
+
+  // Move register to register.
+  void mov(const Register& rd, const Register& rn);
+
+  // Move inverted operand to register.
+  void mvn(const Register& rd, const Operand& operand);
+
+  // System instructions.
+  // Move to register from system register.
+  void mrs(const Register& rt, SystemRegister sysreg);
+
+  // Move from register to system register.
+  void msr(SystemRegister sysreg, const Register& rt);
+
+  // System instruction.
+  void sys(int op1, int crn, int crm, int op2, const Register& rt = xzr);
+
+  // System instruction with pre-encoded op (op1:crn:crm:op2).
+  void sys(int op, const Register& rt = xzr);
+
+  // System data cache operation.
+  void dc(DataCacheOp op, const Register& rt);
+
+  // System instruction cache operation.
+  void ic(InstructionCacheOp op, const Register& rt);
+
+  // System hint.
+  void hint(SystemHint code);
+
+  // Clear exclusive monitor.
+  void clrex(int imm4 = 0xf);
+
+  // Data memory barrier.
+  void dmb(BarrierDomain domain, BarrierType type);
+
+  // Data synchronization barrier.
+  void dsb(BarrierDomain domain, BarrierType type);
+
+  // Instruction synchronization barrier.
+  void isb();
+
+  // Alias for system instructions.
+  // No-op.
+  void nop() {
+    hint(NOP);
+  }
+
+  // FP and NEON instructions.
+  // Move double precision immediate to FP register.
+  void fmov(const VRegister& vd, double imm);
+
+  // Move single precision immediate to FP register.
+  void fmov(const VRegister& vd, float imm);
+
+  // Move FP register to register.
+  void fmov(const Register& rd, const VRegister& fn);
+
+  // Move register to FP register.
+  void fmov(const VRegister& vd, const Register& rn);
+
+  // Move FP register to FP register.
+  void fmov(const VRegister& vd, const VRegister& fn);
+
+  // Move 64-bit register to top half of 128-bit FP register.
+  void fmov(const VRegister& vd, int index, const Register& rn);
+
+  // Move top half of 128-bit FP register to 64-bit register.
+  void fmov(const Register& rd, const VRegister& vn, int index);
+
+  // FP add.
+  void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP subtract.
+  void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP multiply.
+  void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP fused multiply-add.
+  void fmadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             const VRegister& va);
+
+  // FP fused multiply-subtract.
+  void fmsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             const VRegister& va);
+
+  // FP fused multiply-add and negate.
+  void fnmadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              const VRegister& va);
+
+  // FP fused multiply-subtract and negate.
+  void fnmsub(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              const VRegister& va);
+
+  // FP multiply-negate scalar.
+  void fnmul(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP reciprocal exponent scalar.
+  void frecpx(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP divide.
+  void fdiv(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP maximum.
+  void fmax(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP minimum.
+  void fmin(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP maximum number.
+  void fmaxnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP minimum number.
+  void fminnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP absolute.
+  void fabs(const VRegister& vd, const VRegister& vn);
+
+  // FP negate.
+  void fneg(const VRegister& vd, const VRegister& vn);
+
+  // FP square root.
+  void fsqrt(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, nearest with ties to away.
+  void frinta(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, implicit rounding.
+  void frinti(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, toward minus infinity.
+  void frintm(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, nearest with ties to even.
+  void frintn(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, toward plus infinity.
+  void frintp(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, exact, implicit rounding.
+  void frintx(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, towards zero.
+  void frintz(const VRegister& vd, const VRegister& vn);
+
+  void FPCompareMacro(const VRegister& vn,
+                      double value,
+                      FPTrapFlags trap);
+
+  void FPCompareMacro(const VRegister& vn,
+                      const VRegister& vm,
+                      FPTrapFlags trap);
+
+  // FP compare registers.
+  void fcmp(const VRegister& vn, const VRegister& vm);
+
+  // FP compare immediate.
+  void fcmp(const VRegister& vn, double value);
+
+  void FPCCompareMacro(const VRegister& vn,
+                       const VRegister& vm,
+                       StatusFlags nzcv,
+                       Condition cond,
+                       FPTrapFlags trap);
+
+  // FP conditional compare.
+  void fccmp(const VRegister& vn,
+             const VRegister& vm,
+             StatusFlags nzcv,
+             Condition cond);
+
+  // FP signaling compare registers.
+  void fcmpe(const VRegister& vn, const VRegister& vm);
+
+  // FP signaling compare immediate.
+  void fcmpe(const VRegister& vn, double value);
+
+  // FP conditional signaling compare.
+  void fccmpe(const VRegister& vn,
+              const VRegister& vm,
+              StatusFlags nzcv,
+              Condition cond);
+
+  // FP conditional select.
+  void fcsel(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             Condition cond);
+
+  // Common FP Convert functions.
+  void NEONFPConvertToInt(const Register& rd,
+                          const VRegister& vn,
+                          Instr op);
+  void NEONFPConvertToInt(const VRegister& vd,
+                          const VRegister& vn,
+                          Instr op);
+
+  // FP convert between precisions.
+  void fcvt(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to higher precision.
+  void fcvtl(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to higher precision (second part).
+  void fcvtl2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision.
+  void fcvtn(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower prevision (second part).
+  void fcvtn2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision, rounding to odd.
+  void fcvtxn(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision, rounding to odd (second part).
+  void fcvtxn2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to away.
+  void fcvtas(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to away.
+  void fcvtau(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to away.
+  void fcvtas(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to away.
+  void fcvtau(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards -infinity.
+  void fcvtms(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards -infinity.
+  void fcvtmu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards -infinity.
+  void fcvtms(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards -infinity.
+  void fcvtmu(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to even.
+  void fcvtns(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to even.
+  void fcvtnu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to even.
+  void fcvtns(const VRegister& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to even.
+  void fcvtnu(const VRegister& rd, const VRegister& vn);
+
+  // FP convert to signed integer or fixed-point, round towards zero.
+  void fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to unsigned integer or fixed-point, round towards zero.
+  void fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to signed integer or fixed-point, round towards zero.
+  void fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to unsigned integer or fixed-point, round towards zero.
+  void fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to signed integer, round towards +infinity.
+  void fcvtps(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards +infinity.
+  void fcvtpu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards +infinity.
+  void fcvtps(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards +infinity.
+  void fcvtpu(const VRegister& vd, const VRegister& vn);
+
+  // Convert signed integer or fixed point to FP.
+  void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+  // Convert unsigned integer or fixed point to FP.
+  void ucvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+  // Convert signed integer or fixed-point to FP.
+  void scvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+  // Convert unsigned integer or fixed-point to FP.
+  void ucvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+  // Unsigned absolute difference.
+  void uabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed absolute difference.
+  void sabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate.
+  void uaba(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed absolute difference and accumulate.
+  void saba(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Add.
+  void add(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Subtract.
+  void sub(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Unsigned halving add.
+  void uhadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed halving add.
+  void shadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned rounding halving add.
+  void urhadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed rounding halving add.
+  void srhadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned halving sub.
+  void uhsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed halving sub.
+  void shsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating add.
+  void uqadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating add.
+  void sqadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating subtract.
+  void uqsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating subtract.
+  void sqsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add pairwise.
+  void addp(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Add pair of elements scalar.
+  void addp(const VRegister& vd,
+            const VRegister& vn);
+
+  // Multiply-add to accumulator.
+  void mla(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply-subtract to accumulator.
+  void mls(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply.
+  void mul(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply by scalar element.
+  void mul(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Multiply-add by scalar element.
+  void mla(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Multiply-subtract by scalar element.
+  void mls(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Signed long multiply-add by scalar element.
+  void smlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply-add by scalar element (second part).
+  void smlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply-add by scalar element.
+  void umlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply-add by scalar element (second part).
+  void umlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed long multiply-sub by scalar element.
+  void smlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply-sub by scalar element (second part).
+  void smlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply-sub by scalar element.
+  void umlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply-sub by scalar element (second part).
+  void umlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed long multiply by scalar element.
+  void smull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply by scalar element (second part).
+  void smull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply by scalar element.
+  void umull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply by scalar element (second part).
+  void umull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed saturating double long multiply by element.
+  void sqdmull(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating double long multiply by element (second part).
+  void sqdmull2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Signed saturating doubling long multiply-add by element.
+  void sqdmlal(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating doubling long multiply-add by element (second part).
+  void sqdmlal2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Signed saturating doubling long multiply-sub by element.
+  void sqdmlsl(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating doubling long multiply-sub by element (second part).
+  void sqdmlsl2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Compare equal.
+  void cmeq(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare signed greater than or equal.
+  void cmge(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare signed greater than.
+  void cmgt(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare unsigned higher.
+  void cmhi(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare unsigned higher or same.
+  void cmhs(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare bitwise test bits nonzero.
+  void cmtst(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Compare bitwise to zero.
+  void cmeq(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed greater than or equal to zero.
+  void cmge(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed greater than zero.
+  void cmgt(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed less than or equal to zero.
+  void cmle(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed less than zero.
+  void cmlt(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Signed shift left by register.
+  void sshl(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned shift left by register.
+  void ushl(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed saturating shift left by register.
+  void sqshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating shift left by register.
+  void uqshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed rounding shift left by register.
+  void srshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned rounding shift left by register.
+  void urshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating rounding shift left by register.
+  void sqrshl(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned saturating rounding shift left by register.
+  void uqrshl(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Bitwise and.
+  void and_(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Bitwise or.
+  void orr(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise or immediate.
+  void orr(const VRegister& vd,
+           const int imm8,
+           const int left_shift = 0);
+
+  // Move register to register.
+  void mov(const VRegister& vd,
+           const VRegister& vn);
+
+  // Bitwise orn.
+  void orn(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise eor.
+  void eor(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bit clear immediate.
+  void bic(const VRegister& vd,
+           const int imm8,
+           const int left_shift = 0);
+
+  // Bit clear.
+  void bic(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise insert if false.
+  void bif(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise insert if true.
+  void bit(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise select.
+  void bsl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Polynomial multiply.
+  void pmul(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Vector move immediate.
+  void movi(const VRegister& vd,
+            const uint64_t imm,
+            Shift shift = LSL,
+            const int shift_amount = 0);
+
+  // Bitwise not.
+  void mvn(const VRegister& vd,
+           const VRegister& vn);
+
+  // Vector move inverted immediate.
+  void mvni(const VRegister& vd,
+            const int imm8,
+            Shift shift = LSL,
+            const int shift_amount = 0);
+
+  // Signed saturating accumulate of unsigned value.
+  void suqadd(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned saturating accumulate of signed value.
+  void usqadd(const VRegister& vd,
+              const VRegister& vn);
+
+  // Absolute value.
+  void abs(const VRegister& vd,
+           const VRegister& vn);
+
+  // Signed saturating absolute value.
+  void sqabs(const VRegister& vd,
+             const VRegister& vn);
+
+  // Negate.
+  void neg(const VRegister& vd,
+           const VRegister& vn);
+
+  // Signed saturating negate.
+  void sqneg(const VRegister& vd,
+             const VRegister& vn);
+
+  // Bitwise not.
+  void not_(const VRegister& vd,
+            const VRegister& vn);
+
+  // Extract narrow.
+  void xtn(const VRegister& vd,
+           const VRegister& vn);
+
+  // Extract narrow (second part).
+  void xtn2(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed saturating extract narrow.
+  void sqxtn(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed saturating extract narrow (second part).
+  void sqxtn2(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned saturating extract narrow.
+  void uqxtn(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned saturating extract narrow (second part).
+  void uqxtn2(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed saturating extract unsigned narrow.
+  void sqxtun(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed saturating extract unsigned narrow (second part).
+  void sqxtun2(const VRegister& vd,
+               const VRegister& vn);
+
+  // Extract vector from pair of vectors.
+  void ext(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int index);
+
+  // Duplicate vector element to vector or scalar.
+  void dup(const VRegister& vd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Move vector element to scalar.
+  void mov(const VRegister& vd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Duplicate general-purpose register to vector.
+  void dup(const VRegister& vd,
+           const Register& rn);
+
+  // Insert vector element from another vector element.
+  void ins(const VRegister& vd,
+           int vd_index,
+           const VRegister& vn,
+           int vn_index);
+
+  // Move vector element to another vector element.
+  void mov(const VRegister& vd,
+           int vd_index,
+           const VRegister& vn,
+           int vn_index);
+
+  // Insert vector element from general-purpose register.
+  void ins(const VRegister& vd,
+           int vd_index,
+           const Register& rn);
+
+  // Move general-purpose register to a vector element.
+  void mov(const VRegister& vd,
+           int vd_index,
+           const Register& rn);
+
+  // Unsigned move vector element to general-purpose register.
+  void umov(const Register& rd,
+            const VRegister& vn,
+            int vn_index);
+
+  // Move vector element to general-purpose register.
+  void mov(const Register& rd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Signed move vector element to general-purpose register.
+  void smov(const Register& rd,
+            const VRegister& vn,
+            int vn_index);
+
+  // One-element structure load to one register.
+  void ld1(const VRegister& vt,
+           const MemOperand& src);
+
+  // One-element structure load to two registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // One-element structure load to three registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // One-element structure load to four registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // One-element single structure load to one lane.
+  void ld1(const VRegister& vt,
+           int lane,
+           const MemOperand& src);
+
+  // One-element single structure load to all lanes.
+  void ld1r(const VRegister& vt,
+            const MemOperand& src);
+
+  // Two-element structure load.
+  void ld2(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // Two-element single structure load to one lane.
+  void ld2(const VRegister& vt,
+           const VRegister& vt2,
+           int lane,
+           const MemOperand& src);
+
+  // Two-element single structure load to all lanes.
+  void ld2r(const VRegister& vt,
+            const VRegister& vt2,
+            const MemOperand& src);
+
+  // Three-element structure load.
+  void ld3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // Three-element single structure load to one lane.
+  void ld3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           int lane,
+           const MemOperand& src);
+
+  // Three-element single structure load to all lanes.
+  void ld3r(const VRegister& vt,
+            const VRegister& vt2,
+            const VRegister& vt3,
+            const MemOperand& src);
+
+  // Four-element structure load.
+  void ld4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // Four-element single structure load to one lane.
+  void ld4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           int lane,
+           const MemOperand& src);
+
+  // Four-element single structure load to all lanes.
+  void ld4r(const VRegister& vt,
+            const VRegister& vt2,
+            const VRegister& vt3,
+            const VRegister& vt4,
+            const MemOperand& src);
+
+  // Count leading sign bits.
+  void cls(const VRegister& vd,
+           const VRegister& vn);
+
+  // Count leading zero bits (vector).
+  void clz(const VRegister& vd,
+           const VRegister& vn);
+
+  // Population count per byte.
+  void cnt(const VRegister& vd,
+           const VRegister& vn);
+
+  // Reverse bit order.
+  void rbit(const VRegister& vd,
+            const VRegister& vn);
+
+  // Reverse elements in 16-bit halfwords.
+  void rev16(const VRegister& vd,
+             const VRegister& vn);
+
+  // Reverse elements in 32-bit words.
+  void rev32(const VRegister& vd,
+             const VRegister& vn);
+
+  // Reverse elements in 64-bit doublewords.
+  void rev64(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned reciprocal square root estimate.
+  void ursqrte(const VRegister& vd,
+               const VRegister& vn);
+
+  // Unsigned reciprocal estimate.
+  void urecpe(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed pairwise long add.
+  void saddlp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned pairwise long add.
+  void uaddlp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed pairwise long add and accumulate.
+  void sadalp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned pairwise long add and accumulate.
+  void uadalp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Shift left by immediate.
+  void shl(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Signed saturating shift left by immediate.
+  void sqshl(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed saturating shift left unsigned by immediate.
+  void sqshlu(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift left by immediate.
+  void uqshl(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift left long by immediate.
+  void sshll(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift left long by immediate (second part).
+  void sshll2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Signed extend long.
+  void sxtl(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed extend long (second part).
+  void sxtl2(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned shift left long by immediate.
+  void ushll(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned shift left long by immediate (second part).
+  void ushll2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Shift left long by element size.
+  void shll(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Shift left long by element size (second part).
+  void shll2(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned extend long.
+  void uxtl(const VRegister& vd,
+            const VRegister& vn);
+
+  // Unsigned extend long (second part).
+  void uxtl2(const VRegister& vd,
+             const VRegister& vn);
+
+  // Shift left by immediate and insert.
+  void sli(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Shift right by immediate and insert.
+  void sri(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Signed maximum.
+  void smax(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed pairwise maximum.
+  void smaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add across vector.
+  void addv(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed add long across vector.
+  void saddlv(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned add long across vector.
+  void uaddlv(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP maximum number across vector.
+  void fmaxnmv(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP maximum across vector.
+  void fmaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP minimum number across vector.
+  void fminnmv(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP minimum across vector.
+  void fminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed maximum across vector.
+  void smaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed minimum.
+  void smin(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed minimum pairwise.
+  void sminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed minimum across vector.
+  void sminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // One-element structure store from one register.
+  void st1(const VRegister& vt,
+           const MemOperand& src);
+
+  // One-element structure store from two registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // One-element structure store from three registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // One-element structure store from four registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // One-element single structure store from one lane.
+  void st1(const VRegister& vt,
+           int lane,
+           const MemOperand& src);
+
+  // Two-element structure store from two registers.
+  void st2(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // Two-element single structure store from two lanes.
+  void st2(const VRegister& vt,
+           const VRegister& vt2,
+           int lane,
+           const MemOperand& src);
+
+  // Three-element structure store from three registers.
+  void st3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // Three-element single structure store from three lanes.
+  void st3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           int lane,
+           const MemOperand& src);
+
+  // Four-element structure store from four registers.
+  void st4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // Four-element single structure store from four lanes.
+  void st4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           int lane,
+           const MemOperand& src);
+
+  // Unsigned add long.
+  void uaddl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned add long (second part).
+  void uaddl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned add wide.
+  void uaddw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned add wide (second part).
+  void uaddw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed add long.
+  void saddl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed add long (second part).
+  void saddl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed add wide.
+  void saddw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed add wide (second part).
+  void saddw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned subtract long.
+  void usubl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned subtract long (second part).
+  void usubl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned subtract wide.
+  void usubw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned subtract wide (second part).
+  void usubw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed subtract long.
+  void ssubl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed subtract long (second part).
+  void ssubl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed integer subtract wide.
+  void ssubw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed integer subtract wide (second part).
+  void ssubw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned maximum.
+  void umax(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned pairwise maximum.
+  void umaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned maximum across vector.
+  void umaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned minimum.
+  void umin(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned pairwise minimum.
+  void uminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned minimum across vector.
+  void uminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Transpose vectors (primary).
+  void trn1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Transpose vectors (secondary).
+  void trn2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unzip vectors (primary).
+  void uzp1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unzip vectors (secondary).
+  void uzp2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Zip vectors (primary).
+  void zip1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Zip vectors (secondary).
+  void zip2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed shift right by immediate.
+  void sshr(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Unsigned shift right by immediate.
+  void ushr(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Signed rounding shift right by immediate.
+  void srshr(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned rounding shift right by immediate.
+  void urshr(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift right by immediate and accumulate.
+  void ssra(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Unsigned shift right by immediate and accumulate.
+  void usra(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Signed rounding shift right by immediate and accumulate.
+  void srsra(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned rounding shift right by immediate and accumulate.
+  void ursra(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Shift right narrow by immediate.
+  void shrn(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Shift right narrow by immediate (second part).
+  void shrn2(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Rounding shift right narrow by immediate.
+  void rshrn(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Rounding shift right narrow by immediate (second part).
+  void rshrn2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift right narrow by immediate.
+  void uqshrn(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift right narrow by immediate (second part).
+  void uqshrn2(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Unsigned saturating rounding shift right narrow by immediate.
+  void uqrshrn(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Unsigned saturating rounding shift right narrow by immediate (second part).
+  void uqrshrn2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed saturating shift right narrow by immediate.
+  void sqshrn(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Signed saturating shift right narrow by immediate (second part).
+  void sqshrn2(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating rounded shift right narrow by immediate.
+  void sqrshrn(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating rounded shift right narrow by immediate (second part).
+  void sqrshrn2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed saturating shift right unsigned narrow by immediate.
+  void sqshrun(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating shift right unsigned narrow by immediate (second part).
+  void sqshrun2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed sat rounded shift right unsigned narrow by immediate.
+  void sqrshrun(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed sat rounded shift right unsigned narrow by immediate (second part).
+  void sqrshrun2(const VRegister& vd,
+                 const VRegister& vn,
+                 int shift);
+
+  // FP reciprocal step.
+  void frecps(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // FP reciprocal estimate.
+  void frecpe(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP reciprocal square root estimate.
+  void frsqrte(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP reciprocal square root step.
+  void frsqrts(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed absolute difference and accumulate long.
+  void sabal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed absolute difference and accumulate long (second part).
+  void sabal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate long.
+  void uabal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate long (second part).
+  void uabal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed absolute difference long.
+  void sabdl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed absolute difference long (second part).
+  void sabdl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned absolute difference long.
+  void uabdl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned absolute difference long (second part).
+  void uabdl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Polynomial multiply long.
+  void pmull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Polynomial multiply long (second part).
+  void pmull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply-add.
+  void smlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply-add (second part).
+  void smlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned long multiply-add.
+  void umlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply-add (second part).
+  void umlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply-sub.
+  void smlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply-sub (second part).
+  void smlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned long multiply-sub.
+  void umlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply-sub (second part).
+  void umlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply.
+  void smull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply (second part).
+  void smull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed saturating doubling long multiply-add.
+  void sqdmlal(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply-add (second part).
+  void sqdmlal2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling long multiply-subtract.
+  void sqdmlsl(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply-subtract (second part).
+  void sqdmlsl2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling long multiply.
+  void sqdmull(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply (second part).
+  void sqdmull2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling multiply returning high half.
+  void sqdmulh(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating rounding doubling multiply returning high half.
+  void sqrdmulh(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling multiply element returning high half.
+  void sqdmulh(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating rounding doubling multiply element returning high half.
+  void sqrdmulh(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Unsigned long multiply long.
+  void umull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply (second part).
+  void umull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Add narrow returning high half.
+  void addhn(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add narrow returning high half (second part).
+  void addhn2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding add narrow returning high half.
+  void raddhn(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding add narrow returning high half (second part).
+  void raddhn2(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Subtract narrow returning high half.
+  void subhn(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Subtract narrow returning high half (second part).
+  void subhn2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding subtract narrow returning high half.
+  void rsubhn(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding subtract narrow returning high half (second part).
+  void rsubhn2(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP vector multiply accumulate.
+  void fmla(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP vector multiply subtract.
+  void fmls(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP vector multiply extended.
+  void fmulx(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP absolute greater than or equal.
+  void facge(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP absolute greater than.
+  void facgt(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP multiply by element.
+  void fmul(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP fused multiply-add to accumulator by element.
+  void fmla(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP fused multiply-sub from accumulator by element.
+  void fmls(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP multiply extended by element.
+  void fmulx(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // FP compare equal.
+  void fcmeq(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP greater than.
+  void fcmgt(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP greater than or equal.
+  void fcmge(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP compare equal to zero.
+  void fcmeq(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP greater than zero.
+  void fcmgt(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP greater than or equal to zero.
+  void fcmge(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP less than or equal to zero.
+  void fcmle(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP less than to zero.
+  void fcmlt(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP absolute difference.
+  void fabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP pairwise add vector.
+  void faddp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise add scalar.
+  void faddp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise maximum vector.
+  void fmaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise maximum scalar.
+  void fmaxp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise minimum vector.
+  void fminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise minimum scalar.
+  void fminp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise maximum number vector.
+  void fmaxnmp(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP pairwise maximum number scalar.
+  void fmaxnmp(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP pairwise minimum number vector.
+  void fminnmp(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP pairwise minimum number scalar.
+  void fminnmp(const VRegister& vd,
+               const VRegister& vn);
+
+  // Emit generic instructions.
+  // Emit raw instructions into the instruction stream.
+  void dci(Instr raw_inst) { Emit(raw_inst); }
+
+  // Emit 32 bits of data into the instruction stream.
+  void dc32(uint32_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(data);
+  }
+
+  // Emit 64 bits of data into the instruction stream.
+  void dc64(uint64_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit64(data);
+  }
+
+  // Copy a string into the instruction stream, including the terminating NULL
+  // character. The instruction pointer is then aligned correctly for
+  // subsequent instructions.
+  void EmitString(const char * string) {
+    VIXL_ASSERT(string != NULL);
+    VIXL_ASSERT(buffer_monitor_ > 0);
+
+    buffer_->EmitString(string);
+    buffer_->Align();
+  }
+
+  // Code generation helpers.
+
+  // Register encoding.
+  static Instr Rd(CPURegister rd) {
+    VIXL_ASSERT(rd.code() != kSPRegInternalCode);
+    return rd.code() << Rd_offset;
+  }
+
+  static Instr Rn(CPURegister rn) {
+    VIXL_ASSERT(rn.code() != kSPRegInternalCode);
+    return rn.code() << Rn_offset;
+  }
+
+  static Instr Rm(CPURegister rm) {
+    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+    return rm.code() << Rm_offset;
+  }
+
+  static Instr RmNot31(CPURegister rm) {
+    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+    VIXL_ASSERT(!rm.IsZero());
+    return Rm(rm);
+  }
+
+  static Instr Ra(CPURegister ra) {
+    VIXL_ASSERT(ra.code() != kSPRegInternalCode);
+    return ra.code() << Ra_offset;
+  }
+
+  static Instr Rt(CPURegister rt) {
+    VIXL_ASSERT(rt.code() != kSPRegInternalCode);
+    return rt.code() << Rt_offset;
+  }
+
+  static Instr Rt2(CPURegister rt2) {
+    VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
+    return rt2.code() << Rt2_offset;
+  }
+
+  static Instr Rs(CPURegister rs) {
+    VIXL_ASSERT(rs.code() != kSPRegInternalCode);
+    return rs.code() << Rs_offset;
+  }
+
+  // These encoding functions allow the stack pointer to be encoded, and
+  // disallow the zero register.
+  static Instr RdSP(Register rd) {
+    VIXL_ASSERT(!rd.IsZero());
+    return (rd.code() & kRegCodeMask) << Rd_offset;
+  }
+
+  static Instr RnSP(Register rn) {
+    VIXL_ASSERT(!rn.IsZero());
+    return (rn.code() & kRegCodeMask) << Rn_offset;
+  }
+
+  // Flags encoding.
+  static Instr Flags(FlagsUpdate S) {
+    if (S == SetFlags) {
+      return 1 << FlagsUpdate_offset;
+    } else if (S == LeaveFlags) {
+      return 0 << FlagsUpdate_offset;
+    }
+    VIXL_UNREACHABLE();
+    return 0;
+  }
+
+  static Instr Cond(Condition cond) {
+    return cond << Condition_offset;
+  }
+
+  // PC-relative address encoding.
+  static Instr ImmPCRelAddress(int imm21) {
+    VIXL_ASSERT(is_int21(imm21));
+    Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
+    Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
+    Instr immlo = imm << ImmPCRelLo_offset;
+    return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
+  }
+
+  // Branch encoding.
+  static Instr ImmUncondBranch(int imm26) {
+    VIXL_ASSERT(is_int26(imm26));
+    return truncate_to_int26(imm26) << ImmUncondBranch_offset;
+  }
+
+  static Instr ImmCondBranch(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmCondBranch_offset;
+  }
+
+  static Instr ImmCmpBranch(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmCmpBranch_offset;
+  }
+
+  static Instr ImmTestBranch(int imm14) {
+    VIXL_ASSERT(is_int14(imm14));
+    return truncate_to_int14(imm14) << ImmTestBranch_offset;
+  }
+
+  static Instr ImmTestBranchBit(unsigned bit_pos) {
+    VIXL_ASSERT(is_uint6(bit_pos));
+    // Subtract five from the shift offset, as we need bit 5 from bit_pos.
+    unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
+    unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
+    b5 &= ImmTestBranchBit5_mask;
+    b40 &= ImmTestBranchBit40_mask;
+    return b5 | b40;
+  }
+
+  // Data Processing encoding.
+  static Instr SF(Register rd) {
+      return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
+  }
+
+  static Instr ImmAddSub(int imm) {
+    VIXL_ASSERT(IsImmAddSub(imm));
+    if (is_uint12(imm)) {  // No shift required.
+      imm <<= ImmAddSub_offset;
+    } else {
+      imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
+    }
+    return imm;
+  }
+
+  static Instr ImmS(unsigned imms, unsigned reg_size) {
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(imms)) ||
+           ((reg_size == kWRegSize) && is_uint5(imms)));
+    USE(reg_size);
+    return imms << ImmS_offset;
+  }
+
+  static Instr ImmR(unsigned immr, unsigned reg_size) {
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
+           ((reg_size == kWRegSize) && is_uint5(immr)));
+    USE(reg_size);
+    VIXL_ASSERT(is_uint6(immr));
+    return immr << ImmR_offset;
+  }
+
+  static Instr ImmSetBits(unsigned imms, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT(is_uint6(imms));
+    VIXL_ASSERT((reg_size == kXRegSize) || is_uint6(imms + 3));
+    USE(reg_size);
+    return imms << ImmSetBits_offset;
+  }
+
+  static Instr ImmRotate(unsigned immr, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
+           ((reg_size == kWRegSize) && is_uint5(immr)));
+    USE(reg_size);
+    return immr << ImmRotate_offset;
+  }
+
+  static Instr ImmLLiteral(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmLLiteral_offset;
+  }
+
+  static Instr BitN(unsigned bitn, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
+    USE(reg_size);
+    return bitn << BitN_offset;
+  }
+
+  static Instr ShiftDP(Shift shift) {
+    VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
+    return shift << ShiftDP_offset;
+  }
+
+  static Instr ImmDPShift(unsigned amount) {
+    VIXL_ASSERT(is_uint6(amount));
+    return amount << ImmDPShift_offset;
+  }
+
+  static Instr ExtendMode(Extend extend) {
+    return extend << ExtendMode_offset;
+  }
+
+  static Instr ImmExtendShift(unsigned left_shift) {
+    VIXL_ASSERT(left_shift <= 4);
+    return left_shift << ImmExtendShift_offset;
+  }
+
+  static Instr ImmCondCmp(unsigned imm) {
+    VIXL_ASSERT(is_uint5(imm));
+    return imm << ImmCondCmp_offset;
+  }
+
+  static Instr Nzcv(StatusFlags nzcv) {
+    return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
+  }
+
+  // MemOperand offset encoding.
+  static Instr ImmLSUnsigned(int imm12) {
+    VIXL_ASSERT(is_uint12(imm12));
+    return imm12 << ImmLSUnsigned_offset;
+  }
+
+  static Instr ImmLS(int imm9) {
+    VIXL_ASSERT(is_int9(imm9));
+    return truncate_to_int9(imm9) << ImmLS_offset;
+  }
+
+  static Instr ImmLSPair(int imm7, unsigned access_size) {
+    VIXL_ASSERT(((imm7 >> access_size) << access_size) == imm7);
+    int scaled_imm7 = imm7 >> access_size;
+    VIXL_ASSERT(is_int7(scaled_imm7));
+    return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
+  }
+
+  static Instr ImmShiftLS(unsigned shift_amount) {
+    VIXL_ASSERT(is_uint1(shift_amount));
+    return shift_amount << ImmShiftLS_offset;
+  }
+
+  static Instr ImmPrefetchOperation(int imm5) {
+    VIXL_ASSERT(is_uint5(imm5));
+    return imm5 << ImmPrefetchOperation_offset;
+  }
+
+  static Instr ImmException(int imm16) {
+    VIXL_ASSERT(is_uint16(imm16));
+    return imm16 << ImmException_offset;
+  }
+
+  static Instr ImmSystemRegister(int imm15) {
+    VIXL_ASSERT(is_uint15(imm15));
+    return imm15 << ImmSystemRegister_offset;
+  }
+
+  static Instr ImmHint(int imm7) {
+    VIXL_ASSERT(is_uint7(imm7));
+    return imm7 << ImmHint_offset;
+  }
+
+  static Instr CRm(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << CRm_offset;
+  }
+
+  static Instr CRn(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << CRn_offset;
+  }
+
+  static Instr SysOp(int imm14) {
+    VIXL_ASSERT(is_uint14(imm14));
+    return imm14 << SysOp_offset;
+  }
+
+  static Instr ImmSysOp1(int imm3) {
+    VIXL_ASSERT(is_uint3(imm3));
+    return imm3 << SysOp1_offset;
+  }
+
+  static Instr ImmSysOp2(int imm3) {
+    VIXL_ASSERT(is_uint3(imm3));
+    return imm3 << SysOp2_offset;
+  }
+
+  static Instr ImmBarrierDomain(int imm2) {
+    VIXL_ASSERT(is_uint2(imm2));
+    return imm2 << ImmBarrierDomain_offset;
+  }
+
+  static Instr ImmBarrierType(int imm2) {
+    VIXL_ASSERT(is_uint2(imm2));
+    return imm2 << ImmBarrierType_offset;
+  }
+
+  // Move immediates encoding.
+  static Instr ImmMoveWide(uint64_t imm) {
+    VIXL_ASSERT(is_uint16(imm));
+    return static_cast<Instr>(imm << ImmMoveWide_offset);
+  }
+
+  static Instr ShiftMoveWide(int64_t shift) {
+    VIXL_ASSERT(is_uint2(shift));
+    return static_cast<Instr>(shift << ShiftMoveWide_offset);
+  }
+
+  // FP Immediates.
+  static Instr ImmFP32(float imm);
+  static Instr ImmFP64(double imm);
+
+  // FP register type.
+  static Instr FPType(FPRegister fd) {
+    return fd.Is64Bits() ? FP64 : FP32;
+  }
+
+  static Instr FPScale(unsigned scale) {
+    VIXL_ASSERT(is_uint6(scale));
+    return scale << FPScale_offset;
+  }
+
+  // Immediate field checking helpers.
+  static bool IsImmAddSub(int64_t immediate);
+  static bool IsImmConditionalCompare(int64_t immediate);
+  static bool IsImmFP32(float imm);
+  static bool IsImmFP64(double imm);
+  static bool IsImmLogical(uint64_t value,
+                           unsigned width,
+                           unsigned* n = NULL,
+                           unsigned* imm_s = NULL,
+                           unsigned* imm_r = NULL);
+  static bool IsImmLSPair(int64_t offset, unsigned access_size);
+  static bool IsImmLSScaled(int64_t offset, unsigned access_size);
+  static bool IsImmLSUnscaled(int64_t offset);
+  static bool IsImmMovn(uint64_t imm, unsigned reg_size);
+  static bool IsImmMovz(uint64_t imm, unsigned reg_size);
+
+  // Instruction bits for vector format in data processing operations.
+  static Instr VFormat(VRegister vd) {
+    if (vd.Is64Bits()) {
+      switch (vd.lanes()) {
+        case 2: return NEON_2S;
+        case 4: return NEON_4H;
+        case 8: return NEON_8B;
+        default: return 0xffffffff;
+      }
+    } else {
+      VIXL_ASSERT(vd.Is128Bits());
+      switch (vd.lanes()) {
+        case 2: return NEON_2D;
+        case 4: return NEON_4S;
+        case 8: return NEON_8H;
+        case 16: return NEON_16B;
+        default: return 0xffffffff;
+      }
+    }
+  }
+
+  // Instruction bits for vector format in floating point data processing
+  // operations.
+  static Instr FPFormat(VRegister vd) {
+    if (vd.lanes() == 1) {
+      // Floating point scalar formats.
+      VIXL_ASSERT(vd.Is32Bits() || vd.Is64Bits());
+      return vd.Is64Bits() ? FP64 : FP32;
+    }
+
+    // Two lane floating point vector formats.
+    if (vd.lanes() == 2) {
+      VIXL_ASSERT(vd.Is64Bits() || vd.Is128Bits());
+      return vd.Is128Bits() ? NEON_FP_2D : NEON_FP_2S;
+    }
+
+    // Four lane floating point vector format.
+    VIXL_ASSERT((vd.lanes() == 4) && vd.Is128Bits());
+    return NEON_FP_4S;
+  }
+
+  // Instruction bits for vector format in load and store operations.
+  static Instr LSVFormat(VRegister vd) {
+    if (vd.Is64Bits()) {
+      switch (vd.lanes()) {
+        case 1: return LS_NEON_1D;
+        case 2: return LS_NEON_2S;
+        case 4: return LS_NEON_4H;
+        case 8: return LS_NEON_8B;
+        default: return 0xffffffff;
+      }
+    } else {
+      VIXL_ASSERT(vd.Is128Bits());
+      switch (vd.lanes()) {
+        case 2: return LS_NEON_2D;
+        case 4: return LS_NEON_4S;
+        case 8: return LS_NEON_8H;
+        case 16: return LS_NEON_16B;
+        default: return 0xffffffff;
+      }
+    }
+  }
+
+  // Instruction bits for scalar format in data processing operations.
+  static Instr SFormat(VRegister vd) {
+    VIXL_ASSERT(vd.lanes() == 1);
+    switch (vd.SizeInBytes()) {
+      case 1: return NEON_B;
+      case 2: return NEON_H;
+      case 4: return NEON_S;
+      case 8: return NEON_D;
+      default: return 0xffffffff;
+    }
+  }
+
+  static Instr ImmNEONHLM(int index, int num_bits) {
+    int h, l, m;
+    if (num_bits == 3) {
+      VIXL_ASSERT(is_uint3(index));
+      h  = (index >> 2) & 1;
+      l  = (index >> 1) & 1;
+      m  = (index >> 0) & 1;
+    } else if (num_bits == 2) {
+      VIXL_ASSERT(is_uint2(index));
+      h  = (index >> 1) & 1;
+      l  = (index >> 0) & 1;
+      m  = 0;
+    } else {
+      VIXL_ASSERT(is_uint1(index) && (num_bits == 1));
+      h  = (index >> 0) & 1;
+      l  = 0;
+      m  = 0;
+    }
+    return (h << NEONH_offset) | (l << NEONL_offset) | (m << NEONM_offset);
+  }
+
+  static Instr ImmNEONExt(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << ImmNEONExt_offset;
+  }
+
+  static Instr ImmNEON5(Instr format, int index) {
+    VIXL_ASSERT(is_uint4(index));
+    int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+    int imm5 = (index << (s + 1)) | (1 << s);
+    return imm5 << ImmNEON5_offset;
+  }
+
+  static Instr ImmNEON4(Instr format, int index) {
+    VIXL_ASSERT(is_uint4(index));
+    int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+    int imm4 = index << s;
+    return imm4 << ImmNEON4_offset;
+  }
+
+  static Instr ImmNEONabcdefgh(int imm8) {
+    VIXL_ASSERT(is_uint8(imm8));
+    Instr instr;
+    instr  = ((imm8 >> 5) & 7) << ImmNEONabc_offset;
+    instr |= (imm8 & 0x1f) << ImmNEONdefgh_offset;
+    return instr;
+  }
+
+  static Instr NEONCmode(int cmode) {
+    VIXL_ASSERT(is_uint4(cmode));
+    return cmode << NEONCmode_offset;
+  }
+
+  static Instr NEONModImmOp(int op) {
+    VIXL_ASSERT(is_uint1(op));
+    return op << NEONModImmOp_offset;
+  }
+
+  // Size of the code generated since label to the current position.
+  size_t SizeOfCodeGeneratedSince(Label* label) const {
+    VIXL_ASSERT(label->IsBound());
+    return buffer_->OffsetFrom(label->location());
+  }
+
+  size_t SizeOfCodeGenerated() const {
+    return buffer_->CursorOffset();
+  }
+
+  size_t BufferCapacity() const { return buffer_->capacity(); }
+
+  size_t RemainingBufferSpace() const { return buffer_->RemainingBytes(); }
+
+  void EnsureSpaceFor(size_t amount) {
+    if (buffer_->RemainingBytes() < amount) {
+      size_t capacity = buffer_->capacity();
+      size_t size = buffer_->CursorOffset();
+      do {
+        // TODO(all): refine.
+        capacity *= 2;
+      } while ((capacity - size) <  amount);
+      buffer_->Grow(capacity);
+    }
+  }
+
+#ifdef VIXL_DEBUG
+  void AcquireBuffer() {
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+    buffer_monitor_++;
+  }
+
+  void ReleaseBuffer() {
+    buffer_monitor_--;
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+  }
+#endif
+
+  PositionIndependentCodeOption pic() const {
+    return pic_;
+  }
+
+  bool AllowPageOffsetDependentCode() const {
+    return (pic() == PageOffsetDependentCode) ||
+           (pic() == PositionDependentCode);
+  }
+
+  static const Register& AppropriateZeroRegFor(const CPURegister& reg) {
+    return reg.Is64Bits() ? xzr : wzr;
+  }
+
+
+ protected:
+  void LoadStore(const CPURegister& rt,
+                 const MemOperand& addr,
+                 LoadStoreOp op,
+                 LoadStoreScalingOption option = PreferScaledOffset);
+
+  void LoadStorePair(const CPURegister& rt,
+                     const CPURegister& rt2,
+                     const MemOperand& addr,
+                     LoadStorePairOp op);
+  void LoadStoreStruct(const VRegister& vt,
+                       const MemOperand& addr,
+                       NEONLoadStoreMultiStructOp op);
+  void LoadStoreStruct1(const VRegister& vt,
+                        int reg_count,
+                        const MemOperand& addr);
+  void LoadStoreStructSingle(const VRegister& vt,
+                             uint32_t lane,
+                             const MemOperand& addr,
+                             NEONLoadStoreSingleStructOp op);
+  void LoadStoreStructSingleAllLanes(const VRegister& vt,
+                                     const MemOperand& addr,
+                                     NEONLoadStoreSingleStructOp op);
+  void LoadStoreStructVerify(const VRegister& vt,
+                             const MemOperand& addr,
+                             Instr op);
+
+  void Prefetch(PrefetchOperation op,
+                const MemOperand& addr,
+                LoadStoreScalingOption option = PreferScaledOffset);
+
+  // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
+  // reports a bogus uninitialised warning then.
+  void Logical(const Register& rd,
+               const Register& rn,
+               const Operand operand,
+               LogicalOp op);
+  void LogicalImmediate(const Register& rd,
+                        const Register& rn,
+                        unsigned n,
+                        unsigned imm_s,
+                        unsigned imm_r,
+                        LogicalOp op);
+
+  void ConditionalCompare(const Register& rn,
+                          const Operand& operand,
+                          StatusFlags nzcv,
+                          Condition cond,
+                          ConditionalCompareOp op);
+
+  void AddSubWithCarry(const Register& rd,
+                       const Register& rn,
+                       const Operand& operand,
+                       FlagsUpdate S,
+                       AddSubWithCarryOp op);
+
+
+  // Functions for emulating operands not directly supported by the instruction
+  // set.
+  void EmitShift(const Register& rd,
+                 const Register& rn,
+                 Shift shift,
+                 unsigned amount);
+  void EmitExtendShift(const Register& rd,
+                       const Register& rn,
+                       Extend extend,
+                       unsigned left_shift);
+
+  void AddSub(const Register& rd,
+              const Register& rn,
+              const Operand& operand,
+              FlagsUpdate S,
+              AddSubOp op);
+
+  void NEONTable(const VRegister& vd,
+                 const VRegister& vn,
+                 const VRegister& vm,
+                 NEONTableOp op);
+
+  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
+  // registers. Only simple loads are supported; sign- and zero-extension (such
+  // as in LDPSW_x or LDRB_w) are not supported.
+  static LoadStoreOp LoadOpFor(const CPURegister& rt);
+  static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
+                                       const CPURegister& rt2);
+  static LoadStoreOp StoreOpFor(const CPURegister& rt);
+  static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
+                                        const CPURegister& rt2);
+  static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
+    const CPURegister& rt, const CPURegister& rt2);
+  static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
+    const CPURegister& rt, const CPURegister& rt2);
+  static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
+
+
+ private:
+  static uint32_t FP32ToImm8(float imm);
+  static uint32_t FP64ToImm8(double imm);
+
+  // Instruction helpers.
+  void MoveWide(const Register& rd,
+                uint64_t imm,
+                int shift,
+                MoveWideImmediateOp mov_op);
+  void DataProcShiftedRegister(const Register& rd,
+                               const Register& rn,
+                               const Operand& operand,
+                               FlagsUpdate S,
+                               Instr op);
+  void DataProcExtendedRegister(const Register& rd,
+                                const Register& rn,
+                                const Operand& operand,
+                                FlagsUpdate S,
+                                Instr op);
+  void LoadStorePairNonTemporal(const CPURegister& rt,
+                                const CPURegister& rt2,
+                                const MemOperand& addr,
+                                LoadStorePairNonTemporalOp op);
+  void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
+  void ConditionalSelect(const Register& rd,
+                         const Register& rn,
+                         const Register& rm,
+                         Condition cond,
+                         ConditionalSelectOp op);
+  void DataProcessing1Source(const Register& rd,
+                             const Register& rn,
+                             DataProcessing1SourceOp op);
+  void DataProcessing3Source(const Register& rd,
+                             const Register& rn,
+                             const Register& rm,
+                             const Register& ra,
+                             DataProcessing3SourceOp op);
+  void FPDataProcessing1Source(const VRegister& fd,
+                               const VRegister& fn,
+                               FPDataProcessing1SourceOp op);
+  void FPDataProcessing3Source(const VRegister& fd,
+                               const VRegister& fn,
+                               const VRegister& fm,
+                               const VRegister& fa,
+                               FPDataProcessing3SourceOp op);
+  void NEONAcrossLanesL(const VRegister& vd,
+                        const VRegister& vn,
+                        NEONAcrossLanesOp op);
+  void NEONAcrossLanes(const VRegister& vd,
+                       const VRegister& vn,
+                       NEONAcrossLanesOp op);
+  void NEONModifiedImmShiftLsl(const VRegister& vd,
+                               const int imm8,
+                               const int left_shift,
+                               NEONModifiedImmediateOp op);
+  void NEONModifiedImmShiftMsl(const VRegister& vd,
+                               const int imm8,
+                               const int shift_amount,
+                               NEONModifiedImmediateOp op);
+  void NEONFP2Same(const VRegister& vd,
+                   const VRegister& vn,
+                   Instr vop);
+  void NEON3Same(const VRegister& vd,
+                 const VRegister& vn,
+                 const VRegister& vm,
+                 NEON3SameOp vop);
+  void NEONFP3Same(const VRegister& vd,
+                   const VRegister& vn,
+                   const VRegister& vm,
+                   Instr op);
+  void NEON3DifferentL(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       NEON3DifferentOp vop);
+  void NEON3DifferentW(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       NEON3DifferentOp vop);
+  void NEON3DifferentHN(const VRegister& vd,
+                        const VRegister& vn,
+                        const VRegister& vm,
+                        NEON3DifferentOp vop);
+  void NEONFP2RegMisc(const VRegister& vd,
+                      const VRegister& vn,
+                      NEON2RegMiscOp vop,
+                      double value = 0.0);
+  void NEON2RegMisc(const VRegister& vd,
+                    const VRegister& vn,
+                    NEON2RegMiscOp vop,
+                    int value = 0);
+  void NEONFP2RegMisc(const VRegister& vd,
+                      const VRegister& vn,
+                      Instr op);
+  void NEONAddlp(const VRegister& vd,
+                 const VRegister& vn,
+                 NEON2RegMiscOp op);
+  void NEONPerm(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                NEONPermOp op);
+  void NEONFPByElement(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       int vm_index,
+                       NEONByIndexedElementOp op);
+  void NEONByElement(const VRegister& vd,
+                     const VRegister& vn,
+                     const VRegister& vm,
+                     int vm_index,
+                     NEONByIndexedElementOp op);
+  void NEONByElementL(const VRegister& vd,
+                      const VRegister& vn,
+                      const VRegister& vm,
+                      int vm_index,
+                      NEONByIndexedElementOp op);
+  void NEONShiftImmediate(const VRegister& vd,
+                          const VRegister& vn,
+                          NEONShiftImmediateOp op,
+                          int immh_immb);
+  void NEONShiftLeftImmediate(const VRegister& vd,
+                              const VRegister& vn,
+                              int shift,
+                              NEONShiftImmediateOp op);
+  void NEONShiftRightImmediate(const VRegister& vd,
+                               const VRegister& vn,
+                               int shift,
+                               NEONShiftImmediateOp op);
+  void NEONShiftImmediateL(const VRegister& vd,
+                           const VRegister& vn,
+                           int shift,
+                           NEONShiftImmediateOp op);
+  void NEONShiftImmediateN(const VRegister& vd,
+                           const VRegister& vn,
+                           int shift,
+                           NEONShiftImmediateOp op);
+  void NEONXtn(const VRegister& vd,
+               const VRegister& vn,
+               NEON2RegMiscOp vop);
+
+  Instr LoadStoreStructAddrModeField(const MemOperand& addr);
+
+  // Encode the specified MemOperand for the specified access size and scaling
+  // preference.
+  Instr LoadStoreMemOperand(const MemOperand& addr,
+                            unsigned access_size,
+                            LoadStoreScalingOption option);
+
+  // Link the current (not-yet-emitted) instruction to the specified label, then
+  // return an offset to be encoded in the instruction. If the label is not yet
+  // bound, an offset of 0 is returned.
+  ptrdiff_t LinkAndGetByteOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetInstructionOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetPageOffsetTo(Label * label);
+
+  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+  template <int element_shift>
+  ptrdiff_t LinkAndGetOffsetTo(Label* label);
+
+  // Literal load offset are in words (32-bit).
+  ptrdiff_t LinkAndGetWordOffsetTo(RawLiteral* literal);
+
+  // Emit the instruction in buffer_.
+  void Emit(Instr instruction) {
+    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(instruction);
+  }
+
+  // Buffer where the code is emitted.
+  CodeBuffer* buffer_;
+  PositionIndependentCodeOption pic_;
+
+#ifdef VIXL_DEBUG
+  int64_t buffer_monitor_;
+#endif
+};
+
+
+// All Assembler emits MUST acquire/release the underlying code buffer. The
+// helper scope below will do so and optionally ensure the buffer is big enough
+// to receive the emit. It is possible to request the scope not to perform any
+// checks (kNoCheck) if for example it is known in advance the buffer size is
+// adequate or there is some other size checking mechanism in place.
+class CodeBufferCheckScope {
+ public:
+  // Tell whether or not the scope needs to ensure the associated CodeBuffer
+  // has enough space for the requested size.
+  enum CheckPolicy {
+    kNoCheck,
+    kCheck
+  };
+
+  // Tell whether or not the scope should assert the amount of code emitted
+  // within the scope is consistent with the requested amount.
+  enum AssertPolicy {
+    kNoAssert,    // No assert required.
+    kExactSize,   // The code emitted must be exactly size bytes.
+    kMaximumSize  // The code emitted must be at most size bytes.
+  };
+
+  CodeBufferCheckScope(Assembler* assm,
+                       size_t size,
+                       CheckPolicy check_policy = kCheck,
+                       AssertPolicy assert_policy = kMaximumSize)
+      : assm_(assm) {
+    if (check_policy == kCheck) assm->EnsureSpaceFor(size);
+#ifdef VIXL_DEBUG
+    assm->bind(&start_);
+    size_ = size;
+    assert_policy_ = assert_policy;
+    assm->AcquireBuffer();
+#else
+    USE(assert_policy);
+#endif
+  }
+
+  // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
+  explicit CodeBufferCheckScope(Assembler* assm) : assm_(assm) {
+#ifdef VIXL_DEBUG
+    size_ = 0;
+    assert_policy_ = kNoAssert;
+    assm->AcquireBuffer();
+#endif
+  }
+
+  ~CodeBufferCheckScope() {
+#ifdef VIXL_DEBUG
+    assm_->ReleaseBuffer();
+    switch (assert_policy_) {
+      case kNoAssert: break;
+      case kExactSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) == size_);
+        break;
+      case kMaximumSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) <= size_);
+        break;
+      default:
+        VIXL_UNREACHABLE();
+    }
+#endif
+  }
+
+ protected:
+  Assembler* assm_;
+#ifdef VIXL_DEBUG
+  Label start_;
+  size_t size_;
+  AssertPolicy assert_policy_;
+#endif
+};
+
+
+template <typename T>
+void Literal<T>::UpdateValue(T new_value, const Assembler* assembler) {
+  return UpdateValue(new_value, assembler->GetStartAddress<uint8_t*>());
+}
+
+
+template <typename T>
+void Literal<T>::UpdateValue(T high64, T low64, const Assembler* assembler) {
+  return UpdateValue(high64, low64, assembler->GetStartAddress<uint8_t*>());
+}
+
+
+}  // namespace vixl
+
+#endif  // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/qemu/disas/libvixl/a64/constants-a64.h b/qemu/disas/libvixl/vixl/a64/constants-a64.h
index bc1a2c4b9..2caa73af8 100644
--- a/qemu/disas/libvixl/a64/constants-a64.h
+++ b/qemu/disas/libvixl/vixl/a64/constants-a64.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2015, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,14 @@
 namespace vixl {
 
 const unsigned kNumberOfRegisters = 32;
-const unsigned kNumberOfFPRegisters = 32;
+const unsigned kNumberOfVRegisters = 32;
+const unsigned kNumberOfFPRegisters = kNumberOfVRegisters;
+// Callee saved registers are x21-x30(lr).
+const int kNumberOfCalleeSavedRegisters = 10;
+const int kFirstCalleeSavedRegisterIndex = 21;
+// Callee saved FP registers are d8-d15.
+const int kNumberOfCalleeSavedFPRegisters = 8;
+const int kFirstCalleeSavedFPRegisterIndex = 8;
 
 #define REGISTER_CODE_LIST(R)                                                  \
 R(0)  R(1)  R(2)  R(3)  R(4)  R(5)  R(6)  R(7)                                 \
@@ -100,8 +107,10 @@ V_(FPScale, 15, 10, Bits)                                                      \
 V_(ImmLS, 20, 12, SignedBits)                                                  \
 V_(ImmLSUnsigned, 21, 10, Bits)                                                \
 V_(ImmLSPair, 21, 15, SignedBits)                                              \
-V_(SizeLS, 31, 30, Bits)                                                       \
 V_(ImmShiftLS, 12, 12, Bits)                                                   \
+V_(LSOpc, 23, 22, Bits)                                                        \
+V_(LSVector, 26, 26, Bits)                                                     \
+V_(LSSize, 31, 30, Bits)                                                       \
 V_(ImmPrefetchOperation, 4, 0, Bits)                                           \
 V_(PrefetchHint, 4, 3, Bits)                                                   \
 V_(PrefetchTarget, 2, 1, Bits)                                                 \
@@ -116,9 +125,10 @@ V_(ImmHint, 11, 5, Bits)                                                       \
 V_(ImmBarrierDomain, 11, 10, Bits)                                             \
 V_(ImmBarrierType, 9, 8, Bits)                                                 \
                                                                                \
-/* System (MRS, MSR) */                                                        \
+/* System (MRS, MSR, SYS) */                                                   \
 V_(ImmSystemRegister, 19, 5, Bits)                                             \
 V_(SysO0, 19, 19, Bits)                                                        \
+V_(SysOp, 18, 5, Bits)                                                         \
 V_(SysOp1, 18, 16, Bits)                                                       \
 V_(SysOp2, 7, 5, Bits)                                                         \
 V_(CRn, 15, 12, Bits)                                                          \
@@ -130,7 +140,29 @@ V_(LdStXNotExclusive, 23, 23, Bits)                                            \
 V_(LdStXAcquireRelease, 15, 15, Bits)                                          \
 V_(LdStXSizeLog2, 31, 30, Bits)                                                \
 V_(LdStXPair, 21, 21, Bits)                                                    \
-
+                                                                               \
+/* NEON generic fields */                                                      \
+V_(NEONQ, 30, 30, Bits)                                                        \
+V_(NEONSize, 23, 22, Bits)                                                     \
+V_(NEONLSSize, 11, 10, Bits)                                                   \
+V_(NEONS, 12, 12, Bits)                                                        \
+V_(NEONL, 21, 21, Bits)                                                        \
+V_(NEONM, 20, 20, Bits)                                                        \
+V_(NEONH, 11, 11, Bits)                                                        \
+V_(ImmNEONExt, 14, 11, Bits)                                                   \
+V_(ImmNEON5, 20, 16, Bits)                                                     \
+V_(ImmNEON4, 14, 11, Bits)                                                     \
+                                                                               \
+/* NEON Modified Immediate fields */                                           \
+V_(ImmNEONabc, 18, 16, Bits)                                                   \
+V_(ImmNEONdefgh, 9, 5, Bits)                                                   \
+V_(NEONModImmOp, 29, 29, Bits)                                                 \
+V_(NEONCmode, 15, 12, Bits)                                                    \
+                                                                               \
+/* NEON Shift Immediate fields */                                              \
+V_(ImmNEONImmhImmb, 22, 16, Bits)                                              \
+V_(ImmNEONImmh, 22, 19, Bits)                                                  \
+V_(ImmNEONImmb, 18, 16, Bits)
 
 #define SYSTEM_REGISTER_FIELDS_LIST(V_, M_)                                    \
 /* NZCV */                                                                     \
@@ -140,7 +172,6 @@ V_(Z, 30, 30, Bits)                                                            \
 V_(C, 29, 29, Bits)                                                            \
 V_(V, 28, 28, Bits)                                                            \
 M_(NZCV, Flags_mask)                                                           \
-                                                                               \
 /* FPCR */                                                                     \
 V_(AHP, 26, 26, Bits)                                                          \
 V_(DN, 25, 25, Bits)                                                           \
@@ -148,7 +179,6 @@ V_(FZ, 24, 24, Bits)                                                           \
 V_(RMode, 23, 22, Bits)                                                        \
 M_(FPCR, AHP_mask | DN_mask | FZ_mask | RMode_mask)
 
-
 // Fields offsets.
 #define DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, X)                       \
 const int Name##_offset = LowBit;                                              \
@@ -166,22 +196,26 @@ const int ImmPCRel_mask = ImmPCRelLo_mask | ImmPCRelHi_mask;
 
 // Condition codes.
 enum Condition {
-  eq = 0,
-  ne = 1,
-  hs = 2,
-  lo = 3,
-  mi = 4,
-  pl = 5,
-  vs = 6,
-  vc = 7,
-  hi = 8,
-  ls = 9,
-  ge = 10,
-  lt = 11,
-  gt = 12,
-  le = 13,
-  al = 14,
-  nv = 15  // Behaves as always/al.
+  eq = 0,   // Z set            Equal.
+  ne = 1,   // Z clear          Not equal.
+  cs = 2,   // C set            Carry set.
+  cc = 3,   // C clear          Carry clear.
+  mi = 4,   // N set            Negative.
+  pl = 5,   // N clear          Positive or zero.
+  vs = 6,   // V set            Overflow.
+  vc = 7,   // V clear          No overflow.
+  hi = 8,   // C set, Z clear   Unsigned higher.
+  ls = 9,   // C clear or Z set Unsigned lower or same.
+  ge = 10,  // N == V           Greater or equal.
+  lt = 11,  // N != V           Less than.
+  gt = 12,  // Z clear, N == V  Greater than.
+  le = 13,  // Z set or N != V  Less then or equal
+  al = 14,  //                  Always.
+  nv = 15,  // Behaves as always/al.
+
+  // Aliases.
+  hs = cs,  // C set            Unsigned higher or same.
+  lo = cc   // C clear          Unsigned lower.
 };
 
 inline Condition InvertCondition(Condition cond) {
@@ -191,6 +225,11 @@ inline Condition InvertCondition(Condition cond) {
   return static_cast<Condition>(cond ^ 1);
 }
 
+enum FPTrapFlags {
+  EnableTrap   = 1,
+  DisableTrap = 0
+};
+
 enum FlagsUpdate {
   SetFlags   = 1,
   LeaveFlags = 0
@@ -228,7 +267,8 @@ enum Shift {
   LSL = 0x0,
   LSR = 0x1,
   ASR = 0x2,
-  ROR = 0x3
+  ROR = 0x3,
+  MSL = 0x4
 };
 
 enum Extend {
@@ -305,6 +345,32 @@ enum SystemRegister {
           (0x0 << SysOp2_offset)) >> ImmSystemRegister_offset
 };
 
+enum InstructionCacheOp {
+  IVAU = ((0x3 << SysOp1_offset) |
+          (0x7 << CRn_offset) |
+          (0x5 << CRm_offset) |
+          (0x1 << SysOp2_offset)) >> SysOp_offset
+};
+
+enum DataCacheOp {
+  CVAC  = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xa << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  CVAU  = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xb << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  CIVAC = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xe << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  ZVA   = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0x4 << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset
+};
+
 // Instruction enumerations.
 //
 // These are the masks that define a class of instructions, and the list of
@@ -333,6 +399,47 @@ enum GenericInstrField {
   FP64                 = 0x00400000
 };
 
+enum NEONFormatField {
+  NEONFormatFieldMask   = 0x40C00000,
+  NEON_Q                = 0x40000000,
+  NEON_8B               = 0x00000000,
+  NEON_16B              = NEON_8B | NEON_Q,
+  NEON_4H               = 0x00400000,
+  NEON_8H               = NEON_4H | NEON_Q,
+  NEON_2S               = 0x00800000,
+  NEON_4S               = NEON_2S | NEON_Q,
+  NEON_1D               = 0x00C00000,
+  NEON_2D               = 0x00C00000 | NEON_Q
+};
+
+enum NEONFPFormatField {
+  NEONFPFormatFieldMask = 0x40400000,
+  NEON_FP_2S            = FP32,
+  NEON_FP_4S            = FP32 | NEON_Q,
+  NEON_FP_2D            = FP64 | NEON_Q
+};
+
+enum NEONLSFormatField {
+  NEONLSFormatFieldMask = 0x40000C00,
+  LS_NEON_8B            = 0x00000000,
+  LS_NEON_16B           = LS_NEON_8B | NEON_Q,
+  LS_NEON_4H            = 0x00000400,
+  LS_NEON_8H            = LS_NEON_4H | NEON_Q,
+  LS_NEON_2S            = 0x00000800,
+  LS_NEON_4S            = LS_NEON_2S | NEON_Q,
+  LS_NEON_1D            = 0x00000C00,
+  LS_NEON_2D            = LS_NEON_1D | NEON_Q
+};
+
+enum NEONScalarFormatField {
+  NEONScalarFormatFieldMask = 0x00C00000,
+  NEONScalar                = 0x10000000,
+  NEON_B                    = 0x00000000,
+  NEON_H                    = 0x00400000,
+  NEON_S                    = 0x00800000,
+  NEON_D                    = 0x00C00000
+};
+
 // PC relative addressing.
 enum PCRelAddressingOp {
   PCRelAddressingFixed = 0x10000000,
@@ -588,6 +695,13 @@ enum SystemHintOp {
   HINT            = SystemHintFixed | 0x00000000
 };
 
+enum SystemSysOp {
+  SystemSysFixed  = 0xD5080000,
+  SystemSysFMask  = 0xFFF80000,
+  SystemSysMask   = 0xFFF80000,
+  SYS             = SystemSysFixed | 0x00000000
+};
+
 // Exception.
 enum ExceptionOp {
   ExceptionFixed = 0xD4000000,
@@ -640,7 +754,9 @@ enum LoadStorePairAnyOp {
   V(STP, s,   0x04000000),          \
   V(LDP, s,   0x04400000),          \
   V(STP, d,   0x44000000),          \
-  V(LDP, d,   0x44400000)
+  V(LDP, d,   0x44400000),          \
+  V(STP, q,   0x84000000),          \
+  V(LDP, q,   0x84400000)
 
 // Load/store pair (post, pre and offset.)
 enum LoadStorePairOp {
@@ -686,6 +802,7 @@ enum LoadStorePairNonTemporalOp {
   LoadStorePairNonTemporalFixed = 0x28000000,
   LoadStorePairNonTemporalFMask = 0x3B800000,
   LoadStorePairNonTemporalMask  = 0xFFC00000,
+  LoadStorePairNonTemporalLBit = 1 << 22,
   STNP_w = LoadStorePairNonTemporalFixed | STP_w,
   LDNP_w = LoadStorePairNonTemporalFixed | LDP_w,
   STNP_x = LoadStorePairNonTemporalFixed | STP_x,
@@ -693,7 +810,9 @@ enum LoadStorePairNonTemporalOp {
   STNP_s = LoadStorePairNonTemporalFixed | STP_s,
   LDNP_s = LoadStorePairNonTemporalFixed | LDP_s,
   STNP_d = LoadStorePairNonTemporalFixed | STP_d,
-  LDNP_d = LoadStorePairNonTemporalFixed | LDP_d
+  LDNP_d = LoadStorePairNonTemporalFixed | LDP_d,
+  STNP_q = LoadStorePairNonTemporalFixed | STP_q,
+  LDNP_q = LoadStorePairNonTemporalFixed | LDP_q
 };
 
 // Load literal.
@@ -706,7 +825,8 @@ enum LoadLiteralOp {
   LDRSW_x_lit      = LoadLiteralFixed | 0x80000000,
   PRFM_lit         = LoadLiteralFixed | 0xC0000000,
   LDR_s_lit        = LoadLiteralFixed | 0x04000000,
-  LDR_d_lit        = LoadLiteralFixed | 0x44000000
+  LDR_d_lit        = LoadLiteralFixed | 0x44000000,
+  LDR_q_lit        = LoadLiteralFixed | 0x84000000
 };
 
 #define LOAD_STORE_OP_LIST(V)     \
@@ -723,15 +843,21 @@ enum LoadLiteralOp {
   V(LD, RSW, x, 0x80800000),  \
   V(LD, RSB, w, 0x00C00000),  \
   V(LD, RSH, w, 0x40C00000),  \
+  V(ST, R, b,   0x04000000),  \
+  V(ST, R, h,   0x44000000),  \
   V(ST, R, s,   0x84000000),  \
   V(ST, R, d,   0xC4000000),  \
+  V(ST, R, q,   0x04800000),  \
+  V(LD, R, b,   0x04400000),  \
+  V(LD, R, h,   0x44400000),  \
   V(LD, R, s,   0x84400000),  \
-  V(LD, R, d,   0xC4400000)
-
+  V(LD, R, d,   0xC4400000),  \
+  V(LD, R, q,   0x04C00000)
 
 // Load/store (post, pre, offset and unsigned.)
 enum LoadStoreOp {
-  LoadStoreOpMask = 0xC4C00000,
+  LoadStoreMask = 0xC4C00000,
+  LoadStoreVMask = 0x04000000,
   #define LOAD_STORE(A, B, C, D)  \
   A##B##_##C = D
   LOAD_STORE_OP_LIST(LOAD_STORE),
@@ -971,8 +1097,10 @@ enum FPCompareOp {
   FCMP_zero      = FCMP_s_zero,
   FCMPE_s        = FPCompareFixed | 0x00000010,
   FCMPE_d        = FPCompareFixed | FP64 | 0x00000010,
+  FCMPE          = FCMPE_s,
   FCMPE_s_zero   = FPCompareFixed | 0x00000018,
-  FCMPE_d_zero   = FPCompareFixed | FP64 | 0x00000018
+  FCMPE_d_zero   = FPCompareFixed | FP64 | 0x00000018,
+  FCMPE_zero     = FCMPE_s_zero
 };
 
 // Floating point conditional compare.
@@ -1026,6 +1154,10 @@ enum FPDataProcessing1SourceOp {
   FSQRT    = FSQRT_s,
   FCVT_ds  = FPDataProcessing1SourceFixed | 0x00028000,
   FCVT_sd  = FPDataProcessing1SourceFixed | FP64 | 0x00020000,
+  FCVT_hs  = FPDataProcessing1SourceFixed | 0x00038000,
+  FCVT_hd  = FPDataProcessing1SourceFixed | FP64 | 0x00038000,
+  FCVT_sh  = FPDataProcessing1SourceFixed | 0x00C20000,
+  FCVT_dh  = FPDataProcessing1SourceFixed | 0x00C28000,
   FRINTN_s = FPDataProcessing1SourceFixed | 0x00040000,
   FRINTN_d = FPDataProcessing1SourceFixed | FP64 | 0x00040000,
   FRINTN   = FRINTN_s,
@@ -1166,7 +1298,9 @@ enum FPIntegerConvertOp {
   FMOV_ws   = FPIntegerConvertFixed | 0x00060000,
   FMOV_sw   = FPIntegerConvertFixed | 0x00070000,
   FMOV_xd   = FMOV_ws | SixtyFourBits | FP64,
-  FMOV_dx   = FMOV_sw | SixtyFourBits | FP64
+  FMOV_dx   = FMOV_sw | SixtyFourBits | FP64,
+  FMOV_d1_x = FPIntegerConvertFixed | SixtyFourBits | 0x008F0000,
+  FMOV_x_d1 = FPIntegerConvertFixed | SixtyFourBits | 0x008E0000
 };
 
 // Conversion between fixed point and floating point.
@@ -1196,6 +1330,775 @@ enum FPFixedPointConvertOp {
   UCVTF_dx_fixed  = UCVTF_fixed | SixtyFourBits | FP64
 };
 
+// Crypto - two register SHA.
+enum Crypto2RegSHAOp {
+  Crypto2RegSHAFixed = 0x5E280800,
+  Crypto2RegSHAFMask = 0xFF3E0C00
+};
+
+// Crypto - three register SHA.
+enum Crypto3RegSHAOp {
+  Crypto3RegSHAFixed = 0x5E000000,
+  Crypto3RegSHAFMask = 0xFF208C00
+};
+
+// Crypto - AES.
+enum CryptoAESOp {
+  CryptoAESFixed = 0x4E280800,
+  CryptoAESFMask = 0xFF3E0C00
+};
+
+// NEON instructions with two register operands.
+enum NEON2RegMiscOp {
+  NEON2RegMiscFixed = 0x0E200800,
+  NEON2RegMiscFMask = 0x9F3E0C00,
+  NEON2RegMiscMask  = 0xBF3FFC00,
+  NEON2RegMiscUBit  = 0x20000000,
+  NEON_REV64     = NEON2RegMiscFixed | 0x00000000,
+  NEON_REV32     = NEON2RegMiscFixed | 0x20000000,
+  NEON_REV16     = NEON2RegMiscFixed | 0x00001000,
+  NEON_SADDLP    = NEON2RegMiscFixed | 0x00002000,
+  NEON_UADDLP    = NEON_SADDLP | NEON2RegMiscUBit,
+  NEON_SUQADD    = NEON2RegMiscFixed | 0x00003000,
+  NEON_USQADD    = NEON_SUQADD | NEON2RegMiscUBit,
+  NEON_CLS       = NEON2RegMiscFixed | 0x00004000,
+  NEON_CLZ       = NEON2RegMiscFixed | 0x20004000,
+  NEON_CNT       = NEON2RegMiscFixed | 0x00005000,
+  NEON_RBIT_NOT  = NEON2RegMiscFixed | 0x20005000,
+  NEON_SADALP    = NEON2RegMiscFixed | 0x00006000,
+  NEON_UADALP    = NEON_SADALP | NEON2RegMiscUBit,
+  NEON_SQABS     = NEON2RegMiscFixed | 0x00007000,
+  NEON_SQNEG     = NEON2RegMiscFixed | 0x20007000,
+  NEON_CMGT_zero = NEON2RegMiscFixed | 0x00008000,
+  NEON_CMGE_zero = NEON2RegMiscFixed | 0x20008000,
+  NEON_CMEQ_zero = NEON2RegMiscFixed | 0x00009000,
+  NEON_CMLE_zero = NEON2RegMiscFixed | 0x20009000,
+  NEON_CMLT_zero = NEON2RegMiscFixed | 0x0000A000,
+  NEON_ABS       = NEON2RegMiscFixed | 0x0000B000,
+  NEON_NEG       = NEON2RegMiscFixed | 0x2000B000,
+  NEON_XTN       = NEON2RegMiscFixed | 0x00012000,
+  NEON_SQXTUN    = NEON2RegMiscFixed | 0x20012000,
+  NEON_SHLL      = NEON2RegMiscFixed | 0x20013000,
+  NEON_SQXTN     = NEON2RegMiscFixed | 0x00014000,
+  NEON_UQXTN     = NEON_SQXTN | NEON2RegMiscUBit,
+
+  NEON2RegMiscOpcode = 0x0001F000,
+  NEON_RBIT_NOT_opcode = NEON_RBIT_NOT & NEON2RegMiscOpcode,
+  NEON_NEG_opcode = NEON_NEG & NEON2RegMiscOpcode,
+  NEON_XTN_opcode = NEON_XTN & NEON2RegMiscOpcode,
+  NEON_UQXTN_opcode = NEON_UQXTN & NEON2RegMiscOpcode,
+
+  // These instructions use only one bit of the size field. The other bit is
+  // used to distinguish between instructions.
+  NEON2RegMiscFPMask = NEON2RegMiscMask | 0x00800000,
+  NEON_FABS   = NEON2RegMiscFixed | 0x0080F000,
+  NEON_FNEG   = NEON2RegMiscFixed | 0x2080F000,
+  NEON_FCVTN  = NEON2RegMiscFixed | 0x00016000,
+  NEON_FCVTXN = NEON2RegMiscFixed | 0x20016000,
+  NEON_FCVTL  = NEON2RegMiscFixed | 0x00017000,
+  NEON_FRINTN = NEON2RegMiscFixed | 0x00018000,
+  NEON_FRINTA = NEON2RegMiscFixed | 0x20018000,
+  NEON_FRINTP = NEON2RegMiscFixed | 0x00818000,
+  NEON_FRINTM = NEON2RegMiscFixed | 0x00019000,
+  NEON_FRINTX = NEON2RegMiscFixed | 0x20019000,
+  NEON_FRINTZ = NEON2RegMiscFixed | 0x00819000,
+  NEON_FRINTI = NEON2RegMiscFixed | 0x20819000,
+  NEON_FCVTNS = NEON2RegMiscFixed | 0x0001A000,
+  NEON_FCVTNU = NEON_FCVTNS | NEON2RegMiscUBit,
+  NEON_FCVTPS = NEON2RegMiscFixed | 0x0081A000,
+  NEON_FCVTPU = NEON_FCVTPS | NEON2RegMiscUBit,
+  NEON_FCVTMS = NEON2RegMiscFixed | 0x0001B000,
+  NEON_FCVTMU = NEON_FCVTMS | NEON2RegMiscUBit,
+  NEON_FCVTZS = NEON2RegMiscFixed | 0x0081B000,
+  NEON_FCVTZU = NEON_FCVTZS | NEON2RegMiscUBit,
+  NEON_FCVTAS = NEON2RegMiscFixed | 0x0001C000,
+  NEON_FCVTAU = NEON_FCVTAS | NEON2RegMiscUBit,
+  NEON_FSQRT  = NEON2RegMiscFixed | 0x2081F000,
+  NEON_SCVTF  = NEON2RegMiscFixed | 0x0001D000,
+  NEON_UCVTF  = NEON_SCVTF | NEON2RegMiscUBit,
+  NEON_URSQRTE = NEON2RegMiscFixed | 0x2081C000,
+  NEON_URECPE  = NEON2RegMiscFixed | 0x0081C000,
+  NEON_FRSQRTE = NEON2RegMiscFixed | 0x2081D000,
+  NEON_FRECPE  = NEON2RegMiscFixed | 0x0081D000,
+  NEON_FCMGT_zero = NEON2RegMiscFixed | 0x0080C000,
+  NEON_FCMGE_zero = NEON2RegMiscFixed | 0x2080C000,
+  NEON_FCMEQ_zero = NEON2RegMiscFixed | 0x0080D000,
+  NEON_FCMLE_zero = NEON2RegMiscFixed | 0x2080D000,
+  NEON_FCMLT_zero = NEON2RegMiscFixed | 0x0080E000,
+
+  NEON_FCVTL_opcode = NEON_FCVTL & NEON2RegMiscOpcode,
+  NEON_FCVTN_opcode = NEON_FCVTN & NEON2RegMiscOpcode
+};
+
+// NEON instructions with three same-type operands.
+enum NEON3SameOp {
+  NEON3SameFixed = 0x0E200400,
+  NEON3SameFMask = 0x9F200400,
+  NEON3SameMask = 0xBF20FC00,
+  NEON3SameUBit = 0x20000000,
+  NEON_ADD    = NEON3SameFixed | 0x00008000,
+  NEON_ADDP   = NEON3SameFixed | 0x0000B800,
+  NEON_SHADD  = NEON3SameFixed | 0x00000000,
+  NEON_SHSUB  = NEON3SameFixed | 0x00002000,
+  NEON_SRHADD = NEON3SameFixed | 0x00001000,
+  NEON_CMEQ   = NEON3SameFixed | NEON3SameUBit | 0x00008800,
+  NEON_CMGE   = NEON3SameFixed | 0x00003800,
+  NEON_CMGT   = NEON3SameFixed | 0x00003000,
+  NEON_CMHI   = NEON3SameFixed | NEON3SameUBit | NEON_CMGT,
+  NEON_CMHS   = NEON3SameFixed | NEON3SameUBit | NEON_CMGE,
+  NEON_CMTST  = NEON3SameFixed | 0x00008800,
+  NEON_MLA    = NEON3SameFixed | 0x00009000,
+  NEON_MLS    = NEON3SameFixed | 0x20009000,
+  NEON_MUL    = NEON3SameFixed | 0x00009800,
+  NEON_PMUL   = NEON3SameFixed | 0x20009800,
+  NEON_SRSHL  = NEON3SameFixed | 0x00005000,
+  NEON_SQSHL  = NEON3SameFixed | 0x00004800,
+  NEON_SQRSHL = NEON3SameFixed | 0x00005800,
+  NEON_SSHL   = NEON3SameFixed | 0x00004000,
+  NEON_SMAX   = NEON3SameFixed | 0x00006000,
+  NEON_SMAXP  = NEON3SameFixed | 0x0000A000,
+  NEON_SMIN   = NEON3SameFixed | 0x00006800,
+  NEON_SMINP  = NEON3SameFixed | 0x0000A800,
+  NEON_SABD   = NEON3SameFixed | 0x00007000,
+  NEON_SABA   = NEON3SameFixed | 0x00007800,
+  NEON_UABD   = NEON3SameFixed | NEON3SameUBit | NEON_SABD,
+  NEON_UABA   = NEON3SameFixed | NEON3SameUBit | NEON_SABA,
+  NEON_SQADD  = NEON3SameFixed | 0x00000800,
+  NEON_SQSUB  = NEON3SameFixed | 0x00002800,
+  NEON_SUB    = NEON3SameFixed | NEON3SameUBit | 0x00008000,
+  NEON_UHADD  = NEON3SameFixed | NEON3SameUBit | NEON_SHADD,
+  NEON_UHSUB  = NEON3SameFixed | NEON3SameUBit | NEON_SHSUB,
+  NEON_URHADD = NEON3SameFixed | NEON3SameUBit | NEON_SRHADD,
+  NEON_UMAX   = NEON3SameFixed | NEON3SameUBit | NEON_SMAX,
+  NEON_UMAXP  = NEON3SameFixed | NEON3SameUBit | NEON_SMAXP,
+  NEON_UMIN   = NEON3SameFixed | NEON3SameUBit | NEON_SMIN,
+  NEON_UMINP  = NEON3SameFixed | NEON3SameUBit | NEON_SMINP,
+  NEON_URSHL  = NEON3SameFixed | NEON3SameUBit | NEON_SRSHL,
+  NEON_UQADD  = NEON3SameFixed | NEON3SameUBit | NEON_SQADD,
+  NEON_UQRSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQRSHL,
+  NEON_UQSHL  = NEON3SameFixed | NEON3SameUBit | NEON_SQSHL,
+  NEON_UQSUB  = NEON3SameFixed | NEON3SameUBit | NEON_SQSUB,
+  NEON_USHL   = NEON3SameFixed | NEON3SameUBit | NEON_SSHL,
+  NEON_SQDMULH  = NEON3SameFixed | 0x0000B000,
+  NEON_SQRDMULH = NEON3SameFixed | 0x2000B000,
+
+  // NEON floating point instructions with three same-type operands.
+  NEON3SameFPFixed = NEON3SameFixed | 0x0000C000,
+  NEON3SameFPFMask = NEON3SameFMask | 0x0000C000,
+  NEON3SameFPMask = NEON3SameMask | 0x00800000,
+  NEON_FADD    = NEON3SameFixed | 0x0000D000,
+  NEON_FSUB    = NEON3SameFixed | 0x0080D000,
+  NEON_FMUL    = NEON3SameFixed | 0x2000D800,
+  NEON_FDIV    = NEON3SameFixed | 0x2000F800,
+  NEON_FMAX    = NEON3SameFixed | 0x0000F000,
+  NEON_FMAXNM  = NEON3SameFixed | 0x0000C000,
+  NEON_FMAXP   = NEON3SameFixed | 0x2000F000,
+  NEON_FMAXNMP = NEON3SameFixed | 0x2000C000,
+  NEON_FMIN    = NEON3SameFixed | 0x0080F000,
+  NEON_FMINNM  = NEON3SameFixed | 0x0080C000,
+  NEON_FMINP   = NEON3SameFixed | 0x2080F000,
+  NEON_FMINNMP = NEON3SameFixed | 0x2080C000,
+  NEON_FMLA    = NEON3SameFixed | 0x0000C800,
+  NEON_FMLS    = NEON3SameFixed | 0x0080C800,
+  NEON_FMULX   = NEON3SameFixed | 0x0000D800,
+  NEON_FRECPS  = NEON3SameFixed | 0x0000F800,
+  NEON_FRSQRTS = NEON3SameFixed | 0x0080F800,
+  NEON_FABD    = NEON3SameFixed | 0x2080D000,
+  NEON_FADDP   = NEON3SameFixed | 0x2000D000,
+  NEON_FCMEQ   = NEON3SameFixed | 0x0000E000,
+  NEON_FCMGE   = NEON3SameFixed | 0x2000E000,
+  NEON_FCMGT   = NEON3SameFixed | 0x2080E000,
+  NEON_FACGE   = NEON3SameFixed | 0x2000E800,
+  NEON_FACGT   = NEON3SameFixed | 0x2080E800,
+
+  // NEON logical instructions with three same-type operands.
+  NEON3SameLogicalFixed = NEON3SameFixed | 0x00001800,
+  NEON3SameLogicalFMask = NEON3SameFMask | 0x0000F800,
+  NEON3SameLogicalMask = 0xBFE0FC00,
+  NEON3SameLogicalFormatMask = NEON_Q,
+  NEON_AND = NEON3SameLogicalFixed | 0x00000000,
+  NEON_ORR = NEON3SameLogicalFixed | 0x00A00000,
+  NEON_ORN = NEON3SameLogicalFixed | 0x00C00000,
+  NEON_EOR = NEON3SameLogicalFixed | 0x20000000,
+  NEON_BIC = NEON3SameLogicalFixed | 0x00400000,
+  NEON_BIF = NEON3SameLogicalFixed | 0x20C00000,
+  NEON_BIT = NEON3SameLogicalFixed | 0x20800000,
+  NEON_BSL = NEON3SameLogicalFixed | 0x20400000
+};
+
+// NEON instructions with three different-type operands.
+enum NEON3DifferentOp {
+  NEON3DifferentFixed = 0x0E200000,
+  NEON3DifferentFMask = 0x9F200C00,
+  NEON3DifferentMask  = 0xFF20FC00,
+  NEON_ADDHN    = NEON3DifferentFixed | 0x00004000,
+  NEON_ADDHN2   = NEON_ADDHN | NEON_Q,
+  NEON_PMULL    = NEON3DifferentFixed | 0x0000E000,
+  NEON_PMULL2   = NEON_PMULL | NEON_Q,
+  NEON_RADDHN   = NEON3DifferentFixed | 0x20004000,
+  NEON_RADDHN2  = NEON_RADDHN | NEON_Q,
+  NEON_RSUBHN   = NEON3DifferentFixed | 0x20006000,
+  NEON_RSUBHN2  = NEON_RSUBHN | NEON_Q,
+  NEON_SABAL    = NEON3DifferentFixed | 0x00005000,
+  NEON_SABAL2   = NEON_SABAL | NEON_Q,
+  NEON_SABDL    = NEON3DifferentFixed | 0x00007000,
+  NEON_SABDL2   = NEON_SABDL | NEON_Q,
+  NEON_SADDL    = NEON3DifferentFixed | 0x00000000,
+  NEON_SADDL2   = NEON_SADDL | NEON_Q,
+  NEON_SADDW    = NEON3DifferentFixed | 0x00001000,
+  NEON_SADDW2   = NEON_SADDW | NEON_Q,
+  NEON_SMLAL    = NEON3DifferentFixed | 0x00008000,
+  NEON_SMLAL2   = NEON_SMLAL | NEON_Q,
+  NEON_SMLSL    = NEON3DifferentFixed | 0x0000A000,
+  NEON_SMLSL2   = NEON_SMLSL | NEON_Q,
+  NEON_SMULL    = NEON3DifferentFixed | 0x0000C000,
+  NEON_SMULL2   = NEON_SMULL | NEON_Q,
+  NEON_SSUBL    = NEON3DifferentFixed | 0x00002000,
+  NEON_SSUBL2   = NEON_SSUBL | NEON_Q,
+  NEON_SSUBW    = NEON3DifferentFixed | 0x00003000,
+  NEON_SSUBW2   = NEON_SSUBW | NEON_Q,
+  NEON_SQDMLAL  = NEON3DifferentFixed | 0x00009000,
+  NEON_SQDMLAL2 = NEON_SQDMLAL | NEON_Q,
+  NEON_SQDMLSL  = NEON3DifferentFixed | 0x0000B000,
+  NEON_SQDMLSL2 = NEON_SQDMLSL | NEON_Q,
+  NEON_SQDMULL  = NEON3DifferentFixed | 0x0000D000,
+  NEON_SQDMULL2 = NEON_SQDMULL | NEON_Q,
+  NEON_SUBHN    = NEON3DifferentFixed | 0x00006000,
+  NEON_SUBHN2   = NEON_SUBHN | NEON_Q,
+  NEON_UABAL    = NEON_SABAL | NEON3SameUBit,
+  NEON_UABAL2   = NEON_UABAL | NEON_Q,
+  NEON_UABDL    = NEON_SABDL | NEON3SameUBit,
+  NEON_UABDL2   = NEON_UABDL | NEON_Q,
+  NEON_UADDL    = NEON_SADDL | NEON3SameUBit,
+  NEON_UADDL2   = NEON_UADDL | NEON_Q,
+  NEON_UADDW    = NEON_SADDW | NEON3SameUBit,
+  NEON_UADDW2   = NEON_UADDW | NEON_Q,
+  NEON_UMLAL    = NEON_SMLAL | NEON3SameUBit,
+  NEON_UMLAL2   = NEON_UMLAL | NEON_Q,
+  NEON_UMLSL    = NEON_SMLSL | NEON3SameUBit,
+  NEON_UMLSL2   = NEON_UMLSL | NEON_Q,
+  NEON_UMULL    = NEON_SMULL | NEON3SameUBit,
+  NEON_UMULL2   = NEON_UMULL | NEON_Q,
+  NEON_USUBL    = NEON_SSUBL | NEON3SameUBit,
+  NEON_USUBL2   = NEON_USUBL | NEON_Q,
+  NEON_USUBW    = NEON_SSUBW | NEON3SameUBit,
+  NEON_USUBW2   = NEON_USUBW | NEON_Q
+};
+
+// NEON instructions operating across vectors.
+enum NEONAcrossLanesOp {
+  NEONAcrossLanesFixed = 0x0E300800,
+  NEONAcrossLanesFMask = 0x9F3E0C00,
+  NEONAcrossLanesMask  = 0xBF3FFC00,
+  NEON_ADDV   = NEONAcrossLanesFixed | 0x0001B000,
+  NEON_SADDLV = NEONAcrossLanesFixed | 0x00003000,
+  NEON_UADDLV = NEONAcrossLanesFixed | 0x20003000,
+  NEON_SMAXV  = NEONAcrossLanesFixed | 0x0000A000,
+  NEON_SMINV  = NEONAcrossLanesFixed | 0x0001A000,
+  NEON_UMAXV  = NEONAcrossLanesFixed | 0x2000A000,
+  NEON_UMINV  = NEONAcrossLanesFixed | 0x2001A000,
+
+  // NEON floating point across instructions.
+  NEONAcrossLanesFPFixed = NEONAcrossLanesFixed | 0x0000C000,
+  NEONAcrossLanesFPFMask = NEONAcrossLanesFMask | 0x0000C000,
+  NEONAcrossLanesFPMask  = NEONAcrossLanesMask  | 0x00800000,
+
+  NEON_FMAXV   = NEONAcrossLanesFPFixed | 0x2000F000,
+  NEON_FMINV   = NEONAcrossLanesFPFixed | 0x2080F000,
+  NEON_FMAXNMV = NEONAcrossLanesFPFixed | 0x2000C000,
+  NEON_FMINNMV = NEONAcrossLanesFPFixed | 0x2080C000
+};
+
+// NEON instructions with indexed element operand.
+enum NEONByIndexedElementOp {
+  NEONByIndexedElementFixed = 0x0F000000,
+  NEONByIndexedElementFMask = 0x9F000400,
+  NEONByIndexedElementMask  = 0xBF00F400,
+  NEON_MUL_byelement   = NEONByIndexedElementFixed | 0x00008000,
+  NEON_MLA_byelement   = NEONByIndexedElementFixed | 0x20000000,
+  NEON_MLS_byelement   = NEONByIndexedElementFixed | 0x20004000,
+  NEON_SMULL_byelement = NEONByIndexedElementFixed | 0x0000A000,
+  NEON_SMLAL_byelement = NEONByIndexedElementFixed | 0x00002000,
+  NEON_SMLSL_byelement = NEONByIndexedElementFixed | 0x00006000,
+  NEON_UMULL_byelement = NEONByIndexedElementFixed | 0x2000A000,
+  NEON_UMLAL_byelement = NEONByIndexedElementFixed | 0x20002000,
+  NEON_UMLSL_byelement = NEONByIndexedElementFixed | 0x20006000,
+  NEON_SQDMULL_byelement = NEONByIndexedElementFixed | 0x0000B000,
+  NEON_SQDMLAL_byelement = NEONByIndexedElementFixed | 0x00003000,
+  NEON_SQDMLSL_byelement = NEONByIndexedElementFixed | 0x00007000,
+  NEON_SQDMULH_byelement  = NEONByIndexedElementFixed | 0x0000C000,
+  NEON_SQRDMULH_byelement = NEONByIndexedElementFixed | 0x0000D000,
+
+  // Floating point instructions.
+  NEONByIndexedElementFPFixed = NEONByIndexedElementFixed | 0x00800000,
+  NEONByIndexedElementFPMask = NEONByIndexedElementMask | 0x00800000,
+  NEON_FMLA_byelement  = NEONByIndexedElementFPFixed | 0x00001000,
+  NEON_FMLS_byelement  = NEONByIndexedElementFPFixed | 0x00005000,
+  NEON_FMUL_byelement  = NEONByIndexedElementFPFixed | 0x00009000,
+  NEON_FMULX_byelement = NEONByIndexedElementFPFixed | 0x20009000
+};
+
+// NEON register copy.
+enum NEONCopyOp {
+  NEONCopyFixed = 0x0E000400,
+  NEONCopyFMask = 0x9FE08400,
+  NEONCopyMask  = 0x3FE08400,
+  NEONCopyInsElementMask = NEONCopyMask | 0x40000000,
+  NEONCopyInsGeneralMask = NEONCopyMask | 0x40007800,
+  NEONCopyDupElementMask = NEONCopyMask | 0x20007800,
+  NEONCopyDupGeneralMask = NEONCopyDupElementMask,
+  NEONCopyUmovMask       = NEONCopyMask | 0x20007800,
+  NEONCopySmovMask       = NEONCopyMask | 0x20007800,
+  NEON_INS_ELEMENT       = NEONCopyFixed | 0x60000000,
+  NEON_INS_GENERAL       = NEONCopyFixed | 0x40001800,
+  NEON_DUP_ELEMENT       = NEONCopyFixed | 0x00000000,
+  NEON_DUP_GENERAL       = NEONCopyFixed | 0x00000800,
+  NEON_SMOV              = NEONCopyFixed | 0x00002800,
+  NEON_UMOV              = NEONCopyFixed | 0x00003800
+};
+
+// NEON extract.
+enum NEONExtractOp {
+  NEONExtractFixed = 0x2E000000,
+  NEONExtractFMask = 0xBF208400,
+  NEONExtractMask = 0xBFE08400,
+  NEON_EXT = NEONExtractFixed | 0x00000000
+};
+
+enum NEONLoadStoreMultiOp {
+  NEONLoadStoreMultiL    = 0x00400000,
+  NEONLoadStoreMulti1_1v = 0x00007000,
+  NEONLoadStoreMulti1_2v = 0x0000A000,
+  NEONLoadStoreMulti1_3v = 0x00006000,
+  NEONLoadStoreMulti1_4v = 0x00002000,
+  NEONLoadStoreMulti2    = 0x00008000,
+  NEONLoadStoreMulti3    = 0x00004000,
+  NEONLoadStoreMulti4    = 0x00000000
+};
+
+// NEON load/store multiple structures.
+enum NEONLoadStoreMultiStructOp {
+  NEONLoadStoreMultiStructFixed = 0x0C000000,
+  NEONLoadStoreMultiStructFMask = 0xBFBF0000,
+  NEONLoadStoreMultiStructMask  = 0xBFFFF000,
+  NEONLoadStoreMultiStructStore = NEONLoadStoreMultiStructFixed,
+  NEONLoadStoreMultiStructLoad  = NEONLoadStoreMultiStructFixed |
+                                  NEONLoadStoreMultiL,
+  NEON_LD1_1v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_1v,
+  NEON_LD1_2v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_2v,
+  NEON_LD1_3v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_3v,
+  NEON_LD1_4v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_4v,
+  NEON_LD2    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti2,
+  NEON_LD3    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti3,
+  NEON_LD4    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti4,
+  NEON_ST1_1v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_1v,
+  NEON_ST1_2v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_2v,
+  NEON_ST1_3v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_3v,
+  NEON_ST1_4v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_4v,
+  NEON_ST2    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti2,
+  NEON_ST3    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti3,
+  NEON_ST4    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti4
+};
+
+// NEON load/store multiple structures with post-index addressing.
+enum NEONLoadStoreMultiStructPostIndexOp {
+  NEONLoadStoreMultiStructPostIndexFixed = 0x0C800000,
+  NEONLoadStoreMultiStructPostIndexFMask = 0xBFA00000,
+  NEONLoadStoreMultiStructPostIndexMask  = 0xBFE0F000,
+  NEONLoadStoreMultiStructPostIndex = 0x00800000,
+  NEON_LD1_1v_post = NEON_LD1_1v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_2v_post = NEON_LD1_2v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_3v_post = NEON_LD1_3v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_4v_post = NEON_LD1_4v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD2_post = NEON_LD2 | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD3_post = NEON_LD3 | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD4_post = NEON_LD4 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_1v_post = NEON_ST1_1v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_2v_post = NEON_ST1_2v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_3v_post = NEON_ST1_3v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_4v_post = NEON_ST1_4v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST2_post = NEON_ST2 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST3_post = NEON_ST3 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST4_post = NEON_ST4 | NEONLoadStoreMultiStructPostIndex
+};
+
+enum NEONLoadStoreSingleOp {
+  NEONLoadStoreSingle1        = 0x00000000,
+  NEONLoadStoreSingle2        = 0x00200000,
+  NEONLoadStoreSingle3        = 0x00002000,
+  NEONLoadStoreSingle4        = 0x00202000,
+  NEONLoadStoreSingleL        = 0x00400000,
+  NEONLoadStoreSingle_b       = 0x00000000,
+  NEONLoadStoreSingle_h       = 0x00004000,
+  NEONLoadStoreSingle_s       = 0x00008000,
+  NEONLoadStoreSingle_d       = 0x00008400,
+  NEONLoadStoreSingleAllLanes = 0x0000C000,
+  NEONLoadStoreSingleLenMask  = 0x00202000
+};
+
+// NEON load/store single structure.
+enum NEONLoadStoreSingleStructOp {
+  NEONLoadStoreSingleStructFixed = 0x0D000000,
+  NEONLoadStoreSingleStructFMask = 0xBF9F0000,
+  NEONLoadStoreSingleStructMask  = 0xBFFFE000,
+  NEONLoadStoreSingleStructStore = NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructLoad  = NEONLoadStoreSingleStructFixed |
+                                   NEONLoadStoreSingleL,
+  NEONLoadStoreSingleStructLoad1 = NEONLoadStoreSingle1 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad2 = NEONLoadStoreSingle2 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad3 = NEONLoadStoreSingle3 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad4 = NEONLoadStoreSingle4 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructStore1 = NEONLoadStoreSingle1 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore2 = NEONLoadStoreSingle2 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore3 = NEONLoadStoreSingle3 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore4 = NEONLoadStoreSingle4 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEON_LD1_b = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_b,
+  NEON_LD1_h = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_h,
+  NEON_LD1_s = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_s,
+  NEON_LD1_d = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_d,
+  NEON_LD1R  = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingleAllLanes,
+  NEON_ST1_b = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_b,
+  NEON_ST1_h = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_h,
+  NEON_ST1_s = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_s,
+  NEON_ST1_d = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_d,
+
+  NEON_LD2_b = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_b,
+  NEON_LD2_h = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_h,
+  NEON_LD2_s = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_s,
+  NEON_LD2_d = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_d,
+  NEON_LD2R  = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingleAllLanes,
+  NEON_ST2_b = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_b,
+  NEON_ST2_h = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_h,
+  NEON_ST2_s = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_s,
+  NEON_ST2_d = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_d,
+
+  NEON_LD3_b = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_b,
+  NEON_LD3_h = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_h,
+  NEON_LD3_s = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_s,
+  NEON_LD3_d = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_d,
+  NEON_LD3R  = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingleAllLanes,
+  NEON_ST3_b = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_b,
+  NEON_ST3_h = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_h,
+  NEON_ST3_s = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_s,
+  NEON_ST3_d = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_d,
+
+  NEON_LD4_b = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_b,
+  NEON_LD4_h = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_h,
+  NEON_LD4_s = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_s,
+  NEON_LD4_d = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_d,
+  NEON_LD4R  = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingleAllLanes,
+  NEON_ST4_b = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_b,
+  NEON_ST4_h = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_h,
+  NEON_ST4_s = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_s,
+  NEON_ST4_d = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_d
+};
+
+// NEON load/store single structure with post-index addressing.
+enum NEONLoadStoreSingleStructPostIndexOp {
+  NEONLoadStoreSingleStructPostIndexFixed = 0x0D800000,
+  NEONLoadStoreSingleStructPostIndexFMask = 0xBF800000,
+  NEONLoadStoreSingleStructPostIndexMask  = 0xBFE0E000,
+  NEONLoadStoreSingleStructPostIndex = 0x00800000,
+  NEON_LD1_b_post = NEON_LD1_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_h_post = NEON_LD1_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_s_post = NEON_LD1_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_d_post = NEON_LD1_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1R_post  = NEON_LD1R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_b_post = NEON_ST1_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_h_post = NEON_ST1_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_s_post = NEON_ST1_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_d_post = NEON_ST1_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD2_b_post = NEON_LD2_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_h_post = NEON_LD2_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_s_post = NEON_LD2_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_d_post = NEON_LD2_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2R_post  = NEON_LD2R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_b_post = NEON_ST2_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_h_post = NEON_ST2_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_s_post = NEON_ST2_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_d_post = NEON_ST2_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD3_b_post = NEON_LD3_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_h_post = NEON_LD3_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_s_post = NEON_LD3_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_d_post = NEON_LD3_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3R_post  = NEON_LD3R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_b_post = NEON_ST3_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_h_post = NEON_ST3_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_s_post = NEON_ST3_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_d_post = NEON_ST3_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD4_b_post = NEON_LD4_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_h_post = NEON_LD4_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_s_post = NEON_LD4_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_d_post = NEON_LD4_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4R_post  = NEON_LD4R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_b_post = NEON_ST4_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_h_post = NEON_ST4_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_s_post = NEON_ST4_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_d_post = NEON_ST4_d | NEONLoadStoreSingleStructPostIndex
+};
+
+// NEON modified immediate.
+enum NEONModifiedImmediateOp {
+  NEONModifiedImmediateFixed = 0x0F000400,
+  NEONModifiedImmediateFMask = 0x9FF80400,
+  NEONModifiedImmediateOpBit = 0x20000000,
+  NEONModifiedImmediate_MOVI = NEONModifiedImmediateFixed | 0x00000000,
+  NEONModifiedImmediate_MVNI = NEONModifiedImmediateFixed | 0x20000000,
+  NEONModifiedImmediate_ORR  = NEONModifiedImmediateFixed | 0x00001000,
+  NEONModifiedImmediate_BIC  = NEONModifiedImmediateFixed | 0x20001000
+};
+
+// NEON shift immediate.
+enum NEONShiftImmediateOp {
+  NEONShiftImmediateFixed = 0x0F000400,
+  NEONShiftImmediateFMask = 0x9F800400,
+  NEONShiftImmediateMask  = 0xBF80FC00,
+  NEONShiftImmediateUBit  = 0x20000000,
+  NEON_SHL      = NEONShiftImmediateFixed | 0x00005000,
+  NEON_SSHLL    = NEONShiftImmediateFixed | 0x0000A000,
+  NEON_USHLL    = NEONShiftImmediateFixed | 0x2000A000,
+  NEON_SLI      = NEONShiftImmediateFixed | 0x20005000,
+  NEON_SRI      = NEONShiftImmediateFixed | 0x20004000,
+  NEON_SHRN     = NEONShiftImmediateFixed | 0x00008000,
+  NEON_RSHRN    = NEONShiftImmediateFixed | 0x00008800,
+  NEON_UQSHRN   = NEONShiftImmediateFixed | 0x20009000,
+  NEON_UQRSHRN  = NEONShiftImmediateFixed | 0x20009800,
+  NEON_SQSHRN   = NEONShiftImmediateFixed | 0x00009000,
+  NEON_SQRSHRN  = NEONShiftImmediateFixed | 0x00009800,
+  NEON_SQSHRUN  = NEONShiftImmediateFixed | 0x20008000,
+  NEON_SQRSHRUN = NEONShiftImmediateFixed | 0x20008800,
+  NEON_SSHR     = NEONShiftImmediateFixed | 0x00000000,
+  NEON_SRSHR    = NEONShiftImmediateFixed | 0x00002000,
+  NEON_USHR     = NEONShiftImmediateFixed | 0x20000000,
+  NEON_URSHR    = NEONShiftImmediateFixed | 0x20002000,
+  NEON_SSRA     = NEONShiftImmediateFixed | 0x00001000,
+  NEON_SRSRA    = NEONShiftImmediateFixed | 0x00003000,
+  NEON_USRA     = NEONShiftImmediateFixed | 0x20001000,
+  NEON_URSRA    = NEONShiftImmediateFixed | 0x20003000,
+  NEON_SQSHLU   = NEONShiftImmediateFixed | 0x20006000,
+  NEON_SCVTF_imm = NEONShiftImmediateFixed | 0x0000E000,
+  NEON_UCVTF_imm = NEONShiftImmediateFixed | 0x2000E000,
+  NEON_FCVTZS_imm = NEONShiftImmediateFixed | 0x0000F800,
+  NEON_FCVTZU_imm = NEONShiftImmediateFixed | 0x2000F800,
+  NEON_SQSHL_imm = NEONShiftImmediateFixed | 0x00007000,
+  NEON_UQSHL_imm = NEONShiftImmediateFixed | 0x20007000
+};
+
+// NEON table.
+enum NEONTableOp {
+  NEONTableFixed = 0x0E000000,
+  NEONTableFMask = 0xBF208C00,
+  NEONTableExt   = 0x00001000,
+  NEONTableMask  = 0xBF20FC00,
+  NEON_TBL_1v    = NEONTableFixed | 0x00000000,
+  NEON_TBL_2v    = NEONTableFixed | 0x00002000,
+  NEON_TBL_3v    = NEONTableFixed | 0x00004000,
+  NEON_TBL_4v    = NEONTableFixed | 0x00006000,
+  NEON_TBX_1v    = NEON_TBL_1v | NEONTableExt,
+  NEON_TBX_2v    = NEON_TBL_2v | NEONTableExt,
+  NEON_TBX_3v    = NEON_TBL_3v | NEONTableExt,
+  NEON_TBX_4v    = NEON_TBL_4v | NEONTableExt
+};
+
+// NEON perm.
+enum NEONPermOp {
+  NEONPermFixed = 0x0E000800,
+  NEONPermFMask = 0xBF208C00,
+  NEONPermMask  = 0x3F20FC00,
+  NEON_UZP1 = NEONPermFixed | 0x00001000,
+  NEON_TRN1 = NEONPermFixed | 0x00002000,
+  NEON_ZIP1 = NEONPermFixed | 0x00003000,
+  NEON_UZP2 = NEONPermFixed | 0x00005000,
+  NEON_TRN2 = NEONPermFixed | 0x00006000,
+  NEON_ZIP2 = NEONPermFixed | 0x00007000
+};
+
+// NEON scalar instructions with two register operands.
+enum NEONScalar2RegMiscOp {
+  NEONScalar2RegMiscFixed = 0x5E200800,
+  NEONScalar2RegMiscFMask = 0xDF3E0C00,
+  NEONScalar2RegMiscMask = NEON_Q | NEONScalar | NEON2RegMiscMask,
+  NEON_CMGT_zero_scalar = NEON_Q | NEONScalar | NEON_CMGT_zero,
+  NEON_CMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_CMEQ_zero,
+  NEON_CMLT_zero_scalar = NEON_Q | NEONScalar | NEON_CMLT_zero,
+  NEON_CMGE_zero_scalar = NEON_Q | NEONScalar | NEON_CMGE_zero,
+  NEON_CMLE_zero_scalar = NEON_Q | NEONScalar | NEON_CMLE_zero,
+  NEON_ABS_scalar       = NEON_Q | NEONScalar | NEON_ABS,
+  NEON_SQABS_scalar     = NEON_Q | NEONScalar | NEON_SQABS,
+  NEON_NEG_scalar       = NEON_Q | NEONScalar | NEON_NEG,
+  NEON_SQNEG_scalar     = NEON_Q | NEONScalar | NEON_SQNEG,
+  NEON_SQXTN_scalar     = NEON_Q | NEONScalar | NEON_SQXTN,
+  NEON_UQXTN_scalar     = NEON_Q | NEONScalar | NEON_UQXTN,
+  NEON_SQXTUN_scalar    = NEON_Q | NEONScalar | NEON_SQXTUN,
+  NEON_SUQADD_scalar    = NEON_Q | NEONScalar | NEON_SUQADD,
+  NEON_USQADD_scalar    = NEON_Q | NEONScalar | NEON_USQADD,
+
+  NEONScalar2RegMiscOpcode = NEON2RegMiscOpcode,
+  NEON_NEG_scalar_opcode = NEON_NEG_scalar & NEONScalar2RegMiscOpcode,
+
+  NEONScalar2RegMiscFPMask  = NEONScalar2RegMiscMask | 0x00800000,
+  NEON_FRSQRTE_scalar    = NEON_Q | NEONScalar | NEON_FRSQRTE,
+  NEON_FRECPE_scalar     = NEON_Q | NEONScalar | NEON_FRECPE,
+  NEON_SCVTF_scalar      = NEON_Q | NEONScalar | NEON_SCVTF,
+  NEON_UCVTF_scalar      = NEON_Q | NEONScalar | NEON_UCVTF,
+  NEON_FCMGT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGT_zero,
+  NEON_FCMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_FCMEQ_zero,
+  NEON_FCMLT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLT_zero,
+  NEON_FCMGE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGE_zero,
+  NEON_FCMLE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLE_zero,
+  NEON_FRECPX_scalar     = NEONScalar2RegMiscFixed | 0x0081F000,
+  NEON_FCVTNS_scalar     = NEON_Q | NEONScalar | NEON_FCVTNS,
+  NEON_FCVTNU_scalar     = NEON_Q | NEONScalar | NEON_FCVTNU,
+  NEON_FCVTPS_scalar     = NEON_Q | NEONScalar | NEON_FCVTPS,
+  NEON_FCVTPU_scalar     = NEON_Q | NEONScalar | NEON_FCVTPU,
+  NEON_FCVTMS_scalar     = NEON_Q | NEONScalar | NEON_FCVTMS,
+  NEON_FCVTMU_scalar     = NEON_Q | NEONScalar | NEON_FCVTMU,
+  NEON_FCVTZS_scalar     = NEON_Q | NEONScalar | NEON_FCVTZS,
+  NEON_FCVTZU_scalar     = NEON_Q | NEONScalar | NEON_FCVTZU,
+  NEON_FCVTAS_scalar     = NEON_Q | NEONScalar | NEON_FCVTAS,
+  NEON_FCVTAU_scalar     = NEON_Q | NEONScalar | NEON_FCVTAU,
+  NEON_FCVTXN_scalar     = NEON_Q | NEONScalar | NEON_FCVTXN
+};
+
+// NEON scalar instructions with three same-type operands.
+enum NEONScalar3SameOp {
+  NEONScalar3SameFixed = 0x5E200400,
+  NEONScalar3SameFMask = 0xDF200400,
+  NEONScalar3SameMask  = 0xFF20FC00,
+  NEON_ADD_scalar    = NEON_Q | NEONScalar | NEON_ADD,
+  NEON_CMEQ_scalar   = NEON_Q | NEONScalar | NEON_CMEQ,
+  NEON_CMGE_scalar   = NEON_Q | NEONScalar | NEON_CMGE,
+  NEON_CMGT_scalar   = NEON_Q | NEONScalar | NEON_CMGT,
+  NEON_CMHI_scalar   = NEON_Q | NEONScalar | NEON_CMHI,
+  NEON_CMHS_scalar   = NEON_Q | NEONScalar | NEON_CMHS,
+  NEON_CMTST_scalar  = NEON_Q | NEONScalar | NEON_CMTST,
+  NEON_SUB_scalar    = NEON_Q | NEONScalar | NEON_SUB,
+  NEON_UQADD_scalar  = NEON_Q | NEONScalar | NEON_UQADD,
+  NEON_SQADD_scalar  = NEON_Q | NEONScalar | NEON_SQADD,
+  NEON_UQSUB_scalar  = NEON_Q | NEONScalar | NEON_UQSUB,
+  NEON_SQSUB_scalar  = NEON_Q | NEONScalar | NEON_SQSUB,
+  NEON_USHL_scalar   = NEON_Q | NEONScalar | NEON_USHL,
+  NEON_SSHL_scalar   = NEON_Q | NEONScalar | NEON_SSHL,
+  NEON_UQSHL_scalar  = NEON_Q | NEONScalar | NEON_UQSHL,
+  NEON_SQSHL_scalar  = NEON_Q | NEONScalar | NEON_SQSHL,
+  NEON_URSHL_scalar  = NEON_Q | NEONScalar | NEON_URSHL,
+  NEON_SRSHL_scalar  = NEON_Q | NEONScalar | NEON_SRSHL,
+  NEON_UQRSHL_scalar = NEON_Q | NEONScalar | NEON_UQRSHL,
+  NEON_SQRSHL_scalar = NEON_Q | NEONScalar | NEON_SQRSHL,
+  NEON_SQDMULH_scalar = NEON_Q | NEONScalar | NEON_SQDMULH,
+  NEON_SQRDMULH_scalar = NEON_Q | NEONScalar | NEON_SQRDMULH,
+
+  // NEON floating point scalar instructions with three same-type operands.
+  NEONScalar3SameFPFixed = NEONScalar3SameFixed | 0x0000C000,
+  NEONScalar3SameFPFMask = NEONScalar3SameFMask | 0x0000C000,
+  NEONScalar3SameFPMask  = NEONScalar3SameMask | 0x00800000,
+  NEON_FACGE_scalar   = NEON_Q | NEONScalar | NEON_FACGE,
+  NEON_FACGT_scalar   = NEON_Q | NEONScalar | NEON_FACGT,
+  NEON_FCMEQ_scalar   = NEON_Q | NEONScalar | NEON_FCMEQ,
+  NEON_FCMGE_scalar   = NEON_Q | NEONScalar | NEON_FCMGE,
+  NEON_FCMGT_scalar   = NEON_Q | NEONScalar | NEON_FCMGT,
+  NEON_FMULX_scalar   = NEON_Q | NEONScalar | NEON_FMULX,
+  NEON_FRECPS_scalar  = NEON_Q | NEONScalar | NEON_FRECPS,
+  NEON_FRSQRTS_scalar = NEON_Q | NEONScalar | NEON_FRSQRTS,
+  NEON_FABD_scalar    = NEON_Q | NEONScalar | NEON_FABD
+};
+
+// NEON scalar instructions with three different-type operands.
+enum NEONScalar3DiffOp {
+  NEONScalar3DiffFixed = 0x5E200000,
+  NEONScalar3DiffFMask = 0xDF200C00,
+  NEONScalar3DiffMask  = NEON_Q | NEONScalar | NEON3DifferentMask,
+  NEON_SQDMLAL_scalar  = NEON_Q | NEONScalar | NEON_SQDMLAL,
+  NEON_SQDMLSL_scalar  = NEON_Q | NEONScalar | NEON_SQDMLSL,
+  NEON_SQDMULL_scalar  = NEON_Q | NEONScalar | NEON_SQDMULL
+};
+
+// NEON scalar instructions with indexed element operand.
+enum NEONScalarByIndexedElementOp {
+  NEONScalarByIndexedElementFixed = 0x5F000000,
+  NEONScalarByIndexedElementFMask = 0xDF000400,
+  NEONScalarByIndexedElementMask  = 0xFF00F400,
+  NEON_SQDMLAL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMLAL_byelement,
+  NEON_SQDMLSL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMLSL_byelement,
+  NEON_SQDMULL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMULL_byelement,
+  NEON_SQDMULH_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMULH_byelement,
+  NEON_SQRDMULH_byelement_scalar
+    = NEON_Q | NEONScalar | NEON_SQRDMULH_byelement,
+
+  // Floating point instructions.
+  NEONScalarByIndexedElementFPFixed
+    = NEONScalarByIndexedElementFixed | 0x00800000,
+  NEONScalarByIndexedElementFPMask
+    = NEONScalarByIndexedElementMask | 0x00800000,
+  NEON_FMLA_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMLA_byelement,
+  NEON_FMLS_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMLS_byelement,
+  NEON_FMUL_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMUL_byelement,
+  NEON_FMULX_byelement_scalar = NEON_Q | NEONScalar | NEON_FMULX_byelement
+};
+
+// NEON scalar register copy.
+enum NEONScalarCopyOp {
+  NEONScalarCopyFixed = 0x5E000400,
+  NEONScalarCopyFMask = 0xDFE08400,
+  NEONScalarCopyMask  = 0xFFE0FC00,
+  NEON_DUP_ELEMENT_scalar = NEON_Q | NEONScalar | NEON_DUP_ELEMENT
+};
+
+// NEON scalar pairwise instructions.
+enum NEONScalarPairwiseOp {
+  NEONScalarPairwiseFixed = 0x5E300800,
+  NEONScalarPairwiseFMask = 0xDF3E0C00,
+  NEONScalarPairwiseMask  = 0xFFB1F800,
+  NEON_ADDP_scalar    = NEONScalarPairwiseFixed | 0x0081B000,
+  NEON_FMAXNMP_scalar = NEONScalarPairwiseFixed | 0x2000C000,
+  NEON_FMINNMP_scalar = NEONScalarPairwiseFixed | 0x2080C000,
+  NEON_FADDP_scalar   = NEONScalarPairwiseFixed | 0x2000D000,
+  NEON_FMAXP_scalar   = NEONScalarPairwiseFixed | 0x2000F000,
+  NEON_FMINP_scalar   = NEONScalarPairwiseFixed | 0x2080F000
+};
+
+// NEON scalar shift immediate.
+enum NEONScalarShiftImmediateOp {
+  NEONScalarShiftImmediateFixed = 0x5F000400,
+  NEONScalarShiftImmediateFMask = 0xDF800400,
+  NEONScalarShiftImmediateMask  = 0xFF80FC00,
+  NEON_SHL_scalar  = NEON_Q | NEONScalar | NEON_SHL,
+  NEON_SLI_scalar  = NEON_Q | NEONScalar | NEON_SLI,
+  NEON_SRI_scalar  = NEON_Q | NEONScalar | NEON_SRI,
+  NEON_SSHR_scalar = NEON_Q | NEONScalar | NEON_SSHR,
+  NEON_USHR_scalar = NEON_Q | NEONScalar | NEON_USHR,
+  NEON_SRSHR_scalar = NEON_Q | NEONScalar | NEON_SRSHR,
+  NEON_URSHR_scalar = NEON_Q | NEONScalar | NEON_URSHR,
+  NEON_SSRA_scalar = NEON_Q | NEONScalar | NEON_SSRA,
+  NEON_USRA_scalar = NEON_Q | NEONScalar | NEON_USRA,
+  NEON_SRSRA_scalar = NEON_Q | NEONScalar | NEON_SRSRA,
+  NEON_URSRA_scalar = NEON_Q | NEONScalar | NEON_URSRA,
+  NEON_UQSHRN_scalar = NEON_Q | NEONScalar | NEON_UQSHRN,
+  NEON_UQRSHRN_scalar = NEON_Q | NEONScalar | NEON_UQRSHRN,
+  NEON_SQSHRN_scalar = NEON_Q | NEONScalar | NEON_SQSHRN,
+  NEON_SQRSHRN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRN,
+  NEON_SQSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQSHRUN,
+  NEON_SQRSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRUN,
+  NEON_SQSHLU_scalar = NEON_Q | NEONScalar | NEON_SQSHLU,
+  NEON_SQSHL_imm_scalar  = NEON_Q | NEONScalar | NEON_SQSHL_imm,
+  NEON_UQSHL_imm_scalar  = NEON_Q | NEONScalar | NEON_UQSHL_imm,
+  NEON_SCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_SCVTF_imm,
+  NEON_UCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_UCVTF_imm,
+  NEON_FCVTZS_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZS_imm,
+  NEON_FCVTZU_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZU_imm
+};
+
 // Unimplemented and unallocated instructions. These are defined to make fixed
 // bit assertion easier.
 enum UnimplementedOp {
diff --git a/qemu/disas/libvixl/a64/cpu-a64.h b/qemu/disas/libvixl/vixl/a64/cpu-a64.h
index 59b7974a1..cdf09a6af 100644
--- a/qemu/disas/libvixl/a64/cpu-a64.h
+++ b/qemu/disas/libvixl/vixl/a64/cpu-a64.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2014, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -27,8 +27,8 @@
 #ifndef VIXL_CPU_A64_H
 #define VIXL_CPU_A64_H
 
-#include "globals.h"
-#include "instructions-a64.h"
+#include "vixl/globals.h"
+#include "vixl/a64/instructions-a64.h"
 
 namespace vixl {
 
diff --git a/qemu/disas/libvixl/a64/decoder-a64.cc b/qemu/disas/libvixl/vixl/a64/decoder-a64.cc
index 82591ca30..5ba2d3ce0 100644
--- a/qemu/disas/libvixl/a64/decoder-a64.cc
+++ b/qemu/disas/libvixl/vixl/a64/decoder-a64.cc
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2014, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -24,9 +24,9 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "globals.h"
-#include "utils.h"
-#include "a64/decoder-a64.h"
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/decoder-a64.h"
 
 namespace vixl {
 
@@ -271,6 +271,11 @@ void Decoder::DecodeLoadStore(const Instruction* instr) {
               (instr->Bits(27, 24) == 0x9) ||
               (instr->Bits(27, 24) == 0xC) ||
               (instr->Bits(27, 24) == 0xD) );
+  // TODO(all): rearrange the tree to integrate this branch.
+  if ((instr->Bit(28) == 0) && (instr->Bit(29) == 0) && (instr->Bit(26) == 1)) {
+    DecodeNEONLoadStore(instr);
+    return;
+  }
 
   if (instr->Bit(24) == 0) {
     if (instr->Bit(28) == 0) {
@@ -278,7 +283,7 @@ void Decoder::DecodeLoadStore(const Instruction* instr) {
         if (instr->Bit(26) == 0) {
           VisitLoadStoreExclusive(instr);
         } else {
-          DecodeAdvSIMDLoadStore(instr);
+          VIXL_UNREACHABLE();
         }
       } else {
         if ((instr->Bits(31, 30) == 0x3) ||
@@ -483,6 +488,7 @@ void Decoder::DecodeDataProcessing(const Instruction* instr) {
         case 6: {
           if (instr->Bit(29) == 0x1) {
             VisitUnallocated(instr);
+            VIXL_FALLTHROUGH();
           } else {
             if (instr->Bit(30) == 0) {
               if ((instr->Bit(15) == 0x1) ||
@@ -556,18 +562,15 @@ void Decoder::DecodeDataProcessing(const Instruction* instr) {
 void Decoder::DecodeFP(const Instruction* instr) {
   VIXL_ASSERT((instr->Bits(27, 24) == 0xE) ||
               (instr->Bits(27, 24) == 0xF));
-
   if (instr->Bit(28) == 0) {
-    DecodeAdvSIMDDataProcessing(instr);
+    DecodeNEONVectorDataProcessing(instr);
   } else {
-    if (instr->Bit(29) == 1) {
+    if (instr->Bits(31, 30) == 0x3) {
       VisitUnallocated(instr);
+    } else if (instr->Bits(31, 30) == 0x1) {
+      DecodeNEONScalarDataProcessing(instr);
     } else {
-      if (instr->Bits(31, 30) == 0x3) {
-        VisitUnallocated(instr);
-      } else if (instr->Bits(31, 30) == 0x1) {
-        DecodeAdvSIMDDataProcessing(instr);
-      } else {
+      if (instr->Bit(29) == 0) {
         if (instr->Bit(24) == 0) {
           if (instr->Bit(21) == 0) {
             if ((instr->Bit(23) == 1) ||
@@ -674,23 +677,190 @@ void Decoder::DecodeFP(const Instruction* instr) {
             VisitFPDataProcessing3Source(instr);
           }
         }
+      } else {
+        VisitUnallocated(instr);
       }
     }
   }
 }
 
 
-void Decoder::DecodeAdvSIMDLoadStore(const Instruction* instr) {
-  // TODO: Implement Advanced SIMD load/store instruction decode.
+void Decoder::DecodeNEONLoadStore(const Instruction* instr) {
   VIXL_ASSERT(instr->Bits(29, 25) == 0x6);
-  VisitUnimplemented(instr);
+  if (instr->Bit(31) == 0) {
+    if ((instr->Bit(24) == 0) && (instr->Bit(21) == 1)) {
+      VisitUnallocated(instr);
+      return;
+    }
+
+    if (instr->Bit(23) == 0) {
+      if (instr->Bits(20, 16) == 0) {
+        if (instr->Bit(24) == 0) {
+          VisitNEONLoadStoreMultiStruct(instr);
+        } else {
+          VisitNEONLoadStoreSingleStruct(instr);
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+    } else {
+      if (instr->Bit(24) == 0) {
+        VisitNEONLoadStoreMultiStructPostIndex(instr);
+      } else {
+        VisitNEONLoadStoreSingleStructPostIndex(instr);
+      }
+    }
+  } else {
+    VisitUnallocated(instr);
+  }
+}
+
+
+void Decoder::DecodeNEONVectorDataProcessing(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(28, 25) == 0x7);
+  if (instr->Bit(31) == 0) {
+    if (instr->Bit(24) == 0) {
+      if (instr->Bit(21) == 0) {
+        if (instr->Bit(15) == 0) {
+          if (instr->Bit(10) == 0) {
+            if (instr->Bit(29) == 0) {
+              if (instr->Bit(11) == 0) {
+                VisitNEONTable(instr);
+              } else {
+                VisitNEONPerm(instr);
+              }
+            } else {
+              VisitNEONExtract(instr);
+            }
+          } else {
+            if (instr->Bits(23, 22) == 0) {
+              VisitNEONCopy(instr);
+            } else {
+              VisitUnallocated(instr);
+            }
+          }
+        } else {
+          VisitUnallocated(instr);
+        }
+      } else {
+        if (instr->Bit(10) == 0) {
+          if (instr->Bit(11) == 0) {
+            VisitNEON3Different(instr);
+          } else {
+            if (instr->Bits(18, 17) == 0) {
+              if (instr->Bit(20) == 0) {
+                if (instr->Bit(19) == 0) {
+                  VisitNEON2RegMisc(instr);
+                } else {
+                  if (instr->Bits(30, 29) == 0x2) {
+                    VisitCryptoAES(instr);
+                  } else {
+                    VisitUnallocated(instr);
+                  }
+                }
+              } else {
+                if (instr->Bit(19) == 0) {
+                  VisitNEONAcrossLanes(instr);
+                } else {
+                  VisitUnallocated(instr);
+                }
+              }
+            } else {
+              VisitUnallocated(instr);
+            }
+          }
+        } else {
+          VisitNEON3Same(instr);
+        }
+      }
+    } else {
+      if (instr->Bit(10) == 0) {
+        VisitNEONByIndexedElement(instr);
+      } else {
+        if (instr->Bit(23) == 0) {
+          if (instr->Bits(22, 19) == 0) {
+            VisitNEONModifiedImmediate(instr);
+          } else {
+            VisitNEONShiftImmediate(instr);
+          }
+        } else {
+          VisitUnallocated(instr);
+        }
+      }
+    }
+  } else {
+    VisitUnallocated(instr);
+  }
 }
 
 
-void Decoder::DecodeAdvSIMDDataProcessing(const Instruction* instr) {
-  // TODO: Implement Advanced SIMD data processing instruction decode.
-  VIXL_ASSERT(instr->Bits(27, 25) == 0x7);
-  VisitUnimplemented(instr);
+void Decoder::DecodeNEONScalarDataProcessing(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(28, 25) == 0xF);
+  if (instr->Bit(24) == 0) {
+    if (instr->Bit(21) == 0) {
+      if (instr->Bit(15) == 0) {
+        if (instr->Bit(10) == 0) {
+          if (instr->Bit(29) == 0) {
+            if (instr->Bit(11) == 0) {
+              VisitCrypto3RegSHA(instr);
+            } else {
+              VisitUnallocated(instr);
+            }
+          } else {
+            VisitUnallocated(instr);
+          }
+        } else {
+          if (instr->Bits(23, 22) == 0) {
+            VisitNEONScalarCopy(instr);
+          } else {
+            VisitUnallocated(instr);
+          }
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+    } else {
+      if (instr->Bit(10) == 0) {
+        if (instr->Bit(11) == 0) {
+          VisitNEONScalar3Diff(instr);
+        } else {
+          if (instr->Bits(18, 17) == 0) {
+            if (instr->Bit(20) == 0) {
+              if (instr->Bit(19) == 0) {
+                VisitNEONScalar2RegMisc(instr);
+              } else {
+                if (instr->Bit(29) == 0) {
+                  VisitCrypto2RegSHA(instr);
+                } else {
+                  VisitUnallocated(instr);
+                }
+              }
+            } else {
+              if (instr->Bit(19) == 0) {
+                VisitNEONScalarPairwise(instr);
+              } else {
+                VisitUnallocated(instr);
+              }
+            }
+          } else {
+            VisitUnallocated(instr);
+          }
+        }
+      } else {
+        VisitNEONScalar3Same(instr);
+      }
+    }
+  } else {
+    if (instr->Bit(10) == 0) {
+      VisitNEONScalarByIndexedElement(instr);
+    } else {
+      if (instr->Bit(23) == 0) {
+        VisitNEONScalarShiftImmediate(instr);
+      } else {
+        VisitUnallocated(instr);
+      }
+    }
+  }
 }
 
 
diff --git a/qemu/disas/libvixl/a64/decoder-a64.h b/qemu/disas/libvixl/vixl/a64/decoder-a64.h
index fd08d6c1f..b3f04f68f 100644
--- a/qemu/disas/libvixl/a64/decoder-a64.h
+++ b/qemu/disas/libvixl/vixl/a64/decoder-a64.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2014, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -29,13 +29,13 @@
 
 #include <list>
 
-#include "globals.h"
-#include "a64/instructions-a64.h"
+#include "vixl/globals.h"
+#include "vixl/a64/instructions-a64.h"
 
 
 // List macro containing all visitors needed by the decoder class.
 
-#define VISITOR_LIST(V)             \
+#define VISITOR_LIST_THAT_RETURN(V) \
   V(PCRelAddressing)                \
   V(AddSubImmediate)                \
   V(LogicalImmediate)               \
@@ -79,8 +79,39 @@
   V(FPDataProcessing3Source)        \
   V(FPIntegerConvert)               \
   V(FPFixedPointConvert)            \
-  V(Unallocated)                    \
-  V(Unimplemented)
+  V(Crypto2RegSHA)                  \
+  V(Crypto3RegSHA)                  \
+  V(CryptoAES)                      \
+  V(NEON2RegMisc)                   \
+  V(NEON3Different)                 \
+  V(NEON3Same)                      \
+  V(NEONAcrossLanes)                \
+  V(NEONByIndexedElement)           \
+  V(NEONCopy)                       \
+  V(NEONExtract)                    \
+  V(NEONLoadStoreMultiStruct)       \
+  V(NEONLoadStoreMultiStructPostIndex)  \
+  V(NEONLoadStoreSingleStruct)      \
+  V(NEONLoadStoreSingleStructPostIndex) \
+  V(NEONModifiedImmediate)          \
+  V(NEONScalar2RegMisc)             \
+  V(NEONScalar3Diff)                \
+  V(NEONScalar3Same)                \
+  V(NEONScalarByIndexedElement)     \
+  V(NEONScalarCopy)                 \
+  V(NEONScalarPairwise)             \
+  V(NEONScalarShiftImmediate)       \
+  V(NEONShiftImmediate)             \
+  V(NEONTable)                      \
+  V(NEONPerm)                       \
+
+#define VISITOR_LIST_THAT_DONT_RETURN(V)  \
+  V(Unallocated)                          \
+  V(Unimplemented)                        \
+
+#define VISITOR_LIST(V)             \
+  VISITOR_LIST_THAT_RETURN(V)       \
+  VISITOR_LIST_THAT_DONT_RETURN(V)  \
 
 namespace vixl {
 
@@ -222,12 +253,17 @@ class Decoder {
   // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
   // and call the corresponding visitors.
   // On entry, instruction bits 29:25 = 0x6.
-  void DecodeAdvSIMDLoadStore(const Instruction* instr);
+  void DecodeNEONLoadStore(const Instruction* instr);
 
-  // Decode the Advanced SIMD (NEON) data processing part of the instruction
-  // tree, and call the corresponding visitors.
-  // On entry, instruction bits 27:25 = 0x7.
-  void DecodeAdvSIMDDataProcessing(const Instruction* instr);
+  // Decode the Advanced SIMD (NEON) vector data processing part of the
+  // instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 28:25 = 0x7.
+  void DecodeNEONVectorDataProcessing(const Instruction* instr);
+
+  // Decode the Advanced SIMD (NEON) scalar data processing part of the
+  // instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 28:25 = 0xF.
+  void DecodeNEONScalarDataProcessing(const Instruction* instr);
 
  private:
   // Visitors are registered in a list.
diff --git a/qemu/disas/libvixl/vixl/a64/disasm-a64.cc b/qemu/disas/libvixl/vixl/a64/disasm-a64.cc
new file mode 100644
index 000000000..7a58a5c08
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/a64/disasm-a64.cc
@@ -0,0 +1,3491 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdlib>
+#include "vixl/a64/disasm-a64.h"
+
+namespace vixl {
+
+Disassembler::Disassembler() {
+  buffer_size_ = 256;
+  buffer_ = reinterpret_cast<char*>(malloc(buffer_size_));
+  buffer_pos_ = 0;
+  own_buffer_ = true;
+  code_address_offset_ = 0;
+}
+
+
+Disassembler::Disassembler(char* text_buffer, int buffer_size) {
+  buffer_size_ = buffer_size;
+  buffer_ = text_buffer;
+  buffer_pos_ = 0;
+  own_buffer_ = false;
+  code_address_offset_ = 0;
+}
+
+
+Disassembler::~Disassembler() {
+  if (own_buffer_) {
+    free(buffer_);
+  }
+}
+
+
+char* Disassembler::GetOutput() {
+  return buffer_;
+}
+
+
+void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
+                  (instr->ImmAddSub() == 0) ? true : false;
+  const char *mnemonic = "";
+  const char *form = "'Rds, 'Rns, 'IAddSub";
+  const char *form_cmp = "'Rns, 'IAddSub";
+  const char *form_mov = "'Rds, 'Rns";
+
+  switch (instr->Mask(AddSubImmediateMask)) {
+    case ADD_w_imm:
+    case ADD_x_imm: {
+      mnemonic = "add";
+      if (stack_op) {
+        mnemonic = "mov";
+        form = form_mov;
+      }
+      break;
+    }
+    case ADDS_w_imm:
+    case ADDS_x_imm: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_imm:
+    case SUB_x_imm: mnemonic = "sub"; break;
+    case SUBS_w_imm:
+    case SUBS_x_imm: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubShifted(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm'NDP";
+  const char *form_cmp = "'Rn, 'Rm'NDP";
+  const char *form_neg = "'Rd, 'Rm'NDP";
+
+  switch (instr->Mask(AddSubShiftedMask)) {
+    case ADD_w_shift:
+    case ADD_x_shift: mnemonic = "add"; break;
+    case ADDS_w_shift:
+    case ADDS_x_shift: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_shift:
+    case SUB_x_shift: {
+      mnemonic = "sub";
+      if (rn_is_zr) {
+        mnemonic = "neg";
+        form = form_neg;
+      }
+      break;
+    }
+    case SUBS_w_shift:
+    case SUBS_x_shift: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      } else if (rn_is_zr) {
+        mnemonic = "negs";
+        form = form_neg;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubExtended(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  const char *mnemonic = "";
+  Extend mode = static_cast<Extend>(instr->ExtendMode());
+  const char *form = ((mode == UXTX) || (mode == SXTX)) ?
+                     "'Rds, 'Rns, 'Xm'Ext" : "'Rds, 'Rns, 'Wm'Ext";
+  const char *form_cmp = ((mode == UXTX) || (mode == SXTX)) ?
+                         "'Rns, 'Xm'Ext" : "'Rns, 'Wm'Ext";
+
+  switch (instr->Mask(AddSubExtendedMask)) {
+    case ADD_w_ext:
+    case ADD_x_ext: mnemonic = "add"; break;
+    case ADDS_w_ext:
+    case ADDS_x_ext: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_ext:
+    case SUB_x_ext: mnemonic = "sub"; break;
+    case SUBS_w_ext:
+    case SUBS_x_ext: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm";
+  const char *form_neg = "'Rd, 'Rm";
+
+  switch (instr->Mask(AddSubWithCarryMask)) {
+    case ADC_w:
+    case ADC_x: mnemonic = "adc"; break;
+    case ADCS_w:
+    case ADCS_x: mnemonic = "adcs"; break;
+    case SBC_w:
+    case SBC_x: {
+      mnemonic = "sbc";
+      if (rn_is_zr) {
+        mnemonic = "ngc";
+        form = form_neg;
+      }
+      break;
+    }
+    case SBCS_w:
+    case SBCS_x: {
+      mnemonic = "sbcs";
+      if (rn_is_zr) {
+        mnemonic = "ngcs";
+        form = form_neg;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rds, 'Rn, 'ITri";
+
+  if (instr->ImmLogical() == 0) {
+    // The immediate encoded in the instruction is not in the expected format.
+    Format(instr, "unallocated", "(LogicalImmediate)");
+    return;
+  }
+
+  switch (instr->Mask(LogicalImmediateMask)) {
+    case AND_w_imm:
+    case AND_x_imm: mnemonic = "and"; break;
+    case ORR_w_imm:
+    case ORR_x_imm: {
+      mnemonic = "orr";
+      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
+                                                        : kWRegSize;
+      if (rn_is_zr && !IsMovzMovnImm(reg_size, instr->ImmLogical())) {
+        mnemonic = "mov";
+        form = "'Rds, 'ITri";
+      }
+      break;
+    }
+    case EOR_w_imm:
+    case EOR_x_imm: mnemonic = "eor"; break;
+    case ANDS_w_imm:
+    case ANDS_x_imm: {
+      mnemonic = "ands";
+      if (rd_is_zr) {
+        mnemonic = "tst";
+        form = "'Rn, 'ITri";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
+  VIXL_ASSERT((reg_size == kXRegSize) ||
+              ((reg_size == kWRegSize) && (value <= 0xffffffff)));
+
+  // Test for movz: 16 bits set at positions 0, 16, 32 or 48.
+  if (((value & UINT64_C(0xffffffffffff0000)) == 0) ||
+      ((value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+      ((value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+      ((value & UINT64_C(0x0000ffffffffffff)) == 0)) {
+    return true;
+  }
+
+  // Test for movn: NOT(16 bits set at positions 0, 16, 32 or 48).
+  if ((reg_size == kXRegSize) &&
+      (((~value & UINT64_C(0xffffffffffff0000)) == 0) ||
+       ((~value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+       ((~value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+       ((~value & UINT64_C(0x0000ffffffffffff)) == 0))) {
+    return true;
+  }
+  if ((reg_size == kWRegSize) &&
+      (((value & 0xffff0000) == 0xffff0000) ||
+       ((value & 0x0000ffff) == 0x0000ffff))) {
+    return true;
+  }
+  return false;
+}
+
+
+void Disassembler::VisitLogicalShifted(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm'NLo";
+
+  switch (instr->Mask(LogicalShiftedMask)) {
+    case AND_w:
+    case AND_x: mnemonic = "and"; break;
+    case BIC_w:
+    case BIC_x: mnemonic = "bic"; break;
+    case EOR_w:
+    case EOR_x: mnemonic = "eor"; break;
+    case EON_w:
+    case EON_x: mnemonic = "eon"; break;
+    case BICS_w:
+    case BICS_x: mnemonic = "bics"; break;
+    case ANDS_w:
+    case ANDS_x: {
+      mnemonic = "ands";
+      if (rd_is_zr) {
+        mnemonic = "tst";
+        form = "'Rn, 'Rm'NLo";
+      }
+      break;
+    }
+    case ORR_w:
+    case ORR_x: {
+      mnemonic = "orr";
+      if (rn_is_zr && (instr->ImmDPShift() == 0) && (instr->ShiftDP() == LSL)) {
+        mnemonic = "mov";
+        form = "'Rd, 'Rm";
+      }
+      break;
+    }
+    case ORN_w:
+    case ORN_x: {
+      mnemonic = "orn";
+      if (rn_is_zr) {
+        mnemonic = "mvn";
+        form = "'Rd, 'Rm'NLo";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
+
+  switch (instr->Mask(ConditionalCompareRegisterMask)) {
+    case CCMN_w:
+    case CCMN_x: mnemonic = "ccmn"; break;
+    case CCMP_w:
+    case CCMP_x: mnemonic = "ccmp"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
+
+  switch (instr->Mask(ConditionalCompareImmediateMask)) {
+    case CCMN_w_imm:
+    case CCMN_x_imm: mnemonic = "ccmn"; break;
+    case CCMP_w_imm:
+    case CCMP_x_imm: mnemonic = "ccmp"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalSelect(const Instruction* instr) {
+  bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
+  bool rn_is_rm = (instr->Rn() == instr->Rm());
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm, 'Cond";
+  const char *form_test = "'Rd, 'CInv";
+  const char *form_update = "'Rd, 'Rn, 'CInv";
+
+  Condition cond = static_cast<Condition>(instr->Condition());
+  bool invertible_cond = (cond != al) && (cond != nv);
+
+  switch (instr->Mask(ConditionalSelectMask)) {
+    case CSEL_w:
+    case CSEL_x: mnemonic = "csel"; break;
+    case CSINC_w:
+    case CSINC_x: {
+      mnemonic = "csinc";
+      if (rnm_is_zr && invertible_cond) {
+        mnemonic = "cset";
+        form = form_test;
+      } else if (rn_is_rm && invertible_cond) {
+        mnemonic = "cinc";
+        form = form_update;
+      }
+      break;
+    }
+    case CSINV_w:
+    case CSINV_x: {
+      mnemonic = "csinv";
+      if (rnm_is_zr && invertible_cond) {
+        mnemonic = "csetm";
+        form = form_test;
+      } else if (rn_is_rm && invertible_cond) {
+        mnemonic = "cinv";
+        form = form_update;
+      }
+      break;
+    }
+    case CSNEG_w:
+    case CSNEG_x: {
+      mnemonic = "csneg";
+      if (rn_is_rm && invertible_cond) {
+        mnemonic = "cneg";
+        form = form_update;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitBitfield(const Instruction* instr) {
+  unsigned s = instr->ImmS();
+  unsigned r = instr->ImmR();
+  unsigned rd_size_minus_1 =
+    ((instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize) - 1;
+  const char *mnemonic = "";
+  const char *form = "";
+  const char *form_shift_right = "'Rd, 'Rn, 'IBr";
+  const char *form_extend = "'Rd, 'Wn";
+  const char *form_bfiz = "'Rd, 'Rn, 'IBZ-r, 'IBs+1";
+  const char *form_bfx = "'Rd, 'Rn, 'IBr, 'IBs-r+1";
+  const char *form_lsl = "'Rd, 'Rn, 'IBZ-r";
+
+  switch (instr->Mask(BitfieldMask)) {
+    case SBFM_w:
+    case SBFM_x: {
+      mnemonic = "sbfx";
+      form = form_bfx;
+      if (r == 0) {
+        form = form_extend;
+        if (s == 7) {
+          mnemonic = "sxtb";
+        } else if (s == 15) {
+          mnemonic = "sxth";
+        } else if ((s == 31) && (instr->SixtyFourBits() == 1)) {
+          mnemonic = "sxtw";
+        } else {
+          form = form_bfx;
+        }
+      } else if (s == rd_size_minus_1) {
+        mnemonic = "asr";
+        form = form_shift_right;
+      } else if (s < r) {
+        mnemonic = "sbfiz";
+        form = form_bfiz;
+      }
+      break;
+    }
+    case UBFM_w:
+    case UBFM_x: {
+      mnemonic = "ubfx";
+      form = form_bfx;
+      if (r == 0) {
+        form = form_extend;
+        if (s == 7) {
+          mnemonic = "uxtb";
+        } else if (s == 15) {
+          mnemonic = "uxth";
+        } else {
+          form = form_bfx;
+        }
+      }
+      if (s == rd_size_minus_1) {
+        mnemonic = "lsr";
+        form = form_shift_right;
+      } else if (r == s + 1) {
+        mnemonic = "lsl";
+        form = form_lsl;
+      } else if (s < r) {
+        mnemonic = "ubfiz";
+        form = form_bfiz;
+      }
+      break;
+    }
+    case BFM_w:
+    case BFM_x: {
+      mnemonic = "bfxil";
+      form = form_bfx;
+      if (s < r) {
+        mnemonic = "bfi";
+        form = form_bfiz;
+      }
+    }
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitExtract(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
+
+  switch (instr->Mask(ExtractMask)) {
+    case EXTR_w:
+    case EXTR_x: {
+      if (instr->Rn() == instr->Rm()) {
+        mnemonic = "ror";
+        form = "'Rd, 'Rn, 'IExtract";
+      } else {
+        mnemonic = "extr";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
+  switch (instr->Mask(PCRelAddressingMask)) {
+    case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
+    case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
+    default: Format(instr, "unimplemented", "(PCRelAddressing)");
+  }
+}
+
+
+void Disassembler::VisitConditionalBranch(const Instruction* instr) {
+  switch (instr->Mask(ConditionalBranchMask)) {
+    case B_cond: Format(instr, "b.'CBrn", "'TImmCond"); break;
+    default: VIXL_UNREACHABLE();
+  }
+}
+
+
+void Disassembler::VisitUnconditionalBranchToRegister(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Xn";
+
+  switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
+    case BR: mnemonic = "br"; break;
+    case BLR: mnemonic = "blr"; break;
+    case RET: {
+      mnemonic = "ret";
+      if (instr->Rn() == kLinkRegCode) {
+        form = NULL;
+      }
+      break;
+    }
+    default: form = "(UnconditionalBranchToRegister)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'TImmUncn";
+
+  switch (instr->Mask(UnconditionalBranchMask)) {
+    case B: mnemonic = "b"; break;
+    case BL: mnemonic = "bl"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn";
+
+  switch (instr->Mask(DataProcessing1SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_w:           \
+    case A##_x: mnemonic = B; break;
+    FORMAT(RBIT, "rbit");
+    FORMAT(REV16, "rev16");
+    FORMAT(REV, "rev");
+    FORMAT(CLZ, "clz");
+    FORMAT(CLS, "cls");
+    #undef FORMAT
+    case REV32_x: mnemonic = "rev32"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Rd, 'Rn, 'Rm";
+  const char *form_wwx = "'Wd, 'Wn, 'Xm";
+
+  switch (instr->Mask(DataProcessing2SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_w:           \
+    case A##_x: mnemonic = B; break;
+    FORMAT(UDIV, "udiv");
+    FORMAT(SDIV, "sdiv");
+    FORMAT(LSLV, "lsl");
+    FORMAT(LSRV, "lsr");
+    FORMAT(ASRV, "asr");
+    FORMAT(RORV, "ror");
+    #undef FORMAT
+    case CRC32B: mnemonic = "crc32b"; break;
+    case CRC32H: mnemonic = "crc32h"; break;
+    case CRC32W: mnemonic = "crc32w"; break;
+    case CRC32X: mnemonic = "crc32x"; form = form_wwx; break;
+    case CRC32CB: mnemonic = "crc32cb"; break;
+    case CRC32CH: mnemonic = "crc32ch"; break;
+    case CRC32CW: mnemonic = "crc32cw"; break;
+    case CRC32CX: mnemonic = "crc32cx"; form = form_wwx; break;
+    default: form = "(DataProcessing2Source)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
+  bool ra_is_zr = RaIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
+  const char *form_rrr = "'Rd, 'Rn, 'Rm";
+  const char *form_rrrr = "'Rd, 'Rn, 'Rm, 'Ra";
+  const char *form_xww = "'Xd, 'Wn, 'Wm";
+  const char *form_xxx = "'Xd, 'Xn, 'Xm";
+
+  switch (instr->Mask(DataProcessing3SourceMask)) {
+    case MADD_w:
+    case MADD_x: {
+      mnemonic = "madd";
+      form = form_rrrr;
+      if (ra_is_zr) {
+        mnemonic = "mul";
+        form = form_rrr;
+      }
+      break;
+    }
+    case MSUB_w:
+    case MSUB_x: {
+      mnemonic = "msub";
+      form = form_rrrr;
+      if (ra_is_zr) {
+        mnemonic = "mneg";
+        form = form_rrr;
+      }
+      break;
+    }
+    case SMADDL_x: {
+      mnemonic = "smaddl";
+      if (ra_is_zr) {
+        mnemonic = "smull";
+        form = form_xww;
+      }
+      break;
+    }
+    case SMSUBL_x: {
+      mnemonic = "smsubl";
+      if (ra_is_zr) {
+        mnemonic = "smnegl";
+        form = form_xww;
+      }
+      break;
+    }
+    case UMADDL_x: {
+      mnemonic = "umaddl";
+      if (ra_is_zr) {
+        mnemonic = "umull";
+        form = form_xww;
+      }
+      break;
+    }
+    case UMSUBL_x: {
+      mnemonic = "umsubl";
+      if (ra_is_zr) {
+        mnemonic = "umnegl";
+        form = form_xww;
+      }
+      break;
+    }
+    case SMULH_x: {
+      mnemonic = "smulh";
+      form = form_xxx;
+      break;
+    }
+    case UMULH_x: {
+      mnemonic = "umulh";
+      form = form_xxx;
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCompareBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rt, 'TImmCmpa";
+
+  switch (instr->Mask(CompareBranchMask)) {
+    case CBZ_w:
+    case CBZ_x: mnemonic = "cbz"; break;
+    case CBNZ_w:
+    case CBNZ_x: mnemonic = "cbnz"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitTestBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  // If the top bit of the immediate is clear, the tested register is
+  // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
+  // encoded in bit 31 of the instruction, we can reuse the Rt form, which
+  // uses bit 31 (normally "sf") to choose the register size.
+  const char *form = "'Rt, 'IS, 'TImmTest";
+
+  switch (instr->Mask(TestBranchMask)) {
+    case TBZ: mnemonic = "tbz"; break;
+    case TBNZ: mnemonic = "tbnz"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'IMoveImm";
+
+  // Print the shift separately for movk, to make it clear which half word will
+  // be overwritten. Movn and movz print the computed immediate, which includes
+  // shift calculation.
+  switch (instr->Mask(MoveWideImmediateMask)) {
+    case MOVN_w:
+    case MOVN_x:
+      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0)) {
+        if ((instr->SixtyFourBits() == 0) && (instr->ImmMoveWide() == 0xffff)) {
+          mnemonic = "movn";
+        } else {
+          mnemonic = "mov";
+          form = "'Rd, 'IMoveNeg";
+        }
+      } else {
+        mnemonic = "movn";
+      }
+      break;
+    case MOVZ_w:
+    case MOVZ_x:
+      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0))
+        mnemonic = "mov";
+      else
+        mnemonic = "movz";
+      break;
+    case MOVK_w:
+    case MOVK_x: mnemonic = "movk"; form = "'Rd, 'IMoveLSL"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_LIST(V)    \
+  V(STRB_w, "strb", "'Wt")    \
+  V(STRH_w, "strh", "'Wt")    \
+  V(STR_w, "str", "'Wt")      \
+  V(STR_x, "str", "'Xt")      \
+  V(LDRB_w, "ldrb", "'Wt")    \
+  V(LDRH_w, "ldrh", "'Wt")    \
+  V(LDR_w, "ldr", "'Wt")      \
+  V(LDR_x, "ldr", "'Xt")      \
+  V(LDRSB_x, "ldrsb", "'Xt")  \
+  V(LDRSH_x, "ldrsh", "'Xt")  \
+  V(LDRSW_x, "ldrsw", "'Xt")  \
+  V(LDRSB_w, "ldrsb", "'Wt")  \
+  V(LDRSH_w, "ldrsh", "'Wt")  \
+  V(STR_b, "str", "'Bt")      \
+  V(STR_h, "str", "'Ht")      \
+  V(STR_s, "str", "'St")      \
+  V(STR_d, "str", "'Dt")      \
+  V(LDR_b, "ldr", "'Bt")      \
+  V(LDR_h, "ldr", "'Ht")      \
+  V(LDR_s, "ldr", "'St")      \
+  V(LDR_d, "ldr", "'Dt")      \
+  V(STR_q, "str", "'Qt")      \
+  V(LDR_q, "ldr", "'Qt")
+
+void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePreIndex)";
+
+  switch (instr->Mask(LoadStorePreIndexMask)) {
+    #define LS_PREINDEX(A, B, C) \
+    case A##_pre: mnemonic = B; form = C ", ['Xns'ILS]!"; break;
+    LOAD_STORE_LIST(LS_PREINDEX)
+    #undef LS_PREINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePostIndex)";
+
+  switch (instr->Mask(LoadStorePostIndexMask)) {
+    #define LS_POSTINDEX(A, B, C) \
+    case A##_post: mnemonic = B; form = C ", ['Xns]'ILS"; break;
+    LOAD_STORE_LIST(LS_POSTINDEX)
+    #undef LS_POSTINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStoreUnsignedOffset)";
+
+  switch (instr->Mask(LoadStoreUnsignedOffsetMask)) {
+    #define LS_UNSIGNEDOFFSET(A, B, C) \
+    case A##_unsigned: mnemonic = B; form = C ", ['Xns'ILU]"; break;
+    LOAD_STORE_LIST(LS_UNSIGNEDOFFSET)
+    #undef LS_UNSIGNEDOFFSET
+    case PRFM_unsigned: mnemonic = "prfm"; form = "'PrefOp, ['Xns'ILU]";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStoreRegisterOffset)";
+
+  switch (instr->Mask(LoadStoreRegisterOffsetMask)) {
+    #define LS_REGISTEROFFSET(A, B, C) \
+    case A##_reg: mnemonic = B; form = C ", ['Xns, 'Offsetreg]"; break;
+    LOAD_STORE_LIST(LS_REGISTEROFFSET)
+    #undef LS_REGISTEROFFSET
+    case PRFM_reg: mnemonic = "prfm"; form = "'PrefOp, ['Xns, 'Offsetreg]";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Wt, ['Xns'ILS]";
+  const char *form_x = "'Xt, ['Xns'ILS]";
+  const char *form_b = "'Bt, ['Xns'ILS]";
+  const char *form_h = "'Ht, ['Xns'ILS]";
+  const char *form_s = "'St, ['Xns'ILS]";
+  const char *form_d = "'Dt, ['Xns'ILS]";
+  const char *form_q = "'Qt, ['Xns'ILS]";
+  const char *form_prefetch = "'PrefOp, ['Xns'ILS]";
+
+  switch (instr->Mask(LoadStoreUnscaledOffsetMask)) {
+    case STURB_w:  mnemonic = "sturb"; break;
+    case STURH_w:  mnemonic = "sturh"; break;
+    case STUR_w:   mnemonic = "stur"; break;
+    case STUR_x:   mnemonic = "stur"; form = form_x; break;
+    case STUR_b:   mnemonic = "stur"; form = form_b; break;
+    case STUR_h:   mnemonic = "stur"; form = form_h; break;
+    case STUR_s:   mnemonic = "stur"; form = form_s; break;
+    case STUR_d:   mnemonic = "stur"; form = form_d; break;
+    case STUR_q:   mnemonic = "stur"; form = form_q; break;
+    case LDURB_w:  mnemonic = "ldurb"; break;
+    case LDURH_w:  mnemonic = "ldurh"; break;
+    case LDUR_w:   mnemonic = "ldur"; break;
+    case LDUR_x:   mnemonic = "ldur"; form = form_x; break;
+    case LDUR_b:   mnemonic = "ldur"; form = form_b; break;
+    case LDUR_h:   mnemonic = "ldur"; form = form_h; break;
+    case LDUR_s:   mnemonic = "ldur"; form = form_s; break;
+    case LDUR_d:   mnemonic = "ldur"; form = form_d; break;
+    case LDUR_q:   mnemonic = "ldur"; form = form_q; break;
+    case LDURSB_x: form = form_x; VIXL_FALLTHROUGH();
+    case LDURSB_w: mnemonic = "ldursb"; break;
+    case LDURSH_x: form = form_x; VIXL_FALLTHROUGH();
+    case LDURSH_w: mnemonic = "ldursh"; break;
+    case LDURSW_x: mnemonic = "ldursw"; form = form_x; break;
+    case PRFUM:    mnemonic = "prfum"; form = form_prefetch; break;
+    default: form = "(LoadStoreUnscaledOffset)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadLiteral(const Instruction* instr) {
+  const char *mnemonic = "ldr";
+  const char *form = "(LoadLiteral)";
+
+  switch (instr->Mask(LoadLiteralMask)) {
+    case LDR_w_lit: form = "'Wt, 'ILLiteral 'LValue"; break;
+    case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
+    case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
+    case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
+    case LDR_q_lit: form = "'Qt, 'ILLiteral 'LValue"; break;
+    case LDRSW_x_lit: {
+      mnemonic = "ldrsw";
+      form = "'Xt, 'ILLiteral 'LValue";
+      break;
+    }
+    case PRFM_lit: {
+      mnemonic = "prfm";
+      form = "'PrefOp, 'ILLiteral 'LValue";
+      break;
+    }
+    default: mnemonic = "unimplemented";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_PAIR_LIST(V)         \
+  V(STP_w, "stp", "'Wt, 'Wt2", "2")     \
+  V(LDP_w, "ldp", "'Wt, 'Wt2", "2")     \
+  V(LDPSW_x, "ldpsw", "'Xt, 'Xt2", "2") \
+  V(STP_x, "stp", "'Xt, 'Xt2", "3")     \
+  V(LDP_x, "ldp", "'Xt, 'Xt2", "3")     \
+  V(STP_s, "stp", "'St, 'St2", "2")     \
+  V(LDP_s, "ldp", "'St, 'St2", "2")     \
+  V(STP_d, "stp", "'Dt, 'Dt2", "3")     \
+  V(LDP_d, "ldp", "'Dt, 'Dt2", "3")     \
+  V(LDP_q, "ldp", "'Qt, 'Qt2", "4")     \
+  V(STP_q, "stp", "'Qt, 'Qt2", "4")
+
+void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairPostIndex)";
+
+  switch (instr->Mask(LoadStorePairPostIndexMask)) {
+    #define LSP_POSTINDEX(A, B, C, D) \
+    case A##_post: mnemonic = B; form = C ", ['Xns]'ILP" D; break;
+    LOAD_STORE_PAIR_LIST(LSP_POSTINDEX)
+    #undef LSP_POSTINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairPreIndex)";
+
+  switch (instr->Mask(LoadStorePairPreIndexMask)) {
+    #define LSP_PREINDEX(A, B, C, D) \
+    case A##_pre: mnemonic = B; form = C ", ['Xns'ILP" D "]!"; break;
+    LOAD_STORE_PAIR_LIST(LSP_PREINDEX)
+    #undef LSP_PREINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairOffset)";
+
+  switch (instr->Mask(LoadStorePairOffsetMask)) {
+    #define LSP_OFFSET(A, B, C, D) \
+    case A##_off: mnemonic = B; form = C ", ['Xns'ILP" D "]"; break;
+    LOAD_STORE_PAIR_LIST(LSP_OFFSET)
+    #undef LSP_OFFSET
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form;
+
+  switch (instr->Mask(LoadStorePairNonTemporalMask)) {
+    case STNP_w: mnemonic = "stnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+    case LDNP_w: mnemonic = "ldnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+    case STNP_x: mnemonic = "stnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+    case LDNP_x: mnemonic = "ldnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+    case STNP_s: mnemonic = "stnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+    case LDNP_s: mnemonic = "ldnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+    case STNP_d: mnemonic = "stnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+    case LDNP_d: mnemonic = "ldnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+    case STNP_q: mnemonic = "stnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+    case LDNP_q: mnemonic = "ldnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+    default: form = "(LoadStorePairNonTemporal)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form;
+
+  switch (instr->Mask(LoadStoreExclusiveMask)) {
+    case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
+    case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
+    case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
+    case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
+    case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
+    case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
+    case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
+    case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
+    case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
+    case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
+    case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
+    case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
+    case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
+    case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
+    default: form = "(LoadStoreExclusive)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPCompare(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Fn, 'Fm";
+  const char *form_zero = "'Fn, #0.0";
+
+  switch (instr->Mask(FPCompareMask)) {
+    case FCMP_s_zero:
+    case FCMP_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+    case FCMP_s:
+    case FCMP_d: mnemonic = "fcmp"; break;
+    case FCMPE_s_zero:
+    case FCMPE_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+    case FCMPE_s:
+    case FCMPE_d: mnemonic = "fcmpe"; break;
+    default: form = "(FPCompare)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
+  const char *mnemonic = "unmplemented";
+  const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
+
+  switch (instr->Mask(FPConditionalCompareMask)) {
+    case FCCMP_s:
+    case FCCMP_d: mnemonic = "fccmp"; break;
+    case FCCMPE_s:
+    case FCCMPE_d: mnemonic = "fccmpe"; break;
+    default: form = "(FPConditionalCompare)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
+
+  switch (instr->Mask(FPConditionalSelectMask)) {
+    case FCSEL_s:
+    case FCSEL_d: mnemonic = "fcsel"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Fd, 'Fn";
+
+  switch (instr->Mask(FPDataProcessing1SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMOV, "fmov");
+    FORMAT(FABS, "fabs");
+    FORMAT(FNEG, "fneg");
+    FORMAT(FSQRT, "fsqrt");
+    FORMAT(FRINTN, "frintn");
+    FORMAT(FRINTP, "frintp");
+    FORMAT(FRINTM, "frintm");
+    FORMAT(FRINTZ, "frintz");
+    FORMAT(FRINTA, "frinta");
+    FORMAT(FRINTX, "frintx");
+    FORMAT(FRINTI, "frinti");
+    #undef FORMAT
+    case FCVT_ds: mnemonic = "fcvt"; form = "'Dd, 'Sn"; break;
+    case FCVT_sd: mnemonic = "fcvt"; form = "'Sd, 'Dn"; break;
+    case FCVT_hs: mnemonic = "fcvt"; form = "'Hd, 'Sn"; break;
+    case FCVT_sh: mnemonic = "fcvt"; form = "'Sd, 'Hn"; break;
+    case FCVT_dh: mnemonic = "fcvt"; form = "'Dd, 'Hn"; break;
+    case FCVT_hd: mnemonic = "fcvt"; form = "'Hd, 'Dn"; break;
+    default: form = "(FPDataProcessing1Source)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm";
+
+  switch (instr->Mask(FPDataProcessing2SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMUL, "fmul");
+    FORMAT(FDIV, "fdiv");
+    FORMAT(FADD, "fadd");
+    FORMAT(FSUB, "fsub");
+    FORMAT(FMAX, "fmax");
+    FORMAT(FMIN, "fmin");
+    FORMAT(FMAXNM, "fmaxnm");
+    FORMAT(FMINNM, "fminnm");
+    FORMAT(FNMUL, "fnmul");
+    #undef FORMAT
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
+
+  switch (instr->Mask(FPDataProcessing3SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMADD, "fmadd");
+    FORMAT(FMSUB, "fmsub");
+    FORMAT(FNMADD, "fnmadd");
+    FORMAT(FNMSUB, "fnmsub");
+    #undef FORMAT
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "(FPImmediate)";
+
+  switch (instr->Mask(FPImmediateMask)) {
+    case FMOV_s_imm: mnemonic = "fmov"; form = "'Sd, 'IFPSingle"; break;
+    case FMOV_d_imm: mnemonic = "fmov"; form = "'Dd, 'IFPDouble"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(FPIntegerConvert)";
+  const char *form_rf = "'Rd, 'Fn";
+  const char *form_fr = "'Fd, 'Rn";
+
+  switch (instr->Mask(FPIntegerConvertMask)) {
+    case FMOV_ws:
+    case FMOV_xd: mnemonic = "fmov"; form = form_rf; break;
+    case FMOV_sw:
+    case FMOV_dx: mnemonic = "fmov"; form = form_fr; break;
+    case FMOV_d1_x: mnemonic = "fmov"; form = "'Vd.D[1], 'Rn"; break;
+    case FMOV_x_d1: mnemonic = "fmov"; form = "'Rd, 'Vn.D[1]"; break;
+    case FCVTAS_ws:
+    case FCVTAS_xs:
+    case FCVTAS_wd:
+    case FCVTAS_xd: mnemonic = "fcvtas"; form = form_rf; break;
+    case FCVTAU_ws:
+    case FCVTAU_xs:
+    case FCVTAU_wd:
+    case FCVTAU_xd: mnemonic = "fcvtau"; form = form_rf; break;
+    case FCVTMS_ws:
+    case FCVTMS_xs:
+    case FCVTMS_wd:
+    case FCVTMS_xd: mnemonic = "fcvtms"; form = form_rf; break;
+    case FCVTMU_ws:
+    case FCVTMU_xs:
+    case FCVTMU_wd:
+    case FCVTMU_xd: mnemonic = "fcvtmu"; form = form_rf; break;
+    case FCVTNS_ws:
+    case FCVTNS_xs:
+    case FCVTNS_wd:
+    case FCVTNS_xd: mnemonic = "fcvtns"; form = form_rf; break;
+    case FCVTNU_ws:
+    case FCVTNU_xs:
+    case FCVTNU_wd:
+    case FCVTNU_xd: mnemonic = "fcvtnu"; form = form_rf; break;
+    case FCVTZU_xd:
+    case FCVTZU_ws:
+    case FCVTZU_wd:
+    case FCVTZU_xs: mnemonic = "fcvtzu"; form = form_rf; break;
+    case FCVTZS_xd:
+    case FCVTZS_wd:
+    case FCVTZS_xs:
+    case FCVTZS_ws: mnemonic = "fcvtzs"; form = form_rf; break;
+    case FCVTPU_xd:
+    case FCVTPU_ws:
+    case FCVTPU_wd:
+    case FCVTPU_xs: mnemonic = "fcvtpu"; form = form_rf; break;
+    case FCVTPS_xd:
+    case FCVTPS_wd:
+    case FCVTPS_xs:
+    case FCVTPS_ws: mnemonic = "fcvtps"; form = form_rf; break;
+    case SCVTF_sw:
+    case SCVTF_sx:
+    case SCVTF_dw:
+    case SCVTF_dx: mnemonic = "scvtf"; form = form_fr; break;
+    case UCVTF_sw:
+    case UCVTF_sx:
+    case UCVTF_dw:
+    case UCVTF_dx: mnemonic = "ucvtf"; form = form_fr; break;
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Fn, 'IFPFBits";
+  const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
+
+  switch (instr->Mask(FPFixedPointConvertMask)) {
+    case FCVTZS_ws_fixed:
+    case FCVTZS_xs_fixed:
+    case FCVTZS_wd_fixed:
+    case FCVTZS_xd_fixed: mnemonic = "fcvtzs"; break;
+    case FCVTZU_ws_fixed:
+    case FCVTZU_xs_fixed:
+    case FCVTZU_wd_fixed:
+    case FCVTZU_xd_fixed: mnemonic = "fcvtzu"; break;
+    case SCVTF_sw_fixed:
+    case SCVTF_sx_fixed:
+    case SCVTF_dw_fixed:
+    case SCVTF_dx_fixed: mnemonic = "scvtf"; form = form_fr; break;
+    case UCVTF_sw_fixed:
+    case UCVTF_sx_fixed:
+    case UCVTF_dw_fixed:
+    case UCVTF_dx_fixed: mnemonic = "ucvtf"; form = form_fr; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitSystem(const Instruction* instr) {
+  // Some system instructions hijack their Op and Cp fields to represent a
+  // range of immediates instead of indicating a different instruction. This
+  // makes the decoding tricky.
+  const char *mnemonic = "unimplemented";
+  const char *form = "(System)";
+
+  if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
+    switch (instr->Mask(SystemExclusiveMonitorMask)) {
+      case CLREX: {
+        mnemonic = "clrex";
+        form = (instr->CRm() == 0xf) ? NULL : "'IX";
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
+    switch (instr->Mask(SystemSysRegMask)) {
+      case MRS: {
+        mnemonic = "mrs";
+        switch (instr->ImmSystemRegister()) {
+          case NZCV: form = "'Xt, nzcv"; break;
+          case FPCR: form = "'Xt, fpcr"; break;
+          default: form = "'Xt, (unknown)"; break;
+        }
+        break;
+      }
+      case MSR: {
+        mnemonic = "msr";
+        switch (instr->ImmSystemRegister()) {
+          case NZCV: form = "nzcv, 'Xt"; break;
+          case FPCR: form = "fpcr, 'Xt"; break;
+          default: form = "(unknown), 'Xt"; break;
+        }
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
+    switch (instr->ImmHint()) {
+      case NOP: {
+        mnemonic = "nop";
+        form = NULL;
+        break;
+      }
+    }
+  } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
+    switch (instr->Mask(MemBarrierMask)) {
+      case DMB: {
+        mnemonic = "dmb";
+        form = "'M";
+        break;
+      }
+      case DSB: {
+        mnemonic = "dsb";
+        form = "'M";
+        break;
+      }
+      case ISB: {
+        mnemonic = "isb";
+        form = NULL;
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemSysFMask) == SystemSysFixed) {
+    switch (instr->SysOp()) {
+      case IVAU:
+        mnemonic = "ic";
+        form = "ivau, 'Xt";
+        break;
+      case CVAC:
+        mnemonic = "dc";
+        form = "cvac, 'Xt";
+        break;
+      case CVAU:
+        mnemonic = "dc";
+        form = "cvau, 'Xt";
+        break;
+      case CIVAC:
+        mnemonic = "dc";
+        form = "civac, 'Xt";
+        break;
+      case ZVA:
+        mnemonic = "dc";
+        form = "zva, 'Xt";
+        break;
+      default:
+        mnemonic = "sys";
+        if (instr->Rt() == 31) {
+          form = "'G1, 'Kn, 'Km, 'G2";
+        } else {
+          form = "'G1, 'Kn, 'Km, 'G2, 'Xt";
+        }
+        break;
+      }
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitException(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'IDebug";
+
+  switch (instr->Mask(ExceptionMask)) {
+    case HLT: mnemonic = "hlt"; break;
+    case BRK: mnemonic = "brk"; break;
+    case SVC: mnemonic = "svc"; break;
+    case HVC: mnemonic = "hvc"; break;
+    case SMC: mnemonic = "smc"; break;
+    case DCPS1: mnemonic = "dcps1"; form = "{'IDebug}"; break;
+    case DCPS2: mnemonic = "dcps2"; form = "{'IDebug}"; break;
+    case DCPS3: mnemonic = "dcps3"; form = "{'IDebug}"; break;
+    default: form = "(Exception)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCrypto2RegSHA(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCrypto3RegSHA(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCryptoAES(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitNEON2RegMisc(const Instruction* instr) {
+  const char *mnemonic       = "unimplemented";
+  const char *form           = "'Vd.%s, 'Vn.%s";
+  const char *form_cmp_zero  = "'Vd.%s, 'Vn.%s, #0";
+  const char *form_fcmp_zero = "'Vd.%s, 'Vn.%s, #0.0";
+  NEONFormatDecoder nfd(instr);
+
+  static const NEONFormatMap map_lp_ta = {
+    {23, 22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+  };
+
+  static const NEONFormatMap map_cvt_ta = {
+    {22}, {NF_4S, NF_2D}
+  };
+
+  static const NEONFormatMap map_cvt_tb = {
+    {22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S}
+  };
+
+  if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_opcode) {
+    // These instructions all use a two bit size field, except NOT and RBIT,
+    // which use the field to encode the operation.
+    switch (instr->Mask(NEON2RegMiscMask)) {
+      case NEON_REV64:     mnemonic = "rev64"; break;
+      case NEON_REV32:     mnemonic = "rev32"; break;
+      case NEON_REV16:     mnemonic = "rev16"; break;
+      case NEON_SADDLP:
+        mnemonic = "saddlp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_UADDLP:
+        mnemonic = "uaddlp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_SUQADD:    mnemonic = "suqadd"; break;
+      case NEON_USQADD:    mnemonic = "usqadd"; break;
+      case NEON_CLS:       mnemonic = "cls"; break;
+      case NEON_CLZ:       mnemonic = "clz"; break;
+      case NEON_CNT:       mnemonic = "cnt"; break;
+      case NEON_SADALP:
+        mnemonic = "sadalp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_UADALP:
+        mnemonic = "uadalp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_SQABS:     mnemonic = "sqabs"; break;
+      case NEON_SQNEG:     mnemonic = "sqneg"; break;
+      case NEON_CMGT_zero: mnemonic = "cmgt"; form = form_cmp_zero; break;
+      case NEON_CMGE_zero: mnemonic = "cmge"; form = form_cmp_zero; break;
+      case NEON_CMEQ_zero: mnemonic = "cmeq"; form = form_cmp_zero; break;
+      case NEON_CMLE_zero: mnemonic = "cmle"; form = form_cmp_zero; break;
+      case NEON_CMLT_zero: mnemonic = "cmlt"; form = form_cmp_zero; break;
+      case NEON_ABS:       mnemonic = "abs"; break;
+      case NEON_NEG:       mnemonic = "neg"; break;
+      case NEON_RBIT_NOT:
+        switch (instr->FPType()) {
+          case 0: mnemonic = "mvn"; break;
+          case 1: mnemonic = "rbit"; break;
+          default: form = "(NEON2RegMisc)";
+        }
+        nfd.SetFormatMaps(nfd.LogicalFormatMap());
+        break;
+    }
+  } else {
+    // These instructions all use a one bit size field, except XTN, SQXTUN,
+    // SHLL, SQXTN and UQXTN, which use a two bit size field.
+    nfd.SetFormatMaps(nfd.FPFormatMap());
+    switch (instr->Mask(NEON2RegMiscFPMask)) {
+      case NEON_FABS:   mnemonic = "fabs"; break;
+      case NEON_FNEG:   mnemonic = "fneg"; break;
+      case NEON_FCVTN:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtn2" : "fcvtn";
+        nfd.SetFormatMap(0, &map_cvt_tb);
+        nfd.SetFormatMap(1, &map_cvt_ta);
+        break;
+      case NEON_FCVTXN:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtxn2" : "fcvtxn";
+        nfd.SetFormatMap(0, &map_cvt_tb);
+        nfd.SetFormatMap(1, &map_cvt_ta);
+        break;
+      case NEON_FCVTL:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtl2" : "fcvtl";
+        nfd.SetFormatMap(0, &map_cvt_ta);
+        nfd.SetFormatMap(1, &map_cvt_tb);
+        break;
+      case NEON_FRINTN: mnemonic = "frintn"; break;
+      case NEON_FRINTA: mnemonic = "frinta"; break;
+      case NEON_FRINTP: mnemonic = "frintp"; break;
+      case NEON_FRINTM: mnemonic = "frintm"; break;
+      case NEON_FRINTX: mnemonic = "frintx"; break;
+      case NEON_FRINTZ: mnemonic = "frintz"; break;
+      case NEON_FRINTI: mnemonic = "frinti"; break;
+      case NEON_FCVTNS: mnemonic = "fcvtns"; break;
+      case NEON_FCVTNU: mnemonic = "fcvtnu"; break;
+      case NEON_FCVTPS: mnemonic = "fcvtps"; break;
+      case NEON_FCVTPU: mnemonic = "fcvtpu"; break;
+      case NEON_FCVTMS: mnemonic = "fcvtms"; break;
+      case NEON_FCVTMU: mnemonic = "fcvtmu"; break;
+      case NEON_FCVTZS: mnemonic = "fcvtzs"; break;
+      case NEON_FCVTZU: mnemonic = "fcvtzu"; break;
+      case NEON_FCVTAS: mnemonic = "fcvtas"; break;
+      case NEON_FCVTAU: mnemonic = "fcvtau"; break;
+      case NEON_FSQRT:  mnemonic = "fsqrt"; break;
+      case NEON_SCVTF:  mnemonic = "scvtf"; break;
+      case NEON_UCVTF:  mnemonic = "ucvtf"; break;
+      case NEON_URSQRTE: mnemonic = "ursqrte"; break;
+      case NEON_URECPE:  mnemonic = "urecpe";  break;
+      case NEON_FRSQRTE: mnemonic = "frsqrte"; break;
+      case NEON_FRECPE:  mnemonic = "frecpe";  break;
+      case NEON_FCMGT_zero: mnemonic = "fcmgt"; form = form_fcmp_zero; break;
+      case NEON_FCMGE_zero: mnemonic = "fcmge"; form = form_fcmp_zero; break;
+      case NEON_FCMEQ_zero: mnemonic = "fcmeq"; form = form_fcmp_zero; break;
+      case NEON_FCMLE_zero: mnemonic = "fcmle"; form = form_fcmp_zero; break;
+      case NEON_FCMLT_zero: mnemonic = "fcmlt"; form = form_fcmp_zero; break;
+      default:
+        if ((NEON_XTN_opcode <= instr->Mask(NEON2RegMiscOpcode)) &&
+            (instr->Mask(NEON2RegMiscOpcode) <= NEON_UQXTN_opcode)) {
+          nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+          nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+
+          switch (instr->Mask(NEON2RegMiscMask)) {
+            case NEON_XTN:    mnemonic = "xtn"; break;
+            case NEON_SQXTN:  mnemonic = "sqxtn"; break;
+            case NEON_UQXTN:  mnemonic = "uqxtn"; break;
+            case NEON_SQXTUN: mnemonic = "sqxtun"; break;
+            case NEON_SHLL:
+              mnemonic = "shll";
+              nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+              nfd.SetFormatMap(1, nfd.IntegerFormatMap());
+              switch (instr->NEONSize()) {
+                case 0: form = "'Vd.%s, 'Vn.%s, #8"; break;
+                case 1: form = "'Vd.%s, 'Vn.%s, #16"; break;
+                case 2: form = "'Vd.%s, 'Vn.%s, #32"; break;
+                default: form = "(NEON2RegMisc)";
+              }
+          }
+          Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+          return;
+        } else {
+          form = "(NEON2RegMisc)";
+        }
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Same(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+  NEONFormatDecoder nfd(instr);
+
+  if (instr->Mask(NEON3SameLogicalFMask) == NEON3SameLogicalFixed) {
+    switch (instr->Mask(NEON3SameLogicalMask)) {
+      case NEON_AND: mnemonic = "and"; break;
+      case NEON_ORR:
+        mnemonic = "orr";
+        if (instr->Rm() == instr->Rn()) {
+          mnemonic = "mov";
+          form = "'Vd.%s, 'Vn.%s";
+        }
+        break;
+      case NEON_ORN: mnemonic = "orn"; break;
+      case NEON_EOR: mnemonic = "eor"; break;
+      case NEON_BIC: mnemonic = "bic"; break;
+      case NEON_BIF: mnemonic = "bif"; break;
+      case NEON_BIT: mnemonic = "bit"; break;
+      case NEON_BSL: mnemonic = "bsl"; break;
+      default: form = "(NEON3Same)";
+    }
+    nfd.SetFormatMaps(nfd.LogicalFormatMap());
+  } else {
+    static const char *mnemonics[] = {
+        "shadd", "uhadd", "shadd", "uhadd",
+        "sqadd", "uqadd", "sqadd", "uqadd",
+        "srhadd", "urhadd", "srhadd", "urhadd",
+        NULL, NULL, NULL, NULL,  // Handled by logical cases above.
+        "shsub", "uhsub", "shsub", "uhsub",
+        "sqsub", "uqsub", "sqsub", "uqsub",
+        "cmgt", "cmhi", "cmgt", "cmhi",
+        "cmge", "cmhs", "cmge", "cmhs",
+        "sshl", "ushl", "sshl", "ushl",
+        "sqshl", "uqshl", "sqshl", "uqshl",
+        "srshl", "urshl", "srshl", "urshl",
+        "sqrshl", "uqrshl", "sqrshl", "uqrshl",
+        "smax", "umax", "smax", "umax",
+        "smin", "umin", "smin", "umin",
+        "sabd", "uabd", "sabd", "uabd",
+        "saba", "uaba", "saba", "uaba",
+        "add", "sub", "add", "sub",
+        "cmtst", "cmeq", "cmtst", "cmeq",
+        "mla", "mls", "mla", "mls",
+        "mul", "pmul", "mul", "pmul",
+        "smaxp", "umaxp", "smaxp", "umaxp",
+        "sminp", "uminp", "sminp", "uminp",
+        "sqdmulh", "sqrdmulh", "sqdmulh", "sqrdmulh",
+        "addp", "unallocated", "addp", "unallocated",
+        "fmaxnm", "fmaxnmp", "fminnm", "fminnmp",
+        "fmla", "unallocated", "fmls", "unallocated",
+        "fadd", "faddp", "fsub", "fabd",
+        "fmulx", "fmul", "unallocated", "unallocated",
+        "fcmeq", "fcmge", "unallocated", "fcmgt",
+        "unallocated", "facge", "unallocated", "facgt",
+        "fmax", "fmaxp", "fmin", "fminp",
+        "frecps", "fdiv", "frsqrts", "unallocated"};
+
+    // Operation is determined by the opcode bits (15-11), the top bit of
+    // size (23) and the U bit (29).
+    unsigned index = (instr->Bits(15, 11) << 2) | (instr->Bit(23) << 1) |
+                     instr->Bit(29);
+    VIXL_ASSERT(index < (sizeof(mnemonics) / sizeof(mnemonics[0])));
+    mnemonic = mnemonics[index];
+    // Assert that index is not one of the previously handled logical
+    // instructions.
+    VIXL_ASSERT(mnemonic != NULL);
+
+    if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
+      nfd.SetFormatMaps(nfd.FPFormatMap());
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Different(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+
+  NEONFormatDecoder nfd(instr);
+  nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+
+  // Ignore the Q bit. Appending a "2" suffix is handled later.
+  switch (instr->Mask(NEON3DifferentMask) & ~NEON_Q) {
+    case NEON_PMULL:    mnemonic = "pmull";   break;
+    case NEON_SABAL:    mnemonic = "sabal";   break;
+    case NEON_SABDL:    mnemonic = "sabdl";   break;
+    case NEON_SADDL:    mnemonic = "saddl";   break;
+    case NEON_SMLAL:    mnemonic = "smlal";   break;
+    case NEON_SMLSL:    mnemonic = "smlsl";   break;
+    case NEON_SMULL:    mnemonic = "smull";   break;
+    case NEON_SSUBL:    mnemonic = "ssubl";   break;
+    case NEON_SQDMLAL:  mnemonic = "sqdmlal"; break;
+    case NEON_SQDMLSL:  mnemonic = "sqdmlsl"; break;
+    case NEON_SQDMULL:  mnemonic = "sqdmull"; break;
+    case NEON_UABAL:    mnemonic = "uabal";   break;
+    case NEON_UABDL:    mnemonic = "uabdl";   break;
+    case NEON_UADDL:    mnemonic = "uaddl";   break;
+    case NEON_UMLAL:    mnemonic = "umlal";   break;
+    case NEON_UMLSL:    mnemonic = "umlsl";   break;
+    case NEON_UMULL:    mnemonic = "umull";   break;
+    case NEON_USUBL:    mnemonic = "usubl";   break;
+    case NEON_SADDW:
+      mnemonic = "saddw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_SSUBW:
+      mnemonic = "ssubw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_UADDW:
+      mnemonic = "uaddw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_USUBW:
+      mnemonic = "usubw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_ADDHN:
+      mnemonic = "addhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_RADDHN:
+      mnemonic = "raddhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_RSUBHN:
+      mnemonic = "rsubhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_SUBHN:
+      mnemonic = "subhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    default: form = "(NEON3Different)";
+  }
+  Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONAcrossLanes(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, 'Vn.%s";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap(),
+                               NEONFormatDecoder::IntegerFormatMap());
+
+  if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+    nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+    nfd.SetFormatMap(1, nfd.FPFormatMap());
+    switch (instr->Mask(NEONAcrossLanesFPMask)) {
+      case NEON_FMAXV: mnemonic = "fmaxv"; break;
+      case NEON_FMINV: mnemonic = "fminv"; break;
+      case NEON_FMAXNMV: mnemonic = "fmaxnmv"; break;
+      case NEON_FMINNMV: mnemonic = "fminnmv"; break;
+      default: form = "(NEONAcrossLanes)"; break;
+    }
+  } else if (instr->Mask(NEONAcrossLanesFMask) == NEONAcrossLanesFixed) {
+    switch (instr->Mask(NEONAcrossLanesMask)) {
+      case NEON_ADDV:  mnemonic = "addv"; break;
+      case NEON_SMAXV: mnemonic = "smaxv"; break;
+      case NEON_SMINV: mnemonic = "sminv"; break;
+      case NEON_UMAXV: mnemonic = "umaxv"; break;
+      case NEON_UMINV: mnemonic = "uminv"; break;
+      case NEON_SADDLV:
+        mnemonic = "saddlv";
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        break;
+      case NEON_UADDLV:
+        mnemonic = "uaddlv";
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        break;
+      default: form = "(NEONAcrossLanes)"; break;
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form,
+      NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONByIndexedElement(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  bool l_instr = false;
+  bool fp_instr = false;
+
+  const char *form = "'Vd.%s, 'Vn.%s, 'Ve.%s['IVByElemIndex]";
+
+  static const NEONFormatMap map_ta = {
+    {23, 22}, {NF_UNDEF, NF_4S, NF_2D}
+  };
+  NEONFormatDecoder nfd(instr, &map_ta,
+                        NEONFormatDecoder::IntegerFormatMap(),
+                        NEONFormatDecoder::ScalarFormatMap());
+
+  switch (instr->Mask(NEONByIndexedElementMask)) {
+    case NEON_SMULL_byelement:    mnemonic = "smull"; l_instr = true; break;
+    case NEON_UMULL_byelement:    mnemonic = "umull"; l_instr = true; break;
+    case NEON_SMLAL_byelement:    mnemonic = "smlal"; l_instr = true; break;
+    case NEON_UMLAL_byelement:    mnemonic = "umlal"; l_instr = true; break;
+    case NEON_SMLSL_byelement:    mnemonic = "smlsl"; l_instr = true; break;
+    case NEON_UMLSL_byelement:    mnemonic = "umlsl"; l_instr = true; break;
+    case NEON_SQDMULL_byelement:  mnemonic = "sqdmull"; l_instr = true; break;
+    case NEON_SQDMLAL_byelement:  mnemonic = "sqdmlal"; l_instr = true; break;
+    case NEON_SQDMLSL_byelement:  mnemonic = "sqdmlsl"; l_instr = true; break;
+    case NEON_MUL_byelement:      mnemonic = "mul"; break;
+    case NEON_MLA_byelement:      mnemonic = "mla"; break;
+    case NEON_MLS_byelement:      mnemonic = "mls"; break;
+    case NEON_SQDMULH_byelement:  mnemonic = "sqdmulh";  break;
+    case NEON_SQRDMULH_byelement: mnemonic = "sqrdmulh"; break;
+    default:
+      switch (instr->Mask(NEONByIndexedElementFPMask)) {
+        case NEON_FMUL_byelement:  mnemonic = "fmul";  fp_instr = true; break;
+        case NEON_FMLA_byelement:  mnemonic = "fmla";  fp_instr = true; break;
+        case NEON_FMLS_byelement:  mnemonic = "fmls";  fp_instr = true; break;
+        case NEON_FMULX_byelement: mnemonic = "fmulx"; fp_instr = true; break;
+      }
+  }
+
+  if (l_instr) {
+    Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+  } else if (fp_instr) {
+    nfd.SetFormatMap(0, nfd.FPFormatMap());
+    Format(instr, mnemonic, nfd.Substitute(form));
+  } else {
+    nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+    Format(instr, mnemonic, nfd.Substitute(form));
+  }
+}
+
+
+void Disassembler::VisitNEONCopy(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONCopy)";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularFormatMap(),
+                               NEONFormatDecoder::TriangularScalarFormatMap());
+
+  if (instr->Mask(NEONCopyInsElementMask) == NEON_INS_ELEMENT) {
+    mnemonic = "mov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    form = "'Vd.%s['IVInsIndex1], 'Vn.%s['IVInsIndex2]";
+  } else if (instr->Mask(NEONCopyInsGeneralMask) == NEON_INS_GENERAL) {
+    mnemonic = "mov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    if (nfd.GetVectorFormat() == kFormatD) {
+      form = "'Vd.%s['IVInsIndex1], 'Xn";
+    } else {
+      form = "'Vd.%s['IVInsIndex1], 'Wn";
+    }
+  } else if (instr->Mask(NEONCopyUmovMask) == NEON_UMOV) {
+    if (instr->Mask(NEON_Q) || ((instr->ImmNEON5() & 7) == 4)) {
+      mnemonic = "mov";
+    } else {
+      mnemonic = "umov";
+    }
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    if (nfd.GetVectorFormat() == kFormatD) {
+      form = "'Xd, 'Vn.%s['IVInsIndex1]";
+    } else {
+      form = "'Wd, 'Vn.%s['IVInsIndex1]";
+    }
+  } else if (instr->Mask(NEONCopySmovMask) == NEON_SMOV) {
+    mnemonic = "smov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    form = "'Rdq, 'Vn.%s['IVInsIndex1]";
+  } else if (instr->Mask(NEONCopyDupElementMask) == NEON_DUP_ELEMENT) {
+    mnemonic = "dup";
+    form = "'Vd.%s, 'Vn.%s['IVInsIndex1]";
+  } else if (instr->Mask(NEONCopyDupGeneralMask) == NEON_DUP_GENERAL) {
+    mnemonic = "dup";
+    if (nfd.GetVectorFormat() == kFormat2D) {
+      form = "'Vd.%s, 'Xn";
+    } else {
+      form = "'Vd.%s, 'Wn";
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONExtract(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONExtract)";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LogicalFormatMap());
+  if (instr->Mask(NEONExtractMask) == NEON_EXT) {
+    mnemonic = "ext";
+    form = "'Vd.%s, 'Vn.%s, 'Vm.%s, 'IVExtract";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStruct(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreMultiStruct)";
+  const char *form_1v = "{'Vt.%1$s}, ['Xns]";
+  const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns]";
+  const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns]";
+  const char *form_4v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreMultiStructMask)) {
+    case NEON_LD1_1v: mnemonic = "ld1"; form = form_1v; break;
+    case NEON_LD1_2v: mnemonic = "ld1"; form = form_2v; break;
+    case NEON_LD1_3v: mnemonic = "ld1"; form = form_3v; break;
+    case NEON_LD1_4v: mnemonic = "ld1"; form = form_4v; break;
+    case NEON_LD2: mnemonic = "ld2"; form = form_2v; break;
+    case NEON_LD3: mnemonic = "ld3"; form = form_3v; break;
+    case NEON_LD4: mnemonic = "ld4"; form = form_4v; break;
+    case NEON_ST1_1v: mnemonic = "st1"; form = form_1v; break;
+    case NEON_ST1_2v: mnemonic = "st1"; form = form_2v; break;
+    case NEON_ST1_3v: mnemonic = "st1"; form = form_3v; break;
+    case NEON_ST1_4v: mnemonic = "st1"; form = form_4v; break;
+    case NEON_ST2: mnemonic = "st2"; form = form_2v; break;
+    case NEON_ST3: mnemonic = "st3"; form = form_3v; break;
+    case NEON_ST4: mnemonic = "st4"; form = form_4v; break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStructPostIndex(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreMultiStructPostIndex)";
+  const char *form_1v = "{'Vt.%1$s}, ['Xns], 'Xmr1";
+  const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns], 'Xmr2";
+  const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns], 'Xmr3";
+  const char *form_4v =
+      "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmr4";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreMultiStructPostIndexMask)) {
+    case NEON_LD1_1v_post: mnemonic = "ld1"; form = form_1v; break;
+    case NEON_LD1_2v_post: mnemonic = "ld1"; form = form_2v; break;
+    case NEON_LD1_3v_post: mnemonic = "ld1"; form = form_3v; break;
+    case NEON_LD1_4v_post: mnemonic = "ld1"; form = form_4v; break;
+    case NEON_LD2_post: mnemonic = "ld2"; form = form_2v; break;
+    case NEON_LD3_post: mnemonic = "ld3"; form = form_3v; break;
+    case NEON_LD4_post: mnemonic = "ld4"; form = form_4v; break;
+    case NEON_ST1_1v_post: mnemonic = "st1"; form = form_1v; break;
+    case NEON_ST1_2v_post: mnemonic = "st1"; form = form_2v; break;
+    case NEON_ST1_3v_post: mnemonic = "st1"; form = form_3v; break;
+    case NEON_ST1_4v_post: mnemonic = "st1"; form = form_4v; break;
+    case NEON_ST2_post: mnemonic = "st2"; form = form_2v; break;
+    case NEON_ST3_post: mnemonic = "st3"; form = form_3v; break;
+    case NEON_ST4_post: mnemonic = "st4"; form = form_4v; break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStruct(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreSingleStruct)";
+
+  const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns]";
+  const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns]";
+  const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns]";
+  const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreSingleStructMask)) {
+    case NEON_LD1_b: mnemonic = "ld1"; form = form_1b; break;
+    case NEON_LD1_h: mnemonic = "ld1"; form = form_1h; break;
+    case NEON_LD1_s:
+      mnemonic = "ld1";
+      VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_ST1_b: mnemonic = "st1"; form = form_1b; break;
+    case NEON_ST1_h: mnemonic = "st1"; form = form_1h; break;
+    case NEON_ST1_s:
+      mnemonic = "st1";
+      VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_LD1R:
+      mnemonic = "ld1r";
+      form = "{'Vt.%s}, ['Xns]";
+      break;
+    case NEON_LD2_b:
+    case NEON_ST2_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD2_h:
+    case NEON_ST2_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD2_s:
+    case NEON_ST2_s:
+      VIXL_STATIC_ASSERT((NEON_ST2_s | (1 << NEONLSSize_offset)) == NEON_ST2_d);
+      VIXL_STATIC_ASSERT((NEON_LD2_s | (1 << NEONLSSize_offset)) == NEON_LD2_d);
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD2R:
+      mnemonic = "ld2r";
+      form = "{'Vt.%s, 'Vt2.%s}, ['Xns]";
+      break;
+    case NEON_LD3_b:
+    case NEON_ST3_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD3_h:
+    case NEON_ST3_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD3_s:
+    case NEON_ST3_s:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD3R:
+      mnemonic = "ld3r";
+      form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns]";
+      break;
+    case NEON_LD4_b:
+     case NEON_ST4_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD4_h:
+    case NEON_ST4_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD4_s:
+    case NEON_ST4_s:
+      VIXL_STATIC_ASSERT((NEON_LD4_s | (1 << NEONLSSize_offset)) == NEON_LD4_d);
+      VIXL_STATIC_ASSERT((NEON_ST4_s | (1 << NEONLSSize_offset)) == NEON_ST4_d);
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD4R:
+      mnemonic = "ld4r";
+      form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+      break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStructPostIndex(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreSingleStructPostIndex)";
+
+  const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns], 'Xmb1";
+  const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns], 'Xmb2";
+  const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns], 'Xmb4";
+  const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns], 'Xmb8";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreSingleStructPostIndexMask)) {
+    case NEON_LD1_b_post: mnemonic = "ld1"; form = form_1b; break;
+    case NEON_LD1_h_post: mnemonic = "ld1"; form = form_1h; break;
+    case NEON_LD1_s_post:
+      mnemonic = "ld1";
+      VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_ST1_b_post: mnemonic = "st1"; form = form_1b; break;
+    case NEON_ST1_h_post: mnemonic = "st1"; form = form_1h; break;
+    case NEON_ST1_s_post:
+      mnemonic = "st1";
+      VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_LD1R_post:
+      mnemonic = "ld1r";
+      form = "{'Vt.%s}, ['Xns], 'Xmz1";
+      break;
+    case NEON_LD2_b_post:
+    case NEON_ST2_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns], 'Xmb2";
+      break;
+    case NEON_ST2_h_post:
+    case NEON_LD2_h_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns], 'Xmb4";
+      break;
+    case NEON_LD2_s_post:
+    case NEON_ST2_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns], 'Xmb8";
+      else
+        form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns], 'Xmb16";
+      break;
+    case NEON_LD2R_post:
+      mnemonic = "ld2r";
+      form = "{'Vt.%s, 'Vt2.%s}, ['Xns], 'Xmz2";
+      break;
+    case NEON_LD3_b_post:
+    case NEON_ST3_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns], 'Xmb3";
+      break;
+    case NEON_LD3_h_post:
+    case NEON_ST3_h_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns], 'Xmb6";
+      break;
+    case NEON_LD3_s_post:
+    case NEON_ST3_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns], 'Xmb12";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns], 'Xmr3";
+      break;
+    case NEON_LD3R_post:
+      mnemonic = "ld3r";
+      form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns], 'Xmz3";
+      break;
+    case NEON_LD4_b_post:
+    case NEON_ST4_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns], 'Xmb4";
+      break;
+    case NEON_LD4_h_post:
+    case NEON_ST4_h_post:
+      mnemonic = (instr->LdStXLoad()) == 1 ? "ld4" : "st4";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns], 'Xmb8";
+      break;
+    case NEON_LD4_s_post:
+    case NEON_ST4_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns], 'Xmb16";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns], 'Xmb32";
+      break;
+    case NEON_LD4R_post:
+      mnemonic = "ld4r";
+      form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmz4";
+      break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONModifiedImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vt.%s, 'IVMIImm8, lsl 'IVMIShiftAmt1";
+
+  int cmode   = instr->NEONCmode();
+  int cmode_3 = (cmode >> 3) & 1;
+  int cmode_2 = (cmode >> 2) & 1;
+  int cmode_1 = (cmode >> 1) & 1;
+  int cmode_0 = cmode & 1;
+  int q = instr->NEONQ();
+  int op = instr->NEONModImmOp();
+
+  static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+  static const NEONFormatMap map_h = { {30}, {NF_4H, NF_8H} };
+  static const NEONFormatMap map_s = { {30}, {NF_2S, NF_4S} };
+  NEONFormatDecoder nfd(instr, &map_b);
+
+  if (cmode_3 == 0) {
+    if (cmode_0 == 0) {
+      mnemonic = (op == 1) ? "mvni" : "movi";
+    } else {  // cmode<0> == '1'.
+      mnemonic = (op == 1) ? "bic" : "orr";
+    }
+    nfd.SetFormatMap(0, &map_s);
+  } else {  // cmode<3> == '1'.
+    if (cmode_2 == 0) {
+      if (cmode_0 == 0) {
+        mnemonic = (op == 1) ? "mvni" : "movi";
+      } else {  // cmode<0> == '1'.
+        mnemonic = (op == 1) ? "bic" : "orr";
+      }
+      nfd.SetFormatMap(0, &map_h);
+    } else {  // cmode<2> == '1'.
+      if (cmode_1 == 0) {
+        mnemonic = (op == 1) ? "mvni" : "movi";
+        form = "'Vt.%s, 'IVMIImm8, msl 'IVMIShiftAmt2";
+        nfd.SetFormatMap(0, &map_s);
+      } else {   // cmode<1> == '1'.
+        if (cmode_0 == 0) {
+          mnemonic = "movi";
+          if (op == 0) {
+            form = "'Vt.%s, 'IVMIImm8";
+          } else {
+            form = (q == 0) ? "'Dd, 'IVMIImm" : "'Vt.2d, 'IVMIImm";
+          }
+        } else {  // cmode<0> == '1'
+          mnemonic = "fmov";
+          if (op == 0) {
+            form  = "'Vt.%s, 'IVMIImmFPSingle";
+            nfd.SetFormatMap(0, &map_s);
+          } else {
+            if (q == 1) {
+              form = "'Vt.2d, 'IVMIImmFPDouble";
+            }
+          }
+        }
+      }
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONScalar2RegMisc(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form     = "%sd, %sn";
+  const char *form_0   = "%sd, %sn, #0";
+  const char *form_fp0 = "%sd, %sn, #0.0";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+  if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_scalar_opcode) {
+    // These instructions all use a two bit size field, except NOT and RBIT,
+    // which use the field to encode the operation.
+    switch (instr->Mask(NEONScalar2RegMiscMask)) {
+      case NEON_CMGT_zero_scalar: mnemonic = "cmgt"; form = form_0; break;
+      case NEON_CMGE_zero_scalar: mnemonic = "cmge"; form = form_0; break;
+      case NEON_CMLE_zero_scalar: mnemonic = "cmle"; form = form_0; break;
+      case NEON_CMLT_zero_scalar: mnemonic = "cmlt"; form = form_0; break;
+      case NEON_CMEQ_zero_scalar: mnemonic = "cmeq"; form = form_0; break;
+      case NEON_NEG_scalar:       mnemonic = "neg";   break;
+      case NEON_SQNEG_scalar:     mnemonic = "sqneg"; break;
+      case NEON_ABS_scalar:       mnemonic = "abs";   break;
+      case NEON_SQABS_scalar:     mnemonic = "sqabs"; break;
+      case NEON_SUQADD_scalar:    mnemonic = "suqadd"; break;
+      case NEON_USQADD_scalar:    mnemonic = "usqadd"; break;
+      default: form = "(NEONScalar2RegMisc)";
+    }
+  } else {
+    // These instructions all use a one bit size field, except SQXTUN, SQXTN
+    // and UQXTN, which use a two bit size field.
+    nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+    switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
+      case NEON_FRSQRTE_scalar:    mnemonic = "frsqrte"; break;
+      case NEON_FRECPE_scalar:     mnemonic = "frecpe";  break;
+      case NEON_SCVTF_scalar:      mnemonic = "scvtf"; break;
+      case NEON_UCVTF_scalar:      mnemonic = "ucvtf"; break;
+      case NEON_FCMGT_zero_scalar: mnemonic = "fcmgt"; form = form_fp0; break;
+      case NEON_FCMGE_zero_scalar: mnemonic = "fcmge"; form = form_fp0; break;
+      case NEON_FCMLE_zero_scalar: mnemonic = "fcmle"; form = form_fp0; break;
+      case NEON_FCMLT_zero_scalar: mnemonic = "fcmlt"; form = form_fp0; break;
+      case NEON_FCMEQ_zero_scalar: mnemonic = "fcmeq"; form = form_fp0; break;
+      case NEON_FRECPX_scalar:     mnemonic = "frecpx"; break;
+      case NEON_FCVTNS_scalar:     mnemonic = "fcvtns"; break;
+      case NEON_FCVTNU_scalar:     mnemonic = "fcvtnu"; break;
+      case NEON_FCVTPS_scalar:     mnemonic = "fcvtps"; break;
+      case NEON_FCVTPU_scalar:     mnemonic = "fcvtpu"; break;
+      case NEON_FCVTMS_scalar:     mnemonic = "fcvtms"; break;
+      case NEON_FCVTMU_scalar:     mnemonic = "fcvtmu"; break;
+      case NEON_FCVTZS_scalar:     mnemonic = "fcvtzs"; break;
+      case NEON_FCVTZU_scalar:     mnemonic = "fcvtzu"; break;
+      case NEON_FCVTAS_scalar:     mnemonic = "fcvtas"; break;
+      case NEON_FCVTAU_scalar:     mnemonic = "fcvtau"; break;
+      case NEON_FCVTXN_scalar:
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        mnemonic = "fcvtxn";
+        break;
+      default:
+        nfd.SetFormatMap(0, nfd.ScalarFormatMap());
+        nfd.SetFormatMap(1, nfd.LongScalarFormatMap());
+        switch (instr->Mask(NEONScalar2RegMiscMask)) {
+          case NEON_SQXTN_scalar:  mnemonic = "sqxtn"; break;
+          case NEON_UQXTN_scalar:  mnemonic = "uqxtn"; break;
+          case NEON_SQXTUN_scalar: mnemonic = "sqxtun"; break;
+          default: form = "(NEONScalar2RegMisc)";
+        }
+    }
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Diff(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, %sm";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LongScalarFormatMap(),
+                               NEONFormatDecoder::ScalarFormatMap());
+
+  switch (instr->Mask(NEONScalar3DiffMask)) {
+    case NEON_SQDMLAL_scalar  : mnemonic = "sqdmlal"; break;
+    case NEON_SQDMLSL_scalar  : mnemonic = "sqdmlsl"; break;
+    case NEON_SQDMULL_scalar  : mnemonic = "sqdmull"; break;
+    default: form = "(NEONScalar3Diff)";
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Same(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, %sm";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+  if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
+    nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+    switch (instr->Mask(NEONScalar3SameFPMask)) {
+      case NEON_FACGE_scalar:   mnemonic = "facge"; break;
+      case NEON_FACGT_scalar:   mnemonic = "facgt"; break;
+      case NEON_FCMEQ_scalar:   mnemonic = "fcmeq"; break;
+      case NEON_FCMGE_scalar:   mnemonic = "fcmge"; break;
+      case NEON_FCMGT_scalar:   mnemonic = "fcmgt"; break;
+      case NEON_FMULX_scalar:   mnemonic = "fmulx"; break;
+      case NEON_FRECPS_scalar:  mnemonic = "frecps"; break;
+      case NEON_FRSQRTS_scalar: mnemonic = "frsqrts"; break;
+      case NEON_FABD_scalar:    mnemonic = "fabd"; break;
+      default: form = "(NEONScalar3Same)";
+    }
+  } else {
+    switch (instr->Mask(NEONScalar3SameMask)) {
+      case NEON_ADD_scalar:    mnemonic = "add";    break;
+      case NEON_SUB_scalar:    mnemonic = "sub";    break;
+      case NEON_CMEQ_scalar:   mnemonic = "cmeq";   break;
+      case NEON_CMGE_scalar:   mnemonic = "cmge";   break;
+      case NEON_CMGT_scalar:   mnemonic = "cmgt";   break;
+      case NEON_CMHI_scalar:   mnemonic = "cmhi";   break;
+      case NEON_CMHS_scalar:   mnemonic = "cmhs";   break;
+      case NEON_CMTST_scalar:  mnemonic = "cmtst";  break;
+      case NEON_UQADD_scalar:  mnemonic = "uqadd";  break;
+      case NEON_SQADD_scalar:  mnemonic = "sqadd";  break;
+      case NEON_UQSUB_scalar:  mnemonic = "uqsub";  break;
+      case NEON_SQSUB_scalar:  mnemonic = "sqsub";  break;
+      case NEON_USHL_scalar:   mnemonic = "ushl";   break;
+      case NEON_SSHL_scalar:   mnemonic = "sshl";   break;
+      case NEON_UQSHL_scalar:  mnemonic = "uqshl";  break;
+      case NEON_SQSHL_scalar:  mnemonic = "sqshl";  break;
+      case NEON_URSHL_scalar:  mnemonic = "urshl";  break;
+      case NEON_SRSHL_scalar:  mnemonic = "srshl";  break;
+      case NEON_UQRSHL_scalar: mnemonic = "uqrshl"; break;
+      case NEON_SQRSHL_scalar: mnemonic = "sqrshl"; break;
+      case NEON_SQDMULH_scalar:  mnemonic = "sqdmulh";  break;
+      case NEON_SQRDMULH_scalar: mnemonic = "sqrdmulh"; break;
+      default: form = "(NEONScalar3Same)";
+    }
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalarByIndexedElement(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, 'Ve.%s['IVByElemIndex]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+  bool long_instr = false;
+
+  switch (instr->Mask(NEONScalarByIndexedElementMask)) {
+    case NEON_SQDMULL_byelement_scalar:
+      mnemonic = "sqdmull";
+      long_instr = true;
+      break;
+    case NEON_SQDMLAL_byelement_scalar:
+      mnemonic = "sqdmlal";
+      long_instr = true;
+      break;
+    case NEON_SQDMLSL_byelement_scalar:
+      mnemonic = "sqdmlsl";
+      long_instr = true;
+      break;
+    case NEON_SQDMULH_byelement_scalar:
+      mnemonic = "sqdmulh";
+      break;
+    case NEON_SQRDMULH_byelement_scalar:
+      mnemonic = "sqrdmulh";
+      break;
+    default:
+      nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+      switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
+        case NEON_FMUL_byelement_scalar: mnemonic = "fmul"; break;
+        case NEON_FMLA_byelement_scalar: mnemonic = "fmla"; break;
+        case NEON_FMLS_byelement_scalar: mnemonic = "fmls"; break;
+        case NEON_FMULX_byelement_scalar: mnemonic = "fmulx"; break;
+        default: form = "(NEONScalarByIndexedElement)";
+      }
+  }
+
+  if (long_instr) {
+    nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(
+    form, nfd.kPlaceholder, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarCopy(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONScalarCopy)";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularScalarFormatMap());
+
+  if (instr->Mask(NEONScalarCopyMask) == NEON_DUP_ELEMENT_scalar) {
+    mnemonic = "mov";
+    form = "%sd, 'Vn.%s['IVInsIndex1]";
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarPairwise(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, 'Vn.%s";
+  NEONFormatMap map = { {22}, {NF_2S, NF_2D} };
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::FPScalarFormatMap(), &map);
+
+  switch (instr->Mask(NEONScalarPairwiseMask)) {
+    case NEON_ADDP_scalar:    mnemonic = "addp"; break;
+    case NEON_FADDP_scalar:   mnemonic = "faddp"; break;
+    case NEON_FMAXP_scalar:   mnemonic = "fmaxp"; break;
+    case NEON_FMAXNMP_scalar: mnemonic = "fmaxnmp"; break;
+    case NEON_FMINP_scalar:   mnemonic = "fminp"; break;
+    case NEON_FMINNMP_scalar: mnemonic = "fminnmp"; break;
+    default: form = "(NEONScalarPairwise)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form,
+      NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarShiftImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form   = "%sd, %sn, 'Is1";
+  const char *form_2 = "%sd, %sn, 'Is2";
+
+  static const NEONFormatMap map_shift = {
+    {22, 21, 20, 19},
+    {NF_UNDEF, NF_B, NF_H, NF_H, NF_S, NF_S, NF_S, NF_S,
+     NF_D,     NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D}
+  };
+  static const NEONFormatMap map_shift_narrow = {
+    {21, 20, 19},
+    {NF_UNDEF, NF_H, NF_S, NF_S, NF_D, NF_D, NF_D, NF_D}
+  };
+  NEONFormatDecoder nfd(instr, &map_shift);
+
+  if (instr->ImmNEONImmh()) {  // immh has to be non-zero.
+    switch (instr->Mask(NEONScalarShiftImmediateMask)) {
+      case NEON_FCVTZU_imm_scalar: mnemonic = "fcvtzu";    break;
+      case NEON_FCVTZS_imm_scalar: mnemonic = "fcvtzs";   break;
+      case NEON_SCVTF_imm_scalar: mnemonic = "scvtf";    break;
+      case NEON_UCVTF_imm_scalar: mnemonic = "ucvtf";   break;
+      case NEON_SRI_scalar:       mnemonic = "sri";    break;
+      case NEON_SSHR_scalar:      mnemonic = "sshr";   break;
+      case NEON_USHR_scalar:      mnemonic = "ushr";   break;
+      case NEON_SRSHR_scalar:     mnemonic = "srshr";  break;
+      case NEON_URSHR_scalar:     mnemonic = "urshr";  break;
+      case NEON_SSRA_scalar:      mnemonic = "ssra";   break;
+      case NEON_USRA_scalar:      mnemonic = "usra";   break;
+      case NEON_SRSRA_scalar:     mnemonic = "srsra";  break;
+      case NEON_URSRA_scalar:     mnemonic = "ursra";  break;
+      case NEON_SHL_scalar:       mnemonic = "shl";    form = form_2; break;
+      case NEON_SLI_scalar:       mnemonic = "sli";    form = form_2; break;
+      case NEON_SQSHLU_scalar:    mnemonic = "sqshlu"; form = form_2; break;
+      case NEON_SQSHL_imm_scalar: mnemonic = "sqshl";  form = form_2; break;
+      case NEON_UQSHL_imm_scalar: mnemonic = "uqshl";  form = form_2; break;
+      case NEON_UQSHRN_scalar:
+        mnemonic = "uqshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_UQRSHRN_scalar:
+        mnemonic = "uqrshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQSHRN_scalar:
+        mnemonic = "sqshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQRSHRN_scalar:
+        mnemonic = "sqrshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQSHRUN_scalar:
+        mnemonic = "sqshrun";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQRSHRUN_scalar:
+        mnemonic = "sqrshrun";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      default:
+        form = "(NEONScalarShiftImmediate)";
+    }
+  } else {
+    form = "(NEONScalarShiftImmediate)";
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONShiftImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form         = "'Vd.%s, 'Vn.%s, 'Is1";
+  const char *form_shift_2 = "'Vd.%s, 'Vn.%s, 'Is2";
+  const char *form_xtl     = "'Vd.%s, 'Vn.%s";
+
+  // 0001->8H, 001x->4S, 01xx->2D, all others undefined.
+  static const NEONFormatMap map_shift_ta = {
+    {22, 21, 20, 19},
+    {NF_UNDEF, NF_8H, NF_4S, NF_4S, NF_2D, NF_2D, NF_2D, NF_2D}
+  };
+
+  // 00010->8B, 00011->16B, 001x0->4H, 001x1->8H,
+  // 01xx0->2S, 01xx1->4S, 1xxx1->2D, all others undefined.
+  static const NEONFormatMap map_shift_tb = {
+    {22, 21, 20, 19, 30},
+    {NF_UNDEF, NF_UNDEF, NF_8B,    NF_16B, NF_4H,    NF_8H, NF_4H,    NF_8H,
+     NF_2S,    NF_4S,    NF_2S,    NF_4S,  NF_2S,    NF_4S, NF_2S,    NF_4S,
+     NF_UNDEF, NF_2D,    NF_UNDEF, NF_2D,  NF_UNDEF, NF_2D, NF_UNDEF, NF_2D,
+     NF_UNDEF, NF_2D,    NF_UNDEF, NF_2D,  NF_UNDEF, NF_2D, NF_UNDEF, NF_2D}
+  };
+
+  NEONFormatDecoder nfd(instr, &map_shift_tb);
+
+  if (instr->ImmNEONImmh()) {  // immh has to be non-zero.
+    switch (instr->Mask(NEONShiftImmediateMask)) {
+      case NEON_SQSHLU:     mnemonic = "sqshlu"; form = form_shift_2; break;
+      case NEON_SQSHL_imm:  mnemonic = "sqshl";  form = form_shift_2; break;
+      case NEON_UQSHL_imm:  mnemonic = "uqshl";  form = form_shift_2; break;
+      case NEON_SHL:        mnemonic = "shl";    form = form_shift_2; break;
+      case NEON_SLI:        mnemonic = "sli";    form = form_shift_2; break;
+      case NEON_SCVTF_imm:  mnemonic = "scvtf";  break;
+      case NEON_UCVTF_imm:  mnemonic = "ucvtf";  break;
+      case NEON_FCVTZU_imm: mnemonic = "fcvtzu"; break;
+      case NEON_FCVTZS_imm: mnemonic = "fcvtzs"; break;
+      case NEON_SRI:        mnemonic = "sri";    break;
+      case NEON_SSHR:       mnemonic = "sshr";   break;
+      case NEON_USHR:       mnemonic = "ushr";   break;
+      case NEON_SRSHR:      mnemonic = "srshr";  break;
+      case NEON_URSHR:      mnemonic = "urshr";  break;
+      case NEON_SSRA:       mnemonic = "ssra";   break;
+      case NEON_USRA:       mnemonic = "usra";   break;
+      case NEON_SRSRA:      mnemonic = "srsra";  break;
+      case NEON_URSRA:      mnemonic = "ursra";  break;
+      case NEON_SHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "shrn2" : "shrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_RSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "rshrn2" : "rshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_UQSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "uqshrn2" : "uqshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_UQRSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "uqrshrn2" : "uqrshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqshrn2" : "sqshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQRSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqrshrn2" : "sqrshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQSHRUN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqshrun2" : "sqshrun";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQRSHRUN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqrshrun2" : "sqrshrun";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SSHLL:
+        nfd.SetFormatMap(0, &map_shift_ta);
+        if (instr->ImmNEONImmb() == 0 &&
+            CountSetBits(instr->ImmNEONImmh(), 32) == 1) {  // sxtl variant.
+          form = form_xtl;
+          mnemonic = instr->Mask(NEON_Q) ? "sxtl2" : "sxtl";
+        } else {  // sshll variant.
+          form = form_shift_2;
+          mnemonic = instr->Mask(NEON_Q) ? "sshll2" : "sshll";
+        }
+        break;
+      case NEON_USHLL:
+        nfd.SetFormatMap(0, &map_shift_ta);
+        if (instr->ImmNEONImmb() == 0 &&
+            CountSetBits(instr->ImmNEONImmh(), 32) == 1) {  // uxtl variant.
+          form = form_xtl;
+          mnemonic = instr->Mask(NEON_Q) ? "uxtl2" : "uxtl";
+        } else {  // ushll variant.
+          form = form_shift_2;
+          mnemonic = instr->Mask(NEON_Q) ? "ushll2" : "ushll";
+        }
+        break;
+      default: form = "(NEONShiftImmediate)";
+    }
+  } else {
+    form = "(NEONShiftImmediate)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONTable(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONTable)";
+  const char form_1v[] = "'Vd.%%s, {'Vn.16b}, 'Vm.%%s";
+  const char form_2v[] = "'Vd.%%s, {'Vn.16b, v%d.16b}, 'Vm.%%s";
+  const char form_3v[] = "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+  const char form_4v[] =
+      "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+  static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+  NEONFormatDecoder nfd(instr, &map_b);
+
+  switch (instr->Mask(NEONTableMask)) {
+    case NEON_TBL_1v: mnemonic = "tbl"; form = form_1v; break;
+    case NEON_TBL_2v: mnemonic = "tbl"; form = form_2v; break;
+    case NEON_TBL_3v: mnemonic = "tbl"; form = form_3v; break;
+    case NEON_TBL_4v: mnemonic = "tbl"; form = form_4v; break;
+    case NEON_TBX_1v: mnemonic = "tbx"; form = form_1v; break;
+    case NEON_TBX_2v: mnemonic = "tbx"; form = form_2v; break;
+    case NEON_TBX_3v: mnemonic = "tbx"; form = form_3v; break;
+    case NEON_TBX_4v: mnemonic = "tbx"; form = form_4v; break;
+    default: break;
+  }
+
+  char re_form[sizeof(form_4v) + 6];
+  int reg_num = instr->Rn();
+  snprintf(re_form, sizeof(re_form), form,
+           (reg_num + 1) % kNumberOfVRegisters,
+           (reg_num + 2) % kNumberOfVRegisters,
+           (reg_num + 3) % kNumberOfVRegisters);
+
+  Format(instr, mnemonic, nfd.Substitute(re_form));
+}
+
+
+void Disassembler::VisitNEONPerm(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+  NEONFormatDecoder nfd(instr);
+
+  switch (instr->Mask(NEONPermMask)) {
+    case NEON_TRN1: mnemonic = "trn1";   break;
+    case NEON_TRN2: mnemonic = "trn2";  break;
+    case NEON_UZP1: mnemonic = "uzp1"; break;
+    case NEON_UZP2: mnemonic = "uzp2";  break;
+    case NEON_ZIP1: mnemonic = "zip1"; break;
+    case NEON_ZIP2: mnemonic = "zip2"; break;
+    default: form = "(NEONPerm)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitUnimplemented(const Instruction* instr) {
+  Format(instr, "unimplemented", "(Unimplemented)");
+}
+
+
+void Disassembler::VisitUnallocated(const Instruction* instr) {
+  Format(instr, "unallocated", "(Unallocated)");
+}
+
+
+void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
+  // The base disasm does nothing more than disassembling into a buffer.
+}
+
+
+void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
+                                              const CPURegister& reg) {
+  USE(instr);
+  VIXL_ASSERT(reg.IsValid());
+  char reg_char;
+
+  if (reg.IsRegister()) {
+    reg_char = reg.Is64Bits() ? 'x' : 'w';
+  } else {
+    VIXL_ASSERT(reg.IsVRegister());
+    switch (reg.SizeInBits()) {
+      case kBRegSize: reg_char = 'b'; break;
+      case kHRegSize: reg_char = 'h'; break;
+      case kSRegSize: reg_char = 's'; break;
+      case kDRegSize: reg_char = 'd'; break;
+      default:
+        VIXL_ASSERT(reg.Is128Bits());
+        reg_char = 'q';
+    }
+  }
+
+  if (reg.IsVRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
+    // A core or scalar/vector register: [wx]0 - 30, [bhsdq]0 - 31.
+    AppendToOutput("%c%d", reg_char, reg.code());
+  } else if (reg.Aliases(sp)) {
+    // Disassemble w31/x31 as stack pointer wsp/sp.
+    AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
+  } else {
+    // Disassemble w31/x31 as zero register wzr/xzr.
+    AppendToOutput("%czr", reg_char);
+  }
+}
+
+
+void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
+                                                  int64_t offset) {
+  USE(instr);
+  uint64_t abs_offset = offset;
+  char sign = (offset < 0) ? '-' : '+';
+  if (offset < 0) {
+    abs_offset = -abs_offset;
+  }
+  AppendToOutput("#%c0x%" PRIx64, sign, abs_offset);
+}
+
+
+void Disassembler::AppendAddressToOutput(const Instruction* instr,
+                                         const void* addr) {
+  USE(instr);
+  AppendToOutput("(addr 0x%" PRIxPTR ")", reinterpret_cast<uintptr_t>(addr));
+}
+
+
+void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeAddressToOutput(const Instruction* instr,
+                                                     const void* addr) {
+  USE(instr);
+  int64_t rel_addr = CodeRelativeAddress(addr);
+  if (rel_addr >= 0) {
+    AppendToOutput("(addr 0x%" PRIx64 ")", rel_addr);
+  } else {
+    AppendToOutput("(addr -0x%" PRIx64 ")", -rel_addr);
+  }
+}
+
+
+void Disassembler::AppendCodeRelativeCodeAddressToOutput(
+    const Instruction* instr, const void* addr) {
+  AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeDataAddressToOutput(
+    const Instruction* instr, const void* addr) {
+  AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::MapCodeAddress(int64_t base_address,
+                                  const Instruction* instr_address) {
+  set_code_address_offset(
+      base_address - reinterpret_cast<intptr_t>(instr_address));
+}
+int64_t Disassembler::CodeRelativeAddress(const void* addr) {
+  return reinterpret_cast<intptr_t>(addr) + code_address_offset();
+}
+
+
+void Disassembler::Format(const Instruction* instr, const char* mnemonic,
+                          const char* format) {
+  VIXL_ASSERT(mnemonic != NULL);
+  ResetOutput();
+  Substitute(instr, mnemonic);
+  if (format != NULL) {
+    VIXL_ASSERT(buffer_pos_ < buffer_size_);
+    buffer_[buffer_pos_++] = ' ';
+    Substitute(instr, format);
+  }
+  VIXL_ASSERT(buffer_pos_ < buffer_size_);
+  buffer_[buffer_pos_] = 0;
+  ProcessOutput(instr);
+}
+
+
+void Disassembler::Substitute(const Instruction* instr, const char* string) {
+  char chr = *string++;
+  while (chr != '\0') {
+    if (chr == '\'') {
+      string += SubstituteField(instr, string);
+    } else {
+      VIXL_ASSERT(buffer_pos_ < buffer_size_);
+      buffer_[buffer_pos_++] = chr;
+    }
+    chr = *string++;
+  }
+}
+
+
+int Disassembler::SubstituteField(const Instruction* instr,
+                                  const char* format) {
+  switch (format[0]) {
+    // NB. The remaining substitution prefix characters are: GJKUZ.
+    case 'R':  // Register. X or W, selected by sf bit.
+    case 'F':  // FP register. S or D, selected by type field.
+    case 'V':  // Vector register, V, vector format.
+    case 'W':
+    case 'X':
+    case 'B':
+    case 'H':
+    case 'S':
+    case 'D':
+    case 'Q': return SubstituteRegisterField(instr, format);
+    case 'I': return SubstituteImmediateField(instr, format);
+    case 'L': return SubstituteLiteralField(instr, format);
+    case 'N': return SubstituteShiftField(instr, format);
+    case 'P': return SubstitutePrefetchField(instr, format);
+    case 'C': return SubstituteConditionField(instr, format);
+    case 'E': return SubstituteExtendField(instr, format);
+    case 'A': return SubstitutePCRelAddressField(instr, format);
+    case 'T': return SubstituteBranchTargetField(instr, format);
+    case 'O': return SubstituteLSRegOffsetField(instr, format);
+    case 'M': return SubstituteBarrierField(instr, format);
+    case 'K': return SubstituteCrField(instr, format);
+    case 'G': return SubstituteSysOpField(instr, format);
+    default: {
+      VIXL_UNREACHABLE();
+      return 1;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteRegisterField(const Instruction* instr,
+                                          const char* format) {
+  char reg_prefix = format[0];
+  unsigned reg_num = 0;
+  unsigned field_len = 2;
+
+  switch (format[1]) {
+    case 'd':
+      reg_num = instr->Rd();
+      if (format[2] == 'q') {
+        reg_prefix = instr->NEONQ() ? 'X' : 'W';
+        field_len = 3;
+      }
+      break;
+    case 'n': reg_num = instr->Rn(); break;
+    case 'm':
+      reg_num = instr->Rm();
+      switch (format[2]) {
+          // Handle registers tagged with b (bytes), z (instruction), or
+          // r (registers), used for address updates in
+          // NEON load/store instructions.
+        case 'r':
+        case 'b':
+        case 'z': {
+          field_len = 3;
+          char* eimm;
+          int imm = static_cast<int>(strtol(&format[3], &eimm, 10));
+          field_len += eimm - &format[3];
+          if (reg_num == 31) {
+            switch (format[2]) {
+              case 'z':
+                imm *= (1 << instr->NEONLSSize());
+                break;
+              case 'r':
+                imm *= (instr->NEONQ() == 0) ? kDRegSizeInBytes
+                                             : kQRegSizeInBytes;
+                break;
+              case 'b':
+                break;
+            }
+            AppendToOutput("#%d", imm);
+            return field_len;
+          }
+          break;
+        }
+      }
+      break;
+    case 'e':
+      // This is register Rm, but using a 4-bit specifier. Used in NEON
+      // by-element instructions.
+      reg_num = (instr->Rm() & 0xf);
+      break;
+    case 'a': reg_num = instr->Ra(); break;
+    case 's': reg_num = instr->Rs(); break;
+    case 't':
+      reg_num = instr->Rt();
+      if (format[0] == 'V') {
+        if ((format[2] >= '2') && (format[2] <= '4')) {
+          // Handle consecutive vector register specifiers Vt2, Vt3 and Vt4.
+          reg_num = (reg_num + format[2] - '1') % 32;
+          field_len = 3;
+        }
+      } else {
+        if (format[2] == '2') {
+        // Handle register specifier Rt2.
+          reg_num = instr->Rt2();
+          field_len = 3;
+        }
+      }
+      break;
+    default: VIXL_UNREACHABLE();
+  }
+
+  // Increase field length for registers tagged as stack.
+  if (format[2] == 's') {
+    field_len = 3;
+  }
+
+  CPURegister::RegisterType reg_type = CPURegister::kRegister;
+  unsigned reg_size = kXRegSize;
+
+  if (reg_prefix == 'R') {
+    reg_prefix = instr->SixtyFourBits() ? 'X' : 'W';
+  } else if (reg_prefix == 'F') {
+    reg_prefix = ((instr->FPType() & 1) == 0) ? 'S' : 'D';
+  }
+
+  switch (reg_prefix) {
+    case 'W':
+      reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
+    case 'X':
+      reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
+    case 'B':
+      reg_type = CPURegister::kVRegister; reg_size = kBRegSize; break;
+    case 'H':
+      reg_type = CPURegister::kVRegister; reg_size = kHRegSize; break;
+    case 'S':
+      reg_type = CPURegister::kVRegister; reg_size = kSRegSize; break;
+    case 'D':
+      reg_type = CPURegister::kVRegister; reg_size = kDRegSize; break;
+    case 'Q':
+      reg_type = CPURegister::kVRegister; reg_size = kQRegSize; break;
+    case 'V':
+      AppendToOutput("v%d", reg_num);
+      return field_len;
+    default:
+      VIXL_UNREACHABLE();
+  }
+
+  if ((reg_type == CPURegister::kRegister) &&
+      (reg_num == kZeroRegCode) && (format[2] == 's')) {
+    reg_num = kSPRegInternalCode;
+  }
+
+  AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
+
+  return field_len;
+}
+
+
+int Disassembler::SubstituteImmediateField(const Instruction* instr,
+                                           const char* format) {
+  VIXL_ASSERT(format[0] == 'I');
+
+  switch (format[1]) {
+    case 'M': {  // IMoveImm, IMoveNeg or IMoveLSL.
+      if (format[5] == 'L') {
+        AppendToOutput("#0x%" PRIx32, instr->ImmMoveWide());
+        if (instr->ShiftMoveWide() > 0) {
+          AppendToOutput(", lsl #%" PRId32, 16 * instr->ShiftMoveWide());
+        }
+      } else {
+        VIXL_ASSERT((format[5] == 'I') || (format[5] == 'N'));
+        uint64_t imm = static_cast<uint64_t>(instr->ImmMoveWide()) <<
+            (16 * instr->ShiftMoveWide());
+        if (format[5] == 'N')
+          imm = ~imm;
+        if (!instr->SixtyFourBits())
+          imm &= UINT64_C(0xffffffff);
+        AppendToOutput("#0x%" PRIx64, imm);
+      }
+      return 8;
+    }
+    case 'L': {
+      switch (format[2]) {
+        case 'L': {  // ILLiteral - Immediate Load Literal.
+          AppendToOutput("pc%+" PRId32,
+                         instr->ImmLLiteral() << kLiteralEntrySizeLog2);
+          return 9;
+        }
+        case 'S': {  // ILS - Immediate Load/Store.
+          if (instr->ImmLS() != 0) {
+            AppendToOutput(", #%" PRId32, instr->ImmLS());
+          }
+          return 3;
+        }
+        case 'P': {  // ILPx - Immediate Load/Store Pair, x = access size.
+          if (instr->ImmLSPair() != 0) {
+            // format[3] is the scale value. Convert to a number.
+            int scale = 1 << (format[3] - '0');
+            AppendToOutput(", #%" PRId32, instr->ImmLSPair() * scale);
+          }
+          return 4;
+        }
+        case 'U': {  // ILU - Immediate Load/Store Unsigned.
+          if (instr->ImmLSUnsigned() != 0) {
+            int shift = instr->SizeLS();
+            AppendToOutput(", #%" PRId32, instr->ImmLSUnsigned() << shift);
+          }
+          return 3;
+        }
+      }
+    }
+    case 'C': {  // ICondB - Immediate Conditional Branch.
+      int64_t offset = instr->ImmCondBranch() << 2;
+      AppendPCRelativeOffsetToOutput(instr, offset);
+      return 6;
+    }
+    case 'A': {  // IAddSub.
+      VIXL_ASSERT(instr->ShiftAddSub() <= 1);
+      int64_t imm = instr->ImmAddSub() << (12 * instr->ShiftAddSub());
+      AppendToOutput("#0x%" PRIx64 " (%" PRId64 ")", imm, imm);
+      return 7;
+    }
+    case 'F': {  // IFPSingle, IFPDouble or IFPFBits.
+      if (format[3] == 'F') {  // IFPFbits.
+        AppendToOutput("#%" PRId32, 64 - instr->FPScale());
+        return 8;
+      } else {
+        AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmFP(),
+                       format[3] == 'S' ? instr->ImmFP32() : instr->ImmFP64());
+        return 9;
+      }
+    }
+    case 'T': {  // ITri - Immediate Triangular Encoded.
+      AppendToOutput("#0x%" PRIx64, instr->ImmLogical());
+      return 4;
+    }
+    case 'N': {  // INzcv.
+      int nzcv = (instr->Nzcv() << Flags_offset);
+      AppendToOutput("#%c%c%c%c", ((nzcv & NFlag) == 0) ? 'n' : 'N',
+                                  ((nzcv & ZFlag) == 0) ? 'z' : 'Z',
+                                  ((nzcv & CFlag) == 0) ? 'c' : 'C',
+                                  ((nzcv & VFlag) == 0) ? 'v' : 'V');
+      return 5;
+    }
+    case 'P': {  // IP - Conditional compare.
+      AppendToOutput("#%" PRId32, instr->ImmCondCmp());
+      return 2;
+    }
+    case 'B': {  // Bitfields.
+      return SubstituteBitfieldImmediateField(instr, format);
+    }
+    case 'E': {  // IExtract.
+      AppendToOutput("#%" PRId32, instr->ImmS());
+      return 8;
+    }
+    case 'S': {  // IS - Test and branch bit.
+      AppendToOutput("#%" PRId32, (instr->ImmTestBranchBit5() << 5) |
+                                  instr->ImmTestBranchBit40());
+      return 2;
+    }
+    case 's': {  // Is - Shift (immediate).
+      switch (format[2]) {
+        case '1': {  // Is1 - SSHR.
+          int shift = 16 << HighestSetBitPosition(instr->ImmNEONImmh());
+          shift -= instr->ImmNEONImmhImmb();
+          AppendToOutput("#%d", shift);
+          return 3;
+        }
+        case '2': {  // Is2 - SLI.
+          int shift = instr->ImmNEONImmhImmb();
+          shift -= 8 << HighestSetBitPosition(instr->ImmNEONImmh());
+          AppendToOutput("#%d", shift);
+          return 3;
+        }
+        default: {
+          VIXL_UNIMPLEMENTED();
+          return 0;
+        }
+      }
+    }
+    case 'D': {  // IDebug - HLT and BRK instructions.
+      AppendToOutput("#0x%" PRIx32, instr->ImmException());
+      return 6;
+    }
+    case 'V': {  // Immediate Vector.
+      switch (format[2]) {
+        case 'E': {  // IVExtract.
+          AppendToOutput("#%" PRId32, instr->ImmNEONExt());
+          return 9;
+        }
+        case 'B': {  // IVByElemIndex.
+          int vm_index = (instr->NEONH() << 1) | instr->NEONL();
+          if (instr->NEONSize() == 1) {
+            vm_index = (vm_index << 1) | instr->NEONM();
+          }
+          AppendToOutput("%d", vm_index);
+          return strlen("IVByElemIndex");
+        }
+        case 'I': {  // INS element.
+          if (strncmp(format, "IVInsIndex", strlen("IVInsIndex")) == 0) {
+            int rd_index, rn_index;
+            int imm5 = instr->ImmNEON5();
+            int imm4 = instr->ImmNEON4();
+            int tz = CountTrailingZeros(imm5, 32);
+            rd_index = imm5 >> (tz + 1);
+            rn_index = imm4 >> tz;
+            if (strncmp(format, "IVInsIndex1", strlen("IVInsIndex1")) == 0) {
+              AppendToOutput("%d", rd_index);
+              return strlen("IVInsIndex1");
+            } else if (strncmp(format, "IVInsIndex2",
+                       strlen("IVInsIndex2")) == 0) {
+              AppendToOutput("%d", rn_index);
+              return strlen("IVInsIndex2");
+            } else {
+              VIXL_UNIMPLEMENTED();
+              return 0;
+            }
+          }
+          VIXL_FALLTHROUGH();
+        }
+        case 'L': {  // IVLSLane[0123] - suffix indicates access size shift.
+          AppendToOutput("%d", instr->NEONLSIndex(format[8] - '0'));
+          return 9;
+        }
+        case 'M': {  // Modified Immediate cases.
+          if (strncmp(format,
+                      "IVMIImmFPSingle",
+                      strlen("IVMIImmFPSingle")) == 0) {
+            AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+                           instr->ImmNEONFP32());
+            return strlen("IVMIImmFPSingle");
+          } else if (strncmp(format,
+                             "IVMIImmFPDouble",
+                             strlen("IVMIImmFPDouble")) == 0) {
+            AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+                           instr->ImmNEONFP64());
+            return strlen("IVMIImmFPDouble");
+          } else if (strncmp(format, "IVMIImm8", strlen("IVMIImm8")) == 0) {
+            uint64_t imm8 = instr->ImmNEONabcdefgh();
+            AppendToOutput("#0x%" PRIx64, imm8);
+            return strlen("IVMIImm8");
+          } else if (strncmp(format, "IVMIImm", strlen("IVMIImm")) == 0) {
+            uint64_t imm8 = instr->ImmNEONabcdefgh();
+            uint64_t imm = 0;
+            for (int i = 0; i < 8; ++i) {
+              if (imm8 & (1 << i)) {
+                imm |= (UINT64_C(0xff) << (8 * i));
+              }
+            }
+            AppendToOutput("#0x%" PRIx64, imm);
+            return strlen("IVMIImm");
+          } else if (strncmp(format, "IVMIShiftAmt1",
+                             strlen("IVMIShiftAmt1")) == 0) {
+            int cmode = instr->NEONCmode();
+            int shift_amount = 8 * ((cmode >> 1) & 3);
+            AppendToOutput("#%d", shift_amount);
+            return strlen("IVMIShiftAmt1");
+          } else if (strncmp(format, "IVMIShiftAmt2",
+                             strlen("IVMIShiftAmt2")) == 0) {
+            int cmode = instr->NEONCmode();
+            int shift_amount = 8 << (cmode & 1);
+            AppendToOutput("#%d", shift_amount);
+            return strlen("IVMIShiftAmt2");
+          } else {
+            VIXL_UNIMPLEMENTED();
+            return 0;
+          }
+        }
+        default: {
+          VIXL_UNIMPLEMENTED();
+          return 0;
+        }
+      }
+    }
+    case 'X': {  // IX - CLREX instruction.
+      AppendToOutput("#0x%" PRIx32, instr->CRm());
+      return 2;
+    }
+    default: {
+      VIXL_UNIMPLEMENTED();
+      return 0;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
+                                                   const char* format) {
+  VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
+  unsigned r = instr->ImmR();
+  unsigned s = instr->ImmS();
+
+  switch (format[2]) {
+    case 'r': {  // IBr.
+      AppendToOutput("#%d", r);
+      return 3;
+    }
+    case 's': {  // IBs+1 or IBs-r+1.
+      if (format[3] == '+') {
+        AppendToOutput("#%d", s + 1);
+        return 5;
+      } else {
+        VIXL_ASSERT(format[3] == '-');
+        AppendToOutput("#%d", s - r + 1);
+        return 7;
+      }
+    }
+    case 'Z': {  // IBZ-r.
+      VIXL_ASSERT((format[3] == '-') && (format[4] == 'r'));
+      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize;
+      AppendToOutput("#%d", reg_size - r);
+      return 5;
+    }
+    default: {
+      VIXL_UNREACHABLE();
+      return 0;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteLiteralField(const Instruction* instr,
+                                         const char* format) {
+  VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
+  USE(format);
+
+  const void * address = instr->LiteralAddress<const void *>();
+  switch (instr->Mask(LoadLiteralMask)) {
+    case LDR_w_lit:
+    case LDR_x_lit:
+    case LDRSW_x_lit:
+    case LDR_s_lit:
+    case LDR_d_lit:
+    case LDR_q_lit:
+      AppendCodeRelativeDataAddressToOutput(instr, address);
+      break;
+    case PRFM_lit: {
+      // Use the prefetch hint to decide how to print the address.
+      switch (instr->PrefetchHint()) {
+        case 0x0:     // PLD: prefetch for load.
+        case 0x2:     // PST: prepare for store.
+          AppendCodeRelativeDataAddressToOutput(instr, address);
+          break;
+        case 0x1:     // PLI: preload instructions.
+          AppendCodeRelativeCodeAddressToOutput(instr, address);
+          break;
+        case 0x3:     // Unallocated hint.
+          AppendCodeRelativeAddressToOutput(instr, address);
+          break;
+      }
+      break;
+    }
+    default:
+      VIXL_UNREACHABLE();
+  }
+
+  return 6;
+}
+
+
+int Disassembler::SubstituteShiftField(const Instruction* instr,
+                                       const char* format) {
+  VIXL_ASSERT(format[0] == 'N');
+  VIXL_ASSERT(instr->ShiftDP() <= 0x3);
+
+  switch (format[1]) {
+    case 'D': {  // HDP.
+      VIXL_ASSERT(instr->ShiftDP() != ROR);
+      VIXL_FALLTHROUGH();
+    }
+    case 'L': {  // HLo.
+      if (instr->ImmDPShift() != 0) {
+        const char* shift_type[] = {"lsl", "lsr", "asr", "ror"};
+        AppendToOutput(", %s #%" PRId32, shift_type[instr->ShiftDP()],
+                       instr->ImmDPShift());
+      }
+      return 3;
+    }
+    default:
+      VIXL_UNIMPLEMENTED();
+      return 0;
+  }
+}
+
+
+int Disassembler::SubstituteConditionField(const Instruction* instr,
+                                           const char* format) {
+  VIXL_ASSERT(format[0] == 'C');
+  const char* condition_code[] = { "eq", "ne", "hs", "lo",
+                                   "mi", "pl", "vs", "vc",
+                                   "hi", "ls", "ge", "lt",
+                                   "gt", "le", "al", "nv" };
+  int cond;
+  switch (format[1]) {
+    case 'B': cond = instr->ConditionBranch(); break;
+    case 'I': {
+      cond = InvertCondition(static_cast<Condition>(instr->Condition()));
+      break;
+    }
+    default: cond = instr->Condition();
+  }
+  AppendToOutput("%s", condition_code[cond]);
+  return 4;
+}
+
+
+int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
+                                              const char* format) {
+  VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) ||   // Used by `adr`.
+              (strcmp(format, "AddrPCRelPage") == 0));    // Used by `adrp`.
+
+  int64_t offset = instr->ImmPCRel();
+
+  // Compute the target address based on the effective address (after applying
+  // code_address_offset). This is required for correct behaviour of adrp.
+  const Instruction* base = instr + code_address_offset();
+  if (format[9] == 'P') {
+    offset *= kPageSize;
+    base = AlignDown(base, kPageSize);
+  }
+  // Strip code_address_offset before printing, so we can use the
+  // semantically-correct AppendCodeRelativeAddressToOutput.
+  const void* target =
+      reinterpret_cast<const void*>(base + offset - code_address_offset());
+
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendCodeRelativeAddressToOutput(instr, target);
+  return 13;
+}
+
+
+int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
+                                              const char* format) {
+  VIXL_ASSERT(strncmp(format, "TImm", 4) == 0);
+
+  int64_t offset = 0;
+  switch (format[5]) {
+    // BImmUncn - unconditional branch immediate.
+    case 'n': offset = instr->ImmUncondBranch(); break;
+    // BImmCond - conditional branch immediate.
+    case 'o': offset = instr->ImmCondBranch(); break;
+    // BImmCmpa - compare and branch immediate.
+    case 'm': offset = instr->ImmCmpBranch(); break;
+    // BImmTest - test and branch immediate.
+    case 'e': offset = instr->ImmTestBranch(); break;
+    default: VIXL_UNIMPLEMENTED();
+  }
+  offset <<= kInstructionSizeLog2;
+  const void* target_address = reinterpret_cast<const void*>(instr + offset);
+  VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendCodeRelativeCodeAddressToOutput(instr, target_address);
+
+  return 8;
+}
+
+
+int Disassembler::SubstituteExtendField(const Instruction* instr,
+                                        const char* format) {
+  VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
+  VIXL_ASSERT(instr->ExtendMode() <= 7);
+  USE(format);
+
+  const char* extend_mode[] = { "uxtb", "uxth", "uxtw", "uxtx",
+                                "sxtb", "sxth", "sxtw", "sxtx" };
+
+  // If rd or rn is SP, uxtw on 32-bit registers and uxtx on 64-bit
+  // registers becomes lsl.
+  if (((instr->Rd() == kZeroRegCode) || (instr->Rn() == kZeroRegCode)) &&
+      (((instr->ExtendMode() == UXTW) && (instr->SixtyFourBits() == 0)) ||
+       (instr->ExtendMode() == UXTX))) {
+    if (instr->ImmExtendShift() > 0) {
+      AppendToOutput(", lsl #%" PRId32, instr->ImmExtendShift());
+    }
+  } else {
+    AppendToOutput(", %s", extend_mode[instr->ExtendMode()]);
+    if (instr->ImmExtendShift() > 0) {
+      AppendToOutput(" #%" PRId32, instr->ImmExtendShift());
+    }
+  }
+  return 3;
+}
+
+
+int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
+                                             const char* format) {
+  VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
+  const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
+                                "undefined", "undefined", "sxtw", "sxtx" };
+  USE(format);
+
+  unsigned shift = instr->ImmShiftLS();
+  Extend ext = static_cast<Extend>(instr->ExtendMode());
+  char reg_type = ((ext == UXTW) || (ext == SXTW)) ? 'w' : 'x';
+
+  unsigned rm = instr->Rm();
+  if (rm == kZeroRegCode) {
+    AppendToOutput("%czr", reg_type);
+  } else {
+    AppendToOutput("%c%d", reg_type, rm);
+  }
+
+  // Extend mode UXTX is an alias for shift mode LSL here.
+  if (!((ext == UXTX) && (shift == 0))) {
+    AppendToOutput(", %s", extend_mode[ext]);
+    if (shift != 0) {
+      AppendToOutput(" #%d", instr->SizeLS());
+    }
+  }
+  return 9;
+}
+
+
+int Disassembler::SubstitutePrefetchField(const Instruction* instr,
+                                          const char* format) {
+  VIXL_ASSERT(format[0] == 'P');
+  USE(format);
+
+  static const char* hints[] = {"ld", "li", "st"};
+  static const char* stream_options[] = {"keep", "strm"};
+
+  unsigned hint = instr->PrefetchHint();
+  unsigned target = instr->PrefetchTarget() + 1;
+  unsigned stream = instr->PrefetchStream();
+
+  if ((hint >= (sizeof(hints) / sizeof(hints[0]))) || (target > 3)) {
+    // Unallocated prefetch operations.
+    int prefetch_mode = instr->ImmPrefetchOperation();
+    AppendToOutput("#0b%c%c%c%c%c",
+                   (prefetch_mode & (1 << 4)) ? '1' : '0',
+                   (prefetch_mode & (1 << 3)) ? '1' : '0',
+                   (prefetch_mode & (1 << 2)) ? '1' : '0',
+                   (prefetch_mode & (1 << 1)) ? '1' : '0',
+                   (prefetch_mode & (1 << 0)) ? '1' : '0');
+  } else {
+    VIXL_ASSERT(stream < (sizeof(stream_options) / sizeof(stream_options[0])));
+    AppendToOutput("p%sl%d%s", hints[hint], target, stream_options[stream]);
+  }
+  return 6;
+}
+
+int Disassembler::SubstituteBarrierField(const Instruction* instr,
+                                         const char* format) {
+  VIXL_ASSERT(format[0] == 'M');
+  USE(format);
+
+  static const char* options[4][4] = {
+    { "sy (0b0000)", "oshld", "oshst", "osh" },
+    { "sy (0b0100)", "nshld", "nshst", "nsh" },
+    { "sy (0b1000)", "ishld", "ishst", "ish" },
+    { "sy (0b1100)", "ld", "st", "sy" }
+  };
+  int domain = instr->ImmBarrierDomain();
+  int type = instr->ImmBarrierType();
+
+  AppendToOutput("%s", options[domain][type]);
+  return 1;
+}
+
+int Disassembler::SubstituteSysOpField(const Instruction* instr,
+                                       const char* format) {
+  VIXL_ASSERT(format[0] == 'G');
+  int op = -1;
+  switch (format[1]) {
+    case '1': op = instr->SysOp1(); break;
+    case '2': op = instr->SysOp2(); break;
+    default:
+      VIXL_UNREACHABLE();
+  }
+  AppendToOutput("#%d", op);
+  return 2;
+}
+
+int Disassembler::SubstituteCrField(const Instruction* instr,
+                                    const char* format) {
+  VIXL_ASSERT(format[0] == 'K');
+  int cr = -1;
+  switch (format[1]) {
+    case 'n': cr = instr->CRn(); break;
+    case 'm': cr = instr->CRm(); break;
+    default:
+      VIXL_UNREACHABLE();
+  }
+  AppendToOutput("C%d", cr);
+  return 2;
+}
+
+void Disassembler::ResetOutput() {
+  buffer_pos_ = 0;
+  buffer_[buffer_pos_] = 0;
+}
+
+
+void Disassembler::AppendToOutput(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  buffer_pos_ += vsnprintf(&buffer_[buffer_pos_], buffer_size_ - buffer_pos_,
+          format, args);
+  va_end(args);
+}
+
+
+void PrintDisassembler::ProcessOutput(const Instruction* instr) {
+  fprintf(stream_, "0x%016" PRIx64 "  %08" PRIx32 "\t\t%s\n",
+          reinterpret_cast<uint64_t>(instr),
+          instr->InstructionBits(),
+          GetOutput());
+}
+
+}  // namespace vixl
diff --git a/qemu/disas/libvixl/a64/disasm-a64.h b/qemu/disas/libvixl/vixl/a64/disasm-a64.h
index ddfe98be1..930df6ea6 100644
--- a/qemu/disas/libvixl/a64/disasm-a64.h
+++ b/qemu/disas/libvixl/vixl/a64/disasm-a64.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2015, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -27,11 +27,11 @@
 #ifndef VIXL_A64_DISASM_A64_H
 #define VIXL_A64_DISASM_A64_H
 
-#include "globals.h"
-#include "utils.h"
-#include "instructions-a64.h"
-#include "decoder-a64.h"
-#include "assembler-a64.h"
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/instructions-a64.h"
+#include "vixl/a64/decoder-a64.h"
+#include "vixl/a64/assembler-a64.h"
 
 namespace vixl {
 
@@ -55,6 +55,7 @@ class Disassembler: public DecoderVisitor {
   // customize the disassembly output.
 
   // Prints the name of a register.
+  // TODO: This currently doesn't allow renaming of V registers.
   virtual void AppendRegisterNameToOutput(const Instruction* instr,
                                           const CPURegister& reg);
 
@@ -122,7 +123,8 @@ class Disassembler: public DecoderVisitor {
   int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
   int SubstitutePrefetchField(const Instruction* instr, const char* format);
   int SubstituteBarrierField(const Instruction* instr, const char* format);
-
+  int SubstituteSysOpField(const Instruction* instr, const char* format);
+  int SubstituteCrField(const Instruction* instr, const char* format);
   bool RdIsZROrSP(const Instruction* instr) const {
     return (instr->Rd() == kZeroRegCode);
   }
@@ -163,7 +165,6 @@ class Disassembler: public DecoderVisitor {
 class PrintDisassembler: public Disassembler {
  public:
   explicit PrintDisassembler(FILE* stream) : stream_(stream) { }
-  virtual ~PrintDisassembler() { }
 
  protected:
   virtual void ProcessOutput(const Instruction* instr);
diff --git a/qemu/disas/libvixl/vixl/a64/instructions-a64.cc b/qemu/disas/libvixl/vixl/a64/instructions-a64.cc
new file mode 100644
index 000000000..33992f88a
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/a64/instructions-a64.cc
@@ -0,0 +1,622 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "vixl/a64/instructions-a64.h"
+#include "vixl/a64/assembler-a64.h"
+
+namespace vixl {
+
+
+// Floating-point infinity values.
+const float16 kFP16PositiveInfinity = 0x7c00;
+const float16 kFP16NegativeInfinity = 0xfc00;
+const float kFP32PositiveInfinity = rawbits_to_float(0x7f800000);
+const float kFP32NegativeInfinity = rawbits_to_float(0xff800000);
+const double kFP64PositiveInfinity =
+    rawbits_to_double(UINT64_C(0x7ff0000000000000));
+const double kFP64NegativeInfinity =
+    rawbits_to_double(UINT64_C(0xfff0000000000000));
+
+
+// The default NaN values (for FPCR.DN=1).
+const double kFP64DefaultNaN = rawbits_to_double(UINT64_C(0x7ff8000000000000));
+const float kFP32DefaultNaN = rawbits_to_float(0x7fc00000);
+const float16 kFP16DefaultNaN = 0x7e00;
+
+
+static uint64_t RotateRight(uint64_t value,
+                            unsigned int rotate,
+                            unsigned int width) {
+  VIXL_ASSERT(width <= 64);
+  rotate &= 63;
+  return ((value & ((UINT64_C(1) << rotate) - 1)) <<
+          (width - rotate)) | (value >> rotate);
+}
+
+
+static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
+                                    uint64_t value,
+                                    unsigned width) {
+  VIXL_ASSERT((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
+              (width == 32));
+  VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+  uint64_t result = value & ((UINT64_C(1) << width) - 1);
+  for (unsigned i = width; i < reg_size; i *= 2) {
+    result |= (result << i);
+  }
+  return result;
+}
+
+
+bool Instruction::IsLoad() const {
+  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+    return false;
+  }
+
+  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+    return Mask(LoadStorePairLBit) != 0;
+  } else {
+    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+    switch (op) {
+      case LDRB_w:
+      case LDRH_w:
+      case LDR_w:
+      case LDR_x:
+      case LDRSB_w:
+      case LDRSB_x:
+      case LDRSH_w:
+      case LDRSH_x:
+      case LDRSW_x:
+      case LDR_b:
+      case LDR_h:
+      case LDR_s:
+      case LDR_d:
+      case LDR_q: return true;
+      default: return false;
+    }
+  }
+}
+
+
+bool Instruction::IsStore() const {
+  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+    return false;
+  }
+
+  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+    return Mask(LoadStorePairLBit) == 0;
+  } else {
+    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+    switch (op) {
+      case STRB_w:
+      case STRH_w:
+      case STR_w:
+      case STR_x:
+      case STR_b:
+      case STR_h:
+      case STR_s:
+      case STR_d:
+      case STR_q: return true;
+      default: return false;
+    }
+  }
+}
+
+
+// Logical immediates can't encode zero, so a return value of zero is used to
+// indicate a failure case. Specifically, where the constraints on imm_s are
+// not met.
+uint64_t Instruction::ImmLogical() const {
+  unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
+  int32_t n = BitN();
+  int32_t imm_s = ImmSetBits();
+  int32_t imm_r = ImmRotate();
+
+  // An integer is constructed from the n, imm_s and imm_r bits according to
+  // the following table:
+  //
+  //  N   imms    immr    size        S             R
+  //  1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
+  //  0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
+  //  0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
+  //  0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
+  //  0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
+  //  0  11110s  xxxxxr     2    UInt(s)       UInt(r)
+  // (s bits must not be all set)
+  //
+  // A pattern is constructed of size bits, where the least significant S+1
+  // bits are set. The pattern is rotated right by R, and repeated across a
+  // 32 or 64-bit value, depending on destination register width.
+  //
+
+  if (n == 1) {
+    if (imm_s == 0x3f) {
+      return 0;
+    }
+    uint64_t bits = (UINT64_C(1) << (imm_s + 1)) - 1;
+    return RotateRight(bits, imm_r, 64);
+  } else {
+    if ((imm_s >> 1) == 0x1f) {
+      return 0;
+    }
+    for (int width = 0x20; width >= 0x2; width >>= 1) {
+      if ((imm_s & width) == 0) {
+        int mask = width - 1;
+        if ((imm_s & mask) == mask) {
+          return 0;
+        }
+        uint64_t bits = (UINT64_C(1) << ((imm_s & mask) + 1)) - 1;
+        return RepeatBitsAcrossReg(reg_size,
+                                   RotateRight(bits, imm_r & mask, width),
+                                   width);
+      }
+    }
+  }
+  VIXL_UNREACHABLE();
+  return 0;
+}
+
+
+uint32_t Instruction::ImmNEONabcdefgh() const {
+  return ImmNEONabc() << 5 | ImmNEONdefgh();
+}
+
+
+float Instruction::Imm8ToFP32(uint32_t imm8) {
+  //   Imm8: abcdefgh (8 bits)
+  // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
+  // where B is b ^ 1
+  uint32_t bits = imm8;
+  uint32_t bit7 = (bits >> 7) & 0x1;
+  uint32_t bit6 = (bits >> 6) & 0x1;
+  uint32_t bit5_to_0 = bits & 0x3f;
+  uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
+
+  return rawbits_to_float(result);
+}
+
+
+float Instruction::ImmFP32() const {
+  return Imm8ToFP32(ImmFP());
+}
+
+
+double Instruction::Imm8ToFP64(uint32_t imm8) {
+  //   Imm8: abcdefgh (8 bits)
+  // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+  //         0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
+  // where B is b ^ 1
+  uint32_t bits = imm8;
+  uint64_t bit7 = (bits >> 7) & 0x1;
+  uint64_t bit6 = (bits >> 6) & 0x1;
+  uint64_t bit5_to_0 = bits & 0x3f;
+  uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
+
+  return rawbits_to_double(result);
+}
+
+
+double Instruction::ImmFP64() const {
+  return Imm8ToFP64(ImmFP());
+}
+
+
+float Instruction::ImmNEONFP32() const {
+  return Imm8ToFP32(ImmNEONabcdefgh());
+}
+
+
+double Instruction::ImmNEONFP64() const {
+  return Imm8ToFP64(ImmNEONabcdefgh());
+}
+
+
+unsigned CalcLSDataSize(LoadStoreOp op) {
+  VIXL_ASSERT((LSSize_offset + LSSize_width) == (kInstructionSize * 8));
+  unsigned size = static_cast<Instr>(op) >> LSSize_offset;
+  if ((op & LSVector_mask) != 0) {
+    // Vector register memory operations encode the access size in the "size"
+    // and "opc" fields.
+    if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
+      size = kQRegSizeInBytesLog2;
+    }
+  }
+  return size;
+}
+
+
+unsigned CalcLSPairDataSize(LoadStorePairOp op) {
+  VIXL_STATIC_ASSERT(kXRegSizeInBytes == kDRegSizeInBytes);
+  VIXL_STATIC_ASSERT(kWRegSizeInBytes == kSRegSizeInBytes);
+  switch (op) {
+    case STP_q:
+    case LDP_q: return kQRegSizeInBytesLog2;
+    case STP_x:
+    case LDP_x:
+    case STP_d:
+    case LDP_d: return kXRegSizeInBytesLog2;
+    default: return kWRegSizeInBytesLog2;
+  }
+}
+
+
+int Instruction::ImmBranchRangeBitwidth(ImmBranchType branch_type) {
+  switch (branch_type) {
+    case UncondBranchType:
+      return ImmUncondBranch_width;
+    case CondBranchType:
+      return ImmCondBranch_width;
+    case CompareBranchType:
+      return ImmCmpBranch_width;
+    case TestBranchType:
+      return ImmTestBranch_width;
+    default:
+      VIXL_UNREACHABLE();
+      return 0;
+  }
+}
+
+
+int32_t Instruction::ImmBranchForwardRange(ImmBranchType branch_type) {
+  int32_t encoded_max = 1 << (ImmBranchRangeBitwidth(branch_type) - 1);
+  return encoded_max * kInstructionSize;
+}
+
+
+bool Instruction::IsValidImmPCOffset(ImmBranchType branch_type,
+                                     int64_t offset) {
+  return is_intn(ImmBranchRangeBitwidth(branch_type), offset);
+}
+
+
+const Instruction* Instruction::ImmPCOffsetTarget() const {
+  const Instruction * base = this;
+  ptrdiff_t offset;
+  if (IsPCRelAddressing()) {
+    // ADR and ADRP.
+    offset = ImmPCRel();
+    if (Mask(PCRelAddressingMask) == ADRP) {
+      base = AlignDown(base, kPageSize);
+      offset *= kPageSize;
+    } else {
+      VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
+    }
+  } else {
+    // All PC-relative branches.
+    VIXL_ASSERT(BranchType() != UnknownBranchType);
+    // Relative branch offsets are instruction-size-aligned.
+    offset = ImmBranch() << kInstructionSizeLog2;
+  }
+  return base + offset;
+}
+
+
+int Instruction::ImmBranch() const {
+  switch (BranchType()) {
+    case CondBranchType: return ImmCondBranch();
+    case UncondBranchType: return ImmUncondBranch();
+    case CompareBranchType: return ImmCmpBranch();
+    case TestBranchType: return ImmTestBranch();
+    default: VIXL_UNREACHABLE();
+  }
+  return 0;
+}
+
+
+void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
+  if (IsPCRelAddressing()) {
+    SetPCRelImmTarget(target);
+  } else {
+    SetBranchImmTarget(target);
+  }
+}
+
+
+void Instruction::SetPCRelImmTarget(const Instruction* target) {
+  ptrdiff_t imm21;
+  if ((Mask(PCRelAddressingMask) == ADR)) {
+    imm21 = target - this;
+  } else {
+    VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
+    uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
+    uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
+    imm21 = target_page - this_page;
+  }
+  Instr imm = Assembler::ImmPCRelAddress(static_cast<int32_t>(imm21));
+
+  SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+}
+
+
+void Instruction::SetBranchImmTarget(const Instruction* target) {
+  VIXL_ASSERT(((target - this) & 3) == 0);
+  Instr branch_imm = 0;
+  uint32_t imm_mask = 0;
+  int offset = static_cast<int>((target - this) >> kInstructionSizeLog2);
+  switch (BranchType()) {
+    case CondBranchType: {
+      branch_imm = Assembler::ImmCondBranch(offset);
+      imm_mask = ImmCondBranch_mask;
+      break;
+    }
+    case UncondBranchType: {
+      branch_imm = Assembler::ImmUncondBranch(offset);
+      imm_mask = ImmUncondBranch_mask;
+      break;
+    }
+    case CompareBranchType: {
+      branch_imm = Assembler::ImmCmpBranch(offset);
+      imm_mask = ImmCmpBranch_mask;
+      break;
+    }
+    case TestBranchType: {
+      branch_imm = Assembler::ImmTestBranch(offset);
+      imm_mask = ImmTestBranch_mask;
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  SetInstructionBits(Mask(~imm_mask) | branch_imm);
+}
+
+
+void Instruction::SetImmLLiteral(const Instruction* source) {
+  VIXL_ASSERT(IsWordAligned(source));
+  ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
+  Instr imm = Assembler::ImmLLiteral(static_cast<int>(offset));
+  Instr mask = ImmLLiteral_mask;
+
+  SetInstructionBits(Mask(~mask) | imm);
+}
+
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8H || vform == kFormat4S || vform == kFormat2D ||
+              vform == kFormatH || vform == kFormatS || vform == kFormatD);
+  switch (vform) {
+    case kFormat8H: return kFormat8B;
+    case kFormat4S: return kFormat4H;
+    case kFormat2D: return kFormat2S;
+    case kFormatH:  return kFormatB;
+    case kFormatS:  return kFormatH;
+    case kFormatD:  return kFormatS;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S ||
+              vform == kFormatB || vform == kFormatH || vform == kFormatS);
+  switch (vform) {
+    case kFormat8B: return kFormat8H;
+    case kFormat4H: return kFormat4S;
+    case kFormat2S: return kFormat2D;
+    case kFormatB:  return kFormatH;
+    case kFormatH:  return kFormatS;
+    case kFormatS:  return kFormatD;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatFillQ(const VectorFormat vform) {
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return kFormat16B;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H:  return kFormat8H;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S:  return kFormat4S;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D:  return kFormat2D;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform) {
+  switch (vform) {
+    case kFormat4H: return kFormat8B;
+    case kFormat8H: return kFormat16B;
+    case kFormat2S: return kFormat4H;
+    case kFormat4S: return kFormat8H;
+    case kFormat1D: return kFormat2S;
+    case kFormat2D: return kFormat4S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S);
+  switch (vform) {
+    case kFormat8B: return kFormat16B;
+    case kFormat4H: return kFormat8H;
+    case kFormat2S: return kFormat4S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat16B || vform == kFormat8H || vform == kFormat4S);
+  switch (vform) {
+    case kFormat16B: return kFormat8B;
+    case kFormat8H: return kFormat4H;
+    case kFormat4S: return kFormat2S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat ScalarFormatFromLaneSize(int laneSize) {
+  switch (laneSize) {
+    case 8:  return kFormatB;
+    case 16: return kFormatH;
+    case 32: return kFormatS;
+    case 64: return kFormatD;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB: return kBRegSize;
+    case kFormatH: return kHRegSize;
+    case kFormatS: return kSRegSize;
+    case kFormatD: return kDRegSize;
+    case kFormat8B:
+    case kFormat4H:
+    case kFormat2S:
+    case kFormat1D: return kDRegSize;
+    default: return kQRegSize;
+  }
+}
+
+
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform) {
+  return RegisterSizeInBitsFromFormat(vform) / 8;
+}
+
+
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 8;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 16;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 32;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 64;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int LaneSizeInBytesFromFormat(VectorFormat vform) {
+  return LaneSizeInBitsFromFormat(vform) / 8;
+}
+
+
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 0;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 1;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 2;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 3;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int LaneCountFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormat16B: return 16;
+    case kFormat8B:
+    case kFormat8H: return 8;
+    case kFormat4H:
+    case kFormat4S: return 4;
+    case kFormat2S:
+    case kFormat2D: return 2;
+    case kFormat1D:
+    case kFormatB:
+    case kFormatH:
+    case kFormatS:
+    case kFormatD: return 1;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int MaxLaneCountFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 16;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 8;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 4;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 2;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+// Does 'vform' indicate a vector format or a scalar format?
+bool IsVectorFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormatH:
+    case kFormatS:
+    case kFormatD: return false;
+    default: return true;
+  }
+}
+
+
+int64_t MaxIntFromFormat(VectorFormat vform) {
+  return INT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+int64_t MinIntFromFormat(VectorFormat vform) {
+  return INT64_MIN >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+uint64_t MaxUintFromFormat(VectorFormat vform) {
+  return UINT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+}  // namespace vixl
+
diff --git a/qemu/disas/libvixl/vixl/a64/instructions-a64.h b/qemu/disas/libvixl/vixl/a64/instructions-a64.h
new file mode 100644
index 000000000..7e0dbae36
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/a64/instructions-a64.h
@@ -0,0 +1,757 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_INSTRUCTIONS_A64_H_
+#define VIXL_A64_INSTRUCTIONS_A64_H_
+
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/constants-a64.h"
+
+namespace vixl {
+// ISA constants. --------------------------------------------------------------
+
+typedef uint32_t Instr;
+const unsigned kInstructionSize = 4;
+const unsigned kInstructionSizeLog2 = 2;
+const unsigned kLiteralEntrySize = 4;
+const unsigned kLiteralEntrySizeLog2 = 2;
+const unsigned kMaxLoadLiteralRange = 1 * MBytes;
+
+// This is the nominal page size (as used by the adrp instruction); the actual
+// size of the memory pages allocated by the kernel is likely to differ.
+const unsigned kPageSize = 4 * KBytes;
+const unsigned kPageSizeLog2 = 12;
+
+const unsigned kBRegSize = 8;
+const unsigned kBRegSizeLog2 = 3;
+const unsigned kBRegSizeInBytes = kBRegSize / 8;
+const unsigned kBRegSizeInBytesLog2 = kBRegSizeLog2 - 3;
+const unsigned kHRegSize = 16;
+const unsigned kHRegSizeLog2 = 4;
+const unsigned kHRegSizeInBytes = kHRegSize / 8;
+const unsigned kHRegSizeInBytesLog2 = kHRegSizeLog2 - 3;
+const unsigned kWRegSize = 32;
+const unsigned kWRegSizeLog2 = 5;
+const unsigned kWRegSizeInBytes = kWRegSize / 8;
+const unsigned kWRegSizeInBytesLog2 = kWRegSizeLog2 - 3;
+const unsigned kXRegSize = 64;
+const unsigned kXRegSizeLog2 = 6;
+const unsigned kXRegSizeInBytes = kXRegSize / 8;
+const unsigned kXRegSizeInBytesLog2 = kXRegSizeLog2 - 3;
+const unsigned kSRegSize = 32;
+const unsigned kSRegSizeLog2 = 5;
+const unsigned kSRegSizeInBytes = kSRegSize / 8;
+const unsigned kSRegSizeInBytesLog2 = kSRegSizeLog2 - 3;
+const unsigned kDRegSize = 64;
+const unsigned kDRegSizeLog2 = 6;
+const unsigned kDRegSizeInBytes = kDRegSize / 8;
+const unsigned kDRegSizeInBytesLog2 = kDRegSizeLog2 - 3;
+const unsigned kQRegSize = 128;
+const unsigned kQRegSizeLog2 = 7;
+const unsigned kQRegSizeInBytes = kQRegSize / 8;
+const unsigned kQRegSizeInBytesLog2 = kQRegSizeLog2 - 3;
+const uint64_t kWRegMask = UINT64_C(0xffffffff);
+const uint64_t kXRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSRegMask = UINT64_C(0xffffffff);
+const uint64_t kDRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSSignMask = UINT64_C(0x80000000);
+const uint64_t kDSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kWSignMask = UINT64_C(0x80000000);
+const uint64_t kXSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kByteMask = UINT64_C(0xff);
+const uint64_t kHalfWordMask = UINT64_C(0xffff);
+const uint64_t kWordMask = UINT64_C(0xffffffff);
+const uint64_t kXMaxUInt = UINT64_C(0xffffffffffffffff);
+const uint64_t kWMaxUInt = UINT64_C(0xffffffff);
+const int64_t kXMaxInt = INT64_C(0x7fffffffffffffff);
+const int64_t kXMinInt = INT64_C(0x8000000000000000);
+const int32_t kWMaxInt = INT32_C(0x7fffffff);
+const int32_t kWMinInt = INT32_C(0x80000000);
+const unsigned kLinkRegCode = 30;
+const unsigned kZeroRegCode = 31;
+const unsigned kSPRegInternalCode = 63;
+const unsigned kRegCodeMask = 0x1f;
+
+const unsigned kAddressTagOffset = 56;
+const unsigned kAddressTagWidth = 8;
+const uint64_t kAddressTagMask =
+    ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
+VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
+
+// AArch64 floating-point specifics. These match IEEE-754.
+const unsigned kDoubleMantissaBits = 52;
+const unsigned kDoubleExponentBits = 11;
+const unsigned kFloatMantissaBits = 23;
+const unsigned kFloatExponentBits = 8;
+const unsigned kFloat16MantissaBits = 10;
+const unsigned kFloat16ExponentBits = 5;
+
+// Floating-point infinity values.
+extern const float16 kFP16PositiveInfinity;
+extern const float16 kFP16NegativeInfinity;
+extern const float kFP32PositiveInfinity;
+extern const float kFP32NegativeInfinity;
+extern const double kFP64PositiveInfinity;
+extern const double kFP64NegativeInfinity;
+
+// The default NaN values (for FPCR.DN=1).
+extern const float16 kFP16DefaultNaN;
+extern const float kFP32DefaultNaN;
+extern const double kFP64DefaultNaN;
+
+unsigned CalcLSDataSize(LoadStoreOp op);
+unsigned CalcLSPairDataSize(LoadStorePairOp op);
+
+enum ImmBranchType {
+  UnknownBranchType = 0,
+  CondBranchType    = 1,
+  UncondBranchType  = 2,
+  CompareBranchType = 3,
+  TestBranchType    = 4
+};
+
+enum AddrMode {
+  Offset,
+  PreIndex,
+  PostIndex
+};
+
+enum FPRounding {
+  // The first four values are encodable directly by FPCR<RMode>.
+  FPTieEven = 0x0,
+  FPPositiveInfinity = 0x1,
+  FPNegativeInfinity = 0x2,
+  FPZero = 0x3,
+
+  // The final rounding modes are only available when explicitly specified by
+  // the instruction (such as with fcvta). It cannot be set in FPCR.
+  FPTieAway,
+  FPRoundOdd
+};
+
+enum Reg31Mode {
+  Reg31IsStackPointer,
+  Reg31IsZeroRegister
+};
+
+// Instructions. ---------------------------------------------------------------
+
+class Instruction {
+ public:
+  Instr InstructionBits() const {
+    return *(reinterpret_cast<const Instr*>(this));
+  }
+
+  void SetInstructionBits(Instr new_instr) {
+    *(reinterpret_cast<Instr*>(this)) = new_instr;
+  }
+
+  int Bit(int pos) const {
+    return (InstructionBits() >> pos) & 1;
+  }
+
+  uint32_t Bits(int msb, int lsb) const {
+    return unsigned_bitextract_32(msb, lsb, InstructionBits());
+  }
+
+  int32_t SignedBits(int msb, int lsb) const {
+    int32_t bits = *(reinterpret_cast<const int32_t*>(this));
+    return signed_bitextract_32(msb, lsb, bits);
+  }
+
+  Instr Mask(uint32_t mask) const {
+    return InstructionBits() & mask;
+  }
+
+  #define DEFINE_GETTER(Name, HighBit, LowBit, Func)             \
+  int32_t Name() const { return Func(HighBit, LowBit); }
+  INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
+  #undef DEFINE_GETTER
+
+  // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
+  // formed from ImmPCRelLo and ImmPCRelHi.
+  int ImmPCRel() const {
+    int offset =
+        static_cast<int>((ImmPCRelHi() << ImmPCRelLo_width) | ImmPCRelLo());
+    int width = ImmPCRelLo_width + ImmPCRelHi_width;
+    return signed_bitextract_32(width - 1, 0, offset);
+  }
+
+  uint64_t ImmLogical() const;
+  unsigned ImmNEONabcdefgh() const;
+  float ImmFP32() const;
+  double ImmFP64() const;
+  float ImmNEONFP32() const;
+  double ImmNEONFP64() const;
+
+  unsigned SizeLS() const {
+    return CalcLSDataSize(static_cast<LoadStoreOp>(Mask(LoadStoreMask)));
+  }
+
+  unsigned SizeLSPair() const {
+    return CalcLSPairDataSize(
+        static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
+  }
+
+  int NEONLSIndex(int access_size_shift) const {
+    int64_t q = NEONQ();
+    int64_t s = NEONS();
+    int64_t size = NEONLSSize();
+    int64_t index = (q << 3) | (s << 2) | size;
+    return static_cast<int>(index >> access_size_shift);
+  }
+
+  // Helpers.
+  bool IsCondBranchImm() const {
+    return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
+  }
+
+  bool IsUncondBranchImm() const {
+    return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
+  }
+
+  bool IsCompareBranch() const {
+    return Mask(CompareBranchFMask) == CompareBranchFixed;
+  }
+
+  bool IsTestBranch() const {
+    return Mask(TestBranchFMask) == TestBranchFixed;
+  }
+
+  bool IsImmBranch() const {
+    return BranchType() != UnknownBranchType;
+  }
+
+  bool IsPCRelAddressing() const {
+    return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
+  }
+
+  bool IsLogicalImmediate() const {
+    return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
+  }
+
+  bool IsAddSubImmediate() const {
+    return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
+  }
+
+  bool IsAddSubExtended() const {
+    return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
+  }
+
+  bool IsLoadOrStore() const {
+    return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
+  }
+
+  bool IsLoad() const;
+  bool IsStore() const;
+
+  bool IsLoadLiteral() const {
+    // This includes PRFM_lit.
+    return Mask(LoadLiteralFMask) == LoadLiteralFixed;
+  }
+
+  bool IsMovn() const {
+    return (Mask(MoveWideImmediateMask) == MOVN_x) ||
+           (Mask(MoveWideImmediateMask) == MOVN_w);
+  }
+
+  static int ImmBranchRangeBitwidth(ImmBranchType branch_type);
+  static int32_t ImmBranchForwardRange(ImmBranchType branch_type);
+  static bool IsValidImmPCOffset(ImmBranchType branch_type, int64_t offset);
+
+  // Indicate whether Rd can be the stack pointer or the zero register. This
+  // does not check that the instruction actually has an Rd field.
+  Reg31Mode RdMode() const {
+    // The following instructions use sp or wsp as Rd:
+    //  Add/sub (immediate) when not setting the flags.
+    //  Add/sub (extended) when not setting the flags.
+    //  Logical (immediate) when not setting the flags.
+    // Otherwise, r31 is the zero register.
+    if (IsAddSubImmediate() || IsAddSubExtended()) {
+      if (Mask(AddSubSetFlagsBit)) {
+        return Reg31IsZeroRegister;
+      } else {
+        return Reg31IsStackPointer;
+      }
+    }
+    if (IsLogicalImmediate()) {
+      // Of the logical (immediate) instructions, only ANDS (and its aliases)
+      // can set the flags. The others can all write into sp.
+      // Note that some logical operations are not available to
+      // immediate-operand instructions, so we have to combine two masks here.
+      if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
+        return Reg31IsZeroRegister;
+      } else {
+        return Reg31IsStackPointer;
+      }
+    }
+    return Reg31IsZeroRegister;
+  }
+
+  // Indicate whether Rn can be the stack pointer or the zero register. This
+  // does not check that the instruction actually has an Rn field.
+  Reg31Mode RnMode() const {
+    // The following instructions use sp or wsp as Rn:
+    //  All loads and stores.
+    //  Add/sub (immediate).
+    //  Add/sub (extended).
+    // Otherwise, r31 is the zero register.
+    if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
+      return Reg31IsStackPointer;
+    }
+    return Reg31IsZeroRegister;
+  }
+
+  ImmBranchType BranchType() const {
+    if (IsCondBranchImm()) {
+      return CondBranchType;
+    } else if (IsUncondBranchImm()) {
+      return UncondBranchType;
+    } else if (IsCompareBranch()) {
+      return CompareBranchType;
+    } else if (IsTestBranch()) {
+      return TestBranchType;
+    } else {
+      return UnknownBranchType;
+    }
+  }
+
+  // Find the target of this instruction. 'this' may be a branch or a
+  // PC-relative addressing instruction.
+  const Instruction* ImmPCOffsetTarget() const;
+
+  // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
+  // a PC-relative addressing instruction.
+  void SetImmPCOffsetTarget(const Instruction* target);
+  // Patch a literal load instruction to load from 'source'.
+  void SetImmLLiteral(const Instruction* source);
+
+  // The range of a load literal instruction, expressed as 'instr +- range'.
+  // The range is actually the 'positive' range; the branch instruction can
+  // target [instr - range - kInstructionSize, instr + range].
+  static const int kLoadLiteralImmBitwidth = 19;
+  static const int kLoadLiteralRange =
+      (1 << kLoadLiteralImmBitwidth) / 2 - kInstructionSize;
+
+  // Calculate the address of a literal referred to by a load-literal
+  // instruction, and return it as the specified type.
+  //
+  // The literal itself is safely mutable only if the backing buffer is safely
+  // mutable.
+  template <typename T>
+  T LiteralAddress() const {
+    uint64_t base_raw = reinterpret_cast<uint64_t>(this);
+    int64_t offset = ImmLLiteral() << kLiteralEntrySizeLog2;
+    uint64_t address_raw = base_raw + offset;
+
+    // Cast the address using a C-style cast. A reinterpret_cast would be
+    // appropriate, but it can't cast one integral type to another.
+    T address = (T)(address_raw);
+
+    // Assert that the address can be represented by the specified type.
+    VIXL_ASSERT((uint64_t)(address) == address_raw);
+
+    return address;
+  }
+
+  uint32_t Literal32() const {
+    uint32_t literal;
+    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+    return literal;
+  }
+
+  uint64_t Literal64() const {
+    uint64_t literal;
+    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+    return literal;
+  }
+
+  float LiteralFP32() const {
+    return rawbits_to_float(Literal32());
+  }
+
+  double LiteralFP64() const {
+    return rawbits_to_double(Literal64());
+  }
+
+  const Instruction* NextInstruction() const {
+    return this + kInstructionSize;
+  }
+
+  const Instruction* InstructionAtOffset(int64_t offset) const {
+    VIXL_ASSERT(IsWordAligned(this + offset));
+    return this + offset;
+  }
+
+  template<typename T> static Instruction* Cast(T src) {
+    return reinterpret_cast<Instruction*>(src);
+  }
+
+  template<typename T> static const Instruction* CastConst(T src) {
+    return reinterpret_cast<const Instruction*>(src);
+  }
+
+ private:
+  int ImmBranch() const;
+
+  static float Imm8ToFP32(uint32_t imm8);
+  static double Imm8ToFP64(uint32_t imm8);
+
+  void SetPCRelImmTarget(const Instruction* target);
+  void SetBranchImmTarget(const Instruction* target);
+};
+
+
+// Functions for handling NEON vector format information.
+enum VectorFormat {
+  kFormatUndefined = 0xffffffff,
+  kFormat8B  = NEON_8B,
+  kFormat16B = NEON_16B,
+  kFormat4H  = NEON_4H,
+  kFormat8H  = NEON_8H,
+  kFormat2S  = NEON_2S,
+  kFormat4S  = NEON_4S,
+  kFormat1D  = NEON_1D,
+  kFormat2D  = NEON_2D,
+
+  // Scalar formats. We add the scalar bit to distinguish between scalar and
+  // vector enumerations; the bit is always set in the encoding of scalar ops
+  // and always clear for vector ops. Although kFormatD and kFormat1D appear
+  // to be the same, their meaning is subtly different. The first is a scalar
+  // operation, the second a vector operation that only affects one lane.
+  kFormatB = NEON_B | NEONScalar,
+  kFormatH = NEON_H | NEONScalar,
+  kFormatS = NEON_S | NEONScalar,
+  kFormatD = NEON_D | NEONScalar
+};
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform);
+VectorFormat ScalarFormatFromLaneSize(int lanesize);
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatFillQ(const VectorFormat vform);
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform);
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform);
+// TODO: Make the return types of these functions consistent.
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform);
+int LaneSizeInBytesFromFormat(VectorFormat vform);
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform);
+int LaneCountFromFormat(VectorFormat vform);
+int MaxLaneCountFromFormat(VectorFormat vform);
+bool IsVectorFormat(VectorFormat vform);
+int64_t MaxIntFromFormat(VectorFormat vform);
+int64_t MinIntFromFormat(VectorFormat vform);
+uint64_t MaxUintFromFormat(VectorFormat vform);
+
+
+enum NEONFormat {
+  NF_UNDEF = 0,
+  NF_8B    = 1,
+  NF_16B   = 2,
+  NF_4H    = 3,
+  NF_8H    = 4,
+  NF_2S    = 5,
+  NF_4S    = 6,
+  NF_1D    = 7,
+  NF_2D    = 8,
+  NF_B     = 9,
+  NF_H     = 10,
+  NF_S     = 11,
+  NF_D     = 12
+};
+
+static const unsigned kNEONFormatMaxBits = 6;
+
+struct NEONFormatMap {
+  // The bit positions in the instruction to consider.
+  uint8_t bits[kNEONFormatMaxBits];
+
+  // Mapping from concatenated bits to format.
+  NEONFormat map[1 << kNEONFormatMaxBits];
+};
+
+class NEONFormatDecoder {
+ public:
+  enum SubstitutionMode {
+    kPlaceholder,
+    kFormat
+  };
+
+  // Construct a format decoder with increasingly specific format maps for each
+  // subsitution. If no format map is specified, the default is the integer
+  // format map.
+  explicit NEONFormatDecoder(const Instruction* instr) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(IntegerFormatMap());
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format);
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format0,
+                    const NEONFormatMap* format1) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format0, format1);
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format0,
+                    const NEONFormatMap* format1,
+                    const NEONFormatMap* format2) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format0, format1, format2);
+  }
+
+  // Set the format mapping for all or individual substitutions.
+  void SetFormatMaps(const NEONFormatMap* format0,
+                     const NEONFormatMap* format1 = NULL,
+                     const NEONFormatMap* format2 = NULL) {
+    VIXL_ASSERT(format0 != NULL);
+    formats_[0] = format0;
+    formats_[1] = (format1 == NULL) ? formats_[0] : format1;
+    formats_[2] = (format2 == NULL) ? formats_[1] : format2;
+  }
+  void SetFormatMap(unsigned index, const NEONFormatMap* format) {
+    VIXL_ASSERT(index <= (sizeof(formats_) / sizeof(formats_[0])));
+    VIXL_ASSERT(format != NULL);
+    formats_[index] = format;
+  }
+
+  // Substitute %s in the input string with the placeholder string for each
+  // register, ie. "'B", "'H", etc.
+  const char* SubstitutePlaceholders(const char* string) {
+    return Substitute(string, kPlaceholder, kPlaceholder, kPlaceholder);
+  }
+
+  // Substitute %s in the input string with a new string based on the
+  // substitution mode.
+  const char* Substitute(const char* string,
+                         SubstitutionMode mode0 = kFormat,
+                         SubstitutionMode mode1 = kFormat,
+                         SubstitutionMode mode2 = kFormat) {
+    snprintf(form_buffer_, sizeof(form_buffer_), string,
+             GetSubstitute(0, mode0),
+             GetSubstitute(1, mode1),
+             GetSubstitute(2, mode2));
+    return form_buffer_;
+  }
+
+  // Append a "2" to a mnemonic string based of the state of the Q bit.
+  const char* Mnemonic(const char* mnemonic) {
+    if ((instrbits_ & NEON_Q) != 0) {
+      snprintf(mne_buffer_, sizeof(mne_buffer_), "%s2", mnemonic);
+      return mne_buffer_;
+    }
+    return mnemonic;
+  }
+
+  VectorFormat GetVectorFormat(int format_index = 0) {
+    return GetVectorFormat(formats_[format_index]);
+  }
+
+  VectorFormat GetVectorFormat(const NEONFormatMap* format_map) {
+    static const VectorFormat vform[] = {
+      kFormatUndefined,
+      kFormat8B, kFormat16B, kFormat4H, kFormat8H,
+      kFormat2S, kFormat4S, kFormat1D, kFormat2D,
+      kFormatB, kFormatH, kFormatS, kFormatD
+    };
+    VIXL_ASSERT(GetNEONFormat(format_map) < (sizeof(vform) / sizeof(vform[0])));
+    return vform[GetNEONFormat(format_map)];
+  }
+
+  // Built in mappings for common cases.
+
+  // The integer format map uses three bits (Q, size<1:0>) to encode the
+  // "standard" set of NEON integer vector formats.
+  static const NEONFormatMap* IntegerFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22, 30},
+      {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_UNDEF, NF_2D}
+    };
+    return &map;
+  }
+
+  // The long integer format map uses two bits (size<1:0>) to encode the
+  // long set of NEON integer vector formats. These are used in narrow, wide
+  // and long operations.
+  static const NEONFormatMap* LongIntegerFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_8H, NF_4S, NF_2D}
+    };
+    return &map;
+  }
+
+  // The FP format map uses two bits (Q, size<0>) to encode the NEON FP vector
+  // formats: NF_2S, NF_4S, NF_2D.
+  static const NEONFormatMap* FPFormatMap() {
+    // The FP format map assumes two bits (Q, size<0>) are used to encode the
+    // NEON FP vector formats: NF_2S, NF_4S, NF_2D.
+    static const NEONFormatMap map = {
+      {22, 30}, {NF_2S, NF_4S, NF_UNDEF, NF_2D}
+    };
+    return &map;
+  }
+
+  // The load/store format map uses three bits (Q, 11, 10) to encode the
+  // set of NEON vector formats.
+  static const NEONFormatMap* LoadStoreFormatMap() {
+    static const NEONFormatMap map = {
+      {11, 10, 30},
+      {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+    };
+    return &map;
+  }
+
+  // The logical format map uses one bit (Q) to encode the NEON vector format:
+  // NF_8B, NF_16B.
+  static const NEONFormatMap* LogicalFormatMap() {
+    static const NEONFormatMap map = {
+      {30}, {NF_8B, NF_16B}
+    };
+    return &map;
+  }
+
+  // The triangular format map uses between two and five bits to encode the NEON
+  // vector format:
+  // xxx10->8B, xxx11->16B, xx100->4H, xx101->8H
+  // x1000->2S, x1001->4S,  10001->2D, all others undefined.
+  static const NEONFormatMap* TriangularFormatMap() {
+    static const NEONFormatMap map = {
+      {19, 18, 17, 16, 30},
+      {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S,
+       NF_4S, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_UNDEF, NF_2D,
+       NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S, NF_4S, NF_8B, NF_16B,
+       NF_4H, NF_8H, NF_8B, NF_16B}
+    };
+    return &map;
+  }
+
+  // The scalar format map uses two bits (size<1:0>) to encode the NEON scalar
+  // formats: NF_B, NF_H, NF_S, NF_D.
+  static const NEONFormatMap* ScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_B, NF_H, NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The long scalar format map uses two bits (size<1:0>) to encode the longer
+  // NEON scalar formats: NF_H, NF_S, NF_D.
+  static const NEONFormatMap* LongScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_H, NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The FP scalar format map assumes one bit (size<0>) is used to encode the
+  // NEON FP scalar formats: NF_S, NF_D.
+  static const NEONFormatMap* FPScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {22}, {NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The triangular scalar format map uses between one and four bits to encode
+  // the NEON FP scalar formats:
+  // xxx1->B, xx10->H, x100->S, 1000->D, all others undefined.
+  static const NEONFormatMap* TriangularScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {19, 18, 17, 16},
+      {NF_UNDEF, NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B,
+       NF_D,     NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B}
+    };
+    return &map;
+  }
+
+ private:
+  // Get a pointer to a string that represents the format or placeholder for
+  // the specified substitution index, based on the format map and instruction.
+  const char* GetSubstitute(int index, SubstitutionMode mode) {
+    if (mode == kFormat) {
+      return NEONFormatAsString(GetNEONFormat(formats_[index]));
+    }
+    VIXL_ASSERT(mode == kPlaceholder);
+    return NEONFormatAsPlaceholder(GetNEONFormat(formats_[index]));
+  }
+
+  // Get the NEONFormat enumerated value for bits obtained from the
+  // instruction based on the specified format mapping.
+  NEONFormat GetNEONFormat(const NEONFormatMap* format_map) {
+    return format_map->map[PickBits(format_map->bits)];
+  }
+
+  // Convert a NEONFormat into a string.
+  static const char* NEONFormatAsString(NEONFormat format) {
+    static const char* formats[] = {
+      "undefined",
+      "8b", "16b", "4h", "8h", "2s", "4s", "1d", "2d",
+      "b", "h", "s", "d"
+    };
+    VIXL_ASSERT(format < (sizeof(formats) / sizeof(formats[0])));
+    return formats[format];
+  }
+
+  // Convert a NEONFormat into a register placeholder string.
+  static const char* NEONFormatAsPlaceholder(NEONFormat format) {
+    VIXL_ASSERT((format == NF_B) || (format == NF_H) ||
+                (format == NF_S) || (format == NF_D) ||
+                (format == NF_UNDEF));
+    static const char* formats[] = {
+      "undefined",
+      "undefined", "undefined", "undefined", "undefined",
+      "undefined", "undefined", "undefined", "undefined",
+      "'B", "'H", "'S", "'D"
+    };
+    return formats[format];
+  }
+
+  // Select bits from instrbits_ defined by the bits array, concatenate them,
+  // and return the value.
+  uint8_t PickBits(const uint8_t bits[]) {
+    uint8_t result = 0;
+    for (unsigned b = 0; b < kNEONFormatMaxBits; b++) {
+      if (bits[b] == 0) break;
+      result <<= 1;
+      result |= ((instrbits_ & (1 << bits[b])) == 0) ? 0 : 1;
+    }
+    return result;
+  }
+
+  Instr instrbits_;
+  const NEONFormatMap* formats_[3];
+  char form_buffer_[64];
+  char mne_buffer_[16];
+};
+}  // namespace vixl
+
+#endif  // VIXL_A64_INSTRUCTIONS_A64_H_
diff --git a/qemu/disas/libvixl/code-buffer.h b/qemu/disas/libvixl/vixl/code-buffer.h
index da6233dd8..f93ebb6b8 100644
--- a/qemu/disas/libvixl/code-buffer.h
+++ b/qemu/disas/libvixl/vixl/code-buffer.h
@@ -28,7 +28,7 @@
 #define VIXL_CODE_BUFFER_H
 
 #include <string.h>
-#include "globals.h"
+#include "vixl/globals.h"
 
 namespace vixl {
 
diff --git a/qemu/disas/libvixl/utils.cc b/qemu/disas/libvixl/vixl/compiler-intrinsics.cc
index 80b132a11..fd551faeb 100644
--- a/qemu/disas/libvixl/utils.cc
+++ b/qemu/disas/libvixl/vixl/compiler-intrinsics.cc
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2015, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -24,53 +24,13 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include "utils.h"
-#include <stdio.h>
+#include "compiler-intrinsics.h"
 
 namespace vixl {
 
-uint32_t float_to_rawbits(float value) {
-  uint32_t bits = 0;
-  memcpy(&bits, &value, 4);
-  return bits;
-}
-
-
-uint64_t double_to_rawbits(double value) {
-  uint64_t bits = 0;
-  memcpy(&bits, &value, 8);
-  return bits;
-}
-
-
-float rawbits_to_float(uint32_t bits) {
-  float value = 0.0;
-  memcpy(&value, &bits, 4);
-  return value;
-}
-
-
-double rawbits_to_double(uint64_t bits) {
-  double value = 0.0;
-  memcpy(&value, &bits, 8);
-  return value;
-}
-
 
-int CountLeadingZeros(uint64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
-  int count = 0;
-  uint64_t bit_test = UINT64_C(1) << (width - 1);
-  while ((count < width) && ((bit_test & value) == 0)) {
-    count++;
-    bit_test >>= 1;
-  }
-  return count;
-}
-
-
-int CountLeadingSignBits(int64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
+int CountLeadingSignBitsFallBack(int64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
   if (value >= 0) {
     return CountLeadingZeros(value, width) - 1;
   } else {
@@ -79,23 +39,46 @@ int CountLeadingSignBits(int64_t value, int width) {
 }
 
 
-int CountTrailingZeros(uint64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
+int CountLeadingZerosFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+  if (value == 0) {
+    return width;
+  }
   int count = 0;
-  while ((count < width) && (((value >> count) & 1) == 0)) {
-    count++;
+  value = value << (64 - width);
+  if ((value & UINT64_C(0xffffffff00000000)) == 0) {
+    count += 32;
+    value = value << 32;
   }
+  if ((value & UINT64_C(0xffff000000000000)) == 0) {
+    count += 16;
+    value = value << 16;
+  }
+  if ((value & UINT64_C(0xff00000000000000)) == 0) {
+    count += 8;
+    value = value << 8;
+  }
+  if ((value & UINT64_C(0xf000000000000000)) == 0) {
+    count += 4;
+    value = value << 4;
+  }
+  if ((value & UINT64_C(0xc000000000000000)) == 0) {
+    count += 2;
+    value = value << 2;
+  }
+  if ((value & UINT64_C(0x8000000000000000)) == 0) {
+    count += 1;
+  }
+  count += (value == 0);
   return count;
 }
 
 
-int CountSetBits(uint64_t value, int width) {
-  // TODO: Other widths could be added here, as the implementation already
-  // supports them.
-  VIXL_ASSERT((width == 32) || (width == 64));
+int CountSetBitsFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
 
   // Mask out unused bits to ensure that they are not counted.
-  value &= (UINT64_C(0xffffffffffffffff) >> (64-width));
+  value &= (UINT64_C(0xffffffffffffffff) >> (64 - width));
 
   // Add up the set bits.
   // The algorithm works by adding pairs of bit fields together iteratively,
@@ -122,30 +105,40 @@ int CountSetBits(uint64_t value, int width) {
     value = ((value >> shift) & kMasks[i]) + (value & kMasks[i]);
   }
 
-  return value;
-}
-
-
-uint64_t LowestSetBit(uint64_t value) {
-  return value & -value;
-}
-
-
-bool IsPowerOf2(int64_t value) {
-  return (value != 0) && ((value & (value - 1)) == 0);
+  return static_cast<int>(value);
 }
 
 
-unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size) {
-  VIXL_ASSERT((reg_size % 8) == 0);
+int CountTrailingZerosFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
   int count = 0;
-  for (unsigned i = 0; i < (reg_size / 16); i++) {
-    if ((imm & 0xffff) == 0) {
-      count++;
-    }
-    imm >>= 16;
+  value = value << (64 - width);
+  if ((value & UINT64_C(0xffffffff)) == 0) {
+    count += 32;
+    value = value >> 32;
   }
-  return count;
+  if ((value & 0xffff) == 0) {
+    count += 16;
+    value = value >> 16;
+  }
+  if ((value & 0xff) == 0) {
+    count += 8;
+    value = value >> 8;
+  }
+  if ((value & 0xf) == 0) {
+    count += 4;
+    value = value >> 4;
+  }
+  if ((value & 0x3) == 0) {
+    count += 2;
+    value = value >> 2;
+  }
+  if ((value & 0x1) == 0) {
+    count += 1;
+  }
+  count += (value == 0);
+  return count - (64 - width);
 }
 
+
 }  // namespace vixl
diff --git a/qemu/disas/libvixl/vixl/compiler-intrinsics.h b/qemu/disas/libvixl/vixl/compiler-intrinsics.h
new file mode 100644
index 000000000..9431beddb
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/compiler-intrinsics.h
@@ -0,0 +1,155 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+#ifndef VIXL_COMPILER_INTRINSICS_H
+#define VIXL_COMPILER_INTRINSICS_H
+
+#include "globals.h"
+
+namespace vixl {
+
+// Helper to check whether the version of GCC used is greater than the specified
+// requirement.
+#define MAJOR 1000000
+#define MINOR 1000
+#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel)                         \
+    ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR + __GNUC_PATCHLEVEL__) >=      \
+     ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel)                         \
+    ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR) >=                            \
+     ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#else
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel) 0
+#endif
+
+
+#if defined(__clang__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+
+#define COMPILER_HAS_BUILTIN_CLRSB    (__has_builtin(__builtin_clrsb))
+#define COMPILER_HAS_BUILTIN_CLZ      (__has_builtin(__builtin_clz))
+#define COMPILER_HAS_BUILTIN_CTZ      (__has_builtin(__builtin_ctz))
+#define COMPILER_HAS_BUILTIN_FFS      (__has_builtin(__builtin_ffs))
+#define COMPILER_HAS_BUILTIN_POPCOUNT (__has_builtin(__builtin_popcount))
+
+#elif defined(__GNUC__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+// The documentation for these builtins is available at:
+// https://gcc.gnu.org/onlinedocs/gcc-$MAJOR.$MINOR.$PATCHLEVEL/gcc//Other-Builtins.html
+
+# define COMPILER_HAS_BUILTIN_CLRSB    (GCC_VERSION_OR_NEWER(4, 7, 0))
+# define COMPILER_HAS_BUILTIN_CLZ      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_CTZ      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_FFS      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_POPCOUNT (GCC_VERSION_OR_NEWER(3, 4, 0))
+
+#else
+// One can define VIXL_NO_COMPILER_BUILTINS to force using the manually
+// implemented C++ methods.
+
+#define COMPILER_HAS_BUILTIN_BSWAP    false
+#define COMPILER_HAS_BUILTIN_CLRSB    false
+#define COMPILER_HAS_BUILTIN_CLZ      false
+#define COMPILER_HAS_BUILTIN_CTZ      false
+#define COMPILER_HAS_BUILTIN_FFS      false
+#define COMPILER_HAS_BUILTIN_POPCOUNT false
+
+#endif
+
+
+template<typename V>
+inline bool IsPowerOf2(V value) {
+  return (value != 0) && ((value & (value - 1)) == 0);
+}
+
+
+// Declaration of fallback functions.
+int CountLeadingSignBitsFallBack(int64_t value, int width);
+int CountLeadingZerosFallBack(uint64_t value, int width);
+int CountSetBitsFallBack(uint64_t value, int width);
+int CountTrailingZerosFallBack(uint64_t value, int width);
+
+
+// Implementation of intrinsics functions.
+// TODO: The implementations could be improved for sizes different from 32bit
+// and 64bit: we could mask the values and call the appropriate builtin.
+
+template<typename V>
+inline int CountLeadingSignBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLRSB
+  if (width == 32) {
+    return __builtin_clrsb(value);
+  } else if (width == 64) {
+    return __builtin_clrsbll(value);
+  }
+#endif
+  return CountLeadingSignBitsFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountLeadingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLZ
+  if (width == 32) {
+    return (value == 0) ? 32 : __builtin_clz(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return (value == 0) ? 64 : __builtin_clzll(value);
+  }
+#endif
+  return CountLeadingZerosFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountSetBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_POPCOUNT
+  if (width == 32) {
+    return __builtin_popcount(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return __builtin_popcountll(value);
+  }
+#endif
+  return CountSetBitsFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountTrailingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CTZ
+  if (width == 32) {
+    return (value == 0) ? 32 : __builtin_ctz(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return (value == 0) ? 64 : __builtin_ctzll(value);
+  }
+#endif
+  return CountTrailingZerosFallBack(value, width);
+}
+
+}  // namespace vixl
+
+#endif  // VIXL_COMPILER_INTRINSICS_H
+
diff --git a/qemu/disas/libvixl/globals.h b/qemu/disas/libvixl/vixl/globals.h
index 0c2493105..61dc9f7f7 100644
--- a/qemu/disas/libvixl/globals.h
+++ b/qemu/disas/libvixl/vixl/globals.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2015, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -49,20 +49,26 @@
 #include <stdint.h>
 #include <stdlib.h>
 #include <stddef.h>
-#include "platform.h"
+#include "vixl/platform.h"
 
 
 typedef uint8_t byte;
 
+// Type for half-precision (16 bit) floating point numbers.
+typedef uint16_t float16;
+
 const int KBytes = 1024;
 const int MBytes = 1024 * KBytes;
 
-#define VIXL_ABORT() printf("in %s, line %i", __FILE__, __LINE__); abort()
+#define VIXL_ABORT() \
+    do { printf("in %s, line %i", __FILE__, __LINE__); abort(); } while (false)
 #ifdef VIXL_DEBUG
   #define VIXL_ASSERT(condition) assert(condition)
   #define VIXL_CHECK(condition) VIXL_ASSERT(condition)
-  #define VIXL_UNIMPLEMENTED() printf("UNIMPLEMENTED\t"); VIXL_ABORT()
-  #define VIXL_UNREACHABLE() printf("UNREACHABLE\t"); VIXL_ABORT()
+  #define VIXL_UNIMPLEMENTED() \
+    do { fprintf(stderr, "UNIMPLEMENTED\t"); VIXL_ABORT(); } while (false)
+  #define VIXL_UNREACHABLE() \
+    do { fprintf(stderr, "UNREACHABLE\t"); VIXL_ABORT(); } while (false)
 #else
   #define VIXL_ASSERT(condition) ((void) 0)
   #define VIXL_CHECK(condition) assert(condition)
@@ -76,10 +82,70 @@ const int MBytes = 1024 * KBytes;
 #define VIXL_STATIC_ASSERT_LINE(line, condition) \
   typedef char VIXL_CONCAT(STATIC_ASSERT_LINE_, line)[(condition) ? 1 : -1] \
   __attribute__((unused))
-#define VIXL_STATIC_ASSERT(condition) VIXL_STATIC_ASSERT_LINE(__LINE__, condition) //NOLINT
+#define VIXL_STATIC_ASSERT(condition) \
+    VIXL_STATIC_ASSERT_LINE(__LINE__, condition)
+
+template <typename T1>
+inline void USE(T1) {}
+
+template <typename T1, typename T2>
+inline void USE(T1, T2) {}
+
+template <typename T1, typename T2, typename T3>
+inline void USE(T1, T2, T3) {}
+
+template <typename T1, typename T2, typename T3, typename T4>
+inline void USE(T1, T2, T3, T4) {}
+
+#define VIXL_ALIGNMENT_EXCEPTION() \
+    do { fprintf(stderr, "ALIGNMENT EXCEPTION\t"); VIXL_ABORT(); } while (0)
+
+// The clang::fallthrough attribute is used along with the Wimplicit-fallthrough
+// argument to annotate intentional fall-through between switch labels.
+// For more information please refer to:
+// http://clang.llvm.org/docs/AttributeReference.html#fallthrough-clang-fallthrough
+#ifndef __has_warning
+  #define __has_warning(x)  0
+#endif
+
+// Note: This option is only available for Clang. And will only be enabled for
+// C++11(201103L).
+#if __has_warning("-Wimplicit-fallthrough") && __cplusplus >= 201103L
+  #define VIXL_FALLTHROUGH() [[clang::fallthrough]] //NOLINT
+#else
+  #define VIXL_FALLTHROUGH() do {} while (0)
+#endif
+
+#if __cplusplus >= 201103L
+  #define VIXL_NO_RETURN  [[noreturn]] //NOLINT
+#else
+  #define VIXL_NO_RETURN  __attribute__((noreturn))
+#endif
+
+// Some functions might only be marked as "noreturn" for the DEBUG build. This
+// macro should be used for such cases (for more details see what
+// VIXL_UNREACHABLE expands to).
+#ifdef VIXL_DEBUG
+  #define VIXL_DEBUG_NO_RETURN  VIXL_NO_RETURN
+#else
+  #define VIXL_DEBUG_NO_RETURN
+#endif
 
-template <typename T> inline void USE(T) {}
+#ifdef VIXL_INCLUDE_SIMULATOR
+#ifndef VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #define VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE  1
+#endif
+#else
+#ifndef VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #define VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE  0
+#endif
+#if VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #warning "Generating Simulator instructions without Simulator support."
+#endif
+#endif
 
-#define VIXL_ALIGNMENT_EXCEPTION() printf("ALIGNMENT EXCEPTION\t"); VIXL_ABORT()
+#ifdef USE_SIMULATOR
+  #error "Please see the release notes for USE_SIMULATOR."
+#endif
 
 #endif  // VIXL_GLOBALS_H
diff --git a/qemu/disas/libvixl/vixl/invalset.h b/qemu/disas/libvixl/vixl/invalset.h
new file mode 100644
index 000000000..ffdc0237b
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/invalset.h
@@ -0,0 +1,775 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_INVALSET_H_
+#define VIXL_INVALSET_H_
+
+#include <string.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "vixl/globals.h"
+
+namespace vixl {
+
+// We define a custom data structure template and its iterator as `std`
+// containers do not fit the performance requirements for some of our use cases.
+//
+// The structure behaves like an iterable unordered set with special properties
+// and restrictions. "InvalSet" stands for "Invalidatable Set".
+//
+// Restrictions and requirements:
+// - Adding an element already present in the set is illegal. In debug mode,
+//   this is checked at insertion time.
+// - The templated class `ElementType` must provide comparison operators so that
+//   `std::sort()` can be used.
+// - A key must be available to represent invalid elements.
+// - Elements with an invalid key must compare higher or equal to any other
+//   element.
+//
+// Use cases and performance considerations:
+// Our use cases present two specificities that allow us to design this
+// structure to provide fast insertion *and* fast search and deletion
+// operations:
+// - Elements are (generally) inserted in order (sorted according to their key).
+// - A key is available to mark elements as invalid (deleted).
+// The backing `std::vector` allows for fast insertions. When
+// searching for an element we ensure the elements are sorted (this is generally
+// the case) and perform a binary search. When deleting an element we do not
+// free the associated memory immediately. Instead, an element to be deleted is
+// marked with the 'invalid' key. Other methods of the container take care of
+// ignoring entries marked as invalid.
+// To avoid the overhead of the `std::vector` container when only few entries
+// are used, a number of elements are preallocated.
+
+// 'ElementType' and 'KeyType' are respectively the types of the elements and
+// their key.  The structure only reclaims memory when safe to do so, if the
+// number of elements that can be reclaimed is greater than `RECLAIM_FROM` and
+// greater than `<total number of elements> / RECLAIM_FACTOR.
+#define TEMPLATE_INVALSET_P_DECL                                               \
+  class ElementType,                                                           \
+  unsigned N_PREALLOCATED_ELEMENTS,                                            \
+  class KeyType,                                                               \
+  KeyType INVALID_KEY,                                                         \
+  size_t RECLAIM_FROM,                                                         \
+  unsigned RECLAIM_FACTOR
+
+#define TEMPLATE_INVALSET_P_DEF                                                \
+ElementType, N_PREALLOCATED_ELEMENTS,                                          \
+KeyType, INVALID_KEY, RECLAIM_FROM, RECLAIM_FACTOR
+
+template<class S> class InvalSetIterator;  // Forward declaration.
+
+template<TEMPLATE_INVALSET_P_DECL> class InvalSet {
+ public:
+  InvalSet();
+  ~InvalSet();
+
+  static const size_t kNPreallocatedElements = N_PREALLOCATED_ELEMENTS;
+  static const KeyType kInvalidKey = INVALID_KEY;
+
+  // It is illegal to insert an element already present in the set.
+  void insert(const ElementType& element);
+
+  // Looks for the specified element in the set and - if found - deletes it.
+  void erase(const ElementType& element);
+
+  // This indicates the number of (valid) elements stored in this set.
+  size_t size() const;
+
+  // Returns true if no elements are stored in the set.
+  // Note that this does not mean the the backing storage is empty: it can still
+  // contain invalid elements.
+  bool empty() const;
+
+  void clear();
+
+  const ElementType min_element();
+
+  // This returns the key of the minimum element in the set.
+  KeyType min_element_key();
+
+  static bool IsValid(const ElementType& element);
+  static KeyType Key(const ElementType& element);
+  static void SetKey(ElementType* element, KeyType key);
+
+ protected:
+  // Returns a pointer to the element in vector_ if it was found, or NULL
+  // otherwise.
+  ElementType* Search(const ElementType& element);
+
+  // The argument *must* point to an element stored in *this* set.
+  // This function is not allowed to move elements in the backing vector
+  // storage.
+  void EraseInternal(ElementType* element);
+
+  // The elements in the range searched must be sorted.
+  ElementType* BinarySearch(const ElementType& element,
+                            ElementType* start,
+                            ElementType* end) const;
+
+  // Sort the elements.
+  enum SortType {
+    // The 'hard' version guarantees that invalid elements are moved to the end
+    // of the container.
+    kHardSort,
+    // The 'soft' version only guarantees that the elements will be sorted.
+    // Invalid elements may still be present anywhere in the set.
+    kSoftSort
+  };
+  void Sort(SortType sort_type);
+
+  // Delete the elements that have an invalid key. The complexity is linear
+  // with the size of the vector.
+  void Clean();
+
+  const ElementType Front() const;
+  const ElementType Back() const;
+
+  // Delete invalid trailing elements and return the last valid element in the
+  // set.
+  const ElementType CleanBack();
+
+  // Returns a pointer to the start or end of the backing storage.
+  const ElementType* StorageBegin() const;
+  const ElementType* StorageEnd() const;
+  ElementType* StorageBegin();
+  ElementType* StorageEnd();
+
+  // Returns the index of the element within the backing storage. The element
+  // must belong to the backing storage.
+  size_t ElementIndex(const ElementType* element) const;
+
+  // Returns the element at the specified index in the backing storage.
+  const ElementType* ElementAt(size_t index) const;
+  ElementType* ElementAt(size_t index);
+
+  static const ElementType* FirstValidElement(const ElementType* from,
+                                              const ElementType* end);
+
+  void CacheMinElement();
+  const ElementType CachedMinElement() const;
+
+  bool ShouldReclaimMemory() const;
+  void ReclaimMemory();
+
+  bool IsUsingVector() const { return vector_ != NULL; }
+  void set_sorted(bool sorted) { sorted_ = sorted; }
+
+  // We cache some data commonly required by users to improve performance.
+  // We cannot cache pointers to elements as we do not control the backing
+  // storage.
+  bool valid_cached_min_;
+  size_t cached_min_index_;  // Valid iff `valid_cached_min_` is true.
+  KeyType cached_min_key_;         // Valid iff `valid_cached_min_` is true.
+
+  // Indicates whether the elements are sorted.
+  bool sorted_;
+
+  // This represents the number of (valid) elements in this set.
+  size_t size_;
+
+  // The backing storage is either the array of preallocated elements or the
+  // vector. The structure starts by using the preallocated elements, and
+  // transitions (permanently) to using the vector once more than
+  // kNPreallocatedElements are used.
+  // Elements are only invalidated when using the vector. The preallocated
+  // storage always only contains valid elements.
+  ElementType preallocated_[kNPreallocatedElements];
+  std::vector<ElementType>* vector_;
+
+#ifdef VIXL_DEBUG
+  // Iterators acquire and release this monitor. While a set is acquired,
+  // certain operations are illegal to ensure that the iterator will
+  // correctly iterate over the elements in the set.
+  int monitor_;
+  int monitor() const { return monitor_; }
+  void Acquire() { monitor_++; }
+  void Release() {
+    monitor_--;
+    VIXL_ASSERT(monitor_ >= 0);
+  }
+#endif
+
+  friend class InvalSetIterator<InvalSet<TEMPLATE_INVALSET_P_DEF> >;
+  typedef ElementType _ElementType;
+  typedef KeyType _KeyType;
+};
+
+
+template<class S> class InvalSetIterator {
+ private:
+  // Redefine types to mirror the associated set types.
+  typedef typename S::_ElementType ElementType;
+  typedef typename S::_KeyType KeyType;
+
+ public:
+  explicit InvalSetIterator(S* inval_set);
+  ~InvalSetIterator();
+
+  ElementType* Current() const;
+  void Advance();
+  bool Done() const;
+
+  // Mark this iterator as 'done'.
+  void Finish();
+
+  // Delete the current element and advance the iterator to point to the next
+  // element.
+  void DeleteCurrentAndAdvance();
+
+  static bool IsValid(const ElementType& element);
+  static KeyType Key(const ElementType& element);
+
+ protected:
+  void MoveToValidElement();
+
+  // Indicates if the iterator is looking at the vector or at the preallocated
+  // elements.
+  const bool using_vector_;
+  // Used when looking at the preallocated elements, or in debug mode when using
+  // the vector to track how many times the iterator has advanced.
+  size_t index_;
+  typename std::vector<ElementType>::iterator iterator_;
+  S* inval_set_;
+};
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+InvalSet<TEMPLATE_INVALSET_P_DEF>::InvalSet()
+  : valid_cached_min_(false),
+    sorted_(true), size_(0), vector_(NULL) {
+#ifdef VIXL_DEBUG
+  monitor_ = 0;
+#endif
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+InvalSet<TEMPLATE_INVALSET_P_DEF>::~InvalSet() {
+  VIXL_ASSERT(monitor_ == 0);
+  delete vector_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::insert(const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(IsValid(element));
+  VIXL_ASSERT(Search(element) == NULL);
+  set_sorted(empty() || (sorted_ && (element > CleanBack())));
+  if (IsUsingVector()) {
+    vector_->push_back(element);
+  } else {
+    if (size_ < kNPreallocatedElements) {
+      preallocated_[size_] = element;
+    } else {
+      // Transition to using the vector.
+      vector_ = new std::vector<ElementType>(preallocated_,
+                                             preallocated_ + size_);
+      vector_->push_back(element);
+    }
+  }
+  size_++;
+
+  if (valid_cached_min_ && (element < min_element())) {
+    cached_min_index_ = IsUsingVector() ? vector_->size() - 1 : size_ - 1;
+    cached_min_key_ = Key(element);
+    valid_cached_min_ = true;
+  }
+
+  if (ShouldReclaimMemory()) {
+    ReclaimMemory();
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::erase(const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(IsValid(element));
+  ElementType* local_element = Search(element);
+  if (local_element != NULL) {
+    EraseInternal(local_element);
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::Search(
+    const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  if (empty()) {
+    return NULL;
+  }
+  if (ShouldReclaimMemory()) {
+    ReclaimMemory();
+  }
+  if (!sorted_) {
+    Sort(kHardSort);
+  }
+  if (!valid_cached_min_) {
+    CacheMinElement();
+  }
+  return BinarySearch(element, ElementAt(cached_min_index_), StorageEnd());
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+size_t InvalSet<TEMPLATE_INVALSET_P_DEF>::size() const {
+  return size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::empty() const {
+  return size_ == 0;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::clear() {
+  VIXL_ASSERT(monitor() == 0);
+  size_ = 0;
+  if (IsUsingVector()) {
+    vector_->clear();
+  }
+  set_sorted(true);
+  valid_cached_min_ = false;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::min_element() {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(!empty());
+  CacheMinElement();
+  return *ElementAt(cached_min_index_);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+KeyType InvalSet<TEMPLATE_INVALSET_P_DEF>::min_element_key() {
+  VIXL_ASSERT(monitor() == 0);
+  if (valid_cached_min_) {
+    return cached_min_key_;
+  } else {
+    return Key(min_element());
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::IsValid(const ElementType& element) {
+  return Key(element) != kInvalidKey;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::EraseInternal(ElementType* element) {
+  // Note that this function must be safe even while an iterator has acquired
+  // this set.
+  VIXL_ASSERT(element != NULL);
+  size_t deleted_index = ElementIndex(element);
+  if (IsUsingVector()) {
+    VIXL_ASSERT((&(vector_->front()) <= element) &&
+                (element <= &(vector_->back())));
+    SetKey(element, kInvalidKey);
+  } else {
+    VIXL_ASSERT((preallocated_ <= element) &&
+                (element < (preallocated_ + kNPreallocatedElements)));
+    ElementType* end = preallocated_ + kNPreallocatedElements;
+    size_t copy_size = sizeof(*element) * (end - element - 1);
+    memmove(element, element + 1, copy_size);
+  }
+  size_--;
+
+  if (valid_cached_min_ &&
+      (deleted_index == cached_min_index_)) {
+    if (sorted_ && !empty()) {
+      const ElementType* min = FirstValidElement(element, StorageEnd());
+      cached_min_index_ = ElementIndex(min);
+      cached_min_key_ = Key(*min);
+      valid_cached_min_ = true;
+    } else {
+      valid_cached_min_ = false;
+    }
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::BinarySearch(
+    const ElementType& element, ElementType* start, ElementType* end) const {
+  if (start == end) {
+    return NULL;
+  }
+  VIXL_ASSERT(sorted_);
+  VIXL_ASSERT(start < end);
+  VIXL_ASSERT(!empty());
+
+  // Perform a binary search through the elements while ignoring invalid
+  // elements.
+  ElementType* elements = start;
+  size_t low = 0;
+  size_t high = (end - start) - 1;
+  while (low < high) {
+    // Find valid bounds.
+    while (!IsValid(elements[low]) && (low < high)) ++low;
+    while (!IsValid(elements[high]) && (low < high)) --high;
+    VIXL_ASSERT(low <= high);
+    // Avoid overflow when computing the middle index.
+    size_t middle = low / 2 + high / 2 + (low & high & 1);
+    if ((middle == low) || (middle == high)) {
+      break;
+    }
+    while (!IsValid(elements[middle]) && (middle < high - 1)) ++middle;
+    while (!IsValid(elements[middle]) && (low + 1 < middle)) --middle;
+    if (!IsValid(elements[middle])) {
+      break;
+    }
+    if (elements[middle] < element) {
+      low = middle;
+    } else {
+      high = middle;
+    }
+  }
+
+  if (elements[low] == element) return &elements[low];
+  if (elements[high] == element) return &elements[high];
+  return NULL;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::Sort(SortType sort_type) {
+  VIXL_ASSERT(monitor() == 0);
+  if (sort_type == kSoftSort) {
+    if (sorted_) {
+      return;
+    }
+  }
+  if (empty()) {
+    return;
+  }
+
+  Clean();
+  std::sort(StorageBegin(), StorageEnd());
+
+  set_sorted(true);
+  cached_min_index_ = 0;
+  cached_min_key_ = Key(Front());
+  valid_cached_min_ = true;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::Clean() {
+  VIXL_ASSERT(monitor() == 0);
+  if (empty() || !IsUsingVector()) {
+    return;
+  }
+  // Manually iterate through the vector storage to discard invalid elements.
+  ElementType* start = &(vector_->front());
+  ElementType* end = start + vector_->size();
+  ElementType* c = start;
+  ElementType* first_invalid;
+  ElementType* first_valid;
+  ElementType* next_invalid;
+
+  while (c < end && IsValid(*c)) { c++; }
+  first_invalid = c;
+
+  while (c < end) {
+    while (c < end && !IsValid(*c)) { c++; }
+    first_valid = c;
+    while (c < end && IsValid(*c)) { c++; }
+    next_invalid = c;
+
+    ptrdiff_t n_moved_elements = (next_invalid - first_valid);
+    memmove(first_invalid, first_valid,  n_moved_elements * sizeof(*c));
+    first_invalid = first_invalid + n_moved_elements;
+    c = next_invalid;
+  }
+
+  // Delete the trailing invalid elements.
+  vector_->erase(vector_->begin() + (first_invalid - start), vector_->end());
+  VIXL_ASSERT(vector_->size() == size_);
+
+  if (sorted_) {
+    valid_cached_min_ = true;
+    cached_min_index_ = 0;
+    cached_min_key_ = Key(*ElementAt(0));
+  } else {
+    valid_cached_min_ = false;
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::Front() const {
+  VIXL_ASSERT(!empty());
+  return IsUsingVector() ? vector_->front() : preallocated_[0];
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::Back() const {
+  VIXL_ASSERT(!empty());
+  return IsUsingVector() ? vector_->back() : preallocated_[size_ - 1];
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::CleanBack() {
+  VIXL_ASSERT(monitor() == 0);
+  if (IsUsingVector()) {
+    // Delete the invalid trailing elements.
+    typename std::vector<ElementType>::reverse_iterator it = vector_->rbegin();
+    while (!IsValid(*it)) {
+      it++;
+    }
+    vector_->erase(it.base(), vector_->end());
+  }
+  return Back();
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageBegin() const {
+  return IsUsingVector() ? &(vector_->front()) : preallocated_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageEnd() const {
+  return IsUsingVector() ? &(vector_->back()) + 1 : preallocated_ + size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageBegin() {
+  return IsUsingVector() ? &(vector_->front()) : preallocated_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageEnd() {
+  return IsUsingVector() ? &(vector_->back()) + 1 : preallocated_ + size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+size_t InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementIndex(
+    const ElementType* element) const {
+  VIXL_ASSERT((StorageBegin() <= element) && (element < StorageEnd()));
+  return element - StorageBegin();
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementAt(
+    size_t index) const {
+  VIXL_ASSERT(
+      (IsUsingVector() && (index < vector_->size())) || (index < size_));
+  return StorageBegin() + index;
+}
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementAt(size_t index) {
+  VIXL_ASSERT(
+      (IsUsingVector() && (index < vector_->size())) || (index < size_));
+  return StorageBegin() + index;
+}
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::FirstValidElement(
+    const ElementType* from, const ElementType* end) {
+  while ((from < end) && !IsValid(*from)) {
+    from++;
+  }
+  return from;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::CacheMinElement() {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(!empty());
+
+  if (valid_cached_min_) {
+    return;
+  }
+
+  if (sorted_) {
+    const ElementType* min = FirstValidElement(StorageBegin(), StorageEnd());
+    cached_min_index_ = ElementIndex(min);
+    cached_min_key_ = Key(*min);
+    valid_cached_min_ = true;
+  } else {
+    Sort(kHardSort);
+  }
+  VIXL_ASSERT(valid_cached_min_);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::ShouldReclaimMemory() const {
+  if (!IsUsingVector()) {
+    return false;
+  }
+  size_t n_invalid_elements = vector_->size() - size_;
+  return (n_invalid_elements > RECLAIM_FROM) &&
+         (n_invalid_elements > vector_->size() / RECLAIM_FACTOR);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::ReclaimMemory() {
+  VIXL_ASSERT(monitor() == 0);
+  Clean();
+}
+
+
+template<class S>
+InvalSetIterator<S>::InvalSetIterator(S* inval_set)
+    : using_vector_((inval_set != NULL) && inval_set->IsUsingVector()),
+      index_(0),
+      inval_set_(inval_set) {
+  if (inval_set != NULL) {
+    inval_set->Sort(S::kSoftSort);
+#ifdef VIXL_DEBUG
+    inval_set->Acquire();
+#endif
+    if (using_vector_) {
+      iterator_ = typename std::vector<ElementType>::iterator(
+          inval_set_->vector_->begin());
+    }
+    MoveToValidElement();
+  }
+}
+
+
+template<class S>
+InvalSetIterator<S>::~InvalSetIterator() {
+#ifdef VIXL_DEBUG
+  if (inval_set_ != NULL) {
+    inval_set_->Release();
+  }
+#endif
+}
+
+
+template<class S>
+typename S::_ElementType* InvalSetIterator<S>::Current() const {
+  VIXL_ASSERT(!Done());
+  if (using_vector_) {
+    return &(*iterator_);
+  } else {
+    return &(inval_set_->preallocated_[index_]);
+  }
+}
+
+
+template<class S>
+void InvalSetIterator<S>::Advance() {
+  VIXL_ASSERT(!Done());
+  if (using_vector_) {
+    iterator_++;
+#ifdef VIXL_DEBUG
+    index_++;
+#endif
+    MoveToValidElement();
+  } else {
+    index_++;
+  }
+}
+
+
+template<class S>
+bool InvalSetIterator<S>::Done() const {
+  if (using_vector_) {
+    bool done = (iterator_ == inval_set_->vector_->end());
+    VIXL_ASSERT(done == (index_ == inval_set_->size()));
+    return done;
+  } else {
+    return index_ == inval_set_->size();
+  }
+}
+
+
+template<class S>
+void InvalSetIterator<S>::Finish() {
+  VIXL_ASSERT(inval_set_->sorted_);
+  if (using_vector_) {
+    iterator_ = inval_set_->vector_->end();
+  }
+  index_ = inval_set_->size();
+}
+
+
+template<class S>
+void InvalSetIterator<S>::DeleteCurrentAndAdvance() {
+  if (using_vector_) {
+    inval_set_->EraseInternal(&(*iterator_));
+    MoveToValidElement();
+  } else {
+    inval_set_->EraseInternal(inval_set_->preallocated_ + index_);
+  }
+}
+
+
+template<class S>
+bool InvalSetIterator<S>::IsValid(const ElementType& element) {
+  return S::IsValid(element);
+}
+
+
+template<class S>
+typename S::_KeyType InvalSetIterator<S>::Key(const ElementType& element) {
+  return S::Key(element);
+}
+
+
+template<class S>
+void InvalSetIterator<S>::MoveToValidElement() {
+  if (using_vector_) {
+    while ((iterator_ != inval_set_->vector_->end()) && !IsValid(*iterator_)) {
+      iterator_++;
+    }
+  } else {
+    VIXL_ASSERT(inval_set_->empty() || IsValid(inval_set_->preallocated_[0]));
+    // Nothing to do.
+  }
+}
+
+#undef TEMPLATE_INVALSET_P_DECL
+#undef TEMPLATE_INVALSET_P_DEF
+
+}  // namespace vixl
+
+#endif  // VIXL_INVALSET_H_
diff --git a/qemu/disas/libvixl/platform.h b/qemu/disas/libvixl/vixl/platform.h
index de2b110cc..ab588f07f 100644
--- a/qemu/disas/libvixl/platform.h
+++ b/qemu/disas/libvixl/vixl/platform.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2014, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
diff --git a/qemu/disas/libvixl/vixl/utils.cc b/qemu/disas/libvixl/vixl/utils.cc
new file mode 100644
index 000000000..3b8bd75fb
--- /dev/null
+++ b/qemu/disas/libvixl/vixl/utils.cc
@@ -0,0 +1,142 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "vixl/utils.h"
+#include <stdio.h>
+
+namespace vixl {
+
+uint32_t float_to_rawbits(float value) {
+  uint32_t bits = 0;
+  memcpy(&bits, &value, 4);
+  return bits;
+}
+
+
+uint64_t double_to_rawbits(double value) {
+  uint64_t bits = 0;
+  memcpy(&bits, &value, 8);
+  return bits;
+}
+
+
+float rawbits_to_float(uint32_t bits) {
+  float value = 0.0;
+  memcpy(&value, &bits, 4);
+  return value;
+}
+
+
+double rawbits_to_double(uint64_t bits) {
+  double value = 0.0;
+  memcpy(&value, &bits, 8);
+  return value;
+}
+
+
+uint32_t float_sign(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(31, 31, rawbits);
+}
+
+
+uint32_t float_exp(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(30, 23, rawbits);
+}
+
+
+uint32_t float_mantissa(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(22, 0, rawbits);
+}
+
+
+uint32_t double_sign(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return static_cast<uint32_t>(unsigned_bitextract_64(63, 63, rawbits));
+}
+
+
+uint32_t double_exp(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return static_cast<uint32_t>(unsigned_bitextract_64(62, 52, rawbits));
+}
+
+
+uint64_t double_mantissa(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return unsigned_bitextract_64(51, 0, rawbits);
+}
+
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa) {
+  uint32_t bits = (sign << 31) | (exp << 23) | mantissa;
+  return rawbits_to_float(bits);
+}
+
+
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa) {
+  uint64_t bits = (sign << 63) | (exp << 52) | mantissa;
+  return rawbits_to_double(bits);
+}
+
+
+int float16classify(float16 value) {
+  uint16_t exponent_max = (1 << 5) - 1;
+  uint16_t exponent_mask = exponent_max << 10;
+  uint16_t mantissa_mask = (1 << 10) - 1;
+
+  uint16_t exponent = (value & exponent_mask) >> 10;
+  uint16_t mantissa = value & mantissa_mask;
+  if (exponent == 0) {
+    if (mantissa == 0) {
+      return FP_ZERO;
+    }
+    return FP_SUBNORMAL;
+  } else if (exponent == exponent_max) {
+    if (mantissa == 0) {
+      return FP_INFINITE;
+    }
+    return FP_NAN;
+  }
+  return FP_NORMAL;
+}
+
+
+unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size) {
+  VIXL_ASSERT((reg_size % 8) == 0);
+  int count = 0;
+  for (unsigned i = 0; i < (reg_size / 16); i++) {
+    if ((imm & 0xffff) == 0) {
+      count++;
+    }
+    imm >>= 16;
+  }
+  return count;
+}
+
+}  // namespace vixl
diff --git a/qemu/disas/libvixl/utils.h b/qemu/disas/libvixl/vixl/utils.h
index b4406263a..5ab134e24 100644
--- a/qemu/disas/libvixl/utils.h
+++ b/qemu/disas/libvixl/vixl/utils.h
@@ -1,4 +1,4 @@
-// Copyright 2013, ARM Limited
+// Copyright 2015, ARM Limited
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -27,16 +27,17 @@
 #ifndef VIXL_UTILS_H
 #define VIXL_UTILS_H
 
-#include <math.h>
 #include <string.h>
-#include "globals.h"
+#include <cmath>
+#include "vixl/globals.h"
+#include "vixl/compiler-intrinsics.h"
 
 namespace vixl {
 
 // Macros for compile-time format checking.
-#if defined(__GNUC__)
+#if GCC_VERSION_OR_NEWER(4, 4, 0)
 #define PRINTF_CHECK(format_index, varargs_index) \
-  __attribute__((format(printf, format_index, varargs_index)))
+  __attribute__((format(gnu_printf, format_index, varargs_index)))
 #else
 #define PRINTF_CHECK(format_index, varargs_index)
 #endif
@@ -53,9 +54,9 @@ inline bool is_uintn(unsigned n, int64_t x) {
   return !(x >> n);
 }
 
-inline unsigned truncate_to_intn(unsigned n, int64_t x) {
+inline uint32_t truncate_to_intn(unsigned n, int64_t x) {
   VIXL_ASSERT((0 < n) && (n < 64));
-  return (x & ((INT64_C(1) << n) - 1));
+  return static_cast<uint32_t>(x & ((INT64_C(1) << n) - 1));
 }
 
 #define INT_1_TO_63_LIST(V)                                                    \
@@ -73,7 +74,7 @@ inline bool is_int##N(int64_t x) { return is_intn(N, x); }
 #define DECLARE_IS_UINT_N(N)                                                   \
 inline bool is_uint##N(int64_t x) { return is_uintn(N, x); }
 #define DECLARE_TRUNCATE_TO_INT_N(N)                                           \
-inline int truncate_to_int##N(int x) { return truncate_to_intn(N, x); }
+inline uint32_t truncate_to_int##N(int x) { return truncate_to_intn(N, x); }
 INT_1_TO_63_LIST(DECLARE_IS_INT_N)
 INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
 INT_1_TO_63_LIST(DECLARE_TRUNCATE_TO_INT_N)
@@ -104,12 +105,24 @@ uint64_t double_to_rawbits(double value);
 float rawbits_to_float(uint32_t bits);
 double rawbits_to_double(uint64_t bits);
 
+uint32_t float_sign(float val);
+uint32_t float_exp(float val);
+uint32_t float_mantissa(float val);
+uint32_t double_sign(double val);
+uint32_t double_exp(double val);
+uint64_t double_mantissa(double val);
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa);
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa);
+
+// An fpclassify() function for 16-bit half-precision floats.
+int float16classify(float16 value);
 
 // NaN tests.
 inline bool IsSignallingNaN(double num) {
   const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
   uint64_t raw = double_to_rawbits(num);
-  if (isnan(num) && ((raw & kFP64QuietNaNMask) == 0)) {
+  if (std::isnan(num) && ((raw & kFP64QuietNaNMask) == 0)) {
     return true;
   }
   return false;
@@ -119,30 +132,37 @@ inline bool IsSignallingNaN(double num) {
 inline bool IsSignallingNaN(float num) {
   const uint32_t kFP32QuietNaNMask = 0x00400000;
   uint32_t raw = float_to_rawbits(num);
-  if (isnan(num) && ((raw & kFP32QuietNaNMask) == 0)) {
+  if (std::isnan(num) && ((raw & kFP32QuietNaNMask) == 0)) {
     return true;
   }
   return false;
 }
 
 
+inline bool IsSignallingNaN(float16 num) {
+  const uint16_t kFP16QuietNaNMask = 0x0200;
+  return (float16classify(num) == FP_NAN) &&
+         ((num & kFP16QuietNaNMask) == 0);
+}
+
+
 template <typename T>
 inline bool IsQuietNaN(T num) {
-  return isnan(num) && !IsSignallingNaN(num);
+  return std::isnan(num) && !IsSignallingNaN(num);
 }
 
 
 // Convert the NaN in 'num' to a quiet NaN.
 inline double ToQuietNaN(double num) {
   const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
-  VIXL_ASSERT(isnan(num));
+  VIXL_ASSERT(std::isnan(num));
   return rawbits_to_double(double_to_rawbits(num) | kFP64QuietNaNMask);
 }
 
 
 inline float ToQuietNaN(float num) {
   const uint32_t kFP32QuietNaNMask = 0x00400000;
-  VIXL_ASSERT(isnan(num));
+  VIXL_ASSERT(std::isnan(num));
   return rawbits_to_float(float_to_rawbits(num) | kFP32QuietNaNMask);
 }
 
@@ -158,16 +178,71 @@ inline float FusedMultiplyAdd(float op1, float op2, float a) {
 }
 
 
-// Bit counting.
-int CountLeadingZeros(uint64_t value, int width);
-int CountLeadingSignBits(int64_t value, int width);
-int CountTrailingZeros(uint64_t value, int width);
-int CountSetBits(uint64_t value, int width);
-uint64_t LowestSetBit(uint64_t value);
-bool IsPowerOf2(int64_t value);
+inline uint64_t LowestSetBit(uint64_t value) {
+  return value & -value;
+}
+
+
+template<typename T>
+inline int HighestSetBitPosition(T value) {
+  VIXL_ASSERT(value != 0);
+  return (sizeof(value) * 8 - 1) - CountLeadingZeros(value);
+}
+
+
+template<typename V>
+inline int WhichPowerOf2(V value) {
+  VIXL_ASSERT(IsPowerOf2(value));
+  return CountTrailingZeros(value);
+}
+
 
 unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size);
 
+
+template <typename T>
+T ReverseBits(T value) {
+  VIXL_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+              (sizeof(value) == 4) || (sizeof(value) == 8));
+  T result = 0;
+  for (unsigned i = 0; i < (sizeof(value) * 8); i++) {
+    result = (result << 1) | (value & 1);
+    value >>= 1;
+  }
+  return result;
+}
+
+
+template <typename T>
+T ReverseBytes(T value, int block_bytes_log2) {
+  VIXL_ASSERT((sizeof(value) == 4) || (sizeof(value) == 8));
+  VIXL_ASSERT((1U << block_bytes_log2) <= sizeof(value));
+  // Split the 64-bit value into an 8-bit array, where b[0] is the least
+  // significant byte, and b[7] is the most significant.
+  uint8_t bytes[8];
+  uint64_t mask = UINT64_C(0xff00000000000000);
+  for (int i = 7; i >= 0; i--) {
+    bytes[i] = (static_cast<uint64_t>(value) & mask) >> (i * 8);
+    mask >>= 8;
+  }
+
+  // Permutation tables for REV instructions.
+  //  permute_table[0] is used by REV16_x, REV16_w
+  //  permute_table[1] is used by REV32_x, REV_w
+  //  permute_table[2] is used by REV_x
+  VIXL_ASSERT((0 < block_bytes_log2) && (block_bytes_log2 < 4));
+  static const uint8_t permute_table[3][8] = { {6, 7, 4, 5, 2, 3, 0, 1},
+                                               {4, 5, 6, 7, 0, 1, 2, 3},
+                                               {0, 1, 2, 3, 4, 5, 6, 7} };
+  T result = 0;
+  for (int i = 0; i < 8; i++) {
+    result <<= 8;
+    result |= bytes[permute_table[block_bytes_log2 - 1][i]];
+  }
+  return result;
+}
+
+
 // Pointer alignment
 // TODO: rename/refactor to make it specific to instructions.
 template<typename T>
diff --git a/qemu/disas/lm32.c b/qemu/disas/lm32.c
index 1718c86e1..fcc2cde23 100644
--- a/qemu/disas/lm32.c
+++ b/qemu/disas/lm32.c
@@ -18,7 +18,7 @@
  *
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 typedef enum {
diff --git a/qemu/disas/m68k.c b/qemu/disas/m68k.c
index cc0db96ca..8f74ae115 100644
--- a/qemu/disas/m68k.c
+++ b/qemu/disas/m68k.c
@@ -1,9 +1,8 @@
 /* This file is composed of several different files from the upstream
    sourceware.org CVS.  Original file boundaries marked with **** */
 
-#include <string.h>
+#include "qemu/osdep.h"
 #include <math.h>
-#include <stdio.h>
 
 #include "disas/bfd.h"
 
@@ -616,8 +615,6 @@ static const char *const reg_half_names[] =
 /* Maximum length of an instruction.  */
 #define MAXLEN 22
 
-#include <setjmp.h>
-
 struct private
 {
   /* Points to first byte not fetched.  */
diff --git a/qemu/disas/microblaze.c b/qemu/disas/microblaze.c
index c14ab89b7..91b30acbe 100644
--- a/qemu/disas/microblaze.c
+++ b/qemu/disas/microblaze.c
@@ -33,13 +33,10 @@ along with this program; if not, see <http://www.gnu.org/licenses/>. */
  */
 
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #define STATIC_TABLE
 #define DEFINE_TABLE
 
-#define TRUE   1
-#define FALSE  0
-
 #ifndef MICROBLAZE_OPC
 #define MICROBLAZE_OPC
 /* Assembler instructions for Xilinx's microblaze processor
@@ -582,7 +579,6 @@ static const char pvr_register_prefix[] = "rpvr";
 #endif /* MICROBLAZE_OPC */
 
 #include "disas/bfd.h"
-#include <strings.h>
 
 #define get_field_rd(instr) get_field(instr, RD_MASK, RD_LOW)
 #define get_field_r1(instr) get_field(instr, RA_MASK, RA_LOW)
@@ -604,25 +600,6 @@ char * get_field_special (long instr, struct op_code_struct * op);
 unsigned long read_insn_microblaze (bfd_vma memaddr, 
 		      struct disassemble_info *info,
 		      struct op_code_struct **opr);
-enum microblaze_instr get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots);
-short get_delay_slots_microblaze (long inst);
-enum microblaze_instr microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm);
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch);
 
 static char *
 get_field (long instr, long mask, unsigned short low)
@@ -968,134 +945,3 @@ print_insn_microblaze (bfd_vma memaddr, struct disassemble_info * info)
   /* Say how many bytes we consumed? */
   return 4;
 }
-
-enum microblaze_instr
-get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots)
-{
-  struct op_code_struct * op;
-  *isunsignedimm = FALSE;
-
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0)
-    return invalid_inst;
-  else {
-    *isunsignedimm = (op->inst_type == INST_TYPE_RD_R1_UNSIGNED_IMM);
-    *insn_type = op->instr_type;
-    *delay_slots = op->delay_slots;
-    return op->instr;
-  }
-}
-
-short
-get_delay_slots_microblaze (long inst)
-{
-  bfd_boolean isunsignedimm;
-  enum microblaze_instr_type insn_type;
-  enum microblaze_instr op;
-  short delay_slots;
-
-  op = get_insn_microblaze( inst, &isunsignedimm, &insn_type, &delay_slots);
-  if (op == invalid_inst)
-    return 0;
-  else 
-    return delay_slots;
-}
-
-enum microblaze_instr
-microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm)
-{
-  enum microblaze_instr op;
-  bfd_boolean t1;
-  enum microblaze_instr_type t2;
-  short t3;
-
-  op = get_insn_microblaze(insn, &t1, &t2, &t3);
-  *rd = (insn & RD_MASK) >> RD_LOW;
-  *ra = (insn & RA_MASK) >> RA_LOW;
-  *rb = (insn & RB_MASK) >> RB_LOW;
-  t3 = (insn & IMM_MASK) >> IMM_LOW;
-  *imm = (int) t3;
-  return (op);
-}
-
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch)
-{
-  struct op_code_struct * op;
-  long targetaddr = 0;
-
-  *unconditionalbranch = FALSE;
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0) {
-    *targetvalid = FALSE;
-  } else if (op->instr_type == branch_inst) {
-    switch (op->inst_type) {
-    case INST_TYPE_R2:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_R2:
-    case INST_TYPE_R1_R2:
-      targetaddr = r2val;
-      *targetvalid = TRUE;
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      break;
-    case INST_TYPE_IMM:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_IMM:
-    case INST_TYPE_R1_IMM:
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      *targetvalid = TRUE;
-      break;
-    default:
-      *targetvalid = FALSE;
-      break;
-    }
-  } else if (op->instr_type == return_inst) {
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      targetaddr += r1val;
-      *targetvalid = TRUE;
-  } else {
-    *targetvalid = FALSE;
-  }
-  return targetaddr;
-}
diff --git a/qemu/disas/mips.c b/qemu/disas/mips.c
index 01336a838..249931b73 100644
--- a/qemu/disas/mips.c
+++ b/qemu/disas/mips.c
@@ -19,6 +19,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* mips.h.  Mips opcode list for GDB, the GNU debugger.
@@ -1404,6 +1405,10 @@ const struct mips_opcode mips_builtin_opcodes[] =
 {"cmp.sor.d",  "D,S,T", 0x46a00019, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
 {"cmp.sune.d", "D,S,T", 0x46a0001a, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
 {"cmp.sne.d",  "D,S,T", 0x46a0001b, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"dvp",        "",      0x41600024, 0xffffffff, TRAP,                 0, I32R6},
+{"dvp",        "t",     0x41600024, 0xffe0ffff, TRAP|WR_t,            0, I32R6},
+{"evp",        "",      0x41600004, 0xffffffff, TRAP,                 0, I32R6},
+{"evp",        "t",     0x41600004, 0xffe0ffff, TRAP|WR_t,            0, I32R6},
 
 /* MSA */
 {"sll.b",   "+d,+e,+f", 0x7800000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
@@ -2420,9 +2425,11 @@ const struct mips_opcode mips_builtin_opcodes[] =
 {"hibernate","",        0x42000023, 0xffffffff,	0, 			0,		V1	},
 {"ins",     "t,r,+A,+B", 0x7c000004, 0xfc00003f, WR_t|RD_s,    		0,		I33	},
 {"jr",      "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	},
+{"jr",      "s",	0x00000009, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr */
 /* jr.hb is officially MIPS{32,64}R2, but it works on R1 as jr with
    the same hazard barrier effect.  */
 {"jr.hb",   "s",	0x00000408, 0xfc1fffff,	UBD|RD_s,		0,		I32	},
+{"jr.hb",   "s",	0x00000409, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr.hb */
 {"j",       "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	}, /* jr */
 /* SVR4 PIC code requires special handling for j, so it must be a
    macro.  */
diff --git a/qemu/disas/moxie.c b/qemu/disas/moxie.c
index 4c5f18047..70b49ed74 100644
--- a/qemu/disas/moxie.c
+++ b/qemu/disas/moxie.c
@@ -14,7 +14,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #define STATIC_TABLE
 #define DEFINE_TABLE
 
diff --git a/qemu/disas/ppc.c b/qemu/disas/ppc.c
index 99c4cbc3a..478332ba3 100644
--- a/qemu/disas/ppc.c
+++ b/qemu/disas/ppc.c
@@ -18,6 +18,7 @@ the GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with this file; see the file COPYING.  If not,
 see <http://www.gnu.org/licenses/>.  */
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 #define BFD_DEFAULT_TARGET_SIZE 64
 
diff --git a/qemu/disas/s390.c b/qemu/disas/s390.c
index 974460c81..1f167d2ea 100644
--- a/qemu/disas/s390.c
+++ b/qemu/disas/s390.c
@@ -20,6 +20,7 @@
    Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
    02110-1301, USA.  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 
@@ -613,7 +614,7 @@ static const struct s390_operand s390_operands[] =
       names of the instruction format that you can find in the principals
       of operation.
    2) the last part of the definition (y in INSTR_x_y) gives you an idea
-      which operands the binary represenation of the instruction has.
+      which operands the binary representation of the instruction has.
       The meanings of the letters in y are:
       a - access register
       c - control register
@@ -627,7 +628,7 @@ static const struct s390_operand s390_operands[] =
       m - mode field, 4 bit
       0 - operand skipped.
       The order of the letters reflects the layout of the format in
-      storage and not the order of the paramaters of the instructions.
+      storage and not the order of the parameters of the instructions.
       The use of the letters is not a 100% match with the PoP but it is
       quite close.
 
diff --git a/qemu/disas/sh4.c b/qemu/disas/sh4.c
index 020f5ebb7..8b0415dfe 100644
--- a/qemu/disas/sh4.c
+++ b/qemu/disas/sh4.c
@@ -15,7 +15,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 #define DEFINE_TABLE
diff --git a/qemu/disas/sparc.c b/qemu/disas/sparc.c
index f4e356573..64bba8df2 100644
--- a/qemu/disas/sparc.c
+++ b/qemu/disas/sparc.c
@@ -26,7 +26,7 @@
    along with GAS or GDB; see the file COPYING. If not,
    see <http://www.gnu.org/licenses/>.  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* The SPARC opcode table (and other related data) is defined in
@@ -2622,8 +2622,7 @@ build_hash_table (const sparc_opcode **opcode_table,
 
   memset (hash_table, 0, HASH_SIZE * sizeof (hash_table[0]));
   memset (hash_count, 0, HASH_SIZE * sizeof (hash_count[0]));
-  if (hash_buf != NULL)
-    free (hash_buf);
+  free(hash_buf);
   hash_buf = malloc (sizeof (* hash_buf) * num_opcodes);
   for (i = num_opcodes - 1; i >= 0; --i)
     {
diff --git a/qemu/disas/tci.c b/qemu/disas/tci.c
index d7b954e62..1cdf5eeaf 100644
--- a/qemu/disas/tci.c
+++ b/qemu/disas/tci.c
@@ -17,6 +17,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 #include "tcg/tcg.h"
diff --git a/qemu/dma-helpers.c b/qemu/dma-helpers.c
index 4faec5d0c..4ad0bca67 100644
--- a/qemu/dma-helpers.c
+++ b/qemu/dma-helpers.c
@@ -7,6 +7,7 @@
  * (GNU GPL), version 2 or later.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 #include "trace.h"
diff --git a/qemu/docs/bitmaps.md b/qemu/docs/bitmaps.md
index fa87f077f..a2e8d5116 100644
--- a/qemu/docs/bitmaps.md
+++ b/qemu/docs/bitmaps.md
@@ -19,12 +19,20 @@ which is included at the end of this document.
 * A dirty bitmap's name is unique to the node, but bitmaps attached to different
   nodes can share the same name.
 
+* Dirty bitmaps created for internal use by QEMU may be anonymous and have no
+  name, but any user-created bitmaps may not be. There can be any number of
+  anonymous bitmaps per node.
+
+* The name of a user-created bitmap must not be empty ("").
+
 ## Bitmap Modes
 
 * A Bitmap can be "frozen," which means that it is currently in-use by a backup
   operation and cannot be deleted, renamed, written to, reset,
   etc.
 
+* The normal operating mode for a bitmap is "active."
+
 ## Basic QMP Usage
 
 ### Supported Commands ###
@@ -97,11 +105,7 @@ which is included at the end of this document.
 }
 ```
 
-## Transactions (Not yet implemented)
-
-* Transactional commands are forthcoming in a future version,
-  and are not yet available for use. This section serves as
-  documentation of intent for their design and usage.
+## Transactions
 
 ### Justification
 
@@ -323,6 +327,155 @@ full backup as a backing image.
       "event": "BLOCK_JOB_COMPLETED" }
     ```
 
+### Partial Transactional Failures
+
+* Sometimes, a transaction will succeed in launching and return success,
+  but then later the backup jobs themselves may fail. It is possible that
+  a management application may have to deal with a partial backup failure
+  after a successful transaction.
+
+* If multiple backup jobs are specified in a single transaction, when one of
+  them fails, it will not interact with the other backup jobs in any way.
+
+* The job(s) that succeeded will clear the dirty bitmap associated with the
+  operation, but the job(s) that failed will not. It is not "safe" to delete
+  any incremental backups that were created successfully in this scenario,
+  even though others failed.
+
+#### Example
+
+* QMP example highlighting two backup jobs:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ]
+      }
+    }
+    ```
+
+* QMP example response, highlighting one success and one failure:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the first job was completed:
+        ```json
+        { "timestamp": { "seconds": 1447192343, "microseconds": 615698 },
+          "data": { "device": "drive0", "type": "backup",
+                     "speed": 0, "len": 67108864, "offset": 67108864 },
+          "event": "BLOCK_JOB_COMPLETED"
+        }
+        ```
+
+    * Later yet, QEMU sends notice that the second job has failed:
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 683015 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 685853 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+
+* In the above example, "d0-incr-1.qcow2" is valid and must be kept,
+  but "d1-incr-1.qcow2" is invalid and should be deleted. If a VM-wide
+  incremental backup of all drives at a point-in-time is to be made,
+  new backups for both drives will need to be made, taking into account
+  that a new incremental backup for drive0 needs to be based on top of
+  "d0-incr-1.qcow2."
+
+### Grouped Completion Mode
+
+* While jobs launched by transactions normally complete or fail on their own,
+  it is possible to instruct them to complete or fail together as a group.
+
+* QMP transactions take an optional properties structure that can affect
+  the semantics of the transaction.
+
+* The "completion-mode" transaction property can be either "individual"
+  which is the default, legacy behavior described above, or "grouped,"
+  a new behavior detailed below.
+
+* Delayed Completion: In grouped completion mode, no jobs will report
+  success until all jobs are ready to report success.
+
+* Grouped failure: If any job fails in grouped completion mode, all remaining
+  jobs will be cancelled. Any incremental backups will restore their dirty
+  bitmap objects as if no backup command was ever issued.
+
+    * Regardless of if QEMU reports a particular incremental backup job as
+      CANCELLED or as an ERROR, the in-memory bitmap will be restored.
+
+#### Example
+
+* Here's the same example scenario from above with the new property:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ],
+        "properties": {
+          "completion-mode": "grouped"
+        }
+      }
+    }
+    ```
+
+* QMP example response, highlighting a failure for drive2:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the second job has errored out,
+      but that the first job was also cancelled:
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 632377 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640074 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640163 },
+          "data": { "device": "drive0", "type": "backup", "speed": 0,
+                    "len": 67108864, "offset": 16777216 },
+          "event": "BLOCK_JOB_CANCELLED" }
+        ```
+
 <!--
 The FreeBSD Documentation License
 
diff --git a/qemu/docs/blkdebug.txt b/qemu/docs/blkdebug.txt
index b67a36d5c..43d8e8f9c 100644
--- a/qemu/docs/blkdebug.txt
+++ b/qemu/docs/blkdebug.txt
@@ -1,6 +1,6 @@
 Block I/O error injection using blkdebug
 ----------------------------------------
-Copyright (C) 2014 Red Hat Inc
+Copyright (C) 2014-2015 Red Hat Inc
 
 This work is licensed under the terms of the GNU GPL, version 2 or later.  See
 the COPYING file in the top-level directory.
@@ -92,8 +92,9 @@ The core events are:
 
   flush_to_disk - flush the host block device's disk cache
 
-See block/blkdebug.c:event_names[] for the full list of events.  You may need
-to grep block driver source code to understand the meaning of specific events.
+See qapi/block-core.json:BlkdebugEvent for the full list of events.
+You may need to grep block driver source code to understand the
+meaning of specific events.
 
 State transitions
 -----------------
diff --git a/qemu/docs/build-system.txt b/qemu/docs/build-system.txt
new file mode 100644
index 000000000..5ddddeaaf
--- /dev/null
+++ b/qemu/docs/build-system.txt
@@ -0,0 +1,507 @@
+    The QEMU build system architecture
+    ==================================
+
+This document aims to help developers understand the architecture of the
+QEMU build system. As with projects using GNU autotools, the QEMU build
+system has two stages, first the developer runs the "configure" script
+to determine the local build environment characteristics, then they run
+"make" to build the project. There is about where the similarities with
+GNU autotools end, so try to forget what you know about them.
+
+
+Stage 1: configure
+==================
+
+The QEMU configure script is written directly in shell, and should be
+compatible with any POSIX shell, hence it uses #!/bin/sh. An important
+implication of this is that it is important to avoid using bash-isms on
+development platforms where bash is the primary host.
+
+In contrast to autoconf scripts, QEMU's configure is expected to be
+silent while it is checking for features. It will only display output
+when an error occurs, or to show the final feature enablement summary
+on completion.
+
+Adding new checks to the configure script usually comprises the
+following tasks:
+
+ - Initialize one or more variables with the default feature state.
+
+   Ideally features should auto-detect whether they are present,
+   so try to avoid hardcoding the initial state to either enabled
+   or disabled, as that forces the user to pass a --enable-XXX
+   / --disable-XXX flag on every invocation of configure.
+
+ - Add support to the command line arg parser to handle any new
+   --enable-XXX / --disable-XXX flags required by the feature XXX.
+
+ - Add information to the help output message to report on the new
+   feature flag.
+
+ - Add code to perform the actual feature check. As noted above, try to
+   be fully dynamic in checking enablement/disablement.
+
+ - Add code to print out the feature status in the configure summary
+   upon completion.
+
+ - Add any new makefile variables to $config_host_mak on completion.
+
+
+Taking (a simplified version of) the probe for gnutls from configure,
+we have the following pieces:
+
+  # Initial variable state
+  gnutls=""
+
+  ..snip..
+
+  # Configure flag processing
+  --disable-gnutls) gnutls="no"
+  ;;
+  --enable-gnutls) gnutls="yes"
+  ;;
+
+  ..snip..
+
+  # Help output feature message
+  gnutls          GNUTLS cryptography support
+
+  ..snip..
+
+  # Test for gnutls
+  if test "$gnutls" != "no"; then
+     if ! $pkg_config --exists "gnutls"; then
+        gnutls_cflags=`$pkg_config --cflags gnutls`
+        gnutls_libs=`$pkg_config --libs gnutls`
+        libs_softmmu="$gnutls_libs $libs_softmmu"
+        libs_tools="$gnutls_libs $libs_tools"
+        QEMU_CFLAGS="$QEMU_CFLAGS $gnutls_cflags"
+        gnutls="yes"
+     elif test "$gnutls" = "yes"; then
+        feature_not_found "gnutls" "Install gnutls devel"
+     else
+        gnutls="no"
+     fi
+  fi
+
+  ..snip..
+
+  # Completion feature summary
+  echo "GNUTLS support    $gnutls"
+
+  ..snip..
+
+  # Define make variables
+  if test "$gnutls" = "yes" ; then
+     echo "CONFIG_GNUTLS=y" >> $config_host_mak
+  fi
+
+
+Helper functions
+----------------
+
+The configure script provides a variety of helper functions to assist
+developers in checking for system features:
+
+ - do_cc $ARGS...
+
+   Attempt to run the system C compiler passing it $ARGS...
+
+ - do_cxx $ARGS...
+
+   Attempt to run the system C++ compiler passing it $ARGS...
+
+ - compile_object $CFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS. The test program must have been previously written to a file
+   called $TMPC.
+
+ - compile_prog $CFLAGS $LDFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS and link it with the system linker using $LDFLAGS. The test
+   program must have been previously written to a file called $TMPC.
+
+ - has $COMMAND
+
+   Determine if $COMMAND exists in the current environment, either as a
+   shell builtin, or executable binary, returning 0 on success.
+
+ - path_of $COMMAND
+
+   Return the fully qualified path of $COMMAND, printing it to stdout,
+   and returning 0 on success.
+
+ - check_define $NAME
+
+   Determine if the macro $NAME is defined by the system C compiler
+
+ - check_include $NAME
+
+   Determine if the include $NAME file is available to the system C
+   compiler
+
+ - write_c_skeleton
+
+   Write a minimal C program main() function to the temporary file
+   indicated by $TMPC
+
+ - feature_not_found $NAME $REMEDY
+
+   Print a message to stderr that the feature $NAME was not available
+   on the system, suggesting the user try $REMEDY to address the
+   problem.
+
+ - error_exit $MESSAGE $MORE...
+
+   Print $MESSAGE to stderr, followed by $MORE... and then exit from the
+   configure script with non-zero status
+
+ - query_pkg_config $ARGS...
+
+   Run pkg-config passing it $ARGS. If QEMU is doing a static build,
+   then --static will be automatically added to $ARGS
+
+
+Stage 2: makefiles
+==================
+
+The use of GNU make is required with the QEMU build system.
+
+Although the source code is spread across multiple subdirectories, the
+build system should be considered largely non-recursive in nature, in
+contrast to common practices seen with automake. There is some recursive
+invocation of make, but this is related to the things being built,
+rather than the source directory structure.
+
+QEMU currently supports both VPATH and non-VPATH builds, so there are
+three general ways to invoke configure & perform a build.
+
+ - VPATH, build artifacts outside of QEMU source tree entirely
+
+     cd ../
+     mkdir build
+     cd build
+     ../qemu/configure
+     make
+
+ - VPATH, build artifacts in a subdir of QEMU source tree
+
+     mkdir build
+     cd build
+     ../configure
+     make
+
+ - non-VPATH, build artifacts everywhere
+
+     ./configure
+     make
+
+The QEMU maintainers generally recommend that a VPATH build is used by
+developers. Patches to QEMU are expected to ensure VPATH build still
+works.
+
+
+Module structure
+----------------
+
+There are a number of key outputs of the QEMU build system:
+
+ - Tools - qemu-img, qemu-nbd, qga (guest agent), etc
+ - System emulators - qemu-system-$ARCH
+ - Userspace emulators - qemu-$ARCH
+ - Unit tests
+
+The source code is highly modularized, split across many files to
+facilitate building of all of these components with as little duplicated
+compilation as possible. There can be considered to be two distinct
+groups of files, those which are independent of the QEMU emulation
+target and those which are dependent on the QEMU emulation target.
+
+In the target-independent set lives various general purpose helper code,
+such as error handling infrastructure, standard data structures,
+platform portability wrapper functions, etc. This code can be compiled
+once only and the .o files linked into all output binaries.
+
+In the target-dependent set lives CPU emulation, device emulation and
+much glue code. This sometimes also has to be compiled multiple times,
+once for each target being built.
+
+The utility code that is used by all binaries is built into a
+static archive called libqemuutil.a, which is then linked to all the
+binaries. In order to provide hooks that are only needed by some of the
+binaries, code in libqemuutil.a may depend on other functions that are
+not fully implemented by all QEMU binaries. To deal with this there is a
+second library called libqemustub.a which provides dummy stubs for all
+these functions. These will get lazy linked into the binary if the real
+implementation is not present. In this way, the libqemustub.a static
+library can be thought of as a portable implementation of the weak
+symbols concept. All binaries should link to both libqemuutil.a and
+libqemustub.a. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip.. libqemuutil.a libqemustub.a
+
+
+Windows platform portability
+----------------------------
+
+On Windows, all binaries have the suffix '.exe', so all Makefile rules
+which create binaries must include the $(EXESUF) variable on the binary
+name. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip..
+
+This expands to '.exe' on Windows, or '' on other platforms.
+
+A further complication for the system emulator binaries is that
+two separate binaries need to be generated.
+
+The main binary (e.g. qemu-system-x86_64.exe) is linked against the
+Windows console runtime subsystem. These are expected to be run from a
+command prompt window, and so will print stderr to the console that
+launched them.
+
+The second binary generated has a 'w' on the end of its name (e.g.
+qemu-system-x86_64w.exe) and is linked against the Windows graphical
+runtime subsystem. These are expected to be run directly from the
+desktop and will open up a dedicated console window for stderr output.
+
+The Makefile.target will generate the binary for the graphical subsystem
+first, and then use objcopy to relink it against the console subsystem
+to generate the second binary.
+
+
+Object variable naming
+----------------------
+
+The QEMU convention is to define variables to list different groups of
+object files. These are named with the convention $PREFIX-obj-y. For
+example the libqemuutil.a file will be linked with all objects listed
+in a variable 'util-obj-y'. So, for example, util/Makefile.obj will
+contain a set of definitions looking like
+
+  util-obj-y += bitmap.o bitops.o hbitmap.o
+  util-obj-y += fifo8.o
+  util-obj-y += acl.o
+  util-obj-y += error.o qemu-error.o
+
+When there is an object file which needs to be conditionally built based
+on some characteristic of the host system, the configure script will
+define a variable for the conditional. For example, on Windows it will
+define $(CONFIG_POSIX) with a value of 'n' and $(CONFIG_WIN32) with a
+value of 'y'. It is now possible to use the config variables when
+listing object files. For example,
+
+  util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
+  util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
+
+On Windows this expands to
+
+  util-obj-y += oslib-win32.o qemu-thread-win32.o
+  util-obj-n += oslib-posix.o qemu-thread-posix.o
+
+Since libqemutil.a links in $(util-obj-y), the POSIX specific files
+listed against $(util-obj-n) are ignored on the Windows platform builds.
+
+
+CFLAGS / LDFLAGS / LIBS handling
+--------------------------------
+
+There are many different binaries being built with differing purposes,
+and some of them might even be 3rd party libraries pulled in via git
+submodules. As such the use of the global CFLAGS variable is generally
+avoided in QEMU, since it would apply to too many build targets.
+
+Flags that are needed by any QEMU code (i.e. everything *except* GIT
+submodule projects) are put in $(QEMU_CFLAGS) variable. For linker
+flags the $(LIBS) variable is sometimes used, but a couple of more
+targeted variables are preferred. $(libs_softmmu) is used for
+libraries that must be linked to system emulator targets, $(LIBS_TOOLS)
+is used for tools like qemu-img, qemu-nbd, etc and $(LIBS_QGA) is used
+for the QEMU guest agent. There is currently no specific variable for
+the userspace emulator targets as the global $(LIBS), or more targeted
+variables shown below, are sufficient.
+
+In addition to these variables, it is possible to provide cflags and
+libs against individual source code files, by defining variables of the
+form $FILENAME-cflags and $FILENAME-libs. For example, the curl block
+driver needs to link to the libcurl library, so block/Makefile defines
+some variables:
+
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+The scope is a little different between the two variables. The libs get
+used when linking any target binary that includes the curl.o object
+file, while the cflags get used when compiling the curl.c file only.
+
+
+Statically defined files
+------------------------
+
+The following key files are statically defined in the source tree, with
+the rules needed to build QEMU. Their behaviour is influenced by a
+number of dynamically created files listed later.
+
+- Makefile
+
+The main entry point used when invoking make to build all the components
+of QEMU. The default 'all' target will naturally result in the build of
+every component. The various tools and helper binaries are built
+directly via a non-recursive set of rules.
+
+Each system/userspace emulation target needs to have a slightly
+different set of make rules / variables. Thus, make will be recursively
+invoked for each of the emulation targets.
+
+The recursive invocation will end up processing the toplevel
+Makefile.target file (more on that later).
+
+
+- */Makefile.objs
+
+Since the source code is spread across multiple directories, the rules
+for each file are similarly modularized. Thus each subdirectory
+containing .c files will usually also contain a Makefile.objs file.
+These files are not directly invoked by a recursive make, but instead
+they are imported by the top level Makefile and/or Makefile.target
+
+Each Makefile.objs usually just declares a set of variables listing the
+.o files that need building from the source files in the directory. They
+will also define any custom linker or compiler flags. For example in
+block/Makefile.objs
+
+  block-obj-$(CONFIG_LIBISCSI) += iscsi.o
+  block-obj-$(CONFIG_CURL) += curl.o
+
+  ..snip...
+
+  iscsi.o-cflags     := $(LIBISCSI_CFLAGS)
+  iscsi.o-libs       := $(LIBISCSI_LIBS)
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+If there are any rules defined in the Makefile.objs file, they should
+all use $(obj) as a prefix to the target, e.g.
+
+  $(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
+
+
+- Makefile.target
+
+This file provides the entry point used to build each individual system
+or userspace emulator target. Each enabled target has its own
+subdirectory. For example if configure is run with the argument
+'--target-list=x86_64-softmmu', then a sub-directory 'x86_64-softmu'
+will be created, containing a 'Makefile' which symlinks back to
+Makefile.target
+
+So when the recursive '$(MAKE) -C x86_64-softmmu' is invoked, it ends up
+using Makefile.target for the build rules.
+
+
+- rules.mak
+
+This file provides the generic helper rules for invoking build tools, in
+particular the compiler and linker. This also contains the magic (hairy)
+'unnest-vars' function which is used to merge the variable definitions
+from all Makefile.objs in the source tree down into the main Makefile
+context.
+
+
+- default-configs/*.mak
+
+The files under default-configs/ control what emulated hardware is built
+into each QEMU system and userspace emulator targets. They merely
+contain a long list of config variable definitions. For example,
+default-configs/x86_64-softmmu.mak has:
+
+  include pci.mak
+  include sound.mak
+  include usb.mak
+  CONFIG_QXL=$(CONFIG_SPICE)
+  CONFIG_VGA_ISA=y
+  CONFIG_VGA_CIRRUS=y
+  CONFIG_VMWARE_VGA=y
+  CONFIG_VIRTIO_VGA=y
+  ...snip...
+
+These files rarely need changing unless new devices / hardware need to
+be enabled for a particular system/userspace emulation target
+
+
+- tests/Makefile
+
+Rules for building the unit tests. This file is included directly by the
+top level Makefile, so anything defined in this file will influence the
+entire build system. Care needs to be taken when writing rules for tests
+to ensure they only apply to the unit test execution / build.
+
+
+- po/Makefile
+
+Rules for building and installing the binary message catalogs from the
+text .po file sources. This almost never needs changing for any reason.
+
+
+Dynamically created files
+-------------------------
+
+The following files are generated dynamically by configure in order to
+control the behaviour of the statically defined makefiles. This avoids
+the need for QEMU makefiles to go through any pre-processing as seen
+with autotools, where Makefile.am generates Makefile.in which generates
+Makefile.
+
+
+- config-host.mak
+
+When configure has determined the characteristics of the build host it
+will write a long list of variables to config-host.mak file. This
+provides the various install directories, compiler / linker flags and a
+variety of CONFIG_* variables related to optionally enabled features.
+This is imported by the top level Makefile in order to tailor the build
+output.
+
+The variables defined here are those which are applicable to all QEMU
+build outputs. Variables which are potentially different for each
+emulator target are defined by the next file...
+
+It is also used as a dependency checking mechanism. If make sees that
+the modification timestamp on configure is newer than that on
+config-host.mak, then configure will be re-run.
+
+
+- config-host.h
+
+The config-host.h file is used by source code to determine what features
+are enabled. It is generated from the contents of config-host.mak using
+the scripts/create_config program. This extracts all the CONFIG_* variables,
+most of the HOST_* variables and a few other misc variables from
+config-host.mak, formatting them as C preprocessor macros.
+
+
+- $TARGET-NAME/config-target.mak
+
+TARGET-NAME is the name of a system or userspace emulator, for example,
+x86_64-softmmu denotes the system emulator for the x86_64 architecture.
+This file contains the variables which need to vary on a per-target
+basis. For example, it will indicate whether KVM or Xen are enabled for
+the target and any other potential custom libraries needed for linking
+the target.
+
+
+- $TARGET-NAME/config-devices.mak
+
+TARGET-NAME is again the name of a system or userspace emulator. The
+config-devices.mak file is automatically generated by make using the
+scripts/make_device_config.sh program, feeding it the
+default-configs/$TARGET-NAME file as input.
+
+
+- $TARGET-NAME/Makefile
+
+This is the entrypoint used when make recurses to build a single system
+or userspace emulator target. It is merely a symlink back to the
+Makefile.target in the top level.
diff --git a/qemu/docs/libcacard.txt b/qemu/docs/libcacard.txt
deleted file mode 100644
index 8db421d3a..000000000
--- a/qemu/docs/libcacard.txt
+++ /dev/null
@@ -1,483 +0,0 @@
-This file documents the CAC (Common Access Card) library in the libcacard
-subdirectory.
-
-Virtual Smart Card Emulator
-
-This emulator is designed to provide emulation of actual smart cards to a
-virtual card reader running in a guest virtual machine. The emulated smart
-cards can be representations of real smart cards, where the necessary functions
-such as signing, card removal/insertion, etc. are mapped to real, physical
-cards which are shared with the client machine the emulator is running on, or
-the cards could be pure software constructs.
-
-The emulator is structured to allow multiple replaceable or additional pieces,
-so it can be easily modified for future requirements. The primary envisioned
-modifications are:
-
-1) The socket connection to the virtual card reader (presumably a CCID reader,
-but other ISO-7816 compatible readers could be used). The code that handles
-this is in vscclient.c.
-
-2) The virtual card low level emulation. This is currently supplied by using
-NSS. This emulation could be replaced by implementations based on other
-security libraries, including but not limitted to openssl+pkcs#11 library,
-raw pkcs#11, Microsoft CAPI, direct opensc calls, etc. The code that handles
-this is in vcard_emul_nss.c.
-
-3) Emulation for new types of cards. The current implementation emulates the
-original DoD CAC standard with separate pki containers. This emulator lives in
-cac.c. More than one card type emulator could be included. Other cards could
-be emulated as well, including PIV, newer versions of CAC, PKCS #15, etc.
-
---------------------
-Replacing the Socket Based Virtual Reader Interface.
-
-The current implementation contains a replaceable module vscclient.c. The
-current vscclient.c implements a sockets interface to the virtual ccid reader
-on the guest. CCID commands that are pertinent to emulation are passed
-across the socket, and their responses are passed back along that same socket.
-The protocol that vscclient uses is defined in vscard_common.h and connects
-to a qemu ccid usb device. Since this socket runs as a client, vscclient.c
-implements a program with a main entry. It also handles argument parsing for
-the emulator.
-
-An application that wants to use the virtual reader can replace vscclient.c
-with its own implementation that connects to its own CCID reader.  The calls
-that the CCID reader can call are:
-
-      VReaderList * vreader_get_reader_list();
-
-  This function returns a list of virtual readers.  These readers may map to
-  physical devices, or simulated devices depending on vcard the back end. Each
-  reader in the list should represent a reader to the virtual machine. Virtual
-  USB address mapping is left to the CCID reader front end. This call can be
-  made any time to get an updated list. The returned list is a copy of the
-  internal list that can be referenced by the caller without locking. This copy
-  must be freed by the caller with vreader_list_delete when it is no longer
-  needed.
-
-      VReaderListEntry *vreader_list_get_first(VReaderList *);
-
-  This function gets the first entry on the reader list. Along with
-  vreader_list_get_next(), vreader_list_get_first() can be used to walk the
-  reader list returned from vreader_get_reader_list(). VReaderListEntries are
-  part of the list themselves and do not need to be freed separately from the
-  list. If there are no entries on the list, it will return NULL.
-
-      VReaderListEntry *vreader_list_get_next(VReaderListEntry *);
-
-  This function gets the next entry in the list. If there are no more entries
-  it will return NULL.
-
-      VReader * vreader_list_get_reader(VReaderListEntry *)
-
-  This function returns the reader stored in the reader List entry. Caller gets
-  a new reference to a reader. The caller must free its reference when it is
-  finished with vreader_free().
-
-      void vreader_free(VReader *reader);
-
-   This function frees a reference to a reader. Readers are reference counted
-   and are automatically deleted when the last reference is freed.
-
-      void vreader_list_delete(VReaderList *list);
-
-   This function frees the list, all the elements on the list, and all the
-   reader references held by the list.
-
-      VReaderStatus vreader_power_on(VReader *reader, char *atr, int *len);
-
-  This function simulates a card power on. A virtual card does not care about
-  the actual voltage and other physical parameters, but it does care that the
-  card is actually on or off. Cycling the card causes the card to reset. If
-  the caller provides enough space, vreader_power_on will return the ATR of
-  the virtual card. The amount of space provided in atr should be indicated
-  in *len. The function modifies *len to be the actual length of of the
-  returned ATR.
-
-      VReaderStatus vreader_power_off(VReader *reader);
-
-  This function simulates a power off of a virtual card.
-
-      VReaderStatus vreader_xfer_bytes(VReader *reader, unsigne char *send_buf,
-                                       int send_buf_len,
-                                       unsigned char *receive_buf,
-                                       int receive_buf_len);
-
-  This function sends a raw apdu to a card and returns the card's response.
-  The CCID front end should return the response back. Most of the emulation
-  is driven from these APDUs.
-
-      VReaderStatus vreader_card_is_present(VReader *reader);
-
-  This function returns whether or not the reader has a card inserted. The
-  vreader_power_on, vreader_power_off, and vreader_xfer_bytes will return
-  VREADER_NO_CARD.
-
-       const char *vreader_get_name(VReader *reader);
-
-  This function returns the name of the reader. The name comes from the card
-  emulator level and is usually related to the name of the physical reader.
-
-       VReaderID vreader_get_id(VReader *reader);
-
-  This function returns the id of a reader. All readers start out with an id
-  of -1. The application can set the id with vreader_set_id.
-
-       VReaderStatus vreader_get_id(VReader *reader, VReaderID id);
-
-  This function sets the reader id. The application is responsible for making
-  sure that the id is unique for all readers it is actively using.
-
-       VReader *vreader_find_reader_by_id(VReaderID id);
-
-  This function returns the reader which matches the id. If two readers match,
-  only one is returned. The function returns NULL if the id is -1.
-
-       Event *vevent_wait_next_vevent();
-
-  This function blocks waiting for reader and card insertion events. There
-  will be one event for each card insertion, each card removal, each reader
-  insertion and each reader removal. At start up, events are created for all
-  the initial readers found, as well as all the cards that are inserted.
-
-       Event *vevent_get_next_vevent();
-
-  This function returns a pending event if it exists, otherwise it returns
-  NULL. It does not block.
-
-----------------
-Card Type Emulator: Adding a New Virtual Card Type
-
-The ISO 7816 card spec describes 2 types of cards:
- 1) File system cards, where the smartcard is managed by reading and writing
-data to files in a file system. There is currently only boiler plate
-implemented for file system cards.
- 2) VM cards, where the card has loadable applets which perform the card
-functions. The current implementation supports VM cards.
-
-In the case of VM cards, the difference between various types of cards is
-really what applets have been installed in that card. This structure is
-mirrored in card type emulators. The 7816 emulator already handles the basic
-ISO 7186 commands. Card type emulators simply need to add the virtual applets
-which emulate the real card applets. Card type emulators have exactly one
-public entry point:
-
-       VCARDStatus xxx_card_init(VCard *card, const char *flags,
-                               const unsigned char *cert[],
-                               int cert_len[],
-                               VCardKey *key[],
-                               int cert_count);
-
-  The parameters for this are:
-  card       - the virtual card structure which will represent this card.
-  flags      - option flags that may be specific to this card type.
-  cert       - array of binary certificates.
-  cert_len   - array of lengths of each of the certificates specified in cert.
-  key        - array of opaque key structures representing the private keys on
-               the card.
-  cert_count - number of entries in cert, cert_len, and key arrays.
-
-  Any cert, cert_len, or key with the same index are matching sets. That is
-  cert[0] is cert_len[0] long and has the corresponding private key of key[0].
-
-The card type emulator is expected to own the VCardKeys, but it should copy
-any raw cert data it wants to save. It can create new applets and add them to
-the card using the following functions:
-
-       VCardApplet *vcard_new_applet(VCardProcessAPDU apdu_func,
-                                     VCardResetApplet reset_func,
-                                     const unsigned char *aid,
-                                     int aid_len);
-
-  This function creates a new applet. Applet structures store the following
-  information:
-     1) the AID of the applet (set by aid and aid_len).
-     2) a function to handle APDUs for this applet. (set by apdu_func, more on
-        this below).
-     3) a function to reset the applet state when the applet is selected.
-        (set by reset_func, more on this below).
-     3) applet private data, a data pointer used by the card type emulator to
-        store any data or state it needs to complete requests. (set by a
-        separate call).
-     4) applet private data free, a function used to free the applet private
-        data when the applet itself is destroyed.
-  The created applet can be added to the card with vcard_add_applet below.
-
-        void vcard_set_applet_private(VCardApplet *applet,
-                                      VCardAppletPrivate *private,
-                                      VCardAppletPrivateFree private_free);
-  This function sets the private data and the corresponding free function.
-  VCardAppletPrivate is an opaque data structure to the rest of the emulator.
-  The card type emulator can define it any way it wants by defining
-  struct VCardAppletPrivateStruct {};. If there is already a private data
-  structure on the applet, the old one is freed before the new one is set up.
-  passing two NULL clear any existing private data.
-
-         VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-
-  Add an applet onto the list of applets attached to the card. Once an applet
-  has been added, it can be selected by its AID, and then commands will be
-  routed to it VCardProcessAPDU function. This function adopts the applet that
-  is passed into it. Note: 2 applets with the same AID should not be added to
-  the same card. It is permissible to add more than one applet. Multiple applets
-  may have the same VCardPRocessAPDU entry point.
-
-The certs and keys should be attached to private data associated with one or
-more appropriate applets for that card. Control will come to the card type
-emulators once one of its applets are selected through the VCardProcessAPDU
-function it specified when it created the applet.
-
-The signature of VCardResetApplet is:
-        VCardStatus (*VCardResetApplet) (VCard *card, int channel);
-  This function will reset the any internal applet state that needs to be
-  cleared after a select applet call. It should return VCARD_DONE;
-
-The signature of VCardProcessAPDU is:
-        VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                         VCardResponse **response);
-  This function examines the APDU and determines whether it should process
-  the apdu directly, reject the apdu as invalid, or pass the apdu on to
-  the basic 7816 emulator for processing.
-      If the 7816 emulator should process the apdu, then the VCardProcessAPDU
-  should return VCARD_NEXT.
-      If there is an error, then VCardProcessAPDU should return an error
-  response using vcard_make_response and the appropriate 7816 error code
-  (see card_7816t.h) or vcard_make_response with a card type specific error
-  code. It should then return VCARD_DONE.
-      If the apdu can be processed correctly, VCardProcessAPDU should do so,
-  set the response value appropriately for that APDU, and return VCARD_DONE.
-  VCardProcessAPDU should always set the response if it returns VCARD_DONE.
-  It should always either return VCARD_DONE or VCARD_NEXT.
-
-Parsing the APDU --
-
-Prior to processing calling the card type emulator's VCardProcessAPDU function, the emulator has already decoded the APDU header and set several fields:
-
-   apdu->a_data - The raw apdu data bytes.
-   apdu->a_len  - The len of the raw apdu data.
-   apdu->a_body - The start of any post header parameter data.
-   apdu->a_Lc   - The parameter length value.
-   apdu->a_Le   - The expected length of any returned data.
-   apdu->a_cla  - The raw apdu class.
-   apdu->a_channel - The channel (decoded from the class).
-   apdu->a_secure_messaging_type - The decoded secure messaging type
-                                   (from class).
-   apdu->a_type - The decode class type.
-   apdu->a_gen_type - the generic class type (7816, PROPRIETARY, RFU, PTS).
-   apdu->a_ins  - The instruction byte.
-   apdu->a_p1   - Parameter 1.
-   apdu->a_p2   - Parameter 2.
-
-Creating a Response --
-
-The expected result of any APDU call is a response. The card type emulator must
-set *response with an appropriate VCardResponse value if it returns VCARD_DONE.
-Responses could be as simple as returning a 2 byte status word response, to as
-complex as returning a block of data along with a 2 byte response. Which is
-returned will depend on the semantics of the APDU. The following functions will
-create card responses.
-
-        VCardResponse *vcard_make_response(VCard7816Status status);
-
-    This is the most basic function to get a response. This function will
-    return a response the consists solely one 2 byte status code. If that status
-    code is defined in card_7816t.h, then this function is guaranteed to
-    return a response with that status. If a cart type specific status code
-    is passed and vcard_make_response fails to allocate the appropriate memory
-    for that response, then vcard_make_response will return a VCardResponse
-    of VCARD7816_STATUS_EXC_ERROR_MEMORY. In any case, this function is
-    guaranteed to return a valid VCardResponse.
-
-        VCardResponse *vcard_response_new(unsigned char *buf, int len,
-                                          VCard7816Status status);
-
-    This function is similar to vcard_make_response except it includes some
-    returned data with the response. It could also fail to allocate enough
-    memory, in which case it will return NULL.
-
-        VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                                       unsigned char sw2);
-
-    Sometimes in 7816 the response bytes are treated as two separate bytes with
-    split meanings. This function allows you to create a response based on
-    two separate bytes. This function could fail, in which case it will return
-    NULL.
-
-       VCardResponse *vcard_response_new_bytes(unsigned char *buf, int len,
-                                               unsigned char sw1,
-                                               unsigned char sw2);
-
-    This function is the same as vcard_response_new except you may specify
-    the status as two separate bytes like vcard_response_new_status_bytes.
-
-
-Implementing functionality ---
-
-The following helper functions access information about the current card
-and applet.
-
-        VCARDAppletPrivate *vcard_get_current_applet_private(VCard *card,
-                                                             int channel);
-
-    This function returns any private data set by the card type emulator on
-    the currently selected applet. The card type emulator keeps track of the
-    current applet state in this data structure. Any certs and keys associated
-    with a particular applet is also stored here.
-
-        int vcard_emul_get_login_count(VCard *card);
-
-    This function returns the the number of remaining login attempts for this
-    card. If the card emulator does not know, or the card does not have a
-    way of giving this information, this function returns -1.
-
-
-         VCard7816Status vcard_emul_login(VCard *card, unsigned char *pin,
-                                          int pin_len);
-
-    This function logs into the card and returns the standard 7816 status
-    word depending on the success or failure of the call.
-
-         void vcard_emul_delete_key(VCardKey *key);
-
-     This function frees the VCardKey passed in to xxxx_card_init. The card
-     type emulator is responsible for freeing this key when it no longer needs
-     it.
-
-         VCard7816Status vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                           unsigned char *buffer,
-                                           int buffer_size);
-
-     This function does a raw rsa op on the buffer with the given key.
-
-The sample card type emulator is found in cac.c. It implements the cac specific
-applets.  Only those applets needed by the coolkey pkcs#11 driver on the guest
-have been implemented. To support the full range CAC middleware, a complete CAC
-card according to the CAC specs should be implemented here.
-
-------------------------------
-Virtual Card Emulator
-
-This code accesses both real smart cards and simulated smart cards through
-services provided on the client. The current implementation uses NSS, which
-already knows how to talk to various PKCS #11 modules on the client, and is
-portable to most operating systems. A particular emulator can have only one
-virtual card implementation at a time.
-
-The virtual card emulator consists of a series of virtual card services. In
-addition to the services describe above (services starting with
-vcard_emul_xxxx), the virtual card emulator also provides the following
-functions:
-
-    VCardEmulError vcard_emul_init(cont VCardEmulOptions *options);
-
-  The options structure is built by another function in the virtual card
-  interface where a string of virtual card emulator specific strings are
-  mapped to the options. The actual structure is defined by the virtual card
-  emulator and is used to determine the configuration of soft cards, or to
-  determine which physical cards to present to the guest.
-
-  The vcard_emul_init function will build up sets of readers, create any
-  threads that are needed to watch for changes in the reader state. If readers
-  have cards present in them, they are also initialized.
-
-  Readers are created with the function.
-
-          VReader *vreader_new(VReaderEmul *reader_emul,
-                               VReaderEmulFree reader_emul_free);
-
-      The freeFunc is used to free the VReaderEmul * when the reader is
-      destroyed.  The VReaderEmul structure is an opaque structure to the
-      rest of the code, but defined by the virtual card emulator, which can
-      use it to store any reader specific state.
-
-  Once the reader has been created, it can be added to the front end with the
-  call:
-
-           VReaderStatus vreader_add_reader(VReader *reader);
-
-      This function will automatically generate the appropriate new reader
-      events and add the reader to the list.
-
-  To create a new card, the virtual card emulator will call a similar
-  function.
-
-           VCard *vcard_new(VCardEmul *card_emul,
-                            VCardEmulFree card_emul_free);
-
-      Like vreader_new, this function takes a virtual card emulator specific
-      structure which it uses to keep track of the card state.
-
-  Once the card is created, it is attached to a card type emulator with the
-  following function:
-
-            VCardStatus vcard_init(VCard *vcard, VCardEmulType type,
-                                   const char *flags,
-                                   unsigned char *const *certs,
-                                   int *cert_len,
-                                   VCardKey *key[],
-                                   int cert_count);
-
-      The vcard is the value returned from vcard_new. The type is the
-      card type emulator that this card should presented to the guest as.
-      The flags are card type emulator specific options. The certs,
-      cert_len, and keys are all arrays of length cert_count. These are the
-      the same of the parameters xxxx_card_init() accepts.
-
-   Finally the card is associated with its reader by the call:
-
-            VReaderStatus vreader_insert_card(VReader *vreader, VCard *vcard);
-
-      This function, like vreader_add_reader, will take care of any event
-      notification for the card insert.
-
-
-    VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-  Force a card that is present to appear to be removed to the guest, even if
-  that card is a physical card and is present.
-
-
-    VCardEmulError vcard_emul_force_card_insert(VReader *reader);
-
-  Force a card that has been removed by vcard_emul_force_card_remove to be
-  reinserted from the point of view of the guest. This will only work if the
-  card is physically present (which is always true fro a soft card).
-
-     void vcard_emul_get_atr(Vcard *card, unsigned char *atr, int *atr_len);
-
-  Return the virtual ATR for the card. By convention this should be the value
-  VCARD_ATR_PREFIX(size) followed by several ascii bytes related to this
-  particular emulator. For instance the NSS emulator returns
-  {VCARD_ATR_PREFIX(3), 'N', 'S', 'S' }. Do ot return more data then *atr_len;
-
-     void vcard_emul_reset(VCard *card, VCardPower power)
-
-   Set the state of 'card' to the current power level and reset its internal
-   state (logout, etc).
-
--------------------------------------------------------
-List of files and their function:
-README - This file
-card_7816.c - emulate basic 7816 functionality. Parse APDUs.
-card_7816.h - apdu and response services definitions.
-card_7816t.h - 7816 specific structures, types and definitions.
-event.c - event handling code.
-event.h - event handling services definitions.
-eventt.h - event handling structures and types
-vcard.c - handle common virtual card services like creation, destruction, and
-          applet management.
-vcard.h - common virtual card services function definitions.
-vcardt.h - comon virtual card types
-vreader.c - common virtual reader services.
-vreader.h - common virtual reader services definitions.
-vreadert.h - comon virtual reader types.
-vcard_emul_type.c - manage the card type emulators.
-vcard_emul_type.h - definitions for card type emulators.
-cac.c - card type emulator for CAC cards
-vcard_emul.h - virtual card emulator service definitions.
-vcard_emul_nss.c - virtual card emulator implementation for nss.
-vscclient.c - socket connection to guest qemu usb driver.
-vscard_common.h - common header with the guest qemu usb driver.
-mutex.h - header file for machine independent mutexes.
-link_test.c - static test to make sure all the symbols are properly defined.
diff --git a/qemu/docs/memory.txt b/qemu/docs/memory.txt
index 2ceb34894..431d9ca88 100644
--- a/qemu/docs/memory.txt
+++ b/qemu/docs/memory.txt
@@ -26,14 +26,28 @@ These represent memory as seen from the CPU or a device's viewpoint.
 Types of regions
 ----------------
 
-There are four types of memory regions (all represented by a single C type
+There are multiple types of memory regions (all represented by a single C type
 MemoryRegion):
 
 - RAM: a RAM region is simply a range of host memory that can be made available
   to the guest.
+  You typically initialize these with memory_region_init_ram().  Some special
+  purposes require the variants memory_region_init_resizeable_ram(),
+  memory_region_init_ram_from_file(), or memory_region_init_ram_ptr().
 
 - MMIO: a range of guest memory that is implemented by host callbacks;
   each read or write causes a callback to be called on the host.
+  You initialize these with memory_region_init_io(), passing it a
+  MemoryRegionOps structure describing the callbacks.
+
+- ROM: a ROM memory region works like RAM for reads (directly accessing
+  a region of host memory), but like MMIO for writes (invoking a callback).
+  You initialize these with memory_region_init_rom_device().
+
+- IOMMU region: an IOMMU region translates addresses of accesses made to it
+  and forwards them to some other target memory region.  As the name suggests,
+  these are only needed for modelling an IOMMU, not for simple devices.
+  You initialize these with memory_region_init_iommu().
 
 - container: a container simply includes other memory regions, each at
   a different offset.  Containers are useful for grouping several regions
@@ -45,12 +59,22 @@ MemoryRegion):
   can overlay a subregion of RAM with MMIO or ROM, or a PCI controller
   that does not prevent card from claiming overlapping BARs.
 
+  You initialize a pure container with memory_region_init().
+
 - alias: a subsection of another region.  Aliases allow a region to be
   split apart into discontiguous regions.  Examples of uses are memory banks
   used when the guest address space is smaller than the amount of RAM
   addressed, or a memory controller that splits main memory to expose a "PCI
   hole".  Aliases may point to any type of region, including other aliases,
   but an alias may not point back to itself, directly or indirectly.
+  You initialize these with memory_region_init_alias().
+
+- reservation region: a reservation region is primarily for debugging.
+  It claims I/O space that is not supposed to be handled by QEMU itself.
+  The typical use is to track parts of the address space which will be
+  handled by the host kernel when KVM is enabled.
+  You initialize these with memory_region_init_reservation(), or by
+  passing a NULL callback parameter to memory_region_init_io().
 
 It is valid to add subregions to a region which is not a pure container
 (that is, to an MMIO, RAM or ROM region). This means that the region
@@ -156,14 +180,14 @@ aliases that leave holes then the lower priority region will appear in these
 holes too.)
 
 For example, suppose we have a container A of size 0x8000 with two subregions
-B and C. B is a container mapped at 0x2000, size 0x4000, priority 1; C is
-an MMIO region mapped at 0x0, size 0x6000, priority 2. B currently has two
+B and C. B is a container mapped at 0x2000, size 0x4000, priority 2; C is
+an MMIO region mapped at 0x0, size 0x6000, priority 1. B currently has two
 of its own subregions: D of size 0x1000 at offset 0 and E of size 0x1000 at
 offset 0x2000. As a diagram:
 
-        0      1000   2000   3000   4000   5000   6000   7000    8000
-        |------|------|------|------|------|------|------|-------|
-  A:    [                                                       ]
+        0      1000   2000   3000   4000   5000   6000   7000   8000
+        |------|------|------|------|------|------|------|------|
+  A:    [                                                      ]
   C:    [CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC]
   B:                  [                          ]
   D:                  [DDDDD]
@@ -223,7 +247,7 @@ system_memory: container@0-2^48-1
  |
  +---- himem: alias@0x100000000-0x11fffffff ---> #ram (0xe0000000-0xffffffff)
  |
- +---- vga-window: alias@0xa0000-0xbfffff ---> #pci (0xa0000-0xbffff)
+ +---- vga-window: alias@0xa0000-0xbffff ---> #pci (0xa0000-0xbffff)
  |      (prio 1)
  |
  +---- pci-hole: alias@0xe0000000-0xffffffff ---> #pci (0xe0000000-0xffffffff)
@@ -273,8 +297,9 @@ various constraints can be supplied to control how these callbacks are called:
  - .valid.min_access_size, .valid.max_access_size define the access sizes
    (in bytes) which the device accepts; accesses outside this range will
    have device and bus specific behaviour (ignored, or machine check)
- - .valid.aligned specifies that the device only accepts naturally aligned
-   accesses.  Unaligned accesses invoke device and bus specific behaviour.
+ - .valid.unaligned specifies that the *device being modelled* supports
+    unaligned accesses; if false, unaligned accesses will invoke the
+    appropriate bus or CPU specific behaviour.
  - .impl.min_access_size, .impl.max_access_size define the access sizes
    (in bytes) supported by the *implementation*; other access sizes will be
    emulated using the ones available.  For example a 4-byte write will be
@@ -282,5 +307,5 @@ various constraints can be supplied to control how these callbacks are called:
  - .impl.unaligned specifies that the *implementation* supports unaligned
    accesses; if false, unaligned accesses will be emulated by two aligned
    accesses.
- - .old_mmio can be used to ease porting from code using
+ - .old_mmio eases the porting of code that was formerly using
    cpu_register_io_memory(). It should not be used in new code.
diff --git a/qemu/docs/migration.txt b/qemu/docs/migration.txt
index f6df4beb2..90209ab29 100644
--- a/qemu/docs/migration.txt
+++ b/qemu/docs/migration.txt
@@ -291,3 +291,194 @@ save/send this state when we are in the middle of a pio operation
 (that is what ide_drive_pio_state_needed() checks).  If DRQ_STAT is
 not enabled, the values on that fields are garbage and don't need to
 be sent.
+
+= Return path =
+
+In most migration scenarios there is only a single data path that runs
+from the source VM to the destination, typically along a single fd (although
+possibly with another fd or similar for some fast way of throwing pages across).
+
+However, some uses need two way communication; in particular the Postcopy
+destination needs to be able to request pages on demand from the source.
+
+For these scenarios there is a 'return path' from the destination to the source;
+qemu_file_get_return_path(QEMUFile* fwdpath) gives the QEMUFile* for the return
+path.
+
+  Source side
+     Forward path - written by migration thread
+     Return path  - opened by main thread, read by return-path thread
+
+  Destination side
+     Forward path - read by main thread
+     Return path  - opened by main thread, written by main thread AND postcopy
+                    thread (protected by rp_mutex)
+
+= Postcopy =
+'Postcopy' migration is a way to deal with migrations that refuse to converge
+(or take too long to converge) its plus side is that there is an upper bound on
+the amount of migration traffic and time it takes, the down side is that during
+the postcopy phase, a failure of *either* side or the network connection causes
+the guest to be lost.
+
+In postcopy the destination CPUs are started before all the memory has been
+transferred, and accesses to pages that are yet to be transferred cause
+a fault that's translated by QEMU into a request to the source QEMU.
+
+Postcopy can be combined with precopy (i.e. normal migration) so that if precopy
+doesn't finish in a given time the switch is made to postcopy.
+
+=== Enabling postcopy ===
+
+To enable postcopy, issue this command on the monitor prior to the
+start of migration:
+
+migrate_set_capability postcopy-ram on
+
+The normal commands are then used to start a migration, which is still
+started in precopy mode.  Issuing:
+
+migrate_start_postcopy
+
+will now cause the transition from precopy to postcopy.
+It can be issued immediately after migration is started or any
+time later on.  Issuing it after the end of a migration is harmless.
+
+Note: During the postcopy phase, the bandwidth limits set using
+migrate_set_speed is ignored (to avoid delaying requested pages that
+the destination is waiting for).
+
+=== Postcopy device transfer ===
+
+Loading of device data may cause the device emulation to access guest RAM
+that may trigger faults that have to be resolved by the source, as such
+the migration stream has to be able to respond with page data *during* the
+device load, and hence the device data has to be read from the stream completely
+before the device load begins to free the stream up.  This is achieved by
+'packaging' the device data into a blob that's read in one go.
+
+Source behaviour
+
+Until postcopy is entered the migration stream is identical to normal
+precopy, except for the addition of a 'postcopy advise' command at
+the beginning, to tell the destination that postcopy might happen.
+When postcopy starts the source sends the page discard data and then
+forms the 'package' containing:
+
+   Command: 'postcopy listen'
+   The device state
+      A series of sections, identical to the precopy streams device state stream
+      containing everything except postcopiable devices (i.e. RAM)
+   Command: 'postcopy run'
+
+The 'package' is sent as the data part of a Command: 'CMD_PACKAGED', and the
+contents are formatted in the same way as the main migration stream.
+
+During postcopy the source scans the list of dirty pages and sends them
+to the destination without being requested (in much the same way as precopy),
+however when a page request is received from the destination, the dirty page
+scanning restarts from the requested location.  This causes requested pages
+to be sent quickly, and also causes pages directly after the requested page
+to be sent quickly in the hope that those pages are likely to be used
+by the destination soon.
+
+Destination behaviour
+
+Initially the destination looks the same as precopy, with a single thread
+reading the migration stream; the 'postcopy advise' and 'discard' commands
+are processed to change the way RAM is managed, but don't affect the stream
+processing.
+
+------------------------------------------------------------------------------
+                        1      2   3     4 5                      6   7
+main -----DISCARD-CMD_PACKAGED ( LISTEN  DEVICE     DEVICE DEVICE RUN )
+thread                             |       |
+                                   |     (page request)
+                                   |        \___
+                                   v            \
+listen thread:                     --- page -- page -- page -- page -- page --
+
+                                   a   b        c
+------------------------------------------------------------------------------
+
+On receipt of CMD_PACKAGED (1)
+   All the data associated with the package - the ( ... ) section in the
+diagram - is read into memory (into a QEMUSizedBuffer), and the main thread
+recurses into qemu_loadvm_state_main to process the contents of the package (2)
+which contains commands (3,6) and devices (4...)
+
+On receipt of 'postcopy listen' - 3 -(i.e. the 1st command in the package)
+a new thread (a) is started that takes over servicing the migration stream,
+while the main thread carries on loading the package.   It loads normal
+background page data (b) but if during a device load a fault happens (5) the
+returned page (c) is loaded by the listen thread allowing the main threads
+device load to carry on.
+
+The last thing in the CMD_PACKAGED is a 'RUN' command (6) letting the destination
+CPUs start running.
+At the end of the CMD_PACKAGED (7) the main thread returns to normal running behaviour
+and is no longer used by migration, while the listen thread carries
+on servicing page data until the end of migration.
+
+=== Postcopy states ===
+
+Postcopy moves through a series of states (see postcopy_state) from
+ADVISE->DISCARD->LISTEN->RUNNING->END
+
+  Advise:  Set at the start of migration if postcopy is enabled, even
+           if it hasn't had the start command; here the destination
+           checks that its OS has the support needed for postcopy, and performs
+           setup to ensure the RAM mappings are suitable for later postcopy.
+           The destination will fail early in migration at this point if the
+           required OS support is not present.
+           (Triggered by reception of POSTCOPY_ADVISE command)
+
+  Discard: Entered on receipt of the first 'discard' command; prior to
+           the first Discard being performed, hugepages are switched off
+           (using madvise) to ensure that no new huge pages are created
+           during the postcopy phase, and to cause any huge pages that
+           have discards on them to be broken.
+
+  Listen:  The first command in the package, POSTCOPY_LISTEN, switches
+           the destination state to Listen, and starts a new thread
+           (the 'listen thread') which takes over the job of receiving
+           pages off the migration stream, while the main thread carries
+           on processing the blob.  With this thread able to process page
+           reception, the destination now 'sensitises' the RAM to detect
+           any access to missing pages (on Linux using the 'userfault'
+           system).
+
+  Running: POSTCOPY_RUN causes the destination to synchronise all
+           state and start the CPUs and IO devices running.  The main
+           thread now finishes processing the migration package and
+           now carries on as it would for normal precopy migration
+           (although it can't do the cleanup it would do as it
+           finishes a normal migration).
+
+  End:     The listen thread can now quit, and perform the cleanup of migration
+           state, the migration is now complete.
+
+=== Source side page maps ===
+
+The source side keeps two bitmaps during postcopy; 'the migration bitmap'
+and 'unsent map'.  The 'migration bitmap' is basically the same as in
+the precopy case, and holds a bit to indicate that page is 'dirty' -
+i.e. needs sending.  During the precopy phase this is updated as the CPU
+dirties pages, however during postcopy the CPUs are stopped and nothing
+should dirty anything any more.
+
+The 'unsent map' is used for the transition to postcopy. It is a bitmap that
+has a bit cleared whenever a page is sent to the destination, however during
+the transition to postcopy mode it is combined with the migration bitmap
+to form a set of pages that:
+   a) Have been sent but then redirtied (which must be discarded)
+   b) Have not yet been sent - which also must be discarded to cause any
+      transparent huge pages built during precopy to be broken.
+
+Note that the contents of the unsentmap are sacrificed during the calculation
+of the discard set and thus aren't valid once in postcopy.  The dirtymap
+is still valid and is used to ensure that no page is sent more than once.  Any
+request for a page that has already been sent is ignored.  Duplicate requests
+such as this can happen as a page is sent at about the same time the
+destination accesses it.
+
diff --git a/qemu/docs/multiseat.txt b/qemu/docs/multiseat.txt
index ebf244693..807518c8a 100644
--- a/qemu/docs/multiseat.txt
+++ b/qemu/docs/multiseat.txt
@@ -135,7 +135,7 @@ configuration:
             TAG+="seat", ENV{ID_AUTOSEAT}="1"
 
 Patch with this rule has been submitted to upstream udev/systemd, was
-accepted and and should be included in the next systemd release (222).
+accepted and should be included in the next systemd release (222).
 So, if your guest has this or a newer version, multiseat will work just
 fine without any manual guest configuration.
 
diff --git a/qemu/docs/pci_expander_bridge.txt b/qemu/docs/pci_expander_bridge.txt
index d7913fb4a..36750273b 100644
--- a/qemu/docs/pci_expander_bridge.txt
+++ b/qemu/docs/pci_expander_bridge.txt
@@ -23,9 +23,9 @@ A detailed command line would be:
 -m 2G
 -object memory-backend-ram,size=1024M,policy=bind,host-nodes=0,id=ram-node0 -numa node,nodeid=0,cpus=0,memdev=ram-node0
 -object memory-backend-ram,size=1024M,policy=bind,host-nodes=1,id=ram-node1 -numa node,nodeid=1,cpus=1,memdev=ram-node1
--device pxb,id=bridge1,bus=pci.0,numa_node=1,bus_nr=4 -netdev user,id=nd-device e1000,bus=bridge1,addr=0x4,netdev=nd
--device pxb,id=bridge2,bus=pci.0,numa_node=0,bus_nr=8,bus=pci.0 -device e1000,bus=bridge2,addr=0x3
--device pxb,id=bridge3,bus=pci.0,bus_nr=40,bus=pci.0 -drive if=none,id=drive0,file=[img] -device virtio-blk-pci,drive=drive0,scsi=off,bus=bridge3,addr=1
+-device pxb,id=bridge1,bus=pci.0,numa_node=1,bus_nr=4 -netdev user,id=nd -device e1000,bus=bridge1,addr=0x4,netdev=nd
+-device pxb,id=bridge2,bus=pci.0,numa_node=0,bus_nr=8 -device e1000,bus=bridge2,addr=0x3
+-device pxb,id=bridge3,bus=pci.0,bus_nr=40 -drive if=none,id=drive0,file=[img] -device virtio-blk-pci,drive=drive0,scsi=off,bus=bridge3,addr=1
 
 Here you have:
  - 2 NUMA nodes for the guest, 0 and 1. (both mapped to the same NUMA node in host, but you can and should put it in different host NUMA nodes)
@@ -43,7 +43,7 @@ Implementation
 ==============
 The PXB is composed by:
 - HostBridge (TYPE_PXB_HOST)
-  The host bridge allows to register and query the PXB's rPCI root bus in QEMU.
+  The host bridge allows to register and query the PXB's PCI root bus in QEMU.
 - PXBDev(TYPE_PXB_DEVICE)
   It is a regular PCI Device that resides on the piix host-bridge bus and its bus uses the same PCI domain.
   However, the bus behind is exposed through ACPI as a primary PCI bus and starts a new PCI hierarchy.
diff --git a/qemu/docs/qapi-code-gen.txt b/qemu/docs/qapi-code-gen.txt
index 61b5be47f..0e4bafff0 100644
--- a/qemu/docs/qapi-code-gen.txt
+++ b/qemu/docs/qapi-code-gen.txt
@@ -1,7 +1,7 @@
 = How to use the QAPI code generator =
 
 Copyright IBM Corp. 2011
-Copyright (C) 2012-2015 Red Hat, Inc.
+Copyright (C) 2012-2016 Red Hat, Inc.
 
 This work is licensed under the terms of the GNU GPL, version 2 or
 later. See the COPYING file in the top-level directory.
@@ -52,7 +52,7 @@ schema.  The documentation is delimited between two lines of ##, then
 the first line names the expression, an optional overview is provided,
 then individual documentation about each member of 'data' is provided,
 and finally, a 'Since: x.y.z' tag lists the release that introduced
-the expression.  Optional fields are tagged with the phrase
+the expression.  Optional members are tagged with the phrase
 '#optional', often with their default value; and extensions added
 after the expression was first released are also given a '(since
 x.y.z)' comment.  For example:
@@ -106,27 +106,28 @@ Types, commands, and events share a common namespace.  Therefore,
 generally speaking, type definitions should always use CamelCase for
 user-defined type names, while built-in types are lowercase. Type
 definitions should not end in 'Kind', as this namespace is used for
-creating implicit C enums for visiting union types.  Command names,
-and field names within a type, should be all lower case with words
+creating implicit C enums for visiting union types, or in 'List', as
+this namespace is used for creating array types.  Command names,
+and member names within a type, should be all lower case with words
 separated by a hyphen.  However, some existing older commands and
 complex types use underscore; when extending such expressions,
 consistency is preferred over blindly avoiding underscore.  Event
-names should be ALL_CAPS with words separated by underscore.  The
-special string '**' appears for some commands that manually perform
-their own type checking rather than relying on the type-safe code
-produced by the qapi code generators.
+names should be ALL_CAPS with words separated by underscore.  Member
+names cannot start with 'has-' or 'has_', as this is reserved for
+tracking optional members.
 
-Any name (command, event, type, field, or enum value) beginning with
+Any name (command, event, type, member, or enum value) beginning with
 "x-" is marked experimental, and may be withdrawn or changed
-incompatibly in a future release.  Downstream vendors may add
-extensions; such extensions should begin with a prefix matching
-"__RFQDN_" (for the reverse-fully-qualified-domain-name of the
-vendor), even if the rest of the name uses dash (example:
-__com.redhat_drive-mirror).  Other than downstream extensions (with
-leading underscore and the use of dots), all names should begin with a
-letter, and contain only ASCII letters, digits, dash, and underscore.
-It is okay to reuse names that match C keywords; the generator will
-rename a field named "default" in the QAPI to "q_default" in the
+incompatibly in a future release.  All names must begin with a letter,
+and contain only ASCII letters, digits, dash, and underscore.  There
+are two exceptions: enum values may start with a digit, and any
+extensions added by downstream vendors should start with a prefix
+matching "__RFQDN_" (for the reverse-fully-qualified-domain-name of
+the vendor), even if the rest of the name uses dash (example:
+__com.redhat_drive-mirror).  Names beginning with 'q_' are reserved
+for the generator: QMP names that resemble C keywords or other
+problematic strings will be munged in C to use this prefix.  For
+example, a member named "default" in qapi becomes "q_default" in the
 generated C code.
 
 In the rest of this document, usage lines are given for each
@@ -140,17 +141,26 @@ must have a value that forms a struct name.
 
 === Built-in Types ===
 
-The following types are built-in to the parser:
-  'str' - arbitrary UTF-8 string
-  'int' - 64-bit signed integer (although the C code may place further
-          restrictions on acceptable range)
-  'number' - floating point number
-  'bool' - JSON value of true or false
-  'int8', 'int16', 'int32', 'int64' - like 'int', but enforce maximum
-                                      bit size
-  'uint8', 'uint16', 'uint32', 'uint64' - unsigned counterparts
-  'size' - like 'uint64', but allows scaled suffix from command line
-           visitor
+The following types are predefined, and map to C as follows:
+
+  Schema    C          JSON
+  str       char *     any JSON string, UTF-8
+  number    double     any JSON number
+  int       int64_t    a JSON number without fractional part
+                       that fits into the C integer type
+  int8      int8_t     likewise
+  int16     int16_t    likewise
+  int32     int32_t    likewise
+  int64     int64_t    likewise
+  uint8     uint8_t    likewise
+  uint16    uint16_t   likewise
+  uint32    uint32_t   likewise
+  uint64    uint64_t   likewise
+  size      uint64_t   like uint64_t, except StringInputVisitor
+                       accepts size suffixes
+  bool      bool       JSON true or false
+  any       QObject *  any JSON value
+  QType     QType      JSON string matching enum QType values
 
 
 === Includes ===
@@ -163,7 +173,7 @@ The QAPI schema definitions can be modularized using the 'include' directive:
 
 The directive is evaluated recursively, and include paths are relative to the
 file using the directive. Multiple includes of the same file are
-safe.  No other keys should appear in the expression, and the include
+idempotent.  No other keys should appear in the expression, and the include
 value should be a string.
 
 As a matter of style, it is a good idea to have all files be
@@ -177,11 +187,11 @@ prevent incomplete include files.
 
 Usage: { 'struct': STRING, 'data': DICT, '*base': STRUCT-NAME }
 
-A struct is a dictionary containing a single 'data' key whose
-value is a dictionary.  This corresponds to a struct in C or an Object
-in JSON. Each value of the 'data' dictionary must be the name of a
-type, or a one-element array containing a type name.  An example of a
-struct is:
+A struct is a dictionary containing a single 'data' key whose value is
+a dictionary; the dictionary may be empty.  This corresponds to a
+struct in C or an Object in JSON. Each value of the 'data' dictionary
+must be the name of a type, or a one-element array containing a type
+name.  An example of a struct is:
 
  { 'struct': 'MyType',
    'data': { 'member1': 'str', 'member2': 'int', '*member3': 'str' } }
@@ -207,17 +217,18 @@ and must continue to work).
 
 On output structures (only mentioned in the 'returns' side of a command),
 changing from mandatory to optional is in general unsafe (older clients may be
-expecting the field, and could crash if it is missing), although it can be done
-if the only way that the optional argument will be omitted is when it is
-triggered by the presence of a new input flag to the command that older clients
-don't know to send.  Changing from optional to mandatory is safe.
+expecting the member, and could crash if it is missing), although it
+can be done if the only way that the optional argument will be omitted
+is when it is triggered by the presence of a new input flag to the
+command that older clients don't know to send.  Changing from optional
+to mandatory is safe.
 
 A structure that is used in both input and output of various commands
 must consider the backwards compatibility constraints of both directions
 of use.
 
 A struct definition can specify another struct as its base.
-In this case, the fields of the base type are included as top-level fields
+In this case, the members of the base type are included as top-level members
 of the new struct's dictionary in the Client JSON Protocol wire
 format. An example definition is:
 
@@ -227,7 +238,7 @@ format. An example definition is:
    'data': { '*backing': 'str' } }
 
 An example BlockdevOptionsGenericCOWFormat object on the wire could use
-both fields like this:
+both members like this:
 
  { "file": "/some/place/my-image",
    "backing": "/some/place/my-backing-file" }
@@ -236,6 +247,7 @@ both fields like this:
 === Enumeration types ===
 
 Usage: { 'enum': STRING, 'data': ARRAY-OF-STRING }
+       { 'enum': STRING, '*prefix': STRING, 'data': ARRAY-OF-STRING }
 
 An enumeration type is a dictionary containing a single 'data' key
 whose value is a list of strings.  An example enumeration is:
@@ -247,6 +259,13 @@ useful.  The list of strings should be lower case; if an enum name
 represents multiple words, use '-' between words.  The string 'max' is
 not allowed as an enum value, and values should not be repeated.
 
+The enum constants will be named by using a heuristic to turn the
+type name into a set of underscore separated words. For the example
+above, 'MyEnum' will turn into 'MY_ENUM' giving a constant name
+of 'MY_ENUM_VALUE1' for the first value. If the default heuristic
+does not result in a desirable name, the optional 'prefix' member
+can be used when defining the enum.
+
 The enumeration values are passed as strings over the Client JSON
 Protocol, but are encoded as C enum integral values in generated code.
 While the C code starts numbering at 0, it is better to use explicit
@@ -257,42 +276,43 @@ converting between strings and enum values.  Since the wire format
 always passes by name, it is acceptable to reorder or add new
 enumeration members in any location without breaking clients of Client
 JSON Protocol; however, removing enum values would break
-compatibility.  For any struct that has a field that will only contain
-a finite set of string values, using an enum type for that field is
-better than open-coding the field to be type 'str'.
+compatibility.  For any struct that has a member that will only contain
+a finite set of string values, using an enum type for that member is
+better than open-coding the member to be type 'str'.
 
 
 === Union types ===
 
 Usage: { 'union': STRING, 'data': DICT }
-or:    { 'union': STRING, 'data': DICT, 'base': STRUCT-NAME,
+or:    { 'union': STRING, 'data': DICT, 'base': STRUCT-NAME-OR-DICT,
          'discriminator': ENUM-MEMBER-OF-BASE }
 
 Union types are used to let the user choose between several different
 variants for an object.  There are two flavors: simple (no
-discriminator or base), flat (both discriminator and base).  A union
+discriminator or base), and flat (both discriminator and base).  A union
 type is defined using a data dictionary as explained in the following
-paragraphs.
+paragraphs.  The data dictionary for either type of union must not
+be empty.
 
 A simple union type defines a mapping from automatic discriminator
 values to data types like in this example:
 
- { 'struct': 'FileOptions', 'data': { 'filename': 'str' } }
- { 'struct': 'Qcow2Options',
-   'data': { 'backing-file': 'str', 'lazy-refcounts': 'bool' } }
+ { 'struct': 'BlockdevOptionsFile', 'data': { 'filename': 'str' } }
+ { 'struct': 'BlockdevOptionsQcow2',
+   'data': { 'backing': 'str', '*lazy-refcounts': 'bool' } }
 
- { 'union': 'BlockdevOptions',
-   'data': { 'file': 'FileOptions',
-             'qcow2': 'Qcow2Options' } }
+ { 'union': 'BlockdevOptionsSimple',
+   'data': { 'file': 'BlockdevOptionsFile',
+             'qcow2': 'BlockdevOptionsQcow2' } }
 
 In the Client JSON Protocol, a simple union is represented by a
-dictionary that contains the 'type' field as a discriminator, and a
-'data' field that is of the specified data type corresponding to the
+dictionary that contains the 'type' member as a discriminator, and a
+'data' member that is of the specified data type corresponding to the
 discriminator value, as in these examples:
 
- { "type": "file", "data" : { "filename": "/some/place/my-image" } }
- { "type": "qcow2", "data" : { "backing-file": "/some/place/my-image",
-                               "lazy-refcounts": true } }
+ { "type": "file", "data": { "filename": "/some/place/my-image" } }
+ { "type": "qcow2", "data": { "backing": "/some/place/my-image",
+                              "lazy-refcounts": true } }
 
 The generated C code uses a struct containing a union. Additionally,
 an implicit C enum 'NameKind' is created, corresponding to the union
@@ -300,43 +320,43 @@ an implicit C enum 'NameKind' is created, corresponding to the union
 the union can be named 'max', as this would collide with the implicit
 enum.  The value for each branch can be of any type.
 
-
-A flat union definition specifies a struct as its base, and
-avoids nesting on the wire.  All branches of the union must be
-complex types, and the top-level fields of the union dictionary on
-the wire will be combination of fields from both the base type and the
-appropriate branch type (when merging two dictionaries, there must be
-no keys in common).  The 'discriminator' field must be the name of an
-enum-typed member of the base struct.
+A flat union definition avoids nesting on the wire, and specifies a
+set of common members that occur in all variants of the union.  The
+'base' key must specifiy either a type name (the type must be a
+struct, not a union), or a dictionary representing an anonymous type.
+All branches of the union must be complex types, and the top-level
+members of the union dictionary on the wire will be combination of
+members from both the base type and the appropriate branch type (when
+merging two dictionaries, there must be no keys in common).  The
+'discriminator' member must be the name of a non-optional enum-typed
+member of the base struct.
 
 The following example enhances the above simple union example by
-adding a common field 'readonly', renaming the discriminator to
-something more applicable, and reducing the number of {} required on
-the wire:
+adding an optional common member 'read-only', renaming the
+discriminator to something more applicable than the simple union's
+default of 'type', and reducing the number of {} required on the wire:
 
- { 'enum': 'BlockdevDriver', 'data': [ 'raw', 'qcow2' ] }
- { 'struct': 'BlockdevCommonOptions',
-   'data': { 'driver': 'BlockdevDriver', 'readonly': 'bool' } }
+ { 'enum': 'BlockdevDriver', 'data': [ 'file', 'qcow2' ] }
  { 'union': 'BlockdevOptions',
-   'base': 'BlockdevCommonOptions',
+   'base': { 'driver': 'BlockdevDriver', '*read-only': 'bool' },
    'discriminator': 'driver',
-   'data': { 'file': 'FileOptions',
-             'qcow2': 'Qcow2Options' } }
+   'data': { 'file': 'BlockdevOptionsFile',
+             'qcow2': 'BlockdevOptionsQcow2' } }
 
 Resulting in these JSON objects:
 
- { "driver": "file", "readonly": true,
+ { "driver": "file", "read-only": true,
    "filename": "/some/place/my-image" }
- { "driver": "qcow2", "readonly": false,
-   "backing-file": "/some/place/my-image", "lazy-refcounts": true }
+ { "driver": "qcow2", "read-only": false,
+   "backing": "/some/place/my-image", "lazy-refcounts": true }
 
 Notice that in a flat union, the discriminator name is controlled by
 the user, but because it must map to a base member with enum type, the
 code generator can ensure that branches exist for all values of the
 enum (although the order of the keys need not match the declaration of
 the enum).  In the resulting generated C data types, a flat union is
-represented as a struct with the base member fields included directly,
-and then a union of structures for each branch of the struct.
+represented as a struct with the base members included directly, and
+then a union of structures for each branch of the struct.
 
 A simple union can always be re-written as a flat union where the base
 class has a single member named 'type', and where each branch of the
@@ -347,10 +367,9 @@ union has a struct with a single member named 'data'.  That is,
 is identical on the wire to:
 
  { 'enum': 'Enum', 'data': ['one', 'two'] }
- { 'struct': 'Base', 'data': { 'type': 'Enum' } }
  { 'struct': 'Branch1', 'data': { 'data': 'str' } }
  { 'struct': 'Branch2', 'data': { 'data': 'int' } }
- { 'union': 'Flat': 'base': 'Base', 'discriminator': 'type',
+ { 'union': 'Flat': 'base': { 'type': 'Enum' }, 'discriminator': 'type',
    'data': { 'one': 'Branch1', 'two': 'Branch2' } }
 
 
@@ -363,13 +382,10 @@ data types (string, integer, number, or object, but currently not
 array) on the wire.  The definition is similar to a simple union type,
 where each branch of the union names a QAPI type.  For example:
 
- { 'alternate': 'BlockRef',
+ { 'alternate': 'BlockdevRef',
    'data': { 'definition': 'BlockdevOptions',
              'reference': 'str' } }
 
-Just like for a simple union, an implicit C enum 'NameKind' is created
-to enumerate the branches for the alternate 'Name'.
-
 Unlike a union, the discriminator string is never passed on the wire
 for the Client JSON Protocol.  Instead, the value's JSON type serves
 as an implicit discriminator, which in turn means that an alternate
@@ -387,14 +403,14 @@ following example objects:
 
  { "file": "my_existing_block_device_id" }
  { "file": { "driver": "file",
-             "readonly": false,
+             "read-only": false,
              "filename": "/tmp/mydisk.qcow2" } }
 
 
 === Commands ===
 
 Usage: { 'command': STRING, '*data': COMPLEX-TYPE-NAME-OR-DICT,
-         '*returns': TYPE-NAME-OR-DICT,
+         '*returns': TYPE-NAME,
          '*gen': false, '*success-response': false }
 
 Commands are defined by using a dictionary containing several members,
@@ -405,25 +421,23 @@ Client JSON Protocol command exchange.
 The 'data' argument maps to the "arguments" dictionary passed in as
 part of a Client JSON Protocol command.  The 'data' member is optional
 and defaults to {} (an empty dictionary).  If present, it must be the
-string name of a complex type, a one-element array containing the name
-of a complex type, or a dictionary that declares an anonymous type
-with the same semantics as a 'struct' expression, with one exception
-noted below when 'gen' is used.
+string name of a complex type, or a dictionary that declares an
+anonymous type with the same semantics as a 'struct' expression, with
+one exception noted below when 'gen' is used.
 
-The 'returns' member describes what will appear in the "return" field
+The 'returns' member describes what will appear in the "return" member
 of a Client JSON Protocol reply on successful completion of a command.
 The member is optional from the command declaration; if absent, the
-"return" field will be an empty dictionary.  If 'returns' is present,
+"return" member will be an empty dictionary.  If 'returns' is present,
 it must be the string name of a complex or built-in type, a
 one-element array containing the name of a complex or built-in type,
-or a dictionary that declares an anonymous type with the same
-semantics as a 'struct' expression, with one exception noted below
-when 'gen' is used.  Although it is permitted to have the 'returns'
-member name a built-in type or an array of built-in types, any command
-that does this cannot be extended to return additional information in
-the future; thus, new commands should strongly consider returning a
-dictionary-based type or an array of dictionaries, even if the
-dictionary only contains one field at the present.
+with one exception noted below when 'gen' is used.  Although it is
+permitted to have the 'returns' member name a built-in type or an
+array of built-in types, any command that does this cannot be extended
+to return additional information in the future; thus, new commands
+should strongly consider returning a dictionary-based type or an array
+of dictionaries, even if the dictionary only contains one member at the
+present.
 
 All commands in Client JSON Protocol use a dictionary to report
 failure, with no way to specify that in QAPI.  Where the error return
@@ -448,17 +462,14 @@ which would validate this Client JSON Protocol transaction:
  <= { "return": [ { "value": "one" }, { } ] }
 
 In rare cases, QAPI cannot express a type-safe representation of a
-corresponding Client JSON Protocol command.  In these cases, if the
-command expression includes the key 'gen' with boolean value false,
-then the 'data' or 'returns' member that intends to bypass generated
-type-safety and do its own manual validation should use an inline
-dictionary definition, with a value of '**' rather than a valid type
-name for the keys that the generated code will not validate.  Please
-try to avoid adding new commands that rely on this, and instead use
-type-safe unions.  For an example of bypass usage:
+corresponding Client JSON Protocol command.  You then have to suppress
+generation of a marshalling function by including a key 'gen' with
+boolean value false, and instead write your own function.  Please try
+to avoid adding new commands that rely on this, and instead use
+type-safe unions.  For an example of this usage:
 
  { 'command': 'netdev_add',
-   'data': {'type': 'str', 'id': 'str', '*props': '**'},
+   'data': {'type': 'str', 'id': 'str'},
    'gen': false }
 
 Normally, the QAPI schema is used to describe synchronous exchanges,
@@ -468,7 +479,7 @@ response is not possible (although the command will still return a
 normal dictionary error on failure).  When a successful reply is not
 possible, the command expression should include the optional key
 'success-response' with boolean value false.  So far, only QGA makes
-use of this field.
+use of this member.
 
 
 === Events ===
@@ -495,34 +506,255 @@ Resulting in this JSON object:
   "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
 
+== Client JSON Protocol introspection ==
+
+Clients of a Client JSON Protocol commonly need to figure out what
+exactly the server (QEMU) supports.
+
+For this purpose, QMP provides introspection via command
+query-qmp-schema.  QGA currently doesn't support introspection.
+
+While Client JSON Protocol wire compatibility should be maintained
+between qemu versions, we cannot make the same guarantees for
+introspection stability.  For example, one version of qemu may provide
+a non-variant optional member of a struct, and a later version rework
+the member to instead be non-optional and associated with a variant.
+Likewise, one version of qemu may list a member with open-ended type
+'str', and a later version could convert it to a finite set of strings
+via an enum type; or a member may be converted from a specific type to
+an alternate that represents a choice between the original type and
+something else.
+
+query-qmp-schema returns a JSON array of SchemaInfo objects.  These
+objects together describe the wire ABI, as defined in the QAPI schema.
+There is no specified order to the SchemaInfo objects returned; a
+client must search for a particular name throughout the entire array
+to learn more about that name, but is at least guaranteed that there
+will be no collisions between type, command, and event names.
+
+However, the SchemaInfo can't reflect all the rules and restrictions
+that apply to QMP.  It's interface introspection (figuring out what's
+there), not interface specification.  The specification is in the QAPI
+schema.  To understand how QMP is to be used, you need to study the
+QAPI schema.
+
+Like any other command, query-qmp-schema is itself defined in the QAPI
+schema, along with the SchemaInfo type.  This text attempts to give an
+overview how things work.  For details you need to consult the QAPI
+schema.
+
+SchemaInfo objects have common members "name" and "meta-type", and
+additional variant members depending on the value of meta-type.
+
+Each SchemaInfo object describes a wire ABI entity of a certain
+meta-type: a command, event or one of several kinds of type.
+
+SchemaInfo for commands and events have the same name as in the QAPI
+schema.
+
+Command and event names are part of the wire ABI, but type names are
+not.  Therefore, the SchemaInfo for types have auto-generated
+meaningless names.  For readability, the examples in this section use
+meaningful type names instead.
+
+To examine a type, start with a command or event using it, then follow
+references by name.
+
+QAPI schema definitions not reachable that way are omitted.
+
+The SchemaInfo for a command has meta-type "command", and variant
+members "arg-type" and "ret-type".  On the wire, the "arguments"
+member of a client's "execute" command must conform to the object type
+named by "arg-type".  The "return" member that the server passes in a
+success response conforms to the type named by "ret-type".
+
+If the command takes no arguments, "arg-type" names an object type
+without members.  Likewise, if the command returns nothing, "ret-type"
+names an object type without members.
+
+Example: the SchemaInfo for command query-qmp-schema
+
+    { "name": "query-qmp-schema", "meta-type": "command",
+      "arg-type": "q_empty", "ret-type": "SchemaInfoList" }
+
+    Type "q_empty" is an automatic object type without members, and type
+    "SchemaInfoList" is the array of SchemaInfo type.
+
+The SchemaInfo for an event has meta-type "event", and variant member
+"arg-type".  On the wire, a "data" member that the server passes in an
+event conforms to the object type named by "arg-type".
+
+If the event carries no additional information, "arg-type" names an
+object type without members.  The event may not have a data member on
+the wire then.
+
+Each command or event defined with dictionary-valued 'data' in the
+QAPI schema implicitly defines an object type.
+
+Example: the SchemaInfo for EVENT_C from section Events
+
+    { "name": "EVENT_C", "meta-type": "event",
+      "arg-type": "q_obj-EVENT_C-arg" }
+
+    Type "q_obj-EVENT_C-arg" is an implicitly defined object type with
+    the two members from the event's definition.
+
+The SchemaInfo for struct and union types has meta-type "object".
+
+The SchemaInfo for a struct type has variant member "members".
+
+The SchemaInfo for a union type additionally has variant members "tag"
+and "variants".
+
+"members" is a JSON array describing the object's common members, if
+any.  Each element is a JSON object with members "name" (the member's
+name), "type" (the name of its type), and optionally "default".  The
+member is optional if "default" is present.  Currently, "default" can
+only have value null.  Other values are reserved for future
+extensions.  The "members" array is in no particular order; clients
+must search the entire object when learning whether a particular
+member is supported.
+
+Example: the SchemaInfo for MyType from section Struct types
+
+    { "name": "MyType", "meta-type": "object",
+      "members": [
+          { "name": "member1", "type": "str" },
+          { "name": "member2", "type": "int" },
+          { "name": "member3", "type": "str", "default": null } ] }
+
+"tag" is the name of the common member serving as type tag.
+"variants" is a JSON array describing the object's variant members.
+Each element is a JSON object with members "case" (the value of type
+tag this element applies to) and "type" (the name of an object type
+that provides the variant members for this type tag value).  The
+"variants" array is in no particular order, and is not guaranteed to
+list cases in the same order as the corresponding "tag" enum type.
+
+Example: the SchemaInfo for flat union BlockdevOptions from section
+Union types
+
+    { "name": "BlockdevOptions", "meta-type": "object",
+      "members": [
+          { "name": "driver", "type": "BlockdevDriver" },
+          { "name": "read-only", "type": "bool", "default": null } ],
+      "tag": "driver",
+      "variants": [
+          { "case": "file", "type": "BlockdevOptionsFile" },
+          { "case": "qcow2", "type": "BlockdevOptionsQcow2" } ] }
+
+Note that base types are "flattened": its members are included in the
+"members" array.
+
+A simple union implicitly defines an enumeration type for its implicit
+discriminator (called "type" on the wire, see section Union types).
+
+A simple union implicitly defines an object type for each of its
+variants.
+
+Example: the SchemaInfo for simple union BlockdevOptionsSimple from section
+Union types
+
+    { "name": "BlockdevOptionsSimple", "meta-type": "object",
+      "members": [
+          { "name": "type", "type": "BlockdevOptionsSimpleKind" } ],
+      "tag": "type",
+      "variants": [
+          { "case": "file", "type": "q_obj-BlockdevOptionsFile-wrapper" },
+          { "case": "qcow2", "type": "q_obj-BlockdevOptionsQcow2-wrapper" } ] }
+
+    Enumeration type "BlockdevOptionsSimpleKind" and the object types
+    "q_obj-BlockdevOptionsFile-wrapper", "q_obj-BlockdevOptionsQcow2-wrapper"
+    are implicitly defined.
+
+The SchemaInfo for an alternate type has meta-type "alternate", and
+variant member "members".  "members" is a JSON array.  Each element is
+a JSON object with member "type", which names a type.  Values of the
+alternate type conform to exactly one of its member types.  There is
+no guarantee on the order in which "members" will be listed.
+
+Example: the SchemaInfo for BlockdevRef from section Alternate types
+
+    { "name": "BlockdevRef", "meta-type": "alternate",
+      "members": [
+          { "type": "BlockdevOptions" },
+          { "type": "str" } ] }
+
+The SchemaInfo for an array type has meta-type "array", and variant
+member "element-type", which names the array's element type.  Array
+types are implicitly defined.  For convenience, the array's name may
+resemble the element type; however, clients should examine member
+"element-type" instead of making assumptions based on parsing member
+"name".
+
+Example: the SchemaInfo for ['str']
+
+    { "name": "[str]", "meta-type": "array",
+      "element-type": "str" }
+
+The SchemaInfo for an enumeration type has meta-type "enum" and
+variant member "values".  The values are listed in no particular
+order; clients must search the entire enum when learning whether a
+particular value is supported.
+
+Example: the SchemaInfo for MyEnum from section Enumeration types
+
+    { "name": "MyEnum", "meta-type": "enum",
+      "values": [ "value1", "value2", "value3" ] }
+
+The SchemaInfo for a built-in type has the same name as the type in
+the QAPI schema (see section Built-in Types), with one exception
+detailed below.  It has variant member "json-type" that shows how
+values of this type are encoded on the wire.
+
+Example: the SchemaInfo for str
+
+    { "name": "str", "meta-type": "builtin", "json-type": "string" }
+
+The QAPI schema supports a number of integer types that only differ in
+how they map to C.  They are identical as far as SchemaInfo is
+concerned.  Therefore, they get all mapped to a single type "int" in
+SchemaInfo.
+
+As explained above, type names are not part of the wire ABI.  Not even
+the names of built-in types.  Clients should examine member
+"json-type" instead of hard-coding names of built-in types.
+
+
 == Code generation ==
 
-Schemas are fed into 3 scripts to generate all the code/files that, paired
-with the core QAPI libraries, comprise everything required to take JSON
-commands read in by a Client JSON Protocol server, unmarshal the arguments into
-the underlying C types, call into the corresponding C function, and map the
-response back to a Client JSON Protocol response to be returned to the user.
+Schemas are fed into five scripts to generate all the code/files that,
+paired with the core QAPI libraries, comprise everything required to
+take JSON commands read in by a Client JSON Protocol server, unmarshal
+the arguments into the underlying C types, call into the corresponding
+C function, map the response back to a Client JSON Protocol response
+to be returned to the user, and introspect the commands.
 
-As an example, we'll use the following schema, which describes a single
-complex user-defined type (which will produce a C struct, along with a list
-node structure that can be used to chain together a list of such types in
-case we want to accept/return a list of this type with a command), and a
-command which takes that type as a parameter and returns the same type:
+As an example, we'll use the following schema, which describes a
+single complex user-defined type, along with command which takes a
+list of that type as a parameter, and returns a single element of that
+type.  The user is responsible for writing the implementation of
+qmp_my_command(); everything else is produced by the generator.
 
     $ cat example-schema.json
     { 'struct': 'UserDefOne',
-      'data': { 'integer': 'int', 'string': 'str' } }
+      'data': { 'integer': 'int', '*string': 'str' } }
 
     { 'command': 'my-command',
-      'data':    {'arg1': 'UserDefOne'},
+      'data': { 'arg1': ['UserDefOne'] },
       'returns': 'UserDefOne' }
 
     { 'event': 'MY_EVENT' }
 
+For a more thorough look at generated code, the testsuite includes
+tests/qapi-schema/qapi-schema-tests.json that covers more examples of
+what the generator will accept, and compiles the resulting C code as
+part of 'make check-unit'.
+
 === scripts/qapi-types.py ===
 
-Used to generate the C types defined by a schema. The following files are
-created:
+Used to generate the C types defined by a schema, along with
+supporting code. The following files are created:
 
 $(prefix)qapi-types.h - C types corresponding to types defined in
                         the schema you pass in
@@ -537,77 +769,73 @@ Example:
 
     $ python scripts/qapi-types.py --output-dir="qapi-generated" \
     --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qapi-types.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QAPI_TYPES_H
+    #define EXAMPLE_QAPI_TYPES_H
+
+[Built-in types omitted...]
+
+    typedef struct UserDefOne UserDefOne;
+
+    typedef struct UserDefOneList UserDefOneList;
+
+    struct UserDefOne {
+        int64_t integer;
+        bool has_string;
+        char *string;
+    };
+
+    void qapi_free_UserDefOne(UserDefOne *obj);
+
+    struct UserDefOneList {
+        UserDefOneList *next;
+        UserDefOne *value;
+    };
+
+    void qapi_free_UserDefOneList(UserDefOneList *obj);
+
+    #endif
     $ cat qapi-generated/example-qapi-types.c
 [Uninteresting stuff omitted...]
 
-    void qapi_free_UserDefOneList(UserDefOneList *obj)
+    void qapi_free_UserDefOne(UserDefOne *obj)
     {
-        QapiDeallocVisitor *md;
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
         if (!obj) {
             return;
         }
 
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOneList(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOne(v, NULL, &obj, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
-    void qapi_free_UserDefOne(UserDefOne *obj)
+    void qapi_free_UserDefOneList(UserDefOneList *obj)
     {
-        QapiDeallocVisitor *md;
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
         if (!obj) {
             return;
         }
 
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOneList(v, NULL, &obj, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
-    $ cat qapi-generated/example-qapi-types.h
-[Uninteresting stuff omitted...]
-
-    #ifndef EXAMPLE_QAPI_TYPES_H
-    #define EXAMPLE_QAPI_TYPES_H
-
-[Built-in types omitted...]
-
-    typedef struct UserDefOne UserDefOne;
-
-    typedef struct UserDefOneList
-    {
-        union {
-            UserDefOne *value;
-            uint64_t padding;
-        };
-        struct UserDefOneList *next;
-    } UserDefOneList;
-
-[Functions on built-in types omitted...]
-
-    struct UserDefOne
-    {
-        int64_t integer;
-        char *string;
-    };
-
-    void qapi_free_UserDefOneList(UserDefOneList *obj);
-    void qapi_free_UserDefOne(UserDefOne *obj);
-
-    #endif
-
 === scripts/qapi-visit.py ===
 
-Used to generate the visitor functions used to walk through and convert
-a QObject (as provided by QMP) to a native C data structure and
-vice-versa, as well as the visitor function used to dealloc a complex
-schema-defined C type.
+Used to generate the visitor functions used to walk through and
+convert between a native QAPI C data structure and some other format
+(such as QObject); the generated functions are named visit_type_FOO()
+and visit_type_FOO_members().
 
 The following files are generated:
 
@@ -624,79 +852,90 @@ Example:
 
     $ python scripts/qapi-visit.py --output-dir="qapi-generated"
     --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qapi-visit.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QAPI_VISIT_H
+    #define EXAMPLE_QAPI_VISIT_H
+
+[Visitors for built-in types omitted...]
+
+    void visit_type_UserDefOne_members(Visitor *v, UserDefOne *obj, Error **errp);
+    void visit_type_UserDefOne(Visitor *v, const char *name, UserDefOne **obj, Error **errp);
+    void visit_type_UserDefOneList(Visitor *v, const char *name, UserDefOneList **obj, Error **errp);
+
+    #endif
     $ cat qapi-generated/example-qapi-visit.c
 [Uninteresting stuff omitted...]
 
-    static void visit_type_UserDefOne_fields(Visitor *m, UserDefOne **obj, Error **errp)
+    void visit_type_UserDefOne_members(Visitor *v, UserDefOne *obj, Error **errp)
     {
         Error *err = NULL;
-        visit_type_int(m, &(*obj)->integer, "integer", &err);
+
+        visit_type_int(v, "integer", &obj->integer, &err);
         if (err) {
             goto out;
         }
-        visit_type_str(m, &(*obj)->string, "string", &err);
-        if (err) {
-            goto out;
+        if (visit_optional(v, "string", &obj->has_string)) {
+            visit_type_str(v, "string", &obj->string, &err);
+            if (err) {
+                goto out;
+            }
         }
 
     out:
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp)
+    void visit_type_UserDefOne(Visitor *v, const char *name, UserDefOne **obj, Error **errp)
     {
         Error *err = NULL;
 
-        visit_start_struct(m, (void **)obj, "UserDefOne", name, sizeof(UserDefOne), &err);
-        if (!err) {
-            if (*obj) {
-                visit_type_UserDefOne_fields(m, obj, errp);
-            }
-            visit_end_struct(m, &err);
+        visit_start_struct(v, name, (void **)obj, sizeof(UserDefOne), &err);
+        if (err) {
+            goto out;
         }
+        if (!*obj) {
+            goto out_obj;
+        }
+        visit_type_UserDefOne_members(v, *obj, &err);
+        error_propagate(errp, err);
+        err = NULL;
+    out_obj:
+        visit_end_struct(v, &err);
+    out:
         error_propagate(errp, err);
     }
 
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp)
+    void visit_type_UserDefOneList(Visitor *v, const char *name, UserDefOneList **obj, Error **errp)
     {
         Error *err = NULL;
         GenericList *i, **prev;
 
-        visit_start_list(m, name, &err);
+        visit_start_list(v, name, &err);
         if (err) {
             goto out;
         }
 
         for (prev = (GenericList **)obj;
-             !err && (i = visit_next_list(m, prev, &err)) != NULL;
+             !err && (i = visit_next_list(v, prev, sizeof(**obj))) != NULL;
              prev = &i) {
             UserDefOneList *native_i = (UserDefOneList *)i;
-            visit_type_UserDefOne(m, &native_i->value, NULL, &err);
+            visit_type_UserDefOne(v, NULL, &native_i->value, &err);
         }
 
-        error_propagate(errp, err);
-        err = NULL;
-        visit_end_list(m, &err);
+        visit_end_list(v);
     out:
         error_propagate(errp, err);
     }
-    $ cat qapi-generated/example-qapi-visit.h
-[Uninteresting stuff omitted...]
-
-    #ifndef EXAMPLE_QAPI_VISIT_H
-    #define EXAMPLE_QAPI_VISIT_H
-
-[Visitors for built-in types omitted...]
-
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp);
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp);
-
-    #endif
 
 === scripts/qapi-commands.py ===
 
-Used to generate the marshaling/dispatch functions for the commands defined
-in the schema. The following files are generated:
+Used to generate the marshaling/dispatch functions for the commands
+defined in the schema. The generated code implements
+qmp_marshal_COMMAND() (mentioned in qmp-commands.hx, and registered
+automatically), and declares qmp_COMMAND() that the user must
+implement.  The following files are generated:
 
 $(prefix)qmp-marshal.c: command marshal/dispatch functions for each
                         QMP command defined in the schema. Functions
@@ -714,88 +953,88 @@ Example:
 
     $ python scripts/qapi-commands.py --output-dir="qapi-generated"
     --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qmp-commands.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QMP_COMMANDS_H
+    #define EXAMPLE_QMP_COMMANDS_H
+
+    #include "example-qapi-types.h"
+    #include "qapi/qmp/qdict.h"
+    #include "qapi/error.h"
+
+    UserDefOne *qmp_my_command(UserDefOneList *arg1, Error **errp);
+
+    #endif
     $ cat qapi-generated/example-qmp-marshal.c
 [Uninteresting stuff omitted...]
 
-    static void qmp_marshal_output_my_command(UserDefOne *ret_in, QObject **ret_out, Error **errp)
+    static void qmp_marshal_output_UserDefOne(UserDefOne *ret_in, QObject **ret_out, Error **errp)
     {
-        Error *local_err = NULL;
-        QmpOutputVisitor *mo = qmp_output_visitor_new();
-        QapiDeallocVisitor *md;
+        Error *err = NULL;
+        QmpOutputVisitor *qov = qmp_output_visitor_new();
+        QapiDeallocVisitor *qdv;
         Visitor *v;
 
-        v = qmp_output_get_visitor(mo);
-        visit_type_UserDefOne(v, &ret_in, "unused", &local_err);
-        if (local_err) {
+        v = qmp_output_get_visitor(qov);
+        visit_type_UserDefOne(v, "unused", &ret_in, &err);
+        if (err) {
             goto out;
         }
-        *ret_out = qmp_output_get_qobject(mo);
+        *ret_out = qmp_output_get_qobject(qov);
 
     out:
-        error_propagate(errp, local_err);
-        qmp_output_visitor_cleanup(mo);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &ret_in, "unused", NULL);
-        qapi_dealloc_visitor_cleanup(md);
+        error_propagate(errp, err);
+        qmp_output_visitor_cleanup(qov);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOne(v, "unused", &ret_in, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
-    static void qmp_marshal_input_my_command(QDict *args, QObject **ret, Error **errp)
+    static void qmp_marshal_my_command(QDict *args, QObject **ret, Error **errp)
     {
-        Error *local_err = NULL;
-        UserDefOne *retval = NULL;
-        QmpInputVisitor *mi = qmp_input_visitor_new_strict(QOBJECT(args));
-        QapiDeallocVisitor *md;
+        Error *err = NULL;
+        UserDefOne *retval;
+        QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args));
+        QapiDeallocVisitor *qdv;
         Visitor *v;
-        UserDefOne *arg1 = NULL;
+        UserDefOneList *arg1 = NULL;
 
-        v = qmp_input_get_visitor(mi);
-        visit_type_UserDefOne(v, &arg1, "arg1", &local_err);
-        if (local_err) {
+        v = qmp_input_get_visitor(qiv);
+        visit_type_UserDefOneList(v, "arg1", &arg1, &err);
+        if (err) {
             goto out;
         }
 
-        retval = qmp_my_command(arg1, &local_err);
-        if (local_err) {
+        retval = qmp_my_command(arg1, &err);
+        if (err) {
             goto out;
         }
 
-        qmp_marshal_output_my_command(retval, ret, &local_err);
+        qmp_marshal_output_UserDefOne(retval, ret, &err);
 
     out:
-        error_propagate(errp, local_err);
-        qmp_input_visitor_cleanup(mi);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &arg1, "arg1", NULL);
-        qapi_dealloc_visitor_cleanup(md);
-        return;
+        error_propagate(errp, err);
+        qmp_input_visitor_cleanup(qiv);
+        qdv = qapi_dealloc_visitor_new();
+        v = qapi_dealloc_get_visitor(qdv);
+        visit_type_UserDefOneList(v, "arg1", &arg1, NULL);
+        qapi_dealloc_visitor_cleanup(qdv);
     }
 
     static void qmp_init_marshal(void)
     {
-        qmp_register_command("my-command", qmp_marshal_input_my_command, QCO_NO_OPTIONS);
+        qmp_register_command("my-command", qmp_marshal_my_command, QCO_NO_OPTIONS);
     }
 
     qapi_init(qmp_init_marshal);
-    $ cat qapi-generated/example-qmp-commands.h
-[Uninteresting stuff omitted...]
-
-    #ifndef EXAMPLE_QMP_COMMANDS_H
-    #define EXAMPLE_QMP_COMMANDS_H
-
-    #include "example-qapi-types.h"
-    #include "qapi/qmp/qdict.h"
-    #include "qapi/error.h"
-
-    UserDefOne *qmp_my_command(UserDefOne *arg1, Error **errp);
-
-    #endif
 
 === scripts/qapi-event.py ===
 
-Used to generate the event-related C code defined by a schema. The
-following files are created:
+Used to generate the event-related C code defined by a schema, with
+implementations for qapi_event_send_FOO(). The following files are
+created:
 
 $(prefix)qapi-event.h - Function prototypes for each event type, plus an
                         enumeration of all event names
@@ -805,13 +1044,34 @@ Example:
 
     $ python scripts/qapi-event.py --output-dir="qapi-generated"
     --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qapi-event.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QAPI_EVENT_H
+    #define EXAMPLE_QAPI_EVENT_H
+
+    #include "qapi/error.h"
+    #include "qapi/qmp/qdict.h"
+    #include "example-qapi-types.h"
+
+
+    void qapi_event_send_my_event(Error **errp);
+
+    typedef enum example_QAPIEvent {
+        EXAMPLE_QAPI_EVENT_MY_EVENT = 0,
+        EXAMPLE_QAPI_EVENT__MAX = 1,
+    } example_QAPIEvent;
+
+    extern const char *const example_QAPIEvent_lookup[];
+
+    #endif
     $ cat qapi-generated/example-qapi-event.c
 [Uninteresting stuff omitted...]
 
     void qapi_event_send_my_event(Error **errp)
     {
         QDict *qmp;
-        Error *local_err = NULL;
+        Error *err = NULL;
         QMPEventFuncEmit emit;
         emit = qmp_event_get_func_emit();
         if (!emit) {
@@ -820,34 +1080,48 @@ Example:
 
         qmp = qmp_event_build_dict("MY_EVENT");
 
-        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &local_err);
+        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &err);
 
-        error_propagate(errp, local_err);
+        error_propagate(errp, err);
         QDECREF(qmp);
     }
 
-    const char *EXAMPLE_QAPIEvent_lookup[] = {
-        "MY_EVENT",
-        NULL,
+    const char *const example_QAPIEvent_lookup[] = {
+        [EXAMPLE_QAPI_EVENT_MY_EVENT] = "MY_EVENT",
+        [EXAMPLE_QAPI_EVENT__MAX] = NULL,
     };
-    $ cat qapi-generated/example-qapi-event.h
-[Uninteresting stuff omitted...]
 
-    #ifndef EXAMPLE_QAPI_EVENT_H
-    #define EXAMPLE_QAPI_EVENT_H
+=== scripts/qapi-introspect.py ===
 
-    #include "qapi/error.h"
-    #include "qapi/qmp/qdict.h"
-    #include "example-qapi-types.h"
+Used to generate the introspection C code for a schema. The following
+files are created:
 
+$(prefix)qmp-introspect.c - Defines a string holding a JSON
+                            description of the schema.
+$(prefix)qmp-introspect.h - Declares the above string.
 
-    void qapi_event_send_my_event(Error **errp);
+Example:
 
-    extern const char *EXAMPLE_QAPIEvent_lookup[];
-    typedef enum EXAMPLE_QAPIEvent
-    {
-        EXAMPLE_QAPI_EVENT_MY_EVENT = 0,
-        EXAMPLE_QAPI_EVENT_MAX = 1,
-    } EXAMPLE_QAPIEvent;
+    $ python scripts/qapi-introspect.py --output-dir="qapi-generated"
+    --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qmp-introspect.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QMP_INTROSPECT_H
+    #define EXAMPLE_QMP_INTROSPECT_H
+
+    extern const char example_qmp_schema_json[];
 
     #endif
+    $ cat qapi-generated/example-qmp-introspect.c
+[Uninteresting stuff omitted...]
+
+    const char example_qmp_schema_json[] = "["
+        "{\"arg-type\": \"0\", \"meta-type\": \"event\", \"name\": \"MY_EVENT\"}, "
+        "{\"arg-type\": \"1\", \"meta-type\": \"command\", \"name\": \"my-command\", \"ret-type\": \"2\"}, "
+        "{\"members\": [], \"meta-type\": \"object\", \"name\": \"0\"}, "
+        "{\"members\": [{\"name\": \"arg1\", \"type\": \"[2]\"}], \"meta-type\": \"object\", \"name\": \"1\"}, "
+        "{\"members\": [{\"name\": \"integer\", \"type\": \"int\"}, {\"default\": null, \"name\": \"string\", \"type\": \"str\"}], \"meta-type\": \"object\", \"name\": \"2\"}, "
+        "{\"element-type\": \"2\", \"meta-type\": \"array\", \"name\": \"[2]\"}, "
+        "{\"json-type\": \"int\", \"meta-type\": \"builtin\", \"name\": \"int\"}, "
+        "{\"json-type\": \"string\", \"meta-type\": \"builtin\", \"name\": \"str\"}]";
diff --git a/qemu/docs/qcow2-cache.txt b/qemu/docs/qcow2-cache.txt
new file mode 100644
index 000000000..5bb06072d
--- /dev/null
+++ b/qemu/docs/qcow2-cache.txt
@@ -0,0 +1,164 @@
+qcow2 L2/refcount cache configuration
+=====================================
+Copyright (C) 2015 Igalia, S.L.
+Author: Alberto Garcia <berto@igalia.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+Introduction
+------------
+The QEMU qcow2 driver has two caches that can improve the I/O
+performance significantly. However, setting the right cache sizes is
+not a straightforward operation.
+
+This document attempts to give an overview of the L2 and refcount
+caches, and how to configure them.
+
+Please refer to the docs/specs/qcow2.txt file for an in-depth
+technical description of the qcow2 file format.
+
+
+Clusters
+--------
+A qcow2 file is organized in units of constant size called clusters.
+
+The cluster size is configurable, but it must be a power of two and
+its value 512 bytes or higher. QEMU currently defaults to 64 KB
+clusters, and it does not support sizes larger than 2MB.
+
+The 'qemu-img create' command supports specifying the size using the
+cluster_size option:
+
+   qemu-img create -f qcow2 -o cluster_size=128K hd.qcow2 4G
+
+
+The L2 tables
+-------------
+The qcow2 format uses a two-level structure to map the virtual disk as
+seen by the guest to the disk image in the host. These structures are
+called the L1 and L2 tables.
+
+There is one single L1 table per disk image. The table is small and is
+always kept in memory.
+
+There can be many L2 tables, depending on how much space has been
+allocated in the image. Each table is one cluster in size. In order to
+read or write data from the virtual disk, QEMU needs to read its
+corresponding L2 table to find out where that data is located. Since
+reading the table for each I/O operation can be expensive, QEMU keeps
+an L2 cache in memory to speed up disk access.
+
+The size of the L2 cache can be configured, and setting the right
+value can improve the I/O performance significantly.
+
+
+The refcount blocks
+-------------------
+The qcow2 format also mantains a reference count for each cluster.
+Reference counts are used for cluster allocation and internal
+snapshots. The data is stored in a two-level structure similar to the
+L1/L2 tables described above.
+
+The second level structures are called refcount blocks, are also one
+cluster in size and the number is also variable and dependent on the
+amount of allocated space.
+
+Each block contains a number of refcount entries. Their size (in bits)
+is a power of two and must not be higher than 64. It defaults to 16
+bits, but a different value can be set using the refcount_bits option:
+
+   qemu-img create -f qcow2 -o refcount_bits=8 hd.qcow2 4G
+
+QEMU keeps a refcount cache to speed up I/O much like the
+aforementioned L2 cache, and its size can also be configured.
+
+
+Choosing the right cache sizes
+------------------------------
+In order to choose the cache sizes we need to know how they relate to
+the amount of allocated space.
+
+The amount of virtual disk that can be mapped by the L2 and refcount
+caches (in bytes) is:
+
+   disk_size = l2_cache_size * cluster_size / 8
+   disk_size = refcount_cache_size * cluster_size * 8 / refcount_bits
+
+With the default values for cluster_size (64KB) and refcount_bits
+(16), that is
+
+   disk_size = l2_cache_size * 8192
+   disk_size = refcount_cache_size * 32768
+
+So in order to cover n GB of disk space with the default values we
+need:
+
+   l2_cache_size = disk_size_GB * 131072
+   refcount_cache_size = disk_size_GB * 32768
+
+QEMU has a default L2 cache of 1MB (1048576 bytes) and a refcount
+cache of 256KB (262144 bytes), so using the formulas we've just seen
+we have
+
+   1048576 / 131072 = 8 GB of virtual disk covered by that cache
+    262144 /  32768 = 8 GB
+
+
+How to configure the cache sizes
+--------------------------------
+Cache sizes can be configured using the -drive option in the
+command-line, or the 'blockdev-add' QMP command.
+
+There are three options available, and all of them take bytes:
+
+"l2-cache-size":         maximum size of the L2 table cache
+"refcount-cache-size":   maximum size of the refcount block cache
+"cache-size":            maximum size of both caches combined
+
+There are two things that need to be taken into account:
+
+ - Both caches must have a size that is a multiple of the cluster
+   size.
+
+ - If you only set one of the options above, QEMU will automatically
+   adjust the others so that the L2 cache is 4 times bigger than the
+   refcount cache.
+
+This means that these options are equivalent:
+
+   -drive file=hd.qcow2,l2-cache-size=2097152
+   -drive file=hd.qcow2,refcount-cache-size=524288
+   -drive file=hd.qcow2,cache-size=2621440
+
+The reason for this 1/4 ratio is to ensure that both caches cover the
+same amount of disk space. Note however that this is only valid with
+the default value of refcount_bits (16). If you are using a different
+value you might want to calculate both cache sizes yourself since QEMU
+will always use the same 1/4 ratio.
+
+It's also worth mentioning that there's no strict need for both caches
+to cover the same amount of disk space. The refcount cache is used
+much less often than the L2 cache, so it's perfectly reasonable to
+keep it small.
+
+
+Reducing the memory usage
+-------------------------
+It is possible to clean unused cache entries in order to reduce the
+memory usage during periods of low I/O activity.
+
+The parameter "cache-clean-interval" defines an interval (in seconds).
+All cache entries that haven't been accessed during that interval are
+removed from memory.
+
+This example removes all unused cache entries every 15 minutes:
+
+   -drive file=hd.qcow2,cache-clean-interval=900
+
+If unset, the default value for this parameter is 0 and it disables
+this feature.
+
+Note that this functionality currently relies on the MADV_DONTNEED
+argument for madvise() to actually free the memory, so it is not
+useful in systems that don't follow that behavior.
diff --git a/qemu/docs/qmp/qmp-events.txt b/qemu/docs/qmp-events.txt
index d92cc4833..fa7574d67 100644
--- a/qemu/docs/qmp/qmp-events.txt
+++ b/qemu/docs/qmp-events.txt
@@ -28,6 +28,8 @@ Example:
     "data": { "actual": 944766976 },
     "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+Note: this event is rate-limited.
+
 BLOCK_IMAGE_CORRUPTED
 ---------------------
 
@@ -218,6 +220,24 @@ Data:
   },
   "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
 
+DUMP_COMPLETED
+--------------
+
+Emitted when the guest has finished one memory dump.
+
+Data:
+
+- "result": DumpQueryResult type described in qapi-schema.json
+- "error": Error message when dump failed. This is only a
+  human-readable string provided when dump failed. It should not be
+  parsed in any way (json-string, optional)
+
+Example:
+
+{ "event": "DUMP_COMPLETED",
+  "data": {"result": {"total": 1090650112, "status": "completed",
+                      "completed": 1090650112} } }
+
 GUEST_PANICKED
 --------------
 
@@ -296,6 +316,8 @@ Example:
      "data": { "reference": "usr1", "sector-num": 345435, "sectors-count": 5 },
      "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
 
+Note: this event is rate-limited.
+
 QUORUM_REPORT_BAD
 -----------------
 
@@ -303,6 +325,7 @@ Emitted to report a corruption of a Quorum file.
 
 Data:
 
+- "type":          Quorum operation type
 - "error":         Error message (json-string, optional)
                    Only present on failure.  This field contains a human-readable
                    error message.  There are no semantics other than that the
@@ -314,10 +337,20 @@ Data:
 
 Example:
 
+Read operation:
 { "event": "QUORUM_REPORT_BAD",
-     "data": { "node-name": "1.raw", "sector-num": 345435, "sectors-count": 5 },
+     "data": { "node-name": "node0", "sector-num": 345435, "sectors-count": 5,
+               "type": "read" },
      "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
 
+Flush operation:
+{ "event": "QUORUM_REPORT_BAD",
+     "data": { "node-name": "node0", "sector-num": 0, "sectors-count": 2097120,
+               "type": "flush", "error": "Broken pipe" },
+     "timestamp": { "seconds": 1456406829, "microseconds": 291763 } }
+
+Note: this event is rate-limited.
+
 RESET
 -----
 
@@ -358,6 +391,8 @@ Example:
     "data": { "offset": 78 },
     "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+Note: this event is rate-limited.
+
 SHUTDOWN
 --------
 
@@ -488,6 +523,20 @@ Example:
 {"timestamp": {"seconds": 1432121972, "microseconds": 744001},
  "event": "MIGRATION", "data": {"status": "completed"}}
 
+MIGRATION_PASS
+--------------
+
+Emitted from the source side of a migration at the start of each pass
+(when it syncs the dirty bitmap)
+
+Data: None.
+
+  - "pass": An incrementing count (starting at 1 on the first pass)
+
+Example:
+{"timestamp": {"seconds": 1449669631, "microseconds": 239225},
+ "event": "MIGRATION_PASS", "data": {"pass": 2}}
+
 STOP
 ----
 
@@ -632,6 +681,8 @@ Example:
     "data": { "id": "channel0", "open": true },
     "timestamp": { "seconds": 1401385907, "microseconds": 422329 } }
 
+Note: this event is rate-limited separately for each "id".
+
 WAKEUP
 ------
 
@@ -662,3 +713,5 @@ Example:
 
 Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is
 followed respectively by the RESET, SHUTDOWN, or STOP events.
+
+Note: this event is rate-limited.
diff --git a/qemu/docs/qmp/README b/qemu/docs/qmp-intro.txt
index f6a3a031e..f6a3a031e 100644
--- a/qemu/docs/qmp/README
+++ b/qemu/docs/qmp-intro.txt
diff --git a/qemu/docs/qmp/qmp-spec.txt b/qemu/docs/qmp-spec.txt
index 4c28cd943..f8b535601 100644
--- a/qemu/docs/qmp/qmp-spec.txt
+++ b/qemu/docs/qmp-spec.txt
@@ -3,7 +3,7 @@
 0. About This Document
 ======================
 
-Copyright (C) 2009-2015 Red Hat, Inc.
+Copyright (C) 2009-2016 Red Hat, Inc.
 
 This work is licensed under the terms of the GNU GPL, version 2 or
 later. See the COPYING file in the top-level directory.
@@ -175,7 +175,12 @@ The format of asynchronous events is:
 For a listing of supported asynchronous events, please, refer to the
 qmp-events.txt file.
 
-2.5 QGA Synchronization
+Some events are rate-limited to at most one per second.  If additional
+"similar" events arrive within one second, all but the last one are
+dropped, and the last one is delayed.  "Similar" normally means same
+event type.  See qmp-events.txt for details.
+
+2.6 QGA Synchronization
 -----------------------
 
 When using QGA, an additional synchronization feature is built into
@@ -272,7 +277,7 @@ However, Clients must not assume any particular:
 - Amount of errors generated by a command, that is, new errors can be added
   to any existing command in newer versions of the Server
 
-Any command or field name beginning with "x-" is deemed experimental,
+Any command or member name beginning with "x-" is deemed experimental,
 and may be withdrawn or changed in an incompatible manner in a future
 release.
 
diff --git a/qemu/docs/rcu.txt b/qemu/docs/rcu.txt
index 21ecb8106..2f70954e8 100644
--- a/qemu/docs/rcu.txt
+++ b/qemu/docs/rcu.txt
@@ -128,7 +128,7 @@ The core RCU API is small:
         the callback function is g_free, in particular, g_free_rcu can be
         used.  In the above case, one could have written simply:
 
-            g_free_rcu(foo_reclaim, rcu);
+            g_free_rcu(&foo, rcu);
 
      typeof(*p) atomic_rcu_read(p);
 
diff --git a/qemu/docs/replay.txt b/qemu/docs/replay.txt
new file mode 100644
index 000000000..779c6c059
--- /dev/null
+++ b/qemu/docs/replay.txt
@@ -0,0 +1,197 @@
+Copyright (c) 2010-2015 Institute for System Programming
+                        of the Russian Academy of Sciences.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.
+See the COPYING file in the top-level directory.
+
+Record/replay
+-------------
+
+Record/replay functions are used for the reverse execution and deterministic
+replay of qemu execution. This implementation of deterministic replay can
+be used for deterministic debugging of guest code through a gdb remote
+interface.
+
+Execution recording writes a non-deterministic events log, which can be later
+used for replaying the execution anywhere and for unlimited number of times.
+It also supports checkpointing for faster rewinding during reverse debugging.
+Execution replaying reads the log and replays all non-deterministic events
+including external input, hardware clocks, and interrupts.
+
+Deterministic replay has the following features:
+ * Deterministically replays whole system execution and all contents of
+   the memory, state of the hardware devices, clocks, and screen of the VM.
+ * Writes execution log into the file for later replaying for multiple times
+   on different machines.
+ * Supports i386, x86_64, and ARM hardware platforms.
+ * Performs deterministic replay of all operations with keyboard and mouse
+   input devices.
+
+Usage of the record/replay:
+ * First, record the execution, by adding the following arguments to the command line:
+   '-icount shift=7,rr=record,rrfile=replay.bin -net none'.
+   Block devices' images are not actually changed in the recording mode,
+   because all of the changes are written to the temporary overlay file.
+ * Then you can replay it by using another command
+   line option: '-icount shift=7,rr=replay,rrfile=replay.bin -net none'
+ * '-net none' option should also be specified if network replay patches
+   are not applied.
+
+Papers with description of deterministic replay implementation:
+http://www.computer.org/csdl/proceedings/csmr/2012/4666/00/4666a553-abs.html
+http://dl.acm.org/citation.cfm?id=2786805.2803179
+
+Modifications of qemu include:
+ * wrappers for clock and time functions to save their return values in the log
+ * saving different asynchronous events (e.g. system shutdown) into the log
+ * synchronization of the bottom halves execution
+ * synchronization of the threads from thread pool
+ * recording/replaying user input (mouse and keyboard)
+ * adding internal checkpoints for cpu and io synchronization
+
+Non-deterministic events
+------------------------
+
+Our record/replay system is based on saving and replaying non-deterministic
+events (e.g. keyboard input) and simulating deterministic ones (e.g. reading
+from HDD or memory of the VM). Saving only non-deterministic events makes
+log file smaller, simulation faster, and allows using reverse debugging even
+for realtime applications.
+
+The following non-deterministic data from peripheral devices is saved into
+the log: mouse and keyboard input, network packets, audio controller input,
+USB packets, serial port input, and hardware clocks (they are non-deterministic
+too, because their values are taken from the host machine). Inputs from
+simulated hardware, memory of VM, software interrupts, and execution of
+instructions are not saved into the log, because they are deterministic and
+can be replayed by simulating the behavior of virtual machine starting from
+initial state.
+
+We had to solve three tasks to implement deterministic replay: recording
+non-deterministic events, replaying non-deterministic events, and checking
+that there is no divergence between record and replay modes.
+
+We changed several parts of QEMU to make event log recording and replaying.
+Devices' models that have non-deterministic input from external devices were
+changed to write every external event into the execution log immediately.
+E.g. network packets are written into the log when they arrive into the virtual
+network adapter.
+
+All non-deterministic events are coming from these devices. But to
+replay them we need to know at which moments they occur. We specify
+these moments by counting the number of instructions executed between
+every pair of consecutive events.
+
+Instruction counting
+--------------------
+
+QEMU should work in icount mode to use record/replay feature. icount was
+designed to allow deterministic execution in absence of external inputs
+of the virtual machine. We also use icount to control the occurrence of the
+non-deterministic events. The number of instructions elapsed from the last event
+is written to the log while recording the execution. In replay mode we
+can predict when to inject that event using the instruction counter.
+
+Timers
+------
+
+Timers are used to execute callbacks from different subsystems of QEMU
+at the specified moments of time. There are several kinds of timers:
+ * Real time clock. Based on host time and used only for callbacks that
+   do not change the virtual machine state. For this reason real time
+   clock and timers does not affect deterministic replay at all.
+ * Virtual clock. These timers run only during the emulation. In icount
+   mode virtual clock value is calculated using executed instructions counter.
+   That is why it is completely deterministic and does not have to be recorded.
+ * Host clock. This clock is used by device models that simulate real time
+   sources (e.g. real time clock chip). Host clock is the one of the sources
+   of non-determinism. Host clock read operations should be logged to
+   make the execution deterministic.
+ * Virtual real time clock. This clock is similar to real time clock but
+   it is used only for increasing virtual clock while virtual machine is
+   sleeping. Due to its nature it is also non-deterministic as the host clock
+   and has to be logged too.
+
+Checkpoints
+-----------
+
+Replaying of the execution of virtual machine is bound by sources of
+non-determinism. These are inputs from clock and peripheral devices,
+and QEMU thread scheduling. Thread scheduling affect on processing events
+from timers, asynchronous input-output, and bottom halves.
+
+Invocations of timers are coupled with clock reads and changing the state
+of the virtual machine. Reads produce non-deterministic data taken from
+host clock. And VM state changes should preserve their order. Their relative
+order in replay mode must replicate the order of callbacks in record mode.
+To preserve this order we use checkpoints. When a specific clock is processed
+in record mode we save to the log special "checkpoint" event.
+Checkpoints here do not refer to virtual machine snapshots. They are just
+record/replay events used for synchronization.
+
+QEMU in replay mode will try to invoke timers processing in random moment
+of time. That's why we do not process a group of timers until the checkpoint
+event will be read from the log. Such an event allows synchronizing CPU
+execution and timer events.
+
+Two other checkpoints govern the "warping" of the virtual clock.
+While the virtual machine is idle, the virtual clock increments at
+1 ns per *real time* nanosecond.  This is done by setting up a timer
+(called the warp timer) on the virtual real time clock, so that the
+timer fires at the next deadline of the virtual clock; the virtual clock
+is then incremented (which is called "warping" the virtual clock) as
+soon as the timer fires or the CPUs need to go out of the idle state.
+Two functions are used for this purpose; because these actions change
+virtual machine state and must be deterministic, each of them creates a
+checkpoint.  qemu_start_warp_timer checks if the CPUs are idle and if so
+starts accounting real time to virtual clock.  qemu_account_warp_timer
+is called when the CPUs get an interrupt or when the warp timer fires,
+and it warps the virtual clock by the amount of real time that has passed
+since qemu_start_warp_timer.
+
+Bottom halves
+-------------
+
+Disk I/O events are completely deterministic in our model, because
+in both record and replay modes we start virtual machine from the same
+disk state. But callbacks that virtual disk controller uses for reading and
+writing the disk may occur at different moments of time in record and replay
+modes.
+
+Reading and writing requests are created by CPU thread of QEMU. Later these
+requests proceed to block layer which creates "bottom halves". Bottom
+halves consist of callback and its parameters. They are processed when
+main loop locks the global mutex. These locks are not synchronized with
+replaying process because main loop also processes the events that do not
+affect the virtual machine state (like user interaction with monitor).
+
+That is why we had to implement saving and replaying bottom halves callbacks
+synchronously to the CPU execution. When the callback is about to execute
+it is added to the queue in the replay module. This queue is written to the
+log when its callbacks are executed. In replay mode callbacks are not processed
+until the corresponding event is read from the events log file.
+
+Sometimes the block layer uses asynchronous callbacks for its internal purposes
+(like reading or writing VM snapshots or disk image cluster tables). In this
+case bottom halves are not marked as "replayable" and do not saved
+into the log.
+
+Block devices
+-------------
+
+Block devices record/replay module intercepts calls of
+bdrv coroutine functions at the top of block drivers stack.
+To record and replay block operations the drive must be configured
+as following:
+ -drive file=disk.qcow,if=none,id=img-direct
+ -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay
+ -device ide-hd,drive=img-blkreplay
+
+blkreplay driver should be inserted between disk image and virtual driver
+controller. Therefore all disk requests may be recorded and replayed.
+
+All block completion operations are added to the queue in the coroutines.
+Queue is flushed at checkpoints and information about processed requests
+is recorded to the log. In replay phase the queue is matched with
+events read from the log. Therefore block devices requests are processed
+deterministically.
diff --git a/qemu/docs/specs/fw_cfg.txt b/qemu/docs/specs/fw_cfg.txt
index 74351dd18..7a5f8c782 100644
--- a/qemu/docs/specs/fw_cfg.txt
+++ b/qemu/docs/specs/fw_cfg.txt
@@ -76,6 +76,22 @@ increasing address order, similar to memcpy().
 
 Selector Register IOport: 0x510
 Data Register IOport:     0x511
+DMA Address IOport:       0x514
+
+=== ARM Register Locations ===
+
+Selector Register address: Base + 8 (2 bytes)
+Data Register address:     Base + 0 (8 bytes)
+DMA Address address:       Base + 16 (8 bytes)
+
+== ACPI Interface ==
+
+The fw_cfg device is defined with ACPI ID "QEMU0002". Since we expect
+ACPI tables to be passed into the guest through the fw_cfg device itself,
+the guest-side firmware can not use ACPI to find fw_cfg. However, once the
+firmware is finished setting up ACPI tables and hands control over to the
+guest kernel, the latter can use the fw_cfg ACPI node for a more accurate
+inventory of in-use IOport or MMIO regions.
 
 == Firmware Configuration Items ==
 
@@ -86,11 +102,15 @@ by selecting the "signature" item using key 0x0000 (FW_CFG_SIGNATURE),
 and reading four bytes from the data register. If the fw_cfg device is
 present, the four bytes read will contain the characters "QEMU".
 
-=== Revision (Key 0x0001, FW_CFG_ID) ===
+If the DMA interface is available, then reading the DMA Address
+Register returns 0x51454d5520434647 ("QEMU CFG" in big-endian format).
+
+=== Revision / feature bitmap (Key 0x0001, FW_CFG_ID) ===
 
-A 32-bit little-endian unsigned int, this item is used as an interface
-revision number, and is currently set to 1 by QEMU when fw_cfg is
-initialized.
+A 32-bit little-endian unsigned int, this item is used to check for enabled
+features.
+ - Bit 0: traditional interface. Always set.
+ - Bit 1: DMA interface.
 
 === File Directory (Key 0x0019, FW_CFG_FILE_DIR) ===
 
@@ -132,79 +152,56 @@ Selector Reg.    Range Usage
 In practice, the number of allowed firmware configuration items is given
 by the value of FW_CFG_MAX_ENTRY (see fw_cfg.h).
 
-= Host-side API =
-
-The following functions are available to the QEMU programmer for adding
-data to a fw_cfg device during guest initialization (see fw_cfg.h for
-each function's complete prototype):
-
-== fw_cfg_add_bytes() ==
-
-Given a selector key value, starting pointer, and size, create an item
-as a raw "blob" of the given size, available by selecting the given key.
-The data referenced by the starting pointer is only linked, NOT copied,
-into the data structure of the fw_cfg device.
-
-== fw_cfg_add_string() ==
+= Guest-side DMA Interface =
 
-Instead of a starting pointer and size, this function accepts a pointer
-to a NUL-terminated ascii string, and inserts a newly allocated copy of
-the string (including the NUL terminator) into the fw_cfg device data
-structure.
+If bit 1 of the feature bitmap is set, the DMA interface is present. This does
+not replace the existing fw_cfg interface, it is an add-on. This interface
+can be used through the 64-bit wide address register.
 
-== fw_cfg_add_iXX() ==
+The address register is in big-endian format. The value for the register is 0
+at startup and after an operation. A write to the least significant half (at
+offset 4) triggers an operation. This means that operations with 32-bit
+addresses can be triggered with just one write, whereas operations with
+64-bit addresses can be triggered with one 64-bit write or two 32-bit writes,
+starting with the most significant half (at offset 0).
 
-Insert an XX-bit item, where XX may be 16, 32, or 64. These functions
-will convert a 16-, 32-, or 64-bit integer to little-endian, then add
-a dynamically allocated copy of the appropriately sized item to fw_cfg
-under the given selector key value.
+In this register, the physical address of a FWCfgDmaAccess structure in RAM
+should be written. This is the format of the FWCfgDmaAccess structure:
 
-== fw_cfg_add_file() ==
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} FWCfgDmaAccess;
 
-Given a filename (i.e., fw_cfg item name), starting pointer, and size,
-create an item as a raw "blob" of the given size. Unlike fw_cfg_add_bytes()
-above, the next available selector key (above 0x0020, FW_CFG_FILE_FIRST)
-will be used, and a new entry will be added to the file directory structure
-(at key 0x0019), containing the item name, blob size, and automatically
-assigned selector key value. The data referenced by the starting pointer
-is only linked, NOT copied, into the fw_cfg data structure.
+The fields of the structure are in big endian mode, and the field at the lowest
+address is the "control" field.
 
-== fw_cfg_add_file_callback() ==
+The "control" field has the following bits:
+ - Bit 0: Error
+ - Bit 1: Read
+ - Bit 2: Skip
+ - Bit 3: Select. The upper 16 bits are the selected index.
 
-Like fw_cfg_add_file(), but additionally sets pointers to a callback
-function (and opaque argument), which will be executed host-side by
-QEMU each time a byte is read by the guest from this particular item.
+When an operation is triggered, if the "control" field has bit 3 set, the
+upper 16 bits are interpreted as an index of a firmware configuration item.
+This has the same effect as writing the selector register.
 
-NOTE: The callback function is given the opaque argument set by
-fw_cfg_add_file_callback(), but also the current data offset,
-allowing it the option of only acting upon specific offset values
-(e.g., 0, before the first data byte of the selected item is
-returned to the guest).
+If the "control" field has bit 1 set, a read operation will be performed.
+"length" bytes for the current selector and offset will be copied into the
+physical RAM address specified by the "address" field.
 
-== fw_cfg_modify_file() ==
+If the "control" field has bit 2 set (and not bit 1), a skip operation will be
+performed. The offset for the current selector will be advanced "length" bytes.
 
-Given a filename (i.e., fw_cfg item name), starting pointer, and size,
-completely replace the configuration item referenced by the given item
-name with the new given blob. If an existing blob is found, its
-callback information is removed, and a pointer to the old data is
-returned to allow the caller to free it, helping avoid memory leaks.
-If a configuration item does not already exist under the given item
-name, a new item will be created as with fw_cfg_add_file(), and NULL
-is returned to the caller. In any case, the data referenced by the
-starting pointer is only linked, NOT copied, into the fw_cfg data
-structure.
+To check the result, read the "control" field:
+   error bit set        ->  something went wrong.
+   all bits cleared     ->  transfer finished successfully.
+   otherwise            ->  transfer still in progress (doesn't happen
+                            today due to implementation not being async,
+                            but may in the future).
 
-== fw_cfg_add_callback() ==
-
-Like fw_cfg_add_bytes(), but additionally sets pointers to a callback
-function (and opaque argument), which will be executed host-side by
-QEMU each time a guest-side write operation to this particular item
-completes fully overwriting the item's data.
-
-NOTE: This function is deprecated, and will be completely removed
-starting with QEMU v2.4.
-
-== Externally Provided Items ==
+= Externally Provided Items =
 
 As of v2.4, "file" fw_cfg items (i.e., items with selector keys above
 FW_CFG_FILE_FIRST, and with a corresponding entry in the fw_cfg file
@@ -213,14 +210,27 @@ the following syntax:
 
     -fw_cfg [name=]<item_name>,file=<path>
 
-where <item_name> is the fw_cfg item name, and <path> is the location
-on the host file system of a file containing the data to be inserted.
+Or
+
+    -fw_cfg [name=]<item_name>,string=<string>
+
+See QEMU man page for more documentation.
+
+Using item_name with plain ASCII characters only is recommended.
+
+Item names beginning with "opt/" are reserved for users.  QEMU will
+never create entries with such names unless explicitly ordered by the
+user.
+
+To avoid clashes among different users, it is strongly recommended
+that you use names beginning with opt/RFQDN/, where RFQDN is a reverse
+fully qualified domain name you control.  For instance, if SeaBIOS
+wanted to define additional names, the prefix "opt/org.seabios/" would
+be appropriate.
 
-NOTE: Users *SHOULD* choose item names beginning with the prefix "opt/"
-when using the "-fw_cfg" command line option, to avoid conflicting with
-item names used internally by QEMU. For instance:
+For historical reasons, "opt/ovmf/" is reserved for OVMF firmware.
 
-    -fw_cfg name=opt/my_item_name,file=./my_blob.bin
+Prefix "opt/org.qemu/" is reserved for QEMU itself.
 
-Similarly, QEMU developers *SHOULD NOT* use item names prefixed with
-"opt/" when inserting items programmatically, e.g. via fw_cfg_add_file().
+Use of names not beginning with "opt/" is potentially dangerous and
+entirely unsupported.  QEMU will warn if you try.
diff --git a/qemu/docs/specs/ivshmem-spec.txt b/qemu/docs/specs/ivshmem-spec.txt
new file mode 100644
index 000000000..a1f549979
--- /dev/null
+++ b/qemu/docs/specs/ivshmem-spec.txt
@@ -0,0 +1,254 @@
+= Device Specification for Inter-VM shared memory device =
+
+The Inter-VM shared memory device (ivshmem) is designed to share a
+memory region between multiple QEMU processes running different guests
+and the host.  In order for all guests to be able to pick up the
+shared memory area, it is modeled by QEMU as a PCI device exposing
+said memory to the guest as a PCI BAR.
+
+The device can use a shared memory object on the host directly, or it
+can obtain one from an ivshmem server.
+
+In the latter case, the device can additionally interrupt its peers, and
+get interrupted by its peers.
+
+
+== Configuring the ivshmem PCI device ==
+
+There are two basic configurations:
+
+- Just shared memory: -device ivshmem-plain,memdev=HMB,...
+
+  This uses host memory backend HMB.  It should have option "share"
+  set.
+
+- Shared memory plus interrupts: -device ivshmem,chardev=CHR,vectors=N,...
+
+  An ivshmem server must already be running on the host.  The device
+  connects to the server's UNIX domain socket via character device
+  CHR.
+
+  Each peer gets assigned a unique ID by the server.  IDs must be
+  between 0 and 65535.
+
+  Interrupts are message-signaled (MSI-X).  vectors=N configures the
+  number of vectors to use.
+
+For more details on ivshmem device properties, see The QEMU Emulator
+User Documentation (qemu-doc.*).
+
+
+== The ivshmem PCI device's guest interface ==
+
+The device has vendor ID 1af4, device ID 1110, revision 1.  Before
+QEMU 2.6.0, it had revision 0.
+
+=== PCI BARs ===
+
+The ivshmem PCI device has two or three BARs:
+
+- BAR0 holds device registers (256 Byte MMIO)
+- BAR1 holds MSI-X table and PBA (only ivshmem-doorbell)
+- BAR2 maps the shared memory object
+
+There are two ways to use this device:
+
+- If you only need the shared memory part, BAR2 suffices.  This way,
+  you have access to the shared memory in the guest and can use it as
+  you see fit.  Memnic, for example, uses ivshmem this way from guest
+  user space (see http://dpdk.org/browse/memnic).
+
+- If you additionally need the capability for peers to interrupt each
+  other, you need BAR0 and BAR1.  You will most likely want to write a
+  kernel driver to handle interrupts.  Requires the device to be
+  configured for interrupts, obviously.
+
+Before QEMU 2.6.0, BAR2 can initially be invalid if the device is
+configured for interrupts.  It becomes safely accessible only after
+the ivshmem server provided the shared memory.  These devices have PCI
+revision 0 rather than 1.  Guest software should wait for the
+IVPosition register (described below) to become non-negative before
+accessing BAR2.
+
+Revision 0 of the device is not capable to tell guest software whether
+it is configured for interrupts.
+
+=== PCI device registers ===
+
+BAR 0 contains the following registers:
+
+    Offset  Size  Access      On reset  Function
+        0     4   read/write        0   Interrupt Mask
+                                        bit 0: peer interrupt (rev 0)
+                                               reserved       (rev 1)
+                                        bit 1..31: reserved
+        4     4   read/write        0   Interrupt Status
+                                        bit 0: peer interrupt (rev 0)
+                                               reserved       (rev 1)
+                                        bit 1..31: reserved
+        8     4   read-only   0 or ID   IVPosition
+       12     4   write-only      N/A   Doorbell
+                                        bit 0..15: vector
+                                        bit 16..31: peer ID
+       16   240   none            N/A   reserved
+
+Software should only access the registers as specified in column
+"Access".  Reserved bits should be ignored on read, and preserved on
+write.
+
+In revision 0 of the device, Interrupt Status and Mask Register
+together control the legacy INTx interrupt when the device has no
+MSI-X capability: INTx is asserted when the bit-wise AND of Status and
+Mask is non-zero and the device has no MSI-X capability.  Interrupt
+Status Register bit 0 becomes 1 when an interrupt request from a peer
+is received.  Reading the register clears it.
+
+IVPosition Register: if the device is not configured for interrupts,
+this is zero.  Else, it is the device's ID (between 0 and 65535).
+
+Before QEMU 2.6.0, the register may read -1 for a short while after
+reset.  These devices have PCI revision 0 rather than 1.
+
+There is no good way for software to find out whether the device is
+configured for interrupts.  A positive IVPosition means interrupts,
+but zero could be either.
+
+Doorbell Register: writing this register requests to interrupt a peer.
+The written value's high 16 bits are the ID of the peer to interrupt,
+and its low 16 bits select an interrupt vector.
+
+If the device is not configured for interrupts, the write is ignored.
+
+If the interrupt hasn't completed setup, the write is ignored.  The
+device is not capable to tell guest software whether setup is
+complete.  Interrupts can regress to this state on migration.
+
+If the peer with the requested ID isn't connected, or it has fewer
+interrupt vectors connected, the write is ignored.  The device is not
+capable to tell guest software what peers are connected, or how many
+interrupt vectors are connected.
+
+The peer's interrupt for this vector then becomes pending.  There is
+no way for software to clear the pending bit, and a polling mode of
+operation is therefore impossible.
+
+If the peer is a revision 0 device without MSI-X capability, its
+Interrupt Status register is set to 1.  This asserts INTx unless
+masked by the Interrupt Mask register.  The device is not capable to
+communicate the interrupt vector to guest software then.
+
+With multiple MSI-X vectors, different vectors can be used to indicate
+different events have occurred.  The semantics of interrupt vectors
+are left to the application.
+
+
+== Interrupt infrastructure ==
+
+When configured for interrupts, the peers share eventfd objects in
+addition to shared memory.  The shared resources are managed by an
+ivshmem server.
+
+=== The ivshmem server ===
+
+The server listens on a UNIX domain socket.
+
+For each new client that connects to the server, the server
+- picks an ID,
+- creates eventfd file descriptors for the interrupt vectors,
+- sends the ID and the file descriptor for the shared memory to the
+  new client,
+- sends connect notifications for the new client to the other clients
+  (these contain file descriptors for sending interrupts),
+- sends connect notifications for the other clients to the new client,
+  and
+- sends interrupt setup messages to the new client (these contain file
+  descriptors for receiving interrupts).
+
+The first client to connect to the server receives ID zero.
+
+When a client disconnects from the server, the server sends disconnect
+notifications to the other clients.
+
+The next section describes the protocol in detail.
+
+If the server terminates without sending disconnect notifications for
+its connected clients, the clients can elect to continue.  They can
+communicate with each other normally, but won't receive disconnect
+notification on disconnect, and no new clients can connect.  There is
+no way for the clients to connect to a restarted server.  The device
+is not capable to tell guest software whether the server is still up.
+
+Example server code is in contrib/ivshmem-server/.  Not to be used in
+production.  It assumes all clients use the same number of interrupt
+vectors.
+
+A standalone client is in contrib/ivshmem-client/.  It can be useful
+for debugging.
+
+=== The ivshmem Client-Server Protocol ===
+
+An ivshmem device configured for interrupts connects to an ivshmem
+server.  This section details the protocol between the two.
+
+The connection is one-way: the server sends messages to the client.
+Each message consists of a single 8 byte little-endian signed number,
+and may be accompanied by a file descriptor via SCM_RIGHTS.  Both
+client and server close the connection on error.
+
+Note: QEMU currently doesn't close the connection right on error, but
+only when the character device is destroyed.
+
+On connect, the server sends the following messages in order:
+
+1. The protocol version number, currently zero.  The client should
+   close the connection on receipt of versions it can't handle.
+
+2. The client's ID.  This is unique among all clients of this server.
+   IDs must be between 0 and 65535, because the Doorbell register
+   provides only 16 bits for them.
+
+3. The number -1, accompanied by the file descriptor for the shared
+   memory.
+
+4. Connect notifications for existing other clients, if any.  This is
+   a peer ID (number between 0 and 65535 other than the client's ID),
+   repeated N times.  Each repetition is accompanied by one file
+   descriptor.  These are for interrupting the peer with that ID using
+   vector 0,..,N-1, in order.  If the client is configured for fewer
+   vectors, it closes the extra file descriptors.  If it is configured
+   for more, the extra vectors remain unconnected.
+
+5. Interrupt setup.  This is the client's own ID, repeated N times.
+   Each repetition is accompanied by one file descriptor.  These are
+   for receiving interrupts from peers using vector 0,..,N-1, in
+   order.  If the client is configured for fewer vectors, it closes
+   the extra file descriptors.  If it is configured for more, the
+   extra vectors remain unconnected.
+
+From then on, the server sends these kinds of messages:
+
+6. Connection / disconnection notification.  This is a peer ID.
+
+  - If the number comes with a file descriptor, it's a connection
+    notification, exactly like in step 4.
+
+  - Else, it's a disconnection notification for the peer with that ID.
+
+Known bugs:
+
+* The protocol changed incompatibly in QEMU 2.5.  Before, messages
+  were native endian long, and there was no version number.
+
+* The protocol is poorly designed.
+
+=== The ivshmem Client-Client Protocol ===
+
+An ivshmem device configured for interrupts receives eventfd file
+descriptors for interrupting peers and getting interrupted by peers
+from the server, as explained in the previous section.
+
+To interrupt a peer, the device writes the 8-byte integer 1 in native
+byte order to the respective file descriptor.
+
+To receive an interrupt, the device reads and discards as many 8-byte
+integers as it can.
diff --git a/qemu/docs/specs/ivshmem_device_spec.txt b/qemu/docs/specs/ivshmem_device_spec.txt
deleted file mode 100644
index 667a8628f..000000000
--- a/qemu/docs/specs/ivshmem_device_spec.txt
+++ /dev/null
@@ -1,96 +0,0 @@
-
-Device Specification for Inter-VM shared memory device
-------------------------------------------------------
-
-The Inter-VM shared memory device is designed to share a region of memory to
-userspace in multiple virtual guests.  The memory region does not belong to any
-guest, but is a POSIX memory object on the host.  Optionally, the device may
-support sending interrupts to other guests sharing the same memory region.
-
-
-The Inter-VM PCI device
------------------------
-
-*BARs*
-
-The device supports three BARs.  BAR0 is a 1 Kbyte MMIO region to support
-registers.  BAR1 is used for MSI-X when it is enabled in the device.  BAR2 is
-used to map the shared memory object from the host.  The size of BAR2 is
-specified when the guest is started and must be a power of 2 in size.
-
-*Registers*
-
-The device currently supports 4 registers of 32-bits each.  Registers
-are used for synchronization between guests sharing the same memory object when
-interrupts are supported (this requires using the shared memory server).
-
-The server assigns each VM an ID number and sends this ID number to the QEMU
-process when the guest starts.
-
-enum ivshmem_registers {
-    IntrMask = 0,
-    IntrStatus = 4,
-    IVPosition = 8,
-    Doorbell = 12
-};
-
-The first two registers are the interrupt mask and status registers.  Mask and
-status are only used with pin-based interrupts.  They are unused with MSI
-interrupts.
-
-Status Register: The status register is set to 1 when an interrupt occurs.
-
-Mask Register: The mask register is bitwise ANDed with the interrupt status
-and the result will raise an interrupt if it is non-zero.  However, since 1 is
-the only value the status will be set to, it is only the first bit of the mask
-that has any effect.  Therefore interrupts can be masked by setting the first
-bit to 0 and unmasked by setting the first bit to 1.
-
-IVPosition Register: The IVPosition register is read-only and reports the
-guest's ID number.  The guest IDs are non-negative integers.  When using the
-server, since the server is a separate process, the VM ID will only be set when
-the device is ready (shared memory is received from the server and accessible via
-the device).  If the device is not ready, the IVPosition will return -1.
-Applications should ensure that they have a valid VM ID before accessing the
-shared memory.
-
-Doorbell Register:  To interrupt another guest, a guest must write to the
-Doorbell register.  The doorbell register is 32-bits, logically divided into
-two 16-bit fields.  The high 16-bits are the guest ID to interrupt and the low
-16-bits are the interrupt vector to trigger.  The semantics of the value
-written to the doorbell depends on whether the device is using MSI or a regular
-pin-based interrupt.  In short, MSI uses vectors while regular interrupts set the
-status register.
-
-Regular Interrupts
-
-If regular interrupts are used (due to either a guest not supporting MSI or the
-user specifying not to use them on startup) then the value written to the lower
-16-bits of the Doorbell register results is arbitrary and will trigger an
-interrupt in the destination guest.
-
-Message Signalled Interrupts
-
-A ivshmem device may support multiple MSI vectors.  If so, the lower 16-bits
-written to the Doorbell register must be between 0 and the maximum number of
-vectors the guest supports.  The lower 16 bits written to the doorbell is the
-MSI vector that will be raised in the destination guest.  The number of MSI
-vectors is configurable but it is set when the VM is started.
-
-The important thing to remember with MSI is that it is only a signal, no status
-is set (since MSI interrupts are not shared).  All information other than the
-interrupt itself should be communicated via the shared memory region.  Devices
-supporting multiple MSI vectors can use different vectors to indicate different
-events have occurred.  The semantics of interrupt vectors are left to the
-user's discretion.
-
-
-Usage in the Guest
-------------------
-
-The shared memory device is intended to be used with the provided UIO driver.
-Very little configuration is needed.  The guest should map BAR0 to access the
-registers (an array of 32-bit ints allows simple writing) and map BAR2 to
-access the shared memory region itself.  The size of the shared memory region
-is specified when the guest (or shared memory server) is started.  A guest may
-map the whole shared memory region or only part of it.
diff --git a/qemu/docs/specs/parallels.txt b/qemu/docs/specs/parallels.txt
new file mode 100644
index 000000000..b4fe2295f
--- /dev/null
+++ b/qemu/docs/specs/parallels.txt
@@ -0,0 +1,228 @@
+= License =
+
+Copyright (c) 2015 Denis Lunev
+Copyright (c) 2015 Vladimir Sementsov-Ogievskiy
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.
+See the COPYING file in the top-level directory.
+
+= Parallels Expandable Image File Format =
+
+A Parallels expandable image file consists of three consecutive parts:
+    * header
+    * BAT
+    * data area
+
+All numbers in a Parallels expandable image are stored in little-endian byte
+order.
+
+
+== Definitions ==
+
+    Sector    A 512-byte data chunk.
+
+    Cluster   A data chunk of the size specified in the image header.
+              Currently, the default size is 1MiB (2048 sectors). In previous
+              versions, cluster sizes of 63 sectors, 256 and 252 kilobytes were
+              used.
+
+    BAT       Block Allocation Table, an entity that contains information for
+              guest-to-host I/O data address translation.
+
+
+== Header ==
+
+The header is placed at the start of an image and contains the following
+fields:
+
+Bytes:
+   0 - 15:    magic
+              Must contain "WithoutFreeSpace" or "WithouFreSpacExt".
+
+  16 - 19:    version
+              Must be 2.
+
+  20 - 23:    heads
+              Disk geometry parameter for guest.
+
+  24 - 27:    cylinders
+              Disk geometry parameter for guest.
+
+  28 - 31:    tracks
+              Cluster size, in sectors.
+
+  32 - 35:    nb_bat_entries
+              Disk size, in clusters (BAT size).
+
+  36 - 43:    nb_sectors
+              Disk size, in sectors.
+
+              For "WithoutFreeSpace" images:
+              Only the lowest 4 bytes are used. The highest 4 bytes must be
+              cleared in this case.
+
+              For "WithouFreSpacExt" images, there are no such
+              restrictions.
+
+  44 - 47:    in_use
+              Set to 0x746F6E59 when the image is opened by software in R/W
+              mode; set to 0x312e3276 when the image is closed.
+
+              A zero in this field means that the image was opened by an old
+              version of the software that doesn't support Format Extension
+              (see below).
+
+              Other values are not allowed.
+
+  48 - 51:    data_off
+              An offset, in sectors, from the start of the file to the start of
+              the data area.
+
+              For "WithoutFreeSpace" images:
+              - If data_off is zero, the offset is calculated as the end of BAT
+                table plus some padding to ensure sector size alignment.
+              - If data_off is non-zero, the offset should be aligned to sector
+                size. However it is recommended to align it to cluster size for
+                newly created images.
+
+              For "WithouFreSpacExt" images:
+              data_off must be non-zero and aligned to cluster size.
+
+  52 - 55:    flags
+              Miscellaneous flags.
+
+              Bit 0: Empty Image bit. If set, the image should be
+                     considered clear.
+
+              Bits 2-31: Unused.
+
+  56 - 63:    ext_off
+              Format Extension offset, an offset, in sectors, from the start of
+              the file to the start of the Format Extension Cluster.
+
+              ext_off must meet the same requirements as cluster offsets
+              defined by BAT entries (see below).
+
+
+== BAT ==
+
+BAT is placed immediately after the image header. In the file, BAT is a
+contiguous array of 32-bit unsigned little-endian integers with
+(bat_entries * 4) bytes size.
+
+Each BAT entry contains an offset from the start of the file to the
+corresponding cluster. The offset set in clusters for "WithouFreSpacExt" images
+and in sectors for "WithoutFreeSpace" images.
+
+If a BAT entry is zero, the corresponding cluster is not allocated and should
+be considered as filled with zeroes.
+
+Cluster offsets specified by BAT entries must meet the following requirements:
+    - the value must not be lower than data offset (provided by header.data_off
+      or calculated as specified above),
+    - the value must be lower than the desired file size,
+    - the value must be unique among all BAT entries,
+    - the result of (cluster offset - data offset) must be aligned to cluster
+      size.
+
+
+== Data Area ==
+
+The data area is an area from the data offset (provided by header.data_off or
+calculated as specified above) to the end of the file. It represents a
+contiguous array of clusters. Most of them are allocated by the BAT, some may
+be allocated by the ext_off field in the header while other may be allocated by
+extensions. All clusters allocated by ext_off and extensions should meet the
+same requirements as clusters specified by BAT entries.
+
+
+== Format Extension ==
+
+The Format Extension is an area 1 cluster in size that provides additional
+format features. This cluster is addressed by the ext_off field in the header.
+The format of the Format Extension area is the following:
+
+   0 -  7:    magic
+              Must be 0xAB234CEF23DCEA87
+
+   8 - 23:    m_CheckSum
+              The MD5 checksum of the entire Header Extension cluster except
+              the first 24 bytes.
+
+    The above are followed by feature sections or "extensions". The last
+    extension must be "End of features" (see below).
+
+Each feature section has the following format:
+
+   0 -  7:    magic
+              The identifier of the feature:
+              0x0000000000000000 - End of features
+              0x20385FAE252CB34A - Dirty bitmap
+
+   8 - 15:    flags
+              External flags for extension:
+
+              Bit 0: NECESSARY
+                     If the software cannot load the extension (due to an
+                     unknown magic number or error), the file should not be
+                     changed. If this flag is unset and there is an error on
+                     loading the extension, said extension should be dropped.
+
+              Bit 1: TRANSIT
+                     If there is an unknown extension with this flag set,
+                     said extension should be left as is.
+
+              If neither NECESSARY nor TRANSIT are set, the extension should be
+              dropped.
+
+  16 - 19:    data_size
+              The size of the following feature data, in bytes.
+
+  20 - 23:    unused32
+              Align header to 8 bytes boundary.
+
+  variable:   data (data_size bytes)
+
+    The above is followed by padding to the next 8 bytes boundary, then the
+    next extension starts.
+
+    The last extension must be "End of features" with all the fields set to 0.
+
+
+=== Dirty bitmaps feature ===
+
+This feature provides a way of storing dirty bitmaps in the image. The fields
+of its data area are:
+
+   0 -  7:    size
+              The bitmap size, should be equal to disk size in sectors.
+
+   8 - 23:    id
+              An identifier for backup consistency checking.
+
+  24 - 27:    granularity
+              Bitmap granularity, in sectors. I.e., the number of sectors
+              corresponding to one bit of the bitmap. Granularity must be
+              a power of 2.
+
+  28 - 31:    l1_size
+              The number of entries in the L1 table of the bitmap.
+
+  variable:   l1 (64 * l1_size bytes)
+              L1 offset table (in bytes)
+
+A dirty bitmap is stored using a one-level structure for the mapping to host
+clusters - an L1 table.
+
+Given an offset in bytes into the bitmap data, the offset in bytes into the
+image file can be obtained as follows:
+
+    offset = l1_table[offset / cluster_size] + (offset % cluster_size)
+
+If an L1 table entry is 0, the corresponding cluster of the bitmap is assumed
+to be zero.
+
+If an L1 table entry is 1, the corresponding cluster of the bitmap is assumed
+to have all bits set.
+
+If an L1 table entry is not 0 or 1, it allocates a cluster from the data area.
diff --git a/qemu/docs/specs/pci-ids.txt b/qemu/docs/specs/pci-ids.txt
index 0adcb89aa..fd27c677d 100644
--- a/qemu/docs/specs/pci-ids.txt
+++ b/qemu/docs/specs/pci-ids.txt
@@ -15,13 +15,23 @@ The 1000 -> 10ff device ID range is used as follows for virtio-pci devices.
 Note that this allocation separate from the virtio device IDs, which are
 maintained as part of the virtio specification.
 
-1af4:1000  network device
-1af4:1001  block device
-1af4:1002  balloon device
-1af4:1003  console device
-1af4:1004  SCSI host bus adapter device
-1af4:1005  entropy generator device
-1af4:1009  9p filesystem device
+1af4:1000  network device (legacy)
+1af4:1001  block device (legacy)
+1af4:1002  balloon device (legacy)
+1af4:1003  console device (legacy)
+1af4:1004  SCSI host bus adapter device (legacy)
+1af4:1005  entropy generator device (legacy)
+1af4:1009  9p filesystem device (legacy)
+
+1af4:1041  network device (modern)
+1af4:1042  block device (modern)
+1af4:1043  console device (modern)
+1af4:1044  entropy generator device (modern)
+1af4:1045  balloon device (modern)
+1af4:1048  SCSI host bus adapter device (modern)
+1af4:1049  9p filesystem device (modern)
+1af4:1050  virtio gpu device (modern)
+1af4:1052  virtio input device (modern)
 
 1af4:10f0  Available for experimental usage without registration.  Must get
    to      official ID when the code leaves the test lab (i.e. when seeking
diff --git a/qemu/docs/specs/ppc-spapr-hcalls.txt b/qemu/docs/specs/ppc-spapr-hcalls.txt
index 667b3fa00..5bd8eab78 100644
--- a/qemu/docs/specs/ppc-spapr-hcalls.txt
+++ b/qemu/docs/specs/ppc-spapr-hcalls.txt
@@ -41,8 +41,8 @@ When the guest runs in "real mode" (in powerpc lingua this means
 with MMU disabled, ie guest effective == guest physical), it only
 has access to a subset of memory and no IOs.
 
-PAPR provides a set of hypervisor calls to perform cachable or
-non-cachable accesses to any guest physical addresses that the
+PAPR provides a set of hypervisor calls to perform cacheable or
+non-cacheable accesses to any guest physical addresses that the
 guest can use in order to access IO devices while in real mode.
 
 This is typically used by the firmware running in the guest.
diff --git a/qemu/docs/specs/ppc-spapr-hotplug.txt b/qemu/docs/specs/ppc-spapr-hotplug.txt
index 46e07196b..631b0cada 100644
--- a/qemu/docs/specs/ppc-spapr-hotplug.txt
+++ b/qemu/docs/specs/ppc-spapr-hotplug.txt
@@ -302,4 +302,52 @@ consisting of <phys>, <size> and <maxcpus>.
 pseries guests use this property to note the maximum allowed CPUs for the
 guest.
 
+== ibm,dynamic-reconfiguration-memory ==
+
+ibm,dynamic-reconfiguration-memory is a device tree node that represents
+dynamically reconfigurable logical memory blocks (LMB). This node
+is generated only when the guest advertises the support for it via
+ibm,client-architecture-support call. Memory that is not dynamically
+reconfigurable is represented by /memory nodes. The properties of this
+node that are of interest to the sPAPR memory hotplug implementation
+in QEMU are described here.
+
+ibm,lmb-size
+
+This 64bit integer defines the size of each dynamically reconfigurable LMB.
+
+ibm,associativity-lookup-arrays
+
+This property defines a lookup array in which the NUMA associativity
+information for each LMB can be found. It is a property encoded array
+that begins with an integer M, the number of associativity lists followed
+by an integer N, the number of entries per associativity list and terminated
+by M associativity lists each of length N integers.
+
+This property provides the same information as given by ibm,associativity
+property in a /memory node. Each assigned LMB has an index value between
+0 and M-1 which is used as an index into this table to select which
+associativity list to use for the LMB. This index value for each LMB
+is defined in ibm,dynamic-memory property.
+
+ibm,dynamic-memory
+
+This property describes the dynamically reconfigurable memory. It is a
+property encoded array that has an integer N, the number of LMBs followed
+by N LMB list entires.
+
+Each LMB list entry consists of the following elements:
+
+- Logical address of the start of the LMB encoded as a 64bit integer. This
+  corresponds to reg property in /memory node.
+- DRC index of the LMB that corresponds to ibm,my-drc-index property
+  in a /memory node.
+- Four bytes reserved for expansion.
+- Associativity list index for the LMB that is used as an index into
+  ibm,associativity-lookup-arrays property described earlier. This
+  is used to retrieve the right associativity list to be used for this
+  LMB.
+- A 32bit flags word. The bit at bit position 0x00000008 defines whether
+  the LMB is assigned to the the partition as of boot time.
+
 [1] http://thread.gmane.org/gmane.linux.ports.ppc.embedded/75350/focus=106867
diff --git a/qemu/docs/specs/qcow2.txt b/qemu/docs/specs/qcow2.txt
index 121dfc8cc..80cdfd0e9 100644
--- a/qemu/docs/specs/qcow2.txt
+++ b/qemu/docs/specs/qcow2.txt
@@ -103,7 +103,18 @@ in the description of a field.
                     write to an image with unknown auto-clear features if it
                     clears the respective bits from this field first.
 
-                    Bits 0-63:  Reserved (set to 0)
+                    Bit 0:      Bitmaps extension bit
+                                This bit indicates consistency for the bitmaps
+                                extension data.
+
+                                It is an error if this bit is set without the
+                                bitmaps extension present.
+
+                                If the bitmaps extension is present but this
+                                bit is unset, the bitmaps extension data must be
+                                considered inconsistent.
+
+                    Bits 1-63:  Reserved (set to 0)
 
          96 -  99:  refcount_order
                     Describes the width of a reference count block entry (width
@@ -123,6 +134,7 @@ be stored. Each extension has a structure like the following:
                         0x00000000 - End of the header extension area
                         0xE2792ACA - Backing file format name
                         0x6803f857 - Feature name table
+                        0x23852875 - Bitmaps extension
                         other      - Unknown header extension, can be safely
                                      ignored
 
@@ -166,6 +178,36 @@ the header extension data. Each entry look like this:
                     terminated if it has full length)
 
 
+== Bitmaps extension ==
+
+The bitmaps extension is an optional header extension. It provides the ability
+to store bitmaps related to a virtual disk. For now, there is only one bitmap
+type: the dirty tracking bitmap, which tracks virtual disk changes from some
+point in time.
+
+The data of the extension should be considered consistent only if the
+corresponding auto-clear feature bit is set, see autoclear_features above.
+
+The fields of the bitmaps extension are:
+
+    Byte  0 -  3:  nb_bitmaps
+                   The number of bitmaps contained in the image. Must be
+                   greater than or equal to 1.
+
+                   Note: Qemu currently only supports up to 65535 bitmaps per
+                   image.
+
+          4 -  7:  Reserved, must be zero.
+
+          8 - 15:  bitmap_directory_size
+                   Size of the bitmap directory in bytes. It is the cumulative
+                   size of all (nb_bitmaps) bitmap headers.
+
+         16 - 23:  bitmap_directory_offset
+                   Offset into the image file at which the bitmap directory
+                   starts. Must be aligned to a cluster boundary.
+
+
 == Host cluster management ==
 
 qcow2 manages the allocation of host clusters by maintaining a reference count
@@ -257,7 +299,7 @@ L2 table entry:
 
               63:   0 for a cluster that is unused or requires COW, 1 if its
                     refcount is exactly one. This information is only accurate
-                    in L2 tables that are reachable from the the active L1
+                    in L2 tables that are reachable from the active L1
                     table.
 
 Standard Cluster Descriptor:
@@ -360,3 +402,180 @@ Snapshot table entry:
 
         variable:   Padding to round up the snapshot table entry size to the
                     next multiple of 8.
+
+
+== Bitmaps ==
+
+As mentioned above, the bitmaps extension provides the ability to store bitmaps
+related to a virtual disk. This section describes how these bitmaps are stored.
+
+All stored bitmaps are related to the virtual disk stored in the same image, so
+each bitmap size is equal to the virtual disk size.
+
+Each bit of the bitmap is responsible for strictly defined range of the virtual
+disk. For bit number bit_nr the corresponding range (in bytes) will be:
+
+    [bit_nr * bitmap_granularity .. (bit_nr + 1) * bitmap_granularity - 1]
+
+Granularity is a property of the concrete bitmap, see below.
+
+
+=== Bitmap directory ===
+
+Each bitmap saved in the image is described in a bitmap directory entry. The
+bitmap directory is a contiguous area in the image file, whose starting offset
+and length are given by the header extension fields bitmap_directory_offset and
+bitmap_directory_size. The entries of the bitmap directory have variable
+length, depending on the lengths of the bitmap name and extra data. These
+entries are also called bitmap headers.
+
+Structure of a bitmap directory entry:
+
+    Byte 0 -  7:    bitmap_table_offset
+                    Offset into the image file at which the bitmap table
+                    (described below) for the bitmap starts. Must be aligned to
+                    a cluster boundary.
+
+         8 - 11:    bitmap_table_size
+                    Number of entries in the bitmap table of the bitmap.
+
+        12 - 15:    flags
+                    Bit
+                      0: in_use
+                         The bitmap was not saved correctly and may be
+                         inconsistent.
+
+                      1: auto
+                         The bitmap must reflect all changes of the virtual
+                         disk by any application that would write to this qcow2
+                         file (including writes, snapshot switching, etc.). The
+                         type of this bitmap must be 'dirty tracking bitmap'.
+
+                      2: extra_data_compatible
+                         This flags is meaningful when the extra data is
+                         unknown to the software (currently any extra data is
+                         unknown to Qemu).
+                         If it is set, the bitmap may be used as expected, extra
+                         data must be left as is.
+                         If it is not set, the bitmap must not be used, but
+                         both it and its extra data be left as is.
+
+                    Bits 3 - 31 are reserved and must be 0.
+
+             16:    type
+                    This field describes the sort of the bitmap.
+                    Values:
+                      1: Dirty tracking bitmap
+
+                    Values 0, 2 - 255 are reserved.
+
+             17:    granularity_bits
+                    Granularity bits. Valid values: 0 - 63.
+
+                    Note: Qemu currently doesn't support granularity_bits
+                    greater than 31.
+
+                    Granularity is calculated as
+                        granularity = 1 << granularity_bits
+
+                    A bitmap's granularity is how many bytes of the image
+                    accounts for one bit of the bitmap.
+
+        18 - 19:    name_size
+                    Size of the bitmap name. Must be non-zero.
+
+                    Note: Qemu currently doesn't support values greater than
+                    1023.
+
+        20 - 23:    extra_data_size
+                    Size of type-specific extra data.
+
+                    For now, as no extra data is defined, extra_data_size is
+                    reserved and should be zero. If it is non-zero the
+                    behavior is defined by extra_data_compatible flag.
+
+        variable:   extra_data
+                    Extra data for the bitmap, occupying extra_data_size bytes.
+                    Extra data must never contain references to clusters or in
+                    some other way allocate additional clusters.
+
+        variable:   name
+                    The name of the bitmap (not null terminated), occupying
+                    name_size bytes. Must be unique among all bitmap names
+                    within the bitmaps extension.
+
+        variable:   Padding to round up the bitmap directory entry size to the
+                    next multiple of 8. All bytes of the padding must be zero.
+
+
+=== Bitmap table ===
+
+Each bitmap is stored using a one-level structure (as opposed to two-level
+structures like for refcounts and guest clusters mapping) for the mapping of
+bitmap data to host clusters. This structure is called the bitmap table.
+
+Each bitmap table has a variable size (stored in the bitmap directory entry)
+and may use multiple clusters, however, it must be contiguous in the image
+file.
+
+Structure of a bitmap table entry:
+
+    Bit       0:    Reserved and must be zero if bits 9 - 55 are non-zero.
+                    If bits 9 - 55 are zero:
+                      0: Cluster should be read as all zeros.
+                      1: Cluster should be read as all ones.
+
+         1 -  8:    Reserved and must be zero.
+
+         9 - 55:    Bits 9 - 55 of the host cluster offset. Must be aligned to
+                    a cluster boundary. If the offset is 0, the cluster is
+                    unallocated; in that case, bit 0 determines how this
+                    cluster should be treated during reads.
+
+        56 - 63:    Reserved and must be zero.
+
+
+=== Bitmap data ===
+
+As noted above, bitmap data is stored in separate clusters, described by the
+bitmap table. Given an offset (in bytes) into the bitmap data, the offset into
+the image file can be obtained as follows:
+
+    image_offset(bitmap_data_offset) =
+        bitmap_table[bitmap_data_offset / cluster_size] +
+            (bitmap_data_offset % cluster_size)
+
+This offset is not defined if bits 9 - 55 of bitmap table entry are zero (see
+above).
+
+Given an offset byte_nr into the virtual disk and the bitmap's granularity, the
+bit offset into the image file to the corresponding bit of the bitmap can be
+calculated like this:
+
+    bit_offset(byte_nr) =
+        image_offset(byte_nr / granularity / 8) * 8 +
+            (byte_nr / granularity) % 8
+
+If the size of the bitmap data is not a multiple of the cluster size then the
+last cluster of the bitmap data contains some unused tail bits. These bits must
+be zero.
+
+
+=== Dirty tracking bitmaps ===
+
+Bitmaps with 'type' field equal to one are dirty tracking bitmaps.
+
+When the virtual disk is in use dirty tracking bitmap may be 'enabled' or
+'disabled'. While the bitmap is 'enabled', all writes to the virtual disk
+should be reflected in the bitmap. A set bit in the bitmap means that the
+corresponding range of the virtual disk (see above) was written to while the
+bitmap was 'enabled'. An unset bit means that this range was not written to.
+
+The software doesn't have to sync the bitmap in the image file with its
+representation in RAM after each write. Flag 'in_use' should be set while the
+bitmap is not synced.
+
+In the image file the 'enabled' state is reflected by the 'auto' flag. If this
+flag is set, the software must consider the bitmap as 'enabled' and start
+tracking virtual disk changes to this bitmap from the first write to the
+virtual disk. If this flag is not set then the bitmap is disabled.
diff --git a/qemu/docs/specs/rocker.txt b/qemu/docs/specs/rocker.txt
index 1c743515c..d2a82624f 100644
--- a/qemu/docs/specs/rocker.txt
+++ b/qemu/docs/specs/rocker.txt
@@ -297,7 +297,7 @@ but not fired.  If only partial credits are returned, the interrupt remains
 masked but the device generates an interrupt, signaling the driver that more
 outstanding work is available.
 
-(* this masking is unrelated to to the MSI-X interrupt mask register)
+(* this masking is unrelated to the MSI-X interrupt mask register)
 
 Endianness
 ----------
diff --git a/qemu/docs/specs/vhost-user.txt b/qemu/docs/specs/vhost-user.txt
index 650bb1818..777c49cfe 100644
--- a/qemu/docs/specs/vhost-user.txt
+++ b/qemu/docs/specs/vhost-user.txt
@@ -87,6 +87,14 @@ Depending on the request type, payload can be:
    User address: a 64-bit user address
    mmap offset: 64-bit offset where region starts in the mapped memory
 
+* Log description
+   ---------------------------
+   | log size | log offset |
+   ---------------------------
+   log size: size of area used for logging
+   log offset: offset from start of supplied file descriptor
+       where logging starts (i.e. where guest address 0 would be logged)
+
 In QEMU the vhost-user message is implemented with the following struct:
 
 typedef struct VhostUserMsg {
@@ -98,6 +106,7 @@ typedef struct VhostUserMsg {
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
+        VhostUserLog log;
     };
 } QEMU_PACKED VhostUserMsg;
 
@@ -113,12 +122,15 @@ message replies. Most of the requests don't require replies. Here is a list of
 the ones that do:
 
  * VHOST_GET_FEATURES
+ * VHOST_GET_PROTOCOL_FEATURES
  * VHOST_GET_VRING_BASE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
 
 There are several messages that the master sends with file descriptors passed
 in the ancillary data:
 
  * VHOST_SET_MEM_TABLE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
  * VHOST_SET_LOG_FD
  * VHOST_SET_VRING_KICK
  * VHOST_SET_VRING_CALL
@@ -127,6 +139,122 @@ in the ancillary data:
 If Master is unable to send the full message or receives a wrong reply it will
 close the connection. An optional reconnection mechanism can be implemented.
 
+Any protocol extensions are gated by protocol feature bits,
+which allows full backwards compatibility on both master
+and slave.
+As older slaves don't support negotiating protocol features,
+a feature bit was dedicated for this purpose:
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+Starting and stopping rings
+----------------------
+Client must only process each ring when it is started.
+
+Client must only pass data between the ring and the
+backend, when the ring is enabled.
+
+If ring is started but disabled, client must process the
+ring without talking to the backend.
+
+For example, for a networking device, in the disabled state
+client must not supply any new RX packets, but must process
+and discard any TX packets.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated, the ring is initialized
+in an enabled state.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has been negotiated, the ring is initialized
+in a disabled state. Client must not pass data to/from the backend until ring is enabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 1, or after it has been disabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 0.
+
+Each ring is initialized in a stopped state, client must not process it until
+ring is started, or after it has been stopped.
+
+Client must start ring upon receiving a kick (that is, detecting that file
+descriptor is readable) on the descriptor specified by
+VHOST_USER_SET_VRING_KICK, and stop ring upon receiving
+VHOST_USER_GET_VRING_BASE.
+
+While processing the rings (whether they are enabled or not), client must
+support changing some configuration aspects on the fly.
+
+Multiple queue support
+----------------------
+
+Multiple queue is treated as a protocol extension, hence the slave has to
+implement protocol features first. The multiple queues feature is supported
+only when the protocol feature VHOST_USER_PROTOCOL_F_MQ (bit 0) is set.
+
+The max number of queues the slave supports can be queried with message
+VHOST_USER_GET_PROTOCOL_FEATURES. Master should stop when the number of
+requested queues is bigger than that.
+
+As all queues share one connection, the master uses a unique index for each
+queue in the sent message to identify a specified queue. One queue pair
+is enabled initially. More queues are enabled dynamically, by sending
+message VHOST_USER_SET_VRING_ENABLE.
+
+Migration
+---------
+
+During live migration, the master may need to track the modifications
+the slave makes to the memory mapped regions. The client should mark
+the dirty pages in a log. Once it complies to this logging, it may
+declare the VHOST_F_LOG_ALL vhost feature.
+
+To start/stop logging of data/used ring writes, server may send messages
+VHOST_USER_SET_FEATURES with VHOST_F_LOG_ALL and VHOST_USER_SET_VRING_ADDR with
+VHOST_VRING_F_LOG in ring's flags set to 1/0, respectively.
+
+All the modifications to memory pointed by vring "descriptor" should
+be marked. Modifications to "used" vring should be marked if
+VHOST_VRING_F_LOG is part of ring's flags.
+
+Dirty pages are of size:
+#define VHOST_LOG_PAGE 0x1000
+
+The log memory fd is provided in the ancillary data of
+VHOST_USER_SET_LOG_BASE message when the slave has
+VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol feature.
+
+The size of the log is supplied as part of VhostUserMsg
+which should be large enough to cover all known guest
+addresses. Log starts at the supplied offset in the
+supplied file descriptor.
+The log covers from address 0 to the maximum of guest
+regions. In pseudo-code, to mark page at "addr" as dirty:
+
+page = addr / VHOST_LOG_PAGE
+log[page / 8] |= 1 << page % 8
+
+Where addr is the guest physical address.
+
+Use atomic operations, as the log may be concurrently manipulated.
+
+Note that when logging modifications to the used ring (when VHOST_VRING_F_LOG
+is set for this ring), log_guest_addr should be used to calculate the log
+offset: the write to first byte of the used ring is logged at this offset from
+log start. Also note that this value might be outside the legal guest physical
+address range (i.e. does not have to be covered by the VhostUserMemory table),
+but the bit offset of the last byte of the ring must fall within
+the size supplied by VhostUserLog.
+
+VHOST_USER_SET_LOG_FD is an optional message with an eventfd in
+ancillary data, it may be used to inform the master that the log has
+been modified.
+
+Once the source has finished migration, rings will be stopped by
+the source. No further update must be done before rings are
+restarted.
+
+Protocol features
+-----------------
+
+#define VHOST_USER_PROTOCOL_F_MQ             0
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD      1
+#define VHOST_USER_PROTOCOL_F_RARP           2
+
 Message types
 -------------
 
@@ -138,6 +266,8 @@ Message types
       Slave payload: u64
 
       Get from the underlying vhost implementation the features bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
 
  * VHOST_USER_SET_FEATURES
 
@@ -146,6 +276,33 @@ Message types
       Master payload: u64
 
       Enable features in the underlying vhost implementation using a bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_GET_PROTOCOL_FEATURES
+
+      Id: 15
+      Equivalent ioctl: VHOST_GET_FEATURES
+      Master payload: N/A
+      Slave payload: u64
+
+      Get the protocol feature bitmask from the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
+
+ * VHOST_USER_SET_PROTOCOL_FEATURES
+
+      Id: 16
+      Ioctl: VHOST_SET_FEATURES
+      Master payload: u64
+
+      Enable protocol features in the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
 
  * VHOST_USER_SET_OWNER
 
@@ -160,11 +317,13 @@ Message types
  * VHOST_USER_RESET_OWNER
 
       Id: 4
-      Equivalent ioctl: VHOST_RESET_OWNER
       Master payload: N/A
 
-      Issued when a new connection is about to be closed. The Master will no
-      longer own this connection (and will usually close it).
+      This is no longer used. Used to be sent to request disabling
+      all rings, but some clients interpreted it to also discard
+      connection state (this interpretation would lead to bugs).
+      It is recommended that clients either ignore this message,
+      or use it to disable all rings.
 
  * VHOST_USER_SET_MEM_TABLE
 
@@ -182,8 +341,14 @@ Message types
       Id: 6
       Equivalent ioctl: VHOST_SET_LOG_BASE
       Master payload: u64
+      Slave payload: N/A
+
+      Sets logging shared memory space.
+      When slave has VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol
+      feature, the log memory fd is provided in the ancillary data of
+      VHOST_USER_SET_LOG_BASE message, the size and offset of shared
+      memory area provided in the message.
 
-      Sets the logging base address.
 
  * VHOST_USER_SET_LOG_FD
 
@@ -199,7 +364,7 @@ Message types
       Equivalent ioctl: VHOST_SET_VRING_NUM
       Master payload: vring state description
 
-      Sets the number of vrings for this owner.
+      Set the size of the queue.
 
  * VHOST_USER_SET_VRING_ADDR
 
@@ -264,3 +429,38 @@ Message types
       Bits (0-7) of the payload contain the vring index. Bit 8 is the
       invalid FD flag. This flag is set when there is no file descriptor
       in the ancillary data.
+
+ * VHOST_USER_GET_QUEUE_NUM
+
+      Id: 17
+      Equivalent ioctl: N/A
+      Master payload: N/A
+      Slave payload: u64
+
+      Query how many queues the backend supports. This request should be
+      sent only when VHOST_USER_PROTOCOL_F_MQ is set in queried protocol
+      features by VHOST_USER_GET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_SET_VRING_ENABLE
+
+      Id: 18
+      Equivalent ioctl: N/A
+      Master payload: vring state description
+
+      Signal slave to enable or disable corresponding vring.
+      This request should be sent only when VHOST_USER_F_PROTOCOL_FEATURES
+      has been negotiated.
+
+ * VHOST_USER_SEND_RARP
+
+      Id: 19
+      Equivalent ioctl: N/A
+      Master payload: u64
+
+      Ask vhost user backend to broadcast a fake RARP to notify the migration
+      is terminated for guest that does not support GUEST_ANNOUNCE.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
+      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
+      The first 6 bytes of the payload contain the mac address of the guest to
+      allow the vhost user backend to construct and broadcast the fake RARP.
diff --git a/qemu/docs/throttle.txt b/qemu/docs/throttle.txt
new file mode 100644
index 000000000..28204e46c
--- /dev/null
+++ b/qemu/docs/throttle.txt
@@ -0,0 +1,252 @@
+The QEMU throttling infrastructure
+==================================
+Copyright (C) 2016 Igalia, S.L.
+Author: Alberto Garcia <berto@igalia.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+Introduction
+------------
+QEMU includes a throttling module that can be used to set limits to
+I/O operations. The code itself is generic and independent of the I/O
+units, but it is currenly used to limit the number of bytes per second
+and operations per second (IOPS) when performing disk I/O.
+
+This document explains how to use the throttling code in QEMU, and how
+it works internally. The implementation is in throttle.c.
+
+
+Using throttling to limit disk I/O
+----------------------------------
+Two aspects of the disk I/O can be limited: the number of bytes per
+second and the number of operations per second (IOPS). For each one of
+them the user can set a global limit or separate limits for read and
+write operations. This gives us a total of six different parameters.
+
+I/O limits can be set using the throttling.* parameters of -drive, or
+using the QMP 'block_set_io_throttle' command. These are the names of
+the parameters for both cases:
+
+|-----------------------+-----------------------|
+| -drive                | block_set_io_throttle |
+|-----------------------+-----------------------|
+| throttling.iops-total | iops                  |
+| throttling.iops-read  | iops_rd               |
+| throttling.iops-write | iops_wr               |
+| throttling.bps-total  | bps                   |
+| throttling.bps-read   | bps_rd                |
+| throttling.bps-write  | bps_wr                |
+|-----------------------+-----------------------|
+
+It is possible to set limits for both IOPS and bps and the same time,
+and for each case we can decide whether to have separate read and
+write limits or not, but note that if iops-total is set then neither
+iops-read nor iops-write can be set. The same applies to bps-total and
+bps-read/write.
+
+The default value of these parameters is 0, and it means 'unlimited'.
+
+In its most basic usage, the user can add a drive to QEMU with a limit
+of 100 IOPS with the following -drive line:
+
+   -drive file=hd0.qcow2,throttling.iops-total=100
+
+We can do the same using QMP. In this case all these parameters are
+mandatory, so we must set to 0 the ones that we don't want to limit:
+
+   { "execute": "block_set_io_throttle",
+     "arguments": {
+        "device": "virtio0",
+        "iops": 100,
+        "iops_rd": 0,
+        "iops_wr": 0,
+        "bps": 0,
+        "bps_rd": 0,
+        "bps_wr": 0
+     }
+   }
+
+
+I/O bursts
+----------
+In addition to the basic limits we have just seen, QEMU allows the
+user to do bursts of I/O for a configurable amount of time. A burst is
+an amount of I/O that can exceed the basic limit. Bursts are useful to
+allow better performance when there are peaks of activity (the OS
+boots, a service needs to be restarted) while keeping the average
+limits lower the rest of the time.
+
+Two parameters control bursts: their length and the maximum amount of
+I/O they allow. These two can be configured separately for each one of
+the six basic parameters described in the previous section, but in
+this section we'll use 'iops-total' as an example.
+
+The I/O limit during bursts is set using 'iops-total-max', and the
+maximum length (in seconds) is set with 'iops-total-max-length'. So if
+we want to configure a drive with a basic limit of 100 IOPS and allow
+bursts of 2000 IOPS for 60 seconds, we would do it like this (the line
+is split for clarity):
+
+   -drive file=hd0.qcow2,
+          throttling.iops-total=100,
+          throttling.iops-total-max=2000,
+          throttling.iops-total-max-length=60
+
+Or, with QMP:
+
+   { "execute": "block_set_io_throttle",
+     "arguments": {
+        "device": "virtio0",
+        "iops": 100,
+        "iops_rd": 0,
+        "iops_wr": 0,
+        "bps": 0,
+        "bps_rd": 0,
+        "bps_wr": 0,
+        "iops_max": 2000,
+        "iops_max_length": 60,
+     }
+   }
+
+With this, the user can perform I/O on hd0.qcow2 at a rate of 2000
+IOPS for 1 minute before it's throttled down to 100 IOPS.
+
+The user will be able to do bursts again if there's a sufficiently
+long period of time with unused I/O (see below for details).
+
+The default value for 'iops-total-max' is 0 and it means that bursts
+are not allowed. 'iops-total-max-length' can only be set if
+'iops-total-max' is set as well, and its default value is 1 second.
+
+Here's the complete list of parameters for configuring bursts:
+
+|----------------------------------+-----------------------|
+| -drive                           | block_set_io_throttle |
+|----------------------------------+-----------------------|
+| throttling.iops-total-max        | iops_max              |
+| throttling.iops-total-max-length | iops_max_length       |
+| throttling.iops-read-max         | iops_rd_max           |
+| throttling.iops-read-max-length  | iops_rd_max_length    |
+| throttling.iops-write-max        | iops_wr_max           |
+| throttling.iops-write-max-length | iops_wr_max_length    |
+| throttling.bps-total-max         | bps_max               |
+| throttling.bps-total-max-length  | bps_max_length        |
+| throttling.bps-read-max          | bps_rd_max            |
+| throttling.bps-read-max-length   | bps_rd_max_length     |
+| throttling.bps-write-max         | bps_wr_max            |
+| throttling.bps-write-max-length  | bps_wr_max_length     |
+|----------------------------------+-----------------------|
+
+
+Controlling the size of I/O operations
+--------------------------------------
+When applying IOPS limits all I/O operations are treated equally
+regardless of their size. This means that the user can take advantage
+of this in order to circumvent the limits and submit one huge I/O
+request instead of several smaller ones.
+
+QEMU provides a setting called throttling.iops-size to prevent this
+from happening. This setting specifies the size (in bytes) of an I/O
+request for accounting purposes. Larger requests will be counted
+proportionally to this size.
+
+For example, if iops-size is set to 4096 then an 8KB request will be
+counted as two, and a 6KB request will be counted as one and a
+half. This only applies to requests larger than iops-size: smaller
+requests will be always counted as one, no matter their size.
+
+The default value of iops-size is 0 and it means that the size of the
+requests is never taken into account when applying IOPS limits.
+
+
+Applying I/O limits to groups of disks
+--------------------------------------
+In all the examples so far we have seen how to apply limits to the I/O
+performed on individual drives, but QEMU allows grouping drives so
+they all share the same limits.
+
+The way it works is that each drive with I/O limits is assigned to a
+group named using the throttling.group parameter. If this parameter is
+not specified, then the device name (i.e. 'virtio0', 'ide0-hd0') will
+be used as the group name.
+
+Limits set using the throttling.* parameters discussed earlier in this
+document apply to the combined I/O of all members of a group.
+
+Consider this example:
+
+   -drive file=hd1.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd2.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd3.qcow2,throttling.iops-total=3000,throttling.group=bar
+   -drive file=hd4.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd5.qcow2,throttling.iops-total=3000,throttling.group=bar
+   -drive file=hd6.qcow2,throttling.iops-total=5000
+
+Here hd1, hd2 and hd4 are all members of a group named 'foo' with a
+combined IOPS limit of 6000, and hd3 and hd5 are members of 'bar'. hd6
+is left alone (technically it is part of a 1-member group).
+
+Limits are applied in a round-robin fashion so if there are concurrent
+I/O requests on several drives of the same group they will be
+distributed evenly.
+
+When I/O limits are applied to an existing drive using the QMP command
+'block_set_io_throttle', the following things need to be taken into
+account:
+
+   - I/O limits are shared within the same group, so new values will
+     affect all members and overwrite the previous settings. In other
+     words: if different limits are applied to members of the same
+     group, the last one wins.
+
+   - If 'group' is unset it is assumed to be the current group of that
+     drive. If the drive is not in a group yet, it will be added to a
+     group named after the device name.
+
+   - If 'group' is set then the drive will be moved to that group if
+     it was member of a different one. In this case the limits
+     specified in the parameters will be applied to the new group
+     only.
+
+   - I/O limits can be disabled by setting all of them to 0. In this
+     case the device will be removed from its group and the rest of
+     its members will not be affected. The 'group' parameter is
+     ignored.
+
+
+The Leaky Bucket algorithm
+--------------------------
+I/O limits in QEMU are implemented using the leaky bucket algorithm
+(specifically the "Leaky bucket as a meter" variant).
+
+This algorithm uses the analogy of a bucket that leaks water
+constantly. The water that gets into the bucket represents the I/O
+that has been performed, and no more I/O is allowed once the bucket is
+full.
+
+To see the way this corresponds to the throttling parameters in QEMU,
+consider the following values:
+
+  iops-total=100
+  iops-total-max=2000
+  iops-total-max-length=60
+
+  - Water leaks from the bucket at a rate of 100 IOPS.
+  - Water can be added to the bucket at a rate of 2000 IOPS.
+  - The size of the bucket is 2000 x 60 = 120000
+  - If 'iops-total-max-length' is unset then the bucket size is 100.
+
+The bucket is initially empty, therefore water can be added until it's
+full at a rate of 2000 IOPS (the burst rate). Once the bucket is full
+we can only add as much water as it leaks, therefore the I/O rate is
+reduced to 100 IOPS. If we add less water than it leaks then the
+bucket will start to empty, allowing for bursts again.
+
+Note that since water is leaking from the bucket even during bursts,
+it will take a bit more than 60 seconds at 2000 IOPS to fill it
+up. After those 60 seconds the bucket will have leaked 60 x 100 =
+6000, allowing for 3 more seconds of I/O at 2000 IOPS.
+
+Also, due to the way the algorithm works, longer burst can be done at
+a lower I/O rate, e.g. 1000 IOPS during 120 seconds.
diff --git a/qemu/docs/tracing.txt b/qemu/docs/tracing.txt
index 7117c5e7d..0bd6b9cf9 100644
--- a/qemu/docs/tracing.txt
+++ b/qemu/docs/tracing.txt
@@ -157,9 +157,9 @@ performance penalty.
 Note that regardless of the selected trace backend, events with the "disable"
 property will be generated with the "nop" backend.
 
-=== Stderr ===
+=== Log ===
 
-The "stderr" backend sends trace events directly to standard error.  This
+The "log" backend sends trace events directly to standard error.  This
 effectively turns trace events into debug printfs.
 
 This is the simplest backend and can be used together with existing code that
@@ -172,9 +172,6 @@ source tree.  It may not be as powerful as platform-specific or third-party
 trace backends but it is portable.  This is the recommended trace backend
 unless you have specific needs for more advanced backends.
 
-The "simple" backend currently does not capture string arguments, it simply
-records the char* pointer value instead of the string that is pointed to.
-
 === Ftrace ===
 
 The "ftrace" backend writes trace data to ftrace marker. This effectively
@@ -258,11 +255,11 @@ is generated to make use in scripts more convenient.  This step can also be
 performed manually after a build in order to change the binary name in the .stp
 probes:
 
-    scripts/tracetool --dtrace --stap \
-                      --binary path/to/qemu-binary \
-                      --target-type system \
-                      --target-name x86_64 \
-                      <trace-events >qemu.stp
+    scripts/tracetool.py --backends=dtrace --format=stap \
+                         --binary path/to/qemu-binary \
+                         --target-type system \
+                         --target-name x86_64 \
+                         <trace-events >qemu.stp
 
 == Trace event properties ==
 
@@ -347,3 +344,44 @@ This will immediately call:
 and will generate the TCG code to call:
 
     void trace_foo(uint8_t a1, uint32_t a2);
+
+=== "vcpu" ===
+
+Identifies events that trace vCPU-specific information. It implicitly adds a
+"CPUState*" argument, and extends the tracing print format to show the vCPU
+information. If used together with the "tcg" property, it adds a second
+"TCGv_env" argument that must point to the per-target global TCG register that
+points to the vCPU when guest code is executed (usually the "cpu_env" variable).
+
+The following example events:
+
+    foo(uint32_t a) "a=%x"
+    vcpu bar(uint32_t a) "a=%x"
+    tcg vcpu baz(uint32_t a) "a=%x", "a=%x"
+
+Can be used as:
+
+    #include "trace-tcg.h"
+    
+    CPUArchState *env;
+    TCGv_ptr cpu_env;
+    
+    void some_disassembly_func(...)
+    {
+        /* trace emitted at this point */
+        trace_foo(0xd1);
+        /* trace emitted at this point */
+        trace_bar(ENV_GET_CPU(env), 0xd2);
+        /* trace emitted at this point (env) and when guest code is executed (cpu_env) */
+        trace_baz_tcg(ENV_GET_CPU(env), cpu_env, 0xd3);
+    }
+
+If the translating vCPU has address 0xc1 and code is later executed by vCPU
+0xc2, this would be an example output:
+
+    // at guest code translation
+    foo a=0xd1
+    bar cpu=0xc1 a=0xd2
+    baz_trans cpu=0xc1 a=0xd3
+    // at guest code execution
+    baz_exec cpu=0xc2 a=0xd3
diff --git a/qemu/docs/virtio-migration.txt b/qemu/docs/virtio-migration.txt
new file mode 100644
index 000000000..cf66458b9
--- /dev/null
+++ b/qemu/docs/virtio-migration.txt
@@ -0,0 +1,106 @@
+Virtio devices and migration
+============================
+
+Copyright 2015 IBM Corp.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+the COPYING file in the top-level directory.
+
+Saving and restoring the state of virtio devices is a bit of a twisty maze,
+for several reasons:
+- state is distributed between several parts:
+  - virtio core, for common fields like features, number of queues, ...
+  - virtio transport (pci, ccw, ...), for the different proxy devices and
+    transport specific state (msix vectors, indicators, ...)
+  - virtio device (net, blk, ...), for the different device types and their
+    state (mac address, request queue, ...)
+- most fields are saved via the stream interface; subsequently, subsections
+  have been added to make cross-version migration possible
+
+This file attempts to document the current procedure and point out some
+caveats.
+
+
+Save state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    save() function registered
+                                                    via register_savevm()
+virtio_save()                                       <----------
+             ------>      save_config()
+                          - save proxy device
+                          - save transport-specific
+                            device fields
+- save common device
+  fields
+- save common virtqueue
+  fields
+             ------>      save_queue()
+                          - save transport-specific
+                            virtqueue fields
+             ------>                               save_device()
+                                                   - save device-specific
+                                                     fields
+- save subsections
+  - device endianness,
+    if changed from
+    default endianness
+  - 64 bit features, if
+    any high feature bit
+    is set
+  - virtio-1 virtqueue
+    fields, if VERSION_1
+    is set
+
+
+Load state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    load() function registered
+                                                    via register_savevm()
+virtio_load()                                       <----------
+             ------>      load_config()
+                          - load proxy device
+                          - load transport-specific
+                            device fields
+- load common device
+  fields
+- load common virtqueue
+  fields
+             ------>      load_queue()
+                          - load transport-specific
+                            virtqueue fields
+- notify guest
+             ------>                               load_device()
+                                                   - load device-specific
+                                                     fields
+- load subsections
+  - device endianness
+  - 64 bit features
+  - virtio-1 virtqueue
+    fields
+- sanitize endianness
+- sanitize features
+- virtqueue index sanity
+  check
+                                                   - feature-dependent setup
+
+
+Implications of this setup
+==========================
+
+Devices need to be careful in their state processing during load: The
+load_device() procedure is invoked by the core before subsections have
+been loaded. Any code that depends on information transmitted in subsections
+therefore has to be invoked in the device's load() function _after_
+virtio_load() returned (like e.g. code depending on features).
+
+Any extension of the state being migrated should be done in subsections
+added to the core for compatibility reasons. If transport or device specific
+state is added, core needs to invoke a callback from the new subsection.
diff --git a/qemu/docs/win32-qemu-event.promela b/qemu/docs/win32-qemu-event.promela
new file mode 100644
index 000000000..c446a7155
--- /dev/null
+++ b/qemu/docs/win32-qemu-event.promela
@@ -0,0 +1,98 @@
+/*
+ * This model describes the implementation of QemuEvent in
+ * util/qemu-thread-win32.c.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This file is in the public domain.  If you really want a license,
+ * the WTFPL will do.
+ *
+ * To verify it:
+ *     spin -a docs/event.promela
+ *     gcc -O2 pan.c -DSAFETY
+ *     ./a.out
+ */
+
+bool event;
+int value;
+
+/* Primitives for a Win32 event */
+#define RAW_RESET event = false
+#define RAW_SET   event = true
+#define RAW_WAIT  do :: event -> break; od
+
+#if 0
+/* Basic sanity checking: test the Win32 event primitives */
+#define RESET     RAW_RESET
+#define SET       RAW_SET
+#define WAIT      RAW_WAIT
+#else
+/* Full model: layer a userspace-only fast path on top of the RAW_*
+ * primitives.  SET/RESET/WAIT have exactly the same semantics as
+ * RAW_SET/RAW_RESET/RAW_WAIT, but try to avoid invoking them.
+ */
+#define EV_SET 0
+#define EV_FREE 1
+#define EV_BUSY -1
+
+int state = EV_FREE;
+
+int xchg_result;
+#define SET   if :: state != EV_SET ->                                      \
+                    atomic { /* xchg_result=xchg(state, EV_SET) */          \
+                        xchg_result = state;                                \
+                        state = EV_SET;                                     \
+                    }                                                       \
+                    if :: xchg_result == EV_BUSY -> RAW_SET;                \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+
+#define RESET if :: state == EV_SET -> atomic { state = state | EV_FREE; }  \
+                 :: else            -> skip;                                \
+              fi
+
+int tmp1, tmp2;
+#define WAIT  tmp1 = state;                                                 \
+              if :: tmp1 != EV_SET ->                                       \
+                    if :: tmp1 == EV_FREE ->                                \
+                          RAW_RESET;                                        \
+                          atomic { /* tmp2=cas(state, EV_FREE, EV_BUSY) */  \
+                              tmp2 = state;                                 \
+                              if :: tmp2 == EV_FREE -> state = EV_BUSY;     \
+                                 :: else            -> skip;                \
+                              fi;                                           \
+                          }                                                 \
+                          if :: tmp2 == EV_SET -> tmp1 = EV_SET;            \
+                             :: else           -> tmp1 = EV_BUSY;           \
+                          fi;                                               \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                    assert(tmp1 != EV_FREE);                                \
+                    if :: tmp1 == EV_BUSY -> RAW_WAIT;                      \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+#endif
+
+active proctype waiter()
+{
+     if
+         :: !value ->
+             RESET;
+             if
+                 :: !value -> WAIT;
+                 :: else   -> skip;
+             fi;
+        :: else -> skip;
+    fi;
+    assert(value);
+}
+
+active proctype notifier()
+{
+    value = true;
+    SET;
+}
diff --git a/qemu/docs/writing-qmp-commands.txt b/qemu/docs/writing-qmp-commands.txt
index ab1fdd36b..59aa77ae2 100644
--- a/qemu/docs/writing-qmp-commands.txt
+++ b/qemu/docs/writing-qmp-commands.txt
@@ -122,12 +122,12 @@ There are a few things to be noticed:
 Now a little hack is needed. As we're still using the old QMP server we need
 to add the new command to its internal dispatch table. This step won't be
 required in the near future. Open the qmp-commands.hx file and add the
-following in the botton:
+following at the bottom:
 
     {
         .name       = "hello-world",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 You're done. Now build qemu, run it as suggested in the "Testing" section,
@@ -179,7 +179,7 @@ The last step is to update the qmp-commands.hx file:
     {
         .name       = "hello-world",
         .args_type  = "message:s?",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 Notice that the "args_type" member got our "message" argument. The character
@@ -210,7 +210,7 @@ if you don't see these strings, then something went wrong.
 === Errors ===
 
 QMP commands should use the error interface exported by the error.h header
-file. Basically, errors are set by calling the error_set() function.
+file. Basically, most errors are set by calling the error_setg() function.
 
 Let's say we don't accept the string "message" to contain the word "love". If
 it does contain it, we want the "hello-world" command to return an error:
@@ -219,8 +219,7 @@ void qmp_hello_world(bool has_message, const char *message, Error **errp)
 {
     if (has_message) {
         if (strstr(message, "love")) {
-            error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                      "the word 'love' is not allowed");
+            error_setg(errp, "the word 'love' is not allowed");
             return;
         }
         printf("%s\n", message);
@@ -229,10 +228,8 @@ void qmp_hello_world(bool has_message, const char *message, Error **errp)
     }
 }
 
-The first argument to the error_set() function is the Error pointer to pointer,
-which is passed to all QMP functions. The second argument is a ErrorClass
-value, which should be ERROR_CLASS_GENERIC_ERROR most of the time (more
-details about error classes are given below). The third argument is a human
+The first argument to the error_setg() function is the Error pointer
+to pointer, which is passed to all QMP functions. The next argument is a human
 description of the error, this is a free-form printf-like string.
 
 Let's test the example above. Build qemu, run it as defined in the "Testing"
@@ -249,8 +246,9 @@ The QMP server's response should be:
     }
 }
 
-As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR. There
-are two exceptions to this rule:
+As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR
+(done by default when using error_setg()). There are two exceptions to
+this rule:
 
  1. A non-generic ErrorClass value exists* for the failure you want to report
     (eg. DeviceNotFound)
@@ -259,8 +257,8 @@ are two exceptions to this rule:
     want to report, hence you have to add a new ErrorClass value so that they
     can check for it
 
-If the failure you want to report doesn't fall in one of the two cases above,
-just report ERROR_CLASS_GENERIC_ERROR.
+If the failure you want to report falls into one of the two cases above,
+use error_set() with a second argument of an ErrorClass value.
 
  * All existing ErrorClass values are defined in the qapi-schema.json file
 
@@ -461,7 +459,7 @@ The last step is to add the correspoding entry in the qmp-commands.hx file:
     {
         .name       = "query-alarm-clock",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_clock,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_clock,
     },
 
 Time to test the new command. Build qemu, run it as described in the "Testing"
@@ -607,7 +605,7 @@ To test this you have to add the corresponding qmp-commands.hx entry:
     {
         .name       = "query-alarm-methods",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_methods,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_methods,
     },
 
 Now Build qemu, run it as explained in the "Testing" section and try our new
diff --git a/qemu/dump.c b/qemu/dump.c
index 78b7d843c..9726f1f47 100644
--- a/qemu/dump.c
+++ b/qemu/dump.c
@@ -11,7 +11,8 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "elf.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
@@ -22,9 +23,9 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/memory_mapping.h"
 #include "sysemu/cpus.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qmp-commands.h"
+#include "qapi-event.h"
 
 #include <zlib.h>
 #ifdef CONFIG_LZO
@@ -82,12 +83,6 @@ static int dump_cleanup(DumpState *s)
     return 0;
 }
 
-static void dump_error(DumpState *s, const char *reason, Error **errp)
-{
-    dump_cleanup(s);
-    error_setg(errp, "%s", reason);
-}
-
 static int fd_write_vmcore(const void *buf, size_t size, void *opaque)
 {
     DumpState *s = opaque;
@@ -128,7 +123,7 @@ static void write_elf64_header(DumpState *s, Error **errp)
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header", errp);
+        error_setg(errp, "dump: failed to write elf header");
     }
 }
 
@@ -159,7 +154,7 @@ static void write_elf32_header(DumpState *s, Error **errp)
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header", errp);
+        error_setg(errp, "dump: failed to write elf header");
     }
 }
 
@@ -182,7 +177,7 @@ static void write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -205,7 +200,7 @@ static void write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -225,7 +220,7 @@ static void write_elf64_note(DumpState *s, Error **errp)
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -245,7 +240,7 @@ static void write_elf64_notes(WriteCoreDumpFunction f, DumpState *s,
         id = cpu_index(cpu);
         ret = cpu_write_elf64_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes", errp);
+            error_setg(errp, "dump: failed to write elf notes");
             return;
         }
     }
@@ -253,7 +248,7 @@ static void write_elf64_notes(WriteCoreDumpFunction f, DumpState *s,
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf64_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status", errp);
+            error_setg(errp, "dump: failed to write CPU status");
             return;
         }
     }
@@ -275,7 +270,7 @@ static void write_elf32_note(DumpState *s, Error **errp)
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -290,7 +285,7 @@ static void write_elf32_notes(WriteCoreDumpFunction f, DumpState *s,
         id = cpu_index(cpu);
         ret = cpu_write_elf32_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes", errp);
+            error_setg(errp, "dump: failed to write elf notes");
             return;
         }
     }
@@ -298,7 +293,7 @@ static void write_elf32_notes(WriteCoreDumpFunction f, DumpState *s,
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf32_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status", errp);
+            error_setg(errp, "dump: failed to write CPU status");
             return;
         }
     }
@@ -326,7 +321,7 @@ static void write_elf_section(DumpState *s, int type, Error **errp)
 
     ret = fd_write_vmcore(&shdr, shdr_size, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write section header table", errp);
+        error_setg(errp, "dump: failed to write section header table");
     }
 }
 
@@ -336,7 +331,9 @@ static void write_data(DumpState *s, void *buf, int length, Error **errp)
 
     ret = fd_write_vmcore(buf, length, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to save memory", errp);
+        error_setg(errp, "dump: failed to save memory");
+    } else {
+        s->written_size += length;
     }
 }
 
@@ -347,18 +344,18 @@ static void write_memory(DumpState *s, GuestPhysBlock *block, ram_addr_t start,
     int64_t i;
     Error *local_err = NULL;
 
-    for (i = 0; i < size / TARGET_PAGE_SIZE; i++) {
-        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                   TARGET_PAGE_SIZE, &local_err);
+    for (i = 0; i < size / s->dump_info.page_size; i++) {
+        write_data(s, block->host_addr + start + i * s->dump_info.page_size,
+                   s->dump_info.page_size, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
     }
 
-    if ((size % TARGET_PAGE_SIZE) != 0) {
-        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                   size % TARGET_PAGE_SIZE, &local_err);
+    if ((size % s->dump_info.page_size) != 0) {
+        write_data(s, block->host_addr + start + i * s->dump_info.page_size,
+                   size % s->dump_info.page_size, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
@@ -568,11 +565,6 @@ static void dump_begin(DumpState *s, Error **errp)
     }
 }
 
-static void dump_completed(DumpState *s)
-{
-    dump_cleanup(s);
-}
-
 static int get_next_block(DumpState *s, GuestPhysBlock *block)
 {
     while (1) {
@@ -624,8 +616,6 @@ static void dump_iterate(DumpState *s, Error **errp)
         }
 
     } while (!get_next_block(s, block));
-
-    dump_completed(s);
 }
 
 static void create_vmcore(DumpState *s, Error **errp)
@@ -737,7 +727,7 @@ static void create_header32(DumpState *s, Error **errp)
 
     strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
     dh->header_version = cpu_to_dump32(s, 6);
-    block_size = TARGET_PAGE_SIZE;
+    block_size = s->dump_info.page_size;
     dh->block_size = cpu_to_dump32(s, block_size);
     sub_hdr_size = sizeof(struct KdumpSubHeader32) + s->note_size;
     sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
@@ -765,7 +755,7 @@ static void create_header32(DumpState *s, Error **errp)
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header", errp);
+        error_setg(errp, "dump: failed to write disk dump header");
         goto out;
     }
 
@@ -775,7 +765,7 @@ static void create_header32(DumpState *s, Error **errp)
 
     /* 64bit max_mapnr_64 */
     kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
-    kh->phys_base = cpu_to_dump32(s, PHYS_BASE);
+    kh->phys_base = cpu_to_dump32(s, s->dump_info.phys_base);
     kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
 
     offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
@@ -784,7 +774,7 @@ static void create_header32(DumpState *s, Error **errp)
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header", errp);
+        error_setg(errp, "dump: failed to write kdump sub header");
         goto out;
     }
 
@@ -800,7 +790,7 @@ static void create_header32(DumpState *s, Error **errp)
     }
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes", errp);
+        error_setg(errp, "dump: failed to write notes");
         goto out;
     }
 
@@ -837,7 +827,7 @@ static void create_header64(DumpState *s, Error **errp)
 
     strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
     dh->header_version = cpu_to_dump32(s, 6);
-    block_size = TARGET_PAGE_SIZE;
+    block_size = s->dump_info.page_size;
     dh->block_size = cpu_to_dump32(s, block_size);
     sub_hdr_size = sizeof(struct KdumpSubHeader64) + s->note_size;
     sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
@@ -865,7 +855,7 @@ static void create_header64(DumpState *s, Error **errp)
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header", errp);
+        error_setg(errp, "dump: failed to write disk dump header");
         goto out;
     }
 
@@ -875,7 +865,7 @@ static void create_header64(DumpState *s, Error **errp)
 
     /* 64bit max_mapnr_64 */
     kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
-    kh->phys_base = cpu_to_dump64(s, PHYS_BASE);
+    kh->phys_base = cpu_to_dump64(s, s->dump_info.phys_base);
     kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
 
     offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
@@ -884,7 +874,7 @@ static void create_header64(DumpState *s, Error **errp)
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header", errp);
+        error_setg(errp, "dump: failed to write kdump sub header");
         goto out;
     }
 
@@ -901,7 +891,7 @@ static void create_header64(DumpState *s, Error **errp)
 
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes", errp);
+        error_setg(errp, "dump: failed to write notes");
         goto out;
     }
 
@@ -933,6 +923,11 @@ static void write_dump_header(DumpState *s, Error **errp)
     }
 }
 
+static size_t dump_bitmap_get_bufsize(DumpState *s)
+{
+    return s->dump_info.page_size;
+}
+
 /*
  * set dump_bitmap sequencely. the bit before last_pfn is not allowed to be
  * rewritten, so if need to set the first bit, set last_pfn and pfn to 0.
@@ -946,6 +941,8 @@ static int set_dump_bitmap(uint64_t last_pfn, uint64_t pfn, bool value,
     off_t old_offset, new_offset;
     off_t offset_bitmap1, offset_bitmap2;
     uint32_t byte, bit;
+    size_t bitmap_bufsize = dump_bitmap_get_bufsize(s);
+    size_t bits_per_buf = bitmap_bufsize * CHAR_BIT;
 
     /* should not set the previous place */
     assert(last_pfn <= pfn);
@@ -956,14 +953,14 @@ static int set_dump_bitmap(uint64_t last_pfn, uint64_t pfn, bool value,
      * making new_offset be bigger than old_offset can also sync remained data
      * into vmcore.
      */
-    old_offset = BUFSIZE_BITMAP * (last_pfn / PFN_BUFBITMAP);
-    new_offset = BUFSIZE_BITMAP * (pfn / PFN_BUFBITMAP);
+    old_offset = bitmap_bufsize * (last_pfn / bits_per_buf);
+    new_offset = bitmap_bufsize * (pfn / bits_per_buf);
 
     while (old_offset < new_offset) {
         /* calculate the offset and write dump_bitmap */
         offset_bitmap1 = s->offset_dump_bitmap + old_offset;
         if (write_buffer(s->fd, offset_bitmap1, buf,
-                         BUFSIZE_BITMAP) < 0) {
+                         bitmap_bufsize) < 0) {
             return -1;
         }
 
@@ -971,17 +968,17 @@ static int set_dump_bitmap(uint64_t last_pfn, uint64_t pfn, bool value,
         offset_bitmap2 = s->offset_dump_bitmap + s->len_dump_bitmap +
                          old_offset;
         if (write_buffer(s->fd, offset_bitmap2, buf,
-                         BUFSIZE_BITMAP) < 0) {
+                         bitmap_bufsize) < 0) {
             return -1;
         }
 
-        memset(buf, 0, BUFSIZE_BITMAP);
-        old_offset += BUFSIZE_BITMAP;
+        memset(buf, 0, bitmap_bufsize);
+        old_offset += bitmap_bufsize;
     }
 
     /* get the exact place of the bit in the buf, and set it */
-    byte = (pfn % PFN_BUFBITMAP) / CHAR_BIT;
-    bit = (pfn % PFN_BUFBITMAP) % CHAR_BIT;
+    byte = (pfn % bits_per_buf) / CHAR_BIT;
+    bit = (pfn % bits_per_buf) % CHAR_BIT;
     if (value) {
         buf[byte] |= 1u << bit;
     } else {
@@ -991,6 +988,20 @@ static int set_dump_bitmap(uint64_t last_pfn, uint64_t pfn, bool value,
     return 0;
 }
 
+static uint64_t dump_paddr_to_pfn(DumpState *s, uint64_t addr)
+{
+    int target_page_shift = ctz32(s->dump_info.page_size);
+
+    return (addr >> target_page_shift) - ARCH_PFN_OFFSET;
+}
+
+static uint64_t dump_pfn_to_paddr(DumpState *s, uint64_t pfn)
+{
+    int target_page_shift = ctz32(s->dump_info.page_size);
+
+    return (pfn + ARCH_PFN_OFFSET) << target_page_shift;
+}
+
 /*
  * exam every page and return the page frame number and the address of the page.
  * bufptr can be NULL. note: the blocks here is supposed to reflect guest-phys
@@ -1001,16 +1012,16 @@ static bool get_next_page(GuestPhysBlock **blockptr, uint64_t *pfnptr,
                           uint8_t **bufptr, DumpState *s)
 {
     GuestPhysBlock *block = *blockptr;
-    hwaddr addr;
+    hwaddr addr, target_page_mask = ~((hwaddr)s->dump_info.page_size - 1);
     uint8_t *buf;
 
     /* block == NULL means the start of the iteration */
     if (!block) {
         block = QTAILQ_FIRST(&s->guest_phys_blocks.head);
         *blockptr = block;
-        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
-        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
-        *pfnptr = paddr_to_pfn(block->target_start);
+        assert((block->target_start & ~target_page_mask) == 0);
+        assert((block->target_end & ~target_page_mask) == 0);
+        *pfnptr = dump_paddr_to_pfn(s, block->target_start);
         if (bufptr) {
             *bufptr = block->host_addr;
         }
@@ -1018,10 +1029,10 @@ static bool get_next_page(GuestPhysBlock **blockptr, uint64_t *pfnptr,
     }
 
     *pfnptr = *pfnptr + 1;
-    addr = pfn_to_paddr(*pfnptr);
+    addr = dump_pfn_to_paddr(s, *pfnptr);
 
     if ((addr >= block->target_start) &&
-        (addr + TARGET_PAGE_SIZE <= block->target_end)) {
+        (addr + s->dump_info.page_size <= block->target_end)) {
         buf = block->host_addr + (addr - block->target_start);
     } else {
         /* the next page is in the next block */
@@ -1030,9 +1041,9 @@ static bool get_next_page(GuestPhysBlock **blockptr, uint64_t *pfnptr,
         if (!block) {
             return false;
         }
-        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
-        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
-        *pfnptr = paddr_to_pfn(block->target_start);
+        assert((block->target_start & ~target_page_mask) == 0);
+        assert((block->target_end & ~target_page_mask) == 0);
+        *pfnptr = dump_paddr_to_pfn(s, block->target_start);
         buf = block->host_addr;
     }
 
@@ -1050,9 +1061,11 @@ static void write_dump_bitmap(DumpState *s, Error **errp)
     void *dump_bitmap_buf;
     size_t num_dumpable;
     GuestPhysBlock *block_iter = NULL;
+    size_t bitmap_bufsize = dump_bitmap_get_bufsize(s);
+    size_t bits_per_buf = bitmap_bufsize * CHAR_BIT;
 
     /* dump_bitmap_buf is used to store dump_bitmap temporarily */
-    dump_bitmap_buf = g_malloc0(BUFSIZE_BITMAP);
+    dump_bitmap_buf = g_malloc0(bitmap_bufsize);
 
     num_dumpable = 0;
     last_pfn = 0;
@@ -1064,7 +1077,7 @@ static void write_dump_bitmap(DumpState *s, Error **errp)
     while (get_next_page(&block_iter, &pfn, NULL, s)) {
         ret = set_dump_bitmap(last_pfn, pfn, true, dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to set dump_bitmap", errp);
+            error_setg(errp, "dump: failed to set dump_bitmap");
             goto out;
         }
 
@@ -1074,14 +1087,14 @@ static void write_dump_bitmap(DumpState *s, Error **errp)
 
     /*
      * set_dump_bitmap will always leave the recently set bit un-sync. Here we
-     * set last_pfn + PFN_BUFBITMAP to 0 and those set but un-sync bit will be
-     * synchronized into vmcore.
+     * set the remaining bits from last_pfn to the end of the bitmap buffer to
+     * 0. With those set, the un-sync bit will be synchronized into the vmcore.
      */
     if (num_dumpable > 0) {
-        ret = set_dump_bitmap(last_pfn, last_pfn + PFN_BUFBITMAP, false,
+        ret = set_dump_bitmap(last_pfn, last_pfn + bits_per_buf, false,
                               dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to sync dump_bitmap", errp);
+            error_setg(errp, "dump: failed to sync dump_bitmap");
             goto out;
         }
     }
@@ -1098,8 +1111,8 @@ static void prepare_data_cache(DataCache *data_cache, DumpState *s,
 {
     data_cache->fd = s->fd;
     data_cache->data_size = 0;
-    data_cache->buf_size = BUFSIZE_DATA_CACHE;
-    data_cache->buf = g_malloc0(BUFSIZE_DATA_CACHE);
+    data_cache->buf_size = 4 * dump_bitmap_get_bufsize(s);
+    data_cache->buf = g_malloc0(data_cache->buf_size);
     data_cache->offset = offset;
 }
 
@@ -1193,7 +1206,7 @@ static void write_dump_pages(DumpState *s, Error **errp)
     prepare_data_cache(&page_data, s, offset_data);
 
     /* prepare buffer to store compressed data */
-    len_buf_out = get_len_buf_out(TARGET_PAGE_SIZE, s->flag_compress);
+    len_buf_out = get_len_buf_out(s->dump_info.page_size, s->flag_compress);
     assert(len_buf_out != 0);
 
 #ifdef CONFIG_LZO
@@ -1206,19 +1219,19 @@ static void write_dump_pages(DumpState *s, Error **errp)
      * init zero page's page_desc and page_data, because every zero page
      * uses the same page_data
      */
-    pd_zero.size = cpu_to_dump32(s, TARGET_PAGE_SIZE);
+    pd_zero.size = cpu_to_dump32(s, s->dump_info.page_size);
     pd_zero.flags = cpu_to_dump32(s, 0);
     pd_zero.offset = cpu_to_dump64(s, offset_data);
     pd_zero.page_flags = cpu_to_dump64(s, 0);
-    buf = g_malloc0(TARGET_PAGE_SIZE);
-    ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+    buf = g_malloc0(s->dump_info.page_size);
+    ret = write_cache(&page_data, buf, s->dump_info.page_size, false);
     g_free(buf);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write page data (zero page)", errp);
+        error_setg(errp, "dump: failed to write page data (zero page)");
         goto out;
     }
 
-    offset_data += TARGET_PAGE_SIZE;
+    offset_data += s->dump_info.page_size;
 
     /*
      * dump memory to vmcore page by page. zero page will all be resided in the
@@ -1226,11 +1239,11 @@ static void write_dump_pages(DumpState *s, Error **errp)
      */
     while (get_next_page(&block_iter, &pfn_iter, &buf, s)) {
         /* check zero page */
-        if (is_zero_page(buf, TARGET_PAGE_SIZE)) {
+        if (is_zero_page(buf, s->dump_info.page_size)) {
             ret = write_cache(&page_desc, &pd_zero, sizeof(PageDescriptor),
                               false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc", errp);
+                error_setg(errp, "dump: failed to write page desc");
                 goto out;
             }
         } else {
@@ -1248,56 +1261,57 @@ static void write_dump_pages(DumpState *s, Error **errp)
              size_out = len_buf_out;
              if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
                     (compress2(buf_out, (uLongf *)&size_out, buf,
-                               TARGET_PAGE_SIZE, Z_BEST_SPEED) == Z_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                               s->dump_info.page_size, Z_BEST_SPEED) == Z_OK) &&
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_ZLIB);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #ifdef CONFIG_LZO
             } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
-                    (lzo1x_1_compress(buf, TARGET_PAGE_SIZE, buf_out,
+                    (lzo1x_1_compress(buf, s->dump_info.page_size, buf_out,
                     (lzo_uint *)&size_out, wrkmem) == LZO_E_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_LZO);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #endif
 #ifdef CONFIG_SNAPPY
             } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
-                    (snappy_compress((char *)buf, TARGET_PAGE_SIZE,
+                    (snappy_compress((char *)buf, s->dump_info.page_size,
                     (char *)buf_out, &size_out) == SNAPPY_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_SNAPPY);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #endif
             } else {
                 /*
                  * fall back to save in plaintext, size_out should be
-                 * assigned TARGET_PAGE_SIZE
+                 * assigned the target's page size
                  */
                 pd.flags = cpu_to_dump32(s, 0);
-                size_out = TARGET_PAGE_SIZE;
+                size_out = s->dump_info.page_size;
                 pd.size = cpu_to_dump32(s, size_out);
 
-                ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+                ret = write_cache(&page_data, buf,
+                                  s->dump_info.page_size, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
             }
@@ -1309,20 +1323,21 @@ static void write_dump_pages(DumpState *s, Error **errp)
 
             ret = write_cache(&page_desc, &pd, sizeof(PageDescriptor), false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc", errp);
+                error_setg(errp, "dump: failed to write page desc");
                 goto out;
             }
         }
+        s->written_size += s->dump_info.page_size;
     }
 
     ret = write_cache(&page_desc, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_desc", errp);
+        error_setg(errp, "dump: failed to sync cache for page_desc");
         goto out;
     }
     ret = write_cache(&page_data, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_data", errp);
+        error_setg(errp, "dump: failed to sync cache for page_data");
         goto out;
     }
 
@@ -1366,7 +1381,7 @@ static void create_kdump_vmcore(DumpState *s, Error **errp)
 
     ret = write_start_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write start flat header", errp);
+        error_setg(errp, "dump: failed to write start flat header");
         return;
     }
 
@@ -1390,11 +1405,9 @@ static void create_kdump_vmcore(DumpState *s, Error **errp)
 
     ret = write_end_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write end flat header", errp);
+        error_setg(errp, "dump: failed to write end flat header");
         return;
     }
-
-    dump_completed(s);
 }
 
 static ram_addr_t get_start_block(DumpState *s)
@@ -1430,7 +1443,45 @@ static void get_max_mapnr(DumpState *s)
     GuestPhysBlock *last_block;
 
     last_block = QTAILQ_LAST(&s->guest_phys_blocks.head, GuestPhysBlockHead);
-    s->max_mapnr = paddr_to_pfn(last_block->target_end);
+    s->max_mapnr = dump_paddr_to_pfn(s, last_block->target_end);
+}
+
+static DumpState dump_state_global = { .status = DUMP_STATUS_NONE };
+
+static void dump_state_prepare(DumpState *s)
+{
+    /* zero the struct, setting status to active */
+    *s = (DumpState) { .status = DUMP_STATUS_ACTIVE };
+}
+
+bool dump_in_progress(void)
+{
+    DumpState *state = &dump_state_global;
+    return (atomic_read(&state->status) == DUMP_STATUS_ACTIVE);
+}
+
+/* calculate total size of memory to be dumped (taking filter into
+ * acoount.) */
+static int64_t dump_calculate_size(DumpState *s)
+{
+    GuestPhysBlock *block;
+    int64_t size = 0, total = 0, left = 0, right = 0;
+
+    QTAILQ_FOREACH(block, &s->guest_phys_blocks.head, next) {
+        if (s->has_filter) {
+            /* calculate the overlapped region. */
+            left = MAX(s->begin, block->target_start);
+            right = MIN(s->begin + s->length, block->target_end);
+            size = right - left;
+            size = size > 0 ? size : 0;
+        } else {
+            /* count the whole region in */
+            size = (block->target_end - block->target_start);
+        }
+        total += size;
+    }
+
+    return total;
 }
 
 static void dump_init(DumpState *s, int fd, bool has_format,
@@ -1442,6 +1493,10 @@ static void dump_init(DumpState *s, int fd, bool has_format,
     Error *err = NULL;
     int ret;
 
+    s->has_format = has_format;
+    s->format = format;
+    s->written_size = 0;
+
     /* kdump-compressed is conflict with paging and filter */
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
         assert(!paging && !has_filter);
@@ -1472,6 +1527,10 @@ static void dump_init(DumpState *s, int fd, bool has_format,
 
     guest_phys_blocks_init(&s->guest_phys_blocks);
     guest_phys_blocks_append(&s->guest_phys_blocks);
+    s->total_size = dump_calculate_size(s);
+#ifdef DEBUG_DUMP_GUEST_MEMORY
+    fprintf(stderr, "DUMP: total memory to dump: %lu\n", s->total_size);
+#endif
 
     s->start = get_start_block(s);
     if (s->start == -1) {
@@ -1489,6 +1548,10 @@ static void dump_init(DumpState *s, int fd, bool has_format,
         goto cleanup;
     }
 
+    if (!s->dump_info.page_size) {
+        s->dump_info.page_size = TARGET_PAGE_SIZE;
+    }
+
     s->note_size = cpu_get_note_size(s->dump_info.d_class,
                                      s->dump_info.d_machine, nr_cpus);
     if (s->note_size < 0) {
@@ -1512,8 +1575,9 @@ static void dump_init(DumpState *s, int fd, bool has_format,
     get_max_mapnr(s);
 
     uint64_t tmp;
-    tmp = DIV_ROUND_UP(DIV_ROUND_UP(s->max_mapnr, CHAR_BIT), TARGET_PAGE_SIZE);
-    s->len_dump_bitmap = tmp * TARGET_PAGE_SIZE;
+    tmp = DIV_ROUND_UP(DIV_ROUND_UP(s->max_mapnr, CHAR_BIT),
+                       s->dump_info.page_size);
+    s->len_dump_bitmap = tmp * s->dump_info.page_size;
 
     /* init for kdump-compressed format */
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
@@ -1595,8 +1659,60 @@ cleanup:
     dump_cleanup(s);
 }
 
-void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
-                           int64_t begin, bool has_length,
+/* this operation might be time consuming. */
+static void dump_process(DumpState *s, Error **errp)
+{
+    Error *local_err = NULL;
+    DumpQueryResult *result = NULL;
+
+    if (s->has_format && s->format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+        create_kdump_vmcore(s, &local_err);
+    } else {
+        create_vmcore(s, &local_err);
+    }
+
+    /* make sure status is written after written_size updates */
+    smp_wmb();
+    atomic_set(&s->status,
+               (local_err ? DUMP_STATUS_FAILED : DUMP_STATUS_COMPLETED));
+
+    /* send DUMP_COMPLETED message (unconditionally) */
+    result = qmp_query_dump(NULL);
+    /* should never fail */
+    assert(result);
+    qapi_event_send_dump_completed(result, !!local_err, (local_err ? \
+                                   error_get_pretty(local_err) : NULL),
+                                   &error_abort);
+    qapi_free_DumpQueryResult(result);
+
+    error_propagate(errp, local_err);
+    dump_cleanup(s);
+}
+
+static void *dump_thread(void *data)
+{
+    Error *err = NULL;
+    DumpState *s = (DumpState *)data;
+    dump_process(s, &err);
+    error_free(err);
+    return NULL;
+}
+
+DumpQueryResult *qmp_query_dump(Error **errp)
+{
+    DumpQueryResult *result = g_new(DumpQueryResult, 1);
+    DumpState *state = &dump_state_global;
+    result->status = atomic_read(&state->status);
+    /* make sure we are reading status and written_size in order */
+    smp_rmb();
+    result->completed = state->written_size;
+    result->total = state->total_size;
+    return result;
+}
+
+void qmp_dump_guest_memory(bool paging, const char *file,
+                           bool has_detach, bool detach,
+                           bool has_begin, int64_t begin, bool has_length,
                            int64_t length, bool has_format,
                            DumpGuestMemoryFormat format, Error **errp)
 {
@@ -1604,6 +1720,19 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
     int fd = -1;
     DumpState *s;
     Error *local_err = NULL;
+    bool detach_p = false;
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        error_setg(errp, "Dump not allowed during incoming migration.");
+        return;
+    }
+
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
 
     /*
      * kdump-compressed format need the whole memory dumped, so paging or
@@ -1623,6 +1752,9 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
         error_setg(errp, QERR_MISSING_PARAMETER, "begin");
         return;
     }
+    if (has_detach) {
+        detach_p = detach;
+    }
 
     /* check whether lzo/snappy is supported */
 #ifndef CONFIG_LZO
@@ -1661,23 +1793,25 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
         return;
     }
 
-    s = g_malloc0(sizeof(DumpState));
+    s = &dump_state_global;
+    dump_state_prepare(s);
 
     dump_init(s, fd, has_format, format, paging, has_begin,
               begin, length, &local_err);
     if (local_err) {
-        g_free(s);
         error_propagate(errp, local_err);
+        atomic_set(&s->status, DUMP_STATUS_FAILED);
         return;
     }
 
-    if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
-        create_kdump_vmcore(s, errp);
+    if (detach_p) {
+        /* detached dump */
+        qemu_thread_create(&s->dump_thread, "dump_thread", dump_thread,
+                           s, QEMU_THREAD_DETACHED);
     } else {
-        create_vmcore(s, errp);
+        /* sync dump */
+        dump_process(s, errp);
     }
-
-    g_free(s);
 }
 
 DumpGuestMemoryCapability *qmp_query_dump_guest_memory_capability(Error **errp)
diff --git a/qemu/exec.c b/qemu/exec.c
index 0a4a0c5af..c4f903618 100644
--- a/qemu/exec.c
+++ b/qemu/exec.c
@@ -16,13 +16,13 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #ifndef _WIN32
-#include <sys/types.h>
 #include <sys/mman.h>
 #endif
 
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "cpu.h"
 #include "tcg.h"
 #include "hw/hw.h"
@@ -30,7 +30,6 @@
 #include "hw/boards.h"
 #endif
 #include "hw/qdev.h"
-#include "qemu/osdep.h"
 #include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/xen/xen.h"
@@ -49,13 +48,17 @@
 #include "exec/cpu-all.h"
 #include "qemu/rcu_queue.h"
 #include "qemu/main-loop.h"
-#include "exec/cputlb.h"
 #include "translate-all.h"
+#include "sysemu/replay.h"
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "exec/log.h"
 
 #include "qemu/range.h"
+#ifndef _WIN32
+#include "qemu/mmap-alloc.h"
+#endif
 
 //#define DEBUG_SUBPAGE
 
@@ -90,7 +93,7 @@ static MemoryRegion io_mem_unassigned;
 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
 /* current CPU in the current thread. It is only valid inside
    cpu_exec() */
-DEFINE_TLS(CPUState *, current_cpu);
+__thread CPUState *current_cpu;
 /* 0 = Do not count executed instructions.
    1 = Precise instruction counting.
    2 = Adaptive rate instruction counting.  */
@@ -133,6 +136,7 @@ typedef struct PhysPageMap {
 struct AddressSpaceDispatch {
     struct rcu_head rcu;
 
+    MemoryRegionSection *mru_section;
     /* This is a multi-level map on the physical address space.
      * The bottom level has pointers to MemoryRegionSections.
      */
@@ -159,6 +163,21 @@ static void memory_map_init(void);
 static void tcg_commit(MemoryListener *listener);
 
 static MemoryRegion io_mem_watch;
+
+/**
+ * CPUAddressSpace: all the information a CPU needs about an AddressSpace
+ * @cpu: the CPU whose AddressSpace this is
+ * @as: the AddressSpace itself
+ * @memory_dispatch: its dispatch pointer (cached, RCU protected)
+ * @tcg_as_listener: listener for tracking changes to the AddressSpace
+ */
+struct CPUAddressSpace {
+    CPUState *cpu;
+    AddressSpace *as;
+    struct AddressSpaceDispatch *memory_dispatch;
+    MemoryListener tcg_as_listener;
+};
+
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
@@ -290,6 +309,17 @@ static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
     }
 }
 
+static inline bool section_covers_addr(const MemoryRegionSection *section,
+                                       hwaddr addr)
+{
+    /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
+     * the section must cover the entire address space.
+     */
+    return section->size.hi ||
+           range_covers_byte(section->offset_within_address_space,
+                             section->size.lo, addr);
+}
+
 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
                                            Node *nodes, MemoryRegionSection *sections)
 {
@@ -305,9 +335,7 @@ static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
         lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
     }
 
-    if (sections[lp.ptr].size.hi ||
-        range_covers_byte(sections[lp.ptr].offset_within_address_space,
-                          sections[lp.ptr].size.lo, addr)) {
+    if (section_covers_addr(&sections[lp.ptr], addr)) {
         return &sections[lp.ptr];
     } else {
         return &sections[PHYS_SECTION_UNASSIGNED];
@@ -325,14 +353,25 @@ static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
                                                         hwaddr addr,
                                                         bool resolve_subpage)
 {
-    MemoryRegionSection *section;
+    MemoryRegionSection *section = atomic_read(&d->mru_section);
     subpage_t *subpage;
+    bool update;
 
-    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
+    if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
+        section_covers_addr(section, addr)) {
+        update = false;
+    } else {
+        section = phys_page_find(d->phys_map, addr, d->map.nodes,
+                                 d->map.sections);
+        update = true;
+    }
     if (resolve_subpage && section->mr->subpage) {
         subpage = container_of(section->mr, subpage_t, iomem);
         section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
     }
+    if (update) {
+        atomic_set(&d->mru_section, section);
+    }
     return section;
 }
 
@@ -372,18 +411,6 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x
     return section;
 }
 
-static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
-{
-    if (memory_region_is_ram(mr)) {
-        return !(is_write && mr->readonly);
-    }
-    if (memory_region_is_romd(mr)) {
-        return !is_write;
-    }
-
-    return false;
-}
-
 /* Called from RCU critical section */
 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
                                       hwaddr *xlat, hwaddr *plen,
@@ -425,12 +452,13 @@ MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 
 /* Called from RCU critical section */
 MemoryRegionSection *
-address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
+address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
                                   hwaddr *xlat, hwaddr *plen)
 {
     MemoryRegionSection *section;
-    section = address_space_translate_internal(cpu->memory_dispatch,
-                                               addr, xlat, plen, false);
+    AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
+
+    section = address_space_translate_internal(d, addr, xlat, plen, false);
 
     assert(!section->mr->iommu_ops);
     return section;
@@ -478,6 +506,24 @@ static const VMStateDescription vmstate_cpu_common_exception_index = {
     }
 };
 
+static bool cpu_common_crash_occurred_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return cpu->crash_occurred;
+}
+
+static const VMStateDescription vmstate_cpu_common_crash_occurred = {
+    .name = "cpu_common/crash_occurred",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = cpu_common_crash_occurred_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(crash_occurred, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
@@ -491,6 +537,7 @@ const VMStateDescription vmstate_cpu_common = {
     },
     .subsections = (const VMStateDescription*[]) {
         &vmstate_cpu_common_exception_index,
+        &vmstate_cpu_common_crash_occurred,
         NULL
     }
 };
@@ -511,18 +558,38 @@ CPUState *qemu_get_cpu(int index)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
+void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
 {
-    /* We only support one address space per cpu at the moment.  */
-    assert(cpu->as == as);
+    CPUAddressSpace *newas;
 
-    if (cpu->tcg_as_listener) {
-        memory_listener_unregister(cpu->tcg_as_listener);
-    } else {
-        cpu->tcg_as_listener = g_new0(MemoryListener, 1);
+    /* Target code should have set num_ases before calling us */
+    assert(asidx < cpu->num_ases);
+
+    if (asidx == 0) {
+        /* address space 0 gets the convenience alias */
+        cpu->as = as;
+    }
+
+    /* KVM cannot currently support multiple address spaces. */
+    assert(asidx == 0 || !kvm_enabled());
+
+    if (!cpu->cpu_ases) {
+        cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
+    }
+
+    newas = &cpu->cpu_ases[asidx];
+    newas->cpu = cpu;
+    newas->as = as;
+    if (tcg_enabled()) {
+        newas->tcg_as_listener.commit = tcg_commit;
+        memory_listener_register(&newas->tcg_as_listener, as);
     }
-    cpu->tcg_as_listener->commit = tcg_commit;
-    memory_listener_register(cpu->tcg_as_listener, as);
+}
+
+AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
+{
+    /* Return the AddressSpace corresponding to the specified index */
+    return cpu->cpu_ases[asidx].as;
 }
 #endif
 
@@ -577,10 +644,25 @@ void cpu_exec_init(CPUState *cpu, Error **errp)
     int cpu_index;
     Error *local_err = NULL;
 
+    cpu->as = NULL;
+    cpu->num_ases = 0;
+
 #ifndef CONFIG_USER_ONLY
-    cpu->as = &address_space_memory;
     cpu->thread_id = qemu_get_thread_id();
-    cpu_reload_memory_map(cpu);
+
+    /* This is a softmmu CPU object, so create a property for it
+     * so users can wire up its memory. (This can't go in qom/cpu.c
+     * because that file is compiled only once for both user-mode
+     * and system builds.) The default if no link is set up is to use
+     * the system address space.
+     */
+    object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
+                             (Object **)&cpu->memory,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+    cpu->memory = system_memory;
+    object_ref(OBJECT(cpu->memory));
 #endif
 
 #if defined(CONFIG_USER_ONLY)
@@ -601,12 +683,6 @@ void cpu_exec_init(CPUState *cpu, Error **errp)
     if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
         vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
     }
-#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
-    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
-                    cpu_save, cpu_load, cpu->env_ptr);
-    assert(cc->vmsd == NULL);
-    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
-#endif
     if (cc->vmsd != NULL) {
         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
     }
@@ -620,9 +696,11 @@ static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 #else
 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 {
-    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
+    MemTxAttrs attrs;
+    hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
     if (phys != -1) {
-        tb_invalidate_phys_addr(cpu->as,
+        tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
                                 phys | (pc & ~TARGET_PAGE_MASK));
     }
 }
@@ -831,7 +909,7 @@ void cpu_abort(CPUState *cpu, const char *fmt, ...)
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
-    if (qemu_log_enabled()) {
+    if (qemu_log_separate()) {
         qemu_log("qemu: fatal: ");
         qemu_log_vprintf(fmt, ap2);
         qemu_log("\n");
@@ -841,6 +919,7 @@ void cpu_abort(CPUState *cpu, const char *fmt, ...)
     }
     va_end(ap2);
     va_end(ap);
+    replay_finish();
 #if defined(CONFIG_USER_ONLY)
     {
         struct sigaction act;
@@ -860,7 +939,7 @@ static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 
     block = atomic_rcu_read(&ram_list.mru_block);
     if (block && addr - block->offset < block->max_length) {
-        goto found;
+        return block;
     }
     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
         if (addr - block->offset < block->max_length) {
@@ -894,6 +973,7 @@ found:
 
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
 {
+    CPUState *cpu;
     ram_addr_t start1;
     RAMBlock *block;
     ram_addr_t end;
@@ -905,7 +985,9 @@ static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
     block = qemu_get_ram_block(start);
     assert(block == qemu_get_ram_block(end - 1));
     start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
-    cpu_tlb_reset_dirty_all(start1, length);
+    CPU_FOREACH(cpu) {
+        tlb_reset_dirty(cpu, start1, length);
+    }
     rcu_read_unlock();
 }
 
@@ -914,8 +996,9 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
                                               ram_addr_t length,
                                               unsigned client)
 {
+    DirtyMemoryBlocks *blocks;
     unsigned long end, page;
-    bool dirty;
+    bool dirty = false;
 
     if (length == 0) {
         return false;
@@ -923,8 +1006,22 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    dirty = bitmap_test_and_clear_atomic(ram_list.dirty_memory[client],
-                                         page, end - page);
+
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    while (page < end) {
+        unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
+
+        dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
+                                              offset, num);
+        page += num;
+    }
+
+    rcu_read_unlock();
 
     if (dirty && tcg_enabled()) {
         tlb_reset_dirty_range_all(start, length);
@@ -1018,9 +1115,11 @@ static uint16_t phys_section_add(PhysPageMap *map,
 
 static void phys_section_destroy(MemoryRegion *mr)
 {
+    bool have_sub_page = mr->subpage;
+
     memory_region_unref(mr);
 
-    if (mr->subpage) {
+    if (have_sub_page) {
         subpage_t *subpage = container_of(mr, subpage_t, iomem);
         object_unref(OBJECT(&subpage->iomem));
         g_free(subpage);
@@ -1130,87 +1229,83 @@ void qemu_mutex_unlock_ramlist(void)
 }
 
 #ifdef __linux__
-
-#include <sys/vfs.h>
-
-#define HUGETLBFS_MAGIC       0x958458f6
-
-static long gethugepagesize(const char *path, Error **errp)
-{
-    struct statfs fs;
-    int ret;
-
-    do {
-        ret = statfs(path, &fs);
-    } while (ret != 0 && errno == EINTR);
-
-    if (ret != 0) {
-        error_setg_errno(errp, errno, "failed to get page size of file %s",
-                         path);
-        return 0;
-    }
-
-    if (fs.f_type != HUGETLBFS_MAGIC)
-        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
-
-    return fs.f_bsize;
-}
-
 static void *file_ram_alloc(RAMBlock *block,
                             ram_addr_t memory,
                             const char *path,
                             Error **errp)
 {
+    bool unlink_on_error = false;
     char *filename;
     char *sanitized_name;
     char *c;
-    void *area = NULL;
-    int fd;
-    uint64_t hpagesize;
-    Error *local_err = NULL;
-
-    hpagesize = gethugepagesize(path, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        goto error;
-    }
-    block->mr->align = hpagesize;
-
-    if (memory < hpagesize) {
-        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
-                   "or larger than huge page size 0x%" PRIx64,
-                   memory, hpagesize);
-        goto error;
-    }
+    void *area;
+    int fd = -1;
+    int64_t page_size;
 
     if (kvm_enabled() && !kvm_has_sync_mmu()) {
         error_setg(errp,
                    "host lacks kvm mmu notifiers, -mem-path unsupported");
-        goto error;
+        return NULL;
     }
 
-    /* Make name safe to use with mkstemp by replacing '/' with '_'. */
-    sanitized_name = g_strdup(memory_region_name(block->mr));
-    for (c = sanitized_name; *c != '\0'; c++) {
-        if (*c == '/')
-            *c = '_';
+    for (;;) {
+        fd = open(path, O_RDWR);
+        if (fd >= 0) {
+            /* @path names an existing file, use it */
+            break;
+        }
+        if (errno == ENOENT) {
+            /* @path names a file that doesn't exist, create it */
+            fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
+            if (fd >= 0) {
+                unlink_on_error = true;
+                break;
+            }
+        } else if (errno == EISDIR) {
+            /* @path names a directory, create a file there */
+            /* Make name safe to use with mkstemp by replacing '/' with '_'. */
+            sanitized_name = g_strdup(memory_region_name(block->mr));
+            for (c = sanitized_name; *c != '\0'; c++) {
+                if (*c == '/') {
+                    *c = '_';
+                }
+            }
+
+            filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
+                                       sanitized_name);
+            g_free(sanitized_name);
+
+            fd = mkstemp(filename);
+            if (fd >= 0) {
+                unlink(filename);
+                g_free(filename);
+                break;
+            }
+            g_free(filename);
+        }
+        if (errno != EEXIST && errno != EINTR) {
+            error_setg_errno(errp, errno,
+                             "can't open backing store %s for guest RAM",
+                             path);
+            goto error;
+        }
+        /*
+         * Try again on EINTR and EEXIST.  The latter happens when
+         * something else creates the file between our two open().
+         */
     }
 
-    filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
-                               sanitized_name);
-    g_free(sanitized_name);
+    page_size = qemu_fd_getpagesize(fd);
+    block->mr->align = page_size;
 
-    fd = mkstemp(filename);
-    if (fd < 0) {
-        error_setg_errno(errp, errno,
-                         "unable to create backing store for hugepages");
-        g_free(filename);
+    if (memory < page_size) {
+        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
+                   "or larger than page size 0x%" PRIx64,
+                   memory, page_size);
         goto error;
     }
-    unlink(filename);
-    g_free(filename);
 
-    memory = (memory+hpagesize-1) & ~(hpagesize-1);
+    memory = ROUND_UP(memory, page_size);
 
     /*
      * ftruncate is not supported by hugetlbfs in older
@@ -1222,13 +1317,10 @@ static void *file_ram_alloc(RAMBlock *block,
         perror("ftruncate");
     }
 
-    area = mmap(0, memory, PROT_READ | PROT_WRITE,
-                (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
-                fd, 0);
+    area = qemu_ram_mmap(fd, memory, page_size, block->flags & RAM_SHARED);
     if (area == MAP_FAILED) {
         error_setg_errno(errp, errno,
-                         "unable to map backing store for hugepages");
-        close(fd);
+                         "unable to map backing store for guest RAM");
         goto error;
     }
 
@@ -1240,9 +1332,11 @@ static void *file_ram_alloc(RAMBlock *block,
     return area;
 
 error:
-    if (mem_prealloc) {
-        error_report("%s", error_get_pretty(*errp));
-        exit(1);
+    if (unlink_on_error) {
+        unlink(path);
+    }
+    if (fd != -1) {
+        close(fd);
     }
     return NULL;
 }
@@ -1329,6 +1423,11 @@ static RAMBlock *find_ram_block(ram_addr_t addr)
     return NULL;
 }
 
+const char *qemu_ram_get_idstr(RAMBlock *rb)
+{
+    return rb->idstr;
+}
+
 /* Called with iothread lock held.  */
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 {
@@ -1399,7 +1498,7 @@ int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
 
     assert(block);
 
-    newsize = TARGET_PAGE_ALIGN(newsize);
+    newsize = HOST_PAGE_ALIGN(newsize);
 
     if (block->used_length == newsize) {
         return 0;
@@ -1432,11 +1531,53 @@ int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
     return 0;
 }
 
-static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
+/* Called with ram_list.mutex held */
+static void dirty_memory_extend(ram_addr_t old_ram_size,
+                                ram_addr_t new_ram_size)
+{
+    ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
+                                             DIRTY_MEMORY_BLOCK_SIZE);
+    ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
+                                             DIRTY_MEMORY_BLOCK_SIZE);
+    int i;
+
+    /* Only need to extend if block count increased */
+    if (new_num_blocks <= old_num_blocks) {
+        return;
+    }
+
+    for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+        DirtyMemoryBlocks *old_blocks;
+        DirtyMemoryBlocks *new_blocks;
+        int j;
+
+        old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
+        new_blocks = g_malloc(sizeof(*new_blocks) +
+                              sizeof(new_blocks->blocks[0]) * new_num_blocks);
+
+        if (old_num_blocks) {
+            memcpy(new_blocks->blocks, old_blocks->blocks,
+                   old_num_blocks * sizeof(old_blocks->blocks[0]));
+        }
+
+        for (j = old_num_blocks; j < new_num_blocks; j++) {
+            new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
+        }
+
+        atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
+
+        if (old_blocks) {
+            g_free_rcu(old_blocks, rcu);
+        }
+    }
+}
+
+static void ram_block_add(RAMBlock *new_block, Error **errp)
 {
     RAMBlock *block;
     RAMBlock *last_block = NULL;
     ram_addr_t old_ram_size, new_ram_size;
+    Error *err = NULL;
 
     old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
 
@@ -1446,7 +1587,12 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
     if (!new_block->host) {
         if (xen_enabled()) {
             xen_ram_alloc(new_block->offset, new_block->max_length,
-                          new_block->mr);
+                          new_block->mr, &err);
+            if (err) {
+                error_propagate(errp, err);
+                qemu_mutex_unlock_ramlist();
+                return;
+            }
         } else {
             new_block->host = phys_mem_alloc(new_block->max_length,
                                              &new_block->mr->align);
@@ -1455,7 +1601,7 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
                                  "cannot set up guest memory '%s'",
                                  memory_region_name(new_block->mr));
                 qemu_mutex_unlock_ramlist();
-                return -1;
+                return;
             }
             memory_try_enable_merging(new_block->host, new_block->max_length);
         }
@@ -1465,6 +1611,7 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
               (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
     if (new_ram_size > old_ram_size) {
         migration_bitmap_extend(old_ram_size, new_ram_size);
+        dirty_memory_extend(old_ram_size, new_ram_size);
     }
     /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
      * QLIST (which has an RCU-friendly variant) does not have insertion at
@@ -1490,18 +1637,6 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
     ram_list.version++;
     qemu_mutex_unlock_ramlist();
 
-    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
-
-    if (new_ram_size > old_ram_size) {
-        int i;
-
-        /* ram_list.dirty_memory[] is protected by the iothread lock.  */
-        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
-            ram_list.dirty_memory[i] =
-                bitmap_zero_extend(ram_list.dirty_memory[i],
-                                   old_ram_size, new_ram_size);
-       }
-    }
     cpu_physical_memory_set_dirty_range(new_block->offset,
                                         new_block->used_length,
                                         DIRTY_CLIENTS_ALL);
@@ -1514,22 +1649,19 @@ static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
             kvm_setup_guest_memory(new_block->host, new_block->max_length);
         }
     }
-
-    return new_block->offset;
 }
 
 #ifdef __linux__
-ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
-                                    bool share, const char *mem_path,
-                                    Error **errp)
+RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
+                                   bool share, const char *mem_path,
+                                   Error **errp)
 {
     RAMBlock *new_block;
-    ram_addr_t addr;
     Error *local_err = NULL;
 
     if (xen_enabled()) {
         error_setg(errp, "-mem-path not supported with Xen");
-        return -1;
+        return NULL;
     }
 
     if (phys_mem_alloc != qemu_anon_ram_alloc) {
@@ -1540,10 +1672,10 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
          */
         error_setg(errp,
                    "-mem-path not supported with this accelerator");
-        return -1;
+        return NULL;
     }
 
-    size = TARGET_PAGE_ALIGN(size);
+    size = HOST_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     new_block->used_length = size;
@@ -1553,33 +1685,32 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                      mem_path, errp);
     if (!new_block->host) {
         g_free(new_block);
-        return -1;
+        return NULL;
     }
 
-    addr = ram_block_add(new_block, &local_err);
+    ram_block_add(new_block, &local_err);
     if (local_err) {
         g_free(new_block);
         error_propagate(errp, local_err);
-        return -1;
+        return NULL;
     }
-    return addr;
+    return new_block;
 }
 #endif
 
 static
-ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
-                                   void (*resized)(const char*,
-                                                   uint64_t length,
-                                                   void *host),
-                                   void *host, bool resizeable,
-                                   MemoryRegion *mr, Error **errp)
+RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
+                                  void (*resized)(const char*,
+                                                  uint64_t length,
+                                                  void *host),
+                                  void *host, bool resizeable,
+                                  MemoryRegion *mr, Error **errp)
 {
     RAMBlock *new_block;
-    ram_addr_t addr;
     Error *local_err = NULL;
 
-    size = TARGET_PAGE_ALIGN(size);
-    max_size = TARGET_PAGE_ALIGN(max_size);
+    size = HOST_PAGE_ALIGN(size);
+    max_size = HOST_PAGE_ALIGN(max_size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     new_block->resized = resized;
@@ -1594,27 +1725,27 @@ ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
     if (resizeable) {
         new_block->flags |= RAM_RESIZEABLE;
     }
-    addr = ram_block_add(new_block, &local_err);
+    ram_block_add(new_block, &local_err);
     if (local_err) {
         g_free(new_block);
         error_propagate(errp, local_err);
-        return -1;
+        return NULL;
     }
-    return addr;
+    return new_block;
 }
 
-ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr, Error **errp)
 {
     return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
 }
 
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
+RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
 {
     return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
 }
 
-ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
+RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
                                      void (*resized)(const char*,
                                                      uint64_t length,
                                                      void *host),
@@ -1623,25 +1754,6 @@ ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
     return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
 }
 
-void qemu_ram_free_from_ptr(ram_addr_t addr)
-{
-    RAMBlock *block;
-
-    qemu_mutex_lock_ramlist();
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        if (addr == block->offset) {
-            QLIST_REMOVE_RCU(block, next);
-            ram_list.mru_block = NULL;
-            /* Write list before version */
-            smp_wmb();
-            ram_list.version++;
-            g_free_rcu(block, rcu);
-            break;
-        }
-    }
-    qemu_mutex_unlock_ramlist();
-}
-
 static void reclaim_ramblock(RAMBlock *block)
 {
     if (block->flags & RAM_PREALLOC) {
@@ -1650,7 +1762,7 @@ static void reclaim_ramblock(RAMBlock *block)
         xen_invalidate_map_cache_entry(block->host);
 #ifndef _WIN32
     } else if (block->fd >= 0) {
-        munmap(block->host, block->max_length);
+        qemu_ram_munmap(block->host, block->max_length);
         close(block->fd);
 #endif
     } else {
@@ -1659,22 +1771,19 @@ static void reclaim_ramblock(RAMBlock *block)
     g_free(block);
 }
 
-void qemu_ram_free(ram_addr_t addr)
+void qemu_ram_free(RAMBlock *block)
 {
-    RAMBlock *block;
+    if (!block) {
+        return;
+    }
 
     qemu_mutex_lock_ramlist();
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        if (addr == block->offset) {
-            QLIST_REMOVE_RCU(block, next);
-            ram_list.mru_block = NULL;
-            /* Write list before version */
-            smp_wmb();
-            ram_list.version++;
-            call_rcu(block, reclaim_ramblock, rcu);
-            break;
-        }
-    }
+    QLIST_REMOVE_RCU(block, next);
+    ram_list.mru_block = NULL;
+    /* Write list before version */
+    smp_wmb();
+    ram_list.version++;
+    call_rcu(block, reclaim_ramblock, rcu);
     qemu_mutex_unlock_ramlist();
 }
 
@@ -1739,6 +1848,16 @@ int qemu_get_ram_fd(ram_addr_t addr)
     return fd;
 }
 
+void qemu_set_ram_fd(ram_addr_t addr, int fd)
+{
+    RAMBlock *block;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+    block->fd = fd;
+    rcu_read_unlock();
+}
+
 void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
@@ -1756,19 +1875,15 @@ void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
  * or address_space_rw instead. For local memory (e.g. video ram) that the
  * device owns, use memory_region_get_ram_ptr.
  *
- * By the time this function returns, the returned pointer is not protected
- * by RCU anymore.  If the caller is not within an RCU critical section and
- * does not hold the iothread lock, it must have other means of protecting the
- * pointer, such as a reference to the region that includes the incoming
- * ram_addr_t.
+ * Called within RCU critical section.
  */
-void *qemu_get_ram_ptr(ram_addr_t addr)
+void *qemu_get_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
 {
-    RAMBlock *block;
-    void *ptr;
+    RAMBlock *block = ram_block;
 
-    rcu_read_lock();
-    block = qemu_get_ram_block(addr);
+    if (block == NULL) {
+        block = qemu_get_ram_block(addr);
+    }
 
     if (xen_enabled() && block->host == NULL) {
         /* We need to check if the requested address is in the RAM
@@ -1776,56 +1891,59 @@ void *qemu_get_ram_ptr(ram_addr_t addr)
          * In that case just map until the end of the page.
          */
         if (block->offset == 0) {
-            ptr = xen_map_cache(addr, 0, 0);
-            goto unlock;
+            return xen_map_cache(addr, 0, 0);
         }
 
         block->host = xen_map_cache(block->offset, block->max_length, 1);
     }
-    ptr = ramblock_ptr(block, addr - block->offset);
-
-unlock:
-    rcu_read_unlock();
-    return ptr;
+    return ramblock_ptr(block, addr - block->offset);
 }
 
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
  * but takes a size argument.
  *
- * By the time this function returns, the returned pointer is not protected
- * by RCU anymore.  If the caller is not within an RCU critical section and
- * does not hold the iothread lock, it must have other means of protecting the
- * pointer, such as a reference to the region that includes the incoming
- * ram_addr_t.
+ * Called within RCU critical section.
  */
-static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
+static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
+                                 hwaddr *size)
 {
-    void *ptr;
+    RAMBlock *block = ram_block;
+    ram_addr_t offset_inside_block;
     if (*size == 0) {
         return NULL;
     }
-    if (xen_enabled()) {
-        return xen_map_cache(addr, *size, 1);
-    } else {
-        RAMBlock *block;
-        rcu_read_lock();
-        QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-            if (addr - block->offset < block->max_length) {
-                if (addr - block->offset + *size > block->max_length)
-                    *size = block->max_length - addr + block->offset;
-                ptr = ramblock_ptr(block, addr - block->offset);
-                rcu_read_unlock();
-                return ptr;
-            }
+
+    if (block == NULL) {
+        block = qemu_get_ram_block(addr);
+    }
+    offset_inside_block = addr - block->offset;
+    *size = MIN(*size, block->max_length - offset_inside_block);
+
+    if (xen_enabled() && block->host == NULL) {
+        /* We need to check if the requested address is in the RAM
+         * because we don't want to map the entire memory in QEMU.
+         * In that case just map the requested area.
+         */
+        if (block->offset == 0) {
+            return xen_map_cache(addr, *size, 1);
         }
 
-        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
-        abort();
+        block->host = xen_map_cache(block->offset, block->max_length, 1);
     }
+
+    return ramblock_ptr(block, offset_inside_block);
 }
 
-/* Some of the softmmu routines need to translate from a host pointer
- * (typically a TLB entry) back to a ram offset.
+/*
+ * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
+ * in that RAMBlock.
+ *
+ * ptr: Host pointer to look up
+ * round_offset: If true round the result offset down to a page boundary
+ * *ram_addr: set to result ram_addr
+ * *offset: set to result offset within the RAMBlock
+ *
+ * Returns: RAMBlock (or NULL if not found)
  *
  * By the time this function returns, the returned pointer is not protected
  * by RCU anymore.  If the caller is not within an RCU critical section and
@@ -1833,18 +1951,22 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
  * pointer, such as a reference to the region that includes the incoming
  * ram_addr_t.
  */
-MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *ram_addr,
+                                   ram_addr_t *offset)
 {
     RAMBlock *block;
     uint8_t *host = ptr;
-    MemoryRegion *mr;
 
     if (xen_enabled()) {
         rcu_read_lock();
         *ram_addr = xen_ram_addr_from_mapcache(ptr);
-        mr = qemu_get_ram_block(*ram_addr)->mr;
+        block = qemu_get_ram_block(*ram_addr);
+        if (block) {
+            *offset = (host - block->host);
+        }
         rcu_read_unlock();
-        return mr;
+        return block;
     }
 
     rcu_read_lock();
@@ -1867,12 +1989,52 @@ MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
     return NULL;
 
 found:
-    *ram_addr = block->offset + (host - block->host);
-    mr = block->mr;
+    *offset = (host - block->host);
+    if (round_offset) {
+        *offset &= TARGET_PAGE_MASK;
+    }
+    *ram_addr = block->offset + *offset;
     rcu_read_unlock();
-    return mr;
+    return block;
+}
+
+/*
+ * Finds the named RAMBlock
+ *
+ * name: The name of RAMBlock to find
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ */
+RAMBlock *qemu_ram_block_by_name(const char *name)
+{
+    RAMBlock *block;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (!strcmp(name, block->idstr)) {
+            return block;
+        }
+    }
+
+    return NULL;
+}
+
+/* Some of the softmmu routines need to translate from a host pointer
+   (typically a TLB entry) back to a ram offset.  */
+MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+{
+    RAMBlock *block;
+    ram_addr_t offset; /* Not used */
+
+    block = qemu_ram_block_from_host(ptr, false, ram_addr, &offset);
+
+    if (!block) {
+        return NULL;
+    }
+
+    return block->mr;
 }
 
+/* Called within RCU critical section.  */
 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
                                uint64_t val, unsigned size)
 {
@@ -1881,13 +2043,13 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
     }
     switch (size) {
     case 1:
-        stb_p(qemu_get_ram_ptr(ram_addr), val);
+        stb_p(qemu_get_ram_ptr(NULL, ram_addr), val);
         break;
     case 2:
-        stw_p(qemu_get_ram_ptr(ram_addr), val);
+        stw_p(qemu_get_ram_ptr(NULL, ram_addr), val);
         break;
     case 4:
-        stl_p(qemu_get_ram_ptr(ram_addr), val);
+        stl_p(qemu_get_ram_ptr(NULL, ram_addr), val);
         break;
     default:
         abort();
@@ -1900,8 +2062,7 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
     /* we remove the notdirty callback only if the code has been
        flushed */
     if (!cpu_physical_memory_is_clean(ram_addr)) {
-        CPUArchState *env = current_cpu->env_ptr;
-        tlb_set_dirty(env, current_cpu->mem_io_vaddr);
+        tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
     }
 }
 
@@ -1921,6 +2082,7 @@ static const MemoryRegionOps notdirty_mem_ops = {
 static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
 {
     CPUState *cpu = current_cpu;
+    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUArchState *env = cpu->env_ptr;
     target_ulong pc, cs_base;
     target_ulong vaddr;
@@ -1946,6 +2108,11 @@ static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
             wp->hitaddr = vaddr;
             wp->hitattrs = attrs;
             if (!cpu->watchpoint_hit) {
+                if (wp->flags & BP_CPU &&
+                    !cc->debug_check_watchpoint(cpu, wp)) {
+                    wp->flags &= ~BP_WATCHPOINT_HIT;
+                    continue;
+                }
                 cpu->watchpoint_hit = wp;
                 tb_check_watchpoint(cpu);
                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
@@ -1971,17 +2138,19 @@ static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
 {
     MemTxResult res;
     uint64_t data;
+    int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
+    AddressSpace *as = current_cpu->cpu_ases[asidx].as;
 
     check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
     switch (size) {
     case 1:
-        data = address_space_ldub(&address_space_memory, addr, attrs, &res);
+        data = address_space_ldub(as, addr, attrs, &res);
         break;
     case 2:
-        data = address_space_lduw(&address_space_memory, addr, attrs, &res);
+        data = address_space_lduw(as, addr, attrs, &res);
         break;
     case 4:
-        data = address_space_ldl(&address_space_memory, addr, attrs, &res);
+        data = address_space_ldl(as, addr, attrs, &res);
         break;
     default: abort();
     }
@@ -1994,17 +2163,19 @@ static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
                                    MemTxAttrs attrs)
 {
     MemTxResult res;
+    int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
+    AddressSpace *as = current_cpu->cpu_ases[asidx].as;
 
     check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
     switch (size) {
     case 1:
-        address_space_stb(&address_space_memory, addr, val, attrs, &res);
+        address_space_stb(as, addr, val, attrs, &res);
         break;
     case 2:
-        address_space_stw(&address_space_memory, addr, val, attrs, &res);
+        address_space_stw(as, addr, val, attrs, &res);
         break;
     case 4:
-        address_space_stl(&address_space_memory, addr, val, attrs, &res);
+        address_space_stl(as, addr, val, attrs, &res);
         break;
     default: abort();
     }
@@ -2161,9 +2332,11 @@ static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
     return phys_section_add(map, &section);
 }
 
-MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)
+MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
 {
-    AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
+    CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
+    AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
     MemoryRegionSection *sections = d->map.sections;
 
     return sections[index & ~TARGET_PAGE_MASK].mr;
@@ -2222,19 +2395,20 @@ static void mem_commit(MemoryListener *listener)
 
 static void tcg_commit(MemoryListener *listener)
 {
-    CPUState *cpu;
+    CPUAddressSpace *cpuas;
+    AddressSpaceDispatch *d;
 
     /* since each CPU stores ram addresses in its TLB cache, we must
        reset the modified entries */
-    /* XXX: slow ! */
-    CPU_FOREACH(cpu) {
-        /* FIXME: Disentangle the cpu.h circular files deps so we can
-           directly get the right CPU from listener.  */
-        if (cpu->tcg_as_listener != listener) {
-            continue;
-        }
-        cpu_reload_memory_map(cpu);
-    }
+    cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
+    cpu_reloading_memory_map();
+    /* The CPU and TLB are protected by the iothread lock.
+     * We reload the dispatch pointer now because cpu_reloading_memory_map()
+     * may have split the RCU critical section.
+     */
+    d = atomic_rcu_read(&cpuas->as->dispatch);
+    cpuas->memory_dispatch = d;
+    tlb_flush(cpuas->cpu, 1);
 }
 
 void address_space_init_dispatch(AddressSpace *as)
@@ -2374,9 +2548,7 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
     if (l > access_size_max) {
         l = access_size_max;
     }
-    if (l & (l - 1)) {
-        l = 1 << (qemu_fls(l) - 1);
-    }
+    l = pow2floor(l);
 
     return l;
 }
@@ -2404,101 +2576,58 @@ static bool prepare_mmio_access(MemoryRegion *mr)
     return release_lock;
 }
 
-MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
-                             uint8_t *buf, int len, bool is_write)
+/* Called within RCU critical section.  */
+static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
+                                                MemTxAttrs attrs,
+                                                const uint8_t *buf,
+                                                int len, hwaddr addr1,
+                                                hwaddr l, MemoryRegion *mr)
 {
-    hwaddr l;
     uint8_t *ptr;
     uint64_t val;
-    hwaddr addr1;
-    MemoryRegion *mr;
     MemTxResult result = MEMTX_OK;
     bool release_lock = false;
 
-    rcu_read_lock();
-    while (len > 0) {
-        l = len;
-        mr = address_space_translate(as, addr, &addr1, &l, is_write);
-
-        if (is_write) {
-            if (!memory_access_is_direct(mr, is_write)) {
-                release_lock |= prepare_mmio_access(mr);
-                l = memory_access_size(mr, l, addr1);
-                /* XXX: could force current_cpu to NULL to avoid
-                   potential bugs */
-                switch (l) {
-                case 8:
-                    /* 64 bit write access */
-                    val = ldq_p(buf);
-                    result |= memory_region_dispatch_write(mr, addr1, val, 8,
-                                                           attrs);
-                    break;
-                case 4:
-                    /* 32 bit write access */
-                    val = ldl_p(buf);
-                    result |= memory_region_dispatch_write(mr, addr1, val, 4,
-                                                           attrs);
-                    break;
-                case 2:
-                    /* 16 bit write access */
-                    val = lduw_p(buf);
-                    result |= memory_region_dispatch_write(mr, addr1, val, 2,
-                                                           attrs);
-                    break;
-                case 1:
-                    /* 8 bit write access */
-                    val = ldub_p(buf);
-                    result |= memory_region_dispatch_write(mr, addr1, val, 1,
-                                                           attrs);
-                    break;
-                default:
-                    abort();
-                }
-            } else {
-                addr1 += memory_region_get_ram_addr(mr);
-                /* RAM case */
-                ptr = qemu_get_ram_ptr(addr1);
-                memcpy(ptr, buf, l);
-                invalidate_and_set_dirty(mr, addr1, l);
+    for (;;) {
+        if (!memory_access_is_direct(mr, true)) {
+            release_lock |= prepare_mmio_access(mr);
+            l = memory_access_size(mr, l, addr1);
+            /* XXX: could force current_cpu to NULL to avoid
+               potential bugs */
+            switch (l) {
+            case 8:
+                /* 64 bit write access */
+                val = ldq_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 8,
+                                                       attrs);
+                break;
+            case 4:
+                /* 32 bit write access */
+                val = ldl_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 4,
+                                                       attrs);
+                break;
+            case 2:
+                /* 16 bit write access */
+                val = lduw_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 2,
+                                                       attrs);
+                break;
+            case 1:
+                /* 8 bit write access */
+                val = ldub_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 1,
+                                                       attrs);
+                break;
+            default:
+                abort();
             }
         } else {
-            if (!memory_access_is_direct(mr, is_write)) {
-                /* I/O case */
-                release_lock |= prepare_mmio_access(mr);
-                l = memory_access_size(mr, l, addr1);
-                switch (l) {
-                case 8:
-                    /* 64 bit read access */
-                    result |= memory_region_dispatch_read(mr, addr1, &val, 8,
-                                                          attrs);
-                    stq_p(buf, val);
-                    break;
-                case 4:
-                    /* 32 bit read access */
-                    result |= memory_region_dispatch_read(mr, addr1, &val, 4,
-                                                          attrs);
-                    stl_p(buf, val);
-                    break;
-                case 2:
-                    /* 16 bit read access */
-                    result |= memory_region_dispatch_read(mr, addr1, &val, 2,
-                                                          attrs);
-                    stw_p(buf, val);
-                    break;
-                case 1:
-                    /* 8 bit read access */
-                    result |= memory_region_dispatch_read(mr, addr1, &val, 1,
-                                                          attrs);
-                    stb_p(buf, val);
-                    break;
-                default:
-                    abort();
-                }
-            } else {
-                /* RAM case */
-                ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
-                memcpy(buf, ptr, l);
-            }
+            addr1 += memory_region_get_ram_addr(mr);
+            /* RAM case */
+            ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
+            memcpy(ptr, buf, l);
+            invalidate_and_set_dirty(mr, addr1, l);
         }
 
         if (release_lock) {
@@ -2509,8 +2638,14 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
         len -= l;
         buf += l;
         addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
     }
-    rcu_read_unlock();
 
     return result;
 }
@@ -2518,15 +2653,123 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
 MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
                                 const uint8_t *buf, int len)
 {
-    return address_space_rw(as, addr, attrs, (uint8_t *)buf, len, true);
+    hwaddr l;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
+        result = address_space_write_continue(as, addr, attrs, buf, len,
+                                              addr1, l, mr);
+        rcu_read_unlock();
+    }
+
+    return result;
 }
 
-MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
-                               uint8_t *buf, int len)
+/* Called within RCU critical section.  */
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+                                        MemoryRegion *mr)
 {
-    return address_space_rw(as, addr, attrs, buf, len, false);
+    uint8_t *ptr;
+    uint64_t val;
+    MemTxResult result = MEMTX_OK;
+    bool release_lock = false;
+
+    for (;;) {
+        if (!memory_access_is_direct(mr, false)) {
+            /* I/O case */
+            release_lock |= prepare_mmio_access(mr);
+            l = memory_access_size(mr, l, addr1);
+            switch (l) {
+            case 8:
+                /* 64 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 8,
+                                                      attrs);
+                stq_p(buf, val);
+                break;
+            case 4:
+                /* 32 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 4,
+                                                      attrs);
+                stl_p(buf, val);
+                break;
+            case 2:
+                /* 16 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 2,
+                                                      attrs);
+                stw_p(buf, val);
+                break;
+            case 1:
+                /* 8 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 1,
+                                                      attrs);
+                stb_p(buf, val);
+                break;
+            default:
+                abort();
+            }
+        } else {
+            /* RAM case */
+            ptr = qemu_get_ram_ptr(mr->ram_block,
+                                   memory_region_get_ram_addr(mr) + addr1);
+            memcpy(buf, ptr, l);
+        }
+
+        if (release_lock) {
+            qemu_mutex_unlock_iothread();
+            release_lock = false;
+        }
+
+        len -= l;
+        buf += l;
+        addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
+    }
+
+    return result;
+}
+
+MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
+                                    MemTxAttrs attrs, uint8_t *buf, int len)
+{
+    hwaddr l;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
+        result = address_space_read_continue(as, addr, attrs, buf, len,
+                                             addr1, l, mr);
+        rcu_read_unlock();
+    }
+
+    return result;
 }
 
+MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                             uint8_t *buf, int len, bool is_write)
+{
+    if (is_write) {
+        return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
+    } else {
+        return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
+    }
+}
 
 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write)
@@ -2559,7 +2802,7 @@ static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
         } else {
             addr1 += memory_region_get_ram_addr(mr);
             /* ROM/RAM case */
-            ptr = qemu_get_ram_ptr(addr1);
+            ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
             switch (type) {
             case WRITE_DATA:
                 memcpy(ptr, buf, l);
@@ -2652,8 +2895,8 @@ void cpu_register_map_client(QEMUBH *bh)
 void cpu_exec_init_all(void)
 {
     qemu_mutex_init(&ram_list.mutex);
-    memory_map_init();
     io_mem_init();
+    memory_map_init();
     qemu_mutex_init(&map_client_list_lock);
 }
 
@@ -2718,6 +2961,7 @@ void *address_space_map(AddressSpace *as,
     hwaddr l, xlat, base;
     MemoryRegion *mr, *this_mr;
     ram_addr_t raddr;
+    void *ptr;
 
     if (len == 0) {
         return NULL;
@@ -2769,9 +3013,11 @@ void *address_space_map(AddressSpace *as,
     }
 
     memory_region_ref(mr);
-    rcu_read_unlock();
     *plen = done;
-    return qemu_ram_ptr_length(raddr + base, plen);
+    ptr = qemu_ram_ptr_length(mr->ram_block, raddr + base, plen);
+    rcu_read_unlock();
+
+    return ptr;
 }
 
 /* Unmaps a memory region previously mapped by address_space_map().
@@ -2852,7 +3098,8 @@ static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+        ptr = qemu_get_ram_ptr(mr->ram_block,
+                               (memory_region_get_ram_addr(mr)
                                 & TARGET_PAGE_MASK)
                                + addr1);
         switch (endian) {
@@ -2947,7 +3194,8 @@ static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+        ptr = qemu_get_ram_ptr(mr->ram_block,
+                               (memory_region_get_ram_addr(mr)
                                 & TARGET_PAGE_MASK)
                                + addr1);
         switch (endian) {
@@ -3062,7 +3310,8 @@ static inline uint32_t address_space_lduw_internal(AddressSpace *as,
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
+        ptr = qemu_get_ram_ptr(mr->ram_block,
+                               (memory_region_get_ram_addr(mr)
                                 & TARGET_PAGE_MASK)
                                + addr1);
         switch (endian) {
@@ -3147,7 +3396,7 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
         r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
     } else {
         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
         stl_p(ptr, val);
 
         dirty_log_mask = memory_region_get_dirty_log_mask(mr);
@@ -3202,7 +3451,7 @@ static inline void address_space_stl_internal(AddressSpace *as,
     } else {
         /* RAM case */
         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             stl_le_p(ptr, val);
@@ -3312,7 +3561,7 @@ static inline void address_space_stw_internal(AddressSpace *as,
     } else {
         /* RAM case */
         addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             stw_le_p(ptr, val);
@@ -3429,8 +3678,12 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
     target_ulong page;
 
     while (len > 0) {
+        int asidx;
+        MemTxAttrs attrs;
+
         page = addr & TARGET_PAGE_MASK;
-        phys_addr = cpu_get_phys_page_debug(cpu, page);
+        phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
+        asidx = cpu_asidx_from_attrs(cpu, attrs);
         /* if no physical page mapped, return an error */
         if (phys_addr == -1)
             return -1;
@@ -3439,9 +3692,11 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
             l = len;
         phys_addr += (addr & ~TARGET_PAGE_MASK);
         if (is_write) {
-            cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
+            cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
+                                          phys_addr, buf, l);
         } else {
-            address_space_rw(cpu->as, phys_addr, MEMTXATTRS_UNSPECIFIED,
+            address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
+                             MEMTXATTRS_UNSPECIFIED,
                              buf, l, 0);
         }
         len -= l;
@@ -3450,6 +3705,16 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
     }
     return 0;
 }
+
+/*
+ * Allows code that needs to deal with migration bitmaps etc to still be built
+ * target independent.
+ */
+size_t qemu_target_page_bits(void)
+{
+    return TARGET_PAGE_BITS;
+}
+
 #endif
 
 /*
diff --git a/qemu/fpu/softfloat-macros.h b/qemu/fpu/softfloat-macros.h
index 5e030cd8e..9cc6158cb 100644
--- a/qemu/fpu/softfloat-macros.h
+++ b/qemu/fpu/softfloat-macros.h
@@ -99,7 +99,7 @@ this code that are retained.
 | The result is stored in the location pointed to by `zPtr'.
 *----------------------------------------------------------------------------*/
 
-static inline void shift32RightJamming(uint32_t a, int_fast16_t count, uint32_t *zPtr)
+static inline void shift32RightJamming(uint32_t a, int count, uint32_t *zPtr)
 {
     uint32_t z;
 
@@ -125,7 +125,7 @@ static inline void shift32RightJamming(uint32_t a, int_fast16_t count, uint32_t
 | The result is stored in the location pointed to by `zPtr'.
 *----------------------------------------------------------------------------*/
 
-static inline void shift64RightJamming(uint64_t a, int_fast16_t count, uint64_t *zPtr)
+static inline void shift64RightJamming(uint64_t a, int count, uint64_t *zPtr)
 {
     uint64_t z;
 
@@ -161,10 +161,10 @@ static inline void shift64RightJamming(uint64_t a, int_fast16_t count, uint64_t
 
 static inline void
  shift64ExtraRightJamming(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -198,10 +198,10 @@ static inline void
 
 static inline void
  shift128Right(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -233,10 +233,10 @@ static inline void
 
 static inline void
  shift128RightJamming(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -287,14 +287,14 @@ static inline void
      uint64_t a0,
      uint64_t a1,
      uint64_t a2,
-     int_fast16_t count,
+     int count,
      uint64_t *z0Ptr,
      uint64_t *z1Ptr,
      uint64_t *z2Ptr
  )
 {
     uint64_t z0, z1, z2;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z2 = a2;
@@ -342,7 +342,7 @@ static inline void
 
 static inline void
  shortShift128Left(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
 
     *z1Ptr = a1<<count;
@@ -364,14 +364,14 @@ static inline void
      uint64_t a0,
      uint64_t a1,
      uint64_t a2,
-     int_fast16_t count,
+     int count,
      uint64_t *z0Ptr,
      uint64_t *z1Ptr,
      uint64_t *z2Ptr
  )
 {
     uint64_t z0, z1, z2;
-    int8 negCount;
+    int8_t negCount;
 
     z2 = a2<<count;
     z1 = a1<<count;
@@ -428,7 +428,7 @@ static inline void
  )
 {
     uint64_t z0, z1, z2;
-    int8 carry0, carry1;
+    int8_t carry0, carry1;
 
     z2 = a2 + b2;
     carry1 = ( z2 < a2 );
@@ -484,7 +484,7 @@ static inline void
  )
 {
     uint64_t z0, z1, z2;
-    int8 borrow0, borrow1;
+    int8_t borrow0, borrow1;
 
     z2 = a2 - b2;
     borrow1 = ( a2 < b2 );
@@ -635,7 +635,7 @@ static uint64_t estimateDiv128To64( uint64_t a0, uint64_t a1, uint64_t b )
 | value.
 *----------------------------------------------------------------------------*/
 
-static uint32_t estimateSqrt32(int_fast16_t aExp, uint32_t a)
+static uint32_t estimateSqrt32(int aExp, uint32_t a)
 {
     static const uint16_t sqrtOddAdjustments[] = {
         0x0004, 0x0022, 0x005D, 0x00B1, 0x011D, 0x019F, 0x0236, 0x02E0,
@@ -645,7 +645,7 @@ static uint32_t estimateSqrt32(int_fast16_t aExp, uint32_t a)
         0x0A2D, 0x08AF, 0x075A, 0x0629, 0x051A, 0x0429, 0x0356, 0x029E,
         0x0200, 0x0179, 0x0109, 0x00AF, 0x0068, 0x0034, 0x0012, 0x0002
     };
-    int8 index;
+    int8_t index;
     uint32_t z;
 
     index = ( a>>27 ) & 15;
@@ -669,7 +669,7 @@ static uint32_t estimateSqrt32(int_fast16_t aExp, uint32_t a)
 | `a'.  If `a' is zero, 32 is returned.
 *----------------------------------------------------------------------------*/
 
-static int8 countLeadingZeros32( uint32_t a )
+static int8_t countLeadingZeros32( uint32_t a )
 {
 #if SOFTFLOAT_GNUC_PREREQ(3, 4)
     if (a) {
@@ -678,7 +678,7 @@ static int8 countLeadingZeros32( uint32_t a )
         return 32;
     }
 #else
-    static const int8 countLeadingZerosHigh[] = {
+    static const int8_t countLeadingZerosHigh[] = {
         8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
         3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -696,7 +696,7 @@ static int8 countLeadingZeros32( uint32_t a )
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
     };
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = 0;
     if ( a < 0x10000 ) {
@@ -717,7 +717,7 @@ static int8 countLeadingZeros32( uint32_t a )
 | `a'.  If `a' is zero, 64 is returned.
 *----------------------------------------------------------------------------*/
 
-static int8 countLeadingZeros64( uint64_t a )
+static int8_t countLeadingZeros64( uint64_t a )
 {
 #if SOFTFLOAT_GNUC_PREREQ(3, 4)
     if (a) {
@@ -726,7 +726,7 @@ static int8 countLeadingZeros64( uint64_t a )
         return 64;
     }
 #else
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = 0;
     if ( a < ( (uint64_t) 1 )<<32 ) {
diff --git a/qemu/fpu/softfloat-specialize.h b/qemu/fpu/softfloat-specialize.h
index 6dd41d897..a4cbdad45 100644
--- a/qemu/fpu/softfloat-specialize.h
+++ b/qemu/fpu/softfloat-specialize.h
@@ -113,7 +113,7 @@ const float16 float16_default_nan = const_float16(0xFE00);
 #if defined(TARGET_SPARC)
 const float32 float32_default_nan = const_float32(0x7FFFFFFF);
 #elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA) || \
-      defined(TARGET_XTENSA) || defined(TARGET_S390X)
+      defined(TARGET_XTENSA) || defined(TARGET_S390X) || defined(TARGET_TRICORE)
 const float32 float32_default_nan = const_float32(0x7FC00000);
 #elif SNAN_BIT_IS_ONE
 const float32 float32_default_nan = const_float32(0x7FBFFFFF);
@@ -174,7 +174,7 @@ const float128 float128_default_nan
 | should be simply `float_exception_flags |= flags;'.
 *----------------------------------------------------------------------------*/
 
-void float_raise(int8 flags, float_status *status)
+void float_raise(int8_t flags, float_status *status)
 {
     status->float_exception_flags |= flags;
 }
diff --git a/qemu/fpu/softfloat.c b/qemu/fpu/softfloat.c
index f1170fe50..166c48e43 100644
--- a/qemu/fpu/softfloat.c
+++ b/qemu/fpu/softfloat.c
@@ -82,12 +82,11 @@ this code that are retained.
 /* softfloat (and in particular the code in softfloat-specialize.h) is
  * target-dependent and needs the TARGET_* macros.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 
 #include "fpu/softfloat.h"
 
 /* We only need stdlib for abort() */
-#include <stdlib.h>
 
 /*----------------------------------------------------------------------------
 | Primitive arithmetic functions, including multi-word arithmetic, and
@@ -119,7 +118,7 @@ static inline uint32_t extractFloat16Frac(float16 a)
 | Returns the exponent bits of the half-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat16Exp(float16 a)
+static inline int extractFloat16Exp(float16 a)
 {
     return (float16_val(a) >> 10) & 0x1f;
 }
@@ -144,11 +143,11 @@ static inline flag extractFloat16Sign(float16 a)
 | positive or negative integer is returned.
 *----------------------------------------------------------------------------*/
 
-static int32 roundAndPackInt32(flag zSign, uint64_t absZ, float_status *status)
+static int32_t roundAndPackInt32(flag zSign, uint64_t absZ, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int8 roundIncrement, roundBits;
+    int8_t roundIncrement, roundBits;
     int32_t z;
 
     roundingMode = status->float_rounding_mode;
@@ -198,10 +197,10 @@ static int32 roundAndPackInt32(flag zSign, uint64_t absZ, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-static int64 roundAndPackInt64(flag zSign, uint64_t absZ0, uint64_t absZ1,
+static int64_t roundAndPackInt64(flag zSign, uint64_t absZ0, uint64_t absZ1,
                                float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment;
     int64_t z;
 
@@ -255,10 +254,10 @@ static int64 roundAndPackInt64(flag zSign, uint64_t absZ0, uint64_t absZ1,
 | exception is raised and the largest unsigned integer is returned.
 *----------------------------------------------------------------------------*/
 
-static int64 roundAndPackUint64(flag zSign, uint64_t absZ0,
+static int64_t roundAndPackUint64(flag zSign, uint64_t absZ0,
                                 uint64_t absZ1, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment;
 
     roundingMode = status->float_rounding_mode;
@@ -315,7 +314,7 @@ static inline uint32_t extractFloat32Frac( float32 a )
 | Returns the exponent bits of the single-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat32Exp(float32 a)
+static inline int extractFloat32Exp(float32 a)
 {
 
     return ( float32_val(a)>>23 ) & 0xFF;
@@ -356,9 +355,9 @@ float32 float32_squash_input_denormal(float32 a, float_status *status)
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloat32Subnormal(uint32_t aSig, int_fast16_t *zExpPtr, uint32_t *zSigPtr)
+ normalizeFloat32Subnormal(uint32_t aSig, int *zExpPtr, uint32_t *zSigPtr)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros32( aSig ) - 8;
     *zSigPtr = aSig<<shiftCount;
@@ -377,7 +376,7 @@ static void
 | significand.
 *----------------------------------------------------------------------------*/
 
-static inline float32 packFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig)
+static inline float32 packFloat32(flag zSign, int zExp, uint32_t zSig)
 {
 
     return make_float32(
@@ -407,12 +406,12 @@ static inline float32 packFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig)
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 roundAndPackFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig,
+static float32 roundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
                                    float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int8 roundIncrement, roundBits;
+    int8_t roundIncrement, roundBits;
     flag isTiny;
 
     roundingMode = status->float_rounding_mode;
@@ -482,10 +481,10 @@ static float32 roundAndPackFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig,
 *----------------------------------------------------------------------------*/
 
 static float32
- normalizeRoundAndPackFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig,
+ normalizeRoundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
                               float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros32( zSig ) - 1;
     return roundAndPackFloat32(zSign, zExp - shiftCount, zSig<<shiftCount,
@@ -508,7 +507,7 @@ static inline uint64_t extractFloat64Frac( float64 a )
 | Returns the exponent bits of the double-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat64Exp(float64 a)
+static inline int extractFloat64Exp(float64 a)
 {
 
     return ( float64_val(a)>>52 ) & 0x7FF;
@@ -549,9 +548,9 @@ float64 float64_squash_input_denormal(float64 a, float_status *status)
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloat64Subnormal(uint64_t aSig, int_fast16_t *zExpPtr, uint64_t *zSigPtr)
+ normalizeFloat64Subnormal(uint64_t aSig, int *zExpPtr, uint64_t *zSigPtr)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( aSig ) - 11;
     *zSigPtr = aSig<<shiftCount;
@@ -570,7 +569,7 @@ static void
 | significand.
 *----------------------------------------------------------------------------*/
 
-static inline float64 packFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig)
+static inline float64 packFloat64(flag zSign, int zExp, uint64_t zSig)
 {
 
     return make_float64(
@@ -600,12 +599,12 @@ static inline float64 packFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig)
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float64 roundAndPackFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig,
+static float64 roundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
                                    float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int_fast16_t roundIncrement, roundBits;
+    int roundIncrement, roundBits;
     flag isTiny;
 
     roundingMode = status->float_rounding_mode;
@@ -674,10 +673,10 @@ static float64 roundAndPackFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig,
 *----------------------------------------------------------------------------*/
 
 static float64
- normalizeRoundAndPackFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig,
+ normalizeRoundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
                               float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( zSig ) - 1;
     return roundAndPackFloat64(zSign, zExp - shiftCount, zSig<<shiftCount,
@@ -702,7 +701,7 @@ static inline uint64_t extractFloatx80Frac( floatx80 a )
 | value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int32 extractFloatx80Exp( floatx80 a )
+static inline int32_t extractFloatx80Exp( floatx80 a )
 {
 
     return a.high & 0x7FFF;
@@ -729,9 +728,9 @@ static inline flag extractFloatx80Sign( floatx80 a )
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloatx80Subnormal( uint64_t aSig, int32 *zExpPtr, uint64_t *zSigPtr )
+ normalizeFloatx80Subnormal( uint64_t aSig, int32_t *zExpPtr, uint64_t *zSigPtr )
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( aSig );
     *zSigPtr = aSig<<shiftCount;
@@ -744,7 +743,7 @@ static void
 | extended double-precision floating-point value, returning the result.
 *----------------------------------------------------------------------------*/
 
-static inline floatx80 packFloatx80( flag zSign, int32 zExp, uint64_t zSig )
+static inline floatx80 packFloatx80( flag zSign, int32_t zExp, uint64_t zSig )
 {
     floatx80 z;
 
@@ -778,13 +777,13 @@ static inline floatx80 packFloatx80( flag zSign, int32 zExp, uint64_t zSig )
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static floatx80 roundAndPackFloatx80(int8 roundingPrecision, flag zSign,
-                                     int32 zExp, uint64_t zSig0, uint64_t zSig1,
+static floatx80 roundAndPackFloatx80(int8_t roundingPrecision, flag zSign,
+                                     int32_t zExp, uint64_t zSig0, uint64_t zSig1,
                                      float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment, isTiny;
-    int64 roundIncrement, roundMask, roundBits;
+    int64_t roundIncrement, roundMask, roundBits;
 
     roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
@@ -974,12 +973,12 @@ static floatx80 roundAndPackFloatx80(int8 roundingPrecision, flag zSign,
 | normalized.
 *----------------------------------------------------------------------------*/
 
-static floatx80 normalizeRoundAndPackFloatx80(int8 roundingPrecision,
-                                              flag zSign, int32 zExp,
+static floatx80 normalizeRoundAndPackFloatx80(int8_t roundingPrecision,
+                                              flag zSign, int32_t zExp,
                                               uint64_t zSig0, uint64_t zSig1,
                                               float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     if ( zSig0 == 0 ) {
         zSig0 = zSig1;
@@ -1023,7 +1022,7 @@ static inline uint64_t extractFloat128Frac0( float128 a )
 | `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int32 extractFloat128Exp( float128 a )
+static inline int32_t extractFloat128Exp( float128 a )
 {
 
     return ( a.high>>48 ) & 0x7FFF;
@@ -1055,12 +1054,12 @@ static void
  normalizeFloat128Subnormal(
      uint64_t aSig0,
      uint64_t aSig1,
-     int32 *zExpPtr,
+     int32_t *zExpPtr,
      uint64_t *zSig0Ptr,
      uint64_t *zSig1Ptr
  )
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     if ( aSig0 == 0 ) {
         shiftCount = countLeadingZeros64( aSig1 ) - 15;
@@ -1096,7 +1095,7 @@ static void
 *----------------------------------------------------------------------------*/
 
 static inline float128
- packFloat128( flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1 )
+ packFloat128( flag zSign, int32_t zExp, uint64_t zSig0, uint64_t zSig1 )
 {
     float128 z;
 
@@ -1127,11 +1126,11 @@ static inline float128
 | overflow follows the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float128 roundAndPackFloat128(flag zSign, int32 zExp,
+static float128 roundAndPackFloat128(flag zSign, int32_t zExp,
                                      uint64_t zSig0, uint64_t zSig1,
                                      uint64_t zSig2, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment, isTiny;
 
     roundingMode = status->float_rounding_mode;
@@ -1245,11 +1244,11 @@ static float128 roundAndPackFloat128(flag zSign, int32 zExp,
 | point exponent.
 *----------------------------------------------------------------------------*/
 
-static float128 normalizeRoundAndPackFloat128(flag zSign, int32 zExp,
+static float128 normalizeRoundAndPackFloat128(flag zSign, int32_t zExp,
                                               uint64_t zSig0, uint64_t zSig1,
                                               float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
     uint64_t zSig2;
 
     if ( zSig0 == 0 ) {
@@ -1296,8 +1295,8 @@ float32 int32_to_float32(int32_t a, float_status *status)
 float64 int32_to_float64(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig;
 
     if ( a == 0 ) return float64_zero;
@@ -1319,8 +1318,8 @@ float64 int32_to_float64(int32_t a, float_status *status)
 floatx80 int32_to_floatx80(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig;
 
     if ( a == 0 ) return packFloatx80( 0, 0, 0 );
@@ -1341,8 +1340,8 @@ floatx80 int32_to_floatx80(int32_t a, float_status *status)
 float128 int32_to_float128(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig0;
 
     if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
@@ -1363,8 +1362,8 @@ float128 int32_to_float128(int32_t a, float_status *status)
 float32 int64_to_float32(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
+    uint64_t absA;
+    int8_t shiftCount;
 
     if ( a == 0 ) return float32_zero;
     zSign = ( a < 0 );
@@ -1414,8 +1413,8 @@ float64 int64_to_float64(int64_t a, float_status *status)
 floatx80 int64_to_floatx80(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
+    uint64_t absA;
+    int8_t shiftCount;
 
     if ( a == 0 ) return packFloatx80( 0, 0, 0 );
     zSign = ( a < 0 );
@@ -1434,9 +1433,9 @@ floatx80 int64_to_floatx80(int64_t a, float_status *status)
 float128 int64_to_float128(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
-    int32 zExp;
+    uint64_t absA;
+    int8_t shiftCount;
+    int32_t zExp;
     uint64_t zSig0, zSig1;
 
     if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
@@ -1541,10 +1540,11 @@ float128 uint64_to_float128(uint64_t a, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float32_to_int32(float32 a, float_status *status)
+int32_t float32_to_int32(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64;
 
@@ -1572,10 +1572,11 @@ int32 float32_to_int32(float32 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float32_to_int32_round_to_zero(float32 a, float_status *status)
+int32_t float32_to_int32_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     int32_t z;
     a = float32_squash_input_denormal(a, status);
@@ -1617,12 +1618,13 @@ int32 float32_to_int32_round_to_zero(float32 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int_fast16_t float32_to_int16_round_to_zero(float32 a, float_status *status)
+int16_t float32_to_int16_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
-    int32 z;
+    int32_t z;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1666,10 +1668,11 @@ int_fast16_t float32_to_int16_round_to_zero(float32 a, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float32_to_int64(float32 a, float_status *status)
+int64_t float32_to_int64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64, aSigExtra;
     a = float32_squash_input_denormal(a, status);
@@ -1705,10 +1708,11 @@ int64 float32_to_int64(float32 a, float_status *status)
 | raise the inexact exception flag.
 *----------------------------------------------------------------------------*/
 
-uint64 float32_to_uint64(float32 a, float_status *status)
+uint64_t float32_to_uint64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64, aSigExtra;
     a = float32_squash_input_denormal(a, status);
@@ -1750,7 +1754,7 @@ uint64 float32_to_uint64(float32 a, float_status *status)
 | not round to zero will raise the inexact flag.
 *----------------------------------------------------------------------------*/
 
-uint64 float32_to_uint64_round_to_zero(float32 a, float_status *status)
+uint64_t float32_to_uint64_round_to_zero(float32 a, float_status *status)
 {
     signed char current_rounding_mode = status->float_rounding_mode;
     set_float_rounding_mode(float_round_to_zero, status);
@@ -1769,13 +1773,14 @@ uint64 float32_to_uint64_round_to_zero(float32 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float32_to_int64_round_to_zero(float32 a, float_status *status)
+int64_t float32_to_int64_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64;
-    int64 z;
+    int64_t z;
     a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
@@ -1818,7 +1823,7 @@ int64 float32_to_int64_round_to_zero(float32 a, float_status *status)
 float64 float32_to_float64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
     a = float32_squash_input_denormal(a, status);
 
@@ -1850,7 +1855,7 @@ float64 float32_to_float64(float32 a, float_status *status)
 floatx80 float32_to_floatx80(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     a = float32_squash_input_denormal(a, status);
@@ -1882,7 +1887,7 @@ floatx80 float32_to_floatx80(float32 a, float_status *status)
 float128 float32_to_float128(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     a = float32_squash_input_denormal(a, status);
@@ -1914,7 +1919,7 @@ float128 float32_to_float128(float32 a, float_status *status)
 float32 float32_round_to_int(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t lastBitMask, roundBitsMask;
     uint32_t z;
     a = float32_squash_input_denormal(a, status);
@@ -1996,9 +2001,9 @@ float32 float32_round_to_int(float32 a, float_status *status)
 static float32 addFloat32Sigs(float32 a, float32 b, flag zSign,
                               float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -2082,9 +2087,9 @@ static float32 addFloat32Sigs(float32 a, float32 b, flag zSign,
 static float32 subFloat32Sigs(float32 a, float32 b, flag zSign,
                               float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -2208,7 +2213,7 @@ float32 float32_sub(float32 a, float32 b, float_status *status)
 float32 float32_mul(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig;
     uint64_t zSig64;
     uint32_t zSig;
@@ -2273,7 +2278,7 @@ float32 float32_mul(float32 a, float32 b, float_status *status)
 float32 float32_div(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
     a = float32_squash_input_denormal(a, status);
     b = float32_squash_input_denormal(b, status);
@@ -2343,7 +2348,7 @@ float32 float32_div(float32 a, float32 b, float_status *status)
 float32 float32_rem(float32 a, float32 b, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp, bExp, expDiff;
+    int aExp, bExp, expDiff;
     uint32_t aSig, bSig;
     uint32_t q;
     uint64_t aSig64, bSig64, q64;
@@ -2451,7 +2456,7 @@ float32 float32_muladd(float32 a, float32 b, float32 c, int flags,
                        float_status *status)
 {
     flag aSign, bSign, cSign, zSign;
-    int_fast16_t aExp, bExp, cExp, pExp, zExp, expDiff;
+    int aExp, bExp, cExp, pExp, zExp, expDiff;
     uint32_t aSig, bSig, cSig;
     flag pInf, pZero, pSign;
     uint64_t pSig64, cSig64, zSig64;
@@ -2671,7 +2676,7 @@ float32 float32_muladd(float32 a, float32 b, float32 c, int flags,
 float32 float32_sqrt(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, zExp;
+    int aExp, zExp;
     uint32_t aSig, zSig;
     uint64_t rem, term;
     a = float32_squash_input_denormal(a, status);
@@ -2759,7 +2764,7 @@ static const float64 float32_exp2_coefficients[15] =
 float32 float32_exp2(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
     float64 r, x, xn;
     int i;
@@ -2809,7 +2814,7 @@ float32 float32_exp2(float32 a, float_status *status)
 float32 float32_log2(float32 a, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig, zSig, i;
 
     a = float32_squash_input_denormal(a, status);
@@ -3073,10 +3078,11 @@ int float32_unordered_quiet(float32 a, float32 b, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float64_to_int32(float64 a, float_status *status)
+int32_t float64_to_int32(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig;
     a = float64_squash_input_denormal(a, status);
 
@@ -3101,10 +3107,11 @@ int32 float64_to_int32(float64 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float64_to_int32_round_to_zero(float64 a, float_status *status)
+int32_t float64_to_int32_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, savedASig;
     int32_t z;
     a = float64_squash_input_denormal(a, status);
@@ -3150,12 +3157,13 @@ int32 float64_to_int32_round_to_zero(float64 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int_fast16_t float64_to_int16_round_to_zero(float64 a, float_status *status)
+int16_t float64_to_int16_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, savedASig;
-    int32 z;
+    int32_t z;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3201,10 +3209,11 @@ int_fast16_t float64_to_int16_round_to_zero(float64 a, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float64_to_int64(float64 a, float_status *status)
+int64_t float64_to_int64(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, aSigExtra;
     a = float64_squash_input_denormal(a, status);
 
@@ -3244,12 +3253,13 @@ int64 float64_to_int64(float64 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float64_to_int64_round_to_zero(float64 a, float_status *status)
+int64_t float64_to_int64_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig;
-    int64 z;
+    int64_t z;
     a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
@@ -3299,7 +3309,7 @@ int64 float64_to_int64_round_to_zero(float64 a, float_status *status)
 float32 float64_to_float32(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
     uint32_t zSig;
     a = float64_squash_input_denormal(a, status);
@@ -3334,7 +3344,7 @@ float32 float64_to_float32(float64 a, float_status *status)
 | than the desired result exponent whenever `zSig' is a complete, normalized
 | significand.
 *----------------------------------------------------------------------------*/
-static float16 packFloat16(flag zSign, int_fast16_t zExp, uint16_t zSig)
+static float16 packFloat16(flag zSign, int zExp, uint16_t zSig)
 {
     return make_float16(
         (((uint32_t)zSign) << 15) + (((uint32_t)zExp) << 10) + zSig);
@@ -3368,7 +3378,7 @@ static float16 packFloat16(flag zSign, int_fast16_t zExp, uint16_t zSig)
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 roundAndPackFloat16(flag zSign, int_fast16_t zExp,
+static float16 roundAndPackFloat16(flag zSign, int zExp,
                                    uint32_t zSig, flag ieee,
                                    float_status *status)
 {
@@ -3455,7 +3465,7 @@ static float32 roundAndPackFloat16(flag zSign, int_fast16_t zExp,
     return packFloat16(zSign, zExp, zSig >> 13);
 }
 
-static void normalizeFloat16Subnormal(uint32_t aSig, int_fast16_t *zExpPtr,
+static void normalizeFloat16Subnormal(uint32_t aSig, int *zExpPtr,
                                       uint32_t *zSigPtr)
 {
     int8_t shiftCount = countLeadingZeros32(aSig) - 21;
@@ -3469,7 +3479,7 @@ static void normalizeFloat16Subnormal(uint32_t aSig, int_fast16_t *zExpPtr,
 float32 float16_to_float32(float16 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     aSign = extractFloat16Sign(a);
@@ -3496,7 +3506,7 @@ float32 float16_to_float32(float16 a, flag ieee, float_status *status)
 float16 float32_to_float16(float32 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     a = float32_squash_input_denormal(a, status);
@@ -3540,7 +3550,7 @@ float16 float32_to_float16(float32 a, flag ieee, float_status *status)
 float64 float16_to_float64(float16 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     aSign = extractFloat16Sign(a);
@@ -3568,7 +3578,7 @@ float64 float16_to_float64(float16 a, flag ieee, float_status *status)
 float16 float64_to_float16(float64 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
     uint32_t zSig;
 
@@ -3622,7 +3632,7 @@ float16 float64_to_float16(float64 a, flag ieee, float_status *status)
 floatx80 float64_to_floatx80(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
 
     a = float64_squash_input_denormal(a, status);
@@ -3655,7 +3665,7 @@ floatx80 float64_to_floatx80(float64 a, float_status *status)
 float128 float64_to_float128(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, zSig0, zSig1;
 
     a = float64_squash_input_denormal(a, status);
@@ -3688,7 +3698,7 @@ float128 float64_to_float128(float64 a, float_status *status)
 float64 float64_round_to_int(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t lastBitMask, roundBitsMask;
     uint64_t z;
     a = float64_squash_input_denormal(a, status);
@@ -3782,9 +3792,9 @@ float64 float64_trunc_to_int(float64 a, float_status *status)
 static float64 addFloat64Sigs(float64 a, float64 b, flag zSign,
                               float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3868,9 +3878,9 @@ static float64 addFloat64Sigs(float64 a, float64 b, flag zSign,
 static float64 subFloat64Sigs(float64 a, float64 b, flag zSign,
                               float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3994,7 +4004,7 @@ float64 float64_sub(float64 a, float64 b, float_status *status)
 float64 float64_mul(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
 
     a = float64_squash_input_denormal(a, status);
@@ -4057,7 +4067,7 @@ float64 float64_mul(float64 a, float64 b, float_status *status)
 float64 float64_div(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
     uint64_t rem0, rem1;
     uint64_t term0, term1;
@@ -4135,7 +4145,7 @@ float64 float64_div(float64 a, float64 b, float_status *status)
 float64 float64_rem(float64 a, float64 b, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp, bExp, expDiff;
+    int aExp, bExp, expDiff;
     uint64_t aSig, bSig;
     uint64_t q, alternateASig;
     int64_t sigMean;
@@ -4229,7 +4239,7 @@ float64 float64_muladd(float64 a, float64 b, float64 c, int flags,
                        float_status *status)
 {
     flag aSign, bSign, cSign, zSign;
-    int_fast16_t aExp, bExp, cExp, pExp, zExp, expDiff;
+    int aExp, bExp, cExp, pExp, zExp, expDiff;
     uint64_t aSig, bSig, cSig;
     flag pInf, pZero, pSign;
     uint64_t pSig0, pSig1, cSig0, cSig1, zSig0, zSig1;
@@ -4470,7 +4480,7 @@ float64 float64_muladd(float64 a, float64 b, float64 c, int flags,
 float64 float64_sqrt(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, zExp;
+    int aExp, zExp;
     uint64_t aSig, zSig, doubleZSig;
     uint64_t rem0, rem1, term0, term1;
     a = float64_squash_input_denormal(a, status);
@@ -4523,7 +4533,7 @@ float64 float64_sqrt(float64 a, float_status *status)
 float64 float64_log2(float64 a, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, aSig0, aSig1, zSig, i;
     a = float64_squash_input_denormal(a, status);
 
@@ -4790,10 +4800,10 @@ int float64_unordered_quiet(float64 a, float64 b, float_status *status)
 | overflows, the largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 floatx80_to_int32(floatx80 a, float_status *status)
+int32_t floatx80_to_int32(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig;
 
     aSig = extractFloatx80Frac( a );
@@ -4817,10 +4827,10 @@ int32 floatx80_to_int32(floatx80 a, float_status *status)
 | sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 floatx80_to_int32_round_to_zero(floatx80 a, float_status *status)
+int32_t floatx80_to_int32_round_to_zero(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig, savedASig;
     int32_t z;
 
@@ -4864,10 +4874,10 @@ int32 floatx80_to_int32_round_to_zero(floatx80 a, float_status *status)
 | overflows, the largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 floatx80_to_int64(floatx80 a, float_status *status)
+int64_t floatx80_to_int64(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig, aSigExtra;
 
     aSig = extractFloatx80Frac( a );
@@ -4904,12 +4914,12 @@ int64 floatx80_to_int64(floatx80 a, float_status *status)
 | sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 floatx80_to_int64_round_to_zero(floatx80 a, float_status *status)
+int64_t floatx80_to_int64_round_to_zero(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig;
-    int64 z;
+    int64_t z;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -4950,7 +4960,7 @@ int64 floatx80_to_int64_round_to_zero(floatx80 a, float_status *status)
 float32 floatx80_to_float32(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig;
 
     aSig = extractFloatx80Frac( a );
@@ -4978,7 +4988,7 @@ float32 floatx80_to_float32(floatx80 a, float_status *status)
 float64 floatx80_to_float64(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig, zSig;
 
     aSig = extractFloatx80Frac( a );
@@ -5006,7 +5016,7 @@ float64 floatx80_to_float64(floatx80 a, float_status *status)
 float128 floatx80_to_float128(floatx80 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, zSig0, zSig1;
 
     aSig = extractFloatx80Frac( a );
@@ -5030,7 +5040,7 @@ float128 floatx80_to_float128(floatx80 a, float_status *status)
 floatx80 floatx80_round_to_int(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t lastBitMask, roundBitsMask;
     floatx80 z;
 
@@ -5125,9 +5135,9 @@ floatx80 floatx80_round_to_int(floatx80 a, float_status *status)
 static floatx80 addFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
                                 float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
-    int32 expDiff;
+    int32_t expDiff;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -5194,9 +5204,9 @@ static floatx80 addFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
 static floatx80 subFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
                                 float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
-    int32 expDiff;
+    int32_t expDiff;
     floatx80 z;
 
     aSig = extractFloatx80Frac( a );
@@ -5305,7 +5315,7 @@ floatx80 floatx80_sub(floatx80 a, floatx80 b, float_status *status)
 floatx80 floatx80_mul(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
     floatx80 z;
 
@@ -5364,7 +5374,7 @@ floatx80 floatx80_mul(floatx80 a, floatx80 b, float_status *status)
 floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
     uint64_t rem0, rem1, rem2, term0, term1, term2;
     floatx80 z;
@@ -5448,7 +5458,7 @@ floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status)
 floatx80 floatx80_rem(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, zSign;
-    int32 aExp, bExp, expDiff;
+    int32_t aExp, bExp, expDiff;
     uint64_t aSig0, aSig1, bSig;
     uint64_t q, term0, term1, alternateASig0, alternateASig1;
     floatx80 z;
@@ -5546,7 +5556,7 @@ floatx80 floatx80_rem(floatx80 a, floatx80 b, float_status *status)
 floatx80 floatx80_sqrt(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, zExp;
+    int32_t aExp, zExp;
     uint64_t aSig0, aSig1, zSig0, zSig1, doubleZSig0;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
     floatx80 z;
@@ -5854,10 +5864,10 @@ int floatx80_unordered_quiet(floatx80 a, floatx80 b, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float128_to_int32(float128 a, float_status *status)
+int32_t float128_to_int32(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5883,10 +5893,10 @@ int32 float128_to_int32(float128 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float128_to_int32_round_to_zero(float128 a, float_status *status)
+int32_t float128_to_int32_round_to_zero(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1, savedASig;
     int32_t z;
 
@@ -5933,10 +5943,10 @@ int32 float128_to_int32_round_to_zero(float128 a, float_status *status)
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float128_to_int64(float128 a, float_status *status)
+int64_t float128_to_int64(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5976,12 +5986,12 @@ int64 float128_to_int64(float128 a, float_status *status)
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float128_to_int64_round_to_zero(float128 a, float_status *status)
+int64_t float128_to_int64_round_to_zero(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
-    int64 z;
+    int64_t z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6039,7 +6049,7 @@ int64 float128_to_int64_round_to_zero(float128 a, float_status *status)
 float32 float128_to_float32(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
     uint32_t zSig;
 
@@ -6074,7 +6084,7 @@ float32 float128_to_float32(float128 a, float_status *status)
 float64 float128_to_float64(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -6107,7 +6117,7 @@ float64 float128_to_float64(float128 a, float_status *status)
 floatx80 float128_to_floatx80(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -6142,7 +6152,7 @@ floatx80 float128_to_floatx80(float128 a, float_status *status)
 float128 float128_round_to_int(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t lastBitMask, roundBitsMask;
     float128 z;
 
@@ -6281,9 +6291,9 @@ float128 float128_round_to_int(float128 a, float_status *status)
 static float128 addFloat128Sigs(float128 a, float128 b, flag zSign,
                                 float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
-    int32 expDiff;
+    int32_t expDiff;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6372,9 +6382,9 @@ static float128 addFloat128Sigs(float128 a, float128 b, flag zSign,
 static float128 subFloat128Sigs(float128 a, float128 b, flag zSign,
                                 float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1;
-    int32 expDiff;
+    int32_t expDiff;
     float128 z;
 
     aSig1 = extractFloat128Frac1( a );
@@ -6503,7 +6513,7 @@ float128 float128_sub(float128 a, float128 b, float_status *status)
 float128 float128_mul(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2, zSig3;
     float128 z;
 
@@ -6569,7 +6579,7 @@ float128 float128_mul(float128 a, float128 b, float_status *status)
 float128 float128_div(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
     float128 z;
@@ -6659,7 +6669,7 @@ float128 float128_div(float128 a, float128 b, float_status *status)
 float128 float128_rem(float128 a, float128 b, float_status *status)
 {
     flag aSign, zSign;
-    int32 aExp, bExp, expDiff;
+    int32_t aExp, bExp, expDiff;
     uint64_t aSig0, aSig1, bSig0, bSig1, q, term0, term1, term2;
     uint64_t allZero, alternateASig0, alternateASig1, sigMean1;
     int64_t sigMean0;
@@ -6769,7 +6779,7 @@ float128 float128_rem(float128 a, float128 b, float_status *status)
 float128 float128_sqrt(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, zExp;
+    int32_t aExp, zExp;
     uint64_t aSig0, aSig1, zSig0, zSig1, zSig2, doubleZSig0;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
     float128 z;
@@ -7080,10 +7090,10 @@ float64 uint32_to_float64(uint32_t a, float_status *status)
     return int64_to_float64(a, status);
 }
 
-uint32 float32_to_uint32(float32 a, float_status *status)
+uint32_t float32_to_uint32(float32 a, float_status *status)
 {
     int64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float32_to_int64(a, status);
@@ -7099,10 +7109,10 @@ uint32 float32_to_uint32(float32 a, float_status *status)
     return res;
 }
 
-uint32 float32_to_uint32_round_to_zero(float32 a, float_status *status)
+uint32_t float32_to_uint32_round_to_zero(float32 a, float_status *status)
 {
     int64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float32_to_int64_round_to_zero(a, status);
@@ -7118,10 +7128,10 @@ uint32 float32_to_uint32_round_to_zero(float32 a, float_status *status)
     return res;
 }
 
-int_fast16_t float32_to_int16(float32 a, float_status *status)
+int16_t float32_to_int16(float32 a, float_status *status)
 {
     int32_t v;
-    int_fast16_t res;
+    int16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float32_to_int32(a, status);
@@ -7138,10 +7148,10 @@ int_fast16_t float32_to_int16(float32 a, float_status *status)
     return res;
 }
 
-uint_fast16_t float32_to_uint16(float32 a, float_status *status)
+uint16_t float32_to_uint16(float32 a, float_status *status)
 {
     int32_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float32_to_int32(a, status);
@@ -7158,10 +7168,10 @@ uint_fast16_t float32_to_uint16(float32 a, float_status *status)
     return res;
 }
 
-uint_fast16_t float32_to_uint16_round_to_zero(float32 a, float_status *status)
+uint16_t float32_to_uint16_round_to_zero(float32 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float32_to_int64_round_to_zero(a, status);
@@ -7177,10 +7187,10 @@ uint_fast16_t float32_to_uint16_round_to_zero(float32 a, float_status *status)
     return res;
 }
 
-uint32 float64_to_uint32(float64 a, float_status *status)
+uint32_t float64_to_uint32(float64 a, float_status *status)
 {
     uint64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float64_to_uint64(a, status);
@@ -7194,10 +7204,10 @@ uint32 float64_to_uint32(float64 a, float_status *status)
     return res;
 }
 
-uint32 float64_to_uint32_round_to_zero(float64 a, float_status *status)
+uint32_t float64_to_uint32_round_to_zero(float64 a, float_status *status)
 {
     uint64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float64_to_uint64_round_to_zero(a, status);
@@ -7211,10 +7221,10 @@ uint32 float64_to_uint32_round_to_zero(float64 a, float_status *status)
     return res;
 }
 
-int_fast16_t float64_to_int16(float64 a, float_status *status)
+int16_t float64_to_int16(float64 a, float_status *status)
 {
     int64_t v;
-    int_fast16_t res;
+    int16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float64_to_int32(a, status);
@@ -7231,10 +7241,10 @@ int_fast16_t float64_to_int16(float64 a, float_status *status)
     return res;
 }
 
-uint_fast16_t float64_to_uint16(float64 a, float_status *status)
+uint16_t float64_to_uint16(float64 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float64_to_int32(a, status);
@@ -7251,10 +7261,10 @@ uint_fast16_t float64_to_uint16(float64 a, float_status *status)
     return res;
 }
 
-uint_fast16_t float64_to_uint16_round_to_zero(float64 a, float_status *status)
+uint16_t float64_to_uint16_round_to_zero(float64 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
     v = float64_to_int64_round_to_zero(a, status);
@@ -7285,7 +7295,8 @@ uint_fast16_t float64_to_uint16_round_to_zero(float64 a, float_status *status)
 uint64_t float64_to_uint64(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, aSigExtra;
     a = float64_squash_input_denormal(a, status);
 
diff --git a/qemu/fsdev/virtio-9p-marshal.c b/qemu/fsdev/9p-iov-marshal.c
index 20f308b76..fb40bdf0d 100644
--- a/qemu/fsdev/virtio-9p-marshal.c
+++ b/qemu/fsdev/9p-iov-marshal.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2010
  *
@@ -11,52 +11,15 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <glib/gprintf.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/time.h>
 #include <utime.h>
 #include <sys/uio.h>
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
 
-#include "qemu/compiler.h"
-#include "virtio-9p-marshal.h"
+#include "9p-iov-marshal.h"
 #include "qemu/bswap.h"
 
-void v9fs_string_free(V9fsString *str)
-{
-    g_free(str->data);
-    str->data = NULL;
-    str->size = 0;
-}
-
-void v9fs_string_null(V9fsString *str)
-{
-    v9fs_string_free(str);
-}
-
-void GCC_FMT_ATTR(2, 3)
-v9fs_string_sprintf(V9fsString *str, const char *fmt, ...)
-{
-    va_list ap;
-
-    v9fs_string_free(str);
-
-    va_start(ap, fmt);
-    str->size = g_vasprintf(&str->data, fmt, ap);
-    va_end(ap);
-}
-
-void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs)
-{
-    v9fs_string_free(lhs);
-    v9fs_string_sprintf(lhs, "%s", rhs->data);
-}
-
-
 static ssize_t v9fs_packunpack(void *addr, struct iovec *sg, int sg_count,
                                size_t offset, size_t size, int pack)
 {
@@ -108,15 +71,13 @@ ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
     return v9fs_packunpack((void *)src, in_sg, in_num, offset, size, 1);
 }
 
-ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
-                       int bswap, const char *fmt, ...)
+ssize_t v9fs_iov_vunmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                            int bswap, const char *fmt, va_list ap)
 {
     int i;
-    va_list ap;
     ssize_t copied = 0;
     size_t old_offset = offset;
 
-    va_start(ap, fmt);
     for (i = 0; fmt[i]; i++) {
         switch (fmt[i]) {
         case 'b': {
@@ -159,8 +120,8 @@ ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
         }
         case 's': {
             V9fsString *str = va_arg(ap, V9fsString *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "w", &str->size);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "w", &str->size);
             if (copied > 0) {
                 offset += copied;
                 str->data = g_malloc(str->size + 1);
@@ -176,56 +137,70 @@ ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
         }
         case 'Q': {
             V9fsQID *qidp = va_arg(ap, V9fsQID *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap, "bdq",
-                                    &qidp->type, &qidp->version, &qidp->path);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "bdq", &qidp->type, &qidp->version,
+                                        &qidp->path);
             break;
         }
         case 'S': {
             V9fsStat *statp = va_arg(ap, V9fsStat *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "wwdQdddqsssssddd",
-                                    &statp->size, &statp->type, &statp->dev,
-                                    &statp->qid, &statp->mode, &statp->atime,
-                                    &statp->mtime, &statp->length,
-                                    &statp->name, &statp->uid, &statp->gid,
-                                    &statp->muid, &statp->extension,
-                                    &statp->n_uid, &statp->n_gid,
-                                    &statp->n_muid);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "wwdQdddqsssssddd",
+                                        &statp->size, &statp->type,
+                                        &statp->dev, &statp->qid,
+                                        &statp->mode, &statp->atime,
+                                        &statp->mtime, &statp->length,
+                                        &statp->name, &statp->uid,
+                                        &statp->gid, &statp->muid,
+                                        &statp->extension,
+                                        &statp->n_uid, &statp->n_gid,
+                                        &statp->n_muid);
             break;
         }
         case 'I': {
             V9fsIattr *iattr = va_arg(ap, V9fsIattr *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "ddddqqqqq",
-                                    &iattr->valid, &iattr->mode,
-                                    &iattr->uid, &iattr->gid, &iattr->size,
-                                    &iattr->atime_sec, &iattr->atime_nsec,
-                                    &iattr->mtime_sec, &iattr->mtime_nsec);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "ddddqqqqq",
+                                        &iattr->valid, &iattr->mode,
+                                        &iattr->uid, &iattr->gid,
+                                        &iattr->size, &iattr->atime_sec,
+                                        &iattr->atime_nsec,
+                                        &iattr->mtime_sec,
+                                        &iattr->mtime_nsec);
             break;
         }
         default:
             break;
         }
         if (copied < 0) {
-            va_end(ap);
             return copied;
         }
         offset += copied;
     }
-    va_end(ap);
 
     return offset - old_offset;
 }
 
-ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
-                     int bswap, const char *fmt, ...)
+ssize_t v9fs_iov_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                           int bswap, const char *fmt, ...)
 {
-    int i;
+    ssize_t ret;
     va_list ap;
+
+    va_start(ap, fmt);
+    ret = v9fs_iov_vunmarshal(out_sg, out_num, offset, bswap, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+ssize_t v9fs_iov_vmarshal(struct iovec *in_sg, int in_num, size_t offset,
+                          int bswap, const char *fmt, va_list ap)
+{
+    int i;
     ssize_t copied = 0;
     size_t old_offset = offset;
 
-    va_start(ap, fmt);
     for (i = 0; fmt[i]; i++) {
         switch (fmt[i]) {
         case 'b': {
@@ -265,8 +240,8 @@ ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
         }
         case 's': {
             V9fsString *str = va_arg(ap, V9fsString *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                  "w", str->size);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "w", str->size);
             if (copied > 0) {
                 offset += copied;
                 copied = v9fs_pack(in_sg, in_num, offset, str->data, str->size);
@@ -275,49 +250,65 @@ ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
         }
         case 'Q': {
             V9fsQID *qidp = va_arg(ap, V9fsQID *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap, "bdq",
-                                  qidp->type, qidp->version, qidp->path);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap, "bdq",
+                                      qidp->type, qidp->version,
+                                      qidp->path);
             break;
         }
         case 'S': {
             V9fsStat *statp = va_arg(ap, V9fsStat *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                  "wwdQdddqsssssddd",
-                                  statp->size, statp->type, statp->dev,
-                                  &statp->qid, statp->mode, statp->atime,
-                                  statp->mtime, statp->length, &statp->name,
-                                  &statp->uid, &statp->gid, &statp->muid,
-                                  &statp->extension, statp->n_uid,
-                                  statp->n_gid, statp->n_muid);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "wwdQdddqsssssddd",
+                                      statp->size, statp->type, statp->dev,
+                                      &statp->qid, statp->mode, statp->atime,
+                                      statp->mtime, statp->length,
+                                      &statp->name,
+                                      &statp->uid, &statp->gid, &statp->muid,
+                                      &statp->extension, statp->n_uid,
+                                      statp->n_gid, statp->n_muid);
             break;
         }
         case 'A': {
             V9fsStatDotl *statp = va_arg(ap, V9fsStatDotl *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                   "qQdddqqqqqqqqqqqqqqq",
-                                   statp->st_result_mask,
-                                   &statp->qid, statp->st_mode,
-                                   statp->st_uid, statp->st_gid,
-                                   statp->st_nlink, statp->st_rdev,
-                                   statp->st_size, statp->st_blksize,
-                                   statp->st_blocks, statp->st_atime_sec,
-                                   statp->st_atime_nsec, statp->st_mtime_sec,
-                                   statp->st_mtime_nsec, statp->st_ctime_sec,
-                                   statp->st_ctime_nsec, statp->st_btime_sec,
-                                   statp->st_btime_nsec, statp->st_gen,
-                                   statp->st_data_version);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "qQdddqqqqqqqqqqqqqqq",
+                                      statp->st_result_mask,
+                                      &statp->qid, statp->st_mode,
+                                      statp->st_uid, statp->st_gid,
+                                      statp->st_nlink, statp->st_rdev,
+                                      statp->st_size, statp->st_blksize,
+                                      statp->st_blocks, statp->st_atime_sec,
+                                      statp->st_atime_nsec,
+                                      statp->st_mtime_sec,
+                                      statp->st_mtime_nsec,
+                                      statp->st_ctime_sec,
+                                      statp->st_ctime_nsec,
+                                      statp->st_btime_sec,
+                                      statp->st_btime_nsec, statp->st_gen,
+                                      statp->st_data_version);
             break;
         }
         default:
             break;
         }
         if (copied < 0) {
-            va_end(ap);
             return copied;
         }
         offset += copied;
     }
-    va_end(ap);
 
     return offset - old_offset;
 }
+
+ssize_t v9fs_iov_marshal(struct iovec *in_sg, int in_num, size_t offset,
+                         int bswap, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = v9fs_iov_vmarshal(in_sg, in_num, offset, bswap, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
diff --git a/qemu/fsdev/9p-iov-marshal.h b/qemu/fsdev/9p-iov-marshal.h
new file mode 100644
index 000000000..6bccbfb41
--- /dev/null
+++ b/qemu/fsdev/9p-iov-marshal.h
@@ -0,0 +1,18 @@
+#ifndef _QEMU_9P_IOV_MARSHAL_H
+#define _QEMU_9P_IOV_MARSHAL_H
+
+#include "9p-marshal.h"
+
+
+ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
+                  const void *src, size_t size);
+ssize_t v9fs_iov_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                           int bswap, const char *fmt, ...);
+ssize_t v9fs_iov_marshal(struct iovec *in_sg, int in_num, size_t offset,
+                         int bswap, const char *fmt, ...);
+
+ssize_t v9fs_iov_vunmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                            int bswap, const char *fmt, va_list ap);
+ssize_t v9fs_iov_vmarshal(struct iovec *in_sg, int in_num, size_t offset,
+                          int bswap, const char *fmt, va_list ap);
+#endif
diff --git a/qemu/fsdev/9p-marshal.c b/qemu/fsdev/9p-marshal.c
new file mode 100644
index 000000000..183d3667c
--- /dev/null
+++ b/qemu/fsdev/9p-marshal.c
@@ -0,0 +1,51 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include <glib/gprintf.h>
+#include <dirent.h>
+#include <utime.h>
+#include <sys/uio.h>
+
+#include "9p-marshal.h"
+
+void v9fs_string_free(V9fsString *str)
+{
+    g_free(str->data);
+    str->data = NULL;
+    str->size = 0;
+}
+
+void v9fs_string_null(V9fsString *str)
+{
+    v9fs_string_free(str);
+}
+
+void GCC_FMT_ATTR(2, 3)
+v9fs_string_sprintf(V9fsString *str, const char *fmt, ...)
+{
+    va_list ap;
+
+    v9fs_string_free(str);
+
+    va_start(ap, fmt);
+    str->size = g_vasprintf(&str->data, fmt, ap);
+    va_end(ap);
+}
+
+void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs)
+{
+    v9fs_string_free(lhs);
+    v9fs_string_sprintf(lhs, "%s", rhs->data);
+}
diff --git a/qemu/fsdev/virtio-9p-marshal.h b/qemu/fsdev/9p-marshal.h
index 5df65a835..e91b24e9c 100644
--- a/qemu/fsdev/virtio-9p-marshal.h
+++ b/qemu/fsdev/9p-marshal.h
@@ -1,5 +1,5 @@
-#ifndef _QEMU_VIRTIO_9P_MARSHAL_H
-#define _QEMU_VIRTIO_9P_MARSHAL_H
+#ifndef _QEMU_9P_MARSHAL_H
+#define _QEMU_9P_MARSHAL_H
 
 typedef struct V9fsString
 {
@@ -30,7 +30,7 @@ typedef struct V9fsStat
     V9fsString muid;
     /* 9p2000.u */
     V9fsString extension;
-   int32_t n_uid;
+    int32_t n_uid;
     int32_t n_gid;
     int32_t n_muid;
 } V9fsStat;
@@ -81,10 +81,4 @@ extern void v9fs_string_null(V9fsString *str);
 extern void v9fs_string_sprintf(V9fsString *str, const char *fmt, ...);
 extern void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs);
 
-ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
-                  const void *src, size_t size);
-ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
-                       int bswap, const char *fmt, ...);
-ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
-                     int bswap, const char *fmt, ...);
 #endif
diff --git a/qemu/fsdev/Makefile.objs b/qemu/fsdev/Makefile.objs
index c27dad3f6..1b120a4a7 100644
--- a/qemu/fsdev/Makefile.objs
+++ b/qemu/fsdev/Makefile.objs
@@ -1,7 +1,7 @@
 ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
 # Lots of the fsdev/9pcode is pulled in by vl.c via qemu_fsdev_add.
 # only pull in the actual virtio-9p device if we also enabled virtio.
-common-obj-y = qemu-fsdev.o virtio-9p-marshal.o
+common-obj-y = qemu-fsdev.o 9p-marshal.o 9p-iov-marshal.o
 else
 common-obj-y = qemu-fsdev-dummy.o
 endif
diff --git a/qemu/fsdev/file-op-9p.h b/qemu/fsdev/file-op-9p.h
index 956fda091..b8c26024a 100644
--- a/qemu/fsdev/file-op-9p.h
+++ b/qemu/fsdev/file-op-9p.h
@@ -12,11 +12,8 @@
  */
 #ifndef _FILEOP_H
 #define _FILEOP_H
-#include <sys/types.h>
 #include <dirent.h>
-#include <sys/time.h>
 #include <utime.h>
-#include <sys/stat.h>
 #include <sys/uio.h>
 #include <sys/vfs.h>
 
diff --git a/qemu/fsdev/qemu-fsdev-dummy.c b/qemu/fsdev/qemu-fsdev-dummy.c
index 7dc2630a7..7622e86c1 100644
--- a/qemu/fsdev/qemu-fsdev-dummy.c
+++ b/qemu/fsdev/qemu-fsdev-dummy.c
@@ -10,8 +10,7 @@
  * the COPYING file in the top-level directory.
  *
  */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "qemu-fsdev.h"
 #include "qemu/config-file.h"
 #include "qemu/module.h"
diff --git a/qemu/fsdev/qemu-fsdev-opts.c b/qemu/fsdev/qemu-fsdev-opts.c
index 6311c7a7e..88a4ac325 100644
--- a/qemu/fsdev/qemu-fsdev-opts.c
+++ b/qemu/fsdev/qemu-fsdev-opts.c
@@ -5,6 +5,7 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/config-file.h"
 #include "qemu/option.h"
 #include "qemu/module.h"
@@ -82,4 +83,4 @@ static void fsdev_register_config(void)
     qemu_add_opts(&qemu_fsdev_opts);
     qemu_add_opts(&qemu_virtfs_opts);
 }
-machine_init(fsdev_register_config);
+opts_init(fsdev_register_config);
diff --git a/qemu/fsdev/qemu-fsdev.c b/qemu/fsdev/qemu-fsdev.c
index ccfec139a..bf7f0b07f 100644
--- a/qemu/fsdev/qemu-fsdev.c
+++ b/qemu/fsdev/qemu-fsdev.c
@@ -10,13 +10,12 @@
  * the COPYING file in the top-level directory.
  *
  */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "qemu-fsdev.h"
 #include "qemu/queue.h"
-#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 
 static QTAILQ_HEAD(FsDriverEntry_head, FsDriverListEntry) fsdriver_entries =
     QTAILQ_HEAD_INITIALIZER(fsdriver_entries);
@@ -40,7 +39,7 @@ int qemu_fsdev_add(QemuOpts *opts)
     bool ro = qemu_opt_get_bool(opts, "readonly", 0);
 
     if (!fsdev_id) {
-        fprintf(stderr, "fsdev: No id specified\n");
+        error_report("fsdev: No id specified");
         return -1;
     }
 
@@ -52,11 +51,11 @@ int qemu_fsdev_add(QemuOpts *opts)
         }
 
         if (i == ARRAY_SIZE(FsDrivers)) {
-            fprintf(stderr, "fsdev: fsdriver %s not found\n", fsdriver);
+            error_report("fsdev: fsdriver %s not found", fsdriver);
             return -1;
         }
     } else {
-        fprintf(stderr, "fsdev: No fsdriver specified\n");
+        error_report("fsdev: No fsdriver specified");
         return -1;
     }
 
diff --git a/qemu/fsdev/virtfs-proxy-helper.c b/qemu/fsdev/virtfs-proxy-helper.c
index 9097d15c9..54f7ad1c4 100644
--- a/qemu/fsdev/virtfs-proxy-helper.c
+++ b/qemu/fsdev/virtfs-proxy-helper.c
@@ -9,6 +9,7 @@
  * the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <sys/resource.h>
 #include <getopt.h>
 #include <syslog.h>
@@ -23,9 +24,9 @@
 #include "qemu-common.h"
 #include "qemu/sockets.h"
 #include "qemu/xattr.h"
-#include "virtio-9p-marshal.h"
-#include "hw/9pfs/virtio-9p-proxy.h"
-#include "fsdev/virtio-9p-marshal.h"
+#include "9p-iov-marshal.h"
+#include "hw/9pfs/9p-proxy.h"
+#include "fsdev/9p-iov-marshal.h"
 
 #define PROGNAME "virtfs-proxy-helper"
 
@@ -1128,10 +1129,19 @@ int main(int argc, char **argv)
         }
     }
 
+    if (chdir("/") < 0) {
+        do_perror("chdir");
+        goto error;
+    }
+    if (chroot(rpath) < 0) {
+        do_perror("chroot");
+        goto error;
+    }
+
     get_version = false;
 #ifdef FS_IOC_GETVERSION
     /* check whether underlying FS support IOC_GETVERSION */
-    retval = statfs(rpath, &st_fs);
+    retval = statfs("/", &st_fs);
     if (!retval) {
         switch (st_fs.f_type) {
         case EXT2_SUPER_MAGIC:
@@ -1144,16 +1154,7 @@ int main(int argc, char **argv)
     }
 #endif
 
-    if (chdir("/") < 0) {
-        do_perror("chdir");
-        goto error;
-    }
-    if (chroot(rpath) < 0) {
-        do_perror("chroot");
-        goto error;
-    }
     umask(0);
-
     if (init_capabilities() < 0) {
         goto error;
     }
diff --git a/qemu/fsdev/virtfs-proxy-helper.texi b/qemu/fsdev/virtfs-proxy-helper.texi
index e60e3b946..6eb2d5096 100644
--- a/qemu/fsdev/virtfs-proxy-helper.texi
+++ b/qemu/fsdev/virtfs-proxy-helper.texi
@@ -1,6 +1,6 @@
 @example
 @c man begin SYNOPSIS
-usage: virtfs-proxy-helper options
+@command{virtfs-proxy-helper} @var{options}
 @c man end
 @end example
 
@@ -28,8 +28,8 @@ QEMU and proxy helper communicate using this socket. QEMU proxy fs
 driver sends filesystem request to proxy helper and receives the
 response from it.
 
-Proxy helper is designed so that it can drop the root privilege with
-retaining capbilities needed for doing filesystem operations only.
+The proxy helper is designed so that it can drop root privileges except
+for the capabilities needed for doing filesystem operations.
 
 @end table
 @c man end
diff --git a/qemu/gdb-xml/power-vsx.xml b/qemu/gdb-xml/power-vsx.xml
new file mode 100644
index 000000000..fd290e970
--- /dev/null
+++ b/qemu/gdb-xml/power-vsx.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2008-2015 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!-- POWER7 VSX registers that do not overlap existing FP and VMX
+     registers.  -->
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.power.vsx">
+  <reg name="vs0h" bitsize="64" type="uint64"/>
+  <reg name="vs1h" bitsize="64" type="uint64"/>
+  <reg name="vs2h" bitsize="64" type="uint64"/>
+  <reg name="vs3h" bitsize="64" type="uint64"/>
+  <reg name="vs4h" bitsize="64" type="uint64"/>
+  <reg name="vs5h" bitsize="64" type="uint64"/>
+  <reg name="vs6h" bitsize="64" type="uint64"/>
+  <reg name="vs7h" bitsize="64" type="uint64"/>
+  <reg name="vs8h" bitsize="64" type="uint64"/>
+  <reg name="vs9h" bitsize="64" type="uint64"/>
+  <reg name="vs10h" bitsize="64" type="uint64"/>
+  <reg name="vs11h" bitsize="64" type="uint64"/>
+  <reg name="vs12h" bitsize="64" type="uint64"/>
+  <reg name="vs13h" bitsize="64" type="uint64"/>
+  <reg name="vs14h" bitsize="64" type="uint64"/>
+  <reg name="vs15h" bitsize="64" type="uint64"/>
+  <reg name="vs16h" bitsize="64" type="uint64"/>
+  <reg name="vs17h" bitsize="64" type="uint64"/>
+  <reg name="vs18h" bitsize="64" type="uint64"/>
+  <reg name="vs19h" bitsize="64" type="uint64"/>
+  <reg name="vs20h" bitsize="64" type="uint64"/>
+  <reg name="vs21h" bitsize="64" type="uint64"/>
+  <reg name="vs22h" bitsize="64" type="uint64"/>
+  <reg name="vs23h" bitsize="64" type="uint64"/>
+  <reg name="vs24h" bitsize="64" type="uint64"/>
+  <reg name="vs25h" bitsize="64" type="uint64"/>
+  <reg name="vs26h" bitsize="64" type="uint64"/>
+  <reg name="vs27h" bitsize="64" type="uint64"/>
+  <reg name="vs28h" bitsize="64" type="uint64"/>
+  <reg name="vs29h" bitsize="64" type="uint64"/>
+  <reg name="vs30h" bitsize="64" type="uint64"/>
+  <reg name="vs31h" bitsize="64" type="uint64"/>
+</feature>
diff --git a/qemu/gdb-xml/s390-cr.xml b/qemu/gdb-xml/s390-cr.xml
new file mode 100644
index 000000000..5246beaab
--- /dev/null
+++ b/qemu/gdb-xml/s390-cr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.cr">
+  <reg name="cr0" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr1" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr2" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr3" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr4" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr5" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr6" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr7" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr8" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr9" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr10" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr11" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr12" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr13" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr14" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr15" bitsize="64" type="uint64" group="control"/>
+</feature>
diff --git a/qemu/gdb-xml/s390-virt.xml b/qemu/gdb-xml/s390-virt.xml
new file mode 100644
index 000000000..e2e9a7ad3
--- /dev/null
+++ b/qemu/gdb-xml/s390-virt.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.virt">
+  <reg name="ckc" bitsize="64" type="uint64" group="system"/>
+  <reg name="cputm" bitsize="64" type="uint64" group="system"/>
+  <reg name="last_break" bitsize="64" type="code_ptr" group="system"/>
+  <reg name="prefix" bitsize="64" type="data_ptr" group="system"/>
+  <reg name="pp" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_token" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_select" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_compare" bitsize="64" type="uint64" group="system"/>
+</feature>
diff --git a/qemu/gdbstub.c b/qemu/gdbstub.c
index ffe7e6efb..0e431fd4d 100644
--- a/qemu/gdbstub.c
+++ b/qemu/gdbstub.c
@@ -16,17 +16,11 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
-#include "qemu-common.h"
-#ifdef CONFIG_USER_ONLY
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 
+#ifdef CONFIG_USER_ONLY
 #include "qemu.h"
 #else
 #include "monitor/monitor.h"
@@ -540,13 +534,20 @@ static const char *get_feature_xml(const char *p, const char **newp,
             GDBRegisterState *r;
             CPUState *cpu = first_cpu;
 
-            snprintf(target_xml, sizeof(target_xml),
-                     "<?xml version=\"1.0\"?>"
-                     "<!DOCTYPE target SYSTEM \"gdb-target.dtd\">"
-                     "<target>"
-                     "<xi:include href=\"%s\"/>",
-                     cc->gdb_core_xml_file);
-
+            pstrcat(target_xml, sizeof(target_xml),
+                    "<?xml version=\"1.0\"?>"
+                    "<!DOCTYPE target SYSTEM \"gdb-target.dtd\">"
+                    "<target>");
+            if (cc->gdb_arch_name) {
+                gchar *arch = cc->gdb_arch_name(cpu);
+                pstrcat(target_xml, sizeof(target_xml), "<architecture>");
+                pstrcat(target_xml, sizeof(target_xml), arch);
+                pstrcat(target_xml, sizeof(target_xml), "</architecture>");
+                g_free(arch);
+            }
+            pstrcat(target_xml, sizeof(target_xml), "<xi:include href=\"");
+            pstrcat(target_xml, sizeof(target_xml), cc->gdb_core_xml_file);
+            pstrcat(target_xml, sizeof(target_xml), "\"/>");
             for (r = cpu->gdb_regs; r; r = r->next) {
                 pstrcat(target_xml, sizeof(target_xml), "<xi:include href=\"");
                 pstrcat(target_xml, sizeof(target_xml), r->xml);
@@ -956,6 +957,13 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
         if (*p == ',')
             p++;
         len = strtoull(p, NULL, 16);
+
+        /* memtohex() doubles the required space */
+        if (len > MAX_PACKET_LENGTH / 2) {
+            put_packet (s, "E22");
+            break;
+        }
+
         if (target_memory_rw_debug(s->g_cpu, addr, mem_buf, len, false) != 0) {
             put_packet (s, "E14");
         } else {
@@ -970,6 +978,12 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
         len = strtoull(p, (char **)&p, 16);
         if (*p == ':')
             p++;
+
+        /* hextomem() reads 2*len bytes */
+        if (len > strlen(p) / 2) {
+            put_packet (s, "E22");
+            break;
+        }
         hextomem(mem_buf, p, len);
         if (target_memory_rw_debug(s->g_cpu, addr, mem_buf, len,
                                    true) != 0) {
@@ -1107,7 +1121,8 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
             cpu = find_cpu(thread);
             if (cpu != NULL) {
                 cpu_synchronize_state(cpu);
-                len = snprintf((char *)mem_buf, sizeof(mem_buf),
+                /* memtohex() doubles the required space */
+                len = snprintf((char *)mem_buf, sizeof(buf) / 2,
                                "CPU#%d [%s]", cpu->cpu_index,
                                cpu->halted ? "halted " : "running");
                 memtohex(buf, mem_buf, len);
@@ -1136,8 +1151,8 @@ static int gdb_handle_packet(GDBState *s, const char *line_buf)
                 put_packet(s, "E01");
                 break;
             }
-            hextomem(mem_buf, p + 5, len);
             len = len / 2;
+            hextomem(mem_buf, p + 5, len);
             mem_buf[len++] = 0;
             qemu_chr_be_write(s->mon_chr, mem_buf, len);
             put_packet(s, "OK");
@@ -1301,9 +1316,8 @@ send_packet:
     %x  - target_ulong argument printed in hex.
     %lx - 64-bit argument printed in hex.
     %s  - string pointer (target_ulong) and length (int) pair.  */
-void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va)
 {
-    va_list va;
     char *p;
     char *p_end;
     target_ulong addr;
@@ -1317,7 +1331,6 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
 #ifndef CONFIG_USER_ONLY
     vm_stop(RUN_STATE_DEBUG);
 #endif
-    va_start(va, fmt);
     p = s->syscall_buf;
     p_end = &s->syscall_buf[sizeof(s->syscall_buf)];
     *(p++) = 'F';
@@ -1351,7 +1364,6 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
         }
     }
     *p = 0;
-    va_end(va);
 #ifdef CONFIG_USER_ONLY
     put_packet(s, s->syscall_buf);
     gdb_handlesig(s->c_cpu, 0);
@@ -1362,10 +1374,19 @@ void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
        is still in the running state, which can cause packets to be dropped
        and state transition 'T' packets to be sent while the syscall is still
        being processed.  */
-    cpu_exit(s->c_cpu);
+    qemu_cpu_kick(s->c_cpu);
 #endif
 }
 
+void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+{
+    va_list va;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+}
+
 static void gdb_read_byte(GDBState *s, int ch)
 {
     int i, csum;
@@ -1712,6 +1733,7 @@ int gdbserver_start(const char *device)
     char gdbstub_device_name[128];
     CharDriverState *chr = NULL;
     CharDriverState *mon_chr;
+    ChardevCommon common = { 0 };
 
     if (!device)
         return -1;
@@ -1731,7 +1753,7 @@ int gdbserver_start(const char *device)
             sigaction(SIGINT, &act, NULL);
         }
 #endif
-        chr = qemu_chr_new("gdb", device, NULL);
+        chr = qemu_chr_new_noreplay("gdb", device, NULL);
         if (!chr)
             return -1;
 
@@ -1748,7 +1770,7 @@ int gdbserver_start(const char *device)
         qemu_add_vm_change_state_handler(gdb_vm_state_change, NULL);
 
         /* Initialize a monitor terminal for gdb */
-        mon_chr = qemu_chr_alloc();
+        mon_chr = qemu_chr_alloc(&common, &error_abort);
         mon_chr->chr_write = gdb_monitor_write;
         monitor_init(mon_chr, 0);
     } else {
diff --git a/qemu/hmp-commands-info.hx b/qemu/hmp-commands-info.hx
new file mode 100644
index 000000000..52539c310
--- /dev/null
+++ b/qemu/hmp-commands-info.hx
@@ -0,0 +1,809 @@
+HXCOMM Use DEFHEADING() to define headings in both help text and texi
+HXCOMM Text between STEXI and ETEXI are copied to texi version and
+HXCOMM discarded from C version
+HXCOMM DEF(command, args, callback, arg_string, help) is used to construct
+HXCOMM monitor info commands
+HXCOMM HXCOMM can be used for comments, discarded from both texi and C
+
+STEXI
+@table @option
+@item info @var{subcommand}
+@findex info
+Show various information about the system state.
+@table @option
+ETEXI
+
+    {
+        .name       = "version",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the version of QEMU",
+        .mhandler.cmd = hmp_info_version,
+    },
+
+STEXI
+@item info version
+@findex version
+Show the version of QEMU.
+ETEXI
+
+    {
+        .name       = "network",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the network state",
+        .mhandler.cmd = hmp_info_network,
+    },
+
+STEXI
+@item info network
+@findex network
+Show the network state.
+ETEXI
+
+    {
+        .name       = "chardev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the character devices",
+        .mhandler.cmd = hmp_info_chardev,
+    },
+
+STEXI
+@item info chardev
+@findex chardev
+Show the character devices.
+ETEXI
+
+    {
+        .name       = "block",
+        .args_type  = "nodes:-n,verbose:-v,device:B?",
+        .params     = "[-n] [-v] [device]",
+        .help       = "show info of one block device or all block devices "
+                      "(-n: show named nodes; -v: show details)",
+        .mhandler.cmd = hmp_info_block,
+    },
+
+STEXI
+@item info block
+@findex block
+Show info of one block device or all block devices.
+ETEXI
+
+    {
+        .name       = "blockstats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show block device statistics",
+        .mhandler.cmd = hmp_info_blockstats,
+    },
+
+STEXI
+@item info blockstats
+@findex blockstats
+Show block device statistics.
+ETEXI
+
+    {
+        .name       = "block-jobs",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show progress of ongoing block device operations",
+        .mhandler.cmd = hmp_info_block_jobs,
+    },
+
+STEXI
+@item info block-jobs
+@findex block-jobs
+Show progress of ongoing block device operations.
+ETEXI
+
+    {
+        .name       = "registers",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the cpu registers",
+        .mhandler.cmd = hmp_info_registers,
+    },
+
+STEXI
+@item info registers
+@findex registers
+Show the cpu registers.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "lapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show local apic state",
+        .mhandler.cmd = hmp_info_local_apic,
+    },
+#endif
+
+STEXI
+@item info lapic
+@findex lapic
+Show local APIC state
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "ioapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show io apic state",
+        .mhandler.cmd = hmp_info_io_apic,
+    },
+#endif
+
+STEXI
+@item info ioapic
+@findex ioapic
+Show io APIC state
+ETEXI
+
+    {
+        .name       = "cpus",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show infos for each CPU",
+        .mhandler.cmd = hmp_info_cpus,
+    },
+
+STEXI
+@item info cpus
+@findex cpus
+Show infos for each CPU.
+ETEXI
+
+    {
+        .name       = "history",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the command line history",
+        .mhandler.cmd = hmp_info_history,
+    },
+
+STEXI
+@item info history
+@findex history
+Show the command line history.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
+    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
+    {
+        .name       = "irq",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the interrupts statistics (if available)",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_irq,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_irq,
+#else
+        .mhandler.cmd = hmp_info_irq,
+#endif
+    },
+
+STEXI
+@item info irq
+@findex irq
+Show the interrupts statistics (if available).
+ETEXI
+
+    {
+        .name       = "pic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show i8259 (PIC) state",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_pic,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_pic,
+#else
+        .mhandler.cmd = hmp_info_pic,
+#endif
+    },
+#endif
+
+STEXI
+@item info pic
+@findex pic
+Show i8259 (PIC) state.
+ETEXI
+
+    {
+        .name       = "pci",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show PCI info",
+        .mhandler.cmd = hmp_info_pci,
+    },
+
+STEXI
+@item info pci
+@findex pci
+Show PCI information.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
+    defined(TARGET_PPC) || defined(TARGET_XTENSA)
+    {
+        .name       = "tlb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show virtual to physical memory mappings",
+        .mhandler.cmd = hmp_info_tlb,
+    },
+#endif
+
+STEXI
+@item info tlb
+@findex tlb
+Show virtual to physical memory mappings.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "mem",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the active virtual memory mappings",
+        .mhandler.cmd = hmp_info_mem,
+    },
+#endif
+
+STEXI
+@item info mem
+@findex mem
+Show the active virtual memory mappings.
+ETEXI
+
+    {
+        .name       = "mtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory tree",
+        .mhandler.cmd = hmp_info_mtree,
+    },
+
+STEXI
+@item info mtree
+@findex mtree
+Show memory tree.
+ETEXI
+
+    {
+        .name       = "jit",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler info",
+        .mhandler.cmd = hmp_info_jit,
+    },
+
+STEXI
+@item info jit
+@findex jit
+Show dynamic compiler info.
+ETEXI
+
+    {
+        .name       = "opcount",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler opcode counters",
+        .mhandler.cmd = hmp_info_opcount,
+    },
+
+STEXI
+@item info opcount
+@findex opcount
+Show dynamic compiler opcode counters
+ETEXI
+
+    {
+        .name       = "kvm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show KVM information",
+        .mhandler.cmd = hmp_info_kvm,
+    },
+
+STEXI
+@item info kvm
+@findex kvm
+Show KVM information.
+ETEXI
+
+    {
+        .name       = "numa",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show NUMA information",
+        .mhandler.cmd = hmp_info_numa,
+    },
+
+STEXI
+@item info numa
+@findex numa
+Show NUMA information.
+ETEXI
+
+    {
+        .name       = "usb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show guest USB devices",
+        .mhandler.cmd = hmp_info_usb,
+    },
+
+STEXI
+@item info usb
+@findex usb
+Show guest USB devices.
+ETEXI
+
+    {
+        .name       = "usbhost",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show host USB devices",
+        .mhandler.cmd = hmp_info_usbhost,
+    },
+
+STEXI
+@item info usbhost
+@findex usbhost
+Show host USB devices.
+ETEXI
+
+    {
+        .name       = "profile",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show profiling information",
+        .mhandler.cmd = hmp_info_profile,
+    },
+
+STEXI
+@item info profile
+@findex profile
+Show profiling information.
+ETEXI
+
+    {
+        .name       = "capture",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show capture information",
+        .mhandler.cmd = hmp_info_capture,
+    },
+
+STEXI
+@item info capture
+@findex capture
+Show capture information.
+ETEXI
+
+    {
+        .name       = "snapshots",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the currently saved VM snapshots",
+        .mhandler.cmd = hmp_info_snapshots,
+    },
+
+STEXI
+@item info snapshots
+@findex snapshots
+Show the currently saved VM snapshots.
+ETEXI
+
+    {
+        .name       = "status",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM status (running|paused)",
+        .mhandler.cmd = hmp_info_status,
+    },
+
+STEXI
+@item info status
+@findex status
+Show the current VM status (running|paused).
+ETEXI
+
+    {
+        .name       = "mice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show which guest mouse is receiving events",
+        .mhandler.cmd = hmp_info_mice,
+    },
+
+STEXI
+@item info mice
+@findex mice
+Show which guest mouse is receiving events.
+ETEXI
+
+    {
+        .name       = "vnc",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the vnc server status",
+        .mhandler.cmd = hmp_info_vnc,
+    },
+
+STEXI
+@item info vnc
+@findex vnc
+Show the vnc server status.
+ETEXI
+
+#if defined(CONFIG_SPICE)
+    {
+        .name       = "spice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the spice server status",
+        .mhandler.cmd = hmp_info_spice,
+    },
+#endif
+
+STEXI
+@item info spice
+@findex spice
+Show the spice server status.
+ETEXI
+
+    {
+        .name       = "name",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM name",
+        .mhandler.cmd = hmp_info_name,
+    },
+
+STEXI
+@item info name
+@findex name
+Show the current VM name.
+ETEXI
+
+    {
+        .name       = "uuid",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM UUID",
+        .mhandler.cmd = hmp_info_uuid,
+    },
+
+STEXI
+@item info uuid
+@findex uuid
+Show the current VM UUID.
+ETEXI
+
+    {
+        .name       = "cpustats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show CPU statistics",
+        .mhandler.cmd = hmp_info_cpustats,
+    },
+
+STEXI
+@item info cpustats
+@findex cpustats
+Show CPU statistics.
+ETEXI
+
+#if defined(CONFIG_SLIRP)
+    {
+        .name       = "usernet",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show user network stack connection states",
+        .mhandler.cmd = hmp_info_usernet,
+    },
+#endif
+
+STEXI
+@item info usernet
+@findex usernet
+Show user network stack connection states.
+ETEXI
+
+    {
+        .name       = "migrate",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show migration status",
+        .mhandler.cmd = hmp_info_migrate,
+    },
+
+STEXI
+@item info migrate
+@findex migrate
+Show migration status.
+ETEXI
+
+    {
+        .name       = "migrate_capabilities",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration capabilities",
+        .mhandler.cmd = hmp_info_migrate_capabilities,
+    },
+
+STEXI
+@item info migrate_capabilities
+@findex migrate_capabilities
+Show current migration capabilities.
+ETEXI
+
+    {
+        .name       = "migrate_parameters",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration parameters",
+        .mhandler.cmd = hmp_info_migrate_parameters,
+    },
+
+STEXI
+@item info migrate_parameters
+@findex migrate_parameters
+Show current migration parameters.
+ETEXI
+
+    {
+        .name       = "migrate_cache_size",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration xbzrle cache size",
+        .mhandler.cmd = hmp_info_migrate_cache_size,
+    },
+
+STEXI
+@item info migrate_cache_size
+@findex migrate_cache_size
+Show current migration xbzrle cache size.
+ETEXI
+
+    {
+        .name       = "balloon",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show balloon information",
+        .mhandler.cmd = hmp_info_balloon,
+    },
+
+STEXI
+@item info balloon
+@findex balloon
+Show balloon information.
+ETEXI
+
+    {
+        .name       = "qtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show device tree",
+        .mhandler.cmd = hmp_info_qtree,
+    },
+
+STEXI
+@item info qtree
+@findex qtree
+Show device tree.
+ETEXI
+
+    {
+        .name       = "qdm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show qdev device model list",
+        .mhandler.cmd = hmp_info_qdm,
+    },
+
+STEXI
+@item info qdm
+@findex qdm
+Show qdev device model list.
+ETEXI
+
+    {
+        .name       = "qom-tree",
+        .args_type  = "path:s?",
+        .params     = "[path]",
+        .help       = "show QOM composition tree",
+        .mhandler.cmd = hmp_info_qom_tree,
+    },
+
+STEXI
+@item info qom-tree
+@findex qom-tree
+Show QOM composition tree.
+ETEXI
+
+    {
+        .name       = "roms",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show roms",
+        .mhandler.cmd = hmp_info_roms,
+    },
+
+STEXI
+@item info roms
+@findex roms
+Show roms.
+ETEXI
+
+    {
+        .name       = "trace-events",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show available trace-events & their state",
+        .mhandler.cmd = hmp_info_trace_events,
+    },
+
+STEXI
+@item info trace-events
+@findex trace-events
+Show available trace-events & their state.
+ETEXI
+
+    {
+        .name       = "tpm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the TPM device",
+        .mhandler.cmd = hmp_info_tpm,
+    },
+
+STEXI
+@item info tpm
+@findex tpm
+Show the TPM device.
+ETEXI
+
+    {
+        .name       = "memdev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory backends",
+        .mhandler.cmd = hmp_info_memdev,
+    },
+
+STEXI
+@item info memdev
+@findex memdev
+Show memory backends
+ETEXI
+
+    {
+        .name       = "memory-devices",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory devices",
+        .mhandler.cmd = hmp_info_memory_devices,
+    },
+
+STEXI
+@item info memory-devices
+@findex memory-devices
+Show memory devices.
+ETEXI
+
+    {
+        .name       = "iothreads",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show iothreads",
+        .mhandler.cmd = hmp_info_iothreads,
+    },
+
+STEXI
+@item info iothreads
+@findex iothreads
+Show iothread's identifiers.
+ETEXI
+
+    {
+        .name       = "rocker",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker switch",
+        .mhandler.cmd = hmp_rocker,
+    },
+
+STEXI
+@item info rocker @var{name}
+@findex rocker
+Show rocker switch.
+ETEXI
+
+    {
+        .name       = "rocker-ports",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker ports",
+        .mhandler.cmd = hmp_rocker_ports,
+    },
+
+STEXI
+@item info rocker_ports @var{name}-ports
+@findex ocker-ports
+Show rocker ports.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-flows",
+        .args_type  = "name:s,tbl_id:i?",
+        .params     = "name [tbl_id]",
+        .help       = "Show rocker OF-DPA flow tables",
+        .mhandler.cmd = hmp_rocker_of_dpa_flows,
+    },
+
+STEXI
+@item info rocker_of_dpa_flows @var{name} [@var{tbl_id}]
+@findex rocker-of-dpa-flows
+Show rocker OF-DPA flow tables.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-groups",
+        .args_type  = "name:s,type:i?",
+        .params     = "name [type]",
+        .help       = "Show rocker OF-DPA groups",
+        .mhandler.cmd = hmp_rocker_of_dpa_groups,
+    },
+
+STEXI
+@item info rocker-of-dpa-groups @var{name} [@var{type}]
+@findex rocker-of-dpa-groups
+Show rocker OF-DPA groups.
+ETEXI
+
+#if defined(TARGET_S390X)
+    {
+        .name       = "skeys",
+        .args_type  = "addr:l",
+        .params     = "address",
+        .help       = "Display the value of a storage key",
+        .mhandler.cmd = hmp_info_skeys,
+    },
+#endif
+
+STEXI
+@item info skeys @var{address}
+@findex skeys
+Display the value of a storage key (s390 only)
+ETEXI
+
+    {
+        .name       = "dump",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Display the latest dump status",
+        .mhandler.cmd = hmp_info_dump,
+    },
+
+STEXI
+@item info dump
+@findex dump
+Display the latest dump status.
+ETEXI
+
+STEXI
+@end table
+ETEXI
+
+STEXI
+@end table
+ETEXI
diff --git a/qemu/hmp-commands.hx b/qemu/hmp-commands.hx
index d3b7932ff..4f4f60a0d 100644
--- a/qemu/hmp-commands.hx
+++ b/qemu/hmp-commands.hx
@@ -194,8 +194,8 @@ ETEXI
 
     {
         .name       = "change",
-        .args_type  = "device:B,target:F,arg:s?",
-        .params     = "device filename [format]",
+        .args_type  = "device:B,target:F,arg:s?,read-only-mode:s?",
+        .params     = "device filename [format [read-only-mode]]",
         .help       = "change a removable medium, optional format",
         .mhandler.cmd = hmp_change,
     },
@@ -206,7 +206,7 @@ STEXI
 Change the configuration of a device.
 
 @table @option
-@item change @var{diskdevice} @var{filename} [@var{format}]
+@item change @var{diskdevice} @var{filename} [@var{format} [@var{read-only-mode}]]
 Change the medium for a removable disk device to point to @var{filename}. eg
 
 @example
@@ -215,6 +215,20 @@ Change the medium for a removable disk device to point to @var{filename}. eg
 
 @var{format} is optional.
 
+@var{read-only-mode} may be used to change the read-only status of the device.
+It accepts the following values:
+
+@table @var
+@item retain
+Retains the current status; this is the default.
+
+@item read-only
+Makes the device read-only.
+
+@item read-write
+Makes the device writable.
+@end table
+
 @item change vnc @var{display},@var{options}
 Change the configuration of the VNC server. The valid syntax for @var{display}
 and @var{options} are described at @ref{sec_invocation}. eg
@@ -271,6 +285,7 @@ ETEXI
         .params     = "name on|off",
         .help       = "changes status of a specific trace event",
         .mhandler.cmd = hmp_trace_event,
+        .command_completion = trace_event_completion,
     },
 
 STEXI
@@ -675,7 +690,8 @@ ETEXI
 STEXI
 @item device_del @var{id}
 @findex device_del
-Remove device @var{id}.
+Remove device @var{id}. @var{id} may be a short ID
+or a QOM object path.
 ETEXI
 
     {
@@ -1006,6 +1022,23 @@ Set the parameter @var{parameter} for migration.
 ETEXI
 
     {
+        .name       = "migrate_start_postcopy",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Followup to a migration command to switch the migration"
+                      " to postcopy mode. The postcopy-ram capability must "
+                      "be set before the original migration command.",
+        .mhandler.cmd = hmp_migrate_start_postcopy,
+    },
+
+STEXI
+@item migrate_start_postcopy
+@findex migrate_start_postcopy
+Switch in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+ETEXI
+
+    {
         .name       = "client_migrate_info",
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
@@ -1023,10 +1056,11 @@ ETEXI
 
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:-p,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:i?,length:i?",
-        .params     = "[-p] [-z|-l|-s] filename [begin length]",
+        .args_type  = "paging:-p,detach:-d,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:i?,length:i?",
+        .params     = "[-p] [-d] [-z|-l|-s] filename [begin length]",
         .help       = "dump guest memory into file 'filename'.\n\t\t\t"
                       "-p: do paging to get guest's memory mapping.\n\t\t\t"
+                      "-d: return immediately (do not wait for completion).\n\t\t\t"
                       "-z: dump in kdump-compressed format, with zlib compression.\n\t\t\t"
                       "-l: dump in kdump-compressed format, with lzo compression.\n\t\t\t"
                       "-s: dump in kdump-compressed format, with snappy compression.\n\t\t\t"
@@ -1053,6 +1087,22 @@ gdb. Without -z|-l|-s, the dump format is ELF.
             together with begin.
 ETEXI
 
+#if defined(TARGET_S390X)
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .params     = "",
+        .help       = "Save guest storage keys into file 'filename'.\n",
+        .mhandler.cmd = hmp_dump_skeys,
+    },
+#endif
+
+STEXI
+@item dump-skeys @var{filename}
+@findex dump-skeys
+Save guest storage keys to a file.
+ETEXI
+
     {
         .name       = "snapshot_blkdev",
         .args_type  = "reuse:-n,device:B,snapshot-file:s?,format:s?",
@@ -1151,8 +1201,8 @@ ETEXI
 
     {
         .name       = "drive_add",
-        .args_type  = "pci_addr:s,opts:s",
-        .params     = "[[<domain>:]<bus>:]<slot>\n"
+        .args_type  = "node:-n,pci_addr:s,opts:s",
+        .params     = "[-n] [[<domain>:]<bus>:]<slot>\n"
                       "[file=file][,if=type][,bus=n]\n"
                       "[,unit=m][,media=d][,index=i]\n"
                       "[,cyls=c,heads=h,secs=s[,trans=t]]\n"
@@ -1707,121 +1757,5 @@ ETEXI
     },
 
 STEXI
-@item info @var{subcommand}
-@findex info
-Show various information about the system state.
-
-@table @option
-@item info version
-show the version of QEMU
-@item info network
-show the various VLANs and the associated devices
-@item info chardev
-show the character devices
-@item info block
-show the block devices
-@item info blockstats
-show block device statistics
-@item info registers
-show the cpu registers
-@item info cpus
-show infos for each CPU
-@item info history
-show the command line history
-@item info irq
-show the interrupts statistics (if available)
-@item info pic
-show i8259 (PIC) state
-@item info pci
-show emulated PCI device info
-@item info tlb
-show virtual to physical memory mappings (i386, SH4, SPARC, PPC, and Xtensa only)
-@item info mem
-show the active virtual memory mappings (i386 only)
-@item info jit
-show dynamic compiler info
-@item info numa
-show NUMA information
-@item info kvm
-show KVM information
-@item info usb
-show USB devices plugged on the virtual USB hub
-@item info usbhost
-show all USB host devices
-@item info profile
-show profiling information
-@item info capture
-show information about active capturing
-@item info snapshots
-show list of VM snapshots
-@item info status
-show the current VM status (running|paused)
-@item info mice
-show which guest mouse is receiving events
-@item info vnc
-show the vnc server status
-@item info name
-show the current VM name
-@item info uuid
-show the current VM UUID
-@item info cpustats
-show CPU statistics
-@item info usernet
-show user network stack connection states
-@item info migrate
-show migration status
-@item info migrate_capabilities
-show current migration capabilities
-@item info migrate_parameters
-show current migration parameters
-@item info migrate_cache_size
-show current migration XBZRLE cache size
-@item info balloon
-show balloon information
-@item info qtree
-show device tree
-@item info qdm
-show qdev device model list
-@item info qom-tree
-show object composition tree
-@item info roms
-show roms
-@item info tpm
-show the TPM device
-@item info memory-devices
-show the memory devices
-@end table
-ETEXI
-
-STEXI
-@item info trace-events
-show available trace events and their state
-ETEXI
-
-STEXI
-@item rocker @var{name}
-@findex rocker
-Show Rocker(s)
-ETEXI
-
-STEXI
-@item rocker_ports @var{name}
-@findex rocker_ports
-Show Rocker ports
-ETEXI
-
-STEXI
-@item rocker_of_dpa_flows @var{name} [@var{tbl_id}]
-@findex rocker_of_dpa_flows
-Show Rocker OF-DPA flow tables
-ETEXI
-
-STEXI
-@item rocker_of_dpa_groups @var{name} [@var{type}]
-@findex rocker_of_dpa_groups
-Show Rocker OF-DPA groups
-ETEXI
-
-STEXI
 @end table
 ETEXI
diff --git a/qemu/hmp.c b/qemu/hmp.c
index dcc66f159..d51023667 100644
--- a/qemu/hmp.c
+++ b/qemu/hmp.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hmp.h"
 #include "net/net.h"
 #include "net/eth.h"
@@ -27,10 +28,13 @@
 #include "qapi/opts-visitor.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/string-output-visitor.h"
+#include "qapi/util.h"
 #include "qapi-visit.h"
+#include "qom/object_interfaces.h"
 #include "ui/console.h"
 #include "block/qapi.h"
 #include "qemu-io.h"
+#include "qemu/cutils.h"
 
 #ifdef CONFIG_SPICE
 #include <spice/enums.h>
@@ -40,8 +44,7 @@ static void hmp_handle_error(Monitor *mon, Error **errp)
 {
     assert(errp);
     if (*errp) {
-        monitor_printf(mon, "%s\n", error_get_pretty(*errp));
-        error_free(*errp);
+        error_report_err(*errp);
     }
 }
 
@@ -232,6 +235,11 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict)
                        info->xbzrle_cache->overflow);
     }
 
+    if (info->has_x_cpu_throttle_percentage) {
+        monitor_printf(mon, "cpu throttle percentage: %" PRIu64 "\n",
+                       info->x_cpu_throttle_percentage);
+    }
+
     qapi_free_MigrationInfo(info);
     qapi_free_MigrationCapabilityStatusList(caps);
 }
@@ -272,6 +280,12 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
         monitor_printf(mon, " %s: %" PRId64,
             MigrationParameter_lookup[MIGRATION_PARAMETER_DECOMPRESS_THREADS],
             params->decompress_threads);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL],
+            params->x_cpu_throttle_initial);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT],
+            params->x_cpu_throttle_increment);
         monitor_printf(mon, "\n");
     }
 
@@ -299,17 +313,27 @@ void hmp_info_cpus(Monitor *mon, const QDict *qdict)
 
         monitor_printf(mon, "%c CPU #%" PRId64 ":", active, cpu->value->CPU);
 
-        if (cpu->value->has_pc) {
-            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->pc);
-        }
-        if (cpu->value->has_nip) {
-            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->nip);
-        }
-        if (cpu->value->has_npc) {
-            monitor_printf(mon, " npc=0x%016" PRIx64, cpu->value->npc);
-        }
-        if (cpu->value->has_PC) {
-            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->PC);
+        switch (cpu->value->arch) {
+        case CPU_INFO_ARCH_X86:
+            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->u.x86.pc);
+            break;
+        case CPU_INFO_ARCH_PPC:
+            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->u.ppc.nip);
+            break;
+        case CPU_INFO_ARCH_SPARC:
+            monitor_printf(mon, " pc=0x%016" PRIx64,
+                           cpu->value->u.q_sparc.pc);
+            monitor_printf(mon, " npc=0x%016" PRIx64,
+                           cpu->value->u.q_sparc.npc);
+            break;
+        case CPU_INFO_ARCH_MIPS:
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->u.q_mips.PC);
+            break;
+        case CPU_INFO_ARCH_TRICORE:
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->u.tricore.PC);
+            break;
+        default:
+            break;
         }
 
         if (cpu->value->halted) {
@@ -510,6 +534,7 @@ void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
                        " flush_total_time_ns=%" PRId64
                        " rd_merged=%" PRId64
                        " wr_merged=%" PRId64
+                       " idle_time_ns=%" PRId64
                        "\n",
                        stats->value->stats->rd_bytes,
                        stats->value->stats->wr_bytes,
@@ -520,7 +545,8 @@ void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
                        stats->value->stats->rd_total_time_ns,
                        stats->value->stats->flush_total_time_ns,
                        stats->value->stats->rd_merged,
-                       stats->value->stats->wr_merged);
+                       stats->value->stats->wr_merged,
+                       stats->value->stats->idle_time_ns);
     }
 
     qapi_free_BlockStatsList(stats_list);
@@ -534,8 +560,7 @@ void hmp_info_vnc(Monitor *mon, const QDict *qdict)
 
     info = qmp_query_vnc(&err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -558,8 +583,8 @@ void hmp_info_vnc(Monitor *mon, const QDict *qdict)
         for (client = info->clients; client; client = client->next) {
             monitor_printf(mon, "Client:\n");
             monitor_printf(mon, "     address: %s:%s\n",
-                           client->value->base->host,
-                           client->value->base->service);
+                           client->value->host,
+                           client->value->service);
             monitor_printf(mon, "  x509_dname: %s\n",
                            client->value->x509_dname ?
                            client->value->x509_dname : "none");
@@ -627,7 +652,7 @@ void hmp_info_spice(Monitor *mon, const QDict *qdict)
         for (chan = info->channels; chan; chan = chan->next) {
             monitor_printf(mon, "Channel:\n");
             monitor_printf(mon, "     address: %s:%s%s\n",
-                           chan->value->base->host, chan->value->base->port,
+                           chan->value->host, chan->value->port,
                            chan->value->tls ? " [tls]" : "");
             monitor_printf(mon, "     session: %" PRId64 "\n",
                            chan->value->connection_id);
@@ -657,8 +682,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict)
 
     info = qmp_query_balloon(&err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -830,18 +854,18 @@ void hmp_info_tpm(Monitor *mon, const QDict *qdict)
                        c, TpmModel_lookup[ti->model]);
 
         monitor_printf(mon, "  \\ %s: type=%s",
-                       ti->id, TpmTypeOptionsKind_lookup[ti->options->kind]);
+                       ti->id, TpmTypeOptionsKind_lookup[ti->options->type]);
 
-        switch (ti->options->kind) {
+        switch (ti->options->type) {
         case TPM_TYPE_OPTIONS_KIND_PASSTHROUGH:
-            tpo = ti->options->passthrough;
+            tpo = ti->options->u.passthrough.data;
             monitor_printf(mon, "%s%s%s%s",
                            tpo->has_path ? ",path=" : "",
                            tpo->has_path ? tpo->path : "",
                            tpo->has_cancel_path ? ",cancel-path=" : "",
                            tpo->has_cancel_path ? tpo->cancel_path : "");
             break;
-        case TPM_TYPE_OPTIONS_KIND_MAX:
+        case TPM_TYPE_OPTIONS_KIND__MAX:
             break;
         }
         monitor_printf(mon, "\n");
@@ -926,8 +950,7 @@ void hmp_ringbuf_read(Monitor *mon, const QDict *qdict)
 
     data = qmp_ringbuf_read(chardev, size, false, 0, &err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -1020,8 +1043,7 @@ void hmp_balloon(Monitor *mon, const QDict *qdict)
 
     qmp_balloon(value, &err);
     if (err) {
-        monitor_printf(mon, "balloon: %s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
 }
 
@@ -1169,8 +1191,7 @@ void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict)
 
     qmp_migrate_set_cache_size(value, &err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 }
@@ -1189,7 +1210,7 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict)
     MigrationCapabilityStatusList *caps = g_malloc0(sizeof(*caps));
     int i;
 
-    for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+    for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
         if (strcmp(cap, MigrationCapability_lookup[i]) == 0) {
             caps->value = g_malloc0(sizeof(*caps->value));
             caps->value->capability = i;
@@ -1200,16 +1221,14 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict)
         }
     }
 
-    if (i == MIGRATION_CAPABILITY_MAX) {
+    if (i == MIGRATION_CAPABILITY__MAX) {
         error_setg(&err, QERR_INVALID_PARAMETER, cap);
     }
 
     qapi_free_MigrationCapabilityStatusList(caps);
 
     if (err) {
-        monitor_printf(mon, "migrate_set_capability: %s\n",
-                       error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
 }
 
@@ -1221,9 +1240,11 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
     bool has_compress_level = false;
     bool has_compress_threads = false;
     bool has_decompress_threads = false;
+    bool has_x_cpu_throttle_initial = false;
+    bool has_x_cpu_throttle_increment = false;
     int i;
 
-    for (i = 0; i < MIGRATION_PARAMETER_MAX; i++) {
+    for (i = 0; i < MIGRATION_PARAMETER__MAX; i++) {
         if (strcmp(param, MigrationParameter_lookup[i]) == 0) {
             switch (i) {
             case MIGRATION_PARAMETER_COMPRESS_LEVEL:
@@ -1235,23 +1256,29 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
             case MIGRATION_PARAMETER_DECOMPRESS_THREADS:
                 has_decompress_threads = true;
                 break;
+            case MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL:
+                has_x_cpu_throttle_initial = true;
+                break;
+            case MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT:
+                has_x_cpu_throttle_increment = true;
+                break;
             }
             qmp_migrate_set_parameters(has_compress_level, value,
                                        has_compress_threads, value,
                                        has_decompress_threads, value,
+                                       has_x_cpu_throttle_initial, value,
+                                       has_x_cpu_throttle_increment, value,
                                        &err);
             break;
         }
     }
 
-    if (i == MIGRATION_PARAMETER_MAX) {
+    if (i == MIGRATION_PARAMETER__MAX) {
         error_setg(&err, QERR_INVALID_PARAMETER, param);
     }
 
     if (err) {
-        monitor_printf(mon, "migrate_set_parameter: %s\n",
-                       error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
 }
 
@@ -1272,6 +1299,13 @@ void hmp_client_migrate_info(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &err);
 }
 
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    qmp_migrate_start_postcopy(&err);
+    hmp_handle_error(mon, &err);
+}
+
 void hmp_set_password(Monitor *mon, const QDict *qdict)
 {
     const char *protocol  = qdict_get_str(qdict, "protocol");
@@ -1315,24 +1349,46 @@ void hmp_change(Monitor *mon, const QDict *qdict)
     const char *device = qdict_get_str(qdict, "device");
     const char *target = qdict_get_str(qdict, "target");
     const char *arg = qdict_get_try_str(qdict, "arg");
+    const char *read_only = qdict_get_try_str(qdict, "read-only-mode");
+    BlockdevChangeReadOnlyMode read_only_mode = 0;
     Error *err = NULL;
 
-    if (strcmp(device, "vnc") == 0 &&
-            (strcmp(target, "passwd") == 0 ||
-             strcmp(target, "password") == 0)) {
-        if (!arg) {
-            monitor_read_password(mon, hmp_change_read_arg, NULL);
+    if (strcmp(device, "vnc") == 0) {
+        if (read_only) {
+            monitor_printf(mon,
+                           "Parameter 'read-only-mode' is invalid for VNC\n");
             return;
         }
-    }
+        if (strcmp(target, "passwd") == 0 ||
+            strcmp(target, "password") == 0) {
+            if (!arg) {
+                monitor_read_password(mon, hmp_change_read_arg, NULL);
+                return;
+            }
+        }
+        qmp_change("vnc", target, !!arg, arg, &err);
+    } else {
+        if (read_only) {
+            read_only_mode =
+                qapi_enum_parse(BlockdevChangeReadOnlyMode_lookup,
+                                read_only, BLOCKDEV_CHANGE_READ_ONLY_MODE__MAX,
+                                BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN, &err);
+            if (err) {
+                hmp_handle_error(mon, &err);
+                return;
+            }
+        }
 
-    qmp_change(device, target, !!arg, arg, &err);
-    if (err &&
-        error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
-        error_free(err);
-        monitor_read_block_device_key(mon, device, NULL, NULL);
-        return;
+        qmp_blockdev_change_medium(device, target, !!arg, arg,
+                                   !!read_only, read_only_mode, &err);
+        if (err &&
+            error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
+            error_free(err);
+            monitor_read_block_device_key(mon, device, NULL, NULL);
+            return;
+        }
     }
+
     hmp_handle_error(mon, &err);
 }
 
@@ -1359,6 +1415,18 @@ void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict)
                               0,
                               false,
                               0,
+                              false, /* no burst length via HMP */
+                              0,
+                              false,
+                              0,
+                              false,
+                              0,
+                              false,
+                              0,
+                              false,
+                              0,
+                              false,
+                              0,
                               false, /* No default I/O size */
                               0,
                               false,
@@ -1483,8 +1551,7 @@ void hmp_migrate(Monitor *mon, const QDict *qdict)
 
     qmp_migrate(uri, !!blk, blk, !!inc, inc, false, false, &err);
     if (err) {
-        monitor_printf(mon, "migrate: %s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -1533,8 +1600,10 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
     const char *file = qdict_get_str(qdict, "filename");
     bool has_begin = qdict_haskey(qdict, "begin");
     bool has_length = qdict_haskey(qdict, "length");
+    bool has_detach = qdict_haskey(qdict, "detach");
     int64_t begin = 0;
     int64_t length = 0;
+    bool detach = false;
     enum DumpGuestMemoryFormat dump_format = DUMP_GUEST_MEMORY_FORMAT_ELF;
     char *prot;
 
@@ -1562,11 +1631,14 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
     if (has_length) {
         length = qdict_get_int(qdict, "length");
     }
+    if (has_detach) {
+        detach = qdict_get_bool(qdict, "detach");
+    }
 
     prot = g_strconcat("file:", file, NULL);
 
-    qmp_dump_guest_memory(paging, prot, has_begin, begin, has_length, length,
-                          true, dump_format, &err);
+    qmp_dump_guest_memory(paging, prot, true, detach, has_begin, begin,
+                          has_length, length, true, dump_format, &err);
     hmp_handle_error(mon, &err);
     g_free(prot);
 }
@@ -1602,58 +1674,27 @@ void hmp_netdev_del(Monitor *mon, const QDict *qdict)
 void hmp_object_add(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
-    Error *err_end = NULL;
     QemuOpts *opts;
-    char *type = NULL;
-    char *id = NULL;
-    void *dummy = NULL;
     OptsVisitor *ov;
-    QDict *pdict;
+    Object *obj = NULL;
 
     opts = qemu_opts_from_qdict(qemu_find_opts("object"), qdict, &err);
     if (err) {
-        goto out;
+        hmp_handle_error(mon, &err);
+        return;
     }
 
     ov = opts_visitor_new(opts);
-    pdict = qdict_clone_shallow(qdict);
-
-    visit_start_struct(opts_get_visitor(ov), &dummy, NULL, NULL, 0, &err);
-    if (err) {
-        goto out_clean;
-    }
-
-    qdict_del(pdict, "qom-type");
-    visit_type_str(opts_get_visitor(ov), &type, "qom-type", &err);
-    if (err) {
-        goto out_end;
-    }
+    obj = user_creatable_add(qdict, opts_get_visitor(ov), &err);
+    opts_visitor_cleanup(ov);
+    qemu_opts_del(opts);
 
-    qdict_del(pdict, "id");
-    visit_type_str(opts_get_visitor(ov), &id, "id", &err);
     if (err) {
-        goto out_end;
+        hmp_handle_error(mon, &err);
     }
-
-    object_add(type, id, pdict, opts_get_visitor(ov), &err);
-
-out_end:
-    visit_end_struct(opts_get_visitor(ov), &err_end);
-    if (!err && err_end) {
-        qmp_object_del(id, NULL);
+    if (obj) {
+        object_unref(obj);
     }
-    error_propagate(&err, err_end);
-out_clean:
-    opts_visitor_cleanup(ov);
-
-    QDECREF(pdict);
-    qemu_opts_del(opts);
-    g_free(id);
-    g_free(type);
-    g_free(dummy);
-
-out:
-    hmp_handle_error(mon, &err);
 }
 
 void hmp_getfd(Monitor *mon, const QDict *qdict)
@@ -1681,21 +1722,18 @@ void hmp_sendkey(Monitor *mon, const QDict *qdict)
     int has_hold_time = qdict_haskey(qdict, "hold-time");
     int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
     Error *err = NULL;
-    char keyname_buf[16];
     char *separator;
     int keyname_len;
 
     while (1) {
         separator = strchr(keys, '-');
         keyname_len = separator ? separator - keys : strlen(keys);
-        pstrcpy(keyname_buf, sizeof(keyname_buf), keys);
 
         /* Be compatible with old interface, convert user inputted "<" */
-        if (!strncmp(keyname_buf, "<", 1) && keyname_len == 1) {
-            pstrcpy(keyname_buf, sizeof(keyname_buf), "less");
+        if (keys[0] == '<' && keyname_len == 1) {
+            keys = "less";
             keyname_len = 4;
         }
-        keyname_buf[keyname_len] = 0;
 
         keylist = g_malloc0(sizeof(*keylist));
         keylist->value = g_malloc0(sizeof(*keylist->value));
@@ -1708,21 +1746,22 @@ void hmp_sendkey(Monitor *mon, const QDict *qdict)
         }
         tmp = keylist;
 
-        if (strstart(keyname_buf, "0x", NULL)) {
+        if (strstart(keys, "0x", NULL)) {
             char *endp;
-            int value = strtoul(keyname_buf, &endp, 0);
-            if (*endp != '\0') {
+            int value = strtoul(keys, &endp, 0);
+            assert(endp <= keys + keyname_len);
+            if (endp != keys + keyname_len) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_NUMBER;
-            keylist->value->number = value;
+            keylist->value->type = KEY_VALUE_KIND_NUMBER;
+            keylist->value->u.number.data = value;
         } else {
-            int idx = index_from_key(keyname_buf);
-            if (idx == Q_KEY_CODE_MAX) {
+            int idx = index_from_key(keys, keyname_len);
+            if (idx == Q_KEY_CODE__MAX) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_QCODE;
-            keylist->value->qcode = idx;
+            keylist->value->type = KEY_VALUE_KIND_QCODE;
+            keylist->value->u.qcode.data = idx;
         }
 
         if (!separator) {
@@ -1739,7 +1778,7 @@ out:
     return;
 
 err_out:
-    monitor_printf(mon, "invalid parameter: %s\n", keyname_buf);
+    monitor_printf(mon, "invalid parameter: %.*s\n", keyname_len, keys);
     goto out;
 }
 
@@ -1772,7 +1811,7 @@ void hmp_nbd_server_start(Monitor *mon, const QDict *qdict)
         goto exit;
     }
 
-    qmp_nbd_server_start(addr, &local_err);
+    qmp_nbd_server_start(addr, false, NULL, &local_err);
     qapi_free_SocketAddress(addr);
     if (local_err != NULL) {
         goto exit;
@@ -1883,7 +1922,7 @@ void hmp_object_del(Monitor *mon, const QDict *qdict)
     const char *id = qdict_get_str(qdict, "id");
     Error *err = NULL;
 
-    qmp_object_del(id, &err);
+    user_creatable_del(id, &err);
     hmp_handle_error(mon, &err);
 }
 
@@ -1899,8 +1938,8 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)
 
     while (m) {
         ov = string_output_visitor_new(false);
-        visit_type_uint16List(string_output_get_visitor(ov),
-                              &m->value->host_nodes, NULL, NULL);
+        visit_type_uint16List(string_output_get_visitor(ov), NULL,
+                              &m->value->host_nodes, NULL);
         monitor_printf(mon, "memory backend: %d\n", i);
         monitor_printf(mon, "  size:  %" PRId64 "\n", m->value->size);
         monitor_printf(mon, "  merge: %s\n",
@@ -1937,12 +1976,12 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
         value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                di = value->dimm;
+                di = value->u.dimm.data;
 
                 monitor_printf(mon, "Memory device [%s]: \"%s\"\n",
-                               MemoryDeviceInfoKind_lookup[value->kind],
+                               MemoryDeviceInfoKind_lookup[value->type],
                                di->id ? di->id : "");
                 monitor_printf(mon, "  addr: 0x%" PRIx64 "\n", di->addr);
                 monitor_printf(mon, "  slot: %" PRId64 "\n", di->slot);
@@ -1963,6 +2002,19 @@ void hmp_info_memory_devices(Monitor *mon, const QDict *qdict)
     qapi_free_MemoryDeviceInfoList(info_list);
 }
 
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict)
+{
+    IOThreadInfoList *info_list = qmp_query_iothreads(NULL);
+    IOThreadInfoList *info;
+
+    for (info = info_list; info; info = info->next) {
+        monitor_printf(mon, "%s: thread_id=%" PRId64 "\n",
+                       info->value->id, info->value->thread_id);
+    }
+
+    qapi_free_IOThreadInfoList(info_list);
+}
+
 void hmp_qom_list(Monitor *mon, const QDict *qdict)
 {
     const char *path = qdict_get_try_str(qdict, "path");
@@ -2015,11 +2067,11 @@ void hmp_rocker(Monitor *mon, const QDict *qdict)
 {
     const char *name = qdict_get_str(qdict, "name");
     RockerSwitch *rocker;
-    Error *errp = NULL;
+    Error *err = NULL;
 
-    rocker = qmp_query_rocker(name, &errp);
-    if (errp != NULL) {
-        hmp_handle_error(mon, &errp);
+    rocker = qmp_query_rocker(name, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
         return;
     }
 
@@ -2034,11 +2086,11 @@ void hmp_rocker_ports(Monitor *mon, const QDict *qdict)
 {
     RockerPortList *list, *port;
     const char *name = qdict_get_str(qdict, "name");
-    Error *errp = NULL;
+    Error *err = NULL;
 
-    list = qmp_query_rocker_ports(name, &errp);
-    if (errp != NULL) {
-        hmp_handle_error(mon, &errp);
+    list = qmp_query_rocker_ports(name, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
         return;
     }
 
@@ -2063,11 +2115,11 @@ void hmp_rocker_of_dpa_flows(Monitor *mon, const QDict *qdict)
     RockerOfDpaFlowList *list, *info;
     const char *name = qdict_get_str(qdict, "name");
     uint32_t tbl_id = qdict_get_try_int(qdict, "tbl_id", -1);
-    Error *errp = NULL;
+    Error *err = NULL;
 
-    list = qmp_query_rocker_of_dpa_flows(name, tbl_id != -1, tbl_id, &errp);
-    if (errp != NULL) {
-        hmp_handle_error(mon, &errp);
+    list = qmp_query_rocker_of_dpa_flows(name, tbl_id != -1, tbl_id, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
         return;
     }
 
@@ -2213,12 +2265,12 @@ void hmp_rocker_of_dpa_groups(Monitor *mon, const QDict *qdict)
     RockerOfDpaGroupList *list, *g;
     const char *name = qdict_get_str(qdict, "name");
     uint8_t type = qdict_get_try_int(qdict, "type", 9);
-    Error *errp = NULL;
+    Error *err = NULL;
     bool set = false;
 
-    list = qmp_query_rocker_of_dpa_groups(name, type != 9, type, &errp);
-    if (errp != NULL) {
-        hmp_handle_error(mon, &errp);
+    list = qmp_query_rocker_of_dpa_groups(name, type != 9, type, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
         return;
     }
 
@@ -2312,3 +2364,20 @@ void hmp_rocker_of_dpa_groups(Monitor *mon, const QDict *qdict)
 
     qapi_free_RockerOfDpaGroupList(list);
 }
+
+void hmp_info_dump(Monitor *mon, const QDict *qdict)
+{
+    DumpQueryResult *result = qmp_query_dump(NULL);
+
+    assert(result && result->status < DUMP_STATUS__MAX);
+    monitor_printf(mon, "Status: %s\n", DumpStatus_lookup[result->status]);
+
+    if (result->status == DUMP_STATUS_ACTIVE) {
+        float percent = 0;
+        assert(result->total != 0);
+        percent = 100.0 * result->completed / result->total;
+        monitor_printf(mon, "Finished: %.2f %%\n", percent);
+    }
+
+    qapi_free_DumpQueryResult(result);
+}
diff --git a/qemu/hmp.h b/qemu/hmp.h
index 0cf4f2a3d..093d65f5a 100644
--- a/qemu/hmp.h
+++ b/qemu/hmp.h
@@ -39,6 +39,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);
 void hmp_info_pci(Monitor *mon, const QDict *qdict);
 void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_info_tpm(Monitor *mon, const QDict *qdict);
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
@@ -68,6 +69,7 @@ void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict);
 void hmp_client_migrate_info(Monitor *mon, const QDict *qdict);
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict);
 void hmp_set_password(Monitor *mon, const QDict *qdict);
 void hmp_expire_password(Monitor *mon, const QDict *qdict);
 void hmp_eject(Monitor *mon, const QDict *qdict);
@@ -113,6 +115,7 @@ void set_link_completion(ReadLineState *rs, int nb_args, const char *str);
 void netdev_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void ringbuf_write_completion(ReadLineState *rs, int nb_args, const char *str);
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str);
 void watchdog_action_completion(ReadLineState *rs, int nb_args,
                                 const char *str);
 void migrate_set_capability_completion(ReadLineState *rs, int nb_args,
@@ -128,5 +131,6 @@ void hmp_rocker(Monitor *mon, const QDict *qdict);
 void hmp_rocker_ports(Monitor *mon, const QDict *qdict);
 void hmp_rocker_of_dpa_flows(Monitor *mon, const QDict *qdict);
 void hmp_rocker_of_dpa_groups(Monitor *mon, const QDict *qdict);
+void hmp_info_dump(Monitor *mon, const QDict *qdict);
 
 #endif
diff --git a/qemu/hw/9pfs/virtio-9p-handle.c b/qemu/hw/9pfs/9p-handle.c
index 13eabb98a..894041488 100644
--- a/qemu/hw/9pfs/virtio-9p-handle.c
+++ b/qemu/hw/9pfs/9p-handle.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p handle callback
+ * 9p handle callback
  *
  * Copyright IBM, Corp. 2011
  *
@@ -11,16 +11,17 @@
  *
  */
 
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "9p-xattr.h"
 #include <arpa/inet.h>
 #include <pwd.h>
 #include <grp.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include "qemu/xattr.h"
-#include <unistd.h>
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
 #include <linux/magic.h>
@@ -656,12 +657,12 @@ static int handle_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
     const char *path = qemu_opt_get(opts, "path");
 
     if (sec_model) {
-        fprintf(stderr, "Invalid argument security_model specified with handle fsdriver\n");
+        error_report("Invalid argument security_model specified with handle fsdriver");
         return -1;
     }
 
     if (!path) {
-        fprintf(stderr, "fsdev: No path specified.\n");
+        error_report("fsdev: No path specified");
         return -1;
     }
     fse->path = g_strdup(path);
diff --git a/qemu/hw/9pfs/virtio-9p-local.c b/qemu/hw/9pfs/9p-local.c
index f1f2e2573..16f45f485 100644
--- a/qemu/hw/9pfs/virtio-9p-local.c
+++ b/qemu/hw/9pfs/9p-local.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p Posix callback
+ * 9p Posix callback
  *
  * Copyright IBM, Corp. 2010
  *
@@ -11,9 +11,9 @@
  *
  */
 
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "9p-xattr.h"
 #include "fsdev/qemu-fsdev.h"   /* local_ops */
 #include <arpa/inet.h>
 #include <pwd.h>
@@ -21,6 +21,8 @@
 #include <sys/socket.h>
 #include <sys/un.h>
 #include "qemu/xattr.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
 #include <libgen.h>
 #include <linux/fs.h>
 #ifdef CONFIG_LINUX_MAGIC_H
@@ -1210,9 +1212,9 @@ static int local_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
     const char *path = qemu_opt_get(opts, "path");
 
     if (!sec_model) {
-        fprintf(stderr, "security model not specified, "
-                "local fs needs security model\nvalid options are:"
-                "\tsecurity_model=[passthrough|mapped|none]\n");
+        error_report("Security model not specified, local fs needs security model");
+        error_printf("valid options are:"
+                     "\tsecurity_model=[passthrough|mapped-xattr|mapped-file|none]\n");
         return -1;
     }
 
@@ -1226,14 +1228,14 @@ static int local_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
     } else if (!strcmp(sec_model, "mapped-file")) {
         fse->export_flags |= V9FS_SM_MAPPED_FILE;
     } else {
-        fprintf(stderr, "Invalid security model %s specified, valid options are"
-                "\n\t [passthrough|mapped-xattr|mapped-file|none]\n",
-                sec_model);
+        error_report("Invalid security model %s specified", sec_model);
+        error_printf("valid options are:"
+                     "\t[passthrough|mapped-xattr|mapped-file|none]\n");
         return -1;
     }
 
     if (!path) {
-        fprintf(stderr, "fsdev: No path specified.\n");
+        error_report("fsdev: No path specified");
         return -1;
     }
     fse->path = g_strdup(path);
diff --git a/qemu/hw/9pfs/virtio-9p-posix-acl.c b/qemu/hw/9pfs/9p-posix-acl.c
index 09dad071e..ec003181c 100644
--- a/qemu/hw/9pfs/virtio-9p-posix-acl.c
+++ b/qemu/hw/9pfs/9p-posix-acl.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p system.posix* xattr callback
+ * 9p system.posix* xattr callback
  *
  * Copyright IBM, Corp. 2010
  *
@@ -11,12 +11,11 @@
  *
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include "qemu/xattr.h"
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
+#include "9p.h"
 #include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
+#include "9p-xattr.h"
 
 #define MAP_ACL_ACCESS "user.virtfs.system.posix_acl_access"
 #define MAP_ACL_DEFAULT "user.virtfs.system.posix_acl_default"
diff --git a/qemu/hw/9pfs/virtio-9p-proxy.c b/qemu/hw/9pfs/9p-proxy.c
index 1bc7881f0..00a4eb2a7 100644
--- a/qemu/hw/9pfs/virtio-9p-proxy.c
+++ b/qemu/hw/9pfs/9p-proxy.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p Proxy callback
+ * 9p Proxy callback
  *
  * Copyright IBM, Corp. 2011
  *
@@ -9,13 +9,14 @@
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <sys/socket.h>
 #include <sys/un.h>
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
+#include "9p.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 #include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-proxy.h"
+#include "9p-proxy.h"
 
 typedef struct V9fsProxy {
     int sockfd;
@@ -1101,19 +1102,19 @@ static int connect_namedsocket(const char *path)
     struct sockaddr_un helper;
 
     if (strlen(path) >= sizeof(helper.sun_path)) {
-        fprintf(stderr, "Socket name too large\n");
+        error_report("Socket name too long");
         return -1;
     }
     sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
     if (sockfd < 0) {
-        fprintf(stderr, "failed to create socket: %s\n", strerror(errno));
+        error_report("Failed to create socket: %s", strerror(errno));
         return -1;
     }
     strcpy(helper.sun_path, path);
     helper.sun_family = AF_UNIX;
     size = strlen(helper.sun_path) + sizeof(helper.sun_family);
     if (connect(sockfd, (struct sockaddr *)&helper, size) < 0) {
-        fprintf(stderr, "failed to connect to %s: %s\n", path, strerror(errno));
+        error_report("Failed to connect to %s: %s", path, strerror(errno));
         close(sockfd);
         return -1;
     }
@@ -1129,11 +1130,11 @@ static int proxy_parse_opts(QemuOpts *opts, struct FsDriverEntry *fs)
     const char *sock_fd = qemu_opt_get(opts, "sock_fd");
 
     if (!socket && !sock_fd) {
-        fprintf(stderr, "socket and sock_fd none of the option specified\n");
+        error_report("Must specify either socket or sock_fd");
         return -1;
     }
     if (socket && sock_fd) {
-        fprintf(stderr, "Both socket and sock_fd options specified\n");
+        error_report("Both socket and sock_fd options specified");
         return -1;
     }
     if (socket) {
@@ -1156,7 +1157,7 @@ static int proxy_init(FsContext *ctx)
     } else {
         sock_id = atoi(ctx->fs_root);
         if (sock_id < 0) {
-            fprintf(stderr, "socket descriptor not initialized\n");
+            error_report("Socket descriptor not initialized");
         }
     }
     if (sock_id < 0) {
diff --git a/qemu/hw/9pfs/virtio-9p-proxy.h b/qemu/hw/9pfs/9p-proxy.h
index 005c1ad75..ba9ca203d 100644
--- a/qemu/hw/9pfs/virtio-9p-proxy.h
+++ b/qemu/hw/9pfs/9p-proxy.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p Proxy callback
+ * 9p Proxy callback
  *
  * Copyright IBM, Corp. 2011
  *
@@ -9,8 +9,8 @@
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  */
-#ifndef _QEMU_VIRTIO_9P_PROXY_H
-#define _QEMU_VIRTIO_9P_PROXY_H
+#ifndef _QEMU_9P_PROXY_H
+#define _QEMU_9P_PROXY_H
 
 #define PROXY_MAX_IO_SZ (64 * 1024)
 #define V9FS_FD_VALID INT_MAX
@@ -20,9 +20,9 @@
  * marsha/unmarshal doesn't do little endian conversion.
  */
 #define proxy_unmarshal(in_sg, offset, fmt, args...) \
-    v9fs_unmarshal(in_sg, 1, offset, 0, fmt, ##args)
+    v9fs_iov_unmarshal(in_sg, 1, offset, 0, fmt, ##args)
 #define proxy_marshal(out_sg, offset, fmt, args...) \
-    v9fs_marshal(out_sg, 1, offset, 0, fmt, ##args)
+    v9fs_iov_marshal(out_sg, 1, offset, 0, fmt, ##args)
 
 union MsgControl {
     struct cmsghdr cmsg;
diff --git a/qemu/hw/9pfs/virtio-9p-synth.c b/qemu/hw/9pfs/9p-synth.c
index a0ab9a86a..f1475dfd6 100644
--- a/qemu/hw/9pfs/virtio-9p-synth.c
+++ b/qemu/hw/9pfs/9p-synth.c
@@ -12,14 +12,15 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
+#include "9p.h"
+#include "9p-xattr.h"
 #include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-synth.h"
+#include "9p-synth.h"
 #include "qemu/rcu.h"
 #include "qemu/rcu_queue.h"
-#include <sys/stat.h>
+#include "qemu/cutils.h"
 
 /* Root node for synth file system */
 static V9fsSynthNode v9fs_synth_root = {
diff --git a/qemu/hw/9pfs/virtio-9p-synth.h b/qemu/hw/9pfs/9p-synth.h
index ab05a8e78..82962512a 100644
--- a/qemu/hw/9pfs/virtio-9p-synth.h
+++ b/qemu/hw/9pfs/9p-synth.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2011
  *
@@ -10,12 +10,9 @@
  * the COPYING file in the top-level directory.
  *
  */
-#ifndef HW_9PFS_VIRTIO9P_SYNTH_H
-#define HW_9PFS_VIRTIO9P_SYNTH_H 1
+#ifndef HW_9PFS_SYNTH_H
+#define HW_9PFS_SYNTH_H 1
 
-#include <unistd.h>
-#include <sys/types.h>
-#include <limits.h>
 
 typedef struct V9fsSynthNode V9fsSynthNode;
 typedef ssize_t (*v9fs_synth_read)(void *buf, int len, off_t offset,
diff --git a/qemu/hw/9pfs/virtio-9p-xattr-user.c b/qemu/hw/9pfs/9p-xattr-user.c
index 46133e06d..f87530c8b 100644
--- a/qemu/hw/9pfs/virtio-9p-xattr-user.c
+++ b/qemu/hw/9pfs/9p-xattr-user.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p user. xattr callback
+ * 9p user. xattr callback
  *
  * Copyright IBM, Corp. 2010
  *
@@ -11,11 +11,10 @@
  *
  */
 
-#include <sys/types.h>
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
+#include "qemu/osdep.h"
+#include "9p.h"
 #include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
+#include "9p-xattr.h"
 
 
 static ssize_t mp_user_getxattr(FsContext *ctx, const char *path,
diff --git a/qemu/hw/9pfs/virtio-9p-xattr.c b/qemu/hw/9pfs/9p-xattr.c
index 07183887c..5d8595ed9 100644
--- a/qemu/hw/9pfs/virtio-9p-xattr.c
+++ b/qemu/hw/9pfs/9p-xattr.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p  xattr callback
+ * 9p  xattr callback
  *
  * Copyright IBM, Corp. 2010
  *
@@ -11,10 +11,10 @@
  *
  */
 
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
+#include "qemu/osdep.h"
+#include "9p.h"
 #include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
+#include "9p-xattr.h"
 
 
 static XattrOperations *get_xattr_operations(XattrOperations **h,
diff --git a/qemu/hw/9pfs/virtio-9p-xattr.h b/qemu/hw/9pfs/9p-xattr.h
index 327b32b5a..4d39a2026 100644
--- a/qemu/hw/9pfs/virtio-9p-xattr.h
+++ b/qemu/hw/9pfs/9p-xattr.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2010
  *
@@ -10,8 +10,8 @@
  * the COPYING file in the top-level directory.
  *
  */
-#ifndef _QEMU_VIRTIO_9P_XATTR_H
-#define _QEMU_VIRTIO_9P_XATTR_H
+#ifndef _QEMU_9P_XATTR_H
+#define _QEMU_9P_XATTR_H
 
 #include "qemu/xattr.h"
 
diff --git a/qemu/hw/9pfs/virtio-9p.c b/qemu/hw/9pfs/9p.c
index f972731f5..f5e30125f 100644
--- a/qemu/hw/9pfs/virtio-9p.c
+++ b/qemu/hw/9pfs/9p.c
@@ -11,15 +11,17 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio.h"
 #include "hw/i386/pc.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-xattr.h"
-#include "virtio-9p-coth.h"
+#include "9p-xattr.h"
+#include "coth.h"
 #include "trace.h"
 #include "migration/migration.h"
 
@@ -39,6 +41,35 @@ enum {
     Oappend = 0x80,
 };
 
+ssize_t pdu_marshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = virtio_pdu_vmarshal(pdu, offset, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+ssize_t pdu_unmarshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = virtio_pdu_vunmarshal(pdu, offset, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+static void pdu_push_and_notify(V9fsPDU *pdu)
+{
+    virtio_9p_push_and_notify(pdu);
+}
+
 static int omode_to_uflags(int8_t mode)
 {
     int ret = 0;
@@ -563,7 +594,7 @@ static int fid_to_qid(V9fsPDU *pdu, V9fsFidState *fidp, V9fsQID *qidp)
     return 0;
 }
 
-static V9fsPDU *alloc_pdu(V9fsState *s)
+V9fsPDU *pdu_alloc(V9fsState *s)
 {
     V9fsPDU *pdu = NULL;
 
@@ -575,9 +606,10 @@ static V9fsPDU *alloc_pdu(V9fsState *s)
     return pdu;
 }
 
-static void free_pdu(V9fsState *s, V9fsPDU *pdu)
+void pdu_free(V9fsPDU *pdu)
 {
     if (pdu) {
+        V9fsState *s = pdu->s;
         /*
          * Cancelled pdu are added back to the freelist
          * by flush request .
@@ -594,9 +626,10 @@ static void free_pdu(V9fsState *s, V9fsPDU *pdu)
  * because we always expect to have enough space to encode
  * error details
  */
-static void complete_pdu(V9fsState *s, V9fsPDU *pdu, ssize_t len)
+static void pdu_complete(V9fsPDU *pdu, ssize_t len)
 {
     int8_t id = pdu->id + 1; /* Response */
+    V9fsState *s = pdu->s;
 
     if (len < 0) {
         int err = -len;
@@ -627,16 +660,12 @@ static void complete_pdu(V9fsState *s, V9fsPDU *pdu, ssize_t len)
     pdu->size = len;
     pdu->id = id;
 
-    /* push onto queue and notify */
-    virtqueue_push(s->vq, &pdu->elem, len);
-
-    /* FIXME: we should batch these completions */
-    virtio_notify(VIRTIO_DEVICE(s), s->vq);
+    pdu_push_and_notify(pdu);
 
     /* Now wakeup anybody waiting in flush for this request */
     qemu_co_queue_next(&pdu->complete);
 
-    free_pdu(s, pdu);
+    pdu_free(pdu);
 }
 
 static mode_t v9mode_to_mode(uint32_t mode, V9fsString *extension)
@@ -931,7 +960,7 @@ static void v9fs_version(void *opaque)
     offset += err;
     trace_v9fs_version_return(pdu->tag, pdu->id, s->msize, version.data);
 out:
-    complete_pdu(s, pdu, offset);
+    pdu_complete(pdu, offset);
     v9fs_string_free(&version);
 }
 
@@ -995,7 +1024,7 @@ static void v9fs_attach(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&uname);
     v9fs_string_free(&aname);
 }
@@ -1009,7 +1038,6 @@ static void v9fs_stat(void *opaque)
     struct stat stbuf;
     V9fsFidState *fidp;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     err = pdu_unmarshal(pdu, offset, "d", &fid);
     if (err < 0) {
@@ -1042,7 +1070,7 @@ static void v9fs_stat(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static void v9fs_getattr(void *opaque)
@@ -1105,7 +1133,7 @@ static void v9fs_getattr(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, retval);
+    pdu_complete(pdu, retval);
 }
 
 /* Attribute flags */
@@ -1129,7 +1157,6 @@ static void v9fs_setattr(void *opaque)
     size_t offset = 7;
     V9fsIattr v9iattr;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     err = pdu_unmarshal(pdu, offset, "dI", &fid, &v9iattr);
     if (err < 0) {
@@ -1203,7 +1230,7 @@ static void v9fs_setattr(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static int v9fs_walk_marshal(V9fsPDU *pdu, uint16_t nwnames, V9fsQID *qids)
@@ -1245,7 +1272,7 @@ static void v9fs_walk(void *opaque)
 
     err = pdu_unmarshal(pdu, offset, "ddw", &fid, &newfid, &nwnames);
     if (err < 0) {
-        complete_pdu(s, pdu, err);
+        pdu_complete(pdu, err);
         return ;
     }
     offset += err;
@@ -1313,7 +1340,7 @@ out:
     v9fs_path_free(&dpath);
     v9fs_path_free(&path);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     if (nwnames && nwnames <= P9_MAXWELEM) {
         for (name_idx = 0; name_idx < nwnames; name_idx++) {
             v9fs_string_free(&wnames[name_idx]);
@@ -1430,7 +1457,7 @@ static void v9fs_open(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static void v9fs_lcreate(void *opaque)
@@ -1487,7 +1514,7 @@ static void v9fs_lcreate(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -1499,7 +1526,6 @@ static void v9fs_fsync(void *opaque)
     size_t offset = 7;
     V9fsFidState *fidp;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     err = pdu_unmarshal(pdu, offset, "dd", &fid, &datasync);
     if (err < 0) {
@@ -1518,7 +1544,7 @@ static void v9fs_fsync(void *opaque)
     }
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static void v9fs_clunk(void *opaque)
@@ -1551,7 +1577,7 @@ static void v9fs_clunk(void *opaque)
         err = offset;
     }
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static int v9fs_xattr_read(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
@@ -1561,6 +1587,8 @@ static int v9fs_xattr_read(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
     size_t offset = 7;
     int read_count;
     int64_t xattr_len;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
 
     xattr_len = fidp->fs.xattr.len;
     read_count = xattr_len - off;
@@ -1577,7 +1605,8 @@ static int v9fs_xattr_read(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
         return err;
     }
     offset += err;
-    err = v9fs_pack(pdu->elem.in_sg, pdu->elem.in_num, offset,
+
+    err = v9fs_pack(elem->in_sg, elem->in_num, offset,
                     ((char *)fidp->fs.xattr.value) + off,
                     read_count);
     if (err < 0) {
@@ -1667,13 +1696,7 @@ static void v9fs_init_qiov_from_pdu(QEMUIOVector *qiov, V9fsPDU *pdu,
     struct iovec *iov;
     unsigned int niov;
 
-    if (is_write) {
-        iov = pdu->elem.out_sg;
-        niov = pdu->elem.out_num;
-    } else {
-        iov = pdu->elem.in_sg;
-        niov = pdu->elem.in_num;
-    }
+    virtio_init_iov_from_pdu(pdu, &iov, &niov, is_write);
 
     qemu_iovec_init_external(&elem, iov, niov);
     qemu_iovec_init(qiov, niov);
@@ -1761,7 +1784,7 @@ static void v9fs_read(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static size_t v9fs_readdir_data_size(V9fsString *name)
@@ -1848,7 +1871,6 @@ static void v9fs_readdir(void *opaque)
     int32_t count;
     uint32_t max_count;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     retval = pdu_unmarshal(pdu, offset, "dqd", &fid,
                            &initial_offset, &max_count);
@@ -1885,7 +1907,7 @@ static void v9fs_readdir(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, retval);
+    pdu_complete(pdu, retval);
 }
 
 static int v9fs_xattr_write(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
@@ -1952,7 +1974,7 @@ static void v9fs_write(void *opaque)
 
     err = pdu_unmarshal(pdu, offset, "dqd", &fid, &off, &count);
     if (err < 0) {
-        complete_pdu(s, pdu, err);
+        pdu_complete(pdu, err);
         return;
     }
     offset += err;
@@ -2015,7 +2037,7 @@ out:
     put_fid(pdu, fidp);
 out_nofid:
     qemu_iovec_destroy(&qiov_full);
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static void v9fs_create(void *opaque)
@@ -2182,7 +2204,7 @@ static void v9fs_create(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-   complete_pdu(pdu->s, pdu, err);
+   pdu_complete(pdu, err);
    v9fs_string_free(&name);
    v9fs_string_free(&extension);
    v9fs_path_free(&path);
@@ -2229,7 +2251,7 @@ static void v9fs_symlink(void *opaque)
 out:
     put_fid(pdu, dfidp);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
     v9fs_string_free(&symname);
 }
@@ -2245,7 +2267,7 @@ static void v9fs_flush(void *opaque)
 
     err = pdu_unmarshal(pdu, offset, "w", &tag);
     if (err < 0) {
-        complete_pdu(s, pdu, err);
+        pdu_complete(pdu, err);
         return;
     }
     trace_v9fs_flush(pdu->tag, pdu->id, tag);
@@ -2262,15 +2284,14 @@ static void v9fs_flush(void *opaque)
          */
         qemu_co_queue_wait(&cancel_pdu->complete);
         cancel_pdu->cancelled = 0;
-        free_pdu(pdu->s, cancel_pdu);
+        pdu_free(cancel_pdu);
     }
-    complete_pdu(s, pdu, 7);
+    pdu_complete(pdu, 7);
 }
 
 static void v9fs_link(void *opaque)
 {
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
     int32_t dfid, oldfid;
     V9fsFidState *dfidp, *oldfidp;
     V9fsString name;
@@ -2303,7 +2324,7 @@ out:
     put_fid(pdu, dfidp);
 out_nofid:
     v9fs_string_free(&name);
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 /* Only works with path name based fid */
@@ -2348,7 +2369,7 @@ out_err:
     clunk_fid(pdu->s, fidp->fid);
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static void v9fs_unlinkat(void *opaque)
@@ -2392,7 +2413,7 @@ out_err:
     put_fid(pdu, dfidp);
     v9fs_path_free(&path);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -2492,7 +2513,7 @@ static void v9fs_rename(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -2593,7 +2614,7 @@ static void v9fs_renameat(void *opaque)
     }
 
 out_err:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&old_name);
     v9fs_string_free(&new_name);
 }
@@ -2608,7 +2629,6 @@ static void v9fs_wstat(void *opaque)
     struct stat stbuf;
     V9fsFidState *fidp;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     v9fs_stat_init(&v9stat);
     err = pdu_unmarshal(pdu, offset, "dwS", &fid, &unused, &v9stat);
@@ -2690,7 +2710,7 @@ out:
     put_fid(pdu, fidp);
 out_nofid:
     v9fs_stat_free(&v9stat);
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static int v9fs_fill_statfs(V9fsState *s, V9fsPDU *pdu, struct statfs *stbuf)
@@ -2769,7 +2789,7 @@ static void v9fs_statfs(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, retval);
+    pdu_complete(pdu, retval);
 }
 
 static void v9fs_mknod(void *opaque)
@@ -2786,7 +2806,6 @@ static void v9fs_mknod(void *opaque)
     struct stat stbuf;
     V9fsFidState *fidp;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     v9fs_string_init(&name);
     err = pdu_unmarshal(pdu, offset, "dsdddd", &fid, &name, &mode,
@@ -2817,7 +2836,7 @@ static void v9fs_mknod(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -2838,7 +2857,6 @@ static void v9fs_lock(void *opaque)
     V9fsFidState *fidp;
     int32_t fid, err = 0;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     status = P9_LOCK_ERROR;
     v9fs_string_init(&flock.client_id);
@@ -2875,7 +2893,7 @@ out_nofid:
         err += offset;
     }
     trace_v9fs_lock_return(pdu->tag, pdu->id, status);
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&flock.client_id);
 }
 
@@ -2891,7 +2909,6 @@ static void v9fs_getlock(void *opaque)
     V9fsGetlock glock;
     int32_t fid, err = 0;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     v9fs_string_init(&glock.client_id);
     err = pdu_unmarshal(pdu, offset, "dbqqds", &fid, &glock.type,
@@ -2925,7 +2942,7 @@ static void v9fs_getlock(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&glock.client_id);
 }
 
@@ -2969,7 +2986,7 @@ static void v9fs_mkdir(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -3075,7 +3092,7 @@ out:
         put_fid(pdu, xattr_fidp);
     }
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -3090,7 +3107,6 @@ static void v9fs_xattrcreate(void *opaque)
     V9fsFidState *file_fidp;
     V9fsFidState *xattr_fidp;
     V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
 
     v9fs_string_init(&name);
     err = pdu_unmarshal(pdu, offset, "dsqd", &fid, &name, &size, &flags);
@@ -3116,7 +3132,7 @@ static void v9fs_xattrcreate(void *opaque)
     err = offset;
     put_fid(pdu, file_fidp);
 out_nofid:
-    complete_pdu(s, pdu, err);
+    pdu_complete(pdu, err);
     v9fs_string_free(&name);
 }
 
@@ -3156,7 +3172,7 @@ static void v9fs_readlink(void *opaque)
 out:
     put_fid(pdu, fidp);
 out_nofid:
-    complete_pdu(pdu->s, pdu, err);
+    pdu_complete(pdu, err);
 }
 
 static CoroutineEntry *pdu_co_handlers[] = {
@@ -3199,13 +3215,13 @@ static CoroutineEntry *pdu_co_handlers[] = {
 static void v9fs_op_not_supp(void *opaque)
 {
     V9fsPDU *pdu = opaque;
-    complete_pdu(pdu->s, pdu, -EOPNOTSUPP);
+    pdu_complete(pdu, -EOPNOTSUPP);
 }
 
 static void v9fs_fs_ro(void *opaque)
 {
     V9fsPDU *pdu = opaque;
-    complete_pdu(pdu->s, pdu, -EROFS);
+    pdu_complete(pdu, -EROFS);
 }
 
 static inline bool is_read_only_op(V9fsPDU *pdu)
@@ -3235,10 +3251,11 @@ static inline bool is_read_only_op(V9fsPDU *pdu)
     }
 }
 
-static void submit_pdu(V9fsState *s, V9fsPDU *pdu)
+void pdu_submit(V9fsPDU *pdu)
 {
     Coroutine *co;
     CoroutineEntry *handler;
+    V9fsState *s = pdu->s;
 
     if (pdu->id >= ARRAY_SIZE(pdu_co_handlers) ||
         (pdu_co_handlers[pdu->id] == NULL)) {
@@ -3254,45 +3271,108 @@ static void submit_pdu(V9fsState *s, V9fsPDU *pdu)
     qemu_coroutine_enter(co, pdu);
 }
 
-void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq)
+/* Returns 0 on success, 1 on failure. */
+int v9fs_device_realize_common(V9fsState *s, Error **errp)
 {
-    V9fsState *s = (V9fsState *)vdev;
-    V9fsPDU *pdu;
-    ssize_t len;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    int i, len;
+    struct stat stat;
+    FsDriverEntry *fse;
+    V9fsPath path;
+    int rc = 1;
+
+    /* initialize pdu allocator */
+    QLIST_INIT(&s->free_list);
+    QLIST_INIT(&s->active_list);
+    for (i = 0; i < (MAX_REQ - 1); i++) {
+        QLIST_INSERT_HEAD(&s->free_list, &v->pdus[i], next);
+        v->pdus[i].s = s;
+        v->pdus[i].idx = i;
+    }
+
+    v9fs_path_init(&path);
+
+    fse = get_fsdev_fsentry(s->fsconf.fsdev_id);
+
+    if (!fse) {
+        /* We don't have a fsdev identified by fsdev_id */
+        error_setg(errp, "9pfs device couldn't find fsdev with the "
+                   "id = %s",
+                   s->fsconf.fsdev_id ? s->fsconf.fsdev_id : "NULL");
+        goto out;
+    }
+
+    if (!s->fsconf.tag) {
+        /* we haven't specified a mount_tag */
+        error_setg(errp, "fsdev with id %s needs mount_tag arguments",
+                   s->fsconf.fsdev_id);
+        goto out;
+    }
+
+    s->ctx.export_flags = fse->export_flags;
+    s->ctx.fs_root = g_strdup(fse->path);
+    s->ctx.exops.get_st_gen = NULL;
+    len = strlen(s->fsconf.tag);
+    if (len > MAX_TAG_LEN - 1) {
+        error_setg(errp, "mount tag '%s' (%d bytes) is longer than "
+                   "maximum (%d bytes)", s->fsconf.tag, len, MAX_TAG_LEN - 1);
+        goto out;
+    }
 
-    while ((pdu = alloc_pdu(s)) &&
-            (len = virtqueue_pop(vq, &pdu->elem)) != 0) {
-        struct {
-            uint32_t size_le;
-            uint8_t id;
-            uint16_t tag_le;
-        } QEMU_PACKED out;
-        int len;
+    s->tag = g_strdup(s->fsconf.tag);
+    s->ctx.uid = -1;
 
-        pdu->s = s;
-        BUG_ON(pdu->elem.out_num == 0 || pdu->elem.in_num == 0);
-        QEMU_BUILD_BUG_ON(sizeof out != 7);
+    s->ops = fse->ops;
 
-        len = iov_to_buf(pdu->elem.out_sg, pdu->elem.out_num, 0,
-                         &out, sizeof out);
-        BUG_ON(len != sizeof out);
+    s->fid_list = NULL;
+    qemu_co_rwlock_init(&s->rename_lock);
 
-        pdu->size = le32_to_cpu(out.size_le);
+    if (s->ops->init(&s->ctx) < 0) {
+        error_setg(errp, "9pfs Failed to initialize fs-driver with id:%s"
+                   " and export path:%s", s->fsconf.fsdev_id, s->ctx.fs_root);
+        goto out;
+    }
 
-        pdu->id = out.id;
-        pdu->tag = le16_to_cpu(out.tag_le);
+    /*
+     * Check details of export path, We need to use fs driver
+     * call back to do that. Since we are in the init path, we don't
+     * use co-routines here.
+     */
+    if (s->ops->name_to_path(&s->ctx, NULL, "/", &path) < 0) {
+        error_setg(errp,
+                   "error in converting name to path %s", strerror(errno));
+        goto out;
+    }
+    if (s->ops->lstat(&s->ctx, &path, &stat)) {
+        error_setg(errp, "share path %s does not exist", fse->path);
+        goto out;
+    } else if (!S_ISDIR(stat.st_mode)) {
+        error_setg(errp, "share path %s is not a directory", fse->path);
+        goto out;
+    }
+    v9fs_path_free(&path);
 
-        qemu_co_queue_init(&pdu->complete);
-        submit_pdu(s, pdu);
+    rc = 0;
+out:
+    if (rc) {
+        g_free(s->ctx.fs_root);
+        g_free(s->tag);
+        v9fs_path_free(&path);
     }
-    free_pdu(s, pdu);
+    return rc;
+}
+
+void v9fs_device_unrealize_common(V9fsState *s, Error **errp)
+{
+    g_free(s->ctx.fs_root);
+    g_free(s->tag);
 }
 
-static void __attribute__((__constructor__)) virtio_9p_set_fd_limit(void)
+static void __attribute__((__constructor__)) v9fs_set_fd_limit(void)
 {
     struct rlimit rlim;
     if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
-        fprintf(stderr, "Failed to get the resource limit\n");
+        error_report("Failed to get the resource limit");
         exit(1);
     }
     open_fd_hw = rlim.rlim_cur - MIN(400, rlim.rlim_cur/3);
diff --git a/qemu/hw/9pfs/9p.h b/qemu/hw/9pfs/9p.h
new file mode 100644
index 000000000..1a19418a8
--- /dev/null
+++ b/qemu/hw/9pfs/9p.h
@@ -0,0 +1,324 @@
+#ifndef _QEMU_9P_H
+#define _QEMU_9P_H
+
+#include <dirent.h>
+#include <utime.h>
+#include <sys/resource.h>
+#include <glib.h>
+#include "standard-headers/linux/virtio_9p.h"
+#include "hw/virtio/virtio.h"
+#include "fsdev/file-op-9p.h"
+#include "fsdev/9p-iov-marshal.h"
+#include "qemu/thread.h"
+#include "qemu/coroutine.h"
+
+enum {
+    P9_TLERROR = 6,
+    P9_RLERROR,
+    P9_TSTATFS = 8,
+    P9_RSTATFS,
+    P9_TLOPEN = 12,
+    P9_RLOPEN,
+    P9_TLCREATE = 14,
+    P9_RLCREATE,
+    P9_TSYMLINK = 16,
+    P9_RSYMLINK,
+    P9_TMKNOD = 18,
+    P9_RMKNOD,
+    P9_TRENAME = 20,
+    P9_RRENAME,
+    P9_TREADLINK = 22,
+    P9_RREADLINK,
+    P9_TGETATTR = 24,
+    P9_RGETATTR,
+    P9_TSETATTR = 26,
+    P9_RSETATTR,
+    P9_TXATTRWALK = 30,
+    P9_RXATTRWALK,
+    P9_TXATTRCREATE = 32,
+    P9_RXATTRCREATE,
+    P9_TREADDIR = 40,
+    P9_RREADDIR,
+    P9_TFSYNC = 50,
+    P9_RFSYNC,
+    P9_TLOCK = 52,
+    P9_RLOCK,
+    P9_TGETLOCK = 54,
+    P9_RGETLOCK,
+    P9_TLINK = 70,
+    P9_RLINK,
+    P9_TMKDIR = 72,
+    P9_RMKDIR,
+    P9_TRENAMEAT = 74,
+    P9_RRENAMEAT,
+    P9_TUNLINKAT = 76,
+    P9_RUNLINKAT,
+    P9_TVERSION = 100,
+    P9_RVERSION,
+    P9_TAUTH = 102,
+    P9_RAUTH,
+    P9_TATTACH = 104,
+    P9_RATTACH,
+    P9_TERROR = 106,
+    P9_RERROR,
+    P9_TFLUSH = 108,
+    P9_RFLUSH,
+    P9_TWALK = 110,
+    P9_RWALK,
+    P9_TOPEN = 112,
+    P9_ROPEN,
+    P9_TCREATE = 114,
+    P9_RCREATE,
+    P9_TREAD = 116,
+    P9_RREAD,
+    P9_TWRITE = 118,
+    P9_RWRITE,
+    P9_TCLUNK = 120,
+    P9_RCLUNK,
+    P9_TREMOVE = 122,
+    P9_RREMOVE,
+    P9_TSTAT = 124,
+    P9_RSTAT,
+    P9_TWSTAT = 126,
+    P9_RWSTAT,
+};
+
+
+/* qid.types */
+enum {
+    P9_QTDIR = 0x80,
+    P9_QTAPPEND = 0x40,
+    P9_QTEXCL = 0x20,
+    P9_QTMOUNT = 0x10,
+    P9_QTAUTH = 0x08,
+    P9_QTTMP = 0x04,
+    P9_QTSYMLINK = 0x02,
+    P9_QTLINK = 0x01,
+    P9_QTFILE = 0x00,
+};
+
+enum p9_proto_version {
+    V9FS_PROTO_2000U = 0x01,
+    V9FS_PROTO_2000L = 0x02,
+};
+
+#define P9_NOTAG    (u16)(~0)
+#define P9_NOFID    (u32)(~0)
+#define P9_MAXWELEM 16
+
+#define FID_REFERENCED          0x1
+#define FID_NON_RECLAIMABLE     0x2
+static inline char *rpath(FsContext *ctx, const char *path)
+{
+    return g_strdup_printf("%s/%s", ctx->fs_root, path);
+}
+
+/*
+ * ample room for Twrite/Rread header
+ * size[4] Tread/Twrite tag[2] fid[4] offset[8] count[4]
+ */
+#define P9_IOHDRSZ 24
+
+typedef struct V9fsPDU V9fsPDU;
+struct V9fsState;
+
+struct V9fsPDU
+{
+    uint32_t size;
+    uint16_t tag;
+    uint8_t id;
+    uint8_t cancelled;
+    CoQueue complete;
+    struct V9fsState *s;
+    QLIST_ENTRY(V9fsPDU) next;
+    uint32_t idx;
+};
+
+
+/* FIXME
+ * 1) change user needs to set groups and stuff
+ */
+
+#define MAX_REQ         128
+#define MAX_TAG_LEN     32
+
+#define BUG_ON(cond) assert(!(cond))
+
+typedef struct V9fsFidState V9fsFidState;
+
+enum {
+    P9_FID_NONE = 0,
+    P9_FID_FILE,
+    P9_FID_DIR,
+    P9_FID_XATTR,
+};
+
+typedef struct V9fsConf
+{
+    /* tag name for the device */
+    char *tag;
+    char *fsdev_id;
+} V9fsConf;
+
+typedef struct V9fsXattr
+{
+    int64_t copied_len;
+    int64_t len;
+    void *value;
+    V9fsString name;
+    int flags;
+} V9fsXattr;
+
+/*
+ * Filled by fs driver on open and other
+ * calls.
+ */
+union V9fsFidOpenState {
+    int fd;
+    DIR *dir;
+    V9fsXattr xattr;
+    /*
+     * private pointer for fs drivers, that
+     * have its own internal representation of
+     * open files.
+     */
+    void *private;
+};
+
+struct V9fsFidState
+{
+    int fid_type;
+    int32_t fid;
+    V9fsPath path;
+    V9fsFidOpenState fs;
+    V9fsFidOpenState fs_reclaim;
+    int flags;
+    int open_flags;
+    uid_t uid;
+    int ref;
+    int clunked;
+    V9fsFidState *next;
+    V9fsFidState *rclm_lst;
+};
+
+typedef struct V9fsState
+{
+    QLIST_HEAD(, V9fsPDU) free_list;
+    QLIST_HEAD(, V9fsPDU) active_list;
+    V9fsFidState *fid_list;
+    FileOperations *ops;
+    FsContext ctx;
+    char *tag;
+    enum p9_proto_version proto_version;
+    int32_t msize;
+    /*
+     * lock ensuring atomic path update
+     * on rename.
+     */
+    CoRwlock rename_lock;
+    int32_t root_fid;
+    Error *migration_blocker;
+    V9fsConf fsconf;
+} V9fsState;
+
+/* 9p2000.L open flags */
+#define P9_DOTL_RDONLY        00000000
+#define P9_DOTL_WRONLY        00000001
+#define P9_DOTL_RDWR          00000002
+#define P9_DOTL_NOACCESS      00000003
+#define P9_DOTL_CREATE        00000100
+#define P9_DOTL_EXCL          00000200
+#define P9_DOTL_NOCTTY        00000400
+#define P9_DOTL_TRUNC         00001000
+#define P9_DOTL_APPEND        00002000
+#define P9_DOTL_NONBLOCK      00004000
+#define P9_DOTL_DSYNC         00010000
+#define P9_DOTL_FASYNC        00020000
+#define P9_DOTL_DIRECT        00040000
+#define P9_DOTL_LARGEFILE     00100000
+#define P9_DOTL_DIRECTORY     00200000
+#define P9_DOTL_NOFOLLOW      00400000
+#define P9_DOTL_NOATIME       01000000
+#define P9_DOTL_CLOEXEC       02000000
+#define P9_DOTL_SYNC          04000000
+
+/* 9p2000.L at flags */
+#define P9_DOTL_AT_REMOVEDIR         0x200
+
+/* 9P2000.L lock type */
+#define P9_LOCK_TYPE_RDLCK 0
+#define P9_LOCK_TYPE_WRLCK 1
+#define P9_LOCK_TYPE_UNLCK 2
+
+#define P9_LOCK_SUCCESS 0
+#define P9_LOCK_BLOCKED 1
+#define P9_LOCK_ERROR 2
+#define P9_LOCK_GRACE 3
+
+#define P9_LOCK_FLAGS_BLOCK 1
+#define P9_LOCK_FLAGS_RECLAIM 2
+
+typedef struct V9fsFlock
+{
+    uint8_t type;
+    uint32_t flags;
+    uint64_t start; /* absolute offset */
+    uint64_t length;
+    uint32_t proc_id;
+    V9fsString client_id;
+} V9fsFlock;
+
+typedef struct V9fsGetlock
+{
+    uint8_t type;
+    uint64_t start; /* absolute offset */
+    uint64_t length;
+    uint32_t proc_id;
+    V9fsString client_id;
+} V9fsGetlock;
+
+extern int open_fd_hw;
+extern int total_open_fd;
+
+static inline void v9fs_path_write_lock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_wrlock(&s->rename_lock);
+    }
+}
+
+static inline void v9fs_path_read_lock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_rdlock(&s->rename_lock);
+    }
+}
+
+static inline void v9fs_path_unlock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_unlock(&s->rename_lock);
+    }
+}
+
+static inline uint8_t v9fs_request_cancelled(V9fsPDU *pdu)
+{
+    return pdu->cancelled;
+}
+
+extern void v9fs_reclaim_fd(V9fsPDU *pdu);
+extern void v9fs_path_init(V9fsPath *path);
+extern void v9fs_path_free(V9fsPath *path);
+extern void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs);
+extern int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
+                             const char *name, V9fsPath *path);
+extern int v9fs_device_realize_common(V9fsState *s, Error **errp);
+extern void v9fs_device_unrealize_common(V9fsState *s, Error **errp);
+
+ssize_t pdu_marshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...);
+ssize_t pdu_unmarshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...);
+V9fsPDU *pdu_alloc(V9fsState *s);
+void pdu_free(V9fsPDU *pdu);
+void pdu_submit(V9fsPDU *pdu);
+
+#endif
diff --git a/qemu/hw/9pfs/Makefile.objs b/qemu/hw/9pfs/Makefile.objs
index 1e9b595cb..da0ae0cfd 100644
--- a/qemu/hw/9pfs/Makefile.objs
+++ b/qemu/hw/9pfs/Makefile.objs
@@ -1,9 +1,9 @@
-common-obj-y  = virtio-9p.o
-common-obj-y += virtio-9p-local.o virtio-9p-xattr.o
-common-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
-common-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
-common-obj-y += coxattr.o virtio-9p-synth.o
-common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
-common-obj-y += virtio-9p-proxy.o
+common-obj-y  = 9p.o
+common-obj-y += 9p-local.o 9p-xattr.o
+common-obj-y += 9p-xattr-user.o 9p-posix-acl.o
+common-obj-y += coth.o cofs.o codir.o cofile.o
+common-obj-y += coxattr.o 9p-synth.o
+common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  9p-handle.o
+common-obj-y += 9p-proxy.o
 
 obj-y += virtio-9p-device.o
diff --git a/qemu/hw/9pfs/codir.c b/qemu/hw/9pfs/codir.c
index 65ad3298b..91df7f7a7 100644
--- a/qemu/hw/9pfs/codir.c
+++ b/qemu/hw/9pfs/codir.c
@@ -12,10 +12,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 int v9fs_co_readdir_r(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent *dent,
                       struct dirent **result)
diff --git a/qemu/hw/9pfs/cofile.c b/qemu/hw/9pfs/cofile.c
index 2efebf357..293483e0c 100644
--- a/qemu/hw/9pfs/cofile.c
+++ b/qemu/hw/9pfs/cofile.c
@@ -12,10 +12,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode,
                    V9fsStatDotl *v9stat)
diff --git a/qemu/hw/9pfs/cofs.c b/qemu/hw/9pfs/cofs.c
index 42ee614e2..18c81cb3d 100644
--- a/qemu/hw/9pfs/cofs.c
+++ b/qemu/hw/9pfs/cofs.c
@@ -12,10 +12,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 static ssize_t __readlink(V9fsState *s, V9fsPath *path, V9fsString *buf)
 {
diff --git a/qemu/hw/9pfs/coth.c b/qemu/hw/9pfs/coth.c
new file mode 100644
index 000000000..464293ef2
--- /dev/null
+++ b/qemu/hw/9pfs/coth.c
@@ -0,0 +1,42 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
+ *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/thread-pool.h"
+#include "qemu/coroutine.h"
+#include "qemu/main-loop.h"
+#include "coth.h"
+
+/* Called from QEMU I/O thread.  */
+static void coroutine_enter_cb(void *opaque, int ret)
+{
+    Coroutine *co = opaque;
+    qemu_coroutine_enter(co, NULL);
+}
+
+/* Called from worker thread.  */
+static int coroutine_enter_func(void *arg)
+{
+    Coroutine *co = arg;
+    qemu_coroutine_enter(co, NULL);
+    return 0;
+}
+
+void co_run_in_worker_bh(void *opaque)
+{
+    Coroutine *co = opaque;
+    thread_pool_submit_aio(aio_get_thread_pool(qemu_get_aio_context()),
+                           coroutine_enter_func, co, coroutine_enter_cb, co);
+}
diff --git a/qemu/hw/9pfs/virtio-9p-coth.h b/qemu/hw/9pfs/coth.h
index 4f51b250d..209fc6a9a 100644
--- a/qemu/hw/9pfs/virtio-9p-coth.h
+++ b/qemu/hw/9pfs/coth.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2010
  *
@@ -12,20 +12,12 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_9P_COTH_H
-#define _QEMU_VIRTIO_9P_COTH_H
+#ifndef _QEMU_9P_COTH_H
+#define _QEMU_9P_COTH_H
 
 #include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "virtio-9p.h"
-#include <glib.h>
-
-typedef struct V9fsThPool {
-    EventNotifier e;
-
-    GThreadPool *pool;
-    GAsyncQueue *completed;
-} V9fsThPool;
 
 /*
  * we want to use bottom half because we want to make sure the below
@@ -45,7 +37,7 @@ typedef struct V9fsThPool {
         qemu_bh_schedule(co_bh);                                        \
         /*                                                              \
          * yield in qemu thread and re-enter back                       \
-         * in glib worker thread                                        \
+         * in worker thread                                             \
          */                                                             \
         qemu_coroutine_yield();                                         \
         qemu_bh_delete(co_bh);                                          \
diff --git a/qemu/hw/9pfs/coxattr.c b/qemu/hw/9pfs/coxattr.c
index 18ee08df0..6ad96ea9f 100644
--- a/qemu/hw/9pfs/coxattr.c
+++ b/qemu/hw/9pfs/coxattr.c
@@ -12,10 +12,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 int v9fs_co_llistxattr(V9fsPDU *pdu, V9fsPath *path, void *value, size_t size)
 {
diff --git a/qemu/hw/9pfs/virtio-9p-coth.c b/qemu/hw/9pfs/virtio-9p-coth.c
deleted file mode 100644
index 8185c533c..000000000
--- a/qemu/hw/9pfs/virtio-9p-coth.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Virtio 9p backend
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
- *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "fsdev/qemu-fsdev.h"
-#include "qemu/thread.h"
-#include "qemu/event_notifier.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
-
-/* v9fs glib thread pool */
-static V9fsThPool v9fs_pool;
-
-void co_run_in_worker_bh(void *opaque)
-{
-    Coroutine *co = opaque;
-    g_thread_pool_push(v9fs_pool.pool, co, NULL);
-}
-
-static void v9fs_qemu_process_req_done(EventNotifier *e)
-{
-    Coroutine *co;
-
-    event_notifier_test_and_clear(e);
-
-    while ((co = g_async_queue_try_pop(v9fs_pool.completed)) != NULL) {
-        qemu_coroutine_enter(co, NULL);
-    }
-}
-
-static void v9fs_thread_routine(gpointer data, gpointer user_data)
-{
-    Coroutine *co = data;
-
-    qemu_coroutine_enter(co, NULL);
-
-    g_async_queue_push(v9fs_pool.completed, co);
-
-    event_notifier_set(&v9fs_pool.e);
-}
-
-int v9fs_init_worker_threads(void)
-{
-    int ret = 0;
-    V9fsThPool *p = &v9fs_pool;
-    sigset_t set, oldset;
-
-    sigfillset(&set);
-    /* Leave signal handling to the iothread.  */
-    pthread_sigmask(SIG_SETMASK, &set, &oldset);
-
-    p->pool = g_thread_pool_new(v9fs_thread_routine, p, -1, FALSE, NULL);
-    if (!p->pool) {
-        ret = -1;
-        goto err_out;
-    }
-    p->completed = g_async_queue_new();
-    if (!p->completed) {
-        /*
-         * We are going to terminate.
-         * So don't worry about cleanup
-         */
-        ret = -1;
-        goto err_out;
-    }
-    event_notifier_init(&p->e, 0);
-
-    event_notifier_set_handler(&p->e, v9fs_qemu_process_req_done);
-err_out:
-    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
-    return ret;
-}
diff --git a/qemu/hw/9pfs/virtio-9p-device.c b/qemu/hw/9pfs/virtio-9p-device.c
index 93a407c45..a38850ee8 100644
--- a/qemu/hw/9pfs/virtio-9p-device.c
+++ b/qemu/hw/9pfs/virtio-9p-device.c
@@ -11,15 +11,70 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-9p.h"
 #include "hw/i386/pc.h"
 #include "qemu/sockets.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-xattr.h"
-#include "virtio-9p-coth.h"
+#include "9p-xattr.h"
+#include "coth.h"
 #include "hw/virtio/virtio-access.h"
+#include "qemu/iov.h"
+
+void virtio_9p_push_and_notify(V9fsPDU *pdu)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    /* push onto queue and notify */
+    virtqueue_push(v->vq, elem, pdu->size);
+    g_free(elem);
+    v->elems[pdu->idx] = NULL;
+
+    /* FIXME: we should batch these completions */
+    virtio_notify(VIRTIO_DEVICE(v), v->vq);
+}
+
+static void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq)
+{
+    V9fsVirtioState *v = (V9fsVirtioState *)vdev;
+    V9fsState *s = &v->state;
+    V9fsPDU *pdu;
+    ssize_t len;
+
+    while ((pdu = pdu_alloc(s))) {
+        struct {
+            uint32_t size_le;
+            uint8_t id;
+            uint16_t tag_le;
+        } QEMU_PACKED out;
+        VirtQueueElement *elem;
+
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            pdu_free(pdu);
+            break;
+        }
+
+        BUG_ON(elem->out_num == 0 || elem->in_num == 0);
+        QEMU_BUILD_BUG_ON(sizeof out != 7);
+
+        v->elems[pdu->idx] = elem;
+        len = iov_to_buf(elem->out_sg, elem->out_num, 0,
+                         &out, sizeof out);
+        BUG_ON(len != sizeof out);
+
+        pdu->size = le32_to_cpu(out.size_le);
+
+        pdu->id = out.id;
+        pdu->tag = le16_to_cpu(out.tag_le);
+
+        qemu_co_queue_init(&pdu->complete);
+        pdu_submit(pdu);
+    }
+}
 
 static uint64_t virtio_9p_get_features(VirtIODevice *vdev, uint64_t features,
                                        Error **errp)
@@ -32,117 +87,99 @@ static void virtio_9p_get_config(VirtIODevice *vdev, uint8_t *config)
 {
     int len;
     struct virtio_9p_config *cfg;
-    V9fsState *s = VIRTIO_9P(vdev);
+    V9fsVirtioState *v = VIRTIO_9P(vdev);
+    V9fsState *s = &v->state;
 
     len = strlen(s->tag);
     cfg = g_malloc0(sizeof(struct virtio_9p_config) + len);
     virtio_stw_p(vdev, &cfg->tag_len, len);
     /* We don't copy the terminating null to config space */
     memcpy(cfg->tag, s->tag, len);
-    memcpy(config, cfg, s->config_size);
+    memcpy(config, cfg, v->config_size);
     g_free(cfg);
 }
 
+static void virtio_9p_save(QEMUFile *f, void *opaque)
+{
+    virtio_save(VIRTIO_DEVICE(opaque), f);
+}
+
+static int virtio_9p_load(QEMUFile *f, void *opaque, int version_id)
+{
+    return virtio_load(VIRTIO_DEVICE(opaque), f, version_id);
+}
+
 static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
-    V9fsState *s = VIRTIO_9P(dev);
-    int i, len;
-    struct stat stat;
-    FsDriverEntry *fse;
-    V9fsPath path;
-
-    virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P,
-                sizeof(struct virtio_9p_config) + MAX_TAG_LEN);
-
-    /* initialize pdu allocator */
-    QLIST_INIT(&s->free_list);
-    QLIST_INIT(&s->active_list);
-    for (i = 0; i < (MAX_REQ - 1); i++) {
-        QLIST_INSERT_HEAD(&s->free_list, &s->pdus[i], next);
-    }
+    V9fsVirtioState *v = VIRTIO_9P(dev);
+    V9fsState *s = &v->state;
 
-    s->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
+    if (v9fs_device_realize_common(s, errp)) {
+        goto out;
+    }
 
-    v9fs_path_init(&path);
+    v->config_size = sizeof(struct virtio_9p_config) + strlen(s->fsconf.tag);
+    virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P, v->config_size);
+    v->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
+    register_savevm(dev, "virtio-9p", -1, 1, virtio_9p_save, virtio_9p_load, v);
 
-    fse = get_fsdev_fsentry(s->fsconf.fsdev_id);
+out:
+    return;
+}
 
-    if (!fse) {
-        /* We don't have a fsdev identified by fsdev_id */
-        error_setg(errp, "Virtio-9p device couldn't find fsdev with the "
-                   "id = %s",
-                   s->fsconf.fsdev_id ? s->fsconf.fsdev_id : "NULL");
-        goto out;
-    }
+static void virtio_9p_device_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    V9fsVirtioState *v = VIRTIO_9P(dev);
+    V9fsState *s = &v->state;
 
-    if (!s->fsconf.tag) {
-        /* we haven't specified a mount_tag */
-        error_setg(errp, "fsdev with id %s needs mount_tag arguments",
-                   s->fsconf.fsdev_id);
-        goto out;
-    }
+    virtio_cleanup(vdev);
+    unregister_savevm(dev, "virtio-9p", v);
+    v9fs_device_unrealize_common(s, errp);
+}
 
-    s->ctx.export_flags = fse->export_flags;
-    s->ctx.fs_root = g_strdup(fse->path);
-    s->ctx.exops.get_st_gen = NULL;
-    len = strlen(s->fsconf.tag);
-    if (len > MAX_TAG_LEN - 1) {
-        error_setg(errp, "mount tag '%s' (%d bytes) is longer than "
-                   "maximum (%d bytes)", s->fsconf.tag, len, MAX_TAG_LEN - 1);
-        goto out;
-    }
+ssize_t virtio_pdu_vmarshal(V9fsPDU *pdu, size_t offset,
+                            const char *fmt, va_list ap)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
 
-    s->tag = g_strdup(s->fsconf.tag);
-    s->ctx.uid = -1;
+    return v9fs_iov_vmarshal(elem->in_sg, elem->in_num, offset, 1, fmt, ap);
+}
 
-    s->ops = fse->ops;
-    s->config_size = sizeof(struct virtio_9p_config) + len;
-    s->fid_list = NULL;
-    qemu_co_rwlock_init(&s->rename_lock);
+ssize_t virtio_pdu_vunmarshal(V9fsPDU *pdu, size_t offset,
+                              const char *fmt, va_list ap)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
 
-    if (s->ops->init(&s->ctx) < 0) {
-        error_setg(errp, "Virtio-9p Failed to initialize fs-driver with id:%s"
-                   " and export path:%s", s->fsconf.fsdev_id, s->ctx.fs_root);
-        goto out;
-    }
-    if (v9fs_init_worker_threads() < 0) {
-        error_setg(errp, "worker thread initialization failed");
-        goto out;
-    }
+    return v9fs_iov_vunmarshal(elem->out_sg, elem->out_num, offset, 1, fmt, ap);
+}
 
-    /*
-     * Check details of export path, We need to use fs driver
-     * call back to do that. Since we are in the init path, we don't
-     * use co-routines here.
-     */
-    if (s->ops->name_to_path(&s->ctx, NULL, "/", &path) < 0) {
-        error_setg(errp,
-                   "error in converting name to path %s", strerror(errno));
-        goto out;
-    }
-    if (s->ops->lstat(&s->ctx, &path, &stat)) {
-        error_setg(errp, "share path %s does not exist", fse->path);
-        goto out;
-    } else if (!S_ISDIR(stat.st_mode)) {
-        error_setg(errp, "share path %s is not a directory", fse->path);
-        goto out;
+void virtio_init_iov_from_pdu(V9fsPDU *pdu, struct iovec **piov,
+                              unsigned int *pniov, bool is_write)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    if (is_write) {
+        *piov = elem->out_sg;
+        *pniov = elem->out_num;
+    } else {
+        *piov = elem->in_sg;
+        *pniov = elem->in_num;
     }
-    v9fs_path_free(&path);
-
-    return;
-out:
-    g_free(s->ctx.fs_root);
-    g_free(s->tag);
-    virtio_cleanup(vdev);
-    v9fs_path_free(&path);
 }
 
 /* virtio-9p device */
 
 static Property virtio_9p_properties[] = {
-    DEFINE_PROP_STRING("mount_tag", V9fsState, fsconf.tag),
-    DEFINE_PROP_STRING("fsdev", V9fsState, fsconf.fsdev_id),
+    DEFINE_PROP_STRING("mount_tag", V9fsVirtioState, state.fsconf.tag),
+    DEFINE_PROP_STRING("fsdev", V9fsVirtioState, state.fsconf.fsdev_id),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -154,6 +191,7 @@ static void virtio_9p_class_init(ObjectClass *klass, void *data)
     dc->props = virtio_9p_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     vdc->realize = virtio_9p_device_realize;
+    vdc->unrealize = virtio_9p_device_unrealize;
     vdc->get_features = virtio_9p_get_features;
     vdc->get_config = virtio_9p_get_config;
 }
@@ -161,7 +199,7 @@ static void virtio_9p_class_init(ObjectClass *klass, void *data)
 static const TypeInfo virtio_device_info = {
     .name = TYPE_VIRTIO_9P,
     .parent = TYPE_VIRTIO_DEVICE,
-    .instance_size = sizeof(V9fsState),
+    .instance_size = sizeof(V9fsVirtioState),
     .class_init = virtio_9p_class_init,
 };
 
diff --git a/qemu/hw/9pfs/virtio-9p.h b/qemu/hw/9pfs/virtio-9p.h
index 2e7d48857..7f6d88553 100644
--- a/qemu/hw/9pfs/virtio-9p.h
+++ b/qemu/hw/9pfs/virtio-9p.h
@@ -1,394 +1,31 @@
 #ifndef _QEMU_VIRTIO_9P_H
 #define _QEMU_VIRTIO_9P_H
 
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/time.h>
-#include <utime.h>
-#include <sys/resource.h>
-#include <glib.h>
 #include "standard-headers/linux/virtio_9p.h"
 #include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-9p.h"
-#include "fsdev/file-op-9p.h"
-#include "fsdev/virtio-9p-marshal.h"
-#include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "9p.h"
 
-enum {
-    P9_TLERROR = 6,
-    P9_RLERROR,
-    P9_TSTATFS = 8,
-    P9_RSTATFS,
-    P9_TLOPEN = 12,
-    P9_RLOPEN,
-    P9_TLCREATE = 14,
-    P9_RLCREATE,
-    P9_TSYMLINK = 16,
-    P9_RSYMLINK,
-    P9_TMKNOD = 18,
-    P9_RMKNOD,
-    P9_TRENAME = 20,
-    P9_RRENAME,
-    P9_TREADLINK = 22,
-    P9_RREADLINK,
-    P9_TGETATTR = 24,
-    P9_RGETATTR,
-    P9_TSETATTR = 26,
-    P9_RSETATTR,
-    P9_TXATTRWALK = 30,
-    P9_RXATTRWALK,
-    P9_TXATTRCREATE = 32,
-    P9_RXATTRCREATE,
-    P9_TREADDIR = 40,
-    P9_RREADDIR,
-    P9_TFSYNC = 50,
-    P9_RFSYNC,
-    P9_TLOCK = 52,
-    P9_RLOCK,
-    P9_TGETLOCK = 54,
-    P9_RGETLOCK,
-    P9_TLINK = 70,
-    P9_RLINK,
-    P9_TMKDIR = 72,
-    P9_RMKDIR,
-    P9_TRENAMEAT = 74,
-    P9_RRENAMEAT,
-    P9_TUNLINKAT = 76,
-    P9_RUNLINKAT,
-    P9_TVERSION = 100,
-    P9_RVERSION,
-    P9_TAUTH = 102,
-    P9_RAUTH,
-    P9_TATTACH = 104,
-    P9_RATTACH,
-    P9_TERROR = 106,
-    P9_RERROR,
-    P9_TFLUSH = 108,
-    P9_RFLUSH,
-    P9_TWALK = 110,
-    P9_RWALK,
-    P9_TOPEN = 112,
-    P9_ROPEN,
-    P9_TCREATE = 114,
-    P9_RCREATE,
-    P9_TREAD = 116,
-    P9_RREAD,
-    P9_TWRITE = 118,
-    P9_RWRITE,
-    P9_TCLUNK = 120,
-    P9_RCLUNK,
-    P9_TREMOVE = 122,
-    P9_RREMOVE,
-    P9_TSTAT = 124,
-    P9_RSTAT,
-    P9_TWSTAT = 126,
-    P9_RWSTAT,
-};
-
-
-/* qid.types */
-enum {
-    P9_QTDIR = 0x80,
-    P9_QTAPPEND = 0x40,
-    P9_QTEXCL = 0x20,
-    P9_QTMOUNT = 0x10,
-    P9_QTAUTH = 0x08,
-    P9_QTTMP = 0x04,
-    P9_QTSYMLINK = 0x02,
-    P9_QTLINK = 0x01,
-    P9_QTFILE = 0x00,
-};
-
-enum p9_proto_version {
-    V9FS_PROTO_2000U = 0x01,
-    V9FS_PROTO_2000L = 0x02,
-};
-
-#define P9_NOTAG    (u16)(~0)
-#define P9_NOFID    (u32)(~0)
-#define P9_MAXWELEM 16
-
-#define FID_REFERENCED          0x1
-#define FID_NON_RECLAIMABLE     0x2
-static inline char *rpath(FsContext *ctx, const char *path)
-{
-    return g_strdup_printf("%s/%s", ctx->fs_root, path);
-}
-
-/*
- * ample room for Twrite/Rread header
- * size[4] Tread/Twrite tag[2] fid[4] offset[8] count[4]
- */
-#define P9_IOHDRSZ 24
-
-typedef struct V9fsPDU V9fsPDU;
-struct V9fsState;
-
-struct V9fsPDU
-{
-    uint32_t size;
-    uint16_t tag;
-    uint8_t id;
-    uint8_t cancelled;
-    CoQueue complete;
-    VirtQueueElement elem;
-    struct V9fsState *s;
-    QLIST_ENTRY(V9fsPDU) next;
-};
-
-
-/* FIXME
- * 1) change user needs to set groups and stuff
- */
-
-#define MAX_REQ         128
-#define MAX_TAG_LEN     32
-
-#define BUG_ON(cond) assert(!(cond))
-
-typedef struct V9fsFidState V9fsFidState;
-
-enum {
-    P9_FID_NONE = 0,
-    P9_FID_FILE,
-    P9_FID_DIR,
-    P9_FID_XATTR,
-};
-
-typedef struct V9fsXattr
-{
-    int64_t copied_len;
-    int64_t len;
-    void *value;
-    V9fsString name;
-    int flags;
-} V9fsXattr;
-
-/*
- * Filled by fs driver on open and other
- * calls.
- */
-union V9fsFidOpenState {
-    int fd;
-    DIR *dir;
-    V9fsXattr xattr;
-    /*
-     * private pointer for fs drivers, that
-     * have its own internal representation of
-     * open files.
-     */
-    void *private;
-};
-
-struct V9fsFidState
-{
-    int fid_type;
-    int32_t fid;
-    V9fsPath path;
-    V9fsFidOpenState fs;
-    V9fsFidOpenState fs_reclaim;
-    int flags;
-    int open_flags;
-    uid_t uid;
-    int ref;
-    int clunked;
-    V9fsFidState *next;
-    V9fsFidState *rclm_lst;
-};
-
-typedef struct V9fsState
+typedef struct V9fsVirtioState
 {
     VirtIODevice parent_obj;
     VirtQueue *vq;
-    V9fsPDU pdus[MAX_REQ];
-    QLIST_HEAD(, V9fsPDU) free_list;
-    QLIST_HEAD(, V9fsPDU) active_list;
-    V9fsFidState *fid_list;
-    FileOperations *ops;
-    FsContext ctx;
-    char *tag;
     size_t config_size;
-    enum p9_proto_version proto_version;
-    int32_t msize;
-    /*
-     * lock ensuring atomic path update
-     * on rename.
-     */
-    CoRwlock rename_lock;
-    int32_t root_fid;
-    Error *migration_blocker;
-    V9fsConf fsconf;
-} V9fsState;
-
-typedef struct V9fsStatState {
-    V9fsPDU *pdu;
-    size_t offset;
-    V9fsStat v9stat;
-    V9fsFidState *fidp;
-    struct stat stbuf;
-} V9fsStatState;
-
-typedef struct V9fsOpenState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t mode;
-    V9fsFidState *fidp;
-    V9fsQID qid;
-    struct stat stbuf;
-    int iounit;
-} V9fsOpenState;
-
-typedef struct V9fsReadState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t count;
-    int32_t total;
-    int64_t off;
-    V9fsFidState *fidp;
-    struct iovec iov[128]; /* FIXME: bad, bad, bad */
-    struct iovec *sg;
-    off_t dir_pos;
-    struct dirent *dent;
-    struct stat stbuf;
-    V9fsString name;
-    V9fsStat v9stat;
-    int32_t len;
-    int32_t cnt;
-    int32_t max_count;
-} V9fsReadState;
-
-typedef struct V9fsWriteState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t len;
-    int32_t count;
-    int32_t total;
-    int64_t off;
-    V9fsFidState *fidp;
-    struct iovec iov[128]; /* FIXME: bad, bad, bad */
-    struct iovec *sg;
-    int cnt;
-} V9fsWriteState;
-
-typedef struct V9fsMkState {
-    V9fsPDU *pdu;
-    size_t offset;
-    V9fsQID qid;
-    struct stat stbuf;
-    V9fsString name;
-    V9fsString fullname;
-} V9fsMkState;
-
-/* 9p2000.L open flags */
-#define P9_DOTL_RDONLY        00000000
-#define P9_DOTL_WRONLY        00000001
-#define P9_DOTL_RDWR          00000002
-#define P9_DOTL_NOACCESS      00000003
-#define P9_DOTL_CREATE        00000100
-#define P9_DOTL_EXCL          00000200
-#define P9_DOTL_NOCTTY        00000400
-#define P9_DOTL_TRUNC         00001000
-#define P9_DOTL_APPEND        00002000
-#define P9_DOTL_NONBLOCK      00004000
-#define P9_DOTL_DSYNC         00010000
-#define P9_DOTL_FASYNC        00020000
-#define P9_DOTL_DIRECT        00040000
-#define P9_DOTL_LARGEFILE     00100000
-#define P9_DOTL_DIRECTORY     00200000
-#define P9_DOTL_NOFOLLOW      00400000
-#define P9_DOTL_NOATIME       01000000
-#define P9_DOTL_CLOEXEC       02000000
-#define P9_DOTL_SYNC          04000000
-
-/* 9p2000.L at flags */
-#define P9_DOTL_AT_REMOVEDIR         0x200
-
-/* 9P2000.L lock type */
-#define P9_LOCK_TYPE_RDLCK 0
-#define P9_LOCK_TYPE_WRLCK 1
-#define P9_LOCK_TYPE_UNLCK 2
-
-#define P9_LOCK_SUCCESS 0
-#define P9_LOCK_BLOCKED 1
-#define P9_LOCK_ERROR 2
-#define P9_LOCK_GRACE 3
-
-#define P9_LOCK_FLAGS_BLOCK 1
-#define P9_LOCK_FLAGS_RECLAIM 2
-
-typedef struct V9fsFlock
-{
-    uint8_t type;
-    uint32_t flags;
-    uint64_t start; /* absolute offset */
-    uint64_t length;
-    uint32_t proc_id;
-    V9fsString client_id;
-} V9fsFlock;
-
-typedef struct V9fsGetlock
-{
-    uint8_t type;
-    uint64_t start; /* absolute offset */
-    uint64_t length;
-    uint32_t proc_id;
-    V9fsString client_id;
-} V9fsGetlock;
-
-extern int open_fd_hw;
-extern int total_open_fd;
-
-size_t pdu_packunpack(void *addr, struct iovec *sg, int sg_count,
-                      size_t offset, size_t size, int pack);
-
-static inline size_t do_pdu_unpack(void *dst, struct iovec *sg, int sg_count,
-                        size_t offset, size_t size)
-{
-    return pdu_packunpack(dst, sg, sg_count, offset, size, 0);
-}
-
-static inline void v9fs_path_write_lock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_wrlock(&s->rename_lock);
-    }
-}
-
-static inline void v9fs_path_read_lock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_rdlock(&s->rename_lock);
-    }
-}
-
-static inline void v9fs_path_unlock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_unlock(&s->rename_lock);
-    }
-}
-
-static inline uint8_t v9fs_request_cancelled(V9fsPDU *pdu)
-{
-    return pdu->cancelled;
-}
+    V9fsPDU pdus[MAX_REQ];
+    VirtQueueElement *elems[MAX_REQ];
+    V9fsState state;
+} V9fsVirtioState;
 
-extern void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq);
-extern void v9fs_reclaim_fd(V9fsPDU *pdu);
-extern void v9fs_path_init(V9fsPath *path);
-extern void v9fs_path_free(V9fsPath *path);
-extern void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs);
-extern int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
-                             const char *name, V9fsPath *path);
+extern void virtio_9p_push_and_notify(V9fsPDU *pdu);
 
-#define pdu_marshal(pdu, offset, fmt, args...)  \
-    v9fs_marshal(pdu->elem.in_sg, pdu->elem.in_num, offset, 1, fmt, ##args)
-#define pdu_unmarshal(pdu, offset, fmt, args...)  \
-    v9fs_unmarshal(pdu->elem.out_sg, pdu->elem.out_num, offset, 1, fmt, ##args)
+ssize_t virtio_pdu_vmarshal(V9fsPDU *pdu, size_t offset,
+                            const char *fmt, va_list ap);
+ssize_t virtio_pdu_vunmarshal(V9fsPDU *pdu, size_t offset,
+                              const char *fmt, va_list ap);
+void virtio_init_iov_from_pdu(V9fsPDU *pdu, struct iovec **piov,
+                              unsigned int *pniov, bool is_write);
 
 #define TYPE_VIRTIO_9P "virtio-9p-device"
 #define VIRTIO_9P(obj) \
-        OBJECT_CHECK(V9fsState, (obj), TYPE_VIRTIO_9P)
+        OBJECT_CHECK(V9fsVirtioState, (obj), TYPE_VIRTIO_9P)
 
 #endif
diff --git a/qemu/hw/Makefile.objs b/qemu/hw/Makefile.objs
index 73afa41b3..4a07ed434 100644
--- a/qemu/hw/Makefile.objs
+++ b/qemu/hw/Makefile.objs
@@ -13,6 +13,7 @@ devices-dirs-$(CONFIG_SOFTMMU) += ide/
 devices-dirs-$(CONFIG_SOFTMMU) += input/
 devices-dirs-$(CONFIG_SOFTMMU) += intc/
 devices-dirs-$(CONFIG_IPACK) += ipack/
+devices-dirs-$(CONFIG_IPMI) += ipmi/
 devices-dirs-$(CONFIG_SOFTMMU) += isa/
 devices-dirs-$(CONFIG_SOFTMMU) += misc/
 devices-dirs-$(CONFIG_SOFTMMU) += net/
@@ -31,6 +32,7 @@ devices-dirs-$(CONFIG_VIRTIO) += virtio/
 devices-dirs-$(CONFIG_SOFTMMU) += watchdog/
 devices-dirs-$(CONFIG_SOFTMMU) += xen/
 devices-dirs-$(CONFIG_MEM_HOTPLUG) += mem/
+devices-dirs-$(CONFIG_SMBIOS) += smbios/
 devices-dirs-y += core/
 common-obj-y += $(devices-dirs-y)
 obj-y += $(devices-dirs-y)
diff --git a/qemu/hw/acpi/Makefile.objs b/qemu/hw/acpi/Makefile.objs
index 7d3230c2a..faee86c5c 100644
--- a/qemu/hw/acpi/Makefile.objs
+++ b/qemu/hw/acpi/Makefile.objs
@@ -1,7 +1,8 @@
 common-obj-$(CONFIG_ACPI_X86) += core.o piix4.o pcihp.o
 common-obj-$(CONFIG_ACPI_X86_ICH) += ich9.o tco.o
-common-obj-$(CONFIG_ACPI_CPU_HOTPLUG) += cpu_hotplug.o
-common-obj-$(CONFIG_ACPI_MEMORY_HOTPLUG) += memory_hotplug.o
+common-obj-$(CONFIG_ACPI_CPU_HOTPLUG) += cpu_hotplug.o cpu_hotplug_acpi_table.o
+common-obj-$(CONFIG_ACPI_MEMORY_HOTPLUG) += memory_hotplug.o memory_hotplug_acpi_table.o
+obj-$(CONFIG_ACPI_NVDIMM) += nvdimm.o
 common-obj-$(CONFIG_ACPI) += acpi_interface.o
 common-obj-$(CONFIG_ACPI) += bios-linker-loader.o
 common-obj-$(CONFIG_ACPI) += aml-build.o
diff --git a/qemu/hw/acpi/acpi_interface.c b/qemu/hw/acpi/acpi_interface.c
index c181bb226..d82131326 100644
--- a/qemu/hw/acpi/acpi_interface.c
+++ b/qemu/hw/acpi/acpi_interface.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/acpi/acpi_dev_interface.h"
 #include "qemu/module.h"
 
diff --git a/qemu/hw/acpi/aml-build.c b/qemu/hw/acpi/aml-build.c
index 0d4b3247b..ab89ca638 100644
--- a/qemu/hw/acpi/aml-build.c
+++ b/qemu/hw/acpi/aml-build.c
@@ -19,12 +19,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <glib/gprintf.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <assert.h>
-#include <stdbool.h>
-#include <string.h>
 #include "hw/acpi/aml-build.h"
 #include "qemu/bswap.h"
 #include "qemu/bitops.h"
@@ -262,6 +258,34 @@ static void build_append_int(GArray *table, uint64_t value)
     }
 }
 
+/*
+ * Build NAME(XXXX, 0x00000000) where 0x00000000 is encoded as a dword,
+ * and return the offset to 0x00000000 for runtime patching.
+ *
+ * Warning: runtime patching is best avoided. Only use this as
+ * a replacement for DataTableRegion (for guests that don't
+ * support it).
+ */
+int
+build_append_named_dword(GArray *array, const char *name_format, ...)
+{
+    int offset;
+    va_list ap;
+
+    build_append_byte(array, 0x08); /* NameOp */
+    va_start(ap, name_format);
+    build_append_namestringv(array, name_format, ap);
+    va_end(ap);
+
+    build_append_byte(array, 0x0C); /* DWordPrefix */
+
+    offset = array->len;
+    build_append_int_noprefix(array, 0x00000000, 4);
+    assert(array->len == offset + 4);
+
+    return offset;
+}
+
 static GPtrArray *alloc_list;
 
 static Aml *aml_alloc(void)
@@ -427,6 +451,41 @@ Aml *aml_arg(int pos)
     return var;
 }
 
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToInteger */
+Aml *aml_to_integer(Aml *arg)
+{
+    Aml *var = aml_opcode(0x99 /* ToIntegerOp */);
+    aml_append(var, arg);
+    build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    return var;
+}
+
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToHexString */
+Aml *aml_to_hexstring(Aml *src, Aml *dst)
+{
+    Aml *var = aml_opcode(0x98 /* ToHexStringOp */);
+    aml_append(var, src);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
+    return var;
+}
+
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToBuffer */
+Aml *aml_to_buffer(Aml *src, Aml *dst)
+{
+    Aml *var = aml_opcode(0x96 /* ToBufferOp */);
+    aml_append(var, src);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
+    return var;
+}
+
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefStore */
 Aml *aml_store(Aml *val, Aml *target)
 {
@@ -436,44 +495,64 @@ Aml *aml_store(Aml *val, Aml *target)
     return var;
 }
 
-/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAnd */
-Aml *aml_and(Aml *arg1, Aml *arg2)
+/**
+ * build_opcode_2arg_dst:
+ * @op: 1-byte opcode
+ * @arg1: 1st operand
+ * @arg2: 2nd operand
+ * @dst: optional target to store to, set to NULL if it's not required
+ *
+ * An internal helper to compose AML terms that have
+ *   "Op Operand Operand Target"
+ * pattern.
+ *
+ * Returns: The newly allocated and composed according to patter Aml object.
+ */
+static Aml *
+build_opcode_2arg_dst(uint8_t op, Aml *arg1, Aml *arg2, Aml *dst)
 {
-    Aml *var = aml_opcode(0x7B /* AndOp */);
+    Aml *var = aml_opcode(op);
     aml_append(var, arg1);
     aml_append(var, arg2);
-    build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
     return var;
 }
 
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAnd */
+Aml *aml_and(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x7B /* AndOp */, arg1, arg2, dst);
+}
+
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefOr */
-Aml *aml_or(Aml *arg1, Aml *arg2)
+Aml *aml_or(Aml *arg1, Aml *arg2, Aml *dst)
 {
-    Aml *var = aml_opcode(0x7D /* OrOp */);
+    return build_opcode_2arg_dst(0x7D /* OrOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLOr */
+Aml *aml_lor(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x91 /* LOrOp */);
     aml_append(var, arg1);
     aml_append(var, arg2);
-    build_append_byte(var->buf, 0x00 /* NullNameOp */);
     return var;
 }
 
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefShiftLeft */
 Aml *aml_shiftleft(Aml *arg1, Aml *count)
 {
-    Aml *var = aml_opcode(0x79 /* ShiftLeftOp */);
-    aml_append(var, arg1);
-    aml_append(var, count);
-    build_append_byte(var->buf, 0x00); /* NullNameOp */
-    return var;
+    return build_opcode_2arg_dst(0x79 /* ShiftLeftOp */, arg1, count, NULL);
 }
 
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefShiftRight */
-Aml *aml_shiftright(Aml *arg1, Aml *count)
+Aml *aml_shiftright(Aml *arg1, Aml *count, Aml *dst)
 {
-    Aml *var = aml_opcode(0x7A /* ShiftRightOp */);
-    aml_append(var, arg1);
-    aml_append(var, count);
-    build_append_byte(var->buf, 0x00); /* NullNameOp */
-    return var;
+    return build_opcode_2arg_dst(0x7A /* ShiftRightOp */, arg1, count, dst);
 }
 
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLLess */
@@ -486,13 +565,15 @@ Aml *aml_lless(Aml *arg1, Aml *arg2)
 }
 
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAdd */
-Aml *aml_add(Aml *arg1, Aml *arg2)
+Aml *aml_add(Aml *arg1, Aml *arg2, Aml *dst)
 {
-    Aml *var = aml_opcode(0x72 /* AddOp */);
-    aml_append(var, arg1);
-    aml_append(var, arg2);
-    build_append_byte(var->buf, 0x00 /* NullNameOp */);
-    return var;
+    return build_opcode_2arg_dst(0x72 /* AddOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefSubtract */
+Aml *aml_subtract(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x74 /* SubtractOp */, arg1, arg2, dst);
 }
 
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefIncrement */
@@ -503,14 +584,18 @@ Aml *aml_increment(Aml *arg)
     return var;
 }
 
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefDecrement */
+Aml *aml_decrement(Aml *arg)
+{
+    Aml *var = aml_opcode(0x76 /* DecrementOp */);
+    aml_append(var, arg);
+    return var;
+}
+
 /* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefIndex */
 Aml *aml_index(Aml *arg1, Aml *idx)
 {
-    Aml *var = aml_opcode(0x88 /* IndexOp */);
-    aml_append(var, arg1);
-    aml_append(var, idx);
-    build_append_byte(var->buf, 0x00 /* NullNameOp */);
-    return var;
+    return build_opcode_2arg_dst(0x88 /* IndexOp */, arg1, idx, NULL);
 }
 
 /* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefNotify */
@@ -523,6 +608,14 @@ Aml *aml_notify(Aml *arg1, Aml *arg2)
 }
 
 /* helper to call method with 1 argument */
+Aml *aml_call0(const char *method)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    return var;
+}
+
+/* helper to call method with 1 argument */
 Aml *aml_call1(const char *method, Aml *arg1)
 {
     Aml *var = aml_alloc();
@@ -565,6 +658,94 @@ Aml *aml_call4(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4)
 }
 
 /*
+ * ACPI 5.0: 6.4.3.8.1 GPIO Connection Descriptor
+ * Type 1, Large Item Name 0xC
+ */
+
+static Aml *aml_gpio_connection(AmlGpioConnectionType type,
+                                AmlConsumerAndProducer con_and_pro,
+                                uint8_t flags, AmlPinConfig pin_config,
+                                uint16_t output_drive,
+                                uint16_t debounce_timeout,
+                                const uint32_t pin_list[], uint32_t pin_count,
+                                const char *resource_source_name,
+                                const uint8_t *vendor_data,
+                                uint16_t vendor_data_len)
+{
+    Aml *var = aml_alloc();
+    const uint16_t min_desc_len = 0x16;
+    uint16_t resource_source_name_len, length;
+    uint16_t pin_table_offset, resource_source_name_offset, vendor_data_offset;
+    uint32_t i;
+
+    assert(resource_source_name);
+    resource_source_name_len = strlen(resource_source_name) + 1;
+    length = min_desc_len + resource_source_name_len + vendor_data_len;
+    pin_table_offset = min_desc_len + 1;
+    resource_source_name_offset = pin_table_offset + pin_count * 2;
+    vendor_data_offset = resource_source_name_offset + resource_source_name_len;
+
+    build_append_byte(var->buf, 0x8C);  /* GPIO Connection Descriptor */
+    build_append_int_noprefix(var->buf, length, 2); /* Length */
+    build_append_byte(var->buf, 1);     /* Revision ID */
+    build_append_byte(var->buf, type);  /* GPIO Connection Type */
+    /* General Flags (2 bytes) */
+    build_append_int_noprefix(var->buf, con_and_pro, 2);
+    /* Interrupt and IO Flags (2 bytes) */
+    build_append_int_noprefix(var->buf, flags, 2);
+    /* Pin Configuration 0 = Default 1 = Pull-up 2 = Pull-down 3 = No Pull */
+    build_append_byte(var->buf, pin_config);
+    /* Output Drive Strength (2 bytes) */
+    build_append_int_noprefix(var->buf, output_drive, 2);
+    /* Debounce Timeout (2 bytes) */
+    build_append_int_noprefix(var->buf, debounce_timeout, 2);
+    /* Pin Table Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, pin_table_offset, 2);
+    build_append_byte(var->buf, 0);     /* Resource Source Index */
+    /* Resource Source Name Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, resource_source_name_offset, 2);
+    /* Vendor Data Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, vendor_data_offset, 2);
+    /* Vendor Data Length (2 bytes) */
+    build_append_int_noprefix(var->buf, vendor_data_len, 2);
+    /* Pin Number (2n bytes)*/
+    for (i = 0; i < pin_count; i++) {
+        build_append_int_noprefix(var->buf, pin_list[i], 2);
+    }
+
+    /* Resource Source Name */
+    build_append_namestring(var->buf, "%s", resource_source_name);
+    build_append_byte(var->buf, '\0');
+
+    /* Vendor-defined Data */
+    if (vendor_data != NULL) {
+        g_array_append_vals(var->buf, vendor_data, vendor_data_len);
+    }
+
+    return var;
+}
+
+/*
+ * ACPI 5.0: 19.5.53
+ * GpioInt(GPIO Interrupt Connection Resource Descriptor Macro)
+ */
+Aml *aml_gpio_int(AmlConsumerAndProducer con_and_pro,
+                  AmlLevelAndEdge edge_level,
+                  AmlActiveHighAndLow active_level, AmlShared shared,
+                  AmlPinConfig pin_config, uint16_t debounce_timeout,
+                  const uint32_t pin_list[], uint32_t pin_count,
+                  const char *resource_source_name,
+                  const uint8_t *vendor_data, uint16_t vendor_data_len)
+{
+    uint8_t flags = edge_level | (active_level << 1) | (shared << 3);
+
+    return aml_gpio_connection(AML_INTERRUPT_CONNECTION, con_and_pro, flags,
+                               pin_config, 0, debounce_timeout, pin_list,
+                               pin_count, resource_source_name, vendor_data,
+                               vendor_data_len);
+}
+
+/*
  * ACPI 1.0b: 6.4.3.4 32-Bit Fixed Location Memory Range Descriptor
  * (Type 1, Large Item Name 0x6)
  */
@@ -598,23 +779,27 @@ Aml *aml_memory32_fixed(uint32_t addr, uint32_t size,
 Aml *aml_interrupt(AmlConsumerAndProducer con_and_pro,
                    AmlLevelAndEdge level_and_edge,
                    AmlActiveHighAndLow high_and_low, AmlShared shared,
-                   uint32_t irq)
+                   uint32_t *irq_list, uint8_t irq_count)
 {
+    int i;
     Aml *var = aml_alloc();
     uint8_t irq_flags = con_and_pro | (level_and_edge << 1)
                         | (high_and_low << 2) | (shared << 3);
+    const int header_bytes_in_len = 2;
+    uint16_t len = header_bytes_in_len + irq_count * sizeof(uint32_t);
+
+    assert(irq_count > 0);
 
     build_append_byte(var->buf, 0x89); /* Extended irq descriptor */
-    build_append_byte(var->buf, 6); /* Length, bits[7:0] minimum value = 6 */
-    build_append_byte(var->buf, 0); /* Length, bits[15:8] minimum value = 0 */
+    build_append_byte(var->buf, len & 0xFF); /* Length, bits[7:0] */
+    build_append_byte(var->buf, len >> 8); /* Length, bits[15:8] */
     build_append_byte(var->buf, irq_flags); /* Interrupt Vector Information. */
-    build_append_byte(var->buf, 0x01);      /* Interrupt table length = 1 */
+    build_append_byte(var->buf, irq_count);   /* Interrupt table length */
 
-    /* Interrupt Number */
-    build_append_byte(var->buf, extract32(irq, 0, 8));  /* bits[7:0] */
-    build_append_byte(var->buf, extract32(irq, 8, 8));  /* bits[15:8] */
-    build_append_byte(var->buf, extract32(irq, 16, 8)); /* bits[23:16] */
-    build_append_byte(var->buf, extract32(irq, 24, 8)); /* bits[31:24] */
+    /* Interrupt Number List */
+    for (i = 0; i < irq_count; i++) {
+        build_append_int_noprefix(var->buf, irq_list[i], 4);
+    }
     return var;
 }
 
@@ -672,6 +857,26 @@ Aml *aml_equal(Aml *arg1, Aml *arg2)
     return var;
 }
 
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLGreater */
+Aml *aml_lgreater(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x94 /* LGreaterOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLGreaterEqual */
+Aml *aml_lgreater_equal(Aml *arg1, Aml *arg2)
+{
+    /* LGreaterEqualOp := LNotOp LLessOp */
+    Aml *var = aml_opcode(0x92 /* LNotOp */);
+    build_append_byte(var->buf, 0x95 /* LLessOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
 /* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefIfElse */
 Aml *aml_if(Aml *predicate)
 {
@@ -696,11 +901,24 @@ Aml *aml_while(Aml *predicate)
 }
 
 /* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefMethod */
-Aml *aml_method(const char *name, int arg_count)
+Aml *aml_method(const char *name, int arg_count, AmlSerializeFlag sflag)
 {
     Aml *var = aml_bundle(0x14 /* MethodOp */, AML_PACKAGE);
+    int methodflags;
+
+    /*
+     * MethodFlags:
+     *   bit 0-2: ArgCount (0-7)
+     *   bit 3: SerializeFlag
+     *     0: NotSerialized
+     *     1: Serialized
+     *   bit 4-7: reserved (must be 0)
+     */
+    assert(arg_count < 8);
+    methodflags = arg_count | (sflag << 3);
+
     build_append_namestring(var->buf, "%s", name);
-    build_append_byte(var->buf, arg_count); /* MethodFlags: ArgCount */
+    build_append_byte(var->buf, methodflags); /* MethodFlags: ArgCount */
     return var;
 }
 
@@ -752,14 +970,14 @@ Aml *aml_package(uint8_t num_elements)
 
 /* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefOpRegion */
 Aml *aml_operation_region(const char *name, AmlRegionSpace rs,
-                          uint32_t offset, uint32_t len)
+                          Aml *offset, uint32_t len)
 {
     Aml *var = aml_alloc();
     build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
     build_append_byte(var->buf, 0x80); /* OpRegionOp */
     build_append_namestring(var->buf, "%s", name);
     build_append_byte(var->buf, rs);
-    build_append_int(var->buf, offset);
+    aml_append(var, offset);
     build_append_int(var->buf, len);
     return var;
 }
@@ -784,27 +1002,57 @@ Aml *aml_reserved_field(unsigned length)
 }
 
 /* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefField */
-Aml *aml_field(const char *name, AmlAccessType type, AmlUpdateRule rule)
+Aml *aml_field(const char *name, AmlAccessType type, AmlLockRule lock,
+               AmlUpdateRule rule)
 {
     Aml *var = aml_bundle(0x81 /* FieldOp */, AML_EXT_PACKAGE);
     uint8_t flags = rule << 5 | type;
 
+    flags |= lock << 4; /* LockRule at 4 bit offset */
+
     build_append_namestring(var->buf, "%s", name);
     build_append_byte(var->buf, flags);
     return var;
 }
 
-/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefCreateDWordField */
-Aml *aml_create_dword_field(Aml *srcbuf, Aml *index, const char *name)
+static
+Aml *create_field_common(int opcode, Aml *srcbuf, Aml *index, const char *name)
 {
-    Aml *var = aml_alloc();
-    build_append_byte(var->buf, 0x8A); /* CreateDWordFieldOp */
+    Aml *var = aml_opcode(opcode);
     aml_append(var, srcbuf);
     aml_append(var, index);
     build_append_namestring(var->buf, "%s", name);
     return var;
 }
 
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefCreateField */
+Aml *aml_create_field(Aml *srcbuf, Aml *bit_index, Aml *num_bits,
+                      const char *name)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x13); /* CreateFieldOp */
+    aml_append(var, srcbuf);
+    aml_append(var, bit_index);
+    aml_append(var, num_bits);
+    build_append_namestring(var->buf, "%s", name);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefCreateDWordField */
+Aml *aml_create_dword_field(Aml *srcbuf, Aml *index, const char *name)
+{
+    return create_field_common(0x8A /* CreateDWordFieldOp */,
+                               srcbuf, index, name);
+}
+
+/* ACPI 2.0a: 17.2.4.2 Named Objects Encoding: DefCreateQWordField */
+Aml *aml_create_qword_field(Aml *srcbuf, Aml *index, const char *name)
+{
+    return create_field_common(0x8F /* CreateQWordFieldOp */,
+                               srcbuf, index, name);
+}
+
 /* ACPI 1.0b: 16.2.3 Data Objects Encoding: String */
 Aml *aml_string(const char *name_format, ...)
 {
@@ -1065,6 +1313,30 @@ Aml *aml_qword_memory(AmlDecode dec, AmlMinFixed min_fixed,
                              addr_trans, len, flags);
 }
 
+/* ACPI 1.0b: 6.4.2.2 DMA Format/6.4.2.2.1 ASL Macro for DMA Descriptor */
+Aml *aml_dma(AmlDmaType typ, AmlDmaBusMaster bm, AmlTransferSize sz,
+             uint8_t channel)
+{
+    Aml *var = aml_alloc();
+    uint8_t flags = sz | bm << 2 | typ << 5;
+
+    assert(channel < 8);
+    build_append_byte(var->buf, 0x2A);    /* Byte 0: DMA Descriptor */
+    build_append_byte(var->buf, 1U << channel); /* Byte 1: _DMA - DmaChannel */
+    build_append_byte(var->buf, flags);   /* Byte 2 */
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefSleep */
+Aml *aml_sleep(uint64_t msec)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x22); /* SleepOp */
+    aml_append(var, aml_int(msec));
+    return var;
+}
+
 static uint8_t Hex2Byte(const char *src)
 {
     int hi, lo;
@@ -1135,23 +1407,100 @@ Aml *aml_unicode(const char *str)
     return var;
 }
 
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefDerefOf */
+Aml *aml_derefof(Aml *arg)
+{
+    Aml *var = aml_opcode(0x83 /* DerefOfOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefSizeOf */
+Aml *aml_sizeof(Aml *arg)
+{
+    Aml *var = aml_opcode(0x87 /* SizeOfOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefMutex */
+Aml *aml_mutex(const char *name, uint8_t sync_level)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x01); /* MutexOp */
+    build_append_namestring(var->buf, "%s", name);
+    assert(!(sync_level & 0xF0));
+    build_append_byte(var->buf, sync_level);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAcquire */
+Aml *aml_acquire(Aml *mutex, uint16_t timeout)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x23); /* AcquireOp */
+    aml_append(var, mutex);
+    build_append_int_noprefix(var->buf, timeout, sizeof(timeout));
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefRelease */
+Aml *aml_release(Aml *mutex)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x27); /* ReleaseOp */
+    aml_append(var, mutex);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.1 Name Space Modifier Objects Encoding: DefAlias */
+Aml *aml_alias(const char *source_object, const char *alias_object)
+{
+    Aml *var = aml_opcode(0x06 /* AliasOp */);
+    aml_append(var, aml_name("%s", source_object));
+    aml_append(var, aml_name("%s", alias_object));
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefConcat */
+Aml *aml_concatenate(Aml *source1, Aml *source2, Aml *target)
+{
+    return build_opcode_2arg_dst(0x73 /* ConcatOp */, source1, source2,
+                                 target);
+}
+
 void
 build_header(GArray *linker, GArray *table_data,
-             AcpiTableHeader *h, const char *sig, int len, uint8_t rev)
+             AcpiTableHeader *h, const char *sig, int len, uint8_t rev,
+             const char *oem_id, const char *oem_table_id)
 {
     memcpy(&h->signature, sig, 4);
     h->length = cpu_to_le32(len);
     h->revision = rev;
-    memcpy(h->oem_id, ACPI_BUILD_APPNAME6, 6);
-    memcpy(h->oem_table_id, ACPI_BUILD_APPNAME4, 4);
-    memcpy(h->oem_table_id + 4, sig, 4);
+
+    if (oem_id) {
+        strncpy((char *)h->oem_id, oem_id, sizeof h->oem_id);
+    } else {
+        memcpy(h->oem_id, ACPI_BUILD_APPNAME6, 6);
+    }
+
+    if (oem_table_id) {
+        strncpy((char *)h->oem_table_id, oem_table_id, sizeof(h->oem_table_id));
+    } else {
+        memcpy(h->oem_table_id, ACPI_BUILD_APPNAME4, 4);
+        memcpy(h->oem_table_id + 4, sig, 4);
+    }
+
     h->oem_revision = cpu_to_le32(1);
     memcpy(h->asl_compiler_id, ACPI_BUILD_APPNAME4, 4);
     h->asl_compiler_revision = cpu_to_le32(1);
     h->checksum = 0;
     /* Checksum to be filled in by Guest linker */
     bios_linker_loader_add_checksum(linker, ACPI_BUILD_TABLE_FILE,
-                                    table_data->data, h, len, &h->checksum);
+                                    table_data, h, len, &h->checksum);
 }
 
 void *acpi_data_push(GArray *table_data, unsigned size)
@@ -1163,9 +1512,7 @@ void *acpi_data_push(GArray *table_data, unsigned size)
 
 unsigned acpi_data_len(GArray *table)
 {
-#if GLIB_CHECK_VERSION(2, 22, 0)
     assert(g_array_get_element_size(table) == 1);
-#endif
     return table->len;
 }
 
@@ -1194,7 +1541,8 @@ void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre)
 
 /* Build rsdt table */
 void
-build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets)
+build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets,
+           const char *oem_id, const char *oem_table_id)
 {
     AcpiRsdtDescriptorRev1 *rsdt;
     size_t rsdt_len;
@@ -1213,5 +1561,5 @@ build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets)
                                        sizeof(uint32_t));
     }
     build_header(linker, table_data,
-                 (void *)rsdt, "RSDT", rsdt_len, 1);
+                 (void *)rsdt, "RSDT", rsdt_len, 1, oem_id, oem_table_id);
 }
diff --git a/qemu/hw/acpi/bios-linker-loader.c b/qemu/hw/acpi/bios-linker-loader.c
index d9382f826..5153ab151 100644
--- a/qemu/hw/acpi/bios-linker-loader.c
+++ b/qemu/hw/acpi/bios-linker-loader.c
@@ -18,12 +18,20 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/acpi/bios-linker-loader.h"
 #include "hw/nvram/fw_cfg.h"
 
 #include "qemu/bswap.h"
 
+/*
+ * Linker/loader is a paravirtualized interface that passes commands to guest.
+ * The commands can be used to request guest to
+ * - allocate memory chunks and initialize them from QEMU FW CFG files
+ * - link allocated chunks by storing pointer to one chunk into another
+ * - calculate ACPI checksum of part of the chunk and store into same chunk
+ */
 #define BIOS_LINKER_LOADER_FILESZ FW_CFG_MAX_FILE_PATH
 
 struct BiosLinkerLoaderEntry {
@@ -87,6 +95,12 @@ enum {
     BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG = 0x2,
 };
 
+/*
+ * bios_linker_loader_init: allocate a new linker file blob array.
+ *
+ * After initialization, linker commands can be added, and will
+ * be stored in the array.
+ */
 GArray *bios_linker_loader_init(void)
 {
     return g_array_new(false, true /* clear */, 1);
@@ -98,6 +112,16 @@ void *bios_linker_loader_cleanup(GArray *linker)
     return g_array_free(linker, false);
 }
 
+/*
+ * bios_linker_loader_alloc: ask guest to load file into guest memory.
+ *
+ * @linker: linker file blob array
+ * @file: file to be loaded
+ * @alloc_align: required minimal alignment in bytes. Must be a power of 2.
+ * @alloc_fseg: request allocation in FSEG zone (useful for the RSDP ACPI table)
+ *
+ * Note: this command must precede any other linker command using this file.
+ */
 void bios_linker_loader_alloc(GArray *linker,
                               const char *file,
                               uint32_t alloc_align,
@@ -105,35 +129,90 @@ void bios_linker_loader_alloc(GArray *linker,
 {
     BiosLinkerLoaderEntry entry;
 
+    assert(!(alloc_align & (alloc_align - 1)));
+
     memset(&entry, 0, sizeof entry);
     strncpy(entry.alloc.file, file, sizeof entry.alloc.file - 1);
     entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ALLOCATE);
     entry.alloc.align = cpu_to_le32(alloc_align);
-    entry.alloc.zone = cpu_to_le32(alloc_fseg ?
-                                    BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG :
-                                    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH);
+    entry.alloc.zone = alloc_fseg ? BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG :
+                                    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH;
 
     /* Alloc entries must come first, so prepend them */
     g_array_prepend_vals(linker, &entry, sizeof entry);
 }
 
+/*
+ * bios_linker_loader_add_checksum: ask guest to add checksum of file data
+ * into (same) file at the specified pointer.
+ *
+ * Checksum calculation simply sums -X for each byte X in the range
+ * using 8-bit math (i.e. ACPI checksum).
+ *
+ * @linker: linker file blob array
+ * @file: file that includes the checksum to be calculated
+ *        and the data to be checksummed
+ * @table: @file blob contents
+ * @start, @size: range of data to checksum
+ * @checksum: location of the checksum to be patched within file blob
+ *
+ * Notes:
+ * - checksum byte initial value must have been pushed into @table
+ *   and reside at address @checksum.
+ * - @size bytes must have been pushed into @table and reside at address
+ *   @start.
+ * - Guest calculates checksum of specified range of data, result is added to
+ *   initial value at @checksum into copy of @file in Guest memory.
+ * - Range might include the checksum itself.
+ * - To avoid confusion, caller must always put 0x0 at @checksum.
+ * - @file must be loaded into Guest memory using bios_linker_loader_alloc
+ */
 void bios_linker_loader_add_checksum(GArray *linker, const char *file,
-                                     void *table,
+                                     GArray *table,
                                      void *start, unsigned size,
                                      uint8_t *checksum)
 {
     BiosLinkerLoaderEntry entry;
+    ptrdiff_t checksum_offset = (gchar *)checksum - table->data;
+    ptrdiff_t start_offset = (gchar *)start - table->data;
+
+    assert(checksum_offset >= 0);
+    assert(start_offset >= 0);
+    assert(checksum_offset + 1 <= table->len);
+    assert(start_offset + size <= table->len);
+    assert(*checksum == 0x0);
 
     memset(&entry, 0, sizeof entry);
     strncpy(entry.cksum.file, file, sizeof entry.cksum.file - 1);
     entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_CHECKSUM);
-    entry.cksum.offset = cpu_to_le32(checksum - (uint8_t *)table);
-    entry.cksum.start = cpu_to_le32((uint8_t *)start - (uint8_t *)table);
+    entry.cksum.offset = cpu_to_le32(checksum_offset);
+    entry.cksum.start = cpu_to_le32(start_offset);
     entry.cksum.length = cpu_to_le32(size);
 
     g_array_append_vals(linker, &entry, sizeof entry);
 }
 
+/*
+ * bios_linker_loader_add_pointer: ask guest to add address of source file
+ * into destination file at the specified pointer.
+ *
+ * @linker: linker file blob array
+ * @dest_file: destination file that must be changed
+ * @src_file: source file who's address must be taken
+ * @table: @dest_file blob contents array
+ * @pointer: location of the pointer to be patched within destination file blob
+ * @pointer_size: size of pointer to be patched, in bytes
+ *
+ * Notes:
+ * - @pointer_size bytes must have been pushed into @table
+ *   and reside at address @pointer.
+ * - Guest address is added to initial value at @pointer
+ *   into copy of @dest_file in Guest memory.
+ *   e.g. to get start of src_file in guest memory, put 0x0 there
+ *   to get address of a field at offset 0x10 in src_file, put 0x10 there
+ * - Both @dest_file and @src_file must be
+ *   loaded into Guest memory using bios_linker_loader_alloc
+ */
 void bios_linker_loader_add_pointer(GArray *linker,
                                     const char *dest_file,
                                     const char *src_file,
@@ -141,7 +220,10 @@ void bios_linker_loader_add_pointer(GArray *linker,
                                     uint8_t pointer_size)
 {
     BiosLinkerLoaderEntry entry;
-    size_t offset = (gchar *)pointer - table->data;
+    ptrdiff_t offset = (gchar *)pointer - table->data;
+
+    assert(offset >= 0);
+    assert(offset + pointer_size <= table->len);
 
     memset(&entry, 0, sizeof entry);
     strncpy(entry.pointer.dest_file, dest_file,
@@ -149,7 +231,6 @@ void bios_linker_loader_add_pointer(GArray *linker,
     strncpy(entry.pointer.src_file, src_file,
             sizeof entry.pointer.src_file - 1);
     entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_POINTER);
-    assert(table->len >= offset + pointer_size);
     entry.pointer.offset = cpu_to_le32(offset);
     entry.pointer.size = pointer_size;
     assert(pointer_size == 1 || pointer_size == 2 ||
diff --git a/qemu/hw/acpi/core.c b/qemu/hw/acpi/core.c
index fe6215af4..6a2f45214 100644
--- a/qemu/hw/acpi/core.c
+++ b/qemu/hw/acpi/core.c
@@ -18,6 +18,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
@@ -25,7 +26,6 @@
 #include "hw/nvram/fw_cfg.h"
 #include "qemu/config-file.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
 #include "qapi-visit.h"
 #include "qapi-event.h"
 
@@ -67,7 +67,7 @@ static void acpi_register_config(void)
     qemu_add_opts(&qemu_acpi_opts);
 }
 
-machine_init(acpi_register_config);
+opts_init(acpi_register_config);
 
 static int acpi_checksum(const uint8_t *data, int len)
 {
@@ -242,7 +242,7 @@ void acpi_table_add(const QemuOpts *opts, Error **errp)
         OptsVisitor *ov;
 
         ov = opts_visitor_new(opts);
-        visit_type_AcpiTableOptions(opts_get_visitor(ov), &hdrs, NULL, &err);
+        visit_type_AcpiTableOptions(opts_get_visitor(ov), NULL, &hdrs, &err);
         opts_visitor_cleanup(ov);
     }
 
@@ -296,15 +296,7 @@ void acpi_table_add(const QemuOpts *opts, Error **errp)
 out:
     g_free(blob);
     g_strfreev(pathnames);
-
-    if (hdrs != NULL) {
-        QapiDeallocVisitor *dv;
-
-        dv = qapi_dealloc_visitor_new();
-        visit_type_AcpiTableOptions(qapi_dealloc_get_visitor(dv), &hdrs, NULL,
-                                    NULL);
-        qapi_dealloc_visitor_cleanup(dv);
-    }
+    qapi_free_AcpiTableOptions(hdrs);
 
     error_propagate(errp, err);
 }
@@ -349,6 +341,22 @@ uint8_t *acpi_table_next(uint8_t *current)
     }
 }
 
+int acpi_get_slic_oem(AcpiSlicOem *oem)
+{
+    uint8_t *u;
+
+    for (u = acpi_table_first(); u; u = acpi_table_next(u)) {
+        struct acpi_table_header *hdr = (void *)(u - sizeof(hdr->_length));
+
+        if (memcmp(hdr->sig, "SLIC", 4) == 0) {
+            oem->id = hdr->oem_id;
+            oem->table_id = hdr->oem_table_id;
+            return 0;
+        }
+    }
+    return -1;
+}
+
 static void acpi_notify_wakeup(Notifier *notifier, void *data)
 {
     ACPIREGS *ar = container_of(notifier, ACPIREGS, wakeup);
@@ -381,7 +389,7 @@ uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar)
        acpi_pm_tmr_update function uses ns for setting the timer. */
     int64_t d = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     if (d >= muldiv64(ar->tmr.overflow_time,
-                      get_ticks_per_sec(), PM_TIMER_FREQUENCY)) {
+                      NANOSECONDS_PER_SECOND, PM_TIMER_FREQUENCY)) {
         ar->pm1.evt.sts |= ACPI_BITMASK_TIMER_STATUS;
     }
     return ar->pm1.evt.sts;
@@ -475,7 +483,7 @@ void acpi_pm_tmr_update(ACPIREGS *ar, bool enable)
 
     /* schedule a timer interruption if needed */
     if (enable) {
-        expire_time = muldiv64(ar->tmr.overflow_time, get_ticks_per_sec(),
+        expire_time = muldiv64(ar->tmr.overflow_time, NANOSECONDS_PER_SECOND,
                                PM_TIMER_FREQUENCY);
         timer_mod(ar->tmr.timer, expire_time);
     } else {
@@ -528,7 +536,6 @@ void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci,
     ar->tmr.timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, acpi_pm_tmr_timer, ar);
     memory_region_init_io(&ar->tmr.io, memory_region_owner(parent),
                           &acpi_pm_tmr_ops, ar, "acpi-tmr", 4);
-    memory_region_clear_global_locking(&ar->tmr.io);
     memory_region_add_subregion(parent, 8, &ar->tmr.io);
 }
 
@@ -625,8 +632,12 @@ void acpi_pm1_cnt_reset(ACPIREGS *ar)
 void acpi_gpe_init(ACPIREGS *ar, uint8_t len)
 {
     ar->gpe.len = len;
-    ar->gpe.sts = g_malloc0(len / 2);
-    ar->gpe.en = g_malloc0(len / 2);
+    /* Only first len / 2 bytes are ever used,
+     * but the caller in ich9.c migrates full len bytes.
+     * TODO: fix ich9.c and drop the extra allocation.
+     */
+    ar->gpe.sts = g_malloc0(len);
+    ar->gpe.en = g_malloc0(len);
 }
 
 void acpi_gpe_reset(ACPIREGS *ar)
diff --git a/qemu/hw/acpi/cpu_hotplug.c b/qemu/hw/acpi/cpu_hotplug.c
index f5b9972f2..4d86743fd 100644
--- a/qemu/hw/acpi/cpu_hotplug.c
+++ b/qemu/hw/acpi/cpu_hotplug.c
@@ -9,8 +9,11 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/acpi/cpu_hotplug.h"
+#include "qapi/error.h"
+#include "qom/cpu.h"
 
 static uint64_t cpu_status_read(void *opaque, hwaddr addr, unsigned int size)
 {
diff --git a/qemu/hw/acpi/cpu_hotplug_acpi_table.c b/qemu/hw/acpi/cpu_hotplug_acpi_table.c
new file mode 100644
index 000000000..97bb1092a
--- /dev/null
+++ b/qemu/hw/acpi/cpu_hotplug_acpi_table.c
@@ -0,0 +1,136 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/acpi/cpu_hotplug.h"
+
+void build_cpu_hotplug_aml(Aml *ctx)
+{
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    Aml *sb_scope = aml_scope("_SB");
+    uint8_t madt_tmpl[8] = {0x00, 0x08, 0x00, 0x00, 0x00, 0, 0, 0};
+    Aml *cpu_id = aml_arg(0);
+    Aml *cpu_on = aml_local(0);
+    Aml *madt = aml_local(1);
+    Aml *cpus_map = aml_name(CPU_ON_BITMAP);
+    Aml *zero = aml_int(0);
+    Aml *one = aml_int(1);
+
+    /*
+     * _MAT method - creates an madt apic buffer
+     * cpu_id = Arg0 = Processor ID = Local APIC ID
+     * cpu_on = Local0 = CPON flag for this cpu
+     * madt = Local1 = Buffer (in madt apic form) to return
+     */
+    method = aml_method(CPU_MAT_METHOD, 1, AML_NOTSERIALIZED);
+    aml_append(method,
+        aml_store(aml_derefof(aml_index(cpus_map, cpu_id)), cpu_on));
+    aml_append(method,
+        aml_store(aml_buffer(sizeof(madt_tmpl), madt_tmpl), madt));
+    /* Update the processor id, lapic id, and enable/disable status */
+    aml_append(method, aml_store(cpu_id, aml_index(madt, aml_int(2))));
+    aml_append(method, aml_store(cpu_id, aml_index(madt, aml_int(3))));
+    aml_append(method, aml_store(cpu_on, aml_index(madt, aml_int(4))));
+    aml_append(method, aml_return(madt));
+    aml_append(sb_scope, method);
+
+    /*
+     * _STA method - return ON status of cpu
+     * cpu_id = Arg0 = Processor ID = Local APIC ID
+     * cpu_on = Local0 = CPON flag for this cpu
+     */
+    method = aml_method(CPU_STATUS_METHOD, 1, AML_NOTSERIALIZED);
+    aml_append(method,
+        aml_store(aml_derefof(aml_index(cpus_map, cpu_id)), cpu_on));
+    if_ctx = aml_if(cpu_on);
+    {
+        aml_append(if_ctx, aml_return(aml_int(0xF)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(zero));
+    }
+    aml_append(method, else_ctx);
+    aml_append(sb_scope, method);
+
+    method = aml_method(CPU_EJECT_METHOD, 2, AML_NOTSERIALIZED);
+    aml_append(method, aml_sleep(200));
+    aml_append(sb_scope, method);
+
+    method = aml_method(CPU_SCAN_METHOD, 0, AML_NOTSERIALIZED);
+    {
+        Aml *while_ctx, *if_ctx2, *else_ctx2;
+        Aml *bus_check_evt = aml_int(1);
+        Aml *remove_evt = aml_int(3);
+        Aml *status_map = aml_local(5); /* Local5 = active cpu bitmap */
+        Aml *byte = aml_local(2); /* Local2 = last read byte from bitmap */
+        Aml *idx = aml_local(0); /* Processor ID / APIC ID iterator */
+        Aml *is_cpu_on = aml_local(1); /* Local1 = CPON flag for cpu */
+        Aml *status = aml_local(3); /* Local3 = active state for cpu */
+
+        aml_append(method, aml_store(aml_name(CPU_STATUS_MAP), status_map));
+        aml_append(method, aml_store(zero, byte));
+        aml_append(method, aml_store(zero, idx));
+
+        /* While (idx < SizeOf(CPON)) */
+        while_ctx = aml_while(aml_lless(idx, aml_sizeof(cpus_map)));
+        aml_append(while_ctx,
+            aml_store(aml_derefof(aml_index(cpus_map, idx)), is_cpu_on));
+
+        if_ctx = aml_if(aml_and(idx, aml_int(0x07), NULL));
+        {
+            /* Shift down previously read bitmap byte */
+            aml_append(if_ctx, aml_shiftright(byte, one, byte));
+        }
+        aml_append(while_ctx, if_ctx);
+
+        else_ctx = aml_else();
+        {
+            /* Read next byte from cpu bitmap */
+            aml_append(else_ctx, aml_store(aml_derefof(aml_index(status_map,
+                       aml_shiftright(idx, aml_int(3), NULL))), byte));
+        }
+        aml_append(while_ctx, else_ctx);
+
+        aml_append(while_ctx, aml_store(aml_and(byte, one, NULL), status));
+        if_ctx = aml_if(aml_lnot(aml_equal(is_cpu_on, status)));
+        {
+            /* State change - update CPON with new state */
+            aml_append(if_ctx, aml_store(status, aml_index(cpus_map, idx)));
+            if_ctx2 = aml_if(aml_equal(status, one));
+            {
+                aml_append(if_ctx2,
+                    aml_call2(AML_NOTIFY_METHOD, idx, bus_check_evt));
+            }
+            aml_append(if_ctx, if_ctx2);
+            else_ctx2 = aml_else();
+            {
+                aml_append(else_ctx2,
+                    aml_call2(AML_NOTIFY_METHOD, idx, remove_evt));
+            }
+        }
+        aml_append(if_ctx, else_ctx2);
+        aml_append(while_ctx, if_ctx);
+
+        aml_append(while_ctx, aml_increment(idx)); /* go to next cpu */
+        aml_append(method, while_ctx);
+    }
+    aml_append(sb_scope, method);
+
+    aml_append(ctx, sb_scope);
+}
diff --git a/qemu/hw/acpi/ich9.c b/qemu/hw/acpi/ich9.c
index 1c7fcfa9d..27e978f5f 100644
--- a/qemu/hw/acpi/ich9.c
+++ b/qemu/hw/acpi/ich9.c
@@ -23,7 +23,9 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
@@ -239,7 +241,7 @@ static void pm_powerdown_req(Notifier *n, void *opaque)
 }
 
 void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
-                  bool smm_enabled, bool enable_tco,
+                  bool smm_enabled,
                   qemu_irq sci_irq)
 {
     memory_region_init(&pm->io, OBJECT(lpc_pci), "ich9-pm", ICH9_PMIO_SIZE);
@@ -263,10 +265,8 @@ void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
 
     pm->smm_enabled = smm_enabled;
 
-    pm->enable_tco = enable_tco;
-    if (pm->enable_tco) {
-        acpi_pm_tco_init(&pm->tco_regs, &pm->io);
-    }
+    pm->enable_tco = true;
+    acpi_pm_tco_init(&pm->tco_regs, &pm->io);
 
     pm->irq = sci_irq;
     qemu_register_reset(pm_reset, pm);
@@ -282,14 +282,13 @@ void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
     }
 }
 
-static void ich9_pm_get_gpe0_blk(Object *obj, Visitor *v,
-                                 void *opaque, const char *name,
-                                 Error **errp)
+static void ich9_pm_get_gpe0_blk(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     uint32_t value = pm->pm_io_base + ICH9_PMIO_GPE0_STS;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static bool ich9_pm_get_memory_hotplug_support(Object *obj, Error **errp)
@@ -307,25 +306,23 @@ static void ich9_pm_set_memory_hotplug_support(Object *obj, bool value,
     s->pm.acpi_memory_hotplug.is_enabled = value;
 }
 
-static void ich9_pm_get_disable_s3(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void ich9_pm_get_disable_s3(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     uint8_t value = pm->disable_s3;
 
-    visit_type_uint8(v, &value, name, errp);
+    visit_type_uint8(v, name, &value, errp);
 }
 
-static void ich9_pm_set_disable_s3(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void ich9_pm_set_disable_s3(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     Error *local_err = NULL;
     uint8_t value;
 
-    visit_type_uint8(v, &value, name, &local_err);
+    visit_type_uint8(v, name, &value, &local_err);
     if (local_err) {
         goto out;
     }
@@ -334,25 +331,23 @@ out:
     error_propagate(errp, local_err);
 }
 
-static void ich9_pm_get_disable_s4(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void ich9_pm_get_disable_s4(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     uint8_t value = pm->disable_s4;
 
-    visit_type_uint8(v, &value, name, errp);
+    visit_type_uint8(v, name, &value, errp);
 }
 
-static void ich9_pm_set_disable_s4(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void ich9_pm_set_disable_s4(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     Error *local_err = NULL;
     uint8_t value;
 
-    visit_type_uint8(v, &value, name, &local_err);
+    visit_type_uint8(v, name, &value, &local_err);
     if (local_err) {
         goto out;
     }
@@ -361,25 +356,23 @@ out:
     error_propagate(errp, local_err);
 }
 
-static void ich9_pm_get_s4_val(Object *obj, Visitor *v,
-                               void *opaque, const char *name,
-                               Error **errp)
+static void ich9_pm_get_s4_val(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     uint8_t value = pm->s4_val;
 
-    visit_type_uint8(v, &value, name, errp);
+    visit_type_uint8(v, name, &value, errp);
 }
 
-static void ich9_pm_set_s4_val(Object *obj, Visitor *v,
-                               void *opaque, const char *name,
-                               Error **errp)
+static void ich9_pm_set_s4_val(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     Error *local_err = NULL;
     uint8_t value;
 
-    visit_type_uint8(v, &value, name, &local_err);
+    visit_type_uint8(v, name, &value, &local_err);
     if (local_err) {
         goto out;
     }
diff --git a/qemu/hw/acpi/memory_hotplug.c b/qemu/hw/acpi/memory_hotplug.c
index 2ff0d5ce1..f65a3a21e 100644
--- a/qemu/hw/acpi/memory_hotplug.c
+++ b/qemu/hw/acpi/memory_hotplug.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/pc-hotplug.h"
 #include "hw/mem/pc-dimm.h"
@@ -155,6 +156,7 @@ static void acpi_memory_hotplug_write(void *opaque, hwaddr addr, uint64_t data,
                 qapi_event_send_mem_unplug_error(dev->id,
                                                  error_get_pretty(local_err),
                                                  &error_abort);
+                error_free(local_err);
                 break;
             }
             trace_mhp_acpi_pc_dimm_deleted(mem_st->selector);
@@ -230,6 +232,11 @@ void acpi_memory_plug_cb(ACPIREGS *ar, qemu_irq irq, MemHotplugState *mem_st,
                          DeviceState *dev, Error **errp)
 {
     MemStatus *mdev;
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
+
+    if (!dc->hotpluggable) {
+        return;
+    }
 
     mdev = acpi_memory_slot_status(mem_st, dev, errp);
     if (!mdev) {
@@ -238,11 +245,12 @@ void acpi_memory_plug_cb(ACPIREGS *ar, qemu_irq irq, MemHotplugState *mem_st,
 
     mdev->dimm = dev;
     mdev->is_enabled = true;
-    mdev->is_inserting = true;
+    if (dev->hotplugged) {
+        mdev->is_inserting = true;
 
-    /* do ACPI magic */
-    acpi_send_gpe_event(ar, irq, ACPI_MEMORY_HOTPLUG_STATUS);
-    return;
+        /* do ACPI magic */
+        acpi_send_gpe_event(ar, irq, ACPI_MEMORY_HOTPLUG_STATUS);
+    }
 }
 
 void acpi_memory_unplug_request_cb(ACPIREGS *ar, qemu_irq irq,
diff --git a/qemu/hw/acpi/memory_hotplug_acpi_table.c b/qemu/hw/acpi/memory_hotplug_acpi_table.c
new file mode 100644
index 000000000..c75660215
--- /dev/null
+++ b/qemu/hw/acpi/memory_hotplug_acpi_table.c
@@ -0,0 +1,262 @@
+/*
+ * Memory hotplug AML code of DSDT ACPI table
+ *
+ * Copyright (C) 2015 Red Hat Inc
+ *
+ * Author: Igor Mammedov <imammedo@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/acpi/memory_hotplug.h"
+#include "include/hw/acpi/pc-hotplug.h"
+#include "hw/boards.h"
+
+void build_memory_hotplug_aml(Aml *ctx, uint32_t nr_mem,
+                              uint16_t io_base, uint16_t io_len)
+{
+    Aml *ifctx;
+    Aml *method;
+    Aml *pci_scope;
+    Aml *mem_ctrl_dev;
+
+    /* scope for memory hotplug controller device node */
+    pci_scope = aml_scope("_SB.PCI0");
+    mem_ctrl_dev = aml_device(MEMORY_HOTPLUG_DEVICE);
+    {
+        Aml *one = aml_int(1);
+        Aml *zero = aml_int(0);
+        Aml *ret_val = aml_local(0);
+        Aml *slot_arg0 = aml_arg(0);
+        Aml *slots_nr = aml_name(MEMORY_SLOTS_NUMBER);
+        Aml *ctrl_lock = aml_name(MEMORY_SLOT_LOCK);
+        Aml *slot_selector = aml_name(MEMORY_SLOT_SLECTOR);
+
+        aml_append(mem_ctrl_dev, aml_name_decl("_HID", aml_string("PNP0A06")));
+        aml_append(mem_ctrl_dev,
+            aml_name_decl("_UID", aml_string("Memory hotplug resources")));
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        ifctx = aml_if(aml_equal(slots_nr, zero));
+        {
+            aml_append(ifctx, aml_return(zero));
+        }
+        aml_append(method, ifctx);
+        /* present, functioning, decoding, not shown in UI */
+        aml_append(method, aml_return(aml_int(0xB)));
+        aml_append(mem_ctrl_dev, method);
+
+        aml_append(mem_ctrl_dev, aml_mutex(MEMORY_SLOT_LOCK, 0));
+
+        method = aml_method(MEMORY_SLOT_SCAN_METHOD, 0, AML_NOTSERIALIZED);
+        {
+            Aml *else_ctx;
+            Aml *while_ctx;
+            Aml *idx = aml_local(0);
+            Aml *eject_req = aml_int(3);
+            Aml *dev_chk = aml_int(1);
+
+            ifctx = aml_if(aml_equal(slots_nr, zero));
+            {
+                aml_append(ifctx, aml_return(zero));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_store(zero, idx));
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            /* build AML that:
+             * loops over all slots and Notifies DIMMs with
+             * Device Check or Eject Request notifications if
+             * slot has corresponding status bit set and clears
+             * slot status.
+             */
+            while_ctx = aml_while(aml_lless(idx, slots_nr));
+            {
+                Aml *ins_evt = aml_name(MEMORY_SLOT_INSERT_EVENT);
+                Aml *rm_evt = aml_name(MEMORY_SLOT_REMOVE_EVENT);
+
+                aml_append(while_ctx, aml_store(idx, slot_selector));
+                ifctx = aml_if(aml_equal(ins_evt, one));
+                {
+                    aml_append(ifctx,
+                               aml_call2(MEMORY_SLOT_NOTIFY_METHOD,
+                                         idx, dev_chk));
+                    aml_append(ifctx, aml_store(one, ins_evt));
+                }
+                aml_append(while_ctx, ifctx);
+
+                else_ctx = aml_else();
+                ifctx = aml_if(aml_equal(rm_evt, one));
+                {
+                    aml_append(ifctx,
+                        aml_call2(MEMORY_SLOT_NOTIFY_METHOD,
+                                  idx, eject_req));
+                    aml_append(ifctx, aml_store(one, rm_evt));
+                }
+                aml_append(else_ctx, ifctx);
+                aml_append(while_ctx, else_ctx);
+
+                aml_append(while_ctx, aml_add(idx, one, idx));
+            }
+            aml_append(method, while_ctx);
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(one));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_STATUS_METHOD, 1, AML_NOTSERIALIZED);
+        {
+            Aml *slot_enabled = aml_name(MEMORY_SLOT_ENABLED);
+
+            aml_append(method, aml_store(zero, ret_val));
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method,
+                aml_store(aml_to_integer(slot_arg0), slot_selector));
+
+            ifctx = aml_if(aml_equal(slot_enabled, one));
+            {
+                aml_append(ifctx, aml_store(aml_int(0xF), ret_val));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(ret_val));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_CRS_METHOD, 1, AML_SERIALIZED);
+        {
+            Aml *mr64 = aml_name("MR64");
+            Aml *mr32 = aml_name("MR32");
+            Aml *crs_tmpl = aml_resource_template();
+            Aml *minl = aml_name("MINL");
+            Aml *minh = aml_name("MINH");
+            Aml *maxl =  aml_name("MAXL");
+            Aml *maxh =  aml_name("MAXH");
+            Aml *lenl = aml_name("LENL");
+            Aml *lenh = aml_name("LENH");
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+
+            aml_append(crs_tmpl,
+                aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                                 AML_CACHEABLE, AML_READ_WRITE,
+                                 0, 0x0, 0xFFFFFFFFFFFFFFFEULL, 0,
+                                 0xFFFFFFFFFFFFFFFFULL));
+            aml_append(method, aml_name_decl("MR64", crs_tmpl));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(14), "MINL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(18), "MINH"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(38), "LENL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(42), "LENH"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(22), "MAXL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(26), "MAXH"));
+
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_ADDR_HIGH), minh));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_ADDR_LOW), minl));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_SIZE_HIGH), lenh));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_SIZE_LOW), lenl));
+
+            /* 64-bit math: MAX = MIN + LEN - 1 */
+            aml_append(method, aml_add(minl, lenl, maxl));
+            aml_append(method, aml_add(minh, lenh, maxh));
+            ifctx = aml_if(aml_lless(maxl, minl));
+            {
+                aml_append(ifctx, aml_add(maxh, one, maxh));
+            }
+            aml_append(method, ifctx);
+            ifctx = aml_if(aml_lless(maxl, one));
+            {
+                aml_append(ifctx, aml_subtract(maxh, one, maxh));
+            }
+            aml_append(method, ifctx);
+            aml_append(method, aml_subtract(maxl, one, maxl));
+
+            /* return 32-bit _CRS if addr/size is in low mem */
+            /* TODO: remove it since all hotplugged DIMMs are in high mem */
+            ifctx = aml_if(aml_equal(maxh, zero));
+            {
+                crs_tmpl = aml_resource_template();
+                aml_append(crs_tmpl,
+                    aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                     AML_MAX_FIXED, AML_CACHEABLE,
+                                     AML_READ_WRITE,
+                                     0, 0x0, 0xFFFFFFFE, 0,
+                                     0xFFFFFFFF));
+                aml_append(ifctx, aml_name_decl("MR32", crs_tmpl));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(10), "MIN"));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(14), "MAX"));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(22), "LEN"));
+                aml_append(ifctx, aml_store(minl, aml_name("MIN")));
+                aml_append(ifctx, aml_store(maxl, aml_name("MAX")));
+                aml_append(ifctx, aml_store(lenl, aml_name("LEN")));
+
+                aml_append(ifctx, aml_release(ctrl_lock));
+                aml_append(ifctx, aml_return(mr32));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(mr64));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_PROXIMITY_METHOD, 1,
+                            AML_NOTSERIALIZED);
+        {
+            Aml *proximity = aml_name(MEMORY_SLOT_PROXIMITY);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(proximity, ret_val));
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(ret_val));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_OST_METHOD, 4, AML_NOTSERIALIZED);
+        {
+            Aml *ost_evt = aml_name(MEMORY_SLOT_OST_EVENT);
+            Aml *ost_status = aml_name(MEMORY_SLOT_OST_STATUS);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(aml_arg(1), ost_evt));
+            aml_append(method, aml_store(aml_arg(2), ost_status));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_EJECT_METHOD, 2, AML_NOTSERIALIZED);
+        {
+            Aml *eject = aml_name(MEMORY_SLOT_EJECT);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(one, eject));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(mem_ctrl_dev, method);
+    }
+    aml_append(pci_scope, mem_ctrl_dev);
+    aml_append(ctx, pci_scope);
+}
diff --git a/qemu/hw/acpi/nvdimm.c b/qemu/hw/acpi/nvdimm.c
new file mode 100644
index 000000000..9531340e5
--- /dev/null
+++ b/qemu/hw/acpi/nvdimm.c
@@ -0,0 +1,706 @@
+/*
+ * NVDIMM ACPI Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * NFIT is defined in ACPI 6.0: 5.2.25 NVDIMM Firmware Interface Table (NFIT)
+ * and the DSM specification can be found at:
+ *       http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf
+ *
+ * Currently, it only supports PMEM Virtualization.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "hw/acpi/acpi.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/acpi/bios-linker-loader.h"
+#include "hw/nvram/fw_cfg.h"
+#include "hw/mem/nvdimm.h"
+
+static int nvdimm_plugged_device_list(Object *obj, void *opaque)
+{
+    GSList **list = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_NVDIMM)) {
+        DeviceState *dev = DEVICE(obj);
+
+        if (dev->realized) { /* only realized NVDIMMs matter */
+            *list = g_slist_append(*list, DEVICE(obj));
+        }
+    }
+
+    object_child_foreach(obj, nvdimm_plugged_device_list, opaque);
+    return 0;
+}
+
+/*
+ * inquire plugged NVDIMM devices and link them into the list which is
+ * returned to the caller.
+ *
+ * Note: it is the caller's responsibility to free the list to avoid
+ * memory leak.
+ */
+static GSList *nvdimm_get_plugged_device_list(void)
+{
+    GSList *list = NULL;
+
+    object_child_foreach(qdev_get_machine(), nvdimm_plugged_device_list,
+                         &list);
+    return list;
+}
+
+#define NVDIMM_UUID_LE(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7)             \
+   { (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \
+     (b) & 0xff, ((b) >> 8) & 0xff, (c) & 0xff, ((c) >> 8) & 0xff,          \
+     (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }
+
+/*
+ * define Byte Addressable Persistent Memory (PM) Region according to
+ * ACPI 6.0: 5.2.25.1 System Physical Address Range Structure.
+ */
+static const uint8_t nvdimm_nfit_spa_uuid[] =
+      NVDIMM_UUID_LE(0x66f0d379, 0xb4f3, 0x4074, 0xac, 0x43, 0x0d, 0x33,
+                     0x18, 0xb7, 0x8c, 0xdb);
+
+/*
+ * NVDIMM Firmware Interface Table
+ * @signature: "NFIT"
+ *
+ * It provides information that allows OSPM to enumerate NVDIMM present in
+ * the platform and associate system physical address ranges created by the
+ * NVDIMMs.
+ *
+ * It is defined in ACPI 6.0: 5.2.25 NVDIMM Firmware Interface Table (NFIT)
+ */
+struct NvdimmNfitHeader {
+    ACPI_TABLE_HEADER_DEF
+    uint32_t reserved;
+} QEMU_PACKED;
+typedef struct NvdimmNfitHeader NvdimmNfitHeader;
+
+/*
+ * define NFIT structures according to ACPI 6.0: 5.2.25 NVDIMM Firmware
+ * Interface Table (NFIT).
+ */
+
+/*
+ * System Physical Address Range Structure
+ *
+ * It describes the system physical address ranges occupied by NVDIMMs and
+ * the types of the regions.
+ */
+struct NvdimmNfitSpa {
+    uint16_t type;
+    uint16_t length;
+    uint16_t spa_index;
+    uint16_t flags;
+    uint32_t reserved;
+    uint32_t proximity_domain;
+    uint8_t type_guid[16];
+    uint64_t spa_base;
+    uint64_t spa_length;
+    uint64_t mem_attr;
+} QEMU_PACKED;
+typedef struct NvdimmNfitSpa NvdimmNfitSpa;
+
+/*
+ * Memory Device to System Physical Address Range Mapping Structure
+ *
+ * It enables identifying each NVDIMM region and the corresponding SPA
+ * describing the memory interleave
+ */
+struct NvdimmNfitMemDev {
+    uint16_t type;
+    uint16_t length;
+    uint32_t nfit_handle;
+    uint16_t phys_id;
+    uint16_t region_id;
+    uint16_t spa_index;
+    uint16_t dcr_index;
+    uint64_t region_len;
+    uint64_t region_offset;
+    uint64_t region_dpa;
+    uint16_t interleave_index;
+    uint16_t interleave_ways;
+    uint16_t flags;
+    uint16_t reserved;
+} QEMU_PACKED;
+typedef struct NvdimmNfitMemDev NvdimmNfitMemDev;
+
+/*
+ * NVDIMM Control Region Structure
+ *
+ * It describes the NVDIMM and if applicable, Block Control Window.
+ */
+struct NvdimmNfitControlRegion {
+    uint16_t type;
+    uint16_t length;
+    uint16_t dcr_index;
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint16_t revision_id;
+    uint16_t sub_vendor_id;
+    uint16_t sub_device_id;
+    uint16_t sub_revision_id;
+    uint8_t reserved[6];
+    uint32_t serial_number;
+    uint16_t fic;
+    uint16_t num_bcw;
+    uint64_t bcw_size;
+    uint64_t cmd_offset;
+    uint64_t cmd_size;
+    uint64_t status_offset;
+    uint64_t status_size;
+    uint16_t flags;
+    uint8_t reserved2[6];
+} QEMU_PACKED;
+typedef struct NvdimmNfitControlRegion NvdimmNfitControlRegion;
+
+/*
+ * Module serial number is a unique number for each device. We use the
+ * slot id of NVDIMM device to generate this number so that each device
+ * associates with a different number.
+ *
+ * 0x123456 is a magic number we arbitrarily chose.
+ */
+static uint32_t nvdimm_slot_to_sn(int slot)
+{
+    return 0x123456 + slot;
+}
+
+/*
+ * handle is used to uniquely associate nfit_memdev structure with NVDIMM
+ * ACPI device - nfit_memdev.nfit_handle matches with the value returned
+ * by ACPI device _ADR method.
+ *
+ * We generate the handle with the slot id of NVDIMM device and reserve
+ * 0 for NVDIMM root device.
+ */
+static uint32_t nvdimm_slot_to_handle(int slot)
+{
+    return slot + 1;
+}
+
+/*
+ * index uniquely identifies the structure, 0 is reserved which indicates
+ * that the structure is not valid or the associated structure is not
+ * present.
+ *
+ * Each NVDIMM device needs two indexes, one for nfit_spa and another for
+ * nfit_dc which are generated by the slot id of NVDIMM device.
+ */
+static uint16_t nvdimm_slot_to_spa_index(int slot)
+{
+    return (slot + 1) << 1;
+}
+
+/* See the comments of nvdimm_slot_to_spa_index(). */
+static uint32_t nvdimm_slot_to_dcr_index(int slot)
+{
+    return nvdimm_slot_to_spa_index(slot) + 1;
+}
+
+/* ACPI 6.0: 5.2.25.1 System Physical Address Range Structure */
+static void
+nvdimm_build_structure_spa(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitSpa *nfit_spa;
+    uint64_t addr = object_property_get_int(OBJECT(dev), PC_DIMM_ADDR_PROP,
+                                            NULL);
+    uint64_t size = object_property_get_int(OBJECT(dev), PC_DIMM_SIZE_PROP,
+                                            NULL);
+    uint32_t node = object_property_get_int(OBJECT(dev), PC_DIMM_NODE_PROP,
+                                            NULL);
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                            NULL);
+
+    nfit_spa = acpi_data_push(structures, sizeof(*nfit_spa));
+
+    nfit_spa->type = cpu_to_le16(0 /* System Physical Address Range
+                                      Structure */);
+    nfit_spa->length = cpu_to_le16(sizeof(*nfit_spa));
+    nfit_spa->spa_index = cpu_to_le16(nvdimm_slot_to_spa_index(slot));
+
+    /*
+     * Control region is strict as all the device info, such as SN, index,
+     * is associated with slot id.
+     */
+    nfit_spa->flags = cpu_to_le16(1 /* Control region is strictly for
+                                       management during hot add/online
+                                       operation */ |
+                                  2 /* Data in Proximity Domain field is
+                                       valid*/);
+
+    /* NUMA node. */
+    nfit_spa->proximity_domain = cpu_to_le32(node);
+    /* the region reported as PMEM. */
+    memcpy(nfit_spa->type_guid, nvdimm_nfit_spa_uuid,
+           sizeof(nvdimm_nfit_spa_uuid));
+
+    nfit_spa->spa_base = cpu_to_le64(addr);
+    nfit_spa->spa_length = cpu_to_le64(size);
+
+    /* It is the PMEM and can be cached as writeback. */
+    nfit_spa->mem_attr = cpu_to_le64(0x8ULL /* EFI_MEMORY_WB */ |
+                                     0x8000ULL /* EFI_MEMORY_NV */);
+}
+
+/*
+ * ACPI 6.0: 5.2.25.2 Memory Device to System Physical Address Range Mapping
+ * Structure
+ */
+static void
+nvdimm_build_structure_memdev(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitMemDev *nfit_memdev;
+    uint64_t addr = object_property_get_int(OBJECT(dev), PC_DIMM_ADDR_PROP,
+                                            NULL);
+    uint64_t size = object_property_get_int(OBJECT(dev), PC_DIMM_SIZE_PROP,
+                                            NULL);
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                            NULL);
+    uint32_t handle = nvdimm_slot_to_handle(slot);
+
+    nfit_memdev = acpi_data_push(structures, sizeof(*nfit_memdev));
+
+    nfit_memdev->type = cpu_to_le16(1 /* Memory Device to System Address
+                                         Range Map Structure*/);
+    nfit_memdev->length = cpu_to_le16(sizeof(*nfit_memdev));
+    nfit_memdev->nfit_handle = cpu_to_le32(handle);
+
+    /*
+     * associate memory device with System Physical Address Range
+     * Structure.
+     */
+    nfit_memdev->spa_index = cpu_to_le16(nvdimm_slot_to_spa_index(slot));
+    /* associate memory device with Control Region Structure. */
+    nfit_memdev->dcr_index = cpu_to_le16(nvdimm_slot_to_dcr_index(slot));
+
+    /* The memory region on the device. */
+    nfit_memdev->region_len = cpu_to_le64(size);
+    nfit_memdev->region_dpa = cpu_to_le64(addr);
+
+    /* Only one interleave for PMEM. */
+    nfit_memdev->interleave_ways = cpu_to_le16(1);
+}
+
+/*
+ * ACPI 6.0: 5.2.25.5 NVDIMM Control Region Structure.
+ */
+static void nvdimm_build_structure_dcr(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitControlRegion *nfit_dcr;
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                       NULL);
+    uint32_t sn = nvdimm_slot_to_sn(slot);
+
+    nfit_dcr = acpi_data_push(structures, sizeof(*nfit_dcr));
+
+    nfit_dcr->type = cpu_to_le16(4 /* NVDIMM Control Region Structure */);
+    nfit_dcr->length = cpu_to_le16(sizeof(*nfit_dcr));
+    nfit_dcr->dcr_index = cpu_to_le16(nvdimm_slot_to_dcr_index(slot));
+
+    /* vendor: Intel. */
+    nfit_dcr->vendor_id = cpu_to_le16(0x8086);
+    nfit_dcr->device_id = cpu_to_le16(1);
+
+    /* The _DSM method is following Intel's DSM specification. */
+    nfit_dcr->revision_id = cpu_to_le16(1 /* Current Revision supported
+                                             in ACPI 6.0 is 1. */);
+    nfit_dcr->serial_number = cpu_to_le32(sn);
+    nfit_dcr->fic = cpu_to_le16(0x201 /* Format Interface Code. See Chapter
+                                         2: NVDIMM Device Specific Method
+                                         (DSM) in DSM Spec Rev1.*/);
+}
+
+static GArray *nvdimm_build_device_structure(GSList *device_list)
+{
+    GArray *structures = g_array_new(false, true /* clear */, 1);
+
+    for (; device_list; device_list = device_list->next) {
+        DeviceState *dev = device_list->data;
+
+        /* build System Physical Address Range Structure. */
+        nvdimm_build_structure_spa(structures, dev);
+
+        /*
+         * build Memory Device to System Physical Address Range Mapping
+         * Structure.
+         */
+        nvdimm_build_structure_memdev(structures, dev);
+
+        /* build NVDIMM Control Region Structure. */
+        nvdimm_build_structure_dcr(structures, dev);
+    }
+
+    return structures;
+}
+
+static void nvdimm_build_nfit(GSList *device_list, GArray *table_offsets,
+                              GArray *table_data, GArray *linker)
+{
+    GArray *structures = nvdimm_build_device_structure(device_list);
+    unsigned int header;
+
+    acpi_add_table(table_offsets, table_data);
+
+    /* NFIT header. */
+    header = table_data->len;
+    acpi_data_push(table_data, sizeof(NvdimmNfitHeader));
+    /* NVDIMM device structures. */
+    g_array_append_vals(table_data, structures->data, structures->len);
+
+    build_header(linker, table_data,
+                 (void *)(table_data->data + header), "NFIT",
+                 sizeof(NvdimmNfitHeader) + structures->len, 1, NULL, NULL);
+    g_array_free(structures, true);
+}
+
+struct NvdimmDsmIn {
+    uint32_t handle;
+    uint32_t revision;
+    uint32_t function;
+    /* the remaining size in the page is used by arg3. */
+    union {
+        uint8_t arg3[0];
+    };
+} QEMU_PACKED;
+typedef struct NvdimmDsmIn NvdimmDsmIn;
+
+struct NvdimmDsmOut {
+    /* the size of buffer filled by QEMU. */
+    uint32_t len;
+    uint8_t data[0];
+} QEMU_PACKED;
+typedef struct NvdimmDsmOut NvdimmDsmOut;
+
+struct NvdimmDsmFunc0Out {
+    /* the size of buffer filled by QEMU. */
+     uint32_t len;
+     uint32_t supported_func;
+} QEMU_PACKED;
+typedef struct NvdimmDsmFunc0Out NvdimmDsmFunc0Out;
+
+struct NvdimmDsmFuncNoPayloadOut {
+    /* the size of buffer filled by QEMU. */
+     uint32_t len;
+     uint32_t func_ret_status;
+} QEMU_PACKED;
+typedef struct NvdimmDsmFuncNoPayloadOut NvdimmDsmFuncNoPayloadOut;
+
+static uint64_t
+nvdimm_dsm_read(void *opaque, hwaddr addr, unsigned size)
+{
+    nvdimm_debug("BUG: we never read _DSM IO Port.\n");
+    return 0;
+}
+
+static void
+nvdimm_dsm_write(void *opaque, hwaddr addr, uint64_t val, unsigned size)
+{
+    NvdimmDsmIn *in;
+    hwaddr dsm_mem_addr = val;
+
+    nvdimm_debug("dsm memory address %#" HWADDR_PRIx ".\n", dsm_mem_addr);
+
+    /*
+     * The DSM memory is mapped to guest address space so an evil guest
+     * can change its content while we are doing DSM emulation. Avoid
+     * this by copying DSM memory to QEMU local memory.
+     */
+    in = g_malloc(TARGET_PAGE_SIZE);
+    cpu_physical_memory_read(dsm_mem_addr, in, TARGET_PAGE_SIZE);
+
+    le32_to_cpus(&in->revision);
+    le32_to_cpus(&in->function);
+    le32_to_cpus(&in->handle);
+
+    nvdimm_debug("Revision %#x Handler %#x Function %#x.\n", in->revision,
+                 in->handle, in->function);
+
+    /*
+     * function 0 is called to inquire which functions are supported by
+     * OSPM
+     */
+    if (in->function == 0) {
+        NvdimmDsmFunc0Out func0 = {
+            .len = cpu_to_le32(sizeof(func0)),
+             /* No function supported other than function 0 */
+            .supported_func = cpu_to_le32(0),
+        };
+        cpu_physical_memory_write(dsm_mem_addr, &func0, sizeof func0);
+    } else {
+        /* No function except function 0 is supported yet. */
+        NvdimmDsmFuncNoPayloadOut out = {
+            .len = cpu_to_le32(sizeof(out)),
+            .func_ret_status = cpu_to_le32(1)  /* Not Supported */,
+        };
+        cpu_physical_memory_write(dsm_mem_addr, &out, sizeof(out));
+    }
+
+    g_free(in);
+}
+
+static const MemoryRegionOps nvdimm_dsm_ops = {
+    .read = nvdimm_dsm_read,
+    .write = nvdimm_dsm_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+void nvdimm_init_acpi_state(AcpiNVDIMMState *state, MemoryRegion *io,
+                            FWCfgState *fw_cfg, Object *owner)
+{
+    memory_region_init_io(&state->io_mr, owner, &nvdimm_dsm_ops, state,
+                          "nvdimm-acpi-io", NVDIMM_ACPI_IO_LEN);
+    memory_region_add_subregion(io, NVDIMM_ACPI_IO_BASE, &state->io_mr);
+
+    state->dsm_mem = g_array_new(false, true /* clear */, 1);
+    acpi_data_push(state->dsm_mem, TARGET_PAGE_SIZE);
+    fw_cfg_add_file(fw_cfg, NVDIMM_DSM_MEM_FILE, state->dsm_mem->data,
+                    state->dsm_mem->len);
+}
+
+#define NVDIMM_COMMON_DSM      "NCAL"
+#define NVDIMM_ACPI_MEM_ADDR   "MEMA"
+
+static void nvdimm_build_common_dsm(Aml *dev)
+{
+    Aml *method, *ifctx, *function, *dsm_mem, *unpatched, *result_size;
+    uint8_t byte_list[1];
+
+    method = aml_method(NVDIMM_COMMON_DSM, 4, AML_SERIALIZED);
+    function = aml_arg(2);
+    dsm_mem = aml_name(NVDIMM_ACPI_MEM_ADDR);
+
+    /*
+     * do not support any method if DSM memory address has not been
+     * patched.
+     */
+    unpatched = aml_if(aml_equal(dsm_mem, aml_int(0x0)));
+
+    /*
+     * function 0 is called to inquire what functions are supported by
+     * OSPM
+     */
+    ifctx = aml_if(aml_equal(function, aml_int(0)));
+    byte_list[0] = 0 /* No function Supported */;
+    aml_append(ifctx, aml_return(aml_buffer(1, byte_list)));
+    aml_append(unpatched, ifctx);
+
+    /* No function is supported yet. */
+    byte_list[0] = 1 /* Not Supported */;
+    aml_append(unpatched, aml_return(aml_buffer(1, byte_list)));
+    aml_append(method, unpatched);
+
+    /*
+     * The HDLE indicates the DSM function is issued from which device,
+     * it is not used at this time as no function is supported yet.
+     * Currently we make it always be 0 for all the devices and will set
+     * the appropriate value once real function is implemented.
+     */
+    aml_append(method, aml_store(aml_int(0x0), aml_name("HDLE")));
+    aml_append(method, aml_store(aml_arg(1), aml_name("REVS")));
+    aml_append(method, aml_store(aml_arg(2), aml_name("FUNC")));
+
+    /*
+     * tell QEMU about the real address of DSM memory, then QEMU
+     * gets the control and fills the result in DSM memory.
+     */
+    aml_append(method, aml_store(dsm_mem, aml_name("NTFI")));
+
+    result_size = aml_local(1);
+    aml_append(method, aml_store(aml_name("RLEN"), result_size));
+    aml_append(method, aml_store(aml_shiftleft(result_size, aml_int(3)),
+                                 result_size));
+    aml_append(method, aml_create_field(aml_name("ODAT"), aml_int(0),
+                                        result_size, "OBUF"));
+    aml_append(method, aml_concatenate(aml_buffer(0, NULL), aml_name("OBUF"),
+                                       aml_arg(6)));
+    aml_append(method, aml_return(aml_arg(6)));
+    aml_append(dev, method);
+}
+
+static void nvdimm_build_device_dsm(Aml *dev)
+{
+    Aml *method;
+
+    method = aml_method("_DSM", 4, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call4(NVDIMM_COMMON_DSM, aml_arg(0),
+                                  aml_arg(1), aml_arg(2), aml_arg(3))));
+    aml_append(dev, method);
+}
+
+static void nvdimm_build_nvdimm_devices(GSList *device_list, Aml *root_dev)
+{
+    for (; device_list; device_list = device_list->next) {
+        DeviceState *dev = device_list->data;
+        int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                           NULL);
+        uint32_t handle = nvdimm_slot_to_handle(slot);
+        Aml *nvdimm_dev;
+
+        nvdimm_dev = aml_device("NV%02X", slot);
+
+        /*
+         * ACPI 6.0: 9.20 NVDIMM Devices:
+         *
+         * _ADR object that is used to supply OSPM with unique address
+         * of the NVDIMM device. This is done by returning the NFIT Device
+         * handle that is used to identify the associated entries in ACPI
+         * table NFIT or _FIT.
+         */
+        aml_append(nvdimm_dev, aml_name_decl("_ADR", aml_int(handle)));
+
+        nvdimm_build_device_dsm(nvdimm_dev);
+        aml_append(root_dev, nvdimm_dev);
+    }
+}
+
+static void nvdimm_build_ssdt(GSList *device_list, GArray *table_offsets,
+                              GArray *table_data, GArray *linker)
+{
+    Aml *ssdt, *sb_scope, *dev, *field;
+    int mem_addr_offset, nvdimm_ssdt;
+
+    acpi_add_table(table_offsets, table_data);
+
+    ssdt = init_aml_allocator();
+    acpi_data_push(ssdt->buf, sizeof(AcpiTableHeader));
+
+    sb_scope = aml_scope("\\_SB");
+
+    dev = aml_device("NVDR");
+
+    /*
+     * ACPI 6.0: 9.20 NVDIMM Devices:
+     *
+     * The ACPI Name Space device uses _HID of ACPI0012 to identify the root
+     * NVDIMM interface device. Platform firmware is required to contain one
+     * such device in _SB scope if NVDIMMs support is exposed by platform to
+     * OSPM.
+     * For each NVDIMM present or intended to be supported by platform,
+     * platform firmware also exposes an ACPI Namespace Device under the
+     * root device.
+     */
+    aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0012")));
+
+    /* map DSM memory and IO into ACPI namespace. */
+    aml_append(dev, aml_operation_region("NPIO", AML_SYSTEM_IO,
+               aml_int(NVDIMM_ACPI_IO_BASE), NVDIMM_ACPI_IO_LEN));
+    aml_append(dev, aml_operation_region("NRAM", AML_SYSTEM_MEMORY,
+               aml_name(NVDIMM_ACPI_MEM_ADDR), TARGET_PAGE_SIZE));
+
+    /*
+     * DSM notifier:
+     * NTFI: write the address of DSM memory and notify QEMU to emulate
+     *       the access.
+     *
+     * It is the IO port so that accessing them will cause VM-exit, the
+     * control will be transferred to QEMU.
+     */
+    field = aml_field("NPIO", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("NTFI",
+               sizeof(uint32_t) * BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    /*
+     * DSM input:
+     * HDLE: store device's handle, it's zero if the _DSM call happens
+     *       on NVDIMM Root Device.
+     * REVS: store the Arg1 of _DSM call.
+     * FUNC: store the Arg2 of _DSM call.
+     * ARG3: store the Arg3 of _DSM call.
+     *
+     * They are RAM mapping on host so that these accesses never cause
+     * VM-EXIT.
+     */
+    field = aml_field("NRAM", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("HDLE",
+               sizeof(typeof_field(NvdimmDsmIn, handle)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("REVS",
+               sizeof(typeof_field(NvdimmDsmIn, revision)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("FUNC",
+               sizeof(typeof_field(NvdimmDsmIn, function)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("ARG3",
+               (TARGET_PAGE_SIZE - offsetof(NvdimmDsmIn, arg3)) *
+                BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    /*
+     * DSM output:
+     * RLEN: the size of the buffer filled by QEMU.
+     * ODAT: the buffer QEMU uses to store the result.
+     *
+     * Since the page is reused by both input and out, the input data
+     * will be lost after storing new result into ODAT so we should fetch
+     * all the input data before writing the result.
+     */
+    field = aml_field("NRAM", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("RLEN",
+               sizeof(typeof_field(NvdimmDsmOut, len)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("ODAT",
+               (TARGET_PAGE_SIZE - offsetof(NvdimmDsmOut, data)) *
+                     BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    nvdimm_build_common_dsm(dev);
+    nvdimm_build_device_dsm(dev);
+
+    nvdimm_build_nvdimm_devices(device_list, dev);
+
+    aml_append(sb_scope, dev);
+    aml_append(ssdt, sb_scope);
+
+    nvdimm_ssdt = table_data->len;
+
+    /* copy AML table into ACPI tables blob and patch header there */
+    g_array_append_vals(table_data, ssdt->buf->data, ssdt->buf->len);
+    mem_addr_offset = build_append_named_dword(table_data,
+                                               NVDIMM_ACPI_MEM_ADDR);
+
+    bios_linker_loader_alloc(linker, NVDIMM_DSM_MEM_FILE, TARGET_PAGE_SIZE,
+                             false /* high memory */);
+    bios_linker_loader_add_pointer(linker, ACPI_BUILD_TABLE_FILE,
+                                   NVDIMM_DSM_MEM_FILE, table_data,
+                                   table_data->data + mem_addr_offset,
+                                   sizeof(uint32_t));
+    build_header(linker, table_data,
+        (void *)(table_data->data + nvdimm_ssdt),
+        "SSDT", table_data->len - nvdimm_ssdt, 1, NULL, "NVDIMM");
+    free_aml_allocator();
+}
+
+void nvdimm_build_acpi(GArray *table_offsets, GArray *table_data,
+                       GArray *linker)
+{
+    GSList *device_list;
+
+    /* no NVDIMM device is plugged. */
+    device_list = nvdimm_get_plugged_device_list();
+    if (!device_list) {
+        return;
+    }
+    nvdimm_build_nfit(device_list, table_offsets, table_data, linker);
+    nvdimm_build_ssdt(device_list, table_offsets, table_data, linker);
+    g_slist_free(device_list);
+}
diff --git a/qemu/hw/acpi/pcihp.c b/qemu/hw/acpi/pcihp.c
index fbbc4dde4..71f4c4e14 100644
--- a/qemu/hw/acpi/pcihp.c
+++ b/qemu/hw/acpi/pcihp.c
@@ -24,6 +24,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/acpi/pcihp.h"
 
 #include "hw/hw.h"
@@ -34,6 +35,7 @@
 #include "exec/ioport.h"
 #include "exec/address-spaces.h"
 #include "hw/pci/pci_bus.h"
+#include "qapi/error.h"
 #include "qom/qom-qobject.h"
 #include "qapi/qmp/qint.h"
 
diff --git a/qemu/hw/acpi/piix4.c b/qemu/hw/acpi/piix4.c
index 2cd2fee89..16abdf162 100644
--- a/qemu/hw/acpi/piix4.c
+++ b/qemu/hw/acpi/piix4.c
@@ -18,6 +18,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/apm.h"
@@ -25,6 +26,7 @@
 #include "hw/pci/pci.h"
 #include "hw/acpi/acpi.h"
 #include "sysemu/sysemu.h"
+#include "qapi/error.h"
 #include "qemu/range.h"
 #include "exec/ioport.h"
 #include "hw/nvram/fw_cfg.h"
diff --git a/qemu/hw/acpi/tco.c b/qemu/hw/acpi/tco.c
index 7a026c255..8ce7daf23 100644
--- a/qemu/hw/acpi/tco.c
+++ b/qemu/hw/acpi/tco.c
@@ -6,6 +6,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/watchdog.h"
 #include "hw/i386/ich9.h"
diff --git a/qemu/hw/alpha/dp264.c b/qemu/hw/alpha/dp264.c
index f86e7bb83..f1267b544 100644
--- a/qemu/hw/alpha/dp264.c
+++ b/qemu/hw/alpha/dp264.c
@@ -6,16 +6,21 @@
  * that we need to emulate as well.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "elf.h"
 #include "hw/loader.h"
 #include "hw/boards.h"
 #include "alpha_sys.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/mc146818rtc.h"
 #include "hw/ide.h"
 #include "hw/timer/i8254.h"
 #include "hw/char/serial.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 
@@ -104,14 +109,14 @@ static void clipper_init(MachineState *machine)
     palcode_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS,
                                 bios_name ? bios_name : "palcode-clipper");
     if (palcode_filename == NULL) {
-        hw_error("no palcode provided\n");
+        error_report("no palcode provided");
         exit(1);
     }
     size = load_elf(palcode_filename, cpu_alpha_superpage_to_phys,
                     NULL, &palcode_entry, &palcode_low, &palcode_high,
-                    0, EM_ALPHA, 0);
+                    0, EM_ALPHA, 0, 0);
     if (size < 0) {
-        hw_error("could not load palcode '%s'\n", palcode_filename);
+        error_report("could not load palcode '%s'", palcode_filename);
         exit(1);
     }
     g_free(palcode_filename);
@@ -129,9 +134,9 @@ static void clipper_init(MachineState *machine)
 
         size = load_elf(kernel_filename, cpu_alpha_superpage_to_phys,
                         NULL, &kernel_entry, &kernel_low, &kernel_high,
-                        0, EM_ALPHA, 0);
+                        0, EM_ALPHA, 0, 0);
         if (size < 0) {
-            hw_error("could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
 
@@ -148,8 +153,8 @@ static void clipper_init(MachineState *machine)
 
             initrd_size = get_image_size(initrd_filename);
             if (initrd_size < 0) {
-                hw_error("could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
 
@@ -168,17 +173,12 @@ static void clipper_init(MachineState *machine)
     }
 }
 
-static QEMUMachine clipper_machine = {
-    .name = "clipper",
-    .desc = "Alpha DP264/CLIPPER",
-    .init = clipper_init,
-    .max_cpus = 4,
-    .is_default = 1,
-};
-
-static void clipper_machine_init(void)
+static void clipper_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&clipper_machine);
+    mc->desc = "Alpha DP264/CLIPPER";
+    mc->init = clipper_init;
+    mc->max_cpus = 4;
+    mc->is_default = 1;
 }
 
-machine_init(clipper_machine_init);
+DEFINE_MACHINE("clipper", clipper_machine_init)
diff --git a/qemu/hw/alpha/pci.c b/qemu/hw/alpha/pci.c
index d839dd556..5baa0eaf1 100644
--- a/qemu/hw/alpha/pci.c
+++ b/qemu/hw/alpha/pci.c
@@ -6,10 +6,13 @@
  * ??? Sparse memory access not implemented.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "alpha_sys.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
+#include "trace.h"
 
 
 /* Fallback for unassigned PCI I/O operations.  Avoids MCHK.  */
@@ -73,7 +76,7 @@ static uint64_t iack_read(void *opaque, hwaddr addr, unsigned size)
 static void special_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
-    qemu_log("pci: special write cycle");
+    trace_alpha_pci_iack_write();
 }
 
 const MemoryRegionOps alpha_pci_iack_ops = {
diff --git a/qemu/hw/alpha/typhoon.c b/qemu/hw/alpha/typhoon.c
index 421162e1d..97721b535 100644
--- a/qemu/hw/alpha/typhoon.c
+++ b/qemu/hw/alpha/typhoon.c
@@ -6,6 +6,8 @@
  * This work is licensed under the GNU GPL license version 2 or later.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
@@ -920,7 +922,8 @@ PCIBus *typhoon_init(ram_addr_t ram_size, ISABus **isa_bus,
     {
         qemu_irq *isa_irqs;
 
-        *isa_bus = isa_bus_new(NULL, get_system_memory(), &s->pchip.reg_io);
+        *isa_bus = isa_bus_new(NULL, get_system_memory(), &s->pchip.reg_io,
+                               &error_abort);
         isa_irqs = i8259_init(*isa_bus,
                               qemu_allocate_irq(typhoon_set_isa_irq, s, 0));
         isa_bus_irqs(*isa_bus, isa_irqs);
diff --git a/qemu/hw/arm/Makefile.objs b/qemu/hw/arm/Makefile.objs
index cf346c1d0..954c9fe15 100644
--- a/qemu/hw/arm/Makefile.objs
+++ b/qemu/hw/arm/Makefile.objs
@@ -1,6 +1,6 @@
 obj-y += boot.o collie.o exynos4_boards.o gumstix.o highbank.o
 obj-$(CONFIG_DIGIC) += digic_boards.o
-obj-y += integratorcp.o kzm.o mainstone.o musicpal.o nseries.o
+obj-y += integratorcp.o mainstone.o musicpal.o nseries.o
 obj-y += omap_sx1.o palm.o realview.o spitz.o stellaris.o
 obj-y += tosa.o versatilepb.o vexpress.o virt.o xilinx_zynq.o z2.o
 obj-$(CONFIG_ACPI) += virt-acpi-build.o
@@ -11,5 +11,9 @@ obj-y += armv7m.o exynos4210.o pxa2xx.o pxa2xx_gpio.o pxa2xx_pic.o
 obj-$(CONFIG_DIGIC) += digic.o
 obj-y += omap1.o omap2.o strongarm.o
 obj-$(CONFIG_ALLWINNER_A10) += allwinner-a10.o cubieboard.o
+obj-$(CONFIG_RASPI) += bcm2835_peripherals.o bcm2836.o raspi.o
 obj-$(CONFIG_STM32F205_SOC) += stm32f205_soc.o
 obj-$(CONFIG_XLNX_ZYNQMP) += xlnx-zynqmp.o xlnx-ep108.o
+obj-$(CONFIG_FSL_IMX25) += fsl-imx25.o imx25_pdk.o
+obj-$(CONFIG_FSL_IMX31) += fsl-imx31.o kzm.o
+obj-$(CONFIG_ASPEED_SOC) += ast2400.o palmetto-bmc.o
diff --git a/qemu/hw/arm/allwinner-a10.c b/qemu/hw/arm/allwinner-a10.c
index ff249af33..ca15d1c8c 100644
--- a/qemu/hw/arm/allwinner-a10.c
+++ b/qemu/hw/arm/allwinner-a10.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/arm/allwinner-a10.h"
@@ -39,6 +43,9 @@ static void aw_a10_init(Object *obj)
         qemu_check_nic_model(&nd_table[0], TYPE_AW_EMAC);
         qdev_set_nic_properties(DEVICE(&s->emac), &nd_table[0]);
     }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_ALLWINNER_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
 }
 
 static void aw_a10_realize(DeviceState *dev, Error **errp)
@@ -93,6 +100,14 @@ static void aw_a10_realize(DeviceState *dev, Error **errp)
     sysbus_mmio_map(sysbusdev, 0, AW_A10_EMAC_BASE);
     sysbus_connect_irq(sysbusdev, 0, s->irq[55]);
 
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, AW_A10_SATA_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, s->irq[56]);
+
     /* FIXME use a qdev chardev prop instead of serial_hds[] */
     serial_mm_init(get_system_memory(), AW_A10_UART0_REG_BASE, 2, s->irq[1],
                    115200, serial_hds[0], DEVICE_NATIVE_ENDIAN);
@@ -103,6 +118,12 @@ static void aw_a10_class_init(ObjectClass *oc, void *data)
     DeviceClass *dc = DEVICE_CLASS(oc);
 
     dc->realize = aw_a10_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo aw_a10_type_info = {
diff --git a/qemu/hw/arm/armv7m.c b/qemu/hw/arm/armv7m.c
index c6eab6de3..bb2a22d96 100644
--- a/qemu/hw/arm/armv7m.c
+++ b/qemu/hw/arm/armv7m.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/loader.h"
@@ -166,17 +170,15 @@ static void armv7m_reset(void *opaque)
    mem_size is in bytes.
    Returns the NVIC array.  */
 
-qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model)
 {
     ARMCPU *cpu;
     CPUARMState *env;
     DeviceState *nvic;
-    qemu_irq *pic = g_new(qemu_irq, num_irq);
     int image_size;
     uint64_t entry;
     uint64_t lowaddr;
-    int i;
     int big_endian;
     MemoryRegion *hack = g_new(MemoryRegion, 1);
 
@@ -198,9 +200,6 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
     qdev_init_nofail(nvic);
     sysbus_connect_irq(SYS_BUS_DEVICE(nvic), 0,
                        qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ));
-    for (i = 0; i < num_irq; i++) {
-        pic[i] = qdev_get_gpio_in(nvic, i);
-    }
 
 #ifdef TARGET_WORDS_BIGENDIAN
     big_endian = 1;
@@ -215,7 +214,7 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
 
     if (kernel_filename) {
         image_size = load_elf(kernel_filename, NULL, NULL, &entry, &lowaddr,
-                              NULL, big_endian, ELF_MACHINE, 1);
+                              NULL, big_endian, EM_ARM, 1, 0);
         if (image_size < 0) {
             image_size = load_image_targphys(kernel_filename, 0, mem_size);
             lowaddr = 0;
@@ -229,12 +228,12 @@ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
     /* Hack to map an additional page of ram at the top of the address
        space.  This stops qemu complaining about executing code outside RAM
        when returning from an exception.  */
-    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_abort);
+    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_fatal);
     vmstate_register_ram_global(hack);
     memory_region_add_subregion(system_memory, 0xfffff000, hack);
 
     qemu_register_reset(armv7m_reset, cpu);
-    return pic;
+    return nvic;
 }
 
 static Property bitband_properties[] = {
diff --git a/qemu/hw/arm/ast2400.c b/qemu/hw/arm/ast2400.c
new file mode 100644
index 000000000..03f993863
--- /dev/null
+++ b/qemu/hw/arm/ast2400.c
@@ -0,0 +1,140 @@
+/*
+ * AST2400 SoC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ * Jeremy Kerr <jk@ozlabs.org>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/address-spaces.h"
+#include "hw/arm/ast2400.h"
+#include "hw/char/serial.h"
+
+#define AST2400_UART_5_BASE      0x00184000
+#define AST2400_IOMEM_SIZE       0x00200000
+#define AST2400_IOMEM_BASE       0x1E600000
+#define AST2400_VIC_BASE         0x1E6C0000
+#define AST2400_TIMER_BASE       0x1E782000
+
+static const int uart_irqs[] = { 9, 32, 33, 34, 10 };
+static const int timer_irqs[] = { 16, 17, 18, 35, 36, 37, 38, 39, };
+
+/*
+ * IO handlers: simply catch any reads/writes to IO addresses that aren't
+ * handled by a device mapping.
+ */
+
+static uint64_t ast2400_io_read(void *p, hwaddr offset, unsigned size)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: 0x%" HWADDR_PRIx " [%u]\n",
+                  __func__, offset, size);
+    return 0;
+}
+
+static void ast2400_io_write(void *opaque, hwaddr offset, uint64_t value,
+                unsigned size)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: 0x%" HWADDR_PRIx " <- 0x%" PRIx64 " [%u]\n",
+                  __func__, offset, value, size);
+}
+
+static const MemoryRegionOps ast2400_io_ops = {
+    .read = ast2400_io_read,
+    .write = ast2400_io_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ast2400_init(Object *obj)
+{
+    AST2400State *s = AST2400(obj);
+
+    s->cpu = cpu_arm_init("arm926");
+
+    object_initialize(&s->vic, sizeof(s->vic), TYPE_ASPEED_VIC);
+    object_property_add_child(obj, "vic", OBJECT(&s->vic), NULL);
+    qdev_set_parent_bus(DEVICE(&s->vic), sysbus_get_default());
+
+    object_initialize(&s->timerctrl, sizeof(s->timerctrl), TYPE_ASPEED_TIMER);
+    object_property_add_child(obj, "timerctrl", OBJECT(&s->timerctrl), NULL);
+    qdev_set_parent_bus(DEVICE(&s->timerctrl), sysbus_get_default());
+}
+
+static void ast2400_realize(DeviceState *dev, Error **errp)
+{
+    int i;
+    AST2400State *s = AST2400(dev);
+    Error *err = NULL;
+
+    /* IO space */
+    memory_region_init_io(&s->iomem, NULL, &ast2400_io_ops, NULL,
+            "ast2400.io", AST2400_IOMEM_SIZE);
+    memory_region_add_subregion_overlap(get_system_memory(), AST2400_IOMEM_BASE,
+            &s->iomem, -1);
+
+    /* VIC */
+    object_property_set_bool(OBJECT(&s->vic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->vic), 0, AST2400_VIC_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->vic), 0,
+                       qdev_get_gpio_in(DEVICE(s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->vic), 1,
+                       qdev_get_gpio_in(DEVICE(s->cpu), ARM_CPU_FIQ));
+
+    /* Timer */
+    object_property_set_bool(OBJECT(&s->timerctrl), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->timerctrl), 0, AST2400_TIMER_BASE);
+    for (i = 0; i < ARRAY_SIZE(timer_irqs); i++) {
+        qemu_irq irq = qdev_get_gpio_in(DEVICE(&s->vic), timer_irqs[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->timerctrl), i, irq);
+    }
+
+    /* UART - attach an 8250 to the IO space as our UART5 */
+    if (serial_hds[0]) {
+        qemu_irq uart5 = qdev_get_gpio_in(DEVICE(&s->vic), uart_irqs[4]);
+        serial_mm_init(&s->iomem, AST2400_UART_5_BASE, 2,
+                       uart5, 38400, serial_hds[0], DEVICE_LITTLE_ENDIAN);
+    }
+}
+
+static void ast2400_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = ast2400_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo ast2400_type_info = {
+    .name = TYPE_AST2400,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AST2400State),
+    .instance_init = ast2400_init,
+    .class_init = ast2400_class_init,
+};
+
+static void ast2400_register_types(void)
+{
+    type_register_static(&ast2400_type_info);
+}
+
+type_init(ast2400_register_types)
diff --git a/qemu/hw/arm/bcm2835_peripherals.c b/qemu/hw/arm/bcm2835_peripherals.c
new file mode 100644
index 000000000..234d51843
--- /dev/null
+++ b/qemu/hw/arm/bcm2835_peripherals.c
@@ -0,0 +1,312 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/arm/bcm2835_peripherals.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+#include "hw/arm/raspi_platform.h"
+#include "sysemu/char.h"
+
+/* Peripheral base address on the VC (GPU) system bus */
+#define BCM2835_VC_PERI_BASE 0x7e000000
+
+/* Capabilities for SD controller: no DMA, high-speed, default clocks etc. */
+#define BCM2835_SDHC_CAPAREG 0x52034b4
+
+static void bcm2835_peripherals_init(Object *obj)
+{
+    BCM2835PeripheralState *s = BCM2835_PERIPHERALS(obj);
+
+    /* Memory region for peripheral devices, which we export to our parent */
+    memory_region_init(&s->peri_mr, obj,"bcm2835-peripherals", 0x1000000);
+    object_property_add_child(obj, "peripheral-io", OBJECT(&s->peri_mr), NULL);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->peri_mr);
+
+    /* Internal memory region for peripheral bus addresses (not exported) */
+    memory_region_init(&s->gpu_bus_mr, obj, "bcm2835-gpu", (uint64_t)1 << 32);
+    object_property_add_child(obj, "gpu-bus", OBJECT(&s->gpu_bus_mr), NULL);
+
+    /* Internal memory region for request/response communication with
+     * mailbox-addressable peripherals (not exported)
+     */
+    memory_region_init(&s->mbox_mr, obj, "bcm2835-mbox",
+                       MBOX_CHAN_COUNT << MBOX_AS_CHAN_SHIFT);
+
+    /* Interrupt Controller */
+    object_initialize(&s->ic, sizeof(s->ic), TYPE_BCM2835_IC);
+    object_property_add_child(obj, "ic", OBJECT(&s->ic), NULL);
+    qdev_set_parent_bus(DEVICE(&s->ic), sysbus_get_default());
+
+    /* UART0 */
+    s->uart0 = SYS_BUS_DEVICE(object_new("pl011"));
+    object_property_add_child(obj, "uart0", OBJECT(s->uart0), NULL);
+    qdev_set_parent_bus(DEVICE(s->uart0), sysbus_get_default());
+
+    /* AUX / UART1 */
+    object_initialize(&s->aux, sizeof(s->aux), TYPE_BCM2835_AUX);
+    object_property_add_child(obj, "aux", OBJECT(&s->aux), NULL);
+    qdev_set_parent_bus(DEVICE(&s->aux), sysbus_get_default());
+
+    /* Mailboxes */
+    object_initialize(&s->mboxes, sizeof(s->mboxes), TYPE_BCM2835_MBOX);
+    object_property_add_child(obj, "mbox", OBJECT(&s->mboxes), NULL);
+    qdev_set_parent_bus(DEVICE(&s->mboxes), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->mboxes), "mbox-mr",
+                                   OBJECT(&s->mbox_mr), &error_abort);
+
+    /* Framebuffer */
+    object_initialize(&s->fb, sizeof(s->fb), TYPE_BCM2835_FB);
+    object_property_add_child(obj, "fb", OBJECT(&s->fb), NULL);
+    object_property_add_alias(obj, "vcram-size", OBJECT(&s->fb), "vcram-size",
+                              &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->fb), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->fb), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+
+    /* Property channel */
+    object_initialize(&s->property, sizeof(s->property), TYPE_BCM2835_PROPERTY);
+    object_property_add_child(obj, "property", OBJECT(&s->property), NULL);
+    object_property_add_alias(obj, "board-rev", OBJECT(&s->property),
+                              "board-rev", &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->property), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->property), "fb",
+                                   OBJECT(&s->fb), &error_abort);
+    object_property_add_const_link(OBJECT(&s->property), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+
+    /* Extended Mass Media Controller */
+    object_initialize(&s->sdhci, sizeof(s->sdhci), TYPE_SYSBUS_SDHCI);
+    object_property_add_child(obj, "sdhci", OBJECT(&s->sdhci), NULL);
+    qdev_set_parent_bus(DEVICE(&s->sdhci), sysbus_get_default());
+
+    /* DMA Channels */
+    object_initialize(&s->dma, sizeof(s->dma), TYPE_BCM2835_DMA);
+    object_property_add_child(obj, "dma", OBJECT(&s->dma), NULL);
+    qdev_set_parent_bus(DEVICE(&s->dma), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->dma), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+}
+
+static void bcm2835_peripherals_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835PeripheralState *s = BCM2835_PERIPHERALS(dev);
+    Object *obj;
+    MemoryRegion *ram;
+    Error *err = NULL;
+    uint32_t ram_size, vcram_size;
+    CharDriverState *chr;
+    int n;
+
+    obj = object_property_get_link(OBJECT(dev), "ram", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required ram link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    ram = MEMORY_REGION(obj);
+    ram_size = memory_region_size(ram);
+
+    /* Map peripherals and RAM into the GPU address space. */
+    memory_region_init_alias(&s->peri_mr_alias, OBJECT(s),
+                             "bcm2835-peripherals", &s->peri_mr, 0,
+                             memory_region_size(&s->peri_mr));
+
+    memory_region_add_subregion_overlap(&s->gpu_bus_mr, BCM2835_VC_PERI_BASE,
+                                        &s->peri_mr_alias, 1);
+
+    /* RAM is aliased four times (different cache configurations) on the GPU */
+    for (n = 0; n < 4; n++) {
+        memory_region_init_alias(&s->ram_alias[n], OBJECT(s),
+                                 "bcm2835-gpu-ram-alias[*]", ram, 0, ram_size);
+        memory_region_add_subregion_overlap(&s->gpu_bus_mr, (hwaddr)n << 30,
+                                            &s->ram_alias[n], 0);
+    }
+
+    /* Interrupt Controller */
+    object_property_set_bool(OBJECT(&s->ic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, ARMCTRL_IC_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->ic), 0));
+    sysbus_pass_irq(SYS_BUS_DEVICE(s), SYS_BUS_DEVICE(&s->ic));
+
+    /* UART0 */
+    object_property_set_bool(OBJECT(s->uart0), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, UART0_OFFSET,
+                                sysbus_mmio_get_region(s->uart0, 0));
+    sysbus_connect_irq(s->uart0, 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_UART));
+
+    /* AUX / UART1 */
+    /* TODO: don't call qemu_char_get_next_serial() here, instead set
+     * chardev properties for each uart at the board level, once pl011
+     * (uart0) has been updated to avoid qemu_char_get_next_serial()
+     */
+    chr = qemu_char_get_next_serial();
+    if (chr == NULL) {
+        chr = qemu_chr_new("bcm2835.uart1", "null", NULL);
+    }
+    qdev_prop_set_chr(DEVICE(&s->aux), "chardev", chr);
+
+    object_property_set_bool(OBJECT(&s->aux), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, UART1_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->aux), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->aux), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_AUX));
+
+    /* Mailboxes */
+    object_property_set_bool(OBJECT(&s->mboxes), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, ARMCTRL_0_SBM_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->mboxes), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->mboxes), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_ARM_IRQ,
+                               INTERRUPT_ARM_MAILBOX));
+
+    /* Framebuffer */
+    vcram_size = (uint32_t)object_property_get_int(OBJECT(s), "vcram-size",
+                                                   &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_int(OBJECT(&s->fb), ram_size - vcram_size,
+                            "vcram-base", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->fb), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->mbox_mr, MBOX_CHAN_FB << MBOX_AS_CHAN_SHIFT,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->fb), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->fb), 0,
+                       qdev_get_gpio_in(DEVICE(&s->mboxes), MBOX_CHAN_FB));
+
+    /* Property channel */
+    object_property_set_bool(OBJECT(&s->property), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->mbox_mr,
+                MBOX_CHAN_PROPERTY << MBOX_AS_CHAN_SHIFT,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->property), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->property), 0,
+                      qdev_get_gpio_in(DEVICE(&s->mboxes), MBOX_CHAN_PROPERTY));
+
+    /* Extended Mass Media Controller */
+    object_property_set_int(OBJECT(&s->sdhci), BCM2835_SDHC_CAPAREG, "capareg",
+                            &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->sdhci), true, "pending-insert-quirk",
+                             &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->sdhci), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, EMMC_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->sdhci), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sdhci), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_ARASANSDIO));
+    object_property_add_alias(OBJECT(s), "sd-bus", OBJECT(&s->sdhci), "sd-bus",
+                              &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    /* DMA Channels */
+    object_property_set_bool(OBJECT(&s->dma), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, DMA_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->dma), 0));
+    memory_region_add_subregion(&s->peri_mr, DMA15_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->dma), 1));
+
+    for (n = 0; n <= 12; n++) {
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->dma), n,
+                           qdev_get_gpio_in_named(DEVICE(&s->ic),
+                                                  BCM2835_IC_GPU_IRQ,
+                                                  INTERRUPT_DMA0 + n));
+    }
+}
+
+static void bcm2835_peripherals_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = bcm2835_peripherals_realize;
+    /* Reason: realize() method uses qemu_char_get_next_serial() */
+    dc->cannot_instantiate_with_device_add_yet = true;
+}
+
+static const TypeInfo bcm2835_peripherals_type_info = {
+    .name = TYPE_BCM2835_PERIPHERALS,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835PeripheralState),
+    .instance_init = bcm2835_peripherals_init,
+    .class_init = bcm2835_peripherals_class_init,
+};
+
+static void bcm2835_peripherals_register_types(void)
+{
+    type_register_static(&bcm2835_peripherals_type_info);
+}
+
+type_init(bcm2835_peripherals_register_types)
diff --git a/qemu/hw/arm/bcm2836.c b/qemu/hw/arm/bcm2836.c
new file mode 100644
index 000000000..8451190a1
--- /dev/null
+++ b/qemu/hw/arm/bcm2836.c
@@ -0,0 +1,184 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/bcm2836.h"
+#include "hw/arm/raspi_platform.h"
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+
+/* Peripheral base address seen by the CPU */
+#define BCM2836_PERI_BASE       0x3F000000
+
+/* "QA7" (Pi2) interrupt controller and mailboxes etc. */
+#define BCM2836_CONTROL_BASE    0x40000000
+
+static void bcm2836_init(Object *obj)
+{
+    BCM2836State *s = BCM2836(obj);
+    int n;
+
+    for (n = 0; n < BCM2836_NCPUS; n++) {
+        object_initialize(&s->cpus[n], sizeof(s->cpus[n]),
+                          "cortex-a15-" TYPE_ARM_CPU);
+        object_property_add_child(obj, "cpu[*]", OBJECT(&s->cpus[n]),
+                                  &error_abort);
+    }
+
+    object_initialize(&s->control, sizeof(s->control), TYPE_BCM2836_CONTROL);
+    object_property_add_child(obj, "control", OBJECT(&s->control), NULL);
+    qdev_set_parent_bus(DEVICE(&s->control), sysbus_get_default());
+
+    object_initialize(&s->peripherals, sizeof(s->peripherals),
+                      TYPE_BCM2835_PERIPHERALS);
+    object_property_add_child(obj, "peripherals", OBJECT(&s->peripherals),
+                              &error_abort);
+    object_property_add_alias(obj, "board-rev", OBJECT(&s->peripherals),
+                              "board-rev", &error_abort);
+    object_property_add_alias(obj, "vcram-size", OBJECT(&s->peripherals),
+                              "vcram-size", &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->peripherals), sysbus_get_default());
+}
+
+static void bcm2836_realize(DeviceState *dev, Error **errp)
+{
+    BCM2836State *s = BCM2836(dev);
+    Object *obj;
+    Error *err = NULL;
+    int n;
+
+    /* common peripherals from bcm2835 */
+
+    obj = object_property_get_link(OBJECT(dev), "ram", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required ram link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    object_property_add_const_link(OBJECT(&s->peripherals), "ram", obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->peripherals), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_add_alias(OBJECT(s), "sd-bus", OBJECT(&s->peripherals),
+                              "sd-bus", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map_overlap(SYS_BUS_DEVICE(&s->peripherals), 0,
+                            BCM2836_PERI_BASE, 1);
+
+    /* bcm2836 interrupt controller (and mailboxes, etc.) */
+    object_property_set_bool(OBJECT(&s->control), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->control), 0, BCM2836_CONTROL_BASE);
+
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->control), "gpu-irq", 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 1,
+        qdev_get_gpio_in_named(DEVICE(&s->control), "gpu-fiq", 0));
+
+    for (n = 0; n < BCM2836_NCPUS; n++) {
+        /* Mirror bcm2836, which has clusterid set to 0xf
+         * TODO: this should be converted to a property of ARM_CPU
+         */
+        s->cpus[n].mp_affinity = 0xF00 | n;
+
+        /* set periphbase/CBAR value for CPU-local registers */
+        object_property_set_int(OBJECT(&s->cpus[n]),
+                                BCM2836_PERI_BASE + MCORE_OFFSET,
+                                "reset-cbar", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        /* start powered off if not enabled */
+        object_property_set_bool(OBJECT(&s->cpus[n]), n >= s->enabled_cpus,
+                                 "start-powered-off", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        object_property_set_bool(OBJECT(&s->cpus[n]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        /* Connect irq/fiq outputs from the interrupt controller. */
+        qdev_connect_gpio_out_named(DEVICE(&s->control), "irq", n,
+                qdev_get_gpio_in(DEVICE(&s->cpus[n]), ARM_CPU_IRQ));
+        qdev_connect_gpio_out_named(DEVICE(&s->control), "fiq", n,
+                qdev_get_gpio_in(DEVICE(&s->cpus[n]), ARM_CPU_FIQ));
+
+        /* Connect timers from the CPU to the interrupt controller */
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_PHYS,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntpnsirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_VIRT,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntvirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_HYP,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cnthpirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_SEC,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntpsirq", n));
+    }
+}
+
+static Property bcm2836_props[] = {
+    DEFINE_PROP_UINT32("enabled-cpus", BCM2836State, enabled_cpus, BCM2836_NCPUS),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2836_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->props = bcm2836_props;
+    dc->realize = bcm2836_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo bcm2836_type_info = {
+    .name = TYPE_BCM2836,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2836State),
+    .instance_init = bcm2836_init,
+    .class_init = bcm2836_class_init,
+};
+
+static void bcm2836_register_types(void)
+{
+    type_register_static(&bcm2836_type_info);
+}
+
+type_init(bcm2836_register_types)
diff --git a/qemu/hw/arm/boot.c b/qemu/hw/arm/boot.c
index 5b969cda1..587694557 100644
--- a/qemu/hw/arm/boot.c
+++ b/qemu/hw/arm/boot.c
@@ -7,9 +7,12 @@
  * This code is licensed under the GPL.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
+#include "hw/arm/linux-boot-if.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -27,14 +30,15 @@
 #define KERNEL64_LOAD_ADDR 0x00080000
 
 typedef enum {
-    FIXUP_NONE = 0,   /* do nothing */
-    FIXUP_TERMINATOR, /* end of insns */
-    FIXUP_BOARDID,    /* overwrite with board ID number */
-    FIXUP_ARGPTR,     /* overwrite with pointer to kernel args */
-    FIXUP_ENTRYPOINT, /* overwrite with kernel entry point */
-    FIXUP_GIC_CPU_IF, /* overwrite with GIC CPU interface address */
-    FIXUP_BOOTREG,    /* overwrite with boot register address */
-    FIXUP_DSB,        /* overwrite with correct DSB insn for cpu */
+    FIXUP_NONE = 0,     /* do nothing */
+    FIXUP_TERMINATOR,   /* end of insns */
+    FIXUP_BOARDID,      /* overwrite with board ID number */
+    FIXUP_BOARD_SETUP,  /* overwrite with board specific setup code address */
+    FIXUP_ARGPTR,       /* overwrite with pointer to kernel args */
+    FIXUP_ENTRYPOINT,   /* overwrite with kernel entry point */
+    FIXUP_GIC_CPU_IF,   /* overwrite with GIC CPU interface address */
+    FIXUP_BOOTREG,      /* overwrite with boot register address */
+    FIXUP_DSB,          /* overwrite with correct DSB insn for cpu */
     FIXUP_MAX,
 } FixupType;
 
@@ -57,8 +61,17 @@ static const ARMInsnFixup bootloader_aarch64[] = {
     { 0, FIXUP_TERMINATOR }
 };
 
-/* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
+/* A very small bootloader: call the board-setup code (if needed),
+ * set r0-r2, then jump to the kernel.
+ * If we're not calling boot setup code then we don't copy across
+ * the first BOOTLOADER_NO_BOARD_SETUP_OFFSET insns in this array.
+ */
+
 static const ARMInsnFixup bootloader[] = {
+    { 0xe28fe004 }, /* add     lr, pc, #4 */
+    { 0xe51ff004 }, /* ldr     pc, [pc, #-4] */
+    { 0, FIXUP_BOARD_SETUP },
+#define BOOTLOADER_NO_BOARD_SETUP_OFFSET 3
     { 0xe3a00000 }, /* mov     r0, #0 */
     { 0xe59f1004 }, /* ldr     r1, [pc, #4] */
     { 0xe59f2004 }, /* ldr     r2, [pc, #4] */
@@ -130,6 +143,7 @@ static void write_bootloader(const char *name, hwaddr addr,
         case FIXUP_NONE:
             break;
         case FIXUP_BOARDID:
+        case FIXUP_BOARD_SETUP:
         case FIXUP_ARGPTR:
         case FIXUP_ENTRYPOINT:
         case FIXUP_GIC_CPU_IF:
@@ -165,6 +179,57 @@ static void default_write_secondary(ARMCPU *cpu,
                      smpboot, fixupcontext);
 }
 
+void arm_write_secure_board_setup_dummy_smc(ARMCPU *cpu,
+                                            const struct arm_boot_info *info,
+                                            hwaddr mvbar_addr)
+{
+    int n;
+    uint32_t mvbar_blob[] = {
+        /* mvbar_addr: secure monitor vectors
+         * Default unimplemented and unused vectors to spin. Makes it
+         * easier to debug (as opposed to the CPU running away).
+         */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xe1b0f00e, /* movs pc, lr ;SMC exception return */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+    };
+    uint32_t board_setup_blob[] = {
+        /* board setup addr */
+        0xe3a00e00 + (mvbar_addr >> 4), /* mov r0, #mvbar_addr */
+        0xee0c0f30, /* mcr     p15, 0, r0, c12, c0, 1 ;set MVBAR */
+        0xee110f11, /* mrc     p15, 0, r0, c1 , c1, 0 ;read SCR */
+        0xe3800031, /* orr     r0, #0x31              ;enable AW, FW, NS */
+        0xee010f11, /* mcr     p15, 0, r0, c1, c1, 0  ;write SCR */
+        0xe1a0100e, /* mov     r1, lr                 ;save LR across SMC */
+        0xe1600070, /* smc     #0                     ;call monitor to flush SCR */
+        0xe1a0f001, /* mov     pc, r1                 ;return */
+    };
+
+    /* check that mvbar_addr is correctly aligned and relocatable (using MOV) */
+    assert((mvbar_addr & 0x1f) == 0 && (mvbar_addr >> 4) < 0x100);
+
+    /* check that these blobs don't overlap */
+    assert((mvbar_addr + sizeof(mvbar_blob) <= info->board_setup_addr)
+          || (info->board_setup_addr + sizeof(board_setup_blob) <= mvbar_addr));
+
+    for (n = 0; n < ARRAY_SIZE(mvbar_blob); n++) {
+        mvbar_blob[n] = tswap32(mvbar_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup-mvbar", mvbar_blob, sizeof(mvbar_blob),
+                       mvbar_addr);
+
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), info->board_setup_addr);
+}
+
 static void default_reset_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
@@ -373,8 +438,10 @@ static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo,
         return 0;
     }
 
-    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells");
-    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells");
+    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells",
+                                   NULL, &error_fatal);
+    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells",
+                                   NULL, &error_fatal);
     if (acells == 0 || scells == 0) {
         fprintf(stderr, "dtb file invalid (#address-cells or #size-cells 0)\n");
         goto fail;
@@ -452,9 +519,34 @@ static void do_cpu_reset(void *opaque)
     cpu_reset(cs);
     if (info) {
         if (!info->is_linux) {
+            int i;
             /* Jump to the entry point.  */
             uint64_t entry = info->entry;
 
+            switch (info->endianness) {
+            case ARM_ENDIANNESS_LE:
+                env->cp15.sctlr_el[1] &= ~SCTLR_E0E;
+                for (i = 1; i < 4; ++i) {
+                    env->cp15.sctlr_el[i] &= ~SCTLR_EE;
+                }
+                env->uncached_cpsr &= ~CPSR_E;
+                break;
+            case ARM_ENDIANNESS_BE8:
+                env->cp15.sctlr_el[1] |= SCTLR_E0E;
+                for (i = 1; i < 4; ++i) {
+                    env->cp15.sctlr_el[i] |= SCTLR_EE;
+                }
+                env->uncached_cpsr |= CPSR_E;
+                break;
+            case ARM_ENDIANNESS_BE32:
+                env->cp15.sctlr_el[1] |= SCTLR_B;
+                break;
+            case ARM_ENDIANNESS_UNKNOWN:
+                break; /* Board's decision */
+            default:
+                g_assert_not_reached();
+            }
+
             if (!env->aarch64) {
                 env->thumb = info->entry & 1;
                 entry &= 0xfffffffe;
@@ -475,7 +567,9 @@ static void do_cpu_reset(void *opaque)
                  * adjust.
                  */
                 if (env->aarch64) {
+                    env->cp15.scr_el3 |= SCR_RW;
                     if (arm_feature(env, ARM_FEATURE_EL2)) {
+                        env->cp15.hcr_el2 |= HCR_RW;
                         env->pstate = PSTATE_MODE_EL2h;
                     } else {
                         env->pstate = PSTATE_MODE_EL1h;
@@ -483,7 +577,8 @@ static void do_cpu_reset(void *opaque)
                 }
 
                 /* Set to non-secure if not a secure boot */
-                if (!info->secure_boot) {
+                if (!info->secure_boot &&
+                    (cs != first_cpu || !info->secure_board_setup)) {
                     /* Linux expects non-secure state */
                     env->cp15.scr_el3 |= SCR_NS;
                 }
@@ -555,6 +650,76 @@ static void load_image_to_fw_cfg(FWCfgState *fw_cfg, uint16_t size_key,
     fw_cfg_add_bytes(fw_cfg, data_key, data, size);
 }
 
+static int do_arm_linux_init(Object *obj, void *opaque)
+{
+    if (object_dynamic_cast(obj, TYPE_ARM_LINUX_BOOT_IF)) {
+        ARMLinuxBootIf *albif = ARM_LINUX_BOOT_IF(obj);
+        ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_GET_CLASS(obj);
+        struct arm_boot_info *info = opaque;
+
+        if (albifc->arm_linux_init) {
+            albifc->arm_linux_init(albif, info->secure_boot);
+        }
+    }
+    return 0;
+}
+
+static uint64_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry,
+                             uint64_t *lowaddr, uint64_t *highaddr,
+                             int elf_machine)
+{
+    bool elf_is64;
+    union {
+        Elf32_Ehdr h32;
+        Elf64_Ehdr h64;
+    } elf_header;
+    int data_swab = 0;
+    bool big_endian;
+    uint64_t ret = -1;
+    Error *err = NULL;
+
+
+    load_elf_hdr(info->kernel_filename, &elf_header, &elf_is64, &err);
+    if (err) {
+        return ret;
+    }
+
+    if (elf_is64) {
+        big_endian = elf_header.h64.e_ident[EI_DATA] == ELFDATA2MSB;
+        info->endianness = big_endian ? ARM_ENDIANNESS_BE8
+                                      : ARM_ENDIANNESS_LE;
+    } else {
+        big_endian = elf_header.h32.e_ident[EI_DATA] == ELFDATA2MSB;
+        if (big_endian) {
+            if (bswap32(elf_header.h32.e_flags) & EF_ARM_BE8) {
+                info->endianness = ARM_ENDIANNESS_BE8;
+            } else {
+                info->endianness = ARM_ENDIANNESS_BE32;
+                /* In BE32, the CPU has a different view of the per-byte
+                 * address map than the rest of the system. BE32 ELF files
+                 * are organised such that they can be programmed through
+                 * the CPU's per-word byte-reversed view of the world. QEMU
+                 * however loads ELF files independently of the CPU. So
+                 * tell the ELF loader to byte reverse the data for us.
+                 */
+                data_swab = 2;
+            }
+        } else {
+            info->endianness = ARM_ENDIANNESS_LE;
+        }
+    }
+
+    ret = load_elf(info->kernel_filename, NULL, NULL,
+                   pentry, lowaddr, highaddr, big_endian, elf_machine,
+                   1, data_swab);
+    if (ret <= 0) {
+        /* The header loaded but the image didn't */
+        exit(1);
+    }
+
+    return ret;
+}
+
 static void arm_load_kernel_notify(Notifier *notifier, void *data)
 {
     CPUState *cs;
@@ -564,7 +729,6 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
     uint64_t elf_entry, elf_low_addr, elf_high_addr;
     int elf_machine;
     hwaddr entry, kernel_load_offset;
-    int big_endian;
     static const ARMInsnFixup *primary_loader;
     ArmLoadKernelNotifier *n = DO_UPCAST(ArmLoadKernelNotifier,
                                          notifier, notifier);
@@ -572,6 +736,12 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
     struct arm_boot_info *info =
         container_of(n, struct arm_boot_info, load_kernel_notifier);
 
+    /* The board code is not supposed to set secure_board_setup unless
+     * running its code in secure mode is actually possible, and KVM
+     * doesn't support secure.
+     */
+    assert(!(info->secure_board_setup && kvm_enabled()));
+
     /* Load the kernel.  */
     if (!info->kernel_filename || info->firmware_loaded) {
 
@@ -625,6 +795,9 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         elf_machine = EM_AARCH64;
     } else {
         primary_loader = bootloader;
+        if (!info->write_board_setup) {
+            primary_loader += BOOTLOADER_NO_BOARD_SETUP_OFFSET;
+        }
         kernel_load_offset = KERNEL_LOAD_ADDR;
         elf_machine = EM_ARM;
     }
@@ -641,12 +814,6 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
     if (info->nb_cpus == 0)
         info->nb_cpus = 1;
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    big_endian = 1;
-#else
-    big_endian = 0;
-#endif
-
     /* We want to put the initrd far enough into RAM that when the
      * kernel is uncompressed it will not clobber the initrd. However
      * on boards without much RAM we must ensure that we still leave
@@ -661,9 +828,8 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         MIN(info->ram_size / 2, 128 * 1024 * 1024);
 
     /* Assume that raw images are linux kernels, and ELF images are not.  */
-    kernel_size = load_elf(info->kernel_filename, NULL, NULL, &elf_entry,
-                           &elf_low_addr, &elf_high_addr, big_endian,
-                           elf_machine, 1);
+    kernel_size = arm_load_elf(info, &elf_entry, &elf_low_addr,
+                               &elf_high_addr, elf_machine);
     if (kernel_size > 0 && have_dtb(info)) {
         /* If there is still some room left at the base of RAM, try and put
          * the DTB there like we do for images loaded with -bios or -pflash.
@@ -730,6 +896,7 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         info->initrd_size = initrd_size;
 
         fixupcontext[FIXUP_BOARDID] = info->board_id;
+        fixupcontext[FIXUP_BOARD_SETUP] = info->board_setup_addr;
 
         /* for device tree boot, we pass the DTB directly in r2. Otherwise
          * we point to the kernel args.
@@ -778,6 +945,15 @@ static void arm_load_kernel_notify(Notifier *notifier, void *data)
         if (info->nb_cpus > 1) {
             info->write_secondary_boot(cpu, info);
         }
+        if (info->write_board_setup) {
+            info->write_board_setup(cpu, info);
+        }
+
+        /* Notify devices which need to fake up firmware initialization
+         * that we're doing a direct kernel boot.
+         */
+        object_child_foreach_recursive(object_get_root(),
+                                       do_arm_linux_init, info);
     }
     info->is_linux = is_linux;
 
@@ -803,3 +979,16 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
         qemu_register_reset(do_cpu_reset, ARM_CPU(cs));
     }
 }
+
+static const TypeInfo arm_linux_boot_if_info = {
+    .name = TYPE_ARM_LINUX_BOOT_IF,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(ARMLinuxBootIfClass),
+};
+
+static void arm_linux_boot_register_types(void)
+{
+    type_register_static(&arm_linux_boot_if_info);
+}
+
+type_init(arm_linux_boot_register_types)
diff --git a/qemu/hw/arm/collie.c b/qemu/hw/arm/collie.c
index 6c9b82fc5..8bb308a42 100644
--- a/qemu/hw/arm/collie.c
+++ b/qemu/hw/arm/collie.c
@@ -8,6 +8,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/boards.h"
@@ -58,15 +59,10 @@ static void collie_init(MachineState *machine)
     arm_load_kernel(s->cpu, &collie_binfo);
 }
 
-static QEMUMachine collie_machine = {
-    .name = "collie",
-    .desc = "Collie PDA (SA-1110)",
-    .init = collie_init,
-};
-
-static void collie_machine_init(void)
+static void collie_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&collie_machine);
+    mc->desc = "Sharp SL-5500 (Collie) PDA (SA-1110)";
+    mc->init = collie_init;
 }
 
-machine_init(collie_machine_init)
+DEFINE_MACHINE("collie", collie_machine_init)
diff --git a/qemu/hw/arm/cubieboard.c b/qemu/hw/arm/cubieboard.c
index 1582250eb..fbd78ed01 100644
--- a/qemu/hw/arm/cubieboard.c
+++ b/qemu/hw/arm/cubieboard.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
@@ -39,27 +43,26 @@ static void cubieboard_init(MachineState *machine)
 
     object_property_set_int(OBJECT(&s->a10->emac), 1, "phy-addr", &err);
     if (err != NULL) {
-        error_report("Couldn't set phy address: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set phy address: ");
         exit(1);
     }
 
     object_property_set_int(OBJECT(&s->a10->timer), 32768, "clk0-freq", &err);
     if (err != NULL) {
-        error_report("Couldn't set clk0 frequency: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set clk0 frequency: ");
         exit(1);
     }
 
     object_property_set_int(OBJECT(&s->a10->timer), 24000000, "clk1-freq",
                             &err);
     if (err != NULL) {
-        error_report("Couldn't set clk1 frequency: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set clk1 frequency: ");
         exit(1);
     }
 
     object_property_set_bool(OBJECT(s->a10), true, "realized", &err);
     if (err != NULL) {
-        error_report("Couldn't realize Allwinner A10: %s",
-                     error_get_pretty(err));
+        error_reportf_err(err, "Couldn't realize Allwinner A10: ");
         exit(1);
     }
 
@@ -74,16 +77,10 @@ static void cubieboard_init(MachineState *machine)
     arm_load_kernel(&s->a10->cpu, &cubieboard_binfo);
 }
 
-static QEMUMachine cubieboard_machine = {
-    .name = "cubieboard",
-    .desc = "cubietech cubieboard",
-    .init = cubieboard_init,
-};
-
-
-static void cubieboard_machine_init(void)
+static void cubieboard_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&cubieboard_machine);
+    mc->desc = "cubietech cubieboard";
+    mc->init = cubieboard_init;
 }
 
-machine_init(cubieboard_machine_init)
+DEFINE_MACHINE("cubieboard", cubieboard_machine_init)
diff --git a/qemu/hw/arm/digic.c b/qemu/hw/arm/digic.c
index ec8c33060..e0f973032 100644
--- a/qemu/hw/arm/digic.c
+++ b/qemu/hw/arm/digic.c
@@ -20,6 +20,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/arm/digic.h"
 
 #define DIGIC4_TIMER_BASE(n)    (0xc0210000 + (n) * 0x100)
@@ -97,6 +99,12 @@ static void digic_class_init(ObjectClass *oc, void *data)
     DeviceClass *dc = DEVICE_CLASS(oc);
 
     dc->realize = digic_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo digic_type_info = {
diff --git a/qemu/hw/arm/digic_boards.c b/qemu/hw/arm/digic_boards.c
index f8ba9e595..520c8e9ff 100644
--- a/qemu/hw/arm/digic_boards.c
+++ b/qemu/hw/arm/digic_boards.c
@@ -23,6 +23,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -64,8 +68,7 @@ static void digic4_board_init(DigicBoard *board)
     s->digic = DIGIC(object_new(TYPE_DIGIC));
     object_property_set_bool(OBJECT(s->digic), true, "realized", &err);
     if (err != NULL) {
-        error_report("Couldn't realize DIGIC SoC: %s",
-                     error_get_pretty(err));
+        error_reportf_err(err, "Couldn't realize DIGIC SoC: ");
         exit(1);
     }
 
@@ -148,15 +151,10 @@ static void canon_a1100_init(MachineState *machine)
     digic4_board_init(&digic4_board_canon_a1100);
 }
 
-static QEMUMachine canon_a1100 = {
-    .name = "canon-a1100",
-    .desc = "Canon PowerShot A1100 IS",
-    .init = &canon_a1100_init,
-};
-
-static void digic_register_machines(void)
+static void canon_a1100_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&canon_a1100);
+    mc->desc = "Canon PowerShot A1100 IS";
+    mc->init = &canon_a1100_init;
 }
 
-machine_init(digic_register_machines)
+DEFINE_MACHINE("canon-a1100", canon_a1100_machine_init)
diff --git a/qemu/hw/arm/exynos4210.c b/qemu/hw/arm/exynos4210.c
index c55fab813..be3c96d21 100644
--- a/qemu/hw/arm/exynos4210.c
+++ b/qemu/hw/arm/exynos4210.c
@@ -21,6 +21,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
@@ -150,27 +154,18 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     for (n = 0; n < EXYNOS4210_NCPUS; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
 
         /* By default A9 CPUs have EL3 enabled.  This board does not currently
          * support EL3 so the CPU EL3 property is disabled before realization.
          */
         if (object_property_find(cpuobj, "has_el3", NULL)) {
-            object_property_set_bool(cpuobj, false, "has_el3", &err);
-            if (err) {
-                error_report_err(err);
-                exit(1);
-            }
+            object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
         }
 
         s->cpu[n] = ARM_CPU(cpuobj);
         object_property_set_int(cpuobj, EXYNOS4210_SMP_PRIVATE_BASE_ADDR,
                                 "reset-cbar", &error_abort);
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
     }
 
     /*** IRQs ***/
@@ -259,7 +254,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal ROM */
     memory_region_init_ram(&s->irom_mem, NULL, "exynos4210.irom",
-                           EXYNOS4210_IROM_SIZE, &error_abort);
+                           EXYNOS4210_IROM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->irom_mem);
     memory_region_set_readonly(&s->irom_mem, true);
     memory_region_add_subregion(system_mem, EXYNOS4210_IROM_BASE_ADDR,
@@ -275,7 +270,7 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
 
     /* Internal RAM */
     memory_region_init_ram(&s->iram_mem, NULL, "exynos4210.iram",
-                           EXYNOS4210_IRAM_SIZE, &error_abort);
+                           EXYNOS4210_IRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->iram_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_IRAM_BASE_ADDR,
                                 &s->iram_mem);
@@ -284,14 +279,14 @@ Exynos4210State *exynos4210_init(MemoryRegion *system_mem,
     mem_size = ram_size;
     if (mem_size > EXYNOS4210_DRAM_MAX_SIZE) {
         memory_region_init_ram(&s->dram1_mem, NULL, "exynos4210.dram1",
-                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_abort);
+                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_fatal);
         vmstate_register_ram_global(&s->dram1_mem);
         memory_region_add_subregion(system_mem, EXYNOS4210_DRAM1_BASE_ADDR,
                 &s->dram1_mem);
         mem_size = EXYNOS4210_DRAM_MAX_SIZE;
     }
     memory_region_init_ram(&s->dram0_mem, NULL, "exynos4210.dram0", mem_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->dram0_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_DRAM0_BASE_ADDR,
             &s->dram0_mem);
diff --git a/qemu/hw/arm/exynos4_boards.c b/qemu/hw/arm/exynos4_boards.c
index d644db1ef..0efa19405 100644
--- a/qemu/hw/arm/exynos4_boards.c
+++ b/qemu/hw/arm/exynos4_boards.c
@@ -21,6 +21,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/qtest.h"
 #include "hw/sysbus.h"
@@ -74,8 +77,6 @@ static struct arm_boot_info exynos4_board_binfo = {
     .write_secondary_boot = exynos4210_write_secondary,
 };
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS];
-
 static void lan9215_init(uint32_t base, qemu_irq irq)
 {
     DeviceState *dev;
@@ -97,11 +98,12 @@ static void lan9215_init(uint32_t base, qemu_irq irq)
 static Exynos4210State *exynos4_boards_init_common(MachineState *machine,
                                                    Exynos4BoardType board_type)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+
     if (smp_cpus != EXYNOS4210_NCPUS && !qtest_enabled()) {
         fprintf(stderr, "%s board supports only %d CPU cores. Ignoring smp_cpus"
                 " value.\n",
-                exynos4_machines[board_type].name,
-                exynos4_machines[board_type].max_cpus);
+                mc->name, EXYNOS4210_NCPUS);
     }
 
     exynos4_board_binfo.ram_size = exynos4_board_ram_size[board_type];
@@ -145,25 +147,40 @@ static void smdkc210_init(MachineState *machine)
     arm_load_kernel(ARM_CPU(first_cpu), &exynos4_board_binfo);
 }
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS] = {
-    [EXYNOS4_BOARD_NURI] = {
-        .name = "nuri",
-        .desc = "Samsung NURI board (Exynos4210)",
-        .init = nuri_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
-    [EXYNOS4_BOARD_SMDKC210] = {
-        .name = "smdkc210",
-        .desc = "Samsung SMDKC210 board (Exynos4210)",
-        .init = smdkc210_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
+static void nuri_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung NURI board (Exynos4210)";
+    mc->init = nuri_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
+}
+
+static const TypeInfo nuri_type = {
+    .name = MACHINE_TYPE_NAME("nuri"),
+    .parent = TYPE_MACHINE,
+    .class_init = nuri_class_init,
+};
+
+static void smdkc210_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung SMDKC210 board (Exynos4210)";
+    mc->init = smdkc210_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
+}
+
+static const TypeInfo smdkc210_type = {
+    .name = MACHINE_TYPE_NAME("smdkc210"),
+    .parent = TYPE_MACHINE,
+    .class_init = smdkc210_class_init,
 };
 
-static void exynos4_machine_init(void)
+static void exynos4_machines_init(void)
 {
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_NURI]);
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_SMDKC210]);
+    type_register_static(&nuri_type);
+    type_register_static(&smdkc210_type);
 }
 
-machine_init(exynos4_machine_init);
+type_init(exynos4_machines_init)
diff --git a/qemu/hw/arm/fsl-imx25.c b/qemu/hw/arm/fsl-imx25.c
new file mode 100644
index 000000000..2f878b935
--- /dev/null
+++ b/qemu/hw/arm/fsl-imx25.c
@@ -0,0 +1,313 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX25 SOC emulation.
+ *
+ * Based on hw/arm/xlnx-zynqmp.c
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx25.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx25_init(Object *obj)
+{
+    FslIMX25State *s = FSL_IMX25(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm926-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX25_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        object_initialize(&s->gpt[i], sizeof(s->gpt[i]), TYPE_IMX_GPT);
+        qdev_set_parent_bus(DEVICE(&s->gpt[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->fec, sizeof(s->fec), TYPE_IMX_FEC);
+    qdev_set_parent_bus(DEVICE(&s->fec), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx25_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX25State *s = FSL_IMX25(dev);
+    uint8_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX25_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX25_CCM_ADDR);
+
+    /* Initialize all UARTs */
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX25_NUM_UARTS] = {
+            { FSL_IMX25_UART1_ADDR, FSL_IMX25_UART1_IRQ },
+            { FSL_IMX25_UART2_ADDR, FSL_IMX25_UART2_IRQ },
+            { FSL_IMX25_UART3_ADDR, FSL_IMX25_UART3_IRQ },
+            { FSL_IMX25_UART4_ADDR, FSL_IMX25_UART4_IRQ },
+            { FSL_IMX25_UART5_ADDR, FSL_IMX25_UART5_IRQ }
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    /* Initialize all GPT timers */
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpt_table[FSL_IMX25_NUM_GPTS] = {
+            { FSL_IMX25_GPT1_ADDR, FSL_IMX25_GPT1_IRQ },
+            { FSL_IMX25_GPT2_ADDR, FSL_IMX25_GPT2_IRQ },
+            { FSL_IMX25_GPT3_ADDR, FSL_IMX25_GPT3_IRQ },
+            { FSL_IMX25_GPT4_ADDR, FSL_IMX25_GPT4_IRQ }
+        };
+
+        s->gpt[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->gpt[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt[i]), 0, gpt_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpt_table[i].irq));
+    }
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX25_NUM_EPITS] = {
+            { FSL_IMX25_EPIT1_ADDR, FSL_IMX25_EPIT1_IRQ },
+            { FSL_IMX25_EPIT2_ADDR, FSL_IMX25_EPIT2_IRQ }
+        };
+
+        s->epit[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    qdev_set_nic_properties(DEVICE(&s->fec), &nd_table[0]);
+    object_property_set_bool(OBJECT(&s->fec), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->fec), 0, FSL_IMX25_FEC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->fec), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX25_FEC_IRQ));
+
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX25_NUM_I2CS] = {
+            { FSL_IMX25_I2C1_ADDR, FSL_IMX25_I2C1_IRQ },
+            { FSL_IMX25_I2C2_ADDR, FSL_IMX25_I2C2_IRQ },
+            { FSL_IMX25_I2C3_ADDR, FSL_IMX25_I2C3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX25_NUM_GPIOS] = {
+            { FSL_IMX25_GPIO1_ADDR, FSL_IMX25_GPIO1_IRQ },
+            { FSL_IMX25_GPIO2_ADDR, FSL_IMX25_GPIO2_IRQ },
+            { FSL_IMX25_GPIO3_ADDR, FSL_IMX25_GPIO3_IRQ },
+            { FSL_IMX25_GPIO4_ADDR, FSL_IMX25_GPIO4_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* initialize 2 x 16 KB ROM */
+    memory_region_init_rom_device(&s->rom[0], NULL, NULL, NULL,
+                                  "imx25.rom0", FSL_IMX25_ROM0_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM0_ADDR,
+                                &s->rom[0]);
+    memory_region_init_rom_device(&s->rom[1], NULL, NULL, NULL,
+                                  "imx25.rom1", FSL_IMX25_ROM1_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM1_ADDR,
+                                &s->rom[1]);
+
+    /* initialize internal RAM (128 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx25.iram", FSL_IMX25_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (128 KB) is aliased over 128 MB - 128 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx25.iram_alias",
+                             &s->iram, 0, FSL_IMX25_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx25_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx25_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+    dc->desc = "i.MX25 SOC";
+}
+
+static const TypeInfo fsl_imx25_type_info = {
+    .name = TYPE_FSL_IMX25,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX25State),
+    .instance_init = fsl_imx25_init,
+    .class_init = fsl_imx25_class_init,
+};
+
+static void fsl_imx25_register_types(void)
+{
+    type_register_static(&fsl_imx25_type_info);
+}
+
+type_init(fsl_imx25_register_types)
diff --git a/qemu/hw/arm/fsl-imx31.c b/qemu/hw/arm/fsl-imx31.c
new file mode 100644
index 000000000..31a3a8791
--- /dev/null
+++ b/qemu/hw/arm/fsl-imx31.c
@@ -0,0 +1,287 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX31 SOC emulation.
+ *
+ * Based on hw/arm/fsl-imx31.c
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx31.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx31_init(Object *obj)
+{
+    FslIMX31State *s = FSL_IMX31(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm1136-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX31_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->gpt, sizeof(s->gpt), TYPE_IMX_GPT);
+    qdev_set_parent_bus(DEVICE(&s->gpt), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx31_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX31State *s = FSL_IMX31(dev);
+    uint16_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX31_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX31_CCM_ADDR);
+
+    /* Initialize all UARTS */
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX31_NUM_UARTS] = {
+            { FSL_IMX31_UART1_ADDR, FSL_IMX31_UART1_IRQ },
+            { FSL_IMX31_UART2_ADDR, FSL_IMX31_UART2_IRQ },
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    s->gpt.ccm = IMX_CCM(&s->ccm);
+
+    object_property_set_bool(OBJECT(&s->gpt), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt), 0, FSL_IMX31_GPT_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX31_GPT_IRQ));
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX31_NUM_EPITS] = {
+            { FSL_IMX31_EPIT1_ADDR, FSL_IMX31_EPIT1_IRQ },
+            { FSL_IMX31_EPIT2_ADDR, FSL_IMX31_EPIT2_IRQ },
+        };
+
+        s->epit[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX31_NUM_I2CS] = {
+            { FSL_IMX31_I2C1_ADDR, FSL_IMX31_I2C1_IRQ },
+            { FSL_IMX31_I2C2_ADDR, FSL_IMX31_I2C2_IRQ },
+            { FSL_IMX31_I2C3_ADDR, FSL_IMX31_I2C3_IRQ }
+        };
+
+        /* Initialize the I2C */
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        /* Map I2C memory */
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        /* Connect I2C IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX31_NUM_GPIOS] = {
+            { FSL_IMX31_GPIO1_ADDR, FSL_IMX31_GPIO1_IRQ },
+            { FSL_IMX31_GPIO2_ADDR, FSL_IMX31_GPIO2_IRQ },
+            { FSL_IMX31_GPIO3_ADDR, FSL_IMX31_GPIO3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), false, "has-edge-sel",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* On a real system, the first 16k is a `secure boot rom' */
+    memory_region_init_rom_device(&s->secure_rom, NULL, NULL, NULL,
+                                  "imx31.secure_rom",
+                                  FSL_IMX31_SECURE_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SECURE_ROM_ADDR,
+                                &s->secure_rom);
+
+    /* There is also a 16k ROM */
+    memory_region_init_rom_device(&s->rom, NULL, NULL, NULL, "imx31.rom",
+                                  FSL_IMX31_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_ROM_ADDR,
+                                &s->rom);
+
+    /* initialize internal RAM (16 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx31.iram", FSL_IMX31_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (16 KB) is aliased over 256 MB - 16 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx31.iram_alias",
+                             &s->iram, 0, FSL_IMX31_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx31_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx31_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+    dc->desc = "i.MX31 SOC";
+}
+
+static const TypeInfo fsl_imx31_type_info = {
+    .name = TYPE_FSL_IMX31,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX31State),
+    .instance_init = fsl_imx31_init,
+    .class_init = fsl_imx31_class_init,
+};
+
+static void fsl_imx31_register_types(void)
+{
+    type_register_static(&fsl_imx31_type_info);
+}
+
+type_init(fsl_imx31_register_types)
diff --git a/qemu/hw/arm/gumstix.c b/qemu/hw/arm/gumstix.c
index 8103278b1..d59d9ba4e 100644
--- a/qemu/hw/arm/gumstix.c
+++ b/qemu/hw/arm/gumstix.c
@@ -34,6 +34,7 @@
  * # qemu-system-arm -M verdex -pflash flash -monitor null -nographic -m 289
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "net/net.h"
@@ -121,22 +122,38 @@ static void verdex_init(MachineState *machine)
                     qdev_get_gpio_in(cpu->gpio, 99));
 }
 
-static QEMUMachine connex_machine = {
-    .name = "connex",
-    .desc = "Gumstix Connex (PXA255)",
-    .init = connex_init,
+static void connex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Connex (PXA255)";
+    mc->init = connex_init;
+}
+
+static const TypeInfo connex_type = {
+    .name = MACHINE_TYPE_NAME("connex"),
+    .parent = TYPE_MACHINE,
+    .class_init = connex_class_init,
 };
 
-static QEMUMachine verdex_machine = {
-    .name = "verdex",
-    .desc = "Gumstix Verdex (PXA270)",
-    .init = verdex_init,
+static void verdex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Verdex (PXA270)";
+    mc->init = verdex_init;
+}
+
+static const TypeInfo verdex_type = {
+    .name = MACHINE_TYPE_NAME("verdex"),
+    .parent = TYPE_MACHINE,
+    .class_init = verdex_class_init,
 };
 
 static void gumstix_machine_init(void)
 {
-    qemu_register_machine(&connex_machine);
-    qemu_register_machine(&verdex_machine);
+    type_register_static(&connex_type);
+    type_register_static(&verdex_type);
 }
 
-machine_init(gumstix_machine_init);
+type_init(gumstix_machine_init)
diff --git a/qemu/hw/arm/highbank.c b/qemu/hw/arm/highbank.c
index f8353a787..d9930c0d3 100644
--- a/qemu/hw/arm/highbank.c
+++ b/qemu/hw/arm/highbank.c
@@ -17,11 +17,14 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "hw/loader.h"
 #include "net/net.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
@@ -32,10 +35,19 @@
 #define SMP_BOOT_REG            0x40
 #define MPCORE_PERIPHBASE       0xfff10000
 
+#define MVBAR_ADDR              0x200
+#define BOARD_SETUP_ADDR        (MVBAR_ADDR + 8 * sizeof(uint32_t))
+
 #define NIRQ_GIC                160
 
 /* Board init.  */
 
+static void hb_write_board_setup(ARMCPU *cpu,
+                                 const struct arm_boot_info *info)
+{
+    arm_write_secure_board_setup_dummy_smc(cpu, info, MVBAR_ADDR);
+}
+
 static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
 {
     int n;
@@ -223,52 +235,37 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
     MemoryRegion *sysmem;
     char *sysboot_filename;
 
-    if (!cpu_model) {
-        switch (machine_id) {
-        case CALXEDA_HIGHBANK:
-            cpu_model = "cortex-a9";
-            break;
-        case CALXEDA_MIDWAY:
-            cpu_model = "cortex-a15";
-            break;
-        }
+    switch (machine_id) {
+    case CALXEDA_HIGHBANK:
+        cpu_model = "cortex-a9";
+        break;
+    case CALXEDA_MIDWAY:
+        cpu_model = "cortex-a15";
+        break;
     }
 
     for (n = 0; n < smp_cpus; n++) {
         ObjectClass *oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
         Object *cpuobj;
         ARMCPU *cpu;
-        Error *err = NULL;
-
-        if (!oc) {
-            error_report("Unable to find CPU definition");
-            exit(1);
-        }
 
         cpuobj = object_new(object_class_get_name(oc));
         cpu = ARM_CPU(cpuobj);
 
-        /* By default A9 and A15 CPUs have EL3 enabled.  This board does not
-         * currently support EL3 so the CPU EL3 property is disabled before
-         * realization.
-         */
-        if (object_property_find(cpuobj, "has_el3", NULL)) {
-            object_property_set_bool(cpuobj, false, "has_el3", &err);
-            if (err) {
-                error_report_err(err);
-                exit(1);
-            }
+        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_SMC,
+                                "psci-conduit", &error_abort);
+
+        if (n) {
+            /* Secondary CPUs start in PSCI powered-down state */
+            object_property_set_bool(cpuobj, true,
+                                     "start-powered-off", &error_abort);
         }
 
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
             object_property_set_int(cpuobj, MPCORE_PERIPHBASE,
                                     "reset-cbar", &error_abort);
         }
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
         cpu_irq[n] = qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ);
         cpu_fiq[n] = qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_FIQ);
     }
@@ -281,17 +278,19 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
 
     sysram = g_new(MemoryRegion, 1);
     memory_region_init_ram(sysram, NULL, "highbank.sysram", 0x8000,
-                           &error_abort);
+                           &error_fatal);
     memory_region_add_subregion(sysmem, 0xfff88000, sysram);
     if (bios_name != NULL) {
         sysboot_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (sysboot_filename != NULL) {
             if (load_image_targphys(sysboot_filename, 0xfff88000, 0x8000) < 0) {
-                hw_error("Unable to load %s\n", bios_name);
+                error_report("Unable to load %s", bios_name);
+                exit(1);
             }
             g_free(sysboot_filename);
         } else {
-           hw_error("Unable to find %s\n", bios_name);
+            error_report("Unable to find %s", bios_name);
+            exit(1);
         }
     }
 
@@ -378,6 +377,16 @@ static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
     highbank_binfo.loader_start = 0;
     highbank_binfo.write_secondary_boot = hb_write_secondary;
     highbank_binfo.secondary_cpu_reset_hook = hb_reset_secondary;
+    if (!kvm_enabled()) {
+        highbank_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+        highbank_binfo.write_board_setup = hb_write_board_setup;
+        highbank_binfo.secure_board_setup = true;
+    } else {
+        error_report("WARNING: cannot load built-in Monitor support "
+                     "if KVM is enabled. Some guests (such as Linux) "
+                     "may not boot.");
+    }
+
     arm_load_kernel(ARM_CPU(first_cpu), &highbank_binfo);
 }
 
@@ -391,26 +400,42 @@ static void midway_init(MachineState *machine)
     calxeda_init(machine, CALXEDA_MIDWAY);
 }
 
-static QEMUMachine highbank_machine = {
-    .name = "highbank",
-    .desc = "Calxeda Highbank (ECX-1000)",
-    .init = highbank_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void highbank_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Highbank (ECX-1000)";
+    mc->init = highbank_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo highbank_type = {
+    .name = MACHINE_TYPE_NAME("highbank"),
+    .parent = TYPE_MACHINE,
+    .class_init = highbank_class_init,
 };
 
-static QEMUMachine midway_machine = {
-    .name = "midway",
-    .desc = "Calxeda Midway (ECX-2000)",
-    .init = midway_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void midway_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Midway (ECX-2000)";
+    mc->init = midway_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo midway_type = {
+    .name = MACHINE_TYPE_NAME("midway"),
+    .parent = TYPE_MACHINE,
+    .class_init = midway_class_init,
 };
 
 static void calxeda_machines_init(void)
 {
-    qemu_register_machine(&highbank_machine);
-    qemu_register_machine(&midway_machine);
+    type_register_static(&highbank_type);
+    type_register_static(&midway_type);
 }
 
-machine_init(calxeda_machines_init);
+type_init(calxeda_machines_init)
diff --git a/qemu/hw/arm/imx25_pdk.c b/qemu/hw/arm/imx25_pdk.c
new file mode 100644
index 000000000..025b60843
--- /dev/null
+++ b/qemu/hw/arm/imx25_pdk.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * PDK Board System emulation.
+ *
+ * Based on hw/arm/kzm.c
+ *
+ * Copyright (c) 2008 OKL and 2011 NICTA
+ * Written by Hans at OK-Labs
+ * Updated by Peter Chubb.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx25.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
+#include "hw/i2c/i2c.h"
+
+/* Memory map for PDK Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX25 SOC fr support
+ * 0x80000000-0x87ffffff RAM + Alias          EMULATED
+ * 0x90000000-0x9fffffff RAM + Alias          EMULATED
+ * 0xa0000000-0xa7ffffff Flash                IGNORED
+ * 0xa8000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb1ffffff SRAM                 IGNORED
+ * 0xb2000000-0xb3ffffff SRAM                 IGNORED
+ * 0xb4000000-0xb5ffffff CS4                  IGNORED
+ * 0xb6000000-0xb8000fff Reserved             IGNORED
+ * 0xb8001000-0xb8001fff SDRAM CTRL reg       IGNORED
+ * 0xb8002000-0xb8002fff WEIM CTRL reg        IGNORED
+ * 0xb8003000-0xb8003fff M3IF CTRL reg        IGNORED
+ * 0xb8004000-0xb8004fff EMI CTRL reg         IGNORED
+ * 0xb8005000-0xbaffffff Reserved             IGNORED
+ * 0xbb000000-0xbb000fff NAND flash area buf  IGNORED
+ * 0xbb001000-0xbb0011ff NAND flash reserved  IGNORED
+ * 0xbb001200-0xbb001dff Reserved             IGNORED
+ * 0xbb001e00-0xbb001fff NAN flash CTRL reg   IGNORED
+ * 0xbb012000-0xbfffffff Reserved             IGNORED
+ * 0xc0000000-0xffffffff Reserved             IGNORED
+ */
+
+typedef struct IMX25PDK {
+    FslIMX25State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX25PDK;
+
+static struct arm_boot_info imx25_pdk_binfo;
+
+static void imx25_pdk_init(MachineState *machine)
+{
+    IMX25PDK *s = g_new0(IMX25PDK, 1);
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    int i;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX25);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* We need to initialize our memory */
+    if (machine->ram_size > (FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE;
+    }
+
+    memory_region_allocate_system_memory(&s->ram, NULL, "imx25.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX25_SDRAM0_ADDR, FSL_IMX25_SDRAM0_SIZE },
+            { FSL_IMX25_SDRAM1_ADDR, FSL_IMX25_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
+    }
+
+    imx25_pdk_binfo.ram_size = machine->ram_size;
+    imx25_pdk_binfo.kernel_filename = machine->kernel_filename;
+    imx25_pdk_binfo.kernel_cmdline = machine->kernel_cmdline;
+    imx25_pdk_binfo.initrd_filename = machine->initrd_filename;
+    imx25_pdk_binfo.loader_start = FSL_IMX25_SDRAM0_ADDR;
+    imx25_pdk_binfo.board_id = 1771,
+    imx25_pdk_binfo.nb_cpus = 1;
+
+    /*
+     * We test explicitly for qtest here as it is not done (yet?) in
+     * arm_load_kernel(). Without this the "make check" command would
+     * fail.
+     */
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &imx25_pdk_binfo);
+    } else {
+        /*
+         * This I2C device doesn't exist on the real board.
+         * We add it here (only on qtest usage) to be able to do a bit
+         * of simple qtest. See "make check" for details.
+         */
+        i2c_create_slave((I2CBus *)qdev_get_child_bus(DEVICE(&s->soc.i2c[0]),
+                                                      "i2c"),
+                         "ds1338", 0x68);
+    }
+}
+
+static void imx25_pdk_machine_init(MachineClass *mc)
+{
+    mc->desc = "ARM i.MX25 PDK board (ARM926)";
+    mc->init = imx25_pdk_init;
+}
+
+DEFINE_MACHINE("imx25-pdk", imx25_pdk_machine_init)
diff --git a/qemu/hw/arm/integratorcp.c b/qemu/hw/arm/integratorcp.c
index 0fbbf997e..e31bca6e7 100644
--- a/qemu/hw/arm/integratorcp.c
+++ b/qemu/hw/arm/integratorcp.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
@@ -266,7 +270,7 @@ static int integratorcm_init(SysBusDevice *dev)
     s->cm_refcnt_offset = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24,
                                    1000);
     memory_region_init_ram(&s->flash, OBJECT(s), "integrator.flash", 0x100000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->flash);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &integratorcm_ops, s,
@@ -533,7 +537,6 @@ static void integratorcp_init(MachineState *machine)
     qemu_irq pic[32];
     DeviceState *dev, *sic, *icp;
     int i;
-    Error *err = NULL;
 
     if (!cpu_model) {
         cpu_model = "arm926";
@@ -552,18 +555,10 @@ static void integratorcp_init(MachineState *machine)
      * realization.
      */
     if (object_property_find(cpuobj, "has_el3", NULL)) {
-        object_property_set_bool(cpuobj, false, "has_el3", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
     }
 
-    object_property_set_bool(cpuobj, true, "realized", &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
-    }
+    object_property_set_bool(cpuobj, true, "realized", &error_fatal);
 
     cpu = ARM_CPU(cpuobj);
 
@@ -619,18 +614,13 @@ static void integratorcp_init(MachineState *machine)
     arm_load_kernel(cpu, &integrator_binfo);
 }
 
-static QEMUMachine integratorcp_machine = {
-    .name = "integratorcp",
-    .desc = "ARM Integrator/CP (ARM926EJ-S)",
-    .init = integratorcp_init,
-};
-
-static void integratorcp_machine_init(void)
+static void integratorcp_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&integratorcp_machine);
+    mc->desc = "ARM Integrator/CP (ARM926EJ-S)";
+    mc->init = integratorcp_init;
 }
 
-machine_init(integratorcp_machine_init);
+DEFINE_MACHINE("integratorcp", integratorcp_machine_init)
 
 static Property core_properties[] = {
     DEFINE_PROP_UINT32("memsz", IntegratorCMState, memsz, 0),
diff --git a/qemu/hw/arm/kzm.c b/qemu/hw/arm/kzm.c
index 5be0369a5..2c96ee33b 100644
--- a/qemu/hw/arm/kzm.c
+++ b/qemu/hw/arm/kzm.c
@@ -13,141 +13,135 @@
  * i.MX31 SoC
  */
 
-#include "hw/sysbus.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx31.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "exec/address-spaces.h"
-#include "hw/hw.h"
-#include "hw/arm/arm.h"
-#include "hw/devices.h"
 #include "net/net.h"
-#include "sysemu/sysemu.h"
-#include "hw/boards.h"
+#include "hw/devices.h"
 #include "hw/char/serial.h"
-#include "hw/arm/imx.h"
-
-    /* Memory map for Kzm Emulation Baseboard:
-     * 0x00000000-0x00003fff 16k secure ROM       IGNORED
-     * 0x00004000-0x00407fff Reserved             IGNORED
-     * 0x00404000-0x00407fff ROM                  IGNORED
-     * 0x00408000-0x0fffffff Reserved             IGNORED
-     * 0x10000000-0x1fffbfff RAM aliasing         IGNORED
-     * 0x1fffc000-0x1fffffff RAM                  EMULATED
-     * 0x20000000-0x2fffffff Reserved             IGNORED
-     * 0x30000000-0x7fffffff I.MX31 Internal Register Space
-     *   0x43f00000 IO_AREA0
-     *   0x43f90000 UART1                         EMULATED
-     *   0x43f94000 UART2                         EMULATED
-     *   0x68000000 AVIC                          EMULATED
-     *   0x53f80000 CCM                           EMULATED
-     *   0x53f94000 PIT 1                         EMULATED
-     *   0x53f98000 PIT 2                         EMULATED
-     *   0x53f90000 GPT                           EMULATED
-     * 0x80000000-0x87ffffff RAM                  EMULATED
-     * 0x88000000-0x8fffffff RAM Aliasing         EMULATED
-     * 0xa0000000-0xafffffff NAND Flash           IGNORED
-     * 0xb0000000-0xb3ffffff Unavailable          IGNORED
-     * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
-     * 0xb4001000-0xb400100f Board control        IGNORED
-     *  0xb4001003           DIP switch
-     * 0xb4001010-0xb400101f 7-segment LED        IGNORED
-     * 0xb4001020-0xb400102f LED                  IGNORED
-     * 0xb4001030-0xb400103f LED                  IGNORED
-     * 0xb4001040-0xb400104f FPGA, UART           EMULATED
-     * 0xb4001050-0xb400105f FPGA, UART           EMULATED
-     * 0xb4001060-0xb40fffff FPGA                 IGNORED
-     * 0xb6000000-0xb61fffff LAN controller       EMULATED
-     * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
-     * 0xb6300000-0xb7ffffff Free                 IGNORED
-     * 0xb8000000-0xb8004fff Memory control registers IGNORED
-     * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
-     * 0xc4000000-0xffffffff Reserved             IGNORED
-     */
-
-#define KZM_RAMADDRESS (0x80000000)
-#define KZM_FPGA       (0xb4001040)
+#include "sysemu/qtest.h"
+
+/* Memory map for Kzm Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX31 SOC for support
+ * 0x80000000-0x8fffffff RAM                  EMULATED
+ * 0x90000000-0x9fffffff RAM                  EMULATED
+ * 0xa0000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb3ffffff Unavailable          IGNORED
+ * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
+ * 0xb4001000-0xb400100f Board control        IGNORED
+ *  0xb4001003           DIP switch
+ * 0xb4001010-0xb400101f 7-segment LED        IGNORED
+ * 0xb4001020-0xb400102f LED                  IGNORED
+ * 0xb4001030-0xb400103f LED                  IGNORED
+ * 0xb4001040-0xb400104f FPGA, UART           EMULATED
+ * 0xb4001050-0xb400105f FPGA, UART           EMULATED
+ * 0xb4001060-0xb40fffff FPGA                 IGNORED
+ * 0xb6000000-0xb61fffff LAN controller       EMULATED
+ * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
+ * 0xb6300000-0xb7ffffff Free                 IGNORED
+ * 0xb8000000-0xb8004fff Memory control registers IGNORED
+ * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
+ * 0xc4000000-0xffffffff Reserved             IGNORED
+ */
+
+typedef struct IMX31KZM {
+    FslIMX31State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX31KZM;
+
+#define KZM_RAM_ADDR            (FSL_IMX31_SDRAM0_ADDR)
+#define KZM_FPGA_ADDR           (FSL_IMX31_CS4_ADDR + 0x1040)
+#define KZM_LAN9118_ADDR        (FSL_IMX31_CS5_ADDR)
 
 static struct arm_boot_info kzm_binfo = {
-    .loader_start = KZM_RAMADDRESS,
+    .loader_start = KZM_RAM_ADDR,
     .board_id = 1722,
 };
 
 static void kzm_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
-    const char *kernel_filename = machine->kernel_filename;
-    const char *kernel_cmdline = machine->kernel_cmdline;
-    const char *initrd_filename = machine->initrd_filename;
-    ARMCPU *cpu;
-    MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    MemoryRegion *sram = g_new(MemoryRegion, 1);
-    MemoryRegion *ram_alias = g_new(MemoryRegion, 1);
-    DeviceState *dev;
-    DeviceState *ccm;
-
-    if (!cpu_model) {
-        cpu_model = "arm1136";
+    IMX31KZM *s = g_new0(IMX31KZM, 1);
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    unsigned int i;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX31);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* Check the amount of memory is compatible with the SOC */
+    if (machine->ram_size > (FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE;
     }
 
-    cpu = cpu_arm_init(cpu_model);
-    if (!cpu) {
-        fprintf(stderr, "Unable to find CPU definition\n");
-        exit(1);
+    memory_region_allocate_system_memory(&s->ram, NULL, "kzm.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX31_SDRAM0_ADDR, FSL_IMX31_SDRAM0_SIZE },
+            { FSL_IMX31_SDRAM1_ADDR, FSL_IMX31_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
     }
 
-    /* On a real system, the first 16k is a `secure boot rom' */
-
-    memory_region_allocate_system_memory(ram, NULL, "kzm.ram", ram_size);
-    memory_region_add_subregion(address_space_mem, KZM_RAMADDRESS, ram);
-
-    memory_region_init_alias(ram_alias, NULL, "ram.alias", ram, 0, ram_size);
-    memory_region_add_subregion(address_space_mem, 0x88000000, ram_alias);
-
-    memory_region_init_ram(sram, NULL, "kzm.sram", 0x4000, &error_abort);
-    memory_region_add_subregion(address_space_mem, 0x1FFFC000, sram);
-
-    dev = sysbus_create_varargs("imx_avic", 0x68000000,
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ),
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_FIQ),
-                                NULL);
-
-    imx_serial_create(0, 0x43f90000, qdev_get_gpio_in(dev, 45));
-    imx_serial_create(1, 0x43f94000, qdev_get_gpio_in(dev, 32));
-
-    ccm = sysbus_create_simple("imx_ccm", 0x53f80000, NULL);
-
-    imx_timerp_create(0x53f94000, qdev_get_gpio_in(dev, 28), ccm);
-    imx_timerp_create(0x53f98000, qdev_get_gpio_in(dev, 27), ccm);
-    imx_timerg_create(0x53f90000, qdev_get_gpio_in(dev, 29), ccm);
-
     if (nd_table[0].used) {
-        lan9118_init(&nd_table[0], 0xb6000000, qdev_get_gpio_in(dev, 52));
+        lan9118_init(&nd_table[0], KZM_LAN9118_ADDR,
+                     qdev_get_gpio_in(DEVICE(&s->soc.avic), 52));
     }
 
     if (serial_hds[2]) { /* touchscreen */
-        serial_mm_init(address_space_mem, KZM_FPGA+0x10, 0,
-                       qdev_get_gpio_in(dev, 52),
-                       14745600, serial_hds[2],
-                       DEVICE_NATIVE_ENDIAN);
+        serial_mm_init(get_system_memory(), KZM_FPGA_ADDR+0x10, 0,
+                       qdev_get_gpio_in(DEVICE(&s->soc.avic), 52),
+                       14745600, serial_hds[2], DEVICE_NATIVE_ENDIAN);
     }
 
-    kzm_binfo.ram_size = ram_size;
-    kzm_binfo.kernel_filename = kernel_filename;
-    kzm_binfo.kernel_cmdline = kernel_cmdline;
-    kzm_binfo.initrd_filename = initrd_filename;
+    kzm_binfo.ram_size = machine->ram_size;
+    kzm_binfo.kernel_filename = machine->kernel_filename;
+    kzm_binfo.kernel_cmdline = machine->kernel_cmdline;
+    kzm_binfo.initrd_filename = machine->initrd_filename;
     kzm_binfo.nb_cpus = 1;
-    arm_load_kernel(cpu, &kzm_binfo);
-}
 
-static QEMUMachine kzm_machine = {
-    .name = "kzm",
-    .desc = "ARM KZM Emulation Baseboard (ARM1136)",
-    .init = kzm_init,
-};
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &kzm_binfo);
+    }
+}
 
-static void kzm_machine_init(void)
+static void kzm_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&kzm_machine);
+    mc->desc = "ARM KZM Emulation Baseboard (ARM1136)";
+    mc->init = kzm_init;
 }
 
-machine_init(kzm_machine_init)
+DEFINE_MACHINE("kzm", kzm_machine_init)
diff --git a/qemu/hw/arm/mainstone.c b/qemu/hw/arm/mainstone.c
index 0da02a67e..454acc5d2 100644
--- a/qemu/hw/arm/mainstone.c
+++ b/qemu/hw/arm/mainstone.c
@@ -11,6 +11,8 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
@@ -124,7 +126,7 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
     /* Setup CPU & memory */
     mpu = pxa270_init(address_space_mem, mainstone_binfo.ram_size, cpu_model);
     memory_region_init_ram(rom, NULL, "mainstone.rom", MAINSTONE_ROM,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -188,15 +190,10 @@ static void mainstone_init(MachineState *machine)
     mainstone_common_init(get_system_memory(), machine, mainstone, 0x196);
 }
 
-static QEMUMachine mainstone2_machine = {
-    .name = "mainstone",
-    .desc = "Mainstone II (PXA27x)",
-    .init = mainstone_init,
-};
-
-static void mainstone_machine_init(void)
+static void mainstone2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mainstone2_machine);
+    mc->desc = "Mainstone II (PXA27x)";
+    mc->init = mainstone_init;
 }
 
-machine_init(mainstone_machine_init);
+DEFINE_MACHINE("mainstone", mainstone2_machine_init)
diff --git a/qemu/hw/arm/musicpal.c b/qemu/hw/arm/musicpal.c
index 42f66b33e..7a4cc07dd 100644
--- a/qemu/hw/arm/musicpal.c
+++ b/qemu/hw/arm/musicpal.c
@@ -9,6 +9,10 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
@@ -1599,7 +1603,7 @@ static void musicpal_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     memory_region_init_ram(sram, NULL, "musicpal.sram", MP_SRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, MP_SRAM_BASE, sram);
 
@@ -1709,18 +1713,13 @@ static void musicpal_init(MachineState *machine)
     arm_load_kernel(cpu, &musicpal_binfo);
 }
 
-static QEMUMachine musicpal_machine = {
-    .name = "musicpal",
-    .desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)",
-    .init = musicpal_init,
-};
-
-static void musicpal_machine_init(void)
+static void musicpal_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&musicpal_machine);
+    mc->desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)";
+    mc->init = musicpal_init;
 }
 
-machine_init(musicpal_machine_init);
+DEFINE_MACHINE("musicpal", musicpal_machine_init)
 
 static void mv88w8618_wlan_class_init(ObjectClass *klass, void *data)
 {
diff --git a/qemu/hw/arm/netduino2.c b/qemu/hw/arm/netduino2.c
index 8f26780ef..23d792837 100644
--- a/qemu/hw/arm/netduino2.c
+++ b/qemu/hw/arm/netduino2.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/boards.h"
 #include "qemu/error-report.h"
 #include "hw/arm/stm32f205_soc.h"
@@ -29,29 +31,19 @@
 static void netduino2_init(MachineState *machine)
 {
     DeviceState *dev;
-    Error *err = NULL;
 
     dev = qdev_create(NULL, TYPE_STM32F205_SOC);
     if (machine->kernel_filename) {
         qdev_prop_set_string(dev, "kernel-filename", machine->kernel_filename);
     }
     qdev_prop_set_string(dev, "cpu-model", "cortex-m3");
-    object_property_set_bool(OBJECT(dev), true, "realized", &err);
-    if (err != NULL) {
-        error_report("%s", error_get_pretty(err));
-        exit(1);
-    }
+    object_property_set_bool(OBJECT(dev), true, "realized", &error_fatal);
 }
 
-static QEMUMachine netduino2_machine = {
-    .name = "netduino2",
-    .desc = "Netduino 2 Machine",
-    .init = netduino2_init,
-};
-
-static void netduino2_machine_init(void)
+static void netduino2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&netduino2_machine);
+    mc->desc = "Netduino 2 Machine";
+    mc->init = netduino2_init;
 }
 
-machine_init(netduino2_machine_init);
+DEFINE_MACHINE("netduino2", netduino2_machine_init)
diff --git a/qemu/hw/arm/nseries.c b/qemu/hw/arm/nseries.c
index a659e8525..538250555 100644
--- a/qemu/hw/arm/nseries.c
+++ b/qemu/hw/arm/nseries.c
@@ -18,7 +18,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
 #include "hw/arm/omap.h"
 #include "hw/arm/arm.h"
@@ -172,8 +174,8 @@ static void n8x0_nand_setup(struct n800_s *s)
     qdev_prop_set_int32(s->nand, "shift", 1);
     dinfo = drive_get(IF_MTD, 0, 0);
     if (dinfo) {
-        qdev_prop_set_drive_nofail(s->nand, "drive",
-                                   blk_by_legacy_dinfo(dinfo));
+        qdev_prop_set_drive(s->nand, "drive", blk_by_legacy_dinfo(dinfo),
+                            &error_fatal);
     }
     qdev_init_nofail(s->nand);
     sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
@@ -1275,7 +1277,7 @@ static int n8x0_atag_setup(void *p, int model)
     strcpy((void *) w, "hw-build");		/* char component[12] */
     w += 6;
     strcpy((void *) w, "QEMU ");
-    pstrcat((void *) w, 12, qemu_get_version()); /* char version[12] */
+    pstrcat((void *) w, 12, qemu_hw_version()); /* char version[12] */
     w += 6;
 
     tag = (model == 810) ? "1.1.10-qemu" : "1.1.6-qemu";
@@ -1413,24 +1415,40 @@ static void n810_init(MachineState *machine)
     n8x0_init(machine, &n810_binfo, 810);
 }
 
-static QEMUMachine n800_machine = {
-    .name = "n800",
-    .desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)",
-    .init = n800_init,
-    .default_boot_order = "",
+static void n800_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)";
+    mc->init = n800_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n800_type = {
+    .name = MACHINE_TYPE_NAME("n800"),
+    .parent = TYPE_MACHINE,
+    .class_init = n800_class_init,
 };
 
-static QEMUMachine n810_machine = {
-    .name = "n810",
-    .desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)",
-    .init = n810_init,
-    .default_boot_order = "",
+static void n810_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)";
+    mc->init = n810_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n810_type = {
+    .name = MACHINE_TYPE_NAME("n810"),
+    .parent = TYPE_MACHINE,
+    .class_init = n810_class_init,
 };
 
 static void nseries_machine_init(void)
 {
-    qemu_register_machine(&n800_machine);
-    qemu_register_machine(&n810_machine);
+    type_register_static(&n800_type);
+    type_register_static(&n810_type);
 }
 
-machine_init(nseries_machine_init);
+type_init(nseries_machine_init)
diff --git a/qemu/hw/arm/omap1.c b/qemu/hw/arm/omap1.c
index de2b28925..b3cf0ec69 100644
--- a/qemu/hw/arm/omap1.c
+++ b/qemu/hw/arm/omap1.c
@@ -17,6 +17,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
@@ -27,6 +31,8 @@
 #include "sysemu/blockdev.h"
 #include "qemu/range.h"
 #include "hw/sysbus.h"
+#include "qemu/cutils.h"
+#include "qemu/bcd.h"
 
 /* Should signal the TCMI/GPMC */
 uint32_t omap_badwidth_read8(void *opaque, hwaddr addr)
@@ -106,7 +112,7 @@ static inline uint32_t omap_timer_read(struct omap_mpu_timer_s *timer)
 
     if (timer->st && timer->enable && timer->rate)
         return timer->val - muldiv64(distance >> (timer->ptv + 1),
-                                     timer->rate, get_ticks_per_sec());
+                                     timer->rate, NANOSECONDS_PER_SECOND);
     else
         return timer->val;
 }
@@ -124,7 +130,7 @@ static inline void omap_timer_update(struct omap_mpu_timer_s *timer)
     if (timer->enable && timer->st && timer->rate) {
         timer->val = timer->reset_val;	/* Should skip this on clk enable */
         expires = muldiv64((uint64_t) timer->val << (timer->ptv + 1),
-                           get_ticks_per_sec(), timer->rate);
+                           NANOSECONDS_PER_SECOND, timer->rate);
 
         /* If timer expiry would be sooner than in about 1 ms and
          * auto-reload isn't set, then fire immediately.  This is a hack
@@ -132,10 +138,11 @@ static inline void omap_timer_update(struct omap_mpu_timer_s *timer)
          * sets the interval to a very low value and polls the status bit
          * in a busy loop when it wants to sleep just a couple of CPU
          * ticks.  */
-        if (expires > (get_ticks_per_sec() >> 10) || timer->ar)
+        if (expires > (NANOSECONDS_PER_SECOND >> 10) || timer->ar) {
             timer_mod(timer->timer, timer->time + expires);
-        else
+        } else {
             qemu_bh_schedule(timer->tick);
+        }
     } else
         timer_del(timer->timer);
 }
@@ -258,8 +265,7 @@ static struct omap_mpu_timer_s *omap_mpu_timer_init(MemoryRegion *system_memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *)
-            g_malloc0(sizeof(struct omap_mpu_timer_s));
+    struct omap_mpu_timer_s *s = g_new0(struct omap_mpu_timer_s, 1);
 
     s->irq = irq;
     s->clk = clk;
@@ -388,8 +394,7 @@ static struct omap_watchdog_timer_s *omap_wd_timer_init(MemoryRegion *memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *)
-            g_malloc0(sizeof(struct omap_watchdog_timer_s));
+    struct omap_watchdog_timer_s *s = g_new0(struct omap_watchdog_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -495,8 +500,7 @@ static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *)
-            g_malloc0(sizeof(struct omap_32khz_timer_s));
+    struct omap_32khz_timer_s *s = g_new0(struct omap_32khz_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -615,14 +619,14 @@ static void omap_ulpd_pm_write(void *opaque, hwaddr addr,
                 now -= s->ulpd_gauge_start;
 
                 /* 32-kHz ticks */
-                ticks = muldiv64(now, 32768, get_ticks_per_sec());
+                ticks = muldiv64(now, 32768, NANOSECONDS_PER_SECOND);
                 s->ulpd_pm_regs[0x00 >> 2] = (ticks >>  0) & 0xffff;
                 s->ulpd_pm_regs[0x04 >> 2] = (ticks >> 16) & 0xffff;
                 if (ticks >> 32)	/* OVERFLOW_32K */
                     s->ulpd_pm_regs[0x14 >> 2] |= 1 << 2;
 
                 /* High frequency ticks */
-                ticks = muldiv64(now, 12000000, get_ticks_per_sec());
+                ticks = muldiv64(now, 12000000, NANOSECONDS_PER_SECOND);
                 s->ulpd_pm_regs[0x08 >> 2] = (ticks >>  0) & 0xffff;
                 s->ulpd_pm_regs[0x0c >> 2] = (ticks >> 16) & 0xffff;
                 if (ticks >> 32)	/* OVERFLOW_HI_FREQ */
@@ -1236,8 +1240,7 @@ static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
     MemoryRegion *memory, hwaddr base,
     qemu_irq abort_irq, omap_clk clk)
 {
-    struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *)
-            g_malloc0(sizeof(struct omap_tipb_bridge_s));
+    struct omap_tipb_bridge_s *s = g_new0(struct omap_tipb_bridge_s, 1);
 
     s->abort = abort_irq;
     omap_tipb_bridge_reset(s);
@@ -2099,8 +2102,7 @@ static struct omap_mpuio_s *omap_mpuio_init(MemoryRegion *memory,
                 qemu_irq kbd_int, qemu_irq gpio_int, qemu_irq wakeup,
                 omap_clk clk)
 {
-    struct omap_mpuio_s *s = (struct omap_mpuio_s *)
-            g_malloc0(sizeof(struct omap_mpuio_s));
+    struct omap_mpuio_s *s = g_new0(struct omap_mpuio_s, 1);
 
     s->irq = gpio_int;
     s->kbd_irq = kbd_int;
@@ -2292,8 +2294,7 @@ static struct omap_uwire_s *omap_uwire_init(MemoryRegion *system_memory,
                                             qemu_irq dma,
                                             omap_clk clk)
 {
-    struct omap_uwire_s *s = (struct omap_uwire_s *)
-            g_malloc0(sizeof(struct omap_uwire_s));
+    struct omap_uwire_s *s = g_new0(struct omap_uwire_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -2932,8 +2933,7 @@ static struct omap_rtc_s *omap_rtc_init(MemoryRegion *system_memory,
                                         qemu_irq timerirq, qemu_irq alarmirq,
                                         omap_clk clk)
 {
-    struct omap_rtc_s *s = (struct omap_rtc_s *)
-            g_malloc0(sizeof(struct omap_rtc_s));
+    struct omap_rtc_s *s = g_new0(struct omap_rtc_s, 1);
 
     s->irq = timerirq;
     s->alarm = alarmirq;
@@ -3032,7 +3032,7 @@ static void omap_mcbsp_source_tick(void *opaque)
 
     omap_mcbsp_rx_newdata(s);
     timer_mod(s->source_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec());
+                   NANOSECONDS_PER_SECOND);
 }
 
 static void omap_mcbsp_rx_start(struct omap_mcbsp_s *s)
@@ -3078,7 +3078,7 @@ static void omap_mcbsp_sink_tick(void *opaque)
 
     omap_mcbsp_tx_newdata(s);
     timer_mod(s->sink_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec());
+                   NANOSECONDS_PER_SECOND);
 }
 
 static void omap_mcbsp_tx_start(struct omap_mcbsp_s *s)
@@ -3468,8 +3468,7 @@ static struct omap_mcbsp_s *omap_mcbsp_init(MemoryRegion *system_memory,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq *dma, omap_clk clk)
 {
-    struct omap_mcbsp_s *s = (struct omap_mcbsp_s *)
-            g_malloc0(sizeof(struct omap_mcbsp_s));
+    struct omap_mcbsp_s *s = g_new0(struct omap_mcbsp_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -3648,8 +3647,7 @@ static void omap_lpg_clk_update(void *opaque, int line, int on)
 static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
                                         hwaddr base, omap_clk clk)
 {
-    struct omap_lpg_s *s = (struct omap_lpg_s *)
-            g_malloc0(sizeof(struct omap_lpg_s));
+    struct omap_lpg_s *s = g_new0(struct omap_lpg_s, 1);
 
     s->tm = timer_new_ms(QEMU_CLOCK_VIRTUAL, omap_lpg_tick, s);
 
@@ -3853,8 +3851,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
                 const char *core)
 {
     int i;
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[6];
     DriveInfo *dinfo;
     SysBusDevice *busdev;
@@ -3882,7 +3879,7 @@ struct omap_mpu_state_s *omap310_mpu_init(MemoryRegion *system_memory,
                                          s->sdram_size);
     memory_region_add_subregion(system_memory, OMAP_EMIFF_BASE, &s->emiff_ram);
     memory_region_init_ram(&s->imif_ram, NULL, "omap1.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->imif_ram);
     memory_region_add_subregion(system_memory, OMAP_IMIF_BASE, &s->imif_ram);
 
diff --git a/qemu/hw/arm/omap2.c b/qemu/hw/arm/omap2.c
index e39b31729..3a0d77714 100644
--- a/qemu/hw/arm/omap2.c
+++ b/qemu/hw/arm/omap2.c
@@ -18,6 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/boards.h"
@@ -596,8 +600,7 @@ static const MemoryRegionOps omap_eac_ops = {
 static struct omap_eac_s *omap_eac_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_eac_s *s = (struct omap_eac_s *)
-            g_malloc0(sizeof(struct omap_eac_s));
+    struct omap_eac_s *s = g_new0(struct omap_eac_s, 1);
 
     s->irq = irq;
     s->codec.rxdrq = *drq ++;
@@ -788,8 +791,7 @@ static struct omap_sti_s *omap_sti_init(struct omap_target_agent_s *ta,
                 hwaddr channel_base, qemu_irq irq, omap_clk clk,
                 CharDriverState *chr)
 {
-    struct omap_sti_s *s = (struct omap_sti_s *)
-            g_malloc0(sizeof(struct omap_sti_s));
+    struct omap_sti_s *s = g_new0(struct omap_sti_s, 1);
 
     s->irq = irq;
     omap_sti_reset(s);
@@ -1806,8 +1808,7 @@ static struct omap_prcm_s *omap_prcm_init(struct omap_target_agent_s *ta,
                 qemu_irq mpu_int, qemu_irq dsp_int, qemu_irq iva_int,
                 struct omap_mpu_state_s *mpu)
 {
-    struct omap_prcm_s *s = (struct omap_prcm_s *)
-            g_malloc0(sizeof(struct omap_prcm_s));
+    struct omap_prcm_s *s = g_new0(struct omap_prcm_s, 1);
 
     s->irq[0] = mpu_int;
     s->irq[1] = dsp_int;
@@ -2185,8 +2186,7 @@ static void omap_sysctl_reset(struct omap_sysctl_s *s)
 static struct omap_sysctl_s *omap_sysctl_init(struct omap_target_agent_s *ta,
                 omap_clk iclk, struct omap_mpu_state_s *mpu)
 {
-    struct omap_sysctl_s *s = (struct omap_sysctl_s *)
-            g_malloc0(sizeof(struct omap_sysctl_s));
+    struct omap_sysctl_s *s = g_new0(struct omap_sysctl_s, 1);
 
     s->mpu = mpu;
     omap_sysctl_reset(s);
@@ -2248,8 +2248,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                 unsigned long sdram_size,
                 const char *core)
 {
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[4];
     DriveInfo *dinfo;
     int i;
@@ -2276,7 +2275,7 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                                          s->sdram_size);
     memory_region_add_subregion(sysmem, OMAP2_Q2_BASE, &s->sdram);
     memory_region_init_ram(&s->sram, NULL, "omap2.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sram);
     memory_region_add_subregion(sysmem, OMAP2_SRAM_BASE, &s->sram);
 
diff --git a/qemu/hw/arm/omap_sx1.c b/qemu/hw/arm/omap_sx1.c
index 4b0f7f9c4..5d74026cb 100644
--- a/qemu/hw/arm/omap_sx1.c
+++ b/qemu/hw/arm/omap_sx1.c
@@ -25,6 +25,8 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -122,7 +124,7 @@ static void sx1_init(MachineState *machine, const int version)
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space, OMAP_CS0_BASE, flash);
@@ -166,7 +168,7 @@ static void sx1_init(MachineState *machine, const int version)
             (dinfo = drive_get(IF_PFLASH, 0, fl_idx)) != NULL) {
         MemoryRegion *flash_1 = g_new(MemoryRegion, 1);
         memory_region_init_ram(flash_1, NULL, "omap_sx1.flash1-0", flash1_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(flash_1);
         memory_region_set_readonly(flash_1, true);
         memory_region_add_subregion(address_space, OMAP_CS1_BASE, flash_1);
@@ -217,22 +219,38 @@ static void sx1_init_v2(MachineState *machine)
     sx1_init(machine, 2);
 }
 
-static QEMUMachine sx1_machine_v2 = {
-    .name = "sx1",
-    .desc = "Siemens SX1 (OMAP310) V2",
-    .init = sx1_init_v2,
+static void sx1_machine_v2_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V2";
+    mc->init = sx1_init_v2;
+}
+
+static const TypeInfo sx1_machine_v2_type = {
+    .name = MACHINE_TYPE_NAME("sx1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v2_class_init,
 };
 
-static QEMUMachine sx1_machine_v1 = {
-    .name = "sx1-v1",
-    .desc = "Siemens SX1 (OMAP310) V1",
-    .init = sx1_init_v1,
+static void sx1_machine_v1_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V1";
+    mc->init = sx1_init_v1;
+}
+
+static const TypeInfo sx1_machine_v1_type = {
+    .name = MACHINE_TYPE_NAME("sx1-v1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v1_class_init,
 };
 
 static void sx1_machine_init(void)
 {
-    qemu_register_machine(&sx1_machine_v2);
-    qemu_register_machine(&sx1_machine_v1);
+    type_register_static(&sx1_machine_v1_type);
+    type_register_static(&sx1_machine_v2_type);
 }
 
-machine_init(sx1_machine_init);
+type_init(sx1_machine_init)
diff --git a/qemu/hw/arm/palm.c b/qemu/hw/arm/palm.c
index 7f1cfb8f6..7f460732e 100644
--- a/qemu/hw/arm/palm.c
+++ b/qemu/hw/arm/palm.c
@@ -16,6 +16,8 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "sysemu/sysemu.h"
@@ -213,7 +215,7 @@ static void palmte_init(MachineState *machine)
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "palmte.flash", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space_mem, OMAP_CS0_BASE, flash);
@@ -269,15 +271,10 @@ static void palmte_init(MachineState *machine)
     arm_load_kernel(mpu->cpu, &palmte_binfo);
 }
 
-static QEMUMachine palmte_machine = {
-    .name = "cheetah",
-    .desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)",
-    .init = palmte_init,
-};
-
-static void palmte_machine_init(void)
+static void palmte_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&palmte_machine);
+    mc->desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)";
+    mc->init = palmte_init;
 }
 
-machine_init(palmte_machine_init);
+DEFINE_MACHINE("cheetah", palmte_machine_init)
diff --git a/qemu/hw/arm/palmetto-bmc.c b/qemu/hw/arm/palmetto-bmc.c
new file mode 100644
index 000000000..89ebd92b9
--- /dev/null
+++ b/qemu/hw/arm/palmetto-bmc.c
@@ -0,0 +1,68 @@
+/*
+ * OpenPOWER Palmetto BMC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/address-spaces.h"
+#include "hw/arm/arm.h"
+#include "hw/arm/ast2400.h"
+#include "hw/boards.h"
+
+static struct arm_boot_info palmetto_bmc_binfo = {
+    .loader_start = AST2400_SDRAM_BASE,
+    .board_id = 0,
+    .nb_cpus = 1,
+};
+
+typedef struct PalmettoBMCState {
+    AST2400State soc;
+    MemoryRegion ram;
+} PalmettoBMCState;
+
+static void palmetto_bmc_init(MachineState *machine)
+{
+    PalmettoBMCState *bmc;
+
+    bmc = g_new0(PalmettoBMCState, 1);
+    object_initialize(&bmc->soc, (sizeof(bmc->soc)), TYPE_AST2400);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&bmc->soc),
+                              &error_abort);
+
+    memory_region_allocate_system_memory(&bmc->ram, NULL, "ram", ram_size);
+    memory_region_add_subregion(get_system_memory(), AST2400_SDRAM_BASE,
+                                &bmc->ram);
+    object_property_add_const_link(OBJECT(&bmc->soc), "ram", OBJECT(&bmc->ram),
+                                   &error_abort);
+    object_property_set_bool(OBJECT(&bmc->soc), true, "realized",
+                             &error_abort);
+
+    palmetto_bmc_binfo.kernel_filename = machine->kernel_filename;
+    palmetto_bmc_binfo.initrd_filename = machine->initrd_filename;
+    palmetto_bmc_binfo.kernel_cmdline = machine->kernel_cmdline;
+    palmetto_bmc_binfo.ram_size = ram_size;
+    arm_load_kernel(ARM_CPU(first_cpu), &palmetto_bmc_binfo);
+}
+
+static void palmetto_bmc_machine_init(MachineClass *mc)
+{
+    mc->desc = "OpenPOWER Palmetto BMC";
+    mc->init = palmetto_bmc_init;
+    mc->max_cpus = 1;
+    mc->no_sdcard = 1;
+    mc->no_floppy = 1;
+    mc->no_cdrom = 1;
+    mc->no_sdcard = 1;
+    mc->no_parallel = 1;
+}
+
+DEFINE_MACHINE("palmetto-bmc", palmetto_bmc_machine_init);
diff --git a/qemu/hw/arm/pxa2xx.c b/qemu/hw/arm/pxa2xx.c
index ec353f79c..1a8c36033 100644
--- a/qemu/hw/arm/pxa2xx.c
+++ b/qemu/hw/arm/pxa2xx.c
@@ -7,15 +7,20 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
 #include "sysemu/sysemu.h"
 #include "hw/char/serial.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "sysemu/char.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
+#include "qemu/cutils.h"
 
 static struct {
     hwaddr io_base;
@@ -1731,8 +1736,7 @@ static PXA2xxI2SState *pxa2xx_i2s_init(MemoryRegion *sysmem,
                 hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma)
 {
-    PXA2xxI2SState *s = (PXA2xxI2SState *)
-            g_malloc0(sizeof(PXA2xxI2SState));
+    PXA2xxI2SState *s = g_new0(PXA2xxI2SState, 1);
 
     s->irq = irq;
     s->rx_dma = rx_dma;
@@ -1959,7 +1963,7 @@ static void pxa2xx_fir_instance_init(Object *obj)
     PXA2xxFIrState *s = PXA2XX_FIR(obj);
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_fir_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_fir_ops, s,
                           "pxa2xx-fir", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
@@ -2061,7 +2065,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
     PXA2xxState *s;
     int i;
     DriveInfo *dinfo;
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     if (revision && strncmp(revision, "pxa27", 5)) {
         fprintf(stderr, "Machine requires a PXA27x processor.\n");
@@ -2079,11 +2083,11 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa270.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa270.internal", 0x40000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2157,7 +2161,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa27x_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa27x_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa27x_ssp[i].io_base,
@@ -2202,7 +2206,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
     int i;
     DriveInfo *dinfo;
 
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     s->cpu = cpu_arm_init("pxa255");
     if (s->cpu == NULL) {
@@ -2213,11 +2217,11 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa255.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa255.internal",
-                           PXA2XX_INTERNAL_SIZE, &error_abort);
+                           PXA2XX_INTERNAL_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2290,7 +2294,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa255_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa255_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa255_ssp[i].io_base,
diff --git a/qemu/hw/arm/pxa2xx_gpio.c b/qemu/hw/arm/pxa2xx_gpio.c
index c89c8045c..67e7e7094 100644
--- a/qemu/hw/arm/pxa2xx_gpio.c
+++ b/qemu/hw/arm/pxa2xx_gpio.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
diff --git a/qemu/hw/arm/pxa2xx_pic.c b/qemu/hw/arm/pxa2xx_pic.c
index d41ac9341..7e51532cd 100644
--- a/qemu/hw/arm/pxa2xx_pic.c
+++ b/qemu/hw/arm/pxa2xx_pic.c
@@ -8,6 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/sysbus.h"
diff --git a/qemu/hw/arm/raspi.c b/qemu/hw/arm/raspi.c
new file mode 100644
index 000000000..2b295f14c
--- /dev/null
+++ b/qemu/hw/arm/raspi.c
@@ -0,0 +1,172 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/bcm2836.h"
+#include "qemu/error-report.h"
+#include "hw/boards.h"
+#include "hw/loader.h"
+#include "hw/arm/arm.h"
+#include "sysemu/sysemu.h"
+
+#define SMPBOOT_ADDR    0x300 /* this should leave enough space for ATAGS */
+#define MVBAR_ADDR      0x400 /* secure vectors */
+#define BOARDSETUP_ADDR (MVBAR_ADDR + 0x20) /* board setup code */
+#define FIRMWARE_ADDR   0x8000 /* Pi loads kernel.img here by default */
+
+/* Table of Linux board IDs for different Pi versions */
+static const int raspi_boardid[] = {[1] = 0xc42, [2] = 0xc43};
+
+typedef struct RasPiState {
+    BCM2836State soc;
+    MemoryRegion ram;
+} RasPiState;
+
+static void write_smpboot(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    static const uint32_t smpboot[] = {
+        0xe1a0e00f, /*    mov     lr, pc */
+        0xe3a0fe00 + (BOARDSETUP_ADDR >> 4), /* mov pc, BOARDSETUP_ADDR */
+        0xee100fb0, /*    mrc     p15, 0, r0, c0, c0, 5;get core ID */
+        0xe7e10050, /*    ubfx    r0, r0, #0, #2       ;extract LSB */
+        0xe59f5014, /*    ldr     r5, =0x400000CC      ;load mbox base */
+        0xe320f001, /* 1: yield */
+        0xe7953200, /*    ldr     r3, [r5, r0, lsl #4] ;read mbox for our core*/
+        0xe3530000, /*    cmp     r3, #0               ;spin while zero */
+        0x0afffffb, /*    beq     1b */
+        0xe7853200, /*    str     r3, [r5, r0, lsl #4] ;clear mbox */
+        0xe12fff13, /*    bx      r3                   ;jump to target */
+        0x400000cc, /* (constant: mailbox 3 read/clear base) */
+    };
+
+    /* check that we don't overrun board setup vectors */
+    QEMU_BUILD_BUG_ON(SMPBOOT_ADDR + sizeof(smpboot) > MVBAR_ADDR);
+    /* check that board setup address is correctly relocated */
+    QEMU_BUILD_BUG_ON((BOARDSETUP_ADDR & 0xf) != 0
+                      || (BOARDSETUP_ADDR >> 4) >= 0x100);
+
+    rom_add_blob_fixed("raspi_smpboot", smpboot, sizeof(smpboot),
+                       info->smp_loader_start);
+}
+
+static void write_board_setup(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    arm_write_secure_board_setup_dummy_smc(cpu, info, MVBAR_ADDR);
+}
+
+static void reset_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    CPUState *cs = CPU(cpu);
+    cpu_set_pc(cs, info->smp_loader_start);
+}
+
+static void setup_boot(MachineState *machine, int version, size_t ram_size)
+{
+    static struct arm_boot_info binfo;
+    int r;
+
+    binfo.board_id = raspi_boardid[version];
+    binfo.ram_size = ram_size;
+    binfo.nb_cpus = smp_cpus;
+    binfo.board_setup_addr = BOARDSETUP_ADDR;
+    binfo.write_board_setup = write_board_setup;
+    binfo.secure_board_setup = true;
+    binfo.secure_boot = true;
+
+    /* Pi2 requires SMP setup */
+    if (version == 2) {
+        binfo.smp_loader_start = SMPBOOT_ADDR;
+        binfo.write_secondary_boot = write_smpboot;
+        binfo.secondary_cpu_reset_hook = reset_secondary;
+    }
+
+    /* If the user specified a "firmware" image (e.g. UEFI), we bypass
+     * the normal Linux boot process
+     */
+    if (machine->firmware) {
+        /* load the firmware image (typically kernel.img) */
+        r = load_image_targphys(machine->firmware, FIRMWARE_ADDR,
+                                ram_size - FIRMWARE_ADDR);
+        if (r < 0) {
+            error_report("Failed to load firmware from %s", machine->firmware);
+            exit(1);
+        }
+
+        binfo.entry = FIRMWARE_ADDR;
+        binfo.firmware_loaded = true;
+    } else {
+        binfo.kernel_filename = machine->kernel_filename;
+        binfo.kernel_cmdline = machine->kernel_cmdline;
+        binfo.initrd_filename = machine->initrd_filename;
+    }
+
+    arm_load_kernel(ARM_CPU(first_cpu), &binfo);
+}
+
+static void raspi2_init(MachineState *machine)
+{
+    RasPiState *s = g_new0(RasPiState, 1);
+    uint32_t vcram_size;
+    DriveInfo *di;
+    BlockBackend *blk;
+    BusState *bus;
+    DeviceState *carddev;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_BCM2836);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    /* Allocate and map RAM */
+    memory_region_allocate_system_memory(&s->ram, OBJECT(machine), "ram",
+                                         machine->ram_size);
+    /* FIXME: Remove when we have custom CPU address space support */
+    memory_region_add_subregion_overlap(get_system_memory(), 0, &s->ram, 0);
+
+    /* Setup the SOC */
+    object_property_add_const_link(OBJECT(&s->soc), "ram", OBJECT(&s->ram),
+                                   &error_abort);
+    object_property_set_int(OBJECT(&s->soc), smp_cpus, "enabled-cpus",
+                            &error_abort);
+    object_property_set_int(OBJECT(&s->soc), 0xa21041, "board-rev",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_abort);
+
+    /* Create and plug in the SD cards */
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    bus = qdev_get_child_bus(DEVICE(&s->soc), "sd-bus");
+    if (bus == NULL) {
+        error_report("No SD bus found in SOC object");
+        exit(1);
+    }
+    carddev = qdev_create(bus, TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
+    vcram_size = object_property_get_int(OBJECT(&s->soc), "vcram-size",
+                                         &error_abort);
+    setup_boot(machine, 2, machine->ram_size - vcram_size);
+}
+
+static void raspi2_machine_init(MachineClass *mc)
+{
+    mc->desc = "Raspberry Pi 2";
+    mc->init = raspi2_init;
+    mc->block_default_type = IF_SD;
+    mc->no_parallel = 1;
+    mc->no_floppy = 1;
+    mc->no_cdrom = 1;
+    mc->max_cpus = BCM2836_NCPUS;
+    mc->default_ram_size = 1024 * 1024 * 1024;
+};
+DEFINE_MACHINE("raspi2", raspi2_machine_init)
diff --git a/qemu/hw/arm/realview.c b/qemu/hw/arm/realview.c
index ef2788d3e..3222b360e 100644
--- a/qemu/hw/arm/realview.c
+++ b/qemu/hw/arm/realview.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -99,33 +103,21 @@ static void realview_init(MachineState *machine,
 
     for (n = 0; n < smp_cpus; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
 
         /* By default A9,A15 and ARM1176 CPUs have EL3 enabled.  This board
          * does not currently support EL3 so the CPU EL3 property is disabled
          * before realization.
          */
         if (object_property_find(cpuobj, "has_el3", NULL)) {
-            object_property_set_bool(cpuobj, false, "has_el3", &err);
-            if (err) {
-                error_report_err(err);
-                exit(1);
-            }
+            object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
         }
 
         if (is_pb && is_mpcore) {
-            object_property_set_int(cpuobj, periphbase, "reset-cbar", &err);
-            if (err) {
-                error_report_err(err);
-                exit(1);
-            }
+            object_property_set_int(cpuobj, periphbase, "reset-cbar",
+                                    &error_fatal);
         }
 
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
 
         cpu_irq[n] = qdev_get_gpio_in(DEVICE(cpuobj), ARM_CPU_IRQ);
     }
@@ -151,13 +143,13 @@ static void realview_init(MachineState *machine,
         low_ram_size = ram_size - 0x20000000;
         ram_size = 0x20000000;
         memory_region_init_ram(ram_lo, NULL, "realview.lowmem", low_ram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(ram_lo);
         memory_region_add_subregion(sysmem, 0x20000000, ram_lo);
     }
 
     memory_region_init_ram(ram_hi, NULL, "realview.highmem", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hi);
     low_ram_size = ram_size;
     if (low_ram_size > 0x10000000)
@@ -353,7 +345,7 @@ static void realview_init(MachineState *machine,
        BootROM happens to be in ROM/flash or in memory that isn't clobbered
        until after Linux boots the secondary CPUs.  */
     memory_region_init_ram(ram_hack, NULL, "realview.hack", 0x1000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hack);
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
@@ -399,41 +391,73 @@ static void realview_pbx_a9_init(MachineState *machine)
     realview_init(machine, BOARD_PBX_A9);
 }
 
-static QEMUMachine realview_eb_machine = {
-    .name = "realview-eb",
-    .desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)",
-    .init = realview_eb_init,
-    .block_default_type = IF_SCSI,
+static void realview_eb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)";
+    mc->init = realview_eb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo realview_eb_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_class_init,
 };
 
-static QEMUMachine realview_eb_mpcore_machine = {
-    .name = "realview-eb-mpcore",
-    .desc = "ARM RealView Emulation Baseboard (ARM11MPCore)",
-    .init = realview_eb_mpcore_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_eb_mpcore_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM11MPCore)";
+    mc->init = realview_eb_mpcore_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_eb_mpcore_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb-mpcore"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_mpcore_class_init,
 };
 
-static QEMUMachine realview_pb_a8_machine = {
-    .name = "realview-pb-a8",
-    .desc = "ARM RealView Platform Baseboard for Cortex-A8",
-    .init = realview_pb_a8_init,
+static void realview_pb_a8_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard for Cortex-A8";
+    mc->init = realview_pb_a8_init;
+}
+
+static const TypeInfo realview_pb_a8_type = {
+    .name = MACHINE_TYPE_NAME("realview-pb-a8"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pb_a8_class_init,
 };
 
-static QEMUMachine realview_pbx_a9_machine = {
-    .name = "realview-pbx-a9",
-    .desc = "ARM RealView Platform Baseboard Explore for Cortex-A9",
-    .init = realview_pbx_a9_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_pbx_a9_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard Explore for Cortex-A9";
+    mc->init = realview_pbx_a9_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_pbx_a9_type = {
+    .name = MACHINE_TYPE_NAME("realview-pbx-a9"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pbx_a9_class_init,
 };
 
 static void realview_machine_init(void)
 {
-    qemu_register_machine(&realview_eb_machine);
-    qemu_register_machine(&realview_eb_mpcore_machine);
-    qemu_register_machine(&realview_pb_a8_machine);
-    qemu_register_machine(&realview_pbx_a9_machine);
+    type_register_static(&realview_eb_type);
+    type_register_static(&realview_eb_mpcore_type);
+    type_register_static(&realview_pb_a8_type);
+    type_register_static(&realview_pbx_a9_type);
 }
 
-machine_init(realview_machine_init);
+type_init(realview_machine_init)
diff --git a/qemu/hw/arm/spitz.c b/qemu/hw/arm/spitz.c
index 5bf032a63..bf61d63b5 100644
--- a/qemu/hw/arm/spitz.c
+++ b/qemu/hw/arm/spitz.c
@@ -10,13 +10,15 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
 #include "sysemu/sysemu.h"
 #include "hw/pcmcia.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/block/flash.h"
 #include "qemu/timer.h"
 #include "hw/devices.h"
@@ -403,7 +405,7 @@ static void spitz_keyboard_tick(void *opaque)
     }
 
     timer_mod(s->kbdtimer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 32);
+                   NANOSECONDS_PER_SECOND / 32);
 }
 
 static void spitz_keyboard_pre_map(SpitzKeyboardState *s)
@@ -913,7 +915,7 @@ static void spitz_common_init(MachineState *machine,
 
     sl_flash_register(mpu, (model == spitz) ? FLASH_128M : FLASH_1024M);
 
-    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -972,39 +974,71 @@ static void terrier_init(MachineState *machine)
     spitz_common_init(machine, terrier, 0x33f);
 }
 
-static QEMUMachine akitapda_machine = {
-    .name = "akita",
-    .desc = "Akita PDA (PXA270)",
-    .init = akita_init,
+static void akitapda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C1000 (Akita) PDA (PXA270)";
+    mc->init = akita_init;
+}
+
+static const TypeInfo akitapda_type = {
+    .name = MACHINE_TYPE_NAME("akita"),
+    .parent = TYPE_MACHINE,
+    .class_init = akitapda_class_init,
 };
 
-static QEMUMachine spitzpda_machine = {
-    .name = "spitz",
-    .desc = "Spitz PDA (PXA270)",
-    .init = spitz_init,
+static void spitzpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3000 (Spitz) PDA (PXA270)";
+    mc->init = spitz_init;
+}
+
+static const TypeInfo spitzpda_type = {
+    .name = MACHINE_TYPE_NAME("spitz"),
+    .parent = TYPE_MACHINE,
+    .class_init = spitzpda_class_init,
 };
 
-static QEMUMachine borzoipda_machine = {
-    .name = "borzoi",
-    .desc = "Borzoi PDA (PXA270)",
-    .init = borzoi_init,
+static void borzoipda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3100 (Borzoi) PDA (PXA270)";
+    mc->init = borzoi_init;
+}
+
+static const TypeInfo borzoipda_type = {
+    .name = MACHINE_TYPE_NAME("borzoi"),
+    .parent = TYPE_MACHINE,
+    .class_init = borzoipda_class_init,
 };
 
-static QEMUMachine terrierpda_machine = {
-    .name = "terrier",
-    .desc = "Terrier PDA (PXA270)",
-    .init = terrier_init,
+static void terrierpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3200 (Terrier) PDA (PXA270)";
+    mc->init = terrier_init;
+}
+
+static const TypeInfo terrierpda_type = {
+    .name = MACHINE_TYPE_NAME("terrier"),
+    .parent = TYPE_MACHINE,
+    .class_init = terrierpda_class_init,
 };
 
 static void spitz_machine_init(void)
 {
-    qemu_register_machine(&akitapda_machine);
-    qemu_register_machine(&spitzpda_machine);
-    qemu_register_machine(&borzoipda_machine);
-    qemu_register_machine(&terrierpda_machine);
+    type_register_static(&akitapda_type);
+    type_register_static(&spitzpda_type);
+    type_register_static(&borzoipda_type);
+    type_register_static(&terrierpda_type);
 }
 
-machine_init(spitz_machine_init);
+type_init(spitz_machine_init)
 
 static bool is_version_0(void *opaque, int version_id)
 {
@@ -1060,10 +1094,6 @@ static VMStateDescription vmstate_spitz_kbd = {
     },
 };
 
-static Property spitz_keyboard_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -1071,7 +1101,6 @@ static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
 
     k->init = spitz_keyboard_init;
     dc->vmsd = &vmstate_spitz_kbd;
-    dc->props = spitz_keyboard_properties;
 }
 
 static const TypeInfo spitz_keyboard_info = {
diff --git a/qemu/hw/arm/stellaris.c b/qemu/hw/arm/stellaris.c
index cb515ec76..c1766f856 100644
--- a/qemu/hw/arm/stellaris.c
+++ b/qemu/hw/arm/stellaris.c
@@ -7,8 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "qemu/timer.h"
@@ -16,6 +18,7 @@
 #include "net/net.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 #define GPIO_A 0
 #define GPIO_B 1
@@ -98,7 +101,7 @@ static void gptm_reload(gptm_state *s, int n, int reset)
         tick += (int64_t)count * system_clock_scale;
     } else if (s->config == 1) {
         /* 32-bit RTC.  1Hz tick.  */
-        tick += get_ticks_per_sec();
+        tick += NANOSECONDS_PER_SECOND;
     } else if (s->mode[n] == 0xa) {
         /* PWM mode.  Not implemented.  */
     } else {
@@ -675,7 +678,7 @@ static int stellaris_sys_init(uint32_t base, qemu_irq irq,
 {
     ssys_state *s;
 
-    s = (ssys_state *)g_malloc0(sizeof(ssys_state));
+    s = g_new0(ssys_state, 1);
     s->irq = irq;
     s->board = board;
     /* Most devices come preprogrammed with a MAC address in the user data. */
@@ -1176,6 +1179,14 @@ static int stellaris_adc_init(SysBusDevice *sbd)
     return 0;
 }
 
+static
+void do_sys_reset(void *opaque, int n, int level)
+{
+    if (level) {
+        qemu_system_reset_request();
+    }
+}
+
 /* Board init.  */
 static stellaris_board_info stellaris_boards[] = {
   { "LM3S811EVB",
@@ -1210,8 +1221,7 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         0x40024000, 0x40025000, 0x40026000};
     static const int gpio_irq[7] = {0, 1, 2, 3, 4, 30, 31};
 
-    qemu_irq *pic;
-    DeviceState *gpio_dev[7];
+    DeviceState *gpio_dev[7], *nvic;
     qemu_irq gpio_in[7][8];
     qemu_irq gpio_out[7][8];
     qemu_irq adc;
@@ -1231,22 +1241,29 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
 
     /* Flash programming is done via the SCU, so pretend it is ROM.  */
     memory_region_init_ram(flash, NULL, "stellaris.flash", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(system_memory, 0, flash);
 
     memory_region_init_ram(sram, NULL, "stellaris.sram", sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(system_memory, 0x20000000, sram);
 
-    pic = armv7m_init(system_memory, flash_size, NUM_IRQ_LINES,
+    nvic = armv7m_init(system_memory, flash_size, NUM_IRQ_LINES,
                       kernel_filename, cpu_model);
 
+    qdev_connect_gpio_out_named(nvic, "SYSRESETREQ", 0,
+                                qemu_allocate_irq(&do_sys_reset, NULL, 0));
+
     if (board->dc1 & (1 << 16)) {
         dev = sysbus_create_varargs(TYPE_STELLARIS_ADC, 0x40038000,
-                                    pic[14], pic[15], pic[16], pic[17], NULL);
+                                    qdev_get_gpio_in(nvic, 14),
+                                    qdev_get_gpio_in(nvic, 15),
+                                    qdev_get_gpio_in(nvic, 16),
+                                    qdev_get_gpio_in(nvic, 17),
+                                    NULL);
         adc = qdev_get_gpio_in(dev, 0);
     } else {
         adc = NULL;
@@ -1255,19 +1272,21 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         if (board->dc2 & (0x10000 << i)) {
             dev = sysbus_create_simple(TYPE_STELLARIS_GPTM,
                                        0x40030000 + i * 0x1000,
-                                       pic[timer_irq[i]]);
+                                       qdev_get_gpio_in(nvic, timer_irq[i]));
             /* TODO: This is incorrect, but we get away with it because
                the ADC output is only ever pulsed.  */
             qdev_connect_gpio_out(dev, 0, adc);
         }
     }
 
-    stellaris_sys_init(0x400fe000, pic[28], board, nd_table[0].macaddr.a);
+    stellaris_sys_init(0x400fe000, qdev_get_gpio_in(nvic, 28),
+                       board, nd_table[0].macaddr.a);
 
     for (i = 0; i < 7; i++) {
         if (board->dc4 & (1 << i)) {
             gpio_dev[i] = sysbus_create_simple("pl061_luminary", gpio_addr[i],
-                                               pic[gpio_irq[i]]);
+                                               qdev_get_gpio_in(nvic,
+                                                                gpio_irq[i]));
             for (j = 0; j < 8; j++) {
                 gpio_in[i][j] = qdev_get_gpio_in(gpio_dev[i], j);
                 gpio_out[i][j] = NULL;
@@ -1276,7 +1295,8 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     }
 
     if (board->dc2 & (1 << 12)) {
-        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000, pic[8]);
+        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000,
+                                   qdev_get_gpio_in(nvic, 8));
         i2c = (I2CBus *)qdev_get_child_bus(dev, "i2c");
         if (board->peripherals & BP_OLED_I2C) {
             i2c_create_slave(i2c, "ssd0303", 0x3d);
@@ -1286,11 +1306,12 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     for (i = 0; i < 4; i++) {
         if (board->dc2 & (1 << i)) {
             sysbus_create_simple("pl011_luminary", 0x4000c000 + i * 0x1000,
-                                 pic[uart_irq[i]]);
+                                 qdev_get_gpio_in(nvic, uart_irq[i]));
         }
     }
     if (board->dc2 & (1 << 4)) {
-        dev = sysbus_create_simple("pl022", 0x40008000, pic[7]);
+        dev = sysbus_create_simple("pl022", 0x40008000,
+                                   qdev_get_gpio_in(nvic, 7));
         if (board->peripherals & BP_OLED_SSI) {
             void *bus;
             DeviceState *sddev;
@@ -1326,7 +1347,7 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
         qdev_set_nic_properties(enet, &nd_table[0]);
         qdev_init_nofail(enet);
         sysbus_mmio_map(SYS_BUS_DEVICE(enet), 0, 0x40048000);
-        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, pic[42]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, qdev_get_gpio_in(nvic, 42));
     }
     if (board->peripherals & BP_GAMEPAD) {
         qemu_irq gpad_irq[5];
@@ -1366,25 +1387,41 @@ static void lm3s6965evb_init(MachineState *machine)
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
-static QEMUMachine lm3s811evb_machine = {
-    .name = "lm3s811evb",
-    .desc = "Stellaris LM3S811EVB",
-    .init = lm3s811evb_init,
+static void lm3s811evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S811EVB";
+    mc->init = lm3s811evb_init;
+}
+
+static const TypeInfo lm3s811evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s811evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s811evb_class_init,
 };
 
-static QEMUMachine lm3s6965evb_machine = {
-    .name = "lm3s6965evb",
-    .desc = "Stellaris LM3S6965EVB",
-    .init = lm3s6965evb_init,
+static void lm3s6965evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S6965EVB";
+    mc->init = lm3s6965evb_init;
+}
+
+static const TypeInfo lm3s6965evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s6965evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s6965evb_class_init,
 };
 
 static void stellaris_machine_init(void)
 {
-    qemu_register_machine(&lm3s811evb_machine);
-    qemu_register_machine(&lm3s6965evb_machine);
+    type_register_static(&lm3s811evb_type);
+    type_register_static(&lm3s6965evb_type);
 }
 
-machine_init(stellaris_machine_init);
+type_init(stellaris_machine_init)
 
 static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
 {
diff --git a/qemu/hw/arm/stm32f205_soc.c b/qemu/hw/arm/stm32f205_soc.c
index 0f3bdc77b..a5ea1e237 100644
--- a/qemu/hw/arm/stm32f205_soc.c
+++ b/qemu/hw/arm/stm32f205_soc.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/arm/arm.h"
 #include "exec/address-spaces.h"
 #include "hw/arm/stm32f205_soc.h"
@@ -59,9 +63,8 @@ static void stm32f205_soc_initfn(Object *obj)
 static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
 {
     STM32F205State *s = STM32F205_SOC(dev_soc);
-    DeviceState *syscfgdev, *usartdev, *timerdev;
+    DeviceState *syscfgdev, *usartdev, *timerdev, *nvic;
     SysBusDevice *syscfgbusdev, *usartbusdev, *timerbusdev;
-    qemu_irq *pic;
     Error *err = NULL;
     int i;
 
@@ -71,7 +74,7 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     MemoryRegion *flash_alias = g_new(MemoryRegion, 1);
 
     memory_region_init_ram(flash, NULL, "STM32F205.flash", FLASH_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_init_alias(flash_alias, NULL, "STM32F205.flash.alias",
                              flash, 0, FLASH_SIZE);
 
@@ -84,12 +87,12 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     memory_region_add_subregion(system_memory, 0, flash_alias);
 
     memory_region_init_ram(sram, NULL, "STM32F205.sram", SRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(system_memory, SRAM_BASE_ADDRESS, sram);
 
-    pic = armv7m_init(get_system_memory(), FLASH_SIZE, 96,
-                      s->kernel_filename, s->cpu_model);
+    nvic = armv7m_init(get_system_memory(), FLASH_SIZE, 96,
+                       s->kernel_filename, s->cpu_model);
 
     /* System configuration controller */
     syscfgdev = DEVICE(&s->syscfg);
@@ -100,7 +103,7 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
     }
     syscfgbusdev = SYS_BUS_DEVICE(syscfgdev);
     sysbus_mmio_map(syscfgbusdev, 0, 0x40013800);
-    sysbus_connect_irq(syscfgbusdev, 0, pic[71]);
+    sysbus_connect_irq(syscfgbusdev, 0, qdev_get_gpio_in(nvic, 71));
 
     /* Attach UART (uses USART registers) and USART controllers */
     for (i = 0; i < STM_NUM_USARTS; i++) {
@@ -112,7 +115,8 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
         }
         usartbusdev = SYS_BUS_DEVICE(usartdev);
         sysbus_mmio_map(usartbusdev, 0, usart_addr[i]);
-        sysbus_connect_irq(usartbusdev, 0, pic[usart_irq[i]]);
+        sysbus_connect_irq(usartbusdev, 0,
+                           qdev_get_gpio_in(nvic, usart_irq[i]));
     }
 
     /* Timer 2 to 5 */
@@ -126,7 +130,8 @@ static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
         }
         timerbusdev = SYS_BUS_DEVICE(timerdev);
         sysbus_mmio_map(timerbusdev, 0, timer_addr[i]);
-        sysbus_connect_irq(timerbusdev, 0, pic[timer_irq[i]]);
+        sysbus_connect_irq(timerbusdev, 0,
+                           qdev_get_gpio_in(nvic, timer_irq[i]));
     }
 }
 
diff --git a/qemu/hw/arm/strongarm.c b/qemu/hw/arm/strongarm.c
index da9fc1d51..1eeb1ab39 100644
--- a/qemu/hw/arm/strongarm.c
+++ b/qemu/hw/arm/strongarm.c
@@ -27,6 +27,8 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "hw/sysbus.h"
 #include "strongarm.h"
@@ -34,7 +36,8 @@
 #include "hw/arm/arm.h"
 #include "sysemu/char.h"
 #include "sysemu/sysemu.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG
 
@@ -1023,7 +1026,7 @@ static void strongarm_uart_update_parameters(StrongARMUARTState *s)
     ssp.parity = parity;
     ssp.data_bits = data_bits;
     ssp.stop_bits = stop_bits;
-    s->char_transmit_time =  (get_ticks_per_sec() / speed) * frame_size;
+    s->char_transmit_time =  (NANOSECONDS_PER_SECOND / speed) * frame_size;
     if (s->chr) {
         qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
     }
@@ -1588,7 +1591,7 @@ StrongARMState *sa1110_init(MemoryRegion *sysmem,
     StrongARMState *s;
     int i;
 
-    s = g_malloc0(sizeof(StrongARMState));
+    s = g_new0(StrongARMState, 1);
 
     if (!rev) {
         rev = "sa1110-b5";
diff --git a/qemu/hw/arm/sysbus-fdt.c b/qemu/hw/arm/sysbus-fdt.c
index 9d28797c8..5debb3348 100644
--- a/qemu/hw/arm/sysbus-fdt.c
+++ b/qemu/hw/arm/sysbus-fdt.c
@@ -21,6 +21,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include <libfdt.h>
+#include "qemu-common.h"
+#ifdef CONFIG_LINUX
+#include <linux/vfio.h>
+#endif
 #include "hw/arm/sysbus-fdt.h"
 #include "qemu/error-report.h"
 #include "sysemu/device_tree.h"
@@ -28,6 +35,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/vfio/vfio-platform.h"
 #include "hw/vfio/vfio-calxeda-xgmac.h"
+#include "hw/vfio/vfio-amd-xgbe.h"
 #include "hw/arm/fdt.h"
 
 /*
@@ -56,6 +64,146 @@ typedef struct NodeCreationPair {
     int (*add_fdt_node_fn)(SysBusDevice *sbdev, void *opaque);
 } NodeCreationPair;
 
+/* helpers */
+
+typedef struct HostProperty {
+    const char *name;
+    bool optional;
+} HostProperty;
+
+#ifdef CONFIG_LINUX
+
+/**
+ * copy_properties_from_host
+ *
+ * copies properties listed in an array from host device tree to
+ * guest device tree. If a non optional property is not found, the
+ * function asserts. An optional property is ignored if not found
+ * in the host device tree.
+ * @props: array of HostProperty to copy
+ * @nb_props: number of properties in the array
+ * @host_dt: host device tree blob
+ * @guest_dt: guest device tree blob
+ * @node_path: host dt node path where the property is supposed to be
+              found
+ * @nodename: guest node name the properties should be added to
+ */
+static void copy_properties_from_host(HostProperty *props, int nb_props,
+                                      void *host_fdt, void *guest_fdt,
+                                      char *node_path, char *nodename)
+{
+    int i, prop_len;
+    const void *r;
+    Error *err = NULL;
+
+    for (i = 0; i < nb_props; i++) {
+        r = qemu_fdt_getprop(host_fdt, node_path,
+                             props[i].name,
+                             &prop_len,
+                             props[i].optional ? &err : &error_fatal);
+        if (r) {
+            qemu_fdt_setprop(guest_fdt, nodename,
+                             props[i].name, r, prop_len);
+        } else {
+            if (prop_len != -FDT_ERR_NOTFOUND) {
+                /* optional property not returned although property exists */
+                error_report_err(err);
+            } else {
+                error_free(err);
+            }
+        }
+    }
+}
+
+/* clock properties whose values are copied/pasted from host */
+static HostProperty clock_copied_properties[] = {
+    {"compatible", false},
+    {"#clock-cells", false},
+    {"clock-frequency", true},
+    {"clock-output-names", true},
+};
+
+/**
+ * fdt_build_clock_node
+ *
+ * Build a guest clock node, used as a dependency from a passthrough'ed
+ * device. Most information are retrieved from the host clock node.
+ * Also check the host clock is a fixed one.
+ *
+ * @host_fdt: host device tree blob from which info are retrieved
+ * @guest_fdt: guest device tree blob where the clock node is added
+ * @host_phandle: phandle of the clock in host device tree
+ * @guest_phandle: phandle to assign to the guest node
+ */
+static void fdt_build_clock_node(void *host_fdt, void *guest_fdt,
+                                uint32_t host_phandle,
+                                uint32_t guest_phandle)
+{
+    char *node_path = NULL;
+    char *nodename;
+    const void *r;
+    int ret, node_offset, prop_len, path_len = 16;
+
+    node_offset = fdt_node_offset_by_phandle(host_fdt, host_phandle);
+    if (node_offset <= 0) {
+        error_setg(&error_fatal,
+                   "not able to locate clock handle %d in host device tree",
+                   host_phandle);
+    }
+    node_path = g_malloc(path_len);
+    while ((ret = fdt_get_path(host_fdt, node_offset, node_path, path_len))
+            == -FDT_ERR_NOSPACE) {
+        path_len += 16;
+        node_path = g_realloc(node_path, path_len);
+    }
+    if (ret < 0) {
+        error_setg(&error_fatal,
+                   "not able to retrieve node path for clock handle %d",
+                   host_phandle);
+    }
+
+    r = qemu_fdt_getprop(host_fdt, node_path, "compatible", &prop_len,
+                         &error_fatal);
+    if (strcmp(r, "fixed-clock")) {
+        error_setg(&error_fatal,
+                   "clock handle %d is not a fixed clock", host_phandle);
+    }
+
+    nodename = strrchr(node_path, '/');
+    qemu_fdt_add_subnode(guest_fdt, nodename);
+
+    copy_properties_from_host(clock_copied_properties,
+                              ARRAY_SIZE(clock_copied_properties),
+                              host_fdt, guest_fdt,
+                              node_path, nodename);
+
+    qemu_fdt_setprop_cell(guest_fdt, nodename, "phandle", guest_phandle);
+
+    g_free(node_path);
+}
+
+/**
+ * sysfs_to_dt_name: convert the name found in sysfs into the node name
+ * for instance e0900000.xgmac is converted into xgmac@e0900000
+ * @sysfs_name: directory name in sysfs
+ *
+ * returns the device tree name upon success or NULL in case the sysfs name
+ * does not match the expected format
+ */
+static char *sysfs_to_dt_name(const char *sysfs_name)
+{
+    gchar **substrings =  g_strsplit(sysfs_name, ".", 2);
+    char *dt_name = NULL;
+
+    if (!substrings || !substrings[0] || !substrings[1]) {
+        goto out;
+    }
+    dt_name = g_strdup_printf("%s@%s", substrings[1], substrings[0]);
+out:
+    g_strfreev(substrings);
+    return dt_name;
+}
+
 /* Device Specific Code */
 
 /**
@@ -70,7 +218,7 @@ static int add_calxeda_midway_xgmac_fdt_node(SysBusDevice *sbdev, void *opaque)
     PlatformBusDevice *pbus = data->pbus;
     void *fdt = data->fdt;
     const char *parent_node = data->pbus_node_name;
-    int compat_str_len, i, ret = -1;
+    int compat_str_len, i;
     char *nodename;
     uint32_t *irq_attr, *reg_attr;
     uint64_t mmio_base, irq_number;
@@ -93,14 +241,10 @@ static int add_calxeda_midway_xgmac_fdt_node(SysBusDevice *sbdev, void *opaque)
         mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, i);
         reg_attr[2 * i] = cpu_to_be32(mmio_base);
         reg_attr[2 * i + 1] = cpu_to_be32(
-                                memory_region_size(&vdev->regions[i]->mem));
-    }
-    ret = qemu_fdt_setprop(fdt, nodename, "reg", reg_attr,
-                           vbasedev->num_regions * 2 * sizeof(uint32_t));
-    if (ret) {
-        error_report("could not set reg property of node %s", nodename);
-        goto fail_reg;
+                                memory_region_size(vdev->regions[i]->mem));
     }
+    qemu_fdt_setprop(fdt, nodename, "reg", reg_attr,
+                     vbasedev->num_regions * 2 * sizeof(uint32_t));
 
     irq_attr = g_new(uint32_t, vbasedev->num_irqs * 3);
     for (i = 0; i < vbasedev->num_irqs; i++) {
@@ -110,22 +254,173 @@ static int add_calxeda_midway_xgmac_fdt_node(SysBusDevice *sbdev, void *opaque)
         irq_attr[3 * i + 1] = cpu_to_be32(irq_number);
         irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_LEVEL_HI);
     }
-    ret = qemu_fdt_setprop(fdt, nodename, "interrupts",
+    qemu_fdt_setprop(fdt, nodename, "interrupts",
                      irq_attr, vbasedev->num_irqs * 3 * sizeof(uint32_t));
-    if (ret) {
-        error_report("could not set interrupts property of node %s",
-                     nodename);
+    g_free(irq_attr);
+    g_free(reg_attr);
+    g_free(nodename);
+    return 0;
+}
+
+/* AMD xgbe properties whose values are copied/pasted from host */
+static HostProperty amd_xgbe_copied_properties[] = {
+    {"compatible", false},
+    {"dma-coherent", true},
+    {"amd,per-channel-interrupt", true},
+    {"phy-mode", false},
+    {"mac-address", true},
+    {"amd,speed-set", false},
+    {"amd,serdes-blwc", true},
+    {"amd,serdes-cdr-rate", true},
+    {"amd,serdes-pq-skew", true},
+    {"amd,serdes-tx-amp", true},
+    {"amd,serdes-dfe-tap-config", true},
+    {"amd,serdes-dfe-tap-enable", true},
+    {"clock-names", false},
+};
+
+/**
+ * add_amd_xgbe_fdt_node
+ *
+ * Generates the combined xgbe/phy node following kernel >=4.2
+ * binding documentation:
+ * Documentation/devicetree/bindings/net/amd-xgbe.txt:
+ * Also 2 clock nodes are created (dma and ptp)
+ *
+ * Asserts in case of error
+ */
+static int add_amd_xgbe_fdt_node(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformBusFDTData *data = opaque;
+    PlatformBusDevice *pbus = data->pbus;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    VFIOINTp *intp;
+    const char *parent_node = data->pbus_node_name;
+    char **node_path, *nodename, *dt_name;
+    void *guest_fdt = data->fdt, *host_fdt;
+    const void *r;
+    int i, prop_len;
+    uint32_t *irq_attr, *reg_attr, *host_clock_phandles;
+    uint64_t mmio_base, irq_number;
+    uint32_t guest_clock_phandles[2];
+
+    host_fdt = load_device_tree_from_sysfs();
+
+    dt_name = sysfs_to_dt_name(vbasedev->name);
+    if (!dt_name) {
+        error_setg(&error_fatal, "%s incorrect sysfs device name %s",
+                    __func__, vbasedev->name);
+    }
+    node_path = qemu_fdt_node_path(host_fdt, dt_name, vdev->compat,
+                                   &error_fatal);
+    if (!node_path || !node_path[0]) {
+        error_setg(&error_fatal, "%s unable to retrieve node path for %s/%s",
+                   __func__, dt_name, vdev->compat);
     }
+
+    if (node_path[1]) {
+        error_setg(&error_fatal, "%s more than one node matching %s/%s!",
+                   __func__, dt_name, vdev->compat);
+    }
+
+    g_free(dt_name);
+
+    if (vbasedev->num_regions != 5) {
+        error_setg(&error_fatal, "%s Does the host dt node combine XGBE/PHY?",
+                   __func__);
+    }
+
+    /* generate nodes for DMA_CLK and PTP_CLK */
+    r = qemu_fdt_getprop(host_fdt, node_path[0], "clocks",
+                         &prop_len, &error_fatal);
+    if (prop_len != 8) {
+        error_setg(&error_fatal, "%s clocks property should contain 2 handles",
+                   __func__);
+    }
+    host_clock_phandles = (uint32_t *)r;
+    guest_clock_phandles[0] = qemu_fdt_alloc_phandle(guest_fdt);
+    guest_clock_phandles[1] = qemu_fdt_alloc_phandle(guest_fdt);
+
+    /**
+     * clock handles fetched from host dt are in be32 layout whereas
+     * rest of the code uses cpu layout. Also guest clock handles are
+     * in cpu layout.
+     */
+    fdt_build_clock_node(host_fdt, guest_fdt,
+                         be32_to_cpu(host_clock_phandles[0]),
+                         guest_clock_phandles[0]);
+
+    fdt_build_clock_node(host_fdt, guest_fdt,
+                         be32_to_cpu(host_clock_phandles[1]),
+                         guest_clock_phandles[1]);
+
+    /* combined XGBE/PHY node */
+    mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
+    nodename = g_strdup_printf("%s/%s@%" PRIx64, parent_node,
+                               vbasedev->name, mmio_base);
+    qemu_fdt_add_subnode(guest_fdt, nodename);
+
+    copy_properties_from_host(amd_xgbe_copied_properties,
+                       ARRAY_SIZE(amd_xgbe_copied_properties),
+                       host_fdt, guest_fdt,
+                       node_path[0], nodename);
+
+    qemu_fdt_setprop_cells(guest_fdt, nodename, "clocks",
+                           guest_clock_phandles[0],
+                           guest_clock_phandles[1]);
+
+    reg_attr = g_new(uint32_t, vbasedev->num_regions * 2);
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, i);
+        reg_attr[2 * i] = cpu_to_be32(mmio_base);
+        reg_attr[2 * i + 1] = cpu_to_be32(
+                                memory_region_size(vdev->regions[i]->mem));
+    }
+    qemu_fdt_setprop(guest_fdt, nodename, "reg", reg_attr,
+                     vbasedev->num_regions * 2 * sizeof(uint32_t));
+
+    irq_attr = g_new(uint32_t, vbasedev->num_irqs * 3);
+    for (i = 0; i < vbasedev->num_irqs; i++) {
+        irq_number = platform_bus_get_irqn(pbus, sbdev , i)
+                         + data->irq_start;
+        irq_attr[3 * i] = cpu_to_be32(GIC_FDT_IRQ_TYPE_SPI);
+        irq_attr[3 * i + 1] = cpu_to_be32(irq_number);
+        /*
+          * General device interrupt and PCS auto-negotiation interrupts are
+          * level-sensitive while the 4 per-channel interrupts are edge
+          * sensitive
+          */
+        QLIST_FOREACH(intp, &vdev->intp_list, next) {
+            if (intp->pin == i) {
+                break;
+            }
+        }
+        if (intp->flags & VFIO_IRQ_INFO_AUTOMASKED) {
+            irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_LEVEL_HI);
+        } else {
+            irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_EDGE_LO_HI);
+        }
+    }
+    qemu_fdt_setprop(guest_fdt, nodename, "interrupts",
+                     irq_attr, vbasedev->num_irqs * 3 * sizeof(uint32_t));
+
+    g_free(host_fdt);
+    g_strfreev(node_path);
     g_free(irq_attr);
-fail_reg:
     g_free(reg_attr);
     g_free(nodename);
-    return ret;
+    return 0;
 }
 
+#endif /* CONFIG_LINUX */
+
 /* list of supported dynamic sysbus devices */
 static const NodeCreationPair add_fdt_node_functions[] = {
+#ifdef CONFIG_LINUX
     {TYPE_VFIO_CALXEDA_XGMAC, add_calxeda_midway_xgmac_fdt_node},
+    {TYPE_VFIO_AMD_XGBE, add_amd_xgbe_fdt_node},
+#endif
     {"", NULL}, /* last element */
 };
 
diff --git a/qemu/hw/arm/tosa.c b/qemu/hw/arm/tosa.c
index 73572ebe0..4e9494f94 100644
--- a/qemu/hw/arm/tosa.c
+++ b/qemu/hw/arm/tosa.c
@@ -11,6 +11,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
@@ -19,7 +21,7 @@
 #include "hw/pcmcia.h"
 #include "hw/boards.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
@@ -227,7 +229,7 @@ static void tosa_init(MachineState *machine)
 
     mpu = pxa255_init(address_space_mem, tosa_binfo.ram_size);
 
-    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -252,18 +254,13 @@ static void tosa_init(MachineState *machine)
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static QEMUMachine tosapda_machine = {
-    .name = "tosa",
-    .desc = "Tosa PDA (PXA255)",
-    .init = tosa_init,
-};
-
-static void tosapda_machine_init(void)
+static void tosapda_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&tosapda_machine);
+    mc->desc = "Sharp SL-6000 (Tosa) PDA (PXA255)";
+    mc->init = tosa_init;
 }
 
-machine_init(tosapda_machine_init);
+DEFINE_MACHINE("tosa", tosapda_machine_init)
 
 static void tosa_dac_class_init(ObjectClass *klass, void *data)
 {
diff --git a/qemu/hw/arm/versatilepb.c b/qemu/hw/arm/versatilepb.c
index 6c69f4eaa..e5a80c2d2 100644
--- a/qemu/hw/arm/versatilepb.c
+++ b/qemu/hw/arm/versatilepb.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
@@ -192,7 +196,6 @@ static void versatile_init(MachineState *machine, int board_id)
     int n;
     int done_smc = 0;
     DriveInfo *dinfo;
-    Error *err = NULL;
 
     if (!machine->cpu_model) {
         machine->cpu_model = "arm926";
@@ -211,18 +214,10 @@ static void versatile_init(MachineState *machine, int board_id)
      * realization.
      */
     if (object_property_find(cpuobj, "has_el3", NULL)) {
-        object_property_set_bool(cpuobj, false, "has_el3", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
     }
 
-    object_property_set_bool(cpuobj, true, "realized", &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
-    }
+    object_property_set_bool(cpuobj, true, "realized", &error_fatal);
 
     cpu = ARM_CPU(cpuobj);
 
@@ -391,27 +386,43 @@ static void vab_init(MachineState *machine)
     versatile_init(machine, 0x25e);
 }
 
-static QEMUMachine versatilepb_machine = {
-    .name = "versatilepb",
-    .desc = "ARM Versatile/PB (ARM926EJ-S)",
-    .init = vpb_init,
-    .block_default_type = IF_SCSI,
+static void versatilepb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/PB (ARM926EJ-S)";
+    mc->init = vpb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatilepb_type = {
+    .name = MACHINE_TYPE_NAME("versatilepb"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatilepb_class_init,
 };
 
-static QEMUMachine versatileab_machine = {
-    .name = "versatileab",
-    .desc = "ARM Versatile/AB (ARM926EJ-S)",
-    .init = vab_init,
-    .block_default_type = IF_SCSI,
+static void versatileab_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/AB (ARM926EJ-S)";
+    mc->init = vab_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatileab_type = {
+    .name = MACHINE_TYPE_NAME("versatileab"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatileab_class_init,
 };
 
 static void versatile_machine_init(void)
 {
-    qemu_register_machine(&versatilepb_machine);
-    qemu_register_machine(&versatileab_machine);
+    type_register_static(&versatilepb_type);
+    type_register_static(&versatileab_type);
 }
 
-machine_init(versatile_machine_init);
+type_init(versatile_machine_init)
 
 static void vpb_sic_class_init(ObjectClass *klass, void *data)
 {
diff --git a/qemu/hw/arm/vexpress.c b/qemu/hw/arm/vexpress.c
index da217884e..70b3e701e 100644
--- a/qemu/hw/arm/vexpress.c
+++ b/qemu/hw/arm/vexpress.c
@@ -21,6 +21,10 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -168,8 +172,8 @@ typedef struct {
 } VexpressMachineState;
 
 #define TYPE_VEXPRESS_MACHINE   "vexpress"
-#define TYPE_VEXPRESS_A9_MACHINE   "vexpress-a9"
-#define TYPE_VEXPRESS_A15_MACHINE   "vexpress-a15"
+#define TYPE_VEXPRESS_A9_MACHINE   MACHINE_TYPE_NAME("vexpress-a9")
+#define TYPE_VEXPRESS_A15_MACHINE   MACHINE_TYPE_NAME("vexpress-a15")
 #define VEXPRESS_MACHINE(obj) \
     OBJECT_CHECK(VexpressMachineState, (obj), TYPE_VEXPRESS_MACHINE)
 #define VEXPRESS_MACHINE_GET_CLASS(obj) \
@@ -211,7 +215,6 @@ static void init_cpus(const char *cpu_model, const char *privdev,
     /* Create the actual CPUs */
     for (n = 0; n < smp_cpus; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
 
         if (!secure) {
             object_property_set_bool(cpuobj, false, "has_el3", NULL);
@@ -221,11 +224,7 @@ static void init_cpus(const char *cpu_model, const char *privdev,
             object_property_set_int(cpuobj, periphbase,
                                     "reset-cbar", &error_abort);
         }
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
     }
 
     /* Create the private peripheral devices (including the GIC);
@@ -391,7 +390,7 @@ static void a15_daughterboard_init(const VexpressMachineState *vms,
     /* 0x2b0a0000: PL341 dynamic memory controller: not modelled */
     /* 0x2e000000: system SRAM */
     memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0x2e000000, sram);
 
@@ -482,8 +481,10 @@ static void vexpress_modify_dtb(const struct arm_boot_info *info, void *fdt)
     uint32_t acells, scells, intc;
     const VEDBoardInfo *daughterboard = (const VEDBoardInfo *)info;
 
-    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells");
-    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells");
+    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells",
+                                   NULL, &error_fatal);
+    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells",
+                                   NULL, &error_fatal);
     intc = find_int_controller(fdt);
     if (!intc) {
         /* Not fatal, we just won't provide virtio. This will
@@ -541,7 +542,7 @@ static void vexpress_common_init(MachineState *machine)
 {
     VexpressMachineState *vms = VEXPRESS_MACHINE(machine);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_GET_CLASS(machine);
-    VEDBoardInfo *daughterboard = vmc->daughterboard;;
+    VEDBoardInfo *daughterboard = vmc->daughterboard;
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
     uint32_t sys_id;
@@ -671,13 +672,13 @@ static void vexpress_common_init(MachineState *machine)
 
     sram_size = 0x2000000;
     memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, map[VE_SRAM], sram);
 
     vram_size = 0x800000;
     memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(vram);
     memory_region_add_subregion(sysmem, map[VE_VIDEORAM], vram);
 
@@ -747,7 +748,6 @@ static void vexpress_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_MACHINE;
     mc->desc = "ARM Versatile Express";
     mc->init = vexpress_common_init;
     mc->block_default_type = IF_SCSI;
@@ -759,10 +759,9 @@ static void vexpress_a9_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_A9_MACHINE;
     mc->desc = "ARM Versatile Express for Cortex-A9";
 
-    vmc->daughterboard = &a9_daughterboard;;
+    vmc->daughterboard = &a9_daughterboard;
 }
 
 static void vexpress_a15_class_init(ObjectClass *oc, void *data)
@@ -770,7 +769,6 @@ static void vexpress_a15_class_init(ObjectClass *oc, void *data)
     MachineClass *mc = MACHINE_CLASS(oc);
     VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
 
-    mc->name = TYPE_VEXPRESS_A15_MACHINE;
     mc->desc = "ARM Versatile Express for Cortex-A15";
 
     vmc->daughterboard = &a15_daughterboard;
@@ -805,4 +803,4 @@ static void vexpress_machine_init(void)
     type_register_static(&vexpress_a15_info);
 }
 
-machine_init(vexpress_machine_init);
+type_init(vexpress_machine_init);
diff --git a/qemu/hw/arm/virt-acpi-build.c b/qemu/hw/arm/virt-acpi-build.c
index f36514031..f51fe396c 100644
--- a/qemu/hw/arm/virt-acpi-build.c
+++ b/qemu/hw/arm/virt-acpi-build.c
@@ -26,6 +26,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/arm/virt-acpi-build.h"
 #include "qemu/bitmap.h"
@@ -43,20 +45,7 @@
 #include "hw/pci/pci.h"
 
 #define ARM_SPI_BASE 32
-
-typedef struct VirtAcpiCpuInfo {
-    DECLARE_BITMAP(found_cpus, VIRT_ACPI_CPU_ID_LIMIT);
-} VirtAcpiCpuInfo;
-
-static void virt_acpi_get_cpu_info(VirtAcpiCpuInfo *cpuinfo)
-{
-    CPUState *cpu;
-
-    memset(cpuinfo->found_cpus, 0, sizeof cpuinfo->found_cpus);
-    CPU_FOREACH(cpu) {
-        set_bit(cpu->cpu_index, cpuinfo->found_cpus);
-    }
-}
+#define ACPI_POWER_BUTTON_DEVICE "PWRB"
 
 static void acpi_dsdt_add_cpus(Aml *scope, int smp_cpus)
 {
@@ -71,7 +60,7 @@ static void acpi_dsdt_add_cpus(Aml *scope, int smp_cpus)
 }
 
 static void acpi_dsdt_add_uart(Aml *scope, const MemMapEntry *uart_memmap,
-                                           int uart_irq)
+                                           uint32_t uart_irq)
 {
     Aml *dev = aml_device("COM0");
     aml_append(dev, aml_name_decl("_HID", aml_string("ARMH0011")));
@@ -82,7 +71,7 @@ static void acpi_dsdt_add_uart(Aml *scope, const MemMapEntry *uart_memmap,
                                        uart_memmap->size, AML_READ_WRITE));
     aml_append(crs,
                aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
-                             AML_EXCLUSIVE, uart_irq));
+                             AML_EXCLUSIVE, &uart_irq, 1));
     aml_append(dev, aml_name_decl("_CRS", crs));
 
     /* The _ADR entry is used to link this device to the UART described
@@ -93,19 +82,16 @@ static void acpi_dsdt_add_uart(Aml *scope, const MemMapEntry *uart_memmap,
     aml_append(scope, dev);
 }
 
-static void acpi_dsdt_add_rtc(Aml *scope, const MemMapEntry *rtc_memmap,
-                                          int rtc_irq)
+static void acpi_dsdt_add_fw_cfg(Aml *scope, const MemMapEntry *fw_cfg_memmap)
 {
-    Aml *dev = aml_device("RTC0");
-    aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0013")));
-    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+    Aml *dev = aml_device("FWCF");
+    aml_append(dev, aml_name_decl("_HID", aml_string("QEMU0002")));
+    /* device present, functioning, decoding, not shown in UI */
+    aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
 
     Aml *crs = aml_resource_template();
-    aml_append(crs, aml_memory32_fixed(rtc_memmap->base,
-                                       rtc_memmap->size, AML_READ_WRITE));
-    aml_append(crs,
-               aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
-                             AML_EXCLUSIVE, rtc_irq));
+    aml_append(crs, aml_memory32_fixed(fw_cfg_memmap->base,
+                                       fw_cfg_memmap->size, AML_READ_WRITE));
     aml_append(dev, aml_name_decl("_CRS", crs));
     aml_append(scope, dev);
 }
@@ -114,7 +100,7 @@ static void acpi_dsdt_add_flash(Aml *scope, const MemMapEntry *flash_memmap)
 {
     Aml *dev, *crs;
     hwaddr base = flash_memmap->base;
-    hwaddr size = flash_memmap->size;
+    hwaddr size = flash_memmap->size / 2;
 
     dev = aml_device("FLS0");
     aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0015")));
@@ -136,14 +122,14 @@ static void acpi_dsdt_add_flash(Aml *scope, const MemMapEntry *flash_memmap)
 
 static void acpi_dsdt_add_virtio(Aml *scope,
                                  const MemMapEntry *virtio_mmio_memmap,
-                                 int mmio_irq, int num)
+                                 uint32_t mmio_irq, int num)
 {
     hwaddr base = virtio_mmio_memmap->base;
     hwaddr size = virtio_mmio_memmap->size;
-    int irq = mmio_irq;
     int i;
 
     for (i = 0; i < num; i++) {
+        uint32_t irq = mmio_irq + i;
         Aml *dev = aml_device("VR%02u", i);
         aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0005")));
         aml_append(dev, aml_name_decl("_UID", aml_int(i)));
@@ -152,14 +138,15 @@ static void acpi_dsdt_add_virtio(Aml *scope,
         aml_append(crs, aml_memory32_fixed(base, size, AML_READ_WRITE));
         aml_append(crs,
                    aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
-                                 AML_EXCLUSIVE, irq + i));
+                                 AML_EXCLUSIVE, &irq, 1));
         aml_append(dev, aml_name_decl("_CRS", crs));
         aml_append(scope, dev);
         base += size;
     }
 }
 
-static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
+static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
+                              uint32_t irq, bool use_highmem)
 {
     Aml *method, *crs, *ifctx, *UUID, *ifctx1, *elsectx, *buf;
     int i, bus_no;
@@ -179,6 +166,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
     aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
     aml_append(dev, aml_name_decl("_UID", aml_string("PCI0")));
     aml_append(dev, aml_name_decl("_STR", aml_unicode("PCIe 0 Device")));
+    aml_append(dev, aml_name_decl("_CCA", aml_int(1)));
 
     /* Declare the PCI Routing Table. */
     Aml *rt_pkg = aml_package(nr_pcie_buses * PCI_NUM_PINS);
@@ -197,29 +185,30 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
 
     /* Create GSI link device */
     for (i = 0; i < PCI_NUM_PINS; i++) {
+        uint32_t irqs =  irq + i;
         Aml *dev_gsi = aml_device("GSI%d", i);
         aml_append(dev_gsi, aml_name_decl("_HID", aml_string("PNP0C0F")));
         aml_append(dev_gsi, aml_name_decl("_UID", aml_int(0)));
         crs = aml_resource_template();
         aml_append(crs,
                    aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
-                                 AML_EXCLUSIVE, irq + i));
+                                 AML_EXCLUSIVE, &irqs, 1));
         aml_append(dev_gsi, aml_name_decl("_PRS", crs));
         crs = aml_resource_template();
         aml_append(crs,
                    aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
-                                 AML_EXCLUSIVE, irq + i));
+                                 AML_EXCLUSIVE, &irqs, 1));
         aml_append(dev_gsi, aml_name_decl("_CRS", crs));
-        method = aml_method("_SRS", 1);
+        method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
         aml_append(dev_gsi, method);
         aml_append(dev, dev_gsi);
     }
 
-    method = aml_method("_CBA", 0);
+    method = aml_method("_CBA", 0, AML_NOTSERIALIZED);
     aml_append(method, aml_return(aml_int(base_ecam)));
     aml_append(dev, method);
 
-    method = aml_method("_CRS", 0);
+    method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
     Aml *rbuf = aml_resource_template();
     aml_append(rbuf,
         aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
@@ -234,6 +223,17 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
                      AML_ENTIRE_RANGE, 0x0000, 0x0000, size_pio - 1, base_pio,
                      size_pio));
 
+    if (use_highmem) {
+        hwaddr base_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].base;
+        hwaddr size_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].size;
+
+        aml_append(rbuf,
+            aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                             AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000,
+                             base_mmio_high, base_mmio_high, 0x0000,
+                             size_mmio_high));
+    }
+
     aml_append(method, aml_name_decl("RBUF", rbuf));
     aml_append(method, aml_return(rbuf));
     aml_append(dev, method);
@@ -241,7 +241,7 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
     /* Declare an _OSC (OS Control Handoff) method */
     aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
     aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
-    method = aml_method("_OSC", 4);
+    method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
     aml_append(method,
         aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
 
@@ -259,16 +259,16 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
         aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
     aml_append(ifctx, aml_store(aml_name("CDW2"), aml_name("SUPP")));
     aml_append(ifctx, aml_store(aml_name("CDW3"), aml_name("CTRL")));
-    aml_append(ifctx, aml_store(aml_and(aml_name("CTRL"), aml_int(0x1D)),
+    aml_append(ifctx, aml_store(aml_and(aml_name("CTRL"), aml_int(0x1D), NULL),
                                 aml_name("CTRL")));
 
     ifctx1 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(0x1))));
-    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x08)),
+    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x08), NULL),
                                  aml_name("CDW1")));
     aml_append(ifctx, ifctx1);
 
     ifctx1 = aml_if(aml_lnot(aml_equal(aml_name("CDW3"), aml_name("CTRL"))));
-    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x10)),
+    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x10), NULL),
                                  aml_name("CDW1")));
     aml_append(ifctx, ifctx1);
 
@@ -277,13 +277,13 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
     aml_append(method, ifctx);
 
     elsectx = aml_else();
-    aml_append(elsectx, aml_store(aml_or(aml_name("CDW1"), aml_int(4)),
+    aml_append(elsectx, aml_store(aml_or(aml_name("CDW1"), aml_int(4), NULL),
                                   aml_name("CDW1")));
     aml_append(elsectx, aml_return(aml_arg(3)));
     aml_append(method, elsectx);
     aml_append(dev, method);
 
-    method = aml_method("_DSM", 4);
+    method = aml_method("_DSM", 4, AML_NOTSERIALIZED);
 
     /* PCI Firmware Specification 3.0
      * 4.6.1. _DSM for PCI Express Slot Information
@@ -310,6 +310,46 @@ static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap, int irq)
     aml_append(scope, dev);
 }
 
+static void acpi_dsdt_add_gpio(Aml *scope, const MemMapEntry *gpio_memmap,
+                                           uint32_t gpio_irq)
+{
+    Aml *dev = aml_device("GPO0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("ARMH0061")));
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+
+    Aml *crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(gpio_memmap->base, gpio_memmap->size,
+                                       AML_READ_WRITE));
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_EXCLUSIVE, &gpio_irq, 1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    Aml *aei = aml_resource_template();
+    /* Pin 3 for power button */
+    const uint32_t pin_list[1] = {3};
+    aml_append(aei, aml_gpio_int(AML_CONSUMER, AML_EDGE, AML_ACTIVE_HIGH,
+                                 AML_EXCLUSIVE, AML_PULL_UP, 0, pin_list, 1,
+                                 "GPO0", NULL, 0));
+    aml_append(dev, aml_name_decl("_AEI", aei));
+
+    /* _E03 is handle for power button */
+    Aml *method = aml_method("_E03", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_notify(aml_name(ACPI_POWER_BUTTON_DEVICE),
+                                  aml_int(0x80)));
+    aml_append(dev, method);
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_power_button(Aml *scope)
+{
+    Aml *dev = aml_device(ACPI_POWER_BUTTON_DEVICE);
+    aml_append(dev, aml_name_decl("_HID", aml_string("PNP0C0C")));
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+    aml_append(scope, dev);
+}
+
 /* RSDP */
 static GArray *
 build_rsdp(GArray *rsdp_table, GArray *linker, unsigned rsdt)
@@ -334,7 +374,8 @@ build_rsdp(GArray *rsdp_table, GArray *linker, unsigned rsdt)
     rsdp->checksum = 0;
     /* Checksum to be filled by Guest linker */
     bios_linker_loader_add_checksum(linker, ACPI_BUILD_RSDP_FILE,
-                                    rsdp, rsdp, sizeof *rsdp, &rsdp->checksum);
+                                    rsdp_table, rsdp, sizeof *rsdp,
+                                    &rsdp->checksum);
 
     return rsdp_table;
 }
@@ -368,7 +409,8 @@ build_spcr(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
     spcr->pci_device_id = 0xffff;  /* PCI Device ID: not a PCI device */
     spcr->pci_vendor_id = 0xffff;  /* PCI Vendor ID: not a PCI device */
 
-    build_header(linker, table_data, (void *)spcr, "SPCR", sizeof(*spcr), 2);
+    build_header(linker, table_data, (void *)spcr, "SPCR", sizeof(*spcr), 2,
+                 NULL, NULL);
 }
 
 static void
@@ -387,7 +429,7 @@ build_mcfg(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
     mcfg->allocation[0].end_bus_number = (memmap[VIRT_PCIE_ECAM].size
                                           / PCIE_MMCFG_SIZE_MIN) - 1;
 
-    build_header(linker, table_data, (void *)mcfg, "MCFG", len, 1);
+    build_header(linker, table_data, (void *)mcfg, "MCFG", len, 1, NULL, NULL);
 }
 
 /* GTDT */
@@ -403,7 +445,7 @@ build_gtdt(GArray *table_data, GArray *linker)
     gtdt->secure_el1_flags = ACPI_EDGE_SENSITIVE;
 
     gtdt->non_secure_el1_interrupt = ARCH_TIMER_NS_EL1_IRQ + 16;
-    gtdt->non_secure_el1_flags = ACPI_EDGE_SENSITIVE;
+    gtdt->non_secure_el1_flags = ACPI_EDGE_SENSITIVE | ACPI_GTDT_ALWAYS_ON;
 
     gtdt->virtual_timer_interrupt = ARCH_TIMER_VIRT_IRQ + 16;
     gtdt->virtual_timer_flags = ACPI_EDGE_SENSITIVE;
@@ -413,13 +455,12 @@ build_gtdt(GArray *table_data, GArray *linker)
 
     build_header(linker, table_data,
                  (void *)(table_data->data + gtdt_start), "GTDT",
-                 table_data->len - gtdt_start, 2);
+                 table_data->len - gtdt_start, 2, NULL, NULL);
 }
 
 /* MADT */
 static void
-build_madt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info,
-           VirtAcpiCpuInfo *cpuinfo)
+build_madt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
 {
     int madt_start = table_data->len;
     const MemMapEntry *memmap = guest_info->memmap;
@@ -431,37 +472,49 @@ build_madt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info,
 
     madt = acpi_data_push(table_data, sizeof *madt);
 
+    gicd = acpi_data_push(table_data, sizeof *gicd);
+    gicd->type = ACPI_APIC_GENERIC_DISTRIBUTOR;
+    gicd->length = sizeof(*gicd);
+    gicd->base_address = memmap[VIRT_GIC_DIST].base;
+
     for (i = 0; i < guest_info->smp_cpus; i++) {
         AcpiMadtGenericInterrupt *gicc = acpi_data_push(table_data,
                                                      sizeof *gicc);
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(i));
+
         gicc->type = ACPI_APIC_GENERIC_INTERRUPT;
         gicc->length = sizeof(*gicc);
-        gicc->base_address = memmap[VIRT_GIC_CPU].base;
+        if (guest_info->gic_version == 2) {
+            gicc->base_address = memmap[VIRT_GIC_CPU].base;
+        }
         gicc->cpu_interface_number = i;
-        gicc->arm_mpidr = i;
+        gicc->arm_mpidr = armcpu->mp_affinity;
         gicc->uid = i;
-        if (test_bit(i, cpuinfo->found_cpus)) {
-            gicc->flags = cpu_to_le32(ACPI_GICC_ENABLED);
-        }
+        gicc->flags = cpu_to_le32(ACPI_GICC_ENABLED);
     }
 
-    gicd = acpi_data_push(table_data, sizeof *gicd);
-    gicd->type = ACPI_APIC_GENERIC_DISTRIBUTOR;
-    gicd->length = sizeof(*gicd);
-    gicd->base_address = memmap[VIRT_GIC_DIST].base;
-
-    gic_msi = acpi_data_push(table_data, sizeof *gic_msi);
-    gic_msi->type = ACPI_APIC_GENERIC_MSI_FRAME;
-    gic_msi->length = sizeof(*gic_msi);
-    gic_msi->gic_msi_frame_id = 0;
-    gic_msi->base_address = cpu_to_le64(memmap[VIRT_GIC_V2M].base);
-    gic_msi->flags = cpu_to_le32(1);
-    gic_msi->spi_count = cpu_to_le16(NUM_GICV2M_SPIS);
-    gic_msi->spi_base = cpu_to_le16(irqmap[VIRT_GIC_V2M] + ARM_SPI_BASE);
+    if (guest_info->gic_version == 3) {
+        AcpiMadtGenericRedistributor *gicr = acpi_data_push(table_data,
+                                                         sizeof *gicr);
+
+        gicr->type = ACPI_APIC_GENERIC_REDISTRIBUTOR;
+        gicr->length = sizeof(*gicr);
+        gicr->base_address = cpu_to_le64(memmap[VIRT_GIC_REDIST].base);
+        gicr->range_length = cpu_to_le32(memmap[VIRT_GIC_REDIST].size);
+    } else {
+        gic_msi = acpi_data_push(table_data, sizeof *gic_msi);
+        gic_msi->type = ACPI_APIC_GENERIC_MSI_FRAME;
+        gic_msi->length = sizeof(*gic_msi);
+        gic_msi->gic_msi_frame_id = 0;
+        gic_msi->base_address = cpu_to_le64(memmap[VIRT_GIC_V2M].base);
+        gic_msi->flags = cpu_to_le32(1);
+        gic_msi->spi_count = cpu_to_le16(NUM_GICV2M_SPIS);
+        gic_msi->spi_base = cpu_to_le16(irqmap[VIRT_GIC_V2M] + ARM_SPI_BASE);
+    }
 
     build_header(linker, table_data,
                  (void *)(table_data->data + madt_start), "APIC",
-                 table_data->len - madt_start, 3);
+                 table_data->len - madt_start, 3, NULL, NULL);
 }
 
 /* FADT */
@@ -486,7 +539,7 @@ build_fadt(GArray *table_data, GArray *linker, unsigned dsdt)
                                    sizeof fadt->dsdt);
 
     build_header(linker, table_data,
-                 (void *)fadt, "FACP", sizeof(*fadt), 5);
+                 (void *)fadt, "FACP", sizeof(*fadt), 5, NULL, NULL);
 }
 
 /* DSDT */
@@ -501,16 +554,24 @@ build_dsdt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
     /* Reserve space for header */
     acpi_data_push(dsdt->buf, sizeof(AcpiTableHeader));
 
+    /* When booting the VM with UEFI, UEFI takes ownership of the RTC hardware.
+     * While UEFI can use libfdt to disable the RTC device node in the DTB that
+     * it passes to the OS, it cannot modify AML. Therefore, we won't generate
+     * the RTC ACPI device at all when using UEFI.
+     */
     scope = aml_scope("\\_SB");
     acpi_dsdt_add_cpus(scope, guest_info->smp_cpus);
     acpi_dsdt_add_uart(scope, &memmap[VIRT_UART],
                        (irqmap[VIRT_UART] + ARM_SPI_BASE));
-    acpi_dsdt_add_rtc(scope, &memmap[VIRT_RTC],
-                      (irqmap[VIRT_RTC] + ARM_SPI_BASE));
     acpi_dsdt_add_flash(scope, &memmap[VIRT_FLASH]);
+    acpi_dsdt_add_fw_cfg(scope, &memmap[VIRT_FW_CFG]);
     acpi_dsdt_add_virtio(scope, &memmap[VIRT_MMIO],
                     (irqmap[VIRT_MMIO] + ARM_SPI_BASE), NUM_VIRTIO_TRANSPORTS);
-    acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE));
+    acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE),
+                      guest_info->use_highmem);
+    acpi_dsdt_add_gpio(scope, &memmap[VIRT_GPIO],
+                       (irqmap[VIRT_GPIO] + ARM_SPI_BASE));
+    acpi_dsdt_add_power_button(scope);
 
     aml_append(dsdt, scope);
 
@@ -518,7 +579,7 @@ build_dsdt(GArray *table_data, GArray *linker, VirtGuestInfo *guest_info)
     g_array_append_vals(table_data, dsdt->buf->data, dsdt->buf->len);
     build_header(linker, table_data,
         (void *)(table_data->data + table_data->len - dsdt->buf->len),
-        "DSDT", dsdt->buf->len, 2);
+        "DSDT", dsdt->buf->len, 2, NULL, NULL);
     free_aml_allocator();
 }
 
@@ -538,11 +599,8 @@ void virt_acpi_build(VirtGuestInfo *guest_info, AcpiBuildTables *tables)
 {
     GArray *table_offsets;
     unsigned dsdt, rsdt;
-    VirtAcpiCpuInfo cpuinfo;
     GArray *tables_blob = tables->table_data;
 
-    virt_acpi_get_cpu_info(&cpuinfo);
-
     table_offsets = g_array_new(false, true /* clear */,
                                         sizeof(uint32_t));
 
@@ -569,7 +627,7 @@ void virt_acpi_build(VirtGuestInfo *guest_info, AcpiBuildTables *tables)
     build_fadt(tables_blob, tables->linker, dsdt);
 
     acpi_add_table(table_offsets, tables_blob);
-    build_madt(tables_blob, tables->linker, guest_info, &cpuinfo);
+    build_madt(tables_blob, tables->linker, guest_info);
 
     acpi_add_table(table_offsets, tables_blob);
     build_gtdt(tables_blob, tables->linker);
@@ -582,7 +640,7 @@ void virt_acpi_build(VirtGuestInfo *guest_info, AcpiBuildTables *tables)
 
     /* RSDT is pointed to by RSDP */
     rsdt = tables_blob->len;
-    build_rsdt(tables_blob, tables->linker, table_offsets);
+    build_rsdt(tables_blob, tables->linker, table_offsets, NULL, NULL);
 
     /* RSDP is in FSEG memory, so allocate it separately */
     build_rsdp(tables->rsdp, tables->linker, rsdt);
@@ -603,7 +661,7 @@ static void acpi_ram_update(MemoryRegion *mr, GArray *data)
     memory_region_set_dirty(mr, 0, size);
 }
 
-static void virt_acpi_build_update(void *build_opaque, uint32_t offset)
+static void virt_acpi_build_update(void *build_opaque)
 {
     AcpiBuildState *build_state = build_opaque;
     AcpiBuildTables tables;
diff --git a/qemu/hw/arm/virt.c b/qemu/hw/arm/virt.c
index 484689264..56d35c771 100644
--- a/qemu/hw/arm/virt.c
+++ b/qemu/hw/arm/virt.c
@@ -28,6 +28,8 @@
  * This is essentially the same approach kvmtool uses.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -48,6 +50,11 @@
 #include "hw/arm/sysbus-fdt.h"
 #include "hw/platform-bus.h"
 #include "hw/arm/fdt.h"
+#include "hw/intc/arm_gic_common.h"
+#include "kvm_arm.h"
+#include "hw/smbios/smbios.h"
+#include "qapi/visitor.h"
+#include "standard-headers/linux/input.h"
 
 /* Number of external interrupt lines to configure the GIC with */
 #define NUM_IRQS 256
@@ -67,6 +74,7 @@ typedef struct VirtBoardInfo {
     uint32_t clock_phandle;
     uint32_t gic_phandle;
     uint32_t v2m_phandle;
+    bool using_psci;
 } VirtBoardInfo;
 
 typedef struct {
@@ -77,9 +85,11 @@ typedef struct {
 typedef struct {
     MachineState parent;
     bool secure;
+    bool highmem;
+    int32_t gic_version;
 } VirtMachineState;
 
-#define TYPE_VIRT_MACHINE   "virt"
+#define TYPE_VIRT_MACHINE   MACHINE_TYPE_NAME("virt")
 #define VIRT_MACHINE(obj) \
     OBJECT_CHECK(VirtMachineState, (obj), TYPE_VIRT_MACHINE)
 #define VIRT_MACHINE_GET_CLASS(obj) \
@@ -87,6 +97,23 @@ typedef struct {
 #define VIRT_MACHINE_CLASS(klass) \
     OBJECT_CLASS_CHECK(VirtMachineClass, klass, TYPE_VIRT_MACHINE)
 
+/* RAM limit in GB. Since VIRT_MEM starts at the 1GB mark, this means
+ * RAM can go up to the 256GB mark, leaving 256GB of the physical
+ * address space unallocated and free for future use between 256G and 512G.
+ * If we need to provide more RAM to VMs in the future then we need to:
+ *  * allocate a second bank of RAM starting at 2TB and working up
+ *  * fix the DT and ACPI table generation code in QEMU to correctly
+ *    report two split lumps of RAM to the guest
+ *  * fix KVM in the host kernel to allow guests with >40 bit address spaces
+ * (We don't want to fill all the way up to 512GB with RAM because
+ * we might want it for non-RAM purposes later. Conversely it seems
+ * reasonable to assume that anybody configuring a VM with a quarter
+ * of a terabyte of RAM will be doing it on a host with more than a
+ * terabyte of physical address space.)
+ */
+#define RAMLIMIT_GB 255
+#define RAMLIMIT_BYTES (RAMLIMIT_GB * 1024ULL * 1024 * 1024)
+
 /* Addresses and sizes of our components.
  * 0..128MB is space for a flash device so we can run bootrom code such as UEFI.
  * 128MB..256MB is used for miscellaneous device I/O.
@@ -107,22 +134,33 @@ static const MemMapEntry a15memmap[] = {
     [VIRT_GIC_DIST] =           { 0x08000000, 0x00010000 },
     [VIRT_GIC_CPU] =            { 0x08010000, 0x00010000 },
     [VIRT_GIC_V2M] =            { 0x08020000, 0x00001000 },
+    /* The space in between here is reserved for GICv3 CPU/vCPU/HYP */
+    [VIRT_GIC_ITS] =            { 0x08080000, 0x00020000 },
+    /* This redistributor space allows up to 2*64kB*123 CPUs */
+    [VIRT_GIC_REDIST] =         { 0x080A0000, 0x00F60000 },
     [VIRT_UART] =               { 0x09000000, 0x00001000 },
     [VIRT_RTC] =                { 0x09010000, 0x00001000 },
-    [VIRT_FW_CFG] =             { 0x09020000, 0x0000000a },
+    [VIRT_FW_CFG] =             { 0x09020000, 0x00000018 },
+    [VIRT_GPIO] =               { 0x09030000, 0x00001000 },
+    [VIRT_SECURE_UART] =        { 0x09040000, 0x00001000 },
     [VIRT_MMIO] =               { 0x0a000000, 0x00000200 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
     [VIRT_PLATFORM_BUS] =       { 0x0c000000, 0x02000000 },
+    [VIRT_SECURE_MEM] =         { 0x0e000000, 0x01000000 },
     [VIRT_PCIE_MMIO] =          { 0x10000000, 0x2eff0000 },
     [VIRT_PCIE_PIO] =           { 0x3eff0000, 0x00010000 },
     [VIRT_PCIE_ECAM] =          { 0x3f000000, 0x01000000 },
-    [VIRT_MEM] =                { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
+    [VIRT_MEM] =                { 0x40000000, RAMLIMIT_BYTES },
+    /* Second PCIe window, 512GB wide at the 512GB boundary */
+    [VIRT_PCIE_MMIO_HIGH] =   { 0x8000000000ULL, 0x8000000000ULL },
 };
 
 static const int a15irqmap[] = {
     [VIRT_UART] = 1,
     [VIRT_RTC] = 2,
     [VIRT_PCIE] = 3, /* ... to 6 */
+    [VIRT_GPIO] = 7,
+    [VIRT_SECURE_UART] = 8,
     [VIRT_MMIO] = 16, /* ...to 16 + NUM_VIRTIO_TRANSPORTS - 1 */
     [VIRT_GIC_V2M] = 48, /* ...to 48 + NUM_GICV2M_SPIS - 1 */
     [VIRT_PLATFORM_BUS] = 112, /* ...to 112 + PLATFORM_BUS_NUM_IRQS -1 */
@@ -212,6 +250,10 @@ static void fdt_add_psci_node(const VirtBoardInfo *vbi)
     void *fdt = vbi->fdt;
     ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(0));
 
+    if (!vbi->using_psci) {
+        return;
+    }
+
     qemu_fdt_add_subnode(fdt, "/psci");
     if (armcpu->psci_version == 2) {
         const char comp[] = "arm,psci-0.2\0arm,psci";
@@ -249,7 +291,7 @@ static void fdt_add_psci_node(const VirtBoardInfo *vbi)
     qemu_fdt_setprop_cell(fdt, "/psci", "migrate", migrate_fn);
 }
 
-static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
+static void fdt_add_timer_nodes(const VirtBoardInfo *vbi, int gictype)
 {
     /* Note that on A15 h/w these interrupts are level-triggered,
      * but for the GIC implementation provided by both QEMU and KVM
@@ -258,8 +300,11 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
     ARMCPU *armcpu;
     uint32_t irqflags = GIC_FDT_IRQ_FLAGS_EDGE_LO_HI;
 
-    irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
-                         GIC_FDT_IRQ_PPI_CPU_WIDTH, (1 << vbi->smp_cpus) - 1);
+    if (gictype == 2) {
+        irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+                             GIC_FDT_IRQ_PPI_CPU_WIDTH,
+                             (1 << vbi->smp_cpus) - 1);
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/timer");
 
@@ -272,6 +317,7 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
         qemu_fdt_setprop_string(vbi->fdt, "/timer", "compatible",
                                 "arm,armv7-timer");
     }
+    qemu_fdt_setprop(vbi->fdt, "/timer", "always-on", NULL, 0);
     qemu_fdt_setprop_cells(vbi->fdt, "/timer", "interrupts",
                        GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_S_EL1_IRQ, irqflags,
                        GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_NS_EL1_IRQ, irqflags,
@@ -282,9 +328,32 @@ static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
 static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
 {
     int cpu;
+    int addr_cells = 1;
+
+    /*
+     * From Documentation/devicetree/bindings/arm/cpus.txt
+     *  On ARM v8 64-bit systems value should be set to 2,
+     *  that corresponds to the MPIDR_EL1 register size.
+     *  If MPIDR_EL1[63:32] value is equal to 0 on all CPUs
+     *  in the system, #address-cells can be set to 1, since
+     *  MPIDR_EL1[63:32] bits are not used for CPUs
+     *  identification.
+     *
+     *  Here we actually don't know whether our system is 32- or 64-bit one.
+     *  The simplest way to go is to examine affinity IDs of all our CPUs. If
+     *  at least one of them has Aff3 populated, we set #address-cells to 2.
+     */
+    for (cpu = 0; cpu < vbi->smp_cpus; cpu++) {
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(cpu));
+
+        if (armcpu->mp_affinity & ARM_AFF3_MASK) {
+            addr_cells = 2;
+            break;
+        }
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/cpus");
-    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", 0x1);
+    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", addr_cells);
     qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#size-cells", 0x0);
 
     for (cpu = vbi->smp_cpus - 1; cpu >= 0; cpu--) {
@@ -296,12 +365,19 @@ static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
         qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible",
                                     armcpu->dtb_compatible);
 
-        if (vbi->smp_cpus > 1) {
+        if (vbi->using_psci && vbi->smp_cpus > 1) {
             qemu_fdt_setprop_string(vbi->fdt, nodename,
                                         "enable-method", "psci");
         }
 
-        qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg", armcpu->mp_affinity);
+        if (addr_cells == 2) {
+            qemu_fdt_setprop_u64(vbi->fdt, nodename, "reg",
+                                 armcpu->mp_affinity);
+        } else {
+            qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg",
+                                  armcpu->mp_affinity);
+        }
+
         g_free(nodename);
     }
 }
@@ -319,25 +395,36 @@ static void fdt_add_v2m_gic_node(VirtBoardInfo *vbi)
     qemu_fdt_setprop_cell(vbi->fdt, "/intc/v2m", "phandle", vbi->v2m_phandle);
 }
 
-static void fdt_add_gic_node(VirtBoardInfo *vbi)
+static void fdt_add_gic_node(VirtBoardInfo *vbi, int type)
 {
     vbi->gic_phandle = qemu_fdt_alloc_phandle(vbi->fdt);
     qemu_fdt_setprop_cell(vbi->fdt, "/", "interrupt-parent", vbi->gic_phandle);
 
     qemu_fdt_add_subnode(vbi->fdt, "/intc");
-    /* 'cortex-a15-gic' means 'GIC v2' */
-    qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
-                            "arm,cortex-a15-gic");
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#interrupt-cells", 3);
     qemu_fdt_setprop(vbi->fdt, "/intc", "interrupt-controller", NULL, 0);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
-                                     2, vbi->memmap[VIRT_GIC_DIST].base,
-                                     2, vbi->memmap[VIRT_GIC_DIST].size,
-                                     2, vbi->memmap[VIRT_GIC_CPU].base,
-                                     2, vbi->memmap[VIRT_GIC_CPU].size);
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#address-cells", 0x2);
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#size-cells", 0x2);
     qemu_fdt_setprop(vbi->fdt, "/intc", "ranges", NULL, 0);
+    if (type == 3) {
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,gic-v3");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                     2, vbi->memmap[VIRT_GIC_DIST].base,
+                                     2, vbi->memmap[VIRT_GIC_DIST].size,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].base,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].size);
+    } else {
+        /* 'cortex-a15-gic' means 'GIC v2' */
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,cortex-a15-gic");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                      2, vbi->memmap[VIRT_GIC_DIST].base,
+                                      2, vbi->memmap[VIRT_GIC_DIST].size,
+                                      2, vbi->memmap[VIRT_GIC_CPU].base,
+                                      2, vbi->memmap[VIRT_GIC_CPU].size);
+    }
+
     qemu_fdt_setprop_cell(vbi->fdt, "/intc", "phandle", vbi->gic_phandle);
 }
 
@@ -360,29 +447,34 @@ static void create_v2m(VirtBoardInfo *vbi, qemu_irq *pic)
     fdt_add_v2m_gic_node(vbi);
 }
 
-static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic, int type, bool secure)
 {
-    /* We create a standalone GIC v2 */
+    /* We create a standalone GIC */
     DeviceState *gicdev;
     SysBusDevice *gicbusdev;
-    const char *gictype = "arm_gic";
+    const char *gictype;
     int i;
 
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
-    }
+    gictype = (type == 3) ? gicv3_class_name() : gic_class_name();
 
     gicdev = qdev_create(NULL, gictype);
-    qdev_prop_set_uint32(gicdev, "revision", 2);
+    qdev_prop_set_uint32(gicdev, "revision", type);
     qdev_prop_set_uint32(gicdev, "num-cpu", smp_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
      */
     qdev_prop_set_uint32(gicdev, "num-irq", NUM_IRQS + 32);
+    if (!kvm_irqchip_in_kernel()) {
+        qdev_prop_set_bit(gicdev, "has-security-extensions", secure);
+    }
     qdev_init_nofail(gicdev);
     gicbusdev = SYS_BUS_DEVICE(gicdev);
     sysbus_mmio_map(gicbusdev, 0, vbi->memmap[VIRT_GIC_DIST].base);
-    sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    if (type == 3) {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_REDIST].base);
+    } else {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    }
 
     /* Wire the outputs from each CPU's generic timer to the
      * appropriate GIC PPI inputs, and the GIC's IRQ output to
@@ -390,15 +482,23 @@ static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
      */
     for (i = 0; i < smp_cpus; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
-        int ppibase = NUM_IRQS + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int ppibase = NUM_IRQS + i * GIC_INTERNAL + GIC_NR_SGIS;
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs we use for the virt board.
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = ARCH_TIMER_NS_EL1_IRQ,
+            [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ,
+            [GTIMER_HYP]  = ARCH_TIMER_NS_EL2_IRQ,
+            [GTIMER_SEC]  = ARCH_TIMER_S_EL1_IRQ,
+        };
+
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
 
         sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
         sysbus_connect_irq(gicbusdev, i + smp_cpus,
@@ -409,21 +509,29 @@ static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic)
         pic[i] = qdev_get_gpio_in(gicdev, i);
     }
 
-    fdt_add_gic_node(vbi);
+    fdt_add_gic_node(vbi, type);
 
-    create_v2m(vbi, pic);
+    if (type == 2) {
+        create_v2m(vbi, pic);
+    }
 }
 
-static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic, int uart,
+                        MemoryRegion *mem)
 {
     char *nodename;
-    hwaddr base = vbi->memmap[VIRT_UART].base;
-    hwaddr size = vbi->memmap[VIRT_UART].size;
-    int irq = vbi->irqmap[VIRT_UART];
+    hwaddr base = vbi->memmap[uart].base;
+    hwaddr size = vbi->memmap[uart].size;
+    int irq = vbi->irqmap[uart];
     const char compat[] = "arm,pl011\0arm,primecell";
     const char clocknames[] = "uartclk\0apb_pclk";
+    DeviceState *dev = qdev_create(NULL, "pl011");
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
 
-    sysbus_create_simple("pl011", base, pic[irq]);
+    qdev_init_nofail(dev);
+    memory_region_add_subregion(mem, base,
+                                sysbus_mmio_get_region(s, 0));
+    sysbus_connect_irq(s, 0, pic[irq]);
 
     nodename = g_strdup_printf("/pl011@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
@@ -440,7 +548,14 @@ static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
     qemu_fdt_setprop(vbi->fdt, nodename, "clock-names",
                          clocknames, sizeof(clocknames));
 
-    qemu_fdt_setprop_string(vbi->fdt, "/chosen", "stdout-path", nodename);
+    if (uart == VIRT_UART) {
+        qemu_fdt_setprop_string(vbi->fdt, "/chosen", "stdout-path", nodename);
+    } else {
+        /* Mark as not usable by the normal world */
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+    }
+
     g_free(nodename);
 }
 
@@ -467,6 +582,64 @@ static void create_rtc(const VirtBoardInfo *vbi, qemu_irq *pic)
     g_free(nodename);
 }
 
+static DeviceState *gpio_key_dev;
+static void virt_powerdown_req(Notifier *n, void *opaque)
+{
+    /* use gpio Pin 3 for power button event */
+    qemu_set_irq(qdev_get_gpio_in(gpio_key_dev, 0), 1);
+}
+
+static Notifier virt_system_powerdown_notifier = {
+    .notify = virt_powerdown_req
+};
+
+static void create_gpio(const VirtBoardInfo *vbi, qemu_irq *pic)
+{
+    char *nodename;
+    DeviceState *pl061_dev;
+    hwaddr base = vbi->memmap[VIRT_GPIO].base;
+    hwaddr size = vbi->memmap[VIRT_GPIO].size;
+    int irq = vbi->irqmap[VIRT_GPIO];
+    const char compat[] = "arm,pl061\0arm,primecell";
+
+    pl061_dev = sysbus_create_simple("pl061", base, pic[irq]);
+
+    uint32_t phandle = qemu_fdt_alloc_phandle(vbi->fdt);
+    nodename = g_strdup_printf("/pl061@%" PRIx64, base);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                 2, base, 2, size);
+    qemu_fdt_setprop(vbi->fdt, nodename, "compatible", compat, sizeof(compat));
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "#gpio-cells", 2);
+    qemu_fdt_setprop(vbi->fdt, nodename, "gpio-controller", NULL, 0);
+    qemu_fdt_setprop_cells(vbi->fdt, nodename, "interrupts",
+                           GIC_FDT_IRQ_TYPE_SPI, irq,
+                           GIC_FDT_IRQ_FLAGS_LEVEL_HI);
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "clocks", vbi->clock_phandle);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "clock-names", "apb_pclk");
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "phandle", phandle);
+
+    gpio_key_dev = sysbus_create_simple("gpio-key", -1,
+                                        qdev_get_gpio_in(pl061_dev, 3));
+    qemu_fdt_add_subnode(vbi->fdt, "/gpio-keys");
+    qemu_fdt_setprop_string(vbi->fdt, "/gpio-keys", "compatible", "gpio-keys");
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys", "#size-cells", 0);
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys", "#address-cells", 1);
+
+    qemu_fdt_add_subnode(vbi->fdt, "/gpio-keys/poweroff");
+    qemu_fdt_setprop_string(vbi->fdt, "/gpio-keys/poweroff",
+                            "label", "GPIO Key Poweroff");
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys/poweroff", "linux,code",
+                          KEY_POWER);
+    qemu_fdt_setprop_cells(vbi->fdt, "/gpio-keys/poweroff",
+                           "gpios", phandle, 3, 0);
+
+    /* connect powerdown request */
+    qemu_register_powerdown_notifier(&virt_system_powerdown_notifier);
+
+    g_free(nodename);
+}
+
 static void create_virtio_devices(const VirtBoardInfo *vbi, qemu_irq *pic)
 {
     int i;
@@ -532,13 +705,15 @@ static void create_virtio_devices(const VirtBoardInfo *vbi, qemu_irq *pic)
 }
 
 static void create_one_flash(const char *name, hwaddr flashbase,
-                             hwaddr flashsize)
+                             hwaddr flashsize, const char *file,
+                             MemoryRegion *sysmem)
 {
     /* Create and map a single flash device. We use the same
      * parameters as the flash devices on the Versatile Express board.
      */
     DriveInfo *dinfo = drive_get_next(IF_PFLASH);
     DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     const uint64_t sectorlength = 256 * 1024;
 
     if (dinfo) {
@@ -558,19 +733,10 @@ static void create_one_flash(const char *name, hwaddr flashbase,
     qdev_prop_set_string(dev, "name", name);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, flashbase);
-}
-
-static void create_flash(const VirtBoardInfo *vbi)
-{
-    /* Create two flash devices to fill the VIRT_FLASH space in the memmap.
-     * Any file passed via -bios goes in the first of these.
-     */
-    hwaddr flashsize = vbi->memmap[VIRT_FLASH].size / 2;
-    hwaddr flashbase = vbi->memmap[VIRT_FLASH].base;
-    char *nodename;
+    memory_region_add_subregion(sysmem, flashbase,
+                                sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0));
 
-    if (bios_name) {
+    if (file) {
         char *fn;
         int image_size;
 
@@ -580,39 +746,82 @@ static void create_flash(const VirtBoardInfo *vbi)
                          "but you cannot use both options at once");
             exit(1);
         }
-        fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+        fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, file);
         if (!fn) {
-            error_report("Could not find ROM image '%s'", bios_name);
+            error_report("Could not find ROM image '%s'", file);
             exit(1);
         }
-        image_size = load_image_targphys(fn, flashbase, flashsize);
+        image_size = load_image_mr(fn, sysbus_mmio_get_region(sbd, 0));
         g_free(fn);
         if (image_size < 0) {
-            error_report("Could not load ROM image '%s'", bios_name);
+            error_report("Could not load ROM image '%s'", file);
             exit(1);
         }
     }
+}
+
+static void create_flash(const VirtBoardInfo *vbi,
+                         MemoryRegion *sysmem,
+                         MemoryRegion *secure_sysmem)
+{
+    /* Create two flash devices to fill the VIRT_FLASH space in the memmap.
+     * Any file passed via -bios goes in the first of these.
+     * sysmem is the system memory space. secure_sysmem is the secure view
+     * of the system, and the first flash device should be made visible only
+     * there. The second flash device is visible to both secure and nonsecure.
+     * If sysmem == secure_sysmem this means there is no separate Secure
+     * address space and both flash devices are generally visible.
+     */
+    hwaddr flashsize = vbi->memmap[VIRT_FLASH].size / 2;
+    hwaddr flashbase = vbi->memmap[VIRT_FLASH].base;
+    char *nodename;
 
-    create_one_flash("virt.flash0", flashbase, flashsize);
-    create_one_flash("virt.flash1", flashbase + flashsize, flashsize);
+    create_one_flash("virt.flash0", flashbase, flashsize,
+                     bios_name, secure_sysmem);
+    create_one_flash("virt.flash1", flashbase + flashsize, flashsize,
+                     NULL, sysmem);
 
-    nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
-    qemu_fdt_add_subnode(vbi->fdt, nodename);
-    qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
-    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
-                                 2, flashbase, 2, flashsize,
-                                 2, flashbase + flashsize, 2, flashsize);
-    qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
-    g_free(nodename);
+    if (sysmem == secure_sysmem) {
+        /* Report both flash devices as a single node in the DT */
+        nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase, 2, flashsize,
+                                     2, flashbase + flashsize, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        g_free(nodename);
+    } else {
+        /* Report the devices as separate nodes so we can mark one as
+         * only visible to the secure world.
+         */
+        nodename = g_strdup_printf("/secflash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+        g_free(nodename);
+
+        nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase + flashsize, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        g_free(nodename);
+    }
 }
 
-static void create_fw_cfg(const VirtBoardInfo *vbi)
+static void create_fw_cfg(const VirtBoardInfo *vbi, AddressSpace *as)
 {
     hwaddr base = vbi->memmap[VIRT_FW_CFG].base;
     hwaddr size = vbi->memmap[VIRT_FW_CFG].size;
     char *nodename;
 
-    fw_cfg_init_mem_wide(base + 8, base, 8);
+    fw_cfg_init_mem_wide(base + 8, base, 8, base + 16, as);
 
     nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
@@ -658,10 +867,13 @@ static void create_pcie_irq_map(const VirtBoardInfo *vbi, uint32_t gic_phandle,
                            0x7           /* PCI irq */);
 }
 
-static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic,
+                        bool use_highmem)
 {
     hwaddr base_mmio = vbi->memmap[VIRT_PCIE_MMIO].base;
     hwaddr size_mmio = vbi->memmap[VIRT_PCIE_MMIO].size;
+    hwaddr base_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].base;
+    hwaddr size_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].size;
     hwaddr base_pio = vbi->memmap[VIRT_PCIE_PIO].base;
     hwaddr size_pio = vbi->memmap[VIRT_PCIE_PIO].size;
     hwaddr base_ecam = vbi->memmap[VIRT_PCIE_ECAM].base;
@@ -676,6 +888,7 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
     DeviceState *dev;
     char *nodename;
     int i;
+    PCIHostState *pci;
 
     dev = qdev_create(NULL, TYPE_GPEX_HOST);
     qdev_init_nofail(dev);
@@ -698,6 +911,16 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
                              mmio_reg, base_mmio, size_mmio);
     memory_region_add_subregion(get_system_memory(), base_mmio, mmio_alias);
 
+    if (use_highmem) {
+        /* Map high MMIO space */
+        MemoryRegion *high_mmio_alias = g_new0(MemoryRegion, 1);
+
+        memory_region_init_alias(high_mmio_alias, OBJECT(dev), "pcie-mmio-high",
+                                 mmio_reg, base_mmio_high, size_mmio_high);
+        memory_region_add_subregion(get_system_memory(), base_mmio_high,
+                                    high_mmio_alias);
+    }
+
     /* Map IO port space */
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 2, base_pio);
 
@@ -705,6 +928,19 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
         sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, pic[irq + i]);
     }
 
+    pci = PCI_HOST_BRIDGE(dev);
+    if (pci->bus) {
+        for (i = 0; i < nb_nics; i++) {
+            NICInfo *nd = &nd_table[i];
+
+            if (!nd->model) {
+                nd->model = g_strdup("virtio");
+            }
+
+            pci_nic_init_nofail(nd, pci->bus, nd->model, NULL);
+        }
+    }
+
     nodename = g_strdup_printf("/pcie@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
     qemu_fdt_setprop_string(vbi->fdt, nodename,
@@ -715,15 +951,30 @@ static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic)
     qemu_fdt_setprop_cells(vbi->fdt, nodename, "bus-range", 0,
                            nr_pcie_buses - 1);
 
-    qemu_fdt_setprop_cells(vbi->fdt, nodename, "msi-parent", vbi->v2m_phandle);
+    if (vbi->v2m_phandle) {
+        qemu_fdt_setprop_cells(vbi->fdt, nodename, "msi-parent",
+                               vbi->v2m_phandle);
+    }
 
     qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
                                  2, base_ecam, 2, size_ecam);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
-                                 1, FDT_PCI_RANGE_IOPORT, 2, 0,
-                                 2, base_pio, 2, size_pio,
-                                 1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
-                                 2, base_mmio, 2, size_mmio);
+
+    if (use_highmem) {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio,
+                                     1, FDT_PCI_RANGE_MMIO_64BIT,
+                                     2, base_mmio_high,
+                                     2, base_mmio_high, 2, size_mmio_high);
+    } else {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio);
+    }
 
     qemu_fdt_setprop_cell(vbi->fdt, nodename, "#interrupt-cells", 1);
     create_pcie_irq_map(vbi, vbi->gic_phandle, irq, nodename);
@@ -772,6 +1023,27 @@ static void create_platform_bus(VirtBoardInfo *vbi, qemu_irq *pic)
                                 sysbus_mmio_get_region(s, 0));
 }
 
+static void create_secure_ram(VirtBoardInfo *vbi, MemoryRegion *secure_sysmem)
+{
+    MemoryRegion *secram = g_new(MemoryRegion, 1);
+    char *nodename;
+    hwaddr base = vbi->memmap[VIRT_SECURE_MEM].base;
+    hwaddr size = vbi->memmap[VIRT_SECURE_MEM].size;
+
+    memory_region_init_ram(secram, NULL, "virt.secure-ram", size, &error_fatal);
+    vmstate_register_ram_global(secram);
+    memory_region_add_subregion(secure_sysmem, base, secram);
+
+    nodename = g_strdup_printf("/secram@%" PRIx64, base);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "device_type", "memory");
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg", 2, base, 2, size);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+
+    g_free(nodename);
+}
+
 static void *machvirt_dtb(const struct arm_boot_info *binfo, int *fdt_size)
 {
     const VirtBoardInfo *board = (const VirtBoardInfo *)binfo;
@@ -780,12 +1052,42 @@ static void *machvirt_dtb(const struct arm_boot_info *binfo, int *fdt_size)
     return board->fdt;
 }
 
+static void virt_build_smbios(VirtGuestInfo *guest_info)
+{
+    FWCfgState *fw_cfg = guest_info->fw_cfg;
+    uint8_t *smbios_tables, *smbios_anchor;
+    size_t smbios_tables_len, smbios_anchor_len;
+    const char *product = "QEMU Virtual Machine";
+
+    if (!fw_cfg) {
+        return;
+    }
+
+    if (kvm_enabled()) {
+        product = "KVM Virtual Machine";
+    }
+
+    smbios_set_defaults("QEMU", product,
+                        "1.0", false, true, SMBIOS_ENTRY_POINT_30);
+
+    smbios_get_tables(NULL, 0, &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
+}
+
 static
 void virt_guest_info_machine_done(Notifier *notifier, void *data)
 {
     VirtGuestInfoState *guest_info_state = container_of(notifier,
                                               VirtGuestInfoState, machine_done);
     virt_acpi_setup(&guest_info_state->info);
+    virt_build_smbios(&guest_info_state->info);
 }
 
 static void machvirt_init(MachineState *machine)
@@ -793,18 +1095,33 @@ static void machvirt_init(MachineState *machine)
     VirtMachineState *vms = VIRT_MACHINE(machine);
     qemu_irq pic[NUM_IRQS];
     MemoryRegion *sysmem = get_system_memory();
-    int n;
+    MemoryRegion *secure_sysmem = NULL;
+    int gic_version = vms->gic_version;
+    int n, virt_max_cpus;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     const char *cpu_model = machine->cpu_model;
     VirtBoardInfo *vbi;
     VirtGuestInfoState *guest_info_state = g_malloc0(sizeof *guest_info_state);
     VirtGuestInfo *guest_info = &guest_info_state->info;
     char **cpustr;
+    bool firmware_loaded = bios_name || drive_get(IF_PFLASH, 0, 0);
 
     if (!cpu_model) {
         cpu_model = "cortex-a15";
     }
 
+    /* We can probe only here because during property set
+     * KVM is not available yet
+     */
+    if (!gic_version) {
+        gic_version = kvm_arm_vgic_probe();
+        if (!gic_version) {
+            error_report("Unable to determine GIC version supported by host");
+            error_printf("KVM acceleration is probably not supported\n");
+            exit(1);
+        }
+    }
+
     /* Separate the actual CPU model name from any appended features */
     cpustr = g_strsplit(cpu_model, ",", 2);
 
@@ -815,13 +1132,55 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
+    /* If we have an EL3 boot ROM then the assumption is that it will
+     * implement PSCI itself, so disable QEMU's internal implementation
+     * so it doesn't get in the way. Instead of starting secondary
+     * CPUs in PSCI powerdown state we will start them all running and
+     * let the boot ROM sort them out.
+     * The usual case is that we do use QEMU's PSCI implementation.
+     */
+    vbi->using_psci = !(vms->secure && firmware_loaded);
+
+    /* The maximum number of CPUs depends on the GIC version, or on how
+     * many redistributors we can fit into the memory map.
+     */
+    if (gic_version == 3) {
+        virt_max_cpus = vbi->memmap[VIRT_GIC_REDIST].size / 0x20000;
+    } else {
+        virt_max_cpus = GIC_NCPU;
+    }
+
+    if (max_cpus > virt_max_cpus) {
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine 'mach-virt' (%d)",
+                     max_cpus, virt_max_cpus);
+        exit(1);
+    }
+
     vbi->smp_cpus = smp_cpus;
 
     if (machine->ram_size > vbi->memmap[VIRT_MEM].size) {
-        error_report("mach-virt: cannot model more than 30GB RAM");
+        error_report("mach-virt: cannot model more than %dGB RAM", RAMLIMIT_GB);
         exit(1);
     }
 
+    if (vms->secure) {
+        if (kvm_enabled()) {
+            error_report("mach-virt: KVM does not support Security extensions");
+            exit(1);
+        }
+
+        /* The Secure view of the world is the same as the NonSecure,
+         * but with a few extra devices. Create it as a container region
+         * containing the system memory at low priority; any secure-only
+         * devices go in at higher priority and take precedence.
+         */
+        secure_sysmem = g_new(MemoryRegion, 1);
+        memory_region_init(secure_sysmem, OBJECT(machine), "secure-memory",
+                           UINT64_MAX);
+        memory_region_add_subregion_overlap(secure_sysmem, 0, sysmem, -1);
+    }
+
     create_fdt(vbi);
 
     for (n = 0; n < smp_cpus; n++) {
@@ -832,7 +1191,7 @@ static void machvirt_init(MachineState *machine)
         char *cpuopts = g_strdup(cpustr[1]);
 
         if (!oc) {
-            fprintf(stderr, "Unable to find CPU definition\n");
+            error_report("Unable to find CPU definition");
             exit(1);
         }
         cpuobj = object_new(object_class_get_name(oc));
@@ -849,12 +1208,15 @@ static void machvirt_init(MachineState *machine)
             object_property_set_bool(cpuobj, false, "has_el3", NULL);
         }
 
-        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_HVC, "psci-conduit",
-                                NULL);
+        if (vbi->using_psci) {
+            object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_HVC,
+                                    "psci-conduit", NULL);
 
-        /* Secondary CPUs start in PSCI powered-down state */
-        if (n > 0) {
-            object_property_set_bool(cpuobj, true, "start-powered-off", NULL);
+            /* Secondary CPUs start in PSCI powered-down state */
+            if (n > 0) {
+                object_property_set_bool(cpuobj, true,
+                                         "start-powered-off", NULL);
+            }
         }
 
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
@@ -862,10 +1224,17 @@ static void machvirt_init(MachineState *machine)
                                     "reset-cbar", &error_abort);
         }
 
+        object_property_set_link(cpuobj, OBJECT(sysmem), "memory",
+                                 &error_abort);
+        if (vms->secure) {
+            object_property_set_link(cpuobj, OBJECT(secure_sysmem),
+                                     "secure-memory", &error_abort);
+        }
+
         object_property_set_bool(cpuobj, true, "realized", NULL);
     }
     g_strfreev(cpustr);
-    fdt_add_timer_nodes(vbi);
+    fdt_add_timer_nodes(vbi, gic_version);
     fdt_add_cpu_nodes(vbi);
     fdt_add_psci_node(vbi);
 
@@ -873,15 +1242,22 @@ static void machvirt_init(MachineState *machine)
                                          machine->ram_size);
     memory_region_add_subregion(sysmem, vbi->memmap[VIRT_MEM].base, ram);
 
-    create_flash(vbi);
+    create_flash(vbi, sysmem, secure_sysmem ? secure_sysmem : sysmem);
+
+    create_gic(vbi, pic, gic_version, vms->secure);
 
-    create_gic(vbi, pic);
+    create_uart(vbi, pic, VIRT_UART, sysmem);
 
-    create_uart(vbi, pic);
+    if (vms->secure) {
+        create_secure_ram(vbi, secure_sysmem);
+        create_uart(vbi, pic, VIRT_SECURE_UART, secure_sysmem);
+    }
 
     create_rtc(vbi, pic);
 
-    create_pcie(vbi, pic);
+    create_pcie(vbi, pic, vms->highmem);
+
+    create_gpio(vbi, pic);
 
     /* Create mmio transports, so the user can create virtio backends
      * (which will be automatically plugged in to the transports). If
@@ -889,13 +1265,15 @@ static void machvirt_init(MachineState *machine)
      */
     create_virtio_devices(vbi, pic);
 
-    create_fw_cfg(vbi);
+    create_fw_cfg(vbi, &address_space_memory);
     rom_set_fw(fw_cfg_find());
 
     guest_info->smp_cpus = smp_cpus;
     guest_info->fw_cfg = fw_cfg_find();
     guest_info->memmap = vbi->memmap;
     guest_info->irqmap = vbi->irqmap;
+    guest_info->use_highmem = vms->highmem;
+    guest_info->gic_version = gic_version;
     guest_info_state->machine_done.notify = virt_guest_info_machine_done;
     qemu_add_machine_init_done_notifier(&guest_info_state->machine_done);
 
@@ -907,7 +1285,7 @@ static void machvirt_init(MachineState *machine)
     vbi->bootinfo.board_id = -1;
     vbi->bootinfo.loader_start = vbi->memmap[VIRT_MEM].base;
     vbi->bootinfo.get_dtb = machvirt_dtb;
-    vbi->bootinfo.firmware_loaded = bios_name || drive_get(IF_PFLASH, 0, 0);
+    vbi->bootinfo.firmware_loaded = firmware_loaded;
     arm_load_kernel(ARM_CPU(first_cpu), &vbi->bootinfo);
 
     /*
@@ -933,45 +1311,125 @@ static void virt_set_secure(Object *obj, bool value, Error **errp)
     vms->secure = value;
 }
 
-static void virt_instance_init(Object *obj)
+static bool virt_get_highmem(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    return vms->highmem;
+}
+
+static void virt_set_highmem(Object *obj, bool value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->highmem = value;
+}
+
+static char *virt_get_gic_version(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+    const char *val = vms->gic_version == 3 ? "3" : "2";
+
+    return g_strdup(val);
+}
+
+static void virt_set_gic_version(Object *obj, const char *value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    if (!strcmp(value, "3")) {
+        vms->gic_version = 3;
+    } else if (!strcmp(value, "2")) {
+        vms->gic_version = 2;
+    } else if (!strcmp(value, "host")) {
+        vms->gic_version = 0; /* Will probe later */
+    } else {
+        error_setg(errp, "Invalid gic-version value");
+        error_append_hint(errp, "Valid values are 3, 2, host.\n");
+    }
+}
+
+static void virt_machine_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->init = machvirt_init;
+    /* Start max_cpus at the maximum QEMU supports. We'll further restrict
+     * it later in machvirt_init, where we have more information about the
+     * configuration of the particular instance.
+     */
+    mc->max_cpus = MAX_CPUMASK_BITS;
+    mc->has_dynamic_sysbus = true;
+    mc->block_default_type = IF_VIRTIO;
+    mc->no_cdrom = 1;
+    mc->pci_allow_0_address = true;
+}
+
+static const TypeInfo virt_machine_info = {
+    .name          = TYPE_VIRT_MACHINE,
+    .parent        = TYPE_MACHINE,
+    .abstract      = true,
+    .instance_size = sizeof(VirtMachineState),
+    .class_size    = sizeof(VirtMachineClass),
+    .class_init    = virt_machine_class_init,
+};
+
+static void virt_2_6_instance_init(Object *obj)
 {
     VirtMachineState *vms = VIRT_MACHINE(obj);
 
-    /* EL3 is enabled by default on virt */
-    vms->secure = true;
+    /* EL3 is disabled by default on virt: this makes us consistent
+     * between KVM and TCG for this board, and it also allows us to
+     * boot UEFI blobs which assume no TrustZone support.
+     */
+    vms->secure = false;
     object_property_add_bool(obj, "secure", virt_get_secure,
                              virt_set_secure, NULL);
     object_property_set_description(obj, "secure",
                                     "Set on/off to enable/disable the ARM "
                                     "Security Extensions (TrustZone)",
                                     NULL);
+
+    /* High memory is enabled by default */
+    vms->highmem = true;
+    object_property_add_bool(obj, "highmem", virt_get_highmem,
+                             virt_set_highmem, NULL);
+    object_property_set_description(obj, "highmem",
+                                    "Set on/off to enable/disable using "
+                                    "physical address space above 32 bits",
+                                    NULL);
+    /* Default GIC type is v2 */
+    vms->gic_version = 2;
+    object_property_add_str(obj, "gic-version", virt_get_gic_version,
+                        virt_set_gic_version, NULL);
+    object_property_set_description(obj, "gic-version",
+                                    "Set GIC version. "
+                                    "Valid values are 2, 3 and host", NULL);
 }
 
-static void virt_class_init(ObjectClass *oc, void *data)
+static void virt_2_6_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    static GlobalProperty compat_props[] = {
+        { /* end of list */ }
+    };
 
-    mc->name = TYPE_VIRT_MACHINE;
-    mc->desc = "ARM Virtual Machine",
-    mc->init = machvirt_init;
-    mc->max_cpus = 8;
-    mc->has_dynamic_sysbus = true;
-    mc->block_default_type = IF_VIRTIO;
-    mc->no_cdrom = 1;
+    mc->desc = "QEMU 2.6 ARM Virtual Machine";
+    mc->alias = "virt";
+    mc->compat_props = compat_props;
 }
 
 static const TypeInfo machvirt_info = {
-    .name = TYPE_VIRT_MACHINE,
-    .parent = TYPE_MACHINE,
-    .instance_size = sizeof(VirtMachineState),
-    .instance_init = virt_instance_init,
-    .class_size = sizeof(VirtMachineClass),
-    .class_init = virt_class_init,
+    .name = MACHINE_TYPE_NAME("virt-2.6"),
+    .parent = TYPE_VIRT_MACHINE,
+    .instance_init = virt_2_6_instance_init,
+    .class_init = virt_2_6_class_init,
 };
 
 static void machvirt_machine_init(void)
 {
+    type_register_static(&virt_machine_info);
     type_register_static(&machvirt_info);
 }
 
-machine_init(machvirt_machine_init);
+type_init(machvirt_machine_init);
diff --git a/qemu/hw/arm/xilinx_zynq.c b/qemu/hw/arm/xilinx_zynq.c
index a4e7b5c63..98b17c9ae 100644
--- a/qemu/hw/arm/xilinx_zynq.c
+++ b/qemu/hw/arm/xilinx_zynq.c
@@ -15,6 +15,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "net/net.h"
@@ -24,8 +28,10 @@
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "hw/loader.h"
-#include "hw/ssi.h"
+#include "hw/misc/zynq-xadc.h"
+#include "hw/ssi/ssi.h"
 #include "qemu/error-report.h"
+#include "hw/sd/sd.h"
 
 #define NUM_SPI_FLASHES 4
 #define NUM_QSPI_FLASHES 2
@@ -43,6 +49,45 @@ static const int dma_irqs[8] = {
     46, 47, 48, 49, 72, 73, 74, 75
 };
 
+#define BOARD_SETUP_ADDR        0x100
+
+#define SLCR_LOCK_OFFSET        0x004
+#define SLCR_UNLOCK_OFFSET      0x008
+#define SLCR_ARM_PLL_OFFSET     0x100
+
+#define SLCR_XILINX_UNLOCK_KEY  0xdf0d
+#define SLCR_XILINX_LOCK_KEY    0x767b
+
+#define ARMV7_IMM16(x) (extract32((x),  0, 12) | \
+                        extract32((x), 12,  4) << 16)
+
+/* Write immediate val to address r0 + addr. r0 should contain base offset
+ * of the SLCR block. Clobbers r1.
+ */
+
+#define SLCR_WRITE(addr, val) \
+    0xe3001000 + ARMV7_IMM16(extract32((val),  0, 16)), /* movw r1 ... */ \
+    0xe3401000 + ARMV7_IMM16(extract32((val), 16, 16)), /* movt r1 ... */ \
+    0xe5801000 + (addr)
+
+static void zynq_write_board_setup(ARMCPU *cpu,
+                                   const struct arm_boot_info *info)
+{
+    int n;
+    uint32_t board_setup_blob[] = {
+        0xe3a004f8, /* mov r0, #0xf8000000 */
+        SLCR_WRITE(SLCR_UNLOCK_OFFSET, SLCR_XILINX_UNLOCK_KEY),
+        SLCR_WRITE(SLCR_ARM_PLL_OFFSET, 0x00014008),
+        SLCR_WRITE(SLCR_LOCK_OFFSET, SLCR_XILINX_LOCK_KEY),
+        0xe12fff1e, /* bx lr */
+    };
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), BOARD_SETUP_ADDR);
+}
+
 static struct arm_boot_info zynq_binfo = {};
 
 static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
@@ -113,10 +158,11 @@ static void zynq_init(MachineState *machine)
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ext_ram = g_new(MemoryRegion, 1);
     MemoryRegion *ocm_ram = g_new(MemoryRegion, 1);
-    DeviceState *dev;
+    DeviceState *dev, *carddev;
     SysBusDevice *busdev;
+    DriveInfo *di;
+    BlockBackend *blk;
     qemu_irq pic[64];
-    Error *err = NULL;
     int n;
 
     if (!cpu_model) {
@@ -131,29 +177,14 @@ static void zynq_init(MachineState *machine)
      * realization.
      */
     if (object_property_find(OBJECT(cpu), "has_el3", NULL)) {
-        object_property_set_bool(OBJECT(cpu), false, "has_el3", &err);
-        if (err) {
-            error_report_err(err);
-            exit(1);
-        }
-    }
-
-    object_property_set_int(OBJECT(cpu), ZYNQ_BOARD_MIDR, "midr", &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
+        object_property_set_bool(OBJECT(cpu), false, "has_el3", &error_fatal);
     }
 
-    object_property_set_int(OBJECT(cpu), MPCORE_PERIPHBASE, "reset-cbar", &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
-    }
-    object_property_set_bool(OBJECT(cpu), true, "realized", &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
-    }
+    object_property_set_int(OBJECT(cpu), ZYNQ_BOARD_MIDR, "midr",
+                            &error_fatal);
+    object_property_set_int(OBJECT(cpu), MPCORE_PERIPHBASE, "reset-cbar",
+                            &error_fatal);
+    object_property_set_bool(OBJECT(cpu), true, "realized", &error_fatal);
 
     /* max 2GB ram */
     if (ram_size > 0x80000000) {
@@ -167,7 +198,7 @@ static void zynq_init(MachineState *machine)
 
     /* 256K of on-chip memory */
     memory_region_init_ram(ocm_ram, NULL, "zynq.ocm_ram", 256 << 10,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ocm_ram);
     memory_region_add_subregion(address_space_mem, 0xFFFC0000, ocm_ram);
 
@@ -220,11 +251,28 @@ static void zynq_init(MachineState *machine)
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xE0100000);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[56-IRQ_OFFSET]);
 
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
     dev = qdev_create(NULL, "generic-sdhci");
     qdev_init_nofail(dev);
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xE0101000);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[79-IRQ_OFFSET]);
 
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
+    dev = qdev_create(NULL, TYPE_ZYNQ_XADC);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xF8007100);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[39-IRQ_OFFSET]);
+
     dev = qdev_create(NULL, "pl330");
     qdev_prop_set_uint8(dev, "num_chnls",  8);
     qdev_prop_set_uint8(dev, "num_periph_req",  4);
@@ -252,21 +300,19 @@ static void zynq_init(MachineState *machine)
     zynq_binfo.nb_cpus = 1;
     zynq_binfo.board_id = 0xd32;
     zynq_binfo.loader_start = 0;
+    zynq_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+    zynq_binfo.write_board_setup = zynq_write_board_setup;
+
     arm_load_kernel(ARM_CPU(first_cpu), &zynq_binfo);
 }
 
-static QEMUMachine zynq_machine = {
-    .name = "xilinx-zynq-a9",
-    .desc = "Xilinx Zynq Platform Baseboard for Cortex-A9",
-    .init = zynq_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 1,
-    .no_sdcard = 1,
-};
-
-static void zynq_machine_init(void)
+static void zynq_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&zynq_machine);
+    mc->desc = "Xilinx Zynq Platform Baseboard for Cortex-A9";
+    mc->init = zynq_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 1;
+    mc->no_sdcard = 1;
 }
 
-machine_init(zynq_machine_init);
+DEFINE_MACHINE("xilinx-zynq-a9", zynq_machine_init)
diff --git a/qemu/hw/arm/xlnx-ep108.c b/qemu/hw/arm/xlnx-ep108.c
index f94da86cb..5f480182b 100644
--- a/qemu/hw/arm/xlnx-ep108.c
+++ b/qemu/hw/arm/xlnx-ep108.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/arm/xlnx-zynqmp.h"
 #include "hw/boards.h"
 #include "qemu/error-report.h"
@@ -25,42 +29,76 @@ typedef struct XlnxEP108 {
     MemoryRegion ddr_ram;
 } XlnxEP108;
 
-/* Max 2GB RAM */
-#define EP108_MAX_RAM_SIZE 0x80000000ull
-
 static struct arm_boot_info xlnx_ep108_binfo;
 
 static void xlnx_ep108_init(MachineState *machine)
 {
     XlnxEP108 *s = g_new0(XlnxEP108, 1);
-    Error *err = NULL;
+    int i;
+    uint64_t ram_size = machine->ram_size;
+
+    /* Create the memory region to pass to the SoC */
+    if (ram_size > XLNX_ZYNQMP_MAX_RAM_SIZE) {
+        error_report("ERROR: RAM size 0x%" PRIx64 " above max supported of "
+                     "0x%llx", ram_size,
+                     XLNX_ZYNQMP_MAX_RAM_SIZE);
+        exit(1);
+    }
+
+    if (ram_size < 0x08000000) {
+        qemu_log("WARNING: RAM size 0x%" PRIx64 " is small for EP108",
+                 ram_size);
+    }
+
+    memory_region_allocate_system_memory(&s->ddr_ram, NULL, "ddr-ram",
+                                         ram_size);
 
     object_initialize(&s->soc, sizeof(s->soc), TYPE_XLNX_ZYNQMP);
     object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
                               &error_abort);
 
-    object_property_set_bool(OBJECT(&s->soc), true, "realized", &err);
-    if (err) {
-        error_report("%s", error_get_pretty(err));
-        exit(1);
-    }
+    object_property_set_link(OBJECT(&s->soc), OBJECT(&s->ddr_ram),
+                         "ddr-ram", &error_abort);
 
-    if (machine->ram_size > EP108_MAX_RAM_SIZE) {
-        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
-                     "reduced to %llx", machine->ram_size, EP108_MAX_RAM_SIZE);
-        machine->ram_size = EP108_MAX_RAM_SIZE;
-    }
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* Create and plug in the SD cards */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        BusState *bus;
+        DriveInfo *di = drive_get_next(IF_SD);
+        BlockBackend *blk = di ? blk_by_legacy_dinfo(di) : NULL;
+        DeviceState *carddev;
+        char *bus_name;
 
-    if (machine->ram_size <= 0x08000000) {
-        qemu_log("WARNING: RAM size " RAM_ADDR_FMT " is small for EP108",
-                 machine->ram_size);
+        bus_name = g_strdup_printf("sd-bus%d", i);
+        bus = qdev_get_child_bus(DEVICE(&s->soc), bus_name);
+        g_free(bus_name);
+        if (!bus) {
+            error_report("No SD bus found for SD card %d", i);
+            exit(1);
+        }
+        carddev = qdev_create(bus, TYPE_SD_CARD);
+        qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+        object_property_set_bool(OBJECT(carddev), true, "realized",
+                                 &error_fatal);
     }
 
-    memory_region_allocate_system_memory(&s->ddr_ram, NULL, "ddr-ram",
-                                         machine->ram_size);
-    memory_region_add_subregion(get_system_memory(), 0, &s->ddr_ram);
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        SSIBus *spi_bus;
+        DeviceState *flash_dev;
+        qemu_irq cs_line;
+        gchar *bus_name = g_strdup_printf("spi%d", i);
+
+        spi_bus = (SSIBus *)qdev_get_child_bus(DEVICE(&s->soc), bus_name);
+        g_free(bus_name);
 
-    xlnx_ep108_binfo.ram_size = machine->ram_size;
+        flash_dev = ssi_create_slave(spi_bus, "sst25wf080");
+        cs_line = qdev_get_gpio_in_named(flash_dev, SSI_GPIO_CS, 0);
+
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->soc.spi[i]), 1, cs_line);
+    }
+
+    xlnx_ep108_binfo.ram_size = ram_size;
     xlnx_ep108_binfo.kernel_filename = machine->kernel_filename;
     xlnx_ep108_binfo.kernel_cmdline = machine->kernel_cmdline;
     xlnx_ep108_binfo.initrd_filename = machine->initrd_filename;
@@ -68,15 +106,10 @@ static void xlnx_ep108_init(MachineState *machine)
     arm_load_kernel(s->soc.boot_cpu_ptr, &xlnx_ep108_binfo);
 }
 
-static QEMUMachine xlnx_ep108_machine = {
-    .name = "xlnx-ep108",
-    .desc = "Xilinx ZynqMP EP108 board",
-    .init = xlnx_ep108_init,
-};
-
-static void xlnx_ep108_machine_init(void)
+static void xlnx_ep108_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xlnx_ep108_machine);
+    mc->desc = "Xilinx ZynqMP EP108 board";
+    mc->init = xlnx_ep108_init;
 }
 
-machine_init(xlnx_ep108_machine_init);
+DEFINE_MACHINE("xlnx-ep108", xlnx_ep108_machine_init)
diff --git a/qemu/hw/arm/xlnx-zynqmp.c b/qemu/hw/arm/xlnx-zynqmp.c
index 62ef4ceb3..4d504da64 100644
--- a/qemu/hw/arm/xlnx-zynqmp.c
+++ b/qemu/hw/arm/xlnx-zynqmp.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/arm/xlnx-zynqmp.h"
 #include "hw/intc/arm_gic_common.h"
 #include "exec/address-spaces.h"
@@ -28,6 +32,10 @@
 #define GIC_DIST_ADDR       0xf9010000
 #define GIC_CPU_ADDR        0xf9020000
 
+#define SATA_INTR           133
+#define SATA_ADDR           0xFD0C0000
+#define SATA_NUM_PORTS      2
+
 static const uint64_t gem_addr[XLNX_ZYNQMP_NUM_GEMS] = {
     0xFF0B0000, 0xFF0C0000, 0xFF0D0000, 0xFF0E0000,
 };
@@ -44,6 +52,22 @@ static const int uart_intr[XLNX_ZYNQMP_NUM_UARTS] = {
     21, 22,
 };
 
+static const uint64_t sdhci_addr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    0xFF160000, 0xFF170000,
+};
+
+static const int sdhci_intr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    48, 49,
+};
+
+static const uint64_t spi_addr[XLNX_ZYNQMP_NUM_SPIS] = {
+    0xFF040000, 0xFF050000,
+};
+
+static const int spi_intr[XLNX_ZYNQMP_NUM_SPIS] = {
+    19, 20,
+};
+
 typedef struct XlnxZynqMPGICRegion {
     int region_index;
     uint32_t address;
@@ -78,6 +102,11 @@ static void xlnx_zynqmp_init(Object *obj)
                                   &error_abort);
     }
 
+    object_property_add_link(obj, "ddr-ram", TYPE_MEMORY_REGION,
+                             (Object **)&s->ddr_ram,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
+
     object_initialize(&s->gic, sizeof(s->gic), TYPE_ARM_GIC);
     qdev_set_parent_bus(DEVICE(&s->gic), sysbus_get_default());
 
@@ -90,6 +119,22 @@ static void xlnx_zynqmp_init(Object *obj)
         object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_CADENCE_UART);
         qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
     }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_SYSBUS_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        object_initialize(&s->sdhci[i], sizeof(s->sdhci[i]),
+                          TYPE_SYSBUS_SDHCI);
+        qdev_set_parent_bus(DEVICE(&s->sdhci[i]),
+                            sysbus_get_default());
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        object_initialize(&s->spi[i], sizeof(s->spi[i]),
+                          TYPE_XILINX_SPIPS);
+        qdev_set_parent_bus(DEVICE(&s->spi[i]), sysbus_get_default());
+    }
 }
 
 static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
@@ -97,16 +142,63 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
     XlnxZynqMPState *s = XLNX_ZYNQMP(dev);
     MemoryRegion *system_memory = get_system_memory();
     uint8_t i;
+    uint64_t ram_size;
     const char *boot_cpu = s->boot_cpu ? s->boot_cpu : "apu-cpu[0]";
+    ram_addr_t ddr_low_size, ddr_high_size;
     qemu_irq gic_spi[GIC_NUM_SPI_INTR];
     Error *err = NULL;
 
+    ram_size = memory_region_size(s->ddr_ram);
+
+    /* Create the DDR Memory Regions. User friendly checks should happen at
+     * the board level
+     */
+    if (ram_size > XLNX_ZYNQMP_MAX_LOW_RAM_SIZE) {
+        /* The RAM size is above the maximum available for the low DDR.
+         * Create the high DDR memory region as well.
+         */
+        assert(ram_size <= XLNX_ZYNQMP_MAX_RAM_SIZE);
+        ddr_low_size = XLNX_ZYNQMP_MAX_LOW_RAM_SIZE;
+        ddr_high_size = ram_size - XLNX_ZYNQMP_MAX_LOW_RAM_SIZE;
+
+        memory_region_init_alias(&s->ddr_ram_high, NULL,
+                                 "ddr-ram-high", s->ddr_ram,
+                                  ddr_low_size, ddr_high_size);
+        memory_region_add_subregion(get_system_memory(),
+                                    XLNX_ZYNQMP_HIGH_RAM_START,
+                                    &s->ddr_ram_high);
+    } else {
+        /* RAM must be non-zero */
+        assert(ram_size);
+        ddr_low_size = ram_size;
+    }
+
+    memory_region_init_alias(&s->ddr_ram_low, NULL,
+                             "ddr-ram-low", s->ddr_ram,
+                              0, ddr_low_size);
+    memory_region_add_subregion(get_system_memory(), 0, &s->ddr_ram_low);
+
+    /* Create the four OCM banks */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_OCM_BANKS; i++) {
+        char *ocm_name = g_strdup_printf("zynqmp.ocm_ram_bank_%d", i);
+
+        memory_region_init_ram(&s->ocm_ram[i], NULL, ocm_name,
+                               XLNX_ZYNQMP_OCM_RAM_SIZE, &error_fatal);
+        vmstate_register_ram_global(&s->ocm_ram[i]);
+        memory_region_add_subregion(get_system_memory(),
+                                    XLNX_ZYNQMP_OCM_RAM_0_ADDRESS +
+                                        i * XLNX_ZYNQMP_OCM_RAM_SIZE,
+                                    &s->ocm_ram[i]);
+
+        g_free(ocm_name);
+    }
+
     qdev_prop_set_uint32(DEVICE(&s->gic), "num-irq", GIC_NUM_SPI_INTR + 32);
     qdev_prop_set_uint32(DEVICE(&s->gic), "revision", 2);
     qdev_prop_set_uint32(DEVICE(&s->gic), "num-cpu", XLNX_ZYNQMP_NUM_APU_CPUS);
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err) {
-        error_propagate((errp), (err));
+        error_propagate(errp, err);
         return;
     }
     assert(ARRAY_SIZE(xlnx_zynqmp_gic_regions) == XLNX_ZYNQMP_GIC_REGIONS);
@@ -147,16 +239,11 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         g_free(name);
 
         object_property_set_int(OBJECT(&s->apu_cpu[i]), GIC_BASE_ADDR,
-                                "reset-cbar", &err);
-        if (err) {
-            error_propagate((errp), (err));
-            return;
-        }
-
+                                "reset-cbar", &error_abort);
         object_property_set_bool(OBJECT(&s->apu_cpu[i]), true, "realized",
                                  &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
 
@@ -185,22 +272,17 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         g_free(name);
 
         object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "reset-hivecs",
-                                 &err);
-        if (err != NULL) {
-            error_propagate(errp, err);
-            return;
-        }
-
+                                 &error_abort);
         object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "realized",
                                  &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
     }
 
     if (!s->boot_cpu_ptr) {
-        error_setg(errp, "ZynqMP Boot cpu %s not found\n", boot_cpu);
+        error_setg(errp, "ZynqMP Boot cpu %s not found", boot_cpu);
         return;
     }
 
@@ -217,7 +299,7 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
         }
         object_property_set_bool(OBJECT(&s->gem[i]), true, "realized", &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
         sysbus_mmio_map(SYS_BUS_DEVICE(&s->gem[i]), 0, gem_addr[i]);
@@ -228,13 +310,62 @@ static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < XLNX_ZYNQMP_NUM_UARTS; i++) {
         object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
         if (err) {
-            error_propagate((errp), (err));
+            error_propagate(errp, err);
             return;
         }
         sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, uart_addr[i]);
         sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
                            gic_spi[uart_intr[i]]);
     }
+
+    object_property_set_int(OBJECT(&s->sata), SATA_NUM_PORTS, "num-ports",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, SATA_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, gic_spi[SATA_INTR]);
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        char *bus_name;
+
+        object_property_set_bool(OBJECT(&s->sdhci[i]), true,
+                                 "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                        sdhci_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                           gic_spi[sdhci_intr[i]]);
+        /* Alias controller SD bus to the SoC itself */
+        bus_name = g_strdup_printf("sd-bus%d", i);
+        object_property_add_alias(OBJECT(s), bus_name,
+                                  OBJECT(&s->sdhci[i]), "sd-bus",
+                                  &error_abort);
+        g_free(bus_name);
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        gchar *bus_name;
+
+        object_property_set_bool(OBJECT(&s->spi[i]), true, "realized", &err);
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->spi[i]), 0, spi_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->spi[i]), 0,
+                           gic_spi[spi_intr[i]]);
+
+        /* Alias controller SPI bus to the SoC itself */
+        bus_name = g_strdup_printf("spi%d", i);
+        object_property_add_alias(OBJECT(s), bus_name,
+                                  OBJECT(&s->spi[i]), "spi0",
+                                  &error_abort);
+	g_free(bus_name);
+    }
 }
 
 static Property xlnx_zynqmp_props[] = {
@@ -248,6 +379,12 @@ static void xlnx_zynqmp_class_init(ObjectClass *oc, void *data)
 
     dc->props = xlnx_zynqmp_props;
     dc->realize = xlnx_zynqmp_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo xlnx_zynqmp_type_info = {
diff --git a/qemu/hw/arm/z2.c b/qemu/hw/arm/z2.c
index 17355479a..aea895a50 100644
--- a/qemu/hw/arm/z2.c
+++ b/qemu/hw/arm/z2.c
@@ -11,12 +11,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/boards.h"
 #include "sysemu/sysemu.h"
 #include "hw/block/flash.h"
@@ -372,15 +373,10 @@ static void z2_init(MachineState *machine)
     arm_load_kernel(mpu->cpu, &z2_binfo);
 }
 
-static QEMUMachine z2_machine = {
-    .name = "z2",
-    .desc = "Zipit Z2 (PXA27x)",
-    .init = z2_init,
-};
-
-static void z2_machine_init(void)
+static void z2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&z2_machine);
+    mc->desc = "Zipit Z2 (PXA27x)";
+    mc->init = z2_init;
 }
 
-machine_init(z2_machine_init);
+DEFINE_MACHINE("z2", z2_machine_init)
diff --git a/qemu/hw/audio/ac97.c b/qemu/hw/audio/ac97.c
index b17383555..cbd959e0b 100644
--- a/qemu/hw/audio/ac97.c
+++ b/qemu/hw/audio/ac97.c
@@ -17,6 +17,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
diff --git a/qemu/hw/audio/adlib.c b/qemu/hw/audio/adlib.c
index 656eb3773..7836446fc 100644
--- a/qemu/hw/audio/adlib.c
+++ b/qemu/hw/audio/adlib.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -57,11 +59,6 @@ void YMF262UpdateOneQEMU (int which, INT16 *dst, int length);
 #define SHIFT 1
 #endif
 
-#define IO_READ_PROTO(name) \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_ADLIB "adlib"
 #define ADLIB(obj) OBJECT_CHECK(AdlibState, (obj), TYPE_ADLIB)
 
@@ -124,7 +121,7 @@ static void adlib_kill_timers (AdlibState *s)
     }
 }
 
-static IO_WRITE_PROTO (adlib_write)
+static void adlib_write(void *opaque, uint32_t nport, uint32_t val)
 {
     AdlibState *s = opaque;
     int a = nport & 3;
@@ -141,7 +138,7 @@ static IO_WRITE_PROTO (adlib_write)
 #endif
 }
 
-static IO_READ_PROTO (adlib_read)
+static uint32_t adlib_read(void *opaque, uint32_t nport)
 {
     AdlibState *s = opaque;
     uint8_t data;
@@ -173,7 +170,7 @@ static void timer_handler (int c, double interval_Sec)
 
     s->ticking[n] = 1;
 #ifdef DEBUG
-    interval = get_ticks_per_sec () * interval_Sec;
+    interval = NANOSECONDS_PER_SECOND * interval_Sec;
     exp = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + interval;
     s->exp[n] = exp;
 #endif
diff --git a/qemu/hw/audio/cs4231.c b/qemu/hw/audio/cs4231.c
index 6325a8cea..caf97c169 100644
--- a/qemu/hw/audio/cs4231.c
+++ b/qemu/hw/audio/cs4231.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
diff --git a/qemu/hw/audio/cs4231a.c b/qemu/hw/audio/cs4231a.c
index f96f561c7..3ecd0582b 100644
--- a/qemu/hw/audio/cs4231a.c
+++ b/qemu/hw/audio/cs4231a.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -69,6 +70,7 @@ typedef struct CSState {
     uint32_t irq;
     uint32_t dma;
     uint32_t port;
+    IsaDma *isa_dma;
     int shift;
     int dma_running;
     int audio_free;
@@ -264,6 +266,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
 {
     int xtal;
     struct audsettings as;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
 
 #ifdef DEBUG_XLAW
     if (val == 0 || val == 32)
@@ -327,7 +330,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
 
     if (s->dregs[Interface_Configuration] & PEN) {
         if (!s->dma_running) {
-            DMA_hold_DREQ (s->dma);
+            k->hold_DREQ(s->isa_dma, s->dma);
             AUD_set_active_out (s->voice, 1);
             s->transferred = 0;
         }
@@ -335,7 +338,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
     }
     else {
         if (s->dma_running) {
-            DMA_release_DREQ (s->dma);
+            k->release_DREQ(s->isa_dma, s->dma);
             AUD_set_active_out (s->voice, 0);
         }
         s->dma_running = 0;
@@ -344,7 +347,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
 
  error:
     if (s->dma_running) {
-        DMA_release_DREQ (s->dma);
+        k->release_DREQ(s->isa_dma, s->dma);
         AUD_set_active_out (s->voice, 0);
     }
 }
@@ -452,7 +455,8 @@ static void cs_write (void *opaque, hwaddr addr,
             }
             else {
                 if (s->dma_running) {
-                    DMA_release_DREQ (s->dma);
+                    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
+                    k->release_DREQ(s->isa_dma, s->dma);
                     AUD_set_active_out (s->voice, 0);
                     s->dma_running = 0;
                 }
@@ -517,6 +521,7 @@ static int cs_write_audio (CSState *s, int nchan, int dma_pos,
 {
     int temp, net;
     uint8_t tmpbuf[4096];
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
 
     temp = len;
     net = 0;
@@ -531,7 +536,7 @@ static int cs_write_audio (CSState *s, int nchan, int dma_pos,
             to_copy = sizeof (tmpbuf);
         }
 
-        copied = DMA_read_memory (nchan, tmpbuf, dma_pos, to_copy);
+        copied = k->read_memory(s->isa_dma, nchan, tmpbuf, dma_pos, to_copy);
         if (s->tab) {
             int i;
             int16_t linbuf[4096];
@@ -599,7 +604,8 @@ static int cs4231a_pre_load (void *opaque)
     CSState *s = opaque;
 
     if (s->dma_running) {
-        DMA_release_DREQ (s->dma);
+        IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
+        k->release_DREQ(s->isa_dma, s->dma);
         AUD_set_active_out (s->voice, 0);
     }
     s->dma_running = 0;
@@ -655,13 +661,15 @@ static void cs4231a_realizefn (DeviceState *dev, Error **errp)
 {
     ISADevice *d = ISA_DEVICE (dev);
     CSState *s = CS4231A (dev);
+    IsaDmaClass *k;
 
     isa_init_irq (d, &s->pic, s->irq);
+    s->isa_dma = isa_get_dma(isa_bus_from_device(d), s->dma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->dma, cs_dma_read, s);
 
     isa_register_ioport (d, &s->ioports, s->port);
 
-    DMA_register_channel (s->dma, cs_dma_read, s);
-
     AUD_register_card ("cs4231a", &s->card);
 }
 
diff --git a/qemu/hw/audio/es1370.c b/qemu/hw/audio/es1370.c
index 8e7bcf503..8449b5f43 100644
--- a/qemu/hw/audio/es1370.c
+++ b/qemu/hw/audio/es1370.c
@@ -26,6 +26,7 @@
 /* #define VERBOSE_ES1370 */
 #define SILENT_ES1370
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -157,11 +158,6 @@ static const unsigned dac1_samplerate[] = { 5512, 11025, 22050, 44100 };
 #define DAC2_CHANNEL 1
 #define ADC_CHANNEL 2
 
-#define IO_READ_PROTO(n) \
-static uint32_t n (void *opaque, uint32_t addr)
-#define IO_WRITE_PROTO(n) \
-static void n (void *opaque, uint32_t addr, uint32_t val)
-
 static void es1370_dac1_callback (void *opaque, int free);
 static void es1370_dac2_callback (void *opaque, int free);
 static void es1370_adc_callback (void *opaque, int avail);
@@ -293,6 +289,10 @@ struct chan_bits {
                        uint32_t *old_freq, uint32_t *new_freq);
 };
 
+#define TYPE_ES1370 "ES1370"
+#define ES1370(obj) \
+    OBJECT_CHECK(ES1370State, (obj), TYPE_ES1370)
+
 static void es1370_dac1_calc_freq (ES1370State *s, uint32_t ctl,
                                    uint32_t *old_freq, uint32_t *new_freq);
 static void es1370_dac2_and_adc_calc_freq (ES1370State *s, uint32_t ctl,
@@ -474,7 +474,7 @@ static inline uint32_t es1370_fixup (ES1370State *s, uint32_t addr)
     return addr;
 }
 
-IO_WRITE_PROTO (es1370_writeb)
+static void es1370_writeb(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     uint32_t shift, mask;
@@ -512,7 +512,7 @@ IO_WRITE_PROTO (es1370_writeb)
     }
 }
 
-IO_WRITE_PROTO (es1370_writew)
+static void es1370_writew(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     addr = es1370_fixup (s, addr);
@@ -549,7 +549,7 @@ IO_WRITE_PROTO (es1370_writew)
     }
 }
 
-IO_WRITE_PROTO (es1370_writel)
+static void es1370_writel(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -615,7 +615,7 @@ IO_WRITE_PROTO (es1370_writel)
     }
 }
 
-IO_READ_PROTO (es1370_readb)
+static uint32_t es1370_readb(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
@@ -650,7 +650,7 @@ IO_READ_PROTO (es1370_readb)
     return val;
 }
 
-IO_READ_PROTO (es1370_readw)
+static uint32_t es1370_readw(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -692,7 +692,7 @@ IO_READ_PROTO (es1370_readw)
     return val;
 }
 
-IO_READ_PROTO (es1370_readl)
+static uint32_t es1370_readl(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
@@ -1018,7 +1018,7 @@ static void es1370_on_reset (void *opaque)
 
 static void es1370_realize(PCIDevice *dev, Error **errp)
 {
-    ES1370State *s = DO_UPCAST (ES1370State, dev, dev);
+    ES1370State *s = ES1370(dev);
     uint8_t *c = s->dev.config;
 
     c[PCI_STATUS + 1] = PCI_STATUS_DEVSEL_SLOW >> 8;
@@ -1043,7 +1043,7 @@ static void es1370_realize(PCIDevice *dev, Error **errp)
 
 static int es1370_init (PCIBus *bus)
 {
-    pci_create_simple (bus, -1, "ES1370");
+    pci_create_simple (bus, -1, TYPE_ES1370);
     return 0;
 }
 
@@ -1064,7 +1064,7 @@ static void es1370_class_init (ObjectClass *klass, void *data)
 }
 
 static const TypeInfo es1370_info = {
-    .name          = "ES1370",
+    .name          = TYPE_ES1370,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (ES1370State),
     .class_init    = es1370_class_init,
diff --git a/qemu/hw/audio/fmopl.c b/qemu/hw/audio/fmopl.c
index adcef2d3b..731110fe8 100644
--- a/qemu/hw/audio/fmopl.c
+++ b/qemu/hw/audio/fmopl.c
@@ -32,10 +32,7 @@
 
 #define HAS_YM3812	1
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 #include <math.h>
 //#include "driver.h"		/* use M.A.M.E. */
 #include "fmopl.h"
@@ -1177,7 +1174,7 @@ void OPLResetChip(FM_OPL *OPL)
 	OPLWriteReg(OPL,0x03,0); /* Timer2 */
 	OPLWriteReg(OPL,0x04,0); /* IRQ mask clear */
 	for(i = 0xff ; i >= 0x20 ; i-- ) OPLWriteReg(OPL,i,0);
-	/* reset OPerator paramater */
+	/* reset operator parameter */
 	for( c = 0 ; c < OPL->max_ch ; c++ )
 	{
 		OPL_CH *CH = &OPL->P_CH[c];
diff --git a/qemu/hw/audio/gus.c b/qemu/hw/audio/gus.c
index 86223a954..9dd6947be 100644
--- a/qemu/hw/audio/gus.c
+++ b/qemu/hw/audio/gus.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -41,11 +43,6 @@
 #define GUS_ENDIANNESS 0
 #endif
 
-#define IO_READ_PROTO(name) \
-    static uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    static void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_GUS "gus"
 #define GUS(obj) OBJECT_CHECK (GUSState, (obj), TYPE_GUS)
 
@@ -62,16 +59,17 @@ typedef struct GUSState {
     SWVoiceOut *voice;
     int64_t last_ticks;
     qemu_irq pic;
+    IsaDma *isa_dma;
 } GUSState;
 
-IO_READ_PROTO (gus_readb)
+static uint32_t gus_readb(void *opaque, uint32_t nport)
 {
     GUSState *s = opaque;
 
     return gus_read (&s->emu, nport, 1);
 }
 
-IO_WRITE_PROTO (gus_writeb)
+static void gus_writeb(void *opaque, uint32_t nport, uint32_t val)
 {
     GUSState *s = opaque;
 
@@ -172,34 +170,36 @@ void GUS_irqclear (GUSEmuState *emu, int hwirq)
 #endif
 }
 
-void GUS_dmarequest (GUSEmuState *der)
+void GUS_dmarequest (GUSEmuState *emu)
 {
-    /* GUSState *s = (GUSState *) der; */
+    GUSState *s = emu->opaque;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
     ldebug ("dma request %d\n", der->gusdma);
-    DMA_hold_DREQ (der->gusdma);
+    k->hold_DREQ(s->isa_dma, s->emu.gusdma);
 }
 
 static int GUS_read_DMA (void *opaque, int nchan, int dma_pos, int dma_len)
 {
     GUSState *s = opaque;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
     char tmpbuf[4096];
     int pos = dma_pos, mode, left = dma_len - dma_pos;
 
     ldebug ("read DMA %#x %d\n", dma_pos, dma_len);
-    mode = DMA_get_channel_mode (s->emu.gusdma);
+    mode = k->has_autoinitialization(s->isa_dma, s->emu.gusdma);
     while (left) {
         int to_copy = audio_MIN ((size_t) left, sizeof (tmpbuf));
         int copied;
 
         ldebug ("left=%d to_copy=%d pos=%d\n", left, to_copy, pos);
-        copied = DMA_read_memory (nchan, tmpbuf, pos, to_copy);
+        copied = k->read_memory(s->isa_dma, nchan, tmpbuf, pos, to_copy);
         gus_dma_transferdata (&s->emu, tmpbuf, copied, left == copied);
         left -= copied;
         pos += copied;
     }
 
     if (((mode >> 4) & 1) == 0) {
-        DMA_release_DREQ (s->emu.gusdma);
+        k->release_DREQ(s->isa_dma, s->emu.gusdma);
     }
     return dma_len;
 }
@@ -236,6 +236,7 @@ static void gus_realizefn (DeviceState *dev, Error **errp)
 {
     ISADevice *d = ISA_DEVICE(dev);
     GUSState *s = GUS (dev);
+    IsaDmaClass *k;
     struct audsettings as;
 
     AUD_register_card ("gus", &s->card);
@@ -268,7 +269,9 @@ static void gus_realizefn (DeviceState *dev, Error **errp)
     isa_register_portio_list (d, (s->port + 0x100) & 0xf00,
                               gus_portio_list2, s, "gus");
 
-    DMA_register_channel (s->emu.gusdma, GUS_read_DMA, s);
+    s->isa_dma = isa_get_dma(isa_bus_from_device(d), s->emu.gusdma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->emu.gusdma, GUS_read_DMA, s);
     s->emu.himemaddr = s->himem;
     s->emu.gusdatapos = s->emu.himemaddr + 1024 * 1024 + 32;
     s->emu.opaque = s;
diff --git a/qemu/hw/audio/gusemu.h b/qemu/hw/audio/gusemu.h
index 331bb6fec..b7f075126 100644
--- a/qemu/hw/audio/gusemu.h
+++ b/qemu/hw/audio/gusemu.h
@@ -34,7 +34,6 @@
  typedef signed char GUSchar;
  typedef signed short GUSsample;
 #else
- #include <stdint.h>
  typedef int8_t GUSchar;
  typedef uint8_t GUSbyte;
  typedef uint16_t GUSword;
diff --git a/qemu/hw/audio/gusemu_hal.c b/qemu/hw/audio/gusemu_hal.c
index 609669073..973d6b9f4 100644
--- a/qemu/hw/audio/gusemu_hal.c
+++ b/qemu/hw/audio/gusemu_hal.c
@@ -26,6 +26,7 @@
  * TODO: check mixer: see 7.20 of sdk for panning pos (applies to all gus models?)?
  */
 
+#include "qemu/osdep.h"
 #include "gustate.h"
 #include "gusemu.h"
 
diff --git a/qemu/hw/audio/gusemu_mixer.c b/qemu/hw/audio/gusemu_mixer.c
index 6d8d9ced1..701e8fb0e 100644
--- a/qemu/hw/audio/gusemu_mixer.c
+++ b/qemu/hw/audio/gusemu_mixer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "gusemu.h"
 #include "gustate.h"
 
diff --git a/qemu/hw/audio/hda-codec.c b/qemu/hw/audio/hda-codec.c
index 3c03ff566..52d4640e6 100644
--- a/qemu/hw/audio/hda-codec.c
+++ b/qemu/hw/audio/hda-codec.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "intel-hda.h"
diff --git a/qemu/hw/audio/intel-hda.c b/qemu/hw/audio/intel-hda.c
index 433463e9c..d372d4ab9 100644
--- a/qemu/hw/audio/intel-hda.c
+++ b/qemu/hw/audio/intel-hda.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
diff --git a/qemu/hw/audio/lm4549.c b/qemu/hw/audio/lm4549.c
index 380ef603b..a46f2301a 100644
--- a/qemu/hw/audio/lm4549.c
+++ b/qemu/hw/audio/lm4549.c
@@ -13,6 +13,7 @@
  * It supports only one playback voice and no record voice.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "lm4549.h"
@@ -33,7 +34,6 @@ do { printf("lm4549: " fmt , ## __VA_ARGS__); } while (0)
 #endif
 
 #if defined(LM4549_DUMP_DAC_INPUT)
-#include <stdio.h>
 static FILE *fp_dac_input;
 #endif
 
diff --git a/qemu/hw/audio/marvell_88w8618.c b/qemu/hw/audio/marvell_88w8618.c
index 86992677e..a6ca1806b 100644
--- a/qemu/hw/audio/marvell_88w8618.c
+++ b/qemu/hw/audio/marvell_88w8618.c
@@ -9,6 +9,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
diff --git a/qemu/hw/audio/milkymist-ac97.c b/qemu/hw/audio/milkymist-ac97.c
index 28f55e853..6a3b53674 100644
--- a/qemu/hw/audio/milkymist-ac97.c
+++ b/qemu/hw/audio/milkymist-ac97.c
@@ -21,6 +21,7 @@
  *   http://www.milkymist.org/socdoc/ac97.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/audio/pcspk.c b/qemu/hw/audio/pcspk.c
index 5266fb545..f9afc8eda 100644
--- a/qemu/hw/audio/pcspk.c
+++ b/qemu/hw/audio/pcspk.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
diff --git a/qemu/hw/audio/pl041.c b/qemu/hw/audio/pl041.c
index 19982f247..4717bc9b9 100644
--- a/qemu/hw/audio/pl041.c
+++ b/qemu/hw/audio/pl041.c
@@ -20,6 +20,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #include "pl041.h"
diff --git a/qemu/hw/audio/sb16.c b/qemu/hw/audio/sb16.c
index b052de5f7..3a4a57ac3 100644
--- a/qemu/hw/audio/sb16.c
+++ b/qemu/hw/audio/sb16.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -40,11 +41,6 @@
 #define ldebug(...)
 #endif
 
-#define IO_READ_PROTO(name)                             \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name)                                    \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 static const char e3[] = "COPYRIGHT (C) CREATIVE TECHNOLOGY LTD, 1992.";
 
 #define TYPE_SB16 "sb16"
@@ -60,6 +56,8 @@ typedef struct SB16State {
     uint32_t hdma;
     uint32_t port;
     uint32_t ver;
+    IsaDma *isa_dma;
+    IsaDma *isa_hdma;
 
     int in_index;
     int out_data_len;
@@ -170,16 +168,18 @@ static void speaker (SB16State *s, int on)
 static void control (SB16State *s, int hold)
 {
     int dma = s->use_hdma ? s->hdma : s->dma;
+    IsaDma *isa_dma = s->use_hdma ? s->isa_hdma : s->isa_dma;
+    IsaDmaClass *k = ISADMA_GET_CLASS(isa_dma);
     s->dma_running = hold;
 
     ldebug ("hold %d high %d dma %d\n", hold, s->use_hdma, dma);
 
     if (hold) {
-        DMA_hold_DREQ (dma);
+        k->hold_DREQ(isa_dma, dma);
         AUD_set_active_out (s->voice, 1);
     }
     else {
-        DMA_release_DREQ (dma);
+        k->release_DREQ(isa_dma, dma);
         AUD_set_active_out (s->voice, 0);
     }
 }
@@ -762,8 +762,8 @@ static void complete (SB16State *s)
                 freq = s->freq > 0 ? s->freq : 11025;
                 samples = dsp_get_lohi (s) + 1;
                 bytes = samples << s->fmt_stereo << (s->fmt_bits == 16);
-                ticks = muldiv64 (bytes, get_ticks_per_sec (), freq);
-                if (ticks < get_ticks_per_sec () / 1024) {
+                ticks = muldiv64(bytes, NANOSECONDS_PER_SECOND, freq);
+                if (ticks < NANOSECONDS_PER_SECOND / 1024) {
                     qemu_irq_raise (s->pic);
                 }
                 else {
@@ -881,7 +881,7 @@ static void reset (SB16State *s)
     legacy_reset (s);
 }
 
-static IO_WRITE_PROTO (dsp_write)
+static void dsp_write(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     int iport;
@@ -959,7 +959,7 @@ static IO_WRITE_PROTO (dsp_write)
     }
 }
 
-static IO_READ_PROTO (dsp_read)
+static uint32_t dsp_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
     int iport, retval, ack = 0;
@@ -1058,14 +1058,14 @@ static void reset_mixer (SB16State *s)
     }
 }
 
-static IO_WRITE_PROTO (mixer_write_indexb)
+static void mixer_write_indexb(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     (void) nport;
     s->mixer_nreg = val;
 }
 
-static IO_WRITE_PROTO (mixer_write_datab)
+static void mixer_write_datab(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
 
@@ -1121,7 +1121,7 @@ static IO_WRITE_PROTO (mixer_write_datab)
     s->mixer_regs[s->mixer_nreg] = val;
 }
 
-static IO_READ_PROTO (mixer_read)
+static uint32_t mixer_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
 
@@ -1141,6 +1141,8 @@ static IO_READ_PROTO (mixer_read)
 static int write_audio (SB16State *s, int nchan, int dma_pos,
                         int dma_len, int len)
 {
+    IsaDma *isa_dma = nchan == s->dma ? s->isa_dma : s->isa_hdma;
+    IsaDmaClass *k = ISADMA_GET_CLASS(isa_dma);
     int temp, net;
     uint8_t tmpbuf[4096];
 
@@ -1157,7 +1159,7 @@ static int write_audio (SB16State *s, int nchan, int dma_pos,
             to_copy = sizeof (tmpbuf);
         }
 
-        copied = DMA_read_memory (nchan, tmpbuf, dma_pos, to_copy);
+        copied = k->read_memory(isa_dma, nchan, tmpbuf, dma_pos, to_copy);
         copied = AUD_write (s->voice, tmpbuf, copied);
 
         temp -= copied;
@@ -1359,6 +1361,7 @@ static void sb16_realizefn (DeviceState *dev, Error **errp)
 {
     ISADevice *isadev = ISA_DEVICE (dev);
     SB16State *s = SB16 (dev);
+    IsaDmaClass *k;
 
     isa_init_irq (isadev, &s->pic, s->irq);
 
@@ -1377,8 +1380,14 @@ static void sb16_realizefn (DeviceState *dev, Error **errp)
 
     isa_register_portio_list (isadev, s->port, sb16_ioport_list, s, "sb16");
 
-    DMA_register_channel (s->hdma, SB_read_DMA, s);
-    DMA_register_channel (s->dma, SB_read_DMA, s);
+    s->isa_hdma = isa_get_dma(isa_bus_from_device(isadev), s->hdma);
+    k = ISADMA_GET_CLASS(s->isa_hdma);
+    k->register_channel(s->isa_hdma, s->hdma, SB_read_DMA, s);
+
+    s->isa_dma = isa_get_dma(isa_bus_from_device(isadev), s->dma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->dma, SB_read_DMA, s);
+
     s->can_write = 1;
 
     AUD_register_card ("sb16", &s->card);
diff --git a/qemu/hw/audio/wm8750.c b/qemu/hw/audio/wm8750.c
index b50b33140..0c6500e96 100644
--- a/qemu/hw/audio/wm8750.c
+++ b/qemu/hw/audio/wm8750.c
@@ -7,6 +7,7 @@
  * This file is licensed under GNU GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "audio/audio.h"
diff --git a/qemu/hw/block/block.c b/qemu/hw/block/block.c
index f7243e5b9..97a59d4fa 100644
--- a/qemu/hw/block/block.c
+++ b/qemu/hw/block/block.c
@@ -7,9 +7,11 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
 #include "hw/block/block.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 
 void blkconf_serial(BlockConf *conf, char **serial)
diff --git a/qemu/hw/block/cdrom.c b/qemu/hw/block/cdrom.c
index 4e1019c89..da937fe33 100644
--- a/qemu/hw/block/cdrom.c
+++ b/qemu/hw/block/cdrom.c
@@ -25,6 +25,7 @@
 /* ??? Most of the ATAPI emulation is still in ide.c.  It should be moved
    here.  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/scsi/scsi.h"
 
diff --git a/qemu/hw/block/dataplane/virtio-blk.c b/qemu/hw/block/dataplane/virtio-blk.c
index 6106e4615..3cb97c9a2 100644
--- a/qemu/hw/block/dataplane/virtio-blk.c
+++ b/qemu/hw/block/dataplane/virtio-blk.c
@@ -12,13 +12,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "trace.h"
 #include "qemu/iov.h"
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
 #include "hw/virtio/virtio-access.h"
-#include "hw/virtio/dataplane/vring.h"
-#include "hw/virtio/dataplane/vring-accessors.h"
 #include "sysemu/block-backend.h"
 #include "hw/virtio/virtio-blk.h"
 #include "virtio-blk.h"
@@ -27,114 +27,93 @@
 #include "qom/object_interfaces.h"
 
 struct VirtIOBlockDataPlane {
-    bool started;
     bool starting;
     bool stopping;
-    bool disabled;
 
     VirtIOBlkConf *conf;
 
     VirtIODevice *vdev;
-    Vring vring;                    /* virtqueue vring */
+    VirtQueue *vq;                  /* virtqueue vring */
     EventNotifier *guest_notifier;  /* irq */
     QEMUBH *bh;                     /* bh for guest notification */
 
+    Notifier insert_notifier, remove_notifier;
+
     /* Note that these EventNotifiers are assigned by value.  This is
      * fine as long as you do not call event_notifier_cleanup on them
      * (because you don't own the file descriptor or handle; you just
      * use it).
      */
     IOThread *iothread;
-    IOThread internal_iothread_obj;
     AioContext *ctx;
-    EventNotifier host_notifier;    /* doorbell */
 
     /* Operation blocker on BDS */
     Error *blocker;
-    void (*saved_complete_request)(struct VirtIOBlockReq *req,
-                                   unsigned char status);
 };
 
 /* Raise an interrupt to signal guest, if necessary */
-static void notify_guest(VirtIOBlockDataPlane *s)
+void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s)
 {
-    if (!vring_should_notify(s->vdev, &s->vring)) {
-        return;
-    }
-
-    event_notifier_set(s->guest_notifier);
+    qemu_bh_schedule(s->bh);
 }
 
 static void notify_guest_bh(void *opaque)
 {
     VirtIOBlockDataPlane *s = opaque;
 
-    notify_guest(s);
+    if (!virtio_should_notify(s->vdev, s->vq)) {
+        return;
+    }
+
+    event_notifier_set(s->guest_notifier);
 }
 
-static void complete_request_vring(VirtIOBlockReq *req, unsigned char status)
+static void data_plane_set_up_op_blockers(VirtIOBlockDataPlane *s)
 {
-    VirtIOBlockDataPlane *s = req->dev->dataplane;
-    stb_p(&req->in->status, status);
-
-    vring_push(s->vdev, &req->dev->dataplane->vring, &req->elem, req->in_len);
+    assert(!s->blocker);
+    error_setg(&s->blocker, "block device is in use by data plane");
+    blk_op_block_all(s->conf->conf.blk, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_CHANGE, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_SOURCE, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_TARGET, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EJECT, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
+                   s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT,
+                   s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
+                   s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_MIRROR_SOURCE, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker);
+    blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker);
+}
 
-    /* Suppress notification to guest by BH and its scheduled
-     * flag because requests are completed as a batch after io
-     * plug & unplug is introduced, and the BH can still be
-     * executed in dataplane aio context even after it is
-     * stopped, so needn't worry about notification loss with BH.
-     */
-    qemu_bh_schedule(s->bh);
+static void data_plane_remove_op_blockers(VirtIOBlockDataPlane *s)
+{
+    if (s->blocker) {
+        blk_op_unblock_all(s->conf->conf.blk, s->blocker);
+        error_free(s->blocker);
+        s->blocker = NULL;
+    }
 }
 
-static void handle_notify(EventNotifier *e)
+static void data_plane_blk_insert_notifier(Notifier *n, void *data)
 {
-    VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
-                                           host_notifier);
-    VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
+    VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane,
+                                           insert_notifier);
+    assert(s->conf->conf.blk == data);
+    data_plane_set_up_op_blockers(s);
+}
 
-    event_notifier_test_and_clear(&s->host_notifier);
-    blk_io_plug(s->conf->conf.blk);
-    for (;;) {
-        MultiReqBuffer mrb = {};
-        int ret;
-
-        /* Disable guest->host notifies to avoid unnecessary vmexits */
-        vring_disable_notification(s->vdev, &s->vring);
-
-        for (;;) {
-            VirtIOBlockReq *req = virtio_blk_alloc_request(vblk);
-
-            ret = vring_pop(s->vdev, &s->vring, &req->elem);
-            if (ret < 0) {
-                virtio_blk_free_request(req);
-                break; /* no more requests */
-            }
-
-            trace_virtio_blk_data_plane_process_request(s, req->elem.out_num,
-                                                        req->elem.in_num,
-                                                        req->elem.index);
-
-            virtio_blk_handle_request(req, &mrb);
-        }
-
-        if (mrb.num_reqs) {
-            virtio_blk_submit_multireq(s->conf->conf.blk, &mrb);
-        }
-
-        if (likely(ret == -EAGAIN)) { /* vring emptied */
-            /* Re-enable guest->host notifies and stop processing the vring.
-             * But if the guest has snuck in more descriptors, keep processing.
-             */
-            if (vring_enable_notification(s->vdev, &s->vring)) {
-                break;
-            }
-        } else { /* fatal error */
-            break;
-        }
-    }
-    blk_io_unplug(s->conf->conf.blk);
+static void data_plane_blk_remove_notifier(Notifier *n, void *data)
+{
+    VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane,
+                                           remove_notifier);
+    assert(s->conf->conf.blk == data);
+    data_plane_remove_op_blockers(s);
 }
 
 /* Context: QEMU global mutex held */
@@ -143,20 +122,19 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
                                   Error **errp)
 {
     VirtIOBlockDataPlane *s;
-    Error *local_err = NULL;
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 
     *dataplane = NULL;
 
-    if (!conf->data_plane && !conf->iothread) {
+    if (!conf->iothread) {
         return;
     }
 
     /* Don't try if transport does not support notifiers. */
     if (!k->set_guest_notifiers || !k->set_host_notifier) {
         error_setg(errp,
-                   "device is incompatible with x-data-plane "
+                   "device is incompatible with dataplane "
                    "(transport does not support notifiers)");
         return;
     }
@@ -164,11 +142,8 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
     /* If dataplane is (re-)enabled while the guest is running there could be
      * block jobs that can conflict.
      */
-    if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE,
-                          &local_err)) {
-        error_setg(errp, "cannot start dataplane thread: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+    if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
+        error_prepend(errp, "cannot start dataplane thread: ");
         return;
     }
 
@@ -179,36 +154,16 @@ void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
     if (conf->iothread) {
         s->iothread = conf->iothread;
         object_ref(OBJECT(s->iothread));
-    } else {
-        /* Create per-device IOThread if none specified.  This is for
-         * x-data-plane option compatibility.  If x-data-plane is removed we
-         * can drop this.
-         */
-        object_initialize(&s->internal_iothread_obj,
-                          sizeof(s->internal_iothread_obj),
-                          TYPE_IOTHREAD);
-        user_creatable_complete(OBJECT(&s->internal_iothread_obj), &error_abort);
-        s->iothread = &s->internal_iothread_obj;
     }
     s->ctx = iothread_get_aio_context(s->iothread);
     s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
 
-    error_setg(&s->blocker, "block device is in use by data plane");
-    blk_op_block_all(conf->conf.blk, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_CHANGE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_COMMIT_SOURCE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_COMMIT_TARGET, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_EJECT, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
-                   s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_MIRROR, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker);
+    s->insert_notifier.notify = data_plane_blk_insert_notifier;
+    s->remove_notifier.notify = data_plane_blk_remove_notifier;
+    blk_add_insert_bs_notifier(conf->conf.blk, &s->insert_notifier);
+    blk_add_remove_bs_notifier(conf->conf.blk, &s->remove_notifier);
+
+    data_plane_set_up_op_blockers(s);
 
     *dataplane = s;
 }
@@ -221,36 +176,39 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
     }
 
     virtio_blk_data_plane_stop(s);
-    blk_op_unblock_all(s->conf->conf.blk, s->blocker);
-    error_free(s->blocker);
+    data_plane_remove_op_blockers(s);
+    notifier_remove(&s->insert_notifier);
+    notifier_remove(&s->remove_notifier);
     qemu_bh_delete(s->bh);
     object_unref(OBJECT(s->iothread));
     g_free(s);
 }
 
+static void virtio_blk_data_plane_handle_output(VirtIODevice *vdev,
+                                                VirtQueue *vq)
+{
+    VirtIOBlock *s = (VirtIOBlock *)vdev;
+
+    assert(s->dataplane);
+    assert(s->dataplane_started);
+
+    virtio_blk_handle_vq(s, vq);
+}
+
 /* Context: QEMU global mutex held */
 void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
 {
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
-    VirtQueue *vq;
     int r;
 
-    if (s->started || s->disabled) {
-        return;
-    }
-
-    if (s->starting) {
+    if (vblk->dataplane_started || s->starting) {
         return;
     }
 
     s->starting = true;
-
-    vq = virtio_get_queue(s->vdev, 0);
-    if (!vring_setup(&s->vring, s->vdev, 0)) {
-        goto fail_vring;
-    }
+    s->vq = virtio_get_queue(s->vdev, 0);
 
     /* Set up guest notifier (irq) */
     r = k->set_guest_notifiers(qbus->parent, 1, true);
@@ -259,7 +217,7 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
                 "ensure -enable-kvm is set\n", r);
         goto fail_guest_notifiers;
     }
-    s->guest_notifier = virtio_queue_get_guest_notifier(vq);
+    s->guest_notifier = virtio_queue_get_guest_notifier(s->vq);
 
     /* Set up virtqueue notify */
     r = k->set_host_notifier(qbus->parent, 0, true);
@@ -267,33 +225,29 @@ void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
         fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
         goto fail_host_notifier;
     }
-    s->host_notifier = *virtio_queue_get_host_notifier(vq);
-
-    s->saved_complete_request = vblk->complete_request;
-    vblk->complete_request = complete_request_vring;
 
     s->starting = false;
-    s->started = true;
+    vblk->dataplane_started = true;
     trace_virtio_blk_data_plane_start(s);
 
     blk_set_aio_context(s->conf->conf.blk, s->ctx);
 
     /* Kick right away to begin processing requests already in vring */
-    event_notifier_set(virtio_queue_get_host_notifier(vq));
+    event_notifier_set(virtio_queue_get_host_notifier(s->vq));
 
     /* Get this show started by hooking up our callbacks */
     aio_context_acquire(s->ctx);
-    aio_set_event_notifier(s->ctx, &s->host_notifier, handle_notify);
+    virtio_queue_aio_set_host_notifier_handler(s->vq, s->ctx,
+                                               virtio_blk_data_plane_handle_output);
     aio_context_release(s->ctx);
     return;
 
   fail_host_notifier:
     k->set_guest_notifiers(qbus->parent, 1, false);
   fail_guest_notifiers:
-    vring_teardown(&s->vring, s->vdev, 0);
-    s->disabled = true;
-  fail_vring:
+    vblk->dataplane_disabled = true;
     s->starting = false;
+    vblk->dataplane_started = true;
 }
 
 /* Context: QEMU global mutex held */
@@ -303,39 +257,34 @@ void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
 
-
-    /* Better luck next time. */
-    if (s->disabled) {
-        s->disabled = false;
+    if (!vblk->dataplane_started || s->stopping) {
         return;
     }
-    if (!s->started || s->stopping) {
+
+    /* Better luck next time. */
+    if (vblk->dataplane_disabled) {
+        vblk->dataplane_disabled = false;
+        vblk->dataplane_started = false;
         return;
     }
     s->stopping = true;
-    vblk->complete_request = s->saved_complete_request;
     trace_virtio_blk_data_plane_stop(s);
 
     aio_context_acquire(s->ctx);
 
     /* Stop notifications for new requests from guest */
-    aio_set_event_notifier(s->ctx, &s->host_notifier, NULL);
+    virtio_queue_aio_set_host_notifier_handler(s->vq, s->ctx, NULL);
 
     /* Drain and switch bs back to the QEMU main loop */
     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
 
     aio_context_release(s->ctx);
 
-    /* Sync vring state back to virtqueue so that non-dataplane request
-     * processing can continue when we disable the host notifier below.
-     */
-    vring_teardown(&s->vring, s->vdev, 0);
-
     k->set_host_notifier(qbus->parent, 0, false);
 
     /* Clean up guest notifier (irq) */
     k->set_guest_notifiers(qbus->parent, 1, false);
 
-    s->started = false;
+    vblk->dataplane_started = false;
     s->stopping = false;
 }
diff --git a/qemu/hw/block/dataplane/virtio-blk.h b/qemu/hw/block/dataplane/virtio-blk.h
index c88d40e72..0714c11a2 100644
--- a/qemu/hw/block/dataplane/virtio-blk.h
+++ b/qemu/hw/block/dataplane/virtio-blk.h
@@ -26,5 +26,6 @@ void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s);
 void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s);
 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s);
 void virtio_blk_data_plane_drain(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s);
 
 #endif /* HW_DATAPLANE_VIRTIO_BLK_H */
diff --git a/qemu/hw/block/ecc.c b/qemu/hw/block/ecc.c
index 10bb23308..48311d260 100644
--- a/qemu/hw/block/ecc.c
+++ b/qemu/hw/block/ecc.c
@@ -11,6 +11,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 
diff --git a/qemu/hw/block/fdc.c b/qemu/hw/block/fdc.c
index 5e1b67ee4..372227569 100644
--- a/qemu/hw/block/fdc.c
+++ b/qemu/hw/block/fdc.c
@@ -27,8 +27,10 @@
  * way. There are changes in DOR register and DMA is not available.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/fdc.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "hw/isa/isa.h"
@@ -40,14 +42,15 @@
 
 /********************************************************/
 /* debug Floppy devices */
-//#define DEBUG_FLOPPY
 
-#ifdef DEBUG_FLOPPY
+#define DEBUG_FLOPPY 0
+
 #define FLOPPY_DPRINTF(fmt, ...)                                \
-    do { printf("FLOPPY: " fmt , ## __VA_ARGS__); } while (0)
-#else
-#define FLOPPY_DPRINTF(fmt, ...)
-#endif
+    do {                                                        \
+        if (DEBUG_FLOPPY) {                                     \
+            fprintf(stderr, "FLOPPY: " fmt , ## __VA_ARGS__);   \
+        }                                                       \
+    } while (0)
 
 /********************************************************/
 /* Floppy drive emulation                               */
@@ -59,104 +62,82 @@ typedef enum FDriveRate {
     FDRIVE_RATE_1M   = 0x03,  /*   1 Mbps */
 } FDriveRate;
 
+typedef enum FDriveSize {
+    FDRIVE_SIZE_UNKNOWN,
+    FDRIVE_SIZE_350,
+    FDRIVE_SIZE_525,
+} FDriveSize;
+
 typedef struct FDFormat {
-    FDriveType drive;
+    FloppyDriveType drive;
     uint8_t last_sect;
     uint8_t max_track;
     uint8_t max_head;
     FDriveRate rate;
 } FDFormat;
 
+/* In many cases, the total sector size of a format is enough to uniquely
+ * identify it. However, there are some total sector collisions between
+ * formats of different physical size, and these are noted below by
+ * highlighting the total sector size for entries with collisions. */
 static const FDFormat fd_formats[] = {
     /* First entry is default format */
     /* 1.44 MB 3"1/2 floppy disks */
-    { FDRIVE_DRV_144, 18, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 20, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 82, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 83, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 22, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 23, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 24, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 18, 80, 1, FDRIVE_RATE_500K, }, /* 3.5" 2880 */
+    { FLOPPY_DRIVE_TYPE_144, 20, 80, 1, FDRIVE_RATE_500K, }, /* 3.5" 3200 */
+    { FLOPPY_DRIVE_TYPE_144, 21, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 21, 82, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 21, 83, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 22, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 23, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 24, 80, 1, FDRIVE_RATE_500K, },
     /* 2.88 MB 3"1/2 floppy disks */
-    { FDRIVE_DRV_288, 36, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 39, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 40, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 44, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 48, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 36, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 39, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 40, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 44, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 48, 80, 1, FDRIVE_RATE_1M, },
     /* 720 kB 3"1/2 floppy disks */
-    { FDRIVE_DRV_144,  9, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 82, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 83, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 13, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 14, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144,  9, 80, 1, FDRIVE_RATE_250K, }, /* 3.5" 1440 */
+    { FLOPPY_DRIVE_TYPE_144, 10, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 10, 82, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 10, 83, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 13, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 14, 80, 1, FDRIVE_RATE_250K, },
     /* 1.2 MB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120, 15, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 82, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 83, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 20, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 15, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 18, 80, 1, FDRIVE_RATE_500K, }, /* 5.25" 2880 */
+    { FLOPPY_DRIVE_TYPE_120, 18, 82, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 18, 83, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 20, 80, 1, FDRIVE_RATE_500K, }, /* 5.25" 3200 */
     /* 720 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  9, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_120, 11, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  9, 80, 1, FDRIVE_RATE_250K, }, /* 5.25" 1440 */
+    { FLOPPY_DRIVE_TYPE_120, 11, 80, 1, FDRIVE_RATE_250K, },
     /* 360 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  9, 40, 1, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120,  9, 40, 0, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120, 10, 41, 1, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120, 10, 42, 1, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120,  9, 40, 1, FDRIVE_RATE_300K, }, /* 5.25" 720 */
+    { FLOPPY_DRIVE_TYPE_120,  9, 40, 0, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120, 10, 41, 1, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120, 10, 42, 1, FDRIVE_RATE_300K, },
     /* 320 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  8, 40, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_120,  8, 40, 0, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  8, 40, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  8, 40, 0, FDRIVE_RATE_250K, },
     /* 360 kB must match 5"1/4 better than 3"1/2... */
-    { FDRIVE_DRV_144,  9, 80, 0, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144,  9, 80, 0, FDRIVE_RATE_250K, }, /* 3.5" 720 */
     /* end */
-    { FDRIVE_DRV_NONE, -1, -1, 0, 0, },
+    { FLOPPY_DRIVE_TYPE_NONE, -1, -1, 0, 0, },
 };
 
-static void pick_geometry(BlockBackend *blk, int *nb_heads,
-                          int *max_track, int *last_sect,
-                          FDriveType drive_in, FDriveType *drive,
-                          FDriveRate *rate)
+static FDriveSize drive_size(FloppyDriveType drive)
 {
-    const FDFormat *parse;
-    uint64_t nb_sectors, size;
-    int i, first_match, match;
-
-    blk_get_geometry(blk, &nb_sectors);
-    match = -1;
-    first_match = -1;
-    for (i = 0; ; i++) {
-        parse = &fd_formats[i];
-        if (parse->drive == FDRIVE_DRV_NONE) {
-            break;
-        }
-        if (drive_in == parse->drive ||
-            drive_in == FDRIVE_DRV_NONE) {
-            size = (parse->max_head + 1) * parse->max_track *
-                parse->last_sect;
-            if (nb_sectors == size) {
-                match = i;
-                break;
-            }
-            if (first_match == -1) {
-                first_match = i;
-            }
-        }
-    }
-    if (match == -1) {
-        if (first_match == -1) {
-            match = 1;
-        } else {
-            match = first_match;
-        }
-        parse = &fd_formats[match];
+    switch (drive) {
+    case FLOPPY_DRIVE_TYPE_120:
+        return FDRIVE_SIZE_525;
+    case FLOPPY_DRIVE_TYPE_144:
+    case FLOPPY_DRIVE_TYPE_288:
+        return FDRIVE_SIZE_350;
+    default:
+        return FDRIVE_SIZE_UNKNOWN;
     }
-    *nb_heads = parse->max_head + 1;
-    *max_track = parse->max_track;
-    *last_sect = parse->last_sect;
-    *drive = parse->drive;
-    *rate = parse->rate;
 }
 
 #define GET_CUR_DRV(fdctrl) ((fdctrl)->cur_drv)
@@ -178,13 +159,14 @@ typedef struct FDrive {
     FDCtrl *fdctrl;
     BlockBackend *blk;
     /* Drive status */
-    FDriveType drive;
+    FloppyDriveType drive;    /* CMOS drive type        */
     uint8_t perpendicular;    /* 2.88 MB access mode    */
     /* Position */
     uint8_t head;
     uint8_t track;
     uint8_t sect;
     /* Media */
+    FloppyDriveType disk;     /* Current disk type      */
     FDiskFlags flags;
     uint8_t last_sect;        /* Nb sector per track    */
     uint8_t max_track;        /* Nb of tracks           */
@@ -192,16 +174,38 @@ typedef struct FDrive {
     uint8_t ro;               /* Is read-only           */
     uint8_t media_changed;    /* Is media changed       */
     uint8_t media_rate;       /* Data rate of medium    */
+
+    bool media_validated;     /* Have we validated the media? */
 } FDrive;
 
+
+static FloppyDriveType get_fallback_drive_type(FDrive *drv);
+
+/* Hack: FD_SEEK is expected to work on empty drives. However, QEMU
+ * currently goes through some pains to keep seeks within the bounds
+ * established by last_sect and max_track. Correcting this is difficult,
+ * as refactoring FDC code tends to expose nasty bugs in the Linux kernel.
+ *
+ * For now: allow empty drives to have large bounds so we can seek around,
+ * with the understanding that when a diskette is inserted, the bounds will
+ * properly tighten to match the geometry of that inserted medium.
+ */
+static void fd_empty_seek_hack(FDrive *drv)
+{
+    drv->last_sect = 0xFF;
+    drv->max_track = 0xFF;
+}
+
 static void fd_init(FDrive *drv)
 {
     /* Drive */
-    drv->drive = FDRIVE_DRV_NONE;
     drv->perpendicular = 0;
     /* Disk */
+    drv->disk = FLOPPY_DRIVE_TYPE_NONE;
     drv->last_sect = 0;
     drv->max_track = 0;
+    drv->ro = true;
+    drv->media_changed = 1;
 }
 
 #define NUM_SIDES(drv) ((drv)->flags & FDISK_DBL_SIDES ? 2 : 1)
@@ -284,39 +288,149 @@ static void fd_recalibrate(FDrive *drv)
     fd_seek(drv, 0, 0, 1, 1);
 }
 
+/**
+ * Determine geometry based on inserted diskette.
+ * Will not operate on an empty drive.
+ *
+ * @return: 0 on success, -1 if the drive is empty.
+ */
+static int pick_geometry(FDrive *drv)
+{
+    BlockBackend *blk = drv->blk;
+    const FDFormat *parse;
+    uint64_t nb_sectors, size;
+    int i;
+    int match, size_match, type_match;
+    bool magic = drv->drive == FLOPPY_DRIVE_TYPE_AUTO;
+
+    /* We can only pick a geometry if we have a diskette. */
+    if (!drv->blk || !blk_is_inserted(drv->blk) ||
+        drv->drive == FLOPPY_DRIVE_TYPE_NONE)
+    {
+        return -1;
+    }
+
+    /* We need to determine the likely geometry of the inserted medium.
+     * In order of preference, we look for:
+     * (1) The same drive type and number of sectors,
+     * (2) The same diskette size and number of sectors,
+     * (3) The same drive type.
+     *
+     * In all cases, matches that occur higher in the drive table will take
+     * precedence over matches that occur later in the table.
+     */
+    blk_get_geometry(blk, &nb_sectors);
+    match = size_match = type_match = -1;
+    for (i = 0; ; i++) {
+        parse = &fd_formats[i];
+        if (parse->drive == FLOPPY_DRIVE_TYPE_NONE) {
+            break;
+        }
+        size = (parse->max_head + 1) * parse->max_track * parse->last_sect;
+        if (nb_sectors == size) {
+            if (magic || parse->drive == drv->drive) {
+                /* (1) perfect match -- nb_sectors and drive type */
+                goto out;
+            } else if (drive_size(parse->drive) == drive_size(drv->drive)) {
+                /* (2) size match -- nb_sectors and physical medium size */
+                match = (match == -1) ? i : match;
+            } else {
+                /* This is suspicious -- Did the user misconfigure? */
+                size_match = (size_match == -1) ? i : size_match;
+            }
+        } else if (type_match == -1) {
+            if ((parse->drive == drv->drive) ||
+                (magic && (parse->drive == get_fallback_drive_type(drv)))) {
+                /* (3) type match -- nb_sectors mismatch, but matches the type
+                 *     specified explicitly by the user, or matches the fallback
+                 *     default type when using the drive autodetect mechanism */
+                type_match = i;
+            }
+        }
+    }
+
+    /* No exact match found */
+    if (match == -1) {
+        if (size_match != -1) {
+            parse = &fd_formats[size_match];
+            FLOPPY_DPRINTF("User requested floppy drive type '%s', "
+                           "but inserted medium appears to be a "
+                           "%"PRId64" sector '%s' type\n",
+                           FloppyDriveType_lookup[drv->drive],
+                           nb_sectors,
+                           FloppyDriveType_lookup[parse->drive]);
+        }
+        match = type_match;
+    }
+
+    /* No match of any kind found -- fd_format is misconfigured, abort. */
+    if (match == -1) {
+        error_setg(&error_abort, "No candidate geometries present in table "
+                   " for floppy drive type '%s'",
+                   FloppyDriveType_lookup[drv->drive]);
+    }
+
+    parse = &(fd_formats[match]);
+
+ out:
+    if (parse->max_head == 0) {
+        drv->flags &= ~FDISK_DBL_SIDES;
+    } else {
+        drv->flags |= FDISK_DBL_SIDES;
+    }
+    drv->max_track = parse->max_track;
+    drv->last_sect = parse->last_sect;
+    drv->disk = parse->drive;
+    drv->media_rate = parse->rate;
+    return 0;
+}
+
+static void pick_drive_type(FDrive *drv)
+{
+    if (drv->drive != FLOPPY_DRIVE_TYPE_AUTO) {
+        return;
+    }
+
+    if (pick_geometry(drv) == 0) {
+        drv->drive = drv->disk;
+    } else {
+        drv->drive = get_fallback_drive_type(drv);
+    }
+
+    g_assert(drv->drive != FLOPPY_DRIVE_TYPE_AUTO);
+}
+
 /* Revalidate a disk drive after a disk change */
 static void fd_revalidate(FDrive *drv)
 {
-    int nb_heads, max_track, last_sect, ro;
-    FDriveType drive;
-    FDriveRate rate;
+    int rc;
 
     FLOPPY_DPRINTF("revalidate\n");
     if (drv->blk != NULL) {
-        ro = blk_is_read_only(drv->blk);
-        pick_geometry(drv->blk, &nb_heads, &max_track,
-                      &last_sect, drv->drive, &drive, &rate);
+        drv->ro = blk_is_read_only(drv->blk);
         if (!blk_is_inserted(drv->blk)) {
             FLOPPY_DPRINTF("No disk in drive\n");
-        } else {
-            FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n", nb_heads,
-                           max_track, last_sect, ro ? "ro" : "rw");
-        }
-        if (nb_heads == 1) {
-            drv->flags &= ~FDISK_DBL_SIDES;
-        } else {
-            drv->flags |= FDISK_DBL_SIDES;
+            drv->disk = FLOPPY_DRIVE_TYPE_NONE;
+            fd_empty_seek_hack(drv);
+        } else if (!drv->media_validated) {
+            rc = pick_geometry(drv);
+            if (rc) {
+                FLOPPY_DPRINTF("Could not validate floppy drive media");
+            } else {
+                drv->media_validated = true;
+                FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n",
+                               (drv->flags & FDISK_DBL_SIDES) ? 2 : 1,
+                               drv->max_track, drv->last_sect,
+                               drv->ro ? "ro" : "rw");
+            }
         }
-        drv->max_track = max_track;
-        drv->last_sect = last_sect;
-        drv->ro = ro;
-        drv->drive = drive;
-        drv->media_rate = rate;
     } else {
         FLOPPY_DPRINTF("No drive connected\n");
         drv->last_sect = 0;
         drv->max_track = 0;
         drv->flags &= ~FDISK_DBL_SIDES;
+        drv->drive = FLOPPY_DRIVE_TYPE_NONE;
+        drv->disk = FLOPPY_DRIVE_TYPE_NONE;
     }
 }
 
@@ -532,6 +646,7 @@ struct FDCtrl {
     QEMUTimer *result_timer;
     int dma_chann;
     uint8_t phase;
+    IsaDma *dma;
     /* Controller's identification */
     uint8_t version;
     /* HW */
@@ -566,11 +681,17 @@ struct FDCtrl {
     FDrive drives[MAX_FD];
     int reset_sensei;
     uint32_t check_media_rate;
+    FloppyDriveType fallback; /* type=auto failure fallback */
     /* Timers state */
     uint8_t timer0;
     uint8_t timer1;
 };
 
+static FloppyDriveType get_fallback_drive_type(FDrive *drv)
+{
+    return drv->fdctrl->fallback;
+}
+
 #define TYPE_SYSBUS_FDC "base-sysbus-fdc"
 #define SYSBUS_FDC(obj) OBJECT_CHECK(FDCtrlSysBus, (obj), TYPE_SYSBUS_FDC)
 
@@ -1311,7 +1432,8 @@ static void fdctrl_stop_transfer(FDCtrl *fdctrl, uint8_t status0,
     fdctrl->fifo[6] = FD_SECTOR_SC;
     fdctrl->data_dir = FD_DIR_READ;
     if (!(fdctrl->msr & FD_MSR_NONDMA)) {
-        DMA_release_DREQ(fdctrl->dma_chann);
+        IsaDmaClass *k = ISADMA_GET_CLASS(fdctrl->dma);
+        k->release_DREQ(fdctrl->dma, fdctrl->dma_chann);
     }
     fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
     fdctrl->msr &= ~FD_MSR_NONDMA;
@@ -1397,27 +1519,43 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
     }
     fdctrl->eot = fdctrl->fifo[6];
     if (fdctrl->dor & FD_DOR_DMAEN) {
-        int dma_mode;
+        IsaDmaTransferMode dma_mode;
+        IsaDmaClass *k = ISADMA_GET_CLASS(fdctrl->dma);
+        bool dma_mode_ok;
         /* DMA transfer are enabled. Check if DMA channel is well programmed */
-        dma_mode = DMA_get_channel_mode(fdctrl->dma_chann);
-        dma_mode = (dma_mode >> 2) & 3;
+        dma_mode = k->get_transfer_mode(fdctrl->dma, fdctrl->dma_chann);
         FLOPPY_DPRINTF("dma_mode=%d direction=%d (%d - %d)\n",
                        dma_mode, direction,
                        (128 << fdctrl->fifo[5]) *
                        (cur_drv->last_sect - ks + 1), fdctrl->data_len);
-        if (((direction == FD_DIR_SCANE || direction == FD_DIR_SCANL ||
-              direction == FD_DIR_SCANH) && dma_mode == 0) ||
-            (direction == FD_DIR_WRITE && dma_mode == 2) ||
-            (direction == FD_DIR_READ && dma_mode == 1) ||
-            (direction == FD_DIR_VERIFY)) {
+        switch (direction) {
+        case FD_DIR_SCANE:
+        case FD_DIR_SCANL:
+        case FD_DIR_SCANH:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_VERIFY);
+            break;
+        case FD_DIR_WRITE:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_WRITE);
+            break;
+        case FD_DIR_READ:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_READ);
+            break;
+        case FD_DIR_VERIFY:
+            dma_mode_ok = true;
+            break;
+        default:
+            dma_mode_ok = false;
+            break;
+        }
+        if (dma_mode_ok) {
             /* No access is allowed until DMA transfer has completed */
             fdctrl->msr &= ~FD_MSR_RQM;
             if (direction != FD_DIR_VERIFY) {
                 /* Now, we just have to wait for the DMA controller to
                  * recall us...
                  */
-                DMA_hold_DREQ(fdctrl->dma_chann);
-                DMA_schedule(fdctrl->dma_chann);
+                k->hold_DREQ(fdctrl->dma, fdctrl->dma_chann);
+                k->schedule(fdctrl->dma);
             } else {
                 /* Start transfer */
                 fdctrl_transfer_handler(fdctrl, fdctrl->dma_chann, 0,
@@ -1456,12 +1594,14 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
     FDrive *cur_drv;
     int len, start_pos, rel_pos;
     uint8_t status0 = 0x00, status1 = 0x00, status2 = 0x00;
+    IsaDmaClass *k;
 
     fdctrl = opaque;
     if (fdctrl->msr & FD_MSR_RQM) {
         FLOPPY_DPRINTF("Not in DMA transfer mode !\n");
         return 0;
     }
+    k = ISADMA_GET_CLASS(fdctrl->dma);
     cur_drv = get_cur_drv(fdctrl);
     if (fdctrl->data_dir == FD_DIR_SCANE || fdctrl->data_dir == FD_DIR_SCANL ||
         fdctrl->data_dir == FD_DIR_SCANH)
@@ -1500,8 +1640,8 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
         switch (fdctrl->data_dir) {
         case FD_DIR_READ:
             /* READ commands */
-            DMA_write_memory (nchan, fdctrl->fifo + rel_pos,
-                              fdctrl->data_pos, len);
+            k->write_memory(fdctrl->dma, nchan, fdctrl->fifo + rel_pos,
+                            fdctrl->data_pos, len);
             break;
         case FD_DIR_WRITE:
             /* WRITE commands */
@@ -1515,8 +1655,8 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
                 goto transfer_error;
             }
 
-            DMA_read_memory (nchan, fdctrl->fifo + rel_pos,
-                             fdctrl->data_pos, len);
+            k->read_memory(fdctrl->dma, nchan, fdctrl->fifo + rel_pos,
+                           fdctrl->data_pos, len);
             if (blk_write(cur_drv->blk, fd_sector(cur_drv),
                           fdctrl->fifo, 1) < 0) {
                 FLOPPY_DPRINTF("error writing sector %d\n",
@@ -1533,7 +1673,8 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
             {
                 uint8_t tmpbuf[FD_SECTOR_LEN];
                 int ret;
-                DMA_read_memory (nchan, tmpbuf, fdctrl->data_pos, len);
+                k->read_memory(fdctrl->dma, nchan, tmpbuf, fdctrl->data_pos,
+                               len);
                 ret = memcmp(tmpbuf, fdctrl->fifo + rel_pos, len);
                 if (ret == 0) {
                     status2 = FD_SR2_SEH;
@@ -1798,8 +1939,8 @@ static void fdctrl_handle_readid(FDCtrl *fdctrl, int direction)
     FDrive *cur_drv = get_cur_drv(fdctrl);
 
     cur_drv->head = (fdctrl->fifo[1] >> 2) & 1;
-    timer_mod(fdctrl->result_timer,
-                   qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 50));
+    timer_mod(fdctrl->result_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+             (NANOSECONDS_PER_SECOND / 50));
 }
 
 static void fdctrl_handle_format_track(FDCtrl *fdctrl, int direction)
@@ -2185,6 +2326,7 @@ static void fdctrl_change_cb(void *opaque, bool load)
     FDrive *drive = opaque;
 
     drive->media_changed = 1;
+    drive->media_validated = false;
     fd_revalidate(drive);
 }
 
@@ -2214,10 +2356,11 @@ static void fdctrl_connect_drives(FDCtrl *fdctrl, Error **errp)
         }
 
         fd_init(drive);
-        fdctrl_change_cb(drive, 0);
         if (drive->blk) {
             blk_set_dev_ops(drive->blk, &fdctrl_block_ops, drive);
+            pick_drive_type(drive);
         }
+        fd_revalidate(drive);
     }
 }
 
@@ -2233,10 +2376,12 @@ ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds)
     dev = DEVICE(isadev);
 
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fds[1]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
 
@@ -2256,10 +2401,12 @@ void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
     fdctrl = &sys->state;
     fdctrl->dma_chann = dma_chann; /* FIXME */
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fds[1]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
     sbd = SYS_BUS_DEVICE(dev);
@@ -2275,7 +2422,8 @@ void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
 
     dev = qdev_create(NULL, "SUNW,fdtwo");
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
     sys = SYSBUS_FDC(dev);
@@ -2289,6 +2437,10 @@ static void fdctrl_realize_common(FDCtrl *fdctrl, Error **errp)
     int i, j;
     static int command_tables_inited = 0;
 
+    if (fdctrl->fallback == FLOPPY_DRIVE_TYPE_AUTO) {
+        error_setg(errp, "Cannot choose a fallback FDrive type of 'auto'");
+    }
+
     /* Fill 'command_to_handler' lookup table */
     if (!command_tables_inited) {
         command_tables_inited = 1;
@@ -2312,7 +2464,11 @@ static void fdctrl_realize_common(FDCtrl *fdctrl, Error **errp)
     fdctrl->num_floppies = MAX_FD;
 
     if (fdctrl->dma_chann != -1) {
-        DMA_register_channel(fdctrl->dma_chann, &fdctrl_transfer_handler, fdctrl);
+        IsaDmaClass *k;
+        assert(fdctrl->dma);
+        k = ISADMA_GET_CLASS(fdctrl->dma);
+        k->register_channel(fdctrl->dma, fdctrl->dma_chann,
+                            &fdctrl_transfer_handler, fdctrl);
     }
     fdctrl_connect_drives(fdctrl, errp);
 }
@@ -2335,6 +2491,10 @@ static void isabus_fdc_realize(DeviceState *dev, Error **errp)
 
     isa_init_irq(isadev, &fdctrl->irq, isa->irq);
     fdctrl->dma_chann = isa->dma;
+    if (fdctrl->dma_chann != -1) {
+        fdctrl->dma = isa_get_dma(isa_bus_from_device(isadev), isa->dma);
+        assert(fdctrl->dma);
+    }
 
     qdev_set_legacy_instance_id(dev, isa->iobase, 2);
     fdctrl_realize_common(fdctrl, &err);
@@ -2363,6 +2523,8 @@ static void sun4m_fdc_initfn(Object *obj)
     FDCtrlSysBus *sys = SYSBUS_FDC(obj);
     FDCtrl *fdctrl = &sys->state;
 
+    fdctrl->dma_chann = -1;
+
     memory_region_init_io(&fdctrl->iomem, obj, &fdctrl_mem_strict_ops,
                           fdctrl, "fdctrl", 0x08);
     sysbus_init_mmio(sbd, &fdctrl->iomem);
@@ -2389,13 +2551,36 @@ static void sysbus_fdc_common_realize(DeviceState *dev, Error **errp)
     fdctrl_realize_common(fdctrl, errp);
 }
 
-FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i)
+FloppyDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i)
 {
     FDCtrlISABus *isa = ISA_FDC(fdc);
 
     return isa->state.drives[i].drive;
 }
 
+void isa_fdc_get_drive_max_chs(FloppyDriveType type,
+                               uint8_t *maxc, uint8_t *maxh, uint8_t *maxs)
+{
+    const FDFormat *fdf;
+
+    *maxc = *maxh = *maxs = 0;
+    for (fdf = fd_formats; fdf->drive != FLOPPY_DRIVE_TYPE_NONE; fdf++) {
+        if (fdf->drive != type) {
+            continue;
+        }
+        if (*maxc < fdf->max_track) {
+            *maxc = fdf->max_track;
+        }
+        if (*maxh < fdf->max_head) {
+            *maxh = fdf->max_head;
+        }
+        if (*maxs < fdf->last_sect) {
+            *maxs = fdf->last_sect;
+        }
+    }
+    (*maxc)--;
+}
+
 static const VMStateDescription vmstate_isa_fdc ={
     .name = "fdc",
     .version_id = 2,
@@ -2414,6 +2599,15 @@ static Property isa_fdc_properties[] = {
     DEFINE_PROP_DRIVE("driveB", FDCtrlISABus, state.drives[1].blk),
     DEFINE_PROP_BIT("check_media_rate", FDCtrlISABus, state.check_media_rate,
                     0, true),
+    DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlISABus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fdtypeB", FDCtrlISABus, state.drives[1].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_288, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2462,6 +2656,15 @@ static const VMStateDescription vmstate_sysbus_fdc ={
 static Property sysbus_fdc_properties[] = {
     DEFINE_PROP_DRIVE("driveA", FDCtrlSysBus, state.drives[0].blk),
     DEFINE_PROP_DRIVE("driveB", FDCtrlSysBus, state.drives[1].blk),
+    DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlSysBus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fdtypeB", FDCtrlSysBus, state.drives[1].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_144, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2482,6 +2685,12 @@ static const TypeInfo sysbus_fdc_info = {
 
 static Property sun4m_fdc_properties[] = {
     DEFINE_PROP_DRIVE("drive", FDCtrlSysBus, state.drives[0].blk),
+    DEFINE_PROP_DEFAULT("fdtype", FDCtrlSysBus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_144, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/qemu/hw/block/hd-geometry.c b/qemu/hw/block/hd-geometry.c
index b187878fa..6d02192db 100644
--- a/qemu/hw/block/hd-geometry.c
+++ b/qemu/hw/block/hd-geometry.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "trace.h"
diff --git a/qemu/hw/block/m25p80.c b/qemu/hw/block/m25p80.c
index efc43dde6..906b71257 100644
--- a/qemu/hw/block/m25p80.c
+++ b/qemu/hw/block/m25p80.c
@@ -21,10 +21,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/bitops.h"
 
 #ifndef M25P80_ERR_DEBUG
 #define M25P80_ERR_DEBUG 0
@@ -45,7 +47,10 @@
 /* set to allow the page program command to write 0s back to 1. Useful for
  * modelling EEPROM with SPI flash command set
  */
-#define WR_1 0x100
+#define EEPROM 0x100
+
+/* 16 MiB max in 3 byte address mode */
+#define MAX_3BYTES_SIZE 0x1000000
 
 typedef struct FlashPartInfo {
     const char *part_name;
@@ -60,7 +65,7 @@ typedef struct FlashPartInfo {
     uint32_t sector_size;
     uint32_t n_sectors;
     uint32_t page_size;
-    uint8_t flags;
+    uint16_t flags;
 } FlashPartInfo;
 
 /* adapted from linux */
@@ -78,6 +83,30 @@ typedef struct FlashPartInfo {
 #define JEDEC_WINBOND 0xEF
 #define JEDEC_SPANSION 0x01
 
+/* Numonyx (Micron) Configuration register macros */
+#define VCFG_DUMMY 0x1
+#define VCFG_WRAP_SEQUENTIAL 0x2
+#define NVCFG_XIP_MODE_DISABLED (7 << 9)
+#define NVCFG_XIP_MODE_MASK (7 << 9)
+#define VCFG_XIP_MODE_ENABLED (1 << 3)
+#define CFG_DUMMY_CLK_LEN 4
+#define NVCFG_DUMMY_CLK_POS 12
+#define VCFG_DUMMY_CLK_POS 4
+#define EVCFG_OUT_DRIVER_STRENGHT_DEF 7
+#define EVCFG_VPP_ACCELERATOR (1 << 3)
+#define EVCFG_RESET_HOLD_ENABLED (1 << 4)
+#define NVCFG_DUAL_IO_MASK (1 << 2)
+#define EVCFG_DUAL_IO_ENABLED (1 << 6)
+#define NVCFG_QUAD_IO_MASK (1 << 3)
+#define EVCFG_QUAD_IO_ENABLED (1 << 7)
+#define NVCFG_4BYTE_ADDR_MASK (1 << 0)
+#define NVCFG_LOWER_SEGMENT_MASK (1 << 1)
+#define CFG_UPPER_128MB_SEG_ENABLED 0x3
+
+/* Numonyx (Micron) Flag Status Register macros */
+#define FSR_4BYTE_ADDR_MODE_ENABLED 0x1
+#define FSR_FLASH_READY (1 << 7)
+
 static const FlashPartInfo known_devices[] = {
     /* Atmel -- some are (confusingly) marketed as "DataFlash" */
     { INFO("at25fs010",   0x1f6601,      0,  32 << 10,   4, ER_4K) },
@@ -94,6 +123,12 @@ static const FlashPartInfo known_devices[] = {
 
     { INFO("at45db081d",  0x1f2500,      0,  64 << 10,  16, ER_4K) },
 
+    /* Atmel EEPROMS - it is assumed, that don't care bit in command
+     * is set to 0. Block protection is not supported.
+     */
+    { INFO("at25128a-nonjedec", 0x0,     0,         1, 131072, EEPROM) },
+    { INFO("at25256a-nonjedec", 0x0,     0,         1, 262144, EEPROM) },
+
     /* EON -- en25xxx */
     { INFO("en25f32",     0x1c3116,      0,  64 << 10,  64, ER_4K) },
     { INFO("en25p32",     0x1c2016,      0,  64 << 10,  64, 0) },
@@ -163,6 +198,7 @@ static const FlashPartInfo known_devices[] = {
     { INFO("sst25wf010",  0xbf2502,      0,  64 << 10,   2, ER_4K) },
     { INFO("sst25wf020",  0xbf2503,      0,  64 << 10,   4, ER_4K) },
     { INFO("sst25wf040",  0xbf2504,      0,  64 << 10,   8, ER_4K) },
+    { INFO("sst25wf080",  0xbf2505,      0,  64 << 10,  16, ER_4K) },
 
     /* ST Microelectronics -- newer production may have feature updates */
     { INFO("m25p05",      0x202010,      0,  32 << 10,   2, 0) },
@@ -204,8 +240,9 @@ static const FlashPartInfo known_devices[] = {
     { INFO("w25q80bl",    0xef4014,      0,  64 << 10,  16, ER_4K) },
     { INFO("w25q256",     0xef4019,      0,  64 << 10, 512, ER_4K) },
 
-    /* Numonyx -- n25q128 */
     { INFO("n25q128",      0x20ba18,      0,  64 << 10, 256, 0) },
+    { INFO("n25q256a",     0x20ba19,      0,  64 << 10, 512, ER_4K) },
+    { INFO("n25q512a",     0x20ba20,      0,  64 << 10, 1024, ER_4K) },
 };
 
 typedef enum {
@@ -216,21 +253,49 @@ typedef enum {
     WREN = 0x6,
     JEDEC_READ = 0x9f,
     BULK_ERASE = 0xc7,
+    READ_FSR = 0x70,
 
-    READ = 0x3,
-    FAST_READ = 0xb,
+    READ = 0x03,
+    READ4 = 0x13,
+    FAST_READ = 0x0b,
+    FAST_READ4 = 0x0c,
     DOR = 0x3b,
+    DOR4 = 0x3c,
     QOR = 0x6b,
+    QOR4 = 0x6c,
     DIOR = 0xbb,
+    DIOR4 = 0xbc,
     QIOR = 0xeb,
+    QIOR4 = 0xec,
 
-    PP = 0x2,
+    PP = 0x02,
+    PP4 = 0x12,
     DPP = 0xa2,
     QPP = 0x32,
 
     ERASE_4K = 0x20,
+    ERASE4_4K = 0x21,
     ERASE_32K = 0x52,
     ERASE_SECTOR = 0xd8,
+    ERASE4_SECTOR = 0xdc,
+
+    EN_4BYTE_ADDR = 0xB7,
+    EX_4BYTE_ADDR = 0xE9,
+
+    EXTEND_ADDR_READ = 0xC8,
+    EXTEND_ADDR_WRITE = 0xC5,
+
+    RESET_ENABLE = 0x66,
+    RESET_MEMORY = 0x99,
+
+    RNVCR = 0xB5,
+    WNVCR = 0xB1,
+
+    RVCR = 0x85,
+    WVCR = 0x81,
+
+    REVCR = 0x65,
+    WEVCR = 0x61,
 } FlashCMD;
 
 typedef enum {
@@ -244,8 +309,6 @@ typedef enum {
 typedef struct Flash {
     SSISlave parent_obj;
 
-    uint32_t r;
-
     BlockBackend *blk;
 
     uint8_t *storage;
@@ -259,7 +322,13 @@ typedef struct Flash {
     uint8_t needed_bytes;
     uint8_t cmd_in_progress;
     uint64_t cur_addr;
+    uint32_t nonvolatile_cfg;
+    uint32_t volatile_cfg;
+    uint32_t enh_volatile_cfg;
     bool write_enable;
+    bool four_bytes_address_mode;
+    bool reset_enable;
+    uint8_t ear;
 
     int64_t dirty_page;
 
@@ -331,6 +400,7 @@ static void flash_erase(Flash *s, int offset, FlashCMD cmd)
 
     switch (cmd) {
     case ERASE_4K:
+    case ERASE4_4K:
         len = 4 << 10;
         capa_to_assert = ER_4K;
         break;
@@ -339,6 +409,7 @@ static void flash_erase(Flash *s, int offset, FlashCMD cmd)
         capa_to_assert = ER_32K;
         break;
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
         len = s->pi->sector_size;
         break;
     case BULK_ERASE:
@@ -385,7 +456,7 @@ void flash_write8(Flash *s, uint64_t addr, uint8_t data)
                    " -> %" PRIx8 "\n", addr, prev, data);
     }
 
-    if (s->pi->flags & WR_1) {
+    if (s->pi->flags & EEPROM) {
         s->storage[s->cur_addr] = data;
     } else {
         s->storage[s->cur_addr] &= data;
@@ -395,11 +466,43 @@ void flash_write8(Flash *s, uint64_t addr, uint8_t data)
     s->dirty_page = page;
 }
 
+static inline int get_addr_length(Flash *s)
+{
+   /* check if eeprom is in use */
+    if (s->pi->flags == EEPROM) {
+        return 2;
+    }
+
+   switch (s->cmd_in_progress) {
+   case PP4:
+   case READ4:
+   case QIOR4:
+   case ERASE4_4K:
+   case ERASE4_SECTOR:
+   case FAST_READ4:
+   case DOR4:
+   case QOR4:
+   case DIOR4:
+       return 4;
+   default:
+       return s->four_bytes_address_mode ? 4 : 3;
+   }
+}
+
 static void complete_collecting_data(Flash *s)
 {
-    s->cur_addr = s->data[0] << 16;
-    s->cur_addr |= s->data[1] << 8;
-    s->cur_addr |= s->data[2];
+    int i;
+
+    s->cur_addr = 0;
+
+    for (i = 0; i < get_addr_length(s); ++i) {
+        s->cur_addr <<= 8;
+        s->cur_addr |= s->data[i];
+    }
+
+    if (get_addr_length(s) == 3) {
+        s->cur_addr += (s->ear & 0x3) * MAX_3BYTES_SIZE;
+    }
 
     s->state = STATE_IDLE;
 
@@ -407,19 +510,28 @@ static void complete_collecting_data(Flash *s)
     case DPP:
     case QPP:
     case PP:
+    case PP4:
         s->state = STATE_PAGE_PROGRAM;
         break;
     case READ:
+    case READ4:
     case FAST_READ:
+    case FAST_READ4:
     case DOR:
+    case DOR4:
     case QOR:
+    case QOR4:
     case DIOR:
+    case DIOR4:
     case QIOR:
+    case QIOR4:
         s->state = STATE_READ;
         break;
     case ERASE_4K:
+    case ERASE4_4K:
     case ERASE_32K:
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
         flash_erase(s, s->cur_addr, s->cmd_in_progress);
         break;
     case WRSR:
@@ -427,49 +539,128 @@ static void complete_collecting_data(Flash *s)
             s->write_enable = false;
         }
         break;
+    case EXTEND_ADDR_WRITE:
+        s->ear = s->data[0];
+        break;
+    case WNVCR:
+        s->nonvolatile_cfg = s->data[0] | (s->data[1] << 8);
+        break;
+    case WVCR:
+        s->volatile_cfg = s->data[0];
+        break;
+    case WEVCR:
+        s->enh_volatile_cfg = s->data[0];
+        break;
     default:
         break;
     }
 }
 
+static void reset_memory(Flash *s)
+{
+    s->cmd_in_progress = NOP;
+    s->cur_addr = 0;
+    s->ear = 0;
+    s->four_bytes_address_mode = false;
+    s->len = 0;
+    s->needed_bytes = 0;
+    s->pos = 0;
+    s->state = STATE_IDLE;
+    s->write_enable = false;
+    s->reset_enable = false;
+
+    if (((s->pi->jedec >> 16) & 0xFF) == JEDEC_NUMONYX) {
+        s->volatile_cfg = 0;
+        s->volatile_cfg |= VCFG_DUMMY;
+        s->volatile_cfg |= VCFG_WRAP_SEQUENTIAL;
+        if ((s->nonvolatile_cfg & NVCFG_XIP_MODE_MASK)
+                                != NVCFG_XIP_MODE_DISABLED) {
+            s->volatile_cfg |= VCFG_XIP_MODE_ENABLED;
+        }
+        s->volatile_cfg |= deposit32(s->volatile_cfg,
+                            VCFG_DUMMY_CLK_POS,
+                            CFG_DUMMY_CLK_LEN,
+                            extract32(s->nonvolatile_cfg,
+                                        NVCFG_DUMMY_CLK_POS,
+                                        CFG_DUMMY_CLK_LEN)
+                            );
+
+        s->enh_volatile_cfg = 0;
+        s->enh_volatile_cfg |= EVCFG_OUT_DRIVER_STRENGHT_DEF;
+        s->enh_volatile_cfg |= EVCFG_VPP_ACCELERATOR;
+        s->enh_volatile_cfg |= EVCFG_RESET_HOLD_ENABLED;
+        if (s->nonvolatile_cfg & NVCFG_DUAL_IO_MASK) {
+            s->enh_volatile_cfg |= EVCFG_DUAL_IO_ENABLED;
+        }
+        if (s->nonvolatile_cfg & NVCFG_QUAD_IO_MASK) {
+            s->enh_volatile_cfg |= EVCFG_QUAD_IO_ENABLED;
+        }
+        if (!(s->nonvolatile_cfg & NVCFG_4BYTE_ADDR_MASK)) {
+            s->four_bytes_address_mode = true;
+        }
+        if (!(s->nonvolatile_cfg & NVCFG_LOWER_SEGMENT_MASK)) {
+            s->ear = CFG_UPPER_128MB_SEG_ENABLED;
+        }
+    }
+
+    DB_PRINT_L(0, "Reset done.\n");
+}
+
 static void decode_new_cmd(Flash *s, uint32_t value)
 {
     s->cmd_in_progress = value;
     DB_PRINT_L(0, "decoded new command:%x\n", value);
 
+    if (value != RESET_MEMORY) {
+        s->reset_enable = false;
+    }
+
     switch (value) {
 
     case ERASE_4K:
+    case ERASE4_4K:
     case ERASE_32K:
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
     case READ:
+    case READ4:
     case DPP:
     case QPP:
     case PP:
-        s->needed_bytes = 3;
+    case PP4:
+        s->needed_bytes = get_addr_length(s);
         s->pos = 0;
         s->len = 0;
         s->state = STATE_COLLECTING_DATA;
         break;
 
     case FAST_READ:
+    case FAST_READ4:
     case DOR:
+    case DOR4:
     case QOR:
-        s->needed_bytes = 4;
+    case QOR4:
+        s->needed_bytes = get_addr_length(s);
+        if (((s->pi->jedec >> 16) & 0xFF) == JEDEC_NUMONYX) {
+            /* Dummy cycles modeled with bytes writes instead of bits */
+            s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
+        }
         s->pos = 0;
         s->len = 0;
         s->state = STATE_COLLECTING_DATA;
         break;
 
     case DIOR:
+    case DIOR4:
         switch ((s->pi->jedec >> 16) & 0xFF) {
         case JEDEC_WINBOND:
         case JEDEC_SPANSION:
             s->needed_bytes = 4;
             break;
-        case JEDEC_NUMONYX:
         default:
-            s->needed_bytes = 5;
+            s->needed_bytes = get_addr_length(s);
+            /* Dummy cycles modeled with bytes writes instead of bits */
+            s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
         }
         s->pos = 0;
         s->len = 0;
@@ -477,14 +668,16 @@ static void decode_new_cmd(Flash *s, uint32_t value)
         break;
 
     case QIOR:
+    case QIOR4:
         switch ((s->pi->jedec >> 16) & 0xFF) {
         case JEDEC_WINBOND:
         case JEDEC_SPANSION:
             s->needed_bytes = 6;
             break;
-        case JEDEC_NUMONYX:
         default:
-            s->needed_bytes = 8;
+            s->needed_bytes = get_addr_length(s);
+            /* Dummy cycles modeled with bytes writes instead of bits */
+            s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
         }
         s->pos = 0;
         s->len = 0;
@@ -514,6 +707,16 @@ static void decode_new_cmd(Flash *s, uint32_t value)
         s->state = STATE_READING_DATA;
         break;
 
+    case READ_FSR:
+        s->data[0] = FSR_FLASH_READY;
+        if (s->four_bytes_address_mode) {
+            s->data[0] |= FSR_4BYTE_ADDR_MODE_ENABLED;
+        }
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+
     case JEDEC_READ:
         DB_PRINT_L(0, "populated jedec code\n");
         s->data[0] = (s->pi->jedec >> 16) & 0xff;
@@ -541,6 +744,77 @@ static void decode_new_cmd(Flash *s, uint32_t value)
         break;
     case NOP:
         break;
+    case EN_4BYTE_ADDR:
+        s->four_bytes_address_mode = true;
+        break;
+    case EX_4BYTE_ADDR:
+        s->four_bytes_address_mode = false;
+        break;
+    case EXTEND_ADDR_READ:
+        s->data[0] = s->ear;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case EXTEND_ADDR_WRITE:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RNVCR:
+        s->data[0] = s->nonvolatile_cfg & 0xFF;
+        s->data[1] = (s->nonvolatile_cfg >> 8) & 0xFF;
+        s->pos = 0;
+        s->len = 2;
+        s->state = STATE_READING_DATA;
+        break;
+    case WNVCR:
+        if (s->write_enable) {
+            s->needed_bytes = 2;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RVCR:
+        s->data[0] = s->volatile_cfg & 0xFF;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case WVCR:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case REVCR:
+        s->data[0] = s->enh_volatile_cfg & 0xFF;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case WEVCR:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RESET_ENABLE:
+        s->reset_enable = true;
+        break;
+    case RESET_MEMORY:
+        if (s->reset_enable) {
+            reset_memory(s);
+        }
+        break;
     default:
         qemu_log_mask(LOG_GUEST_ERROR, "M25P80: Unknown cmd %x\n", value);
         break;
@@ -647,14 +921,26 @@ static int m25p80_init(SSISlave *ss)
     return 0;
 }
 
+static void m25p80_reset(DeviceState *d)
+{
+    Flash *s = M25P80(d);
+
+    reset_memory(s);
+}
+
 static void m25p80_pre_save(void *opaque)
 {
     flash_sync_dirty((Flash *)opaque, -1);
 }
 
+static Property m25p80_properties[] = {
+    DEFINE_PROP_UINT32("nonvolatile-cfg", Flash, nonvolatile_cfg, 0x8FFF),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static const VMStateDescription vmstate_m25p80 = {
     .name = "xilinx_spi",
-    .version_id = 1,
+    .version_id = 2,
     .minimum_version_id = 1,
     .pre_save = m25p80_pre_save,
     .fields = (VMStateField[]) {
@@ -666,6 +952,12 @@ static const VMStateDescription vmstate_m25p80 = {
         VMSTATE_UINT8(cmd_in_progress, Flash),
         VMSTATE_UINT64(cur_addr, Flash),
         VMSTATE_BOOL(write_enable, Flash),
+        VMSTATE_BOOL_V(reset_enable, Flash, 2),
+        VMSTATE_UINT8_V(ear, Flash, 2),
+        VMSTATE_BOOL_V(four_bytes_address_mode, Flash, 2),
+        VMSTATE_UINT32_V(nonvolatile_cfg, Flash, 2),
+        VMSTATE_UINT32_V(volatile_cfg, Flash, 2),
+        VMSTATE_UINT32_V(enh_volatile_cfg, Flash, 2),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -681,6 +973,8 @@ static void m25p80_class_init(ObjectClass *klass, void *data)
     k->set_cs = m25p80_cs;
     k->cs_polarity = SSI_CS_LOW;
     dc->vmsd = &vmstate_m25p80;
+    dc->props = m25p80_properties;
+    dc->reset = m25p80_reset;
     mc->pi = data;
 }
 
diff --git a/qemu/hw/block/nand.c b/qemu/hw/block/nand.c
index 61d2cec03..29c659681 100644
--- a/qemu/hw/block/nand.c
+++ b/qemu/hw/block/nand.c
@@ -18,10 +18,12 @@
 
 #ifndef NAND_IO
 
-# include "hw/hw.h"
-# include "hw/block/flash.h"
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 
 # define NAND_CMD_READ0		0x00
@@ -522,8 +524,8 @@ void nand_setio(DeviceState *dev, uint32_t value)
 
     if (s->ale) {
         unsigned int shift = s->addrlen * 8;
-        unsigned int mask = ~(0xff << shift);
-        unsigned int v = value << shift;
+        uint64_t mask = ~(0xffull << shift);
+        uint64_t v = (uint64_t)value << shift;
 
         s->addr = (s->addr & mask) | v;
         s->addrlen ++;
@@ -635,7 +637,7 @@ DeviceState *nand_init(BlockBackend *blk, int manf_id, int chip_id)
     qdev_prop_set_uint8(dev, "manufacturer_id", manf_id);
     qdev_prop_set_uint8(dev, "chip_id", chip_id);
     if (blk) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk);
+        qdev_prop_set_drive(dev, "drive", blk, &error_fatal);
     }
 
     qdev_init_nofail(dev);
@@ -712,7 +714,7 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         memset(s->storage + (PAGE(addr) << OOB_SHIFT),
                         0xff, OOB_SIZE << s->erase_shift);
         i = SECTOR(addr);
-        page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
+        page = SECTOR(addr + (1 << (ADDR_SHIFT + s->erase_shift)));
         for (; i < page; i ++)
             if (blk_write(s->blk, i, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
diff --git a/qemu/hw/block/nvme.c b/qemu/hw/block/nvme.c
index 40d488032..173988ee8 100644
--- a/qemu/hw/block/nvme.c
+++ b/qemu/hw/block/nvme.c
@@ -20,11 +20,13 @@
  *      -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>
  */
 
+#include "qemu/osdep.h"
 #include <hw/block/block.h>
 #include <hw/hw.h>
 #include <hw/pci/msix.h>
 #include <hw/pci/pci.h>
 #include "sysemu/sysemu.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "sysemu/block-backend.h"
 
@@ -201,10 +203,11 @@ static void nvme_rw_cb(void *opaque, int ret)
     NvmeCtrl *n = sq->ctrl;
     NvmeCQueue *cq = n->cq[sq->cqid];
 
-    block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
     if (!ret) {
+        block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_SUCCESS;
     } else {
+        block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_INTERNAL_DEV_ERROR;
     }
     if (req->has_sg) {
@@ -238,18 +241,22 @@ static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     uint64_t data_size = (uint64_t)nlb << data_shift;
     uint64_t aio_slba  = slba << (data_shift - BDRV_SECTOR_BITS);
     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
+    enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 
     if ((slba + nlb) > ns->id_ns.nsze) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_LBA_RANGE | NVME_DNR;
     }
+
     if (nvme_map_prp(&req->qsg, prp1, prp2, data_size, n)) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
+
     assert((nlb << data_shift) == req->qsg.size);
 
     req->has_sg = true;
-    dma_acct_start(n->conf.blk, &req->acct, &req->qsg,
-                   is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
+    dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
     req->aiocb = is_write ?
         dma_blk_write(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req) :
         dma_blk_read(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req);
@@ -805,7 +812,7 @@ static int nvme_init(PCIDevice *pci_dev)
 
     n->num_namespaces = 1;
     n->num_queues = 64;
-    n->reg_size = 1 << qemu_fls(0x1004 + 2 * (n->num_queues + 1) * 4);
+    n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
     n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
@@ -910,45 +917,13 @@ static void nvme_class_init(ObjectClass *oc, void *data)
     dc->vmsd = &nvme_vmstate;
 }
 
-static void nvme_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
-{
-    NvmeCtrl *s = NVME(obj);
-
-    visit_type_int32(v, &s->conf.bootindex, name, errp);
-}
-
-static void nvme_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void nvme_instance_init(Object *obj)
 {
     NvmeCtrl *s = NVME(obj);
-    int32_t boot_index;
-    Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    /* check whether bootindex is present in fw_boot_order list  */
-    check_boot_index(boot_index, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    /* change bootindex to a new one */
-    s->conf.bootindex = boot_index;
-
-out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-}
-
-static void nvme_instance_init(Object *obj)
-{
-    object_property_add(obj, "bootindex", "int32",
-                        nvme_get_bootindex,
-                        nvme_set_bootindex, NULL, NULL, NULL);
-    object_property_set_int(obj, -1, "bootindex", NULL);
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/namespace@1,0",
+                                  DEVICE(obj), &error_abort);
 }
 
 static const TypeInfo nvme_info = {
diff --git a/qemu/hw/block/nvme.h b/qemu/hw/block/nvme.h
index bf3a3ccac..8fb0c1075 100644
--- a/qemu/hw/block/nvme.h
+++ b/qemu/hw/block/nvme.h
@@ -1,5 +1,6 @@
 #ifndef HW_NVME_H
 #define HW_NVME_H
+#include "qemu/cutils.h"
 
 typedef struct NvmeBar {
     uint64_t    cap;
diff --git a/qemu/hw/block/onenand.c b/qemu/hw/block/onenand.c
index 1b2c89375..883f4b1fa 100644
--- a/qemu/hw/block/onenand.c
+++ b/qemu/hw/block/onenand.c
@@ -18,6 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -786,7 +788,7 @@ static int onenand_initfn(SysBusDevice *sbd)
     s->otp = memset(g_malloc((64 + 2) << PAGE_SHIFT),
                     0xff, (64 + 2) << PAGE_SHIFT);
     memory_region_init_ram(&s->ram, OBJECT(s), "onenand.ram",
-                           0xc000 << s->shift, &error_abort);
+                           0xc000 << s->shift, &error_fatal);
     vmstate_register_ram_global(&s->ram);
     ram = memory_region_get_ram_ptr(&s->ram);
     s->boot[0] = ram + (0x0000 << s->shift);
diff --git a/qemu/hw/block/pflash_cfi01.c b/qemu/hw/block/pflash_cfi01.c
index 2ba6c7729..106a77523 100644
--- a/qemu/hw/block/pflash_cfi01.c
+++ b/qemu/hw/block/pflash_cfi01.c
@@ -36,14 +36,17 @@
  * It does not implement much more ...
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "exec/address-spaces.h"
 #include "qemu/host-utils.h"
 #include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
 
 #define PFLASH_BUG(fmt, ...) \
 do { \
@@ -95,6 +98,7 @@ struct pflash_t {
     MemoryRegion mem;
     char *name;
     void *storage;
+    VMChangeStateEntry *vmstate;
 };
 
 static int pflash_post_load(void *opaque, int version_id);
@@ -942,13 +946,25 @@ MemoryRegion *pflash_cfi01_get_memory(pflash_t *fl)
     return &fl->mem;
 }
 
+static void postload_update_cb(void *opaque, int running, RunState state)
+{
+    pflash_t *pfl = opaque;
+
+    /* This is called after bdrv_invalidate_cache_all.  */
+    qemu_del_vm_change_state_handler(pfl->vmstate);
+    pfl->vmstate = NULL;
+
+    DPRINTF("%s: updating bdrv for %s\n", __func__, pfl->name);
+    pflash_update(pfl, 0, pfl->sector_len * pfl->nb_blocs);
+}
+
 static int pflash_post_load(void *opaque, int version_id)
 {
     pflash_t *pfl = opaque;
 
     if (!pfl->ro) {
-        DPRINTF("%s: updating bdrv for %s\n", __func__, pfl->name);
-        pflash_update(pfl, 0, pfl->sector_len * pfl->nb_blocs);
+        pfl->vmstate = qemu_add_vm_change_state_handler(postload_update_cb,
+                                                        pfl);
     }
     return 0;
 }
diff --git a/qemu/hw/block/pflash_cfi02.c b/qemu/hw/block/pflash_cfi02.c
index 074a005f6..b13172c6e 100644
--- a/qemu/hw/block/pflash_cfi02.c
+++ b/qemu/hw/block/pflash_cfi02.c
@@ -35,8 +35,10 @@
  * It does not implement multiple sectors erase
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
@@ -430,8 +432,8 @@ static void pflash_write (pflash_t *pfl, hwaddr offset,
             }
             pfl->status = 0x00;
             /* Let's wait 5 seconds before chip erase is done */
-            timer_mod(pfl->timer,
-                           qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() * 5));
+            timer_mod(pfl->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND * 5));
             break;
         case 0x30:
             /* Sector erase */
@@ -445,8 +447,8 @@ static void pflash_write (pflash_t *pfl, hwaddr offset,
             }
             pfl->status = 0x00;
             /* Let's wait 1/2 second before sector erase is done */
-            timer_mod(pfl->timer,
-                           qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 2));
+            timer_mod(pfl->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND / 2));
             break;
         default:
             DPRINTF("%s: invalid command %02x (wc 5)\n", __func__, cmd);
diff --git a/qemu/hw/block/tc58128.c b/qemu/hw/block/tc58128.c
index 728f1c3b6..7909d5041 100644
--- a/qemu/hw/block/tc58128.c
+++ b/qemu/hw/block/tc58128.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "hw/loader.h"
diff --git a/qemu/hw/block/virtio-blk.c b/qemu/hw/block/virtio-blk.c
index 1556c9cf5..3f88f8cf5 100644
--- a/qemu/hw/block/virtio-blk.c
+++ b/qemu/hw/block/virtio-blk.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/error-report.h"
@@ -20,7 +22,6 @@
 #include "sysemu/blockdev.h"
 #include "hw/virtio/virtio-blk.h"
 #include "dataplane/virtio-blk.h"
-#include "migration/migration.h"
 #include "block/scsi.h"
 #ifdef __linux__
 # include <scsi/sg.h>
@@ -28,26 +29,23 @@
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 
-VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s)
+void virtio_blk_init_request(VirtIOBlock *s, VirtIOBlockReq *req)
 {
-    VirtIOBlockReq *req = g_slice_new(VirtIOBlockReq);
     req->dev = s;
     req->qiov.size = 0;
     req->in_len = 0;
     req->next = NULL;
     req->mr_next = NULL;
-    return req;
 }
 
 void virtio_blk_free_request(VirtIOBlockReq *req)
 {
     if (req) {
-        g_slice_free(VirtIOBlockReq, req);
+        g_free(req);
     }
 }
 
-static void virtio_blk_complete_request(VirtIOBlockReq *req,
-                                        unsigned char status)
+static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
 {
     VirtIOBlock *s = req->dev;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
@@ -56,12 +54,11 @@ static void virtio_blk_complete_request(VirtIOBlockReq *req,
 
     stb_p(&req->in->status, status);
     virtqueue_push(s->vq, &req->elem, req->in_len);
-    virtio_notify(vdev, s->vq);
-}
-
-static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
-{
-    req->dev->complete_request(req, status);
+    if (s->dataplane_started && !s->dataplane_disabled) {
+        virtio_blk_data_plane_notify(s->dataplane);
+    } else {
+        virtio_notify(vdev, s->vq);
+    }
 }
 
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
@@ -72,11 +69,14 @@ static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
     VirtIOBlock *s = req->dev;
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* Break the link as the next request is going to be parsed from the
+         * ring again. Otherwise we may end up doing a double completion! */
+        req->mr_next = NULL;
         req->next = s->rq;
         s->rq = req;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        block_acct_done(blk_get_stats(s->blk), &req->acct);
+        block_acct_failed(blk_get_stats(s->blk), &req->acct);
         virtio_blk_free_request(req);
     }
 
@@ -189,13 +189,11 @@ out:
 
 static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s)
 {
-    VirtIOBlockReq *req = virtio_blk_alloc_request(s);
+    VirtIOBlockReq *req = virtqueue_pop(s->vq, sizeof(VirtIOBlockReq));
 
-    if (!virtqueue_pop(s->vq, &req->elem)) {
-        virtio_blk_free_request(req);
-        return NULL;
+    if (req) {
+        virtio_blk_init_request(s, req);
     }
-
     return req;
 }
 
@@ -404,24 +402,16 @@ void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb)
     for (i = 0; i < mrb->num_reqs; i++) {
         VirtIOBlockReq *req = mrb->reqs[i];
         if (num_reqs > 0) {
-            bool merge = true;
-
-            /* merge would exceed maximum number of IOVs */
-            if (niov + req->qiov.niov > IOV_MAX) {
-                merge = false;
-            }
-
-            /* merge would exceed maximum transfer length of backend device */
-            if (req->qiov.size / BDRV_SECTOR_SIZE + nb_sectors > max_xfer_len) {
-                merge = false;
-            }
-
-            /* requests are not sequential */
-            if (sector_num + nb_sectors != req->sector_num) {
-                merge = false;
-            }
-
-            if (!merge) {
+            /*
+             * NOTE: We cannot merge the requests in below situations:
+             * 1. requests are not sequential
+             * 2. merge would exceed maximum number of IOVs
+             * 3. merge would exceed maximum transfer length of backend device
+             */
+            if (sector_num + nb_sectors != req->sector_num ||
+                niov > blk_get_max_iov(blk) - req->qiov.niov ||
+                req->qiov.size / BDRV_SECTOR_SIZE > max_xfer_len ||
+                nb_sectors > max_xfer_len - req->qiov.size / BDRV_SECTOR_SIZE) {
                 submit_requests(blk, mrb, start, num_reqs, niov);
                 num_reqs = 0;
             }
@@ -536,6 +526,8 @@ void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
         if (!virtio_blk_sect_range_ok(req->dev, req->sector_num,
                                       req->qiov.size)) {
             virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
+            block_acct_invalid(blk_get_stats(req->dev->blk),
+                               is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
             virtio_blk_free_request(req);
             return;
         }
@@ -586,19 +578,12 @@ void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
     }
 }
 
-static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
 {
-    VirtIOBlock *s = VIRTIO_BLK(vdev);
     VirtIOBlockReq *req;
     MultiReqBuffer mrb = {};
 
-    /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
-     * dataplane here instead of waiting for .set_status().
-     */
-    if (s->dataplane) {
-        virtio_blk_data_plane_start(s->dataplane);
-        return;
-    }
+    blk_io_plug(s->blk);
 
     while ((req = virtio_blk_get_request(s))) {
         virtio_blk_handle_request(req, &mrb);
@@ -607,6 +592,24 @@ static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
     if (mrb.num_reqs) {
         virtio_blk_submit_multireq(s->blk, &mrb);
     }
+
+    blk_io_unplug(s->blk);
+}
+
+static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOBlock *s = (VirtIOBlock *)vdev;
+
+    if (s->dataplane) {
+        /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+         * dataplane here instead of waiting for .set_status().
+         */
+        virtio_blk_data_plane_start(s->dataplane);
+        if (!s->dataplane_disabled) {
+            return;
+        }
+    }
+    virtio_blk_handle_vq(s, vq);
 }
 
 static void virtio_blk_dma_restart_bh(void *opaque)
@@ -731,7 +734,7 @@ static uint64_t virtio_blk_get_features(VirtIODevice *vdev, uint64_t features,
     virtio_add_feature(&features, VIRTIO_BLK_F_GEOMETRY);
     virtio_add_feature(&features, VIRTIO_BLK_F_TOPOLOGY);
     virtio_add_feature(&features, VIRTIO_BLK_F_BLK_SIZE);
-    if (__virtio_has_feature(features, VIRTIO_F_VERSION_1)) {
+    if (virtio_has_feature(features, VIRTIO_F_VERSION_1)) {
         if (s->conf.scsi) {
             error_setg(errp, "Please set scsi=off for virtio-blk devices in order to use virtio 1.0");
             return 0;
@@ -782,10 +785,11 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
      *
      * s->blk would erroneously be placed in writethrough mode.
      */
-    if (!virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) {
         aio_context_acquire(blk_get_aio_context(s->blk));
         blk_set_enable_write_cache(s->blk,
-                                   virtio_has_feature(vdev, VIRTIO_BLK_F_WCE));
+                                   virtio_vdev_has_feature(vdev,
+                                                           VIRTIO_BLK_F_WCE));
         aio_context_release(blk_get_aio_context(s->blk));
     }
 }
@@ -793,6 +797,11 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
 static void virtio_blk_save(QEMUFile *f, void *opaque)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
+    VirtIOBlock *s = VIRTIO_BLK(vdev);
+
+    if (s->dataplane) {
+        virtio_blk_data_plane_stop(s->dataplane);
+    }
 
     virtio_save(vdev, f);
 }
@@ -804,8 +813,7 @@ static void virtio_blk_save_device(VirtIODevice *vdev, QEMUFile *f)
 
     while (req) {
         qemu_put_sbyte(f, 1);
-        qemu_put_buffer(f, (unsigned char *)&req->elem,
-                        sizeof(VirtQueueElement));
+        qemu_put_virtqueue_element(f, &req->elem);
         req = req->next;
     }
     qemu_put_sbyte(f, 0);
@@ -828,16 +836,11 @@ static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
     VirtIOBlock *s = VIRTIO_BLK(vdev);
 
     while (qemu_get_sbyte(f)) {
-        VirtIOBlockReq *req = virtio_blk_alloc_request(s);
-        qemu_get_buffer(f, (unsigned char *)&req->elem,
-                        sizeof(VirtQueueElement));
+        VirtIOBlockReq *req;
+        req = qemu_get_virtqueue_element(f, sizeof(VirtIOBlockReq));
+        virtio_blk_init_request(s, req);
         req->next = s->rq;
         s->rq = req;
-
-        virtqueue_map_sg(req->elem.in_sg, req->elem.in_addr,
-            req->elem.in_num, 1);
-        virtqueue_map_sg(req->elem.out_sg, req->elem.out_addr,
-            req->elem.out_num, 0);
     }
 
     return 0;
@@ -854,36 +857,6 @@ static const BlockDevOps virtio_block_ops = {
     .resize_cb = virtio_blk_resize,
 };
 
-/* Disable dataplane thread during live migration since it does not
- * update the dirty memory bitmap yet.
- */
-static void virtio_blk_migration_state_changed(Notifier *notifier, void *data)
-{
-    VirtIOBlock *s = container_of(notifier, VirtIOBlock,
-                                  migration_state_notifier);
-    MigrationState *mig = data;
-    Error *err = NULL;
-
-    if (migration_in_setup(mig)) {
-        if (!s->dataplane) {
-            return;
-        }
-        virtio_blk_data_plane_destroy(s->dataplane);
-        s->dataplane = NULL;
-    } else if (migration_has_finished(mig) ||
-               migration_has_failed(mig)) {
-        if (s->dataplane) {
-            return;
-        }
-        blk_drain_all(); /* complete in-flight non-dataplane requests */
-        virtio_blk_data_plane_create(VIRTIO_DEVICE(s), &s->conf,
-                                     &s->dataplane, &err);
-        if (err != NULL) {
-            error_report_err(err);
-        }
-    }
-}
-
 static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
@@ -918,15 +891,12 @@ static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
     s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
     s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output);
-    s->complete_request = virtio_blk_complete_request;
     virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         virtio_cleanup(vdev);
         return;
     }
-    s->migration_state_notifier.notify = virtio_blk_migration_state_changed;
-    add_migration_state_change_notifier(&s->migration_state_notifier);
 
     s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
     register_savevm(dev, "virtio-blk", virtio_blk_id++, 2,
@@ -942,7 +912,6 @@ static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
 
-    remove_migration_state_change_notifier(&s->migration_state_notifier);
     virtio_blk_data_plane_destroy(s->dataplane);
     s->dataplane = NULL;
     qemu_del_vm_change_state_handler(s->change);
@@ -970,11 +939,10 @@ static Property virtio_blk_properties[] = {
     DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
     DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
 #ifdef __linux__
-    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, true),
+    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, false),
 #endif
     DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
                     true),
-    DEFINE_PROP_BIT("x-data-plane", VirtIOBlock, conf.data_plane, 0, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/qemu/hw/block/xen_blkif.h b/qemu/hw/block/xen_blkif.h
index 711b69274..c68487cb3 100644
--- a/qemu/hw/block/xen_blkif.h
+++ b/qemu/hw/block/xen_blkif.h
@@ -85,8 +85,10 @@ static inline void blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_reque
 		d->nr_sectors = s->nr_sectors;
 		return;
 	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
+	/* prevent the compiler from optimizing the code and using src->nr_segments instead */
+	barrier();
+	if (n > dst->nr_segments)
+		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
 		dst->seg[i] = src->seg[i];
 }
@@ -106,8 +108,10 @@ static inline void blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_reque
 		d->nr_sectors = s->nr_sectors;
 		return;
 	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
+	/* prevent the compiler from optimizing the code and using src->nr_segments instead */
+	barrier();
+	if (n > dst->nr_segments)
+		n = dst->nr_segments;
 	for (i = 0; i < n; i++)
 		dst->seg[i] = src->seg[i];
 }
diff --git a/qemu/hw/block/xen_disk.c b/qemu/hw/block/xen_disk.c
index 267d8a8c7..d4ce380fe 100644
--- a/qemu/hw/block/xen_disk.c
+++ b/qemu/hw/block/xen_disk.c
@@ -19,19 +19,8 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <inttypes.h>
-#include <time.h>
-#include <fcntl.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/mman.h>
 #include <sys/uio.h>
 
@@ -40,6 +29,7 @@
 #include "xen_blkif.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
 
@@ -76,7 +66,6 @@ struct ioreq {
     off_t               start;
     QEMUIOVector        v;
     int                 presync;
-    int                 postsync;
     uint8_t             mapped;
 
     /* grant mapping */
@@ -145,7 +134,6 @@ static void ioreq_reset(struct ioreq *ioreq)
     ioreq->status = 0;
     ioreq->start = 0;
     ioreq->presync = 0;
-    ioreq->postsync = 0;
     ioreq->mapped = 0;
 
     memset(ioreq->domids, 0, sizeof(ioreq->domids));
@@ -174,11 +162,11 @@ static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
 static void destroy_grant(gpointer pgnt)
 {
     PersistentGrant *grant = pgnt;
-    XenGnttab gnt = grant->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = grant->blkdev->xendev.gnttabdev;
 
-    if (xc_gnttab_munmap(gnt, grant->page, 1) != 0) {
+    if (xengnttab_unmap(gnt, grant->page, 1) != 0) {
         xen_be_printf(&grant->blkdev->xendev, 0,
-                      "xc_gnttab_munmap failed: %s\n",
+                      "xengnttab_unmap failed: %s\n",
                       strerror(errno));
     }
     grant->blkdev->persistent_gnt_count--;
@@ -191,11 +179,11 @@ static void remove_persistent_region(gpointer data, gpointer dev)
 {
     PersistentRegion *region = data;
     struct XenBlkDev *blkdev = dev;
-    XenGnttab gnt = blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = blkdev->xendev.gnttabdev;
 
-    if (xc_gnttab_munmap(gnt, region->addr, region->num) != 0) {
+    if (xengnttab_unmap(gnt, region->addr, region->num) != 0) {
         xen_be_printf(&blkdev->xendev, 0,
-                      "xc_gnttab_munmap region %p failed: %s\n",
+                      "xengnttab_unmap region %p failed: %s\n",
                       region->addr, strerror(errno));
     }
     xen_be_printf(&blkdev->xendev, 3,
@@ -330,7 +318,7 @@ err:
 
 static void ioreq_unmap(struct ioreq *ioreq)
 {
-    XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = ioreq->blkdev->xendev.gnttabdev;
     int i;
 
     if (ioreq->num_unmap == 0 || ioreq->mapped == 0) {
@@ -340,8 +328,9 @@ static void ioreq_unmap(struct ioreq *ioreq)
         if (!ioreq->pages) {
             return;
         }
-        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
-            xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
+        if (xengnttab_unmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
+            xen_be_printf(&ioreq->blkdev->xendev, 0,
+                          "xengnttab_unmap failed: %s\n",
                           strerror(errno));
         }
         ioreq->blkdev->cnt_map -= ioreq->num_unmap;
@@ -351,8 +340,9 @@ static void ioreq_unmap(struct ioreq *ioreq)
             if (!ioreq->page[i]) {
                 continue;
             }
-            if (xc_gnttab_munmap(gnt, ioreq->page[i], 1) != 0) {
-                xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
+            if (xengnttab_unmap(gnt, ioreq->page[i], 1) != 0) {
+                xen_be_printf(&ioreq->blkdev->xendev, 0,
+                              "xengnttab_unmap failed: %s\n",
                               strerror(errno));
             }
             ioreq->blkdev->cnt_map--;
@@ -364,7 +354,7 @@ static void ioreq_unmap(struct ioreq *ioreq)
 
 static int ioreq_map(struct ioreq *ioreq)
 {
-    XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = ioreq->blkdev->xendev.gnttabdev;
     uint32_t domids[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     uint32_t refs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     void *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
@@ -415,7 +405,7 @@ static int ioreq_map(struct ioreq *ioreq)
     }
 
     if (batch_maps && new_maps) {
-        ioreq->pages = xc_gnttab_map_grant_refs
+        ioreq->pages = xengnttab_map_grant_refs
             (gnt, new_maps, domids, refs, ioreq->prot);
         if (ioreq->pages == NULL) {
             xen_be_printf(&ioreq->blkdev->xendev, 0,
@@ -431,7 +421,7 @@ static int ioreq_map(struct ioreq *ioreq)
         ioreq->blkdev->cnt_map += new_maps;
     } else if (new_maps)  {
         for (i = 0; i < new_maps; i++) {
-            ioreq->page[i] = xc_gnttab_map_grant_ref
+            ioreq->page[i] = xengnttab_map_grant_ref
                 (gnt, domids[i], refs[i], ioreq->prot);
             if (ioreq->page[i] == NULL) {
                 xen_be_printf(&ioreq->blkdev->xendev, 0,
@@ -521,12 +511,6 @@ static void qemu_aio_complete(void *opaque, int ret)
     if (ioreq->aio_inflight > 0) {
         return;
     }
-    if (ioreq->postsync) {
-        ioreq->postsync = 0;
-        ioreq->aio_inflight++;
-        blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
-        return;
-    }
 
     ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
     ioreq_unmap(ioreq);
@@ -538,7 +522,11 @@ static void qemu_aio_complete(void *opaque, int ret)
             break;
         }
     case BLKIF_OP_READ:
-        block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        if (ioreq->status == BLKIF_RSP_OKAY) {
+            block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        } else {
+            block_acct_failed(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        }
         break;
     case BLKIF_OP_DISCARD:
     default:
@@ -577,7 +565,9 @@ static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
         }
 
         block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
-                         ioreq->v.size, BLOCK_ACCT_WRITE);
+                         ioreq->v.size,
+                         ioreq->req.operation == BLKIF_OP_WRITE ?
+                         BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
         ioreq->aio_inflight++;
         blk_aio_writev(blkdev->blk, ioreq->start / BLOCK_SIZE,
                        &ioreq->v, ioreq->v.size / BLOCK_SIZE,
@@ -721,6 +711,23 @@ static void blk_handle_requests(struct XenBlkDev *blkdev)
 
         /* parse them */
         if (ioreq_parse(ioreq) != 0) {
+
+            switch (ioreq->req.operation) {
+            case BLKIF_OP_READ:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_READ);
+                break;
+            case BLKIF_OP_WRITE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_WRITE);
+                break;
+            case BLKIF_OP_FLUSH_DISKCACHE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_FLUSH);
+            default:
+                break;
+            };
+
             if (blk_send_response_one(ioreq)) {
                 xen_be_send_notify(&blkdev->xendev);
             }
@@ -764,9 +771,9 @@ static void blk_alloc(struct XenDevice *xendev)
     if (xen_mode != XEN_EMULATE) {
         batch_maps = 1;
     }
-    if (xc_gnttab_set_max_grants(xendev->gnttabdev,
+    if (xengnttab_set_max_grants(xendev->gnttabdev,
             MAX_GRANTS(max_requests, BLKIF_MAX_SEGMENTS_PER_REQUEST)) < 0) {
-        xen_be_printf(xendev, 0, "xc_gnttab_set_max_grants failed: %s\n",
+        xen_be_printf(xendev, 0, "xengnttab_set_max_grants failed: %s\n",
                       strerror(errno));
     }
 }
@@ -811,6 +818,9 @@ static int blk_init(struct XenDevice *xendev)
     if (!strcmp("aio", blkdev->fileproto)) {
         blkdev->fileproto = "raw";
     }
+    if (!strcmp("vhd", blkdev->fileproto)) {
+        blkdev->fileproto = "vpc";
+    }
     if (blkdev->mode == NULL) {
         blkdev->mode = xenstore_read_be_str(&blkdev->xendev, "mode");
     }
@@ -879,12 +889,14 @@ static int blk_connect(struct XenDevice *xendev)
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
     int pers, index, qflags;
     bool readonly = true;
+    bool writethrough = true;
 
     /* read-only ? */
     if (blkdev->directiosafe) {
         qflags = BDRV_O_NOCACHE | BDRV_O_NATIVE_AIO;
     } else {
-        qflags = BDRV_O_CACHE_WB;
+        qflags = 0;
+        writethrough = false;
     }
     if (strcmp(blkdev->mode, "w") == 0) {
         qflags |= BDRV_O_RDWR;
@@ -908,7 +920,7 @@ static int blk_connect(struct XenDevice *xendev)
 
         /* setup via xenbus -> create new block driver instance */
         xen_be_printf(&blkdev->xendev, 2, "create new bdrv (xenbus setup)\n");
-        blkdev->blk = blk_new_open(blkdev->dev, blkdev->filename, NULL, options,
+        blkdev->blk = blk_new_open(blkdev->filename, NULL, options,
                                    qflags, &local_err);
         if (!blkdev->blk) {
             xen_be_printf(&blkdev->xendev, 0, "error: %s\n",
@@ -916,6 +928,7 @@ static int blk_connect(struct XenDevice *xendev)
             error_free(local_err);
             return -1;
         }
+        blk_set_enable_write_cache(blkdev->blk, !writethrough);
     } else {
         /* setup via qemu cmdline -> already setup for us */
         xen_be_printf(&blkdev->xendev, 2, "get configured bdrv (cmdline setup)\n");
@@ -932,9 +945,11 @@ static int blk_connect(struct XenDevice *xendev)
     blk_attach_dev_nofail(blkdev->blk, blkdev);
     blkdev->file_size = blk_getlength(blkdev->blk);
     if (blkdev->file_size < 0) {
+        BlockDriverState *bs = blk_bs(blkdev->blk);
+        const char *drv_name = bs ? bdrv_get_format_name(bs) : NULL;
         xen_be_printf(&blkdev->xendev, 1, "blk_getlength: %d (%s) | drv %s\n",
                       (int)blkdev->file_size, strerror(-blkdev->file_size),
-                      bdrv_get_format_name(blk_bs(blkdev->blk)) ?: "-");
+                      drv_name ?: "-");
         blkdev->file_size = 0;
     }
 
@@ -971,7 +986,7 @@ static int blk_connect(struct XenDevice *xendev)
         }
     }
 
-    blkdev->sring = xc_gnttab_map_grant_ref(blkdev->xendev.gnttabdev,
+    blkdev->sring = xengnttab_map_grant_ref(blkdev->xendev.gnttabdev,
                                             blkdev->xendev.dom,
                                             blkdev->ring_ref,
                                             PROT_READ | PROT_WRITE);
@@ -1036,7 +1051,7 @@ static void blk_disconnect(struct XenDevice *xendev)
     xen_be_unbind_evtchn(&blkdev->xendev);
 
     if (blkdev->sring) {
-        xc_gnttab_munmap(blkdev->xendev.gnttabdev, blkdev->sring, 1);
+        xengnttab_unmap(blkdev->xendev.gnttabdev, blkdev->sring, 1);
         blkdev->cnt_map--;
         blkdev->sring = NULL;
     }
diff --git a/qemu/hw/bt/core.c b/qemu/hw/bt/core.c
index 0ffc94889..615f0af07 100644
--- a/qemu/hw/bt/core.c
+++ b/qemu/hw/bt/core.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
diff --git a/qemu/hw/bt/hci-csr.c b/qemu/hw/bt/hci-csr.c
index 7b9b91608..2e970b656 100644
--- a/qemu/hw/bt/hci-csr.c
+++ b/qemu/hw/bt/hci-csr.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "qemu/timer.h"
@@ -362,7 +363,7 @@ static int csrhci_ioctl(struct CharDriverState *chr, int cmd, void *arg)
     switch (cmd) {
     case CHR_IOCTL_SERIAL_SET_PARAMS:
         ssp = (QEMUSerialSetParams *) arg;
-        s->baud_delay = get_ticks_per_sec() / ssp->speed;
+        s->baud_delay = NANOSECONDS_PER_SECOND / ssp->speed;
         /* Moments later... (but shorter than 100ms) */
         s->modem_state |= CHR_TIOCM_CTS;
         break;
@@ -388,7 +389,7 @@ static void csrhci_reset(struct csrhci_s *s)
     s->out_len = 0;
     s->out_size = FIFO_LEN;
     s->in_len = 0;
-    s->baud_delay = get_ticks_per_sec();
+    s->baud_delay = NANOSECONDS_PER_SECOND;
     s->enable = 0;
     s->in_hdr = INT_MAX;
     s->in_data = INT_MAX;
diff --git a/qemu/hw/bt/hci.c b/qemu/hw/bt/hci.c
index 7ea3dc6b7..7d5220509 100644
--- a/qemu/hw/bt/hci.c
+++ b/qemu/hw/bt/hci.c
@@ -18,11 +18,16 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/replay.h"
+#include "qemu/cutils.h"
 
 struct bt_hci_s {
     uint8_t *(*evt_packet)(void *opaque);
@@ -72,6 +77,8 @@ struct bt_hci_s {
 
     struct HCIInfo info;
     struct bt_device_s device;
+
+    Error *replay_blocker;
 };
 
 #define DEFAULT_RSSI_DBM	20
@@ -595,7 +602,7 @@ static void bt_hci_inquiry_result(struct bt_hci_s *hci,
 static void bt_hci_mod_timer_1280ms(QEMUTimer *timer, int period)
 {
     timer_mod(timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(period << 7, get_ticks_per_sec(), 100));
+                     (uint64_t)(period << 7) * 10000000);
 }
 
 static void bt_hci_inquiry_start(struct bt_hci_s *hci, int length)
@@ -1099,7 +1106,7 @@ static int bt_hci_mode_change(struct bt_hci_s *hci, uint16_t handle,
     bt_hci_event_status(hci, HCI_SUCCESS);
 
     timer_mod(link->acl_mode_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(interval * 625, get_ticks_per_sec(), 1000000));
+                                    ((uint64_t)interval * 625) * 1000);
     bt_hci_lmp_mode_change_master(hci, link->link, mode, interval);
 
     return 0;
@@ -1151,8 +1158,7 @@ static void bt_hci_reset(struct bt_hci_s *hci)
     hci->event_mask[7] = 0x00;
     hci->device.inquiry_scan = 0;
     hci->device.page_scan = 0;
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
     hci->device.lmp_name = NULL;
     hci->device.class[0] = 0x00;
     hci->device.class[1] = 0x00;
@@ -1829,8 +1835,7 @@ static void bt_submit_hci(struct HCIInfo *info,
     case cmd_opcode_pack(OGF_HOST_CTL, OCF_CHANGE_LOCAL_NAME):
         LENGTH_CHECK(change_local_name);
 
-        if (hci->device.lmp_name)
-            g_free((void *) hci->device.lmp_name);
+        g_free((void *) hci->device.lmp_name);
         hci->device.lmp_name = g_strndup(PARAM(change_local_name, name),
                         sizeof(PARAM(change_local_name, name)));
         bt_hci_event_complete_status(hci, HCI_SUCCESS);
@@ -2191,6 +2196,9 @@ struct HCIInfo *bt_new_hci(struct bt_scatternet_s *net)
 
     s->device.handle_destroy = bt_hci_destroy;
 
+    error_setg(&s->replay_blocker, QERR_REPLAY_NOT_SUPPORTED, "-bt hci");
+    replay_add_blocker(s->replay_blocker);
+
     return &s->info;
 }
 
@@ -2231,8 +2239,7 @@ static void bt_hci_done(struct HCIInfo *info)
 
     bt_device_done(&hci->device);
 
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
 
     /* Be gentle and send DISCONNECT to all connected peers and those
      * currently waiting for us to accept or reject a connection request.
diff --git a/qemu/hw/bt/hid.c b/qemu/hw/bt/hid.c
index af494e1e0..f6affbbb4 100644
--- a/qemu/hw/bt/hid.c
+++ b/qemu/hw/bt/hid.c
@@ -18,6 +18,7 @@
  * with this program; if not, if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "ui/console.h"
diff --git a/qemu/hw/bt/l2cap.c b/qemu/hw/bt/l2cap.c
index 591e04778..806525194 100644
--- a/qemu/hw/bt/l2cap.c
+++ b/qemu/hw/bt/l2cap.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/bt.h"
diff --git a/qemu/hw/bt/sdp.c b/qemu/hw/bt/sdp.c
index c90374795..be26009b0 100644
--- a/qemu/hw/bt/sdp.c
+++ b/qemu/hw/bt/sdp.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/bt.h"
 
@@ -42,7 +43,7 @@ struct bt_l2cap_sdp_state_s {
 
 static ssize_t sdp_datalen(const uint8_t **element, ssize_t *left)
 {
-    size_t len = *(*element) ++ & SDP_DSIZE_MASK;
+    uint32_t len = *(*element) ++ & SDP_DSIZE_MASK;
 
     if (!*left)
         return -1;
@@ -150,12 +151,14 @@ static ssize_t sdp_svc_search(struct bt_l2cap_sdp_state_s *sdp,
         if (seqlen < 3 || len < seqlen)
             return -SDP_INVALID_SYNTAX;
         len -= seqlen;
-
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -278,8 +281,11 @@ static ssize_t sdp_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_attr_match(record, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_attr_match(record, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_attr_match(record, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
@@ -393,8 +399,11 @@ static ssize_t sdp_svc_search_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -413,8 +422,11 @@ static ssize_t sdp_svc_search_attr_get(struct bt_l2cap_sdp_state_s *sdp,
         while (seqlen)
             if (sdp_svc_attr_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_attr_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_attr_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
diff --git a/qemu/hw/char/Makefile.objs b/qemu/hw/char/Makefile.objs
index 5931cc840..69a553cd8 100644
--- a/qemu/hw/char/Makefile.objs
+++ b/qemu/hw/char/Makefile.objs
@@ -16,6 +16,7 @@ obj-$(CONFIG_SH4) += sh_serial.o
 obj-$(CONFIG_PSERIES) += spapr_vty.o
 obj-$(CONFIG_DIGIC) += digic-uart.o
 obj-$(CONFIG_STM32F2XX_USART) += stm32f2xx_usart.o
+obj-$(CONFIG_RASPI) += bcm2835_aux.o
 
 common-obj-$(CONFIG_ETRAXFS) += etraxfs_ser.o
 common-obj-$(CONFIG_ISA_DEBUG) += debugcon.o
diff --git a/qemu/hw/char/bcm2835_aux.c b/qemu/hw/char/bcm2835_aux.c
new file mode 100644
index 000000000..0394d11a8
--- /dev/null
+++ b/qemu/hw/char/bcm2835_aux.c
@@ -0,0 +1,316 @@
+/*
+ * BCM2835 (Raspberry Pi / Pi 2) Aux block (mini UART and SPI).
+ * Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ * Based on pl011.c, copyright terms below:
+ *
+ * Arm PrimeCell PL011 UART
+ *
+ * Copyright (c) 2006 CodeSourcery.
+ * Written by Paul Brook
+ *
+ * This code is licensed under the GPL.
+ *
+ * At present only the core UART functions (data path for tx/rx) are
+ * implemented. The following features/registers are unimplemented:
+ *  - Line/modem control
+ *  - Scratch register
+ *  - Extra control
+ *  - Baudrate
+ *  - SPI interfaces
+ */
+
+#include "qemu/osdep.h"
+#include "hw/char/bcm2835_aux.h"
+
+#define AUX_IRQ         0x0
+#define AUX_ENABLES     0x4
+#define AUX_MU_IO_REG   0x40
+#define AUX_MU_IER_REG  0x44
+#define AUX_MU_IIR_REG  0x48
+#define AUX_MU_LCR_REG  0x4c
+#define AUX_MU_MCR_REG  0x50
+#define AUX_MU_LSR_REG  0x54
+#define AUX_MU_MSR_REG  0x58
+#define AUX_MU_SCRATCH  0x5c
+#define AUX_MU_CNTL_REG 0x60
+#define AUX_MU_STAT_REG 0x64
+#define AUX_MU_BAUD_REG 0x68
+
+/* bits in IER/IIR registers */
+#define TX_INT  0x1
+#define RX_INT  0x2
+
+static void bcm2835_aux_update(BCM2835AuxState *s)
+{
+    /* signal an interrupt if either:
+     * 1. rx interrupt is enabled and we have a non-empty rx fifo, or
+     * 2. the tx interrupt is enabled (since we instantly drain the tx fifo)
+     */
+    s->iir = 0;
+    if ((s->ier & RX_INT) && s->read_count != 0) {
+        s->iir |= RX_INT;
+    }
+    if (s->ier & TX_INT) {
+        s->iir |= TX_INT;
+    }
+    qemu_set_irq(s->irq, s->iir != 0);
+}
+
+static uint64_t bcm2835_aux_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835AuxState *s = opaque;
+    uint32_t c, res;
+
+    switch (offset) {
+    case AUX_IRQ:
+        return s->iir != 0;
+
+    case AUX_ENABLES:
+        return 1; /* mini UART permanently enabled */
+
+    case AUX_MU_IO_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        c = s->read_fifo[s->read_pos];
+        if (s->read_count > 0) {
+            s->read_count--;
+            if (++s->read_pos == BCM2835_AUX_RX_FIFO_LEN) {
+                s->read_pos = 0;
+            }
+        }
+        if (s->chr) {
+            qemu_chr_accept_input(s->chr);
+        }
+        bcm2835_aux_update(s);
+        return c;
+
+    case AUX_MU_IER_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        return 0xc0 | s->ier; /* FIFO enables always read 1 */
+
+    case AUX_MU_IIR_REG:
+        res = 0xc0; /* FIFO enables */
+        /* The spec is unclear on what happens when both tx and rx
+         * interrupts are active, besides that this cannot occur. At
+         * present, we choose to prioritise the rx interrupt, since
+         * the tx fifo is always empty. */
+        if (s->read_count != 0) {
+            res |= 0x4;
+        } else {
+            res |= 0x2;
+        }
+        if (s->iir == 0) {
+            res |= 0x1;
+        }
+        return res;
+
+    case AUX_MU_LCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_LCR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_MCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MCR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_LSR_REG:
+        res = 0x60; /* tx idle, empty */
+        if (s->read_count != 0) {
+            res |= 0x1;
+        }
+        return res;
+
+    case AUX_MU_MSR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MSR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_SCRATCH:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_SCRATCH unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_CNTL_REG:
+        return 0x3; /* tx, rx enabled */
+
+    case AUX_MU_STAT_REG:
+        res = 0x30e; /* space in the output buffer, empty tx fifo, idle tx/rx */
+        if (s->read_count > 0) {
+            res |= 0x1; /* data in input buffer */
+            assert(s->read_count < BCM2835_AUX_RX_FIFO_LEN);
+            res |= ((uint32_t)s->read_count) << 16; /* rx fifo fill level */
+        }
+        return res;
+
+    case AUX_MU_BAUD_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_BAUD_REG unsupported\n", __func__);
+        return 0;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+}
+
+static void bcm2835_aux_write(void *opaque, hwaddr offset, uint64_t value,
+                              unsigned size)
+{
+    BCM2835AuxState *s = opaque;
+    unsigned char ch;
+
+    switch (offset) {
+    case AUX_ENABLES:
+        if (value != 1) {
+            qemu_log_mask(LOG_UNIMP, "%s: unsupported attempt to enable SPI "
+                          "or disable UART\n", __func__);
+        }
+        break;
+
+    case AUX_MU_IO_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        ch = value;
+        if (s->chr) {
+            qemu_chr_fe_write(s->chr, &ch, 1);
+        }
+        break;
+
+    case AUX_MU_IER_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        s->ier = value & (TX_INT | RX_INT);
+        bcm2835_aux_update(s);
+        break;
+
+    case AUX_MU_IIR_REG:
+        if (value & 0x2) {
+            s->read_count = 0;
+        }
+        break;
+
+    case AUX_MU_LCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_LCR_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_MCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MCR_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_SCRATCH:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_SCRATCH unsupported\n", __func__);
+        break;
+
+    case AUX_MU_CNTL_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_CNTL_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_BAUD_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_BAUD_REG unsupported\n", __func__);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+    }
+
+    bcm2835_aux_update(s);
+}
+
+static int bcm2835_aux_can_receive(void *opaque)
+{
+    BCM2835AuxState *s = opaque;
+
+    return s->read_count < BCM2835_AUX_RX_FIFO_LEN;
+}
+
+static void bcm2835_aux_put_fifo(void *opaque, uint8_t value)
+{
+    BCM2835AuxState *s = opaque;
+    int slot;
+
+    slot = s->read_pos + s->read_count;
+    if (slot >= BCM2835_AUX_RX_FIFO_LEN) {
+        slot -= BCM2835_AUX_RX_FIFO_LEN;
+    }
+    s->read_fifo[slot] = value;
+    s->read_count++;
+    if (s->read_count == BCM2835_AUX_RX_FIFO_LEN) {
+        /* buffer full */
+    }
+    bcm2835_aux_update(s);
+}
+
+static void bcm2835_aux_receive(void *opaque, const uint8_t *buf, int size)
+{
+    bcm2835_aux_put_fifo(opaque, *buf);
+}
+
+static const MemoryRegionOps bcm2835_aux_ops = {
+    .read = bcm2835_aux_read,
+    .write = bcm2835_aux_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_aux = {
+    .name = TYPE_BCM2835_AUX,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY(read_fifo, BCM2835AuxState,
+                            BCM2835_AUX_RX_FIFO_LEN),
+        VMSTATE_UINT8(read_pos, BCM2835AuxState),
+        VMSTATE_UINT8(read_count, BCM2835AuxState),
+        VMSTATE_UINT8(ier, BCM2835AuxState),
+        VMSTATE_UINT8(iir, BCM2835AuxState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_aux_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    BCM2835AuxState *s = BCM2835_AUX(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &bcm2835_aux_ops, s,
+                          TYPE_BCM2835_AUX, 0x100);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+}
+
+static void bcm2835_aux_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835AuxState *s = BCM2835_AUX(dev);
+
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, bcm2835_aux_can_receive,
+                              bcm2835_aux_receive, NULL, s);
+    }
+}
+
+static Property bcm2835_aux_props[] = {
+    DEFINE_PROP_CHR("chardev", BCM2835AuxState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void bcm2835_aux_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = bcm2835_aux_realize;
+    dc->vmsd = &vmstate_bcm2835_aux;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+    dc->props = bcm2835_aux_props;
+}
+
+static const TypeInfo bcm2835_aux_info = {
+    .name          = TYPE_BCM2835_AUX,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835AuxState),
+    .instance_init = bcm2835_aux_init,
+    .class_init    = bcm2835_aux_class_init,
+};
+
+static void bcm2835_aux_register_types(void)
+{
+    type_register_static(&bcm2835_aux_info);
+}
+
+type_init(bcm2835_aux_register_types)
diff --git a/qemu/hw/char/cadence_uart.c b/qemu/hw/char/cadence_uart.c
index 9d379e5b1..797787823 100644
--- a/qemu/hw/char/cadence_uart.c
+++ b/qemu/hw/char/cadence_uart.c
@@ -16,6 +16,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/char/cadence_uart.h"
 
 #ifdef CADENCE_UART_ERR_DEBUG
@@ -204,7 +205,7 @@ static void uart_parameters_setup(CadenceUARTState *s)
     }
 
     packet_size += ssp.data_bits + ssp.stop_bits;
-    s->char_tx_time = (get_ticks_per_sec() / ssp.speed) * packet_size;
+    s->char_tx_time = (NANOSECONDS_PER_SECOND / ssp.speed) * packet_size;
     if (s->chr) {
         qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
     }
@@ -374,6 +375,9 @@ static void uart_write(void *opaque, hwaddr offset,
 
     DB_PRINT(" offset:%x data:%08x\n", (unsigned)offset, (unsigned)value);
     offset >>= 2;
+    if (offset >= CADENCE_UART_R_MAX) {
+        return;
+    }
     switch (offset) {
     case R_IER: /* ier (wts imr) */
         s->r[R_IMR] |= value;
@@ -478,7 +482,7 @@ static void cadence_uart_init(Object *obj)
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
 
-    s->char_tx_time = (get_ticks_per_sec() / 9600) * 10;
+    s->char_tx_time = (NANOSECONDS_PER_SECOND / 9600) * 10;
 }
 
 static int cadence_uart_post_load(void *opaque, int version_id)
diff --git a/qemu/hw/char/debugcon.c b/qemu/hw/char/debugcon.c
index 36f1c4adb..e7f025ec6 100644
--- a/qemu/hw/char/debugcon.c
+++ b/qemu/hw/char/debugcon.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/char.h"
 #include "hw/isa/isa.h"
diff --git a/qemu/hw/char/digic-uart.c b/qemu/hw/char/digic-uart.c
index 6d44576ff..d3bc533d7 100644
--- a/qemu/hw/char/digic-uart.c
+++ b/qemu/hw/char/digic-uart.c
@@ -26,6 +26,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
diff --git a/qemu/hw/char/escc.c b/qemu/hw/char/escc.c
index ba653efd6..7bf09a007 100644
--- a/qemu/hw/char/escc.c
+++ b/qemu/hw/char/escc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/char/escc.h"
@@ -714,7 +715,7 @@ MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
     return &d->mmio;
 }
 
-static const uint8_t qcode_to_keycode[Q_KEY_CODE_MAX] = {
+static const uint8_t qcode_to_keycode[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_SHIFT]         = 99,
     [Q_KEY_CODE_SHIFT_R]       = 110,
     [Q_KEY_CODE_ALT]           = 19,
@@ -841,14 +842,16 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
 {
     ChannelState *s = (ChannelState *)dev;
     int qcode, keycode;
+    InputKeyEvent *key;
 
-    assert(evt->kind == INPUT_EVENT_KIND_KEY);
-    qcode = qemu_input_key_value_to_qcode(evt->key->key);
+    assert(evt->type == INPUT_EVENT_KIND_KEY);
+    key = evt->u.key.data;
+    qcode = qemu_input_key_value_to_qcode(key->key);
     trace_escc_sunkbd_event_in(qcode, QKeyCode_lookup[qcode],
-                               evt->key->down);
+                               key->down);
 
     if (qcode == Q_KEY_CODE_CAPS_LOCK) {
-        if (evt->key->down) {
+        if (key->down) {
             s->caps_lock_mode ^= 1;
             if (s->caps_lock_mode == 2) {
                 return; /* Drop second press */
@@ -862,7 +865,7 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
     }
 
     if (qcode == Q_KEY_CODE_NUM_LOCK) {
-        if (evt->key->down) {
+        if (key->down) {
             s->num_lock_mode ^= 1;
             if (s->num_lock_mode == 2) {
                 return; /* Drop second press */
@@ -876,7 +879,7 @@ static void sunkbd_handle_event(DeviceState *dev, QemuConsole *src,
     }
 
     keycode = qcode_to_keycode[qcode];
-    if (!evt->key->down) {
+    if (!key->down) {
         keycode |= 0x80;
     }
     trace_escc_sunkbd_event_out(keycode);
@@ -1035,6 +1038,7 @@ static void escc_class_init(ObjectClass *klass, void *data)
     dc->reset = escc_reset;
     dc->vmsd = &vmstate_escc;
     dc->props = escc_properties;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo escc_info = {
diff --git a/qemu/hw/char/etraxfs_ser.c b/qemu/hw/char/etraxfs_ser.c
index 857c13621..146b387e7 100644
--- a/qemu/hw/char/etraxfs_ser.c
+++ b/qemu/hw/char/etraxfs_ser.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/log.h"
@@ -165,7 +166,7 @@ static void serial_receive(void *opaque, const uint8_t *buf, int size)
 
     /* Got a byte.  */
     if (s->rx_fifo_len >= 16) {
-        qemu_log("WARNING: UART dropped char.\n");
+        D(qemu_log("WARNING: UART dropped char.\n"));
         return;
     }
 
@@ -182,15 +183,13 @@ static void serial_receive(void *opaque, const uint8_t *buf, int size)
 static int serial_can_receive(void *opaque)
 {
     ETRAXSerial *s = opaque;
-    int r;
 
     /* Is the receiver enabled?  */
     if (!(s->regs[RW_REC_CTRL] & (1 << 3))) {
         return 0;
     }
 
-    r = sizeof(s->rx_fifo) - s->rx_fifo_len;
-    return r;
+    return sizeof(s->rx_fifo) - s->rx_fifo_len;
 }
 
 static void serial_event(void *opaque, int event)
diff --git a/qemu/hw/char/exynos4210_uart.c b/qemu/hw/char/exynos4210_uart.c
index 7614e5860..885ecc027 100644
--- a/qemu/hw/char/exynos4210_uart.c
+++ b/qemu/hw/char/exynos4210_uart.c
@@ -19,7 +19,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
 
@@ -234,10 +236,8 @@ static int fifo_empty_elements_number(Exynos4210UartFIFO *q)
 
 static void fifo_reset(Exynos4210UartFIFO *q)
 {
-    if (q->data != NULL) {
-        g_free(q->data);
-        q->data = NULL;
-    }
+    g_free(q->data);
+    q->data = NULL;
 
     q->data = (uint8_t *)g_malloc0(q->size);
 
@@ -597,15 +597,17 @@ DeviceState *exynos4210_uart_create(hwaddr addr,
 
     if (!chr) {
         if (channel >= MAX_SERIAL_PORTS) {
-            hw_error("Only %d serial ports are supported by QEMU.\n",
-                     MAX_SERIAL_PORTS);
+            error_report("Only %d serial ports are supported by QEMU",
+                         MAX_SERIAL_PORTS);
+            exit(1);
         }
         chr = serial_hds[channel];
         if (!chr) {
             snprintf(label, ARRAY_SIZE(label), "%s%d", chr_name, channel);
             chr = qemu_chr_new(label, "null", NULL);
             if (!(chr)) {
-                hw_error("Can't assign serial port to UART%d.\n", channel);
+                error_report("Can't assign serial port to UART%d", channel);
+                exit(1);
             }
         }
     }
diff --git a/qemu/hw/char/grlib_apbuart.c b/qemu/hw/char/grlib_apbuart.c
index 35ef66177..871524c82 100644
--- a/qemu/hw/char/grlib_apbuart.c
+++ b/qemu/hw/char/grlib_apbuart.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 
diff --git a/qemu/hw/char/imx_serial.c b/qemu/hw/char/imx_serial.c
index f3fbc776b..6df74ac7c 100644
--- a/qemu/hw/char/imx_serial.c
+++ b/qemu/hw/char/imx_serial.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2008 OKL
  * Originally Written by Hans Jiang
  * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -17,63 +18,25 @@
  *     is a real serial device.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
+#include "qemu/osdep.h"
+#include "hw/char/imx_serial.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
-#include "hw/arm/imx.h"
 
-//#define DEBUG_SERIAL 1
-#ifdef DEBUG_SERIAL
-#define DPRINTF(fmt, args...) \
-do { printf("imx_serial: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-//#define DEBUG_IMPLEMENTATION 1
-#ifdef DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_serial: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_UART
+#define DEBUG_IMX_UART 0
 #endif
 
-#define TYPE_IMX_SERIAL "imx-serial"
-#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
-
-typedef struct IMXSerialState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    int32_t readbuff;
-
-    uint32_t usr1;
-    uint32_t usr2;
-    uint32_t ucr1;
-    uint32_t ucr2;
-    uint32_t uts1;
-
-    /*
-     * The registers below are implemented just so that the
-     * guest OS sees what it has written
-     */
-    uint32_t onems;
-    uint32_t ufcr;
-    uint32_t ubmr;
-    uint32_t ubrc;
-    uint32_t ucr3;
-
-    qemu_irq irq;
-    CharDriverState *chr;
-} IMXSerialState;
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_UART) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_SERIAL, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_serial = {
-    .name = "imx-serial",
+    .name = TYPE_IMX_SERIAL,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -91,61 +54,14 @@ static const VMStateDescription vmstate_imx_serial = {
     },
 };
 
-
-#define URXD_CHARRDY    (1<<15)   /* character read is valid */
-#define URXD_ERR        (1<<14)   /* Character has error */
-#define URXD_BRK        (1<<11)   /* Break received */
-
-#define USR1_PARTYER    (1<<15)   /* Parity Error */
-#define USR1_RTSS       (1<<14)   /* RTS pin status */
-#define USR1_TRDY       (1<<13)   /* Tx ready */
-#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
-#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
-#define USR1_FRAMERR    (1<<10)   /* Framing error  */
-#define USR1_RRDY       (1<<9)    /* receiver ready */
-#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
-#define USR1_DTRD       (1<<7)    /* DTR changed */
-#define USR1_RXDS       (1<<6)    /* Receiver is idle */
-#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
-#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
-
-#define USR2_ADET       (1<<15)   /* Autobaud complete */
-#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
-#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
-#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
-#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
-#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
-#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
-#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
-#define USR2_WAKE       (1<<7)    /* Start bit detected */
-#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
-#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
-#define USR2_RTSF       (1<<4)    /* RTS transition */
-#define USR2_TXDC       (1<<3)    /* Transmission complete */
-#define USR2_BRCD       (1<<2)    /* Break condition detected */
-#define USR2_ORE        (1<<1)    /* Overrun error */
-#define USR2_RDR        (1<<0)    /* Receive data ready */
-
-#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
-#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
-#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
-#define UCR1_UARTEN     (1<<0)    /* UART Enable */
-
-#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
-#define UCR2_RXEN       (1<<1)    /* Receiver enable */
-#define UCR2_SRST       (1<<0)    /* Reset complete */
-
-#define UTS1_TXEMPTY    (1<<6)
-#define UTS1_RXEMPTY    (1<<5)
-#define UTS1_TXFULL     (1<<4)
-#define UTS1_RXFULL     (1<<3)
-
 static void imx_update(IMXSerialState *s)
 {
     uint32_t flags;
 
     flags = (s->usr1 & s->ucr1) & (USR1_TRDY|USR1_RRDY);
-    if (!(s->ucr1 & UCR1_TXMPTYEN)) {
+    if (s->ucr1 & UCR1_TXMPTYEN) {
+        flags |= (s->uts1 & UTS1_TXEMPTY);
+    } else {
         flags &= ~USR1_TRDY;
     }
 
@@ -192,7 +108,8 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
     IMXSerialState *s = (IMXSerialState *)opaque;
     uint32_t c;
 
-    DPRINTF("read(offset=%x)\n", offset >> 2);
+    DPRINTF("read(offset=0x%" HWADDR_PRIx ")\n", offset);
+
     switch (offset >> 2) {
     case 0x0: /* URXD */
         c = s->readbuff;
@@ -203,7 +120,9 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
             s->usr2 &= ~USR2_RDR;
             s->uts1 |= UTS1_RXEMPTY;
             imx_update(s);
-            qemu_chr_accept_input(s->chr);
+            if (s->chr) {
+                qemu_chr_accept_input(s->chr);
+            }
         }
         return c;
 
@@ -242,20 +161,20 @@ static uint64_t imx_serial_read(void *opaque, hwaddr offset,
         return 0x0; /* TODO */
 
     default:
-        IPRINTF("imx_serial_read: bad offset: 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         return 0;
     }
 }
 
 static void imx_serial_write(void *opaque, hwaddr offset,
-                      uint64_t value, unsigned size)
+                             uint64_t value, unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
     unsigned char ch;
 
-    DPRINTF("write(offset=%x, value = %x) to %s\n",
-            offset >> 2,
-            (unsigned int)value, s->chr ? s->chr->label : "NODEV");
+    DPRINTF("write(offset=0x%" HWADDR_PRIx ", value = 0x%x) to %s\n",
+            offset, (unsigned int)value, s->chr ? s->chr->label : "NODEV");
 
     switch (offset >> 2) {
     case 0x10: /* UTXD */
@@ -273,7 +192,9 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x20: /* UCR1 */
         s->ucr1 = value & 0xffff;
+
         DPRINTF("write(ucr1=%x)\n", (unsigned int)value);
+
         imx_update(s);
         break;
 
@@ -290,7 +211,9 @@ static void imx_serial_write(void *opaque, hwaddr offset,
         }
         if (value & UCR2_RXEN) {
             if (!(s->ucr2 & UCR2_RXEN)) {
-                qemu_chr_accept_input(s->chr);
+                if (s->chr) {
+                    qemu_chr_accept_input(s->chr);
+                }
             }
         }
         s->ucr2 = value & 0xffff;
@@ -298,25 +221,25 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x25: /* USR1 */
         value &= USR1_AWAKE | USR1_AIRINT | USR1_DTRD | USR1_AGTIM |
-            USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
+                 USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
         s->usr1 &= ~value;
         break;
 
     case 0x26: /* USR2 */
-       /*
-        * Writing 1 to some bits clears them; all other
-        * values are ignored
-        */
+        /*
+         * Writing 1 to some bits clears them; all other
+         * values are ignored
+         */
         value &= USR2_ADET | USR2_DTRF | USR2_IDLE | USR2_ACST |
-            USR2_RIDELT | USR2_IRINT | USR2_WAKE |
-            USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
+                 USR2_RIDELT | USR2_IRINT | USR2_WAKE |
+                 USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
         s->usr2 &= ~value;
         break;
 
-        /*
-         * Linux expects to see what it writes to these registers
-         * We don't currently alter the baud rate
-         */
+    /*
+     * Linux expects to see what it writes to these registers
+     * We don't currently alter the baud rate
+     */
     case 0x29: /* UBIR */
         s->ubrc = value & 0xffff;
         break;
@@ -339,12 +262,14 @@ static void imx_serial_write(void *opaque, hwaddr offset,
 
     case 0x2d: /* UTS1 */
     case 0x23: /* UCR4 */
-        IPRINTF("Unimplemented Register %x written to\n", offset >> 2);
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: Unimplemented reg 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         /* TODO */
         break;
 
     default:
-        IPRINTF("imx_serial_write: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
     }
 }
 
@@ -357,7 +282,9 @@ static int imx_can_receive(void *opaque)
 static void imx_put_data(void *opaque, uint32_t value)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
+
     DPRINTF("received char\n");
+
     s->usr1 |= USR1_RRDY;
     s->usr2 |= USR2_RDR;
     s->uts1 &= ~UTS1_RXEMPTY;
@@ -384,62 +311,30 @@ static const struct MemoryRegionOps imx_serial_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int imx_serial_init(SysBusDevice *dev)
+static void imx_serial_realize(DeviceState *dev, Error **errp)
 {
     IMXSerialState *s = IMX_SERIAL(dev);
 
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &imx_serial_ops, s,
-                          "imx-serial", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-    sysbus_init_irq(dev, &s->irq);
-
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, imx_can_receive, imx_receive,
                               imx_event, s);
     } else {
-        DPRINTF("No char dev for uart at 0x%lx\n",
-                (unsigned long)s->iomem.ram_addr);
+        DPRINTF("No char dev for uart\n");
     }
-
-    return 0;
 }
 
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
+static void imx_serial_init(Object *obj)
 {
-    DeviceState *dev;
-    SysBusDevice *bus;
-    CharDriverState *chr;
-    const char chr_name[] = "serial";
-    char label[ARRAY_SIZE(chr_name) + 1];
-
-    dev = qdev_create(NULL, TYPE_IMX_SERIAL);
-
-    if (uart >= MAX_SERIAL_PORTS) {
-        hw_error("Cannot assign uart %d: QEMU supports only %d ports\n",
-                 uart, MAX_SERIAL_PORTS);
-    }
-    chr = serial_hds[uart];
-    if (!chr) {
-        snprintf(label, ARRAY_SIZE(label), "%s%d", chr_name, uart);
-        chr = qemu_chr_new(label, "null", NULL);
-        if (!(chr)) {
-            hw_error("Can't assign serial port to imx-uart%d.\n", uart);
-        }
-    }
-
-    qdev_prop_set_chr(dev, "chardev", chr);
-    bus = SYS_BUS_DEVICE(dev);
-    qdev_init_nofail(dev);
-    if (addr != (hwaddr)-1) {
-        sysbus_mmio_map(bus, 0, addr);
-    }
-    sysbus_connect_irq(bus, 0, irq);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    IMXSerialState *s = IMX_SERIAL(obj);
 
+    memory_region_init_io(&s->iomem, obj, &imx_serial_ops, s,
+                          TYPE_IMX_SERIAL, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
 }
 
-
-static Property imx32_serial_properties[] = {
+static Property imx_serial_properties[] = {
     DEFINE_PROP_CHR("chardev", IMXSerialState, chr),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -447,21 +342,21 @@ static Property imx32_serial_properties[] = {
 static void imx_serial_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = imx_serial_init;
+    dc->realize = imx_serial_realize;
     dc->vmsd = &vmstate_imx_serial;
     dc->reset = imx_serial_reset_at_boot;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
     dc->desc = "i.MX series UART";
-    dc->props = imx32_serial_properties;
+    dc->props = imx_serial_properties;
 }
 
 static const TypeInfo imx_serial_info = {
-    .name = TYPE_IMX_SERIAL,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(IMXSerialState),
-    .class_init = imx_serial_class_init,
+    .name           = TYPE_IMX_SERIAL,
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(IMXSerialState),
+    .instance_init  = imx_serial_init,
+    .class_init     = imx_serial_class_init,
 };
 
 static void imx_serial_register_types(void)
diff --git a/qemu/hw/char/ipoctal232.c b/qemu/hw/char/ipoctal232.c
index c8d5cdb36..bc0ae4980 100644
--- a/qemu/hw/char/ipoctal232.c
+++ b/qemu/hw/char/ipoctal232.c
@@ -8,6 +8,7 @@
  * later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ipack/ipack.h"
 #include "qemu/bitops.h"
 #include "sysemu/char.h"
diff --git a/qemu/hw/char/lm32_juart.c b/qemu/hw/char/lm32_juart.c
index 62763f2f4..5bf8acfe8 100644
--- a/qemu/hw/char/lm32_juart.c
+++ b/qemu/hw/char/lm32_juart.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/char/lm32_uart.c b/qemu/hw/char/lm32_uart.c
index 837a46e8e..036813d0f 100644
--- a/qemu/hw/char/lm32_uart.c
+++ b/qemu/hw/char/lm32_uart.c
@@ -22,6 +22,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/char/mcf_uart.c b/qemu/hw/char/mcf_uart.c
index 98fd44e66..3c0438fd7 100644
--- a/qemu/hw/char/mcf_uart.c
+++ b/qemu/hw/char/mcf_uart.c
@@ -5,6 +5,7 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "sysemu/char.h"
@@ -126,7 +127,7 @@ static void mcf_uart_do_tx(mcf_uart_state *s)
 static void mcf_do_command(mcf_uart_state *s, uint8_t cmd)
 {
     /* Misc command.  */
-    switch ((cmd >> 4) & 3) {
+    switch ((cmd >> 4) & 7) {
     case 0: /* No-op.  */
         break;
     case 1: /* Reset mode register pointer.  */
diff --git a/qemu/hw/char/milkymist-uart.c b/qemu/hw/char/milkymist-uart.c
index 9b89b7e63..03b36b223 100644
--- a/qemu/hw/char/milkymist-uart.c
+++ b/qemu/hw/char/milkymist-uart.c
@@ -21,6 +21,7 @@
  *   http://www.milkymist.org/socdoc/uart.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/char/omap_uart.c b/qemu/hw/char/omap_uart.c
index 88f20943e..415bec5fa 100644
--- a/qemu/hw/char/omap_uart.c
+++ b/qemu/hw/char/omap_uart.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
@@ -55,8 +56,7 @@ struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
 {
-    struct omap_uart_s *s = (struct omap_uart_s *)
-            g_malloc0(sizeof(struct omap_uart_s));
+    struct omap_uart_s *s = g_new0(struct omap_uart_s, 1);
 
     s->base = base;
     s->fclk = fclk;
diff --git a/qemu/hw/char/parallel.c b/qemu/hw/char/parallel.c
index c2b553f0d..11c78fed8 100644
--- a/qemu/hw/char/parallel.c
+++ b/qemu/hw/char/parallel.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/char.h"
 #include "hw/isa/isa.h"
diff --git a/qemu/hw/char/pl011.c b/qemu/hw/char/pl011.c
index eac6fac09..210c87b4c 100644
--- a/qemu/hw/char/pl011.c
+++ b/qemu/hw/char/pl011.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 
diff --git a/qemu/hw/char/sclpconsole-lm.c b/qemu/hw/char/sclpconsole-lm.c
index 02ac80b65..7d4ff8120 100644
--- a/qemu/hw/char/sclpconsole-lm.c
+++ b/qemu/hw/char/sclpconsole-lm.c
@@ -13,6 +13,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
diff --git a/qemu/hw/char/sclpconsole.c b/qemu/hw/char/sclpconsole.c
index b014c7f52..45997ff4a 100644
--- a/qemu/hw/char/sclpconsole.c
+++ b/qemu/hw/char/sclpconsole.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <hw/qdev.h>
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
diff --git a/qemu/hw/char/serial-isa.c b/qemu/hw/char/serial-isa.c
index f3db024d0..1594ec4db 100644
--- a/qemu/hw/char/serial-isa.c
+++ b/qemu/hw/char/serial-isa.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/char/serial.h"
 #include "hw/isa/isa.h"
 
diff --git a/qemu/hw/char/serial-pci.c b/qemu/hw/char/serial-pci.c
index 1c8b9be5c..303104dd1 100644
--- a/qemu/hw/char/serial-pci.c
+++ b/qemu/hw/char/serial-pci.c
@@ -25,6 +25,8 @@
 
 /* see docs/specs/pci-serial.txt */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/char/serial.h"
 #include "hw/pci/pci.h"
 
diff --git a/qemu/hw/char/serial.c b/qemu/hw/char/serial.c
index 513d73c27..6d815b5c6 100644
--- a/qemu/hw/char/serial.c
+++ b/qemu/hw/char/serial.c
@@ -23,8 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/char/serial.h"
 #include "sysemu/char.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -177,7 +179,7 @@ static void serial_update_parameters(SerialState *s)
     ssp.parity = parity;
     ssp.data_bits = data_bits;
     ssp.stop_bits = stop_bits;
-    s->char_transmit_time =  (get_ticks_per_sec() / speed) * frame_size;
+    s->char_transmit_time =  (NANOSECONDS_PER_SECOND / speed) * frame_size;
     qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
 
     DPRINTF("speed=%d parity=%c data=%d stop=%d\n",
@@ -215,8 +217,10 @@ static void serial_update_msl(SerialState *s)
     /* The real 16550A apparently has a 250ns response latency to line status changes.
        We'll be lazy and poll only every 10ms, and only poll it at all if MSI interrupts are turned on */
 
-    if (s->poll_msl)
-        timer_mod(s->modem_status_poll, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + get_ticks_per_sec() / 100);
+    if (s->poll_msl) {
+        timer_mod(s->modem_status_poll, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  NANOSECONDS_PER_SECOND / 100);
+    }
 }
 
 static gboolean serial_xmit(GIOChannel *chan, GIOCondition cond, void *opaque)
@@ -822,7 +826,7 @@ static void serial_reset(void *opaque)
     s->mcr = UART_MCR_OUT2;
     s->scr = 0;
     s->tsr_retry = 0;
-    s->char_transmit_time = (get_ticks_per_sec() / 9600) * 10;
+    s->char_transmit_time = (NANOSECONDS_PER_SECOND / 9600) * 10;
     s->poll_msl = 0;
 
     s->timeout_ipending = 0;
@@ -888,18 +892,13 @@ SerialState *serial_init(int base, qemu_irq irq, int baudbase,
                          CharDriverState *chr, MemoryRegion *system_io)
 {
     SerialState *s;
-    Error *err = NULL;
 
     s = g_malloc0(sizeof(SerialState));
 
     s->irq = irq;
     s->baudbase = baudbase;
     s->chr = chr;
-    serial_realize_core(s, &err);
-    if (err != NULL) {
-        error_report_err(err);
-        exit(1);
-    }
+    serial_realize_core(s, &error_fatal);
 
     vmstate_register(NULL, base, &vmstate_serial, s);
 
@@ -949,7 +948,6 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
                             CharDriverState *chr, enum device_endian end)
 {
     SerialState *s;
-    Error *err = NULL;
 
     s = g_malloc0(sizeof(SerialState));
 
@@ -958,11 +956,7 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
     s->baudbase = baudbase;
     s->chr = chr;
 
-    serial_realize_core(s, &err);
-    if (err != NULL) {
-        error_report_err(err);
-        exit(1);
-    }
+    serial_realize_core(s, &error_fatal);
     vmstate_register(NULL, base, &vmstate_serial, s);
 
     memory_region_init_io(&s->io, NULL, &serial_mm_ops[end], s,
diff --git a/qemu/hw/char/sh_serial.c b/qemu/hw/char/sh_serial.c
index 9328dd1b5..4c55dcb7d 100644
--- a/qemu/hw/char/sh_serial.c
+++ b/qemu/hw/char/sh_serial.c
@@ -24,6 +24,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/char.h"
diff --git a/qemu/hw/char/spapr_vty.c b/qemu/hw/char/spapr_vty.c
index 36b328b9a..3498d7b05 100644
--- a/qemu/hw/char/spapr_vty.c
+++ b/qemu/hw/char/spapr_vty.c
@@ -1,3 +1,7 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/qdev.h"
 #include "sysemu/char.h"
 #include "hw/ppc/spapr.h"
diff --git a/qemu/hw/char/stm32f2xx_usart.c b/qemu/hw/char/stm32f2xx_usart.c
index c9d3a1be7..a94d61ceb 100644
--- a/qemu/hw/char/stm32f2xx_usart.c
+++ b/qemu/hw/char/stm32f2xx_usart.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/char/stm32f2xx_usart.h"
 
 #ifndef STM_USART_ERR_DEBUG
diff --git a/qemu/hw/char/virtio-console.c b/qemu/hw/char/virtio-console.c
index 2a867cb4e..2e36481a7 100644
--- a/qemu/hw/char/virtio-console.c
+++ b/qemu/hw/char/virtio-console.c
@@ -10,6 +10,7 @@
  * the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 #include "qemu/error-report.h"
 #include "trace.h"
diff --git a/qemu/hw/char/virtio-serial-bus.c b/qemu/hw/char/virtio-serial-bus.c
index bc56f5d52..6e5de6dec 100644
--- a/qemu/hw/char/virtio-serial-bus.c
+++ b/qemu/hw/char/virtio-serial-bus.c
@@ -18,6 +18,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
 #include "monitor/monitor.h"
 #include "qemu/error-report.h"
@@ -76,13 +78,13 @@ static VirtIOSerialPort *find_port_by_name(char *name)
 static bool use_multiport(VirtIOSerial *vser)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(vser);
-    return virtio_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT);
+    return virtio_vdev_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT);
 }
 
 static size_t write_to_port(VirtIOSerialPort *port,
                             const uint8_t *buf, size_t size)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtQueue *vq;
     size_t offset;
 
@@ -95,15 +97,17 @@ static size_t write_to_port(VirtIOSerialPort *port,
     while (offset < size) {
         size_t len;
 
-        if (!virtqueue_pop(vq, &elem)) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
             break;
         }
 
-        len = iov_from_buf(elem.in_sg, elem.in_num, 0,
+        len = iov_from_buf(elem->in_sg, elem->in_num, 0,
                            buf + offset, size - offset);
         offset += len;
 
-        virtqueue_push(vq, &elem, len);
+        virtqueue_push(vq, elem, len);
+        g_free(elem);
     }
 
     virtio_notify(VIRTIO_DEVICE(port->vser), vq);
@@ -112,13 +116,18 @@ static size_t write_to_port(VirtIOSerialPort *port,
 
 static void discard_vq_data(VirtQueue *vq, VirtIODevice *vdev)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
 
     if (!virtio_queue_ready(vq)) {
         return;
     }
-    while (virtqueue_pop(vq, &elem)) {
-        virtqueue_push(vq, &elem, 0);
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+        virtqueue_push(vq, elem, 0);
+        g_free(elem);
     }
     virtio_notify(vdev, vq);
 }
@@ -137,21 +146,22 @@ static void do_flush_queued_data(VirtIOSerialPort *port, VirtQueue *vq,
         unsigned int i;
 
         /* Pop an elem only if we haven't left off a previous one mid-way */
-        if (!port->elem.out_num) {
-            if (!virtqueue_pop(vq, &port->elem)) {
+        if (!port->elem) {
+            port->elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+            if (!port->elem) {
                 break;
             }
             port->iov_idx = 0;
             port->iov_offset = 0;
         }
 
-        for (i = port->iov_idx; i < port->elem.out_num; i++) {
+        for (i = port->iov_idx; i < port->elem->out_num; i++) {
             size_t buf_size;
             ssize_t ret;
 
-            buf_size = port->elem.out_sg[i].iov_len - port->iov_offset;
+            buf_size = port->elem->out_sg[i].iov_len - port->iov_offset;
             ret = vsc->have_data(port,
-                                  port->elem.out_sg[i].iov_base
+                                  port->elem->out_sg[i].iov_base
                                   + port->iov_offset,
                                   buf_size);
             if (port->throttled) {
@@ -166,8 +176,9 @@ static void do_flush_queued_data(VirtIOSerialPort *port, VirtQueue *vq,
         if (port->throttled) {
             break;
         }
-        virtqueue_push(vq, &port->elem, 0);
-        port->elem.out_num = 0;
+        virtqueue_push(vq, port->elem, 0);
+        g_free(port->elem);
+        port->elem = NULL;
     }
     virtio_notify(vdev, vq);
 }
@@ -184,22 +195,26 @@ static void flush_queued_data(VirtIOSerialPort *port)
 
 static size_t send_control_msg(VirtIOSerial *vser, void *buf, size_t len)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtQueue *vq;
 
     vq = vser->c_ivq;
     if (!virtio_queue_ready(vq)) {
         return 0;
     }
-    if (!virtqueue_pop(vq, &elem)) {
+
+    elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+    if (!elem) {
         return 0;
     }
 
     /* TODO: detect a buffer that's too short, set NEEDS_RESET */
-    iov_from_buf(elem.in_sg, elem.in_num, 0, buf, len);
+    iov_from_buf(elem->in_sg, elem->in_num, 0, buf, len);
 
-    virtqueue_push(vq, &elem, len);
+    virtqueue_push(vq, elem, len);
     virtio_notify(VIRTIO_DEVICE(vser), vq);
+    g_free(elem);
+
     return len;
 }
 
@@ -413,7 +428,7 @@ static void control_in(VirtIODevice *vdev, VirtQueue *vq)
 
 static void control_out(VirtIODevice *vdev, VirtQueue *vq)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtIOSerial *vser;
     uint8_t *buf;
     size_t len;
@@ -422,10 +437,15 @@ static void control_out(VirtIODevice *vdev, VirtQueue *vq)
 
     len = 0;
     buf = NULL;
-    while (virtqueue_pop(vq, &elem)) {
+    for (;;) {
         size_t cur_len;
 
-        cur_len = iov_size(elem.out_sg, elem.out_num);
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        cur_len = iov_size(elem->out_sg, elem->out_num);
         /*
          * Allocate a new buf only if we didn't have one previously or
          * if the size of the buf differs
@@ -436,10 +456,11 @@ static void control_out(VirtIODevice *vdev, VirtQueue *vq)
             buf = g_malloc(cur_len);
             len = cur_len;
         }
-        iov_to_buf(elem.out_sg, elem.out_num, 0, buf, cur_len);
+        iov_to_buf(elem->out_sg, elem->out_num, 0, buf, cur_len);
 
         handle_control_message(vser, buf, cur_len);
-        virtqueue_push(vq, &elem, 0);
+        virtqueue_push(vq, elem, 0);
+        g_free(elem);
     }
     g_free(buf);
     virtio_notify(vdev, vq);
@@ -619,16 +640,14 @@ static void virtio_serial_save_device(VirtIODevice *vdev, QEMUFile *f)
         qemu_put_byte(f, port->host_connected);
 
 	elem_popped = 0;
-        if (port->elem.out_num) {
+        if (port->elem) {
             elem_popped = 1;
         }
         qemu_put_be32s(f, &elem_popped);
         if (elem_popped) {
             qemu_put_be32s(f, &port->iov_idx);
             qemu_put_be64s(f, &port->iov_offset);
-
-            qemu_put_buffer(f, (unsigned char *)&port->elem,
-                            sizeof(port->elem));
+            qemu_put_virtqueue_element(f, port->elem);
         }
     }
 }
@@ -703,12 +722,8 @@ static int fetch_active_ports_list(QEMUFile *f, int version_id,
                 qemu_get_be32s(f, &port->iov_idx);
                 qemu_get_be64s(f, &port->iov_offset);
 
-                qemu_get_buffer(f, (unsigned char *)&port->elem,
-                                sizeof(port->elem));
-                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
-                                 port->elem.in_num, 1);
-                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
-                                 port->elem.out_num, 1);
+                port->elem =
+                    qemu_get_virtqueue_element(f, sizeof(VirtQueueElement));
 
                 /*
                  *  Port was throttled on source machine.  Let's
@@ -801,7 +816,7 @@ static const TypeInfo virtser_bus_info = {
 
 static void virtser_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent)
 {
-    VirtIOSerialPort *port = DO_UPCAST(VirtIOSerialPort, dev, qdev);
+    VirtIOSerialPort *port = VIRTIO_SERIAL_PORT(qdev);
 
     monitor_printf(mon, "%*sport %d, guest %s, host %s, throttle %s\n",
                    indent, "", port->id,
@@ -930,7 +945,7 @@ static void virtser_port_device_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    port->elem.out_num = 0;
+    port->elem = NULL;
 }
 
 static void virtser_port_device_plug(HotplugHandler *hotplug_dev,
diff --git a/qemu/hw/char/xen_console.c b/qemu/hw/char/xen_console.c
index eb7f450ab..cbf1dccbb 100644
--- a/qemu/hw/char/xen_console.c
+++ b/qemu/hw/char/xen_console.c
@@ -19,14 +19,9 @@
  *  with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include <sys/select.h>
-#include <fcntl.h>
-#include <unistd.h>
 #include <termios.h>
-#include <stdarg.h>
 #include <sys/mman.h>
 
 #include "hw/hw.h"
@@ -228,12 +223,12 @@ static int con_initialise(struct XenDevice *xendev)
 	con->buffer.max_capacity = limit;
 
     if (!xendev->dev) {
-        con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
-                                          XC_PAGE_SIZE,
+        xen_pfn_t mfn = con->ring_ref;
+        con->sring = xenforeignmemory_map(xen_fmem, con->xendev.dom,
                                           PROT_READ|PROT_WRITE,
-                                          con->ring_ref);
+                                          1, &mfn, NULL);
     } else {
-        con->sring = xc_gnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
+        con->sring = xengnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
                                              con->ring_ref,
                                              PROT_READ|PROT_WRITE);
     }
@@ -265,9 +260,6 @@ static void con_disconnect(struct XenDevice *xendev)
 {
     struct XenConsole *con = container_of(xendev, struct XenConsole, xendev);
 
-    if (!xendev->dev) {
-        return;
-    }
     if (con->chr) {
         qemu_chr_add_handlers(con->chr, NULL, NULL, NULL, NULL);
         qemu_chr_fe_release(con->chr);
@@ -275,12 +267,12 @@ static void con_disconnect(struct XenDevice *xendev)
     xen_be_unbind_evtchn(&con->xendev);
 
     if (con->sring) {
-        if (!xendev->gnttabdev) {
-            munmap(con->sring, XC_PAGE_SIZE);
+        if (!xendev->dev) {
+            xenforeignmemory_unmap(xen_fmem, con->sring, 1);
         } else {
-            xc_gnttab_munmap(xendev->gnttabdev, con->sring, 1);
+            xengnttab_unmap(xendev->gnttabdev, con->sring, 1);
         }
-	con->sring = NULL;
+        con->sring = NULL;
     }
 }
 
diff --git a/qemu/hw/char/xilinx_uartlite.c b/qemu/hw/char/xilinx_uartlite.c
index ef883a898..911af4a0d 100644
--- a/qemu/hw/char/xilinx_uartlite.c
+++ b/qemu/hw/char/xilinx_uartlite.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 
diff --git a/qemu/hw/core/empty_slot.c b/qemu/hw/core/empty_slot.c
index 612b1093a..c1b9c2b10 100644
--- a/qemu/hw/core/empty_slot.c
+++ b/qemu/hw/core/empty_slot.c
@@ -9,6 +9,7 @@
  * version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/empty_slot.h"
diff --git a/qemu/hw/core/fw-path-provider.c b/qemu/hw/core/fw-path-provider.c
index 7442d322d..33b99830e 100644
--- a/qemu/hw/core/fw-path-provider.c
+++ b/qemu/hw/core/fw-path-provider.c
@@ -15,6 +15,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/fw-path-provider.h"
 
 char *fw_path_provider_get_dev_path(FWPathProvider *p, BusState *bus,
diff --git a/qemu/hw/core/hotplug.c b/qemu/hw/core/hotplug.c
index 4e0107455..645cfca1b 100644
--- a/qemu/hw/core/hotplug.c
+++ b/qemu/hw/core/hotplug.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hotplug.h"
 #include "qemu/module.h"
 
diff --git a/qemu/hw/core/irq.c b/qemu/hw/core/irq.c
index 8a62a36d5..49ff2e64f 100644
--- a/qemu/hw/core/irq.c
+++ b/qemu/hw/core/irq.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/irq.h"
 #include "qom/object.h"
diff --git a/qemu/hw/core/loader.c b/qemu/hw/core/loader.c
index 216eeeb91..c0499571c 100644
--- a/qemu/hw/core/loader.c
+++ b/qemu/hw/core/loader.c
@@ -42,6 +42,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "disas/disas.h"
 #include "monitor/monitor.h"
@@ -51,12 +53,11 @@
 #include "hw/nvram/fw_cfg.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "qemu/cutils.h"
 
 #include <zlib.h>
 
-bool option_rom_has_mr = false;
-bool rom_file_has_mr = true;
-
 static int roms_loaded;
 
 /* return the size or -1 if error */
@@ -148,6 +149,28 @@ int load_image_targphys(const char *filename,
     return size;
 }
 
+int load_image_mr(const char *filename, MemoryRegion *mr)
+{
+    int size;
+
+    if (!memory_access_is_direct(mr, false)) {
+        /* Can only load an image into RAM or ROM */
+        return -1;
+    }
+
+    size = get_image_size(filename);
+
+    if (size > memory_region_size(mr)) {
+        return -1;
+    }
+    if (size > 0) {
+        if (rom_add_file_mr(filename, mr, -1) < 0) {
+            return -1;
+        }
+    }
+    return size;
+}
+
 void pstrcpy_targphys(const char *name, hwaddr dest, int buf_size,
                       const char *source)
 {
@@ -333,10 +356,66 @@ const char *load_elf_strerror(int error)
     }
 }
 
+void load_elf_hdr(const char *filename, void *hdr, bool *is64, Error **errp)
+{
+    int fd;
+    uint8_t e_ident_local[EI_NIDENT];
+    uint8_t *e_ident;
+    size_t hdr_size, off;
+    bool is64l;
+
+    if (!hdr) {
+        hdr = e_ident_local;
+    }
+    e_ident = hdr;
+
+    fd = open(filename, O_RDONLY | O_BINARY);
+    if (fd < 0) {
+        error_setg_errno(errp, errno, "Failed to open file: %s", filename);
+        return;
+    }
+    if (read(fd, hdr, EI_NIDENT) != EI_NIDENT) {
+        error_setg_errno(errp, errno, "Failed to read file: %s", filename);
+        goto fail;
+    }
+    if (e_ident[0] != ELFMAG0 ||
+        e_ident[1] != ELFMAG1 ||
+        e_ident[2] != ELFMAG2 ||
+        e_ident[3] != ELFMAG3) {
+        error_setg(errp, "Bad ELF magic");
+        goto fail;
+    }
+
+    is64l = e_ident[EI_CLASS] == ELFCLASS64;
+    hdr_size = is64l ? sizeof(Elf64_Ehdr) : sizeof(Elf32_Ehdr);
+    if (is64) {
+        *is64 = is64l;
+    }
+
+    off = EI_NIDENT;
+    while (hdr != e_ident_local && off < hdr_size) {
+        size_t br = read(fd, hdr + off, hdr_size - off);
+        switch (br) {
+        case 0:
+            error_setg(errp, "File too short: %s", filename);
+            goto fail;
+        case -1:
+            error_setg_errno(errp, errno, "Failed to read file: %s",
+                             filename);
+            goto fail;
+        }
+        off += br;
+    }
+
+fail:
+    close(fd);
+}
+
 /* return < 0 if error, otherwise the number of bytes loaded in memory */
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
-             uint64_t *highaddr, int big_endian, int elf_machine, int clear_lsb)
+             uint64_t *highaddr, int big_endian, int elf_machine,
+             int clear_lsb, int data_swab)
 {
     int fd, data_order, target_data_order, must_swab, ret = ELF_LOAD_FAILED;
     uint8_t e_ident[EI_NIDENT];
@@ -375,10 +454,12 @@ int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
     lseek(fd, 0, SEEK_SET);
     if (e_ident[EI_CLASS] == ELFCLASS64) {
         ret = load_elf64(filename, fd, translate_fn, translate_opaque, must_swab,
-                         pentry, lowaddr, highaddr, elf_machine, clear_lsb);
+                         pentry, lowaddr, highaddr, elf_machine, clear_lsb,
+                         data_swab);
     } else {
         ret = load_elf32(filename, fd, translate_fn, translate_opaque, must_swab,
-                         pentry, lowaddr, highaddr, elf_machine, clear_lsb);
+                         pentry, lowaddr, highaddr, elf_machine, clear_lsb,
+                         data_swab);
     }
 
  fail:
@@ -594,8 +675,7 @@ static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr,
     ret = hdr->ih_size;
 
 out:
-    if (data)
-        g_free(data);
+    g_free(data);
     close(fd);
     return ret;
 }
@@ -741,7 +821,7 @@ static void *rom_set_mr(Rom *rom, Object *owner, const char *name)
     memory_region_init_resizeable_ram(rom->mr, owner, name,
                                       rom->datasize, rom->romsize,
                                       fw_cfg_resized,
-                                      &error_abort);
+                                      &error_fatal);
     memory_region_set_readonly(rom->mr, true);
     vmstate_register_ram_global(rom->mr);
 
@@ -753,8 +833,9 @@ static void *rom_set_mr(Rom *rom, Object *owner, const char *name)
 
 int rom_add_file(const char *file, const char *fw_dir,
                  hwaddr addr, int32_t bootindex,
-                 bool option_rom)
+                 bool option_rom, MemoryRegion *mr)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
     Rom *rom;
     int rc, fd = -1;
     char devpath[100];
@@ -811,7 +892,7 @@ int rom_add_file(const char *file, const char *fw_dir,
                  basename);
         snprintf(devpath, sizeof(devpath), "/rom@%s", fw_file_name);
 
-        if ((!option_rom || option_rom_has_mr) && rom_file_has_mr) {
+        if ((!option_rom || mc->option_rom_has_mr) && mc->rom_file_has_mr) {
             data = rom_set_mr(rom, OBJECT(fw_cfg), devpath);
         } else {
             data = rom->data;
@@ -819,7 +900,12 @@ int rom_add_file(const char *file, const char *fw_dir,
 
         fw_cfg_add_file(fw_cfg, fw_file_name, data, rom->romsize);
     } else {
-        snprintf(devpath, sizeof(devpath), "/rom@" TARGET_FMT_plx, addr);
+        if (mr) {
+            rom->mr = mr;
+            snprintf(devpath, sizeof(devpath), "/rom@%s", file);
+        } else {
+            snprintf(devpath, sizeof(devpath), "/rom@" TARGET_FMT_plx, addr);
+        }
     }
 
     add_boot_device_path(bootindex, NULL, devpath);
@@ -839,6 +925,7 @@ MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len,
                    size_t max_len, hwaddr addr, const char *fw_file_name,
                    FWCfgReadCallback fw_callback, void *callback_opaque)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
     Rom *rom;
     MemoryRegion *mr = NULL;
 
@@ -856,7 +943,7 @@ MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len,
 
         snprintf(devpath, sizeof(devpath), "/rom@%s", fw_file_name);
 
-        if (rom_file_has_mr) {
+        if (mc->rom_file_has_mr) {
             data = rom_set_mr(rom, OBJECT(fw_cfg), devpath);
             mr = rom->mr;
         } else {
@@ -892,12 +979,12 @@ int rom_add_elf_program(const char *name, void *data, size_t datasize,
 
 int rom_add_vga(const char *file)
 {
-    return rom_add_file(file, "vgaroms", 0, -1, true);
+    return rom_add_file(file, "vgaroms", 0, -1, true, NULL);
 }
 
 int rom_add_option(const char *file, int32_t bootindex)
 {
-    return rom_add_file(file, "genroms", 0, bootindex, true);
+    return rom_add_file(file, "genroms", 0, bootindex, true, NULL);
 }
 
 static void rom_reset(void *unused)
@@ -966,6 +1053,20 @@ void rom_set_fw(FWCfgState *f)
     fw_cfg = f;
 }
 
+void rom_set_order_override(int order)
+{
+    if (!fw_cfg)
+        return;
+    fw_cfg_set_order_override(fw_cfg, order);
+}
+
+void rom_reset_order_override(void)
+{
+    if (!fw_cfg)
+        return;
+    fw_cfg_reset_order_override(fw_cfg);
+}
+
 static Rom *find_rom(hwaddr addr)
 {
     Rom *rom;
diff --git a/qemu/hw/core/machine.c b/qemu/hw/core/machine.c
index ac4654e9d..6dbbc85b9 100644
--- a/qemu/hw/core/machine.c
+++ b/qemu/hw/core/machine.c
@@ -10,11 +10,15 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "hw/boards.h"
+#include "qapi/error.h"
+#include "qapi-visit.h"
 #include "qapi/visitor.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 static char *machine_get_accel(Object *obj, Error **errp)
 {
@@ -31,33 +35,60 @@ static void machine_set_accel(Object *obj, const char *value, Error **errp)
     ms->accel = g_strdup(value);
 }
 
-static void machine_set_kernel_irqchip(Object *obj, bool value, Error **errp)
+static void machine_set_kernel_irqchip(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
 {
+    Error *err = NULL;
     MachineState *ms = MACHINE(obj);
+    OnOffSplit mode;
 
-    ms->kernel_irqchip_allowed = value;
-    ms->kernel_irqchip_required = value;
+    visit_type_OnOffSplit(v, name, &mode, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    } else {
+        switch (mode) {
+        case ON_OFF_SPLIT_ON:
+            ms->kernel_irqchip_allowed = true;
+            ms->kernel_irqchip_required = true;
+            ms->kernel_irqchip_split = false;
+            break;
+        case ON_OFF_SPLIT_OFF:
+            ms->kernel_irqchip_allowed = false;
+            ms->kernel_irqchip_required = false;
+            ms->kernel_irqchip_split = false;
+            break;
+        case ON_OFF_SPLIT_SPLIT:
+            ms->kernel_irqchip_allowed = true;
+            ms->kernel_irqchip_required = true;
+            ms->kernel_irqchip_split = true;
+            break;
+        default:
+            abort();
+        }
+    }
 }
 
 static void machine_get_kvm_shadow_mem(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
+                                       const char *name, void *opaque,
                                        Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     int64_t value = ms->kvm_shadow_mem;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void machine_set_kvm_shadow_mem(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
+                                       const char *name, void *opaque,
                                        Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -142,24 +173,24 @@ static void machine_set_dumpdtb(Object *obj, const char *value, Error **errp)
 }
 
 static void machine_get_phandle_start(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
-                                       Error **errp)
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     int64_t value = ms->phandle_start;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void machine_set_phandle_start(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
-                                       Error **errp)
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -226,6 +257,20 @@ static void machine_set_usb(Object *obj, bool value, Error **errp)
     ms->usb_disabled = !value;
 }
 
+static bool machine_get_igd_gfx_passthru(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->igd_gfx_passthru;
+}
+
+static void machine_set_igd_gfx_passthru(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->igd_gfx_passthru = value;
+}
+
 static char *machine_get_firmware(Object *obj, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
@@ -269,6 +314,21 @@ static bool machine_get_suppress_vmdesc(Object *obj, Error **errp)
     return ms->suppress_vmdesc;
 }
 
+static void machine_set_enforce_config_section(Object *obj, bool value,
+                                             Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->enforce_config_section = value;
+}
+
+static bool machine_get_enforce_config_section(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->enforce_config_section;
+}
+
 static int error_on_sysbus_device(SysBusDevice *sbdev, void *opaque)
 {
     error_report("Option '-device %s' cannot be handled by this machine",
@@ -300,6 +360,18 @@ static void machine_class_init(ObjectClass *oc, void *data)
 
     /* Default 128 MB as guest ram size */
     mc->default_ram_size = 128 * M_BYTE;
+    mc->rom_file_has_mr = true;
+}
+
+static void machine_class_base_init(ObjectClass *oc, void *data)
+{
+    if (!object_class_is_abstract(oc)) {
+        MachineClass *mc = MACHINE_CLASS(oc);
+        const char *cname = object_class_get_name(oc);
+        assert(g_str_has_suffix(cname, TYPE_MACHINE_SUFFIX));
+        mc->name = g_strndup(cname,
+                            strlen(cname) - strlen(TYPE_MACHINE_SUFFIX));
+    }
 }
 
 static void machine_initfn(Object *obj)
@@ -316,12 +388,12 @@ static void machine_initfn(Object *obj)
     object_property_set_description(obj, "accel",
                                     "Accelerator list",
                                     NULL);
-    object_property_add_bool(obj, "kernel-irqchip",
-                             NULL,
-                             machine_set_kernel_irqchip,
-                             NULL);
+    object_property_add(obj, "kernel-irqchip", "OnOffSplit",
+                        NULL,
+                        machine_set_kernel_irqchip,
+                        NULL, NULL, NULL);
     object_property_set_description(obj, "kernel-irqchip",
-                                    "Use KVM in-kernel irqchip",
+                                    "Configure KVM in-kernel irqchip",
                                     NULL);
     object_property_add(obj, "kvm-shadow-mem", "int",
                         machine_get_kvm_shadow_mem,
@@ -388,6 +460,12 @@ static void machine_initfn(Object *obj)
     object_property_set_description(obj, "usb",
                                     "Set on/off to enable/disable usb",
                                     NULL);
+    object_property_add_bool(obj, "igd-passthru",
+                             machine_get_igd_gfx_passthru,
+                             machine_set_igd_gfx_passthru, NULL);
+    object_property_set_description(obj, "igd-passthru",
+                                    "Set on/off to enable/disable igd passthrou",
+                                    NULL);
     object_property_add_str(obj, "firmware",
                             machine_get_firmware,
                             machine_set_firmware, NULL);
@@ -406,6 +484,12 @@ static void machine_initfn(Object *obj)
     object_property_set_description(obj, "suppress-vmdesc",
                                     "Set on to disable self-describing migration",
                                     NULL);
+    object_property_add_bool(obj, "enforce-config-section",
+                             machine_get_enforce_config_section,
+                             machine_set_enforce_config_section, NULL);
+    object_property_set_description(obj, "enforce-config-section",
+                                    "Set on to enforce configuration section migration",
+                                    NULL);
 
     /* Register notifier when init is done for sysbus sanity checks */
     ms->sysbus_notifier.notify = machine_init_notify;
@@ -431,11 +515,6 @@ bool machine_usb(MachineState *machine)
     return machine->usb;
 }
 
-bool machine_iommu(MachineState *machine)
-{
-    return machine->iommu;
-}
-
 bool machine_kernel_irqchip_allowed(MachineState *machine)
 {
     return machine->kernel_irqchip_allowed;
@@ -446,6 +525,11 @@ bool machine_kernel_irqchip_required(MachineState *machine)
     return machine->kernel_irqchip_required;
 }
 
+bool machine_kernel_irqchip_split(MachineState *machine)
+{
+    return machine->kernel_irqchip_split;
+}
+
 int machine_kvm_shadow_mem(MachineState *machine)
 {
     return machine->kvm_shadow_mem;
@@ -472,6 +556,7 @@ static const TypeInfo machine_info = {
     .abstract = true,
     .class_size = sizeof(MachineClass),
     .class_init    = machine_class_init,
+    .class_base_init = machine_class_base_init,
     .instance_size = sizeof(MachineState),
     .instance_init = machine_initfn,
     .instance_finalize = machine_finalize,
diff --git a/qemu/hw/core/nmi.c b/qemu/hw/core/nmi.c
index de1d1f8cb..e8bcc4177 100644
--- a/qemu/hw/core/nmi.c
+++ b/qemu/hw/core/nmi.c
@@ -19,13 +19,15 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/nmi.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "monitor/monitor.h"
 
 struct do_nmi_s {
     int cpu_index;
-    Error *errp;
+    Error *err;
     bool handled;
 };
 
@@ -40,8 +42,8 @@ static int do_nmi(Object *o, void *opaque)
         NMIClass *nc = NMI_GET_CLASS(n);
 
         ns->handled = true;
-        nc->nmi_monitor_handler(n, ns->cpu_index, &ns->errp);
-        if (ns->errp) {
+        nc->nmi_monitor_handler(n, ns->cpu_index, &ns->err);
+        if (ns->err) {
             return -1;
         }
     }
@@ -59,13 +61,13 @@ void nmi_monitor_handle(int cpu_index, Error **errp)
 {
     struct do_nmi_s ns = {
         .cpu_index = cpu_index,
-        .errp = NULL,
+        .err = NULL,
         .handled = false
     };
 
     nmi_children(object_get_root(), &ns);
     if (ns.handled) {
-        error_propagate(errp, ns.errp);
+        error_propagate(errp, ns.err);
     } else {
         error_setg(errp, QERR_UNSUPPORTED);
     }
diff --git a/qemu/hw/core/null-machine.c b/qemu/hw/core/null-machine.c
index 1ec7c3bbe..0351ba782 100644
--- a/qemu/hw/core/null-machine.c
+++ b/qemu/hw/core/null-machine.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
@@ -19,17 +20,11 @@ static void machine_none_init(MachineState *machine)
 {
 }
 
-static QEMUMachine machine_none = {
-    .name = "none",
-    .desc = "empty machine",
-    .init = machine_none_init,
-    .max_cpus = 0,
-};
-
-static void register_machines(void)
+static void machine_none_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&machine_none);
+    mc->desc = "empty machine";
+    mc->init = machine_none_init;
+    mc->max_cpus = 0;
 }
 
-machine_init(register_machines);
-
+DEFINE_MACHINE("none", machine_none_machine_init)
diff --git a/qemu/hw/core/platform-bus.c b/qemu/hw/core/platform-bus.c
index 70e051890..36f84ab72 100644
--- a/qemu/hw/core/platform-bus.c
+++ b/qemu/hw/core/platform-bus.c
@@ -19,8 +19,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/platform-bus.h"
 #include "exec/address-spaces.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 
 
@@ -106,31 +108,29 @@ static void plaform_bus_refresh_irqs(PlatformBusDevice *pbus)
     pbus->done_gathering = true;
 }
 
-static int platform_bus_map_irq(PlatformBusDevice *pbus, SysBusDevice *sbdev,
-                                int n)
+static void platform_bus_map_irq(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                 int n)
 {
     int max_irqs = pbus->num_irqs;
     int irqn;
 
     if (sysbus_is_irq_connected(sbdev, n)) {
         /* IRQ is already mapped, nothing to do */
-        return 0;
+        return;
     }
 
     irqn = find_first_zero_bit(pbus->used_irqs, max_irqs);
     if (irqn >= max_irqs) {
-        hw_error("Platform Bus: Can not fit IRQ line");
-        return -1;
+        error_report("Platform Bus: Can not fit IRQ line");
+        exit(1);
     }
 
     set_bit(irqn, pbus->used_irqs);
     sysbus_connect_irq(sbdev, n, pbus->irqs[irqn]);
-
-    return 0;
 }
 
-static int platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
-                                 int n)
+static void platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                  int n)
 {
     MemoryRegion *sbdev_mr = sysbus_mmio_get_region(sbdev, n);
     uint64_t size = memory_region_size(sbdev_mr);
@@ -140,7 +140,7 @@ static int platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
 
     if (memory_region_is_mapped(sbdev_mr)) {
         /* Region is already mapped, nothing to do */
-        return 0;
+        return;
     }
 
     /*
@@ -155,13 +155,13 @@ static int platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
     }
 
     if (!found_region) {
-        hw_error("Platform Bus: Can not fit MMIO region of size %"PRIx64, size);
+        error_report("Platform Bus: Can not fit MMIO region of size %"PRIx64,
+                     size);
+        exit(1);
     }
 
     /* Map the device's region into our Platform Bus MMIO space */
     memory_region_add_subregion(&pbus->mmio, off, sbdev_mr);
-
-    return 0;
 }
 
 /*
diff --git a/qemu/hw/core/ptimer.c b/qemu/hw/core/ptimer.c
index 8437bd6e8..153c83513 100644
--- a/qemu/hw/core/ptimer.c
+++ b/qemu/hw/core/ptimer.c
@@ -5,10 +5,12 @@
  *
  * This code is licensed under the GNU LGPL.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
 #include "qemu/host-utils.h"
+#include "sysemu/replay.h"
 
 struct ptimer_state
 {
@@ -27,7 +29,7 @@ struct ptimer_state
 static void ptimer_trigger(ptimer_state *s)
 {
     if (s->bh) {
-        qemu_bh_schedule(s->bh);
+        replay_bh_schedule_event(s->bh);
     }
 }
 
diff --git a/qemu/hw/core/qdev-properties-system.c b/qemu/hw/core/qdev-properties-system.c
index 921e799db..891219ae0 100644
--- a/qemu/hw/core/qdev-properties-system.c
+++ b/qemu/hw/core/qdev-properties-system.c
@@ -10,8 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
@@ -30,7 +32,7 @@ static void get_pointer(Object *obj, Visitor *v, Property *prop,
     char *p;
 
     p = *ptr ? print(*ptr) : g_strdup("");
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
     g_free(p);
 }
 
@@ -50,7 +52,7 @@ static void set_pointer(Object *obj, Visitor *v, Property *prop,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -111,14 +113,14 @@ static char *print_drive(void *ptr)
     return g_strdup(blk_name(ptr));
 }
 
-static void get_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_drive(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     get_pointer(obj, v, opaque, print_drive, name, errp);
 }
 
-static void set_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_drive(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     set_pointer(obj, v, opaque, parse_drive, name, errp);
 }
@@ -172,14 +174,14 @@ static char *print_chr(void *ptr)
     return g_strdup(val);
 }
 
-static void get_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void get_chr(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     get_pointer(obj, v, opaque, print_chr, name, errp);
 }
 
-static void set_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void set_chr(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     set_pointer(obj, v, opaque, parse_chr, name, errp);
 }
@@ -193,20 +195,20 @@ PropertyInfo qdev_prop_chr = {
 };
 
 /* --- netdev device --- */
-static void get_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_netdev(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     NICPeers *peers_ptr = qdev_get_prop_ptr(dev, prop);
     char *p = g_strdup(peers_ptr->ncs[0] ? peers_ptr->ncs[0]->name : "");
 
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
     g_free(p);
 }
 
-static void set_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_netdev(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -222,7 +224,7 @@ static void set_netdev(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -292,8 +294,8 @@ static int print_vlan(DeviceState *dev, Property *prop, char *dest, size_t len)
     return snprintf(dest, len, "<null>");
 }
 
-static void get_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_vlan(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -307,11 +309,11 @@ static void get_vlan(Object *obj, Visitor *v, void *opaque,
         }
     }
 
-    visit_type_int32(v, &id, name, errp);
+    visit_type_int32(v, name, &id, errp);
 }
 
-static void set_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_vlan(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -326,7 +328,7 @@ static void set_vlan(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_int32(v, &id, name, &local_err);
+    visit_type_int32(v, name, &id, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -364,18 +366,6 @@ void qdev_prop_set_drive(DeviceState *dev, const char *name,
                             name, errp);
 }
 
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
-                                BlockBackend *value)
-{
-    Error *err = NULL;
-
-    qdev_prop_set_drive(dev, name, value, &err);
-    if (err) {
-        error_report_err(err);
-        exit(1);
-    }
-}
-
 void qdev_prop_set_chr(DeviceState *dev, const char *name,
                        CharDriverState *value)
 {
diff --git a/qemu/hw/core/qdev-properties.c b/qemu/hw/core/qdev-properties.c
index 04fd80a4d..737d29c63 100644
--- a/qemu/hw/core/qdev-properties.c
+++ b/qemu/hw/core/qdev-properties.c
@@ -1,5 +1,8 @@
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
+#include "hw/pci/pci.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
@@ -41,19 +44,18 @@ void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
     return ptr;
 }
 
-static void get_enum(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_enum(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     int *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_enum(v, ptr, prop->info->enum_table,
-                    prop->info->name, prop->name, errp);
+    visit_type_enum(v, prop->name, ptr, prop->info->enum_table, errp);
 }
 
-static void set_enum(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_enum(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -64,8 +66,7 @@ static void set_enum(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_enum(v, ptr, prop->info->enum_table,
-                    prop->info->name, prop->name, errp);
+    visit_type_enum(v, prop->name, ptr, prop->info->enum_table, errp);
 }
 
 /* Bit */
@@ -87,19 +88,19 @@ static void bit_prop_set(DeviceState *dev, Property *props, bool val)
     }
 }
 
-static void prop_get_bit(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void prop_get_bit(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint32_t *p = qdev_get_prop_ptr(dev, prop);
     bool value = (*p & qdev_get_prop_mask(prop)) != 0;
 
-    visit_type_bool(v, &value, name, errp);
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void prop_set_bit(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void prop_set_bit(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -111,7 +112,7 @@ static void prop_set_bit(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -145,19 +146,19 @@ static void bit64_prop_set(DeviceState *dev, Property *props, bool val)
     }
 }
 
-static void prop_get_bit64(Object *obj, Visitor *v, void *opaque,
-                           const char *name, Error **errp)
+static void prop_get_bit64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *p = qdev_get_prop_ptr(dev, prop);
     bool value = (*p & qdev_get_prop_mask64(prop)) != 0;
 
-    visit_type_bool(v, &value, name, errp);
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void prop_set_bit64(Object *obj, Visitor *v, void *opaque,
-                           const char *name, Error **errp)
+static void prop_set_bit64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -169,7 +170,7 @@ static void prop_set_bit64(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -186,18 +187,18 @@ PropertyInfo qdev_prop_bit64 = {
 
 /* --- bool --- */
 
-static void get_bool(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_bool(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     bool *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_bool(v, ptr, name, errp);
+    visit_type_bool(v, name, ptr, errp);
 }
 
-static void set_bool(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_bool(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -208,7 +209,7 @@ static void set_bool(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_bool(v, ptr, name, errp);
+    visit_type_bool(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_bool = {
@@ -219,18 +220,18 @@ PropertyInfo qdev_prop_bool = {
 
 /* --- 8bit integer --- */
 
-static void get_uint8(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_uint8(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint8_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint8(v, ptr, name, errp);
+    visit_type_uint8(v, name, ptr, errp);
 }
 
-static void set_uint8(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_uint8(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -241,7 +242,7 @@ static void set_uint8(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_uint8(v, ptr, name, errp);
+    visit_type_uint8(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint8 = {
@@ -252,18 +253,18 @@ PropertyInfo qdev_prop_uint8 = {
 
 /* --- 16bit integer --- */
 
-static void get_uint16(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint16(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint16_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint16(v, ptr, name, errp);
+    visit_type_uint16(v, name, ptr, errp);
 }
 
-static void set_uint16(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint16(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -274,7 +275,7 @@ static void set_uint16(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_uint16(v, ptr, name, errp);
+    visit_type_uint16(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint16 = {
@@ -285,18 +286,18 @@ PropertyInfo qdev_prop_uint16 = {
 
 /* --- 32bit integer --- */
 
-static void get_uint32(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint32(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint32(v, ptr, name, errp);
+    visit_type_uint32(v, name, ptr, errp);
 }
 
-static void set_uint32(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint32(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -307,21 +308,21 @@ static void set_uint32(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_uint32(v, ptr, name, errp);
+    visit_type_uint32(v, name, ptr, errp);
 }
 
-static void get_int32(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_int32(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_int32(v, ptr, name, errp);
+    visit_type_int32(v, name, ptr, errp);
 }
 
-static void set_int32(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_int32(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -332,7 +333,7 @@ static void set_int32(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_int32(v, ptr, name, errp);
+    visit_type_int32(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint32 = {
@@ -349,18 +350,18 @@ PropertyInfo qdev_prop_int32 = {
 
 /* --- 64bit integer --- */
 
-static void get_uint64(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint64(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint64(v, ptr, name, errp);
+    visit_type_uint64(v, name, ptr, errp);
 }
 
-static void set_uint64(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint64(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -371,7 +372,7 @@ static void set_uint64(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_uint64(v, ptr, name, errp);
+    visit_type_uint64(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint64 = {
@@ -388,8 +389,8 @@ static void release_string(Object *obj, const char *name, void *opaque)
     g_free(*(char **)qdev_get_prop_ptr(DEVICE(obj), prop));
 }
 
-static void get_string(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_string(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -397,14 +398,14 @@ static void get_string(Object *obj, Visitor *v, void *opaque,
 
     if (!*ptr) {
         char *str = (char *)"";
-        visit_type_str(v, &str, name, errp);
+        visit_type_str(v, name, &str, errp);
     } else {
-        visit_type_str(v, ptr, name, errp);
+        visit_type_str(v, name, ptr, errp);
     }
 }
 
-static void set_string(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_string(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -417,14 +418,12 @@ static void set_string(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
-    if (*ptr) {
-        g_free(*ptr);
-    }
+    g_free(*ptr);
     *ptr = str;
 }
 
@@ -449,8 +448,8 @@ PropertyInfo qdev_prop_ptr = {
  *   01:02:03:04:05:06
  *   01-02-03-04-05-06
  */
-static void get_mac(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void get_mac(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -462,11 +461,11 @@ static void get_mac(Object *obj, Visitor *v, void *opaque,
              mac->a[0], mac->a[1], mac->a[2],
              mac->a[3], mac->a[4], mac->a[5]);
 
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
 }
 
-static void set_mac(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void set_mac(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -480,7 +479,7 @@ static void set_mac(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -519,6 +518,16 @@ PropertyInfo qdev_prop_macaddr = {
     .set   = set_mac,
 };
 
+/* --- on/off/auto --- */
+
+PropertyInfo qdev_prop_on_off_auto = {
+    .name = "OnOffAuto",
+    .description = "on/off/auto",
+    .enum_table = OnOffAuto_lookup,
+    .get = get_enum,
+    .set = set_enum,
+};
+
 /* --- lost tick policy --- */
 
 QEMU_BUILD_BUG_ON(sizeof(LostTickPolicy) != sizeof(int));
@@ -543,13 +552,24 @@ PropertyInfo qdev_prop_bios_chs_trans = {
     .set = set_enum,
 };
 
+/* --- FDC default drive types */
+
+PropertyInfo qdev_prop_fdc_drive_type = {
+    .name = "FdcDriveType",
+    .description = "FDC drive type, "
+                   "144/288/120/none/auto",
+    .enum_table = FloppyDriveType_lookup,
+    .get = get_enum,
+    .set = set_enum
+};
+
 /* --- pci address --- */
 
 /*
  * bus-local address, i.e. "$slot" or "$slot.$fn"
  */
-static void set_pci_devfn(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void set_pci_devfn(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -563,11 +583,11 @@ static void set_pci_devfn(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_free(local_err);
         local_err = NULL;
-        visit_type_int32(v, &value, name, &local_err);
+        visit_type_int32(v, name, &value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
         } else if (value < -1 || value > 255) {
@@ -619,8 +639,8 @@ PropertyInfo qdev_prop_pci_devfn = {
 
 /* --- blocksize --- */
 
-static void set_blocksize(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void set_blocksize(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -634,7 +654,7 @@ static void set_blocksize(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_uint16(v, &value, name, &local_err);
+    visit_type_uint16(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -666,8 +686,8 @@ PropertyInfo qdev_prop_blocksize = {
 
 /* --- pci host address --- */
 
-static void get_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void get_pci_host_devaddr(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -680,15 +700,15 @@ static void get_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
                   addr->domain, addr->bus, addr->slot, addr->function);
     assert(rc == sizeof(buffer) - 1);
 
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
 }
 
 /*
  * Parse [<domain>:]<bus>:<slot>.<func>
  *   if <domain> is not supplied, it's assumed to be 0.
  */
-static void set_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void set_pci_host_devaddr(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -705,7 +725,7 @@ static void set_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -801,8 +821,8 @@ static void array_element_release(Object *obj, const char *name, void *opaque)
     g_free(p);
 }
 
-static void set_prop_arraylen(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void set_prop_arraylen(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     /* Setter for the property which defines the length of a
      * variable-sized property array. As well as actually setting the
@@ -827,7 +847,7 @@ static void set_prop_arraylen(Object *obj, Visitor *v, void *opaque,
                    name);
         return;
     }
-    visit_type_uint32(v, alenptr, name, &local_err);
+    visit_type_uint32(v, name, alenptr, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1065,10 +1085,8 @@ static void qdev_prop_set_globals_for_type(DeviceState *dev,
         object_property_parse(OBJECT(dev), prop->value, prop->property, &err);
         if (err != NULL) {
             assert(prop->user_provided);
-            error_report("Warning: global %s.%s=%s ignored (%s)",
-                         prop->driver, prop->property, prop->value,
-                         error_get_pretty(err));
-            error_free(err);
+            error_reportf_err(err, "Warning: global %s.%s=%s ignored: ",
+                              prop->driver, prop->property, prop->value);
             return;
         }
     }
@@ -1086,24 +1104,24 @@ void qdev_prop_set_globals(DeviceState *dev)
 
 /* --- 64bit unsigned int 'size' type --- */
 
-static void get_size(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_size(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_size(v, ptr, name, errp);
+    visit_type_size(v, name, ptr, errp);
 }
 
-static void set_size(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_size(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_size(v, ptr, name, errp);
+    visit_type_size(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_size = {
diff --git a/qemu/hw/core/qdev.c b/qemu/hw/core/qdev.c
index b2f404a76..db41aa1f2 100644
--- a/qemu/hw/core/qdev.c
+++ b/qemu/hw/core/qdev.c
@@ -25,10 +25,10 @@
    inherit from a particular bus (e.g. PCI or I2C) rather than
    this API directly.  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "hw/fw-path-provider.h"
 #include "sysemu/sysemu.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/visitor.h"
 #include "qapi/qmp/qjson.h"
@@ -325,6 +325,11 @@ void qdev_reset_all(DeviceState *dev)
     qdev_walk_children(dev, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
 }
 
+void qdev_reset_all_fn(void *opaque)
+{
+    qdev_reset_all(DEVICE(opaque));
+}
+
 void qbus_reset_all(BusState *bus)
 {
     qbus_walk_children(bus, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
@@ -365,9 +370,8 @@ void qdev_init_nofail(DeviceState *dev)
 
     object_property_set_bool(OBJECT(dev), true, "realized", &err);
     if (err) {
-        error_report("Initialization of device %s failed: %s",
-                     object_get_typename(OBJECT(dev)),
-                     error_get_pretty(err));
+        error_reportf_err(err, "Initialization of device %s failed: ",
+                          object_get_typename(OBJECT(dev)));
         exit(1);
     }
 }
@@ -417,17 +421,21 @@ void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-in");
 
     assert(gpio_list->num_out == 0 || !name);
     gpio_list->in = qemu_extend_irqs(gpio_list->in, gpio_list->num_in, handler,
                                      dev, n);
 
+    if (!name) {
+        name = "unnamed-gpio-in";
+    }
     for (i = gpio_list->num_in; i < gpio_list->num_in + n; i++) {
+        gchar *propname = g_strdup_printf("%s[%u]", name, i);
+
         object_property_add_child(OBJECT(dev), propname,
                                   OBJECT(gpio_list->in[i]), &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
 
     gpio_list->num_in += n;
 }
@@ -442,20 +450,25 @@ void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-out");
 
     assert(gpio_list->num_in == 0 || !name);
-    gpio_list->num_out += n;
 
+    if (!name) {
+        name = "unnamed-gpio-out";
+    }
+    memset(pins, 0, sizeof(*pins) * n);
     for (i = 0; i < n; ++i) {
-        memset(&pins[i], 0, sizeof(*pins));
+        gchar *propname = g_strdup_printf("%s[%u]", name,
+                                          gpio_list->num_out + i);
+
         object_property_add_link(OBJECT(dev), propname, TYPE_IRQ,
                                  (Object **)&pins[i],
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE,
                                  &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
+    gpio_list->num_out += n;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -506,7 +519,7 @@ qemu_irq qdev_get_gpio_out_connector(DeviceState *dev, const char *name, int n)
     return ret;
 }
 
-/* disconnect a GPIO ouput, returning the disconnected input (if any) */
+/* disconnect a GPIO output, returning the disconnected input (if any) */
 
 static qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev,
                                                const char *name, int n)
@@ -567,6 +580,12 @@ void qdev_pass_gpios(DeviceState *dev, DeviceState *container,
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
 {
     BusState *bus;
+    Object *child = object_resolve_path_component(OBJECT(dev), name);
+
+    bus = (BusState *)object_dynamic_cast(child, TYPE_BUS);
+    if (bus) {
+        return bus;
+    }
 
     QLIST_FOREACH(bus, &dev->child_bus, sibling) {
         if (strcmp(name, bus->name) == 0) {
@@ -874,8 +893,9 @@ char *qdev_get_dev_path(DeviceState *dev)
  * Legacy property handling
  */
 
-static void qdev_get_legacy_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void qdev_get_legacy_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -884,7 +904,7 @@ static void qdev_get_legacy_property(Object *obj, Visitor *v, void *opaque,
     char *ptr = buffer;
 
     prop->info->print(dev, prop, buffer, sizeof(buffer));
-    visit_type_str(v, &ptr, name, errp);
+    visit_type_str(v, name, &ptr, errp);
 }
 
 /**
@@ -1120,7 +1140,6 @@ post_realize_fail:
 
 fail:
     error_propagate(errp, local_err);
-    return;
 }
 
 static bool device_get_hotpluggable(Object *obj, Error **errp)
@@ -1194,7 +1213,6 @@ static void device_finalize(Object *obj)
     NamedGPIOList *ngl, *next;
 
     DeviceState *dev = DEVICE(obj);
-    qemu_opts_del(dev->opts);
 
     QLIST_FOREACH_SAFE(ngl, &dev->gpios, node, next) {
         QLIST_REMOVE(ngl, node);
@@ -1242,6 +1260,9 @@ static void device_unparent(Object *obj)
         qapi_event_send_device_deleted(!!dev->id, dev->id, path, &error_abort);
         g_free(path);
     }
+
+    qemu_opts_del(dev->opts);
+    dev->opts = NULL;
 }
 
 static void device_class_init(ObjectClass *class, void *data)
diff --git a/qemu/hw/core/stream.c b/qemu/hw/core/stream.c
index e6a05a543..4439ecdf0 100644
--- a/qemu/hw/core/stream.c
+++ b/qemu/hw/core/stream.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/stream.h"
 
 size_t
diff --git a/qemu/hw/core/sysbus.c b/qemu/hw/core/sysbus.c
index 3c5862989..a7dbe2b32 100644
--- a/qemu/hw/core/sysbus.c
+++ b/qemu/hw/core/sysbus.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "monitor/monitor.h"
 #include "exec/address-spaces.h"
diff --git a/qemu/hw/cpu/Makefile.objs b/qemu/hw/cpu/Makefile.objs
index 6381238cc..0954a1872 100644
--- a/qemu/hw/cpu/Makefile.objs
+++ b/qemu/hw/cpu/Makefile.objs
@@ -2,5 +2,4 @@ obj-$(CONFIG_ARM11MPCORE) += arm11mpcore.o
 obj-$(CONFIG_REALVIEW) += realview_mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
-obj-$(CONFIG_ICC_BUS) += icc_bus.o
 
diff --git a/qemu/hw/cpu/a15mpcore.c b/qemu/hw/cpu/a15mpcore.c
index acc419e11..bc05152fd 100644
--- a/qemu/hw/cpu/a15mpcore.c
+++ b/qemu/hw/cpu/a15mpcore.c
@@ -18,8 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/a15mpcore.h"
 #include "sysemu/kvm.h"
+#include "kvm_arm.h"
 
 static void a15mp_priv_set_irq(void *opaque, int irq, int level)
 {
@@ -33,16 +36,11 @@ static void a15mp_priv_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     A15MPPrivState *s = A15MPCORE_PRIV(obj);
     DeviceState *gicdev;
-    const char *gictype = "arm_gic";
-
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
-    }
 
     memory_region_init(&s->container, obj, "a15mp-priv-container", 0x8000);
     sysbus_init_mmio(sbd, &s->container);
 
-    object_initialize(&s->gic, sizeof(s->gic), gictype);
+    object_initialize(&s->gic, sizeof(s->gic), gic_class_name());
     gicdev = DEVICE(&s->gic);
     qdev_set_parent_bus(gicdev, sysbus_get_default());
     qdev_prop_set_uint32(gicdev, "revision", 2);
@@ -56,10 +54,23 @@ static void a15mp_priv_realize(DeviceState *dev, Error **errp)
     SysBusDevice *busdev;
     int i;
     Error *err = NULL;
+    bool has_el3;
+    Object *cpuobj;
 
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    if (!kvm_irqchip_in_kernel()) {
+        /* Make the GIC's TZ support match the CPUs. We assume that
+         * either all the CPUs have TZ, or none do.
+         */
+        cpuobj = OBJECT(qemu_get_cpu(0));
+        has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+            object_property_get_bool(cpuobj, "has_el3", &error_abort);
+        qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+    }
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
@@ -79,20 +90,27 @@ static void a15mp_priv_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < s->num_cpu; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
         int ppibase = s->num_irq - 32 + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs used on the A15:
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = 30,
+            [GTIMER_VIRT] = 27,
+            [GTIMER_HYP]  = 26,
+            [GTIMER_SEC]  = 29,
+        };
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
     }
 
     /* Memory map (addresses are offsets from PERIPHBASE):
      *  0x0000-0x0fff -- reserved
      *  0x1000-0x1fff -- GIC Distributor
-     *  0x2000-0x2fff -- GIC CPU interface
+     *  0x2000-0x3fff -- GIC CPU interface
      *  0x4000-0x4fff -- GIC virtual interface control (not modelled)
      *  0x5000-0x5fff -- GIC virtual interface control (not modelled)
      *  0x6000-0x7fff -- GIC virtual CPU interface (not modelled)
diff --git a/qemu/hw/cpu/a9mpcore.c b/qemu/hw/cpu/a9mpcore.c
index c09358c6e..5459ae8c1 100644
--- a/qemu/hw/cpu/a9mpcore.c
+++ b/qemu/hw/cpu/a9mpcore.c
@@ -8,6 +8,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/a9mpcore.h"
 
 static void a9mp_priv_set_irq(void *opaque, int irq, int level)
@@ -49,6 +51,8 @@ static void a9mp_priv_realize(DeviceState *dev, Error **errp)
                  *wdtbusdev;
     Error *err = NULL;
     int i;
+    bool has_el3;
+    Object *cpuobj;
 
     scudev = DEVICE(&s->scu);
     qdev_prop_set_uint32(scudev, "num-cpu", s->num_cpu);
@@ -62,6 +66,15 @@ static void a9mp_priv_realize(DeviceState *dev, Error **errp)
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    /* Make the GIC's TZ support match the CPUs. We assume that
+     * either all the CPUs have TZ, or none do.
+     */
+    cpuobj = OBJECT(qemu_get_cpu(0));
+    has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+        object_property_get_bool(cpuobj, "has_el3", &error_abort);
+    qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
diff --git a/qemu/hw/cpu/arm11mpcore.c b/qemu/hw/cpu/arm11mpcore.c
index 717d3e4f8..eb244658b 100644
--- a/qemu/hw/cpu/arm11mpcore.c
+++ b/qemu/hw/cpu/arm11mpcore.c
@@ -7,6 +7,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/arm11mpcore.h"
 #include "hw/intc/realview_gic.h"
 
diff --git a/qemu/hw/cpu/icc_bus.c b/qemu/hw/cpu/icc_bus.c
deleted file mode 100644
index 6646ea2b3..000000000
--- a/qemu/hw/cpu/icc_bus.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* icc_bus.c
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#include "hw/cpu/icc_bus.h"
-#include "hw/sysbus.h"
-
-/* icc-bridge implementation */
-
-static const TypeInfo icc_bus_info = {
-    .name = TYPE_ICC_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(ICCBus),
-};
-
-
-/* icc-device implementation */
-
-static void icc_device_realize(DeviceState *dev, Error **errp)
-{
-    ICCDeviceClass *idc = ICC_DEVICE_GET_CLASS(dev);
-
-    /* convert to QOM */
-    if (idc->realize) {
-        idc->realize(dev, errp);
-    }
-
-}
-
-static void icc_device_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    dc->realize = icc_device_realize;
-    dc->bus_type = TYPE_ICC_BUS;
-}
-
-static const TypeInfo icc_device_info = {
-    .name = TYPE_ICC_DEVICE,
-    .parent = TYPE_DEVICE,
-    .abstract = true,
-    .instance_size = sizeof(ICCDevice),
-    .class_size = sizeof(ICCDeviceClass),
-    .class_init = icc_device_class_init,
-};
-
-
-/*  icc-bridge implementation */
-
-typedef struct ICCBridgeState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    ICCBus icc_bus;
-    MemoryRegion apic_container;
-} ICCBridgeState;
-
-#define ICC_BRIDGE(obj) OBJECT_CHECK(ICCBridgeState, (obj), TYPE_ICC_BRIDGE)
-
-static void icc_bridge_init(Object *obj)
-{
-    ICCBridgeState *s = ICC_BRIDGE(obj);
-    SysBusDevice *sb = SYS_BUS_DEVICE(obj);
-
-    qbus_create_inplace(&s->icc_bus, sizeof(s->icc_bus), TYPE_ICC_BUS,
-                        DEVICE(s), "icc");
-
-    /* Do not change order of registering regions,
-     * APIC must be first registered region, board maps it by 0 index
-     */
-    memory_region_init(&s->apic_container, obj, "icc-apic-container",
-                       APIC_SPACE_SIZE);
-    sysbus_init_mmio(sb, &s->apic_container);
-    s->icc_bus.apic_address_space = &s->apic_container;
-}
-
-static void icc_bridge_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-}
-
-static const TypeInfo icc_bridge_info = {
-    .name  = TYPE_ICC_BRIDGE,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_init  = icc_bridge_init,
-    .instance_size  = sizeof(ICCBridgeState),
-    .class_init = icc_bridge_class_init,
-};
-
-
-static void icc_bus_register_types(void)
-{
-    type_register_static(&icc_bus_info);
-    type_register_static(&icc_device_info);
-    type_register_static(&icc_bridge_info);
-}
-
-type_init(icc_bus_register_types)
diff --git a/qemu/hw/cpu/realview_mpcore.c b/qemu/hw/cpu/realview_mpcore.c
index c39a2da42..39d4ebeb1 100644
--- a/qemu/hw/cpu/realview_mpcore.c
+++ b/qemu/hw/cpu/realview_mpcore.c
@@ -8,6 +8,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/arm11mpcore.h"
 #include "hw/intc/realview_gic.h"
 
diff --git a/qemu/hw/cris/axis_dev88.c b/qemu/hw/cris/axis_dev88.c
index 3cae480fa..9f5865874 100644
--- a/qemu/hw/cris/axis_dev88.c
+++ b/qemu/hw/cris/axis_dev88.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/block/flash.h"
@@ -138,7 +142,7 @@ static void tempsensor_clkedge(struct tempsensor_t *s,
                     s->count = 16;
 
                     if ((s->regs[0] & 0xff) == 0) {
-                        /* 25 degrees celcius.  */
+                        /* 25 degrees celsius.  */
                         s->shiftreg = 0x0b9f;
                     } else if ((s->regs[0] & 0xff) == 0xff) {
                         /* Sensor ID, 0x8100 LM70.  */
@@ -277,7 +281,7 @@ void axisdev88_init(MachineState *machine)
     /* The ETRAX-FS has 128Kb on chip ram, the docs refer to it as the 
        internal memory.  */
     memory_region_init_ram(phys_intmem, NULL, "axisdev88.chipram", INTMEM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_intmem);
     memory_region_add_subregion(address_space_mem, 0x38000000, phys_intmem);
 
@@ -351,16 +355,11 @@ void axisdev88_init(MachineState *machine)
     }
 }
 
-static QEMUMachine axisdev88_machine = {
-    .name = "axis-dev88",
-    .desc = "AXIS devboard 88",
-    .init = axisdev88_init,
-    .is_default = 1,
-};
-
-static void axisdev88_machine_init(void)
+static void axisdev88_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&axisdev88_machine);
+    mc->desc = "AXIS devboard 88";
+    mc->init = axisdev88_init;
+    mc->is_default = 1;
 }
 
-machine_init(axisdev88_machine_init);
+DEFINE_MACHINE("axis-dev88", axisdev88_machine_init)
diff --git a/qemu/hw/cris/boot.c b/qemu/hw/cris/boot.c
index 622f353c9..f896ed7f8 100644
--- a/qemu/hw/cris/boot.c
+++ b/qemu/hw/cris/boot.c
@@ -22,10 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "boot.h"
+#include "qemu/cutils.h"
 
 static void main_cpu_reset(void *opaque)
 {
@@ -72,7 +76,7 @@ void cris_load_image(CRISCPU *cpu, struct cris_load_info *li)
     /* Boots a kernel elf binary, os/linux-2.6/vmlinux from the axis 
        devboard SDK.  */
     image_size = load_elf(li->image_filename, translate_kernel_address, NULL,
-                          &entry, NULL, &high, 0, ELF_MACHINE, 0);
+                          &entry, NULL, &high, 0, EM_CRIS, 0, 0);
     li->entry = entry;
     if (image_size < 0) {
         /* Takes a kimage from the axis devboard SDK.  */
diff --git a/qemu/hw/display/Makefile.objs b/qemu/hw/display/Makefile.objs
index dd8ea76d1..d99780eeb 100644
--- a/qemu/hw/display/Makefile.objs
+++ b/qemu/hw/display/Makefile.objs
@@ -27,6 +27,7 @@ endif
 obj-$(CONFIG_OMAP) += omap_dss.o
 obj-$(CONFIG_OMAP) += omap_lcdc.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_lcd.o
+obj-$(CONFIG_RASPI) += bcm2835_fb.o
 obj-$(CONFIG_SM501) += sm501.o
 obj-$(CONFIG_TCX) += tcx.o
 obj-$(CONFIG_CG3) += cg3.o
@@ -35,6 +36,10 @@ obj-$(CONFIG_VGA) += vga.o
 
 common-obj-$(CONFIG_QXL) += qxl.o qxl-logger.o qxl-render.o
 
-obj-$(CONFIG_VIRTIO) += virtio-gpu.o
+obj-$(CONFIG_VIRTIO) += virtio-gpu.o virtio-gpu-3d.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio-gpu-pci.o
 obj-$(CONFIG_VIRTIO_VGA) += virtio-vga.o
+virtio-gpu.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu.o-libs += $(VIRGL_LIBS)
+virtio-gpu-3d.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu-3d.o-libs += $(VIRGL_LIBS)
diff --git a/qemu/hw/display/ads7846.c b/qemu/hw/display/ads7846.c
index 3f35369bb..05aa2d1e6 100644
--- a/qemu/hw/display/ads7846.c
+++ b/qemu/hw/display/ads7846.c
@@ -10,7 +10,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 #include "ui/console.h"
 
 typedef struct {
diff --git a/qemu/hw/display/bcm2835_fb.c b/qemu/hw/display/bcm2835_fb.c
new file mode 100644
index 000000000..506f1d3d9
--- /dev/null
+++ b/qemu/hw/display/bcm2835_fb.c
@@ -0,0 +1,425 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Refactoring for Pi2 Copyright (c) 2015, Microsoft. Written by Andrew Baumann.
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * Heavily based on milkymist-vgafb.c, copyright terms below:
+ *  QEMU model of the Milkymist VGA framebuffer.
+ *
+ *  Copyright (c) 2010-2012 Michael Walle <michael@walle.cc>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/display/bcm2835_fb.h"
+#include "hw/display/framebuffer.h"
+#include "ui/pixel_ops.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+
+#define DEFAULT_VCRAM_SIZE 0x4000000
+#define BCM2835_FB_OFFSET  0x00100000
+
+static void fb_invalidate_display(void *opaque)
+{
+    BCM2835FBState *s = BCM2835_FB(opaque);
+
+    s->invalidate = true;
+}
+
+static void draw_line_src16(void *opaque, uint8_t *dst, const uint8_t *src,
+                            int width, int deststep)
+{
+    BCM2835FBState *s = opaque;
+    uint16_t rgb565;
+    uint32_t rgb888;
+    uint8_t r, g, b;
+    DisplaySurface *surface = qemu_console_surface(s->con);
+    int bpp = surface_bits_per_pixel(surface);
+
+    while (width--) {
+        switch (s->bpp) {
+        case 8:
+            /* lookup palette starting at video ram base
+             * TODO: cache translation, rather than doing this each time!
+             */
+            rgb888 = ldl_le_phys(&s->dma_as, s->vcram_base + (*src << 2));
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src++;
+            break;
+        case 16:
+            rgb565 = lduw_le_p(src);
+            r = ((rgb565 >> 11) & 0x1f) << 3;
+            g = ((rgb565 >>  5) & 0x3f) << 2;
+            b = ((rgb565 >>  0) & 0x1f) << 3;
+            src += 2;
+            break;
+        case 24:
+            rgb888 = ldl_le_p(src);
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src += 3;
+            break;
+        case 32:
+            rgb888 = ldl_le_p(src);
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src += 4;
+            break;
+        default:
+            r = 0;
+            g = 0;
+            b = 0;
+            break;
+        }
+
+        if (s->pixo == 0) {
+            /* swap to BGR pixel format */
+            uint8_t tmp = r;
+            r = b;
+            b = tmp;
+        }
+
+        switch (bpp) {
+        case 8:
+            *dst++ = rgb_to_pixel8(r, g, b);
+            break;
+        case 15:
+            *(uint16_t *)dst = rgb_to_pixel15(r, g, b);
+            dst += 2;
+            break;
+        case 16:
+            *(uint16_t *)dst = rgb_to_pixel16(r, g, b);
+            dst += 2;
+            break;
+        case 24:
+            rgb888 = rgb_to_pixel24(r, g, b);
+            *dst++ = rgb888 & 0xff;
+            *dst++ = (rgb888 >> 8) & 0xff;
+            *dst++ = (rgb888 >> 16) & 0xff;
+            break;
+        case 32:
+            *(uint32_t *)dst = rgb_to_pixel32(r, g, b);
+            dst += 4;
+            break;
+        default:
+            return;
+        }
+    }
+}
+
+static void fb_update_display(void *opaque)
+{
+    BCM2835FBState *s = opaque;
+    DisplaySurface *surface = qemu_console_surface(s->con);
+    int first = 0;
+    int last = 0;
+    int src_width = 0;
+    int dest_width = 0;
+
+    if (s->lock || !s->xres) {
+        return;
+    }
+
+    src_width = s->xres * (s->bpp >> 3);
+    dest_width = s->xres;
+
+    switch (surface_bits_per_pixel(surface)) {
+    case 0:
+        return;
+    case 8:
+        break;
+    case 15:
+        dest_width *= 2;
+        break;
+    case 16:
+        dest_width *= 2;
+        break;
+    case 24:
+        dest_width *= 3;
+        break;
+    case 32:
+        dest_width *= 4;
+        break;
+    default:
+        hw_error("bcm2835_fb: bad color depth\n");
+        break;
+    }
+
+    if (s->invalidate) {
+        framebuffer_update_memory_section(&s->fbsection, s->dma_mr, s->base,
+                                          s->yres, src_width);
+    }
+
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
+                               src_width, dest_width, 0, s->invalidate,
+                               draw_line_src16, s, &first, &last);
+
+    if (first >= 0) {
+        dpy_gfx_update(s->con, 0, first, s->xres, last - first + 1);
+    }
+
+    s->invalidate = false;
+}
+
+static void bcm2835_fb_mbox_push(BCM2835FBState *s, uint32_t value)
+{
+    value &= ~0xf;
+
+    s->lock = true;
+
+    s->xres = ldl_le_phys(&s->dma_as, value);
+    s->yres = ldl_le_phys(&s->dma_as, value + 4);
+    s->xres_virtual = ldl_le_phys(&s->dma_as, value + 8);
+    s->yres_virtual = ldl_le_phys(&s->dma_as, value + 12);
+    s->bpp = ldl_le_phys(&s->dma_as, value + 20);
+    s->xoffset = ldl_le_phys(&s->dma_as, value + 24);
+    s->yoffset = ldl_le_phys(&s->dma_as, value + 28);
+
+    s->base = s->vcram_base | (value & 0xc0000000);
+    s->base += BCM2835_FB_OFFSET;
+
+    /* TODO - Manage properly virtual resolution */
+
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    stl_le_phys(&s->dma_as, value + 16, s->pitch);
+    stl_le_phys(&s->dma_as, value + 32, s->base);
+    stl_le_phys(&s->dma_as, value + 36, s->size);
+
+    s->invalidate = true;
+    qemu_console_resize(s->con, s->xres, s->yres);
+    s->lock = false;
+}
+
+void bcm2835_fb_reconfigure(BCM2835FBState *s, uint32_t *xres, uint32_t *yres,
+                            uint32_t *xoffset, uint32_t *yoffset, uint32_t *bpp,
+                            uint32_t *pixo, uint32_t *alpha)
+{
+    s->lock = true;
+
+    /* TODO: input validation! */
+    if (xres) {
+        s->xres = *xres;
+    }
+    if (yres) {
+        s->yres = *yres;
+    }
+    if (xoffset) {
+        s->xoffset = *xoffset;
+    }
+    if (yoffset) {
+        s->yoffset = *yoffset;
+    }
+    if (bpp) {
+        s->bpp = *bpp;
+    }
+    if (pixo) {
+        s->pixo = *pixo;
+    }
+    if (alpha) {
+        s->alpha = *alpha;
+    }
+
+    /* TODO - Manage properly virtual resolution */
+
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    s->invalidate = true;
+    qemu_console_resize(s->con, s->xres, s->yres);
+    s->lock = false;
+}
+
+static uint64_t bcm2835_fb_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835FBState *s = opaque;
+    uint32_t res = 0;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        res = MBOX_CHAN_FB;
+        s->pending = false;
+        qemu_set_irq(s->mbox_irq, 0);
+        break;
+
+    case MBOX_AS_PENDING:
+        res = s->pending;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_fb_write(void *opaque, hwaddr offset, uint64_t value,
+                             unsigned size)
+{
+    BCM2835FBState *s = opaque;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        /* bcm2835_mbox should check our pending status before pushing */
+        assert(!s->pending);
+        s->pending = true;
+        bcm2835_fb_mbox_push(s, value);
+        qemu_set_irq(s->mbox_irq, 1);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+}
+
+static const MemoryRegionOps bcm2835_fb_ops = {
+    .read = bcm2835_fb_read,
+    .write = bcm2835_fb_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_fb = {
+    .name = TYPE_BCM2835_FB,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(lock, BCM2835FBState),
+        VMSTATE_BOOL(invalidate, BCM2835FBState),
+        VMSTATE_BOOL(pending, BCM2835FBState),
+        VMSTATE_UINT32(xres, BCM2835FBState),
+        VMSTATE_UINT32(yres, BCM2835FBState),
+        VMSTATE_UINT32(xres_virtual, BCM2835FBState),
+        VMSTATE_UINT32(yres_virtual, BCM2835FBState),
+        VMSTATE_UINT32(xoffset, BCM2835FBState),
+        VMSTATE_UINT32(yoffset, BCM2835FBState),
+        VMSTATE_UINT32(bpp, BCM2835FBState),
+        VMSTATE_UINT32(base, BCM2835FBState),
+        VMSTATE_UINT32(pitch, BCM2835FBState),
+        VMSTATE_UINT32(size, BCM2835FBState),
+        VMSTATE_UINT32(pixo, BCM2835FBState),
+        VMSTATE_UINT32(alpha, BCM2835FBState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const GraphicHwOps vgafb_ops = {
+    .invalidate  = fb_invalidate_display,
+    .gfx_update  = fb_update_display,
+};
+
+static void bcm2835_fb_init(Object *obj)
+{
+    BCM2835FBState *s = BCM2835_FB(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_fb_ops, s, TYPE_BCM2835_FB,
+                          0x10);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->mbox_irq);
+}
+
+static void bcm2835_fb_reset(DeviceState *dev)
+{
+    BCM2835FBState *s = BCM2835_FB(dev);
+
+    s->pending = false;
+
+    s->xres_virtual = s->xres;
+    s->yres_virtual = s->yres;
+    s->xoffset = 0;
+    s->yoffset = 0;
+    s->base = s->vcram_base + BCM2835_FB_OFFSET;
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    s->invalidate = true;
+    s->lock = false;
+}
+
+static void bcm2835_fb_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835FBState *s = BCM2835_FB(dev);
+    Error *err = NULL;
+    Object *obj;
+
+    if (s->vcram_base == 0) {
+        error_setg(errp, "%s: required vcram-base property not set", __func__);
+        return;
+    }
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    bcm2835_fb_reset(dev);
+
+    s->con = graphic_console_init(dev, 0, &vgafb_ops, s);
+    qemu_console_resize(s->con, s->xres, s->yres);
+}
+
+static Property bcm2835_fb_props[] = {
+    DEFINE_PROP_UINT32("vcram-base", BCM2835FBState, vcram_base, 0),/*required*/
+    DEFINE_PROP_UINT32("vcram-size", BCM2835FBState, vcram_size,
+                       DEFAULT_VCRAM_SIZE),
+    DEFINE_PROP_UINT32("xres", BCM2835FBState, xres, 640),
+    DEFINE_PROP_UINT32("yres", BCM2835FBState, yres, 480),
+    DEFINE_PROP_UINT32("bpp", BCM2835FBState, bpp, 16),
+    DEFINE_PROP_UINT32("pixo", BCM2835FBState, pixo, 1), /* 1=RGB, 0=BGR */
+    DEFINE_PROP_UINT32("alpha", BCM2835FBState, alpha, 2), /* alpha ignored */
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2835_fb_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = bcm2835_fb_props;
+    dc->realize = bcm2835_fb_realize;
+    dc->reset = bcm2835_fb_reset;
+    dc->vmsd = &vmstate_bcm2835_fb;
+}
+
+static TypeInfo bcm2835_fb_info = {
+    .name          = TYPE_BCM2835_FB,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835FBState),
+    .class_init    = bcm2835_fb_class_init,
+    .instance_init = bcm2835_fb_init,
+};
+
+static void bcm2835_fb_register_types(void)
+{
+    type_register_static(&bcm2835_fb_info);
+}
+
+type_init(bcm2835_fb_register_types)
diff --git a/qemu/hw/display/blizzard.c b/qemu/hw/display/blizzard.c
index 5019bbbef..c231960d9 100644
--- a/qemu/hw/display/blizzard.c
+++ b/qemu/hw/display/blizzard.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "hw/devices.h"
diff --git a/qemu/hw/display/cg3.c b/qemu/hw/display/cg3.c
index 34dcbc311..fc0d97fa4 100644
--- a/qemu/hw/display/cg3.c
+++ b/qemu/hw/display/cg3.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "ui/console.h"
@@ -280,12 +282,12 @@ static void cg3_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     CG3State *s = CG3(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "cg3.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+    memory_region_init_ram(&s->rom, obj, "cg3.prom", FCODE_MAX_ROM_SIZE,
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
-    memory_region_init_io(&s->reg, NULL, &cg3_reg_ops, s, "cg3.reg",
+    memory_region_init_io(&s->reg, obj, &cg3_reg_ops, s, "cg3.reg",
                           CG3_REG_SIZE);
     sysbus_init_mmio(sbd, &s->reg);
 }
@@ -310,7 +312,7 @@ static void cg3_realizefn(DeviceState *dev, Error **errp)
     }
 
     memory_region_init_ram(&s->vram_mem, NULL, "cg3.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vmstate_register_ram_global(&s->vram_mem);
     sysbus_init_mmio(sbd, &s->vram_mem);
diff --git a/qemu/hw/display/cirrus_vga.c b/qemu/hw/display/cirrus_vga.c
index 5198037d8..3d712d592 100644
--- a/qemu/hw/display/cirrus_vga.c
+++ b/qemu/hw/display/cirrus_vga.c
@@ -26,6 +26,8 @@
  * Reference: Finn Thogersons' VGADOC4b
  *   available at http://home.worldonline.dk/~finth/
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "ui/console.h"
@@ -275,14 +277,14 @@ static bool blit_region_is_unsafe(struct CirrusVGAState *s,
             + ((int64_t)s->cirrus_blt_height-1) * pitch;
         int32_t max = addr
             + s->cirrus_blt_width;
-        if (min < 0 || max >= s->vga.vram_size) {
+        if (min < 0 || max > s->vga.vram_size) {
             return true;
         }
     } else {
         int64_t max = addr
             + ((int64_t)s->cirrus_blt_height-1) * pitch
             + s->cirrus_blt_width;
-        if (max >= s->vga.vram_size) {
+        if (max > s->vga.vram_size) {
             return true;
         }
     }
diff --git a/qemu/hw/display/exynos4210_fimd.c b/qemu/hw/display/exynos4210_fimd.c
index 603ef5056..728eb214a 100644
--- a/qemu/hw/display/exynos4210_fimd.c
+++ b/qemu/hw/display/exynos4210_fimd.c
@@ -22,6 +22,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/sysbus.h"
 #include "ui/console.h"
@@ -1354,9 +1355,7 @@ static void exynos4210_fimd_reset(DeviceState *d)
         fimd_update_get_alpha(s, w);
     }
 
-    if (s->ifb != NULL) {
-        g_free(s->ifb);
-    }
+    g_free(s->ifb);
     s->ifb = NULL;
 
     exynos4210_fimd_invalidate(s);
diff --git a/qemu/hw/display/framebuffer.c b/qemu/hw/display/framebuffer.c
index 7f075ce77..df51358e7 100644
--- a/qemu/hw/display/framebuffer.c
+++ b/qemu/hw/display/framebuffer.c
@@ -17,6 +17,7 @@
    - Remove all DisplayState knowledge from devices.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "framebuffer.h"
diff --git a/qemu/hw/display/g364fb.c b/qemu/hw/display/g364fb.c
index 7f83a007b..70ef2c745 100644
--- a/qemu/hw/display/g364fb.c
+++ b/qemu/hw/display/g364fb.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/error-report.h"
 #include "ui/console.h"
diff --git a/qemu/hw/display/jazz_led.c b/qemu/hw/display/jazz_led.c
index 12b1707cb..09dcdb46a 100644
--- a/qemu/hw/display/jazz_led.c
+++ b/qemu/hw/display/jazz_led.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
@@ -233,8 +234,10 @@ static void jazz_led_text_update(void *opaque, console_ch_t *chardata)
 
     /* TODO: draw the segments */
     snprintf(buf, 2, "%02hhx\n", s->segments);
-    console_write_ch(chardata++, 0x00200100 | buf[0]);
-    console_write_ch(chardata++, 0x00200100 | buf[1]);
+    console_write_ch(chardata++, ATTR2CHTYPE(buf[0], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
+    console_write_ch(chardata++, ATTR2CHTYPE(buf[1], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
 
     dpy_text_update(s->con, 0, 0, 2, 1);
 }
diff --git a/qemu/hw/display/milkymist-tmu2.c b/qemu/hw/display/milkymist-tmu2.c
index 3e1d0b9c2..9bc88f93b 100644
--- a/qemu/hw/display/milkymist-tmu2.c
+++ b/qemu/hw/display/milkymist-tmu2.c
@@ -24,14 +24,15 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 #include "qemu/error-report.h"
 
 #include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 
 enum {
     R_CTL = 0,
diff --git a/qemu/hw/display/milkymist-vgafb.c b/qemu/hw/display/milkymist-vgafb.c
index ab3074fad..19ca25647 100644
--- a/qemu/hw/display/milkymist-vgafb.c
+++ b/qemu/hw/display/milkymist-vgafb.c
@@ -22,6 +22,7 @@
  *   http://www.milkymist.org/socdoc/vgafb.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/display/omap_dss.c b/qemu/hw/display/omap_dss.c
index f1fef2767..783e9e131 100644
--- a/qemu/hw/display/omap_dss.c
+++ b/qemu/hw/display/omap_dss.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -1051,8 +1052,7 @@ struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2)
 {
-    struct omap_dss_s *s = (struct omap_dss_s *)
-            g_malloc0(sizeof(struct omap_dss_s));
+    struct omap_dss_s *s = g_new0(struct omap_dss_s, 1);
 
     s->irq = irq;
     s->drq = drq;
diff --git a/qemu/hw/display/omap_lcd_template.h b/qemu/hw/display/omap_lcd_template.h
index e5dd44716..f0ce71fd6 100644
--- a/qemu/hw/display/omap_lcd_template.h
+++ b/qemu/hw/display/omap_lcd_template.h
@@ -136,7 +136,7 @@ static void glue(draw_line12_, DEPTH)(void *opaque,
     uint8_t r, g, b;
 
     do {
-        v = lduw_p((void *) s);
+        v = lduw_le_p((void *) s);
         r = (v >> 4) & 0xf0;
         g = v & 0xf0;
         b = (v << 4) & 0xf0;
@@ -159,7 +159,7 @@ static void glue(draw_line16_, DEPTH)(void *opaque,
     uint8_t r, g, b;
 
     do {
-        v = lduw_p((void *) s);
+        v = lduw_le_p((void *) s);
         r = (v >> 8) & 0xf8;
         g = (v >> 3) & 0xfc;
         b = (v << 3) & 0xf8;
diff --git a/qemu/hw/display/omap_lcdc.c b/qemu/hw/display/omap_lcdc.c
index a7c6cd79b..ce1058bf8 100644
--- a/qemu/hw/display/omap_lcdc.c
+++ b/qemu/hw/display/omap_lcdc.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -403,8 +404,7 @@ struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk)
 {
-    struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *)
-            g_malloc0(sizeof(struct omap_lcd_panel_s));
+    struct omap_lcd_panel_s *s = g_new0(struct omap_lcd_panel_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/qemu/hw/display/pl110.c b/qemu/hw/display/pl110.c
index ef1a7b1a5..d589959f1 100644
--- a/qemu/hw/display/pl110.c
+++ b/qemu/hw/display/pl110.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GNU LGPL
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "ui/console.h"
 #include "framebuffer.h"
diff --git a/qemu/hw/display/pxa2xx_lcd.c b/qemu/hw/display/pxa2xx_lcd.c
index 494700d07..845521c5b 100644
--- a/qemu/hw/display/pxa2xx_lcd.c
+++ b/qemu/hw/display/pxa2xx_lcd.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/pxa.h"
@@ -309,10 +310,10 @@ static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
         }
 
         cpu_physical_memory_read(descptr, &desc, sizeof(desc));
-        s->dma_ch[i].descriptor = tswap32(desc.fdaddr);
-        s->dma_ch[i].source = tswap32(desc.fsaddr);
-        s->dma_ch[i].id = tswap32(desc.fidr);
-        s->dma_ch[i].command = tswap32(desc.ldcmd);
+        s->dma_ch[i].descriptor = le32_to_cpu(desc.fdaddr);
+        s->dma_ch[i].source = le32_to_cpu(desc.fsaddr);
+        s->dma_ch[i].id = le32_to_cpu(desc.fidr);
+        s->dma_ch[i].command = le32_to_cpu(desc.ldcmd);
     }
 }
 
diff --git a/qemu/hw/display/qxl-logger.c b/qemu/hw/display/qxl-logger.c
index d944d3fdb..2ec6d8fa3 100644
--- a/qemu/hw/display/qxl-logger.c
+++ b/qemu/hw/display/qxl-logger.c
@@ -19,6 +19,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 #include "qxl.h"
 
diff --git a/qemu/hw/display/qxl-render.c b/qemu/hw/display/qxl-render.c
index a542087fc..9ad9d9e0f 100644
--- a/qemu/hw/display/qxl-render.c
+++ b/qemu/hw/display/qxl-render.c
@@ -19,6 +19,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qxl.h"
 #include "trace.h"
 
@@ -159,7 +160,7 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
 /*
  * use ssd.lock to protect render_update_cookie_num.
  * qxl_render_update is called by io thread or vcpu thread, and the completion
- * callbacks are called by spice_server thread, defering to bh called from the
+ * callbacks are called by spice_server thread, deferring to bh called from the
  * io thread.
  */
 void qxl_render_update(PCIQXLDevice *qxl)
diff --git a/qemu/hw/display/qxl.c b/qemu/hw/display/qxl.c
index 2288238d0..919dc5cd3 100644
--- a/qemu/hw/display/qxl.c
+++ b/qemu/hw/display/qxl.c
@@ -18,8 +18,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h>
-#include <stdint.h>
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
@@ -1156,7 +1156,9 @@ static void qxl_soft_reset(PCIQXLDevice *d)
     trace_qxl_soft_reset(d->id);
     qxl_check_state(d);
     qxl_clear_guest_bug(d);
+    qemu_mutex_lock(&d->async_lock);
     d->current_async = QXL_UNDEFINED_IO;
+    qemu_mutex_unlock(&d->async_lock);
 
     if (d->id == 0) {
         qxl_enter_vga_mode(d);
@@ -1970,14 +1972,14 @@ static void qxl_realize_common(PCIQXLDevice *qxl, Error **errp)
 
     qxl->rom_size = qxl_rom_size();
     memory_region_init_ram(&qxl->rom_bar, OBJECT(qxl), "qxl.vrom",
-                           qxl->rom_size, &error_abort);
+                           qxl->rom_size, &error_fatal);
     vmstate_register_ram(&qxl->rom_bar, &qxl->pci.qdev);
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
     qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, OBJECT(qxl), "qxl.vram",
-                           qxl->vram_size, &error_abort);
+                           qxl->vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, OBJECT(qxl), "qxl.vram32",
                              &qxl->vram_bar, 0, qxl->vram32_size);
@@ -2079,7 +2081,7 @@ static void qxl_realize_secondary(PCIDevice *dev, Error **errp)
     qxl->id = device_id++;
     qxl_init_ramsize(qxl);
     memory_region_init_ram(&qxl->vga.vram, OBJECT(dev), "qxl.vgavram",
-                           qxl->vga.vram_size, &error_abort);
+                           qxl->vga.vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vga.vram, &qxl->pci.qdev);
     qxl->vga.vram_ptr = memory_region_get_ram_ptr(&qxl->vga.vram);
     qxl->vga.con = graphic_console_init(DEVICE(dev), 0, &qxl_ops, qxl);
@@ -2156,7 +2158,7 @@ static int qxl_post_load(void *opaque, int version)
         qxl_create_guest_primary(d, 1, QXL_SYNC);
 
         /* replay surface-create and cursor-set commands */
-        cmds = g_malloc0(sizeof(QXLCommandExt) * (d->ssd.num_surfaces + 1));
+        cmds = g_new0(QXLCommandExt, d->ssd.num_surfaces + 1);
         for (in = 0, out = 0; in < d->ssd.num_surfaces; in++) {
             if (d->guest_surfaces.cmds[in] == 0) {
                 continue;
diff --git a/qemu/hw/display/sm501.c b/qemu/hw/display/sm501.c
index 15a5ba800..5f7101210 100644
--- a/qemu/hw/display/sm501.c
+++ b/qemu/hw/display/sm501.c
@@ -22,7 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/char/serial.h"
 #include "ui/console.h"
@@ -1411,7 +1414,7 @@ void sm501_init(MemoryRegion *address_space_mem, uint32_t base,
 
     /* allocate local memory */
     memory_region_init_ram(&s->local_mem_region, NULL, "sm501.local",
-                           local_mem_bytes, &error_abort);
+                           local_mem_bytes, &error_fatal);
     vmstate_register_ram_global(&s->local_mem_region);
     memory_region_set_log(&s->local_mem_region, true, DIRTY_MEMORY_VGA);
     s->local_mem = memory_region_get_ram_ptr(&s->local_mem_region);
diff --git a/qemu/hw/display/ssd0303.c b/qemu/hw/display/ssd0303.c
index f6804fb51..d3017563f 100644
--- a/qemu/hw/display/ssd0303.c
+++ b/qemu/hw/display/ssd0303.c
@@ -10,6 +10,7 @@
 /* The controller can support a variety of different displays, but we only
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 #include "ui/console.h"
 
diff --git a/qemu/hw/display/ssd0323.c b/qemu/hw/display/ssd0323.c
index 97270077e..14c1bf339 100644
--- a/qemu/hw/display/ssd0323.c
+++ b/qemu/hw/display/ssd0323.c
@@ -10,7 +10,8 @@
 /* The controller can support a variety of different displays, but we only
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 #include "ui/console.h"
 
 //#define DEBUG_SSD0323 1
diff --git a/qemu/hw/display/tc6393xb.c b/qemu/hw/display/tc6393xb.c
index f5f3f3e69..da3ceceb0 100644
--- a/qemu/hw/display/tc6393xb.c
+++ b/qemu/hw/display/tc6393xb.c
@@ -10,6 +10,8 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "hw/block/flash.h"
@@ -584,7 +586,7 @@ TC6393xbState *tc6393xb_init(MemoryRegion *sysmem, uint32_t base, qemu_irq irq)
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     memory_region_init_ram(&s->vram, NULL, "tc6393xb.vram", 0x100000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
     memory_region_add_subregion(sysmem, base + 0x100000, &s->vram);
diff --git a/qemu/hw/display/tcx.c b/qemu/hw/display/tcx.c
index 6acdc2d28..8e26aae80 100644
--- a/qemu/hw/display/tcx.c
+++ b/qemu/hw/display/tcx.c
@@ -22,7 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "cpu.h" /* FIXME shouldn't use TARGET_PAGE_SIZE */
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
 #include "hw/loader.h"
@@ -944,57 +947,55 @@ static void tcx_initfn(Object *obj)
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     TCXState *s = TCX(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "tcx.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+    memory_region_init_ram(&s->rom, obj, "tcx.prom", FCODE_MAX_ROM_SIZE,
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
     /* 2/STIP : Stippler */
-    memory_region_init_io(&s->stip, OBJECT(s), &tcx_stip_ops, s, "tcx.stip",
+    memory_region_init_io(&s->stip, obj, &tcx_stip_ops, s, "tcx.stip",
                           TCX_STIP_NREGS);
     sysbus_init_mmio(sbd, &s->stip);
 
     /* 3/BLIT : Blitter */
-    memory_region_init_io(&s->blit, OBJECT(s), &tcx_blit_ops, s, "tcx.blit",
+    memory_region_init_io(&s->blit, obj, &tcx_blit_ops, s, "tcx.blit",
                           TCX_BLIT_NREGS);
     sysbus_init_mmio(sbd, &s->blit);
 
     /* 5/RSTIP : Raw Stippler */
-    memory_region_init_io(&s->rstip, OBJECT(s), &tcx_rstip_ops, s, "tcx.rstip",
+    memory_region_init_io(&s->rstip, obj, &tcx_rstip_ops, s, "tcx.rstip",
                           TCX_RSTIP_NREGS);
     sysbus_init_mmio(sbd, &s->rstip);
 
     /* 6/RBLIT : Raw Blitter */
-    memory_region_init_io(&s->rblit, OBJECT(s), &tcx_rblit_ops, s, "tcx.rblit",
+    memory_region_init_io(&s->rblit, obj, &tcx_rblit_ops, s, "tcx.rblit",
                           TCX_RBLIT_NREGS);
     sysbus_init_mmio(sbd, &s->rblit);
 
     /* 7/TEC : ??? */
-    memory_region_init_io(&s->tec, OBJECT(s), &tcx_dummy_ops, s,
-                          "tcx.tec", TCX_TEC_NREGS);
+    memory_region_init_io(&s->tec, obj, &tcx_dummy_ops, s, "tcx.tec",
+                          TCX_TEC_NREGS);
     sysbus_init_mmio(sbd, &s->tec);
 
     /* 8/CMAP : DAC */
-    memory_region_init_io(&s->dac, OBJECT(s), &tcx_dac_ops, s,
-                          "tcx.dac", TCX_DAC_NREGS);
+    memory_region_init_io(&s->dac, obj, &tcx_dac_ops, s, "tcx.dac",
+                          TCX_DAC_NREGS);
     sysbus_init_mmio(sbd, &s->dac);
 
     /* 9/THC : Cursor */
-    memory_region_init_io(&s->thc, OBJECT(s), &tcx_thc_ops, s, "tcx.thc",
+    memory_region_init_io(&s->thc, obj, &tcx_thc_ops, s, "tcx.thc",
                           TCX_THC_NREGS);
     sysbus_init_mmio(sbd, &s->thc);
 
     /* 11/DHC : ??? */
-    memory_region_init_io(&s->dhc, OBJECT(s), &tcx_dummy_ops, s, "tcx.dhc",
+    memory_region_init_io(&s->dhc, obj, &tcx_dummy_ops, s, "tcx.dhc",
                           TCX_DHC_NREGS);
     sysbus_init_mmio(sbd, &s->dhc);
 
     /* 12/ALT : ??? */
-    memory_region_init_io(&s->alt, OBJECT(s), &tcx_dummy_ops, s, "tcx.alt",
+    memory_region_init_io(&s->alt, obj, &tcx_dummy_ops, s, "tcx.alt",
                           TCX_ALT_NREGS);
     sysbus_init_mmio(sbd, &s->alt);
-
-    return;
 }
 
 static void tcx_realizefn(DeviceState *dev, Error **errp)
@@ -1007,7 +1008,7 @@ static void tcx_realizefn(DeviceState *dev, Error **errp)
     char *fcode_filename;
 
     memory_region_init_ram(&s->vram_mem, OBJECT(s), "tcx.vram",
-                           s->vram_size * (1 + 4 + 4), &error_abort);
+                           s->vram_size * (1 + 4 + 4), &error_fatal);
     vmstate_register_ram_global(&s->vram_mem);
     memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vram_base = memory_region_get_ram_ptr(&s->vram_mem);
diff --git a/qemu/hw/display/vga-isa-mm.c b/qemu/hw/display/vga-isa-mm.c
index 4efc22278..51ccbccc4 100644
--- a/qemu/hw/display/vga-isa-mm.c
+++ b/qemu/hw/display/vga-isa-mm.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/i386/pc.h"
diff --git a/qemu/hw/display/vga-isa.c b/qemu/hw/display/vga-isa.c
index 7f3c98941..f5aff1cbe 100644
--- a/qemu/hw/display/vga-isa.c
+++ b/qemu/hw/display/vga-isa.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/i386/pc.h"
diff --git a/qemu/hw/display/vga-pci.c b/qemu/hw/display/vga-pci.c
index 1dfa331e6..ac9a76499 100644
--- a/qemu/hw/display/vga-pci.c
+++ b/qemu/hw/display/vga-pci.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/pci/pci.h"
diff --git a/qemu/hw/display/vga.c b/qemu/hw/display/vga.c
index b35d523e6..4a55ec6db 100644
--- a/qemu/hw/display/vga.c
+++ b/qemu/hw/display/vga.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "vga.h"
 #include "ui/console.h"
@@ -140,6 +142,13 @@ static uint32_t expand4[256];
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
+static void vbe_update_vgaregs(VGACommonState *s);
+
+static inline bool vbe_enabled(VGACommonState *s)
+{
+    return s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED;
+}
+
 static void vga_update_memory_access(VGACommonState *s)
 {
     hwaddr base, offset, size;
@@ -177,6 +186,7 @@ static void vga_update_memory_access(VGACommonState *s)
             size = 0x8000;
             break;
         }
+        assert(offset + size <= s->vram_size);
         memory_region_init_alias(&s->chain4_alias, memory_region_owner(&s->vram),
                                  "vga.chain4", &s->vram, offset, size);
         memory_region_add_subregion_overlap(s->legacy_address_space, base,
@@ -234,9 +244,9 @@ static void vga_precise_update_retrace_info(VGACommonState *s)
 
     r->total_chars = vtotal_lines * htotal_chars;
     if (r->freq) {
-        r->ticks_per_char = get_ticks_per_sec() / (r->total_chars * r->freq);
+        r->ticks_per_char = NANOSECONDS_PER_SECOND / (r->total_chars * r->freq);
     } else {
-        r->ticks_per_char = get_ticks_per_sec() / chars_per_sec;
+        r->ticks_per_char = NANOSECONDS_PER_SECOND / chars_per_sec;
     }
 
     r->vstart = vretr_start_line;
@@ -264,7 +274,7 @@ static void vga_precise_update_retrace_info(VGACommonState *s)
         "dots = %d\n"
         "ticks/char = %" PRId64 "\n"
         "\n",
-        (double) get_ticks_per_sec() / (r->ticks_per_char * r->total_chars),
+        (double) NANOSECONDS_PER_SECOND / (r->ticks_per_char * r->total_chars),
         htotal_chars,
         hretr_start_char,
         hretr_skew_chars,
@@ -476,6 +486,7 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         printf("vga: write SR%x = 0x%02x\n", s->sr_index, val);
 #endif
         s->sr[s->sr_index] = val & sr_mask[s->sr_index];
+        vbe_update_vgaregs(s);
         if (s->sr_index == VGA_SEQ_CLOCK_MODE) {
             s->update_retrace_info(s);
         }
@@ -507,6 +518,7 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         printf("vga: write GR%x = 0x%02x\n", s->gr_index, val);
 #endif
         s->gr[s->gr_index] = val & gr_mask[s->gr_index];
+        vbe_update_vgaregs(s);
         vga_update_memory_access(s);
         break;
     case VGA_CRT_IM:
@@ -525,10 +537,12 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
             if (s->cr_index == VGA_CRTC_OVERFLOW) {
                 s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x10) |
                     (val & 0x10);
+                vbe_update_vgaregs(s);
             }
             return;
         }
         s->cr[s->cr_index] = val;
+        vbe_update_vgaregs(s);
 
         switch(s->cr_index) {
         case VGA_CRTC_H_TOTAL:
@@ -561,7 +575,7 @@ static void vbe_fixup_regs(VGACommonState *s)
     uint16_t *r = s->vbe_regs;
     uint32_t bits, linelength, maxy, offset;
 
-    if (!(r[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
+    if (!vbe_enabled(s)) {
         /* vbe is turned off -- nothing to do */
         return;
     }
@@ -636,6 +650,49 @@ static void vbe_fixup_regs(VGACommonState *s)
     s->vbe_start_addr  = offset / 4;
 }
 
+/* we initialize the VGA graphic mode */
+static void vbe_update_vgaregs(VGACommonState *s)
+{
+    int h, shift_control;
+
+    if (!vbe_enabled(s)) {
+        /* vbe is turned off -- nothing to do */
+        return;
+    }
+
+    /* graphic mode + memory map 1 */
+    s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
+        VGA_GR06_GRAPHICS_MODE;
+    s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
+    s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
+    /* width */
+    s->cr[VGA_CRTC_H_DISP] =
+        (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
+    /* height (only meaningful if < 1024) */
+    h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
+    s->cr[VGA_CRTC_V_DISP_END] = h;
+    s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
+        ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
+    /* line compare to 1023 */
+    s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
+    s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
+    s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
+
+    if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
+        shift_control = 0;
+        s->sr[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
+    } else {
+        shift_control = 2;
+        /* set chain 4 mode */
+        s->sr[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
+        /* activate all planes */
+        s->sr[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
+    }
+    s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
+        (shift_control << 5);
+    s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
+}
+
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
@@ -712,13 +769,10 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         case VBE_DISPI_INDEX_Y_OFFSET:
             s->vbe_regs[s->vbe_index] = val;
             vbe_fixup_regs(s);
+            vbe_update_vgaregs(s);
             break;
         case VBE_DISPI_INDEX_BANK:
-            if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-              val &= (s->vbe_bank_mask >> 2);
-            } else {
-              val &= s->vbe_bank_mask;
-            }
+            val &= s->vbe_bank_mask;
             s->vbe_regs[s->vbe_index] = val;
             s->bank_offset = (val << 16);
             vga_update_memory_access(s);
@@ -726,52 +780,19 @@ void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         case VBE_DISPI_INDEX_ENABLE:
             if ((val & VBE_DISPI_ENABLED) &&
                 !(s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
-                int h, shift_control;
 
                 s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_X_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_Y_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_ENABLE] |= VBE_DISPI_ENABLED;
                 vbe_fixup_regs(s);
+                vbe_update_vgaregs(s);
 
                 /* clear the screen */
                 if (!(val & VBE_DISPI_NOCLEARMEM)) {
                     memset(s->vram_ptr, 0,
                            s->vbe_regs[VBE_DISPI_INDEX_YRES] * s->vbe_line_offset);
                 }
-
-                /* we initialize the VGA graphic mode */
-                /* graphic mode + memory map 1 */
-                s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
-                    VGA_GR06_GRAPHICS_MODE;
-                s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
-                s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
-                /* width */
-                s->cr[VGA_CRTC_H_DISP] =
-                    (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
-                /* height (only meaningful if < 1024) */
-                h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
-                s->cr[VGA_CRTC_V_DISP_END] = h;
-                s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
-                    ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
-                /* line compare to 1023 */
-                s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
-                s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
-                s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
-
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-                    shift_control = 0;
-                    s->sr[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
-                } else {
-                    shift_control = 2;
-                    /* set chain 4 mode */
-                    s->sr[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
-                    /* activate all planes */
-                    s->sr[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
-                }
-                s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
-                    (shift_control << 5);
-                s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
             } else {
                 s->bank_offset = 0;
             }
@@ -817,13 +838,21 @@ uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr)
 
     if (s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
         /* chain 4 mode : simplest access */
+        assert(addr < s->vram_size);
         ret = s->vram_ptr[addr];
     } else if (s->gr[VGA_GFX_MODE] & 0x10) {
         /* odd/even mode (aka text mode mapping) */
         plane = (s->gr[VGA_GFX_PLANE_READ] & 2) | (addr & 1);
-        ret = s->vram_ptr[((addr & ~1) << 1) | plane];
+        addr = ((addr & ~1) << 1) | plane;
+        if (addr >= s->vram_size) {
+            return 0xff;
+        }
+        ret = s->vram_ptr[addr];
     } else {
         /* standard VGA latched access */
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return 0xff;
+        }
         s->latch = ((uint32_t *)s->vram_ptr)[addr];
 
         if (!(s->gr[VGA_GFX_MODE] & 0x08)) {
@@ -880,6 +909,7 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         plane = addr & 3;
         mask = (1 << plane);
         if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
+            assert(addr < s->vram_size);
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: chain4: [0x" TARGET_FMT_plx "]\n", addr);
@@ -893,6 +923,9 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         mask = (1 << plane);
         if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
             addr = ((addr & ~1) << 1) | plane;
+            if (addr >= s->vram_size) {
+                return;
+            }
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: odd/even: [0x" TARGET_FMT_plx "]\n", addr);
@@ -966,6 +999,9 @@ void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
         mask = s->sr[VGA_SEQ_PLANE_WRITE];
         s->plane_updated |= mask; /* only used to detect font change */
         write_mask = mask16[mask];
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return;
+        }
         ((uint32_t *)s->vram_ptr)[addr] =
             (((uint32_t *)s->vram_ptr)[addr] & ~write_mask) |
             (val & write_mask);
@@ -1044,7 +1080,7 @@ static void vga_get_offsets(VGACommonState *s,
 {
     uint32_t start_addr, line_offset, line_compare;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         line_offset = s->vbe_line_offset;
         start_addr = s->vbe_start_addr;
         line_compare = 65535;
@@ -1369,7 +1405,7 @@ static int vga_get_bpp(VGACommonState *s)
 {
     int ret;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         ret = s->vbe_regs[VBE_DISPI_INDEX_BPP];
     } else {
         ret = 0;
@@ -1381,7 +1417,7 @@ static void vga_get_resolution(VGACommonState *s, int *pwidth, int *pheight)
 {
     int width, height;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         width = s->vbe_regs[VBE_DISPI_INDEX_XRES];
         height = s->vbe_regs[VBE_DISPI_INDEX_YRES];
     } else {
@@ -1979,7 +2015,8 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
     width = (s->last_width - size) / 2;
     dst = chardata + s->last_width + width;
     for (i = 0; i < size; i ++)
-        console_write_ch(dst ++, 0x00200100 | msg_buffer[i]);
+        console_write_ch(dst ++, ATTR2CHTYPE(msg_buffer[i], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
 
     dpy_text_update(s->con, 0, 0, s->last_width, height);
 }
@@ -2139,7 +2176,7 @@ void vga_common_init(VGACommonState *s, Object *obj, bool global_vmstate)
 
     s->is_vbe_vmstate = 1;
     memory_region_init_ram(&s->vram, obj, "vga.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram(&s->vram, global_vmstate ? NULL : DEVICE(obj));
     xen_register_framebuffer(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
diff --git a/qemu/hw/display/vga_int.h b/qemu/hw/display/vga_int.h
index 40ba6a420..bdb43a5a3 100644
--- a/qemu/hw/display/vga_int.h
+++ b/qemu/hw/display/vga_int.h
@@ -25,7 +25,6 @@
 #define HW_VGA_INT_H 1
 
 #include <hw/hw.h>
-#include "qapi/error.h"
 #include "exec/memory.h"
 
 #define ST01_V_RETRACE      0x08
diff --git a/qemu/hw/display/virtio-gpu-3d.c b/qemu/hw/display/virtio-gpu-3d.c
new file mode 100644
index 000000000..fa192946a
--- /dev/null
+++ b/qemu/hw/display/virtio-gpu-3d.c
@@ -0,0 +1,606 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/iov.h"
+#include "trace.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-gpu.h"
+
+#ifdef CONFIG_VIRGL
+
+#include "virglrenderer.h"
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs;
+
+static void virgl_cmd_create_resource_2d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_2d c2d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c2d);
+    trace_virtio_gpu_cmd_res_create_2d(c2d.resource_id, c2d.format,
+                                       c2d.width, c2d.height);
+
+    args.handle = c2d.resource_id;
+    args.target = 2;
+    args.format = c2d.format;
+    args.bind = (1 << 1);
+    args.width = c2d.width;
+    args.height = c2d.height;
+    args.depth = 1;
+    args.array_size = 1;
+    args.last_level = 0;
+    args.nr_samples = 0;
+    args.flags = VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_create_resource_3d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_3d c3d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c3d);
+    trace_virtio_gpu_cmd_res_create_3d(c3d.resource_id, c3d.format,
+                                       c3d.width, c3d.height, c3d.depth);
+
+    args.handle = c3d.resource_id;
+    args.target = c3d.target;
+    args.format = c3d.format;
+    args.bind = c3d.bind;
+    args.width = c3d.width;
+    args.height = c3d.height;
+    args.depth = c3d.depth;
+    args.array_size = c3d.array_size;
+    args.last_level = c3d.last_level;
+    args.nr_samples = c3d.nr_samples;
+    args.flags = c3d.flags;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_resource_unref(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_unref unref;
+
+    VIRTIO_GPU_FILL_CMD(unref);
+    trace_virtio_gpu_cmd_res_unref(unref.resource_id);
+
+    virgl_renderer_resource_unref(unref.resource_id);
+}
+
+static void virgl_cmd_context_create(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_create cc;
+
+    VIRTIO_GPU_FILL_CMD(cc);
+    trace_virtio_gpu_cmd_ctx_create(cc.hdr.ctx_id,
+                                    cc.debug_name);
+
+    virgl_renderer_context_create(cc.hdr.ctx_id, cc.nlen,
+                                  cc.debug_name);
+}
+
+static void virgl_cmd_context_destroy(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_destroy cd;
+
+    VIRTIO_GPU_FILL_CMD(cd);
+    trace_virtio_gpu_cmd_ctx_destroy(cd.hdr.ctx_id);
+
+    virgl_renderer_context_destroy(cd.hdr.ctx_id);
+}
+
+static void virtio_gpu_rect_update(VirtIOGPU *g, int idx, int x, int y,
+                                int width, int height)
+{
+    if (!g->scanout[idx].con) {
+        return;
+    }
+
+    dpy_gl_update(g->scanout[idx].con, x, y, width, height);
+}
+
+static void virgl_cmd_resource_flush(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_flush rf;
+    int i;
+
+    VIRTIO_GPU_FILL_CMD(rf);
+    trace_virtio_gpu_cmd_res_flush(rf.resource_id,
+                                   rf.r.width, rf.r.height, rf.r.x, rf.r.y);
+
+    for (i = 0; i < VIRTIO_GPU_MAX_SCANOUT; i++) {
+        if (g->scanout[i].resource_id != rf.resource_id) {
+            continue;
+        }
+        virtio_gpu_rect_update(g, i, rf.r.x, rf.r.y, rf.r.width, rf.r.height);
+    }
+}
+
+static void virgl_cmd_set_scanout(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_set_scanout ss;
+    struct virgl_renderer_resource_info info;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(ss);
+    trace_virtio_gpu_cmd_set_scanout(ss.scanout_id, ss.resource_id,
+                                     ss.r.width, ss.r.height, ss.r.x, ss.r.y);
+
+    if (ss.scanout_id >= VIRTIO_GPU_MAX_SCANOUT) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal scanout id specified %d",
+                      __func__, ss.scanout_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID;
+        return;
+    }
+    g->enable = 1;
+
+    memset(&info, 0, sizeof(info));
+
+    if (ss.resource_id && ss.r.width && ss.r.height) {
+        ret = virgl_renderer_resource_get_info(ss.resource_id, &info);
+        if (ret == -1) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "%s: illegal resource specified %d\n",
+                          __func__, ss.resource_id);
+            cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+            return;
+        }
+        qemu_console_resize(g->scanout[ss.scanout_id].con,
+                            ss.r.width, ss.r.height);
+        virgl_renderer_force_ctx_0();
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, info.tex_id,
+                       info.flags & 1 /* FIXME: Y_0_TOP */,
+                       ss.r.x, ss.r.y, ss.r.width, ss.r.height);
+    } else {
+        if (ss.scanout_id != 0) {
+            dpy_gfx_replace_surface(g->scanout[ss.scanout_id].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, 0, false,
+                       0, 0, 0, 0);
+    }
+    g->scanout[ss.scanout_id].resource_id = ss.resource_id;
+}
+
+static void virgl_cmd_submit_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_cmd_submit cs;
+    void *buf;
+    size_t s;
+
+    VIRTIO_GPU_FILL_CMD(cs);
+    trace_virtio_gpu_cmd_ctx_submit(cs.hdr.ctx_id, cs.size);
+
+    buf = g_malloc(cs.size);
+    s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
+                   sizeof(cs), buf, cs.size);
+    if (s != cs.size) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: size mismatch (%zd/%d)",
+                      __func__, s, cs.size);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        goto out;
+    }
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->stats.req_3d++;
+        g->stats.bytes_3d += cs.size;
+    }
+
+    virgl_renderer_submit_cmd(buf, cs.hdr.ctx_id, cs.size / 4);
+
+out:
+    g_free(buf);
+}
+
+static void virgl_cmd_transfer_to_host_2d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_to_host_2d t2d;
+    struct virtio_gpu_box box;
+
+    VIRTIO_GPU_FILL_CMD(t2d);
+    trace_virtio_gpu_cmd_res_xfer_toh_2d(t2d.resource_id);
+
+    box.x = t2d.r.x;
+    box.y = t2d.r.y;
+    box.z = 0;
+    box.w = t2d.r.width;
+    box.h = t2d.r.height;
+    box.d = 1;
+
+    virgl_renderer_transfer_write_iov(t2d.resource_id,
+                                      0,
+                                      0,
+                                      0,
+                                      0,
+                                      (struct virgl_box *)&box,
+                                      t2d.offset, NULL, 0);
+}
+
+static void virgl_cmd_transfer_to_host_3d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d t3d;
+
+    VIRTIO_GPU_FILL_CMD(t3d);
+    trace_virtio_gpu_cmd_res_xfer_toh_3d(t3d.resource_id);
+
+    virgl_renderer_transfer_write_iov(t3d.resource_id,
+                                      t3d.hdr.ctx_id,
+                                      t3d.level,
+                                      t3d.stride,
+                                      t3d.layer_stride,
+                                      (struct virgl_box *)&t3d.box,
+                                      t3d.offset, NULL, 0);
+}
+
+static void
+virgl_cmd_transfer_from_host_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d tf3d;
+
+    VIRTIO_GPU_FILL_CMD(tf3d);
+    trace_virtio_gpu_cmd_res_xfer_fromh_3d(tf3d.resource_id);
+
+    virgl_renderer_transfer_read_iov(tf3d.resource_id,
+                                     tf3d.hdr.ctx_id,
+                                     tf3d.level,
+                                     tf3d.stride,
+                                     tf3d.layer_stride,
+                                     (struct virgl_box *)&tf3d.box,
+                                     tf3d.offset, NULL, 0);
+}
+
+
+static void virgl_resource_attach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_attach_backing att_rb;
+    struct iovec *res_iovs;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(att_rb);
+    trace_virtio_gpu_cmd_res_back_attach(att_rb.resource_id);
+
+    ret = virtio_gpu_create_mapping_iov(&att_rb, cmd, &res_iovs);
+    if (ret != 0) {
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        return;
+    }
+
+    virgl_renderer_resource_attach_iov(att_rb.resource_id,
+                                       res_iovs, att_rb.nr_entries);
+}
+
+static void virgl_resource_detach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_detach_backing detach_rb;
+    struct iovec *res_iovs = NULL;
+    int num_iovs = 0;
+
+    VIRTIO_GPU_FILL_CMD(detach_rb);
+    trace_virtio_gpu_cmd_res_back_detach(detach_rb.resource_id);
+
+    virgl_renderer_resource_detach_iov(detach_rb.resource_id,
+                                       &res_iovs,
+                                       &num_iovs);
+    if (res_iovs == NULL || num_iovs == 0) {
+        return;
+    }
+    virtio_gpu_cleanup_mapping_iov(res_iovs, num_iovs);
+}
+
+
+static void virgl_cmd_ctx_attach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource att_res;
+
+    VIRTIO_GPU_FILL_CMD(att_res);
+    trace_virtio_gpu_cmd_ctx_res_attach(att_res.hdr.ctx_id,
+                                        att_res.resource_id);
+
+    virgl_renderer_ctx_attach_resource(att_res.hdr.ctx_id, att_res.resource_id);
+}
+
+static void virgl_cmd_ctx_detach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource det_res;
+
+    VIRTIO_GPU_FILL_CMD(det_res);
+    trace_virtio_gpu_cmd_ctx_res_detach(det_res.hdr.ctx_id,
+                                        det_res.resource_id);
+
+    virgl_renderer_ctx_detach_resource(det_res.hdr.ctx_id, det_res.resource_id);
+}
+
+static void virgl_cmd_get_capset_info(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset_info info;
+    struct virtio_gpu_resp_capset_info resp;
+
+    VIRTIO_GPU_FILL_CMD(info);
+
+    if (info.capset_index == 0) {
+        resp.capset_id = VIRTIO_GPU_CAPSET_VIRGL;
+        virgl_renderer_get_cap_set(resp.capset_id,
+                                   &resp.capset_max_version,
+                                   &resp.capset_max_size);
+    } else {
+        resp.capset_max_version = 0;
+        resp.capset_max_size = 0;
+    }
+    resp.hdr.type = VIRTIO_GPU_RESP_OK_CAPSET_INFO;
+    virtio_gpu_ctrl_response(g, cmd, &resp.hdr, sizeof(resp));
+}
+
+static void virgl_cmd_get_capset(VirtIOGPU *g,
+                                 struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset gc;
+    struct virtio_gpu_resp_capset *resp;
+    uint32_t max_ver, max_size;
+    VIRTIO_GPU_FILL_CMD(gc);
+
+    virgl_renderer_get_cap_set(gc.capset_id, &max_ver,
+                               &max_size);
+    resp = g_malloc(sizeof(*resp) + max_size);
+
+    resp->hdr.type = VIRTIO_GPU_RESP_OK_CAPSET;
+    virgl_renderer_fill_caps(gc.capset_id,
+                             gc.capset_version,
+                             (void *)resp->capset_data);
+    virtio_gpu_ctrl_response(g, cmd, &resp->hdr, sizeof(*resp) + max_size);
+    g_free(resp);
+}
+
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    VIRTIO_GPU_FILL_CMD(cmd->cmd_hdr);
+
+    cmd->waiting = g->renderer_blocked;
+    if (cmd->waiting) {
+        return;
+    }
+
+    virgl_renderer_force_ctx_0();
+    switch (cmd->cmd_hdr.type) {
+    case VIRTIO_GPU_CMD_CTX_CREATE:
+        virgl_cmd_context_create(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DESTROY:
+        virgl_cmd_context_destroy(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_2D:
+        virgl_cmd_create_resource_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_3D:
+        virgl_cmd_create_resource_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SUBMIT_3D:
+        virgl_cmd_submit_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D:
+        virgl_cmd_transfer_to_host_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D:
+        virgl_cmd_transfer_to_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D:
+        virgl_cmd_transfer_from_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING:
+        virgl_resource_attach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING:
+        virgl_resource_detach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SET_SCANOUT:
+        virgl_cmd_set_scanout(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_FLUSH:
+        virgl_cmd_resource_flush(g, cmd);
+       break;
+    case VIRTIO_GPU_CMD_RESOURCE_UNREF:
+        virgl_cmd_resource_unref(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_attach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_detach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET_INFO:
+        virgl_cmd_get_capset_info(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET:
+        virgl_cmd_get_capset(g, cmd);
+        break;
+
+    case VIRTIO_GPU_CMD_GET_DISPLAY_INFO:
+        virtio_gpu_get_display_info(g, cmd);
+        break;
+    default:
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        break;
+    }
+
+    if (cmd->finished) {
+        return;
+    }
+    if (cmd->error) {
+        fprintf(stderr, "%s: ctrl 0x%x, error 0x%x\n", __func__,
+                cmd->cmd_hdr.type, cmd->error);
+        virtio_gpu_ctrl_response_nodata(g, cmd, cmd->error);
+        return;
+    }
+    if (!(cmd->cmd_hdr.flags & VIRTIO_GPU_FLAG_FENCE)) {
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        return;
+    }
+
+    trace_virtio_gpu_fence_ctrl(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+    virgl_renderer_create_fence(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+}
+
+static void virgl_write_fence(void *opaque, uint32_t fence)
+{
+    VirtIOGPU *g = opaque;
+    struct virtio_gpu_ctrl_command *cmd, *tmp;
+
+    QTAILQ_FOREACH_SAFE(cmd, &g->fenceq, next, tmp) {
+        /*
+	 * the guest can end up emitting fences out of order
+	 * so we should check all fenced cmds not just the first one.
+	 */
+        if (cmd->cmd_hdr.fence_id > fence) {
+            continue;
+        }
+        trace_virtio_gpu_fence_resp(cmd->cmd_hdr.fence_id);
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        QTAILQ_REMOVE(&g->fenceq, cmd, next);
+        g_free(cmd);
+        g->inflight--;
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            fprintf(stderr, "inflight: %3d (-)\r", g->inflight);
+        }
+    }
+}
+
+static virgl_renderer_gl_context
+virgl_create_context(void *opaque, int scanout_idx,
+                     struct virgl_renderer_gl_ctx_param *params)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext ctx;
+    QEMUGLParams qparams;
+
+    qparams.major_ver = params->major_ver;
+    qparams.minor_ver = params->minor_ver;
+
+    ctx = dpy_gl_ctx_create(g->scanout[scanout_idx].con, &qparams);
+    return (virgl_renderer_gl_context)ctx;
+}
+
+static void virgl_destroy_context(void *opaque, virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    dpy_gl_ctx_destroy(g->scanout[0].con, qctx);
+}
+
+static int virgl_make_context_current(void *opaque, int scanout_idx,
+                                      virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    return dpy_gl_ctx_make_current(g->scanout[scanout_idx].con, qctx);
+}
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs = {
+    .version             = 1,
+    .write_fence         = virgl_write_fence,
+    .create_gl_context   = virgl_create_context,
+    .destroy_gl_context  = virgl_destroy_context,
+    .make_current        = virgl_make_context_current,
+};
+
+static void virtio_gpu_print_stats(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    if (g->stats.requests) {
+        fprintf(stderr, "stats: vq req %4d, %3d -- 3D %4d (%5d)\n",
+                g->stats.requests,
+                g->stats.max_inflight,
+                g->stats.req_3d,
+                g->stats.bytes_3d);
+        g->stats.requests     = 0;
+        g->stats.max_inflight = 0;
+        g->stats.req_3d       = 0;
+        g->stats.bytes_3d     = 0;
+    } else {
+        fprintf(stderr, "stats: idle\r");
+    }
+    timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+}
+
+static void virtio_gpu_fence_poll(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    virgl_renderer_poll();
+    virtio_gpu_process_cmdq(g);
+    if (!QTAILQ_EMPTY(&g->cmdq) || !QTAILQ_EMPTY(&g->fenceq)) {
+        timer_mod(g->fence_poll, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 10);
+    }
+}
+
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g)
+{
+    virtio_gpu_fence_poll(g);
+}
+
+void virtio_gpu_virgl_reset(VirtIOGPU *g)
+{
+    int i;
+
+    /* virgl_renderer_reset() ??? */
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        if (i != 0) {
+            dpy_gfx_replace_surface(g->scanout[i].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[i].con, 0, false, 0, 0, 0, 0);
+    }
+}
+
+int virtio_gpu_virgl_init(VirtIOGPU *g)
+{
+    int ret;
+
+    ret = virgl_renderer_init(g, 0, &virtio_gpu_3d_cbs);
+    if (ret != 0) {
+        return ret;
+    }
+
+    g->fence_poll = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                 virtio_gpu_fence_poll, g);
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->print_stats = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                      virtio_gpu_print_stats, g);
+        timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+    }
+    return 0;
+}
+
+#endif /* CONFIG_VIRGL */
diff --git a/qemu/hw/display/virtio-gpu-pci.c b/qemu/hw/display/virtio-gpu-pci.c
index 5bc62cf34..a71b230d3 100644
--- a/qemu/hw/display/virtio-gpu-pci.c
+++ b/qemu/hw/display/virtio-gpu-pci.c
@@ -6,10 +6,11 @@
  * Authors:
  *  Dave Airlie
  *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/virtio/virtio.h"
 #include "hw/virtio/virtio-bus.h"
diff --git a/qemu/hw/display/virtio-gpu.c b/qemu/hw/display/virtio-gpu.c
index a67d927f5..c181fb364 100644
--- a/qemu/hw/display/virtio-gpu.c
+++ b/qemu/hw/display/virtio-gpu.c
@@ -7,10 +7,11 @@
  *     Dave Airlie <airlied@redhat.com>
  *     Gerd Hoffmann <kraxel@redhat.com>
  *
- * This work is licensed under the terms of the GNU GPL, version 2.
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/iov.h"
 #include "ui/console.h"
@@ -22,6 +23,23 @@
 static struct virtio_gpu_simple_resource*
 virtio_gpu_find_resource(VirtIOGPU *g, uint32_t resource_id);
 
+#ifdef CONFIG_VIRGL
+#include "virglrenderer.h"
+#define VIRGL(_g, _virgl, _simple, ...)                     \
+    do {                                                    \
+        if (_g->use_virgl_renderer) {                       \
+            _virgl(__VA_ARGS__);                            \
+        } else {                                            \
+            _simple(__VA_ARGS__);                           \
+        }                                                   \
+    } while (0)
+#else
+#define VIRGL(_g, _virgl, _simple, ...)                 \
+    do {                                                \
+        _simple(__VA_ARGS__);                           \
+    } while (0)
+#endif
+
 static void update_cursor_data_simple(VirtIOGPU *g,
                                       struct virtio_gpu_scanout *s,
                                       uint32_t resource_id)
@@ -45,16 +63,49 @@ static void update_cursor_data_simple(VirtIOGPU *g,
            pixels * sizeof(uint32_t));
 }
 
+#ifdef CONFIG_VIRGL
+
+static void update_cursor_data_virgl(VirtIOGPU *g,
+                                     struct virtio_gpu_scanout *s,
+                                     uint32_t resource_id)
+{
+    uint32_t width, height;
+    uint32_t pixels, *data;
+
+    data = virgl_renderer_get_cursor_data(resource_id, &width, &height);
+    if (!data) {
+        return;
+    }
+
+    if (width != s->current_cursor->width ||
+        height != s->current_cursor->height) {
+        return;
+    }
+
+    pixels = s->current_cursor->width * s->current_cursor->height;
+    memcpy(s->current_cursor->data, data, pixels * sizeof(uint32_t));
+    free(data);
+}
+
+#endif
+
 static void update_cursor(VirtIOGPU *g, struct virtio_gpu_update_cursor *cursor)
 {
     struct virtio_gpu_scanout *s;
+    bool move = cursor->hdr.type != VIRTIO_GPU_CMD_MOVE_CURSOR;
 
     if (cursor->pos.scanout_id >= g->conf.max_outputs) {
         return;
     }
     s = &g->scanout[cursor->pos.scanout_id];
 
-    if (cursor->hdr.type != VIRTIO_GPU_CMD_MOVE_CURSOR) {
+    trace_virtio_gpu_update_cursor(cursor->pos.scanout_id,
+                                   cursor->pos.x,
+                                   cursor->pos.y,
+                                   move ? "move" : "update",
+                                   cursor->resource_id);
+
+    if (move) {
         if (!s->current_cursor) {
             s->current_cursor = cursor_alloc(64, 64);
         }
@@ -63,7 +114,8 @@ static void update_cursor(VirtIOGPU *g, struct virtio_gpu_update_cursor *cursor)
         s->current_cursor->hot_y = cursor->hot_y;
 
         if (cursor->resource_id > 0) {
-            update_cursor_data_simple(g, s, cursor->resource_id);
+            VIRGL(g, update_cursor_data_virgl, update_cursor_data_simple,
+                  g, s, cursor->resource_id);
         }
         dpy_cursor_define(s->con, s->current_cursor);
     }
@@ -92,9 +144,23 @@ static void virtio_gpu_set_config(VirtIODevice *vdev, const uint8_t *config)
 static uint64_t virtio_gpu_get_features(VirtIODevice *vdev, uint64_t features,
                                         Error **errp)
 {
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        features |= (1 << VIRTIO_GPU_F_VIRGL);
+    }
     return features;
 }
 
+static void virtio_gpu_set_features(VirtIODevice *vdev, uint64_t features)
+{
+    static const uint32_t virgl = (1 << VIRTIO_GPU_F_VIRGL);
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    g->use_virgl_renderer = ((features & virgl) == virgl);
+    trace_virtio_gpu_features(g->use_virgl_renderer);
+}
+
 static void virtio_gpu_notify_event(VirtIOGPU *g, uint32_t event_type)
 {
     g->virtio_config.events_read |= event_type;
@@ -506,10 +572,7 @@ static void virtio_gpu_set_scanout(VirtIOGPU *g,
         scanout->width != ss.r.width ||
         scanout->height != ss.r.height) {
         /* realloc the surface ptr */
-        scanout->ds = qemu_create_displaysurface_from
-            (ss.r.width, ss.r.height, format,
-             pixman_image_get_stride(res->image),
-             (uint8_t *)pixman_image_get_data(res->image) + offset);
+        scanout->ds = qemu_create_displaysurface_pixman(res->image);
         if (!scanout->ds) {
             cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
             return;
@@ -563,7 +626,6 @@ int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
                           __func__, ab->resource_id, i);
             virtio_gpu_cleanup_mapping_iov(*iov, i);
             g_free(ents);
-            g_free(*iov);
             *iov = NULL;
             return -1;
         }
@@ -580,12 +642,12 @@ void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count)
         cpu_physical_memory_unmap(iov[i].iov_base, iov[i].iov_len, 1,
                                   iov[i].iov_len);
     }
+    g_free(iov);
 }
 
 static void virtio_gpu_cleanup_mapping(struct virtio_gpu_simple_resource *res)
 {
     virtio_gpu_cleanup_mapping_iov(res->iov, res->iov_cnt);
-    g_free(res->iov);
     res->iov = NULL;
     res->iov_cnt = 0;
 }
@@ -690,6 +752,39 @@ static void virtio_gpu_handle_cursor_cb(VirtIODevice *vdev, VirtQueue *vq)
     qemu_bh_schedule(g->cursor_bh);
 }
 
+void virtio_gpu_process_cmdq(VirtIOGPU *g)
+{
+    struct virtio_gpu_ctrl_command *cmd;
+
+    while (!QTAILQ_EMPTY(&g->cmdq)) {
+        cmd = QTAILQ_FIRST(&g->cmdq);
+
+        /* process command */
+        VIRGL(g, virtio_gpu_virgl_process_cmd, virtio_gpu_simple_process_cmd,
+              g, cmd);
+        if (cmd->waiting) {
+            break;
+        }
+        QTAILQ_REMOVE(&g->cmdq, cmd, next);
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            g->stats.requests++;
+        }
+
+        if (!cmd->finished) {
+            QTAILQ_INSERT_TAIL(&g->fenceq, cmd, next);
+            g->inflight++;
+            if (virtio_gpu_stats_enabled(g->conf)) {
+                if (g->stats.max_inflight < g->inflight) {
+                    g->stats.max_inflight = g->inflight;
+                }
+                fprintf(stderr, "inflight: %3d (+)\r", g->inflight);
+            }
+        } else {
+            g_free(cmd);
+        }
+    }
+}
+
 static void virtio_gpu_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOGPU *g = VIRTIO_GPU(vdev);
@@ -699,25 +794,30 @@ static void virtio_gpu_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
         return;
     }
 
-    cmd = g_new(struct virtio_gpu_ctrl_command, 1);
-    while (virtqueue_pop(vq, &cmd->elem)) {
+#ifdef CONFIG_VIRGL
+    if (!g->renderer_inited && g->use_virgl_renderer) {
+        virtio_gpu_virgl_init(g);
+        g->renderer_inited = true;
+    }
+#endif
+
+    cmd = virtqueue_pop(vq, sizeof(struct virtio_gpu_ctrl_command));
+    while (cmd) {
         cmd->vq = vq;
         cmd->error = 0;
         cmd->finished = false;
-        g->stats.requests++;
+        cmd->waiting = false;
+        QTAILQ_INSERT_TAIL(&g->cmdq, cmd, next);
+        cmd = virtqueue_pop(vq, sizeof(struct virtio_gpu_ctrl_command));
+    }
 
-        virtio_gpu_simple_process_cmd(g, cmd);
-        if (!cmd->finished) {
-            QTAILQ_INSERT_TAIL(&g->fenceq, cmd, next);
-            g->stats.inflight++;
-            if (g->stats.max_inflight < g->stats.inflight) {
-                g->stats.max_inflight = g->stats.inflight;
-            }
-            fprintf(stderr, "inflight: %3d (+)\r", g->stats.inflight);
-            cmd = g_new(struct virtio_gpu_ctrl_command, 1);
-        }
+    virtio_gpu_process_cmdq(g);
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_fence_poll(g);
     }
-    g_free(cmd);
+#endif
 }
 
 static void virtio_gpu_ctrl_bh(void *opaque)
@@ -729,15 +829,20 @@ static void virtio_gpu_ctrl_bh(void *opaque)
 static void virtio_gpu_handle_cursor(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOGPU *g = VIRTIO_GPU(vdev);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     size_t s;
     struct virtio_gpu_update_cursor cursor_info;
 
     if (!virtio_queue_ready(vq)) {
         return;
     }
-    while (virtqueue_pop(vq, &elem)) {
-        s = iov_to_buf(elem.out_sg, elem.out_num, 0,
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        s = iov_to_buf(elem->out_sg, elem->out_num, 0,
                        &cursor_info, sizeof(cursor_info));
         if (s != sizeof(cursor_info)) {
             qemu_log_mask(LOG_GUEST_ERROR,
@@ -746,8 +851,9 @@ static void virtio_gpu_handle_cursor(VirtIODevice *vdev, VirtQueue *vq)
         } else {
             update_cursor(g, &cursor_info);
         }
-        virtqueue_push(vq, &elem, 0);
+        virtqueue_push(vq, elem, 0);
         virtio_notify(vdev, vq);
+        g_free(elem);
     }
 }
 
@@ -793,17 +899,34 @@ static int virtio_gpu_ui_info(void *opaque, uint32_t idx, QemuUIInfo *info)
     return 0;
 }
 
+static void virtio_gpu_gl_block(void *opaque, bool block)
+{
+    VirtIOGPU *g = opaque;
+
+    g->renderer_blocked = block;
+    if (!block) {
+        virtio_gpu_process_cmdq(g);
+    }
+}
+
 const GraphicHwOps virtio_gpu_ops = {
     .invalidate = virtio_gpu_invalidate_display,
     .gfx_update = virtio_gpu_update_display,
     .text_update = virtio_gpu_text_update,
     .ui_info = virtio_gpu_ui_info,
+    .gl_block = virtio_gpu_gl_block,
+};
+
+static const VMStateDescription vmstate_virtio_gpu_unmigratable = {
+    .name = "virtio-gpu",
+    .unmigratable = 1,
 };
 
 static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
     VirtIOGPU *g = VIRTIO_GPU(qdev);
+    bool have_virgl;
     int i;
 
     g->config_size = sizeof(struct virtio_gpu_config);
@@ -814,12 +937,30 @@ static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
     g->req_state[0].width = 1024;
     g->req_state[0].height = 768;
 
-    g->ctrl_vq   = virtio_add_queue(vdev, 64, virtio_gpu_handle_ctrl_cb);
-    g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+    g->use_virgl_renderer = false;
+#if !defined(CONFIG_VIRGL) || defined(HOST_WORDS_BIGENDIAN)
+    have_virgl = false;
+#else
+    have_virgl = display_opengl;
+#endif
+    if (!have_virgl) {
+        g->conf.flags &= ~(1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED);
+    }
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        /* use larger control queue in 3d mode */
+        g->ctrl_vq   = virtio_add_queue(vdev, 256, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+        g->virtio_config.num_capsets = 1;
+    } else {
+        g->ctrl_vq   = virtio_add_queue(vdev, 64, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+    }
 
     g->ctrl_bh = qemu_bh_new(virtio_gpu_ctrl_bh, g);
     g->cursor_bh = qemu_bh_new(virtio_gpu_cursor_bh, g);
     QTAILQ_INIT(&g->reslist);
+    QTAILQ_INIT(&g->cmdq);
     QTAILQ_INIT(&g->fenceq);
 
     g->enabled_output_bitmask = 1;
@@ -832,6 +973,8 @@ static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
             dpy_gfx_replace_surface(g->scanout[i].con, NULL);
         }
     }
+
+    vmstate_register(qdev, -1, &vmstate_virtio_gpu_unmigratable, g);
 }
 
 static void virtio_gpu_instance_init(Object *obj)
@@ -869,10 +1012,23 @@ static void virtio_gpu_reset(VirtIODevice *vdev)
         g->scanout[i].ds = NULL;
     }
     g->enabled_output_bitmask = 1;
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_reset(g);
+        g->use_virgl_renderer = 0;
+    }
+#endif
 }
 
 static Property virtio_gpu_properties[] = {
     DEFINE_PROP_UINT32("max_outputs", VirtIOGPU, conf.max_outputs, 1),
+#ifdef CONFIG_VIRGL
+    DEFINE_PROP_BIT("virgl", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_VIRGL_ENABLED, true),
+    DEFINE_PROP_BIT("stats", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_STATS_ENABLED, false),
+#endif
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -885,6 +1041,7 @@ static void virtio_gpu_class_init(ObjectClass *klass, void *data)
     vdc->get_config = virtio_gpu_get_config;
     vdc->set_config = virtio_gpu_set_config;
     vdc->get_features = virtio_gpu_get_features;
+    vdc->set_features = virtio_gpu_set_features;
 
     vdc->reset = virtio_gpu_reset;
 
@@ -917,3 +1074,14 @@ QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_mem_entry)               != 16);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_attach_backing) != 32);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_detach_backing) != 32);
 QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_display_info)       != 408);
+
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_transfer_host_3d)        != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_create_3d)      != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_create)              != 96);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_destroy)             != 24);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_resource)            != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_cmd_submit)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset_info)         != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset_info)        != 40);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset)             != 24);
diff --git a/qemu/hw/display/virtio-vga.c b/qemu/hw/display/virtio-vga.c
index f7e539fe9..e58b165ae 100644
--- a/qemu/hw/display/virtio-vga.c
+++ b/qemu/hw/display/virtio-vga.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "ui/console.h"
@@ -65,11 +66,21 @@ static int virtio_vga_ui_info(void *opaque, uint32_t idx, QemuUIInfo *info)
     return -1;
 }
 
+static void virtio_vga_gl_block(void *opaque, bool block)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (virtio_gpu_ops.gl_block) {
+        virtio_gpu_ops.gl_block(&vvga->vdev, block);
+    }
+}
+
 static const GraphicHwOps virtio_vga_ops = {
     .invalidate = virtio_vga_invalidate_display,
     .gfx_update = virtio_vga_update_display,
     .text_update = virtio_vga_text_update,
     .ui_info = virtio_vga_ui_info,
+    .gl_block = virtio_vga_gl_block,
 };
 
 /* VGA device wrapper around PCI device around virtio GPU */
diff --git a/qemu/hw/display/vmware_vga.c b/qemu/hw/display/vmware_vga.c
index 7f397d3c2..0c63fa851 100644
--- a/qemu/hw/display/vmware_vga.c
+++ b/qemu/hw/display/vmware_vga.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "trace.h"
@@ -488,10 +490,10 @@ static inline int vmsvga_fill_rect(struct vmsvga_state_s *s,
 #endif
 
 struct vmsvga_cursor_definition_s {
-    int width;
-    int height;
+    uint32_t width;
+    uint32_t height;
     int id;
-    int bpp;
+    uint32_t bpp;
     int hot_x;
     int hot_y;
     uint32_t mask[1024];
@@ -658,7 +660,10 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             cursor.bpp = vmsvga_fifo_read(s);
 
             args = SVGA_BITMAP_SIZE(x, y) + SVGA_PIXMAP_SIZE(x, y, cursor.bpp);
-            if (SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
+            if (cursor.width > 256 ||
+                cursor.height > 256 ||
+                cursor.bpp > 32 ||
+                SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
                 SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image) {
                     goto badcmd;
             }
@@ -1244,7 +1249,7 @@ static void vmsvga_init(DeviceState *dev, struct vmsvga_state_s *s,
 
     s->fifo_size = SVGA_FIFO_SIZE;
     memory_region_init_ram(&s->fifo_ram, NULL, "vmsvga.fifo", s->fifo_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->fifo_ram);
     s->fifo_ptr = memory_region_get_ram_ptr(&s->fifo_ram);
 
diff --git a/qemu/hw/display/xenfb.c b/qemu/hw/display/xenfb.c
index 5e324ef62..9866dfda5 100644
--- a/qemu/hw/display/xenfb.c
+++ b/qemu/hw/display/xenfb.c
@@ -24,16 +24,8 @@
  *  with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
 
 #include "hw/hw.h"
 #include "ui/console.h"
@@ -95,23 +87,24 @@ struct XenFB {
 
 static int common_bind(struct common *c)
 {
-    uint64_t mfn;
+    uint64_t val;
+    xen_pfn_t mfn;
 
-    if (xenstore_read_fe_uint64(&c->xendev, "page-ref", &mfn) == -1)
+    if (xenstore_read_fe_uint64(&c->xendev, "page-ref", &val) == -1)
 	return -1;
-    assert(mfn == (xen_pfn_t)mfn);
+    mfn = (xen_pfn_t)val;
+    assert(val == mfn);
 
     if (xenstore_read_fe_int(&c->xendev, "event-channel", &c->xendev.remote_port) == -1)
 	return -1;
 
-    c->page = xc_map_foreign_range(xen_xc, c->xendev.dom,
-				   XC_PAGE_SIZE,
-				   PROT_READ | PROT_WRITE, mfn);
+    c->page = xenforeignmemory_map(xen_fmem, c->xendev.dom,
+                                   PROT_READ | PROT_WRITE, 1, &mfn, NULL);
     if (c->page == NULL)
 	return -1;
 
     xen_be_bind_evtchn(&c->xendev);
-    xen_be_printf(&c->xendev, 1, "ring mfn %"PRIx64", remote-port %d, local-port %d\n",
+    xen_be_printf(&c->xendev, 1, "ring mfn %"PRI_xen_pfn", remote-port %d, local-port %d\n",
 		  mfn, c->xendev.remote_port, c->xendev.local_port);
 
     return 0;
@@ -121,7 +114,7 @@ static void common_unbind(struct common *c)
 {
     xen_be_unbind_evtchn(&c->xendev);
     if (c->page) {
-	munmap(c->page, XC_PAGE_SIZE);
+        xenforeignmemory_unmap(xen_fmem, c->page, 1);
 	c->page = NULL;
     }
 }
@@ -248,9 +241,7 @@ static int xenfb_send_motion(struct XenInput *xenfb,
     event.type = XENKBD_TYPE_MOTION;
     event.motion.rel_x = rel_x;
     event.motion.rel_y = rel_y;
-#if __XEN_LATEST_INTERFACE_VERSION__ >= 0x00030207
     event.motion.rel_z = rel_z;
-#endif
 
     return xenfb_kbd_event(xenfb, &event);
 }
@@ -265,12 +256,7 @@ static int xenfb_send_position(struct XenInput *xenfb,
     event.type = XENKBD_TYPE_POS;
     event.pos.abs_x = abs_x;
     event.pos.abs_y = abs_y;
-#if __XEN_LATEST_INTERFACE_VERSION__ == 0x00030207
-    event.pos.abs_z = z;
-#endif
-#if __XEN_LATEST_INTERFACE_VERSION__ >= 0x00030208
     event.pos.rel_z = z;
-#endif
 
     return xenfb_kbd_event(xenfb, &event);
 }
@@ -494,15 +480,15 @@ static int xenfb_map_fb(struct XenFB *xenfb)
     fbmfns = g_malloc0(sizeof(xen_pfn_t) * xenfb->fbpages);
 
     xenfb_copy_mfns(mode, n_fbdirs, pgmfns, pd);
-    map = xc_map_foreign_pages(xen_xc, xenfb->c.xendev.dom,
-			       PROT_READ, pgmfns, n_fbdirs);
+    map = xenforeignmemory_map(xen_fmem, xenfb->c.xendev.dom,
+                               PROT_READ, n_fbdirs, pgmfns, NULL);
     if (map == NULL)
 	goto out;
     xenfb_copy_mfns(mode, xenfb->fbpages, fbmfns, map);
-    munmap(map, n_fbdirs * XC_PAGE_SIZE);
+    xenforeignmemory_unmap(xen_fmem, map, n_fbdirs);
 
-    xenfb->pixels = xc_map_foreign_pages(xen_xc, xenfb->c.xendev.dom,
-            PROT_READ, fbmfns, xenfb->fbpages);
+    xenfb->pixels = xenforeignmemory_map(xen_fmem, xenfb->c.xendev.dom,
+            PROT_READ, xenfb->fbpages, fbmfns, NULL);
     if (xenfb->pixels == NULL)
 	goto out;
 
@@ -784,18 +770,21 @@ static void xenfb_invalidate(void *opaque)
 
 static void xenfb_handle_events(struct XenFB *xenfb)
 {
-    uint32_t prod, cons;
+    uint32_t prod, cons, out_cons;
     struct xenfb_page *page = xenfb->c.page;
 
     prod = page->out_prod;
-    if (prod == page->out_cons)
-	return;
+    out_cons = page->out_cons;
+    if (prod - out_cons > XENFB_OUT_RING_LEN) {
+        return;
+    }
     xen_rmb();		/* ensure we see ring contents up to prod */
-    for (cons = page->out_cons; cons != prod; cons++) {
+    for (cons = out_cons; cons != prod; cons++) {
 	union xenfb_out_event *event = &XENFB_OUT_RING_REF(page, cons);
+        uint8_t type = event->type;
 	int x, y, w, h;
 
-	switch (event->type) {
+	switch (type) {
 	case XENFB_TYPE_UPDATE:
 	    if (xenfb->up_count == UP_QUEUE)
 		xenfb->up_fullscreen = 1;
@@ -909,6 +898,7 @@ static void fb_disconnect(struct XenDevice *xendev)
      *   Replacing the framebuffer with anonymous shared memory
      *   instead.  This releases the guest pages and keeps qemu happy.
      */
+    xenforeignmemory_unmap(xen_fmem, fb->pixels, fb->fbpages);
     fb->pixels = mmap(fb->pixels, fb->fbpages * XC_PAGE_SIZE,
                       PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON,
                       -1, 0);
diff --git a/qemu/hw/dma/Makefile.objs b/qemu/hw/dma/Makefile.objs
index 0e65ed0d7..a1abbcf74 100644
--- a/qemu/hw/dma/Makefile.objs
+++ b/qemu/hw/dma/Makefile.objs
@@ -11,3 +11,4 @@ common-obj-$(CONFIG_SUN4M) += sun4m_iommu.o
 
 obj-$(CONFIG_OMAP) += omap_dma.o soc_dma.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_dma.o
+obj-$(CONFIG_RASPI) += bcm2835_dma.o
diff --git a/qemu/hw/dma/bcm2835_dma.c b/qemu/hw/dma/bcm2835_dma.c
new file mode 100644
index 000000000..542117599
--- /dev/null
+++ b/qemu/hw/dma/bcm2835_dma.c
@@ -0,0 +1,409 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/dma/bcm2835_dma.h"
+
+/* DMA CS Control and Status bits */
+#define BCM2708_DMA_ACTIVE      (1 << 0)
+#define BCM2708_DMA_END         (1 << 1) /* GE */
+#define BCM2708_DMA_INT         (1 << 2)
+#define BCM2708_DMA_ISPAUSED    (1 << 4)  /* Pause requested or not active */
+#define BCM2708_DMA_ISHELD      (1 << 5)  /* Is held by DREQ flow control */
+#define BCM2708_DMA_ERR         (1 << 8)
+#define BCM2708_DMA_ABORT       (1 << 30) /* stop current CB, go to next, WO */
+#define BCM2708_DMA_RESET       (1 << 31) /* WO, self clearing */
+
+/* DMA control block "info" field bits */
+#define BCM2708_DMA_INT_EN      (1 << 0)
+#define BCM2708_DMA_TDMODE      (1 << 1)
+#define BCM2708_DMA_WAIT_RESP   (1 << 3)
+#define BCM2708_DMA_D_INC       (1 << 4)
+#define BCM2708_DMA_D_WIDTH     (1 << 5)
+#define BCM2708_DMA_D_DREQ      (1 << 6)
+#define BCM2708_DMA_D_IGNORE    (1 << 7)
+#define BCM2708_DMA_S_INC       (1 << 8)
+#define BCM2708_DMA_S_WIDTH     (1 << 9)
+#define BCM2708_DMA_S_DREQ      (1 << 10)
+#define BCM2708_DMA_S_IGNORE    (1 << 11)
+
+/* Register offsets */
+#define BCM2708_DMA_CS          0x00 /* Control and Status */
+#define BCM2708_DMA_ADDR        0x04 /* Control block address */
+/* the current control block appears in the following registers - read only */
+#define BCM2708_DMA_INFO        0x08
+#define BCM2708_DMA_SOURCE_AD   0x0c
+#define BCM2708_DMA_DEST_AD     0x10
+#define BCM2708_DMA_TXFR_LEN    0x14
+#define BCM2708_DMA_STRIDE      0x18
+#define BCM2708_DMA_NEXTCB      0x1C
+#define BCM2708_DMA_DEBUG       0x20
+
+#define BCM2708_DMA_INT_STATUS  0xfe0 /* Interrupt status of each channel */
+#define BCM2708_DMA_ENABLE      0xff0 /* Global enable bits for each channel */
+
+#define BCM2708_DMA_CS_RW_MASK  0x30ff0001 /* All RW bits in DMA_CS */
+
+static void bcm2835_dma_update(BCM2835DMAState *s, unsigned c)
+{
+    BCM2835DMAChan *ch = &s->chan[c];
+    uint32_t data, xlen, ylen;
+    int16_t dst_stride, src_stride;
+
+    if (!(s->enable & (1 << c))) {
+        return;
+    }
+
+    while ((s->enable & (1 << c)) && (ch->conblk_ad != 0)) {
+        /* CB fetch */
+        ch->ti = ldl_le_phys(&s->dma_as, ch->conblk_ad);
+        ch->source_ad = ldl_le_phys(&s->dma_as, ch->conblk_ad + 4);
+        ch->dest_ad = ldl_le_phys(&s->dma_as, ch->conblk_ad + 8);
+        ch->txfr_len = ldl_le_phys(&s->dma_as, ch->conblk_ad + 12);
+        ch->stride = ldl_le_phys(&s->dma_as, ch->conblk_ad + 16);
+        ch->nextconbk = ldl_le_phys(&s->dma_as, ch->conblk_ad + 20);
+
+        if (ch->ti & BCM2708_DMA_TDMODE) {
+            /* 2D transfer mode */
+            ylen = (ch->txfr_len >> 16) & 0x3fff;
+            xlen = ch->txfr_len & 0xffff;
+            dst_stride = ch->stride >> 16;
+            src_stride = ch->stride & 0xffff;
+        } else {
+            ylen = 1;
+            xlen = ch->txfr_len;
+            dst_stride = 0;
+            src_stride = 0;
+        }
+
+        while (ylen != 0) {
+            /* Normal transfer mode */
+            while (xlen != 0) {
+                if (ch->ti & BCM2708_DMA_S_IGNORE) {
+                    /* Ignore reads */
+                    data = 0;
+                } else {
+                    data = ldl_le_phys(&s->dma_as, ch->source_ad);
+                }
+                if (ch->ti & BCM2708_DMA_S_INC) {
+                    ch->source_ad += 4;
+                }
+
+                if (ch->ti & BCM2708_DMA_D_IGNORE) {
+                    /* Ignore writes */
+                } else {
+                    stl_le_phys(&s->dma_as, ch->dest_ad, data);
+                }
+                if (ch->ti & BCM2708_DMA_D_INC) {
+                    ch->dest_ad += 4;
+                }
+
+                /* update remaining transfer length */
+                xlen -= 4;
+                if (ch->ti & BCM2708_DMA_TDMODE) {
+                    ch->txfr_len = (ylen << 16) | xlen;
+                } else {
+                    ch->txfr_len = xlen;
+                }
+            }
+
+            if (--ylen != 0) {
+                ch->source_ad += src_stride;
+                ch->dest_ad += dst_stride;
+            }
+        }
+        ch->cs |= BCM2708_DMA_END;
+        if (ch->ti & BCM2708_DMA_INT_EN) {
+            ch->cs |= BCM2708_DMA_INT;
+            s->int_status |= (1 << c);
+            qemu_set_irq(ch->irq, 1);
+        }
+
+        /* Process next CB */
+        ch->conblk_ad = ch->nextconbk;
+    }
+
+    ch->cs &= ~BCM2708_DMA_ACTIVE;
+    ch->cs |= BCM2708_DMA_ISPAUSED;
+}
+
+static void bcm2835_dma_chan_reset(BCM2835DMAChan *ch)
+{
+    ch->cs = 0;
+    ch->conblk_ad = 0;
+}
+
+static uint64_t bcm2835_dma_read(BCM2835DMAState *s, hwaddr offset,
+                                 unsigned size, unsigned c)
+{
+    BCM2835DMAChan *ch;
+    uint32_t res = 0;
+
+    assert(size == 4);
+    assert(c < BCM2835_DMA_NCHANS);
+
+    ch = &s->chan[c];
+
+    switch (offset) {
+    case BCM2708_DMA_CS:
+        res = ch->cs;
+        break;
+    case BCM2708_DMA_ADDR:
+        res = ch->conblk_ad;
+        break;
+    case BCM2708_DMA_INFO:
+        res = ch->ti;
+        break;
+    case BCM2708_DMA_SOURCE_AD:
+        res = ch->source_ad;
+        break;
+    case BCM2708_DMA_DEST_AD:
+        res = ch->dest_ad;
+        break;
+    case BCM2708_DMA_TXFR_LEN:
+        res = ch->txfr_len;
+        break;
+    case BCM2708_DMA_STRIDE:
+        res = ch->stride;
+        break;
+    case BCM2708_DMA_NEXTCB:
+        res = ch->nextconbk;
+        break;
+    case BCM2708_DMA_DEBUG:
+        res = ch->debug;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        break;
+    }
+    return res;
+}
+
+static void bcm2835_dma_write(BCM2835DMAState *s, hwaddr offset,
+                              uint64_t value, unsigned size, unsigned c)
+{
+    BCM2835DMAChan *ch;
+    uint32_t oldcs;
+
+    assert(size == 4);
+    assert(c < BCM2835_DMA_NCHANS);
+
+    ch = &s->chan[c];
+
+    switch (offset) {
+    case BCM2708_DMA_CS:
+        oldcs = ch->cs;
+        if (value & BCM2708_DMA_RESET) {
+            bcm2835_dma_chan_reset(ch);
+        }
+        if (value & BCM2708_DMA_ABORT) {
+            /* abort is a no-op, since we always run to completion */
+        }
+        if (value & BCM2708_DMA_END) {
+            ch->cs &= ~BCM2708_DMA_END;
+        }
+        if (value & BCM2708_DMA_INT) {
+            ch->cs &= ~BCM2708_DMA_INT;
+            s->int_status &= ~(1 << c);
+            qemu_set_irq(ch->irq, 0);
+        }
+        ch->cs &= ~BCM2708_DMA_CS_RW_MASK;
+        ch->cs |= (value & BCM2708_DMA_CS_RW_MASK);
+        if (!(oldcs & BCM2708_DMA_ACTIVE) && (ch->cs & BCM2708_DMA_ACTIVE)) {
+            bcm2835_dma_update(s, c);
+        }
+        break;
+    case BCM2708_DMA_ADDR:
+        ch->conblk_ad = value;
+        break;
+    case BCM2708_DMA_DEBUG:
+        ch->debug = value;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        break;
+    }
+}
+
+static uint64_t bcm2835_dma0_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835DMAState *s = opaque;
+
+    if (offset < 0xf00) {
+        return bcm2835_dma_read(s, (offset & 0xff), size, (offset >> 8) & 0xf);
+    } else {
+        switch (offset) {
+        case BCM2708_DMA_INT_STATUS:
+            return s->int_status;
+        case BCM2708_DMA_ENABLE:
+            return s->enable;
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                          __func__, offset);
+            return 0;
+        }
+    }
+}
+
+static uint64_t bcm2835_dma15_read(void *opaque, hwaddr offset, unsigned size)
+{
+    return bcm2835_dma_read(opaque, (offset & 0xff), size, 15);
+}
+
+static void bcm2835_dma0_write(void *opaque, hwaddr offset, uint64_t value,
+                               unsigned size)
+{
+    BCM2835DMAState *s = opaque;
+
+    if (offset < 0xf00) {
+        bcm2835_dma_write(s, (offset & 0xff), value, size, (offset >> 8) & 0xf);
+    } else {
+        switch (offset) {
+        case BCM2708_DMA_INT_STATUS:
+            break;
+        case BCM2708_DMA_ENABLE:
+            s->enable = (value & 0xffff);
+            break;
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                          __func__, offset);
+        }
+    }
+
+}
+
+static void bcm2835_dma15_write(void *opaque, hwaddr offset, uint64_t value,
+                                unsigned size)
+{
+    bcm2835_dma_write(opaque, (offset & 0xff), value, size, 15);
+}
+
+static const MemoryRegionOps bcm2835_dma0_ops = {
+    .read = bcm2835_dma0_read,
+    .write = bcm2835_dma0_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const MemoryRegionOps bcm2835_dma15_ops = {
+    .read = bcm2835_dma15_read,
+    .write = bcm2835_dma15_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_dma_chan = {
+    .name = TYPE_BCM2835_DMA "-chan",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(cs, BCM2835DMAChan),
+        VMSTATE_UINT32(conblk_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(ti, BCM2835DMAChan),
+        VMSTATE_UINT32(source_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(dest_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(txfr_len, BCM2835DMAChan),
+        VMSTATE_UINT32(stride, BCM2835DMAChan),
+        VMSTATE_UINT32(nextconbk, BCM2835DMAChan),
+        VMSTATE_UINT32(debug, BCM2835DMAChan),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_bcm2835_dma = {
+    .name = TYPE_BCM2835_DMA,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_ARRAY(chan, BCM2835DMAState, BCM2835_DMA_NCHANS, 1,
+                             vmstate_bcm2835_dma_chan, BCM2835DMAChan),
+        VMSTATE_UINT32(int_status, BCM2835DMAState),
+        VMSTATE_UINT32(enable, BCM2835DMAState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_dma_init(Object *obj)
+{
+    BCM2835DMAState *s = BCM2835_DMA(obj);
+    int n;
+
+    /* DMA channels 0-14 occupy a contiguous block of IO memory, along
+     * with the global enable and interrupt status bits. Channel 15
+     * has the same register map, but is mapped at a discontiguous
+     * address in a separate IO block.
+     */
+    memory_region_init_io(&s->iomem0, OBJECT(s), &bcm2835_dma0_ops, s,
+                          TYPE_BCM2835_DMA, 0x1000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem0);
+
+    memory_region_init_io(&s->iomem15, OBJECT(s), &bcm2835_dma15_ops, s,
+                          TYPE_BCM2835_DMA "-chan15", 0x100);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem15);
+
+    for (n = 0; n < 16; n++) {
+        sysbus_init_irq(SYS_BUS_DEVICE(s), &s->chan[n].irq);
+    }
+}
+
+static void bcm2835_dma_reset(DeviceState *dev)
+{
+    BCM2835DMAState *s = BCM2835_DMA(dev);
+    int n;
+
+    s->enable = 0xffff;
+    s->int_status = 0;
+    for (n = 0; n < BCM2835_DMA_NCHANS; n++) {
+        bcm2835_dma_chan_reset(&s->chan[n]);
+    }
+}
+
+static void bcm2835_dma_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835DMAState *s = BCM2835_DMA(dev);
+    Error *err = NULL;
+    Object *obj;
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    bcm2835_dma_reset(dev);
+}
+
+static void bcm2835_dma_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = bcm2835_dma_realize;
+    dc->reset = bcm2835_dma_reset;
+    dc->vmsd = &vmstate_bcm2835_dma;
+}
+
+static TypeInfo bcm2835_dma_info = {
+    .name          = TYPE_BCM2835_DMA,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835DMAState),
+    .class_init    = bcm2835_dma_class_init,
+    .instance_init = bcm2835_dma_init,
+};
+
+static void bcm2835_dma_register_types(void)
+{
+    type_register_static(&bcm2835_dma_info);
+}
+
+type_init(bcm2835_dma_register_types)
diff --git a/qemu/hw/dma/etraxfs_dma.c b/qemu/hw/dma/etraxfs_dma.c
index 359951341..d5650eb88 100644
--- a/qemu/hw/dma/etraxfs_dma.c
+++ b/qemu/hw/dma/etraxfs_dma.c
@@ -21,8 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdio.h>
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "exec/address-spaces.h"
 #include "qemu-common.h"
@@ -441,13 +440,16 @@ static int channel_out_run(struct fs_dma_ctrl *ctrl, int c)
 		D(printf("channel %d pushes %x %u bytes eop=%u\n", c,
 		         saved_data_buf, len, out_eop));
 
-		if (ctrl->channels[c].client->client.push)
-			ctrl->channels[c].client->client.push(
-				ctrl->channels[c].client->client.opaque,
-				buf, len, out_eop);
-		else
+		if (ctrl->channels[c].client->client.push) {
+                        if (len > 0) {
+				ctrl->channels[c].client->client.push(
+					ctrl->channels[c].client->client.opaque,
+					buf, len, out_eop);
+			}
+		} else {
 			printf("WARNING: DMA ch%d dataloss,"
 			       " no attached client.\n", c);
+		}
 
 		saved_data_buf += len;
 
diff --git a/qemu/hw/dma/i82374.c b/qemu/hw/dma/i82374.c
index b8ad2e64e..6c0f975df 100644
--- a/qemu/hw/dma/i82374.c
+++ b/qemu/hw/dma/i82374.c
@@ -22,8 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/isa.h"
 
+#define TYPE_I82374 "i82374"
+#define I82374(obj) OBJECT_CHECK(I82374State, (obj), TYPE_I82374)
+
 //#define DEBUG_I82374
 
 #ifdef DEBUG_I82374
@@ -37,8 +41,10 @@ do {} while (0)
 do { fprintf(stderr, "i82374 ERROR: " fmt , ## __VA_ARGS__); } while (0)
 
 typedef struct I82374State {
+    ISADevice parent_obj;
+
+    uint32_t iobase;
     uint8_t commands[8];
-    qemu_irq out;
     PortioList port_list;
 } I82374State;
 
@@ -99,32 +105,6 @@ static uint32_t i82374_read_descriptor(void *opaque, uint32_t nport)
     return val;
 }
 
-static void i82374_realize(I82374State *s, Error **errp)
-{
-    DMA_init(1, &s->out);
-    memset(s->commands, 0, sizeof(s->commands));
-}
-
-#define TYPE_I82374 "i82374"
-#define I82374(obj) OBJECT_CHECK(ISAi82374State, (obj), TYPE_I82374)
-
-typedef struct ISAi82374State {
-    ISADevice parent_obj;
-
-    uint32_t iobase;
-    I82374State state;
-} ISAi82374State;
-
-static const VMStateDescription vmstate_isa_i82374 = {
-    .name = "isa-i82374",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .fields = (VMStateField[]) {
-        VMSTATE_STRUCT(state, ISAi82374State, 0, vmstate_i82374, I82374State),
-        VMSTATE_END_OF_LIST()
-    },
-};
-
 static const MemoryRegionPortio i82374_portio_list[] = {
     { 0x0A, 1, 1, .read = i82374_read_isr, },
     { 0x10, 8, 1, .write = i82374_write_command, },
@@ -134,23 +114,21 @@ static const MemoryRegionPortio i82374_portio_list[] = {
     PORTIO_END_OF_LIST(),
 };
 
-static void i82374_isa_realize(DeviceState *dev, Error **errp)
+static void i82374_realize(DeviceState *dev, Error **errp)
 {
-    ISAi82374State *isa = I82374(dev);
-    I82374State *s = &isa->state;
+    I82374State *s = I82374(dev);
 
-    portio_list_init(&s->port_list, OBJECT(isa), i82374_portio_list, s,
+    portio_list_init(&s->port_list, OBJECT(s), i82374_portio_list, s,
                      "i82374");
-    portio_list_add(&s->port_list, isa_address_space_io(&isa->parent_obj),
-                    isa->iobase);
-
-    i82374_realize(s, errp);
+    portio_list_add(&s->port_list, isa_address_space_io(&s->parent_obj),
+                    s->iobase);
 
-    qdev_init_gpio_out(dev, &s->out, 1);
+    DMA_init(isa_bus_from_device(ISA_DEVICE(dev)), 1);
+    memset(s->commands, 0, sizeof(s->commands));
 }
 
 static Property i82374_properties[] = {
-    DEFINE_PROP_UINT32("iobase", ISAi82374State, iobase, 0x400),
+    DEFINE_PROP_UINT32("iobase", I82374State, iobase, 0x400),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -158,21 +136,21 @@ static void i82374_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     
-    dc->realize = i82374_isa_realize;
-    dc->vmsd = &vmstate_isa_i82374;
+    dc->realize = i82374_realize;
+    dc->vmsd = &vmstate_i82374;
     dc->props = i82374_properties;
 }
 
-static const TypeInfo i82374_isa_info = {
+static const TypeInfo i82374_info = {
     .name  = TYPE_I82374,
     .parent = TYPE_ISA_DEVICE,
-    .instance_size  = sizeof(ISAi82374State),
+    .instance_size  = sizeof(I82374State),
     .class_init = i82374_class_init,
 };
 
 static void i82374_register_types(void)
 {
-    type_register_static(&i82374_isa_info);
+    type_register_static(&i82374_info);
 }
 
 type_init(i82374_register_types)
diff --git a/qemu/hw/dma/i8257.c b/qemu/hw/dma/i8257.c
index a414029be..f345c5476 100644
--- a/qemu/hw/dma/i8257.c
+++ b/qemu/hw/dma/i8257.c
@@ -21,11 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
+#include "hw/isa/i8257.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
 
+#define I8257(obj) \
+    OBJECT_CHECK(I8257State, (obj), TYPE_I8257)
+
 /* #define DEBUG_DMA */
 
 #define dolog(...) fprintf (stderr, "dma: " __VA_ARGS__)
@@ -37,33 +42,9 @@
 #define ldebug(...)
 #endif
 
-struct dma_regs {
-    int now[2];
-    uint16_t base[2];
-    uint8_t mode;
-    uint8_t page;
-    uint8_t pageh;
-    uint8_t dack;
-    uint8_t eop;
-    DMA_transfer_handler transfer_handler;
-    void *opaque;
-};
-
 #define ADDR 0
 #define COUNT 1
 
-static struct dma_cont {
-    uint8_t status;
-    uint8_t command;
-    uint8_t mask;
-    uint8_t flip_flop;
-    int dshift;
-    struct dma_regs regs[4];
-    qemu_irq *cpu_request_exit;
-    MemoryRegion channel_io;
-    MemoryRegion cont_io;
-} dma_controllers[2];
-
 enum {
     CMD_MEMORY_TO_MEMORY = 0x01,
     CMD_FIXED_ADDRESS    = 0x02,
@@ -79,13 +60,13 @@ enum {
 
 };
 
-static void DMA_run (void);
+static void i8257_dma_run(void *opaque);
 
-static int channels[8] = {-1, 2, 3, 1, -1, -1, -1, 0};
+static const int channels[8] = {-1, 2, 3, 1, -1, -1, -1, 0};
 
-static void write_page (void *opaque, uint32_t nport, uint32_t data)
+static void i8257_write_page(void *opaque, uint32_t nport, uint32_t data)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -96,9 +77,9 @@ static void write_page (void *opaque, uint32_t nport, uint32_t data)
     d->regs[ichan].page = data;
 }
 
-static void write_pageh (void *opaque, uint32_t nport, uint32_t data)
+static void i8257_write_pageh(void *opaque, uint32_t nport, uint32_t data)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -109,9 +90,9 @@ static void write_pageh (void *opaque, uint32_t nport, uint32_t data)
     d->regs[ichan].pageh = data;
 }
 
-static uint32_t read_page (void *opaque, uint32_t nport)
+static uint32_t i8257_read_page(void *opaque, uint32_t nport)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -122,9 +103,9 @@ static uint32_t read_page (void *opaque, uint32_t nport)
     return d->regs[ichan].page;
 }
 
-static uint32_t read_pageh (void *opaque, uint32_t nport)
+static uint32_t i8257_read_pageh(void *opaque, uint32_t nport)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -135,16 +116,16 @@ static uint32_t read_pageh (void *opaque, uint32_t nport)
     return d->regs[ichan].pageh;
 }
 
-static inline void init_chan (struct dma_cont *d, int ichan)
+static inline void i8257_init_chan(I8257State *d, int ichan)
 {
-    struct dma_regs *r;
+    I8257Regs *r;
 
     r = d->regs + ichan;
     r->now[ADDR] = r->base[ADDR] << d->dshift;
     r->now[COUNT] = 0;
 }
 
-static inline int getff (struct dma_cont *d)
+static inline int i8257_getff(I8257State *d)
 {
     int ff;
 
@@ -153,11 +134,11 @@ static inline int getff (struct dma_cont *d)
     return ff;
 }
 
-static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
+static uint64_t i8257_read_chan(void *opaque, hwaddr nport, unsigned size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan, nreg, iport, ff, val, dir;
-    struct dma_regs *r;
+    I8257Regs *r;
 
     iport = (nport >> d->dshift) & 0x0f;
     ichan = iport >> 1;
@@ -165,7 +146,7 @@ static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
     r = d->regs + ichan;
 
     dir = ((r->mode >> 5) & 1) ? -1 : 1;
-    ff = getff (d);
+    ff = i8257_getff(d);
     if (nreg)
         val = (r->base[COUNT] << d->dshift) - r->now[COUNT];
     else
@@ -175,29 +156,29 @@ static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
     return (val >> (d->dshift + (ff << 3))) & 0xff;
 }
 
-static void write_chan(void *opaque, hwaddr nport, uint64_t data,
-                       unsigned size)
+static void i8257_write_chan(void *opaque, hwaddr nport, uint64_t data,
+                             unsigned int size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, ichan, nreg;
-    struct dma_regs *r;
+    I8257Regs *r;
 
     iport = (nport >> d->dshift) & 0x0f;
     ichan = iport >> 1;
     nreg = iport & 1;
     r = d->regs + ichan;
-    if (getff (d)) {
+    if (i8257_getff(d)) {
         r->base[nreg] = (r->base[nreg] & 0xff) | ((data << 8) & 0xff00);
-        init_chan (d, ichan);
+        i8257_init_chan(d, ichan);
     } else {
         r->base[nreg] = (r->base[nreg] & 0xff00) | (data & 0xff);
     }
 }
 
-static void write_cont(void *opaque, hwaddr nport, uint64_t data,
-                       unsigned size)
+static void i8257_write_cont(void *opaque, hwaddr nport, uint64_t data,
+                             unsigned int size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, ichan = 0;
 
     iport = (nport >> d->dshift) & 0x0f;
@@ -219,7 +200,7 @@ static void write_cont(void *opaque, hwaddr nport, uint64_t data,
             d->status &= ~(1 << (ichan + 4));
         }
         d->status &= ~(1 << ichan);
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x02:                  /* single mask */
@@ -227,7 +208,7 @@ static void write_cont(void *opaque, hwaddr nport, uint64_t data,
             d->mask |= 1 << (data & 3);
         else
             d->mask &= ~(1 << (data & 3));
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x03:                  /* mode */
@@ -262,12 +243,12 @@ static void write_cont(void *opaque, hwaddr nport, uint64_t data,
 
     case 0x06:                  /* clear mask for all channels */
         d->mask = 0;
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x07:                  /* write mask for all channels */
         d->mask = data;
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     default:
@@ -283,9 +264,9 @@ static void write_cont(void *opaque, hwaddr nport, uint64_t data,
 #endif
 }
 
-static uint64_t read_cont(void *opaque, hwaddr nport, unsigned size)
+static uint64_t i8257_read_cont(void *opaque, hwaddr nport, unsigned size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, val;
 
     iport = (nport >> d->dshift) & 0x0f;
@@ -306,37 +287,43 @@ static uint64_t read_cont(void *opaque, hwaddr nport, unsigned size)
     return val;
 }
 
-int DMA_get_channel_mode (int nchan)
+static IsaDmaTransferMode i8257_dma_get_transfer_mode(IsaDma *obj, int nchan)
 {
-    return dma_controllers[nchan > 3].regs[nchan & 3].mode;
+    I8257State *d = I8257(obj);
+    return (d->regs[nchan & 3].mode >> 2) & 3;
 }
 
-void DMA_hold_DREQ (int nchan)
+static bool i8257_dma_has_autoinitialization(IsaDma *obj, int nchan)
 {
-    int ncont, ichan;
+    I8257State *d = I8257(obj);
+    return (d->regs[nchan & 3].mode >> 4) & 1;
+}
+
+static void i8257_dma_hold_DREQ(IsaDma *obj, int nchan)
+{
+    I8257State *d = I8257(obj);
+    int ichan;
 
-    ncont = nchan > 3;
     ichan = nchan & 3;
-    linfo ("held cont=%d chan=%d\n", ncont, ichan);
-    dma_controllers[ncont].status |= 1 << (ichan + 4);
-    DMA_run();
+    d->status |= 1 << (ichan + 4);
+    i8257_dma_run(d);
 }
 
-void DMA_release_DREQ (int nchan)
+static void i8257_dma_release_DREQ(IsaDma *obj, int nchan)
 {
-    int ncont, ichan;
+    I8257State *d = I8257(obj);
+    int ichan;
 
-    ncont = nchan > 3;
     ichan = nchan & 3;
-    linfo ("released cont=%d chan=%d\n", ncont, ichan);
-    dma_controllers[ncont].status &= ~(1 << (ichan + 4));
-    DMA_run();
+    d->status &= ~(1 << (ichan + 4));
+    i8257_dma_run(d);
 }
 
-static void channel_run (int ncont, int ichan)
+static void i8257_channel_run(I8257State *d, int ichan)
 {
+    int ncont = d->dshift;
     int n;
-    struct dma_regs *r = &dma_controllers[ncont].regs[ichan];
+    I8257Regs *r = &d->regs[ichan];
 #ifdef DEBUG_DMA
     int dir, opmode;
 
@@ -355,68 +342,64 @@ static void channel_run (int ncont, int ichan)
                              r->now[COUNT], (r->base[COUNT] + 1) << ncont);
     r->now[COUNT] = n;
     ldebug ("dma_pos %d size %d\n", n, (r->base[COUNT] + 1) << ncont);
+    if (n == (r->base[COUNT] + 1) << ncont) {
+        ldebug("transfer done\n");
+        d->status |= (1 << ichan);
+    }
 }
 
-static QEMUBH *dma_bh;
-
-static void DMA_run (void)
+static void i8257_dma_run(void *opaque)
 {
-    struct dma_cont *d;
-    int icont, ichan;
+    I8257State *d = opaque;
+    int ichan;
     int rearm = 0;
-    static int running = 0;
 
-    if (running) {
+    if (d->running) {
         rearm = 1;
         goto out;
     } else {
-        running = 1;
+        d->running = 1;
     }
 
-    d = dma_controllers;
+    for (ichan = 0; ichan < 4; ichan++) {
+        int mask;
 
-    for (icont = 0; icont < 2; icont++, d++) {
-        for (ichan = 0; ichan < 4; ichan++) {
-            int mask;
+        mask = 1 << ichan;
 
-            mask = 1 << ichan;
-
-            if ((0 == (d->mask & mask)) && (0 != (d->status & (mask << 4)))) {
-                channel_run (icont, ichan);
-                rearm = 1;
-            }
+        if ((0 == (d->mask & mask)) && (0 != (d->status & (mask << 4)))) {
+            i8257_channel_run(d, ichan);
+            rearm = 1;
         }
     }
 
-    running = 0;
+    d->running = 0;
 out:
-    if (rearm)
-        qemu_bh_schedule_idle(dma_bh);
-}
-
-static void DMA_run_bh(void *unused)
-{
-    DMA_run();
+    if (rearm) {
+        qemu_bh_schedule_idle(d->dma_bh);
+        d->dma_bh_scheduled = true;
+    }
 }
 
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
+static void i8257_dma_register_channel(IsaDma *obj, int nchan,
+                                       IsaDmaTransferHandler transfer_handler,
+                                       void *opaque)
 {
-    struct dma_regs *r;
-    int ichan, ncont;
+    I8257State *d = I8257(obj);
+    I8257Regs *r;
+    int ichan;
 
-    ncont = nchan > 3;
     ichan = nchan & 3;
 
-    r = dma_controllers[ncont].regs + ichan;
+    r = d->regs + ichan;
     r->transfer_handler = transfer_handler;
     r->opaque = opaque;
 }
 
-int DMA_read_memory (int nchan, void *buf, int pos, int len)
+static int i8257_dma_read_memory(IsaDma *obj, int nchan, void *buf, int pos,
+                                 int len)
 {
-    struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
+    I8257State *d = I8257(obj);
+    I8257Regs *r = &d->regs[nchan & 3];
     hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
@@ -436,9 +419,11 @@ int DMA_read_memory (int nchan, void *buf, int pos, int len)
     return len;
 }
 
-int DMA_write_memory (int nchan, void *buf, int pos, int len)
+static int i8257_dma_write_memory(IsaDma *obj, int nchan, void *buf, int pos,
+                                 int len)
 {
-    struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
+    I8257State *s = I8257(obj);
+    I8257Regs *r = &s->regs[nchan & 3];
     hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
@@ -458,21 +443,25 @@ int DMA_write_memory (int nchan, void *buf, int pos, int len)
     return len;
 }
 
-/* request the emulator to transfer a new DMA memory block ASAP */
-void DMA_schedule(int nchan)
+/* request the emulator to transfer a new DMA memory block ASAP (even
+ * if the idle bottom half would not have exited the iothread yet).
+ */
+static void i8257_dma_schedule(IsaDma *obj)
 {
-    struct dma_cont *d = &dma_controllers[nchan > 3];
-
-    qemu_irq_pulse(*d->cpu_request_exit);
+    I8257State *d = I8257(obj);
+    if (d->dma_bh_scheduled) {
+        qemu_notify_event();
+    }
 }
 
-static void dma_reset(void *opaque)
+static void i8257_reset(DeviceState *dev)
 {
-    struct dma_cont *d = opaque;
-    write_cont(d, (0x05 << d->dshift), 0, 1);
+    I8257State *d = I8257(dev);
+    i8257_write_cont(d, (0x05 << d->dshift), 0, 1);
 }
 
-static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
+static int i8257_phony_handler(void *opaque, int nchan, int dma_pos,
+                               int dma_len)
 {
     trace_i8257_unregistered_dma(nchan, dma_pos, dma_len);
     return dma_pos;
@@ -480,8 +469,8 @@ static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
 
 
 static const MemoryRegionOps channel_io_ops = {
-    .read = read_chan,
-    .write = write_chan,
+    .read = i8257_read_chan,
+    .write = i8257_write_chan,
     .endianness = DEVICE_NATIVE_ENDIAN,
     .impl = {
         .min_access_size = 1,
@@ -491,21 +480,21 @@ static const MemoryRegionOps channel_io_ops = {
 
 /* IOport from page_base */
 static const MemoryRegionPortio page_portio_list[] = {
-    { 0x01, 3, 1, .write = write_page, .read = read_page, },
-    { 0x07, 1, 1, .write = write_page, .read = read_page, },
+    { 0x01, 3, 1, .write = i8257_write_page, .read = i8257_read_page, },
+    { 0x07, 1, 1, .write = i8257_write_page, .read = i8257_read_page, },
     PORTIO_END_OF_LIST(),
 };
 
 /* IOport from pageh_base */
 static const MemoryRegionPortio pageh_portio_list[] = {
-    { 0x01, 3, 1, .write = write_pageh, .read = read_pageh, },
-    { 0x07, 3, 1, .write = write_pageh, .read = read_pageh, },
+    { 0x01, 3, 1, .write = i8257_write_pageh, .read = i8257_read_pageh, },
+    { 0x07, 3, 1, .write = i8257_write_pageh, .read = i8257_read_pageh, },
     PORTIO_END_OF_LIST(),
 };
 
 static const MemoryRegionOps cont_io_ops = {
-    .read = read_cont,
-    .write = write_cont,
+    .read = i8257_read_cont,
+    .write = i8257_write_cont,
     .endianness = DEVICE_NATIVE_ENDIAN,
     .impl = {
         .min_access_size = 1,
@@ -513,86 +502,142 @@ static const MemoryRegionOps cont_io_ops = {
     },
 };
 
-/* dshift = 0: 8 bit DMA, 1 = 16 bit DMA */
-static void dma_init2(struct dma_cont *d, int base, int dshift,
-                      int page_base, int pageh_base,
-                      qemu_irq *cpu_request_exit)
+static const VMStateDescription vmstate_i8257_regs = {
+    .name = "dma_regs",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32_ARRAY(now, I8257Regs, 2),
+        VMSTATE_UINT16_ARRAY(base, I8257Regs, 2),
+        VMSTATE_UINT8(mode, I8257Regs),
+        VMSTATE_UINT8(page, I8257Regs),
+        VMSTATE_UINT8(pageh, I8257Regs),
+        VMSTATE_UINT8(dack, I8257Regs),
+        VMSTATE_UINT8(eop, I8257Regs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int i8257_post_load(void *opaque, int version_id)
 {
-    int i;
+    I8257State *d = opaque;
+    i8257_dma_run(d);
 
-    d->dshift = dshift;
-    d->cpu_request_exit = cpu_request_exit;
+    return 0;
+}
+
+static const VMStateDescription vmstate_i8257 = {
+    .name = "dma",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = i8257_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(command, I8257State),
+        VMSTATE_UINT8(mask, I8257State),
+        VMSTATE_UINT8(flip_flop, I8257State),
+        VMSTATE_INT32(dshift, I8257State),
+        VMSTATE_STRUCT_ARRAY(regs, I8257State, 4, 1, vmstate_i8257_regs,
+                             I8257Regs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void i8257_realize(DeviceState *dev, Error **errp)
+{
+    ISADevice *isa = ISA_DEVICE(dev);
+    I8257State *d = I8257(dev);
+    int i;
 
     memory_region_init_io(&d->channel_io, NULL, &channel_io_ops, d,
                           "dma-chan", 8 << d->dshift);
-    memory_region_add_subregion(isa_address_space_io(NULL),
-                                base, &d->channel_io);
+    memory_region_add_subregion(isa_address_space_io(isa),
+                                d->base, &d->channel_io);
 
-    isa_register_portio_list(NULL, page_base, page_portio_list, d,
+    isa_register_portio_list(isa, d->page_base, page_portio_list, d,
                              "dma-page");
-    if (pageh_base >= 0) {
-        isa_register_portio_list(NULL, pageh_base, pageh_portio_list, d,
+    if (d->pageh_base >= 0) {
+        isa_register_portio_list(isa, d->pageh_base, pageh_portio_list, d,
                                  "dma-pageh");
     }
 
-    memory_region_init_io(&d->cont_io, NULL, &cont_io_ops, d, "dma-cont",
-                          8 << d->dshift);
-    memory_region_add_subregion(isa_address_space_io(NULL),
-                                base + (8 << d->dshift), &d->cont_io);
+    memory_region_init_io(&d->cont_io, OBJECT(isa), &cont_io_ops, d,
+                          "dma-cont", 8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(isa),
+                                d->base + (8 << d->dshift), &d->cont_io);
 
-    qemu_register_reset(dma_reset, d);
-    dma_reset(d);
-    for (i = 0; i < ARRAY_SIZE (d->regs); ++i) {
-        d->regs[i].transfer_handler = dma_phony_handler;
+    for (i = 0; i < ARRAY_SIZE(d->regs); ++i) {
+        d->regs[i].transfer_handler = i8257_phony_handler;
     }
+
+    d->dma_bh = qemu_bh_new(i8257_dma_run, d);
 }
 
-static const VMStateDescription vmstate_dma_regs = {
-    .name = "dma_regs",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_INT32_ARRAY(now, struct dma_regs, 2),
-        VMSTATE_UINT16_ARRAY(base, struct dma_regs, 2),
-        VMSTATE_UINT8(mode, struct dma_regs),
-        VMSTATE_UINT8(page, struct dma_regs),
-        VMSTATE_UINT8(pageh, struct dma_regs),
-        VMSTATE_UINT8(dack, struct dma_regs),
-        VMSTATE_UINT8(eop, struct dma_regs),
-        VMSTATE_END_OF_LIST()
-    }
+static Property i8257_properties[] = {
+    DEFINE_PROP_INT32("base", I8257State, base, 0x00),
+    DEFINE_PROP_INT32("page-base", I8257State, page_base, 0x80),
+    DEFINE_PROP_INT32("pageh-base", I8257State, pageh_base, 0x480),
+    DEFINE_PROP_INT32("dshift", I8257State, dshift, 0),
+    DEFINE_PROP_END_OF_LIST()
 };
 
-static int dma_post_load(void *opaque, int version_id)
+static void i8257_class_init(ObjectClass *klass, void *data)
 {
-    DMA_run();
-
-    return 0;
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IsaDmaClass *idc = ISADMA_CLASS(klass);
+
+    dc->realize = i8257_realize;
+    dc->reset = i8257_reset;
+    dc->vmsd = &vmstate_i8257;
+    dc->props = i8257_properties;
+
+    idc->get_transfer_mode = i8257_dma_get_transfer_mode;
+    idc->has_autoinitialization = i8257_dma_has_autoinitialization;
+    idc->read_memory = i8257_dma_read_memory;
+    idc->write_memory = i8257_dma_write_memory;
+    idc->hold_DREQ = i8257_dma_hold_DREQ;
+    idc->release_DREQ = i8257_dma_release_DREQ;
+    idc->schedule = i8257_dma_schedule;
+    idc->register_channel = i8257_dma_register_channel;
 }
 
-static const VMStateDescription vmstate_dma = {
-    .name = "dma",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .post_load = dma_post_load,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT8(command, struct dma_cont),
-        VMSTATE_UINT8(mask, struct dma_cont),
-        VMSTATE_UINT8(flip_flop, struct dma_cont),
-        VMSTATE_INT32(dshift, struct dma_cont),
-        VMSTATE_STRUCT_ARRAY(regs, struct dma_cont, 4, 1, vmstate_dma_regs, struct dma_regs),
-        VMSTATE_END_OF_LIST()
+static const TypeInfo i8257_info = {
+    .name = TYPE_I8257,
+    .parent = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(I8257State),
+    .class_init = i8257_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_ISADMA },
+        { }
     }
 };
 
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+static void i8257_register_types(void)
+{
+    type_register_static(&i8257_info);
+}
+
+type_init(i8257_register_types)
+
+void DMA_init(ISABus *bus, int high_page_enable)
 {
-    dma_init2(&dma_controllers[0], 0x00, 0, 0x80,
-              high_page_enable ? 0x480 : -1, cpu_request_exit);
-    dma_init2(&dma_controllers[1], 0xc0, 1, 0x88,
-              high_page_enable ? 0x488 : -1, cpu_request_exit);
-    vmstate_register (NULL, 0, &vmstate_dma, &dma_controllers[0]);
-    vmstate_register (NULL, 1, &vmstate_dma, &dma_controllers[1]);
-
-    dma_bh = qemu_bh_new(DMA_run_bh, NULL);
+    ISADevice *isa1, *isa2;
+    DeviceState *d;
+
+    isa1 = isa_create(bus, TYPE_I8257);
+    d = DEVICE(isa1);
+    qdev_prop_set_int32(d, "base", 0x00);
+    qdev_prop_set_int32(d, "page-base", 0x80);
+    qdev_prop_set_int32(d, "pageh-base", high_page_enable ? 0x480 : -1);
+    qdev_prop_set_int32(d, "dshift", 0);
+    qdev_init_nofail(d);
+
+    isa2 = isa_create(bus, TYPE_I8257);
+    d = DEVICE(isa2);
+    qdev_prop_set_int32(d, "base", 0xc0);
+    qdev_prop_set_int32(d, "page-base", 0x88);
+    qdev_prop_set_int32(d, "pageh-base", high_page_enable ? 0x488 : -1);
+    qdev_prop_set_int32(d, "dshift", 1);
+    qdev_init_nofail(d);
+
+    isa_bus_dma(bus, ISADMA(isa1), ISADMA(isa2));
 }
diff --git a/qemu/hw/dma/omap_dma.c b/qemu/hw/dma/omap_dma.c
index 97c57a03c..700cd6b43 100644
--- a/qemu/hw/dma/omap_dma.c
+++ b/qemu/hw/dma/omap_dma.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -1626,8 +1627,7 @@ struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 enum omap_dma_model model)
 {
     int num_irqs, memsize, i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     if (model <= omap_dma_3_1) {
         num_irqs = 6;
@@ -2061,8 +2061,7 @@ struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 int chans, omap_clk iclk, omap_clk fclk)
 {
     int i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     s->model = omap_dma_4;
     s->chans = chans;
diff --git a/qemu/hw/dma/pl080.c b/qemu/hw/dma/pl080.c
index b89b4744f..9318108b8 100644
--- a/qemu/hw/dma/pl080.c
+++ b/qemu/hw/dma/pl080.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 
diff --git a/qemu/hw/dma/pl330.c b/qemu/hw/dma/pl330.c
index 5be3df521..ea89ecb00 100644
--- a/qemu/hw/dma/pl330.c
+++ b/qemu/hw/dma/pl330.c
@@ -14,7 +14,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/dma.h"
 
diff --git a/qemu/hw/dma/puv3_dma.c b/qemu/hw/dma/puv3_dma.c
index 101bd7f8a..b97a6c176 100644
--- a/qemu/hw/dma/puv3_dma.c
+++ b/qemu/hw/dma/puv3_dma.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/qemu/hw/dma/pxa2xx_dma.c b/qemu/hw/dma/pxa2xx_dma.c
index d4501fb4c..2306abc35 100644
--- a/qemu/hw/dma/pxa2xx_dma.c
+++ b/qemu/hw/dma/pxa2xx_dma.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/sysbus.h"
@@ -459,9 +460,8 @@ static int pxa2xx_dma_init(SysBusDevice *sbd)
         return -1;
     }
 
-    s->chan = g_malloc0(sizeof(PXA2xxDMAChannel) * s->channels);
+    s->chan = g_new0(PXA2xxDMAChannel, s->channels);
 
-    memset(s->chan, 0, sizeof(PXA2xxDMAChannel) * s->channels);
     for (i = 0; i < s->channels; i ++)
         s->chan[i].state = DCSR_STOPINTR;
 
diff --git a/qemu/hw/dma/rc4030.c b/qemu/hw/dma/rc4030.c
index 3efa6de35..a06c2359a 100644
--- a/qemu/hw/dma/rc4030.c
+++ b/qemu/hw/dma/rc4030.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/sysbus.h"
@@ -111,7 +112,7 @@ static void set_next_tick(rc4030State *s)
     tm_hz = 1000 / (s->itr + 1);
 
     timer_mod(s->periodic_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / tm_hz);
+                   NANOSECONDS_PER_SECOND / tm_hz);
 }
 
 /* called for accesses to rc4030 */
diff --git a/qemu/hw/dma/soc_dma.c b/qemu/hw/dma/soc_dma.c
index c06aabb40..9bb499bf9 100644
--- a/qemu/hw/dma/soc_dma.c
+++ b/qemu/hw/dma/soc_dma.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/arm/soc_dma.h"
@@ -269,11 +270,10 @@ void soc_dma_port_add_fifo(struct soc_dma_s *soc, hwaddr virt_base,
         if (entry->type == soc_dma_port_mem) {
             if (entry->addr <= virt_base &&
                             entry->addr + entry->u.mem.size > virt_base) {
-                fprintf(stderr, "%s: FIFO at " TARGET_FMT_lx
-                                " collides with RAM region at " TARGET_FMT_lx
-                                "-" TARGET_FMT_lx "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) entry->addr, (target_ulong)
+                fprintf(stderr, "%s: FIFO at %"PRIx64
+                                " collides with RAM region at %"PRIx64
+                                "-%"PRIx64 "\n", __func__,
+                                virt_base, entry->addr,
                                 (entry->addr + entry->u.mem.size));
                 exit(-1);
             }
@@ -284,10 +284,9 @@ void soc_dma_port_add_fifo(struct soc_dma_s *soc, hwaddr virt_base,
             while (entry < dma->memmap + dma->memmap_size &&
                             entry->addr <= virt_base) {
                 if (entry->addr == virt_base && entry->u.fifo.out == out) {
-                    fprintf(stderr, "%s: FIFO at " TARGET_FMT_lx
-                                    " collides FIFO at " TARGET_FMT_lx "\n",
-                                    __FUNCTION__, (target_ulong) virt_base,
-                                    (target_ulong) entry->addr);
+                    fprintf(stderr, "%s: FIFO at %"PRIx64
+                                    " collides FIFO at %"PRIx64 "\n",
+                                    __func__, virt_base, entry->addr);
                     exit(-1);
                 }
 
@@ -322,13 +321,11 @@ void soc_dma_port_add_mem(struct soc_dma_s *soc, uint8_t *phys_base,
             if ((entry->addr >= virt_base && entry->addr < virt_base + size) ||
                             (entry->addr <= virt_base &&
                              entry->addr + entry->u.mem.size > virt_base)) {
-                fprintf(stderr, "%s: RAM at " TARGET_FMT_lx "-" TARGET_FMT_lx
-                                " collides with RAM region at " TARGET_FMT_lx
-                                "-" TARGET_FMT_lx "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) (virt_base + size),
-                                (target_ulong) entry->addr, (target_ulong)
-                                (entry->addr + entry->u.mem.size));
+                fprintf(stderr, "%s: RAM at %"PRIx64 "-%"PRIx64
+                                " collides with RAM region at %"PRIx64
+                                "-%"PRIx64 "\n", __func__,
+                                virt_base, virt_base + size,
+                                entry->addr, entry->addr + entry->u.mem.size);
                 exit(-1);
             }
 
@@ -337,12 +334,11 @@ void soc_dma_port_add_mem(struct soc_dma_s *soc, uint8_t *phys_base,
         } else {
             if (entry->addr >= virt_base &&
                             entry->addr < virt_base + size) {
-                fprintf(stderr, "%s: RAM at " TARGET_FMT_lx "-" TARGET_FMT_lx
-                                " collides with FIFO at " TARGET_FMT_lx
-                                "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) (virt_base + size),
-                                (target_ulong) entry->addr);
+                fprintf(stderr, "%s: RAM at %"PRIx64 "-%"PRIx64
+                                " collides with FIFO at %"PRIx64
+                                "\n", __func__,
+                                virt_base, virt_base + size,
+                                entry->addr);
                 exit(-1);
             }
 
diff --git a/qemu/hw/dma/sparc32_dma.c b/qemu/hw/dma/sparc32_dma.c
index e6a453ce5..9d545e412 100644
--- a/qemu/hw/dma/sparc32_dma.c
+++ b/qemu/hw/dma/sparc32_dma.c
@@ -25,6 +25,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sparc/sparc32_dma.h"
 #include "hw/sparc/sun4m.h"
diff --git a/qemu/hw/dma/sun4m_iommu.c b/qemu/hw/dma/sun4m_iommu.c
index 9a488bc9b..b3cbc54c2 100644
--- a/qemu/hw/dma/sun4m_iommu.c
+++ b/qemu/hw/dma/sun4m_iommu.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
diff --git a/qemu/hw/dma/xilinx_axidma.c b/qemu/hw/dma/xilinx_axidma.c
index cf842a3cc..a4753e55a 100644
--- a/qemu/hw/dma/xilinx_axidma.c
+++ b/qemu/hw/dma/xilinx_axidma.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
@@ -133,7 +135,7 @@ struct XilinxAXIDMA {
 };
 
 /*
- * Helper calls to extract info from desriptors and other trivial
+ * Helper calls to extract info from descriptors and other trivial
  * state from regs.
  */
 static inline int stream_desc_sof(struct SDesc *d)
@@ -177,16 +179,6 @@ static inline int streamid_from_addr(hwaddr addr)
     return sid;
 }
 
-#ifdef DEBUG_ENET
-static void stream_desc_show(struct SDesc *d)
-{
-    qemu_log("buffer_addr  = " PRIx64 "\n", d->buffer_address);
-    qemu_log("nxtdesc      = " PRIx64 "\n", d->nxtdesc);
-    qemu_log("control      = %x\n", d->control);
-    qemu_log("status       = %x\n", d->status);
-}
-#endif
-
 static void stream_desc_load(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
diff --git a/qemu/hw/gpio/Makefile.objs b/qemu/hw/gpio/Makefile.objs
index 1abcf1798..a43c7cf44 100644
--- a/qemu/hw/gpio/Makefile.objs
+++ b/qemu/hw/gpio/Makefile.objs
@@ -3,5 +3,7 @@ common-obj-$(CONFIG_PL061) += pl061.o
 common-obj-$(CONFIG_PUV3) += puv3_gpio.o
 common-obj-$(CONFIG_ZAURUS) += zaurus.o
 common-obj-$(CONFIG_E500) += mpc8xxx.o
+common-obj-$(CONFIG_GPIO_KEY) += gpio_key.o
 
 obj-$(CONFIG_OMAP) += omap_gpio.o
+obj-$(CONFIG_IMX) += imx_gpio.o
diff --git a/qemu/hw/gpio/gpio_key.c b/qemu/hw/gpio/gpio_key.c
new file mode 100644
index 000000000..ef287727b
--- /dev/null
+++ b/qemu/hw/gpio/gpio_key.c
@@ -0,0 +1,104 @@
+/*
+ * GPIO key
+ *
+ * Copyright (c) 2016 Linaro Limited
+ *
+ * Author: Shannon Zhao <shannon.zhao@linaro.org>
+ *
+ * Emulate a (human) keypress -- when the key is triggered by
+ * setting the incoming gpio line, the outbound irq line is
+ * raised for 100ms before being dropped again.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+
+#define TYPE_GPIOKEY "gpio-key"
+#define GPIOKEY(obj) OBJECT_CHECK(GPIOKEYState, (obj), TYPE_GPIOKEY)
+#define GPIO_KEY_LATENCY 100 /* 100ms */
+
+typedef struct GPIOKEYState {
+    SysBusDevice parent_obj;
+
+    QEMUTimer *timer;
+    qemu_irq irq;
+} GPIOKEYState;
+
+static const VMStateDescription vmstate_gpio_key = {
+    .name = "gpio-key",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER_PTR(timer, GPIOKEYState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void gpio_key_reset(DeviceState *dev)
+{
+    GPIOKEYState *s = GPIOKEY(dev);
+
+    timer_del(s->timer);
+}
+
+static void gpio_key_timer_expired(void *opaque)
+{
+    GPIOKEYState *s = (GPIOKEYState *)opaque;
+
+    qemu_set_irq(s->irq, 0);
+    timer_del(s->timer);
+}
+
+static void gpio_key_set_irq(void *opaque, int irq, int level)
+{
+    GPIOKEYState *s = (GPIOKEYState *)opaque;
+
+    qemu_set_irq(s->irq, 1);
+    timer_mod(s->timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + GPIO_KEY_LATENCY);
+}
+
+static void gpio_key_realize(DeviceState *dev, Error **errp)
+{
+    GPIOKEYState *s = GPIOKEY(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_in(dev, gpio_key_set_irq, 1);
+    s->timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, gpio_key_timer_expired, s);
+}
+
+static void gpio_key_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = gpio_key_realize;
+    dc->vmsd = &vmstate_gpio_key;
+    dc->reset = &gpio_key_reset;
+}
+
+static const TypeInfo gpio_key_info = {
+    .name          = TYPE_GPIOKEY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GPIOKEYState),
+    .class_init    = gpio_key_class_init,
+};
+
+static void gpio_key_register_types(void)
+{
+    type_register_static(&gpio_key_info);
+}
+
+type_init(gpio_key_register_types)
diff --git a/qemu/hw/gpio/imx_gpio.c b/qemu/hw/gpio/imx_gpio.c
new file mode 100644
index 000000000..ed7e247f5
--- /dev/null
+++ b/qemu/hw/gpio/imx_gpio.c
@@ -0,0 +1,350 @@
+/*
+ * i.MX processors GPIO emulation.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/gpio/imx_gpio.h"
+
+#ifndef DEBUG_IMX_GPIO
+#define DEBUG_IMX_GPIO 0
+#endif
+
+typedef enum IMXGPIOLevel {
+    IMX_GPIO_LEVEL_LOW = 0,
+    IMX_GPIO_LEVEL_HIGH = 1,
+} IMXGPIOLevel;
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPIO) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPIO, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_gpio_reg_name(uint32_t reg)
+{
+    switch (reg) {
+    case DR_ADDR:
+        return "DR";
+    case GDIR_ADDR:
+        return "GDIR";
+    case PSR_ADDR:
+        return "PSR";
+    case ICR1_ADDR:
+        return "ICR1";
+    case ICR2_ADDR:
+        return "ICR2";
+    case IMR_ADDR:
+        return "IMR";
+    case ISR_ADDR:
+        return "ISR";
+    case EDGE_SEL_ADDR:
+        return "EDGE_SEL";
+    default:
+        return "[?]";
+    }
+}
+
+static void imx_gpio_update_int(IMXGPIOState *s)
+{
+    if (s->has_upper_pin_irq) {
+        qemu_set_irq(s->irq[0], (s->isr & s->imr & 0x0000FFFF) ? 1 : 0);
+        qemu_set_irq(s->irq[1], (s->isr & s->imr & 0xFFFF0000) ? 1 : 0);
+    } else {
+        qemu_set_irq(s->irq[0], (s->isr & s->imr) ? 1 : 0);
+    }
+}
+
+static void imx_gpio_set_int_line(IMXGPIOState *s, int line, IMXGPIOLevel level)
+{
+    /* if this signal isn't configured as an input signal, nothing to do */
+    if (!extract32(s->gdir, line, 1)) {
+        return;
+    }
+
+    /* When set, EDGE_SEL overrides the ICR config */
+    if (extract32(s->edge_sel, line, 1)) {
+        /* we detect interrupt on rising and falling edge */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    } else if (extract64(s->icr, 2*line + 1, 1)) {
+        /* interrupt is edge sensitive */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            if (extract64(s->icr, 2*line, 1) != level) {
+                s->isr = deposit32(s->isr, line, 1, 1);
+            }
+        }
+    } else {
+        /* interrupt is level sensitive */
+        if (extract64(s->icr, 2*line, 1) == level) {
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    }
+}
+
+static void imx_gpio_set(void *opaque, int line, int level)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    IMXGPIOLevel imx_level = level ? IMX_GPIO_LEVEL_HIGH : IMX_GPIO_LEVEL_LOW;
+
+    imx_gpio_set_int_line(s, line, imx_level);
+
+    /* this is an input signal, so set PSR */
+    s->psr = deposit32(s->psr, line, 1, imx_level);
+
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_set_all_int_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        IMXGPIOLevel imx_level = extract32(s->psr, i, 1);
+        imx_gpio_set_int_line(s, i, imx_level);
+    }
+
+    imx_gpio_update_int(s);
+}
+
+static inline void imx_gpio_set_all_output_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        /*
+         * if the line is set as output, then forward the line
+         * level to its user.
+         */
+        if (extract32(s->gdir, i, 1) && s->output[i]) {
+            qemu_set_irq(s->output[i], extract32(s->dr, i, 1));
+        }
+    }
+}
+
+static uint64_t imx_gpio_read(void *opaque, hwaddr offset, unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    uint32_t reg_value = 0;
+
+    switch (offset) {
+    case DR_ADDR:
+        /*
+         * depending on the "line" configuration, the bit values
+         * are coming either from DR or PSR
+         */
+        reg_value = (s->dr & s->gdir) | (s->psr & ~s->gdir);
+        break;
+
+    case GDIR_ADDR:
+        reg_value = s->gdir;
+        break;
+
+    case PSR_ADDR:
+        reg_value = s->psr & ~s->gdir;
+        break;
+
+    case ICR1_ADDR:
+        reg_value = extract64(s->icr, 0, 32);
+        break;
+
+    case ICR2_ADDR:
+        reg_value = extract64(s->icr, 32, 32);
+        break;
+
+    case IMR_ADDR:
+        reg_value = s->imr;
+        break;
+
+    case ISR_ADDR:
+        reg_value = s->isr;
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            reg_value = s->edge_sel;
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    DPRINTF("(%s) = 0x%" PRIx32 "\n", imx_gpio_reg_name(offset), reg_value);
+
+    return reg_value;
+}
+
+static void imx_gpio_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+
+    DPRINTF("(%s, value = 0x%" PRIx32 ")\n", imx_gpio_reg_name(offset),
+            (uint32_t)value);
+
+    switch (offset) {
+    case DR_ADDR:
+        s->dr = value;
+        imx_gpio_set_all_output_lines(s);
+        break;
+
+    case GDIR_ADDR:
+        s->gdir = value;
+        imx_gpio_set_all_output_lines(s);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR1_ADDR:
+        s->icr = deposit64(s->icr, 0, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR2_ADDR:
+        s->icr = deposit64(s->icr, 32, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case IMR_ADDR:
+        s->imr = value;
+        imx_gpio_update_int(s);
+        break;
+
+    case ISR_ADDR:
+        s->isr |= ~value;
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            s->edge_sel = value;
+            imx_gpio_set_all_int_lines(s);
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    return;
+}
+
+static const MemoryRegionOps imx_gpio_ops = {
+    .read = imx_gpio_read,
+    .write = imx_gpio_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription vmstate_imx_gpio = {
+    .name = TYPE_IMX_GPIO,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(dr, IMXGPIOState),
+        VMSTATE_UINT32(gdir, IMXGPIOState),
+        VMSTATE_UINT32(psr, IMXGPIOState),
+        VMSTATE_UINT64(icr, IMXGPIOState),
+        VMSTATE_UINT32(imr, IMXGPIOState),
+        VMSTATE_UINT32(isr, IMXGPIOState),
+        VMSTATE_BOOL(has_edge_sel, IMXGPIOState),
+        VMSTATE_UINT32(edge_sel, IMXGPIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property imx_gpio_properties[] = {
+    DEFINE_PROP_BOOL("has-edge-sel", IMXGPIOState, has_edge_sel, true),
+    DEFINE_PROP_BOOL("has-upper-pin-irq", IMXGPIOState, has_upper_pin_irq,
+                     false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_gpio_reset(DeviceState *dev)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    s->dr       = 0;
+    s->gdir     = 0;
+    s->psr      = 0;
+    s->icr      = 0;
+    s->imr      = 0;
+    s->isr      = 0;
+    s->edge_sel = 0;
+
+    imx_gpio_set_all_output_lines(s);
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_realize(DeviceState *dev, Error **errp)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_gpio_ops, s,
+                          TYPE_IMX_GPIO, IMX_GPIO_MEM_SIZE);
+
+    qdev_init_gpio_in(DEVICE(s), imx_gpio_set, IMX_GPIO_PIN_COUNT);
+    qdev_init_gpio_out(DEVICE(s), s->output, IMX_GPIO_PIN_COUNT);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq[0]);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq[1]);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+}
+
+static void imx_gpio_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = imx_gpio_realize;
+    dc->reset = imx_gpio_reset;
+    dc->props = imx_gpio_properties;
+    dc->vmsd = &vmstate_imx_gpio;
+    dc->desc = "i.MX GPIO controller";
+}
+
+static const TypeInfo imx_gpio_info = {
+    .name = TYPE_IMX_GPIO,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXGPIOState),
+    .class_init = imx_gpio_class_init,
+};
+
+static void imx_gpio_register_types(void)
+{
+    type_register_static(&imx_gpio_info);
+}
+
+type_init(imx_gpio_register_types)
diff --git a/qemu/hw/gpio/max7310.c b/qemu/hw/gpio/max7310.c
index 2f59b134e..1bd5eaf91 100644
--- a/qemu/hw/gpio/max7310.c
+++ b/qemu/hw/gpio/max7310.c
@@ -7,6 +7,7 @@
  * This file is licensed under GNU GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 
 #define TYPE_MAX7310 "max7310"
diff --git a/qemu/hw/gpio/mpc8xxx.c b/qemu/hw/gpio/mpc8xxx.c
index 1aeaaaaf0..d14971946 100644
--- a/qemu/hw/gpio/mpc8xxx.c
+++ b/qemu/hw/gpio/mpc8xxx.c
@@ -19,6 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #define TYPE_MPC8XXX_GPIO "mpc8xxx_gpio"
diff --git a/qemu/hw/gpio/omap_gpio.c b/qemu/hw/gpio/omap_gpio.c
index d92f8cfba..9b1b004fc 100644
--- a/qemu/hw/gpio/omap_gpio.c
+++ b/qemu/hw/gpio/omap_gpio.c
@@ -18,9 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 
 struct omap_gpio_s {
     qemu_irq irq;
@@ -682,7 +684,8 @@ static int omap_gpio_init(SysBusDevice *sbd)
     struct omap_gpif_s *s = OMAP1_GPIO(dev);
 
     if (!s->clk) {
-        hw_error("omap-gpio: clk not connected\n");
+        error_report("omap-gpio: clk not connected");
+        return -1;
     }
     qdev_init_gpio_in(dev, omap_gpio_set, 16);
     qdev_init_gpio_out(dev, s->omap1.handler, 16);
@@ -700,25 +703,35 @@ static int omap2_gpio_init(SysBusDevice *sbd)
     int i;
 
     if (!s->iclk) {
-        hw_error("omap2-gpio: iclk not connected\n");
+        error_report("omap2-gpio: iclk not connected");
+        return -1;
     }
+
+    s->modulecount = s->mpu_model < omap2430 ? 4
+                   : s->mpu_model < omap3430 ? 5
+                   : 6;
+
+    for (i = 0; i < s->modulecount; i++) {
+        if (!s->fclk[i]) {
+            error_report("omap2-gpio: fclk%d not connected", i);
+            return -1;
+        }
+    }
+
     if (s->mpu_model < omap3430) {
-        s->modulecount = (s->mpu_model < omap2430) ? 4 : 5;
         memory_region_init_io(&s->iomem, OBJECT(s), &omap2_gpif_top_ops, s,
                               "omap2.gpio", 0x1000);
         sysbus_init_mmio(sbd, &s->iomem);
-    } else {
-        s->modulecount = 6;
     }
-    s->modules = g_malloc0(s->modulecount * sizeof(struct omap2_gpio_s));
-    s->handler = g_malloc0(s->modulecount * 32 * sizeof(qemu_irq));
+
+    s->modules = g_new0(struct omap2_gpio_s, s->modulecount);
+    s->handler = g_new0(qemu_irq, s->modulecount * 32);
     qdev_init_gpio_in(dev, omap2_gpio_set, s->modulecount * 32);
     qdev_init_gpio_out(dev, s->handler, s->modulecount * 32);
+
     for (i = 0; i < s->modulecount; i++) {
         struct omap2_gpio_s *m = &s->modules[i];
-        if (!s->fclk[i]) {
-            hw_error("omap2-gpio: fclk%d not connected\n", i);
-        }
+
         m->revision = (s->mpu_model < omap3430) ? 0x18 : 0x25;
         m->handler = &s->handler[i * 32];
         sysbus_init_irq(sbd, &m->irq[0]); /* mpu irq */
@@ -728,6 +741,7 @@ static int omap2_gpio_init(SysBusDevice *sbd)
                               "omap.gpio-module", 0x1000);
         sysbus_init_mmio(sbd, &m->iomem);
     }
+
     return 0;
 }
 
diff --git a/qemu/hw/gpio/pl061.c b/qemu/hw/gpio/pl061.c
index 4ba730b47..29dc7fc38 100644
--- a/qemu/hw/gpio/pl061.c
+++ b/qemu/hw/gpio/pl061.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 //#define DEBUG_PL061 1
@@ -55,17 +56,17 @@ typedef struct PL061State {
     uint32_t slr;
     uint32_t den;
     uint32_t cr;
-    uint32_t float_high;
     uint32_t amsel;
     qemu_irq irq;
     qemu_irq out[8];
     const unsigned char *id;
+    uint32_t rsvd_start; /* reserved area: [rsvd_start, 0xfcc] */
 } PL061State;
 
 static const VMStateDescription vmstate_pl061 = {
     .name = "pl061",
-    .version_id = 3,
-    .minimum_version_id = 3,
+    .version_id = 4,
+    .minimum_version_id = 4,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(locked, PL061State),
         VMSTATE_UINT32(data, PL061State),
@@ -87,7 +88,6 @@ static const VMStateDescription vmstate_pl061 = {
         VMSTATE_UINT32(slr, PL061State),
         VMSTATE_UINT32(den, PL061State),
         VMSTATE_UINT32(cr, PL061State),
-        VMSTATE_UINT32(float_high, PL061State),
         VMSTATE_UINT32_V(amsel, PL061State, 2),
         VMSTATE_END_OF_LIST()
     }
@@ -153,12 +153,15 @@ static uint64_t pl061_read(void *opaque, hwaddr offset,
 {
     PL061State *s = (PL061State *)opaque;
 
-    if (offset >= 0xfd0 && offset < 0x1000) {
-        return s->id[(offset - 0xfd0) >> 2];
-    }
     if (offset < 0x400) {
         return s->data & (offset >> 2);
     }
+    if (offset >= s->rsvd_start && offset <= 0xfcc) {
+        goto err_out;
+    }
+    if (offset >= 0xfd0 && offset < 0x1000) {
+        return s->id[(offset - 0xfd0) >> 2];
+    }
     switch (offset) {
     case 0x400: /* Direction */
         return s->dir;
@@ -199,10 +202,12 @@ static uint64_t pl061_read(void *opaque, hwaddr offset,
     case 0x528: /* Analog mode select */
         return s->amsel;
     default:
-        qemu_log_mask(LOG_GUEST_ERROR,
-                      "pl061_read: Bad offset %x\n", (int)offset);
-        return 0;
+        break;
     }
+err_out:
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "pl061_read: Bad offset %x\n", (int)offset);
+    return 0;
 }
 
 static void pl061_write(void *opaque, hwaddr offset,
@@ -217,6 +222,9 @@ static void pl061_write(void *opaque, hwaddr offset,
         pl061_update(s);
         return;
     }
+    if (offset >= s->rsvd_start) {
+        goto err_out;
+    }
     switch (offset) {
     case 0x400: /* Direction */
         s->dir = value & 0xff;
@@ -275,16 +283,41 @@ static void pl061_write(void *opaque, hwaddr offset,
         s->amsel = value & 0xff;
         break;
     default:
-        qemu_log_mask(LOG_GUEST_ERROR,
-                      "pl061_write: Bad offset %x\n", (int)offset);
+        goto err_out;
     }
     pl061_update(s);
+    return;
+err_out:
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "pl061_write: Bad offset %x\n", (int)offset);
 }
 
-static void pl061_reset(PL061State *s)
+static void pl061_reset(DeviceState *dev)
 {
-  s->locked = 1;
-  s->cr = 0xff;
+    PL061State *s = PL061(dev);
+
+    /* reset values from PL061 TRM, Stellaris LM3S5P31 & LM3S8962 Data Sheet */
+    s->data = 0;
+    s->old_out_data = 0;
+    s->old_in_data = 0;
+    s->dir = 0;
+    s->isense = 0;
+    s->ibe = 0;
+    s->iev = 0;
+    s->im = 0;
+    s->istate = 0;
+    s->afsel = 0;
+    s->dr2r = 0xff;
+    s->dr4r = 0;
+    s->dr8r = 0;
+    s->odr = 0;
+    s->pur = 0;
+    s->pdr = 0;
+    s->slr = 0;
+    s->den = 0;
+    s->locked = 1;
+    s->cr = 0xff;
+    s->amsel = 0;
 }
 
 static void pl061_set_irq(void * opaque, int irq, int level)
@@ -317,7 +350,7 @@ static int pl061_initfn(SysBusDevice *sbd)
     sysbus_init_irq(sbd, &s->irq);
     qdev_init_gpio_in(dev, pl061_set_irq, 8);
     qdev_init_gpio_out(dev, s->out, 8);
-    pl061_reset(s);
+
     return 0;
 }
 
@@ -326,6 +359,7 @@ static void pl061_luminary_init(Object *obj)
     PL061State *s = PL061(obj);
 
     s->id = pl061_id_luminary;
+    s->rsvd_start = 0x52c;
 }
 
 static void pl061_init(Object *obj)
@@ -333,6 +367,7 @@ static void pl061_init(Object *obj)
     PL061State *s = PL061(obj);
 
     s->id = pl061_id;
+    s->rsvd_start = 0x424;
 }
 
 static void pl061_class_init(ObjectClass *klass, void *data)
@@ -342,6 +377,7 @@ static void pl061_class_init(ObjectClass *klass, void *data)
 
     k->init = pl061_initfn;
     dc->vmsd = &vmstate_pl061;
+    dc->reset = &pl061_reset;
 }
 
 static const TypeInfo pl061_info = {
diff --git a/qemu/hw/gpio/puv3_gpio.c b/qemu/hw/gpio/puv3_gpio.c
index 39840aa73..445afccf9 100644
--- a/qemu/hw/gpio/puv3_gpio.c
+++ b/qemu/hw/gpio/puv3_gpio.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/qemu/hw/gpio/zaurus.c b/qemu/hw/gpio/zaurus.c
index 24a77272d..555da281c 100644
--- a/qemu/hw/gpio/zaurus.c
+++ b/qemu/hw/gpio/zaurus.c
@@ -15,6 +15,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/sharpsl.h"
 #include "hw/sysbus.h"
@@ -235,10 +236,6 @@ static const VMStateDescription vmstate_scoop_regs = {
     },
 };
 
-static Property scoop_sysbus_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -247,7 +244,6 @@ static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
     k->init = scoop_init;
     dc->desc = "Scoop2 Sharp custom ASIC";
     dc->vmsd = &vmstate_scoop_regs;
-    dc->props = scoop_sysbus_properties;
 }
 
 static const TypeInfo scoop_sysbus_info = {
diff --git a/qemu/hw/i2c/Makefile.objs b/qemu/hw/i2c/Makefile.objs
index 0f130608c..aeb8f38d7 100644
--- a/qemu/hw/i2c/Makefile.objs
+++ b/qemu/hw/i2c/Makefile.objs
@@ -4,4 +4,5 @@ common-obj-$(CONFIG_ACPI_X86) += smbus_ich9.o
 common-obj-$(CONFIG_APM) += pm_smbus.o
 common-obj-$(CONFIG_BITBANG_I2C) += bitbang_i2c.o
 common-obj-$(CONFIG_EXYNOS4) += exynos4210_i2c.o
+common-obj-$(CONFIG_IMX_I2C) += imx_i2c.o
 obj-$(CONFIG_OMAP) += omap_i2c.o
diff --git a/qemu/hw/i2c/bitbang_i2c.c b/qemu/hw/i2c/bitbang_i2c.c
index 6d1bb03d6..6ed206020 100644
--- a/qemu/hw/i2c/bitbang_i2c.c
+++ b/qemu/hw/i2c/bitbang_i2c.c
@@ -9,6 +9,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "bitbang_i2c.h"
 #include "hw/sysbus.h"
diff --git a/qemu/hw/i2c/core.c b/qemu/hw/i2c/core.c
index 5a6402634..ba22104af 100644
--- a/qemu/hw/i2c/core.c
+++ b/qemu/hw/i2c/core.c
@@ -7,6 +7,7 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 
 struct I2CBus
diff --git a/qemu/hw/i2c/exynos4210_i2c.c b/qemu/hw/i2c/exynos4210_i2c.c
index fb99dfda1..8c2a2c163 100644
--- a/qemu/hw/i2c/exynos4210_i2c.c
+++ b/qemu/hw/i2c/exynos4210_i2c.c
@@ -20,6 +20,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 #include "hw/sysbus.h"
 #include "hw/i2c/i2c.h"
diff --git a/qemu/hw/i2c/imx_i2c.c b/qemu/hw/i2c/imx_i2c.c
new file mode 100644
index 000000000..a01e43ebe
--- /dev/null
+++ b/qemu/hw/i2c/imx_i2c.c
@@ -0,0 +1,337 @@
+/*
+ *  i.MX I2C Bus Serial Interface Emulation
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/i2c/i2c.h"
+
+#ifndef DEBUG_IMX_I2C
+#define DEBUG_IMX_I2C 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_I2C) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_I2C, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_i2c_get_regname(unsigned offset)
+{
+    switch (offset) {
+    case IADR_ADDR:
+        return "IADR";
+    case IFDR_ADDR:
+        return "IFDR";
+    case I2CR_ADDR:
+        return "I2CR";
+    case I2SR_ADDR:
+        return "I2SR";
+    case I2DR_ADDR:
+        return "I2DR";
+    default:
+        return "[?]";
+    }
+}
+
+static inline bool imx_i2c_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IEN;
+}
+
+static inline bool imx_i2c_interrupt_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IIEN;
+}
+
+static inline bool imx_i2c_is_master(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_MSTA;
+}
+
+static void imx_i2c_reset(DeviceState *dev)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    if (s->address != ADDR_RESET) {
+        i2c_end_transfer(s->bus);
+    }
+
+    s->address    = ADDR_RESET;
+    s->iadr       = IADR_RESET;
+    s->ifdr       = IFDR_RESET;
+    s->i2cr       = I2CR_RESET;
+    s->i2sr       = I2SR_RESET;
+    s->i2dr_read  = I2DR_RESET;
+    s->i2dr_write = I2DR_RESET;
+}
+
+static inline void imx_i2c_raise_interrupt(IMXI2CState *s)
+{
+    /*
+     * raise an interrupt if the device is enabled and it is configured
+     * to generate some interrupts.
+     */
+    if (imx_i2c_is_enabled(s) && imx_i2c_interrupt_is_enabled(s)) {
+        s->i2sr |= I2SR_IIF;
+        qemu_irq_raise(s->irq);
+    }
+}
+
+static uint64_t imx_i2c_read(void *opaque, hwaddr offset,
+                             unsigned size)
+{
+    uint16_t value;
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    switch (offset) {
+    case IADR_ADDR:
+        value = s->iadr;
+        break;
+    case IFDR_ADDR:
+        value = s->ifdr;
+        break;
+    case I2CR_ADDR:
+        value = s->i2cr;
+        break;
+    case I2SR_ADDR:
+        value = s->i2sr;
+        break;
+    case I2DR_ADDR:
+        value = s->i2dr_read;
+
+        if (imx_i2c_is_master(s)) {
+            int ret = 0xff;
+
+            if (s->address == ADDR_RESET) {
+                /* something is wrong as the address is not set */
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "without specifying the slave address\n",
+                              TYPE_IMX_I2C, __func__);
+            } else if (s->i2cr & I2CR_MTX) {
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "but MTX is set\n", TYPE_IMX_I2C, __func__);
+            } else {
+                /* get the next byte */
+                ret = i2c_recv(s->bus);
+
+                if (ret >= 0) {
+                    imx_i2c_raise_interrupt(s);
+                } else {
+                    qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: read failed "
+                                  "for device 0x%02x\n", TYPE_IMX_I2C,
+                                  __func__, s->address);
+                    ret = 0xff;
+                }
+            }
+
+            s->i2dr_read = ret;
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        value = 0;
+        break;
+    }
+
+    DPRINTF("read %s [0x%" HWADDR_PRIx "] -> 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, value);
+
+    return (uint64_t)value;
+}
+
+static void imx_i2c_write(void *opaque, hwaddr offset,
+                          uint64_t value, unsigned size)
+{
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    DPRINTF("write %s [0x%" HWADDR_PRIx "] <- 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, (int)value);
+
+    value &= 0xff;
+
+    switch (offset) {
+    case IADR_ADDR:
+        s->iadr = value & IADR_MASK;
+        /* i2c_set_slave_address(s->bus, (uint8_t)s->iadr); */
+        break;
+    case IFDR_ADDR:
+        s->ifdr = value & IFDR_MASK;
+        break;
+    case I2CR_ADDR:
+        if (imx_i2c_is_enabled(s) && ((value & I2CR_IEN) == 0)) {
+            /* This is a soft reset. IADR is preserved during soft resets */
+            uint16_t iadr = s->iadr;
+            imx_i2c_reset(DEVICE(s));
+            s->iadr = iadr;
+        } else { /* normal write */
+            s->i2cr = value & I2CR_MASK;
+
+            if (imx_i2c_is_master(s)) {
+                /* set the bus to busy */
+                s->i2sr |= I2SR_IBB;
+            } else { /* slave mode */
+                /* bus is not busy anymore */
+                s->i2sr &= ~I2SR_IBB;
+
+                /*
+                 * if we unset the master mode then it ends the ongoing
+                 * transfer if any
+                 */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                }
+            }
+
+            if (s->i2cr & I2CR_RSTA) { /* Restart */
+                /* if this is a restart then it ends the ongoing transfer */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                    s->i2cr &= ~I2CR_RSTA;
+                }
+            }
+        }
+        break;
+    case I2SR_ADDR:
+        /*
+         * if the user writes 0 to IIF then lower the interrupt and
+         * reset the bit
+         */
+        if ((s->i2sr & I2SR_IIF) && !(value & I2SR_IIF)) {
+            s->i2sr &= ~I2SR_IIF;
+            qemu_irq_lower(s->irq);
+        }
+
+        /*
+         * if the user writes 0 to IAL, reset the bit
+         */
+        if ((s->i2sr & I2SR_IAL) && !(value & I2SR_IAL)) {
+            s->i2sr &= ~I2SR_IAL;
+        }
+
+        break;
+    case I2DR_ADDR:
+        /* if the device is not enabled, nothing to do */
+        if (!imx_i2c_is_enabled(s)) {
+            break;
+        }
+
+        s->i2dr_write = value & I2DR_MASK;
+
+        if (imx_i2c_is_master(s)) {
+            /* If this is the first write cycle then it is the slave addr */
+            if (s->address == ADDR_RESET) {
+                if (i2c_start_transfer(s->bus, extract32(s->i2dr_write, 1, 7),
+                                       extract32(s->i2dr_write, 0, 1))) {
+                    /* if non zero is returned, the adress is not valid */
+                    s->i2sr |= I2SR_RXAK;
+                } else {
+                    s->address = s->i2dr_write;
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            } else { /* This is a normal data write */
+                if (i2c_send(s->bus, s->i2dr_write)) {
+                    /* if the target return non zero then end the transfer */
+                    s->i2sr |= I2SR_RXAK;
+                    s->address = ADDR_RESET;
+                    i2c_end_transfer(s->bus);
+                } else {
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            }
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps imx_i2c_ops = {
+    .read = imx_i2c_read,
+    .write = imx_i2c_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription imx_i2c_vmstate = {
+    .name = TYPE_IMX_I2C,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(address, IMXI2CState),
+        VMSTATE_UINT16(iadr, IMXI2CState),
+        VMSTATE_UINT16(ifdr, IMXI2CState),
+        VMSTATE_UINT16(i2cr, IMXI2CState),
+        VMSTATE_UINT16(i2sr, IMXI2CState),
+        VMSTATE_UINT16(i2dr_read, IMXI2CState),
+        VMSTATE_UINT16(i2dr_write, IMXI2CState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void imx_i2c_realize(DeviceState *dev, Error **errp)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_i2c_ops, s, TYPE_IMX_I2C,
+                          IMX_I2C_MEM_SIZE);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq);
+    s->bus = i2c_init_bus(DEVICE(dev), "i2c");
+}
+
+static void imx_i2c_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &imx_i2c_vmstate;
+    dc->reset = imx_i2c_reset;
+    dc->realize = imx_i2c_realize;
+    dc->desc = "i.MX I2C Controller";
+}
+
+static const TypeInfo imx_i2c_type_info = {
+    .name = TYPE_IMX_I2C,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXI2CState),
+    .class_init = imx_i2c_class_init,
+};
+
+static void imx_i2c_register_types(void)
+{
+    type_register_static(&imx_i2c_type_info);
+}
+
+type_init(imx_i2c_register_types)
diff --git a/qemu/hw/i2c/omap_i2c.c b/qemu/hw/i2c/omap_i2c.c
index b6f544a22..67fbbff8e 100644
--- a/qemu/hw/i2c/omap_i2c.c
+++ b/qemu/hw/i2c/omap_i2c.c
@@ -16,10 +16,12 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 
 #define TYPE_OMAP_I2C "omap_i2c"
 #define OMAP_I2C(obj) OBJECT_CHECK(OMAPI2CState, (obj), TYPE_OMAP_I2C)
@@ -449,12 +451,15 @@ static int omap_i2c_init(SysBusDevice *sbd)
     OMAPI2CState *s = OMAP_I2C(dev);
 
     if (!s->fclk) {
-        hw_error("omap_i2c: fclk not connected\n");
+        error_report("omap_i2c: fclk not connected");
+        return -1;
     }
     if (s->revision >= OMAP2_INTR_REV && !s->iclk) {
         /* Note that OMAP1 doesn't have a separate interface clock */
-        hw_error("omap_i2c: iclk not connected\n");
+        error_report("omap_i2c: iclk not connected");
+        return -1;
     }
+
     sysbus_init_irq(sbd, &s->irq);
     sysbus_init_irq(sbd, &s->drq[0]);
     sysbus_init_irq(sbd, &s->drq[1]);
diff --git a/qemu/hw/i2c/pm_smbus.c b/qemu/hw/i2c/pm_smbus.c
index ce1713d26..6fc3923f5 100644
--- a/qemu/hw/i2c/pm_smbus.c
+++ b/qemu/hw/i2c/pm_smbus.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see
  * <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i2c/pm_smbus.h"
diff --git a/qemu/hw/i2c/smbus.c b/qemu/hw/i2c/smbus.c
index 6e27ae8bd..3979b3dad 100644
--- a/qemu/hw/i2c/smbus.c
+++ b/qemu/hw/i2c/smbus.c
@@ -9,6 +9,7 @@
 
 /* TODO: Implement PEC.  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/i2c/smbus.h"
diff --git a/qemu/hw/i2c/smbus_eeprom.c b/qemu/hw/i2c/smbus_eeprom.c
index 72c09cba6..5b7bd891b 100644
--- a/qemu/hw/i2c/smbus_eeprom.c
+++ b/qemu/hw/i2c/smbus_eeprom.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/i2c/smbus.h"
diff --git a/qemu/hw/i2c/smbus_ich9.c b/qemu/hw/i2c/smbus_ich9.c
index 91d4d322c..498f03e83 100644
--- a/qemu/hw/i2c/smbus_ich9.c
+++ b/qemu/hw/i2c/smbus_ich9.c
@@ -24,6 +24,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i2c/pm_smbus.h"
diff --git a/qemu/hw/i2c/versatile_i2c.c b/qemu/hw/i2c/versatile_i2c.c
index 3c0c2c106..fee3bc761 100644
--- a/qemu/hw/i2c/versatile_i2c.c
+++ b/qemu/hw/i2c/versatile_i2c.c
@@ -21,6 +21,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "bitbang_i2c.h"
 
diff --git a/qemu/hw/i386/Makefile.objs b/qemu/hw/i386/Makefile.objs
index bd4f147f9..b52d5b875 100644
--- a/qemu/hw/i386/Makefile.objs
+++ b/qemu/hw/i386/Makefile.objs
@@ -1,5 +1,5 @@
 obj-$(CONFIG_KVM) += kvm/
-obj-y += multiboot.o smbios.o
+obj-y += multiboot.o
 obj-y += pc.o pc_piix.o pc_q35.o
 obj-y += pc_sysfw.o
 obj-y += intel_iommu.o
@@ -7,27 +7,4 @@ obj-$(CONFIG_XEN) += ../xenpv/ xen/
 
 obj-y += kvmvapic.o
 obj-y += acpi-build.o
-hw/i386/acpi-build.o: hw/i386/acpi-build.c \
-	hw/i386/acpi-dsdt.hex hw/i386/q35-acpi-dsdt.hex
-
-iasl-option=$(shell if test -z "`$(1) $(2) 2>&1 > /dev/null`" \
-    ; then echo "$(2)"; else echo "$(3)"; fi ;)
-
-ifdef IASL
-#IASL Present. Generate hex files from .dsl
-hw/i386/%.hex: $(SRC_PATH)/hw/i386/%.dsl $(SRC_PATH)/scripts/acpi_extract_preprocess.py $(SRC_PATH)/scripts/acpi_extract.py
-	$(call quiet-command, $(CPP) -x c -P $(QEMU_DGFLAGS) $(QEMU_INCLUDES) $< -o $*.dsl.i.orig, "  CPP $(TARGET_DIR)$*.dsl.i.orig")
-	$(call quiet-command, $(PYTHON) $(SRC_PATH)/scripts/acpi_extract_preprocess.py $*.dsl.i.orig > $*.dsl.i, "  ACPI_PREPROCESS $(TARGET_DIR)$*.dsl.i")
-	$(call quiet-command, $(IASL) $(call iasl-option,$(IASL),-Pn,) -vs -l -tc -p $* $*.dsl.i $(if $(V), , > /dev/null) 2>&1 ,"  IASL $(TARGET_DIR)$*.dsl.i")
-	$(call quiet-command, $(PYTHON) $(SRC_PATH)/scripts/acpi_extract.py $*.lst > $*.off, "  ACPI_EXTRACT $(TARGET_DIR)$*.off")
-	$(call quiet-command, cat $*.off > $@, "  CAT $(TARGET_DIR)$@")
-else
-#IASL Not present. Restore pre-generated hex files.
-hw/i386/%.hex: $(SRC_PATH)/hw/i386/%.hex.generated
-	$(call quiet-command, cp -f $< $@, "  CP $(TARGET_DIR)$@")
-endif
-
-.PHONY: cleanhex
-cleanhex:
-	rm -f hw/i386/*hex
-clean: cleanhex
+obj-y += pci-assign-load-rom.o
diff --git a/qemu/hw/i386/acpi-build.c b/qemu/hw/i386/acpi-build.c
index 46eddb8e4..64770034f 100644
--- a/qemu/hw/i386/acpi-build.c
+++ b/qemu/hw/i386/acpi-build.c
@@ -20,12 +20,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "acpi-build.h"
-#include <stddef.h>
 #include <glib.h>
 #include "qemu-common.h"
 #include "qemu/bitmap.h"
-#include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "hw/pci/pci.h"
 #include "qom/cpu.h"
@@ -38,10 +38,12 @@
 #include "hw/acpi/bios-linker-loader.h"
 #include "hw/loader.h"
 #include "hw/isa/isa.h"
+#include "hw/block/fdc.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "sysemu/tpm.h"
 #include "hw/acpi/tpm.h"
 #include "sysemu/tpm_backend.h"
+#include "hw/timer/mc146818rtc_regs.h"
 
 /* Supported chipsets: */
 #include "hw/acpi/piix4.h"
@@ -50,9 +52,7 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/pci-host/q35.h"
 #include "hw/i386/intel_iommu.h"
-
-#include "hw/i386/q35-acpi-dsdt.hex"
-#include "hw/i386/acpi-dsdt.hex"
+#include "hw/timer/hpet.h"
 
 #include "hw/acpi/aml-build.h"
 
@@ -77,10 +77,6 @@
 #define ACPI_BUILD_DPRINTF(fmt, ...)
 #endif
 
-typedef struct AcpiCpuInfo {
-    DECLARE_BITMAP(found_cpus, ACPI_CPU_HOTPLUG_ID_LIMIT);
-} AcpiCpuInfo;
-
 typedef struct AcpiMcfgInfo {
     uint64_t mcfg_base;
     uint32_t mcfg_size;
@@ -106,6 +102,7 @@ typedef struct AcpiPmInfo {
 } AcpiPmInfo;
 
 typedef struct AcpiMiscInfo {
+    bool is_piix4;
     bool has_hpet;
     TPMVersion tpm_version;
     const unsigned char *dsdt_code;
@@ -121,47 +118,6 @@ typedef struct AcpiBuildPciBusHotplugState {
     bool pcihp_bridge_en;
 } AcpiBuildPciBusHotplugState;
 
-static void acpi_get_dsdt(AcpiMiscInfo *info)
-{
-    Object *piix = piix4_pm_find();
-    Object *lpc = ich9_lpc_find();
-    assert(!!piix != !!lpc);
-
-    if (piix) {
-        info->dsdt_code = AcpiDsdtAmlCode;
-        info->dsdt_size = sizeof AcpiDsdtAmlCode;
-    }
-    if (lpc) {
-        info->dsdt_code = Q35AcpiDsdtAmlCode;
-        info->dsdt_size = sizeof Q35AcpiDsdtAmlCode;
-    }
-}
-
-static
-int acpi_add_cpu_info(Object *o, void *opaque)
-{
-    AcpiCpuInfo *cpu = opaque;
-    uint64_t apic_id;
-
-    if (object_dynamic_cast(o, TYPE_CPU)) {
-        apic_id = object_property_get_int(o, "apic-id", NULL);
-        assert(apic_id < ACPI_CPU_HOTPLUG_ID_LIMIT);
-
-        set_bit(apic_id, cpu->found_cpus);
-    }
-
-    object_child_foreach(o, acpi_add_cpu_info, opaque);
-    return 0;
-}
-
-static void acpi_get_cpu_info(AcpiCpuInfo *cpu)
-{
-    Object *root = object_get_root();
-
-    memset(cpu->found_cpus, 0, sizeof cpu->found_cpus);
-    object_child_foreach(root, acpi_add_cpu_info, cpu);
-}
-
 static void acpi_get_pm_info(AcpiPmInfo *pm)
 {
     Object *piix = piix4_pm_find();
@@ -169,6 +125,7 @@ static void acpi_get_pm_info(AcpiPmInfo *pm)
     Object *obj = NULL;
     QObject *o;
 
+    pm->cpu_hp_io_base = 0;
     pm->pcihp_io_base = 0;
     pm->pcihp_io_len = 0;
     if (piix) {
@@ -234,6 +191,17 @@ static void acpi_get_pm_info(AcpiPmInfo *pm)
 
 static void acpi_get_misc_info(AcpiMiscInfo *info)
 {
+    Object *piix = piix4_pm_find();
+    Object *lpc = ich9_lpc_find();
+    assert(!!piix != !!lpc);
+
+    if (piix) {
+        info->is_piix4 = true;
+    }
+    if (lpc) {
+        info->is_piix4 = false;
+    }
+
     info->has_hpet = hpet_find();
     info->tpm_version = tpm_get_version();
     info->pvpanic_port = pvpanic_port();
@@ -294,7 +262,7 @@ static void acpi_align_size(GArray *blob, unsigned align)
 
 /* FACS */
 static void
-build_facs(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
+build_facs(GArray *table_data, GArray *linker)
 {
     AcpiFacsDescriptorRev1 *facs = acpi_data_push(table_data, sizeof *facs);
     memcpy(&facs->signature, "FACS", 4);
@@ -333,13 +301,15 @@ static void fadt_setup(AcpiFadtDescriptorRev1 *fadt, AcpiPmInfo *pm)
     if (max_cpus > 8) {
         fadt->flags |= cpu_to_le32(1 << ACPI_FADT_F_FORCE_APIC_CLUSTER_MODEL);
     }
+    fadt->century = RTC_CENTURY;
 }
 
 
 /* FADT */
 static void
 build_fadt(GArray *table_data, GArray *linker, AcpiPmInfo *pm,
-           unsigned facs, unsigned dsdt)
+           unsigned facs, unsigned dsdt,
+           const char *oem_id, const char *oem_table_id)
 {
     AcpiFadtDescriptorRev1 *fadt = acpi_data_push(table_data, sizeof(*fadt));
 
@@ -360,13 +330,14 @@ build_fadt(GArray *table_data, GArray *linker, AcpiPmInfo *pm,
     fadt_setup(fadt, pm);
 
     build_header(linker, table_data,
-                 (void *)fadt, "FACP", sizeof(*fadt), 1);
+                 (void *)fadt, "FACP", sizeof(*fadt), 1, oem_id, oem_table_id);
 }
 
 static void
-build_madt(GArray *table_data, GArray *linker, AcpiCpuInfo *cpu,
-           PcGuestInfo *guest_info)
+build_madt(GArray *table_data, GArray *linker, PCMachineState *pcms)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(pcms);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(MACHINE(pcms));
     int madt_start = table_data->len;
 
     AcpiMultipleApicTable *madt;
@@ -379,18 +350,28 @@ build_madt(GArray *table_data, GArray *linker, AcpiCpuInfo *cpu,
     madt->local_apic_address = cpu_to_le32(APIC_DEFAULT_ADDRESS);
     madt->flags = cpu_to_le32(1);
 
-    for (i = 0; i < guest_info->apic_id_limit; i++) {
+    for (i = 0; i < apic_ids->len; i++) {
         AcpiMadtProcessorApic *apic = acpi_data_push(table_data, sizeof *apic);
+        int apic_id = apic_ids->cpus[i].arch_id;
+
         apic->type = ACPI_APIC_PROCESSOR;
         apic->length = sizeof(*apic);
-        apic->processor_id = i;
-        apic->local_apic_id = i;
-        if (test_bit(i, cpu->found_cpus)) {
+        apic->processor_id = apic_id;
+        apic->local_apic_id = apic_id;
+        if (apic_ids->cpus[i].cpu != NULL) {
             apic->flags = cpu_to_le32(1);
         } else {
+            /* ACPI spec says that LAPIC entry for non present
+             * CPU may be omitted from MADT or it must be marked
+             * as disabled. However omitting non present CPU from
+             * MADT breaks hotplug on linux. So possible CPUs
+             * should be put in MADT but kept disabled.
+             */
             apic->flags = cpu_to_le32(0);
         }
     }
+    g_free(apic_ids);
+
     io_apic = acpi_data_push(table_data, sizeof *io_apic);
     io_apic->type = ACPI_APIC_IO;
     io_apic->length = sizeof(*io_apic);
@@ -399,7 +380,7 @@ build_madt(GArray *table_data, GArray *linker, AcpiCpuInfo *cpu,
     io_apic->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS);
     io_apic->interrupt = cpu_to_le32(0);
 
-    if (guest_info->apic_xrupt_override) {
+    if (pcms->apic_xrupt_override) {
         intsrcovr = acpi_data_push(table_data, sizeof *intsrcovr);
         intsrcovr->type   = ACPI_APIC_XRUPT_OVERRIDE;
         intsrcovr->length = sizeof(*intsrcovr);
@@ -430,7 +411,7 @@ build_madt(GArray *table_data, GArray *linker, AcpiCpuInfo *cpu,
 
     build_header(linker, table_data,
                  (void *)(table_data->data + madt_start), "APIC",
-                 table_data->len - madt_start, 1);
+                 table_data->len - madt_start, 1, NULL, NULL);
 }
 
 /* Assign BSEL property to all buses.  In the future, this can be changed
@@ -468,7 +449,7 @@ static void build_append_pcihp_notify_entry(Aml *method, int slot)
     Aml *if_ctx;
     int32_t devfn = PCI_DEVFN(slot, 0);
 
-    if_ctx = aml_if(aml_and(aml_arg(0), aml_int(0x1U << slot)));
+    if_ctx = aml_if(aml_and(aml_arg(0), aml_int(0x1U << slot), NULL));
     aml_append(if_ctx, aml_notify(aml_name("S%.02X", devfn), aml_arg(1)));
     aml_append(method, if_ctx);
 }
@@ -486,7 +467,7 @@ static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
         int64_t bsel_val = qint_get_int(qobject_to_qint(bsel));
 
         aml_append(parent_scope, aml_name_decl("BSEL", aml_int(bsel_val)));
-        notify_method = aml_method("DVNT", 2);
+        notify_method = aml_method("DVNT", 2, AML_NOTSERIALIZED);
     }
 
     for (i = 0; i < ARRAY_SIZE(bus->devices); i += PCI_FUNC_MAX) {
@@ -502,7 +483,7 @@ static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
                 dev = aml_device("S%.02X", PCI_DEVFN(slot, 0));
                 aml_append(dev, aml_name_decl("_SUN", aml_int(slot)));
                 aml_append(dev, aml_name_decl("_ADR", aml_int(slot << 16)));
-                method = aml_method("_EJ0", 1);
+                method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
                 aml_append(method,
                     aml_call2("PCEJ", aml_name("BSEL"), aml_name("_SUN"))
                 );
@@ -545,22 +526,22 @@ static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
                 s3d = 0;
             }
 
-            method = aml_method("_S1D", 0);
+            method = aml_method("_S1D", 0, AML_NOTSERIALIZED);
             aml_append(method, aml_return(aml_int(0)));
             aml_append(dev, method);
 
-            method = aml_method("_S2D", 0);
+            method = aml_method("_S2D", 0, AML_NOTSERIALIZED);
             aml_append(method, aml_return(aml_int(0)));
             aml_append(dev, method);
 
-            method = aml_method("_S3D", 0);
+            method = aml_method("_S3D", 0, AML_NOTSERIALIZED);
             aml_append(method, aml_return(aml_int(s3d)));
             aml_append(dev, method);
         } else if (hotplug_enabled_dev) {
             /* add _SUN/_EJ0 to make slot hotpluggable  */
             aml_append(dev, aml_name_decl("_SUN", aml_int(slot)));
 
-            method = aml_method("_EJ0", 1);
+            method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
             aml_append(method,
                 aml_call2("PCEJ", aml_name("BSEL"), aml_name("_SUN"))
             );
@@ -589,7 +570,7 @@ static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
     /* Append PCNT method to notify about events on local and child buses.
      * Add unconditionally for root since DSDT expects it.
      */
-    method = aml_method("PCNT", 0);
+    method = aml_method("PCNT", 0, AML_NOTSERIALIZED);
 
     /* If bus supports hotplug select it and notify about local events */
     if (bsel) {
@@ -615,6 +596,23 @@ static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
     qobject_decref(bsel);
 }
 
+/**
+ * build_prt_entry:
+ * @link_name: link name for PCI route entry
+ *
+ * build AML package containing a PCI route entry for @link_name
+ */
+static Aml *build_prt_entry(const char *link_name)
+{
+    Aml *a_zero = aml_int(0);
+    Aml *pkg = aml_package(4);
+    aml_append(pkg, a_zero);
+    aml_append(pkg, a_zero);
+    aml_append(pkg, aml_name("%s", link_name));
+    aml_append(pkg, a_zero);
+    return pkg;
+}
+
 /*
  * initialize_route - Initialize the interrupt routing rule
  * through a specific LINK:
@@ -625,12 +623,8 @@ static Aml *initialize_route(Aml *route, const char *link_name,
                              Aml *lnk_idx, int idx)
 {
     Aml *if_ctx = aml_if(aml_equal(lnk_idx, aml_int(idx)));
-    Aml *pkg = aml_package(4);
+    Aml *pkg = build_prt_entry(link_name);
 
-    aml_append(pkg, aml_int(0));
-    aml_append(pkg, aml_int(0));
-    aml_append(pkg, aml_name("%s", link_name));
-    aml_append(pkg, aml_int(0));
     aml_append(if_ctx, aml_store(pkg, route));
 
     return if_ctx;
@@ -646,11 +640,11 @@ static Aml *initialize_route(Aml *route, const char *link_name,
  * The hash function is  (slot + pin) & 3 -> "LNK[D|A|B|C]".
  *
  */
-static Aml *build_prt(void)
+static Aml *build_prt(bool is_pci0_prt)
 {
     Aml *method, *while_ctx, *pin, *res;
 
-    method = aml_method("_PRT", 0);
+    method = aml_method("_PRT", 0, AML_NOTSERIALIZED);
     res = aml_local(0);
     pin = aml_local(1);
     aml_append(method, aml_store(aml_package(128), res));
@@ -665,24 +659,49 @@ static Aml *build_prt(void)
 
         /* slot = pin >> 2 */
         aml_append(while_ctx,
-                   aml_store(aml_shiftright(pin, aml_int(2)), slot));
+                   aml_store(aml_shiftright(pin, aml_int(2), NULL), slot));
         /* lnk_idx = (slot + pin) & 3 */
         aml_append(while_ctx,
-                   aml_store(aml_and(aml_add(pin, slot), aml_int(3)), lnk_idx));
+            aml_store(aml_and(aml_add(pin, slot, NULL), aml_int(3), NULL),
+                      lnk_idx));
 
         /* route[2] = "LNK[D|A|B|C]", selection based on pin % 3  */
         aml_append(while_ctx, initialize_route(route, "LNKD", lnk_idx, 0));
-        aml_append(while_ctx, initialize_route(route, "LNKA", lnk_idx, 1));
+        if (is_pci0_prt) {
+            Aml *if_device_1, *if_pin_4, *else_pin_4;
+
+            /* device 1 is the power-management device, needs SCI */
+            if_device_1 = aml_if(aml_equal(lnk_idx, aml_int(1)));
+            {
+                if_pin_4 = aml_if(aml_equal(pin, aml_int(4)));
+                {
+                    aml_append(if_pin_4,
+                        aml_store(build_prt_entry("LNKS"), route));
+                }
+                aml_append(if_device_1, if_pin_4);
+                else_pin_4 = aml_else();
+                {
+                    aml_append(else_pin_4,
+                        aml_store(build_prt_entry("LNKA"), route));
+                }
+                aml_append(if_device_1, else_pin_4);
+            }
+            aml_append(while_ctx, if_device_1);
+        } else {
+            aml_append(while_ctx, initialize_route(route, "LNKA", lnk_idx, 1));
+        }
         aml_append(while_ctx, initialize_route(route, "LNKB", lnk_idx, 2));
         aml_append(while_ctx, initialize_route(route, "LNKC", lnk_idx, 3));
 
         /* route[0] = 0x[slot]FFFF */
         aml_append(while_ctx,
-            aml_store(aml_or(aml_shiftleft(slot, aml_int(16)), aml_int(0xFFFF)),
+            aml_store(aml_or(aml_shiftleft(slot, aml_int(16)), aml_int(0xFFFF),
+                             NULL),
                       aml_index(route, aml_int(0))));
         /* route[1] = pin & 3 */
         aml_append(while_ctx,
-            aml_store(aml_and(pin, aml_int(3)), aml_index(route, aml_int(1))));
+            aml_store(aml_and(pin, aml_int(3), NULL),
+                      aml_index(route, aml_int(1))));
         /* res[pin] = route */
         aml_append(while_ctx, aml_store(route, aml_index(res, pin)));
         /* pin++ */
@@ -761,16 +780,59 @@ static void crs_replace_with_free_ranges(GPtrArray *ranges,
     g_ptr_array_free(free_ranges, false);
 }
 
+/*
+ * crs_range_merge - merges adjacent ranges in the given array.
+ * Array elements are deleted and replaced with the merged ranges.
+ */
+static void crs_range_merge(GPtrArray *range)
+{
+    GPtrArray *tmp =  g_ptr_array_new_with_free_func(crs_range_free);
+    CrsRangeEntry *entry;
+    uint64_t range_base, range_limit;
+    int i;
+
+    if (!range->len) {
+        return;
+    }
+
+    g_ptr_array_sort(range, crs_range_compare);
+
+    entry = g_ptr_array_index(range, 0);
+    range_base = entry->base;
+    range_limit = entry->limit;
+    for (i = 1; i < range->len; i++) {
+        entry = g_ptr_array_index(range, i);
+        if (entry->base - 1 == range_limit) {
+            range_limit = entry->limit;
+        } else {
+            crs_range_insert(tmp, range_base, range_limit);
+            range_base = entry->base;
+            range_limit = entry->limit;
+        }
+    }
+    crs_range_insert(tmp, range_base, range_limit);
+
+    g_ptr_array_set_size(range, 0);
+    for (i = 0; i < tmp->len; i++) {
+        entry = g_ptr_array_index(tmp, i);
+        crs_range_insert(range, entry->base, entry->limit);
+    }
+    g_ptr_array_free(tmp, true);
+}
+
 static Aml *build_crs(PCIHostState *host,
                       GPtrArray *io_ranges, GPtrArray *mem_ranges)
 {
     Aml *crs = aml_resource_template();
+    GPtrArray *host_io_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    GPtrArray *host_mem_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    CrsRangeEntry *entry;
     uint8_t max_bus = pci_bus_num(host->bus);
     uint8_t type;
     int devfn;
+    int i;
 
     for (devfn = 0; devfn < ARRAY_SIZE(host->bus->devices); devfn++) {
-        int i;
         uint64_t range_base, range_limit;
         PCIDevice *dev = host->bus->devices[devfn];
 
@@ -793,26 +855,9 @@ static Aml *build_crs(PCIHostState *host,
             }
 
             if (r->type & PCI_BASE_ADDRESS_SPACE_IO) {
-                aml_append(crs,
-                    aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
-                                AML_POS_DECODE, AML_ENTIRE_RANGE,
-                                0,
-                                range_base,
-                                range_limit,
-                                0,
-                                range_limit - range_base + 1));
-                crs_range_insert(io_ranges, range_base, range_limit);
+                crs_range_insert(host_io_ranges, range_base, range_limit);
             } else { /* "memory" */
-                aml_append(crs,
-                    aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
-                                     AML_MAX_FIXED, AML_NON_CACHEABLE,
-                                     AML_READ_WRITE,
-                                     0,
-                                     range_base,
-                                     range_limit,
-                                     0,
-                                     range_limit - range_base + 1));
-                crs_range_insert(mem_ranges, range_base, range_limit);
+                crs_range_insert(host_mem_ranges, range_base, range_limit);
             }
         }
 
@@ -831,15 +876,7 @@ static Aml *build_crs(PCIHostState *host,
              * that do not support multiple root buses
              */
             if (range_base && range_base <= range_limit) {
-                aml_append(crs,
-                           aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
-                                       AML_POS_DECODE, AML_ENTIRE_RANGE,
-                                       0,
-                                       range_base,
-                                       range_limit,
-                                       0,
-                                       range_limit - range_base + 1));
-                crs_range_insert(io_ranges, range_base, range_limit);
+                crs_range_insert(host_io_ranges, range_base, range_limit);
             }
 
             range_base =
@@ -852,16 +889,7 @@ static Aml *build_crs(PCIHostState *host,
              * that do not support multiple root buses
              */
             if (range_base && range_base <= range_limit) {
-                aml_append(crs,
-                           aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
-                                            AML_MAX_FIXED, AML_NON_CACHEABLE,
-                                            AML_READ_WRITE,
-                                            0,
-                                            range_base,
-                                            range_limit,
-                                            0,
-                                            range_limit - range_base + 1));
-                crs_range_insert(mem_ranges, range_base, range_limit);
+                crs_range_insert(host_mem_ranges, range_base, range_limit);
             }
 
             range_base =
@@ -874,20 +902,36 @@ static Aml *build_crs(PCIHostState *host,
              * that do not support multiple root buses
              */
             if (range_base && range_base <= range_limit) {
-                aml_append(crs,
-                           aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
-                                            AML_MAX_FIXED, AML_NON_CACHEABLE,
-                                            AML_READ_WRITE,
-                                            0,
-                                            range_base,
-                                            range_limit,
-                                            0,
-                                            range_limit - range_base + 1));
-                crs_range_insert(mem_ranges, range_base, range_limit);
+                crs_range_insert(host_mem_ranges, range_base, range_limit);
             }
         }
     }
 
+    crs_range_merge(host_io_ranges);
+    for (i = 0; i < host_io_ranges->len; i++) {
+        entry = g_ptr_array_index(host_io_ranges, i);
+        aml_append(crs,
+                   aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
+                               AML_POS_DECODE, AML_ENTIRE_RANGE,
+                               0, entry->base, entry->limit, 0,
+                               entry->limit - entry->base + 1));
+        crs_range_insert(io_ranges, entry->base, entry->limit);
+    }
+    g_ptr_array_free(host_io_ranges, true);
+
+    crs_range_merge(host_mem_ranges);
+    for (i = 0; i < host_mem_ranges->len; i++) {
+        entry = g_ptr_array_index(host_mem_ranges, i);
+        aml_append(crs,
+                   aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                    AML_MAX_FIXED, AML_NON_CACHEABLE,
+                                    AML_READ_WRITE,
+                                    0, entry->base, entry->limit, 0,
+                                    entry->limit - entry->base + 1));
+        crs_range_insert(mem_ranges, entry->base, entry->limit);
+    }
+    g_ptr_array_free(host_mem_ranges, true);
+
     aml_append(crs,
         aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
                             0,
@@ -899,33 +943,1151 @@ static Aml *build_crs(PCIHostState *host,
     return crs;
 }
 
+static void build_processor_devices(Aml *sb_scope, MachineState *machine,
+                                    AcpiPmInfo *pm)
+{
+    int i, apic_idx;
+    Aml *dev;
+    Aml *crs;
+    Aml *pkg;
+    Aml *field;
+    Aml *ifctx;
+    Aml *method;
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    /* The current AML generator can cover the APIC ID range [0..255],
+     * inclusive, for VCPU hotplug. */
+    QEMU_BUILD_BUG_ON(ACPI_CPU_HOTPLUG_ID_LIMIT > 256);
+    g_assert(pcms->apic_id_limit <= ACPI_CPU_HOTPLUG_ID_LIMIT);
+
+    /* create PCI0.PRES device and its _CRS to reserve CPU hotplug MMIO */
+    dev = aml_device("PCI0." stringify(CPU_HOTPLUG_RESOURCE_DEVICE));
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A06")));
+    aml_append(dev,
+        aml_name_decl("_UID", aml_string("CPU Hotplug resources"))
+    );
+    /* device present, functioning, decoding, not shown in UI */
+    aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_io(AML_DECODE16, pm->cpu_hp_io_base, pm->cpu_hp_io_base, 1,
+               pm->cpu_hp_io_len)
+    );
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(sb_scope, dev);
+    /* declare CPU hotplug MMIO region and PRS field to access it */
+    aml_append(sb_scope, aml_operation_region(
+        "PRST", AML_SYSTEM_IO, aml_int(pm->cpu_hp_io_base), pm->cpu_hp_io_len));
+    field = aml_field("PRST", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRS", 256));
+    aml_append(sb_scope, field);
+
+    /* build Processor object for each processor */
+    for (i = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        assert(apic_id < ACPI_CPU_HOTPLUG_ID_LIMIT);
+
+        dev = aml_processor(apic_id, 0, 0, "CP%.02X", apic_id);
+
+        method = aml_method("_MAT", 0, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call1(CPU_MAT_METHOD, aml_int(apic_id))));
+        aml_append(dev, method);
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call1(CPU_STATUS_METHOD, aml_int(apic_id))));
+        aml_append(dev, method);
+
+        method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call2(CPU_EJECT_METHOD, aml_int(apic_id),
+                aml_arg(0)))
+        );
+        aml_append(dev, method);
+
+        aml_append(sb_scope, dev);
+    }
+
+    /* build this code:
+     *   Method(NTFY, 2) {If (LEqual(Arg0, 0x00)) {Notify(CP00, Arg1)} ...}
+     */
+    /* Arg0 = Processor ID = APIC ID */
+    method = aml_method(AML_NOTIFY_METHOD, 2, AML_NOTSERIALIZED);
+    for (i = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        ifctx = aml_if(aml_equal(aml_arg(0), aml_int(apic_id)));
+        aml_append(ifctx,
+            aml_notify(aml_name("CP%.02X", apic_id), aml_arg(1))
+        );
+        aml_append(method, ifctx);
+    }
+    aml_append(sb_scope, method);
+
+    /* build "Name(CPON, Package() { One, One, ..., Zero, Zero, ... })"
+     *
+     * Note: The ability to create variable-sized packages was first
+     * introduced in ACPI 2.0. ACPI 1.0 only allowed fixed-size packages
+     * ith up to 255 elements. Windows guests up to win2k8 fail when
+     * VarPackageOp is used.
+     */
+    pkg = pcms->apic_id_limit <= 255 ? aml_package(pcms->apic_id_limit) :
+                                       aml_varpackage(pcms->apic_id_limit);
+
+    for (i = 0, apic_idx = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        for (; apic_idx < apic_id; apic_idx++) {
+            aml_append(pkg, aml_int(0));
+        }
+        aml_append(pkg, aml_int(apic_ids->cpus[i].cpu ? 1 : 0));
+        apic_idx = apic_id + 1;
+    }
+    aml_append(sb_scope, aml_name_decl(CPU_ON_BITMAP, pkg));
+    g_free(apic_ids);
+}
+
+static void build_memory_devices(Aml *sb_scope, int nr_mem,
+                                 uint16_t io_base, uint16_t io_len)
+{
+    int i;
+    Aml *scope;
+    Aml *crs;
+    Aml *field;
+    Aml *dev;
+    Aml *method;
+    Aml *ifctx;
+
+    /* build memory devices */
+    assert(nr_mem <= ACPI_MAX_RAM_SLOTS);
+    scope = aml_scope("\\_SB.PCI0." MEMORY_HOTPLUG_DEVICE);
+    aml_append(scope,
+        aml_name_decl(MEMORY_SLOTS_NUMBER, aml_int(nr_mem))
+    );
+
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_io(AML_DECODE16, io_base, io_base, 0, io_len)
+    );
+    aml_append(scope, aml_name_decl("_CRS", crs));
+
+    aml_append(scope, aml_operation_region(
+        MEMORY_HOTPLUG_IO_REGION, AML_SYSTEM_IO,
+        aml_int(io_base), io_len)
+    );
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_DWORD_ACC,
+                      AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_ADDR_LOW, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_ADDR_HIGH, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_SIZE_LOW, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_SIZE_HIGH, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_PROXIMITY, 32));
+    aml_append(scope, field);
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_BYTE_ACC,
+                      AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_reserved_field(160 /* bits, Offset(20) */));
+    aml_append(field, /* 1 if enabled, read only */
+        aml_named_field(MEMORY_SLOT_ENABLED, 1));
+    aml_append(field,
+        /*(read) 1 if has a insert event. (write) 1 to clear event */
+        aml_named_field(MEMORY_SLOT_INSERT_EVENT, 1));
+    aml_append(field,
+        /* (read) 1 if has a remove event. (write) 1 to clear event */
+        aml_named_field(MEMORY_SLOT_REMOVE_EVENT, 1));
+    aml_append(field,
+        /* initiates device eject, write only */
+        aml_named_field(MEMORY_SLOT_EJECT, 1));
+    aml_append(scope, field);
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_DWORD_ACC,
+                      AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, /* DIMM selector, write only */
+        aml_named_field(MEMORY_SLOT_SLECTOR, 32));
+    aml_append(field, /* _OST event code, write only */
+        aml_named_field(MEMORY_SLOT_OST_EVENT, 32));
+    aml_append(field, /* _OST status code, write only */
+        aml_named_field(MEMORY_SLOT_OST_STATUS, 32));
+    aml_append(scope, field);
+    aml_append(sb_scope, scope);
+
+    for (i = 0; i < nr_mem; i++) {
+        #define BASEPATH "\\_SB.PCI0." MEMORY_HOTPLUG_DEVICE "."
+        const char *s;
+
+        dev = aml_device("MP%02X", i);
+        aml_append(dev, aml_name_decl("_UID", aml_string("0x%02X", i)));
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C80")));
+
+        method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_CRS_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_STATUS_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_PXM", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_PROXIMITY_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_OST", 3, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_OST_METHOD;
+
+        aml_append(method, aml_return(aml_call4(
+            s, aml_name("_UID"), aml_arg(0), aml_arg(1), aml_arg(2)
+        )));
+        aml_append(dev, method);
+
+        method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_EJECT_METHOD;
+        aml_append(method, aml_return(aml_call2(
+                   s, aml_name("_UID"), aml_arg(0))));
+        aml_append(dev, method);
+
+        aml_append(sb_scope, dev);
+    }
+
+    /* build Method(MEMORY_SLOT_NOTIFY_METHOD, 2) {
+     *     If (LEqual(Arg0, 0x00)) {Notify(MP00, Arg1)} ... }
+     */
+    method = aml_method(MEMORY_SLOT_NOTIFY_METHOD, 2, AML_NOTSERIALIZED);
+    for (i = 0; i < nr_mem; i++) {
+        ifctx = aml_if(aml_equal(aml_arg(0), aml_int(i)));
+        aml_append(ifctx,
+            aml_notify(aml_name("MP%.02X", i), aml_arg(1))
+        );
+        aml_append(method, ifctx);
+    }
+    aml_append(sb_scope, method);
+}
+
+static void build_hpet_aml(Aml *table)
+{
+    Aml *crs;
+    Aml *field;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *scope = aml_scope("_SB");
+    Aml *dev = aml_device("HPET");
+    Aml *zero = aml_int(0);
+    Aml *id = aml_local(0);
+    Aml *period = aml_local(1);
+
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0103")));
+    aml_append(dev, aml_name_decl("_UID", zero));
+
+    aml_append(dev,
+        aml_operation_region("HPTM", AML_SYSTEM_MEMORY, aml_int(HPET_BASE),
+                             HPET_LEN));
+    field = aml_field("HPTM", AML_DWORD_ACC, AML_LOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("VEND", 32));
+    aml_append(field, aml_named_field("PRD", 32));
+    aml_append(dev, field);
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("VEND"), id));
+    aml_append(method, aml_store(aml_name("PRD"), period));
+    aml_append(method, aml_shiftright(id, aml_int(16), id));
+    if_ctx = aml_if(aml_lor(aml_equal(id, zero),
+                            aml_equal(id, aml_int(0xffff))));
+    {
+        aml_append(if_ctx, aml_return(zero));
+    }
+    aml_append(method, if_ctx);
+
+    if_ctx = aml_if(aml_lor(aml_equal(period, zero),
+                            aml_lgreater(period, aml_int(100000000))));
+    {
+        aml_append(if_ctx, aml_return(zero));
+    }
+    aml_append(method, if_ctx);
+
+    aml_append(method, aml_return(aml_int(0x0F)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(HPET_BASE, HPET_LEN, AML_READ_ONLY));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static Aml *build_fdinfo_aml(int idx, FloppyDriveType type)
+{
+    Aml *dev, *fdi;
+    uint8_t maxc, maxh, maxs;
+
+    isa_fdc_get_drive_max_chs(type, &maxc, &maxh, &maxs);
+
+    dev = aml_device("FLP%c", 'A' + idx);
+
+    aml_append(dev, aml_name_decl("_ADR", aml_int(idx)));
+
+    fdi = aml_package(16);
+    aml_append(fdi, aml_int(idx));  /* Drive Number */
+    aml_append(fdi,
+        aml_int(cmos_get_fd_drive_type(type)));  /* Device Type */
+    /*
+     * the values below are the limits of the drive, and are thus independent
+     * of the inserted media
+     */
+    aml_append(fdi, aml_int(maxc));  /* Maximum Cylinder Number */
+    aml_append(fdi, aml_int(maxs));  /* Maximum Sector Number */
+    aml_append(fdi, aml_int(maxh));  /* Maximum Head Number */
+    /*
+     * SeaBIOS returns the below values for int 0x13 func 0x08 regardless of
+     * the drive type, so shall we
+     */
+    aml_append(fdi, aml_int(0xAF));  /* disk_specify_1 */
+    aml_append(fdi, aml_int(0x02));  /* disk_specify_2 */
+    aml_append(fdi, aml_int(0x25));  /* disk_motor_wait */
+    aml_append(fdi, aml_int(0x02));  /* disk_sector_siz */
+    aml_append(fdi, aml_int(0x12));  /* disk_eot */
+    aml_append(fdi, aml_int(0x1B));  /* disk_rw_gap */
+    aml_append(fdi, aml_int(0xFF));  /* disk_dtl */
+    aml_append(fdi, aml_int(0x6C));  /* disk_formt_gap */
+    aml_append(fdi, aml_int(0xF6));  /* disk_fill */
+    aml_append(fdi, aml_int(0x0F));  /* disk_head_sttl */
+    aml_append(fdi, aml_int(0x08));  /* disk_motor_strt */
+
+    aml_append(dev, aml_name_decl("_FDI", fdi));
+    return dev;
+}
+
+static Aml *build_fdc_device_aml(ISADevice *fdc)
+{
+    int i;
+    Aml *dev;
+    Aml *crs;
+
+#define ACPI_FDE_MAX_FD 4
+    uint32_t fde_buf[5] = {
+        0, 0, 0, 0,     /* presence of floppy drives #0 - #3 */
+        cpu_to_le32(2)  /* tape presence (2 == never present) */
+    };
+
+    dev = aml_device("FDC0");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0700")));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x03F2, 0x03F2, 0x00, 0x04));
+    aml_append(crs, aml_io(AML_DECODE16, 0x03F7, 0x03F7, 0x00, 0x01));
+    aml_append(crs, aml_irq_no_flags(6));
+    aml_append(crs,
+        aml_dma(AML_COMPATIBILITY, AML_NOTBUSMASTER, AML_TRANSFER8, 2));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    for (i = 0; i < MIN(MAX_FD, ACPI_FDE_MAX_FD); i++) {
+        FloppyDriveType type = isa_fdc_get_drive_type(fdc, i);
+
+        if (type < FLOPPY_DRIVE_TYPE_NONE) {
+            fde_buf[i] = cpu_to_le32(1);  /* drive present */
+            aml_append(dev, build_fdinfo_aml(i, type));
+        }
+    }
+    aml_append(dev, aml_name_decl("_FDE",
+               aml_buffer(sizeof(fde_buf), (uint8_t *)fde_buf)));
+
+    return dev;
+}
+
+static Aml *build_rtc_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+
+    dev = aml_device("RTC");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0B00")));
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0070, 0x0070, 0x10, 0x02));
+    aml_append(crs, aml_irq_no_flags(8));
+    aml_append(crs, aml_io(AML_DECODE16, 0x0072, 0x0072, 0x02, 0x06));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_kbd_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+
+    dev = aml_device("KBD");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0303")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_int(0x0f)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0060, 0x0060, 0x01, 0x01));
+    aml_append(crs, aml_io(AML_DECODE16, 0x0064, 0x0064, 0x01, 0x01));
+    aml_append(crs, aml_irq_no_flags(1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_mouse_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+
+    dev = aml_device("MOU");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0F13")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_int(0x0f)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_irq_no_flags(12));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_lpt_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    Aml *zero = aml_int(0);
+    Aml *is_present = aml_local(0);
+
+    dev = aml_device("LPT");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0400")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("LPEN"), is_present));
+    if_ctx = aml_if(aml_equal(is_present, zero));
+    {
+        aml_append(if_ctx, aml_return(aml_int(0x00)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(aml_int(0x0f)));
+    }
+    aml_append(method, else_ctx);
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0378, 0x0378, 0x08, 0x08));
+    aml_append(crs, aml_irq_no_flags(7));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_com_device_aml(uint8_t uid)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    Aml *zero = aml_int(0);
+    Aml *is_present = aml_local(0);
+    const char *enabled_field = "CAEN";
+    uint8_t irq = 4;
+    uint16_t io_port = 0x03F8;
+
+    assert(uid == 1 || uid == 2);
+    if (uid == 2) {
+        enabled_field = "CBEN";
+        irq = 3;
+        io_port = 0x02F8;
+    }
+
+    dev = aml_device("COM%d", uid);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0501")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("%s", enabled_field), is_present));
+    if_ctx = aml_if(aml_equal(is_present, zero));
+    {
+        aml_append(if_ctx, aml_return(aml_int(0x00)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(aml_int(0x0f)));
+    }
+    aml_append(method, else_ctx);
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, io_port, io_port, 0x00, 0x08));
+    aml_append(crs, aml_irq_no_flags(irq));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static void build_isa_devices_aml(Aml *table)
+{
+    ISADevice *fdc = pc_find_fdc0();
+
+    Aml *scope = aml_scope("_SB.PCI0.ISA");
+
+    aml_append(scope, build_rtc_device_aml());
+    aml_append(scope, build_kbd_device_aml());
+    aml_append(scope, build_mouse_device_aml());
+    if (fdc) {
+        aml_append(scope, build_fdc_device_aml(fdc));
+    }
+    aml_append(scope, build_lpt_device_aml());
+    aml_append(scope, build_com_device_aml(1));
+    aml_append(scope, build_com_device_aml(2));
+
+    aml_append(table, scope);
+}
+
+static void build_dbg_aml(Aml *table)
+{
+    Aml *field;
+    Aml *method;
+    Aml *while_ctx;
+    Aml *scope = aml_scope("\\");
+    Aml *buf = aml_local(0);
+    Aml *len = aml_local(1);
+    Aml *idx = aml_local(2);
+
+    aml_append(scope,
+       aml_operation_region("DBG", AML_SYSTEM_IO, aml_int(0x0402), 0x01));
+    field = aml_field("DBG", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("DBGB", 8));
+    aml_append(scope, field);
+
+    method = aml_method("DBUG", 1, AML_NOTSERIALIZED);
+
+    aml_append(method, aml_to_hexstring(aml_arg(0), buf));
+    aml_append(method, aml_to_buffer(buf, buf));
+    aml_append(method, aml_subtract(aml_sizeof(buf), aml_int(1), len));
+    aml_append(method, aml_store(aml_int(0), idx));
+
+    while_ctx = aml_while(aml_lless(idx, len));
+    aml_append(while_ctx,
+        aml_store(aml_derefof(aml_index(buf, idx)), aml_name("DBGB")));
+    aml_append(while_ctx, aml_increment(idx));
+    aml_append(method, while_ctx);
+
+    aml_append(method, aml_store(aml_int(0x0A), aml_name("DBGB")));
+    aml_append(scope, method);
+
+    aml_append(table, scope);
+}
+
+static Aml *build_link_dev(const char *name, uint8_t uid, Aml *reg)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    uint32_t irqs[] = {5, 10, 11};
+
+    dev = aml_device("%s", name);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_SHARED, irqs, ARRAY_SIZE(irqs)));
+    aml_append(dev, aml_name_decl("_PRS", crs));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call1("IQST", reg)));
+    aml_append(dev, method);
+
+    method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_or(reg, aml_int(0x80), reg));
+    aml_append(dev, method);
+
+    method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call1("IQCR", reg)));
+    aml_append(dev, method);
+
+    method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+    aml_append(method, aml_create_dword_field(aml_arg(0), aml_int(5), "PRRI"));
+    aml_append(method, aml_store(aml_name("PRRI"), reg));
+    aml_append(dev, method);
+
+    return dev;
+ }
+
+static Aml *build_gsi_link_dev(const char *name, uint8_t uid, uint8_t gsi)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    uint32_t irqs;
+
+    dev = aml_device("%s", name);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    crs = aml_resource_template();
+    irqs = gsi;
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_SHARED, &irqs, 1));
+    aml_append(dev, aml_name_decl("_PRS", crs));
+
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    /*
+     * _DIS can be no-op because the interrupt cannot be disabled.
+     */
+    method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+    aml_append(dev, method);
+
+    method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+    aml_append(dev, method);
+
+    return dev;
+}
+
+/* _CRS method - get current settings */
+static Aml *build_iqcr_method(bool is_piix4)
+{
+    Aml *if_ctx;
+    uint32_t irqs;
+    Aml *method = aml_method("IQCR", 1, AML_SERIALIZED);
+    Aml *crs = aml_resource_template();
+
+    irqs = 0;
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL,
+                                  AML_ACTIVE_HIGH, AML_SHARED, &irqs, 1));
+    aml_append(method, aml_name_decl("PRR0", crs));
+
+    aml_append(method,
+        aml_create_dword_field(aml_name("PRR0"), aml_int(5), "PRRI"));
+
+    if (is_piix4) {
+        if_ctx = aml_if(aml_lless(aml_arg(0), aml_int(0x80)));
+        aml_append(if_ctx, aml_store(aml_arg(0), aml_name("PRRI")));
+        aml_append(method, if_ctx);
+    } else {
+        aml_append(method,
+            aml_store(aml_and(aml_arg(0), aml_int(0xF), NULL),
+                      aml_name("PRRI")));
+    }
+
+    aml_append(method, aml_return(aml_name("PRR0")));
+    return method;
+}
+
+/* _STA method - get status */
+static Aml *build_irq_status_method(void)
+{
+    Aml *if_ctx;
+    Aml *method = aml_method("IQST", 1, AML_NOTSERIALIZED);
+
+    if_ctx = aml_if(aml_and(aml_int(0x80), aml_arg(0), NULL));
+    aml_append(if_ctx, aml_return(aml_int(0x09)));
+    aml_append(method, if_ctx);
+    aml_append(method, aml_return(aml_int(0x0B)));
+    return method;
+}
+
+static void build_piix4_pci0_int(Aml *table)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *field;
+    Aml *method;
+    uint32_t irqs;
+    Aml *sb_scope = aml_scope("_SB");
+    Aml *pci0_scope = aml_scope("PCI0");
+
+    aml_append(pci0_scope, build_prt(true));
+    aml_append(sb_scope, pci0_scope);
+
+    field = aml_field("PCI0.ISA.P40C", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRQ0", 8));
+    aml_append(field, aml_named_field("PRQ1", 8));
+    aml_append(field, aml_named_field("PRQ2", 8));
+    aml_append(field, aml_named_field("PRQ3", 8));
+    aml_append(sb_scope, field);
+
+    aml_append(sb_scope, build_irq_status_method());
+    aml_append(sb_scope, build_iqcr_method(true));
+
+    aml_append(sb_scope, build_link_dev("LNKA", 0, aml_name("PRQ0")));
+    aml_append(sb_scope, build_link_dev("LNKB", 1, aml_name("PRQ1")));
+    aml_append(sb_scope, build_link_dev("LNKC", 2, aml_name("PRQ2")));
+    aml_append(sb_scope, build_link_dev("LNKD", 3, aml_name("PRQ3")));
+
+    dev = aml_device("LNKS");
+    {
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+        aml_append(dev, aml_name_decl("_UID", aml_int(4)));
+
+        crs = aml_resource_template();
+        irqs = 9;
+        aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL,
+                                      AML_ACTIVE_HIGH, AML_SHARED,
+                                      &irqs, 1));
+        aml_append(dev, aml_name_decl("_PRS", crs));
+
+        /* The SCI cannot be disabled and is always attached to GSI 9,
+         * so these are no-ops.  We only need this link to override the
+         * polarity to active high and match the content of the MADT.
+         */
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_return(aml_int(0x0b)));
+        aml_append(dev, method);
+
+        method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+        aml_append(dev, method);
+
+        method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_return(aml_name("_PRS")));
+        aml_append(dev, method);
+
+        method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+        aml_append(dev, method);
+    }
+    aml_append(sb_scope, dev);
+
+    aml_append(table, sb_scope);
+}
+
+static void append_q35_prt_entry(Aml *ctx, uint32_t nr, const char *name)
+{
+    int i;
+    int head;
+    Aml *pkg;
+    char base = name[3] < 'E' ? 'A' : 'E';
+    char *s = g_strdup(name);
+    Aml *a_nr = aml_int((nr << 16) | 0xffff);
+
+    assert(strlen(s) == 4);
+
+    head = name[3] - base;
+    for (i = 0; i < 4; i++) {
+        if (head + i > 3) {
+            head = i * -1;
+        }
+        s[3] = base + head + i;
+        pkg = aml_package(4);
+        aml_append(pkg, a_nr);
+        aml_append(pkg, aml_int(i));
+        aml_append(pkg, aml_name("%s", s));
+        aml_append(pkg, aml_int(0));
+        aml_append(ctx, pkg);
+    }
+    g_free(s);
+}
+
+static Aml *build_q35_routing_table(const char *str)
+{
+    int i;
+    Aml *pkg;
+    char *name = g_strdup_printf("%s ", str);
+
+    pkg = aml_package(128);
+    for (i = 0; i < 0x18; i++) {
+            name[3] = 'E' + (i & 0x3);
+            append_q35_prt_entry(pkg, i, name);
+    }
+
+    name[3] = 'E';
+    append_q35_prt_entry(pkg, 0x18, name);
+
+    /* INTA -> PIRQA for slot 25 - 31, see the default value of D<N>IR */
+    for (i = 0x0019; i < 0x1e; i++) {
+        name[3] = 'A';
+        append_q35_prt_entry(pkg, i, name);
+    }
+
+    /* PCIe->PCI bridge. use PIRQ[E-H] */
+    name[3] = 'E';
+    append_q35_prt_entry(pkg, 0x1e, name);
+    name[3] = 'A';
+    append_q35_prt_entry(pkg, 0x1f, name);
+
+    g_free(name);
+    return pkg;
+}
+
+static void build_q35_pci0_int(Aml *table)
+{
+    Aml *field;
+    Aml *method;
+    Aml *sb_scope = aml_scope("_SB");
+    Aml *pci0_scope = aml_scope("PCI0");
+
+    /* Zero => PIC mode, One => APIC Mode */
+    aml_append(table, aml_name_decl("PICF", aml_int(0)));
+    method = aml_method("_PIC", 1, AML_NOTSERIALIZED);
+    {
+        aml_append(method, aml_store(aml_arg(0), aml_name("PICF")));
+    }
+    aml_append(table, method);
+
+    aml_append(pci0_scope,
+        aml_name_decl("PRTP", build_q35_routing_table("LNK")));
+    aml_append(pci0_scope,
+        aml_name_decl("PRTA", build_q35_routing_table("GSI")));
+
+    method = aml_method("_PRT", 0, AML_NOTSERIALIZED);
+    {
+        Aml *if_ctx;
+        Aml *else_ctx;
+
+        /* PCI IRQ routing table, example from ACPI 2.0a specification,
+           section 6.2.8.1 */
+        /* Note: we provide the same info as the PCI routing
+           table of the Bochs BIOS */
+        if_ctx = aml_if(aml_equal(aml_name("PICF"), aml_int(0)));
+        aml_append(if_ctx, aml_return(aml_name("PRTP")));
+        aml_append(method, if_ctx);
+        else_ctx = aml_else();
+        aml_append(else_ctx, aml_return(aml_name("PRTA")));
+        aml_append(method, else_ctx);
+    }
+    aml_append(pci0_scope, method);
+    aml_append(sb_scope, pci0_scope);
+
+    field = aml_field("PCI0.ISA.PIRQ", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRQA", 8));
+    aml_append(field, aml_named_field("PRQB", 8));
+    aml_append(field, aml_named_field("PRQC", 8));
+    aml_append(field, aml_named_field("PRQD", 8));
+    aml_append(field, aml_reserved_field(0x20));
+    aml_append(field, aml_named_field("PRQE", 8));
+    aml_append(field, aml_named_field("PRQF", 8));
+    aml_append(field, aml_named_field("PRQG", 8));
+    aml_append(field, aml_named_field("PRQH", 8));
+    aml_append(sb_scope, field);
+
+    aml_append(sb_scope, build_irq_status_method());
+    aml_append(sb_scope, build_iqcr_method(false));
+
+    aml_append(sb_scope, build_link_dev("LNKA", 0, aml_name("PRQA")));
+    aml_append(sb_scope, build_link_dev("LNKB", 1, aml_name("PRQB")));
+    aml_append(sb_scope, build_link_dev("LNKC", 2, aml_name("PRQC")));
+    aml_append(sb_scope, build_link_dev("LNKD", 3, aml_name("PRQD")));
+    aml_append(sb_scope, build_link_dev("LNKE", 4, aml_name("PRQE")));
+    aml_append(sb_scope, build_link_dev("LNKF", 5, aml_name("PRQF")));
+    aml_append(sb_scope, build_link_dev("LNKG", 6, aml_name("PRQG")));
+    aml_append(sb_scope, build_link_dev("LNKH", 7, aml_name("PRQH")));
+
+    aml_append(sb_scope, build_gsi_link_dev("GSIA", 0x10, 0x10));
+    aml_append(sb_scope, build_gsi_link_dev("GSIB", 0x11, 0x11));
+    aml_append(sb_scope, build_gsi_link_dev("GSIC", 0x12, 0x12));
+    aml_append(sb_scope, build_gsi_link_dev("GSID", 0x13, 0x13));
+    aml_append(sb_scope, build_gsi_link_dev("GSIE", 0x14, 0x14));
+    aml_append(sb_scope, build_gsi_link_dev("GSIF", 0x15, 0x15));
+    aml_append(sb_scope, build_gsi_link_dev("GSIG", 0x16, 0x16));
+    aml_append(sb_scope, build_gsi_link_dev("GSIH", 0x17, 0x17));
+
+    aml_append(table, sb_scope);
+}
+
+static void build_q35_isa_bridge(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+    Aml *field;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("ISA");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x001F0000)));
+
+    /* ICH9 PCI to ISA irq remapping */
+    aml_append(dev, aml_operation_region("PIRQ", AML_PCI_CONFIG,
+                                         aml_int(0x60), 0x0C));
+
+    aml_append(dev, aml_operation_region("LPCD", AML_PCI_CONFIG,
+                                         aml_int(0x80), 0x02));
+    field = aml_field("LPCD", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("COMA", 3));
+    aml_append(field, aml_reserved_field(1));
+    aml_append(field, aml_named_field("COMB", 3));
+    aml_append(field, aml_reserved_field(1));
+    aml_append(field, aml_named_field("LPTD", 2));
+    aml_append(dev, field);
+
+    aml_append(dev, aml_operation_region("LPCE", AML_PCI_CONFIG,
+                                         aml_int(0x82), 0x02));
+    /* enable bits */
+    field = aml_field("LPCE", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("CAEN", 1));
+    aml_append(field, aml_named_field("CBEN", 1));
+    aml_append(field, aml_named_field("LPEN", 1));
+    aml_append(dev, field);
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_pm(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("PX13");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x00010003)));
+
+    aml_append(dev, aml_operation_region("P13C", AML_PCI_CONFIG,
+                                         aml_int(0x00), 0xff));
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_isa_bridge(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+    Aml *field;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("ISA");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x00010000)));
+
+    /* PIIX PCI to ISA irq remapping */
+    aml_append(dev, aml_operation_region("P40C", AML_PCI_CONFIG,
+                                         aml_int(0x60), 0x04));
+    /* enable bits */
+    field = aml_field("^PX13.P13C", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    /* Offset(0x5f),, 7, */
+    aml_append(field, aml_reserved_field(0x2f8));
+    aml_append(field, aml_reserved_field(7));
+    aml_append(field, aml_named_field("LPEN", 1));
+    /* Offset(0x67),, 3, */
+    aml_append(field, aml_reserved_field(0x38));
+    aml_append(field, aml_reserved_field(3));
+    aml_append(field, aml_named_field("CAEN", 1));
+    aml_append(field, aml_reserved_field(3));
+    aml_append(field, aml_named_field("CBEN", 1));
+    aml_append(dev, field);
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_pci_hotplug(Aml *table)
+{
+    Aml *scope;
+    Aml *field;
+    Aml *method;
+
+    scope =  aml_scope("_SB.PCI0");
+
+    aml_append(scope,
+        aml_operation_region("PCST", AML_SYSTEM_IO, aml_int(0xae00), 0x08));
+    field = aml_field("PCST", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("PCIU", 32));
+    aml_append(field, aml_named_field("PCID", 32));
+    aml_append(scope, field);
+
+    aml_append(scope,
+        aml_operation_region("SEJ", AML_SYSTEM_IO, aml_int(0xae08), 0x04));
+    field = aml_field("SEJ", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("B0EJ", 32));
+    aml_append(scope, field);
+
+    aml_append(scope,
+        aml_operation_region("BNMR", AML_SYSTEM_IO, aml_int(0xae10), 0x04));
+    field = aml_field("BNMR", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("BNUM", 32));
+    aml_append(scope, field);
+
+    aml_append(scope, aml_mutex("BLCK", 0));
+
+    method = aml_method("PCEJ", 2, AML_NOTSERIALIZED);
+    aml_append(method, aml_acquire(aml_name("BLCK"), 0xFFFF));
+    aml_append(method, aml_store(aml_arg(0), aml_name("BNUM")));
+    aml_append(method,
+        aml_store(aml_shiftleft(aml_int(1), aml_arg(1)), aml_name("B0EJ")));
+    aml_append(method, aml_release(aml_name("BLCK")));
+    aml_append(method, aml_return(aml_int(0)));
+    aml_append(scope, method);
+
+    aml_append(table, scope);
+}
+
+static Aml *build_q35_osc_method(void)
+{
+    Aml *if_ctx;
+    Aml *if_ctx2;
+    Aml *else_ctx;
+    Aml *method;
+    Aml *a_cwd1 = aml_name("CDW1");
+    Aml *a_ctrl = aml_name("CTRL");
+
+    method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
+    aml_append(method, aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
+
+    if_ctx = aml_if(aml_equal(
+        aml_arg(0), aml_touuid("33DB4D5B-1FF7-401C-9657-7441C03DD766")));
+    aml_append(if_ctx, aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2"));
+    aml_append(if_ctx, aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
+
+    aml_append(if_ctx, aml_store(aml_name("CDW2"), aml_name("SUPP")));
+    aml_append(if_ctx, aml_store(aml_name("CDW3"), a_ctrl));
+
+    /*
+     * Always allow native PME, AER (no dependencies)
+     * Never allow SHPC (no SHPC controller in this system)
+     */
+    aml_append(if_ctx, aml_and(a_ctrl, aml_int(0x1D), a_ctrl));
+
+    if_ctx2 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(1))));
+    /* Unknown revision */
+    aml_append(if_ctx2, aml_or(a_cwd1, aml_int(0x08), a_cwd1));
+    aml_append(if_ctx, if_ctx2);
+
+    if_ctx2 = aml_if(aml_lnot(aml_equal(aml_name("CDW3"), a_ctrl)));
+    /* Capabilities bits were masked */
+    aml_append(if_ctx2, aml_or(a_cwd1, aml_int(0x10), a_cwd1));
+    aml_append(if_ctx, if_ctx2);
+
+    /* Update DWORD3 in the buffer */
+    aml_append(if_ctx, aml_store(a_ctrl, aml_name("CDW3")));
+    aml_append(method, if_ctx);
+
+    else_ctx = aml_else();
+    /* Unrecognized UUID */
+    aml_append(else_ctx, aml_or(a_cwd1, aml_int(4), a_cwd1));
+    aml_append(method, else_ctx);
+
+    aml_append(method, aml_return(aml_arg(3)));
+    return method;
+}
+
 static void
-build_ssdt(GArray *table_data, GArray *linker,
-           AcpiCpuInfo *cpu, AcpiPmInfo *pm, AcpiMiscInfo *misc,
-           PcPciInfo *pci, PcGuestInfo *guest_info)
+build_dsdt(GArray *table_data, GArray *linker,
+           AcpiPmInfo *pm, AcpiMiscInfo *misc,
+           PcPciInfo *pci, MachineState *machine)
 {
-    MachineState *machine = MACHINE(qdev_get_machine());
-    uint32_t nr_mem = machine->ram_slots;
-    unsigned acpi_cpus = guest_info->apic_id_limit;
-    Aml *ssdt, *sb_scope, *scope, *pkg, *dev, *method, *crs, *field, *ifctx;
-    PCIBus *bus = NULL;
-    GPtrArray *io_ranges = g_ptr_array_new_with_free_func(crs_range_free);
-    GPtrArray *mem_ranges = g_ptr_array_new_with_free_func(crs_range_free);
     CrsRangeEntry *entry;
+    Aml *dsdt, *sb_scope, *scope, *dev, *method, *field, *pkg, *crs;
+    GPtrArray *mem_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    GPtrArray *io_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    PCMachineState *pcms = PC_MACHINE(machine);
+    uint32_t nr_mem = machine->ram_slots;
     int root_bus_limit = 0xFF;
+    PCIBus *bus = NULL;
     int i;
 
-    ssdt = init_aml_allocator();
-    /* The current AML generator can cover the APIC ID range [0..255],
-     * inclusive, for VCPU hotplug. */
-    QEMU_BUILD_BUG_ON(ACPI_CPU_HOTPLUG_ID_LIMIT > 256);
-    g_assert(acpi_cpus <= ACPI_CPU_HOTPLUG_ID_LIMIT);
+    dsdt = init_aml_allocator();
 
     /* Reserve space for header */
-    acpi_data_push(ssdt->buf, sizeof(AcpiTableHeader));
+    acpi_data_push(dsdt->buf, sizeof(AcpiTableHeader));
+
+    build_dbg_aml(dsdt);
+    if (misc->is_piix4) {
+        sb_scope = aml_scope("_SB");
+        dev = aml_device("PCI0");
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A03")));
+        aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+        aml_append(dev, aml_name_decl("_UID", aml_int(1)));
+        aml_append(sb_scope, dev);
+        aml_append(dsdt, sb_scope);
+
+        build_hpet_aml(dsdt);
+        build_piix4_pm(dsdt);
+        build_piix4_isa_bridge(dsdt);
+        build_isa_devices_aml(dsdt);
+        build_piix4_pci_hotplug(dsdt);
+        build_piix4_pci0_int(dsdt);
+    } else {
+        sb_scope = aml_scope("_SB");
+        aml_append(sb_scope,
+            aml_operation_region("PCST", AML_SYSTEM_IO, aml_int(0xae00), 0x0c));
+        aml_append(sb_scope,
+            aml_operation_region("PCSB", AML_SYSTEM_IO, aml_int(0xae0c), 0x01));
+        field = aml_field("PCSB", AML_ANY_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+        aml_append(field, aml_named_field("PCIB", 8));
+        aml_append(sb_scope, field);
+        aml_append(dsdt, sb_scope);
+
+        sb_scope = aml_scope("_SB");
+        dev = aml_device("PCI0");
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A08")));
+        aml_append(dev, aml_name_decl("_CID", aml_eisaid("PNP0A03")));
+        aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+        aml_append(dev, aml_name_decl("_UID", aml_int(1)));
+        aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
+        aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
+        aml_append(dev, build_q35_osc_method());
+        aml_append(sb_scope, dev);
+        aml_append(dsdt, sb_scope);
+
+        build_hpet_aml(dsdt);
+        build_q35_isa_bridge(dsdt);
+        build_isa_devices_aml(dsdt);
+        build_q35_pci0_int(dsdt);
+    }
 
-    /* Extra PCI root buses are implemented  only for i440fx */
-    bus = find_i440fx();
+    build_cpu_hotplug_aml(dsdt);
+    build_memory_hotplug_aml(dsdt, nr_mem, pm->mem_hp_io_base,
+                             pm->mem_hp_io_len);
+
+    scope =  aml_scope("_GPE");
+    {
+        aml_append(scope, aml_name_decl("_HID", aml_string("ACPI0006")));
+
+        aml_append(scope, aml_method("_L00", 0, AML_NOTSERIALIZED));
+
+        if (misc->is_piix4) {
+            method = aml_method("_E01", 0, AML_NOTSERIALIZED);
+            aml_append(method,
+                aml_acquire(aml_name("\\_SB.PCI0.BLCK"), 0xFFFF));
+            aml_append(method, aml_call0("\\_SB.PCI0.PCNT"));
+            aml_append(method, aml_release(aml_name("\\_SB.PCI0.BLCK")));
+            aml_append(scope, method);
+        } else {
+            aml_append(scope, aml_method("_L01", 0, AML_NOTSERIALIZED));
+        }
+
+        method = aml_method("_E02", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_call0("\\_SB." CPU_SCAN_METHOD));
+        aml_append(scope, method);
+
+        method = aml_method("_E03", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_call0(MEMORY_HOTPLUG_HANDLER_PATH));
+        aml_append(scope, method);
+
+        aml_append(scope, aml_method("_L04", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L05", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L06", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L07", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L08", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L09", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0A", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0B", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0C", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0D", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0E", 0, AML_NOTSERIALIZED));
+        aml_append(scope, aml_method("_L0F", 0, AML_NOTSERIALIZED));
+    }
+    aml_append(dsdt, scope);
+
+    bus = PC_MACHINE(machine)->bus;
     if (bus) {
         QLIST_FOREACH(bus, &bus->child, sibling) {
             uint8_t bus_num = pci_bus_num(bus);
@@ -950,12 +2112,12 @@ build_ssdt(GArray *table_data, GArray *linker,
                 aml_append(dev, aml_name_decl("_PXM", aml_int(numa_node)));
             }
 
-            aml_append(dev, build_prt());
+            aml_append(dev, build_prt(false));
             crs = build_crs(PCI_HOST_BRIDGE(BUS(bus)->parent),
                             io_ranges, mem_ranges);
             aml_append(dev, aml_name_decl("_CRS", crs));
             aml_append(scope, dev);
-            aml_append(ssdt, scope);
+            aml_append(dsdt, scope);
         }
     }
 
@@ -1005,6 +2167,11 @@ build_ssdt(GArray *table_data, GArray *linker,
                              0, pci->w64.begin, pci->w64.end - 1, 0,
                              pci->w64.end - pci->w64.begin));
     }
+
+    if (misc->tpm_version != TPM_VERSION_UNSPEC) {
+        aml_append(crs, aml_memory32_fixed(TPM_TIS_ADDR_BASE,
+                   TPM_TIS_ADDR_SIZE, AML_READ_WRITE));
+    }
     aml_append(scope, aml_name_decl("_CRS", crs));
 
     /* reserve GPE0 block resources */
@@ -1039,7 +2206,7 @@ build_ssdt(GArray *table_data, GArray *linker,
         aml_append(dev, aml_name_decl("_CRS", crs));
         aml_append(scope, dev);
     }
-    aml_append(ssdt, scope);
+    aml_append(dsdt, scope);
 
     /*  create S3_ / S4_ / S5_ packages if necessary */
     scope = aml_scope("\\");
@@ -1068,7 +2235,36 @@ build_ssdt(GArray *table_data, GArray *linker,
     aml_append(pkg, aml_int(0)); /* reserved */
     aml_append(pkg, aml_int(0)); /* reserved */
     aml_append(scope, aml_name_decl("_S5", pkg));
-    aml_append(ssdt, scope);
+    aml_append(dsdt, scope);
+
+    /* create fw_cfg node, unconditionally */
+    {
+        /* when using port i/o, the 8-bit data register *always* overlaps
+         * with half of the 16-bit control register. Hence, the total size
+         * of the i/o region used is FW_CFG_CTL_SIZE; when using DMA, the
+         * DMA control register is located at FW_CFG_DMA_IO_BASE + 4 */
+        uint8_t io_size = object_property_get_bool(OBJECT(pcms->fw_cfg),
+                                                   "dma_enabled", NULL) ?
+                          ROUND_UP(FW_CFG_CTL_SIZE, 4) + sizeof(dma_addr_t) :
+                          FW_CFG_CTL_SIZE;
+
+        scope = aml_scope("\\_SB.PCI0");
+        dev = aml_device("FWCF");
+
+        aml_append(dev, aml_name_decl("_HID", aml_string("QEMU0002")));
+
+        /* device present, functioning, decoding, not shown in UI */
+        aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+
+        crs = aml_resource_template();
+        aml_append(crs,
+            aml_io(AML_DECODE16, FW_CFG_IO_BASE, FW_CFG_IO_BASE, 0x01, io_size)
+        );
+        aml_append(dev, aml_name_decl("_CRS", crs));
+
+        aml_append(scope, dev);
+        aml_append(dsdt, scope);
+    }
 
     if (misc->applesmc_io_base) {
         scope = aml_scope("\\_SB.PCI0.ISA");
@@ -1087,7 +2283,7 @@ build_ssdt(GArray *table_data, GArray *linker,
         aml_append(dev, aml_name_decl("_CRS", crs));
 
         aml_append(scope, dev);
-        aml_append(ssdt, scope);
+        aml_append(dsdt, scope);
     }
 
     if (misc->pvpanic_port) {
@@ -1103,214 +2299,33 @@ build_ssdt(GArray *table_data, GArray *linker,
         aml_append(dev, aml_name_decl("_CRS", crs));
 
         aml_append(dev, aml_operation_region("PEOR", AML_SYSTEM_IO,
-                                              misc->pvpanic_port, 1));
-        field = aml_field("PEOR", AML_BYTE_ACC, AML_PRESERVE);
+                                              aml_int(misc->pvpanic_port), 1));
+        field = aml_field("PEOR", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
         aml_append(field, aml_named_field("PEPT", 8));
         aml_append(dev, field);
 
         /* device present, functioning, decoding, shown in UI */
         aml_append(dev, aml_name_decl("_STA", aml_int(0xF)));
 
-        method = aml_method("RDPT", 0);
+        method = aml_method("RDPT", 0, AML_NOTSERIALIZED);
         aml_append(method, aml_store(aml_name("PEPT"), aml_local(0)));
         aml_append(method, aml_return(aml_local(0)));
         aml_append(dev, method);
 
-        method = aml_method("WRPT", 1);
+        method = aml_method("WRPT", 1, AML_NOTSERIALIZED);
         aml_append(method, aml_store(aml_arg(0), aml_name("PEPT")));
         aml_append(dev, method);
 
         aml_append(scope, dev);
-        aml_append(ssdt, scope);
+        aml_append(dsdt, scope);
     }
 
     sb_scope = aml_scope("\\_SB");
     {
-        /* create PCI0.PRES device and its _CRS to reserve CPU hotplug MMIO */
-        dev = aml_device("PCI0." stringify(CPU_HOTPLUG_RESOURCE_DEVICE));
-        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A06")));
-        aml_append(dev,
-            aml_name_decl("_UID", aml_string("CPU Hotplug resources"))
-        );
-        /* device present, functioning, decoding, not shown in UI */
-        aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
-        crs = aml_resource_template();
-        aml_append(crs,
-            aml_io(AML_DECODE16, pm->cpu_hp_io_base, pm->cpu_hp_io_base, 1,
-                   pm->cpu_hp_io_len)
-        );
-        aml_append(dev, aml_name_decl("_CRS", crs));
-        aml_append(sb_scope, dev);
-        /* declare CPU hotplug MMIO region and PRS field to access it */
-        aml_append(sb_scope, aml_operation_region(
-            "PRST", AML_SYSTEM_IO, pm->cpu_hp_io_base, pm->cpu_hp_io_len));
-        field = aml_field("PRST", AML_BYTE_ACC, AML_PRESERVE);
-        aml_append(field, aml_named_field("PRS", 256));
-        aml_append(sb_scope, field);
-
-        /* build Processor object for each processor */
-        for (i = 0; i < acpi_cpus; i++) {
-            dev = aml_processor(i, 0, 0, "CP%.02X", i);
-
-            method = aml_method("_MAT", 0);
-            aml_append(method, aml_return(aml_call1("CPMA", aml_int(i))));
-            aml_append(dev, method);
-
-            method = aml_method("_STA", 0);
-            aml_append(method, aml_return(aml_call1("CPST", aml_int(i))));
-            aml_append(dev, method);
-
-            method = aml_method("_EJ0", 1);
-            aml_append(method,
-                aml_return(aml_call2("CPEJ", aml_int(i), aml_arg(0)))
-            );
-            aml_append(dev, method);
+        build_processor_devices(sb_scope, machine, pm);
 
-            aml_append(sb_scope, dev);
-        }
-
-        /* build this code:
-         *   Method(NTFY, 2) {If (LEqual(Arg0, 0x00)) {Notify(CP00, Arg1)} ...}
-         */
-        /* Arg0 = Processor ID = APIC ID */
-        method = aml_method("NTFY", 2);
-        for (i = 0; i < acpi_cpus; i++) {
-            ifctx = aml_if(aml_equal(aml_arg(0), aml_int(i)));
-            aml_append(ifctx,
-                aml_notify(aml_name("CP%.02X", i), aml_arg(1))
-            );
-            aml_append(method, ifctx);
-        }
-        aml_append(sb_scope, method);
-
-        /* build "Name(CPON, Package() { One, One, ..., Zero, Zero, ... })"
-         *
-         * Note: The ability to create variable-sized packages was first
-         * introduced in ACPI 2.0. ACPI 1.0 only allowed fixed-size packages
-         * ith up to 255 elements. Windows guests up to win2k8 fail when
-         * VarPackageOp is used.
-         */
-        pkg = acpi_cpus <= 255 ? aml_package(acpi_cpus) :
-                                 aml_varpackage(acpi_cpus);
-
-        for (i = 0; i < acpi_cpus; i++) {
-            uint8_t b = test_bit(i, cpu->found_cpus) ? 0x01 : 0x00;
-            aml_append(pkg, aml_int(b));
-        }
-        aml_append(sb_scope, aml_name_decl("CPON", pkg));
-
-        /* build memory devices */
-        assert(nr_mem <= ACPI_MAX_RAM_SLOTS);
-        scope = aml_scope("\\_SB.PCI0." stringify(MEMORY_HOTPLUG_DEVICE));
-        aml_append(scope,
-            aml_name_decl(stringify(MEMORY_SLOTS_NUMBER), aml_int(nr_mem))
-        );
-
-        crs = aml_resource_template();
-        aml_append(crs,
-            aml_io(AML_DECODE16, pm->mem_hp_io_base, pm->mem_hp_io_base, 0,
-                   pm->mem_hp_io_len)
-        );
-        aml_append(scope, aml_name_decl("_CRS", crs));
-
-        aml_append(scope, aml_operation_region(
-            stringify(MEMORY_HOTPLUG_IO_REGION), AML_SYSTEM_IO,
-            pm->mem_hp_io_base, pm->mem_hp_io_len)
-        );
-
-        field = aml_field(stringify(MEMORY_HOTPLUG_IO_REGION), AML_DWORD_ACC,
-                          AML_PRESERVE);
-        aml_append(field, /* read only */
-            aml_named_field(stringify(MEMORY_SLOT_ADDR_LOW), 32));
-        aml_append(field, /* read only */
-            aml_named_field(stringify(MEMORY_SLOT_ADDR_HIGH), 32));
-        aml_append(field, /* read only */
-            aml_named_field(stringify(MEMORY_SLOT_SIZE_LOW), 32));
-        aml_append(field, /* read only */
-            aml_named_field(stringify(MEMORY_SLOT_SIZE_HIGH), 32));
-        aml_append(field, /* read only */
-            aml_named_field(stringify(MEMORY_SLOT_PROXIMITY), 32));
-        aml_append(scope, field);
-
-        field = aml_field(stringify(MEMORY_HOTPLUG_IO_REGION), AML_BYTE_ACC,
-                          AML_WRITE_AS_ZEROS);
-        aml_append(field, aml_reserved_field(160 /* bits, Offset(20) */));
-        aml_append(field, /* 1 if enabled, read only */
-            aml_named_field(stringify(MEMORY_SLOT_ENABLED), 1));
-        aml_append(field,
-            /*(read) 1 if has a insert event. (write) 1 to clear event */
-            aml_named_field(stringify(MEMORY_SLOT_INSERT_EVENT), 1));
-        aml_append(field,
-            /* (read) 1 if has a remove event. (write) 1 to clear event */
-            aml_named_field(stringify(MEMORY_SLOT_REMOVE_EVENT), 1));
-        aml_append(field,
-            /* initiates device eject, write only */
-            aml_named_field(stringify(MEMORY_SLOT_EJECT), 1));
-        aml_append(scope, field);
-
-        field = aml_field(stringify(MEMORY_HOTPLUG_IO_REGION), AML_DWORD_ACC,
-                          AML_PRESERVE);
-        aml_append(field, /* DIMM selector, write only */
-            aml_named_field(stringify(MEMORY_SLOT_SLECTOR), 32));
-        aml_append(field, /* _OST event code, write only */
-            aml_named_field(stringify(MEMORY_SLOT_OST_EVENT), 32));
-        aml_append(field, /* _OST status code, write only */
-            aml_named_field(stringify(MEMORY_SLOT_OST_STATUS), 32));
-        aml_append(scope, field);
-
-        aml_append(sb_scope, scope);
-
-        for (i = 0; i < nr_mem; i++) {
-            #define BASEPATH "\\_SB.PCI0." stringify(MEMORY_HOTPLUG_DEVICE) "."
-            const char *s;
-
-            dev = aml_device("MP%02X", i);
-            aml_append(dev, aml_name_decl("_UID", aml_string("0x%02X", i)));
-            aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C80")));
-
-            method = aml_method("_CRS", 0);
-            s = BASEPATH stringify(MEMORY_SLOT_CRS_METHOD);
-            aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
-            aml_append(dev, method);
-
-            method = aml_method("_STA", 0);
-            s = BASEPATH stringify(MEMORY_SLOT_STATUS_METHOD);
-            aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
-            aml_append(dev, method);
-
-            method = aml_method("_PXM", 0);
-            s = BASEPATH stringify(MEMORY_SLOT_PROXIMITY_METHOD);
-            aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
-            aml_append(dev, method);
-
-            method = aml_method("_OST", 3);
-            s = BASEPATH stringify(MEMORY_SLOT_OST_METHOD);
-            aml_append(method, aml_return(aml_call4(
-                s, aml_name("_UID"), aml_arg(0), aml_arg(1), aml_arg(2)
-            )));
-            aml_append(dev, method);
-
-            method = aml_method("_EJ0", 1);
-            s = BASEPATH stringify(MEMORY_SLOT_EJECT_METHOD);
-            aml_append(method, aml_return(aml_call2(
-                       s, aml_name("_UID"), aml_arg(0))));
-            aml_append(dev, method);
-
-            aml_append(sb_scope, dev);
-        }
-
-        /* build Method(MEMORY_SLOT_NOTIFY_METHOD, 2) {
-         *     If (LEqual(Arg0, 0x00)) {Notify(MP00, Arg1)} ... }
-         */
-        method = aml_method(stringify(MEMORY_SLOT_NOTIFY_METHOD), 2);
-        for (i = 0; i < nr_mem; i++) {
-            ifctx = aml_if(aml_equal(aml_arg(0), aml_int(i)));
-            aml_append(ifctx,
-                aml_notify(aml_name("MP%.02X", i), aml_arg(1))
-            );
-            aml_append(method, ifctx);
-        }
-        aml_append(sb_scope, method);
+        build_memory_devices(sb_scope, nr_mem, pm->mem_hp_io_base,
+                             pm->mem_hp_io_len);
 
         {
             Object *pci_host;
@@ -1333,7 +2348,12 @@ build_ssdt(GArray *table_data, GArray *linker,
                     crs = aml_resource_template();
                     aml_append(crs, aml_memory32_fixed(TPM_TIS_ADDR_BASE,
                                TPM_TIS_ADDR_SIZE, AML_READ_WRITE));
-                    aml_append(crs, aml_irq_no_flags(TPM_TIS_IRQ));
+                    /*
+                        FIXME: TPM_TIS_IRQ=5 conflicts with PNP0C0F irqs,
+                        Rewrite to take IRQ from TPM device model and
+                        fix default IRQ value there to use some unused IRQ
+                     */
+                    /* aml_append(crs, aml_irq_no_flags(TPM_TIS_IRQ)); */
                     aml_append(dev, aml_name_decl("_CRS", crs));
                     aml_append(scope, dev);
                 }
@@ -1341,14 +2361,14 @@ build_ssdt(GArray *table_data, GArray *linker,
                 aml_append(sb_scope, scope);
             }
         }
-        aml_append(ssdt, sb_scope);
+        aml_append(dsdt, sb_scope);
     }
 
     /* copy AML table into ACPI tables blob and patch header there */
-    g_array_append_vals(table_data, ssdt->buf->data, ssdt->buf->len);
+    g_array_append_vals(table_data, dsdt->buf->data, dsdt->buf->len);
     build_header(linker, table_data,
-        (void *)(table_data->data + table_data->len - ssdt->buf->len),
-        "SSDT", ssdt->buf->len, 1);
+        (void *)(table_data->data + table_data->len - dsdt->buf->len),
+        "DSDT", dsdt->buf->len, 1, NULL, NULL);
     free_aml_allocator();
 }
 
@@ -1364,7 +2384,7 @@ build_hpet(GArray *table_data, GArray *linker)
     hpet->timer_block_id = cpu_to_le32(0x8086a201);
     hpet->addr.address = cpu_to_le64(HPET_BASE);
     build_header(linker, table_data,
-                 (void *)hpet, "HPET", sizeof(*hpet), 1);
+                 (void *)hpet, "HPET", sizeof(*hpet), 1, NULL, NULL);
 }
 
 static void
@@ -1387,7 +2407,7 @@ build_tpm_tcpa(GArray *table_data, GArray *linker, GArray *tcpalog)
                                    sizeof(tcpa->log_area_start_address));
 
     build_header(linker, table_data,
-                 (void *)tcpa, "TCPA", sizeof(*tcpa), 2);
+                 (void *)tcpa, "TCPA", sizeof(*tcpa), 2, NULL, NULL);
 
     acpi_data_push(tcpalog, TPM_LOG_AREA_MINIMUM_SIZE);
 }
@@ -1404,7 +2424,7 @@ build_tpm2(GArray *table_data, GArray *linker)
     tpm2_ptr->start_method = cpu_to_le32(TPM2_START_METHOD_MMIO);
 
     build_header(linker, table_data,
-                 (void *)tpm2_ptr, "TPM2", sizeof(*tpm2_ptr), 4);
+                 (void *)tpm2_ptr, "TPM2", sizeof(*tpm2_ptr), 4, NULL, NULL);
 }
 
 typedef enum {
@@ -1428,7 +2448,7 @@ acpi_build_srat_memory(AcpiSratMemoryAffinity *numamem, uint64_t base,
 }
 
 static void
-build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
+build_srat(GArray *table_data, GArray *linker, MachineState *machine)
 {
     AcpiSystemResourceAffinityTable *srat;
     AcpiSratProcessorAffinity *core;
@@ -1438,7 +2458,9 @@ build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
     uint64_t curnode;
     int srat_start, numa_start, slots;
     uint64_t mem_len, mem_base, next_base;
-    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
     ram_addr_t hotplugabble_address_space_size =
         object_property_get_int(OBJECT(pcms), PC_MACHINE_MEMHP_REGION_SIZE,
                                 NULL);
@@ -1447,14 +2469,15 @@ build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
 
     srat = acpi_data_push(table_data, sizeof *srat);
     srat->reserved1 = cpu_to_le32(1);
-    core = (void *)(srat + 1);
 
-    for (i = 0; i < guest_info->apic_id_limit; ++i) {
+    for (i = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
         core = acpi_data_push(table_data, sizeof *core);
         core->type = ACPI_SRAT_PROCESSOR;
         core->length = sizeof(*core);
-        core->local_apic_id = i;
-        curnode = guest_info->node_cpu[i];
+        core->local_apic_id = apic_id;
+        curnode = pcms->node_cpu[apic_id];
         core->proximity_lo = curnode;
         memset(core->proximity_hi, 0, 3);
         core->local_sapic_eid = 0;
@@ -1471,33 +2494,33 @@ build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
     numamem = acpi_data_push(table_data, sizeof *numamem);
     acpi_build_srat_memory(numamem, 0, 640*1024, 0, MEM_AFFINITY_ENABLED);
     next_base = 1024 * 1024;
-    for (i = 1; i < guest_info->numa_nodes + 1; ++i) {
+    for (i = 1; i < pcms->numa_nodes + 1; ++i) {
         mem_base = next_base;
-        mem_len = guest_info->node_mem[i - 1];
+        mem_len = pcms->node_mem[i - 1];
         if (i == 1) {
             mem_len -= 1024 * 1024;
         }
         next_base = mem_base + mem_len;
 
         /* Cut out the ACPI_PCI hole */
-        if (mem_base <= guest_info->ram_size_below_4g &&
-            next_base > guest_info->ram_size_below_4g) {
-            mem_len -= next_base - guest_info->ram_size_below_4g;
+        if (mem_base <= pcms->below_4g_mem_size &&
+            next_base > pcms->below_4g_mem_size) {
+            mem_len -= next_base - pcms->below_4g_mem_size;
             if (mem_len > 0) {
                 numamem = acpi_data_push(table_data, sizeof *numamem);
                 acpi_build_srat_memory(numamem, mem_base, mem_len, i - 1,
                                        MEM_AFFINITY_ENABLED);
             }
             mem_base = 1ULL << 32;
-            mem_len = next_base - guest_info->ram_size_below_4g;
-            next_base += (1ULL << 32) - guest_info->ram_size_below_4g;
+            mem_len = next_base - pcms->below_4g_mem_size;
+            next_base += (1ULL << 32) - pcms->below_4g_mem_size;
         }
         numamem = acpi_data_push(table_data, sizeof *numamem);
         acpi_build_srat_memory(numamem, mem_base, mem_len, i - 1,
                                MEM_AFFINITY_ENABLED);
     }
     slots = (table_data->len - numa_start) / sizeof *numamem;
-    for (; slots < guest_info->numa_nodes + 2; slots++) {
+    for (; slots < pcms->numa_nodes + 2; slots++) {
         numamem = acpi_data_push(table_data, sizeof *numamem);
         acpi_build_srat_memory(numamem, 0, 0, 0, MEM_AFFINITY_NOFLAGS);
     }
@@ -1518,7 +2541,8 @@ build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
     build_header(linker, table_data,
                  (void *)(table_data->data + srat_start),
                  "SRAT",
-                 table_data->len - srat_start, 1);
+                 table_data->len - srat_start, 1, NULL, NULL);
+    g_free(apic_ids);
 }
 
 static void
@@ -1547,7 +2571,7 @@ build_mcfg_q35(GArray *table_data, GArray *linker, AcpiMcfgInfo *info)
     } else {
         sig = "MCFG";
     }
-    build_header(linker, table_data, (void *)mcfg, sig, len, 1);
+    build_header(linker, table_data, (void *)mcfg, sig, len, 1, NULL, NULL);
 }
 
 static void
@@ -1571,22 +2595,7 @@ build_dmar_q35(GArray *table_data, GArray *linker)
     drhd->address = cpu_to_le64(Q35_HOST_BRIDGE_IOMMU_ADDR);
 
     build_header(linker, table_data, (void *)(table_data->data + dmar_start),
-                 "DMAR", table_data->len - dmar_start, 1);
-}
-
-static void
-build_dsdt(GArray *table_data, GArray *linker, AcpiMiscInfo *misc)
-{
-    AcpiTableHeader *dsdt;
-
-    assert(misc->dsdt_code && misc->dsdt_size);
-
-    dsdt = acpi_data_push(table_data, misc->dsdt_size);
-    memcpy(dsdt, misc->dsdt_code, misc->dsdt_size);
-
-    memset(dsdt, 0, sizeof *dsdt);
-    build_header(linker, table_data, dsdt, "DSDT",
-                 misc->dsdt_size, 1);
+                 "DMAR", table_data->len - dmar_start, 1, NULL, NULL);
 }
 
 static GArray *
@@ -1608,7 +2617,8 @@ build_rsdp(GArray *rsdp_table, GArray *linker, unsigned rsdt)
     rsdp->checksum = 0;
     /* Checksum to be filled by Guest linker */
     bios_linker_loader_add_checksum(linker, ACPI_BUILD_RSDP_FILE,
-                                    rsdp, rsdp, sizeof *rsdp, &rsdp->checksum);
+                                    rsdp_table, rsdp, sizeof *rsdp,
+                                    &rsdp->checksum);
 
     return rsdp_table;
 }
@@ -1619,7 +2629,6 @@ struct AcpiBuildState {
     MemoryRegion *table_mr;
     /* Is table patched? */
     uint8_t patched;
-    PcGuestInfo *guest_info;
     void *rsdp;
     MemoryRegion *rsdp_mr;
     MemoryRegion *linker_mr;
@@ -1658,11 +2667,12 @@ static bool acpi_has_iommu(void)
 }
 
 static
-void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
+void acpi_build(AcpiBuildTables *tables, MachineState *machine)
 {
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     GArray *table_offsets;
-    unsigned facs, ssdt, dsdt, rsdt;
-    AcpiCpuInfo cpu;
+    unsigned facs, dsdt, rsdt, fadt;
     AcpiPmInfo pm;
     AcpiMiscInfo misc;
     AcpiMcfgInfo mcfg;
@@ -1670,12 +2680,12 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
     uint8_t *u;
     size_t aml_len = 0;
     GArray *tables_blob = tables->table_data;
+    AcpiSlicOem slic_oem = { .id = NULL, .table_id = NULL };
 
-    acpi_get_cpu_info(&cpu);
     acpi_get_pm_info(&pm);
-    acpi_get_dsdt(&misc);
     acpi_get_misc_info(&misc);
     acpi_get_pci_info(&pci);
+    acpi_get_slic_oem(&slic_oem);
 
     table_offsets = g_array_new(false, true /* clear */,
                                         sizeof(uint32_t));
@@ -1691,11 +2701,11 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
      * requirements.
      */
     facs = tables_blob->len;
-    build_facs(tables_blob, tables->linker, guest_info);
+    build_facs(tables_blob, tables->linker);
 
     /* DSDT is pointed to by FADT */
     dsdt = tables_blob->len;
-    build_dsdt(tables_blob, tables->linker, &misc);
+    build_dsdt(tables_blob, tables->linker, &pm, &misc, &pci, machine);
 
     /* Count the size of the DSDT and SSDT, we will need it for legacy
      * sizing of ACPI tables.
@@ -1703,17 +2713,14 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
     aml_len += tables_blob->len - dsdt;
 
     /* ACPI tables pointed to by RSDT */
+    fadt = tables_blob->len;
     acpi_add_table(table_offsets, tables_blob);
-    build_fadt(tables_blob, tables->linker, &pm, facs, dsdt);
-
-    ssdt = tables_blob->len;
-    acpi_add_table(table_offsets, tables_blob);
-    build_ssdt(tables_blob, tables->linker, &cpu, &pm, &misc, &pci,
-               guest_info);
-    aml_len += tables_blob->len - ssdt;
+    build_fadt(tables_blob, tables->linker, &pm, facs, dsdt,
+               slic_oem.id, slic_oem.table_id);
+    aml_len += tables_blob->len - fadt;
 
     acpi_add_table(table_offsets, tables_blob);
-    build_madt(tables_blob, tables->linker, &cpu, guest_info);
+    build_madt(tables_blob, tables->linker, pcms);
 
     if (misc.has_hpet) {
         acpi_add_table(table_offsets, tables_blob);
@@ -1728,9 +2735,9 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
             build_tpm2(tables_blob, tables->linker);
         }
     }
-    if (guest_info->numa_nodes) {
+    if (pcms->numa_nodes) {
         acpi_add_table(table_offsets, tables_blob);
-        build_srat(tables_blob, tables->linker, guest_info);
+        build_srat(tables_blob, tables->linker, machine);
     }
     if (acpi_get_mcfg(&mcfg)) {
         acpi_add_table(table_offsets, tables_blob);
@@ -1740,6 +2747,9 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
         acpi_add_table(table_offsets, tables_blob);
         build_dmar_q35(tables_blob, tables->linker);
     }
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_build_acpi(table_offsets, tables_blob, tables->linker);
+    }
 
     /* Add tables supplied by user (if any) */
     for (u = acpi_table_first(); u; u = acpi_table_next(u)) {
@@ -1751,7 +2761,8 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
 
     /* RSDT is pointed to by RSDP */
     rsdt = tables_blob->len;
-    build_rsdt(tables_blob, tables->linker, table_offsets);
+    build_rsdt(tables_blob, tables->linker, table_offsets,
+               slic_oem.id, slic_oem.table_id);
 
     /* RSDP is in FSEG memory, so allocate it separately */
     build_rsdp(tables->rsdp, tables->linker, rsdt);
@@ -1773,12 +2784,12 @@ void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
      *
      * All this is for PIIX4, since QEMU 2.0 didn't support Q35 migration.
      */
-    if (guest_info->legacy_acpi_table_size) {
+    if (pcmc->legacy_acpi_table_size) {
         /* Subtracting aml_len gives the size of fixed tables.  Then add the
          * size of the PIIX4 DSDT/SSDT in QEMU 2.0.
          */
         int legacy_aml_len =
-            guest_info->legacy_acpi_table_size +
+            pcmc->legacy_acpi_table_size +
             ACPI_BUILD_LEGACY_CPU_AML_SIZE * max_cpus;
         int legacy_table_size =
             ROUND_UP(tables_blob->len - aml_len + legacy_aml_len,
@@ -1817,7 +2828,7 @@ static void acpi_ram_update(MemoryRegion *mr, GArray *data)
     memory_region_set_dirty(mr, 0, size);
 }
 
-static void acpi_build_update(void *build_opaque, uint32_t offset)
+static void acpi_build_update(void *build_opaque)
 {
     AcpiBuildState *build_state = build_opaque;
     AcpiBuildTables tables;
@@ -1830,7 +2841,7 @@ static void acpi_build_update(void *build_opaque, uint32_t offset)
 
     acpi_build_tables_init(&tables);
 
-    acpi_build(build_state->guest_info, &tables);
+    acpi_build(&tables, MACHINE(qdev_get_machine()));
 
     acpi_ram_update(build_state->table_mr, tables.table_data);
 
@@ -1868,17 +2879,19 @@ static const VMStateDescription vmstate_acpi_build = {
     },
 };
 
-void acpi_setup(PcGuestInfo *guest_info)
+void acpi_setup(void)
 {
+    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     AcpiBuildTables tables;
     AcpiBuildState *build_state;
 
-    if (!guest_info->fw_cfg) {
+    if (!pcms->fw_cfg) {
         ACPI_BUILD_DPRINTF("No fw cfg. Bailing out.\n");
         return;
     }
 
-    if (!guest_info->has_acpi_build) {
+    if (!pcmc->has_acpi_build) {
         ACPI_BUILD_DPRINTF("ACPI build disabled. Bailing out.\n");
         return;
     }
@@ -1890,12 +2903,10 @@ void acpi_setup(PcGuestInfo *guest_info)
 
     build_state = g_malloc0(sizeof *build_state);
 
-    build_state->guest_info = guest_info;
-
     acpi_set_pci_info();
 
     acpi_build_tables_init(&tables);
-    acpi_build(build_state->guest_info, &tables);
+    acpi_build(&tables, MACHINE(pcms));
 
     /* Now expose it all to Guest */
     build_state->table_mr = acpi_add_rom_blob(build_state, tables.table_data,
@@ -1906,10 +2917,10 @@ void acpi_setup(PcGuestInfo *guest_info)
     build_state->linker_mr =
         acpi_add_rom_blob(build_state, tables.linker, "etc/table-loader", 0);
 
-    fw_cfg_add_file(guest_info->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
+    fw_cfg_add_file(pcms->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
                     tables.tcpalog->data, acpi_data_len(tables.tcpalog));
 
-    if (!guest_info->rsdp_in_ram) {
+    if (!pcmc->rsdp_in_ram) {
         /*
          * Keep for compatibility with old machine types.
          * Though RSDP is small, its contents isn't immutable, so
@@ -1918,7 +2929,7 @@ void acpi_setup(PcGuestInfo *guest_info)
         uint32_t rsdp_size = acpi_data_len(tables.rsdp);
 
         build_state->rsdp = g_memdup(tables.rsdp->data, rsdp_size);
-        fw_cfg_add_file_callback(guest_info->fw_cfg, ACPI_BUILD_RSDP_FILE,
+        fw_cfg_add_file_callback(pcms->fw_cfg, ACPI_BUILD_RSDP_FILE,
                                  acpi_build_update, build_state,
                                  build_state->rsdp, rsdp_size);
         build_state->rsdp_mr = NULL;
diff --git a/qemu/hw/i386/acpi-build.h b/qemu/hw/i386/acpi-build.h
index e57b1aafd..007332e51 100644
--- a/qemu/hw/i386/acpi-build.h
+++ b/qemu/hw/i386/acpi-build.h
@@ -2,8 +2,6 @@
 #ifndef HW_I386_ACPI_BUILD_H
 #define HW_I386_ACPI_BUILD_H
 
-#include "qemu/typedefs.h"
-
-void acpi_setup(PcGuestInfo *);
+void acpi_setup(void);
 
 #endif
diff --git a/qemu/hw/i386/acpi-dsdt-cpu-hotplug.dsl b/qemu/hw/i386/acpi-dsdt-cpu-hotplug.dsl
deleted file mode 100644
index 1aff74627..000000000
--- a/qemu/hw/i386/acpi-dsdt-cpu-hotplug.dsl
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * CPU hotplug
- ****************************************************************/
-
-Scope(\_SB) {
-    /* Objects filled in by run-time generated SSDT */
-    External(NTFY, MethodObj)
-    External(CPON, PkgObj)
-    External(PRS, FieldUnitObj)
-
-    /* Methods called by run-time generated SSDT Processor objects */
-    Method(CPMA, 1, NotSerialized) {
-        // _MAT method - create an madt apic buffer
-        // Arg0 = Processor ID = Local APIC ID
-        // Local0 = CPON flag for this cpu
-        Store(DerefOf(Index(CPON, Arg0)), Local0)
-        // Local1 = Buffer (in madt apic form) to return
-        Store(Buffer(8) {0x00, 0x08, 0x00, 0x00, 0x00, 0, 0, 0}, Local1)
-        // Update the processor id, lapic id, and enable/disable status
-        Store(Arg0, Index(Local1, 2))
-        Store(Arg0, Index(Local1, 3))
-        Store(Local0, Index(Local1, 4))
-        Return (Local1)
-    }
-    Method(CPST, 1, NotSerialized) {
-        // _STA method - return ON status of cpu
-        // Arg0 = Processor ID = Local APIC ID
-        // Local0 = CPON flag for this cpu
-        Store(DerefOf(Index(CPON, Arg0)), Local0)
-        If (Local0) {
-            Return (0xF)
-        } Else {
-            Return (0x0)
-        }
-    }
-    Method(CPEJ, 2, NotSerialized) {
-        // _EJ0 method - eject callback
-        Sleep(200)
-    }
-
-#define CPU_STATUS_LEN ACPI_GPE_PROC_LEN
-    Method(PRSC, 0) {
-        // Local5 = active cpu bitmap
-        Store(PRS, Local5)
-        // Local2 = last read byte from bitmap
-        Store(Zero, Local2)
-        // Local0 = Processor ID / APIC ID iterator
-        Store(Zero, Local0)
-        While (LLess(Local0, SizeOf(CPON))) {
-            // Local1 = CPON flag for this cpu
-            Store(DerefOf(Index(CPON, Local0)), Local1)
-            If (And(Local0, 0x07)) {
-                // Shift down previously read bitmap byte
-                ShiftRight(Local2, 1, Local2)
-            } Else {
-                // Read next byte from cpu bitmap
-                Store(DerefOf(Index(Local5, ShiftRight(Local0, 3))), Local2)
-            }
-            // Local3 = active state for this cpu
-            Store(And(Local2, 1), Local3)
-
-            If (LNotEqual(Local1, Local3)) {
-                // State change - update CPON with new state
-                Store(Local3, Index(CPON, Local0))
-                // Do CPU notify
-                If (LEqual(Local3, 1)) {
-                    NTFY(Local0, 1)
-                } Else {
-                    NTFY(Local0, 3)
-                }
-            }
-            Increment(Local0)
-        }
-    }
-}
diff --git a/qemu/hw/i386/acpi-dsdt-dbug.dsl b/qemu/hw/i386/acpi-dsdt-dbug.dsl
deleted file mode 100644
index 86230f75a..000000000
--- a/qemu/hw/i386/acpi-dsdt-dbug.dsl
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * Debugging
- ****************************************************************/
-
-Scope(\) {
-    /* Debug Output */
-    OperationRegion(DBG, SystemIO, 0x0402, 0x01)
-    Field(DBG, ByteAcc, NoLock, Preserve) {
-        DBGB,   8,
-    }
-
-    /* Debug method - use this method to send output to the QEMU
-     * BIOS debug port.  This method handles strings, integers,
-     * and buffers.  For example: DBUG("abc") DBUG(0x123) */
-    Method(DBUG, 1) {
-        ToHexString(Arg0, Local0)
-        ToBuffer(Local0, Local0)
-        Subtract(SizeOf(Local0), 1, Local1)
-        Store(Zero, Local2)
-        While (LLess(Local2, Local1)) {
-            Store(DerefOf(Index(Local0, Local2)), DBGB)
-            Increment(Local2)
-        }
-        Store(0x0A, DBGB)
-    }
-}
diff --git a/qemu/hw/i386/acpi-dsdt-hpet.dsl b/qemu/hw/i386/acpi-dsdt-hpet.dsl
deleted file mode 100644
index 44961b87a..000000000
--- a/qemu/hw/i386/acpi-dsdt-hpet.dsl
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * HPET
- ****************************************************************/
-
-Scope(\_SB) {
-    Device(HPET) {
-        Name(_HID, EISAID("PNP0103"))
-        Name(_UID, 0)
-        OperationRegion(HPTM, SystemMemory, 0xFED00000, 0x400)
-        Field(HPTM, DWordAcc, Lock, Preserve) {
-            VEND, 32,
-            PRD, 32,
-        }
-        Method(_STA, 0, NotSerialized) {
-            Store(VEND, Local0)
-            Store(PRD, Local1)
-            ShiftRight(Local0, 16, Local0)
-            If (LOr(LEqual(Local0, 0), LEqual(Local0, 0xffff))) {
-                Return (0x0)
-            }
-            If (LOr(LEqual(Local1, 0), LGreater(Local1, 100000000))) {
-                Return (0x0)
-            }
-            Return (0x0F)
-        }
-        Name(_CRS, ResourceTemplate() {
-            Memory32Fixed(ReadOnly,
-                0xFED00000,         // Address Base
-                0x00000400,         // Address Length
-                )
-        })
-    }
-}
diff --git a/qemu/hw/i386/acpi-dsdt-isa.dsl b/qemu/hw/i386/acpi-dsdt-isa.dsl
deleted file mode 100644
index 89caa1649..000000000
--- a/qemu/hw/i386/acpi-dsdt-isa.dsl
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* Common legacy ISA style devices. */
-Scope(\_SB.PCI0.ISA) {
-
-    Device(RTC) {
-        Name(_HID, EisaId("PNP0B00"))
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0070, 0x0070, 0x10, 0x02)
-            IRQNoFlags() { 8 }
-            IO(Decode16, 0x0072, 0x0072, 0x02, 0x06)
-        })
-    }
-
-    Device(KBD) {
-        Name(_HID, EisaId("PNP0303"))
-        Method(_STA, 0, NotSerialized) {
-            Return (0x0f)
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0060, 0x0060, 0x01, 0x01)
-            IO(Decode16, 0x0064, 0x0064, 0x01, 0x01)
-            IRQNoFlags() { 1 }
-        })
-    }
-
-    Device(MOU) {
-        Name(_HID, EisaId("PNP0F13"))
-        Method(_STA, 0, NotSerialized) {
-            Return (0x0f)
-        }
-        Name(_CRS, ResourceTemplate() {
-            IRQNoFlags() { 12 }
-        })
-    }
-
-    Device(FDC0) {
-        Name(_HID, EisaId("PNP0700"))
-        Method(_STA, 0, NotSerialized) {
-            Store(FDEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x03F2, 0x03F2, 0x00, 0x04)
-            IO(Decode16, 0x03F7, 0x03F7, 0x00, 0x01)
-            IRQNoFlags() { 6 }
-            DMA(Compatibility, NotBusMaster, Transfer8) { 2 }
-        })
-    }
-
-    Device(LPT) {
-        Name(_HID, EisaId("PNP0400"))
-        Method(_STA, 0, NotSerialized) {
-            Store(LPEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0378, 0x0378, 0x08, 0x08)
-            IRQNoFlags() { 7 }
-        })
-    }
-
-    Device(COM1) {
-        Name(_HID, EisaId("PNP0501"))
-        Name(_UID, 0x01)
-        Method(_STA, 0, NotSerialized) {
-            Store(CAEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x03F8, 0x03F8, 0x00, 0x08)
-            IRQNoFlags() { 4 }
-        })
-    }
-
-    Device(COM2) {
-        Name(_HID, EisaId("PNP0501"))
-        Name(_UID, 0x02)
-        Method(_STA, 0, NotSerialized) {
-            Store(CBEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x02F8, 0x02F8, 0x00, 0x08)
-            IRQNoFlags() { 3 }
-        })
-    }
-}
diff --git a/qemu/hw/i386/acpi-dsdt-mem-hotplug.dsl b/qemu/hw/i386/acpi-dsdt-mem-hotplug.dsl
deleted file mode 100644
index c2bb6a160..000000000
--- a/qemu/hw/i386/acpi-dsdt-mem-hotplug.dsl
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-    External(MEMORY_SLOT_NOTIFY_METHOD, MethodObj)
-
-    Scope(\_SB.PCI0) {
-        Device(MEMORY_HOTPLUG_DEVICE) {
-            Name(_HID, "PNP0A06")
-            Name(_UID, "Memory hotplug resources")
-            External(MEMORY_SLOTS_NUMBER, IntObj)
-
-            /* Memory hotplug IO registers */
-            External(MEMORY_SLOT_ADDR_LOW, FieldUnitObj) // read only
-            External(MEMORY_SLOT_ADDR_HIGH, FieldUnitObj) // read only
-            External(MEMORY_SLOT_SIZE_LOW, FieldUnitObj) // read only
-            External(MEMORY_SLOT_SIZE_HIGH, FieldUnitObj) // read only
-            External(MEMORY_SLOT_PROXIMITY, FieldUnitObj) // read only
-            External(MEMORY_SLOT_ENABLED, FieldUnitObj) // 1 if enabled, read only
-            External(MEMORY_SLOT_INSERT_EVENT, FieldUnitObj) // (read) 1 if has a insert event. (write) 1 to clear event
-            External(MEMORY_SLOT_REMOVE_EVENT, FieldUnitObj) // (read) 1 if has a remove event. (write) 1 to clear event
-            External(MEMORY_SLOT_EJECT, FieldUnitObj) // initiates device eject, write only
-            External(MEMORY_SLOT_SLECTOR, FieldUnitObj) // DIMM selector, write only
-            External(MEMORY_SLOT_OST_EVENT, FieldUnitObj) // _OST event code, write only
-            External(MEMORY_SLOT_OST_STATUS, FieldUnitObj) // _OST status code, write only
-
-            Method(_STA, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                    Return(0x0)
-                }
-                /* present, functioning, decoding, not shown in UI */
-                Return(0xB)
-            }
-
-            Mutex (MEMORY_SLOT_LOCK, 0)
-
-            Method(MEMORY_SLOT_SCAN_METHOD, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                     Return(Zero)
-                }
-
-                Store(Zero, Local0) // Mem devs iterrator
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                while (LLess(Local0, MEMORY_SLOTS_NUMBER)) {
-                    Store(Local0, MEMORY_SLOT_SLECTOR) // select Local0 DIMM
-                    If (LEqual(MEMORY_SLOT_INSERT_EVENT, One)) { // Memory device needs check
-                        MEMORY_SLOT_NOTIFY_METHOD(Local0, 1)
-                        Store(1, MEMORY_SLOT_INSERT_EVENT)
-                    } Elseif (LEqual(MEMORY_SLOT_REMOVE_EVENT, One)) { // Ejection request
-                        MEMORY_SLOT_NOTIFY_METHOD(Local0, 3)
-                        Store(1, MEMORY_SLOT_REMOVE_EVENT)
-                    }
-                    Add(Local0, One, Local0) // goto next DIMM
-                }
-                Release(MEMORY_SLOT_LOCK)
-                Return(One)
-            }
-
-            Method(MEMORY_SLOT_STATUS_METHOD, 1) {
-                Store(Zero, Local0)
-
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                If (LEqual(MEMORY_SLOT_ENABLED, One)) {
-                    Store(0xF, Local0)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_CRS_METHOD, 1, Serialized) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                Name(MR64, ResourceTemplate() {
-                    QWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                    Cacheable, ReadWrite,
-                    0x0000000000000000,        // Address Space Granularity
-                    0x0000000000000000,        // Address Range Minimum
-                    0xFFFFFFFFFFFFFFFE,        // Address Range Maximum
-                    0x0000000000000000,        // Address Translation Offset
-                    0xFFFFFFFFFFFFFFFF,        // Address Length
-                    ,, MW64, AddressRangeMemory, TypeStatic)
-                })
-
-                CreateDWordField(MR64, 14, MINL)
-                CreateDWordField(MR64, 18, MINH)
-                CreateDWordField(MR64, 38, LENL)
-                CreateDWordField(MR64, 42, LENH)
-                CreateDWordField(MR64, 22, MAXL)
-                CreateDWordField(MR64, 26, MAXH)
-
-                Store(MEMORY_SLOT_ADDR_HIGH, MINH)
-                Store(MEMORY_SLOT_ADDR_LOW, MINL)
-                Store(MEMORY_SLOT_SIZE_HIGH, LENH)
-                Store(MEMORY_SLOT_SIZE_LOW, LENL)
-
-                // 64-bit math: MAX = MIN + LEN - 1
-                Add(MINL, LENL, MAXL)
-                Add(MINH, LENH, MAXH)
-                If (LLess(MAXL, MINL)) {
-                    Add(MAXH, One, MAXH)
-                }
-                If (LLess(MAXL, One)) {
-                    Subtract(MAXH, One, MAXH)
-                }
-                Subtract(MAXL, One, MAXL)
-
-                If (LEqual(MAXH, Zero)){
-                    Name(MR32, ResourceTemplate() {
-                        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                        Cacheable, ReadWrite,
-                        0x00000000,        // Address Space Granularity
-                        0x00000000,        // Address Range Minimum
-                        0xFFFFFFFE,        // Address Range Maximum
-                        0x00000000,        // Address Translation Offset
-                        0xFFFFFFFF,        // Address Length
-                        ,, MW32, AddressRangeMemory, TypeStatic)
-                    })
-                    CreateDWordField(MR32, MW32._MIN, MIN)
-                    CreateDWordField(MR32, MW32._MAX, MAX)
-                    CreateDWordField(MR32, MW32._LEN, LEN)
-                    Store(MINL, MIN)
-                    Store(MAXL, MAX)
-                    Store(LENL, LEN)
-
-                    Release(MEMORY_SLOT_LOCK)
-                    Return(MR32)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(MR64)
-            }
-
-            Method(MEMORY_SLOT_PROXIMITY_METHOD, 1) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(MEMORY_SLOT_PROXIMITY, Local0)
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_OST_METHOD, 4) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(Arg1, MEMORY_SLOT_OST_EVENT)
-                Store(Arg2, MEMORY_SLOT_OST_STATUS)
-                Release(MEMORY_SLOT_LOCK)
-            }
-
-            Method(MEMORY_SLOT_EJECT_METHOD, 2) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(1, MEMORY_SLOT_EJECT)
-                Release(MEMORY_SLOT_LOCK)
-            }
-        } // Device()
-    } // Scope()
diff --git a/qemu/hw/i386/acpi-dsdt.dsl b/qemu/hw/i386/acpi-dsdt.dsl
deleted file mode 100644
index a2d84ecf8..000000000
--- a/qemu/hw/i386/acpi-dsdt.dsl
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Bochs/QEMU ACPI DSDT ASL definition
- *
- * Copyright (c) 2006 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2 as published by the Free Software Foundation.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-ACPI_EXTRACT_ALL_CODE AcpiDsdtAmlCode
-
-DefinitionBlock (
-    "acpi-dsdt.aml",    // Output Filename
-    "DSDT",             // Signature
-    0x01,               // DSDT Compliance Revision
-    "BXPC",             // OEMID
-    "BXDSDT",           // TABLE ID
-    0x1                 // OEM Revision
-    )
-{
-
-#include "acpi-dsdt-dbug.dsl"
-
-    Scope(\_SB) {
-        Device(PCI0) {
-            Name(_HID, EisaId("PNP0A03"))
-            Name(_ADR, 0x00)
-            Name(_UID, 1)
-//#define PX13 S0B_
-//            External(PX13, DeviceObj)
-        }
-    }
-
-#include "acpi-dsdt-hpet.dsl"
-
-
-/****************************************************************
- * PIIX4 PM
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        Device(PX13) {
-            Name(_ADR, 0x00010003)
-            OperationRegion(P13C, PCI_Config, 0x00, 0xff)
-        }
-    }
-
-
-/****************************************************************
- * PIIX3 ISA bridge
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-
-        External(ISA, DeviceObj)
-
-        Device(ISA) {
-            Name(_ADR, 0x00010000)
-
-            /* PIIX PCI to ISA irq remapping */
-            OperationRegion(P40C, PCI_Config, 0x60, 0x04)
-
-            /* enable bits */
-            Field(\_SB.PCI0.PX13.P13C, AnyAcc, NoLock, Preserve) {
-                Offset(0x5f),
-                , 7,
-                LPEN, 1,         // LPT
-                Offset(0x67),
-                , 3,
-                CAEN, 1,         // COM1
-                , 3,
-                CBEN, 1,         // COM2
-            }
-            Name(FDEN, 1)
-        }
-    }
-
-#include "acpi-dsdt-isa.dsl"
-
-
-/****************************************************************
- * PCI hotplug
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        OperationRegion(PCST, SystemIO, 0xae00, 0x08)
-        Field(PCST, DWordAcc, NoLock, WriteAsZeros) {
-            PCIU, 32,
-            PCID, 32,
-        }
-
-        OperationRegion(SEJ, SystemIO, 0xae08, 0x04)
-        Field(SEJ, DWordAcc, NoLock, WriteAsZeros) {
-            B0EJ, 32,
-        }
-
-        OperationRegion(BNMR, SystemIO, 0xae10, 0x04)
-        Field(BNMR, DWordAcc, NoLock, WriteAsZeros) {
-            BNUM, 32,
-        }
-
-        /* Lock to protect access to fields above. */
-        Mutex(BLCK, 0)
-
-        /* Methods called by bulk generated PCI devices below */
-
-        /* Methods called by hotplug devices */
-        Method(PCEJ, 2, NotSerialized) {
-            // _EJ0 method - eject callback
-            Acquire(BLCK, 0xFFFF)
-            Store(Arg0, BNUM)
-            Store(ShiftLeft(1, Arg1), B0EJ)
-            Release(BLCK)
-            Return (0x0)
-        }
-
-        /* Hotplug notification method supplied by SSDT */
-        External(\_SB.PCI0.PCNT, MethodObj)
-    }
-
-
-/****************************************************************
- * PCI IRQs
- ****************************************************************/
-
-    Scope(\_SB) {
-        Scope(PCI0) {
-            Method (_PRT, 0) {
-                Store(Package(128) {}, Local0)
-                Store(Zero, Local1)
-                While(LLess(Local1, 128)) {
-                    // slot = pin >> 2
-                    Store(ShiftRight(Local1, 2), Local2)
-
-                    // lnk = (slot + pin) & 3
-                    Store(And(Add(Local1, Local2), 3), Local3)
-                    If (LEqual(Local3, 0)) {
-                        Store(Package(4) { Zero, Zero, LNKD, Zero }, Local4)
-                    }
-                    If (LEqual(Local3, 1)) {
-                        // device 1 is the power-management device, needs SCI
-                        If (LEqual(Local1, 4)) {
-                            Store(Package(4) { Zero, Zero, LNKS, Zero }, Local4)
-                        } Else {
-                            Store(Package(4) { Zero, Zero, LNKA, Zero }, Local4)
-                        }
-                    }
-                    If (LEqual(Local3, 2)) {
-                        Store(Package(4) { Zero, Zero, LNKB, Zero }, Local4)
-                    }
-                    If (LEqual(Local3, 3)) {
-                        Store(Package(4) { Zero, Zero, LNKC, Zero }, Local4)
-                    }
-
-                    // Complete the interrupt routing entry:
-                    //    Package(4) { 0x[slot]FFFF, [pin], [link], 0) }
-
-                    Store(Or(ShiftLeft(Local2, 16), 0xFFFF), Index(Local4, 0))
-                    Store(And(Local1, 3),                    Index(Local4, 1))
-                    Store(Local4,                            Index(Local0, Local1))
-
-                    Increment(Local1)
-                }
-
-                Return(Local0)
-            }
-        }
-
-        Field(PCI0.ISA.P40C, ByteAcc, NoLock, Preserve) {
-            PRQ0,   8,
-            PRQ1,   8,
-            PRQ2,   8,
-            PRQ3,   8
-        }
-
-        Method(IQST, 1, NotSerialized) {
-            // _STA method - get status
-            If (And(0x80, Arg0)) {
-                Return (0x09)
-            }
-            Return (0x0B)
-        }
-        Method(IQCR, 1, Serialized) {
-            // _CRS method - get current settings
-            Name(PRR0, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 0 }
-            })
-            CreateDWordField(PRR0, 0x05, PRRI)
-            If (LLess(Arg0, 0x80)) {
-                Store(Arg0, PRRI)
-            }
-            Return (PRR0)
-        }
-
-#define define_link(link, uid, reg)                             \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    5, 10, 11                                   \
-                }                                               \
-            })                                                  \
-            Method(_STA, 0, NotSerialized) {                    \
-                Return (IQST(reg))                              \
-            }                                                   \
-            Method(_DIS, 0, NotSerialized) {                    \
-                Or(reg, 0x80, reg)                              \
-            }                                                   \
-            Method(_CRS, 0, NotSerialized) {                    \
-                Return (IQCR(reg))                              \
-            }                                                   \
-            Method(_SRS, 1, NotSerialized) {                    \
-                CreateDWordField(Arg0, 0x05, PRRI)              \
-                Store(PRRI, reg)                                \
-            }                                                   \
-        }
-
-        define_link(LNKA, 0, PRQ0)
-        define_link(LNKB, 1, PRQ1)
-        define_link(LNKC, 2, PRQ2)
-        define_link(LNKD, 3, PRQ3)
-
-        Device(LNKS) {
-            Name(_HID, EISAID("PNP0C0F"))
-            Name(_UID, 4)
-            Name(_PRS, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 9 }
-            })
-
-            // The SCI cannot be disabled and is always attached to GSI 9,
-            // so these are no-ops.  We only need this link to override the
-            // polarity to active high and match the content of the MADT.
-            Method(_STA, 0, NotSerialized) { Return (0x0b) }
-            Method(_DIS, 0, NotSerialized) { }
-            Method(_CRS, 0, NotSerialized) { Return (_PRS) }
-            Method(_SRS, 1, NotSerialized) { }
-        }
-    }
-
-#include "hw/acpi/pc-hotplug.h"
-#define CPU_STATUS_BASE PIIX4_CPU_HOTPLUG_IO_BASE
-#include "acpi-dsdt-cpu-hotplug.dsl"
-#include "acpi-dsdt-mem-hotplug.dsl"
-
-
-/****************************************************************
- * General purpose events
- ****************************************************************/
-    Scope(\_GPE) {
-        Name(_HID, "ACPI0006")
-
-        Method(_L00) {
-        }
-        Method(_E01) {
-            // PCI hotplug event
-            Acquire(\_SB.PCI0.BLCK, 0xFFFF)
-            \_SB.PCI0.PCNT()
-            Release(\_SB.PCI0.BLCK)
-        }
-        Method(_E02) {
-            // CPU hotplug event
-            \_SB.PRSC()
-        }
-        Method(_E03) {
-            // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
-        }
-        Method(_L04) {
-        }
-        Method(_L05) {
-        }
-        Method(_L06) {
-        }
-        Method(_L07) {
-        }
-        Method(_L08) {
-        }
-        Method(_L09) {
-        }
-        Method(_L0A) {
-        }
-        Method(_L0B) {
-        }
-        Method(_L0C) {
-        }
-        Method(_L0D) {
-        }
-        Method(_L0E) {
-        }
-        Method(_L0F) {
-        }
-    }
-}
diff --git a/qemu/hw/i386/acpi-dsdt.hex.generated b/qemu/hw/i386/acpi-dsdt.hex.generated
deleted file mode 100644
index ecaa4a548..000000000
--- a/qemu/hw/i386/acpi-dsdt.hex.generated
+++ /dev/null
@@ -1,2972 +0,0 @@
-static unsigned char AcpiDsdtAmlCode[] = {
-0x44,
-0x53,
-0x44,
-0x54,
-0x9a,
-0xb,
-0x0,
-0x0,
-0x1,
-0xf8,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x44,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x7,
-0x11,
-0x14,
-0x20,
-0x10,
-0x49,
-0x4,
-0x5c,
-0x0,
-0x5b,
-0x80,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0xb,
-0x2,
-0x4,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0x44,
-0x42,
-0x47,
-0x42,
-0x8,
-0x14,
-0x2c,
-0x44,
-0x42,
-0x55,
-0x47,
-0x1,
-0x98,
-0x68,
-0x60,
-0x96,
-0x60,
-0x60,
-0x74,
-0x87,
-0x60,
-0x1,
-0x61,
-0x70,
-0x0,
-0x62,
-0xa2,
-0x10,
-0x95,
-0x62,
-0x61,
-0x70,
-0x83,
-0x88,
-0x60,
-0x62,
-0x0,
-0x44,
-0x42,
-0x47,
-0x42,
-0x75,
-0x62,
-0x70,
-0xa,
-0xa,
-0x44,
-0x42,
-0x47,
-0x42,
-0x10,
-0x22,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x1b,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x10,
-0x4d,
-0x8,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x45,
-0x8,
-0x48,
-0x50,
-0x45,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x1,
-0x3,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x0,
-0xc,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0xb,
-0x0,
-0x4,
-0x5b,
-0x81,
-0x10,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x13,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x20,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x20,
-0x14,
-0x36,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x60,
-0x70,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x61,
-0x7a,
-0x60,
-0xa,
-0x10,
-0x60,
-0xa0,
-0xc,
-0x91,
-0x93,
-0x60,
-0x0,
-0x93,
-0x60,
-0xb,
-0xff,
-0xff,
-0xa4,
-0x0,
-0xa0,
-0xe,
-0x91,
-0x93,
-0x61,
-0x0,
-0x94,
-0x61,
-0xc,
-0x0,
-0xe1,
-0xf5,
-0x5,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x11,
-0xa,
-0xe,
-0x86,
-0x9,
-0x0,
-0x0,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0x0,
-0x4,
-0x0,
-0x0,
-0x79,
-0x0,
-0x10,
-0x25,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x19,
-0x50,
-0x58,
-0x31,
-0x33,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x3,
-0x0,
-0x1,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x31,
-0x33,
-0x43,
-0x2,
-0x0,
-0xa,
-0xff,
-0x10,
-0x46,
-0x5,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x49,
-0x4,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0x1,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x34,
-0x30,
-0x43,
-0x2,
-0xa,
-0x60,
-0xa,
-0x4,
-0x5b,
-0x81,
-0x26,
-0x5e,
-0x2e,
-0x50,
-0x58,
-0x31,
-0x33,
-0x50,
-0x31,
-0x33,
-0x43,
-0x0,
-0x0,
-0x48,
-0x2f,
-0x0,
-0x7,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x1,
-0x0,
-0x38,
-0x0,
-0x3,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x1,
-0x0,
-0x3,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x1,
-0x8,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x1,
-0x10,
-0x4c,
-0x1b,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x5b,
-0x82,
-0x2d,
-0x52,
-0x54,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xb,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x70,
-0x0,
-0x70,
-0x0,
-0x10,
-0x2,
-0x22,
-0x0,
-0x1,
-0x47,
-0x1,
-0x72,
-0x0,
-0x72,
-0x0,
-0x2,
-0x6,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x37,
-0x4b,
-0x42,
-0x44,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x3,
-0x3,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x60,
-0x0,
-0x60,
-0x0,
-0x1,
-0x1,
-0x47,
-0x1,
-0x64,
-0x0,
-0x64,
-0x0,
-0x1,
-0x1,
-0x22,
-0x2,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x27,
-0x4d,
-0x4f,
-0x55,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xf,
-0x13,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x8,
-0xa,
-0x5,
-0x22,
-0x0,
-0x10,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4a,
-0x4,
-0x46,
-0x44,
-0x43,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x7,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1b,
-0xa,
-0x18,
-0x47,
-0x1,
-0xf2,
-0x3,
-0xf2,
-0x3,
-0x0,
-0x4,
-0x47,
-0x1,
-0xf7,
-0x3,
-0xf7,
-0x3,
-0x0,
-0x1,
-0x22,
-0x40,
-0x0,
-0x2a,
-0x4,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x3e,
-0x4c,
-0x50,
-0x54,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x4,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x78,
-0x3,
-0x78,
-0x3,
-0x8,
-0x8,
-0x22,
-0x80,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x31,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x3,
-0xf8,
-0x3,
-0x0,
-0x8,
-0x22,
-0x10,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x46,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x32,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x2,
-0xf8,
-0x2,
-0x0,
-0x8,
-0x22,
-0x8,
-0x0,
-0x79,
-0x0,
-0x10,
-0x48,
-0x8,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x54,
-0x1,
-0xb,
-0x0,
-0xae,
-0xa,
-0x8,
-0x5b,
-0x81,
-0x10,
-0x50,
-0x43,
-0x53,
-0x54,
-0x43,
-0x50,
-0x43,
-0x49,
-0x55,
-0x20,
-0x50,
-0x43,
-0x49,
-0x44,
-0x20,
-0x5b,
-0x80,
-0x53,
-0x45,
-0x4a,
-0x5f,
-0x1,
-0xb,
-0x8,
-0xae,
-0xa,
-0x4,
-0x5b,
-0x81,
-0xb,
-0x53,
-0x45,
-0x4a,
-0x5f,
-0x43,
-0x42,
-0x30,
-0x45,
-0x4a,
-0x20,
-0x5b,
-0x80,
-0x42,
-0x4e,
-0x4d,
-0x52,
-0x1,
-0xb,
-0x10,
-0xae,
-0xa,
-0x4,
-0x5b,
-0x81,
-0xb,
-0x42,
-0x4e,
-0x4d,
-0x52,
-0x43,
-0x42,
-0x4e,
-0x55,
-0x4d,
-0x20,
-0x5b,
-0x1,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x14,
-0x25,
-0x50,
-0x43,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x23,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x68,
-0x42,
-0x4e,
-0x55,
-0x4d,
-0x70,
-0x79,
-0x1,
-0x69,
-0x0,
-0x42,
-0x30,
-0x45,
-0x4a,
-0x5b,
-0x27,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x0,
-0x10,
-0x4e,
-0x36,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x10,
-0x4b,
-0xa,
-0x50,
-0x43,
-0x49,
-0x30,
-0x14,
-0x44,
-0xa,
-0x5f,
-0x50,
-0x52,
-0x54,
-0x0,
-0x70,
-0x12,
-0x2,
-0x80,
-0x60,
-0x70,
-0x0,
-0x61,
-0xa2,
-0x42,
-0x9,
-0x95,
-0x61,
-0xa,
-0x80,
-0x70,
-0x7a,
-0x61,
-0xa,
-0x2,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x72,
-0x61,
-0x62,
-0x0,
-0xa,
-0x3,
-0x0,
-0x63,
-0xa0,
-0x10,
-0x93,
-0x63,
-0x0,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x64,
-0xa0,
-0x24,
-0x93,
-0x63,
-0x1,
-0xa0,
-0x11,
-0x93,
-0x61,
-0xa,
-0x4,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x53,
-0x0,
-0x64,
-0xa1,
-0xd,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x64,
-0xa0,
-0x11,
-0x93,
-0x63,
-0xa,
-0x2,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x64,
-0xa0,
-0x11,
-0x93,
-0x63,
-0xa,
-0x3,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x64,
-0x70,
-0x7d,
-0x79,
-0x62,
-0xa,
-0x10,
-0x0,
-0xb,
-0xff,
-0xff,
-0x0,
-0x88,
-0x64,
-0x0,
-0x0,
-0x70,
-0x7b,
-0x61,
-0xa,
-0x3,
-0x0,
-0x88,
-0x64,
-0x1,
-0x0,
-0x70,
-0x64,
-0x88,
-0x60,
-0x61,
-0x0,
-0x75,
-0x61,
-0xa4,
-0x60,
-0x5b,
-0x81,
-0x24,
-0x2f,
-0x3,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x50,
-0x34,
-0x30,
-0x43,
-0x1,
-0x50,
-0x52,
-0x51,
-0x30,
-0x8,
-0x50,
-0x52,
-0x51,
-0x31,
-0x8,
-0x50,
-0x52,
-0x51,
-0x32,
-0x8,
-0x50,
-0x52,
-0x51,
-0x33,
-0x8,
-0x14,
-0x13,
-0x49,
-0x51,
-0x53,
-0x54,
-0x1,
-0xa0,
-0x9,
-0x7b,
-0xa,
-0x80,
-0x68,
-0x0,
-0xa4,
-0xa,
-0x9,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x36,
-0x49,
-0x51,
-0x43,
-0x52,
-0x9,
-0x8,
-0x50,
-0x52,
-0x52,
-0x30,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8a,
-0x50,
-0x52,
-0x52,
-0x30,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa0,
-0xb,
-0x95,
-0x68,
-0xa,
-0x80,
-0x70,
-0x68,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa4,
-0x50,
-0x52,
-0x52,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x30,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x31,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x31,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x32,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x32,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x33,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x33,
-0x5b,
-0x82,
-0x4f,
-0x4,
-0x4c,
-0x4e,
-0x4b,
-0x53,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x4,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x9,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x6,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x14,
-0xb,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x10,
-0x4d,
-0xc,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x14,
-0x35,
-0x43,
-0x50,
-0x4d,
-0x41,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0x70,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x61,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x2,
-0x0,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x3,
-0x0,
-0x70,
-0x60,
-0x88,
-0x61,
-0xa,
-0x4,
-0x0,
-0xa4,
-0x61,
-0x14,
-0x1a,
-0x43,
-0x50,
-0x53,
-0x54,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0xa0,
-0x5,
-0x60,
-0xa4,
-0xa,
-0xf,
-0xa1,
-0x3,
-0xa4,
-0x0,
-0x14,
-0xa,
-0x43,
-0x50,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x22,
-0xa,
-0xc8,
-0x14,
-0x4a,
-0x6,
-0x50,
-0x52,
-0x53,
-0x43,
-0x0,
-0x70,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x65,
-0x70,
-0x0,
-0x62,
-0x70,
-0x0,
-0x60,
-0xa2,
-0x46,
-0x5,
-0x95,
-0x60,
-0x87,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0x61,
-0xa0,
-0xa,
-0x7b,
-0x60,
-0xa,
-0x7,
-0x0,
-0x7a,
-0x62,
-0x1,
-0x62,
-0xa1,
-0xc,
-0x70,
-0x83,
-0x88,
-0x65,
-0x7a,
-0x60,
-0xa,
-0x3,
-0x0,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x62,
-0x1,
-0x0,
-0x63,
-0xa0,
-0x22,
-0x92,
-0x93,
-0x61,
-0x63,
-0x70,
-0x63,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0xa0,
-0xa,
-0x93,
-0x63,
-0x1,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0xa1,
-0x8,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0xa,
-0x3,
-0x75,
-0x60,
-0x10,
-0x44,
-0x2a,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x47,
-0x29,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x50,
-0x4e,
-0x50,
-0x30,
-0x41,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x4d,
-0x65,
-0x6d,
-0x6f,
-0x72,
-0x79,
-0x20,
-0x68,
-0x6f,
-0x74,
-0x70,
-0x6c,
-0x75,
-0x67,
-0x20,
-0x72,
-0x65,
-0x73,
-0x6f,
-0x75,
-0x72,
-0x63,
-0x65,
-0x73,
-0x0,
-0x14,
-0x13,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x5b,
-0x1,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x14,
-0x4a,
-0x4,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0xa2,
-0x25,
-0x95,
-0x60,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x70,
-0x60,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0x13,
-0x93,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x4d,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0x70,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x72,
-0x60,
-0x1,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x1,
-0x14,
-0x2d,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x1,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0xb,
-0x93,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x70,
-0xa,
-0xf,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x41,
-0x18,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x9,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x8,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x12,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x2a,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x1a,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x14,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x11,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0xa0,
-0x44,
-0x7,
-0x93,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x0,
-0x8,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xa,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xe,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0x16,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x70,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x14,
-0x24,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x1,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x28,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x4,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x69,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x70,
-0x6a,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x10,
-0x45,
-0xd,
-0x5f,
-0x47,
-0x50,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x41,
-0x43,
-0x50,
-0x49,
-0x30,
-0x30,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x30,
-0x0,
-0x14,
-0x39,
-0x5f,
-0x45,
-0x30,
-0x31,
-0x0,
-0x5b,
-0x23,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x50,
-0x43,
-0x4e,
-0x54,
-0x5b,
-0x27,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0x14,
-0x10,
-0x5f,
-0x45,
-0x30,
-0x32,
-0x0,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x43,
-0x14,
-0x19,
-0x5f,
-0x45,
-0x30,
-0x33,
-0x0,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x34,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x35,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x37,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x38,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x39,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x41,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x42,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x43,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x44,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x45,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x46,
-0x0
-};
diff --git a/qemu/hw/i386/intel_iommu.c b/qemu/hw/i386/intel_iommu.c
index 08055a8d8..347718f93 100644
--- a/qemu/hw/i386/intel_iommu.c
+++ b/qemu/hw/i386/intel_iommu.c
@@ -19,9 +19,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "intel_iommu_internal.h"
+#include "hw/pci/pci.h"
 
 /*#define DEBUG_INTEL_IOMMU*/
 #ifdef DEBUG_INTEL_IOMMU
@@ -151,14 +153,27 @@ static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
     return entry->domain_id == domain_id;
 }
 
+/* The shift of an addr for a certain level of paging structure */
+static inline uint32_t vtd_slpt_level_shift(uint32_t level)
+{
+    return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
+}
+
+static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
+{
+    return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
+}
+
 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
                                         gpointer user_data)
 {
     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
     VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
-    uint64_t gfn = info->gfn & info->mask;
+    uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
+    uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
     return (entry->domain_id == info->domain_id) &&
-            ((entry->gfn & info->mask) == gfn);
+            (((entry->gfn & info->mask) == gfn) ||
+             (entry->gfn == gfn_tlb));
 }
 
 /* Reset all the gen of VTDAddressSpace to zero and set the gen of
@@ -166,19 +181,17 @@ static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
  */
 static void vtd_reset_context_cache(IntelIOMMUState *s)
 {
-    VTDAddressSpace **pvtd_as;
     VTDAddressSpace *vtd_as;
-    uint32_t bus_it;
+    VTDBus *vtd_bus;
+    GHashTableIter bus_it;
     uint32_t devfn_it;
 
+    g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
+
     VTD_DPRINTF(CACHE, "global context_cache_gen=1");
-    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {
-        pvtd_as = s->address_spaces[bus_it];
-        if (!pvtd_as) {
-            continue;
-        }
+    while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
         for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (!vtd_as) {
                 continue;
             }
@@ -194,24 +207,46 @@ static void vtd_reset_iotlb(IntelIOMMUState *s)
     g_hash_table_remove_all(s->iotlb);
 }
 
+static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint8_t source_id,
+                                  uint32_t level)
+{
+    return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
+           ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
+}
+
+static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
+{
+    return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
+}
+
 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
                                        hwaddr addr)
 {
+    VTDIOTLBEntry *entry;
     uint64_t key;
+    int level;
+
+    for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
+        key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
+                                source_id, level);
+        entry = g_hash_table_lookup(s->iotlb, &key);
+        if (entry) {
+            goto out;
+        }
+    }
 
-    key = (addr >> VTD_PAGE_SHIFT_4K) |
-           ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
-    return g_hash_table_lookup(s->iotlb, &key);
-
+out:
+    return entry;
 }
 
 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
                              uint16_t domain_id, hwaddr addr, uint64_t slpte,
-                             bool read_flags, bool write_flags)
+                             bool read_flags, bool write_flags,
+                             uint32_t level)
 {
     VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
     uint64_t *key = g_malloc(sizeof(*key));
-    uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K;
+    uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
 
     VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
                 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
@@ -226,7 +261,8 @@ static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
     entry->slpte = slpte;
     entry->read_flags = read_flags;
     entry->write_flags = write_flags;
-    *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
+    entry->mask = vtd_slpt_level_page_mask(level);
+    *key = vtd_get_iotlb_key(gfn, source_id, level);
     g_hash_table_replace(s->iotlb, key, entry);
 }
 
@@ -501,12 +537,6 @@ static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
     return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
 }
 
-/* The shift of an addr for a certain level of paging structure */
-static inline uint32_t vtd_slpt_level_shift(uint32_t level)
-{
-    return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
-}
-
 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
 {
     return slpte & VTD_SL_PT_BASE_ADDR_MASK;
@@ -754,14 +784,15 @@ static inline bool vtd_is_interrupt_addr(hwaddr addr)
  * @is_write: The access is a write operation
  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
  */
-static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
+static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
                                    uint8_t devfn, hwaddr addr, bool is_write,
                                    IOMMUTLBEntry *entry)
 {
     IntelIOMMUState *s = vtd_as->iommu_state;
     VTDContextEntry ce;
+    uint8_t bus_num = pci_bus_num(bus);
     VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
-    uint64_t slpte;
+    uint64_t slpte, page_mask;
     uint32_t level;
     uint16_t source_id = vtd_make_source_id(bus_num, devfn);
     int ret_fr;
@@ -801,6 +832,7 @@ static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
         slpte = iotlb_entry->slpte;
         reads = iotlb_entry->read_flags;
         writes = iotlb_entry->write_flags;
+        page_mask = iotlb_entry->mask;
         goto out;
     }
     /* Try to fetch context-entry from cache first */
@@ -847,12 +879,13 @@ static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
         return;
     }
 
+    page_mask = vtd_slpt_level_page_mask(level);
     vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
-                     reads, writes);
+                     reads, writes, level);
 out:
-    entry->iova = addr & VTD_PAGE_MASK_4K;
-    entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K;
-    entry->addr_mask = ~VTD_PAGE_MASK_4K;
+    entry->iova = addr & page_mask;
+    entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
+    entry->addr_mask = ~page_mask;
     entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
 }
 
@@ -874,6 +907,29 @@ static void vtd_context_global_invalidate(IntelIOMMUState *s)
     }
 }
 
+
+/* Find the VTD address space currently associated with a given bus number,
+ */
+static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
+{
+    VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
+    if (!vtd_bus) {
+        /* Iterate over the registered buses to find the one
+         * which currently hold this bus number, and update the bus_num lookup table:
+         */
+        GHashTableIter iter;
+
+        g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
+        while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
+            if (pci_bus_num(vtd_bus->bus) == bus_num) {
+                s->vtd_as_by_bus_num[bus_num] = vtd_bus;
+                return vtd_bus;
+            }
+        }
+    }
+    return vtd_bus;
+}
+
 /* Do a context-cache device-selective invalidation.
  * @func_mask: FM field after shifting
  */
@@ -882,7 +938,7 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s,
                                           uint16_t func_mask)
 {
     uint16_t mask;
-    VTDAddressSpace **pvtd_as;
+    VTDBus *vtd_bus;
     VTDAddressSpace *vtd_as;
     uint16_t devfn;
     uint16_t devfn_it;
@@ -903,11 +959,11 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s,
     }
     VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
                     " mask %"PRIu16, source_id, mask);
-    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];
-    if (pvtd_as) {
+    vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
+    if (vtd_bus) {
         devfn = VTD_SID_TO_DEVFN(source_id);
         for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
                 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
                             devfn_it);
@@ -967,7 +1023,7 @@ static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
 
     assert(am <= VTD_MAMV);
     info.domain_id = domain_id;
-    info.gfn = addr >> VTD_PAGE_SHIFT_4K;
+    info.addr = addr;
     info.mask = ~((1 << am) - 1);
     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
 }
@@ -1805,11 +1861,11 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
         return ret;
     }
 
-    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,
+    vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
                            is_write, &ret);
     VTD_DPRINTF(MMU,
                 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
-                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,
+                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
                 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
                 vtd_as->devfn, addr, ret.translated_addr);
     return ret;
@@ -1839,6 +1895,38 @@ static Property vtd_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
+{
+    uintptr_t key = (uintptr_t)bus;
+    VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
+    VTDAddressSpace *vtd_dev_as;
+
+    if (!vtd_bus) {
+        /* No corresponding free() */
+        vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
+        vtd_bus->bus = bus;
+        key = (uintptr_t)bus;
+        g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
+    }
+
+    vtd_dev_as = vtd_bus->dev_as[devfn];
+
+    if (!vtd_dev_as) {
+        vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
+
+        vtd_dev_as->bus = bus;
+        vtd_dev_as->devfn = (uint8_t)devfn;
+        vtd_dev_as->iommu_state = s;
+        vtd_dev_as->context_cache_entry.context_cache_gen = 0;
+        memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
+                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
+        address_space_init(&vtd_dev_as->as,
+                           &vtd_dev_as->iommu, "intel_iommu");
+    }
+    return vtd_dev_as;
+}
+
 /* Do the initialization. It will also be called when reset, so pay
  * attention when adding new initialization stuff.
  */
@@ -1861,7 +1949,7 @@ static void vtd_init(IntelIOMMUState *s)
     s->iq_last_desc_type = VTD_INV_DESC_NONE;
     s->next_frcd_reg = 0;
     s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
-             VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI;
+             VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
     s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
 
     vtd_reset_context_cache(s);
@@ -1931,13 +2019,15 @@ static void vtd_realize(DeviceState *dev, Error **errp)
     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
 
     VTD_DPRINTF(GENERAL, "");
-    memset(s->address_spaces, 0, sizeof(s->address_spaces));
+    memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
     memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
                           "intel_iommu", DMAR_REG_SIZE);
     sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
     /* No corresponding destroy */
     s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
                                      g_free, g_free);
+    s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
+                                              g_free, g_free);
     vtd_init(s);
 }
 
diff --git a/qemu/hw/i386/intel_iommu_internal.h b/qemu/hw/i386/intel_iommu_internal.h
index ba288ab1d..e5f514c6e 100644
--- a/qemu/hw/i386/intel_iommu_internal.h
+++ b/qemu/hw/i386/intel_iommu_internal.h
@@ -113,6 +113,7 @@
 
 /* The shift of source_id in the key of IOTLB hash table */
 #define VTD_IOTLB_SID_SHIFT         36
+#define VTD_IOTLB_LVL_SHIFT         44
 #define VTD_IOTLB_MAX_SIZE          1024    /* Max size of the hash table */
 
 /* IOTLB_REG */
@@ -185,9 +186,10 @@
 #define VTD_CAP_ND                  (((VTD_DOMAIN_ID_SHIFT - 4) / 2) & 7ULL)
 #define VTD_MGAW                    39  /* Maximum Guest Address Width */
 #define VTD_CAP_MGAW                (((VTD_MGAW - 1) & 0x3fULL) << 16)
-#define VTD_MAMV                    9ULL
+#define VTD_MAMV                    18ULL
 #define VTD_CAP_MAMV                (VTD_MAMV << 48)
 #define VTD_CAP_PSI                 (1ULL << 39)
+#define VTD_CAP_SLLPS               ((1ULL << 34) | (1ULL << 35))
 
 /* Supported Adjusted Guest Address Widths */
 #define VTD_CAP_SAGAW_SHIFT         8
@@ -320,7 +322,7 @@ typedef struct VTDInvDesc VTDInvDesc;
 /* Information about page-selective IOTLB invalidate */
 struct VTDIOTLBPageInvInfo {
     uint16_t domain_id;
-    uint64_t gfn;
+    uint64_t addr;
     uint8_t mask;
 };
 typedef struct VTDIOTLBPageInvInfo VTDIOTLBPageInvInfo;
diff --git a/qemu/hw/i386/kvm/apic.c b/qemu/hw/i386/kvm/apic.c
index 5b470562a..3c7c8fa00 100644
--- a/qemu/hw/i386/kvm/apic.c
+++ b/qemu/hw/i386/kvm/apic.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL version 2.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/pci/msi.h"
 #include "sysemu/kvm.h"
@@ -185,7 +186,7 @@ static void kvm_apic_realize(DeviceState *dev, Error **errp)
                           APIC_SPACE_SIZE);
 
     if (kvm_has_gsi_routing()) {
-        msi_supported = true;
+        msi_nonbroken = true;
     }
 }
 
diff --git a/qemu/hw/i386/kvm/clock.c b/qemu/hw/i386/kvm/clock.c
index 0593a3f1f..a3b300cad 100644
--- a/qemu/hw/i386/kvm/clock.c
+++ b/qemu/hw/i386/kvm/clock.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/hw/i386/kvm/i8254.c b/qemu/hw/i386/kvm/i8254.c
index 90eea10df..a4462e5ca 100644
--- a/qemu/hw/i386/kvm/i8254.c
+++ b/qemu/hw/i386/kvm/i8254.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/i8254.h"
diff --git a/qemu/hw/i386/kvm/i8259.c b/qemu/hw/i386/kvm/i8259.c
index 53e3ca8c6..2b207de01 100644
--- a/qemu/hw/i386/kvm/i8259.c
+++ b/qemu/hw/i386/kvm/i8259.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL version 2.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/isa/i8259_internal.h"
 #include "hw/i386/apic_internal.h"
 #include "sysemu/kvm.h"
diff --git a/qemu/hw/i386/kvm/ioapic.c b/qemu/hw/i386/kvm/ioapic.c
index d2a6c4cf6..8eb2c7a70 100644
--- a/qemu/hw/i386/kvm/ioapic.c
+++ b/qemu/hw/i386/kvm/ioapic.c
@@ -10,6 +10,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "monitor/monitor.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/i386/apic_internal.h"
@@ -110,6 +112,15 @@ static void kvm_ioapic_put(IOAPICCommonState *s)
     }
 }
 
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    IOAPICCommonState s;
+
+    kvm_ioapic_get(&s);
+
+    ioapic_print_redtbl(mon, &s);
+}
+
 static void kvm_ioapic_reset(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/qemu/hw/i386/kvm/pci-assign.c b/qemu/hw/i386/kvm/pci-assign.c
index 74d22f4fd..bf425a2b9 100644
--- a/qemu/hw/i386/kvm/pci-assign.c
+++ b/qemu/hw/i386/kvm/pci-assign.c
@@ -20,12 +20,9 @@
  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  */
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/io.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "qemu/error-report.h"
@@ -37,6 +34,7 @@
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
 #include "kvm_i386.h"
+#include "hw/pci/pci-assign.h"
 
 #define MSIX_PAGE_SIZE 0x1000
 
@@ -48,17 +46,6 @@
 #define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
 #define IORESOURCE_MEM_64   0x00100000
 
-//#define DEVICE_ASSIGNMENT_DEBUG
-
-#ifdef DEVICE_ASSIGNMENT_DEBUG
-#define DEBUG(fmt, ...)                                       \
-    do {                                                      \
-        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
-    } while (0)
-#else
-#define DEBUG(fmt, ...)
-#endif
-
 typedef struct PCIRegion {
     int type;           /* Memory or port I/O */
     int valid;
@@ -781,7 +768,7 @@ static char *assign_failed_examine(const AssignedDevice *dev)
         "*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
         "pci-stub/bind\n"
         "*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/remove_id\n"
-        "***",
+        "***\n",
         ns, dev->host.domain, dev->host.bus, dev->host.slot,
         dev->host.function, vendor_id, device_id,
         dev->host.domain, dev->host.bus, dev->host.slot, dev->host.function,
@@ -789,7 +776,7 @@ static char *assign_failed_examine(const AssignedDevice *dev)
         dev->host.function, vendor_id, device_id);
 
 fail:
-    return g_strdup("Couldn't find out why.");
+    return g_strdup("Couldn't find out why.\n");
 }
 
 static void assign_device(AssignedDevice *dev, Error **errp)
@@ -823,8 +810,9 @@ static void assign_device(AssignedDevice *dev, Error **errp)
             char *cause;
 
             cause = assign_failed_examine(dev);
-            error_setg_errno(errp, -r, "Failed to assign device \"%s\"\n%s",
-                             dev->dev.qdev.id, cause);
+            error_setg_errno(errp, -r, "Failed to assign device \"%s\"",
+                             dev->dev.qdev.id);
+            error_append_hint(errp, "%s", cause);
             g_free(cause);
             break;
         }
@@ -923,11 +911,10 @@ retry:
             dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK;
             goto retry;
         }
-        error_setg_errno(errp, -r,
-                         "Failed to assign irq for \"%s\"\n"
-                         "Perhaps you are assigning a device "
-                         "that shares an IRQ with another device?",
+        error_setg_errno(errp, -r, "Failed to assign irq for \"%s\"",
                          dev->dev.qdev.id);
+        error_append_hint(errp, "Perhaps you are assigning a device "
+                          "that shares an IRQ with another device?\n");
         return r;
     }
 
@@ -990,7 +977,7 @@ static void assigned_dev_update_msi(PCIDevice *pci_dev)
         MSIMessage msg = msi_get_message(pci_dev, 0);
         int virq;
 
-        virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        virq = kvm_irqchip_add_msi_route(kvm_state, msg, pci_dev);
         if (virq < 0) {
             perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route");
             return;
@@ -1028,7 +1015,7 @@ static void assigned_dev_update_msi_msg(PCIDevice *pci_dev)
     }
 
     kvm_irqchip_update_msi_route(kvm_state, assigned_dev->msi_virq[0],
-                                 msi_get_message(pci_dev, 0));
+                                 msi_get_message(pci_dev, 0), pci_dev);
 }
 
 static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
@@ -1094,7 +1081,7 @@ static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
 
         msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32);
         msg.data = entry->data;
-        r = kvm_irqchip_add_msi_route(kvm_state, msg);
+        r = kvm_irqchip_add_msi_route(kvm_state, msg, pci_dev);
         if (r < 0) {
             return r;
         }
@@ -1494,7 +1481,7 @@ static int assigned_device_pci_cap_init(PCIDevice *pci_dev, Error **errp)
          * error bits, leave the rest. */
         status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS);
         status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN);
-        status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn;
+        status |= pci_requester_id(pci_dev);
         status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL |
                     PCI_X_STATUS_SPL_ERR);
         pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status);
@@ -1613,7 +1600,8 @@ static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr,
                 msg.data = entry->data;
 
                 ret = kvm_irqchip_update_msi_route(kvm_state,
-                                                   adev->msi_virq[i], msg);
+                                                   adev->msi_virq[i], msg,
+                                                   pdev);
                 if (ret) {
                     error_report("Error updating irq routing entry (%d)", ret);
                 }
@@ -1896,73 +1884,15 @@ static void assign_register_types(void)
 
 type_init(assign_register_types)
 
-/*
- * Scan the assigned devices for the devices that have an option ROM, and then
- * load the corresponding ROM data to RAM. If an error occurs while loading an
- * option ROM, we just ignore that option ROM and continue with the next one.
- */
 static void assigned_dev_load_option_rom(AssignedDevice *dev)
 {
-    char name[32], rom_file[64];
-    FILE *fp;
-    uint8_t val;
-    struct stat st;
-    void *ptr;
-
-    /* If loading ROM from file, pci handles it */
-    if (dev->dev.romfile || !dev->dev.rom_bar) {
-        return;
-    }
+    int size = 0;
 
-    snprintf(rom_file, sizeof(rom_file),
-             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
-             dev->host.domain, dev->host.bus, dev->host.slot,
-             dev->host.function);
+    pci_assign_dev_load_option_rom(&dev->dev, OBJECT(dev), &size,
+                                   dev->host.domain, dev->host.bus,
+                                   dev->host.slot, dev->host.function);
 
-    if (stat(rom_file, &st)) {
-        return;
-    }
-
-    if (access(rom_file, F_OK)) {
-        error_report("pci-assign: Insufficient privileges for %s", rom_file);
-        return;
-    }
-
-    /* Write "1" to the ROM file to enable it */
-    fp = fopen(rom_file, "r+");
-    if (fp == NULL) {
-        return;
+    if (!size) {
+        error_report("pci-assign: Invalid ROM.");
     }
-    val = 1;
-    if (fwrite(&val, 1, 1, fp) != 1) {
-        goto close_rom;
-    }
-    fseek(fp, 0, SEEK_SET);
-
-    snprintf(name, sizeof(name), "%s.rom",
-            object_get_typename(OBJECT(dev)));
-    memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size,
-                           &error_abort);
-    vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
-    ptr = memory_region_get_ram_ptr(&dev->dev.rom);
-    memset(ptr, 0xff, st.st_size);
-
-    if (!fread(ptr, 1, st.st_size, fp)) {
-        error_report("pci-assign: Cannot read from host %s", rom_file);
-        error_printf("Device option ROM contents are probably invalid "
-                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
-                     "or load from file with romfile=\n");
-        goto close_rom;
-    }
-
-    pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom);
-    dev->dev.has_rom = true;
-close_rom:
-    /* Write "0" to disable ROM */
-    fseek(fp, 0, SEEK_SET);
-    val = 0;
-    if (!fwrite(&val, 1, 1, fp)) {
-        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
-    }
-    fclose(fp);
 }
diff --git a/qemu/hw/i386/kvmvapic.c b/qemu/hw/i386/kvmvapic.c
index c6d34b254..c69f37404 100644
--- a/qemu/hw/i386/kvmvapic.c
+++ b/qemu/hw/i386/kvmvapic.c
@@ -8,6 +8,7 @@
  * (at your option) any later version. See the COPYING file in the
  * top-level directory.
  */
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
@@ -634,13 +635,18 @@ static int vapic_prepare(VAPICROMState *s)
 static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
                         unsigned int size)
 {
-    CPUState *cs = current_cpu;
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    hwaddr rom_paddr;
     VAPICROMState *s = opaque;
+    X86CPU *cpu;
+    CPUX86State *env;
+    hwaddr rom_paddr;
 
-    cpu_synchronize_state(cs);
+    if (!current_cpu) {
+        return;
+    }
+
+    cpu_synchronize_state(current_cpu);
+    cpu = X86_CPU(current_cpu);
+    env = &cpu->env;
 
     /*
      * The VAPIC supports two PIO-based hypercalls, both via port 0x7E.
diff --git a/qemu/hw/i386/multiboot.c b/qemu/hw/i386/multiboot.c
index 1adbe9e25..387caa67d 100644
--- a/qemu/hw/i386/multiboot.c
+++ b/qemu/hw/i386/multiboot.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/nvram/fw_cfg.h"
 #include "multiboot.h"
@@ -195,7 +198,8 @@ int load_multiboot(FWCfgState *fw_cfg,
         }
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_low, &elf_high, 0, ELF_MACHINE, 0);
+                               &elf_low, &elf_high, 0, I386_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "Error while loading elf kernel\n");
             exit(1);
diff --git a/qemu/hw/i386/pc.c b/qemu/hw/i386/pc.c
index 7661ea9cd..99437e0b7 100644
--- a/qemu/hw/i386/pc.c
+++ b/qemu/hw/i386/pc.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
@@ -33,7 +34,7 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/timer/hpet.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "multiboot.h"
@@ -59,13 +60,13 @@
 #include "qemu/error-report.h"
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/cpu_hotplug.h"
-#include "hw/cpu/icc_bus.h"
 #include "hw/boards.h"
 #include "hw/pci/pci_host.h"
 #include "acpi-build.h"
 #include "hw/mem/pc-dimm.h"
 #include "qapi/visitor.h"
 #include "qapi-visit.h"
+#include "qom/cpu.h"
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -77,16 +78,6 @@
 #define DPRINTF(fmt, ...)
 #endif
 
-/* Leave a chunk of memory at the top of RAM for the BIOS ACPI tables
- * (128K) and other BIOS datastructures (less than 4K reported to be used at
- * the moment, 32K should be enough for a while).  */
-static unsigned acpi_data_size = 0x20000 + 0x8000;
-void pc_set_legacy_acpi_data_size(void)
-{
-    acpi_data_size = 0x10000;
-}
-
-#define BIOS_CFG_IOPORT 0x510
 #define FW_CFG_ACPI_TABLES (FW_CFG_ARCH_LOCAL + 0)
 #define FW_CFG_SMBIOS_ENTRIES (FW_CFG_ARCH_LOCAL + 1)
 #define FW_CFG_IRQ0_OVERRIDE (FW_CFG_ARCH_LOCAL + 2)
@@ -208,24 +199,24 @@ static void pic_irq_request(void *opaque, int irq, int level)
 
 #define REG_EQUIPMENT_BYTE          0x14
 
-static int cmos_get_fd_drive_type(FDriveType fd0)
+int cmos_get_fd_drive_type(FloppyDriveType fd0)
 {
     int val;
 
     switch (fd0) {
-    case FDRIVE_DRV_144:
+    case FLOPPY_DRIVE_TYPE_144:
         /* 1.44 Mb 3"5 drive */
         val = 4;
         break;
-    case FDRIVE_DRV_288:
+    case FLOPPY_DRIVE_TYPE_288:
         /* 2.88 Mb 3"5 drive */
         val = 5;
         break;
-    case FDRIVE_DRV_120:
+    case FLOPPY_DRIVE_TYPE_120:
         /* 1.2 Mb 5"5 drive */
         val = 2;
         break;
-    case FDRIVE_DRV_NONE:
+    case FLOPPY_DRIVE_TYPE_NONE:
     default:
         val = 0;
         break;
@@ -296,7 +287,8 @@ static void pc_boot_set(void *opaque, const char *boot_device, Error **errp)
 static void pc_cmos_init_floppy(ISADevice *rtc_state, ISADevice *floppy)
 {
     int val, nb, i;
-    FDriveType fd_type[2] = { FDRIVE_DRV_NONE, FDRIVE_DRV_NONE };
+    FloppyDriveType fd_type[2] = { FLOPPY_DRIVE_TYPE_NONE,
+                                   FLOPPY_DRIVE_TYPE_NONE };
 
     /* floppy type */
     if (floppy) {
@@ -310,10 +302,10 @@ static void pc_cmos_init_floppy(ISADevice *rtc_state, ISADevice *floppy)
 
     val = rtc_get_memory(rtc_state, REG_EQUIPMENT_BYTE);
     nb = 0;
-    if (fd_type[0] < FDRIVE_DRV_NONE) {
+    if (fd_type[0] != FLOPPY_DRIVE_TYPE_NONE) {
         nb++;
     }
-    if (fd_type[1] < FDRIVE_DRV_NONE) {
+    if (fd_type[1] != FLOPPY_DRIVE_TYPE_NONE) {
         nb++;
     }
     switch (nb) {
@@ -369,6 +361,31 @@ static const char * const fdc_container_path[] = {
     "/unattached", "/peripheral", "/peripheral-anon"
 };
 
+/*
+ * Locate the FDC at IO address 0x3f0, in order to configure the CMOS registers
+ * and ACPI objects.
+ */
+ISADevice *pc_find_fdc0(void)
+{
+    int i;
+    Object *container;
+    CheckFdcState state = { 0 };
+
+    for (i = 0; i < ARRAY_SIZE(fdc_container_path); i++) {
+        container = container_get(qdev_get_machine(), fdc_container_path[i]);
+        object_child_foreach(container, check_fdc, &state);
+    }
+
+    if (state.multiple) {
+        error_report("warning: multiple floppy disk controllers with "
+                     "iobase=0x3f0 have been found");
+        error_printf("the one being picked for CMOS setup might not reflect "
+                     "your intent");
+    }
+
+    return state.floppy;
+}
+
 static void pc_cmos_init_late(void *opaque)
 {
     pc_cmos_init_late_arg *arg = opaque;
@@ -377,8 +394,6 @@ static void pc_cmos_init_late(void *opaque)
     int8_t heads, sectors;
     int val;
     int i, trans;
-    Object *container;
-    CheckFdcState state = { 0 };
 
     val = 0;
     if (ide_get_geometry(arg->idebus[0], 0,
@@ -408,46 +423,28 @@ static void pc_cmos_init_late(void *opaque)
     }
     rtc_set_memory(s, 0x39, val);
 
-    /*
-     * Locate the FDC at IO address 0x3f0, and configure the CMOS registers
-     * accordingly.
-     */
-    for (i = 0; i < ARRAY_SIZE(fdc_container_path); i++) {
-        container = container_get(qdev_get_machine(), fdc_container_path[i]);
-        object_child_foreach(container, check_fdc, &state);
-    }
-
-    if (state.multiple) {
-        error_report("warning: multiple floppy disk controllers with "
-                     "iobase=0x3f0 have been found;\n"
-                     "the one being picked for CMOS setup might not reflect "
-                     "your intent");
-    }
-    pc_cmos_init_floppy(s, state.floppy);
+    pc_cmos_init_floppy(s, pc_find_fdc0());
 
     qemu_unregister_reset(pc_cmos_init_late, opaque);
 }
 
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
+void pc_cmos_init(PCMachineState *pcms,
                   BusState *idebus0, BusState *idebus1,
                   ISADevice *s)
 {
     int val;
     static pc_cmos_init_late_arg arg;
-    PCMachineState *pc_machine = PC_MACHINE(machine);
-    Error *local_err = NULL;
 
     /* various important CMOS locations needed by PC/Bochs bios */
 
     /* memory size */
     /* base memory (first MiB) */
-    val = MIN(ram_size / 1024, 640);
+    val = MIN(pcms->below_4g_mem_size / 1024, 640);
     rtc_set_memory(s, 0x15, val);
     rtc_set_memory(s, 0x16, val >> 8);
     /* extended memory (next 64MiB) */
-    if (ram_size > 1024 * 1024) {
-        val = (ram_size - 1024 * 1024) / 1024;
+    if (pcms->below_4g_mem_size > 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 1024 * 1024) / 1024;
     } else {
         val = 0;
     }
@@ -458,8 +455,8 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     rtc_set_memory(s, 0x30, val);
     rtc_set_memory(s, 0x31, val >> 8);
     /* memory between 16MiB and 4GiB */
-    if (ram_size > 16 * 1024 * 1024) {
-        val = (ram_size - 16 * 1024 * 1024) / 65536;
+    if (pcms->below_4g_mem_size > 16 * 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 16 * 1024 * 1024) / 65536;
     } else {
         val = 0;
     }
@@ -468,7 +465,7 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     rtc_set_memory(s, 0x34, val);
     rtc_set_memory(s, 0x35, val >> 8);
     /* memory above 4GiB */
-    val = above_4g_mem_size / 65536;
+    val = pcms->above_4g_mem_size / 65536;
     rtc_set_memory(s, 0x5b, val);
     rtc_set_memory(s, 0x5c, val >> 8);
     rtc_set_memory(s, 0x5d, val >> 16);
@@ -476,19 +473,15 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
     /* set the number of CPU */
     rtc_set_memory(s, 0x5f, smp_cpus - 1);
 
-    object_property_add_link(OBJECT(machine), "rtc_state",
+    object_property_add_link(OBJECT(pcms), "rtc_state",
                              TYPE_ISA_DEVICE,
-                             (Object **)&pc_machine->rtc,
+                             (Object **)&pcms->rtc,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
-    object_property_set_link(OBJECT(machine), OBJECT(s),
+    object_property_set_link(OBJECT(pcms), OBJECT(s),
                              "rtc_state", &error_abort);
 
-    set_boot_dev(s, boot_device, &local_err);
-    if (local_err) {
-        error_report_err(local_err);
-        exit(1);
-    }
+    set_boot_dev(s, MACHINE(pcms)->boot_order, &error_fatal);
 
     val = 0;
     val |= 0x02; /* FPU is there */
@@ -706,28 +699,51 @@ static uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index)
     }
 }
 
-/* Calculates the limit to CPU APIC ID values
- *
- * This function returns the limit for the APIC ID value, so that all
- * CPU APIC IDs are < pc_apic_id_limit().
- *
- * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
- */
-static unsigned int pc_apic_id_limit(unsigned int max_cpus)
+static void pc_build_smbios(FWCfgState *fw_cfg)
 {
-    return x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+    uint8_t *smbios_tables, *smbios_anchor;
+    size_t smbios_tables_len, smbios_anchor_len;
+    struct smbios_phys_mem_area *mem_array;
+    unsigned i, array_count;
+
+    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
+    if (smbios_tables) {
+        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
+                         smbios_tables, smbios_tables_len);
+    }
+
+    /* build the array of physical mem area from e820 table */
+    mem_array = g_malloc0(sizeof(*mem_array) * e820_get_num_entries());
+    for (i = 0, array_count = 0; i < e820_get_num_entries(); i++) {
+        uint64_t addr, len;
+
+        if (e820_get_entry(i, E820_RAM, &addr, &len)) {
+            mem_array[array_count].address = addr;
+            mem_array[array_count].length = len;
+            array_count++;
+        }
+    }
+    smbios_get_tables(mem_array, array_count,
+                      &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+    g_free(mem_array);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
 }
 
-static FWCfgState *bochs_bios_init(void)
+static FWCfgState *bochs_bios_init(AddressSpace *as, PCMachineState *pcms)
 {
     FWCfgState *fw_cfg;
-    uint8_t *smbios_tables, *smbios_anchor;
-    size_t smbios_tables_len, smbios_anchor_len;
     uint64_t *numa_fw_cfg;
     int i, j;
-    unsigned int apic_id_limit = pc_apic_id_limit(max_cpus);
 
-    fw_cfg = fw_cfg_init_io(BIOS_CFG_IOPORT);
+    fw_cfg = fw_cfg_init_io_dma(FW_CFG_IO_BASE, FW_CFG_IO_BASE + 4, as);
+
     /* FW_CFG_MAX_CPUS is a bit confusing/problematic on x86:
      *
      * SeaBIOS needs FW_CFG_MAX_CPUS for CPU hotplug, but the CPU hotplug
@@ -742,26 +758,13 @@ static FWCfgState *bochs_bios_init(void)
      * [1] The only kind of "CPU identifier" used between SeaBIOS and QEMU is
      *     the APIC ID, not the "CPU index"
      */
-    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)apic_id_limit);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)pcms->apic_id_limit);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES,
                      acpi_tables, acpi_tables_len);
     fw_cfg_add_i32(fw_cfg, FW_CFG_IRQ0_OVERRIDE, kvm_allows_irq0_override());
 
-    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
-    if (smbios_tables) {
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
-                         smbios_tables, smbios_tables_len);
-    }
-
-    smbios_get_tables(&smbios_tables, &smbios_tables_len,
-                      &smbios_anchor, &smbios_anchor_len);
-    if (smbios_anchor) {
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
-                        smbios_tables, smbios_tables_len);
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
-                        smbios_anchor, smbios_anchor_len);
-    }
+    pc_build_smbios(fw_cfg);
 
     fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE,
                      &e820_reserve, sizeof(e820_reserve));
@@ -773,11 +776,11 @@ static FWCfgState *bochs_bios_init(void)
      * of nodes, one word for each VCPU->node and one word for each node to
      * hold the amount of memory.
      */
-    numa_fw_cfg = g_new0(uint64_t, 1 + apic_id_limit + nb_numa_nodes);
+    numa_fw_cfg = g_new0(uint64_t, 1 + pcms->apic_id_limit + nb_numa_nodes);
     numa_fw_cfg[0] = cpu_to_le64(nb_numa_nodes);
     for (i = 0; i < max_cpus; i++) {
         unsigned int apic_id = x86_cpu_apic_id_from_index(i);
-        assert(apic_id < apic_id_limit);
+        assert(apic_id < pcms->apic_id_limit);
         for (j = 0; j < nb_numa_nodes; j++) {
             if (test_bit(i, numa_info[j].node_cpu)) {
                 numa_fw_cfg[apic_id + 1] = cpu_to_le64(j);
@@ -786,10 +789,11 @@ static FWCfgState *bochs_bios_init(void)
         }
     }
     for (i = 0; i < nb_numa_nodes; i++) {
-        numa_fw_cfg[apic_id_limit + 1 + i] = cpu_to_le64(numa_info[i].node_mem);
+        numa_fw_cfg[pcms->apic_id_limit + 1 + i] =
+            cpu_to_le64(numa_info[i].node_mem);
     }
     fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, numa_fw_cfg,
-                     (1 + apic_id_limit + nb_numa_nodes) *
+                     (1 + pcms->apic_id_limit + nb_numa_nodes) *
                      sizeof(*numa_fw_cfg));
 
     return fw_cfg;
@@ -809,11 +813,8 @@ static long get_file_size(FILE *f)
     return size;
 }
 
-static void load_linux(FWCfgState *fw_cfg,
-                       const char *kernel_filename,
-                       const char *initrd_filename,
-                       const char *kernel_cmdline,
-                       hwaddr max_ram_size)
+static void load_linux(PCMachineState *pcms,
+                       FWCfgState *fw_cfg)
 {
     uint16_t protocol;
     int setup_size, kernel_size, initrd_size = 0, cmdline_size;
@@ -822,6 +823,11 @@ static void load_linux(FWCfgState *fw_cfg,
     hwaddr real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
     FILE *f;
     char *vmode;
+    MachineState *machine = MACHINE(pcms);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+    const char *kernel_filename = machine->kernel_filename;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
 
     /* Align to 16 bytes as a paranoia measure */
     cmdline_size = (strlen(kernel_cmdline)+16) & ~15;
@@ -886,8 +892,8 @@ static void load_linux(FWCfgState *fw_cfg,
         initrd_max = 0x37ffffff;
     }
 
-    if (initrd_max >= max_ram_size - acpi_data_size) {
-        initrd_max = max_ram_size - acpi_data_size - 1;
+    if (initrd_max >= pcms->below_4g_mem_size - pcmc->acpi_data_size) {
+        initrd_max = pcms->below_4g_mem_size - pcmc->acpi_data_size - 1;
     }
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
@@ -965,6 +971,10 @@ static void load_linux(FWCfgState *fw_cfg,
         setup_size = 4;
     }
     setup_size = (setup_size+1)*512;
+    if (setup_size > kernel_size) {
+        fprintf(stderr, "qemu: invalid kernel header\n");
+        exit(1);
+    }
     kernel_size -= setup_size;
 
     setup  = g_malloc(setup_size);
@@ -1031,23 +1041,16 @@ void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
 }
 
 static X86CPU *pc_new_cpu(const char *cpu_model, int64_t apic_id,
-                          DeviceState *icc_bridge, Error **errp)
+                          Error **errp)
 {
     X86CPU *cpu = NULL;
     Error *local_err = NULL;
 
-    if (icc_bridge == NULL) {
-        error_setg(&local_err, "Invalid icc-bridge value");
-        goto out;
-    }
-
     cpu = cpu_x86_create(cpu_model, &local_err);
     if (local_err != NULL) {
         goto out;
     }
 
-    qdev_set_parent_bus(DEVICE(cpu), qdev_get_child_bus(icc_bridge, "icc"));
-
     object_property_set_int(OBJECT(cpu), apic_id, "apic-id", &local_err);
     object_property_set_bool(OBJECT(cpu), true, "realized", &local_err);
 
@@ -1060,12 +1063,10 @@ out:
     return cpu;
 }
 
-static const char *current_cpu_model;
-
 void pc_hot_add_cpu(const int64_t id, Error **errp)
 {
-    DeviceState *icc_bridge;
     X86CPU *cpu;
+    MachineState *machine = MACHINE(qdev_get_machine());
     int64_t apic_id = x86_cpu_apic_id_from_index(id);
     Error *local_err = NULL;
 
@@ -1093,9 +1094,7 @@ void pc_hot_add_cpu(const int64_t id, Error **errp)
         return;
     }
 
-    icc_bridge = DEVICE(object_resolve_path_type("icc-bridge",
-                                                 TYPE_ICC_BRIDGE, NULL));
-    cpu = pc_new_cpu(current_cpu_model, apic_id, icc_bridge, &local_err);
+    cpu = pc_new_cpu(machine->cpu_model, apic_id, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1103,45 +1102,46 @@ void pc_hot_add_cpu(const int64_t id, Error **errp)
     object_unref(OBJECT(cpu));
 }
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge)
+void pc_cpus_init(PCMachineState *pcms)
 {
     int i;
     X86CPU *cpu = NULL;
-    Error *error = NULL;
-    unsigned long apic_id_limit;
+    MachineState *machine = MACHINE(pcms);
 
     /* init CPUs */
-    if (cpu_model == NULL) {
+    if (machine->cpu_model == NULL) {
 #ifdef TARGET_X86_64
-        cpu_model = "qemu64";
+        machine->cpu_model = "qemu64";
 #else
-        cpu_model = "qemu32";
+        machine->cpu_model = "qemu32";
 #endif
     }
-    current_cpu_model = cpu_model;
 
-    apic_id_limit = pc_apic_id_limit(max_cpus);
-    if (apic_id_limit > ACPI_CPU_HOTPLUG_ID_LIMIT) {
-        error_report("max_cpus is too large. APIC ID of last CPU is %lu",
-                     apic_id_limit - 1);
+    /* Calculates the limit to CPU APIC ID values
+     *
+     * Limit for the APIC ID value, so that all
+     * CPU APIC IDs are < pcms->apic_id_limit.
+     *
+     * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
+     */
+    pcms->apic_id_limit = x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+    if (pcms->apic_id_limit > ACPI_CPU_HOTPLUG_ID_LIMIT) {
+        error_report("max_cpus is too large. APIC ID of last CPU is %u",
+                     pcms->apic_id_limit - 1);
         exit(1);
     }
 
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = pc_new_cpu(cpu_model, x86_cpu_apic_id_from_index(i),
-                         icc_bridge, &error);
-        if (error) {
-            error_report_err(error);
-            exit(1);
+    pcms->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
+                                    sizeof(CPUArchId) * max_cpus);
+    for (i = 0; i < max_cpus; i++) {
+        pcms->possible_cpus->cpus[i].arch_id = x86_cpu_apic_id_from_index(i);
+        pcms->possible_cpus->len++;
+        if (i < smp_cpus) {
+            cpu = pc_new_cpu(machine->cpu_model, x86_cpu_apic_id_from_index(i),
+                             &error_fatal);
+            pcms->possible_cpus->cpus[i].cpu = CPU(cpu);
+            object_unref(OBJECT(cpu));
         }
-        object_unref(OBJECT(cpu));
-    }
-
-    /* map APIC MMIO area if CPU has APIC */
-    if (cpu && cpu->apic_state) {
-        /* XXX: what if the base changes? */
-        sysbus_mmio_map_overlap(SYS_BUS_DEVICE(icc_bridge), 0,
-                                APIC_DEFAULT_ADDRESS, 0x1000);
     }
 
     /* tell smbios about cpuid version and features */
@@ -1156,18 +1156,12 @@ typedef struct PcRomPciInfo {
     uint64_t w64_max;
 } PcRomPciInfo;
 
-typedef struct PcGuestInfoState {
-    PcGuestInfo info;
-    Notifier machine_done;
-} PcGuestInfoState;
-
 static
-void pc_guest_info_machine_done(Notifier *notifier, void *data)
+void pc_machine_done(Notifier *notifier, void *data)
 {
-    PcGuestInfoState *guest_info_state = container_of(notifier,
-                                                      PcGuestInfoState,
-                                                      machine_done);
-    PCIBus *bus = find_i440fx();
+    PCMachineState *pcms = container_of(notifier,
+                                        PCMachineState, machine_done);
+    PCIBus *bus = pcms->bus;
 
     if (bus) {
         int extra_hosts = 0;
@@ -1178,52 +1172,45 @@ void pc_guest_info_machine_done(Notifier *notifier, void *data)
                 extra_hosts++;
             }
         }
-        if (extra_hosts && guest_info_state->info.fw_cfg) {
+        if (extra_hosts && pcms->fw_cfg) {
             uint64_t *val = g_malloc(sizeof(*val));
             *val = cpu_to_le64(extra_hosts);
-            fw_cfg_add_file(guest_info_state->info.fw_cfg,
+            fw_cfg_add_file(pcms->fw_cfg,
                     "etc/extra-pci-roots", val, sizeof(*val));
         }
     }
 
-    acpi_setup(&guest_info_state->info);
+    acpi_setup();
 }
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size)
+void pc_guest_info_init(PCMachineState *pcms)
 {
-    PcGuestInfoState *guest_info_state = g_malloc0(sizeof *guest_info_state);
-    PcGuestInfo *guest_info = &guest_info_state->info;
     int i, j;
 
-    guest_info->ram_size_below_4g = below_4g_mem_size;
-    guest_info->ram_size = below_4g_mem_size + above_4g_mem_size;
-    guest_info->apic_id_limit = pc_apic_id_limit(max_cpus);
-    guest_info->apic_xrupt_override = kvm_allows_irq0_override();
-    guest_info->numa_nodes = nb_numa_nodes;
-    guest_info->node_mem = g_malloc0(guest_info->numa_nodes *
-                                    sizeof *guest_info->node_mem);
+    pcms->apic_xrupt_override = kvm_allows_irq0_override();
+    pcms->numa_nodes = nb_numa_nodes;
+    pcms->node_mem = g_malloc0(pcms->numa_nodes *
+                                    sizeof *pcms->node_mem);
     for (i = 0; i < nb_numa_nodes; i++) {
-        guest_info->node_mem[i] = numa_info[i].node_mem;
+        pcms->node_mem[i] = numa_info[i].node_mem;
     }
 
-    guest_info->node_cpu = g_malloc0(guest_info->apic_id_limit *
-                                     sizeof *guest_info->node_cpu);
+    pcms->node_cpu = g_malloc0(pcms->apic_id_limit *
+                                     sizeof *pcms->node_cpu);
 
     for (i = 0; i < max_cpus; i++) {
         unsigned int apic_id = x86_cpu_apic_id_from_index(i);
-        assert(apic_id < guest_info->apic_id_limit);
+        assert(apic_id < pcms->apic_id_limit);
         for (j = 0; j < nb_numa_nodes; j++) {
             if (test_bit(i, numa_info[j].node_cpu)) {
-                guest_info->node_cpu[apic_id] = j;
+                pcms->node_cpu[apic_id] = j;
                 break;
             }
         }
     }
 
-    guest_info_state->machine_done.notify = pc_guest_info_machine_done;
-    qemu_add_machine_init_done_notifier(&guest_info_state->machine_done);
-    return guest_info;
+    pcms->machine_done.notify = pc_machine_done;
+    qemu_add_machine_init_done_notifier(&pcms->machine_done);
 }
 
 /* setup pci memory address space mapping into system address space */
@@ -1256,54 +1243,46 @@ void pc_acpi_init(const char *default_dsdt)
 
         acpi_table_add_builtin(opts, &err);
         if (err) {
-            error_report("WARNING: failed to load %s: %s", filename,
-                         error_get_pretty(err));
-            error_free(err);
+            error_reportf_err(err, "WARNING: failed to load %s: ",
+                              filename);
         }
         g_free(filename);
     }
 }
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
-                           PcGuestInfo *guest_info)
+void xen_load_linux(PCMachineState *pcms)
 {
     int i;
     FWCfgState *fw_cfg;
 
-    assert(kernel_filename != NULL);
+    assert(MACHINE(pcms)->kernel_filename != NULL);
 
-    fw_cfg = fw_cfg_init_io(BIOS_CFG_IOPORT);
+    fw_cfg = fw_cfg_init_io(FW_CFG_IO_BASE);
     rom_set_fw(fw_cfg);
 
-    load_linux(fw_cfg, kernel_filename, initrd_filename,
-               kernel_cmdline, below_4g_mem_size);
+    load_linux(pcms, fw_cfg);
     for (i = 0; i < nb_option_roms; i++) {
         assert(!strcmp(option_rom[i].name, "linuxboot.bin") ||
                !strcmp(option_rom[i].name, "multiboot.bin"));
         rom_add_option(option_rom[i].name, option_rom[i].bootindex);
     }
-    guest_info->fw_cfg = fw_cfg;
-    return fw_cfg;
+    pcms->fw_cfg = fw_cfg;
 }
 
-FWCfgState *pc_memory_init(MachineState *machine,
-                           MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
-                           MemoryRegion *rom_memory,
-                           MemoryRegion **ram_memory,
-                           PcGuestInfo *guest_info)
+void pc_memory_init(PCMachineState *pcms,
+                    MemoryRegion *system_memory,
+                    MemoryRegion *rom_memory,
+                    MemoryRegion **ram_memory)
 {
     int linux_boot, i;
     MemoryRegion *ram, *option_rom_mr;
     MemoryRegion *ram_below_4g, *ram_above_4g;
     FWCfgState *fw_cfg;
-    PCMachineState *pcms = PC_MACHINE(machine);
+    MachineState *machine = MACHINE(pcms);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
 
-    assert(machine->ram_size == below_4g_mem_size + above_4g_mem_size);
+    assert(machine->ram_size == pcms->below_4g_mem_size +
+                                pcms->above_4g_mem_size);
 
     linux_boot = (machine->kernel_filename != NULL);
 
@@ -1317,19 +1296,20 @@ FWCfgState *pc_memory_init(MachineState *machine,
     *ram_memory = ram;
     ram_below_4g = g_malloc(sizeof(*ram_below_4g));
     memory_region_init_alias(ram_below_4g, NULL, "ram-below-4g", ram,
-                             0, below_4g_mem_size);
+                             0, pcms->below_4g_mem_size);
     memory_region_add_subregion(system_memory, 0, ram_below_4g);
-    e820_add_entry(0, below_4g_mem_size, E820_RAM);
-    if (above_4g_mem_size > 0) {
+    e820_add_entry(0, pcms->below_4g_mem_size, E820_RAM);
+    if (pcms->above_4g_mem_size > 0) {
         ram_above_4g = g_malloc(sizeof(*ram_above_4g));
         memory_region_init_alias(ram_above_4g, NULL, "ram-above-4g", ram,
-                                 below_4g_mem_size, above_4g_mem_size);
+                                 pcms->below_4g_mem_size,
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(system_memory, 0x100000000ULL,
                                     ram_above_4g);
-        e820_add_entry(0x100000000ULL, above_4g_mem_size, E820_RAM);
+        e820_add_entry(0x100000000ULL, pcms->above_4g_mem_size, E820_RAM);
     }
 
-    if (!guest_info->has_reserved_memory &&
+    if (!pcmc->has_reserved_memory &&
         (machine->ram_slots ||
          (machine->maxram_size > machine->ram_size))) {
         MachineClass *mc = MACHINE_GET_CLASS(machine);
@@ -1340,7 +1320,7 @@ FWCfgState *pc_memory_init(MachineState *machine,
     }
 
     /* initialize hotplug memory address space */
-    if (guest_info->has_reserved_memory &&
+    if (pcmc->has_reserved_memory &&
         (machine->ram_size < machine->maxram_size)) {
         ram_addr_t hotplug_mem_size =
             machine->maxram_size - machine->ram_size;
@@ -1359,9 +1339,9 @@ FWCfgState *pc_memory_init(MachineState *machine,
         }
 
         pcms->hotplug_memory.base =
-            ROUND_UP(0x100000000ULL + above_4g_mem_size, 1ULL << 30);
+            ROUND_UP(0x100000000ULL + pcms->above_4g_mem_size, 1ULL << 30);
 
-        if (pcms->enforce_aligned_dimm) {
+        if (pcmc->enforce_aligned_dimm) {
             /* size hotplug region assuming 1G page max alignment per slot */
             hotplug_mem_size += (1ULL << 30) * machine->ram_slots;
         }
@@ -1380,36 +1360,41 @@ FWCfgState *pc_memory_init(MachineState *machine,
     }
 
     /* Initialize PC system firmware */
-    pc_system_firmware_init(rom_memory, guest_info->isapc_ram_fw);
+    pc_system_firmware_init(rom_memory, !pcmc->pci_enabled);
 
     option_rom_mr = g_malloc(sizeof(*option_rom_mr));
     memory_region_init_ram(option_rom_mr, NULL, "pc.rom", PC_ROM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(option_rom_mr);
     memory_region_add_subregion_overlap(rom_memory,
                                         PC_ROM_MIN_VGA,
                                         option_rom_mr,
                                         1);
 
-    fw_cfg = bochs_bios_init();
+    fw_cfg = bochs_bios_init(&address_space_memory, pcms);
+
     rom_set_fw(fw_cfg);
 
-    if (guest_info->has_reserved_memory && pcms->hotplug_memory.base) {
+    if (pcmc->has_reserved_memory && pcms->hotplug_memory.base) {
         uint64_t *val = g_malloc(sizeof(*val));
-        *val = cpu_to_le64(ROUND_UP(pcms->hotplug_memory.base, 0x1ULL << 30));
+        PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+        uint64_t res_mem_end = pcms->hotplug_memory.base;
+
+        if (!pcmc->broken_reserved_end) {
+            res_mem_end += memory_region_size(&pcms->hotplug_memory.mr);
+        }
+        *val = cpu_to_le64(ROUND_UP(res_mem_end, 0x1ULL << 30));
         fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
     }
 
     if (linux_boot) {
-        load_linux(fw_cfg, machine->kernel_filename, machine->initrd_filename,
-                   machine->kernel_cmdline, below_4g_mem_size);
+        load_linux(pcms, fw_cfg);
     }
 
     for (i = 0; i < nb_option_roms; i++) {
         rom_add_option(option_rom[i].name, option_rom[i].bootindex);
     }
-    guest_info->fw_cfg = fw_cfg;
-    return fw_cfg;
+    pcms->fw_cfg = fw_cfg;
 }
 
 qemu_irq pc_allocate_cpu_irq(void)
@@ -1421,6 +1406,7 @@ DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     DeviceState *dev = NULL;
 
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_VGA);
     if (pci_bus) {
         PCIDevice *pcidev = pci_vga_init(pci_bus);
         dev = pcidev ? &pcidev->qdev : NULL;
@@ -1428,18 +1414,10 @@ DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
         ISADevice *isadev = isa_vga_init(isa_bus);
         dev = isadev ? DEVICE(isadev) : NULL;
     }
+    rom_reset_order_override();
     return dev;
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static const MemoryRegionOps ioport80_io_ops = {
     .write = ioport80_write,
     .read = ioport80_read,
@@ -1464,7 +1442,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
                           bool create_fdctrl,
                           bool no_vmport,
-                          uint32 hpet_irqs)
+                          uint32_t hpet_irqs)
 {
     int i;
     DriveInfo *fd[MAX_FD];
@@ -1474,7 +1452,6 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     qemu_irq rtc_irq = NULL;
     qemu_irq *a20_line;
     ISADevice *i8042, *port92, *vmmouse, *pit = NULL;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ioport80_io = g_new(MemoryRegion, 1);
     MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1);
 
@@ -1519,7 +1496,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     qemu_register_boot_set(pc_boot_set, *rtc_state);
 
     if (!xen_enabled()) {
-        if (kvm_irqchip_in_kernel()) {
+        if (kvm_pit_in_kernel()) {
             pit = kvm_pit_init(isa_bus, 0x40);
         } else {
             pit = pit_init(isa_bus, 0x40, pit_isa_irq, pit_alt_irq);
@@ -1551,8 +1528,7 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     port92 = isa_create_simple(isa_bus, "port92");
     port92_init(port92, &a20_line[1]);
 
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
@@ -1567,6 +1543,7 @@ void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     int i;
 
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_NIC);
     for (i = 0; i < nb_nics; i++) {
         NICInfo *nd = &nd_table[i];
 
@@ -1576,6 +1553,7 @@ void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus)
             pci_nic_init_nofail(nd, pci_bus, "e1000", NULL);
         }
     }
+    rom_reset_order_override();
 }
 
 void pc_pci_device_init(PCIBus *pci_bus)
@@ -1595,7 +1573,7 @@ void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name)
     SysBusDevice *d;
     unsigned int i;
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_ioapic_in_kernel()) {
         dev = qdev_create(NULL, "kvm-ioapic");
     } else {
         dev = qdev_create(NULL, "ioapic");
@@ -1619,12 +1597,13 @@ static void pc_dimm_plug(HotplugHandler *hotplug_dev,
     HotplugHandlerClass *hhc;
     Error *local_err = NULL;
     PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     PCDIMMDevice *dimm = PC_DIMM(dev);
     PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
     MemoryRegion *mr = ddc->get_memory_region(dimm);
     uint64_t align = TARGET_PAGE_SIZE;
 
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
+    if (memory_region_get_alignment(mr) && pcmc->enforce_aligned_dimm) {
         align = memory_region_get_alignment(mr);
     }
 
@@ -1689,9 +1668,19 @@ static void pc_dimm_unplug(HotplugHandler *hotplug_dev,
     error_propagate(errp, local_err);
 }
 
+static int pc_apic_cmp(const void *a, const void *b)
+{
+   CPUArchId *apic_a = (CPUArchId *)a;
+   CPUArchId *apic_b = (CPUArchId *)b;
+
+   return apic_a->arch_id - apic_b->arch_id;
+}
+
 static void pc_cpu_plug(HotplugHandler *hotplug_dev,
                         DeviceState *dev, Error **errp)
 {
+    CPUClass *cc = CPU_GET_CLASS(dev);
+    CPUArchId apic_id, *found_cpu;
     HotplugHandlerClass *hhc;
     Error *local_err = NULL;
     PCMachineState *pcms = PC_MACHINE(hotplug_dev);
@@ -1714,6 +1703,13 @@ static void pc_cpu_plug(HotplugHandler *hotplug_dev,
 
     /* increment the number of CPUs */
     rtc_set_memory(pcms->rtc, 0x5f, rtc_get_memory(pcms->rtc, 0x5f) + 1);
+
+    apic_id.arch_id = cc->get_arch_id(CPU(dev));
+    found_cpu = bsearch(&apic_id, pcms->possible_cpus->cpus,
+        pcms->possible_cpus->len, sizeof(*pcms->possible_cpus->cpus),
+        pc_apic_cmp);
+    assert(found_cpu);
+    found_cpu->cpu = CPU(dev);
 out:
     error_propagate(errp, local_err);
 }
@@ -1765,42 +1761,43 @@ static HotplugHandler *pc_get_hotpug_handler(MachineState *machine,
 }
 
 static void
-pc_machine_get_hotplug_memory_region_size(Object *obj, Visitor *v, void *opaque,
-                                          const char *name, Error **errp)
+pc_machine_get_hotplug_memory_region_size(Object *obj, Visitor *v,
+                                          const char *name, void *opaque,
+                                          Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     int64_t value = memory_region_size(&pcms->hotplug_memory.mr);
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void pc_machine_get_max_ram_below_4g(Object *obj, Visitor *v,
-                                         void *opaque, const char *name,
-                                         Error **errp)
+                                            const char *name, void *opaque,
+                                            Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     uint64_t value = pcms->max_ram_below_4g;
 
-    visit_type_size(v, &value, name, errp);
+    visit_type_size(v, name, &value, errp);
 }
 
 static void pc_machine_set_max_ram_below_4g(Object *obj, Visitor *v,
-                                         void *opaque, const char *name,
-                                         Error **errp)
+                                            const char *name, void *opaque,
+                                            Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     Error *error = NULL;
     uint64_t value;
 
-    visit_type_size(v, &value, name, &error);
+    visit_type_size(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
     }
     if (value > (1ULL << 32)) {
-        error_set(&error, ERROR_CLASS_GENERIC_ERROR,
-                  "Machine option 'max-ram-below-4g=%"PRIu64
-                  "' expects size less than or equal to 4G", value);
+        error_setg(&error,
+                   "Machine option 'max-ram-below-4g=%"PRIu64
+                   "' expects size less than or equal to 4G", value);
         error_propagate(errp, error);
         return;
     }
@@ -1814,21 +1811,21 @@ static void pc_machine_set_max_ram_below_4g(Object *obj, Visitor *v,
     pcms->max_ram_below_4g = value;
 }
 
-static void pc_machine_get_vmport(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void pc_machine_get_vmport(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     OnOffAuto vmport = pcms->vmport;
 
-    visit_type_OnOffAuto(v, &vmport, name, errp);
+    visit_type_OnOffAuto(v, name, &vmport, errp);
 }
 
-static void pc_machine_set_vmport(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void pc_machine_set_vmport(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
 
-    visit_type_OnOffAuto(v, &pcms->vmport, name, errp);
+    visit_type_OnOffAuto(v, name, &pcms->vmport, errp);
 }
 
 bool pc_machine_is_smm_enabled(PCMachineState *pcms)
@@ -1856,28 +1853,35 @@ bool pc_machine_is_smm_enabled(PCMachineState *pcms)
     return false;
 }
 
-static void pc_machine_get_smm(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void pc_machine_get_smm(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     OnOffAuto smm = pcms->smm;
 
-    visit_type_OnOffAuto(v, &smm, name, errp);
+    visit_type_OnOffAuto(v, name, &smm, errp);
+}
+
+static void pc_machine_set_smm(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+
+    visit_type_OnOffAuto(v, name, &pcms->smm, errp);
 }
 
-static void pc_machine_set_smm(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static bool pc_machine_get_nvdimm(Object *obj, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
 
-    visit_type_OnOffAuto(v, &pcms->smm, name, errp);
+    return pcms->acpi_nvdimm_state.is_enabled;
 }
 
-static bool pc_machine_get_aligned_dimm(Object *obj, Error **errp)
+static void pc_machine_set_nvdimm(Object *obj, bool value, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
 
-    return pcms->enforce_aligned_dimm;
+    pcms->acpi_nvdimm_state.is_enabled = value;
 }
 
 static void pc_machine_initfn(Object *obj)
@@ -1886,47 +1890,77 @@ static void pc_machine_initfn(Object *obj)
 
     object_property_add(obj, PC_MACHINE_MEMHP_REGION_SIZE, "int",
                         pc_machine_get_hotplug_memory_region_size,
-                        NULL, NULL, NULL, NULL);
+                        NULL, NULL, NULL, &error_abort);
 
     pcms->max_ram_below_4g = 1ULL << 32; /* 4G */
     object_property_add(obj, PC_MACHINE_MAX_RAM_BELOW_4G, "size",
                         pc_machine_get_max_ram_below_4g,
                         pc_machine_set_max_ram_below_4g,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_MAX_RAM_BELOW_4G,
                                     "Maximum ram below the 4G boundary (32bit boundary)",
-                                    NULL);
+                                    &error_abort);
 
     pcms->smm = ON_OFF_AUTO_AUTO;
     object_property_add(obj, PC_MACHINE_SMM, "OnOffAuto",
                         pc_machine_get_smm,
                         pc_machine_set_smm,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_SMM,
                                     "Enable SMM (pc & q35)",
-                                    NULL);
+                                    &error_abort);
 
     pcms->vmport = ON_OFF_AUTO_AUTO;
     object_property_add(obj, PC_MACHINE_VMPORT, "OnOffAuto",
                         pc_machine_get_vmport,
                         pc_machine_set_vmport,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
     object_property_set_description(obj, PC_MACHINE_VMPORT,
                                     "Enable vmport (pc & q35)",
-                                    NULL);
+                                    &error_abort);
+
+    /* nvdimm is disabled on default. */
+    pcms->acpi_nvdimm_state.is_enabled = false;
+    object_property_add_bool(obj, PC_MACHINE_NVDIMM, pc_machine_get_nvdimm,
+                             pc_machine_set_nvdimm, &error_abort);
+}
+
+static void pc_machine_reset(void)
+{
+    CPUState *cs;
+    X86CPU *cpu;
+
+    qemu_devices_reset();
+
+    /* Reset APIC after devices have been reset to cancel
+     * any changes that qemu_devices_reset() might have done.
+     */
+    CPU_FOREACH(cs) {
+        cpu = X86_CPU(cs);
 
-    pcms->enforce_aligned_dimm = true;
-    object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
-                             pc_machine_get_aligned_dimm,
-                             NULL, NULL);
+        if (cpu->apic_state) {
+            device_reset(cpu->apic_state);
+        }
+    }
 }
 
 static unsigned pc_cpu_index_to_socket_id(unsigned cpu_index)
 {
-    unsigned pkg_id, core_id, smt_id;
+    X86CPUTopoInfo topo;
     x86_topo_ids_from_idx(smp_cores, smp_threads, cpu_index,
-                          &pkg_id, &core_id, &smt_id);
-    return pkg_id;
+                          &topo);
+    return topo.pkg_id;
+}
+
+static CPUArchIdList *pc_possible_cpu_arch_ids(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+    int len = sizeof(CPUArchIdList) +
+              sizeof(CPUArchId) * (pcms->possible_cpus->len);
+    CPUArchIdList *list = g_malloc(len);
+
+    memcpy(list, pcms->possible_cpus, len);
+    return list;
 }
 
 static void pc_machine_class_init(ObjectClass *oc, void *data)
@@ -1936,8 +1970,26 @@ static void pc_machine_class_init(ObjectClass *oc, void *data)
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
 
     pcmc->get_hotplug_handler = mc->get_hotplug_handler;
+    pcmc->pci_enabled = true;
+    pcmc->has_acpi_build = true;
+    pcmc->rsdp_in_ram = true;
+    pcmc->smbios_defaults = true;
+    pcmc->smbios_uuid_encoded = true;
+    pcmc->gigabyte_align = true;
+    pcmc->has_reserved_memory = true;
+    pcmc->kvmclock_enabled = true;
+    pcmc->enforce_aligned_dimm = true;
+    /* BIOS ACPI tables: 128K. Other BIOS datastructures: less than 4K reported
+     * to be used at the moment, 32K should be enough for a while.  */
+    pcmc->acpi_data_size = 0x20000 + 0x8000;
+    pcmc->save_tsc_khz = true;
     mc->get_hotplug_handler = pc_get_hotpug_handler;
     mc->cpu_index_to_socket_id = pc_cpu_index_to_socket_id;
+    mc->possible_cpu_arch_ids = pc_possible_cpu_arch_ids;
+    mc->default_boot_order = "cad";
+    mc->hot_add_cpu = pc_hot_add_cpu;
+    mc->max_cpus = 255;
+    mc->reset = pc_machine_reset;
     hc->plug = pc_machine_device_plug_cb;
     hc->unplug_request = pc_machine_device_unplug_request_cb;
     hc->unplug = pc_machine_device_unplug_cb;
diff --git a/qemu/hw/i386/pc_piix.c b/qemu/hw/i386/pc_piix.c
index a896624f8..7f50116bc 100644
--- a/qemu/hw/i386/pc_piix.c
+++ b/qemu/hw/i386/pc_piix.c
@@ -22,13 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/usb.h"
@@ -39,7 +40,6 @@
 #include "hw/kvm/clock.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
-#include "hw/cpu/icc_bus.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/block-backend.h"
 #include "hw/i2c/smbus.h"
@@ -50,9 +50,11 @@
 #include "cpu.h"
 #include "qemu/error-report.h"
 #ifdef CONFIG_XEN
-#  include <xen/hvm/hvm_info_table.h>
+#include <xen/hvm/hvm_info_table.h>
+#include "hw/xen/xen_pt.h"
 #endif
 #include "migration/migration.h"
+#include "kvm_i386.h"
 
 #define MAX_IDE_BUS 2
 
@@ -60,29 +62,15 @@ static const int ide_iobase[MAX_IDE_BUS] = { 0x1f0, 0x170 };
 static const int ide_iobase2[MAX_IDE_BUS] = { 0x3f6, 0x376 };
 static const int ide_irq[MAX_IDE_BUS] = { 14, 15 };
 
-static bool pci_enabled = true;
-static bool has_acpi_build = true;
-static bool rsdp_in_ram = true;
-static int legacy_acpi_table_size;
-static bool smbios_defaults = true;
-static bool smbios_legacy_mode;
-static bool smbios_uuid_encoded = true;
-/* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
- * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
- * pages in the host.
- */
-static bool gigabyte_align = true;
-static bool has_reserved_memory = true;
-static bool kvmclock_enabled = true;
-
 /* PC hardware initialisation */
-static void pc_init1(MachineState *machine)
+static void pc_init1(MachineState *machine,
+                     const char *host_type, const char *pci_type)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     MemoryRegion *system_memory = get_system_memory();
     MemoryRegion *system_io = get_system_io();
     int i;
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     PCII440FXState *i440fx_state;
@@ -97,8 +85,6 @@ static void pc_init1(MachineState *machine)
     MemoryRegion *ram_memory;
     MemoryRegion *pci_memory;
     MemoryRegion *rom_memory;
-    DeviceState *icc_bridge;
-    PcGuestInfo *guest_info;
     ram_addr_t lowmem;
 
     /* Check whether RAM fits below 4G (leaving 1/2 GByte for IO memory).
@@ -109,7 +95,7 @@ static void pc_init1(MachineState *machine)
      * breaking migration.
      */
     if (machine->ram_size >= 0xe0000000) {
-        lowmem = gigabyte_align ? 0xc0000000 : 0xe0000000;
+        lowmem = pcmc->gigabyte_align ? 0xc0000000 : 0xe0000000;
     } else {
         lowmem = 0xe0000000;
     }
@@ -117,41 +103,35 @@ static void pc_init1(MachineState *machine)
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(qemu limit, user limit).
      */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
+    if (lowmem > pcms->max_ram_below_4g) {
+        lowmem = pcms->max_ram_below_4g;
         if (machine->ram_size - lowmem > lowmem &&
             lowmem & ((1ULL << 30) - 1)) {
             error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
                          ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+                         pcms->max_ram_below_4g);
         }
     }
 
     if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
+        pcms->above_4g_mem_size = machine->ram_size - lowmem;
+        pcms->below_4g_mem_size = lowmem;
     } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = machine->ram_size;
     }
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
-        fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
-        exit(1);
+    if (xen_enabled()) {
+        xen_hvm_init(pcms, &ram_memory);
     }
 
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
+    pc_cpus_init(pcms);
 
-    if (kvm_enabled() && kvmclock_enabled) {
+    if (kvm_enabled() && pcmc->kvmclock_enabled) {
         kvmclock_create();
     }
 
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pci_memory = g_new(MemoryRegion, 1);
         memory_region_init(pci_memory, NULL, "pci", UINT64_MAX);
         rom_memory = pci_memory;
@@ -160,60 +140,54 @@ static void pc_init1(MachineState *machine)
         rom_memory = system_memory;
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
-
-    guest_info->has_acpi_build = has_acpi_build;
-    guest_info->legacy_acpi_table_size = legacy_acpi_table_size;
+    pc_guest_info_init(pcms);
 
-    guest_info->isapc_ram_fw = !pci_enabled;
-    guest_info->has_reserved_memory = has_reserved_memory;
-    guest_info->rsdp_in_ram = rsdp_in_ram;
-
-    if (smbios_defaults) {
+    if (pcmc->smbios_defaults) {
         MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (i440FX + PIIX, 1996)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, pcmc->smbios_legacy_mode,
+                            pcmc->smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        pc_memory_init(machine, system_memory,
-                       below_4g_mem_size, above_4g_mem_size,
-                       rom_memory, &ram_memory, guest_info);
+        pc_memory_init(pcms, system_memory,
+                       rom_memory, &ram_memory);
     } else if (machine->kernel_filename != NULL) {
         /* For xen HVM direct kernel boot, load linux here */
-        xen_load_linux(machine->kernel_filename,
-                       machine->kernel_cmdline,
-                       machine->initrd_filename,
-                       below_4g_mem_size,
-                       guest_info);
+        xen_load_linux(pcms);
     }
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
-    if (kvm_irqchip_in_kernel()) {
-        kvm_pc_setup_irq_routing(pci_enabled);
+    if (kvm_ioapic_in_kernel()) {
+        kvm_pc_setup_irq_routing(pcmc->pci_enabled);
         gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
         gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
     }
 
-    if (pci_enabled) {
-        pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
+    if (pcmc->pci_enabled) {
+        pci_bus = i440fx_init(host_type,
+                              pci_type,
+                              &i440fx_state, &piix3_devfn, &isa_bus, gsi,
                               system_memory, system_io, machine->ram_size,
-                              below_4g_mem_size,
-                              above_4g_mem_size,
+                              pcms->below_4g_mem_size,
+                              pcms->above_4g_mem_size,
                               pci_memory, ram_memory);
+        pcms->bus = pci_bus;
     } else {
         pci_bus = NULL;
         i440fx_state = NULL;
-        isa_bus = isa_bus_new(NULL, get_system_memory(), system_io);
+        isa_bus = isa_bus_new(NULL, get_system_memory(), system_io,
+                              &error_abort);
         no_hpet = 1;
     }
     isa_bus_irqs(isa_bus, gsi);
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_pic_in_kernel()) {
         i8259 = kvm_i8259_init(isa_bus);
     } else if (xen_enabled()) {
         i8259 = xen_interrupt_controller_init();
@@ -225,28 +199,27 @@ static void pc_init1(MachineState *machine)
         gsi_state->i8259_irq[i] = i8259[i];
     }
     g_free(i8259);
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         ioapic_init_gsi(gsi_state, "i440fx");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
-    pc_vga_init(isa_bus, pci_enabled ? pci_bus : NULL);
+    pc_vga_init(isa_bus, pcmc->pci_enabled ? pci_bus : NULL);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO__MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, true,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0x4);
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0x4);
 
     pc_nic_init(isa_bus, pci_bus);
 
     ide_drive_get(hd, ARRAY_SIZE(hd));
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         PCIDevice *dev;
         if (xen_enabled()) {
             dev = pci_piix3_xen_ide_init(pci_bus, hd, piix3_devfn + 1);
@@ -271,14 +244,13 @@ static void pc_init1(MachineState *machine)
         }
     }
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
-    if (pci_enabled && usb_enabled()) {
+    if (pcmc->pci_enabled && usb_enabled()) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
     }
 
-    if (pci_enabled && acpi_enabled) {
+    if (pcmc->pci_enabled && acpi_enabled) {
         DeviceState *piix4_pm;
         I2CBus *smbus;
 
@@ -286,24 +258,36 @@ static void pc_init1(MachineState *machine)
         /* TODO: Populate SPD eeprom data.  */
         smbus = piix4_pm_init(pci_bus, piix3_devfn + 3, 0xb100,
                               gsi[9], smi_irq,
-                              pc_machine_is_smm_enabled(pc_machine),
+                              pc_machine_is_smm_enabled(pcms),
                               &piix4_pm);
         smbus_eeprom_init(smbus, 8, NULL, 0);
 
         object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                                  TYPE_HOTPLUG_HANDLER,
-                                 (Object **)&pc_machine->acpi_dev,
+                                 (Object **)&pcms->acpi_dev,
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
         object_property_set_link(OBJECT(machine), OBJECT(piix4_pm),
                                  PC_MACHINE_ACPI_DEVICE_PROP, &error_abort);
     }
 
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pc_pci_device_init(pci_bus);
     }
+
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_init_acpi_state(&pcms->acpi_nvdimm_state, system_io,
+                               pcms->fw_cfg, OBJECT(pcms));
+    }
 }
 
+/* Looking for a pc_compat_2_4() function? It doesn't exist.
+ * pc_compat_*() functions that run on machine-init time and
+ * change global QEMU state are deprecated. Please don't create
+ * one, and implement any pc-*-2.4 (and newer) compat code in
+ * HW_COMPAT_*, PC_COMPAT_*, or * pc_*_machine_options().
+ */
+
 static void pc_compat_2_3(MachineState *machine)
 {
     PCMachineState *pcms = PC_MACHINE(machine);
@@ -318,80 +302,29 @@ static void pc_compat_2_3(MachineState *machine)
 static void pc_compat_2_2(MachineState *machine)
 {
     pc_compat_2_3(machine);
-    rsdp_in_ram = false;
-    x86_cpu_compat_set_features("kvm64", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("kvm32", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Conroe", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Penryn", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Nehalem", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("SandyBridge", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G1", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G2", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G3", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G4", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G5", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
     machine->suppress_vmdesc = true;
 }
 
 static void pc_compat_2_1(MachineState *machine)
 {
-    PCMachineState *pcms = PC_MACHINE(machine);
-
     pc_compat_2_2(machine);
-    smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    x86_cpu_change_kvm_default("svm", NULL);
 }
 
 static void pc_compat_2_0(MachineState *machine)
 {
     pc_compat_2_1(machine);
-    /* This value depends on the actual DSDT and SSDT compiled into
-     * the source QEMU; unfortunately it depends on the binary and
-     * not on the machine type, so we cannot make pc-i440fx-1.7 work on
-     * both QEMU 1.7 and QEMU 2.0.
-     *
-     * Large variations cause migration to fail for more than one
-     * consecutive value of the "-smp" maxcpus option.
-     *
-     * For small variations of the kind caused by different iasl versions,
-     * the 4k rounding usually leaves slack.  However, there could be still
-     * one or two values that break.  For QEMU 1.7 and QEMU 2.0 the
-     * slack is only ~10 bytes before one "-smp maxcpus" value breaks!
-     *
-     * 6652 is valid for QEMU 2.0, the right value for pc-i440fx-1.7 on
-     * QEMU 1.7 it is 6414.  For RHEL/CentOS 7.0 it is 6418.
-     */
-    legacy_acpi_table_size = 6652;
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    pc_set_legacy_acpi_data_size();
 }
 
 static void pc_compat_1_7(MachineState *machine)
 {
     pc_compat_2_0(machine);
-    smbios_defaults = false;
-    gigabyte_align = false;
-    option_rom_has_mr = true;
-    legacy_acpi_table_size = 6414;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    x86_cpu_change_kvm_default("x2apic", NULL);
 }
 
 static void pc_compat_1_6(MachineState *machine)
 {
     pc_compat_1_7(machine);
-    rom_file_has_mr = false;
-    has_acpi_build = false;
 }
 
 static void pc_compat_1_5(MachineState *machine)
@@ -402,8 +335,6 @@ static void pc_compat_1_5(MachineState *machine)
 static void pc_compat_1_4(MachineState *machine)
 {
     pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
 static void pc_compat_1_3(MachineState *machine)
@@ -416,40 +347,46 @@ static void pc_compat_1_3(MachineState *machine)
 static void pc_compat_1_2(MachineState *machine)
 {
     pc_compat_1_3(machine);
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, 1 << KVM_FEATURE_PV_EOI);
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
 }
 
 /* PC compat function for pc-0.10 to pc-0.13 */
 static void pc_compat_0_13(MachineState *machine)
 {
     pc_compat_1_2(machine);
-    kvmclock_enabled = false;
 }
 
 static void pc_init_isa(MachineState *machine)
 {
-    pci_enabled = false;
-    has_acpi_build = false;
-    smbios_defaults = false;
-    gigabyte_align = false;
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    option_rom_has_mr = true;
-    rom_file_has_mr = false;
     if (!machine->cpu_model) {
         machine->cpu_model = "486";
     }
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, 1 << KVM_FEATURE_PV_EOI);
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
     enable_compat_apic_id_mode();
-    pc_init1(machine);
+    pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, TYPE_I440FX_PCI_DEVICE);
 }
 
 #ifdef CONFIG_XEN
+static void pc_xen_hvm_init_pci(MachineState *machine)
+{
+    const char *pci_type = has_igd_gfx_passthru ?
+                TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE : TYPE_I440FX_PCI_DEVICE;
+
+    pc_init1(machine,
+             TYPE_I440FX_PCI_HOST_BRIDGE,
+             pci_type);
+}
+
 static void pc_xen_hvm_init(MachineState *machine)
 {
     PCIBus *bus;
 
-    pc_init1(machine);
+    if (!xen_enabled()) {
+        error_report("xenfv machine requires the xen accelerator");
+        exit(1);
+    }
+
+    pc_xen_hvm_init_pci(machine);
 
     bus = pci_find_primary_bus();
     if (bus != NULL) {
@@ -465,27 +402,55 @@ static void pc_xen_hvm_init(MachineState *machine)
         if (compat) { \
             compat(machine); \
         } \
-        pc_init1(machine); \
+        pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, \
+                 TYPE_I440FX_PCI_DEVICE); \
     } \
     DEFINE_PC_MACHINE(suffix, name, pc_init_##suffix, optionfn)
 
 static void pc_i440fx_machine_options(MachineClass *m)
 {
-    pc_default_machine_options(m);
     m->family = "pc_piix";
     m->desc = "Standard PC (i440FX + PIIX, 1996)";
     m->hot_add_cpu = pc_hot_add_cpu;
+    m->default_machine_opts = "firmware=bios-256k.bin";
+    m->default_display = "std";
 }
 
-static void pc_i440fx_2_4_machine_options(MachineClass *m)
+static void pc_i440fx_2_6_machine_options(MachineClass *m)
 {
     pc_i440fx_machine_options(m);
-    m->default_machine_opts = "firmware=bios-256k.bin";
-    m->default_display = "std";
     m->alias = "pc";
     m->is_default = 1;
 }
 
+DEFINE_I440FX_MACHINE(v2_6, "pc-i440fx-2.6", NULL,
+                      pc_i440fx_2_6_machine_options);
+
+
+static void pc_i440fx_2_5_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_6_machine_options(m);
+    m->alias = NULL;
+    m->is_default = 0;
+    pcmc->save_tsc_khz = false;
+    m->legacy_fw_cfg_order = 1;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_5);
+}
+
+DEFINE_I440FX_MACHINE(v2_5, "pc-i440fx-2.5", NULL,
+                      pc_i440fx_2_5_machine_options);
+
+
+static void pc_i440fx_2_4_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
+}
+
 DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
                       pc_i440fx_2_4_machine_options)
 
@@ -493,8 +458,7 @@ DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
 static void pc_i440fx_2_3_machine_options(MachineClass *m)
 {
     pc_i440fx_2_4_machine_options(m);
-    m->alias = NULL;
-    m->is_default = 0;
+    m->hw_version = "2.3.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_3);
 }
 
@@ -504,8 +468,11 @@ DEFINE_I440FX_MACHINE(v2_3, "pc-i440fx-2.3", pc_compat_2_3,
 
 static void pc_i440fx_2_2_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_2_3_machine_options(m);
+    m->hw_version = "2.2.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_2);
+    pcmc->rsdp_in_ram = false;
 }
 
 DEFINE_I440FX_MACHINE(v2_2, "pc-i440fx-2.2", pc_compat_2_2,
@@ -514,9 +481,13 @@ DEFINE_I440FX_MACHINE(v2_2, "pc-i440fx-2.2", pc_compat_2_2,
 
 static void pc_i440fx_2_1_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_2_2_machine_options(m);
+    m->hw_version = "2.1.0";
     m->default_display = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_1);
+    pcmc->smbios_uuid_encoded = false;
+    pcmc->enforce_aligned_dimm = false;
 }
 
 DEFINE_I440FX_MACHINE(v2_1, "pc-i440fx-2.1", pc_compat_2_1,
@@ -526,8 +497,30 @@ DEFINE_I440FX_MACHINE(v2_1, "pc-i440fx-2.1", pc_compat_2_1,
 
 static void pc_i440fx_2_0_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_2_1_machine_options(m);
+    m->hw_version = "2.0.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_2_0);
+    pcmc->smbios_legacy_mode = true;
+    pcmc->has_reserved_memory = false;
+    /* This value depends on the actual DSDT and SSDT compiled into
+     * the source QEMU; unfortunately it depends on the binary and
+     * not on the machine type, so we cannot make pc-i440fx-1.7 work on
+     * both QEMU 1.7 and QEMU 2.0.
+     *
+     * Large variations cause migration to fail for more than one
+     * consecutive value of the "-smp" maxcpus option.
+     *
+     * For small variations of the kind caused by different iasl versions,
+     * the 4k rounding usually leaves slack.  However, there could be still
+     * one or two values that break.  For QEMU 1.7 and QEMU 2.0 the
+     * slack is only ~10 bytes before one "-smp maxcpus" value breaks!
+     *
+     * 6652 is valid for QEMU 2.0, the right value for pc-i440fx-1.7 on
+     * QEMU 1.7 it is 6414.  For RHEL/CentOS 7.0 it is 6418.
+     */
+    pcmc->legacy_acpi_table_size = 6652;
+    pcmc->acpi_data_size = 0x10000;
 }
 
 DEFINE_I440FX_MACHINE(v2_0, "pc-i440fx-2.0", pc_compat_2_0,
@@ -536,9 +529,15 @@ DEFINE_I440FX_MACHINE(v2_0, "pc-i440fx-2.0", pc_compat_2_0,
 
 static void pc_i440fx_1_7_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_2_0_machine_options(m);
+    m->hw_version = "1.7.0";
     m->default_machine_opts = NULL;
+    m->option_rom_has_mr = true;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_7);
+    pcmc->smbios_defaults = false;
+    pcmc->gigabyte_align = false;
+    pcmc->legacy_acpi_table_size = 6414;
 }
 
 DEFINE_I440FX_MACHINE(v1_7, "pc-i440fx-1.7", pc_compat_1_7,
@@ -547,8 +546,12 @@ DEFINE_I440FX_MACHINE(v1_7, "pc-i440fx-1.7", pc_compat_1_7,
 
 static void pc_i440fx_1_6_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_1_7_machine_options(m);
+    m->hw_version = "1.6.0";
+    m->rom_file_has_mr = false;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_6);
+    pcmc->has_acpi_build = false;
 }
 
 DEFINE_I440FX_MACHINE(v1_6, "pc-i440fx-1.6", pc_compat_1_6,
@@ -558,6 +561,7 @@ DEFINE_I440FX_MACHINE(v1_6, "pc-i440fx-1.6", pc_compat_1_6,
 static void pc_i440fx_1_5_machine_options(MachineClass *m)
 {
     pc_i440fx_1_6_machine_options(m);
+    m->hw_version = "1.5.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_5);
 }
 
@@ -568,6 +572,7 @@ DEFINE_I440FX_MACHINE(v1_5, "pc-i440fx-1.5", pc_compat_1_5,
 static void pc_i440fx_1_4_machine_options(MachineClass *m)
 {
     pc_i440fx_1_5_machine_options(m);
+    m->hw_version = "1.4.0";
     m->hot_add_cpu = NULL;
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_4);
 }
@@ -600,6 +605,7 @@ DEFINE_I440FX_MACHINE(v1_4, "pc-i440fx-1.4", pc_compat_1_4,
 static void pc_i440fx_1_3_machine_options(MachineClass *m)
 {
     pc_i440fx_1_4_machine_options(m);
+    m->hw_version = "1.3.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_3);
 }
 
@@ -638,6 +644,7 @@ DEFINE_I440FX_MACHINE(v1_3, "pc-1.3", pc_compat_1_3,
 static void pc_i440fx_1_2_machine_options(MachineClass *m)
 {
     pc_i440fx_1_3_machine_options(m);
+    m->hw_version = "1.2.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_2);
 }
 
@@ -680,6 +687,7 @@ DEFINE_I440FX_MACHINE(v1_2, "pc-1.2", pc_compat_1_2,
 static void pc_i440fx_1_1_machine_options(MachineClass *m)
 {
     pc_i440fx_1_2_machine_options(m);
+    m->hw_version = "1.1.0";
     SET_MACHINE_COMPAT(m, PC_COMPAT_1_1);
 }
 
@@ -797,9 +805,11 @@ DEFINE_I440FX_MACHINE(v0_14, "pc-0.14", pc_compat_1_2,
 
 static void pc_i440fx_0_13_machine_options(MachineClass *m)
 {
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     pc_i440fx_0_14_machine_options(m);
     m->hw_version = "0.13";
     SET_MACHINE_COMPAT(m, PC_COMPAT_0_13);
+    pcmc->kvmclock_enabled = false;
 }
 
 DEFINE_I440FX_MACHINE(v0_13, "pc-0.13", pc_compat_0_13,
@@ -906,12 +916,132 @@ static void pc_i440fx_0_10_machine_options(MachineClass *m)
 DEFINE_I440FX_MACHINE(v0_10, "pc-0.10", pc_compat_0_13,
                       pc_i440fx_0_10_machine_options);
 
+typedef struct {
+    uint16_t gpu_device_id;
+    uint16_t pch_device_id;
+    uint8_t pch_revision_id;
+} IGDDeviceIDInfo;
+
+/* In real world different GPU should have different PCH. But actually
+ * the different PCH DIDs likely map to different PCH SKUs. We do the
+ * same thing for the GPU. For PCH, the different SKUs are going to be
+ * all the same silicon design and implementation, just different
+ * features turn on and off with fuses. The SW interfaces should be
+ * consistent across all SKUs in a given family (eg LPT). But just same
+ * features may not be supported.
+ *
+ * Most of these different PCH features probably don't matter to the
+ * Gfx driver, but obviously any difference in display port connections
+ * will so it should be fine with any PCH in case of passthrough.
+ *
+ * So currently use one PCH version, 0x8c4e, to cover all HSW(Haswell)
+ * scenarios, 0x9cc3 for BDW(Broadwell).
+ */
+static const IGDDeviceIDInfo igd_combo_id_infos[] = {
+    /* HSW Classic */
+    {0x0402, 0x8c4e, 0x04}, /* HSWGT1D, HSWD_w7 */
+    {0x0406, 0x8c4e, 0x04}, /* HSWGT1M, HSWM_w7 */
+    {0x0412, 0x8c4e, 0x04}, /* HSWGT2D, HSWD_w7 */
+    {0x0416, 0x8c4e, 0x04}, /* HSWGT2M, HSWM_w7 */
+    {0x041E, 0x8c4e, 0x04}, /* HSWGT15D, HSWD_w7 */
+    /* HSW ULT */
+    {0x0A06, 0x8c4e, 0x04}, /* HSWGT1UT, HSWM_w7 */
+    {0x0A16, 0x8c4e, 0x04}, /* HSWGT2UT, HSWM_w7 */
+    {0x0A26, 0x8c4e, 0x06}, /* HSWGT3UT, HSWM_w7 */
+    {0x0A2E, 0x8c4e, 0x04}, /* HSWGT3UT28W, HSWM_w7 */
+    {0x0A1E, 0x8c4e, 0x04}, /* HSWGT2UX, HSWM_w7 */
+    {0x0A0E, 0x8c4e, 0x04}, /* HSWGT1ULX, HSWM_w7 */
+    /* HSW CRW */
+    {0x0D26, 0x8c4e, 0x04}, /* HSWGT3CW, HSWM_w7 */
+    {0x0D22, 0x8c4e, 0x04}, /* HSWGT3CWDT, HSWD_w7 */
+    /* HSW Server */
+    {0x041A, 0x8c4e, 0x04}, /* HSWSVGT2, HSWD_w7 */
+    /* HSW SRVR */
+    {0x040A, 0x8c4e, 0x04}, /* HSWSVGT1, HSWD_w7 */
+    /* BSW */
+    {0x1606, 0x9cc3, 0x03}, /* BDWULTGT1, BDWM_w7 */
+    {0x1616, 0x9cc3, 0x03}, /* BDWULTGT2, BDWM_w7 */
+    {0x1626, 0x9cc3, 0x03}, /* BDWULTGT3, BDWM_w7 */
+    {0x160E, 0x9cc3, 0x03}, /* BDWULXGT1, BDWM_w7 */
+    {0x161E, 0x9cc3, 0x03}, /* BDWULXGT2, BDWM_w7 */
+    {0x1602, 0x9cc3, 0x03}, /* BDWHALOGT1, BDWM_w7 */
+    {0x1612, 0x9cc3, 0x03}, /* BDWHALOGT2, BDWM_w7 */
+    {0x1622, 0x9cc3, 0x03}, /* BDWHALOGT3, BDWM_w7 */
+    {0x162B, 0x9cc3, 0x03}, /* BDWHALO28W, BDWM_w7 */
+    {0x162A, 0x9cc3, 0x03}, /* BDWGT3WRKS, BDWM_w7 */
+    {0x162D, 0x9cc3, 0x03}, /* BDWGT3SRVR, BDWM_w7 */
+};
+
+static void isa_bridge_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->desc        = "ISA bridge faked to support IGD PT";
+    k->vendor_id    = PCI_VENDOR_ID_INTEL;
+    k->class_id     = PCI_CLASS_BRIDGE_ISA;
+};
+
+static TypeInfo isa_bridge_info = {
+    .name          = "igd-passthrough-isa-bridge",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIDevice),
+    .class_init = isa_bridge_class_init,
+};
+
+static void pt_graphics_register_types(void)
+{
+    type_register_static(&isa_bridge_info);
+}
+type_init(pt_graphics_register_types)
+
+void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id)
+{
+    struct PCIDevice *bridge_dev;
+    int i, num;
+    uint16_t pch_dev_id = 0xffff;
+    uint8_t pch_rev_id;
+
+    num = ARRAY_SIZE(igd_combo_id_infos);
+    for (i = 0; i < num; i++) {
+        if (gpu_dev_id == igd_combo_id_infos[i].gpu_device_id) {
+            pch_dev_id = igd_combo_id_infos[i].pch_device_id;
+            pch_rev_id = igd_combo_id_infos[i].pch_revision_id;
+        }
+    }
+
+    if (pch_dev_id == 0xffff) {
+        return;
+    }
+
+    /* Currently IGD drivers always need to access PCH by 1f.0. */
+    bridge_dev = pci_create_simple(bus, PCI_DEVFN(0x1f, 0),
+                                   "igd-passthrough-isa-bridge");
+
+    /*
+     * Note that vendor id is always PCI_VENDOR_ID_INTEL.
+     */
+    if (!bridge_dev) {
+        fprintf(stderr, "set igd-passthrough-isa-bridge failed!\n");
+        return;
+    }
+    pci_config_set_device_id(bridge_dev->config, pch_dev_id);
+    pci_config_set_revision(bridge_dev->config, pch_rev_id);
+}
 
 static void isapc_machine_options(MachineClass *m)
 {
-    pc_common_machine_options(m);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
     m->desc = "ISA-only PC";
     m->max_cpus = 1;
+    m->option_rom_has_mr = true;
+    m->rom_file_has_mr = false;
+    pcmc->pci_enabled = false;
+    pcmc->has_acpi_build = false;
+    pcmc->smbios_defaults = false;
+    pcmc->gigabyte_align = false;
+    pcmc->smbios_legacy_mode = true;
+    pcmc->has_reserved_memory = false;
 }
 
 DEFINE_PC_MACHINE(isapc, "isapc", pc_init_isa,
@@ -921,7 +1051,6 @@ DEFINE_PC_MACHINE(isapc, "isapc", pc_init_isa,
 #ifdef CONFIG_XEN
 static void xenfv_machine_options(MachineClass *m)
 {
-    pc_common_machine_options(m);
     m->desc = "Xen Fully-virtualized PC";
     m->max_cpus = HVM_MAX_VCPUS;
     m->default_machine_opts = "accel=xen";
diff --git a/qemu/hw/i386/pc_q35.c b/qemu/hw/i386/pc_q35.c
index 974aead5a..04aae8958 100644
--- a/qemu/hw/i386/pc_q35.c
+++ b/qemu/hw/i386/pc_q35.c
@@ -27,6 +27,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "sysemu/arch_init.h"
@@ -38,54 +39,40 @@
 #include "hw/kvm/clock.h"
 #include "hw/pci-host/q35.h"
 #include "exec/address-spaces.h"
+#include "hw/i386/pc.h"
 #include "hw/i386/ich9.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/ide/pci.h"
 #include "hw/ide/ahci.h"
 #include "hw/usb.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/error-report.h"
 #include "migration/migration.h"
 
 /* ICH9 AHCI has 6 ports */
 #define MAX_SATA_PORTS     6
 
-static bool has_acpi_build = true;
-static bool rsdp_in_ram = true;
-static bool smbios_defaults = true;
-static bool smbios_legacy_mode;
-static bool smbios_uuid_encoded = true;
-/* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
- * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
- * pages in the host.
- */
-static bool gigabyte_align = true;
-static bool has_reserved_memory = true;
-
 /* PC hardware initialisation */
 static void pc_q35_init(MachineState *machine)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     Q35PCIHost *q35_host;
     PCIHostState *phb;
     PCIBus *host_bus;
     PCIDevice *lpc;
     BusState *idebus[MAX_SATA_PORTS];
     ISADevice *rtc_state;
+    MemoryRegion *system_io = get_system_io();
     MemoryRegion *pci_memory;
     MemoryRegion *rom_memory;
     MemoryRegion *ram_memory;
     GSIState *gsi_state;
     ISABus *isa_bus;
-    int pci_enabled = 1;
     qemu_irq *gsi;
     qemu_irq *i8259;
     int i;
     ICH9LPCState *ich9_lpc;
     PCIDevice *ahci;
-    DeviceState *icc_bridge;
-    PcGuestInfo *guest_info;
     ram_addr_t lowmem;
     DriveInfo *hd[MAX_SATA_PORTS];
     MachineClass *mc = MACHINE_GET_CLASS(machine);
@@ -96,11 +83,9 @@ static void pc_q35_init(MachineState *machine)
      * If it doesn't, we need to split it in chunks below and above 4G.
      * In any case, try to make sure that guest addresses aligned at
      * 1G boundaries get mapped to host addresses aligned at 1G boundaries.
-     * For old machine types, use whatever split we used historically to avoid
-     * breaking migration.
      */
     if (machine->ram_size >= 0xb0000000) {
-        lowmem = gigabyte_align ? 0x80000000 : 0xb0000000;
+        lowmem = 0x80000000;
     } else {
         lowmem = 0xb0000000;
     }
@@ -108,41 +93,34 @@ static void pc_q35_init(MachineState *machine)
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(qemu limit, user limit).
      */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
+    if (lowmem > pcms->max_ram_below_4g) {
+        lowmem = pcms->max_ram_below_4g;
         if (machine->ram_size - lowmem > lowmem &&
             lowmem & ((1ULL << 30) - 1)) {
             error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
                          ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+                         pcms->max_ram_below_4g);
         }
     }
 
     if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
+        pcms->above_4g_mem_size = machine->ram_size - lowmem;
+        pcms->below_4g_mem_size = lowmem;
     } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = machine->ram_size;
     }
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
-        fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
-        exit(1);
+    if (xen_enabled()) {
+        xen_hvm_init(pcms, &ram_memory);
     }
 
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
-    pc_acpi_init("q35-acpi-dsdt.aml");
+    pc_cpus_init(pcms);
 
     kvmclock_create();
 
     /* pci enabled */
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pci_memory = g_new(MemoryRegion, 1);
         memory_region_init(pci_memory, NULL, "pci", UINT64_MAX);
         rom_memory = pci_memory;
@@ -151,34 +129,26 @@ static void pc_q35_init(MachineState *machine)
         rom_memory = get_system_memory();
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
-    guest_info->isapc_ram_fw = false;
-    guest_info->has_acpi_build = has_acpi_build;
-    guest_info->has_reserved_memory = has_reserved_memory;
-    guest_info->rsdp_in_ram = rsdp_in_ram;
-
-    /* Migration was not supported in 2.0 for Q35, so do not bother
-     * with this hack (see hw/i386/acpi-build.c).
-     */
-    guest_info->legacy_acpi_table_size = 0;
+    pc_guest_info_init(pcms);
 
-    if (smbios_defaults) {
+    if (pcmc->smbios_defaults) {
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (Q35 + ICH9, 2009)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, pcmc->smbios_legacy_mode,
+                            pcmc->smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        pc_memory_init(machine, get_system_memory(),
-                       below_4g_mem_size, above_4g_mem_size,
-                       rom_memory, &ram_memory, guest_info);
+        pc_memory_init(pcms, get_system_memory(),
+                       rom_memory, &ram_memory);
     }
 
     /* irq lines */
     gsi_state = g_malloc0(sizeof(*gsi_state));
-    if (kvm_irqchip_in_kernel()) {
-        kvm_pc_setup_irq_routing(pci_enabled);
+    if (kvm_ioapic_in_kernel()) {
+        kvm_pc_setup_irq_routing(pcmc->pci_enabled);
         gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
@@ -192,14 +162,14 @@ static void pc_q35_init(MachineState *machine)
     q35_host->mch.ram_memory = ram_memory;
     q35_host->mch.pci_address_space = pci_memory;
     q35_host->mch.system_memory = get_system_memory();
-    q35_host->mch.address_space_io = get_system_io();
-    q35_host->mch.below_4g_mem_size = below_4g_mem_size;
-    q35_host->mch.above_4g_mem_size = above_4g_mem_size;
-    q35_host->mch.guest_info = guest_info;
+    q35_host->mch.address_space_io = system_io;
+    q35_host->mch.below_4g_mem_size = pcms->below_4g_mem_size;
+    q35_host->mch.above_4g_mem_size = pcms->above_4g_mem_size;
     /* pci */
     qdev_init_nofail(DEVICE(q35_host));
     phb = PCI_HOST_BRIDGE(q35_host);
     host_bus = phb->bus;
+    pcms->bus = phb->bus;
     /* create ISA bus */
     lpc = pci_create_simple_multifunction(host_bus, PCI_DEVFN(ICH9_LPC_DEV,
                                           ICH9_LPC_FUNC), true,
@@ -207,7 +177,7 @@ static void pc_q35_init(MachineState *machine)
 
     object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                              TYPE_HOTPLUG_HANDLER,
-                             (Object **)&pc_machine->acpi_dev,
+                             (Object **)&pcms->acpi_dev,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
     object_property_set_link(OBJECT(machine), OBJECT(lpc),
@@ -224,7 +194,7 @@ static void pc_q35_init(MachineState *machine)
     /*end early*/
     isa_bus_irqs(isa_bus, gsi);
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_pic_in_kernel()) {
         i8259 = kvm_i8259_init(isa_bus);
     } else if (xen_enabled()) {
         i8259 = xen_interrupt_controller_init();
@@ -235,24 +205,23 @@ static void pc_q35_init(MachineState *machine)
     for (i = 0; i < ISA_NUM_IRQS; i++) {
         gsi_state->i8259_irq[i] = i8259[i];
     }
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         ioapic_init_gsi(gsi_state, "q35");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO__MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, !mc->no_floppy,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0xff0104);
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0xff0104);
 
     /* connect pm stuff to lpc */
-    ich9_lpc_pm_init(lpc, pc_machine_is_smm_enabled(pc_machine), !mc->no_tco);
+    ich9_lpc_pm_init(lpc, pc_machine_is_smm_enabled(pcms));
 
     /* ahci and SATA device, for q35 1 ahci controller is built-in */
     ahci = pci_create_simple_multifunction(host_bus,
@@ -276,99 +245,19 @@ static void pc_q35_init(MachineState *machine)
                                     0xb100),
                       8, NULL, 0);
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
     /* the rest devices to which pci devfn is automatically assigned */
     pc_vga_init(isa_bus, host_bus);
     pc_nic_init(isa_bus, host_bus);
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pc_pci_device_init(host_bus);
     }
-}
 
-static void pc_compat_2_3(MachineState *machine)
-{
-    PCMachineState *pcms = PC_MACHINE(machine);
-    savevm_skip_section_footers();
-    if (kvm_enabled()) {
-        pcms->smm = ON_OFF_AUTO_OFF;
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_init_acpi_state(&pcms->acpi_nvdimm_state, system_io,
+                               pcms->fw_cfg, OBJECT(pcms));
     }
-    global_state_set_optional();
-    savevm_skip_configuration();
-}
-
-static void pc_compat_2_2(MachineState *machine)
-{
-    pc_compat_2_3(machine);
-    rsdp_in_ram = false;
-    x86_cpu_compat_set_features("kvm64", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("kvm32", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Conroe", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Penryn", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Nehalem", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("SandyBridge", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G1", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G2", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G3", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G4", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Opteron_G5", FEAT_1_EDX, 0, CPUID_VME);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Haswell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_F16C);
-    x86_cpu_compat_set_features("Broadwell", FEAT_1_ECX, 0, CPUID_EXT_RDRAND);
-    machine->suppress_vmdesc = true;
-}
-
-static void pc_compat_2_1(MachineState *machine)
-{
-    PCMachineState *pcms = PC_MACHINE(machine);
-
-    pc_compat_2_2(machine);
-    pcms->enforce_aligned_dimm = false;
-    smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-}
-
-static void pc_compat_2_0(MachineState *machine)
-{
-    pc_compat_2_1(machine);
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    pc_set_legacy_acpi_data_size();
-}
-
-static void pc_compat_1_7(MachineState *machine)
-{
-    pc_compat_2_0(machine);
-    smbios_defaults = false;
-    gigabyte_align = false;
-    option_rom_has_mr = true;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
-}
-
-static void pc_compat_1_6(MachineState *machine)
-{
-    pc_compat_1_7(machine);
-    rom_file_has_mr = false;
-    has_acpi_build = false;
-}
-
-static void pc_compat_1_5(MachineState *machine)
-{
-    pc_compat_1_6(machine);
-}
-
-static void pc_compat_1_4(MachineState *machine)
-{
-    pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
 #define DEFINE_Q35_MACHINE(suffix, name, compatfn, optionfn) \
@@ -385,108 +274,45 @@ static void pc_compat_1_4(MachineState *machine)
 
 static void pc_q35_machine_options(MachineClass *m)
 {
-    pc_default_machine_options(m);
     m->family = "pc_q35";
     m->desc = "Standard PC (Q35 + ICH9, 2009)";
     m->hot_add_cpu = pc_hot_add_cpu;
     m->units_per_default_bus = 1;
-}
-
-static void pc_q35_2_4_machine_options(MachineClass *m)
-{
-    pc_q35_machine_options(m);
     m->default_machine_opts = "firmware=bios-256k.bin";
     m->default_display = "std";
     m->no_floppy = 1;
-    m->no_tco = 0;
-    m->alias = "q35";
 }
 
-DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
-                   pc_q35_2_4_machine_options);
-
-
-static void pc_q35_2_3_machine_options(MachineClass *m)
-{
-    pc_q35_2_4_machine_options(m);
-    m->no_floppy = 0;
-    m->no_tco = 1;
-    m->alias = NULL;
-    SET_MACHINE_COMPAT(m, PC_COMPAT_2_3);
-}
-
-DEFINE_Q35_MACHINE(v2_3, "pc-q35-2.3", pc_compat_2_3,
-                   pc_q35_2_3_machine_options);
-
-
-static void pc_q35_2_2_machine_options(MachineClass *m)
-{
-    pc_q35_2_3_machine_options(m);
-    SET_MACHINE_COMPAT(m, PC_COMPAT_2_2);
-}
-
-DEFINE_Q35_MACHINE(v2_2, "pc-q35-2.2", pc_compat_2_2,
-                   pc_q35_2_2_machine_options);
-
-
-static void pc_q35_2_1_machine_options(MachineClass *m)
-{
-    pc_q35_2_2_machine_options(m);
-    m->default_display = NULL;
-    SET_MACHINE_COMPAT(m, PC_COMPAT_2_1);
-}
-
-DEFINE_Q35_MACHINE(v2_1, "pc-q35-2.1", pc_compat_2_1,
-                   pc_q35_2_1_machine_options);
-
-
-static void pc_q35_2_0_machine_options(MachineClass *m)
-{
-    pc_q35_2_1_machine_options(m);
-    SET_MACHINE_COMPAT(m, PC_COMPAT_2_0);
-}
-
-DEFINE_Q35_MACHINE(v2_0, "pc-q35-2.0", pc_compat_2_0,
-                   pc_q35_2_0_machine_options);
-
-
-static void pc_q35_1_7_machine_options(MachineClass *m)
-{
-    pc_q35_2_0_machine_options(m);
-    m->default_machine_opts = NULL;
-    SET_MACHINE_COMPAT(m, PC_COMPAT_1_7);
-}
-
-DEFINE_Q35_MACHINE(v1_7, "pc-q35-1.7", pc_compat_1_7,
-                   pc_q35_1_7_machine_options);
-
-
-static void pc_q35_1_6_machine_options(MachineClass *m)
+static void pc_q35_2_6_machine_options(MachineClass *m)
 {
     pc_q35_machine_options(m);
-    SET_MACHINE_COMPAT(m, PC_COMPAT_1_6);
+    m->alias = "q35";
 }
 
-DEFINE_Q35_MACHINE(v1_6, "pc-q35-1.6", pc_compat_1_6,
-                   pc_q35_1_6_machine_options);
+DEFINE_Q35_MACHINE(v2_6, "pc-q35-2.6", NULL,
+                   pc_q35_2_6_machine_options);
 
-
-static void pc_q35_1_5_machine_options(MachineClass *m)
+static void pc_q35_2_5_machine_options(MachineClass *m)
 {
-    pc_q35_1_6_machine_options(m);
-    SET_MACHINE_COMPAT(m, PC_COMPAT_1_5);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_6_machine_options(m);
+    m->alias = NULL;
+    pcmc->save_tsc_khz = false;
+    m->legacy_fw_cfg_order = 1;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_5);
 }
 
-DEFINE_Q35_MACHINE(v1_5, "pc-q35-1.5", pc_compat_1_5,
-                   pc_q35_1_5_machine_options);
+DEFINE_Q35_MACHINE(v2_5, "pc-q35-2.5", NULL,
+                   pc_q35_2_5_machine_options);
 
-
-static void pc_q35_1_4_machine_options(MachineClass *m)
+static void pc_q35_2_4_machine_options(MachineClass *m)
 {
-    pc_q35_1_5_machine_options(m);
-    m->hot_add_cpu = NULL;
-    SET_MACHINE_COMPAT(m, PC_COMPAT_1_4);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-DEFINE_Q35_MACHINE(v1_4, "pc-q35-1.4", pc_compat_1_4,
-                   pc_q35_1_4_machine_options);
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+                   pc_q35_2_4_machine_options);
diff --git a/qemu/hw/i386/pc_sysfw.c b/qemu/hw/i386/pc_sysfw.c
index 662d99768..f915ad0a3 100644
--- a/qemu/hw/i386/pc_sysfw.c
+++ b/qemu/hw/i386/pc_sysfw.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/block-backend.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
@@ -56,7 +58,7 @@ static void pc_isa_bios_init(MemoryRegion *rom_memory,
     isa_bios_size = MIN(flash_size, 128 * 1024);
     isa_bios = g_malloc(sizeof(*isa_bios));
     memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(isa_bios);
     memory_region_add_subregion_overlap(rom_memory,
                                         0x100000 - isa_bios_size,
@@ -193,7 +195,7 @@ static void old_pc_system_rom_init(MemoryRegion *rom_memory, bool isapc_ram_fw)
         goto bios_error;
     }
     bios = g_malloc(sizeof(*bios));
-    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_abort);
+    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_fatal);
     vmstate_register_ram_global(bios);
     if (!isapc_ram_fw) {
         memory_region_set_readonly(bios, true);
diff --git a/qemu/hw/i386/pci-assign-load-rom.c b/qemu/hw/i386/pci-assign-load-rom.c
new file mode 100644
index 000000000..4bbb08c95
--- /dev/null
+++ b/qemu/hw/i386/pci-assign-load-rom.c
@@ -0,0 +1,85 @@
+/*
+ * This is splited from hw/i386/kvm/pci-assign.c
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "qemu/error-report.h"
+#include "ui/console.h"
+#include "hw/loader.h"
+#include "monitor/monitor.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci-assign.h"
+
+/*
+ * Scan the assigned devices for the devices that have an option ROM, and then
+ * load the corresponding ROM data to RAM. If an error occurs while loading an
+ * option ROM, we just ignore that option ROM and continue with the next one.
+ */
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function)
+{
+    char name[32], rom_file[64];
+    FILE *fp;
+    uint8_t val;
+    struct stat st;
+    void *ptr = NULL;
+
+    /* If loading ROM from file, pci handles it */
+    if (dev->romfile || !dev->rom_bar) {
+        return NULL;
+    }
+
+    snprintf(rom_file, sizeof(rom_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
+             domain, bus, slot, function);
+
+    if (stat(rom_file, &st)) {
+        return NULL;
+    }
+
+    /* Write "1" to the ROM file to enable it */
+    fp = fopen(rom_file, "r+");
+    if (fp == NULL) {
+        error_report("pci-assign: Cannot open %s: %s", rom_file, strerror(errno));
+        return NULL;
+    }
+    val = 1;
+    if (fwrite(&val, 1, 1, fp) != 1) {
+        goto close_rom;
+    }
+    fseek(fp, 0, SEEK_SET);
+
+    snprintf(name, sizeof(name), "%s.rom", object_get_typename(owner));
+    memory_region_init_ram(&dev->rom, owner, name, st.st_size, &error_abort);
+    vmstate_register_ram(&dev->rom, &dev->qdev);
+    ptr = memory_region_get_ram_ptr(&dev->rom);
+    memset(ptr, 0xff, st.st_size);
+
+    if (!fread(ptr, 1, st.st_size, fp)) {
+        error_report("pci-assign: Cannot read from host %s", rom_file);
+        error_printf("Device option ROM contents are probably invalid "
+                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=\n");
+        goto close_rom;
+    }
+
+    pci_register_bar(dev, PCI_ROM_SLOT, 0, &dev->rom);
+    dev->has_rom = true;
+    *size = st.st_size;
+close_rom:
+    /* Write "0" to disable ROM */
+    fseek(fp, 0, SEEK_SET);
+    val = 0;
+    if (!fwrite(&val, 1, 1, fp)) {
+        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
+    }
+    fclose(fp);
+
+    return ptr;
+}
diff --git a/qemu/hw/i386/q35-acpi-dsdt.dsl b/qemu/hw/i386/q35-acpi-dsdt.dsl
deleted file mode 100644
index 16eaca3fa..000000000
--- a/qemu/hw/i386/q35-acpi-dsdt.dsl
+++ /dev/null
@@ -1,435 +0,0 @@
-/*
- * Bochs/QEMU ACPI DSDT ASL definition
- *
- * Copyright (c) 2006 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2 as published by the Free Software Foundation.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- */
-/*
- * Copyright (c) 2010 Isaku Yamahata
- *                    yamahata at valinux co jp
- * Based on acpi-dsdt.dsl, but heavily modified for q35 chipset.
- */
-
-ACPI_EXTRACT_ALL_CODE Q35AcpiDsdtAmlCode
-
-DefinitionBlock (
-    "q35-acpi-dsdt.aml",// Output Filename
-    "DSDT",             // Signature
-    0x01,               // DSDT Compliance Revision
-    "BXPC",             // OEMID
-    "BXDSDT",           // TABLE ID
-    0x2                 // OEM Revision
-    )
-{
-
-#include "acpi-dsdt-dbug.dsl"
-
-    Scope(\_SB) {
-        OperationRegion(PCST, SystemIO, 0xae00, 0x0c)
-        OperationRegion(PCSB, SystemIO, 0xae0c, 0x01)
-        Field(PCSB, AnyAcc, NoLock, WriteAsZeros) {
-            PCIB, 8,
-        }
-    }
-
-
-/****************************************************************
- * PCI Bus definition
- ****************************************************************/
-    Scope(\_SB) {
-        Device(PCI0) {
-            Name(_HID, EisaId("PNP0A08"))
-            Name(_CID, EisaId("PNP0A03"))
-            Name(_ADR, 0x00)
-            Name(_UID, 1)
-
-            External(ISA, DeviceObj)
-
-            // _OSC: based on sample of ACPI3.0b spec
-            Name(SUPP, 0) // PCI _OSC Support Field value
-            Name(CTRL, 0) // PCI _OSC Control Field value
-            Method(_OSC, 4) {
-                // Create DWORD-addressable fields from the Capabilities Buffer
-                CreateDWordField(Arg3, 0, CDW1)
-
-                // Check for proper UUID
-                If (LEqual(Arg0, ToUUID("33DB4D5B-1FF7-401C-9657-7441C03DD766"))) {
-                    // Create DWORD-addressable fields from the Capabilities Buffer
-                    CreateDWordField(Arg3, 4, CDW2)
-                    CreateDWordField(Arg3, 8, CDW3)
-
-                    // Save Capabilities DWORD2 & 3
-                    Store(CDW2, SUPP)
-                    Store(CDW3, CTRL)
-
-                    // Always allow native PME, AER (no dependencies)
-                    // Never allow SHPC (no SHPC controller in this system)
-                    And(CTRL, 0x1D, CTRL)
-
-#if 0 // For now, nothing to do
-                    If (Not(And(CDW1, 1))) { // Query flag clear?
-                        // Disable GPEs for features granted native control.
-                        If (And(CTRL, 0x01)) { // Hot plug control granted?
-                            Store(0, HPCE) // clear the hot plug SCI enable bit
-                            Store(1, HPCS) // clear the hot plug SCI status bit
-                        }
-                        If (And(CTRL, 0x04)) { // PME control granted?
-                            Store(0, PMCE) // clear the PME SCI enable bit
-                            Store(1, PMCS) // clear the PME SCI status bit
-                        }
-                        If (And(CTRL, 0x10)) { // OS restoring PCI Express cap structure?
-                            // Set status to not restore PCI Express cap structure
-                            // upon resume from S3
-                            Store(1, S3CR)
-                        }
-                    }
-#endif
-                    If (LNotEqual(Arg1, One)) {
-                        // Unknown revision
-                        Or(CDW1, 0x08, CDW1)
-                    }
-                    If (LNotEqual(CDW3, CTRL)) {
-                        // Capabilities bits were masked
-                        Or(CDW1, 0x10, CDW1)
-                    }
-                    // Update DWORD3 in the buffer
-                    Store(CTRL, CDW3)
-                } Else {
-                    Or(CDW1, 4, CDW1) // Unrecognized UUID
-                }
-                Return (Arg3)
-            }
-        }
-    }
-
-#include "acpi-dsdt-hpet.dsl"
-
-
-/****************************************************************
- * LPC ISA bridge
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        /* PCI D31:f0 LPC ISA bridge */
-        Device(ISA) {
-            Name (_ADR, 0x001F0000)  // _ADR: Address
-
-            /* ICH9 PCI to ISA irq remapping */
-            OperationRegion(PIRQ, PCI_Config, 0x60, 0x0C)
-
-            OperationRegion(LPCD, PCI_Config, 0x80, 0x2)
-            Field(LPCD, AnyAcc, NoLock, Preserve) {
-                COMA,   3,
-                    ,   1,
-                COMB,   3,
-
-                Offset(0x01),
-                LPTD,   2,
-                    ,   2,
-                FDCD,   2
-            }
-            OperationRegion(LPCE, PCI_Config, 0x82, 0x2)
-            Field(LPCE, AnyAcc, NoLock, Preserve) {
-                CAEN,   1,
-                CBEN,   1,
-                LPEN,   1,
-                FDEN,   1
-            }
-        }
-    }
-
-#include "acpi-dsdt-isa.dsl"
-
-
-/****************************************************************
- * PCI IRQs
- ****************************************************************/
-
-    /* Zero => PIC mode, One => APIC Mode */
-    Name(\PICF, Zero)
-    Method(\_PIC, 1, NotSerialized) {
-        Store(Arg0, \PICF)
-    }
-
-    Scope(\_SB) {
-        Scope(PCI0) {
-#define prt_slot_lnk(nr, lnk0, lnk1, lnk2, lnk3)  \
-    Package() { nr##ffff, 0, lnk0, 0 },           \
-    Package() { nr##ffff, 1, lnk1, 0 },           \
-    Package() { nr##ffff, 2, lnk2, 0 },           \
-    Package() { nr##ffff, 3, lnk3, 0 }
-
-#define prt_slot_lnkA(nr) prt_slot_lnk(nr, LNKA, LNKB, LNKC, LNKD)
-#define prt_slot_lnkB(nr) prt_slot_lnk(nr, LNKB, LNKC, LNKD, LNKA)
-#define prt_slot_lnkC(nr) prt_slot_lnk(nr, LNKC, LNKD, LNKA, LNKB)
-#define prt_slot_lnkD(nr) prt_slot_lnk(nr, LNKD, LNKA, LNKB, LNKC)
-
-#define prt_slot_lnkE(nr) prt_slot_lnk(nr, LNKE, LNKF, LNKG, LNKH)
-#define prt_slot_lnkF(nr) prt_slot_lnk(nr, LNKF, LNKG, LNKH, LNKE)
-#define prt_slot_lnkG(nr) prt_slot_lnk(nr, LNKG, LNKH, LNKE, LNKF)
-#define prt_slot_lnkH(nr) prt_slot_lnk(nr, LNKH, LNKE, LNKF, LNKG)
-
-            Name(PRTP, package() {
-                prt_slot_lnkE(0x0000),
-                prt_slot_lnkF(0x0001),
-                prt_slot_lnkG(0x0002),
-                prt_slot_lnkH(0x0003),
-                prt_slot_lnkE(0x0004),
-                prt_slot_lnkF(0x0005),
-                prt_slot_lnkG(0x0006),
-                prt_slot_lnkH(0x0007),
-                prt_slot_lnkE(0x0008),
-                prt_slot_lnkF(0x0009),
-                prt_slot_lnkG(0x000a),
-                prt_slot_lnkH(0x000b),
-                prt_slot_lnkE(0x000c),
-                prt_slot_lnkF(0x000d),
-                prt_slot_lnkG(0x000e),
-                prt_slot_lnkH(0x000f),
-                prt_slot_lnkE(0x0010),
-                prt_slot_lnkF(0x0011),
-                prt_slot_lnkG(0x0012),
-                prt_slot_lnkH(0x0013),
-                prt_slot_lnkE(0x0014),
-                prt_slot_lnkF(0x0015),
-                prt_slot_lnkG(0x0016),
-                prt_slot_lnkH(0x0017),
-                prt_slot_lnkE(0x0018),
-
-                /* INTA -> PIRQA for slot 25 - 31
-                   see the default value of D<N>IR */
-                prt_slot_lnkA(0x0019),
-                prt_slot_lnkA(0x001a),
-                prt_slot_lnkA(0x001b),
-                prt_slot_lnkA(0x001c),
-                prt_slot_lnkA(0x001d),
-
-                /* PCIe->PCI bridge. use PIRQ[E-H] */
-                prt_slot_lnkE(0x001e),
-
-                prt_slot_lnkA(0x001f)
-            })
-
-#define prt_slot_gsi(nr, gsi0, gsi1, gsi2, gsi3)  \
-    Package() { nr##ffff, 0, gsi0, 0 },           \
-    Package() { nr##ffff, 1, gsi1, 0 },           \
-    Package() { nr##ffff, 2, gsi2, 0 },           \
-    Package() { nr##ffff, 3, gsi3, 0 }
-
-#define prt_slot_gsiA(nr) prt_slot_gsi(nr, GSIA, GSIB, GSIC, GSID)
-#define prt_slot_gsiB(nr) prt_slot_gsi(nr, GSIB, GSIC, GSID, GSIA)
-#define prt_slot_gsiC(nr) prt_slot_gsi(nr, GSIC, GSID, GSIA, GSIB)
-#define prt_slot_gsiD(nr) prt_slot_gsi(nr, GSID, GSIA, GSIB, GSIC)
-
-#define prt_slot_gsiE(nr) prt_slot_gsi(nr, GSIE, GSIF, GSIG, GSIH)
-#define prt_slot_gsiF(nr) prt_slot_gsi(nr, GSIF, GSIG, GSIH, GSIE)
-#define prt_slot_gsiG(nr) prt_slot_gsi(nr, GSIG, GSIH, GSIE, GSIF)
-#define prt_slot_gsiH(nr) prt_slot_gsi(nr, GSIH, GSIE, GSIF, GSIG)
-
-            Name(PRTA, package() {
-                prt_slot_gsiE(0x0000),
-                prt_slot_gsiF(0x0001),
-                prt_slot_gsiG(0x0002),
-                prt_slot_gsiH(0x0003),
-                prt_slot_gsiE(0x0004),
-                prt_slot_gsiF(0x0005),
-                prt_slot_gsiG(0x0006),
-                prt_slot_gsiH(0x0007),
-                prt_slot_gsiE(0x0008),
-                prt_slot_gsiF(0x0009),
-                prt_slot_gsiG(0x000a),
-                prt_slot_gsiH(0x000b),
-                prt_slot_gsiE(0x000c),
-                prt_slot_gsiF(0x000d),
-                prt_slot_gsiG(0x000e),
-                prt_slot_gsiH(0x000f),
-                prt_slot_gsiE(0x0010),
-                prt_slot_gsiF(0x0011),
-                prt_slot_gsiG(0x0012),
-                prt_slot_gsiH(0x0013),
-                prt_slot_gsiE(0x0014),
-                prt_slot_gsiF(0x0015),
-                prt_slot_gsiG(0x0016),
-                prt_slot_gsiH(0x0017),
-                prt_slot_gsiE(0x0018),
-
-                /* INTA -> PIRQA for slot 25 - 31, but 30
-                   see the default value of D<N>IR */
-                prt_slot_gsiA(0x0019),
-                prt_slot_gsiA(0x001a),
-                prt_slot_gsiA(0x001b),
-                prt_slot_gsiA(0x001c),
-                prt_slot_gsiA(0x001d),
-
-                /* PCIe->PCI bridge. use PIRQ[E-H] */
-                prt_slot_gsiE(0x001e),
-
-                prt_slot_gsiA(0x001f)
-            })
-
-            Method(_PRT, 0, NotSerialized) {
-                /* PCI IRQ routing table, example from ACPI 2.0a specification,
-                   section 6.2.8.1 */
-                /* Note: we provide the same info as the PCI routing
-                   table of the Bochs BIOS */
-                If (LEqual(\PICF, Zero)) {
-                    Return (PRTP)
-                } Else {
-                    Return (PRTA)
-                }
-            }
-        }
-
-        Field(PCI0.ISA.PIRQ, ByteAcc, NoLock, Preserve) {
-            PRQA,   8,
-            PRQB,   8,
-            PRQC,   8,
-            PRQD,   8,
-
-            Offset(0x08),
-            PRQE,   8,
-            PRQF,   8,
-            PRQG,   8,
-            PRQH,   8
-        }
-
-        Method(IQST, 1, NotSerialized) {
-            // _STA method - get status
-            If (And(0x80, Arg0)) {
-                Return (0x09)
-            }
-            Return (0x0B)
-        }
-        Method(IQCR, 1, Serialized) {
-            // _CRS method - get current settings
-            Name(PRR0, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 0 }
-            })
-            CreateDWordField(PRR0, 0x05, PRRI)
-            Store(And(Arg0, 0x0F), PRRI)
-            Return (PRR0)
-        }
-
-#define define_link(link, uid, reg)                             \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    5, 10, 11                                   \
-                }                                               \
-            })                                                  \
-            Method(_STA, 0, NotSerialized) {                    \
-                Return (IQST(reg))                              \
-            }                                                   \
-            Method(_DIS, 0, NotSerialized) {                    \
-                Or(reg, 0x80, reg)                              \
-            }                                                   \
-            Method(_CRS, 0, NotSerialized) {                    \
-                Return (IQCR(reg))                              \
-            }                                                   \
-            Method(_SRS, 1, NotSerialized) {                    \
-                CreateDWordField(Arg0, 0x05, PRRI)              \
-                Store(PRRI, reg)                                \
-            }                                                   \
-        }
-
-        define_link(LNKA, 0, PRQA)
-        define_link(LNKB, 1, PRQB)
-        define_link(LNKC, 2, PRQC)
-        define_link(LNKD, 3, PRQD)
-        define_link(LNKE, 4, PRQE)
-        define_link(LNKF, 5, PRQF)
-        define_link(LNKG, 6, PRQG)
-        define_link(LNKH, 7, PRQH)
-
-#define define_gsi_link(link, uid, gsi)                         \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    gsi                                         \
-                }                                               \
-            })                                                  \
-            Name(_CRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    gsi                                         \
-                }                                               \
-            })                                                  \
-            Method(_SRS, 1, NotSerialized) {                    \
-            }                                                   \
-        }
-
-        define_gsi_link(GSIA, 0, 0x10)
-        define_gsi_link(GSIB, 0, 0x11)
-        define_gsi_link(GSIC, 0, 0x12)
-        define_gsi_link(GSID, 0, 0x13)
-        define_gsi_link(GSIE, 0, 0x14)
-        define_gsi_link(GSIF, 0, 0x15)
-        define_gsi_link(GSIG, 0, 0x16)
-        define_gsi_link(GSIH, 0, 0x17)
-    }
-
-#include "hw/acpi/pc-hotplug.h"
-#define CPU_STATUS_BASE ICH9_CPU_HOTPLUG_IO_BASE
-#include "acpi-dsdt-cpu-hotplug.dsl"
-#include "acpi-dsdt-mem-hotplug.dsl"
-
-
-/****************************************************************
- * General purpose events
- ****************************************************************/
-    Scope(\_GPE) {
-        Name(_HID, "ACPI0006")
-
-        Method(_L00) {
-        }
-        Method(_L01) {
-        }
-        Method(_E02) {
-            // CPU hotplug event
-            \_SB.PRSC()
-        }
-        Method(_E03) {
-            // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
-        }
-        Method(_L04) {
-        }
-        Method(_L05) {
-        }
-        Method(_L06) {
-        }
-        Method(_L07) {
-        }
-        Method(_L08) {
-        }
-        Method(_L09) {
-        }
-        Method(_L0A) {
-        }
-        Method(_L0B) {
-        }
-        Method(_L0C) {
-        }
-        Method(_L0D) {
-        }
-        Method(_L0E) {
-        }
-        Method(_L0F) {
-        }
-    }
-}
diff --git a/qemu/hw/i386/q35-acpi-dsdt.hex.generated b/qemu/hw/i386/q35-acpi-dsdt.hex.generated
deleted file mode 100644
index ed9a2cc8e..000000000
--- a/qemu/hw/i386/q35-acpi-dsdt.hex.generated
+++ /dev/null
@@ -1,7610 +0,0 @@
-static unsigned char Q35AcpiDsdtAmlCode[] = {
-0x44,
-0x53,
-0x44,
-0x54,
-0xb8,
-0x1d,
-0x0,
-0x0,
-0x1,
-0x35,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x44,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x2,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x7,
-0x11,
-0x14,
-0x20,
-0x10,
-0x49,
-0x4,
-0x5c,
-0x0,
-0x5b,
-0x80,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0xb,
-0x2,
-0x4,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0x44,
-0x42,
-0x47,
-0x42,
-0x8,
-0x14,
-0x2c,
-0x44,
-0x42,
-0x55,
-0x47,
-0x1,
-0x98,
-0x68,
-0x60,
-0x96,
-0x60,
-0x60,
-0x74,
-0x87,
-0x60,
-0x1,
-0x61,
-0x70,
-0x0,
-0x62,
-0xa2,
-0x10,
-0x95,
-0x62,
-0x61,
-0x70,
-0x83,
-0x88,
-0x60,
-0x62,
-0x0,
-0x44,
-0x42,
-0x47,
-0x42,
-0x75,
-0x62,
-0x70,
-0xa,
-0xa,
-0x44,
-0x42,
-0x47,
-0x42,
-0x10,
-0x29,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x54,
-0x1,
-0xb,
-0x0,
-0xae,
-0xa,
-0xc,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x42,
-0x1,
-0xb,
-0xc,
-0xae,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x50,
-0x43,
-0x53,
-0x42,
-0x40,
-0x50,
-0x43,
-0x49,
-0x42,
-0x8,
-0x10,
-0x4f,
-0xc,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x47,
-0xc,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x8,
-0x8,
-0x5f,
-0x43,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x53,
-0x55,
-0x50,
-0x50,
-0x0,
-0x8,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x0,
-0x14,
-0x44,
-0x9,
-0x5f,
-0x4f,
-0x53,
-0x43,
-0x4,
-0x8a,
-0x6b,
-0x0,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa0,
-0x46,
-0x7,
-0x93,
-0x68,
-0x11,
-0x13,
-0xa,
-0x10,
-0x5b,
-0x4d,
-0xdb,
-0x33,
-0xf7,
-0x1f,
-0x1c,
-0x40,
-0x96,
-0x57,
-0x74,
-0x41,
-0xc0,
-0x3d,
-0xd7,
-0x66,
-0x8a,
-0x6b,
-0xa,
-0x4,
-0x43,
-0x44,
-0x57,
-0x32,
-0x8a,
-0x6b,
-0xa,
-0x8,
-0x43,
-0x44,
-0x57,
-0x33,
-0x70,
-0x43,
-0x44,
-0x57,
-0x32,
-0x53,
-0x55,
-0x50,
-0x50,
-0x70,
-0x43,
-0x44,
-0x57,
-0x33,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x7b,
-0x43,
-0x54,
-0x52,
-0x4c,
-0xa,
-0x1d,
-0x43,
-0x54,
-0x52,
-0x4c,
-0xa0,
-0x10,
-0x92,
-0x93,
-0x69,
-0x1,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x8,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa0,
-0x16,
-0x92,
-0x93,
-0x43,
-0x44,
-0x57,
-0x33,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x10,
-0x43,
-0x44,
-0x57,
-0x31,
-0x70,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x43,
-0x44,
-0x57,
-0x33,
-0xa1,
-0xc,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x4,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa4,
-0x6b,
-0x10,
-0x4d,
-0x8,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x45,
-0x8,
-0x48,
-0x50,
-0x45,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x1,
-0x3,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x0,
-0xc,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0xb,
-0x0,
-0x4,
-0x5b,
-0x81,
-0x10,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x13,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x20,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x20,
-0x14,
-0x36,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x60,
-0x70,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x61,
-0x7a,
-0x60,
-0xa,
-0x10,
-0x60,
-0xa0,
-0xc,
-0x91,
-0x93,
-0x60,
-0x0,
-0x93,
-0x60,
-0xb,
-0xff,
-0xff,
-0xa4,
-0x0,
-0xa0,
-0xe,
-0x91,
-0x93,
-0x61,
-0x0,
-0x94,
-0x61,
-0xc,
-0x0,
-0xe1,
-0xf5,
-0x5,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x11,
-0xa,
-0xe,
-0x86,
-0x9,
-0x0,
-0x0,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0x0,
-0x4,
-0x0,
-0x0,
-0x79,
-0x0,
-0x10,
-0x4c,
-0x7,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x4f,
-0x6,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0x1f,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x49,
-0x52,
-0x51,
-0x2,
-0xa,
-0x60,
-0xa,
-0xc,
-0x5b,
-0x80,
-0x4c,
-0x50,
-0x43,
-0x44,
-0x2,
-0xa,
-0x80,
-0xa,
-0x2,
-0x5b,
-0x81,
-0x20,
-0x4c,
-0x50,
-0x43,
-0x44,
-0x0,
-0x43,
-0x4f,
-0x4d,
-0x41,
-0x3,
-0x0,
-0x1,
-0x43,
-0x4f,
-0x4d,
-0x42,
-0x3,
-0x0,
-0x1,
-0x4c,
-0x50,
-0x54,
-0x44,
-0x2,
-0x0,
-0x2,
-0x46,
-0x44,
-0x43,
-0x44,
-0x2,
-0x5b,
-0x80,
-0x4c,
-0x50,
-0x43,
-0x45,
-0x2,
-0xa,
-0x82,
-0xa,
-0x2,
-0x5b,
-0x81,
-0x1a,
-0x4c,
-0x50,
-0x43,
-0x45,
-0x0,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x1,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x1,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x1,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x1,
-0x10,
-0x4c,
-0x1b,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x5b,
-0x82,
-0x2d,
-0x52,
-0x54,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xb,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x70,
-0x0,
-0x70,
-0x0,
-0x10,
-0x2,
-0x22,
-0x0,
-0x1,
-0x47,
-0x1,
-0x72,
-0x0,
-0x72,
-0x0,
-0x2,
-0x6,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x37,
-0x4b,
-0x42,
-0x44,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x3,
-0x3,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x60,
-0x0,
-0x60,
-0x0,
-0x1,
-0x1,
-0x47,
-0x1,
-0x64,
-0x0,
-0x64,
-0x0,
-0x1,
-0x1,
-0x22,
-0x2,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x27,
-0x4d,
-0x4f,
-0x55,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xf,
-0x13,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x8,
-0xa,
-0x5,
-0x22,
-0x0,
-0x10,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4a,
-0x4,
-0x46,
-0x44,
-0x43,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x7,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1b,
-0xa,
-0x18,
-0x47,
-0x1,
-0xf2,
-0x3,
-0xf2,
-0x3,
-0x0,
-0x4,
-0x47,
-0x1,
-0xf7,
-0x3,
-0xf7,
-0x3,
-0x0,
-0x1,
-0x22,
-0x40,
-0x0,
-0x2a,
-0x4,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x3e,
-0x4c,
-0x50,
-0x54,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x4,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x78,
-0x3,
-0x78,
-0x3,
-0x8,
-0x8,
-0x22,
-0x80,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x31,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x3,
-0xf8,
-0x3,
-0x0,
-0x8,
-0x22,
-0x10,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x46,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x32,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x2,
-0xf8,
-0x2,
-0x0,
-0x8,
-0x22,
-0x8,
-0x0,
-0x79,
-0x0,
-0x8,
-0x50,
-0x49,
-0x43,
-0x46,
-0x0,
-0x14,
-0xc,
-0x5f,
-0x50,
-0x49,
-0x43,
-0x1,
-0x70,
-0x68,
-0x50,
-0x49,
-0x43,
-0x46,
-0x10,
-0x8e,
-0x55,
-0x1,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x10,
-0x43,
-0xea,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x50,
-0x52,
-0x54,
-0x50,
-0x12,
-0x4b,
-0x73,
-0x80,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x8,
-0x50,
-0x52,
-0x54,
-0x41,
-0x12,
-0x4b,
-0x73,
-0x80,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x14,
-0x1a,
-0x5f,
-0x50,
-0x52,
-0x54,
-0x0,
-0xa0,
-0xc,
-0x93,
-0x50,
-0x49,
-0x43,
-0x46,
-0x0,
-0xa4,
-0x50,
-0x52,
-0x54,
-0x50,
-0xa1,
-0x6,
-0xa4,
-0x50,
-0x52,
-0x54,
-0x41,
-0x5b,
-0x81,
-0x3a,
-0x2f,
-0x3,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x50,
-0x49,
-0x52,
-0x51,
-0x1,
-0x50,
-0x52,
-0x51,
-0x41,
-0x8,
-0x50,
-0x52,
-0x51,
-0x42,
-0x8,
-0x50,
-0x52,
-0x51,
-0x43,
-0x8,
-0x50,
-0x52,
-0x51,
-0x44,
-0x8,
-0x0,
-0x20,
-0x50,
-0x52,
-0x51,
-0x45,
-0x8,
-0x50,
-0x52,
-0x51,
-0x46,
-0x8,
-0x50,
-0x52,
-0x51,
-0x47,
-0x8,
-0x50,
-0x52,
-0x51,
-0x48,
-0x8,
-0x14,
-0x13,
-0x49,
-0x51,
-0x53,
-0x54,
-0x1,
-0xa0,
-0x9,
-0x7b,
-0xa,
-0x80,
-0x68,
-0x0,
-0xa4,
-0xa,
-0x9,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x34,
-0x49,
-0x51,
-0x43,
-0x52,
-0x9,
-0x8,
-0x50,
-0x52,
-0x52,
-0x30,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8a,
-0x50,
-0x52,
-0x52,
-0x30,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x7b,
-0x68,
-0xa,
-0xf,
-0x0,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa4,
-0x50,
-0x52,
-0x52,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x41,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x41,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x42,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x42,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x43,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x43,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x44,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x44,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x4,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x45,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x45,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x5,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x46,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x46,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x6,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x47,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x47,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x7,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x48,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x48,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x10,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x10,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x11,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x11,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x12,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x12,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x13,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x13,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x14,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x14,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x46,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x15,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x15,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x47,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x16,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x16,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x48,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x17,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x17,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x10,
-0x4d,
-0xc,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x14,
-0x35,
-0x43,
-0x50,
-0x4d,
-0x41,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0x70,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x61,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x2,
-0x0,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x3,
-0x0,
-0x70,
-0x60,
-0x88,
-0x61,
-0xa,
-0x4,
-0x0,
-0xa4,
-0x61,
-0x14,
-0x1a,
-0x43,
-0x50,
-0x53,
-0x54,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0xa0,
-0x5,
-0x60,
-0xa4,
-0xa,
-0xf,
-0xa1,
-0x3,
-0xa4,
-0x0,
-0x14,
-0xa,
-0x43,
-0x50,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x22,
-0xa,
-0xc8,
-0x14,
-0x4a,
-0x6,
-0x50,
-0x52,
-0x53,
-0x43,
-0x0,
-0x70,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x65,
-0x70,
-0x0,
-0x62,
-0x70,
-0x0,
-0x60,
-0xa2,
-0x46,
-0x5,
-0x95,
-0x60,
-0x87,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0x61,
-0xa0,
-0xa,
-0x7b,
-0x60,
-0xa,
-0x7,
-0x0,
-0x7a,
-0x62,
-0x1,
-0x62,
-0xa1,
-0xc,
-0x70,
-0x83,
-0x88,
-0x65,
-0x7a,
-0x60,
-0xa,
-0x3,
-0x0,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x62,
-0x1,
-0x0,
-0x63,
-0xa0,
-0x22,
-0x92,
-0x93,
-0x61,
-0x63,
-0x70,
-0x63,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0xa0,
-0xa,
-0x93,
-0x63,
-0x1,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0xa1,
-0x8,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0xa,
-0x3,
-0x75,
-0x60,
-0x10,
-0x44,
-0x2a,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x47,
-0x29,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x50,
-0x4e,
-0x50,
-0x30,
-0x41,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x4d,
-0x65,
-0x6d,
-0x6f,
-0x72,
-0x79,
-0x20,
-0x68,
-0x6f,
-0x74,
-0x70,
-0x6c,
-0x75,
-0x67,
-0x20,
-0x72,
-0x65,
-0x73,
-0x6f,
-0x75,
-0x72,
-0x63,
-0x65,
-0x73,
-0x0,
-0x14,
-0x13,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x5b,
-0x1,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x14,
-0x4a,
-0x4,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0xa2,
-0x25,
-0x95,
-0x60,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x70,
-0x60,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0x13,
-0x93,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x4d,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0x70,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x72,
-0x60,
-0x1,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x1,
-0x14,
-0x2d,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x1,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0xb,
-0x93,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x70,
-0xa,
-0xf,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x41,
-0x18,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x9,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x8,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x12,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x2a,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x1a,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x14,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x11,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0xa0,
-0x44,
-0x7,
-0x93,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x0,
-0x8,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xa,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xe,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0x16,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x70,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x14,
-0x24,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x1,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x28,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x4,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x69,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x70,
-0x6a,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x10,
-0x42,
-0xa,
-0x5f,
-0x47,
-0x50,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x41,
-0x43,
-0x50,
-0x49,
-0x30,
-0x30,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x30,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x31,
-0x0,
-0x14,
-0x10,
-0x5f,
-0x45,
-0x30,
-0x32,
-0x0,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x43,
-0x14,
-0x19,
-0x5f,
-0x45,
-0x30,
-0x33,
-0x0,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x34,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x35,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x37,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x38,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x39,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x41,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x42,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x43,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x44,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x45,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x46,
-0x0
-};
diff --git a/qemu/hw/i386/xen/xen_apic.c b/qemu/hw/i386/xen/xen_apic.c
index f5acd6a09..21d68ee04 100644
--- a/qemu/hw/i386/xen/xen_apic.c
+++ b/qemu/hw/i386/xen/xen_apic.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL version 2 or
  * later. See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/pci/msi.h"
 #include "hw/xen/xen.h"
@@ -43,11 +44,7 @@ static void xen_apic_realize(DeviceState *dev, Error **errp)
     s->vapic_control = 0;
     memory_region_init_io(&s->io_memory, OBJECT(s), &xen_apic_io_ops, s,
                           "xen-apic-msi", APIC_SPACE_SIZE);
-
-#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
-    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
-    msi_supported = true;
-#endif
+    msi_nonbroken = true;
 }
 
 static void xen_apic_set_base(APICCommonState *s, uint64_t val)
diff --git a/qemu/hw/i386/xen/xen_platform.c b/qemu/hw/i386/xen/xen_platform.c
index 28b324a6f..aa7839324 100644
--- a/qemu/hw/i386/xen/xen_platform.c
+++ b/qemu/hw/i386/xen/xen_platform.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#include <assert.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/ide.h"
@@ -35,6 +35,7 @@
 #include "trace.h"
 #include "exec/address-spaces.h"
 #include "sysemu/block-backend.h"
+#include "qemu/error-report.h"
 
 #include <xenguest.h>
 
@@ -384,11 +385,17 @@ static const VMStateDescription vmstate_xen_platform = {
     }
 };
 
-static int xen_platform_initfn(PCIDevice *dev)
+static void xen_platform_realize(PCIDevice *dev, Error **errp)
 {
     PCIXenPlatformState *d = XEN_PLATFORM(dev);
     uint8_t *pci_conf;
 
+    /* Device will crash on reset if xen is not initialized */
+    if (!xen_enabled()) {
+        error_setg(errp, "xen-platform device requires the Xen accelerator");
+        return;
+    }
+
     pci_conf = dev->config;
 
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
@@ -406,8 +413,6 @@ static int xen_platform_initfn(PCIDevice *dev)
                      &d->mmio_bar);
 
     platform_fixed_ioport_init(d);
-
-    return 0;
 }
 
 static void platform_reset(DeviceState *dev)
@@ -422,7 +427,7 @@ static void xen_platform_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_platform_initfn;
+    k->realize = xen_platform_realize;
     k->vendor_id = PCI_VENDOR_ID_XEN;
     k->device_id = PCI_DEVICE_ID_XEN_PLATFORM;
     k->class_id = PCI_CLASS_OTHERS << 8 | 0x80;
diff --git a/qemu/hw/i386/xen/xen_pvdevice.c b/qemu/hw/i386/xen/xen_pvdevice.c
index c2189473b..c093b3445 100644
--- a/qemu/hw/i386/xen/xen_pvdevice.c
+++ b/qemu/hw/i386/xen/xen_pvdevice.c
@@ -29,6 +29,8 @@
  * SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "trace.h"
@@ -69,14 +71,16 @@ static const MemoryRegionOps xen_pv_mmio_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int xen_pv_init(PCIDevice *pci_dev)
+static void xen_pv_realize(PCIDevice *pci_dev, Error **errp)
 {
     XenPVDevice *d = XEN_PV_DEVICE(pci_dev);
     uint8_t *pci_conf;
 
     /* device-id property must always be supplied */
-    if (d->device_id == 0xffff)
-	    return -1;
+    if (d->device_id == 0xffff) {
+        error_setg(errp, "Device ID invalid, it must always be supplied");
+        return;
+    }
 
     pci_conf = pci_dev->config;
 
@@ -97,8 +101,6 @@ static int xen_pv_init(PCIDevice *pci_dev)
 
     pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_MEM_PREFETCH,
                      &d->mmio);
-
-    return 0;
 }
 
 static Property xen_pv_props[] = {
@@ -114,7 +116,7 @@ static void xen_pv_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_pv_init;
+    k->realize = xen_pv_realize;
     k->class_id = PCI_CLASS_SYSTEM_OTHER;
     dc->desc = "Xen PV Device";
     dc->props = xen_pv_props;
diff --git a/qemu/hw/ide/ahci.c b/qemu/hw/ide/ahci.c
index 48749c1dc..f244bc01c 100644
--- a/qemu/hw/ide/ahci.c
+++ b/qemu/hw/ide/ahci.c
@@ -21,11 +21,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/pci/msi.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
-#include <hw/sysbus.h>
 
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
@@ -47,10 +47,9 @@ do { \
 static void check_cmd(AHCIState *s, int port);
 static int handle_cmd(AHCIState *s, int port, uint8_t slot);
 static void ahci_reset_port(AHCIState *s, int port);
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis);
+static bool ahci_write_fis_d2h(AHCIDevice *ad);
 static void ahci_init_d2h(AHCIDevice *ad);
 static int ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit);
-static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes);
 static bool ahci_map_clb_address(AHCIDevice *ad);
 static bool ahci_map_fis_address(AHCIDevice *ad);
 static void ahci_unmap_clb_address(AHCIDevice *ad);
@@ -122,9 +121,9 @@ static uint32_t  ahci_port_read(AHCIState *s, int port, int offset)
 
 static void ahci_irq_raise(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "raise irq\n");
 
@@ -137,9 +136,9 @@ static void ahci_irq_raise(AHCIState *s, AHCIDevice *dev)
 
 static void ahci_irq_lower(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "lower irq\n");
 
@@ -200,52 +199,38 @@ static void map_page(AddressSpace *as, uint8_t **ptr, uint64_t addr,
  * Check the cmd register to see if we should start or stop
  * the DMA or FIS RX engines.
  *
- * @ad: Device to engage.
- * @allow_stop: Allow device to transition from started to stopped?
- *   'no' is useful for migration post_load, which does not expect a transition.
+ * @ad: Device to dis/engage.
  *
  * @return 0 on success, -1 on error.
  */
-static int ahci_cond_start_engines(AHCIDevice *ad, bool allow_stop)
+static int ahci_cond_start_engines(AHCIDevice *ad)
 {
     AHCIPortRegs *pr = &ad->port_regs;
-
-    if (pr->cmd & PORT_CMD_START) {
-        if (ahci_map_clb_address(ad)) {
-            pr->cmd |= PORT_CMD_LIST_ON;
-        } else {
+    bool cmd_start = pr->cmd & PORT_CMD_START;
+    bool cmd_on    = pr->cmd & PORT_CMD_LIST_ON;
+    bool fis_start = pr->cmd & PORT_CMD_FIS_RX;
+    bool fis_on    = pr->cmd & PORT_CMD_FIS_ON;
+
+    if (cmd_start && !cmd_on) {
+        if (!ahci_map_clb_address(ad)) {
+            pr->cmd &= ~PORT_CMD_START;
             error_report("AHCI: Failed to start DMA engine: "
                          "bad command list buffer address");
             return -1;
         }
-    } else if (pr->cmd & PORT_CMD_LIST_ON) {
-        if (allow_stop) {
-            ahci_unmap_clb_address(ad);
-            pr->cmd = pr->cmd & ~(PORT_CMD_LIST_ON);
-        } else {
-            error_report("AHCI: DMA engine should be off, "
-                         "but appears to still be running");
-            return -1;
-        }
+    } else if (!cmd_start && cmd_on) {
+        ahci_unmap_clb_address(ad);
     }
 
-    if (pr->cmd & PORT_CMD_FIS_RX) {
-        if (ahci_map_fis_address(ad)) {
-            pr->cmd |= PORT_CMD_FIS_ON;
-        } else {
+    if (fis_start && !fis_on) {
+        if (!ahci_map_fis_address(ad)) {
+            pr->cmd &= ~PORT_CMD_FIS_RX;
             error_report("AHCI: Failed to start FIS receive engine: "
                          "bad FIS receive buffer address");
             return -1;
         }
-    } else if (pr->cmd & PORT_CMD_FIS_ON) {
-        if (allow_stop) {
-            ahci_unmap_fis_address(ad);
-            pr->cmd = pr->cmd & ~(PORT_CMD_FIS_ON);
-        } else {
-            error_report("AHCI: FIS receive engine should be off, "
-                         "but appears to still be running");
-            return -1;
-        }
+    } else if (!fis_start && fis_on) {
+        ahci_unmap_fis_address(ad);
     }
 
     return 0;
@@ -287,8 +272,8 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             pr->cmd = (pr->cmd & PORT_CMD_RO_MASK) |
                       (val & ~(PORT_CMD_RO_MASK|PORT_CMD_ICC_MASK));
 
-            /* Check FIS RX and CLB engines, allow transition to false: */
-            ahci_cond_start_engines(&s->dev[port], true);
+            /* Check FIS RX and CLB engines */
+            ahci_cond_start_engines(&s->dev[port]);
 
             /* XXX usually the FIS would be pending on the bus here and
                    issuing deferred until the OS enables FIS receival.
@@ -297,7 +282,6 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
             if ((pr->cmd & PORT_CMD_FIS_ON) &&
                 !s->dev[port].init_d2h_sent) {
                 ahci_init_d2h(&s->dev[port]);
-                s->dev[port].init_d2h_sent = true;
             }
 
             check_cmd(s, port);
@@ -381,17 +365,23 @@ static uint64_t ahci_mem_read(void *opaque, hwaddr addr, unsigned size)
     int ofst = addr - aligned;
     uint64_t lo = ahci_mem_read_32(opaque, aligned);
     uint64_t hi;
+    uint64_t val;
 
     /* if < 8 byte read does not cross 4 byte boundary */
     if (ofst + size <= 4) {
-        return lo >> (ofst * 8);
+        val = lo >> (ofst * 8);
+    } else {
+        g_assert_cmpint(size, >, 1);
+
+        /* If the 64bit read is unaligned, we will produce undefined
+         * results. AHCI does not support unaligned 64bit reads. */
+        hi = ahci_mem_read_32(opaque, aligned + 4);
+        val = (hi << 32 | lo) >> (ofst * 8);
     }
-    g_assert_cmpint(size, >, 1);
 
-    /* If the 64bit read is unaligned, we will produce undefined
-     * results. AHCI does not support unaligned 64bit reads. */
-    hi = ahci_mem_read_32(opaque, aligned + 4);
-    return (hi << 32 | lo) >> (ofst * 8);
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return val;
 }
 
 
@@ -400,6 +390,9 @@ static void ahci_mem_write(void *opaque, hwaddr addr,
 {
     AHCIState *s = opaque;
 
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+
     /* Only aligned reads are allowed on AHCI */
     if (addr & 3) {
         fprintf(stderr, "ahci: Mis-aligned write to addr 0x"
@@ -540,20 +533,33 @@ static void ahci_check_cmd_bh(void *opaque)
 
 static void ahci_init_d2h(AHCIDevice *ad)
 {
-    uint8_t init_fis[20];
     IDEState *ide_state = &ad->port.ifs[0];
+    AHCIPortRegs *pr = &ad->port_regs;
 
-    memset(init_fis, 0, sizeof(init_fis));
-
-    init_fis[4] = 1;
-    init_fis[12] = 1;
+    if (ad->init_d2h_sent) {
+        return;
+    }
 
-    if (ide_state->drive_kind == IDE_CD) {
-        init_fis[5] = ide_state->lcyl;
-        init_fis[6] = ide_state->hcyl;
+    if (ahci_write_fis_d2h(ad)) {
+        ad->init_d2h_sent = true;
+        /* We're emulating receiving the first Reg H2D Fis from the device;
+         * Update the SIG register, but otherwise proceed as normal. */
+        pr->sig = ((uint32_t)ide_state->hcyl << 24) |
+            (ide_state->lcyl << 16) |
+            (ide_state->sector << 8) |
+            (ide_state->nsector & 0xFF);
     }
+}
 
-    ahci_write_fis_d2h(ad, init_fis);
+static void ahci_set_signature(AHCIDevice *ad, uint32_t sig)
+{
+    IDEState *s = &ad->port.ifs[0];
+    s->hcyl = sig >> 24 & 0xFF;
+    s->lcyl = sig >> 16 & 0xFF;
+    s->sector = sig >> 8 & 0xFF;
+    s->nsector = sig & 0xFF;
+
+    DPRINTF(ad->port_no, "set hcyl:lcyl:sect:nsect = 0x%08x\n", sig);
 }
 
 static void ahci_reset_port(AHCIState *s, int port)
@@ -604,17 +610,11 @@ static void ahci_reset_port(AHCIState *s, int port)
     }
 
     s->dev[port].port_state = STATE_RUN;
-    if (!ide_state->blk) {
-        pr->sig = 0;
-        ide_state->status = SEEK_STAT | WRERR_STAT;
-    } else if (ide_state->drive_kind == IDE_CD) {
-        pr->sig = SATA_SIGNATURE_CDROM;
-        ide_state->lcyl = 0x14;
-        ide_state->hcyl = 0xeb;
-        DPRINTF(port, "set lcyl = %d\n", ide_state->lcyl);
+    if (ide_state->drive_kind == IDE_CD) {
+        ahci_set_signature(d, SATA_SIGNATURE_CDROM);\
         ide_state->status = SEEK_STAT | WRERR_STAT | READY_STAT;
     } else {
-        pr->sig = SATA_SIGNATURE_DISK;
+        ahci_set_signature(d, SATA_SIGNATURE_DISK);
         ide_state->status = SEEK_STAT | WRERR_STAT;
     }
 
@@ -643,11 +643,22 @@ static bool ahci_map_fis_address(AHCIDevice *ad)
     AHCIPortRegs *pr = &ad->port_regs;
     map_page(ad->hba->as, &ad->res_fis,
              ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
-    return ad->res_fis != NULL;
+    if (ad->res_fis != NULL) {
+        pr->cmd |= PORT_CMD_FIS_ON;
+        return true;
+    }
+
+    pr->cmd &= ~PORT_CMD_FIS_ON;
+    return false;
 }
 
 static void ahci_unmap_fis_address(AHCIDevice *ad)
 {
+    if (ad->res_fis == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL FIS address\n");
+        return;
+    }
+    ad->port_regs.cmd &= ~PORT_CMD_FIS_ON;
     dma_memory_unmap(ad->hba->as, ad->res_fis, 256,
                      DMA_DIRECTION_FROM_DEVICE, 256);
     ad->res_fis = NULL;
@@ -659,11 +670,22 @@ static bool ahci_map_clb_address(AHCIDevice *ad)
     ad->cur_cmd = NULL;
     map_page(ad->hba->as, &ad->lst,
              ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
-    return ad->lst != NULL;
+    if (ad->lst != NULL) {
+        pr->cmd |= PORT_CMD_LIST_ON;
+        return true;
+    }
+
+    pr->cmd &= ~PORT_CMD_LIST_ON;
+    return false;
 }
 
 static void ahci_unmap_clb_address(AHCIDevice *ad)
 {
+    if (ad->lst == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL CLB address\n");
+        return;
+    }
+    ad->port_regs.cmd &= ~PORT_CMD_LIST_ON;
     dma_memory_unmap(ad->hba->as, ad->lst, 1024,
                      DMA_DIRECTION_FROM_DEVICE, 1024);
     ad->lst = NULL;
@@ -750,7 +772,7 @@ static void ahci_write_fis_pio(AHCIDevice *ad, uint16_t len)
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_PIOS_FIS);
 }
 
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
+static bool ahci_write_fis_d2h(AHCIDevice *ad)
 {
     AHCIPortRegs *pr = &ad->port_regs;
     uint8_t *d2h_fis;
@@ -758,7 +780,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     IDEState *s = &ad->port.ifs[0];
 
     if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
-        return;
+        return false;
     }
 
     d2h_fis = &ad->res_fis[RES_FIS_RFIS];
@@ -791,6 +813,7 @@ static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
     }
 
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_D2H_REG_FIS);
+    return true;
 }
 
 static int prdt_tbl_entry_size(const AHCI_SG *tbl)
@@ -799,8 +822,21 @@ static int prdt_tbl_entry_size(const AHCI_SG *tbl)
     return (le32_to_cpu(tbl->flags_size) & AHCI_PRDT_SIZE_MASK) + 1;
 }
 
+/**
+ * Fetch entries in a guest-provided PRDT and convert it into a QEMU SGlist.
+ * @ad: The AHCIDevice for whom we are building the SGList.
+ * @sglist: The SGList target to add PRD entries to.
+ * @cmd: The AHCI Command Header that describes where the PRDT is.
+ * @limit: The remaining size of the S/ATA transaction, in bytes.
+ * @offset: The number of bytes already transferred, in bytes.
+ *
+ * The AHCI PRDT can describe up to 256GiB. S/ATA only support transactions of
+ * up to 32MiB as of ATA8-ACS3 rev 1b, assuming a 512 byte sector size. We stop
+ * building the sglist from the PRDT as soon as we hit @limit bytes,
+ * which is <= INT32_MAX/2GiB.
+ */
 static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
-                                AHCICmdHdr *cmd, int64_t limit, int32_t offset)
+                                AHCICmdHdr *cmd, int64_t limit, uint64_t offset)
 {
     uint16_t opts = le16_to_cpu(cmd->opts);
     uint16_t prdtl = le16_to_cpu(cmd->prdtl);
@@ -818,14 +854,6 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
     IDEBus *bus = &ad->port;
     BusState *qbus = BUS(bus);
 
-    /*
-     * Note: AHCI PRDT can describe up to 256GiB. SATA/ATA only support
-     * transactions of up to 32MiB as of ATA8-ACS3 rev 1b, assuming a
-     * 512 byte sector size. We limit the PRDT in this implementation to
-     * a reasonably large 2GiB, which can accommodate the maximum transfer
-     * request for sector sizes up to 32K.
-     */
-
     if (!prdtl) {
         DPRINTF(ad->port_no, "no sg list given by guest: 0x%08x\n", opts);
         return -1;
@@ -875,13 +903,6 @@ static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
             qemu_sglist_add(sglist, le64_to_cpu(tbl[i].addr),
                             MIN(prdt_tbl_entry_size(&tbl[i]),
                                 limit - sglist->size));
-            if (sglist->size > INT32_MAX) {
-                error_report("AHCI Physical Region Descriptor Table describes "
-                             "more than 2 GiB.\n");
-                qemu_sglist_destroy(sglist);
-                r = -1;
-                goto out;
-            }
         }
     }
 
@@ -898,6 +919,7 @@ static void ncq_err(NCQTransferState *ncq_tfs)
     ide_state->error = ABRT_ERR;
     ide_state->status = READY_STAT | ERR_STAT;
     ncq_tfs->drive->port_regs.scr_err |= (1 << ncq_tfs->tag);
+    ncq_tfs->used = 0;
 }
 
 static void ncq_finish(NCQTransferState *ncq_tfs)
@@ -1290,7 +1312,7 @@ out:
     s->data_ptr = s->data_end;
 
     /* Update number of transferred bytes, destroy sglist */
-    ahci_commit_buf(dma, size);
+    dma_buf_commit(s, size);
 
     s->end_transfer_func(s);
 
@@ -1332,9 +1354,8 @@ static void ahci_restart(IDEDMA *dma)
 }
 
 /**
- * Called in DMA R/W chains to read the PRDT, utilizing ahci_populate_sglist.
- * Not currently invoked by PIO R/W chains,
- * which invoke ahci_populate_sglist via ahci_start_transfer.
+ * Called in DMA and PIO R/W chains to read the PRDT.
+ * Not shared with NCQ pathways.
  */
 static int32_t ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit)
 {
@@ -1353,21 +1374,16 @@ static int32_t ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit)
 }
 
 /**
- * Destroys the scatter-gather list,
- * and updates the command header with a bytes-read value.
- * called explicitly via ahci_dma_rw_buf (ATAPI DMA),
- * and ahci_start_transfer (PIO R/W),
- * and called via callback from ide_dma_cb for DMA R/W paths.
+ * Updates the command header with a bytes-read value.
+ * Called via dma_buf_commit, for both DMA and PIO paths.
+ * sglist destruction is handled within dma_buf_commit.
  */
 static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-    IDEState *s = &ad->port.ifs[0];
 
     tx_bytes += le32_to_cpu(ad->cur_cmd->status);
     ad->cur_cmd->status = cpu_to_le32(tx_bytes);
-
-    qemu_sglist_destroy(&s->sg);
 }
 
 static int ahci_dma_rw_buf(IDEDMA *dma, int is_write)
@@ -1388,10 +1404,9 @@ static int ahci_dma_rw_buf(IDEDMA *dma, int is_write)
     }
 
     /* free sglist, update byte count */
-    ahci_commit_buf(dma, l);
+    dma_buf_commit(s, l);
 
     s->io_buffer_index += l;
-    s->io_buffer_offset += l;
 
     DPRINTF(ad->port_no, "len=%#x\n", l);
 
@@ -1405,7 +1420,7 @@ static void ahci_cmd_done(IDEDMA *dma)
     DPRINTF(ad->port_no, "cmd done\n");
 
     /* update d2h status */
-    ahci_write_fis_d2h(ad, NULL);
+    ahci_write_fis_d2h(ad);
 
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
@@ -1429,7 +1444,17 @@ static const IDEDMAOps ahci_dma_ops = {
     .cmd_done = ahci_cmd_done,
 };
 
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
+void ahci_init(AHCIState *s, DeviceState *qdev)
+{
+    s->container = qdev;
+    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
+    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
+                          "ahci", AHCI_MEM_BAR_SIZE);
+    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
+                          "ahci-idp", 32);
+}
+
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
 {
     qemu_irq *irqs;
     int i;
@@ -1438,14 +1463,7 @@ void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
     s->ports = ports;
     s->dev = g_new0(AHCIDevice, ports);
     ahci_reg_init(s);
-    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
-    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
-                          "ahci", AHCI_MEM_BAR_SIZE);
-    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
-                          "ahci-idp", 32);
-
     irqs = qemu_allocate_irqs(ahci_irq_set, s, s->ports);
-
     for (i = 0; i < s->ports; i++) {
         AHCIDevice *ad = &s->dev[i];
 
@@ -1542,14 +1560,28 @@ static int ahci_state_post_load(void *opaque, int version_id)
     int i, j;
     struct AHCIDevice *ad;
     NCQTransferState *ncq_tfs;
+    AHCIPortRegs *pr;
     AHCIState *s = opaque;
 
     for (i = 0; i < s->ports; i++) {
         ad = &s->dev[i];
+        pr = &ad->port_regs;
+
+        if (!(pr->cmd & PORT_CMD_START) && (pr->cmd & PORT_CMD_LIST_ON)) {
+            error_report("AHCI: DMA engine should be off, but status bit "
+                         "indicates it is still running.");
+            return -1;
+        }
+        if (!(pr->cmd & PORT_CMD_FIS_RX) && (pr->cmd & PORT_CMD_FIS_ON)) {
+            error_report("AHCI: FIS RX engine should be off, but status bit "
+                         "indicates it is still running.");
+            return -1;
+        }
 
-        /* Only remap the CLB address if appropriate, disallowing a state
-         * transition from 'on' to 'off' it should be consistent here. */
-        if (ahci_cond_start_engines(ad, false) != 0) {
+        /* After a migrate, the DMA/FIS engines are "off" and
+         * need to be conditionally restarted */
+        pr->cmd &= ~(PORT_CMD_LIST_ON | PORT_CMD_FIS_ON);
+        if (ahci_cond_start_engines(ad) != 0) {
             return -1;
         }
 
@@ -1625,18 +1657,6 @@ const VMStateDescription vmstate_ahci = {
     },
 };
 
-#define TYPE_SYSBUS_AHCI "sysbus-ahci"
-#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
-
-typedef struct SysbusAHCIState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    AHCIState ahci;
-    uint32_t num_ports;
-} SysbusAHCIState;
-
 static const VMStateDescription vmstate_sysbus_ahci = {
     .name = "sysbus-ahci",
     .fields = (VMStateField[]) {
@@ -1652,17 +1672,24 @@ static void sysbus_ahci_reset(DeviceState *dev)
     ahci_reset(&s->ahci);
 }
 
-static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+static void sysbus_ahci_init(Object *obj)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
-    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    ahci_init(&s->ahci, dev, &address_space_memory, s->num_ports);
+    ahci_init(&s->ahci, DEVICE(obj));
 
     sysbus_init_mmio(sbd, &s->ahci.mem);
     sysbus_init_irq(sbd, &s->ahci.irq);
 }
 
+static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+
+    ahci_realize(&s->ahci, dev, &address_space_memory, s->num_ports);
+}
+
 static Property sysbus_ahci_properties[] = {
     DEFINE_PROP_UINT32("num-ports", SysbusAHCIState, num_ports, 1),
     DEFINE_PROP_END_OF_LIST(),
@@ -1683,12 +1710,108 @@ static const TypeInfo sysbus_ahci_info = {
     .name          = TYPE_SYSBUS_AHCI,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysbusAHCIState),
+    .instance_init = sysbus_ahci_init,
     .class_init    = sysbus_ahci_class_init,
 };
 
+#define ALLWINNER_AHCI_BISTAFR    ((0xa0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTCR     ((0xa4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTFCTR   ((0xa8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTSR     ((0xac - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTDECR   ((0xb0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR0    ((0xb4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR1    ((0xb8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_OOBR       ((0xbc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS0R    ((0xc0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS1R    ((0xc4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS2R    ((0xc8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TIMER1MS   ((0xe0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM1R   ((0xe8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM2R   ((0xec - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PPARAMR    ((0xf0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TESTR      ((0xf4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_VERSIONR   ((0xf8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_IDR        ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_RWCR       ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+
+static uint64_t allwinner_ahci_mem_read(void *opaque, hwaddr addr,
+                                        unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+    uint64_t val = a->regs[addr/4];
+
+    switch (addr / 4) {
+    case ALLWINNER_AHCI_PHYCS0R:
+        val |= 0x2 << 28;
+        break;
+    case ALLWINNER_AHCI_PHYCS2R:
+        val &= ~(0x1 << 24);
+        break;
+    }
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return  val;
+}
+
+static void allwinner_ahci_mem_write(void *opaque, hwaddr addr,
+                                     uint64_t val, unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    a->regs[addr/4] = val;
+}
+
+static const MemoryRegionOps allwinner_ahci_mem_ops = {
+    .read = allwinner_ahci_mem_read,
+    .write = allwinner_ahci_mem_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void allwinner_ahci_init(Object *obj)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    AllwinnerAHCIState *a = ALLWINNER_AHCI(obj);
+
+    memory_region_init_io(&a->mmio, OBJECT(obj), &allwinner_ahci_mem_ops, a,
+                          "allwinner-ahci", ALLWINNER_AHCI_MMIO_SIZE);
+    memory_region_add_subregion(&s->ahci.mem, ALLWINNER_AHCI_MMIO_OFF,
+                                &a->mmio);
+}
+
+static const VMStateDescription vmstate_allwinner_ahci = {
+    .name = "allwinner-ahci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, AllwinnerAHCIState,
+                             ALLWINNER_AHCI_MMIO_SIZE/4),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void allwinner_ahci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_allwinner_ahci;
+}
+
+static const TypeInfo allwinner_ahci_info = {
+    .name          = TYPE_ALLWINNER_AHCI,
+    .parent        = TYPE_SYSBUS_AHCI,
+    .instance_size = sizeof(AllwinnerAHCIState),
+    .instance_init = allwinner_ahci_init,
+    .class_init    = allwinner_ahci_class_init,
+};
+
 static void sysbus_ahci_register_types(void)
 {
     type_register_static(&sysbus_ahci_info);
+    type_register_static(&allwinner_ahci_info);
 }
 
 type_init(sysbus_ahci_register_types)
diff --git a/qemu/hw/ide/ahci.h b/qemu/hw/ide/ahci.h
index 79a463d93..bc777ed5c 100644
--- a/qemu/hw/ide/ahci.h
+++ b/qemu/hw/ide/ahci.h
@@ -24,6 +24,8 @@
 #ifndef HW_IDE_AHCI_H
 #define HW_IDE_AHCI_H
 
+#include <hw/sysbus.h>
+
 #define AHCI_MEM_BAR_SIZE         0x1000
 #define AHCI_MAX_PORTS            32
 #define AHCI_MAX_SG               168 /* hardware max is 64K */
@@ -285,6 +287,8 @@ struct AHCIDevice {
 };
 
 typedef struct AHCIState {
+    DeviceState *container;
+
     AHCIDevice *dev;
     AHCIControlRegs control_regs;
     MemoryRegion mem;
@@ -362,11 +366,40 @@ typedef struct SDBFIS {
     uint32_t payload;
 } QEMU_PACKED SDBFIS;
 
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
+void ahci_init(AHCIState *s, DeviceState *qdev);
 void ahci_uninit(AHCIState *s);
 
 void ahci_reset(AHCIState *s);
 
 void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd);
 
+#define TYPE_SYSBUS_AHCI "sysbus-ahci"
+#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
+
+typedef struct SysbusAHCIState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    AHCIState ahci;
+    uint32_t num_ports;
+} SysbusAHCIState;
+
+#define TYPE_ALLWINNER_AHCI "allwinner-ahci"
+#define ALLWINNER_AHCI(obj) OBJECT_CHECK(AllwinnerAHCIState, (obj), \
+                       TYPE_ALLWINNER_AHCI)
+
+#define ALLWINNER_AHCI_MMIO_OFF  0x80
+#define ALLWINNER_AHCI_MMIO_SIZE 0x80
+
+struct AllwinnerAHCIState {
+    /*< private >*/
+    SysbusAHCIState parent_obj;
+    /*< public >*/
+
+    MemoryRegion mmio;
+    uint32_t regs[ALLWINNER_AHCI_MMIO_SIZE/4];
+};
+
 #endif /* HW_IDE_AHCI_H */
diff --git a/qemu/hw/ide/atapi.c b/qemu/hw/ide/atapi.c
index 79dd16710..2bb606c1c 100644
--- a/qemu/hw/ide/atapi.c
+++ b/qemu/hw/ide/atapi.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ide/internal.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/block-backend.h"
@@ -105,33 +106,99 @@ static void cd_data_to_raw(uint8_t *buf, int lba)
     memset(buf, 0, 288);
 }
 
-static int cd_read_sector(IDEState *s, int lba, uint8_t *buf, int sector_size)
+static int
+cd_read_sector_sync(IDEState *s)
 {
     int ret;
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
 
-    switch(sector_size) {
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_sync: lba=%d\n", s->lba);
+#endif
+
+    switch (s->cd_sector_size) {
     case 2048:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        ret = blk_read(s->blk, (int64_t)s->lba << 2,
+                       s->io_buffer, 4);
         break;
     case 2352:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf + 16, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
-        if (ret < 0)
-            return ret;
-        cd_data_to_raw(buf, lba);
+        ret = blk_read(s->blk, (int64_t)s->lba << 2,
+                       s->io_buffer + 16, 4);
+        if (ret >= 0) {
+            cd_data_to_raw(s->io_buffer, s->lba);
+        }
         break;
     default:
-        ret = -EIO;
-        break;
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EIO;
+    }
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        s->lba++;
+        s->io_buffer_index = 0;
     }
+
     return ret;
 }
 
+static void cd_read_sector_cb(void *opaque, int ret)
+{
+    IDEState *s = opaque;
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_cb: lba=%d ret=%d\n", s->lba, ret);
+#endif
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        ide_atapi_io_error(s, ret);
+        return;
+    }
+
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
+    if (s->cd_sector_size == 2352) {
+        cd_data_to_raw(s->io_buffer, s->lba);
+    }
+
+    s->lba++;
+    s->io_buffer_index = 0;
+    s->status &= ~BUSY_STAT;
+
+    ide_atapi_cmd_reply_end(s);
+}
+
+static int cd_read_sector(IDEState *s)
+{
+    if (s->cd_sector_size != 2048 && s->cd_sector_size != 2352) {
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EINVAL;
+    }
+
+    s->iov.iov_base = (s->cd_sector_size == 2352) ?
+                      s->io_buffer + 16 : s->io_buffer;
+
+    s->iov.iov_len = 4 * BDRV_SECTOR_SIZE;
+    qemu_iovec_init_external(&s->qiov, &s->iov, 1);
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector: lba=%d\n", s->lba);
+#endif
+
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+
+    ide_buffered_readv(s, (int64_t)s->lba << 2, &s->qiov, 4,
+                       cd_read_sector_cb, s);
+
+    s->status |= BUSY_STAT;
+    return 0;
+}
+
 void ide_atapi_cmd_ok(IDEState *s)
 {
     s->error = 0;
@@ -167,6 +234,17 @@ void ide_atapi_io_error(IDEState *s, int ret)
     }
 }
 
+static uint16_t atapi_byte_count_limit(IDEState *s)
+{
+    uint16_t bcl;
+
+    bcl = s->lcyl | (s->hcyl << 8);
+    if (bcl == 0xffff) {
+        return 0xfffe;
+    }
+    return bcl;
+}
+
 /* The whole ATAPI transfer logic is handled in this function */
 void ide_atapi_cmd_reply_end(IDEState *s)
 {
@@ -182,18 +260,27 @@ void ide_atapi_cmd_reply_end(IDEState *s)
         ide_atapi_cmd_ok(s);
         ide_set_irq(s->bus);
 #ifdef DEBUG_IDE_ATAPI
-        printf("status=0x%x\n", s->status);
+        printf("end of transfer, status=0x%x\n", s->status);
 #endif
     } else {
         /* see if a new sector must be read */
         if (s->lba != -1 && s->io_buffer_index >= s->cd_sector_size) {
-            ret = cd_read_sector(s, s->lba, s->io_buffer, s->cd_sector_size);
-            if (ret < 0) {
-                ide_atapi_io_error(s, ret);
+            if (!s->elementary_transfer_size) {
+                ret = cd_read_sector(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                }
                 return;
+            } else {
+                /* rebuffering within an elementary transfer is
+                 * only possible with a sync request because we
+                 * end up with a race condition otherwise */
+                ret = cd_read_sector_sync(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                    return;
+                }
             }
-            s->lba++;
-            s->io_buffer_index = 0;
         }
         if (s->elementary_transfer_size > 0) {
             /* there are some data left to transmit in this elementary
@@ -209,12 +296,10 @@ void ide_atapi_cmd_reply_end(IDEState *s)
         } else {
             /* a new transfer is needed */
             s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO;
-            byte_count_limit = s->lcyl | (s->hcyl << 8);
+            byte_count_limit = atapi_byte_count_limit(s);
 #ifdef DEBUG_IDE_ATAPI
             printf("byte_count_limit=%d\n", byte_count_limit);
 #endif
-            if (byte_count_limit == 0xffff)
-                byte_count_limit--;
             size = s->packet_transfer_size;
             if (size > byte_count_limit) {
                 /* byte count limit must be even if this case */
@@ -275,7 +360,6 @@ static void ide_atapi_cmd_read_pio(IDEState *s, int lba, int nb_sectors,
     s->io_buffer_index = sector_size;
     s->cd_sector_size = sector_size;
 
-    s->status = READY_STAT | SEEK_STAT;
     ide_atapi_cmd_reply_end(s);
 }
 
@@ -291,15 +375,18 @@ static void ide_atapi_cmd_check_status(IDEState *s)
 }
 /* ATAPI DMA support */
 
-/* XXX: handle read errors */
 static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret)
 {
     IDEState *s = opaque;
     int data_offset, n;
 
     if (ret < 0) {
-        ide_atapi_io_error(s, ret);
-        goto eot;
+        if (ide_handle_rw_error(s, -ret, ide_dma_cmd_to_retry(s->dma_cmd))) {
+            if (s->bus->error_status) {
+                return;
+            }
+            goto eot;
+        }
     }
 
     if (s->io_buffer_size > 0) {
@@ -351,13 +438,17 @@ static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret)
     s->bus->dma->iov.iov_len = n * 4 * 512;
     qemu_iovec_init_external(&s->bus->dma->qiov, &s->bus->dma->iov, 1);
 
-    s->bus->dma->aiocb = blk_aio_readv(s->blk, (int64_t)s->lba << 2,
-                                       &s->bus->dma->qiov, n * 4,
-                                       ide_atapi_cmd_read_dma_cb, s);
+    s->bus->dma->aiocb = ide_buffered_readv(s, (int64_t)s->lba << 2,
+                                            &s->bus->dma->qiov, n * 4,
+                                            ide_atapi_cmd_read_dma_cb, s);
     return;
 
 eot:
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
     ide_set_inactive(s, false);
 }
 
@@ -393,21 +484,16 @@ static void ide_atapi_cmd_read(IDEState *s, int lba, int nb_sectors,
     }
 }
 
-
-/* Called by *_restart_bh when the transfer function points
- * to ide_atapi_cmd
- */
 void ide_atapi_dma_restart(IDEState *s)
 {
     /*
-     * I'm not sure we have enough stored to restart the command
-     * safely, so give the guest an error it should recover from.
-     * I'm assuming most guests will try to recover from something
-     * listed as a medium error on a CD; it seems to work on Linux.
-     * This would be more of a problem if we did any other type of
-     * DMA operation.
+     * At this point we can just re-evaluate the packet command and start over.
+     * The presence of ->dma_cb callback in the pre_save ensures that the packet
+     * command has been completely sent and we can safely restart command.
      */
-    ide_atapi_cmd_error(s, MEDIUM_ERROR, ASC_NO_SEEK_COMPLETE);
+    s->unit = s->bus->retry_unit;
+    s->bus->dma->ops->restart_dma(s->bus->dma);
+    ide_atapi_cmd(s);
 }
 
 static inline uint8_t ide_atapi_set_profile(uint8_t *buf, uint8_t *index,
@@ -737,7 +823,6 @@ static void cmd_inquiry(IDEState *s, uint8_t *buf)
  out:
     buf[size_idx] = idx - preamble_len;
     ide_atapi_cmd_reply(s, idx, max_len);
-    return;
 }
 
 static void cmd_get_configuration(IDEState *s, uint8_t *buf)
@@ -1169,20 +1254,28 @@ enum {
      * 4.1.8)
      */
     CHECK_READY = 0x02,
+
+    /*
+     * Commands flagged with NONDATA do not in any circumstances return
+     * any data via ide_atapi_cmd_reply. These commands are exempt from
+     * the normal byte_count_limit constraints.
+     * See ATA8-ACS3 "7.21.5 Byte Count Limit"
+     */
+    NONDATA = 0x04,
 };
 
-static const struct {
+static const struct AtapiCmd {
     void (*handler)(IDEState *s, uint8_t *buf);
     int flags;
 } atapi_cmd_table[0x100] = {
-    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY },
+    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY | NONDATA },
     [ 0x03 ] = { cmd_request_sense,                 ALLOW_UA },
     [ 0x12 ] = { cmd_inquiry,                       ALLOW_UA },
-    [ 0x1b ] = { cmd_start_stop_unit,               0 }, /* [1] */
-    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  0 },
+    [ 0x1b ] = { cmd_start_stop_unit,               NONDATA }, /* [1] */
+    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  NONDATA },
     [ 0x25 ] = { cmd_read_cdvd_capacity,            CHECK_READY },
     [ 0x28 ] = { cmd_read, /* (10) */               CHECK_READY },
-    [ 0x2b ] = { cmd_seek,                          CHECK_READY },
+    [ 0x2b ] = { cmd_seek,                          CHECK_READY | NONDATA },
     [ 0x43 ] = { cmd_read_toc_pma_atip,             CHECK_READY },
     [ 0x46 ] = { cmd_get_configuration,             ALLOW_UA },
     [ 0x4a ] = { cmd_get_event_status_notification, ALLOW_UA },
@@ -1190,7 +1283,7 @@ static const struct {
     [ 0x5a ] = { cmd_mode_sense, /* (10) */         0 },
     [ 0xa8 ] = { cmd_read, /* (12) */               CHECK_READY },
     [ 0xad ] = { cmd_read_dvd_structure,            CHECK_READY },
-    [ 0xbb ] = { cmd_set_speed,                     0 },
+    [ 0xbb ] = { cmd_set_speed,                     NONDATA },
     [ 0xbd ] = { cmd_mechanism_status,              0 },
     [ 0xbe ] = { cmd_read_cd,                       CHECK_READY },
     /* [1] handler detects and reports not ready condition itself */
@@ -1198,9 +1291,9 @@ static const struct {
 
 void ide_atapi_cmd(IDEState *s)
 {
-    uint8_t *buf;
+    uint8_t *buf = s->io_buffer;
+    const struct AtapiCmd *cmd = &atapi_cmd_table[s->io_buffer[0]];
 
-    buf = s->io_buffer;
 #ifdef DEBUG_IDE_ATAPI
     {
         int i;
@@ -1211,14 +1304,14 @@ void ide_atapi_cmd(IDEState *s)
         printf("\n");
     }
 #endif
+
     /*
      * If there's a UNIT_ATTENTION condition pending, only command flagged with
      * ALLOW_UA are allowed to complete. with other commands getting a CHECK
      * condition response unless a higher priority status, defined by the drive
      * here, is pending.
      */
-    if (s->sense_key == UNIT_ATTENTION &&
-        !(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA)) {
+    if (s->sense_key == UNIT_ATTENTION && !(cmd->flags & ALLOW_UA)) {
         ide_atapi_cmd_check_status(s);
         return;
     }
@@ -1229,7 +1322,7 @@ void ide_atapi_cmd(IDEState *s)
      * GET_EVENT_STATUS_NOTIFICATION to detect such tray open/close
      * states rely on this behavior.
      */
-    if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
+    if (!(cmd->flags & ALLOW_UA) &&
         !s->tray_open && blk_is_inserted(s->blk) && s->cdrom_changed) {
 
         if (s->cdrom_changed == 1) {
@@ -1244,16 +1337,29 @@ void ide_atapi_cmd(IDEState *s)
     }
 
     /* Report a Not Ready condition if appropriate for the command */
-    if ((atapi_cmd_table[s->io_buffer[0]].flags & CHECK_READY) &&
+    if ((cmd->flags & CHECK_READY) &&
         (!media_present(s) || !blk_is_inserted(s->blk)))
     {
         ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
         return;
     }
 
+    /* Nondata commands permit the byte_count_limit to be 0.
+     * If this is a data-transferring PIO command and BCL is 0,
+     * we abort at the /ATA/ level, not the ATAPI level.
+     * See ATA8 ACS3 section 7.17.6.49 and 7.21.5 */
+    if (cmd->handler && !(cmd->flags & NONDATA)) {
+        /* TODO: Check IDENTIFY data word 125 for default BCL (currently 0) */
+        if (!(atapi_byte_count_limit(s) || s->atapi_dma)) {
+            /* TODO: Move abort back into core.c and make static inline again */
+            ide_abort_command(s);
+            return;
+        }
+    }
+
     /* Execute the command */
-    if (atapi_cmd_table[s->io_buffer[0]].handler) {
-        atapi_cmd_table[s->io_buffer[0]].handler(s, buf);
+    if (cmd->handler) {
+        cmd->handler(s, buf);
         return;
     }
 
diff --git a/qemu/hw/ide/cmd646.c b/qemu/hw/ide/cmd646.c
index 66fb9d96d..49294a531 100644
--- a/qemu/hw/ide/cmd646.c
+++ b/qemu/hw/ide/cmd646.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
@@ -417,6 +418,7 @@ static void cmd646_ide_class_init(ObjectClass *klass, void *data)
     k->config_read = cmd646_pci_config_read;
     k->config_write = cmd646_pci_config_write;
     dc->props = cmd646_ide_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo cmd646_ide_info = {
diff --git a/qemu/hw/ide/core.c b/qemu/hw/ide/core.c
index 50449cae0..41e6a2dc4 100644
--- a/qemu/hw/ide/core.c
+++ b/qemu/hw/ide/core.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
@@ -32,6 +33,7 @@
 #include "sysemu/dma.h"
 #include "hw/block/block.h"
 #include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
 #include <hw/ide/internal.h>
 
@@ -55,7 +57,6 @@ static const int smart_attributes[][12] = {
     { 190,  0x03, 0x00, 0x45, 0x45, 0x1f, 0x00, 0x1f, 0x1f, 0x00, 0x00, 0x32},
 };
 
-static int ide_handle_rw_error(IDEState *s, int error, int op);
 static void ide_dummy_transfer_stop(IDEState *s);
 
 static void padstr(char *str, const char *src, int len)
@@ -457,7 +458,7 @@ BlockAIOCB *ide_issue_trim(BlockBackend *blk,
     return &iocb->common;
 }
 
-static inline void ide_abort_command(IDEState *s)
+void ide_abort_command(IDEState *s)
 {
     ide_transfer_stop(s);
     s->status = READY_STAT | ERR_STAT;
@@ -486,13 +487,27 @@ static void ide_cmd_done(IDEState *s)
     }
 }
 
-void ide_transfer_stop(IDEState *s)
+static void ide_transfer_halt(IDEState *s,
+                              void(*end_transfer_func)(IDEState *),
+                              bool notify)
 {
-    s->end_transfer_func = ide_transfer_stop;
+    s->end_transfer_func = end_transfer_func;
     s->data_ptr = s->io_buffer;
     s->data_end = s->io_buffer;
     s->status &= ~DRQ_STAT;
-    ide_cmd_done(s);
+    if (notify) {
+        ide_cmd_done(s);
+    }
+}
+
+void ide_transfer_stop(IDEState *s)
+{
+    ide_transfer_halt(s, ide_transfer_stop, true);
+}
+
+static void ide_transfer_cancel(IDEState *s)
+{
+    ide_transfer_halt(s, ide_transfer_cancel, false);
 }
 
 int64_t ide_get_sector(IDEState *s)
@@ -561,6 +576,98 @@ static bool ide_sect_range_ok(IDEState *s,
     return true;
 }
 
+static void ide_buffered_readv_cb(void *opaque, int ret)
+{
+    IDEBufferedRequest *req = opaque;
+    if (!req->orphaned) {
+        if (!ret) {
+            qemu_iovec_from_buf(req->original_qiov, 0, req->iov.iov_base,
+                                req->original_qiov->size);
+        }
+        req->original_cb(req->original_opaque, ret);
+    }
+    QLIST_REMOVE(req, list);
+    qemu_vfree(req->iov.iov_base);
+    g_free(req);
+}
+
+#define MAX_BUFFERED_REQS 16
+
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque)
+{
+    BlockAIOCB *aioreq;
+    IDEBufferedRequest *req;
+    int c = 0;
+
+    QLIST_FOREACH(req, &s->buffered_requests, list) {
+        c++;
+    }
+    if (c > MAX_BUFFERED_REQS) {
+        return blk_abort_aio_request(s->blk, cb, opaque, -EIO);
+    }
+
+    req = g_new0(IDEBufferedRequest, 1);
+    req->original_qiov = iov;
+    req->original_cb = cb;
+    req->original_opaque = opaque;
+    req->iov.iov_base = qemu_blockalign(blk_bs(s->blk), iov->size);
+    req->iov.iov_len = iov->size;
+    qemu_iovec_init_external(&req->qiov, &req->iov, 1);
+
+    aioreq = blk_aio_readv(s->blk, sector_num, &req->qiov, nb_sectors,
+                           ide_buffered_readv_cb, req);
+
+    QLIST_INSERT_HEAD(&s->buffered_requests, req, list);
+    return aioreq;
+}
+
+/**
+ * Cancel all pending DMA requests.
+ * Any buffered DMA requests are instantly canceled,
+ * but any pending unbuffered DMA requests must be waited on.
+ */
+void ide_cancel_dma_sync(IDEState *s)
+{
+    IDEBufferedRequest *req;
+
+    /* First invoke the callbacks of all buffered requests
+     * and flag those requests as orphaned. Ideally there
+     * are no unbuffered (Scatter Gather DMA Requests or
+     * write requests) pending and we can avoid to drain. */
+    QLIST_FOREACH(req, &s->buffered_requests, list) {
+        if (!req->orphaned) {
+#ifdef DEBUG_IDE
+            printf("%s: invoking cb %p of buffered request %p with"
+                   " -ECANCELED\n", __func__, req->original_cb, req);
+#endif
+            req->original_cb(req->original_opaque, -ECANCELED);
+        }
+        req->orphaned = true;
+    }
+
+    /*
+     * We can't cancel Scatter Gather DMA in the middle of the
+     * operation or a partial (not full) DMA transfer would reach
+     * the storage so we wait for completion instead (we beahve
+     * like if the DMA was completed by the time the guest trying
+     * to cancel dma with bmdma_cmd_writeb with BM_CMD_START not
+     * set).
+     *
+     * In the future we'll be able to safely cancel the I/O if the
+     * whole DMA operation will be submitted to disk with a single
+     * aio operation with preadv/pwritev.
+     */
+    if (s->bus->dma->aiocb) {
+#ifdef DEBUG_IDE
+        printf("%s: draining all remaining requests", __func__);
+#endif
+        blk_drain(s->blk);
+        assert(s->bus->dma->aiocb == NULL);
+    }
+}
+
 static void ide_sector_read(IDEState *s);
 
 static void ide_sector_read_cb(void *opaque, int ret)
@@ -574,7 +681,6 @@ static void ide_sector_read_cb(void *opaque, int ret)
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
     if (ret != 0) {
         if (ide_handle_rw_error(s, -ret, IDE_RETRY_PIO |
                                 IDE_RETRY_READ)) {
@@ -582,6 +688,8 @@ static void ide_sector_read_cb(void *opaque, int ret)
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
@@ -591,7 +699,6 @@ static void ide_sector_read_cb(void *opaque, int ret)
     s->nsector -= n;
     /* Allow the guest to read the io_buffer */
     ide_transfer_start(s, s->io_buffer, n * BDRV_SECTOR_SIZE, ide_sector_read);
-    s->io_buffer_offset += 512 * n;
     ide_set_irq(s->bus);
 }
 
@@ -622,6 +729,7 @@ static void ide_sector_read(IDEState *s)
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
         return;
     }
 
@@ -631,15 +739,16 @@ static void ide_sector_read(IDEState *s)
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
                      n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-    s->pio_aiocb = blk_aio_readv(s->blk, sector_num, &s->qiov, n,
-                                 ide_sector_read_cb, s);
+    s->pio_aiocb = ide_buffered_readv(s, sector_num, &s->qiov, n,
+                                      ide_sector_read_cb, s);
 }
 
-static void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
+void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
 {
     if (s->bus->dma->ops->commit_buf) {
         s->bus->dma->ops->commit_buf(s->bus->dma, tx_bytes);
     }
+    s->io_buffer_offset += tx_bytes;
     qemu_sglist_destroy(&s->sg);
 }
 
@@ -663,7 +772,7 @@ void ide_dma_error(IDEState *s)
     ide_set_irq(s->bus);
 }
 
-static int ide_handle_rw_error(IDEState *s, int error, int op)
+int ide_handle_rw_error(IDEState *s, int error, int op)
 {
     bool is_read = (op & IDE_RETRY_READ) != 0;
     BlockErrorAction action = blk_get_error_action(s->blk, is_read, error);
@@ -672,8 +781,11 @@ static int ide_handle_rw_error(IDEState *s, int error, int op)
         assert(s->bus->retry_unit == s->unit);
         s->bus->error_status = op;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
-        if (op & IDE_RETRY_DMA) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        if (IS_IDE_RETRY_DMA(op)) {
             ide_dma_error(s);
+        } else if (IS_IDE_RETRY_ATAPI(op)) {
+            ide_atapi_io_error(s, -error);
         } else {
             ide_rw_error(s);
         }
@@ -693,14 +805,7 @@ static void ide_dma_cb(void *opaque, int ret)
         return;
     }
     if (ret < 0) {
-        int op = IDE_RETRY_DMA;
-
-        if (s->dma_cmd == IDE_DMA_READ)
-            op |= IDE_RETRY_READ;
-        else if (s->dma_cmd == IDE_DMA_TRIM)
-            op |= IDE_RETRY_TRIM;
-
-        if (ide_handle_rw_error(s, -ret, op)) {
+        if (ide_handle_rw_error(s, -ret, ide_dma_cmd_to_retry(s->dma_cmd))) {
             return;
         }
     }
@@ -750,6 +855,7 @@ static void ide_dma_cb(void *opaque, int ret)
     if ((s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) &&
         !ide_sect_range_ok(s, sector_num, n)) {
         ide_dma_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), s->acct.type);
         return;
     }
 
@@ -767,6 +873,8 @@ static void ide_dma_cb(void *opaque, int ret)
                                         ide_issue_trim, ide_dma_cb, s,
                                         DMA_DIRECTION_TO_DEVICE);
         break;
+    default:
+        abort();
     }
     return;
 
@@ -826,7 +934,6 @@ static void ide_sector_write_cb(void *opaque, int ret)
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
 
     s->pio_aiocb = NULL;
     s->status &= ~BUSY_STAT;
@@ -837,12 +944,13 @@ static void ide_sector_write_cb(void *opaque, int ret)
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
     }
     s->nsector -= n;
-    s->io_buffer_offset += 512 * n;
 
     ide_set_sector(s, ide_get_sector(s) + n);
     if (s->nsector == 0) {
@@ -864,8 +972,8 @@ static void ide_sector_write_cb(void *opaque, int ret)
            that at the expense of slower write performances. Use this
            option _only_ to install Windows 2000. You must disable it
            for normal use. */
-        timer_mod(s->sector_write_timer,
-                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 1000));
+        timer_mod(s->sector_write_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  (NANOSECONDS_PER_SECOND / 1000));
     } else {
         ide_set_irq(s->bus);
     }
@@ -888,6 +996,7 @@ static void ide_sector_write(IDEState *s)
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_WRITE);
         return;
     }
 
@@ -896,7 +1005,7 @@ static void ide_sector_write(IDEState *s)
     qemu_iovec_init_external(&s->qiov, &s->iov, 1);
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
-                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
+                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
     s->pio_aiocb = blk_aio_writev(s->blk, sector_num, &s->qiov, n,
                                   ide_sector_write_cb, s);
 }
@@ -1122,11 +1231,86 @@ void ide_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
+static void ide_reset(IDEState *s)
+{
+#ifdef DEBUG_IDE
+    printf("ide: reset\n");
+#endif
+
+    if (s->pio_aiocb) {
+        blk_aio_cancel(s->pio_aiocb);
+        s->pio_aiocb = NULL;
+    }
+
+    if (s->drive_kind == IDE_CFATA)
+        s->mult_sectors = 0;
+    else
+        s->mult_sectors = MAX_MULT_SECTORS;
+    /* ide regs */
+    s->feature = 0;
+    s->error = 0;
+    s->nsector = 0;
+    s->sector = 0;
+    s->lcyl = 0;
+    s->hcyl = 0;
+
+    /* lba48 */
+    s->hob_feature = 0;
+    s->hob_sector = 0;
+    s->hob_nsector = 0;
+    s->hob_lcyl = 0;
+    s->hob_hcyl = 0;
+
+    s->select = 0xa0;
+    s->status = READY_STAT | SEEK_STAT;
+
+    s->lba48 = 0;
+
+    /* ATAPI specific */
+    s->sense_key = 0;
+    s->asc = 0;
+    s->cdrom_changed = 0;
+    s->packet_transfer_size = 0;
+    s->elementary_transfer_size = 0;
+    s->io_buffer_index = 0;
+    s->cd_sector_size = 0;
+    s->atapi_dma = 0;
+    s->tray_locked = 0;
+    s->tray_open = 0;
+    /* ATA DMA state */
+    s->io_buffer_size = 0;
+    s->req_nb_sectors = 0;
+
+    ide_set_signature(s);
+    /* init the transfer handler so that 0xffff is returned on data
+       accesses */
+    s->end_transfer_func = ide_dummy_transfer_stop;
+    ide_dummy_transfer_stop(s);
+    s->media_changed = 0;
+}
+
 static bool cmd_nop(IDEState *s, uint8_t cmd)
 {
     return true;
 }
 
+static bool cmd_device_reset(IDEState *s, uint8_t cmd)
+{
+    /* Halt PIO (in the DRQ phase), then DMA */
+    ide_transfer_cancel(s);
+    ide_cancel_dma_sync(s);
+
+    /* Reset any PIO commands, reset signature, etc */
+    ide_reset(s);
+
+    /* RESET: ATA8-ACS3 7.10.4 "Normal Outputs";
+     * ATA8-ACS3 Table 184 "Device Signatures for Normal Output" */
+    s->status = 0x00;
+
+    /* Do not overwrite status register */
+    return false;
+}
+
 static bool cmd_data_set_management(IDEState *s, uint8_t cmd)
 {
     switch (s->feature) {
@@ -1443,15 +1627,6 @@ static bool cmd_exec_dev_diagnostic(IDEState *s, uint8_t cmd)
     return false;
 }
 
-static bool cmd_device_reset(IDEState *s, uint8_t cmd)
-{
-    ide_set_signature(s);
-    s->status = 0x00; /* NOTE: READY is _not_ set */
-    s->error = 0x01;
-
-    return false;
-}
-
 static bool cmd_packet(IDEState *s, uint8_t cmd)
 {
     /* overlapping commands not supported */
@@ -1462,6 +1637,9 @@ static bool cmd_packet(IDEState *s, uint8_t cmd)
 
     s->status = READY_STAT | SEEK_STAT;
     s->atapi_dma = s->feature & 1;
+    if (s->atapi_dma) {
+        s->dma_cmd = IDE_DMA_ATAPI;
+    }
     s->nsector = 1;
     ide_transfer_start(s, s->io_buffer, ATAPI_PACKET_SIZE,
                        ide_atapi_cmd);
@@ -1747,11 +1925,11 @@ static const struct {
 } ide_cmd_table[0x100] = {
     /* NOP not implemented, mandatory for CD */
     [CFA_REQ_EXT_ERROR_CODE]      = { cmd_cfa_req_ext_error_code, CFA_OK },
-    [WIN_DSM]                     = { cmd_data_set_management, ALL_OK },
+    [WIN_DSM]                     = { cmd_data_set_management, HD_CFA_OK },
     [WIN_DEVICE_RESET]            = { cmd_device_reset, CD_OK },
     [WIN_RECAL]                   = { cmd_nop, HD_CFA_OK | SET_DSC},
     [WIN_READ]                    = { cmd_read_pio, ALL_OK },
-    [WIN_READ_ONCE]               = { cmd_read_pio, ALL_OK },
+    [WIN_READ_ONCE]               = { cmd_read_pio, HD_CFA_OK },
     [WIN_READ_EXT]                = { cmd_read_pio, HD_CFA_OK },
     [WIN_READDMA_EXT]             = { cmd_read_dma, HD_CFA_OK },
     [WIN_READ_NATIVE_MAX_EXT]     = { cmd_read_native_max, HD_CFA_OK | SET_DSC },
@@ -1770,12 +1948,12 @@ static const struct {
     [CFA_TRANSLATE_SECTOR]        = { cmd_cfa_translate_sector, CFA_OK },
     [WIN_DIAGNOSE]                = { cmd_exec_dev_diagnostic, ALL_OK },
     [WIN_SPECIFY]                 = { cmd_nop, HD_CFA_OK | SET_DSC },
-    [WIN_STANDBYNOW2]             = { cmd_nop, ALL_OK },
-    [WIN_IDLEIMMEDIATE2]          = { cmd_nop, ALL_OK },
-    [WIN_STANDBY2]                = { cmd_nop, ALL_OK },
-    [WIN_SETIDLE2]                = { cmd_nop, ALL_OK },
-    [WIN_CHECKPOWERMODE2]         = { cmd_check_power_mode, ALL_OK | SET_DSC },
-    [WIN_SLEEPNOW2]               = { cmd_nop, ALL_OK },
+    [WIN_STANDBYNOW2]             = { cmd_nop, HD_CFA_OK },
+    [WIN_IDLEIMMEDIATE2]          = { cmd_nop, HD_CFA_OK },
+    [WIN_STANDBY2]                = { cmd_nop, HD_CFA_OK },
+    [WIN_SETIDLE2]                = { cmd_nop, HD_CFA_OK },
+    [WIN_CHECKPOWERMODE2]         = { cmd_check_power_mode, HD_CFA_OK | SET_DSC },
+    [WIN_SLEEPNOW2]               = { cmd_nop, HD_CFA_OK },
     [WIN_PACKETCMD]               = { cmd_packet, CD_OK },
     [WIN_PIDENTIFY]               = { cmd_identify_packet, CD_OK },
     [WIN_SMART]                   = { cmd_smart, HD_CFA_OK | SET_DSC },
@@ -1789,19 +1967,19 @@ static const struct {
     [WIN_WRITEDMA]                = { cmd_write_dma, HD_CFA_OK },
     [WIN_WRITEDMA_ONCE]           = { cmd_write_dma, HD_CFA_OK },
     [CFA_WRITE_MULTI_WO_ERASE]    = { cmd_write_multiple, CFA_OK },
-    [WIN_STANDBYNOW1]             = { cmd_nop, ALL_OK },
-    [WIN_IDLEIMMEDIATE]           = { cmd_nop, ALL_OK },
-    [WIN_STANDBY]                 = { cmd_nop, ALL_OK },
-    [WIN_SETIDLE1]                = { cmd_nop, ALL_OK },
-    [WIN_CHECKPOWERMODE1]         = { cmd_check_power_mode, ALL_OK | SET_DSC },
-    [WIN_SLEEPNOW1]               = { cmd_nop, ALL_OK },
+    [WIN_STANDBYNOW1]             = { cmd_nop, HD_CFA_OK },
+    [WIN_IDLEIMMEDIATE]           = { cmd_nop, HD_CFA_OK },
+    [WIN_STANDBY]                 = { cmd_nop, HD_CFA_OK },
+    [WIN_SETIDLE1]                = { cmd_nop, HD_CFA_OK },
+    [WIN_CHECKPOWERMODE1]         = { cmd_check_power_mode, HD_CFA_OK | SET_DSC },
+    [WIN_SLEEPNOW1]               = { cmd_nop, HD_CFA_OK },
     [WIN_FLUSH_CACHE]             = { cmd_flush_cache, ALL_OK },
     [WIN_FLUSH_CACHE_EXT]         = { cmd_flush_cache, HD_CFA_OK },
     [WIN_IDENTIFY]                = { cmd_identify, ALL_OK },
     [WIN_SETFEATURES]             = { cmd_set_features, ALL_OK | SET_DSC },
     [IBM_SENSE_CONDITION]         = { cmd_ibm_sense_condition, CFA_OK | SET_DSC },
     [CFA_WEAR_LEVEL]              = { cmd_cfa_erase_sectors, HD_CFA_OK | SET_DSC },
-    [WIN_READ_NATIVE_MAX]         = { cmd_read_native_max, ALL_OK | SET_DSC },
+    [WIN_READ_NATIVE_MAX]         = { cmd_read_native_max, HD_CFA_OK | SET_DSC },
 };
 
 static bool ide_cmd_permitted(IDEState *s, uint32_t cmd)
@@ -1824,9 +2002,13 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
         return;
     }
 
-    /* Only DEVICE RESET is allowed while BSY or/and DRQ are set */
-    if ((s->status & (BUSY_STAT|DRQ_STAT)) && val != WIN_DEVICE_RESET)
-        return;
+    /* Only RESET is allowed while BSY and/or DRQ are set,
+     * and only to ATAPI devices. */
+    if (s->status & (BUSY_STAT|DRQ_STAT)) {
+        if (val != WIN_DEVICE_RESET || s->drive_kind != IDE_CD) {
+            return;
+        }
+    }
 
     if (!ide_cmd_permitted(s, val)) {
         ide_abort_command(s);
@@ -2126,64 +2308,6 @@ static void ide_dummy_transfer_stop(IDEState *s)
     s->io_buffer[3] = 0xff;
 }
 
-static void ide_reset(IDEState *s)
-{
-#ifdef DEBUG_IDE
-    printf("ide: reset\n");
-#endif
-
-    if (s->pio_aiocb) {
-        blk_aio_cancel(s->pio_aiocb);
-        s->pio_aiocb = NULL;
-    }
-
-    if (s->drive_kind == IDE_CFATA)
-        s->mult_sectors = 0;
-    else
-        s->mult_sectors = MAX_MULT_SECTORS;
-    /* ide regs */
-    s->feature = 0;
-    s->error = 0;
-    s->nsector = 0;
-    s->sector = 0;
-    s->lcyl = 0;
-    s->hcyl = 0;
-
-    /* lba48 */
-    s->hob_feature = 0;
-    s->hob_sector = 0;
-    s->hob_nsector = 0;
-    s->hob_lcyl = 0;
-    s->hob_hcyl = 0;
-
-    s->select = 0xa0;
-    s->status = READY_STAT | SEEK_STAT;
-
-    s->lba48 = 0;
-
-    /* ATAPI specific */
-    s->sense_key = 0;
-    s->asc = 0;
-    s->cdrom_changed = 0;
-    s->packet_transfer_size = 0;
-    s->elementary_transfer_size = 0;
-    s->io_buffer_index = 0;
-    s->cd_sector_size = 0;
-    s->atapi_dma = 0;
-    s->tray_locked = 0;
-    s->tray_open = 0;
-    /* ATA DMA state */
-    s->io_buffer_size = 0;
-    s->req_nb_sectors = 0;
-
-    ide_set_signature(s);
-    /* init the transfer handler so that 0xffff is returned on data
-       accesses */
-    s->end_transfer_func = ide_dummy_transfer_stop;
-    ide_dummy_transfer_stop(s);
-    s->media_changed = 0;
-}
-
 void ide_bus_reset(IDEBus *bus)
 {
     bus->unit = 0;
@@ -2313,7 +2437,7 @@ int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
     if (version) {
         pstrcpy(s->version, sizeof(s->version), version);
     } else {
-        pstrcpy(s->version, sizeof(s->version), qemu_get_version());
+        pstrcpy(s->version, sizeof(s->version), qemu_hw_version());
     }
 
     ide_reset(s);
@@ -2400,15 +2524,13 @@ static void ide_restart_bh(void *opaque)
         if (s->bus->dma->ops->restart) {
             s->bus->dma->ops->restart(s->bus->dma);
         }
-    }
-
-    if (error_status & IDE_RETRY_DMA) {
+    } else if (IS_IDE_RETRY_DMA(error_status)) {
         if (error_status & IDE_RETRY_TRIM) {
             ide_restart_dma(s, IDE_DMA_TRIM);
         } else {
             ide_restart_dma(s, is_read ? IDE_DMA_READ : IDE_DMA_WRITE);
         }
-    } else if (error_status & IDE_RETRY_PIO) {
+    } else if (IS_IDE_RETRY_PIO(error_status)) {
         if (is_read) {
             ide_sector_read(s);
         } else {
@@ -2416,15 +2538,11 @@ static void ide_restart_bh(void *opaque)
         }
     } else if (error_status & IDE_RETRY_FLUSH) {
         ide_flush_cache(s);
+    } else if (IS_IDE_RETRY_ATAPI(error_status)) {
+        assert(s->end_transfer_func == ide_atapi_cmd);
+        ide_atapi_dma_restart(s);
     } else {
-        /*
-         * We've not got any bits to tell us about ATAPI - but
-         * we do have the end_transfer_func that tells us what
-         * we're trying to do.
-         */
-        if (s->end_transfer_func == ide_atapi_cmd) {
-            ide_atapi_dma_restart(s);
-        }
+        abort();
     }
 }
 
diff --git a/qemu/hw/ide/ich.c b/qemu/hw/ide/ich.c
index 350c7f1c7..0a13334ba 100644
--- a/qemu/hw/ide/ich.c
+++ b/qemu/hw/ide/ich.c
@@ -60,6 +60,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/pci/msi.h>
 #include <hw/i386/pc.h>
@@ -97,6 +98,13 @@ static void pci_ich9_reset(DeviceState *dev)
     ahci_reset(&d->ahci);
 }
 
+static void pci_ich9_ahci_init(Object *obj)
+{
+    struct AHCIPCIState *d = ICH_AHCI(obj);
+
+    ahci_init(&d->ahci, DEVICE(obj));
+}
+
 static void pci_ich9_ahci_realize(PCIDevice *dev, Error **errp)
 {
     struct AHCIPCIState *d;
@@ -104,7 +112,7 @@ static void pci_ich9_ahci_realize(PCIDevice *dev, Error **errp)
     uint8_t *sata_cap;
     d = ICH_AHCI(dev);
 
-    ahci_init(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
+    ahci_realize(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
 
     pci_config_set_prog_interface(dev->config, AHCI_PROGMODE_MAJOR_REV_1);
 
@@ -171,6 +179,7 @@ static const TypeInfo ich_ahci_info = {
     .name          = TYPE_ICH9_AHCI,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(AHCIPCIState),
+    .instance_init = pci_ich9_ahci_init,
     .class_init    = ich_ahci_class_init,
 };
 
diff --git a/qemu/hw/ide/internal.h b/qemu/hw/ide/internal.h
index 30fdcbc5f..d2c458f57 100644
--- a/qemu/hw/ide/internal.h
+++ b/qemu/hw/ide/internal.h
@@ -338,11 +338,22 @@ enum ide_dma_cmd {
     IDE_DMA_READ,
     IDE_DMA_WRITE,
     IDE_DMA_TRIM,
+    IDE_DMA_ATAPI,
 };
 
 #define ide_cmd_is_read(s) \
 	((s)->dma_cmd == IDE_DMA_READ)
 
+typedef struct IDEBufferedRequest {
+    QLIST_ENTRY(IDEBufferedRequest) list;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    QEMUIOVector *original_qiov;
+    BlockCompletionFunc *original_cb;
+    void *original_opaque;
+    bool orphaned;
+} IDEBufferedRequest;
+
 /* NOTE: IDEState represents in fact one drive */
 struct IDEState {
     IDEBus *bus;
@@ -396,8 +407,9 @@ struct IDEState {
     BlockAIOCB *pio_aiocb;
     struct iovec iov;
     QEMUIOVector qiov;
+    QLIST_HEAD(, IDEBufferedRequest) buffered_requests;
     /* ATA DMA state */
-    int32_t io_buffer_offset;
+    uint64_t io_buffer_offset;
     int32_t io_buffer_size;
     QEMUSGList sg;
     /* PIO transfer handling */
@@ -495,13 +507,45 @@ struct IDEDevice {
 };
 
 /* These are used for the error_status field of IDEBus */
+#define IDE_RETRY_MASK 0xf8
 #define IDE_RETRY_DMA  0x08
 #define IDE_RETRY_PIO  0x10
+#define IDE_RETRY_ATAPI 0x20 /* reused IDE_RETRY_READ bit */
 #define IDE_RETRY_READ  0x20
 #define IDE_RETRY_FLUSH 0x40
 #define IDE_RETRY_TRIM 0x80
 #define IDE_RETRY_HBA  0x100
 
+#define IS_IDE_RETRY_DMA(_status) \
+    ((_status) & IDE_RETRY_DMA)
+
+#define IS_IDE_RETRY_PIO(_status) \
+    ((_status) & IDE_RETRY_PIO)
+
+/*
+ * The method of the IDE_RETRY_ATAPI determination is to use a previously
+ * impossible bit combination as a new status value.
+ */
+#define IS_IDE_RETRY_ATAPI(_status)   \
+    (((_status) & IDE_RETRY_MASK) == IDE_RETRY_ATAPI)
+
+static inline uint8_t ide_dma_cmd_to_retry(uint8_t dma_cmd)
+{
+    switch (dma_cmd) {
+    case IDE_DMA_READ:
+        return IDE_RETRY_DMA | IDE_RETRY_READ;
+    case IDE_DMA_WRITE:
+        return IDE_RETRY_DMA;
+    case IDE_DMA_TRIM:
+        return IDE_RETRY_DMA | IDE_RETRY_TRIM;
+    case IDE_DMA_ATAPI:
+        return IDE_RETRY_ATAPI;
+    default:
+        break;
+    }
+    return 0;
+}
+
 static inline IDEState *idebus_active_if(IDEBus *bus)
 {
     return bus->ifs + bus->unit;
@@ -536,7 +580,9 @@ int64_t ide_get_sector(IDEState *s);
 void ide_set_sector(IDEState *s, int64_t sector_num);
 
 void ide_start_dma(IDEState *s, BlockCompletionFunc *cb);
+void dma_buf_commit(IDEState *s, uint32_t tx_bytes);
 void ide_dma_error(IDEState *s);
+void ide_abort_command(IDEState *s);
 
 void ide_atapi_cmd_ok(IDEState *s);
 void ide_atapi_cmd_error(IDEState *s, int sense_key, int asc);
@@ -570,6 +616,10 @@ void ide_set_inactive(IDEState *s, bool more);
 BlockAIOCB *ide_issue_trim(BlockBackend *blk,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque);
+void ide_cancel_dma_sync(IDEState *s);
 
 /* hw/ide/atapi.c */
 void ide_atapi_cmd(IDEState *s);
@@ -580,4 +630,6 @@ void ide_bus_new(IDEBus *idebus, size_t idebus_size, DeviceState *dev,
                  int bus_id, int max_units);
 IDEDevice *ide_create_drive(IDEBus *bus, int unit, DriveInfo *drive);
 
+int ide_handle_rw_error(IDEState *s, int error, int op);
+
 #endif /* HW_IDE_INTERNAL_H */
diff --git a/qemu/hw/ide/isa.c b/qemu/hw/ide/isa.c
index 9f80503fa..eba567c87 100644
--- a/qemu/hw/ide/isa.c
+++ b/qemu/hw/ide/isa.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/isa/isa.h>
diff --git a/qemu/hw/ide/macio.c b/qemu/hw/ide/macio.c
index 66ac2baa9..76256eb8a 100644
--- a/qemu/hw/ide/macio.c
+++ b/qemu/hw/ide/macio.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/ppc/mac_dbdma.h"
@@ -119,8 +120,8 @@ static void pmac_dma_read(BlockBackend *blk,
     MACIO_DPRINTF("--- Block read transfer - sector_num: %" PRIx64 "  "
                   "nsector: %x\n", (offset >> 9), (bytes >> 9));
 
-    m->aiocb = blk_aio_readv(blk, (offset >> 9), &io->iov, (bytes >> 9),
-                             cb, io);
+    s->bus->dma->aiocb = blk_aio_readv(blk, (offset >> 9), &io->iov,
+                             (bytes >> 9), cb, io);
 }
 
 static void pmac_dma_write(BlockBackend *blk,
@@ -204,8 +205,8 @@ static void pmac_dma_write(BlockBackend *blk,
     MACIO_DPRINTF("--- Block write transfer - sector_num: %" PRIx64 "  "
                   "nsector: %x\n", (offset >> 9), (bytes >> 9));
 
-    m->aiocb = blk_aio_writev(blk, (offset >> 9), &io->iov, (bytes >> 9),
-                              cb, io);
+    s->bus->dma->aiocb = blk_aio_writev(blk, (offset >> 9), &io->iov,
+                             (bytes >> 9), cb, io);
 }
 
 static void pmac_dma_trim(BlockBackend *blk,
@@ -231,8 +232,8 @@ static void pmac_dma_trim(BlockBackend *blk,
     s->io_buffer_index += io->len;
     io->len = 0;
 
-    m->aiocb = ide_issue_trim(blk, (offset >> 9), &io->iov, (bytes >> 9),
-                              cb, io);
+    s->bus->dma->aiocb = ide_issue_trim(blk, (offset >> 9), &io->iov,
+                             (bytes >> 9), cb, io);
 }
 
 static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
@@ -280,16 +281,20 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
     }
 
     /* Calculate current offset */
-    offset = (int64_t)(s->lba << 11) + s->io_buffer_index;
+    offset = ((int64_t)s->lba << 11) + s->io_buffer_index;
 
     pmac_dma_read(s->blk, offset, io->len, pmac_ide_atapi_transfer_cb, io);
     return;
 
 done:
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
-    io->dma_end(opaque);
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
 
-    return;
+    ide_set_inactive(s, false);
+    io->dma_end(opaque);
 }
 
 static void pmac_ide_transfer_cb(void *opaque, int ret)
@@ -303,7 +308,6 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
 
     if (ret < 0) {
         MACIO_DPRINTF("DMA error: %d\n", ret);
-        m->aiocb = NULL;
         ide_dma_error(s);
         goto done;
     }
@@ -342,14 +346,22 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
     case IDE_DMA_TRIM:
         pmac_dma_trim(s->blk, offset, io->len, pmac_ide_transfer_cb, io);
         break;
+    default:
+        abort();
     }
 
     return;
 
 done:
     if (s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) {
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        if (ret < 0) {
+            block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        } else {
+            block_acct_done(blk_get_stats(s->blk), &s->acct);
+        }
     }
+
+    ide_set_inactive(s, false);
     io->dma_end(opaque);
 }
 
@@ -387,8 +399,9 @@ static void pmac_ide_transfer(DBDMA_io *io)
 static void pmac_ide_flush(DBDMA_io *io)
 {
     MACIOIDEState *m = io->opaque;
+    IDEState *s = idebus_active_if(&m->bus);
 
-    if (m->aiocb) {
+    if (s->bus->dma->aiocb) {
         blk_drain_all();
     }
 }
@@ -506,11 +519,12 @@ static const MemoryRegionOps pmac_ide_ops = {
 
 static const VMStateDescription vmstate_pmac = {
     .name = "ide",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
         VMSTATE_IDE_BUS(bus, MACIOIDEState),
         VMSTATE_IDE_DRIVES(bus.ifs, MACIOIDEState),
+        VMSTATE_BOOL(dma_active, MACIOIDEState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -590,6 +604,7 @@ static void macio_ide_class_init(ObjectClass *oc, void *data)
     dc->realize = macio_ide_realizefn;
     dc->reset = macio_ide_reset;
     dc->vmsd = &vmstate_pmac;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo macio_ide_type_info = {
diff --git a/qemu/hw/ide/microdrive.c b/qemu/hw/ide/microdrive.c
index 6639dd488..5c9db8047 100644
--- a/qemu/hw/ide/microdrive.c
+++ b/qemu/hw/ide/microdrive.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pcmcia.h>
diff --git a/qemu/hw/ide/mmio.c b/qemu/hw/ide/mmio.c
index b6ce62ac5..493f65a1d 100644
--- a/qemu/hw/ide/mmio.c
+++ b/qemu/hw/ide/mmio.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/block-backend.h"
diff --git a/qemu/hw/ide/pci.c b/qemu/hw/ide/pci.c
index d31ff885b..8d56a00b1 100644
--- a/qemu/hw/ide/pci.c
+++ b/qemu/hw/ide/pci.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
@@ -103,13 +104,6 @@ static int32_t bmdma_prepare_buf(IDEDMA *dma, int32_t limit)
                 qemu_sglist_add(&s->sg, bm->cur_prd_addr, sg_len);
             }
 
-            /* Note: We limit the max transfer to be 2GiB.
-             * This should accommodate the largest ATA transaction
-             * for LBA48 (65,536 sectors) and 32K sector sizes. */
-            if (s->sg.size > INT32_MAX) {
-                error_report("IDE: sglist describes more than 2GiB.");
-                break;
-            }
             bm->cur_prd_addr += l;
             bm->cur_prd_len -= l;
             s->io_buffer_size += l;
@@ -240,22 +234,7 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
     /* Ignore writes to SSBM if it keeps the old value */
     if ((val & BM_CMD_START) != (bm->cmd & BM_CMD_START)) {
         if (!(val & BM_CMD_START)) {
-            /*
-             * We can't cancel Scatter Gather DMA in the middle of the
-             * operation or a partial (not full) DMA transfer would reach
-             * the storage so we wait for completion instead (we beahve
-             * like if the DMA was completed by the time the guest trying
-             * to cancel dma with bmdma_cmd_writeb with BM_CMD_START not
-             * set).
-             *
-             * In the future we'll be able to safely cancel the I/O if the
-             * whole DMA operation will be submitted to disk with a single
-             * aio operation with preadv/pwritev.
-             */
-            if (bm->bus->dma->aiocb) {
-                blk_drain_all();
-                assert(bm->bus->dma->aiocb == NULL);
-            }
+            ide_cancel_dma_sync(idebus_active_if(bm->bus));
             bm->status &= ~BM_STATUS_DMAING;
         } else {
             bm->cur_addr = bm->addr;
@@ -329,6 +308,10 @@ static void ide_bmdma_pre_save(void *opaque)
     BMDMAState *bm = opaque;
     uint8_t abused_bits = BM_MIGRATION_COMPAT_STATUS_BITS;
 
+    if (!(bm->status & BM_STATUS_DMAING) && bm->dma_cb) {
+        bm->bus->error_status =
+            ide_dma_cmd_to_retry(bmdma_active_if(bm)->dma_cmd);
+    }
     bm->migration_retry_unit = bm->bus->retry_unit;
     bm->migration_retry_sector_num = bm->bus->retry_sector_num;
     bm->migration_retry_nsector = bm->bus->retry_nsector;
diff --git a/qemu/hw/ide/piix.c b/qemu/hw/ide/piix.c
index 5a26c86a8..6d76ce980 100644
--- a/qemu/hw/ide/piix.c
+++ b/qemu/hw/ide/piix.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
@@ -188,6 +189,7 @@ int pci_piix3_xen_ide_unplug(DeviceState *dev)
                 idedev = pci_ide->bus[di->bus].slave;
             }
             idedev->conf.blk = NULL;
+            monitor_remove_blk(blk);
             blk_unref(blk);
         }
     }
@@ -257,22 +259,10 @@ static const TypeInfo piix3_ide_info = {
     .class_init    = piix3_ide_class_init,
 };
 
-static void piix3_ide_xen_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->realize = pci_piix_ide_realize;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371SB_1;
-    k->class_id = PCI_CLASS_STORAGE_IDE;
-    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-}
-
 static const TypeInfo piix3_ide_xen_info = {
     .name          = "piix3-ide-xen",
     .parent        = TYPE_PCI_IDE,
-    .class_init    = piix3_ide_xen_class_init,
+    .class_init    = piix3_ide_class_init,
 };
 
 static void piix4_ide_class_init(ObjectClass *klass, void *data)
diff --git a/qemu/hw/ide/qdev.c b/qemu/hw/ide/qdev.c
index 788b36133..4bc74a32d 100644
--- a/qemu/hw/ide/qdev.c
+++ b/qemu/hw/ide/qdev.c
@@ -16,8 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include "sysemu/dma.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include <hw/ide/internal.h>
 #include "sysemu/block-backend.h"
@@ -118,7 +120,8 @@ IDEDevice *ide_create_drive(IDEBus *bus, int unit, DriveInfo *drive)
 
     dev = qdev_create(&bus->qbus, drive->media_cd ? "ide-cd" : "ide-hd");
     qdev_prop_set_uint32(dev, "unit", unit);
-    qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(drive));
+    qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(drive),
+                        &error_fatal);
     qdev_init_nofail(dev);
     return DO_UPCAST(IDEDevice, qdev, dev);
 }
@@ -171,7 +174,7 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
 
     blkconf_serial(&dev->conf, &dev->serial);
     if (kind != IDE_CD) {
-        blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255, &err);
+        blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, &err);
         if (err) {
             error_report_err(err);
             return -1;
@@ -198,22 +201,22 @@ static int ide_dev_initfn(IDEDevice *dev, IDEDriveKind kind)
     return 0;
 }
 
-static void ide_dev_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void ide_dev_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     IDEDevice *d = IDE_DEVICE(obj);
 
-    visit_type_int32(v, &d->conf.bootindex, name, errp);
+    visit_type_int32(v, name, &d->conf.bootindex, errp);
 }
 
-static void ide_dev_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void ide_dev_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     IDEDevice *d = IDE_DEVICE(obj);
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
diff --git a/qemu/hw/ide/via.c b/qemu/hw/ide/via.c
index e2da9ef71..d3f72267a 100644
--- a/qemu/hw/ide/via.c
+++ b/qemu/hw/ide/via.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <hw/hw.h>
 #include <hw/i386/pc.h>
 #include <hw/pci/pci.h>
diff --git a/qemu/hw/input/Makefile.objs b/qemu/hw/input/Makefile.objs
index 624ba7ea4..7715d7230 100644
--- a/qemu/hw/input/Makefile.objs
+++ b/qemu/hw/input/Makefile.objs
@@ -8,9 +8,9 @@ common-obj-$(CONFIG_STELLARIS_INPUT) += stellaris_input.o
 common-obj-$(CONFIG_TSC2005) += tsc2005.o
 common-obj-$(CONFIG_VMMOUSE) += vmmouse.o
 
-ifeq ($(CONFIG_LINUX),y)
 common-obj-$(CONFIG_VIRTIO) += virtio-input.o
 common-obj-$(CONFIG_VIRTIO) += virtio-input-hid.o
+ifeq ($(CONFIG_LINUX),y)
 common-obj-$(CONFIG_VIRTIO) += virtio-input-host.o
 endif
 
diff --git a/qemu/hw/input/adb.c b/qemu/hw/input/adb.c
index a18eea265..f0ad0d447 100644
--- a/qemu/hw/input/adb.c
+++ b/qemu/hw/input/adb.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/input/adb.h"
 #include "ui/console.h"
@@ -88,7 +89,7 @@ int adb_request(ADBBusState *s, uint8_t *obuf, const uint8_t *buf, int len)
 }
 
 /* XXX: move that to cuda ? */
-int adb_poll(ADBBusState *s, uint8_t *obuf)
+int adb_poll(ADBBusState *s, uint8_t *obuf, uint16_t poll_mask)
 {
     ADBDevice *d;
     int olen, i;
@@ -99,13 +100,15 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
         if (s->poll_index >= s->nb_devices)
             s->poll_index = 0;
         d = s->devices[s->poll_index];
-        buf[0] = ADB_READREG | (d->devaddr << 4);
-        olen = adb_request(s, obuf + 1, buf, 1);
-        /* if there is data, we poll again the same device */
-        if (olen > 0) {
-            obuf[0] = buf[0];
-            olen++;
-            break;
+        if ((1 << d->devaddr) & poll_mask) {
+            buf[0] = ADB_READREG | (d->devaddr << 4);
+            olen = adb_request(s, obuf + 1, buf, 1);
+            /* if there is data, we poll again the same device */
+            if (olen > 0) {
+                obuf[0] = buf[0];
+                olen++;
+                break;
+            }
         }
         s->poll_index++;
     }
@@ -362,6 +365,7 @@ static void adb_kbd_class_init(ObjectClass *oc, void *data)
 
     akc->parent_realize = dc->realize;
     dc->realize = adb_kbd_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_kbd_request;
     dc->reset = adb_kbd_reset;
@@ -566,6 +570,7 @@ static void adb_mouse_class_init(ObjectClass *oc, void *data)
 
     amc->parent_realize = dc->realize;
     dc->realize = adb_mouse_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_mouse_request;
     dc->reset = adb_mouse_reset;
diff --git a/qemu/hw/input/hid.c b/qemu/hw/input/hid.c
index 21ebd9e71..d92c7463b 100644
--- a/qemu/hw/input/hid.c
+++ b/qemu/hw/input/hid.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "qemu/timer.h"
@@ -44,7 +45,7 @@ static const uint8_t hid_usage_keys[0x100] = {
     0xe2, 0x2c, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e,
     0x3f, 0x40, 0x41, 0x42, 0x43, 0x53, 0x47, 0x5f,
     0x60, 0x61, 0x56, 0x5c, 0x5d, 0x5e, 0x57, 0x59,
-    0x5a, 0x5b, 0x62, 0x63, 0x00, 0x00, 0x00, 0x44,
+    0x5a, 0x5b, 0x62, 0x63, 0x00, 0x00, 0x64, 0x44,
     0x45, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e,
     0xe8, 0xe9, 0x71, 0x72, 0x73, 0x00, 0x00, 0x00,
     0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00,
@@ -95,7 +96,7 @@ void hid_set_next_idle(HIDState *hs)
 {
     if (hs->idle) {
         uint64_t expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                               get_ticks_per_sec() * hs->idle * 4 / 1000;
+                               NANOSECONDS_PER_SECOND * hs->idle * 4 / 1000;
         if (!hs->idle_timer) {
             hs->idle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, hid_idle_timer, hs);
         }
@@ -108,44 +109,49 @@ void hid_set_next_idle(HIDState *hs)
 static void hid_pointer_event(DeviceState *dev, QemuConsole *src,
                               InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = 0x01,
         [INPUT_BUTTON_RIGHT]  = 0x02,
         [INPUT_BUTTON_MIDDLE] = 0x04,
     };
     HIDState *hs = (HIDState *)dev;
     HIDPointerEvent *e;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
 
     assert(hs->n < QUEUE_LENGTH);
     e = &hs->ptr.queue[(hs->head + hs->n) & QUEUE_MASK];
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy += evt->rel->value;
+        move = evt->u.rel.data;
+        if (move->axis == INPUT_AXIS_X) {
+            e->xdx += move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            e->ydy += move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_ABS:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx = evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy = evt->rel->value;
+        move = evt->u.abs.data;
+        if (move->axis == INPUT_AXIS_X) {
+            e->xdx = move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            e->ydy = move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            e->buttons_state |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            e->buttons_state |= bmap[btn->button];
+            if (btn->button == INPUT_BUTTON_WHEEL_UP) {
                 e->dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 e->dz++;
             }
         } else {
-            e->buttons_state &= ~bmap[evt->btn->button];
+            e->buttons_state &= ~bmap[btn->button];
         }
         break;
 
@@ -222,9 +228,10 @@ static void hid_keyboard_event(DeviceState *dev, QemuConsole *src,
     HIDState *hs = (HIDState *)dev;
     int scancodes[3], i, count;
     int slot;
+    InputKeyEvent *key = evt->u.key.data;
 
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     if (hs->n + count > QUEUE_LENGTH) {
         fprintf(stderr, "usb-kbd: warning: key event queue full\n");
diff --git a/qemu/hw/input/lm832x.c b/qemu/hw/input/lm832x.c
index 530a6e01f..539682cac 100644
--- a/qemu/hw/input/lm832x.c
+++ b/qemu/hw/input/lm832x.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/input/milkymist-softusb.c b/qemu/hw/input/milkymist-softusb.c
index 7b0f4db88..40dfca157 100644
--- a/qemu/hw/input/milkymist-softusb.c
+++ b/qemu/hw/input/milkymist-softusb.c
@@ -21,6 +21,8 @@
  *   not available yet
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -255,12 +257,12 @@ static int milkymist_softusb_init(SysBusDevice *dev)
 
     /* register pmem and dmem */
     memory_region_init_ram(&s->pmem, OBJECT(s), "milkymist-softusb.pmem",
-                           s->pmem_size, &error_abort);
+                           s->pmem_size, &error_fatal);
     vmstate_register_ram_global(&s->pmem);
     s->pmem_ptr = memory_region_get_ram_ptr(&s->pmem);
     sysbus_init_mmio(dev, &s->pmem);
     memory_region_init_ram(&s->dmem, OBJECT(s), "milkymist-softusb.dmem",
-                           s->dmem_size, &error_abort);
+                           s->dmem_size, &error_fatal);
     vmstate_register_ram_global(&s->dmem);
     s->dmem_ptr = memory_region_get_ram_ptr(&s->dmem);
     sysbus_init_mmio(dev, &s->dmem);
diff --git a/qemu/hw/input/pckbd.c b/qemu/hw/input/pckbd.c
index ddac69df6..1d932ec19 100644
--- a/qemu/hw/input/pckbd.c
+++ b/qemu/hw/input/pckbd.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/i386/pc.h"
diff --git a/qemu/hw/input/pl050.c b/qemu/hw/input/pl050.c
index c1b08d5a4..3092b0fe3 100644
--- a/qemu/hw/input/pl050.c
+++ b/qemu/hw/input/pl050.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/input/ps2.h"
 
diff --git a/qemu/hw/input/ps2.c b/qemu/hw/input/ps2.c
index fdbe565e6..a8aa36f5c 100644
--- a/qemu/hw/input/ps2.c
+++ b/qemu/hw/input/ps2.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/input/ps2.h"
 #include "ui/console.h"
@@ -181,10 +182,11 @@ static void ps2_keyboard_event(DeviceState *dev, QemuConsole *src,
 {
     PS2KbdState *s = (PS2KbdState *)dev;
     int scancodes[3], i, count;
+    InputKeyEvent *key = evt->u.key.data;
 
     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         ps2_put_keycode(s, scancodes[i]);
@@ -382,36 +384,40 @@ static void ps2_mouse_send_packet(PS2MouseState *s)
 static void ps2_mouse_event(DeviceState *dev, QemuConsole *src,
                             InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE] = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]  = MOUSE_EVENT_RBUTTON,
     };
     PS2MouseState *s = (PS2MouseState *)dev;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
 
     /* check if deltas are recorded when disabled */
     if (!(s->mouse_status & MOUSE_STATUS_ENABLED))
         return;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            s->mouse_dx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            s->mouse_dy -= evt->rel->value;
+        move = evt->u.rel.data;
+        if (move->axis == INPUT_AXIS_X) {
+            s->mouse_dx += move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            s->mouse_dy -= move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->mouse_buttons |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            s->mouse_buttons |= bmap[btn->button];
+            if (btn->button == INPUT_BUTTON_WHEEL_UP) {
                 s->mouse_dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 s->mouse_dz++;
             }
         } else {
-            s->mouse_buttons &= ~bmap[evt->btn->button];
+            s->mouse_buttons &= ~bmap[btn->button];
         }
         break;
 
@@ -622,7 +628,7 @@ static void ps2_kbd_reset(void *opaque)
     ps2_common_reset(&s->common);
     s->scan_enabled = 0;
     s->translate = 0;
-    s->scancode_set = 0;
+    s->scancode_set = 2;
 }
 
 static void ps2_mouse_reset(void *opaque)
diff --git a/qemu/hw/input/pxa2xx_keypad.c b/qemu/hw/input/pxa2xx_keypad.c
index 85011145e..2b70bbb95 100644
--- a/qemu/hw/input/pxa2xx_keypad.c
+++ b/qemu/hw/input/pxa2xx_keypad.c
@@ -11,6 +11,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "ui/console.h"
diff --git a/qemu/hw/input/stellaris_input.c b/qemu/hw/input/stellaris_input.c
index 0609e8086..99168bfee 100644
--- a/qemu/hw/input/stellaris_input.c
+++ b/qemu/hw/input/stellaris_input.c
@@ -6,6 +6,7 @@
  *
  * This code is licensed under the GPL.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "ui/console.h"
@@ -69,14 +70,14 @@ static const VMStateDescription vmstate_stellaris_gamepad = {
     }
 };
 
-/* Returns an array 5 ouput slots.  */
+/* Returns an array of 5 output slots.  */
 void stellaris_gamepad_init(int n, qemu_irq *irq, const int *keycode)
 {
     gamepad_state *s;
     int i;
 
-    s = (gamepad_state *)g_malloc0(sizeof (gamepad_state));
-    s->buttons = (gamepad_button *)g_malloc0(n * sizeof (gamepad_button));
+    s = g_new0(gamepad_state, 1);
+    s->buttons = g_new0(gamepad_button, n);
     for (i = 0; i < n; i++) {
         s->buttons[i].irq = irq[i];
         s->buttons[i].keycode = keycode[i];
diff --git a/qemu/hw/input/tsc2005.c b/qemu/hw/input/tsc2005.c
index 21d4f4dbb..9b359aaec 100644
--- a/qemu/hw/input/tsc2005.c
+++ b/qemu/hw/input/tsc2005.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "ui/console.h"
@@ -290,7 +291,8 @@ static void tsc2005_pin_update(TSC2005State *s)
     s->precision = s->nextprecision;
     s->function = s->nextfunction;
     s->pdst = !s->pnd0;	/* Synchronised on internal clock */
-    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() >> 7);
+    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+        (NANOSECONDS_PER_SECOND >> 7);
     timer_mod(s->timer, expires);
 }
 
diff --git a/qemu/hw/input/tsc210x.c b/qemu/hw/input/tsc210x.c
index fae338563..93ca374fc 100644
--- a/qemu/hw/input/tsc210x.c
+++ b/qemu/hw/input/tsc210x.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "qemu/timer.h"
@@ -834,7 +835,8 @@ static void tsc210x_pin_update(TSC210xState *s)
     s->busy = 1;
     s->precision = s->nextprecision;
     s->function = s->nextfunction;
-    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() >> 10);
+    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+        (NANOSECONDS_PER_SECOND >> 10);
     timer_mod(s->timer, expires);
 }
 
@@ -1086,9 +1088,7 @@ uWireSlave *tsc2102_init(qemu_irq pint)
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 160;
     s->y = 160;
     s->pressure = 0;
@@ -1135,9 +1135,7 @@ uWireSlave *tsc2301_init(qemu_irq penirq, qemu_irq kbirq, qemu_irq dav)
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 400;
     s->y = 240;
     s->pressure = 0;
diff --git a/qemu/hw/input/virtio-input-hid.c b/qemu/hw/input/virtio-input-hid.c
index 4d85dad4d..3ee0c1814 100644
--- a/qemu/hw/input/virtio-input-hid.c
+++ b/qemu/hw/input/virtio-input-hid.c
@@ -4,6 +4,7 @@
  * top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 
 #include "hw/qdev.h"
@@ -21,7 +22,7 @@
 
 /* ----------------------------------------------------------------- */
 
-static const unsigned int keymap_qcode[Q_KEY_CODE_MAX] = {
+static const unsigned int keymap_qcode[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_ESC]                 = KEY_ESC,
     [Q_KEY_CODE_1]                   = KEY_1,
     [Q_KEY_CODE_2]                   = KEY_2,
@@ -120,6 +121,8 @@ static const unsigned int keymap_qcode[Q_KEY_CODE_MAX] = {
 
     [Q_KEY_CODE_CTRL_R]              = KEY_RIGHTCTRL,
     [Q_KEY_CODE_SYSRQ]               = KEY_SYSRQ,
+    [Q_KEY_CODE_PRINT]               = KEY_SYSRQ,
+    [Q_KEY_CODE_PAUSE]               = KEY_PAUSE,
     [Q_KEY_CODE_ALT_R]               = KEY_RIGHTALT,
 
     [Q_KEY_CODE_HOME]                = KEY_HOME,
@@ -138,7 +141,7 @@ static const unsigned int keymap_qcode[Q_KEY_CODE_MAX] = {
     [Q_KEY_CODE_MENU]                = KEY_MENU,
 };
 
-static const unsigned int keymap_button[INPUT_BUTTON_MAX] = {
+static const unsigned int keymap_button[INPUT_BUTTON__MAX] = {
     [INPUT_BUTTON_LEFT]              = BTN_LEFT,
     [INPUT_BUTTON_RIGHT]             = BTN_RIGHT,
     [INPUT_BUTTON_MIDDLE]            = BTN_MIDDLE,
@@ -146,12 +149,12 @@ static const unsigned int keymap_button[INPUT_BUTTON_MAX] = {
     [INPUT_BUTTON_WHEEL_DOWN]        = BTN_GEAR_DOWN,
 };
 
-static const unsigned int axismap_rel[INPUT_AXIS_MAX] = {
+static const unsigned int axismap_rel[INPUT_AXIS__MAX] = {
     [INPUT_AXIS_X]                   = REL_X,
     [INPUT_AXIS_Y]                   = REL_Y,
 };
 
-static const unsigned int axismap_abs[INPUT_AXIS_MAX] = {
+static const unsigned int axismap_abs[INPUT_AXIS__MAX] = {
     [INPUT_AXIS_X]                   = ABS_X,
     [INPUT_AXIS_Y]                   = ABS_Y,
 };
@@ -190,45 +193,53 @@ static void virtio_input_handle_event(DeviceState *dev, QemuConsole *src,
     VirtIOInput *vinput = VIRTIO_INPUT(dev);
     virtio_input_event event;
     int qcode;
+    InputKeyEvent *key;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_KEY:
-        qcode = qemu_input_key_value_to_qcode(evt->key->key);
+        key = evt->u.key.data;
+        qcode = qemu_input_key_value_to_qcode(key->key);
         if (qcode && keymap_qcode[qcode]) {
             event.type  = cpu_to_le16(EV_KEY);
             event.code  = cpu_to_le16(keymap_qcode[qcode]);
-            event.value = cpu_to_le32(evt->key->down ? 1 : 0);
+            event.value = cpu_to_le32(key->down ? 1 : 0);
             virtio_input_send(vinput, &event);
         } else {
-            if (evt->key->down) {
+            if (key->down) {
                 fprintf(stderr, "%s: unmapped key: %d [%s]\n", __func__,
                         qcode, QKeyCode_lookup[qcode]);
             }
         }
         break;
     case INPUT_EVENT_KIND_BTN:
-        if (keymap_button[evt->btn->button]) {
+        btn = evt->u.btn.data;
+        if (keymap_button[btn->button]) {
             event.type  = cpu_to_le16(EV_KEY);
-            event.code  = cpu_to_le16(keymap_button[evt->btn->button]);
-            event.value = cpu_to_le32(evt->btn->down ? 1 : 0);
+            event.code  = cpu_to_le16(keymap_button[btn->button]);
+            event.value = cpu_to_le32(btn->down ? 1 : 0);
             virtio_input_send(vinput, &event);
         } else {
-            if (evt->btn->down) {
+            if (btn->down) {
                 fprintf(stderr, "%s: unmapped button: %d [%s]\n", __func__,
-                        evt->btn->button, InputButton_lookup[evt->btn->button]);
+                        btn->button,
+                        InputButton_lookup[btn->button]);
             }
         }
         break;
     case INPUT_EVENT_KIND_REL:
+        move = evt->u.rel.data;
         event.type  = cpu_to_le16(EV_REL);
-        event.code  = cpu_to_le16(axismap_rel[evt->rel->axis]);
-        event.value = cpu_to_le32(evt->rel->value);
+        event.code  = cpu_to_le16(axismap_rel[move->axis]);
+        event.value = cpu_to_le32(move->value);
         virtio_input_send(vinput, &event);
         break;
     case INPUT_EVENT_KIND_ABS:
+        move = evt->u.abs.data;
         event.type  = cpu_to_le16(EV_ABS);
-        event.code  = cpu_to_le16(axismap_abs[evt->abs->axis]);
-        event.value = cpu_to_le32(evt->abs->value);
+        event.code  = cpu_to_le16(axismap_abs[move->axis]);
+        event.value = cpu_to_le32(move->value);
         virtio_input_send(vinput, &event);
         break;
     default:
@@ -473,12 +484,12 @@ static struct virtio_input_config virtio_tablet_config[] = {
         .select    = VIRTIO_INPUT_CFG_ABS_INFO,
         .subsel    = ABS_X,
         .size      = sizeof(virtio_input_absinfo),
-        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE),
+        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE - 1),
     },{
         .select    = VIRTIO_INPUT_CFG_ABS_INFO,
         .subsel    = ABS_Y,
         .size      = sizeof(virtio_input_absinfo),
-        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE),
+        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE - 1),
     },
     { /* end of list */ },
 };
diff --git a/qemu/hw/input/virtio-input-host.c b/qemu/hw/input/virtio-input-host.c
index 8978f16ba..cb79e8002 100644
--- a/qemu/hw/input/virtio-input-host.c
+++ b/qemu/hw/input/virtio-input-host.c
@@ -4,6 +4,8 @@
  * top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/sockets.h"
 
@@ -68,13 +70,39 @@ static void virtio_input_bits_config(VirtIOInputHost *vih,
     virtio_input_add_config(VIRTIO_INPUT(vih), &bits);
 }
 
+static void virtio_input_abs_config(VirtIOInputHost *vih, int axis)
+{
+    virtio_input_config config;
+    struct input_absinfo absinfo;
+    int rc;
+
+    rc = ioctl(vih->fd, EVIOCGABS(axis), &absinfo);
+    if (rc < 0) {
+        return;
+    }
+
+    memset(&config, 0, sizeof(config));
+    config.select = VIRTIO_INPUT_CFG_ABS_INFO;
+    config.subsel = axis;
+    config.size   = sizeof(virtio_input_absinfo);
+
+    config.u.abs.min  = cpu_to_le32(absinfo.minimum);
+    config.u.abs.max  = cpu_to_le32(absinfo.maximum);
+    config.u.abs.fuzz = cpu_to_le32(absinfo.fuzz);
+    config.u.abs.flat = cpu_to_le32(absinfo.flat);
+    config.u.abs.res  = cpu_to_le32(absinfo.resolution);
+
+    virtio_input_add_config(VIRTIO_INPUT(vih), &config);
+}
+
 static void virtio_input_host_realize(DeviceState *dev, Error **errp)
 {
     VirtIOInputHost *vih = VIRTIO_INPUT_HOST(dev);
     VirtIOInput *vinput = VIRTIO_INPUT(dev);
-    virtio_input_config id;
+    virtio_input_config id, *abs;
     struct input_id ids;
-    int rc, ver;
+    int rc, ver, i, axis;
+    uint8_t byte;
 
     if (!vih->evdev) {
         error_setg(errp, "evdev property is required");
@@ -123,6 +151,23 @@ static void virtio_input_host_realize(DeviceState *dev, Error **errp)
     virtio_input_bits_config(vih, EV_ABS, ABS_CNT);
     virtio_input_bits_config(vih, EV_MSC, MSC_CNT);
     virtio_input_bits_config(vih, EV_SW,  SW_CNT);
+    virtio_input_bits_config(vih, EV_LED, LED_CNT);
+
+    abs = virtio_input_find_config(VIRTIO_INPUT(vih),
+        VIRTIO_INPUT_CFG_EV_BITS, EV_ABS);
+    if (abs) {
+        for (i = 0; i < abs->size; i++) {
+            byte = abs->u.bitmap[i];
+            axis = 8 * i;
+            while (byte) {
+                if (byte & 1) {
+                    virtio_input_abs_config(vih, axis);
+                }
+                axis++;
+                byte >>= 1;
+            }
+        }
+    }
 
     qemu_set_fd_handler(vih->fd, virtio_input_host_event, NULL, vih);
     return;
@@ -143,6 +188,28 @@ static void virtio_input_host_unrealize(DeviceState *dev, Error **errp)
     }
 }
 
+static void virtio_input_host_handle_status(VirtIOInput *vinput,
+                                            virtio_input_event *event)
+{
+    VirtIOInputHost *vih = VIRTIO_INPUT_HOST(vinput);
+    struct input_event evdev;
+    int rc;
+
+    if (gettimeofday(&evdev.time, NULL)) {
+        perror("virtio_input_host_handle_status: gettimeofday");
+        return;
+    }
+
+    evdev.type = le16_to_cpu(event->type);
+    evdev.code = le16_to_cpu(event->code);
+    evdev.value = le32_to_cpu(event->value);
+
+    rc = write(vih->fd, &evdev, sizeof(evdev));
+    if (rc == -1) {
+        perror("virtio_input_host_handle_status: write");
+    }
+}
+
 static const VMStateDescription vmstate_virtio_input_host = {
     .name = "virtio-input-host",
     .unmigratable = 1,
@@ -162,6 +229,7 @@ static void virtio_input_host_class_init(ObjectClass *klass, void *data)
     dc->props          = virtio_input_host_properties;
     vic->realize       = virtio_input_host_realize;
     vic->unrealize     = virtio_input_host_unrealize;
+    vic->handle_status = virtio_input_host_handle_status;
 }
 
 static void virtio_input_host_init(Object *obj)
diff --git a/qemu/hw/input/virtio-input.c b/qemu/hw/input/virtio-input.c
index 7b25d2769..f59749a94 100644
--- a/qemu/hw/input/virtio-input.c
+++ b/qemu/hw/input/virtio-input.c
@@ -4,6 +4,8 @@
  * top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
 
 #include "hw/qdev.h"
@@ -12,14 +14,20 @@
 
 #include "standard-headers/linux/input.h"
 
+#define VIRTIO_INPUT_VM_VERSION 1
+
 /* ----------------------------------------------------------------- */
 
 void virtio_input_send(VirtIOInput *vinput, virtio_input_event *event)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     unsigned have, need;
     int i, len;
 
+    if (!vinput->active) {
+        return;
+    }
+
     /* queue up events ... */
     if (vinput->qindex == vinput->qsize) {
         vinput->qsize++;
@@ -45,14 +53,16 @@ void virtio_input_send(VirtIOInput *vinput, virtio_input_event *event)
 
     /* ... and finally pass them to the guest */
     for (i = 0; i < vinput->qindex; i++) {
-        if (!virtqueue_pop(vinput->evt, &elem)) {
+        elem = virtqueue_pop(vinput->evt, sizeof(VirtQueueElement));
+        if (!elem) {
             /* should not happen, we've checked for space beforehand */
             fprintf(stderr, "%s: Huh?  No vq elem available ...\n", __func__);
             return;
         }
-        len = iov_from_buf(elem.in_sg, elem.in_num,
+        len = iov_from_buf(elem->in_sg, elem->in_num,
                            0, vinput->queue+i, sizeof(virtio_input_event));
-        virtqueue_push(vinput->evt, &elem, len);
+        virtqueue_push(vinput->evt, elem, len);
+        g_free(elem);
     }
     virtio_notify(VIRTIO_DEVICE(vinput), vinput->evt);
     vinput->qindex = 0;
@@ -68,24 +78,30 @@ static void virtio_input_handle_sts(VirtIODevice *vdev, VirtQueue *vq)
     VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vdev);
     VirtIOInput *vinput = VIRTIO_INPUT(vdev);
     virtio_input_event event;
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     int len;
 
-    while (virtqueue_pop(vinput->sts, &elem)) {
+    for (;;) {
+        elem = virtqueue_pop(vinput->sts, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
         memset(&event, 0, sizeof(event));
-        len = iov_to_buf(elem.out_sg, elem.out_num,
+        len = iov_to_buf(elem->out_sg, elem->out_num,
                          0, &event, sizeof(event));
         if (vic->handle_status) {
             vic->handle_status(vinput, &event);
         }
-        virtqueue_push(vinput->sts, &elem, len);
+        virtqueue_push(vinput->sts, elem, len);
+        g_free(elem);
     }
     virtio_notify(vdev, vinput->sts);
 }
 
-static virtio_input_config *virtio_input_find_config(VirtIOInput *vinput,
-                                                     uint8_t select,
-                                                     uint8_t subsel)
+virtio_input_config *virtio_input_find_config(VirtIOInput *vinput,
+                                              uint8_t select,
+                                              uint8_t subsel)
 {
     VirtIOInputConfig *cfg;
 
@@ -200,6 +216,38 @@ static void virtio_input_reset(VirtIODevice *vdev)
     }
 }
 
+static void virtio_input_save(QEMUFile *f, void *opaque)
+{
+    VirtIOInput *vinput = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(vinput);
+
+    virtio_save(vdev, f);
+}
+
+static int virtio_input_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIOInput *vinput = opaque;
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vinput);
+    VirtIODevice *vdev = VIRTIO_DEVICE(vinput);
+    int ret;
+
+    if (version_id != VIRTIO_INPUT_VM_VERSION) {
+        return -EINVAL;
+    }
+
+    ret = virtio_load(vdev, f, version_id);
+    if (ret) {
+        return ret;
+    }
+
+    /* post_load() */
+    vinput->active = vdev->status & VIRTIO_CONFIG_S_DRIVER_OK;
+    if (vic->change_active) {
+        vic->change_active(vinput);
+    }
+    return 0;
+}
+
 static void virtio_input_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(dev);
@@ -231,14 +279,20 @@ static void virtio_input_device_realize(DeviceState *dev, Error **errp)
                 vinput->cfg_size);
     vinput->evt = virtio_add_queue(vdev, 64, virtio_input_handle_evt);
     vinput->sts = virtio_add_queue(vdev, 64, virtio_input_handle_sts);
+
+    register_savevm(dev, "virtio-input", -1, VIRTIO_INPUT_VM_VERSION,
+                    virtio_input_save, virtio_input_load, vinput);
 }
 
 static void virtio_input_device_unrealize(DeviceState *dev, Error **errp)
 {
     VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(dev);
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VirtIOInput *vinput = VIRTIO_INPUT(dev);
     Error *local_err = NULL;
 
+    unregister_savevm(dev, "virtio-input", vinput);
+
     if (vic->unrealize) {
         vic->unrealize(dev, &local_err);
         if (local_err) {
diff --git a/qemu/hw/input/vmmouse.c b/qemu/hw/input/vmmouse.c
index d7b1c76f5..6d15a887c 100644
--- a/qemu/hw/input/vmmouse.c
+++ b/qemu/hw/input/vmmouse.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/input/ps2.h"
diff --git a/qemu/hw/intc/Makefile.objs b/qemu/hw/intc/Makefile.objs
index 092d8a80a..0e47f0f9e 100644
--- a/qemu/hw/intc/Makefile.objs
+++ b/qemu/hw/intc/Makefile.objs
@@ -12,19 +12,23 @@ common-obj-$(CONFIG_IOAPIC) += ioapic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gicv2m.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_common.o
 common-obj-$(CONFIG_OPENPIC) += openpic.o
 
 obj-$(CONFIG_APIC) += apic.o apic_common.o
 obj-$(CONFIG_ARM_GIC_KVM) += arm_gic_kvm.o
+obj-$(call land,$(CONFIG_ARM_GIC_KVM),$(TARGET_AARCH64)) += arm_gicv3_kvm.o
 obj-$(CONFIG_STELLARIS) += armv7m_nvic.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_gic.o exynos4210_combiner.o
 obj-$(CONFIG_GRLIB) += grlib_irqmp.o
 obj-$(CONFIG_IOAPIC) += ioapic.o
 obj-$(CONFIG_OMAP) += omap_intc.o
 obj-$(CONFIG_OPENPIC_KVM) += openpic_kvm.o
+obj-$(CONFIG_RASPI) += bcm2835_ic.o bcm2836_control.o
 obj-$(CONFIG_SH4) += sh_intc.o
 obj-$(CONFIG_XICS) += xics.o
 obj-$(CONFIG_XICS_KVM) += xics_kvm.o
 obj-$(CONFIG_ALLWINNER_A10_PIC) += allwinner-a10-pic.o
 obj-$(CONFIG_S390_FLIC) += s390_flic.o
 obj-$(CONFIG_S390_FLIC_KVM) += s390_flic_kvm.o
+obj-$(CONFIG_ASPEED_SOC) += aspeed_vic.o
diff --git a/qemu/hw/intc/allwinner-a10-pic.c b/qemu/hw/intc/allwinner-a10-pic.c
index eed7621f1..dc971a160 100644
--- a/qemu/hw/intc/allwinner-a10-pic.c
+++ b/qemu/hw/intc/allwinner-a10-pic.c
@@ -15,6 +15,7 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/hw/intc/apic.c b/qemu/hw/intc/apic.c
index 77b639cce..28c2ea540 100644
--- a/qemu/hw/intc/apic.c
+++ b/qemu/hw/intc/apic.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
+#include "qemu/osdep.h"
 #include "qemu/thread.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/i386/apic.h"
@@ -51,14 +52,6 @@ static int apic_ffs_bit(uint32_t value)
     return ctz32(value);
 }
 
-static inline void apic_set_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    tab[i] |= mask;
-}
-
 static inline void apic_reset_bit(uint32_t *tab, int index)
 {
     int i, mask;
@@ -67,14 +60,6 @@ static inline void apic_reset_bit(uint32_t *tab, int index)
     tab[i] &= ~mask;
 }
 
-static inline int apic_get_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    return !!(tab[i] & mask);
-}
-
 /* return -1 if no bit is set */
 static int get_highest_priority_int(uint32_t *tab)
 {
@@ -318,7 +303,7 @@ static uint8_t apic_get_tpr(APICCommonState *s)
     return s->tpr >> 4;
 }
 
-static int apic_get_ppr(APICCommonState *s)
+int apic_get_ppr(APICCommonState *s)
 {
     int tpr, isrv, ppr;
 
@@ -739,7 +724,7 @@ static uint32_t apic_mem_readl(void *opaque, hwaddr addr)
         val = s->divide_conf;
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         val = 0;
         break;
     }
@@ -852,7 +837,7 @@ static void apic_mem_writel(void *opaque, hwaddr addr, uint32_t val)
         }
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         break;
     }
 }
@@ -889,7 +874,7 @@ static void apic_realize(DeviceState *dev, Error **errp)
     s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, apic_timer, s);
     local_apics[s->idx] = s;
 
-    msi_supported = true;
+    msi_nonbroken = true;
 }
 
 static void apic_class_init(ObjectClass *klass, void *data)
diff --git a/qemu/hw/intc/apic_common.c b/qemu/hw/intc/apic_common.c
index 0032b97c5..4abe145c6 100644
--- a/qemu/hw/intc/apic_common.c
+++ b/qemu/hw/intc/apic_common.c
@@ -17,6 +17,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "trace.h"
@@ -296,7 +298,6 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
     APICCommonClass *info;
     static DeviceState *vapic;
     static int apic_no;
-    static bool mmio_registered;
 
     if (apic_no >= MAX_APICS) {
         error_setg(errp, "%s initialization failed.",
@@ -307,11 +308,6 @@ static void apic_common_realize(DeviceState *dev, Error **errp)
 
     info = APIC_COMMON_GET_CLASS(s);
     info->realize(dev, errp);
-    if (!mmio_registered) {
-        ICCBus *b = ICC_BUS(qdev_get_parent_bus(dev));
-        memory_region_add_subregion(b->apic_address_space, 0, &s->io_memory);
-        mmio_registered = true;
-    }
 
     /* Note: We need at least 1M to map the VAPIC option ROM */
     if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK &&
@@ -425,13 +421,12 @@ static Property apic_properties_common[] = {
 
 static void apic_common_class_init(ObjectClass *klass, void *data)
 {
-    ICCDeviceClass *idc = ICC_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->vmsd = &vmstate_apic_common;
     dc->reset = apic_reset_common;
     dc->props = apic_properties_common;
-    idc->realize = apic_common_realize;
+    dc->realize = apic_common_realize;
     /*
      * Reason: APIC and CPU need to be wired up by
      * x86_cpu_apic_create()
@@ -441,7 +436,7 @@ static void apic_common_class_init(ObjectClass *klass, void *data)
 
 static const TypeInfo apic_common_type = {
     .name = TYPE_APIC_COMMON,
-    .parent = TYPE_ICC_DEVICE,
+    .parent = TYPE_DEVICE,
     .instance_size = sizeof(APICCommonState),
     .class_size = sizeof(APICCommonClass),
     .class_init = apic_common_class_init,
diff --git a/qemu/hw/intc/arm_gic.c b/qemu/hw/intc/arm_gic.c
index 454bfd7df..f55124174 100644
--- a/qemu/hw/intc/arm_gic.c
+++ b/qemu/hw/intc/arm_gic.c
@@ -18,8 +18,10 @@
  *  armv7m_nvic device.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "gic_internal.h"
+#include "qapi/error.h"
 #include "qom/cpu.h"
 
 //#define DEBUG_GIC
@@ -31,11 +33,17 @@ do { fprintf(stderr, "arm_gic: " fmt , ## __VA_ARGS__); } while (0)
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-static const uint8_t gic_id[] = {
-    0x90, 0x13, 0x04, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+static const uint8_t gic_id_11mpcore[] = {
+    0x00, 0x00, 0x00, 0x00, 0x90, 0x13, 0x04, 0x00, 0x0d, 0xf0, 0x05, 0xb1
 };
 
-#define NUM_CPU(s) ((s)->num_cpu)
+static const uint8_t gic_id_gicv1[] = {
+    0x04, 0x00, 0x00, 0x00, 0x90, 0xb3, 0x1b, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+};
+
+static const uint8_t gic_id_gicv2[] = {
+    0x04, 0x00, 0x00, 0x00, 0x90, 0xb4, 0x2b, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+};
 
 static inline int gic_get_current_cpu(GICState *s)
 {
@@ -64,7 +72,7 @@ void gic_update(GICState *s)
     int cpu;
     int cm;
 
-    for (cpu = 0; cpu < NUM_CPU(s); cpu++) {
+    for (cpu = 0; cpu < s->num_cpu; cpu++) {
         cm = 1 << cpu;
         s->current_pending[cpu] = 1023;
         if (!(s->ctlr & (GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1))
@@ -219,15 +227,99 @@ static uint16_t gic_get_current_pending_irq(GICState *s, int cpu,
     return pending_irq;
 }
 
-static void gic_set_running_irq(GICState *s, int cpu, int irq)
+static int gic_get_group_priority(GICState *s, int cpu, int irq)
 {
-    s->running_irq[cpu] = irq;
-    if (irq == 1023) {
-        s->running_priority[cpu] = 0x100;
+    /* Return the group priority of the specified interrupt
+     * (which is the top bits of its priority, with the number
+     * of bits masked determined by the applicable binary point register).
+     */
+    int bpr;
+    uint32_t mask;
+
+    if (gic_has_groups(s) &&
+        !(s->cpu_ctlr[cpu] & GICC_CTLR_CBPR) &&
+        GIC_TEST_GROUP(irq, (1 << cpu))) {
+        bpr = s->abpr[cpu];
     } else {
-        s->running_priority[cpu] = GIC_GET_PRIORITY(irq, cpu);
+        bpr = s->bpr[cpu];
     }
-    gic_update(s);
+
+    /* a BPR of 0 means the group priority bits are [7:1];
+     * a BPR of 1 means they are [7:2], and so on down to
+     * a BPR of 7 meaning no group priority bits at all.
+     */
+    mask = ~0U << ((bpr & 7) + 1);
+
+    return GIC_GET_PRIORITY(irq, cpu) & mask;
+}
+
+static void gic_activate_irq(GICState *s, int cpu, int irq)
+{
+    /* Set the appropriate Active Priority Register bit for this IRQ,
+     * and update the running priority.
+     */
+    int prio = gic_get_group_priority(s, cpu, irq);
+    int preemption_level = prio >> (GIC_MIN_BPR + 1);
+    int regno = preemption_level / 32;
+    int bitno = preemption_level % 32;
+
+    if (gic_has_groups(s) && GIC_TEST_GROUP(irq, (1 << cpu))) {
+        s->nsapr[regno][cpu] |= (1 << bitno);
+    } else {
+        s->apr[regno][cpu] |= (1 << bitno);
+    }
+
+    s->running_priority[cpu] = prio;
+    GIC_SET_ACTIVE(irq, 1 << cpu);
+}
+
+static int gic_get_prio_from_apr_bits(GICState *s, int cpu)
+{
+    /* Recalculate the current running priority for this CPU based
+     * on the set bits in the Active Priority Registers.
+     */
+    int i;
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t apr = s->apr[i][cpu] | s->nsapr[i][cpu];
+        if (!apr) {
+            continue;
+        }
+        return (i * 32 + ctz32(apr)) << (GIC_MIN_BPR + 1);
+    }
+    return 0x100;
+}
+
+static void gic_drop_prio(GICState *s, int cpu, int group)
+{
+    /* Drop the priority of the currently active interrupt in the
+     * specified group.
+     *
+     * Note that we can guarantee (because of the requirement to nest
+     * GICC_IAR reads [which activate an interrupt and raise priority]
+     * with GICC_EOIR writes [which drop the priority for the interrupt])
+     * that the interrupt we're being called for is the highest priority
+     * active interrupt, meaning that it has the lowest set bit in the
+     * APR registers.
+     *
+     * If the guest does not honour the ordering constraints then the
+     * behaviour of the GIC is UNPREDICTABLE, which for us means that
+     * the values of the APR registers might become incorrect and the
+     * running priority will be wrong, so interrupts that should preempt
+     * might not do so, and interrupts that should not preempt might do so.
+     */
+    int i;
+
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t *papr = group ? &s->nsapr[i][cpu] : &s->apr[i][cpu];
+        if (!*papr) {
+            continue;
+        }
+        /* Clear lowest set bit */
+        *papr &= *papr - 1;
+        break;
+    }
+
+    s->running_priority[cpu] = gic_get_prio_from_apr_bits(s, cpu);
 }
 
 uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
@@ -239,7 +331,7 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
      * for the case where this GIC supports grouping and the pending interrupt
      * is in the wrong group.
      */
-    irq = gic_get_current_pending_irq(s, cpu, attrs);;
+    irq = gic_get_current_pending_irq(s, cpu, attrs);
 
     if (irq >= GIC_MAXIRQ) {
         DPRINTF("ACK, no pending interrupt or it is hidden: %d\n", irq);
@@ -250,7 +342,6 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
         DPRINTF("ACK, pending interrupt (%d) has insufficient priority\n", irq);
         return 1023;
     }
-    s->last_active[irq][cpu] = s->running_irq[cpu];
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Clear pending flags for both level and edge triggered interrupts.
@@ -281,7 +372,8 @@ uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
         }
     }
 
-    gic_set_running_irq(s, cpu, irq);
+    gic_activate_irq(s, cpu, irq);
+    gic_update(s);
     DPRINTF("ACK %d\n", irq);
     return ret;
 }
@@ -409,10 +501,46 @@ static uint8_t gic_get_running_priority(GICState *s, int cpu, MemTxAttrs attrs)
     }
 }
 
+/* Return true if we should split priority drop and interrupt deactivation,
+ * ie whether the relevant EOIMode bit is set.
+ */
+static bool gic_eoi_split(GICState *s, int cpu, MemTxAttrs attrs)
+{
+    if (s->revision != 2) {
+        /* Before GICv2 prio-drop and deactivate are not separable */
+        return false;
+    }
+    if (s->security_extn && !attrs.secure) {
+        return s->cpu_ctlr[cpu] & GICC_CTLR_EOIMODE_NS;
+    }
+    return s->cpu_ctlr[cpu] & GICC_CTLR_EOIMODE;
+}
+
+static void gic_deactivate_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
+{
+    int cm = 1 << cpu;
+    int group = gic_has_groups(s) && GIC_TEST_GROUP(irq, cm);
+
+    if (!gic_eoi_split(s, cpu, attrs)) {
+        /* This is UNPREDICTABLE; we choose to ignore it */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_deactivate_irq: GICC_DIR write when EOIMode clear");
+        return;
+    }
+
+    if (s->security_extn && !attrs.secure && !group) {
+        DPRINTF("Non-secure DI for Group0 interrupt %d ignored\n", irq);
+        return;
+    }
+
+    GIC_CLEAR_ACTIVE(irq, cm);
+}
+
 void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
 {
-    int update = 0;
     int cm = 1 << cpu;
+    int group;
+
     DPRINTF("EOI %d\n", irq);
     if (irq >= s->num_irq) {
         /* This handles two cases:
@@ -425,8 +553,9 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
          */
         return;
     }
-    if (s->running_irq[cpu] == 1023)
+    if (s->running_priority[cpu] == 0x100) {
         return; /* No active IRQ.  */
+    }
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Mark level triggered interrupts as pending if they are still
@@ -435,11 +564,12 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
             && GIC_TEST_LEVEL(irq, cm) && (GIC_TARGET(irq) & cm) != 0) {
             DPRINTF("Set %d pending mask %x\n", irq, cm);
             GIC_SET_PENDING(irq, cm);
-            update = 1;
         }
     }
 
-    if (s->security_extn && !attrs.secure && !GIC_TEST_GROUP(irq, cm)) {
+    group = gic_has_groups(s) && GIC_TEST_GROUP(irq, cm);
+
+    if (s->security_extn && !attrs.secure && !group) {
         DPRINTF("Non-secure EOI for Group0 interrupt %d ignored\n", irq);
         return;
     }
@@ -449,23 +579,13 @@ void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
      * i.e. go ahead and complete the irq anyway.
      */
 
-    if (irq != s->running_irq[cpu]) {
-        /* Complete an IRQ that is not currently running.  */
-        int tmp = s->running_irq[cpu];
-        while (s->last_active[tmp][cpu] != 1023) {
-            if (s->last_active[tmp][cpu] == irq) {
-                s->last_active[tmp][cpu] = s->last_active[irq][cpu];
-                break;
-            }
-            tmp = s->last_active[tmp][cpu];
-        }
-        if (update) {
-            gic_update(s);
-        }
-    } else {
-        /* Complete the current running IRQ.  */
-        gic_set_running_irq(s, cpu, s->last_active[s->running_irq[cpu]][cpu]);
+    gic_drop_prio(s, cpu, group);
+
+    /* In GICv2 the guest can choose to split priority-drop and deactivate */
+    if (!gic_eoi_split(s, cpu, attrs)) {
+        GIC_CLEAR_ACTIVE(irq, cm);
     }
+    gic_update(s);
 }
 
 static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
@@ -494,7 +614,7 @@ static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
         if (offset == 4)
             /* Interrupt Controller Type Register */
             return ((s->num_irq / 32) - 1)
-                    | ((NUM_CPU(s) - 1) << 5)
+                    | ((s->num_cpu - 1) << 5)
                     | (s->security_extn << 10);
         if (offset < 0x08)
             return 0;
@@ -612,14 +732,31 @@ static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
         }
 
         res = s->sgi_pending[irq][cpu];
-    } else if (offset < 0xfe0) {
+    } else if (offset < 0xfd0) {
         goto bad_reg;
-    } else /* offset >= 0xfe0 */ {
+    } else if (offset < 0x1000) {
         if (offset & 3) {
             res = 0;
         } else {
-            res = gic_id[(offset - 0xfe0) >> 2];
+            switch (s->revision) {
+            case REV_11MPCORE:
+                res = gic_id_11mpcore[(offset - 0xfd0) >> 2];
+                break;
+            case 1:
+                res = gic_id_gicv1[(offset - 0xfd0) >> 2];
+                break;
+            case 2:
+                res = gic_id_gicv2[(offset - 0xfd0) >> 2];
+                break;
+            case REV_NVIC:
+                /* Shouldn't be able to get here */
+                abort();
+            default:
+                res = 0;
+            }
         }
+    } else {
+        g_assert_not_reached();
     }
     return res;
 bad_reg:
@@ -922,11 +1059,67 @@ static MemTxResult gic_dist_write(void *opaque, hwaddr offset, uint64_t data,
     }
 }
 
-static const MemoryRegionOps gic_dist_ops = {
-    .read_with_attrs = gic_dist_read,
-    .write_with_attrs = gic_dist_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
+static inline uint32_t gic_apr_ns_view(GICState *s, int cpu, int regno)
+{
+    /* Return the Nonsecure view of GICC_APR<regno>. This is the
+     * second half of GICC_NSAPR.
+     */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            return s->nsapr[regno + 2][cpu];
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            return s->nsapr[regno + 1][cpu];
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 16, 16);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 8, 8);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static inline void gic_apr_write_ns_view(GICState *s, int cpu, int regno,
+                                         uint32_t value)
+{
+    /* Write the Nonsecure view of GICC_APR<regno>. */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            s->nsapr[regno + 2][cpu] = value;
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            s->nsapr[regno + 1][cpu] = value;
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 16, 16, value);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 8, 8, value);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
 
 static MemTxResult gic_cpu_read(GICState *s, int cpu, int offset,
                                 uint64_t *data, MemTxAttrs attrs)
@@ -968,8 +1161,31 @@ static MemTxResult gic_cpu_read(GICState *s, int cpu, int offset,
         }
         break;
     case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        *data = s->apr[(offset - 0xd0) / 4][cpu];
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            *data = 0;
+        } else if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            *data = gic_apr_ns_view(s, regno, cpu);
+        } else {
+            *data = s->apr[regno][cpu];
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2 || !gic_has_groups(s) ||
+            (s->security_extn && !attrs.secure)) {
+            *data = 0;
+        } else {
+            *data = s->nsapr[regno][cpu];
+        }
         break;
+    }
     default:
         qemu_log_mask(LOG_GUEST_ERROR,
                       "gic_cpu_read: Bad offset %x\n", (int)offset);
@@ -1007,7 +1223,36 @@ static MemTxResult gic_cpu_write(GICState *s, int cpu, int offset,
         }
         break;
     case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        qemu_log_mask(LOG_UNIMP, "Writing APR not implemented\n");
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            gic_apr_write_ns_view(s, regno, cpu, value);
+        } else {
+            s->apr[regno][cpu] = value;
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (!gic_has_groups(s) || (s->security_extn && !attrs.secure)) {
+            return MEMTX_OK;
+        }
+        s->nsapr[regno][cpu] = value;
+        break;
+    }
+    case 0x1000:
+        /* GICC_DIR */
+        gic_deactivate_irq(s, cpu, value & 0x3ff, attrs);
         break;
     default:
         qemu_log_mask(LOG_GUEST_ERROR,
@@ -1056,10 +1301,17 @@ static MemTxResult gic_do_cpu_write(void *opaque, hwaddr addr,
     return gic_cpu_write(s, id, addr, value, attrs);
 }
 
-static const MemoryRegionOps gic_thiscpu_ops = {
-    .read_with_attrs = gic_thiscpu_read,
-    .write_with_attrs = gic_thiscpu_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+static const MemoryRegionOps gic_ops[2] = {
+    {
+        .read_with_attrs = gic_dist_read,
+        .write_with_attrs = gic_dist_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    },
+    {
+        .read_with_attrs = gic_thiscpu_read,
+        .write_with_attrs = gic_thiscpu_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    }
 };
 
 static const MemoryRegionOps gic_cpu_ops = {
@@ -1068,31 +1320,10 @@ static const MemoryRegionOps gic_cpu_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
+/* This function is used by nvic model */
 void gic_init_irqs_and_distributor(GICState *s)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
-    int i;
-
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    if (s->revision != REV_NVIC) {
-        i += (GIC_INTERNAL * s->num_cpu);
-    }
-    qdev_init_gpio_in(DEVICE(s), gic_set_irq, i);
-    for (i = 0; i < NUM_CPU(s); i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
-    }
-    for (i = 0; i < NUM_CPU(s); i++) {
-        sysbus_init_irq(sbd, &s->parent_fiq[i]);
-    }
-    memory_region_init_io(&s->iomem, OBJECT(s), &gic_dist_ops, s,
-                          "gic_dist", 0x1000);
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 }
 
 static void arm_gic_realize(DeviceState *dev, Error **errp)
@@ -1110,28 +1341,22 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    gic_init_irqs_and_distributor(s);
+    /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 
-    /* Memory regions for the CPU interfaces (NVIC doesn't have these):
-     * a region for "CPU interface for this core", then a region for
-     * "CPU interface for core 0", "for core 1", ...
+    /* Extra core-specific regions for the CPU interfaces. This is
+     * necessary for "franken-GIC" implementations, for example on
+     * Exynos 4.
      * NB that the memory region size of 0x100 applies for the 11MPCore
      * and also cores following the GIC v1 spec (ie A9).
      * GIC v2 defines a larger memory region (0x1000) so this will need
      * to be extended when we implement A15.
      */
-    memory_region_init_io(&s->cpuiomem[0], OBJECT(s), &gic_thiscpu_ops, s,
-                          "gic_cpu", 0x100);
-    for (i = 0; i < NUM_CPU(s); i++) {
+    for (i = 0; i < s->num_cpu; i++) {
         s->backref[i] = s;
         memory_region_init_io(&s->cpuiomem[i+1], OBJECT(s), &gic_cpu_ops,
                               &s->backref[i], "gic_cpu", 0x100);
-    }
-    /* Distributor */
-    sysbus_init_mmio(sbd, &s->iomem);
-    /* cpu interfaces (one for "current cpu" plus one per cpu) */
-    for (i = 0; i <= NUM_CPU(s); i++) {
-        sysbus_init_mmio(sbd, &s->cpuiomem[i]);
+        sysbus_init_mmio(sbd, &s->cpuiomem[i+1]);
     }
 }
 
diff --git a/qemu/hw/intc/arm_gic_common.c b/qemu/hw/intc/arm_gic_common.c
index a64d0714e..0a1f56af1 100644
--- a/qemu/hw/intc/arm_gic_common.c
+++ b/qemu/hw/intc/arm_gic_common.c
@@ -18,7 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "gic_internal.h"
+#include "hw/arm/linux-boot-if.h"
 
 static void gic_pre_save(void *opaque)
 {
@@ -59,8 +62,8 @@ static const VMStateDescription vmstate_gic_irq_state = {
 
 static const VMStateDescription vmstate_gic = {
     .name = "arm_gic",
-    .version_id = 10,
-    .minimum_version_id = 10,
+    .version_id = 12,
+    .minimum_version_id = 12,
     .pre_save = gic_pre_save,
     .post_load = gic_post_load,
     .fields = (VMStateField[]) {
@@ -71,19 +74,59 @@ static const VMStateDescription vmstate_gic = {
         VMSTATE_UINT8_ARRAY(irq_target, GICState, GIC_MAXIRQ),
         VMSTATE_UINT8_2DARRAY(priority1, GICState, GIC_INTERNAL, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(priority2, GICState, GIC_MAXIRQ - GIC_INTERNAL),
-        VMSTATE_UINT16_2DARRAY(last_active, GICState, GIC_MAXIRQ, GIC_NCPU),
         VMSTATE_UINT8_2DARRAY(sgi_pending, GICState, GIC_NR_SGIS, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(priority_mask, GICState, GIC_NCPU),
-        VMSTATE_UINT16_ARRAY(running_irq, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(running_priority, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(current_pending, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(bpr, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(abpr, GICState, GIC_NCPU),
         VMSTATE_UINT32_2DARRAY(apr, GICState, GIC_NR_APRS, GIC_NCPU),
+        VMSTATE_UINT32_2DARRAY(nsapr, GICState, GIC_NR_APRS, GIC_NCPU),
         VMSTATE_END_OF_LIST()
     }
 };
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i = s->num_irq - GIC_INTERNAL;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] SPIs
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    if (s->revision != REV_NVIC) {
+        i += (GIC_INTERNAL * s->num_cpu);
+    }
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_fiq[i]);
+    }
+
+    /* Distributor */
+    memory_region_init_io(&s->iomem, OBJECT(s), ops, s, "gic_dist", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    if (s->revision != REV_NVIC) {
+        /* This is the main CPU interface "for this core". It is always
+         * present because it is required by both software emulation and KVM.
+         * NVIC is not handled here because its CPU interface is different,
+         * neither it can use KVM.
+         */
+        memory_region_init_io(&s->cpuiomem[0], OBJECT(s), ops ? &ops[1] : NULL,
+                              s, "gic_cpu", s->revision == 2 ? 0x2000 : 0x100);
+        sysbus_init_mmio(sbd, &s->cpuiomem[0]);
+    }
+}
+
 static void arm_gic_common_realize(DeviceState *dev, Error **errp)
 {
     GICState *s = ARM_GIC_COMMON(dev);
@@ -124,21 +167,35 @@ static void arm_gic_common_reset(DeviceState *dev)
 {
     GICState *s = ARM_GIC_COMMON(dev);
     int i, j;
+    int resetprio;
+
+    /* If we're resetting a TZ-aware GIC as if secure firmware
+     * had set it up ready to start a kernel in non-secure,
+     * we need to set interrupt priorities to a "zero for the
+     * NS view" value. This is particularly critical for the
+     * priority_mask[] values, because if they are zero then NS
+     * code cannot ever rewrite the priority to anything else.
+     */
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        resetprio = 0x80;
+    } else {
+        resetprio = 0;
+    }
+
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
         if (s->revision == REV_11MPCORE) {
             s->priority_mask[i] = 0xf0;
         } else {
-            s->priority_mask[i] = 0;
+            s->priority_mask[i] = resetprio;
         }
         s->current_pending[i] = 1023;
-        s->running_irq[i] = 1023;
         s->running_priority[i] = 0x100;
         s->cpu_ctlr[i] = 0;
         s->bpr[i] = GIC_MIN_BPR;
         s->abpr[i] = GIC_MIN_ABPR;
         for (j = 0; j < GIC_INTERNAL; j++) {
-            s->priority1[j][i] = 0;
+            s->priority1[j][i] = resetprio;
         }
         for (j = 0; j < GIC_NR_SGIS; j++) {
             s->sgi_pending[j][i] = 0;
@@ -150,7 +207,7 @@ static void arm_gic_common_reset(DeviceState *dev)
     }
 
     for (i = 0; i < ARRAY_SIZE(s->priority2); i++) {
-        s->priority2[i] = 0;
+        s->priority2[i] = resetprio;
     }
 
     for (i = 0; i < GIC_MAXIRQ; i++) {
@@ -161,9 +218,32 @@ static void arm_gic_common_reset(DeviceState *dev)
             s->irq_target[i] = 0;
         }
     }
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        for (i = 0; i < GIC_MAXIRQ; i++) {
+            GIC_SET_GROUP(i, ALL_CPU_MASK);
+        }
+    }
+
     s->ctlr = 0;
 }
 
+static void arm_gic_common_linux_init(ARMLinuxBootIf *obj,
+                                      bool secure_boot)
+{
+    GICState *s = ARM_GIC_COMMON(obj);
+
+    if (s->security_extn && !secure_boot) {
+        /* We're directly booting a kernel into NonSecure. If this GIC
+         * implements the security extensions then we must configure it
+         * to have all the interrupts be NonSecure (this is a job that
+         * is done by the Secure boot firmware in real hardware, and in
+         * this mode QEMU is acting as a minimalist firmware-and-bootloader
+         * equivalent).
+         */
+        s->irq_reset_nonsecure = true;
+    }
+}
+
 static Property arm_gic_common_properties[] = {
     DEFINE_PROP_UINT32("num-cpu", GICState, num_cpu, 1),
     DEFINE_PROP_UINT32("num-irq", GICState, num_irq, 32),
@@ -180,11 +260,13 @@ static Property arm_gic_common_properties[] = {
 static void arm_gic_common_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_CLASS(klass);
 
     dc->reset = arm_gic_common_reset;
     dc->realize = arm_gic_common_realize;
     dc->props = arm_gic_common_properties;
     dc->vmsd = &vmstate_gic;
+    albifc->arm_linux_init = arm_gic_common_linux_init;
 }
 
 static const TypeInfo arm_gic_common_type = {
@@ -194,6 +276,10 @@ static const TypeInfo arm_gic_common_type = {
     .class_size = sizeof(ARMGICCommonClass),
     .class_init = arm_gic_common_class_init,
     .abstract = true,
+    .interfaces = (InterfaceInfo []) {
+        { TYPE_ARM_LINUX_BOOT_IF },
+        { },
+    },
 };
 
 static void register_types(void)
diff --git a/qemu/hw/intc/arm_gic_kvm.c b/qemu/hw/intc/arm_gic_kvm.c
index f56bff1af..bc85ab769 100644
--- a/qemu/hw/intc/arm_gic_kvm.c
+++ b/qemu/hw/intc/arm_gic_kvm.c
@@ -19,10 +19,14 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
+#include "migration/migration.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "gic_internal.h"
+#include "vgic_common.h"
 
 //#define DEBUG_GIC_KVM
 
@@ -52,7 +56,7 @@ typedef struct KVMARMGICClass {
     void (*parent_reset)(DeviceState *dev);
 } KVMARMGICClass;
 
-static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
 {
     /* Meaning of the 'irq' parameter:
      *  [0..N-1] : external interrupts
@@ -63,10 +67,9 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
      * has separate fields in the irq number for type,
      * CPU number and interrupt number.
      */
-    GICState *s = (GICState *)opaque;
     int kvm_irq, irqtype, cpu;
 
-    if (irq < (s->num_irq - GIC_INTERNAL)) {
+    if (irq < (num_irq - GIC_INTERNAL)) {
         /* External interrupt. The kernel numbers these like the GIC
          * hardware, with external interrupt IDs starting after the
          * internal ones.
@@ -77,7 +80,7 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
     } else {
         /* Internal interrupt: decode into (cpu, interrupt id) */
         irqtype = KVM_ARM_IRQ_TYPE_PPI;
-        irq -= (s->num_irq - GIC_INTERNAL);
+        irq -= (num_irq - GIC_INTERNAL);
         cpu = irq / GIC_INTERNAL;
         irq %= GIC_INTERNAL;
     }
@@ -87,69 +90,36 @@ static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
     kvm_set_irq(kvm_state, kvm_irq, !!level);
 }
 
-static bool kvm_arm_gic_can_save_restore(GICState *s)
-{
-    return s->dev_fd >= 0;
-}
-
-static bool kvm_gic_supports_attr(GICState *s, int group, int attrnum)
+static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
 {
-    struct kvm_device_attr attr = {
-        .group = group,
-        .attr = attrnum,
-        .flags = 0,
-    };
-
-    if (s->dev_fd == -1) {
-        return false;
-    }
+    GICState *s = (GICState *)opaque;
 
-    return kvm_device_ioctl(s->dev_fd, KVM_HAS_DEVICE_ATTR, &attr) == 0;
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
 }
 
-static void kvm_gic_access(GICState *s, int group, int offset,
-                                   int cpu, uint32_t *val, bool write)
+static bool kvm_arm_gic_can_save_restore(GICState *s)
 {
-    struct kvm_device_attr attr;
-    int type;
-    int err;
-
-    cpu = cpu & 0xff;
-
-    attr.flags = 0;
-    attr.group = group;
-    attr.attr = (((uint64_t)cpu << KVM_DEV_ARM_VGIC_CPUID_SHIFT) &
-                 KVM_DEV_ARM_VGIC_CPUID_MASK) |
-                (((uint64_t)offset << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) &
-                 KVM_DEV_ARM_VGIC_OFFSET_MASK);
-    attr.addr = (uintptr_t)val;
-
-    if (write) {
-        type = KVM_SET_DEVICE_ATTR;
-    } else {
-        type = KVM_GET_DEVICE_ATTR;
-    }
-
-    err = kvm_device_ioctl(s->dev_fd, type, &attr);
-    if (err < 0) {
-        fprintf(stderr, "KVM_{SET/GET}_DEVICE_ATTR failed: %s\n",
-                strerror(-err));
-        abort();
-    }
+    return s->dev_fd >= 0;
 }
 
+#define KVM_VGIC_ATTR(offset, cpu) \
+    ((((uint64_t)(cpu) << KVM_DEV_ARM_VGIC_CPUID_SHIFT) & \
+      KVM_DEV_ARM_VGIC_CPUID_MASK) | \
+     (((uint64_t)(offset) << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) & \
+      KVM_DEV_ARM_VGIC_OFFSET_MASK))
+
 static void kvm_gicd_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 static void kvm_gicc_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 #define for_each_irq_reg(_ctr, _max_irq, _field_width) \
@@ -340,11 +310,6 @@ static void kvm_arm_gic_put(GICState *s)
     int num_cpu;
     int num_irq;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot put kernel gic state, no kernel interface");
-            return;
-    }
-
     /* Note: We do the restore in a slightly different order than the save
      * (where the order doesn't matter and is simply ordered according to the
      * register offset values */
@@ -444,11 +409,6 @@ static void kvm_arm_gic_get(GICState *s)
     int i;
     int cpu;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot get kernel gic state, no kernel interface");
-            return;
-    }
-
     /*****************************************************************
      * Distributor State
      */
@@ -536,14 +496,16 @@ static void kvm_arm_gic_reset(DeviceState *dev)
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
 
     kgc->parent_reset(dev);
-    kvm_arm_gic_put(s);
+
+    if (kvm_arm_gic_can_save_restore(s)) {
+        kvm_arm_gic_put(s);
+    }
 }
 
 static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
 {
     int i;
     GICState *s = KVM_ARM_GIC(dev);
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
     Error *local_err = NULL;
     int ret;
@@ -560,58 +522,37 @@ static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
         return;
     }
 
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    i += (GIC_INTERNAL * s->num_cpu);
-    qdev_init_gpio_in(dev, kvm_arm_gic_set_irq, i);
+    gic_init_irqs_and_mmio(s, kvm_arm_gicv2_set_irq, NULL);
 
     for (i = 0; i < s->num_irq - GIC_INTERNAL; i++) {
         qemu_irq irq = qdev_get_gpio_in(dev, i);
         kvm_irqchip_set_qemuirq_gsi(kvm_state, irq, i);
     }
 
-    /* We never use our outbound IRQ/FIQ lines but provide them so that
-     * we maintain the same interface as the non-KVM GIC.
-     */
-    for (i = 0; i < s->num_cpu; i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
-    }
-    for (i = 0; i < s->num_cpu; i++) {
-        sysbus_init_irq(sbd, &s->parent_fiq[i]);
-    }
-
     /* Try to create the device via the device control API */
     s->dev_fd = -1;
     ret = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V2, false);
     if (ret >= 0) {
         s->dev_fd = ret;
+
+        /* Newstyle API is used, we may have attributes */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
+            uint32_t numirqs = s->num_irq;
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0,
+                              &numirqs, true);
+        }
+        /* Tell the kernel to complete VGIC initialization now */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                                  KVM_DEV_ARM_VGIC_CTRL_INIT)) {
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                              KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+        }
     } else if (ret != -ENODEV && ret != -ENOTSUP) {
         error_setg_errno(errp, -ret, "error creating in-kernel VGIC");
         return;
     }
 
-    if (kvm_gic_supports_attr(s, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
-        uint32_t numirqs = s->num_irq;
-        kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, 0, &numirqs, 1);
-    }
-
-    /* Tell the kernel to complete VGIC initialization now */
-    if (kvm_gic_supports_attr(s, KVM_DEV_ARM_VGIC_GRP_CTRL,
-                              KVM_DEV_ARM_VGIC_CTRL_INIT)) {
-        kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_CTRL,
-                          KVM_DEV_ARM_VGIC_CTRL_INIT, 0, 0, 1);
-    }
-
     /* Distributor */
-    memory_region_init_reservation(&s->iomem, OBJECT(s),
-                                   "kvm-gic_dist", 0x1000);
-    sysbus_init_mmio(sbd, &s->iomem);
     kvm_arm_register_device(&s->iomem,
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_DIST,
@@ -622,15 +563,18 @@ static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
      * provide the "interface for core #N" memory regions, because
      * cores with a VGIC don't have those.
      */
-    memory_region_init_reservation(&s->cpuiomem[0], OBJECT(s),
-                                   "kvm-gic_cpu", 0x1000);
-    sysbus_init_mmio(sbd, &s->cpuiomem[0]);
     kvm_arm_register_device(&s->cpuiomem[0],
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_CPU,
                             KVM_DEV_ARM_VGIC_GRP_ADDR,
                             KVM_VGIC_V2_ADDR_TYPE_CPU,
                             s->dev_fd);
+
+    if (!kvm_arm_gic_can_save_restore(s)) {
+        error_setg(&s->migration_blocker, "This operating system kernel does "
+                                          "not support vGICv2 migration");
+        migrate_add_blocker(s->migration_blocker);
+    }
 }
 
 static void kvm_arm_gic_class_init(ObjectClass *klass, void *data)
diff --git a/qemu/hw/intc/arm_gicv2m.c b/qemu/hw/intc/arm_gicv2m.c
index 43d1976c4..e8b5177dc 100644
--- a/qemu/hw/intc/arm_gicv2m.c
+++ b/qemu/hw/intc/arm_gicv2m.c
@@ -25,6 +25,8 @@
  * identification registers and with a single non-secure MSI register frame.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
 #include "hw/pci/msi.h"
 
@@ -147,7 +149,7 @@ static void gicv2m_realize(DeviceState *dev, Error **errp)
         sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->spi[i]);
     }
 
-    msi_supported = true;
+    msi_nonbroken = true;
     kvm_gsi_direct_mapping = true;
     kvm_msi_via_irqfd_allowed = kvm_irqfds_enabled();
 }
diff --git a/qemu/hw/intc/arm_gicv3_common.c b/qemu/hw/intc/arm_gicv3_common.c
new file mode 100644
index 000000000..b9d3824f2
--- /dev/null
+++ b/qemu/hw/intc/arm_gicv3_common.c
@@ -0,0 +1,142 @@
+/*
+ * ARM GICv3 support - common bits of emulated and KVM kernel model
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Written by Peter Maydell
+ * Extended to 64 cores by Shlomo Pongratz
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/intc/arm_gicv3_common.h"
+
+static void gicv3_pre_save(void *opaque)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->pre_save) {
+        c->pre_save(s);
+    }
+}
+
+static int gicv3_post_load(void *opaque, int version_id)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->post_load) {
+        c->post_load(s);
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_gicv3 = {
+    .name = "arm_gicv3",
+    .unmigratable = 1,
+    .pre_save = gicv3_pre_save,
+    .post_load = gicv3_post_load,
+};
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] spi
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    i = s->num_irq - GIC_INTERNAL + GIC_INTERNAL * s->num_cpu;
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    s->parent_irq = g_malloc(s->num_cpu * sizeof(qemu_irq));
+    s->parent_fiq = g_malloc(s->num_cpu * sizeof(qemu_irq));
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_fiq[i]);
+    }
+
+    memory_region_init_io(&s->iomem_dist, OBJECT(s), ops, s,
+                          "gicv3_dist", 0x10000);
+    memory_region_init_io(&s->iomem_redist, OBJECT(s), ops ? &ops[1] : NULL, s,
+                          "gicv3_redist", 0x20000 * s->num_cpu);
+
+    sysbus_init_mmio(sbd, &s->iomem_dist);
+    sysbus_init_mmio(sbd, &s->iomem_redist);
+}
+
+static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+
+    /* revision property is actually reserved and currently used only in order
+     * to keep the interface compatible with GICv2 code, avoiding extra
+     * conditions. However, in future it could be used, for example, if we
+     * implement GICv4.
+     */
+    if (s->revision != 3) {
+        error_setg(errp, "unsupported GIC revision %d", s->revision);
+        return;
+    }
+}
+
+static void arm_gicv3_common_reset(DeviceState *dev)
+{
+    /* TODO */
+}
+
+static Property arm_gicv3_common_properties[] = {
+    DEFINE_PROP_UINT32("num-cpu", GICv3State, num_cpu, 1),
+    DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
+    DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
+    DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void arm_gicv3_common_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = arm_gicv3_common_reset;
+    dc->realize = arm_gicv3_common_realize;
+    dc->props = arm_gicv3_common_properties;
+    dc->vmsd = &vmstate_gicv3;
+}
+
+static const TypeInfo arm_gicv3_common_type = {
+    .name = TYPE_ARM_GICV3_COMMON,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GICv3State),
+    .class_size = sizeof(ARMGICv3CommonClass),
+    .class_init = arm_gicv3_common_class_init,
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&arm_gicv3_common_type);
+}
+
+type_init(register_types)
diff --git a/qemu/hw/intc/arm_gicv3_kvm.c b/qemu/hw/intc/arm_gicv3_kvm.c
new file mode 100644
index 000000000..acc173004
--- /dev/null
+++ b/qemu/hw/intc/arm_gicv3_kvm.c
@@ -0,0 +1,151 @@
+/*
+ * ARM Generic Interrupt Controller using KVM in-kernel support
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Pavel Fedin
+ * Based on vGICv2 code by Peter Maydell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/intc/arm_gicv3_common.h"
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
+#include "vgic_common.h"
+
+#ifdef DEBUG_GICV3_KVM
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "kvm_gicv3: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+#define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3"
+#define KVM_ARM_GICV3(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_CLASS(klass) \
+     OBJECT_CLASS_CHECK(KVMARMGICv3Class, (klass), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(KVMARMGICv3Class, (obj), TYPE_KVM_ARM_GICV3)
+
+typedef struct KVMARMGICv3Class {
+    ARMGICv3CommonClass parent_class;
+    DeviceRealize parent_realize;
+    void (*parent_reset)(DeviceState *dev);
+} KVMARMGICv3Class;
+
+static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level)
+{
+    GICv3State *s = (GICv3State *)opaque;
+
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
+}
+
+static void kvm_arm_gicv3_put(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot put kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_get(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot get kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_reset(DeviceState *dev)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+
+    DPRINTF("Reset\n");
+
+    kgc->parent_reset(dev);
+    kvm_arm_gicv3_put(s);
+}
+
+static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = KVM_ARM_GICV3(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+    Error *local_err = NULL;
+
+    DPRINTF("kvm_arm_gicv3_realize\n");
+
+    kgc->parent_realize(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (s->security_extn) {
+        error_setg(errp, "the in-kernel VGICv3 does not implement the "
+                   "security extensions");
+        return;
+    }
+
+    gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL);
+
+    /* Try to create the device via the device control API */
+    s->dev_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false);
+    if (s->dev_fd < 0) {
+        error_setg_errno(errp, -s->dev_fd, "error creating in-kernel VGIC");
+        return;
+    }
+
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+                      0, &s->num_irq, true);
+
+    /* Tell the kernel to complete VGIC initialization now */
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                      KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+
+    kvm_arm_register_device(&s->iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd);
+    kvm_arm_register_device(&s->iomem_redist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_REDIST, s->dev_fd);
+}
+
+static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
+
+    agcc->pre_save = kvm_arm_gicv3_get;
+    agcc->post_load = kvm_arm_gicv3_put;
+    kgc->parent_realize = dc->realize;
+    kgc->parent_reset = dc->reset;
+    dc->realize = kvm_arm_gicv3_realize;
+    dc->reset = kvm_arm_gicv3_reset;
+}
+
+static const TypeInfo kvm_arm_gicv3_info = {
+    .name = TYPE_KVM_ARM_GICV3,
+    .parent = TYPE_ARM_GICV3_COMMON,
+    .instance_size = sizeof(GICv3State),
+    .class_init = kvm_arm_gicv3_class_init,
+    .class_size = sizeof(KVMARMGICv3Class),
+};
+
+static void kvm_arm_gicv3_register_types(void)
+{
+    type_register_static(&kvm_arm_gicv3_info);
+}
+
+type_init(kvm_arm_gicv3_register_types)
diff --git a/qemu/hw/intc/armv7m_nvic.c b/qemu/hw/intc/armv7m_nvic.c
index e13b729e1..669e82adf 100644
--- a/qemu/hw/intc/armv7m_nvic.c
+++ b/qemu/hw/intc/armv7m_nvic.c
@@ -10,6 +10,9 @@
  * NVIC.  Much of that is also implemented here.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "hw/arm/arm.h"
@@ -28,6 +31,7 @@ typedef struct {
     MemoryRegion gic_iomem_alias;
     MemoryRegion container;
     uint32_t num_irq;
+    qemu_irq sysresetreq;
 } nvic_state;
 
 #define TYPE_NVIC "armv7m_nvic"
@@ -185,26 +189,25 @@ static uint32_t nvic_readl(nvic_state *s, uint32_t offset)
         return cpu->midr;
     case 0xd04: /* Interrupt Control State.  */
         /* VECTACTIVE */
-        val = s->gic.running_irq[0];
+        cpu = ARM_CPU(current_cpu);
+        val = cpu->env.v7m.exception;
         if (val == 1023) {
             val = 0;
         } else if (val >= 32) {
             val -= 16;
         }
-        /* RETTOBASE */
-        if (s->gic.running_irq[0] == 1023
-                || s->gic.last_active[s->gic.running_irq[0]][0] == 1023) {
-            val |= (1 << 11);
-        }
         /* VECTPENDING */
         if (s->gic.current_pending[0] != 1023)
             val |= (s->gic.current_pending[0] << 12);
-        /* ISRPENDING */
+        /* ISRPENDING and RETTOBASE */
         for (irq = 32; irq < s->num_irq; irq++) {
             if (s->gic.irq_state[irq].pending) {
                 val |= (1 << 22);
                 break;
             }
+            if (irq != cpu->env.v7m.exception && s->gic.irq_state[irq].active) {
+                val |= (1 << 11);
+            }
         }
         /* PENDSTSET */
         if (s->gic.irq_state[ARMV7M_EXCP_SYSTICK].pending)
@@ -349,10 +352,13 @@ static void nvic_writel(nvic_state *s, uint32_t offset, uint32_t value)
         break;
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         if ((value >> 16) == 0x05fa) {
+            if (value & 4) {
+                qemu_irq_pulse(s->sysresetreq);
+            }
             if (value & 2) {
                 qemu_log_mask(LOG_UNIMP, "VECTCLRACTIVE unimplemented\n");
             }
-            if (value & 5) {
+            if (value & 1) {
                 qemu_log_mask(LOG_UNIMP, "AIRCR system reset unimplemented\n");
             }
             if (value & 0x700) {
@@ -536,11 +542,14 @@ static void armv7m_nvic_instance_init(Object *obj)
      * value in the GICState struct.
      */
     GICState *s = ARM_GIC_COMMON(obj);
+    DeviceState *dev = DEVICE(obj);
+    nvic_state *nvic = NVIC(obj);
     /* The ARM v7m may have anything from 0 to 496 external interrupt
      * IRQ lines. We default to 64. Other boards may differ and should
      * set the num-irq property appropriately.
      */
     s->num_irq = 64;
+    qdev_init_gpio_out_named(dev, &nvic->sysresetreq, "SYSRESETREQ", 1);
 }
 
 static void armv7m_nvic_class_init(ObjectClass *klass, void *data)
diff --git a/qemu/hw/intc/aspeed_vic.c b/qemu/hw/intc/aspeed_vic.c
new file mode 100644
index 000000000..19a0ff748
--- /dev/null
+++ b/qemu/hw/intc/aspeed_vic.c
@@ -0,0 +1,339 @@
+/*
+ * ASPEED Interrupt Controller (New)
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2015, 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+/* The hardware exposes two register sets, a legacy set and a 'new' set. The
+ * model implements the 'new' register set, and logs warnings on accesses to
+ * the legacy IO space.
+ *
+ * The hardware uses 32bit registers to manage 51 IRQs, with low and high
+ * registers for each conceptual register. The device model's implementation
+ * uses 64bit data types to store both low and high register values (in the one
+ * member), but must cope with access offset values in multiples of 4 passed to
+ * the callbacks. As such the read() and write() implementations process the
+ * provided offset to understand whether the access is requesting the lower or
+ * upper 32 bits of the 64bit member.
+ *
+ * Additionally, the "Interrupt Enable", "Edge Status" and "Software Interrupt"
+ * fields have separate "enable"/"status" and "clear" registers, where set bits
+ * are written to one or the other to change state (avoiding a
+ * read-modify-write sequence).
+ */
+
+#include "qemu/osdep.h"
+#include <inttypes.h>
+#include "hw/intc/aspeed_vic.h"
+#include "qemu/bitops.h"
+#include "trace.h"
+
+#define AVIC_NEW_BASE_OFFSET 0x80
+
+#define AVIC_L_MASK 0xFFFFFFFFU
+#define AVIC_H_MASK 0x0007FFFFU
+#define AVIC_EVENT_W_MASK (0x78000ULL << 32)
+
+static void aspeed_vic_update(AspeedVICState *s)
+{
+    uint64_t new = (s->raw & s->enable);
+    uint64_t flags;
+
+    flags = new & s->select;
+    trace_aspeed_vic_update_fiq(!!flags);
+    qemu_set_irq(s->fiq, !!flags);
+
+    flags = new & ~s->select;
+    trace_aspeed_vic_update_irq(!!flags);
+    qemu_set_irq(s->irq, !!flags);
+}
+
+static void aspeed_vic_set_irq(void *opaque, int irq, int level)
+{
+    uint64_t irq_mask;
+    bool raise;
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (irq > ASPEED_VIC_NR_IRQS) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Invalid interrupt number: %d\n",
+                      __func__, irq);
+        return;
+    }
+
+    trace_aspeed_vic_set_irq(irq, level);
+
+    irq_mask = BIT(irq);
+    if (s->sense & irq_mask) {
+        /* level-triggered */
+        if (s->event & irq_mask) {
+            /* high-sensitive */
+            raise = level;
+        } else {
+            /* low-sensitive */
+            raise = !level;
+        }
+        s->raw = deposit64(s->raw, irq, 1, raise);
+    } else {
+        uint64_t old_level = s->level & irq_mask;
+
+        /* edge-triggered */
+        if (s->dual_edge & irq_mask) {
+            raise = (!!old_level) != (!!level);
+        } else {
+            if (s->event & irq_mask) {
+                /* rising-sensitive */
+                raise = !old_level && level;
+            } else {
+                /* falling-sensitive */
+                raise = old_level && !level;
+            }
+        }
+        if (raise) {
+            s->raw = deposit64(s->raw, irq, 1, raise);
+        }
+    }
+    s->level = deposit64(s->level, irq, 1, level);
+    aspeed_vic_update(s);
+}
+
+static uint64_t aspeed_vic_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint64_t val;
+    const bool high = !!(offset & 0x4);
+    hwaddr n_offset = (offset & ~0x4);
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (offset < AVIC_NEW_BASE_OFFSET) {
+        qemu_log_mask(LOG_UNIMP, "%s: Ignoring read from legacy registers "
+                      "at 0x%" HWADDR_PRIx "[%u]\n", __func__, offset, size);
+        return 0;
+    }
+
+    n_offset -= AVIC_NEW_BASE_OFFSET;
+
+    switch (n_offset) {
+    case 0x0: /* IRQ Status */
+        val = s->raw & ~s->select & s->enable;
+        break;
+    case 0x08: /* FIQ Status */
+        val = s->raw & s->select & s->enable;
+        break;
+    case 0x10: /* Raw Interrupt Status */
+        val = s->raw;
+        break;
+    case 0x18: /* Interrupt Selection */
+        val = s->select;
+        break;
+    case 0x20: /* Interrupt Enable */
+        val = s->enable;
+        break;
+    case 0x30: /* Software Interrupt */
+        val = s->trigger;
+        break;
+    case 0x40: /* Interrupt Sensitivity */
+        val = s->sense;
+        break;
+    case 0x48: /* Interrupt Both Edge Trigger Control */
+        val = s->dual_edge;
+        break;
+    case 0x50: /* Interrupt Event */
+        val = s->event;
+        break;
+    case 0x60: /* Edge Triggered Interrupt Status */
+        val = s->raw & ~s->sense;
+        break;
+        /* Illegal */
+    case 0x28: /* Interrupt Enable Clear */
+    case 0x38: /* Software Interrupt Clear */
+    case 0x58: /* Edge Triggered Interrupt Clear */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Read of write-only register with offset 0x%"
+                      HWADDR_PRIx "\n", __func__, offset);
+        val = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad register at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        val = 0;
+        break;
+    }
+    if (high) {
+        val = extract64(val, 32, 19);
+    }
+    trace_aspeed_vic_read(offset, size, val);
+    return val;
+}
+
+static void aspeed_vic_write(void *opaque, hwaddr offset, uint64_t data,
+                             unsigned size)
+{
+    const bool high = !!(offset & 0x4);
+    hwaddr n_offset = (offset & ~0x4);
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (offset < AVIC_NEW_BASE_OFFSET) {
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Ignoring write to legacy registers at 0x%"
+                      HWADDR_PRIx "[%u] <- 0x%" PRIx64 "\n", __func__, offset,
+                      size, data);
+        return;
+    }
+
+    n_offset -= AVIC_NEW_BASE_OFFSET;
+    trace_aspeed_vic_write(offset, size, data);
+
+    /* Given we have members using separate enable/clear registers, deposit64()
+     * isn't quite the tool for the job. Instead, relocate the incoming bits to
+     * the required bit offset based on the provided access address
+     */
+    if (high) {
+        data &= AVIC_H_MASK;
+        data <<= 32;
+    } else {
+        data &= AVIC_L_MASK;
+    }
+
+    switch (n_offset) {
+    case 0x18: /* Interrupt Selection */
+        /* Register has deposit64() semantics - overwrite requested 32 bits */
+        if (high) {
+            s->select &= AVIC_L_MASK;
+        } else {
+            s->select &= ((uint64_t) AVIC_H_MASK) << 32;
+        }
+        s->select |= data;
+        break;
+    case 0x20: /* Interrupt Enable */
+        s->enable |= data;
+        break;
+    case 0x28: /* Interrupt Enable Clear */
+        s->enable &= ~data;
+        break;
+    case 0x30: /* Software Interrupt */
+        qemu_log_mask(LOG_UNIMP, "%s: Software interrupts unavailable. "
+                      "IRQs requested: 0x%016" PRIx64 "\n", __func__, data);
+        break;
+    case 0x38: /* Software Interrupt Clear */
+        qemu_log_mask(LOG_UNIMP, "%s: Software interrupts unavailable. "
+                      "IRQs to be cleared: 0x%016" PRIx64 "\n", __func__, data);
+        break;
+    case 0x50: /* Interrupt Event */
+        /* Register has deposit64() semantics - overwrite the top four valid
+         * IRQ bits, as only the top four IRQs (GPIOs) can change their event
+         * type */
+        if (high) {
+            s->event &= ~AVIC_EVENT_W_MASK;
+            s->event |= (data & AVIC_EVENT_W_MASK);
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "Ignoring invalid write to interrupt event register");
+        }
+        break;
+    case 0x58: /* Edge Triggered Interrupt Clear */
+        s->raw &= ~(data & ~s->sense);
+        break;
+    case 0x00: /* IRQ Status */
+    case 0x08: /* FIQ Status */
+    case 0x10: /* Raw Interrupt Status */
+    case 0x40: /* Interrupt Sensitivity */
+    case 0x48: /* Interrupt Both Edge Trigger Control */
+    case 0x60: /* Edge Triggered Interrupt Status */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Write of read-only register with offset 0x%"
+                      HWADDR_PRIx "\n", __func__, offset);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad register at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        break;
+    }
+    aspeed_vic_update(s);
+}
+
+static const MemoryRegionOps aspeed_vic_ops = {
+    .read = aspeed_vic_read,
+    .write = aspeed_vic_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .valid.unaligned = false,
+};
+
+static void aspeed_vic_reset(DeviceState *dev)
+{
+    AspeedVICState *s = ASPEED_VIC(dev);
+
+    s->level = 0;
+    s->raw = 0;
+    s->select = 0;
+    s->enable = 0;
+    s->trigger = 0;
+    s->sense = 0x1F07FFF8FFFFULL;
+    s->dual_edge = 0xF800070000ULL;
+    s->event = 0x5F07FFF8FFFFULL;
+}
+
+#define AVIC_IO_REGION_SIZE 0x20000
+
+static void aspeed_vic_realize(DeviceState *dev, Error **errp)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedVICState *s = ASPEED_VIC(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_vic_ops, s,
+                          TYPE_ASPEED_VIC, AVIC_IO_REGION_SIZE);
+
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    qdev_init_gpio_in(dev, aspeed_vic_set_irq, ASPEED_VIC_NR_IRQS);
+    sysbus_init_irq(sbd, &s->irq);
+    sysbus_init_irq(sbd, &s->fiq);
+}
+
+static const VMStateDescription vmstate_aspeed_vic = {
+    .name = "aspeed.new-vic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(level, AspeedVICState),
+        VMSTATE_UINT64(raw, AspeedVICState),
+        VMSTATE_UINT64(select, AspeedVICState),
+        VMSTATE_UINT64(enable, AspeedVICState),
+        VMSTATE_UINT64(trigger, AspeedVICState),
+        VMSTATE_UINT64(sense, AspeedVICState),
+        VMSTATE_UINT64(dual_edge, AspeedVICState),
+        VMSTATE_UINT64(event, AspeedVICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void aspeed_vic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->realize = aspeed_vic_realize;
+    dc->reset = aspeed_vic_reset;
+    dc->desc = "ASPEED Interrupt Controller (New)";
+    dc->vmsd = &vmstate_aspeed_vic;
+}
+
+static const TypeInfo aspeed_vic_info = {
+    .name = TYPE_ASPEED_VIC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedVICState),
+    .class_init = aspeed_vic_class_init,
+};
+
+static void aspeed_vic_register_types(void)
+{
+    type_register_static(&aspeed_vic_info);
+}
+
+type_init(aspeed_vic_register_types);
diff --git a/qemu/hw/intc/bcm2835_ic.c b/qemu/hw/intc/bcm2835_ic.c
new file mode 100644
index 000000000..80513b28f
--- /dev/null
+++ b/qemu/hw/intc/bcm2835_ic.c
@@ -0,0 +1,237 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Refactoring for Pi2 Copyright (c) 2015, Microsoft. Written by Andrew Baumann.
+ * This code is licensed under the GNU GPLv2 and later.
+ * Heavily based on pl190.c, copyright terms below:
+ *
+ * Arm PrimeCell PL190 Vector Interrupt Controller
+ *
+ * Copyright (c) 2006 CodeSourcery.
+ * Written by Paul Brook
+ *
+ * This code is licensed under the GPL.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/intc/bcm2835_ic.h"
+
+#define GPU_IRQS 64
+#define ARM_IRQS 8
+
+#define IRQ_PENDING_BASIC       0x00 /* IRQ basic pending */
+#define IRQ_PENDING_1           0x04 /* IRQ pending 1 */
+#define IRQ_PENDING_2           0x08 /* IRQ pending 2 */
+#define FIQ_CONTROL             0x0C /* FIQ register */
+#define IRQ_ENABLE_1            0x10 /* Interrupt enable register 1 */
+#define IRQ_ENABLE_2            0x14 /* Interrupt enable register 2 */
+#define IRQ_ENABLE_BASIC        0x18 /* Base interrupt enable register */
+#define IRQ_DISABLE_1           0x1C /* Interrupt disable register 1 */
+#define IRQ_DISABLE_2           0x20 /* Interrupt disable register 2 */
+#define IRQ_DISABLE_BASIC       0x24 /* Base interrupt disable register */
+
+/* Update interrupts.  */
+static void bcm2835_ic_update(BCM2835ICState *s)
+{
+    bool set = false;
+
+    if (s->fiq_enable) {
+        if (s->fiq_select >= GPU_IRQS) {
+            /* ARM IRQ */
+            set = extract32(s->arm_irq_level, s->fiq_select - GPU_IRQS, 1);
+        } else {
+            set = extract64(s->gpu_irq_level, s->fiq_select, 1);
+        }
+    }
+    qemu_set_irq(s->fiq, set);
+
+    set = (s->gpu_irq_level & s->gpu_irq_enable)
+        || (s->arm_irq_level & s->arm_irq_enable);
+    qemu_set_irq(s->irq, set);
+
+}
+
+static void bcm2835_ic_set_gpu_irq(void *opaque, int irq, int level)
+{
+    BCM2835ICState *s = opaque;
+
+    assert(irq >= 0 && irq < 64);
+    s->gpu_irq_level = deposit64(s->gpu_irq_level, irq, 1, level != 0);
+    bcm2835_ic_update(s);
+}
+
+static void bcm2835_ic_set_arm_irq(void *opaque, int irq, int level)
+{
+    BCM2835ICState *s = opaque;
+
+    assert(irq >= 0 && irq < 8);
+    s->arm_irq_level = deposit32(s->arm_irq_level, irq, 1, level != 0);
+    bcm2835_ic_update(s);
+}
+
+static const int irq_dups[] = { 7, 9, 10, 18, 19, 53, 54, 55, 56, 57, 62 };
+
+static uint64_t bcm2835_ic_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835ICState *s = opaque;
+    uint32_t res = 0;
+    uint64_t gpu_pending = s->gpu_irq_level & s->gpu_irq_enable;
+    int i;
+
+    switch (offset) {
+    case IRQ_PENDING_BASIC:
+        /* bits 0-7: ARM irqs */
+        res = s->arm_irq_level & s->arm_irq_enable;
+
+        /* bits 8 & 9: pending registers 1 & 2 */
+        res |= (((uint32_t)gpu_pending) != 0) << 8;
+        res |= ((gpu_pending >> 32) != 0) << 9;
+
+        /* bits 10-20: selected GPU IRQs */
+        for (i = 0; i < ARRAY_SIZE(irq_dups); i++) {
+            res |= extract64(gpu_pending, irq_dups[i], 1) << (i + 10);
+        }
+        break;
+    case IRQ_PENDING_1:
+        res = gpu_pending;
+        break;
+    case IRQ_PENDING_2:
+        res = gpu_pending >> 32;
+        break;
+    case FIQ_CONTROL:
+        res = (s->fiq_enable << 7) | s->fiq_select;
+        break;
+    case IRQ_ENABLE_1:
+        res = s->gpu_irq_enable;
+        break;
+    case IRQ_ENABLE_2:
+        res = s->gpu_irq_enable >> 32;
+        break;
+    case IRQ_ENABLE_BASIC:
+        res = s->arm_irq_enable;
+        break;
+    case IRQ_DISABLE_1:
+        res = ~s->gpu_irq_enable;
+        break;
+    case IRQ_DISABLE_2:
+        res = ~s->gpu_irq_enable >> 32;
+        break;
+    case IRQ_DISABLE_BASIC:
+        res = ~s->arm_irq_enable;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_ic_write(void *opaque, hwaddr offset, uint64_t val,
+                             unsigned size)
+{
+    BCM2835ICState *s = opaque;
+
+    switch (offset) {
+    case FIQ_CONTROL:
+        s->fiq_select = extract32(val, 0, 7);
+        s->fiq_enable = extract32(val, 7, 1);
+        break;
+    case IRQ_ENABLE_1:
+        s->gpu_irq_enable |= val;
+        break;
+    case IRQ_ENABLE_2:
+        s->gpu_irq_enable |= val << 32;
+        break;
+    case IRQ_ENABLE_BASIC:
+        s->arm_irq_enable |= val & 0xff;
+        break;
+    case IRQ_DISABLE_1:
+        s->gpu_irq_enable &= ~val;
+        break;
+    case IRQ_DISABLE_2:
+        s->gpu_irq_enable &= ~(val << 32);
+        break;
+    case IRQ_DISABLE_BASIC:
+        s->arm_irq_enable &= ~val & 0xff;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+    bcm2835_ic_update(s);
+}
+
+static const MemoryRegionOps bcm2835_ic_ops = {
+    .read = bcm2835_ic_read,
+    .write = bcm2835_ic_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static void bcm2835_ic_reset(DeviceState *d)
+{
+    BCM2835ICState *s = BCM2835_IC(d);
+
+    s->gpu_irq_enable = 0;
+    s->arm_irq_enable = 0;
+    s->fiq_enable = false;
+    s->fiq_select = 0;
+}
+
+static void bcm2835_ic_init(Object *obj)
+{
+    BCM2835ICState *s = BCM2835_IC(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_ic_ops, s, TYPE_BCM2835_IC,
+                          0x200);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+
+    qdev_init_gpio_in_named(DEVICE(s), bcm2835_ic_set_gpu_irq,
+                            BCM2835_IC_GPU_IRQ, GPU_IRQS);
+    qdev_init_gpio_in_named(DEVICE(s), bcm2835_ic_set_arm_irq,
+                            BCM2835_IC_ARM_IRQ, ARM_IRQS);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->fiq);
+}
+
+static const VMStateDescription vmstate_bcm2835_ic = {
+    .name = TYPE_BCM2835_IC,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(gpu_irq_level, BCM2835ICState),
+        VMSTATE_UINT64(gpu_irq_enable, BCM2835ICState),
+        VMSTATE_UINT8(arm_irq_level, BCM2835ICState),
+        VMSTATE_UINT8(arm_irq_enable, BCM2835ICState),
+        VMSTATE_BOOL(fiq_enable, BCM2835ICState),
+        VMSTATE_UINT8(fiq_select, BCM2835ICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_ic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = bcm2835_ic_reset;
+    dc->vmsd = &vmstate_bcm2835_ic;
+}
+
+static TypeInfo bcm2835_ic_info = {
+    .name          = TYPE_BCM2835_IC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835ICState),
+    .class_init    = bcm2835_ic_class_init,
+    .instance_init = bcm2835_ic_init,
+};
+
+static void bcm2835_ic_register_types(void)
+{
+    type_register_static(&bcm2835_ic_info);
+}
+
+type_init(bcm2835_ic_register_types)
diff --git a/qemu/hw/intc/bcm2836_control.c b/qemu/hw/intc/bcm2836_control.c
new file mode 100644
index 000000000..d0271810c
--- /dev/null
+++ b/qemu/hw/intc/bcm2836_control.c
@@ -0,0 +1,304 @@
+/*
+ * Rasperry Pi 2 emulation ARM control logic module.
+ * Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * Based on bcm2835_ic.c (Raspberry Pi emulation) (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * At present, only implements interrupt routing, and mailboxes (i.e.,
+ * not local timer, PMU interrupt, or AXI counters).
+ *
+ * Ref:
+ * https://www.raspberrypi.org/documentation/hardware/raspberrypi/bcm2836/QA7_rev3.4.pdf
+ */
+
+#include "qemu/osdep.h"
+#include "hw/intc/bcm2836_control.h"
+
+#define REG_GPU_ROUTE           0x0c
+#define REG_TIMERCONTROL        0x40
+#define REG_MBOXCONTROL         0x50
+#define REG_IRQSRC              0x60
+#define REG_FIQSRC              0x70
+#define REG_MBOX0_WR            0x80
+#define REG_MBOX0_RDCLR         0xc0
+#define REG_LIMIT              0x100
+
+#define IRQ_BIT(cntrl, num) (((cntrl) & (1 << (num))) != 0)
+#define FIQ_BIT(cntrl, num) (((cntrl) & (1 << ((num) + 4))) != 0)
+
+#define IRQ_CNTPSIRQ    0
+#define IRQ_CNTPNSIRQ   1
+#define IRQ_CNTHPIRQ    2
+#define IRQ_CNTVIRQ     3
+#define IRQ_MAILBOX0    4
+#define IRQ_MAILBOX1    5
+#define IRQ_MAILBOX2    6
+#define IRQ_MAILBOX3    7
+#define IRQ_GPU         8
+#define IRQ_PMU         9
+#define IRQ_AXI         10
+#define IRQ_TIMER       11
+#define IRQ_MAX         IRQ_TIMER
+
+static void deliver_local(BCM2836ControlState *s, uint8_t core, uint8_t irq,
+                          uint32_t controlreg, uint8_t controlidx)
+{
+    if (FIQ_BIT(controlreg, controlidx)) {
+        /* deliver a FIQ */
+        s->fiqsrc[core] |= (uint32_t)1 << irq;
+    } else if (IRQ_BIT(controlreg, controlidx)) {
+        /* deliver an IRQ */
+        s->irqsrc[core] |= (uint32_t)1 << irq;
+    } else {
+        /* the interrupt is masked */
+    }
+}
+
+/* Update interrupts.  */
+static void bcm2836_control_update(BCM2836ControlState *s)
+{
+    int i, j;
+
+    /* reset pending IRQs/FIQs */
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        s->irqsrc[i] = s->fiqsrc[i] = 0;
+    }
+
+    /* apply routing logic, update status regs */
+    if (s->gpu_irq) {
+        assert(s->route_gpu_irq < BCM2836_NCORES);
+        s->irqsrc[s->route_gpu_irq] |= (uint32_t)1 << IRQ_GPU;
+    }
+
+    if (s->gpu_fiq) {
+        assert(s->route_gpu_fiq < BCM2836_NCORES);
+        s->fiqsrc[s->route_gpu_fiq] |= (uint32_t)1 << IRQ_GPU;
+    }
+
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        /* handle local timer interrupts for this core */
+        if (s->timerirqs[i]) {
+            assert(s->timerirqs[i] < (1 << (IRQ_CNTVIRQ + 1))); /* sane mask? */
+            for (j = 0; j <= IRQ_CNTVIRQ; j++) {
+                if ((s->timerirqs[i] & (1 << j)) != 0) {
+                    /* local interrupt j is set */
+                    deliver_local(s, i, j, s->timercontrol[i], j);
+                }
+            }
+        }
+
+        /* handle mailboxes for this core */
+        for (j = 0; j < BCM2836_MBPERCORE; j++) {
+            if (s->mailboxes[i * BCM2836_MBPERCORE + j] != 0) {
+                /* mailbox j is set */
+                deliver_local(s, i, j + IRQ_MAILBOX0, s->mailboxcontrol[i], j);
+            }
+        }
+    }
+
+    /* call set_irq appropriately for each output */
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        qemu_set_irq(s->irq[i], s->irqsrc[i] != 0);
+        qemu_set_irq(s->fiq[i], s->fiqsrc[i] != 0);
+    }
+}
+
+static void bcm2836_control_set_local_irq(void *opaque, int core, int local_irq,
+                                          int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    assert(core >= 0 && core < BCM2836_NCORES);
+    assert(local_irq >= 0 && local_irq <= IRQ_CNTVIRQ);
+
+    s->timerirqs[core] = deposit32(s->timerirqs[core], local_irq, 1, !!level);
+
+    bcm2836_control_update(s);
+}
+
+/* XXX: the following wrapper functions are a kludgy workaround,
+ * needed because I can't seem to pass useful information in the "irq"
+ * parameter when using named interrupts. Feel free to clean this up!
+ */
+
+static void bcm2836_control_set_local_irq0(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 0, level);
+}
+
+static void bcm2836_control_set_local_irq1(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 1, level);
+}
+
+static void bcm2836_control_set_local_irq2(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 2, level);
+}
+
+static void bcm2836_control_set_local_irq3(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 3, level);
+}
+
+static void bcm2836_control_set_gpu_irq(void *opaque, int irq, int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    s->gpu_irq = level;
+
+    bcm2836_control_update(s);
+}
+
+static void bcm2836_control_set_gpu_fiq(void *opaque, int irq, int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    s->gpu_fiq = level;
+
+    bcm2836_control_update(s);
+}
+
+static uint64_t bcm2836_control_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2836ControlState *s = opaque;
+
+    if (offset == REG_GPU_ROUTE) {
+        assert(s->route_gpu_fiq < BCM2836_NCORES
+               && s->route_gpu_irq < BCM2836_NCORES);
+        return ((uint32_t)s->route_gpu_fiq << 2) | s->route_gpu_irq;
+    } else if (offset >= REG_TIMERCONTROL && offset < REG_MBOXCONTROL) {
+        return s->timercontrol[(offset - REG_TIMERCONTROL) >> 2];
+    } else if (offset >= REG_MBOXCONTROL && offset < REG_IRQSRC) {
+        return s->mailboxcontrol[(offset - REG_MBOXCONTROL) >> 2];
+    } else if (offset >= REG_IRQSRC && offset < REG_FIQSRC) {
+        return s->irqsrc[(offset - REG_IRQSRC) >> 2];
+    } else if (offset >= REG_FIQSRC && offset < REG_MBOX0_WR) {
+        return s->fiqsrc[(offset - REG_FIQSRC) >> 2];
+    } else if (offset >= REG_MBOX0_RDCLR && offset < REG_LIMIT) {
+        return s->mailboxes[(offset - REG_MBOX0_RDCLR) >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+}
+
+static void bcm2836_control_write(void *opaque, hwaddr offset,
+                                  uint64_t val, unsigned size)
+{
+    BCM2836ControlState *s = opaque;
+
+    if (offset == REG_GPU_ROUTE) {
+        s->route_gpu_irq = val & 0x3;
+        s->route_gpu_fiq = (val >> 2) & 0x3;
+    } else if (offset >= REG_TIMERCONTROL && offset < REG_MBOXCONTROL) {
+        s->timercontrol[(offset - REG_TIMERCONTROL) >> 2] = val & 0xff;
+    } else if (offset >= REG_MBOXCONTROL && offset < REG_IRQSRC) {
+        s->mailboxcontrol[(offset - REG_MBOXCONTROL) >> 2] = val & 0xff;
+    } else if (offset >= REG_MBOX0_WR && offset < REG_MBOX0_RDCLR) {
+        s->mailboxes[(offset - REG_MBOX0_WR) >> 2] |= val;
+    } else if (offset >= REG_MBOX0_RDCLR && offset < REG_LIMIT) {
+        s->mailboxes[(offset - REG_MBOX0_RDCLR) >> 2] &= ~val;
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+
+    bcm2836_control_update(s);
+}
+
+static const MemoryRegionOps bcm2836_control_ops = {
+    .read = bcm2836_control_read,
+    .write = bcm2836_control_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static void bcm2836_control_reset(DeviceState *d)
+{
+    BCM2836ControlState *s = BCM2836_CONTROL(d);
+    int i;
+
+    s->route_gpu_irq = s->route_gpu_fiq = 0;
+
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        s->timercontrol[i] = 0;
+        s->mailboxcontrol[i] = 0;
+    }
+
+    for (i = 0; i < BCM2836_NCORES * BCM2836_MBPERCORE; i++) {
+        s->mailboxes[i] = 0;
+    }
+}
+
+static void bcm2836_control_init(Object *obj)
+{
+    BCM2836ControlState *s = BCM2836_CONTROL(obj);
+    DeviceState *dev = DEVICE(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2836_control_ops, s,
+                          TYPE_BCM2836_CONTROL, REG_LIMIT);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+
+    /* inputs from each CPU core */
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq0, "cntpsirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq1, "cntpnsirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq2, "cnthpirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq3, "cntvirq",
+                            BCM2836_NCORES);
+
+    /* IRQ and FIQ inputs from upstream bcm2835 controller */
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_gpu_irq, "gpu-irq", 1);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_gpu_fiq, "gpu-fiq", 1);
+
+    /* outputs to CPU cores */
+    qdev_init_gpio_out_named(dev, s->irq, "irq", BCM2836_NCORES);
+    qdev_init_gpio_out_named(dev, s->fiq, "fiq", BCM2836_NCORES);
+}
+
+static const VMStateDescription vmstate_bcm2836_control = {
+    .name = TYPE_BCM2836_CONTROL,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(mailboxes, BCM2836ControlState,
+                             BCM2836_NCORES * BCM2836_MBPERCORE),
+        VMSTATE_UINT8(route_gpu_irq, BCM2836ControlState),
+        VMSTATE_UINT8(route_gpu_fiq, BCM2836ControlState),
+        VMSTATE_UINT32_ARRAY(timercontrol, BCM2836ControlState, BCM2836_NCORES),
+        VMSTATE_UINT32_ARRAY(mailboxcontrol, BCM2836ControlState,
+                             BCM2836_NCORES),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2836_control_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = bcm2836_control_reset;
+    dc->vmsd = &vmstate_bcm2836_control;
+}
+
+static TypeInfo bcm2836_control_info = {
+    .name          = TYPE_BCM2836_CONTROL,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2836ControlState),
+    .class_init    = bcm2836_control_class_init,
+    .instance_init = bcm2836_control_init,
+};
+
+static void bcm2836_control_register_types(void)
+{
+    type_register_static(&bcm2836_control_info);
+}
+
+type_init(bcm2836_control_register_types)
diff --git a/qemu/hw/intc/etraxfs_pic.c b/qemu/hw/intc/etraxfs_pic.c
index bd588681e..48f947706 100644
--- a/qemu/hw/intc/etraxfs_pic.c
+++ b/qemu/hw/intc/etraxfs_pic.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 //#include "pc.h"
diff --git a/qemu/hw/intc/exynos4210_combiner.c b/qemu/hw/intc/exynos4210_combiner.c
index a6b702890..dc0c90326 100644
--- a/qemu/hw/intc/exynos4210_combiner.c
+++ b/qemu/hw/intc/exynos4210_combiner.c
@@ -27,6 +27,7 @@
  * IRQs are passed to GIC through Combiner.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #include "hw/arm/exynos4210.h"
diff --git a/qemu/hw/intc/exynos4210_gic.c b/qemu/hw/intc/exynos4210_gic.c
index b2a4950bc..4f7e89f7b 100644
--- a/qemu/hw/intc/exynos4210_gic.c
+++ b/qemu/hw/intc/exynos4210_gic.c
@@ -20,6 +20,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu-common.h"
 #include "hw/irq.h"
diff --git a/qemu/hw/intc/grlib_irqmp.c b/qemu/hw/intc/grlib_irqmp.c
index d1813f76b..f5ca8f752 100644
--- a/qemu/hw/intc/grlib_irqmp.c
+++ b/qemu/hw/intc/grlib_irqmp.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "cpu.h"
 
diff --git a/qemu/hw/intc/heathrow_pic.c b/qemu/hw/intc/heathrow_pic.c
index 9ff3119ed..171f5ed81 100644
--- a/qemu/hw/intc/heathrow_pic.c
+++ b/qemu/hw/intc/heathrow_pic.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 
diff --git a/qemu/hw/intc/i8259.c b/qemu/hw/intc/i8259.c
index 0f5c0259a..bb43669b9 100644
--- a/qemu/hw/intc/i8259.c
+++ b/qemu/hw/intc/i8259.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -229,7 +230,7 @@ int pic_read_irq(DeviceState *d)
     printf("IRQ%d latency=%0.3fus\n",
            irq,
            (double)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                    irq_time[irq]) * 1000000.0 / get_ticks_per_sec());
+                    irq_time[irq]) * 1000000.0 / NANOSECONDS_PER_SECOND);
 #endif
     DPRINTF("pic_interrupt: irq=%d\n", irq);
     return intno;
diff --git a/qemu/hw/intc/i8259_common.c b/qemu/hw/intc/i8259_common.c
index fbf26e557..3a850b0c6 100644
--- a/qemu/hw/intc/i8259_common.c
+++ b/qemu/hw/intc/i8259_common.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/i8259_internal.h"
 
diff --git a/qemu/hw/intc/imx_avic.c b/qemu/hw/intc/imx_avic.c
index e48f66c8f..702765577 100644
--- a/qemu/hw/intc/imx_avic.c
+++ b/qemu/hw/intc/imx_avic.c
@@ -7,6 +7,7 @@
  * Copyright (c) 2008 OKL
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under the GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
@@ -14,67 +15,23 @@
  * TODO: implement vectors.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "qemu/host-utils.h"
+#include "qemu/osdep.h"
+#include "hw/intc/imx_avic.h"
 
-#define DEBUG_INT 1
-#undef DEBUG_INT /* comment out for debugging */
-
-#ifdef DEBUG_INT
-#define DPRINTF(fmt, args...) \
-do { printf("imx_avic: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_AVIC
+#define DEBUG_IMX_AVIC 0
 #endif
 
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_avic: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_AVIC_NUM_IRQS 64
-
-/* Interrupt Control Bits */
-#define ABFLAG (1<<25)
-#define ABFEN (1<<24)
-#define NIDIS (1<<22) /* Normal Interrupt disable */
-#define FIDIS (1<<21) /* Fast interrupt disable */
-#define NIAD  (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
-#define FIAD  (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
-#define NM    (1<<18) /* Normal interrupt mode */
-
-
-#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
-#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
-
-#define TYPE_IMX_AVIC "imx_avic"
-#define IMX_AVIC(obj) \
-    OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
-
-typedef struct IMXAVICState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    uint64_t pending;
-    uint64_t enabled;
-    uint64_t is_fiq;
-    uint32_t intcntl;
-    uint32_t intmask;
-    qemu_irq irq;
-    qemu_irq fiq;
-    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
-} IMXAVICState;
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_AVIC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_AVIC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_avic = {
-    .name = "imx-avic",
+    .name = TYPE_IMX_AVIC,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -88,8 +45,6 @@ static const VMStateDescription vmstate_imx_avic = {
     },
 };
 
-
-
 static inline int imx_avic_prio(IMXAVICState *s, int irq)
 {
     uint32_t word = irq / PRIO_PER_WORD;
@@ -151,8 +106,8 @@ static uint64_t imx_avic_read(void *opaque,
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
 
+    DPRINTF("read(offset = 0x%" HWADDR_PRIx ")\n", offset);
 
-    DPRINTF("read(offset = 0x%x)\n", offset >> 2);
     switch (offset >> 2) {
     case 0: /* INTCNTL */
         return s->intcntl;
@@ -249,7 +204,8 @@ static uint64_t imx_avic_read(void *opaque,
         return 0x4;
 
     default:
-        IPRINTF("imx_avic_read: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
         return 0;
     }
 }
@@ -261,13 +217,13 @@ static void imx_avic_write(void *opaque, hwaddr offset,
 
     /* Vector Registers not yet supported */
     if (offset >= 0x100 && offset <= 0x2fc) {
-        IPRINTF("imx_avic_write to vector register %d ignored\n",
-                (unsigned int)((offset - 0x100) >> 2));
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: vector %d ignored\n",
+                      TYPE_IMX_AVIC, __func__, (int)((offset - 0x100) >> 2));
         return;
     }
 
-    DPRINTF("imx_avic_write(0x%x) = %x\n",
-            (unsigned int)offset>>2, (unsigned int)val);
+    DPRINTF("(0x%" HWADDR_PRIx ") = 0x%x\n", offset, (unsigned int)val);
+
     switch (offset >> 2) {
     case 0: /* Interrupt Control Register, INTCNTL */
         s->intcntl = val & (ABFEN | NIDIS | FIDIS | NIAD | FIAD | NM);
@@ -341,7 +297,8 @@ static void imx_avic_write(void *opaque, hwaddr offset,
         return;
 
     default:
-        IPRINTF("imx_avic_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
     }
     imx_avic_update(s);
 }
@@ -370,7 +327,7 @@ static int imx_avic_init(SysBusDevice *sbd)
     IMXAVICState *s = IMX_AVIC(dev);
 
     memory_region_init_io(&s->iomem, OBJECT(s), &imx_avic_ops, s,
-                          "imx_avic", 0x1000);
+                          TYPE_IMX_AVIC, 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
 
     qdev_init_gpio_in(dev, imx_avic_set_irq, IMX_AVIC_NUM_IRQS);
diff --git a/qemu/hw/intc/ioapic.c b/qemu/hw/intc/ioapic.c
index b52793238..378e663f6 100644
--- a/qemu/hw/intc/ioapic.c
+++ b/qemu/hw/intc/ioapic.c
@@ -20,10 +20,14 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "monitor/monitor.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/ioapic_internal.h"
+#include "include/hw/pci/msi.h"
+#include "sysemu/kvm.h"
 
 //#define DEBUG_IOAPIC
 
@@ -34,6 +38,10 @@
 #define DPRINTF(fmt, ...)
 #endif
 
+#define APIC_DELIVERY_MODE_SHIFT 8
+#define APIC_POLARITY_SHIFT 14
+#define APIC_TRIG_MODE_SHIFT 15
+
 static IOAPICCommonState *ioapics[MAX_IOAPICS];
 
 /* global variable from ioapic_common.c */
@@ -53,6 +61,8 @@ static void ioapic_service(IOAPICCommonState *s)
     for (i = 0; i < IOAPIC_NUM_PINS; i++) {
         mask = 1 << i;
         if (s->irr & mask) {
+            int coalesce = 0;
+
             entry = s->ioredtbl[i];
             if (!(entry & IOAPIC_LVT_MASKED)) {
                 trig_mode = ((entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1);
@@ -63,6 +73,7 @@ static void ioapic_service(IOAPICCommonState *s)
                 if (trig_mode == IOAPIC_TRIGGER_EDGE) {
                     s->irr &= ~mask;
                 } else {
+                    coalesce = s->ioredtbl[i] & IOAPIC_LVT_REMOTE_IRR;
                     s->ioredtbl[i] |= IOAPIC_LVT_REMOTE_IRR;
                 }
                 if (delivery_mode == IOAPIC_DM_EXTINT) {
@@ -70,8 +81,23 @@ static void ioapic_service(IOAPICCommonState *s)
                 } else {
                     vector = entry & IOAPIC_VECTOR_MASK;
                 }
-                apic_deliver_irq(dest, dest_mode, delivery_mode,
-                                 vector, trig_mode);
+#ifdef CONFIG_KVM
+                if (kvm_irqchip_is_split()) {
+                    if (trig_mode == IOAPIC_TRIGGER_EDGE) {
+                        kvm_set_irq(kvm_state, i, 1);
+                        kvm_set_irq(kvm_state, i, 0);
+                    } else {
+                        if (!coalesce) {
+                            kvm_set_irq(kvm_state, i, 1);
+                        }
+                    }
+                    continue;
+                }
+#else
+                (void)coalesce;
+#endif
+                apic_deliver_irq(dest, dest_mode, delivery_mode, vector,
+                                 trig_mode);
             }
         }
     }
@@ -98,7 +124,9 @@ static void ioapic_set_irq(void *opaque, int vector, int level)
             /* level triggered */
             if (level) {
                 s->irr |= mask;
-                ioapic_service(s);
+                if (!(entry & IOAPIC_LVT_REMOTE_IRR)) {
+                    ioapic_service(s);
+                }
             } else {
                 s->irr &= ~mask;
             }
@@ -113,6 +141,44 @@ static void ioapic_set_irq(void *opaque, int vector, int level)
     }
 }
 
+static void ioapic_update_kvm_routes(IOAPICCommonState *s)
+{
+#ifdef CONFIG_KVM
+    int i;
+
+    if (kvm_irqchip_is_split()) {
+        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+            uint64_t entry = s->ioredtbl[i];
+            uint8_t trig_mode;
+            uint8_t delivery_mode;
+            uint8_t dest;
+            uint8_t dest_mode;
+            uint64_t pin_polarity;
+            MSIMessage msg;
+
+            trig_mode = ((entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1);
+            dest = entry >> IOAPIC_LVT_DEST_SHIFT;
+            dest_mode = (entry >> IOAPIC_LVT_DEST_MODE_SHIFT) & 1;
+            pin_polarity = (entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1;
+            delivery_mode =
+                (entry >> IOAPIC_LVT_DELIV_MODE_SHIFT) & IOAPIC_DM_MASK;
+
+            msg.address = APIC_DEFAULT_ADDRESS;
+            msg.address |= dest_mode << 2;
+            msg.address |= dest << 12;
+
+            msg.data = entry & IOAPIC_VECTOR_MASK;
+            msg.data |= delivery_mode << APIC_DELIVERY_MODE_SHIFT;
+            msg.data |= pin_polarity << APIC_POLARITY_SHIFT;
+            msg.data |= trig_mode << APIC_TRIG_MODE_SHIFT;
+
+            kvm_irqchip_update_msi_route(kvm_state, i, msg, NULL);
+        }
+        kvm_irqchip_commit_routes(kvm_state);
+    }
+#endif
+}
+
 void ioapic_eoi_broadcast(int vector)
 {
     IOAPICCommonState *s;
@@ -137,6 +203,17 @@ void ioapic_eoi_broadcast(int vector)
     }
 }
 
+void ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    int i;
+
+    for (i = 0; i < MAX_IOAPICS; i++) {
+        if (ioapics[i] != 0) {
+            ioapic_print_redtbl(mon, ioapics[i]);
+        }
+    }
+}
+
 static uint64_t
 ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
 {
@@ -154,15 +231,13 @@ ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
         }
         switch (s->ioregsel) {
         case IOAPIC_REG_ID:
+        case IOAPIC_REG_ARB:
             val = s->id << IOAPIC_ID_SHIFT;
             break;
         case IOAPIC_REG_VER:
             val = IOAPIC_VERSION |
                 ((IOAPIC_NUM_PINS - 1) << IOAPIC_VER_ENTRIES_SHIFT);
             break;
-        case IOAPIC_REG_ARB:
-            val = 0;
-            break;
         default:
             index = (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1;
             if (index >= 0 && index < IOAPIC_NUM_PINS) {
@@ -217,6 +292,8 @@ ioapic_mem_write(void *opaque, hwaddr addr, uint64_t val,
         }
         break;
     }
+
+    ioapic_update_kvm_routes(s);
 }
 
 static const MemoryRegionOps ioapic_io_ops = {
diff --git a/qemu/hw/intc/ioapic_common.c b/qemu/hw/intc/ioapic_common.c
index 8b7d11806..1b7ec5ec2 100644
--- a/qemu/hw/intc/ioapic_common.c
+++ b/qemu/hw/intc/ioapic_common.c
@@ -19,6 +19,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "monitor/monitor.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/sysbus.h"
@@ -31,6 +34,60 @@
  */
 int ioapic_no;
 
+static void ioapic_irr_dump(Monitor *mon, const char *name, uint32_t bitmap)
+{
+    int i;
+
+    monitor_printf(mon, "%-10s ", name);
+    if (bitmap == 0) {
+        monitor_printf(mon, "(none)\n");
+        return;
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        if (bitmap & (1 << i)) {
+            monitor_printf(mon, "%-2u ", i);
+        }
+    }
+    monitor_printf(mon, "\n");
+}
+
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s)
+{
+    static const char *delm_str[] = {
+        "fixed", "lowest", "SMI", "...", "NMI", "INIT", "...", "extINT"};
+    uint32_t remote_irr = 0;
+    int i;
+
+    monitor_printf(mon, "ioapic id=0x%02x sel=0x%02x", s->id, s->ioregsel);
+    if (s->ioregsel) {
+        monitor_printf(mon, " (redir[%u])\n",
+                       (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1);
+    } else {
+        monitor_printf(mon, "\n");
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        uint64_t entry = s->ioredtbl[i];
+        uint32_t delm = (uint32_t)((entry & IOAPIC_LVT_DELIV_MODE) >>
+                                   IOAPIC_LVT_DELIV_MODE_SHIFT);
+        monitor_printf(mon, "pin %-2u 0x%016"PRIx64" dest=%"PRIx64
+                       " vec=%-3"PRIu64" %s %-5s %-6s %-6s %s\n",
+                       i, entry,
+                       (entry >> IOAPIC_LVT_DEST_SHIFT) &
+                            (entry & IOAPIC_LVT_DEST_MODE ? 0xff : 0xf),
+                       entry & IOAPIC_VECTOR_MASK,
+                       entry & IOAPIC_LVT_POLARITY ? "active-lo" : "active-hi",
+                       entry & IOAPIC_LVT_TRIGGER_MODE ? "level" : "edge",
+                       entry & IOAPIC_LVT_MASKED ? "masked" : "",
+                       delm_str[delm],
+                       entry & IOAPIC_LVT_DEST_MODE ? "logical" : "physical");
+
+        remote_irr |= entry & IOAPIC_LVT_TRIGGER_MODE ?
+                        (entry & IOAPIC_LVT_REMOTE_IRR ? (1 << i) : 0) : 0;
+    }
+    ioapic_irr_dump(mon, "IRR", s->irr);
+    ioapic_irr_dump(mon, "Remote IRR", remote_irr);
+}
+
 void ioapic_reset_common(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/qemu/hw/intc/lm32_pic.c b/qemu/hw/intc/lm32_pic.c
index 641ee4727..edc08f184 100644
--- a/qemu/hw/intc/lm32_pic.c
+++ b/qemu/hw/intc/lm32_pic.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
diff --git a/qemu/hw/intc/omap_intc.c b/qemu/hw/intc/omap_intc.c
index e9b38a3c6..336882510 100644
--- a/qemu/hw/intc/omap_intc.c
+++ b/qemu/hw/intc/omap_intc.c
@@ -17,9 +17,11 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 
 /* Interrupt Handlers */
 struct omap_intr_handler_bank_s {
@@ -367,7 +369,8 @@ static int omap_intc_init(SysBusDevice *sbd)
     struct omap_intr_handler_s *s = OMAP_INTC(dev);
 
     if (!s->iclk) {
-        hw_error("omap-intc: clk not connected\n");
+        error_report("omap-intc: clk not connected");
+        return -1;
     }
     s->nbanks = 1;
     sysbus_init_irq(sbd, &s->parent_intr[0]);
@@ -608,10 +611,12 @@ static int omap2_intc_init(SysBusDevice *sbd)
     struct omap_intr_handler_s *s = OMAP_INTC(dev);
 
     if (!s->iclk) {
-        hw_error("omap2-intc: iclk not connected\n");
+        error_report("omap2-intc: iclk not connected");
+        return -1;
     }
     if (!s->fclk) {
-        hw_error("omap2-intc: fclk not connected\n");
+        error_report("omap2-intc: fclk not connected");
+        return -1;
     }
     s->level_only = 1;
     s->nbanks = 3;
diff --git a/qemu/hw/intc/openpic.c b/qemu/hw/intc/openpic.c
index 14ab0e31b..2d3769310 100644
--- a/qemu/hw/intc/openpic.c
+++ b/qemu/hw/intc/openpic.c
@@ -33,6 +33,7 @@
  * Serial interrupts, as implemented in Raven chipset are not supported yet.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -40,6 +41,7 @@
 #include "hw/ppc/ppc_e500.h"
 #include "hw/sysbus.h"
 #include "hw/pci/msi.h"
+#include "qapi/error.h"
 #include "qemu/bitops.h"
 #include "qapi/qmp/qerror.h"
 
@@ -1374,7 +1376,7 @@ static void fsl_common_init(OpenPICState *opp)
 
     opp->irq_msi = 224;
 
-    msi_supported = true;
+    msi_nonbroken = true;
     for (i = 0; i < opp->fsl->max_ext; i++) {
         opp->src[i].level = false;
     }
@@ -1643,6 +1645,7 @@ static void openpic_class_init(ObjectClass *oc, void *data)
     dc->props = openpic_properties;
     dc->reset = openpic_reset;
     dc->vmsd = &vmstate_openpic;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo openpic_info = {
diff --git a/qemu/hw/intc/openpic_kvm.c b/qemu/hw/intc/openpic_kvm.c
index f7cac585a..e47e94f2c 100644
--- a/qemu/hw/intc/openpic_kvm.c
+++ b/qemu/hw/intc/openpic_kvm.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
@@ -238,7 +240,7 @@ static void kvm_openpic_realize(DeviceState *dev, Error **errp)
     memory_listener_register(&opp->mem_listener, &address_space_memory);
 
     /* indicate pic capabilities */
-    msi_supported = true;
+    msi_nonbroken = true;
     kvm_kernel_irqchip = true;
     kvm_async_interrupts_allowed = true;
 
@@ -275,6 +277,7 @@ static void kvm_openpic_class_init(ObjectClass *oc, void *data)
     dc->realize = kvm_openpic_realize;
     dc->props = kvm_openpic_properties;
     dc->reset = kvm_openpic_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo kvm_openpic_info = {
diff --git a/qemu/hw/intc/pl190.c b/qemu/hw/intc/pl190.c
index 2bf359a76..5ecbc4a48 100644
--- a/qemu/hw/intc/pl190.c
+++ b/qemu/hw/intc/pl190.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 /* The number of virtual priority levels.  16 user vectors plus the
diff --git a/qemu/hw/intc/puv3_intc.c b/qemu/hw/intc/puv3_intc.c
index c2803d07d..ef8488aac 100644
--- a/qemu/hw/intc/puv3_intc.c
+++ b/qemu/hw/intc/puv3_intc.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #undef DEBUG_PUV3
diff --git a/qemu/hw/intc/realview_gic.c b/qemu/hw/intc/realview_gic.c
index 6c812961c..50bbab66e 100644
--- a/qemu/hw/intc/realview_gic.c
+++ b/qemu/hw/intc/realview_gic.c
@@ -7,6 +7,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/intc/realview_gic.h"
 
 static void realview_gic_set_irq(void *opaque, int irq, int level)
diff --git a/qemu/hw/intc/s390_flic.c b/qemu/hw/intc/s390_flic.c
index 02e10b750..bc75fa7d9 100644
--- a/qemu/hw/intc/s390_flic.c
+++ b/qemu/hw/intc/s390_flic.c
@@ -10,6 +10,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
 #include "migration/qemu-file.h"
diff --git a/qemu/hw/intc/s390_flic_kvm.c b/qemu/hw/intc/s390_flic_kvm.c
index b471e7a41..02449b390 100644
--- a/qemu/hw/intc/s390_flic_kvm.c
+++ b/qemu/hw/intc/s390_flic_kvm.c
@@ -10,6 +10,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
@@ -228,6 +229,8 @@ static int kvm_s390_add_adapter_routes(S390FLICState *fs,
         routes->gsi[i] = ret;
         routes->adapter.ind_offset++;
     }
+    kvm_irqchip_commit_routes(kvm_state);
+
     /* Restore passed-in structure to original state. */
     routes->adapter.ind_offset = ind_offset;
     return 0;
diff --git a/qemu/hw/intc/sh_intc.c b/qemu/hw/intc/sh_intc.c
index 55c76e4af..6ce2a8084 100644
--- a/qemu/hw/intc/sh_intc.c
+++ b/qemu/hw/intc/sh_intc.c
@@ -8,6 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sh4/sh_intc.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
diff --git a/qemu/hw/intc/slavio_intctl.c b/qemu/hw/intc/slavio_intctl.c
index f22aba031..c9486ed99 100644
--- a/qemu/hw/intc/slavio_intctl.c
+++ b/qemu/hw/intc/slavio_intctl.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "monitor/monitor.h"
 #include "hw/sysbus.h"
diff --git a/qemu/hw/intc/vgic_common.h b/qemu/hw/intc/vgic_common.h
new file mode 100644
index 000000000..80d919eb9
--- /dev/null
+++ b/qemu/hw/intc/vgic_common.h
@@ -0,0 +1,35 @@
+/*
+ * ARM KVM vGIC utility functions
+ *
+ * Copyright (c) 2015 Samsung Electronics
+ * Written by Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_ARM_VGIC_COMMON_H
+#define QEMU_ARM_VGIC_COMMON_H
+
+/**
+ * kvm_arm_gic_set_irq - Send an IRQ to the in-kernel vGIC
+ * @num_irq: Total number of IRQs configured for the GIC instance
+ * @irq: qemu internal IRQ line number:
+ *  [0..N-1] : external interrupts
+ *  [N..N+31] : PPI (internal) interrupts for CPU 0
+ *  [N+32..N+63] : PPI (internal interrupts for CPU 1
+ * @level: level of the IRQ line.
+ */
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level);
+
+#endif
diff --git a/qemu/hw/intc/xics.c b/qemu/hw/intc/xics.c
index 924b1ae3c..8659be017 100644
--- a/qemu/hw/intc/xics.c
+++ b/qemu/hw/intc/xics.c
@@ -25,6 +25,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "trace.h"
 #include "qemu/timer.h"
@@ -88,24 +92,24 @@ static void xics_common_reset(DeviceState *d)
     device_reset(DEVICE(icp->ics));
 }
 
-static void xics_prop_get_nr_irqs(Object *obj, Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void xics_prop_get_nr_irqs(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     XICSState *icp = XICS_COMMON(obj);
     int64_t value = icp->nr_irqs;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void xics_prop_set_nr_irqs(Object *obj, Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void xics_prop_set_nr_irqs(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     XICSState *icp = XICS_COMMON(obj);
     XICSStateClass *info = XICS_COMMON_GET_CLASS(icp);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -122,17 +126,17 @@ static void xics_prop_set_nr_irqs(Object *obj, Visitor *v,
 }
 
 static void xics_prop_get_nr_servers(Object *obj, Visitor *v,
-                                     void *opaque, const char *name,
+                                     const char *name, void *opaque,
                                      Error **errp)
 {
     XICSState *icp = XICS_COMMON(obj);
     int64_t value = icp->nr_servers;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void xics_prop_set_nr_servers(Object *obj, Visitor *v,
-                                     void *opaque, const char *name,
+                                     const char *name, void *opaque,
                                      Error **errp)
 {
     XICSState *icp = XICS_COMMON(obj);
@@ -140,7 +144,7 @@ static void xics_prop_set_nr_servers(Object *obj, Visitor *v,
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -711,7 +715,7 @@ static int ics_find_free_block(ICSState *ics, int num, int alignnum)
     return -1;
 }
 
-int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi)
+int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi, Error **errp)
 {
     ICSState *ics = &icp->ics[src];
     int irq;
@@ -719,14 +723,14 @@ int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi)
     if (irq_hint) {
         assert(src == xics_find_source(icp, irq_hint));
         if (!ICS_IRQ_FREE(ics, irq_hint - ics->offset)) {
-            trace_xics_alloc_failed_hint(src, irq_hint);
+            error_setg(errp, "can't allocate IRQ %d: already in use", irq_hint);
             return -1;
         }
         irq = irq_hint;
     } else {
         irq = ics_find_free_block(ics, 1, 1);
         if (irq < 0) {
-            trace_xics_alloc_failed_no_left(src);
+            error_setg(errp, "can't allocate IRQ: no IRQ left");
             return -1;
         }
         irq += ics->offset;
@@ -739,10 +743,11 @@ int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi)
 }
 
 /*
- * Allocate block of consequtive IRQs, returns a number of the first.
+ * Allocate block of consecutive IRQs, and return the number of the first IRQ in the block.
  * If align==true, aligns the first IRQ number to num.
  */
-int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align)
+int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align,
+                     Error **errp)
 {
     int i, first = -1;
     ICSState *ics = &icp->ics[src];
@@ -762,6 +767,10 @@ int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align)
     } else {
         first = ics_find_free_block(ics, num, 1);
     }
+    if (first < 0) {
+        error_setg(errp, "can't find a free %d-IRQ block", num);
+        return -1;
+    }
 
     if (first >= 0) {
         for (i = first; i < first + num; ++i) {
@@ -848,7 +857,7 @@ static target_ulong h_xirr_x(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     uint32_t xirr = icp_accept(ss);
 
     args[0] = xirr;
-    args[1] = cpu_get_real_ticks();
+    args[1] = cpu_get_host_ticks();
     return H_SUCCESS;
 }
 
diff --git a/qemu/hw/intc/xics_kvm.c b/qemu/hw/intc/xics_kvm.c
index d58729cfa..9029d9ee0 100644
--- a/qemu/hw/intc/xics_kvm.c
+++ b/qemu/hw/intc/xics_kvm.c
@@ -25,6 +25,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "trace.h"
 #include "hw/ppc/spapr.h"
diff --git a/qemu/hw/intc/xilinx_intc.c b/qemu/hw/intc/xilinx_intc.c
index 12804ab72..9d8139bc6 100644
--- a/qemu/hw/intc/xilinx_intc.c
+++ b/qemu/hw/intc/xilinx_intc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 
diff --git a/qemu/hw/ipack/ipack.c b/qemu/hw/ipack/ipack.c
index 59bfe2869..5f99ed9a7 100644
--- a/qemu/hw/ipack/ipack.c
+++ b/qemu/hw/ipack/ipack.c
@@ -8,6 +8,8 @@
  * later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/ipack/ipack.h"
 
 IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot)
diff --git a/qemu/hw/ipack/tpci200.c b/qemu/hw/ipack/tpci200.c
index 1df02ee85..fdda6f414 100644
--- a/qemu/hw/ipack/tpci200.c
+++ b/qemu/hw/ipack/tpci200.c
@@ -8,10 +8,10 @@
  * later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ipack/ipack.h"
 #include "hw/pci/pci.h"
 #include "qemu/bitops.h"
-#include <stdio.h>
 
 /* #define DEBUG_TPCI */
 
diff --git a/qemu/hw/ipmi/Makefile.objs b/qemu/hw/ipmi/Makefile.objs
new file mode 100644
index 000000000..a90318d5b
--- /dev/null
+++ b/qemu/hw/ipmi/Makefile.objs
@@ -0,0 +1,5 @@
+common-obj-$(CONFIG_IPMI) += ipmi.o
+common-obj-$(CONFIG_IPMI_LOCAL) += ipmi_bmc_sim.o
+common-obj-$(CONFIG_IPMI_LOCAL) += ipmi_bmc_extern.o
+common-obj-$(CONFIG_ISA_IPMI_KCS) += isa_ipmi_kcs.o
+common-obj-$(CONFIG_ISA_IPMI_BT) += isa_ipmi_bt.o
diff --git a/qemu/hw/ipmi/ipmi.c b/qemu/hw/ipmi/ipmi.c
new file mode 100644
index 000000000..6adec1e99
--- /dev/null
+++ b/qemu/hw/ipmi/ipmi.c
@@ -0,0 +1,151 @@
+/*
+ * QEMU IPMI emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "sysemu/sysemu.h"
+#include "qmp-commands.h"
+#include "qom/object_interfaces.h"
+#include "qapi/visitor.h"
+
+static int ipmi_do_hw_op(IPMIInterface *s, enum ipmi_op op, int checkonly)
+{
+    switch (op) {
+    case IPMI_RESET_CHASSIS:
+        if (checkonly) {
+            return 0;
+        }
+        qemu_system_reset_request();
+        return 0;
+
+    case IPMI_POWEROFF_CHASSIS:
+        if (checkonly) {
+            return 0;
+        }
+        qemu_system_powerdown_request();
+        return 0;
+
+    case IPMI_SEND_NMI:
+        if (checkonly) {
+            return 0;
+        }
+        qmp_inject_nmi(NULL);
+        return 0;
+
+    case IPMI_POWERCYCLE_CHASSIS:
+    case IPMI_PULSE_DIAG_IRQ:
+    case IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP:
+    case IPMI_POWERON_CHASSIS:
+    default:
+        return IPMI_CC_COMMAND_NOT_SUPPORTED;
+    }
+}
+
+static void ipmi_interface_class_init(ObjectClass *class, void *data)
+{
+    IPMIInterfaceClass *ik = IPMI_INTERFACE_CLASS(class);
+
+    ik->do_hw_op = ipmi_do_hw_op;
+}
+
+static TypeInfo ipmi_interface_type_info = {
+    .name = TYPE_IPMI_INTERFACE,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(IPMIInterfaceClass),
+    .class_init = ipmi_interface_class_init,
+};
+
+static void isa_ipmi_bmc_check(Object *obj, const char *name,
+                               Object *val, Error **errp)
+{
+    IPMIBmc *bmc = IPMI_BMC(val);
+
+    if (bmc->intf)
+        error_setg(errp, "BMC object is already in use");
+}
+
+void ipmi_bmc_find_and_link(Object *obj, Object **bmc)
+{
+    object_property_add_link(obj, "bmc", TYPE_IPMI_BMC, bmc,
+                             isa_ipmi_bmc_check,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
+
+static Property ipmi_bmc_properties[] = {
+    DEFINE_PROP_UINT8("slave_addr",  IPMIBmc, slave_addr, 0x20),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void bmc_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->props = ipmi_bmc_properties;
+}
+
+static TypeInfo ipmi_bmc_type_info = {
+    .name = TYPE_IPMI_BMC,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(IPMIBmc),
+    .abstract = true,
+    .class_size = sizeof(IPMIBmcClass),
+    .class_init = bmc_class_init,
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&ipmi_interface_type_info);
+    type_register_static(&ipmi_bmc_type_info);
+}
+
+type_init(ipmi_register_types)
+
+static IPMIFwInfo *ipmi_fw_info;
+static unsigned int ipmi_fw_info_len;
+
+static uint32_t current_uuid = 1;
+
+void ipmi_add_fwinfo(IPMIFwInfo *info, Error **errp)
+{
+    info->uuid = current_uuid++;
+    ipmi_fw_info = g_realloc(ipmi_fw_info,
+                             sizeof(*ipmi_fw_info) * (ipmi_fw_info_len + 1));
+    ipmi_fw_info[ipmi_fw_info_len] = *info;
+}
+
+IPMIFwInfo *ipmi_first_fwinfo(void)
+{
+    return ipmi_fw_info;
+}
+
+IPMIFwInfo *ipmi_next_fwinfo(IPMIFwInfo *current)
+{
+    current++;
+    if (current >= &ipmi_fw_info[ipmi_fw_info_len]) {
+        return NULL;
+    }
+    return current;
+}
diff --git a/qemu/hw/ipmi/ipmi_bmc_extern.c b/qemu/hw/ipmi/ipmi_bmc_extern.c
new file mode 100644
index 000000000..fe12112a2
--- /dev/null
+++ b/qemu/hw/ipmi/ipmi_bmc_extern.c
@@ -0,0 +1,519 @@
+/*
+ * IPMI BMC external connection
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * This is designed to connect with OpenIPMI's lanserv serial interface
+ * using the "VM" connection type.  See that for details.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/timer.h"
+#include "sysemu/char.h"
+#include "sysemu/sysemu.h"
+#include "hw/ipmi/ipmi.h"
+
+#define VM_MSG_CHAR        0xA0 /* Marks end of message */
+#define VM_CMD_CHAR        0xA1 /* Marks end of a command */
+#define VM_ESCAPE_CHAR     0xAA /* Set bit 4 from the next byte to 0 */
+
+#define VM_PROTOCOL_VERSION        1
+#define VM_CMD_VERSION             0xff /* A version number byte follows */
+#define VM_CMD_NOATTN              0x00
+#define VM_CMD_ATTN                0x01
+#define VM_CMD_ATTN_IRQ            0x02
+#define VM_CMD_POWEROFF            0x03
+#define VM_CMD_RESET               0x04
+#define VM_CMD_ENABLE_IRQ          0x05 /* Enable/disable the messaging irq */
+#define VM_CMD_DISABLE_IRQ         0x06
+#define VM_CMD_SEND_NMI            0x07
+#define VM_CMD_CAPABILITIES        0x08
+#define   VM_CAPABILITIES_POWER    0x01
+#define   VM_CAPABILITIES_RESET    0x02
+#define   VM_CAPABILITIES_IRQ      0x04
+#define   VM_CAPABILITIES_NMI      0x08
+#define   VM_CAPABILITIES_ATTN     0x10
+#define VM_CMD_FORCEOFF            0x09
+
+#define TYPE_IPMI_BMC_EXTERN "ipmi-bmc-extern"
+#define IPMI_BMC_EXTERN(obj) OBJECT_CHECK(IPMIBmcExtern, (obj), \
+                                        TYPE_IPMI_BMC_EXTERN)
+typedef struct IPMIBmcExtern {
+    IPMIBmc parent;
+
+    CharDriverState *chr;
+
+    bool connected;
+
+    unsigned char inbuf[MAX_IPMI_MSG_SIZE + 2];
+    unsigned int inpos;
+    bool in_escape;
+    bool in_too_many;
+    bool waiting_rsp;
+    bool sending_cmd;
+
+    unsigned char outbuf[(MAX_IPMI_MSG_SIZE + 2) * 2 + 1];
+    unsigned int outpos;
+    unsigned int outlen;
+
+    struct QEMUTimer *extern_timer;
+
+    /* A reset event is pending to be sent upstream. */
+    bool send_reset;
+} IPMIBmcExtern;
+
+static int can_receive(void *opaque);
+static void receive(void *opaque, const uint8_t *buf, int size);
+static void chr_event(void *opaque, int event);
+
+static unsigned char
+ipmb_checksum(const unsigned char *data, int size, unsigned char start)
+{
+        unsigned char csum = start;
+
+        for (; size > 0; size--, data++) {
+                csum += *data;
+        }
+        return csum;
+}
+
+static void continue_send(IPMIBmcExtern *ibe)
+{
+    if (ibe->outlen == 0) {
+        goto check_reset;
+    }
+ send:
+    ibe->outpos += qemu_chr_fe_write(ibe->chr, ibe->outbuf + ibe->outpos,
+                                     ibe->outlen - ibe->outpos);
+    if (ibe->outpos < ibe->outlen) {
+        /* Not fully transmitted, try again in a 10ms */
+        timer_mod_ns(ibe->extern_timer,
+                     qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 10000000);
+    } else {
+        /* Sent */
+        ibe->outlen = 0;
+        ibe->outpos = 0;
+        if (!ibe->sending_cmd) {
+            ibe->waiting_rsp = true;
+        } else {
+            ibe->sending_cmd = false;
+        }
+    check_reset:
+        if (ibe->connected && ibe->send_reset) {
+            /* Send the reset */
+            ibe->outbuf[0] = VM_CMD_RESET;
+            ibe->outbuf[1] = VM_CMD_CHAR;
+            ibe->outlen = 2;
+            ibe->outpos = 0;
+            ibe->send_reset = false;
+            ibe->sending_cmd = true;
+            goto send;
+        }
+
+        if (ibe->waiting_rsp) {
+            /* Make sure we get a response within 4 seconds. */
+            timer_mod_ns(ibe->extern_timer,
+                         qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 4000000000ULL);
+        }
+    }
+    return;
+}
+
+static void extern_timeout(void *opaque)
+{
+    IPMIBmcExtern *ibe = opaque;
+    IPMIInterface *s = ibe->parent.intf;
+
+    if (ibe->connected) {
+        if (ibe->waiting_rsp && (ibe->outlen == 0)) {
+            IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+            /* The message response timed out, return an error. */
+            ibe->waiting_rsp = false;
+            ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+            ibe->inbuf[2] = ibe->outbuf[2];
+            ibe->inbuf[3] = IPMI_CC_TIMEOUT;
+            k->handle_rsp(s, ibe->outbuf[0], ibe->inbuf + 1, 3);
+        } else {
+            continue_send(ibe);
+        }
+    }
+}
+
+static void addchar(IPMIBmcExtern *ibe, unsigned char ch)
+{
+    switch (ch) {
+    case VM_MSG_CHAR:
+    case VM_CMD_CHAR:
+    case VM_ESCAPE_CHAR:
+        ibe->outbuf[ibe->outlen] = VM_ESCAPE_CHAR;
+        ibe->outlen++;
+        ch |= 0x10;
+        /* No break */
+
+    default:
+        ibe->outbuf[ibe->outlen] = ch;
+        ibe->outlen++;
+    }
+}
+
+static void ipmi_bmc_extern_handle_command(IPMIBmc *b,
+                                       uint8_t *cmd, unsigned int cmd_len,
+                                       unsigned int max_cmd_len,
+                                       uint8_t msg_id)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(b);
+    IPMIInterface *s = ibe->parent.intf;
+    uint8_t err = 0, csum;
+    unsigned int i;
+
+    if (ibe->outlen) {
+        /* We already have a command queued.  Shouldn't ever happen. */
+        fprintf(stderr, "IPMI KCS: Got command when not finished with the"
+                " previous commmand\n");
+        abort();
+    }
+
+    /* If it's too short or it was truncated, return an error. */
+    if (cmd_len < 2) {
+        err = IPMI_CC_REQUEST_DATA_LENGTH_INVALID;
+    } else if ((cmd_len > max_cmd_len) || (cmd_len > MAX_IPMI_MSG_SIZE)) {
+        err = IPMI_CC_REQUEST_DATA_TRUNCATED;
+    } else if (!ibe->connected) {
+        err = IPMI_CC_BMC_INIT_IN_PROGRESS;
+    }
+    if (err) {
+        IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+        unsigned char rsp[3];
+        rsp[0] = cmd[0] | 0x04;
+        rsp[1] = cmd[1];
+        rsp[2] = err;
+        ibe->waiting_rsp = false;
+        k->handle_rsp(s, msg_id, rsp, 3);
+        goto out;
+    }
+
+    addchar(ibe, msg_id);
+    for (i = 0; i < cmd_len; i++) {
+        addchar(ibe, cmd[i]);
+    }
+    csum = ipmb_checksum(&msg_id, 1, 0);
+    addchar(ibe, -ipmb_checksum(cmd, cmd_len, csum));
+
+    ibe->outbuf[ibe->outlen] = VM_MSG_CHAR;
+    ibe->outlen++;
+
+    /* Start the transmit */
+    continue_send(ibe);
+
+ out:
+    return;
+}
+
+static void handle_hw_op(IPMIBmcExtern *ibe, unsigned char hw_op)
+{
+    IPMIInterface *s = ibe->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    switch (hw_op) {
+    case VM_CMD_VERSION:
+        /* We only support one version at this time. */
+        break;
+
+    case VM_CMD_NOATTN:
+        k->set_atn(s, 0, 0);
+        break;
+
+    case VM_CMD_ATTN:
+        k->set_atn(s, 1, 0);
+        break;
+
+    case VM_CMD_ATTN_IRQ:
+        k->set_atn(s, 1, 1);
+        break;
+
+    case VM_CMD_POWEROFF:
+        k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0);
+        break;
+
+    case VM_CMD_RESET:
+        k->do_hw_op(s, IPMI_RESET_CHASSIS, 0);
+        break;
+
+    case VM_CMD_ENABLE_IRQ:
+        k->set_irq_enable(s, 1);
+        break;
+
+    case VM_CMD_DISABLE_IRQ:
+        k->set_irq_enable(s, 0);
+        break;
+
+    case VM_CMD_SEND_NMI:
+        k->do_hw_op(s, IPMI_SEND_NMI, 0);
+        break;
+
+    case VM_CMD_FORCEOFF:
+        qemu_system_shutdown_request();
+        break;
+    }
+}
+
+static void handle_msg(IPMIBmcExtern *ibe)
+{
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(ibe->parent.intf);
+
+    if (ibe->in_escape) {
+        ipmi_debug("msg escape not ended\n");
+        return;
+    }
+    if (ibe->inpos < 5) {
+        ipmi_debug("msg too short\n");
+        return;
+    }
+    if (ibe->in_too_many) {
+        ibe->inbuf[3] = IPMI_CC_REQUEST_DATA_TRUNCATED;
+        ibe->inpos = 4;
+    } else if (ipmb_checksum(ibe->inbuf, ibe->inpos, 0) != 0) {
+        ipmi_debug("msg checksum failure\n");
+        return;
+    } else {
+        ibe->inpos--; /* Remove checkum */
+    }
+
+    timer_del(ibe->extern_timer);
+    ibe->waiting_rsp = false;
+    k->handle_rsp(ibe->parent.intf, ibe->inbuf[0], ibe->inbuf + 1, ibe->inpos - 1);
+}
+
+static int can_receive(void *opaque)
+{
+    return 1;
+}
+
+static void receive(void *opaque, const uint8_t *buf, int size)
+{
+    IPMIBmcExtern *ibe = opaque;
+    int i;
+    unsigned char hw_op;
+
+    for (i = 0; i < size; i++) {
+        unsigned char ch = buf[i];
+
+        switch (ch) {
+        case VM_MSG_CHAR:
+            handle_msg(ibe);
+            ibe->in_too_many = false;
+            ibe->inpos = 0;
+            break;
+
+        case VM_CMD_CHAR:
+            if (ibe->in_too_many) {
+                ipmi_debug("cmd in too many\n");
+                ibe->in_too_many = false;
+                ibe->inpos = 0;
+                break;
+            }
+            if (ibe->in_escape) {
+                ipmi_debug("cmd in escape\n");
+                ibe->in_too_many = false;
+                ibe->inpos = 0;
+                ibe->in_escape = false;
+                break;
+            }
+            ibe->in_too_many = false;
+            if (ibe->inpos < 1) {
+                break;
+            }
+            hw_op = ibe->inbuf[0];
+            ibe->inpos = 0;
+            goto out_hw_op;
+            break;
+
+        case VM_ESCAPE_CHAR:
+            ibe->in_escape = true;
+            break;
+
+        default:
+            if (ibe->in_escape) {
+                ch &= ~0x10;
+                ibe->in_escape = false;
+            }
+            if (ibe->in_too_many) {
+                break;
+            }
+            if (ibe->inpos >= sizeof(ibe->inbuf)) {
+                ibe->in_too_many = true;
+                break;
+            }
+            ibe->inbuf[ibe->inpos] = ch;
+            ibe->inpos++;
+            break;
+        }
+    }
+    return;
+
+ out_hw_op:
+    handle_hw_op(ibe, hw_op);
+}
+
+static void chr_event(void *opaque, int event)
+{
+    IPMIBmcExtern *ibe = opaque;
+    IPMIInterface *s = ibe->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned char v;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        ibe->connected = true;
+        ibe->outpos = 0;
+        ibe->outlen = 0;
+        addchar(ibe, VM_CMD_VERSION);
+        addchar(ibe, VM_PROTOCOL_VERSION);
+        ibe->outbuf[ibe->outlen] = VM_CMD_CHAR;
+        ibe->outlen++;
+        addchar(ibe, VM_CMD_CAPABILITIES);
+        v = VM_CAPABILITIES_IRQ | VM_CAPABILITIES_ATTN;
+        if (k->do_hw_op(ibe->parent.intf, IPMI_POWEROFF_CHASSIS, 1) == 0) {
+            v |= VM_CAPABILITIES_POWER;
+        }
+        if (k->do_hw_op(ibe->parent.intf, IPMI_RESET_CHASSIS, 1) == 0) {
+            v |= VM_CAPABILITIES_RESET;
+        }
+        if (k->do_hw_op(ibe->parent.intf, IPMI_SEND_NMI, 1) == 0) {
+            v |= VM_CAPABILITIES_NMI;
+        }
+        addchar(ibe, v);
+        ibe->outbuf[ibe->outlen] = VM_CMD_CHAR;
+        ibe->outlen++;
+        ibe->sending_cmd = false;
+        continue_send(ibe);
+        break;
+
+    case CHR_EVENT_CLOSED:
+        if (!ibe->connected) {
+            return;
+        }
+        ibe->connected = false;
+        if (ibe->waiting_rsp) {
+            ibe->waiting_rsp = false;
+            ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+            ibe->inbuf[2] = ibe->outbuf[2];
+            ibe->inbuf[3] = IPMI_CC_BMC_INIT_IN_PROGRESS;
+            k->handle_rsp(s, ibe->outbuf[0], ibe->inbuf + 1, 3);
+        }
+        break;
+    }
+}
+
+static void ipmi_bmc_extern_handle_reset(IPMIBmc *b)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(b);
+
+    ibe->send_reset = true;
+    continue_send(ibe);
+}
+
+static void ipmi_bmc_extern_realize(DeviceState *dev, Error **errp)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(dev);
+
+    if (!ibe->chr) {
+        error_setg(errp, "IPMI external bmc requires chardev attribute");
+        return;
+    }
+
+    qemu_chr_add_handlers(ibe->chr, can_receive, receive, chr_event, ibe);
+}
+
+static int ipmi_bmc_extern_post_migrate(void *opaque, int version_id)
+{
+    IPMIBmcExtern *ibe = opaque;
+
+    /*
+     * We don't directly restore waiting_rsp, Instead, we return an
+     * error on the interface if a response was being waited for.
+     */
+    if (ibe->waiting_rsp) {
+        IPMIInterface *ii = ibe->parent.intf;
+        IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+        ibe->waiting_rsp = false;
+        ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+        ibe->inbuf[2] = ibe->outbuf[2];
+        ibe->inbuf[3] = IPMI_CC_BMC_INIT_IN_PROGRESS;
+        iic->handle_rsp(ii, ibe->outbuf[0], ibe->inbuf + 1, 3);
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_ipmi_bmc_extern = {
+    .name = TYPE_IPMI_BMC_EXTERN,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = ipmi_bmc_extern_post_migrate,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(send_reset, IPMIBmcExtern),
+        VMSTATE_BOOL(waiting_rsp, IPMIBmcExtern),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ipmi_bmc_extern_init(Object *obj)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(obj);
+
+    ibe->extern_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, extern_timeout, ibe);
+    vmstate_register(NULL, 0, &vmstate_ipmi_bmc_extern, ibe);
+}
+
+static Property ipmi_bmc_extern_properties[] = {
+    DEFINE_PROP_CHR("chardev", IPMIBmcExtern, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ipmi_bmc_extern_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIBmcClass *bk = IPMI_BMC_CLASS(oc);
+
+    bk->handle_command = ipmi_bmc_extern_handle_command;
+    bk->handle_reset = ipmi_bmc_extern_handle_reset;
+    dc->realize = ipmi_bmc_extern_realize;
+    dc->props = ipmi_bmc_extern_properties;
+}
+
+static const TypeInfo ipmi_bmc_extern_type = {
+    .name          = TYPE_IPMI_BMC_EXTERN,
+    .parent        = TYPE_IPMI_BMC,
+    .instance_size = sizeof(IPMIBmcExtern),
+    .instance_init = ipmi_bmc_extern_init,
+    .class_init    = ipmi_bmc_extern_class_init,
+ };
+
+static void ipmi_bmc_extern_register_types(void)
+{
+    type_register_static(&ipmi_bmc_extern_type);
+}
+
+type_init(ipmi_bmc_extern_register_types)
diff --git a/qemu/hw/ipmi/ipmi_bmc_sim.c b/qemu/hw/ipmi/ipmi_bmc_sim.c
new file mode 100644
index 000000000..dc9c14cd2
--- /dev/null
+++ b/qemu/hw/ipmi/ipmi_bmc_sim.c
@@ -0,0 +1,1810 @@
+/*
+ * IPMI BMC emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "hw/ipmi/ipmi.h"
+#include "qemu/error-report.h"
+
+#define IPMI_NETFN_CHASSIS            0x00
+
+#define IPMI_CMD_GET_CHASSIS_CAPABILITIES 0x00
+#define IPMI_CMD_GET_CHASSIS_STATUS       0x01
+#define IPMI_CMD_CHASSIS_CONTROL          0x02
+#define IPMI_CMD_GET_SYS_RESTART_CAUSE    0x09
+
+#define IPMI_NETFN_SENSOR_EVENT       0x04
+
+#define IPMI_CMD_SET_SENSOR_EVT_ENABLE    0x28
+#define IPMI_CMD_GET_SENSOR_EVT_ENABLE    0x29
+#define IPMI_CMD_REARM_SENSOR_EVTS        0x2a
+#define IPMI_CMD_GET_SENSOR_EVT_STATUS    0x2b
+#define IPMI_CMD_GET_SENSOR_READING       0x2d
+#define IPMI_CMD_SET_SENSOR_TYPE          0x2e
+#define IPMI_CMD_GET_SENSOR_TYPE          0x2f
+
+/* #define IPMI_NETFN_APP             0x06 In ipmi.h */
+
+#define IPMI_CMD_GET_DEVICE_ID            0x01
+#define IPMI_CMD_COLD_RESET               0x02
+#define IPMI_CMD_WARM_RESET               0x03
+#define IPMI_CMD_SET_ACPI_POWER_STATE     0x06
+#define IPMI_CMD_GET_ACPI_POWER_STATE     0x07
+#define IPMI_CMD_GET_DEVICE_GUID          0x08
+#define IPMI_CMD_RESET_WATCHDOG_TIMER     0x22
+#define IPMI_CMD_SET_WATCHDOG_TIMER       0x24
+#define IPMI_CMD_GET_WATCHDOG_TIMER       0x25
+#define IPMI_CMD_SET_BMC_GLOBAL_ENABLES   0x2e
+#define IPMI_CMD_GET_BMC_GLOBAL_ENABLES   0x2f
+#define IPMI_CMD_CLR_MSG_FLAGS            0x30
+#define IPMI_CMD_GET_MSG_FLAGS            0x31
+#define IPMI_CMD_GET_MSG                  0x33
+#define IPMI_CMD_SEND_MSG                 0x34
+#define IPMI_CMD_READ_EVT_MSG_BUF         0x35
+
+#define IPMI_NETFN_STORAGE            0x0a
+
+#define IPMI_CMD_GET_SDR_REP_INFO         0x20
+#define IPMI_CMD_GET_SDR_REP_ALLOC_INFO   0x21
+#define IPMI_CMD_RESERVE_SDR_REP          0x22
+#define IPMI_CMD_GET_SDR                  0x23
+#define IPMI_CMD_ADD_SDR                  0x24
+#define IPMI_CMD_PARTIAL_ADD_SDR          0x25
+#define IPMI_CMD_DELETE_SDR               0x26
+#define IPMI_CMD_CLEAR_SDR_REP            0x27
+#define IPMI_CMD_GET_SDR_REP_TIME         0x28
+#define IPMI_CMD_SET_SDR_REP_TIME         0x29
+#define IPMI_CMD_ENTER_SDR_REP_UPD_MODE   0x2A
+#define IPMI_CMD_EXIT_SDR_REP_UPD_MODE    0x2B
+#define IPMI_CMD_RUN_INIT_AGENT           0x2C
+#define IPMI_CMD_GET_SEL_INFO             0x40
+#define IPMI_CMD_GET_SEL_ALLOC_INFO       0x41
+#define IPMI_CMD_RESERVE_SEL              0x42
+#define IPMI_CMD_GET_SEL_ENTRY            0x43
+#define IPMI_CMD_ADD_SEL_ENTRY            0x44
+#define IPMI_CMD_PARTIAL_ADD_SEL_ENTRY    0x45
+#define IPMI_CMD_DELETE_SEL_ENTRY         0x46
+#define IPMI_CMD_CLEAR_SEL                0x47
+#define IPMI_CMD_GET_SEL_TIME             0x48
+#define IPMI_CMD_SET_SEL_TIME             0x49
+
+
+/* Same as a timespec struct. */
+struct ipmi_time {
+    long tv_sec;
+    long tv_nsec;
+};
+
+#define MAX_SEL_SIZE 128
+
+typedef struct IPMISel {
+    uint8_t sel[MAX_SEL_SIZE][16];
+    unsigned int next_free;
+    long time_offset;
+    uint16_t reservation;
+    uint8_t last_addition[4];
+    uint8_t last_clear[4];
+    uint8_t overflow;
+} IPMISel;
+
+#define MAX_SDR_SIZE 16384
+
+typedef struct IPMISdr {
+    uint8_t sdr[MAX_SDR_SIZE];
+    unsigned int next_free;
+    uint16_t next_rec_id;
+    uint16_t reservation;
+    uint8_t last_addition[4];
+    uint8_t last_clear[4];
+    uint8_t overflow;
+} IPMISdr;
+
+typedef struct IPMISensor {
+    uint8_t status;
+    uint8_t reading;
+    uint16_t states_suppt;
+    uint16_t assert_suppt;
+    uint16_t deassert_suppt;
+    uint16_t states;
+    uint16_t assert_states;
+    uint16_t deassert_states;
+    uint16_t assert_enable;
+    uint16_t deassert_enable;
+    uint8_t  sensor_type;
+    uint8_t  evt_reading_type_code;
+} IPMISensor;
+#define IPMI_SENSOR_GET_PRESENT(s)       ((s)->status & 0x01)
+#define IPMI_SENSOR_SET_PRESENT(s, v)    ((s)->status = (s->status & ~0x01) | \
+                                             !!(v))
+#define IPMI_SENSOR_GET_SCAN_ON(s)       ((s)->status & 0x40)
+#define IPMI_SENSOR_SET_SCAN_ON(s, v)    ((s)->status = (s->status & ~0x40) | \
+                                             ((!!(v)) << 6))
+#define IPMI_SENSOR_GET_EVENTS_ON(s)     ((s)->status & 0x80)
+#define IPMI_SENSOR_SET_EVENTS_ON(s, v)  ((s)->status = (s->status & ~0x80) | \
+                                             ((!!(v)) << 7))
+#define IPMI_SENSOR_GET_RET_STATUS(s)    ((s)->status & 0xc0)
+#define IPMI_SENSOR_SET_RET_STATUS(s, v) ((s)->status = (s->status & ~0xc0) | \
+                                             (v & 0xc0))
+#define IPMI_SENSOR_IS_DISCRETE(s) ((s)->evt_reading_type_code != 1)
+
+#define MAX_SENSORS 20
+#define IPMI_WATCHDOG_SENSOR 0
+
+typedef struct IPMIBmcSim IPMIBmcSim;
+typedef struct RspBuffer RspBuffer;
+
+#define MAX_NETFNS 64
+
+typedef struct IPMICmdHandler {
+    void (*cmd_handler)(IPMIBmcSim *s,
+                        uint8_t *cmd, unsigned int cmd_len,
+                        RspBuffer *rsp);
+    unsigned int cmd_len_min;
+} IPMICmdHandler;
+
+typedef struct IPMINetfn {
+    unsigned int cmd_nums;
+    const IPMICmdHandler *cmd_handlers;
+} IPMINetfn;
+
+typedef struct IPMIRcvBufEntry {
+    QTAILQ_ENTRY(IPMIRcvBufEntry) entry;
+    uint8_t len;
+    uint8_t buf[MAX_IPMI_MSG_SIZE];
+} IPMIRcvBufEntry;
+
+#define TYPE_IPMI_BMC_SIMULATOR "ipmi-bmc-sim"
+#define IPMI_BMC_SIMULATOR(obj) OBJECT_CHECK(IPMIBmcSim, (obj), \
+                                        TYPE_IPMI_BMC_SIMULATOR)
+struct IPMIBmcSim {
+    IPMIBmc parent;
+
+    QEMUTimer *timer;
+
+    uint8_t bmc_global_enables;
+    uint8_t msg_flags;
+
+    bool     watchdog_initialized;
+    uint8_t  watchdog_use;
+    uint8_t  watchdog_action;
+    uint8_t  watchdog_pretimeout; /* In seconds */
+    bool     watchdog_expired;
+    uint16_t watchdog_timeout; /* in 100's of milliseconds */
+
+    bool     watchdog_running;
+    bool     watchdog_preaction_ran;
+    int64_t  watchdog_expiry;
+
+    uint8_t device_id;
+    uint8_t ipmi_version;
+    uint8_t device_rev;
+    uint8_t fwrev1;
+    uint8_t fwrev2;
+    uint8_t mfg_id[3];
+    uint8_t product_id[2];
+
+    uint8_t restart_cause;
+
+    uint8_t acpi_power_state[2];
+    uint8_t uuid[16];
+
+    IPMISel sel;
+    IPMISdr sdr;
+    IPMISensor sensors[MAX_SENSORS];
+
+    /* Odd netfns are for responses, so we only need the even ones. */
+    const IPMINetfn *netfns[MAX_NETFNS / 2];
+
+    QemuMutex lock;
+    /* We allow one event in the buffer */
+    uint8_t evtbuf[16];
+
+    QTAILQ_HEAD(, IPMIRcvBufEntry) rcvbufs;
+};
+
+#define IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK        (1 << 3)
+#define IPMI_BMC_MSG_FLAG_EVT_BUF_FULL                 (1 << 1)
+#define IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE                (1 << 0)
+#define IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK_SET(s) \
+    (IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK & (s)->msg_flags)
+#define IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(s) \
+    (IPMI_BMC_MSG_FLAG_EVT_BUF_FULL & (s)->msg_flags)
+#define IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(s) \
+    (IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE & (s)->msg_flags)
+
+#define IPMI_BMC_RCV_MSG_QUEUE_INT_BIT    0
+#define IPMI_BMC_EVBUF_FULL_INT_BIT       1
+#define IPMI_BMC_EVENT_MSG_BUF_BIT        2
+#define IPMI_BMC_EVENT_LOG_BIT            3
+#define IPMI_BMC_MSG_INTS_ON(s) ((s)->bmc_global_enables & \
+                                 (1 << IPMI_BMC_RCV_MSG_QUEUE_INT_BIT))
+#define IPMI_BMC_EVBUF_FULL_INT_ENABLED(s) ((s)->bmc_global_enables & \
+                                        (1 << IPMI_BMC_EVBUF_FULL_INT_BIT))
+#define IPMI_BMC_EVENT_LOG_ENABLED(s) ((s)->bmc_global_enables & \
+                                       (1 << IPMI_BMC_EVENT_LOG_BIT))
+#define IPMI_BMC_EVENT_MSG_BUF_ENABLED(s) ((s)->bmc_global_enables & \
+                                           (1 << IPMI_BMC_EVENT_MSG_BUF_BIT))
+
+#define IPMI_BMC_WATCHDOG_USE_MASK 0xc7
+#define IPMI_BMC_WATCHDOG_ACTION_MASK 0x77
+#define IPMI_BMC_WATCHDOG_GET_USE(s) ((s)->watchdog_use & 0x7)
+#define IPMI_BMC_WATCHDOG_GET_DONT_LOG(s) (((s)->watchdog_use >> 7) & 0x1)
+#define IPMI_BMC_WATCHDOG_GET_DONT_STOP(s) (((s)->watchdog_use >> 6) & 0x1)
+#define IPMI_BMC_WATCHDOG_GET_PRE_ACTION(s) (((s)->watchdog_action >> 4) & 0x7)
+#define IPMI_BMC_WATCHDOG_PRE_NONE               0
+#define IPMI_BMC_WATCHDOG_PRE_SMI                1
+#define IPMI_BMC_WATCHDOG_PRE_NMI                2
+#define IPMI_BMC_WATCHDOG_PRE_MSG_INT            3
+#define IPMI_BMC_WATCHDOG_GET_ACTION(s) ((s)->watchdog_action & 0x7)
+#define IPMI_BMC_WATCHDOG_ACTION_NONE            0
+#define IPMI_BMC_WATCHDOG_ACTION_RESET           1
+#define IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN      2
+#define IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE     3
+
+struct RspBuffer {
+    uint8_t buffer[MAX_IPMI_MSG_SIZE];
+    unsigned int len;
+};
+
+#define RSP_BUFFER_INITIALIZER { }
+
+static inline void rsp_buffer_set_error(RspBuffer *rsp, uint8_t byte)
+{
+    rsp->buffer[2] = byte;
+}
+
+/* Add a byte to the response. */
+static inline void rsp_buffer_push(RspBuffer *rsp, uint8_t byte)
+{
+    if (rsp->len >= sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        return;
+    }
+    rsp->buffer[rsp->len++] = byte;
+}
+
+static inline void rsp_buffer_pushmore(RspBuffer *rsp, uint8_t *bytes,
+                                       unsigned int n)
+{
+    if (rsp->len + n >= sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        return;
+    }
+
+    memcpy(&rsp->buffer[rsp->len], bytes, n);
+    rsp->len += n;
+}
+
+static void ipmi_sim_handle_timeout(IPMIBmcSim *ibs);
+
+static void ipmi_gettime(struct ipmi_time *time)
+{
+    int64_t stime;
+
+    stime = qemu_clock_get_ns(QEMU_CLOCK_HOST);
+    time->tv_sec = stime / 1000000000LL;
+    time->tv_nsec = stime % 1000000000LL;
+}
+
+static int64_t ipmi_getmonotime(void)
+{
+    return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+}
+
+static void ipmi_timeout(void *opaque)
+{
+    IPMIBmcSim *ibs = opaque;
+
+    ipmi_sim_handle_timeout(ibs);
+}
+
+static void set_timestamp(IPMIBmcSim *ibs, uint8_t *ts)
+{
+    unsigned int val;
+    struct ipmi_time now;
+
+    ipmi_gettime(&now);
+    val = now.tv_sec + ibs->sel.time_offset;
+    ts[0] = val & 0xff;
+    ts[1] = (val >> 8) & 0xff;
+    ts[2] = (val >> 16) & 0xff;
+    ts[3] = (val >> 24) & 0xff;
+}
+
+static void sdr_inc_reservation(IPMISdr *sdr)
+{
+    sdr->reservation++;
+    if (sdr->reservation == 0) {
+        sdr->reservation = 1;
+    }
+}
+
+static int sdr_add_entry(IPMIBmcSim *ibs,
+                         const struct ipmi_sdr_header *sdrh_entry,
+                         unsigned int len, uint16_t *recid)
+{
+    struct ipmi_sdr_header *sdrh =
+        (struct ipmi_sdr_header *) &ibs->sdr.sdr[ibs->sdr.next_free];
+
+    if ((len < IPMI_SDR_HEADER_SIZE) || (len > 255)) {
+        return 1;
+    }
+
+    if (ipmi_sdr_length(sdrh_entry) != len) {
+        return 1;
+    }
+
+    if (ibs->sdr.next_free + len > MAX_SDR_SIZE) {
+        ibs->sdr.overflow = 1;
+        return 1;
+    }
+
+    memcpy(sdrh, sdrh_entry, len);
+    sdrh->rec_id[0] = ibs->sdr.next_rec_id & 0xff;
+    sdrh->rec_id[1] = (ibs->sdr.next_rec_id >> 8) & 0xff;
+    sdrh->sdr_version = 0x51; /* Conform to IPMI 1.5 spec */
+
+    if (recid) {
+        *recid = ibs->sdr.next_rec_id;
+    }
+    ibs->sdr.next_rec_id++;
+    set_timestamp(ibs, ibs->sdr.last_addition);
+    ibs->sdr.next_free += len;
+    sdr_inc_reservation(&ibs->sdr);
+    return 0;
+}
+
+static int sdr_find_entry(IPMISdr *sdr, uint16_t recid,
+                          unsigned int *retpos, uint16_t *nextrec)
+{
+    unsigned int pos = *retpos;
+
+    while (pos < sdr->next_free) {
+        struct ipmi_sdr_header *sdrh =
+            (struct ipmi_sdr_header *) &sdr->sdr[pos];
+        uint16_t trec = ipmi_sdr_recid(sdrh);
+        unsigned int nextpos = pos + ipmi_sdr_length(sdrh);
+
+        if (trec == recid) {
+            if (nextrec) {
+                if (nextpos >= sdr->next_free) {
+                    *nextrec = 0xffff;
+                } else {
+                    *nextrec = (sdr->sdr[nextpos] |
+                                (sdr->sdr[nextpos + 1] << 8));
+                }
+            }
+            *retpos = pos;
+            return 0;
+        }
+        pos = nextpos;
+    }
+    return 1;
+}
+
+static void sel_inc_reservation(IPMISel *sel)
+{
+    sel->reservation++;
+    if (sel->reservation == 0) {
+        sel->reservation = 1;
+    }
+}
+
+/* Returns 1 if the SEL is full and can't hold the event. */
+static int sel_add_event(IPMIBmcSim *ibs, uint8_t *event)
+{
+    event[0] = 0xff;
+    event[1] = 0xff;
+    set_timestamp(ibs, event + 3);
+    if (ibs->sel.next_free == MAX_SEL_SIZE) {
+        ibs->sel.overflow = 1;
+        return 1;
+    }
+    event[0] = ibs->sel.next_free & 0xff;
+    event[1] = (ibs->sel.next_free >> 8) & 0xff;
+    memcpy(ibs->sel.last_addition, event + 3, 4);
+    memcpy(ibs->sel.sel[ibs->sel.next_free], event, 16);
+    ibs->sel.next_free++;
+    sel_inc_reservation(&ibs->sel);
+    return 0;
+}
+
+static int attn_set(IPMIBmcSim *ibs)
+{
+    return IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(ibs)
+        || IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(ibs)
+        || IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK_SET(ibs);
+}
+
+static int attn_irq_enabled(IPMIBmcSim *ibs)
+{
+    return (IPMI_BMC_MSG_INTS_ON(ibs) && IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(ibs))
+        || (IPMI_BMC_EVBUF_FULL_INT_ENABLED(ibs) &&
+            IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(ibs));
+}
+
+static void gen_event(IPMIBmcSim *ibs, unsigned int sens_num, uint8_t deassert,
+                      uint8_t evd1, uint8_t evd2, uint8_t evd3)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    uint8_t evt[16];
+    IPMISensor *sens = ibs->sensors + sens_num;
+
+    if (!IPMI_BMC_EVENT_MSG_BUF_ENABLED(ibs)) {
+        return;
+    }
+    if (!IPMI_SENSOR_GET_EVENTS_ON(sens)) {
+        return;
+    }
+
+    evt[2] = 0x2; /* System event record */
+    evt[7] = ibs->parent.slave_addr;
+    evt[8] = 0;
+    evt[9] = 0x04; /* Format version */
+    evt[10] = sens->sensor_type;
+    evt[11] = sens_num;
+    evt[12] = sens->evt_reading_type_code | (!!deassert << 7);
+    evt[13] = evd1;
+    evt[14] = evd2;
+    evt[15] = evd3;
+
+    if (IPMI_BMC_EVENT_LOG_ENABLED(ibs)) {
+        sel_add_event(ibs, evt);
+    }
+
+    if (ibs->msg_flags & IPMI_BMC_MSG_FLAG_EVT_BUF_FULL) {
+        return;
+    }
+
+    memcpy(ibs->evtbuf, evt, 16);
+    ibs->msg_flags |= IPMI_BMC_MSG_FLAG_EVT_BUF_FULL;
+    k->set_atn(s, 1, attn_irq_enabled(ibs));
+}
+
+static void sensor_set_discrete_bit(IPMIBmcSim *ibs, unsigned int sensor,
+                                    unsigned int bit, unsigned int val,
+                                    uint8_t evd1, uint8_t evd2, uint8_t evd3)
+{
+    IPMISensor *sens;
+    uint16_t mask;
+
+    if (sensor >= MAX_SENSORS) {
+        return;
+    }
+    if (bit >= 16) {
+        return;
+    }
+
+    mask = (1 << bit);
+    sens = ibs->sensors + sensor;
+    if (val) {
+        sens->states |= mask & sens->states_suppt;
+        if (sens->assert_states & mask) {
+            return; /* Already asserted */
+        }
+        sens->assert_states |= mask & sens->assert_suppt;
+        if (sens->assert_enable & mask & sens->assert_states) {
+            /* Send an event on assert */
+            gen_event(ibs, sensor, 0, evd1, evd2, evd3);
+        }
+    } else {
+        sens->states &= ~(mask & sens->states_suppt);
+        if (sens->deassert_states & mask) {
+            return; /* Already deasserted */
+        }
+        sens->deassert_states |= mask & sens->deassert_suppt;
+        if (sens->deassert_enable & mask & sens->deassert_states) {
+            /* Send an event on deassert */
+            gen_event(ibs, sensor, 1, evd1, evd2, evd3);
+        }
+    }
+}
+
+static void ipmi_init_sensors_from_sdrs(IPMIBmcSim *s)
+{
+    unsigned int i, pos;
+    IPMISensor *sens;
+
+    for (i = 0; i < MAX_SENSORS; i++) {
+        memset(s->sensors + i, 0, sizeof(*sens));
+    }
+
+    pos = 0;
+    for (i = 0; !sdr_find_entry(&s->sdr, i, &pos, NULL); i++) {
+        struct ipmi_sdr_compact *sdr =
+            (struct ipmi_sdr_compact *) &s->sdr.sdr[pos];
+        unsigned int len = sdr->header.rec_length;
+
+        if (len < 20) {
+            continue;
+        }
+        if (sdr->header.rec_type != IPMI_SDR_COMPACT_TYPE) {
+            continue; /* Not a sensor SDR we set from */
+        }
+
+        if (sdr->sensor_owner_number >= MAX_SENSORS) {
+            continue;
+        }
+        sens = s->sensors + sdr->sensor_owner_number;
+
+        IPMI_SENSOR_SET_PRESENT(sens, 1);
+        IPMI_SENSOR_SET_SCAN_ON(sens, (sdr->sensor_init >> 6) & 1);
+        IPMI_SENSOR_SET_EVENTS_ON(sens, (sdr->sensor_init >> 5) & 1);
+        sens->assert_suppt = sdr->assert_mask[0] | (sdr->assert_mask[1] << 8);
+        sens->deassert_suppt =
+            sdr->deassert_mask[0] | (sdr->deassert_mask[1] << 8);
+        sens->states_suppt =
+            sdr->discrete_mask[0] | (sdr->discrete_mask[1] << 8);
+        sens->sensor_type = sdr->sensor_type;
+        sens->evt_reading_type_code = sdr->reading_type & 0x7f;
+
+        /* Enable all the events that are supported. */
+        sens->assert_enable = sens->assert_suppt;
+        sens->deassert_enable = sens->deassert_suppt;
+    }
+}
+
+static int ipmi_register_netfn(IPMIBmcSim *s, unsigned int netfn,
+                               const IPMINetfn *netfnd)
+{
+    if ((netfn & 1) || (netfn >= MAX_NETFNS) || (s->netfns[netfn / 2])) {
+        return -1;
+    }
+    s->netfns[netfn / 2] = netfnd;
+    return 0;
+}
+
+static const IPMICmdHandler *ipmi_get_handler(IPMIBmcSim *ibs,
+                                              unsigned int netfn,
+                                              unsigned int cmd)
+{
+    const IPMICmdHandler *hdl;
+
+    if (netfn & 1 || netfn >= MAX_NETFNS || !ibs->netfns[netfn / 2]) {
+        return NULL;
+    }
+
+    if (cmd >= ibs->netfns[netfn / 2]->cmd_nums) {
+        return NULL;
+    }
+
+    hdl = &ibs->netfns[netfn / 2]->cmd_handlers[cmd];
+    if (!hdl->cmd_handler) {
+        return NULL;
+    }
+
+    return hdl;
+}
+
+static void next_timeout(IPMIBmcSim *ibs)
+{
+    int64_t next;
+    if (ibs->watchdog_running) {
+        next = ibs->watchdog_expiry;
+    } else {
+        /* Wait a minute */
+        next = ipmi_getmonotime() + 60 * 1000000000LL;
+    }
+    timer_mod_ns(ibs->timer, next);
+}
+
+static void ipmi_sim_handle_command(IPMIBmc *b,
+                                    uint8_t *cmd, unsigned int cmd_len,
+                                    unsigned int max_cmd_len,
+                                    uint8_t msg_id)
+{
+    IPMIBmcSim *ibs = IPMI_BMC_SIMULATOR(b);
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    const IPMICmdHandler *hdl;
+    RspBuffer rsp = RSP_BUFFER_INITIALIZER;
+
+    /* Set up the response, set the low bit of NETFN. */
+    /* Note that max_rsp_len must be at least 3 */
+    if (sizeof(rsp.buffer) < 3) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        goto out;
+    }
+
+    rsp_buffer_push(&rsp, cmd[0] | 0x04);
+    rsp_buffer_push(&rsp, cmd[1]);
+    rsp_buffer_push(&rsp, 0); /* Assume success */
+
+    /* If it's too short or it was truncated, return an error. */
+    if (cmd_len < 2) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        goto out;
+    }
+    if (cmd_len > max_cmd_len) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        goto out;
+    }
+
+    if ((cmd[0] & 0x03) != 0) {
+        /* Only have stuff on LUN 0 */
+        rsp_buffer_set_error(&rsp, IPMI_CC_COMMAND_INVALID_FOR_LUN);
+        goto out;
+    }
+
+    hdl = ipmi_get_handler(ibs, cmd[0] >> 2, cmd[1]);
+    if (!hdl) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_INVALID_CMD);
+        goto out;
+    }
+
+    if (cmd_len < hdl->cmd_len_min) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        goto out;
+    }
+
+    hdl->cmd_handler(ibs, cmd, cmd_len, &rsp);
+
+ out:
+    k->handle_rsp(s, msg_id, rsp.buffer, rsp.len);
+
+    next_timeout(ibs);
+}
+
+static void ipmi_sim_handle_timeout(IPMIBmcSim *ibs)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    if (!ibs->watchdog_running) {
+        goto out;
+    }
+
+    if (!ibs->watchdog_preaction_ran) {
+        switch (IPMI_BMC_WATCHDOG_GET_PRE_ACTION(ibs)) {
+        case IPMI_BMC_WATCHDOG_PRE_NMI:
+            ibs->msg_flags |= IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK;
+            k->do_hw_op(s, IPMI_SEND_NMI, 0);
+            sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 8, 1,
+                                    0xc8, (2 << 4) | 0xf, 0xff);
+            break;
+
+        case IPMI_BMC_WATCHDOG_PRE_MSG_INT:
+            ibs->msg_flags |= IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK;
+            k->set_atn(s, 1, attn_irq_enabled(ibs));
+            sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 8, 1,
+                                    0xc8, (3 << 4) | 0xf, 0xff);
+            break;
+
+        default:
+            goto do_full_expiry;
+        }
+
+        ibs->watchdog_preaction_ran = 1;
+        /* Issued the pretimeout, do the rest of the timeout now. */
+        ibs->watchdog_expiry = ipmi_getmonotime();
+        ibs->watchdog_expiry += ibs->watchdog_pretimeout * 1000000000LL;
+        goto out;
+    }
+
+ do_full_expiry:
+    ibs->watchdog_running = 0; /* Stop the watchdog on a timeout */
+    ibs->watchdog_expired |= (1 << IPMI_BMC_WATCHDOG_GET_USE(ibs));
+    switch (IPMI_BMC_WATCHDOG_GET_ACTION(ibs)) {
+    case IPMI_BMC_WATCHDOG_ACTION_NONE:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 0, 1,
+                                0xc0, ibs->watchdog_use & 0xf, 0xff);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_RESET:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 1, 1,
+                                0xc1, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_RESET_CHASSIS, 0);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 2, 1,
+                                0xc2, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 2, 1,
+                                0xc3, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 0);
+        break;
+    }
+
+ out:
+    next_timeout(ibs);
+}
+
+static void chassis_capabilities(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+}
+
+static void chassis_status(IPMIBmcSim *ibs,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, 0x61); /* Unknown power restore, power is on */
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, 0);
+}
+
+static void chassis_control(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    switch (cmd[2] & 0xf) {
+    case 0: /* power down */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0));
+        break;
+    case 1: /* power up */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERON_CHASSIS, 0));
+        break;
+    case 2: /* power cycle */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 0));
+        break;
+    case 3: /* hard reset */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_RESET_CHASSIS, 0));
+        break;
+    case 4: /* pulse diagnostic interrupt */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_PULSE_DIAG_IRQ, 0));
+        break;
+    case 5: /* soft shutdown via ACPI by overtemp emulation */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s,
+                                          IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP, 0));
+        break;
+    default:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void chassis_get_sys_restart_cause(IPMIBmcSim *ibs,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           RspBuffer *rsp)
+
+{
+    rsp_buffer_push(rsp, ibs->restart_cause & 0xf); /* Restart Cause */
+    rsp_buffer_push(rsp, 0);  /* Channel 0 */
+}
+
+static void get_device_id(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->device_id);
+    rsp_buffer_push(rsp, ibs->device_rev & 0xf);
+    rsp_buffer_push(rsp, ibs->fwrev1 & 0x7f);
+    rsp_buffer_push(rsp, ibs->fwrev2);
+    rsp_buffer_push(rsp, ibs->ipmi_version);
+    rsp_buffer_push(rsp, 0x07); /* sensor, SDR, and SEL. */
+    rsp_buffer_push(rsp, ibs->mfg_id[0]);
+    rsp_buffer_push(rsp, ibs->mfg_id[1]);
+    rsp_buffer_push(rsp, ibs->mfg_id[2]);
+    rsp_buffer_push(rsp, ibs->product_id[0]);
+    rsp_buffer_push(rsp, ibs->product_id[1]);
+}
+
+static void set_global_enables(IPMIBmcSim *ibs, uint8_t val)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    bool irqs_on;
+
+    ibs->bmc_global_enables = val;
+
+    irqs_on = val & (IPMI_BMC_EVBUF_FULL_INT_BIT |
+                     IPMI_BMC_RCV_MSG_QUEUE_INT_BIT);
+
+    k->set_irq_enable(s, irqs_on);
+}
+
+static void cold_reset(IPMIBmcSim *ibs,
+                       uint8_t *cmd, unsigned int cmd_len,
+                       RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    /* Disable all interrupts */
+    set_global_enables(ibs, 1 << IPMI_BMC_EVENT_LOG_BIT);
+
+    if (k->reset) {
+        k->reset(s, true);
+    }
+}
+
+static void warm_reset(IPMIBmcSim *ibs,
+                       uint8_t *cmd, unsigned int cmd_len,
+                       RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    if (k->reset) {
+        k->reset(s, false);
+    }
+}
+static void set_acpi_power_state(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    ibs->acpi_power_state[0] = cmd[2];
+    ibs->acpi_power_state[1] = cmd[3];
+}
+
+static void get_acpi_power_state(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->acpi_power_state[0]);
+    rsp_buffer_push(rsp, ibs->acpi_power_state[1]);
+}
+
+static void get_device_guid(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    unsigned int i;
+
+    for (i = 0; i < 16; i++) {
+        rsp_buffer_push(rsp, ibs->uuid[i]);
+    }
+}
+
+static void set_bmc_global_enables(IPMIBmcSim *ibs,
+                                   uint8_t *cmd, unsigned int cmd_len,
+                                   RspBuffer *rsp)
+{
+    set_global_enables(ibs, cmd[2]);
+}
+
+static void get_bmc_global_enables(IPMIBmcSim *ibs,
+                                   uint8_t *cmd, unsigned int cmd_len,
+                                   RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->bmc_global_enables);
+}
+
+static void clr_msg_flags(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    ibs->msg_flags &= ~cmd[2];
+    k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+}
+
+static void get_msg_flags(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->msg_flags);
+}
+
+static void read_evt_msg_buf(IPMIBmcSim *ibs,
+                             uint8_t *cmd, unsigned int cmd_len,
+                             RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned int i;
+
+    if (!(ibs->msg_flags & IPMI_BMC_MSG_FLAG_EVT_BUF_FULL)) {
+        rsp_buffer_set_error(rsp, 0x80);
+        return;
+    }
+    for (i = 0; i < 16; i++) {
+        rsp_buffer_push(rsp, ibs->evtbuf[i]);
+    }
+    ibs->msg_flags &= ~IPMI_BMC_MSG_FLAG_EVT_BUF_FULL;
+    k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+}
+
+static void get_msg(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    IPMIRcvBufEntry *msg;
+
+    qemu_mutex_lock(&ibs->lock);
+    if (QTAILQ_EMPTY(&ibs->rcvbufs)) {
+        rsp_buffer_set_error(rsp, 0x80); /* Queue empty */
+        goto out;
+    }
+    rsp_buffer_push(rsp, 0); /* Channel 0 */
+    msg = QTAILQ_FIRST(&ibs->rcvbufs);
+    rsp_buffer_pushmore(rsp, msg->buf, msg->len);
+    QTAILQ_REMOVE(&ibs->rcvbufs, msg, entry);
+    g_free(msg);
+
+    if (QTAILQ_EMPTY(&ibs->rcvbufs)) {
+        IPMIInterface *s = ibs->parent.intf;
+        IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+        ibs->msg_flags &= ~IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE;
+        k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+    }
+
+out:
+    qemu_mutex_unlock(&ibs->lock);
+    return;
+}
+
+static unsigned char
+ipmb_checksum(unsigned char *data, int size, unsigned char csum)
+{
+    for (; size > 0; size--, data++) {
+            csum += *data;
+    }
+
+    return -csum;
+}
+
+static void send_msg(IPMIBmcSim *ibs,
+                     uint8_t *cmd, unsigned int cmd_len,
+                     RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    IPMIRcvBufEntry *msg;
+    uint8_t *buf;
+    uint8_t netfn, rqLun, rsLun, rqSeq;
+
+    if (cmd[2] != 0) {
+        /* We only handle channel 0 with no options */
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    if (cmd_len < 10) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        return;
+    }
+
+    if (cmd[3] != 0x40) {
+        /* We only emulate a MC at address 0x40. */
+        rsp_buffer_set_error(rsp, 0x83); /* NAK on write */
+        return;
+    }
+
+    cmd += 3; /* Skip the header. */
+    cmd_len -= 3;
+
+    /*
+     * At this point we "send" the message successfully.  Any error will
+     * be returned in the response.
+     */
+    if (ipmb_checksum(cmd, cmd_len, 0) != 0 ||
+        cmd[3] != 0x20) { /* Improper response address */
+        return; /* No response */
+    }
+
+    netfn = cmd[1] >> 2;
+    rqLun = cmd[4] & 0x3;
+    rsLun = cmd[1] & 0x3;
+    rqSeq = cmd[4] >> 2;
+
+    if (rqLun != 2) {
+        /* We only support LUN 2 coming back to us. */
+        return;
+    }
+
+    msg = g_malloc(sizeof(*msg));
+    msg->buf[0] = ((netfn | 1) << 2) | rqLun; /* NetFN, and make a response */
+    msg->buf[1] = ipmb_checksum(msg->buf, 1, 0);
+    msg->buf[2] = cmd[0]; /* rsSA */
+    msg->buf[3] = (rqSeq << 2) | rsLun;
+    msg->buf[4] = cmd[5]; /* Cmd */
+    msg->buf[5] = 0; /* Completion Code */
+    msg->len = 6;
+
+    if ((cmd[1] >> 2) != IPMI_NETFN_APP || cmd[5] != IPMI_CMD_GET_DEVICE_ID) {
+        /* Not a command we handle. */
+        msg->buf[5] = IPMI_CC_INVALID_CMD;
+        goto end_msg;
+    }
+
+    buf = msg->buf + msg->len; /* After the CC */
+    buf[0] = 0;
+    buf[1] = 0;
+    buf[2] = 0;
+    buf[3] = 0;
+    buf[4] = 0x51;
+    buf[5] = 0;
+    buf[6] = 0;
+    buf[7] = 0;
+    buf[8] = 0;
+    buf[9] = 0;
+    buf[10] = 0;
+    msg->len += 11;
+
+ end_msg:
+    msg->buf[msg->len] = ipmb_checksum(msg->buf, msg->len, 0);
+    msg->len++;
+    qemu_mutex_lock(&ibs->lock);
+    QTAILQ_INSERT_TAIL(&ibs->rcvbufs, msg, entry);
+    ibs->msg_flags |= IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE;
+    k->set_atn(s, 1, attn_irq_enabled(ibs));
+    qemu_mutex_unlock(&ibs->lock);
+}
+
+static void do_watchdog_reset(IPMIBmcSim *ibs)
+{
+    if (IPMI_BMC_WATCHDOG_GET_ACTION(ibs) ==
+        IPMI_BMC_WATCHDOG_ACTION_NONE) {
+        ibs->watchdog_running = 0;
+        return;
+    }
+    ibs->watchdog_preaction_ran = 0;
+
+
+    /* Timeout is in tenths of a second, offset is in seconds */
+    ibs->watchdog_expiry = ipmi_getmonotime();
+    ibs->watchdog_expiry += ibs->watchdog_timeout * 100000000LL;
+    if (IPMI_BMC_WATCHDOG_GET_PRE_ACTION(ibs) != IPMI_BMC_WATCHDOG_PRE_NONE) {
+        ibs->watchdog_expiry -= ibs->watchdog_pretimeout * 1000000000LL;
+    }
+    ibs->watchdog_running = 1;
+}
+
+static void reset_watchdog_timer(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    if (!ibs->watchdog_initialized) {
+        rsp_buffer_set_error(rsp, 0x80);
+        return;
+    }
+    do_watchdog_reset(ibs);
+}
+
+static void set_watchdog_timer(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned int val;
+
+    val = cmd[2] & 0x7; /* Validate use */
+    if (val == 0 || val > 5) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    val = cmd[3] & 0x7; /* Validate action */
+    switch (val) {
+    case IPMI_BMC_WATCHDOG_ACTION_NONE:
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_RESET:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_RESET_CHASSIS, 1));
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 1));
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 1));
+        break;
+
+    default:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+    }
+    if (rsp->buffer[2]) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    val = (cmd[3] >> 4) & 0x7; /* Validate preaction */
+    switch (val) {
+    case IPMI_BMC_WATCHDOG_PRE_MSG_INT:
+    case IPMI_BMC_WATCHDOG_PRE_NONE:
+        break;
+
+    case IPMI_BMC_WATCHDOG_PRE_NMI:
+        if (!k->do_hw_op(s, IPMI_SEND_NMI, 1)) {
+            /* NMI not supported. */
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+            return;
+        }
+        break;
+
+    default:
+        /* We don't support PRE_SMI */
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    ibs->watchdog_initialized = 1;
+    ibs->watchdog_use = cmd[2] & IPMI_BMC_WATCHDOG_USE_MASK;
+    ibs->watchdog_action = cmd[3] & IPMI_BMC_WATCHDOG_ACTION_MASK;
+    ibs->watchdog_pretimeout = cmd[4];
+    ibs->watchdog_expired &= ~cmd[5];
+    ibs->watchdog_timeout = cmd[6] | (((uint16_t) cmd[7]) << 8);
+    if (ibs->watchdog_running & IPMI_BMC_WATCHDOG_GET_DONT_STOP(ibs)) {
+        do_watchdog_reset(ibs);
+    } else {
+        ibs->watchdog_running = 0;
+    }
+}
+
+static void get_watchdog_timer(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->watchdog_use);
+    rsp_buffer_push(rsp, ibs->watchdog_action);
+    rsp_buffer_push(rsp, ibs->watchdog_pretimeout);
+    rsp_buffer_push(rsp, ibs->watchdog_expired);
+    if (ibs->watchdog_running) {
+        long timeout;
+        timeout = ((ibs->watchdog_expiry - ipmi_getmonotime() + 50000000)
+                   / 100000000);
+        rsp_buffer_push(rsp, timeout & 0xff);
+        rsp_buffer_push(rsp, (timeout >> 8) & 0xff);
+    } else {
+        rsp_buffer_push(rsp, 0);
+        rsp_buffer_push(rsp, 0);
+    }
+}
+
+static void get_sdr_rep_info(IPMIBmcSim *ibs,
+                             uint8_t *cmd, unsigned int cmd_len,
+                             RspBuffer *rsp)
+{
+    unsigned int i;
+
+    rsp_buffer_push(rsp, 0x51); /* Conform to IPMI 1.5 spec */
+    rsp_buffer_push(rsp, ibs->sdr.next_rec_id & 0xff);
+    rsp_buffer_push(rsp, (ibs->sdr.next_rec_id >> 8) & 0xff);
+    rsp_buffer_push(rsp, (MAX_SDR_SIZE - ibs->sdr.next_free) & 0xff);
+    rsp_buffer_push(rsp, ((MAX_SDR_SIZE - ibs->sdr.next_free) >> 8) & 0xff);
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sdr.last_addition[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sdr.last_clear[i]);
+    }
+    /* Only modal support, reserve supported */
+    rsp_buffer_push(rsp, (ibs->sdr.overflow << 7) | 0x22);
+}
+
+static void reserve_sdr_rep(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->sdr.reservation & 0xff);
+    rsp_buffer_push(rsp, (ibs->sdr.reservation >> 8) & 0xff);
+}
+
+static void get_sdr(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    unsigned int pos;
+    uint16_t nextrec;
+    struct ipmi_sdr_header *sdrh;
+
+    if (cmd[6]) {
+        if ((cmd[2] | (cmd[3] << 8)) != ibs->sdr.reservation) {
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+            return;
+        }
+    }
+
+    pos = 0;
+    if (sdr_find_entry(&ibs->sdr, cmd[4] | (cmd[5] << 8),
+                       &pos, &nextrec)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+
+    sdrh = (struct ipmi_sdr_header *) &ibs->sdr.sdr[pos];
+
+    if (cmd[6] > ipmi_sdr_length(sdrh)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_PARM_OUT_OF_RANGE);
+        return;
+    }
+
+    rsp_buffer_push(rsp, nextrec & 0xff);
+    rsp_buffer_push(rsp, (nextrec >> 8) & 0xff);
+
+    if (cmd[7] == 0xff) {
+        cmd[7] = ipmi_sdr_length(sdrh) - cmd[6];
+    }
+
+    if ((cmd[7] + rsp->len) > sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES);
+        return;
+    }
+
+    rsp_buffer_pushmore(rsp, ibs->sdr.sdr + pos + cmd[6], cmd[7]);
+}
+
+static void add_sdr(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    uint16_t recid;
+    struct ipmi_sdr_header *sdrh = (struct ipmi_sdr_header *) cmd + 2;
+
+    if (sdr_add_entry(ibs, sdrh, cmd_len - 2, &recid)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    rsp_buffer_push(rsp, recid & 0xff);
+    rsp_buffer_push(rsp, (recid >> 8) & 0xff);
+}
+
+static void clear_sdr_rep(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    if ((cmd[2] | (cmd[3] << 8)) != ibs->sdr.reservation) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+        return;
+    }
+
+    if (cmd[4] != 'C' || cmd[5] != 'L' || cmd[6] != 'R') {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xaa) {
+        ibs->sdr.next_free = 0;
+        ibs->sdr.overflow = 0;
+        set_timestamp(ibs, ibs->sdr.last_clear);
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+        sdr_inc_reservation(&ibs->sdr);
+    } else if (cmd[7] == 0) {
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+    } else {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void get_sel_info(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    unsigned int i, val;
+
+    rsp_buffer_push(rsp, 0x51); /* Conform to IPMI 1.5 */
+    rsp_buffer_push(rsp, ibs->sel.next_free & 0xff);
+    rsp_buffer_push(rsp, (ibs->sel.next_free >> 8) & 0xff);
+    val = (MAX_SEL_SIZE - ibs->sel.next_free) * 16;
+    rsp_buffer_push(rsp, val & 0xff);
+    rsp_buffer_push(rsp, (val >> 8) & 0xff);
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sel.last_addition[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sel.last_clear[i]);
+    }
+    /* Only support Reserve SEL */
+    rsp_buffer_push(rsp, (ibs->sel.overflow << 7) | 0x02);
+}
+
+static void reserve_sel(IPMIBmcSim *ibs,
+                        uint8_t *cmd, unsigned int cmd_len,
+                        RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->sel.reservation & 0xff);
+    rsp_buffer_push(rsp, (ibs->sel.reservation >> 8) & 0xff);
+}
+
+static void get_sel_entry(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    unsigned int val;
+
+    if (cmd[6]) {
+        if ((cmd[2] | (cmd[3] << 8)) != ibs->sel.reservation) {
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+            return;
+        }
+    }
+    if (ibs->sel.next_free == 0) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    if (cmd[6] > 15) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xff) {
+        cmd[7] = 16;
+    } else if ((cmd[7] + cmd[6]) > 16) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    } else {
+        cmd[7] += cmd[6];
+    }
+
+    val = cmd[4] | (cmd[5] << 8);
+    if (val == 0xffff) {
+        val = ibs->sel.next_free - 1;
+    } else if (val >= ibs->sel.next_free) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    if ((val + 1) == ibs->sel.next_free) {
+        rsp_buffer_push(rsp, 0xff);
+        rsp_buffer_push(rsp, 0xff);
+    } else {
+        rsp_buffer_push(rsp, (val + 1) & 0xff);
+        rsp_buffer_push(rsp, ((val + 1) >> 8) & 0xff);
+    }
+    for (; cmd[6] < cmd[7]; cmd[6]++) {
+        rsp_buffer_push(rsp, ibs->sel.sel[val][cmd[6]]);
+    }
+}
+
+static void add_sel_entry(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    if (sel_add_event(ibs, cmd + 2)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_OUT_OF_SPACE);
+        return;
+    }
+    /* sel_add_event fills in the record number. */
+    rsp_buffer_push(rsp, cmd[2]);
+    rsp_buffer_push(rsp, cmd[3]);
+}
+
+static void clear_sel(IPMIBmcSim *ibs,
+                      uint8_t *cmd, unsigned int cmd_len,
+                      RspBuffer *rsp)
+{
+    if ((cmd[2] | (cmd[3] << 8)) != ibs->sel.reservation) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+        return;
+    }
+
+    if (cmd[4] != 'C' || cmd[5] != 'L' || cmd[6] != 'R') {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xaa) {
+        ibs->sel.next_free = 0;
+        ibs->sel.overflow = 0;
+        set_timestamp(ibs, ibs->sdr.last_clear);
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+        sel_inc_reservation(&ibs->sel);
+    } else if (cmd[7] == 0) {
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+    } else {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void get_sel_time(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    uint32_t val;
+    struct ipmi_time now;
+
+    ipmi_gettime(&now);
+    val = now.tv_sec + ibs->sel.time_offset;
+    rsp_buffer_push(rsp, val & 0xff);
+    rsp_buffer_push(rsp, (val >> 8) & 0xff);
+    rsp_buffer_push(rsp, (val >> 16) & 0xff);
+    rsp_buffer_push(rsp, (val >> 24) & 0xff);
+}
+
+static void set_sel_time(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    uint32_t val;
+    struct ipmi_time now;
+
+    val = cmd[2] | (cmd[3] << 8) | (cmd[4] << 16) | (cmd[5] << 24);
+    ipmi_gettime(&now);
+    ibs->sel.time_offset = now.tv_sec - ((long) val);
+}
+
+static void set_sensor_evt_enable(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    switch ((cmd[3] >> 4) & 0x3) {
+    case 0: /* Do not change */
+        break;
+    case 1: /* Enable bits */
+        if (cmd_len > 4) {
+            sens->assert_enable |= cmd[4];
+        }
+        if (cmd_len > 5) {
+            sens->assert_enable |= cmd[5] << 8;
+        }
+        if (cmd_len > 6) {
+            sens->deassert_enable |= cmd[6];
+        }
+        if (cmd_len > 7) {
+            sens->deassert_enable |= cmd[7] << 8;
+        }
+        break;
+    case 2: /* Disable bits */
+        if (cmd_len > 4) {
+            sens->assert_enable &= ~cmd[4];
+        }
+        if (cmd_len > 5) {
+            sens->assert_enable &= ~(cmd[5] << 8);
+        }
+        if (cmd_len > 6) {
+            sens->deassert_enable &= ~cmd[6];
+        }
+        if (cmd_len > 7) {
+            sens->deassert_enable &= ~(cmd[7] << 8);
+        }
+        break;
+    case 3:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    IPMI_SENSOR_SET_RET_STATUS(sens, cmd[3]);
+}
+
+static void get_sensor_evt_enable(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->assert_enable & 0xff);
+    rsp_buffer_push(rsp, (sens->assert_enable >> 8) & 0xff);
+    rsp_buffer_push(rsp, sens->deassert_enable & 0xff);
+    rsp_buffer_push(rsp, (sens->deassert_enable >> 8) & 0xff);
+}
+
+static void rearm_sensor_evts(IPMIBmcSim *ibs,
+                              uint8_t *cmd, unsigned int cmd_len,
+                              RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+
+    if ((cmd[3] & 0x80) == 0) {
+        /* Just clear everything */
+        sens->states = 0;
+        return;
+    }
+}
+
+static void get_sensor_evt_status(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->reading);
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->assert_states & 0xff);
+    rsp_buffer_push(rsp, (sens->assert_states >> 8) & 0xff);
+    rsp_buffer_push(rsp, sens->deassert_states & 0xff);
+    rsp_buffer_push(rsp, (sens->deassert_states >> 8) & 0xff);
+}
+
+static void get_sensor_reading(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->reading);
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->states & 0xff);
+    if (IPMI_SENSOR_IS_DISCRETE(sens)) {
+        rsp_buffer_push(rsp, (sens->states >> 8) & 0xff);
+    }
+}
+
+static void set_sensor_type(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    sens->sensor_type = cmd[3];
+    sens->evt_reading_type_code = cmd[4] & 0x7f;
+}
+
+static void get_sensor_type(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->sensor_type);
+    rsp_buffer_push(rsp, sens->evt_reading_type_code);
+}
+
+
+static const IPMICmdHandler chassis_cmds[] = {
+    [IPMI_CMD_GET_CHASSIS_CAPABILITIES] = { chassis_capabilities },
+    [IPMI_CMD_GET_CHASSIS_STATUS] = { chassis_status },
+    [IPMI_CMD_CHASSIS_CONTROL] = { chassis_control, 3 },
+    [IPMI_CMD_GET_SYS_RESTART_CAUSE] = { chassis_get_sys_restart_cause }
+};
+static const IPMINetfn chassis_netfn = {
+    .cmd_nums = ARRAY_SIZE(chassis_cmds),
+    .cmd_handlers = chassis_cmds
+};
+
+static const IPMICmdHandler sensor_event_cmds[] = {
+    [IPMI_CMD_SET_SENSOR_EVT_ENABLE] = { set_sensor_evt_enable, 4 },
+    [IPMI_CMD_GET_SENSOR_EVT_ENABLE] = { get_sensor_evt_enable, 3 },
+    [IPMI_CMD_REARM_SENSOR_EVTS] = { rearm_sensor_evts, 4 },
+    [IPMI_CMD_GET_SENSOR_EVT_STATUS] = { get_sensor_evt_status, 3 },
+    [IPMI_CMD_GET_SENSOR_READING] = { get_sensor_reading, 3 },
+    [IPMI_CMD_SET_SENSOR_TYPE] = { set_sensor_type, 5 },
+    [IPMI_CMD_GET_SENSOR_TYPE] = { get_sensor_type, 3 },
+};
+static const IPMINetfn sensor_event_netfn = {
+    .cmd_nums = ARRAY_SIZE(sensor_event_cmds),
+    .cmd_handlers = sensor_event_cmds
+};
+
+static const IPMICmdHandler app_cmds[] = {
+    [IPMI_CMD_GET_DEVICE_ID] = { get_device_id },
+    [IPMI_CMD_COLD_RESET] = { cold_reset },
+    [IPMI_CMD_WARM_RESET] = { warm_reset },
+    [IPMI_CMD_SET_ACPI_POWER_STATE] = { set_acpi_power_state, 4 },
+    [IPMI_CMD_GET_ACPI_POWER_STATE] = { get_acpi_power_state },
+    [IPMI_CMD_GET_DEVICE_GUID] = { get_device_guid },
+    [IPMI_CMD_SET_BMC_GLOBAL_ENABLES] = { set_bmc_global_enables, 3 },
+    [IPMI_CMD_GET_BMC_GLOBAL_ENABLES] = { get_bmc_global_enables },
+    [IPMI_CMD_CLR_MSG_FLAGS] = { clr_msg_flags, 3 },
+    [IPMI_CMD_GET_MSG_FLAGS] = { get_msg_flags },
+    [IPMI_CMD_GET_MSG] = { get_msg },
+    [IPMI_CMD_SEND_MSG] = { send_msg, 3 },
+    [IPMI_CMD_READ_EVT_MSG_BUF] = { read_evt_msg_buf },
+    [IPMI_CMD_RESET_WATCHDOG_TIMER] = { reset_watchdog_timer },
+    [IPMI_CMD_SET_WATCHDOG_TIMER] = { set_watchdog_timer, 8 },
+    [IPMI_CMD_GET_WATCHDOG_TIMER] = { get_watchdog_timer },
+};
+static const IPMINetfn app_netfn = {
+    .cmd_nums = ARRAY_SIZE(app_cmds),
+    .cmd_handlers = app_cmds
+};
+
+static const IPMICmdHandler storage_cmds[] = {
+    [IPMI_CMD_GET_SDR_REP_INFO] = { get_sdr_rep_info },
+    [IPMI_CMD_RESERVE_SDR_REP] = { reserve_sdr_rep },
+    [IPMI_CMD_GET_SDR] = { get_sdr, 8 },
+    [IPMI_CMD_ADD_SDR] = { add_sdr },
+    [IPMI_CMD_CLEAR_SDR_REP] = { clear_sdr_rep, 8 },
+    [IPMI_CMD_GET_SEL_INFO] = { get_sel_info },
+    [IPMI_CMD_RESERVE_SEL] = { reserve_sel },
+    [IPMI_CMD_GET_SEL_ENTRY] = { get_sel_entry, 8 },
+    [IPMI_CMD_ADD_SEL_ENTRY] = { add_sel_entry, 18 },
+    [IPMI_CMD_CLEAR_SEL] = { clear_sel, 8 },
+    [IPMI_CMD_GET_SEL_TIME] = { get_sel_time, 6 },
+    [IPMI_CMD_SET_SEL_TIME] = { set_sel_time },
+};
+
+static const IPMINetfn storage_netfn = {
+    .cmd_nums = ARRAY_SIZE(storage_cmds),
+    .cmd_handlers = storage_cmds
+};
+
+static void register_cmds(IPMIBmcSim *s)
+{
+    ipmi_register_netfn(s, IPMI_NETFN_CHASSIS, &chassis_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_SENSOR_EVENT, &sensor_event_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_APP, &app_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_STORAGE, &storage_netfn);
+}
+
+static uint8_t init_sdrs[] = {
+    /* Watchdog device */
+    0x00, 0x00, 0x51, 0x02,   35, 0x20, 0x00, 0x00,
+    0x23, 0x01, 0x63, 0x00, 0x23, 0x6f, 0x0f, 0x01,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8,
+    'W',  'a',  't',  'c',  'h',  'd',  'o',  'g',
+};
+
+static void ipmi_sdr_init(IPMIBmcSim *ibs)
+{
+    unsigned int i;
+    int len;
+    size_t sdrs_size;
+    uint8_t *sdrs;
+
+    sdrs_size = sizeof(init_sdrs);
+    sdrs = init_sdrs;
+
+    for (i = 0; i < sdrs_size; i += len) {
+        struct ipmi_sdr_header *sdrh;
+
+        if (i + IPMI_SDR_HEADER_SIZE > sdrs_size) {
+            error_report("Problem with recid 0x%4.4x", i);
+            return;
+        }
+        sdrh = (struct ipmi_sdr_header *) &sdrs[i];
+        len = ipmi_sdr_length(sdrh);
+        if (i + len > sdrs_size) {
+            error_report("Problem with recid 0x%4.4x", i);
+            return;
+        }
+        sdr_add_entry(ibs, sdrh, len, NULL);
+    }
+}
+
+static const VMStateDescription vmstate_ipmi_sim = {
+    .name = TYPE_IPMI_BMC_SIMULATOR,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT8(bmc_global_enables, IPMIBmcSim),
+        VMSTATE_UINT8(msg_flags, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_initialized, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_use, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_action, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_pretimeout, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_expired, IPMIBmcSim),
+        VMSTATE_UINT16(watchdog_timeout, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_running, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_preaction_ran, IPMIBmcSim),
+        VMSTATE_INT64(watchdog_expiry, IPMIBmcSim),
+        VMSTATE_UINT8_ARRAY(evtbuf, IPMIBmcSim, 16),
+        VMSTATE_UINT8(sensors[IPMI_WATCHDOG_SENSOR].status, IPMIBmcSim),
+        VMSTATE_UINT8(sensors[IPMI_WATCHDOG_SENSOR].reading, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].states, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].assert_states, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].deassert_states,
+                       IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].assert_enable, IPMIBmcSim),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ipmi_sim_realize(DeviceState *dev, Error **errp)
+{
+    IPMIBmc *b = IPMI_BMC(dev);
+    unsigned int i;
+    IPMIBmcSim *ibs = IPMI_BMC_SIMULATOR(b);
+
+    qemu_mutex_init(&ibs->lock);
+    QTAILQ_INIT(&ibs->rcvbufs);
+
+    ibs->bmc_global_enables = (1 << IPMI_BMC_EVENT_LOG_BIT);
+    ibs->device_id = 0x20;
+    ibs->ipmi_version = 0x02; /* IPMI 2.0 */
+    ibs->restart_cause = 0;
+    for (i = 0; i < 4; i++) {
+        ibs->sel.last_addition[i] = 0xff;
+        ibs->sel.last_clear[i] = 0xff;
+        ibs->sdr.last_addition[i] = 0xff;
+        ibs->sdr.last_clear[i] = 0xff;
+    }
+
+    ipmi_sdr_init(ibs);
+
+    ibs->acpi_power_state[0] = 0;
+    ibs->acpi_power_state[1] = 0;
+
+    if (qemu_uuid_set) {
+        memcpy(&ibs->uuid, qemu_uuid, 16);
+    } else {
+        memset(&ibs->uuid, 0, 16);
+    }
+
+    ipmi_init_sensors_from_sdrs(ibs);
+    register_cmds(ibs);
+
+    ibs->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, ipmi_timeout, ibs);
+
+    vmstate_register(NULL, 0, &vmstate_ipmi_sim, ibs);
+}
+
+static void ipmi_sim_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIBmcClass *bk = IPMI_BMC_CLASS(oc);
+
+    dc->realize = ipmi_sim_realize;
+    bk->handle_command = ipmi_sim_handle_command;
+}
+
+static const TypeInfo ipmi_sim_type = {
+    .name          = TYPE_IPMI_BMC_SIMULATOR,
+    .parent        = TYPE_IPMI_BMC,
+    .instance_size = sizeof(IPMIBmcSim),
+    .class_init    = ipmi_sim_class_init,
+};
+
+static void ipmi_sim_register_types(void)
+{
+    type_register_static(&ipmi_sim_type);
+}
+
+type_init(ipmi_sim_register_types)
diff --git a/qemu/hw/ipmi/isa_ipmi_bt.c b/qemu/hw/ipmi/isa_ipmi_bt.c
new file mode 100644
index 000000000..aaea12ecd
--- /dev/null
+++ b/qemu/hw/ipmi/isa_ipmi_bt.c
@@ -0,0 +1,530 @@
+/*
+ * QEMU ISA IPMI BT emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/isa/isa.h"
+#include "hw/i386/pc.h"
+
+/* Control register */
+#define IPMI_BT_CLR_WR_BIT         0
+#define IPMI_BT_CLR_RD_BIT         1
+#define IPMI_BT_H2B_ATN_BIT        2
+#define IPMI_BT_B2H_ATN_BIT        3
+#define IPMI_BT_SMS_ATN_BIT        4
+#define IPMI_BT_HBUSY_BIT          6
+#define IPMI_BT_BBUSY_BIT          7
+
+#define IPMI_BT_CLR_WR_MASK        (1 << IPMI_BT_CLR_WR_BIT)
+#define IPMI_BT_GET_CLR_WR(d)      (((d) >> IPMI_BT_CLR_WR_BIT) & 0x1)
+#define IPMI_BT_SET_CLR_WR(d, v)   (d) = (((d) & ~IPMI_BT_CLR_WR_MASK) | \
+                                       (((v & 1) << IPMI_BT_CLR_WR_BIT)))
+
+#define IPMI_BT_CLR_RD_MASK        (1 << IPMI_BT_CLR_RD_BIT)
+#define IPMI_BT_GET_CLR_RD(d)      (((d) >> IPMI_BT_CLR_RD_BIT) & 0x1)
+#define IPMI_BT_SET_CLR_RD(d, v)   (d) = (((d) & ~IPMI_BT_CLR_RD_MASK) | \
+                                       (((v & 1) << IPMI_BT_CLR_RD_BIT)))
+
+#define IPMI_BT_H2B_ATN_MASK       (1 << IPMI_BT_H2B_ATN_BIT)
+#define IPMI_BT_GET_H2B_ATN(d)     (((d) >> IPMI_BT_H2B_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_H2B_ATN(d, v)  (d) = (((d) & ~IPMI_BT_H2B_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_H2B_ATN_BIT)))
+
+#define IPMI_BT_B2H_ATN_MASK       (1 << IPMI_BT_B2H_ATN_BIT)
+#define IPMI_BT_GET_B2H_ATN(d)     (((d) >> IPMI_BT_B2H_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_ATN(d, v)  (d) = (((d) & ~IPMI_BT_B2H_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_ATN_BIT)))
+
+#define IPMI_BT_SMS_ATN_MASK       (1 << IPMI_BT_SMS_ATN_BIT)
+#define IPMI_BT_GET_SMS_ATN(d)     (((d) >> IPMI_BT_SMS_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_SMS_ATN(d, v)  (d) = (((d) & ~IPMI_BT_SMS_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_SMS_ATN_BIT)))
+
+#define IPMI_BT_HBUSY_MASK         (1 << IPMI_BT_HBUSY_BIT)
+#define IPMI_BT_GET_HBUSY(d)       (((d) >> IPMI_BT_HBUSY_BIT) & 0x1)
+#define IPMI_BT_SET_HBUSY(d, v)    (d) = (((d) & ~IPMI_BT_HBUSY_MASK) | \
+                                       (((v & 1) << IPMI_BT_HBUSY_BIT)))
+
+#define IPMI_BT_BBUSY_MASK         (1 << IPMI_BT_BBUSY_BIT)
+#define IPMI_BT_GET_BBUSY(d)       (((d) >> IPMI_BT_BBUSY_BIT) & 0x1)
+#define IPMI_BT_SET_BBUSY(d, v)    (d) = (((d) & ~IPMI_BT_BBUSY_MASK) | \
+                                       (((v & 1) << IPMI_BT_BBUSY_BIT)))
+
+
+/* Mask register */
+#define IPMI_BT_B2H_IRQ_EN_BIT     0
+#define IPMI_BT_B2H_IRQ_BIT        1
+
+#define IPMI_BT_B2H_IRQ_EN_MASK      (1 << IPMI_BT_B2H_IRQ_EN_BIT)
+#define IPMI_BT_GET_B2H_IRQ_EN(d)    (((d) >> IPMI_BT_B2H_IRQ_EN_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_IRQ_EN(d, v) (d) = (((d) & ~IPMI_BT_B2H_IRQ_EN_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_IRQ_EN_BIT)))
+
+#define IPMI_BT_B2H_IRQ_MASK         (1 << IPMI_BT_B2H_IRQ_BIT)
+#define IPMI_BT_GET_B2H_IRQ(d)       (((d) >> IPMI_BT_B2H_IRQ_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_IRQ(d, v)    (d) = (((d) & ~IPMI_BT_B2H_IRQ_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_IRQ_BIT)))
+
+typedef struct IPMIBT {
+    IPMIBmc *bmc;
+
+    bool do_wake;
+
+    qemu_irq irq;
+
+    uint32_t io_base;
+    unsigned long io_length;
+    MemoryRegion io;
+
+    bool obf_irq_set;
+    bool atn_irq_set;
+    bool use_irq;
+    bool irqs_enabled;
+
+    uint8_t outmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t outpos;
+    uint32_t outlen;
+
+    uint8_t inmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t inlen;
+
+    uint8_t control_reg;
+    uint8_t mask_reg;
+
+    /*
+     * This is a response number that we send with the command to make
+     * sure that the response matches the command.
+     */
+    uint8_t waiting_rsp;
+    uint8_t waiting_seq;
+} IPMIBT;
+
+#define IPMI_CMD_GET_BT_INTF_CAP        0x36
+
+static void ipmi_bt_handle_event(IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (ib->inlen < 4) {
+        goto out;
+    }
+    /* Note that overruns are handled by handle_command */
+    if (ib->inmsg[0] != (ib->inlen - 1)) {
+        /* Length mismatch, just ignore. */
+        IPMI_BT_SET_BBUSY(ib->control_reg, 1);
+        ib->inlen = 0;
+        goto out;
+    }
+    if ((ib->inmsg[1] == (IPMI_NETFN_APP << 2)) &&
+                        (ib->inmsg[3] == IPMI_CMD_GET_BT_INTF_CAP)) {
+        /* We handle this one ourselves. */
+        ib->outmsg[0] = 9;
+        ib->outmsg[1] = ib->inmsg[1] | 0x04;
+        ib->outmsg[2] = ib->inmsg[2];
+        ib->outmsg[3] = ib->inmsg[3];
+        ib->outmsg[4] = 0;
+        ib->outmsg[5] = 1; /* Only support 1 outstanding request. */
+        if (sizeof(ib->inmsg) > 0xff) { /* Input buffer size */
+            ib->outmsg[6] = 0xff;
+        } else {
+            ib->outmsg[6] = (unsigned char) sizeof(ib->inmsg);
+        }
+        if (sizeof(ib->outmsg) > 0xff) { /* Output buffer size */
+            ib->outmsg[7] = 0xff;
+        } else {
+            ib->outmsg[7] = (unsigned char) sizeof(ib->outmsg);
+        }
+        ib->outmsg[8] = 10; /* Max request to response time */
+        ib->outmsg[9] = 0; /* Don't recommend retries */
+        ib->outlen = 10;
+        IPMI_BT_SET_BBUSY(ib->control_reg, 0);
+        IPMI_BT_SET_B2H_ATN(ib->control_reg, 1);
+        if (ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_IRQ(ib->mask_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+        goto out;
+    }
+    ib->waiting_seq = ib->inmsg[2];
+    ib->inmsg[2] = ib->inmsg[1];
+    {
+        IPMIBmcClass *bk = IPMI_BMC_GET_CLASS(ib->bmc);
+        bk->handle_command(ib->bmc, ib->inmsg + 2, ib->inlen - 2,
+                           sizeof(ib->inmsg), ib->waiting_rsp);
+    }
+ out:
+    return;
+}
+
+static void ipmi_bt_handle_rsp(IPMIInterface *ii, uint8_t msg_id,
+                                unsigned char *rsp, unsigned int rsp_len)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (ib->waiting_rsp == msg_id) {
+        ib->waiting_rsp++;
+        if (rsp_len > (sizeof(ib->outmsg) - 2)) {
+            ib->outmsg[0] = 4;
+            ib->outmsg[1] = rsp[0];
+            ib->outmsg[2] = ib->waiting_seq;
+            ib->outmsg[3] = rsp[1];
+            ib->outmsg[4] = IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES;
+            ib->outlen = 5;
+        } else {
+            ib->outmsg[0] = rsp_len + 1;
+            ib->outmsg[1] = rsp[0];
+            ib->outmsg[2] = ib->waiting_seq;
+            memcpy(ib->outmsg + 3, rsp + 1, rsp_len - 1);
+            ib->outlen = rsp_len + 2;
+        }
+        IPMI_BT_SET_BBUSY(ib->control_reg, 0);
+        IPMI_BT_SET_B2H_ATN(ib->control_reg, 1);
+        if (ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_IRQ(ib->mask_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+    }
+}
+
+
+static uint64_t ipmi_bt_ioport_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+    uint32_t ret = 0xff;
+
+    switch (addr & 3) {
+    case 0:
+        ret = ib->control_reg;
+        break;
+    case 1:
+        if (ib->outpos < ib->outlen) {
+            ret = ib->outmsg[ib->outpos];
+            ib->outpos++;
+            if (ib->outpos == ib->outlen) {
+                ib->outpos = 0;
+                ib->outlen = 0;
+            }
+        } else {
+            ret = 0xff;
+        }
+        break;
+    case 2:
+        ret = ib->mask_reg;
+        break;
+    }
+    return ret;
+}
+
+static void ipmi_bt_signal(IPMIBT *ib, IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    ib->do_wake = 1;
+    while (ib->do_wake) {
+        ib->do_wake = 0;
+        iic->handle_if_event(ii);
+    }
+}
+
+static void ipmi_bt_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                 unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    switch (addr & 3) {
+    case 0:
+        if (IPMI_BT_GET_CLR_WR(val)) {
+            ib->inlen = 0;
+        }
+        if (IPMI_BT_GET_CLR_RD(val)) {
+            ib->outpos = 0;
+        }
+        if (IPMI_BT_GET_B2H_ATN(val)) {
+            IPMI_BT_SET_B2H_ATN(ib->control_reg, 0);
+        }
+        if (IPMI_BT_GET_SMS_ATN(val)) {
+            IPMI_BT_SET_SMS_ATN(ib->control_reg, 0);
+        }
+        if (IPMI_BT_GET_HBUSY(val)) {
+            /* Toggle */
+            IPMI_BT_SET_HBUSY(ib->control_reg,
+                              !IPMI_BT_GET_HBUSY(ib->control_reg));
+        }
+        if (IPMI_BT_GET_H2B_ATN(val)) {
+            IPMI_BT_SET_BBUSY(ib->control_reg, 1);
+            ipmi_bt_signal(ib, ii);
+        }
+        break;
+
+    case 1:
+        if (ib->inlen < sizeof(ib->inmsg)) {
+            ib->inmsg[ib->inlen] = val;
+        }
+        ib->inlen++;
+        break;
+
+    case 2:
+        if (IPMI_BT_GET_B2H_IRQ_EN(val) !=
+                        IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            if (IPMI_BT_GET_B2H_IRQ_EN(val)) {
+                if (IPMI_BT_GET_B2H_ATN(ib->control_reg) ||
+                        IPMI_BT_GET_SMS_ATN(ib->control_reg)) {
+                    IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+                    qemu_irq_raise(ib->irq);
+                }
+                IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 1);
+            } else {
+                if (IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+                    IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+                    qemu_irq_lower(ib->irq);
+                }
+                IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 0);
+            }
+        }
+        if (IPMI_BT_GET_B2H_IRQ(val) && IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+        break;
+    }
+}
+
+static const MemoryRegionOps ipmi_bt_io_ops = {
+    .read = ipmi_bt_ioport_read,
+    .write = ipmi_bt_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ipmi_bt_set_atn(IPMIInterface *ii, int val, int irq)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (!!val == IPMI_BT_GET_SMS_ATN(ib->control_reg)) {
+        return;
+    }
+
+    IPMI_BT_SET_SMS_ATN(ib->control_reg, val);
+    if (val) {
+        if (irq && ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_ATN(ib->control_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+    } else {
+        if (!IPMI_BT_GET_B2H_ATN(ib->control_reg) &&
+                IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+    }
+}
+
+static void ipmi_bt_handle_reset(IPMIInterface *ii, bool is_cold)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (is_cold) {
+        /* Disable the BT interrupt on reset */
+        if (IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+        IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 0);
+    }
+}
+
+static void ipmi_bt_set_irq_enable(IPMIInterface *ii, int val)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    ib->irqs_enabled = val;
+}
+
+static void ipmi_bt_init(IPMIInterface *ii, Error **errp)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    ib->io_length = 3;
+
+    memory_region_init_io(&ib->io, NULL, &ipmi_bt_io_ops, ii, "ipmi-bt", 3);
+}
+
+static void ipmi_bt_class_init(IPMIInterfaceClass *iic)
+{
+    iic->init = ipmi_bt_init;
+    iic->set_atn = ipmi_bt_set_atn;
+    iic->handle_rsp = ipmi_bt_handle_rsp;
+    iic->handle_if_event = ipmi_bt_handle_event;
+    iic->set_irq_enable = ipmi_bt_set_irq_enable;
+    iic->reset = ipmi_bt_handle_reset;
+}
+
+
+#define TYPE_ISA_IPMI_BT "isa-ipmi-bt"
+#define ISA_IPMI_BT(obj) OBJECT_CHECK(ISAIPMIBTDevice, (obj), \
+                                       TYPE_ISA_IPMI_BT)
+
+typedef struct ISAIPMIBTDevice {
+    ISADevice dev;
+    int32_t isairq;
+    IPMIBT bt;
+    IPMIFwInfo fwinfo;
+} ISAIPMIBTDevice;
+
+static void isa_ipmi_bt_realize(DeviceState *dev, Error **errp)
+{
+    ISADevice *isadev = ISA_DEVICE(dev);
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(dev);
+    IPMIInterface *ii = IPMI_INTERFACE(dev);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    if (!iib->bt.bmc) {
+        error_setg(errp, "IPMI device requires a bmc attribute to be set");
+        return;
+    }
+
+    iib->bt.bmc->intf = ii;
+
+    iic->init(ii, errp);
+    if (*errp)
+        return;
+
+    if (iib->isairq > 0) {
+        isa_init_irq(isadev, &iib->bt.irq, iib->isairq);
+        iib->bt.use_irq = 1;
+    }
+
+    qdev_set_legacy_instance_id(dev, iib->bt.io_base, iib->bt.io_length);
+
+    isa_register_ioport(isadev, &iib->bt.io, iib->bt.io_base);
+
+    iib->fwinfo.interface_name = "bt";
+    iib->fwinfo.interface_type = IPMI_SMBIOS_BT;
+    iib->fwinfo.ipmi_spec_major_revision = 2;
+    iib->fwinfo.ipmi_spec_minor_revision = 0;
+    iib->fwinfo.base_address = iib->bt.io_base;
+    iib->fwinfo.register_length = iib->bt.io_length;
+    iib->fwinfo.register_spacing = 1;
+    iib->fwinfo.memspace = IPMI_MEMSPACE_IO;
+    iib->fwinfo.irq_type = IPMI_LEVEL_IRQ;
+    iib->fwinfo.interrupt_number = iib->isairq;
+    iib->fwinfo.acpi_parent = "\\_SB.PCI0.ISA";
+    iib->fwinfo.i2c_slave_address = iib->bt.bmc->slave_addr;
+    ipmi_add_fwinfo(&iib->fwinfo, errp);
+}
+
+static const VMStateDescription vmstate_ISAIPMIBTDevice = {
+    .name = TYPE_IPMI_INTERFACE,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(bt.obf_irq_set, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.atn_irq_set, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.use_irq, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.irqs_enabled, ISAIPMIBTDevice),
+        VMSTATE_UINT32(bt.outpos, ISAIPMIBTDevice),
+        VMSTATE_VBUFFER_UINT32(bt.outmsg, ISAIPMIBTDevice, 1, NULL, 0,
+                               bt.outlen),
+        VMSTATE_VBUFFER_UINT32(bt.inmsg, ISAIPMIBTDevice, 1, NULL, 0,
+                               bt.inlen),
+        VMSTATE_UINT8(bt.control_reg, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.mask_reg, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.waiting_rsp, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.waiting_seq, ISAIPMIBTDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void isa_ipmi_bt_init(Object *obj)
+{
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(obj);
+
+    ipmi_bmc_find_and_link(obj, (Object **) &iib->bt.bmc);
+
+    vmstate_register(NULL, 0, &vmstate_ISAIPMIBTDevice, iib);
+}
+
+static void *isa_ipmi_bt_get_backend_data(IPMIInterface *ii)
+{
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(ii);
+
+    return &iib->bt;
+}
+
+static Property ipmi_isa_properties[] = {
+    DEFINE_PROP_UINT32("ioport", ISAIPMIBTDevice, bt.io_base,  0xe4),
+    DEFINE_PROP_INT32("irq",   ISAIPMIBTDevice, isairq,  5),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void isa_ipmi_bt_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_CLASS(oc);
+
+    dc->realize = isa_ipmi_bt_realize;
+    dc->props = ipmi_isa_properties;
+
+    iic->get_backend_data = isa_ipmi_bt_get_backend_data;
+    ipmi_bt_class_init(iic);
+}
+
+static const TypeInfo isa_ipmi_bt_info = {
+    .name          = TYPE_ISA_IPMI_BT,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISAIPMIBTDevice),
+    .instance_init = isa_ipmi_bt_init,
+    .class_init    = isa_ipmi_bt_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_IPMI_INTERFACE },
+        { }
+    }
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&isa_ipmi_bt_info);
+}
+
+type_init(ipmi_register_types)
diff --git a/qemu/hw/ipmi/isa_ipmi_kcs.c b/qemu/hw/ipmi/isa_ipmi_kcs.c
new file mode 100644
index 000000000..2742ce06c
--- /dev/null
+++ b/qemu/hw/ipmi/isa_ipmi_kcs.c
@@ -0,0 +1,495 @@
+/*
+ * QEMU ISA IPMI KCS emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/isa/isa.h"
+#include "hw/i386/pc.h"
+
+#define IPMI_KCS_OBF_BIT        0
+#define IPMI_KCS_IBF_BIT        1
+#define IPMI_KCS_SMS_ATN_BIT    2
+#define IPMI_KCS_CD_BIT         3
+
+#define IPMI_KCS_OBF_MASK          (1 << IPMI_KCS_OBF_BIT)
+#define IPMI_KCS_GET_OBF(d)        (((d) >> IPMI_KCS_OBF_BIT) & 0x1)
+#define IPMI_KCS_SET_OBF(d, v)     (d) = (((d) & ~IPMI_KCS_OBF_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_OBF_BIT))
+#define IPMI_KCS_IBF_MASK          (1 << IPMI_KCS_IBF_BIT)
+#define IPMI_KCS_GET_IBF(d)        (((d) >> IPMI_KCS_IBF_BIT) & 0x1)
+#define IPMI_KCS_SET_IBF(d, v)     (d) = (((d) & ~IPMI_KCS_IBF_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_IBF_BIT))
+#define IPMI_KCS_SMS_ATN_MASK      (1 << IPMI_KCS_SMS_ATN_BIT)
+#define IPMI_KCS_GET_SMS_ATN(d)    (((d) >> IPMI_KCS_SMS_ATN_BIT) & 0x1)
+#define IPMI_KCS_SET_SMS_ATN(d, v) (d) = (((d) & ~IPMI_KCS_SMS_ATN_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_SMS_ATN_BIT))
+#define IPMI_KCS_CD_MASK           (1 << IPMI_KCS_CD_BIT)
+#define IPMI_KCS_GET_CD(d)         (((d) >> IPMI_KCS_CD_BIT) & 0x1)
+#define IPMI_KCS_SET_CD(d, v)      (d) = (((d) & ~IPMI_KCS_CD_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_CD_BIT))
+
+#define IPMI_KCS_IDLE_STATE        0
+#define IPMI_KCS_READ_STATE        1
+#define IPMI_KCS_WRITE_STATE       2
+#define IPMI_KCS_ERROR_STATE       3
+
+#define IPMI_KCS_GET_STATE(d)    (((d) >> 6) & 0x3)
+#define IPMI_KCS_SET_STATE(d, v) ((d) = ((d) & ~0xc0) | (((v) & 0x3) << 6))
+
+#define IPMI_KCS_ABORT_STATUS_CMD       0x60
+#define IPMI_KCS_WRITE_START_CMD        0x61
+#define IPMI_KCS_WRITE_END_CMD          0x62
+#define IPMI_KCS_READ_CMD               0x68
+
+#define IPMI_KCS_STATUS_NO_ERR          0x00
+#define IPMI_KCS_STATUS_ABORTED_ERR     0x01
+#define IPMI_KCS_STATUS_BAD_CC_ERR      0x02
+#define IPMI_KCS_STATUS_LENGTH_ERR      0x06
+
+typedef struct IPMIKCS {
+    IPMIBmc *bmc;
+
+    bool do_wake;
+
+    qemu_irq irq;
+
+    uint32_t io_base;
+    unsigned long io_length;
+    MemoryRegion io;
+
+    bool obf_irq_set;
+    bool atn_irq_set;
+    bool use_irq;
+    bool irqs_enabled;
+
+    uint8_t outmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t outpos;
+    uint32_t outlen;
+
+    uint8_t inmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t inlen;
+    bool write_end;
+
+    uint8_t status_reg;
+    uint8_t data_out_reg;
+
+    int16_t data_in_reg; /* -1 means not written */
+    int16_t cmd_reg;
+
+    /*
+     * This is a response number that we send with the command to make
+     * sure that the response matches the command.
+     */
+    uint8_t waiting_rsp;
+} IPMIKCS;
+
+#define SET_OBF() \
+    do {                                                                      \
+        IPMI_KCS_SET_OBF(ik->status_reg, 1);                                  \
+        if (ik->use_irq && ik->irqs_enabled && !ik->obf_irq_set) {            \
+            ik->obf_irq_set = 1;                                              \
+            if (!ik->atn_irq_set) {                                           \
+                qemu_irq_raise(ik->irq);                                      \
+            }                                                                 \
+        }                                                                     \
+    } while (0)
+
+static void ipmi_kcs_signal(IPMIKCS *ik, IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    ik->do_wake = 1;
+    while (ik->do_wake) {
+        ik->do_wake = 0;
+        iic->handle_if_event(ii);
+    }
+}
+
+static void ipmi_kcs_handle_event(IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (ik->cmd_reg == IPMI_KCS_ABORT_STATUS_CMD) {
+        if (IPMI_KCS_GET_STATE(ik->status_reg) != IPMI_KCS_ERROR_STATE) {
+            ik->waiting_rsp++; /* Invalidate the message */
+            ik->outmsg[0] = IPMI_KCS_STATUS_ABORTED_ERR;
+            ik->outlen = 1;
+            ik->outpos = 0;
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+            SET_OBF();
+        }
+        goto out;
+    }
+
+    switch (IPMI_KCS_GET_STATE(ik->status_reg)) {
+    case IPMI_KCS_IDLE_STATE:
+        if (ik->cmd_reg == IPMI_KCS_WRITE_START_CMD) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_WRITE_STATE);
+            ik->cmd_reg = -1;
+            ik->write_end = 0;
+            ik->inlen = 0;
+            SET_OBF();
+        }
+        break;
+
+    case IPMI_KCS_READ_STATE:
+    handle_read:
+        if (ik->outpos >= ik->outlen) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_IDLE_STATE);
+            SET_OBF();
+        } else if (ik->data_in_reg == IPMI_KCS_READ_CMD) {
+            ik->data_out_reg = ik->outmsg[ik->outpos];
+            ik->outpos++;
+            SET_OBF();
+        } else {
+            ik->outmsg[0] = IPMI_KCS_STATUS_BAD_CC_ERR;
+            ik->outlen = 1;
+            ik->outpos = 0;
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+            SET_OBF();
+            goto out;
+        }
+        break;
+
+    case IPMI_KCS_WRITE_STATE:
+        if (ik->data_in_reg != -1) {
+            /*
+             * Don't worry about input overrun here, that will be
+             * handled in the BMC.
+             */
+            if (ik->inlen < sizeof(ik->inmsg)) {
+                ik->inmsg[ik->inlen] = ik->data_in_reg;
+            }
+            ik->inlen++;
+        }
+        if (ik->write_end) {
+            IPMIBmcClass *bk = IPMI_BMC_GET_CLASS(ik->bmc);
+            ik->outlen = 0;
+            ik->write_end = 0;
+            ik->outpos = 0;
+            bk->handle_command(ik->bmc, ik->inmsg, ik->inlen, sizeof(ik->inmsg),
+                               ik->waiting_rsp);
+            goto out_noibf;
+        } else if (ik->cmd_reg == IPMI_KCS_WRITE_END_CMD) {
+            ik->cmd_reg = -1;
+            ik->write_end = 1;
+        }
+        SET_OBF();
+        break;
+
+    case IPMI_KCS_ERROR_STATE:
+        if (ik->data_in_reg != -1) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_READ_STATE);
+            ik->data_in_reg = IPMI_KCS_READ_CMD;
+            goto handle_read;
+        }
+        break;
+    }
+
+    if (ik->cmd_reg != -1) {
+        /* Got an invalid command */
+        ik->outmsg[0] = IPMI_KCS_STATUS_BAD_CC_ERR;
+        ik->outlen = 1;
+        ik->outpos = 0;
+        IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+    }
+
+ out:
+    ik->cmd_reg = -1;
+    ik->data_in_reg = -1;
+    IPMI_KCS_SET_IBF(ik->status_reg, 0);
+ out_noibf:
+    return;
+}
+
+static void ipmi_kcs_handle_rsp(IPMIInterface *ii, uint8_t msg_id,
+                                unsigned char *rsp, unsigned int rsp_len)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (ik->waiting_rsp == msg_id) {
+        ik->waiting_rsp++;
+        if (rsp_len > sizeof(ik->outmsg)) {
+            ik->outmsg[0] = rsp[0];
+            ik->outmsg[1] = rsp[1];
+            ik->outmsg[2] = IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES;
+            ik->outlen = 3;
+        } else {
+            memcpy(ik->outmsg, rsp, rsp_len);
+            ik->outlen = rsp_len;
+        }
+        IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_READ_STATE);
+        ik->data_in_reg = IPMI_KCS_READ_CMD;
+        ipmi_kcs_signal(ik, ii);
+    }
+}
+
+
+static uint64_t ipmi_kcs_ioport_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+    uint32_t ret;
+
+    switch (addr & 1) {
+    case 0:
+        ret = ik->data_out_reg;
+        IPMI_KCS_SET_OBF(ik->status_reg, 0);
+        if (ik->obf_irq_set) {
+            ik->obf_irq_set = 0;
+            if (!ik->atn_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+        break;
+    case 1:
+        ret = ik->status_reg;
+        if (ik->atn_irq_set) {
+            ik->atn_irq_set = 0;
+            if (!ik->obf_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+        break;
+    }
+    return ret;
+}
+
+static void ipmi_kcs_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (IPMI_KCS_GET_IBF(ik->status_reg)) {
+        return;
+    }
+
+    switch (addr & 1) {
+    case 0:
+        ik->data_in_reg = val;
+        break;
+
+    case 1:
+        ik->cmd_reg = val;
+        break;
+    }
+    IPMI_KCS_SET_IBF(ik->status_reg, 1);
+    ipmi_kcs_signal(ik, ii);
+}
+
+const MemoryRegionOps ipmi_kcs_io_ops = {
+    .read = ipmi_kcs_ioport_read,
+    .write = ipmi_kcs_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ipmi_kcs_set_atn(IPMIInterface *ii, int val, int irq)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    IPMI_KCS_SET_SMS_ATN(ik->status_reg, val);
+    if (val) {
+        if (irq && !ik->atn_irq_set && ik->use_irq && ik->irqs_enabled) {
+            ik->atn_irq_set = 1;
+            if (!ik->obf_irq_set) {
+                qemu_irq_raise(ik->irq);
+            }
+        }
+    } else {
+        if (ik->atn_irq_set) {
+            ik->atn_irq_set = 0;
+            if (!ik->obf_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+    }
+}
+
+static void ipmi_kcs_set_irq_enable(IPMIInterface *ii, int val)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    ik->irqs_enabled = val;
+}
+
+static void ipmi_kcs_init(IPMIInterface *ii, Error **errp)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    ik->io_length = 2;
+    memory_region_init_io(&ik->io, NULL, &ipmi_kcs_io_ops, ii, "ipmi-kcs", 2);
+}
+
+static void ipmi_kcs_class_init(IPMIInterfaceClass *iic)
+{
+    iic->init = ipmi_kcs_init;
+    iic->set_atn = ipmi_kcs_set_atn;
+    iic->handle_rsp = ipmi_kcs_handle_rsp;
+    iic->handle_if_event = ipmi_kcs_handle_event;
+    iic->set_irq_enable = ipmi_kcs_set_irq_enable;
+}
+
+
+#define TYPE_ISA_IPMI_KCS "isa-ipmi-kcs"
+#define ISA_IPMI_KCS(obj) OBJECT_CHECK(ISAIPMIKCSDevice, (obj), \
+                                       TYPE_ISA_IPMI_KCS)
+
+typedef struct ISAIPMIKCSDevice {
+    ISADevice dev;
+    int32_t isairq;
+    IPMIKCS kcs;
+    IPMIFwInfo fwinfo;
+} ISAIPMIKCSDevice;
+
+static void ipmi_isa_realize(DeviceState *dev, Error **errp)
+{
+    ISADevice *isadev = ISA_DEVICE(dev);
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(dev);
+    IPMIInterface *ii = IPMI_INTERFACE(dev);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    if (!iik->kcs.bmc) {
+        error_setg(errp, "IPMI device requires a bmc attribute to be set");
+        return;
+    }
+
+    iik->kcs.bmc->intf = ii;
+
+    iic->init(ii, errp);
+    if (*errp)
+        return;
+
+    if (iik->isairq > 0) {
+        isa_init_irq(isadev, &iik->kcs.irq, iik->isairq);
+        iik->kcs.use_irq = 1;
+    }
+
+    qdev_set_legacy_instance_id(dev, iik->kcs.io_base, iik->kcs.io_length);
+
+    isa_register_ioport(isadev, &iik->kcs.io, iik->kcs.io_base);
+
+    iik->fwinfo.interface_name = "kcs";
+    iik->fwinfo.interface_type = IPMI_SMBIOS_KCS;
+    iik->fwinfo.ipmi_spec_major_revision = 2;
+    iik->fwinfo.ipmi_spec_minor_revision = 0;
+    iik->fwinfo.base_address = iik->kcs.io_base;
+    iik->fwinfo.i2c_slave_address = iik->kcs.bmc->slave_addr;
+    iik->fwinfo.register_length = iik->kcs.io_length;
+    iik->fwinfo.register_spacing = 1;
+    iik->fwinfo.memspace = IPMI_MEMSPACE_IO;
+    iik->fwinfo.irq_type = IPMI_LEVEL_IRQ;
+    iik->fwinfo.interrupt_number = iik->isairq;
+    iik->fwinfo.acpi_parent = "\\_SB.PCI0.ISA";
+    ipmi_add_fwinfo(&iik->fwinfo, errp);
+}
+
+const VMStateDescription vmstate_ISAIPMIKCSDevice = {
+    .name = TYPE_IPMI_INTERFACE,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(kcs.obf_irq_set, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.atn_irq_set, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.use_irq, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.irqs_enabled, ISAIPMIKCSDevice),
+        VMSTATE_UINT32(kcs.outpos, ISAIPMIKCSDevice),
+        VMSTATE_VBUFFER_UINT32(kcs.outmsg, ISAIPMIKCSDevice, 1, NULL, 0,
+                               kcs.outlen),
+        VMSTATE_VBUFFER_UINT32(kcs.inmsg, ISAIPMIKCSDevice, 1, NULL, 0,
+                               kcs.inlen),
+        VMSTATE_BOOL(kcs.write_end, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.status_reg, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.data_out_reg, ISAIPMIKCSDevice),
+        VMSTATE_INT16(kcs.data_in_reg, ISAIPMIKCSDevice),
+        VMSTATE_INT16(kcs.cmd_reg, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.waiting_rsp, ISAIPMIKCSDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void isa_ipmi_kcs_init(Object *obj)
+{
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(obj);
+
+    ipmi_bmc_find_and_link(obj, (Object **) &iik->kcs.bmc);
+
+    vmstate_register(NULL, 0, &vmstate_ISAIPMIKCSDevice, iik);
+}
+
+static void *isa_ipmi_kcs_get_backend_data(IPMIInterface *ii)
+{
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(ii);
+
+    return &iik->kcs;
+}
+
+static Property ipmi_isa_properties[] = {
+    DEFINE_PROP_UINT32("ioport", ISAIPMIKCSDevice, kcs.io_base,  0xca2),
+    DEFINE_PROP_INT32("irq",   ISAIPMIKCSDevice, isairq,  5),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void isa_ipmi_kcs_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_CLASS(oc);
+
+    dc->realize = ipmi_isa_realize;
+    dc->props = ipmi_isa_properties;
+
+    iic->get_backend_data = isa_ipmi_kcs_get_backend_data;
+    ipmi_kcs_class_init(iic);
+}
+
+static const TypeInfo isa_ipmi_kcs_info = {
+    .name          = TYPE_ISA_IPMI_KCS,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISAIPMIKCSDevice),
+    .instance_init = isa_ipmi_kcs_init,
+    .class_init    = isa_ipmi_kcs_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_IPMI_INTERFACE },
+        { }
+    }
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&isa_ipmi_kcs_info);
+}
+
+type_init(ipmi_register_types)
diff --git a/qemu/hw/isa/apm.c b/qemu/hw/isa/apm.c
index 26ab17021..e232b0da0 100644
--- a/qemu/hw/isa/apm.c
+++ b/qemu/hw/isa/apm.c
@@ -20,6 +20,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/apm.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
diff --git a/qemu/hw/isa/i82378.c b/qemu/hw/isa/i82378.c
index fcf97d86a..4d29a9900 100644
--- a/qemu/hw/isa/i82378.c
+++ b/qemu/hw/isa/i82378.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
 #include "hw/timer/i8254.h"
@@ -75,7 +76,10 @@ static void i82378_realize(PCIDevice *pci, Error **errp)
     pci_config_set_interrupt_pin(pci_conf, 1); /* interrupt pin 0 */
 
     isabus = isa_bus_new(dev, get_system_memory(),
-                         pci_address_space_io(pci));
+                         pci_address_space_io(pci), errp);
+    if (!isabus) {
+        return;
+    }
 
     /* This device has:
        2 82C59 (irq)
@@ -100,7 +104,6 @@ static void i82378_realize(PCIDevice *pci, Error **errp)
 
     /* 2 82C37 (dma) */
     isa = isa_create_simple(isabus, "i82374");
-    qdev_connect_gpio_out(DEVICE(isa), 0, s->out[1]);
 
     /* timer */
     isa_create_simple(isabus, "mc146818rtc");
@@ -111,7 +114,7 @@ static void i82378_init(Object *obj)
     DeviceState *dev = DEVICE(obj);
     I82378State *s = I82378(obj);
 
-    qdev_init_gpio_out(dev, s->out, 2);
+    qdev_init_gpio_out(dev, s->out, 1);
     qdev_init_gpio_in(dev, i82378_request_pic_irq, 16);
 }
 
diff --git a/qemu/hw/isa/isa-bus.c b/qemu/hw/isa/isa-bus.c
index 43e0cd8dd..7aa115caf 100644
--- a/qemu/hw/isa/isa-bus.c
+++ b/qemu/hw/isa/isa-bus.c
@@ -16,6 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "monitor/monitor.h"
 #include "hw/sysbus.h"
@@ -36,6 +38,12 @@ static void isa_bus_class_init(ObjectClass *klass, void *data)
     k->get_fw_dev_path = isabus_get_fw_dev_path;
 }
 
+static const TypeInfo isa_dma_info = {
+    .name = TYPE_ISADMA,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(IsaDmaClass),
+};
+
 static const TypeInfo isa_bus_info = {
     .name = TYPE_ISA_BUS,
     .parent = TYPE_BUS,
@@ -44,10 +52,10 @@ static const TypeInfo isa_bus_info = {
 };
 
 ISABus *isa_bus_new(DeviceState *dev, MemoryRegion* address_space,
-                    MemoryRegion *address_space_io)
+                    MemoryRegion *address_space_io, Error **errp)
 {
     if (isabus) {
-        fprintf(stderr, "Can't create a second ISA bus\n");
+        error_setg(errp, "Can't create a second ISA bus");
         return NULL;
     }
     if (!dev) {
@@ -63,9 +71,6 @@ ISABus *isa_bus_new(DeviceState *dev, MemoryRegion* address_space,
 
 void isa_bus_irqs(ISABus *bus, qemu_irq *irqs)
 {
-    if (!bus) {
-        hw_error("Can't set isa irqs with no isa bus present.");
-    }
     bus->irqs = irqs;
 }
 
@@ -92,6 +97,20 @@ void isa_init_irq(ISADevice *dev, qemu_irq *p, int isairq)
     dev->nirqs++;
 }
 
+void isa_bus_dma(ISABus *bus, IsaDma *dma8, IsaDma *dma16)
+{
+    assert(bus && dma8 && dma16);
+    assert(!bus->dma[0] && !bus->dma[1]);
+    bus->dma[0] = dma8;
+    bus->dma[1] = dma16;
+}
+
+IsaDma *isa_get_dma(ISABus *bus, int nchan)
+{
+    assert(bus);
+    return bus->dma[nchan > 3 ? 1 : 0];
+}
+
 static inline void isa_init_ioport(ISADevice *dev, uint16_t ioport)
 {
     if (dev && (dev->ioport_id == 0 || ioport < dev->ioport_id)) {
@@ -137,10 +156,6 @@ ISADevice *isa_create(ISABus *bus, const char *name)
 {
     DeviceState *dev;
 
-    if (!bus) {
-        hw_error("Tried to create isa device %s with no isa bus present.",
-                 name);
-    }
     dev = qdev_create(BUS(bus), name);
     return ISA_DEVICE(dev);
 }
@@ -149,10 +164,6 @@ ISADevice *isa_try_create(ISABus *bus, const char *name)
 {
     DeviceState *dev;
 
-    if (!bus) {
-        hw_error("Tried to create isa device %s with no isa bus present.",
-                 name);
-    }
     dev = qdev_try_create(BUS(bus), name);
     return ISA_DEVICE(dev);
 }
@@ -233,6 +244,7 @@ static const TypeInfo isa_device_type_info = {
 
 static void isabus_register_types(void)
 {
+    type_register_static(&isa_dma_info);
     type_register_static(&isa_bus_info);
     type_register_static(&isabus_bridge_info);
     type_register_static(&isa_device_type_info);
diff --git a/qemu/hw/isa/lpc_ich9.c b/qemu/hw/isa/lpc_ich9.c
index 360699f6f..99cd3ba9e 100644
--- a/qemu/hw/isa/lpc_ich9.c
+++ b/qemu/hw/isa/lpc_ich9.c
@@ -27,7 +27,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "qapi/visitor.h"
 #include "qemu/range.h"
@@ -368,13 +370,13 @@ static void ich9_set_sci(void *opaque, int irq_num, int level)
     }
 }
 
-void ich9_lpc_pm_init(PCIDevice *lpc_pci, bool smm_enabled, bool enable_tco)
+void ich9_lpc_pm_init(PCIDevice *lpc_pci, bool smm_enabled)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pci);
     qemu_irq sci_irq;
 
     sci_irq = qemu_allocate_irq(ich9_set_sci, lpc, 0);
-    ich9_pm_init(lpc_pci, &lpc->pm, smm_enabled, enable_tco, sci_irq);
+    ich9_pm_init(lpc_pci, &lpc->pm, smm_enabled, sci_irq);
     ich9_lpc_reset(&lpc->d.qdev);
 }
 
@@ -394,7 +396,7 @@ static void ich9_apm_ctrl_changed(uint32_t val, void *arg)
 
     /* SMI_EN = PMBASE + 30. SMI control and enable register */
     if (lpc->pm.smi_en & ICH9_PMIO_SMI_EN_APMC_EN) {
-        cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+        cpu_interrupt(current_cpu, CPU_INTERRUPT_SMI);
     }
 }
 
@@ -408,18 +410,18 @@ ich9_lpc_pmbase_update(ICH9LPCState *lpc)
     ich9_pm_iospace_update(&lpc->pm, pm_io_base);
 }
 
-/* config:RBCA */
-static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rbca_old)
+/* config:RCBA */
+static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rcba_old)
 {
-    uint32_t rbca = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
+    uint32_t rcba = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
 
-    if (rbca_old & ICH9_LPC_RCBA_EN) {
-            memory_region_del_subregion(get_system_memory(), &lpc->rbca_mem);
+    if (rcba_old & ICH9_LPC_RCBA_EN) {
+        memory_region_del_subregion(get_system_memory(), &lpc->rcrb_mem);
     }
-    if (rbca & ICH9_LPC_RCBA_EN) {
-            memory_region_add_subregion_overlap(get_system_memory(),
-                                                rbca & ICH9_LPC_RCBA_BA_MASK,
-                                                &lpc->rbca_mem, 1);
+    if (rcba & ICH9_LPC_RCBA_EN) {
+        memory_region_add_subregion_overlap(get_system_memory(),
+                                            rcba & ICH9_LPC_RCBA_BA_MASK,
+                                            &lpc->rcrb_mem, 1);
     }
 }
 
@@ -443,7 +445,7 @@ static int ich9_lpc_post_load(void *opaque, int version_id)
     ICH9LPCState *lpc = opaque;
 
     ich9_lpc_pmbase_update(lpc);
-    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RBCA_EN */);
+    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RCBA_EN */);
     ich9_lpc_pmcon_update(lpc);
     return 0;
 }
@@ -452,14 +454,14 @@ static void ich9_lpc_config_write(PCIDevice *d,
                                   uint32_t addr, uint32_t val, int len)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
-    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    uint32_t rcba_old = pci_get_long(d->config + ICH9_LPC_RCBA);
 
     pci_default_write_config(d, addr, val, len);
     if (ranges_overlap(addr, len, ICH9_LPC_PMBASE, 4)) {
         ich9_lpc_pmbase_update(lpc);
     }
     if (ranges_overlap(addr, len, ICH9_LPC_RCBA, 4)) {
-        ich9_lpc_rcba_update(lpc, rbca_old);
+        ich9_lpc_rcba_update(lpc, rcba_old);
     }
     if (ranges_overlap(addr, len, ICH9_LPC_PIRQA_ROUT, 4)) {
         pci_bus_fire_intx_routing_notifier(lpc->d.bus);
@@ -476,7 +478,7 @@ static void ich9_lpc_reset(DeviceState *qdev)
 {
     PCIDevice *d = PCI_DEVICE(qdev);
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
-    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    uint32_t rcba_old = pci_get_long(d->config + ICH9_LPC_RCBA);
     int i;
 
     for (i = 0; i < 4; i++) {
@@ -495,13 +497,14 @@ static void ich9_lpc_reset(DeviceState *qdev)
     ich9_cc_reset(lpc);
 
     ich9_lpc_pmbase_update(lpc);
-    ich9_lpc_rcba_update(lpc, rbca_old);
+    ich9_lpc_rcba_update(lpc, rcba_old);
 
     lpc->sci_level = 0;
     lpc->rst_cnt = 0;
 }
 
-static const MemoryRegionOps rbca_mmio_ops = {
+/* root complex register block is mapped into memory space */
+static const MemoryRegionOps rcrb_mmio_ops = {
     .read = ich9_cc_read,
     .write = ich9_cc_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
@@ -569,14 +572,13 @@ Object *ich9_lpc_find(void)
     return o;
 }
 
-static void ich9_lpc_get_sci_int(Object *obj, Visitor *v,
-                                 void *opaque, const char *name,
-                                 Error **errp)
+static void ich9_lpc_get_sci_int(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(obj);
     uint32_t value = ich9_lpc_sci_irq(lpc);
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void ich9_lpc_add_properties(ICH9LPCState *lpc)
@@ -602,18 +604,22 @@ static void ich9_lpc_initfn(Object *obj)
     ich9_lpc_add_properties(lpc);
 }
 
-static int ich9_lpc_init(PCIDevice *d)
+static void ich9_lpc_realize(PCIDevice *d, Error **errp)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
     ISABus *isa_bus;
 
-    isa_bus = isa_bus_new(DEVICE(d), get_system_memory(), get_system_io());
+    isa_bus = isa_bus_new(DEVICE(d), get_system_memory(), get_system_io(),
+                          errp);
+    if (!isa_bus) {
+        return;
+    }
 
     pci_set_long(d->wmask + ICH9_LPC_PMBASE,
                  ICH9_LPC_PMBASE_BASE_ADDRESS_MASK);
 
-    memory_region_init_io(&lpc->rbca_mem, OBJECT(d), &rbca_mmio_ops, lpc,
-                            "lpc-rbca-mmio", ICH9_CC_SIZE);
+    memory_region_init_io(&lpc->rcrb_mem, OBJECT(d), &rcrb_mmio_ops, lpc,
+                          "lpc-rcrb-mmio", ICH9_CC_SIZE);
 
     lpc->isa_bus = isa_bus;
 
@@ -628,7 +634,6 @@ static int ich9_lpc_init(PCIDevice *d)
     memory_region_add_subregion_overlap(pci_address_space_io(d),
                                         ICH9_RST_CNT_IOPORT, &lpc->rst_cnt_mem,
                                         1);
-    return 0;
 }
 
 static void ich9_device_plug_cb(HotplugHandler *hotplug_dev,
@@ -706,7 +711,7 @@ static void ich9_lpc_class_init(ObjectClass *klass, void *data)
 
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     dc->reset = ich9_lpc_reset;
-    k->init = ich9_lpc_init;
+    k->realize = ich9_lpc_realize;
     dc->vmsd = &vmstate_ich9_lpc;
     dc->props = ich9_lpc_properties;
     k->config_write = ich9_lpc_config_write;
diff --git a/qemu/hw/isa/pc87312.c b/qemu/hw/isa/pc87312.c
index 3b1fcec53..c3ebf3e7a 100644
--- a/qemu/hw/isa/pc87312.c
+++ b/qemu/hw/isa/pc87312.c
@@ -23,7 +23,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/pc87312.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
@@ -324,14 +326,14 @@ static void pc87312_realize(DeviceState *dev, Error **errp)
         /* FIXME use a qdev drive property instead of drive_get() */
         drive = drive_get(IF_FLOPPY, 0, 0);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveA",
-                                       blk_by_legacy_dinfo(drive));
+            qdev_prop_set_drive(d, "driveA", blk_by_legacy_dinfo(drive),
+                                &error_fatal);
         }
         /* FIXME use a qdev drive property instead of drive_get() */
         drive = drive_get(IF_FLOPPY, 0, 1);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveB",
-                                       blk_by_legacy_dinfo(drive));
+            qdev_prop_set_drive(d, "driveB", blk_by_legacy_dinfo(drive),
+                                &error_fatal);
         }
         qdev_init_nofail(d);
         s->fdc.dev = isa;
diff --git a/qemu/hw/isa/piix4.c b/qemu/hw/isa/piix4.c
index 2c59e91ff..5500fcc4d 100644
--- a/qemu/hw/isa/piix4.c
+++ b/qemu/hw/isa/piix4.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
@@ -90,8 +91,10 @@ static void piix4_realize(PCIDevice *dev, Error **errp)
 {
     PIIX4State *d = PIIX4_PCI_DEVICE(dev);
 
-    isa_bus_new(DEVICE(d), pci_address_space(dev),
-                pci_address_space_io(dev));
+    if (!isa_bus_new(DEVICE(d), pci_address_space(dev),
+                     pci_address_space_io(dev), errp)) {
+        return;
+    }
     piix4_dev = &d->dev;
     qemu_register_reset(piix4_reset, d);
 }
diff --git a/qemu/hw/isa/vt82c686.c b/qemu/hw/isa/vt82c686.c
index 252e1d714..41d5254f8 100644
--- a/qemu/hw/isa/vt82c686.c
+++ b/qemu/hw/isa/vt82c686.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/vt82c686.h"
@@ -440,7 +441,10 @@ static void vt82c686b_realize(PCIDevice *d, Error **errp)
     int i;
 
     isa_bus = isa_bus_new(DEVICE(d), get_system_memory(),
-                          pci_address_space_io(d));
+                          pci_address_space_io(d), errp);
+    if (!isa_bus) {
+        return;
+    }
 
     pci_conf = d->config;
     pci_config_set_prog_interface(pci_conf, 0x0);
diff --git a/qemu/hw/lm32/lm32_boards.c b/qemu/hw/lm32/lm32_boards.c
index 70f48d3b1..c0290560f 100644
--- a/qemu/hw/lm32/lm32_boards.c
+++ b/qemu/hw/lm32/lm32_boards.c
@@ -17,6 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -142,7 +145,7 @@ static void lm32_evr_init(MachineState *machine)
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -244,7 +247,7 @@ static void lm32_uclinux_init(MachineState *machine)
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -292,24 +295,40 @@ static void lm32_uclinux_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine lm32_evr_machine = {
-    .name = "lm32-evr",
-    .desc = "LatticeMico32 EVR32 eval system",
-    .init = lm32_evr_init,
-    .is_default = 1,
+static void lm32_evr_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "LatticeMico32 EVR32 eval system";
+    mc->init = lm32_evr_init;
+    mc->is_default = 1;
+}
+
+static const TypeInfo lm32_evr_type = {
+    .name = MACHINE_TYPE_NAME("lm32-evr"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_evr_class_init,
 };
 
-static QEMUMachine lm32_uclinux_machine = {
-    .name = "lm32-uclinux",
-    .desc = "lm32 platform for uClinux and u-boot by Theobroma Systems",
-    .init = lm32_uclinux_init,
-    .is_default = 0,
+static void lm32_uclinux_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lm32 platform for uClinux and u-boot by Theobroma Systems";
+    mc->init = lm32_uclinux_init;
+    mc->is_default = 0;
+}
+
+static const TypeInfo lm32_uclinux_type = {
+    .name = MACHINE_TYPE_NAME("lm32-uclinux"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_uclinux_class_init,
 };
 
 static void lm32_machine_init(void)
 {
-    qemu_register_machine(&lm32_uclinux_machine);
-    qemu_register_machine(&lm32_evr_machine);
+    type_register_static(&lm32_evr_type);
+    type_register_static(&lm32_uclinux_type);
 }
 
-machine_init(lm32_machine_init);
+type_init(lm32_machine_init)
diff --git a/qemu/hw/lm32/lm32_hwsetup.h b/qemu/hw/lm32/lm32_hwsetup.h
index 838754d5d..b71e6eafb 100644
--- a/qemu/hw/lm32/lm32_hwsetup.h
+++ b/qemu/hw/lm32/lm32_hwsetup.h
@@ -26,6 +26,7 @@
 #define QEMU_HW_LM32_HWSETUP_H
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "hw/loader.h"
 
 typedef struct {
diff --git a/qemu/hw/lm32/milkymist-hw.h b/qemu/hw/lm32/milkymist-hw.h
index 8d20cac1d..c8dfb4d2d 100644
--- a/qemu/hw/lm32/milkymist-hw.h
+++ b/qemu/hw/lm32/milkymist-hw.h
@@ -88,7 +88,8 @@ static inline DeviceState *milkymist_pfpu_create(hwaddr base,
 
 #ifdef CONFIG_OPENGL
 #include <X11/Xlib.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 static const int glx_fbconfig_attr[] = {
     GLX_GREEN_SIZE, 5,
     GLX_GREEN_SIZE, 6,
diff --git a/qemu/hw/lm32/milkymist.c b/qemu/hw/lm32/milkymist.c
index e755f5b24..96e6f4dc2 100644
--- a/qemu/hw/lm32/milkymist.c
+++ b/qemu/hw/lm32/milkymist.c
@@ -17,6 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -30,6 +33,7 @@
 #include "milkymist-hw.h"
 #include "lm32.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 #define BIOS_FILENAME    "mmone-bios.bin"
 #define BIOS_OFFSET      0x00860000
@@ -176,7 +180,7 @@ milkymist_init(MachineState *machine)
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -209,16 +213,11 @@ milkymist_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine milkymist_machine = {
-    .name = "milkymist",
-    .desc = "Milkymist One",
-    .init = milkymist_init,
-    .is_default = 0,
-};
-
-static void milkymist_machine_init(void)
+static void milkymist_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&milkymist_machine);
+    mc->desc = "Milkymist One";
+    mc->init = milkymist_init;
+    mc->is_default = 0;
 }
 
-machine_init(milkymist_machine_init);
+DEFINE_MACHINE("milkymist", milkymist_machine_init)
diff --git a/qemu/hw/m68k/an5206.c b/qemu/hw/m68k/an5206.c
index f63ab2b94..142bab98c 100644
--- a/qemu/hw/m68k/an5206.c
+++ b/qemu/hw/m68k/an5206.c
@@ -6,12 +6,17 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "exec/address-spaces.h"
+#include "qemu/error-report.h"
 #include "sysemu/qtest.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
@@ -39,7 +44,8 @@ static void an5206_init(MachineState *machine)
     }
     cpu = cpu_m68k_init(cpu_model);
     if (!cpu) {
-        hw_error("Unable to find m68k CPU definition\n");
+        error_report("Unable to find m68k CPU definition");
+        exit(1);
     }
     env = &cpu->env;
 
@@ -54,7 +60,7 @@ static void an5206_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_abort);
+    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, AN5206_RAMBAR_ADDR, sram);
 
@@ -70,7 +76,7 @@ static void an5206_init(MachineState *machine)
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -89,15 +95,10 @@ static void an5206_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine an5206_machine = {
-    .name = "an5206",
-    .desc = "Arnewsh 5206",
-    .init = an5206_init,
-};
-
-static void an5206_machine_init(void)
+static void an5206_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&an5206_machine);
+    mc->desc = "Arnewsh 5206";
+    mc->init = an5206_init;
 }
 
-machine_init(an5206_machine_init);
+DEFINE_MACHINE("an5206", an5206_machine_init)
diff --git a/qemu/hw/m68k/dummy_m68k.c b/qemu/hw/m68k/dummy_m68k.c
index 5b77d930e..0b11d2074 100644
--- a/qemu/hw/m68k/dummy_m68k.c
+++ b/qemu/hw/m68k/dummy_m68k.c
@@ -6,6 +6,9 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -49,7 +52,7 @@ static void dummy_m68k_init(MachineState *machine)
     /* Load kernel.  */
     if (kernel_filename) {
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, 1, EM_68K, 0, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -72,15 +75,10 @@ static void dummy_m68k_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine dummy_m68k_machine = {
-    .name = "dummy",
-    .desc = "Dummy board",
-    .init = dummy_m68k_init,
-};
-
-static void dummy_m68k_machine_init(void)
+static void dummy_m68k_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&dummy_m68k_machine);
+    mc->desc = "Dummy board";
+    mc->init = dummy_m68k_init;
 }
 
-machine_init(dummy_m68k_machine_init);
+DEFINE_MACHINE("dummy", dummy_m68k_machine_init)
diff --git a/qemu/hw/m68k/mcf5206.c b/qemu/hw/m68k/mcf5206.c
index 1727a4685..e14896e52 100644
--- a/qemu/hw/m68k/mcf5206.c
+++ b/qemu/hw/m68k/mcf5206.c
@@ -5,6 +5,9 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/m68k/mcf5208.c b/qemu/hw/m68k/mcf5208.c
index 326a42d27..24155574f 100644
--- a/qemu/hw/m68k/mcf5208.c
+++ b/qemu/hw/m68k/mcf5208.c
@@ -5,6 +5,10 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "qemu/timer.h"
@@ -222,7 +226,7 @@ static void mcf5208evb_init(MachineState *machine)
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_abort);
+    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, 0x80000000, sram);
 
@@ -275,7 +279,7 @@ static void mcf5208evb_init(MachineState *machine)
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -294,16 +298,11 @@ static void mcf5208evb_init(MachineState *machine)
     env->pc = entry;
 }
 
-static QEMUMachine mcf5208evb_machine = {
-    .name = "mcf5208evb",
-    .desc = "MCF5206EVB",
-    .init = mcf5208evb_init,
-    .is_default = 1,
-};
-
-static void mcf5208evb_machine_init(void)
+static void mcf5208evb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mcf5208evb_machine);
+    mc->desc = "MCF5206EVB";
+    mc->init = mcf5208evb_init;
+    mc->is_default = 1;
 }
 
-machine_init(mcf5208evb_machine_init);
+DEFINE_MACHINE("mcf5208evb", mcf5208evb_machine_init)
diff --git a/qemu/hw/m68k/mcf_intc.c b/qemu/hw/m68k/mcf_intc.c
index f13c7f3ae..cf581324e 100644
--- a/qemu/hw/m68k/mcf_intc.c
+++ b/qemu/hw/m68k/mcf_intc.c
@@ -5,6 +5,9 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "exec/address-spaces.h"
diff --git a/qemu/hw/mem/Makefile.objs b/qemu/hw/mem/Makefile.objs
index b000fb42b..f12f8b97a 100644
--- a/qemu/hw/mem/Makefile.objs
+++ b/qemu/hw/mem/Makefile.objs
@@ -1 +1,2 @@
 common-obj-$(CONFIG_MEM_HOTPLUG) += pc-dimm.o
+common-obj-$(CONFIG_NVDIMM) += nvdimm.o
diff --git a/qemu/hw/mem/nvdimm.c b/qemu/hw/mem/nvdimm.c
new file mode 100644
index 000000000..0a602f28b
--- /dev/null
+++ b/qemu/hw/mem/nvdimm.c
@@ -0,0 +1,47 @@
+/*
+ * Non-Volatile Dual In-line Memory Module Virtualization Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * Currently, it only supports PMEM Virtualization.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "hw/mem/nvdimm.h"
+
+static void nvdimm_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    /* nvdimm hotplug has not been supported yet. */
+    dc->hotpluggable = false;
+}
+
+static TypeInfo nvdimm_info = {
+    .name          = TYPE_NVDIMM,
+    .parent        = TYPE_PC_DIMM,
+    .class_init    = nvdimm_class_init,
+};
+
+static void nvdimm_register_types(void)
+{
+    type_register_static(&nvdimm_info);
+}
+
+type_init(nvdimm_register_types)
diff --git a/qemu/hw/mem/pc-dimm.c b/qemu/hw/mem/pc-dimm.c
index bb04862de..9e7de5682 100644
--- a/qemu/hw/mem/pc-dimm.c
+++ b/qemu/hw/mem/pc-dimm.c
@@ -18,13 +18,16 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
+#include "qemu/osdep.h"
 #include "hw/mem/pc-dimm.h"
+#include "qapi/error.h"
 #include "qemu/config-file.h"
 #include "qapi/visitor.h"
 #include "qemu/range.h"
 #include "sysemu/numa.h"
 #include "sysemu/kvm.h"
 #include "trace.h"
+#include "hw/virtio/vhost.h"
 
 typedef struct pc_dimms_capacity {
      uint64_t size;
@@ -95,6 +98,12 @@ void pc_dimm_memory_plug(DeviceState *dev, MemoryHotplugState *hpms,
         goto out;
     }
 
+    if (!vhost_has_free_slot()) {
+        error_setg(&local_err, "a used vhost backend has no free"
+                               " memory slots left");
+        goto out;
+    }
+
     memory_region_add_subregion(&hpms->mr, addr - hpms->base, mr);
     vmstate_register_ram(mr, dev);
     numa_set_mem_node_id(addr, memory_region_size(mr), dimm->node);
@@ -172,7 +181,7 @@ int qmp_pc_dimm_device_list(Object *obj, void *opaque)
                                                NULL);
             di->memdev = object_get_canonical_path(OBJECT(dimm->hostmem));
 
-            info->dimm = di;
+            info->u.dimm.data = di;
             elem->value = info;
             elem->next = NULL;
             **prev = elem;
@@ -184,32 +193,6 @@ int qmp_pc_dimm_device_list(Object *obj, void *opaque)
     return 0;
 }
 
-ram_addr_t get_current_ram_size(void)
-{
-    MemoryDeviceInfoList *info_list = NULL;
-    MemoryDeviceInfoList **prev = &info_list;
-    MemoryDeviceInfoList *info;
-    ram_addr_t size = ram_size;
-
-    qmp_pc_dimm_device_list(qdev_get_machine(), &prev);
-    for (info = info_list; info; info = info->next) {
-        MemoryDeviceInfo *value = info->value;
-
-        if (value) {
-            switch (value->kind) {
-            case MEMORY_DEVICE_INFO_KIND_DIMM:
-                size += value->dimm->size;
-                break;
-            default:
-                break;
-            }
-        }
-    }
-    qapi_free_MemoryDeviceInfoList(info_list);
-
-    return size;
-}
-
 static int pc_dimm_slot2bitmap(Object *obj, void *opaque)
 {
     unsigned long *bitmap = opaque;
@@ -365,8 +348,8 @@ static Property pc_dimm_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void pc_dimm_get_size(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void pc_dimm_get_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     int64_t value;
     MemoryRegion *mr;
@@ -375,22 +358,29 @@ static void pc_dimm_get_size(Object *obj, Visitor *v, void *opaque,
     mr = host_memory_backend_get_memory(dimm->hostmem, errp);
     value = memory_region_size(mr);
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void pc_dimm_check_memdev_is_busy(Object *obj, const char *name,
                                       Object *val, Error **errp)
 {
     MemoryRegion *mr;
+    Error *local_err = NULL;
 
-    mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp);
+    mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), &local_err);
+    if (local_err) {
+        goto out;
+    }
     if (memory_region_is_mapped(mr)) {
         char *path = object_get_canonical_path_component(val);
-        error_setg(errp, "can't use already busy memdev: %s", path);
+        error_setg(&local_err, "can't use already busy memdev: %s", path);
         g_free(path);
     } else {
-        qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
+        qdev_prop_allow_set_link_before_realize(obj, name, val, &local_err);
     }
+
+out:
+    error_propagate(errp, local_err);
 }
 
 static void pc_dimm_init(Object *obj)
@@ -414,10 +404,11 @@ static void pc_dimm_realize(DeviceState *dev, Error **errp)
         error_setg(errp, "'" PC_DIMM_MEMDEV_PROP "' property is not set");
         return;
     }
-    if ((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) {
+    if (((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) ||
+        (!nb_numa_nodes && dimm->node)) {
         error_setg(errp, "'DIMM property " PC_DIMM_NODE_PROP " has value %"
                    PRIu32 "' which exceeds the number of numa nodes: %d",
-                   dimm->node, nb_numa_nodes);
+                   dimm->node, nb_numa_nodes ? nb_numa_nodes : 1);
         return;
     }
 }
diff --git a/qemu/hw/microblaze/boot.c b/qemu/hw/microblaze/boot.c
index 3e8820f36..9eebb1a52 100644
--- a/qemu/hw/microblaze/boot.c
+++ b/qemu/hw/microblaze/boot.c
@@ -24,6 +24,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
@@ -32,6 +35,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/cutils.h"
 
 #include "boot.h"
 
@@ -141,12 +145,12 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
                                &entry, &low, &high,
-                               big_endian, ELF_MACHINE, 0);
+                               big_endian, EM_MICROBLAZE, 0, 0);
         base32 = entry;
         if (base32 == 0xc0000000) {
             kernel_size = load_elf(kernel_filename, translate_kernel_address,
                                    NULL, &entry, NULL, NULL,
-                                   big_endian, ELF_MACHINE, 0);
+                                   big_endian, EM_MICROBLAZE, 0, 0);
         }
         /* Always boot into physical ram.  */
         boot_info.bootstrap_pc = (uint32_t)entry;
diff --git a/qemu/hw/microblaze/petalogix_ml605_mmu.c b/qemu/hw/microblaze/petalogix_ml605_mmu.c
index ed84a37e6..07527b677 100644
--- a/qemu/hw/microblaze/petalogix_ml605_mmu.c
+++ b/qemu/hw/microblaze/petalogix_ml605_mmu.c
@@ -25,6 +25,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
@@ -35,7 +39,7 @@
 #include "sysemu/block-backend.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 
 #include "boot.h"
 
@@ -81,6 +85,7 @@ petalogix_ml605_init(MachineState *machine)
 
     /* init CPUs */
     cpu = MICROBLAZE_CPU(object_new(TYPE_MICROBLAZE_CPU));
+    object_property_set_str(OBJECT(cpu), "8.10.a", "version", &error_abort);
     /* Use FPU but don't use floating point conversion and square
      * root instructions
      */
@@ -92,12 +97,12 @@ petalogix_ml605_init(MachineState *machine)
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL, "petalogix_ml605.lmb_bram",
-                           LMB_BRAM_SIZE, &error_abort);
+                           LMB_BRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(address_space_mem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_ml605.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, MEMORY_BASEADDR, phys_ram);
 
@@ -206,16 +211,11 @@ petalogix_ml605_init(MachineState *machine)
 
 }
 
-static QEMUMachine petalogix_ml605_machine = {
-    .name = "petalogix-ml605",
-    .desc = "PetaLogix linux refdesign for xilinx ml605 little endian",
-    .init = petalogix_ml605_init,
-    .is_default = 0,
-};
-
-static void petalogix_ml605_machine_init(void)
+static void petalogix_ml605_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_ml605_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx ml605 little endian";
+    mc->init = petalogix_ml605_init;
+    mc->is_default = 0;
 }
 
-machine_init(petalogix_ml605_machine_init);
+DEFINE_MACHINE("petalogix-ml605", petalogix_ml605_machine_init)
diff --git a/qemu/hw/microblaze/petalogix_s3adsp1800_mmu.c b/qemu/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 0c2140c3f..f821e1cfe 100644
--- a/qemu/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/qemu/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -23,6 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
@@ -66,17 +70,18 @@ petalogix_s3adsp1800_init(MachineState *machine)
     MemoryRegion *sysmem = get_system_memory();
 
     cpu = MICROBLAZE_CPU(object_new(TYPE_MICROBLAZE_CPU));
+    object_property_set_str(OBJECT(cpu), "7.10.d", "version", &error_abort);
     object_property_set_bool(OBJECT(cpu), true, "realized", &error_abort);
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL,
                            "petalogix_s3adsp1800.lmb_bram", LMB_BRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(sysmem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_s3adsp1800.ram",
-                           ram_size, &error_abort);
+                           ram_size, &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(sysmem, ddr_base, phys_ram);
 
@@ -124,16 +129,11 @@ petalogix_s3adsp1800_init(MachineState *machine)
                            NULL);
 }
 
-static QEMUMachine petalogix_s3adsp1800_machine = {
-    .name = "petalogix-s3adsp1800",
-    .desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800",
-    .init = petalogix_s3adsp1800_init,
-    .is_default = 1,
-};
-
-static void petalogix_s3adsp1800_machine_init(void)
+static void petalogix_s3adsp1800_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_s3adsp1800_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800";
+    mc->init = petalogix_s3adsp1800_init;
+    mc->is_default = 1;
 }
 
-machine_init(petalogix_s3adsp1800_machine_init);
+DEFINE_MACHINE("petalogix-s3adsp1800", petalogix_s3adsp1800_machine_init)
diff --git a/qemu/hw/mips/Makefile.objs b/qemu/hw/mips/Makefile.objs
index 9633f3a57..9352a1c06 100644
--- a/qemu/hw/mips/Makefile.objs
+++ b/qemu/hw/mips/Makefile.objs
@@ -3,3 +3,4 @@ obj-y += addr.o cputimer.o mips_int.o
 obj-$(CONFIG_JAZZ) += mips_jazz.o
 obj-$(CONFIG_FULONG) += mips_fulong2e.o
 obj-y += gt64xxx_pci.o
+obj-$(CONFIG_MIPS_CPS) += cps.o
diff --git a/qemu/hw/mips/addr.c b/qemu/hw/mips/addr.c
index ff3b95260..e4e86b4a7 100644
--- a/qemu/hw/mips/addr.c
+++ b/qemu/hw/mips/addr.c
@@ -20,6 +20,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 
diff --git a/qemu/hw/mips/cps.c b/qemu/hw/mips/cps.c
new file mode 100644
index 000000000..1bafbbb27
--- /dev/null
+++ b/qemu/hw/mips/cps.c
@@ -0,0 +1,180 @@
+/*
+ * Coherent Processing System emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/mips/cps.h"
+#include "hw/mips/mips.h"
+#include "hw/mips/cpudevs.h"
+#include "sysemu/kvm.h"
+
+qemu_irq get_cps_irq(MIPSCPSState *s, int pin_number)
+{
+    MIPSCPU *cpu = MIPS_CPU(first_cpu);
+    CPUMIPSState *env = &cpu->env;
+
+    assert(pin_number < s->num_irq);
+
+    /* TODO: return GIC pins once implemented */
+    return env->irq[pin_number];
+}
+
+static void mips_cps_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSCPSState *s = MIPS_CPS(obj);
+
+    /* Cover entire address space as there do not seem to be any
+     * constraints for the base address of CPC and GIC. */
+    memory_region_init(&s->container, obj, "mips-cps-container", UINT64_MAX);
+    sysbus_init_mmio(sbd, &s->container);
+}
+
+static void main_cpu_reset(void *opaque)
+{
+    MIPSCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
+
+    cpu_reset(cs);
+
+    /* All VPs are halted on reset. Leave powering up to CPC. */
+    cs->halted = 1;
+}
+
+static bool cpu_mips_itu_supported(CPUMIPSState *env)
+{
+    bool is_mt = (env->CP0_Config5 & (1 << CP0C5_VP)) ||
+                 (env->CP0_Config3 & (1 << CP0C3_MT));
+
+    return is_mt && !kvm_enabled();
+}
+
+static void mips_cps_realize(DeviceState *dev, Error **errp)
+{
+    MIPSCPSState *s = MIPS_CPS(dev);
+    CPUMIPSState *env;
+    MIPSCPU *cpu;
+    int i;
+    Error *err = NULL;
+    target_ulong gcr_base;
+    bool itu_present = false;
+
+    for (i = 0; i < s->num_vp; i++) {
+        cpu = cpu_mips_init(s->cpu_model);
+        if (cpu == NULL) {
+            error_setg(errp, "%s: CPU initialization failed\n",  __func__);
+            return;
+        }
+        env = &cpu->env;
+
+        /* Init internal devices */
+        cpu_mips_irq_init_cpu(env);
+        cpu_mips_clock_init(env);
+        if (cpu_mips_itu_supported(env)) {
+            itu_present = true;
+            /* Attach ITC Tag to the VP */
+            env->itc_tag = mips_itu_get_tag_region(&s->itu);
+        }
+        qemu_register_reset(main_cpu_reset, cpu);
+    }
+
+    cpu = MIPS_CPU(first_cpu);
+    env = &cpu->env;
+
+    /* Inter-Thread Communication Unit */
+    if (itu_present) {
+        object_initialize(&s->itu, sizeof(s->itu), TYPE_MIPS_ITU);
+        qdev_set_parent_bus(DEVICE(&s->itu), sysbus_get_default());
+
+        object_property_set_int(OBJECT(&s->itu), 16, "num-fifo", &err);
+        object_property_set_int(OBJECT(&s->itu), 16, "num-semaphores", &err);
+        object_property_set_bool(OBJECT(&s->itu), true, "realized", &err);
+        if (err != NULL) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        memory_region_add_subregion(&s->container, 0,
+                           sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->itu), 0));
+    }
+
+    /* Cluster Power Controller */
+    object_initialize(&s->cpc, sizeof(s->cpc), TYPE_MIPS_CPC);
+    qdev_set_parent_bus(DEVICE(&s->cpc), sysbus_get_default());
+
+    object_property_set_int(OBJECT(&s->cpc), s->num_vp, "num-vp", &err);
+    object_property_set_int(OBJECT(&s->cpc), 1, "vp-start-running", &err);
+    object_property_set_bool(OBJECT(&s->cpc), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->container, 0,
+                            sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->cpc), 0));
+
+    /* Global Configuration Registers */
+    gcr_base = env->CP0_CMGCRBase << 4;
+
+    object_initialize(&s->gcr, sizeof(s->gcr), TYPE_MIPS_GCR);
+    qdev_set_parent_bus(DEVICE(&s->gcr), sysbus_get_default());
+
+    object_property_set_int(OBJECT(&s->gcr), s->num_vp, "num-vp", &err);
+    object_property_set_int(OBJECT(&s->gcr), 0x800, "gcr-rev", &err);
+    object_property_set_int(OBJECT(&s->gcr), gcr_base, "gcr-base", &err);
+    object_property_set_link(OBJECT(&s->gcr), OBJECT(&s->cpc.mr), "cpc", &err);
+    object_property_set_bool(OBJECT(&s->gcr), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->container, gcr_base,
+                            sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->gcr), 0));
+}
+
+static Property mips_cps_properties[] = {
+    DEFINE_PROP_UINT32("num-vp", MIPSCPSState, num_vp, 1),
+    DEFINE_PROP_UINT32("num-irq", MIPSCPSState, num_irq, 8),
+    DEFINE_PROP_STRING("cpu-model", MIPSCPSState, cpu_model),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void mips_cps_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = mips_cps_realize;
+    dc->props = mips_cps_properties;
+}
+
+static const TypeInfo mips_cps_info = {
+    .name = TYPE_MIPS_CPS,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSCPSState),
+    .instance_init = mips_cps_init,
+    .class_init = mips_cps_class_init,
+};
+
+static void mips_cps_register_types(void)
+{
+    type_register_static(&mips_cps_info);
+}
+
+type_init(mips_cps_register_types)
diff --git a/qemu/hw/mips/cputimer.c b/qemu/hw/mips/cputimer.c
index 577c9aeab..efb227d06 100644
--- a/qemu/hw/mips/cputimer.c
+++ b/qemu/hw/mips/cputimer.c
@@ -20,23 +20,32 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 #include "qemu/timer.h"
 #include "sysemu/kvm.h"
 
-#define TIMER_FREQ	100 * 1000 * 1000
+#define TIMER_PERIOD 10 /* 10 ns period for 100 Mhz frequency */
 
 /* XXX: do not use a global */
 uint32_t cpu_mips_get_random (CPUMIPSState *env)
 {
-    static uint32_t lfsr = 1;
+    static uint32_t seed = 1;
     static uint32_t prev_idx = 0;
     uint32_t idx;
+    uint32_t nb_rand_tlb = env->tlb->nb_tlb - env->CP0_Wired;
+
+    if (nb_rand_tlb == 1) {
+        return env->tlb->nb_tlb - 1;
+    }
+
     /* Don't return same value twice, so get another value */
     do {
-        lfsr = (lfsr >> 1) ^ (-(lfsr & 1u) & 0xd0000001u);
-        idx = lfsr % (env->tlb->nb_tlb - env->CP0_Wired) + env->CP0_Wired;
+        /* Use a simple algorithm of Linear Congruential Generator
+         * from ISO/IEC 9899 standard. */
+        seed = 1103515245 * seed + 12345;
+        idx = (seed >> 16) % nb_rand_tlb + env->CP0_Wired;
     } while (idx == prev_idx);
     prev_idx = idx;
     return idx;
@@ -49,9 +58,8 @@ static void cpu_mips_timer_update(CPUMIPSState *env)
     uint32_t wait;
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    wait = env->CP0_Compare - env->CP0_Count -
-	    (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    wait = env->CP0_Compare - env->CP0_Count - (uint32_t)(now / TIMER_PERIOD);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(env->timer, next);
 }
 
@@ -79,8 +87,7 @@ uint32_t cpu_mips_get_count (CPUMIPSState *env)
             cpu_mips_timer_expire(env);
         }
 
-        return env->CP0_Count +
-            (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
+        return env->CP0_Count + (uint32_t)(now / TIMER_PERIOD);
     }
 }
 
@@ -95,9 +102,8 @@ void cpu_mips_store_count (CPUMIPSState *env, uint32_t count)
         env->CP0_Count = count;
     else {
         /* Store new count register */
-        env->CP0_Count =
-            count - (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                       TIMER_FREQ, get_ticks_per_sec());
+        env->CP0_Count = count -
+               (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / TIMER_PERIOD);
         /* Update timer timer */
         cpu_mips_timer_update(env);
     }
@@ -121,8 +127,8 @@ void cpu_mips_start_count(CPUMIPSState *env)
 void cpu_mips_stop_count(CPUMIPSState *env)
 {
     /* Store the current value */
-    env->CP0_Count += (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                         TIMER_FREQ, get_ticks_per_sec());
+    env->CP0_Count += (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
+                                 TIMER_PERIOD);
 }
 
 static void mips_timer_cb (void *opaque)
diff --git a/qemu/hw/mips/gt64xxx_pci.c b/qemu/hw/mips/gt64xxx_pci.c
index 10fcca33f..3f4523df2 100644
--- a/qemu/hw/mips/gt64xxx_pci.c
+++ b/qemu/hw/mips/gt64xxx_pci.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/pci/pci.h"
@@ -275,7 +276,8 @@ static void check_reserved_space (hwaddr *start,
 
 static void gt64120_isd_mapping(GT64120State *s)
 {
-    hwaddr start = s->regs[GT_ISD] << 21;
+    /* Bits 14:0 of ISD map to bits 35:21 of the start address.  */
+    hwaddr start = ((hwaddr)s->regs[GT_ISD] << 21) & 0xFFFE00000ull;
     hwaddr length = 0x1000;
 
     if (s->ISD_length) {
@@ -1192,7 +1194,7 @@ static int gt64120_init(SysBusDevice *dev)
     return 0;
 }
 
-static int gt64120_pci_init(PCIDevice *d)
+static void gt64120_pci_realize(PCIDevice *d, Error **errp)
 {
     /* FIXME: Malta specific hw assumptions ahead */
     pci_set_word(d->config + PCI_COMMAND, 0);
@@ -1206,8 +1208,6 @@ static int gt64120_pci_init(PCIDevice *d)
     pci_set_long(d->config + PCI_BASE_ADDRESS_4, 0x14000000);
     pci_set_long(d->config + PCI_BASE_ADDRESS_5, 0x14000001);
     pci_set_byte(d->config + 0x3d, 0x01);
-
-    return 0;
 }
 
 static void gt64120_pci_class_init(ObjectClass *klass, void *data)
@@ -1215,7 +1215,7 @@ static void gt64120_pci_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = gt64120_pci_init;
+    k->realize = gt64120_pci_realize;
     k->vendor_id = PCI_VENDOR_ID_MARVELL;
     k->device_id = PCI_DEVICE_ID_MARVELL_GT6412X;
     k->revision = 0x10;
diff --git a/qemu/hw/mips/mips_fulong2e.c b/qemu/hw/mips/mips_fulong2e.c
index dea941ad8..bdb716e72 100644
--- a/qemu/hw/mips/mips_fulong2e.c
+++ b/qemu/hw/mips/mips_fulong2e.c
@@ -18,6 +18,8 @@
  * http://www.loongsondeveloper.com/doc/Loongson2EUserGuide.pdf
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
@@ -116,7 +118,7 @@ static int64_t load_kernel (CPUMIPSState *env)
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, (uint64_t *)&kernel_low,
-                 (uint64_t *)&kernel_high, 0, ELF_MACHINE, 1) < 0) {
+                 (uint64_t *)&kernel_high, 0, EM_MIPS, 1, 0) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -251,15 +253,6 @@ static void network_init (PCIBus *pci_bus)
     }
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void mips_fulong2e_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
@@ -274,7 +267,6 @@ static void mips_fulong2e_init(MachineState *machine)
     long bios_size;
     int64_t kernel_entry;
     qemu_irq *i8259;
-    qemu_irq *cpu_exit_irq;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     I2CBus *smbus;
@@ -304,7 +296,7 @@ static void mips_fulong2e_init(MachineState *machine)
     /* allocate RAM */
     memory_region_allocate_system_memory(ram, NULL, "fulong2e.ram", ram_size);
     memory_region_init_ram(bios, NULL, "fulong2e.bios", bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -375,8 +367,7 @@ static void mips_fulong2e_init(MachineState *machine)
 
     /* init other devices */
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
@@ -392,15 +383,10 @@ static void mips_fulong2e_init(MachineState *machine)
     network_init(pci_bus);
 }
 
-static QEMUMachine mips_fulong2e_machine = {
-    .name = "fulong2e",
-    .desc = "Fulong 2e mini pc",
-    .init = mips_fulong2e_init,
-};
-
-static void mips_fulong2e_machine_init(void)
+static void mips_fulong2e_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_fulong2e_machine);
+    mc->desc = "Fulong 2e mini pc";
+    mc->init = mips_fulong2e_init;
 }
 
-machine_init(mips_fulong2e_machine_init);
+DEFINE_MACHINE("fulong2e", mips_fulong2e_machine_init)
diff --git a/qemu/hw/mips/mips_int.c b/qemu/hw/mips/mips_int.c
index d740046ba..59081f9d1 100644
--- a/qemu/hw/mips/mips_int.c
+++ b/qemu/hw/mips/mips_int.c
@@ -20,6 +20,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 #include "cpu.h"
diff --git a/qemu/hw/mips/mips_jazz.c b/qemu/hw/mips/mips_jazz.c
index 9d60633ef..ac7c64125 100644
--- a/qemu/hw/mips/mips_jazz.c
+++ b/qemu/hw/mips/mips_jazz.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -44,6 +45,7 @@
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
 #include "qemu/error-report.h"
+#include "qemu/help_option.h"
 
 enum jazz_model_e
 {
@@ -104,15 +106,6 @@ static const MemoryRegionOps dma_dummy_ops = {
 #define MAGNUM_BIOS_SIZE_MAX 0x7e000
 #define MAGNUM_BIOS_SIZE (BIOS_SIZE < MAGNUM_BIOS_SIZE_MAX ? BIOS_SIZE : MAGNUM_BIOS_SIZE_MAX)
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static CPUUnassignedAccess real_do_unassigned_access;
 static void mips_jazz_do_unassigned_access(CPUState *cpu, hwaddr addr,
                                            bool is_write, bool is_exec,
@@ -150,7 +143,6 @@ static void mips_jazz_init(MachineState *machine,
     ISADevice *pit;
     DriveInfo *fds[MAX_FD];
     qemu_irq esp_reset, dma_enable;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *bios = g_new(MemoryRegion, 1);
     MemoryRegion *bios2 = g_new(MemoryRegion, 1);
@@ -184,7 +176,7 @@ static void mips_jazz_init(MachineState *machine,
     memory_region_add_subregion(address_space, 0, ram);
 
     memory_region_init_ram(bios, NULL, "mips_jazz.bios", MAGNUM_BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
     memory_region_init_alias(bios2, NULL, "mips_jazz.bios", bios,
@@ -229,13 +221,12 @@ static void mips_jazz_init(MachineState *machine,
     memory_region_init(isa_mem, NULL, "isa-mem", 0x01000000);
     memory_region_add_subregion(address_space, 0x90000000, isa_io);
     memory_region_add_subregion(address_space, 0x91000000, isa_mem);
-    isa_bus = isa_bus_new(NULL, isa_mem, isa_io);
+    isa_bus = isa_bus_new(NULL, isa_mem, isa_io, &error_abort);
 
     /* ISA devices */
     i8259 = i8259_init(isa_bus, env->irq[4]);
     isa_bus_irqs(isa_bus, i8259);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
     pcspk_init(isa_bus, pit);
 
@@ -252,7 +243,7 @@ static void mips_jazz_init(MachineState *machine,
             /* Simple ROM, so user doesn't have to provide one */
             MemoryRegion *rom_mr = g_new(MemoryRegion, 1);
             memory_region_init_ram(rom_mr, NULL, "g364fb.rom", 0x80000,
-                                   &error_abort);
+                                   &error_fatal);
             vmstate_register_ram_global(rom_mr);
             memory_region_set_readonly(rom_mr, true);
             uint8_t *rom = memory_region_get_ram_ptr(rom_mr);
@@ -307,7 +298,8 @@ static void mips_jazz_init(MachineState *machine,
     for (n = 0; n < MAX_FD; n++) {
         fds[n] = drive_get(IF_FLOPPY, 0, n);
     }
-    fdctrl_init_sysbus(qdev_get_gpio_in(rc4030, 1), 0, 0x80003000, fds);
+    /* FIXME: we should enable DMA with a custom IsaDma device */
+    fdctrl_init_sysbus(qdev_get_gpio_in(rc4030, 1), -1, 0x80003000, fds);
 
     /* Real time clock */
     rtc_init(isa_bus, 1980, NULL);
@@ -360,24 +352,40 @@ void mips_pica61_init(MachineState *machine)
     mips_jazz_init(machine, JAZZ_PICA61);
 }
 
-static QEMUMachine mips_magnum_machine = {
-    .name = "magnum",
-    .desc = "MIPS Magnum",
-    .init = mips_magnum_init,
-    .block_default_type = IF_SCSI,
+static void mips_magnum_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "MIPS Magnum";
+    mc->init = mips_magnum_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_magnum_type = {
+    .name = MACHINE_TYPE_NAME("magnum"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_magnum_class_init,
 };
 
-static QEMUMachine mips_pica61_machine = {
-    .name = "pica61",
-    .desc = "Acer Pica 61",
-    .init = mips_pica61_init,
-    .block_default_type = IF_SCSI,
+static void mips_pica61_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Acer Pica 61";
+    mc->init = mips_pica61_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_pica61_type = {
+    .name = MACHINE_TYPE_NAME("pica61"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_pica61_class_init,
 };
 
 static void mips_jazz_machine_init(void)
 {
-    qemu_register_machine(&mips_magnum_machine);
-    qemu_register_machine(&mips_pica61_machine);
+    type_register_static(&mips_magnum_type);
+    type_register_static(&mips_pica61_type);
 }
 
-machine_init(mips_jazz_machine_init);
+type_init(mips_jazz_machine_init)
diff --git a/qemu/hw/mips/mips_malta.c b/qemu/hw/mips/mips_malta.c
index 3082e7534..fa769e5c0 100644
--- a/qemu/hw/mips/mips_malta.c
+++ b/qemu/hw/mips/mips_malta.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
@@ -54,6 +57,7 @@
 #include "hw/empty_slot.h"
 #include "sysemu/kvm.h"
 #include "exec/semihost.h"
+#include "hw/mips/cps.h"
 
 //#define DEBUG_BOARD_INIT
 
@@ -92,6 +96,7 @@ typedef struct {
 typedef struct {
     SysBusDevice parent_obj;
 
+    MIPSCPSState *cps;
     qemu_irq *i8259;
 } MaltaState;
 
@@ -605,8 +610,8 @@ static void network_init(PCIBus *pci_bus)
      a3 - RAM size in bytes
 */
 
-static void write_bootloader (CPUMIPSState *env, uint8_t *base,
-                              int64_t run_addr, int64_t kernel_entry)
+static void write_bootloader(uint8_t *base, int64_t run_addr,
+                             int64_t kernel_entry)
 {
     uint32_t *p;
 
@@ -795,7 +800,7 @@ static int64_t load_kernel (void)
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, NULL, (uint64_t *)&kernel_high,
-                 big_endian, ELF_MACHINE, 1) < 0) {
+                 big_endian, EM_MIPS, 1, 0) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -901,16 +906,77 @@ static void main_cpu_reset(void *opaque)
 
     if (kvm_enabled()) {
         /* Start running from the bootloader we wrote to end of RAM */
-        env->active_tc.PC = 0x40000000 + loaderparams.ram_size;
+        env->active_tc.PC = 0x40000000 + loaderparams.ram_low_size;
     }
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
+static void create_cpu_without_cps(const char *cpu_model,
+                                   qemu_irq *cbus_irq, qemu_irq *i8259_irq)
 {
-    CPUState *cpu = current_cpu;
+    CPUMIPSState *env;
+    MIPSCPU *cpu;
+    int i;
 
-    if (cpu && level) {
-        cpu_exit(cpu);
+    for (i = 0; i < smp_cpus; i++) {
+        cpu = cpu_mips_init(cpu_model);
+        if (cpu == NULL) {
+            fprintf(stderr, "Unable to find CPU definition\n");
+            exit(1);
+        }
+        env = &cpu->env;
+
+        /* Init internal devices */
+        cpu_mips_irq_init_cpu(env);
+        cpu_mips_clock_init(env);
+        qemu_register_reset(main_cpu_reset, cpu);
+    }
+
+    cpu = MIPS_CPU(first_cpu);
+    env = &cpu->env;
+    *i8259_irq = env->irq[2];
+    *cbus_irq = env->irq[4];
+}
+
+static void create_cps(MaltaState *s, const char *cpu_model,
+                       qemu_irq *cbus_irq, qemu_irq *i8259_irq)
+{
+    Error *err = NULL;
+    s->cps = g_new0(MIPSCPSState, 1);
+
+    object_initialize(s->cps, sizeof(MIPSCPSState), TYPE_MIPS_CPS);
+    qdev_set_parent_bus(DEVICE(s->cps), sysbus_get_default());
+
+    object_property_set_str(OBJECT(s->cps), cpu_model, "cpu-model", &err);
+    object_property_set_int(OBJECT(s->cps), smp_cpus, "num-vp", &err);
+    object_property_set_bool(OBJECT(s->cps), true, "realized", &err);
+    if (err != NULL) {
+        error_report("%s", error_get_pretty(err));
+        exit(1);
+    }
+
+    sysbus_mmio_map_overlap(SYS_BUS_DEVICE(s->cps), 0, 0, 1);
+
+    /* FIXME: When GIC is present then we should use GIC's IRQ 3.
+       Until then CPS exposes CPU's IRQs thus use the default IRQ 2. */
+    *i8259_irq = get_cps_irq(s->cps, 2);
+    *cbus_irq = NULL;
+}
+
+static void create_cpu(MaltaState *s, const char *cpu_model,
+                       qemu_irq *cbus_irq, qemu_irq *i8259_irq)
+{
+    if (cpu_model == NULL) {
+#ifdef TARGET_MIPS64
+        cpu_model = "20Kc";
+#else
+        cpu_model = "24Kf";
+#endif
+    }
+
+    if ((smp_cpus > 1) && cpu_supports_cps_smp(cpu_model)) {
+        create_cps(s, cpu_model, cbus_irq, i8259_irq);
+    } else {
+        create_cpu_without_cps(cpu_model, cbus_irq, i8259_irq);
     }
 }
 
@@ -919,7 +985,6 @@ void mips_malta_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
     ram_addr_t ram_low_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -936,10 +1001,8 @@ void mips_malta_init(MachineState *machine)
     int64_t kernel_entry, bootloader_run_addr;
     PCIBus *pci_bus;
     ISABus *isa_bus;
-    MIPSCPU *cpu;
-    CPUMIPSState *env;
     qemu_irq *isa_irq;
-    qemu_irq *cpu_exit_irq;
+    qemu_irq cbus_irq, i8259_irq;
     int piix4_devfn;
     I2CBus *smbus;
     int i;
@@ -969,30 +1032,8 @@ void mips_malta_init(MachineState *machine)
         }
     }
 
-    /* init CPUs */
-    if (cpu_model == NULL) {
-#ifdef TARGET_MIPS64
-        cpu_model = "20Kc";
-#else
-        cpu_model = "24Kf";
-#endif
-    }
-
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_mips_init(cpu_model);
-        if (cpu == NULL) {
-            fprintf(stderr, "Unable to find CPU definition\n");
-            exit(1);
-        }
-        env = &cpu->env;
-
-        /* Init internal devices */
-        cpu_mips_irq_init_cpu(env);
-        cpu_mips_clock_init(env);
-        qemu_register_reset(main_cpu_reset, cpu);
-    }
-    cpu = MIPS_CPU(first_cpu);
-    env = &cpu->env;
+    /* create CPU */
+    create_cpu(s, machine->cpu_model, &cbus_irq, &i8259_irq);
 
     /* allocate RAM */
     if (ram_size > (2048u << 20)) {
@@ -1033,7 +1074,7 @@ void mips_malta_init(MachineState *machine)
 #endif
     /* FPGA */
     /* The CBUS UART is attached to the MIPS CPU INT2 pin, ie interrupt 4 */
-    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[4], serial_hds[2]);
+    malta_fpga_init(system_memory, FPGA_ADDRESS, cbus_irq, serial_hds[2]);
 
     /* Load firmware in flash / BIOS. */
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
@@ -1070,11 +1111,11 @@ void mips_malta_init(MachineState *machine)
         loaderparams.initrd_filename = initrd_filename;
         kernel_entry = load_kernel();
 
-        write_bootloader(env, memory_region_get_ram_ptr(bios),
+        write_bootloader(memory_region_get_ram_ptr(bios),
                          bootloader_run_addr, kernel_entry);
         if (kvm_enabled()) {
             /* Write the bootloader code @ the end of RAM, 1MB reserved */
-            write_bootloader(env, memory_region_get_ram_ptr(ram_low_preio) +
+            write_bootloader(memory_region_get_ram_ptr(ram_low_preio) +
                                     ram_low_size,
                              bootloader_run_addr, kernel_entry);
         }
@@ -1130,7 +1171,7 @@ void mips_malta_init(MachineState *machine)
      * regions are not executable.
      */
     memory_region_init_ram(bios_copy, NULL, "bios.1fc", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     if (!rom_copy(memory_region_get_ram_ptr(bios_copy),
                   FLASH_ADDRESS, BIOS_SIZE)) {
         memcpy(memory_region_get_ram_ptr(bios_copy),
@@ -1142,10 +1183,6 @@ void mips_malta_init(MachineState *machine)
     /* Board ID = 0x420 (Malta Board with CoreLV) */
     stl_p(memory_region_get_ram_ptr(bios_copy) + 0x10, 0x00000420);
 
-    /* Init internal devices */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
-
     /*
      * We have a circular dependency problem: pci_bus depends on isa_irq,
      * isa_irq is provided by i8259, i8259 depends on ISA, ISA depends
@@ -1165,7 +1202,7 @@ void mips_malta_init(MachineState *machine)
 
     /* Interrupt controller */
     /* The 8259 is attached to the MIPS CPU INT0 pin, ie interrupt 2 */
-    s->i8259 = i8259_init(isa_bus, env->irq[2]);
+    s->i8259 = i8259_init(isa_bus, i8259_irq);
 
     isa_bus_irqs(isa_bus, s->i8259);
     pci_piix4_ide_init(pci_bus, hd, piix4_devfn + 1);
@@ -1175,8 +1212,7 @@ void mips_malta_init(MachineState *machine)
     smbus_eeprom_init(smbus, 8, smbus_eeprom_buf, smbus_eeprom_size);
     g_free(smbus_eeprom_buf);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
@@ -1216,23 +1252,19 @@ static const TypeInfo mips_malta_device = {
     .class_init    = mips_malta_class_init,
 };
 
-static QEMUMachine mips_malta_machine = {
-    .name = "malta",
-    .desc = "MIPS Malta Core LV",
-    .init = mips_malta_init,
-    .max_cpus = 16,
-    .is_default = 1,
-};
-
-static void mips_malta_register_types(void)
+static void mips_malta_machine_init(MachineClass *mc)
 {
-    type_register_static(&mips_malta_device);
+    mc->desc = "MIPS Malta Core LV";
+    mc->init = mips_malta_init;
+    mc->max_cpus = 16;
+    mc->is_default = 1;
 }
 
-static void mips_malta_machine_init(void)
+DEFINE_MACHINE("malta", mips_malta_machine_init)
+
+static void mips_malta_register_types(void)
 {
-    qemu_register_machine(&mips_malta_machine);
+    type_register_static(&mips_malta_device);
 }
 
 type_init(mips_malta_register_types)
-machine_init(mips_malta_machine_init);
diff --git a/qemu/hw/mips/mips_mipssim.c b/qemu/hw/mips/mips_mipssim.c
index 61f74a631..a2c2a1646 100644
--- a/qemu/hw/mips/mips_mipssim.c
+++ b/qemu/hw/mips/mips_mipssim.c
@@ -24,6 +24,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -69,7 +73,7 @@ static int64_t load_kernel(void)
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1, 0);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -174,7 +178,7 @@ mips_mipssim_init(MachineState *machine)
     memory_region_allocate_system_memory(ram, NULL, "mips_mipssim.ram",
                                          ram_size);
     memory_region_init_ram(bios, NULL, "mips_mipssim.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -231,15 +235,10 @@ mips_mipssim_init(MachineState *machine)
         mipsnet_init(0x4200, env->irq[2], &nd_table[0]);
 }
 
-static QEMUMachine mips_mipssim_machine = {
-    .name = "mipssim",
-    .desc = "MIPS MIPSsim platform",
-    .init = mips_mipssim_init,
-};
-
-static void mips_mipssim_machine_init(void)
+static void mips_mipssim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_mipssim_machine);
+    mc->desc = "MIPS MIPSsim platform";
+    mc->init = mips_mipssim_init;
 }
 
-machine_init(mips_mipssim_machine_init);
+DEFINE_MACHINE("mipssim", mips_mipssim_machine_init)
diff --git a/qemu/hw/mips/mips_r4k.c b/qemu/hw/mips/mips_r4k.c
index f4dcacd86..21aca981c 100644
--- a/qemu/hw/mips/mips_r4k.c
+++ b/qemu/hw/mips/mips_r4k.c
@@ -7,6 +7,10 @@
  * All peripherial devices are attached to this "bus" with
  * the standard PC ISA addresses.
 */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -87,7 +91,7 @@ static int64_t load_kernel(void)
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1, 0);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -233,7 +237,7 @@ void mips_r4k_init(MachineState *machine)
     if ((bios_size > 0) && (bios_size <= BIOS_SIZE)) {
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "mips_r4k.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         memory_region_set_readonly(bios, true);
         memory_region_add_subregion(get_system_memory(), 0x1fc00000, bios);
@@ -252,9 +256,7 @@ void mips_r4k_init(MachineState *machine)
         fprintf(stderr, "qemu: Warning, could not load MIPS bios '%s'\n",
 		bios_name);
     }
-    if (filename) {
-        g_free(filename);
-    }
+    g_free(filename);
 
     if (kernel_filename) {
         loaderparams.ram_size = ram_size;
@@ -274,7 +276,7 @@ void mips_r4k_init(MachineState *machine)
     memory_region_init(isa_mem, NULL, "isa-mem", 0x01000000);
     memory_region_add_subregion(get_system_memory(), 0x14000000, isa_io);
     memory_region_add_subregion(get_system_memory(), 0x10000000, isa_mem);
-    isa_bus = isa_bus_new(NULL, isa_mem, get_system_io());
+    isa_bus = isa_bus_new(NULL, isa_mem, get_system_io(), &error_abort);
 
     /* The PIC is attached to the MIPS CPU INT0 pin */
     i8259 = i8259_init(isa_bus, env->irq[2]);
@@ -300,15 +302,10 @@ void mips_r4k_init(MachineState *machine)
     isa_create_simple(isa_bus, "i8042");
 }
 
-static QEMUMachine mips_machine = {
-    .name = "mips",
-    .desc = "mips r4k platform",
-    .init = mips_r4k_init,
-};
-
-static void mips_machine_init(void)
+static void mips_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_machine);
+    mc->desc = "mips r4k platform";
+    mc->init = mips_r4k_init;
 }
 
-machine_init(mips_machine_init);
+DEFINE_MACHINE("mips", mips_machine_init)
diff --git a/qemu/hw/misc/Makefile.objs b/qemu/hw/misc/Makefile.objs
index 4aa76ffec..93f952880 100644
--- a/qemu/hw/misc/Makefile.objs
+++ b/qemu/hw/misc/Makefile.objs
@@ -26,6 +26,9 @@ obj-$(CONFIG_NSERIES) += cbus.o
 obj-$(CONFIG_ECCMEMCTL) += eccmemctl.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_pmu.o
 obj-$(CONFIG_IMX) += imx_ccm.o
+obj-$(CONFIG_IMX) += imx31_ccm.o
+obj-$(CONFIG_IMX) += imx25_ccm.o
+obj-$(CONFIG_IMX) += imx6_ccm.o
 obj-$(CONFIG_MILKYMIST) += milkymist-hpdmc.o
 obj-$(CONFIG_MILKYMIST) += milkymist-pfpu.o
 obj-$(CONFIG_MAINSTONE) += mst_fpga.o
@@ -34,9 +37,16 @@ obj-$(CONFIG_OMAP) += omap_gpmc.o
 obj-$(CONFIG_OMAP) += omap_l4.o
 obj-$(CONFIG_OMAP) += omap_sdrc.o
 obj-$(CONFIG_OMAP) += omap_tap.o
+obj-$(CONFIG_RASPI) += bcm2835_mbox.o
+obj-$(CONFIG_RASPI) += bcm2835_property.o
 obj-$(CONFIG_SLAVIO) += slavio_misc.o
 obj-$(CONFIG_ZYNQ) += zynq_slcr.o
+obj-$(CONFIG_ZYNQ) += zynq-xadc.o
 obj-$(CONFIG_STM32F2XX_SYSCFG) += stm32f2xx_syscfg.o
+obj-$(CONFIG_MIPS_CPS) += mips_cmgcr.o
+obj-$(CONFIG_MIPS_CPS) += mips_cpc.o
+obj-$(CONFIG_MIPS_ITU) += mips_itu.o
 
 obj-$(CONFIG_PVPANIC) += pvpanic.o
 obj-$(CONFIG_EDU) += edu.o
+obj-$(CONFIG_HYPERV_TESTDEV) += hyperv_testdev.o
diff --git a/qemu/hw/misc/a9scu.c b/qemu/hw/misc/a9scu.c
index 443494590..3e8ad8cd7 100644
--- a/qemu/hw/misc/a9scu.c
+++ b/qemu/hw/misc/a9scu.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/misc/a9scu.h"
 
 static uint64_t a9_scu_read(void *opaque, hwaddr offset,
diff --git a/qemu/hw/misc/applesmc.c b/qemu/hw/misc/applesmc.c
index 6bd61e782..77fab5b9d 100644
--- a/qemu/hw/misc/applesmc.c
+++ b/qemu/hw/misc/applesmc.c
@@ -30,6 +30,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "ui/console.h"
diff --git a/qemu/hw/misc/arm11scu.c b/qemu/hw/misc/arm11scu.c
index a79167544..5e54b494b 100644
--- a/qemu/hw/misc/arm11scu.c
+++ b/qemu/hw/misc/arm11scu.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/misc/arm11scu.h"
 
 static uint64_t mpcore_scu_read(void *opaque, hwaddr offset,
diff --git a/qemu/hw/misc/arm_integrator_debug.c b/qemu/hw/misc/arm_integrator_debug.c
index 99b720fbb..902605fef 100644
--- a/qemu/hw/misc/arm_integrator_debug.c
+++ b/qemu/hw/misc/arm_integrator_debug.c
@@ -14,6 +14,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
@@ -79,7 +80,7 @@ static void intdbg_control_init(Object *obj)
     SysBusDevice *sd = SYS_BUS_DEVICE(obj);
     IntegratorDebugState *s = INTEGRATOR_DEBUG(obj);
 
-    memory_region_init_io(&s->iomem, NULL, &intdbg_control_ops,
+    memory_region_init_io(&s->iomem, obj, &intdbg_control_ops,
                           NULL, "dbg-leds", 0x1000000);
     sysbus_init_mmio(sd, &s->iomem);
 }
diff --git a/qemu/hw/misc/arm_l2x0.c b/qemu/hw/misc/arm_l2x0.c
index 9e220c9a5..7e179f1a4 100644
--- a/qemu/hw/misc/arm_l2x0.c
+++ b/qemu/hw/misc/arm_l2x0.c
@@ -18,6 +18,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 /* L2C-310 r3p2 */
diff --git a/qemu/hw/misc/arm_sysctl.c b/qemu/hw/misc/arm_sysctl.c
index 3fad6f86d..34d90d523 100644
--- a/qemu/hw/misc/arm_sysctl.c
+++ b/qemu/hw/misc/arm_sysctl.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
@@ -170,7 +171,8 @@ static uint64_t arm_sysctl_read(void *opaque, hwaddr offset,
     case 0x58: /* BOOTCS */
         return 0;
     case 0x5c: /* 24MHz */
-        return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24000000, get_ticks_per_sec());
+        return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24000000,
+                        NANOSECONDS_PER_SECOND);
     case 0x60: /* MISC */
         return 0;
     case 0x84: /* PROCID0 */
diff --git a/qemu/hw/misc/bcm2835_mbox.c b/qemu/hw/misc/bcm2835_mbox.c
new file mode 100644
index 000000000..263280fd4
--- /dev/null
+++ b/qemu/hw/misc/bcm2835_mbox.c
@@ -0,0 +1,335 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * This file models the system mailboxes, which are used for
+ * communication with low-bandwidth GPU peripherals. Refs:
+ *   https://github.com/raspberrypi/firmware/wiki/Mailboxes
+ *   https://github.com/raspberrypi/firmware/wiki/Accessing-mailboxes
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/misc/bcm2835_mbox.h"
+
+#define MAIL0_PEEK   0x90
+#define MAIL0_SENDER 0x94
+#define MAIL1_STATUS 0xb8
+
+/* Mailbox status register */
+#define MAIL0_STATUS 0x98
+#define ARM_MS_FULL       0x80000000
+#define ARM_MS_EMPTY      0x40000000
+#define ARM_MS_LEVEL      0x400000FF /* Max. value depends on mailbox depth */
+
+/* MAILBOX config/status register */
+#define MAIL0_CONFIG 0x9c
+/* ANY write to this register clears the error bits! */
+#define ARM_MC_IHAVEDATAIRQEN    0x00000001 /* mbox irq enable:  has data */
+#define ARM_MC_IHAVESPACEIRQEN   0x00000002 /* mbox irq enable:  has space */
+#define ARM_MC_OPPISEMPTYIRQEN   0x00000004 /* mbox irq enable: Opp is empty */
+#define ARM_MC_MAIL_CLEAR        0x00000008 /* mbox clear write 1, then  0 */
+#define ARM_MC_IHAVEDATAIRQPEND  0x00000010 /* mbox irq pending:  has space */
+#define ARM_MC_IHAVESPACEIRQPEND 0x00000020 /* mbox irq pending: Opp is empty */
+#define ARM_MC_OPPISEMPTYIRQPEND 0x00000040 /* mbox irq pending */
+/* Bit 7 is unused */
+#define ARM_MC_ERRNOOWN   0x00000100 /* error : none owner read from mailbox */
+#define ARM_MC_ERROVERFLW 0x00000200 /* error : write to fill mailbox */
+#define ARM_MC_ERRUNDRFLW 0x00000400 /* error : read from empty mailbox */
+
+static void mbox_update_status(BCM2835Mbox *mb)
+{
+    mb->status &= ~(ARM_MS_EMPTY | ARM_MS_FULL);
+    if (mb->count == 0) {
+        mb->status |= ARM_MS_EMPTY;
+    } else if (mb->count == MBOX_SIZE) {
+        mb->status |= ARM_MS_FULL;
+    }
+}
+
+static void mbox_reset(BCM2835Mbox *mb)
+{
+    int n;
+
+    mb->count = 0;
+    mb->config = 0;
+    for (n = 0; n < MBOX_SIZE; n++) {
+        mb->reg[n] = MBOX_INVALID_DATA;
+    }
+    mbox_update_status(mb);
+}
+
+static uint32_t mbox_pull(BCM2835Mbox *mb, int index)
+{
+    int n;
+    uint32_t val;
+
+    assert(mb->count > 0);
+    assert(index < mb->count);
+
+    val = mb->reg[index];
+    for (n = index + 1; n < mb->count; n++) {
+        mb->reg[n - 1] = mb->reg[n];
+    }
+    mb->count--;
+    mb->reg[mb->count] = MBOX_INVALID_DATA;
+
+    mbox_update_status(mb);
+
+    return val;
+}
+
+static void mbox_push(BCM2835Mbox *mb, uint32_t val)
+{
+    assert(mb->count < MBOX_SIZE);
+    mb->reg[mb->count++] = val;
+    mbox_update_status(mb);
+}
+
+static void bcm2835_mbox_update(BCM2835MboxState *s)
+{
+    uint32_t value;
+    bool set;
+    int n;
+
+    s->mbox_irq_disabled = true;
+
+    /* Get pending responses and put them in the vc->arm mbox,
+     * as long as it's not full
+     */
+    for (n = 0; n < MBOX_CHAN_COUNT; n++) {
+        while (s->available[n] && !(s->mbox[0].status & ARM_MS_FULL)) {
+            value = ldl_le_phys(&s->mbox_as, n << MBOX_AS_CHAN_SHIFT);
+            assert(value != MBOX_INVALID_DATA); /* Pending interrupt but no data */
+            mbox_push(&s->mbox[0], value);
+        }
+    }
+
+    /* TODO (?): Try to push pending requests from the arm->vc mbox */
+
+    /* Re-enable calls from the IRQ routine */
+    s->mbox_irq_disabled = false;
+
+    /* Update ARM IRQ status */
+    set = false;
+    s->mbox[0].config &= ~ARM_MC_IHAVEDATAIRQPEND;
+    if (!(s->mbox[0].status & ARM_MS_EMPTY)) {
+        s->mbox[0].config |= ARM_MC_IHAVEDATAIRQPEND;
+        if (s->mbox[0].config & ARM_MC_IHAVEDATAIRQEN) {
+            set = true;
+        }
+    }
+    qemu_set_irq(s->arm_irq, set);
+}
+
+static void bcm2835_mbox_set_irq(void *opaque, int irq, int level)
+{
+    BCM2835MboxState *s = opaque;
+
+    s->available[irq] = level;
+
+    /* avoid recursively calling bcm2835_mbox_update when the interrupt
+     * status changes due to the ldl_phys call within that function
+     */
+    if (!s->mbox_irq_disabled) {
+        bcm2835_mbox_update(s);
+    }
+}
+
+static uint64_t bcm2835_mbox_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835MboxState *s = opaque;
+    uint32_t res = 0;
+
+    offset &= 0xff;
+
+    switch (offset) {
+    case 0x80 ... 0x8c: /* MAIL0_READ */
+        if (s->mbox[0].status & ARM_MS_EMPTY) {
+            res = MBOX_INVALID_DATA;
+        } else {
+            res = mbox_pull(&s->mbox[0], 0);
+        }
+        break;
+
+    case MAIL0_PEEK:
+        res = s->mbox[0].reg[0];
+        break;
+
+    case MAIL0_SENDER:
+        break;
+
+    case MAIL0_STATUS:
+        res = s->mbox[0].status;
+        break;
+
+    case MAIL0_CONFIG:
+        res = s->mbox[0].config;
+        break;
+
+    case MAIL1_STATUS:
+        res = s->mbox[1].status;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    bcm2835_mbox_update(s);
+
+    return res;
+}
+
+static void bcm2835_mbox_write(void *opaque, hwaddr offset,
+                               uint64_t value, unsigned size)
+{
+    BCM2835MboxState *s = opaque;
+    hwaddr childaddr;
+    uint8_t ch;
+
+    offset &= 0xff;
+
+    switch (offset) {
+    case MAIL0_SENDER:
+        break;
+
+    case MAIL0_CONFIG:
+        s->mbox[0].config &= ~ARM_MC_IHAVEDATAIRQEN;
+        s->mbox[0].config |= value & ARM_MC_IHAVEDATAIRQEN;
+        break;
+
+    case 0xa0 ... 0xac: /* MAIL1_WRITE */
+        if (s->mbox[1].status & ARM_MS_FULL) {
+            /* Mailbox full */
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: mailbox full\n", __func__);
+        } else {
+            ch = value & 0xf;
+            if (ch < MBOX_CHAN_COUNT) {
+                childaddr = ch << MBOX_AS_CHAN_SHIFT;
+                if (ldl_le_phys(&s->mbox_as, childaddr + MBOX_AS_PENDING)) {
+                    /* Child busy, push delayed. Push it in the arm->vc mbox */
+                    mbox_push(&s->mbox[1], value);
+                } else {
+                    /* Push it directly to the child device */
+                    stl_le_phys(&s->mbox_as, childaddr, value);
+                }
+            } else {
+                /* Invalid channel number */
+                qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid channel %u\n",
+                              __func__, ch);
+            }
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+
+    bcm2835_mbox_update(s);
+}
+
+static const MemoryRegionOps bcm2835_mbox_ops = {
+    .read = bcm2835_mbox_read,
+    .write = bcm2835_mbox_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+/* vmstate of a single mailbox */
+static const VMStateDescription vmstate_bcm2835_mbox_box = {
+    .name = TYPE_BCM2835_MBOX "_box",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, BCM2835Mbox, MBOX_SIZE),
+        VMSTATE_UINT32(count, BCM2835Mbox),
+        VMSTATE_UINT32(status, BCM2835Mbox),
+        VMSTATE_UINT32(config, BCM2835Mbox),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* vmstate of the entire device */
+static const VMStateDescription vmstate_bcm2835_mbox = {
+    .name = TYPE_BCM2835_MBOX,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL_ARRAY(available, BCM2835MboxState, MBOX_CHAN_COUNT),
+        VMSTATE_STRUCT_ARRAY(mbox, BCM2835MboxState, 2, 1,
+                             vmstate_bcm2835_mbox_box, BCM2835Mbox),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_mbox_init(Object *obj)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_mbox_ops, s,
+                          TYPE_BCM2835_MBOX, 0x400);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->arm_irq);
+    qdev_init_gpio_in(DEVICE(s), bcm2835_mbox_set_irq, MBOX_CHAN_COUNT);
+}
+
+static void bcm2835_mbox_reset(DeviceState *dev)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(dev);
+    int n;
+
+    mbox_reset(&s->mbox[0]);
+    mbox_reset(&s->mbox[1]);
+    s->mbox_irq_disabled = false;
+    for (n = 0; n < MBOX_CHAN_COUNT; n++) {
+        s->available[n] = false;
+    }
+}
+
+static void bcm2835_mbox_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(dev);
+    Object *obj;
+    Error *err = NULL;
+
+    obj = object_property_get_link(OBJECT(dev), "mbox-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required mbox-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->mbox_mr = MEMORY_REGION(obj);
+    address_space_init(&s->mbox_as, s->mbox_mr, NULL);
+    bcm2835_mbox_reset(dev);
+}
+
+static void bcm2835_mbox_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = bcm2835_mbox_realize;
+    dc->reset = bcm2835_mbox_reset;
+    dc->vmsd = &vmstate_bcm2835_mbox;
+}
+
+static TypeInfo bcm2835_mbox_info = {
+    .name          = TYPE_BCM2835_MBOX,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835MboxState),
+    .class_init    = bcm2835_mbox_class_init,
+    .instance_init = bcm2835_mbox_init,
+};
+
+static void bcm2835_mbox_register_types(void)
+{
+    type_register_static(&bcm2835_mbox_info);
+}
+
+type_init(bcm2835_mbox_register_types)
diff --git a/qemu/hw/misc/bcm2835_property.c b/qemu/hw/misc/bcm2835_property.c
new file mode 100644
index 000000000..530411f84
--- /dev/null
+++ b/qemu/hw/misc/bcm2835_property.c
@@ -0,0 +1,424 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/misc/bcm2835_property.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+#include "sysemu/dma.h"
+
+/* https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface */
+
+static void bcm2835_property_mbox_push(BCM2835PropertyState *s, uint32_t value)
+{
+    uint32_t tag;
+    uint32_t bufsize;
+    uint32_t tot_len;
+    size_t resplen;
+    uint32_t tmp;
+    int n;
+    uint32_t offset, length, color;
+    uint32_t xres, yres, xoffset, yoffset, bpp, pixo, alpha;
+    uint32_t *newxres = NULL, *newyres = NULL, *newxoffset = NULL,
+        *newyoffset = NULL, *newbpp = NULL, *newpixo = NULL, *newalpha = NULL;
+
+    value &= ~0xf;
+
+    s->addr = value;
+
+    tot_len = ldl_le_phys(&s->dma_as, value);
+
+    /* @(addr + 4) : Buffer response code */
+    value = s->addr + 8;
+    while (value + 8 <= s->addr + tot_len) {
+        tag = ldl_le_phys(&s->dma_as, value);
+        bufsize = ldl_le_phys(&s->dma_as, value + 4);
+        /* @(value + 8) : Request/response indicator */
+        resplen = 0;
+        switch (tag) {
+        case 0x00000000: /* End tag */
+            break;
+        case 0x00000001: /* Get firmware revision */
+            stl_le_phys(&s->dma_as, value + 12, 346337);
+            resplen = 4;
+            break;
+        case 0x00010001: /* Get board model */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x get board model NYI\n", tag);
+            resplen = 4;
+            break;
+        case 0x00010002: /* Get board revision */
+            stl_le_phys(&s->dma_as, value + 12, s->board_rev);
+            resplen = 4;
+            break;
+        case 0x00010003: /* Get board MAC address */
+            resplen = sizeof(s->macaddr.a);
+            dma_memory_write(&s->dma_as, value + 12, s->macaddr.a, resplen);
+            break;
+        case 0x00010004: /* Get board serial */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x get board serial NYI\n", tag);
+            resplen = 8;
+            break;
+        case 0x00010005: /* Get ARM memory */
+            /* base */
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            /* size */
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->vcram_base);
+            resplen = 8;
+            break;
+        case 0x00010006: /* Get VC memory */
+            /* base */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->vcram_base);
+            /* size */
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->vcram_size);
+            resplen = 8;
+            break;
+        case 0x00028001: /* Set power state */
+            /* Assume that whatever device they asked for exists,
+             * and we'll just claim we set it to the desired state
+             */
+            tmp = ldl_le_phys(&s->dma_as, value + 16);
+            stl_le_phys(&s->dma_as, value + 16, (tmp & 1));
+            resplen = 8;
+            break;
+
+        /* Clocks */
+
+        case 0x00030001: /* Get clock state */
+            stl_le_phys(&s->dma_as, value + 16, 0x1);
+            resplen = 8;
+            break;
+
+        case 0x00038001: /* Set clock state */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x set clock state NYI\n", tag);
+            resplen = 8;
+            break;
+
+        case 0x00030002: /* Get clock rate */
+        case 0x00030004: /* Get max clock rate */
+        case 0x00030007: /* Get min clock rate */
+            switch (ldl_le_phys(&s->dma_as, value + 12)) {
+            case 1: /* EMMC */
+                stl_le_phys(&s->dma_as, value + 16, 50000000);
+                break;
+            case 2: /* UART */
+                stl_le_phys(&s->dma_as, value + 16, 3000000);
+                break;
+            default:
+                stl_le_phys(&s->dma_as, value + 16, 700000000);
+                break;
+            }
+            resplen = 8;
+            break;
+
+        case 0x00038002: /* Set clock rate */
+        case 0x00038004: /* Set max clock rate */
+        case 0x00038007: /* Set min clock rate */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x set clock rates NYI\n", tag);
+            resplen = 8;
+            break;
+
+        /* Temperature */
+
+        case 0x00030006: /* Get temperature */
+            stl_le_phys(&s->dma_as, value + 16, 25000);
+            resplen = 8;
+            break;
+
+        case 0x0003000A: /* Get max temperature */
+            stl_le_phys(&s->dma_as, value + 16, 99000);
+            resplen = 8;
+            break;
+
+        /* Frame buffer */
+
+        case 0x00040001: /* Allocate buffer */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->base);
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->size);
+            resplen = 8;
+            break;
+        case 0x00048001: /* Release buffer */
+            resplen = 0;
+            break;
+        case 0x00040002: /* Blank screen */
+            resplen = 4;
+            break;
+        case 0x00040003: /* Get display width/height */
+        case 0x00040004:
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->xres);
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->yres);
+            resplen = 8;
+            break;
+        case 0x00044003: /* Test display width/height */
+        case 0x00044004:
+            resplen = 8;
+            break;
+        case 0x00048003: /* Set display width/height */
+        case 0x00048004:
+            xres = ldl_le_phys(&s->dma_as, value + 12);
+            newxres = &xres;
+            yres = ldl_le_phys(&s->dma_as, value + 16);
+            newyres = &yres;
+            resplen = 8;
+            break;
+        case 0x00040005: /* Get depth */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->bpp);
+            resplen = 4;
+            break;
+        case 0x00044005: /* Test depth */
+            resplen = 4;
+            break;
+        case 0x00048005: /* Set depth */
+            bpp = ldl_le_phys(&s->dma_as, value + 12);
+            newbpp = &bpp;
+            resplen = 4;
+            break;
+        case 0x00040006: /* Get pixel order */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->pixo);
+            resplen = 4;
+            break;
+        case 0x00044006: /* Test pixel order */
+            resplen = 4;
+            break;
+        case 0x00048006: /* Set pixel order */
+            pixo = ldl_le_phys(&s->dma_as, value + 12);
+            newpixo = &pixo;
+            resplen = 4;
+            break;
+        case 0x00040007: /* Get alpha */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->alpha);
+            resplen = 4;
+            break;
+        case 0x00044007: /* Test pixel alpha */
+            resplen = 4;
+            break;
+        case 0x00048007: /* Set alpha */
+            alpha = ldl_le_phys(&s->dma_as, value + 12);
+            newalpha = &alpha;
+            resplen = 4;
+            break;
+        case 0x00040008: /* Get pitch */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->pitch);
+            resplen = 4;
+            break;
+        case 0x00040009: /* Get virtual offset */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->xoffset);
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->yoffset);
+            resplen = 8;
+            break;
+        case 0x00044009: /* Test virtual offset */
+            resplen = 8;
+            break;
+        case 0x00048009: /* Set virtual offset */
+            xoffset = ldl_le_phys(&s->dma_as, value + 12);
+            newxoffset = &xoffset;
+            yoffset = ldl_le_phys(&s->dma_as, value + 16);
+            newyoffset = &yoffset;
+            resplen = 8;
+            break;
+        case 0x0004000a: /* Get/Test/Set overscan */
+        case 0x0004400a:
+        case 0x0004800a:
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            stl_le_phys(&s->dma_as, value + 16, 0);
+            stl_le_phys(&s->dma_as, value + 20, 0);
+            stl_le_phys(&s->dma_as, value + 24, 0);
+            resplen = 16;
+            break;
+        case 0x0004800b: /* Set palette */
+            offset = ldl_le_phys(&s->dma_as, value + 12);
+            length = ldl_le_phys(&s->dma_as, value + 16);
+            n = 0;
+            while (n < length - offset) {
+                color = ldl_le_phys(&s->dma_as, value + 20 + (n << 2));
+                stl_le_phys(&s->dma_as,
+                            s->fbdev->vcram_base + ((offset + n) << 2), color);
+                n++;
+            }
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            resplen = 4;
+            break;
+
+        case 0x00060001: /* Get DMA channels */
+            /* channels 2-5 */
+            stl_le_phys(&s->dma_as, value + 12, 0x003C);
+            resplen = 4;
+            break;
+
+        case 0x00050001: /* Get command line */
+            resplen = 0;
+            break;
+
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "bcm2835_property: unhandled tag %08x\n", tag);
+            break;
+        }
+
+        if (tag == 0) {
+            break;
+        }
+
+        stl_le_phys(&s->dma_as, value + 8, (1 << 31) | resplen);
+        value += bufsize + 12;
+    }
+
+    /* Reconfigure framebuffer if required */
+    if (newxres || newyres || newxoffset || newyoffset || newbpp || newpixo
+        || newalpha) {
+        bcm2835_fb_reconfigure(s->fbdev, newxres, newyres, newxoffset,
+                               newyoffset, newbpp, newpixo, newalpha);
+    }
+
+    /* Buffer response code */
+    stl_le_phys(&s->dma_as, s->addr + 4, (1 << 31));
+}
+
+static uint64_t bcm2835_property_read(void *opaque, hwaddr offset,
+                                      unsigned size)
+{
+    BCM2835PropertyState *s = opaque;
+    uint32_t res = 0;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        res = MBOX_CHAN_PROPERTY | s->addr;
+        s->pending = false;
+        qemu_set_irq(s->mbox_irq, 0);
+        break;
+
+    case MBOX_AS_PENDING:
+        res = s->pending;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_property_write(void *opaque, hwaddr offset,
+                                   uint64_t value, unsigned size)
+{
+    BCM2835PropertyState *s = opaque;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        /* bcm2835_mbox should check our pending status before pushing */
+        assert(!s->pending);
+        s->pending = true;
+        bcm2835_property_mbox_push(s, value);
+        qemu_set_irq(s->mbox_irq, 1);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+}
+
+static const MemoryRegionOps bcm2835_property_ops = {
+    .read = bcm2835_property_read,
+    .write = bcm2835_property_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_property = {
+    .name = TYPE_BCM2835_PROPERTY,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_MACADDR(macaddr, BCM2835PropertyState),
+        VMSTATE_UINT32(addr, BCM2835PropertyState),
+        VMSTATE_BOOL(pending, BCM2835PropertyState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_property_init(Object *obj)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &bcm2835_property_ops, s,
+                          TYPE_BCM2835_PROPERTY, 0x10);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->mbox_irq);
+}
+
+static void bcm2835_property_reset(DeviceState *dev)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(dev);
+
+    s->pending = false;
+}
+
+static void bcm2835_property_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(dev);
+    Object *obj;
+    Error *err = NULL;
+
+    obj = object_property_get_link(OBJECT(dev), "fb", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required fb link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->fbdev = BCM2835_FB(obj);
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    /* TODO: connect to MAC address of USB NIC device, once we emulate it */
+    qemu_macaddr_default_if_unset(&s->macaddr);
+
+    bcm2835_property_reset(dev);
+}
+
+static Property bcm2835_property_props[] = {
+    DEFINE_PROP_UINT32("board-rev", BCM2835PropertyState, board_rev, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2835_property_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = bcm2835_property_props;
+    dc->realize = bcm2835_property_realize;
+    dc->vmsd = &vmstate_bcm2835_property;
+}
+
+static TypeInfo bcm2835_property_info = {
+    .name          = TYPE_BCM2835_PROPERTY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835PropertyState),
+    .class_init    = bcm2835_property_class_init,
+    .instance_init = bcm2835_property_init,
+};
+
+static void bcm2835_property_register_types(void)
+{
+    type_register_static(&bcm2835_property_info);
+}
+
+type_init(bcm2835_property_register_types)
diff --git a/qemu/hw/misc/cbus.c b/qemu/hw/misc/cbus.c
index 495d5078f..0c207e310 100644
--- a/qemu/hw/misc/cbus.c
+++ b/qemu/hw/misc/cbus.c
@@ -20,7 +20,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "hw/hw.h"
 #include "hw/irq.h"
 #include "hw/devices.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/hw/misc/debugexit.c b/qemu/hw/misc/debugexit.c
index 69a1b004c..84fa1a5b9 100644
--- a/qemu/hw/misc/debugexit.c
+++ b/qemu/hw/misc/debugexit.c
@@ -7,6 +7,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 
diff --git a/qemu/hw/misc/eccmemctl.c b/qemu/hw/misc/eccmemctl.c
index 8bad6f682..a0071f3ea 100644
--- a/qemu/hw/misc/eccmemctl.c
+++ b/qemu/hw/misc/eccmemctl.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
diff --git a/qemu/hw/misc/edu.c b/qemu/hw/misc/edu.c
index fe50b42af..888ba49a0 100644
--- a/qemu/hw/misc/edu.c
+++ b/qemu/hw/misc/edu.c
@@ -22,6 +22,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "qemu/timer.h"
 #include "qemu/main-loop.h" /* iothread mutex */
@@ -327,7 +328,7 @@ static void *edu_fact_thread(void *opaque)
     return NULL;
 }
 
-static int pci_edu_init(PCIDevice *pdev)
+static void pci_edu_realize(PCIDevice *pdev, Error **errp)
 {
     EduState *edu = DO_UPCAST(EduState, pdev, pdev);
     uint8_t *pci_conf = pdev->config;
@@ -344,8 +345,6 @@ static int pci_edu_init(PCIDevice *pdev)
     memory_region_init_io(&edu->mmio, OBJECT(edu), &edu_mmio_ops, edu,
                     "edu-mmio", 1 << 20);
     pci_register_bar(pdev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &edu->mmio);
-
-    return 0;
 }
 
 static void pci_edu_uninit(PCIDevice *pdev)
@@ -364,12 +363,12 @@ static void pci_edu_uninit(PCIDevice *pdev)
     timer_del(&edu->dma_timer);
 }
 
-static void edu_obj_uint64(Object *obj, struct Visitor *v, void *opaque,
-                const char *name, Error **errp)
+static void edu_obj_uint64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     uint64_t *val = opaque;
 
-    visit_type_uint64(v, val, name, errp);
+    visit_type_uint64(v, name, val, errp);
 }
 
 static void edu_instance_init(Object *obj)
@@ -385,7 +384,7 @@ static void edu_class_init(ObjectClass *class, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(class);
 
-    k->init = pci_edu_init;
+    k->realize = pci_edu_realize;
     k->exit = pci_edu_uninit;
     k->vendor_id = PCI_VENDOR_ID_QEMU;
     k->device_id = 0x11e8;
diff --git a/qemu/hw/misc/exynos4210_pmu.c b/qemu/hw/misc/exynos4210_pmu.c
index 2b118c725..889abadfe 100644
--- a/qemu/hw/misc/exynos4210_pmu.c
+++ b/qemu/hw/misc/exynos4210_pmu.c
@@ -24,6 +24,7 @@
  * uses PMU INFORM5 register as a holding pen.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #ifndef DEBUG_PMU
diff --git a/qemu/hw/misc/hyperv_testdev.c b/qemu/hw/misc/hyperv_testdev.c
new file mode 100644
index 000000000..1883fd7f2
--- /dev/null
+++ b/qemu/hw/misc/hyperv_testdev.c
@@ -0,0 +1,168 @@
+/*
+ * QEMU KVM Hyper-V test device to support Hyper-V kvm-unit-tests
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/qdev.h"
+#include "hw/isa/isa.h"
+#include "sysemu/kvm.h"
+#include "linux/kvm.h"
+#include "target-i386/hyperv.h"
+#include "kvm_i386.h"
+
+#define HV_TEST_DEV_MAX_SINT_ROUTES 64
+
+struct HypervTestDev {
+    ISADevice parent_obj;
+    MemoryRegion sint_control;
+    HvSintRoute *sint_route[HV_TEST_DEV_MAX_SINT_ROUTES];
+};
+typedef struct HypervTestDev HypervTestDev;
+
+#define TYPE_HYPERV_TEST_DEV "hyperv-testdev"
+#define HYPERV_TEST_DEV(obj) \
+        OBJECT_CHECK(HypervTestDev, (obj), TYPE_HYPERV_TEST_DEV)
+
+enum {
+    HV_TEST_DEV_SINT_ROUTE_CREATE = 1,
+    HV_TEST_DEV_SINT_ROUTE_DESTROY,
+    HV_TEST_DEV_SINT_ROUTE_SET_SINT
+};
+
+static int alloc_sint_route_index(HypervTestDev *dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dev->sint_route); i++) {
+        if (dev->sint_route[i] == NULL) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+static void free_sint_route_index(HypervTestDev *dev, int i)
+{
+    assert(i >= 0 && i < ARRAY_SIZE(dev->sint_route));
+    dev->sint_route[i] = NULL;
+}
+
+static int find_sint_route_index(HypervTestDev *dev, uint32_t vcpu_id,
+                                 uint32_t sint)
+{
+    HvSintRoute *sint_route;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dev->sint_route); i++) {
+        sint_route = dev->sint_route[i];
+        if (sint_route && sint_route->vcpu_id == vcpu_id &&
+            sint_route->sint == sint) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+static void hv_synic_test_dev_control(HypervTestDev *dev, uint32_t ctl,
+                                      uint32_t vcpu_id, uint32_t sint)
+{
+    int i;
+    HvSintRoute *sint_route;
+
+    switch (ctl) {
+    case HV_TEST_DEV_SINT_ROUTE_CREATE:
+        i = alloc_sint_route_index(dev);
+        assert(i >= 0);
+        sint_route = kvm_hv_sint_route_create(vcpu_id, sint, NULL);
+        assert(sint_route);
+        dev->sint_route[i] = sint_route;
+        break;
+    case HV_TEST_DEV_SINT_ROUTE_DESTROY:
+        i = find_sint_route_index(dev, vcpu_id, sint);
+        assert(i >= 0);
+        sint_route = dev->sint_route[i];
+        kvm_hv_sint_route_destroy(sint_route);
+        free_sint_route_index(dev, i);
+        break;
+    case HV_TEST_DEV_SINT_ROUTE_SET_SINT:
+        i = find_sint_route_index(dev, vcpu_id, sint);
+        assert(i >= 0);
+        sint_route = dev->sint_route[i];
+        kvm_hv_sint_route_set_sint(sint_route);
+        break;
+    default:
+        break;
+    }
+}
+
+static void hv_test_dev_control(void *opaque, hwaddr addr, uint64_t data,
+                                uint32_t len)
+{
+    HypervTestDev *dev = HYPERV_TEST_DEV(opaque);
+    uint8_t ctl;
+
+    ctl = (data >> 16ULL) & 0xFF;
+    switch (ctl) {
+    case HV_TEST_DEV_SINT_ROUTE_CREATE:
+    case HV_TEST_DEV_SINT_ROUTE_DESTROY:
+    case HV_TEST_DEV_SINT_ROUTE_SET_SINT: {
+        uint8_t sint = data & 0xFF;
+        uint8_t vcpu_id = (data >> 8ULL) & 0xFF;
+        hv_synic_test_dev_control(dev, ctl, vcpu_id, sint);
+        break;
+    }
+    default:
+        break;
+    }
+}
+
+static const MemoryRegionOps synic_test_sint_ops = {
+    .write = hv_test_dev_control,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void hv_test_dev_realizefn(DeviceState *d, Error **errp)
+{
+    ISADevice *isa = ISA_DEVICE(d);
+    HypervTestDev *dev = HYPERV_TEST_DEV(d);
+    MemoryRegion *io = isa_address_space_io(isa);
+
+    memset(dev->sint_route, 0, sizeof(dev->sint_route));
+    memory_region_init_io(&dev->sint_control, OBJECT(dev),
+                          &synic_test_sint_ops, dev,
+                          "hyperv-testdev-ctl", 4);
+    memory_region_add_subregion(io, 0x3000, &dev->sint_control);
+}
+
+static void hv_test_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->realize = hv_test_dev_realizefn;
+}
+
+static const TypeInfo hv_test_dev_info = {
+    .name           = TYPE_HYPERV_TEST_DEV,
+    .parent         = TYPE_ISA_DEVICE,
+    .instance_size  = sizeof(HypervTestDev),
+    .class_init     = hv_test_dev_class_init,
+};
+
+static void hv_test_dev_register_types(void)
+{
+    type_register_static(&hv_test_dev_info);
+}
+type_init(hv_test_dev_register_types);
diff --git a/qemu/hw/misc/imx25_ccm.c b/qemu/hw/misc/imx25_ccm.c
new file mode 100644
index 000000000..225604d82
--- /dev/null
+++ b/qemu/hw/misc/imx25_ccm.c
@@ -0,0 +1,317 @@
+/*
+ * IMX25 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx25_ccm.h"
+
+#ifndef DEBUG_IMX25_CCM
+#define DEBUG_IMX25_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX25_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX25_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx25_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case IMX25_CCM_MPCTL_REG:
+        return "mpctl";
+    case IMX25_CCM_UPCTL_REG:
+        return "upctl";
+    case IMX25_CCM_CCTL_REG:
+        return "cctl";
+    case IMX25_CCM_CGCR0_REG:
+        return "cgcr0";
+    case IMX25_CCM_CGCR1_REG:
+        return "cgcr1";
+    case IMX25_CCM_CGCR2_REG:
+        return "cgcr2";
+    case IMX25_CCM_PCDR0_REG:
+        return "pcdr0";
+    case IMX25_CCM_PCDR1_REG:
+        return "pcdr1";
+    case IMX25_CCM_PCDR2_REG:
+        return "pcdr2";
+    case IMX25_CCM_PCDR3_REG:
+        return "pcdr3";
+    case IMX25_CCM_RCSR_REG:
+        return "rcsr";
+    case IMX25_CCM_CRDR_REG:
+        return "crdr";
+    case IMX25_CCM_DCVR0_REG:
+        return "dcvr0";
+    case IMX25_CCM_DCVR1_REG:
+        return "dcvr1";
+    case IMX25_CCM_DCVR2_REG:
+        return "dcvr2";
+    case IMX25_CCM_DCVR3_REG:
+        return "dcvr3";
+    case IMX25_CCM_LTR0_REG:
+        return "ltr0";
+    case IMX25_CCM_LTR1_REG:
+        return "ltr1";
+    case IMX25_CCM_LTR2_REG:
+        return "ltr2";
+    case IMX25_CCM_LTR3_REG:
+        return "ltr3";
+    case IMX25_CCM_LTBR0_REG:
+        return "ltbr0";
+    case IMX25_CCM_LTBR1_REG:
+        return "ltbr1";
+    case IMX25_CCM_PMCR0_REG:
+        return "pmcr0";
+    case IMX25_CCM_PMCR1_REG:
+        return "pmcr1";
+    case IMX25_CCM_PMCR2_REG:
+        return "pmcr2";
+    case IMX25_CCM_MCR_REG:
+        return "mcr";
+    case IMX25_CCM_LPIMR0_REG:
+        return "lpimr0";
+    case IMX25_CCM_LPIMR1_REG:
+        return "lpimr1";
+    default:
+        sprintf(unknown, "[%d ?]", reg);
+        return unknown;
+    }
+}
+#define CKIH_FREQ 24000000 /* 24MHz crystal input */
+
+static const VMStateDescription vmstate_imx25_ccm = {
+    .name = TYPE_IMX25_CCM,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, IMX25CCMState, IMX25_CCM_MAX_REG),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint32_t imx25_ccm_get_mpll_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    if (EXTRACT(s->reg[IMX25_CCM_CCTL_REG], MPLL_BYPASS)) {
+        freq = CKIH_FREQ;
+    } else {
+        freq = imx_ccm_calc_pll(s->reg[IMX25_CCM_MPCTL_REG], CKIH_FREQ);
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_mcu_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    freq = imx25_ccm_get_mpll_clk(dev);
+
+    if (EXTRACT(s->reg[IMX25_CCM_CCTL_REG], ARM_SRC)) {
+        freq = (freq * 3 / 4);
+    }
+
+    freq = freq / (1 + EXTRACT(s->reg[IMX25_CCM_CCTL_REG], ARM_CLK_DIV));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_ahb_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    freq = imx25_ccm_get_mcu_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX25_CCM_CCTL_REG], AHB_CLK_DIV));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_ipg_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+
+    freq = imx25_ccm_get_ahb_clk(dev) / 2;
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+    DPRINTF("Clock = %d)\n", clock);
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+    case CLK_IPG_HIGH:
+        freq = imx25_ccm_get_ipg_clk(dev);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX25_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx25_ccm_reset(DeviceState *dev)
+{
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    DPRINTF("\n");
+
+    memset(s->reg, 0, IMX25_CCM_MAX_REG * sizeof(uint32_t));
+    s->reg[IMX25_CCM_MPCTL_REG] = 0x800b2c01;
+    s->reg[IMX25_CCM_UPCTL_REG] = 0x84042800;
+    /* 
+     * The value below gives:
+     * CPU = 133 MHz, AHB = 66,5 MHz, IPG = 33 MHz. 
+     */
+    s->reg[IMX25_CCM_CCTL_REG]  = 0xd0030000;
+    s->reg[IMX25_CCM_CGCR0_REG] = 0x028A0100;
+    s->reg[IMX25_CCM_CGCR1_REG] = 0x04008100;
+    s->reg[IMX25_CCM_CGCR2_REG] = 0x00000438;
+    s->reg[IMX25_CCM_PCDR0_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR1_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR2_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR3_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PMCR0_REG] = 0x00A00000;
+    s->reg[IMX25_CCM_PMCR1_REG] = 0x0000A030;
+    s->reg[IMX25_CCM_PMCR2_REG] = 0x0000A030;
+    s->reg[IMX25_CCM_MCR_REG]   = 0x43000000;
+
+    /*
+     * default boot will change the reset values to allow:
+     * CPU = 399 MHz, AHB = 133 MHz, IPG = 66,5 MHz. 
+     * For some reason, this doesn't work. With the value below, linux
+     * detects a 88 MHz IPG CLK instead of 66,5 MHz.
+    s->reg[IMX25_CCM_CCTL_REG]  = 0x20032000;
+     */
+}
+
+static uint64_t imx25_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMX25CCMState *s = (IMX25CCMState *)opaque;
+
+    if (offset < 0x70) {
+        value = s->reg[offset >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX25_CCM, __func__, offset);
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx25_ccm_reg_name(offset >> 2),
+            value);
+
+    return value;
+}
+
+static void imx25_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                            unsigned size)
+{
+    IMX25CCMState *s = (IMX25CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx25_ccm_reg_name(offset >> 2),
+            (uint32_t)value);
+
+    if (offset < 0x70) {
+        /*
+         * We will do a better implementation later. In particular some bits
+         * cannot be written to.
+         */
+        s->reg[offset >> 2] = value;
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX25_CCM, __func__, offset);
+    }
+}
+
+static const struct MemoryRegionOps imx25_ccm_ops = {
+    .read = imx25_ccm_read,
+    .write = imx25_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx25_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX25CCMState *s = IMX25_CCM(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx25_ccm_ops, s,
+                          TYPE_IMX25_CCM, 0x1000);
+    sysbus_init_mmio(sd, &s->iomem);
+}
+
+static void imx25_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx25_ccm_reset;
+    dc->vmsd = &vmstate_imx25_ccm;
+    dc->desc = "i.MX25 Clock Control Module";
+
+    ccm->get_clock_frequency = imx25_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx25_ccm_info = {
+    .name          = TYPE_IMX25_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX25CCMState),
+    .instance_init = imx25_ccm_init,
+    .class_init    = imx25_ccm_class_init,
+};
+
+static void imx25_ccm_register_types(void)
+{
+    type_register_static(&imx25_ccm_info);
+}
+
+type_init(imx25_ccm_register_types)
diff --git a/qemu/hw/misc/imx31_ccm.c b/qemu/hw/misc/imx31_ccm.c
new file mode 100644
index 000000000..80c164716
--- /dev/null
+++ b/qemu/hw/misc/imx31_ccm.c
@@ -0,0 +1,344 @@
+/*
+ * IMX31 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the i.MX31 CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx31_ccm.h"
+
+#define CKIH_FREQ 26000000 /* 26MHz crystal input */
+
+#ifndef DEBUG_IMX31_CCM
+#define DEBUG_IMX31_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX31_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX31_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx31_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case IMX31_CCM_CCMR_REG:
+        return "CCMR";
+    case IMX31_CCM_PDR0_REG:
+        return "PDR0";
+    case IMX31_CCM_PDR1_REG:
+        return "PDR1";
+    case IMX31_CCM_RCSR_REG:
+        return "RCSR";
+    case IMX31_CCM_MPCTL_REG:
+        return "MPCTL";
+    case IMX31_CCM_UPCTL_REG:
+        return "UPCTL";
+    case IMX31_CCM_SPCTL_REG:
+        return "SPCTL";
+    case IMX31_CCM_COSR_REG:
+        return "COSR";
+    case IMX31_CCM_CGR0_REG:
+        return "CGR0";
+    case IMX31_CCM_CGR1_REG:
+        return "CGR1";
+    case IMX31_CCM_CGR2_REG:
+        return "CGR2";
+    case IMX31_CCM_WIMR_REG:
+        return "WIMR";
+    case IMX31_CCM_LDC_REG:
+        return "LDC";
+    case IMX31_CCM_DCVR0_REG:
+        return "DCVR0";
+    case IMX31_CCM_DCVR1_REG:
+        return "DCVR1";
+    case IMX31_CCM_DCVR2_REG:
+        return "DCVR2";
+    case IMX31_CCM_DCVR3_REG:
+        return "DCVR3";
+    case IMX31_CCM_LTR0_REG:
+        return "LTR0";
+    case IMX31_CCM_LTR1_REG:
+        return "LTR1";
+    case IMX31_CCM_LTR2_REG:
+        return "LTR2";
+    case IMX31_CCM_LTR3_REG:
+        return "LTR3";
+    case IMX31_CCM_LTBR0_REG:
+        return "LTBR0";
+    case IMX31_CCM_LTBR1_REG:
+        return "LTBR1";
+    case IMX31_CCM_PMCR0_REG:
+        return "PMCR0";
+    case IMX31_CCM_PMCR1_REG:
+        return "PMCR1";
+    case IMX31_CCM_PDR2_REG:
+        return "PDR2";
+    default:
+        sprintf(unknown, "[%d ?]", reg);
+        return unknown;
+    }
+}
+
+static const VMStateDescription vmstate_imx31_ccm = {
+    .name = TYPE_IMX31_CCM,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, IMX31CCMState, IMX31_CCM_MAX_REG),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint32_t imx31_ccm_get_pll_ref_clk(IMXCCMState *dev)
+{
+    uint32_t freq = 0;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    if ((s->reg[IMX31_CCM_CCMR_REG] & CCMR_PRCS) == 2) {
+        if (s->reg[IMX31_CCM_CCMR_REG] & CCMR_FPME) {
+            freq = CKIL_FREQ;
+            if (s->reg[IMX31_CCM_CCMR_REG] & CCMR_FPMF) {
+                freq *= 1024;
+            }
+        } 
+    } else {
+        freq = CKIH_FREQ;
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_mpll_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx_ccm_calc_pll(s->reg[IMX31_CCM_MPCTL_REG],
+                            imx31_ccm_get_pll_ref_clk(dev));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_mcu_main_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    if ((s->reg[IMX31_CCM_CCMR_REG] & CCMR_MDS) ||
+        !(s->reg[IMX31_CCM_CCMR_REG] & CCMR_MPE)) {
+        freq = imx31_ccm_get_pll_ref_clk(dev);
+    } else {
+        freq = imx31_ccm_get_mpll_clk(dev);
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_hclk_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx31_ccm_get_mcu_main_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX31_CCM_PDR0_REG], MAX));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_ipg_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx31_ccm_get_hclk_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX31_CCM_PDR0_REG], IPG));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+    case CLK_IPG_HIGH:
+        freq = imx31_ccm_get_ipg_clk(dev);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX31_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx31_ccm_reset(DeviceState *dev)
+{
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    DPRINTF("()\n");
+
+    memset(s->reg, 0, sizeof(uint32_t) * IMX31_CCM_MAX_REG);
+
+    s->reg[IMX31_CCM_CCMR_REG]   = 0x074b0b7d;
+    s->reg[IMX31_CCM_PDR0_REG]   = 0xff870b48;
+    s->reg[IMX31_CCM_PDR1_REG]   = 0x49fcfe7f;
+    s->reg[IMX31_CCM_RCSR_REG]   = 0x007f0000;
+    s->reg[IMX31_CCM_MPCTL_REG]  = 0x04001800;
+    s->reg[IMX31_CCM_UPCTL_REG]  = 0x04051c03;
+    s->reg[IMX31_CCM_SPCTL_REG]  = 0x04043001;
+    s->reg[IMX31_CCM_COSR_REG]   = 0x00000280;
+    s->reg[IMX31_CCM_CGR0_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_CGR1_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_CGR2_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_WIMR_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_LTR1_REG]   = 0x00004040;
+    s->reg[IMX31_CCM_PMCR0_REG]  = 0x80209828;
+    s->reg[IMX31_CCM_PMCR1_REG]  = 0x00aa0000;
+    s->reg[IMX31_CCM_PDR2_REG]   = 0x00000285;
+}
+
+static uint64_t imx31_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMX31CCMState *s = (IMX31CCMState *)opaque;
+
+    if ((offset >> 2) < IMX31_CCM_MAX_REG) {
+        value = s->reg[offset >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX31_CCM, __func__, offset);
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx31_ccm_reg_name(offset >> 2),
+            value);
+
+    return (uint64_t)value;
+}
+
+static void imx31_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                            unsigned size)
+{
+    IMX31CCMState *s = (IMX31CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx31_ccm_reg_name(offset >> 2),
+            (uint32_t)value);
+
+    switch (offset >> 2) {
+    case IMX31_CCM_CCMR_REG:
+        s->reg[IMX31_CCM_CCMR_REG] = CCMR_FPMF | (value & 0x3b6fdfff);
+        break;
+    case IMX31_CCM_PDR0_REG:
+        s->reg[IMX31_CCM_PDR0_REG] = value & 0xff9f3fff;
+        break;
+    case IMX31_CCM_PDR1_REG:
+        s->reg[IMX31_CCM_PDR1_REG] = value;
+        break;
+    case IMX31_CCM_MPCTL_REG:
+        s->reg[IMX31_CCM_MPCTL_REG] = value & 0xbfff3fff;
+        break;
+    case IMX31_CCM_SPCTL_REG:
+        s->reg[IMX31_CCM_SPCTL_REG] = value & 0xbfff3fff;
+        break;
+    case IMX31_CCM_CGR0_REG:
+        s->reg[IMX31_CCM_CGR0_REG] = value;
+        break;
+    case IMX31_CCM_CGR1_REG:
+        s->reg[IMX31_CCM_CGR1_REG] = value;
+        break;
+    case IMX31_CCM_CGR2_REG:
+        s->reg[IMX31_CCM_CGR2_REG] = value;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX31_CCM, __func__, offset);
+        break;
+    }
+}
+
+static const struct MemoryRegionOps imx31_ccm_ops = {
+    .read = imx31_ccm_read,
+    .write = imx31_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+
+};
+
+static void imx31_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX31CCMState *s = IMX31_CCM(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx31_ccm_ops, s,
+                          TYPE_IMX31_CCM, 0x1000);
+    sysbus_init_mmio(sd, &s->iomem);
+}
+
+static void imx31_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc  = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx31_ccm_reset;
+    dc->vmsd  = &vmstate_imx31_ccm;
+    dc->desc  = "i.MX31 Clock Control Module";
+
+    ccm->get_clock_frequency = imx31_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx31_ccm_info = {
+    .name          = TYPE_IMX31_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX31CCMState),
+    .instance_init = imx31_ccm_init,
+    .class_init    = imx31_ccm_class_init,
+};
+
+static void imx31_ccm_register_types(void)
+{
+    type_register_static(&imx31_ccm_info);
+}
+
+type_init(imx31_ccm_register_types)
diff --git a/qemu/hw/misc/imx6_ccm.c b/qemu/hw/misc/imx6_ccm.c
new file mode 100644
index 000000000..4e1d49da6
--- /dev/null
+++ b/qemu/hw/misc/imx6_ccm.c
@@ -0,0 +1,774 @@
+/*
+ * IMX6 Clock Control Module
+ *
+ * Copyright (c) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx6_ccm.h"
+
+#ifndef DEBUG_IMX6_CCM
+#define DEBUG_IMX6_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX6_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX6_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx6_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case CCM_CCR:
+        return "CCR";
+    case CCM_CCDR:
+        return "CCDR";
+    case CCM_CSR:
+        return "CSR";
+    case CCM_CCSR:
+        return "CCSR";
+    case CCM_CACRR:
+        return "CACRR";
+    case CCM_CBCDR:
+        return "CBCDR";
+    case CCM_CBCMR:
+        return "CBCMR";
+    case CCM_CSCMR1:
+        return "CSCMR1";
+    case CCM_CSCMR2:
+        return "CSCMR2";
+    case CCM_CSCDR1:
+        return "CSCDR1";
+    case CCM_CS1CDR:
+        return "CS1CDR";
+    case CCM_CS2CDR:
+        return "CS2CDR";
+    case CCM_CDCDR:
+        return "CDCDR";
+    case CCM_CHSCCDR:
+        return "CHSCCDR";
+    case CCM_CSCDR2:
+        return "CSCDR2";
+    case CCM_CSCDR3:
+        return "CSCDR3";
+    case CCM_CDHIPR:
+        return "CDHIPR";
+    case CCM_CTOR:
+        return "CTOR";
+    case CCM_CLPCR:
+        return "CLPCR";
+    case CCM_CISR:
+        return "CISR";
+    case CCM_CIMR:
+        return "CIMR";
+    case CCM_CCOSR:
+        return "CCOSR";
+    case CCM_CGPR:
+        return "CGPR";
+    case CCM_CCGR0:
+        return "CCGR0";
+    case CCM_CCGR1:
+        return "CCGR1";
+    case CCM_CCGR2:
+        return "CCGR2";
+    case CCM_CCGR3:
+        return "CCGR3";
+    case CCM_CCGR4:
+        return "CCGR4";
+    case CCM_CCGR5:
+        return "CCGR5";
+    case CCM_CCGR6:
+        return "CCGR6";
+    case CCM_CMEOR:
+        return "CMEOR";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+static char const *imx6_analog_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case CCM_ANALOG_PLL_ARM:
+        return "PLL_ARM";
+    case CCM_ANALOG_PLL_ARM_SET:
+        return "PLL_ARM_SET";
+    case CCM_ANALOG_PLL_ARM_CLR:
+        return "PLL_ARM_CLR";
+    case CCM_ANALOG_PLL_ARM_TOG:
+        return "PLL_ARM_TOG";
+    case CCM_ANALOG_PLL_USB1:
+        return "PLL_USB1";
+    case CCM_ANALOG_PLL_USB1_SET:
+        return "PLL_USB1_SET";
+    case CCM_ANALOG_PLL_USB1_CLR:
+        return "PLL_USB1_CLR";
+    case CCM_ANALOG_PLL_USB1_TOG:
+        return "PLL_USB1_TOG";
+    case CCM_ANALOG_PLL_USB2:
+        return "PLL_USB2";
+    case CCM_ANALOG_PLL_USB2_SET:
+        return "PLL_USB2_SET";
+    case CCM_ANALOG_PLL_USB2_CLR:
+        return "PLL_USB2_CLR";
+    case CCM_ANALOG_PLL_USB2_TOG:
+        return "PLL_USB2_TOG";
+    case CCM_ANALOG_PLL_SYS:
+        return "PLL_SYS";
+    case CCM_ANALOG_PLL_SYS_SET:
+        return "PLL_SYS_SET";
+    case CCM_ANALOG_PLL_SYS_CLR:
+        return "PLL_SYS_CLR";
+    case CCM_ANALOG_PLL_SYS_TOG:
+        return "PLL_SYS_TOG";
+    case CCM_ANALOG_PLL_SYS_SS:
+        return "PLL_SYS_SS";
+    case CCM_ANALOG_PLL_SYS_NUM:
+        return "PLL_SYS_NUM";
+    case CCM_ANALOG_PLL_SYS_DENOM:
+        return "PLL_SYS_DENOM";
+    case CCM_ANALOG_PLL_AUDIO:
+        return "PLL_AUDIO";
+    case CCM_ANALOG_PLL_AUDIO_SET:
+        return "PLL_AUDIO_SET";
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+        return "PLL_AUDIO_CLR";
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+        return "PLL_AUDIO_TOG";
+    case CCM_ANALOG_PLL_AUDIO_NUM:
+        return "PLL_AUDIO_NUM";
+    case CCM_ANALOG_PLL_AUDIO_DENOM:
+        return "PLL_AUDIO_DENOM";
+    case CCM_ANALOG_PLL_VIDEO:
+        return "PLL_VIDEO";
+    case CCM_ANALOG_PLL_VIDEO_SET:
+        return "PLL_VIDEO_SET";
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+        return "PLL_VIDEO_CLR";
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+        return "PLL_VIDEO_TOG";
+    case CCM_ANALOG_PLL_VIDEO_NUM:
+        return "PLL_VIDEO_NUM";
+    case CCM_ANALOG_PLL_VIDEO_DENOM:
+        return "PLL_VIDEO_DENOM";
+    case CCM_ANALOG_PLL_MLB:
+        return "PLL_MLB";
+    case CCM_ANALOG_PLL_MLB_SET:
+        return "PLL_MLB_SET";
+    case CCM_ANALOG_PLL_MLB_CLR:
+        return "PLL_MLB_CLR";
+    case CCM_ANALOG_PLL_MLB_TOG:
+        return "PLL_MLB_TOG";
+    case CCM_ANALOG_PLL_ENET:
+        return "PLL_ENET";
+    case CCM_ANALOG_PLL_ENET_SET:
+        return "PLL_ENET_SET";
+    case CCM_ANALOG_PLL_ENET_CLR:
+        return "PLL_ENET_CLR";
+    case CCM_ANALOG_PLL_ENET_TOG:
+        return "PLL_ENET_TOG";
+    case CCM_ANALOG_PFD_480:
+        return "PFD_480";
+    case CCM_ANALOG_PFD_480_SET:
+        return "PFD_480_SET";
+    case CCM_ANALOG_PFD_480_CLR:
+        return "PFD_480_CLR";
+    case CCM_ANALOG_PFD_480_TOG:
+        return "PFD_480_TOG";
+    case CCM_ANALOG_PFD_528:
+        return "PFD_528";
+    case CCM_ANALOG_PFD_528_SET:
+        return "PFD_528_SET";
+    case CCM_ANALOG_PFD_528_CLR:
+        return "PFD_528_CLR";
+    case CCM_ANALOG_PFD_528_TOG:
+        return "PFD_528_TOG";
+    case CCM_ANALOG_MISC0:
+        return "MISC0";
+    case CCM_ANALOG_MISC0_SET:
+        return "MISC0_SET";
+    case CCM_ANALOG_MISC0_CLR:
+        return "MISC0_CLR";
+    case CCM_ANALOG_MISC0_TOG:
+        return "MISC0_TOG";
+    case CCM_ANALOG_MISC2:
+        return "MISC2";
+    case CCM_ANALOG_MISC2_SET:
+        return "MISC2_SET";
+    case CCM_ANALOG_MISC2_CLR:
+        return "MISC2_CLR";
+    case CCM_ANALOG_MISC2_TOG:
+        return "MISC2_TOG";
+    case PMU_REG_1P1:
+        return "PMU_REG_1P1";
+    case PMU_REG_3P0:
+        return "PMU_REG_3P0";
+    case PMU_REG_2P5:
+        return "PMU_REG_2P5";
+    case PMU_REG_CORE:
+        return "PMU_REG_CORE";
+    case PMU_MISC1:
+        return "PMU_MISC1";
+    case PMU_MISC1_SET:
+        return "PMU_MISC1_SET";
+    case PMU_MISC1_CLR:
+        return "PMU_MISC1_CLR";
+    case PMU_MISC1_TOG:
+        return "PMU_MISC1_TOG";
+    case USB_ANALOG_DIGPROG:
+        return "USB_ANALOG_DIGPROG";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+#define CKIH_FREQ 24000000 /* 24MHz crystal input */
+
+static const VMStateDescription vmstate_imx6_ccm = {
+    .name = TYPE_IMX6_CCM,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(ccm, IMX6CCMState, CCM_MAX),
+        VMSTATE_UINT32_ARRAY(analog, IMX6CCMState, CCM_ANALOG_MAX),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint64_t imx6_analog_get_pll2_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 24000000;
+
+    if (EXTRACT(dev->analog[CCM_ANALOG_PLL_SYS], DIV_SELECT)) {
+        freq *= 22;
+    } else {
+        freq *= 20;
+    }
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_pll2_pfd0_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_pll2_clk(dev) * 18
+           / EXTRACT(dev->analog[CCM_ANALOG_PFD_528], PFD0_FRAC);
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_pll2_pfd2_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_pll2_clk(dev) * 18
+           / EXTRACT(dev->analog[CCM_ANALOG_PFD_528], PFD2_FRAC);
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_periph_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    switch (EXTRACT(dev->ccm[CCM_CBCMR], PRE_PERIPH_CLK_SEL)) {
+    case 0:
+        freq = imx6_analog_get_pll2_clk(dev);
+        break;
+    case 1:
+        freq = imx6_analog_get_pll2_pfd2_clk(dev);
+        break;
+    case 2:
+        freq = imx6_analog_get_pll2_pfd0_clk(dev);
+        break;
+    case 3:
+        freq = imx6_analog_get_pll2_pfd2_clk(dev) / 2;
+        break;
+    default:
+        /* We should never get there */
+        g_assert_not_reached();
+        break;
+    }
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_ahb_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_periph_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CBCDR], AHB_PODF));
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_ipg_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_ccm_get_ahb_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CBCDR], IPG_PODF));;
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_per_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_ccm_get_ipg_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CSCMR1], PERCLK_PODF));
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint32_t imx6_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+    IMX6CCMState *s = IMX6_CCM(dev);
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+        freq = imx6_ccm_get_ipg_clk(s);
+        break;
+    case CLK_IPG_HIGH:
+        freq = imx6_ccm_get_per_clk(s);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX6_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx6_ccm_reset(DeviceState *dev)
+{
+    IMX6CCMState *s = IMX6_CCM(dev);
+
+    DPRINTF("\n");
+
+    s->ccm[CCM_CCR] = 0x040116FF;
+    s->ccm[CCM_CCDR] = 0x00000000;
+    s->ccm[CCM_CSR] = 0x00000010;
+    s->ccm[CCM_CCSR] = 0x00000100;
+    s->ccm[CCM_CACRR] = 0x00000000;
+    s->ccm[CCM_CBCDR] = 0x00018D40;
+    s->ccm[CCM_CBCMR] = 0x00022324;
+    s->ccm[CCM_CSCMR1] = 0x00F00000;
+    s->ccm[CCM_CSCMR2] = 0x02B92F06;
+    s->ccm[CCM_CSCDR1] = 0x00490B00;
+    s->ccm[CCM_CS1CDR] = 0x0EC102C1;
+    s->ccm[CCM_CS2CDR] = 0x000736C1;
+    s->ccm[CCM_CDCDR] = 0x33F71F92;
+    s->ccm[CCM_CHSCCDR] = 0x0002A150;
+    s->ccm[CCM_CSCDR2] = 0x0002A150;
+    s->ccm[CCM_CSCDR3] = 0x00014841;
+    s->ccm[CCM_CDHIPR] = 0x00000000;
+    s->ccm[CCM_CTOR] = 0x00000000;
+    s->ccm[CCM_CLPCR] = 0x00000079;
+    s->ccm[CCM_CISR] = 0x00000000;
+    s->ccm[CCM_CIMR] = 0xFFFFFFFF;
+    s->ccm[CCM_CCOSR] = 0x000A0001;
+    s->ccm[CCM_CGPR] = 0x0000FE62;
+    s->ccm[CCM_CCGR0] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR1] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR2] = 0xFC3FFFFF;
+    s->ccm[CCM_CCGR3] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR4] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR5] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR6] = 0xFFFFFFFF;
+    s->ccm[CCM_CMEOR] = 0xFFFFFFFF;
+
+    s->analog[CCM_ANALOG_PLL_ARM] = 0x00013042;
+    s->analog[CCM_ANALOG_PLL_USB1] = 0x00012000;
+    s->analog[CCM_ANALOG_PLL_USB2] = 0x00012000;
+    s->analog[CCM_ANALOG_PLL_SYS] = 0x00013001;
+    s->analog[CCM_ANALOG_PLL_SYS_SS] = 0x00000000;
+    s->analog[CCM_ANALOG_PLL_SYS_NUM] = 0x00000000;
+    s->analog[CCM_ANALOG_PLL_SYS_DENOM] = 0x00000012;
+    s->analog[CCM_ANALOG_PLL_AUDIO] = 0x00011006;
+    s->analog[CCM_ANALOG_PLL_AUDIO_NUM] = 0x05F5E100;
+    s->analog[CCM_ANALOG_PLL_AUDIO_DENOM] = 0x2964619C;
+    s->analog[CCM_ANALOG_PLL_VIDEO] = 0x0001100C;
+    s->analog[CCM_ANALOG_PLL_VIDEO_NUM] = 0x05F5E100;
+    s->analog[CCM_ANALOG_PLL_VIDEO_DENOM] = 0x10A24447;
+    s->analog[CCM_ANALOG_PLL_MLB] = 0x00010000;
+    s->analog[CCM_ANALOG_PLL_ENET] = 0x00011001;
+    s->analog[CCM_ANALOG_PFD_480] = 0x1311100C;
+    s->analog[CCM_ANALOG_PFD_528] = 0x1018101B;
+
+    s->analog[PMU_REG_1P1] = 0x00001073;
+    s->analog[PMU_REG_3P0] = 0x00000F74;
+    s->analog[PMU_REG_2P5] = 0x00005071;
+    s->analog[PMU_REG_CORE] = 0x00402010;
+    s->analog[PMU_MISC0] = 0x04000000;
+    s->analog[PMU_MISC1] = 0x00000000;
+    s->analog[PMU_MISC2] = 0x00272727;
+
+    s->analog[USB_ANALOG_USB1_VBUS_DETECT] = 0x00000004;
+    s->analog[USB_ANALOG_USB1_CHRG_DETECT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_VBUS_DETECT_STAT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_CHRG_DETECT_STAT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_MISC] = 0x00000002;
+    s->analog[USB_ANALOG_USB2_VBUS_DETECT] = 0x00000004;
+    s->analog[USB_ANALOG_USB2_CHRG_DETECT] = 0x00000000;
+    s->analog[USB_ANALOG_USB2_MISC] = 0x00000002;
+    s->analog[USB_ANALOG_DIGPROG] = 0x00000000;
+
+    /* all PLLs need to be locked */
+    s->analog[CCM_ANALOG_PLL_ARM]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_USB1]  |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_USB2]  |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_SYS]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_AUDIO] |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_VIDEO] |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_MLB]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_ENET]  |= CCM_ANALOG_PLL_LOCK;
+}
+
+static uint64_t imx6_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    value = s->ccm[index];
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx6_ccm_reg_name(index), value);
+
+    return (uint64_t)value;
+}
+
+static void imx6_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx6_ccm_reg_name(index),
+            (uint32_t)value);
+
+    /*
+     * We will do a better implementation later. In particular some bits
+     * cannot be written to.
+     */
+    s->ccm[index] = (uint32_t)value;
+}
+
+static uint64_t imx6_analog_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value;
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    switch (index) {
+    case CCM_ANALOG_PLL_ARM_SET:
+    case CCM_ANALOG_PLL_USB1_SET:
+    case CCM_ANALOG_PLL_USB2_SET:
+    case CCM_ANALOG_PLL_SYS_SET:
+    case CCM_ANALOG_PLL_AUDIO_SET:
+    case CCM_ANALOG_PLL_VIDEO_SET:
+    case CCM_ANALOG_PLL_MLB_SET:
+    case CCM_ANALOG_PLL_ENET_SET:
+    case CCM_ANALOG_PFD_480_SET:
+    case CCM_ANALOG_PFD_528_SET:
+    case CCM_ANALOG_MISC0_SET:
+    case PMU_MISC1_SET:
+    case CCM_ANALOG_MISC2_SET:
+    case USB_ANALOG_USB1_VBUS_DETECT_SET:
+    case USB_ANALOG_USB1_CHRG_DETECT_SET:
+    case USB_ANALOG_USB1_MISC_SET:
+    case USB_ANALOG_USB2_VBUS_DETECT_SET:
+    case USB_ANALOG_USB2_CHRG_DETECT_SET:
+    case USB_ANALOG_USB2_MISC_SET:
+        /*
+         * All REG_NAME_SET register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 1];
+        break;
+    case CCM_ANALOG_PLL_ARM_CLR:
+    case CCM_ANALOG_PLL_USB1_CLR:
+    case CCM_ANALOG_PLL_USB2_CLR:
+    case CCM_ANALOG_PLL_SYS_CLR:
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+    case CCM_ANALOG_PLL_MLB_CLR:
+    case CCM_ANALOG_PLL_ENET_CLR:
+    case CCM_ANALOG_PFD_480_CLR:
+    case CCM_ANALOG_PFD_528_CLR:
+    case CCM_ANALOG_MISC0_CLR:
+    case PMU_MISC1_CLR:
+    case CCM_ANALOG_MISC2_CLR:
+    case USB_ANALOG_USB1_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB1_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB1_MISC_CLR:
+    case USB_ANALOG_USB2_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB2_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB2_MISC_CLR:
+        /*
+         * All REG_NAME_CLR register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 2];
+        break;
+    case CCM_ANALOG_PLL_ARM_TOG:
+    case CCM_ANALOG_PLL_USB1_TOG:
+    case CCM_ANALOG_PLL_USB2_TOG:
+    case CCM_ANALOG_PLL_SYS_TOG:
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+    case CCM_ANALOG_PLL_MLB_TOG:
+    case CCM_ANALOG_PLL_ENET_TOG:
+    case CCM_ANALOG_PFD_480_TOG:
+    case CCM_ANALOG_PFD_528_TOG:
+    case CCM_ANALOG_MISC0_TOG:
+    case PMU_MISC1_TOG:
+    case CCM_ANALOG_MISC2_TOG:
+    case USB_ANALOG_USB1_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB1_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB1_MISC_TOG:
+    case USB_ANALOG_USB2_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB2_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB2_MISC_TOG:
+        /*
+         * All REG_NAME_TOG register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 3];
+        break;
+    default:
+        value = s->analog[index];
+        break;
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx6_analog_reg_name(index), value);
+
+    return (uint64_t)value;
+}
+
+static void imx6_analog_write(void *opaque, hwaddr offset, uint64_t value,
+                              unsigned size)
+{
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx6_analog_reg_name(index),
+            (uint32_t)value);
+
+    switch (index) {
+    case CCM_ANALOG_PLL_ARM_SET:
+    case CCM_ANALOG_PLL_USB1_SET:
+    case CCM_ANALOG_PLL_USB2_SET:
+    case CCM_ANALOG_PLL_SYS_SET:
+    case CCM_ANALOG_PLL_AUDIO_SET:
+    case CCM_ANALOG_PLL_VIDEO_SET:
+    case CCM_ANALOG_PLL_MLB_SET:
+    case CCM_ANALOG_PLL_ENET_SET:
+    case CCM_ANALOG_PFD_480_SET:
+    case CCM_ANALOG_PFD_528_SET:
+    case CCM_ANALOG_MISC0_SET:
+    case PMU_MISC1_SET:
+    case CCM_ANALOG_MISC2_SET:
+    case USB_ANALOG_USB1_VBUS_DETECT_SET:
+    case USB_ANALOG_USB1_CHRG_DETECT_SET:
+    case USB_ANALOG_USB1_MISC_SET:
+    case USB_ANALOG_USB2_VBUS_DETECT_SET:
+    case USB_ANALOG_USB2_CHRG_DETECT_SET:
+    case USB_ANALOG_USB2_MISC_SET:
+        /*
+         * All REG_NAME_SET register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, setting bits passed in the value.
+         */
+        s->analog[index - 1] |= value;
+        break;
+    case CCM_ANALOG_PLL_ARM_CLR:
+    case CCM_ANALOG_PLL_USB1_CLR:
+    case CCM_ANALOG_PLL_USB2_CLR:
+    case CCM_ANALOG_PLL_SYS_CLR:
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+    case CCM_ANALOG_PLL_MLB_CLR:
+    case CCM_ANALOG_PLL_ENET_CLR:
+    case CCM_ANALOG_PFD_480_CLR:
+    case CCM_ANALOG_PFD_528_CLR:
+    case CCM_ANALOG_MISC0_CLR:
+    case PMU_MISC1_CLR:
+    case CCM_ANALOG_MISC2_CLR:
+    case USB_ANALOG_USB1_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB1_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB1_MISC_CLR:
+    case USB_ANALOG_USB2_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB2_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB2_MISC_CLR:
+        /*
+         * All REG_NAME_CLR register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, unsetting bits passed in the value.
+         */
+        s->analog[index - 2] &= ~value;
+        break;
+    case CCM_ANALOG_PLL_ARM_TOG:
+    case CCM_ANALOG_PLL_USB1_TOG:
+    case CCM_ANALOG_PLL_USB2_TOG:
+    case CCM_ANALOG_PLL_SYS_TOG:
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+    case CCM_ANALOG_PLL_MLB_TOG:
+    case CCM_ANALOG_PLL_ENET_TOG:
+    case CCM_ANALOG_PFD_480_TOG:
+    case CCM_ANALOG_PFD_528_TOG:
+    case CCM_ANALOG_MISC0_TOG:
+    case PMU_MISC1_TOG:
+    case CCM_ANALOG_MISC2_TOG:
+    case USB_ANALOG_USB1_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB1_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB1_MISC_TOG:
+    case USB_ANALOG_USB2_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB2_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB2_MISC_TOG:
+        /*
+         * All REG_NAME_TOG register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, toggling bits passed in the value.
+         */
+        s->analog[index - 3] ^= value;
+        break;
+    default:
+        /*
+         * We will do a better implementation later. In particular some bits
+         * cannot be written to.
+         */
+        s->analog[index] = value;
+        break;
+    }
+}
+
+static const struct MemoryRegionOps imx6_ccm_ops = {
+    .read = imx6_ccm_read,
+    .write = imx6_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static const struct MemoryRegionOps imx6_analog_ops = {
+    .read = imx6_analog_read,
+    .write = imx6_analog_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx6_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX6CCMState *s = IMX6_CCM(obj);
+
+    /* initialize a container for the all memory range */
+    memory_region_init(&s->container, OBJECT(dev), TYPE_IMX6_CCM, 0x5000);
+
+    /* We initialize an IO memory region for the CCM part */
+    memory_region_init_io(&s->ioccm, OBJECT(dev), &imx6_ccm_ops, s,
+                          TYPE_IMX6_CCM ".ccm", CCM_MAX * sizeof(uint32_t));
+
+    /* Add the CCM as a subregion at offset 0 */
+    memory_region_add_subregion(&s->container, 0, &s->ioccm);
+
+    /* We initialize an IO memory region for the ANALOG part */
+    memory_region_init_io(&s->ioanalog, OBJECT(dev), &imx6_analog_ops, s,
+                          TYPE_IMX6_CCM ".analog",
+                          CCM_ANALOG_MAX * sizeof(uint32_t));
+
+    /* Add the ANALOG as a subregion at offset 0x4000 */
+    memory_region_add_subregion(&s->container, 0x4000, &s->ioanalog);
+
+    sysbus_init_mmio(sd, &s->container);
+}
+
+static void imx6_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx6_ccm_reset;
+    dc->vmsd = &vmstate_imx6_ccm;
+    dc->desc = "i.MX6 Clock Control Module";
+
+    ccm->get_clock_frequency = imx6_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx6_ccm_info = {
+    .name          = TYPE_IMX6_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX6CCMState),
+    .instance_init = imx6_ccm_init,
+    .class_init    = imx6_ccm_class_init,
+};
+
+static void imx6_ccm_register_types(void)
+{
+    type_register_static(&imx6_ccm_info);
+}
+
+type_init(imx6_ccm_register_types)
diff --git a/qemu/hw/misc/imx_ccm.c b/qemu/hw/misc/imx_ccm.c
index 092028863..986d890ca 100644
--- a/qemu/hw/misc/imx_ccm.c
+++ b/qemu/hw/misc/imx_ccm.c
@@ -2,138 +2,51 @@
  * IMX31 Clock Control Module
  *
  * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
- * To get the timer frequencies right, we need to emulate at least part of
- * the CCM.
+ * This is an abstract base class used to get a common interface to
+ * retrieve the CCM frequencies from the various i.MX SOC.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "sysemu/sysemu.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/misc/imx_ccm.h"
 
-#define CKIH_FREQ 26000000 /* 26MHz crystal input */
-#define CKIL_FREQ    32768 /* nominal 32khz clock */
-
-
-//#define DEBUG_CCM 1
-#ifdef DEBUG_CCM
-#define DPRINTF(fmt, args...) \
-do { printf("imx_ccm: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_CCM
+#define DEBUG_IMX_CCM 0
 #endif
 
-static int imx_ccm_post_load(void *opaque, int version_id);
-
-#define TYPE_IMX_CCM "imx_ccm"
-#define IMX_CCM(obj) OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
-
-typedef struct IMXCCMState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-
-    uint32_t ccmr;
-    uint32_t pdr0;
-    uint32_t pdr1;
-    uint32_t mpctl;
-    uint32_t spctl;
-    uint32_t cgr[3];
-    uint32_t pmcr0;
-    uint32_t pmcr1;
-
-    /* Frequencies precalculated on register changes */
-    uint32_t pll_refclk_freq;
-    uint32_t mcu_clk_freq;
-    uint32_t hsp_clk_freq;
-    uint32_t ipg_clk_freq;
-} IMXCCMState;
-
-static const VMStateDescription vmstate_imx_ccm = {
-    .name = "imx-ccm",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT32(ccmr, IMXCCMState),
-        VMSTATE_UINT32(pdr0, IMXCCMState),
-        VMSTATE_UINT32(pdr1, IMXCCMState),
-        VMSTATE_UINT32(mpctl, IMXCCMState),
-        VMSTATE_UINT32(spctl, IMXCCMState),
-        VMSTATE_UINT32_ARRAY(cgr, IMXCCMState, 3),
-        VMSTATE_UINT32(pmcr0, IMXCCMState),
-        VMSTATE_UINT32(pmcr1, IMXCCMState),
-        VMSTATE_UINT32(pll_refclk_freq, IMXCCMState),
-        VMSTATE_END_OF_LIST()
-    },
-    .post_load = imx_ccm_post_load,
-};
-
-/* CCMR */
-#define CCMR_FPME (1<<0)
-#define CCMR_MPE  (1<<3)
-#define CCMR_MDS  (1<<7)
-#define CCMR_FPMF (1<<26)
-#define CCMR_PRCS (3<<1)
-
-/* PDR0 */
-#define PDR0_MCU_PODF_SHIFT (0)
-#define PDR0_MCU_PODF_MASK (0x7)
-#define PDR0_MAX_PODF_SHIFT (3)
-#define PDR0_MAX_PODF_MASK (0x7)
-#define PDR0_IPG_PODF_SHIFT (6)
-#define PDR0_IPG_PODF_MASK (0x3)
-#define PDR0_NFC_PODF_SHIFT (8)
-#define PDR0_NFC_PODF_MASK (0x7)
-#define PDR0_HSP_PODF_SHIFT (11)
-#define PDR0_HSP_PODF_MASK (0x7)
-#define PDR0_PER_PODF_SHIFT (16)
-#define PDR0_PER_PODF_MASK (0x1f)
-#define PDR0_CSI_PODF_SHIFT (23)
-#define PDR0_CSI_PODF_MASK (0x1ff)
-
-#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
-                              & PDR0_##name##_PODF_MASK)
-#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
-                             PDR0_##name##_PODF_SHIFT)
-/* PLL control registers */
-#define PD(v) (((v) >> 26) & 0xf)
-#define MFD(v) (((v) >> 16) & 0x3ff)
-#define MFI(v) (((v) >> 10) & 0xf);
-#define MFN(v) ((v) & 0x3ff)
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
-#define PLL_PD(x)               (((x) & 0xf) << 26)
-#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
-#define PLL_MFI(x)              (((x) & 0xf) << 10)
-#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
 
-uint32_t imx_clock_frequency(DeviceState *dev, IMXClk clock)
+uint32_t imx_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
 {
-    IMXCCMState *s = IMX_CCM(dev);
+    uint32_t freq = 0;
+    IMXCCMClass *klass = IMX_GET_CLASS(dev);
 
-    switch (clock) {
-    case NOCLK:
-        return 0;
-    case MCU:
-        return s->mcu_clk_freq;
-    case HSP:
-        return s->hsp_clk_freq;
-    case IPG:
-        return s->ipg_clk_freq;
-    case CLK_32k:
-        return CKIL_FREQ;
+    if (klass->get_clock_frequency) {
+        freq = klass->get_clock_frequency(dev, clock);
     }
-    return 0;
+
+    DPRINTF("(clock = %d) = %d\n", clock, freq);
+
+    return freq;
 }
 
 /*
  * Calculate PLL output frequency
  */
-static uint32_t calc_pll(uint32_t pllreg, uint32_t base_freq)
+uint32_t imx_ccm_calc_pll(uint32_t pllreg, uint32_t base_freq)
 {
+    int32_t freq;
     int32_t mfn = MFN(pllreg);  /* Numerator */
     uint32_t mfi = MFI(pllreg); /* Integer part */
     uint32_t mfd = 1 + MFD(pllreg); /* Denominator */
@@ -142,180 +55,26 @@ static uint32_t calc_pll(uint32_t pllreg, uint32_t base_freq)
     if (mfi < 5) {
         mfi = 5;
     }
+
     /* mfn is 10-bit signed twos-complement */
     mfn <<= 32 - 10;
     mfn >>= 32 - 10;
 
-    return ((2 * (base_freq >> 10) * (mfi * mfd + mfn)) /
+    freq = ((2 * (base_freq >> 10) * (mfi * mfd + mfn)) /
             (mfd * pd)) << 10;
-}
-
-static void update_clocks(IMXCCMState *s)
-{
-    /*
-     * If we ever emulate more clocks, this should switch to a data-driven
-     * approach
-     */
-
-    if ((s->ccmr & CCMR_PRCS) == 2) {
-        s->pll_refclk_freq = CKIL_FREQ * 1024;
-    } else {
-        s->pll_refclk_freq = CKIH_FREQ;
-    }
-
-    /* ipg_clk_arm aka MCU clock */
-    if ((s->ccmr & CCMR_MDS) || !(s->ccmr & CCMR_MPE)) {
-        s->mcu_clk_freq = s->pll_refclk_freq;
-    } else {
-        s->mcu_clk_freq = calc_pll(s->mpctl, s->pll_refclk_freq);
-    }
-
-    /* High-speed clock */
-    s->hsp_clk_freq = s->mcu_clk_freq / (1 + EXTRACT(s->pdr0, HSP));
-    s->ipg_clk_freq = s->hsp_clk_freq / (1 + EXTRACT(s->pdr0, IPG));
 
-    DPRINTF("Clocks: mcu %uMHz, HSP %uMHz, IPG %uHz\n",
-            s->mcu_clk_freq / 1000000,
-            s->hsp_clk_freq / 1000000,
-            s->ipg_clk_freq);
-}
-
-static void imx_ccm_reset(DeviceState *dev)
-{
-    IMXCCMState *s = IMX_CCM(dev);
-
-    s->ccmr = 0x074b0b7b;
-    s->pdr0 = 0xff870b48;
-    s->pdr1 = 0x49fcfe7f;
-    s->mpctl = PLL_PD(1) | PLL_MFD(0) | PLL_MFI(6) | PLL_MFN(0);
-    s->cgr[0] = s->cgr[1] = s->cgr[2] = 0xffffffff;
-    s->spctl = PLL_PD(1) | PLL_MFD(4) | PLL_MFI(0xc) | PLL_MFN(1);
-    s->pmcr0 = 0x80209828;
-
-    update_clocks(s);
-}
-
-static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
-                                unsigned size)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    DPRINTF("read(offset=%x)", offset >> 2);
-    switch (offset >> 2) {
-    case 0: /* CCMR */
-        DPRINTF(" ccmr = 0x%x\n", s->ccmr);
-        return s->ccmr;
-    case 1:
-        DPRINTF(" pdr0 = 0x%x\n", s->pdr0);
-        return s->pdr0;
-    case 2:
-        DPRINTF(" pdr1 = 0x%x\n", s->pdr1);
-        return s->pdr1;
-    case 4:
-        DPRINTF(" mpctl = 0x%x\n", s->mpctl);
-        return s->mpctl;
-    case 6:
-        DPRINTF(" spctl = 0x%x\n", s->spctl);
-        return s->spctl;
-    case 8:
-        DPRINTF(" cgr0 = 0x%x\n", s->cgr[0]);
-        return s->cgr[0];
-    case 9:
-        DPRINTF(" cgr1 = 0x%x\n", s->cgr[1]);
-        return s->cgr[1];
-    case 10:
-        DPRINTF(" cgr2 = 0x%x\n", s->cgr[2]);
-        return s->cgr[2];
-    case 18: /* LTR1 */
-        return 0x00004040;
-    case 23:
-        DPRINTF(" pcmr0 = 0x%x\n", s->pmcr0);
-        return s->pmcr0;
-    }
-    DPRINTF(" return 0\n");
-    return 0;
-}
-
-static void imx_ccm_write(void *opaque, hwaddr offset,
-                          uint64_t value, unsigned size)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    DPRINTF("write(offset=%x, value = %x)\n",
-            offset >> 2, (unsigned int)value);
-    switch (offset >> 2) {
-    case 0:
-        s->ccmr = CCMR_FPMF | (value & 0x3b6fdfff);
-        break;
-    case 1:
-        s->pdr0 = value & 0xff9f3fff;
-        break;
-    case 2:
-        s->pdr1 = value;
-        break;
-    case 4:
-        s->mpctl = value & 0xbfff3fff;
-        break;
-    case 6:
-        s->spctl = value & 0xbfff3fff;
-        break;
-    case 8:
-        s->cgr[0] = value;
-        return;
-    case 9:
-        s->cgr[1] = value;
-        return;
-    case 10:
-        s->cgr[2] = value;
-        return;
-
-    default:
-        return;
-    }
-    update_clocks(s);
-}
-
-static const struct MemoryRegionOps imx_ccm_ops = {
-    .read = imx_ccm_read,
-    .write = imx_ccm_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static int imx_ccm_init(SysBusDevice *dev)
-{
-    IMXCCMState *s = IMX_CCM(dev);
-
-    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_ccm_ops, s,
-                          "imx_ccm", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
-}
-
-static int imx_ccm_post_load(void *opaque, int version_id)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    update_clocks(s);
-    return 0;
-}
-
-static void imx_ccm_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DPRINTF("(pllreg = 0x%08x, base_freq = %d) = %d\n", pllreg, base_freq,
+            freq);
 
-    sbc->init = imx_ccm_init;
-    dc->reset = imx_ccm_reset;
-    dc->vmsd = &vmstate_imx_ccm;
-    dc->desc = "i.MX Clock Control Module";
+    return freq;
 }
 
 static const TypeInfo imx_ccm_info = {
-    .name = TYPE_IMX_CCM,
-    .parent = TYPE_SYS_BUS_DEVICE,
+    .name          = TYPE_IMX_CCM,
+    .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXCCMState),
-    .class_init = imx_ccm_class_init,
+    .class_size    = sizeof(IMXCCMClass),
+    .abstract      = true,
 };
 
 static void imx_ccm_register_types(void)
diff --git a/qemu/hw/misc/ivshmem.c b/qemu/hw/misc/ivshmem.c
index cc76989a3..e40f23bfc 100644
--- a/qemu/hw/misc/ivshmem.c
+++ b/qemu/hw/misc/ivshmem.c
@@ -16,39 +16,57 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "sysemu/kvm.h"
 #include "migration/migration.h"
 #include "qemu/error-report.h"
 #include "qemu/event_notifier.h"
-#include "qemu/fifo8.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/char.h"
+#include "sysemu/hostmem.h"
+#include "sysemu/qtest.h"
+#include "qapi/visitor.h"
+#include "exec/ram_addr.h"
+
+#include "hw/misc/ivshmem.h"
 
 #include <sys/mman.h>
-#include <sys/types.h>
-#include <limits.h>
 
 #define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
 #define PCI_DEVICE_ID_IVSHMEM   0x1110
 
+#define IVSHMEM_MAX_PEERS UINT16_MAX
 #define IVSHMEM_IOEVENTFD   0
 #define IVSHMEM_MSI     1
 
-#define IVSHMEM_PEER    0
-#define IVSHMEM_MASTER  1
-
 #define IVSHMEM_REG_BAR_SIZE 0x100
 
-//#define DEBUG_IVSHMEM
-#ifdef DEBUG_IVSHMEM
-#define IVSHMEM_DPRINTF(fmt, ...)        \
-    do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define IVSHMEM_DPRINTF(fmt, ...)
-#endif
+#define IVSHMEM_DEBUG 0
+#define IVSHMEM_DPRINTF(fmt, ...)                       \
+    do {                                                \
+        if (IVSHMEM_DEBUG) {                            \
+            printf("IVSHMEM: " fmt, ## __VA_ARGS__);    \
+        }                                               \
+    } while (0)
+
+#define TYPE_IVSHMEM_COMMON "ivshmem-common"
+#define IVSHMEM_COMMON(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON)
+
+#define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
+#define IVSHMEM_PLAIN(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN)
+
+#define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
+#define IVSHMEM_DOORBELL(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL)
 
 #define TYPE_IVSHMEM "ivshmem"
 #define IVSHMEM(obj) \
@@ -59,51 +77,50 @@ typedef struct Peer {
     EventNotifier *eventfds;
 } Peer;
 
-typedef struct EventfdEntry {
+typedef struct MSIVector {
     PCIDevice *pdev;
-    int vector;
-} EventfdEntry;
+    int virq;
+} MSIVector;
 
 typedef struct IVShmemState {
     /*< private >*/
     PCIDevice parent_obj;
     /*< public >*/
 
+    uint32_t features;
+
+    /* exactly one of these two may be set */
+    HostMemoryBackend *hostmem; /* with interrupts */
+    CharDriverState *server_chr; /* without interrupts */
+
+    /* registers */
     uint32_t intrmask;
     uint32_t intrstatus;
-    uint32_t doorbell;
-
-    CharDriverState **eventfd_chr;
-    CharDriverState *server_chr;
-    Fifo8 incoming_fifo;
-    MemoryRegion ivshmem_mmio;
+    int vm_id;
 
-    /* We might need to register the BAR before we actually have the memory.
-     * So prepare a container MemoryRegion for the BAR immediately and
-     * add a subregion when we have the memory.
-     */
-    MemoryRegion bar;
-    MemoryRegion ivshmem;
-    uint64_t ivshmem_size; /* size of shared memory region */
-    uint32_t ivshmem_attr;
-    uint32_t ivshmem_64bit;
-    int shm_fd; /* shared memory file descriptor */
+    /* BARs */
+    MemoryRegion ivshmem_mmio;  /* BAR 0 (registers) */
+    MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */
+    MemoryRegion server_bar2;   /* used with server_chr */
 
+    /* interrupt support */
     Peer *peers;
-    int nb_peers; /* how many guests we have space for */
-    int max_peer; /* maximum numbered peer */
-
-    int vm_id;
+    int nb_peers;               /* space in @peers[] */
     uint32_t vectors;
-    uint32_t features;
-    EventfdEntry *eventfd_table;
+    MSIVector *msi_vectors;
+    uint64_t msg_buf;           /* buffer for receiving server messages */
+    int msg_buffered_bytes;     /* #bytes in @msg_buf */
 
+    /* migration stuff */
+    OnOffAuto master;
     Error *migration_blocker;
 
-    char * shmobj;
-    char * sizearg;
-    char * role;
-    int role_val;   /* scalar to avoid multiple string comparisons */
+    /* legacy cruft */
+    char *role;
+    char *shmobj;
+    char *sizearg;
+    size_t legacy_size;
+    uint32_t not_legacy_32bit;
 } IVShmemState;
 
 /* registers for the Inter-VM shared memory device */
@@ -119,16 +136,34 @@ static inline uint32_t ivshmem_has_feature(IVShmemState *ivs,
     return (ivs->features & (1 << feature));
 }
 
-static inline bool is_power_of_two(uint64_t x) {
-    return (x & (x - 1)) == 0;
+static inline bool ivshmem_is_master(IVShmemState *s)
+{
+    assert(s->master != ON_OFF_AUTO_AUTO);
+    return s->master == ON_OFF_AUTO_ON;
 }
 
-/* accessing registers - based on rtl8139 */
-static void ivshmem_update_irq(IVShmemState *s, int val)
+static void ivshmem_update_irq(IVShmemState *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
-    int isr;
-    isr = (s->intrstatus & s->intrmask) & 0xffffffff;
+    uint32_t isr = s->intrstatus & s->intrmask;
+
+    /*
+     * Do nothing unless the device actually uses INTx.  Here's how
+     * the device variants signal interrupts, what they put in PCI
+     * config space:
+     * Device variant    Interrupt  Interrupt Pin  MSI-X cap.
+     * ivshmem-plain         none            0         no
+     * ivshmem-doorbell     MSI-X            1        yes(1)
+     * ivshmem,msi=off       INTx            1         no
+     * ivshmem,msi=on       MSI-X            1(2)     yes(1)
+     * (1) if guest enabled MSI-X
+     * (2) the device lies
+     * Leads to the condition for doing nothing:
+     */
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)
+        || !d->config[PCI_INTERRUPT_PIN]) {
+        return;
+    }
 
     /* don't print ISR resets */
     if (isr) {
@@ -136,7 +171,7 @@ static void ivshmem_update_irq(IVShmemState *s, int val)
                         isr ? 1 : 0, s->intrstatus, s->intrmask);
     }
 
-    pci_set_irq(d, (isr != 0));
+    pci_set_irq(d, isr != 0);
 }
 
 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
@@ -144,8 +179,7 @@ static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
     IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val);
 
     s->intrmask = val;
-
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
@@ -153,7 +187,6 @@ static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
     uint32_t ret = s->intrmask;
 
     IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret);
-
     return ret;
 }
 
@@ -162,8 +195,7 @@ static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
     IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val);
 
     s->intrstatus = val;
-
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
@@ -172,9 +204,7 @@ static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
 
     /* reading ISR clears all interrupts */
     s->intrstatus = 0;
-
-    ivshmem_update_irq(s, 0);
-
+    ivshmem_update_irq(s);
     return ret;
 }
 
@@ -201,7 +231,7 @@ static void ivshmem_io_write(void *opaque, hwaddr addr,
 
         case DOORBELL:
             /* check that dest VM ID is reasonable */
-            if (dest > s->max_peer) {
+            if (dest >= s->nb_peers) {
                 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
                 break;
             }
@@ -210,10 +240,13 @@ static void ivshmem_io_write(void *opaque, hwaddr addr,
             if (vector < s->peers[dest].nb_eventfds) {
                 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
                 event_notifier_set(&s->peers[dest].eventfds[vector]);
+            } else {
+                IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
+                                vector, dest);
             }
             break;
         default:
-            IVSHMEM_DPRINTF("Invalid VM Doorbell VM %d\n", dest);
+            IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr);
     }
 }
 
@@ -235,12 +268,7 @@ static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
             break;
 
         case IVPOSITION:
-            /* return my VM ID if the memory is mapped */
-            if (s->shm_fd > 0) {
-                ret = s->vm_id;
-            } else {
-                ret = -1;
-            }
+            ret = s->vm_id;
             break;
 
         default:
@@ -261,105 +289,95 @@ static const MemoryRegionOps ivshmem_mmio_ops = {
     },
 };
 
-static void ivshmem_receive(void *opaque, const uint8_t *buf, int size)
+static void ivshmem_vector_notify(void *opaque)
 {
-    IVShmemState *s = opaque;
-
-    ivshmem_IntrStatus_write(s, *buf);
+    MSIVector *entry = opaque;
+    PCIDevice *pdev = entry->pdev;
+    IVShmemState *s = IVSHMEM_COMMON(pdev);
+    int vector = entry - s->msi_vectors;
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
 
-    IVSHMEM_DPRINTF("ivshmem_receive 0x%02x\n", *buf);
-}
+    if (!event_notifier_test_and_clear(n)) {
+        return;
+    }
 
-static int ivshmem_can_receive(void * opaque)
-{
-    return 8;
+    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        if (msix_enabled(pdev)) {
+            msix_notify(pdev, vector);
+        }
+    } else {
+        ivshmem_IntrStatus_write(s, 1);
+    }
 }
 
-static void ivshmem_event(void *opaque, int event)
+static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
+                                 MSIMessage msg)
 {
-    IVSHMEM_DPRINTF("ivshmem_event %d\n", event);
-}
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    MSIVector *v = &s->msi_vectors[vector];
+    int ret;
 
-static void fake_irqfd(void *opaque, const uint8_t *buf, int size) {
+    IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
 
-    EventfdEntry *entry = opaque;
-    PCIDevice *pdev = entry->pdev;
+    ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
+    if (ret < 0) {
+        return ret;
+    }
 
-    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, entry->vector);
-    msix_notify(pdev, entry->vector);
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
 }
 
-static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *n,
-                                                  int vector)
+static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
 {
-    /* create a event character device based on the passed eventfd */
-    IVShmemState *s = opaque;
-    CharDriverState * chr;
-    int eventfd = event_notifier_get_fd(n);
-
-    chr = qemu_chr_open_eventfd(eventfd);
-
-    if (chr == NULL) {
-        error_report("creating eventfd for eventfd %d failed", eventfd);
-        exit(1);
-    }
-    qemu_chr_fe_claim_no_fail(chr);
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    int ret;
 
-    /* if MSI is supported we need multiple interrupts */
-    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        s->eventfd_table[vector].pdev = PCI_DEVICE(s);
-        s->eventfd_table[vector].vector = vector;
+    IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
 
-        qemu_chr_add_handlers(chr, ivshmem_can_receive, fake_irqfd,
-                      ivshmem_event, &s->eventfd_table[vector]);
-    } else {
-        qemu_chr_add_handlers(chr, ivshmem_can_receive, ivshmem_receive,
-                      ivshmem_event, s);
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n,
+                                                s->msi_vectors[vector].virq);
+    if (ret != 0) {
+        error_report("remove_irqfd_notifier_gsi failed");
     }
+}
 
-    return chr;
+static void ivshmem_vector_poll(PCIDevice *dev,
+                                unsigned int vector_start,
+                                unsigned int vector_end)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    unsigned int vector;
 
-}
+    IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
 
-static int check_shm_size(IVShmemState *s, int fd) {
-    /* check that the guest isn't going to try and map more memory than the
-     * the object has allocated return -1 to indicate error */
+    vector_end = MIN(vector_end, s->vectors);
 
-    struct stat buf;
+    for (vector = vector_start; vector < vector_end; vector++) {
+        EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
 
-    if (fstat(fd, &buf) < 0) {
-        error_report("exiting: fstat on fd %d failed: %s",
-                     fd, strerror(errno));
-        return -1;
-    }
+        if (!msix_is_masked(dev, vector)) {
+            continue;
+        }
 
-    if (s->ivshmem_size > buf.st_size) {
-        error_report("Requested memory size greater"
-                     " than shared object size (%" PRIu64 " > %" PRIu64")",
-                     s->ivshmem_size, (uint64_t)buf.st_size);
-        return -1;
-    } else {
-        return 0;
+        if (event_notifier_test_and_clear(notifier)) {
+            msix_set_pending(dev, vector);
+        }
     }
 }
 
-/* create the shared memory BAR when we are not using the server, so we can
- * create the BAR and map the memory immediately */
-static void create_shared_memory_BAR(IVShmemState *s, int fd) {
-
-    void * ptr;
-
-    s->shm_fd = fd;
-
-    ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+static void watch_vector_notifier(IVShmemState *s, EventNotifier *n,
+                                 int vector)
+{
+    int eventfd = event_notifier_get_fd(n);
 
-    memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2",
-                               s->ivshmem_size, ptr);
-    vmstate_register_ram(&s->ivshmem, DEVICE(s));
-    memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
+    assert(!s->msi_vectors[vector].pdev);
+    s->msi_vectors[vector].pdev = PCI_DEVICE(s);
 
-    /* region for shared memory */
-    pci_register_bar(PCI_DEVICE(s), 2, s->ivshmem_attr, &s->bar);
+    qemu_set_fd_handler(eventfd, ivshmem_vector_notify,
+                        NULL, &s->msi_vectors[vector]);
 }
 
 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
@@ -382,25 +400,22 @@ static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i)
                               &s->peers[posn].eventfds[i]);
 }
 
-static void close_guest_eventfds(IVShmemState *s, int posn)
+static void close_peer_eventfds(IVShmemState *s, int posn)
 {
-    int i, guest_curr_max;
+    int i, n;
 
-    if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
-        return;
-    }
-    if (posn < 0 || posn >= s->nb_peers) {
-        return;
-    }
+    assert(posn >= 0 && posn < s->nb_peers);
+    n = s->peers[posn].nb_eventfds;
 
-    guest_curr_max = s->peers[posn].nb_eventfds;
-
-    memory_region_transaction_begin();
-    for (i = 0; i < guest_curr_max; i++) {
-        ivshmem_del_eventfd(s, posn, i);
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
+        memory_region_transaction_begin();
+        for (i = 0; i < n; i++) {
+            ivshmem_del_eventfd(s, posn, i);
+        }
+        memory_region_transaction_commit();
     }
-    memory_region_transaction_commit();
-    for (i = 0; i < guest_curr_max; i++) {
+
+    for (i = 0; i < n; i++) {
         event_notifier_cleanup(&s->peers[posn].eventfds[i]);
     }
 
@@ -408,187 +423,320 @@ static void close_guest_eventfds(IVShmemState *s, int posn)
     s->peers[posn].nb_eventfds = 0;
 }
 
-/* this function increase the dynamic storage need to store data about other
- * guests */
-static int increase_dynamic_storage(IVShmemState *s, int new_min_size)
+static void resize_peers(IVShmemState *s, int nb_peers)
 {
+    int old_nb_peers = s->nb_peers;
+    int i;
 
-    int j, old_nb_alloc;
-
-    /* check for integer overflow */
-    if (new_min_size >= INT_MAX / sizeof(Peer) - 1 || new_min_size <= 0) {
-        return -1;
-    }
+    assert(nb_peers > old_nb_peers);
+    IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers);
 
-    old_nb_alloc = s->nb_peers;
+    s->peers = g_realloc(s->peers, nb_peers * sizeof(Peer));
+    s->nb_peers = nb_peers;
 
-    if (new_min_size >= s->nb_peers) {
-        /* +1 because #new_min_size is used as last array index */
-        s->nb_peers = new_min_size + 1;
-    } else {
-        return 0;
+    for (i = old_nb_peers; i < nb_peers; i++) {
+        s->peers[i].eventfds = g_new0(EventNotifier, s->vectors);
+        s->peers[i].nb_eventfds = 0;
     }
+}
 
-    IVSHMEM_DPRINTF("bumping storage to %d guests\n", s->nb_peers);
-    s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer));
+static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector,
+                                     Error **errp)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    MSIMessage msg = msix_get_message(pdev, vector);
+    int ret;
 
-    /* zero out new pointers */
-    for (j = old_nb_alloc; j < s->nb_peers; j++) {
-        s->peers[j].eventfds = NULL;
-        s->peers[j].nb_eventfds = 0;
+    IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
+    assert(!s->msi_vectors[vector].pdev);
+
+    ret = kvm_irqchip_add_msi_route(kvm_state, msg, pdev);
+    if (ret < 0) {
+        error_setg(errp, "kvm_irqchip_add_msi_route failed");
+        return;
     }
 
-    return 0;
+    s->msi_vectors[vector].virq = ret;
+    s->msi_vectors[vector].pdev = pdev;
 }
 
-static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
+static void setup_interrupt(IVShmemState *s, int vector, Error **errp)
 {
-    IVShmemState *s = opaque;
-    int incoming_fd, tmp_fd;
-    int guest_max_eventfd;
-    long incoming_posn;
-
-    if (fifo8_is_empty(&s->incoming_fifo) && size == sizeof(incoming_posn)) {
-        memcpy(&incoming_posn, buf, size);
-    } else {
-        const uint8_t *p;
-        uint32_t num;
-
-        IVSHMEM_DPRINTF("short read of %d bytes\n", size);
-        num = MAX(size, sizeof(long) - fifo8_num_used(&s->incoming_fifo));
-        fifo8_push_all(&s->incoming_fifo, buf, num);
-        if (fifo8_num_used(&s->incoming_fifo) < sizeof(incoming_posn)) {
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
+        ivshmem_has_feature(s, IVSHMEM_MSI);
+    PCIDevice *pdev = PCI_DEVICE(s);
+    Error *err = NULL;
+
+    IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
+
+    if (!with_irqfd) {
+        IVSHMEM_DPRINTF("with eventfd\n");
+        watch_vector_notifier(s, n, vector);
+    } else if (msix_enabled(pdev)) {
+        IVSHMEM_DPRINTF("with irqfd\n");
+        ivshmem_add_kvm_msi_virq(s, vector, &err);
+        if (err) {
+            error_propagate(errp, err);
             return;
         }
-        size -= num;
-        buf += num;
-        p = fifo8_pop_buf(&s->incoming_fifo, sizeof(incoming_posn), &num);
-        g_assert(num == sizeof(incoming_posn));
-        memcpy(&incoming_posn, p, sizeof(incoming_posn));
-        if (size > 0) {
-            fifo8_push_all(&s->incoming_fifo, buf, size);
+
+        if (!msix_is_masked(pdev, vector)) {
+            kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
+                                               s->msi_vectors[vector].virq);
+            /* TODO handle error */
         }
+    } else {
+        /* it will be delayed until msix is enabled, in write_config */
+        IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
     }
+}
 
-    if (incoming_posn < -1) {
-        IVSHMEM_DPRINTF("invalid incoming_posn %ld\n", incoming_posn);
+static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)
+{
+    struct stat buf;
+    size_t size;
+    void *ptr;
+
+    if (s->ivshmem_bar2) {
+        error_setg(errp, "server sent unexpected shared memory message");
+        close(fd);
+        return;
+    }
+
+    if (fstat(fd, &buf) < 0) {
+        error_setg_errno(errp, errno,
+            "can't determine size of shared memory sent by server");
+        close(fd);
         return;
     }
 
-    /* pick off s->server_chr->msgfd and store it, posn should accompany msg */
-    tmp_fd = qemu_chr_fe_get_msgfd(s->server_chr);
-    IVSHMEM_DPRINTF("posn is %ld, fd is %d\n", incoming_posn, tmp_fd);
+    size = buf.st_size;
 
-    /* make sure we have enough space for this guest */
-    if (incoming_posn >= s->nb_peers) {
-        if (increase_dynamic_storage(s, incoming_posn) < 0) {
-            error_report("increase_dynamic_storage() failed");
-            if (tmp_fd != -1) {
-                close(tmp_fd);
-            }
+    /* Legacy cruft */
+    if (s->legacy_size != SIZE_MAX) {
+        if (size < s->legacy_size) {
+            error_setg(errp, "server sent only %zd bytes of shared memory",
+                       (size_t)buf.st_size);
+            close(fd);
             return;
         }
+        size = s->legacy_size;
     }
 
-    if (tmp_fd == -1) {
-        /* if posn is positive and unseen before then this is our posn*/
-        if ((incoming_posn >= 0) &&
-                            (s->peers[incoming_posn].eventfds == NULL)) {
-            /* receive our posn */
-            s->vm_id = incoming_posn;
-            return;
-        } else {
-            /* otherwise an fd == -1 means an existing guest has gone away */
-            IVSHMEM_DPRINTF("posn %ld has gone away\n", incoming_posn);
-            close_guest_eventfds(s, incoming_posn);
-            return;
-        }
+    /* mmap the region and map into the BAR2 */
+    ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (ptr == MAP_FAILED) {
+        error_setg_errno(errp, errno, "Failed to mmap shared memory");
+        close(fd);
+        return;
     }
+    memory_region_init_ram_ptr(&s->server_bar2, OBJECT(s),
+                               "ivshmem.bar2", size, ptr);
+    qemu_set_ram_fd(memory_region_get_ram_addr(&s->server_bar2), fd);
+    s->ivshmem_bar2 = &s->server_bar2;
+}
 
-    /* because of the implementation of get_msgfd, we need a dup */
-    incoming_fd = dup(tmp_fd);
+static void process_msg_disconnect(IVShmemState *s, uint16_t posn,
+                                   Error **errp)
+{
+    IVSHMEM_DPRINTF("posn %d has gone away\n", posn);
+    if (posn >= s->nb_peers || posn == s->vm_id) {
+        error_setg(errp, "invalid peer %d", posn);
+        return;
+    }
+    close_peer_eventfds(s, posn);
+}
 
-    if (incoming_fd == -1) {
-        error_report("could not allocate file descriptor %s", strerror(errno));
-        close(tmp_fd);
+static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd,
+                                Error **errp)
+{
+    Peer *peer = &s->peers[posn];
+    int vector;
+
+    /*
+     * The N-th connect message for this peer comes with the file
+     * descriptor for vector N-1.  Count messages to find the vector.
+     */
+    if (peer->nb_eventfds >= s->vectors) {
+        error_setg(errp, "Too many eventfd received, device has %d vectors",
+                   s->vectors);
+        close(fd);
         return;
     }
+    vector = peer->nb_eventfds++;
+
+    IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd);
+    event_notifier_init_fd(&peer->eventfds[vector], fd);
+    fcntl_setfl(fd, O_NONBLOCK); /* msix/irqfd poll non block */
+
+    if (posn == s->vm_id) {
+        setup_interrupt(s, vector, errp);
+        /* TODO do we need to handle the error? */
+    }
 
-    /* if the position is -1, then it's shared memory region fd */
-    if (incoming_posn == -1) {
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
+        ivshmem_add_eventfd(s, posn, vector);
+    }
+}
 
-        void * map_ptr;
+static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp)
+{
+    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
 
-        s->max_peer = 0;
+    if (msg < -1 || msg > IVSHMEM_MAX_PEERS) {
+        error_setg(errp, "server sent invalid message %" PRId64, msg);
+        close(fd);
+        return;
+    }
 
-        if (check_shm_size(s, incoming_fd) == -1) {
-            exit(1);
-        }
+    if (msg == -1) {
+        process_msg_shmem(s, fd, errp);
+        return;
+    }
 
-        /* mmap the region and map into the BAR2 */
-        map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED,
-                                                            incoming_fd, 0);
-        memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s),
-                                   "ivshmem.bar2", s->ivshmem_size, map_ptr);
-        vmstate_register_ram(&s->ivshmem, DEVICE(s));
+    if (msg >= s->nb_peers) {
+        resize_peers(s, msg + 1);
+    }
 
-        IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n",
-                        map_ptr, s->ivshmem_size);
+    if (fd >= 0) {
+        process_msg_connect(s, msg, fd, errp);
+    } else {
+        process_msg_disconnect(s, msg, errp);
+    }
+}
 
-        memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
+static int ivshmem_can_receive(void *opaque)
+{
+    IVShmemState *s = opaque;
 
-        /* only store the fd if it is successfully mapped */
-        s->shm_fd = incoming_fd;
+    assert(s->msg_buffered_bytes < sizeof(s->msg_buf));
+    return sizeof(s->msg_buf) - s->msg_buffered_bytes;
+}
 
+static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
+{
+    IVShmemState *s = opaque;
+    Error *err = NULL;
+    int fd;
+    int64_t msg;
+
+    assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf));
+    memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size);
+    s->msg_buffered_bytes += size;
+    if (s->msg_buffered_bytes < sizeof(s->msg_buf)) {
         return;
     }
+    msg = le64_to_cpu(s->msg_buf);
+    s->msg_buffered_bytes = 0;
 
-    /* each guest has an array of eventfds, and we keep track of how many
-     * guests for each VM */
-    guest_max_eventfd = s->peers[incoming_posn].nb_eventfds;
+    fd = qemu_chr_fe_get_msgfd(s->server_chr);
+    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
 
-    if (guest_max_eventfd == 0) {
-        /* one eventfd per MSI vector */
-        s->peers[incoming_posn].eventfds = g_new(EventNotifier, s->vectors);
+    process_msg(s, msg, fd, &err);
+    if (err) {
+        error_report_err(err);
     }
+}
 
-    /* this is an eventfd for a particular guest VM */
-    IVSHMEM_DPRINTF("eventfds[%ld][%d] = %d\n", incoming_posn,
-                    guest_max_eventfd, incoming_fd);
-    event_notifier_init_fd(&s->peers[incoming_posn].eventfds[guest_max_eventfd],
-                           incoming_fd);
+static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp)
+{
+    int64_t msg;
+    int n, ret;
+
+    n = 0;
+    do {
+        ret = qemu_chr_fe_read_all(s->server_chr, (uint8_t *)&msg + n,
+                                 sizeof(msg) - n);
+        if (ret < 0 && ret != -EINTR) {
+            error_setg_errno(errp, -ret, "read from server failed");
+            return INT64_MIN;
+        }
+        n += ret;
+    } while (n < sizeof(msg));
 
-    /* increment count for particular guest */
-    s->peers[incoming_posn].nb_eventfds++;
+    *pfd = qemu_chr_fe_get_msgfd(s->server_chr);
+    return msg;
+}
 
-    /* keep track of the maximum VM ID */
-    if (incoming_posn > s->max_peer) {
-        s->max_peer = incoming_posn;
-    }
+static void ivshmem_recv_setup(IVShmemState *s, Error **errp)
+{
+    Error *err = NULL;
+    int64_t msg;
+    int fd;
 
-    if (incoming_posn == s->vm_id) {
-        s->eventfd_chr[guest_max_eventfd] = create_eventfd_chr_device(s,
-                   &s->peers[s->vm_id].eventfds[guest_max_eventfd],
-                   guest_max_eventfd);
+    msg = ivshmem_recv_msg(s, &fd, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    if (msg != IVSHMEM_PROTOCOL_VERSION) {
+        error_setg(errp, "server sent version %" PRId64 ", expecting %d",
+                   msg, IVSHMEM_PROTOCOL_VERSION);
+        return;
+    }
+    if (fd != -1) {
+        error_setg(errp, "server sent invalid version message");
+        return;
     }
 
-    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
-        ivshmem_add_eventfd(s, incoming_posn, guest_max_eventfd);
+    /*
+     * ivshmem-server sends the remaining initial messages in a fixed
+     * order, but the device has always accepted them in any order.
+     * Stay as compatible as practical, just in case people use
+     * servers that behave differently.
+     */
+
+    /*
+     * ivshmem_device_spec.txt has always required the ID message
+     * right here, and ivshmem-server has always complied.  However,
+     * older versions of the device accepted it out of order, but
+     * broke when an interrupt setup message arrived before it.
+     */
+    msg = ivshmem_recv_msg(s, &fd, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) {
+        error_setg(errp, "server sent invalid ID message");
+        return;
     }
+    s->vm_id = msg;
+
+    /*
+     * Receive more messages until we got shared memory.
+     */
+    do {
+        msg = ivshmem_recv_msg(s, &fd, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        process_msg(s, msg, fd, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    } while (msg != -1);
+
+    /*
+     * This function must either map the shared memory or fail.  The
+     * loop above ensures that: it terminates normally only after it
+     * successfully processed the server's shared memory message.
+     * Assert that actually mapped the shared memory:
+     */
+    assert(s->ivshmem_bar2);
 }
 
 /* Select the MSI-X vectors used by device.
  * ivshmem maps events to vectors statically, so
  * we just enable all vectors on init and after reset. */
-static void ivshmem_use_msix(IVShmemState * s)
+static void ivshmem_msix_vector_use(IVShmemState *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
     int i;
 
-    if (!msix_present(d)) {
-        return;
-    }
-
     for (i = 0; i < s->vectors; i++) {
         msix_vector_use(d, i);
     }
@@ -596,295 +744,579 @@ static void ivshmem_use_msix(IVShmemState * s)
 
 static void ivshmem_reset(DeviceState *d)
 {
-    IVShmemState *s = IVSHMEM(d);
+    IVShmemState *s = IVSHMEM_COMMON(d);
 
     s->intrstatus = 0;
-    ivshmem_use_msix(s);
+    s->intrmask = 0;
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        ivshmem_msix_vector_use(s);
+    }
 }
 
-static uint64_t ivshmem_get_size(IVShmemState * s) {
+static int ivshmem_setup_interrupts(IVShmemState *s)
+{
+    /* allocate QEMU callback data for receiving interrupts */
+    s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector));
 
-    uint64_t value;
-    char *ptr;
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
+            return -1;
+        }
 
-    value = strtoull(s->sizearg, &ptr, 10);
-    switch (*ptr) {
-        case 0: case 'M': case 'm':
-            value <<= 20;
-            break;
-        case 'G': case 'g':
-            value <<= 30;
-            break;
-        default:
-            error_report("invalid ram size: %s", s->sizearg);
-            exit(1);
+        IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
+        ivshmem_msix_vector_use(s);
     }
 
-    /* BARs must be a power of 2 */
-    if (!is_power_of_two(value)) {
-        error_report("size must be power of 2");
-        exit(1);
+    return 0;
+}
+
+static void ivshmem_enable_irqfd(IVShmemState *s)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
+
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        Error *err = NULL;
+
+        ivshmem_add_kvm_msi_virq(s, i, &err);
+        if (err) {
+            error_report_err(err);
+            /* TODO do we need to handle the error? */
+        }
     }
 
-    return value;
+    if (msix_set_vector_notifiers(pdev,
+                                  ivshmem_vector_unmask,
+                                  ivshmem_vector_mask,
+                                  ivshmem_vector_poll)) {
+        error_report("ivshmem: msix_set_vector_notifiers failed");
+    }
 }
 
-static void ivshmem_setup_msi(IVShmemState * s)
+static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
 {
-    if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
-        IVSHMEM_DPRINTF("msix initialization failed\n");
-        exit(1);
-    }
+    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
 
-    IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
+    if (s->msi_vectors[vector].pdev == NULL) {
+        return;
+    }
 
-    /* allocate QEMU char devices for receiving interrupts */
-    s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
+    /* it was cleaned when masked in the frontend. */
+    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
 
-    ivshmem_use_msix(s);
+    s->msi_vectors[vector].pdev = NULL;
 }
 
-static void ivshmem_save(QEMUFile* f, void *opaque)
+static void ivshmem_disable_irqfd(IVShmemState *s)
 {
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-
-    IVSHMEM_DPRINTF("ivshmem_save\n");
-    pci_device_save(pci_dev, f);
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
 
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_save(pci_dev, f);
-    } else {
-        qemu_put_be32(f, proxy->intrstatus);
-        qemu_put_be32(f, proxy->intrmask);
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        ivshmem_remove_kvm_msi_virq(s, i);
     }
 
+    msix_unset_vector_notifiers(pdev);
 }
 
-static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
+static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
+                                 uint32_t val, int len)
 {
-    IVSHMEM_DPRINTF("ivshmem_load\n");
+    IVShmemState *s = IVSHMEM_COMMON(pdev);
+    int is_enabled, was_enabled = msix_enabled(pdev);
 
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-    int ret;
+    pci_default_write_config(pdev, address, val, len);
+    is_enabled = msix_enabled(pdev);
 
-    if (version_id > 0) {
-        return -EINVAL;
+    if (kvm_msi_via_irqfd_enabled()) {
+        if (!was_enabled && is_enabled) {
+            ivshmem_enable_irqfd(s);
+        } else if (was_enabled && !is_enabled) {
+            ivshmem_disable_irqfd(s);
+        }
     }
+}
 
-    if (proxy->role_val == IVSHMEM_PEER) {
-        error_report("'peer' devices are not migratable");
-        return -EINVAL;
+static void ivshmem_common_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    Error *err = NULL;
+    uint8_t *pci_conf;
+    uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
+
+    /* IRQFD requires MSI */
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
+        !ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        error_setg(errp, "ioeventfd/irqfd requires MSI");
+        return;
     }
 
-    ret = pci_device_load(pci_dev, f);
-    if (ret) {
-        return ret;
+    pci_conf = dev->config;
+    pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
+
+    memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
+                          "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
+
+    /* region for registers*/
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
+                     &s->ivshmem_mmio);
+
+    if (!s->not_legacy_32bit) {
+        attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
     }
 
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_load(pci_dev, f);
-	ivshmem_use_msix(proxy);
+    if (s->hostmem != NULL) {
+        IVSHMEM_DPRINTF("using hostmem\n");
+
+        s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem,
+                                                         &error_abort);
     } else {
-        proxy->intrstatus = qemu_get_be32(f);
-        proxy->intrmask = qemu_get_be32(f);
+        assert(s->server_chr);
+
+        IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
+                        s->server_chr->filename);
+
+        /* we allocate enough space for 16 peers and grow as needed */
+        resize_peers(s, 16);
+
+        /*
+         * Receive setup messages from server synchronously.
+         * Older versions did it asynchronously, but that creates a
+         * number of entertaining race conditions.
+         */
+        ivshmem_recv_setup(s, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) {
+            error_setg(errp,
+                       "master must connect to the server before any peers");
+            return;
+        }
+
+        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive,
+                              ivshmem_read, NULL, s);
+
+        if (ivshmem_setup_interrupts(s) < 0) {
+            error_setg(errp, "failed to initialize interrupts");
+            return;
+        }
     }
 
-    return 0;
-}
+    vmstate_register_ram(s->ivshmem_bar2, DEVICE(s));
+    pci_register_bar(PCI_DEVICE(s), 2, attr, s->ivshmem_bar2);
 
-static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
-				 uint32_t val, int len)
-{
-    pci_default_write_config(pci_dev, address, val, len);
+    if (s->master == ON_OFF_AUTO_AUTO) {
+        s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
+    }
+
+    if (!ivshmem_is_master(s)) {
+        error_setg(&s->migration_blocker,
+                   "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
+        migrate_add_blocker(s->migration_blocker);
+    }
 }
 
-static int pci_ivshmem_init(PCIDevice *dev)
+static void ivshmem_exit(PCIDevice *dev)
 {
-    IVShmemState *s = IVSHMEM(dev);
-    uint8_t *pci_conf;
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    int i;
 
-    if (s->sizearg == NULL)
-        s->ivshmem_size = 4 << 20; /* 4 MB default */
-    else {
-        s->ivshmem_size = ivshmem_get_size(s);
+    if (s->migration_blocker) {
+        migrate_del_blocker(s->migration_blocker);
+        error_free(s->migration_blocker);
     }
 
-    fifo8_create(&s->incoming_fifo, sizeof(long));
+    if (memory_region_is_mapped(s->ivshmem_bar2)) {
+        if (!s->hostmem) {
+            void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2);
+            int fd;
 
-    register_savevm(DEVICE(dev), "ivshmem", 0, 0, ivshmem_save, ivshmem_load,
-                                                                        dev);
+            if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) {
+                error_report("Failed to munmap shared memory %s",
+                             strerror(errno));
+            }
 
-    /* IRQFD requires MSI */
-    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
-        !ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        error_report("ioeventfd/irqfd requires MSI");
-        exit(1);
+            fd = qemu_get_ram_fd(memory_region_get_ram_addr(s->ivshmem_bar2));
+            close(fd);
+        }
+
+        vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev));
     }
 
-    /* check that role is reasonable */
-    if (s->role) {
-        if (strncmp(s->role, "peer", 5) == 0) {
-            s->role_val = IVSHMEM_PEER;
-        } else if (strncmp(s->role, "master", 7) == 0) {
-            s->role_val = IVSHMEM_MASTER;
-        } else {
-            error_report("'role' must be 'peer' or 'master'");
-            exit(1);
+    if (s->peers) {
+        for (i = 0; i < s->nb_peers; i++) {
+            close_peer_eventfds(s, i);
         }
-    } else {
-        s->role_val = IVSHMEM_MASTER; /* default */
+        g_free(s->peers);
     }
 
-    if (s->role_val == IVSHMEM_PEER) {
-        error_setg(&s->migration_blocker,
-                   "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
-        migrate_add_blocker(s->migration_blocker);
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_uninit_exclusive_bar(dev);
     }
 
-    pci_conf = dev->config;
-    pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
+    g_free(s->msi_vectors);
+}
 
-    pci_config_set_interrupt_pin(pci_conf, 1);
+static int ivshmem_pre_load(void *opaque)
+{
+    IVShmemState *s = opaque;
 
-    s->shm_fd = 0;
+    if (!ivshmem_is_master(s)) {
+        error_report("'peer' devices are not migratable");
+        return -EINVAL;
+    }
 
-    memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
-                          "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
+    return 0;
+}
 
-    /* region for registers*/
-    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
-                     &s->ivshmem_mmio);
+static int ivshmem_post_load(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
 
-    memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size);
-    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
-        PCI_BASE_ADDRESS_MEM_PREFETCH;
-    if (s->ivshmem_64bit) {
-        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        ivshmem_msix_vector_use(s);
     }
+    return 0;
+}
 
-    if ((s->server_chr != NULL) &&
-                        (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
-        /* if we get a UNIX socket as the parameter we will talk
-         * to the ivshmem server to receive the memory region */
+static void ivshmem_common_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-        if (s->shmobj != NULL) {
-            error_report("WARNING: do not specify both 'chardev' "
-                         "and 'shm' with ivshmem");
-        }
+    k->realize = ivshmem_common_realize;
+    k->exit = ivshmem_exit;
+    k->config_write = ivshmem_write_config;
+    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
+    k->device_id = PCI_DEVICE_ID_IVSHMEM;
+    k->class_id = PCI_CLASS_MEMORY_RAM;
+    k->revision = 1;
+    dc->reset = ivshmem_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->desc = "Inter-VM shared memory";
+}
 
-        IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
-                        s->server_chr->filename);
+static const TypeInfo ivshmem_common_info = {
+    .name          = TYPE_IVSHMEM_COMMON,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(IVShmemState),
+    .abstract      = true,
+    .class_init    = ivshmem_common_class_init,
+};
 
-        if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-            ivshmem_setup_msi(s);
-        }
+static void ivshmem_check_memdev_is_busy(Object *obj, const char *name,
+                                         Object *val, Error **errp)
+{
+    MemoryRegion *mr;
 
-        /* we allocate enough space for 16 guests and grow as needed */
-        s->nb_peers = 16;
-        s->vm_id = -1;
+    mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), &error_abort);
+    if (memory_region_is_mapped(mr)) {
+        char *path = object_get_canonical_path_component(val);
+        error_setg(errp, "can't use already busy memdev: %s", path);
+        g_free(path);
+    } else {
+        qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
+    }
+}
+
+static const VMStateDescription ivshmem_plain_vmsd = {
+    .name = TYPE_IVSHMEM_PLAIN,
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+        VMSTATE_UINT32(intrstatus, IVShmemState),
+        VMSTATE_UINT32(intrmask, IVShmemState),
+        VMSTATE_END_OF_LIST()
+    },
+};
 
-        /* allocate/initialize space for interrupt handling */
-        s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
+static Property ivshmem_plain_properties[] = {
+    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
-        pci_register_bar(dev, 2, s->ivshmem_attr, &s->bar);
+static void ivshmem_plain_init(Object *obj)
+{
+    IVShmemState *s = IVSHMEM_PLAIN(obj);
 
-        s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
+    object_property_add_link(obj, "memdev", TYPE_MEMORY_BACKEND,
+                             (Object **)&s->hostmem,
+                             ivshmem_check_memdev_is_busy,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
 
-        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read,
-                     ivshmem_event, s);
-    } else {
-        /* just map the file immediately, we're not using a server */
-        int fd;
+static void ivshmem_plain_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
 
-        if (s->shmobj == NULL) {
-            error_report("Must specify 'chardev' or 'shm' to ivshmem");
-            exit(1);
-        }
+    if (!s->hostmem) {
+        error_setg(errp, "You must specify a 'memdev'");
+        return;
+    }
 
-        IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj);
+    ivshmem_common_realize(dev, errp);
+}
 
-        /* try opening with O_EXCL and if it succeeds zero the memory
-         * by truncating to 0 */
-        if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR|O_EXCL,
-                        S_IRWXU|S_IRWXG|S_IRWXO)) > 0) {
-           /* truncate file to length PCI device's memory */
-            if (ftruncate(fd, s->ivshmem_size) != 0) {
-                error_report("could not truncate shared file");
-            }
+static void ivshmem_plain_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-        } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR,
-                        S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
-            error_report("could not open shared file");
-            exit(1);
+    k->realize = ivshmem_plain_realize;
+    dc->props = ivshmem_plain_properties;
+    dc->vmsd = &ivshmem_plain_vmsd;
+}
 
-        }
+static const TypeInfo ivshmem_plain_info = {
+    .name          = TYPE_IVSHMEM_PLAIN,
+    .parent        = TYPE_IVSHMEM_COMMON,
+    .instance_size = sizeof(IVShmemState),
+    .instance_init = ivshmem_plain_init,
+    .class_init    = ivshmem_plain_class_init,
+};
 
-        if (check_shm_size(s, fd) == -1) {
-            exit(1);
-        }
+static const VMStateDescription ivshmem_doorbell_vmsd = {
+    .name = TYPE_IVSHMEM_DOORBELL,
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+        VMSTATE_MSIX(parent_obj, IVShmemState),
+        VMSTATE_UINT32(intrstatus, IVShmemState),
+        VMSTATE_UINT32(intrmask, IVShmemState),
+        VMSTATE_END_OF_LIST()
+    },
+};
 
-        create_shared_memory_BAR(s, fd);
+static Property ivshmem_doorbell_properties[] = {
+    DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
+    DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
+    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
+                    true),
+    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
+static void ivshmem_doorbell_init(Object *obj)
+{
+    IVShmemState *s = IVSHMEM_DOORBELL(obj);
+
+    s->features |= (1 << IVSHMEM_MSI);
+    s->legacy_size = SIZE_MAX;  /* whatever the server sends */
+}
+
+static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+
+    if (!s->server_chr) {
+        error_setg(errp, "You must specify a 'chardev'");
+        return;
     }
 
-    dev->config_write = ivshmem_write_config;
+    ivshmem_common_realize(dev, errp);
+}
 
-    return 0;
+static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = ivshmem_doorbell_realize;
+    dc->props = ivshmem_doorbell_properties;
+    dc->vmsd = &ivshmem_doorbell_vmsd;
 }
 
-static void pci_ivshmem_uninit(PCIDevice *dev)
+static const TypeInfo ivshmem_doorbell_info = {
+    .name          = TYPE_IVSHMEM_DOORBELL,
+    .parent        = TYPE_IVSHMEM_COMMON,
+    .instance_size = sizeof(IVShmemState),
+    .instance_init = ivshmem_doorbell_init,
+    .class_init    = ivshmem_doorbell_class_init,
+};
+
+static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id)
 {
-    IVShmemState *s = IVSHMEM(dev);
+    IVShmemState *s = opaque;
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int ret;
 
-    if (s->migration_blocker) {
-        migrate_del_blocker(s->migration_blocker);
-        error_free(s->migration_blocker);
+    IVSHMEM_DPRINTF("ivshmem_load_old\n");
+
+    if (version_id != 0) {
+        return -EINVAL;
+    }
+
+    ret = ivshmem_pre_load(s);
+    if (ret) {
+        return ret;
+    }
+
+    ret = pci_device_load(pdev, f);
+    if (ret) {
+        return ret;
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_load(pdev, f);
+        ivshmem_msix_vector_use(s);
+    } else {
+        s->intrstatus = qemu_get_be32(f);
+        s->intrmask = qemu_get_be32(f);
     }
 
-    memory_region_del_subregion(&s->bar, &s->ivshmem);
-    vmstate_unregister_ram(&s->ivshmem, DEVICE(dev));
-    unregister_savevm(DEVICE(dev), "ivshmem", s);
-    fifo8_destroy(&s->incoming_fifo);
+    return 0;
 }
 
+static bool test_msix(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+
+    return ivshmem_has_feature(s, IVSHMEM_MSI);
+}
+
+static bool test_no_msix(void *opaque, int version_id)
+{
+    return !test_msix(opaque, version_id);
+}
+
+static const VMStateDescription ivshmem_vmsd = {
+    .name = "ivshmem",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+
+        VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix),
+        VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix),
+        VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix),
+
+        VMSTATE_END_OF_LIST()
+    },
+    .load_state_old = ivshmem_load_old,
+    .minimum_version_id_old = 0
+};
+
 static Property ivshmem_properties[] = {
     DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
     DEFINE_PROP_STRING("size", IVShmemState, sizearg),
     DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
-    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, false),
+    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
+                    false),
     DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
     DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
     DEFINE_PROP_STRING("role", IVShmemState, role),
-    DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1),
+    DEFINE_PROP_UINT32("use64", IVShmemState, not_legacy_32bit, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void desugar_shm(IVShmemState *s)
+{
+    Object *obj;
+    char *path;
+
+    obj = object_new("memory-backend-file");
+    path = g_strdup_printf("/dev/shm/%s", s->shmobj);
+    object_property_set_str(obj, path, "mem-path", &error_abort);
+    g_free(path);
+    object_property_set_int(obj, s->legacy_size, "size", &error_abort);
+    object_property_set_bool(obj, true, "share", &error_abort);
+    object_property_add_child(OBJECT(s), "internal-shm-backend", obj,
+                              &error_abort);
+    user_creatable_complete(obj, &error_abort);
+    s->hostmem = MEMORY_BACKEND(obj);
+}
+
+static void ivshmem_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+
+    if (!qtest_enabled()) {
+        error_report("ivshmem is deprecated, please use ivshmem-plain"
+                     " or ivshmem-doorbell instead");
+    }
+
+    if (!!s->server_chr + !!s->shmobj != 1) {
+        error_setg(errp, "You must specify either 'shm' or 'chardev'");
+        return;
+    }
+
+    if (s->sizearg == NULL) {
+        s->legacy_size = 4 << 20; /* 4 MB default */
+    } else {
+        char *end;
+        int64_t size = qemu_strtosz(s->sizearg, &end);
+        if (size < 0 || (size_t)size != size || *end != '\0'
+            || !is_power_of_2(size)) {
+            error_setg(errp, "Invalid size %s", s->sizearg);
+            return;
+        }
+        s->legacy_size = size;
+    }
+
+    /* check that role is reasonable */
+    if (s->role) {
+        if (strncmp(s->role, "peer", 5) == 0) {
+            s->master = ON_OFF_AUTO_OFF;
+        } else if (strncmp(s->role, "master", 7) == 0) {
+            s->master = ON_OFF_AUTO_ON;
+        } else {
+            error_setg(errp, "'role' must be 'peer' or 'master'");
+            return;
+        }
+    } else {
+        s->master = ON_OFF_AUTO_AUTO;
+    }
+
+    if (s->shmobj) {
+        desugar_shm(s);
+    }
+
+    /*
+     * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a
+     * bald-faced lie then.  But it's a backwards compatible lie.
+     */
+    pci_config_set_interrupt_pin(dev->config, 1);
+
+    ivshmem_common_realize(dev, errp);
+}
+
 static void ivshmem_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ivshmem_init;
-    k->exit = pci_ivshmem_uninit;
-    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
-    k->device_id = PCI_DEVICE_ID_IVSHMEM;
-    k->class_id = PCI_CLASS_MEMORY_RAM;
-    dc->reset = ivshmem_reset;
+    k->realize = ivshmem_realize;
+    k->revision = 0;
+    dc->desc = "Inter-VM shared memory (legacy)";
     dc->props = ivshmem_properties;
-    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->vmsd = &ivshmem_vmsd;
 }
 
 static const TypeInfo ivshmem_info = {
     .name          = TYPE_IVSHMEM,
-    .parent        = TYPE_PCI_DEVICE,
+    .parent        = TYPE_IVSHMEM_COMMON,
     .instance_size = sizeof(IVShmemState),
     .class_init    = ivshmem_class_init,
 };
 
 static void ivshmem_register_types(void)
 {
+    type_register_static(&ivshmem_common_info);
+    type_register_static(&ivshmem_plain_info);
+    type_register_static(&ivshmem_doorbell_info);
     type_register_static(&ivshmem_info);
 }
 
diff --git a/qemu/hw/misc/macio/cuda.c b/qemu/hw/misc/macio/cuda.c
index f3984e3a2..f15f30110 100644
--- a/qemu/hw/misc/macio/cuda.c
+++ b/qemu/hw/misc/macio/cuda.c
@@ -22,11 +22,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/input/adb.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
 /* XXX: implement all timer modes */
 
@@ -57,6 +59,8 @@
 #define IER_SET		0x80		/* set bits in IER */
 #define IER_CLR		0		/* clear bits in IER */
 #define SR_INT		0x04		/* Shift register full/empty */
+#define SR_DATA_INT	0x08
+#define SR_CLOCK_INT	0x10
 #define T1_INT          0x40            /* Timer 1 interrupt */
 #define T2_INT          0x20            /* Timer 2 interrupt */
 
@@ -103,11 +107,28 @@
 #define CUDA_COMBINED_FORMAT_IIC	0x25
 
 #define CUDA_TIMER_FREQ (4700000 / 6)
-#define CUDA_ADB_POLL_FREQ 50
 
 /* CUDA returns time_t's offset from Jan 1, 1904, not 1970 */
 #define RTC_OFFSET                      2082844800
 
+/* CUDA registers */
+#define CUDA_REG_B       0x00
+#define CUDA_REG_A       0x01
+#define CUDA_REG_DIRB    0x02
+#define CUDA_REG_DIRA    0x03
+#define CUDA_REG_T1CL    0x04
+#define CUDA_REG_T1CH    0x05
+#define CUDA_REG_T1LL    0x06
+#define CUDA_REG_T1LH    0x07
+#define CUDA_REG_T2CL    0x08
+#define CUDA_REG_T2CH    0x09
+#define CUDA_REG_SR      0x0a
+#define CUDA_REG_ACR     0x0b
+#define CUDA_REG_PCR     0x0c
+#define CUDA_REG_IFR     0x0d
+#define CUDA_REG_IER     0x0e
+#define CUDA_REG_ANH     0x0f
+
 static void cuda_update(CUDAState *s);
 static void cuda_receive_packet_from_host(CUDAState *s,
                                           const uint8_t *data, int len);
@@ -116,47 +137,48 @@ static void cuda_timer_update(CUDAState *s, CUDATimer *ti,
 
 static void cuda_update_irq(CUDAState *s)
 {
-    if (s->ifr & s->ier & (SR_INT | T1_INT)) {
+    if (s->ifr & s->ier & (SR_INT | T1_INT | T2_INT)) {
         qemu_irq_raise(s->irq);
     } else {
         qemu_irq_lower(s->irq);
     }
 }
 
-static uint64_t get_tb(uint64_t freq)
+static uint64_t get_tb(uint64_t time, uint64_t freq)
 {
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                    freq, get_ticks_per_sec());
+    return muldiv64(time, freq, NANOSECONDS_PER_SECOND);
 }
 
-static unsigned int get_counter(CUDATimer *s)
+static unsigned int get_counter(CUDATimer *ti)
 {
     int64_t d;
     unsigned int counter;
     uint64_t tb_diff;
+    uint64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* Reverse of the tb calculation algorithm that Mac OS X uses on bootup. */
-    tb_diff = get_tb(s->frequency) - s->load_time;
-    d = (tb_diff * 0xBF401675E5DULL) / (s->frequency << 24);
+    tb_diff = get_tb(current_time, ti->frequency) - ti->load_time;
+    d = (tb_diff * 0xBF401675E5DULL) / (ti->frequency << 24);
 
-    if (s->index == 0) {
+    if (ti->index == 0) {
         /* the timer goes down from latch to -1 (period of latch + 2) */
-        if (d <= (s->counter_value + 1)) {
-            counter = (s->counter_value - d) & 0xffff;
+        if (d <= (ti->counter_value + 1)) {
+            counter = (ti->counter_value - d) & 0xffff;
         } else {
-            counter = (d - (s->counter_value + 1)) % (s->latch + 2);
-            counter = (s->latch - counter) & 0xffff;
+            counter = (d - (ti->counter_value + 1)) % (ti->latch + 2);
+            counter = (ti->latch - counter) & 0xffff;
         }
     } else {
-        counter = (s->counter_value - d) & 0xffff;
+        counter = (ti->counter_value - d) & 0xffff;
     }
     return counter;
 }
 
 static void set_counter(CUDAState *s, CUDATimer *ti, unsigned int val)
 {
-    CUDA_DPRINTF("T%d.counter=%d\n", 1 + (ti->timer == NULL), val);
-    ti->load_time = get_tb(s->frequency);
+    CUDA_DPRINTF("T%d.counter=%d\n", 1 + ti->index, val);
+    ti->load_time = get_tb(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           s->frequency);
     ti->counter_value = val;
     cuda_timer_update(s, ti, ti->load_time);
 }
@@ -168,7 +190,7 @@ static int64_t get_next_irq_time(CUDATimer *s, int64_t current_time)
 
     /* current counter value */
     d = muldiv64(current_time - s->load_time,
-                 CUDA_TIMER_FREQ, get_ticks_per_sec());
+                 CUDA_TIMER_FREQ, NANOSECONDS_PER_SECOND);
     /* the timer goes down from latch to -1 (period of latch + 2) */
     if (d <= (s->counter_value + 1)) {
         counter = (s->counter_value - d) & 0xffff;
@@ -187,7 +209,7 @@ static int64_t get_next_irq_time(CUDATimer *s, int64_t current_time)
     }
     CUDA_DPRINTF("latch=%d counter=%" PRId64 " delta_next=%" PRId64 "\n",
                  s->latch, d, next_time - d);
-    next_time = muldiv64(next_time, get_ticks_per_sec(), CUDA_TIMER_FREQ) +
+    next_time = muldiv64(next_time, NANOSECONDS_PER_SECOND, CUDA_TIMER_FREQ) +
         s->load_time;
     if (next_time <= current_time)
         next_time = current_time + 1;
@@ -199,7 +221,7 @@ static void cuda_timer_update(CUDAState *s, CUDATimer *ti,
 {
     if (!ti->timer)
         return;
-    if ((s->acr & T1MODE) != T1MODE_CONT) {
+    if (ti->index == 0 && (s->acr & T1MODE) != T1MODE_CONT) {
         timer_del(ti->timer);
     } else {
         ti->next_irq_time = get_next_irq_time(ti, current_time);
@@ -217,6 +239,41 @@ static void cuda_timer1(void *opaque)
     cuda_update_irq(s);
 }
 
+static void cuda_timer2(void *opaque)
+{
+    CUDAState *s = opaque;
+    CUDATimer *ti = &s->timers[1];
+
+    cuda_timer_update(s, ti, ti->next_irq_time);
+    s->ifr |= T2_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_set_sr_int(void *opaque)
+{
+    CUDAState *s = opaque;
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+    s->ifr |= SR_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_delay_set_sr_int(CUDAState *s)
+{
+    int64_t expire;
+
+    if (s->dirb == 0xff) {
+        /* Not in Mac OS, fire the IRQ directly */
+        cuda_set_sr_int(s);
+        return;
+    }
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+
+    expire = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 300 * SCALE_US;
+    timer_mod(s->sr_delay_timer, expire);
+}
+
 static uint32_t cuda_readb(void *opaque, hwaddr addr)
 {
     CUDAState *s = opaque;
@@ -224,66 +281,68 @@ static uint32_t cuda_readb(void *opaque, hwaddr addr)
 
     addr = (addr >> 9) & 0xf;
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         val = s->b;
         break;
-    case 1:
+    case CUDA_REG_A:
         val = s->a;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         val = s->dirb;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         val = s->dira;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         val = get_counter(&s->timers[0]) & 0xff;
         s->ifr &= ~T1_INT;
         cuda_update_irq(s);
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         val = get_counter(&s->timers[0]) >> 8;
         cuda_update_irq(s);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         val = s->timers[0].latch & 0xff;
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         /* XXX: check this */
         val = (s->timers[0].latch >> 8) & 0xff;
         break;
-    case 8:
+    case CUDA_REG_T2CL:
         val = get_counter(&s->timers[1]) & 0xff;
         s->ifr &= ~T2_INT;
+        cuda_update_irq(s);
         break;
-    case 9:
+    case CUDA_REG_T2CH:
         val = get_counter(&s->timers[1]) >> 8;
         break;
-    case 10:
+    case CUDA_REG_SR:
         val = s->sr;
-        s->ifr &= ~SR_INT;
+        s->ifr &= ~(SR_INT | SR_CLOCK_INT | SR_DATA_INT);
         cuda_update_irq(s);
         break;
-    case 11:
+    case CUDA_REG_ACR:
         val = s->acr;
         break;
-    case 12:
+    case CUDA_REG_PCR:
         val = s->pcr;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         val = s->ifr;
-        if (s->ifr & s->ier)
+        if (s->ifr & s->ier) {
             val |= 0x80;
+        }
         break;
-    case 14:
+    case CUDA_REG_IER:
         val = s->ier | 0x80;
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         val = s->anh;
         break;
     }
-    if (addr != 13 || val != 0) {
+    if (addr != CUDA_REG_IFR || val != 0) {
         CUDA_DPRINTF("read: reg=0x%x val=%02x\n", (int)addr, val);
     }
 
@@ -298,61 +357,65 @@ static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
     CUDA_DPRINTF("write: reg=0x%x val=%02x\n", (int)addr, val);
 
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         s->b = val;
         cuda_update(s);
         break;
-    case 1:
+    case CUDA_REG_A:
         s->a = val;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         s->dirb = val;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         s->dira = val;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         set_counter(s, &s->timers[0], s->timers[0].latch);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 8:
-        s->timers[1].latch = val;
-        set_counter(s, &s->timers[1], val);
+    case CUDA_REG_T2CL:
+        s->timers[1].latch = (s->timers[1].latch & 0xff00) | val;
         break;
-    case 9:
-        set_counter(s, &s->timers[1], (val << 8) | s->timers[1].latch);
+    case CUDA_REG_T2CH:
+        /* To ensure T2 generates an interrupt on zero crossing with the
+           common timer code, write the value directly from the latch to
+           the counter */
+        s->timers[1].latch = (s->timers[1].latch & 0xff) | (val << 8);
+        s->ifr &= ~T2_INT;
+        set_counter(s, &s->timers[1], s->timers[1].latch);
         break;
-    case 10:
+    case CUDA_REG_SR:
         s->sr = val;
         break;
-    case 11:
+    case CUDA_REG_ACR:
         s->acr = val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         cuda_update(s);
         break;
-    case 12:
+    case CUDA_REG_PCR:
         s->pcr = val;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         /* reset bits */
         s->ifr &= ~val;
         cuda_update_irq(s);
         break;
-    case 14:
+    case CUDA_REG_IER:
         if (val & IER_SET) {
             /* set bits */
             s->ier |= val & 0x7f;
@@ -363,7 +426,7 @@ static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
         cuda_update_irq(s);
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         s->anh = val;
         break;
     }
@@ -384,8 +447,7 @@ static void cuda_update(CUDAState *s)
                 if (s->data_out_index < sizeof(s->data_out)) {
                     CUDA_DPRINTF("send: %02x\n", s->sr);
                     s->data_out[s->data_out_index++] = s->sr;
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         } else {
@@ -398,8 +460,7 @@ static void cuda_update(CUDAState *s)
                     if (s->data_in_index >= s->data_in_size) {
                         s->b = (s->b | TREQ);
                     }
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         }
@@ -411,15 +472,13 @@ static void cuda_update(CUDAState *s)
                 s->b = (s->b | TREQ);
             else
                 s->b = (s->b & ~TREQ);
-            s->ifr |= SR_INT;
-            cuda_update_irq(s);
+            cuda_delay_set_sr_int(s);
         } else {
             if (!(s->last_b & TIP)) {
                 /* handle end of host to cuda transfer */
                 packet_received = (s->data_out_index > 0);
                 /* always an IRQ at the end of transfer */
-                s->ifr |= SR_INT;
-                cuda_update_irq(s);
+                cuda_delay_set_sr_int(s);
             }
             /* signal if there is data to read */
             if (s->data_in_index < s->data_in_size) {
@@ -456,8 +515,7 @@ static void cuda_send_packet_to_host(CUDAState *s,
     s->data_in_size = len;
     s->data_in_index = 0;
     cuda_update(s);
-    s->ifr |= SR_INT;
-    cuda_update_irq(s);
+    cuda_delay_set_sr_int(s);
 }
 
 static void cuda_adb_poll(void *opaque)
@@ -466,7 +524,7 @@ static void cuda_adb_poll(void *opaque)
     uint8_t obuf[ADB_MAX_OUT_LEN + 2];
     int olen;
 
-    olen = adb_poll(&s->adb_bus, obuf + 2);
+    olen = adb_poll(&s->adb_bus, obuf + 2, s->adb_poll_mask);
     if (olen > 0) {
         obuf[0] = ADB_PACKET;
         obuf[1] = 0x40; /* polled data */
@@ -474,75 +532,213 @@ static void cuda_adb_poll(void *opaque)
     }
     timer_mod(s->adb_poll_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   (get_ticks_per_sec() / CUDA_ADB_POLL_FREQ));
+                   (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
 }
 
-static void cuda_receive_packet(CUDAState *s,
-                                const uint8_t *data, int len)
+/* description of commands */
+typedef struct CudaCommand {
+    uint8_t command;
+    const char *name;
+    bool (*handler)(CUDAState *s,
+                    const uint8_t *in_args, int in_len,
+                    uint8_t *out_args, int *out_len);
+} CudaCommand;
+
+static bool cuda_cmd_autopoll(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
 {
-    uint8_t obuf[16];
     int autopoll;
+
+    if (in_len != 1) {
+        return false;
+    }
+
+    autopoll = (in_data[0] != 0);
+    if (autopoll != s->autopoll) {
+        s->autopoll = autopoll;
+        if (autopoll) {
+            timer_mod(s->adb_poll_timer,
+                      qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
+        } else {
+            timer_del(s->adb_poll_timer);
+        }
+    }
+    return true;
+}
+
+static bool cuda_cmd_set_autorate(CUDAState *s,
+                                  const uint8_t *in_data, int in_len,
+                                  uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    /* we don't want a period of 0 ms */
+    /* FIXME: check what real hardware does */
+    if (in_data[0] == 0) {
+        return false;
+    }
+
+    s->autopoll_rate_ms = in_data[0];
+    if (s->autopoll) {
+        timer_mod(s->adb_poll_timer,
+                  qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
+    }
+    return true;
+}
+
+static bool cuda_cmd_set_device_list(CUDAState *s,
+                                     const uint8_t *in_data, int in_len,
+                                     uint8_t *out_data, int *out_len)
+{
+    if (in_len != 2) {
+        return false;
+    }
+
+    s->adb_poll_mask = (((uint16_t)in_data[0]) << 8) | in_data[1];
+    return true;
+}
+
+static bool cuda_cmd_powerdown(CUDAState *s,
+                               const uint8_t *in_data, int in_len,
+                               uint8_t *out_data, int *out_len)
+{
+    if (in_len != 0) {
+        return false;
+    }
+
+    qemu_system_shutdown_request();
+    return true;
+}
+
+static bool cuda_cmd_reset_system(CUDAState *s,
+                                  const uint8_t *in_data, int in_len,
+                                  uint8_t *out_data, int *out_len)
+{
+    if (in_len != 0) {
+        return false;
+    }
+
+    qemu_system_reset_request();
+    return true;
+}
+
+static bool cuda_cmd_set_file_server_flag(CUDAState *s,
+                                          const uint8_t *in_data, int in_len,
+                                          uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    qemu_log_mask(LOG_UNIMP,
+                  "CUDA: unimplemented command FILE_SERVER_FLAG %d\n",
+                  in_data[0]);
+    return true;
+}
+
+static bool cuda_cmd_set_power_message(CUDAState *s,
+                                       const uint8_t *in_data, int in_len,
+                                       uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    qemu_log_mask(LOG_UNIMP,
+                  "CUDA: unimplemented command SET_POWER_MESSAGE %d\n",
+                  in_data[0]);
+    return true;
+}
+
+static bool cuda_cmd_get_time(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
+{
     uint32_t ti;
 
-    switch(data[0]) {
-    case CUDA_AUTOPOLL:
-        autopoll = (data[1] != 0);
-        if (autopoll != s->autopoll) {
-            s->autopoll = autopoll;
-            if (autopoll) {
-                timer_mod(s->adb_poll_timer,
-                               qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                               (get_ticks_per_sec() / CUDA_ADB_POLL_FREQ));
+    if (in_len != 0) {
+        return false;
+    }
+
+    ti = s->tick_offset + (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                           / NANOSECONDS_PER_SECOND);
+    out_data[0] = ti >> 24;
+    out_data[1] = ti >> 16;
+    out_data[2] = ti >> 8;
+    out_data[3] = ti;
+    *out_len = 4;
+    return true;
+}
+
+static bool cuda_cmd_set_time(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
+{
+    uint32_t ti;
+
+    if (in_len != 4) {
+        return false;
+    }
+
+    ti = (((uint32_t)in_data[0]) << 24) + (((uint32_t)in_data[1]) << 16)
+         + (((uint32_t)in_data[2]) << 8) + in_data[3];
+    s->tick_offset = ti - (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                           / NANOSECONDS_PER_SECOND);
+    return true;
+}
+
+static const CudaCommand handlers[] = {
+    { CUDA_AUTOPOLL, "AUTOPOLL", cuda_cmd_autopoll },
+    { CUDA_SET_AUTO_RATE, "SET_AUTO_RATE",  cuda_cmd_set_autorate },
+    { CUDA_SET_DEVICE_LIST, "SET_DEVICE_LIST", cuda_cmd_set_device_list },
+    { CUDA_POWERDOWN, "POWERDOWN", cuda_cmd_powerdown },
+    { CUDA_RESET_SYSTEM, "RESET_SYSTEM", cuda_cmd_reset_system },
+    { CUDA_FILE_SERVER_FLAG, "FILE_SERVER_FLAG",
+      cuda_cmd_set_file_server_flag },
+    { CUDA_SET_POWER_MESSAGES, "SET_POWER_MESSAGES",
+      cuda_cmd_set_power_message },
+    { CUDA_GET_TIME, "GET_TIME", cuda_cmd_get_time },
+    { CUDA_SET_TIME, "SET_TIME", cuda_cmd_set_time },
+};
+
+static void cuda_receive_packet(CUDAState *s,
+                                const uint8_t *data, int len)
+{
+    uint8_t obuf[16] = { CUDA_PACKET, 0, data[0] };
+    int i, out_len = 0;
+
+    for (i = 0; i < ARRAY_SIZE(handlers); i++) {
+        const CudaCommand *desc = &handlers[i];
+        if (desc->command == data[0]) {
+            CUDA_DPRINTF("handling command %s\n", desc->name);
+            out_len = 0;
+            if (desc->handler(s, data + 1, len - 1, obuf + 3, &out_len)) {
+                cuda_send_packet_to_host(s, obuf, 3 + out_len);
             } else {
-                timer_del(s->adb_poll_timer);
+                qemu_log_mask(LOG_GUEST_ERROR,
+                              "CUDA: %s: wrong parameters %d\n",
+                              desc->name, len);
+                obuf[0] = ERROR_PACKET;
+                obuf[1] = 0x5; /* bad parameters */
+                obuf[2] = CUDA_PACKET;
+                obuf[3] = data[0];
+                cuda_send_packet_to_host(s, obuf, 4);
             }
+            return;
         }
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = data[1];
-        cuda_send_packet_to_host(s, obuf, 2);
-        break;
-    case CUDA_SET_TIME:
-        ti = (((uint32_t)data[1]) << 24) + (((uint32_t)data[2]) << 16) + (((uint32_t)data[3]) << 8) + data[4];
-        s->tick_offset = ti - (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
-        cuda_send_packet_to_host(s, obuf, 3);
-        break;
-    case CUDA_GET_TIME:
-        ti = s->tick_offset + (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
-        obuf[3] = ti >> 24;
-        obuf[4] = ti >> 16;
-        obuf[5] = ti >> 8;
-        obuf[6] = ti;
-        cuda_send_packet_to_host(s, obuf, 7);
-        break;
-    case CUDA_FILE_SERVER_FLAG:
-    case CUDA_SET_DEVICE_LIST:
-    case CUDA_SET_AUTO_RATE:
-    case CUDA_SET_POWER_MESSAGES:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        break;
-    case CUDA_POWERDOWN:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        qemu_system_shutdown_request();
-        break;
-    case CUDA_RESET_SYSTEM:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        qemu_system_reset_request();
-        break;
-    default:
-        break;
     }
+
+    qemu_log_mask(LOG_GUEST_ERROR, "CUDA: unknown command 0x%02x\n", data[0]);
+    obuf[0] = ERROR_PACKET;
+    obuf[1] = 0x2; /* unknown command */
+    obuf[2] = CUDA_PACKET;
+    obuf[3] = data[0];
+    cuda_send_packet_to_host(s, obuf, 4);
 }
 
 static void cuda_receive_packet_from_host(CUDAState *s,
@@ -560,19 +756,21 @@ static void cuda_receive_packet_from_host(CUDAState *s,
     switch(data[0]) {
     case ADB_PACKET:
         {
-            uint8_t obuf[ADB_MAX_OUT_LEN + 2];
+            uint8_t obuf[ADB_MAX_OUT_LEN + 3];
             int olen;
             olen = adb_request(&s->adb_bus, obuf + 2, data + 1, len - 1);
             if (olen > 0) {
                 obuf[0] = ADB_PACKET;
                 obuf[1] = 0x00;
+                cuda_send_packet_to_host(s, obuf, olen + 2);
             } else {
                 /* error */
                 obuf[0] = ADB_PACKET;
                 obuf[1] = -olen;
+                obuf[2] = data[1];
                 olen = 0;
+                cuda_send_packet_to_host(s, obuf, olen + 3);
             }
-            cuda_send_packet_to_host(s, obuf, olen + 2);
         }
         break;
     case CUDA_PACKET:
@@ -638,15 +836,17 @@ static const VMStateDescription vmstate_cuda_timer = {
 
 static const VMStateDescription vmstate_cuda = {
     .name = "cuda",
-    .version_id = 2,
-    .minimum_version_id = 2,
+    .version_id = 4,
+    .minimum_version_id = 4,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(a, CUDAState),
         VMSTATE_UINT8(b, CUDAState),
+        VMSTATE_UINT8(last_b, CUDAState),
         VMSTATE_UINT8(dira, CUDAState),
         VMSTATE_UINT8(dirb, CUDAState),
         VMSTATE_UINT8(sr, CUDAState),
         VMSTATE_UINT8(acr, CUDAState),
+        VMSTATE_UINT8(last_acr, CUDAState),
         VMSTATE_UINT8(pcr, CUDAState),
         VMSTATE_UINT8(ifr, CUDAState),
         VMSTATE_UINT8(ier, CUDAState),
@@ -655,12 +855,15 @@ static const VMStateDescription vmstate_cuda = {
         VMSTATE_INT32(data_in_index, CUDAState),
         VMSTATE_INT32(data_out_index, CUDAState),
         VMSTATE_UINT8(autopoll, CUDAState),
+        VMSTATE_UINT8(autopoll_rate_ms, CUDAState),
+        VMSTATE_UINT16(adb_poll_mask, CUDAState),
         VMSTATE_BUFFER(data_in, CUDAState),
         VMSTATE_BUFFER(data_out, CUDAState),
         VMSTATE_UINT32(tick_offset, CUDAState),
         VMSTATE_STRUCT_ARRAY(timers, CUDAState, 2, 1,
                              vmstate_cuda_timer, CUDATimer),
         VMSTATE_TIMER_PTR(adb_poll_timer, CUDAState),
+        VMSTATE_TIMER_PTR(sr_delay_timer, CUDAState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -671,7 +874,7 @@ static void cuda_reset(DeviceState *dev)
 
     s->b = 0;
     s->a = 0;
-    s->dirb = 0;
+    s->dirb = 0xff;
     s->dira = 0;
     s->sr = 0;
     s->acr = 0;
@@ -688,8 +891,9 @@ static void cuda_reset(DeviceState *dev)
     s->timers[0].latch = 0xffff;
     set_counter(s, &s->timers[0], 0xffff);
 
-    s->timers[1].latch = 0;
-    set_counter(s, &s->timers[1], 0xffff);
+    s->timers[1].latch = 0xffff;
+
+    s->sr_delay_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_set_sr_int, s);
 }
 
 static void cuda_realizefn(DeviceState *dev, Error **errp)
@@ -699,12 +903,15 @@ static void cuda_realizefn(DeviceState *dev, Error **errp)
 
     s->timers[0].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer1, s);
     s->timers[0].frequency = s->frequency;
-    s->timers[1].frequency = s->frequency;
+    s->timers[1].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer2, s);
+    s->timers[1].frequency = (SCALE_US * 6000) / 4700;
 
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
 
     s->adb_poll_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_adb_poll, s);
+    s->autopoll_rate_ms = 20;
+    s->adb_poll_mask = 0xffff;
 }
 
 static void cuda_initfn(Object *obj)
@@ -713,7 +920,7 @@ static void cuda_initfn(Object *obj)
     CUDAState *s = CUDA(obj);
     int i;
 
-    memory_region_init_io(&s->mem, NULL, &cuda_ops, s, "cuda", 0x2000);
+    memory_region_init_io(&s->mem, obj, &cuda_ops, s, "cuda", 0x2000);
     sysbus_init_mmio(d, &s->mem);
     sysbus_init_irq(d, &s->irq);
 
@@ -738,6 +945,7 @@ static void cuda_class_init(ObjectClass *oc, void *data)
     dc->reset = cuda_reset;
     dc->vmsd = &vmstate_cuda;
     dc->props = cuda_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo cuda_type_info = {
diff --git a/qemu/hw/misc/macio/mac_dbdma.c b/qemu/hw/misc/macio/mac_dbdma.c
index b25e8511b..6051f17db 100644
--- a/qemu/hw/misc/macio/mac_dbdma.c
+++ b/qemu/hw/misc/macio/mac_dbdma.c
@@ -36,6 +36,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/ppc/mac_dbdma.h"
@@ -556,12 +557,13 @@ void DBDMA_register_channel(void *dbdma, int nchan, qemu_irq irq,
 
     DBDMA_DPRINTF("DBDMA_register_channel 0x%x\n", nchan);
 
+    assert(rw);
+    assert(flush);
+
     ch->irq = irq;
-    ch->channel = nchan;
     ch->rw = rw;
     ch->flush = flush;
     ch->io.opaque = opaque;
-    ch->io.channel = ch;
 }
 
 static void
@@ -590,10 +592,11 @@ dbdma_control_write(DBDMA_channel *ch)
     if ((ch->regs[DBDMA_STATUS] & RUN) && !(status & RUN)) {
         /* RUN is cleared */
         status &= ~(ACTIVE|DEAD);
-        if ((status & FLUSH) && ch->flush) {
-            ch->flush(&ch->io);
-            status &= ~FLUSH;
-        }
+    }
+
+    if ((status & FLUSH) && ch->flush) {
+        ch->flush(&ch->io);
+        status &= ~FLUSH;
     }
 
     DBDMA_DPRINTF("    status 0x%08x\n", status);
@@ -603,9 +606,6 @@ dbdma_control_write(DBDMA_channel *ch)
     if (status & ACTIVE) {
         DBDMA_kick(dbdma_from_ch(ch));
     }
-    if ((status & FLUSH) && ch->flush) {
-        ch->flush(&ch->io);
-    }
 }
 
 static void dbdma_write(void *opaque, hwaddr addr,
@@ -715,20 +715,52 @@ static const MemoryRegionOps dbdma_ops = {
     },
 };
 
-static const VMStateDescription vmstate_dbdma_channel = {
-    .name = "dbdma_channel",
+static const VMStateDescription vmstate_dbdma_io = {
+    .name = "dbdma_io",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(addr, struct DBDMA_io),
+        VMSTATE_INT32(len, struct DBDMA_io),
+        VMSTATE_INT32(is_last, struct DBDMA_io),
+        VMSTATE_INT32(is_dma_out, struct DBDMA_io),
+        VMSTATE_BOOL(processing, struct DBDMA_io),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_dbdma_cmd = {
+    .name = "dbdma_cmd",
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
+        VMSTATE_UINT16(req_count, dbdma_cmd),
+        VMSTATE_UINT16(command, dbdma_cmd),
+        VMSTATE_UINT32(phy_addr, dbdma_cmd),
+        VMSTATE_UINT32(cmd_dep, dbdma_cmd),
+        VMSTATE_UINT16(res_count, dbdma_cmd),
+        VMSTATE_UINT16(xfer_status, dbdma_cmd),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_dbdma_channel = {
+    .name = "dbdma_channel",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
         VMSTATE_UINT32_ARRAY(regs, struct DBDMA_channel, DBDMA_REGS),
+        VMSTATE_STRUCT(io, struct DBDMA_channel, 0, vmstate_dbdma_io, DBDMA_io),
+        VMSTATE_STRUCT(current, struct DBDMA_channel, 0, vmstate_dbdma_cmd,
+                       dbdma_cmd),
         VMSTATE_END_OF_LIST()
     }
 };
 
 static const VMStateDescription vmstate_dbdma = {
     .name = "dbdma",
-    .version_id = 2,
-    .minimum_version_id = 2,
+    .version_id = 3,
+    .minimum_version_id = 3,
     .fields = (VMStateField[]) {
         VMSTATE_STRUCT_ARRAY(channels, DBDMAState, DBDMA_CHANNELS, 1,
                              vmstate_dbdma_channel, DBDMA_channel),
@@ -745,6 +777,20 @@ static void dbdma_reset(void *opaque)
         memset(s->channels[i].regs, 0, DBDMA_SIZE);
 }
 
+static void dbdma_unassigned_rw(DBDMA_io *io)
+{
+    DBDMA_channel *ch = io->channel;
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: use of unassigned channel %d\n",
+                  __func__, ch->channel);
+}
+
+static void dbdma_unassigned_flush(DBDMA_io *io)
+{
+    DBDMA_channel *ch = io->channel;
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: use of unassigned channel %d\n",
+                  __func__, ch->channel);
+}
+
 void* DBDMA_init (MemoryRegion **dbdma_mem)
 {
     DBDMAState *s;
@@ -754,7 +800,13 @@ void* DBDMA_init (MemoryRegion **dbdma_mem)
 
     for (i = 0; i < DBDMA_CHANNELS; i++) {
         DBDMA_io *io = &s->channels[i].io;
+        DBDMA_channel *ch = &s->channels[i];
         qemu_iovec_init(&io->iov, 1);
+
+        ch->rw = dbdma_unassigned_rw;
+        ch->flush = dbdma_unassigned_flush;
+        ch->channel = i;
+        ch->io.channel = ch;
     }
 
     memory_region_init_io(&s->mem, NULL, &dbdma_ops, s, "dbdma", 0x1000);
diff --git a/qemu/hw/misc/macio/macio.c b/qemu/hw/misc/macio/macio.c
index e3c0242d4..be03926b9 100644
--- a/qemu/hw/misc/macio/macio.c
+++ b/qemu/hw/misc/macio/macio.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -105,10 +107,10 @@ static void macio_escc_legacy_setup(MacIOState *macio_state)
         0xF0, 0xE0,
     };
 
-    memory_region_init(escc_legacy, NULL, "escc-legacy", 256);
+    memory_region_init(escc_legacy, OBJECT(macio_state), "escc-legacy", 256);
     for (i = 0; i < ARRAY_SIZE(maps); i += 2) {
         MemoryRegion *port = g_new(MemoryRegion, 1);
-        memory_region_init_alias(port, NULL, "escc-legacy-port",
+        memory_region_init_alias(port, OBJECT(macio_state), "escc-legacy-port",
                                  macio_state->escc_mem, maps[i+1], 0x2);
         memory_region_add_subregion(escc_legacy, maps[i], port);
     }
@@ -131,6 +133,10 @@ static void macio_common_realize(PCIDevice *d, Error **errp)
     MacIOState *s = MACIO(d);
     SysBusDevice *sysbus_dev;
     Error *err = NULL;
+    MemoryRegion *dbdma_mem;
+
+    s->dbdma = DBDMA_init(&dbdma_mem);
+    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
 
     object_property_set_bool(OBJECT(&s->cuda), true, "realized", &err);
     if (err) {
@@ -248,7 +254,7 @@ static uint64_t timer_read(void *opaque, hwaddr addr, unsigned size)
     uint64_t systime = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     uint64_t kltime;
 
-    kltime = muldiv64(systime, 4194300, get_ticks_per_sec() * 4);
+    kltime = muldiv64(systime, 4194300, NANOSECONDS_PER_SECOND * 4);
     kltime = muldiv64(kltime, 18432000, 1048575);
 
     switch (addr) {
@@ -328,16 +334,12 @@ static void macio_newworld_init(Object *obj)
 static void macio_instance_init(Object *obj)
 {
     MacIOState *s = MACIO(obj);
-    MemoryRegion *dbdma_mem;
 
-    memory_region_init(&s->bar, NULL, "macio", 0x80000);
+    memory_region_init(&s->bar, obj, "macio", 0x80000);
 
     object_initialize(&s->cuda, sizeof(s->cuda), TYPE_CUDA);
     qdev_set_parent_bus(DEVICE(&s->cuda), sysbus_get_default());
     object_property_add_child(obj, "cuda", OBJECT(&s->cuda), NULL);
-
-    s->dbdma = DBDMA_init(&dbdma_mem);
-    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
 }
 
 static const VMStateDescription vmstate_macio_oldworld = {
@@ -393,6 +395,7 @@ static void macio_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
     dc->props = macio_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo macio_oldworld_type_info = {
diff --git a/qemu/hw/misc/max111x.c b/qemu/hw/misc/max111x.c
index bef3651d6..9014f0f70 100644
--- a/qemu/hw/misc/max111x.c
+++ b/qemu/hw/misc/max111x.c
@@ -10,7 +10,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 
 typedef struct {
     SSISlave parent_obj;
diff --git a/qemu/hw/misc/milkymist-hpdmc.c b/qemu/hw/misc/milkymist-hpdmc.c
index f5f4c1b34..b97000fc4 100644
--- a/qemu/hw/misc/milkymist-hpdmc.c
+++ b/qemu/hw/misc/milkymist-hpdmc.c
@@ -21,6 +21,7 @@
  *   http://www.milkymist.org/socdoc/hpdmc.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/misc/milkymist-pfpu.c b/qemu/hw/misc/milkymist-pfpu.c
index 08b604f13..57acd7b36 100644
--- a/qemu/hw/misc/milkymist-pfpu.c
+++ b/qemu/hw/misc/milkymist-pfpu.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/misc/mips_cmgcr.c b/qemu/hw/misc/mips_cmgcr.c
new file mode 100644
index 000000000..37be23995
--- /dev/null
+++ b/qemu/hw/misc/mips_cmgcr.c
@@ -0,0 +1,160 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
+ * Authors: Sanjay Lal <sanjayl@kymasys.com>
+ *
+ * Copyright (C) 2015 Imagination Technologies
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "hw/misc/mips_cmgcr.h"
+#include "hw/misc/mips_cpc.h"
+
+static inline bool is_cpc_connected(MIPSGCRState *s)
+{
+    return s->cpc_mr != NULL;
+}
+
+static inline void update_cpc_base(MIPSGCRState *gcr, uint64_t val)
+{
+    if (is_cpc_connected(gcr)) {
+        gcr->cpc_base = val & GCR_CPC_BASE_MSK;
+        memory_region_transaction_begin();
+        memory_region_set_address(gcr->cpc_mr,
+                                  gcr->cpc_base & GCR_CPC_BASE_CPCBASE_MSK);
+        memory_region_set_enabled(gcr->cpc_mr,
+                                  gcr->cpc_base & GCR_CPC_BASE_CPCEN_MSK);
+        memory_region_transaction_commit();
+    }
+}
+
+/* Read GCR registers */
+static uint64_t gcr_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSGCRState *gcr = (MIPSGCRState *) opaque;
+
+    switch (addr) {
+    /* Global Control Block Register */
+    case GCR_CONFIG_OFS:
+        /* Set PCORES to 0 */
+        return 0;
+    case GCR_BASE_OFS:
+        return gcr->gcr_base;
+    case GCR_REV_OFS:
+        return gcr->gcr_rev;
+    case GCR_CPC_BASE_OFS:
+        return gcr->cpc_base;
+    case GCR_CPC_STATUS_OFS:
+        return is_cpc_connected(gcr);
+    case GCR_L2_CONFIG_OFS:
+        /* L2 BYPASS */
+        return GCR_L2_CONFIG_BYPASS_MSK;
+        /* Core-Local and Core-Other Control Blocks */
+    case MIPS_CLCB_OFS + GCR_CL_CONFIG_OFS:
+    case MIPS_COCB_OFS + GCR_CL_CONFIG_OFS:
+        /* Set PVP to # of VPs - 1 */
+        return gcr->num_vps - 1;
+    case MIPS_CLCB_OFS + GCR_CL_OTHER_OFS:
+        return 0;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Read %d bytes at GCR offset 0x%" HWADDR_PRIx
+                      "\n", size, addr);
+        return 0;
+    }
+    return 0;
+}
+
+/* Write GCR registers */
+static void gcr_write(void *opaque, hwaddr addr, uint64_t data, unsigned size)
+{
+    MIPSGCRState *gcr = (MIPSGCRState *)opaque;
+
+    switch (addr) {
+    case GCR_CPC_BASE_OFS:
+        update_cpc_base(gcr, data);
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Write %d bytes at GCR offset 0x%" HWADDR_PRIx
+                      " 0x%" PRIx64 "\n", size, addr, data);
+        break;
+    }
+}
+
+static const MemoryRegionOps gcr_ops = {
+    .read = gcr_read,
+    .write = gcr_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .max_access_size = 8,
+    },
+};
+
+static void mips_gcr_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSGCRState *s = MIPS_GCR(obj);
+
+    object_property_add_link(obj, "cpc", TYPE_MEMORY_REGION,
+                             (Object **)&s->cpc_mr,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &gcr_ops, s,
+                          "mips-gcr", GCR_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void mips_gcr_reset(DeviceState *dev)
+{
+    MIPSGCRState *s = MIPS_GCR(dev);
+
+    update_cpc_base(s, 0);
+}
+
+static const VMStateDescription vmstate_mips_gcr = {
+    .name = "mips-gcr",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc_base, MIPSGCRState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property mips_gcr_properties[] = {
+    DEFINE_PROP_INT32("num-vp", MIPSGCRState, num_vps, 1),
+    DEFINE_PROP_INT32("gcr-rev", MIPSGCRState, gcr_rev, 0x800),
+    DEFINE_PROP_UINT64("gcr-base", MIPSGCRState, gcr_base, GCR_BASE_ADDR),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_gcr_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->props = mips_gcr_properties;
+    dc->vmsd = &vmstate_mips_gcr;
+    dc->reset = mips_gcr_reset;
+}
+
+static const TypeInfo mips_gcr_info = {
+    .name          = TYPE_MIPS_GCR,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSGCRState),
+    .instance_init = mips_gcr_init,
+    .class_init    = mips_gcr_class_init,
+};
+
+static void mips_gcr_register_types(void)
+{
+    type_register_static(&mips_gcr_info);
+}
+
+type_init(mips_gcr_register_types)
diff --git a/qemu/hw/misc/mips_cpc.c b/qemu/hw/misc/mips_cpc.c
new file mode 100644
index 000000000..d2b8e42da
--- /dev/null
+++ b/qemu/hw/misc/mips_cpc.c
@@ -0,0 +1,177 @@
+/*
+ * Cluster Power Controller emulation
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+
+#include "hw/misc/mips_cpc.h"
+
+static inline uint64_t cpc_vp_run_mask(MIPSCPCState *cpc)
+{
+    return (1ULL << cpc->num_vp) - 1;
+}
+
+static void cpc_run_vp(MIPSCPCState *cpc, uint64_t vp_run)
+{
+    CPUState *cs = first_cpu;
+
+    CPU_FOREACH(cs) {
+        uint64_t i = 1ULL << cs->cpu_index;
+        if (i & vp_run & ~cpc->vp_running) {
+            cpu_interrupt(cs, CPU_INTERRUPT_WAKE);
+            cpc->vp_running |= i;
+        }
+    }
+}
+
+static void cpc_stop_vp(MIPSCPCState *cpc, uint64_t vp_stop)
+{
+    CPUState *cs = first_cpu;
+
+    CPU_FOREACH(cs) {
+        uint64_t i = 1ULL << cs->cpu_index;
+        if (i & vp_stop & cpc->vp_running) {
+            cs->halted = 1;
+            cpu_reset_interrupt(cs, CPU_INTERRUPT_WAKE);
+            cpc->vp_running &= ~i;
+        }
+    }
+}
+
+static void cpc_write(void *opaque, hwaddr offset, uint64_t data,
+                      unsigned size)
+{
+    MIPSCPCState *s = opaque;
+
+    switch (offset) {
+    case CPC_CL_BASE_OFS + CPC_VP_RUN_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_RUN_OFS:
+        cpc_run_vp(s, data & cpc_vp_run_mask(s));
+        break;
+    case CPC_CL_BASE_OFS + CPC_VP_STOP_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_STOP_OFS:
+        cpc_stop_vp(s, data & cpc_vp_run_mask(s));
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Bad offset 0x%x\n",  __func__, (int)offset);
+        break;
+    }
+
+    return;
+}
+
+static uint64_t cpc_read(void *opaque, hwaddr offset, unsigned size)
+{
+    MIPSCPCState *s = opaque;
+
+    switch (offset) {
+    case CPC_CL_BASE_OFS + CPC_VP_RUNNING_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_RUNNING_OFS:
+        return s->vp_running;
+    default:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Bad offset 0x%x\n",  __func__, (int)offset);
+        return 0;
+    }
+}
+
+static const MemoryRegionOps cpc_ops = {
+    .read = cpc_read,
+    .write = cpc_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .max_access_size = 8,
+    },
+};
+
+static void mips_cpc_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSCPCState *s = MIPS_CPC(obj);
+
+    memory_region_init_io(&s->mr, OBJECT(s), &cpc_ops, s, "mips-cpc",
+                          CPC_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->mr);
+}
+
+static void mips_cpc_realize(DeviceState *dev, Error **errp)
+{
+    MIPSCPCState *s = MIPS_CPC(dev);
+
+    if (s->vp_start_running > cpc_vp_run_mask(s)) {
+        error_setg(errp,
+                   "incorrect vp_start_running 0x%" PRIx64 " for num_vp = %d",
+                   s->vp_running, s->num_vp);
+        return;
+    }
+}
+
+static void mips_cpc_reset(DeviceState *dev)
+{
+    MIPSCPCState *s = MIPS_CPC(dev);
+
+    /* Reflect the fact that all VPs are halted on reset */
+    s->vp_running = 0;
+
+    /* Put selected VPs into run state */
+    cpc_run_vp(s, s->vp_start_running);
+}
+
+static const VMStateDescription vmstate_mips_cpc = {
+    .name = "mips-cpc",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(vp_running, MIPSCPCState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property mips_cpc_properties[] = {
+    DEFINE_PROP_UINT32("num-vp", MIPSCPCState, num_vp, 0x1),
+    DEFINE_PROP_UINT64("vp-start-running", MIPSCPCState, vp_start_running, 0x1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_cpc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = mips_cpc_realize;
+    dc->reset = mips_cpc_reset;
+    dc->vmsd = &vmstate_mips_cpc;
+    dc->props = mips_cpc_properties;
+}
+
+static const TypeInfo mips_cpc_info = {
+    .name          = TYPE_MIPS_CPC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSCPCState),
+    .instance_init = mips_cpc_init,
+    .class_init    = mips_cpc_class_init,
+};
+
+static void mips_cpc_register_types(void)
+{
+    type_register_static(&mips_cpc_info);
+}
+
+type_init(mips_cpc_register_types)
diff --git a/qemu/hw/misc/mips_itu.c b/qemu/hw/misc/mips_itu.c
new file mode 100644
index 000000000..da5455062
--- /dev/null
+++ b/qemu/hw/misc/mips_itu.c
@@ -0,0 +1,521 @@
+/*
+ * Inter-Thread Communication Unit emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "hw/misc/mips_itu.h"
+
+#define ITC_TAG_ADDRSPACE_SZ (ITC_ADDRESSMAP_NUM * 8)
+/* Initialize as 4kB area to fit all 32 cells with default 128B grain.
+   Storage may be resized by the software. */
+#define ITC_STORAGE_ADDRSPACE_SZ 0x1000
+
+#define ITC_FIFO_NUM_MAX 16
+#define ITC_SEMAPH_NUM_MAX 16
+#define ITC_AM1_NUMENTRIES_OFS 20
+
+#define ITC_CELL_PV_MAX_VAL 0xFFFF
+
+#define ITC_CELL_TAG_FIFO_DEPTH 28
+#define ITC_CELL_TAG_FIFO_PTR 18
+#define ITC_CELL_TAG_FIFO 17
+#define ITC_CELL_TAG_T 16
+#define ITC_CELL_TAG_F 1
+#define ITC_CELL_TAG_E 0
+
+#define ITC_AM0_BASE_ADDRESS_MASK 0xFFFFFC00ULL
+#define ITC_AM0_EN_MASK 0x1
+
+#define ITC_AM1_ADDR_MASK_MASK 0x1FC00
+#define ITC_AM1_ENTRY_GRAIN_MASK 0x7
+
+typedef enum ITCView {
+    ITCVIEW_BYPASS  = 0,
+    ITCVIEW_CONTROL = 1,
+    ITCVIEW_EF_SYNC = 2,
+    ITCVIEW_EF_TRY  = 3,
+    ITCVIEW_PV_SYNC = 4,
+    ITCVIEW_PV_TRY  = 5
+} ITCView;
+
+MemoryRegion *mips_itu_get_tag_region(MIPSITUState *itu)
+{
+    return &itu->tag_io;
+}
+
+static uint64_t itc_tag_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSITUState *tag = (MIPSITUState *)opaque;
+    uint64_t index = addr >> 3;
+
+    if (index >= ITC_ADDRESSMAP_NUM) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Read 0x%" PRIx64 "\n", addr);
+        return 0;
+    }
+
+    return tag->ITCAddressMap[index];
+}
+
+static void itc_reconfigure(MIPSITUState *tag)
+{
+    uint64_t *am = &tag->ITCAddressMap[0];
+    MemoryRegion *mr = &tag->storage_io;
+    hwaddr address = am[0] & ITC_AM0_BASE_ADDRESS_MASK;
+    uint64_t size = (1 << 10) + (am[1] & ITC_AM1_ADDR_MASK_MASK);
+    bool is_enabled = (am[0] & ITC_AM0_EN_MASK) != 0;
+
+    memory_region_transaction_begin();
+    if (!(size & (size - 1))) {
+        memory_region_set_size(mr, size);
+    }
+    memory_region_set_address(mr, address);
+    memory_region_set_enabled(mr, is_enabled);
+    memory_region_transaction_commit();
+}
+
+static void itc_tag_write(void *opaque, hwaddr addr,
+                          uint64_t data, unsigned size)
+{
+    MIPSITUState *tag = (MIPSITUState *)opaque;
+    uint64_t *am = &tag->ITCAddressMap[0];
+    uint64_t am_old, mask;
+    uint64_t index = addr >> 3;
+
+    switch (index) {
+    case 0:
+        mask = ITC_AM0_BASE_ADDRESS_MASK | ITC_AM0_EN_MASK;
+        break;
+    case 1:
+        mask = ITC_AM1_ADDR_MASK_MASK | ITC_AM1_ENTRY_GRAIN_MASK;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "Bad write 0x%" PRIx64 "\n", addr);
+        return;
+    }
+
+    am_old = am[index];
+    am[index] = (data & mask) | (am_old & ~mask);
+    if (am_old != am[index]) {
+        itc_reconfigure(tag);
+    }
+}
+
+static const MemoryRegionOps itc_tag_ops = {
+    .read = itc_tag_read,
+    .write = itc_tag_write,
+    .impl = {
+        .max_access_size = 8,
+    },
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static inline uint32_t get_num_cells(MIPSITUState *s)
+{
+    return s->num_fifo + s->num_semaphores;
+}
+
+static inline ITCView get_itc_view(hwaddr addr)
+{
+    return (addr >> 3) & 0xf;
+}
+
+static inline int get_cell_stride_shift(const MIPSITUState *s)
+{
+    /* Minimum interval (for EntryGain = 0) is 128 B */
+    return 7 + (s->ITCAddressMap[1] & ITC_AM1_ENTRY_GRAIN_MASK);
+}
+
+static inline ITCStorageCell *get_cell(MIPSITUState *s,
+                                       hwaddr addr)
+{
+    uint32_t cell_idx = addr >> get_cell_stride_shift(s);
+    uint32_t num_cells = get_num_cells(s);
+
+    if (cell_idx >= num_cells) {
+        cell_idx = num_cells - 1;
+    }
+
+    return &s->cell[cell_idx];
+}
+
+static void wake_blocked_threads(ITCStorageCell *c)
+{
+    CPUState *cs;
+    CPU_FOREACH(cs) {
+        if (cs->halted && (c->blocked_threads & (1ULL << cs->cpu_index))) {
+            cpu_interrupt(cs, CPU_INTERRUPT_WAKE);
+        }
+    }
+    c->blocked_threads = 0;
+}
+
+static void QEMU_NORETURN block_thread_and_exit(ITCStorageCell *c)
+{
+    c->blocked_threads |= 1ULL << current_cpu->cpu_index;
+    cpu_restore_state(current_cpu, current_cpu->mem_io_pc);
+    current_cpu->halted = 1;
+    current_cpu->exception_index = EXCP_HLT;
+    cpu_loop_exit(current_cpu);
+}
+
+/* ITC Bypass View */
+
+static inline uint64_t view_bypass_read(ITCStorageCell *c)
+{
+    if (c->tag.FIFO) {
+        return c->data[c->fifo_out];
+    } else {
+        return c->data[0];
+    }
+}
+
+static inline void view_bypass_write(ITCStorageCell *c, uint64_t val)
+{
+    if (c->tag.FIFO && (c->tag.FIFOPtr > 0)) {
+        int idx = (c->fifo_out + c->tag.FIFOPtr - 1) % ITC_CELL_DEPTH;
+        c->data[idx] = val;
+    }
+
+    /* ignore a write to the semaphore cell */
+}
+
+/* ITC Control View */
+
+static inline uint64_t view_control_read(ITCStorageCell *c)
+{
+    return ((uint64_t)c->tag.FIFODepth << ITC_CELL_TAG_FIFO_DEPTH) |
+           (c->tag.FIFOPtr << ITC_CELL_TAG_FIFO_PTR) |
+           (c->tag.FIFO << ITC_CELL_TAG_FIFO) |
+           (c->tag.T << ITC_CELL_TAG_T) |
+           (c->tag.E << ITC_CELL_TAG_E) |
+           (c->tag.F << ITC_CELL_TAG_F);
+}
+
+static inline void view_control_write(ITCStorageCell *c, uint64_t val)
+{
+    c->tag.T = (val >> ITC_CELL_TAG_T) & 1;
+    c->tag.E = (val >> ITC_CELL_TAG_E) & 1;
+    c->tag.F = (val >> ITC_CELL_TAG_F) & 1;
+
+    if (c->tag.E) {
+        c->tag.FIFOPtr = 0;
+    }
+}
+
+/* ITC Empty/Full View */
+
+static uint64_t view_ef_common_read(ITCStorageCell *c, bool blocking)
+{
+    uint64_t ret = 0;
+
+    if (!c->tag.FIFO) {
+        return 0;
+    }
+
+    c->tag.F = 0;
+
+    if (blocking && c->tag.E) {
+        block_thread_and_exit(c);
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+
+    if (c->tag.FIFOPtr > 0) {
+        ret = c->data[c->fifo_out];
+        c->fifo_out = (c->fifo_out + 1) % ITC_CELL_DEPTH;
+        c->tag.FIFOPtr--;
+    }
+
+    if (c->tag.FIFOPtr == 0) {
+        c->tag.E = 1;
+    }
+
+    return ret;
+}
+
+static uint64_t view_ef_sync_read(ITCStorageCell *c)
+{
+    return view_ef_common_read(c, true);
+}
+
+static uint64_t view_ef_try_read(ITCStorageCell *c)
+{
+    return view_ef_common_read(c, false);
+}
+
+static inline void view_ef_common_write(ITCStorageCell *c, uint64_t val,
+                                        bool blocking)
+{
+    if (!c->tag.FIFO) {
+        return;
+    }
+
+    c->tag.E = 0;
+
+    if (blocking && c->tag.F) {
+        block_thread_and_exit(c);
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+
+    if (c->tag.FIFOPtr < ITC_CELL_DEPTH) {
+        int idx = (c->fifo_out + c->tag.FIFOPtr) % ITC_CELL_DEPTH;
+        c->data[idx] = val;
+        c->tag.FIFOPtr++;
+    }
+
+    if (c->tag.FIFOPtr == ITC_CELL_DEPTH) {
+        c->tag.F = 1;
+    }
+}
+
+static void view_ef_sync_write(ITCStorageCell *c, uint64_t val)
+{
+    view_ef_common_write(c, val, true);
+}
+
+static void view_ef_try_write(ITCStorageCell *c, uint64_t val)
+{
+    view_ef_common_write(c, val, false);
+}
+
+/* ITC P/V View */
+
+static uint64_t view_pv_common_read(ITCStorageCell *c, bool blocking)
+{
+    uint64_t ret = c->data[0];
+
+    if (c->tag.FIFO) {
+        return 0;
+    }
+
+    if (c->data[0] > 0) {
+        c->data[0]--;
+    } else if (blocking) {
+        block_thread_and_exit(c);
+    }
+
+    return ret;
+}
+
+static uint64_t view_pv_sync_read(ITCStorageCell *c)
+{
+    return view_pv_common_read(c, true);
+}
+
+static uint64_t view_pv_try_read(ITCStorageCell *c)
+{
+    return view_pv_common_read(c, false);
+}
+
+static inline void view_pv_common_write(ITCStorageCell *c)
+{
+    if (c->tag.FIFO) {
+        return;
+    }
+
+    if (c->data[0] < ITC_CELL_PV_MAX_VAL) {
+        c->data[0]++;
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+}
+
+static void view_pv_sync_write(ITCStorageCell *c)
+{
+    view_pv_common_write(c);
+}
+
+static void view_pv_try_write(ITCStorageCell *c)
+{
+    view_pv_common_write(c);
+}
+
+static uint64_t itc_storage_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSITUState *s = (MIPSITUState *)opaque;
+    ITCStorageCell *cell = get_cell(s, addr);
+    ITCView view = get_itc_view(addr);
+    uint64_t ret = -1;
+
+    switch (view) {
+    case ITCVIEW_BYPASS:
+        ret = view_bypass_read(cell);
+        break;
+    case ITCVIEW_CONTROL:
+        ret = view_control_read(cell);
+        break;
+    case ITCVIEW_EF_SYNC:
+        ret = view_ef_sync_read(cell);
+        break;
+    case ITCVIEW_EF_TRY:
+        ret = view_ef_try_read(cell);
+        break;
+    case ITCVIEW_PV_SYNC:
+        ret = view_pv_sync_read(cell);
+        break;
+    case ITCVIEW_PV_TRY:
+        ret = view_pv_try_read(cell);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "itc_storage_read: Bad ITC View %d\n", (int)view);
+        break;
+    }
+
+    return ret;
+}
+
+static void itc_storage_write(void *opaque, hwaddr addr, uint64_t data,
+                              unsigned size)
+{
+    MIPSITUState *s = (MIPSITUState *)opaque;
+    ITCStorageCell *cell = get_cell(s, addr);
+    ITCView view = get_itc_view(addr);
+
+    switch (view) {
+    case ITCVIEW_BYPASS:
+        view_bypass_write(cell, data);
+        break;
+    case ITCVIEW_CONTROL:
+        view_control_write(cell, data);
+        break;
+    case ITCVIEW_EF_SYNC:
+        view_ef_sync_write(cell, data);
+        break;
+    case ITCVIEW_EF_TRY:
+        view_ef_try_write(cell, data);
+        break;
+    case ITCVIEW_PV_SYNC:
+        view_pv_sync_write(cell);
+        break;
+    case ITCVIEW_PV_TRY:
+        view_pv_try_write(cell);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "itc_storage_write: Bad ITC View %d\n", (int)view);
+        break;
+    }
+
+}
+
+static const MemoryRegionOps itc_storage_ops = {
+    .read = itc_storage_read,
+    .write = itc_storage_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void itc_reset_cells(MIPSITUState *s)
+{
+    int i;
+
+    memset(s->cell, 0, get_num_cells(s) * sizeof(s->cell[0]));
+
+    for (i = 0; i < s->num_fifo; i++) {
+        s->cell[i].tag.E = 1;
+        s->cell[i].tag.FIFO = 1;
+        s->cell[i].tag.FIFODepth = ITC_CELL_DEPTH_SHIFT;
+    }
+}
+
+static void mips_itu_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSITUState *s = MIPS_ITU(obj);
+
+    memory_region_init_io(&s->storage_io, OBJECT(s), &itc_storage_ops, s,
+                          "mips-itc-storage", ITC_STORAGE_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->storage_io);
+
+    memory_region_init_io(&s->tag_io, OBJECT(s), &itc_tag_ops, s,
+                          "mips-itc-tag", ITC_TAG_ADDRSPACE_SZ);
+}
+
+static void mips_itu_realize(DeviceState *dev, Error **errp)
+{
+    MIPSITUState *s = MIPS_ITU(dev);
+
+    if (s->num_fifo > ITC_FIFO_NUM_MAX) {
+        error_setg(errp, "Exceed maximum number of FIFO cells: %d",
+                   s->num_fifo);
+        return;
+    }
+    if (s->num_semaphores > ITC_SEMAPH_NUM_MAX) {
+        error_setg(errp, "Exceed maximum number of Semaphore cells: %d",
+                   s->num_semaphores);
+        return;
+    }
+
+    s->cell = g_new(ITCStorageCell, get_num_cells(s));
+}
+
+static void mips_itu_reset(DeviceState *dev)
+{
+    MIPSITUState *s = MIPS_ITU(dev);
+
+    s->ITCAddressMap[0] = 0;
+    s->ITCAddressMap[1] =
+        ((ITC_STORAGE_ADDRSPACE_SZ - 1) & ITC_AM1_ADDR_MASK_MASK) |
+        (get_num_cells(s) << ITC_AM1_NUMENTRIES_OFS);
+    itc_reconfigure(s);
+
+    itc_reset_cells(s);
+}
+
+static Property mips_itu_properties[] = {
+    DEFINE_PROP_INT32("num-fifo", MIPSITUState, num_fifo,
+                      ITC_FIFO_NUM_MAX),
+    DEFINE_PROP_INT32("num-semaphores", MIPSITUState, num_semaphores,
+                      ITC_SEMAPH_NUM_MAX),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_itu_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = mips_itu_properties;
+    dc->realize = mips_itu_realize;
+    dc->reset = mips_itu_reset;
+}
+
+static const TypeInfo mips_itu_info = {
+    .name          = TYPE_MIPS_ITU,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSITUState),
+    .instance_init = mips_itu_init,
+    .class_init    = mips_itu_class_init,
+};
+
+static void mips_itu_register_types(void)
+{
+    type_register_static(&mips_itu_info);
+}
+
+type_init(mips_itu_register_types)
diff --git a/qemu/hw/misc/mst_fpga.c b/qemu/hw/misc/mst_fpga.c
index d5090799f..48d7dfb2d 100644
--- a/qemu/hw/misc/mst_fpga.c
+++ b/qemu/hw/misc/mst_fpga.c
@@ -10,6 +10,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/qemu/hw/misc/omap_clk.c b/qemu/hw/misc/omap_clk.c
index 80a3c50e1..19151d07d 100644
--- a/qemu/hw/misc/omap_clk.c
+++ b/qemu/hw/misc/omap_clk.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -1239,7 +1240,7 @@ void omap_clk_init(struct omap_mpu_state_s *mpu)
     for (i = onchip_clks, count = 0; *i; i ++)
         if ((*i)->flags & flag)
             count ++;
-    mpu->clks = (struct clk *) g_malloc0(sizeof(struct clk) * (count + 1));
+    mpu->clks = g_new0(struct clk, count + 1);
     for (i = onchip_clks, j = mpu->clks; *i; i ++)
         if ((*i)->flags & flag) {
             memcpy(j, *i, sizeof(struct clk));
diff --git a/qemu/hw/misc/omap_gpmc.c b/qemu/hw/misc/omap_gpmc.c
index 74fc91c8e..67d8e2f02 100644
--- a/qemu/hw/misc/omap_gpmc.c
+++ b/qemu/hw/misc/omap_gpmc.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "hw/arm/omap.h"
@@ -826,8 +827,7 @@ struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
                                    qemu_irq irq, qemu_irq drq)
 {
     int cs;
-    struct omap_gpmc_s *s = (struct omap_gpmc_s *)
-            g_malloc0(sizeof(struct omap_gpmc_s));
+    struct omap_gpmc_s *s = g_new0(struct omap_gpmc_s, 1);
 
     memory_region_init_io(&s->iomem, NULL, &omap_gpmc_ops, s, "omap-gpmc", 0x1000);
     memory_region_add_subregion(get_system_memory(), base, &s->iomem);
diff --git a/qemu/hw/misc/omap_l4.c b/qemu/hw/misc/omap_l4.c
index 245ceac84..88c533a0f 100644
--- a/qemu/hw/misc/omap_l4.c
+++ b/qemu/hw/misc/omap_l4.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
diff --git a/qemu/hw/misc/omap_sdrc.c b/qemu/hw/misc/omap_sdrc.c
index 3de0c0e9d..dff37ecaf 100644
--- a/qemu/hw/misc/omap_sdrc.c
+++ b/qemu/hw/misc/omap_sdrc.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -157,8 +158,7 @@ static const MemoryRegionOps omap_sdrc_ops = {
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
                                    hwaddr base)
 {
-    struct omap_sdrc_s *s = (struct omap_sdrc_s *)
-            g_malloc0(sizeof(struct omap_sdrc_s));
+    struct omap_sdrc_s *s = g_new0(struct omap_sdrc_s, 1);
 
     omap_sdrc_reset(s);
 
diff --git a/qemu/hw/misc/omap_tap.c b/qemu/hw/misc/omap_tap.c
index 6f02bb9e4..e6ea8ee23 100644
--- a/qemu/hw/misc/omap_tap.c
+++ b/qemu/hw/misc/omap_tap.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
diff --git a/qemu/hw/misc/pc-testdev.c b/qemu/hw/misc/pc-testdev.c
index 18e94e07b..086893dcc 100644
--- a/qemu/hw/misc/pc-testdev.c
+++ b/qemu/hw/misc/pc-testdev.c
@@ -35,7 +35,7 @@
  * git://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git
 */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 #if defined(CONFIG_POSIX)
 #include <sys/mman.h>
 #endif
diff --git a/qemu/hw/misc/pci-testdev.c b/qemu/hw/misc/pci-testdev.c
index 26b9b8617..2f2e98977 100644
--- a/qemu/hw/misc/pci-testdev.c
+++ b/qemu/hw/misc/pci-testdev.c
@@ -17,10 +17,10 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "qemu/event_notifier.h"
-#include "qemu/osdep.h"
 
 typedef struct PCITestDevHdr {
     uint8_t test;
@@ -239,6 +239,7 @@ static void pci_testdev_realize(PCIDevice *pci_dev, Error **errp)
     uint8_t *pci_conf;
     char *name;
     int r, i;
+    bool fastmmio = kvm_ioeventfd_any_length_enabled();
 
     pci_conf = pci_dev->config;
 
@@ -261,8 +262,12 @@ static void pci_testdev_realize(PCIDevice *pci_dev, Error **errp)
         memcpy(test->hdr->name, name, strlen(name) + 1);
         g_free(name);
         test->hdr->offset = cpu_to_le32(IOTEST_SIZE(i) + i * IOTEST_ACCESS_WIDTH);
-        test->size = IOTEST_ACCESS_WIDTH;
         test->match_data = strcmp(IOTEST_TEST(i), "wildcard-eventfd");
+        if (fastmmio && IOTEST_IS_MEM(i) && !test->match_data) {
+            test->size = 0;
+        } else {
+            test->size = IOTEST_ACCESS_WIDTH;
+        }
         test->hdr->test = i;
         test->hdr->data = test->match_data ? IOTEST_DATAMATCH : IOTEST_NOMATCH;
         test->hdr->width = IOTEST_ACCESS_WIDTH;
diff --git a/qemu/hw/misc/puv3_pm.c b/qemu/hw/misc/puv3_pm.c
index 37f23695d..577cebaac 100644
--- a/qemu/hw/misc/puv3_pm.c
+++ b/qemu/hw/misc/puv3_pm.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/qemu/hw/misc/pvpanic.c b/qemu/hw/misc/pvpanic.c
index 994f8af8e..0ac1e6ac9 100644
--- a/qemu/hw/misc/pvpanic.c
+++ b/qemu/hw/misc/pvpanic.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qjson.h"
 #include "sysemu/sysemu.h"
@@ -41,8 +42,7 @@ static void handle_event(int event)
     }
 
     if (event & PVPANIC_PANICKED) {
-        qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
-        vm_stop(RUN_STATE_GUEST_PANICKED);
+        qemu_system_guest_panicked();
         return;
     }
 }
diff --git a/qemu/hw/misc/sga.c b/qemu/hw/misc/sga.c
index 83d2fd9d3..03b006d6f 100644
--- a/qemu/hw/misc/sga.c
+++ b/qemu/hw/misc/sga.c
@@ -24,6 +24,7 @@
  * sgabios code originally available at code.google.com/p/sgabios
  *
  */
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
 #include "hw/loader.h"
diff --git a/qemu/hw/misc/slavio_misc.c b/qemu/hw/misc/slavio_misc.c
index ec50f1075..edd5de070 100644
--- a/qemu/hw/misc/slavio_misc.c
+++ b/qemu/hw/misc/slavio_misc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/misc/stm32f2xx_syscfg.c b/qemu/hw/misc/stm32f2xx_syscfg.c
index 4ae4042bf..d0d7076ef 100644
--- a/qemu/hw/misc/stm32f2xx_syscfg.c
+++ b/qemu/hw/misc/stm32f2xx_syscfg.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/misc/stm32f2xx_syscfg.h"
 
 #ifndef STM_SYSCFG_ERR_DEBUG
diff --git a/qemu/hw/misc/tmp105.c b/qemu/hw/misc/tmp105.c
index f3fe8b81f..f5c2472b5 100644
--- a/qemu/hw/misc/tmp105.c
+++ b/qemu/hw/misc/tmp105.c
@@ -18,9 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "tmp105.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 static void tmp105_interrupt_update(TMP105State *s)
@@ -52,26 +54,26 @@ static void tmp105_alarm_update(TMP105State *s)
     tmp105_interrupt_update(s);
 }
 
-static void tmp105_get_temperature(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void tmp105_get_temperature(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     TMP105State *s = TMP105(obj);
     int64_t value = s->temperature * 1000 / 256;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 /* Units are 0.001 centigrades relative to 0 C.  s->temperature is 8.8
  * fixed point, so units are 1/256 centigrades.  A simple ratio will do.
  */
-static void tmp105_set_temperature(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void tmp105_set_temperature(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     TMP105State *s = TMP105(obj);
     Error *local_err = NULL;
     int64_t temp;
 
-    visit_type_int(v, &temp, name, &local_err);
+    visit_type_int(v, name, &temp, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
diff --git a/qemu/hw/misc/vmport.c b/qemu/hw/misc/vmport.c
index cd5716a46..689678980 100644
--- a/qemu/hw/misc/vmport.c
+++ b/qemu/hw/misc/vmport.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/i386/pc.h"
diff --git a/qemu/hw/misc/zynq-xadc.c b/qemu/hw/misc/zynq-xadc.c
new file mode 100644
index 000000000..71fbccd79
--- /dev/null
+++ b/qemu/hw/misc/zynq-xadc.c
@@ -0,0 +1,303 @@
+/*
+ * ADC registers for Xilinx Zynq Platform
+ *
+ * Copyright (c) 2015 Guenter Roeck
+ * Based on hw/misc/zynq_slcr.c, written by Michal Simek
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/misc/zynq-xadc.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+
+enum {
+    CFG                = 0x000 / 4,
+    INT_STS,
+    INT_MASK,
+    MSTS,
+    CMDFIFO,
+    RDFIFO,
+    MCTL,
+};
+
+#define CFG_ENABLE              BIT(31)
+#define CFG_CFIFOTH_SHIFT       20
+#define CFG_CFIFOTH_LENGTH      4
+#define CFG_DFIFOTH_SHIFT       16
+#define CFG_DFIFOTH_LENGTH      4
+#define CFG_WEDGE               BIT(13)
+#define CFG_REDGE               BIT(12)
+#define CFG_TCKRATE_SHIFT       8
+#define CFG_TCKRATE_LENGTH      2
+
+#define CFG_TCKRATE_DIV(x)      (0x1 << (x - 1))
+
+#define CFG_IGAP_SHIFT          0
+#define CFG_IGAP_LENGTH         5
+
+#define INT_CFIFO_LTH           BIT(9)
+#define INT_DFIFO_GTH           BIT(8)
+#define INT_OT                  BIT(7)
+#define INT_ALM_SHIFT           0
+#define INT_ALM_LENGTH          7
+#define INT_ALM_MASK            (((1 << INT_ALM_LENGTH) - 1) << INT_ALM_SHIFT)
+
+#define INT_ALL (INT_CFIFO_LTH | INT_DFIFO_GTH | INT_OT | INT_ALM_MASK)
+
+#define MSTS_CFIFO_LVL_SHIFT    16
+#define MSTS_CFIFO_LVL_LENGTH   4
+#define MSTS_DFIFO_LVL_SHIFT    12
+#define MSTS_DFIFO_LVL_LENGTH   4
+#define MSTS_CFIFOF             BIT(11)
+#define MSTS_CFIFOE             BIT(10)
+#define MSTS_DFIFOF             BIT(9)
+#define MSTS_DFIFOE             BIT(8)
+#define MSTS_OT                 BIT(7)
+#define MSTS_ALM_SHIFT          0
+#define MSTS_ALM_LENGTH         7
+
+#define MCTL_RESET              BIT(4)
+
+#define CMD_NOP                 0x00
+#define CMD_READ                0x01
+#define CMD_WRITE               0x02
+
+static void zynq_xadc_update_ints(ZynqXADCState *s)
+{
+
+    /* We are fast, commands are actioned instantly so the CFIFO is always
+     * empty (and below threshold).
+     */
+    s->regs[INT_STS] |= INT_CFIFO_LTH;
+
+    if (s->xadc_dfifo_entries >
+        extract32(s->regs[CFG], CFG_DFIFOTH_SHIFT, CFG_DFIFOTH_LENGTH)) {
+        s->regs[INT_STS] |= INT_DFIFO_GTH;
+    }
+
+    qemu_set_irq(s->qemu_irq, !!(s->regs[INT_STS] & ~s->regs[INT_MASK]));
+}
+
+static void zynq_xadc_reset(DeviceState *d)
+{
+    ZynqXADCState *s = ZYNQ_XADC(d);
+
+    s->regs[CFG] = 0x14 << CFG_IGAP_SHIFT |
+                   CFG_TCKRATE_DIV(4) << CFG_TCKRATE_SHIFT | CFG_REDGE;
+    s->regs[INT_STS] = INT_CFIFO_LTH;
+    s->regs[INT_MASK] = 0xffffffff;
+    s->regs[CMDFIFO] = 0;
+    s->regs[RDFIFO] = 0;
+    s->regs[MCTL] = MCTL_RESET;
+
+    memset(s->xadc_regs, 0, sizeof(s->xadc_regs));
+    memset(s->xadc_dfifo, 0, sizeof(s->xadc_dfifo));
+    s->xadc_dfifo_entries = 0;
+
+    zynq_xadc_update_ints(s);
+}
+
+static uint16_t xadc_pop_dfifo(ZynqXADCState *s)
+{
+    uint16_t rv = s->xadc_dfifo[0];
+    int i;
+
+    if (s->xadc_dfifo_entries > 0) {
+        s->xadc_dfifo_entries--;
+    }
+    for (i = 0; i < s->xadc_dfifo_entries; i++) {
+        s->xadc_dfifo[i] = s->xadc_dfifo[i + 1];
+    }
+    s->xadc_dfifo[s->xadc_dfifo_entries] = 0;
+    zynq_xadc_update_ints(s);
+    return rv;
+}
+
+static void xadc_push_dfifo(ZynqXADCState *s, uint16_t regval)
+{
+    if (s->xadc_dfifo_entries < ZYNQ_XADC_FIFO_DEPTH) {
+        s->xadc_dfifo[s->xadc_dfifo_entries++] = s->xadc_read_reg_previous;
+    }
+    s->xadc_read_reg_previous = regval;
+    zynq_xadc_update_ints(s);
+}
+
+static bool zynq_xadc_check_offset(hwaddr offset, bool rnw)
+{
+    switch (offset) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        return true;
+    case RDFIFO:
+    case MSTS:
+        return rnw;     /* read only */
+    case CMDFIFO:
+        return !rnw;    /* write only */
+    default:
+        return false;
+    }
+}
+
+static uint64_t zynq_xadc_read(void *opaque, hwaddr offset, unsigned size)
+{
+    ZynqXADCState *s = opaque;
+    int reg = offset / 4;
+    uint32_t rv = 0;
+
+    if (!zynq_xadc_check_offset(reg, true)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid read access to "
+                      "addr %" HWADDR_PRIx "\n", offset);
+        return 0;
+    }
+
+    switch (reg) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        rv = s->regs[reg];
+        break;
+    case MSTS:
+        rv = MSTS_CFIFOE;
+        rv |= s->xadc_dfifo_entries << MSTS_DFIFO_LVL_SHIFT;
+        if (!s->xadc_dfifo_entries) {
+            rv |= MSTS_DFIFOE;
+        } else if (s->xadc_dfifo_entries == ZYNQ_XADC_FIFO_DEPTH) {
+            rv |= MSTS_DFIFOF;
+        }
+        break;
+    case RDFIFO:
+        rv = xadc_pop_dfifo(s);
+        break;
+    }
+    return rv;
+}
+
+static void zynq_xadc_write(void *opaque, hwaddr offset, uint64_t val,
+                            unsigned size)
+{
+    ZynqXADCState *s = (ZynqXADCState *)opaque;
+    int reg = offset / 4;
+    int xadc_reg;
+    int xadc_cmd;
+    int xadc_data;
+
+    if (!zynq_xadc_check_offset(reg, false)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid write access "
+                      "to addr %" HWADDR_PRIx "\n", offset);
+        return;
+    }
+
+    switch (reg) {
+    case CFG:
+        s->regs[CFG] = val;
+        break;
+    case INT_STS:
+        s->regs[INT_STS] &= ~val;
+        break;
+    case INT_MASK:
+        s->regs[INT_MASK] = val & INT_ALL;
+        break;
+    case CMDFIFO:
+        xadc_cmd  = extract32(val, 26,  4);
+        xadc_reg  = extract32(val, 16, 10);
+        xadc_data = extract32(val,  0, 16);
+
+        if (s->regs[MCTL] & MCTL_RESET) {
+            qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Sending command "
+                          "while comm channel held in reset: %" PRIx32 "\n",
+                          (uint32_t) val);
+            break;
+        }
+
+        if (xadc_reg >= ZYNQ_XADC_NUM_ADC_REGS && xadc_cmd != CMD_NOP) {
+            qemu_log_mask(LOG_GUEST_ERROR, "read/write op to invalid xadc "
+                          "reg 0x%x\n", xadc_reg);
+            break;
+        }
+
+        switch (xadc_cmd) {
+        case CMD_READ:
+            xadc_push_dfifo(s, s->xadc_regs[xadc_reg]);
+            break;
+        case CMD_WRITE:
+            s->xadc_regs[xadc_reg] = xadc_data;
+            /* fallthrough */
+        case CMD_NOP:
+            xadc_push_dfifo(s, 0);
+            break;
+        }
+        break;
+    case MCTL:
+        s->regs[MCTL] = val & 0x00fffeff;
+        break;
+    }
+    zynq_xadc_update_ints(s);
+}
+
+static const MemoryRegionOps xadc_ops = {
+    .read = zynq_xadc_read,
+    .write = zynq_xadc_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void zynq_xadc_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    ZynqXADCState *s = ZYNQ_XADC(obj);
+
+    memory_region_init_io(&s->iomem, obj, &xadc_ops, s, "zynq-xadc",
+                          ZYNQ_XADC_MMIO_SIZE);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->qemu_irq);
+}
+
+static const VMStateDescription vmstate_zynq_xadc = {
+    .name = "zynq-xadc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, ZynqXADCState, ZYNQ_XADC_NUM_IO_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_regs, ZynqXADCState,
+                             ZYNQ_XADC_NUM_ADC_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_dfifo, ZynqXADCState,
+                             ZYNQ_XADC_FIFO_DEPTH),
+        VMSTATE_UINT16(xadc_read_reg_previous, ZynqXADCState),
+        VMSTATE_UINT16(xadc_dfifo_entries, ZynqXADCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void zynq_xadc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_zynq_xadc;
+    dc->reset = zynq_xadc_reset;
+}
+
+static const TypeInfo zynq_xadc_info = {
+    .class_init = zynq_xadc_class_init,
+    .name  = TYPE_ZYNQ_XADC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(ZynqXADCState),
+    .instance_init = zynq_xadc_init,
+};
+
+static void zynq_xadc_register_types(void)
+{
+    type_register_static(&zynq_xadc_info);
+}
+
+type_init(zynq_xadc_register_types)
diff --git a/qemu/hw/misc/zynq_slcr.c b/qemu/hw/misc/zynq_slcr.c
index 964f2532f..b1b7591ef 100644
--- a/qemu/hw/misc/zynq_slcr.c
+++ b/qemu/hw/misc/zynq_slcr.c
@@ -14,6 +14,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/sysbus.h"
@@ -393,12 +394,12 @@ static void zynq_slcr_write(void *opaque, hwaddr offset,
         return;
     }
 
-    if (!s->regs[LOCKSTA]) {
-        s->regs[offset / 4] = val;
-    } else {
-        DB_PRINT("SCLR registers are locked. Unlock them first\n");
+    if (s->regs[LOCKSTA]) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "SCLR registers are locked. Unlock them first\n");
         return;
     }
+    s->regs[offset] = val;
 
     switch (offset) {
     case PSS_RST_CTRL:
diff --git a/qemu/hw/moxie/moxiesim.c b/qemu/hw/moxie/moxiesim.c
index 80bcc5b4f..3069834cf 100644
--- a/qemu/hw/moxie/moxiesim.c
+++ b/qemu/hw/moxie/moxiesim.c
@@ -24,6 +24,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
@@ -34,6 +38,7 @@
 #include "hw/loader.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
+#include "elf.h"
 
 #define PHYS_MEM_BASE 0x80000000
 
@@ -52,8 +57,8 @@ static void load_kernel(MoxieCPU *cpu, LoaderParams *loader_params)
     ram_addr_t initrd_offset;
 
     kernel_size = load_elf(loader_params->kernel_filename,  NULL, NULL,
-                           &entry, &kernel_low, &kernel_high, 1,
-                           ELF_MACHINE, 0);
+                           &entry, &kernel_low, &kernel_high, 1, EM_MOXIE,
+                           0, 0);
 
     if (kernel_size <= 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
@@ -123,11 +128,11 @@ static void moxiesim_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, cpu);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, ram_base, ram);
 
-    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0x1000, rom);
 
@@ -146,16 +151,11 @@ static void moxiesim_init(MachineState *machine)
     }
 }
 
-static QEMUMachine moxiesim_machine = {
-    .name = "moxiesim",
-    .desc = "Moxie simulator platform",
-    .init = moxiesim_init,
-    .is_default = 1,
-};
-
-static void moxie_machine_init(void)
+static void moxiesim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&moxiesim_machine);
+    mc->desc = "Moxie simulator platform";
+    mc->init = moxiesim_init;
+    mc->is_default = 1;
 }
 
-machine_init(moxie_machine_init)
+DEFINE_MACHINE("moxiesim", moxiesim_machine_init)
diff --git a/qemu/hw/net/Makefile.objs b/qemu/hw/net/Makefile.objs
index 98801739e..64d044923 100644
--- a/qemu/hw/net/Makefile.objs
+++ b/qemu/hw/net/Makefile.objs
@@ -19,6 +19,7 @@ common-obj-$(CONFIG_XGMAC) += xgmac.o
 common-obj-$(CONFIG_MIPSNET) += mipsnet.o
 common-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
 common-obj-$(CONFIG_ALLWINNER_EMAC) += allwinner_emac.o
+common-obj-$(CONFIG_IMX_FEC) += imx_fec.o
 
 common-obj-$(CONFIG_CADENCE) += cadence_gem.o
 common-obj-$(CONFIG_STELLARIS_ENET) += stellaris_enet.o
diff --git a/qemu/hw/net/allwinner_emac.c b/qemu/hw/net/allwinner_emac.c
index 0407dee6d..16d4b63ba 100644
--- a/qemu/hw/net/allwinner_emac.c
+++ b/qemu/hw/net/allwinner_emac.c
@@ -16,6 +16,7 @@
  * GNU General Public License for more details.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "qemu/fifo8.h"
diff --git a/qemu/hw/net/cadence_gem.c b/qemu/hw/net/cadence_gem.c
index 494a346cf..0346f3e33 100644
--- a/qemu/hw/net/cadence_gem.c
+++ b/qemu/hw/net/cadence_gem.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h> /* For crc32 */
 
 #include "hw/net/cadence_gem.h"
@@ -677,6 +678,10 @@ static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     } else {
         unsigned crc_val;
 
+        if (size > sizeof(rxbuf) - sizeof(crc_val)) {
+            size = sizeof(rxbuf) - sizeof(crc_val);
+        }
+        bytes_to_copy = size;
         /* The application wants the FCS field, which QEMU does not provide.
          * We must try and calculate one.
          */
@@ -862,6 +867,14 @@ static void gem_transmit(CadenceGEMState *s)
             break;
         }
 
+        if (tx_desc_get_length(desc) > sizeof(tx_packet) - (p - tx_packet)) {
+            DB_PRINT("TX descriptor @ 0x%x too large: size 0x%x space 0x%x\n",
+                     (unsigned)packet_desc_addr,
+                     (unsigned)tx_desc_get_length(desc),
+                     sizeof(tx_packet) - (p - tx_packet));
+            break;
+        }
+
         /* Gather this fragment of the packet from "dma memory" to our contig.
          * buffer.
          */
@@ -951,7 +964,7 @@ static void gem_phy_reset(CadenceGEMState *s)
     s->phy_regs[PHY_REG_1000BTSTAT] = 0x7C00;
     s->phy_regs[PHY_REG_EXTSTAT] = 0x3000;
     s->phy_regs[PHY_REG_PHYSPCFC_CTL] = 0x0078;
-    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0xBC00;
+    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0x7C00;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL] = 0x0C60;
     s->phy_regs[PHY_REG_LED] = 0x4100;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL2] = 0x000A;
@@ -964,6 +977,7 @@ static void gem_reset(DeviceState *d)
 {
     int i;
     CadenceGEMState *s = CADENCE_GEM(d);
+    const uint8_t *a;
 
     DB_PRINT("\n");
 
@@ -982,6 +996,11 @@ static void gem_reset(DeviceState *d)
     s->regs[GEM_DESCONF5] = 0x002f2145;
     s->regs[GEM_DESCONF6] = 0x00000200;
 
+    /* Set MAC address */
+    a = &s->conf.macaddr.a[0];
+    s->regs[GEM_SPADDR1LO] = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24);
+    s->regs[GEM_SPADDR1HI] = a[4] | (a[5] << 8);
+
     for (i = 0; i < 4; i++) {
         s->sar_active[i] = false;
     }
diff --git a/qemu/hw/net/dp8393x.c b/qemu/hw/net/dp8393x.c
index ab607e484..0fa652c39 100644
--- a/qemu/hw/net/dp8393x.c
+++ b/qemu/hw/net/dp8393x.c
@@ -17,9 +17,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "net/net.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include <zlib.h>
 
@@ -292,7 +294,7 @@ static void dp8393x_set_next_tick(dp8393xState *s)
 
     ticks = s->regs[SONIC_WT1] << 16 | s->regs[SONIC_WT0];
     s->wt_last_update = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    delay = get_ticks_per_sec() * ticks / 5000000;
+    delay = NANOSECONDS_PER_SECOND * ticks / 5000000;
     timer_mod(s->watchdog, s->wt_last_update + delay);
 }
 
diff --git a/qemu/hw/net/e1000.c b/qemu/hw/net/e1000.c
index 09c9e9d53..8e79b550e 100644
--- a/qemu/hw/net/e1000.c
+++ b/qemu/hw/net/e1000.c
@@ -25,6 +25,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -37,24 +38,26 @@
 
 #include "e1000_regs.h"
 
+static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
 #define E1000_DEBUG
 
 #ifdef E1000_DEBUG
 enum {
-    DEBUG_GENERAL,	DEBUG_IO,	DEBUG_MMIO,	DEBUG_INTERRUPT,
-    DEBUG_RX,		DEBUG_TX,	DEBUG_MDIC,	DEBUG_EEPROM,
-    DEBUG_UNKNOWN,	DEBUG_TXSUM,	DEBUG_TXERR,	DEBUG_RXERR,
+    DEBUG_GENERAL,      DEBUG_IO,       DEBUG_MMIO,     DEBUG_INTERRUPT,
+    DEBUG_RX,           DEBUG_TX,       DEBUG_MDIC,     DEBUG_EEPROM,
+    DEBUG_UNKNOWN,      DEBUG_TXSUM,    DEBUG_TXERR,    DEBUG_RXERR,
     DEBUG_RXFILTER,     DEBUG_PHY,      DEBUG_NOTYET,
 };
-#define DBGBIT(x)	(1<<DEBUG_##x)
+#define DBGBIT(x)    (1<<DEBUG_##x)
 static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 
-#define	DBGOUT(what, fmt, ...) do { \
+#define DBGOUT(what, fmt, ...) do { \
     if (debugflags & DBGBIT(what)) \
         fprintf(stderr, "e1000: " fmt, ## __VA_ARGS__); \
     } while (0)
 #else
-#define	DBGOUT(what, fmt, ...) do {} while (0)
+#define DBGOUT(what, fmt, ...) do {} while (0)
 #endif
 
 #define IOPORT_SIZE       0x40
@@ -118,7 +121,7 @@ typedef struct E1000State_st {
     } tx;
 
     struct {
-        uint32_t val_in;	// shifted in from guest driver
+        uint32_t val_in;    /* shifted in from guest driver */
         uint16_t bitnum_in;
         uint16_t bitnum_out;
         uint16_t reading;
@@ -135,11 +138,15 @@ typedef struct E1000State_st {
 /* Compatibility flags for migration to/from qemu 1.3.0 and older */
 #define E1000_FLAG_AUTONEG_BIT 0
 #define E1000_FLAG_MIT_BIT 1
+#define E1000_FLAG_MAC_BIT 2
 #define E1000_FLAG_AUTONEG (1 << E1000_FLAG_AUTONEG_BIT)
 #define E1000_FLAG_MIT (1 << E1000_FLAG_MIT_BIT)
+#define E1000_FLAG_MAC (1 << E1000_FLAG_MAC_BIT)
     uint32_t compat_flags;
 } E1000State;
 
+#define chkflag(x)     (s->compat_flags & E1000_FLAG_##x)
+
 typedef struct E1000BaseClass {
     PCIDeviceClass parent_class;
     uint16_t phy_id2;
@@ -155,20 +162,36 @@ typedef struct E1000BaseClass {
 #define E1000_DEVICE_GET_CLASS(obj) \
     OBJECT_GET_CLASS(E1000BaseClass, (obj), TYPE_E1000_BASE)
 
-#define	defreg(x)	x = (E1000_##x>>2)
+#define defreg(x)    x = (E1000_##x>>2)
 enum {
-    defreg(CTRL),	defreg(EECD),	defreg(EERD),	defreg(GPRC),
-    defreg(GPTC),	defreg(ICR),	defreg(ICS),	defreg(IMC),
-    defreg(IMS),	defreg(LEDCTL),	defreg(MANC),	defreg(MDIC),
-    defreg(MPC),	defreg(PBA),	defreg(RCTL),	defreg(RDBAH),
-    defreg(RDBAL),	defreg(RDH),	defreg(RDLEN),	defreg(RDT),
-    defreg(STATUS),	defreg(SWSM),	defreg(TCTL),	defreg(TDBAH),
-    defreg(TDBAL),	defreg(TDH),	defreg(TDLEN),	defreg(TDT),
-    defreg(TORH),	defreg(TORL),	defreg(TOTH),	defreg(TOTL),
-    defreg(TPR),	defreg(TPT),	defreg(TXDCTL),	defreg(WUFC),
-    defreg(RA),		defreg(MTA),	defreg(CRCERRS),defreg(VFTA),
-    defreg(VET),        defreg(RDTR),   defreg(RADV),   defreg(TADV),
-    defreg(ITR),
+    defreg(CTRL),    defreg(EECD),    defreg(EERD),    defreg(GPRC),
+    defreg(GPTC),    defreg(ICR),     defreg(ICS),     defreg(IMC),
+    defreg(IMS),     defreg(LEDCTL),  defreg(MANC),    defreg(MDIC),
+    defreg(MPC),     defreg(PBA),     defreg(RCTL),    defreg(RDBAH),
+    defreg(RDBAL),   defreg(RDH),     defreg(RDLEN),   defreg(RDT),
+    defreg(STATUS),  defreg(SWSM),    defreg(TCTL),    defreg(TDBAH),
+    defreg(TDBAL),   defreg(TDH),     defreg(TDLEN),   defreg(TDT),
+    defreg(TORH),    defreg(TORL),    defreg(TOTH),    defreg(TOTL),
+    defreg(TPR),     defreg(TPT),     defreg(TXDCTL),  defreg(WUFC),
+    defreg(RA),      defreg(MTA),     defreg(CRCERRS), defreg(VFTA),
+    defreg(VET),     defreg(RDTR),    defreg(RADV),    defreg(TADV),
+    defreg(ITR),     defreg(FCRUC),   defreg(TDFH),    defreg(TDFT),
+    defreg(TDFHS),   defreg(TDFTS),   defreg(TDFPC),   defreg(RDFH),
+    defreg(RDFT),    defreg(RDFHS),   defreg(RDFTS),   defreg(RDFPC),
+    defreg(IPAV),    defreg(WUC),     defreg(WUS),     defreg(AIT),
+    defreg(IP6AT),   defreg(IP4AT),   defreg(FFLT),    defreg(FFMT),
+    defreg(FFVT),    defreg(WUPM),    defreg(PBM),     defreg(SCC),
+    defreg(ECOL),    defreg(MCC),     defreg(LATECOL), defreg(COLC),
+    defreg(DC),      defreg(TNCRS),   defreg(SEC),     defreg(CEXTERR),
+    defreg(RLEC),    defreg(XONRXC),  defreg(XONTXC),  defreg(XOFFRXC),
+    defreg(XOFFTXC), defreg(RFC),     defreg(RJC),     defreg(RNBC),
+    defreg(TSCTFC),  defreg(MGTPRC),  defreg(MGTPDC),  defreg(MGTPTC),
+    defreg(RUC),     defreg(ROC),     defreg(GORCL),   defreg(GORCH),
+    defreg(GOTCL),   defreg(GOTCH),   defreg(BPRC),    defreg(MPRC),
+    defreg(TSCTC),   defreg(PRC64),   defreg(PRC127),  defreg(PRC255),
+    defreg(PRC511),  defreg(PRC1023), defreg(PRC1522), defreg(PTC64),
+    defreg(PTC127),  defreg(PTC255),  defreg(PTC511),  defreg(PTC1023),
+    defreg(PTC1522), defreg(MPTC),    defreg(BPTC)
 };
 
 static void
@@ -193,8 +216,7 @@ e1000_link_up(E1000State *s)
 static bool
 have_autoneg(E1000State *s)
 {
-    return (s->compat_flags & E1000_FLAG_AUTONEG) &&
-           (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
+    return chkflag(AUTONEG) && (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
 }
 
 static void
@@ -226,18 +248,18 @@ enum { NPHYWRITEOPS = ARRAY_SIZE(phyreg_writeops) };
 
 enum { PHY_R = 1, PHY_W = 2, PHY_RW = PHY_R | PHY_W };
 static const char phy_regcap[0x20] = {
-    [PHY_STATUS] = PHY_R,	[M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_ID1] = PHY_R,		[M88E1000_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_CTRL] = PHY_RW,	[PHY_1000T_CTRL] = PHY_RW,
-    [PHY_LP_ABILITY] = PHY_R,	[PHY_1000T_STATUS] = PHY_R,
-    [PHY_AUTONEG_ADV] = PHY_RW,	[M88E1000_RX_ERR_CNTR] = PHY_R,
-    [PHY_ID2] = PHY_R,		[M88E1000_PHY_SPEC_STATUS] = PHY_R,
+    [PHY_STATUS]      = PHY_R,     [M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
+    [PHY_ID1]         = PHY_R,     [M88E1000_PHY_SPEC_CTRL]     = PHY_RW,
+    [PHY_CTRL]        = PHY_RW,    [PHY_1000T_CTRL]             = PHY_RW,
+    [PHY_LP_ABILITY]  = PHY_R,     [PHY_1000T_STATUS]           = PHY_R,
+    [PHY_AUTONEG_ADV] = PHY_RW,    [M88E1000_RX_ERR_CNTR]       = PHY_R,
+    [PHY_ID2]         = PHY_R,     [M88E1000_PHY_SPEC_STATUS]   = PHY_R,
     [PHY_AUTONEG_EXP] = PHY_R,
 };
 
 /* PHY_ID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */
 static const uint16_t phy_reg_init[] = {
-    [PHY_CTRL] =   MII_CR_SPEED_SELECT_MSB |
+    [PHY_CTRL]   = MII_CR_SPEED_SELECT_MSB |
                    MII_CR_FULL_DUPLEX |
                    MII_CR_AUTO_NEG_EN,
 
@@ -264,15 +286,15 @@ static const uint16_t phy_reg_init[] = {
 };
 
 static const uint32_t mac_reg_init[] = {
-    [PBA] =     0x00100030,
-    [LEDCTL] =  0x602,
-    [CTRL] =    E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
+    [PBA]     = 0x00100030,
+    [LEDCTL]  = 0x602,
+    [CTRL]    = E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
                 E1000_CTRL_SPD_1000 | E1000_CTRL_SLU,
-    [STATUS] =  0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
+    [STATUS]  = 0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
                 E1000_STATUS_ASDV | E1000_STATUS_MTXCKOK |
                 E1000_STATUS_SPEED_1000 | E1000_STATUS_FD |
                 E1000_STATUS_LU,
-    [MANC] =    E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
+    [MANC]    = E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
                 E1000_MANC_ARP_EN | E1000_MANC_0298_EN |
                 E1000_MANC_RMCP_EN,
 };
@@ -319,7 +341,7 @@ set_interrupt_cause(E1000State *s, int index, uint32_t val)
         if (s->mit_timer_on) {
             return;
         }
-        if (s->compat_flags & E1000_FLAG_MIT) {
+        if (chkflag(MIT)) {
             /* Compute the next mitigation delay according to pending
              * interrupts and the current values of RADV (provided
              * RDTR!=0), TADV and ITR.
@@ -335,6 +357,14 @@ set_interrupt_cause(E1000State *s, int index, uint32_t val)
             }
             mit_update_delay(&mit_delay, s->mac_reg[ITR]);
 
+            /*
+             * According to e1000 SPEC, the Ethernet controller guarantees
+             * a maximum observable interrupt rate of 7813 interrupts/sec.
+             * Thus if mit_delay < 500 then the delay should be set to the
+             * minimum delay possible which is 500.
+             */
+            mit_delay = (mit_delay < 500) ? 500 : mit_delay;
+
             if (mit_delay) {
                 s->mit_timer_on = 1;
                 timer_mod(s->mit_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
@@ -510,17 +540,19 @@ set_eecd(E1000State *s, int index, uint32_t val)
 
     s->eecd_state.old_eecd = val & (E1000_EECD_SK | E1000_EECD_CS |
             E1000_EECD_DI|E1000_EECD_FWE_MASK|E1000_EECD_REQ);
-    if (!(E1000_EECD_CS & val))			// CS inactive; nothing to do
-	return;
-    if (E1000_EECD_CS & (val ^ oldval)) {	// CS rise edge; reset state
-	s->eecd_state.val_in = 0;
-	s->eecd_state.bitnum_in = 0;
-	s->eecd_state.bitnum_out = 0;
-	s->eecd_state.reading = 0;
+    if (!(E1000_EECD_CS & val)) {            /* CS inactive; nothing to do */
+        return;
     }
-    if (!(E1000_EECD_SK & (val ^ oldval)))	// no clock edge
+    if (E1000_EECD_CS & (val ^ oldval)) {    /* CS rise edge; reset state */
+        s->eecd_state.val_in = 0;
+        s->eecd_state.bitnum_in = 0;
+        s->eecd_state.bitnum_out = 0;
+        s->eecd_state.reading = 0;
+    }
+    if (!(E1000_EECD_SK & (val ^ oldval))) {    /* no clock edge */
         return;
-    if (!(E1000_EECD_SK & val)) {		// falling edge
+    }
+    if (!(E1000_EECD_SK & val)) {               /* falling edge */
         s->eecd_state.bitnum_out++;
         return;
     }
@@ -565,6 +597,56 @@ putsum(uint8_t *data, uint32_t n, uint32_t sloc, uint32_t css, uint32_t cse)
     }
 }
 
+static inline void
+inc_reg_if_not_full(E1000State *s, int index)
+{
+    if (s->mac_reg[index] != 0xffffffff) {
+        s->mac_reg[index]++;
+    }
+}
+
+static inline void
+inc_tx_bcast_or_mcast_count(E1000State *s, const unsigned char *arr)
+{
+    if (!memcmp(arr, bcast, sizeof bcast)) {
+        inc_reg_if_not_full(s, BPTC);
+    } else if (arr[0] & 1) {
+        inc_reg_if_not_full(s, MPTC);
+    }
+}
+
+static void
+grow_8reg_if_not_full(E1000State *s, int index, int size)
+{
+    uint64_t sum = s->mac_reg[index] | (uint64_t)s->mac_reg[index+1] << 32;
+
+    if (sum + size < sum) {
+        sum = ~0ULL;
+    } else {
+        sum += size;
+    }
+    s->mac_reg[index] = sum;
+    s->mac_reg[index+1] = sum >> 32;
+}
+
+static void
+increase_size_stats(E1000State *s, const int *size_regs, int size)
+{
+    if (size > 1023) {
+        inc_reg_if_not_full(s, size_regs[5]);
+    } else if (size > 511) {
+        inc_reg_if_not_full(s, size_regs[4]);
+    } else if (size > 255) {
+        inc_reg_if_not_full(s, size_regs[3]);
+    } else if (size > 127) {
+        inc_reg_if_not_full(s, size_regs[2]);
+    } else if (size > 64) {
+        inc_reg_if_not_full(s, size_regs[1]);
+    } else if (size == 64) {
+        inc_reg_if_not_full(s, size_regs[0]);
+    }
+}
+
 static inline int
 vlan_enabled(E1000State *s)
 {
@@ -602,40 +684,49 @@ fcs_len(E1000State *s)
 static void
 e1000_send_packet(E1000State *s, const uint8_t *buf, int size)
 {
+    static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
+                                    PTC1023, PTC1522 };
+
     NetClientState *nc = qemu_get_queue(s->nic);
     if (s->phy_reg[PHY_CTRL] & MII_CR_LOOPBACK) {
         nc->info->receive(nc, buf, size);
     } else {
         qemu_send_packet(nc, buf, size);
     }
+    inc_tx_bcast_or_mcast_count(s, buf);
+    increase_size_stats(s, PTCregs, size);
 }
 
 static void
 xmit_seg(E1000State *s)
 {
     uint16_t len, *sp;
-    unsigned int frames = s->tx.tso_frames, css, sofar, n;
+    unsigned int frames = s->tx.tso_frames, css, sofar;
     struct e1000_tx *tp = &s->tx;
 
     if (tp->tse && tp->cptse) {
         css = tp->ipcss;
         DBGOUT(TXSUM, "frames %d size %d ipcss %d\n",
                frames, tp->size, css);
-        if (tp->ip) {		// IPv4
+        if (tp->ip) {    /* IPv4 */
             stw_be_p(tp->data+css+2, tp->size - css);
             stw_be_p(tp->data+css+4,
-                          be16_to_cpup((uint16_t *)(tp->data+css+4))+frames);
-        } else			// IPv6
+                     be16_to_cpup((uint16_t *)(tp->data+css+4))+frames);
+        } else {         /* IPv6 */
             stw_be_p(tp->data+css+4, tp->size - css);
+        }
         css = tp->tucss;
         len = tp->size - css;
         DBGOUT(TXSUM, "tcp %d tucss %d len %d\n", tp->tcp, css, len);
         if (tp->tcp) {
             sofar = frames * tp->mss;
             stl_be_p(tp->data+css+4, ldl_be_p(tp->data+css+4)+sofar); /* seq */
-            if (tp->paylen - sofar > tp->mss)
-                tp->data[css + 13] &= ~9;		// PSH, FIN
-        } else	// UDP
+            if (tp->paylen - sofar > tp->mss) {
+                tp->data[css + 13] &= ~9;    /* PSH, FIN */
+            } else if (frames) {
+                inc_reg_if_not_full(s, TSCTC);
+            }
+        } else    /* UDP */
             stw_be_p(tp->data+css+4, len);
         if (tp->sum_needed & E1000_TXD_POPTS_TXSM) {
             unsigned int phsum;
@@ -657,13 +748,15 @@ xmit_seg(E1000State *s)
         memmove(tp->data, tp->data + 4, 8);
         memcpy(tp->data + 8, tp->vlan_header, 4);
         e1000_send_packet(s, tp->vlan, tp->size + 4);
-    } else
+    } else {
         e1000_send_packet(s, tp->data, tp->size);
-    s->mac_reg[TPT]++;
-    s->mac_reg[GPTC]++;
-    n = s->mac_reg[TOTL];
-    if ((s->mac_reg[TOTL] += s->tx.size) < n)
-        s->mac_reg[TOTH]++;
+    }
+
+    inc_reg_if_not_full(s, TPT);
+    grow_8reg_if_not_full(s, TOTL, s->tx.size);
+    s->mac_reg[GPTC] = s->mac_reg[TPT];
+    s->mac_reg[GOTCL] = s->mac_reg[TOTL];
+    s->mac_reg[GOTCH] = s->mac_reg[TOTH];
 }
 
 static void
@@ -679,7 +772,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
     struct e1000_tx *tp = &s->tx;
 
     s->mit_ide |= (txd_lower & E1000_TXD_CMD_IDE);
-    if (dtype == E1000_TXD_CMD_DEXT) {	// context descriptor
+    if (dtype == E1000_TXD_CMD_DEXT) {    /* context descriptor */
         op = le32_to_cpu(xp->cmd_and_length);
         tp->ipcss = xp->lower_setup.ip_fields.ipcss;
         tp->ipcso = xp->lower_setup.ip_fields.ipcso;
@@ -694,7 +787,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
         tp->tcp = (op & E1000_TXD_CMD_TCP) ? 1 : 0;
         tp->tse = (op & E1000_TXD_CMD_TSE) ? 1 : 0;
         tp->tso_frames = 0;
-        if (tp->tucso == 0) {	// this is probably wrong
+        if (tp->tucso == 0) {    /* this is probably wrong */
             DBGOUT(TXSUM, "TCP/UDP: cso 0!\n");
             tp->tucso = tp->tucss + (tp->tcp ? 16 : 6);
         }
@@ -718,7 +811,7 @@ process_tx_desc(E1000State *s, struct e1000_tx_desc *dp)
         stw_be_p(tp->vlan_header + 2,
                       le16_to_cpu(dp->upper.fields.special));
     }
-        
+
     addr = le64_to_cpu(dp->buffer_addr);
     if (tp->tse && tp->cptse) {
         msh = tp->hdr_len + tp->mss;
@@ -819,7 +912,8 @@ start_xmit(E1000State *s)
          * bogus values to TDT/TDLEN.
          * there's nothing too intelligent we could do about this.
          */
-        if (s->mac_reg[TDH] == tdh_start) {
+        if (s->mac_reg[TDH] == tdh_start ||
+            tdh_start >= s->mac_reg[TDLEN] / sizeof(desc)) {
             DBGOUT(TXERR, "TDH wraparound @%x, TDT %x, TDLEN %x\n",
                    tdh_start, s->mac_reg[TDT], s->mac_reg[TDLEN]);
             break;
@@ -831,9 +925,9 @@ start_xmit(E1000State *s)
 static int
 receive_filter(E1000State *s, const uint8_t *buf, int size)
 {
-    static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
     static const int mta_shift[] = {4, 3, 2, 0};
     uint32_t f, rctl = s->mac_reg[RCTL], ra[2], *rp;
+    int isbcast = !memcmp(buf, bcast, sizeof bcast), ismcast = (buf[0] & 1);
 
     if (is_vlan_packet(s, buf) && vlan_rx_filter_enabled(s)) {
         uint16_t vid = be16_to_cpup((uint16_t *)(buf + 14));
@@ -843,14 +937,19 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
             return 0;
     }
 
-    if (rctl & E1000_RCTL_UPE)			// promiscuous
+    if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */
         return 1;
+    }
 
-    if ((buf[0] & 1) && (rctl & E1000_RCTL_MPE))	// promiscuous mcast
+    if (ismcast && (rctl & E1000_RCTL_MPE)) {          /* promiscuous mcast */
+        inc_reg_if_not_full(s, MPRC);
         return 1;
+    }
 
-    if ((rctl & E1000_RCTL_BAM) && !memcmp(buf, bcast, sizeof bcast))
+    if (isbcast && (rctl & E1000_RCTL_BAM)) {          /* broadcast enabled */
+        inc_reg_if_not_full(s, BPRC);
         return 1;
+    }
 
     for (rp = s->mac_reg + RA; rp < s->mac_reg + RA + 32; rp += 2) {
         if (!(rp[1] & E1000_RAH_AV))
@@ -870,8 +969,10 @@ receive_filter(E1000State *s, const uint8_t *buf, int size)
 
     f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
     f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff;
-    if (s->mac_reg[MTA + (f >> 5)] & (1 << (f & 0x1f)))
+    if (s->mac_reg[MTA + (f >> 5)] & (1 << (f & 0x1f))) {
+        inc_reg_if_not_full(s, MPRC);
         return 1;
+    }
     DBGOUT(RXFILTER,
            "dropping, inexact filter mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] %x\n",
            buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
@@ -960,6 +1061,8 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
     size_t desc_offset;
     size_t desc_size;
     size_t total_size;
+    static const int PRCregs[6] = { PRC64, PRC127, PRC255, PRC511,
+                                    PRC1023, PRC1522 };
 
     if (!(s->mac_reg[STATUS] & E1000_STATUS_LU)) {
         return -1;
@@ -973,6 +1076,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
     if (size < sizeof(min_buf)) {
         iov_to_buf(iov, iovcnt, 0, min_buf, size);
         memset(&min_buf[size], 0, sizeof(min_buf) - size);
+        inc_reg_if_not_full(s, RUC);
         min_iov.iov_base = filter_buf = min_buf;
         min_iov.iov_len = size = sizeof(min_buf);
         iovcnt = 1;
@@ -988,6 +1092,7 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         (size > MAXIMUM_ETHERNET_VLAN_SIZE
         && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)))
         && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
+        inc_reg_if_not_full(s, ROC);
         return size;
     }
 
@@ -1065,7 +1170,8 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         if (++s->mac_reg[RDH] * sizeof(desc) >= s->mac_reg[RDLEN])
             s->mac_reg[RDH] = 0;
         /* see comment in start_xmit; same here */
-        if (s->mac_reg[RDH] == rdh_start) {
+        if (s->mac_reg[RDH] == rdh_start ||
+            rdh_start >= s->mac_reg[RDLEN] / sizeof(desc)) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
                    rdh_start, s->mac_reg[RDT], s->mac_reg[RDLEN]);
             set_ics(s, 0, E1000_ICS_RXO);
@@ -1073,16 +1179,17 @@ e1000_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
         }
     } while (desc_offset < total_size);
 
-    s->mac_reg[GPRC]++;
-    s->mac_reg[TPR]++;
+    increase_size_stats(s, PRCregs, total_size);
+    inc_reg_if_not_full(s, TPR);
+    s->mac_reg[GPRC] = s->mac_reg[TPR];
     /* TOR - Total Octets Received:
      * This register includes bytes received in a packet from the <Destination
      * Address> field through the <CRC> field, inclusively.
+     * Always include FCS length (4) in size.
      */
-    n = s->mac_reg[TORL] + size + /* Always include FCS length. */ 4;
-    if (n < s->mac_reg[TORL])
-        s->mac_reg[TORH]++;
-    s->mac_reg[TORL] = n;
+    grow_8reg_if_not_full(s, TORL, size+4);
+    s->mac_reg[GORCL] = s->mac_reg[TORL];
+    s->mac_reg[GORCH] = s->mac_reg[TORH];
 
     n = E1000_ICS_RXT0;
     if ((rdt = s->mac_reg[RDT]) < s->mac_reg[RDH])
@@ -1114,6 +1221,30 @@ mac_readreg(E1000State *s, int index)
 }
 
 static uint32_t
+mac_low4_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xf;
+}
+
+static uint32_t
+mac_low11_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x7ff;
+}
+
+static uint32_t
+mac_low13_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x1fff;
+}
+
+static uint32_t
+mac_low16_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xffff;
+}
+
+static uint32_t
 mac_icr_read(E1000State *s, int index)
 {
     uint32_t ret = s->mac_reg[ICR];
@@ -1206,46 +1337,144 @@ set_ims(E1000State *s, int index, uint32_t val)
     set_ics(s, 0, 0);
 }
 
-#define getreg(x)	[x] = mac_readreg
+#define getreg(x)    [x] = mac_readreg
 static uint32_t (*macreg_readops[])(E1000State *, int) = {
-    getreg(PBA),	getreg(RCTL),	getreg(TDH),	getreg(TXDCTL),
-    getreg(WUFC),	getreg(TDT),	getreg(CTRL),	getreg(LEDCTL),
-    getreg(MANC),	getreg(MDIC),	getreg(SWSM),	getreg(STATUS),
-    getreg(TORL),	getreg(TOTL),	getreg(IMS),	getreg(TCTL),
-    getreg(RDH),	getreg(RDT),	getreg(VET),	getreg(ICS),
-    getreg(TDBAL),	getreg(TDBAH),	getreg(RDBAH),	getreg(RDBAL),
-    getreg(TDLEN),      getreg(RDLEN),  getreg(RDTR),   getreg(RADV),
-    getreg(TADV),       getreg(ITR),
-
-    [TOTH] = mac_read_clr8,	[TORH] = mac_read_clr8,	[GPRC] = mac_read_clr4,
-    [GPTC] = mac_read_clr4,	[TPR] = mac_read_clr4,	[TPT] = mac_read_clr4,
-    [ICR] = mac_icr_read,	[EECD] = get_eecd,	[EERD] = flash_eerd_read,
-    [CRCERRS ... MPC] = &mac_readreg,
-    [RA ... RA+31] = &mac_readreg,
-    [MTA ... MTA+127] = &mac_readreg,
+    getreg(PBA),      getreg(RCTL),     getreg(TDH),      getreg(TXDCTL),
+    getreg(WUFC),     getreg(TDT),      getreg(CTRL),     getreg(LEDCTL),
+    getreg(MANC),     getreg(MDIC),     getreg(SWSM),     getreg(STATUS),
+    getreg(TORL),     getreg(TOTL),     getreg(IMS),      getreg(TCTL),
+    getreg(RDH),      getreg(RDT),      getreg(VET),      getreg(ICS),
+    getreg(TDBAL),    getreg(TDBAH),    getreg(RDBAH),    getreg(RDBAL),
+    getreg(TDLEN),    getreg(RDLEN),    getreg(RDTR),     getreg(RADV),
+    getreg(TADV),     getreg(ITR),      getreg(FCRUC),    getreg(IPAV),
+    getreg(WUC),      getreg(WUS),      getreg(SCC),      getreg(ECOL),
+    getreg(MCC),      getreg(LATECOL),  getreg(COLC),     getreg(DC),
+    getreg(TNCRS),    getreg(SEC),      getreg(CEXTERR),  getreg(RLEC),
+    getreg(XONRXC),   getreg(XONTXC),   getreg(XOFFRXC),  getreg(XOFFTXC),
+    getreg(RFC),      getreg(RJC),      getreg(RNBC),     getreg(TSCTFC),
+    getreg(MGTPRC),   getreg(MGTPDC),   getreg(MGTPTC),   getreg(GORCL),
+    getreg(GOTCL),
+
+    [TOTH]    = mac_read_clr8,      [TORH]    = mac_read_clr8,
+    [GOTCH]   = mac_read_clr8,      [GORCH]   = mac_read_clr8,
+    [PRC64]   = mac_read_clr4,      [PRC127]  = mac_read_clr4,
+    [PRC255]  = mac_read_clr4,      [PRC511]  = mac_read_clr4,
+    [PRC1023] = mac_read_clr4,      [PRC1522] = mac_read_clr4,
+    [PTC64]   = mac_read_clr4,      [PTC127]  = mac_read_clr4,
+    [PTC255]  = mac_read_clr4,      [PTC511]  = mac_read_clr4,
+    [PTC1023] = mac_read_clr4,      [PTC1522] = mac_read_clr4,
+    [GPRC]    = mac_read_clr4,      [GPTC]    = mac_read_clr4,
+    [TPT]     = mac_read_clr4,      [TPR]     = mac_read_clr4,
+    [RUC]     = mac_read_clr4,      [ROC]     = mac_read_clr4,
+    [BPRC]    = mac_read_clr4,      [MPRC]    = mac_read_clr4,
+    [TSCTC]   = mac_read_clr4,      [BPTC]    = mac_read_clr4,
+    [MPTC]    = mac_read_clr4,
+    [ICR]     = mac_icr_read,       [EECD]    = get_eecd,
+    [EERD]    = flash_eerd_read,
+    [RDFH]    = mac_low13_read,     [RDFT]    = mac_low13_read,
+    [RDFHS]   = mac_low13_read,     [RDFTS]   = mac_low13_read,
+    [RDFPC]   = mac_low13_read,
+    [TDFH]    = mac_low11_read,     [TDFT]    = mac_low11_read,
+    [TDFHS]   = mac_low13_read,     [TDFTS]   = mac_low13_read,
+    [TDFPC]   = mac_low13_read,
+    [AIT]     = mac_low16_read,
+
+    [CRCERRS ... MPC]   = &mac_readreg,
+    [IP6AT ... IP6AT+3] = &mac_readreg,    [IP4AT ... IP4AT+6] = &mac_readreg,
+    [FFLT ... FFLT+6]   = &mac_low11_read,
+    [RA ... RA+31]      = &mac_readreg,
+    [WUPM ... WUPM+31]  = &mac_readreg,
+    [MTA ... MTA+127]   = &mac_readreg,
     [VFTA ... VFTA+127] = &mac_readreg,
+    [FFMT ... FFMT+254] = &mac_low4_read,
+    [FFVT ... FFVT+254] = &mac_readreg,
+    [PBM ... PBM+16383] = &mac_readreg,
 };
 enum { NREADOPS = ARRAY_SIZE(macreg_readops) };
 
-#define putreg(x)	[x] = mac_writereg
+#define putreg(x)    [x] = mac_writereg
 static void (*macreg_writeops[])(E1000State *, int, uint32_t) = {
-    putreg(PBA),	putreg(EERD),	putreg(SWSM),	putreg(WUFC),
-    putreg(TDBAL),	putreg(TDBAH),	putreg(TXDCTL),	putreg(RDBAH),
-    putreg(RDBAL),	putreg(LEDCTL), putreg(VET),
-    [TDLEN] = set_dlen,	[RDLEN] = set_dlen,	[TCTL] = set_tctl,
-    [TDT] = set_tctl,	[MDIC] = set_mdic,	[ICS] = set_ics,
-    [TDH] = set_16bit,	[RDH] = set_16bit,	[RDT] = set_rdt,
-    [IMC] = set_imc,	[IMS] = set_ims,	[ICR] = set_icr,
-    [EECD] = set_eecd,	[RCTL] = set_rx_control, [CTRL] = set_ctrl,
-    [RDTR] = set_16bit, [RADV] = set_16bit,     [TADV] = set_16bit,
-    [ITR] = set_16bit,
-    [RA ... RA+31] = &mac_writereg,
-    [MTA ... MTA+127] = &mac_writereg,
+    putreg(PBA),      putreg(EERD),     putreg(SWSM),     putreg(WUFC),
+    putreg(TDBAL),    putreg(TDBAH),    putreg(TXDCTL),   putreg(RDBAH),
+    putreg(RDBAL),    putreg(LEDCTL),   putreg(VET),      putreg(FCRUC),
+    putreg(TDFH),     putreg(TDFT),     putreg(TDFHS),    putreg(TDFTS),
+    putreg(TDFPC),    putreg(RDFH),     putreg(RDFT),     putreg(RDFHS),
+    putreg(RDFTS),    putreg(RDFPC),    putreg(IPAV),     putreg(WUC),
+    putreg(WUS),      putreg(AIT),
+
+    [TDLEN]  = set_dlen,   [RDLEN]  = set_dlen,       [TCTL] = set_tctl,
+    [TDT]    = set_tctl,   [MDIC]   = set_mdic,       [ICS]  = set_ics,
+    [TDH]    = set_16bit,  [RDH]    = set_16bit,      [RDT]  = set_rdt,
+    [IMC]    = set_imc,    [IMS]    = set_ims,        [ICR]  = set_icr,
+    [EECD]   = set_eecd,   [RCTL]   = set_rx_control, [CTRL] = set_ctrl,
+    [RDTR]   = set_16bit,  [RADV]   = set_16bit,      [TADV] = set_16bit,
+    [ITR]    = set_16bit,
+
+    [IP6AT ... IP6AT+3] = &mac_writereg, [IP4AT ... IP4AT+6] = &mac_writereg,
+    [FFLT ... FFLT+6]   = &mac_writereg,
+    [RA ... RA+31]      = &mac_writereg,
+    [WUPM ... WUPM+31]  = &mac_writereg,
+    [MTA ... MTA+127]   = &mac_writereg,
     [VFTA ... VFTA+127] = &mac_writereg,
+    [FFMT ... FFMT+254] = &mac_writereg, [FFVT ... FFVT+254] = &mac_writereg,
+    [PBM ... PBM+16383] = &mac_writereg,
 };
 
 enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) };
 
+enum { MAC_ACCESS_PARTIAL = 1, MAC_ACCESS_FLAG_NEEDED = 2 };
+
+#define markflag(x)    ((E1000_FLAG_##x << 2) | MAC_ACCESS_FLAG_NEEDED)
+/* In the array below the meaning of the bits is: [f|f|f|f|f|f|n|p]
+ * f - flag bits (up to 6 possible flags)
+ * n - flag needed
+ * p - partially implenented */
+static const uint8_t mac_reg_access[0x8000] = {
+    [RDTR]    = markflag(MIT),    [TADV]    = markflag(MIT),
+    [RADV]    = markflag(MIT),    [ITR]     = markflag(MIT),
+
+    [IPAV]    = markflag(MAC),    [WUC]     = markflag(MAC),
+    [IP6AT]   = markflag(MAC),    [IP4AT]   = markflag(MAC),
+    [FFVT]    = markflag(MAC),    [WUPM]    = markflag(MAC),
+    [ECOL]    = markflag(MAC),    [MCC]     = markflag(MAC),
+    [DC]      = markflag(MAC),    [TNCRS]   = markflag(MAC),
+    [RLEC]    = markflag(MAC),    [XONRXC]  = markflag(MAC),
+    [XOFFTXC] = markflag(MAC),    [RFC]     = markflag(MAC),
+    [TSCTFC]  = markflag(MAC),    [MGTPRC]  = markflag(MAC),
+    [WUS]     = markflag(MAC),    [AIT]     = markflag(MAC),
+    [FFLT]    = markflag(MAC),    [FFMT]    = markflag(MAC),
+    [SCC]     = markflag(MAC),    [FCRUC]   = markflag(MAC),
+    [LATECOL] = markflag(MAC),    [COLC]    = markflag(MAC),
+    [SEC]     = markflag(MAC),    [CEXTERR] = markflag(MAC),
+    [XONTXC]  = markflag(MAC),    [XOFFRXC] = markflag(MAC),
+    [RJC]     = markflag(MAC),    [RNBC]    = markflag(MAC),
+    [MGTPDC]  = markflag(MAC),    [MGTPTC]  = markflag(MAC),
+    [RUC]     = markflag(MAC),    [ROC]     = markflag(MAC),
+    [GORCL]   = markflag(MAC),    [GORCH]   = markflag(MAC),
+    [GOTCL]   = markflag(MAC),    [GOTCH]   = markflag(MAC),
+    [BPRC]    = markflag(MAC),    [MPRC]    = markflag(MAC),
+    [TSCTC]   = markflag(MAC),    [PRC64]   = markflag(MAC),
+    [PRC127]  = markflag(MAC),    [PRC255]  = markflag(MAC),
+    [PRC511]  = markflag(MAC),    [PRC1023] = markflag(MAC),
+    [PRC1522] = markflag(MAC),    [PTC64]   = markflag(MAC),
+    [PTC127]  = markflag(MAC),    [PTC255]  = markflag(MAC),
+    [PTC511]  = markflag(MAC),    [PTC1023] = markflag(MAC),
+    [PTC1522] = markflag(MAC),    [MPTC]    = markflag(MAC),
+    [BPTC]    = markflag(MAC),
+
+    [TDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [PBM]   = markflag(MAC) | MAC_ACCESS_PARTIAL,
+};
+
 static void
 e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned size)
@@ -1254,9 +1483,20 @@ e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
     unsigned int index = (addr & 0x1ffff) >> 2;
 
     if (index < NWRITEOPS && macreg_writeops[index]) {
-        macreg_writeops[index](s, index, val);
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Writing to register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            macreg_writeops[index](s, index, val);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO write attempt to disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
     } else if (index < NREADOPS && macreg_readops[index]) {
-        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n", index<<2, val);
+        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n",
+               index<<2, val);
     } else {
         DBGOUT(UNKNOWN, "MMIO unknown write addr=0x%08x,val=0x%08"PRIx64"\n",
                index<<2, val);
@@ -1269,11 +1509,21 @@ e1000_mmio_read(void *opaque, hwaddr addr, unsigned size)
     E1000State *s = opaque;
     unsigned int index = (addr & 0x1ffff) >> 2;
 
-    if (index < NREADOPS && macreg_readops[index])
-    {
-        return macreg_readops[index](s, index);
+    if (index < NREADOPS && macreg_readops[index]) {
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Reading register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            return macreg_readops[index](s, index);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO read attempt of disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
+    } else {
+        DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     }
-    DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     return 0;
 }
 
@@ -1340,7 +1590,7 @@ static int e1000_post_load(void *opaque, int version_id)
     E1000State *s = opaque;
     NetClientState *nc = qemu_get_queue(s->nic);
 
-    if (!(s->compat_flags & E1000_FLAG_MIT)) {
+    if (!chkflag(MIT)) {
         s->mac_reg[ITR] = s->mac_reg[RDTR] = s->mac_reg[RADV] =
             s->mac_reg[TADV] = 0;
         s->mit_irq_level = false;
@@ -1367,7 +1617,14 @@ static bool e1000_mit_state_needed(void *opaque)
 {
     E1000State *s = opaque;
 
-    return s->compat_flags & E1000_FLAG_MIT;
+    return chkflag(MIT);
+}
+
+static bool e1000_full_mac_needed(void *opaque)
+{
+    E1000State *s = opaque;
+
+    return chkflag(MAC);
 }
 
 static const VMStateDescription vmstate_e1000_mit_state = {
@@ -1385,6 +1642,17 @@ static const VMStateDescription vmstate_e1000_mit_state = {
     }
 };
 
+static const VMStateDescription vmstate_e1000_full_mac_state = {
+    .name = "e1000/full_mac_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = e1000_full_mac_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(mac_reg, E1000State, 0x8000),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_e1000 = {
     .name = "e1000",
     .version_id = 2,
@@ -1464,6 +1732,7 @@ static const VMStateDescription vmstate_e1000 = {
     },
     .subsections = (const VMStateDescription*[]) {
         &vmstate_e1000_mit_state,
+        &vmstate_e1000_full_mac_state,
         NULL
     }
 };
@@ -1596,6 +1865,8 @@ static Property e1000_properties[] = {
                     compat_flags, E1000_FLAG_AUTONEG_BIT, true),
     DEFINE_PROP_BIT("mitigation", E1000State,
                     compat_flags, E1000_FLAG_MIT_BIT, true),
+    DEFINE_PROP_BIT("extra_mac_registers", E1000State,
+                    compat_flags, E1000_FLAG_MAC_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1647,7 +1918,7 @@ static const TypeInfo e1000_base_info = {
 
 static const E1000Info e1000_devices[] = {
     {
-        .name      = "e1000-82540em",
+        .name      = "e1000",
         .device_id = E1000_DEV_ID_82540EM,
         .revision  = 0x03,
         .phy_id2   = E1000_PHY_ID2_8254xx_DEFAULT,
@@ -1666,11 +1937,6 @@ static const E1000Info e1000_devices[] = {
     },
 };
 
-static const TypeInfo e1000_default_info = {
-    .name          = "e1000",
-    .parent        = "e1000-82540em",
-};
-
 static void e1000_register_types(void)
 {
     int i;
@@ -1688,7 +1954,6 @@ static void e1000_register_types(void)
 
         type_register(&type_info);
     }
-    type_register_static(&e1000_default_info);
 }
 
 type_init(e1000_register_types)
diff --git a/qemu/hw/net/e1000_regs.h b/qemu/hw/net/e1000_regs.h
index 60b96aaf1..1c40244ab 100644
--- a/qemu/hw/net/e1000_regs.h
+++ b/qemu/hw/net/e1000_regs.h
@@ -158,7 +158,8 @@
 #define E1000_PHY_CTRL     0x00F10  /* PHY Control Register in CSR */
 #define FEXTNVM_SW_CONFIG  0x0001
 #define E1000_PBA      0x01000  /* Packet Buffer Allocation - RW */
-#define E1000_PBS      0x01008  /* Packet Buffer Size */
+#define E1000_PBM      0x10000  /* Packet Buffer Memory - RW */
+#define E1000_PBS      0x01008  /* Packet Buffer Size - RW */
 #define E1000_EEMNGCTL 0x01010  /* MNG EEprom Control */
 #define E1000_FLASH_UPDATES 1000
 #define E1000_EEARBC   0x01024  /* EEPROM Auto Read Bus Control */
@@ -191,6 +192,11 @@
 #define E1000_RAID     0x02C08  /* Receive Ack Interrupt Delay - RW */
 #define E1000_TXDMAC   0x03000  /* TX DMA Control - RW */
 #define E1000_KABGTXD  0x03004  /* AFE Band Gap Transmit Ref Data */
+#define E1000_RDFH     0x02410  /* Receive Data FIFO Head Register - RW */
+#define E1000_RDFT     0x02418  /* Receive Data FIFO Tail Register - RW */
+#define E1000_RDFHS    0x02420  /* Receive Data FIFO Head Saved Register - RW */
+#define E1000_RDFTS    0x02428  /* Receive Data FIFO Tail Saved Register - RW */
+#define E1000_RDFPC    0x02430  /* Receive Data FIFO Packet Count - RW */
 #define E1000_TDFH     0x03410  /* TX Data FIFO Head - RW */
 #define E1000_TDFT     0x03418  /* TX Data FIFO Tail - RW */
 #define E1000_TDFHS    0x03420  /* TX Data FIFO Head Saved - RW */
diff --git a/qemu/hw/net/eepro100.c b/qemu/hw/net/eepro100.c
index 60333b7fc..9b4b9b59d 100644
--- a/qemu/hw/net/eepro100.c
+++ b/qemu/hw/net/eepro100.c
@@ -40,7 +40,7 @@
  *      * Wake-on-LAN is not implemented.
  */
 
-#include <stddef.h>             /* offsetof */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -774,6 +774,11 @@ static void tx_command(EEPRO100State *s)
 #if 0
         uint16_t tx_buffer_el = lduw_le_pci_dma(&s->dev, tbd_address + 6);
 #endif
+        if (tx_buffer_size == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
         tbd_address += 8;
         TRACE(RXTX, logout
             ("TBD (simplified mode): buffer address 0x%08x, size 0x%04x\n",
@@ -855,6 +860,10 @@ static void set_multicast_list(EEPRO100State *s)
 
 static void action_command(EEPRO100State *s)
 {
+    /* The loop below won't stop if it gets special handcrafted data.
+       Therefore we limit the number of iterations. */
+    unsigned max_loop_count = 16;
+
     for (;;) {
         bool bit_el;
         bool bit_s;
@@ -870,6 +879,13 @@ static void action_command(EEPRO100State *s)
 #if 0
         bool bit_sf = ((s->tx.command & COMMAND_SF) != 0);
 #endif
+
+        if (max_loop_count-- == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
+
         s->cu_offset = s->tx.link;
         TRACE(OTHER,
               logout("val=(cu start), status=0x%04x, command=0x%04x, link=0x%08x\n",
diff --git a/qemu/hw/net/etraxfs_eth.c b/qemu/hw/net/etraxfs_eth.c
index d6002750f..05495ec40 100644
--- a/qemu/hw/net/etraxfs_eth.c
+++ b/qemu/hw/net/etraxfs_eth.c
@@ -22,10 +22,11 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/cris/etraxfs.h"
+#include "qemu/error-report.h"
 
 #define D(x)
 
@@ -589,7 +590,8 @@ static int fs_eth_init(SysBusDevice *sbd)
     ETRAXFSEthState *s = ETRAX_FS_ETH(dev);
 
     if (!s->dma_out || !s->dma_in) {
-        hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
+        error_report("Unconnected ETRAX-FS Ethernet MAC");
+        return -1;
     }
 
     s->dma_out->client.push = eth_tx_push;
diff --git a/qemu/hw/net/fsl_etsec/etsec.c b/qemu/hw/net/fsl_etsec/etsec.c
index 0f5cf4477..1e35f7f8c 100644
--- a/qemu/hw/net/fsl_etsec/etsec.c
+++ b/qemu/hw/net/fsl_etsec/etsec.c
@@ -26,6 +26,7 @@
  * This implementation doesn't include ring priority, TCP/IP Off-Load, QoS.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -353,7 +354,7 @@ static ssize_t etsec_receive(NetClientState *nc,
     etsec->need_flush = false;
     ret = etsec_rx_ring_write(etsec, buf, size);
     if (ret == 0) {
-        /* The packet will be queued, let's flush it when buffer is avilable
+        /* The packet will be queued, let's flush it when buffer is available
          * again. */
         etsec->need_flush = true;
     }
diff --git a/qemu/hw/net/fsl_etsec/miim.c b/qemu/hw/net/fsl_etsec/miim.c
index 1931b74e6..6bba01c82 100644
--- a/qemu/hw/net/fsl_etsec/miim.c
+++ b/qemu/hw/net/fsl_etsec/miim.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "etsec.h"
 #include "registers.h"
 
diff --git a/qemu/hw/net/fsl_etsec/registers.c b/qemu/hw/net/fsl_etsec/registers.c
index a7bbfa113..46ce7a84b 100644
--- a/qemu/hw/net/fsl_etsec/registers.c
+++ b/qemu/hw/net/fsl_etsec/registers.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "registers.h"
 
 const eTSEC_Register_Definition eTSEC_registers_def[] = {
diff --git a/qemu/hw/net/fsl_etsec/registers.h b/qemu/hw/net/fsl_etsec/registers.h
index 7ad768647..6fb96842b 100644
--- a/qemu/hw/net/fsl_etsec/registers.h
+++ b/qemu/hw/net/fsl_etsec/registers.h
@@ -24,7 +24,6 @@
 #ifndef _ETSEC_REGISTERS_H_
 #define _ETSEC_REGISTERS_H_
 
-#include <stdint.h>
 
 enum eTSEC_Register_Access_Type {
     ACC_RW      = 1,            /* Read/Write */
diff --git a/qemu/hw/net/fsl_etsec/rings.c b/qemu/hw/net/fsl_etsec/rings.c
index 68e7b6d16..ed1de7da9 100644
--- a/qemu/hw/net/fsl_etsec/rings.c
+++ b/qemu/hw/net/fsl_etsec/rings.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "net/checksum.h"
 
 #include "etsec.h"
@@ -464,9 +465,7 @@ static void rx_init_frame(eTSEC *etsec, const uint8_t *buf, size_t size)
         etsec->rx_fcb_size = 0;
     }
 
-    if (etsec->rx_buffer != NULL) {
-        g_free(etsec->rx_buffer);
-    }
+    g_free(etsec->rx_buffer);
 
     /* Do not copy the frame for now */
     etsec->rx_buffer     = (uint8_t *)buf;
diff --git a/qemu/hw/net/imx_fec.c b/qemu/hw/net/imx_fec.c
new file mode 100644
index 000000000..e60e3380e
--- /dev/null
+++ b/qemu/hw/net/imx_fec.c
@@ -0,0 +1,711 @@
+/*
+ * i.MX Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/net/imx_fec.h"
+#include "sysemu/dma.h"
+
+/* For crc32 */
+#include <zlib.h>
+
+#ifndef DEBUG_IMX_FEC
+#define DEBUG_IMX_FEC 0
+#endif
+
+#define FEC_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_FEC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_FEC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+#ifndef DEBUG_IMX_PHY
+#define DEBUG_IMX_PHY 0
+#endif
+
+#define PHY_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_PHY) { \
+            fprintf(stderr, "[%s.phy]%s: " fmt , TYPE_IMX_FEC, \
+                                                 __func__, ##args); \
+        } \
+    } while (0)
+
+static const VMStateDescription vmstate_imx_fec = {
+    .name = TYPE_IMX_FEC,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(irq_state, IMXFECState),
+        VMSTATE_UINT32(eir, IMXFECState),
+        VMSTATE_UINT32(eimr, IMXFECState),
+        VMSTATE_UINT32(rx_enabled, IMXFECState),
+        VMSTATE_UINT32(rx_descriptor, IMXFECState),
+        VMSTATE_UINT32(tx_descriptor, IMXFECState),
+        VMSTATE_UINT32(ecr, IMXFECState),
+        VMSTATE_UINT32(mmfr, IMXFECState),
+        VMSTATE_UINT32(mscr, IMXFECState),
+        VMSTATE_UINT32(mibc, IMXFECState),
+        VMSTATE_UINT32(rcr, IMXFECState),
+        VMSTATE_UINT32(tcr, IMXFECState),
+        VMSTATE_UINT32(tfwr, IMXFECState),
+        VMSTATE_UINT32(frsr, IMXFECState),
+        VMSTATE_UINT32(erdsr, IMXFECState),
+        VMSTATE_UINT32(etdsr, IMXFECState),
+        VMSTATE_UINT32(emrbr, IMXFECState),
+        VMSTATE_UINT32(miigsk_cfgr, IMXFECState),
+        VMSTATE_UINT32(miigsk_enr, IMXFECState),
+
+        VMSTATE_UINT32(phy_status, IMXFECState),
+        VMSTATE_UINT32(phy_control, IMXFECState),
+        VMSTATE_UINT32(phy_advertise, IMXFECState),
+        VMSTATE_UINT32(phy_int, IMXFECState),
+        VMSTATE_UINT32(phy_int_mask, IMXFECState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define PHY_INT_ENERGYON            (1 << 7)
+#define PHY_INT_AUTONEG_COMPLETE    (1 << 6)
+#define PHY_INT_FAULT               (1 << 5)
+#define PHY_INT_DOWN                (1 << 4)
+#define PHY_INT_AUTONEG_LP          (1 << 3)
+#define PHY_INT_PARFAULT            (1 << 2)
+#define PHY_INT_AUTONEG_PAGE        (1 << 1)
+
+static void imx_fec_update(IMXFECState *s);
+
+/*
+ * The MII phy could raise a GPIO to the processor which in turn
+ * could be handled as an interrpt by the OS.
+ * For now we don't handle any GPIO/interrupt line, so the OS will
+ * have to poll for the PHY status.
+ */
+static void phy_update_irq(IMXFECState *s)
+{
+    imx_fec_update(s);
+}
+
+static void phy_update_link(IMXFECState *s)
+{
+    /* Autonegotiation status mirrors link status.  */
+    if (qemu_get_queue(s->nic)->link_down) {
+        PHY_PRINTF("link is down\n");
+        s->phy_status &= ~0x0024;
+        s->phy_int |= PHY_INT_DOWN;
+    } else {
+        PHY_PRINTF("link is up\n");
+        s->phy_status |= 0x0024;
+        s->phy_int |= PHY_INT_ENERGYON;
+        s->phy_int |= PHY_INT_AUTONEG_COMPLETE;
+    }
+    phy_update_irq(s);
+}
+
+static void imx_fec_set_link(NetClientState *nc)
+{
+    phy_update_link(IMX_FEC(qemu_get_nic_opaque(nc)));
+}
+
+static void phy_reset(IMXFECState *s)
+{
+    s->phy_status = 0x7809;
+    s->phy_control = 0x3000;
+    s->phy_advertise = 0x01e1;
+    s->phy_int_mask = 0;
+    s->phy_int = 0;
+    phy_update_link(s);
+}
+
+static uint32_t do_phy_read(IMXFECState *s, int reg)
+{
+    uint32_t val;
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return 0;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        val = s->phy_control;
+        break;
+    case 1:     /* Basic Status */
+        val = s->phy_status;
+        break;
+    case 2:     /* ID1 */
+        val = 0x0007;
+        break;
+    case 3:     /* ID2 */
+        val = 0xc0d1;
+        break;
+    case 4:     /* Auto-neg advertisement */
+        val = s->phy_advertise;
+        break;
+    case 5:     /* Auto-neg Link Partner Ability */
+        val = 0x0f71;
+        break;
+    case 6:     /* Auto-neg Expansion */
+        val = 1;
+        break;
+    case 29:    /* Interrupt source.  */
+        val = s->phy_int;
+        s->phy_int = 0;
+        phy_update_irq(s);
+        break;
+    case 30:    /* Interrupt mask */
+        val = s->phy_int_mask;
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy]%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    }
+
+    PHY_PRINTF("read 0x%04x @ %d\n", val, reg);
+
+    return val;
+}
+
+static void do_phy_write(IMXFECState *s, int reg, uint32_t val)
+{
+    PHY_PRINTF("write 0x%04x @ %d\n", val, reg);
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        if (val & 0x8000) {
+            phy_reset(s);
+        } else {
+            s->phy_control = val & 0x7980;
+            /* Complete autonegotiation immediately.  */
+            if (val & 0x1000) {
+                s->phy_status |= 0x0020;
+            }
+        }
+        break;
+    case 4:     /* Auto-neg advertisement */
+        s->phy_advertise = (val & 0x2d7f) | 0x80;
+        break;
+    case 30:    /* Interrupt mask */
+        s->phy_int_mask = val & 0xff;
+        phy_update_irq(s);
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy)%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    }
+}
+
+static void imx_fec_read_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_read(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_fec_write_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_write(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_fec_update(IMXFECState *s)
+{
+    uint32_t active;
+    uint32_t changed;
+
+    active = s->eir & s->eimr;
+    changed = active ^ s->irq_state;
+    if (changed) {
+        qemu_set_irq(s->irq, active);
+    }
+    s->irq_state = active;
+}
+
+static void imx_fec_do_tx(IMXFECState *s)
+{
+    int frame_size = 0;
+    uint8_t frame[FEC_MAX_FRAME_SIZE];
+    uint8_t *ptr = frame;
+    uint32_t addr = s->tx_descriptor;
+
+    while (1) {
+        IMXFECBufDesc bd;
+        int len;
+
+        imx_fec_read_bd(&bd, addr);
+        FEC_PRINTF("tx_bd %x flags %04x len %d data %08x\n",
+                   addr, bd.flags, bd.length, bd.data);
+        if ((bd.flags & FEC_BD_R) == 0) {
+            /* Run out of descriptors to transmit.  */
+            break;
+        }
+        len = bd.length;
+        if (frame_size + len > FEC_MAX_FRAME_SIZE) {
+            len = FEC_MAX_FRAME_SIZE - frame_size;
+            s->eir |= FEC_INT_BABT;
+        }
+        dma_memory_read(&address_space_memory, bd.data, ptr, len);
+        ptr += len;
+        frame_size += len;
+        if (bd.flags & FEC_BD_L) {
+            /* Last buffer in frame.  */
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
+            ptr = frame;
+            frame_size = 0;
+            s->eir |= FEC_INT_TXF;
+        }
+        s->eir |= FEC_INT_TXB;
+        bd.flags &= ~FEC_BD_R;
+        /* Write back the modified descriptor.  */
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & FEC_BD_W) != 0) {
+            addr = s->etdsr;
+        } else {
+            addr += 8;
+        }
+    }
+
+    s->tx_descriptor = addr;
+
+    imx_fec_update(s);
+}
+
+static void imx_fec_enable_rx(IMXFECState *s)
+{
+    IMXFECBufDesc bd;
+    uint32_t tmp;
+
+    imx_fec_read_bd(&bd, s->rx_descriptor);
+
+    tmp = ((bd.flags & FEC_BD_E) != 0);
+
+    if (!tmp) {
+        FEC_PRINTF("RX buffer full\n");
+    } else if (!s->rx_enabled) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+
+    s->rx_enabled = tmp;
+}
+
+static void imx_fec_reset(DeviceState *d)
+{
+    IMXFECState *s = IMX_FEC(d);
+
+    /* Reset the FEC */
+    s->eir = 0;
+    s->eimr = 0;
+    s->rx_enabled = 0;
+    s->ecr = 0;
+    s->mscr = 0;
+    s->mibc = 0xc0000000;
+    s->rcr = 0x05ee0001;
+    s->tcr = 0;
+    s->tfwr = 0;
+    s->frsr = 0x500;
+    s->miigsk_cfgr = 0;
+    s->miigsk_enr = 0x6;
+
+    /* We also reset the PHY */
+    phy_reset(s);
+}
+
+static uint64_t imx_fec_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IMXFECState *s = IMX_FEC(opaque);
+
+    FEC_PRINTF("reading from @ 0x%" HWADDR_PRIx "\n", addr);
+
+    switch (addr & 0x3ff) {
+    case 0x004:
+        return s->eir;
+    case 0x008:
+        return s->eimr;
+    case 0x010:
+        return s->rx_enabled ? (1 << 24) : 0;   /* RDAR */
+    case 0x014:
+        return 0;   /* TDAR */
+    case 0x024:
+        return s->ecr;
+    case 0x040:
+        return s->mmfr;
+    case 0x044:
+        return s->mscr;
+    case 0x064:
+        return s->mibc; /* MIBC */
+    case 0x084:
+        return s->rcr;
+    case 0x0c4:
+        return s->tcr;
+    case 0x0e4:     /* PALR */
+        return (s->conf.macaddr.a[0] << 24)
+               | (s->conf.macaddr.a[1] << 16)
+               | (s->conf.macaddr.a[2] << 8)
+               | s->conf.macaddr.a[3];
+        break;
+    case 0x0e8:     /* PAUR */
+        return (s->conf.macaddr.a[4] << 24)
+               | (s->conf.macaddr.a[5] << 16)
+               | 0x8808;
+    case 0x0ec:
+        return 0x10000; /* OPD */
+    case 0x118:
+        return 0;
+    case 0x11c:
+        return 0;
+    case 0x120:
+        return 0;
+    case 0x124:
+        return 0;
+    case 0x144:
+        return s->tfwr;
+    case 0x14c:
+        return 0x600;
+    case 0x150:
+        return s->frsr;
+    case 0x180:
+        return s->erdsr;
+    case 0x184:
+        return s->etdsr;
+    case 0x188:
+        return s->emrbr;
+    case 0x300:
+        return s->miigsk_cfgr;
+    case 0x308:
+        return s->miigsk_enr;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_FEC, __func__, addr);
+        return 0;
+    }
+}
+
+static void imx_fec_write(void *opaque, hwaddr addr,
+                          uint64_t value, unsigned size)
+{
+    IMXFECState *s = IMX_FEC(opaque);
+
+    FEC_PRINTF("writing 0x%08x @ 0x%" HWADDR_PRIx "\n", (int)value, addr);
+
+    switch (addr & 0x3ff) {
+    case 0x004: /* EIR */
+        s->eir &= ~value;
+        break;
+    case 0x008: /* EIMR */
+        s->eimr = value;
+        break;
+    case 0x010: /* RDAR */
+        if ((s->ecr & FEC_EN) && !s->rx_enabled) {
+            imx_fec_enable_rx(s);
+        }
+        break;
+    case 0x014: /* TDAR */
+        if (s->ecr & FEC_EN) {
+            imx_fec_do_tx(s);
+        }
+        break;
+    case 0x024: /* ECR */
+        s->ecr = value;
+        if (value & FEC_RESET) {
+            imx_fec_reset(DEVICE(s));
+        }
+        if ((s->ecr & FEC_EN) == 0) {
+            s->rx_enabled = 0;
+        }
+        break;
+    case 0x040: /* MMFR */
+        /* store the value */
+        s->mmfr = value;
+        if (extract32(value, 28, 1)) {
+            do_phy_write(s, extract32(value, 18, 9), extract32(value, 0, 16));
+        } else {
+            s->mmfr = do_phy_read(s, extract32(value, 18, 9));
+        }
+        /* raise the interrupt as the PHY operation is done */
+        s->eir |= FEC_INT_MII;
+        break;
+    case 0x044: /* MSCR */
+        s->mscr = value & 0xfe;
+        break;
+    case 0x064: /* MIBC */
+        /* TODO: Implement MIB.  */
+        s->mibc = (value & 0x80000000) ? 0xc0000000 : 0;
+        break;
+    case 0x084: /* RCR */
+        s->rcr = value & 0x07ff003f;
+        /* TODO: Implement LOOP mode.  */
+        break;
+    case 0x0c4: /* TCR */
+        /* We transmit immediately, so raise GRA immediately.  */
+        s->tcr = value;
+        if (value & 1) {
+            s->eir |= FEC_INT_GRA;
+        }
+        break;
+    case 0x0e4: /* PALR */
+        s->conf.macaddr.a[0] = value >> 24;
+        s->conf.macaddr.a[1] = value >> 16;
+        s->conf.macaddr.a[2] = value >> 8;
+        s->conf.macaddr.a[3] = value;
+        break;
+    case 0x0e8: /* PAUR */
+        s->conf.macaddr.a[4] = value >> 24;
+        s->conf.macaddr.a[5] = value >> 16;
+        break;
+    case 0x0ec: /* OPDR */
+        break;
+    case 0x118: /* IAUR */
+    case 0x11c: /* IALR */
+    case 0x120: /* GAUR */
+    case 0x124: /* GALR */
+        /* TODO: implement MAC hash filtering.  */
+        break;
+    case 0x144: /* TFWR */
+        s->tfwr = value & 3;
+        break;
+    case 0x14c: /* FRBR */
+        /* FRBR writes ignored.  */
+        break;
+    case 0x150: /* FRSR */
+        s->frsr = (value & 0x3fc) | 0x400;
+        break;
+    case 0x180: /* ERDSR */
+        s->erdsr = value & ~3;
+        s->rx_descriptor = s->erdsr;
+        break;
+    case 0x184: /* ETDSR */
+        s->etdsr = value & ~3;
+        s->tx_descriptor = s->etdsr;
+        break;
+    case 0x188: /* EMRBR */
+        s->emrbr = value & 0x7f0;
+        break;
+    case 0x300: /* MIIGSK_CFGR */
+        s->miigsk_cfgr = value & 0x53;
+        break;
+    case 0x308: /* MIIGSK_ENR */
+        s->miigsk_enr = (value & 0x2) ? 0x6 : 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_FEC, __func__, addr);
+        break;
+    }
+
+    imx_fec_update(s);
+}
+
+static int imx_fec_can_receive(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    return s->rx_enabled;
+}
+
+static ssize_t imx_fec_receive(NetClientState *nc, const uint8_t *buf,
+                               size_t len)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+    IMXFECBufDesc bd;
+    uint32_t flags = 0;
+    uint32_t addr;
+    uint32_t crc;
+    uint32_t buf_addr;
+    uint8_t *crc_ptr;
+    unsigned int buf_len;
+    size_t size = len;
+
+    FEC_PRINTF("len %d\n", (int)size);
+
+    if (!s->rx_enabled) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Unexpected packet\n",
+                      TYPE_IMX_FEC, __func__);
+        return 0;
+    }
+
+    /* 4 bytes for the CRC.  */
+    size += 4;
+    crc = cpu_to_be32(crc32(~0, buf, size));
+    crc_ptr = (uint8_t *) &crc;
+
+    /* Huge frames are truncted.  */
+    if (size > FEC_MAX_FRAME_SIZE) {
+        size = FEC_MAX_FRAME_SIZE;
+        flags |= FEC_BD_TR | FEC_BD_LG;
+    }
+
+    /* Frames larger than the user limit just set error flags.  */
+    if (size > (s->rcr >> 16)) {
+        flags |= FEC_BD_LG;
+    }
+
+    addr = s->rx_descriptor;
+    while (size > 0) {
+        imx_fec_read_bd(&bd, addr);
+        if ((bd.flags & FEC_BD_E) == 0) {
+            /* No descriptors available.  Bail out.  */
+            /*
+             * FIXME: This is wrong. We should probably either
+             * save the remainder for when more RX buffers are
+             * available, or flag an error.
+             */
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Lost end of frame\n",
+                          TYPE_IMX_FEC, __func__);
+            break;
+        }
+        buf_len = (size <= s->emrbr) ? size : s->emrbr;
+        bd.length = buf_len;
+        size -= buf_len;
+
+        FEC_PRINTF("rx_bd 0x%x length %d\n", addr, bd.length);
+
+        /* The last 4 bytes are the CRC.  */
+        if (size < 4) {
+            buf_len += size - 4;
+        }
+        buf_addr = bd.data;
+        dma_memory_write(&address_space_memory, buf_addr, buf, buf_len);
+        buf += buf_len;
+        if (size < 4) {
+            dma_memory_write(&address_space_memory, buf_addr + buf_len,
+                             crc_ptr, 4 - size);
+            crc_ptr += 4 - size;
+        }
+        bd.flags &= ~FEC_BD_E;
+        if (size == 0) {
+            /* Last buffer in frame.  */
+            bd.flags |= flags | FEC_BD_L;
+            FEC_PRINTF("rx frame flags %04x\n", bd.flags);
+            s->eir |= FEC_INT_RXF;
+        } else {
+            s->eir |= FEC_INT_RXB;
+        }
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & FEC_BD_W) != 0) {
+            addr = s->erdsr;
+        } else {
+            addr += 8;
+        }
+    }
+    s->rx_descriptor = addr;
+    imx_fec_enable_rx(s);
+    imx_fec_update(s);
+    return len;
+}
+
+static const MemoryRegionOps imx_fec_ops = {
+    .read = imx_fec_read,
+    .write = imx_fec_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void imx_fec_cleanup(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    s->nic = NULL;
+}
+
+static NetClientInfo net_imx_fec_info = {
+    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .size = sizeof(NICState),
+    .can_receive = imx_fec_can_receive,
+    .receive = imx_fec_receive,
+    .cleanup = imx_fec_cleanup,
+    .link_status_changed = imx_fec_set_link,
+};
+
+
+static void imx_fec_realize(DeviceState *dev, Error **errp)
+{
+    IMXFECState *s = IMX_FEC(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_fec_ops, s,
+                          TYPE_IMX_FEC, 0x400);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+
+    s->conf.peers.ncs[0] = nd_table[0].netdev;
+
+    s->nic = qemu_new_nic(&net_imx_fec_info, &s->conf,
+                          object_get_typename(OBJECT(dev)), DEVICE(dev)->id,
+                          s);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
+}
+
+static Property imx_fec_properties[] = {
+    DEFINE_NIC_PROPERTIES(IMXFECState, conf),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_fec_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_imx_fec;
+    dc->reset = imx_fec_reset;
+    dc->props = imx_fec_properties;
+    dc->realize = imx_fec_realize;
+    dc->desc = "i.MX FEC Ethernet Controller";
+}
+
+static const TypeInfo imx_fec_info = {
+    .name = TYPE_IMX_FEC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXFECState),
+    .class_init = imx_fec_class_init,
+};
+
+static void imx_fec_register_types(void)
+{
+    type_register_static(&imx_fec_info);
+}
+
+type_init(imx_fec_register_types)
diff --git a/qemu/hw/net/lan9118.c b/qemu/hw/net/lan9118.c
index 4f0e840f0..08dc474d6 100644
--- a/qemu/hw/net/lan9118.c
+++ b/qemu/hw/net/lan9118.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/devices.h"
@@ -56,6 +57,8 @@ do { fprintf(stderr, "lan9118: error: " fmt , ## __VA_ARGS__);} while (0)
 #define CSR_E2P_CMD     0xb0
 #define CSR_E2P_DATA    0xb4
 
+#define E2P_CMD_MAC_ADDR_LOADED 0x100
+
 /* IRQ_CFG */
 #define IRQ_INT         0x00001000
 #define IRQ_EN          0x00000100
@@ -352,14 +355,14 @@ static void lan9118_reload_eeprom(lan9118_state *s)
 {
     int i;
     if (s->eeprom[0] != 0xa5) {
-        s->e2p_cmd &= ~0x10;
+        s->e2p_cmd &= ~E2P_CMD_MAC_ADDR_LOADED;
         DPRINTF("MACADDR load failed\n");
         return;
     }
     for (i = 0; i < 6; i++) {
         s->conf.macaddr.a[i] = s->eeprom[i + 1];
     }
-    s->e2p_cmd |= 0x10;
+    s->e2p_cmd |= E2P_CMD_MAC_ADDR_LOADED;
     DPRINTF("MACADDR loaded from eeprom\n");
     lan9118_mac_changed(s);
 }
@@ -902,7 +905,8 @@ static void do_mac_write(lan9118_state *s, int reg, uint32_t val)
          */
         break;
     default:
-        hw_error("lan9118: Unimplemented MAC register write: %d = 0x%x\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register write: %d = 0x%x\n",
                  s->mac_cmd & 0xf, val);
     }
 }
@@ -930,14 +934,16 @@ static uint32_t do_mac_read(lan9118_state *s, int reg)
     case MAC_FLOW:
         return s->mac_flow;
     default:
-        hw_error("lan9118: Unimplemented MAC register read: %d\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register read: %d\n",
                  s->mac_cmd & 0xf);
+        return 0;
     }
 }
 
 static void lan9118_eeprom_cmd(lan9118_state *s, int cmd, int addr)
 {
-    s->e2p_cmd = (s->e2p_cmd & 0x10) | (cmd << 28) | addr;
+    s->e2p_cmd = (s->e2p_cmd & E2P_CMD_MAC_ADDR_LOADED) | (cmd << 28) | addr;
     switch (cmd) {
     case 0:
         s->e2p_data = s->eeprom[addr];
@@ -1128,7 +1134,8 @@ static void lan9118_writel(void *opaque, hwaddr offset,
         break;
 
     default:
-        hw_error("lan9118_write: Bad reg 0x%x = %x\n", (int)offset, (int)val);
+        qemu_log_mask(LOG_GUEST_ERROR, "lan9118_write: Bad reg 0x%x = %x\n",
+                      (int)offset, (int)val);
         break;
     }
     lan9118_update(s);
@@ -1246,7 +1253,7 @@ static uint64_t lan9118_readl(void *opaque, hwaddr offset,
     case CSR_E2P_DATA:
         return s->e2p_data;
     }
-    hw_error("lan9118_read: Bad reg 0x%x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR, "lan9118_read: Bad reg 0x%x\n", (int)offset);
     return 0;
 }
 
diff --git a/qemu/hw/net/lance.c b/qemu/hw/net/lance.c
index 780b39d65..6253d2103 100644
--- a/qemu/hw/net/lance.c
+++ b/qemu/hw/net/lance.c
@@ -35,6 +35,7 @@
  * http://www.ibiblio.org/pub/historic-linux/early-ports/Sparc/NCR/NCR92C990.txt
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/net/mcf_fec.c b/qemu/hw/net/mcf_fec.c
index 21928f9f3..7c0398ed9 100644
--- a/qemu/hw/net/mcf_fec.c
+++ b/qemu/hw/net/mcf_fec.c
@@ -5,6 +5,7 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "net/net.h"
 #include "hw/m68k/mcf.h"
diff --git a/qemu/hw/net/milkymist-minimac2.c b/qemu/hw/net/milkymist-minimac2.c
index 5d1cf0851..1e147c33c 100644
--- a/qemu/hw/net/milkymist-minimac2.c
+++ b/qemu/hw/net/milkymist-minimac2.c
@@ -22,6 +22,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h" /* FIXME: why does this use TARGET_PAGE_ALIGN? */
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -463,7 +467,7 @@ static int milkymist_minimac2_init(SysBusDevice *sbd)
 
     /* register buffers memory */
     memory_region_init_ram(&s->buffers, OBJECT(dev), "milkymist-minimac2.buffers",
-                           buffers_size, &error_abort);
+                           buffers_size, &error_fatal);
     vmstate_register_ram_global(&s->buffers);
     s->rx0_buf = memory_region_get_ram_ptr(&s->buffers);
     s->rx1_buf = s->rx0_buf + MINIMAC2_BUFFER_SIZE;
diff --git a/qemu/hw/net/mipsnet.c b/qemu/hw/net/mipsnet.c
index f261011a2..740cd98ff 100644
--- a/qemu/hw/net/mipsnet.c
+++ b/qemu/hw/net/mipsnet.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "net/net.h"
 #include "trace.h"
diff --git a/qemu/hw/net/ne2000-isa.c b/qemu/hw/net/ne2000-isa.c
index 17e7199f7..a7f5a9464 100644
--- a/qemu/hw/net/ne2000-isa.c
+++ b/qemu/hw/net/ne2000-isa.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -28,6 +29,7 @@
 #include "net/net.h"
 #include "ne2000.h"
 #include "exec/address-spaces.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 #define TYPE_ISA_NE2000 "ne2k_isa"
@@ -44,7 +46,6 @@ typedef struct ISANE2000State {
 static NetClientInfo net_ne2000_isa_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
 };
 
@@ -94,24 +95,26 @@ static void isa_ne2000_class_initfn(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
 }
 
-static void isa_ne2000_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void isa_ne2000_get_bootindex(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     ISANE2000State *isa = ISA_NE2000(obj);
     NE2000State *s = &isa->ne2000;
 
-    visit_type_int32(v, &s->c.bootindex, name, errp);
+    visit_type_int32(v, name, &s->c.bootindex, errp);
 }
 
-static void isa_ne2000_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void isa_ne2000_set_bootindex(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     ISANE2000State *isa = ISA_NE2000(obj);
     NE2000State *s = &isa->ne2000;
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
diff --git a/qemu/hw/net/ne2000.c b/qemu/hw/net/ne2000.c
index 2bdb4c927..f0feaf96b 100644
--- a/qemu/hw/net/ne2000.c
+++ b/qemu/hw/net/ne2000.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -154,6 +155,10 @@ static int ne2000_buffer_full(NE2000State *s)
 {
     int avail, index, boundary;
 
+    if (s->stop <= s->start) {
+        return 1;
+    }
+
     index = s->curpag << 8;
     boundary = s->boundary << 8;
     if (index < boundary)
@@ -165,15 +170,6 @@ static int ne2000_buffer_full(NE2000State *s)
     return 0;
 }
 
-int ne2000_can_receive(NetClientState *nc)
-{
-    NE2000State *s = qemu_get_nic_opaque(nc);
-
-    if (s->cmd & E8390_STOP)
-        return 1;
-    return !ne2000_buffer_full(s);
-}
-
 #define MIN_BUF_SIZE 60
 
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
@@ -476,8 +472,9 @@ static inline void ne2000_mem_writel(NE2000State *s, uint32_t addr,
                                      uint32_t val)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         stl_le_p(s->mem + addr, val);
     }
 }
@@ -506,8 +503,9 @@ static inline uint32_t ne2000_mem_readw(NE2000State *s, uint32_t addr)
 static inline uint32_t ne2000_mem_readl(NE2000State *s, uint32_t addr)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         return ldl_le_p(s->mem + addr);
     } else {
         return 0xffffffff;
@@ -716,7 +714,6 @@ void ne2000_setup_io(NE2000State *s, DeviceState *dev, unsigned size)
 static NetClientInfo net_ne2000_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
 };
 
diff --git a/qemu/hw/net/ne2000.h b/qemu/hw/net/ne2000.h
index e500306aa..d022b28fc 100644
--- a/qemu/hw/net/ne2000.h
+++ b/qemu/hw/net/ne2000.h
@@ -34,7 +34,6 @@ typedef struct NE2000State {
 void ne2000_setup_io(NE2000State *s, DeviceState *dev, unsigned size);
 extern const VMStateDescription vmstate_ne2000;
 void ne2000_reset(NE2000State *s);
-int ne2000_can_receive(NetClientState *nc);
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
 
 #endif
diff --git a/qemu/hw/net/opencores_eth.c b/qemu/hw/net/opencores_eth.c
index 3642046ef..c6094fbb5 100644
--- a/qemu/hw/net/opencores_eth.c
+++ b/qemu/hw/net/opencores_eth.c
@@ -31,6 +31,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
@@ -84,7 +85,7 @@ static void mii_reset(Mii *s)
 {
     memset(s->regs, 0, sizeof(s->regs));
     s->regs[MII_BMCR] = 0x1000;
-    s->regs[MII_BMSR] = 0x7848; /* no ext regs */
+    s->regs[MII_BMSR] = 0x7868; /* no ext regs */
     s->regs[MII_PHYIDR1] = 0x2000;
     s->regs[MII_PHYIDR2] = 0x5c90;
     s->regs[MII_ANAR] = 0x01e1;
diff --git a/qemu/hw/net/pcnet-pci.c b/qemu/hw/net/pcnet-pci.c
index b4d60b812..595439a65 100644
--- a/qemu/hw/net/pcnet-pci.c
+++ b/qemu/hw/net/pcnet-pci.c
@@ -27,6 +27,7 @@
  * AMD Publication# 19436  Rev:E  Amendment/0  Issue Date: June 2000
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
 #include "hw/loader.h"
diff --git a/qemu/hw/net/pcnet.c b/qemu/hw/net/pcnet.c
index 34373767d..198a01f92 100644
--- a/qemu/hw/net/pcnet.c
+++ b/qemu/hw/net/pcnet.c
@@ -35,6 +35,7 @@
  * http://www.ibiblio.org/pub/historic-linux/early-ports/Sparc/NCR/NCR92C990.txt
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "net/net.h"
 #include "qemu/timer.h"
@@ -670,8 +671,7 @@ static inline hwaddr pcnet_rdra_addr(PCNetState *s, int idx)
 static inline int64_t pcnet_get_next_poll_time(PCNetState *s, int64_t current_time)
 {
     int64_t next_time = current_time +
-        muldiv64(65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s)),
-                 get_ticks_per_sec(), 33000000L);
+                        (65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s))) * 30;
     if (next_time <= current_time)
         next_time = current_time + 1;
     return next_time;
@@ -1065,6 +1065,12 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             int pktcount = 0;
 
             if (!s->looptest) {
+                if (size > 4092) {
+#ifdef PCNET_DEBUG_RMD
+                    fprintf(stderr, "pcnet: truncates rx packet.\n");
+#endif
+                    size = 4092;
+                }
                 memcpy(src, buf, size);
                 /* no need to compute the CRC */
                 src[size] = 0;
@@ -1085,7 +1091,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
                 uint32_t fcs = ~0;
                 uint8_t *p = src;
 
-                while (p != &src[size-4])
+                while (p != &src[size])
                     CRC(fcs, *p++);
                 crc_err = (*(uint32_t *)p != htonl(fcs));
             }
@@ -1234,8 +1240,10 @@ static void pcnet_transmit(PCNetState *s)
         bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
 
         /* if multi-tmd packet outsizes s->buffer then skip it silently.
-           Note: this is not what real hw does */
-        if (s->xmit_pos + bcnt > sizeof(s->buffer)) {
+         * Note: this is not what real hw does.
+         * Last four bytes of s->buffer are used to store CRC FCS code.
+         */
+        if (s->xmit_pos + bcnt > sizeof(s->buffer) - 4) {
             s->xmit_pos = -1;
             goto txdone;
         }
diff --git a/qemu/hw/net/rocker/qmp-norocker.c b/qemu/hw/net/rocker/qmp-norocker.c
index 49b498b64..6acbcdb02 100644
--- a/qemu/hw/net/rocker/qmp-norocker.c
+++ b/qemu/hw/net/rocker/qmp-norocker.c
@@ -15,6 +15,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qmp-commands.h"
 #include "qapi/qmp/qerror.h"
diff --git a/qemu/hw/net/rocker/rocker.c b/qemu/hw/net/rocker/rocker.c
index 47d080fd3..30f2ce417 100644
--- a/qemu/hw/net/rocker/rocker.c
+++ b/qemu/hw/net/rocker/rocker.c
@@ -15,6 +15,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/msix.h"
@@ -42,6 +43,7 @@ struct rocker {
 
     /* switch configuration */
     char *name;                  /* switch name */
+    char *world_name;            /* world name */
     uint32_t fp_ports;           /* front-panel port count */
     NICPeers *fp_ports_peers;
     MACAddr fp_start_macaddr;    /* front-panel port 0 mac addr */
@@ -101,8 +103,7 @@ RockerSwitch *qmp_query_rocker(const char *name, Error **errp)
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
@@ -122,8 +123,7 @@ RockerPortList *qmp_query_rocker_ports(const char *name, Error **errp)
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
@@ -234,6 +234,9 @@ static int tx_consume(Rocker *r, DescInfo *info)
         frag_addr = rocker_tlv_get_le64(tlvs[ROCKER_TLV_TX_FRAG_ATTR_ADDR]);
         frag_len = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_FRAG_ATTR_LEN]);
 
+        if (iovcnt >= ROCKER_TX_FRAGS_MAX) {
+            goto err_too_many_frags;
+        }
         iov[iovcnt].iov_len = frag_len;
         iov[iovcnt].iov_base = g_malloc(frag_len);
         if (!iov[iovcnt].iov_base) {
@@ -246,10 +249,7 @@ static int tx_consume(Rocker *r, DescInfo *info)
             err = -ROCKER_ENXIO;
             goto err_bad_io;
         }
-
-        if (++iovcnt > ROCKER_TX_FRAGS_MAX) {
-            goto err_too_many_frags;
-        }
+        iovcnt++;
     }
 
     if (iovcnt) {
@@ -265,9 +265,7 @@ err_bad_io:
 err_no_mem:
 err_bad_attr:
     for (i = 0; i < ROCKER_TX_FRAGS_MAX; i++) {
-        if (iov[i].iov_base) {
-            g_free(iov[i].iov_base);
-        }
+        g_free(iov[i].iov_base);
     }
 
     return err;
@@ -403,7 +401,13 @@ static int cmd_set_port_settings(Rocker *r,
 
     if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE]) {
         mode = rocker_tlv_get_u8(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE]);
-        fp_port_set_world(fp_port, r->worlds[mode]);
+        if (mode >= ROCKER_WORLD_TYPE_MAX) {
+            return -ROCKER_EINVAL;
+        }
+        /* We don't support world change. */
+        if (!fp_port_check_world(fp_port, r->worlds[mode])) {
+            return -ROCKER_EINVAL;
+        }
     }
 
     if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING]) {
@@ -1283,6 +1287,18 @@ static void rocker_msix_uninit(Rocker *r)
     rocker_msix_vectors_unuse(r, ROCKER_MSIX_VEC_COUNT(r->fp_ports));
 }
 
+static World *rocker_world_type_by_name(Rocker *r, const char *name)
+{
+    int i;
+
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (strcmp(name, world_name(r->worlds[i])) == 0) {
+            return r->worlds[i];
+	}
+    }
+    return NULL;
+}
+
 static int pci_rocker_init(PCIDevice *dev)
 {
     Rocker *r = to_rocker(dev);
@@ -1294,14 +1310,27 @@ static int pci_rocker_init(PCIDevice *dev)
     /* allocate worlds */
 
     r->worlds[ROCKER_WORLD_TYPE_OF_DPA] = of_dpa_world_alloc(r);
-    r->world_dflt = r->worlds[ROCKER_WORLD_TYPE_OF_DPA];
 
     for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
         if (!r->worlds[i]) {
+            err = -ENOMEM;
             goto err_world_alloc;
         }
     }
 
+    if (!r->world_name) {
+        r->world_name = g_strdup(world_name(r->worlds[ROCKER_WORLD_TYPE_OF_DPA]));
+    }
+
+    r->world_dflt = rocker_world_type_by_name(r, r->world_name);
+    if (!r->world_dflt) {
+        fprintf(stderr,
+                "rocker: requested world \"%s\" does not exist\n",
+                r->world_name);
+        err = -EINVAL;
+        goto err_world_type_by_name;
+    }
+
     /* set up memory-mapped region at BAR0 */
 
     memory_region_init_io(&r->mmio, OBJECT(r), &rocker_mmio_ops, r,
@@ -1364,7 +1393,7 @@ static int pci_rocker_init(PCIDevice *dev)
         r->fp_ports = ROCKER_FP_PORTS_MAX;
     }
 
-    r->rings = g_malloc(sizeof(DescRing *) * rocker_pci_ring_count(r));
+    r->rings = g_new(DescRing *, rocker_pci_ring_count(r));
     if (!r->rings) {
         goto err_rings_alloc;
     }
@@ -1435,6 +1464,7 @@ err_duplicate:
 err_msix_init:
     object_unparent(OBJECT(&r->msix_bar));
     object_unparent(OBJECT(&r->mmio));
+err_world_type_by_name:
 err_world_alloc:
     for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
         if (r->worlds[i]) {
@@ -1506,6 +1536,7 @@ static void rocker_reset(DeviceState *dev)
 
 static Property rocker_properties[] = {
     DEFINE_PROP_STRING("name", Rocker, name),
+    DEFINE_PROP_STRING("world", Rocker, world_name),
     DEFINE_PROP_MACADDR("fp_start_macaddr", Rocker,
                         fp_start_macaddr),
     DEFINE_PROP_UINT64("switch_id", Rocker,
diff --git a/qemu/hw/net/rocker/rocker_desc.c b/qemu/hw/net/rocker/rocker_desc.c
index 9d896fe47..ac02797b7 100644
--- a/qemu/hw/net/rocker/rocker_desc.c
+++ b/qemu/hw/net/rocker/rocker_desc.c
@@ -14,6 +14,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
@@ -136,15 +137,13 @@ bool desc_ring_set_size(DescRing *ring, uint32_t size)
     }
 
     for (i = 0; i < ring->size; i++) {
-        if (ring->info[i].buf) {
-            g_free(ring->info[i].buf);
-        }
+        g_free(ring->info[i].buf);
     }
 
     ring->size = size;
     ring->head = ring->tail = 0;
 
-    ring->info = g_realloc(ring->info, size * sizeof(DescInfo));
+    ring->info = g_renew(DescInfo, ring->info, size);
     if (!ring->info) {
         return false;
     }
@@ -347,7 +346,7 @@ DescRing *desc_ring_alloc(Rocker *r, int index)
 {
     DescRing *ring;
 
-    ring = g_malloc0(sizeof(DescRing));
+    ring = g_new0(DescRing, 1);
     if (!ring) {
         return NULL;
     }
@@ -360,9 +359,7 @@ DescRing *desc_ring_alloc(Rocker *r, int index)
 
 void desc_ring_free(DescRing *ring)
 {
-    if (ring->info) {
-        g_free(ring->info);
-    }
+    g_free(ring->info);
     g_free(ring);
 }
 
diff --git a/qemu/hw/net/rocker/rocker_fp.c b/qemu/hw/net/rocker/rocker_fp.c
index c693ae508..0149899c6 100644
--- a/qemu/hw/net/rocker/rocker_fp.c
+++ b/qemu/hw/net/rocker/rocker_fp.c
@@ -14,6 +14,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "net/clients.h"
 
 #include "rocker.h"
@@ -185,6 +186,11 @@ void fp_port_set_world(FpPort *port, World *world)
     port->world = world;
 }
 
+bool fp_port_check_world(FpPort *port, World *world)
+{
+    return port->world == world;
+}
+
 bool fp_port_enabled(FpPort *port)
 {
     return port->enabled;
@@ -218,7 +224,7 @@ FpPort *fp_port_alloc(Rocker *r, char *sw_name,
                       MACAddr *start_mac, unsigned int index,
                       NICPeers *peers)
 {
-    FpPort *port = g_malloc0(sizeof(FpPort));
+    FpPort *port = g_new0(FpPort, 1);
 
     if (!port) {
         return NULL;
diff --git a/qemu/hw/net/rocker/rocker_fp.h b/qemu/hw/net/rocker/rocker_fp.h
index ab80fd833..04592bbfd 100644
--- a/qemu/hw/net/rocker/rocker_fp.h
+++ b/qemu/hw/net/rocker/rocker_fp.h
@@ -40,6 +40,7 @@ int fp_port_set_settings(FpPort *port, uint32_t speed,
 bool fp_port_from_pport(uint32_t pport, uint32_t *port);
 World *fp_port_get_world(FpPort *port);
 void fp_port_set_world(FpPort *port, World *world);
+bool fp_port_check_world(FpPort *port, World *world);
 bool fp_port_enabled(FpPort *port);
 void fp_port_enable(FpPort *port);
 void fp_port_disable(FpPort *port);
diff --git a/qemu/hw/net/rocker/rocker_of_dpa.c b/qemu/hw/net/rocker/rocker_of_dpa.c
index 874fb01d6..0a134ebca 100644
--- a/qemu/hw/net/rocker/rocker_of_dpa.c
+++ b/qemu/hw/net/rocker/rocker_of_dpa.c
@@ -14,6 +14,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "net/eth.h"
 #include "qemu/iov.h"
 #include "qemu/timer.h"
@@ -367,7 +368,7 @@ static OfDpaFlow *of_dpa_flow_alloc(uint64_t cookie)
     OfDpaFlow *flow;
     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) / 1000;
 
-    flow = g_malloc0(sizeof(OfDpaFlow));
+    flow = g_new0(OfDpaFlow, 1);
     if (!flow) {
         return NULL;
     }
@@ -811,7 +812,7 @@ static int of_dpa_group_get_stats(OfDpa *of_dpa, uint32_t id)
 
 static OfDpaGroup *of_dpa_group_alloc(uint32_t id)
 {
-    OfDpaGroup *group = g_malloc0(sizeof(OfDpaGroup));
+    OfDpaGroup *group = g_new0(OfDpaGroup, 1);
 
     if (!group) {
         return NULL;
@@ -2039,15 +2040,14 @@ static int of_dpa_cmd_add_l2_flood(OfDpa *of_dpa, OfDpaGroup *group,
     group->l2_flood.group_count =
         rocker_tlv_get_le16(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_COUNT]);
 
-    tlvs = g_malloc0((group->l2_flood.group_count + 1) *
-                     sizeof(RockerTlv *));
+    tlvs = g_new0(RockerTlv *, group->l2_flood.group_count + 1);
     if (!tlvs) {
         return -ROCKER_ENOMEM;
     }
 
     g_free(group->l2_flood.group_ids);
     group->l2_flood.group_ids =
-        g_malloc0(group->l2_flood.group_count * sizeof(uint32_t));
+        g_new0(uint32_t, group->l2_flood.group_count);
     if (!group->l2_flood.group_ids) {
         err = -ROCKER_ENOMEM;
         goto err_out;
@@ -2463,15 +2463,13 @@ RockerOfDpaFlowList *qmp_query_rocker_of_dpa_flows(const char *name,
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
     w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
     if (!w) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s doesn't have OF-DPA world", name);
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
         return NULL;
     }
 
@@ -2598,15 +2596,13 @@ RockerOfDpaGroupList *qmp_query_rocker_of_dpa_groups(const char *name,
 
     r = rocker_find(name);
     if (!r) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s not found", name);
+        error_setg(errp, "rocker %s not found", name);
         return NULL;
     }
 
     w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
     if (!w) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "rocker %s doesn't have OF-DPA world", name);
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
         return NULL;
     }
 
@@ -2618,6 +2614,7 @@ RockerOfDpaGroupList *qmp_query_rocker_of_dpa_groups(const char *name,
 }
 
 static WorldOps of_dpa_ops = {
+    .name = "ofdpa",
     .init = of_dpa_init,
     .uninit = of_dpa_uninit,
     .ig = of_dpa_ig,
diff --git a/qemu/hw/net/rocker/rocker_world.c b/qemu/hw/net/rocker/rocker_world.c
index a6b18f175..89777e968 100644
--- a/qemu/hw/net/rocker/rocker_world.c
+++ b/qemu/hw/net/rocker/rocker_world.c
@@ -14,6 +14,7 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 
 #include "rocker.h"
@@ -97,10 +98,5 @@ enum rocker_world_type world_type(World *world)
 
 const char *world_name(World *world)
 {
-    switch (world->type) {
-    case ROCKER_WORLD_TYPE_OF_DPA:
-        return "OF_DPA";
-    default:
-        return "unknown";
-    }
+    return world->ops->name;
 }
diff --git a/qemu/hw/net/rocker/rocker_world.h b/qemu/hw/net/rocker/rocker_world.h
index 18d277b92..58ade4733 100644
--- a/qemu/hw/net/rocker/rocker_world.h
+++ b/qemu/hw/net/rocker/rocker_world.h
@@ -33,6 +33,7 @@ typedef int (world_cmd)(World *world, DescInfo *info,
                         RockerTlv *cmd_info_tlv);
 
 typedef struct world_ops {
+    const char *name;
     world_init *init;
     world_uninit *uninit;
     world_ig *ig;
diff --git a/qemu/hw/net/rtl8139.c b/qemu/hw/net/rtl8139.c
index edbb61ccf..1e5ec149f 100644
--- a/qemu/hw/net/rtl8139.c
+++ b/qemu/hw/net/rtl8139.c
@@ -43,12 +43,13 @@
  *                                  Added rx/tx buffer reset when enabling rx/tx operation
  *
  *  2010-Feb-04  Frediano Ziglio:   Rewrote timer support using QEMU timer only
- *                                  when strictly needed (required for for
+ *                                  when strictly needed (required for
  *                                  Darwin)
  *  2011-Mar-22  Benjamin Poirier:  Implemented VLAN offloading
  */
 
 /* For crc32 */
+#include "qemu/osdep.h"
 #include <zlib.h>
 
 #include "hw/hw.h"
@@ -56,6 +57,7 @@
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "net/net.h"
+#include "net/eth.h"
 #include "hw/loader.h"
 #include "sysemu/sysemu.h"
 #include "qemu/iov.h"
@@ -63,7 +65,7 @@
 /* debug RTL8139 card */
 //#define DEBUG_RTL8139 1
 
-#define PCI_FREQUENCY 33000000L
+#define PCI_PERIOD 30    /* 30 ns period = 33.333333 Mhz frequency */
 
 #define SET_MASKED(input, mask, curr) \
     ( ( (input) & ~(mask) ) | ( (curr) & (mask) ) )
@@ -72,11 +74,7 @@
 #define MOD2(input, size) \
     ( ( input ) & ( size - 1 )  )
 
-#define ETHER_ADDR_LEN 6
 #define ETHER_TYPE_LEN 2
-#define ETH_HLEN (ETHER_ADDR_LEN * 2 + ETHER_TYPE_LEN)
-#define ETH_P_IP    0x0800      /* Internet Protocol packet */
-#define ETH_P_8021Q 0x8100      /* 802.1Q VLAN Extended Header  */
 #define ETH_MTU     1500
 
 #define VLAN_TCI_LEN 2
@@ -1016,8 +1014,8 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         /* write VLAN info to descriptor variables. */
         if (s->CpCmd & CPlusRxVLAN && be16_to_cpup((uint16_t *)
-                &buf[ETHER_ADDR_LEN * 2]) == ETH_P_8021Q) {
-            dot1q_buf = &buf[ETHER_ADDR_LEN * 2];
+                &buf[ETH_ALEN * 2]) == ETH_P_VLAN) {
+            dot1q_buf = &buf[ETH_ALEN * 2];
             size -= VLAN_HLEN;
             /* if too small buffer, use the tailroom added duing expansion */
             if (size < MIN_BUF_SIZE) {
@@ -1058,10 +1056,10 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         /* receive/copy to target memory */
         if (dot1q_buf) {
-            pci_dma_write(d, rx_addr, buf, 2 * ETHER_ADDR_LEN);
-            pci_dma_write(d, rx_addr + 2 * ETHER_ADDR_LEN,
-                          buf + 2 * ETHER_ADDR_LEN + VLAN_HLEN,
-                          size - 2 * ETHER_ADDR_LEN);
+            pci_dma_write(d, rx_addr, buf, 2 * ETH_ALEN);
+            pci_dma_write(d, rx_addr + 2 * ETH_ALEN,
+                          buf + 2 * ETH_ALEN + VLAN_HLEN,
+                          size - 2 * ETH_ALEN);
         } else {
             pci_dma_write(d, rx_addr, buf, size);
         }
@@ -1148,7 +1146,9 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
 
         /* if receiver buffer is empty then avail == 0 */
 
-        if (avail != 0 && size + 8 >= avail)
+#define RX_ALIGN(x) (((x) + 3) & ~0x3)
+
+        if (avail != 0 && RX_ALIGN(size + 8) >= avail)
         {
             DPRINTF("rx overflow: rx buffer length %d head 0x%04x "
                 "read 0x%04x === available 0x%04x need 0x%04x\n",
@@ -1157,7 +1157,7 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
             s->IntrStatus |= RxOverflow;
             ++s->RxMissed;
             rtl8139_update_irq(s);
-            return size_;
+            return 0;
         }
 
         packet_header |= RxStatusOK;
@@ -1176,7 +1176,7 @@ static ssize_t rtl8139_do_receive(NetClientState *nc, const uint8_t *buf, size_t
         rtl8139_write_buffer(s, (uint8_t *)&val, 4);
 
         /* correct buffer write pointer */
-        s->RxBufAddr = MOD2((s->RxBufAddr + 3) & ~0x3, s->RxBufferSize);
+        s->RxBufAddr = MOD2(RX_ALIGN(s->RxBufAddr), s->RxBufferSize);
 
         /* now we can signal we have received something */
 
@@ -1783,12 +1783,12 @@ static void rtl8139_transfer_frame(RTL8139State *s, uint8_t *buf, int size,
         return;
     }
 
-    if (dot1q_buf && size >= ETHER_ADDR_LEN * 2) {
+    if (dot1q_buf && size >= ETH_ALEN * 2) {
         iov = (struct iovec[3]) {
-            { .iov_base = buf, .iov_len = ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf, .iov_len = ETH_ALEN * 2 },
             { .iov_base = (void *) dot1q_buf, .iov_len = VLAN_HLEN },
-            { .iov_base = buf + ETHER_ADDR_LEN * 2,
-                .iov_len = size - ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf + ETH_ALEN * 2,
+                .iov_len = size - ETH_ALEN * 2 },
         };
 
         memcpy(vlan_iov, iov, sizeof(vlan_iov));
@@ -1868,64 +1868,12 @@ static int rtl8139_transmit_one(RTL8139State *s, int descriptor)
 }
 
 /* structures and macros for task offloading */
-typedef struct ip_header
-{
-    uint8_t  ip_ver_len;    /* version and header length */
-    uint8_t  ip_tos;        /* type of service */
-    uint16_t ip_len;        /* total length */
-    uint16_t ip_id;         /* identification */
-    uint16_t ip_off;        /* fragment offset field */
-    uint8_t  ip_ttl;        /* time to live */
-    uint8_t  ip_p;          /* protocol */
-    uint16_t ip_sum;        /* checksum */
-    uint32_t ip_src,ip_dst; /* source and dest address */
-} ip_header;
-
-#define IP_HEADER_VERSION_4 4
-#define IP_HEADER_VERSION(ip) ((ip->ip_ver_len >> 4)&0xf)
-#define IP_HEADER_LENGTH(ip) (((ip->ip_ver_len)&0xf) << 2)
-
-typedef struct tcp_header
-{
-    uint16_t th_sport;		/* source port */
-    uint16_t th_dport;		/* destination port */
-    uint32_t th_seq;			/* sequence number */
-    uint32_t th_ack;			/* acknowledgement number */
-    uint16_t th_offset_flags; /* data offset, reserved 6 bits, TCP protocol flags */
-    uint16_t th_win;			/* window */
-    uint16_t th_sum;			/* checksum */
-    uint16_t th_urp;			/* urgent pointer */
-} tcp_header;
-
-typedef struct udp_header
-{
-    uint16_t uh_sport; /* source port */
-    uint16_t uh_dport; /* destination port */
-    uint16_t uh_ulen;  /* udp length */
-    uint16_t uh_sum;   /* udp checksum */
-} udp_header;
-
-typedef struct ip_pseudo_header
-{
-    uint32_t ip_src;
-    uint32_t ip_dst;
-    uint8_t  zeros;
-    uint8_t  ip_proto;
-    uint16_t ip_payload;
-} ip_pseudo_header;
-
-#define IP_PROTO_TCP 6
-#define IP_PROTO_UDP 17
-
 #define TCP_HEADER_DATA_OFFSET(tcp) (((be16_to_cpu(tcp->th_offset_flags) >> 12)&0xf) << 2)
 #define TCP_FLAGS_ONLY(flags) ((flags)&0x3f)
 #define TCP_HEADER_FLAGS(tcp) TCP_FLAGS_ONLY(be16_to_cpu(tcp->th_offset_flags))
 
 #define TCP_HEADER_CLEAR_FLAGS(tcp, off) ((tcp)->th_offset_flags &= cpu_to_be16(~TCP_FLAGS_ONLY(off)))
 
-#define TCP_FLAG_FIN  0x01
-#define TCP_FLAG_PUSH 0x08
-
 /* produces ones' complement sum of data */
 static uint16_t ones_complement_sum(uint8_t *data, size_t len)
 {
@@ -2098,7 +2046,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
     }
 
     /* transfer ownership to target */
-    txdw0 &= ~CP_RX_OWN;
+    txdw0 &= ~CP_TX_OWN;
 
     /* reset error indicator bits */
     txdw0 &= ~CP_TX_STATUS_UNF;
@@ -2134,7 +2082,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                 bswap16(txdw1 & CP_TX_VLAN_TAG_MASK));
 
             dot1q_buffer = (uint16_t *) dot1q_buffer_space;
-            dot1q_buffer[0] = cpu_to_be16(ETH_P_8021Q);
+            dot1q_buffer[0] = cpu_to_be16(ETH_P_VLAN);
             /* BE + le_to_cpu() + ~cpu_to_le()~ = BE */
             dot1q_buffer[1] = cpu_to_le16(txdw1 & CP_TX_VLAN_TAG_MASK);
         } else {
@@ -2151,12 +2099,12 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
             DPRINTF("+++ C+ mode offloaded task checksum\n");
 
             /* Large enough for Ethernet and IP headers? */
-            if (saved_size < ETH_HLEN + sizeof(ip_header)) {
+            if (saved_size < ETH_HLEN + sizeof(struct ip_header)) {
                 goto skip_offload;
             }
 
             /* ip packet header */
-            ip_header *ip = NULL;
+            struct ip_header *ip = NULL;
             int hlen = 0;
             uint8_t  ip_protocol = 0;
             uint16_t ip_data_len = 0;
@@ -2172,11 +2120,15 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
 
             DPRINTF("+++ C+ mode has IP packet\n");
 
-            /* not aligned */
+            /* Note on memory alignment: eth_payload_data is 16-bit aligned
+             * since saved_buffer is allocated with g_malloc() and ETH_HLEN is
+             * even.  32-bit accesses must use ldl/stl wrappers to avoid
+             * unaligned accesses.
+             */
             eth_payload_data = saved_buffer + ETH_HLEN;
             eth_payload_len  = saved_size   - ETH_HLEN;
 
-            ip = (ip_header*)eth_payload_data;
+            ip = (struct ip_header*)eth_payload_data;
 
             if (IP_HEADER_VERSION(ip) != IP_HEADER_VERSION_4) {
                 DPRINTF("+++ C+ mode packet has bad IP version %d "
@@ -2185,8 +2137,8 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                 goto skip_offload;
             }
 
-            hlen = IP_HEADER_LENGTH(ip);
-            if (hlen < sizeof(ip_header) || hlen > eth_payload_len) {
+            hlen = IP_HDR_GET_LEN(ip);
+            if (hlen < sizeof(struct ip_header) || hlen > eth_payload_len) {
                 goto skip_offload;
             }
 
@@ -2269,7 +2221,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                     }
 
                     DPRINTF("+++ C+ mode TSO TCP seqno %08x\n",
-                        be32_to_cpu(p_tcp_hdr->th_seq));
+                            ldl_be_p(&p_tcp_hdr->th_seq));
 
                     /* add 4 TCP pseudoheader fields */
                     /* copy IP source and destination fields */
@@ -2287,7 +2239,7 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                     /* keep PUSH and FIN flags only for the last frame */
                     if (!is_last_frame)
                     {
-                        TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TCP_FLAG_PUSH|TCP_FLAG_FIN);
+                        TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TH_PUSH | TH_FIN);
                     }
 
                     /* recalculate TCP checksum */
@@ -2325,7 +2277,8 @@ static int rtl8139_cplus_transmit_one(RTL8139State *s)
                         0, (uint8_t *) dot1q_buffer);
 
                     /* add transferred count to TCP sequence number */
-                    p_tcp_hdr->th_seq = cpu_to_be32(chunk_size + be32_to_cpu(p_tcp_hdr->th_seq));
+                    stl_be_p(&p_tcp_hdr->th_seq,
+                             chunk_size + ldl_be_p(&p_tcp_hdr->th_seq));
                     ++send_count;
                 }
 
@@ -2881,8 +2834,7 @@ static void rtl8139_io_writew(void *opaque, uint8_t addr, uint32_t val)
 
 static void rtl8139_set_next_tctr_time(RTL8139State *s)
 {
-    const uint64_t ns_per_period =
-        muldiv64(0x100000000LL, get_ticks_per_sec(), PCI_FREQUENCY);
+    const uint64_t ns_per_period = (uint64_t)PCI_PERIOD << 32;
 
     DPRINTF("entered rtl8139_set_next_tctr_time\n");
 
@@ -2900,7 +2852,7 @@ static void rtl8139_set_next_tctr_time(RTL8139State *s)
     if (!s->TimerInt) {
         timer_del(s->timer);
     } else {
-        uint64_t delta = muldiv64(s->TimerInt, get_ticks_per_sec(), PCI_FREQUENCY);
+        uint64_t delta = (uint64_t)s->TimerInt * PCI_PERIOD;
         if (s->TCTR_base + delta <= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)) {
             delta += ns_per_period;
         }
@@ -3174,8 +3126,8 @@ static uint32_t rtl8139_io_readl(void *opaque, uint8_t addr)
             break;
 
         case Timer:
-            ret = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base,
-                           PCI_FREQUENCY, get_ticks_per_sec());
+            ret = (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base) /
+                  PCI_PERIOD;
             DPRINTF("TCTR Timer read val=0x%08x\n", ret);
             break;
 
@@ -3269,8 +3221,7 @@ static void rtl8139_pre_save(void *opaque)
     int64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* for migration to older versions */
-    s->TCTR = muldiv64(current_time - s->TCTR_base, PCI_FREQUENCY,
-                       get_ticks_per_sec());
+    s->TCTR = (current_time - s->TCTR_base) / PCI_PERIOD;
     s->rtl8139_mmio_io_addr_dummy = 0;
 }
 
@@ -3438,10 +3389,8 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
 {
     RTL8139State *s = RTL8139(dev);
 
-    if (s->cplus_txbuffer) {
-        g_free(s->cplus_txbuffer);
-        s->cplus_txbuffer = NULL;
-    }
+    g_free(s->cplus_txbuffer);
+    s->cplus_txbuffer = NULL;
     timer_del(s->timer);
     timer_free(s->timer);
     qemu_del_nic(s->nic);
diff --git a/qemu/hw/net/smc91c111.c b/qemu/hw/net/smc91c111.c
index 74e06e6c7..21c1b8f54 100644
--- a/qemu/hw/net/smc91c111.c
+++ b/qemu/hw/net/smc91c111.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/devices.h"
@@ -124,6 +125,25 @@ static void smc91c111_update(smc91c111_state *s)
     qemu_set_irq(s->irq, level);
 }
 
+static int smc91c111_can_receive(smc91c111_state *s)
+{
+    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST)) {
+        return 1;
+    }
+    if (s->allocated == (1 << NUM_PACKETS) - 1 ||
+        s->rx_fifo_len == NUM_PACKETS) {
+        return 0;
+    }
+    return 1;
+}
+
+static inline void smc91c111_flush_queued_packets(smc91c111_state *s)
+{
+    if (smc91c111_can_receive(s)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+}
+
 /* Try to allocate a packet.  Returns 0x80 on failure.  */
 static int smc91c111_allocate_packet(smc91c111_state *s)
 {
@@ -164,6 +184,7 @@ static void smc91c111_pop_rx_fifo(smc91c111_state *s)
     } else {
         s->int_level &= ~INT_RCV;
     }
+    smc91c111_flush_queued_packets(s);
     smc91c111_update(s);
 }
 
@@ -185,7 +206,7 @@ static void smc91c111_release_packet(smc91c111_state *s, int packet)
     s->allocated &= ~(1 << packet);
     if (s->tx_alloc == 0x80)
         smc91c111_tx_alloc(s);
-    qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    smc91c111_flush_queued_packets(s);
 }
 
 /* Flush the TX FIFO.  */
@@ -311,6 +332,7 @@ static void smc91c111_writeb(void *opaque, hwaddr offset,
             if (s->rcr & RCR_SOFT_RST) {
                 smc91c111_reset(DEVICE(s));
             }
+            smc91c111_flush_queued_packets(s);
             return;
         case 10: case 11: /* RPCR */
             /* Ignored */
@@ -636,15 +658,11 @@ static uint32_t smc91c111_readl(void *opaque, hwaddr offset)
     return val;
 }
 
-static int smc91c111_can_receive(NetClientState *nc)
+static int smc91c111_can_receive_nc(NetClientState *nc)
 {
     smc91c111_state *s = qemu_get_nic_opaque(nc);
 
-    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST))
-        return 1;
-    if (s->allocated == (1 << NUM_PACKETS) - 1)
-        return 0;
-    return 1;
+    return smc91c111_can_receive(s);
 }
 
 static ssize_t smc91c111_receive(NetClientState *nc, const uint8_t *buf, size_t size)
@@ -739,7 +757,7 @@ static const MemoryRegionOps smc91c111_mem_ops = {
 static NetClientInfo net_smc91c111_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
-    .can_receive = smc91c111_can_receive,
+    .can_receive = smc91c111_can_receive_nc,
     .receive = smc91c111_receive,
 };
 
diff --git a/qemu/hw/net/spapr_llan.c b/qemu/hw/net/spapr_llan.c
index 1ca5e9ce6..a647f25d9 100644
--- a/qemu/hw/net/spapr_llan.c
+++ b/qemu/hw/net/spapr_llan.c
@@ -24,6 +24,9 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "net/net.h"
 #include "hw/qdev.h"
@@ -44,6 +47,10 @@
 #define DPRINTF(fmt...)
 #endif
 
+/* Compatibility flags for migration */
+#define SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT  0
+#define SPAPRVLAN_FLAG_RX_BUF_POOLS      (1 << SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT)
+
 /*
  * Virtual LAN device
  */
@@ -85,6 +92,15 @@ typedef uint64_t vlan_bd_t;
 #define VIO_SPAPR_VLAN_DEVICE(obj) \
      OBJECT_CHECK(VIOsPAPRVLANDevice, (obj), TYPE_VIO_SPAPR_VLAN_DEVICE)
 
+#define RX_POOL_MAX_BDS 4096
+#define RX_MAX_POOLS 5
+
+typedef struct {
+    int32_t bufsize;
+    int32_t count;
+    vlan_bd_t bds[RX_POOL_MAX_BDS];
+} RxBufPool;
+
 typedef struct VIOsPAPRVLANDevice {
     VIOsPAPRDevice sdev;
     NICConf nicconf;
@@ -93,6 +109,8 @@ typedef struct VIOsPAPRVLANDevice {
     target_ulong buf_list;
     uint32_t add_buf_ptr, use_buf_ptr, rx_bufs;
     target_ulong rxq_ptr;
+    uint32_t compat_flags;             /* Compatability flags for migration */
+    RxBufPool *rx_pool[RX_MAX_POOLS];  /* Receive buffer descriptor pools */
 } VIOsPAPRVLANDevice;
 
 static int spapr_vlan_can_receive(NetClientState *nc)
@@ -102,6 +120,73 @@ static int spapr_vlan_can_receive(NetClientState *nc)
     return (dev->isopen && dev->rx_bufs > 0);
 }
 
+/**
+ * Get buffer descriptor from one of our receive buffer pools
+ */
+static vlan_bd_t spapr_vlan_get_rx_bd_from_pool(VIOsPAPRVLANDevice *dev,
+                                                size_t size)
+{
+    vlan_bd_t bd;
+    int pool;
+
+    for (pool = 0; pool < RX_MAX_POOLS; pool++) {
+        if (dev->rx_pool[pool]->count > 0 &&
+            dev->rx_pool[pool]->bufsize >= size + 8) {
+            break;
+        }
+    }
+    if (pool == RX_MAX_POOLS) {
+        /* Failed to find a suitable buffer */
+        return 0;
+    }
+
+    DPRINTF("Found buffer: pool=%d count=%d rxbufs=%d\n", pool,
+            dev->rx_pool[pool]->count, dev->rx_bufs);
+
+    /* Remove the buffer from the pool */
+    dev->rx_pool[pool]->count--;
+    bd = dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count];
+    dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count] = 0;
+
+    return bd;
+}
+
+/**
+ * Get buffer descriptor from the receive buffer list page that has been
+ * supplied by the guest with the H_REGISTER_LOGICAL_LAN call
+ */
+static vlan_bd_t spapr_vlan_get_rx_bd_from_page(VIOsPAPRVLANDevice *dev,
+                                                size_t size)
+{
+    int buf_ptr = dev->use_buf_ptr;
+    vlan_bd_t bd;
+
+    do {
+        buf_ptr += 8;
+        if (buf_ptr >= VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF) {
+            buf_ptr = VLAN_RX_BDS_OFF;
+        }
+
+        bd = vio_ldq(&dev->sdev, dev->buf_list + buf_ptr);
+        DPRINTF("use_buf_ptr=%d bd=0x%016llx\n",
+                buf_ptr, (unsigned long long)bd);
+    } while ((!(bd & VLAN_BD_VALID) || VLAN_BD_LEN(bd) < size + 8)
+             && buf_ptr != dev->use_buf_ptr);
+
+    if (!(bd & VLAN_BD_VALID) || VLAN_BD_LEN(bd) < size + 8) {
+        /* Failed to find a suitable buffer */
+        return 0;
+    }
+
+    /* Remove the buffer from the pool */
+    dev->use_buf_ptr = buf_ptr;
+    vio_stq(&dev->sdev, dev->buf_list + dev->use_buf_ptr, 0);
+
+    DPRINTF("Found buffer: ptr=%d rxbufs=%d\n", dev->use_buf_ptr, dev->rx_bufs);
+
+    return bd;
+}
+
 static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
                                   size_t size)
 {
@@ -109,7 +194,6 @@ static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
     VIOsPAPRDevice *sdev = VIO_SPAPR_DEVICE(dev);
     vlan_bd_t rxq_bd = vio_ldq(sdev, dev->buf_list + VLAN_RXQ_BD_OFF);
     vlan_bd_t bd;
-    int buf_ptr = dev->use_buf_ptr;
     uint64_t handle;
     uint8_t control;
 
@@ -124,29 +208,16 @@ static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
         return -1;
     }
 
-    do {
-        buf_ptr += 8;
-        if (buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
-            buf_ptr = VLAN_RX_BDS_OFF;
-        }
-
-        bd = vio_ldq(sdev, dev->buf_list + buf_ptr);
-        DPRINTF("use_buf_ptr=%d bd=0x%016llx\n",
-                buf_ptr, (unsigned long long)bd);
-    } while ((!(bd & VLAN_BD_VALID) || (VLAN_BD_LEN(bd) < (size + 8)))
-             && (buf_ptr != dev->use_buf_ptr));
-
-    if (!(bd & VLAN_BD_VALID) || (VLAN_BD_LEN(bd) < (size + 8))) {
-        /* Failed to find a suitable buffer */
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        bd = spapr_vlan_get_rx_bd_from_pool(dev, size);
+    } else {
+        bd = spapr_vlan_get_rx_bd_from_page(dev, size);
+    }
+    if (!bd) {
         return -1;
     }
 
-    /* Remove the buffer from the pool */
     dev->rx_bufs--;
-    dev->use_buf_ptr = buf_ptr;
-    vio_stq(sdev, dev->buf_list + dev->use_buf_ptr, 0);
-
-    DPRINTF("Found buffer: ptr=%d num=%d\n", dev->use_buf_ptr, dev->rx_bufs);
 
     /* Transfer the packet data */
     if (spapr_vio_dma_write(sdev, VLAN_BD_ADDR(bd) + 8, buf, size) < 0) {
@@ -194,13 +265,31 @@ static NetClientInfo net_spapr_vlan_info = {
     .receive = spapr_vlan_receive,
 };
 
+static void spapr_vlan_reset_rx_pool(RxBufPool *rxp)
+{
+    /*
+     * Use INT_MAX as bufsize so that unused buffers are moved to the end
+     * of the list during the qsort in spapr_vlan_add_rxbuf_to_pool() later.
+     */
+    rxp->bufsize = INT_MAX;
+    rxp->count = 0;
+    memset(rxp->bds, 0, sizeof(rxp->bds));
+}
+
 static void spapr_vlan_reset(VIOsPAPRDevice *sdev)
 {
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(sdev);
+    int i;
 
     dev->buf_list = 0;
     dev->rx_bufs = 0;
     dev->isopen = 0;
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            spapr_vlan_reset_rx_pool(dev->rx_pool[i]);
+        }
+    }
 }
 
 static void spapr_vlan_realize(VIOsPAPRDevice *sdev, Error **errp)
@@ -217,10 +306,31 @@ static void spapr_vlan_realize(VIOsPAPRDevice *sdev, Error **errp)
 static void spapr_vlan_instance_init(Object *obj)
 {
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(obj);
+    int i;
 
     device_add_bootindex_property(obj, &dev->nicconf.bootindex,
                                   "bootindex", "",
                                   DEVICE(dev), NULL);
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            dev->rx_pool[i] = g_new(RxBufPool, 1);
+            spapr_vlan_reset_rx_pool(dev->rx_pool[i]);
+        }
+    }
+}
+
+static void spapr_vlan_instance_finalize(Object *obj)
+{
+    VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(obj);
+    int i;
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            g_free(dev->rx_pool[i]);
+            dev->rx_pool[i] = NULL;
+        }
+    }
 }
 
 void spapr_vlan_create(VIOsPAPRBus *bus, NICInfo *nd)
@@ -371,6 +481,113 @@ static target_ulong h_free_logical_lan(PowerPCCPU *cpu,
     return H_SUCCESS;
 }
 
+/**
+ * Used for qsort, this function compares two RxBufPools by size.
+ */
+static int rx_pool_size_compare(const void *p1, const void *p2)
+{
+    const RxBufPool *pool1 = *(RxBufPool **)p1;
+    const RxBufPool *pool2 = *(RxBufPool **)p2;
+
+    if (pool1->bufsize < pool2->bufsize) {
+        return -1;
+    }
+    return pool1->bufsize > pool2->bufsize;
+}
+
+/**
+ * Search for a matching buffer pool with exact matching size,
+ * or return -1 if no matching pool has been found.
+ */
+static int spapr_vlan_get_rx_pool_id(VIOsPAPRVLANDevice *dev, int size)
+{
+    int pool;
+
+    for (pool = 0; pool < RX_MAX_POOLS; pool++) {
+        if (dev->rx_pool[pool]->bufsize == size) {
+            return pool;
+        }
+    }
+
+    return -1;
+}
+
+/**
+ * Enqueuing receive buffer by adding it to one of our receive buffer pools
+ */
+static target_long spapr_vlan_add_rxbuf_to_pool(VIOsPAPRVLANDevice *dev,
+                                                target_ulong buf)
+{
+    int size = VLAN_BD_LEN(buf);
+    int pool;
+
+    pool = spapr_vlan_get_rx_pool_id(dev, size);
+    if (pool < 0) {
+        /*
+         * No matching pool found? Try to use a new one. If the guest used all
+         * pools before, but changed the size of one pool inbetween, we might
+         * need to recycle that pool here (if it's empty already). Thus scan
+         * all buffer pools now, starting with the last (likely empty) one.
+         */
+        for (pool = RX_MAX_POOLS - 1; pool >= 0 ; pool--) {
+            if (dev->rx_pool[pool]->count == 0) {
+                dev->rx_pool[pool]->bufsize = size;
+                /*
+                 * Sort pools by size so that spapr_vlan_receive()
+                 * can later find the smallest buffer pool easily.
+                 */
+                qsort(dev->rx_pool, RX_MAX_POOLS, sizeof(dev->rx_pool[0]),
+                      rx_pool_size_compare);
+                pool = spapr_vlan_get_rx_pool_id(dev, size);
+                DPRINTF("created RX pool %d for size %lld\n", pool,
+                        VLAN_BD_LEN(buf));
+                break;
+            }
+        }
+    }
+    /* Still no usable pool? Give up */
+    if (pool < 0 || dev->rx_pool[pool]->count >= RX_POOL_MAX_BDS) {
+        return H_RESOURCE;
+    }
+
+    DPRINTF("h_add_llan_buf():  Add buf using pool %i (size %lli, count=%i)\n",
+            pool, VLAN_BD_LEN(buf), dev->rx_pool[pool]->count);
+
+    dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count++] = buf;
+
+    return 0;
+}
+
+/**
+ * This is the old way of enqueuing receive buffers: Add it to the rx queue
+ * page that has been supplied by the guest (which is quite limited in size).
+ */
+static target_long spapr_vlan_add_rxbuf_to_page(VIOsPAPRVLANDevice *dev,
+                                                target_ulong buf)
+{
+    vlan_bd_t bd;
+
+    if (dev->rx_bufs >= VLAN_MAX_BUFS) {
+        return H_RESOURCE;
+    }
+
+    do {
+        dev->add_buf_ptr += 8;
+        if (dev->add_buf_ptr >= VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF) {
+            dev->add_buf_ptr = VLAN_RX_BDS_OFF;
+        }
+
+        bd = vio_ldq(&dev->sdev, dev->buf_list + dev->add_buf_ptr);
+    } while (bd & VLAN_BD_VALID);
+
+    vio_stq(&dev->sdev, dev->buf_list + dev->add_buf_ptr, buf);
+
+    DPRINTF("h_add_llan_buf():  Added buf  ptr=%d  rx_bufs=%d bd=0x%016llx\n",
+            dev->add_buf_ptr, dev->rx_bufs, (unsigned long long)buf);
+
+    return 0;
+}
+
 static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
                                              sPAPRMachineState *spapr,
                                              target_ulong opcode,
@@ -380,7 +597,7 @@ static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
     target_ulong buf = args[1];
     VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(sdev);
-    vlan_bd_t bd;
+    target_long ret;
 
     DPRINTF("H_ADD_LOGICAL_LAN_BUFFER(0x" TARGET_FMT_lx
             ", 0x" TARGET_FMT_lx ")\n", reg, buf);
@@ -396,29 +613,23 @@ static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
         return H_PARAMETER;
     }
 
-    if (!dev->isopen || dev->rx_bufs >= VLAN_MAX_BUFS) {
+    if (!dev->isopen) {
         return H_RESOURCE;
     }
 
-    do {
-        dev->add_buf_ptr += 8;
-        if (dev->add_buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
-            dev->add_buf_ptr = VLAN_RX_BDS_OFF;
-        }
-
-        bd = vio_ldq(sdev, dev->buf_list + dev->add_buf_ptr);
-    } while (bd & VLAN_BD_VALID);
-
-    vio_stq(sdev, dev->buf_list + dev->add_buf_ptr, buf);
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        ret = spapr_vlan_add_rxbuf_to_pool(dev, buf);
+    } else {
+        ret = spapr_vlan_add_rxbuf_to_page(dev, buf);
+    }
+    if (ret) {
+        return ret;
+    }
 
     dev->rx_bufs++;
 
     qemu_flush_queued_packets(qemu_get_queue(dev->nic));
 
-    DPRINTF("h_add_logical_lan_buffer():  Added buf  ptr=%d  rx_bufs=%d"
-            " bd=0x%016llx\n", dev->add_buf_ptr, dev->rx_bufs,
-            (unsigned long long)buf);
-
     return H_SUCCESS;
 }
 
@@ -508,9 +719,44 @@ static target_ulong h_multicast_ctrl(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 static Property spapr_vlan_properties[] = {
     DEFINE_SPAPR_PROPERTIES(VIOsPAPRVLANDevice, sdev),
     DEFINE_NIC_PROPERTIES(VIOsPAPRVLANDevice, nicconf),
+    DEFINE_PROP_BIT("use-rx-buffer-pools", VIOsPAPRVLANDevice,
+                    compat_flags, SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static bool spapr_vlan_rx_buffer_pools_needed(void *opaque)
+{
+    VIOsPAPRVLANDevice *dev = opaque;
+
+    return (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) != 0;
+}
+
+static const VMStateDescription vmstate_rx_buffer_pool = {
+    .name = "spapr_llan/rx_buffer_pool",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_vlan_rx_buffer_pools_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(bufsize, RxBufPool),
+        VMSTATE_INT32(count, RxBufPool),
+        VMSTATE_UINT64_ARRAY(bds, RxBufPool, RX_POOL_MAX_BDS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_rx_pools = {
+    .name = "spapr_llan/rx_pools",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_vlan_rx_buffer_pools_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_ARRAY_OF_POINTER_TO_STRUCT(rx_pool, VIOsPAPRVLANDevice,
+                                           RX_MAX_POOLS, 1,
+                                           vmstate_rx_buffer_pool, RxBufPool),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_spapr_llan = {
     .name = "spapr_llan",
     .version_id = 1,
@@ -527,6 +773,10 @@ static const VMStateDescription vmstate_spapr_llan = {
 
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_rx_pools,
+        NULL
+    }
 };
 
 static void spapr_vlan_class_init(ObjectClass *klass, void *data)
@@ -553,6 +803,7 @@ static const TypeInfo spapr_vlan_info = {
     .instance_size = sizeof(VIOsPAPRVLANDevice),
     .class_init    = spapr_vlan_class_init,
     .instance_init = spapr_vlan_instance_init,
+    .instance_finalize = spapr_vlan_instance_finalize,
 };
 
 static void spapr_vlan_register_types(void)
diff --git a/qemu/hw/net/stellaris_enet.c b/qemu/hw/net/stellaris_enet.c
index 21a47735d..688089494 100644
--- a/qemu/hw/net/stellaris_enet.c
+++ b/qemu/hw/net/stellaris_enet.c
@@ -6,6 +6,7 @@
  *
  * This code is licensed under the GPL.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include <zlib.h>
@@ -235,8 +236,18 @@ static ssize_t stellaris_enet_receive(NetClientState *nc, const uint8_t *buf, si
     n = s->next_packet + s->np;
     if (n >= 31)
         n -= 31;
-    s->np++;
 
+    if (size >= sizeof(s->rx[n].data) - 6) {
+        /* If the packet won't fit into the
+         * emulated 2K RAM, this is reported
+         * as a FIFO overrun error.
+         */
+        s->ris |= SE_INT_FOV;
+        stellaris_enet_update(s);
+        return -1;
+    }
+
+    s->np++;
     s->rx[n].len = size + 6;
     p = s->rx[n].data;
     *(p++) = (size + 6);
diff --git a/qemu/hw/net/vhost_net.c b/qemu/hw/net/vhost_net.c
index 5c1d11f51..6e1032fc1 100644
--- a/qemu/hw/net/vhost_net.c
+++ b/qemu/hw/net/vhost_net.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
@@ -21,24 +22,20 @@
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
 
-#include "config.h"
 
 #ifdef CONFIG_VHOST_NET
 #include <linux/vhost.h>
 #include <sys/socket.h>
 #include <linux/kvm.h>
-#include <fcntl.h>
 #include <netpacket/packet.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <netinet/in.h>
 
-#include <stdio.h>
 
 #include "standard-headers/linux/virtio_ring.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/virtio-bus.h"
-#include "hw/virtio/virtio-access.h"
 
 struct vhost_net {
     struct vhost_dev dev;
@@ -77,13 +74,9 @@ static const int user_feature_bits[] = {
     VIRTIO_NET_F_HOST_ECN,
     VIRTIO_NET_F_HOST_UFO,
     VIRTIO_NET_F_MRG_RXBUF,
-    VIRTIO_NET_F_STATUS,
-    VIRTIO_NET_F_CTRL_VQ,
-    VIRTIO_NET_F_CTRL_RX,
-    VIRTIO_NET_F_CTRL_VLAN,
-    VIRTIO_NET_F_CTRL_RX_EXTRA,
-    VIRTIO_NET_F_CTRL_MAC_ADDR,
-    VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
+
+    /* This bit implies RARP isn't sent by QEMU out of band */
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
 
     VIRTIO_NET_F_MQ,
 
@@ -122,6 +115,11 @@ void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
     vhost_ack_features(&net->dev, vhost_net_get_feature_bits(net), features);
 }
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return net->dev.max_queues;
+}
+
 static int vhost_net_get_fd(NetClientState *backend)
 {
     switch (backend->info->type) {
@@ -143,6 +141,11 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options)
         fprintf(stderr, "vhost-net requires net backend to be setup\n");
         goto fail;
     }
+    net->nc = options->net_backend;
+
+    net->dev.max_queues = 1;
+    net->dev.nvqs = 2;
+    net->dev.vqs = net->vqs;
 
     if (backend_kernel) {
         r = vhost_net_get_fd(options->net_backend);
@@ -152,14 +155,15 @@ struct vhost_net *vhost_net_init(VhostNetOptions *options)
         net->dev.backend_features = qemu_has_vnet_hdr(options->net_backend)
             ? 0 : (1ULL << VHOST_NET_F_VIRTIO_NET_HDR);
         net->backend = r;
+        net->dev.protocol_features = 0;
     } else {
         net->dev.backend_features = 0;
+        net->dev.protocol_features = 0;
         net->backend = -1;
-    }
-    net->nc = options->net_backend;
 
-    net->dev.nvqs = 2;
-    net->dev.vqs = net->vqs;
+        /* vhost-user needs vq_index to initiate a specific queue pair */
+        net->dev.vq_index = net->nc->queue_index * net->dev.nvqs;
+    }
 
     r = vhost_dev_init(&net->dev, options->opaque,
                        options->backend_type);
@@ -192,27 +196,6 @@ static void vhost_net_set_vq_index(struct vhost_net *net, int vq_index)
     net->dev.vq_index = vq_index;
 }
 
-static int vhost_net_set_vnet_endian(VirtIODevice *dev, NetClientState *peer,
-                                     bool set)
-{
-    int r = 0;
-
-    if (virtio_has_feature(dev, VIRTIO_F_VERSION_1) ||
-        (virtio_legacy_is_cross_endian(dev) && !virtio_is_big_endian(dev))) {
-        r = qemu_set_vnet_le(peer, set);
-        if (r) {
-            error_report("backend does not support LE vnet headers");
-        }
-    } else if (virtio_legacy_is_cross_endian(dev)) {
-        r = qemu_set_vnet_be(peer, set);
-        if (r) {
-            error_report("backend does not support BE vnet headers");
-        }
-    }
-
-    return r;
-}
-
 static int vhost_net_start_one(struct vhost_net *net,
                                VirtIODevice *dev)
 {
@@ -241,8 +224,7 @@ static int vhost_net_start_one(struct vhost_net *net,
         file.fd = net->backend;
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                      &file);
+            r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             if (r < 0) {
                 r = -errno;
                 goto fail;
@@ -255,8 +237,7 @@ fail:
     if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
         while (file.index-- > 0) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
+            int r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -278,15 +259,7 @@ static void vhost_net_stop_one(struct vhost_net *net,
     if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
             const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
-            assert(r >= 0);
-        }
-    } else if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
-        for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
-            const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_RESET_OWNER,
-                                          NULL);
+            int r = vhost_ops->vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -307,23 +280,28 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
-        r = -ENOSYS;
-        goto err;
-    }
-
-    r = vhost_net_set_vnet_endian(dev, ncs[0].peer, true);
-    if (r < 0) {
-        goto err;
+        return -ENOSYS;
     }
 
     for (i = 0; i < total_queues; i++) {
-        vhost_net_set_vq_index(get_vhost_net(ncs[i].peer), i * 2);
-    }
+        struct vhost_net *net;
+
+        net = get_vhost_net(ncs[i].peer);
+        vhost_net_set_vq_index(net, i * 2);
+
+        /* Suppress the masking guest notifiers on vhost user
+         * because vhost user doesn't interrupt masking/unmasking
+         * properly.
+         */
+        if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
+                dev->use_guest_notifier_mask = false;
+        }
+     }
 
     r = k->set_guest_notifiers(qbus->parent, total_queues * 2, true);
     if (r < 0) {
         error_report("Error binding guest notifier: %d", -r);
-        goto err_endian;
+        goto err;
     }
 
     for (i = 0; i < total_queues; i++) {
@@ -345,8 +323,6 @@ err_start:
         fprintf(stderr, "vhost guest notifier cleanup failed: %d\n", e);
         fflush(stderr);
     }
-err_endian:
-    vhost_net_set_vnet_endian(dev, ncs[0].peer, false);
 err:
     return r;
 }
@@ -369,8 +345,6 @@ void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs,
         fflush(stderr);
     }
     assert(r >= 0);
-
-    assert(vhost_net_set_vnet_endian(dev, ncs[0].peer, false) >= 0);
 }
 
 void vhost_net_cleanup(struct vhost_net *net)
@@ -379,6 +353,18 @@ void vhost_net_cleanup(struct vhost_net *net)
     g_free(net);
 }
 
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    int r = -1;
+
+    if (vhost_ops->vhost_migration_done) {
+        r = vhost_ops->vhost_migration_done(&net->dev, mac_addr);
+    }
+
+    return r;
+}
+
 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
 {
     return vhost_virtqueue_pending(&net->dev, idx);
@@ -411,7 +397,25 @@ VHostNetState *get_vhost_net(NetClientState *nc)
 
     return vhost_net;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    if (vhost_ops->vhost_set_vring_enable) {
+        return vhost_ops->vhost_set_vring_enable(&net->dev, enable);
+    }
+
+    return 0;
+}
+
 #else
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return 1;
+}
+
 struct vhost_net *vhost_net_init(VhostNetOptions *options)
 {
     error_report("vhost-net support is not compiled in");
@@ -452,8 +456,18 @@ void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
 {
 }
 
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    return -1;
+}
+
 VHostNetState *get_vhost_net(NetClientState *nc)
 {
     return 0;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    return 0;
+}
 #endif
diff --git a/qemu/hw/net/virtio-net.c b/qemu/hw/net/virtio-net.c
index 151083954..5798f87d8 100644
--- a/qemu/hw/net/virtio-net.c
+++ b/qemu/hw/net/virtio-net.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 #include "hw/virtio/virtio.h"
 #include "net/net.h"
@@ -86,8 +87,8 @@ static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config)
 
     memcpy(&netcfg, config, n->config_size);
 
-    if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
-        !virtio_has_feature(vdev, VIRTIO_F_VERSION_1) &&
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
+        !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) &&
         memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
         memcpy(n->mac, netcfg.mac, ETH_ALEN);
         qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
@@ -128,6 +129,13 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
     if (!n->vhost_started) {
         int r, i;
 
+        if (n->needs_vnet_hdr_swap) {
+            error_report("backend does not support %s vnet headers; "
+                         "falling back on userspace virtio",
+                         virtio_is_big_endian(vdev) ? "BE" : "LE");
+            return;
+        }
+
         /* Any packets outstanding? Purge them to avoid touching rings
          * when vhost is running.
          */
@@ -152,6 +160,59 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
     }
 }
 
+static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
+                                          NetClientState *peer,
+                                          bool enable)
+{
+    if (virtio_is_big_endian(vdev)) {
+        return qemu_set_vnet_be(peer, enable);
+    } else {
+        return qemu_set_vnet_le(peer, enable);
+    }
+}
+
+static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs,
+                                       int queues, bool enable)
+{
+    int i;
+
+    for (i = 0; i < queues; i++) {
+        if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 &&
+            enable) {
+            while (--i >= 0) {
+                virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false);
+            }
+
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(n);
+    int queues = n->multiqueue ? n->max_queues : 1;
+
+    if (virtio_net_started(n, status)) {
+        /* Before using the device, we tell the network backend about the
+         * endianness to use when parsing vnet headers. If the backend
+         * can't do it, we fallback onto fixing the headers in the core
+         * virtio-net code.
+         */
+        n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs,
+                                                            queues, true);
+    } else if (virtio_net_started(n, vdev->status)) {
+        /* After using the device, we need to reset the network backend to
+         * the default (guest native endianness), otherwise the guest may
+         * lose network connectivity if it is rebooted into a different
+         * endianness.
+         */
+        virtio_net_set_vnet_endian(vdev, n->nic->ncs, queues, false);
+    }
+}
+
 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
 {
     VirtIONet *n = VIRTIO_NET(vdev);
@@ -159,6 +220,7 @@ static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
     int i;
     uint8_t queue_status;
 
+    virtio_net_vnet_endian_status(n, status);
     virtio_net_vhost_status(n, status);
 
     for (i = 0; i < n->max_queues; i++) {
@@ -304,7 +366,7 @@ static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc)
     info->multicast_table = str_list;
     info->vlan_table = get_vlan_table(n);
 
-    if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) {
         info->vlan = RX_STATE_ALL;
     } else if (!info->vlan_table) {
         info->vlan = RX_STATE_NONE;
@@ -406,6 +468,10 @@ static int peer_attach(VirtIONet *n, int index)
         return 0;
     }
 
+    if (nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 1);
+    }
+
     if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
         return 0;
     }
@@ -421,6 +487,10 @@ static int peer_detach(VirtIONet *n, int index)
         return 0;
     }
 
+    if (nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 0);
+    }
+
     if (nc->peer->info->type !=  NET_CLIENT_OPTIONS_KIND_TAP) {
         return 0;
     }
@@ -529,13 +599,13 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features)
     int i;
 
     virtio_net_set_multiqueue(n,
-                              __virtio_has_feature(features, VIRTIO_NET_F_MQ));
+                              virtio_has_feature(features, VIRTIO_NET_F_MQ));
 
     virtio_net_set_mrg_rx_bufs(n,
-                               __virtio_has_feature(features,
-                                                    VIRTIO_NET_F_MRG_RXBUF),
-                               __virtio_has_feature(features,
-                                                    VIRTIO_F_VERSION_1));
+                               virtio_has_feature(features,
+                                                  VIRTIO_NET_F_MRG_RXBUF),
+                               virtio_has_feature(features,
+                                                  VIRTIO_F_VERSION_1));
 
     if (n->has_vnet_hdr) {
         n->curr_guest_offloads =
@@ -552,7 +622,7 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features)
         vhost_net_ack_features(get_vhost_net(nc->peer), features);
     }
 
-    if (__virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) {
+    if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) {
         memset(n->vlans, 0, MAX_VLAN >> 3);
     } else {
         memset(n->vlans, 0xff, MAX_VLAN >> 3);
@@ -599,7 +669,7 @@ static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd,
     uint64_t offloads;
     size_t s;
 
-    if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
         return VIRTIO_NET_ERR;
     }
 
@@ -810,20 +880,24 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
     VirtIONet *n = VIRTIO_NET(vdev);
     struct virtio_net_ctrl_hdr ctrl;
     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     size_t s;
     struct iovec *iov, *iov2;
     unsigned int iov_cnt;
 
-    while (virtqueue_pop(vq, &elem)) {
-        if (iov_size(elem.in_sg, elem.in_num) < sizeof(status) ||
-            iov_size(elem.out_sg, elem.out_num) < sizeof(ctrl)) {
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+        if (iov_size(elem->in_sg, elem->in_num) < sizeof(status) ||
+            iov_size(elem->out_sg, elem->out_num) < sizeof(ctrl)) {
             error_report("virtio-net ctrl missing headers");
             exit(1);
         }
 
-        iov_cnt = elem.out_num;
-        iov2 = iov = g_memdup(elem.out_sg, sizeof(struct iovec) * elem.out_num);
+        iov_cnt = elem->out_num;
+        iov2 = iov = g_memdup(elem->out_sg, sizeof(struct iovec) * elem->out_num);
         s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl));
         iov_discard_front(&iov, &iov_cnt, sizeof(ctrl));
         if (s != sizeof(ctrl)) {
@@ -842,12 +916,13 @@ static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
             status = virtio_net_handle_offloads(n, ctrl.cmd, iov, iov_cnt);
         }
 
-        s = iov_from_buf(elem.in_sg, elem.in_num, 0, &status, sizeof(status));
+        s = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, sizeof(status));
         assert(s == sizeof(status));
 
-        virtqueue_push(vq, &elem, sizeof(status));
+        virtqueue_push(vq, elem, sizeof(status));
         virtio_notify(vdev, vq);
         g_free(iov2);
+        g_free(elem);
     }
 }
 
@@ -949,7 +1024,10 @@ static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt,
         void *wbuf = (void *)buf;
         work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
                                     size - n->host_hdr_len);
-        virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
+
+        if (n->needs_vnet_hdr_swap) {
+            virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
+        }
         iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
     } else {
         struct virtio_net_hdr hdr = {
@@ -1036,13 +1114,14 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
     offset = i = 0;
 
     while (offset < size) {
-        VirtQueueElement elem;
+        VirtQueueElement *elem;
         int len, total;
-        const struct iovec *sg = elem.in_sg;
+        const struct iovec *sg;
 
         total = 0;
 
-        if (virtqueue_pop(q->rx_vq, &elem) == 0) {
+        elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement));
+        if (!elem) {
             if (i == 0)
                 return -1;
             error_report("virtio-net unexpected empty queue: "
@@ -1055,21 +1134,22 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
             exit(1);
         }
 
-        if (elem.in_num < 1) {
+        if (elem->in_num < 1) {
             error_report("virtio-net receive queue contains no in buffers");
             exit(1);
         }
 
+        sg = elem->in_sg;
         if (i == 0) {
             assert(offset == 0);
             if (n->mergeable_rx_bufs) {
                 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
-                                    sg, elem.in_num,
+                                    sg, elem->in_num,
                                     offsetof(typeof(mhdr), num_buffers),
                                     sizeof(mhdr.num_buffers));
             }
 
-            receive_header(n, sg, elem.in_num, buf, size);
+            receive_header(n, sg, elem->in_num, buf, size);
             offset = n->host_hdr_len;
             total += n->guest_hdr_len;
             guest_offset = n->guest_hdr_len;
@@ -1078,7 +1158,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
         }
 
         /* copy in packet.  ugh */
-        len = iov_from_buf(sg, elem.in_num, guest_offset,
+        len = iov_from_buf(sg, elem->in_num, guest_offset,
                            buf + offset, size - offset);
         total += len;
         offset += len;
@@ -1086,18 +1166,14 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
          * must have consumed the complete packet.
          * Otherwise, drop it. */
         if (!n->mergeable_rx_bufs && offset < size) {
-#if 0
-            error_report("virtio-net truncated non-mergeable packet: "
-                         "i %zd mergeable %d offset %zd, size %zd, "
-                         "guest hdr len %zd, host hdr len %zd",
-                         i, n->mergeable_rx_bufs,
-                         offset, size, n->guest_hdr_len, n->host_hdr_len);
-#endif
+            virtqueue_discard(q->rx_vq, elem, total);
+            g_free(elem);
             return size;
         }
 
         /* signal other side */
-        virtqueue_fill(q->rx_vq, &elem, total, i++);
+        virtqueue_fill(q->rx_vq, elem, total, i++);
+        g_free(elem);
     }
 
     if (mhdr_cnt) {
@@ -1121,10 +1197,11 @@ static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
     VirtIONetQueue *q = virtio_net_get_subqueue(nc);
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
 
-    virtqueue_push(q->tx_vq, &q->async_tx.elem, 0);
+    virtqueue_push(q->tx_vq, q->async_tx.elem, 0);
     virtio_notify(vdev, q->tx_vq);
 
-    q->async_tx.elem.out_num = q->async_tx.len = 0;
+    g_free(q->async_tx.elem);
+    q->async_tx.elem = NULL;
 
     virtio_queue_set_notification(q->tx_vq, 1);
     virtio_net_flush_tx(q);
@@ -1135,25 +1212,31 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
 {
     VirtIONet *n = q->n;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     int32_t num_packets = 0;
     int queue_index = vq2q(virtio_get_queue_index(q->tx_vq));
     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return num_packets;
     }
 
-    if (q->async_tx.elem.out_num) {
+    if (q->async_tx.elem) {
         virtio_queue_set_notification(q->tx_vq, 0);
         return num_packets;
     }
 
-    while (virtqueue_pop(q->tx_vq, &elem)) {
-        ssize_t ret, len;
-        unsigned int out_num = elem.out_num;
-        struct iovec *out_sg = &elem.out_sg[0];
-        struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1];
+    for (;;) {
+        ssize_t ret;
+        unsigned int out_num;
+        struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg;
         struct virtio_net_hdr_mrg_rxbuf mhdr;
 
+        elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        out_num = elem->out_num;
+        out_sg = elem->out_sg;
         if (out_num < 1) {
             error_report("virtio-net header not in first element");
             exit(1);
@@ -1165,7 +1248,7 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
                 error_report("virtio-net header incorrect");
                 exit(1);
             }
-            if (virtio_needs_swap(vdev)) {
+            if (n->needs_vnet_hdr_swap) {
                 virtio_net_hdr_swap(vdev, (void *) &mhdr);
                 sg2[0].iov_base = &mhdr;
                 sg2[0].iov_len = n->guest_hdr_len;
@@ -1196,21 +1279,18 @@ static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
             out_sg = sg;
         }
 
-        len = n->guest_hdr_len;
-
         ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
                                       out_sg, out_num, virtio_net_tx_complete);
         if (ret == 0) {
             virtio_queue_set_notification(q->tx_vq, 0);
             q->async_tx.elem = elem;
-            q->async_tx.len  = len;
             return -EBUSY;
         }
 
-        len += ret;
 drop:
-        virtqueue_push(q->tx_vq, &elem, 0);
+        virtqueue_push(q->tx_vq, elem, 0);
         virtio_notify(vdev, q->tx_vq);
+        g_free(elem);
 
         if (++num_packets >= n->tx_burst) {
             break;
@@ -1453,7 +1533,7 @@ static void virtio_net_save_device(VirtIODevice *vdev, QEMUFile *f)
         }
     }
 
-    if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
         qemu_put_be64(f, n->curr_guest_offloads);
     }
 }
@@ -1462,11 +1542,33 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
 {
     VirtIONet *n = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
+    int ret;
 
     if (version_id < 2 || version_id > VIRTIO_NET_VM_VERSION)
         return -EINVAL;
 
-    return virtio_load(vdev, f, version_id);
+    ret = virtio_load(vdev, f, version_id);
+    if (ret) {
+        return ret;
+    }
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
+        n->curr_guest_offloads = qemu_get_be64(f);
+    } else {
+        n->curr_guest_offloads = virtio_net_supported_guest_offloads(n);
+    }
+
+    if (peer_has_vnet_hdr(n)) {
+        virtio_net_apply_guest_offloads(n);
+    }
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
+        virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
+        n->announce_counter = SELF_ANNOUNCE_ROUNDS;
+        timer_mod(n->announce_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL));
+    }
+
+    return 0;
 }
 
 static int virtio_net_load_device(VirtIODevice *vdev, QEMUFile *f,
@@ -1479,7 +1581,8 @@ static int virtio_net_load_device(VirtIODevice *vdev, QEMUFile *f,
     n->vqs[0].tx_waiting = qemu_get_be32(f);
 
     virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f),
-                               virtio_has_feature(vdev, VIRTIO_F_VERSION_1));
+                               virtio_vdev_has_feature(vdev,
+                                                       VIRTIO_F_VERSION_1));
 
     if (version_id >= 3)
         n->status = qemu_get_be16(f);
@@ -1562,16 +1665,6 @@ static int virtio_net_load_device(VirtIODevice *vdev, QEMUFile *f,
         }
     }
 
-    if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
-        n->curr_guest_offloads = qemu_get_be64(f);
-    } else {
-        n->curr_guest_offloads = virtio_net_supported_guest_offloads(n);
-    }
-
-    if (peer_has_vnet_hdr(n)) {
-        virtio_net_apply_guest_offloads(n);
-    }
-
     virtio_net_set_queues(n);
 
     /* Find the first multicast entry in the saved MAC filter */
@@ -1589,12 +1682,6 @@ static int virtio_net_load_device(VirtIODevice *vdev, QEMUFile *f,
         qemu_get_subqueue(n->nic, i)->link_down = link_down;
     }
 
-    if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
-        virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
-        n->announce_counter = SELF_ANNOUNCE_ROUNDS;
-        timer_mod(n->announce_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL));
-    }
-
     return 0;
 }
 
diff --git a/qemu/hw/net/vmware_utils.h b/qemu/hw/net/vmware_utils.h
index 1099df669..c0dbb2ff4 100644
--- a/qemu/hw/net/vmware_utils.h
+++ b/qemu/hw/net/vmware_utils.h
@@ -18,10 +18,7 @@
 #define VMWARE_UTILS_H
 
 #include "qemu/range.h"
-
-#ifndef VMW_SHPRN
-#define VMW_SHPRN(fmt, ...) do {} while (0)
-#endif
+#include "vmxnet_debug.h"
 
 /*
  * Shared memory access functions with byte swap support
@@ -52,7 +49,7 @@ vmw_shmem_rw(hwaddr addr, void *buf, int len, int is_write)
 }
 
 static inline void
-vmw_shmem_set(hwaddr addr, uint8 val, int len)
+vmw_shmem_set(hwaddr addr, uint8_t val, int len)
 {
     int i;
     VMW_SHPRN("SHMEM set: %" PRIx64 ", len: %d (value 0x%X)", addr, len, val);
diff --git a/qemu/hw/net/vmxnet3.c b/qemu/hw/net/vmxnet3.c
index 59b06b841..093a71e12 100644
--- a/qemu/hw/net/vmxnet3.c
+++ b/qemu/hw/net/vmxnet3.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -36,12 +37,28 @@
 #define VMXNET3_MSIX_BAR_SIZE 0x2000
 #define MIN_BUF_SIZE 60
 
+/* Compatability flags for migration */
+#define VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT 0
+#define VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS \
+    (1 << VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT)
+#define VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT 1
+#define VMXNET3_COMPAT_FLAG_DISABLE_PCIE \
+    (1 << VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT)
+
+#define VMXNET3_EXP_EP_OFFSET (0x48)
+#define VMXNET3_MSI_OFFSET(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0x50 : 0x84)
+#define VMXNET3_MSIX_OFFSET(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0 : 0x9c)
+#define VMXNET3_DSN_OFFSET     (0x100)
+
 #define VMXNET3_BAR0_IDX      (0)
 #define VMXNET3_BAR1_IDX      (1)
 #define VMXNET3_MSIX_BAR_IDX  (2)
 
 #define VMXNET3_OFF_MSIX_TABLE (0x000)
-#define VMXNET3_OFF_MSIX_PBA   (0x800)
+#define VMXNET3_OFF_MSIX_PBA(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0x800 : 0x1000)
 
 /* Link speed in Mbps should be shifted by 16 */
 #define VMXNET3_LINK_SPEED      (1000 << 16)
@@ -50,7 +67,7 @@
 #define VMXNET3_LINK_STATUS_UP  0x1
 
 /* Least significant bit should be set for revision and version */
-#define VMXNET3_DEVICE_VERSION    0x1
+#define VMXNET3_UPT_REVISION      0x1
 #define VMXNET3_DEVICE_REVISION   0x1
 
 /* Number of interrupt vectors for non-MSIx modes */
@@ -108,9 +125,19 @@
 
 #define VMXNET_FLAG_IS_SET(field, flag) (((field) & (flag)) == (flag))
 
+typedef struct VMXNET3Class {
+    PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
+} VMXNET3Class;
+
 #define TYPE_VMXNET3 "vmxnet3"
 #define VMXNET3(obj) OBJECT_CHECK(VMXNET3State, (obj), TYPE_VMXNET3)
 
+#define VMXNET3_DEVICE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(VMXNET3Class, (klass), TYPE_VMXNET3)
+#define VMXNET3_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(VMXNET3Class, (obj), TYPE_VMXNET3)
+
 /* Cyclic ring abstraction */
 typedef struct {
     hwaddr pa;
@@ -138,7 +165,7 @@ static inline void vmxnet3_ring_init(Vmxnet3Ring *ring,
 }
 
 #define VMXNET3_RING_DUMP(macro, ring_name, ridx, r)                         \
-    macro("%s#%d: base %" PRIx64 " size %lu cell_size %lu gen %d next %lu",  \
+    macro("%s#%d: base %" PRIx64 " size %zu cell_size %zu gen %d next %zu",  \
           (ring_name), (ridx),                                               \
           (r)->pa, (r)->size, (r)->cell_size, (r)->gen, (r)->next)
 
@@ -313,6 +340,9 @@ typedef struct {
         MACAddr *mcast_list;
         uint32_t mcast_list_len;
         uint32_t mcast_list_buff_size; /* needed for live migration. */
+
+        /* Compatability flags for migration */
+        uint32_t compat_flags;
 } VMXNET3State;
 
 /* Interrupt management */
@@ -493,7 +523,7 @@ vmxnet3_dec_rx_completion_counter(VMXNET3State *s, int qidx)
     vmxnet3_ring_dec(&s->rxq_descr[qidx].comp_ring);
 }
 
-static void vmxnet3_complete_packet(VMXNET3State *s, int qidx, uint32 tx_ridx)
+static void vmxnet3_complete_packet(VMXNET3State *s, int qidx, uint32_t tx_ridx)
 {
     struct Vmxnet3_TxCompDesc txcq_descr;
 
@@ -729,9 +759,7 @@ static void vmxnet3_process_tx_queue(VMXNET3State *s, int qidx)
         }
 
         if (txd.eop) {
-            if (!s->skip_current_tx_pkt) {
-                vmxnet_tx_pkt_parse(s->tx_pkt);
-
+            if (!s->skip_current_tx_pkt && vmxnet_tx_pkt_parse(s->tx_pkt)) {
                 if (s->needs_vlan) {
                     vmxnet_tx_pkt_setup_vlan_header(s->tx_pkt, s->tci);
                 }
@@ -927,9 +955,9 @@ static void vmxnet3_rx_need_csum_calculate(struct VmxnetRxPkt *pkt,
 
     /* Validate packet len: csum_start + scum_offset + length of csum field */
     if (pkt_len < (vhdr->csum_start + vhdr->csum_offset + 2)) {
-        VMW_PKPRN("packet len:%d < csum_start(%d) + csum_offset(%d) + 2, "
+        VMW_PKPRN("packet len:%zu < csum_start(%d) + csum_offset(%d) + 2, "
                   "cannot calculate checksum",
-                  len, vhdr->csum_start, vhdr->csum_offset);
+                  pkt_len, vhdr->csum_start, vhdr->csum_offset);
         return;
     }
 
@@ -1165,9 +1193,13 @@ vmxnet3_io_bar0_write(void *opaque, hwaddr addr,
 static uint64_t
 vmxnet3_io_bar0_read(void *opaque, hwaddr addr, unsigned size)
 {
+    VMXNET3State *s = opaque;
+
     if (VMW_IS_MULTIREG_ADDR(addr, VMXNET3_REG_IMR,
                         VMXNET3_MAX_INTRS, VMXNET3_REG_ALIGN)) {
-        g_assert_not_reached();
+        int l = VMW_MULTIREG_IDX_BY_ADDR(addr, VMXNET3_REG_IMR,
+                                         VMXNET3_REG_ALIGN);
+        return s->interrupt_states[l].is_masked;
     }
 
     VMW_CBPRN("BAR0 unknown read [%" PRIx64 "], size %d", addr, size);
@@ -1192,8 +1224,13 @@ static void vmxnet3_reset_mac(VMXNET3State *s)
 
 static void vmxnet3_deactivate_device(VMXNET3State *s)
 {
-    VMW_CBPRN("Deactivating vmxnet3...");
-    s->device_active = false;
+    if (s->device_active) {
+        VMW_CBPRN("Deactivating vmxnet3...");
+        vmxnet_tx_pkt_reset(s->tx_pkt);
+        vmxnet_tx_pkt_uninit(s->tx_pkt);
+        vmxnet_rx_pkt_uninit(s->rx_pkt);
+        s->device_active = false;
+    }
 }
 
 static void vmxnet3_reset(VMXNET3State *s)
@@ -1202,7 +1239,6 @@ static void vmxnet3_reset(VMXNET3State *s)
 
     vmxnet3_deactivate_device(s);
     vmxnet3_reset_interrupt_states(s);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
     s->drv_shmem = 0;
     s->tx_sop = true;
     s->skip_current_tx_pkt = false;
@@ -1287,6 +1323,10 @@ static uint32_t vmxnet3_get_interrupt_config(VMXNET3State *s)
 static void vmxnet3_fill_stats(VMXNET3State *s)
 {
     int i;
+
+    if (!s->device_active)
+        return;
+
     for (i = 0; i < s->txq_num; i++) {
         cpu_physical_memory_write(s->txq_descr[i].tx_stats_pa,
                                   &s->txq_descr[i].txq_stats,
@@ -1425,6 +1465,12 @@ static void vmxnet3_activate_device(VMXNET3State *s)
         return;
     }
 
+    /* Verify if device is active */
+    if (s->device_active) {
+        VMW_CFPRN("Vmxnet3 device is active");
+        return;
+    }
+
     vmxnet3_adjust_by_guest_type(s);
     vmxnet3_update_features(s);
     vmxnet3_update_pm_state(s);
@@ -1621,7 +1667,7 @@ static void vmxnet3_handle_command(VMXNET3State *s, uint64_t cmd)
         break;
 
     case VMXNET3_CMD_QUIESCE_DEV:
-        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - pause the device");
+        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - deactivate the device");
         vmxnet3_deactivate_device(s);
         break;
 
@@ -1629,6 +1675,23 @@ static void vmxnet3_handle_command(VMXNET3State *s, uint64_t cmd)
         VMW_CBPRN("Set: VMXNET3_CMD_GET_CONF_INTR - interrupt configuration");
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        VMW_CBPRN("Set: VMXNET3_CMD_GET_ADAPTIVE_RING_INFO - "
+                  "adaptive ring info flags");
+        break;
+
+    case VMXNET3_CMD_GET_DID_LO:
+        VMW_CBPRN("Set: Get lower part of device ID");
+        break;
+
+    case VMXNET3_CMD_GET_DID_HI:
+        VMW_CBPRN("Set: Get upper part of device ID");
+        break;
+
+    case VMXNET3_CMD_GET_DEV_EXTRA_INFO:
+        VMW_CBPRN("Set: Get device extra info");
+        break;
+
     default:
         VMW_CBPRN("Received unknown command: %" PRIx64, cmd);
         break;
@@ -1641,13 +1704,14 @@ static uint64_t vmxnet3_get_command_status(VMXNET3State *s)
 
     switch (s->last_command) {
     case VMXNET3_CMD_ACTIVATE_DEV:
-        ret = (s->device_active) ? 0 : -1;
+        ret = (s->device_active) ? 0 : 1;
         VMW_CFPRN("Device active: %" PRIx64, ret);
         break;
 
     case VMXNET3_CMD_RESET_DEV:
     case VMXNET3_CMD_QUIESCE_DEV:
     case VMXNET3_CMD_GET_QUEUE_STATUS:
+    case VMXNET3_CMD_GET_DEV_EXTRA_INFO:
         ret = 0;
         break;
 
@@ -1668,9 +1732,21 @@ static uint64_t vmxnet3_get_command_status(VMXNET3State *s)
         ret = vmxnet3_get_interrupt_config(s);
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        ret = VMXNET3_DISABLE_ADAPTIVE_RING;
+        break;
+
+    case VMXNET3_CMD_GET_DID_LO:
+        ret = PCI_DEVICE_ID_VMWARE_VMXNET3;
+        break;
+
+    case VMXNET3_CMD_GET_DID_HI:
+        ret = VMXNET3_DEVICE_REVISION;
+        break;
+
     default:
         VMW_WRPRN("Received request for unknown command: %x", s->last_command);
-        ret = -1;
+        ret = 0;
         break;
     }
 
@@ -1726,7 +1802,7 @@ vmxnet3_io_bar1_write(void *opaque,
          * shared address only after we get the high part
          */
         if (val == 0) {
-            s->device_active = false;
+            vmxnet3_deactivate_device(s);
         }
         s->temp_shared_guest_driver_memory = val;
         s->drv_shmem = 0;
@@ -1801,7 +1877,7 @@ vmxnet3_io_bar1_read(void *opaque, hwaddr addr, unsigned size)
         /* UPT Version Report Selection */
         case VMXNET3_REG_UVRS:
             VMW_CBPRN("Read BAR1 [VMXNET3_REG_UVRS], size %d", size);
-            ret = VMXNET3_DEVICE_VERSION;
+            ret = VMXNET3_UPT_REVISION;
             break;
 
         /* Command */
@@ -1959,7 +2035,7 @@ vmxnet3_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         vmxnet_rx_pkt_attach_data(s->rx_pkt, buf, size, s->rx_vlan_stripping);
         bytes_indicated = vmxnet3_indicate_packet(s) ? size : -1;
         if (bytes_indicated < size) {
-            VMW_PKPRN("RX: %lu of %lu bytes indicated", bytes_indicated, size);
+            VMW_PKPRN("RX: %zu of %zu bytes indicated", bytes_indicated, size);
         }
     } else {
         VMW_PKPRN("Packet dropped by RX filter");
@@ -1988,7 +2064,6 @@ static void vmxnet3_set_link_status(NetClientState *nc)
 static NetClientInfo net_vmxnet3_info = {
         .type = NET_CLIENT_OPTIONS_KIND_NIC,
         .size = sizeof(NICState),
-        .can_receive = vmxnet3_can_receive,
         .receive = vmxnet3_receive,
         .link_status_changed = vmxnet3_set_link_status,
 };
@@ -2001,16 +2076,13 @@ static bool vmxnet3_peer_has_vnet_hdr(VMXNET3State *s)
         return true;
     }
 
-    VMW_WRPRN("Peer has no virtio extension. Task offloads will be emulated.");
     return false;
 }
 
 static void vmxnet3_net_uninit(VMXNET3State *s)
 {
     g_free(s->mcast_list);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
-    vmxnet_tx_pkt_uninit(s->tx_pkt);
-    vmxnet_rx_pkt_uninit(s->rx_pkt);
+    vmxnet3_deactivate_device(s);
     qemu_del_nic(s->nic);
 }
 
@@ -2030,7 +2102,7 @@ static void vmxnet3_net_init(VMXNET3State *s)
 
     s->link_status_and_speed = VMXNET3_LINK_SPEED | VMXNET3_LINK_STATUS_UP;
 
-    VMW_CFPRN("Permanent MAC: " MAC_FMT, MAC_ARG(s->perm_mac.a));
+    VMW_CFPRN("Permanent MAC: " VMXNET_MF, VMXNET_MA(s->perm_mac.a));
 
     s->nic = qemu_new_nic(&net_vmxnet3_info, &s->conf,
                           object_get_typename(OBJECT(s)),
@@ -2088,8 +2160,8 @@ vmxnet3_init_msix(VMXNET3State *s)
                         &s->msix_bar,
                         VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_TABLE,
                         &s->msix_bar,
-                        VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_PBA,
-                        0);
+                        VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_PBA(s),
+                        VMXNET3_MSIX_OFFSET(s));
 
     if (0 > res) {
         VMW_WRPRN("Failed to initialize MSI-X, error %d", res);
@@ -2117,7 +2189,6 @@ vmxnet3_cleanup_msix(VMXNET3State *s)
     }
 }
 
-#define VMXNET3_MSI_OFFSET        (0x50)
 #define VMXNET3_USE_64BIT         (true)
 #define VMXNET3_PER_VECTOR_MASK   (false)
 
@@ -2127,7 +2198,7 @@ vmxnet3_init_msi(VMXNET3State *s)
     PCIDevice *d = PCI_DEVICE(s);
     int res;
 
-    res = msi_init(d, VMXNET3_MSI_OFFSET, VMXNET3_MAX_NMSIX_INTRS,
+    res = msi_init(d, VMXNET3_MSI_OFFSET(s), VMXNET3_MAX_NMSIX_INTRS,
                    VMXNET3_USE_64BIT, VMXNET3_PER_VECTOR_MASK);
     if (0 > res) {
         VMW_WRPRN("Failed to initialize MSI, error %d", res);
@@ -2184,6 +2255,22 @@ static const MemoryRegionOps b1_ops = {
     },
 };
 
+static uint8_t *vmxnet3_device_serial_num(VMXNET3State *s)
+{
+    static uint64_t dsn_payload;
+    uint8_t *dsnp = (uint8_t *)&dsn_payload;
+
+    dsnp[0] = 0xfe;
+    dsnp[1] = s->conf.macaddr.a[3];
+    dsnp[2] = s->conf.macaddr.a[4];
+    dsnp[3] = s->conf.macaddr.a[5];
+    dsnp[4] = s->conf.macaddr.a[0];
+    dsnp[5] = s->conf.macaddr.a[1];
+    dsnp[6] = s->conf.macaddr.a[2];
+    dsnp[7] = 0xff;
+    return dsnp;
+}
+
 static void vmxnet3_pci_realize(PCIDevice *pci_dev, Error **errp)
 {
     DeviceState *dev = DEVICE(pci_dev);
@@ -2221,6 +2308,17 @@ static void vmxnet3_pci_realize(PCIDevice *pci_dev, Error **errp)
 
     vmxnet3_net_init(s);
 
+    if (pci_is_express(pci_dev)) {
+        if (pci_bus_is_express(pci_dev->bus)) {
+            pcie_endpoint_cap_init(pci_dev, VMXNET3_EXP_EP_OFFSET);
+        }
+
+        pcie_add_capability(pci_dev, PCI_EXT_CAP_ID_DSN, 0x1,
+                            VMXNET3_DSN_OFFSET, PCI_EXT_CAP_DSN_SIZEOF);
+        memcpy(pci_dev->config + VMXNET3_DSN_OFFSET + 4,
+               vmxnet3_device_serial_num(s), sizeof(uint64_t));
+    }
+
     register_savevm(dev, "vmxnet3-msix", -1, 1,
                     vmxnet3_msix_save, vmxnet3_msix_load, s);
 }
@@ -2490,6 +2588,29 @@ static const VMStateInfo int_state_info = {
     .put = vmxnet3_put_int_state
 };
 
+static bool vmxnet3_vmstate_need_pcie_device(void *opaque)
+{
+    VMXNET3State *s = VMXNET3(opaque);
+
+    return !(s->compat_flags & VMXNET3_COMPAT_FLAG_DISABLE_PCIE);
+}
+
+static bool vmxnet3_vmstate_test_pci_device(void *opaque, int version_id)
+{
+    return !vmxnet3_vmstate_need_pcie_device(opaque);
+}
+
+static const VMStateDescription vmstate_vmxnet3_pcie_device = {
+    .name = "vmxnet3/pcie",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vmxnet3_vmstate_need_pcie_device,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, VMXNET3State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_vmxnet3 = {
     .name = "vmxnet3",
     .version_id = 1,
@@ -2497,7 +2618,9 @@ static const VMStateDescription vmstate_vmxnet3 = {
     .pre_save = vmxnet3_pre_save,
     .post_load = vmxnet3_post_load,
     .fields = (VMStateField[]) {
-            VMSTATE_PCI_DEVICE(parent_obj, VMXNET3State),
+            VMSTATE_STRUCT_TEST(parent_obj, VMXNET3State,
+                                vmxnet3_vmstate_test_pci_device, 0,
+                                vmstate_pci_device, PCIDevice),
             VMSTATE_BOOL(rx_packets_compound, VMXNET3State),
             VMSTATE_BOOL(rx_vlan_stripping, VMXNET3State),
             VMSTATE_BOOL(lro_supported, VMXNET3State),
@@ -2532,19 +2655,38 @@ static const VMStateDescription vmstate_vmxnet3 = {
     },
     .subsections = (const VMStateDescription*[]) {
         &vmxstate_vmxnet3_mcast_list,
+        &vmstate_vmxnet3_pcie_device,
         NULL
     }
 };
 
 static Property vmxnet3_properties[] = {
     DEFINE_NIC_PROPERTIES(VMXNET3State, conf),
+    DEFINE_PROP_BIT("x-old-msi-offsets", VMXNET3State, compat_flags,
+                    VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", VMXNET3State, compat_flags,
+                    VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void vmxnet3_realize(DeviceState *qdev, Error **errp)
+{
+    VMXNET3Class *vc = VMXNET3_DEVICE_GET_CLASS(qdev);
+    PCIDevice *pci_dev = PCI_DEVICE(qdev);
+    VMXNET3State *s = VMXNET3(qdev);
+
+    if (!(s->compat_flags & VMXNET3_COMPAT_FLAG_DISABLE_PCIE)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    vc->parent_dc_realize(qdev, errp);
+}
+
 static void vmxnet3_class_init(ObjectClass *class, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(class);
     PCIDeviceClass *c = PCI_DEVICE_CLASS(class);
+    VMXNET3Class *vc = VMXNET3_DEVICE_CLASS(class);
 
     c->realize = vmxnet3_pci_realize;
     c->exit = vmxnet3_pci_uninit;
@@ -2554,6 +2696,8 @@ static void vmxnet3_class_init(ObjectClass *class, void *data)
     c->class_id = PCI_CLASS_NETWORK_ETHERNET;
     c->subsystem_vendor_id = PCI_VENDOR_ID_VMWARE;
     c->subsystem_id = PCI_DEVICE_ID_VMWARE_VMXNET3;
+    vc->parent_dc_realize = dc->realize;
+    dc->realize = vmxnet3_realize;
     dc->desc = "VMWare Paravirtualized Ethernet v3";
     dc->reset = vmxnet3_qdev_reset;
     dc->vmsd = &vmstate_vmxnet3;
@@ -2564,6 +2708,7 @@ static void vmxnet3_class_init(ObjectClass *class, void *data)
 static const TypeInfo vmxnet3_info = {
     .name          = TYPE_VMXNET3,
     .parent        = TYPE_PCI_DEVICE,
+    .class_size    = sizeof(VMXNET3Class),
     .instance_size = sizeof(VMXNET3State),
     .class_init    = vmxnet3_class_init,
     .instance_init = vmxnet3_instance_init,
diff --git a/qemu/hw/net/vmxnet3.h b/qemu/hw/net/vmxnet3.h
index f987d7126..f7006afe9 100644
--- a/qemu/hw/net/vmxnet3.h
+++ b/qemu/hw/net/vmxnet3.h
@@ -198,9 +198,13 @@ enum {
     VMXNET3_CMD_GET_DID_LO,                               /* 0xF00D0005 */
     VMXNET3_CMD_GET_DID_HI,                               /* 0xF00D0006 */
     VMXNET3_CMD_GET_DEV_EXTRA_INFO,                       /* 0xF00D0007 */
-    VMXNET3_CMD_GET_CONF_INTR                             /* 0xF00D0008 */
+    VMXNET3_CMD_GET_CONF_INTR,                            /* 0xF00D0008 */
+    VMXNET3_CMD_GET_ADAPTIVE_RING_INFO                    /* 0xF00D0009 */
 };
 
+/* Adaptive Ring Info Flags */
+#define VMXNET3_DISABLE_ADAPTIVE_RING 1
+
 /*
  *    Little Endian layout of bitfields -
  *    Byte 0 :    7.....len.....0
diff --git a/qemu/hw/net/vmxnet_debug.h b/qemu/hw/net/vmxnet_debug.h
index 96dae0f91..96495dbb1 100644
--- a/qemu/hw/net/vmxnet_debug.h
+++ b/qemu/hw/net/vmxnet_debug.h
@@ -20,94 +20,127 @@
 
 #define VMXNET_DEVICE_NAME "vmxnet3"
 
-/* #define VMXNET_DEBUG_CB */
 #define VMXNET_DEBUG_WARNINGS
 #define VMXNET_DEBUG_ERRORS
-/* #define VMXNET_DEBUG_INTERRUPTS */
-/* #define VMXNET_DEBUG_CONFIG */
-/* #define VMXNET_DEBUG_RINGS */
-/* #define VMXNET_DEBUG_PACKETS */
-/* #define VMXNET_DEBUG_SHMEM_ACCESS */
+
+#undef VMXNET_DEBUG_CB
+#undef VMXNET_DEBUG_INTERRUPTS
+#undef VMXNET_DEBUG_CONFIG
+#undef VMXNET_DEBUG_RINGS
+#undef VMXNET_DEBUG_PACKETS
+#undef VMXNET_DEBUG_SHMEM_ACCESS
+
+#ifdef VMXNET_DEBUG_CB
+#  define VMXNET_DEBUG_CB_ENABLED 1
+#else
+#  define VMXNET_DEBUG_CB_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_WARNINGS
+#  define VMXNET_DEBUG_WARNINGS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_WARNINGS_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_ERRORS
+#  define VMXNET_DEBUG_ERRORS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_ERRORS_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_CONFIG
+#  define VMXNET_DEBUG_CONFIG_ENABLED 1
+#else
+#  define VMXNET_DEBUG_CONFIG_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_RINGS
+#  define VMXNET_DEBUG_RINGS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_RINGS_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_PACKETS
+#  define VMXNET_DEBUG_PACKETS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_PACKETS_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_INTERRUPTS
+#  define VMXNET_DEBUG_INTERRUPTS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_INTERRUPTS_ENABLED 0
+#endif
 
 #ifdef VMXNET_DEBUG_SHMEM_ACCESS
+#  define VMXNET_DEBUG_SHMEM_ACCESS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_SHMEM_ACCESS_ENABLED 0
+#endif
+
 #define VMW_SHPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][SH][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_SHMEM_ACCESS_ENABLED) {                              \
+            printf("[%s][SH][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+       }                                                                      \
     } while (0)
-#else
-#define VMW_SHPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_CB
 #define VMW_CBPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][CB][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_CB_ENABLED) {                                        \
+            printf("[%s][CB][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_CBPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_PACKETS
 #define VMW_PKPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][PK][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_PACKETS_ENABLED) {                                   \
+            printf("[%s][PK][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_PKPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_WARNINGS
 #define VMW_WRPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][WR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_WARNINGS_ENABLED) {                                  \
+            printf("[%s][WR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_WRPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_ERRORS
 #define VMW_ERPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][ER][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_ERRORS_ENABLED) {                                    \
+            printf("[%s][ER][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_ERPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_INTERRUPTS
 #define VMW_IRPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][IR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_INTERRUPTS_ENABLED) {                                \
+            printf("[%s][IR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_IRPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_CONFIG
 #define VMW_CFPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][CF][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_CONFIG_ENABLED) {                                    \
+            printf("[%s][CF][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_CFPRN(fmt, ...) do {} while (0)
-#endif
 
-#ifdef VMXNET_DEBUG_RINGS
 #define VMW_RIPRN(fmt, ...)                                                   \
     do {                                                                      \
-        printf("[%s][RI][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
+        if (VMXNET_DEBUG_RINGS_ENABLED) {                                     \
+            printf("[%s][RI][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
     } while (0)
-#else
-#define VMW_RIPRN(fmt, ...) do {} while (0)
-#endif
 
 #define VMXNET_MF       "%02X:%02X:%02X:%02X:%02X:%02X"
 #define VMXNET_MA(a)    (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5]
diff --git a/qemu/hw/net/vmxnet_rx_pkt.c b/qemu/hw/net/vmxnet_rx_pkt.c
index aa5462931..21bb46e68 100644
--- a/qemu/hw/net/vmxnet_rx_pkt.c
+++ b/qemu/hw/net/vmxnet_rx_pkt.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "vmxnet_rx_pkt.h"
 #include "net/eth.h"
 #include "qemu-common.h"
diff --git a/qemu/hw/net/vmxnet_rx_pkt.h b/qemu/hw/net/vmxnet_rx_pkt.h
index a425846b5..0a45c1ba0 100644
--- a/qemu/hw/net/vmxnet_rx_pkt.h
+++ b/qemu/hw/net/vmxnet_rx_pkt.h
@@ -18,8 +18,6 @@
 #ifndef VMXNET_RX_PKT_H
 #define VMXNET_RX_PKT_H
 
-#include "stdint.h"
-#include "stdbool.h"
 #include "net/eth.h"
 
 /* defines to enable packet dump functions */
diff --git a/qemu/hw/net/vmxnet_tx_pkt.c b/qemu/hw/net/vmxnet_tx_pkt.c
index f7344c4cb..91e1e08fd 100644
--- a/qemu/hw/net/vmxnet_tx_pkt.c
+++ b/qemu/hw/net/vmxnet_tx_pkt.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "vmxnet_tx_pkt.h"
 #include "net/eth.h"
@@ -142,11 +143,24 @@ static bool vmxnet_tx_pkt_parse_headers(struct VmxnetTxPkt *pkt)
 
     bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
                             ETH_MAX_L2_HDR_LEN);
-    if (bytes_read < ETH_MAX_L2_HDR_LEN) {
+    if (bytes_read < sizeof(struct eth_header)) {
+        l2_hdr->iov_len = 0;
+        return false;
+    }
+
+    l2_hdr->iov_len = sizeof(struct eth_header);
+    switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
+    case ETH_P_VLAN:
+        l2_hdr->iov_len += sizeof(struct vlan_header);
+        break;
+    case ETH_P_DVLAN:
+        l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
+        break;
+    }
+
+    if (bytes_read < l2_hdr->iov_len) {
         l2_hdr->iov_len = 0;
         return false;
-    } else {
-        l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
     }
 
     l3_proto = eth_get_l3_proto(l2_hdr->iov_base, l2_hdr->iov_len);
diff --git a/qemu/hw/net/vmxnet_tx_pkt.h b/qemu/hw/net/vmxnet_tx_pkt.h
index 57121a6fe..f51e98ad9 100644
--- a/qemu/hw/net/vmxnet_tx_pkt.h
+++ b/qemu/hw/net/vmxnet_tx_pkt.h
@@ -18,8 +18,6 @@
 #ifndef VMXNET_TX_PKT_H
 #define VMXNET_TX_PKT_H
 
-#include "stdint.h"
-#include "stdbool.h"
 #include "net/eth.h"
 #include "exec/hwaddr.h"
 
diff --git a/qemu/hw/net/xen_nic.c b/qemu/hw/net/xen_nic.c
index d7cbfc103..7281730d9 100644
--- a/qemu/hw/net/xen_nic.c
+++ b/qemu/hw/net/xen_nic.c
@@ -19,19 +19,9 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <inttypes.h>
-#include <fcntl.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/socket.h>
 #include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/mman.h>
 #include <sys/wait.h>
 
@@ -169,7 +159,7 @@ static void net_tx_packets(struct XenNetDev *netdev)
                           (txreq.flags & NETTXF_more_data)      ? " more_data"      : "",
                           (txreq.flags & NETTXF_extra_info)     ? " extra_info"     : "");
 
-            page = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+            page = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                            netdev->xendev.dom,
                                            txreq.gref, PROT_READ);
             if (page == NULL) {
@@ -191,7 +181,7 @@ static void net_tx_packets(struct XenNetDev *netdev)
                 qemu_send_packet(qemu_get_queue(netdev->nic),
                                  page + txreq.offset, txreq.size);
             }
-            xc_gnttab_munmap(netdev->xendev.gnttabdev, page, 1);
+            xengnttab_unmap(netdev->xendev.gnttabdev, page, 1);
             net_tx_response(netdev, &txreq, NETIF_RSP_OKAY);
         }
         if (!netdev->tx_work) {
@@ -261,7 +251,7 @@ static ssize_t net_rx_packet(NetClientState *nc, const uint8_t *buf, size_t size
     memcpy(&rxreq, RING_GET_REQUEST(&netdev->rx_ring, rc), sizeof(rxreq));
     netdev->rx_ring.req_cons = ++rc;
 
-    page = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    page = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                    netdev->xendev.dom,
                                    rxreq.gref, PROT_WRITE);
     if (page == NULL) {
@@ -271,7 +261,7 @@ static ssize_t net_rx_packet(NetClientState *nc, const uint8_t *buf, size_t size
         return -1;
     }
     memcpy(page + NET_IP_ALIGN, buf, size);
-    xc_gnttab_munmap(netdev->xendev.gnttabdev, page, 1);
+    xengnttab_unmap(netdev->xendev.gnttabdev, page, 1);
     net_rx_response(netdev, &rxreq, NETIF_RSP_OKAY, NET_IP_ALIGN, size, 0);
 
     return size;
@@ -343,19 +333,19 @@ static int net_connect(struct XenDevice *xendev)
         return -1;
     }
 
-    netdev->txs = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    netdev->txs = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                           netdev->xendev.dom,
                                           netdev->tx_ring_ref,
                                           PROT_READ | PROT_WRITE);
     if (!netdev->txs) {
         return -1;
     }
-    netdev->rxs = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    netdev->rxs = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                           netdev->xendev.dom,
                                           netdev->rx_ring_ref,
                                           PROT_READ | PROT_WRITE);
     if (!netdev->rxs) {
-        xc_gnttab_munmap(netdev->xendev.gnttabdev, netdev->txs, 1);
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->txs, 1);
         netdev->txs = NULL;
         return -1;
     }
@@ -380,11 +370,11 @@ static void net_disconnect(struct XenDevice *xendev)
     xen_be_unbind_evtchn(&netdev->xendev);
 
     if (netdev->txs) {
-        xc_gnttab_munmap(netdev->xendev.gnttabdev, netdev->txs, 1);
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->txs, 1);
         netdev->txs = NULL;
     }
     if (netdev->rxs) {
-        xc_gnttab_munmap(netdev->xendev.gnttabdev, netdev->rxs, 1);
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->rxs, 1);
         netdev->rxs = NULL;
     }
 }
diff --git a/qemu/hw/net/xgmac.c b/qemu/hw/net/xgmac.c
index 15fb68194..0c5f793bd 100644
--- a/qemu/hw/net/xgmac.c
+++ b/qemu/hw/net/xgmac.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/log.h"
diff --git a/qemu/hw/net/xilinx_axienet.c b/qemu/hw/net/xilinx_axienet.c
index d63c42324..de23ab5dc 100644
--- a/qemu/hw/net/xilinx_axienet.c
+++ b/qemu/hw/net/xilinx_axienet.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 #include "net/net.h"
 #include "net/checksum.h"
diff --git a/qemu/hw/net/xilinx_ethlite.c b/qemu/hw/net/xilinx_ethlite.c
index ad6b55306..bc846e709 100644
--- a/qemu/hw/net/xilinx_ethlite.c
+++ b/qemu/hw/net/xilinx_ethlite.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h" /* FIXME should not use tswap* */
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
diff --git a/qemu/hw/nvram/ds1225y.c b/qemu/hw/nvram/ds1225y.c
index 332598b25..57d5ab215 100644
--- a/qemu/hw/nvram/ds1225y.c
+++ b/qemu/hw/nvram/ds1225y.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
diff --git a/qemu/hw/nvram/eeprom93xx.c b/qemu/hw/nvram/eeprom93xx.c
index 0af4d6707..2c16fc23d 100644
--- a/qemu/hw/nvram/eeprom93xx.c
+++ b/qemu/hw/nvram/eeprom93xx.c
@@ -35,6 +35,7 @@
  * - No emulation of EEPROM timings.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/nvram/eeprom93xx.h"
 
diff --git a/qemu/hw/nvram/fw_cfg.c b/qemu/hw/nvram/fw_cfg.c
index 88481b78c..999f48028 100644
--- a/qemu/hw/nvram/fw_cfg.c
+++ b/qemu/hw/nvram/fw_cfg.c
@@ -21,16 +21,19 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
 #include "hw/isa/isa.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/sysbus.h"
+#include "hw/boards.h"
 #include "trace.h"
 #include "qemu/error-report.h"
 #include "qemu/config-file.h"
+#include "qemu/cutils.h"
 
-#define FW_CFG_SIZE 2
 #define FW_CFG_NAME "fw_cfg"
 #define FW_CFG_PATH "/machine/" FW_CFG_NAME
 
@@ -42,6 +45,18 @@
 #define FW_CFG_IO(obj)  OBJECT_CHECK(FWCfgIoState,  (obj), TYPE_FW_CFG_IO)
 #define FW_CFG_MEM(obj) OBJECT_CHECK(FWCfgMemState, (obj), TYPE_FW_CFG_MEM)
 
+/* FW_CFG_VERSION bits */
+#define FW_CFG_VERSION      0x01
+#define FW_CFG_VERSION_DMA  0x02
+
+/* FW_CFG_DMA_CONTROL bits */
+#define FW_CFG_DMA_CTL_ERROR   0x01
+#define FW_CFG_DMA_CTL_READ    0x02
+#define FW_CFG_DMA_CTL_SKIP    0x04
+#define FW_CFG_DMA_CTL_SELECT  0x08
+
+#define FW_CFG_DMA_SIGNATURE 0x51454d5520434647ULL /* "QEMU CFG" */
+
 typedef struct FWCfgEntry {
     uint32_t len;
     uint8_t *data;
@@ -55,10 +70,18 @@ struct FWCfgState {
     /*< public >*/
 
     FWCfgEntry entries[2][FW_CFG_MAX_ENTRY];
+    int entry_order[FW_CFG_MAX_ENTRY];
     FWCfgFiles *files;
     uint16_t cur_entry;
     uint32_t cur_offset;
     Notifier machine_ready;
+
+    int fw_cfg_order_override;
+
+    bool dma_enabled;
+    dma_addr_t dma_addr;
+    AddressSpace *dma_as;
+    MemoryRegion dma_iomem;
 };
 
 struct FWCfgIoState {
@@ -67,7 +90,7 @@ struct FWCfgIoState {
     /*< public >*/
 
     MemoryRegion comb_iomem;
-    uint32_t iobase;
+    uint32_t iobase, dma_iobase;
 };
 
 struct FWCfgMemState {
@@ -187,9 +210,7 @@ static void fw_cfg_bootsplash(FWCfgState *s)
             g_free(filename);
             return;
         }
-        if (boot_splash_filedata != NULL) {
-            g_free(boot_splash_filedata);
-        }
+        g_free(boot_splash_filedata);
         boot_splash_filedata = (uint8_t *)file_data;
         boot_splash_filedata_size = file_size;
 
@@ -236,7 +257,8 @@ static void fw_cfg_write(FWCfgState *s, uint8_t value)
 
 static int fw_cfg_select(FWCfgState *s, uint16_t key)
 {
-    int ret;
+    int arch, ret;
+    FWCfgEntry *e;
 
     s->cur_offset = 0;
     if ((key & FW_CFG_ENTRY_MASK) >= FW_CFG_MAX_ENTRY) {
@@ -245,41 +267,45 @@ static int fw_cfg_select(FWCfgState *s, uint16_t key)
     } else {
         s->cur_entry = key;
         ret = 1;
-    }
-
-    trace_fw_cfg_select(s, key, ret);
-    return ret;
-}
-
-static uint8_t fw_cfg_read(FWCfgState *s)
-{
-    int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
-    FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
-    uint8_t ret;
-
-    if (s->cur_entry == FW_CFG_INVALID || !e->data || s->cur_offset >= e->len)
-        ret = 0;
-    else {
+        /* entry successfully selected, now run callback if present */
+        arch = !!(key & FW_CFG_ARCH_LOCAL);
+        e = &s->entries[arch][key & FW_CFG_ENTRY_MASK];
         if (e->read_callback) {
-            e->read_callback(e->callback_opaque, s->cur_offset);
+            e->read_callback(e->callback_opaque);
         }
-        ret = e->data[s->cur_offset++];
     }
 
-    trace_fw_cfg_read(s, ret);
+    trace_fw_cfg_select(s, key, ret);
     return ret;
 }
 
-static uint64_t fw_cfg_data_mem_read(void *opaque, hwaddr addr,
-                                     unsigned size)
+static uint64_t fw_cfg_data_read(void *opaque, hwaddr addr, unsigned size)
 {
     FWCfgState *s = opaque;
+    int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
+    FWCfgEntry *e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+                    &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
     uint64_t value = 0;
-    unsigned i;
 
-    for (i = 0; i < size; ++i) {
-        value = (value << 8) | fw_cfg_read(s);
+    assert(size > 0 && size <= sizeof(value));
+    if (s->cur_entry != FW_CFG_INVALID && e->data && s->cur_offset < e->len) {
+        /* The least significant 'size' bytes of the return value are
+         * expected to contain a string preserving portion of the item
+         * data, padded with zeros on the right in case we run out early.
+         * In technical terms, we're composing the host-endian representation
+         * of the big endian interpretation of the fw_cfg string.
+         */
+        do {
+            value = (value << 8) | e->data[s->cur_offset++];
+        } while (--size && s->cur_offset < e->len);
+        /* If size is still not zero, we *did* run out early, so continue
+         * left-shifting, to add the appropriate number of padding zeros
+         * on the right.
+         */
+        value <<= 8 * size;
     }
+
+    trace_fw_cfg_read(s, value);
     return value;
 }
 
@@ -294,6 +320,126 @@ static void fw_cfg_data_mem_write(void *opaque, hwaddr addr,
     } while (i);
 }
 
+static void fw_cfg_dma_transfer(FWCfgState *s)
+{
+    dma_addr_t len;
+    FWCfgDmaAccess dma;
+    int arch;
+    FWCfgEntry *e;
+    int read;
+    dma_addr_t dma_addr;
+
+    /* Reset the address before the next access */
+    dma_addr = s->dma_addr;
+    s->dma_addr = 0;
+
+    if (dma_memory_read(s->dma_as, dma_addr, &dma, sizeof(dma))) {
+        stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                   FW_CFG_DMA_CTL_ERROR);
+        return;
+    }
+
+    dma.address = be64_to_cpu(dma.address);
+    dma.length = be32_to_cpu(dma.length);
+    dma.control = be32_to_cpu(dma.control);
+
+    if (dma.control & FW_CFG_DMA_CTL_SELECT) {
+        fw_cfg_select(s, dma.control >> 16);
+    }
+
+    arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
+    e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+        &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
+
+    if (dma.control & FW_CFG_DMA_CTL_READ) {
+        read = 1;
+    } else if (dma.control & FW_CFG_DMA_CTL_SKIP) {
+        read = 0;
+    } else {
+        dma.length = 0;
+    }
+
+    dma.control = 0;
+
+    while (dma.length > 0 && !(dma.control & FW_CFG_DMA_CTL_ERROR)) {
+        if (s->cur_entry == FW_CFG_INVALID || !e->data ||
+                                s->cur_offset >= e->len) {
+            len = dma.length;
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_set(s->dma_as, dma.address, 0, len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+        } else {
+            if (dma.length <= (e->len - s->cur_offset)) {
+                len = dma.length;
+            } else {
+                len = (e->len - s->cur_offset);
+            }
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_write(s->dma_as, dma.address,
+                                    &e->data[s->cur_offset], len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+            s->cur_offset += len;
+        }
+
+        dma.address += len;
+        dma.length  -= len;
+
+    }
+
+    stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                dma.control);
+
+    trace_fw_cfg_read(s, 0);
+}
+
+static uint64_t fw_cfg_dma_mem_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    /* Return a signature value (and handle various read sizes) */
+    return extract64(FW_CFG_DMA_SIGNATURE, (8 - addr - size) * 8, size * 8);
+}
+
+static void fw_cfg_dma_mem_write(void *opaque, hwaddr addr,
+                                 uint64_t value, unsigned size)
+{
+    FWCfgState *s = opaque;
+
+    if (size == 4) {
+        if (addr == 0) {
+            /* FWCfgDmaAccess high address */
+            s->dma_addr = value << 32;
+        } else if (addr == 4) {
+            /* FWCfgDmaAccess low address */
+            s->dma_addr |= value;
+            fw_cfg_dma_transfer(s);
+        }
+    } else if (size == 8 && addr == 0) {
+        s->dma_addr = value;
+        fw_cfg_dma_transfer(s);
+    }
+}
+
+static bool fw_cfg_dma_mem_valid(void *opaque, hwaddr addr,
+                                  unsigned size, bool is_write)
+{
+    return !is_write || ((size == 4 && (addr == 0 || addr == 4)) ||
+                         (size == 8 && addr == 0));
+}
+
 static bool fw_cfg_data_mem_valid(void *opaque, hwaddr addr,
                                   unsigned size, bool is_write)
 {
@@ -312,12 +458,6 @@ static bool fw_cfg_ctl_mem_valid(void *opaque, hwaddr addr,
     return is_write && size == 2;
 }
 
-static uint64_t fw_cfg_comb_read(void *opaque, hwaddr addr,
-                                 unsigned size)
-{
-    return fw_cfg_read(opaque);
-}
-
 static void fw_cfg_comb_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
@@ -344,7 +484,7 @@ static const MemoryRegionOps fw_cfg_ctl_mem_ops = {
 };
 
 static const MemoryRegionOps fw_cfg_data_mem_ops = {
-    .read = fw_cfg_data_mem_read,
+    .read = fw_cfg_data_read,
     .write = fw_cfg_data_mem_write,
     .endianness = DEVICE_BIG_ENDIAN,
     .valid = {
@@ -355,17 +495,27 @@ static const MemoryRegionOps fw_cfg_data_mem_ops = {
 };
 
 static const MemoryRegionOps fw_cfg_comb_mem_ops = {
-    .read = fw_cfg_comb_read,
+    .read = fw_cfg_data_read,
     .write = fw_cfg_comb_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid.accepts = fw_cfg_comb_valid,
 };
 
+static const MemoryRegionOps fw_cfg_dma_mem_ops = {
+    .read = fw_cfg_dma_mem_read,
+    .write = fw_cfg_dma_mem_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .valid.accepts = fw_cfg_dma_mem_valid,
+    .valid.max_access_size = 8,
+    .impl.max_access_size = 8,
+};
+
 static void fw_cfg_reset(DeviceState *d)
 {
     FWCfgState *s = FW_CFG(d);
 
-    fw_cfg_select(s, 0);
+    /* we never register a read callback for FW_CFG_SIGNATURE */
+    fw_cfg_select(s, FW_CFG_SIGNATURE);
 }
 
 /* Save restore 32 bit int as uint16_t
@@ -401,6 +551,22 @@ static bool is_version_1(void *opaque, int version_id)
     return version_id == 1;
 }
 
+static bool fw_cfg_dma_enabled(void *opaque)
+{
+    FWCfgState *s = opaque;
+
+    return s->dma_enabled;
+}
+
+static const VMStateDescription vmstate_fw_cfg_dma = {
+    .name = "fw_cfg/dma",
+    .needed = fw_cfg_dma_enabled,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(dma_addr, FWCfgState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_fw_cfg = {
     .name = "fw_cfg",
     .version_id = 2,
@@ -410,6 +576,10 @@ static const VMStateDescription vmstate_fw_cfg = {
         VMSTATE_UINT16_HACK(cur_offset, FWCfgState, is_version_1),
         VMSTATE_UINT32_V(cur_offset, FWCfgState, 2),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fw_cfg_dma,
+        NULL,
     }
 };
 
@@ -499,12 +669,87 @@ void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value)
     fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
+void fw_cfg_set_order_override(FWCfgState *s, int order)
+{
+    assert(s->fw_cfg_order_override == 0);
+    s->fw_cfg_order_override = order;
+}
+
+void fw_cfg_reset_order_override(FWCfgState *s)
+{
+    assert(s->fw_cfg_order_override != 0);
+    s->fw_cfg_order_override = 0;
+}
+
+/*
+ * This is the legacy order list.  For legacy systems, files are in
+ * the fw_cfg in the order defined below, by the "order" value.  Note
+ * that some entries (VGA ROMs, NIC option ROMS, etc.) go into a
+ * specific area, but there may be more than one and they occur in the
+ * order that the user specifies them on the command line.  Those are
+ * handled in a special manner, using the order override above.
+ *
+ * For non-legacy, the files are sorted by filename to avoid this kind
+ * of complexity in the future.
+ *
+ * This is only for x86, other arches don't implement versioning so
+ * they won't set legacy mode.
+ */
+static struct {
+    const char *name;
+    int order;
+} fw_cfg_order[] = {
+    { "etc/boot-menu-wait", 10 },
+    { "bootsplash.jpg", 11 },
+    { "bootsplash.bmp", 12 },
+    { "etc/boot-fail-wait", 15 },
+    { "etc/smbios/smbios-tables", 20 },
+    { "etc/smbios/smbios-anchor", 30 },
+    { "etc/e820", 40 },
+    { "etc/reserved-memory-end", 50 },
+    { "genroms/kvmvapic.bin", 55 },
+    { "genroms/linuxboot.bin", 60 },
+    { }, /* VGA ROMs from pc_vga_init come here, 70. */
+    { }, /* NIC option ROMs from pc_nic_init come here, 80. */
+    { "etc/system-states", 90 },
+    { }, /* User ROMs come here, 100. */
+    { }, /* Device FW comes here, 110. */
+    { "etc/extra-pci-roots", 120 },
+    { "etc/acpi/tables", 130 },
+    { "etc/table-loader", 140 },
+    { "etc/tpm/log", 150 },
+    { "etc/acpi/rsdp", 160 },
+    { "bootorder", 170 },
+
+#define FW_CFG_ORDER_OVERRIDE_LAST 200
+};
+
+static int get_fw_cfg_order(FWCfgState *s, const char *name)
+{
+    int i;
+
+    if (s->fw_cfg_order_override > 0)
+	return s->fw_cfg_order_override;
+
+    for (i = 0; i < ARRAY_SIZE(fw_cfg_order); i++) {
+	if (fw_cfg_order[i].name == NULL)
+	    continue;
+	if (strcmp(name, fw_cfg_order[i].name) == 0)
+	    return fw_cfg_order[i].order;
+    }
+    /* Stick unknown stuff at the end. */
+    error_report("warning: Unknown firmware file in legacy mode: %s\n", name);
+    return FW_CFG_ORDER_OVERRIDE_LAST;
+}
+
 void fw_cfg_add_file_callback(FWCfgState *s,  const char *filename,
                               FWCfgReadCallback callback, void *callback_opaque,
                               void *data, size_t len)
 {
-    int i, index;
+    int i, index, count;
     size_t dsize;
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+    int order = 0;
 
     if (!s->files) {
         dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
@@ -512,13 +757,48 @@ void fw_cfg_add_file_callback(FWCfgState *s,  const char *filename,
         fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, s->files, dsize);
     }
 
-    index = be32_to_cpu(s->files->count);
-    assert(index < FW_CFG_FILE_SLOTS);
+    count = be32_to_cpu(s->files->count);
+    assert(count < FW_CFG_FILE_SLOTS);
+
+    /* Find the insertion point. */
+    if (mc->legacy_fw_cfg_order) {
+        /*
+         * Sort by order. For files with the same order, we keep them
+         * in the sequence in which they were added.
+         */
+        order = get_fw_cfg_order(s, filename);
+        for (index = count;
+             index > 0 && order < s->entry_order[index - 1];
+             index--);
+    } else {
+        /* Sort by file name. */
+        for (index = count;
+             index > 0 && strcmp(filename, s->files->f[index - 1].name) < 0;
+             index--);
+    }
 
-    pstrcpy(s->files->f[index].name, sizeof(s->files->f[index].name),
-            filename);
-    for (i = 0; i < index; i++) {
-        if (strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
+    /*
+     * Move all the entries from the index point and after down one
+     * to create a slot for the new entry.  Because calculations are
+     * being done with the index, make it so that "i" is the current
+     * index and "i - 1" is the one being copied from, thus the
+     * unusual start and end in the for statement.
+     */
+    for (i = count + 1; i > index; i--) {
+        s->files->f[i] = s->files->f[i - 1];
+        s->files->f[i].select = cpu_to_be16(FW_CFG_FILE_FIRST + i);
+        s->entries[0][FW_CFG_FILE_FIRST + i] =
+            s->entries[0][FW_CFG_FILE_FIRST + i - 1];
+        s->entry_order[i] = s->entry_order[i - 1];
+    }
+
+    memset(&s->files->f[index], 0, sizeof(FWCfgFile));
+    memset(&s->entries[0][FW_CFG_FILE_FIRST + index], 0, sizeof(FWCfgEntry));
+
+    pstrcpy(s->files->f[index].name, sizeof(s->files->f[index].name), filename);
+    for (i = 0; i <= count; i++) {
+        if (i != index &&
+            strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
             error_report("duplicate fw_cfg file name: %s",
                          s->files->f[index].name);
             exit(1);
@@ -530,9 +810,10 @@ void fw_cfg_add_file_callback(FWCfgState *s,  const char *filename,
 
     s->files->f[index].size   = cpu_to_be32(len);
     s->files->f[index].select = cpu_to_be16(FW_CFG_FILE_FIRST + index);
+    s->entry_order[index] = order;
     trace_fw_cfg_add_file(s, index, s->files->f[index].name, len);
 
-    s->files->count = cpu_to_be32(index+1);
+    s->files->count = cpu_to_be32(count+1);
 }
 
 void fw_cfg_add_file(FWCfgState *s,  const char *filename,
@@ -595,7 +876,6 @@ static void fw_cfg_init1(DeviceState *dev)
     qdev_init_nofail(dev);
 
     fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (char *)"QEMU", 4);
-    fw_cfg_add_i32(s, FW_CFG_ID, 1);
     fw_cfg_add_bytes(s, FW_CFG_UUID, qemu_uuid, 16);
     fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)(display_type == DT_NOGRAPHIC));
     fw_cfg_add_i16(s, FW_CFG_NB_CPUS, (uint16_t)smp_cpus);
@@ -607,25 +887,57 @@ static void fw_cfg_init1(DeviceState *dev)
     qemu_add_machine_init_done_notifier(&s->machine_ready);
 }
 
-FWCfgState *fw_cfg_init_io(uint32_t iobase)
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as)
 {
     DeviceState *dev;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_iobase && dma_as;
 
     dev = qdev_create(NULL, TYPE_FW_CFG_IO);
     qdev_prop_set_uint32(dev, "iobase", iobase);
+    qdev_prop_set_uint32(dev, "dma_iobase", dma_iobase);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
+    }
+
     fw_cfg_init1(dev);
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        /* 64 bits for the address field */
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+
+        version |= FW_CFG_VERSION_DMA;
+    }
+
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
+
+    return s;
+}
 
-    return FW_CFG(dev);
+FWCfgState *fw_cfg_init_io(uint32_t iobase)
+{
+    return fw_cfg_init_io_dma(iobase, 0, NULL);
 }
 
-FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
-                                 uint32_t data_width)
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as)
 {
     DeviceState *dev;
     SysBusDevice *sbd;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_addr && dma_as;
 
     dev = qdev_create(NULL, TYPE_FW_CFG_MEM);
     qdev_prop_set_uint32(dev, "data_width", data_width);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
+    }
 
     fw_cfg_init1(dev);
 
@@ -633,13 +945,25 @@ FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
     sysbus_mmio_map(sbd, 0, ctl_addr);
     sysbus_mmio_map(sbd, 1, data_addr);
 
-    return FW_CFG(dev);
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+        sysbus_mmio_map(sbd, 2, dma_addr);
+        version |= FW_CFG_VERSION_DMA;
+    }
+
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
+
+    return s;
 }
 
 FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr)
 {
     return fw_cfg_init_mem_wide(ctl_addr, data_addr,
-                                fw_cfg_data_mem_ops.valid.max_access_size);
+                                fw_cfg_data_mem_ops.valid.max_access_size,
+                                0, NULL);
 }
 
 
@@ -666,6 +990,9 @@ static const TypeInfo fw_cfg_info = {
 
 static Property fw_cfg_io_properties[] = {
     DEFINE_PROP_UINT32("iobase", FWCfgIoState, iobase, -1),
+    DEFINE_PROP_UINT32("dma_iobase", FWCfgIoState, dma_iobase, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgIoState, parent_obj.dma_enabled,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -674,9 +1001,19 @@ static void fw_cfg_io_realize(DeviceState *dev, Error **errp)
     FWCfgIoState *s = FW_CFG_IO(dev);
     SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
 
+    /* when using port i/o, the 8-bit data register ALWAYS overlaps
+     * with half of the 16-bit control register. Hence, the total size
+     * of the i/o region used is FW_CFG_CTL_SIZE */
     memory_region_init_io(&s->comb_iomem, OBJECT(s), &fw_cfg_comb_mem_ops,
-                          FW_CFG(s), "fwcfg", FW_CFG_SIZE);
+                          FW_CFG(s), "fwcfg", FW_CFG_CTL_SIZE);
     sysbus_add_io(sbd, s->iobase, &s->comb_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_add_io(sbd, s->dma_iobase, &FW_CFG(s)->dma_iomem);
+    }
 }
 
 static void fw_cfg_io_class_init(ObjectClass *klass, void *data)
@@ -697,6 +1034,8 @@ static const TypeInfo fw_cfg_io_info = {
 
 static Property fw_cfg_mem_properties[] = {
     DEFINE_PROP_UINT32("data_width", FWCfgMemState, data_width, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgMemState, parent_obj.dma_enabled,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -707,7 +1046,7 @@ static void fw_cfg_mem_realize(DeviceState *dev, Error **errp)
     const MemoryRegionOps *data_ops = &fw_cfg_data_mem_ops;
 
     memory_region_init_io(&s->ctl_iomem, OBJECT(s), &fw_cfg_ctl_mem_ops,
-                          FW_CFG(s), "fwcfg.ctl", FW_CFG_SIZE);
+                          FW_CFG(s), "fwcfg.ctl", FW_CFG_CTL_SIZE);
     sysbus_init_mmio(sbd, &s->ctl_iomem);
 
     if (s->data_width > data_ops->valid.max_access_size) {
@@ -725,6 +1064,13 @@ static void fw_cfg_mem_realize(DeviceState *dev, Error **errp)
     memory_region_init_io(&s->data_iomem, OBJECT(s), data_ops, FW_CFG(s),
                           "fwcfg.data", data_ops->valid.max_access_size);
     sysbus_init_mmio(sbd, &s->data_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_init_mmio(sbd, &FW_CFG(s)->dma_iomem);
+    }
 }
 
 static void fw_cfg_mem_class_init(ObjectClass *klass, void *data)
diff --git a/qemu/hw/nvram/mac_nvram.c b/qemu/hw/nvram/mac_nvram.c
index d35f8a312..24f61212b 100644
--- a/qemu/hw/nvram/mac_nvram.c
+++ b/qemu/hw/nvram/mac_nvram.c
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/nvram/openbios_firmware_abi.h"
 #include "sysemu/sysemu.h"
 #include "hw/ppc/mac.h"
+#include "qemu/cutils.h"
 #include <zlib.h>
 
 /* debug NVR */
@@ -48,7 +50,8 @@ static void macio_nvram_writeb(void *opaque, hwaddr addr,
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     s->data[addr] = value;
-    NVR_DPRINTF("writeb addr %04" PHYS_PRIx " val %" PRIx64 "\n", addr, value);
+    NVR_DPRINTF("writeb addr %04" HWADDR_PRIx " val %" PRIx64 "\n",
+                addr, value);
 }
 
 static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
@@ -59,7 +62,8 @@ static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     value = s->data[addr];
-    NVR_DPRINTF("readb addr %04x val %x\n", (int)addr, value);
+    NVR_DPRINTF("readb addr %04" HWADDR_PRIx " val %" PRIx32 "\n",
+                addr, value);
 
     return value;
 }
@@ -123,6 +127,7 @@ static void macio_nvram_class_init(ObjectClass *oc, void *data)
     dc->reset = macio_nvram_reset;
     dc->vmsd = &vmstate_macio_nvram;
     dc->props = macio_nvram_properties;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo macio_nvram_type_info = {
diff --git a/qemu/hw/nvram/spapr_nvram.c b/qemu/hw/nvram/spapr_nvram.c
index fcaa77dd9..802636ef3 100644
--- a/qemu/hw/nvram/spapr_nvram.c
+++ b/qemu/hw/nvram/spapr_nvram.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include <libfdt.h>
 
 #include "sysemu/block-backend.h"
diff --git a/qemu/hw/openrisc/cputimer.c b/qemu/hw/openrisc/cputimer.c
index 9c5494510..a98c799de 100644
--- a/qemu/hw/openrisc/cputimer.c
+++ b/qemu/hw/openrisc/cputimer.c
@@ -18,11 +18,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 
-#define TIMER_FREQ    (20 * 1000 * 1000)    /* 20MHz */
+#define TIMER_PERIOD 50 /* 50 ns period for 20 MHz timer */
 
 /* The time when TTCR changes */
 static uint64_t last_clk;
@@ -36,8 +37,7 @@ void cpu_openrisc_count_update(OpenRISCCPU *cpu)
         return;
     }
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    cpu->env.ttcr += (uint32_t)muldiv64(now - last_clk, TIMER_FREQ,
-                                        get_ticks_per_sec());
+    cpu->env.ttcr += (uint32_t)((now - last_clk) / TIMER_PERIOD);
     last_clk = now;
 }
 
@@ -59,7 +59,7 @@ void cpu_openrisc_timer_update(OpenRISCCPU *cpu)
     } else {
         wait = (cpu->env.ttmr & TTMR_TP) - (cpu->env.ttcr & TTMR_TP);
     }
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(cpu->env.timer, next);
 }
 
diff --git a/qemu/hw/openrisc/openrisc_sim.c b/qemu/hw/openrisc/openrisc_sim.c
index 1da0657dd..6d06d5be0 100644
--- a/qemu/hw/openrisc/openrisc_sim.c
+++ b/qemu/hw/openrisc/openrisc_sim.c
@@ -18,6 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "elf.h"
@@ -68,7 +72,8 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
 
     if (kernel_filename && !qtest_enabled()) {
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &elf_entry, NULL, NULL, 1, ELF_MACHINE, 1);
+                               &elf_entry, NULL, NULL, 1, EM_OPENRISC,
+                               1, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename,
@@ -114,7 +119,7 @@ static void openrisc_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
@@ -132,17 +137,12 @@ static void openrisc_sim_init(MachineState *machine)
     cpu_openrisc_load_kernel(ram_size, kernel_filename, cpu);
 }
 
-static QEMUMachine openrisc_sim_machine = {
-    .name = "or32-sim",
-    .desc = "or32 simulation",
-    .init = openrisc_sim_init,
-    .max_cpus = 1,
-    .is_default = 1,
-};
-
-static void openrisc_sim_machine_init(void)
+static void openrisc_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&openrisc_sim_machine);
+    mc->desc = "or32 simulation";
+    mc->init = openrisc_sim_init;
+    mc->max_cpus = 1;
+    mc->is_default = 1;
 }
 
-machine_init(openrisc_sim_machine_init);
+DEFINE_MACHINE("or32-sim", openrisc_sim_machine_init)
diff --git a/qemu/hw/openrisc/pic_cpu.c b/qemu/hw/openrisc/pic_cpu.c
index 2af1d6013..569b443f5 100644
--- a/qemu/hw/openrisc/pic_cpu.c
+++ b/qemu/hw/openrisc/pic_cpu.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "cpu.h"
 
diff --git a/qemu/hw/pci-bridge/dec.c b/qemu/hw/pci-bridge/dec.c
index 28d0ff9c8..840c96198 100644
--- a/qemu/hw/pci-bridge/dec.c
+++ b/qemu/hw/pci-bridge/dec.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "dec.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
@@ -51,9 +52,9 @@ static int dec_map_irq(PCIDevice *pci_dev, int irq_num)
     return irq_num;
 }
 
-static int dec_pci_bridge_initfn(PCIDevice *pci_dev)
+static void dec_pci_bridge_realize(PCIDevice *pci_dev, Error **errp)
 {
-    return pci_bridge_initfn(pci_dev, TYPE_PCI_BUS);
+    pci_bridge_initfn(pci_dev, TYPE_PCI_BUS);
 }
 
 static void dec_21154_pci_bridge_class_init(ObjectClass *klass, void *data)
@@ -61,7 +62,7 @@ static void dec_21154_pci_bridge_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = dec_pci_bridge_initfn;
+    k->realize = dec_pci_bridge_realize;
     k->exit = pci_bridge_exitfn;
     k->vendor_id = PCI_VENDOR_ID_DEC;
     k->device_id = PCI_DEVICE_ID_DEC_21154;
diff --git a/qemu/hw/pci-bridge/i82801b11.c b/qemu/hw/pci-bridge/i82801b11.c
index 7e79bc01e..2404e7eba 100644
--- a/qemu/hw/pci-bridge/i82801b11.c
+++ b/qemu/hw/pci-bridge/i82801b11.c
@@ -41,6 +41,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/ich9.h"
 
@@ -61,10 +62,7 @@ static int i82801b11_bridge_initfn(PCIDevice *d)
 {
     int rc;
 
-    rc = pci_bridge_initfn(d, TYPE_PCI_BUS);
-    if (rc < 0) {
-        return rc;
-    }
+    pci_bridge_initfn(d, TYPE_PCI_BUS);
 
     rc = pci_bridge_ssvid_init(d, I82801ba_SSVID_OFFSET,
                                I82801ba_SSVID_SVID, I82801ba_SSVID_SSID);
@@ -80,6 +78,14 @@ err_bridge:
     return rc;
 }
 
+static const VMStateDescription i82801b11_bridge_dev_vmstate = {
+    .name = "i82801b11_bridge",
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, PCIBridge),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void i82801b11_bridge_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
@@ -91,6 +97,7 @@ static void i82801b11_bridge_class_init(ObjectClass *klass, void *data)
     k->revision = ICH9_D2P_A2_REVISION;
     k->init = i82801b11_bridge_initfn;
     k->config_write = pci_bridge_write_config;
+    dc->vmsd = &i82801b11_bridge_dev_vmstate;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
diff --git a/qemu/hw/pci-bridge/ioh3420.c b/qemu/hw/pci-bridge/ioh3420.c
index cce2fdd8e..0937fa34b 100644
--- a/qemu/hw/pci-bridge/ioh3420.c
+++ b/qemu/hw/pci-bridge/ioh3420.c
@@ -20,6 +20,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
@@ -97,11 +98,7 @@ static int ioh3420_initfn(PCIDevice *d)
     PCIESlot *s = PCIE_SLOT(d);
     int rc;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = pci_bridge_ssvid_init(d, IOH_EP_SSVID_OFFSET,
@@ -129,7 +126,7 @@ static int ioh3420_initfn(PCIDevice *d)
         goto err_pcie_cap;
     }
     pcie_cap_root_init(d);
-    rc = pcie_aer_init(d, IOH_EP_AER_OFFSET);
+    rc = pcie_aer_init(d, IOH_EP_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
diff --git a/qemu/hw/pci-bridge/pci_bridge_dev.c b/qemu/hw/pci-bridge/pci_bridge_dev.c
index 26aded9f0..7b582e96a 100644
--- a/qemu/hw/pci-bridge/pci_bridge_dev.c
+++ b/qemu/hw/pci-bridge/pci_bridge_dev.c
@@ -19,6 +19,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
@@ -52,10 +54,8 @@ static int pci_bridge_dev_initfn(PCIDevice *dev)
     PCIBridgeDev *bridge_dev = PCI_BRIDGE_DEV(dev);
     int err;
 
-    err = pci_bridge_initfn(dev, TYPE_PCI_BUS);
-    if (err) {
-        goto bridge_error;
-    }
+    pci_bridge_initfn(dev, TYPE_PCI_BUS);
+
     if (bridge_dev->flags & (1 << PCI_BRIDGE_DEV_F_SHPC_REQ)) {
         dev->config[PCI_INTERRUPT_PIN] = 0x1;
         memory_region_init(&bridge_dev->bar, OBJECT(dev), "shpc-bar",
@@ -73,7 +73,7 @@ static int pci_bridge_dev_initfn(PCIDevice *dev)
         goto slotid_error;
     }
     if ((bridge_dev->flags & (1 << PCI_BRIDGE_DEV_F_MSI_REQ)) &&
-        msi_supported) {
+        msi_nonbroken) {
         err = msi_init(dev, 0, 1, true, true);
         if (err < 0) {
             goto msi_error;
@@ -94,7 +94,7 @@ slotid_error:
     }
 shpc_error:
     pci_bridge_exitfn(dev);
-bridge_error:
+
     return err;
 }
 
diff --git a/qemu/hw/pci-bridge/pci_expander_bridge.c b/qemu/hw/pci-bridge/pci_expander_bridge.c
index 57f8a3762..ba320bd85 100644
--- a/qemu/hw/pci-bridge/pci_expander_bridge.c
+++ b/qemu/hw/pci-bridge/pci_expander_bridge.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/pci_host.h"
@@ -23,6 +24,9 @@
 #define TYPE_PXB_BUS "pxb-bus"
 #define PXB_BUS(obj) OBJECT_CHECK(PXBBus, (obj), TYPE_PXB_BUS)
 
+#define TYPE_PXB_PCIE_BUS "pxb-pcie-bus"
+#define PXB_PCIE_BUS(obj) OBJECT_CHECK(PXBBus, (obj), TYPE_PXB_PCIE_BUS)
+
 typedef struct PXBBus {
     /*< private >*/
     PCIBus parent_obj;
@@ -34,6 +38,9 @@ typedef struct PXBBus {
 #define TYPE_PXB_DEVICE "pxb"
 #define PXB_DEV(obj) OBJECT_CHECK(PXBDev, (obj), TYPE_PXB_DEVICE)
 
+#define TYPE_PXB_PCIE_DEVICE "pxb-pcie"
+#define PXB_PCIE_DEV(obj) OBJECT_CHECK(PXBDev, (obj), TYPE_PXB_PCIE_DEVICE)
+
 typedef struct PXBDev {
     /*< private >*/
     PCIDevice parent_obj;
@@ -43,13 +50,18 @@ typedef struct PXBDev {
     uint16_t numa_node;
 } PXBDev;
 
+static PXBDev *convert_to_pxb(PCIDevice *dev)
+{
+    return pci_bus_is_express(dev->bus) ? PXB_PCIE_DEV(dev) : PXB_DEV(dev);
+}
+
 static GList *pxb_dev_list;
 
 #define TYPE_PXB_HOST "pxb-host"
 
 static int pxb_bus_num(PCIBus *bus)
 {
-    PXBDev *pxb = PXB_DEV(bus->parent_dev);
+    PXBDev *pxb = convert_to_pxb(bus->parent_dev);
 
     return pxb->bus_nr;
 }
@@ -61,7 +73,7 @@ static bool pxb_is_root(PCIBus *bus)
 
 static uint16_t pxb_bus_numa_node(PCIBus *bus)
 {
-    PXBDev *pxb = PXB_DEV(bus->parent_dev);
+    PXBDev *pxb = convert_to_pxb(bus->parent_dev);
 
     return pxb->numa_node;
 }
@@ -82,10 +94,18 @@ static const TypeInfo pxb_bus_info = {
     .class_init    = pxb_bus_class_init,
 };
 
+static const TypeInfo pxb_pcie_bus_info = {
+    .name          = TYPE_PXB_PCIE_BUS,
+    .parent        = TYPE_PCIE_BUS,
+    .instance_size = sizeof(PXBBus),
+    .class_init    = pxb_bus_class_init,
+};
+
 static const char *pxb_host_root_bus_path(PCIHostState *host_bridge,
                                           PCIBus *rootbus)
 {
-    PXBBus *bus = PXB_BUS(rootbus);
+    PXBBus *bus = pci_bus_is_express(rootbus) ?
+                  PXB_PCIE_BUS(rootbus) : PXB_BUS(rootbus);
 
     snprintf(bus->bus_path, 8, "0000:%02x", pxb_bus_num(rootbus));
     return bus->bus_path;
@@ -103,7 +123,7 @@ static char *pxb_host_ofw_unit_address(const SysBusDevice *dev)
 
     pxb_host = PCI_HOST_BRIDGE(dev);
     pxb_bus = pxb_host->bus;
-    pxb_dev = PXB_DEV(pxb_bus->parent_dev);
+    pxb_dev = convert_to_pxb(pxb_bus->parent_dev);
     position = g_list_index(pxb_dev_list, pxb_dev);
     assert(position >= 0);
 
@@ -193,10 +213,10 @@ static gint pxb_compare(gconstpointer a, gconstpointer b)
            0;
 }
 
-static int pxb_dev_initfn(PCIDevice *dev)
+static int pxb_dev_init_common(PCIDevice *dev, bool pcie)
 {
-    PXBDev *pxb = PXB_DEV(dev);
-    DeviceState *ds, *bds;
+    PXBDev *pxb = convert_to_pxb(dev);
+    DeviceState *ds, *bds = NULL;
     PCIBus *bus;
     const char *dev_name = NULL;
 
@@ -211,26 +231,31 @@ static int pxb_dev_initfn(PCIDevice *dev)
     }
 
     ds = qdev_create(NULL, TYPE_PXB_HOST);
-    bus = pci_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS);
+    if (pcie) {
+        bus = pci_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_PCIE_BUS);
+    } else {
+        bus = pci_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS);
+        bds = qdev_create(BUS(bus), "pci-bridge");
+        bds->id = dev_name;
+        qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr);
+        qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false);
+    }
 
     bus->parent_dev = dev;
     bus->address_space_mem = dev->bus->address_space_mem;
     bus->address_space_io = dev->bus->address_space_io;
     bus->map_irq = pxb_map_irq_fn;
 
-    bds = qdev_create(BUS(bus), "pci-bridge");
-    bds->id = dev_name;
-    qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr);
-    qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false);
-
     PCI_HOST_BRIDGE(ds)->bus = bus;
 
     if (pxb_register_bus(dev, bus)) {
-        return -EINVAL;
+        goto err_register_bus;
     }
 
     qdev_init_nofail(ds);
-    qdev_init_nofail(bds);
+    if (bds) {
+        qdev_init_nofail(bds);
+    }
 
     pci_word_test_and_set_mask(dev->config + PCI_STATUS,
                                PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK);
@@ -238,17 +263,33 @@ static int pxb_dev_initfn(PCIDevice *dev)
 
     pxb_dev_list = g_list_insert_sorted(pxb_dev_list, pxb, pxb_compare);
     return 0;
+
+err_register_bus:
+    object_unref(OBJECT(bds));
+    object_unparent(OBJECT(bus));
+    object_unref(OBJECT(ds));
+    return -EINVAL;
+}
+
+static int pxb_dev_initfn(PCIDevice *dev)
+{
+    if (pci_bus_is_express(dev->bus)) {
+        error_report("pxb devices cannot reside on a PCIe bus!");
+        return -EINVAL;
+    }
+
+    return pxb_dev_init_common(dev, false);
 }
 
 static void pxb_dev_exitfn(PCIDevice *pci_dev)
 {
-    PXBDev *pxb = PXB_DEV(pci_dev);
+    PXBDev *pxb = convert_to_pxb(pci_dev);
 
     pxb_dev_list = g_list_remove(pxb_dev_list, pxb);
 }
 
 static Property pxb_dev_properties[] = {
-    /* Note: 0 is not a legal a PXB bus number. */
+    /* Note: 0 is not a legal PXB bus number. */
     DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0),
     DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED),
     DEFINE_PROP_END_OF_LIST(),
@@ -267,6 +308,7 @@ static void pxb_dev_class_init(ObjectClass *klass, void *data)
 
     dc->desc = "PCI Expander Bridge";
     dc->props = pxb_dev_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pxb_dev_info = {
@@ -276,11 +318,46 @@ static const TypeInfo pxb_dev_info = {
     .class_init    = pxb_dev_class_init,
 };
 
+static int pxb_pcie_dev_initfn(PCIDevice *dev)
+{
+    if (!pci_bus_is_express(dev->bus)) {
+        error_report("pxb-pcie devices cannot reside on a PCI bus!");
+        return -EINVAL;
+    }
+
+    return pxb_dev_init_common(dev, true);
+}
+
+static void pxb_pcie_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = pxb_pcie_dev_initfn;
+    k->exit = pxb_dev_exitfn;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_PXB_PCIE;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+
+    dc->desc = "PCI Express Expander Bridge";
+    dc->props = pxb_dev_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo pxb_pcie_dev_info = {
+    .name          = TYPE_PXB_PCIE_DEVICE,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PXBDev),
+    .class_init    = pxb_pcie_dev_class_init,
+};
+
 static void pxb_register_types(void)
 {
     type_register_static(&pxb_bus_info);
+    type_register_static(&pxb_pcie_bus_info);
     type_register_static(&pxb_host_info);
     type_register_static(&pxb_dev_info);
+    type_register_static(&pxb_pcie_dev_info);
 }
 
 type_init(pxb_register_types)
diff --git a/qemu/hw/pci-bridge/xio3130_downstream.c b/qemu/hw/pci-bridge/xio3130_downstream.c
index b3a647926..cf1ee63ab 100644
--- a/qemu/hw/pci-bridge/xio3130_downstream.c
+++ b/qemu/hw/pci-bridge/xio3130_downstream.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
@@ -60,11 +61,7 @@ static int xio3130_downstream_initfn(PCIDevice *d)
     PCIESlot *s = PCIE_SLOT(d);
     int rc;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = msi_init(d, XIO3130_MSI_OFFSET, XIO3130_MSI_NR_VECTOR,
@@ -92,7 +89,7 @@ static int xio3130_downstream_initfn(PCIDevice *d)
         goto err_pcie_cap;
     }
     pcie_cap_arifwd_init(d);
-    rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+    rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
diff --git a/qemu/hw/pci-bridge/xio3130_upstream.c b/qemu/hw/pci-bridge/xio3130_upstream.c
index eada5828a..164ef58c4 100644
--- a/qemu/hw/pci-bridge/xio3130_upstream.c
+++ b/qemu/hw/pci-bridge/xio3130_upstream.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
@@ -56,11 +57,7 @@ static int xio3130_upstream_initfn(PCIDevice *d)
     PCIEPort *p = PCIE_PORT(d);
     int rc;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = msi_init(d, XIO3130_MSI_OFFSET, XIO3130_MSI_NR_VECTOR,
@@ -81,7 +78,7 @@ static int xio3130_upstream_initfn(PCIDevice *d)
     }
     pcie_cap_flr_init(d);
     pcie_cap_deverr_init(d);
-    rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+    rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
diff --git a/qemu/hw/pci-host/apb.c b/qemu/hw/pci-host/apb.c
index 599768e2d..aaef7bb3a 100644
--- a/qemu/hw/pci-host/apb.c
+++ b/qemu/hw/pci-host/apb.c
@@ -27,6 +27,7 @@
    Ultrasparc PCI host is called the PCI Bus Module (PBM).  The APB is
    the secondary PCI bridge.  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
@@ -634,12 +635,7 @@ static void pci_apb_set_irq(void *opaque, int irq_num, int level)
 
 static int apb_pci_bridge_initfn(PCIDevice *dev)
 {
-    int rc;
-
-    rc = pci_bridge_initfn(dev, TYPE_PCI_BUS);
-    if (rc < 0) {
-        return rc;
-    }
+    pci_bridge_initfn(dev, TYPE_PCI_BUS);
 
     /*
      * command register:
diff --git a/qemu/hw/pci-host/bonito.c b/qemu/hw/pci-host/bonito.c
index 3a731fe18..1999ece59 100644
--- a/qemu/hw/pci-host/bonito.c
+++ b/qemu/hw/pci-host/bonito.c
@@ -37,7 +37,7 @@
  * north bridge address to pci address.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
@@ -180,8 +180,6 @@
 #define PCI_ADDR(busno,devno,funno,regno)  \
     ((((busno)<<16)&0xff0000) + (((devno)<<11)&0xf800) + (((funno)<<8)&0x700) + (regno))
 
-#define TYPE_BONITO_PCI_HOST_BRIDGE "Bonito-pcihost"
-
 typedef struct BonitoState BonitoState;
 
 typedef struct PCIBonitoState
@@ -215,17 +213,20 @@ typedef struct PCIBonitoState
 
 } PCIBonitoState;
 
-#define BONITO_PCI_HOST_BRIDGE(obj) \
-    OBJECT_CHECK(BonitoState, (obj), TYPE_BONITO_PCI_HOST_BRIDGE)
-
 struct BonitoState {
     PCIHostState parent_obj;
-
     qemu_irq *pic;
-
     PCIBonitoState *pci_dev;
 };
 
+#define TYPE_BONITO_PCI_HOST_BRIDGE "Bonito-pcihost"
+#define BONITO_PCI_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(BonitoState, (obj), TYPE_BONITO_PCI_HOST_BRIDGE)
+
+#define TYPE_PCI_BONITO "Bonito"
+#define PCI_BONITO(obj) \
+    OBJECT_CHECK(PCIBonitoState, (obj), TYPE_PCI_BONITO)
+
 static void bonito_writel(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
@@ -355,6 +356,10 @@ static uint64_t bonito_ldma_readl(void *opaque, hwaddr addr,
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)];
 
     return val;
@@ -365,6 +370,10 @@ static void bonito_ldma_writel(void *opaque, hwaddr addr,
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return;
+    }
+
     ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
@@ -384,6 +393,10 @@ static uint64_t bonito_cop_readl(void *opaque, hwaddr addr,
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)];
 
     return val;
@@ -394,6 +407,10 @@ static void bonito_cop_writel(void *opaque, hwaddr addr,
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return;
+    }
+
     ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
@@ -707,7 +724,7 @@ static int bonito_pcihost_initfn(SysBusDevice *dev)
 
 static void bonito_realize(PCIDevice *dev, Error **errp)
 {
-    PCIBonitoState *s = DO_UPCAST(PCIBonitoState, dev, dev);
+    PCIBonitoState *s = PCI_BONITO(dev);
     SysBusDevice *sysbus = SYS_BUS_DEVICE(s->pcihost);
     PCIHostState *phb = PCI_HOST_BRIDGE(s->pcihost);
 
@@ -783,8 +800,8 @@ PCIBus *bonito_init(qemu_irq *pic)
     qdev_init_nofail(dev);
 
     /* set the pcihost pointer before bonito_initfn is called */
-    d = pci_create(phb->bus, PCI_DEVFN(0, 0), "Bonito");
-    s = DO_UPCAST(PCIBonitoState, dev, d);
+    d = pci_create(phb->bus, PCI_DEVFN(0, 0), TYPE_PCI_BONITO);
+    s = PCI_BONITO(d);
     s->pcihost = pcihost;
     pcihost->pci_dev = s;
     qdev_init_nofail(DEVICE(d));
@@ -812,7 +829,7 @@ static void bonito_class_init(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo bonito_info = {
-    .name          = "Bonito",
+    .name          = TYPE_PCI_BONITO,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBonitoState),
     .class_init    = bonito_class_init,
diff --git a/qemu/hw/pci-host/gpex.c b/qemu/hw/pci-host/gpex.c
index 9d8fb5a49..66055ee5c 100644
--- a/qemu/hw/pci-host/gpex.c
+++ b/qemu/hw/pci-host/gpex.c
@@ -28,6 +28,7 @@
  * http://www.kernel.org/doc/Documentation/devicetree/bindings/pci/host-generic-pci.txt
  * http://www.firmware.org/1275/practice/imap/imap0_9d.pdf
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci-host/gpex.h"
 
diff --git a/qemu/hw/pci-host/grackle.c b/qemu/hw/pci-host/grackle.c
index bfe707a1a..8f9121615 100644
--- a/qemu/hw/pci-host/grackle.c
+++ b/qemu/hw/pci-host/grackle.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -146,8 +147,10 @@ static const TypeInfo grackle_pci_info = {
 static void pci_grackle_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     k->init = pci_grackle_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo grackle_pci_host_info = {
diff --git a/qemu/hw/pci-host/pam.c b/qemu/hw/pci-host/pam.c
index 17d826cba..e361ecb7e 100644
--- a/qemu/hw/pci-host/pam.c
+++ b/qemu/hw/pci-host/pam.c
@@ -27,6 +27,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "sysemu/sysemu.h"
 #include "hw/pci-host/pam.h"
diff --git a/qemu/hw/pci-host/piix.c b/qemu/hw/pci-host/piix.c
index ad55f9966..df2b0e26f 100644
--- a/qemu/hw/pci-host/piix.c
+++ b/qemu/hw/pci-host/piix.c
@@ -22,25 +22,27 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/range.h"
 #include "hw/xen/xen.h"
 #include "hw/pci-host/pam.h"
 #include "sysemu/sysemu.h"
 #include "hw/i386/ioapic.h"
 #include "qapi/visitor.h"
+#include "qemu/error-report.h"
 
 /*
  * I440FX chipset data sheet.
  * http://download.intel.com/design/chipsets/datashts/29054901.pdf
  */
 
-#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
 #define I440FX_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(I440FXState, (obj), TYPE_I440FX_PCI_HOST_BRIDGE)
 
@@ -95,7 +97,6 @@ typedef struct PIIX3State {
 #define PIIX3_PCI_DEVICE(obj) \
     OBJECT_CHECK(PIIX3State, (obj), TYPE_PIIX3_PCI_DEVICE)
 
-#define TYPE_I440FX_PCI_DEVICE "i440FX"
 #define I440FX_PCI_DEVICE(obj) \
     OBJECT_CHECK(PCII440FXState, (obj), TYPE_I440FX_PCI_DEVICE)
 
@@ -117,6 +118,11 @@ struct PCII440FXState {
 #define I440FX_PAM_SIZE 7
 #define I440FX_SMRAM    0x72
 
+/* Older coreboot versions (4.0 and older) read a config register that doesn't
+ * exist in real hardware, to get the RAM size from QEMU.
+ */
+#define I440FX_COREBOOT_RAM_SIZE 0x57
+
 static void piix3_set_irq(void *opaque, int pirq, int level);
 static PCIINTxRoute piix3_route_intx_pin_to_irq(void *opaque, int pci_intx);
 static void piix3_write_config_xen(PCIDevice *dev,
@@ -211,39 +217,39 @@ static const VMStateDescription vmstate_i440fx = {
 };
 
 static void i440fx_pcihost_get_pci_hole_start(Object *obj, Visitor *v,
-                                              void *opaque, const char *name,
+                                              const char *name, void *opaque,
                                               Error **errp)
 {
     I440FXState *s = I440FX_PCI_HOST_BRIDGE(obj);
     uint32_t value = s->pci_info.w32.begin;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole_end(Object *obj, Visitor *v,
-                                            void *opaque, const char *name,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     I440FXState *s = I440FX_PCI_HOST_BRIDGE(obj);
     uint32_t value = s->pci_info.w32.end;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole64_start(Object *obj, Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
+                                                const char *name,
+                                                void *opaque, Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
     Range w64;
 
     pci_bus_get_w64_range(h->bus, &w64);
 
-    visit_type_uint64(v, &w64.begin, name, errp);
+    visit_type_uint64(v, name, &w64.begin, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole64_end(Object *obj, Visitor *v,
-                                              void *opaque, const char *name,
+                                              const char *name, void *opaque,
                                               Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
@@ -251,7 +257,7 @@ static void i440fx_pcihost_get_pci_hole64_end(Object *obj, Visitor *v,
 
     pci_bus_get_w64_range(h->bus, &w64);
 
-    visit_type_uint64(v, &w64.end, name, errp);
+    visit_type_uint64(v, name, &w64.end, errp);
 }
 
 static void i440fx_pcihost_initfn(Object *obj)
@@ -298,9 +304,14 @@ static void i440fx_pcihost_realize(DeviceState *dev, Error **errp)
 static void i440fx_realize(PCIDevice *dev, Error **errp)
 {
     dev->config[I440FX_SMRAM] = 0x02;
+
+    if (object_property_get_bool(qdev_get_machine(), "iommu", NULL)) {
+        error_report("warning: i440fx doesn't support emulated iommu");
+    }
 }
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state,
                     int *piix3_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
@@ -320,7 +331,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     unsigned i;
     I440FXState *i440fx;
 
-    dev = qdev_create(NULL, TYPE_I440FX_PCI_HOST_BRIDGE);
+    dev = qdev_create(NULL, host_type);
     s = PCI_HOST_BRIDGE(dev);
     b = pci_bus_new(dev, NULL, pci_address_space,
                     address_space_io, 0, TYPE_PCI_BUS);
@@ -328,7 +339,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     object_property_add_child(qdev_get_machine(), "i440fx", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 
-    d = pci_create_simple(b, 0, TYPE_I440FX_PCI_DEVICE);
+    d = pci_create_simple(b, 0, pci_type);
     *pi440fx_state = I440FX_PCI_DEVICE(d);
     f = *pi440fx_state;
     f->system_memory = address_space_mem;
@@ -394,7 +405,7 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
     if (ram_size > 255) {
         ram_size = 255;
     }
-    d->config[0x57] = ram_size;
+    d->config[I440FX_COREBOOT_RAM_SIZE] = ram_size;
 
     i440fx_update_memory_mappings(f);
 
@@ -642,8 +653,10 @@ static void piix3_realize(PCIDevice *dev, Error **errp)
 {
     PIIX3State *d = PIIX3_PCI_DEVICE(dev);
 
-    isa_bus_new(DEVICE(d), get_system_memory(),
-                pci_address_space_io(dev));
+    if (!isa_bus_new(DEVICE(d), get_system_memory(),
+                     pci_address_space_io(dev), errp)) {
+        return;
+    }
 
     memory_region_init_io(&d->rcr_mem, OBJECT(dev), &rcr_ops, d,
                           "piix3-reset-control", 1);
@@ -735,6 +748,95 @@ static const TypeInfo i440fx_info = {
     .class_init    = i440fx_class_init,
 };
 
+/* IGD Passthrough Host Bridge. */
+typedef struct {
+    uint8_t offset;
+    uint8_t len;
+} IGDHostInfo;
+
+/* Here we just expose minimal host bridge offset subset. */
+static const IGDHostInfo igd_host_bridge_infos[] = {
+    {0x08, 2},  /* revision id */
+    {0x2c, 2},  /* sybsystem vendor id */
+    {0x2e, 2},  /* sybsystem id */
+    {0x50, 2},  /* SNB: processor graphics control register */
+    {0x52, 2},  /* processor graphics control register */
+    {0xa4, 4},  /* SNB: graphics base of stolen memory */
+    {0xa8, 4},  /* SNB: base of GTT stolen memory */
+};
+
+static int host_pci_config_read(int pos, int len, uint32_t *val)
+{
+    char path[PATH_MAX];
+    int config_fd;
+    ssize_t size = sizeof(path);
+    /* Access real host bridge. */
+    int rc = snprintf(path, size, "/sys/bus/pci/devices/%04x:%02x:%02x.%d/%s",
+                      0, 0, 0, 0, "config");
+    int ret = 0;
+
+    if (rc >= size || rc < 0) {
+        return -ENODEV;
+    }
+
+    config_fd = open(path, O_RDWR);
+    if (config_fd < 0) {
+        return -ENODEV;
+    }
+
+    if (lseek(config_fd, pos, SEEK_SET) != pos) {
+        ret = -errno;
+        goto out;
+    }
+
+    do {
+        rc = read(config_fd, (uint8_t *)val, len);
+    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
+    if (rc != len) {
+        ret = -errno;
+    }
+
+out:
+    close(config_fd);
+    return ret;
+}
+
+static int igd_pt_i440fx_initfn(struct PCIDevice *pci_dev)
+{
+    uint32_t val = 0;
+    int rc, i, num;
+    int pos, len;
+
+    num = ARRAY_SIZE(igd_host_bridge_infos);
+    for (i = 0; i < num; i++) {
+        pos = igd_host_bridge_infos[i].offset;
+        len = igd_host_bridge_infos[i].len;
+        rc = host_pci_config_read(pos, len, &val);
+        if (rc) {
+            return -ENODEV;
+        }
+        pci_default_write_config(pci_dev, pos, val, len);
+    }
+
+    return 0;
+}
+
+static void igd_passthrough_i440fx_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = igd_pt_i440fx_initfn;
+    dc->desc = "IGD Passthrough Host bridge";
+}
+
+static const TypeInfo igd_passthrough_i440fx_info = {
+    .name          = TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE,
+    .parent        = TYPE_I440FX_PCI_DEVICE,
+    .instance_size = sizeof(PCII440FXState),
+    .class_init    = igd_passthrough_i440fx_class_init,
+};
+
 static const char *i440fx_pcihost_root_bus_path(PCIHostState *host_bridge,
                                                 PCIBus *rootbus)
 {
@@ -776,6 +878,7 @@ static const TypeInfo i440fx_pcihost_info = {
 static void i440fx_register_types(void)
 {
     type_register_static(&i440fx_info);
+    type_register_static(&igd_passthrough_i440fx_info);
     type_register_static(&piix3_pci_type_info);
     type_register_static(&piix3_info);
     type_register_static(&piix3_xen_info);
diff --git a/qemu/hw/pci-host/ppce500.c b/qemu/hw/pci-host/ppce500.c
index 613ba73c6..e502bc050 100644
--- a/qemu/hw/pci-host/ppce500.c
+++ b/qemu/hw/pci-host/ppce500.c
@@ -14,6 +14,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/e500-ccsr.h"
 #include "hw/pci/pci.h"
@@ -140,7 +141,7 @@ static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
     case PPCE500_PCI_OW3:
     case PPCE500_PCI_OW4:
         idx = (addr >> 5) & 0x7;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_POTAR:
             value = pci->pob[idx].potar;
             break;
@@ -162,7 +163,7 @@ static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
     case PPCE500_PCI_IW2:
     case PPCE500_PCI_IW1:
         idx = ((addr >> 5) & 0x3) - 1;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_PITAR:
             value = pci->pib[idx].pitar;
             break;
diff --git a/qemu/hw/pci-host/prep.c b/qemu/hw/pci-host/prep.c
index c63f45d21..487e32ecb 100644
--- a/qemu/hw/pci-host/prep.c
+++ b/qemu/hw/pci-host/prep.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
@@ -302,7 +304,7 @@ static void raven_realize(PCIDevice *d, Error **errp)
     d->config[0x34] = 0x00; // capabilities_pointer
 
     memory_region_init_ram(&s->bios, OBJECT(s), "bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_readonly(&s->bios, true);
     memory_region_add_subregion(get_system_memory(), (uint32_t)(-BIOS_SIZE),
                                 &s->bios);
@@ -312,7 +314,7 @@ static void raven_realize(PCIDevice *d, Error **errp)
         if (filename) {
             if (s->elf_machine != EM_NONE) {
                 bios_size = load_elf(filename, NULL, NULL, NULL,
-                                     NULL, NULL, 1, s->elf_machine, 0);
+                                     NULL, NULL, 1, s->elf_machine, 0, 0);
             }
             if (bios_size < 0) {
                 bios_size = get_image_size(filename);
@@ -326,11 +328,10 @@ static void raven_realize(PCIDevice *d, Error **errp)
             }
         }
         if (bios_size < 0 || bios_size > BIOS_SIZE) {
+            /* FIXME should error_setg() */
             hw_error("qemu: could not load bios image '%s'\n", s->bios_name);
         }
-        if (filename) {
-            g_free(filename);
-        }
+        g_free(filename);
     }
 }
 
@@ -357,8 +358,9 @@ static void raven_class_init(ObjectClass *klass, void *data)
     dc->desc = "PReP Host Bridge - Motorola Raven";
     dc->vmsd = &vmstate_raven;
     /*
-     * PCI-facing part of the host bridge, not usable without the
-     * host-facing part, which can't be device_add'ed, yet.
+     * Reason: PCI-facing part of the host bridge, not usable without
+     * the host-facing part, which can't be device_add'ed, yet.
+     * Reason: realize() method uses hw_error().
      */
     dc->cannot_instantiate_with_device_add_yet = true;
 }
diff --git a/qemu/hw/pci-host/q35.c b/qemu/hw/pci-host/q35.c
index bd7409456..70f897e3a 100644
--- a/qemu/hw/pci-host/q35.c
+++ b/qemu/hw/pci-host/q35.c
@@ -27,8 +27,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci-host/q35.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 /****************************************************************************
@@ -67,27 +69,27 @@ static const char *q35_host_root_bus_path(PCIHostState *host_bridge,
 }
 
 static void q35_host_get_pci_hole_start(Object *obj, Visitor *v,
-                                        void *opaque, const char *name,
+                                        const char *name, void *opaque,
                                         Error **errp)
 {
     Q35PCIHost *s = Q35_HOST_DEVICE(obj);
     uint32_t value = s->mch.pci_info.w32.begin;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void q35_host_get_pci_hole_end(Object *obj, Visitor *v,
-                                      void *opaque, const char *name,
+                                      const char *name, void *opaque,
                                       Error **errp)
 {
     Q35PCIHost *s = Q35_HOST_DEVICE(obj);
     uint32_t value = s->mch.pci_info.w32.end;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void q35_host_get_pci_hole64_start(Object *obj, Visitor *v,
-                                          void *opaque, const char *name,
+                                          const char *name, void *opaque,
                                           Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
@@ -95,11 +97,11 @@ static void q35_host_get_pci_hole64_start(Object *obj, Visitor *v,
 
     pci_bus_get_w64_range(h->bus, &w64);
 
-    visit_type_uint64(v, &w64.begin, name, errp);
+    visit_type_uint64(v, name, &w64.begin, errp);
 }
 
 static void q35_host_get_pci_hole64_end(Object *obj, Visitor *v,
-                                        void *opaque, const char *name,
+                                        const char *name, void *opaque,
                                         Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
@@ -107,17 +109,16 @@ static void q35_host_get_pci_hole64_end(Object *obj, Visitor *v,
 
     pci_bus_get_w64_range(h->bus, &w64);
 
-    visit_type_uint64(v, &w64.end, name, errp);
+    visit_type_uint64(v, name, &w64.end, errp);
 }
 
-static void q35_host_get_mmcfg_size(Object *obj, Visitor *v,
-                                    void *opaque, const char *name,
-                                    Error **errp)
+static void q35_host_get_mmcfg_size(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     PCIExpressHost *e = PCIE_HOST_BRIDGE(obj);
     uint32_t value = e->size;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static Property mch_props[] = {
@@ -426,31 +427,12 @@ static void mch_reset(DeviceState *qdev)
 static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
 {
     IntelIOMMUState *s = opaque;
-    VTDAddressSpace **pvtd_as;
-    int bus_num = pci_bus_num(bus);
+    VTDAddressSpace *vtd_as;
 
-    assert(0 <= bus_num && bus_num <= VTD_PCI_BUS_MAX);
     assert(0 <= devfn && devfn <= VTD_PCI_DEVFN_MAX);
 
-    pvtd_as = s->address_spaces[bus_num];
-    if (!pvtd_as) {
-        /* No corresponding free() */
-        pvtd_as = g_malloc0(sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
-        s->address_spaces[bus_num] = pvtd_as;
-    }
-    if (!pvtd_as[devfn]) {
-        pvtd_as[devfn] = g_malloc0(sizeof(VTDAddressSpace));
-
-        pvtd_as[devfn]->bus_num = (uint8_t)bus_num;
-        pvtd_as[devfn]->devfn = (uint8_t)devfn;
-        pvtd_as[devfn]->iommu_state = s;
-        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;
-        memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),
-                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
-        address_space_init(&pvtd_as[devfn]->as,
-                           &pvtd_as[devfn]->iommu, "intel_iommu");
-    }
-    return &pvtd_as[devfn]->as;
+    vtd_as = vtd_find_add_as(s, bus, devfn);
+    return &vtd_as->as;
 }
 
 static void mch_init_dmar(MCHPCIState *mch)
@@ -525,7 +507,7 @@ static void mch_realize(PCIDevice *d, Error **errp)
                  PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE, PAM_EXPAN_SIZE);
     }
     /* Intel IOMMU (VT-d) */
-    if (machine_iommu(current_machine)) {
+    if (object_property_get_bool(qdev_get_machine(), "iommu", NULL)) {
         mch_init_dmar(mch);
     }
 }
diff --git a/qemu/hw/pci-host/uninorth.c b/qemu/hw/pci-host/uninorth.c
index f0144eb7b..15b105423 100644
--- a/qemu/hw/pci-host/uninorth.c
+++ b/qemu/hw/pci-host/uninorth.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -119,7 +120,7 @@ static void unin_data_write(void *opaque, hwaddr addr,
 {
     UNINState *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
-    UNIN_DPRINTF("write addr %" TARGET_FMT_plx " len %d val %"PRIx64"\n",
+    UNIN_DPRINTF("write addr " TARGET_FMT_plx " len %d val %"PRIx64"\n",
                  addr, len, val);
     pci_data_write(phb->bus,
                    unin_get_config_reg(phb->config_reg, addr),
@@ -136,7 +137,7 @@ static uint64_t unin_data_read(void *opaque, hwaddr addr,
     val = pci_data_read(phb->bus,
                         unin_get_config_reg(phb->config_reg, addr),
                         len);
-    UNIN_DPRINTF("read addr %" TARGET_FMT_plx " len %d val %x\n",
+    UNIN_DPRINTF("read addr " TARGET_FMT_plx " len %d val %x\n",
                  addr, len, val);
     return val;
 }
@@ -330,6 +331,15 @@ static void unin_agp_pci_host_realize(PCIDevice *d, Error **errp)
     d->config[0x0C] = 0x08; // cache_line_size
     d->config[0x0D] = 0x10; // latency_timer
     //    d->config[0x34] = 0x80; // capabilities_pointer
+    /*
+     * Set kMacRISCPCIAddressSelect (0x48) register to indicate PCI
+     * memory space with base 0x80000000, size 0x10000000 for Apple's
+     * AppleMacRiscPCI driver
+     */
+    d->config[0x48] = 0x0;
+    d->config[0x49] = 0x0;
+    d->config[0x4a] = 0x0;
+    d->config[0x4b] = 0x1;
 }
 
 static void u3_agp_pci_host_realize(PCIDevice *d, Error **errp)
@@ -446,8 +456,10 @@ static const TypeInfo unin_internal_pci_host_info = {
 static void pci_unin_main_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_main_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_main_info = {
@@ -460,8 +472,10 @@ static const TypeInfo pci_unin_main_info = {
 static void pci_u3_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_u3_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_u3_agp_info = {
@@ -474,8 +488,10 @@ static const TypeInfo pci_u3_agp_info = {
 static void pci_unin_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_agp_info = {
@@ -488,8 +504,10 @@ static const TypeInfo pci_unin_agp_info = {
 static void pci_unin_internal_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_internal_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_internal_info = {
diff --git a/qemu/hw/pci-host/versatile.c b/qemu/hw/pci-host/versatile.c
index 6d2355309..339ec2c50 100644
--- a/qemu/hw/pci-host/versatile.c
+++ b/qemu/hw/pci-host/versatile.c
@@ -7,6 +7,7 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
@@ -500,6 +501,8 @@ static void pci_vpb_class_init(ObjectClass *klass, void *data)
     dc->reset = pci_vpb_reset;
     dc->vmsd = &pci_vpb_vmstate;
     dc->props = pci_vpb_properties;
+    /* Reason: object_unref() hangs */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo pci_vpb_info = {
@@ -521,10 +524,19 @@ static void pci_realview_init(Object *obj)
     s->mem_win_size[2] = 0x08000000;
 }
 
+static void pci_realview_class_init(ObjectClass *class, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(class);
+
+    /* Reason: object_unref() hangs */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
 static const TypeInfo pci_realview_info = {
     .name          = "realview_pci",
     .parent        = TYPE_VERSATILE_PCI,
     .instance_init = pci_realview_init,
+    .class_init    = pci_realview_class_init,
 };
 
 static void versatile_pci_register_types(void)
diff --git a/qemu/hw/pci/msi.c b/qemu/hw/pci/msi.c
index f9c048442..e0e64c2d9 100644
--- a/qemu/hw/pci/msi.c
+++ b/qemu/hw/pci/msi.c
@@ -18,7 +18,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/msi.h"
+#include "hw/xen/xen.h"
 #include "qemu/range.h"
 
 /* PCI_MSI_ADDRESS_LO */
@@ -32,8 +34,21 @@
 
 #define PCI_MSI_VECTORS_MAX     32
 
-/* Flag for interrupt controller to declare MSI/MSI-X support */
-bool msi_supported;
+/*
+ * Flag for interrupt controllers to declare broken MSI/MSI-X support.
+ * values: false - broken; true - non-broken.
+ *
+ * Setting this flag to false will remove MSI/MSI-X capability from all devices.
+ *
+ * It is preferrable for controllers to set this to true (non-broken) even if
+ * they do not actually support MSI/MSI-X: guests normally probe the controller
+ * type and do not attempt to enable MSI/MSI-X with interrupt controllers not
+ * supporting such, so removing the capability is not required, and
+ * it seems cleaner to have a given device look the same for all boards.
+ *
+ * TODO: some existing controllers violate the above rule. Identify and fix them.
+ */
+bool msi_nonbroken;
 
 /* If we get rid of cap allocator, we won't need this. */
 static inline uint8_t msi_cap_sizeof(uint16_t flags)
@@ -158,7 +173,7 @@ int msi_init(struct PCIDevice *dev, uint8_t offset,
     uint8_t cap_size;
     int config_offset;
 
-    if (!msi_supported) {
+    if (!msi_nonbroken) {
         return -ENOTSUP;
     }
 
@@ -253,13 +268,19 @@ void msi_reset(PCIDevice *dev)
 static bool msi_is_masked(const PCIDevice *dev, unsigned int vector)
 {
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
-    uint32_t mask;
+    uint32_t mask, data;
+    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
     assert(vector < PCI_MSI_VECTORS_MAX);
 
     if (!(flags & PCI_MSI_FLAGS_MASKBIT)) {
         return false;
     }
 
+    data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
+    if (xen_is_pirq_msi(data)) {
+        return false;
+    }
+
     mask = pci_get_long(dev->config +
                         msi_mask_off(dev, flags & PCI_MSI_FLAGS_64BIT));
     return mask & (1U << vector);
@@ -294,7 +315,7 @@ void msi_send_message(PCIDevice *dev, MSIMessage msg)
 {
     MemTxAttrs attrs = {};
 
-    attrs.stream_id = (pci_bus_num(dev->bus) << 8) | dev->devfn;
+    attrs.requester_id = pci_requester_id(dev);
     address_space_stl_le(&dev->bus_master_as, msg.address, msg.data,
                          attrs, NULL);
 }
diff --git a/qemu/hw/pci/msix.c b/qemu/hw/pci/msix.c
index 7716bf364..b75f0e9c4 100644
--- a/qemu/hw/pci/msix.c
+++ b/qemu/hw/pci/msix.c
@@ -14,10 +14,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "hw/pci/pci.h"
+#include "hw/xen/xen.h"
 #include "qemu/range.h"
 
 #define MSIX_CAP_LENGTH 12
@@ -77,8 +79,15 @@ static void msix_clr_pending(PCIDevice *dev, int vector)
 
 static bool msix_vector_masked(PCIDevice *dev, unsigned int vector, bool fmask)
 {
-    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
-    return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
+    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE;
+    uint8_t *data = &dev->msix_table[offset + PCI_MSIX_ENTRY_DATA];
+    /* MSIs on Xen can be remapped into pirqs. In those cases, masking
+     * and unmasking go through the PV evtchn path. */
+    if (xen_enabled() && xen_is_pirq_msi(pci_get_long(data))) {
+        return false;
+    }
+    return fmask || dev->msix_table[offset + PCI_MSIX_ENTRY_VECTOR_CTRL] &
+        PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
 bool msix_is_masked(PCIDevice *dev, unsigned int vector)
@@ -200,8 +209,14 @@ static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
     return pci_get_long(dev->msix_pba + addr);
 }
 
+static void msix_pba_mmio_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
+{
+}
+
 static const MemoryRegionOps msix_pba_mmio_ops = {
     .read = msix_pba_mmio_read,
+    .write = msix_pba_mmio_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
@@ -234,7 +249,7 @@ int msix_init(struct PCIDevice *dev, unsigned short nentries,
     uint8_t *config;
 
     /* Nothing to do if MSI is not supported by interrupt controller */
-    if (!msi_supported) {
+    if (!msi_nonbroken) {
         return -ENOTSUP;
     }
 
@@ -314,9 +329,7 @@ int msix_init_exclusive_bar(PCIDevice *dev, unsigned short nentries,
         bar_size = bar_pba_offset + bar_pba_size;
     }
 
-    if (bar_size & (bar_size - 1)) {
-        bar_size = 1 << qemu_fls(bar_size);
-    }
+    bar_size = pow2ceil(bar_size);
 
     name = g_strdup_printf("%s-msix", dev->name);
     memory_region_init(&dev->msix_exclusive_bar, OBJECT(dev), name, bar_size);
diff --git a/qemu/hw/pci/pci-stub.c b/qemu/hw/pci/pci-stub.c
index 063a7c242..36d2c430c 100644
--- a/qemu/hw/pci/pci-stub.c
+++ b/qemu/hw/pci/pci-stub.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
 #include "qapi/qmp/qerror.h"
diff --git a/qemu/hw/pci/pci.c b/qemu/hw/pci/pci.c
index a017614d4..bb605efae 100644
--- a/qemu/hw/pci/pci.c
+++ b/qemu/hw/pci/pci.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bridge.h"
@@ -38,6 +39,8 @@
 #include "hw/pci/msix.h"
 #include "exec/address-spaces.h"
 #include "hw/hotplug.h"
+#include "hw/boards.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -276,9 +279,9 @@ static void pcibus_reset(BusState *qbus)
     }
 }
 
-static void pci_host_bus_register(PCIBus *bus, DeviceState *parent)
+static void pci_host_bus_register(DeviceState *host)
 {
-    PCIHostState *host_bridge = PCI_HOST_BRIDGE(parent);
+    PCIHostState *host_bridge = PCI_HOST_BRIDGE(host);
 
     QLIST_INSERT_HEAD(&pci_host_bridges, host_bridge, next);
 }
@@ -329,7 +332,6 @@ const char *pci_root_bus_path(PCIDevice *dev)
 }
 
 static void pci_bus_init(PCIBus *bus, DeviceState *parent,
-                         const char *name,
                          MemoryRegion *address_space_mem,
                          MemoryRegion *address_space_io,
                          uint8_t devfn_min)
@@ -342,7 +344,7 @@ static void pci_bus_init(PCIBus *bus, DeviceState *parent,
     /* host bridge */
     QLIST_INIT(&bus->child);
 
-    pci_host_bus_register(bus, parent);
+    pci_host_bus_register(parent);
 }
 
 bool pci_bus_is_express(PCIBus *bus)
@@ -362,8 +364,7 @@ void pci_bus_new_inplace(PCIBus *bus, size_t bus_size, DeviceState *parent,
                          uint8_t devfn_min, const char *typename)
 {
     qbus_create_inplace(bus, bus_size, typename, parent, name);
-    pci_bus_init(bus, parent, name, address_space_mem,
-                 address_space_io, devfn_min);
+    pci_bus_init(bus, parent, address_space_mem, address_space_io, devfn_min);
 }
 
 PCIBus *pci_bus_new(DeviceState *parent, const char *name,
@@ -374,8 +375,7 @@ PCIBus *pci_bus_new(DeviceState *parent, const char *name,
     PCIBus *bus;
 
     bus = PCI_BUS(qbus_create(typename, parent, name));
-    pci_bus_init(bus, parent, name, address_space_mem,
-                 address_space_io, devfn_min);
+    pci_bus_init(bus, parent, address_space_mem, address_space_io, devfn_min);
     return bus;
 }
 
@@ -846,6 +846,16 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
     PCIConfigWriteFunc *config_write = pc->config_write;
     Error *local_err = NULL;
     AddressSpace *dma_as;
+    DeviceState *dev = DEVICE(pci_dev);
+
+    pci_dev->bus = bus;
+    /* Only pci bridges can be attached to extra PCI root buses */
+    if (pci_bus_is_root(bus) && bus->parent_dev && !pc->is_bridge) {
+        error_setg(errp,
+                   "PCI: Only PCI/PCIe bridges can be plugged into %s",
+                    bus->parent_dev->name);
+        return NULL;
+    }
 
     if (devfn < 0) {
         for(devfn = bus->devfn_min ; devfn < ARRAY_SIZE(bus->devices);
@@ -863,9 +873,17 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
                    PCI_SLOT(devfn), PCI_FUNC(devfn), name,
                    bus->devices[devfn]->name);
         return NULL;
+    } else if (dev->hotplugged &&
+               pci_get_function_0(pci_dev)) {
+        error_setg(errp, "PCI: slot %d function 0 already ocuppied by %s,"
+                   " new func %s cannot be exposed to guest.",
+                   PCI_SLOT(devfn),
+                   bus->devices[PCI_DEVFN(PCI_SLOT(devfn), 0)]->name,
+                   name);
+
+       return NULL;
     }
 
-    pci_dev->bus = bus;
     pci_dev->devfn = devfn;
     dma_as = pci_device_iommu_address_space(pci_dev);
 
@@ -1065,6 +1083,10 @@ static pcibus_t pci_bar_address(PCIDevice *d,
     pcibus_t new_addr, last_addr;
     int bar = pci_bar(d, reg);
     uint16_t cmd = pci_get_word(d->config + PCI_COMMAND);
+    Object *machine = qdev_get_machine();
+    ObjectClass *oc = object_get_class(machine);
+    MachineClass *mc = MACHINE_CLASS(oc);
+    bool allow_0_address = mc->pci_allow_0_address;
 
     if (type & PCI_BASE_ADDRESS_SPACE_IO) {
         if (!(cmd & PCI_COMMAND_IO)) {
@@ -1075,7 +1097,8 @@ static pcibus_t pci_bar_address(PCIDevice *d,
         /* Check if 32 bit BAR wraps around explicitly.
          * TODO: make priorities correct and remove this work around.
          */
-        if (last_addr <= new_addr || new_addr == 0 || last_addr >= UINT32_MAX) {
+        if (last_addr <= new_addr || last_addr >= UINT32_MAX ||
+            (!allow_0_address && new_addr == 0)) {
             return PCI_BAR_UNMAPPED;
         }
         return new_addr;
@@ -1099,8 +1122,8 @@ static pcibus_t pci_bar_address(PCIDevice *d,
     /* XXX: as we cannot support really dynamic
        mappings, we handle specific values as invalid
        mappings. */
-    if (last_addr <= new_addr || new_addr == 0 ||
-        last_addr == PCI_BAR_UNMAPPED) {
+    if (last_addr <= new_addr || last_addr == PCI_BAR_UNMAPPED ||
+        (!allow_0_address && new_addr == 0)) {
         return PCI_BAR_UNMAPPED;
     }
 
@@ -1148,16 +1171,16 @@ static void pci_update_mappings(PCIDevice *d)
         /* now do the real mapping */
         if (r->addr != PCI_BAR_UNMAPPED) {
             trace_pci_update_mappings_del(d, pci_bus_num(d->bus),
-                                          PCI_FUNC(d->devfn),
                                           PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
                                           i, r->addr, r->size);
             memory_region_del_subregion(r->address_space, r->memory);
         }
         r->addr = new_addr;
         if (r->addr != PCI_BAR_UNMAPPED) {
             trace_pci_update_mappings_add(d, pci_bus_num(d->bus),
-                                          PCI_FUNC(d->devfn),
                                           PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
                                           i, r->addr, r->size);
             memory_region_add_subregion_overlap(r->address_space,
                                                 r->addr, r->memory, 1);
@@ -2065,9 +2088,7 @@ static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
         g_free(path);
         return;
     }
-    if (size & (size - 1)) {
-        size = 1 << qemu_fls(size);
-    }
+    size = pow2ceil(size);
 
     vmsd = qdev_get_vmsd(DEVICE(pdev));
 
@@ -2077,7 +2098,7 @@ static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
         snprintf(name, sizeof(name), "%s.rom", object_get_typename(OBJECT(pdev)));
     }
     pdev->has_rom = true;
-    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_abort);
+    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_fatal);
     vmstate_register_ram(&pdev->rom, &pdev->qdev);
     ptr = memory_region_get_ram_ptr(&pdev->rom);
     load_image(path, ptr);
@@ -2379,17 +2400,14 @@ static void pci_device_class_init(ObjectClass *klass, void *data)
 AddressSpace *pci_device_iommu_address_space(PCIDevice *dev)
 {
     PCIBus *bus = PCI_BUS(dev->bus);
+    PCIBus *iommu_bus = bus;
 
-    if (bus->iommu_fn) {
-        return bus->iommu_fn(bus, bus->iommu_opaque, dev->devfn);
+    while(iommu_bus && !iommu_bus->iommu_fn && iommu_bus->parent_dev) {
+        iommu_bus = PCI_BUS(iommu_bus->parent_dev->bus);
     }
-
-    if (bus->parent_dev) {
-        /** We are ignoring the bus master DMA bit of the bridge
-         *  as it would complicate things such as VFIO for no good reason */
-        return pci_device_iommu_address_space(bus->parent_dev);
+    if (iommu_bus && iommu_bus->iommu_fn) {
+        return iommu_bus->iommu_fn(bus, iommu_bus->iommu_opaque, dev->devfn);
     }
-
     return &address_space_memory;
 }
 
@@ -2453,6 +2471,33 @@ void pci_bus_get_w64_range(PCIBus *bus, Range *range)
     pci_for_each_device_under_bus(bus, pci_dev_get_w64, range);
 }
 
+static bool pcie_has_upstream_port(PCIDevice *dev)
+{
+    PCIDevice *parent_dev = pci_bridge_get_device(dev->bus);
+
+    /* Device associated with an upstream port.
+     * As there are several types of these, it's easier to check the
+     * parent device: upstream ports are always connected to
+     * root or downstream ports.
+     */
+    return parent_dev &&
+        pci_is_express(parent_dev) &&
+        parent_dev->exp.exp_cap &&
+        (pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_ROOT_PORT ||
+         pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_DOWNSTREAM);
+}
+
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev)
+{
+    if(pcie_has_upstream_port(pci_dev)) {
+        /* With an upstream PCIe port, we only support 1 device at slot 0 */
+        return pci_dev->bus->devices[0];
+    } else {
+        /* Other bus types might support multiple devices at slots 0-31 */
+        return pci_dev->bus->devices[PCI_DEVFN(PCI_SLOT(pci_dev->devfn), 0)];
+    }
+}
+
 static const TypeInfo pci_device_type_info = {
     .name = TYPE_PCI_DEVICE,
     .parent = TYPE_DEVICE,
diff --git a/qemu/hw/pci/pci_bridge.c b/qemu/hw/pci/pci_bridge.c
index 40c97b155..3cf30bd33 100644
--- a/qemu/hw/pci/pci_bridge.c
+++ b/qemu/hw/pci/pci_bridge.c
@@ -29,6 +29,7 @@
  *                    VA Linux Systems Japan K.K.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_bus.h"
 #include "qemu/range.h"
@@ -332,7 +333,7 @@ void pci_bridge_reset(DeviceState *qdev)
 }
 
 /* default qdev initialization function for PCI-to-PCI bridge */
-int pci_bridge_initfn(PCIDevice *dev, const char *typename)
+void pci_bridge_initfn(PCIDevice *dev, const char *typename)
 {
     PCIBus *parent = dev->bus;
     PCIBridge *br = PCI_BRIDGE(dev);
@@ -378,7 +379,6 @@ int pci_bridge_initfn(PCIDevice *dev, const char *typename)
     br->windows = pci_bridge_region_init(br);
     QLIST_INIT(&sec_bus->child);
     QLIST_INSERT_HEAD(&parent->child, sec_bus, sibling);
-    return 0;
 }
 
 /* default qdev clean up function for PCI-to-PCI bridge */
diff --git a/qemu/hw/pci/pci_host.c b/qemu/hw/pci/pci_host.c
index 3e26f9256..5eaa935cb 100644
--- a/qemu/hw/pci/pci_host.c
+++ b/qemu/hw/pci/pci_host.c
@@ -18,8 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
+#include "hw/pci/pci_bus.h"
 #include "trace.h"
 
 /* debug PCI */
@@ -52,6 +54,13 @@ void pci_host_config_write_common(PCIDevice *pci_dev, uint32_t addr,
                                   uint32_t limit, uint32_t val, uint32_t len)
 {
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return;
+    }
+
     trace_pci_cfg_write(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                         PCI_FUNC(pci_dev->devfn), addr, val);
     pci_dev->config_write(pci_dev, addr, val, MIN(len, limit - addr));
@@ -63,6 +72,13 @@ uint32_t pci_host_config_read_common(PCIDevice *pci_dev, uint32_t addr,
     uint32_t ret;
 
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return ~0x0;
+    }
+
     ret = pci_dev->config_read(pci_dev, addr, MIN(len, limit - addr));
     trace_pci_cfg_read(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                        PCI_FUNC(pci_dev->devfn), addr, ret);
diff --git a/qemu/hw/pci/pcie.c b/qemu/hw/pci/pcie.c
index 6e28985bd..728386ada 100644
--- a/qemu/hw/pci/pcie.c
+++ b/qemu/hw/pci/pcie.c
@@ -18,6 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pcie.h"
@@ -249,25 +251,43 @@ void pcie_cap_slot_hotplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
         return;
     }
 
-    /* TODO: multifunction hot-plug.
-     * Right now, only a device of function = 0 is allowed to be
-     * hot plugged/unplugged.
+    /* To enable multifunction hot-plug, we just ensure the function
+     * 0 added last. When function 0 is added, we set the sltsta and
+     * inform OS via event notification.
      */
-    assert(PCI_FUNC(pci_dev->devfn) == 0);
+    if (pci_get_function_0(pci_dev)) {
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+                                   PCI_EXP_SLTSTA_PDS);
+        pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
+                            PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+    }
+}
 
-    pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
-                               PCI_EXP_SLTSTA_PDS);
-    pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
-                        PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
+{
+    object_unparent(OBJECT(dev));
 }
 
 void pcie_cap_slot_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
                                          DeviceState *dev, Error **errp)
 {
     uint8_t *exp_cap;
+    PCIDevice *pci_dev = PCI_DEVICE(dev);
+    PCIBus *bus = pci_dev->bus;
 
     pcie_cap_slot_hotplug_common(PCI_DEVICE(hotplug_dev), dev, &exp_cap, errp);
 
+    /* In case user cancel the operation of multi-function hot-add,
+     * remove the function that is unexposed to guest individually,
+     * without interaction with guest.
+     */
+    if (pci_dev->devfn &&
+        !bus->devices[0]) {
+        pcie_unplug_device(bus, pci_dev, NULL);
+
+        return;
+    }
+
     pcie_cap_slot_push_attention_button(PCI_DEVICE(hotplug_dev));
 }
 
@@ -378,11 +398,6 @@ void pcie_cap_slot_reset(PCIDevice *dev)
     hotplug_event_update_event_status(dev);
 }
 
-static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
-{
-    object_unparent(OBJECT(dev));
-}
-
 void pcie_cap_slot_write_config(PCIDevice *dev,
                                 uint32_t addr, uint32_t val, int len)
 {
@@ -413,13 +428,13 @@ void pcie_cap_slot_write_config(PCIDevice *dev,
      */
     if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
         ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
-            PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
-            pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
-                                pcie_unplug_device, NULL);
+        PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
+        pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                            pcie_unplug_device, NULL);
 
-            pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
-                                         PCI_EXP_SLTSTA_PDS);
-            pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+        pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
+                                     PCI_EXP_SLTSTA_PDS);
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
                                        PCI_EXP_SLTSTA_PDC);
     }
 
@@ -594,7 +609,7 @@ void pcie_add_capability(PCIDevice *dev,
 
     assert(offset >= PCI_CONFIG_SPACE_SIZE);
     assert(offset < offset + size);
-    assert(offset + size < PCIE_CONFIG_SPACE_SIZE);
+    assert(offset + size <= PCIE_CONFIG_SPACE_SIZE);
     assert(size >= 8);
     assert(pci_is_express(dev));
 
diff --git a/qemu/hw/pci/pcie_aer.c b/qemu/hw/pci/pcie_aer.c
index f1847ac21..e2d4e68ba 100644
--- a/qemu/hw/pci/pcie_aer.c
+++ b/qemu/hw/pci/pcie_aer.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "qapi/qmp/types.h"
 #include "monitor/monitor.h"
@@ -94,12 +95,12 @@ static void aer_log_clear_all_err(PCIEAERLog *aer_log)
     aer_log->log_num = 0;
 }
 
-int pcie_aer_init(PCIDevice *dev, uint16_t offset)
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size)
 {
     PCIExpressDevice *exp;
 
     pcie_add_capability(dev, PCI_EXT_CAP_ID_ERR, PCI_ERR_VER,
-                        offset, PCI_ERR_SIZEOF);
+                        offset, size);
     exp = &dev->exp;
     exp->aer_cap = offset;
 
@@ -370,7 +371,7 @@ static void pcie_aer_msg_root_port(PCIDevice *dev, const PCIEAERMsg *msg)
  *
  * Walk up the bus tree from the device, propagate the error message.
  */
-static void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
 {
     uint8_t type;
 
@@ -827,10 +828,6 @@ typedef struct PCIEAERErrorName {
  */
 static const struct PCIEAERErrorName pcie_aer_error_list[] = {
     {
-        .name = "TRAIN",
-        .val = PCI_ERR_UNC_TRAIN,
-        .correctable = false,
-    }, {
         .name = "DLP",
         .val = PCI_ERR_UNC_DLP,
         .correctable = false,
@@ -983,7 +980,7 @@ static int do_pcie_aer_inject_error(Monitor *mon,
         }
     }
     err.status = error_status;
-    err.source_id = (pci_bus_num(dev->bus) << 8) | dev->devfn;
+    err.source_id = pci_requester_id(dev);
 
     err.flags = 0;
     if (correctable) {
diff --git a/qemu/hw/pci/pcie_host.c b/qemu/hw/pci/pcie_host.c
index d8afba863..dcebf57ed 100644
--- a/qemu/hw/pci/pcie_host.c
+++ b/qemu/hw/pci/pcie_host.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pcie_host.h"
diff --git a/qemu/hw/pci/pcie_port.c b/qemu/hw/pci/pcie_port.c
index 40ca8d5d1..6432b9ac1 100644
--- a/qemu/hw/pci/pcie_port.c
+++ b/qemu/hw/pci/pcie_port.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pcie_port.h"
 #include "hw/hotplug.h"
 
diff --git a/qemu/hw/pci/shpc.c b/qemu/hw/pci/shpc.c
index bfb4d31b6..3dcd472eb 100644
--- a/qemu/hw/pci/shpc.c
+++ b/qemu/hw/pci/shpc.c
@@ -1,6 +1,6 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
-#include <strings.h>
-#include <stdint.h>
 #include "qemu/range.h"
 #include "qemu/error-report.h"
 #include "hw/pci/shpc.h"
diff --git a/qemu/hw/pci/slotid_cap.c b/qemu/hw/pci/slotid_cap.c
index 1c01d346c..aec1e9166 100644
--- a/qemu/hw/pci/slotid_cap.c
+++ b/qemu/hw/pci/slotid_cap.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/pci/slotid_cap.h"
 #include "hw/pci/pci.h"
 #include "qemu/error-report.h"
diff --git a/qemu/hw/pcmcia/pcmcia.c b/qemu/hw/pcmcia/pcmcia.c
index 78efe5a67..195672186 100644
--- a/qemu/hw/pcmcia/pcmcia.c
+++ b/qemu/hw/pcmcia/pcmcia.c
@@ -4,6 +4,7 @@
  * Copyright 2013 SUSE LINUX Products GmbH
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/pcmcia.h"
diff --git a/qemu/hw/pcmcia/pxa2xx.c b/qemu/hw/pcmcia/pxa2xx.c
index a7e187743..20c9c753d 100644
--- a/qemu/hw/pcmcia/pxa2xx.c
+++ b/qemu/hw/pcmcia/pxa2xx.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/pcmcia.h"
@@ -163,7 +164,7 @@ static void pxa2xx_pcmcia_initfn(Object *obj)
     sysbus_init_mmio(sbd, &s->container_mem);
 
     /* Socket I/O Memory Space */
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_pcmcia_io_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_pcmcia_io_ops, s,
                           "pxa2xx-pcmcia-io", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x00000000,
                                 &s->iomem);
@@ -171,13 +172,13 @@ static void pxa2xx_pcmcia_initfn(Object *obj)
     /* Then next 64 MB is reserved */
 
     /* Socket Attribute Memory Space */
-    memory_region_init_io(&s->attr_iomem, NULL, &pxa2xx_pcmcia_attr_ops, s,
+    memory_region_init_io(&s->attr_iomem, obj, &pxa2xx_pcmcia_attr_ops, s,
                           "pxa2xx-pcmcia-attribute", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x08000000,
                                 &s->attr_iomem);
 
     /* Socket Common Memory Space */
-    memory_region_init_io(&s->common_iomem, NULL, &pxa2xx_pcmcia_common_ops, s,
+    memory_region_init_io(&s->common_iomem, obj, &pxa2xx_pcmcia_common_ops, s,
                           "pxa2xx-pcmcia-common", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x0c000000,
                                 &s->common_iomem);
diff --git a/qemu/hw/ppc/Makefile.objs b/qemu/hw/ppc/Makefile.objs
index c8ab06e7f..c1ffc7771 100644
--- a/qemu/hw/ppc/Makefile.objs
+++ b/qemu/hw/ppc/Makefile.objs
@@ -3,7 +3,7 @@ obj-y += ppc.o ppc_booke.o
 # IBM pSeries (sPAPR)
 obj-$(CONFIG_PSERIES) += spapr.o spapr_vio.o spapr_events.o
 obj-$(CONFIG_PSERIES) += spapr_hcall.o spapr_iommu.o spapr_rtas.o
-obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o
+obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o spapr_rng.o
 ifeq ($(CONFIG_PCI)$(CONFIG_PSERIES)$(CONFIG_LINUX), yyy)
 obj-y += spapr_pci_vfio.o
 endif
diff --git a/qemu/hw/ppc/e500.c b/qemu/hw/ppc/e500.c
index d300846c3..ee1c60b82 100644
--- a/qemu/hw/ppc/e500.c
+++ b/qemu/hw/ppc/e500.c
@@ -14,7 +14,8 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "e500-ccsr.h"
@@ -751,8 +752,8 @@ static qemu_irq *ppce500_init_mpic(MachineState *machine, PPCE500Params *params,
             dev = ppce500_init_mpic_kvm(params, irqs, &err);
         }
         if (machine_kernel_irqchip_required(machine) && !dev) {
-            error_report("kernel_irqchip requested but unavailable: %s",
-                         error_get_pretty(err));
+            error_reportf_err(err,
+                              "kernel_irqchip requested but unavailable: ");
             exit(1);
         }
     }
@@ -1017,7 +1018,7 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
 
     bios_size = load_elf(filename, NULL, NULL, &bios_entry, &loadaddr, NULL,
-                         1, ELF_MACHINE, 0);
+                         1, PPC_ELF_MACHINE, 0, 0);
     if (bios_size < 0) {
         /*
          * Hrm. No ELF image? Try a uImage, maybe someone is giving us an
@@ -1048,10 +1049,6 @@ void ppce500_init(MachineState *machine, PPCE500Params *params)
     boot_info->entry = bios_entry;
     boot_info->dt_base = dt_base;
     boot_info->dt_size = dt_size;
-
-    if (kvm_enabled()) {
-        kvmppc_init();
-    }
 }
 
 static int e500_ccsr_initfn(SysBusDevice *dev)
diff --git a/qemu/hw/ppc/e500plat.c b/qemu/hw/ppc/e500plat.c
index 14b14eaa7..b00565c3d 100644
--- a/qemu/hw/ppc/e500plat.c
+++ b/qemu/hw/ppc/e500plat.c
@@ -9,7 +9,7 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "hw/boards.h"
@@ -57,17 +57,12 @@ static void e500plat_init(MachineState *machine)
     ppce500_init(machine, &params);
 }
 
-static QEMUMachine e500plat_machine = {
-    .name = "ppce500",
-    .desc = "generic paravirt e500 platform",
-    .init = e500plat_init,
-    .max_cpus = 32,
-    .has_dynamic_sysbus = true,
-};
-
-static void e500plat_machine_init(void)
+static void e500plat_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&e500plat_machine);
+    mc->desc = "generic paravirt e500 platform";
+    mc->init = e500plat_init;
+    mc->max_cpus = 32;
+    mc->has_dynamic_sysbus = true;
 }
 
-machine_init(e500plat_machine_init);
+DEFINE_MACHINE("ppce500", e500plat_machine_init)
diff --git a/qemu/hw/ppc/mac.h b/qemu/hw/ppc/mac.h
index 8bdba30c1..5764b86c2 100644
--- a/qemu/hw/ppc/mac.h
+++ b/qemu/hw/ppc/mac.h
@@ -103,11 +103,16 @@ typedef struct CUDAState {
     uint8_t last_b;
     uint8_t last_acr;
 
+    /* MacOS 9 is racy and requires a delay upon setting the SR_INT bit */
+    QEMUTimer *sr_delay_timer;
+
     int data_in_size;
     int data_in_index;
     int data_out_index;
 
     qemu_irq irq;
+    uint16_t adb_poll_mask;
+    uint8_t autopoll_rate_ms;
     uint8_t autopoll;
     uint8_t data_in[128];
     uint8_t data_out[16];
@@ -131,7 +136,6 @@ typedef struct MACIOIDEState {
 
     MemoryRegion mem;
     IDEBus bus;
-    BlockAIOCB *aiocb;
     IDEDMA dma;
     void *dbdma;
     bool dma_active;
diff --git a/qemu/hw/ppc/mac_newworld.c b/qemu/hw/ppc/mac_newworld.c
index 77d5c819e..32e88b378 100644
--- a/qemu/hw/ppc/mac_newworld.c
+++ b/qemu/hw/ppc/mac_newworld.c
@@ -46,6 +46,8 @@
  * 0001:05:0c.0 IDE interface [0101]: Broadcom K2 SATA [1166:0240]
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/mac.h"
@@ -62,12 +64,14 @@
 #include "hw/ide.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "hw/usb.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/sysbus.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -207,7 +211,7 @@ static void ppc_core99_init(MachineState *machine)
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_core99.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -219,14 +223,14 @@ static void ppc_core99_init(MachineState *machine)
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, NULL, NULL, NULL,
-                             NULL, NULL, 1, ELF_MACHINE, 0);
+                             NULL, NULL, 1, PPC_ELF_MACHINE, 0, 0);
 
         g_free(filename);
     } else {
         bios_size = -1;
     }
     if (bios_size < 0 || bios_size > BIOS_SIZE) {
-        hw_error("qemu: could not load PowerPC bios '%s'\n", bios_name);
+        error_report("could not load PowerPC bios '%s'", bios_name);
         exit(1);
     }
 
@@ -242,7 +246,8 @@ static void ppc_core99_init(MachineState *machine)
         kernel_base = KERNEL_LOAD_ADDR;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -252,7 +257,7 @@ static void ppc_core99_init(MachineState *machine)
                                               kernel_base,
                                               ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -261,8 +266,8 @@ static void ppc_core99_init(MachineState *machine)
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
             cmdline_base = round_page(initrd_base + initrd_size);
@@ -344,7 +349,7 @@ static void ppc_core99_init(MachineState *machine)
             break;
 #endif /* defined(TARGET_PPC64) */
         default:
-            hw_error("Bus model not supported on mac99 machine\n");
+            error_report("Bus model not supported on mac99 machine");
             exit(1);
         }
     }
@@ -371,12 +376,13 @@ static void ppc_core99_init(MachineState *machine)
         /* 970 gets a U3 bus */
         pci_bus = pci_pmac_u3_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99_U3;
-        machine->usb |= defaults_enabled() && !machine->usb_disabled;
     } else {
         pci_bus = pci_pmac_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99;
     }
 
+    machine->usb |= defaults_enabled() && !machine->usb_disabled;
+
     /* Timebase Frequency */
     if (kvm_enabled()) {
         tbfreq = kvmppc_get_tbfreq();
@@ -508,7 +514,6 @@ static void core99_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
 
-    mc->name = "mac99";
     mc->desc = "Mac99 based PowerMAC";
     mc->init = ppc_core99_init;
     mc->max_cpus = MAX_CPUS;
@@ -517,7 +522,7 @@ static void core99_machine_class_init(ObjectClass *oc, void *data)
 }
 
 static const TypeInfo core99_machine_info = {
-    .name          = "mac99-machine",
+    .name          = MACHINE_TYPE_NAME("mac99"),
     .parent        = TYPE_MACHINE,
     .class_init    = core99_machine_class_init,
 };
diff --git a/qemu/hw/ppc/mac_oldworld.c b/qemu/hw/ppc/mac_oldworld.c
index 06fdbaf58..a9bb1c27d 100644
--- a/qemu/hw/ppc/mac_oldworld.c
+++ b/qemu/hw/ppc/mac_oldworld.c
@@ -23,6 +23,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "mac.h"
@@ -38,10 +40,12 @@
 #include "hw/ide.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -135,7 +139,7 @@ static void ppc_heathrow_init(MachineState *machine)
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_heathrow.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -147,13 +151,13 @@ static void ppc_heathrow_init(MachineState *machine)
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, 0, NULL, NULL, NULL, NULL,
-                             1, ELF_MACHINE, 0);
+                             1, PPC_ELF_MACHINE, 0, 0);
         g_free(filename);
     } else {
         bios_size = -1;
     }
     if (bios_size < 0 || bios_size > BIOS_SIZE) {
-        hw_error("qemu: could not load PowerPC bios '%s'\n", bios_name);
+        error_report("could not load PowerPC bios '%s'", bios_name);
         exit(1);
     }
 
@@ -168,7 +172,8 @@ static void ppc_heathrow_init(MachineState *machine)
 #endif
         kernel_base = KERNEL_LOAD_ADDR;
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -178,8 +183,7 @@ static void ppc_heathrow_init(MachineState *machine)
                                               kernel_base,
                                               ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n",
-                      kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -188,8 +192,8 @@ static void ppc_heathrow_init(MachineState *machine)
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
             cmdline_base = round_page(initrd_base + initrd_size);
@@ -246,7 +250,8 @@ static void ppc_heathrow_init(MachineState *machine)
                 ((qemu_irq *)env->irq_inputs)[PPC6xx_INPUT_INT];
             break;
         default:
-            hw_error("Bus model not supported on OldWorld Mac machine\n");
+            error_report("Bus model not supported on OldWorld Mac machine");
+            exit(1);
         }
     }
 
@@ -259,7 +264,8 @@ static void ppc_heathrow_init(MachineState *machine)
 
     /* init basic PC hardware */
     if (PPC_INPUT(env) != PPC_FLAGS_INPUT_6xx) {
-        hw_error("Only 6xx bus is supported on heathrow machine\n");
+        error_report("Only 6xx bus is supported on heathrow machine");
+        exit(1);
     }
     pic = heathrow_pic_init(&pic_mem, 1, heathrow_irqs);
     pci_bus = pci_grackle_init(0xfec00000, pic,
@@ -357,21 +363,17 @@ static int heathrow_kvm_type(const char *arg)
     return 2;
 }
 
-static QEMUMachine heathrow_machine = {
-    .name = "g3beige",
-    .desc = "Heathrow based PowerMAC",
-    .init = ppc_heathrow_init,
-    .max_cpus = MAX_CPUS,
+static void heathrow_machine_init(MachineClass *mc)
+{
+    mc->desc = "Heathrow based PowerMAC";
+    mc->init = ppc_heathrow_init;
+    mc->max_cpus = MAX_CPUS;
 #ifndef TARGET_PPC64
-    .is_default = 1,
+    mc->is_default = 1;
 #endif
-    .default_boot_order = "cd", /* TOFIX "cad" when Mac floppy is implemented */
-    .kvm_type = heathrow_kvm_type,
-};
-
-static void heathrow_machine_init(void)
-{
-    qemu_register_machine(&heathrow_machine);
+    /* TOFIX "cad" when Mac floppy is implemented */
+    mc->default_boot_order = "cd";
+    mc->kvm_type = heathrow_kvm_type;
 }
 
-machine_init(heathrow_machine_init);
+DEFINE_MACHINE("g3beige", heathrow_machine_init)
diff --git a/qemu/hw/ppc/mpc8544_guts.c b/qemu/hw/ppc/mpc8544_guts.c
index a10abe978..ba69178d6 100644
--- a/qemu/hw/ppc/mpc8544_guts.c
+++ b/qemu/hw/ppc/mpc8544_guts.c
@@ -17,6 +17,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
diff --git a/qemu/hw/ppc/mpc8544ds.c b/qemu/hw/ppc/mpc8544ds.c
index 3a3b141e4..27b828901 100644
--- a/qemu/hw/ppc/mpc8544ds.c
+++ b/qemu/hw/ppc/mpc8544ds.c
@@ -9,7 +9,7 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "hw/boards.h"
@@ -50,16 +50,11 @@ static void mpc8544ds_init(MachineState *machine)
 }
 
 
-static QEMUMachine ppce500_machine = {
-    .name = "mpc8544ds",
-    .desc = "mpc8544ds",
-    .init = mpc8544ds_init,
-    .max_cpus = 15,
-};
-
-static void ppce500_machine_init(void)
+static void ppce500_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ppce500_machine);
+    mc->desc = "mpc8544ds";
+    mc->init = mpc8544ds_init;
+    mc->max_cpus = 15;
 }
 
-machine_init(ppce500_machine_init);
+DEFINE_MACHINE("mpc8544ds", ppce500_machine_init)
diff --git a/qemu/hw/ppc/ppc.c b/qemu/hw/ppc/ppc.c
index b77e30357..38ff2e159 100644
--- a/qemu/hw/ppc/ppc.c
+++ b/qemu/hw/ppc/ppc.c
@@ -21,6 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc_e500.h"
@@ -462,7 +465,7 @@ void ppce500_set_mpic_proxy(bool enabled)
 uint64_t cpu_ppc_get_tb(ppc_tb_t *tb_env, uint64_t vmclk, int64_t tb_offset)
 {
     /* TB time in tb periods */
-    return muldiv64(vmclk, tb_env->tb_freq, get_ticks_per_sec()) + tb_offset;
+    return muldiv64(vmclk, tb_env->tb_freq, NANOSECONDS_PER_SECOND) + tb_offset;
 }
 
 uint64_t cpu_ppc_load_tbl (CPUPPCState *env)
@@ -503,7 +506,9 @@ uint32_t cpu_ppc_load_tbu (CPUPPCState *env)
 static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,
                                     int64_t *tb_offsetp, uint64_t value)
 {
-    *tb_offsetp = value - muldiv64(vmclk, tb_env->tb_freq, get_ticks_per_sec());
+    *tb_offsetp = value -
+        muldiv64(vmclk, tb_env->tb_freq, NANOSECONDS_PER_SECOND);
+
     LOG_TB("%s: tb %016" PRIx64 " offset %08" PRIx64 "\n",
                 __func__, value, *tb_offsetp);
 }
@@ -637,11 +642,11 @@ static inline uint32_t _cpu_ppc_load_decr(CPUPPCState *env, uint64_t next)
 
     diff = next - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     if (diff >= 0) {
-        decr = muldiv64(diff, tb_env->decr_freq, get_ticks_per_sec());
+        decr = muldiv64(diff, tb_env->decr_freq, NANOSECONDS_PER_SECOND);
     } else if (tb_env->flags & PPC_TIMER_BOOKE) {
         decr = 0;
     }  else {
-        decr = -muldiv64(-diff, tb_env->decr_freq, get_ticks_per_sec());
+        decr = -muldiv64(-diff, tb_env->decr_freq, NANOSECONDS_PER_SECOND);
     }
     LOG_TB("%s: %08" PRIx32 "\n", __func__, decr);
 
@@ -673,7 +678,8 @@ uint64_t cpu_ppc_load_purr (CPUPPCState *env)
 
     diff = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - tb_env->purr_start;
 
-    return tb_env->purr_load + muldiv64(diff, tb_env->tb_freq, get_ticks_per_sec());
+    return tb_env->purr_load +
+        muldiv64(diff, tb_env->tb_freq, NANOSECONDS_PER_SECOND);
 }
 
 /* When decrementer expires,
@@ -749,7 +755,7 @@ static void __cpu_ppc_store_decr(PowerPCCPU *cpu, uint64_t *nextp,
 
     /* Calculate the next timer event */
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    next = now + muldiv64(value, get_ticks_per_sec(), tb_env->decr_freq);
+    next = now + muldiv64(value, NANOSECONDS_PER_SECOND, tb_env->decr_freq);
     *nextp = next;
 
     /* Adjust timer */
@@ -834,7 +840,7 @@ static void cpu_ppc_set_tb_clk (void *opaque, uint32_t freq)
 static void timebase_pre_save(void *opaque)
 {
     PPCTimebase *tb = opaque;
-    uint64_t ticks = cpu_get_real_ticks();
+    uint64_t ticks = cpu_get_host_ticks();
     PowerPCCPU *first_ppc_cpu = POWERPC_CPU(first_cpu);
 
     if (!first_ppc_cpu->env.tb_env) {
@@ -878,7 +884,7 @@ static int timebase_post_load(void *opaque, int version_id)
                                      NANOSECONDS_PER_SECOND);
     guest_tb = tb_remote->guest_timebase + MIN(0, migration_duration_tb);
 
-    tb_off_adj = guest_tb - cpu_get_real_ticks();
+    tb_off_adj = guest_tb - cpu_get_host_ticks();
 
     tb_off = first_ppc_cpu->env.tb_env->tb_offset;
     trace_ppc_tb_adjust(tb_off, tb_off_adj, tb_off_adj - tb_off,
@@ -1010,7 +1016,7 @@ static void cpu_4xx_fit_cb (void *opaque)
         /* Cannot occur, but makes gcc happy */
         return;
     }
-    next = now + muldiv64(next, get_ticks_per_sec(), tb_env->tb_freq);
+    next = now + muldiv64(next, NANOSECONDS_PER_SECOND, tb_env->tb_freq);
     if (next == now)
         next++;
     timer_mod(ppc40x_timer->fit_timer, next);
@@ -1041,7 +1047,7 @@ static void start_stop_pit (CPUPPCState *env, ppc_tb_t *tb_env, int is_excp)
                     __func__, ppc40x_timer->pit_reload);
         now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         next = now + muldiv64(ppc40x_timer->pit_reload,
-                              get_ticks_per_sec(), tb_env->decr_freq);
+                              NANOSECONDS_PER_SECOND, tb_env->decr_freq);
         if (is_excp)
             next += tb_env->decr_next - now;
         if (next == now)
@@ -1106,7 +1112,7 @@ static void cpu_4xx_wdt_cb (void *opaque)
         /* Cannot occur, but makes gcc happy */
         return;
     }
-    next = now + muldiv64(next, get_ticks_per_sec(), tb_env->decr_freq);
+    next = now + muldiv64(next, NANOSECONDS_PER_SECOND, tb_env->decr_freq);
     if (next == now)
         next++;
     LOG_TB("%s: TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx "\n", __func__,
diff --git a/qemu/hw/ppc/ppc405_boards.c b/qemu/hw/ppc/ppc405_boards.c
index ec6c4cbaf..4b2f07aec 100644
--- a/qemu/hw/ppc/ppc405_boards.c
+++ b/qemu/hw/ppc/ppc405_boards.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "ppc405.h"
@@ -215,7 +219,8 @@ static void ref405ep_init(MachineState *machine)
                         33333333, &pic, kernel_filename == NULL ? 0 : 1);
     /* allocate SRAM */
     sram_size = 512 * 1024;
-    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size, &error_abort);
+    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size,
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0xFFF00000, sram);
     /* allocate and load BIOS */
@@ -250,7 +255,7 @@ static void ref405ep_init(MachineState *machine)
 #endif
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "ef405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
 
         if (bios_name == NULL)
@@ -368,10 +373,18 @@ static void ref405ep_init(MachineState *machine)
 #endif
 }
 
-static QEMUMachine ref405ep_machine = {
-    .name = "ref405ep",
-    .desc = "ref405ep",
-    .init = ref405ep_init,
+static void ref405ep_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ref405ep";
+    mc->init = ref405ep_init;
+}
+
+static const TypeInfo ref405ep_type = {
+    .name = MACHINE_TYPE_NAME("ref405ep"),
+    .parent = TYPE_MACHINE,
+    .class_init = ref405ep_class_init,
 };
 
 /*****************************************************************************/
@@ -399,7 +412,7 @@ struct taihu_cpld_t {
     uint8_t reg1;
 };
 
-static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
+static uint64_t taihu_cpld_read(void *opaque, hwaddr addr, unsigned size)
 {
     taihu_cpld_t *cpld;
     uint32_t ret;
@@ -420,8 +433,8 @@ static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
     return ret;
 }
 
-static void taihu_cpld_writeb (void *opaque,
-                               hwaddr addr, uint32_t value)
+static void taihu_cpld_write(void *opaque, hwaddr addr,
+                             uint64_t value, unsigned size)
 {
     taihu_cpld_t *cpld;
 
@@ -438,48 +451,12 @@ static void taihu_cpld_writeb (void *opaque,
     }
 }
 
-static uint32_t taihu_cpld_readw (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 1);
-
-    return ret;
-}
-
-static void taihu_cpld_writew (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writeb(opaque, addr, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 1, value & 0xFF);
-}
-
-static uint32_t taihu_cpld_readl (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 24;
-    ret |= taihu_cpld_readb(opaque, addr + 1) << 16;
-    ret |= taihu_cpld_readb(opaque, addr + 2) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 3);
-
-    return ret;
-}
-
-static void taihu_cpld_writel (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writel(opaque, addr, (value >> 24) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 1, (value >> 16) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 2, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 3, value & 0xFF);
-}
-
 static const MemoryRegionOps taihu_cpld_ops = {
-    .old_mmio = {
-        .read = { taihu_cpld_readb, taihu_cpld_readw, taihu_cpld_readl, },
-        .write = { taihu_cpld_writeb, taihu_cpld_writew, taihu_cpld_writel, },
+    .read = taihu_cpld_read,
+    .write = taihu_cpld_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
     },
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
@@ -579,7 +556,7 @@ static void taihu_405ep_init(MachineState *machine)
             bios_name = BIOS_FILENAME;
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "taihu_405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (filename) {
@@ -664,16 +641,24 @@ static void taihu_405ep_init(MachineState *machine)
 #endif
 }
 
-static QEMUMachine taihu_machine = {
-    .name = "taihu",
-    .desc = "taihu",
-    .init = taihu_405ep_init,
+static void taihu_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "taihu";
+    mc->init = taihu_405ep_init;
+}
+
+static const TypeInfo taihu_type = {
+    .name = MACHINE_TYPE_NAME("taihu"),
+    .parent = TYPE_MACHINE,
+    .class_init = taihu_class_init,
 };
 
 static void ppc405_machine_init(void)
 {
-    qemu_register_machine(&ref405ep_machine);
-    qemu_register_machine(&taihu_machine);
+    type_register_static(&ref405ep_type);
+    type_register_static(&taihu_type);
 }
 
-machine_init(ppc405_machine_init);
+type_init(ppc405_machine_init)
diff --git a/qemu/hw/ppc/ppc405_uc.c b/qemu/hw/ppc/ppc405_uc.c
index c77434ae0..d6d3fc2c4 100644
--- a/qemu/hw/ppc/ppc405_uc.c
+++ b/qemu/hw/ppc/ppc405_uc.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/boards.h"
@@ -975,7 +979,7 @@ static void ppc405_ocm_init(CPUPPCState *env)
     ocm = g_malloc0(sizeof(ppc405_ocm_t));
     /* XXX: Size is 4096 or 0x04000000 */
     memory_region_init_ram(&ocm->isarc_ram, NULL, "ppc405.ocm", 4096,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&ocm->isarc_ram);
     memory_region_init_alias(&ocm->dsarc_ram, NULL, "ppc405.dsarc", &ocm->isarc_ram,
                              0, 4096);
@@ -1352,7 +1356,7 @@ static uint32_t ppc4xx_gpt_readl (void *opaque, hwaddr addr)
     case 0x00:
         /* Time base counter */
         ret = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + gpt->tb_offset,
-                       gpt->tb_freq, get_ticks_per_sec());
+                       gpt->tb_freq, NANOSECONDS_PER_SECOND);
         break;
     case 0x10:
         /* Output enable */
@@ -1407,7 +1411,7 @@ static void ppc4xx_gpt_writel (void *opaque,
     switch (addr) {
     case 0x00:
         /* Time base counter */
-        gpt->tb_offset = muldiv64(value, get_ticks_per_sec(), gpt->tb_freq)
+        gpt->tb_offset = muldiv64(value, NANOSECONDS_PER_SECOND, gpt->tb_freq)
             - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         ppc4xx_gpt_compute_timer(gpt);
         break;
diff --git a/qemu/hw/ppc/ppc440_bamboo.c b/qemu/hw/ppc/ppc440_bamboo.c
index 032fa803d..5c535b18a 100644
--- a/qemu/hw/ppc/ppc440_bamboo.c
+++ b/qemu/hw/ppc/ppc440_bamboo.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "net/net.h"
 #include "hw/hw.h"
@@ -256,7 +256,8 @@ static void bamboo_init(MachineState *machine)
                               NULL, NULL);
         if (success < 0) {
             success = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               &elf_lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
             entry = elf_entry;
             loadaddr = elf_lowaddr;
         }
@@ -288,20 +289,12 @@ static void bamboo_init(MachineState *machine)
             exit(1);
         }
     }
-
-    if (kvm_enabled())
-        kvmppc_init();
 }
 
-static QEMUMachine bamboo_machine = {
-    .name = "bamboo",
-    .desc = "bamboo",
-    .init = bamboo_init,
-};
-
-static void bamboo_machine_init(void)
+static void bamboo_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&bamboo_machine);
+    mc->desc = "bamboo";
+    mc->init = bamboo_init;
 }
 
-machine_init(bamboo_machine_init);
+DEFINE_MACHINE("bamboo", bamboo_machine_init)
diff --git a/qemu/hw/ppc/ppc4xx_devs.c b/qemu/hw/ppc/ppc4xx_devs.c
index 2f38ff7d2..7d59018fc 100644
--- a/qemu/hw/ppc/ppc4xx_devs.c
+++ b/qemu/hw/ppc/ppc4xx_devs.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc4xx.h"
diff --git a/qemu/hw/ppc/ppc4xx_pci.c b/qemu/hw/ppc/ppc4xx_pci.c
index 0bb3cdb46..683218e5c 100644
--- a/qemu/hw/ppc/ppc4xx_pci.c
+++ b/qemu/hw/ppc/ppc4xx_pci.c
@@ -19,6 +19,7 @@
 /* This file implements emulation of the 32-bit PCI controller found in some
  * 4xx SoCs, such as the 440EP. */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc4xx.h"
diff --git a/qemu/hw/ppc/ppc_booke.c b/qemu/hw/ppc/ppc_booke.c
index 8b94da6b0..ab8d026c3 100644
--- a/qemu/hw/ppc/ppc_booke.c
+++ b/qemu/hw/ppc/ppc_booke.c
@@ -21,6 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "qemu/timer.h"
@@ -162,7 +165,7 @@ static void booke_update_fixed_timer(CPUPPCState         *env,
         ticks += delta_tick;
     }
 
-    *next = now + muldiv64(ticks, get_ticks_per_sec(), tb_env->tb_freq);
+    *next = now + muldiv64(ticks, NANOSECONDS_PER_SECOND, tb_env->tb_freq);
     if ((*next < now) || (*next > INT64_MAX)) {
         /* Overflow, so assume the biggest number the qemu timer supports. */
         *next = INT64_MAX;
diff --git a/qemu/hw/ppc/ppce500_spin.c b/qemu/hw/ppc/ppce500_spin.c
index a99f7b039..76bd78bfd 100644
--- a/qemu/hw/ppc/ppce500_spin.c
+++ b/qemu/hw/ppc/ppce500_spin.c
@@ -27,6 +27,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
diff --git a/qemu/hw/ppc/prep.c b/qemu/hw/ppc/prep.c
index 45b5f62d6..3ffb85e60 100644
--- a/qemu/hw/ppc/prep.c
+++ b/qemu/hw/ppc/prep.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/timer/m48t59.h"
 #include "hw/i386/pc.h"
@@ -33,6 +34,7 @@
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/ppc.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "qemu/log.h"
 #include "hw/ide.h"
 #include "hw/loader.h"
@@ -42,10 +44,9 @@
 #include "sysemu/arch_init.h"
 #include "sysemu/qtest.h"
 #include "exec/address-spaces.h"
+#include "trace.h"
 #include "elf.h"
-
-//#define HARD_DEBUG_PPC_IO
-//#define DEBUG_PPC_IO
+#include "qemu/cutils.h"
 
 /* SMP is not enabled, for now */
 #define MAX_CPUS 1
@@ -57,26 +58,6 @@
 #define KERNEL_LOAD_ADDR 0x01000000
 #define INITRD_LOAD_ADDR 0x01800000
 
-#if defined (HARD_DEBUG_PPC_IO) && !defined (DEBUG_PPC_IO)
-#define DEBUG_PPC_IO
-#endif
-
-#if defined (HARD_DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...)                         \
-do {                                                     \
-    if (qemu_loglevel_mask(CPU_LOG_IOPORT)) {            \
-        qemu_log("%s: " fmt, __func__ , ## __VA_ARGS__); \
-    } else {                                             \
-        printf("%s : " fmt, __func__ , ## __VA_ARGS__);  \
-    }                                                    \
-} while (0)
-#elif defined (DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...) \
-qemu_log_mask(CPU_LOG_IOPORT, fmt, ## __VA_ARGS__)
-#else
-#define PPC_IO_DPRINTF(fmt, ...) do { } while (0)
-#endif
-
 /* Constants for devices init */
 static const int ide_iobase[2] = { 0x1f0, 0x170 };
 static const int ide_iobase2[2] = { 0x3f6, 0x376 };
@@ -199,8 +180,7 @@ static void PREP_io_800_writeb (void *opaque, uint32_t addr, uint32_t val)
 {
     sysctrl_t *sysctrl = opaque;
 
-    PPC_IO_DPRINTF("0x%08" PRIx32 " => 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, val);
+    trace_prep_io_800_writeb(addr - PPC_IO_BASE, val);
     switch (addr) {
     case 0x0092:
         /* Special port 92 */
@@ -327,8 +307,7 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
         printf("ERROR: unaffected IO port: %04" PRIx32 " read\n", addr);
         break;
     }
-    PPC_IO_DPRINTF("0x%08" PRIx32 " <= 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, retval);
+    trace_prep_io_800_readb(addr - PPC_IO_BASE, retval);
 
     return retval;
 }
@@ -336,15 +315,6 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
 
 #define NVRAM_SIZE        0x2000
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void ppc_prep_reset(void *opaque)
 {
     PowerPCCPU *cpu = opaque;
@@ -565,7 +535,7 @@ static void ppc_prep_init(MachineState *machine)
         kernel_size = load_image_targphys(kernel_filename, kernel_base,
                                           ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -574,8 +544,9 @@ static void ppc_prep_init(MachineState *machine)
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                          initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
+                exit(1);
             }
         } else {
             initrd_base = 0;
@@ -602,7 +573,8 @@ static void ppc_prep_init(MachineState *machine)
     }
 
     if (PPC_INPUT(env) != PPC_FLAGS_INPUT_6xx) {
-        hw_error("Only 6xx bus is supported on PREP machine\n");
+        error_report("Only 6xx bus is supported on PREP machine");
+        exit(1);
     }
 
     dev = qdev_create(NULL, "raven-pcihost");
@@ -610,7 +582,7 @@ static void ppc_prep_init(MachineState *machine)
         bios_name = BIOS_FILENAME;
     }
     qdev_prop_set_string(dev, "bios-name", bios_name);
-    qdev_prop_set_uint32(dev, "elf-machine", ELF_MACHINE);
+    qdev_prop_set_uint32(dev, "elf-machine", PPC_ELF_MACHINE);
     pcihost = PCI_HOST_BRIDGE(dev);
     object_property_add_child(qdev_get_machine(), "raven", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
@@ -626,8 +598,6 @@ static void ppc_prep_init(MachineState *machine)
     cpu = POWERPC_CPU(first_cpu);
     qdev_connect_gpio_out(&pci->qdev, 0,
                           cpu->env.irq_inputs[PPC6xx_INPUT_INT]);
-    qdev_connect_gpio_out(&pci->qdev, 1,
-                          qemu_allocate_irq(cpu_request_exit, NULL, 0));
     sysbus_connect_irq(&pcihost->busdev, 0, qdev_get_gpio_in(&pci->qdev, 9));
     sysbus_connect_irq(&pcihost->busdev, 1, qdev_get_gpio_in(&pci->qdev, 11));
     sysbus_connect_irq(&pcihost->busdev, 2, qdev_get_gpio_in(&pci->qdev, 9));
@@ -698,17 +668,12 @@ static void ppc_prep_init(MachineState *machine)
                          graphic_width, graphic_height, graphic_depth);
 }
 
-static QEMUMachine prep_machine = {
-    .name = "prep",
-    .desc = "PowerPC PREP platform",
-    .init = ppc_prep_init,
-    .max_cpus = MAX_CPUS,
-    .default_boot_order = "cad",
-};
-
-static void prep_machine_init(void)
+static void prep_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&prep_machine);
+    mc->desc = "PowerPC PREP platform";
+    mc->init = ppc_prep_init;
+    mc->max_cpus = MAX_CPUS;
+    mc->default_boot_order = "cad";
 }
 
-machine_init(prep_machine_init);
+DEFINE_MACHINE("prep", prep_machine_init)
diff --git a/qemu/hw/ppc/spapr.c b/qemu/hw/ppc/spapr.c
index a6f19473c..b69995e0d 100644
--- a/qemu/hw/ppc/spapr.c
+++ b/qemu/hw/ppc/spapr.c
@@ -24,15 +24,19 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/numa.h"
 #include "hw/hw.h"
 #include "hw/fw-path-provider.h"
 #include "elf.h"
 #include "net/net.h"
+#include "sysemu/device_tree.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
+#include "sysemu/device_tree.h"
 #include "kvm_ppc.h"
 #include "migration/migration.h"
 #include "mmu-hash64.h"
@@ -60,6 +64,7 @@
 #include "hw/nmi.h"
 
 #include "hw/compat.h"
+#include "qemu/cutils.h"
 
 #include <libfdt.h>
 
@@ -73,7 +78,7 @@
  *
  * We load our kernel at 4M, leaving space for SLOF initial image
  */
-#define FDT_MAX_SIZE            0x40000
+#define FDT_MAX_SIZE            0x100000
 #define RTAS_MAX_SIZE           0x10000
 #define RTAS_MAX_ADDR           0x80000000 /* RTAS must stay below that */
 #define FW_MAX_SIZE             0x400000
@@ -85,8 +90,6 @@
 
 #define TIMEBASE_FREQ           512000000ULL
 
-#define MAX_CPUS                255
-
 #define PHANDLE_XICP            0x00001111
 
 #define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
@@ -110,7 +113,7 @@ static XICSState *try_create_xics(const char *type, int nr_servers,
 }
 
 static XICSState *xics_system_init(MachineState *machine,
-                                   int nr_servers, int nr_irqs)
+                                   int nr_servers, int nr_irqs, Error **errp)
 {
     XICSState *icp = NULL;
 
@@ -121,13 +124,15 @@ static XICSState *xics_system_init(MachineState *machine,
             icp = try_create_xics(TYPE_KVM_XICS, nr_servers, nr_irqs, &err);
         }
         if (machine_kernel_irqchip_required(machine) && !icp) {
-            error_report("kernel_irqchip requested but unavailable: %s",
-                         error_get_pretty(err));
+            error_reportf_err(err,
+                              "kernel_irqchip requested but unavailable: ");
+        } else {
+            error_free(err);
         }
     }
 
     if (!icp) {
-        icp = try_create_xics(TYPE_XICS, nr_servers, nr_irqs, &error_abort);
+        icp = try_create_xics(TYPE_XICS, nr_servers, nr_irqs, errp);
     }
 
     return icp;
@@ -373,8 +378,16 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
                           qemu_uuid[14], qemu_uuid[15]);
 
     _FDT((fdt_property_string(fdt, "vm,uuid", buf)));
+    if (qemu_uuid_set) {
+        _FDT((fdt_property_string(fdt, "system-id", buf)));
+    }
     g_free(buf);
 
+    if (qemu_get_vm_name()) {
+        _FDT((fdt_property_string(fdt, "ibm,partition-name",
+                                  qemu_get_vm_name())));
+    }
+
     _FDT((fdt_property_cell(fdt, "#address-cells", 0x2)));
     _FDT((fdt_property_cell(fdt, "#size-cells", 0x2)));
 
@@ -427,6 +440,10 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     _FDT((fdt_property_cell(fdt, "rtas-event-scan-rate",
                             RTAS_EVENT_SCAN_RATE)));
 
+    if (msi_nonbroken) {
+        _FDT((fdt_property(fdt, "ibm,change-msix-capable", NULL, 0)));
+    }
+
     /*
      * According to PAPR, rtas ibm,os-term does not guarantee a return
      * back to the guest cpu.
@@ -481,10 +498,11 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
              * Older KVM versions with older guest kernels were broken with the
              * magic page, don't allow the guest to map it.
              */
-            kvmppc_get_hypercall(first_cpu->env_ptr, hypercall,
-                                 sizeof(hypercall));
-            _FDT((fdt_property(fdt, "hcall-instructions", hypercall,
-                              sizeof(hypercall))));
+            if (!kvmppc_get_hypercall(first_cpu->env_ptr, hypercall,
+                                      sizeof(hypercall))) {
+                _FDT((fdt_property(fdt, "hcall-instructions", hypercall,
+                                   sizeof(hypercall))));
+            }
         }
         _FDT((fdt_end_node(fdt)));
     }
@@ -495,44 +513,7 @@ static void *spapr_create_fdt_skel(hwaddr initrd_base,
     return fdt;
 }
 
-int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
-                                 target_ulong addr, target_ulong size)
-{
-    void *fdt, *fdt_skel;
-    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
-
-    size -= sizeof(hdr);
-
-    /* Create sceleton */
-    fdt_skel = g_malloc0(size);
-    _FDT((fdt_create(fdt_skel, size)));
-    _FDT((fdt_begin_node(fdt_skel, "")));
-    _FDT((fdt_end_node(fdt_skel)));
-    _FDT((fdt_finish(fdt_skel)));
-    fdt = g_malloc0(size);
-    _FDT((fdt_open_into(fdt_skel, fdt, size)));
-    g_free(fdt_skel);
-
-    /* Fix skeleton up */
-    _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
-
-    /* Pack resulting tree */
-    _FDT((fdt_pack(fdt)));
-
-    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
-        trace_spapr_cas_failed(size);
-        return -1;
-    }
-
-    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
-    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
-    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
-    g_free(fdt);
-
-    return 0;
-}
-
-static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
+static int spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
                                        hwaddr size)
 {
     uint32_t associativity[] = {
@@ -555,6 +536,7 @@ static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
                       sizeof(mem_reg_property))));
     _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
                       sizeof(associativity))));
+    return off;
 }
 
 static int spapr_populate_memory(sPAPRMachineState *spapr, void *fdt)
@@ -620,11 +602,27 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
     uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
     uint32_t page_sizes_prop[64];
     size_t page_sizes_prop_size;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("smp-opts"), NULL);
-    unsigned sockets = opts ? qemu_opt_get_number(opts, "sockets", 0) : 0;
-    uint32_t cpus_per_socket = sockets ? (smp_cpus / sockets) : 1;
+    uint32_t vcpus_per_socket = smp_threads * smp_cores;
     uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
 
+    /* Note: we keep CI large pages off for now because a 64K capable guest
+     * provisioned with large pages might otherwise try to map a qemu
+     * framebuffer (or other kind of memory mapped PCI BAR) using 64K pages
+     * even if that qemu runs on a 4k host.
+     *
+     * We can later add this bit back when we are confident this is not
+     * an issue (!HV KVM or 64K host)
+     */
+    uint8_t pa_features_206[] = { 6, 0,
+        0xf6, 0x1f, 0xc7, 0x00, 0x80, 0xc0 };
+    uint8_t pa_features_207[] = { 24, 0,
+        0xf6, 0x1f, 0xc7, 0xc0, 0x80, 0xf0,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x80, 0x00,
+        0x80, 0x00, 0x80, 0x00, 0x80, 0x00 };
+    uint8_t *pa_features;
+    size_t pa_size;
+
     _FDT((fdt_setprop_cell(fdt, offset, "reg", index)));
     _FDT((fdt_setprop_string(fdt, offset, "device_type", "cpu")));
 
@@ -653,6 +651,7 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
 
     _FDT((fdt_setprop_cell(fdt, offset, "timebase-frequency", tbfreq)));
     _FDT((fdt_setprop_cell(fdt, offset, "clock-frequency", cpufreq)));
+    _FDT((fdt_setprop_cell(fdt, offset, "slb-size", env->slb_nr)));
     _FDT((fdt_setprop_cell(fdt, offset, "ibm,slb-size", env->slb_nr)));
     _FDT((fdt_setprop_string(fdt, offset, "status", "okay")));
     _FDT((fdt_setprop(fdt, offset, "64-bit", NULL, 0)));
@@ -690,8 +689,21 @@ static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
                           page_sizes_prop, page_sizes_prop_size)));
     }
 
+    /* Do the ibm,pa-features property, adjust it for ci-large-pages */
+    if (env->mmu_model == POWERPC_MMU_2_06) {
+        pa_features = pa_features_206;
+        pa_size = sizeof(pa_features_206);
+    } else /* env->mmu_model == POWERPC_MMU_2_07 */ {
+        pa_features = pa_features_207;
+        pa_size = sizeof(pa_features_207);
+    }
+    if (env->ci_large_pages) {
+        pa_features[3] |= 0x20;
+    }
+    _FDT((fdt_setprop(fdt, offset, "ibm,pa-features", pa_features, pa_size)));
+
     _FDT((fdt_setprop_cell(fdt, offset, "ibm,chip-id",
-                           cs->cpu_index / cpus_per_socket)));
+                           cs->cpu_index / vcpus_per_socket)));
 
     _FDT((fdt_setprop(fdt, offset, "ibm,pft-size",
                       pft_size_prop, sizeof(pft_size_prop))));
@@ -738,12 +750,162 @@ static void spapr_populate_cpus_dt_node(void *fdt, sPAPRMachineState *spapr)
 
 }
 
+/*
+ * Adds ibm,dynamic-reconfiguration-memory node.
+ * Refer to docs/specs/ppc-spapr-hotplug.txt for the documentation
+ * of this device tree node.
+ */
+static int spapr_populate_drconf_memory(sPAPRMachineState *spapr, void *fdt)
+{
+    MachineState *machine = MACHINE(spapr);
+    int ret, i, offset;
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t prop_lmb_size[] = {0, cpu_to_be32(lmb_size)};
+    uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
+    uint32_t *int_buf, *cur_index, buf_len;
+    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
+
+    /*
+     * Don't create the node if there are no DR LMBs.
+     */
+    if (!nr_lmbs) {
+        return 0;
+    }
+
+    /*
+     * Allocate enough buffer size to fit in ibm,dynamic-memory
+     * or ibm,associativity-lookup-arrays
+     */
+    buf_len = MAX(nr_lmbs * SPAPR_DR_LMB_LIST_ENTRY_SIZE + 1, nr_nodes * 4 + 2)
+              * sizeof(uint32_t);
+    cur_index = int_buf = g_malloc0(buf_len);
+
+    offset = fdt_add_subnode(fdt, 0, "ibm,dynamic-reconfiguration-memory");
+
+    ret = fdt_setprop(fdt, offset, "ibm,lmb-size", prop_lmb_size,
+                    sizeof(prop_lmb_size));
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-flags-mask", 0xff);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-preservation-time", 0x0);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,dynamic-memory */
+    int_buf[0] = cpu_to_be32(nr_lmbs);
+    cur_index++;
+    for (i = 0; i < nr_lmbs; i++) {
+        sPAPRDRConnector *drc;
+        sPAPRDRConnectorClass *drck;
+        uint64_t addr = i * lmb_size + spapr->hotplug_memory.base;;
+        uint32_t *dynamic_memory = cur_index;
+
+        drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB,
+                                       addr/lmb_size);
+        g_assert(drc);
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+        dynamic_memory[0] = cpu_to_be32(addr >> 32);
+        dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
+        dynamic_memory[2] = cpu_to_be32(drck->get_index(drc));
+        dynamic_memory[3] = cpu_to_be32(0); /* reserved */
+        dynamic_memory[4] = cpu_to_be32(numa_get_node(addr, NULL));
+        if (addr < machine->ram_size ||
+                    memory_region_present(get_system_memory(), addr)) {
+            dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_ASSIGNED);
+        } else {
+            dynamic_memory[5] = cpu_to_be32(0);
+        }
+
+        cur_index += SPAPR_DR_LMB_LIST_ENTRY_SIZE;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,dynamic-memory", int_buf, buf_len);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,associativity-lookup-arrays */
+    cur_index = int_buf;
+    int_buf[0] = cpu_to_be32(nr_nodes);
+    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
+    cur_index += 2;
+    for (i = 0; i < nr_nodes; i++) {
+        uint32_t associativity[] = {
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(i)
+        };
+        memcpy(cur_index, associativity, sizeof(associativity));
+        cur_index += 4;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
+            (cur_index - int_buf) * sizeof(uint32_t));
+out:
+    g_free(int_buf);
+    return ret;
+}
+
+int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update)
+{
+    void *fdt, *fdt_skel;
+    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    size -= sizeof(hdr);
+
+    /* Create sceleton */
+    fdt_skel = g_malloc0(size);
+    _FDT((fdt_create(fdt_skel, size)));
+    _FDT((fdt_begin_node(fdt_skel, "")));
+    _FDT((fdt_end_node(fdt_skel)));
+    _FDT((fdt_finish(fdt_skel)));
+    fdt = g_malloc0(size);
+    _FDT((fdt_open_into(fdt_skel, fdt, size)));
+    g_free(fdt_skel);
+
+    /* Fixup cpu nodes */
+    if (cpu_update) {
+        _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
+    }
+
+    /* Generate ibm,dynamic-reconfiguration-memory node if required */
+    if (memory_update && smc->dr_lmb_enabled) {
+        _FDT((spapr_populate_drconf_memory(spapr, fdt)));
+    }
+
+    /* Pack resulting tree */
+    _FDT((fdt_pack(fdt)));
+
+    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
+        trace_spapr_cas_failed(size);
+        return -1;
+    }
+
+    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
+    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
+    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
+    g_free(fdt);
+
+    return 0;
+}
+
 static void spapr_finalize_fdt(sPAPRMachineState *spapr,
                                hwaddr fdt_addr,
                                hwaddr rtas_addr,
                                hwaddr rtas_size)
 {
     MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
     const char *boot_device = machine->boot_order;
     int ret, i;
     size_t cb = 0;
@@ -768,13 +930,20 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         exit(1);
     }
 
-    QLIST_FOREACH(phb, &spapr->phbs, list) {
-        ret = spapr_populate_pci_dt(phb, PHANDLE_XICP, fdt);
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL)) {
+        ret = spapr_rng_populate_dt(fdt);
+        if (ret < 0) {
+            fprintf(stderr, "could not set up rng device in the fdt\n");
+            exit(1);
+        }
     }
 
-    if (ret < 0) {
-        fprintf(stderr, "couldn't setup PCI devices in fdt\n");
-        exit(1);
+    QLIST_FOREACH(phb, &spapr->phbs, list) {
+        ret = spapr_populate_pci_dt(phb, PHANDLE_XICP, fdt);
+        if (ret < 0) {
+            error_report("couldn't setup PCI devices in fdt");
+            exit(1);
+        }
     }
 
     /* RTAS */
@@ -814,6 +983,10 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         spapr_populate_chosen_stdout(fdt, spapr->vio_bus);
     }
 
+    if (smc->dr_lmb_enabled) {
+        _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB));
+    }
+
     _FDT((fdt_pack(fdt)));
 
     if (fdt_totalsize(fdt) > FDT_MAX_SIZE) {
@@ -822,6 +995,7 @@ static void spapr_finalize_fdt(sPAPRMachineState *spapr,
         exit(1);
     }
 
+    qemu_fdt_dumpdtb(fdt, fdt_totalsize(fdt));
     cpu_physical_memory_write(fdt_addr, fdt, fdt_totalsize(fdt));
 
     g_free(bootlist);
@@ -851,45 +1025,93 @@ static void emulate_spapr_hypercall(PowerPCCPU *cpu)
 #define CLEAN_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) &= tswap64(~HPTE64_V_HPTE_DIRTY))
 #define DIRTY_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) |= tswap64(HPTE64_V_HPTE_DIRTY))
 
-static void spapr_reset_htab(sPAPRMachineState *spapr)
+/*
+ * Get the fd to access the kernel htab, re-opening it if necessary
+ */
+static int get_htab_fd(sPAPRMachineState *spapr)
 {
-    long shift;
-    int index;
+    if (spapr->htab_fd >= 0) {
+        return spapr->htab_fd;
+    }
+
+    spapr->htab_fd = kvmppc_get_htab_fd(false);
+    if (spapr->htab_fd < 0) {
+        error_report("Unable to open fd for reading hash table from KVM: %s",
+                     strerror(errno));
+    }
 
-    /* allocate hash page table.  For now we always make this 16mb,
-     * later we should probably make it scale to the size of guest
-     * RAM */
+    return spapr->htab_fd;
+}
 
-    shift = kvmppc_reset_htab(spapr->htab_shift);
+static void close_htab_fd(sPAPRMachineState *spapr)
+{
+    if (spapr->htab_fd >= 0) {
+        close(spapr->htab_fd);
+    }
+    spapr->htab_fd = -1;
+}
 
-    if (shift > 0) {
-        /* Kernel handles htab, we don't need to allocate one */
-        spapr->htab_shift = shift;
-        kvmppc_kern_htab = true;
+static int spapr_hpt_shift_for_ramsize(uint64_t ramsize)
+{
+    int shift;
+
+    /* We aim for a hash table of size 1/128 the size of RAM (rounded
+     * up).  The PAPR recommendation is actually 1/64 of RAM size, but
+     * that's much more than is needed for Linux guests */
+    shift = ctz64(pow2ceil(ramsize)) - 7;
+    shift = MAX(shift, 18); /* Minimum architected size */
+    shift = MIN(shift, 46); /* Maximum architected size */
+    return shift;
+}
 
-        /* Tell readers to update their file descriptor */
-        if (spapr->htab_fd >= 0) {
-            spapr->htab_fd_stale = true;
+static void spapr_reallocate_hpt(sPAPRMachineState *spapr, int shift,
+                                 Error **errp)
+{
+    long rc;
+
+    /* Clean up any HPT info from a previous boot */
+    g_free(spapr->htab);
+    spapr->htab = NULL;
+    spapr->htab_shift = 0;
+    close_htab_fd(spapr);
+
+    rc = kvmppc_reset_htab(shift);
+    if (rc < 0) {
+        /* kernel-side HPT needed, but couldn't allocate one */
+        error_setg_errno(errp, errno,
+                         "Failed to allocate KVM HPT of order %d (try smaller maxmem?)",
+                         shift);
+        /* This is almost certainly fatal, but if the caller really
+         * wants to carry on with shift == 0, it's welcome to try */
+    } else if (rc > 0) {
+        /* kernel-side HPT allocated */
+        if (rc != shift) {
+            error_setg(errp,
+                       "Requested order %d HPT, but kernel allocated order %ld (try smaller maxmem?)",
+                       shift, rc);
         }
+
+        spapr->htab_shift = shift;
+        spapr->htab = NULL;
     } else {
+        /* kernel-side HPT not needed, allocate in userspace instead */
+        size_t size = 1ULL << shift;
+        int i;
+
+        spapr->htab = qemu_memalign(size, size);
         if (!spapr->htab) {
-            /* Allocate an htab if we don't yet have one */
-            spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
+            error_setg_errno(errp, errno,
+                             "Could not allocate HPT of order %d", shift);
+            return;
         }
 
-        /* And clear it */
-        memset(spapr->htab, 0, HTAB_SIZE(spapr));
+        memset(spapr->htab, 0, size);
+        spapr->htab_shift = shift;
 
-        for (index = 0; index < HTAB_SIZE(spapr) / HASH_PTE_SIZE_64; index++) {
-            DIRTY_HPTE(HPTE(spapr->htab, index));
+        for (i = 0; i < size / HASH_PTE_SIZE_64; i++) {
+            DIRTY_HPTE(HPTE(spapr->htab, i));
         }
     }
-
-    /* Update the RMA size if necessary */
-    if (spapr->vrma_adjust) {
-        spapr->rma_size = kvmppc_rma_size(spapr_node0_size(),
-                                          spapr->htab_shift);
-    }
 }
 
 static int find_unknown_sysbus_device(SysBusDevice *sbdev, void *opaque)
@@ -909,39 +1131,26 @@ static int find_unknown_sysbus_device(SysBusDevice *sbdev, void *opaque)
     return 0;
 }
 
-/*
- * A guest reset will cause spapr->htab_fd to become stale if being used.
- * Reopen the file descriptor to make sure the whole HTAB is properly read.
- */
-static int spapr_check_htab_fd(sPAPRMachineState *spapr)
-{
-    int rc = 0;
-
-    if (spapr->htab_fd_stale) {
-        close(spapr->htab_fd);
-        spapr->htab_fd = kvmppc_get_htab_fd(false);
-        if (spapr->htab_fd < 0) {
-            error_report("Unable to open fd for reading hash table from KVM: "
-                         "%s", strerror(errno));
-            rc = -1;
-        }
-        spapr->htab_fd_stale = false;
-    }
-
-    return rc;
-}
-
 static void ppc_spapr_reset(void)
 {
-    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
     PowerPCCPU *first_ppc_cpu;
     uint32_t rtas_limit;
 
     /* Check for unknown sysbus devices */
     foreach_dynamic_sysbus_device(find_unknown_sysbus_device, NULL);
 
-    /* Reset the hash table & recalc the RMA */
-    spapr_reset_htab(spapr);
+    /* Allocate and/or reset the hash page table */
+    spapr_reallocate_hpt(spapr,
+                         spapr_hpt_shift_for_ramsize(machine->maxram_size),
+                         &error_fatal);
+
+    /* Update the RMA size if necessary */
+    if (spapr->vrma_adjust) {
+        spapr->rma_size = kvmppc_rma_size(spapr_node0_size(),
+                                          spapr->htab_shift);
+    }
 
     qemu_devices_reset();
 
@@ -987,24 +1196,8 @@ static void spapr_cpu_reset(void *opaque)
 
     env->spr[SPR_HIOR] = 0;
 
-    env->external_htab = (uint8_t *)spapr->htab;
-    if (kvm_enabled() && !env->external_htab) {
-        /*
-         * HV KVM, set external_htab to 1 so our ppc_hash64_load_hpte*
-         * functions do the right thing.
-         */
-        env->external_htab = (void *)1;
-    }
-    env->htab_base = -1;
-    /*
-     * htab_mask is the mask used to normalize hash value to PTEG index.
-     * htab_shift is log2 of hash table size.
-     * We have 8 hpte per group, and each hpte is 16 bytes.
-     * ie have 128 bytes per hpte entry.
-     */
-    env->htab_mask = (1ULL << (spapr->htab_shift - 7)) - 1;
-    env->spr[SPR_SDR1] = (target_ulong)(uintptr_t)spapr->htab |
-        (spapr->htab_shift - 18);
+    ppc_hash64_set_external_hpt(cpu, spapr->htab, spapr->htab_shift,
+                                &error_fatal);
 }
 
 static void spapr_create_nvram(sPAPRMachineState *spapr)
@@ -1013,7 +1206,8 @@ static void spapr_create_nvram(sPAPRMachineState *spapr)
     DriveInfo *dinfo = drive_get(IF_PFLASH, 0, 0);
 
     if (dinfo) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(dinfo));
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                            &error_fatal);
     }
 
     qdev_init_nofail(dev);
@@ -1033,7 +1227,7 @@ static void spapr_rtc_create(sPAPRMachineState *spapr)
 }
 
 /* Returns whether we want to use VGA or not */
-static int spapr_vga_init(PCIBus *pci_bus)
+static bool spapr_vga_init(PCIBus *pci_bus, Error **errp)
 {
     switch (vga_interface_type) {
     case VGA_NONE:
@@ -1041,11 +1235,12 @@ static int spapr_vga_init(PCIBus *pci_bus)
     case VGA_DEVICE:
         return true;
     case VGA_STD:
+    case VGA_VIRTIO:
         return pci_vga_init(pci_bus) != NULL;
     default:
-        fprintf(stderr, "This vga model is not supported,"
-                "currently it only supports -vga std\n");
-        exit(0);
+        error_setg(errp,
+                   "Unsupported VGA mode, only -vga std or -vga virtio is supported");
+        return false;
     }
 }
 
@@ -1099,14 +1294,6 @@ static int htab_save_setup(QEMUFile *f, void *opaque)
         spapr->htab_first_pass = true;
     } else {
         assert(kvm_enabled());
-
-        spapr->htab_fd = kvmppc_get_htab_fd(false);
-        spapr->htab_fd_stale = false;
-        if (spapr->htab_fd < 0) {
-            fprintf(stderr, "Unable to open fd for reading hash table from KVM: %s\n",
-                    strerror(errno));
-            return -1;
-        }
     }
 
 
@@ -1116,6 +1303,7 @@ static int htab_save_setup(QEMUFile *f, void *opaque)
 static void htab_save_first_pass(QEMUFile *f, sPAPRMachineState *spapr,
                                  int64_t max_ns)
 {
+    bool has_timeout = max_ns != -1;
     int htabslots = HTAB_SIZE(spapr) / HASH_PTE_SIZE_64;
     int index = spapr->htab_save_index;
     int64_t starttime = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
@@ -1149,7 +1337,8 @@ static void htab_save_first_pass(QEMUFile *f, sPAPRMachineState *spapr,
             qemu_put_buffer(f, HPTE(spapr->htab, chunkstart),
                             HASH_PTE_SIZE_64 * n_valid);
 
-            if ((qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
+            if (has_timeout &&
+                (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
                 break;
             }
         }
@@ -1246,6 +1435,7 @@ static int htab_save_later_pass(QEMUFile *f, sPAPRMachineState *spapr,
 static int htab_save_iterate(QEMUFile *f, void *opaque)
 {
     sPAPRMachineState *spapr = opaque;
+    int fd;
     int rc = 0;
 
     /* Iteration header */
@@ -1254,13 +1444,12 @@ static int htab_save_iterate(QEMUFile *f, void *opaque)
     if (!spapr->htab) {
         assert(kvm_enabled());
 
-        rc = spapr_check_htab_fd(spapr);
-        if (rc < 0) {
-            return rc;
+        fd = get_htab_fd(spapr);
+        if (fd < 0) {
+            return fd;
         }
 
-        rc = kvmppc_save_htab(f, spapr->htab_fd,
-                              MAX_KVM_BUF_SIZE, MAX_ITERATION_NS);
+        rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, MAX_ITERATION_NS);
         if (rc < 0) {
             return rc;
         }
@@ -1281,6 +1470,7 @@ static int htab_save_iterate(QEMUFile *f, void *opaque)
 static int htab_save_complete(QEMUFile *f, void *opaque)
 {
     sPAPRMachineState *spapr = opaque;
+    int fd;
 
     /* Iteration header */
     qemu_put_be32(f, 0);
@@ -1290,18 +1480,20 @@ static int htab_save_complete(QEMUFile *f, void *opaque)
 
         assert(kvm_enabled());
 
-        rc = spapr_check_htab_fd(spapr);
-        if (rc < 0) {
-            return rc;
+        fd = get_htab_fd(spapr);
+        if (fd < 0) {
+            return fd;
         }
 
-        rc = kvmppc_save_htab(f, spapr->htab_fd, MAX_KVM_BUF_SIZE, -1);
+        rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, -1);
         if (rc < 0) {
             return rc;
         }
-        close(spapr->htab_fd);
-        spapr->htab_fd = -1;
+        close_htab_fd(spapr);
     } else {
+        if (spapr->htab_first_pass) {
+            htab_save_first_pass(f, spapr, -1);
+        }
         htab_save_later_pass(f, spapr, -1);
     }
 
@@ -1320,15 +1512,19 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
     int fd = -1;
 
     if (version_id < 1 || version_id > 1) {
-        fprintf(stderr, "htab_load() bad version\n");
+        error_report("htab_load() bad version");
         return -EINVAL;
     }
 
     section_hdr = qemu_get_be32(f);
 
     if (section_hdr) {
-        /* First section, just the hash shift */
-        if (spapr->htab_shift != section_hdr) {
+        Error *local_err = NULL;
+
+        /* First section gives the htab size */
+        spapr_reallocate_hpt(spapr, section_hdr, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
             return -EINVAL;
         }
         return 0;
@@ -1339,8 +1535,8 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
 
         fd = kvmppc_get_htab_fd(true);
         if (fd < 0) {
-            fprintf(stderr, "Unable to open fd to restore KVM hash table: %s\n",
-                    strerror(errno));
+            error_report("Unable to open fd to restore KVM hash table: %s",
+                         strerror(errno));
         }
     }
 
@@ -1360,9 +1556,9 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
         if ((index + n_valid + n_invalid) >
             (HTAB_SIZE(spapr) / HASH_PTE_SIZE_64)) {
             /* Bad index in stream */
-            fprintf(stderr, "htab_load() bad index %d (%hd+%hd entries) "
-                    "in htab stream (htab_shift=%d)\n", index, n_valid, n_invalid,
-                    spapr->htab_shift);
+            error_report(
+                "htab_load() bad index %d (%hd+%hd entries) in htab stream (htab_shift=%d)",
+                index, n_valid, n_invalid, spapr->htab_shift);
             return -EINVAL;
         }
 
@@ -1398,7 +1594,7 @@ static int htab_load(QEMUFile *f, void *opaque, int version_id)
 static SaveVMHandlers savevm_htab_handlers = {
     .save_live_setup = htab_save_setup,
     .save_live_iterate = htab_save_iterate,
-    .save_live_complete = htab_save_complete,
+    .save_live_complete_precopy = htab_save_complete,
     .load_state = htab_load,
 };
 
@@ -1409,26 +1605,24 @@ static void spapr_boot_set(void *opaque, const char *boot_device,
     machine->boot_order = g_strdup(boot_device);
 }
 
-static void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu)
+static void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu,
+                           Error **errp)
 {
     CPUPPCState *env = &cpu->env;
 
     /* Set time-base frequency to 512 MHz */
     cpu_ppc_tb_init(env, TIMEBASE_FREQ);
 
-    /* PAPR always has exception vectors in RAM not ROM. To ensure this,
-     * MSR[IP] should never be set.
-     */
-    env->msr_mask &= ~(1 << 6);
-
-    /* Tell KVM that we're in PAPR mode */
-    if (kvm_enabled()) {
-        kvmppc_set_papr(cpu);
-    }
+    /* Enable PAPR mode in TCG or KVM */
+    cpu_ppc_set_papr(cpu);
 
     if (cpu->max_compat) {
-        if (ppc_set_compat(cpu, cpu->max_compat) < 0) {
-            exit(1);
+        Error *local_err = NULL;
+
+        ppc_set_compat(cpu, cpu->max_compat, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
         }
     }
 
@@ -1437,10 +1631,84 @@ static void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu)
     qemu_register_reset(spapr_cpu_reset, cpu);
 }
 
+/*
+ * Reset routine for LMB DR devices.
+ *
+ * Unlike PCI DR devices, LMB DR devices explicitly register this reset
+ * routine. Reset for PCI DR devices will be handled by PHB reset routine
+ * when it walks all its children devices. LMB devices reset occurs
+ * as part of spapr_ppc_reset().
+ */
+static void spapr_drc_reset(void *opaque)
+{
+    sPAPRDRConnector *drc = opaque;
+    DeviceState *d = DEVICE(drc);
+
+    if (d) {
+        device_reset(d);
+    }
+}
+
+static void spapr_create_lmb_dr_connectors(sPAPRMachineState *spapr)
+{
+    MachineState *machine = MACHINE(spapr);
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
+    int i;
+
+    for (i = 0; i < nr_lmbs; i++) {
+        sPAPRDRConnector *drc;
+        uint64_t addr;
+
+        addr = i * lmb_size + spapr->hotplug_memory.base;
+        drc = spapr_dr_connector_new(OBJECT(spapr), SPAPR_DR_CONNECTOR_TYPE_LMB,
+                                     addr/lmb_size);
+        qemu_register_reset(spapr_drc_reset, drc);
+    }
+}
+
+/*
+ * If RAM size, maxmem size and individual node mem sizes aren't aligned
+ * to SPAPR_MEMORY_BLOCK_SIZE(256MB), then refuse to start the guest
+ * since we can't support such unaligned sizes with DRCONF_MEMORY.
+ */
+static void spapr_validate_node_memory(MachineState *machine, Error **errp)
+{
+    int i;
+
+    if (machine->ram_size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(errp, "Memory size 0x" RAM_ADDR_FMT
+                   " is not aligned to %llu MiB",
+                   machine->ram_size,
+                   SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+        return;
+    }
+
+    if (machine->maxram_size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(errp, "Maximum memory size 0x" RAM_ADDR_FMT
+                   " is not aligned to %llu MiB",
+                   machine->ram_size,
+                   SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+        return;
+    }
+
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (numa_info[i].node_mem % SPAPR_MEMORY_BLOCK_SIZE) {
+            error_setg(errp,
+                       "Node %d memory size 0x%" PRIx64
+                       " is not aligned to %llu MiB",
+                       i, numa_info[i].node_mem,
+                       SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+            return;
+        }
+    }
+}
+
 /* pSeries LPAR / sPAPR hardware init */
 static void ppc_spapr_init(MachineState *machine)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -1459,7 +1727,7 @@ static void ppc_spapr_init(MachineState *machine)
     bool kernel_le = false;
     char *filename;
 
-    msi_supported = true;
+    msi_nonbroken = true;
 
     QLIST_INIT(&spapr->phbs);
 
@@ -1494,30 +1762,23 @@ static void ppc_spapr_init(MachineState *machine)
     }
 
     if (spapr->rma_size > node0_size) {
-        fprintf(stderr, "Error: Numa node 0 has to span the RMA (%#08"HWADDR_PRIx")\n",
-                spapr->rma_size);
+        error_report("Numa node 0 has to span the RMA (%#08"HWADDR_PRIx")",
+                     spapr->rma_size);
         exit(1);
     }
 
     /* Setup a load limit for the ramdisk leaving room for SLOF and FDT */
     load_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR) - FW_OVERHEAD;
 
-    /* We aim for a hash table of size 1/128 the size of RAM.  The
-     * normal rule of thumb is 1/64 the size of RAM, but that's much
-     * more than needed for the Linux guests we support. */
-    spapr->htab_shift = 18; /* Minimum architected size */
-    while (spapr->htab_shift <= 46) {
-        if ((1ULL << (spapr->htab_shift + 7)) >= machine->ram_size) {
-            break;
-        }
-        spapr->htab_shift++;
-    }
-
     /* Set up Interrupt Controller before we create the VCPUs */
     spapr->icp = xics_system_init(machine,
                                   DIV_ROUND_UP(max_cpus * kvmppc_smt_threads(),
                                                smp_threads),
-                                  XICS_IRQS);
+                                  XICS_IRQS, &error_fatal);
+
+    if (smc->dr_lmb_enabled) {
+        spapr_validate_node_memory(machine, &error_fatal);
+    }
 
     /* init CPUs */
     if (machine->cpu_model == NULL) {
@@ -1526,15 +1787,16 @@ static void ppc_spapr_init(MachineState *machine)
     for (i = 0; i < smp_cpus; i++) {
         cpu = cpu_ppc_init(machine->cpu_model);
         if (cpu == NULL) {
-            fprintf(stderr, "Unable to find PowerPC CPU definition\n");
+            error_report("Unable to find PowerPC CPU definition");
             exit(1);
         }
-        spapr_cpu_init(spapr, cpu);
+        spapr_cpu_init(spapr, cpu, &error_fatal);
     }
 
     if (kvm_enabled()) {
         /* Enable H_LOGICAL_CI_* so SLOF can talk to in-kernel devices */
         kvmppc_enable_logical_ci_hcalls();
+        kvmppc_enable_set_mode_hcall();
     }
 
     /* allocate RAM */
@@ -1550,6 +1812,29 @@ static void ppc_spapr_init(MachineState *machine)
         memory_region_add_subregion(sysmem, 0, rma_region);
     }
 
+    /* initialize hotplug memory address space */
+    if (machine->ram_size < machine->maxram_size) {
+        ram_addr_t hotplug_mem_size = machine->maxram_size - machine->ram_size;
+
+        if (machine->ram_slots > SPAPR_MAX_RAM_SLOTS) {
+            error_report("Specified number of memory slots %"
+                         PRIu64" exceeds max supported %d",
+                         machine->ram_slots, SPAPR_MAX_RAM_SLOTS);
+            exit(1);
+        }
+
+        spapr->hotplug_memory.base = ROUND_UP(machine->ram_size,
+                                              SPAPR_HOTPLUG_MEM_ALIGN);
+        memory_region_init(&spapr->hotplug_memory.mr, OBJECT(spapr),
+                           "hotplug-memory", hotplug_mem_size);
+        memory_region_add_subregion(sysmem, spapr->hotplug_memory.base,
+                                    &spapr->hotplug_memory.mr);
+    }
+
+    if (smc->dr_lmb_enabled) {
+        spapr_create_lmb_dr_connectors(spapr);
+    }
+
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
     if (!filename) {
         error_report("Could not find LPAR rtas '%s'", "spapr-rtas.bin");
@@ -1610,13 +1895,17 @@ static void ppc_spapr_init(MachineState *machine)
     }
 
     /* Graphics */
-    if (spapr_vga_init(phb->bus)) {
+    if (spapr_vga_init(phb->bus, &error_fatal)) {
         spapr->has_graphics = true;
         machine->usb |= defaults_enabled() && !machine->usb_disabled;
     }
 
     if (machine->usb) {
-        pci_create_simple(phb->bus, -1, "pci-ohci");
+        if (smc->use_ohci_by_default) {
+            pci_create_simple(phb->bus, -1, "pci-ohci");
+        } else {
+            pci_create_simple(phb->bus, -1, "nec-usb-xhci");
+        }
 
         if (spapr->has_graphics) {
             USBBus *usb_bus = usb_bus_find(-1);
@@ -1627,8 +1916,9 @@ static void ppc_spapr_init(MachineState *machine)
     }
 
     if (spapr->rma_size < (MIN_RMA_SLOF << 20)) {
-        fprintf(stderr, "qemu: pSeries SLOF firmware requires >= "
-                "%ldM guest RMA (Real Mode Area memory)\n", MIN_RMA_SLOF);
+        error_report(
+            "pSeries SLOF firmware requires >= %ldM guest RMA (Real Mode Area memory)",
+            MIN_RMA_SLOF);
         exit(1);
     }
 
@@ -1636,16 +1926,18 @@ static void ppc_spapr_init(MachineState *machine)
         uint64_t lowaddr = 0;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size == ELF_LOAD_WRONG_ENDIAN) {
             kernel_size = load_elf(kernel_filename,
                                    translate_kernel_address, NULL,
-                                   NULL, &lowaddr, NULL, 0, ELF_MACHINE, 0);
+                                   NULL, &lowaddr, NULL, 0, PPC_ELF_MACHINE,
+                                   0, 0);
             kernel_le = kernel_size > 0;
         }
         if (kernel_size < 0) {
-            fprintf(stderr, "qemu: error loading %s: %s\n",
-                    kernel_filename, load_elf_strerror(kernel_size));
+            error_report("error loading %s: %s",
+                         kernel_filename, load_elf_strerror(kernel_size));
             exit(1);
         }
 
@@ -1658,8 +1950,8 @@ static void ppc_spapr_init(MachineState *machine)
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               load_limit - initrd_base);
             if (initrd_size < 0) {
-                fprintf(stderr, "qemu: could not load initial ram disk '%s'\n",
-                        initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
         } else {
@@ -1796,6 +2088,9 @@ static void spapr_set_kvm_type(Object *obj, const char *value, Error **errp)
 
 static void spapr_machine_initfn(Object *obj)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
+
+    spapr->htab_fd = -1;
     object_property_add_str(obj, "kvm-type",
                             spapr_get_kvm_type, spapr_set_kvm_type, NULL);
     object_property_set_description(obj, "kvm-type",
@@ -1803,6 +2098,13 @@ static void spapr_machine_initfn(Object *obj)
                                     NULL);
 }
 
+static void spapr_machine_finalizefn(Object *obj)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
+
+    g_free(spapr->kvm_type);
+}
+
 static void ppc_cpu_do_nmi_on_cpu(void *arg)
 {
     CPUState *cs = arg;
@@ -1820,22 +2122,177 @@ static void spapr_nmi(NMIState *n, int cpu_index, Error **errp)
     }
 }
 
+static void spapr_add_lmbs(DeviceState *dev, uint64_t addr, uint64_t size,
+                           uint32_t node, Error **errp)
+{
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    uint32_t nr_lmbs = size/SPAPR_MEMORY_BLOCK_SIZE;
+    int i, fdt_offset, fdt_size;
+    void *fdt;
+
+    /*
+     * Check for DRC connectors and send hotplug notification to the
+     * guest only in case of hotplugged memory. This allows cold plugged
+     * memory to be specified at boot time.
+     */
+    if (!dev->hotplugged) {
+        return;
+    }
+
+    for (i = 0; i < nr_lmbs; i++) {
+        drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB,
+                addr/SPAPR_MEMORY_BLOCK_SIZE);
+        g_assert(drc);
+
+        fdt = create_device_tree(&fdt_size);
+        fdt_offset = spapr_populate_memory_node(fdt, node, addr,
+                                                SPAPR_MEMORY_BLOCK_SIZE);
+
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+        drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, errp);
+        addr += SPAPR_MEMORY_BLOCK_SIZE;
+    }
+    spapr_hotplug_req_add_by_count(SPAPR_DR_CONNECTOR_TYPE_LMB, nr_lmbs);
+}
+
+static void spapr_memory_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                              uint32_t node, Error **errp)
+{
+    Error *local_err = NULL;
+    sPAPRMachineState *ms = SPAPR_MACHINE(hotplug_dev);
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *mr = ddc->get_memory_region(dimm);
+    uint64_t align = memory_region_get_alignment(mr);
+    uint64_t size = memory_region_size(mr);
+    uint64_t addr;
+
+    if (size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(&local_err, "Hotplugged memory size must be a multiple of "
+                      "%lld MB", SPAPR_MEMORY_BLOCK_SIZE/M_BYTE);
+        goto out;
+    }
+
+    pc_dimm_memory_plug(dev, &ms->hotplug_memory, mr, align, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
+    if (local_err) {
+        pc_dimm_memory_unplug(dev, &ms->hotplug_memory, mr);
+        goto out;
+    }
+
+    spapr_add_lmbs(dev, addr, size, node, &error_abort);
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        int node;
+
+        if (!smc->dr_lmb_enabled) {
+            error_setg(errp, "Memory hotplug not supported for this machine");
+            return;
+        }
+        node = object_property_get_int(OBJECT(dev), PC_DIMM_NODE_PROP, errp);
+        if (*errp) {
+            return;
+        }
+        if (node < 0 || node >= MAX_NODES) {
+            error_setg(errp, "Invaild node %d", node);
+            return;
+        }
+
+        /*
+         * Currently PowerPC kernel doesn't allow hot-adding memory to
+         * memory-less node, but instead will silently add the memory
+         * to the first node that has some memory. This causes two
+         * unexpected behaviours for the user.
+         *
+         * - Memory gets hotplugged to a different node than what the user
+         *   specified.
+         * - Since pc-dimm subsystem in QEMU still thinks that memory belongs
+         *   to memory-less node, a reboot will set things accordingly
+         *   and the previously hotplugged memory now ends in the right node.
+         *   This appears as if some memory moved from one node to another.
+         *
+         * So until kernel starts supporting memory hotplug to memory-less
+         * nodes, just prevent such attempts upfront in QEMU.
+         */
+        if (nb_numa_nodes && !numa_info[node].node_mem) {
+            error_setg(errp, "Can't hotplug memory to memory-less node %d",
+                       node);
+            return;
+        }
+
+        spapr_memory_plug(hotplug_dev, dev, node, errp);
+    }
+}
+
+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        error_setg(errp, "Memory hot unplug not supported by sPAPR");
+    }
+}
+
+static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
+                                             DeviceState *dev)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        return HOTPLUG_HANDLER(machine);
+    }
+    return NULL;
+}
+
+static unsigned spapr_cpu_index_to_socket_id(unsigned cpu_index)
+{
+    /* Allocate to NUMA nodes on a "socket" basis (not that concept of
+     * socket means much for the paravirtualized PAPR platform) */
+    return cpu_index / smp_threads / smp_cores;
+}
+
 static void spapr_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(oc);
     FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+
+    mc->desc = "pSeries Logical Partition (PAPR compliant)";
 
+    /*
+     * We set up the default / latest behaviour here.  The class_init
+     * functions for the specific versioned machine types can override
+     * these details for backwards compatibility
+     */
     mc->init = ppc_spapr_init;
     mc->reset = ppc_spapr_reset;
     mc->block_default_type = IF_SCSI;
-    mc->max_cpus = MAX_CPUS;
+    mc->max_cpus = MAX_CPUMASK_BITS;
     mc->no_parallel = 1;
     mc->default_boot_order = "";
     mc->default_ram_size = 512 * M_BYTE;
     mc->kvm_type = spapr_kvm_type;
     mc->has_dynamic_sysbus = true;
+    mc->pci_allow_0_address = true;
+    mc->get_hotplug_handler = spapr_get_hotpug_handler;
+    hc->plug = spapr_machine_device_plug;
+    hc->unplug = spapr_machine_device_unplug;
+    mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id;
 
+    smc->dr_lmb_enabled = true;
     fwc->get_dev_path = spapr_get_fw_dev_path;
     nc->nmi_monitor_handler = spapr_nmi;
 }
@@ -1846,153 +2303,183 @@ static const TypeInfo spapr_machine_info = {
     .abstract      = true,
     .instance_size = sizeof(sPAPRMachineState),
     .instance_init = spapr_machine_initfn,
+    .instance_finalize = spapr_machine_finalizefn,
     .class_size    = sizeof(sPAPRMachineClass),
     .class_init    = spapr_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_FW_PATH_PROVIDER },
         { TYPE_NMI },
+        { TYPE_HOTPLUG_HANDLER },
         { }
     },
 };
 
-#define SPAPR_COMPAT_2_3 \
-        HW_COMPAT_2_3 \
-        {\
-            .driver   = "spapr-pci-host-bridge",\
-            .property = "dynamic-reconfiguration",\
-            .value    = "off",\
-        },
-
-#define SPAPR_COMPAT_2_2 \
-        SPAPR_COMPAT_2_3 \
-        HW_COMPAT_2_2 \
-        {\
-            .driver   = TYPE_SPAPR_PCI_HOST_BRIDGE,\
-            .property = "mem_win_size",\
-            .value    = "0x20000000",\
-        },
-
-#define SPAPR_COMPAT_2_1 \
-        SPAPR_COMPAT_2_2 \
-        HW_COMPAT_2_1
+#define DEFINE_SPAPR_MACHINE(suffix, verstr, latest)                 \
+    static void spapr_machine_##suffix##_class_init(ObjectClass *oc, \
+                                                    void *data)      \
+    {                                                                \
+        MachineClass *mc = MACHINE_CLASS(oc);                        \
+        spapr_machine_##suffix##_class_options(mc);                  \
+        if (latest) {                                                \
+            mc->alias = "pseries";                                   \
+            mc->is_default = 1;                                      \
+        }                                                            \
+    }                                                                \
+    static void spapr_machine_##suffix##_instance_init(Object *obj)  \
+    {                                                                \
+        MachineState *machine = MACHINE(obj);                        \
+        spapr_machine_##suffix##_instance_options(machine);          \
+    }                                                                \
+    static const TypeInfo spapr_machine_##suffix##_info = {          \
+        .name = MACHINE_TYPE_NAME("pseries-" verstr),                \
+        .parent = TYPE_SPAPR_MACHINE,                                \
+        .class_init = spapr_machine_##suffix##_class_init,           \
+        .instance_init = spapr_machine_##suffix##_instance_init,     \
+    };                                                               \
+    static void spapr_machine_register_##suffix(void)                \
+    {                                                                \
+        type_register(&spapr_machine_##suffix##_info);               \
+    }                                                                \
+    type_init(spapr_machine_register_##suffix)
 
-static void spapr_compat_2_3(Object *obj)
+/*
+ * pseries-2.6
+ */
+static void spapr_machine_2_6_instance_options(MachineState *machine)
 {
-    savevm_skip_section_footers();
-    global_state_set_optional();
 }
 
-static void spapr_compat_2_2(Object *obj)
+static void spapr_machine_2_6_class_options(MachineClass *mc)
 {
-    spapr_compat_2_3(obj);
+    /* Defaults for the latest behaviour inherited from the base class */
 }
 
-static void spapr_compat_2_1(Object *obj)
-{
-    spapr_compat_2_2(obj);
-}
+DEFINE_SPAPR_MACHINE(2_6, "2.6", true);
+
+/*
+ * pseries-2.5
+ */
+#define SPAPR_COMPAT_2_5 \
+    HW_COMPAT_2_5 \
+    { \
+        .driver   = "spapr-vlan", \
+        .property = "use-rx-buffer-pools", \
+        .value    = "off", \
+    },
 
-static void spapr_machine_2_3_instance_init(Object *obj)
+static void spapr_machine_2_5_instance_options(MachineState *machine)
 {
-    spapr_compat_2_3(obj);
-    spapr_machine_initfn(obj);
 }
 
-static void spapr_machine_2_2_instance_init(Object *obj)
+static void spapr_machine_2_5_class_options(MachineClass *mc)
 {
-    spapr_compat_2_2(obj);
-    spapr_machine_initfn(obj);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
+    spapr_machine_2_6_class_options(mc);
+    smc->use_ohci_by_default = true;
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5);
 }
 
-static void spapr_machine_2_1_instance_init(Object *obj)
+DEFINE_SPAPR_MACHINE(2_5, "2.5", false);
+
+/*
+ * pseries-2.4
+ */
+#define SPAPR_COMPAT_2_4 \
+        SPAPR_COMPAT_2_5 \
+        HW_COMPAT_2_4
+
+static void spapr_machine_2_4_instance_options(MachineState *machine)
 {
-    spapr_compat_2_1(obj);
-    spapr_machine_initfn(obj);
+    spapr_machine_2_5_instance_options(machine);
 }
 
-static void spapr_machine_2_1_class_init(ObjectClass *oc, void *data)
+static void spapr_machine_2_4_class_options(MachineClass *mc)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
-    static GlobalProperty compat_props[] = {
-        SPAPR_COMPAT_2_1
-        { /* end of list */ }
-    };
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
 
-    mc->name = "pseries-2.1";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.1";
-    mc->compat_props = compat_props;
+    spapr_machine_2_5_class_options(mc);
+    smc->dr_lmb_enabled = false;
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_4);
 }
 
-static const TypeInfo spapr_machine_2_1_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.1",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_1_class_init,
-    .instance_init = spapr_machine_2_1_instance_init,
-};
+DEFINE_SPAPR_MACHINE(2_4, "2.4", false);
+
+/*
+ * pseries-2.3
+ */
+#define SPAPR_COMPAT_2_3 \
+        SPAPR_COMPAT_2_4 \
+        HW_COMPAT_2_3 \
+        {\
+            .driver   = "spapr-pci-host-bridge",\
+            .property = "dynamic-reconfiguration",\
+            .value    = "off",\
+        },
 
-static void spapr_machine_2_2_class_init(ObjectClass *oc, void *data)
+static void spapr_machine_2_3_instance_options(MachineState *machine)
 {
-    static GlobalProperty compat_props[] = {
-        SPAPR_COMPAT_2_2
-        { /* end of list */ }
-    };
-    MachineClass *mc = MACHINE_CLASS(oc);
+    spapr_machine_2_4_instance_options(machine);
+    savevm_skip_section_footers();
+    global_state_set_optional();
+    savevm_skip_configuration();
+}
 
-    mc->name = "pseries-2.2";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.2";
-    mc->compat_props = compat_props;
+static void spapr_machine_2_3_class_options(MachineClass *mc)
+{
+    spapr_machine_2_4_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_3);
 }
+DEFINE_SPAPR_MACHINE(2_3, "2.3", false);
 
-static const TypeInfo spapr_machine_2_2_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.2",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_2_class_init,
-    .instance_init = spapr_machine_2_2_instance_init,
-};
+/*
+ * pseries-2.2
+ */
 
-static void spapr_machine_2_3_class_init(ObjectClass *oc, void *data)
+#define SPAPR_COMPAT_2_2 \
+        SPAPR_COMPAT_2_3 \
+        HW_COMPAT_2_2 \
+        {\
+            .driver   = TYPE_SPAPR_PCI_HOST_BRIDGE,\
+            .property = "mem_win_size",\
+            .value    = "0x20000000",\
+        },
+
+static void spapr_machine_2_2_instance_options(MachineState *machine)
 {
-    static GlobalProperty compat_props[] = {
-        SPAPR_COMPAT_2_3
-        { /* end of list */ }
-    };
-    MachineClass *mc = MACHINE_CLASS(oc);
+    spapr_machine_2_3_instance_options(machine);
+    machine->suppress_vmdesc = true;
+}
 
-    mc->name = "pseries-2.3";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.3";
-    mc->compat_props = compat_props;
+static void spapr_machine_2_2_class_options(MachineClass *mc)
+{
+    spapr_machine_2_3_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_2);
 }
+DEFINE_SPAPR_MACHINE(2_2, "2.2", false);
 
-static const TypeInfo spapr_machine_2_3_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.3",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_3_class_init,
-    .instance_init = spapr_machine_2_3_instance_init,
-};
+/*
+ * pseries-2.1
+ */
+#define SPAPR_COMPAT_2_1 \
+        SPAPR_COMPAT_2_2 \
+        HW_COMPAT_2_1
 
-static void spapr_machine_2_4_class_init(ObjectClass *oc, void *data)
+static void spapr_machine_2_1_instance_options(MachineState *machine)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
-
-    mc->name = "pseries-2.4";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.4";
-    mc->alias = "pseries";
-    mc->is_default = 1;
+    spapr_machine_2_2_instance_options(machine);
 }
 
-static const TypeInfo spapr_machine_2_4_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.4",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_4_class_init,
-};
+static void spapr_machine_2_1_class_options(MachineClass *mc)
+{
+    spapr_machine_2_2_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_1);
+}
+DEFINE_SPAPR_MACHINE(2_1, "2.1", false);
 
 static void spapr_machine_register_types(void)
 {
     type_register_static(&spapr_machine_info);
-    type_register_static(&spapr_machine_2_1_info);
-    type_register_static(&spapr_machine_2_2_info);
-    type_register_static(&spapr_machine_2_3_info);
-    type_register_static(&spapr_machine_2_4_info);
 }
 
 type_init(spapr_machine_register_types)
diff --git a/qemu/hw/ppc/spapr_drc.c b/qemu/hw/ppc/spapr_drc.c
index ee874326e..1f5f1d790 100644
--- a/qemu/hw/ppc/spapr_drc.c
+++ b/qemu/hw/ppc/spapr_drc.c
@@ -10,11 +10,16 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu/cutils.h"
 #include "hw/ppc/spapr_drc.h"
 #include "qom/object.h"
 #include "hw/qdev.h"
 #include "qapi/visitor.h"
 #include "qemu/error-report.h"
+#include "hw/ppc/spapr.h" /* for RTAS return codes */
 
 /* #define DEBUG_SPAPR_DRC */
 
@@ -32,7 +37,7 @@
 
 #define DRC_CONTAINER_PATH "/dr-connector"
 #define DRC_INDEX_TYPE_SHIFT 28
-#define DRC_INDEX_ID_MASK (~(~0 << DRC_INDEX_TYPE_SHIFT))
+#define DRC_INDEX_ID_MASK ((1ULL << DRC_INDEX_TYPE_SHIFT) - 1)
 
 static sPAPRDRConnectorTypeShift get_type_shift(sPAPRDRConnectorType type)
 {
@@ -59,13 +64,23 @@ static uint32_t get_index(sPAPRDRConnector *drc)
             (drc->id & DRC_INDEX_ID_MASK);
 }
 
-static int set_isolation_state(sPAPRDRConnector *drc,
-                               sPAPRDRIsolationState state)
+static uint32_t set_isolation_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state)
 {
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
     DPRINTFN("drc: %x, set_isolation_state: %x", get_index(drc), state);
 
+    if (state == SPAPR_DR_ISOLATION_STATE_UNISOLATED) {
+        /* cannot unisolate a non-existant resource, and, or resources
+         * which are in an 'UNUSABLE' allocation state. (PAPR 2.7, 13.5.3.5)
+         */
+        if (!drc->dev ||
+            drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
     drc->isolation_state = state;
 
     if (drc->isolation_state == SPAPR_DR_ISOLATION_STATE_ISOLATED) {
@@ -89,24 +104,35 @@ static int set_isolation_state(sPAPRDRConnector *drc,
         drc->configured = false;
     }
 
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
-static int set_indicator_state(sPAPRDRConnector *drc,
-                               sPAPRDRIndicatorState state)
+static uint32_t set_indicator_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state)
 {
     DPRINTFN("drc: %x, set_indicator_state: %x", get_index(drc), state);
     drc->indicator_state = state;
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
-static int set_allocation_state(sPAPRDRConnector *drc,
-                                sPAPRDRAllocationState state)
+static uint32_t set_allocation_state(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state)
 {
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
     DPRINTFN("drc: %x, set_allocation_state: %x", get_index(drc), state);
 
+    if (state == SPAPR_DR_ALLOCATION_STATE_USABLE) {
+        /* if there's no resource/device associated with the DRC, there's
+         * no way for us to put it in an allocation state consistent with
+         * being 'USABLE'. PAPR 2.7, 13.5.3.4 documents that this should
+         * result in an RTAS return code of -3 / "no such indicator"
+         */
+        if (!drc->dev) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
     if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI) {
         drc->allocation_state = state;
         if (drc->awaiting_release &&
@@ -116,7 +142,7 @@ static int set_allocation_state(sPAPRDRConnector *drc,
                          drc->detach_cb_opaque, NULL);
         }
     }
-    return 0;
+    return RTAS_OUT_SUCCESS;
 }
 
 static uint32_t get_type(sPAPRDRConnector *drc)
@@ -150,6 +176,12 @@ static void set_configured(sPAPRDRConnector *drc)
     drc->configured = true;
 }
 
+/* has the guest been notified of device attachment? */
+static void set_signalled(sPAPRDRConnector *drc)
+{
+    drc->signalled = true;
+}
+
 /*
  * dr-entity-sense sensor value
  * returned via get-sensor-state RTAS calls
@@ -157,10 +189,8 @@ static void set_configured(sPAPRDRConnector *drc)
  * based on the current allocation/indicator/power states
  * for the DR connector.
  */
-static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
+static uint32_t entity_sense(sPAPRDRConnector *drc, sPAPRDREntitySense *state)
 {
-    sPAPRDREntitySense state;
-
     if (drc->dev) {
         if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI &&
             drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
@@ -169,7 +199,7 @@ static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
              * Otherwise, report the state as USABLE/PRESENT,
              * as we would for PCI.
              */
-            state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
         } else {
             /* this assumes all PCI devices are assigned to
              * a 'live insertion' power domain, where QEMU
@@ -177,39 +207,39 @@ static sPAPRDREntitySense entity_sense(sPAPRDRConnector *drc)
              * to the guest. present, non-PCI resources are
              * unaffected by power state.
              */
-            state = SPAPR_DR_ENTITY_SENSE_PRESENT;
+            *state = SPAPR_DR_ENTITY_SENSE_PRESENT;
         }
     } else {
         if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
             /* PCI devices, and only PCI devices, use EMPTY
              * in cases where we'd otherwise use UNUSABLE
              */
-            state = SPAPR_DR_ENTITY_SENSE_EMPTY;
+            *state = SPAPR_DR_ENTITY_SENSE_EMPTY;
         } else {
-            state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
         }
     }
 
     DPRINTFN("drc: %x, entity_sense: %x", get_index(drc), state);
-    return state;
+    return RTAS_OUT_SUCCESS;
 }
 
-static void prop_get_index(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void prop_get_index(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
     uint32_t value = (uint32_t)drck->get_index(drc);
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
-static void prop_get_type(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void prop_get_type(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
     uint32_t value = (uint32_t)drck->get_type(drc);
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static char *prop_get_name(Object *obj, Error **errp)
@@ -219,23 +249,31 @@ static char *prop_get_name(Object *obj, Error **errp)
     return g_strdup(drck->get_name(drc));
 }
 
-static void prop_get_entity_sense(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void prop_get_entity_sense(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    uint32_t value = (uint32_t)drck->entity_sense(drc);
-    visit_type_uint32(v, &value, name, errp);
+    uint32_t value;
+
+    drck->entity_sense(drc, &value);
+    visit_type_uint32(v, name, &value, errp);
 }
 
-static void prop_get_fdt(Object *obj, Visitor *v, void *opaque,
-                        const char *name, Error **errp)
+static void prop_get_fdt(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    Error *err = NULL;
     int fdt_offset_next, fdt_offset, fdt_depth;
     void *fdt;
 
     if (!drc->fdt) {
+        visit_start_struct(v, name, NULL, 0, &err);
+        if (!err) {
+            visit_end_struct(v, &err);
+        }
+        error_propagate(errp, err);
         return;
     }
 
@@ -254,24 +292,39 @@ static void prop_get_fdt(Object *obj, Visitor *v, void *opaque,
         case FDT_BEGIN_NODE:
             fdt_depth++;
             name = fdt_get_name(fdt, fdt_offset, &name_len);
-            visit_start_struct(v, NULL, NULL, name, 0, NULL);
+            visit_start_struct(v, name, NULL, 0, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             break;
         case FDT_END_NODE:
             /* shouldn't ever see an FDT_END_NODE before FDT_BEGIN_NODE */
             g_assert(fdt_depth > 0);
-            visit_end_struct(v, NULL);
+            visit_end_struct(v, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             fdt_depth--;
             break;
         case FDT_PROP: {
             int i;
             prop = fdt_get_property_by_offset(fdt, fdt_offset, &prop_len);
             name = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
-            visit_start_list(v, name, NULL);
+            visit_start_list(v, name, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
             for (i = 0; i < prop_len; i++) {
-                visit_type_uint8(v, (uint8_t *)&prop->data[i], NULL, NULL);
-
+                visit_type_uint8(v, NULL, (uint8_t *)&prop->data[i], &err);
+                if (err) {
+                    error_propagate(errp, err);
+                    return;
+                }
             }
-            visit_end_list(v, NULL);
+            visit_end_list(v);
             break;
         }
         default:
@@ -310,7 +363,18 @@ static void attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt,
     drc->dev = d;
     drc->fdt = fdt;
     drc->fdt_start_offset = fdt_start_offset;
-    drc->configured = false;
+    drc->configured = coldplug;
+    /* 'logical' DR resources such as memory/cpus are in some cases treated
+     * as a pool of resources from which the guest is free to choose from
+     * based on only a count. for resources that can be assigned in this
+     * fashion, we must assume the resource is signalled immediately
+     * since a single hotplug request might make an arbitrary number of
+     * such attached resources available to the guest, as opposed to
+     * 'physical' DR resources such as PCI where each device/resource is
+     * signalled individually.
+     */
+    drc->signalled = (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI)
+                     ? true : coldplug;
 
     object_property_add_link(OBJECT(drc), "device",
                              object_get_typename(OBJECT(drc->dev)),
@@ -327,6 +391,26 @@ static void detach(sPAPRDRConnector *drc, DeviceState *d,
     drc->detach_cb = detach_cb;
     drc->detach_cb_opaque = detach_cb_opaque;
 
+    /* if we've signalled device presence to the guest, or if the guest
+     * has gone ahead and configured the device (via manually-executed
+     * device add via drmgr in guest, namely), we need to wait
+     * for the guest to quiesce the device before completing detach.
+     * Otherwise, we can assume the guest hasn't seen it and complete the
+     * detach immediately. Note that there is a small race window
+     * just before, or during, configuration, which is this context
+     * refers mainly to fetching the device tree via RTAS.
+     * During this window the device access will be arbitrated by
+     * associated DRC, which will simply fail the RTAS calls as invalid.
+     * This is recoverable within guest and current implementations of
+     * drmgr should be able to cope.
+     */
+    if (!drc->signalled && !drc->configured) {
+        /* if the guest hasn't seen the device we can't rely on it to
+         * set it back to an isolated state via RTAS, so do it here manually
+         */
+        drc->isolation_state = SPAPR_DR_ISOLATION_STATE_ISOLATED;
+    }
+
     if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_ISOLATED) {
         DPRINTFN("awaiting transition to isolated state before removal");
         drc->awaiting_release = true;
@@ -365,6 +449,7 @@ static void reset(DeviceState *d)
 {
     sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(d);
     sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDREntitySense state;
 
     DPRINTFN("drc reset: %x", drck->get_index(drc));
     /* immediately upon reset we can safely assume DRCs whose devices
@@ -392,6 +477,11 @@ static void reset(DeviceState *d)
             drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_UNUSABLE);
         }
     }
+
+    drck->entity_sense(drc, &state);
+    if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+        drck->set_signalled(drc);
+    }
 }
 
 static void realize(DeviceState *d, Error **errp)
@@ -418,8 +508,7 @@ static void realize(DeviceState *d, Error **errp)
     object_property_add_alias(root_container, link_name,
                               drc->owner, child_name, &err);
     if (err) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         object_unref(OBJECT(drc));
     }
     g_free(child_name);
@@ -439,8 +528,7 @@ static void unrealize(DeviceState *d, Error **errp)
     snprintf(name, sizeof(name), "%x", drck->get_index(drc));
     object_property_del(root_container, name, &err);
     if (err) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         object_unref(OBJECT(drc));
     }
 }
@@ -451,14 +539,17 @@ sPAPRDRConnector *spapr_dr_connector_new(Object *owner,
 {
     sPAPRDRConnector *drc =
         SPAPR_DR_CONNECTOR(object_new(TYPE_SPAPR_DR_CONNECTOR));
+    char *prop_name;
 
     g_assert(type);
 
     drc->type = type;
     drc->id = id;
     drc->owner = owner;
-    object_property_add_child(owner, "dr-connector[*]", OBJECT(drc), NULL);
+    prop_name = g_strdup_printf("dr-connector[%"PRIu32"]", get_index(drc));
+    object_property_add_child(owner, prop_name, OBJECT(drc), NULL);
     object_property_set_bool(OBJECT(drc), true, "realized", NULL);
+    g_free(prop_name);
 
     /* human-readable name for a DRC to encode into the DT
      * description. this is mainly only used within a guest in place
@@ -549,6 +640,11 @@ static void spapr_dr_connector_class_init(ObjectClass *k, void *data)
     drck->attach = attach;
     drck->detach = detach;
     drck->release_pending = release_pending;
+    drck->set_signalled = set_signalled;
+    /*
+     * Reason: it crashes FIXME find and document the real reason
+     */
+    dk->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo spapr_dr_connector_info = {
@@ -632,6 +728,7 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
 {
     Object *root_container;
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
     uint32_t drc_count = 0;
     GArray *drc_indexes, *drc_power_domains;
     GString *drc_names, *drc_types;
@@ -655,7 +752,8 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
      */
     root_container = container_get(object_get_root(), DRC_CONTAINER_PATH);
 
-    QTAILQ_FOREACH(prop, &root_container->properties, node) {
+    object_property_iter_init(&iter, root_container);
+    while ((prop = object_property_iter_next(&iter))) {
         Object *obj;
         sPAPRDRConnector *drc;
         sPAPRDRConnectorClass *drck;
diff --git a/qemu/hw/ppc/spapr_events.c b/qemu/hw/ppc/spapr_events.c
index f626eb7b3..049fb1b32 100644
--- a/qemu/hw/ppc/spapr_events.c
+++ b/qemu/hw/ppc/spapr_events.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
@@ -35,7 +37,8 @@
 #include "hw/pci/pci.h"
 #include "hw/pci-host/spapr.h"
 #include "hw/ppc/spapr_drc.h"
-
+#include "qemu/help_option.h"
+#include "qemu/bcd.h"
 #include <libfdt.h>
 
 struct rtas_error_log {
@@ -386,7 +389,16 @@ static void spapr_powerdown_req(Notifier *n, void *opaque)
     qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->check_exception_irq));
 }
 
-static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
+static void spapr_hotplug_set_signalled(uint32_t drc_index)
+{
+    sPAPRDRConnector *drc = spapr_dr_connector_by_index(drc_index);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->set_signalled(drc);
+}
+
+static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
+                                    sPAPRDRConnectorType drc_type,
+                                    uint32_t drc)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     struct hp_log_full *new_hp;
@@ -395,8 +407,6 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
     struct rtas_event_log_v6_maina *maina;
     struct rtas_event_log_v6_mainb *mainb;
     struct rtas_event_log_v6_hp *hp;
-    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    sPAPRDRConnectorType drc_type = drck->get_type(drc);
 
     new_hp = g_malloc0(sizeof(struct hp_log_full));
     hdr = &new_hp->hdr;
@@ -427,13 +437,17 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
     hp->hdr.section_length = cpu_to_be16(sizeof(*hp));
     hp->hdr.section_version = 1; /* includes extended modifier */
     hp->hotplug_action = hp_action;
-
+    hp->hotplug_identifier = hp_id;
 
     switch (drc_type) {
     case SPAPR_DR_CONNECTOR_TYPE_PCI:
-        hp->drc.index = cpu_to_be32(drck->get_index(drc));
-        hp->hotplug_identifier = RTAS_LOG_V6_HP_ID_DRC_INDEX;
         hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_PCI;
+        if (hp->hotplug_action == RTAS_LOG_V6_HP_ACTION_ADD) {
+            spapr_hotplug_set_signalled(drc);
+        }
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_LMB:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY;
         break;
     default:
         /* we shouldn't be signaling hotplug events for resources
@@ -443,19 +457,49 @@ static void spapr_hotplug_req_event(sPAPRDRConnector *drc, uint8_t hp_action)
         return;
     }
 
+    if (hp_id == RTAS_LOG_V6_HP_ID_DRC_COUNT) {
+        hp->drc.count = cpu_to_be32(drc);
+    } else if (hp_id == RTAS_LOG_V6_HP_ID_DRC_INDEX) {
+        hp->drc.index = cpu_to_be32(drc);
+    }
+
     rtas_event_log_queue(RTAS_LOG_TYPE_HOTPLUG, new_hp, true);
 
     qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->check_exception_irq));
 }
 
-void spapr_hotplug_req_add_event(sPAPRDRConnector *drc)
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32_t index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, index);
+}
+
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32_t index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, index);
+}
+
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count)
 {
-    spapr_hotplug_req_event(drc, RTAS_LOG_V6_HP_ACTION_ADD);
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, count);
 }
 
-void spapr_hotplug_req_remove_event(sPAPRDRConnector *drc)
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count)
 {
-    spapr_hotplug_req_event(drc, RTAS_LOG_V6_HP_ACTION_REMOVE);
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, count);
 }
 
 static void check_exception(PowerPCCPU *cpu, sPAPRMachineState *spapr,
@@ -556,7 +600,8 @@ out_no_events:
 void spapr_events_init(sPAPRMachineState *spapr)
 {
     QTAILQ_INIT(&spapr->pending_events);
-    spapr->check_exception_irq = xics_alloc(spapr->icp, 0, 0, false);
+    spapr->check_exception_irq = xics_alloc(spapr->icp, 0, 0, false,
+                                            &error_fatal);
     spapr->epow_notifier.notify = spapr_powerdown_req;
     qemu_register_powerdown_notifier(&spapr->epow_notifier);
     spapr_rtas_register(RTAS_CHECK_EXCEPTION, "check-exception",
diff --git a/qemu/hw/ppc/spapr_hcall.c b/qemu/hw/ppc/spapr_hcall.c
index 652ddf6e3..8f40602a5 100644
--- a/qemu/hw/ppc/spapr_hcall.c
+++ b/qemu/hw/ppc/spapr_hcall.c
@@ -1,3 +1,5 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "cpu.h"
 #include "helper_regs.h"
@@ -37,40 +39,10 @@ static void set_spr(CPUState *cs, int spr, target_ulong value,
     run_on_cpu(cs, do_spr_sync, &s);
 }
 
-static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
-                                     target_ulong pte_index)
+static bool has_spr(PowerPCCPU *cpu, int spr)
 {
-    target_ulong rb, va_low;
-
-    rb = (v & ~0x7fULL) << 16; /* AVA field */
-    va_low = pte_index >> 3;
-    if (v & HPTE64_V_SECONDARY) {
-        va_low = ~va_low;
-    }
-    /* xor vsid from AVA */
-    if (!(v & HPTE64_V_1TB_SEG)) {
-        va_low ^= v >> 12;
-    } else {
-        va_low ^= v >> 24;
-    }
-    va_low &= 0x7ff;
-    if (v & HPTE64_V_LARGE) {
-        rb |= 1;                         /* L field */
-#if 0 /* Disable that P7 specific bit for now */
-        if (r & 0xff000) {
-            /* non-16MB large page, must be 64k */
-            /* (masks depend on page size) */
-            rb |= 0x1000;                /* page encoding in LP field */
-            rb |= (va_low & 0x7f) << 16; /* 7b of VA in AVA/LP field */
-            rb |= (va_low & 0xfe);       /* AVAL field */
-        }
-#endif
-    } else {
-        /* 4kB page */
-        rb |= (va_low & 0x7ff) << 12;   /* remaining 11b of AVA */
-    }
-    rb |= (v >> 54) & 0x300;            /* B field */
-    return rb;
+    /* We can test whether the SPR is defined by checking for a valid name */
+    return cpu->env.spr_cb[spr].name != NULL;
 }
 
 static inline bool valid_pte_index(CPUPPCState *env, target_ulong pte_index)
@@ -84,42 +56,44 @@ static inline bool valid_pte_index(CPUPPCState *env, target_ulong pte_index)
     return true;
 }
 
+static bool is_ram_address(sPAPRMachineState *spapr, hwaddr addr)
+{
+    MachineState *machine = MACHINE(spapr);
+    MemoryHotplugState *hpms = &spapr->hotplug_memory;
+
+    if (addr < machine->ram_size) {
+        return true;
+    }
+    if ((addr >= hpms->base)
+        && ((addr - hpms->base) < memory_region_size(&hpms->mr))) {
+        return true;
+    }
+
+    return false;
+}
+
 static target_ulong h_enter(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                             target_ulong opcode, target_ulong *args)
 {
-    MachineState *machine = MACHINE(spapr);
     CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong pteh = args[2];
     target_ulong ptel = args[3];
-    target_ulong page_shift = 12;
+    unsigned apshift, spshift;
     target_ulong raddr;
     target_ulong index;
     uint64_t token;
 
-    /* only handle 4k and 16M pages for now */
-    if (pteh & HPTE64_V_LARGE) {
-#if 0 /* We don't support 64k pages yet */
-        if ((ptel & 0xf000) == 0x1000) {
-            /* 64k page */
-        } else
-#endif
-        if ((ptel & 0xff000) == 0) {
-            /* 16M page */
-            page_shift = 24;
-            /* lowest AVA bit must be 0 for 16M pages */
-            if (pteh & 0x80) {
-                return H_PARAMETER;
-            }
-        } else {
-            return H_PARAMETER;
-        }
+    apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel, &spshift);
+    if (!apshift) {
+        /* Bad page size encoding */
+        return H_PARAMETER;
     }
 
-    raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << page_shift) - 1);
+    raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
 
-    if (raddr < machine->ram_size) {
+    if (is_ram_address(spapr, raddr)) {
         /* Regular RAM - should have WIMG=0010 */
         if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
             return H_PARAMETER;
@@ -145,24 +119,24 @@ static target_ulong h_enter(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         pte_index &= ~7ULL;
         token = ppc_hash64_start_access(cpu, pte_index);
         for (; index < 8; index++) {
-            if ((ppc_hash64_load_hpte0(env, token, index) & HPTE64_V_VALID) == 0) {
+            if (!(ppc_hash64_load_hpte0(cpu, token, index) & HPTE64_V_VALID)) {
                 break;
             }
         }
-        ppc_hash64_stop_access(token);
+        ppc_hash64_stop_access(cpu, token);
         if (index == 8) {
             return H_PTEG_FULL;
         }
     } else {
         token = ppc_hash64_start_access(cpu, pte_index);
-        if (ppc_hash64_load_hpte0(env, token, 0) & HPTE64_V_VALID) {
-            ppc_hash64_stop_access(token);
+        if (ppc_hash64_load_hpte0(cpu, token, 0) & HPTE64_V_VALID) {
+            ppc_hash64_stop_access(cpu, token);
             return H_PTEG_FULL;
         }
-        ppc_hash64_stop_access(token);
+        ppc_hash64_stop_access(cpu, token);
     }
 
-    ppc_hash64_store_hpte(env, pte_index + index,
+    ppc_hash64_store_hpte(cpu, pte_index + index,
                           pteh | HPTE64_V_HPTE_DIRTY, ptel);
 
     args[0] = pte_index + index;
@@ -176,22 +150,23 @@ typedef enum {
     REMOVE_HW = 3,
 } RemoveResult;
 
-static RemoveResult remove_hpte(CPUPPCState *env, target_ulong ptex,
+static RemoveResult remove_hpte(PowerPCCPU *cpu, target_ulong ptex,
                                 target_ulong avpn,
                                 target_ulong flags,
                                 target_ulong *vp, target_ulong *rp)
 {
+    CPUPPCState *env = &cpu->env;
     uint64_t token;
-    target_ulong v, r, rb;
+    target_ulong v, r;
 
     if (!valid_pte_index(env, ptex)) {
         return REMOVE_PARM;
     }
 
-    token = ppc_hash64_start_access(ppc_env_get_cpu(env), ptex);
-    v = ppc_hash64_load_hpte0(env, token, 0);
-    r = ppc_hash64_load_hpte1(env, token, 0);
-    ppc_hash64_stop_access(token);
+    token = ppc_hash64_start_access(cpu, ptex);
+    v = ppc_hash64_load_hpte0(cpu, token, 0);
+    r = ppc_hash64_load_hpte1(cpu, token, 0);
+    ppc_hash64_stop_access(cpu, token);
 
     if ((v & HPTE64_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
@@ -200,22 +175,20 @@ static RemoveResult remove_hpte(CPUPPCState *env, target_ulong ptex,
     }
     *vp = v;
     *rp = r;
-    ppc_hash64_store_hpte(env, ptex, HPTE64_V_HPTE_DIRTY, 0);
-    rb = compute_tlbie_rb(v, r, ptex);
-    ppc_tlb_invalidate_one(env, rb);
+    ppc_hash64_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
+    ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
     return REMOVE_SUCCESS;
 }
 
 static target_ulong h_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                              target_ulong opcode, target_ulong *args)
 {
-    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
     RemoveResult ret;
 
-    ret = remove_hpte(env, pte_index, avpn, flags,
+    ret = remove_hpte(cpu, pte_index, avpn, flags,
                       &args[0], &args[1]);
 
     switch (ret) {
@@ -256,7 +229,6 @@ static target_ulong h_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                   target_ulong opcode, target_ulong *args)
 {
-    CPUPPCState *env = &cpu->env;
     int i;
 
     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
@@ -278,7 +250,7 @@ static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
             return H_PARAMETER;
         }
 
-        ret = remove_hpte(env, *tsh & H_BULK_REMOVE_PTEX, tsl,
+        ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
                           (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
                           &v, &r);
 
@@ -308,16 +280,16 @@ static target_ulong h_protect(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
     uint64_t token;
-    target_ulong v, r, rb;
+    target_ulong v, r;
 
     if (!valid_pte_index(env, pte_index)) {
         return H_PARAMETER;
     }
 
     token = ppc_hash64_start_access(cpu, pte_index);
-    v = ppc_hash64_load_hpte0(env, token, 0);
-    r = ppc_hash64_load_hpte1(env, token, 0);
-    ppc_hash64_stop_access(token);
+    v = ppc_hash64_load_hpte0(cpu, token, 0);
+    r = ppc_hash64_load_hpte1(cpu, token, 0);
+    ppc_hash64_stop_access(cpu, token);
 
     if ((v & HPTE64_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
@@ -329,12 +301,11 @@ static target_ulong h_protect(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     r |= (flags << 55) & HPTE64_R_PP0;
     r |= (flags << 48) & HPTE64_R_KEY_HI;
     r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
-    rb = compute_tlbie_rb(v, r, pte_index);
-    ppc_hash64_store_hpte(env, pte_index,
+    ppc_hash64_store_hpte(cpu, pte_index,
                           (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
-    ppc_tlb_invalidate_one(env, rb);
+    ppc_hash64_tlb_flush_hpte(cpu, pte_index, v, r);
     /* Don't need a memory barrier, due to qemu's global lock */
-    ppc_hash64_store_hpte(env, pte_index, v | HPTE64_V_HPTE_DIRTY, r);
+    ppc_hash64_store_hpte(cpu, pte_index, v | HPTE64_V_HPTE_DIRTY, r);
     return H_SUCCESS;
 }
 
@@ -368,11 +339,111 @@ static target_ulong h_read(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     return H_SUCCESS;
 }
 
+static target_ulong h_set_sprg0(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    cpu_synchronize_state(CPU(cpu));
+    cpu->env.spr[SPR_SPRG0] = args[0];
+
+    return H_SUCCESS;
+}
+
 static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                target_ulong opcode, target_ulong *args)
 {
-    /* FIXME: actually implement this */
-    return H_HARDWARE;
+    if (!has_spr(cpu, SPR_DABR)) {
+        return H_HARDWARE;              /* DABR register not available */
+    }
+    cpu_synchronize_state(CPU(cpu));
+
+    if (has_spr(cpu, SPR_DABRX)) {
+        cpu->env.spr[SPR_DABRX] = 0x3;  /* Use Problem and Privileged state */
+    } else if (!(args[0] & 0x4)) {      /* Breakpoint Translation set? */
+        return H_RESERVED_DABR;
+    }
+
+    cpu->env.spr[SPR_DABR] = args[0];
+    return H_SUCCESS;
+}
+
+static target_ulong h_set_xdabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    target_ulong dabrx = args[1];
+
+    if (!has_spr(cpu, SPR_DABR) || !has_spr(cpu, SPR_DABRX)) {
+        return H_HARDWARE;
+    }
+
+    if ((dabrx & ~0xfULL) != 0 || (dabrx & H_DABRX_HYPERVISOR) != 0
+        || (dabrx & (H_DABRX_KERNEL | H_DABRX_USER)) == 0) {
+        return H_PARAMETER;
+    }
+
+    cpu_synchronize_state(CPU(cpu));
+    cpu->env.spr[SPR_DABRX] = dabrx;
+    cpu->env.spr[SPR_DABR] = args[0];
+
+    return H_SUCCESS;
+}
+
+static target_ulong h_page_init(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    target_ulong flags = args[0];
+    hwaddr dst = args[1];
+    hwaddr src = args[2];
+    hwaddr len = TARGET_PAGE_SIZE;
+    uint8_t *pdst, *psrc;
+    target_long ret = H_SUCCESS;
+
+    if (flags & ~(H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE
+                  | H_COPY_PAGE | H_ZERO_PAGE)) {
+        qemu_log_mask(LOG_UNIMP, "h_page_init: Bad flags (" TARGET_FMT_lx "\n",
+                      flags);
+        return H_PARAMETER;
+    }
+
+    /* Map-in destination */
+    if (!is_ram_address(spapr, dst) || (dst & ~TARGET_PAGE_MASK) != 0) {
+        return H_PARAMETER;
+    }
+    pdst = cpu_physical_memory_map(dst, &len, 1);
+    if (!pdst || len != TARGET_PAGE_SIZE) {
+        return H_PARAMETER;
+    }
+
+    if (flags & H_COPY_PAGE) {
+        /* Map-in source, copy to destination, and unmap source again */
+        if (!is_ram_address(spapr, src) || (src & ~TARGET_PAGE_MASK) != 0) {
+            ret = H_PARAMETER;
+            goto unmap_out;
+        }
+        psrc = cpu_physical_memory_map(src, &len, 0);
+        if (!psrc || len != TARGET_PAGE_SIZE) {
+            ret = H_PARAMETER;
+            goto unmap_out;
+        }
+        memcpy(pdst, psrc, len);
+        cpu_physical_memory_unmap(psrc, len, 0, len);
+    } else if (flags & H_ZERO_PAGE) {
+        memset(pdst, 0, len);          /* Just clear the destination page */
+    }
+
+    if (kvm_enabled() && (flags & H_ICACHE_SYNCHRONIZE) != 0) {
+        kvmppc_dcbst_range(cpu, pdst, len);
+    }
+    if (flags & (H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE)) {
+        if (kvm_enabled()) {
+            kvmppc_icbi_range(cpu, pdst, len);
+        } else {
+            tb_flush(CPU(cpu));
+        }
+    }
+
+unmap_out:
+    cpu_physical_memory_unmap(pdst, TARGET_PAGE_SIZE, 1, len);
+    return ret;
 }
 
 #define FLAGS_REGISTER_VPA         0x0000200000000000ULL
@@ -753,7 +824,6 @@ static target_ulong h_set_mode_resource_addr_trans_mode(PowerPCCPU *cpu,
 {
     CPUState *cs;
     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
-    target_ulong prefix;
 
     if (!(pcc->insns_flags2 & PPC2_ISA207S)) {
         return H_P2;
@@ -765,25 +835,12 @@ static target_ulong h_set_mode_resource_addr_trans_mode(PowerPCCPU *cpu,
         return H_P4;
     }
 
-    switch (mflags) {
-    case H_SET_MODE_ADDR_TRANS_NONE:
-        prefix = 0;
-        break;
-    case H_SET_MODE_ADDR_TRANS_0001_8000:
-        prefix = 0x18000;
-        break;
-    case H_SET_MODE_ADDR_TRANS_C000_0000_0000_4000:
-        prefix = 0xC000000000004000ULL;
-        break;
-    default:
+    if (mflags == AIL_RESERVED) {
         return H_UNSUPPORTED_FLAG;
     }
 
     CPU_FOREACH(cs) {
-        CPUPPCState *env = &POWERPC_CPU(cpu)->env;
-
         set_spr(cs, SPR_LPCR, mflags << LPCR_AIL_SHIFT, LPCR_AIL);
-        env->excp_prefix = prefix;
     }
 
     return H_SUCCESS;
@@ -808,10 +865,36 @@ static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     return ret;
 }
 
+/*
+ * Return the offset to the requested option vector @vector in the
+ * option vector table @table.
+ */
+static target_ulong cas_get_option_vector(int vector, target_ulong table)
+{
+    int i;
+    char nr_vectors, nr_entries;
+
+    if (!table) {
+        return 0;
+    }
+
+    nr_vectors = (ldl_phys(&address_space_memory, table) >> 24) + 1;
+    if (!vector || vector > nr_vectors) {
+        return 0;
+    }
+    table++; /* skip nr option vectors */
+
+    for (i = 0; i < vector - 1; i++) {
+        nr_entries = ldl_phys(&address_space_memory, table) >> 24;
+        table += nr_entries + 2;
+    }
+    return table;
+}
+
 typedef struct {
     PowerPCCPU *cpu;
     uint32_t cpu_version;
-    int ret;
+    Error *err;
 } SetCompatState;
 
 static void do_set_compat(void *arg)
@@ -819,7 +902,7 @@ static void do_set_compat(void *arg)
     SetCompatState *s = arg;
 
     cpu_synchronize_state(CPU(s->cpu));
-    s->ret = ppc_set_compat(s->cpu, s->cpu_version);
+    ppc_set_compat(s->cpu, s->cpu_version, &s->err);
 }
 
 #define get_compat_level(cpuver) ( \
@@ -828,27 +911,31 @@ static void do_set_compat(void *arg)
     ((cpuver) == CPU_POWERPC_LOGICAL_2_06_PLUS) ? 2061 : \
     ((cpuver) == CPU_POWERPC_LOGICAL_2_07) ? 2070 : 0)
 
+#define OV5_DRCONF_MEMORY 0x20
+
 static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
                                                   sPAPRMachineState *spapr,
                                                   target_ulong opcode,
                                                   target_ulong *args)
 {
-    target_ulong list = args[0];
+    target_ulong list = ppc64_phys_to_real(args[0]);
+    target_ulong ov_table, ov5;
     PowerPCCPUClass *pcc_ = POWERPC_CPU_GET_CLASS(cpu_);
     CPUState *cs;
-    bool cpu_match = false;
+    bool cpu_match = false, cpu_update = true, memory_update = false;
     unsigned old_cpu_version = cpu_->cpu_version;
     unsigned compat_lvl = 0, cpu_version = 0;
     unsigned max_lvl = get_compat_level(cpu_->max_compat);
     int counter;
+    char ov5_byte2;
 
     /* Parse PVR list */
     for (counter = 0; counter < 512; ++counter) {
         uint32_t pvr, pvr_mask;
 
-        pvr_mask = rtas_ld(list, 0);
+        pvr_mask = ldl_be_phys(&address_space_memory, list);
         list += 4;
-        pvr = rtas_ld(list, 0);
+        pvr = ldl_be_phys(&address_space_memory, list);
         list += 4;
 
         trace_spapr_cas_pvr_try(pvr);
@@ -890,8 +977,6 @@ static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
         }
     }
 
-    /* For the future use: here @list points to the first capability */
-
     /* Parsing finished */
     trace_spapr_cas_pvr(cpu_->cpu_version, cpu_match,
                         cpu_version, pcc_->pcr_mask);
@@ -902,27 +987,38 @@ static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
             SetCompatState s = {
                 .cpu = POWERPC_CPU(cs),
                 .cpu_version = cpu_version,
-                .ret = 0
+                .err = NULL,
             };
 
             run_on_cpu(cs, do_set_compat, &s);
 
-            if (s.ret < 0) {
-                fprintf(stderr, "Unable to set compatibility mode\n");
+            if (s.err) {
+                error_report_err(s.err);
                 return H_HARDWARE;
             }
         }
     }
 
     if (!cpu_version) {
-        return H_SUCCESS;
+        cpu_update = false;
     }
 
-    if (!list) {
+    /* For the future use: here @ov_table points to the first option vector */
+    ov_table = list;
+
+    ov5 = cas_get_option_vector(5, ov_table);
+    if (!ov5) {
         return H_SUCCESS;
     }
 
-    if (spapr_h_cas_compose_response(spapr, args[1], args[2])) {
+    /* @list now points to OV 5 */
+    ov5_byte2 = ldub_phys(&address_space_memory, ov5 + 2);
+    if (ov5_byte2 & OV5_DRCONF_MEMORY) {
+        memory_update = true;
+    }
+
+    if (spapr_h_cas_compose_response(spapr, args[1], args[2],
+                                     cpu_update, memory_update)) {
         qemu_system_reset_request();
     }
 
@@ -971,7 +1067,8 @@ target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
         }
     }
 
-    hcall_dprintf("Unimplemented hcall 0x" TARGET_FMT_lx "\n", opcode);
+    qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x" TARGET_FMT_lx "\n",
+                  opcode);
     return H_FUNCTION;
 }
 
@@ -986,13 +1083,17 @@ static void hypercall_register_types(void)
     /* hcall-bulk */
     spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
 
-    /* hcall-dabr */
-    spapr_register_hypercall(H_SET_DABR, h_set_dabr);
-
     /* hcall-splpar */
     spapr_register_hypercall(H_REGISTER_VPA, h_register_vpa);
     spapr_register_hypercall(H_CEDE, h_cede);
 
+    /* processor register resource access h-calls */
+    spapr_register_hypercall(H_SET_SPRG0, h_set_sprg0);
+    spapr_register_hypercall(H_SET_DABR, h_set_dabr);
+    spapr_register_hypercall(H_SET_XDABR, h_set_xdabr);
+    spapr_register_hypercall(H_PAGE_INIT, h_page_init);
+    spapr_register_hypercall(H_SET_MODE, h_set_mode);
+
     /* "debugger" hcalls (also used by SLOF). Note: We do -not- differenciate
      * here between the "CI" and the "CACHE" variants, they will use whatever
      * mapping attributes qemu is using. When using KVM, the kernel will
@@ -1009,8 +1110,6 @@ static void hypercall_register_types(void)
     /* qemu/KVM-PPC specific hcalls */
     spapr_register_hypercall(KVMPPC_H_RTAS, h_rtas);
 
-    spapr_register_hypercall(H_SET_MODE, h_set_mode);
-
     /* ibm,client-architecture-support support */
     spapr_register_hypercall(KVMPPC_H_CAS, h_client_architecture_support);
 }
diff --git a/qemu/hw/ppc/spapr_iommu.c b/qemu/hw/ppc/spapr_iommu.c
index f61504e0c..7dd458846 100644
--- a/qemu/hw/ppc/spapr_iommu.c
+++ b/qemu/hw/ppc/spapr_iommu.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/kvm.h"
 #include "hw/qdev.h"
@@ -146,7 +147,7 @@ static int spapr_tce_table_realize(DeviceState *dev)
         tcet->table = kvmppc_create_spapr_tce(tcet->liobn,
                                               window_size,
                                               &tcet->fd,
-                                              tcet->vfio_accel);
+                                              tcet->need_vfio);
     }
 
     if (!tcet->table) {
@@ -168,11 +169,43 @@ static int spapr_tce_table_realize(DeviceState *dev)
     return 0;
 }
 
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio)
+{
+    size_t table_size = tcet->nb_table * sizeof(uint64_t);
+    void *newtable;
+
+    if (need_vfio == tcet->need_vfio) {
+        /* Nothing to do */
+        return;
+    }
+
+    if (!need_vfio) {
+        /* FIXME: We don't support transition back to KVM accelerated
+         * TCEs yet */
+        return;
+    }
+
+    tcet->need_vfio = true;
+
+    if (tcet->fd < 0) {
+        /* Table is already in userspace, nothing to be do */
+        return;
+    }
+
+    newtable = g_malloc(table_size);
+    memcpy(newtable, tcet->table, table_size);
+
+    kvmppc_remove_spapr_tce(tcet->table, tcet->fd, tcet->nb_table);
+
+    tcet->fd = -1;
+    tcet->table = newtable;
+}
+
 sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
                                    uint64_t bus_offset,
                                    uint32_t page_shift,
                                    uint32_t nb_table,
-                                   bool vfio_accel)
+                                   bool need_vfio)
 {
     sPAPRTCETable *tcet;
     char tmp[64];
@@ -192,7 +225,7 @@ sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
     tcet->bus_offset = bus_offset;
     tcet->page_shift = page_shift;
     tcet->nb_table = nb_table;
-    tcet->vfio_accel = vfio_accel;
+    tcet->need_vfio = need_vfio;
 
     snprintf(tmp, sizeof(tmp), "tce-table-%x", liobn);
     object_property_add_child(OBJECT(owner), tmp, OBJECT(tcet), NULL);
diff --git a/qemu/hw/ppc/spapr_pci.c b/qemu/hw/ppc/spapr_pci.c
index a8f79d800..573e635bf 100644
--- a/qemu/hw/ppc/spapr_pci.c
+++ b/qemu/hw/ppc/spapr_pci.c
@@ -22,6 +22,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
@@ -41,6 +45,8 @@
 #include "hw/ppc/spapr_drc.h"
 #include "sysemu/device_tree.h"
 
+#include "hw/vfio/vfio.h"
+
 /* Copied from the kernel arch/powerpc/platforms/pseries/msi.c */
 #define RTAS_QUERY_FN           0
 #define RTAS_CHANGE_FN          1
@@ -140,7 +146,7 @@ static void rtas_ibm_read_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
 
@@ -206,7 +212,7 @@ static void rtas_ibm_write_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     val = rtas_ld(args, 4);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
@@ -269,16 +275,17 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                 target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int func = rtas_ld(args, 3);
     unsigned int req_num = rtas_ld(args, 4); /* 0 == remove all */
     unsigned int seq_num = rtas_ld(args, 5);
     unsigned int ret_intr_type;
-    unsigned int irq, max_irqs = 0, num = 0;
+    unsigned int irq, max_irqs = 0;
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
     spapr_pci_msi *msi;
     int *config_addr_key;
+    Error *err = NULL;
 
     switch (func) {
     case RTAS_CHANGE_MSI_FN:
@@ -304,9 +311,10 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
 
+    msi = (spapr_pci_msi *) g_hash_table_lookup(phb->msi, &config_addr);
+
     /* Releasing MSIs */
     if (!req_num) {
-        msi = (spapr_pci_msi *) g_hash_table_lookup(phb->msi, &config_addr);
         if (!msi) {
             trace_spapr_pci_msi("Releasing wrong config", config_addr);
             rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
@@ -315,10 +323,10 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 
         xics_free(spapr->icp, msi->first_irq, msi->num);
         if (msi_present(pdev)) {
-            spapr_msi_setmsg(pdev, 0, false, 0, num);
+            spapr_msi_setmsg(pdev, 0, false, 0, 0);
         }
         if (msix_present(pdev)) {
-            spapr_msi_setmsg(pdev, 0, true, 0, num);
+            spapr_msi_setmsg(pdev, 0, true, 0, 0);
         }
         g_hash_table_remove(phb->msi, &config_addr);
 
@@ -352,13 +360,20 @@ static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 
     /* Allocate MSIs */
     irq = xics_alloc_block(spapr->icp, 0, req_num, false,
-                           ret_intr_type == RTAS_TYPE_MSI);
-    if (!irq) {
-        error_report("Cannot allocate MSIs for device %x", config_addr);
+                           ret_intr_type == RTAS_TYPE_MSI, &err);
+    if (err) {
+        error_reportf_err(err, "Can't allocate MSIs for device %x: ",
+                          config_addr);
         rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
         return;
     }
 
+    /* Release previous MSIs */
+    if (msi) {
+        xics_free(spapr->icp, msi->first_irq, msi->num);
+        g_hash_table_remove(phb->msi, &config_addr);
+    }
+
     /* Setup MSI/MSIX vectors in the device (via cfgspace or MSIX BAR) */
     spapr_msi_setmsg(pdev, SPAPR_PCI_MSI_WINDOW, ret_intr_type == RTAS_TYPE_MSIX,
                      irq, req_num);
@@ -375,7 +390,9 @@ out:
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
     rtas_st(rets, 1, req_num);
     rtas_st(rets, 2, ++seq_num);
-    rtas_st(rets, 3, ret_intr_type);
+    if (nret > 3) {
+        rtas_st(rets, 3, ret_intr_type);
+    }
 
     trace_spapr_pci_rtas_ibm_change_msi(config_addr, func, req_num, irq);
 }
@@ -389,7 +406,7 @@ static void rtas_ibm_query_interrupt_source_number(PowerPCCPU *cpu,
                                                    target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int intr_src_num = -1, ioa_intr_num = rtas_ld(args, 3);
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
@@ -428,8 +445,6 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
                                     target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
-    PCIDevice *pdev;
     uint32_t addr, option;
     uint64_t buid;
     int ret;
@@ -438,7 +453,7 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     addr = rtas_ld(args, 0);
     option = rtas_ld(args, 3);
 
@@ -447,18 +462,11 @@ static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    pdev = pci_find_device(PCI_HOST_BRIDGE(sphb)->bus,
-                           (addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
-    if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_set_option) {
-        goto param_error_exit;
-    }
-
-    ret = spc->eeh_set_option(sphb, addr, option);
+    ret = spapr_phb_vfio_eeh_set_option(sphb, addr, option);
     rtas_st(rets, 0, ret);
     return;
 
@@ -473,7 +481,6 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
                                            target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
     PCIDevice *pdev;
     uint32_t addr, option;
     uint64_t buid;
@@ -482,14 +489,13 @@ static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_set_option) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
@@ -529,7 +535,6 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
                                             target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
     uint64_t buid;
     int state, ret;
 
@@ -537,18 +542,17 @@ static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_get_state) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
-    ret = spc->eeh_get_state(sphb, &state);
+    ret = spapr_phb_vfio_eeh_get_state(sphb, &state);
     rtas_st(rets, 0, ret);
     if (ret != RTAS_OUT_SUCCESS) {
         return;
@@ -573,7 +577,6 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
                                     target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
     uint32_t option;
     uint64_t buid;
     int ret;
@@ -582,19 +585,18 @@ static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     option = rtas_ld(args, 3);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_reset) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
-    ret = spc->eeh_reset(sphb, option);
+    ret = spapr_phb_vfio_eeh_reset(sphb, option);
     rtas_st(rets, 0, ret);
     return;
 
@@ -609,7 +611,6 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
                                   target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
     uint64_t buid;
     int ret;
 
@@ -617,18 +618,17 @@ static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_configure) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
-    ret = spc->eeh_configure(sphb);
+    ret = spapr_phb_vfio_eeh_configure(sphb);
     rtas_st(rets, 0, ret);
     return;
 
@@ -644,7 +644,6 @@ static void rtas_ibm_slot_error_detail(PowerPCCPU *cpu,
                                        target_ulong rets)
 {
     sPAPRPHBState *sphb;
-    sPAPRPHBClass *spc;
     int option;
     uint64_t buid;
 
@@ -652,14 +651,13 @@ static void rtas_ibm_slot_error_detail(PowerPCCPU *cpu,
         goto param_error_exit;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     sphb = spapr_pci_find_phb(spapr, buid);
     if (!sphb) {
         goto param_error_exit;
     }
 
-    spc = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(sphb);
-    if (!spc->eeh_set_option) {
+    if (!spapr_phb_eeh_available(sphb)) {
         goto param_error_exit;
     }
 
@@ -955,6 +953,7 @@ static int spapr_populate_pci_child_dt(PCIDevice *dev, void *fdt, int offset,
     int pci_status, err;
     char *buf = NULL;
     uint32_t drc_index = spapr_phb_get_pci_drc_index(sphb, dev);
+    uint32_t max_msi, max_msix;
 
     if (pci_default_read_config(dev, PCI_HEADER_TYPE, 1) ==
         PCI_HEADER_TYPE_BRIDGE) {
@@ -1035,8 +1034,15 @@ static int spapr_populate_pci_child_dt(PCIDevice *dev, void *fdt, int offset,
                           RESOURCE_CELLS_ADDRESS));
     _FDT(fdt_setprop_cell(fdt, offset, "#size-cells",
                           RESOURCE_CELLS_SIZE));
-    _FDT(fdt_setprop_cell(fdt, offset, "ibm,req#msi-x",
-                          RESOURCE_CELLS_SIZE));
+
+    max_msi = msi_nr_vectors_allocated(dev);
+    if (max_msi) {
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,req#msi", max_msi));
+    }
+    max_msix = dev->msix_entries_nr;
+    if (max_msix) {
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,req#msi-x", max_msix));
+    }
 
     populate_resource_props(dev, &rp);
     _FDT(fdt_setprop(fdt, offset, "reg", (uint8_t *)rp.reg, rp.reg_len));
@@ -1080,6 +1086,12 @@ static void spapr_phb_add_pci_device(sPAPRDRConnector *drc,
     void *fdt = NULL;
     int fdt_start_offset = 0, fdt_size;
 
+    if (object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+        sPAPRTCETable *tcet = spapr_tce_find_by_liobn(phb->dma_liobn);
+
+        spapr_tce_set_need_vfio(tcet, true);
+    }
+
     if (dev->hotplugged) {
         fdt = create_device_tree(&fdt_size);
         fdt_start_offset = spapr_create_pci_child_dt(phb, pdev, fdt, 0);
@@ -1123,14 +1135,21 @@ static void spapr_phb_remove_pci_device(sPAPRDRConnector *drc,
     drck->detach(drc, DEVICE(pdev), spapr_phb_remove_pci_device_cb, phb, errp);
 }
 
-static sPAPRDRConnector *spapr_phb_get_pci_drc(sPAPRPHBState *phb,
-                                               PCIDevice *pdev)
+static sPAPRDRConnector *spapr_phb_get_pci_func_drc(sPAPRPHBState *phb,
+                                                    uint32_t busnr,
+                                                    int32_t devfn)
 {
-    uint32_t busnr = pci_bus_num(PCI_BUS(qdev_get_parent_bus(DEVICE(pdev))));
     return spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_PCI,
                                     (phb->index << 16) |
                                     (busnr << 8) |
-                                    pdev->devfn);
+                                    devfn);
+}
+
+static sPAPRDRConnector *spapr_phb_get_pci_drc(sPAPRPHBState *phb,
+                                               PCIDevice *pdev)
+{
+    uint32_t busnr = pci_bus_num(PCI_BUS(qdev_get_parent_bus(DEVICE(pdev))));
+    return spapr_phb_get_pci_func_drc(phb, busnr, pdev->devfn);
 }
 
 static uint32_t spapr_phb_get_pci_drc_index(sPAPRPHBState *phb,
@@ -1154,6 +1173,8 @@ static void spapr_phb_hot_plug_child(HotplugHandler *plug_handler,
     PCIDevice *pdev = PCI_DEVICE(plugged_dev);
     sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev);
     Error *local_err = NULL;
+    PCIBus *bus = PCI_BUS(qdev_get_parent_bus(DEVICE(pdev)));
+    uint32_t slotnr = PCI_SLOT(pdev->devfn);
 
     /* if DR is disabled we don't need to do anything in the case of
      * hotplug or coldplug callbacks
@@ -1171,13 +1192,44 @@ static void spapr_phb_hot_plug_child(HotplugHandler *plug_handler,
 
     g_assert(drc);
 
+    /* Following the QEMU convention used for PCIe multifunction
+     * hotplug, we do not allow functions to be hotplugged to a
+     * slot that already has function 0 present
+     */
+    if (plugged_dev->hotplugged && bus->devices[PCI_DEVFN(slotnr, 0)] &&
+        PCI_FUNC(pdev->devfn) != 0) {
+        error_setg(errp, "PCI: slot %d function 0 already ocuppied by %s,"
+                   " additional functions can no longer be exposed to guest.",
+                   slotnr, bus->devices[PCI_DEVFN(slotnr, 0)]->name);
+        return;
+    }
+
     spapr_phb_add_pci_device(drc, phb, pdev, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
-    if (plugged_dev->hotplugged) {
-        spapr_hotplug_req_add_event(drc);
+
+    /* If this is function 0, signal hotplug for all the device functions.
+     * Otherwise defer sending the hotplug event.
+     */
+    if (plugged_dev->hotplugged && PCI_FUNC(pdev->devfn) == 0) {
+        int i;
+
+        for (i = 0; i < 8; i++) {
+            sPAPRDRConnector *func_drc;
+            sPAPRDRConnectorClass *func_drck;
+            sPAPRDREntitySense state;
+
+            func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                  PCI_DEVFN(slotnr, i));
+            func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+            func_drck->entity_sense(func_drc, &state);
+
+            if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+                spapr_hotplug_req_add_by_index(func_drc);
+            }
+        }
     }
 }
 
@@ -1200,12 +1252,51 @@ static void spapr_phb_hot_unplug_child(HotplugHandler *plug_handler,
 
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
     if (!drck->release_pending(drc)) {
+        PCIBus *bus = PCI_BUS(qdev_get_parent_bus(DEVICE(pdev)));
+        uint32_t slotnr = PCI_SLOT(pdev->devfn);
+        sPAPRDRConnector *func_drc;
+        sPAPRDRConnectorClass *func_drck;
+        sPAPRDREntitySense state;
+        int i;
+
+        /* ensure any other present functions are pending unplug */
+        if (PCI_FUNC(pdev->devfn) == 0) {
+            for (i = 1; i < 8; i++) {
+                func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                      PCI_DEVFN(slotnr, i));
+                func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+                func_drck->entity_sense(func_drc, &state);
+                if (state == SPAPR_DR_ENTITY_SENSE_PRESENT
+                    && !func_drck->release_pending(func_drc)) {
+                    error_setg(errp,
+                               "PCI: slot %d, function %d still present. "
+                               "Must unplug all non-0 functions first.",
+                               slotnr, i);
+                    return;
+                }
+            }
+        }
+
         spapr_phb_remove_pci_device(drc, phb, pdev, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
-        spapr_hotplug_req_remove_event(drc);
+
+        /* if this isn't func 0, defer unplug event. otherwise signal removal
+         * for all present functions
+         */
+        if (PCI_FUNC(pdev->devfn) == 0) {
+            for (i = 7; i >= 0; i--) {
+                func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                      PCI_DEVFN(slotnr, i));
+                func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+                func_drck->entity_sense(func_drc, &state);
+                if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+                    spapr_hotplug_req_remove_by_index(func_drc);
+                }
+            }
+        }
     }
 }
 
@@ -1215,11 +1306,12 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
     SysBusDevice *s = SYS_BUS_DEVICE(dev);
     sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
-    sPAPRPHBClass *info = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(s);
     char *namebuf;
     int i;
     PCIBus *bus;
     uint64_t msi_window_size = 4096;
+    sPAPRTCETable *tcet;
+    uint32_t nb_table;
 
     if (sphb->index != (uint32_t)-1) {
         hwaddr windows_base;
@@ -1350,10 +1442,12 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
     /* Initialize the LSI table */
     for (i = 0; i < PCI_NUM_PINS; i++) {
         uint32_t irq;
+        Error *local_err = NULL;
 
-        irq = xics_alloc_block(spapr->icp, 0, 1, true, false);
-        if (!irq) {
-            error_setg(errp, "spapr_allocate_lsi failed");
+        irq = xics_alloc_block(spapr->icp, 0, 1, true, false, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            error_prepend(errp, "can't allocate LSIs: ");
             return;
         }
 
@@ -1369,33 +1463,20 @@ static void spapr_phb_realize(DeviceState *dev, Error **errp)
         }
     }
 
-    if (!info->finish_realize) {
-        error_setg(errp, "finish_realize not defined");
-        return;
-    }
-
-    info->finish_realize(sphb, errp);
-
-    sphb->msi = g_hash_table_new_full(g_int_hash, g_int_equal, g_free, g_free);
-}
-
-static void spapr_phb_finish_realize(sPAPRPHBState *sphb, Error **errp)
-{
-    sPAPRTCETable *tcet;
-    uint32_t nb_table;
-
-    nb_table = SPAPR_PCI_DMA32_SIZE >> SPAPR_TCE_PAGE_SHIFT;
+    nb_table = sphb->dma_win_size >> SPAPR_TCE_PAGE_SHIFT;
     tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn,
                                0, SPAPR_TCE_PAGE_SHIFT, nb_table, false);
     if (!tcet) {
         error_setg(errp, "Unable to create TCE table for %s",
                    sphb->dtbusname);
-        return ;
+        return;
     }
 
     /* Register default 32bit DMA window */
-    memory_region_add_subregion(&sphb->iommu_root, 0,
+    memory_region_add_subregion(&sphb->iommu_root, sphb->dma_win_addr,
                                 spapr_tce_get_iommu(tcet));
+
+    sphb->msi = g_hash_table_new_full(g_int_hash, g_int_equal, g_free, g_free);
 }
 
 static int spapr_phb_children_reset(Object *child, void *opaque)
@@ -1413,6 +1494,10 @@ static void spapr_phb_reset(DeviceState *qdev)
 {
     /* Reset the IOMMU state */
     object_child_foreach(OBJECT(qdev), spapr_phb_children_reset, NULL);
+
+    if (spapr_phb_eeh_available(SPAPR_PCI_HOST_BRIDGE(qdev))) {
+        spapr_phb_vfio_reset(qdev);
+    }
 }
 
 static Property spapr_phb_properties[] = {
@@ -1427,6 +1512,9 @@ static Property spapr_phb_properties[] = {
                        SPAPR_PCI_IO_WIN_SIZE),
     DEFINE_PROP_BOOL("dynamic-reconfiguration", sPAPRPHBState, dr_enabled,
                      true),
+    /* Default DMA window is 0..1GB */
+    DEFINE_PROP_UINT64("dma_win_addr", sPAPRPHBState, dma_win_addr, 0),
+    DEFINE_PROP_UINT64("dma_win_size", sPAPRPHBState, dma_win_size, 0x40000000),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1460,10 +1548,8 @@ static void spapr_pci_pre_save(void *opaque)
     gpointer key, value;
     int i;
 
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
     sphb->msi_devs_num = g_hash_table_size(sphb->msi);
     if (!sphb->msi_devs_num) {
         return;
@@ -1490,10 +1576,8 @@ static int spapr_pci_post_load(void *opaque, int version_id)
                          sizeof(sphb->msi_devs[i].value));
         g_hash_table_insert(sphb->msi, key, value);
     }
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
     sphb->msi_devs_num = 0;
 
     return 0;
@@ -1533,7 +1617,6 @@ static void spapr_phb_class_init(ObjectClass *klass, void *data)
 {
     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
-    sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_CLASS(klass);
     HotplugHandlerClass *hp = HOTPLUG_HANDLER_CLASS(klass);
 
     hc->root_bus_path = spapr_phb_root_bus_path;
@@ -1543,7 +1626,6 @@ static void spapr_phb_class_init(ObjectClass *klass, void *data)
     dc->vmsd = &vmstate_spapr_pci;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     dc->cannot_instantiate_with_device_add_yet = false;
-    spc->finish_realize = spapr_phb_finish_realize;
     hp->plug = spapr_phb_hot_plug_child;
     hp->unplug = spapr_phb_hot_unplug_child;
 }
@@ -1553,7 +1635,6 @@ static const TypeInfo spapr_phb_info = {
     .parent        = TYPE_PCI_HOST_BRIDGE,
     .instance_size = sizeof(sPAPRPHBState),
     .class_init    = spapr_phb_class_init,
-    .class_size    = sizeof(sPAPRPHBClass),
     .interfaces    = (InterfaceInfo[]) {
         { TYPE_HOTPLUG_HANDLER },
         { }
@@ -1735,6 +1816,9 @@ int spapr_populate_pci_dt(sPAPRPHBState *phb,
                      sizeof(interrupt_map)));
 
     tcet = spapr_tce_find_by_liobn(SPAPR_PCI_LIOBN(phb->index, 0));
+    if (!tcet) {
+        return -1;
+    }
     spapr_dma_dt(fdt, bus_off, "ibm,dma-window",
                  tcet->liobn, tcet->bus_offset,
                  tcet->nb_table << tcet->page_shift);
@@ -1770,7 +1854,7 @@ void spapr_pci_rtas_init(void)
                         rtas_ibm_read_pci_config);
     spapr_rtas_register(RTAS_IBM_WRITE_PCI_CONFIG, "ibm,write-pci-config",
                         rtas_ibm_write_pci_config);
-    if (msi_supported) {
+    if (msi_nonbroken) {
         spapr_rtas_register(RTAS_IBM_QUERY_INTERRUPT_SOURCE_NUMBER,
                             "ibm,query-interrupt-source-number",
                             rtas_ibm_query_interrupt_source_number);
diff --git a/qemu/hw/ppc/spapr_pci_vfio.c b/qemu/hw/ppc/spapr_pci_vfio.c
index cca45ed31..cbd3d23c9 100644
--- a/qemu/hw/ppc/spapr_pci_vfio.c
+++ b/qemu/hw/ppc/spapr_pci_vfio.c
@@ -17,73 +17,51 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/ppc/spapr.h"
 #include "hw/pci-host/spapr.h"
 #include "hw/pci/msix.h"
 #include "linux/vfio.h"
 #include "hw/vfio/vfio.h"
+#include "qemu/error-report.h"
 
-static Property spapr_phb_vfio_properties[] = {
-    DEFINE_PROP_INT32("iommu", sPAPRPHBVFIOState, iommugroupid, -1),
-    DEFINE_PROP_END_OF_LIST(),
-};
+#define TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE "spapr-pci-vfio-host-bridge"
 
-static void spapr_phb_vfio_finish_realize(sPAPRPHBState *sphb, Error **errp)
-{
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_iommu_spapr_tce_info info = { .argsz = sizeof(info) };
-    int ret;
-    sPAPRTCETable *tcet;
-    uint32_t liobn = svphb->phb.dma_liobn;
+#define SPAPR_PCI_VFIO_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(sPAPRPHBVFIOState, (obj), TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE)
 
-    if (svphb->iommugroupid == -1) {
-        error_setg(errp, "Wrong IOMMU group ID %d", svphb->iommugroupid);
-        return;
-    }
+typedef struct sPAPRPHBVFIOState sPAPRPHBVFIOState;
 
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_CHECK_EXTENSION,
-                               (void *) VFIO_SPAPR_TCE_IOMMU);
-    if (ret != 1) {
-        error_setg_errno(errp, -ret,
-                         "spapr-vfio: SPAPR extension is not supported");
-        return;
-    }
+struct sPAPRPHBVFIOState {
+    sPAPRPHBState phb;
 
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
-    if (ret) {
-        error_setg_errno(errp, -ret,
-                         "spapr-vfio: get info from container failed");
-        return;
-    }
+    int32_t iommugroupid;
+};
 
-    tcet = spapr_tce_new_table(DEVICE(sphb), liobn, info.dma32_window_start,
-                               SPAPR_TCE_PAGE_SHIFT,
-                               info.dma32_window_size >> SPAPR_TCE_PAGE_SHIFT,
-                               true);
-    if (!tcet) {
-        error_setg(errp, "spapr-vfio: failed to create VFIO TCE table");
-        return;
-    }
+static Property spapr_phb_vfio_properties[] = {
+    DEFINE_PROP_INT32("iommu", sPAPRPHBVFIOState, iommugroupid, -1),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
-    /* Register default 32bit DMA window */
-    memory_region_add_subregion(&sphb->iommu_root, tcet->bus_offset,
-                                spapr_tce_get_iommu(tcet));
+static void spapr_phb_vfio_instance_init(Object *obj)
+{
+    error_report("spapr-pci-vfio-host-bridge is deprecated");
 }
 
-static void spapr_phb_vfio_eeh_reenable(sPAPRPHBVFIOState *svphb)
+bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
 {
-    struct vfio_eeh_pe_op op = {
-        .argsz = sizeof(op),
-        .op    = VFIO_EEH_PE_ENABLE
-    };
+    return vfio_eeh_as_ok(&sphb->iommu_as);
+}
 
-    vfio_container_ioctl(&svphb->phb.iommu_as,
-                         svphb->iommugroupid, VFIO_EEH_PE_OP, &op);
+static void spapr_phb_vfio_eeh_reenable(sPAPRPHBState *sphb)
+{
+    vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_ENABLE);
 }
 
-static void spapr_phb_vfio_reset(DeviceState *qdev)
+void spapr_phb_vfio_reset(DeviceState *qdev)
 {
     /*
      * The PE might be in frozen state. To reenable the EEH
@@ -91,19 +69,18 @@ static void spapr_phb_vfio_reset(DeviceState *qdev)
      * ensures that the contained PCI devices will work properly
      * after reboot.
      */
-    spapr_phb_vfio_eeh_reenable(SPAPR_PCI_VFIO_HOST_BRIDGE(qdev));
+    spapr_phb_vfio_eeh_reenable(SPAPR_PCI_HOST_BRIDGE(qdev));
 }
 
-static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
-                                         unsigned int addr, int option)
+int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                  unsigned int addr, int option)
 {
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
+    uint32_t op;
     int ret;
 
     switch (option) {
     case RTAS_EEH_DISABLE:
-        op.op = VFIO_EEH_PE_DISABLE;
+        op = VFIO_EEH_PE_DISABLE;
         break;
     case RTAS_EEH_ENABLE: {
         PCIHostState *phb;
@@ -117,25 +94,24 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
         phb = PCI_HOST_BRIDGE(sphb);
         pdev = pci_find_device(phb->bus,
                                (addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
-        if (!pdev) {
+        if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
             return RTAS_OUT_PARAM_ERROR;
         }
 
-        op.op = VFIO_EEH_PE_ENABLE;
+        op = VFIO_EEH_PE_ENABLE;
         break;
     }
     case RTAS_EEH_THAW_IO:
-        op.op = VFIO_EEH_PE_UNFREEZE_IO;
+        op = VFIO_EEH_PE_UNFREEZE_IO;
         break;
     case RTAS_EEH_THAW_DMA:
-        op.op = VFIO_EEH_PE_UNFREEZE_DMA;
+        op = VFIO_EEH_PE_UNFREEZE_DMA;
         break;
     default:
         return RTAS_OUT_PARAM_ERROR;
     }
 
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_EEH_PE_OP, &op);
+    ret = vfio_eeh_as_op(&sphb->iommu_as, op);
     if (ret < 0) {
         return RTAS_OUT_HW_ERROR;
     }
@@ -143,15 +119,11 @@ static int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
     return RTAS_OUT_SUCCESS;
 }
 
-static int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state)
+int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state)
 {
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
     int ret;
 
-    op.op = VFIO_EEH_PE_GET_STATE;
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_EEH_PE_OP, &op);
+    ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_GET_STATE);
     if (ret < 0) {
         return RTAS_OUT_PARAM_ERROR;
     }
@@ -203,30 +175,28 @@ static void spapr_phb_vfio_eeh_pre_reset(sPAPRPHBState *sphb)
        pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
 }
 
-static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
+int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
 {
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
+    uint32_t op;
     int ret;
 
     switch (option) {
     case RTAS_SLOT_RESET_DEACTIVATE:
-        op.op = VFIO_EEH_PE_RESET_DEACTIVATE;
+        op = VFIO_EEH_PE_RESET_DEACTIVATE;
         break;
     case RTAS_SLOT_RESET_HOT:
         spapr_phb_vfio_eeh_pre_reset(sphb);
-        op.op = VFIO_EEH_PE_RESET_HOT;
+        op = VFIO_EEH_PE_RESET_HOT;
         break;
     case RTAS_SLOT_RESET_FUNDAMENTAL:
         spapr_phb_vfio_eeh_pre_reset(sphb);
-        op.op = VFIO_EEH_PE_RESET_FUNDAMENTAL;
+        op = VFIO_EEH_PE_RESET_FUNDAMENTAL;
         break;
     default:
         return RTAS_OUT_PARAM_ERROR;
     }
 
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_EEH_PE_OP, &op);
+    ret = vfio_eeh_as_op(&sphb->iommu_as, op);
     if (ret < 0) {
         return RTAS_OUT_HW_ERROR;
     }
@@ -234,15 +204,11 @@ static int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
     return RTAS_OUT_SUCCESS;
 }
 
-static int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
+int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
 {
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_eeh_pe_op op = { .argsz = sizeof(op) };
     int ret;
 
-    op.op = VFIO_EEH_PE_CONFIGURE;
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_EEH_PE_OP, &op);
+    ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_CONFIGURE);
     if (ret < 0) {
         return RTAS_OUT_PARAM_ERROR;
     }
@@ -253,23 +219,16 @@ static int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
 static void spapr_phb_vfio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_CLASS(klass);
 
     dc->props = spapr_phb_vfio_properties;
-    dc->reset = spapr_phb_vfio_reset;
-    spc->finish_realize = spapr_phb_vfio_finish_realize;
-    spc->eeh_set_option = spapr_phb_vfio_eeh_set_option;
-    spc->eeh_get_state = spapr_phb_vfio_eeh_get_state;
-    spc->eeh_reset = spapr_phb_vfio_eeh_reset;
-    spc->eeh_configure = spapr_phb_vfio_eeh_configure;
 }
 
 static const TypeInfo spapr_phb_vfio_info = {
     .name          = TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE,
     .parent        = TYPE_SPAPR_PCI_HOST_BRIDGE,
     .instance_size = sizeof(sPAPRPHBVFIOState),
+    .instance_init = spapr_phb_vfio_instance_init,
     .class_init    = spapr_phb_vfio_class_init,
-    .class_size    = sizeof(sPAPRPHBClass),
 };
 
 static void spapr_pci_vfio_register_types(void)
diff --git a/qemu/hw/ppc/spapr_rng.c b/qemu/hw/ppc/spapr_rng.c
new file mode 100644
index 000000000..80515eb54
--- /dev/null
+++ b/qemu/hw/ppc/spapr_rng.c
@@ -0,0 +1,191 @@
+/*
+ * QEMU sPAPR random number generator "device" for H_RANDOM hypercall
+ *
+ * Copyright 2015 Thomas Huth, Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/device_tree.h"
+#include "sysemu/rng.h"
+#include "hw/ppc/spapr.h"
+#include "kvm_ppc.h"
+
+#define SPAPR_RNG(obj) \
+    OBJECT_CHECK(sPAPRRngState, (obj), TYPE_SPAPR_RNG)
+
+struct sPAPRRngState {
+    /*< private >*/
+    DeviceState ds;
+    RngBackend *backend;
+    bool use_kvm;
+};
+typedef struct sPAPRRngState sPAPRRngState;
+
+struct HRandomData {
+    QemuSemaphore sem;
+    union {
+        uint64_t v64;
+        uint8_t v8[8];
+    } val;
+    int received;
+};
+typedef struct HRandomData HRandomData;
+
+/* Callback function for the RngBackend */
+static void random_recv(void *dest, const void *src, size_t size)
+{
+    HRandomData *hrdp = dest;
+
+    if (src && size > 0) {
+        assert(size + hrdp->received <= sizeof(hrdp->val.v8));
+        memcpy(&hrdp->val.v8[hrdp->received], src, size);
+        hrdp->received += size;
+    }
+
+    qemu_sem_post(&hrdp->sem);
+}
+
+/* Handler for the H_RANDOM hypercall */
+static target_ulong h_random(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                             target_ulong opcode, target_ulong *args)
+{
+    sPAPRRngState *rngstate;
+    HRandomData hrdata;
+
+    rngstate = SPAPR_RNG(object_resolve_path_type("", TYPE_SPAPR_RNG, NULL));
+
+    if (!rngstate || !rngstate->backend) {
+        return H_HARDWARE;
+    }
+
+    qemu_sem_init(&hrdata.sem, 0);
+    hrdata.val.v64 = 0;
+    hrdata.received = 0;
+
+    while (hrdata.received < 8) {
+        rng_backend_request_entropy(rngstate->backend, 8 - hrdata.received,
+                                    random_recv, &hrdata);
+        qemu_mutex_unlock_iothread();
+        qemu_sem_wait(&hrdata.sem);
+        qemu_mutex_lock_iothread();
+    }
+
+    qemu_sem_destroy(&hrdata.sem);
+    args[0] = hrdata.val.v64;
+
+    return H_SUCCESS;
+}
+
+static void spapr_rng_instance_init(Object *obj)
+{
+    sPAPRRngState *rngstate = SPAPR_RNG(obj);
+
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL) != NULL) {
+        error_report("spapr-rng can not be instantiated twice!");
+        return;
+    }
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&rngstate->backend,
+                             object_property_allow_set_link,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+    object_property_set_description(obj, "rng",
+                                    "ID of the random number generator backend",
+                                    NULL);
+}
+
+static void spapr_rng_realize(DeviceState *dev, Error **errp)
+{
+
+    sPAPRRngState *rngstate = SPAPR_RNG(dev);
+
+    if (rngstate->use_kvm) {
+        if (kvmppc_enable_hwrng() == 0) {
+            return;
+        }
+        /*
+         * If user specified both, use-kvm and a backend, we fall back to
+         * the backend now. If not, provide an appropriate error message.
+         */
+        if (!rngstate->backend) {
+            error_setg(errp, "Could not initialize in-kernel H_RANDOM call!");
+            return;
+        }
+    }
+
+    if (rngstate->backend) {
+        spapr_register_hypercall(H_RANDOM, h_random);
+    } else {
+        error_setg(errp, "spapr-rng needs an RNG backend!");
+    }
+}
+
+int spapr_rng_populate_dt(void *fdt)
+{
+    int node;
+    int ret;
+
+    node = qemu_fdt_add_subnode(fdt, "/ibm,platform-facilities");
+    if (node <= 0) {
+        return -1;
+    }
+    ret = fdt_setprop_string(fdt, node, "device_type",
+                             "ibm,platform-facilities");
+    ret |= fdt_setprop_cell(fdt, node, "#address-cells", 0x1);
+    ret |= fdt_setprop_cell(fdt, node, "#size-cells", 0x0);
+
+    node = fdt_add_subnode(fdt, node, "ibm,random-v1");
+    if (node <= 0) {
+        return -1;
+    }
+    ret |= fdt_setprop_string(fdt, node, "compatible", "ibm,random");
+
+    return ret ? -1 : 0;
+}
+
+static Property spapr_rng_properties[] = {
+    DEFINE_PROP_BOOL("use-kvm", sPAPRRngState, use_kvm, false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void spapr_rng_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = spapr_rng_realize;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->props = spapr_rng_properties;
+    dc->hotpluggable = false;
+}
+
+static const TypeInfo spapr_rng_info = {
+    .name          = TYPE_SPAPR_RNG,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(sPAPRRngState),
+    .instance_init = spapr_rng_instance_init,
+    .class_init    = spapr_rng_class_init,
+};
+
+static void spapr_rng_register_type(void)
+{
+    type_register_static(&spapr_rng_info);
+}
+type_init(spapr_rng_register_type)
diff --git a/qemu/hw/ppc/spapr_rtas.c b/qemu/hw/ppc/spapr_rtas.c
index 2986f94f0..f07325831 100644
--- a/qemu/hw/ppc/spapr_rtas.c
+++ b/qemu/hw/ppc/spapr_rtas.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
@@ -34,9 +35,11 @@
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/spapr_vio.h"
 #include "qapi-event.h"
+#include "hw/boards.h"
 
 #include <libfdt.h>
 #include "hw/ppc/spapr_drc.h"
+#include "qemu/cutils.h"
 
 /* #define DEBUG_SPAPR */
 
@@ -111,6 +114,7 @@ static void rtas_power_off(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         return;
     }
     qemu_system_shutdown_request();
+    cpu_stop_current();
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
 }
 
@@ -214,7 +218,7 @@ static void rtas_stop_self(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     CPUPPCState *env = &cpu->env;
 
     cs->halted = 1;
-    cpu_exit(cs);
+    qemu_cpu_kick(cs);
     /*
      * While stopping a CPU, the guest calls H_CPPR which
      * effectively disables interrupts on XICS level.
@@ -227,6 +231,19 @@ static void rtas_stop_self(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     env->msr = 0;
 }
 
+static inline int sysparm_st(target_ulong addr, target_ulong len,
+                             const void *val, uint16_t vallen)
+{
+    hwaddr phys = ppc64_phys_to_real(addr);
+
+    if (len < 2) {
+        return RTAS_OUT_SYSPARM_PARAM_ERROR;
+    }
+    stw_be_phys(&address_space_memory, phys, vallen);
+    cpu_physical_memory_write(phys + 2, val, MIN(len - 2, vallen));
+    return RTAS_OUT_SUCCESS;
+}
+
 static void rtas_ibm_get_system_parameter(PowerPCCPU *cpu,
                                           sPAPRMachineState *spapr,
                                           uint32_t token, uint32_t nargs,
@@ -236,24 +253,30 @@ static void rtas_ibm_get_system_parameter(PowerPCCPU *cpu,
     target_ulong parameter = rtas_ld(args, 0);
     target_ulong buffer = rtas_ld(args, 1);
     target_ulong length = rtas_ld(args, 2);
-    target_ulong ret = RTAS_OUT_SUCCESS;
+    target_ulong ret;
 
     switch (parameter) {
     case RTAS_SYSPARM_SPLPAR_CHARACTERISTICS: {
-        char *param_val = g_strdup_printf("MaxEntCap=%d,MaxPlatProcs=%d",
-                                          max_cpus, smp_cpus);
-        rtas_st_buffer(buffer, length, (uint8_t *)param_val, strlen(param_val));
+        char *param_val = g_strdup_printf("MaxEntCap=%d,"
+                                          "DesMem=%llu,"
+                                          "DesProcs=%d,"
+                                          "MaxPlatProcs=%d",
+                                          max_cpus,
+                                          current_machine->ram_size / M_BYTE,
+                                          smp_cpus,
+                                          max_cpus);
+        ret = sysparm_st(buffer, length, param_val, strlen(param_val) + 1);
         g_free(param_val);
         break;
     }
     case RTAS_SYSPARM_DIAGNOSTICS_RUN_MODE: {
         uint8_t param_val = DIAGNOSTICS_RUN_MODE_DISABLED;
 
-        rtas_st_buffer(buffer, length, &param_val, sizeof(param_val));
+        ret = sysparm_st(buffer, length, &param_val, sizeof(param_val));
         break;
     }
     case RTAS_SYSPARM_UUID:
-        rtas_st_buffer(buffer, length, qemu_uuid, (qemu_uuid_set ? 16 : 0));
+        ret = sysparm_st(buffer, length, qemu_uuid, (qemu_uuid_set ? 16 : 0));
         break;
     default:
         ret = RTAS_OUT_NOT_SUPPORTED;
@@ -365,12 +388,13 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     uint32_t sensor_type;
     uint32_t sensor_index;
     uint32_t sensor_state;
+    uint32_t ret = RTAS_OUT_SUCCESS;
     sPAPRDRConnector *drc;
     sPAPRDRConnectorClass *drck;
 
     if (nargs != 3 || nret != 1) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
 
     sensor_type = rtas_ld(args, 0);
@@ -386,8 +410,8 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
     if (!drc) {
         DPRINTF("rtas_set_indicator: invalid sensor/DRC index: %xh\n",
                 sensor_index);
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
 
@@ -406,19 +430,20 @@ static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                 spapr_ccs_remove(spapr, ccs);
             }
         }
-        drck->set_isolation_state(drc, sensor_state);
+        ret = drck->set_isolation_state(drc, sensor_state);
         break;
     case RTAS_SENSOR_TYPE_DR:
-        drck->set_indicator_state(drc, sensor_state);
+        ret = drck->set_indicator_state(drc, sensor_state);
         break;
     case RTAS_SENSOR_TYPE_ALLOCATION_STATE:
-        drck->set_allocation_state(drc, sensor_state);
+        ret = drck->set_allocation_state(drc, sensor_state);
         break;
     default:
         goto out_unimplemented;
     }
 
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+out:
+    rtas_st(rets, 0, ret);
     return;
 
 out_unimplemented:
@@ -435,13 +460,14 @@ static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 {
     uint32_t sensor_type;
     uint32_t sensor_index;
+    uint32_t sensor_state = 0;
     sPAPRDRConnector *drc;
     sPAPRDRConnectorClass *drck;
-    uint32_t entity_sense;
+    uint32_t ret = RTAS_OUT_SUCCESS;
 
     if (nargs != 2 || nret != 2) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
 
     sensor_type = rtas_ld(args, 0);
@@ -451,22 +477,23 @@ static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         /* currently only DR-related sensors are implemented */
         DPRINTF("rtas_get_sensor_state: sensor/indicator not implemented: %d\n",
                 sensor_type);
-        rtas_st(rets, 0, RTAS_OUT_NOT_SUPPORTED);
-        return;
+        ret = RTAS_OUT_NOT_SUPPORTED;
+        goto out;
     }
 
     drc = spapr_dr_connector_by_index(sensor_index);
     if (!drc) {
         DPRINTF("rtas_get_sensor_state: invalid sensor/DRC index: %xh\n",
                 sensor_index);
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
     }
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
-    entity_sense = drck->entity_sense(drc);
+    ret = drck->entity_sense(drc, &sensor_state);
 
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-    rtas_st(rets, 1, entity_sense);
+out:
+    rtas_st(rets, 0, ret);
+    rtas_st(rets, 1, sensor_state);
 }
 
 /* configure-connector work area offsets, int32_t units for field
@@ -481,6 +508,13 @@ static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 #define CC_VAL_DATA_OFFSET ((CC_IDX_PROP_DATA_OFFSET + 1) * 4)
 #define CC_WA_LEN 4096
 
+static void configure_connector_st(target_ulong addr, target_ulong offset,
+                                   const void *buf, size_t len)
+{
+    cpu_physical_memory_write(ppc64_phys_to_real(addr + offset),
+                              buf, MIN(len, CC_WA_LEN - offset));
+}
+
 static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
                                          sPAPRMachineState *spapr,
                                          uint32_t token, uint32_t nargs,
@@ -515,6 +549,12 @@ static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
 
     drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
     fdt = drck->get_fdt(drc, NULL);
+    if (!fdt) {
+        DPRINTF("rtas_ibm_configure_connector: Missing FDT for DRC index: %xh\n",
+                drc_index);
+        rc = SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE;
+        goto out;
+    }
 
     ccs = spapr_ccs_find(spapr, drc_index);
     if (!ccs) {
@@ -540,8 +580,7 @@ static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
             /* provide the name of the next OF node */
             wa_offset = CC_VAL_DATA_OFFSET;
             rtas_st(wa_addr, CC_IDX_NODE_NAME_OFFSET, wa_offset);
-            rtas_st_buffer_direct(wa_addr + wa_offset, CC_WA_LEN - wa_offset,
-                                  (uint8_t *)name, strlen(name) + 1);
+            configure_connector_st(wa_addr, wa_offset, name, strlen(name) + 1);
             resp = SPAPR_DR_CC_RESPONSE_NEXT_CHILD;
             break;
         case FDT_END_NODE:
@@ -566,8 +605,7 @@ static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
             /* provide the name of the next OF property */
             wa_offset = CC_VAL_DATA_OFFSET;
             rtas_st(wa_addr, CC_IDX_PROP_NAME_OFFSET, wa_offset);
-            rtas_st_buffer_direct(wa_addr + wa_offset, CC_WA_LEN - wa_offset,
-                                  (uint8_t *)name, strlen(name) + 1);
+            configure_connector_st(wa_addr, wa_offset, name, strlen(name) + 1);
 
             /* provide the length and value of the OF property. data gets
              * placed immediately after NULL terminator of the OF property's
@@ -576,9 +614,7 @@ static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
             wa_offset += strlen(name) + 1,
             rtas_st(wa_addr, CC_IDX_PROP_LEN, prop_len);
             rtas_st(wa_addr, CC_IDX_PROP_DATA_OFFSET, wa_offset);
-            rtas_st_buffer_direct(wa_addr + wa_offset, CC_WA_LEN - wa_offset,
-                                  (uint8_t *)((struct fdt_property *)prop)->data,
-                                  prop_len);
+            configure_connector_st(wa_addr, wa_offset, prop->data, prop_len);
             resp = SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY;
             break;
         case FDT_END:
@@ -631,17 +667,11 @@ target_ulong spapr_rtas_call(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 
 void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn)
 {
-    if (!((token >= RTAS_TOKEN_BASE) && (token < RTAS_TOKEN_MAX))) {
-        fprintf(stderr, "RTAS invalid token 0x%x\n", token);
-        exit(1);
-    }
+    assert((token >= RTAS_TOKEN_BASE) && (token < RTAS_TOKEN_MAX));
 
     token -= RTAS_TOKEN_BASE;
-    if (rtas_table[token].name) {
-        fprintf(stderr, "RTAS call \"%s\" is registered already as 0x%x\n",
-                rtas_table[token].name, token);
-        exit(1);
-    }
+
+    assert(!rtas_table[token].name);
 
     rtas_table[token].name = name;
     rtas_table[token].fn = fn;
@@ -654,6 +684,9 @@ int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
     int i;
     uint32_t lrdr_capacity[5];
     MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    uint64_t max_hotplug_addr = spapr->hotplug_memory.base +
+                                memory_region_size(&spapr->hotplug_memory.mr);
 
     ret = fdt_add_mem_rsv(fdt, rtas_addr, rtas_size);
     if (ret < 0) {
@@ -703,8 +736,8 @@ int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
 
     }
 
-    lrdr_capacity[0] = cpu_to_be32(((uint64_t)machine->maxram_size) >> 32);
-    lrdr_capacity[1] = cpu_to_be32(machine->maxram_size & 0xffffffff);
+    lrdr_capacity[0] = cpu_to_be32(max_hotplug_addr >> 32);
+    lrdr_capacity[1] = cpu_to_be32(max_hotplug_addr & 0xffffffff);
     lrdr_capacity[2] = 0;
     lrdr_capacity[3] = cpu_to_be32(SPAPR_MEMORY_BLOCK_SIZE);
     lrdr_capacity[4] = cpu_to_be32(max_cpus/smp_threads);
diff --git a/qemu/hw/ppc/spapr_rtc.c b/qemu/hw/ppc/spapr_rtc.c
index 34b27db70..3a17ac42e 100644
--- a/qemu/hw/ppc/spapr_rtc.c
+++ b/qemu/hw/ppc/spapr_rtc.c
@@ -25,11 +25,13 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/ppc/spapr.h"
 #include "qapi-event.h"
+#include "qemu/cutils.h"
 
 #define SPAPR_RTC(obj) \
     OBJECT_CHECK(sPAPRRTCState, (obj), TYPE_SPAPR_RTC)
@@ -200,7 +202,6 @@ static const TypeInfo spapr_rtc_info = {
     .name          = TYPE_SPAPR_RTC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(sPAPRRTCState),
-    .class_size = sizeof(XICSStateClass),
     .class_init    = spapr_rtc_class_init,
 };
 
diff --git a/qemu/hw/ppc/spapr_vio.c b/qemu/hw/ppc/spapr_vio.c
index c51eb8e24..8aa021fde 100644
--- a/qemu/hw/ppc/spapr_vio.c
+++ b/qemu/hw/ppc/spapr_vio.c
@@ -19,6 +19,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
@@ -388,7 +390,7 @@ static void rtas_quiesce(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 
 static VIOsPAPRDevice *reg_conflict(VIOsPAPRDevice *dev)
 {
-    VIOsPAPRBus *bus = DO_UPCAST(VIOsPAPRBus, bus, dev->qdev.parent_bus);
+    VIOsPAPRBus *bus = SPAPR_VIO_BUS(dev->qdev.parent_bus);
     BusChild *kid;
     VIOsPAPRDevice *other;
 
@@ -430,6 +432,7 @@ static void spapr_vio_busdev_realize(DeviceState *qdev, Error **errp)
     VIOsPAPRDevice *dev = (VIOsPAPRDevice *)qdev;
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
     char *id;
+    Error *local_err = NULL;
 
     if (dev->reg != -1) {
         /*
@@ -449,7 +452,7 @@ static void spapr_vio_busdev_realize(DeviceState *qdev, Error **errp)
         }
     } else {
         /* Need to assign an address */
-        VIOsPAPRBus *bus = DO_UPCAST(VIOsPAPRBus, bus, dev->qdev.parent_bus);
+        VIOsPAPRBus *bus = SPAPR_VIO_BUS(dev->qdev.parent_bus);
 
         do {
             dev->reg = bus->next_reg++;
@@ -462,9 +465,9 @@ static void spapr_vio_busdev_realize(DeviceState *qdev, Error **errp)
         dev->qdev.id = id;
     }
 
-    dev->irq = xics_alloc(spapr->icp, 0, dev->irq, false);
-    if (!dev->irq) {
-        error_setg(errp, "can't allocate IRQ");
+    dev->irq = xics_alloc(spapr->icp, 0, dev->irq, false, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         return;
     }
 
@@ -523,13 +526,12 @@ VIOsPAPRBus *spapr_vio_bus_init(void)
     DeviceState *dev;
 
     /* Create bridge device */
-    dev = qdev_create(NULL, "spapr-vio-bridge");
+    dev = qdev_create(NULL, TYPE_SPAPR_VIO_BRIDGE);
     qdev_init_nofail(dev);
 
     /* Create bus on bridge device */
-
     qbus = qbus_create(TYPE_SPAPR_VIO_BUS, dev, "spapr-vio");
-    bus = DO_UPCAST(VIOsPAPRBus, bus, qbus);
+    bus = SPAPR_VIO_BUS(qbus);
     bus->next_reg = 0x71000000;
 
     /* hcall-vio */
@@ -567,9 +569,8 @@ static void spapr_vio_bridge_class_init(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo spapr_vio_bridge_info = {
-    .name          = "spapr-vio-bridge",
+    .name          = TYPE_SPAPR_VIO_BRIDGE,
     .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(SysBusDevice),
     .class_init    = spapr_vio_bridge_class_init,
 };
 
diff --git a/qemu/hw/ppc/virtex_ml507.c b/qemu/hw/ppc/virtex_ml507.c
index de86f7c64..b807a08c2 100644
--- a/qemu/hw/ppc/virtex_ml507.c
+++ b/qemu/hw/ppc/virtex_ml507.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/char/serial.h"
@@ -257,7 +258,8 @@ static void virtex_init(MachineState *machine)
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &entry, &low, &high, 1, ELF_MACHINE, 0);
+                               &entry, &low, &high, 1, PPC_ELF_MACHINE,
+                               0, 0);
         boot_info.bootstrap_pc = entry & 0x00ffffff;
 
         if (kernel_size < 0) {
@@ -297,15 +299,10 @@ static void virtex_init(MachineState *machine)
     env->load_info = &boot_info;
 }
 
-static QEMUMachine virtex_machine = {
-    .name = "virtex-ml507",
-    .desc = "Xilinx Virtex ML507 reference design",
-    .init = virtex_init,
-};
-
-static void virtex_machine_init(void)
+static void virtex_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&virtex_machine);
+    mc->desc = "Xilinx Virtex ML507 reference design";
+    mc->init = virtex_init;
 }
 
-machine_init(virtex_machine_init);
+DEFINE_MACHINE("virtex-ml507", virtex_machine_init)
diff --git a/qemu/hw/s390x/Makefile.objs b/qemu/hw/s390x/Makefile.objs
index 27cd75a93..220361782 100644
--- a/qemu/hw/s390x/Makefile.objs
+++ b/qemu/hw/s390x/Makefile.objs
@@ -1,4 +1,4 @@
-obj-y = s390-virtio-bus.o s390-virtio.o
+obj-y += s390-virtio.o
 obj-y += s390-virtio-hcall.o
 obj-y += sclp.o
 obj-y += event-facility.o
@@ -9,3 +9,5 @@ obj-y += css.o
 obj-y += s390-virtio-ccw.o
 obj-y += virtio-ccw.o
 obj-y += s390-pci-bus.o s390-pci-inst.o
+obj-y += s390-skeys.o
+obj-$(CONFIG_KVM) += s390-skeys-kvm.o
diff --git a/qemu/hw/s390x/css.c b/qemu/hw/s390x/css.c
index 5df450e00..3a1d91958 100644
--- a/qemu/hw/s390x/css.c
+++ b/qemu/hw/s390x/css.c
@@ -9,6 +9,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include <hw/qdev.h>
 #include "qemu/bitops.h"
 #include "exec/address-spaces.h"
@@ -49,6 +50,7 @@ typedef struct IoAdapter {
 
 typedef struct ChannelSubSys {
     QTAILQ_HEAD(, CrwContainer) pending_crws;
+    bool sei_pending;
     bool do_crw_mchk;
     bool crws_lost;
     uint8_t max_cssid;
@@ -58,9 +60,81 @@ typedef struct ChannelSubSys {
     CssImage *css[MAX_CSSID + 1];
     uint8_t default_cssid;
     QTAILQ_HEAD(, IoAdapter) io_adapters;
+    QTAILQ_HEAD(, IndAddr) indicator_addresses;
 } ChannelSubSys;
 
-static ChannelSubSys *channel_subsys;
+static ChannelSubSys channel_subsys = {
+    .pending_crws = QTAILQ_HEAD_INITIALIZER(channel_subsys.pending_crws),
+    .do_crw_mchk = true,
+    .sei_pending = false,
+    .do_crw_mchk = true,
+    .crws_lost = false,
+    .chnmon_active = false,
+    .io_adapters = QTAILQ_HEAD_INITIALIZER(channel_subsys.io_adapters),
+    .indicator_addresses =
+        QTAILQ_HEAD_INITIALIZER(channel_subsys.indicator_addresses),
+};
+
+IndAddr *get_indicator(hwaddr ind_addr, int len)
+{
+    IndAddr *indicator;
+
+    QTAILQ_FOREACH(indicator, &channel_subsys.indicator_addresses, sibling) {
+        if (indicator->addr == ind_addr) {
+            indicator->refcnt++;
+            return indicator;
+        }
+    }
+    indicator = g_new0(IndAddr, 1);
+    indicator->addr = ind_addr;
+    indicator->len = len;
+    indicator->refcnt = 1;
+    QTAILQ_INSERT_TAIL(&channel_subsys.indicator_addresses,
+                       indicator, sibling);
+    return indicator;
+}
+
+static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
+                               bool do_map)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
+}
+
+void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    assert(indicator->refcnt > 0);
+    indicator->refcnt--;
+    if (indicator->refcnt > 0) {
+        return;
+    }
+    QTAILQ_REMOVE(&channel_subsys.indicator_addresses, indicator, sibling);
+    if (indicator->map) {
+        s390_io_adapter_map(adapter, indicator->map, false);
+    }
+    g_free(indicator);
+}
+
+int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    int ret;
+
+    if (indicator->map) {
+        return 0; /* already mapped is not an error */
+    }
+    indicator->map = indicator->addr;
+    ret = s390_io_adapter_map(adapter, indicator->map, true);
+    if ((ret != 0) && (ret != -ENOSYS)) {
+        goto out_err;
+    }
+    return 0;
+
+out_err:
+    indicator->map = 0;
+    return ret;
+}
 
 int css_create_css_image(uint8_t cssid, bool default_image)
 {
@@ -68,12 +142,12 @@ int css_create_css_image(uint8_t cssid, bool default_image)
     if (cssid > MAX_CSSID) {
         return -EINVAL;
     }
-    if (channel_subsys->css[cssid]) {
+    if (channel_subsys.css[cssid]) {
         return -EBUSY;
     }
-    channel_subsys->css[cssid] = g_malloc0(sizeof(CssImage));
+    channel_subsys.css[cssid] = g_malloc0(sizeof(CssImage));
     if (default_image) {
-        channel_subsys->default_cssid = cssid;
+        channel_subsys.default_cssid = cssid;
     }
     return 0;
 }
@@ -88,7 +162,7 @@ int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
     S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
 
     *id = 0;
-    QTAILQ_FOREACH(adapter, &channel_subsys->io_adapters, sibling) {
+    QTAILQ_FOREACH(adapter, &channel_subsys.io_adapters, sibling) {
         if ((adapter->type == type) && (adapter->isc == isc)) {
             *id = adapter->id;
             found = true;
@@ -108,7 +182,7 @@ int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
         adapter->id = *id;
         adapter->isc = isc;
         adapter->type = type;
-        QTAILQ_INSERT_TAIL(&channel_subsys->io_adapters, adapter, sibling);
+        QTAILQ_INSERT_TAIL(&channel_subsys.io_adapters, adapter, sibling);
     } else {
         g_free(adapter);
         fprintf(stderr, "Unexpected error %d when registering adapter %d\n",
@@ -120,7 +194,7 @@ out:
 
 uint16_t css_build_subchannel_id(SubchDev *sch)
 {
-    if (channel_subsys->max_cssid > 0) {
+    if (channel_subsys.max_cssid > 0) {
         return (sch->cssid << 8) | (1 << 3) | (sch->ssid << 1) | 1;
     }
     return (sch->ssid << 1) | 1;
@@ -261,11 +335,15 @@ static CCW1 copy_ccw_from_guest(hwaddr addr, bool fmt1)
         ret.flags = tmp0.flags;
         ret.count = be16_to_cpu(tmp0.count);
         ret.cda = be16_to_cpu(tmp0.cda1) | (tmp0.cda0 << 16);
+        if ((ret.cmd_code & 0x0f) == CCW_CMD_TIC) {
+            ret.cmd_code &= 0x0f;
+        }
     }
     return ret;
 }
 
-static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
+static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr,
+                             bool suspend_allowed)
 {
     int ret;
     bool check_len;
@@ -287,9 +365,13 @@ static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
         ((ccw.cmd_code & 0xf0) != 0)) {
         return -EINVAL;
     }
+    if (!sch->ccw_fmt_1 && (ccw.count == 0) &&
+        (ccw.cmd_code != CCW_CMD_TIC)) {
+        return -EINVAL;
+    }
 
     if (ccw.flags & CCW_FLAG_SUSPEND) {
-        return -EINPROGRESS;
+        return suspend_allowed ? -EINPROGRESS : -EINVAL;
     }
 
     check_len = !((ccw.flags & CCW_FLAG_SLI) && !(ccw.flags & CCW_FLAG_DC));
@@ -387,11 +469,14 @@ static void sch_handle_start_func(SubchDev *sch, ORB *orb)
     SCSW *s = &sch->curr_status.scsw;
     int path;
     int ret;
+    bool suspend_allowed;
 
     /* Path management: In our simple css, we always choose the only path. */
     path = 0x80;
 
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
+        s->cstat = 0;
+        s->dstat = 0;
         /* Look at the orb and try to execute the channel program. */
         assert(orb != NULL); /* resume does not pass an orb */
         p->intparm = orb->intparm;
@@ -404,12 +489,15 @@ static void sch_handle_start_func(SubchDev *sch, ORB *orb)
         }
         sch->ccw_fmt_1 = !!(orb->ctrl0 & ORB_CTRL0_MASK_FMT);
         sch->ccw_no_data_cnt = 0;
+        suspend_allowed = !!(orb->ctrl0 & ORB_CTRL0_MASK_SPND);
     } else {
         s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
+        /* The channel program had been suspended before. */
+        suspend_allowed = true;
     }
     sch->last_cmd_valid = false;
     do {
-        ret = css_interpret_ccw(sch, sch->channel_prog);
+        ret = css_interpret_ccw(sch, sch->channel_prog, suspend_allowed);
         switch (ret) {
         case -EAGAIN:
             /* ccw chain, continue processing */
@@ -692,7 +780,7 @@ int css_do_csch(SubchDev *sch)
 
     /* Trigger the clear function. */
     s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
-    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_FCTL_CLEAR_FUNC;
+    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_ACTL_CLEAR_PEND;
 
     do_subchannel_work(sch, NULL);
     ret = 0;
@@ -767,12 +855,12 @@ static void css_update_chnmon(SubchDev *sch)
 
         offset = sch->curr_status.pmcw.mbi << 5;
         count = address_space_lduw(&address_space_memory,
-                                   channel_subsys->chnmon_area + offset,
+                                   channel_subsys.chnmon_area + offset,
                                    MEMTXATTRS_UNSPECIFIED,
                                    NULL);
         count++;
         address_space_stw(&address_space_memory,
-                          channel_subsys->chnmon_area + offset, count,
+                          channel_subsys.chnmon_area + offset, count,
                           MEMTXATTRS_UNSPECIFIED, NULL);
     }
 }
@@ -801,7 +889,7 @@ int css_do_ssch(SubchDev *sch, ORB *orb)
     }
 
     /* If monitoring is active, update counter. */
-    if (channel_subsys->chnmon_active) {
+    if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
     sch->channel_prog = orb->cpa;
@@ -883,8 +971,14 @@ int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
         /* If a unit check is pending, copy sense data. */
         if ((s->dstat & SCSW_DSTAT_UNIT_CHECK) &&
             (p->chars & PMCW_CHARS_MASK_CSENSE)) {
+            int i;
+
             irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF | SCSW_FLAGS_MASK_ECTL;
+            /* Attention: sense_data is already BE! */
             memcpy(irb.ecw, sch->sense_data, sizeof(sch->sense_data));
+            for (i = 0; i < ARRAY_SIZE(irb.ecw); i++) {
+                irb.ecw[i] = be32_to_cpu(irb.ecw[i]);
+            }
             irb.esw[1] = 0x01000000 | (sizeof(sch->sense_data) << 8);
         }
     }
@@ -954,16 +1048,16 @@ int css_do_stcrw(CRW *crw)
     CrwContainer *crw_cont;
     int ret;
 
-    crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws);
+    crw_cont = QTAILQ_FIRST(&channel_subsys.pending_crws);
     if (crw_cont) {
-        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+        QTAILQ_REMOVE(&channel_subsys.pending_crws, crw_cont, sibling);
         copy_crw_to_guest(crw, &crw_cont->crw);
         g_free(crw_cont);
         ret = 0;
     } else {
         /* List was empty, turn crw machine checks on again. */
         memset(crw, 0, sizeof(*crw));
-        channel_subsys->do_crw_mchk = true;
+        channel_subsys.do_crw_mchk = true;
         ret = 1;
     }
 
@@ -982,12 +1076,12 @@ void css_undo_stcrw(CRW *crw)
 
     crw_cont = g_try_malloc0(sizeof(CrwContainer));
     if (!crw_cont) {
-        channel_subsys->crws_lost = true;
+        channel_subsys.crws_lost = true;
         return;
     }
     copy_crw_from_guest(&crw_cont->crw, crw);
 
-    QTAILQ_INSERT_HEAD(&channel_subsys->pending_crws, crw_cont, sibling);
+    QTAILQ_INSERT_HEAD(&channel_subsys.pending_crws, crw_cont, sibling);
 }
 
 int css_do_tpi(IOIntCode *int_code, int lowcore)
@@ -1005,9 +1099,9 @@ int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid, uint8_t l_chpid,
     CssImage *css;
 
     if (!m && !cssid) {
-        css = channel_subsys->css[channel_subsys->default_cssid];
+        css = channel_subsys.css[channel_subsys.default_cssid];
     } else {
-        css = channel_subsys->css[cssid];
+        css = channel_subsys.css[cssid];
     }
     if (!css) {
         return 0;
@@ -1042,15 +1136,15 @@ void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo)
 {
     /* dct is currently ignored (not really meaningful for our devices) */
     /* TODO: Don't ignore mbk. */
-    if (update && !channel_subsys->chnmon_active) {
+    if (update && !channel_subsys.chnmon_active) {
         /* Enable measuring. */
-        channel_subsys->chnmon_area = mbo;
-        channel_subsys->chnmon_active = true;
+        channel_subsys.chnmon_area = mbo;
+        channel_subsys.chnmon_active = true;
     }
-    if (!update && channel_subsys->chnmon_active) {
+    if (!update && channel_subsys.chnmon_active) {
         /* Disable measuring. */
-        channel_subsys->chnmon_area = 0;
-        channel_subsys->chnmon_active = false;
+        channel_subsys.chnmon_area = 0;
+        channel_subsys.chnmon_active = false;
     }
 }
 
@@ -1078,7 +1172,7 @@ int css_do_rsch(SubchDev *sch)
     }
 
     /* If monitoring is active, update counter. */
-    if (channel_subsys->chnmon_active) {
+    if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
 
@@ -1094,23 +1188,23 @@ int css_do_rchp(uint8_t cssid, uint8_t chpid)
 {
     uint8_t real_cssid;
 
-    if (cssid > channel_subsys->max_cssid) {
+    if (cssid > channel_subsys.max_cssid) {
         return -EINVAL;
     }
-    if (channel_subsys->max_cssid == 0) {
-        real_cssid = channel_subsys->default_cssid;
+    if (channel_subsys.max_cssid == 0) {
+        real_cssid = channel_subsys.default_cssid;
     } else {
         real_cssid = cssid;
     }
-    if (!channel_subsys->css[real_cssid]) {
+    if (!channel_subsys.css[real_cssid]) {
         return -EINVAL;
     }
 
-    if (!channel_subsys->css[real_cssid]->chpids[chpid].in_use) {
+    if (!channel_subsys.css[real_cssid]->chpids[chpid].in_use) {
         return -ENODEV;
     }
 
-    if (!channel_subsys->css[real_cssid]->chpids[chpid].is_virtual) {
+    if (!channel_subsys.css[real_cssid]->chpids[chpid].is_virtual) {
         fprintf(stderr,
                 "rchp unsupported for non-virtual chpid %x.%02x!\n",
                 real_cssid, chpid);
@@ -1119,8 +1213,8 @@ int css_do_rchp(uint8_t cssid, uint8_t chpid)
 
     /* We don't really use a channel path, so we're done here. */
     css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT,
-                  channel_subsys->max_cssid > 0 ? 1 : 0, chpid);
-    if (channel_subsys->max_cssid > 0) {
+                  channel_subsys.max_cssid > 0 ? 1 : 0, chpid);
+    if (channel_subsys.max_cssid > 0) {
         css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT, 0, real_cssid << 8);
     }
     return 0;
@@ -1131,13 +1225,13 @@ bool css_schid_final(int m, uint8_t cssid, uint8_t ssid, uint16_t schid)
     SubchSet *set;
     uint8_t real_cssid;
 
-    real_cssid = (!m && (cssid == 0)) ? channel_subsys->default_cssid : cssid;
+    real_cssid = (!m && (cssid == 0)) ? channel_subsys.default_cssid : cssid;
     if (real_cssid > MAX_CSSID || ssid > MAX_SSID ||
-        !channel_subsys->css[real_cssid] ||
-        !channel_subsys->css[real_cssid]->sch_set[ssid]) {
+        !channel_subsys.css[real_cssid] ||
+        !channel_subsys.css[real_cssid]->sch_set[ssid]) {
         return true;
     }
-    set = channel_subsys->css[real_cssid]->sch_set[ssid];
+    set = channel_subsys.css[real_cssid]->sch_set[ssid];
     return schid > find_last_bit(set->schids_used,
                                  (MAX_SCHID + 1) / sizeof(unsigned long));
 }
@@ -1150,7 +1244,7 @@ static int css_add_virtual_chpid(uint8_t cssid, uint8_t chpid, uint8_t type)
     if (cssid > MAX_CSSID) {
         return -EINVAL;
     }
-    css = channel_subsys->css[cssid];
+    css = channel_subsys.css[cssid];
     if (!css) {
         return -EINVAL;
     }
@@ -1171,7 +1265,7 @@ void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type)
     PMCW *p = &sch->curr_status.pmcw;
     SCSW *s = &sch->curr_status.scsw;
     int i;
-    CssImage *css = channel_subsys->css[sch->cssid];
+    CssImage *css = channel_subsys.css[sch->cssid];
 
     assert(css != NULL);
     memset(p, 0, sizeof(PMCW));
@@ -1197,27 +1291,27 @@ SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid, uint16_t schid)
 {
     uint8_t real_cssid;
 
-    real_cssid = (!m && (cssid == 0)) ? channel_subsys->default_cssid : cssid;
+    real_cssid = (!m && (cssid == 0)) ? channel_subsys.default_cssid : cssid;
 
-    if (!channel_subsys->css[real_cssid]) {
+    if (!channel_subsys.css[real_cssid]) {
         return NULL;
     }
 
-    if (!channel_subsys->css[real_cssid]->sch_set[ssid]) {
+    if (!channel_subsys.css[real_cssid]->sch_set[ssid]) {
         return NULL;
     }
 
-    return channel_subsys->css[real_cssid]->sch_set[ssid]->sch[schid];
+    return channel_subsys.css[real_cssid]->sch_set[ssid]->sch[schid];
 }
 
 bool css_subch_visible(SubchDev *sch)
 {
-    if (sch->ssid > channel_subsys->max_ssid) {
+    if (sch->ssid > channel_subsys.max_ssid) {
         return false;
     }
 
-    if (sch->cssid != channel_subsys->default_cssid) {
-        return (channel_subsys->max_cssid > 0);
+    if (sch->cssid != channel_subsys.default_cssid) {
+        return (channel_subsys.max_cssid > 0);
     }
 
     return true;
@@ -1225,20 +1319,20 @@ bool css_subch_visible(SubchDev *sch)
 
 bool css_present(uint8_t cssid)
 {
-    return (channel_subsys->css[cssid] != NULL);
+    return (channel_subsys.css[cssid] != NULL);
 }
 
 bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno)
 {
-    if (!channel_subsys->css[cssid]) {
+    if (!channel_subsys.css[cssid]) {
         return false;
     }
-    if (!channel_subsys->css[cssid]->sch_set[ssid]) {
+    if (!channel_subsys.css[cssid]->sch_set[ssid]) {
         return false;
     }
 
     return !!test_bit(devno,
-                      channel_subsys->css[cssid]->sch_set[ssid]->devnos_used);
+                      channel_subsys.css[cssid]->sch_set[ssid]->devnos_used);
 }
 
 void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
@@ -1249,13 +1343,13 @@ void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
 
     trace_css_assign_subch(sch ? "assign" : "deassign", cssid, ssid, schid,
                            devno);
-    if (!channel_subsys->css[cssid]) {
+    if (!channel_subsys.css[cssid]) {
         fprintf(stderr,
                 "Suspicious call to %s (%x.%x.%04x) for non-existing css!\n",
                 __func__, cssid, ssid, schid);
         return;
     }
-    css = channel_subsys->css[cssid];
+    css = channel_subsys.css[cssid];
 
     if (!css->sch_set[ssid]) {
         css->sch_set[ssid] = g_malloc0(sizeof(SubchSet));
@@ -1280,7 +1374,7 @@ void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid)
     /* TODO: Maybe use a static crw pool? */
     crw_cont = g_try_malloc0(sizeof(CrwContainer));
     if (!crw_cont) {
-        channel_subsys->crws_lost = true;
+        channel_subsys.crws_lost = true;
         return;
     }
     crw_cont->crw.flags = (rsc << 8) | erc;
@@ -1288,15 +1382,15 @@ void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid)
         crw_cont->crw.flags |= CRW_FLAGS_MASK_C;
     }
     crw_cont->crw.rsid = rsid;
-    if (channel_subsys->crws_lost) {
+    if (channel_subsys.crws_lost) {
         crw_cont->crw.flags |= CRW_FLAGS_MASK_R;
-        channel_subsys->crws_lost = false;
+        channel_subsys.crws_lost = false;
     }
 
-    QTAILQ_INSERT_TAIL(&channel_subsys->pending_crws, crw_cont, sibling);
+    QTAILQ_INSERT_TAIL(&channel_subsys.pending_crws, crw_cont, sibling);
 
-    if (channel_subsys->do_crw_mchk) {
-        channel_subsys->do_crw_mchk = false;
+    if (channel_subsys.do_crw_mchk) {
+        channel_subsys.do_crw_mchk = false;
         /* Inject crw pending machine check. */
         s390_crw_mchk();
     }
@@ -1311,9 +1405,9 @@ void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
     if (add && !hotplugged) {
         return;
     }
-    if (channel_subsys->max_cssid == 0) {
+    if (channel_subsys.max_cssid == 0) {
         /* Default cssid shows up as 0. */
-        guest_cssid = (cssid == channel_subsys->default_cssid) ? 0 : cssid;
+        guest_cssid = (cssid == channel_subsys.default_cssid) ? 0 : cssid;
     } else {
         /* Show real cssid to the guest. */
         guest_cssid = cssid;
@@ -1322,14 +1416,14 @@ void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
      * Only notify for higher subchannel sets/channel subsystems if the
      * guest has enabled it.
      */
-    if ((ssid > channel_subsys->max_ssid) ||
-        (guest_cssid > channel_subsys->max_cssid) ||
-        ((channel_subsys->max_cssid == 0) &&
-         (cssid != channel_subsys->default_cssid))) {
+    if ((ssid > channel_subsys.max_ssid) ||
+        (guest_cssid > channel_subsys.max_cssid) ||
+        ((channel_subsys.max_cssid == 0) &&
+         (cssid != channel_subsys.default_cssid))) {
         return;
     }
-    chain_crw = (channel_subsys->max_ssid > 0) ||
-            (channel_subsys->max_cssid > 0);
+    chain_crw = (channel_subsys.max_ssid > 0) ||
+            (channel_subsys.max_cssid > 0);
     css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, chain_crw ? 1 : 0, schid);
     if (chain_crw) {
         css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, 0,
@@ -1344,20 +1438,28 @@ void css_generate_chp_crws(uint8_t cssid, uint8_t chpid)
 
 void css_generate_css_crws(uint8_t cssid)
 {
-    css_queue_crw(CRW_RSC_CSS, 0, 0, cssid);
+    if (!channel_subsys.sei_pending) {
+        css_queue_crw(CRW_RSC_CSS, 0, 0, cssid);
+    }
+    channel_subsys.sei_pending = true;
+}
+
+void css_clear_sei_pending(void)
+{
+    channel_subsys.sei_pending = false;
 }
 
 int css_enable_mcsse(void)
 {
     trace_css_enable_facility("mcsse");
-    channel_subsys->max_cssid = MAX_CSSID;
+    channel_subsys.max_cssid = MAX_CSSID;
     return 0;
 }
 
 int css_enable_mss(void)
 {
     trace_css_enable_facility("mss");
-    channel_subsys->max_ssid = MAX_SSID;
+    channel_subsys.max_ssid = MAX_SSID;
     return 0;
 }
 
@@ -1415,7 +1517,6 @@ void subch_device_save(SubchDev *s, QEMUFile *f)
     }
     qemu_put_byte(f, s->ccw_fmt_1);
     qemu_put_byte(f, s->ccw_no_data_cnt);
-    return;
 }
 
 int subch_device_load(SubchDev *s, QEMUFile *f)
@@ -1481,27 +1582,15 @@ int subch_device_load(SubchDev *s, QEMUFile *f)
      */
     if (s->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ENA) {
         if (s->ssid) {
-            channel_subsys->max_ssid = MAX_SSID;
+            channel_subsys.max_ssid = MAX_SSID;
         }
-        if (s->cssid != channel_subsys->default_cssid) {
-            channel_subsys->max_cssid = MAX_CSSID;
+        if (s->cssid != channel_subsys.default_cssid) {
+            channel_subsys.max_cssid = MAX_CSSID;
         }
     }
     return 0;
 }
 
-
-static void css_init(void)
-{
-    channel_subsys = g_malloc0(sizeof(*channel_subsys));
-    QTAILQ_INIT(&channel_subsys->pending_crws);
-    channel_subsys->do_crw_mchk = true;
-    channel_subsys->crws_lost = false;
-    channel_subsys->chnmon_active = false;
-    QTAILQ_INIT(&channel_subsys->io_adapters);
-}
-machine_init(css_init);
-
 void css_reset_sch(SubchDev *sch)
 {
     PMCW *p = &sch->curr_status.pmcw;
@@ -1539,18 +1628,19 @@ void css_reset(void)
     CrwContainer *crw_cont;
 
     /* Clean up monitoring. */
-    channel_subsys->chnmon_active = false;
-    channel_subsys->chnmon_area = 0;
+    channel_subsys.chnmon_active = false;
+    channel_subsys.chnmon_area = 0;
 
     /* Clear pending CRWs. */
-    while ((crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws))) {
-        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+    while ((crw_cont = QTAILQ_FIRST(&channel_subsys.pending_crws))) {
+        QTAILQ_REMOVE(&channel_subsys.pending_crws, crw_cont, sibling);
         g_free(crw_cont);
     }
-    channel_subsys->do_crw_mchk = true;
-    channel_subsys->crws_lost = false;
+    channel_subsys.sei_pending = false;
+    channel_subsys.do_crw_mchk = true;
+    channel_subsys.crws_lost = false;
 
     /* Reset maximum ids. */
-    channel_subsys->max_cssid = 0;
-    channel_subsys->max_ssid = 0;
+    channel_subsys.max_cssid = 0;
+    channel_subsys.max_ssid = 0;
 }
diff --git a/qemu/hw/s390x/css.h b/qemu/hw/s390x/css.h
index a09bb1f87..a320eea59 100644
--- a/qemu/hw/s390x/css.h
+++ b/qemu/hw/s390x/css.h
@@ -12,6 +12,8 @@
 #ifndef CSS_H
 #define CSS_H
 
+#include "hw/s390x/adapter.h"
+#include "hw/s390x/s390_flic.h"
 #include "ioinst.h"
 
 /* Channel subsystem constants. */
@@ -86,6 +88,18 @@ struct SubchDev {
     void *driver_data;
 };
 
+typedef struct IndAddr {
+    hwaddr addr;
+    uint64_t map;
+    unsigned long refcnt;
+    int len;
+    QTAILQ_ENTRY(IndAddr) sibling;
+} IndAddr;
+
+IndAddr *get_indicator(hwaddr ind_addr, int len);
+void release_indicator(AdapterInfo *adapter, IndAddr *indicator);
+int map_indicator(AdapterInfo *adapter, IndAddr *indicator);
+
 typedef SubchDev *(*css_subch_cb_func)(uint8_t m, uint8_t cssid, uint8_t ssid,
                                        uint16_t schid);
 void subch_device_save(SubchDev *s, QEMUFile *f);
@@ -103,6 +117,7 @@ void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
                            int hotplugged, int add);
 void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
 void css_generate_css_crws(uint8_t cssid);
+void css_clear_sei_pending(void);
 void css_adapter_interrupt(uint8_t isc);
 
 #define CSS_IO_ADAPTER_VIRTIO 1
diff --git a/qemu/hw/s390x/event-facility.c b/qemu/hw/s390x/event-facility.c
index 0c700effb..34b2faf01 100644
--- a/qemu/hw/s390x/event-facility.c
+++ b/qemu/hw/s390x/event-facility.c
@@ -15,6 +15,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 
 #include "hw/s390x/sclp.h"
@@ -31,8 +33,6 @@ struct SCLPEventFacility {
     unsigned int receive_mask;
 };
 
-static SCLPEvent cpu_hotplug;
-
 /* return true if any child has event pending set */
 static bool event_pending(SCLPEventFacility *ef)
 {
@@ -240,12 +240,13 @@ static void read_event_data(SCLPEventFacility *ef, SCCB *sccb)
         sclp_active_selection_mask = sclp_cp_receive_mask;
         break;
     case SCLP_SELECTIVE_READ:
-        if (!(sclp_cp_receive_mask & be32_to_cpu(red->mask))) {
+        sclp_active_selection_mask = be32_to_cpu(red->mask);
+        if (!sclp_cp_receive_mask ||
+            (sclp_active_selection_mask & ~sclp_cp_receive_mask)) {
             sccb->h.response_code =
                     cpu_to_be16(SCLP_RC_INVALID_SELECTION_MASK);
             goto out;
         }
-        sclp_active_selection_mask = be32_to_cpu(red->mask);
         break;
     default:
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
@@ -286,8 +287,26 @@ out:
 
 #define TYPE_SCLP_EVENTS_BUS "s390-sclp-events-bus"
 
+static void sclp_events_bus_realize(BusState *bus, Error **errp)
+{
+    BusChild *kid;
+
+    /* TODO: recursive realization has to be done in common code */
+    QTAILQ_FOREACH(kid, &bus->children, sibling) {
+        DeviceState *dev = kid->child;
+
+        object_property_set_bool(OBJECT(dev), true, "realized", errp);
+        if (*errp) {
+            return;
+        }
+    }
+}
+
 static void sclp_events_bus_class_init(ObjectClass *klass, void *data)
 {
+    BusClass *bc = BUS_CLASS(klass);
+
+    bc->realize = sclp_events_bus_realize;
 }
 
 static const TypeInfo sclp_events_bus_info = {
@@ -324,26 +343,26 @@ static const VMStateDescription vmstate_event_facility = {
      }
 };
 
-static int init_event_facility(SCLPEventFacility *event_facility)
+static void init_event_facility(Object *obj)
 {
-    DeviceState *sdev = DEVICE(event_facility);
-    DeviceState *quiesce;
+    SCLPEventFacility *event_facility = EVENT_FACILITY(obj);
+    DeviceState *sdev = DEVICE(obj);
+    Object *new;
 
     /* Spawn a new bus for SCLP events */
     qbus_create_inplace(&event_facility->sbus, sizeof(event_facility->sbus),
                         TYPE_SCLP_EVENTS_BUS, sdev, NULL);
 
-    quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
-    if (!quiesce) {
-        return -1;
-    }
-    qdev_init_nofail(quiesce);
-
-    object_initialize(&cpu_hotplug, sizeof(cpu_hotplug), TYPE_SCLP_CPU_HOTPLUG);
-    qdev_set_parent_bus(DEVICE(&cpu_hotplug), BUS(&event_facility->sbus));
-    object_property_set_bool(OBJECT(&cpu_hotplug), true, "realized", NULL);
+    new = object_new(TYPE_SCLP_QUIESCE);
+    object_property_add_child(obj, TYPE_SCLP_QUIESCE, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
 
-    return 0;
+    new = object_new(TYPE_SCLP_CPU_HOTPLUG);
+    object_property_add_child(obj, TYPE_SCLP_CPU_HOTPLUG, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
+    /* the facility will automatically realize the devices via the bus */
 }
 
 static void reset_event_facility(DeviceState *dev)
@@ -362,7 +381,6 @@ static void init_event_facility_class(ObjectClass *klass, void *data)
     dc->reset = reset_event_facility;
     dc->vmsd = &vmstate_event_facility;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    k->init = init_event_facility;
     k->command_handler = command_handler;
     k->event_pending = event_pending;
 }
@@ -370,6 +388,7 @@ static void init_event_facility_class(ObjectClass *klass, void *data)
 static const TypeInfo sclp_event_facility_info = {
     .name          = TYPE_SCLP_EVENT_FACILITY,
     .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_init = init_event_facility,
     .instance_size = sizeof(SCLPEventFacility),
     .class_init    = init_event_facility_class,
     .class_size    = sizeof(SCLPEventFacilityClass),
diff --git a/qemu/hw/s390x/ipl.c b/qemu/hw/s390x/ipl.c
index 2e0a8b6e0..f10420027 100644
--- a/qemu/hw/s390x/ipl.c
+++ b/qemu/hw/s390x/ipl.c
@@ -11,11 +11,12 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "cpu.h"
 #include "elf.h"
 #include "hw/loader.h"
-#include "hw/sysbus.h"
 #include "hw/s390x/virtio-ccw.h"
 #include "hw/s390x/css.h"
 #include "ipl.h"
@@ -29,44 +30,6 @@
 #define ZIPL_IMAGE_START                0x009000UL
 #define IPL_PSW_MASK                    (PSW_MASK_32 | PSW_MASK_64)
 
-#define TYPE_S390_IPL "s390-ipl"
-#define S390_IPL(obj) \
-    OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
-#if 0
-#define S390_IPL_CLASS(klass) \
-    OBJECT_CLASS_CHECK(S390IPLState, (klass), TYPE_S390_IPL)
-#define S390_IPL_GET_CLASS(obj) \
-    OBJECT_GET_CLASS(S390IPLState, (obj), TYPE_S390_IPL)
-#endif
-
-typedef struct S390IPLClass {
-    /*< private >*/
-    SysBusDeviceClass parent_class;
-    /*< public >*/
-
-    void (*parent_reset) (SysBusDevice *dev);
-} S390IPLClass;
-
-typedef struct S390IPLState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    uint64_t start_addr;
-    uint64_t bios_start_addr;
-    bool enforce_bios;
-    IplParameterBlock iplb;
-    bool iplb_valid;
-    bool reipl_requested;
-
-    /*< public >*/
-    char *kernel;
-    char *initrd;
-    char *cmdline;
-    char *firmware;
-    uint8_t cssid;
-    uint8_t ssid;
-    uint16_t devno;
-} S390IPLState;
-
 static const VMStateDescription vmstate_iplb = {
     .name = "ipl/iplb",
     .version_id = 0,
@@ -95,6 +58,11 @@ static const VMStateDescription vmstate_ipl = {
      }
 };
 
+static S390IPLState *get_ipl_device(void)
+{
+    return S390_IPL(object_resolve_path_type("", TYPE_S390_IPL, NULL));
+}
+
 static uint64_t bios_translate_addr(void *opaque, uint64_t srcaddr)
 {
     uint64_t dstaddr = *(uint64_t *) opaque;
@@ -105,11 +73,12 @@ static uint64_t bios_translate_addr(void *opaque, uint64_t srcaddr)
     return srcaddr + dstaddr;
 }
 
-static int s390_ipl_init(SysBusDevice *dev)
+static void s390_ipl_realize(DeviceState *dev, Error **errp)
 {
     S390IPLState *ipl = S390_IPL(dev);
     uint64_t pentry = KERN_IMAGE_START;
     int kernel_size;
+    Error *err = NULL;
 
     int bios_size;
     char *bios_filename;
@@ -127,17 +96,18 @@ static int s390_ipl_init(SysBusDevice *dev)
 
         bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (bios_filename == NULL) {
-            hw_error("could not find stage1 bootloader\n");
+            error_setg(&err, "could not find stage1 bootloader");
+            goto error;
         }
 
         bios_size = load_elf(bios_filename, bios_translate_addr, &fwbase,
                              &ipl->bios_start_addr, NULL, NULL, 1,
-                             ELF_MACHINE, 0);
+                             EM_S390, 0, 0);
         if (bios_size > 0) {
             /* Adjust ELF start address to final location */
             ipl->bios_start_addr += fwbase;
         } else {
-            /* Try to load non-ELF file (e.g. s390-zipl.rom) */
+            /* Try to load non-ELF file (e.g. s390-ccw.img) */
             bios_size = load_image_targphys(bios_filename, ZIPL_IMAGE_START,
                                             4096);
             ipl->bios_start_addr = ZIPL_IMAGE_START;
@@ -145,7 +115,8 @@ static int s390_ipl_init(SysBusDevice *dev)
         g_free(bios_filename);
 
         if (bios_size == -1) {
-            hw_error("could not load bootloader '%s'\n", bios_name);
+            error_setg(&err, "could not load bootloader '%s'", bios_name);
+            goto error;
         }
 
         /* default boot target is the bios */
@@ -154,13 +125,13 @@ static int s390_ipl_init(SysBusDevice *dev)
 
     if (ipl->kernel) {
         kernel_size = load_elf(ipl->kernel, NULL, NULL, &pentry, NULL,
-                               NULL, 1, ELF_MACHINE, 0);
+                               NULL, 1, EM_S390, 0, 0);
         if (kernel_size < 0) {
             kernel_size = load_image_targphys(ipl->kernel, 0, ram_size);
         }
         if (kernel_size < 0) {
-            fprintf(stderr, "could not load kernel '%s'\n", ipl->kernel);
-            return -1;
+            error_setg(&err, "could not load kernel '%s'", ipl->kernel);
+            goto error;
         }
         /*
          * Is it a Linux kernel (starting at 0x10000)? If yes, we fill in the
@@ -187,9 +158,8 @@ static int s390_ipl_init(SysBusDevice *dev)
             initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
                                               ram_size - initrd_offset);
             if (initrd_size == -1) {
-                fprintf(stderr, "qemu: could not load initrd '%s'\n",
-                        ipl->initrd);
-                exit(1);
+                error_setg(&err, "could not load initrd '%s'", ipl->initrd);
+                goto error;
             }
 
             /*
@@ -200,7 +170,9 @@ static int s390_ipl_init(SysBusDevice *dev)
             stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
         }
     }
-    return 0;
+    qemu_register_reset(qdev_reset_all_fn, dev);
+error:
+    error_propagate(errp, err);
 }
 
 static Property s390_ipl_properties[] = {
@@ -218,7 +190,7 @@ static Property s390_ipl_properties[] = {
  * - -1 if no valid boot device was found
  * - ccw id of the boot device otherwise
  */
-static uint64_t s390_update_iplstate(CPUS390XState *env, S390IPLState *ipl)
+static uint64_t s390_update_iplstate(S390IPLState *ipl)
 {
     DeviceState *dev_st;
 
@@ -251,25 +223,19 @@ out:
     return (uint32_t) (ipl->cssid << 24 | ipl->ssid << 16 | ipl->devno);
 }
 
-int s390_ipl_update_diag308(IplParameterBlock *iplb)
+void s390_ipl_update_diag308(IplParameterBlock *iplb)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
-    if (ipl) {
-        ipl->iplb = *iplb;
-        ipl->iplb_valid = true;
-        return 0;
-    }
-    return -1;
+    ipl->iplb = *iplb;
+    ipl->iplb_valid = true;
 }
 
 IplParameterBlock *s390_ipl_get_iplb(void)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
-    if (!ipl || !ipl->iplb_valid) {
+    if (!ipl->iplb_valid) {
         return NULL;
     }
     return &ipl->iplb;
@@ -277,41 +243,40 @@ IplParameterBlock *s390_ipl_get_iplb(void)
 
 void s390_reipl_request(void)
 {
-    S390IPLState *ipl;
+    S390IPLState *ipl = get_ipl_device();
 
-    ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
     ipl->reipl_requested = true;
     qemu_system_reset_request();
 }
 
+void s390_ipl_prepare_cpu(S390CPU *cpu)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    cpu->env.psw.addr = ipl->start_addr;
+    cpu->env.psw.mask = IPL_PSW_MASK;
+
+    if (!ipl->kernel || ipl->iplb_valid) {
+        cpu->env.psw.addr = ipl->bios_start_addr;
+        cpu->env.regs[7] = s390_update_iplstate(ipl);
+    }
+}
+
 static void s390_ipl_reset(DeviceState *dev)
 {
     S390IPLState *ipl = S390_IPL(dev);
-    S390CPU *cpu = S390_CPU(qemu_get_cpu(0));
-    CPUS390XState *env = &cpu->env;
-
-    env->psw.addr = ipl->start_addr;
-    env->psw.mask = IPL_PSW_MASK;
 
     if (!ipl->reipl_requested) {
         ipl->iplb_valid = false;
     }
     ipl->reipl_requested = false;
-
-    if (!ipl->kernel || ipl->iplb_valid) {
-        env->psw.addr = ipl->bios_start_addr;
-        env->regs[7] = s390_update_iplstate(env, ipl);
-    }
-
-    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
 }
 
 static void s390_ipl_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = s390_ipl_init;
+    dc->realize = s390_ipl_realize;
     dc->props = s390_ipl_properties;
     dc->reset = s390_ipl_reset;
     dc->vmsd = &vmstate_ipl;
@@ -320,8 +285,8 @@ static void s390_ipl_class_init(ObjectClass *klass, void *data)
 
 static const TypeInfo s390_ipl_info = {
     .class_init = s390_ipl_class_init,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .name  = "s390-ipl",
+    .parent = TYPE_DEVICE,
+    .name  = TYPE_S390_IPL,
     .instance_size  = sizeof(S390IPLState),
 };
 
diff --git a/qemu/hw/s390x/ipl.h b/qemu/hw/s390x/ipl.h
index 70497bc65..6b48ed7b9 100644
--- a/qemu/hw/s390x/ipl.h
+++ b/qemu/hw/s390x/ipl.h
@@ -12,14 +12,42 @@
 #ifndef HW_S390_IPL_H
 #define HW_S390_IPL_H
 
+#include "hw/qdev.h"
+#include "cpu.h"
+
 typedef struct IplParameterBlock {
       uint8_t  reserved1[110];
       uint16_t devno;
       uint8_t  reserved2[88];
 } IplParameterBlock;
 
-int s390_ipl_update_diag308(IplParameterBlock *iplb);
+void s390_ipl_update_diag308(IplParameterBlock *iplb);
+void s390_ipl_prepare_cpu(S390CPU *cpu);
 IplParameterBlock *s390_ipl_get_iplb(void);
 void s390_reipl_request(void);
 
+#define TYPE_S390_IPL "s390-ipl"
+#define S390_IPL(obj) OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
+
+struct S390IPLState {
+    /*< private >*/
+    DeviceState parent_obj;
+    uint64_t start_addr;
+    uint64_t bios_start_addr;
+    bool enforce_bios;
+    IplParameterBlock iplb;
+    bool iplb_valid;
+    bool reipl_requested;
+
+    /*< public >*/
+    char *kernel;
+    char *initrd;
+    char *cmdline;
+    char *firmware;
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t devno;
+};
+typedef struct S390IPLState S390IPLState;
+
 #endif
diff --git a/qemu/hw/s390x/s390-pci-bus.c b/qemu/hw/s390x/s390-pci-bus.c
index 560b66a50..918b58543 100644
--- a/qemu/hw/s390x/s390-pci-bus.c
+++ b/qemu/hw/s390x/s390-pci-bus.c
@@ -11,6 +11,9 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "s390-pci-bus.h"
 #include <hw/pci/pci_bus.h>
 #include <hw/pci/msi.h>
@@ -123,7 +126,6 @@ void s390_pci_sclp_configure(int configure, SCCB *sccb)
     }
 
     psccb->header.response_code = cpu_to_be16(rc);
-    return;
 }
 
 static uint32_t s390_pci_get_pfid(PCIDevice *pdev)
@@ -308,9 +310,8 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, mr);
-    S390pciState *s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)
-                                           ->qbus.parent);
+    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
+    S390pciState *s;
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
         .iova = 0,
@@ -319,8 +320,13 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
         .perm = IOMMU_NONE,
     };
 
+    if (!pbdev->configured || !pbdev->pdev || !(pbdev->fh & FH_ENABLED)) {
+        return ret;
+    }
+
     DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
 
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)->qbus.parent);
     /* s390 does not have an APIC mapped to main storage so we use
      * a separate AddressSpace only for msix notifications
      */
@@ -425,6 +431,10 @@ static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
         return;
     }
 
+    if (!(pbdev->fh & FH_ENABLED)) {
+        return;
+    }
+
     ind_bit = pbdev->routes.adapter.ind_offset;
     sum_bit = pbdev->routes.adapter.summary_offset;
 
@@ -435,8 +445,6 @@ static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
         io_int_word = (pbdev->isc << 27) | IO_INT_WORD_AI;
         s390_io_interrupt(0, 0, 0, io_int_word);
     }
-
-    return;
 }
 
 static uint64_t s390_msi_ctrl_read(void *opaque, hwaddr addr, unsigned size)
@@ -450,14 +458,32 @@ static const MemoryRegionOps s390_msi_ctrl_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable)
+{
+    pbdev->configured = false;
+
+    if (enable) {
+        uint64_t size = pbdev->pal - pbdev->pba + 1;
+        memory_region_init_iommu(&pbdev->iommu_mr, OBJECT(&pbdev->mr),
+                                 &s390_iommu_ops, "iommu-s390", size);
+        memory_region_add_subregion(&pbdev->mr, pbdev->pba, &pbdev->iommu_mr);
+    } else {
+        memory_region_del_subregion(&pbdev->mr, &pbdev->iommu_mr);
+    }
+
+    pbdev->configured = true;
+}
+
 static void s390_pcihost_init_as(S390pciState *s)
 {
     int i;
+    S390PCIBusDevice *pbdev;
 
     for (i = 0; i < PCI_SLOT_MAX; i++) {
-        memory_region_init_iommu(&s->pbdev[i].mr, OBJECT(s),
-                                 &s390_iommu_ops, "iommu-s390", UINT64_MAX);
-        address_space_init(&s->pbdev[i].as, &s->pbdev[i].mr, "iommu-pci");
+        pbdev = &s->pbdev[i];
+        memory_region_init(&pbdev->mr, OBJECT(s),
+                           "iommu-root-s390", UINT64_MAX);
+        address_space_init(&pbdev->as, &pbdev->mr, "iommu-pci");
     }
 
     memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
@@ -500,7 +526,7 @@ static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
         return 0;
     }
 
-    ctrl = pci_host_config_read_common(pbdev->pdev, pos + PCI_CAP_FLAGS,
+    ctrl = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_FLAGS,
              pci_config_size(pbdev->pdev), sizeof(ctrl));
     table = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_TABLE,
              pci_config_size(pbdev->pdev), sizeof(table));
@@ -539,7 +565,6 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
         s390_pci_generate_plug_event(HP_EVENT_TO_CONFIGURED,
                                      pbdev->fh, pbdev->fid);
     }
-    return;
 }
 
 static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
@@ -574,7 +599,7 @@ static void s390_pcihost_class_init(ObjectClass *klass, void *data)
     k->init = s390_pcihost_init;
     hc->plug = s390_pcihost_hot_plug;
     hc->unplug = s390_pcihost_hot_unplug;
-    msi_supported = true;
+    msi_nonbroken = true;
 }
 
 static const TypeInfo s390_pcihost_info = {
diff --git a/qemu/hw/s390x/s390-pci-bus.h b/qemu/hw/s390x/s390-pci-bus.h
index 464a92eed..59fd5c958 100644
--- a/qemu/hw/s390x/s390-pci-bus.h
+++ b/qemu/hw/s390x/s390-pci-bus.h
@@ -23,6 +23,7 @@
 #define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"
 #define FH_VIRT 0x00ff0000
 #define ENABLE_BIT_OFFSET 31
+#define FH_ENABLED (1 << ENABLE_BIT_OFFSET)
 #define S390_PCIPT_ADAPTER 2
 
 #define S390_PCI_HOST_BRIDGE(obj) \
@@ -231,6 +232,9 @@ typedef struct S390PCIBusDevice {
     AdapterRoutes routes;
     AddressSpace as;
     MemoryRegion mr;
+    MemoryRegion iommu_mr;
+    IndAddr *summary_ind;
+    IndAddr *indicator;
 } S390PCIBusDevice;
 
 typedef struct S390pciState {
@@ -244,6 +248,7 @@ typedef struct S390pciState {
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(int configure, SCCB *sccb);
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable);
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
diff --git a/qemu/hw/s390x/s390-pci-inst.c b/qemu/hw/s390x/s390-pci-inst.c
index f9151a9af..b28e7d14f 100644
--- a/qemu/hw/s390x/s390-pci-inst.c
+++ b/qemu/hw/s390x/s390-pci-inst.c
@@ -11,6 +11,9 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "s390-pci-inst.h"
 #include "s390-pci-bus.h"
 #include <exec/memory-internal.h>
@@ -105,7 +108,8 @@ static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
             pci_get_word(pbdev->pdev->config + PCI_DEVICE_ID));
         stw_p(&rrb->response.fh_list[idx - resume_token].vendor_id,
             pci_get_word(pbdev->pdev->config + PCI_VENDOR_ID));
-        stl_p(&rrb->response.fh_list[idx - resume_token].config, 0x80000000);
+        stl_p(&rrb->response.fh_list[idx - resume_token].config,
+            pbdev->configured << 31);
         stl_p(&rrb->response.fh_list[idx - resume_token].fid, pbdev->fid);
         stl_p(&rrb->response.fh_list[idx - resume_token].fh, pbdev->fh);
 
@@ -208,12 +212,12 @@ int clp_service_call(S390CPU *cpu, uint8_t r2)
 
         switch (reqsetpci->oc) {
         case CLP_SET_ENABLE_PCI_FN:
-            pbdev->fh = pbdev->fh | 1 << ENABLE_BIT_OFFSET;
+            pbdev->fh = pbdev->fh | FH_ENABLED;
             stl_p(&ressetpci->fh, pbdev->fh);
             stw_p(&ressetpci->hdr.rsp, CLP_RC_OK);
             break;
         case CLP_SET_DISABLE_PCI_FN:
-            pbdev->fh = pbdev->fh & ~(1 << ENABLE_BIT_OFFSET);
+            pbdev->fh = pbdev->fh & ~FH_ENABLED;
             pbdev->error_state = false;
             pbdev->lgstg_blocked = false;
             stl_p(&ressetpci->fh, pbdev->fh);
@@ -313,7 +317,7 @@ int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     offset = env->regs[r2 + 1];
 
     pbdev = s390_pci_find_dev_by_fh(fh);
-    if (!pbdev) {
+    if (!pbdev || !(pbdev->fh & FH_ENABLED)) {
         DPRINTF("pcilg no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         return 0;
@@ -430,7 +434,7 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     offset = env->regs[r2 + 1];
 
     pbdev = s390_pci_find_dev_by_fh(fh);
-    if (!pbdev) {
+    if (!pbdev || !(pbdev->fh & FH_ENABLED)) {
         DPRINTF("pcistg no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         return 0;
@@ -521,14 +525,13 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     end = start + env->regs[r2 + 1];
 
     pbdev = s390_pci_find_dev_by_fh(fh);
-
-    if (!pbdev) {
+    if (!pbdev || !(pbdev->fh & FH_ENABLED)) {
         DPRINTF("rpcit no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         goto out;
     }
 
-    mr = pci_device_iommu_address_space(pbdev->pdev)->root;
+    mr = &pbdev->iommu_mr;
     while (start < end) {
         entry = mr->iommu_ops->translate(mr, start, 0);
 
@@ -586,7 +589,7 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
     }
 
     pbdev = s390_pci_find_dev_by_fh(fh);
-    if (!pbdev) {
+    if (!pbdev || !(pbdev->fh & FH_ENABLED)) {
         DPRINTF("pcistb no pci dev fh 0x%x\n", fh);
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         return 0;
@@ -620,19 +623,19 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
 
 static int reg_irqs(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
 {
-    int ret;
-    S390FLICState *fs = s390_get_flic();
-    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+    int ret, len;
 
     ret = css_register_io_adapter(S390_PCIPT_ADAPTER,
                                   FIB_DATA_ISC(ldl_p(&fib.data)), true, false,
                                   &pbdev->routes.adapter.adapter_id);
     assert(ret == 0);
 
-    fsc->io_adapter_map(fs, pbdev->routes.adapter.adapter_id,
-        ldq_p(&fib.aisb), true);
-    fsc->io_adapter_map(fs, pbdev->routes.adapter.adapter_id,
-        ldq_p(&fib.aibv), true);
+    pbdev->summary_ind = get_indicator(ldq_p(&fib.aisb), sizeof(uint64_t));
+    len = BITS_TO_LONGS(FIB_DATA_NOI(ldl_p(&fib.data))) * sizeof(unsigned long);
+    pbdev->indicator = get_indicator(ldq_p(&fib.aibv), len);
+
+    map_indicator(&pbdev->routes.adapter, pbdev->summary_ind);
+    map_indicator(&pbdev->routes.adapter, pbdev->indicator);
 
     pbdev->routes.adapter.summary_addr = ldq_p(&fib.aisb);
     pbdev->routes.adapter.summary_offset = FIB_DATA_AISBO(ldl_p(&fib.data));
@@ -648,12 +651,11 @@ static int reg_irqs(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
 
 static int dereg_irqs(S390PCIBusDevice *pbdev)
 {
-    S390FLICState *fs = s390_get_flic();
-    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
-
-    fsc->io_adapter_map(fs, pbdev->routes.adapter.adapter_id,
-                        pbdev->routes.adapter.ind_addr, false);
+    release_indicator(&pbdev->routes.adapter, pbdev->summary_ind);
+    release_indicator(&pbdev->routes.adapter, pbdev->indicator);
 
+    pbdev->summary_ind = NULL;
+    pbdev->indicator = NULL;
     pbdev->routes.adapter.summary_addr = 0;
     pbdev->routes.adapter.summary_offset = 0;
     pbdev->routes.adapter.ind_addr = 0;
@@ -689,6 +691,9 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
     pbdev->pba = pba;
     pbdev->pal = pal;
     pbdev->g_iota = g_iota;
+
+    s390_pcihost_iommu_configure(pbdev, true);
+
     return 0;
 }
 
@@ -697,6 +702,8 @@ static void dereg_ioat(S390PCIBusDevice *pbdev)
     pbdev->pba = 0;
     pbdev->pal = 0;
     pbdev->g_iota = 0;
+
+    s390_pcihost_iommu_configure(pbdev, false);
 }
 
 int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
@@ -722,7 +729,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
     }
 
     pbdev = s390_pci_find_dev_by_fh(fh);
-    if (!pbdev) {
+    if (!pbdev || !(pbdev->fh & FH_ENABLED)) {
         DPRINTF("mpcifc no pci dev fh 0x%x\n", fh);
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         return 0;
@@ -814,7 +821,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
            ((uint32_t)pbdev->sum << 7) | pbdev->routes.adapter.summary_offset;
     stl_p(&fib.data, data);
 
-    if (pbdev->fh >> ENABLE_BIT_OFFSET) {
+    if (pbdev->fh & FH_ENABLED) {
         fib.fc |= 0x80;
     }
 
diff --git a/qemu/hw/s390x/s390-skeys-kvm.c b/qemu/hw/s390x/s390-skeys-kvm.c
new file mode 100644
index 000000000..131da56bb
--- /dev/null
+++ b/qemu/hw/s390x/s390-skeys-kvm.c
@@ -0,0 +1,76 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/s390x/storage-keys.h"
+#include "sysemu/kvm.h"
+#include "qemu/error-report.h"
+
+static int kvm_s390_skeys_enabled(S390SKeysState *ss)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint8_t single_key;
+    int r;
+
+    r = skeyclass->get_skeys(ss, 0, 1, &single_key);
+    if (r != 0 && r != KVM_S390_GET_SKEYS_NONE) {
+        error_report("S390_GET_KEYS error %d", r);
+    }
+    return (r == 0);
+}
+
+static int kvm_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_GET_SKEYS, &args);
+}
+
+static int kvm_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_SET_SKEYS, &args);
+}
+
+static void kvm_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = kvm_s390_skeys_enabled;
+    skeyclass->get_skeys = kvm_s390_skeys_get;
+    skeyclass->set_skeys = kvm_s390_skeys_set;
+}
+
+static const TypeInfo kvm_s390_skeys_info = {
+    .name          = TYPE_KVM_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = kvm_s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+};
+
+static void kvm_s390_skeys_register_types(void)
+{
+    type_register_static(&kvm_s390_skeys_info);
+}
+
+type_init(kvm_s390_skeys_register_types)
diff --git a/qemu/hw/s390x/s390-skeys.c b/qemu/hw/s390x/s390-skeys.c
new file mode 100644
index 000000000..6528ffed1
--- /dev/null
+++ b/qemu/hw/s390x/s390-skeys.c
@@ -0,0 +1,415 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "qmp-commands.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/storage-keys.h"
+#include "qemu/error-report.h"
+
+#define S390_SKEYS_BUFFER_SIZE 131072  /* Room for 128k storage keys */
+#define S390_SKEYS_SAVE_FLAG_EOS 0x01
+#define S390_SKEYS_SAVE_FLAG_SKEYS 0x02
+#define S390_SKEYS_SAVE_FLAG_ERROR 0x04
+
+S390SKeysState *s390_get_skeys_device(void)
+{
+    S390SKeysState *ss;
+
+    ss = S390_SKEYS(object_resolve_path_type("", TYPE_S390_SKEYS, NULL));
+    assert(ss);
+    return ss;
+}
+
+void s390_skeys_init(void)
+{
+    Object *obj;
+
+    if (kvm_enabled()) {
+        obj = object_new(TYPE_KVM_S390_SKEYS);
+    } else {
+        obj = object_new(TYPE_QEMU_S390_SKEYS);
+    }
+    object_property_add_child(qdev_get_machine(), TYPE_S390_SKEYS,
+                              obj, NULL);
+    object_unref(obj);
+
+    qdev_init_nofail(DEVICE(obj));
+}
+
+static void write_keys(QEMUFile *f, uint8_t *keys, uint64_t startgfn,
+                       uint64_t count, Error **errp)
+{
+    uint64_t curpage = startgfn;
+    uint64_t maxpage = curpage + count - 1;
+    const char *fmt = "page=%03" PRIx64 ": key(%d) => ACC=%X, FP=%d, REF=%d,"
+                      " ch=%d, reserved=%d\n";
+    char buf[128];
+    int len;
+
+    for (; curpage <= maxpage; curpage++) {
+        uint8_t acc = (*keys & 0xF0) >> 4;
+        int fp =  (*keys & 0x08);
+        int ref = (*keys & 0x04);
+        int ch = (*keys & 0x02);
+        int res = (*keys & 0x01);
+
+        len = snprintf(buf, sizeof(buf), fmt, curpage,
+                       *keys, acc, fp, ref, ch, res);
+        assert(len < sizeof(buf));
+        qemu_put_buffer(f, (uint8_t *)buf, len);
+        keys++;
+    }
+}
+
+void hmp_info_skeys(Monitor *mon, const QDict *qdict)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint8_t key;
+    int r;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        monitor_printf(mon, "Error: This guest is not using storage keys\n");
+        return;
+    }
+
+    r = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
+    if (r < 0) {
+        monitor_printf(mon, "Error: %s\n", strerror(-r));
+        return;
+    }
+
+    monitor_printf(mon, "  key: 0x%X\n", key);
+}
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    Error *err = NULL;
+
+    qmp_dump_skeys(filename, &err);
+    if (err) {
+        error_report_err(err);
+    }
+}
+
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    const uint64_t total_count = ram_size / TARGET_PAGE_SIZE;
+    uint64_t handled_count = 0, cur_count;
+    Error *lerr = NULL;
+    vaddr cur_gfn = 0;
+    uint8_t *buf;
+    int ret;
+    QEMUFile *f;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        error_setg(errp, "This guest is not using storage keys - "
+                         "nothing to dump");
+        return;
+    }
+
+    f = qemu_fopen(filename, "wb");
+    if (!f) {
+        error_setg_file_open(errp, errno, filename);
+        return;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_setg(errp, "Could not allocate memory");
+        goto out;
+    }
+
+    /* we'll only dump initial memory for now */
+    while (handled_count < total_count) {
+        /* Calculate how many keys to ask for & handle overflow case */
+        cur_count = MIN(total_count - handled_count, S390_SKEYS_BUFFER_SIZE);
+
+        ret = skeyclass->get_skeys(ss, cur_gfn, cur_count, buf);
+        if (ret < 0) {
+            error_setg(errp, "get_keys error %d", ret);
+            goto out_free;
+        }
+
+        /* write keys to stream */
+        write_keys(f, buf, cur_gfn, cur_count, &lerr);
+        if (lerr) {
+            goto out_free;
+        }
+
+        cur_gfn += cur_count;
+        handled_count += cur_count;
+    }
+
+out_free:
+    error_propagate(errp, lerr);
+    g_free(buf);
+out:
+    qemu_fclose(f);
+}
+
+static void qemu_s390_skeys_init(Object *obj)
+{
+    QEMUS390SKeysState *skeys = QEMU_S390_SKEYS(obj);
+    MachineState *machine = MACHINE(qdev_get_machine());
+
+    skeys->key_count = machine->maxram_size / TARGET_PAGE_SIZE;
+    skeys->keydata = g_malloc0(skeys->key_count);
+}
+
+static int qemu_s390_skeys_enabled(S390SKeysState *ss)
+{
+    return 1;
+}
+
+/*
+ * TODO: for memory hotplug support qemu_s390_skeys_set and qemu_s390_skeys_get
+ * will have to make sure that the given gfn belongs to a memory region and not
+ * a memory hole.
+ */
+static int qemu_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Setting storage keys for page beyond the end "
+                     "of memory: gfn=%" PRIx64 " count=%" PRId64,
+                     start_gfn, count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        skeydev->keydata[start_gfn + i] = keys[i];
+    }
+    return 0;
+}
+
+static int qemu_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                               uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Getting storage keys for page beyond the end "
+                     "of memory: gfn=%" PRIx64 " count=%" PRId64,
+                     start_gfn, count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        keys[i] = skeydev->keydata[start_gfn + i];
+    }
+    return 0;
+}
+
+static void qemu_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = qemu_s390_skeys_enabled;
+    skeyclass->get_skeys = qemu_s390_skeys_get;
+    skeyclass->set_skeys = qemu_s390_skeys_set;
+}
+
+static const TypeInfo qemu_s390_skeys_info = {
+    .name          = TYPE_QEMU_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_init = qemu_s390_skeys_init,
+    .instance_size = sizeof(QEMUS390SKeysState),
+    .class_init    = qemu_s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+};
+
+static void s390_storage_keys_save(QEMUFile *f, void *opaque)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t pages_left = ram_size / TARGET_PAGE_SIZE;
+    uint64_t read_count, eos = S390_SKEYS_SAVE_FLAG_EOS;
+    vaddr cur_gfn = 0;
+    int error = 0;
+    uint8_t *buf;
+
+    if (!skeyclass->skeys_enabled(ss)) {
+        goto end_stream;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_report("storage key save could not allocate memory");
+        goto end_stream;
+    }
+
+    /* We only support initial memory. Standby memory is not handled yet. */
+    qemu_put_be64(f, (cur_gfn * TARGET_PAGE_SIZE) | S390_SKEYS_SAVE_FLAG_SKEYS);
+    qemu_put_be64(f, pages_left);
+
+    while (pages_left) {
+        read_count = MIN(pages_left, S390_SKEYS_BUFFER_SIZE);
+
+        if (!error) {
+            error = skeyclass->get_skeys(ss, cur_gfn, read_count, buf);
+            if (error) {
+                /*
+                 * If error: we want to fill the stream with valid data instead
+                 * of stopping early so we pad the stream with 0x00 values and
+                 * use S390_SKEYS_SAVE_FLAG_ERROR to indicate failure to the
+                 * reading side.
+                 */
+                error_report("S390_GET_KEYS error %d", error);
+                memset(buf, 0, S390_SKEYS_BUFFER_SIZE);
+                eos = S390_SKEYS_SAVE_FLAG_ERROR;
+            }
+        }
+
+        qemu_put_buffer(f, buf, read_count);
+        cur_gfn += read_count;
+        pages_left -= read_count;
+    }
+
+    g_free(buf);
+end_stream:
+    qemu_put_be64(f, eos);
+}
+
+static int s390_storage_keys_load(QEMUFile *f, void *opaque, int version_id)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    int ret = 0;
+
+    while (!ret) {
+        ram_addr_t addr;
+        int flags;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        switch (flags) {
+        case S390_SKEYS_SAVE_FLAG_SKEYS: {
+            const uint64_t total_count = qemu_get_be64(f);
+            uint64_t handled_count = 0, cur_count;
+            uint64_t cur_gfn = addr / TARGET_PAGE_SIZE;
+            uint8_t *buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+
+            if (!buf) {
+                error_report("storage key load could not allocate memory");
+                ret = -ENOMEM;
+                break;
+            }
+
+            while (handled_count < total_count) {
+                cur_count = MIN(total_count - handled_count,
+                                S390_SKEYS_BUFFER_SIZE);
+                qemu_get_buffer(f, buf, cur_count);
+
+                ret = skeyclass->set_skeys(ss, cur_gfn, cur_count, buf);
+                if (ret < 0) {
+                    error_report("S390_SET_KEYS error %d", ret);
+                    break;
+                }
+                handled_count += cur_count;
+                cur_gfn += cur_count;
+            }
+            g_free(buf);
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_ERROR: {
+            error_report("Storage key data is incomplete");
+            ret = -EINVAL;
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_EOS:
+            /* normal exit */
+            return 0;
+        default:
+            error_report("Unexpected storage key flag data: %#x", flags);
+            ret = -EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+static inline bool s390_skeys_get_migration_enabled(Object *obj, Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    return ss->migration_enabled;
+}
+
+static inline void s390_skeys_set_migration_enabled(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    /* Prevent double registration of savevm handler */
+    if (ss->migration_enabled == value) {
+        return;
+    }
+
+    ss->migration_enabled = value;
+
+    if (ss->migration_enabled) {
+        register_savevm(NULL, TYPE_S390_SKEYS, 0, 1, s390_storage_keys_save,
+                        s390_storage_keys_load, ss);
+    } else {
+        unregister_savevm(DEVICE(ss), TYPE_S390_SKEYS, ss);
+    }
+}
+
+static void s390_skeys_instance_init(Object *obj)
+{
+    object_property_add_bool(obj, "migration-enabled",
+                             s390_skeys_get_migration_enabled,
+                             s390_skeys_set_migration_enabled, NULL);
+    object_property_set_bool(obj, true, "migration-enabled", NULL);
+}
+
+static void s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+static const TypeInfo s390_skeys_info = {
+    .name          = TYPE_S390_SKEYS,
+    .parent        = TYPE_DEVICE,
+    .instance_init = s390_skeys_instance_init,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+    .abstract = true,
+};
+
+static void qemu_s390_skeys_register_types(void)
+{
+    type_register_static(&s390_skeys_info);
+    type_register_static(&qemu_s390_skeys_info);
+}
+
+type_init(qemu_s390_skeys_register_types)
diff --git a/qemu/hw/s390x/s390-virtio-bus.c b/qemu/hw/s390x/s390-virtio-bus.c
deleted file mode 100644
index 77aec8a5b..000000000
--- a/qemu/hw/s390x/s390-virtio-bus.c
+++ /dev/null
@@ -1,763 +0,0 @@
-/*
- * QEMU S390 virtio target
- *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "hw/hw.h"
-#include "sysemu/block-backend.h"
-#include "sysemu/sysemu.h"
-#include "hw/boards.h"
-#include "hw/loader.h"
-#include "elf.h"
-#include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-rng.h"
-#include "hw/virtio/virtio-serial.h"
-#include "hw/virtio/virtio-net.h"
-#include "hw/virtio/vhost-scsi.h"
-#include "hw/sysbus.h"
-#include "sysemu/kvm.h"
-
-#include "hw/s390x/s390-virtio-bus.h"
-#include "hw/virtio/virtio-bus.h"
-
-/* #define DEBUG_S390 */
-
-#ifdef DEBUG_S390
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-#define VIRTIO_S390_QUEUE_MAX 64
-
-static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOS390Device *dev);
-
-static const TypeInfo s390_virtio_bus_info = {
-    .name = TYPE_S390_VIRTIO_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(VirtIOS390Bus),
-};
-
-static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev);
-
-/* length of VirtIO device pages */
-const hwaddr virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
-
-static void s390_virtio_bus_reset(void *opaque)
-{
-    VirtIOS390Bus *bus = opaque;
-    bus->next_ring = bus->dev_page + TARGET_PAGE_SIZE;
-}
-
-void s390_virtio_reset_idx(VirtIOS390Device *dev)
-{
-    int i;
-    hwaddr idx_addr;
-    uint8_t num_vq;
-
-    num_vq = s390_virtio_device_num_vq(dev);
-    for (i = 0; i < num_vq; i++) {
-        idx_addr = virtio_queue_get_avail_addr(dev->vdev, i) +
-            VIRTIO_VRING_AVAIL_IDX_OFFS;
-        address_space_stw(&address_space_memory, idx_addr, 0,
-                          MEMTXATTRS_UNSPECIFIED, NULL);
-        idx_addr = virtio_queue_get_avail_addr(dev->vdev, i) +
-            virtio_queue_get_avail_size(dev->vdev, i);
-        address_space_stw(&address_space_memory, idx_addr, 0,
-                          MEMTXATTRS_UNSPECIFIED, NULL);
-        idx_addr = virtio_queue_get_used_addr(dev->vdev, i) +
-            VIRTIO_VRING_USED_IDX_OFFS;
-        address_space_stw(&address_space_memory, idx_addr, 0,
-                          MEMTXATTRS_UNSPECIFIED, NULL);
-        idx_addr = virtio_queue_get_used_addr(dev->vdev, i) +
-            virtio_queue_get_used_size(dev->vdev, i);
-        address_space_stw(&address_space_memory, idx_addr, 0,
-                          MEMTXATTRS_UNSPECIFIED, NULL);
-    }
-}
-
-VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size)
-{
-    VirtIOS390Bus *bus;
-    BusState *_bus;
-    DeviceState *dev;
-
-    /* Create bridge device */
-    dev = qdev_create(NULL, "s390-virtio-bridge");
-    qdev_init_nofail(dev);
-
-    /* Create bus on bridge device */
-
-    _bus = qbus_create(TYPE_S390_VIRTIO_BUS, dev, "s390-virtio");
-    bus = DO_UPCAST(VirtIOS390Bus, bus, _bus);
-
-    bus->dev_page = *ram_size;
-    bus->dev_offs = bus->dev_page;
-    bus->next_ring = bus->dev_page + TARGET_PAGE_SIZE;
-
-    /* Enable hotplugging */
-    qbus_set_hotplug_handler(_bus, dev, &error_abort);
-
-    /* Allocate RAM for VirtIO device pages (descriptors, queues, rings) */
-    *ram_size += S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
-
-    qemu_register_reset(s390_virtio_bus_reset, bus);
-    return bus;
-}
-
-static void s390_virtio_device_init(VirtIOS390Device *dev,
-                                    VirtIODevice *vdev)
-{
-    VirtIOS390Bus *bus;
-    int dev_len;
-
-    bus = DO_UPCAST(VirtIOS390Bus, bus, dev->qdev.parent_bus);
-    dev->vdev = vdev;
-    dev->dev_offs = bus->dev_offs;
-    dev->feat_len = sizeof(uint32_t); /* always keep 32 bits features */
-
-    dev_len = VIRTIO_DEV_OFFS_CONFIG;
-    dev_len += s390_virtio_device_num_vq(dev) * VIRTIO_VQCONFIG_LEN;
-    dev_len += dev->feat_len * 2;
-    dev_len += virtio_bus_get_vdev_config_len(&dev->bus);
-
-    bus->dev_offs += dev_len;
-
-    s390_virtio_device_sync(dev);
-    s390_virtio_reset_idx(dev);
-    if (dev->qdev.hotplugged) {
-        s390_virtio_irq(VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
-    }
-}
-
-static void s390_virtio_net_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    DeviceState *qdev = DEVICE(s390_dev);
-    VirtIONetS390 *dev = VIRTIO_NET_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    Error *err = NULL;
-
-    virtio_net_set_netclient_name(&dev->vdev, qdev->id,
-                                  object_get_typename(OBJECT(qdev)));
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_net_instance_init(Object *obj)
-{
-    VirtIONetS390 *dev = VIRTIO_NET_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_NET);
-    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
-                              "bootindex", &error_abort);
-}
-
-static void s390_virtio_blk_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    VirtIOBlkS390 *dev = VIRTIO_BLK_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    Error *err = NULL;
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_blk_instance_init(Object *obj)
-{
-    VirtIOBlkS390 *dev = VIRTIO_BLK_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_BLK);
-    object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev),"iothread",
-                              &error_abort);
-    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
-                              "bootindex", &error_abort);
-}
-
-static void s390_virtio_serial_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    VirtIOSerialS390 *dev = VIRTIO_SERIAL_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    DeviceState *qdev = DEVICE(s390_dev);
-    Error *err = NULL;
-    VirtIOS390Bus *bus;
-    char *bus_name;
-
-    bus = DO_UPCAST(VirtIOS390Bus, bus, qdev->parent_bus);
-
-    /*
-     * For command line compatibility, this sets the virtio-serial-device bus
-     * name as before.
-     */
-    if (qdev->id) {
-        bus_name = g_strdup_printf("%s.0", qdev->id);
-        virtio_device_set_child_bus_name(VIRTIO_DEVICE(vdev), bus_name);
-        g_free(bus_name);
-    }
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-    bus->console = s390_dev;
-}
-
-static void s390_virtio_serial_instance_init(Object *obj)
-{
-    VirtIOSerialS390 *dev = VIRTIO_SERIAL_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_SERIAL);
-}
-
-static void s390_virtio_scsi_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    VirtIOSCSIS390 *dev = VIRTIO_SCSI_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    DeviceState *qdev = DEVICE(s390_dev);
-    Error *err = NULL;
-    char *bus_name;
-
-    /*
-     * For command line compatibility, this sets the virtio-scsi-device bus
-     * name as before.
-     */
-    if (qdev->id) {
-        bus_name = g_strdup_printf("%s.0", qdev->id);
-        virtio_device_set_child_bus_name(VIRTIO_DEVICE(vdev), bus_name);
-        g_free(bus_name);
-    }
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_scsi_instance_init(Object *obj)
-{
-    VirtIOSCSIS390 *dev = VIRTIO_SCSI_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_SCSI);
-}
-
-#ifdef CONFIG_VHOST_SCSI
-static void s390_vhost_scsi_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    VHostSCSIS390 *dev = VHOST_SCSI_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    Error *err = NULL;
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_vhost_scsi_instance_init(Object *obj)
-{
-    VHostSCSIS390 *dev = VHOST_SCSI_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VHOST_SCSI);
-}
-#endif
-
-
-static void s390_virtio_rng_realize(VirtIOS390Device *s390_dev, Error **errp)
-{
-    VirtIORNGS390 *dev = VIRTIO_RNG_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    Error *err = NULL;
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
-    }
-
-    object_property_set_link(OBJECT(dev),
-                             OBJECT(dev->vdev.conf.rng), "rng",
-                             NULL);
-
-    s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_rng_instance_init(Object *obj)
-{
-    VirtIORNGS390 *dev = VIRTIO_RNG_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_RNG);
-    object_property_add_alias(obj, "rng", OBJECT(&dev->vdev),
-                              "rng", &error_abort);
-}
-
-static uint64_t s390_virtio_device_vq_token(VirtIOS390Device *dev, int vq)
-{
-    ram_addr_t token_off;
-
-    token_off = (dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG) +
-                (vq * VIRTIO_VQCONFIG_LEN) +
-                VIRTIO_VQCONFIG_OFFS_TOKEN;
-
-    return address_space_ldq_be(&address_space_memory, token_off,
-                                MEMTXATTRS_UNSPECIFIED, NULL);
-}
-
-static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev)
-{
-    VirtIODevice *vdev = dev->vdev;
-    int num_vq;
-
-    for (num_vq = 0; num_vq < VIRTIO_S390_QUEUE_MAX; num_vq++) {
-        if (!virtio_queue_get_num(vdev, num_vq)) {
-            break;
-        }
-    }
-
-    return num_vq;
-}
-
-static ram_addr_t s390_virtio_next_ring(VirtIOS390Bus *bus)
-{
-    ram_addr_t r = bus->next_ring;
-
-    bus->next_ring += VIRTIO_RING_LEN;
-    return r;
-}
-
-void s390_virtio_device_sync(VirtIOS390Device *dev)
-{
-    VirtIOS390Bus *bus = DO_UPCAST(VirtIOS390Bus, bus, dev->qdev.parent_bus);
-    ram_addr_t cur_offs;
-    uint8_t num_vq;
-    int i;
-
-    virtio_reset(dev->vdev);
-
-    /* Sync dev space */
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_TYPE,
-                      dev->vdev->device_id,
-                      MEMTXATTRS_UNSPECIFIED,
-                      NULL);
-
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_NUM_VQ,
-                      s390_virtio_device_num_vq(dev),
-                      MEMTXATTRS_UNSPECIFIED,
-                      NULL);
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_FEATURE_LEN,
-                      dev->feat_len,
-                      MEMTXATTRS_UNSPECIFIED,
-                      NULL);
-
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG_LEN,
-                      dev->vdev->config_len,
-                      MEMTXATTRS_UNSPECIFIED,
-                      NULL);
-
-    num_vq = s390_virtio_device_num_vq(dev);
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_NUM_VQ, num_vq,
-                      MEMTXATTRS_UNSPECIFIED, NULL);
-
-    /* Sync virtqueues */
-    for (i = 0; i < num_vq; i++) {
-        ram_addr_t vq = (dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG) +
-                        (i * VIRTIO_VQCONFIG_LEN);
-        ram_addr_t vring;
-
-        vring = s390_virtio_next_ring(bus);
-        virtio_queue_set_addr(dev->vdev, i, vring);
-        virtio_queue_set_vector(dev->vdev, i, i);
-        address_space_stq_be(&address_space_memory,
-                             vq + VIRTIO_VQCONFIG_OFFS_ADDRESS, vring,
-                             MEMTXATTRS_UNSPECIFIED, NULL);
-        address_space_stw_be(&address_space_memory,
-                             vq + VIRTIO_VQCONFIG_OFFS_NUM,
-                             virtio_queue_get_num(dev->vdev, i),
-                             MEMTXATTRS_UNSPECIFIED,
-                             NULL);
-    }
-
-    cur_offs = dev->dev_offs;
-    cur_offs += VIRTIO_DEV_OFFS_CONFIG;
-    cur_offs += num_vq * VIRTIO_VQCONFIG_LEN;
-
-    /* Sync feature bitmap */
-    address_space_stl_le(&address_space_memory, cur_offs,
-                         dev->vdev->host_features,
-                         MEMTXATTRS_UNSPECIFIED, NULL);
-
-    dev->feat_offs = cur_offs + dev->feat_len;
-    cur_offs += dev->feat_len * 2;
-
-    /* Sync config space */
-    virtio_bus_get_vdev_config(&dev->bus, dev->vdev->config);
-
-    cpu_physical_memory_write(cur_offs,
-                              dev->vdev->config, dev->vdev->config_len);
-    cur_offs += dev->vdev->config_len;
-}
-
-void s390_virtio_device_update_status(VirtIOS390Device *dev)
-{
-    VirtIODevice *vdev = dev->vdev;
-    uint32_t features;
-
-    virtio_set_status(vdev,
-                      address_space_ldub(&address_space_memory,
-                                         dev->dev_offs + VIRTIO_DEV_OFFS_STATUS,
-                                         MEMTXATTRS_UNSPECIFIED, NULL));
-
-    /* Update guest supported feature bitmap */
-
-    features = bswap32(address_space_ldl_be(&address_space_memory,
-                                            dev->feat_offs,
-                                            MEMTXATTRS_UNSPECIFIED, NULL));
-    virtio_set_features(vdev, features);
-}
-
-/* Find a device by vring address */
-VirtIOS390Device *s390_virtio_bus_find_vring(VirtIOS390Bus *bus,
-                                             ram_addr_t mem,
-                                             int *vq_num)
-{
-    BusChild *kid;
-    int i;
-
-    QTAILQ_FOREACH(kid, &bus->bus.children, sibling) {
-        VirtIOS390Device *dev = (VirtIOS390Device *)kid->child;
-
-        for (i = 0; i < VIRTIO_S390_QUEUE_MAX; i++) {
-            if (!virtio_queue_get_addr(dev->vdev, i))
-                break;
-            if (virtio_queue_get_addr(dev->vdev, i) == mem) {
-                if (vq_num) {
-                    *vq_num = i;
-                }
-                return dev;
-            }
-        }
-    }
-
-    return NULL;
-}
-
-/* Find a device by device descriptor location */
-VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem)
-{
-    BusChild *kid;
-
-    QTAILQ_FOREACH(kid, &bus->bus.children, sibling) {
-        VirtIOS390Device *dev = (VirtIOS390Device *)kid->child;
-        if (dev->dev_offs == mem) {
-            return dev;
-        }
-    }
-
-    return NULL;
-}
-
-/* DeviceState to VirtIOS390Device. Note: used on datapath,
- * be careful and test performance if you change this.
- */
-static inline VirtIOS390Device *to_virtio_s390_device_fast(DeviceState *d)
-{
-    return container_of(d, VirtIOS390Device, qdev);
-}
-
-/* DeviceState to VirtIOS390Device. TODO: use QOM. */
-static inline VirtIOS390Device *to_virtio_s390_device(DeviceState *d)
-{
-    return container_of(d, VirtIOS390Device, qdev);
-}
-
-static void virtio_s390_notify(DeviceState *d, uint16_t vector)
-{
-    VirtIOS390Device *dev = to_virtio_s390_device_fast(d);
-    uint64_t token = s390_virtio_device_vq_token(dev, vector);
-
-    s390_virtio_irq(0, token);
-}
-
-static void virtio_s390_device_plugged(DeviceState *d, Error **errp)
-{
-    VirtIOS390Device *dev = to_virtio_s390_device(d);
-    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
-    int n = virtio_get_num_queues(vdev);
-
-    if (n > VIRTIO_S390_QUEUE_MAX) {
-        error_setg(errp, "The nubmer of virtqueues %d "
-                   "exceeds s390 limit %d", n,
-                   VIRTIO_S390_QUEUE_MAX);
-    }
-}
-
-/**************** S390 Virtio Bus Device Descriptions *******************/
-
-static void s390_virtio_net_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->realize = s390_virtio_net_realize;
-    set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
-}
-
-static const TypeInfo s390_virtio_net = {
-    .name          = TYPE_VIRTIO_NET_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIONetS390),
-    .instance_init = s390_virtio_net_instance_init,
-    .class_init    = s390_virtio_net_class_init,
-};
-
-static void s390_virtio_blk_class_init(ObjectClass *klass, void *data)
-{
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-    DeviceClass *dc = DEVICE_CLASS(klass);
-
-    k->realize = s390_virtio_blk_realize;
-    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-}
-
-static const TypeInfo s390_virtio_blk = {
-    .name          = "virtio-blk-s390",
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOBlkS390),
-    .instance_init = s390_virtio_blk_instance_init,
-    .class_init    = s390_virtio_blk_class_init,
-};
-
-static Property s390_virtio_serial_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_virtio_serial_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->realize = s390_virtio_serial_realize;
-    dc->props = s390_virtio_serial_properties;
-    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
-}
-
-static const TypeInfo s390_virtio_serial = {
-    .name          = TYPE_VIRTIO_SERIAL_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOSerialS390),
-    .instance_init = s390_virtio_serial_instance_init,
-    .class_init    = s390_virtio_serial_class_init,
-};
-
-static void s390_virtio_rng_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->realize = s390_virtio_rng_realize;
-    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-}
-
-static const TypeInfo s390_virtio_rng = {
-    .name          = TYPE_VIRTIO_RNG_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIORNGS390),
-    .instance_init = s390_virtio_rng_instance_init,
-    .class_init    = s390_virtio_rng_class_init,
-};
-
-static void s390_virtio_busdev_realize(DeviceState *dev, Error **errp)
-{
-    VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
-    VirtIOS390DeviceClass *_info = VIRTIO_S390_DEVICE_GET_CLASS(dev);
-
-    virtio_s390_bus_new(&_dev->bus, sizeof(_dev->bus), _dev);
-
-    _info->realize(_dev, errp);
-}
-
-static void s390_virtio_busdev_reset(DeviceState *dev)
-{
-    VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
-
-    virtio_reset(_dev->vdev);
-}
-
-static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-
-    dc->realize = s390_virtio_busdev_realize;
-    dc->bus_type = TYPE_S390_VIRTIO_BUS;
-    dc->reset = s390_virtio_busdev_reset;
-}
-
-static const TypeInfo virtio_s390_device_info = {
-    .name = TYPE_VIRTIO_S390_DEVICE,
-    .parent = TYPE_DEVICE,
-    .instance_size = sizeof(VirtIOS390Device),
-    .class_init = virtio_s390_device_class_init,
-    .class_size = sizeof(VirtIOS390DeviceClass),
-    .abstract = true,
-};
-
-static void s390_virtio_scsi_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->realize = s390_virtio_scsi_realize;
-    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-}
-
-static const TypeInfo s390_virtio_scsi = {
-    .name          = TYPE_VIRTIO_SCSI_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOSCSIS390),
-    .instance_init = s390_virtio_scsi_instance_init,
-    .class_init    = s390_virtio_scsi_class_init,
-};
-
-#ifdef CONFIG_VHOST_SCSI
-static void s390_vhost_scsi_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->realize = s390_vhost_scsi_realize;
-    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-}
-
-static const TypeInfo s390_vhost_scsi = {
-    .name          = TYPE_VHOST_SCSI_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VHostSCSIS390),
-    .instance_init = s390_vhost_scsi_instance_init,
-    .class_init    = s390_vhost_scsi_class_init,
-};
-#endif
-
-/***************** S390 Virtio Bus Bridge Device *******************/
-/* Only required to have the virtio bus as child in the system bus */
-
-static int s390_virtio_bridge_init(SysBusDevice *dev)
-{
-    /* nothing */
-    return 0;
-}
-
-static void s390_virtio_bridge_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-    DeviceClass *dc = DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_bridge_init;
-    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-}
-
-static const TypeInfo s390_virtio_bridge_info = {
-    .name          = "s390-virtio-bridge",
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(SysBusDevice),
-    .class_init    = s390_virtio_bridge_class_init,
-    .interfaces = (InterfaceInfo[]) {
-        { TYPE_HOTPLUG_HANDLER },
-        { }
-    }
-};
-
-/* virtio-s390-bus */
-
-static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOS390Device *dev)
-{
-    DeviceState *qdev = DEVICE(dev);
-    char virtio_bus_name[] = "virtio-bus";
-
-    qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_S390_BUS,
-                        qdev, virtio_bus_name);
-}
-
-static void virtio_s390_bus_class_init(ObjectClass *klass, void *data)
-{
-    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
-    BusClass *bus_class = BUS_CLASS(klass);
-    bus_class->max_dev = 1;
-    k->notify = virtio_s390_notify;
-    k->device_plugged = virtio_s390_device_plugged;
-}
-
-static const TypeInfo virtio_s390_bus_info = {
-    .name          = TYPE_VIRTIO_S390_BUS,
-    .parent        = TYPE_VIRTIO_BUS,
-    .instance_size = sizeof(VirtioS390BusState),
-    .class_init    = virtio_s390_bus_class_init,
-};
-
-static void s390_virtio_register_types(void)
-{
-    type_register_static(&virtio_s390_bus_info);
-    type_register_static(&s390_virtio_bus_info);
-    type_register_static(&virtio_s390_device_info);
-    type_register_static(&s390_virtio_serial);
-    type_register_static(&s390_virtio_blk);
-    type_register_static(&s390_virtio_net);
-    type_register_static(&s390_virtio_scsi);
-#ifdef CONFIG_VHOST_SCSI
-    type_register_static(&s390_vhost_scsi);
-#endif
-    type_register_static(&s390_virtio_rng);
-    type_register_static(&s390_virtio_bridge_info);
-}
-
-type_init(s390_virtio_register_types)
diff --git a/qemu/hw/s390x/s390-virtio-bus.h b/qemu/hw/s390x/s390-virtio-bus.h
deleted file mode 100644
index 7ad295e68..000000000
--- a/qemu/hw/s390x/s390-virtio-bus.h
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * QEMU S390x VirtIO BUS definitions
- *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef HW_S390_VIRTIO_BUS_H
-#define HW_S390_VIRTIO_BUS_H 1
-
-#include <stddef.h>
-
-#include "standard-headers/asm-s390/kvm_virtio.h"
-#include "standard-headers/linux/virtio_ring.h"
-#include "hw/virtio/virtio-blk.h"
-#include "hw/virtio/virtio-net.h"
-#include "hw/virtio/virtio-rng.h"
-#include "hw/virtio/virtio-serial.h"
-#include "hw/virtio/virtio-scsi.h"
-#include "hw/virtio/virtio-bus.h"
-#ifdef CONFIG_VHOST_SCSI
-#include "hw/virtio/vhost-scsi.h"
-#endif
-
-typedef struct kvm_device_desc KvmDeviceDesc;
-
-#define VIRTIO_DEV_OFFS_TYPE        offsetof(KvmDeviceDesc, type)
-#define VIRTIO_DEV_OFFS_NUM_VQ      offsetof(KvmDeviceDesc, num_vq)
-#define VIRTIO_DEV_OFFS_FEATURE_LEN offsetof(KvmDeviceDesc, feature_len)
-#define VIRTIO_DEV_OFFS_CONFIG_LEN  offsetof(KvmDeviceDesc, config_len)
-#define VIRTIO_DEV_OFFS_STATUS      offsetof(KvmDeviceDesc, status)
-#define VIRTIO_DEV_OFFS_CONFIG      offsetof(KvmDeviceDesc, config)
-
-typedef struct kvm_vqconfig KvmVqConfig;
-#define VIRTIO_VQCONFIG_OFFS_TOKEN   offsetof(KvmVqConfig,token)    /* 64 bit */
-#define VIRTIO_VQCONFIG_OFFS_ADDRESS offsetof(KvmVqConfig, address) /* 64 bit */
-#define VIRTIO_VQCONFIG_OFFS_NUM     offsetof(KvmVqConfig, num)     /* 16 bit */
-#define VIRTIO_VQCONFIG_LEN          sizeof(KvmVqConfig)
-
-#define VIRTIO_RING_LEN			(TARGET_PAGE_SIZE * 3)
-#define VIRTIO_VRING_AVAIL_IDX_OFFS offsetof(struct vring_avail, idx)
-#define VIRTIO_VRING_USED_IDX_OFFS  offsetof(struct vring_used, idx)
-#define S390_DEVICE_PAGES		512
-
-#define TYPE_VIRTIO_S390_DEVICE "virtio-s390-device"
-#define VIRTIO_S390_DEVICE(obj) \
-     OBJECT_CHECK(VirtIOS390Device, (obj), TYPE_VIRTIO_S390_DEVICE)
-#define VIRTIO_S390_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(VirtIOS390DeviceClass, (klass), TYPE_VIRTIO_S390_DEVICE)
-#define VIRTIO_S390_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(VirtIOS390DeviceClass, (obj), TYPE_VIRTIO_S390_DEVICE)
-
-#define TYPE_S390_VIRTIO_BUS "s390-virtio-bus"
-#define S390_VIRTIO_BUS(obj) \
-     OBJECT_CHECK(VirtIOS390Bus, (obj), TYPE_S390_VIRTIO_BUS)
-
-/* virtio-s390-bus */
-
-typedef struct VirtioBusState VirtioS390BusState;
-typedef struct VirtioBusClass VirtioS390BusClass;
-
-#define TYPE_VIRTIO_S390_BUS "virtio-s390-bus"
-#define VIRTIO_S390_BUS(obj) \
-        OBJECT_CHECK(VirtioS390BusState, (obj), TYPE_VIRTIO_S390_BUS)
-#define VIRTIO_S390_BUS_GET_CLASS(obj) \
-        OBJECT_GET_CLASS(VirtioS390BusClass, obj, TYPE_VIRTIO_S390_BUS)
-#define VIRTIO_S390_BUS_CLASS(klass) \
-        OBJECT_CLASS_CHECK(VirtioS390BusClass, klass, TYPE_VIRTIO_S390_BUS)
-
-
-typedef struct VirtIOS390Device VirtIOS390Device;
-
-typedef struct VirtIOS390DeviceClass {
-    DeviceClass qdev;
-    void (*realize)(VirtIOS390Device *dev, Error **errp);
-} VirtIOS390DeviceClass;
-
-struct VirtIOS390Device {
-    DeviceState qdev;
-    ram_addr_t dev_offs;
-    ram_addr_t feat_offs;
-    uint8_t feat_len;
-    VirtIODevice *vdev;
-    VirtioBusState bus;
-};
-
-typedef struct VirtIOS390Bus {
-    BusState bus;
-
-    VirtIOS390Device *console;
-    ram_addr_t dev_page;
-    ram_addr_t dev_offs;
-    ram_addr_t next_ring;
-} VirtIOS390Bus;
-
-
-void s390_virtio_device_update_status(VirtIOS390Device *dev);
-
-VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size);
-
-VirtIOS390Device *s390_virtio_bus_find_vring(VirtIOS390Bus *bus,
-                                             ram_addr_t mem, int *vq_num);
-VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem);
-void s390_virtio_device_sync(VirtIOS390Device *dev);
-void s390_virtio_reset_idx(VirtIOS390Device *dev);
-
-/* virtio-blk-s390 */
-
-#define TYPE_VIRTIO_BLK_S390 "virtio-blk-s390"
-#define VIRTIO_BLK_S390(obj) \
-        OBJECT_CHECK(VirtIOBlkS390, (obj), TYPE_VIRTIO_BLK_S390)
-
-typedef struct VirtIOBlkS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOBlock vdev;
-} VirtIOBlkS390;
-
-/* virtio-scsi-s390 */
-
-#define TYPE_VIRTIO_SCSI_S390 "virtio-scsi-s390"
-#define VIRTIO_SCSI_S390(obj) \
-        OBJECT_CHECK(VirtIOSCSIS390, (obj), TYPE_VIRTIO_SCSI_S390)
-
-typedef struct VirtIOSCSIS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOSCSI vdev;
-} VirtIOSCSIS390;
-
-/* virtio-serial-s390 */
-
-#define TYPE_VIRTIO_SERIAL_S390 "virtio-serial-s390"
-#define VIRTIO_SERIAL_S390(obj) \
-        OBJECT_CHECK(VirtIOSerialS390, (obj), TYPE_VIRTIO_SERIAL_S390)
-
-typedef struct VirtIOSerialS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOSerial vdev;
-} VirtIOSerialS390;
-
-/* virtio-net-s390 */
-
-#define TYPE_VIRTIO_NET_S390 "virtio-net-s390"
-#define VIRTIO_NET_S390(obj) \
-        OBJECT_CHECK(VirtIONetS390, (obj), TYPE_VIRTIO_NET_S390)
-
-typedef struct VirtIONetS390 {
-    VirtIOS390Device parent_obj;
-    VirtIONet vdev;
-} VirtIONetS390;
-
-/* vhost-scsi-s390 */
-
-#ifdef CONFIG_VHOST_SCSI
-#define TYPE_VHOST_SCSI_S390 "vhost-scsi-s390"
-#define VHOST_SCSI_S390(obj) \
-        OBJECT_CHECK(VHostSCSIS390, (obj), TYPE_VHOST_SCSI_S390)
-
-typedef struct VHostSCSIS390 {
-    VirtIOS390Device parent_obj;
-    VHostSCSI vdev;
-} VHostSCSIS390;
-#endif
-
-/* virtio-rng-s390 */
-
-#define TYPE_VIRTIO_RNG_S390 "virtio-rng-s390"
-#define VIRTIO_RNG_S390(obj) \
-        OBJECT_CHECK(VirtIORNGS390, (obj), TYPE_VIRTIO_RNG_S390)
-
-typedef struct VirtIORNGS390 {
-    VirtIOS390Device parent_obj;
-    VirtIORNG vdev;
-} VirtIORNGS390;
-
-#endif
diff --git a/qemu/hw/s390x/s390-virtio-ccw.c b/qemu/hw/s390x/s390-virtio-ccw.c
index 4c51d1a5b..e3df9c78b 100644
--- a/qemu/hw/s390x/s390-virtio-ccw.c
+++ b/qemu/hw/s390x/s390-virtio-ccw.c
@@ -9,6 +9,10 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
 #include "s390-virtio.h"
@@ -19,41 +23,27 @@
 #include "virtio-ccw.h"
 #include "qemu/config-file.h"
 #include "s390-pci-bus.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/compat.h"
+#include "hw/s390x/s390-virtio-ccw.h"
+
+static const char *const reset_dev_types[] = {
+    "virtual-css-bridge",
+    "s390-sclp-event-facility",
+    "s390-flic",
+    "diag288",
+};
 
-#define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
-
-#define S390_CCW_MACHINE(obj) \
-    OBJECT_CHECK(S390CcwMachineState, (obj), TYPE_S390_CCW_MACHINE)
-
-typedef struct S390CcwMachineState {
-    /*< private >*/
-    MachineState parent_obj;
-
-    /*< public >*/
-    bool aes_key_wrap;
-    bool dea_key_wrap;
-} S390CcwMachineState;
-
-void io_subsystem_reset(void)
+void subsystem_reset(void)
 {
-    DeviceState *css, *sclp, *flic, *diag288;
+    DeviceState *dev;
+    int i;
 
-    css = DEVICE(object_resolve_path_type("", "virtual-css-bridge", NULL));
-    if (css) {
-        qdev_reset_all(css);
-    }
-    sclp = DEVICE(object_resolve_path_type("",
-                  "s390-sclp-event-facility", NULL));
-    if (sclp) {
-        qdev_reset_all(sclp);
-    }
-    flic = DEVICE(object_resolve_path_type("", "s390-flic", NULL));
-    if (flic) {
-        qdev_reset_all(flic);
-    }
-    diag288 = DEVICE(object_resolve_path_type("", "diag288", NULL));
-    if (diag288) {
-        qdev_reset_all(diag288);
+    for (i = 0; i < ARRAY_SIZE(reset_dev_types); i++) {
+        dev = DEVICE(object_resolve_path_type("", reset_dev_types[i], NULL));
+        if (dev) {
+            qdev_reset_all(dev);
+        }
     }
 }
 
@@ -99,58 +89,30 @@ static void virtio_ccw_register_hcalls(void)
                                    virtio_ccw_hcall_early_printk);
 }
 
-static void ccw_init(MachineState *machine)
+void s390_memory_init(ram_addr_t mem_size)
 {
-    ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
-    uint8_t *storage_keys;
+
+    /* allocate RAM for core */
+    memory_region_allocate_system_memory(ram, NULL, "s390.ram", mem_size);
+    memory_region_add_subregion(sysmem, 0, ram);
+
+    /* Initialize storage key device */
+    s390_skeys_init();
+}
+
+static void ccw_init(MachineState *machine)
+{
     int ret;
     VirtualCssBus *css_bus;
     DeviceState *dev;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    ram_addr_t pad_size = 0;
-    ram_addr_t maxmem = qemu_opt_get_size(opts, "maxmem", my_ram_size);
-    ram_addr_t standby_mem_size = maxmem - my_ram_size;
-    uint64_t kvm_limit;
-
-    /* The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     * The variable 'mhd->increment_size' is an exponent of 2 that can be
-     * used to calculate the size (in bytes) of an increment. */
-    mhd->increment_size = 20;
-    while ((my_ram_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
-    while ((standby_mem_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
 
-    /* The core and standby memory areas need to be aligned with
-     * the increment size.  In effect, this can cause the
-     * user-specified memory size to be rounded down to align
-     * with the nearest increment boundary. */
-    standby_mem_size = standby_mem_size >> mhd->increment_size
-                                        << mhd->increment_size;
-    my_ram_size = my_ram_size >> mhd->increment_size
-                              << mhd->increment_size;
-
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
-    machine->maxram_size = my_ram_size + standby_mem_size;
-
-    ret = s390_set_memory_limit(machine->maxram_size, &kvm_limit);
-    if (ret == -E2BIG) {
-        hw_error("qemu: host supports a maximum of %" PRIu64 " GB",
-                 kvm_limit >> 30);
-    } else if (ret) {
-        hw_error("qemu: setting the guest size failed");
-    }
+    s390_sclp_init();
+    s390_memory_init(machine->ram_size);
 
     /* get a BUS */
     css_bus = virtual_css_bus_init();
-    s390_sclp_init();
     s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
                       machine->initrd_filename, "s390-ccw.img", true);
     s390_flic_init();
@@ -163,27 +125,8 @@ static void ccw_init(MachineState *machine)
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* allocate RAM for core */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
-
-    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
-       calculate the pad size necessary to force this boundary. */
-    if (standby_mem_size) {
-        if (my_ram_size % MEM_SECTION_SIZE) {
-            pad_size = MEM_SECTION_SIZE - my_ram_size % MEM_SECTION_SIZE;
-        }
-        my_ram_size += standby_mem_size + pad_size;
-        mhd->pad_size = pad_size;
-        mhd->standby_mem_size = standby_mem_size;
-    }
-
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
     /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
+    s390_init_cpus(machine);
 
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
@@ -203,12 +146,54 @@ static void ccw_init(MachineState *machine)
                     gtod_save, gtod_load, kvm_state);
 }
 
+static void s390_cpu_plug(HotplugHandler *hotplug_dev,
+                        DeviceState *dev, Error **errp)
+{
+    gchar *name;
+    S390CPU *cpu = S390_CPU(dev);
+    CPUState *cs = CPU(dev);
+
+    name = g_strdup_printf("cpu[%i]", cpu->env.cpu_num);
+    object_property_set_link(OBJECT(hotplug_dev), OBJECT(cs), name,
+                             errp);
+    g_free(name);
+}
+
+static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        s390_cpu_plug(hotplug_dev, dev, errp);
+    }
+}
+
+static HotplugHandler *s390_get_hotplug_handler(MachineState *machine,
+                                                DeviceState *dev)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        return HOTPLUG_HANDLER(machine);
+    }
+    return NULL;
+}
+
+static void s390_hot_add_cpu(const int64_t id, Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    Error *err = NULL;
+
+    s390x_new_cpu(machine->cpu_model, id, &err);
+    error_propagate(errp, err);
+}
+
 static void ccw_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
 
     mc->init = ccw_init;
+    mc->reset = s390_machine_reset;
+    mc->hot_add_cpu = s390_hot_add_cpu;
     mc->block_default_type = IF_VIRTIO;
     mc->no_cdrom = 1;
     mc->no_floppy = 1;
@@ -217,6 +202,8 @@ static void ccw_machine_class_init(ObjectClass *oc, void *data)
     mc->no_sdcard = 1;
     mc->use_sclp = 1;
     mc->max_cpus = 255;
+    mc->get_hotplug_handler = s390_get_hotplug_handler;
+    hc->plug = s390_machine_device_plug;
     nc->nmi_monitor_handler = s390_nmi;
 }
 
@@ -278,30 +265,117 @@ static const TypeInfo ccw_machine_info = {
     .class_init    = ccw_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_NMI },
+        { TYPE_HOTPLUG_HANDLER},
         { }
     },
 };
 
-static void ccw_machine_2_4_class_init(ObjectClass *oc, void *data)
+#define DEFINE_CCW_MACHINE(suffix, verstr, latest)                            \
+    static void ccw_machine_##suffix##_class_init(ObjectClass *oc,            \
+                                                  void *data)                 \
+    {                                                                         \
+        MachineClass *mc = MACHINE_CLASS(oc);                                 \
+        ccw_machine_##suffix##_class_options(mc);                             \
+        mc->desc = "VirtIO-ccw based S390 machine v" verstr;                  \
+        if (latest) {                                                         \
+            mc->alias = "s390-ccw-virtio";                                    \
+            mc->is_default = 1;                                               \
+        }                                                                     \
+    }                                                                         \
+    static void ccw_machine_##suffix##_instance_init(Object *obj)             \
+    {                                                                         \
+        MachineState *machine = MACHINE(obj);                                 \
+        ccw_machine_##suffix##_instance_options(machine);                     \
+    }                                                                         \
+    static const TypeInfo ccw_machine_##suffix##_info = {                     \
+        .name = MACHINE_TYPE_NAME("s390-ccw-virtio-" verstr),                 \
+        .parent = TYPE_S390_CCW_MACHINE,                                      \
+        .class_init = ccw_machine_##suffix##_class_init,                      \
+        .instance_init = ccw_machine_##suffix##_instance_init,                \
+    };                                                                        \
+    static void ccw_machine_register_##suffix(void)                           \
+    {                                                                         \
+        type_register_static(&ccw_machine_##suffix##_info);                   \
+    }                                                                         \
+    type_init(ccw_machine_register_##suffix)
+
+#define CCW_COMPAT_2_5 \
+        HW_COMPAT_2_5
+
+#define CCW_COMPAT_2_4 \
+        CCW_COMPAT_2_5 \
+        HW_COMPAT_2_4 \
+        {\
+            .driver   = TYPE_S390_SKEYS,\
+            .property = "migration-enabled",\
+            .value    = "off",\
+        },{\
+            .driver   = "virtio-blk-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-balloon-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-serial-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-9p-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-rng-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-net-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "vhost-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },
+
+static void ccw_machine_2_6_instance_options(MachineState *machine)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
+}
 
-    mc->name = "s390-ccw-virtio-2.4";
-    mc->alias = "s390-ccw-virtio";
-    mc->desc = "VirtIO-ccw based S390 machine v2.4";
-    mc->is_default = 1;
+static void ccw_machine_2_6_class_options(MachineClass *mc)
+{
 }
+DEFINE_CCW_MACHINE(2_6, "2.6", true);
 
-static const TypeInfo ccw_machine_2_4_info = {
-    .name          = TYPE_S390_CCW_MACHINE "2.4",
-    .parent        = TYPE_S390_CCW_MACHINE,
-    .class_init    = ccw_machine_2_4_class_init,
-};
+static void ccw_machine_2_5_instance_options(MachineState *machine)
+{
+}
+
+static void ccw_machine_2_5_class_options(MachineClass *mc)
+{
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_5);
+}
+DEFINE_CCW_MACHINE(2_5, "2.5", false);
+
+static void ccw_machine_2_4_instance_options(MachineState *machine)
+{
+    ccw_machine_2_5_instance_options(machine);
+}
+
+static void ccw_machine_2_4_class_options(MachineClass *mc)
+{
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_4);
+}
+DEFINE_CCW_MACHINE(2_4, "2.4", false);
 
 static void ccw_machine_register_types(void)
 {
     type_register_static(&ccw_machine_info);
-    type_register_static(&ccw_machine_2_4_info);
 }
 
 type_init(ccw_machine_register_types)
diff --git a/qemu/hw/s390x/s390-virtio-hcall.c b/qemu/hw/s390x/s390-virtio-hcall.c
index c7bdc2005..23d67d617 100644
--- a/qemu/hw/s390x/s390-virtio-hcall.c
+++ b/qemu/hw/s390x/s390-virtio-hcall.c
@@ -9,6 +9,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "hw/s390x/s390-virtio.h"
 
diff --git a/qemu/hw/s390x/s390-virtio.c b/qemu/hw/s390x/s390-virtio.c
index 1284e77b2..544c61643 100644
--- a/qemu/hw/s390x/s390-virtio.c
+++ b/qemu/hw/s390x/s390-virtio.c
@@ -21,8 +21,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
@@ -30,14 +33,15 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "hw/virtio/virtio.h"
-#include "hw/sysbus.h"
 #include "sysemu/kvm.h"
 #include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
 
-#include "hw/s390x/s390-virtio-bus.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/s390-virtio.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/s390x/ipl.h"
 #include "cpu.h"
 
 //#define DEBUG_S390
@@ -51,94 +55,20 @@
 #endif
 
 #define MAX_BLK_DEVS                    10
-#define ZIPL_FILENAME                   "s390-zipl.rom"
-#define TYPE_S390_MACHINE               "s390-machine"
 
 #define S390_TOD_CLOCK_VALUE_MISSING    0x00
 #define S390_TOD_CLOCK_VALUE_PRESENT    0x01
 
-static VirtIOS390Bus *s390_bus;
-static S390CPU **ipi_states;
+static S390CPU **cpu_states;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
-    if (cpu_addr >= smp_cpus) {
+    if (cpu_addr >= max_cpus) {
         return NULL;
     }
 
-    return ipi_states[cpu_addr];
-}
-
-static int s390_virtio_hcall_notify(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    int r = 0, i;
-
-    if (mem > ram_size) {
-        VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus, mem, &i);
-        if (dev) {
-            /*
-             * Older kernels will use the virtqueue before setting DRIVER_OK.
-             * In this case the feature bits are not yet up to date, meaning
-             * that several funny things can happen, e.g. the guest thinks
-             * EVENT_IDX is on and QEMU thinks it is off. Let's force a feature
-             * and status sync.
-             */
-            if (!(dev->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
-                s390_virtio_device_update_status(dev);
-            }
-            virtio_queue_notify(dev->vdev, i);
-        } else {
-            r = -EINVAL;
-        }
-    } else {
-        /* Early printk */
-    }
-    return r;
-}
-
-static int s390_virtio_hcall_reset(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    VirtIOS390Device *dev;
-
-    dev = s390_virtio_bus_find_mem(s390_bus, mem);
-    if (dev == NULL) {
-        return -EINVAL;
-    }
-    virtio_reset(dev->vdev);
-    address_space_stb(&address_space_memory,
-                      dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0,
-                      MEMTXATTRS_UNSPECIFIED, NULL);
-    s390_virtio_device_sync(dev);
-    s390_virtio_reset_idx(dev);
-
-    return 0;
-}
-
-static int s390_virtio_hcall_set_status(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    int r = 0;
-    VirtIOS390Device *dev;
-
-    dev = s390_virtio_bus_find_mem(s390_bus, mem);
-    if (dev) {
-        s390_virtio_device_update_status(dev);
-    } else {
-        r = -EINVAL;
-    }
-    return r;
-}
-
-static void s390_virtio_register_hcalls(void)
-{
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_NOTIFY,
-                                   s390_virtio_hcall_notify);
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_RESET,
-                                   s390_virtio_hcall_reset);
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_SET_STATUS,
-                                   s390_virtio_hcall_set_status);
+    /* Fast lookup via CPU ID */
+    return cpu_states[cpu_addr];
 }
 
 void s390_init_ipl_dev(const char *kernel_filename,
@@ -147,9 +77,9 @@ void s390_init_ipl_dev(const char *kernel_filename,
                        const char *firmware,
                        bool enforce_bios)
 {
-    DeviceState *dev;
+    Object *new = object_new(TYPE_S390_IPL);
+    DeviceState *dev = DEVICE(new);
 
-    dev  = qdev_create(NULL, "s390-ipl");
     if (kernel_filename) {
         qdev_prop_set_string(dev, "kernel", kernel_filename);
     }
@@ -159,32 +89,35 @@ void s390_init_ipl_dev(const char *kernel_filename,
     qdev_prop_set_string(dev, "cmdline", kernel_cmdline);
     qdev_prop_set_string(dev, "firmware", firmware);
     qdev_prop_set_bit(dev, "enforce_bios", enforce_bios);
-    object_property_add_child(qdev_get_machine(), "s390-ipl",
-                              OBJECT(dev), NULL);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_IPL,
+                              new, NULL);
+    object_unref(new);
     qdev_init_nofail(dev);
 }
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys)
+void s390_init_cpus(MachineState *machine)
 {
     int i;
+    gchar *name;
 
-    if (cpu_model == NULL) {
-        cpu_model = "host";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = "host";
     }
 
-    ipi_states = g_malloc(sizeof(S390CPU *) * smp_cpus);
-
-    for (i = 0; i < smp_cpus; i++) {
-        S390CPU *cpu;
-        CPUState *cs;
+    cpu_states = g_new0(S390CPU *, max_cpus);
 
-        cpu = cpu_s390x_init(cpu_model);
-        cs = CPU(cpu);
+    for (i = 0; i < max_cpus; i++) {
+        name = g_strdup_printf("cpu[%i]", i);
+        object_property_add_link(OBJECT(machine), name, TYPE_S390_CPU,
+                                 (Object **) &cpu_states[i],
+                                 object_property_allow_set_link,
+                                 OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                                 &error_abort);
+        g_free(name);
+    }
 
-        ipi_states[i] = cpu;
-        cs->halted = 1;
-        cs->exception_index = EXCP_HLT;
-        cpu->env.storage_keys = storage_keys;
+    for (i = 0; i < smp_cpus; i++) {
+        s390x_new_cpu(machine->cpu_model, i, &error_fatal);
     }
 }
 
@@ -201,10 +134,7 @@ void s390_create_virtio_net(BusState *bus, const char *name)
             nd->model = g_strdup("virtio");
         }
 
-        if (strcmp(nd->model, "virtio")) {
-            fprintf(stderr, "S390 only supports VirtIO nics\n");
-            exit(1);
-        }
+        qemu_check_nic_model(nd, "virtio");
 
         dev = qdev_create(bus, name);
         qdev_set_nic_properties(dev, nd);
@@ -257,68 +187,6 @@ int gtod_load(QEMUFile *f, void *opaque, int version_id)
     return 0;
 }
 
-/* PC hardware initialisation */
-static void s390_init(MachineState *machine)
-{
-    ram_addr_t my_ram_size = machine->ram_size;
-    MemoryRegion *sysmem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int increment_size = 20;
-    uint8_t *storage_keys;
-    void *virtio_region;
-    hwaddr virtio_region_len;
-    hwaddr virtio_region_start;
-
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or
-     * fewer.
-     */
-    while ((my_ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
-    }
-    my_ram_size = my_ram_size >> increment_size << increment_size;
-
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
-
-    /* get a BUS */
-    s390_bus = s390_virtio_bus_init(&my_ram_size);
-    s390_sclp_init();
-    s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
-                      machine->initrd_filename, ZIPL_FILENAME, false);
-    s390_flic_init();
-
-    /* register hypercalls */
-    s390_virtio_register_hcalls();
-
-    /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
-
-    /* clear virtio region */
-    virtio_region_len = my_ram_size - ram_size;
-    virtio_region_start = ram_size;
-    virtio_region = cpu_physical_memory_map(virtio_region_start,
-                                            &virtio_region_len, true);
-    memset(virtio_region, 0, virtio_region_len);
-    cpu_physical_memory_unmap(virtio_region, virtio_region_len, 1,
-                              virtio_region_len);
-
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
-    /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
-
-    /* Create VirtIO network adapters */
-    s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
-
-    /* Register savevm handler for guest TOD clock */
-    register_savevm(NULL, "todclock", 0, 1, gtod_save, gtod_load, NULL);
-}
-
 void s390_nmi(NMIState *n, int cpu_index, Error **errp)
 {
     CPUState *cs = qemu_get_cpu(cpu_index);
@@ -328,39 +196,15 @@ void s390_nmi(NMIState *n, int cpu_index, Error **errp)
     }
 }
 
-static void s390_machine_class_init(ObjectClass *oc, void *data)
+void s390_machine_reset(void)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
-    NMIClass *nc = NMI_CLASS(oc);
-
-    mc->name = "s390-virtio";
-    mc->alias = "s390";
-    mc->desc = "VirtIO based S390 machine";
-    mc->init = s390_init;
-    mc->block_default_type = IF_VIRTIO;
-    mc->max_cpus = 255;
-    mc->no_serial = 1;
-    mc->no_parallel = 1;
-    mc->use_virtcon = 1;
-    mc->no_floppy = 1;
-    mc->no_cdrom = 1;
-    mc->no_sdcard = 1;
-    nc->nmi_monitor_handler = s390_nmi;
-}
+    S390CPU *ipl_cpu = S390_CPU(qemu_get_cpu(0));
 
-static const TypeInfo s390_machine_info = {
-    .name          = TYPE_S390_MACHINE,
-    .parent        = TYPE_MACHINE,
-    .class_init    = s390_machine_class_init,
-    .interfaces = (InterfaceInfo[]) {
-        { TYPE_NMI },
-        { }
-    },
-};
-
-static void s390_machine_register_types(void)
-{
-    type_register_static(&s390_machine_info);
-}
+    qemu_devices_reset();
+    s390_cmma_reset();
+    s390_crypto_reset();
 
-type_init(s390_machine_register_types)
+    /* all cpus are stopped - configure and start the ipl cpu only */
+    s390_ipl_prepare_cpu(ipl_cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, ipl_cpu);
+}
diff --git a/qemu/hw/s390x/s390-virtio.h b/qemu/hw/s390x/s390-virtio.h
index c84785395..ffd014cb5 100644
--- a/qemu/hw/s390x/s390-virtio.h
+++ b/qemu/hw/s390x/s390-virtio.h
@@ -19,7 +19,7 @@
 typedef int (*s390_virtio_fn)(const uint64_t *args);
 void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn);
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys);
+void s390_init_cpus(MachineState *machine);
 void s390_init_ipl_dev(const char *kernel_filename,
                        const char *kernel_cmdline,
                        const char *initrd_filename,
@@ -27,4 +27,6 @@ void s390_init_ipl_dev(const char *kernel_filename,
                        bool enforce_bios);
 void s390_create_virtio_net(BusState *bus, const char *name);
 void s390_nmi(NMIState *n, int cpu_index, Error **errp);
+void s390_machine_reset(void);
+void s390_memory_init(ram_addr_t mem_size);
 #endif
diff --git a/qemu/hw/s390x/sclp.c b/qemu/hw/s390x/sclp.c
index b3a6c5e5a..85dbe1b60 100644
--- a/qemu/hw/s390x/sclp.c
+++ b/qemu/hw/s390x/sclp.c
@@ -12,42 +12,34 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
-#include "qemu/config-file.h"
+#include "hw/boards.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
 #include "hw/s390x/s390-pci-bus.h"
 
-static inline SCLPEventFacility *get_event_facility(void)
+static inline SCLPDevice *get_sclp_device(void)
 {
-    ObjectProperty *op = object_property_find(qdev_get_machine(),
-                                              TYPE_SCLP_EVENT_FACILITY,
-                                              NULL);
-    assert(op);
-    return op->opaque;
+    return SCLP(object_resolve_path_type("", TYPE_SCLP, NULL));
 }
 
 /* Provide information about the configuration, CPUs and storage */
-static void read_SCP_info(SCCB *sccb)
+static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    MachineState *machine = MACHINE(qdev_get_machine());
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
     int cpu_count = 0;
     int i = 0;
-    int increment_size = 20;
     int rnsize, rnmax;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    int slots = qemu_opt_get_number(opts, "slots", 0);
-    int max_avail_slots = s390_get_memslot_count(kvm_state);
-
-    if (slots > max_avail_slots) {
-        slots = max_avail_slots;
-    }
+    int slots = MIN(machine->ram_slots, s390_get_memslot_count(kvm_state));
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -66,23 +58,8 @@ static void read_SCP_info(SCCB *sccb)
     read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
                                         SCLP_HAS_PCI_RECONFIG);
 
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     */
-    while ((ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
-    }
-    rnmax = ram_size >> increment_size;
-
     /* Memory Hotplug is only supported for the ccw machine type */
     if (mhd) {
-        while ((mhd->standby_mem_size >> increment_size) >
-               MAX_STORAGE_INCREMENTS) {
-            increment_size++;
-        }
-        assert(increment_size == mhd->increment_size);
-
         mhd->standby_subregion_size = MEM_SECTION_SIZE;
         /* Deduct the memory slot already used for core */
         if (slots > 0) {
@@ -108,13 +85,11 @@ static void read_SCP_info(SCCB *sccb)
         }
         mhd->padded_ram_size = ram_size + mhd->pad_size;
         mhd->rzm = 1 << mhd->increment_size;
-        rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
-             >> mhd->increment_size);
 
         read_info->facilities |= cpu_to_be64(SCLP_FC_ASSIGN_ATTACH_READ_STOR);
     }
 
-    rnsize = 1 << (increment_size - 20);
+    rnsize = 1 << (sclp->increment_size - 20);
     if (rnsize <= 128) {
         read_info->rnsize = rnsize;
     } else {
@@ -122,6 +97,7 @@ static void read_SCP_info(SCCB *sccb)
         read_info->rnsize2 = cpu_to_be32(rnsize);
     }
 
+    rnmax = machine->maxram_size >> sclp->increment_size;
     if (rnmax < 0x10000) {
         read_info->rnmax = cpu_to_be16(rnmax);
     } else {
@@ -132,14 +108,17 @@ static void read_SCP_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element0_info(SCCB *sccb)
+static void read_storage_element0_info(SCLPDevice *sclp, SCCB *sccb)
 {
     int i, assigned;
     int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((ram_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -158,12 +137,15 @@ static void read_storage_element0_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element1_info(SCCB *sccb)
+static void read_storage_element1_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((mhd->standby_mem_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -179,13 +161,17 @@ static void read_storage_element1_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
 }
 
-static void attach_storage_element(SCCB *sccb, uint16_t element)
+static void attach_storage_element(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element)
 {
     int i, assigned, subincrement_id;
     AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if (element != 1) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
@@ -203,20 +189,26 @@ static void attach_storage_element(SCCB *sccb, uint16_t element)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void assign_storage(SCCB *sccb)
+static void assign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     uint64_t this_subregion_size;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t assign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    assign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     if ((assign_addr % MEM_SECTION_SIZE == 0) &&
         (assign_addr >= mhd->padded_ram_size)) {
         /* Re-use existing memory region if found */
         mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        memory_region_unref(mr);
         if (!mr) {
 
             MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
@@ -241,7 +233,13 @@ static void assign_storage(SCCB *sccb)
                 this_subregion_size = mhd->standby_subregion_size;
             }
 
-            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size, &error_abort);
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size,
+                                   &error_fatal);
+            /* This is a hack to make memory hotunplug work again. Once we have
+             * subdevices, we have to unparent them when unassigning memory,
+             * instead of doing it via the ref count of the MemoryRegion. */
+            object_ref(OBJECT(standby_ram));
+            object_unparent(OBJECT(standby_ram));
             vmstate_register_ram_global(standby_ram);
             memory_region_add_subregion(sysmem, offset, standby_ram);
         }
@@ -252,15 +250,20 @@ static void assign_storage(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void unassign_storage(SCCB *sccb)
+static void unassign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t unassign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     /* if the addr is a multiple of 256 MB */
     if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
         (unassign_addr >= mhd->padded_ram_size)) {
@@ -269,6 +272,7 @@ static void unassign_storage(SCCB *sccb)
 
         /* find the specified memory region and destroy it */
         mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        memory_region_unref(mr);
         if (mr) {
             int i;
             int is_removable = 1;
@@ -287,8 +291,7 @@ static void unassign_storage(SCCB *sccb)
             }
             if (is_removable) {
                 memory_region_del_subregion(sysmem, mr);
-                object_unparent(OBJECT(mr));
-                g_free(mr);
+                object_unref(OBJECT(mr));
             }
         }
     }
@@ -296,7 +299,7 @@ static void unassign_storage(SCCB *sccb)
 }
 
 /* Provide information about the CPU */
-static void sclp_read_cpu_info(SCCB *sccb)
+static void sclp_read_cpu_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadCpuInfo *cpu_info = (ReadCpuInfo *) sccb;
     CPUState *cpu;
@@ -323,34 +326,35 @@ static void sclp_read_cpu_info(SCCB *sccb)
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void sclp_execute(SCCB *sccb, uint32_t code)
+static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     switch (code & SCLP_CMD_CODE_MASK) {
     case SCLP_CMDW_READ_SCP_INFO:
     case SCLP_CMDW_READ_SCP_INFO_FORCED:
-        read_SCP_info(sccb);
+        sclp_c->read_SCP_info(sclp, sccb);
         break;
     case SCLP_CMDW_READ_CPU_INFO:
-        sclp_read_cpu_info(sccb);
+        sclp_c->read_cpu_info(sclp, sccb);
         break;
     case SCLP_READ_STORAGE_ELEMENT_INFO:
         if (code & 0xff00) {
-            read_storage_element1_info(sccb);
+            sclp_c->read_storage_element1_info(sclp, sccb);
         } else {
-            read_storage_element0_info(sccb);
+            sclp_c->read_storage_element0_info(sclp, sccb);
         }
         break;
     case SCLP_ATTACH_STORAGE_ELEMENT:
-        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        sclp_c->attach_storage_element(sclp, sccb, (code & 0xff00) >> 8);
         break;
     case SCLP_ASSIGN_STORAGE:
-        assign_storage(sccb);
+        sclp_c->assign_storage(sclp, sccb);
         break;
     case SCLP_UNASSIGN_STORAGE:
-        unassign_storage(sccb);
+        sclp_c->unassign_storage(sclp, sccb);
         break;
     case SCLP_CMDW_CONFIGURE_PCI:
         s390_pci_sclp_configure(1, sccb);
@@ -366,6 +370,8 @@ static void sclp_execute(SCCB *sccb, uint32_t code)
 
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
     int r = 0;
     SCCB work_sccb;
 
@@ -400,20 +406,20 @@ int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
         goto out;
     }
 
-    sclp_execute((SCCB *)&work_sccb, code);
+    sclp_c->execute(sclp, (SCCB *)&work_sccb, code);
 
     cpu_physical_memory_write(sccb, &work_sccb,
                               be16_to_cpu(work_sccb.h.length));
 
-    sclp_service_interrupt(sccb);
+    sclp_c->service_interrupt(sclp, sccb);
 
 out:
     return r;
 }
 
-void sclp_service_interrupt(uint32_t sccb)
+static void service_interrupt(SCLPDevice *sclp, uint32_t sccb)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     uint32_t param = sccb & ~3;
@@ -428,17 +434,149 @@ void sclp_service_interrupt(uint32_t sccb)
     s390_sclp_extint(param);
 }
 
+void sclp_service_interrupt(uint32_t sccb)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+
+    sclp_c->service_interrupt(sclp, sccb);
+}
+
 /* qemu object creation and initialization functions */
 
 void s390_sclp_init(void)
 {
-    DeviceState *dev  = qdev_create(NULL, TYPE_SCLP_EVENT_FACILITY);
+    Object *new = object_new(TYPE_SCLP);
 
-    object_property_add_child(qdev_get_machine(), TYPE_SCLP_EVENT_FACILITY,
-                              OBJECT(dev), NULL);
-    qdev_init_nofail(dev);
+    object_property_add_child(qdev_get_machine(), TYPE_SCLP, new,
+                              NULL);
+    object_unref(OBJECT(new));
+    qdev_init_nofail(DEVICE(new));
 }
 
+static void sclp_realize(DeviceState *dev, Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    SCLPDevice *sclp = SCLP(dev);
+    Error *err = NULL;
+    uint64_t hw_limit;
+    int ret;
+
+    object_property_set_bool(OBJECT(sclp->event_facility), true, "realized",
+                             &err);
+    if (err) {
+        goto out;
+    }
+    /*
+     * qdev_device_add searches the sysbus for TYPE_SCLP_EVENTS_BUS. As long
+     * as we can't find a fitting bus via the qom tree, we have to add the
+     * event facility to the sysbus, so e.g. a sclp console can be created.
+     */
+    qdev_set_parent_bus(DEVICE(sclp->event_facility), sysbus_get_default());
+
+    ret = s390_set_memory_limit(machine->maxram_size, &hw_limit);
+    if (ret == -E2BIG) {
+        error_setg(&err, "qemu: host supports a maximum of %" PRIu64 " GB",
+                   hw_limit >> 30);
+    } else if (ret) {
+        error_setg(&err, "qemu: setting the guest size failed");
+    }
+
+out:
+    error_propagate(errp, err);
+}
+
+static void sclp_memory_init(SCLPDevice *sclp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    ram_addr_t initial_mem = machine->ram_size;
+    ram_addr_t max_mem = machine->maxram_size;
+    ram_addr_t standby_mem = max_mem - initial_mem;
+    ram_addr_t pad_mem = 0;
+    int increment_size = 20;
+
+    /* The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     * The variable 'increment_size' is an exponent of 2 that can be
+     * used to calculate the size (in bytes) of an increment. */
+    while ((initial_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
+    }
+    if (machine->ram_slots) {
+        while ((standby_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+            increment_size++;
+        }
+    }
+    sclp->increment_size = increment_size;
+
+    /* The core and standby memory areas need to be aligned with
+     * the increment size.  In effect, this can cause the
+     * user-specified memory size to be rounded down to align
+     * with the nearest increment boundary. */
+    initial_mem = initial_mem >> increment_size << increment_size;
+    standby_mem = standby_mem >> increment_size << increment_size;
+
+    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
+       calculate the pad size necessary to force this boundary. */
+    if (machine->ram_slots && standby_mem) {
+        sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
+
+        if (initial_mem % MEM_SECTION_SIZE) {
+            pad_mem = MEM_SECTION_SIZE - initial_mem % MEM_SECTION_SIZE;
+        }
+        mhd->increment_size = increment_size;
+        mhd->pad_size = pad_mem;
+        mhd->standby_mem_size = standby_mem;
+    }
+    machine->ram_size = initial_mem;
+    machine->maxram_size = initial_mem + pad_mem + standby_mem;
+    /* let's propagate the changed ram size into the global variable. */
+    ram_size = initial_mem;
+}
+
+static void sclp_init(Object *obj)
+{
+    SCLPDevice *sclp = SCLP(obj);
+    Object *new;
+
+    new = object_new(TYPE_SCLP_EVENT_FACILITY);
+    object_property_add_child(obj, TYPE_SCLP_EVENT_FACILITY, new, NULL);
+    object_unref(new);
+    sclp->event_facility = EVENT_FACILITY(new);
+
+    sclp_memory_init(sclp);
+}
+
+static void sclp_class_init(ObjectClass *oc, void *data)
+{
+    SCLPDeviceClass *sc = SCLP_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->desc = "SCLP (Service-Call Logical Processor)";
+    dc->realize = sclp_realize;
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+
+    sc->read_SCP_info = read_SCP_info;
+    sc->read_storage_element0_info = read_storage_element0_info;
+    sc->read_storage_element1_info = read_storage_element1_info;
+    sc->attach_storage_element = attach_storage_element;
+    sc->assign_storage = assign_storage;
+    sc->unassign_storage = unassign_storage;
+    sc->read_cpu_info = sclp_read_cpu_info;
+    sc->execute = sclp_execute;
+    sc->service_interrupt = service_interrupt;
+}
+
+static TypeInfo sclp_info = {
+    .name = TYPE_SCLP,
+    .parent = TYPE_DEVICE,
+    .instance_init = sclp_init,
+    .instance_size = sizeof(SCLPDevice),
+    .class_init = sclp_class_init,
+    .class_size = sizeof(SCLPDeviceClass),
+};
+
 sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void)
 {
     DeviceState *dev;
@@ -475,5 +613,6 @@ static TypeInfo sclp_memory_hotplug_dev_info = {
 static void register_types(void)
 {
     type_register_static(&sclp_memory_hotplug_dev_info);
+    type_register_static(&sclp_info);
 }
 type_init(register_types);
diff --git a/qemu/hw/s390x/sclpcpu.c b/qemu/hw/s390x/sclpcpu.c
index 2fe8b5aa4..b1f3ef8c7 100644
--- a/qemu/hw/s390x/sclpcpu.c
+++ b/qemu/hw/s390x/sclpcpu.c
@@ -12,6 +12,7 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
@@ -25,13 +26,16 @@ typedef struct ConfigMgtData {
     uint8_t event_qualifier;
 } QEMU_PACKED ConfigMgtData;
 
-static qemu_irq *irq_cpu_hotplug; /* Only used in this file */
-
 #define EVENT_QUAL_CPU_CHANGE  1
 
 void raise_irq_cpu_hotplug(void)
 {
-    qemu_irq_raise(*irq_cpu_hotplug);
+    Object *obj = object_resolve_path_type("", TYPE_SCLP_CPU_HOTPLUG, NULL);
+
+    SCLP_EVENT(obj)->event_pending = true;
+
+    /* Trigger SCLP read operation */
+    sclp_service_interrupt(0);
 }
 
 static unsigned int send_mask(void)
@@ -70,36 +74,19 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
     return 1;
 }
 
-static void trigger_signal(void *opaque, int n, int level)
-{
-    SCLPEvent *event = opaque;
-    event->event_pending = true;
-
-    /* Trigger SCLP read operation */
-    sclp_service_interrupt(0);
-}
-
-static int irq_cpu_hotplug_init(SCLPEvent *event)
-{
-    irq_cpu_hotplug = qemu_allocate_irqs(trigger_signal, event, 1);
-    return 0;
-}
-
 static void cpu_class_init(ObjectClass *oc, void *data)
 {
     SCLPEventClass *k = SCLP_EVENT_CLASS(oc);
     DeviceClass *dc = DEVICE_CLASS(oc);
 
-    k->init = irq_cpu_hotplug_init;
     k->get_send_mask = send_mask;
     k->get_receive_mask = receive_mask;
     k->read_event_data = read_event_data;
-    k->write_event_data = NULL;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo sclp_cpu_info = {
-    .name          = "sclp-cpu-hotplug",
+    .name          = TYPE_SCLP_CPU_HOTPLUG,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = cpu_class_init,
diff --git a/qemu/hw/s390x/sclpquiesce.c b/qemu/hw/s390x/sclpquiesce.c
index ffa555313..c0ecab9c3 100644
--- a/qemu/hw/s390x/sclpquiesce.c
+++ b/qemu/hw/s390x/sclpquiesce.c
@@ -11,6 +11,7 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include <hw/qdev.h>
 #include "sysemu/sysemu.h"
 #include "hw/s390x/sclp.h"
@@ -66,7 +67,7 @@ static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
 }
 
 static const VMStateDescription vmstate_sclpquiesce = {
-    .name = "sclpquiesce",
+    .name = TYPE_SCLP_QUIESCE,
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
@@ -127,7 +128,7 @@ static void quiesce_class_init(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo sclp_quiesce_info = {
-    .name          = "sclpquiesce",
+    .name          = TYPE_SCLP_QUIESCE,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = quiesce_class_init,
diff --git a/qemu/hw/s390x/virtio-ccw.c b/qemu/hw/s390x/virtio-ccw.c
index d36373e88..d51642db0 100644
--- a/qemu/hw/s390x/virtio-ccw.c
+++ b/qemu/hw/s390x/virtio-ccw.c
@@ -10,6 +10,8 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
@@ -31,69 +33,6 @@
 #include "virtio-ccw.h"
 #include "trace.h"
 
-static QTAILQ_HEAD(, IndAddr) indicator_addresses =
-    QTAILQ_HEAD_INITIALIZER(indicator_addresses);
-
-static IndAddr *get_indicator(hwaddr ind_addr, int len)
-{
-    IndAddr *indicator;
-
-    QTAILQ_FOREACH(indicator, &indicator_addresses, sibling) {
-        if (indicator->addr == ind_addr) {
-            indicator->refcnt++;
-            return indicator;
-        }
-    }
-    indicator = g_new0(IndAddr, 1);
-    indicator->addr = ind_addr;
-    indicator->len = len;
-    indicator->refcnt = 1;
-    QTAILQ_INSERT_TAIL(&indicator_addresses, indicator, sibling);
-    return indicator;
-}
-
-static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
-                               bool do_map)
-{
-    S390FLICState *fs = s390_get_flic();
-    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
-
-    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
-}
-
-static void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
-{
-    assert(indicator->refcnt > 0);
-    indicator->refcnt--;
-    if (indicator->refcnt > 0) {
-        return;
-    }
-    QTAILQ_REMOVE(&indicator_addresses, indicator, sibling);
-    if (indicator->map) {
-        s390_io_adapter_map(adapter, indicator->map, false);
-    }
-    g_free(indicator);
-}
-
-static int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
-{
-    int ret;
-
-    if (indicator->map) {
-        return 0; /* already mapped is not an error */
-    }
-    indicator->map = indicator->addr;
-    ret = s390_io_adapter_map(adapter, indicator->map, true);
-    if ((ret != 0) && (ret != -ENOSYS)) {
-        goto out_err;
-    }
-    return 0;
-
-out_err:
-    indicator->map = 0;
-    return ret;
-}
-
 static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtioCcwDevice *dev);
 
@@ -307,11 +246,18 @@ static int virtio_ccw_set_vqs(SubchDev *sch, VqInfoBlock *info,
     if (!desc) {
         virtio_queue_set_vector(vdev, index, VIRTIO_NO_VECTOR);
     } else {
-        /* Fail if we don't have a big enough queue. */
-        /* TODO: Add interface to handle vring.num changing */
-        if (virtio_queue_get_num(vdev, index) > num) {
+        if (info) {
+            /* virtio-1 allows changing the ring size. */
+            if (virtio_queue_get_num(vdev, index) < num) {
+                /* Fail if we exceed the maximum number. */
+                return -EINVAL;
+            }
+            virtio_queue_set_num(vdev, index, num);
+        } else if (virtio_queue_get_num(vdev, index) > num) {
+            /* Fail if we don't have a big enough queue. */
             return -EINVAL;
         }
+        /* We ignore possible increased num for legacy for compatibility. */
         virtio_queue_set_vector(vdev, index, index);
     }
     /* tell notify handler in case of config change */
@@ -460,16 +406,19 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
                                                 MEMTXATTRS_UNSPECIFIED,
                                                 NULL);
             if (features.index == 0) {
-                features.features = (uint32_t)vdev->host_features;
-            } else if (features.index == 1) {
-                features.features = (uint32_t)(vdev->host_features >> 32);
+                if (dev->revision >= 1) {
+                    /* Don't offer legacy features for modern devices. */
+                    features.features = (uint32_t)
+                        (vdev->host_features & ~VIRTIO_LEGACY_FEATURES);
+                } else {
+                    features.features = (uint32_t)vdev->host_features;
+                }
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
                 /*
-                 * Don't offer version 1 to the guest if it did not
-                 * negotiate at least revision 1.
+                 * Only offer feature bits beyond 31 if the guest has
+                 * negotiated at least revision 1.
                  */
-                if (dev->revision <= 0) {
-                    features.features &= ~(1 << (VIRTIO_F_VERSION_1 - 32));
-                }
+                features.features = (uint32_t)(vdev->host_features >> 32);
             } else {
                 /* Return zeroes if the guest supports more feature bits. */
                 features.features = 0;
@@ -508,14 +457,12 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
                 virtio_set_features(vdev,
                                     (vdev->guest_features & 0xffffffff00000000ULL) |
                                     features.features);
-            } else if (features.index == 1) {
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
                 /*
-                 * The guest should not set version 1 if it didn't
-                 * negotiate a revision >= 1.
+                 * If the guest did not negotiate at least revision 1,
+                 * we did not offer it any feature bits beyond 31. Such a
+                 * guest passing us any bit here is therefore buggy.
                  */
-                if (dev->revision <= 0) {
-                    features.features &= ~(1 << (VIRTIO_F_VERSION_1 - 32));
-                }
                 virtio_set_features(vdev,
                                     (vdev->guest_features & 0x00000000ffffffffULL) |
                                     ((uint64_t)features.features << 32));
@@ -766,7 +713,7 @@ static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
          * need to fetch it here. Nothing to do for now, though.
          */
         if (dev->revision >= 0 ||
-            revinfo.revision > virtio_ccw_rev_max(vdev)) {
+            revinfo.revision > virtio_ccw_rev_max(dev)) {
             ret = -ENOSYS;
             break;
         }
@@ -1169,7 +1116,8 @@ static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
     SubchDev *sch = dev->sch;
     uint64_t indicators;
 
-    if (vector >= 128) {
+    /* queue indicators + secondary indicators */
+    if (vector >= VIRTIO_CCW_QUEUE_MAX + 64) {
         return;
     }
 
@@ -1539,10 +1487,25 @@ static void virtio_ccw_device_plugged(DeviceState *d, Error **errp)
 
     sch->id.cu_model = virtio_bus_get_vdev_id(&dev->bus);
 
+    if (dev->max_rev >= 1) {
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
+    }
+
     css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid,
                           d->hotplugged, 1);
 }
 
+static void virtio_ccw_post_plugged(DeviceState *d, Error **errp)
+{
+   VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+   VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+
+   if (!virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        /* A backend didn't support modern virtio. */
+       dev->max_rev = 0;
+   }
+}
+
 static void virtio_ccw_device_unplugged(DeviceState *d)
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
@@ -1555,6 +1518,8 @@ static Property virtio_ccw_net_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1582,6 +1547,8 @@ static Property virtio_ccw_blk_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1609,6 +1576,8 @@ static Property virtio_ccw_serial_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1636,6 +1605,8 @@ static Property virtio_ccw_balloon_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1663,6 +1634,8 @@ static Property virtio_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1689,6 +1662,8 @@ static const TypeInfo virtio_ccw_scsi = {
 #ifdef CONFIG_VHOST_SCSI
 static Property vhost_ccw_scsi_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1727,6 +1702,8 @@ static Property virtio_ccw_rng_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1865,6 +1842,7 @@ static void virtio_ccw_bus_class_init(ObjectClass *klass, void *data)
     k->save_config = virtio_ccw_save_config;
     k->load_config = virtio_ccw_load_config;
     k->device_plugged = virtio_ccw_device_plugged;
+    k->post_plugged = virtio_ccw_post_plugged;
     k->device_unplugged = virtio_ccw_device_unplugged;
 }
 
@@ -1880,6 +1858,8 @@ static Property virtio_ccw_9p_properties[] = {
     DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
             VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/qemu/hw/s390x/virtio-ccw.h b/qemu/hw/s390x/virtio-ccw.h
index 692ddd731..66c831ba8 100644
--- a/qemu/hw/s390x/virtio-ccw.h
+++ b/qemu/hw/s390x/virtio-ccw.h
@@ -23,7 +23,8 @@
 #include <hw/virtio/virtio-balloon.h>
 #include <hw/virtio/virtio-rng.h>
 #include <hw/virtio/virtio-bus.h>
-#include <hw/s390x/s390_flic.h>
+
+#include "css.h"
 
 #define VIRTUAL_CSSID 0xfe
 
@@ -75,19 +76,12 @@ typedef struct VirtIOCCWDeviceClass {
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD   (1 << VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT)
 
-typedef struct IndAddr {
-    hwaddr addr;
-    uint64_t map;
-    unsigned long refcnt;
-    int len;
-    QTAILQ_ENTRY(IndAddr) sibling;
-} IndAddr;
-
 struct VirtioCcwDevice {
     DeviceState parent_obj;
     SubchDev *sch;
     char *bus_id;
     int revision;
+    uint32_t max_rev;
     VirtioBusState bus;
     bool ioeventfd_started;
     bool ioeventfd_disabled;
@@ -102,9 +96,10 @@ struct VirtioCcwDevice {
 };
 
 /* The maximum virtio revision we support. */
-static inline int virtio_ccw_rev_max(VirtIODevice *vdev)
+#define VIRTIO_CCW_MAX_REV 1
+static inline int virtio_ccw_rev_max(VirtioCcwDevice *dev)
 {
-    return 0;
+    return dev->max_rev;
 }
 
 /* virtual css bus type */
@@ -208,7 +203,7 @@ VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch);
 
 typedef struct V9fsCCWState {
     VirtioCcwDevice parent_obj;
-    V9fsState vdev;
+    V9fsVirtioState vdev;
 } V9fsCCWState;
 
 #endif /* CONFIG_VIRTFS */
diff --git a/qemu/hw/scsi/Makefile.objs b/qemu/hw/scsi/Makefile.objs
index 40c79d34c..5a2248be3 100644
--- a/qemu/hw/scsi/Makefile.objs
+++ b/qemu/hw/scsi/Makefile.objs
@@ -1,6 +1,7 @@
 common-obj-y += scsi-disk.o
 common-obj-y += scsi-generic.o scsi-bus.o
 common-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
+common-obj-$(CONFIG_MPTSAS_SCSI_PCI) += mptsas.o mptconfig.o mptendian.o
 common-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
 common-obj-$(CONFIG_VMW_PVSCSI_SCSI_PCI) += vmw_pvscsi.o
 common-obj-$(CONFIG_ESP) += esp.o
diff --git a/qemu/hw/scsi/esp-pci.c b/qemu/hw/scsi/esp-pci.c
index 8d2242d0a..595f88b35 100644
--- a/qemu/hw/scsi/esp-pci.c
+++ b/qemu/hw/scsi/esp-pci.c
@@ -23,10 +23,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/nvram/eeprom93xx.h"
 #include "hw/scsi/esp.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 
 #define TYPE_AM53C974_DEVICE "am53c974"
diff --git a/qemu/hw/scsi/esp.c b/qemu/hw/scsi/esp.c
index 272d13d63..8961be2f3 100644
--- a/qemu/hw/scsi/esp.c
+++ b/qemu/hw/scsi/esp.c
@@ -23,9 +23,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/scsi/esp.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 
 /*
diff --git a/qemu/hw/scsi/lsi53c895a.c b/qemu/hw/scsi/lsi53c895a.c
index c5b0cc5ca..df205cdaf 100644
--- a/qemu/hw/scsi/lsi53c895a.c
+++ b/qemu/hw/scsi/lsi53c895a.c
@@ -13,7 +13,7 @@
  * as well-behaved operating systems will not try to use them.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
diff --git a/qemu/hw/scsi/megasas.c b/qemu/hw/scsi/megasas.c
index a04369c5a..a63a58155 100644
--- a/qemu/hw/scsi/megasas.c
+++ b/qemu/hw/scsi/megasas.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "sysemu/dma.h"
@@ -431,7 +432,7 @@ static uint64_t megasas_fw_time(void)
 static uint64_t megasas_get_sata_addr(uint16_t id)
 {
     uint64_t addr = (0x1221ULL << 48);
-    return addr & (id << 24);
+    return addr | ((uint64_t)id << 24);
 }
 
 /*
@@ -718,7 +719,7 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
     BusChild *kid;
     int num_pd_disks = 0;
 
-    memset(&info, 0x0, cmd->iov_size);
+    memset(&info, 0x0, dcmd_size);
     if (cmd->iov_size < dcmd_size) {
         trace_megasas_dcmd_invalid_xfer_len(cmd->index, cmd->iov_size,
                                             dcmd_size);
@@ -744,7 +745,7 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
     info.device.type = MFI_INFO_DEV_SAS3G;
     info.device.port_count = 8;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t pd_id;
 
         if (num_pd_disks < 8) {
@@ -757,7 +758,7 @@ static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
 
     memcpy(info.product_name, base_class->product_name, 24);
     snprintf(info.serial_number, 32, "%s", s->hba_serial);
-    snprintf(info.package_version, 0x60, "%s-QEMU", QEMU_VERSION);
+    snprintf(info.package_version, 0x60, "%s-QEMU", qemu_hw_version());
     memcpy(info.image_component[0].name, "APP", 3);
     snprintf(info.image_component[0].version, 10, "%s-QEMU",
              base_class->product_version);
@@ -960,7 +961,7 @@ static int megasas_dcmd_pd_get_list(MegasasState *s, MegasasCmd *cmd)
         max_pd_disks = MFI_MAX_SYS_PDS;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t pd_id;
 
         if (num_pd_disks >= max_pd_disks)
@@ -1136,7 +1137,7 @@ static int megasas_dcmd_ld_get_list(MegasasState *s, MegasasCmd *cmd)
         max_ld_disks = MFI_MAX_LD;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
         if (num_ld_disks >= max_ld_disks) {
             break;
@@ -1187,7 +1188,7 @@ static int megasas_dcmd_ld_list_query(MegasasState *s, MegasasCmd *cmd)
         max_ld_disks = MFI_MAX_LD;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
         if (num_ld_disks >= max_ld_disks) {
             break;
@@ -1327,7 +1328,7 @@ static int megasas_dcmd_cfg_read(MegasasState *s, MegasasCmd *cmd)
     ld_offset = array_offset + sizeof(struct mfi_array) * num_pd_disks;
 
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t sdev_id = ((sdev->id & 0xFF) << 8) | (sdev->lun & 0xFF);
         struct mfi_array *array;
         struct mfi_ld_config *ld;
@@ -2237,7 +2238,7 @@ static void megasas_soft_reset(MegasasState *s)
          * after the initial reset.
          */
         QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-            SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+            SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
             sdev->unit_attention = SENSE_CODE(NO_SENSE);
             scsi_device_unit_attention_reported(sdev);
diff --git a/qemu/hw/scsi/mpi.h b/qemu/hw/scsi/mpi.h
new file mode 100644
index 000000000..0568e1950
--- /dev/null
+++ b/qemu/hw/scsi/mpi.h
@@ -0,0 +1,1153 @@
+/*-
+ * Based on FreeBSD sys/dev/mpt/mpilib headers.
+ *
+ * Copyright (c) 2000-2010, LSI Logic Corporation and its contributors.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon including
+ *    a substantially similar Disclaimer requirement for further binary
+ *    redistribution.
+ * 3. Neither the name of the LSI Logic Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE COPYRIGHT
+ * OWNER OR CONTRIBUTOR IS ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef MPI_H
+#define MPI_H
+
+enum {
+    MPI_FUNCTION_SCSI_IO_REQUEST                = 0x00,
+    MPI_FUNCTION_SCSI_TASK_MGMT                 = 0x01,
+    MPI_FUNCTION_IOC_INIT                       = 0x02,
+    MPI_FUNCTION_IOC_FACTS                      = 0x03,
+    MPI_FUNCTION_CONFIG                         = 0x04,
+    MPI_FUNCTION_PORT_FACTS                     = 0x05,
+    MPI_FUNCTION_PORT_ENABLE                    = 0x06,
+    MPI_FUNCTION_EVENT_NOTIFICATION             = 0x07,
+    MPI_FUNCTION_EVENT_ACK                      = 0x08,
+    MPI_FUNCTION_FW_DOWNLOAD                    = 0x09,
+    MPI_FUNCTION_TARGET_CMD_BUFFER_POST         = 0x0A,
+    MPI_FUNCTION_TARGET_ASSIST                  = 0x0B,
+    MPI_FUNCTION_TARGET_STATUS_SEND             = 0x0C,
+    MPI_FUNCTION_TARGET_MODE_ABORT              = 0x0D,
+    MPI_FUNCTION_FC_LINK_SRVC_BUF_POST          = 0x0E,
+    MPI_FUNCTION_FC_LINK_SRVC_RSP               = 0x0F,
+    MPI_FUNCTION_FC_EX_LINK_SRVC_SEND           = 0x10,
+    MPI_FUNCTION_FC_ABORT                       = 0x11,
+    MPI_FUNCTION_FW_UPLOAD                      = 0x12,
+    MPI_FUNCTION_FC_COMMON_TRANSPORT_SEND       = 0x13,
+    MPI_FUNCTION_FC_PRIMITIVE_SEND              = 0x14,
+
+    MPI_FUNCTION_RAID_ACTION                    = 0x15,
+    MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH       = 0x16,
+
+    MPI_FUNCTION_TOOLBOX                        = 0x17,
+
+    MPI_FUNCTION_SCSI_ENCLOSURE_PROCESSOR       = 0x18,
+
+    MPI_FUNCTION_MAILBOX                        = 0x19,
+
+    MPI_FUNCTION_SMP_PASSTHROUGH                = 0x1A,
+    MPI_FUNCTION_SAS_IO_UNIT_CONTROL            = 0x1B,
+    MPI_FUNCTION_SATA_PASSTHROUGH               = 0x1C,
+
+    MPI_FUNCTION_DIAG_BUFFER_POST               = 0x1D,
+    MPI_FUNCTION_DIAG_RELEASE                   = 0x1E,
+
+    MPI_FUNCTION_SCSI_IO_32                     = 0x1F,
+
+    MPI_FUNCTION_LAN_SEND                       = 0x20,
+    MPI_FUNCTION_LAN_RECEIVE                    = 0x21,
+    MPI_FUNCTION_LAN_RESET                      = 0x22,
+
+    MPI_FUNCTION_TARGET_ASSIST_EXTENDED         = 0x23,
+    MPI_FUNCTION_TARGET_CMD_BUF_BASE_POST       = 0x24,
+    MPI_FUNCTION_TARGET_CMD_BUF_LIST_POST       = 0x25,
+
+    MPI_FUNCTION_INBAND_BUFFER_POST             = 0x28,
+    MPI_FUNCTION_INBAND_SEND                    = 0x29,
+    MPI_FUNCTION_INBAND_RSP                     = 0x2A,
+    MPI_FUNCTION_INBAND_ABORT                   = 0x2B,
+
+    MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET         = 0x40,
+    MPI_FUNCTION_IO_UNIT_RESET                  = 0x41,
+    MPI_FUNCTION_HANDSHAKE                      = 0x42,
+    MPI_FUNCTION_REPLY_FRAME_REMOVAL            = 0x43,
+    MPI_FUNCTION_HOST_PAGEBUF_ACCESS_CONTROL    = 0x44,
+};
+
+/****************************************************************************/
+/*  Registers                                                               */
+/****************************************************************************/
+
+enum {
+    MPI_IOC_STATE_RESET                 = 0x00000000,
+    MPI_IOC_STATE_READY                 = 0x10000000,
+    MPI_IOC_STATE_OPERATIONAL           = 0x20000000,
+    MPI_IOC_STATE_FAULT                 = 0x40000000,
+
+    MPI_DOORBELL_OFFSET                 = 0x00000000,
+    MPI_DOORBELL_ACTIVE                 = 0x08000000, /* DoorbellUsed */
+    MPI_DOORBELL_WHO_INIT_MASK          = 0x07000000,
+    MPI_DOORBELL_WHO_INIT_SHIFT         = 24,
+    MPI_DOORBELL_FUNCTION_MASK          = 0xFF000000,
+    MPI_DOORBELL_FUNCTION_SHIFT         = 24,
+    MPI_DOORBELL_ADD_DWORDS_MASK        = 0x00FF0000,
+    MPI_DOORBELL_ADD_DWORDS_SHIFT       = 16,
+    MPI_DOORBELL_DATA_MASK              = 0x0000FFFF,
+    MPI_DOORBELL_FUNCTION_SPECIFIC_MASK = 0x0000FFFF,
+
+    MPI_DB_HPBAC_VALUE_MASK             = 0x0000F000,
+    MPI_DB_HPBAC_ENABLE_ACCESS          = 0x01,
+    MPI_DB_HPBAC_DISABLE_ACCESS         = 0x02,
+    MPI_DB_HPBAC_FREE_BUFFER            = 0x03,
+
+    MPI_WRITE_SEQUENCE_OFFSET           = 0x00000004,
+    MPI_WRSEQ_KEY_VALUE_MASK            = 0x0000000F,
+    MPI_WRSEQ_1ST_KEY_VALUE             = 0x04,
+    MPI_WRSEQ_2ND_KEY_VALUE             = 0x0B,
+    MPI_WRSEQ_3RD_KEY_VALUE             = 0x02,
+    MPI_WRSEQ_4TH_KEY_VALUE             = 0x07,
+    MPI_WRSEQ_5TH_KEY_VALUE             = 0x0D,
+
+    MPI_DIAGNOSTIC_OFFSET               = 0x00000008,
+    MPI_DIAG_CLEAR_FLASH_BAD_SIG        = 0x00000400,
+    MPI_DIAG_PREVENT_IOC_BOOT           = 0x00000200,
+    MPI_DIAG_DRWE                       = 0x00000080,
+    MPI_DIAG_FLASH_BAD_SIG              = 0x00000040,
+    MPI_DIAG_RESET_HISTORY              = 0x00000020,
+    MPI_DIAG_RW_ENABLE                  = 0x00000010,
+    MPI_DIAG_RESET_ADAPTER              = 0x00000004,
+    MPI_DIAG_DISABLE_ARM                = 0x00000002,
+    MPI_DIAG_MEM_ENABLE                 = 0x00000001,
+
+    MPI_TEST_BASE_ADDRESS_OFFSET        = 0x0000000C,
+
+    MPI_DIAG_RW_DATA_OFFSET             = 0x00000010,
+
+    MPI_DIAG_RW_ADDRESS_OFFSET          = 0x00000014,
+
+    MPI_HOST_INTERRUPT_STATUS_OFFSET    = 0x00000030,
+    MPI_HIS_IOP_DOORBELL_STATUS         = 0x80000000,
+    MPI_HIS_REPLY_MESSAGE_INTERRUPT     = 0x00000008,
+    MPI_HIS_DOORBELL_INTERRUPT          = 0x00000001,
+
+    MPI_HOST_INTERRUPT_MASK_OFFSET      = 0x00000034,
+    MPI_HIM_RIM                         = 0x00000008,
+    MPI_HIM_DIM                         = 0x00000001,
+
+    MPI_REQUEST_QUEUE_OFFSET            = 0x00000040,
+    MPI_REQUEST_POST_FIFO_OFFSET        = 0x00000040,
+
+    MPI_REPLY_QUEUE_OFFSET              = 0x00000044,
+    MPI_REPLY_POST_FIFO_OFFSET          = 0x00000044,
+    MPI_REPLY_FREE_FIFO_OFFSET          = 0x00000044,
+
+    MPI_HI_PRI_REQUEST_QUEUE_OFFSET     = 0x00000048,
+};
+
+#define MPI_ADDRESS_REPLY_A_BIT          0x80000000
+
+/****************************************************************************/
+/*  Scatter/gather elements                                                 */
+/****************************************************************************/
+
+typedef struct MPISGEntry {
+    uint32_t                FlagsLength;
+    union
+    {
+        uint32_t            Address32;
+        uint64_t            Address64;
+    } u;
+} QEMU_PACKED MPISGEntry;
+
+/* Flags field bit definitions */
+
+enum {
+    MPI_SGE_FLAGS_LAST_ELEMENT              = 0x80000000,
+    MPI_SGE_FLAGS_END_OF_BUFFER             = 0x40000000,
+    MPI_SGE_FLAGS_ELEMENT_TYPE_MASK         = 0x30000000,
+    MPI_SGE_FLAGS_LOCAL_ADDRESS             = 0x08000000,
+    MPI_SGE_FLAGS_DIRECTION                 = 0x04000000,
+    MPI_SGE_FLAGS_64_BIT_ADDRESSING         = 0x02000000,
+    MPI_SGE_FLAGS_END_OF_LIST               = 0x01000000,
+
+    MPI_SGE_LENGTH_MASK                     = 0x00FFFFFF,
+    MPI_SGE_CHAIN_LENGTH_MASK               = 0x0000FFFF,
+
+    MPI_SGE_FLAGS_TRANSACTION_ELEMENT       = 0x00000000,
+    MPI_SGE_FLAGS_SIMPLE_ELEMENT            = 0x10000000,
+    MPI_SGE_FLAGS_CHAIN_ELEMENT             = 0x30000000,
+
+    /* Direction */
+
+    MPI_SGE_FLAGS_IOC_TO_HOST               = 0x00000000,
+    MPI_SGE_FLAGS_HOST_TO_IOC               = 0x04000000,
+
+    MPI_SGE_CHAIN_OFFSET_MASK               = 0x00FF0000,
+};
+
+#define MPI_SGE_CHAIN_OFFSET_SHIFT 16
+
+/****************************************************************************/
+/* Standard message request header for all request messages                 */
+/****************************************************************************/
+
+typedef struct MPIRequestHeader {
+    uint8_t                 Reserved[2];      /* function specific */
+    uint8_t                 ChainOffset;
+    uint8_t                 Function;
+    uint8_t                 Reserved1[3];     /* function specific */
+    uint8_t                 MsgFlags;
+    uint32_t                MsgContext;
+} QEMU_PACKED MPIRequestHeader;
+
+
+typedef struct MPIDefaultReply {
+    uint8_t                 Reserved[2];      /* function specific */
+    uint8_t                 MsgLength;
+    uint8_t                 Function;
+    uint8_t                 Reserved1[3];     /* function specific */
+    uint8_t                 MsgFlags;
+    uint32_t                MsgContext;
+    uint8_t                 Reserved2[2];     /* function specific */
+    uint16_t                IOCStatus;
+    uint32_t                IOCLogInfo;
+} QEMU_PACKED MPIDefaultReply;
+
+/* MsgFlags definition for all replies */
+
+#define MPI_MSGFLAGS_CONTINUATION_REPLY         (0x80)
+
+enum {
+
+    /************************************************************************/
+    /*  Common IOCStatus values for all replies                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SUCCESS                   = 0x0000,
+    MPI_IOCSTATUS_INVALID_FUNCTION          = 0x0001,
+    MPI_IOCSTATUS_BUSY                      = 0x0002,
+    MPI_IOCSTATUS_INVALID_SGL               = 0x0003,
+    MPI_IOCSTATUS_INTERNAL_ERROR            = 0x0004,
+    MPI_IOCSTATUS_RESERVED                  = 0x0005,
+    MPI_IOCSTATUS_INSUFFICIENT_RESOURCES    = 0x0006,
+    MPI_IOCSTATUS_INVALID_FIELD             = 0x0007,
+    MPI_IOCSTATUS_INVALID_STATE             = 0x0008,
+    MPI_IOCSTATUS_OP_STATE_NOT_SUPPORTED    = 0x0009,
+
+    /************************************************************************/
+    /*  Config IOCStatus values                                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_CONFIG_INVALID_ACTION     = 0x0020,
+    MPI_IOCSTATUS_CONFIG_INVALID_TYPE       = 0x0021,
+    MPI_IOCSTATUS_CONFIG_INVALID_PAGE       = 0x0022,
+    MPI_IOCSTATUS_CONFIG_INVALID_DATA       = 0x0023,
+    MPI_IOCSTATUS_CONFIG_NO_DEFAULTS        = 0x0024,
+    MPI_IOCSTATUS_CONFIG_CANT_COMMIT        = 0x0025,
+
+    /************************************************************************/
+    /*  SCSIIO Reply = SPI & FCP, initiator values                           */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SCSI_RECOVERED_ERROR      = 0x0040,
+    MPI_IOCSTATUS_SCSI_INVALID_BUS          = 0x0041,
+    MPI_IOCSTATUS_SCSI_INVALID_TARGETID     = 0x0042,
+    MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE     = 0x0043,
+    MPI_IOCSTATUS_SCSI_DATA_OVERRUN         = 0x0044,
+    MPI_IOCSTATUS_SCSI_DATA_UNDERRUN        = 0x0045,
+    MPI_IOCSTATUS_SCSI_IO_DATA_ERROR        = 0x0046,
+    MPI_IOCSTATUS_SCSI_PROTOCOL_ERROR       = 0x0047,
+    MPI_IOCSTATUS_SCSI_TASK_TERMINATED      = 0x0048,
+    MPI_IOCSTATUS_SCSI_RESIDUAL_MISMATCH    = 0x0049,
+    MPI_IOCSTATUS_SCSI_TASK_MGMT_FAILED     = 0x004A,
+    MPI_IOCSTATUS_SCSI_IOC_TERMINATED       = 0x004B,
+    MPI_IOCSTATUS_SCSI_EXT_TERMINATED       = 0x004C,
+
+    /************************************************************************/
+    /*  For use by SCSI Initiator and SCSI Target end-to-end data protection*/
+    /************************************************************************/
+
+    MPI_IOCSTATUS_EEDP_GUARD_ERROR          = 0x004D,
+    MPI_IOCSTATUS_EEDP_REF_TAG_ERROR        = 0x004E,
+    MPI_IOCSTATUS_EEDP_APP_TAG_ERROR        = 0x004F,
+
+    /************************************************************************/
+    /*  SCSI Target values                                                  */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_TARGET_PRIORITY_IO         = 0x0060,
+    MPI_IOCSTATUS_TARGET_INVALID_PORT        = 0x0061,
+    MPI_IOCSTATUS_TARGET_INVALID_IO_INDEX    = 0x0062,
+    MPI_IOCSTATUS_TARGET_ABORTED             = 0x0063,
+    MPI_IOCSTATUS_TARGET_NO_CONN_RETRYABLE   = 0x0064,
+    MPI_IOCSTATUS_TARGET_NO_CONNECTION       = 0x0065,
+    MPI_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH = 0x006A,
+    MPI_IOCSTATUS_TARGET_STS_DATA_NOT_SENT   = 0x006B,
+    MPI_IOCSTATUS_TARGET_DATA_OFFSET_ERROR   = 0x006D,
+    MPI_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA = 0x006E,
+    MPI_IOCSTATUS_TARGET_IU_TOO_SHORT        = 0x006F,
+    MPI_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT     = 0x0070,
+    MPI_IOCSTATUS_TARGET_NAK_RECEIVED        = 0x0071,
+
+    /************************************************************************/
+    /*  Fibre Channel Direct Access values                                  */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_FC_ABORTED                = 0x0066,
+    MPI_IOCSTATUS_FC_RX_ID_INVALID          = 0x0067,
+    MPI_IOCSTATUS_FC_DID_INVALID            = 0x0068,
+    MPI_IOCSTATUS_FC_NODE_LOGGED_OUT        = 0x0069,
+    MPI_IOCSTATUS_FC_EXCHANGE_CANCELED      = 0x006C,
+
+    /************************************************************************/
+    /*  LAN values                                                          */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_LAN_DEVICE_NOT_FOUND      = 0x0080,
+    MPI_IOCSTATUS_LAN_DEVICE_FAILURE        = 0x0081,
+    MPI_IOCSTATUS_LAN_TRANSMIT_ERROR        = 0x0082,
+    MPI_IOCSTATUS_LAN_TRANSMIT_ABORTED      = 0x0083,
+    MPI_IOCSTATUS_LAN_RECEIVE_ERROR         = 0x0084,
+    MPI_IOCSTATUS_LAN_RECEIVE_ABORTED       = 0x0085,
+    MPI_IOCSTATUS_LAN_PARTIAL_PACKET        = 0x0086,
+    MPI_IOCSTATUS_LAN_CANCELED              = 0x0087,
+
+    /************************************************************************/
+    /*  Serial Attached SCSI values                                         */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SAS_SMP_REQUEST_FAILED    = 0x0090,
+    MPI_IOCSTATUS_SAS_SMP_DATA_OVERRUN      = 0x0091,
+
+    /************************************************************************/
+    /*  Inband values                                                       */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_INBAND_ABORTED            = 0x0098,
+    MPI_IOCSTATUS_INBAND_NO_CONNECTION      = 0x0099,
+
+    /************************************************************************/
+    /*  Diagnostic Tools values                                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_DIAGNOSTIC_RELEASED       = 0x00A0,
+
+    /************************************************************************/
+    /*  IOCStatus flag to indicate that log info is available               */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE   = 0x8000,
+    MPI_IOCSTATUS_MASK                      = 0x7FFF,
+
+    /************************************************************************/
+    /*  LogInfo Types                                                       */
+    /************************************************************************/
+
+    MPI_IOCLOGINFO_TYPE_MASK                = 0xF0000000,
+    MPI_IOCLOGINFO_TYPE_SHIFT               = 28,
+    MPI_IOCLOGINFO_TYPE_NONE                = 0x0,
+    MPI_IOCLOGINFO_TYPE_SCSI                = 0x1,
+    MPI_IOCLOGINFO_TYPE_FC                  = 0x2,
+    MPI_IOCLOGINFO_TYPE_SAS                 = 0x3,
+    MPI_IOCLOGINFO_TYPE_ISCSI               = 0x4,
+    MPI_IOCLOGINFO_LOG_DATA_MASK            = 0x0FFFFFFF,
+};
+
+/****************************************************************************/
+/*  SCSI IO messages and associated structures                              */
+/****************************************************************************/
+
+typedef struct MPIMsgSCSIIORequest {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 ChainOffset;        /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 CDBLength;          /* 04h */
+    uint8_t                 SenseBufferLength;  /* 05h */
+    uint8_t                 Reserved;           /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 LUN[8];             /* 0Ch */
+    uint32_t                Control;            /* 14h */
+    uint8_t                 CDB[16];            /* 18h */
+    uint32_t                DataLength;         /* 28h */
+    uint32_t                SenseBufferLowAddr; /* 2Ch */
+} QEMU_PACKED MPIMsgSCSIIORequest;
+
+/* SCSI IO MsgFlags bits */
+
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH              (0x01)
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH_32           (0x00)
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH_64           (0x01)
+
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOCATION           (0x02)
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOC_HOST           (0x00)
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOC_IOC            (0x02)
+
+#define MPI_SCSIIO_MSGFLGS_CMD_DETERMINES_DATA_DIR  (0x04)
+
+/* SCSI IO LUN fields */
+
+#define MPI_SCSIIO_LUN_FIRST_LEVEL_ADDRESSING   (0x0000FFFF)
+#define MPI_SCSIIO_LUN_SECOND_LEVEL_ADDRESSING  (0xFFFF0000)
+#define MPI_SCSIIO_LUN_THIRD_LEVEL_ADDRESSING   (0x0000FFFF)
+#define MPI_SCSIIO_LUN_FOURTH_LEVEL_ADDRESSING  (0xFFFF0000)
+#define MPI_SCSIIO_LUN_LEVEL_1_WORD             (0xFF00)
+#define MPI_SCSIIO_LUN_LEVEL_1_DWORD            (0x0000FF00)
+
+/* SCSI IO Control bits */
+
+#define MPI_SCSIIO_CONTROL_DATADIRECTION_MASK   (0x03000000)
+#define MPI_SCSIIO_CONTROL_NODATATRANSFER       (0x00000000)
+#define MPI_SCSIIO_CONTROL_WRITE                (0x01000000)
+#define MPI_SCSIIO_CONTROL_READ                 (0x02000000)
+
+#define MPI_SCSIIO_CONTROL_ADDCDBLEN_MASK       (0x3C000000)
+#define MPI_SCSIIO_CONTROL_ADDCDBLEN_SHIFT      (26)
+
+#define MPI_SCSIIO_CONTROL_TASKATTRIBUTE_MASK   (0x00000700)
+#define MPI_SCSIIO_CONTROL_SIMPLEQ              (0x00000000)
+#define MPI_SCSIIO_CONTROL_HEADOFQ              (0x00000100)
+#define MPI_SCSIIO_CONTROL_ORDEREDQ             (0x00000200)
+#define MPI_SCSIIO_CONTROL_ACAQ                 (0x00000400)
+#define MPI_SCSIIO_CONTROL_UNTAGGED             (0x00000500)
+#define MPI_SCSIIO_CONTROL_NO_DISCONNECT        (0x00000700)
+
+#define MPI_SCSIIO_CONTROL_TASKMANAGE_MASK      (0x00FF0000)
+#define MPI_SCSIIO_CONTROL_OBSOLETE             (0x00800000)
+#define MPI_SCSIIO_CONTROL_CLEAR_ACA_RSV        (0x00400000)
+#define MPI_SCSIIO_CONTROL_TARGET_RESET         (0x00200000)
+#define MPI_SCSIIO_CONTROL_LUN_RESET_RSV        (0x00100000)
+#define MPI_SCSIIO_CONTROL_RESERVED             (0x00080000)
+#define MPI_SCSIIO_CONTROL_CLR_TASK_SET_RSV     (0x00040000)
+#define MPI_SCSIIO_CONTROL_ABORT_TASK_SET       (0x00020000)
+#define MPI_SCSIIO_CONTROL_RESERVED2            (0x00010000)
+
+/* SCSI IO reply structure */
+typedef struct MPIMsgSCSIIOReply
+{
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 MsgLength;          /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 CDBLength;          /* 04h */
+    uint8_t                 SenseBufferLength;  /* 05h */
+    uint8_t                 Reserved;           /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 SCSIStatus;         /* 0Ch */
+    uint8_t                 SCSIState;          /* 0Dh */
+    uint16_t                IOCStatus;          /* 0Eh */
+    uint32_t                IOCLogInfo;         /* 10h */
+    uint32_t                TransferCount;      /* 14h */
+    uint32_t                SenseCount;         /* 18h */
+    uint32_t                ResponseInfo;       /* 1Ch */
+    uint16_t                TaskTag;            /* 20h */
+    uint16_t                Reserved1;          /* 22h */
+} QEMU_PACKED MPIMsgSCSIIOReply;
+
+/* SCSI IO Reply SCSIStatus values (SAM-2 status codes) */
+
+#define MPI_SCSI_STATUS_SUCCESS                 (0x00)
+#define MPI_SCSI_STATUS_CHECK_CONDITION         (0x02)
+#define MPI_SCSI_STATUS_CONDITION_MET           (0x04)
+#define MPI_SCSI_STATUS_BUSY                    (0x08)
+#define MPI_SCSI_STATUS_INTERMEDIATE            (0x10)
+#define MPI_SCSI_STATUS_INTERMEDIATE_CONDMET    (0x14)
+#define MPI_SCSI_STATUS_RESERVATION_CONFLICT    (0x18)
+#define MPI_SCSI_STATUS_COMMAND_TERMINATED      (0x22)
+#define MPI_SCSI_STATUS_TASK_SET_FULL           (0x28)
+#define MPI_SCSI_STATUS_ACA_ACTIVE              (0x30)
+
+#define MPI_SCSI_STATUS_FCPEXT_DEVICE_LOGGED_OUT    (0x80)
+#define MPI_SCSI_STATUS_FCPEXT_NO_LINK              (0x81)
+#define MPI_SCSI_STATUS_FCPEXT_UNASSIGNED           (0x82)
+
+
+/* SCSI IO Reply SCSIState values */
+
+#define MPI_SCSI_STATE_AUTOSENSE_VALID          (0x01)
+#define MPI_SCSI_STATE_AUTOSENSE_FAILED         (0x02)
+#define MPI_SCSI_STATE_NO_SCSI_STATUS           (0x04)
+#define MPI_SCSI_STATE_TERMINATED               (0x08)
+#define MPI_SCSI_STATE_RESPONSE_INFO_VALID      (0x10)
+#define MPI_SCSI_STATE_QUEUE_TAG_REJECTED       (0x20)
+
+/* SCSI IO Reply ResponseInfo values */
+/* (FCP-1 RSP_CODE values and SPI-3 Packetized Failure codes) */
+
+#define MPI_SCSI_RSP_INFO_FUNCTION_COMPLETE     (0x00000000)
+#define MPI_SCSI_RSP_INFO_FCP_BURST_LEN_ERROR   (0x01000000)
+#define MPI_SCSI_RSP_INFO_CMND_FIELDS_INVALID   (0x02000000)
+#define MPI_SCSI_RSP_INFO_FCP_DATA_RO_ERROR     (0x03000000)
+#define MPI_SCSI_RSP_INFO_TASK_MGMT_UNSUPPORTED (0x04000000)
+#define MPI_SCSI_RSP_INFO_TASK_MGMT_FAILED      (0x05000000)
+#define MPI_SCSI_RSP_INFO_SPI_LQ_INVALID_TYPE   (0x06000000)
+
+#define MPI_SCSI_TASKTAG_UNKNOWN                (0xFFFF)
+
+
+/****************************************************************************/
+/*  SCSI Task Management messages                                           */
+/****************************************************************************/
+
+typedef struct MPIMsgSCSITaskMgmt {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 ChainOffset;        /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 Reserved;           /* 04h */
+    uint8_t                 TaskType;           /* 05h */
+    uint8_t                 Reserved1;          /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 LUN[8];             /* 0Ch */
+    uint32_t                Reserved2[7];       /* 14h */
+    uint32_t                TaskMsgContext;     /* 30h */
+} QEMU_PACKED MPIMsgSCSITaskMgmt;
+
+enum {
+    /* TaskType values */
+
+    MPI_SCSITASKMGMT_TASKTYPE_ABORT_TASK            = 0x01,
+    MPI_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET         = 0x02,
+    MPI_SCSITASKMGMT_TASKTYPE_TARGET_RESET          = 0x03,
+    MPI_SCSITASKMGMT_TASKTYPE_RESET_BUS             = 0x04,
+    MPI_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET    = 0x05,
+    MPI_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET        = 0x06,
+    MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK            = 0x07,
+    MPI_SCSITASKMGMT_TASKTYPE_CLR_ACA               = 0x08,
+
+    /* MsgFlags bits */
+
+    MPI_SCSITASKMGMT_MSGFLAGS_DO_NOT_SEND_TASK_IU   = 0x01,
+
+    MPI_SCSITASKMGMT_MSGFLAGS_TARGET_RESET_OPTION   = 0x00,
+    MPI_SCSITASKMGMT_MSGFLAGS_LIP_RESET_OPTION      = 0x02,
+    MPI_SCSITASKMGMT_MSGFLAGS_LIPRESET_RESET_OPTION = 0x04,
+
+    MPI_SCSITASKMGMT_MSGFLAGS_SOFT_RESET_OPTION     = 0x08,
+};
+
+/* SCSI Task Management Reply */
+typedef struct MPIMsgSCSITaskMgmtReply {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 MsgLength;          /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 ResponseCode;       /* 04h */
+    uint8_t                 TaskType;           /* 05h */
+    uint8_t                 Reserved1;          /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 Reserved2[2];       /* 0Ch */
+    uint16_t                IOCStatus;          /* 0Eh */
+    uint32_t                IOCLogInfo;         /* 10h */
+    uint32_t                TerminationCount;   /* 14h */
+} QEMU_PACKED MPIMsgSCSITaskMgmtReply;
+
+/* ResponseCode values */
+enum {
+    MPI_SCSITASKMGMT_RSP_TM_COMPLETE                = 0x00,
+    MPI_SCSITASKMGMT_RSP_INVALID_FRAME              = 0x02,
+    MPI_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED           = 0x04,
+    MPI_SCSITASKMGMT_RSP_TM_FAILED                  = 0x05,
+    MPI_SCSITASKMGMT_RSP_TM_SUCCEEDED               = 0x08,
+    MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN             = 0x09,
+    MPI_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC           = 0x80,
+};
+
+/****************************************************************************/
+/*  IOCInit message                                                         */
+/****************************************************************************/
+
+typedef struct MPIMsgIOCInit {
+    uint8_t                 WhoInit;                    /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Flags;                      /* 04h */
+    uint8_t                 MaxDevices;                 /* 05h */
+    uint8_t                 MaxBuses;                   /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                ReplyFrameSize;             /* 0Ch */
+    uint8_t                 Reserved1[2];               /* 0Eh */
+    uint32_t                HostMfaHighAddr;            /* 10h */
+    uint32_t                SenseBufferHighAddr;        /* 14h */
+    uint32_t                ReplyFifoHostSignalingAddr; /* 18h */
+    MPISGEntry              HostPageBufferSGE;          /* 1Ch */
+    uint16_t                MsgVersion;                 /* 28h */
+    uint16_t                HeaderVersion;              /* 2Ah */
+} QEMU_PACKED MPIMsgIOCInit;
+
+enum {
+    /* WhoInit values */
+
+    MPI_WHOINIT_NO_ONE                              = 0x00,
+    MPI_WHOINIT_SYSTEM_BIOS                         = 0x01,
+    MPI_WHOINIT_ROM_BIOS                            = 0x02,
+    MPI_WHOINIT_PCI_PEER                            = 0x03,
+    MPI_WHOINIT_HOST_DRIVER                         = 0x04,
+    MPI_WHOINIT_MANUFACTURER                        = 0x05,
+
+    /* Flags values */
+
+    MPI_IOCINIT_FLAGS_HOST_PAGE_BUFFER_PERSISTENT   = 0x04,
+    MPI_IOCINIT_FLAGS_REPLY_FIFO_HOST_SIGNAL        = 0x02,
+    MPI_IOCINIT_FLAGS_DISCARD_FW_IMAGE              = 0x01,
+
+    /* MsgVersion */
+
+    MPI_IOCINIT_MSGVERSION_MAJOR_MASK               = 0xFF00,
+    MPI_IOCINIT_MSGVERSION_MAJOR_SHIFT              = 8,
+    MPI_IOCINIT_MSGVERSION_MINOR_MASK               = 0x00FF,
+    MPI_IOCINIT_MSGVERSION_MINOR_SHIFT              = 0,
+
+    /* HeaderVersion */
+
+    MPI_IOCINIT_HEADERVERSION_UNIT_MASK             = 0xFF00,
+    MPI_IOCINIT_HEADERVERSION_UNIT_SHIFT            = 8,
+    MPI_IOCINIT_HEADERVERSION_DEV_MASK              = 0x00FF,
+    MPI_IOCINIT_HEADERVERSION_DEV_SHIFT             = 0,
+};
+
+typedef struct MPIMsgIOCInitReply {
+    uint8_t                 WhoInit;                    /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Flags;                      /* 04h */
+    uint8_t                 MaxDevices;                 /* 05h */
+    uint8_t                 MaxBuses;                   /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgIOCInitReply;
+
+
+
+/****************************************************************************/
+/*  IOC Facts message                                                       */
+/****************************************************************************/
+
+typedef struct MPIMsgIOCFacts {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 01h */
+    uint8_t                 Function;                   /* 02h */
+    uint8_t                 Reserved1[3];               /* 03h */
+    uint8_t                 MsgFlags;                   /* 04h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgIOCFacts;
+
+/* IOC Facts Reply */
+typedef struct MPIMsgIOCFactsReply {
+    uint16_t                MsgVersion;                 /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                HeaderVersion;              /* 04h */
+    uint8_t                 IOCNumber;                  /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                IOCExceptions;              /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+    uint8_t                 MaxChainDepth;              /* 14h */
+    uint8_t                 WhoInit;                    /* 15h */
+    uint8_t                 BlockSize;                  /* 16h */
+    uint8_t                 Flags;                      /* 17h */
+    uint16_t                ReplyQueueDepth;            /* 18h */
+    uint16_t                RequestFrameSize;           /* 1Ah */
+    uint16_t                Reserved_0101_FWVersion;    /* 1Ch */ /* obsolete 16-bit FWVersion */
+    uint16_t                ProductID;                  /* 1Eh */
+    uint32_t                CurrentHostMfaHighAddr;     /* 20h */
+    uint16_t                GlobalCredits;              /* 24h */
+    uint8_t                 NumberOfPorts;              /* 26h */
+    uint8_t                 EventState;                 /* 27h */
+    uint32_t                CurrentSenseBufferHighAddr; /* 28h */
+    uint16_t                CurReplyFrameSize;          /* 2Ch */
+    uint8_t                 MaxDevices;                 /* 2Eh */
+    uint8_t                 MaxBuses;                   /* 2Fh */
+    uint32_t                FWImageSize;                /* 30h */
+    uint32_t                IOCCapabilities;            /* 34h */
+    uint8_t                 FWVersionDev;               /* 38h */
+    uint8_t                 FWVersionUnit;              /* 39h */
+    uint8_t                 FWVersionMinor;             /* 3ah */
+    uint8_t                 FWVersionMajor;             /* 3bh */
+    uint16_t                HighPriorityQueueDepth;     /* 3Ch */
+    uint16_t                Reserved2;                  /* 3Eh */
+    MPISGEntry              HostPageBufferSGE;          /* 40h */
+    uint32_t                ReplyFifoHostSignalingAddr; /* 4Ch */
+} QEMU_PACKED MPIMsgIOCFactsReply;
+
+enum {
+    MPI_IOCFACTS_MSGVERSION_MAJOR_MASK              = 0xFF00,
+    MPI_IOCFACTS_MSGVERSION_MAJOR_SHIFT             = 8,
+    MPI_IOCFACTS_MSGVERSION_MINOR_MASK              = 0x00FF,
+    MPI_IOCFACTS_MSGVERSION_MINOR_SHIFT             = 0,
+
+    MPI_IOCFACTS_HDRVERSION_UNIT_MASK               = 0xFF00,
+    MPI_IOCFACTS_HDRVERSION_UNIT_SHIFT              = 8,
+    MPI_IOCFACTS_HDRVERSION_DEV_MASK                = 0x00FF,
+    MPI_IOCFACTS_HDRVERSION_DEV_SHIFT               = 0,
+
+    MPI_IOCFACTS_EXCEPT_CONFIG_CHECKSUM_FAIL        = 0x0001,
+    MPI_IOCFACTS_EXCEPT_RAID_CONFIG_INVALID         = 0x0002,
+    MPI_IOCFACTS_EXCEPT_FW_CHECKSUM_FAIL            = 0x0004,
+    MPI_IOCFACTS_EXCEPT_PERSISTENT_TABLE_FULL       = 0x0008,
+    MPI_IOCFACTS_EXCEPT_METADATA_UNSUPPORTED        = 0x0010,
+
+    MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT             = 0x01,
+    MPI_IOCFACTS_FLAGS_REPLY_FIFO_HOST_SIGNAL       = 0x02,
+    MPI_IOCFACTS_FLAGS_HOST_PAGE_BUFFER_PERSISTENT  = 0x04,
+
+    MPI_IOCFACTS_EVENTSTATE_DISABLED                = 0x00,
+    MPI_IOCFACTS_EVENTSTATE_ENABLED                 = 0x01,
+
+    MPI_IOCFACTS_CAPABILITY_HIGH_PRI_Q              = 0x00000001,
+    MPI_IOCFACTS_CAPABILITY_REPLY_HOST_SIGNAL       = 0x00000002,
+    MPI_IOCFACTS_CAPABILITY_QUEUE_FULL_HANDLING     = 0x00000004,
+    MPI_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER       = 0x00000008,
+    MPI_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER         = 0x00000010,
+    MPI_IOCFACTS_CAPABILITY_EXTENDED_BUFFER         = 0x00000020,
+    MPI_IOCFACTS_CAPABILITY_EEDP                    = 0x00000040,
+    MPI_IOCFACTS_CAPABILITY_BIDIRECTIONAL           = 0x00000080,
+    MPI_IOCFACTS_CAPABILITY_MULTICAST               = 0x00000100,
+    MPI_IOCFACTS_CAPABILITY_SCSIIO32                = 0x00000200,
+    MPI_IOCFACTS_CAPABILITY_NO_SCSIIO16             = 0x00000400,
+    MPI_IOCFACTS_CAPABILITY_TLR                     = 0x00000800,
+};
+
+/****************************************************************************/
+/*  Port Facts message and Reply                                            */
+/****************************************************************************/
+
+typedef struct MPIMsgPortFacts {
+     uint8_t                Reserved[2];                /* 00h */
+     uint8_t                ChainOffset;                /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint8_t                Reserved1[2];               /* 04h */
+     uint8_t                PortNumber;                 /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgPortFacts;
+
+typedef struct MPIMsgPortFactsReply {
+     uint16_t               Reserved;                   /* 00h */
+     uint8_t                MsgLength;                  /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint16_t               Reserved1;                  /* 04h */
+     uint8_t                PortNumber;                 /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+     uint16_t               Reserved2;                  /* 0Ch */
+     uint16_t               IOCStatus;                  /* 0Eh */
+     uint32_t               IOCLogInfo;                 /* 10h */
+     uint8_t                Reserved3;                  /* 14h */
+     uint8_t                PortType;                   /* 15h */
+     uint16_t               MaxDevices;                 /* 16h */
+     uint16_t               PortSCSIID;                 /* 18h */
+     uint16_t               ProtocolFlags;              /* 1Ah */
+     uint16_t               MaxPostedCmdBuffers;        /* 1Ch */
+     uint16_t               MaxPersistentIDs;           /* 1Eh */
+     uint16_t               MaxLanBuckets;              /* 20h */
+     uint8_t                MaxInitiators;              /* 22h */
+     uint8_t                Reserved4;                  /* 23h */
+     uint32_t               Reserved5;                  /* 24h */
+} QEMU_PACKED MPIMsgPortFactsReply;
+
+
+enum {
+    /* PortTypes values */
+    MPI_PORTFACTS_PORTTYPE_INACTIVE         = 0x00,
+    MPI_PORTFACTS_PORTTYPE_SCSI             = 0x01,
+    MPI_PORTFACTS_PORTTYPE_FC               = 0x10,
+    MPI_PORTFACTS_PORTTYPE_ISCSI            = 0x20,
+    MPI_PORTFACTS_PORTTYPE_SAS              = 0x30,
+
+    /* ProtocolFlags values */
+    MPI_PORTFACTS_PROTOCOL_LOGBUSADDR       = 0x01,
+    MPI_PORTFACTS_PROTOCOL_LAN              = 0x02,
+    MPI_PORTFACTS_PROTOCOL_TARGET           = 0x04,
+    MPI_PORTFACTS_PROTOCOL_INITIATOR        = 0x08,
+};
+
+
+/****************************************************************************/
+/*  Port Enable Message                                                     */
+/****************************************************************************/
+
+typedef struct MPIMsgPortEnable {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[2];               /* 04h */
+    uint8_t                 PortNumber;                 /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgPortEnable;
+
+typedef struct MPIMsgPortEnableReply {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[2];               /* 04h */
+    uint8_t                 PortNumber;                 /* 05h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgPortEnableReply;
+
+/****************************************************************************/
+/*  Event Notification messages                                             */
+/****************************************************************************/
+
+typedef struct MPIMsgEventNotify {
+    uint8_t                 Switch;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgEventNotify;
+
+/* Event Notification Reply */
+
+typedef struct MPIMsgEventNotifyReply {
+     uint16_t               EventDataLength;            /* 00h */
+     uint8_t                MsgLength;                  /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint8_t                Reserved1[2];               /* 04h */
+     uint8_t                AckRequired;                /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+     uint8_t                Reserved2[2];               /* 0Ch */
+     uint16_t               IOCStatus;                  /* 0Eh */
+     uint32_t               IOCLogInfo;                 /* 10h */
+     uint32_t               Event;                      /* 14h */
+     uint32_t               EventContext;               /* 18h */
+     uint32_t               Data[1];                    /* 1Ch */
+} QEMU_PACKED MPIMsgEventNotifyReply;
+
+/* Event Acknowledge */
+
+typedef struct MPIMsgEventAck {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint32_t                Event;                      /* 0Ch */
+    uint32_t                EventContext;               /* 10h */
+} QEMU_PACKED MPIMsgEventAck;
+
+typedef struct MPIMsgEventAckReply {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgEventAckReply;
+
+enum {
+    /* Switch */
+
+    MPI_EVENT_NOTIFICATION_SWITCH_OFF   = 0x00,
+    MPI_EVENT_NOTIFICATION_SWITCH_ON    = 0x01,
+
+    /* Event */
+
+    MPI_EVENT_NONE                          = 0x00000000,
+    MPI_EVENT_LOG_DATA                      = 0x00000001,
+    MPI_EVENT_STATE_CHANGE                  = 0x00000002,
+    MPI_EVENT_UNIT_ATTENTION                = 0x00000003,
+    MPI_EVENT_IOC_BUS_RESET                 = 0x00000004,
+    MPI_EVENT_EXT_BUS_RESET                 = 0x00000005,
+    MPI_EVENT_RESCAN                        = 0x00000006,
+    MPI_EVENT_LINK_STATUS_CHANGE            = 0x00000007,
+    MPI_EVENT_LOOP_STATE_CHANGE             = 0x00000008,
+    MPI_EVENT_LOGOUT                        = 0x00000009,
+    MPI_EVENT_EVENT_CHANGE                  = 0x0000000A,
+    MPI_EVENT_INTEGRATED_RAID               = 0x0000000B,
+    MPI_EVENT_SCSI_DEVICE_STATUS_CHANGE     = 0x0000000C,
+    MPI_EVENT_ON_BUS_TIMER_EXPIRED          = 0x0000000D,
+    MPI_EVENT_QUEUE_FULL                    = 0x0000000E,
+    MPI_EVENT_SAS_DEVICE_STATUS_CHANGE      = 0x0000000F,
+    MPI_EVENT_SAS_SES                       = 0x00000010,
+    MPI_EVENT_PERSISTENT_TABLE_FULL         = 0x00000011,
+    MPI_EVENT_SAS_PHY_LINK_STATUS           = 0x00000012,
+    MPI_EVENT_SAS_DISCOVERY_ERROR           = 0x00000013,
+    MPI_EVENT_IR_RESYNC_UPDATE              = 0x00000014,
+    MPI_EVENT_IR2                           = 0x00000015,
+    MPI_EVENT_SAS_DISCOVERY                 = 0x00000016,
+    MPI_EVENT_SAS_BROADCAST_PRIMITIVE       = 0x00000017,
+    MPI_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE = 0x00000018,
+    MPI_EVENT_SAS_INIT_TABLE_OVERFLOW       = 0x00000019,
+    MPI_EVENT_SAS_SMP_ERROR                 = 0x0000001A,
+    MPI_EVENT_SAS_EXPANDER_STATUS_CHANGE    = 0x0000001B,
+    MPI_EVENT_LOG_ENTRY_ADDED               = 0x00000021,
+
+    /* AckRequired field values */
+
+    MPI_EVENT_NOTIFICATION_ACK_NOT_REQUIRED = 0x00,
+    MPI_EVENT_NOTIFICATION_ACK_REQUIRED     = 0x01,
+};
+
+/****************************************************************************
+*   Config Request Message
+****************************************************************************/
+
+typedef struct MPIMsgConfig {
+    uint8_t                 Action;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                ExtPageLength;              /* 04h */
+    uint8_t                 ExtPageType;                /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint8_t                 Reserved2[8];               /* 0Ch */
+    uint8_t                 PageVersion;                /* 14h */
+    uint8_t                 PageLength;                 /* 15h */
+    uint8_t                 PageNumber;                 /* 16h */
+    uint8_t                 PageType;                   /* 17h */
+    uint32_t                PageAddress;                /* 18h */
+    MPISGEntry              PageBufferSGE;              /* 1Ch */
+} QEMU_PACKED MPIMsgConfig;
+
+/* Action field values */
+
+enum {
+    MPI_CONFIG_ACTION_PAGE_HEADER               = 0x00,
+    MPI_CONFIG_ACTION_PAGE_READ_CURRENT         = 0x01,
+    MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT        = 0x02,
+    MPI_CONFIG_ACTION_PAGE_DEFAULT              = 0x03,
+    MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM          = 0x04,
+    MPI_CONFIG_ACTION_PAGE_READ_DEFAULT         = 0x05,
+    MPI_CONFIG_ACTION_PAGE_READ_NVRAM           = 0x06,
+};
+
+
+/* Config Reply Message */
+typedef struct MPIMsgConfigReply {
+    uint8_t                 Action;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                ExtPageLength;              /* 04h */
+    uint8_t                 ExtPageType;                /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint8_t                 Reserved2[2];               /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+    uint8_t                 PageVersion;                /* 14h */
+    uint8_t                 PageLength;                 /* 15h */
+    uint8_t                 PageNumber;                 /* 16h */
+    uint8_t                 PageType;                   /* 17h */
+} QEMU_PACKED MPIMsgConfigReply;
+
+enum {
+    /* PageAddress field values */
+    MPI_CONFIG_PAGEATTR_READ_ONLY               = 0x00,
+    MPI_CONFIG_PAGEATTR_CHANGEABLE              = 0x10,
+    MPI_CONFIG_PAGEATTR_PERSISTENT              = 0x20,
+    MPI_CONFIG_PAGEATTR_RO_PERSISTENT           = 0x30,
+    MPI_CONFIG_PAGEATTR_MASK                    = 0xF0,
+
+    MPI_CONFIG_PAGETYPE_IO_UNIT                 = 0x00,
+    MPI_CONFIG_PAGETYPE_IOC                     = 0x01,
+    MPI_CONFIG_PAGETYPE_BIOS                    = 0x02,
+    MPI_CONFIG_PAGETYPE_SCSI_PORT               = 0x03,
+    MPI_CONFIG_PAGETYPE_SCSI_DEVICE             = 0x04,
+    MPI_CONFIG_PAGETYPE_FC_PORT                 = 0x05,
+    MPI_CONFIG_PAGETYPE_FC_DEVICE               = 0x06,
+    MPI_CONFIG_PAGETYPE_LAN                     = 0x07,
+    MPI_CONFIG_PAGETYPE_RAID_VOLUME             = 0x08,
+    MPI_CONFIG_PAGETYPE_MANUFACTURING           = 0x09,
+    MPI_CONFIG_PAGETYPE_RAID_PHYSDISK           = 0x0A,
+    MPI_CONFIG_PAGETYPE_INBAND                  = 0x0B,
+    MPI_CONFIG_PAGETYPE_EXTENDED                = 0x0F,
+    MPI_CONFIG_PAGETYPE_MASK                    = 0x0F,
+
+    MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT          = 0x10,
+    MPI_CONFIG_EXTPAGETYPE_SAS_EXPANDER         = 0x11,
+    MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE           = 0x12,
+    MPI_CONFIG_EXTPAGETYPE_SAS_PHY              = 0x13,
+    MPI_CONFIG_EXTPAGETYPE_LOG                  = 0x14,
+    MPI_CONFIG_EXTPAGETYPE_ENCLOSURE            = 0x15,
+
+    MPI_SCSI_PORT_PGAD_PORT_MASK                = 0x000000FF,
+
+    MPI_SCSI_DEVICE_FORM_MASK                   = 0xF0000000,
+    MPI_SCSI_DEVICE_FORM_BUS_TID                = 0x00000000,
+    MPI_SCSI_DEVICE_TARGET_ID_MASK              = 0x000000FF,
+    MPI_SCSI_DEVICE_TARGET_ID_SHIFT             = 0,
+    MPI_SCSI_DEVICE_BUS_MASK                    = 0x0000FF00,
+    MPI_SCSI_DEVICE_BUS_SHIFT                   = 8,
+    MPI_SCSI_DEVICE_FORM_TARGET_MODE            = 0x10000000,
+    MPI_SCSI_DEVICE_TM_RESPOND_ID_MASK          = 0x000000FF,
+    MPI_SCSI_DEVICE_TM_RESPOND_ID_SHIFT         = 0,
+    MPI_SCSI_DEVICE_TM_BUS_MASK                 = 0x0000FF00,
+    MPI_SCSI_DEVICE_TM_BUS_SHIFT                = 8,
+    MPI_SCSI_DEVICE_TM_INIT_ID_MASK             = 0x00FF0000,
+    MPI_SCSI_DEVICE_TM_INIT_ID_SHIFT            = 16,
+
+    MPI_FC_PORT_PGAD_PORT_MASK                  = 0xF0000000,
+    MPI_FC_PORT_PGAD_PORT_SHIFT                 = 28,
+    MPI_FC_PORT_PGAD_FORM_MASK                  = 0x0F000000,
+    MPI_FC_PORT_PGAD_FORM_INDEX                 = 0x01000000,
+    MPI_FC_PORT_PGAD_INDEX_MASK                 = 0x0000FFFF,
+    MPI_FC_PORT_PGAD_INDEX_SHIFT                = 0,
+
+    MPI_FC_DEVICE_PGAD_PORT_MASK                = 0xF0000000,
+    MPI_FC_DEVICE_PGAD_PORT_SHIFT               = 28,
+    MPI_FC_DEVICE_PGAD_FORM_MASK                = 0x0F000000,
+    MPI_FC_DEVICE_PGAD_FORM_NEXT_DID            = 0x00000000,
+    MPI_FC_DEVICE_PGAD_ND_PORT_MASK             = 0xF0000000,
+    MPI_FC_DEVICE_PGAD_ND_PORT_SHIFT            = 28,
+    MPI_FC_DEVICE_PGAD_ND_DID_MASK              = 0x00FFFFFF,
+    MPI_FC_DEVICE_PGAD_ND_DID_SHIFT             = 0,
+    MPI_FC_DEVICE_PGAD_FORM_BUS_TID             = 0x01000000,
+    MPI_FC_DEVICE_PGAD_BT_BUS_MASK              = 0x0000FF00,
+    MPI_FC_DEVICE_PGAD_BT_BUS_SHIFT             = 8,
+    MPI_FC_DEVICE_PGAD_BT_TID_MASK              = 0x000000FF,
+    MPI_FC_DEVICE_PGAD_BT_TID_SHIFT             = 0,
+
+    MPI_PHYSDISK_PGAD_PHYSDISKNUM_MASK          = 0x000000FF,
+    MPI_PHYSDISK_PGAD_PHYSDISKNUM_SHIFT         = 0,
+
+    MPI_SAS_EXPAND_PGAD_FORM_MASK             = 0xF0000000,
+    MPI_SAS_EXPAND_PGAD_FORM_SHIFT            = 28,
+    MPI_SAS_EXPAND_PGAD_FORM_GET_NEXT_HANDLE  = 0x00000000,
+    MPI_SAS_EXPAND_PGAD_FORM_HANDLE_PHY_NUM   = 0x00000001,
+    MPI_SAS_EXPAND_PGAD_FORM_HANDLE           = 0x00000002,
+    MPI_SAS_EXPAND_PGAD_GNH_MASK_HANDLE       = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_GNH_SHIFT_HANDLE      = 0,
+    MPI_SAS_EXPAND_PGAD_HPN_MASK_PHY          = 0x00FF0000,
+    MPI_SAS_EXPAND_PGAD_HPN_SHIFT_PHY         = 16,
+    MPI_SAS_EXPAND_PGAD_HPN_MASK_HANDLE       = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_HPN_SHIFT_HANDLE      = 0,
+    MPI_SAS_EXPAND_PGAD_H_MASK_HANDLE         = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_H_SHIFT_HANDLE        = 0,
+
+    MPI_SAS_DEVICE_PGAD_FORM_MASK               = 0xF0000000,
+    MPI_SAS_DEVICE_PGAD_FORM_SHIFT              = 28,
+    MPI_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE    = 0x00000000,
+    MPI_SAS_DEVICE_PGAD_FORM_BUS_TARGET_ID      = 0x00000001,
+    MPI_SAS_DEVICE_PGAD_FORM_HANDLE             = 0x00000002,
+    MPI_SAS_DEVICE_PGAD_GNH_HANDLE_MASK         = 0x0000FFFF,
+    MPI_SAS_DEVICE_PGAD_GNH_HANDLE_SHIFT        = 0,
+    MPI_SAS_DEVICE_PGAD_BT_BUS_MASK             = 0x0000FF00,
+    MPI_SAS_DEVICE_PGAD_BT_BUS_SHIFT            = 8,
+    MPI_SAS_DEVICE_PGAD_BT_TID_MASK             = 0x000000FF,
+    MPI_SAS_DEVICE_PGAD_BT_TID_SHIFT            = 0,
+    MPI_SAS_DEVICE_PGAD_H_HANDLE_MASK           = 0x0000FFFF,
+    MPI_SAS_DEVICE_PGAD_H_HANDLE_SHIFT          = 0,
+
+    MPI_SAS_PHY_PGAD_FORM_MASK                  = 0xF0000000,
+    MPI_SAS_PHY_PGAD_FORM_SHIFT                 = 28,
+    MPI_SAS_PHY_PGAD_FORM_PHY_NUMBER            = 0x0,
+    MPI_SAS_PHY_PGAD_FORM_PHY_TBL_INDEX         = 0x1,
+    MPI_SAS_PHY_PGAD_PHY_NUMBER_MASK            = 0x000000FF,
+    MPI_SAS_PHY_PGAD_PHY_NUMBER_SHIFT           = 0,
+    MPI_SAS_PHY_PGAD_PHY_TBL_INDEX_MASK         = 0x0000FFFF,
+    MPI_SAS_PHY_PGAD_PHY_TBL_INDEX_SHIFT        = 0,
+
+    MPI_SAS_ENCLOS_PGAD_FORM_MASK               = 0xF0000000,
+    MPI_SAS_ENCLOS_PGAD_FORM_SHIFT              = 28,
+    MPI_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE    = 0x00000000,
+    MPI_SAS_ENCLOS_PGAD_FORM_HANDLE             = 0x00000001,
+    MPI_SAS_ENCLOS_PGAD_GNH_HANDLE_MASK         = 0x0000FFFF,
+    MPI_SAS_ENCLOS_PGAD_GNH_HANDLE_SHIFT        = 0,
+    MPI_SAS_ENCLOS_PGAD_H_HANDLE_MASK           = 0x0000FFFF,
+    MPI_SAS_ENCLOS_PGAD_H_HANDLE_SHIFT          = 0,
+};
+
+/* Too many structs and definitions... see mptconfig.c for the few
+ * that are used.
+ */
+
+/****************************************************************************/
+/*  Firmware Upload message and associated structures                       */
+/****************************************************************************/
+
+enum {
+    /* defines for using the ProductId field */
+    MPI_FW_HEADER_PID_TYPE_MASK                     = 0xF000,
+    MPI_FW_HEADER_PID_TYPE_SCSI                     = 0x0000,
+    MPI_FW_HEADER_PID_TYPE_FC                       = 0x1000,
+    MPI_FW_HEADER_PID_TYPE_SAS                      = 0x2000,
+
+    MPI_FW_HEADER_PID_PROD_MASK                     = 0x0F00,
+    MPI_FW_HEADER_PID_PROD_INITIATOR_SCSI           = 0x0100,
+    MPI_FW_HEADER_PID_PROD_TARGET_INITIATOR_SCSI    = 0x0200,
+    MPI_FW_HEADER_PID_PROD_TARGET_SCSI              = 0x0300,
+    MPI_FW_HEADER_PID_PROD_IM_SCSI                  = 0x0400,
+    MPI_FW_HEADER_PID_PROD_IS_SCSI                  = 0x0500,
+    MPI_FW_HEADER_PID_PROD_CTX_SCSI                 = 0x0600,
+    MPI_FW_HEADER_PID_PROD_IR_SCSI                  = 0x0700,
+
+    MPI_FW_HEADER_PID_FAMILY_MASK                   = 0x00FF,
+
+    /* SCSI */
+    MPI_FW_HEADER_PID_FAMILY_1030A0_SCSI            = 0x0001,
+    MPI_FW_HEADER_PID_FAMILY_1030B0_SCSI            = 0x0002,
+    MPI_FW_HEADER_PID_FAMILY_1030B1_SCSI            = 0x0003,
+    MPI_FW_HEADER_PID_FAMILY_1030C0_SCSI            = 0x0004,
+    MPI_FW_HEADER_PID_FAMILY_1020A0_SCSI            = 0x0005,
+    MPI_FW_HEADER_PID_FAMILY_1020B0_SCSI            = 0x0006,
+    MPI_FW_HEADER_PID_FAMILY_1020B1_SCSI            = 0x0007,
+    MPI_FW_HEADER_PID_FAMILY_1020C0_SCSI            = 0x0008,
+    MPI_FW_HEADER_PID_FAMILY_1035A0_SCSI            = 0x0009,
+    MPI_FW_HEADER_PID_FAMILY_1035B0_SCSI            = 0x000A,
+    MPI_FW_HEADER_PID_FAMILY_1030TA0_SCSI           = 0x000B,
+    MPI_FW_HEADER_PID_FAMILY_1020TA0_SCSI           = 0x000C,
+
+    /* Fibre Channel */
+    MPI_FW_HEADER_PID_FAMILY_909_FC                 = 0x0000,
+    MPI_FW_HEADER_PID_FAMILY_919_FC                 = 0x0001, /* 919 and 929     */
+    MPI_FW_HEADER_PID_FAMILY_919X_FC                = 0x0002, /* 919X and 929X   */
+    MPI_FW_HEADER_PID_FAMILY_919XL_FC               = 0x0003, /* 919XL and 929XL */
+    MPI_FW_HEADER_PID_FAMILY_939X_FC                = 0x0004, /* 939X and 949X   */
+    MPI_FW_HEADER_PID_FAMILY_959_FC                 = 0x0005,
+    MPI_FW_HEADER_PID_FAMILY_949E_FC                = 0x0006,
+
+    /* SAS */
+    MPI_FW_HEADER_PID_FAMILY_1064_SAS               = 0x0001,
+    MPI_FW_HEADER_PID_FAMILY_1068_SAS               = 0x0002,
+    MPI_FW_HEADER_PID_FAMILY_1078_SAS               = 0x0003,
+    MPI_FW_HEADER_PID_FAMILY_106xE_SAS              = 0x0004, /* 1068E, 1066E, and 1064E */
+};
+
+#endif
diff --git a/qemu/hw/scsi/mptconfig.c b/qemu/hw/scsi/mptconfig.c
new file mode 100644
index 000000000..707185469
--- /dev/null
+++ b/qemu/hw/scsi/mptconfig.c
@@ -0,0 +1,905 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation - configuration pages
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ */
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/scsi/scsi.h"
+
+#include "mptsas.h"
+#include "mpi.h"
+#include "trace.h"
+
+/* Generic functions for marshaling and unmarshaling.  */
+
+#define repl1(x) x
+#define repl2(x) x x
+#define repl3(x) x x x
+#define repl4(x) x x x x
+#define repl5(x) x x x x x
+#define repl6(x) x x x x x x
+#define repl7(x) x x x x x x x
+#define repl8(x) x x x x x x x x
+
+#define repl(n, x) glue(repl, n)(x)
+
+typedef union PackValue {
+    uint64_t ll;
+    char *str;
+} PackValue;
+
+static size_t vfill(uint8_t *data, size_t size, const char *fmt, va_list ap)
+{
+    size_t ofs;
+    PackValue val;
+    const char *p;
+
+    ofs = 0;
+    p = fmt;
+    while (*p) {
+        memset(&val, 0, sizeof(val));
+        switch (*p) {
+        case '*':
+            p++;
+            break;
+        case 'b':
+        case 'w':
+        case 'l':
+            val.ll = va_arg(ap, int);
+            break;
+        case 'q':
+            val.ll = va_arg(ap, int64_t);
+            break;
+        case 's':
+            val.str = va_arg(ap, void *);
+            break;
+        }
+        switch (*p++) {
+        case 'b':
+            if (data) {
+                stb_p(data + ofs, val.ll);
+            }
+            ofs++;
+            break;
+        case 'w':
+            if (data) {
+                stw_le_p(data + ofs, val.ll);
+            }
+            ofs += 2;
+            break;
+        case 'l':
+            if (data) {
+                stl_le_p(data + ofs, val.ll);
+            }
+            ofs += 4;
+            break;
+        case 'q':
+            if (data) {
+                stq_le_p(data + ofs, val.ll);
+            }
+            ofs += 8;
+            break;
+        case 's':
+            {
+                int cnt = atoi(p);
+                if (data) {
+                    if (val.str) {
+                        strncpy((void *)data + ofs, val.str, cnt);
+                    } else {
+                        memset((void *)data + ofs, 0, cnt);
+                    }
+                }
+                ofs += cnt;
+                break;
+            }
+        }
+    }
+
+    return ofs;
+}
+
+static size_t vpack(uint8_t **p_data, const char *fmt, va_list ap1)
+{
+    size_t size = 0;
+    uint8_t *data = NULL;
+
+    if (p_data) {
+        va_list ap2;
+
+        va_copy(ap2, ap1);
+        size = vfill(NULL, 0, fmt, ap2);
+        *p_data = data = g_malloc(size);
+        va_end(ap2);
+    }
+    return vfill(data, size, fmt, ap1);
+}
+
+static size_t fill(uint8_t *data, size_t size, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vfill(data, size, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+/* Functions to build the page header and fill in the length, always used
+ * through the macros.
+ */
+
+#define MPTSAS_CONFIG_PACK(number, type, version, fmt, ...)                  \
+    mptsas_config_pack(data, "b*bbb" fmt, version, number, type,             \
+                       ## __VA_ARGS__)
+
+static size_t mptsas_config_pack(uint8_t **data, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vpack(data, fmt, ap);
+    va_end(ap);
+
+    if (data) {
+        assert(ret < 256 && (ret % 4) == 0);
+        stb_p(*data + 1, ret / 4);
+    }
+    return ret;
+}
+
+#define MPTSAS_CONFIG_PACK_EXT(number, type, version, fmt, ...)              \
+    mptsas_config_pack_ext(data, "b*bbb*wb*b" fmt, version, number,          \
+                           MPI_CONFIG_PAGETYPE_EXTENDED, type, ## __VA_ARGS__)
+
+static size_t mptsas_config_pack_ext(uint8_t **data, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vpack(data, fmt, ap);
+    va_end(ap);
+
+    if (data) {
+        assert(ret < 65536 && (ret % 4) == 0);
+        stw_le_p(*data + 4, ret / 4);
+    }
+    return ret;
+}
+
+/* Manufacturing pages */
+
+static
+size_t mptsas_config_manufacturing_0(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "s16s8s16s16s16",
+                              "QEMU MPT Fusion",
+                              "2.5",
+                              "QEMU MPT Fusion",
+                              "QEMU",
+                              "0000111122223333");
+}
+
+static
+size_t mptsas_config_manufacturing_1(MPTSASState *s, uint8_t **data, int address)
+{
+    /* VPD - all zeros */
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "s256");
+}
+
+static
+size_t mptsas_config_manufacturing_2(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "wb*b*l",
+                              pcic->device_id, pcic->revision);
+}
+
+static
+size_t mptsas_config_manufacturing_3(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "wb*b*l",
+                              pcic->device_id, pcic->revision);
+}
+
+static
+size_t mptsas_config_manufacturing_4(MPTSASState *s, uint8_t **data, int address)
+{
+    /* All zeros */
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x05,
+                              "*l*b*b*b*b*b*b*w*s56*l*l*l*l*l*l"
+                              "*b*b*w*b*b*w*l*l");
+}
+
+static
+size_t mptsas_config_manufacturing_5(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(5, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x02,
+                              "q*b*b*w*l*l", s->sas_addr);
+}
+
+static
+size_t mptsas_config_manufacturing_6(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(6, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_7(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(7, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l*l*l*s16*b*b*w", MPTSAS_NUM_PORTS);
+}
+
+static
+size_t mptsas_config_manufacturing_8(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(8, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_9(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(9, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_10(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(10, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+/* I/O unit pages */
+
+static
+size_t mptsas_config_io_unit_0(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+    uint64_t unique_value = 0x53504D554D4551LL;  /* "QEMUMPTx" */
+
+    unique_value |= (uint64_t)pci->devfn << 56;
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x00,
+                              "q", unique_value);
+}
+
+static
+size_t mptsas_config_io_unit_1(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x02, "l",
+                              0x41 /* single function, RAID disabled */ );
+}
+
+static
+size_t mptsas_config_io_unit_2(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+    uint8_t devfn = pci->devfn;
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x02,
+                              "llbbw*b*b*w*b*b*w*b*b*w*l",
+                              0, 0x100, 0 /* pci bus? */, devfn, 0);
+}
+
+static
+size_t mptsas_config_io_unit_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x01,
+                              "*b*b*w*l");
+}
+
+static
+size_t mptsas_config_io_unit_4(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x00, "*l*l*q");
+}
+
+/* I/O controller pages */
+
+static
+size_t mptsas_config_ioc_0(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_IOC, 0x01,
+                              "*l*lwwb*b*b*blww",
+                              pcic->vendor_id, pcic->device_id, pcic->revision,
+                              pcic->subsystem_vendor_id,
+                              pcic->subsystem_id);
+}
+
+static
+size_t mptsas_config_ioc_1(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_IOC, 0x03,
+                              "*l*l*b*b*b*b");
+}
+
+static
+size_t mptsas_config_ioc_2(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_IOC, 0x04,
+                              "*l*b*b*b*b");
+}
+
+static
+size_t mptsas_config_ioc_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_4(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_5(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(5, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*l*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_6(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(6, MPI_CONFIG_PAGETYPE_IOC, 0x01,
+                              "*l*b*b*b*b*b*b*b*b*b*b*w*l*l*l*l*b*b*w"
+                              "*w*w*w*w*l*l*l");
+}
+
+/* SAS I/O unit pages (extended) */
+
+#define MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE 16
+
+#define MPI_SAS_IOUNIT0_RATE_FAILED_SPEED_NEGOTIATION 0x02
+#define MPI_SAS_IOUNIT0_RATE_1_5                      0x08
+#define MPI_SAS_IOUNIT0_RATE_3_0                      0x09
+
+#define MPI_SAS_DEVICE_INFO_NO_DEVICE                 0x00000000
+#define MPI_SAS_DEVICE_INFO_END_DEVICE                0x00000001
+#define MPI_SAS_DEVICE_INFO_SSP_TARGET                0x00000400
+
+#define MPI_SAS_DEVICE0_ASTATUS_NO_ERRORS             0x00
+
+#define MPI_SAS_DEVICE0_FLAGS_DEVICE_PRESENT          0x0001
+#define MPI_SAS_DEVICE0_FLAGS_DEVICE_MAPPED           0x0002
+#define MPI_SAS_DEVICE0_FLAGS_MAPPING_PERSISTENT      0x0004
+
+
+
+static SCSIDevice *mptsas_phy_get_device(MPTSASState *s, int i,
+                                         int *phy_handle, int *dev_handle)
+{
+    SCSIDevice *d = scsi_device_find(&s->bus, 0, i, 0);
+
+    if (phy_handle) {
+        *phy_handle = i + 1;
+    }
+    if (dev_handle) {
+        *dev_handle = d ? i + 1 + MPTSAS_NUM_PORTS : 0;
+    }
+    return d;
+}
+
+static
+size_t mptsas_config_sas_io_unit_0(MPTSASState *s, uint8_t **data, int address)
+{
+    size_t size = MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x04,
+                                         "*w*wb*b*w"
+                                         repl(MPTSAS_NUM_PORTS, "*s16"),
+                                         MPTSAS_NUM_PORTS);
+
+    if (data) {
+        size_t ofs = size - MPTSAS_NUM_PORTS * MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE;
+        int i;
+
+        for (i = 0; i < MPTSAS_NUM_PORTS; i++) {
+            int phy_handle, dev_handle;
+            SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+            fill(*data + ofs, MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE,
+                 "bbbblwwl", i, 0, 0,
+                 (dev
+                  ? MPI_SAS_IOUNIT0_RATE_3_0
+                  : MPI_SAS_IOUNIT0_RATE_FAILED_SPEED_NEGOTIATION),
+                 (dev
+                  ? MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET
+                  : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                 dev_handle,
+                 dev_handle,
+                 0);
+            ofs += MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE;
+        }
+        assert(ofs == size);
+    }
+    return size;
+}
+
+#define MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE 12
+
+static
+size_t mptsas_config_sas_io_unit_1(MPTSASState *s, uint8_t **data, int address)
+{
+    size_t size = MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x07,
+                                         "*w*w*w*wb*b*b*b"
+                                         repl(MPTSAS_NUM_PORTS, "*s12"),
+                                         MPTSAS_NUM_PORTS);
+
+    if (data) {
+        size_t ofs = size - MPTSAS_NUM_PORTS * MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE;
+        int i;
+
+        for (i = 0; i < MPTSAS_NUM_PORTS; i++) {
+            SCSIDevice *dev = mptsas_phy_get_device(s, i, NULL, NULL);
+            fill(*data + ofs, MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE,
+                 "bbbblww", i, 0, 0,
+                 (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5,
+                 (dev
+                  ? MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET
+                  : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                 0, 0);
+            ofs += MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE;
+        }
+        assert(ofs == size);
+    }
+    return size;
+}
+
+static
+size_t mptsas_config_sas_io_unit_2(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK_EXT(2, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x06,
+                                  "*b*b*w*w*w*b*b*w");
+}
+
+static
+size_t mptsas_config_sas_io_unit_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK_EXT(3, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x06,
+                                  "*l*l*l*l*l*l*l*l*l");
+}
+
+/* SAS PHY pages (extended) */
+
+static int mptsas_phy_addr_get(MPTSASState *s, int address)
+{
+    int i;
+    if ((address >> MPI_SAS_PHY_PGAD_FORM_SHIFT) == 0) {
+        i = address & 255;
+    } else if ((address >> MPI_SAS_PHY_PGAD_FORM_SHIFT) == 1) {
+        i = address & 65535;
+    } else {
+        return -EINVAL;
+    }
+
+    if (i >= MPTSAS_NUM_PORTS) {
+        return -EINVAL;
+    }
+
+    return i;
+}
+
+static
+size_t mptsas_config_phy_0(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_phy_addr_get(s, address);
+    SCSIDevice *dev;
+
+    if (i < 0) {
+        trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 0);
+        return i;
+    }
+
+    dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+    trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 0);
+
+    return MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_PHY, 0x01,
+                                  "w*wqwb*blbb*b*b*l",
+                                  dev_handle, s->sas_addr, dev_handle, i,
+                                  (dev
+                                   ? MPI_SAS_DEVICE_INFO_END_DEVICE /* | MPI_SAS_DEVICE_INFO_SSP_TARGET?? */
+                                   : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                                  (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5,
+                                  (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5);
+}
+
+static
+size_t mptsas_config_phy_1(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_phy_addr_get(s, address);
+
+    if (i < 0) {
+        trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 1);
+        return i;
+    }
+
+    (void) mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+    trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 1);
+
+    return MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_PHY, 0x01,
+                                  "*l*l*l*l*l");
+}
+
+/* SAS device pages (extended) */
+
+static int mptsas_device_addr_get(MPTSASState *s, int address)
+{
+    uint32_t handle, i;
+    uint32_t form = address >> MPI_SAS_PHY_PGAD_FORM_SHIFT;
+    if (form == MPI_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE) {
+        handle = address & MPI_SAS_DEVICE_PGAD_GNH_HANDLE_MASK;
+        do {
+            if (handle == 65535) {
+                handle = MPTSAS_NUM_PORTS + 1;
+            } else {
+                ++handle;
+            }
+            i = handle - 1 - MPTSAS_NUM_PORTS;
+        } while (i < MPTSAS_NUM_PORTS && !scsi_device_find(&s->bus, 0, i, 0));
+
+    } else if (form == MPI_SAS_DEVICE_PGAD_FORM_BUS_TARGET_ID) {
+        if (address & MPI_SAS_DEVICE_PGAD_BT_BUS_MASK) {
+            return -EINVAL;
+        }
+        i = address & MPI_SAS_DEVICE_PGAD_BT_TID_MASK;
+
+    } else if (form == MPI_SAS_DEVICE_PGAD_FORM_HANDLE) {
+        handle = address & MPI_SAS_DEVICE_PGAD_H_HANDLE_MASK;
+        i = handle - 1 - MPTSAS_NUM_PORTS;
+
+    } else {
+        return -EINVAL;
+    }
+
+    if (i >= MPTSAS_NUM_PORTS) {
+        return -EINVAL;
+    }
+
+    return i;
+}
+
+static
+size_t mptsas_config_sas_device_0(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 0);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x05,
+                                  "*w*wqwbbwbblwb*b",
+                                  dev->wwn, phy_handle, i,
+                                  MPI_SAS_DEVICE0_ASTATUS_NO_ERRORS,
+                                  dev_handle, i, 0,
+                                  MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET,
+                                  (MPI_SAS_DEVICE0_FLAGS_DEVICE_PRESENT |
+                                   MPI_SAS_DEVICE0_FLAGS_DEVICE_MAPPED |
+                                   MPI_SAS_DEVICE0_FLAGS_MAPPING_PERSISTENT), i);
+}
+
+static
+size_t mptsas_config_sas_device_1(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 1);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x00,
+                                  "*lq*lwbb*s20",
+                                  dev->wwn, dev_handle, i, 0);
+}
+
+static
+size_t mptsas_config_sas_device_2(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 2);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(2, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x01,
+                                  "ql", dev->wwn, 0);
+}
+
+typedef struct MPTSASConfigPage {
+    uint8_t number;
+    uint8_t type;
+    size_t (*mpt_config_build)(MPTSASState *s, uint8_t **data, int address);
+} MPTSASConfigPage;
+
+static const MPTSASConfigPage mptsas_config_pages[] = {
+    {
+        0, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_4,
+    }, {
+        5, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_5,
+    }, {
+        6, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_6,
+    }, {
+        7, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_7,
+    }, {
+        8, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_8,
+    }, {
+        9, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_9,
+    }, {
+        10, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_10,
+    }, {
+        0, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_4,
+    }, {
+        0, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_4,
+    }, {
+        5, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_5,
+    }, {
+        6, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_6,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_1,
+    }, {
+        2, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_2,
+    }, {
+        3, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_3,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_PHY,
+        mptsas_config_phy_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_PHY,
+        mptsas_config_phy_1,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_1,
+    }, {
+       2,  MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_2,
+    }
+};
+
+static const MPTSASConfigPage *mptsas_find_config_page(int type, int number)
+{
+    const MPTSASConfigPage *page;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(mptsas_config_pages); i++) {
+        page = &mptsas_config_pages[i];
+        if (page->type == type && page->number == number) {
+            return page;
+        }
+    }
+
+    return NULL;
+}
+
+void mptsas_process_config(MPTSASState *s, MPIMsgConfig *req)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+
+    MPIMsgConfigReply reply;
+    const MPTSASConfigPage *page;
+    size_t length;
+    uint8_t type;
+    uint8_t *data = NULL;
+    uint32_t flags_and_length;
+    uint32_t dmalen;
+    uint64_t pa;
+
+    mptsas_fix_config_endianness(req);
+
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    /* Copy common bits from the request into the reply. */
+    memset(&reply, 0, sizeof(reply));
+    reply.Action      = req->Action;
+    reply.Function    = req->Function;
+    reply.MsgContext  = req->MsgContext;
+    reply.MsgLength   = sizeof(reply) / 4;
+    reply.PageType    = req->PageType;
+    reply.PageNumber  = req->PageNumber;
+    reply.PageLength  = req->PageLength;
+    reply.PageVersion = req->PageVersion;
+
+    type = req->PageType & MPI_CONFIG_PAGETYPE_MASK;
+    if (type == MPI_CONFIG_PAGETYPE_EXTENDED) {
+        type = req->ExtPageType;
+        if (type <= MPI_CONFIG_PAGETYPE_MASK) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_TYPE;
+            goto out;
+        }
+
+        reply.ExtPageType = req->ExtPageType;
+    }
+
+    page = mptsas_find_config_page(type, req->PageNumber);
+
+    switch(req->Action) {
+    case MPI_CONFIG_ACTION_PAGE_DEFAULT:
+    case MPI_CONFIG_ACTION_PAGE_HEADER:
+    case MPI_CONFIG_ACTION_PAGE_READ_NVRAM:
+    case MPI_CONFIG_ACTION_PAGE_READ_CURRENT:
+    case MPI_CONFIG_ACTION_PAGE_READ_DEFAULT:
+    case MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT:
+    case MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM:
+        break;
+
+    default:
+        reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_ACTION;
+        goto out;
+    }
+
+    if (!page) {
+        page = mptsas_find_config_page(type, 1);
+        if (page) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        } else {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_TYPE;
+        }
+        goto out;
+    }
+
+    if (req->Action == MPI_CONFIG_ACTION_PAGE_DEFAULT ||
+        req->Action == MPI_CONFIG_ACTION_PAGE_HEADER) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+            goto out;
+        } else {
+            goto done;
+        }
+    }
+
+    if (req->Action == MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT ||
+        req->Action == MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        } else {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_CANT_COMMIT;
+        }
+        goto out;
+    }
+
+    flags_and_length = req->PageBufferSGE.FlagsLength;
+    dmalen = flags_and_length & MPI_SGE_LENGTH_MASK;
+    if (dmalen == 0) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+            goto out;
+        } else {
+            goto done;
+        }
+    }
+
+    if (flags_and_length & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+        pa = req->PageBufferSGE.u.Address64;
+    } else {
+        pa = req->PageBufferSGE.u.Address32;
+    }
+
+    /* Only read actions left.  */
+    length = page->mpt_config_build(s, &data, req->PageAddress);
+    if ((ssize_t)length < 0) {
+        reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        goto out;
+    } else {
+        assert(data[2] == page->number);
+        pci_dma_write(pci, pa, data, MIN(length, dmalen));
+        goto done;
+    }
+
+    abort();
+
+done:
+    if (type > MPI_CONFIG_PAGETYPE_MASK) {
+        reply.ExtPageLength = length / 4;
+        reply.ExtPageType   = req->ExtPageType;
+    } else {
+        reply.PageLength    = length / 4;
+    }
+
+out:
+    mptsas_fix_config_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+    g_free(data);
+}
diff --git a/qemu/hw/scsi/mptendian.c b/qemu/hw/scsi/mptendian.c
new file mode 100644
index 000000000..b7fe2a2a3
--- /dev/null
+++ b/qemu/hw/scsi/mptendian.c
@@ -0,0 +1,204 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation
+ * Endianness conversion for MPI data structures
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Authors: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "sysemu/dma.h"
+#include "sysemu/block-backend.h"
+#include "hw/pci/msi.h"
+#include "qemu/iov.h"
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "trace.h"
+
+#include "mptsas.h"
+#include "mpi.h"
+
+static void mptsas_fix_sgentry_endianness(MPISGEntry *sge)
+{
+    le32_to_cpus(&sge->FlagsLength);
+    if (sge->FlagsLength & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+       le64_to_cpus(&sge->u.Address64);
+    } else {
+       le32_to_cpus(&sge->u.Address32);
+    }
+}
+
+static void mptsas_fix_sgentry_endianness_reply(MPISGEntry *sge)
+{
+    if (sge->FlagsLength & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+       cpu_to_le64s(&sge->u.Address64);
+    } else {
+       cpu_to_le32s(&sge->u.Address32);
+    }
+    cpu_to_le32s(&sge->FlagsLength);
+}
+
+void mptsas_fix_scsi_io_endianness(MPIMsgSCSIIORequest *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->Control);
+    le32_to_cpus(&req->DataLength);
+    le32_to_cpus(&req->SenseBufferLowAddr);
+}
+
+void mptsas_fix_scsi_io_reply_endianness(MPIMsgSCSIIOReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->TransferCount);
+    cpu_to_le32s(&reply->SenseCount);
+    cpu_to_le32s(&reply->ResponseInfo);
+    cpu_to_le16s(&reply->TaskTag);
+}
+
+void mptsas_fix_scsi_task_mgmt_endianness(MPIMsgSCSITaskMgmt *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->TaskMsgContext);
+}
+
+void mptsas_fix_scsi_task_mgmt_reply_endianness(MPIMsgSCSITaskMgmtReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->TerminationCount);
+}
+
+void mptsas_fix_ioc_init_endianness(MPIMsgIOCInit *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le16_to_cpus(&req->ReplyFrameSize);
+    le32_to_cpus(&req->HostMfaHighAddr);
+    le32_to_cpus(&req->SenseBufferHighAddr);
+    le32_to_cpus(&req->ReplyFifoHostSignalingAddr);
+    mptsas_fix_sgentry_endianness(&req->HostPageBufferSGE);
+    le16_to_cpus(&req->MsgVersion);
+    le16_to_cpus(&req->HeaderVersion);
+}
+
+void mptsas_fix_ioc_init_reply_endianness(MPIMsgIOCInitReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_ioc_facts_endianness(MPIMsgIOCFacts *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_ioc_facts_reply_endianness(MPIMsgIOCFactsReply *reply)
+{
+    cpu_to_le16s(&reply->MsgVersion);
+    cpu_to_le16s(&reply->HeaderVersion);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCExceptions);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le16s(&reply->ReplyQueueDepth);
+    cpu_to_le16s(&reply->RequestFrameSize);
+    cpu_to_le16s(&reply->ProductID);
+    cpu_to_le32s(&reply->CurrentHostMfaHighAddr);
+    cpu_to_le16s(&reply->GlobalCredits);
+    cpu_to_le32s(&reply->CurrentSenseBufferHighAddr);
+    cpu_to_le16s(&reply->CurReplyFrameSize);
+    cpu_to_le32s(&reply->FWImageSize);
+    cpu_to_le32s(&reply->IOCCapabilities);
+    cpu_to_le16s(&reply->HighPriorityQueueDepth);
+    mptsas_fix_sgentry_endianness_reply(&reply->HostPageBufferSGE);
+    cpu_to_le32s(&reply->ReplyFifoHostSignalingAddr);
+}
+
+void mptsas_fix_config_endianness(MPIMsgConfig *req)
+{
+    le16_to_cpus(&req->ExtPageLength);
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->PageAddress);
+    mptsas_fix_sgentry_endianness(&req->PageBufferSGE);
+}
+
+void mptsas_fix_config_reply_endianness(MPIMsgConfigReply *reply)
+{
+    cpu_to_le16s(&reply->ExtPageLength);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_port_facts_endianness(MPIMsgPortFacts *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_port_facts_reply_endianness(MPIMsgPortFactsReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le16s(&reply->MaxDevices);
+    cpu_to_le16s(&reply->PortSCSIID);
+    cpu_to_le16s(&reply->ProtocolFlags);
+    cpu_to_le16s(&reply->MaxPostedCmdBuffers);
+    cpu_to_le16s(&reply->MaxPersistentIDs);
+    cpu_to_le16s(&reply->MaxLanBuckets);
+}
+
+void mptsas_fix_port_enable_endianness(MPIMsgPortEnable *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_port_enable_reply_endianness(MPIMsgPortEnableReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_event_notification_endianness(MPIMsgEventNotify *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_event_notification_reply_endianness(MPIMsgEventNotifyReply *reply)
+{
+    int length = reply->EventDataLength;
+    int i;
+
+    cpu_to_le16s(&reply->EventDataLength);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->Event);
+    cpu_to_le32s(&reply->EventContext);
+
+    /* Really depends on the event kind.  This will do for now.  */
+    for (i = 0; i < length; i++) {
+        cpu_to_le32s(&reply->Data[i]);
+    }
+}
+
diff --git a/qemu/hw/scsi/mptsas.c b/qemu/hw/scsi/mptsas.c
new file mode 100644
index 000000000..499c1465a
--- /dev/null
+++ b/qemu/hw/scsi/mptsas.c
@@ -0,0 +1,1442 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation
+ * Based on the QEMU Megaraid emulator
+ *
+ * Copyright (c) 2009-2012 Hannes Reinecke, SUSE Labs
+ * Copyright (c) 2012 Verizon, Inc.
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Authors: Don Slutz, Paolo Bonzini
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "sysemu/dma.h"
+#include "sysemu/block-backend.h"
+#include "hw/pci/msi.h"
+#include "qemu/iov.h"
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "trace.h"
+
+#include "mptsas.h"
+#include "mpi.h"
+
+#define NAA_LOCALLY_ASSIGNED_ID 0x3ULL
+#define IEEE_COMPANY_LOCALLY_ASSIGNED 0x525400
+
+#define TYPE_MPTSAS1068 "mptsas1068"
+
+#define MPT_SAS(obj) \
+    OBJECT_CHECK(MPTSASState, (obj), TYPE_MPTSAS1068)
+
+#define MPTSAS1068_PRODUCT_ID                  \
+    (MPI_FW_HEADER_PID_FAMILY_1068_SAS |       \
+     MPI_FW_HEADER_PID_PROD_INITIATOR_SCSI |   \
+     MPI_FW_HEADER_PID_TYPE_SAS)
+
+struct MPTSASRequest {
+    MPIMsgSCSIIORequest scsi_io;
+    SCSIRequest *sreq;
+    QEMUSGList qsg;
+    MPTSASState *dev;
+
+    QTAILQ_ENTRY(MPTSASRequest) next;
+};
+
+static void mptsas_update_interrupt(MPTSASState *s)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    uint32_t state = s->intr_status & ~(s->intr_mask | MPI_HIS_IOP_DOORBELL_STATUS);
+
+    if (s->msi_in_use && msi_enabled(pci)) {
+        if (state) {
+            trace_mptsas_irq_msi(s);
+            msi_notify(pci, 0);
+        }
+    }
+
+    trace_mptsas_irq_intx(s, !!state);
+    pci_set_irq(pci, !!state);
+}
+
+static void mptsas_set_fault(MPTSASState *s, uint32_t code)
+{
+    if ((s->state & MPI_IOC_STATE_FAULT) == 0) {
+        s->state = MPI_IOC_STATE_FAULT | code;
+    }
+}
+
+#define MPTSAS_FIFO_INVALID(s, name)                     \
+    ((s)->name##_head > ARRAY_SIZE((s)->name) ||         \
+     (s)->name##_tail > ARRAY_SIZE((s)->name))
+
+#define MPTSAS_FIFO_EMPTY(s, name)                       \
+    ((s)->name##_head == (s)->name##_tail)
+
+#define MPTSAS_FIFO_FULL(s, name)                        \
+    ((s)->name##_head == ((s)->name##_tail + 1) % ARRAY_SIZE((s)->name))
+
+#define MPTSAS_FIFO_GET(s, name) ({                      \
+    uint32_t _val = (s)->name[(s)->name##_head++];       \
+    (s)->name##_head %= ARRAY_SIZE((s)->name);           \
+    _val;                                                \
+})
+
+#define MPTSAS_FIFO_PUT(s, name, val) do {       \
+    (s)->name[(s)->name##_tail++] = (val);       \
+    (s)->name##_tail %= ARRAY_SIZE((s)->name);   \
+} while(0)
+
+static void mptsas_post_reply(MPTSASState *s, MPIDefaultReply *reply)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    uint32_t addr_lo;
+
+    if (MPTSAS_FIFO_EMPTY(s, reply_free) || MPTSAS_FIFO_FULL(s, reply_post)) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        return;
+    }
+
+    addr_lo = MPTSAS_FIFO_GET(s, reply_free);
+
+    pci_dma_write(pci, addr_lo | s->host_mfa_high_addr, reply,
+                  MIN(s->reply_frame_size, 4 * reply->MsgLength));
+
+    MPTSAS_FIFO_PUT(s, reply_post, MPI_ADDRESS_REPLY_A_BIT | (addr_lo >> 1));
+
+    s->intr_status |= MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        s->doorbell_state = DOORBELL_NONE;
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+    }
+    mptsas_update_interrupt(s);
+}
+
+void mptsas_reply(MPTSASState *s, MPIDefaultReply *reply)
+{
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        /* The reply is sent out in 16 bit chunks, while the size
+         * in the reply is in 32 bit units.
+         */
+        s->doorbell_state = DOORBELL_READ;
+        s->doorbell_reply_idx = 0;
+        s->doorbell_reply_size = reply->MsgLength * 2;
+        memcpy(s->doorbell_reply, reply, s->doorbell_reply_size * 2);
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+        mptsas_update_interrupt(s);
+    } else {
+        mptsas_post_reply(s, reply);
+    }
+}
+
+static void mptsas_turbo_reply(MPTSASState *s, uint32_t msgctx)
+{
+    if (MPTSAS_FIFO_FULL(s, reply_post)) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        return;
+    }
+
+    /* The reply is just the message context ID (bit 31 = clear). */
+    MPTSAS_FIFO_PUT(s, reply_post, msgctx);
+
+    s->intr_status |= MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+    mptsas_update_interrupt(s);
+}
+
+#define MPTSAS_MAX_REQUEST_SIZE 52
+
+static const int mpi_request_sizes[] = {
+    [MPI_FUNCTION_SCSI_IO_REQUEST]    = sizeof(MPIMsgSCSIIORequest),
+    [MPI_FUNCTION_SCSI_TASK_MGMT]     = sizeof(MPIMsgSCSITaskMgmt),
+    [MPI_FUNCTION_IOC_INIT]           = sizeof(MPIMsgIOCInit),
+    [MPI_FUNCTION_IOC_FACTS]          = sizeof(MPIMsgIOCFacts),
+    [MPI_FUNCTION_CONFIG]             = sizeof(MPIMsgConfig),
+    [MPI_FUNCTION_PORT_FACTS]         = sizeof(MPIMsgPortFacts),
+    [MPI_FUNCTION_PORT_ENABLE]        = sizeof(MPIMsgPortEnable),
+    [MPI_FUNCTION_EVENT_NOTIFICATION] = sizeof(MPIMsgEventNotify),
+};
+
+static dma_addr_t mptsas_ld_sg_base(MPTSASState *s, uint32_t flags_and_length,
+                                    dma_addr_t *sgaddr)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    dma_addr_t addr;
+
+    if (flags_and_length & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+        addr = ldq_le_pci_dma(pci, *sgaddr + 4);
+        *sgaddr += 12;
+    } else {
+        addr = ldl_le_pci_dma(pci, *sgaddr + 4);
+        *sgaddr += 8;
+    }
+    return addr;
+}
+
+static int mptsas_build_sgl(MPTSASState *s, MPTSASRequest *req, hwaddr addr)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    hwaddr next_chain_addr;
+    uint32_t left;
+    hwaddr sgaddr;
+    uint32_t chain_offset;
+
+    chain_offset = req->scsi_io.ChainOffset;
+    next_chain_addr = addr + chain_offset * sizeof(uint32_t);
+    sgaddr = addr + sizeof(MPIMsgSCSIIORequest);
+    pci_dma_sglist_init(&req->qsg, pci, 4);
+    left = req->scsi_io.DataLength;
+
+    for(;;) {
+        dma_addr_t addr, len;
+        uint32_t flags_and_length;
+
+        flags_and_length = ldl_le_pci_dma(pci, sgaddr);
+        len = flags_and_length & MPI_SGE_LENGTH_MASK;
+        if ((flags_and_length & MPI_SGE_FLAGS_ELEMENT_TYPE_MASK)
+            != MPI_SGE_FLAGS_SIMPLE_ELEMENT ||
+            (!len &&
+             !(flags_and_length & MPI_SGE_FLAGS_END_OF_LIST) &&
+             !(flags_and_length & MPI_SGE_FLAGS_END_OF_BUFFER))) {
+            return MPI_IOCSTATUS_INVALID_SGL;
+        }
+
+        len = MIN(len, left);
+        if (!len) {
+            /* We reached the desired transfer length, ignore extra
+             * elements of the s/g list.
+             */
+            break;
+        }
+
+        addr = mptsas_ld_sg_base(s, flags_and_length, &sgaddr);
+        qemu_sglist_add(&req->qsg, addr, len);
+        left -= len;
+
+        if (flags_and_length & MPI_SGE_FLAGS_END_OF_LIST) {
+            break;
+        }
+
+        if (flags_and_length & MPI_SGE_FLAGS_LAST_ELEMENT) {
+            if (!chain_offset) {
+                break;
+            }
+
+            flags_and_length = ldl_le_pci_dma(pci, next_chain_addr);
+            if ((flags_and_length & MPI_SGE_FLAGS_ELEMENT_TYPE_MASK)
+                != MPI_SGE_FLAGS_CHAIN_ELEMENT) {
+                return MPI_IOCSTATUS_INVALID_SGL;
+            }
+
+            sgaddr = mptsas_ld_sg_base(s, flags_and_length, &next_chain_addr);
+            chain_offset =
+                (flags_and_length & MPI_SGE_CHAIN_OFFSET_MASK) >> MPI_SGE_CHAIN_OFFSET_SHIFT;
+            next_chain_addr = sgaddr + chain_offset * sizeof(uint32_t);
+        }
+    }
+    return 0;
+}
+
+static void mptsas_free_request(MPTSASRequest *req)
+{
+    MPTSASState *s = req->dev;
+
+    if (req->sreq != NULL) {
+        req->sreq->hba_private = NULL;
+        scsi_req_unref(req->sreq);
+        req->sreq = NULL;
+        QTAILQ_REMOVE(&s->pending, req, next);
+    }
+    qemu_sglist_destroy(&req->qsg);
+    g_free(req);
+}
+
+static int mptsas_scsi_device_find(MPTSASState *s, int bus, int target,
+                                   uint8_t *lun, SCSIDevice **sdev)
+{
+    if (bus != 0) {
+        return MPI_IOCSTATUS_SCSI_INVALID_BUS;
+    }
+
+    if (target >= s->max_devices) {
+        return MPI_IOCSTATUS_SCSI_INVALID_TARGETID;
+    }
+
+    *sdev = scsi_device_find(&s->bus, bus, target, lun[1]);
+    if (!*sdev) {
+        return MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE;
+    }
+
+    return 0;
+}
+
+static int mptsas_process_scsi_io_request(MPTSASState *s,
+                                          MPIMsgSCSIIORequest *scsi_io,
+                                          hwaddr addr)
+{
+    MPTSASRequest *req;
+    MPIMsgSCSIIOReply reply;
+    SCSIDevice *sdev;
+    int status;
+
+    mptsas_fix_scsi_io_endianness(scsi_io);
+
+    trace_mptsas_process_scsi_io_request(s, scsi_io->Bus, scsi_io->TargetID,
+                                         scsi_io->LUN[1], scsi_io->DataLength);
+
+    status = mptsas_scsi_device_find(s, scsi_io->Bus, scsi_io->TargetID,
+                                     scsi_io->LUN, &sdev);
+    if (status) {
+        goto bad;
+    }
+
+    req = g_new(MPTSASRequest, 1);
+    QTAILQ_INSERT_TAIL(&s->pending, req, next);
+    req->scsi_io = *scsi_io;
+    req->dev = s;
+
+    status = mptsas_build_sgl(s, req, addr);
+    if (status) {
+        goto free_bad;
+    }
+
+    if (req->qsg.size < scsi_io->DataLength) {
+        trace_mptsas_sgl_overflow(s, scsi_io->MsgContext, scsi_io->DataLength,
+                                  req->qsg.size);
+        status = MPI_IOCSTATUS_INVALID_SGL;
+        goto free_bad;
+    }
+
+    req->sreq = scsi_req_new(sdev, scsi_io->MsgContext,
+                            scsi_io->LUN[1], scsi_io->CDB, req);
+
+    if (req->sreq->cmd.xfer > scsi_io->DataLength) {
+        goto overrun;
+    }
+    switch (scsi_io->Control & MPI_SCSIIO_CONTROL_DATADIRECTION_MASK) {
+    case MPI_SCSIIO_CONTROL_NODATATRANSFER:
+        if (req->sreq->cmd.mode != SCSI_XFER_NONE) {
+            goto overrun;
+        }
+        break;
+
+    case MPI_SCSIIO_CONTROL_WRITE:
+        if (req->sreq->cmd.mode != SCSI_XFER_TO_DEV) {
+            goto overrun;
+        }
+        break;
+
+    case MPI_SCSIIO_CONTROL_READ:
+        if (req->sreq->cmd.mode != SCSI_XFER_FROM_DEV) {
+            goto overrun;
+        }
+        break;
+    }
+
+    if (scsi_req_enqueue(req->sreq)) {
+        scsi_req_continue(req->sreq);
+    }
+    return 0;
+
+overrun:
+    trace_mptsas_scsi_overflow(s, scsi_io->MsgContext, req->sreq->cmd.xfer,
+                               scsi_io->DataLength);
+    status = MPI_IOCSTATUS_SCSI_DATA_OVERRUN;
+free_bad:
+    mptsas_free_request(req);
+bad:
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID          = scsi_io->TargetID;
+    reply.Bus               = scsi_io->Bus;
+    reply.MsgLength         = sizeof(reply) / 4;
+    reply.Function          = scsi_io->Function;
+    reply.CDBLength         = scsi_io->CDBLength;
+    reply.SenseBufferLength = scsi_io->SenseBufferLength;
+    reply.MsgContext        = scsi_io->MsgContext;
+    reply.SCSIState         = MPI_SCSI_STATE_NO_SCSI_STATUS;
+    reply.IOCStatus         = status;
+
+    mptsas_fix_scsi_io_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+
+    return 0;
+}
+
+typedef struct {
+    Notifier                notifier;
+    MPTSASState             *s;
+    MPIMsgSCSITaskMgmtReply *reply;
+} MPTSASCancelNotifier;
+
+static void mptsas_cancel_notify(Notifier *notifier, void *data)
+{
+    MPTSASCancelNotifier *n = container_of(notifier,
+                                           MPTSASCancelNotifier,
+                                           notifier);
+
+    /* Abusing IOCLogInfo to store the expected number of requests... */
+    if (++n->reply->TerminationCount == n->reply->IOCLogInfo) {
+        n->reply->IOCLogInfo = 0;
+        mptsas_fix_scsi_task_mgmt_reply_endianness(n->reply);
+        mptsas_post_reply(n->s, (MPIDefaultReply *)n->reply);
+        g_free(n->reply);
+    }
+    g_free(n);
+}
+
+static void mptsas_process_scsi_task_mgmt(MPTSASState *s, MPIMsgSCSITaskMgmt *req)
+{
+    MPIMsgSCSITaskMgmtReply reply;
+    MPIMsgSCSITaskMgmtReply *reply_async;
+    int status, count;
+    SCSIDevice *sdev;
+    SCSIRequest *r, *next;
+    BusChild *kid;
+
+    mptsas_fix_scsi_task_mgmt_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID   = req->TargetID;
+    reply.Bus        = req->Bus;
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.TaskType   = req->TaskType;
+    reply.MsgContext = req->MsgContext;
+
+    switch (req->TaskType) {
+    case MPI_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
+    case MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+
+        QTAILQ_FOREACH_SAFE(r, &sdev->requests, next, next) {
+            MPTSASRequest *cmd_req = r->hba_private;
+            if (cmd_req && cmd_req->scsi_io.MsgContext == req->TaskMsgContext) {
+                break;
+            }
+        }
+        if (r) {
+            /*
+             * Assert that the request has not been completed yet, we
+             * check for it in the loop above.
+             */
+            assert(r->hba_private);
+            if (req->TaskType == MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK) {
+                /* "If the specified command is present in the task set, then
+                 * return a service response set to FUNCTION SUCCEEDED".
+                 */
+                reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_SUCCEEDED;
+            } else {
+                MPTSASCancelNotifier *notifier;
+
+                reply_async = g_memdup(&reply, sizeof(MPIMsgSCSITaskMgmtReply));
+                reply_async->IOCLogInfo = INT_MAX;
+
+                count = 1;
+                notifier = g_new(MPTSASCancelNotifier, 1);
+                notifier->s = s;
+                notifier->reply = reply_async;
+                notifier->notifier.notify = mptsas_cancel_notify;
+                scsi_req_cancel_async(r, &notifier->notifier);
+                goto reply_maybe_async;
+            }
+        }
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
+    case MPI_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+
+        reply_async = g_memdup(&reply, sizeof(MPIMsgSCSITaskMgmtReply));
+        reply_async->IOCLogInfo = INT_MAX;
+
+        count = 0;
+        QTAILQ_FOREACH_SAFE(r, &sdev->requests, next, next) {
+            if (r->hba_private) {
+                MPTSASCancelNotifier *notifier;
+
+                count++;
+                notifier = g_new(MPTSASCancelNotifier, 1);
+                notifier->s = s;
+                notifier->reply = reply_async;
+                notifier->notifier.notify = mptsas_cancel_notify;
+                scsi_req_cancel_async(r, &notifier->notifier);
+            }
+        }
+
+reply_maybe_async:
+        if (reply_async->TerminationCount < count) {
+            reply_async->IOCLogInfo = count;
+            return;
+        }
+        g_free(reply_async);
+        reply.TerminationCount = count;
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+        qdev_reset_all(&sdev->qdev);
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
+        if (req->Bus != 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_SCSI_INVALID_BUS;
+            goto out;
+        }
+        if (req->TargetID > s->max_devices) {
+            reply.IOCStatus = MPI_IOCSTATUS_SCSI_INVALID_TARGETID;
+            goto out;
+        }
+
+        QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
+            sdev = SCSI_DEVICE(kid->child);
+            if (sdev->channel == 0 && sdev->id == req->TargetID) {
+                qdev_reset_all(kid->child);
+            }
+        }
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_RESET_BUS:
+        qbus_reset_all(&s->bus.qbus);
+        break;
+
+    default:
+        reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED;
+        break;
+    }
+
+out:
+    mptsas_fix_scsi_task_mgmt_reply_endianness(&reply);
+    mptsas_post_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_ioc_init(MPTSASState *s, MPIMsgIOCInit *req)
+{
+    MPIMsgIOCInitReply reply;
+
+    mptsas_fix_ioc_init_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    s->who_init               = req->WhoInit;
+    s->reply_frame_size       = req->ReplyFrameSize;
+    s->max_buses              = req->MaxBuses;
+    s->max_devices            = req->MaxDevices ? req->MaxDevices : 256;
+    s->host_mfa_high_addr     = (hwaddr)req->HostMfaHighAddr << 32;
+    s->sense_buffer_high_addr = (hwaddr)req->SenseBufferHighAddr << 32;
+
+    if (s->state == MPI_IOC_STATE_READY) {
+        s->state = MPI_IOC_STATE_OPERATIONAL;
+    }
+
+    memset(&reply, 0, sizeof(reply));
+    reply.WhoInit    = s->who_init;
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.MaxDevices = s->max_devices;
+    reply.MaxBuses   = s->max_buses;
+    reply.MsgContext = req->MsgContext;
+
+    mptsas_fix_ioc_init_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_ioc_facts(MPTSASState *s,
+                                     MPIMsgIOCFacts *req)
+{
+    MPIMsgIOCFactsReply reply;
+
+    mptsas_fix_ioc_facts_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgVersion                 = 0x0105;
+    reply.MsgLength                  = sizeof(reply) / 4;
+    reply.Function                   = req->Function;
+    reply.MsgContext                 = req->MsgContext;
+    reply.MaxChainDepth              = MPTSAS_MAXIMUM_CHAIN_DEPTH;
+    reply.WhoInit                    = s->who_init;
+    reply.BlockSize                  = MPTSAS_MAX_REQUEST_SIZE / sizeof(uint32_t);
+    reply.ReplyQueueDepth            = ARRAY_SIZE(s->reply_post) - 1;
+    QEMU_BUILD_BUG_ON(ARRAY_SIZE(s->reply_post) != ARRAY_SIZE(s->reply_free));
+
+    reply.RequestFrameSize           = 128;
+    reply.ProductID                  = MPTSAS1068_PRODUCT_ID;
+    reply.CurrentHostMfaHighAddr     = s->host_mfa_high_addr >> 32;
+    reply.GlobalCredits              = ARRAY_SIZE(s->request_post) - 1;
+    reply.NumberOfPorts              = MPTSAS_NUM_PORTS;
+    reply.CurrentSenseBufferHighAddr = s->sense_buffer_high_addr >> 32;
+    reply.CurReplyFrameSize          = s->reply_frame_size;
+    reply.MaxDevices                 = s->max_devices;
+    reply.MaxBuses                   = s->max_buses;
+    reply.FWVersionDev               = 0;
+    reply.FWVersionUnit              = 0x92;
+    reply.FWVersionMinor             = 0x32;
+    reply.FWVersionMajor             = 0x1;
+
+    mptsas_fix_ioc_facts_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_port_facts(MPTSASState *s,
+                                     MPIMsgPortFacts *req)
+{
+    MPIMsgPortFactsReply reply;
+
+    mptsas_fix_port_facts_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.PortNumber = req->PortNumber;
+    reply.MsgContext = req->MsgContext;
+
+    if (req->PortNumber < MPTSAS_NUM_PORTS) {
+        reply.PortType      = MPI_PORTFACTS_PORTTYPE_SAS;
+        reply.MaxDevices    = MPTSAS_NUM_PORTS;
+        reply.PortSCSIID    = MPTSAS_NUM_PORTS;
+        reply.ProtocolFlags = MPI_PORTFACTS_PROTOCOL_LOGBUSADDR | MPI_PORTFACTS_PROTOCOL_INITIATOR;
+    }
+
+    mptsas_fix_port_facts_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_port_enable(MPTSASState *s,
+                                       MPIMsgPortEnable *req)
+{
+    MPIMsgPortEnableReply reply;
+
+    mptsas_fix_port_enable_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.PortNumber = req->PortNumber;
+    reply.Function   = req->Function;
+    reply.MsgContext = req->MsgContext;
+
+    mptsas_fix_port_enable_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_event_notification(MPTSASState *s,
+                                              MPIMsgEventNotify *req)
+{
+    MPIMsgEventNotifyReply reply;
+
+    mptsas_fix_event_notification_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    /* Don't even bother storing whether event notification is enabled,
+     * since it is not accessible.
+     */
+
+    memset(&reply, 0, sizeof(reply));
+    reply.EventDataLength = sizeof(reply.Data) / 4;
+    reply.MsgLength       = sizeof(reply) / 4;
+    reply.Function        = req->Function;
+
+    /* This is set because events are sent through the reply FIFOs.  */
+    reply.MsgFlags        = MPI_MSGFLAGS_CONTINUATION_REPLY;
+
+    reply.MsgContext      = req->MsgContext;
+    reply.Event           = MPI_EVENT_EVENT_CHANGE;
+    reply.Data[0]         = !!req->Switch;
+
+    mptsas_fix_event_notification_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_message(MPTSASState *s, MPIRequestHeader *req)
+{
+    trace_mptsas_process_message(s, req->Function, req->MsgContext);
+    switch (req->Function) {
+    case MPI_FUNCTION_SCSI_TASK_MGMT:
+        mptsas_process_scsi_task_mgmt(s, (MPIMsgSCSITaskMgmt *)req);
+        break;
+
+    case MPI_FUNCTION_IOC_INIT:
+        mptsas_process_ioc_init(s, (MPIMsgIOCInit *)req);
+        break;
+
+    case MPI_FUNCTION_IOC_FACTS:
+        mptsas_process_ioc_facts(s, (MPIMsgIOCFacts *)req);
+        break;
+
+    case MPI_FUNCTION_PORT_FACTS:
+        mptsas_process_port_facts(s, (MPIMsgPortFacts *)req);
+        break;
+
+    case MPI_FUNCTION_PORT_ENABLE:
+        mptsas_process_port_enable(s, (MPIMsgPortEnable *)req);
+        break;
+
+    case MPI_FUNCTION_EVENT_NOTIFICATION:
+        mptsas_process_event_notification(s, (MPIMsgEventNotify *)req);
+        break;
+
+    case MPI_FUNCTION_CONFIG:
+        mptsas_process_config(s, (MPIMsgConfig *)req);
+        break;
+
+    default:
+        trace_mptsas_unhandled_cmd(s, req->Function, 0);
+        mptsas_set_fault(s, MPI_IOCSTATUS_INVALID_FUNCTION);
+        break;
+    }
+}
+
+static void mptsas_fetch_request(MPTSASState *s)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    char req[MPTSAS_MAX_REQUEST_SIZE];
+    MPIRequestHeader *hdr = (MPIRequestHeader *)req;
+    hwaddr addr;
+    int size;
+
+    if (s->state != MPI_IOC_STATE_OPERATIONAL) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INVALID_STATE);
+        return;
+    }
+
+    /* Read the message header from the guest first. */
+    addr = s->host_mfa_high_addr | MPTSAS_FIFO_GET(s, request_post);
+    pci_dma_read(pci, addr, req, sizeof(hdr));
+
+    if (hdr->Function < ARRAY_SIZE(mpi_request_sizes) &&
+        mpi_request_sizes[hdr->Function]) {
+        /* Read the rest of the request based on the type.  Do not
+         * reread everything, as that could cause a TOC/TOU mismatch
+         * and leak data from the QEMU stack.
+         */
+        size = mpi_request_sizes[hdr->Function];
+        assert(size <= MPTSAS_MAX_REQUEST_SIZE);
+        pci_dma_read(pci, addr + sizeof(hdr), &req[sizeof(hdr)],
+                     size - sizeof(hdr));
+    }
+
+    if (hdr->Function == MPI_FUNCTION_SCSI_IO_REQUEST) {
+        /* SCSI I/O requests are separate from mptsas_process_message
+         * because they cannot be sent through the doorbell yet.
+         */
+        mptsas_process_scsi_io_request(s, (MPIMsgSCSIIORequest *)req, addr);
+    } else {
+        mptsas_process_message(s, (MPIRequestHeader *)req);
+    }
+}
+
+static void mptsas_fetch_requests(void *opaque)
+{
+    MPTSASState *s = opaque;
+
+    while (!MPTSAS_FIFO_EMPTY(s, request_post)) {
+        mptsas_fetch_request(s);
+    }
+}
+
+static void mptsas_soft_reset(MPTSASState *s)
+{
+    uint32_t save_mask;
+
+    trace_mptsas_reset(s);
+
+    /* Temporarily disable interrupts */
+    save_mask = s->intr_mask;
+    s->intr_mask = MPI_HIM_DIM | MPI_HIM_RIM;
+    mptsas_update_interrupt(s);
+
+    qbus_reset_all(&s->bus.qbus);
+    s->intr_status = 0;
+    s->intr_mask = save_mask;
+
+    s->reply_free_tail = 0;
+    s->reply_free_head = 0;
+    s->reply_post_tail = 0;
+    s->reply_post_head = 0;
+    s->request_post_tail = 0;
+    s->request_post_head = 0;
+    qemu_bh_cancel(s->request_bh);
+
+    s->state = MPI_IOC_STATE_READY;
+}
+
+static uint32_t mptsas_doorbell_read(MPTSASState *s)
+{
+    uint32_t ret;
+
+    ret = (s->who_init << MPI_DOORBELL_WHO_INIT_SHIFT) & MPI_DOORBELL_WHO_INIT_MASK;
+    ret |= s->state;
+    switch (s->doorbell_state) {
+    case DOORBELL_NONE:
+        break;
+
+    case DOORBELL_WRITE:
+        ret |= MPI_DOORBELL_ACTIVE;
+        break;
+
+    case DOORBELL_READ:
+        /* Get rid of the IOC fault code.  */
+        ret &= ~MPI_DOORBELL_DATA_MASK;
+
+        assert(s->intr_status & MPI_HIS_DOORBELL_INTERRUPT);
+        assert(s->doorbell_reply_idx <= s->doorbell_reply_size);
+
+        ret |= MPI_DOORBELL_ACTIVE;
+        if (s->doorbell_reply_idx < s->doorbell_reply_size) {
+            /* For more information about this endian switch, see the
+             * commit message for commit 36b62ae ("fw_cfg: fix endianness in
+             * fw_cfg_data_mem_read() / _write()", 2015-01-16).
+             */
+            ret |= le16_to_cpu(s->doorbell_reply[s->doorbell_reply_idx++]);
+        }
+        break;
+
+    default:
+        abort();
+    }
+
+    return ret;
+}
+
+static void mptsas_doorbell_write(MPTSASState *s, uint32_t val)
+{
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        if (s->doorbell_idx < s->doorbell_cnt) {
+            /* For more information about this endian switch, see the
+             * commit message for commit 36b62ae ("fw_cfg: fix endianness in
+             * fw_cfg_data_mem_read() / _write()", 2015-01-16).
+             */
+            s->doorbell_msg[s->doorbell_idx++] = cpu_to_le32(val);
+            if (s->doorbell_idx == s->doorbell_cnt) {
+                mptsas_process_message(s, (MPIRequestHeader *)s->doorbell_msg);
+            }
+        }
+        return;
+    }
+
+    switch ((val & MPI_DOORBELL_FUNCTION_MASK) >> MPI_DOORBELL_FUNCTION_SHIFT) {
+    case MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET:
+        mptsas_soft_reset(s);
+        break;
+    case MPI_FUNCTION_IO_UNIT_RESET:
+        break;
+    case MPI_FUNCTION_HANDSHAKE:
+        s->doorbell_state = DOORBELL_WRITE;
+        s->doorbell_idx = 0;
+        s->doorbell_cnt = (val & MPI_DOORBELL_ADD_DWORDS_MASK)
+            >> MPI_DOORBELL_ADD_DWORDS_SHIFT;
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+        mptsas_update_interrupt(s);
+        break;
+    default:
+        trace_mptsas_unhandled_doorbell_cmd(s, val);
+        break;
+    }
+}
+
+static void mptsas_write_sequence_write(MPTSASState *s, uint32_t val)
+{
+    /* If the diagnostic register is enabled, any write to this register
+     * will disable it.  Otherwise, the guest has to do a magic five-write
+     * sequence.
+     */
+    if (s->diagnostic & MPI_DIAG_DRWE) {
+        goto disable;
+    }
+
+    switch (s->diagnostic_idx) {
+    case 0:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_1ST_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 1:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_2ND_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 2:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_3RD_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 3:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_4TH_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 4:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_5TH_KEY_VALUE) {
+            goto disable;
+        }
+        /* Prepare Spaceball One for departure, and change the
+         * combination on my luggage!
+         */
+        s->diagnostic |= MPI_DIAG_DRWE;
+        break;
+    }
+    s->diagnostic_idx++;
+    return;
+
+disable:
+    s->diagnostic &= ~MPI_DIAG_DRWE;
+    s->diagnostic_idx = 0;
+}
+
+static int mptsas_hard_reset(MPTSASState *s)
+{
+    mptsas_soft_reset(s);
+
+    s->intr_mask = MPI_HIM_DIM | MPI_HIM_RIM;
+
+    s->host_mfa_high_addr = 0;
+    s->sense_buffer_high_addr = 0;
+    s->reply_frame_size = 0;
+    s->max_devices = MPTSAS_NUM_PORTS;
+    s->max_buses = 1;
+
+    return 0;
+}
+
+static void mptsas_interrupt_status_write(MPTSASState *s)
+{
+    switch (s->doorbell_state) {
+    case DOORBELL_NONE:
+    case DOORBELL_WRITE:
+        s->intr_status &= ~MPI_HIS_DOORBELL_INTERRUPT;
+        break;
+
+    case DOORBELL_READ:
+        /* The reply can be read continuously, so leave the interrupt up.  */
+        assert(s->intr_status & MPI_HIS_DOORBELL_INTERRUPT);
+        if (s->doorbell_reply_idx == s->doorbell_reply_size) {
+            s->doorbell_state = DOORBELL_NONE;
+        }
+        break;
+
+    default:
+        abort();
+    }
+    mptsas_update_interrupt(s);
+}
+
+static uint32_t mptsas_reply_post_read(MPTSASState *s)
+{
+    uint32_t ret;
+
+    if (!MPTSAS_FIFO_EMPTY(s, reply_post)) {
+        ret = MPTSAS_FIFO_GET(s, reply_post);
+    } else {
+        ret = -1;
+        s->intr_status &= ~MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+        mptsas_update_interrupt(s);
+    }
+
+    return ret;
+}
+
+static uint64_t mptsas_mmio_read(void *opaque, hwaddr addr,
+                                  unsigned size)
+{
+    MPTSASState *s = opaque;
+    uint32_t ret = 0;
+
+    switch (addr & ~3) {
+    case MPI_DOORBELL_OFFSET:
+        ret = mptsas_doorbell_read(s);
+        break;
+
+    case MPI_DIAGNOSTIC_OFFSET:
+        ret = s->diagnostic;
+        break;
+
+    case MPI_HOST_INTERRUPT_STATUS_OFFSET:
+        ret = s->intr_status;
+        break;
+
+    case MPI_HOST_INTERRUPT_MASK_OFFSET:
+        ret = s->intr_mask;
+        break;
+
+    case MPI_REPLY_POST_FIFO_OFFSET:
+        ret = mptsas_reply_post_read(s);
+        break;
+
+    default:
+        trace_mptsas_mmio_unhandled_read(s, addr);
+        break;
+    }
+    trace_mptsas_mmio_read(s, addr, ret);
+    return ret;
+}
+
+static void mptsas_mmio_write(void *opaque, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    MPTSASState *s = opaque;
+
+    trace_mptsas_mmio_write(s, addr, val);
+    switch (addr) {
+    case MPI_DOORBELL_OFFSET:
+        mptsas_doorbell_write(s, val);
+        break;
+
+    case MPI_WRITE_SEQUENCE_OFFSET:
+        mptsas_write_sequence_write(s, val);
+        break;
+
+    case MPI_DIAGNOSTIC_OFFSET:
+        if (val & MPI_DIAG_RESET_ADAPTER) {
+            mptsas_hard_reset(s);
+        }
+        break;
+
+    case MPI_HOST_INTERRUPT_STATUS_OFFSET:
+        mptsas_interrupt_status_write(s);
+        break;
+
+    case MPI_HOST_INTERRUPT_MASK_OFFSET:
+        s->intr_mask = val & (MPI_HIM_RIM | MPI_HIM_DIM);
+        mptsas_update_interrupt(s);
+        break;
+
+    case MPI_REQUEST_POST_FIFO_OFFSET:
+        if (MPTSAS_FIFO_FULL(s, request_post)) {
+            mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        } else {
+            MPTSAS_FIFO_PUT(s, request_post, val & ~0x03);
+            qemu_bh_schedule(s->request_bh);
+        }
+        break;
+
+    case MPI_REPLY_FREE_FIFO_OFFSET:
+        if (MPTSAS_FIFO_FULL(s, reply_free)) {
+            mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        } else {
+            MPTSAS_FIFO_PUT(s, reply_free, val);
+        }
+        break;
+
+    default:
+        trace_mptsas_mmio_unhandled_write(s, addr, val);
+        break;
+    }
+}
+
+static const MemoryRegionOps mptsas_mmio_ops = {
+    .read = mptsas_mmio_read,
+    .write = mptsas_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static const MemoryRegionOps mptsas_port_ops = {
+    .read = mptsas_mmio_read,
+    .write = mptsas_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static uint64_t mptsas_diag_read(void *opaque, hwaddr addr,
+                                   unsigned size)
+{
+    MPTSASState *s = opaque;
+    trace_mptsas_diag_read(s, addr, 0);
+    return 0;
+}
+
+static void mptsas_diag_write(void *opaque, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    MPTSASState *s = opaque;
+    trace_mptsas_diag_write(s, addr, val);
+}
+
+static const MemoryRegionOps mptsas_diag_ops = {
+    .read = mptsas_diag_read,
+    .write = mptsas_diag_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static QEMUSGList *mptsas_get_sg_list(SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+
+    return &req->qsg;
+}
+
+static void mptsas_command_complete(SCSIRequest *sreq,
+        uint32_t status, size_t resid)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    MPTSASState *s = req->dev;
+    uint8_t sense_buf[SCSI_SENSE_BUF_SIZE];
+    uint8_t sense_len;
+
+    hwaddr sense_buffer_addr = req->dev->sense_buffer_high_addr |
+            req->scsi_io.SenseBufferLowAddr;
+
+    trace_mptsas_command_complete(s, req->scsi_io.MsgContext, status, resid);
+
+    sense_len = scsi_req_get_sense(sreq, sense_buf, SCSI_SENSE_BUF_SIZE);
+    if (sense_len > 0) {
+        pci_dma_write(PCI_DEVICE(s), sense_buffer_addr, sense_buf,
+                      MIN(req->scsi_io.SenseBufferLength, sense_len));
+    }
+
+    if (sreq->status != GOOD || resid ||
+        req->dev->doorbell_state == DOORBELL_WRITE) {
+        MPIMsgSCSIIOReply reply;
+
+        memset(&reply, 0, sizeof(reply));
+        reply.TargetID          = req->scsi_io.TargetID;
+        reply.Bus               = req->scsi_io.Bus;
+        reply.MsgLength         = sizeof(reply) / 4;
+        reply.Function          = req->scsi_io.Function;
+        reply.CDBLength         = req->scsi_io.CDBLength;
+        reply.SenseBufferLength = req->scsi_io.SenseBufferLength;
+        reply.MsgFlags          = req->scsi_io.MsgFlags;
+        reply.MsgContext        = req->scsi_io.MsgContext;
+        reply.SCSIStatus        = sreq->status;
+        if (sreq->status == GOOD) {
+            reply.TransferCount = req->scsi_io.DataLength - resid;
+            if (resid) {
+                reply.IOCStatus     = MPI_IOCSTATUS_SCSI_DATA_UNDERRUN;
+            }
+        } else {
+            reply.SCSIState     = MPI_SCSI_STATE_AUTOSENSE_VALID;
+            reply.SenseCount    = sense_len;
+            reply.IOCStatus     = MPI_IOCSTATUS_SCSI_DATA_UNDERRUN;
+        }
+
+        mptsas_fix_scsi_io_reply_endianness(&reply);
+        mptsas_post_reply(req->dev, (MPIDefaultReply *)&reply);
+    } else {
+        mptsas_turbo_reply(req->dev, req->scsi_io.MsgContext);
+    }
+
+    mptsas_free_request(req);
+}
+
+static void mptsas_request_cancelled(SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    MPIMsgSCSIIOReply reply;
+
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID          = req->scsi_io.TargetID;
+    reply.Bus               = req->scsi_io.Bus;
+    reply.MsgLength         = sizeof(reply) / 4;
+    reply.Function          = req->scsi_io.Function;
+    reply.CDBLength         = req->scsi_io.CDBLength;
+    reply.SenseBufferLength = req->scsi_io.SenseBufferLength;
+    reply.MsgFlags          = req->scsi_io.MsgFlags;
+    reply.MsgContext        = req->scsi_io.MsgContext;
+    reply.SCSIState         = MPI_SCSI_STATE_NO_SCSI_STATUS;
+    reply.IOCStatus         = MPI_IOCSTATUS_SCSI_TASK_TERMINATED;
+
+    mptsas_fix_scsi_io_reply_endianness(&reply);
+    mptsas_post_reply(req->dev, (MPIDefaultReply *)&reply);
+    mptsas_free_request(req);
+}
+
+static void mptsas_save_request(QEMUFile *f, SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    int i;
+
+    qemu_put_buffer(f, (unsigned char *)&req->scsi_io, sizeof(req->scsi_io));
+    qemu_put_be32(f, req->qsg.nsg);
+    for (i = 0; i < req->qsg.nsg; i++) {
+        qemu_put_be64(f, req->qsg.sg[i].base);
+        qemu_put_be64(f, req->qsg.sg[i].len);
+    }
+}
+
+static void *mptsas_load_request(QEMUFile *f, SCSIRequest *sreq)
+{
+    SCSIBus *bus = sreq->bus;
+    MPTSASState *s = container_of(bus, MPTSASState, bus);
+    PCIDevice *pci = PCI_DEVICE(s);
+    MPTSASRequest *req;
+    int i, n;
+
+    req = g_new(MPTSASRequest, 1);
+    qemu_get_buffer(f, (unsigned char *)&req->scsi_io, sizeof(req->scsi_io));
+
+    n = qemu_get_be32(f);
+    /* TODO: add a way for SCSIBusInfo's load_request to fail,
+     * and fail migration instead of asserting here.
+     * When we do, we might be able to re-enable NDEBUG below.
+     */
+#ifdef NDEBUG
+#error building with NDEBUG is not supported
+#endif
+    assert(n >= 0);
+
+    pci_dma_sglist_init(&req->qsg, pci, n);
+    for (i = 0; i < n; i++) {
+        uint64_t base = qemu_get_be64(f);
+        uint64_t len = qemu_get_be64(f);
+        qemu_sglist_add(&req->qsg, base, len);
+    }
+
+    scsi_req_ref(sreq);
+    req->sreq = sreq;
+    req->dev = s;
+
+    return req;
+}
+
+static const struct SCSIBusInfo mptsas_scsi_info = {
+    .tcq = true,
+    .max_target = MPTSAS_NUM_PORTS,
+    .max_lun = 1,
+
+    .get_sg_list = mptsas_get_sg_list,
+    .complete = mptsas_command_complete,
+    .cancel = mptsas_request_cancelled,
+    .save_request = mptsas_save_request,
+    .load_request = mptsas_load_request,
+};
+
+static void mptsas_scsi_init(PCIDevice *dev, Error **errp)
+{
+    DeviceState *d = DEVICE(dev);
+    MPTSASState *s = MPT_SAS(dev);
+
+    dev->config[PCI_LATENCY_TIMER] = 0;
+    dev->config[PCI_INTERRUPT_PIN] = 0x01;
+
+    memory_region_init_io(&s->mmio_io, OBJECT(s), &mptsas_mmio_ops, s,
+                          "mptsas-mmio", 0x4000);
+    memory_region_init_io(&s->port_io, OBJECT(s), &mptsas_port_ops, s,
+                          "mptsas-io", 256);
+    memory_region_init_io(&s->diag_io, OBJECT(s), &mptsas_diag_ops, s,
+                          "mptsas-diag", 0x10000);
+
+    if (s->msi_available &&
+        msi_init(dev, 0, 1, true, false) >= 0) {
+        s->msi_in_use = true;
+    }
+
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->port_io);
+    pci_register_bar(dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY |
+                                 PCI_BASE_ADDRESS_MEM_TYPE_32, &s->mmio_io);
+    pci_register_bar(dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY |
+                                 PCI_BASE_ADDRESS_MEM_TYPE_32, &s->diag_io);
+
+    if (!s->sas_addr) {
+        s->sas_addr = ((NAA_LOCALLY_ASSIGNED_ID << 24) |
+                       IEEE_COMPANY_LOCALLY_ASSIGNED) << 36;
+        s->sas_addr |= (pci_bus_num(dev->bus) << 16);
+        s->sas_addr |= (PCI_SLOT(dev->devfn) << 8);
+        s->sas_addr |= PCI_FUNC(dev->devfn);
+    }
+    s->max_devices = MPTSAS_NUM_PORTS;
+
+    s->request_bh = qemu_bh_new(mptsas_fetch_requests, s);
+
+    QTAILQ_INIT(&s->pending);
+
+    scsi_bus_new(&s->bus, sizeof(s->bus), &dev->qdev, &mptsas_scsi_info, NULL);
+    if (!d->hotplugged) {
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
+    }
+}
+
+static void mptsas_scsi_uninit(PCIDevice *dev)
+{
+    MPTSASState *s = MPT_SAS(dev);
+
+    qemu_bh_delete(s->request_bh);
+    if (s->msi_in_use) {
+        msi_uninit(dev);
+    }
+}
+
+static void mptsas_reset(DeviceState *dev)
+{
+    MPTSASState *s = MPT_SAS(dev);
+
+    mptsas_hard_reset(s);
+}
+
+static int mptsas_post_load(void *opaque, int version_id)
+{
+    MPTSASState *s = opaque;
+
+    if (s->doorbell_idx > s->doorbell_cnt ||
+        s->doorbell_cnt > ARRAY_SIZE(s->doorbell_msg) ||
+        s->doorbell_reply_idx > s->doorbell_reply_size ||
+        s->doorbell_reply_size > ARRAY_SIZE(s->doorbell_reply) ||
+        MPTSAS_FIFO_INVALID(s, request_post) ||
+        MPTSAS_FIFO_INVALID(s, reply_post) ||
+        MPTSAS_FIFO_INVALID(s, reply_free) ||
+        s->diagnostic_idx > 4) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_mptsas = {
+    .name = "mptsas",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .post_load = mptsas_post_load,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, MPTSASState),
+        VMSTATE_BOOL(msi_in_use, MPTSASState),
+
+        VMSTATE_UINT32(state, MPTSASState),
+        VMSTATE_UINT8(who_init, MPTSASState),
+        VMSTATE_UINT8(doorbell_state, MPTSASState),
+        VMSTATE_UINT32_ARRAY(doorbell_msg, MPTSASState, 256),
+        VMSTATE_INT32(doorbell_idx, MPTSASState),
+        VMSTATE_INT32(doorbell_cnt, MPTSASState),
+
+        VMSTATE_UINT16_ARRAY(doorbell_reply, MPTSASState, 256),
+        VMSTATE_INT32(doorbell_reply_idx, MPTSASState),
+        VMSTATE_INT32(doorbell_reply_size, MPTSASState),
+
+        VMSTATE_UINT32(diagnostic, MPTSASState),
+        VMSTATE_UINT8(diagnostic_idx, MPTSASState),
+
+        VMSTATE_UINT32(intr_status, MPTSASState),
+        VMSTATE_UINT32(intr_mask, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(request_post, MPTSASState,
+                             MPTSAS_REQUEST_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(request_post_head, MPTSASState),
+        VMSTATE_UINT16(request_post_tail, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(reply_post, MPTSASState,
+                             MPTSAS_REPLY_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(reply_post_head, MPTSASState),
+        VMSTATE_UINT16(reply_post_tail, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(reply_free, MPTSASState,
+                             MPTSAS_REPLY_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(reply_free_head, MPTSASState),
+        VMSTATE_UINT16(reply_free_tail, MPTSASState),
+
+        VMSTATE_UINT16(max_buses, MPTSASState),
+        VMSTATE_UINT16(max_devices, MPTSASState),
+        VMSTATE_UINT16(reply_frame_size, MPTSASState),
+        VMSTATE_UINT64(host_mfa_high_addr, MPTSASState),
+        VMSTATE_UINT64(sense_buffer_high_addr, MPTSASState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property mptsas_properties[] = {
+    DEFINE_PROP_UINT64("sas_address", MPTSASState, sas_addr, 0),
+    /* TODO: test MSI support under Windows */
+    DEFINE_PROP_BIT("msi", MPTSASState, msi_available, 0, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mptsas1068_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
+
+    pc->realize = mptsas_scsi_init;
+    pc->exit = mptsas_scsi_uninit;
+    pc->romfile = 0;
+    pc->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
+    pc->device_id = PCI_DEVICE_ID_LSI_SAS1068;
+    pc->subsystem_vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
+    pc->subsystem_id = 0x8000;
+    pc->class_id = PCI_CLASS_STORAGE_SCSI;
+    dc->props = mptsas_properties;
+    dc->reset = mptsas_reset;
+    dc->vmsd = &vmstate_mptsas;
+    dc->desc = "LSI SAS 1068";
+}
+
+static const TypeInfo mptsas_info = {
+    .name = TYPE_MPTSAS1068,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(MPTSASState),
+    .class_init = mptsas1068_class_init,
+};
+
+static void mptsas_register_types(void)
+{
+    type_register(&mptsas_info);
+}
+
+type_init(mptsas_register_types)
diff --git a/qemu/hw/scsi/mptsas.h b/qemu/hw/scsi/mptsas.h
new file mode 100644
index 000000000..595f81fb5
--- /dev/null
+++ b/qemu/hw/scsi/mptsas.h
@@ -0,0 +1,100 @@
+#ifndef MPTSAS_H
+#define MPTSAS_H
+
+#include "mpi.h"
+
+#define MPTSAS_NUM_PORTS 8
+#define MPTSAS_MAX_FRAMES 2048     /* Firmware limit at 65535 */
+
+#define MPTSAS_REQUEST_QUEUE_DEPTH 128
+#define MPTSAS_REPLY_QUEUE_DEPTH   128
+
+#define MPTSAS_MAXIMUM_CHAIN_DEPTH 0x22
+
+typedef struct MPTSASState MPTSASState;
+typedef struct MPTSASRequest MPTSASRequest;
+
+enum {
+    DOORBELL_NONE,
+    DOORBELL_WRITE,
+    DOORBELL_READ
+};
+
+struct MPTSASState {
+    PCIDevice dev;
+    MemoryRegion mmio_io;
+    MemoryRegion port_io;
+    MemoryRegion diag_io;
+    QEMUBH *request_bh;
+
+    uint32_t msi_available;
+    uint64_t sas_addr;
+
+    bool msi_in_use;
+
+    /* Doorbell register */
+    uint32_t state;
+    uint8_t who_init;
+    uint8_t doorbell_state;
+
+    /* Buffer for requests that are sent through the doorbell register.  */
+    uint32_t doorbell_msg[256];
+    int doorbell_idx;
+    int doorbell_cnt;
+
+    uint16_t doorbell_reply[256];
+    int doorbell_reply_idx;
+    int doorbell_reply_size;
+
+    /* Other registers */
+    uint8_t diagnostic_idx;
+    uint32_t diagnostic;
+    uint32_t intr_mask;
+    uint32_t intr_status;
+
+    /* Request queues */
+    uint32_t request_post[MPTSAS_REQUEST_QUEUE_DEPTH + 1];
+    uint16_t request_post_head;
+    uint16_t request_post_tail;
+
+    uint32_t reply_post[MPTSAS_REPLY_QUEUE_DEPTH + 1];
+    uint16_t reply_post_head;
+    uint16_t reply_post_tail;
+
+    uint32_t reply_free[MPTSAS_REPLY_QUEUE_DEPTH + 1];
+    uint16_t reply_free_head;
+    uint16_t reply_free_tail;
+
+    /* IOC Facts */
+    hwaddr host_mfa_high_addr;
+    hwaddr sense_buffer_high_addr;
+    uint16_t max_devices;
+    uint16_t max_buses;
+    uint16_t reply_frame_size;
+
+    SCSIBus bus;
+    QTAILQ_HEAD(, MPTSASRequest) pending;
+};
+
+void mptsas_fix_scsi_io_endianness(MPIMsgSCSIIORequest *req);
+void mptsas_fix_scsi_io_reply_endianness(MPIMsgSCSIIOReply *reply);
+void mptsas_fix_scsi_task_mgmt_endianness(MPIMsgSCSITaskMgmt *req);
+void mptsas_fix_scsi_task_mgmt_reply_endianness(MPIMsgSCSITaskMgmtReply *reply);
+void mptsas_fix_ioc_init_endianness(MPIMsgIOCInit *req);
+void mptsas_fix_ioc_init_reply_endianness(MPIMsgIOCInitReply *reply);
+void mptsas_fix_ioc_facts_endianness(MPIMsgIOCFacts *req);
+void mptsas_fix_ioc_facts_reply_endianness(MPIMsgIOCFactsReply *reply);
+void mptsas_fix_config_endianness(MPIMsgConfig *req);
+void mptsas_fix_config_reply_endianness(MPIMsgConfigReply *reply);
+void mptsas_fix_port_facts_endianness(MPIMsgPortFacts *req);
+void mptsas_fix_port_facts_reply_endianness(MPIMsgPortFactsReply *reply);
+void mptsas_fix_port_enable_endianness(MPIMsgPortEnable *req);
+void mptsas_fix_port_enable_reply_endianness(MPIMsgPortEnableReply *reply);
+void mptsas_fix_event_notification_endianness(MPIMsgEventNotify *req);
+void mptsas_fix_event_notification_reply_endianness(MPIMsgEventNotifyReply *reply);
+
+void mptsas_reply(MPTSASState *s, MPIDefaultReply *reply);
+
+void mptsas_process_config(MPTSASState *s, MPIMsgConfig *req);
+
+#endif /* MPTSAS_H */
diff --git a/qemu/hw/scsi/scsi-bus.c b/qemu/hw/scsi/scsi-bus.c
index f0ae4625f..ad6f398c3 100644
--- a/qemu/hw/scsi/scsi-bus.c
+++ b/qemu/hw/scsi/scsi-bus.c
@@ -1,4 +1,6 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -7,6 +9,7 @@
 #include "sysemu/blockdev.h"
 #include "trace.h"
 #include "sysemu/dma.h"
+#include "qemu/cutils.h"
 
 static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
@@ -136,7 +139,8 @@ static void scsi_dma_restart_cb(void *opaque, int running, RunState state)
         return;
     }
     if (!s->bh) {
-        s->bh = qemu_bh_new(scsi_dma_restart_bh, s);
+        AioContext *ctx = blk_get_aio_context(s->conf.blk);
+        s->bh = aio_bh_new(ctx, scsi_dma_restart_bh, s);
         qemu_bh_schedule(s->bh);
     }
 }
@@ -452,7 +456,7 @@ static bool scsi_target_emulate_inquiry(SCSITargetReq *r)
         r->buf[7] = 0x10 | (r->req.bus->info->tcq ? 0x02 : 0); /* Sync, TCQ.  */
         memcpy(&r->buf[8], "QEMU    ", 8);
         memcpy(&r->buf[16], "QEMU TARGET     ", 16);
-        pstrcpy((char *) &r->buf[32], 4, qemu_get_version());
+        pstrcpy((char *) &r->buf[32], 4, qemu_hw_version());
     }
     return true;
 }
@@ -557,7 +561,7 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
     const int memset_off = offsetof(SCSIRequest, sense)
                            + sizeof(req->sense);
 
-    req = g_slice_alloc(reqops->size);
+    req = g_malloc(reqops->size);
     memset((uint8_t *)req + memset_off, 0, reqops->size - memset_off);
     req->refcount = 1;
     req->bus = bus;
@@ -987,7 +991,6 @@ static int scsi_req_xfer(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
         }
         /* fall through */
     case READ_10:
-    case RECOVER_BUFFERED_DATA:
     case READ_12:
     case READ_16:
         cmd->xfer *= dev->blocksize;
@@ -1621,7 +1624,7 @@ void scsi_req_unref(SCSIRequest *req)
         }
         object_unref(OBJECT(req->dev));
         object_unref(OBJECT(qbus->parent));
-        g_slice_free1(req->ops->size, req);
+        g_free(req);
     }
 }
 
@@ -1759,8 +1762,14 @@ void scsi_req_cancel_async(SCSIRequest *req, Notifier *notifier)
         notifier_list_add(&req->cancel_notifiers, notifier);
     }
     if (req->io_canceled) {
+        /* A blk_aio_cancel_async is pending; when it finishes,
+         * scsi_req_cancel_complete will be called and will
+         * call the notifier we just added.  Just wait for that.
+         */
+        assert(req->aiocb);
         return;
     }
+    /* Dropped in scsi_req_cancel_complete.  */
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
@@ -1777,6 +1786,8 @@ void scsi_req_cancel(SCSIRequest *req)
     if (!req->enqueued) {
         return;
     }
+    assert(!req->io_canceled);
+    /* Dropped in scsi_req_cancel_complete.  */
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
@@ -1840,17 +1851,19 @@ void scsi_device_purge_requests(SCSIDevice *sdev, SCSISense sense)
 {
     SCSIRequest *req;
 
+    aio_context_acquire(blk_get_aio_context(sdev->conf.blk));
     while (!QTAILQ_EMPTY(&sdev->requests)) {
         req = QTAILQ_FIRST(&sdev->requests);
-        scsi_req_cancel(req);
+        scsi_req_cancel_async(req, NULL);
     }
-
+    blk_drain(sdev->conf.blk);
+    aio_context_release(blk_get_aio_context(sdev->conf.blk));
     scsi_device_set_ua(sdev, sense);
 }
 
 static char *scsibus_get_dev_path(DeviceState *dev)
 {
-    SCSIDevice *d = DO_UPCAST(SCSIDevice, qdev, dev);
+    SCSIDevice *d = SCSI_DEVICE(dev);
     DeviceState *hba = dev->parent_bus->parent;
     char *id;
     char *path;
@@ -2023,7 +2036,7 @@ static void scsi_device_class_init(ObjectClass *klass, void *data)
 static void scsi_dev_instance_init(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
-    SCSIDevice *s = DO_UPCAST(SCSIDevice, qdev, dev);
+    SCSIDevice *s = SCSI_DEVICE(dev);
 
     device_add_bootindex_property(obj, &s->conf.bootindex,
                                   "bootindex", NULL,
diff --git a/qemu/hw/scsi/scsi-disk.c b/qemu/hw/scsi/scsi-disk.c
index 64f069473..c3ce54a20 100644
--- a/qemu/hw/scsi/scsi-disk.c
+++ b/qemu/hw/scsi/scsi-disk.c
@@ -28,7 +28,8 @@ do { printf("scsi-disk: " fmt , ## __VA_ARGS__); } while (0)
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -37,6 +38,7 @@ do { printf("scsi-disk: " fmt , ## __VA_ARGS__); } while (0)
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "sysemu/dma.h"
+#include "qemu/cutils.h"
 
 #ifdef __linux
 #include <scsi/sg.h>
@@ -76,8 +78,6 @@ struct SCSIDiskState
     bool media_changed;
     bool media_event;
     bool eject_request;
-    uint64_t wwn;
-    uint64_t port_wwn;
     uint16_t port_index;
     uint64_t max_unmap_size;
     uint64_t max_io_size;
@@ -90,7 +90,7 @@ struct SCSIDiskState
     bool tray_locked;
 };
 
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error);
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed);
 
 static void scsi_free_request(SCSIRequest *req)
 {
@@ -169,18 +169,18 @@ static void scsi_aio_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     scsi_req_complete(&r->req, GOOD);
 
 done:
@@ -217,6 +217,8 @@ static void scsi_write_do_fua(SCSIDiskReq *r)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
@@ -235,22 +237,17 @@ done:
     scsi_req_unref(&r->req);
 }
 
-static void scsi_dma_complete_noio(void *opaque, int ret)
+static void scsi_dma_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+    assert(r->req.aiocb == NULL);
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -271,9 +268,17 @@ done:
 static void scsi_dma_complete(void *opaque, int ret)
 {
     SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
     assert(r->req.aiocb != NULL);
-    scsi_dma_complete_noio(opaque, ret);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_dma_complete_noio(r, ret);
 }
 
 static void scsi_read_complete(void * opaque, int ret)
@@ -284,18 +289,18 @@ static void scsi_read_complete(void * opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     DPRINTF("Data ready tag=0x%x len=%zd\n", r->req.tag, r->qiov.size);
 
     n = r->qiov.size / 512;
@@ -308,23 +313,20 @@ done:
 }
 
 /* Actually issue a read to the block device.  */
-static void scsi_do_read(void *opaque, int ret)
+static void scsi_do_read(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = opaque;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint32_t n;
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
+    assert (r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -349,6 +351,22 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_do_read_cb(void *opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_do_read(opaque, ret);
+}
+
 /* Read more data from scsi device into buffer.  */
 static void scsi_read_data(SCSIRequest *req)
 {
@@ -384,7 +402,7 @@ static void scsi_read_data(SCSIRequest *req)
     if (first && scsi_is_cmd_fua(&r->req.cmd)) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
                          BLOCK_ACCT_FLUSH);
-        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read, r);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read_cb, r);
     } else {
         scsi_do_read(r, 0);
     }
@@ -397,14 +415,17 @@ static void scsi_read_data(SCSIRequest *req)
  * scsi_handle_rw_error always manages its reference counts, independent
  * of the return value.
  */
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed)
 {
-    bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
+    bool is_read = (r->req.cmd.mode == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     BlockErrorAction action = blk_get_error_action(s->qdev.conf.blk,
                                                    is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_REPORT) {
+        if (acct_failed) {
+            block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+        }
         switch (error) {
         case ENOMEDIUM:
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
@@ -430,23 +451,19 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
-static void scsi_write_complete(void * opaque, int ret)
+static void scsi_write_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint32_t n;
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
+    assert (r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -467,6 +484,22 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_write_complete(void * opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_write_complete_noio(r, ret);
+}
+
 static void scsi_write_data(SCSIRequest *req)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
@@ -480,18 +513,18 @@ static void scsi_write_data(SCSIRequest *req)
     scsi_req_ref(&r->req);
     if (r->req.cmd.mode != SCSI_XFER_TO_DEV) {
         DPRINTF("Data transfer direction invalid\n");
-        scsi_write_complete(r, -EINVAL);
+        scsi_write_complete_noio(r, -EINVAL);
         return;
     }
 
     if (!r->req.sg && !r->qiov.size) {
         /* Called for the first time.  Ask the driver to send us more data.  */
         r->started = true;
-        scsi_write_complete(r, 0);
+        scsi_write_complete_noio(r, 0);
         return;
     }
     if (s->tray_open) {
-        scsi_write_complete(r, -ENOMEDIUM);
+        scsi_write_complete_noio(r, -ENOMEDIUM);
         return;
     }
 
@@ -500,7 +533,7 @@ static void scsi_write_data(SCSIRequest *req)
         if (r->req.sg) {
             scsi_dma_complete_noio(r, 0);
         } else {
-            scsi_write_complete(r, 0);
+            scsi_write_complete_noio(r, 0);
         }
         return;
     }
@@ -599,21 +632,21 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
             memcpy(outbuf+buflen, str, id_len);
             buflen += id_len;
 
-            if (s->wwn) {
+            if (s->qdev.wwn) {
                 outbuf[buflen++] = 0x1; // Binary
                 outbuf[buflen++] = 0x3; // NAA
                 outbuf[buflen++] = 0;   // reserved
                 outbuf[buflen++] = 8;
-                stq_be_p(&outbuf[buflen], s->wwn);
+                stq_be_p(&outbuf[buflen], s->qdev.wwn);
                 buflen += 8;
             }
 
-            if (s->port_wwn) {
+            if (s->qdev.port_wwn) {
                 outbuf[buflen++] = 0x61; // SAS / Binary
                 outbuf[buflen++] = 0x93; // PIV / Target port / NAA
                 outbuf[buflen++] = 0;    // reserved
                 outbuf[buflen++] = 8;
-                stq_be_p(&outbuf[buflen], s->port_wwn);
+                stq_be_p(&outbuf[buflen], s->qdev.port_wwn);
                 buflen += 8;
             }
 
@@ -1557,22 +1590,24 @@ typedef struct UnmapCBData {
     int count;
 } UnmapCBData;
 
-static void scsi_unmap_complete(void *opaque, int ret)
+static void scsi_unmap_complete(void *opaque, int ret);
+
+static void scsi_unmap_complete_noio(UnmapCBData *data, int ret)
 {
-    UnmapCBData *data = opaque;
     SCSIDiskReq *r = data->r;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint64_t sector_num;
     uint32_t nb_sectors;
 
-    r->req.aiocb = NULL;
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, false)) {
             goto done;
         }
     }
@@ -1601,6 +1636,17 @@ done:
     g_free(data);
 }
 
+static void scsi_unmap_complete(void *opaque, int ret)
+{
+    UnmapCBData *data = opaque;
+    SCSIDiskReq *r = data->r;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    scsi_unmap_complete_noio(data, ret);
+}
+
 static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
@@ -1638,7 +1684,7 @@ static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 
     /* The matching unref is in scsi_unmap_complete, before data is freed.  */
     scsi_req_ref(&r->req);
-    scsi_unmap_complete(data, 0);
+    scsi_unmap_complete_noio(data, 0);
     return;
 
 invalid_param_len:
@@ -1665,24 +1711,29 @@ static void scsi_write_same_complete(void *opaque, int ret)
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
     }
 
     if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
+        if (scsi_handle_rw_error(r, -ret, true)) {
             goto done;
         }
     }
 
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+
     data->nb_sectors -= data->iov.iov_len / 512;
     data->sector += data->iov.iov_len / 512;
     data->iov.iov_len = MIN(data->nb_sectors * 512, data->iov.iov_len);
     if (data->iov.iov_len) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
                          data->iov.iov_len, BLOCK_ACCT_WRITE);
+        /* blk_aio_write doesn't like the qiov size being different from
+         * nb_sectors, make sure they match.
+         */
+        qemu_iovec_init_external(&data->qiov, &data->iov, 1);
         r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, data->sector,
                                       &data->qiov, data->iov.iov_len / 512,
                                       scsi_write_same_complete, data);
@@ -2280,7 +2331,7 @@ static void scsi_realize(SCSIDevice *dev, Error **errp)
     }
 
     if (!s->version) {
-        s->version = g_strdup(qemu_get_version());
+        s->version = g_strdup(qemu_hw_version());
     }
     if (!s->vendor) {
         s->vendor = g_strdup("QEMU");
@@ -2523,6 +2574,7 @@ static void scsi_block_realize(SCSIDevice *dev, Error **errp)
     s->features |= (1 << SCSI_DISK_F_NO_REMOVABLE_DEVOPS);
 
     scsi_realize(&s->qdev, errp);
+    scsi_generic_read_device_identification(&s->qdev);
 }
 
 static bool scsi_block_is_passthrough(SCSIDiskState *s, uint8_t *buf)
@@ -2616,8 +2668,8 @@ static Property scsi_hd_properties[] = {
                     SCSI_DISK_F_REMOVABLE, false),
     DEFINE_PROP_BIT("dpofua", SCSIDiskState, features,
                     SCSI_DISK_F_DPOFUA, false),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
     DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
                        DEFAULT_MAX_UNMAP_SIZE),
@@ -2666,8 +2718,8 @@ static const TypeInfo scsi_hd_info = {
 
 static Property scsi_cd_properties[] = {
     DEFINE_SCSI_DISK_PROPERTIES(),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
     DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
                        DEFAULT_MAX_IO_SIZE),
@@ -2731,8 +2783,8 @@ static Property scsi_disk_properties[] = {
                     SCSI_DISK_F_REMOVABLE, false),
     DEFINE_PROP_BIT("dpofua", SCSIDiskState, features,
                     SCSI_DISK_F_DPOFUA, false),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
     DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
                        DEFAULT_MAX_UNMAP_SIZE),
diff --git a/qemu/hw/scsi/scsi-generic.c b/qemu/hw/scsi/scsi-generic.c
index e53470f85..7459465f6 100644
--- a/qemu/hw/scsi/scsi-generic.c
+++ b/qemu/hw/scsi/scsi-generic.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
@@ -31,10 +33,6 @@ do { printf("scsi-generic: " fmt , ## __VA_ARGS__); } while (0)
 #define BADF(fmt, ...) \
 do { fprintf(stderr, "scsi-generic: " fmt , ## __VA_ARGS__); } while (0)
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
 #include <scsi/sg.h>
 #include "block/scsi.h"
 
@@ -88,12 +86,12 @@ static void scsi_free_request(SCSIRequest *req)
 }
 
 /* Helper function for command completion.  */
-static void scsi_command_complete(void *opaque, int ret)
+static void scsi_command_complete_noio(SCSIGenericReq *r, int ret)
 {
     int status;
-    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
 
-    r->req.aiocb = NULL;
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
@@ -142,6 +140,15 @@ done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_command_complete(void *opaque, int ret)
+{
+    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+    scsi_command_complete_noio(r, ret);
+}
+
 static int execute_command(BlockBackend *blk,
                            SCSIGenericReq *r, int direction,
                            BlockCompletionFunc *complete)
@@ -172,33 +179,51 @@ static void scsi_read_complete(void * opaque, int ret)
     SCSIDevice *s = r->req.dev;
     int len;
 
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
+
     len = r->io_header.dxfer_len - r->io_header.resid;
     DPRINTF("Data ready tag=0x%x len=%d\n", r->req.tag, len);
 
     r->len = -1;
     if (len == 0) {
-        scsi_command_complete(r, 0);
-    } else {
-        /* Snoop READ CAPACITY output to set the blocksize.  */
-        if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
-            (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
-            s->blocksize = ldl_be_p(&r->buf[4]);
-            s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
-        } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
-                   (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
-            s->blocksize = ldl_be_p(&r->buf[8]);
-            s->max_lba = ldq_be_p(&r->buf[0]);
-        }
-        blk_set_guest_block_size(s->conf.blk, s->blocksize);
+        scsi_command_complete_noio(r, 0);
+        return;
+    }
 
-        scsi_req_data(&r->req, len);
-        scsi_req_unref(&r->req);
+    /* Snoop READ CAPACITY output to set the blocksize.  */
+    if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
+        (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
+        s->blocksize = ldl_be_p(&r->buf[4]);
+        s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
+    } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
+               (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
+        s->blocksize = ldl_be_p(&r->buf[8]);
+        s->max_lba = ldq_be_p(&r->buf[0]);
+    }
+    blk_set_guest_block_size(s->conf.blk, s->blocksize);
+
+    /* Patch MODE SENSE device specific parameters if the BDS is opened
+     * readonly.
+     */
+    if ((s->type == TYPE_DISK || s->type == TYPE_TAPE) &&
+        blk_is_read_only(s->conf.blk) &&
+        (r->req.cmd.buf[0] == MODE_SENSE ||
+         r->req.cmd.buf[0] == MODE_SENSE_10) &&
+        (r->req.cmd.buf[1] & 0x8) == 0) {
+        if (r->req.cmd.buf[0] == MODE_SENSE) {
+            r->buf[2] |= 0x80;
+        } else  {
+            r->buf[3] |= 0x80;
+        }
     }
+    scsi_req_data(&r->req, len);
+    scsi_req_unref(&r->req);
 }
 
 /* Read more data from scsi device into buffer.  */
@@ -213,14 +238,14 @@ static void scsi_read_data(SCSIRequest *req)
     /* The request is used as the AIO opaque value, so add a ref.  */
     scsi_req_ref(&r->req);
     if (r->len == -1) {
-        scsi_command_complete(r, 0);
+        scsi_command_complete_noio(r, 0);
         return;
     }
 
     ret = execute_command(s->conf.blk, r, SG_DXFER_FROM_DEV,
                           scsi_read_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -230,9 +255,12 @@ static void scsi_write_complete(void * opaque, int ret)
     SCSIDevice *s = r->req.dev;
 
     DPRINTF("scsi_write_complete() ret = %d\n", ret);
+
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
 
@@ -242,7 +270,7 @@ static void scsi_write_complete(void * opaque, int ret)
         DPRINTF("block size %d\n", s->blocksize);
     }
 
-    scsi_command_complete(r, ret);
+    scsi_command_complete_noio(r, ret);
 }
 
 /* Write data to a scsi device.  Returns nonzero on failure.
@@ -264,7 +292,7 @@ static void scsi_write_data(SCSIRequest *req)
     scsi_req_ref(&r->req);
     ret = execute_command(s->conf.blk, r, SG_DXFER_TO_DEV, scsi_write_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -306,7 +334,7 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
         ret = execute_command(s->conf.blk, r, SG_DXFER_NONE,
                               scsi_command_complete);
         if (ret < 0) {
-            scsi_command_complete(r, ret);
+            scsi_command_complete_noio(r, ret);
             return 0;
         }
         return 0;
@@ -328,6 +356,96 @@ static int32_t scsi_send_command(SCSIRequest *req, uint8_t *cmd)
     }
 }
 
+static int read_naa_id(const uint8_t *p, uint64_t *p_wwn)
+{
+    int i;
+
+    if ((p[1] & 0xF) == 3) {
+        /* NAA designator type */
+        if (p[3] != 8) {
+            return -EINVAL;
+        }
+        *p_wwn = ldq_be_p(p + 4);
+        return 0;
+    }
+
+    if ((p[1] & 0xF) == 8) {
+        /* SCSI name string designator type */
+        if (p[3] < 20 || memcmp(&p[4], "naa.", 4)) {
+            return -EINVAL;
+        }
+        if (p[3] > 20 && p[24] != ',') {
+            return -EINVAL;
+        }
+        *p_wwn = 0;
+        for (i = 8; i < 24; i++) {
+            char c = toupper(p[i]);
+            c -= (c >= '0' && c <= '9' ? '0' : 'A' - 10);
+            *p_wwn = (*p_wwn << 4) | c;
+        }
+        return 0;
+    }
+
+    return -EINVAL;
+}
+
+void scsi_generic_read_device_identification(SCSIDevice *s)
+{
+    uint8_t cmd[6];
+    uint8_t buf[250];
+    uint8_t sensebuf[8];
+    sg_io_hdr_t io_header;
+    int ret;
+    int i, len;
+
+    memset(cmd, 0, sizeof(cmd));
+    memset(buf, 0, sizeof(buf));
+    cmd[0] = INQUIRY;
+    cmd[1] = 1;
+    cmd[2] = 0x83;
+    cmd[4] = sizeof(buf);
+
+    memset(&io_header, 0, sizeof(io_header));
+    io_header.interface_id = 'S';
+    io_header.dxfer_direction = SG_DXFER_FROM_DEV;
+    io_header.dxfer_len = sizeof(buf);
+    io_header.dxferp = buf;
+    io_header.cmdp = cmd;
+    io_header.cmd_len = sizeof(cmd);
+    io_header.mx_sb_len = sizeof(sensebuf);
+    io_header.sbp = sensebuf;
+    io_header.timeout = 6000; /* XXX */
+
+    ret = blk_ioctl(s->conf.blk, SG_IO, &io_header);
+    if (ret < 0 || io_header.driver_status || io_header.host_status) {
+        return;
+    }
+
+    len = MIN((buf[2] << 8) | buf[3], sizeof(buf) - 4);
+    for (i = 0; i + 3 <= len; ) {
+        const uint8_t *p = &buf[i + 4];
+        uint64_t wwn;
+
+        if (i + (p[3] + 4) > len) {
+            break;
+        }
+
+        if ((p[1] & 0x10) == 0) {
+            /* Associated with the logical unit */
+            if (read_naa_id(p, &wwn) == 0) {
+                s->wwn = wwn;
+            }
+        } else if ((p[1] & 0x10) == 0x10) {
+            /* Associated with the target port */
+            if (read_naa_id(p, &wwn) == 0) {
+                s->port_wwn = wwn;
+            }
+        }
+
+        i += p[3] + 4;
+    }
+}
+
 static int get_stream_blocksize(BlockBackend *blk)
 {
     uint8_t cmd[6];
@@ -431,6 +549,8 @@ static void scsi_generic_realize(SCSIDevice *s, Error **errp)
     }
 
     DPRINTF("block size %d\n", s->blocksize);
+
+    scsi_generic_read_device_identification(s);
 }
 
 const SCSIReqOps scsi_generic_req_ops = {
diff --git a/qemu/hw/scsi/spapr_vscsi.c b/qemu/hw/scsi/spapr_vscsi.c
index 891424fae..b00edf7fd 100644
--- a/qemu/hw/scsi/spapr_vscsi.c
+++ b/qemu/hw/scsi/spapr_vscsi.c
@@ -31,6 +31,9 @@
  *  - Add indirect descriptors support
  *  - Maybe do autosense (PAPR seems to mandate it, linux doesn't care)
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -750,7 +753,6 @@ static void vscsi_report_luns(VSCSIState *s, vscsi_req *req)
     len = n+8;
 
     resp_data = g_malloc0(len);
-    memset(resp_data, 0, len);
     stl_be_p(resp_data, n);
     i = found_lun0 ? 8 : 16;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
diff --git a/qemu/hw/scsi/vhost-scsi.c b/qemu/hw/scsi/vhost-scsi.c
index a69918bef..9261d51da 100644
--- a/qemu/hw/scsi/vhost-scsi.c
+++ b/qemu/hw/scsi/vhost-scsi.c
@@ -14,8 +14,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
-#include "config.h"
 #include "qemu/error-report.h"
 #include "qemu/queue.h"
 #include "monitor/monitor.h"
@@ -26,6 +27,8 @@
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 #include "hw/fw-path-provider.h"
+#include "linux/vhost.h"
+#include "qemu/cutils.h"
 
 /* Features supported by host kernel. */
 static const int kernel_feature_bits[] = {
@@ -45,7 +48,7 @@ static int vhost_scsi_set_endpoint(VHostSCSI *s)
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    ret = vhost_ops->vhost_call(&s->dev, VHOST_SCSI_SET_ENDPOINT, &backend);
+    ret = vhost_ops->vhost_scsi_set_endpoint(&s->dev, &backend);
     if (ret < 0) {
         return -errno;
     }
@@ -60,7 +63,7 @@ static void vhost_scsi_clear_endpoint(VHostSCSI *s)
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    vhost_ops->vhost_call(&s->dev, VHOST_SCSI_CLEAR_ENDPOINT, &backend);
+    vhost_ops->vhost_scsi_clear_endpoint(&s->dev, &backend);
 }
 
 static int vhost_scsi_start(VHostSCSI *s)
@@ -76,8 +79,7 @@ static int vhost_scsi_start(VHostSCSI *s)
         return -ENOSYS;
     }
 
-    ret = vhost_ops->vhost_call(&s->dev,
-                                VHOST_SCSI_GET_ABI_VERSION, &abi_version);
+    ret = vhost_ops->vhost_scsi_get_abi_version(&s->dev, &abi_version);
     if (ret < 0) {
         return -errno;
     }
@@ -118,7 +120,7 @@ static int vhost_scsi_start(VHostSCSI *s)
      * enabling/disabling irqfd.
      */
     for (i = 0; i < s->dev.nvqs; i++) {
-        vhost_virtqueue_mask(&s->dev, vdev, i, false);
+        vhost_virtqueue_mask(&s->dev, vdev, s->dev.vq_index + i, false);
     }
 
     return ret;
@@ -217,11 +219,9 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp)
     }
 
     if (vs->conf.vhostfd) {
-        vhostfd = monitor_fd_param(cur_mon, vs->conf.vhostfd, &err);
+        vhostfd = monitor_fd_param(cur_mon, vs->conf.vhostfd, errp);
         if (vhostfd == -1) {
-            error_setg(errp, "vhost-scsi: unable to parse vhostfd: %s",
-                       error_get_pretty(err));
-            error_free(err);
+            error_prepend(errp, "vhost-scsi: unable to parse vhostfd: ");
             return;
         }
     } else {
@@ -277,6 +277,7 @@ static void vhost_scsi_unrealize(DeviceState *dev, Error **errp)
     /* This will stop vhost backend. */
     vhost_scsi_set_status(vdev, 0);
 
+    vhost_dev_cleanup(&s->dev);
     g_free(s->dev.vqs);
 
     virtio_scsi_common_unrealize(dev, errp);
@@ -291,7 +292,7 @@ static char *vhost_scsi_get_fw_dev_path(FWPathProvider *p, BusState *bus,
 {
     VHostSCSI *s = VHOST_SCSI(dev);
     /* format: channel@channel/vhost-scsi@target,lun */
-    return g_strdup_printf("channel@%x/%s@%x,%x", s->channel,
+    return g_strdup_printf("/channel@%x/%s@%x,%x", s->channel,
                            qdev_fw_name(dev), s->target, s->lun);
 }
 
diff --git a/qemu/hw/scsi/virtio-scsi-dataplane.c b/qemu/hw/scsi/virtio-scsi-dataplane.c
index 5575648a9..1a49f1e4b 100644
--- a/qemu/hw/scsi/virtio-scsi-dataplane.c
+++ b/qemu/hw/scsi/virtio-scsi-dataplane.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
@@ -18,7 +19,6 @@
 #include <block/scsi.h>
 #include <hw/virtio/virtio-bus.h>
 #include "hw/virtio/virtio-access.h"
-#include "stdio.h"
 
 /* Context: QEMU global mutex held */
 void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread)
@@ -38,120 +38,57 @@ void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread)
     }
 }
 
-static VirtIOSCSIVring *virtio_scsi_vring_init(VirtIOSCSI *s,
-                                               VirtQueue *vq,
-                                               EventNotifierHandler *handler,
-                                               int n)
+static void virtio_scsi_data_plane_handle_cmd(VirtIODevice *vdev,
+                                              VirtQueue *vq)
 {
-    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
-    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
-    VirtIOSCSIVring *r;
-    int rc;
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
 
-    /* Set up virtqueue notify */
-    rc = k->set_host_notifier(qbus->parent, n, true);
-    if (rc != 0) {
-        fprintf(stderr, "virtio-scsi: Failed to set host notifier (%d)\n",
-                rc);
-        s->dataplane_fenced = true;
-        return NULL;
-    }
-
-    r = g_slice_new(VirtIOSCSIVring);
-    r->host_notifier = *virtio_queue_get_host_notifier(vq);
-    r->guest_notifier = *virtio_queue_get_guest_notifier(vq);
-    aio_set_event_notifier(s->ctx, &r->host_notifier, handler);
-
-    r->parent = s;
-
-    if (!vring_setup(&r->vring, VIRTIO_DEVICE(s), n)) {
-        fprintf(stderr, "virtio-scsi: VRing setup failed\n");
-        goto fail_vring;
-    }
-    return r;
-
-fail_vring:
-    aio_set_event_notifier(s->ctx, &r->host_notifier, NULL);
-    k->set_host_notifier(qbus->parent, n, false);
-    g_slice_free(VirtIOSCSIVring, r);
-    return NULL;
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_cmd_vq(s, vq);
 }
 
-VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
-                                         VirtIOSCSIVring *vring)
+static void virtio_scsi_data_plane_handle_ctrl(VirtIODevice *vdev,
+                                               VirtQueue *vq)
 {
-    VirtIOSCSIReq *req = virtio_scsi_init_req(s, NULL);
-    int r;
-
-    req->vring = vring;
-    r = vring_pop((VirtIODevice *)s, &vring->vring, &req->elem);
-    if (r < 0) {
-        virtio_scsi_free_req(req);
-        req = NULL;
-    }
-    return req;
-}
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
 
-void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req)
-{
-    VirtIODevice *vdev = VIRTIO_DEVICE(req->vring->parent);
-
-    vring_push(vdev, &req->vring->vring, &req->elem,
-               req->qsgl.size + req->resp_iov.size);
-
-    if (vring_should_notify(vdev, &req->vring->vring)) {
-        event_notifier_set(&req->vring->guest_notifier);
-    }
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_ctrl_vq(s, vq);
 }
 
-static void virtio_scsi_iothread_handle_ctrl(EventNotifier *notifier)
+static void virtio_scsi_data_plane_handle_event(VirtIODevice *vdev,
+                                                VirtQueue *vq)
 {
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = VIRTIO_SCSI(vring->parent);
-    VirtIOSCSIReq *req;
-
-    event_notifier_test_and_clear(notifier);
-    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
-        virtio_scsi_handle_ctrl_req(s, req);
-    }
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_event_vq(s, vq);
 }
 
-static void virtio_scsi_iothread_handle_event(EventNotifier *notifier)
+static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue *vq, int n,
+                                  void (*fn)(VirtIODevice *vdev, VirtQueue *vq))
 {
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = vring->parent;
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
-
-    event_notifier_test_and_clear(notifier);
+    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+    int rc;
 
-    if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
-        return;
+    /* Set up virtqueue notify */
+    rc = k->set_host_notifier(qbus->parent, n, true);
+    if (rc != 0) {
+        fprintf(stderr, "virtio-scsi: Failed to set host notifier (%d)\n",
+                rc);
+        s->dataplane_fenced = true;
+        return rc;
     }
 
-    if (s->events_dropped) {
-        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
-    }
+    virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, fn);
+    return 0;
 }
 
-static void virtio_scsi_iothread_handle_cmd(EventNotifier *notifier)
+void virtio_scsi_dataplane_notify(VirtIODevice *vdev, VirtIOSCSIReq *req)
 {
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = (VirtIOSCSI *)vring->parent;
-    VirtIOSCSIReq *req, *next;
-    QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
-
-    event_notifier_test_and_clear(notifier);
-    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
-        if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
-            QTAILQ_INSERT_TAIL(&reqs, req, next);
-        }
-    }
-
-    QTAILQ_FOREACH_SAFE(req, &reqs, next, next) {
-        virtio_scsi_handle_cmd_req_submit(s, req);
+    if (virtio_should_notify(vdev, req->vq)) {
+        event_notifier_set(virtio_queue_get_guest_notifier(req->vq));
     }
 }
 
@@ -161,43 +98,10 @@ static void virtio_scsi_clear_aio(VirtIOSCSI *s)
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
     int i;
 
-    if (s->ctrl_vring) {
-        aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
-    }
-    if (s->event_vring) {
-        aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
-    }
-    if (s->cmd_vrings) {
-        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
-            aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
-        }
-    }
-}
-
-static void virtio_scsi_vring_teardown(VirtIOSCSI *s)
-{
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
-    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
-    int i;
-
-    if (s->ctrl_vring) {
-        vring_teardown(&s->ctrl_vring->vring, vdev, 0);
-        g_slice_free(VirtIOSCSIVring, s->ctrl_vring);
-        s->ctrl_vring = NULL;
-    }
-    if (s->event_vring) {
-        vring_teardown(&s->event_vring->vring, vdev, 1);
-        g_slice_free(VirtIOSCSIVring, s->event_vring);
-        s->event_vring = NULL;
-    }
-    if (s->cmd_vrings) {
-        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
-            vring_teardown(&s->cmd_vrings[i]->vring, vdev, 2 + i);
-            g_slice_free(VirtIOSCSIVring, s->cmd_vrings[i]);
-            s->cmd_vrings[i] = NULL;
-        }
-        free(s->cmd_vrings);
-        s->cmd_vrings = NULL;
+    virtio_queue_aio_set_host_notifier_handler(vs->ctrl_vq, s->ctx, NULL);
+    virtio_queue_aio_set_host_notifier_handler(vs->event_vq, s->ctx, NULL);
+    for (i = 0; i < vs->conf.num_queues; i++) {
+        virtio_queue_aio_set_host_notifier_handler(vs->cmd_vqs[i], s->ctx, NULL);
     }
 }
 
@@ -224,30 +128,24 @@ void virtio_scsi_dataplane_start(VirtIOSCSI *s)
     if (rc != 0) {
         fprintf(stderr, "virtio-scsi: Failed to set guest notifiers (%d), "
                 "ensure -enable-kvm is set\n", rc);
-        s->dataplane_fenced = true;
         goto fail_guest_notifiers;
     }
 
     aio_context_acquire(s->ctx);
-    s->ctrl_vring = virtio_scsi_vring_init(s, vs->ctrl_vq,
-                                           virtio_scsi_iothread_handle_ctrl,
-                                           0);
-    if (!s->ctrl_vring) {
+    rc = virtio_scsi_vring_init(s, vs->ctrl_vq, 0,
+                                virtio_scsi_data_plane_handle_ctrl);
+    if (rc) {
         goto fail_vrings;
     }
-    s->event_vring = virtio_scsi_vring_init(s, vs->event_vq,
-                                            virtio_scsi_iothread_handle_event,
-                                            1);
-    if (!s->event_vring) {
+    rc = virtio_scsi_vring_init(s, vs->event_vq, 1,
+                                virtio_scsi_data_plane_handle_event);
+    if (rc) {
         goto fail_vrings;
     }
-    s->cmd_vrings = g_new(VirtIOSCSIVring *, vs->conf.num_queues);
     for (i = 0; i < vs->conf.num_queues; i++) {
-        s->cmd_vrings[i] =
-            virtio_scsi_vring_init(s, vs->cmd_vqs[i],
-                                   virtio_scsi_iothread_handle_cmd,
-                                   i + 2);
-        if (!s->cmd_vrings[i]) {
+        rc = virtio_scsi_vring_init(s, vs->cmd_vqs[i], i + 2,
+                                    virtio_scsi_data_plane_handle_cmd);
+        if (rc) {
             goto fail_vrings;
         }
     }
@@ -260,13 +158,14 @@ void virtio_scsi_dataplane_start(VirtIOSCSI *s)
 fail_vrings:
     virtio_scsi_clear_aio(s);
     aio_context_release(s->ctx);
-    virtio_scsi_vring_teardown(s);
     for (i = 0; i < vs->conf.num_queues + 2; i++) {
         k->set_host_notifier(qbus->parent, i, false);
     }
     k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, false);
 fail_guest_notifiers:
+    s->dataplane_fenced = true;
     s->dataplane_starting = false;
+    s->dataplane_started = true;
 }
 
 /* Context: QEMU global mutex held */
@@ -277,12 +176,14 @@ void virtio_scsi_dataplane_stop(VirtIOSCSI *s)
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
     int i;
 
+    if (!s->dataplane_started || s->dataplane_stopping) {
+        return;
+    }
+
     /* Better luck next time. */
     if (s->dataplane_fenced) {
         s->dataplane_fenced = false;
-        return;
-    }
-    if (!s->dataplane_started || s->dataplane_stopping) {
+        s->dataplane_started = false;
         return;
     }
     s->dataplane_stopping = true;
@@ -290,21 +191,12 @@ void virtio_scsi_dataplane_stop(VirtIOSCSI *s)
 
     aio_context_acquire(s->ctx);
 
-    aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
-    aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
-    for (i = 0; i < vs->conf.num_queues; i++) {
-        aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
-    }
+    virtio_scsi_clear_aio(s);
 
     blk_drain_all(); /* ensure there are no in-flight requests */
 
     aio_context_release(s->ctx);
 
-    /* Sync vring state back to virtqueue so that non-dataplane request
-     * processing can continue when we disable the host notifier below.
-     */
-    virtio_scsi_vring_teardown(s);
-
     for (i = 0; i < vs->conf.num_queues + 2; i++) {
         k->set_host_notifier(qbus->parent, i, false);
     }
diff --git a/qemu/hw/scsi/virtio-scsi.c b/qemu/hw/scsi/virtio-scsi.c
index 811c3da8b..30415c6a9 100644
--- a/qemu/hw/scsi/virtio-scsi.c
+++ b/qemu/hw/scsi/virtio-scsi.c
@@ -13,6 +13,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "standard-headers/linux/virtio_ids.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "qemu/error-report.h"
@@ -22,7 +24,6 @@
 #include <block/scsi.h>
 #include <hw/virtio/virtio-bus.h>
 #include "hw/virtio/virtio-access.h"
-#include "migration/migration.h"
 
 static inline int virtio_scsi_get_lun(uint8_t *lun)
 {
@@ -40,29 +41,23 @@ static inline SCSIDevice *virtio_scsi_device_find(VirtIOSCSI *s, uint8_t *lun)
     return scsi_device_find(&s->bus, 0, lun[1], virtio_scsi_get_lun(lun));
 }
 
-VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
+void virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq, VirtIOSCSIReq *req)
 {
-    VirtIOSCSIReq *req;
-    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)s;
-    const size_t zero_skip = offsetof(VirtIOSCSIReq, elem)
-                             + sizeof(VirtQueueElement);
+    const size_t zero_skip =
+        offsetof(VirtIOSCSIReq, resp_iov) + sizeof(req->resp_iov);
 
-    req = g_slice_alloc(sizeof(*req) + vs->cdb_size);
     req->vq = vq;
     req->dev = s;
     qemu_sglist_init(&req->qsgl, DEVICE(s), 8, &address_space_memory);
     qemu_iovec_init(&req->resp_iov, 1);
     memset((uint8_t *)req + zero_skip, 0, sizeof(*req) - zero_skip);
-    return req;
 }
 
 void virtio_scsi_free_req(VirtIOSCSIReq *req)
 {
-    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)req->dev;
-
     qemu_iovec_destroy(&req->resp_iov);
     qemu_sglist_destroy(&req->qsgl);
-    g_slice_free1(sizeof(*req) + vs->cdb_size, req);
+    g_free(req);
 }
 
 static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
@@ -72,11 +67,10 @@ static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
     qemu_iovec_from_buf(&req->resp_iov, 0, &req->resp, req->resp_size);
-    if (req->vring) {
-        assert(req->vq == NULL);
-        virtio_scsi_vring_push_notify(req);
+    virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
+    if (s->dataplane_started && !s->dataplane_fenced) {
+        virtio_scsi_dataplane_notify(vdev, req);
     } else {
-        virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
         virtio_notify(vdev, vq);
     }
 
@@ -145,7 +139,7 @@ static int virtio_scsi_parse_req(VirtIOSCSIReq *req,
      *
      * TODO: always disable this workaround for virtio 1.0 devices.
      */
-    if (!virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_ANY_LAYOUT)) {
         if (req->elem.out_num) {
             req_size = req->elem.out_sg[0].iov_len;
         }
@@ -176,11 +170,14 @@ static int virtio_scsi_parse_req(VirtIOSCSIReq *req,
 
 static VirtIOSCSIReq *virtio_scsi_pop_req(VirtIOSCSI *s, VirtQueue *vq)
 {
-    VirtIOSCSIReq *req = virtio_scsi_init_req(s, vq);
-    if (!virtqueue_pop(vq, &req->elem)) {
-        virtio_scsi_free_req(req);
+    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)s;
+    VirtIOSCSIReq *req;
+
+    req = virtqueue_pop(vq, sizeof(VirtIOSCSIReq) + vs->cdb_size);
+    if (!req) {
         return NULL;
     }
+    virtio_scsi_init_req(s, vq, req);
     return req;
 }
 
@@ -192,7 +189,7 @@ static void virtio_scsi_save_request(QEMUFile *f, SCSIRequest *sreq)
 
     assert(n < vs->conf.num_queues);
     qemu_put_be32s(f, &n);
-    qemu_put_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
+    qemu_put_virtqueue_element(f, &req->elem);
 }
 
 static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
@@ -205,17 +202,8 @@ static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
 
     qemu_get_be32s(f, &n);
     assert(n < vs->conf.num_queues);
-    req = virtio_scsi_init_req(s, vs->cmd_vqs[n]);
-    qemu_get_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
-    /* TODO: add a way for SCSIBusInfo's load_request to fail,
-     * and fail migration instead of asserting here.
-     * When we do, we might be able to re-enable NDEBUG below.
-     */
-#ifdef NDEBUG
-#error building with NDEBUG is not supported
-#endif
-    assert(req->elem.in_num <= ARRAY_SIZE(req->elem.in_sg));
-    assert(req->elem.out_num <= ARRAY_SIZE(req->elem.out_sg));
+    req = qemu_get_virtqueue_element(f, sizeof(VirtIOSCSIReq) + vs->cdb_size);
+    virtio_scsi_init_req(s, vs->cmd_vqs[n], req);
 
     if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICmdReq) + vs->cdb_size,
                               sizeof(VirtIOSCSICmdResp) + vs->sense_size) < 0) {
@@ -245,7 +233,7 @@ static void virtio_scsi_cancel_notify(Notifier *notifier, void *data)
     if (--n->tmf_req->remaining == 0) {
         virtio_scsi_complete_req(n->tmf_req);
     }
-    g_slice_free(VirtIOSCSICancelNotifier, n);
+    g_free(n);
 }
 
 /* Return 0 if the request is ready to be completed and return to guest;
@@ -259,7 +247,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
     int target;
     int ret = 0;
 
-    if (s->dataplane_started) {
+    if (s->dataplane_started && d) {
         assert(blk_get_aio_context(d->conf.blk) == s->ctx);
     }
     /* Here VIRTIO_SCSI_S_OK means "FUNCTION COMPLETE".  */
@@ -296,7 +284,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                 VirtIOSCSICancelNotifier *notifier;
 
                 req->remaining = 1;
-                notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                notifier = g_new(VirtIOSCSICancelNotifier, 1);
                 notifier->tmf_req = req;
                 notifier->notifier.notify = virtio_scsi_cancel_notify;
                 scsi_req_cancel_async(r, &notifier->notifier);
@@ -345,7 +333,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
                     VirtIOSCSICancelNotifier *notifier;
 
                     req->remaining++;
-                    notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                    notifier = g_new(VirtIOSCSICancelNotifier, 1);
                     notifier->notifier.notify = virtio_scsi_cancel_notify;
                     notifier->tmf_req = req;
                     scsi_req_cancel_async(r, &notifier->notifier);
@@ -361,7 +349,7 @@ static int virtio_scsi_do_tmf(VirtIOSCSI *s, VirtIOSCSIReq *req)
         target = req->req.tmf.lun[1];
         s->resetting++;
         QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-             d = DO_UPCAST(SCSIDevice, qdev, kid->child);
+             d = SCSI_DEVICE(kid->child);
              if (d->channel == 0 && d->id == target) {
                 qdev_reset_all(&d->qdev);
              }
@@ -386,7 +374,7 @@ fail:
     return ret;
 }
 
-void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     VirtIODevice *vdev = (VirtIODevice *)s;
     uint32_t type;
@@ -424,20 +412,28 @@ void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
     }
 }
 
-static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq)
 {
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
     VirtIOSCSIReq *req;
 
-    if (s->ctx && !s->dataplane_disabled) {
-        virtio_scsi_dataplane_start(s);
-        return;
-    }
     while ((req = virtio_scsi_pop_req(s, vq))) {
         virtio_scsi_handle_ctrl_req(s, req);
     }
 }
 
+static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+
+    if (s->ctx) {
+        virtio_scsi_dataplane_start(s);
+        if (!s->dataplane_fenced) {
+            return;
+        }
+    }
+    virtio_scsi_handle_ctrl_vq(s, vq);
+}
+
 static void virtio_scsi_complete_cmd_req(VirtIOSCSIReq *req)
 {
     /* Sense data is not in req->resp and is copied separately
@@ -520,7 +516,7 @@ static void virtio_scsi_fail_cmd_req(VirtIOSCSIReq *req)
     virtio_scsi_complete_cmd_req(req);
 }
 
-bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     VirtIOSCSICommon *vs = &s->parent_obj;
     SCSIDevice *d;
@@ -562,7 +558,7 @@ bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
     return true;
 }
 
-void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     SCSIRequest *sreq = req->sreq;
     if (scsi_req_enqueue(sreq)) {
@@ -572,17 +568,11 @@ void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
     scsi_req_unref(sreq);
 }
 
-static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq)
 {
-    /* use non-QOM casts in the data path */
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
     VirtIOSCSIReq *req, *next;
     QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
 
-    if (s->ctx && !s->dataplane_disabled) {
-        virtio_scsi_dataplane_start(s);
-        return;
-    }
     while ((req = virtio_scsi_pop_req(s, vq))) {
         if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
             QTAILQ_INSERT_TAIL(&reqs, req, next);
@@ -594,6 +584,20 @@ static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
     }
 }
 
+static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+{
+    /* use non-QOM casts in the data path */
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+
+    if (s->ctx) {
+        virtio_scsi_dataplane_start(s);
+        if (!s->dataplane_fenced) {
+            return;
+        }
+    }
+    virtio_scsi_handle_cmd_vq(s, vq);
+}
+
 static void virtio_scsi_get_config(VirtIODevice *vdev,
                                    uint8_t *config)
 {
@@ -662,6 +666,11 @@ static void virtio_scsi_reset(VirtIODevice *vdev)
 static void virtio_scsi_save(QEMUFile *f, void *opaque)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+
+    if (s->dataplane_started) {
+        virtio_scsi_dataplane_stop(s);
+    }
     virtio_save(vdev, f);
 }
 
@@ -694,11 +703,7 @@ void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
         aio_context_acquire(s->ctx);
     }
 
-    if (s->dataplane_started) {
-        req = virtio_scsi_pop_req_vring(s, s->event_vring);
-    } else {
-        req = virtio_scsi_pop_req(s, vs->event_vq);
-    }
+    req = virtio_scsi_pop_req(s, vs->event_vq);
     if (!req) {
         s->events_dropped = true;
         goto out;
@@ -736,17 +741,24 @@ out:
     }
 }
 
+void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq)
+{
+    if (s->events_dropped) {
+        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
+    }
+}
+
 static void virtio_scsi_handle_event(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
 
-    if (s->ctx && !s->dataplane_disabled) {
+    if (s->ctx) {
         virtio_scsi_dataplane_start(s);
-        return;
-    }
-    if (s->events_dropped) {
-        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
+        if (!s->dataplane_fenced) {
+            return;
+        }
     }
+    virtio_scsi_handle_event_vq(s, vq);
 }
 
 static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
@@ -754,13 +766,29 @@ static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
-    if (virtio_has_feature(vdev, VIRTIO_SCSI_F_CHANGE) &&
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_CHANGE) &&
         dev->type != TYPE_ROM) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_PARAM_CHANGE,
                                sense.asc | (sense.ascq << 8));
     }
 }
 
+static void virtio_scsi_blk_insert_notifier(Notifier *n, void *data)
+{
+    VirtIOSCSIBlkChangeNotifier *cn = DO_UPCAST(VirtIOSCSIBlkChangeNotifier,
+                                                n, n);
+    assert(cn->sd->conf.blk == data);
+    blk_op_block_all(cn->sd->conf.blk, cn->s->blocker);
+}
+
+static void virtio_scsi_blk_remove_notifier(Notifier *n, void *data)
+{
+    VirtIOSCSIBlkChangeNotifier *cn = DO_UPCAST(VirtIOSCSIBlkChangeNotifier,
+                                                n, n);
+    assert(cn->sd->conf.blk == data);
+    blk_op_unblock_all(cn->sd->conf.blk, cn->s->blocker);
+}
+
 static void virtio_scsi_hotplug(HotplugHandler *hotplug_dev, DeviceState *dev,
                                 Error **errp)
 {
@@ -768,7 +796,9 @@ static void virtio_scsi_hotplug(HotplugHandler *hotplug_dev, DeviceState *dev,
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
     SCSIDevice *sd = SCSI_DEVICE(dev);
 
-    if (s->ctx && !s->dataplane_disabled) {
+    if (s->ctx && !s->dataplane_fenced) {
+        VirtIOSCSIBlkChangeNotifier *insert_notifier, *remove_notifier;
+
         if (blk_op_is_blocked(sd->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
             return;
         }
@@ -776,9 +806,23 @@ static void virtio_scsi_hotplug(HotplugHandler *hotplug_dev, DeviceState *dev,
         aio_context_acquire(s->ctx);
         blk_set_aio_context(sd->conf.blk, s->ctx);
         aio_context_release(s->ctx);
+
+        insert_notifier = g_new0(VirtIOSCSIBlkChangeNotifier, 1);
+        insert_notifier->n.notify = virtio_scsi_blk_insert_notifier;
+        insert_notifier->s = s;
+        insert_notifier->sd = sd;
+        blk_add_insert_bs_notifier(sd->conf.blk, &insert_notifier->n);
+        QTAILQ_INSERT_TAIL(&s->insert_notifiers, insert_notifier, next);
+
+        remove_notifier = g_new0(VirtIOSCSIBlkChangeNotifier, 1);
+        remove_notifier->n.notify = virtio_scsi_blk_remove_notifier;
+        remove_notifier->s = s;
+        remove_notifier->sd = sd;
+        blk_add_remove_bs_notifier(sd->conf.blk, &remove_notifier->n);
+        QTAILQ_INSERT_TAIL(&s->remove_notifiers, remove_notifier, next);
     }
 
-    if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
         virtio_scsi_push_event(s, sd,
                                VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_RESCAN);
@@ -791,8 +835,9 @@ static void virtio_scsi_hotunplug(HotplugHandler *hotplug_dev, DeviceState *dev,
     VirtIODevice *vdev = VIRTIO_DEVICE(hotplug_dev);
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
     SCSIDevice *sd = SCSI_DEVICE(dev);
+    VirtIOSCSIBlkChangeNotifier *insert_notifier, *remove_notifier;
 
-    if (virtio_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
         virtio_scsi_push_event(s, sd,
                                VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_REMOVED);
@@ -801,6 +846,25 @@ static void virtio_scsi_hotunplug(HotplugHandler *hotplug_dev, DeviceState *dev,
     if (s->ctx) {
         blk_op_unblock_all(sd->conf.blk, s->blocker);
     }
+
+    QTAILQ_FOREACH(insert_notifier, &s->insert_notifiers, next) {
+        if (insert_notifier->sd == sd) {
+            notifier_remove(&insert_notifier->n);
+            QTAILQ_REMOVE(&s->insert_notifiers, insert_notifier, next);
+            g_free(insert_notifier);
+            break;
+        }
+    }
+
+    QTAILQ_FOREACH(remove_notifier, &s->remove_notifiers, next) {
+        if (remove_notifier->sd == sd) {
+            notifier_remove(&remove_notifier->n);
+            QTAILQ_REMOVE(&s->remove_notifiers, remove_notifier, next);
+            g_free(remove_notifier);
+            break;
+        }
+    }
+
     qdev_simple_device_unplug_cb(hotplug_dev, dev, errp);
 }
 
@@ -856,31 +920,6 @@ void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
     }
 }
 
-/* Disable dataplane thread during live migration since it does not
- * update the dirty memory bitmap yet.
- */
-static void virtio_scsi_migration_state_changed(Notifier *notifier, void *data)
-{
-    VirtIOSCSI *s = container_of(notifier, VirtIOSCSI,
-                                 migration_state_notifier);
-    MigrationState *mig = data;
-
-    if (migration_in_setup(mig)) {
-        if (!s->dataplane_started) {
-            return;
-        }
-        virtio_scsi_dataplane_stop(s);
-        s->dataplane_disabled = true;
-    } else if (migration_has_finished(mig) ||
-               migration_has_failed(mig)) {
-        if (s->dataplane_started) {
-            return;
-        }
-        blk_drain_all(); /* complete in-flight non-dataplane requests */
-        s->dataplane_disabled = false;
-    }
-}
-
 static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
@@ -911,10 +950,11 @@ static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
 
     register_savevm(dev, "virtio-scsi", virtio_scsi_id++, 1,
                     virtio_scsi_save, virtio_scsi_load, s);
-    s->migration_state_notifier.notify = virtio_scsi_migration_state_changed;
-    add_migration_state_change_notifier(&s->migration_state_notifier);
 
     error_setg(&s->blocker, "block device is in use by data plane");
+
+    QTAILQ_INIT(&s->insert_notifiers);
+    QTAILQ_INIT(&s->remove_notifiers);
 }
 
 static void virtio_scsi_instance_init(Object *obj)
@@ -943,8 +983,6 @@ static void virtio_scsi_device_unrealize(DeviceState *dev, Error **errp)
     error_free(s->blocker);
 
     unregister_savevm(dev, "virtio-scsi", s);
-    remove_migration_state_change_notifier(&s->migration_state_notifier);
-
     virtio_scsi_common_unrealize(dev, errp);
 }
 
diff --git a/qemu/hw/scsi/vmw_pvscsi.c b/qemu/hw/scsi/vmw_pvscsi.c
index 9c71f31fe..e690b4ec0 100644
--- a/qemu/hw/scsi/vmw_pvscsi.c
+++ b/qemu/hw/scsi/vmw_pvscsi.c
@@ -25,6 +25,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/scsi/scsi.h"
 #include <block/scsi.h>
 #include "hw/pci/msi.h"
@@ -32,7 +34,6 @@
 #include "trace.h"
 
 
-#define PVSCSI_MSI_OFFSET        (0x50)
 #define PVSCSI_USE_64BIT         (true)
 #define PVSCSI_PER_VECTOR_MASK   (false)
 
@@ -49,9 +50,33 @@
     (stl_le_pci_dma(&container_of(m, PVSCSIState, rings)->parent_obj, \
                  (m)->rs_pa + offsetof(struct PVSCSIRingsState, field), val))
 
+typedef struct PVSCSIClass {
+    PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
+} PVSCSIClass;
+
 #define TYPE_PVSCSI "pvscsi"
 #define PVSCSI(obj) OBJECT_CHECK(PVSCSIState, (obj), TYPE_PVSCSI)
 
+#define PVSCSI_DEVICE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(PVSCSIClass, (klass), TYPE_PVSCSI)
+#define PVSCSI_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(PVSCSIClass, (obj), TYPE_PVSCSI)
+
+/* Compatability flags for migration */
+#define PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT 0
+#define PVSCSI_COMPAT_OLD_PCI_CONFIGURATION \
+    (1 << PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT)
+#define PVSCSI_COMPAT_DISABLE_PCIE_BIT 1
+#define PVSCSI_COMPAT_DISABLE_PCIE \
+    (1 << PVSCSI_COMPAT_DISABLE_PCIE_BIT)
+
+#define PVSCSI_USE_OLD_PCI_CONFIGURATION(s) \
+    ((s)->compat_flags & PVSCSI_COMPAT_OLD_PCI_CONFIGURATION)
+#define PVSCSI_MSI_OFFSET(s) \
+    (PVSCSI_USE_OLD_PCI_CONFIGURATION(s) ? 0x50 : 0x7c)
+#define PVSCSI_EXP_EP_OFFSET (0x40)
+
 typedef struct PVSCSIRingInfo {
     uint64_t            rs_pa;
     uint32_t            txr_len_mask;
@@ -100,6 +125,8 @@ typedef struct {
 
     PVSCSIRingInfo rings;                /* Data transfer rings manager      */
     uint32_t resetting;                  /* Reset in progress                */
+
+    uint32_t compat_flags;
 } PVSCSIState;
 
 typedef struct PVSCSIRequest {
@@ -1019,7 +1046,7 @@ pvscsi_init_msi(PVSCSIState *s)
     int res;
     PCIDevice *d = PCI_DEVICE(s);
 
-    res = msi_init(d, PVSCSI_MSI_OFFSET, PVSCSI_MSIX_NUM_VECTORS,
+    res = msi_init(d, PVSCSI_MSI_OFFSET(s), PVSCSI_MSIX_NUM_VECTORS,
                    PVSCSI_USE_64BIT, PVSCSI_PER_VECTOR_MASK);
     if (res < 0) {
         trace_pvscsi_init_msi_fail(res);
@@ -1069,9 +1096,16 @@ pvscsi_init(PCIDevice *pci_dev)
 
     trace_pvscsi_state("init");
 
-    /* PCI subsystem ID */
-    pci_dev->config[PCI_SUBSYSTEM_ID] = 0x00;
-    pci_dev->config[PCI_SUBSYSTEM_ID + 1] = 0x10;
+    /* PCI subsystem ID, subsystem vendor ID, revision */
+    if (PVSCSI_USE_OLD_PCI_CONFIGURATION(s)) {
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_ID, 0x1000);
+    } else {
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_VENDOR_ID,
+                     PCI_VENDOR_ID_VMWARE);
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_ID,
+                     PCI_DEVICE_ID_VMWARE_PVSCSI);
+        pci_config_set_revision(pci_dev->config, 0x2);
+    }
 
     /* PCI latency timer = 255 */
     pci_dev->config[PCI_LATENCY_TIMER] = 0xff;
@@ -1085,6 +1119,10 @@ pvscsi_init(PCIDevice *pci_dev)
 
     pvscsi_init_msi(s);
 
+    if (pci_is_express(pci_dev) && pci_bus_is_express(pci_dev->bus)) {
+        pcie_endpoint_cap_init(pci_dev, PVSCSI_EXP_EP_OFFSET);
+    }
+
     s->completion_worker = qemu_bh_new(pvscsi_process_completion_queue, s);
     if (!s->completion_worker) {
         pvscsi_cleanup_msi(s);
@@ -1139,6 +1177,27 @@ pvscsi_post_load(void *opaque, int version_id)
     return 0;
 }
 
+static bool pvscsi_vmstate_need_pcie_device(void *opaque)
+{
+    PVSCSIState *s = PVSCSI(opaque);
+
+    return !(s->compat_flags & PVSCSI_COMPAT_DISABLE_PCIE);
+}
+
+static bool pvscsi_vmstate_test_pci_device(void *opaque, int version_id)
+{
+    return !pvscsi_vmstate_need_pcie_device(opaque);
+}
+
+static const VMStateDescription vmstate_pvscsi_pcie_device = {
+    .name = "pvscsi/pcie",
+    .needed = pvscsi_vmstate_need_pcie_device,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, PVSCSIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_pvscsi = {
     .name = "pvscsi",
     .version_id = 0,
@@ -1146,7 +1205,9 @@ static const VMStateDescription vmstate_pvscsi = {
     .pre_save = pvscsi_pre_save,
     .post_load = pvscsi_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_PCI_DEVICE(parent_obj, PVSCSIState),
+        VMSTATE_STRUCT_TEST(parent_obj, PVSCSIState,
+                            pvscsi_vmstate_test_pci_device, 0,
+                            vmstate_pci_device, PCIDevice),
         VMSTATE_UINT8(msi_used, PVSCSIState),
         VMSTATE_UINT32(resetting, PVSCSIState),
         VMSTATE_UINT64(reg_interrupt_status, PVSCSIState),
@@ -1171,18 +1232,40 @@ static const VMStateDescription vmstate_pvscsi = {
         VMSTATE_UINT64(rings.filled_cmp_ptr, PVSCSIState),
 
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_pvscsi_pcie_device,
+        NULL
     }
 };
 
 static Property pvscsi_properties[] = {
     DEFINE_PROP_UINT8("use_msg", PVSCSIState, use_msg, 1),
+    DEFINE_PROP_BIT("x-old-pci-configuration", PVSCSIState, compat_flags,
+                    PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", PVSCSIState, compat_flags,
+                    PVSCSI_COMPAT_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void pvscsi_realize(DeviceState *qdev, Error **errp)
+{
+    PVSCSIClass *pvs_c = PVSCSI_DEVICE_GET_CLASS(qdev);
+    PCIDevice *pci_dev = PCI_DEVICE(qdev);
+    PVSCSIState *s = PVSCSI(qdev);
+
+    if (!(s->compat_flags & PVSCSI_COMPAT_DISABLE_PCIE)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    pvs_c->parent_dc_realize(qdev, errp);
+}
+
 static void pvscsi_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    PVSCSIClass *pvs_k = PVSCSI_DEVICE_CLASS(klass);
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->init = pvscsi_init;
@@ -1191,6 +1274,8 @@ static void pvscsi_class_init(ObjectClass *klass, void *data)
     k->device_id = PCI_DEVICE_ID_VMWARE_PVSCSI;
     k->class_id = PCI_CLASS_STORAGE_SCSI;
     k->subsystem_id = 0x1000;
+    pvs_k->parent_dc_realize = dc->realize;
+    dc->realize = pvscsi_realize;
     dc->reset = pvscsi_reset;
     dc->vmsd = &vmstate_pvscsi;
     dc->props = pvscsi_properties;
@@ -1202,6 +1287,7 @@ static void pvscsi_class_init(ObjectClass *klass, void *data)
 static const TypeInfo pvscsi_info = {
     .name          = TYPE_PVSCSI,
     .parent        = TYPE_PCI_DEVICE,
+    .class_size    = sizeof(PVSCSIClass),
     .instance_size = sizeof(PVSCSIState),
     .class_init    = pvscsi_class_init,
     .interfaces = (InterfaceInfo[]) {
diff --git a/qemu/hw/sd/Makefile.objs b/qemu/hw/sd/Makefile.objs
index f1aed83d9..31c83308f 100644
--- a/qemu/hw/sd/Makefile.objs
+++ b/qemu/hw/sd/Makefile.objs
@@ -1,6 +1,6 @@
 common-obj-$(CONFIG_PL181) += pl181.o
 common-obj-$(CONFIG_SSI_SD) += ssi-sd.o
-common-obj-$(CONFIG_SD) += sd.o
+common-obj-$(CONFIG_SD) += sd.o core.o
 common-obj-$(CONFIG_SDHCI) += sdhci.o
 
 obj-$(CONFIG_MILKYMIST) += milkymist-memcard.o
diff --git a/qemu/hw/sd/core.c b/qemu/hw/sd/core.c
new file mode 100644
index 000000000..14c2bdf27
--- /dev/null
+++ b/qemu/hw/sd/core.c
@@ -0,0 +1,146 @@
+/*
+ * SD card bus interface code.
+ *
+ * Copyright (c) 2015 Linaro Limited
+ *
+ * Author:
+ *  Peter Maydell <peter.maydell@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/qdev-core.h"
+#include "sysemu/block-backend.h"
+#include "hw/sd/sd.h"
+
+static SDState *get_card(SDBus *sdbus)
+{
+    /* We only ever have one child on the bus so just return it */
+    BusChild *kid = QTAILQ_FIRST(&sdbus->qbus.children);
+
+    if (!kid) {
+        return NULL;
+    }
+    return SD_CARD(kid->child);
+}
+
+int sdbus_do_command(SDBus *sdbus, SDRequest *req, uint8_t *response)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->do_command(card, req, response);
+    }
+
+    return 0;
+}
+
+void sdbus_write_data(SDBus *sdbus, uint8_t value)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        sc->write_data(card, value);
+    }
+}
+
+uint8_t sdbus_read_data(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->read_data(card);
+    }
+
+    return 0;
+}
+
+bool sdbus_data_ready(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->data_ready(card);
+    }
+
+    return false;
+}
+
+bool sdbus_get_inserted(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->get_inserted(card);
+    }
+
+    return false;
+}
+
+bool sdbus_get_readonly(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->get_readonly(card);
+    }
+
+    return false;
+}
+
+void sdbus_set_inserted(SDBus *sdbus, bool inserted)
+{
+    SDBusClass *sbc = SD_BUS_GET_CLASS(sdbus);
+    BusState *qbus = BUS(sdbus);
+
+    if (sbc->set_inserted) {
+        sbc->set_inserted(qbus->parent, inserted);
+    }
+}
+
+void sdbus_set_readonly(SDBus *sdbus, bool readonly)
+{
+    SDBusClass *sbc = SD_BUS_GET_CLASS(sdbus);
+    BusState *qbus = BUS(sdbus);
+
+    if (sbc->set_readonly) {
+        sbc->set_readonly(qbus->parent, readonly);
+    }
+}
+
+static const TypeInfo sd_bus_info = {
+    .name = TYPE_SD_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_size = sizeof(SDBusClass),
+};
+
+static void sd_bus_register_types(void)
+{
+    type_register_static(&sd_bus_info);
+}
+
+type_init(sd_bus_register_types)
diff --git a/qemu/hw/sd/milkymist-memcard.c b/qemu/hw/sd/milkymist-memcard.c
index 2209ef1d5..c04ff02fa 100644
--- a/qemu/hw/sd/milkymist-memcard.c
+++ b/qemu/hw/sd/milkymist-memcard.c
@@ -21,6 +21,7 @@
  *   http://www.milkymist.org/socdoc/memcard.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
@@ -28,7 +29,7 @@
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 enum {
     ENABLE_CMD_TX   = (1<<0),
diff --git a/qemu/hw/sd/omap_mmc.c b/qemu/hw/sd/omap_mmc.c
index d072deca1..e934cd365 100644
--- a/qemu/hw/sd/omap_mmc.c
+++ b/qemu/hw/sd/omap_mmc.c
@@ -16,9 +16,10 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 struct omap_mmc_s {
     qemu_irq irq;
@@ -578,8 +579,7 @@ struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 BlockBackend *blk,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
@@ -605,8 +605,7 @@ struct omap_mmc_s *omap2_mmc_init(struct omap_target_agent_s *ta,
                 BlockBackend *blk, qemu_irq irq, qemu_irq dma[],
                 omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/qemu/hw/sd/pl181.c b/qemu/hw/sd/pl181.c
index 11fcd479d..e87abb205 100644
--- a/qemu/hw/sd/pl181.c
+++ b/qemu/hw/sd/pl181.c
@@ -7,10 +7,11 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/sysbus.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 //#define DEBUG_PL181 1
 
@@ -46,7 +47,7 @@ typedef struct PL181State {
     int32_t fifo_pos;
     int32_t fifo_len;
     /* The linux 2.6.21 driver is buggy, and misbehaves if new data arrives
-       while it is reading the FIFO.  We hack around this be defering
+       while it is reading the FIFO.  We hack around this by deferring
        subsequent transfers until after the driver polls the status word.
        http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=4446/1
      */
diff --git a/qemu/hw/sd/pxa2xx_mmci.c b/qemu/hw/sd/pxa2xx_mmci.c
index d1fe6d58e..3deccf02c 100644
--- a/qemu/hw/sd/pxa2xx_mmci.c
+++ b/qemu/hw/sd/pxa2xx_mmci.c
@@ -10,18 +10,34 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
+#include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 #include "hw/qdev.h"
+#include "hw/qdev-properties.h"
+#include "qemu/error-report.h"
+
+#define TYPE_PXA2XX_MMCI "pxa2xx-mmci"
+#define PXA2XX_MMCI(obj) OBJECT_CHECK(PXA2xxMMCIState, (obj), TYPE_PXA2XX_MMCI)
+
+#define TYPE_PXA2XX_MMCI_BUS "pxa2xx-mmci-bus"
+#define PXA2XX_MMCI_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_PXA2XX_MMCI_BUS)
 
 struct PXA2xxMMCIState {
+    SysBusDevice parent_obj;
+
     MemoryRegion iomem;
     qemu_irq irq;
     qemu_irq rx_dma;
     qemu_irq tx_dma;
+    qemu_irq inserted;
+    qemu_irq readonly;
 
-    SDState *card;
+    BlockBackend *blk;
+    SDBus sdbus;
 
     uint32_t status;
     uint32_t clkrt;
@@ -29,25 +45,70 @@ struct PXA2xxMMCIState {
     uint32_t cmdat;
     uint32_t resp_tout;
     uint32_t read_tout;
-    int blklen;
-    int numblk;
+    int32_t blklen;
+    int32_t numblk;
     uint32_t intmask;
     uint32_t intreq;
-    int cmd;
+    int32_t cmd;
     uint32_t arg;
 
-    int active;
-    int bytesleft;
+    int32_t active;
+    int32_t bytesleft;
     uint8_t tx_fifo[64];
-    int tx_start;
-    int tx_len;
+    uint32_t tx_start;
+    uint32_t tx_len;
     uint8_t rx_fifo[32];
-    int rx_start;
-    int rx_len;
+    uint32_t rx_start;
+    uint32_t rx_len;
     uint16_t resp_fifo[9];
-    int resp_len;
+    uint32_t resp_len;
 
-    int cmdreq;
+    int32_t cmdreq;
+};
+
+static bool pxa2xx_mmci_vmstate_validate(void *opaque, int version_id)
+{
+    PXA2xxMMCIState *s = opaque;
+
+    return s->tx_start < ARRAY_SIZE(s->tx_fifo)
+        && s->rx_start < ARRAY_SIZE(s->rx_fifo)
+        && s->tx_len <= ARRAY_SIZE(s->tx_fifo)
+        && s->rx_len <= ARRAY_SIZE(s->rx_fifo)
+        && s->resp_len <= ARRAY_SIZE(s->resp_fifo);
+}
+
+
+static const VMStateDescription vmstate_pxa2xx_mmci = {
+    .name = "pxa2xx-mmci",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(status, PXA2xxMMCIState),
+        VMSTATE_UINT32(clkrt, PXA2xxMMCIState),
+        VMSTATE_UINT32(spi, PXA2xxMMCIState),
+        VMSTATE_UINT32(cmdat, PXA2xxMMCIState),
+        VMSTATE_UINT32(resp_tout, PXA2xxMMCIState),
+        VMSTATE_UINT32(read_tout, PXA2xxMMCIState),
+        VMSTATE_INT32(blklen, PXA2xxMMCIState),
+        VMSTATE_INT32(numblk, PXA2xxMMCIState),
+        VMSTATE_UINT32(intmask, PXA2xxMMCIState),
+        VMSTATE_UINT32(intreq, PXA2xxMMCIState),
+        VMSTATE_INT32(cmd, PXA2xxMMCIState),
+        VMSTATE_UINT32(arg, PXA2xxMMCIState),
+        VMSTATE_INT32(cmdreq, PXA2xxMMCIState),
+        VMSTATE_INT32(active, PXA2xxMMCIState),
+        VMSTATE_INT32(bytesleft, PXA2xxMMCIState),
+        VMSTATE_UINT32(tx_start, PXA2xxMMCIState),
+        VMSTATE_UINT32(tx_len, PXA2xxMMCIState),
+        VMSTATE_UINT32(rx_start, PXA2xxMMCIState),
+        VMSTATE_UINT32(rx_len, PXA2xxMMCIState),
+        VMSTATE_UINT32(resp_len, PXA2xxMMCIState),
+        VMSTATE_VALIDATE("fifo size incorrect", pxa2xx_mmci_vmstate_validate),
+        VMSTATE_UINT8_ARRAY(tx_fifo, PXA2xxMMCIState, 64),
+        VMSTATE_UINT8_ARRAY(rx_fifo, PXA2xxMMCIState, 32),
+        VMSTATE_UINT16_ARRAY(resp_fifo, PXA2xxMMCIState, 9),
+        VMSTATE_END_OF_LIST()
+    }
 };
 
 #define MMC_STRPCL	0x00	/* MMC Clock Start/Stop register */
@@ -121,7 +182,7 @@ static void pxa2xx_mmci_fifo_update(PXA2xxMMCIState *s)
 
     if (s->cmdat & CMDAT_WR_RD) {
         while (s->bytesleft && s->tx_len) {
-            sd_write_data(s->card, s->tx_fifo[s->tx_start ++]);
+            sdbus_write_data(&s->sdbus, s->tx_fifo[s->tx_start++]);
             s->tx_start &= 0x1f;
             s->tx_len --;
             s->bytesleft --;
@@ -131,7 +192,7 @@ static void pxa2xx_mmci_fifo_update(PXA2xxMMCIState *s)
     } else
         while (s->bytesleft && s->rx_len < 32) {
             s->rx_fifo[(s->rx_start + (s->rx_len ++)) & 0x1f] =
-                sd_read_data(s->card);
+                sdbus_read_data(&s->sdbus);
             s->bytesleft --;
             s->intreq |= INT_RXFIFO_REQ;
         }
@@ -165,7 +226,7 @@ static void pxa2xx_mmci_wakequeues(PXA2xxMMCIState *s)
     request.arg = s->arg;
     request.crc = 0;	/* FIXME */
 
-    rsplen = sd_do_command(s->card, &request, response);
+    rsplen = sdbus_do_command(&s->sdbus, &request, response);
     s->intreq |= INT_END_CMD;
 
     memset(s->resp_fifo, 0, sizeof(s->resp_fifo));
@@ -391,114 +452,147 @@ static const MemoryRegionOps pxa2xx_mmci_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pxa2xx_mmci_save(QEMUFile *f, void *opaque)
+PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
+                hwaddr base,
+                BlockBackend *blk, qemu_irq irq,
+                qemu_irq rx_dma, qemu_irq tx_dma)
 {
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    int i;
-
-    qemu_put_be32s(f, &s->status);
-    qemu_put_be32s(f, &s->clkrt);
-    qemu_put_be32s(f, &s->spi);
-    qemu_put_be32s(f, &s->cmdat);
-    qemu_put_be32s(f, &s->resp_tout);
-    qemu_put_be32s(f, &s->read_tout);
-    qemu_put_be32(f, s->blklen);
-    qemu_put_be32(f, s->numblk);
-    qemu_put_be32s(f, &s->intmask);
-    qemu_put_be32s(f, &s->intreq);
-    qemu_put_be32(f, s->cmd);
-    qemu_put_be32s(f, &s->arg);
-    qemu_put_be32(f, s->cmdreq);
-    qemu_put_be32(f, s->active);
-    qemu_put_be32(f, s->bytesleft);
-
-    qemu_put_byte(f, s->tx_len);
-    for (i = 0; i < s->tx_len; i ++)
-        qemu_put_byte(f, s->tx_fifo[(s->tx_start + i) & 63]);
-
-    qemu_put_byte(f, s->rx_len);
-    for (i = 0; i < s->rx_len; i ++)
-        qemu_put_byte(f, s->rx_fifo[(s->rx_start + i) & 31]);
-
-    qemu_put_byte(f, s->resp_len);
-    for (i = s->resp_len; i < 9; i ++)
-        qemu_put_be16s(f, &s->resp_fifo[i]);
+    DeviceState *dev, *carddev;
+    SysBusDevice *sbd;
+    PXA2xxMMCIState *s;
+    Error *err = NULL;
+
+    dev = qdev_create(NULL, TYPE_PXA2XX_MMCI);
+    s = PXA2XX_MMCI(dev);
+    sbd = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(sbd, 0, base);
+    sysbus_connect_irq(sbd, 0, irq);
+    qdev_connect_gpio_out_named(dev, "rx-dma", 0, rx_dma);
+    qdev_connect_gpio_out_named(dev, "tx-dma", 0, tx_dma);
+
+    /* Create and plug in the sd card */
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &err);
+    if (err) {
+        error_report("failed to init SD card: %s", error_get_pretty(err));
+        return NULL;
+    }
+    object_property_set_bool(OBJECT(carddev), true, "realized", &err);
+    if (err) {
+        error_report("failed to init SD card: %s", error_get_pretty(err));
+        return NULL;
+    }
+
+    return s;
 }
 
-static int pxa2xx_mmci_load(QEMUFile *f, void *opaque, int version_id)
+static void pxa2xx_mmci_set_inserted(DeviceState *dev, bool inserted)
 {
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    int i;
-
-    qemu_get_be32s(f, &s->status);
-    qemu_get_be32s(f, &s->clkrt);
-    qemu_get_be32s(f, &s->spi);
-    qemu_get_be32s(f, &s->cmdat);
-    qemu_get_be32s(f, &s->resp_tout);
-    qemu_get_be32s(f, &s->read_tout);
-    s->blklen = qemu_get_be32(f);
-    s->numblk = qemu_get_be32(f);
-    qemu_get_be32s(f, &s->intmask);
-    qemu_get_be32s(f, &s->intreq);
-    s->cmd = qemu_get_be32(f);
-    qemu_get_be32s(f, &s->arg);
-    s->cmdreq = qemu_get_be32(f);
-    s->active = qemu_get_be32(f);
-    s->bytesleft = qemu_get_be32(f);
-
-    s->tx_len = qemu_get_byte(f);
-    s->tx_start = 0;
-    if (s->tx_len >= sizeof(s->tx_fifo) || s->tx_len < 0)
-        return -EINVAL;
-    for (i = 0; i < s->tx_len; i ++)
-        s->tx_fifo[i] = qemu_get_byte(f);
+    PXA2xxMMCIState *s = PXA2XX_MMCI(dev);
 
-    s->rx_len = qemu_get_byte(f);
-    s->rx_start = 0;
-    if (s->rx_len >= sizeof(s->rx_fifo) || s->rx_len < 0)
-        return -EINVAL;
-    for (i = 0; i < s->rx_len; i ++)
-        s->rx_fifo[i] = qemu_get_byte(f);
+    qemu_set_irq(s->inserted, inserted);
+}
 
-    s->resp_len = qemu_get_byte(f);
-    if (s->resp_len > 9 || s->resp_len < 0)
-        return -EINVAL;
-    for (i = s->resp_len; i < 9; i ++)
-         qemu_get_be16s(f, &s->resp_fifo[i]);
+static void pxa2xx_mmci_set_readonly(DeviceState *dev, bool readonly)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(dev);
 
-    return 0;
+    qemu_set_irq(s->readonly, readonly);
 }
 
-PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
-                hwaddr base,
-                BlockBackend *blk, qemu_irq irq,
-                qemu_irq rx_dma, qemu_irq tx_dma)
+void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
+                          qemu_irq coverswitch)
 {
-    PXA2xxMMCIState *s;
+    DeviceState *dev = DEVICE(s);
+
+    s->readonly = readonly;
+    s->inserted = coverswitch;
+
+    pxa2xx_mmci_set_inserted(dev, sdbus_get_inserted(&s->sdbus));
+    pxa2xx_mmci_set_readonly(dev, sdbus_get_readonly(&s->sdbus));
+}
+
+static void pxa2xx_mmci_reset(DeviceState *d)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(d);
+
+    s->status = 0;
+    s->clkrt = 0;
+    s->spi = 0;
+    s->cmdat = 0;
+    s->resp_tout = 0;
+    s->read_tout = 0;
+    s->blklen = 0;
+    s->numblk = 0;
+    s->intmask = 0;
+    s->intreq = 0;
+    s->cmd = 0;
+    s->arg = 0;
+    s->active = 0;
+    s->bytesleft = 0;
+    s->tx_start = 0;
+    s->tx_len = 0;
+    s->rx_start = 0;
+    s->rx_len = 0;
+    s->resp_len = 0;
+    s->cmdreq = 0;
+    memset(s->tx_fifo, 0, sizeof(s->tx_fifo));
+    memset(s->rx_fifo, 0, sizeof(s->rx_fifo));
+    memset(s->resp_fifo, 0, sizeof(s->resp_fifo));
+}
 
-    s = (PXA2xxMMCIState *) g_malloc0(sizeof(PXA2xxMMCIState));
-    s->irq = irq;
-    s->rx_dma = rx_dma;
-    s->tx_dma = tx_dma;
+static void pxa2xx_mmci_instance_init(Object *obj)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    DeviceState *dev = DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_mmci_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_mmci_ops, s,
                           "pxa2xx-mmci", 0x00100000);
-    memory_region_add_subregion(sysmem, base, &s->iomem);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_out_named(dev, &s->rx_dma, "rx-dma", 1);
+    qdev_init_gpio_out_named(dev, &s->tx_dma, "tx-dma", 1);
 
-    /* Instantiate the actual storage */
-    s->card = sd_init(blk, false);
-    if (s->card == NULL) {
-        exit(1);
-    }
+    qbus_create_inplace(&s->sdbus, sizeof(s->sdbus),
+                        TYPE_PXA2XX_MMCI_BUS, DEVICE(obj), "sd-bus");
+}
 
-    register_savevm(NULL, "pxa2xx_mmci", 0, 0,
-                    pxa2xx_mmci_save, pxa2xx_mmci_load, s);
+static void pxa2xx_mmci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
-    return s;
+    dc->vmsd = &vmstate_pxa2xx_mmci;
+    dc->reset = pxa2xx_mmci_reset;
 }
 
-void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
-                qemu_irq coverswitch)
+static void pxa2xx_mmci_bus_class_init(ObjectClass *klass, void *data)
 {
-    sd_set_cb(s->card, readonly, coverswitch);
+    SDBusClass *sbc = SD_BUS_CLASS(klass);
+
+    sbc->set_inserted = pxa2xx_mmci_set_inserted;
+    sbc->set_readonly = pxa2xx_mmci_set_readonly;
 }
+
+static const TypeInfo pxa2xx_mmci_info = {
+    .name = TYPE_PXA2XX_MMCI,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PXA2xxMMCIState),
+    .instance_init = pxa2xx_mmci_instance_init,
+    .class_init = pxa2xx_mmci_class_init,
+};
+
+static const TypeInfo pxa2xx_mmci_bus_info = {
+    .name = TYPE_PXA2XX_MMCI_BUS,
+    .parent = TYPE_SD_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_init = pxa2xx_mmci_bus_class_init,
+};
+
+static void pxa2xx_mmci_register_types(void)
+{
+    type_register_static(&pxa2xx_mmci_info);
+    type_register_static(&pxa2xx_mmci_bus_info);
+}
+
+type_init(pxa2xx_mmci_register_types)
diff --git a/qemu/hw/sd/sd.c b/qemu/hw/sd/sd.c
index a1ff465a6..b66e5d2db 100644
--- a/qemu/hw/sd/sd.c
+++ b/qemu/hw/sd/sd.c
@@ -29,10 +29,16 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
+#include "qapi/error.h"
 #include "qemu/bitmap.h"
+#include "hw/qdev-properties.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
 
 //#define DEBUG_SD 1
 
@@ -43,7 +49,9 @@ do { fprintf(stderr, "SD: " fmt , ## __VA_ARGS__); } while (0)
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-#define ACMD41_ENQUIRY_MASK 0x00ffffff
+#define ACMD41_ENQUIRY_MASK     0x00ffffff
+#define OCR_POWER_UP            0x80000000
+#define OCR_POWER_DELAY_NS      500000 /* 0.5ms */
 
 typedef enum {
     sd_r0 = 0,    /* no response */
@@ -77,9 +85,12 @@ enum SDCardStates {
 };
 
 struct SDState {
+    DeviceState parent_obj;
+
     uint32_t mode;    /* current card mode, one of SDCardModes */
     int32_t state;    /* current card state, one of SDCardStates */
     uint32_t ocr;
+    QEMUTimer *ocr_power_timer;
     uint8_t scr[8];
     uint8_t cid[16];
     uint8_t csd[16];
@@ -92,6 +103,7 @@ struct SDState {
     int32_t wpgrps_size;
     uint64_t size;
     uint32_t blk_len;
+    uint32_t multi_blk_cnt;
     uint32_t erase_start;
     uint32_t erase_end;
     uint8_t pwd[16];
@@ -193,8 +205,17 @@ static uint16_t sd_crc16(void *message, size_t width)
 
 static void sd_set_ocr(SDState *sd)
 {
-    /* All voltages OK, card power-up OK, Standard Capacity SD Memory Card */
-    sd->ocr = 0x80ffff00;
+    /* All voltages OK, Standard Capacity SD Memory Card, not yet powered up */
+    sd->ocr = 0x00ffff00;
+}
+
+static void sd_ocr_powerup(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* Set powered up bit in OCR */
+    assert(!(sd->ocr & OCR_POWER_UP));
+    sd->ocr |= OCR_POWER_UP;
 }
 
 static void sd_set_scr(SDState *sd)
@@ -389,8 +410,9 @@ static inline uint64_t sd_addr_to_wpnum(uint64_t addr)
     return addr >> (HWBLOCK_SHIFT + SECTOR_SHIFT + WPGROUP_SHIFT);
 }
 
-static void sd_reset(SDState *sd)
+static void sd_reset(DeviceState *dev)
 {
+    SDState *sd = SD_CARD(dev);
     uint64_t size;
     uint64_t sect;
 
@@ -412,8 +434,7 @@ static void sd_reset(SDState *sd)
     sd_set_cardstatus(sd);
     sd_set_sdstatus(sd);
 
-    if (sd->wp_groups)
-        g_free(sd->wp_groups);
+    g_free(sd->wp_groups);
     sd->wp_switch = sd->blk ? blk_is_read_only(sd->blk) : false;
     sd->wpgrps_size = sect;
     sd->wp_groups = bitmap_new(sd->wpgrps_size);
@@ -424,16 +445,44 @@ static void sd_reset(SDState *sd)
     sd->blk_len = 0x200;
     sd->pwd_len = 0;
     sd->expecting_acmd = false;
+    sd->multi_blk_cnt = 0;
+}
+
+static bool sd_get_inserted(SDState *sd)
+{
+    return sd->blk && blk_is_inserted(sd->blk);
+}
+
+static bool sd_get_readonly(SDState *sd)
+{
+    return sd->wp_switch;
 }
 
 static void sd_cardchange(void *opaque, bool load)
 {
     SDState *sd = opaque;
+    DeviceState *dev = DEVICE(sd);
+    SDBus *sdbus = SD_BUS(qdev_get_parent_bus(dev));
+    bool inserted = sd_get_inserted(sd);
+    bool readonly = sd_get_readonly(sd);
 
-    qemu_set_irq(sd->inserted_cb, blk_is_inserted(sd->blk));
-    if (blk_is_inserted(sd->blk)) {
-        sd_reset(sd);
-        qemu_set_irq(sd->readonly_cb, sd->wp_switch);
+    if (inserted) {
+        sd_reset(dev);
+    }
+
+    /* The IRQ notification is for legacy non-QOM SD controller devices;
+     * QOMified controllers use the SDBus APIs.
+     */
+    if (sdbus) {
+        sdbus_set_inserted(sdbus, inserted);
+        if (inserted) {
+            sdbus_set_readonly(sdbus, readonly);
+        }
+    } else {
+        qemu_set_irq(sd->inserted_cb, inserted);
+        if (inserted) {
+            qemu_set_irq(sd->readonly_cb, readonly);
+        }
     }
 }
 
@@ -441,10 +490,44 @@ static const BlockDevOps sd_block_ops = {
     .change_media_cb = sd_cardchange,
 };
 
+static bool sd_ocr_vmstate_needed(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* Include the OCR state (and timer) if it is not yet powered up */
+    return !(sd->ocr & OCR_POWER_UP);
+}
+
+static const VMStateDescription sd_ocr_vmstate = {
+    .name = "sd-card/ocr-state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = sd_ocr_vmstate_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(ocr, SDState),
+        VMSTATE_TIMER_PTR(ocr_power_timer, SDState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static int sd_vmstate_pre_load(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* If the OCR state is not included (prior versions, or not
+     * needed), then the OCR must be set as powered up. If the OCR state
+     * is included, this will be replaced by the state restore.
+     */
+    sd_ocr_powerup(sd);
+
+    return 0;
+}
+
 static const VMStateDescription sd_vmstate = {
     .name = "sd-card",
     .version_id = 1,
     .minimum_version_id = 1,
+    .pre_load = sd_vmstate_pre_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(mode, SDState),
         VMSTATE_INT32(state, SDState),
@@ -456,6 +539,7 @@ static const VMStateDescription sd_vmstate = {
         VMSTATE_UINT32(vhs, SDState),
         VMSTATE_BITMAP(wp_groups, SDState, 0, wpgrps_size),
         VMSTATE_UINT32(blk_len, SDState),
+        VMSTATE_UINT32(multi_blk_cnt, SDState),
         VMSTATE_UINT32(erase_start, SDState),
         VMSTATE_UINT32(erase_end, SDState),
         VMSTATE_UINT8_ARRAY(pwd, SDState, 16),
@@ -470,34 +554,35 @@ static const VMStateDescription sd_vmstate = {
         VMSTATE_BUFFER_POINTER_UNSAFE(buf, SDState, 1, 512),
         VMSTATE_BOOL(enable, SDState),
         VMSTATE_END_OF_LIST()
-    }
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &sd_ocr_vmstate,
+        NULL
+    },
 };
 
-/* We do not model the chip select pin, so allow the board to select
-   whether card should be in SSI or MMC/SD mode.  It is also up to the
-   board to ensure that ssi transfers only occur when the chip select
-   is asserted.  */
+/* Legacy initialization function for use by non-qdevified callers */
 SDState *sd_init(BlockBackend *blk, bool is_spi)
 {
-    SDState *sd;
-
-    if (blk && blk_is_read_only(blk)) {
-        fprintf(stderr, "sd_init: Cannot use read-only drive\n");
+    Object *obj;
+    DeviceState *dev;
+    Error *err = NULL;
+
+    obj = object_new(TYPE_SD_CARD);
+    dev = DEVICE(obj);
+    qdev_prop_set_drive(dev, "drive", blk, &err);
+    if (err) {
+        error_report("sd_init failed: %s", error_get_pretty(err));
         return NULL;
     }
-
-    sd = (SDState *) g_malloc0(sizeof(SDState));
-    sd->buf = blk_blockalign(blk, 512);
-    sd->spi = is_spi;
-    sd->enable = true;
-    sd->blk = blk;
-    sd_reset(sd);
-    if (sd->blk) {
-        blk_attach_dev_nofail(sd->blk, sd);
-        blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
+    qdev_prop_set_bit(dev, "spi", is_spi);
+    object_property_set_bool(obj, true, "realized", &err);
+    if (err) {
+        error_report("sd_init failed: %s", error_get_pretty(err));
+        return NULL;
     }
-    vmstate_register(NULL, -1, &sd_vmstate, sd);
-    return sd;
+
+    return SD_CARD(dev);
 }
 
 void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert)
@@ -666,8 +751,16 @@ static sd_rsp_type_t sd_normal_command(SDState *sd,
     /* Not interpreting this as an app command */
     sd->card_status &= ~APP_CMD;
 
-    if (sd_cmd_type[req.cmd] == sd_ac || sd_cmd_type[req.cmd] == sd_adtc)
+    if (sd_cmd_type[req.cmd & 0x3F] == sd_ac
+        || sd_cmd_type[req.cmd & 0x3F] == sd_adtc) {
         rca = req.arg >> 16;
+    }
+
+    /* CMD23 (set block count) must be immediately followed by CMD18 or CMD25
+     * if not, its effects are cancelled */
+    if (sd->multi_blk_cnt != 0 && !(req.cmd == 18 || req.cmd == 25)) {
+        sd->multi_blk_cnt = 0;
+    }
 
     DPRINTF("CMD%d 0x%08x state %d\n", req.cmd, req.arg, sd->state);
     switch (req.cmd) {
@@ -679,7 +772,7 @@ static sd_rsp_type_t sd_normal_command(SDState *sd,
 
         default:
             sd->state = sd_idle_state;
-            sd_reset(sd);
+            sd_reset(DEVICE(sd));
             return sd->spi ? sd_r1 : sd_r0;
         }
         break;
@@ -964,6 +1057,17 @@ static sd_rsp_type_t sd_normal_command(SDState *sd,
         }
         break;
 
+    case 23:    /* CMD23: SET_BLOCK_COUNT */
+        switch (sd->state) {
+        case sd_transfer_state:
+            sd->multi_blk_cnt = req.arg;
+            return sd_r1;
+
+        default:
+            break;
+        }
+        break;
+
     /* Block write commands (Class 4) */
     case 24:	/* CMD24:  WRITE_SINGLE_BLOCK */
         if (sd->spi)
@@ -1196,16 +1300,17 @@ static sd_rsp_type_t sd_normal_command(SDState *sd,
 
     default:
     bad_cmd:
-        fprintf(stderr, "SD: Unknown CMD%i\n", req.cmd);
+        qemu_log_mask(LOG_GUEST_ERROR, "SD: Unknown CMD%i\n", req.cmd);
         return sd_illegal;
 
     unimplemented_cmd:
         /* Commands that are recognised but not yet implemented in SPI mode.  */
-        fprintf(stderr, "SD: CMD%i not implemented in SPI mode\n", req.cmd);
+        qemu_log_mask(LOG_UNIMP, "SD: CMD%i not implemented in SPI mode\n",
+                      req.cmd);
         return sd_illegal;
     }
 
-    fprintf(stderr, "SD: CMD%i in a wrong state\n", req.cmd);
+    qemu_log_mask(LOG_GUEST_ERROR, "SD: CMD%i in a wrong state\n", req.cmd);
     return sd_illegal;
 }
 
@@ -1273,9 +1378,28 @@ static sd_rsp_type_t sd_app_command(SDState *sd,
         }
         switch (sd->state) {
         case sd_idle_state:
+            /* If it's the first ACMD41 since reset, we need to decide
+             * whether to power up. If this is not an enquiry ACMD41,
+             * we immediately report power on and proceed below to the
+             * ready state, but if it is, we set a timer to model a
+             * delay for power up. This works around a bug in EDK2
+             * UEFI, which sends an initial enquiry ACMD41, but
+             * assumes that the card is in ready state as soon as it
+             * sees the power up bit set. */
+            if (!(sd->ocr & OCR_POWER_UP)) {
+                if ((req.arg & ACMD41_ENQUIRY_MASK) != 0) {
+                    timer_del(sd->ocr_power_timer);
+                    sd_ocr_powerup(sd);
+                } else if (!timer_pending(sd->ocr_power_timer)) {
+                    timer_mod_ns(sd->ocr_power_timer,
+                                 (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                                  + OCR_POWER_DELAY_NS));
+                }
+            }
+
             /* We accept any voltage.  10000 V is nothing.
              *
-             * We don't model init delay so just advance straight to ready state
+             * Once we're powered up, we advance straight to ready state
              * unless it's an enquiry ACMD41 (bits 23:0 == 0).
              */
             if (req.arg & ACMD41_ENQUIRY_MASK) {
@@ -1318,7 +1442,7 @@ static sd_rsp_type_t sd_app_command(SDState *sd,
         return sd_normal_command(sd, req);
     }
 
-    fprintf(stderr, "SD: ACMD%i in a wrong state\n", req.cmd);
+    qemu_log_mask(LOG_GUEST_ERROR, "SD: ACMD%i in a wrong state\n", req.cmd);
     return sd_illegal;
 }
 
@@ -1338,7 +1462,8 @@ static int cmd_valid_while_locked(SDState *sd, SDRequest *req)
     if (req->cmd == 16 || req->cmd == 55) {
         return 1;
     }
-    return sd_cmd_class[req->cmd] == 0 || sd_cmd_class[req->cmd] == 7;
+    return sd_cmd_class[req->cmd & 0x3F] == 0
+            || sd_cmd_class[req->cmd & 0x3F] == 7;
 }
 
 int sd_do_command(SDState *sd, SDRequest *req,
@@ -1361,7 +1486,7 @@ int sd_do_command(SDState *sd, SDRequest *req,
         if (!cmd_valid_while_locked(sd, req)) {
             sd->card_status |= ILLEGAL_COMMAND;
             sd->expecting_acmd = false;
-            fprintf(stderr, "SD: Card is locked\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "SD: Card is locked\n");
             rtype = sd_illegal;
             goto send_response;
         }
@@ -1519,7 +1644,8 @@ void sd_write_data(SDState *sd, uint8_t value)
         return;
 
     if (sd->state != sd_receivingdata_state) {
-        fprintf(stderr, "sd_write_data: not in Receiving-Data state\n");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "sd_write_data: not in Receiving-Data state\n");
         return;
     }
 
@@ -1563,6 +1689,14 @@ void sd_write_data(SDState *sd, uint8_t value)
             sd->csd[14] |= 0x40;
 
             /* Bzzzzzzztt .... Operation complete.  */
+            if (sd->multi_blk_cnt != 0) {
+                if (--sd->multi_blk_cnt == 0) {
+                    /* Stop! */
+                    sd->state = sd_transfer_state;
+                    break;
+                }
+            }
+
             sd->state = sd_receivingdata_state;
         }
         break;
@@ -1630,7 +1764,7 @@ void sd_write_data(SDState *sd, uint8_t value)
         break;
 
     default:
-        fprintf(stderr, "sd_write_data: unknown command\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "sd_write_data: unknown command\n");
         break;
     }
 }
@@ -1645,7 +1779,8 @@ uint8_t sd_read_data(SDState *sd)
         return 0x00;
 
     if (sd->state != sd_sendingdata_state) {
-        fprintf(stderr, "sd_read_data: not in Sending-Data state\n");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "sd_read_data: not in Sending-Data state\n");
         return 0x00;
     }
 
@@ -1709,6 +1844,15 @@ uint8_t sd_read_data(SDState *sd)
         if (sd->data_offset >= io_len) {
             sd->data_start += io_len;
             sd->data_offset = 0;
+
+            if (sd->multi_blk_cnt != 0) {
+                if (--sd->multi_blk_cnt == 0) {
+                    /* Stop! */
+                    sd->state = sd_transfer_state;
+                    break;
+                }
+            }
+
             if (sd->data_start + io_len > sd->size) {
                 sd->card_status |= ADDRESS_ERROR;
                 break;
@@ -1747,7 +1891,7 @@ uint8_t sd_read_data(SDState *sd)
         break;
 
     default:
-        fprintf(stderr, "sd_read_data: unknown command\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "sd_read_data: unknown command\n");
         return 0x00;
     }
 
@@ -1763,3 +1907,73 @@ void sd_enable(SDState *sd, bool enable)
 {
     sd->enable = enable;
 }
+
+static void sd_instance_init(Object *obj)
+{
+    SDState *sd = SD_CARD(obj);
+
+    sd->enable = true;
+    sd->ocr_power_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sd_ocr_powerup, sd);
+}
+
+static void sd_realize(DeviceState *dev, Error **errp)
+{
+    SDState *sd = SD_CARD(dev);
+
+    if (sd->blk && blk_is_read_only(sd->blk)) {
+        error_setg(errp, "Cannot use read-only drive as SD card");
+        return;
+    }
+
+    sd->buf = blk_blockalign(sd->blk, 512);
+
+    if (sd->blk) {
+        blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
+    }
+}
+
+static Property sd_properties[] = {
+    DEFINE_PROP_DRIVE("drive", SDState, blk),
+    /* We do not model the chip select pin, so allow the board to select
+     * whether card should be in SSI or MMC/SD mode.  It is also up to the
+     * board to ensure that ssi transfers only occur when the chip select
+     * is asserted.  */
+    DEFINE_PROP_BOOL("spi", SDState, spi, false),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void sd_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SDCardClass *sc = SD_CARD_CLASS(klass);
+
+    dc->realize = sd_realize;
+    dc->props = sd_properties;
+    dc->vmsd = &sd_vmstate;
+    dc->reset = sd_reset;
+    dc->bus_type = TYPE_SD_BUS;
+
+    sc->do_command = sd_do_command;
+    sc->write_data = sd_write_data;
+    sc->read_data = sd_read_data;
+    sc->data_ready = sd_data_ready;
+    sc->enable = sd_enable;
+    sc->get_inserted = sd_get_inserted;
+    sc->get_readonly = sd_get_readonly;
+}
+
+static const TypeInfo sd_info = {
+    .name = TYPE_SD_CARD,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(SDState),
+    .class_size = sizeof(SDCardClass),
+    .class_init = sd_class_init,
+    .instance_init = sd_instance_init,
+};
+
+static void sd_register_types(void)
+{
+    type_register_static(&sd_info);
+}
+
+type_init(sd_register_types)
diff --git a/qemu/hw/sd/sdhci.h b/qemu/hw/sd/sdhci-internal.h
index 3352d23d6..161177cf3 100644
--- a/qemu/hw/sd/sdhci.h
+++ b/qemu/hw/sd/sdhci-internal.h
@@ -21,14 +21,10 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#ifndef SDHCI_INTERNAL_H
+#define SDHCI_INTERNAL_H
 
-#ifndef SDHCI_H
-#define SDHCI_H
-
-#include "qemu-common.h"
-#include "hw/pci/pci.h"
-#include "hw/sysbus.h"
-#include "hw/sd.h"
+#include "hw/sd/sdhci.h"
 
 /* R/W SDMA System Address register 0x0 */
 #define SDHC_SYSAD                     0x00
@@ -220,7 +216,7 @@
 #define SD_HOST_SPECv2_VERS             0x2401
 
 #define SDHC_REGISTERS_MAP_SIZE         0x100
-#define SDHC_INSERTION_DELAY            (get_ticks_per_sec())
+#define SDHC_INSERTION_DELAY            (NANOSECONDS_PER_SECOND)
 #define SDHC_TRANSFER_DELAY             100
 #define SDHC_ADMA_DESCS_PER_DELAY       5
 #define SDHC_CMD_RESPONSE               (3 << 0)
@@ -231,65 +227,6 @@ enum {
     sdhc_gap_write  = 2   /* SDHC stopped at block gap during write operation */
 };
 
-/* SD/MMC host controller state */
-typedef struct SDHCIState {
-    union {
-        PCIDevice pcidev;
-        SysBusDevice busdev;
-    };
-    SDState *card;
-    MemoryRegion iomem;
-
-    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
-    QEMUTimer *transfer_timer;
-    qemu_irq eject_cb;
-    qemu_irq ro_cb;
-    qemu_irq irq;
-
-    uint32_t sdmasysad;    /* SDMA System Address register */
-    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
-    uint16_t blkcnt;       /* Blocks count for current transfer */
-    uint32_t argument;     /* Command Argument Register */
-    uint16_t trnmod;       /* Transfer Mode Setting Register */
-    uint16_t cmdreg;       /* Command Register */
-    uint32_t rspreg[4];    /* Response Registers 0-3 */
-    uint32_t prnsts;       /* Present State Register */
-    uint8_t  hostctl;      /* Host Control Register */
-    uint8_t  pwrcon;       /* Power control Register */
-    uint8_t  blkgap;       /* Block Gap Control Register */
-    uint8_t  wakcon;       /* WakeUp Control Register */
-    uint16_t clkcon;       /* Clock control Register */
-    uint8_t  timeoutcon;   /* Timeout Control Register */
-    uint8_t  admaerr;      /* ADMA Error Status Register */
-    uint16_t norintsts;    /* Normal Interrupt Status Register */
-    uint16_t errintsts;    /* Error Interrupt Status Register */
-    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
-    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
-    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
-    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
-    uint16_t acmd12errsts; /* Auto CMD12 error status register */
-    uint64_t admasysaddr;  /* ADMA System Address Register */
-
-    uint32_t capareg;      /* Capabilities Register */
-    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
-    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
-    uint32_t buf_maxsz;
-    uint16_t data_count;   /* current element in FIFO buffer */
-    uint8_t  stopped_state;/* Current SDHC state */
-    /* Buffer Data Port Register - virtual access point to R and W buffers */
-    /* Software Reset Register - always reads as 0 */
-    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
-    /* Force Event Error Interrupt Register- write only */
-    /* RO Host Controller Version Register always reads as 0x2401 */
-} SDHCIState;
-
 extern const VMStateDescription sdhci_vmstate;
 
-#define TYPE_PCI_SDHCI "sdhci-pci"
-#define PCI_SDHCI(obj) OBJECT_CHECK(SDHCIState, (obj), TYPE_PCI_SDHCI)
-
-#define TYPE_SYSBUS_SDHCI "generic-sdhci"
-#define SYSBUS_SDHCI(obj)                               \
-     OBJECT_CHECK(SDHCIState, (obj), TYPE_SYSBUS_SDHCI)
-
-#endif /* SDHCI_H */
+#endif
diff --git a/qemu/hw/sd/sdhci.c b/qemu/hw/sd/sdhci.c
index e63367ba5..d28b5871f 100644
--- a/qemu/hw/sd/sdhci.c
+++ b/qemu/hw/sd/sdhci.c
@@ -22,38 +22,41 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
-
-#include "sdhci.h"
+#include "sdhci-internal.h"
 
 /* host controller debug messages */
 #ifndef SDHC_DEBUG
 #define SDHC_DEBUG                        0
 #endif
 
-#if SDHC_DEBUG == 0
-    #define DPRINT_L1(fmt, args...)       do { } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        do { } while (0)
-#elif SDHC_DEBUG == 1
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#else
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#endif
+#define DPRINT_L1(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define DPRINT_L2(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG > 1) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define ERRPRINT(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC ERROR: " fmt, ## args); \
+        } \
+    } while (0)
+
+#define TYPE_SDHCI_BUS "sdhci-bus"
+#define SDHCI_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_SDHCI_BUS)
 
 /* Default SD/MMC host controller features information, which will be
  * presented in CAPABILITIES register of generic SD host controller at reset.
@@ -145,9 +148,9 @@ static void sdhci_raise_insertion_irq(void *opaque)
     }
 }
 
-static void sdhci_insert_eject_cb(void *opaque, int irq, int level)
+static void sdhci_set_inserted(DeviceState *dev, bool level)
 {
-    SDHCIState *s = (SDHCIState *)opaque;
+    SDHCIState *s = (SDHCIState *)dev;
     DPRINT_L1("Card state changed: %s!\n", level ? "insert" : "eject");
 
     if ((s->norintsts & SDHC_NIS_REMOVE) && level) {
@@ -172,9 +175,9 @@ static void sdhci_insert_eject_cb(void *opaque, int irq, int level)
     }
 }
 
-static void sdhci_card_readonly_cb(void *opaque, int irq, int level)
+static void sdhci_set_readonly(DeviceState *dev, bool level)
 {
-    SDHCIState *s = (SDHCIState *)opaque;
+    SDHCIState *s = (SDHCIState *)dev;
 
     if (level) {
         s->prnsts &= ~SDHC_WRITE_PROTECT;
@@ -186,6 +189,8 @@ static void sdhci_card_readonly_cb(void *opaque, int irq, int level)
 
 static void sdhci_reset(SDHCIState *s)
 {
+    DeviceState *dev = DEVICE(s);
+
     timer_del(s->insert_timer);
     timer_del(s->transfer_timer);
     /* Set all registers to 0. Capabilities registers are not cleared
@@ -193,9 +198,28 @@ static void sdhci_reset(SDHCIState *s)
      * initialization */
     memset(&s->sdmasysad, 0, (uintptr_t)&s->capareg - (uintptr_t)&s->sdmasysad);
 
-    sd_set_cb(s->card, s->ro_cb, s->eject_cb);
+    /* Reset other state based on current card insertion/readonly status */
+    sdhci_set_inserted(dev, sdbus_get_inserted(&s->sdbus));
+    sdhci_set_readonly(dev, sdbus_get_readonly(&s->sdbus));
+
     s->data_count = 0;
     s->stopped_state = sdhc_not_stopped;
+    s->pending_insert_state = false;
+}
+
+static void sdhci_poweron_reset(DeviceState *dev)
+{
+    /* QOM (ie power-on) reset. This is identical to reset
+     * commanded via device register apart from handling of the
+     * 'pending insert on powerup' quirk.
+     */
+    SDHCIState *s = (SDHCIState *)dev;
+
+    sdhci_reset(s);
+
+    if (s->pending_insert_quirk) {
+        s->pending_insert_state = true;
+    }
 }
 
 static void sdhci_data_transfer(void *opaque);
@@ -211,7 +235,7 @@ static void sdhci_send_command(SDHCIState *s)
     request.cmd = s->cmdreg >> 8;
     request.arg = s->argument;
     DPRINT_L1("sending CMD%u ARG[0x%08x]\n", request.cmd, request.arg);
-    rlen = sd_do_command(s->card, &request, response);
+    rlen = sdbus_do_command(&s->sdbus, &request, response);
 
     if (s->cmdreg & SDHC_CMD_RESPONSE) {
         if (rlen == 4) {
@@ -243,9 +267,6 @@ static void sdhci_send_command(SDHCIState *s)
             (s->cmdreg & SDHC_CMD_RESPONSE) == SDHC_CMD_RSP_WITH_BUSY) {
             s->norintsts |= SDHC_NIS_TRSCMP;
         }
-    } else if (rlen != 0 && (s->errintstsen & SDHC_EISEN_CMDIDX)) {
-        s->errintsts |= SDHC_EIS_CMDIDX;
-        s->norintsts |= SDHC_NIS_ERR;
     }
 
     if (s->norintstsen & SDHC_NISEN_CMDCMP) {
@@ -270,7 +291,7 @@ static void sdhci_end_transfer(SDHCIState *s)
         request.cmd = 0x0C;
         request.arg = 0;
         DPRINT_L1("Automatically issue CMD%d %08x\n", request.cmd, request.arg);
-        sd_do_command(s->card, &request, response);
+        sdbus_do_command(&s->sdbus, &request, response);
         /* Auto CMD12 response goes to the upper Response register */
         s->rspreg[3] = (response[0] << 24) | (response[1] << 16) |
                 (response[2] << 8) | response[3];
@@ -302,7 +323,7 @@ static void sdhci_read_block_from_card(SDHCIState *s)
     }
 
     for (index = 0; index < (s->blksize & 0x0fff); index++) {
-        s->fifo_buffer[index] = sd_read_data(s->card);
+        s->fifo_buffer[index] = sdbus_read_data(&s->sdbus);
     }
 
     /* New data now available for READ through Buffer Port Register */
@@ -395,7 +416,7 @@ static void sdhci_write_block_to_card(SDHCIState *s)
     }
 
     for (index = 0; index < (s->blksize & 0x0fff); index++) {
-        sd_write_data(s->card, s->fifo_buffer[index]);
+        sdbus_write_data(&s->sdbus, s->fifo_buffer[index]);
     }
 
     /* Next data can be written through BUFFER DATORT register */
@@ -477,7 +498,7 @@ static void sdhci_sdma_transfer_multi_blocks(SDHCIState *s)
         while (s->blkcnt) {
             if (s->data_count == 0) {
                 for (n = 0; n < block_size; n++) {
-                    s->fifo_buffer[n] = sd_read_data(s->card);
+                    s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
                 }
             }
             begin = s->data_count;
@@ -518,7 +539,7 @@ static void sdhci_sdma_transfer_multi_blocks(SDHCIState *s)
             s->sdmasysad += s->data_count - begin;
             if (s->data_count == block_size) {
                 for (n = 0; n < block_size; n++) {
-                    sd_write_data(s->card, s->fifo_buffer[n]);
+                    sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
                 }
                 s->data_count = 0;
                 if (s->trnmod & SDHC_TRNS_BLK_CNT_EN) {
@@ -550,7 +571,7 @@ static void sdhci_sdma_transfer_single_block(SDHCIState *s)
 
     if (s->trnmod & SDHC_TRNS_READ) {
         for (n = 0; n < datacnt; n++) {
-            s->fifo_buffer[n] = sd_read_data(s->card);
+            s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
         }
         dma_memory_write(&address_space_memory, s->sdmasysad, s->fifo_buffer,
                          datacnt);
@@ -558,7 +579,7 @@ static void sdhci_sdma_transfer_single_block(SDHCIState *s)
         dma_memory_read(&address_space_memory, s->sdmasysad, s->fifo_buffer,
                         datacnt);
         for (n = 0; n < datacnt; n++) {
-            sd_write_data(s->card, s->fifo_buffer[n]);
+            sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
         }
     }
 
@@ -662,7 +683,7 @@ static void sdhci_do_adma(SDHCIState *s)
                 while (length) {
                     if (s->data_count == 0) {
                         for (n = 0; n < block_size; n++) {
-                            s->fifo_buffer[n] = sd_read_data(s->card);
+                            s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
                         }
                     }
                     begin = s->data_count;
@@ -703,7 +724,7 @@ static void sdhci_do_adma(SDHCIState *s)
                     dscr.addr += s->data_count - begin;
                     if (s->data_count == block_size) {
                         for (n = 0; n < block_size; n++) {
-                            sd_write_data(s->card, s->fifo_buffer[n]);
+                            sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
                         }
                         s->data_count = 0;
                         if (s->trnmod & SDHC_TRNS_BLK_CNT_EN) {
@@ -719,7 +740,8 @@ static void sdhci_do_adma(SDHCIState *s)
             break;
         case SDHC_ADMA_ATTR_ACT_LINK:   /* link to next descriptor table */
             s->admasysaddr = dscr.addr;
-            DPRINT_L1("ADMA link: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA link: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             break;
         default:
             s->admasysaddr += dscr.incr;
@@ -727,7 +749,8 @@ static void sdhci_do_adma(SDHCIState *s)
         }
 
         if (dscr.attr & SDHC_ADMA_ATTR_INT) {
-            DPRINT_L1("ADMA interrupt: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA interrupt: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             if (s->norintstsen & SDHC_NISEN_DMA) {
                 s->norintsts |= SDHC_NIS_DMA;
             }
@@ -815,7 +838,7 @@ static void sdhci_data_transfer(void *opaque)
             break;
         }
     } else {
-        if ((s->trnmod & SDHC_TRNS_READ) && sd_data_ready(s->card)) {
+        if ((s->trnmod & SDHC_TRNS_READ) && sdbus_data_ready(&s->sdbus)) {
             s->prnsts |= SDHC_DOING_READ | SDHC_DATA_INHIBIT |
                     SDHC_DAT_LINE_ACTIVE;
             sdhci_read_block_from_card(s);
@@ -829,7 +852,7 @@ static void sdhci_data_transfer(void *opaque)
 
 static bool sdhci_can_issue_command(SDHCIState *s)
 {
-    if (!SDHC_CLOCK_IS_ON(s->clkcon) || !(s->pwrcon & SDHC_POWER_ON) ||
+    if (!SDHC_CLOCK_IS_ON(s->clkcon) ||
         (((s->prnsts & SDHC_DATA_INHIBIT) || s->stopped_state) &&
         ((s->cmdreg & SDHC_CMD_DATA_PRESENT) ||
         ((s->cmdreg & SDHC_CMD_RESPONSE) == SDHC_CMD_RSP_WITH_BUSY &&
@@ -1006,6 +1029,16 @@ sdhci_write(void *opaque, hwaddr offset, uint64_t val, unsigned size)
             MASKED_WRITE(s->blksize, mask, value);
             MASKED_WRITE(s->blkcnt, mask >> 16, value >> 16);
         }
+
+        /* Limit block size to the maximum buffer size */
+        if (extract32(s->blksize, 0, 12) > s->buf_maxsz) {
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Size 0x%x is larger than " \
+                          "the maximum buffer 0x%x", __func__, s->blksize,
+                          s->buf_maxsz);
+
+            s->blksize = deposit32(s->blksize, 0, 12, s->buf_maxsz);
+        }
+
         break;
     case SDHC_ARGUMENT:
         MASKED_WRITE(s->argument, mask, value);
@@ -1078,6 +1111,13 @@ sdhci_write(void *opaque, hwaddr offset, uint64_t val, unsigned size)
         } else {
             s->norintsts &= ~SDHC_NIS_ERR;
         }
+        /* Quirk for Raspberry Pi: pending card insert interrupt
+         * appears when first enabled after power on */
+        if ((s->norintstsen & SDHC_NISEN_INSERT) && s->pending_insert_state) {
+            assert(s->pending_insert_quirk);
+            s->norintsts |= SDHC_NIS_INSERT;
+            s->pending_insert_state = false;
+        }
         sdhci_update_irq(s);
         break;
     case SDHC_NORINTSIGEN:
@@ -1144,17 +1184,8 @@ static inline unsigned int sdhci_get_fifolen(SDHCIState *s)
 
 static void sdhci_initfn(SDHCIState *s)
 {
-    DriveInfo *di;
-
-    /* FIXME use a qdev drive property instead of drive_get_next() */
-    di = drive_get_next(IF_SD);
-    s->card = sd_init(di ? blk_by_legacy_dinfo(di) : NULL, false);
-    if (s->card == NULL) {
-        exit(1);
-    }
-    s->eject_cb = qemu_allocate_irq(sdhci_insert_eject_cb, s, 0);
-    s->ro_cb = qemu_allocate_irq(sdhci_card_readonly_cb, s, 0);
-    sd_set_cb(s->card, s->ro_cb, s->eject_cb);
+    qbus_create_inplace(&s->sdbus, sizeof(s->sdbus),
+                        TYPE_SDHCI_BUS, DEVICE(s), "sd-bus");
 
     s->insert_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sdhci_raise_insertion_irq, s);
     s->transfer_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sdhci_data_transfer, s);
@@ -1169,12 +1200,28 @@ static void sdhci_uninitfn(SDHCIState *s)
     qemu_free_irq(s->eject_cb);
     qemu_free_irq(s->ro_cb);
 
-    if (s->fifo_buffer) {
-        g_free(s->fifo_buffer);
-        s->fifo_buffer = NULL;
-    }
+    g_free(s->fifo_buffer);
+    s->fifo_buffer = NULL;
 }
 
+static bool sdhci_pending_insert_vmstate_needed(void *opaque)
+{
+    SDHCIState *s = opaque;
+
+    return s->pending_insert_state;
+}
+
+static const VMStateDescription sdhci_pending_insert_vmstate = {
+    .name = "sdhci/pending-insert",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = sdhci_pending_insert_vmstate_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(pending_insert_state, SDHCIState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 const VMStateDescription sdhci_vmstate = {
     .name = "sdhci",
     .version_id = 1,
@@ -1209,12 +1256,16 @@ const VMStateDescription sdhci_vmstate = {
         VMSTATE_TIMER_PTR(insert_timer, SDHCIState),
         VMSTATE_TIMER_PTR(transfer_timer, SDHCIState),
         VMSTATE_END_OF_LIST()
-    }
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &sdhci_pending_insert_vmstate,
+        NULL
+    },
 };
 
 /* Capabilities registers provide information on supported features of this
  * specific host controller implementation */
-static Property sdhci_properties[] = {
+static Property sdhci_pci_properties[] = {
     DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
             SDHC_CAPAB_REG_DEFAULT),
     DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
@@ -1253,9 +1304,8 @@ static void sdhci_pci_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_SYSTEM_SDHCI;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->vmsd = &sdhci_vmstate;
-    dc->props = sdhci_properties;
-    /* Reason: realize() method uses drive_get_next() */
-    dc->cannot_instantiate_with_device_add_yet = true;
+    dc->props = sdhci_pci_properties;
+    dc->reset = sdhci_poweron_reset;
 }
 
 static const TypeInfo sdhci_pci_info = {
@@ -1265,9 +1315,19 @@ static const TypeInfo sdhci_pci_info = {
     .class_init = sdhci_pci_class_init,
 };
 
+static Property sdhci_sysbus_properties[] = {
+    DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
+            SDHC_CAPAB_REG_DEFAULT),
+    DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
+    DEFINE_PROP_BOOL("pending-insert-quirk", SDHCIState, pending_insert_quirk,
+                     false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void sdhci_sysbus_init(Object *obj)
 {
     SDHCIState *s = SYSBUS_SDHCI(obj);
+
     sdhci_initfn(s);
 }
 
@@ -1295,10 +1355,9 @@ static void sdhci_sysbus_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     dc->vmsd = &sdhci_vmstate;
-    dc->props = sdhci_properties;
+    dc->props = sdhci_sysbus_properties;
     dc->realize = sdhci_sysbus_realize;
-    /* Reason: instance_init() method uses drive_get_next() */
-    dc->cannot_instantiate_with_device_add_yet = true;
+    dc->reset = sdhci_poweron_reset;
 }
 
 static const TypeInfo sdhci_sysbus_info = {
@@ -1310,10 +1369,26 @@ static const TypeInfo sdhci_sysbus_info = {
     .class_init = sdhci_sysbus_class_init,
 };
 
+static void sdhci_bus_class_init(ObjectClass *klass, void *data)
+{
+    SDBusClass *sbc = SD_BUS_CLASS(klass);
+
+    sbc->set_inserted = sdhci_set_inserted;
+    sbc->set_readonly = sdhci_set_readonly;
+}
+
+static const TypeInfo sdhci_bus_info = {
+    .name = TYPE_SDHCI_BUS,
+    .parent = TYPE_SD_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_init = sdhci_bus_class_init,
+};
+
 static void sdhci_register_types(void)
 {
     type_register_static(&sdhci_pci_info);
     type_register_static(&sdhci_sysbus_info);
+    type_register_static(&sdhci_bus_info);
 }
 
 type_init(sdhci_register_types)
diff --git a/qemu/hw/sd/ssi-sd.c b/qemu/hw/sd/ssi-sd.c
index e4b2d4f83..075e4ed5d 100644
--- a/qemu/hw/sd/ssi-sd.c
+++ b/qemu/hw/sd/ssi-sd.c
@@ -10,10 +10,11 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/ssi.h"
-#include "hw/sd.h"
+#include "hw/ssi/ssi.h"
+#include "hw/sd/sd.h"
 
 //#define DEBUG_SSI_SD 1
 
diff --git a/qemu/hw/sh4/r2d.c b/qemu/hw/sh4/r2d.c
index 5e22ed79b..db373c70c 100644
--- a/qemu/hw/sh4/r2d.c
+++ b/qemu/hw/sh4/r2d.c
@@ -23,6 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
@@ -255,7 +259,7 @@ static void r2d_init(MachineState *machine)
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate memory space */
-    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_abort);
+    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(sdram);
     memory_region_add_subregion(address_space_mem, SDRAM_BASE, sdram);
     /* Register peripherals */
@@ -338,9 +342,9 @@ static void r2d_init(MachineState *machine)
         }
 
         /* initialization which should be done by firmware */
-        boot_params.loader_type = 1;
-        boot_params.initrd_start = INITRD_LOAD_OFFSET;
-        boot_params.initrd_size = initrd_size;
+        boot_params.loader_type = tswap32(1);
+        boot_params.initrd_start = tswap32(INITRD_LOAD_OFFSET);
+        boot_params.initrd_size = tswap32(initrd_size);
     }
 
     if (kernel_cmdline) {
@@ -354,15 +358,10 @@ static void r2d_init(MachineState *machine)
                        SDRAM_BASE + BOOT_PARAMS_OFFSET);
 }
 
-static QEMUMachine r2d_machine = {
-    .name = "r2d",
-    .desc = "r2d-plus board",
-    .init = r2d_init,
-};
-
-static void r2d_machine_init(void)
+static void r2d_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&r2d_machine);
+    mc->desc = "r2d-plus board";
+    mc->init = r2d_init;
 }
 
-machine_init(r2d_machine_init);
+DEFINE_MACHINE("r2d", r2d_machine_init)
diff --git a/qemu/hw/sh4/sh7750.c b/qemu/hw/sh4/sh7750.c
index 5dda5de34..a1ea760f6 100644
--- a/qemu/hw/sh4/sh7750.c
+++ b/qemu/hw/sh4/sh7750.c
@@ -22,7 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/hw/sh4/sh7750_regnames.c b/qemu/hw/sh4/sh7750_regnames.c
index 52ac1cc78..34b4f99b8 100644
--- a/qemu/hw/sh4/sh7750_regnames.c
+++ b/qemu/hw/sh4/sh7750_regnames.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sh7750_regs.h"
diff --git a/qemu/hw/sh4/sh_pci.c b/qemu/hw/sh4/sh_pci.c
index a2f6d9e0b..e820a3230 100644
--- a/qemu/hw/sh4/sh_pci.c
+++ b/qemu/hw/sh4/sh_pci.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/sh4/sh.h"
 #include "hw/pci/pci.h"
@@ -151,12 +152,11 @@ static int sh_pci_device_init(SysBusDevice *dev)
     return 0;
 }
 
-static int sh_pci_host_init(PCIDevice *d)
+static void sh_pci_host_realize(PCIDevice *d, Error **errp)
 {
     pci_set_word(d->config + PCI_COMMAND, PCI_COMMAND_WAIT);
     pci_set_word(d->config + PCI_STATUS, PCI_STATUS_CAP_LIST |
                  PCI_STATUS_FAST_BACK | PCI_STATUS_DEVSEL_MEDIUM);
-    return 0;
 }
 
 static void sh_pci_host_class_init(ObjectClass *klass, void *data)
@@ -164,7 +164,7 @@ static void sh_pci_host_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = sh_pci_host_init;
+    k->realize = sh_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_HITACHI;
     k->device_id = PCI_DEVICE_ID_HITACHI_SH7751R;
     /*
diff --git a/qemu/hw/sh4/shix.c b/qemu/hw/sh4/shix.c
index f93f98e56..ccc9e7589 100644
--- a/qemu/hw/sh4/shix.c
+++ b/qemu/hw/sh4/shix.c
@@ -27,6 +27,10 @@
 
    More information in target-sh4/README.sh4
 */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/sysemu.h"
@@ -59,16 +63,16 @@ static void shix_init(MachineState *machine)
     }
 
     /* Allocate memory space */
-    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_abort);
+    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(sysmem, 0x00000000, rom);
     memory_region_init_ram(&sdram[0], NULL, "shix.sdram1", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[0]);
     memory_region_add_subregion(sysmem, 0x08000000, &sdram[0]);
     memory_region_init_ram(&sdram[1], NULL, "shix.sdram2", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[1]);
     memory_region_add_subregion(sysmem, 0x0c000000, &sdram[1]);
 
@@ -87,16 +91,11 @@ static void shix_init(MachineState *machine)
     tc58128_init(s, "shix_linux_nand.bin", NULL);
 }
 
-static QEMUMachine shix_machine = {
-    .name = "shix",
-    .desc = "shix card",
-    .init = shix_init,
-    .is_default = 1,
-};
-
-static void shix_machine_init(void)
+static void shix_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&shix_machine);
+    mc->desc = "shix card";
+    mc->init = shix_init;
+    mc->is_default = 1;
 }
 
-machine_init(shix_machine_init);
+DEFINE_MACHINE("shix", shix_machine_init)
diff --git a/qemu/hw/smbios/Makefile.objs b/qemu/hw/smbios/Makefile.objs
new file mode 100644
index 000000000..f69a92f96
--- /dev/null
+++ b/qemu/hw/smbios/Makefile.objs
@@ -0,0 +1 @@
+common-obj-$(CONFIG_SMBIOS) += smbios.o
diff --git a/qemu/hw/i386/smbios.c b/qemu/hw/smbios/smbios.c
index 1341e0234..cb8a11110 100644
--- a/qemu/hw/i386/smbios.c
+++ b/qemu/hw/smbios/smbios.c
@@ -15,14 +15,15 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpus.h"
-#include "hw/i386/pc.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/loader.h"
-
+#include "exec/cpu-common.h"
 
 /* legacy structures and constants for <= 2.0 machines */
 struct smbios_header {
@@ -56,7 +57,9 @@ static uint8_t *smbios_tables;
 static size_t smbios_tables_len;
 static unsigned smbios_table_max;
 static unsigned smbios_table_cnt;
-static struct smbios_entry_point ep;
+static SmbiosEntryPointType smbios_ep_type = SMBIOS_ENTRY_POINT_21;
+
+static SmbiosEntryPoint ep;
 
 static int smbios_type4_count = 0;
 static bool smbios_immutable;
@@ -318,7 +321,7 @@ static void smbios_register_config(void)
     qemu_add_opts(&qemu_smbios_opts);
 }
 
-machine_init(smbios_register_config);
+opts_init(smbios_register_config);
 
 static void smbios_validate_table(void)
 {
@@ -772,11 +775,12 @@ void smbios_set_cpuid(uint32_t version, uint32_t features)
 
 void smbios_set_defaults(const char *manufacturer, const char *product,
                          const char *version, bool legacy_mode,
-                         bool uuid_encoded)
+                         bool uuid_encoded, SmbiosEntryPointType ep_type)
 {
     smbios_have_defaults = true;
     smbios_legacy = legacy_mode;
     smbios_uuid_encoded = uuid_encoded;
+    smbios_ep_type = ep_type;
 
     /* drop unwanted version of command-line file blob(s) */
     if (smbios_legacy) {
@@ -809,32 +813,61 @@ void smbios_set_defaults(const char *manufacturer, const char *product,
 
 static void smbios_entry_point_setup(void)
 {
-    memcpy(ep.anchor_string, "_SM_", 4);
-    memcpy(ep.intermediate_anchor_string, "_DMI_", 5);
-    ep.length = sizeof(struct smbios_entry_point);
-    ep.entry_point_revision = 0; /* formatted_area reserved, per spec v2.1+ */
-    memset(ep.formatted_area, 0, 5);
-
-    /* compliant with smbios spec v2.8 */
-    ep.smbios_major_version = 2;
-    ep.smbios_minor_version = 8;
-    ep.smbios_bcd_revision = 0x28;
-
-    /* set during table construction, but BIOS may override: */
-    ep.structure_table_length = cpu_to_le16(smbios_tables_len);
-    ep.max_structure_size = cpu_to_le16(smbios_table_max);
-    ep.number_of_structures = cpu_to_le16(smbios_table_cnt);
-
-    /* BIOS must recalculate: */
-    ep.checksum = 0;
-    ep.intermediate_checksum = 0;
-    ep.structure_table_address = cpu_to_le32(0);
+    switch (smbios_ep_type) {
+    case SMBIOS_ENTRY_POINT_21:
+        memcpy(ep.ep21.anchor_string, "_SM_", 4);
+        memcpy(ep.ep21.intermediate_anchor_string, "_DMI_", 5);
+        ep.ep21.length = sizeof(struct smbios_21_entry_point);
+        ep.ep21.entry_point_revision = 0; /* formatted_area reserved */
+        memset(ep.ep21.formatted_area, 0, 5);
+
+        /* compliant with smbios spec v2.8 */
+        ep.ep21.smbios_major_version = 2;
+        ep.ep21.smbios_minor_version = 8;
+        ep.ep21.smbios_bcd_revision = 0x28;
+
+        /* set during table construction, but BIOS may override: */
+        ep.ep21.structure_table_length = cpu_to_le16(smbios_tables_len);
+        ep.ep21.max_structure_size = cpu_to_le16(smbios_table_max);
+        ep.ep21.number_of_structures = cpu_to_le16(smbios_table_cnt);
+
+        /* BIOS must recalculate */
+        ep.ep21.checksum = 0;
+        ep.ep21.intermediate_checksum = 0;
+        ep.ep21.structure_table_address = cpu_to_le32(0);
+
+        break;
+    case SMBIOS_ENTRY_POINT_30:
+        memcpy(ep.ep30.anchor_string, "_SM3_", 5);
+        ep.ep30.length = sizeof(struct smbios_30_entry_point);
+        ep.ep30.entry_point_revision = 1;
+        ep.ep30.reserved = 0;
+
+        /* compliant with smbios spec 3.0 */
+        ep.ep30.smbios_major_version = 3;
+        ep.ep30.smbios_minor_version = 0;
+        ep.ep30.smbios_doc_rev = 0;
+
+        /* set during table construct, but BIOS might override */
+        ep.ep30.structure_table_max_size = cpu_to_le32(smbios_tables_len);
+
+        /* BIOS must recalculate */
+        ep.ep30.checksum = 0;
+        ep.ep30.structure_table_address = cpu_to_le64(0);
+
+        break;
+    default:
+        abort();
+        break;
+    }
 }
 
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
                        uint8_t **anchor, size_t *anchor_len)
 {
-    unsigned i, dimm_cnt, instance;
+    unsigned i, dimm_cnt;
 
     if (smbios_legacy) {
         *tables = *anchor = NULL;
@@ -867,11 +900,9 @@ void smbios_get_tables(uint8_t **tables, size_t *tables_len,
             smbios_build_type_17_table(i, GET_DIMM_SZ);
         }
 
-        for (i = 0, instance = 0; i < e820_get_num_entries(); i++) {
-            uint64_t address, length;
-            if (e820_get_entry(i, E820_RAM, &address, &length)) {
-                smbios_build_type_19_table(instance++, address, length);
-            }
+        for (i = 0; i < mem_array_size; i++) {
+            smbios_build_type_19_table(i, mem_array[i].address,
+                                       mem_array[i].length);
         }
 
         smbios_build_type_32_table();
@@ -886,7 +917,15 @@ void smbios_get_tables(uint8_t **tables, size_t *tables_len,
     *tables = smbios_tables;
     *tables_len = smbios_tables_len;
     *anchor = (uint8_t *)&ep;
-    *anchor_len = sizeof(struct smbios_entry_point);
+
+    /* calculate length based on anchor string */
+    if (!strncmp((char *)&ep, "_SM_", 4)) {
+        *anchor_len = sizeof(struct smbios_21_entry_point);
+    } else if (!strncmp((char *)&ep, "_SM3_", 5)) {
+        *anchor_len = sizeof(struct smbios_30_entry_point);
+    } else {
+        abort();
+    }
 }
 
 static void save_opt(const char **dest, QemuOpts *opts, const char *name)
@@ -900,7 +939,6 @@ static void save_opt(const char **dest, QemuOpts *opts, const char *name)
 
 void smbios_entry_add(QemuOpts *opts)
 {
-    Error *local_err = NULL;
     const char *val;
 
     assert(!smbios_immutable);
@@ -911,11 +949,7 @@ void smbios_entry_add(QemuOpts *opts)
         int size;
         struct smbios_table *table; /* legacy mode only */
 
-        qemu_opts_validate(opts, qemu_smbios_file_opts, &local_err);
-        if (local_err) {
-            error_report_err(local_err);
-            exit(1);
-        }
+        qemu_opts_validate(opts, qemu_smbios_file_opts, &error_fatal);
 
         size = get_image_size(val);
         if (size == -1 || size < sizeof(struct smbios_structure_header)) {
@@ -997,11 +1031,7 @@ void smbios_entry_add(QemuOpts *opts)
 
         switch (type) {
         case 0:
-            qemu_opts_validate(opts, qemu_smbios_type0_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type0_opts, &error_fatal);
             save_opt(&type0.vendor, opts, "vendor");
             save_opt(&type0.version, opts, "version");
             save_opt(&type0.date, opts, "date");
@@ -1017,11 +1047,7 @@ void smbios_entry_add(QemuOpts *opts)
             }
             return;
         case 1:
-            qemu_opts_validate(opts, qemu_smbios_type1_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type1_opts, &error_fatal);
             save_opt(&type1.manufacturer, opts, "manufacturer");
             save_opt(&type1.product, opts, "product");
             save_opt(&type1.version, opts, "version");
@@ -1039,11 +1065,7 @@ void smbios_entry_add(QemuOpts *opts)
             }
             return;
         case 2:
-            qemu_opts_validate(opts, qemu_smbios_type2_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type2_opts, &error_fatal);
             save_opt(&type2.manufacturer, opts, "manufacturer");
             save_opt(&type2.product, opts, "product");
             save_opt(&type2.version, opts, "version");
@@ -1052,11 +1074,7 @@ void smbios_entry_add(QemuOpts *opts)
             save_opt(&type2.location, opts, "location");
             return;
         case 3:
-            qemu_opts_validate(opts, qemu_smbios_type3_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type3_opts, &error_fatal);
             save_opt(&type3.manufacturer, opts, "manufacturer");
             save_opt(&type3.version, opts, "version");
             save_opt(&type3.serial, opts, "serial");
@@ -1064,11 +1082,7 @@ void smbios_entry_add(QemuOpts *opts)
             save_opt(&type3.sku, opts, "sku");
             return;
         case 4:
-            qemu_opts_validate(opts, qemu_smbios_type4_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type4_opts, &error_fatal);
             save_opt(&type4.sock_pfx, opts, "sock_pfx");
             save_opt(&type4.manufacturer, opts, "manufacturer");
             save_opt(&type4.version, opts, "version");
@@ -1077,11 +1091,7 @@ void smbios_entry_add(QemuOpts *opts)
             save_opt(&type4.part, opts, "part");
             return;
         case 17:
-            qemu_opts_validate(opts, qemu_smbios_type17_opts, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            qemu_opts_validate(opts, qemu_smbios_type17_opts, &error_fatal);
             save_opt(&type17.loc_pfx, opts, "loc_pfx");
             save_opt(&type17.bank, opts, "bank");
             save_opt(&type17.manufacturer, opts, "manufacturer");
diff --git a/qemu/hw/sparc/leon3.c b/qemu/hw/sparc/leon3.c
index 7f5dcd6d8..dbae41f3a 100644
--- a/qemu/hw/sparc/leon3.c
+++ b/qemu/hw/sparc/leon3.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
@@ -156,7 +160,7 @@ static void leon3_generic_hw_init(MachineState *machine)
 
     /* Allocate BIOS */
     prom_size = 8 * 1024 * 1024; /* 8Mb */
-    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_abort);
+    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_fatal);
     vmstate_register_ram_global(prom);
     memory_region_set_readonly(prom, true);
     memory_region_add_subregion(address_space_mem, 0x00000000, prom);
@@ -193,7 +197,7 @@ static void leon3_generic_hw_init(MachineState *machine)
         uint64_t entry;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1 /* big endian */, ELF_MACHINE, 0);
+                               1 /* big endian */, EM_SPARC, 0, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "qemu: could not load kernel '%s'\n",
                     kernel_filename);
@@ -216,15 +220,10 @@ static void leon3_generic_hw_init(MachineState *machine)
     }
 }
 
-static QEMUMachine leon3_generic_machine = {
-    .name     = "leon3_generic",
-    .desc     = "Leon-3 generic",
-    .init     = leon3_generic_hw_init,
-};
-
-static void leon3_machine_init(void)
+static void leon3_generic_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&leon3_generic_machine);
+    mc->desc = "Leon-3 generic";
+    mc->init = leon3_generic_hw_init;
 }
 
-machine_init(leon3_machine_init);
+DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)
diff --git a/qemu/hw/sparc/sun4m.c b/qemu/hw/sparc/sun4m.c
index 68ac4d8bb..7bfc00abc 100644
--- a/qemu/hw/sparc/sun4m.c
+++ b/qemu/hw/sparc/sun4m.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
@@ -42,6 +46,7 @@
 #include "elf.h"
 #include "sysemu/block-backend.h"
 #include "trace.h"
+#include "qemu/cutils.h"
 
 /*
  * Sun4m architecture was used in the following machines:
@@ -95,29 +100,7 @@ struct sun4m_hwdef {
     uint8_t nvram_machine_id;
 };
 
-int DMA_get_channel_mode (int nchan)
-{
-    return 0;
-}
-int DMA_read_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-int DMA_write_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-void DMA_hold_DREQ (int nchan) {}
-void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
-
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
-{
-}
-
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
+void DMA_init(ISABus *bus, int high_page_enable)
 {
 }
 
@@ -300,7 +283,7 @@ static unsigned long sun4m_load_kernel(const char *kernel_filename,
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, NULL, 1, EM_SPARC, 0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, KERNEL_LOAD_ADDR,
                                     RAM_size - KERNEL_LOAD_ADDR, bswap_needed,
@@ -641,7 +624,7 @@ static int idreg_init1(SysBusDevice *dev)
     IDRegState *s = MACIO_ID_REGISTER(dev);
 
     memory_region_init_ram(&s->mem, OBJECT(s),
-                           "sun4m.idreg", sizeof(idreg_data), &error_abort);
+                           "sun4m.idreg", sizeof(idreg_data), &error_fatal);
     vmstate_register_ram_global(&s->mem);
     memory_region_set_readonly(&s->mem, true);
     sysbus_init_mmio(dev, &s->mem);
@@ -688,7 +671,7 @@ static int afx_init1(SysBusDevice *dev)
 {
     AFXState *s = TCX_AFX(dev);
 
-    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_abort);
+    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_fatal);
     vmstate_register_ram_global(&s->mem);
     sysbus_init_mmio(dev, &s->mem);
     return 0;
@@ -744,7 +727,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr, NULL,
-                       NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, 1, EM_SPARC, 0, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -763,7 +746,7 @@ static int prom_init1(SysBusDevice *dev)
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4m.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -1420,80 +1403,152 @@ static void sbook_init(MachineState *machine)
     sun4m_hw_init(&sun4m_hwdefs[8], machine);
 }
 
-static QEMUMachine ss5_machine = {
-    .name = "SS-5",
-    .desc = "Sun4m platform, SPARCstation 5",
-    .init = ss5_init,
-    .block_default_type = IF_SCSI,
-    .is_default = 1,
-    .default_boot_order = "c",
+static void ss5_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 5";
+    mc->init = ss5_init;
+    mc->block_default_type = IF_SCSI;
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss5_type = {
+    .name = MACHINE_TYPE_NAME("SS-5"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss5_class_init,
 };
 
-static QEMUMachine ss10_machine = {
-    .name = "SS-10",
-    .desc = "Sun4m platform, SPARCstation 10",
-    .init = ss10_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss10_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 10";
+    mc->init = ss10_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss10_type = {
+    .name = MACHINE_TYPE_NAME("SS-10"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss10_class_init,
 };
 
-static QEMUMachine ss600mp_machine = {
-    .name = "SS-600MP",
-    .desc = "Sun4m platform, SPARCserver 600MP",
-    .init = ss600mp_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss600mp_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCserver 600MP";
+    mc->init = ss600mp_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss600mp_type = {
+    .name = MACHINE_TYPE_NAME("SS-600MP"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss600mp_class_init,
 };
 
-static QEMUMachine ss20_machine = {
-    .name = "SS-20",
-    .desc = "Sun4m platform, SPARCstation 20",
-    .init = ss20_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss20_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 20";
+    mc->init = ss20_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss20_type = {
+    .name = MACHINE_TYPE_NAME("SS-20"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss20_class_init,
 };
 
-static QEMUMachine voyager_machine = {
-    .name = "Voyager",
-    .desc = "Sun4m platform, SPARCstation Voyager",
-    .init = vger_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void voyager_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation Voyager";
+    mc->init = vger_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo voyager_type = {
+    .name = MACHINE_TYPE_NAME("Voyager"),
+    .parent = TYPE_MACHINE,
+    .class_init = voyager_class_init,
 };
 
-static QEMUMachine ss_lx_machine = {
-    .name = "LX",
-    .desc = "Sun4m platform, SPARCstation LX",
-    .init = ss_lx_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss_lx_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation LX";
+    mc->init = ss_lx_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss_lx_type = {
+    .name = MACHINE_TYPE_NAME("LX"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss_lx_class_init,
 };
 
-static QEMUMachine ss4_machine = {
-    .name = "SS-4",
-    .desc = "Sun4m platform, SPARCstation 4",
-    .init = ss4_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss4_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 4";
+    mc->init = ss4_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss4_type = {
+    .name = MACHINE_TYPE_NAME("SS-4"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss4_class_init,
 };
 
-static QEMUMachine scls_machine = {
-    .name = "SPARCClassic",
-    .desc = "Sun4m platform, SPARCClassic",
-    .init = scls_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void scls_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCClassic";
+    mc->init = scls_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo scls_type = {
+    .name = MACHINE_TYPE_NAME("SPARCClassic"),
+    .parent = TYPE_MACHINE,
+    .class_init = scls_class_init,
 };
 
-static QEMUMachine sbook_machine = {
-    .name = "SPARCbook",
-    .desc = "Sun4m platform, SPARCbook",
-    .init = sbook_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void sbook_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCbook";
+    mc->init = sbook_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sbook_type = {
+    .name = MACHINE_TYPE_NAME("SPARCbook"),
+    .parent = TYPE_MACHINE,
+    .class_init = sbook_class_init,
 };
 
 static void sun4m_register_types(void)
@@ -1502,20 +1557,16 @@ static void sun4m_register_types(void)
     type_register_static(&afx_info);
     type_register_static(&prom_info);
     type_register_static(&ram_info);
-}
 
-static void sun4m_machine_init(void)
-{
-    qemu_register_machine(&ss5_machine);
-    qemu_register_machine(&ss10_machine);
-    qemu_register_machine(&ss600mp_machine);
-    qemu_register_machine(&ss20_machine);
-    qemu_register_machine(&voyager_machine);
-    qemu_register_machine(&ss_lx_machine);
-    qemu_register_machine(&ss4_machine);
-    qemu_register_machine(&scls_machine);
-    qemu_register_machine(&sbook_machine);
+    type_register_static(&ss5_type);
+    type_register_static(&ss10_type);
+    type_register_static(&ss600mp_type);
+    type_register_static(&ss20_type);
+    type_register_static(&voyager_type);
+    type_register_static(&ss_lx_type);
+    type_register_static(&ss4_type);
+    type_register_static(&scls_type);
+    type_register_static(&sbook_type);
 }
 
 type_init(sun4m_register_types)
-machine_init(sun4m_machine_init);
diff --git a/qemu/hw/sparc64/sun4u.c b/qemu/hw/sparc64/sun4u.c
index 30cfa0e0a..3165e18eb 100644
--- a/qemu/hw/sparc64/sun4u.c
+++ b/qemu/hw/sparc64/sun4u.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci-host/apb.h"
@@ -40,6 +44,7 @@
 #include "elf.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_IRQ
 //#define DEBUG_EBUS
@@ -98,29 +103,7 @@ typedef struct EbusState {
     MemoryRegion bar1;
 } EbusState;
 
-int DMA_get_channel_mode (int nchan)
-{
-    return 0;
-}
-int DMA_read_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-int DMA_write_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-void DMA_hold_DREQ (int nchan) {}
-void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
-
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
-{
-}
-
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
+void DMA_init(ISABus *bus, int high_page_enable)
 {
 }
 
@@ -208,7 +191,7 @@ static uint64_t sun4u_load_kernel(const char *kernel_filename,
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, NULL, NULL, kernel_entry,
-                               kernel_addr, &kernel_top, 1, ELF_MACHINE, 0);
+                               kernel_addr, &kernel_top, 1, EM_SPARCV9, 0, 0);
         if (kernel_size < 0) {
             *kernel_addr = KERNEL_LOAD_ADDR;
             *kernel_entry = KERNEL_LOAD_ADDR;
@@ -358,37 +341,19 @@ typedef struct ResetData {
     uint64_t prom_addr;
 } ResetData;
 
-void cpu_put_timer(QEMUFile *f, CPUTimer *s)
-{
-    qemu_put_be32s(f, &s->frequency);
-    qemu_put_be32s(f, &s->disabled);
-    qemu_put_be64s(f, &s->disabled_mask);
-    qemu_put_sbe64s(f, &s->clock_offset);
-
-    timer_put(f, s->qtimer);
-}
-
-void cpu_get_timer(QEMUFile *f, CPUTimer *s)
-{
-    qemu_get_be32s(f, &s->frequency);
-    qemu_get_be32s(f, &s->disabled);
-    qemu_get_be64s(f, &s->disabled_mask);
-    qemu_get_sbe64s(f, &s->clock_offset);
-
-    timer_get(f, s->qtimer);
-}
-
 static CPUTimer *cpu_timer_create(const char *name, SPARCCPU *cpu,
                                   QEMUBHFunc *cb, uint32_t frequency,
-                                  uint64_t disabled_mask)
+                                  uint64_t disabled_mask, uint64_t npt_mask)
 {
     CPUTimer *timer = g_malloc0(sizeof (CPUTimer));
 
     timer->name = name;
     timer->frequency = frequency;
     timer->disabled_mask = disabled_mask;
+    timer->npt_mask = npt_mask;
 
     timer->disabled = 1;
+    timer->npt = 1;
     timer->clock_offset = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     timer->qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cb, cpu);
@@ -484,27 +449,27 @@ static void hstick_irq(void *opaque)
 
 static int64_t cpu_to_timer_ticks(int64_t cpu_ticks, uint32_t frequency)
 {
-    return muldiv64(cpu_ticks, get_ticks_per_sec(), frequency);
+    return muldiv64(cpu_ticks, NANOSECONDS_PER_SECOND, frequency);
 }
 
 static uint64_t timer_to_cpu_ticks(int64_t timer_ticks, uint32_t frequency)
 {
-    return muldiv64(timer_ticks, frequency, get_ticks_per_sec());
+    return muldiv64(timer_ticks, frequency, NANOSECONDS_PER_SECOND);
 }
 
 void cpu_tick_set_count(CPUTimer *timer, uint64_t count)
 {
-    uint64_t real_count = count & ~timer->disabled_mask;
-    uint64_t disabled_bit = count & timer->disabled_mask;
+    uint64_t real_count = count & ~timer->npt_mask;
+    uint64_t npt_bit = count & timer->npt_mask;
 
     int64_t vm_clock_offset = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
                     cpu_to_timer_ticks(real_count, timer->frequency);
 
-    TIMER_DPRINTF("%s set_count count=0x%016lx (%s) p=%p\n",
+    TIMER_DPRINTF("%s set_count count=0x%016lx (npt %s) p=%p\n",
                   timer->name, real_count,
-                  timer->disabled?"disabled":"enabled", timer);
+                  timer->npt ? "disabled" : "enabled", timer);
 
-    timer->disabled = disabled_bit ? 1 : 0;
+    timer->npt = npt_bit ? 1 : 0;
     timer->clock_offset = vm_clock_offset;
 }
 
@@ -514,12 +479,13 @@ uint64_t cpu_tick_get_count(CPUTimer *timer)
                     qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - timer->clock_offset,
                     timer->frequency);
 
-    TIMER_DPRINTF("%s get_count count=0x%016lx (%s) p=%p\n",
+    TIMER_DPRINTF("%s get_count count=0x%016lx (npt %s) p=%p\n",
            timer->name, real_count,
-           timer->disabled?"disabled":"enabled", timer);
+           timer->npt ? "disabled" : "enabled", timer);
 
-    if (timer->disabled)
-        real_count |= timer->disabled_mask;
+    if (timer->npt) {
+        real_count |= timer->npt_mask;
+    }
 
     return real_count;
 }
@@ -593,13 +559,14 @@ pci_ebus_init(PCIBus *bus, int devfn, qemu_irq *irqs)
     return isa_bus;
 }
 
-static int
-pci_ebus_init1(PCIDevice *pci_dev)
+static void pci_ebus_realize(PCIDevice *pci_dev, Error **errp)
 {
     EbusState *s = DO_UPCAST(EbusState, pci_dev, pci_dev);
 
-    isa_bus_new(DEVICE(pci_dev), get_system_memory(),
-                pci_address_space_io(pci_dev));
+    if (!isa_bus_new(DEVICE(pci_dev), get_system_memory(),
+                     pci_address_space_io(pci_dev), errp)) {
+        return;
+    }
 
     pci_dev->config[0x04] = 0x06; // command = bus master, pci mem
     pci_dev->config[0x05] = 0x00;
@@ -614,14 +581,13 @@ pci_ebus_init1(PCIDevice *pci_dev)
     memory_region_init_alias(&s->bar1, OBJECT(s), "bar1", get_system_io(),
                              0, 0x4000);
     pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &s->bar1);
-    return 0;
 }
 
 static void ebus_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ebus_init1;
+    k->realize = pci_ebus_realize;
     k->vendor_id = PCI_VENDOR_ID_SUN;
     k->device_id = PCI_DEVICE_ID_SUN_EBUS;
     k->revision = 0x01;
@@ -671,7 +637,7 @@ static void prom_init(hwaddr addr, const char *bios_name)
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr,
-                       NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, NULL, 1, EM_SPARCV9, 0, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -690,7 +656,7 @@ static int prom_init1(SysBusDevice *dev)
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4u.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -734,7 +700,7 @@ static int ram_init1(SysBusDevice *dev)
     RamDevice *d = SUN4U_RAM(dev);
 
     memory_region_init_ram(&d->ram, OBJECT(d), "sun4u.ram", d->size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&d->ram);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
@@ -799,13 +765,16 @@ static SPARCCPU *cpu_devinit(const char *cpu_model, const struct hwdef *hwdef)
     env = &cpu->env;
 
     env->tick = cpu_timer_create("tick", cpu, tick_irq,
-                                  tick_frequency, TICK_NPT_MASK);
+                                  tick_frequency, TICK_INT_DIS,
+                                  TICK_NPT_MASK);
 
     env->stick = cpu_timer_create("stick", cpu, stick_irq,
-                                   stick_frequency, TICK_INT_DIS);
+                                   stick_frequency, TICK_INT_DIS,
+                                   TICK_NPT_MASK);
 
     env->hstick = cpu_timer_create("hstick", cpu, hstick_irq,
-                                    hstick_frequency, TICK_INT_DIS);
+                                    hstick_frequency, TICK_INT_DIS,
+                                    TICK_NPT_MASK);
 
     reset_info = g_malloc0(sizeof(ResetData));
     reset_info->cpu = cpu;
@@ -829,6 +798,7 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
     qemu_irq *ivec_irqs, *pbm_irqs;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     DriveInfo *fd[MAX_FD];
+    DeviceState *dev;
     FWCfgState *fw_cfg;
 
     /* init CPUs */
@@ -865,10 +835,22 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
     pci_cmd646_ide_init(pci_bus, hd, 1);
 
     isa_create_simple(isa_bus, "i8042");
+
+    /* Floppy */
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
     }
-    fdctrl_init_isa(isa_bus, fd);
+    dev = DEVICE(isa_create(isa_bus, TYPE_ISA_FDC));
+    if (fd[0]) {
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fd[0]),
+                            &error_abort);
+    }
+    if (fd[1]) {
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fd[1]),
+                            &error_abort);
+    }
+    qdev_prop_set_uint32(dev, "dma", -1);
+    qdev_init_nofail(dev);
 
     /* Map NVRAM into I/O (ebus) space */
     nvram = m48t59_init(NULL, 0, 0, NVRAM_SIZE, 1968, 59);
@@ -965,29 +947,53 @@ static void niagara_init(MachineState *machine)
     sun4uv_init(get_system_memory(), machine, &hwdefs[2]);
 }
 
-static QEMUMachine sun4u_machine = {
-    .name = "sun4u",
-    .desc = "Sun4u platform",
-    .init = sun4u_init,
-    .max_cpus = 1, // XXX for now
-    .is_default = 1,
-    .default_boot_order = "c",
+static void sun4u_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4u platform";
+    mc->init = sun4u_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4u_type = {
+    .name = MACHINE_TYPE_NAME("sun4u"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4u_class_init,
 };
 
-static QEMUMachine sun4v_machine = {
-    .name = "sun4v",
-    .desc = "Sun4v platform",
-    .init = sun4v_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void sun4v_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform";
+    mc->init = sun4v_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4v_type = {
+    .name = MACHINE_TYPE_NAME("sun4v"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4v_class_init,
 };
 
-static QEMUMachine niagara_machine = {
-    .name = "Niagara",
-    .desc = "Sun4v platform, Niagara",
-    .init = niagara_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void niagara_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform, Niagara";
+    mc->init = niagara_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo niagara_type = {
+    .name = MACHINE_TYPE_NAME("Niagara"),
+    .parent = TYPE_MACHINE,
+    .class_init = niagara_class_init,
 };
 
 static void sun4u_register_types(void)
@@ -995,14 +1001,10 @@ static void sun4u_register_types(void)
     type_register_static(&ebus_info);
     type_register_static(&prom_info);
     type_register_static(&ram_info);
-}
 
-static void sun4u_machine_init(void)
-{
-    qemu_register_machine(&sun4u_machine);
-    qemu_register_machine(&sun4v_machine);
-    qemu_register_machine(&niagara_machine);
+    type_register_static(&sun4u_type);
+    type_register_static(&sun4v_type);
+    type_register_static(&niagara_type);
 }
 
 type_init(sun4u_register_types)
-machine_init(sun4u_machine_init);
diff --git a/qemu/hw/ssi/omap_spi.c b/qemu/hw/ssi/omap_spi.c
index 119e325a6..22034656b 100644
--- a/qemu/hw/ssi/omap_spi.c
+++ b/qemu/hw/ssi/omap_spi.c
@@ -19,6 +19,7 @@
  * with this program; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -342,8 +343,7 @@ static const MemoryRegionOps omap_mcspi_ops = {
 struct omap_mcspi_s *omap_mcspi_init(struct omap_target_agent_s *ta, int chnum,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mcspi_s *s = (struct omap_mcspi_s *)
-            g_malloc0(sizeof(struct omap_mcspi_s));
+    struct omap_mcspi_s *s = g_new0(struct omap_mcspi_s, 1);
     struct omap_mcspi_ch_s *ch = s->ch;
 
     s->irq = irq;
diff --git a/qemu/hw/ssi/pl022.c b/qemu/hw/ssi/pl022.c
index 61d568f36..564a0d36e 100644
--- a/qemu/hw/ssi/pl022.c
+++ b/qemu/hw/ssi/pl022.c
@@ -7,8 +7,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 
 //#define DEBUG_PL022 1
 
diff --git a/qemu/hw/ssi/ssi.c b/qemu/hw/ssi/ssi.c
index 2aab79ba7..9791c0d94 100644
--- a/qemu/hw/ssi/ssi.c
+++ b/qemu/hw/ssi/ssi.c
@@ -12,7 +12,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 
 struct SSIBus {
     BusState parent_obj;
diff --git a/qemu/hw/ssi/xilinx_spi.c b/qemu/hw/ssi/xilinx_spi.c
index 620573cac..33482f04d 100644
--- a/qemu/hw/ssi/xilinx_spi.c
+++ b/qemu/hw/ssi/xilinx_spi.c
@@ -24,12 +24,13 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/log.h"
 #include "qemu/fifo8.h"
 
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 
 #ifdef XILINX_SPI_ERR_DEBUG
 #define DB_PRINT(...) do { \
diff --git a/qemu/hw/ssi/xilinx_spips.c b/qemu/hw/ssi/xilinx_spips.c
index 0910f5479..e2b77dc3d 100644
--- a/qemu/hw/ssi/xilinx_spips.c
+++ b/qemu/hw/ssi/xilinx_spips.c
@@ -22,13 +22,15 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
 #include "qemu/fifo8.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "qemu/bitops.h"
+#include "hw/ssi/xilinx_spips.h"
 
 #ifndef XILINX_SPIPS_ERR_DEBUG
 #define XILINX_SPIPS_ERR_DEBUG 0
@@ -103,8 +105,6 @@
 
 #define R_MOD_ID            (0xFC / 4)
 
-#define R_MAX (R_MOD_ID+1)
-
 /* size of TXRX FIFOs */
 #define RXFF_A          32
 #define TXFF_A          32
@@ -135,30 +135,6 @@ typedef enum {
 } FlashCMD;
 
 typedef struct {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    MemoryRegion mmlqspi;
-
-    qemu_irq irq;
-    int irqline;
-
-    uint8_t num_cs;
-    uint8_t num_busses;
-
-    uint8_t snoop_state;
-    qemu_irq *cs_lines;
-    SSIBus **spi;
-
-    Fifo8 rx_fifo;
-    Fifo8 tx_fifo;
-
-    uint8_t num_txrx_bytes;
-
-    uint32_t regs[R_MAX];
-} XilinxSPIPS;
-
-typedef struct {
     XilinxSPIPS parent_obj;
 
     uint8_t lqspi_buf[LQSPI_CACHE_SIZE];
@@ -174,19 +150,6 @@ typedef struct XilinxSPIPSClass {
     uint32_t tx_fifo_size;
 } XilinxSPIPSClass;
 
-#define TYPE_XILINX_SPIPS "xlnx.ps7-spi"
-#define TYPE_XILINX_QSPIPS "xlnx.ps7-qspi"
-
-#define XILINX_SPIPS(obj) \
-     OBJECT_CHECK(XilinxSPIPS, (obj), TYPE_XILINX_SPIPS)
-#define XILINX_SPIPS_CLASS(klass) \
-     OBJECT_CLASS_CHECK(XilinxSPIPSClass, (klass), TYPE_XILINX_SPIPS)
-#define XILINX_SPIPS_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(XilinxSPIPSClass, (obj), TYPE_XILINX_SPIPS)
-
-#define XILINX_QSPIPS(obj) \
-     OBJECT_CHECK(XilinxQSPIPS, (obj), TYPE_XILINX_QSPIPS)
-
 static inline int num_effective_busses(XilinxSPIPS *s)
 {
     return (s->regs[R_LQSPI_CFG] & LQSPI_CFG_SEP_BUS &&
@@ -257,7 +220,7 @@ static void xilinx_spips_reset(DeviceState *d)
     XilinxSPIPS *s = XILINX_SPIPS(d);
 
     int i;
-    for (i = 0; i < R_MAX; i++) {
+    for (i = 0; i < XLNX_SPIPS_R_MAX; i++) {
         s->regs[i] = 0;
     }
 
@@ -664,7 +627,7 @@ static void xilinx_spips_realize(DeviceState *dev, Error **errp)
     }
 
     memory_region_init_io(&s->iomem, OBJECT(s), xsc->reg_ops, s,
-                          "spi", R_MAX*4);
+                          "spi", XLNX_SPIPS_R_MAX * 4);
     sysbus_init_mmio(sbd, &s->iomem);
 
     s->irqline = -1;
@@ -708,7 +671,7 @@ static const VMStateDescription vmstate_xilinx_spips = {
     .fields = (VMStateField[]) {
         VMSTATE_FIFO8(tx_fifo, XilinxSPIPS),
         VMSTATE_FIFO8(rx_fifo, XilinxSPIPS),
-        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, R_MAX),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, XLNX_SPIPS_R_MAX),
         VMSTATE_UINT8(snoop_state, XilinxSPIPS),
         VMSTATE_END_OF_LIST()
     }
diff --git a/qemu/hw/timer/Makefile.objs b/qemu/hw/timer/Makefile.objs
index 133bd0d45..003c14fa2 100644
--- a/qemu/hw/timer/Makefile.objs
+++ b/qemu/hw/timer/Makefile.objs
@@ -25,7 +25,6 @@ obj-$(CONFIG_OMAP) += omap_gptimer.o
 obj-$(CONFIG_OMAP) += omap_synctimer.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_timer.o
 obj-$(CONFIG_SH4) += sh_timer.o
-obj-$(CONFIG_TUSB6010) += tusb6010.o
 obj-$(CONFIG_DIGIC) += digic-timer.o
 
 obj-$(CONFIG_MC146818RTC) += mc146818rtc.o
@@ -33,3 +32,4 @@ obj-$(CONFIG_MC146818RTC) += mc146818rtc.o
 obj-$(CONFIG_ALLWINNER_A10_PIT) += allwinner-a10-pit.o
 
 common-obj-$(CONFIG_STM32F2XX_TIMER) += stm32f2xx_timer.o
+common-obj-$(CONFIG_ASPEED_SOC) += aspeed_timer.o
diff --git a/qemu/hw/timer/a9gtimer.c b/qemu/hw/timer/a9gtimer.c
index dd4aae8b3..afe577c76 100644
--- a/qemu/hw/timer/a9gtimer.c
+++ b/qemu/hw/timer/a9gtimer.c
@@ -20,10 +20,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/timer/a9gtimer.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "qemu/log.h"
+#include "qom/cpu.h"
 
 #ifndef A9_GTIMER_ERR_DEBUG
 #define A9_GTIMER_ERR_DEBUG 0
diff --git a/qemu/hw/timer/allwinner-a10-pit.c b/qemu/hw/timer/allwinner-a10-pit.c
index 34124fe3d..51cdc98f3 100644
--- a/qemu/hw/timer/allwinner-a10-pit.c
+++ b/qemu/hw/timer/allwinner-a10-pit.c
@@ -15,6 +15,7 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/allwinner-a10-pit.h"
diff --git a/qemu/hw/timer/arm_mptimer.c b/qemu/hw/timer/arm_mptimer.c
index 3e59c2a28..d66bbf01b 100644
--- a/qemu/hw/timer/arm_mptimer.c
+++ b/qemu/hw/timer/arm_mptimer.c
@@ -19,7 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/timer/arm_mptimer.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qom/cpu.h"
 
@@ -220,8 +222,9 @@ static void arm_mptimer_realize(DeviceState *dev, Error **errp)
     int i;
 
     if (s->num_cpu < 1 || s->num_cpu > ARM_MPTIMER_MAX_CPUS) {
-        hw_error("%s: num-cpu must be between 1 and %d\n",
-                 __func__, ARM_MPTIMER_MAX_CPUS);
+        error_setg(errp, "num-cpu must be between 1 and %d",
+                   ARM_MPTIMER_MAX_CPUS);
+        return;
     }
     /* We implement one timer block per CPU, and expose multiple MMIO regions:
      *  * region 0 is "timer for this core"
diff --git a/qemu/hw/timer/arm_timer.c b/qemu/hw/timer/arm_timer.c
index d53f39ad6..f1ede5f53 100644
--- a/qemu/hw/timer/arm_timer.c
+++ b/qemu/hw/timer/arm_timer.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
@@ -276,21 +277,25 @@ static const VMStateDescription vmstate_sp804 = {
     }
 };
 
-static int sp804_init(SysBusDevice *sbd)
+static void sp804_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    SP804State *s = SP804(dev);
+    SP804State *s = SP804(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     sysbus_init_irq(sbd, &s->irq);
+    memory_region_init_io(&s->iomem, obj, &sp804_ops, s,
+                          "sp804", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void sp804_realize(DeviceState *dev, Error **errp)
+{
+    SP804State *s = SP804(dev);
+
     s->timer[0] = arm_timer_init(s->freq0);
     s->timer[1] = arm_timer_init(s->freq1);
     s->timer[0]->irq = qemu_allocate_irq(sp804_set_irq, s, 0);
     s->timer[1]->irq = qemu_allocate_irq(sp804_set_irq, s, 1);
-    memory_region_init_io(&s->iomem, OBJECT(s), &sp804_ops, s,
-                          "sp804", 0x1000);
-    sysbus_init_mmio(sbd, &s->iomem);
-    vmstate_register(dev, -1, &vmstate_sp804, s);
-    return 0;
 }
 
 /* Integrator/CP timer module.  */
@@ -343,9 +348,10 @@ static const MemoryRegionOps icp_pit_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int icp_pit_init(SysBusDevice *dev)
+static void icp_pit_init(Object *obj)
 {
-    icp_pit_state *s = INTEGRATOR_PIT(dev);
+    icp_pit_state *s = INTEGRATOR_PIT(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     /* Timer 0 runs at the system clock speed (40MHz).  */
     s->timer[0] = arm_timer_init(40000000);
@@ -357,26 +363,18 @@ static int icp_pit_init(SysBusDevice *dev)
     sysbus_init_irq(dev, &s->timer[1]->irq);
     sysbus_init_irq(dev, &s->timer[2]->irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &icp_pit_ops, s,
+    memory_region_init_io(&s->iomem, obj, &icp_pit_ops, s,
                           "icp_pit", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
     /* This device has no state to save/restore.  The component timers will
        save themselves.  */
-    return 0;
-}
-
-static void icp_pit_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
-
-    sdc->init = icp_pit_init;
 }
 
 static const TypeInfo icp_pit_info = {
     .name          = TYPE_INTEGRATOR_PIT,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pit_state),
-    .class_init    = icp_pit_class_init,
+    .instance_init = icp_pit_init,
 };
 
 static Property sp804_properties[] = {
@@ -387,17 +385,18 @@ static Property sp804_properties[] = {
 
 static void sp804_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *k = DEVICE_CLASS(klass);
 
-    sdc->init = sp804_init;
+    k->realize = sp804_realize;
     k->props = sp804_properties;
+    k->vmsd = &vmstate_sp804;
 }
 
 static const TypeInfo sp804_info = {
     .name          = TYPE_SP804,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SP804State),
+    .instance_init = sp804_init,
     .class_init    = sp804_class_init,
 };
 
diff --git a/qemu/hw/timer/aspeed_timer.c b/qemu/hw/timer/aspeed_timer.c
new file mode 100644
index 000000000..51e8303cd
--- /dev/null
+++ b/qemu/hw/timer/aspeed_timer.c
@@ -0,0 +1,449 @@
+/*
+ * ASPEED AST2400 Timer
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/ptimer.h"
+#include "hw/sysbus.h"
+#include "hw/timer/aspeed_timer.h"
+#include "qemu-common.h"
+#include "qemu/bitops.h"
+#include "qemu/main-loop.h"
+#include "qemu/timer.h"
+#include "trace.h"
+
+#define TIMER_NR_REGS 4
+
+#define TIMER_CTRL_BITS 4
+#define TIMER_CTRL_MASK ((1 << TIMER_CTRL_BITS) - 1)
+
+#define TIMER_CLOCK_USE_EXT true
+#define TIMER_CLOCK_EXT_HZ 1000000
+#define TIMER_CLOCK_USE_APB false
+#define TIMER_CLOCK_APB_HZ 24000000
+
+#define TIMER_REG_STATUS 0
+#define TIMER_REG_RELOAD 1
+#define TIMER_REG_MATCH_FIRST 2
+#define TIMER_REG_MATCH_SECOND 3
+
+#define TIMER_FIRST_CAP_PULSE 4
+
+enum timer_ctrl_op {
+    op_enable = 0,
+    op_external_clock,
+    op_overflow_interrupt,
+    op_pulse_enable
+};
+
+/**
+ * Avoid mutual references between AspeedTimerCtrlState and AspeedTimer
+ * structs, as it's a waste of memory. The ptimer BH callback needs to know
+ * whether a specific AspeedTimer is enabled, but this information is held in
+ * AspeedTimerCtrlState. So, provide a helper to hoist ourselves from an
+ * arbitrary AspeedTimer to AspeedTimerCtrlState.
+ */
+static inline AspeedTimerCtrlState *timer_to_ctrl(AspeedTimer *t)
+{
+    const AspeedTimer (*timers)[] = (void *)t - (t->id * sizeof(*t));
+    return container_of(timers, AspeedTimerCtrlState, timers);
+}
+
+static inline bool timer_ctrl_status(AspeedTimer *t, enum timer_ctrl_op op)
+{
+    return !!(timer_to_ctrl(t)->ctrl & BIT(t->id * TIMER_CTRL_BITS + op));
+}
+
+static inline bool timer_enabled(AspeedTimer *t)
+{
+    return timer_ctrl_status(t, op_enable);
+}
+
+static inline bool timer_overflow_interrupt(AspeedTimer *t)
+{
+    return timer_ctrl_status(t, op_overflow_interrupt);
+}
+
+static inline bool timer_can_pulse(AspeedTimer *t)
+{
+    return t->id >= TIMER_FIRST_CAP_PULSE;
+}
+
+static void aspeed_timer_expire(void *opaque)
+{
+    AspeedTimer *t = opaque;
+
+    /* Only support interrupts on match values of zero for the moment - this is
+     * sufficient to boot an aspeed_defconfig Linux kernel.
+     *
+     * TODO: matching on arbitrary values (see e.g. hw/timer/a9gtimer.c)
+     */
+    bool match = !(t->match[0] && t->match[1]);
+    bool interrupt = timer_overflow_interrupt(t) || match;
+    if (timer_enabled(t) && interrupt) {
+        t->level = !t->level;
+        qemu_set_irq(t->irq, t->level);
+    }
+}
+
+static uint64_t aspeed_timer_get_value(AspeedTimer *t, int reg)
+{
+    uint64_t value;
+
+    switch (reg) {
+    case TIMER_REG_STATUS:
+        value = ptimer_get_count(t->timer);
+        break;
+    case TIMER_REG_RELOAD:
+        value = t->reload;
+        break;
+    case TIMER_REG_MATCH_FIRST:
+    case TIMER_REG_MATCH_SECOND:
+        value = t->match[reg - 2];
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: Programming error: unexpected reg: %d\n",
+                      __func__, reg);
+        value = 0;
+        break;
+    }
+    return value;
+}
+
+static uint64_t aspeed_timer_read(void *opaque, hwaddr offset, unsigned size)
+{
+    AspeedTimerCtrlState *s = opaque;
+    const int reg = (offset & 0xf) / 4;
+    uint64_t value;
+
+    switch (offset) {
+    case 0x30: /* Control Register */
+        value = s->ctrl;
+        break;
+    case 0x34: /* Control Register 2 */
+        value = s->ctrl2;
+        break;
+    case 0x00 ... 0x2c: /* Timers 1 - 4 */
+        value = aspeed_timer_get_value(&s->timers[(offset >> 4)], reg);
+        break;
+    case 0x40 ... 0x8c: /* Timers 5 - 8 */
+        value = aspeed_timer_get_value(&s->timers[(offset >> 4) - 1], reg);
+        break;
+    /* Illegal */
+    case 0x38:
+    case 0x3C:
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                __func__, offset);
+        value = 0;
+        break;
+    }
+    trace_aspeed_timer_read(offset, size, value);
+    return value;
+}
+
+static void aspeed_timer_set_value(AspeedTimerCtrlState *s, int timer, int reg,
+                                   uint32_t value)
+{
+    AspeedTimer *t;
+
+    trace_aspeed_timer_set_value(timer, reg, value);
+    t = &s->timers[timer];
+    switch (reg) {
+    case TIMER_REG_STATUS:
+        if (timer_enabled(t)) {
+            ptimer_set_count(t->timer, value);
+        }
+        break;
+    case TIMER_REG_RELOAD:
+        t->reload = value;
+        ptimer_set_limit(t->timer, value, 1);
+        break;
+    case TIMER_REG_MATCH_FIRST:
+    case TIMER_REG_MATCH_SECOND:
+        if (value) {
+            /* Non-zero match values are unsupported. As such an interrupt will
+             * always be triggered when the timer reaches zero even if the
+             * overflow interrupt control bit is clear.
+             */
+            qemu_log_mask(LOG_UNIMP, "%s: Match value unsupported by device: "
+                    "0x%" PRIx32 "\n", __func__, value);
+        } else {
+            t->match[reg - 2] = value;
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: Programming error: unexpected reg: %d\n",
+                      __func__, reg);
+        break;
+    }
+}
+
+/* Control register operations are broken out into helpers that can be
+ * explictly called on aspeed_timer_reset(), but also from
+ * aspeed_timer_ctrl_op().
+ */
+
+static void aspeed_timer_ctrl_enable(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_enable(t->id, enable);
+    if (enable) {
+        ptimer_run(t->timer, 0);
+    } else {
+        ptimer_stop(t->timer);
+        ptimer_set_limit(t->timer, t->reload, 1);
+    }
+}
+
+static void aspeed_timer_ctrl_external_clock(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_external_clock(t->id, enable);
+    if (enable) {
+        ptimer_set_freq(t->timer, TIMER_CLOCK_EXT_HZ);
+    } else {
+        ptimer_set_freq(t->timer, TIMER_CLOCK_APB_HZ);
+    }
+}
+
+static void aspeed_timer_ctrl_overflow_interrupt(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_overflow_interrupt(t->id, enable);
+}
+
+static void aspeed_timer_ctrl_pulse_enable(AspeedTimer *t, bool enable)
+{
+    if (timer_can_pulse(t)) {
+        trace_aspeed_timer_ctrl_pulse_enable(t->id, enable);
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "%s: Timer does not support pulse mode\n", __func__);
+    }
+}
+
+/**
+ * Given the actions are fixed in number and completely described in helper
+ * functions, dispatch with a lookup table rather than manage control flow with
+ * a switch statement.
+ */
+static void (*const ctrl_ops[])(AspeedTimer *, bool) = {
+    [op_enable] = aspeed_timer_ctrl_enable,
+    [op_external_clock] = aspeed_timer_ctrl_external_clock,
+    [op_overflow_interrupt] = aspeed_timer_ctrl_overflow_interrupt,
+    [op_pulse_enable] = aspeed_timer_ctrl_pulse_enable,
+};
+
+/**
+ * Conditionally affect changes chosen by a timer's control bit.
+ *
+ * The aspeed_timer_ctrl_op() interface is convenient for the
+ * aspeed_timer_set_ctrl() function as the "no change" early exit can be
+ * calculated for all operations, which cleans up the caller code. However the
+ * interface isn't convenient for the reset function where we want to enter a
+ * specific state without artificially constructing old and new values that
+ * will fall through the change guard (and motivates extracting the actions
+ * out to helper functions).
+ *
+ * @t: The timer to manipulate
+ * @op: The type of operation to be performed
+ * @old: The old state of the timer's control bits
+ * @new: The incoming state for the timer's control bits
+ */
+static void aspeed_timer_ctrl_op(AspeedTimer *t, enum timer_ctrl_op op,
+                                 uint8_t old, uint8_t new)
+{
+    const uint8_t mask = BIT(op);
+    const bool enable = !!(new & mask);
+    const bool changed = ((old ^ new) & mask);
+    if (!changed) {
+        return;
+    }
+    ctrl_ops[op](t, enable);
+}
+
+static void aspeed_timer_set_ctrl(AspeedTimerCtrlState *s, uint32_t reg)
+{
+    int i;
+    int shift;
+    uint8_t t_old, t_new;
+    AspeedTimer *t;
+    const uint8_t enable_mask = BIT(op_enable);
+
+    /* Handle a dependency between the 'enable' and remaining three
+     * configuration bits - i.e. if more than one bit in the control set has
+     * changed, including the 'enable' bit, then we want either disable the
+     * timer and perform configuration, or perform configuration and then
+     * enable the timer
+     */
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        t = &s->timers[i];
+        shift = (i * TIMER_CTRL_BITS);
+        t_old = (s->ctrl >> shift) & TIMER_CTRL_MASK;
+        t_new = (reg >> shift) & TIMER_CTRL_MASK;
+
+        /* If we are disabling, do so first */
+        if ((t_old & enable_mask) && !(t_new & enable_mask)) {
+            aspeed_timer_ctrl_enable(t, false);
+        }
+        aspeed_timer_ctrl_op(t, op_external_clock, t_old, t_new);
+        aspeed_timer_ctrl_op(t, op_overflow_interrupt, t_old, t_new);
+        aspeed_timer_ctrl_op(t, op_pulse_enable, t_old, t_new);
+        /* If we are enabling, do so last */
+        if (!(t_old & enable_mask) && (t_new & enable_mask)) {
+            aspeed_timer_ctrl_enable(t, true);
+        }
+    }
+    s->ctrl = reg;
+}
+
+static void aspeed_timer_set_ctrl2(AspeedTimerCtrlState *s, uint32_t value)
+{
+    trace_aspeed_timer_set_ctrl2(value);
+}
+
+static void aspeed_timer_write(void *opaque, hwaddr offset, uint64_t value,
+                               unsigned size)
+{
+    const uint32_t tv = (uint32_t)(value & 0xFFFFFFFF);
+    const int reg = (offset & 0xf) / 4;
+    AspeedTimerCtrlState *s = opaque;
+
+    switch (offset) {
+    /* Control Registers */
+    case 0x30:
+        aspeed_timer_set_ctrl(s, tv);
+        break;
+    case 0x34:
+        aspeed_timer_set_ctrl2(s, tv);
+        break;
+    /* Timer Registers */
+    case 0x00 ... 0x2c:
+        aspeed_timer_set_value(s, (offset >> TIMER_NR_REGS), reg, tv);
+        break;
+    case 0x40 ... 0x8c:
+        aspeed_timer_set_value(s, (offset >> TIMER_NR_REGS) - 1, reg, tv);
+        break;
+    /* Illegal */
+    case 0x38:
+    case 0x3C:
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps aspeed_timer_ops = {
+    .read = aspeed_timer_read,
+    .write = aspeed_timer_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .valid.unaligned = false,
+};
+
+static void aspeed_init_one_timer(AspeedTimerCtrlState *s, uint8_t id)
+{
+    QEMUBH *bh;
+    AspeedTimer *t = &s->timers[id];
+
+    t->id = id;
+    bh = qemu_bh_new(aspeed_timer_expire, t);
+    t->timer = ptimer_init(bh);
+}
+
+static void aspeed_timer_realize(DeviceState *dev, Error **errp)
+{
+    int i;
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedTimerCtrlState *s = ASPEED_TIMER(dev);
+
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        aspeed_init_one_timer(s, i);
+        sysbus_init_irq(sbd, &s->timers[i].irq);
+    }
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_timer_ops, s,
+                          TYPE_ASPEED_TIMER, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void aspeed_timer_reset(DeviceState *dev)
+{
+    int i;
+    AspeedTimerCtrlState *s = ASPEED_TIMER(dev);
+
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        AspeedTimer *t = &s->timers[i];
+        /* Explictly call helpers to avoid any conditional behaviour through
+         * aspeed_timer_set_ctrl().
+         */
+        aspeed_timer_ctrl_enable(t, false);
+        aspeed_timer_ctrl_external_clock(t, TIMER_CLOCK_USE_APB);
+        aspeed_timer_ctrl_overflow_interrupt(t, false);
+        aspeed_timer_ctrl_pulse_enable(t, false);
+        t->level = 0;
+        t->reload = 0;
+        t->match[0] = 0;
+        t->match[1] = 0;
+    }
+    s->ctrl = 0;
+    s->ctrl2 = 0;
+}
+
+static const VMStateDescription vmstate_aspeed_timer = {
+    .name = "aspeed.timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(id, AspeedTimer),
+        VMSTATE_INT32(level, AspeedTimer),
+        VMSTATE_PTIMER(timer, AspeedTimer),
+        VMSTATE_UINT32(reload, AspeedTimer),
+        VMSTATE_UINT32_ARRAY(match, AspeedTimer, 2),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_aspeed_timer_state = {
+    .name = "aspeed.timerctrl",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(ctrl, AspeedTimerCtrlState),
+        VMSTATE_UINT32(ctrl2, AspeedTimerCtrlState),
+        VMSTATE_STRUCT_ARRAY(timers, AspeedTimerCtrlState,
+                             ASPEED_TIMER_NR_TIMERS, 1, vmstate_aspeed_timer,
+                             AspeedTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void timer_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = aspeed_timer_realize;
+    dc->reset = aspeed_timer_reset;
+    dc->desc = "ASPEED Timer";
+    dc->vmsd = &vmstate_aspeed_timer_state;
+}
+
+static const TypeInfo aspeed_timer_info = {
+    .name = TYPE_ASPEED_TIMER,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedTimerCtrlState),
+    .class_init = timer_class_init,
+};
+
+static void aspeed_timer_register_types(void)
+{
+    type_register_static(&aspeed_timer_info);
+}
+
+type_init(aspeed_timer_register_types)
diff --git a/qemu/hw/timer/cadence_ttc.c b/qemu/hw/timer/cadence_ttc.c
index 35bc88033..03f5b9c20 100644
--- a/qemu/hw/timer/cadence_ttc.c
+++ b/qemu/hw/timer/cadence_ttc.c
@@ -16,6 +16,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 
diff --git a/qemu/hw/timer/digic-timer.c b/qemu/hw/timer/digic-timer.c
index 7e28e7e5f..5b97e1e1a 100644
--- a/qemu/hw/timer/digic-timer.c
+++ b/qemu/hw/timer/digic-timer.c
@@ -26,6 +26,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/main-loop.h"
diff --git a/qemu/hw/timer/ds1338.c b/qemu/hw/timer/ds1338.c
index ec6dbeeab..0112949e2 100644
--- a/qemu/hw/timer/ds1338.c
+++ b/qemu/hw/timer/ds1338.c
@@ -10,7 +10,10 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "hw/i2c/i2c.h"
+#include "qemu/bcd.h"
 
 /* Size of NVRAM including both the user-accessible area and the
  * secondary register area.
diff --git a/qemu/hw/timer/etraxfs_timer.c b/qemu/hw/timer/etraxfs_timer.c
index aee4990eb..36d8f462c 100644
--- a/qemu/hw/timer/etraxfs_timer.c
+++ b/qemu/hw/timer/etraxfs_timer.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/timer/exynos4210_mct.c b/qemu/hw/timer/exynos4210_mct.c
index 015bbaf1b..ae69345f0 100644
--- a/qemu/hw/timer/exynos4210_mct.c
+++ b/qemu/hw/timer/exynos4210_mct.c
@@ -52,6 +52,7 @@
  * there is no way to avoid frequently events).
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu/main-loop.h"
@@ -1421,10 +1422,11 @@ static const MemoryRegionOps exynos4210_mct_ops = {
 };
 
 /* MCT init */
-static int exynos4210_mct_init(SysBusDevice *dev)
+static void exynos4210_mct_init(Object *obj)
 {
     int i;
-    Exynos4210MCTState *s = EXYNOS4210_MCT(dev);
+    Exynos4210MCTState *s = EXYNOS4210_MCT(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     QEMUBH *bh[2];
 
     /* Global timer */
@@ -1449,19 +1451,15 @@ static int exynos4210_mct_init(SysBusDevice *dev)
         sysbus_init_irq(dev, &s->l_timer[i].irq);
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_mct_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_mct_ops, s,
                           "exynos4210-mct", MCT_SFR_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_mct_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_mct_init;
     dc->reset = exynos4210_mct_reset;
     dc->vmsd = &vmstate_exynos4210_mct_state;
 }
@@ -1470,6 +1468,7 @@ static const TypeInfo exynos4210_mct_info = {
     .name          = TYPE_EXYNOS4210_MCT,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210MCTState),
+    .instance_init = exynos4210_mct_init,
     .class_init    = exynos4210_mct_class_init,
 };
 
diff --git a/qemu/hw/timer/exynos4210_pwm.c b/qemu/hw/timer/exynos4210_pwm.c
index 1c1a2b8ff..0e9e2e9bf 100644
--- a/qemu/hw/timer/exynos4210_pwm.c
+++ b/qemu/hw/timer/exynos4210_pwm.c
@@ -20,6 +20,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
@@ -379,9 +380,10 @@ static const MemoryRegionOps exynos4210_pwm_ops = {
 /*
  * PWM timer initialization
  */
-static int exynos4210_pwm_init(SysBusDevice *dev)
+static void exynos4210_pwm_init(Object *obj)
 {
-    Exynos4210PWMState *s = EXYNOS4210_PWM(dev);
+    Exynos4210PWMState *s = EXYNOS4210_PWM(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     int i;
     QEMUBH *bh;
 
@@ -393,19 +395,15 @@ static int exynos4210_pwm_init(SysBusDevice *dev)
         s->timer[i].parent = s;
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_pwm_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_pwm_ops, s,
                           "exynos4210-pwm", EXYNOS4210_PWM_REG_MEM_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_pwm_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_pwm_init;
     dc->reset = exynos4210_pwm_reset;
     dc->vmsd = &vmstate_exynos4210_pwm_state;
 }
@@ -414,6 +412,7 @@ static const TypeInfo exynos4210_pwm_info = {
     .name          = TYPE_EXYNOS4210_PWM,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PWMState),
+    .instance_init = exynos4210_pwm_init,
     .class_init    = exynos4210_pwm_class_init,
 };
 
diff --git a/qemu/hw/timer/exynos4210_rtc.c b/qemu/hw/timer/exynos4210_rtc.c
index bf2ee9f80..da4dd451b 100644
--- a/qemu/hw/timer/exynos4210_rtc.c
+++ b/qemu/hw/timer/exynos4210_rtc.c
@@ -25,9 +25,11 @@
  *  CLKOUTEN Bit[9] not used
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
+#include "qemu/bcd.h"
 #include "hw/ptimer.h"
 
 #include "hw/hw.h"
@@ -546,9 +548,10 @@ static const MemoryRegionOps exynos4210_rtc_ops = {
 /*
  * RTC timer initialization
  */
-static int exynos4210_rtc_init(SysBusDevice *dev)
+static void exynos4210_rtc_init(Object *obj)
 {
-    Exynos4210RTCState *s = EXYNOS4210_RTC(dev);
+    Exynos4210RTCState *s = EXYNOS4210_RTC(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     QEMUBH *bh;
 
     bh = qemu_bh_new(exynos4210_rtc_tick, s);
@@ -563,19 +566,15 @@ static int exynos4210_rtc_init(SysBusDevice *dev)
     sysbus_init_irq(dev, &s->alm_irq);
     sysbus_init_irq(dev, &s->tick_irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_rtc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_rtc_ops, s,
                           "exynos4210-rtc", EXYNOS4210_RTC_REG_MEM_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_rtc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_rtc_init;
     dc->reset = exynos4210_rtc_reset;
     dc->vmsd = &vmstate_exynos4210_rtc_state;
 }
@@ -584,6 +583,7 @@ static const TypeInfo exynos4210_rtc_info = {
     .name          = TYPE_EXYNOS4210_RTC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210RTCState),
+    .instance_init = exynos4210_rtc_init,
     .class_init    = exynos4210_rtc_class_init,
 };
 
diff --git a/qemu/hw/timer/grlib_gptimer.c b/qemu/hw/timer/grlib_gptimer.c
index d655bb2a2..dd000f5af 100644
--- a/qemu/hw/timer/grlib_gptimer.c
+++ b/qemu/hw/timer/grlib_gptimer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
diff --git a/qemu/hw/timer/hpet.c b/qemu/hw/timer/hpet.c
index 2bb62211c..a2c18b30c 100644
--- a/qemu/hw/timer/hpet.c
+++ b/qemu/hw/timer/hpet.c
@@ -24,9 +24,11 @@
  * This driver attempts to emulate an HPET device in software.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "ui/console.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "hw/timer/hpet.h"
@@ -116,22 +118,22 @@ static uint32_t timer_enabled(HPETTimer *t)
 
 static uint32_t hpet_time_after(uint64_t a, uint64_t b)
 {
-    return ((int32_t)(b) - (int32_t)(a) < 0);
+    return ((int32_t)(b - a) < 0);
 }
 
 static uint32_t hpet_time_after64(uint64_t a, uint64_t b)
 {
-    return ((int64_t)(b) - (int64_t)(a) < 0);
+    return ((int64_t)(b - a) < 0);
 }
 
 static uint64_t ticks_to_ns(uint64_t value)
 {
-    return (muldiv64(value, HPET_CLK_PERIOD, FS_PER_NS));
+    return value * HPET_CLK_PERIOD;
 }
 
 static uint64_t ns_to_ticks(uint64_t value)
 {
-    return (muldiv64(value, FS_PER_NS, HPET_CLK_PERIOD));
+    return value / HPET_CLK_PERIOD;
 }
 
 static uint64_t hpet_fixup_reg(uint64_t new, uint64_t old, uint64_t mask)
@@ -199,12 +201,7 @@ static void update_irq(struct HPETTimer *timer, int set)
     if (!set || !timer_enabled(timer) || !hpet_enabled(timer->state)) {
         s->isr &= ~mask;
         if (!timer_fsb_route(timer)) {
-            /* fold the ICH PIRQ# pin's internal inversion logic into hpet */
-            if (route >= ISA_NUM_IRQS) {
-                qemu_irq_raise(s->irqs[route]);
-            } else {
-                qemu_irq_lower(s->irqs[route]);
-            }
+            qemu_irq_lower(s->irqs[route]);
         }
     } else if (timer_fsb_route(timer)) {
         address_space_stl_le(&address_space_memory, timer->fsb >> 32,
@@ -212,12 +209,7 @@ static void update_irq(struct HPETTimer *timer, int set)
                              NULL);
     } else if (timer->config & HPET_TN_TYPE_LEVEL) {
         s->isr |= mask;
-        /* fold the ICH PIRQ# pin's internal inversion logic into hpet */
-        if (route >= ISA_NUM_IRQS) {
-            qemu_irq_lower(s->irqs[route]);
-        } else {
-            qemu_irq_raise(s->irqs[route]);
-        }
+        qemu_irq_raise(s->irqs[route]);
     } else {
         s->isr &= ~mask;
         qemu_irq_pulse(s->irqs[route]);
@@ -713,7 +705,7 @@ static void hpet_init(Object *obj)
     HPETState *s = HPET(obj);
 
     /* HPET Area */
-    memory_region_init_io(&s->iomem, obj, &hpet_ram_ops, s, "hpet", 0x400);
+    memory_region_init_io(&s->iomem, obj, &hpet_ram_ops, s, "hpet", HPET_LEN);
     sysbus_init_mmio(sbd, &s->iomem);
 }
 
@@ -758,7 +750,7 @@ static void hpet_realize(DeviceState *dev, Error **errp)
     /* 64-bit main counter; LegacyReplacementRoute. */
     s->capability = 0x8086a001ULL;
     s->capability |= (s->num_timers - 1) << HPET_ID_NUM_TIM_SHIFT;
-    s->capability |= ((HPET_CLK_PERIOD) << 32);
+    s->capability |= ((uint64_t)(HPET_CLK_PERIOD * FS_PER_NS) << 32);
 
     qdev_init_gpio_in(dev, hpet_handle_legacy_irq, 2);
     qdev_init_gpio_out(dev, &s->pit_enabled, 1);
diff --git a/qemu/hw/timer/i8254.c b/qemu/hw/timer/i8254.c
index 9b65a3369..5e61ad50a 100644
--- a/qemu/hw/timer/i8254.c
+++ b/qemu/hw/timer/i8254.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -52,7 +53,7 @@ static int pit_get_count(PITChannelState *s)
     int counter;
 
     d = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch(s->mode) {
     case 0:
     case 1:
@@ -262,7 +263,7 @@ static void pit_irq_timer_update(PITChannelState *s, int64_t current_time)
 #ifdef DEBUG_PIT
     printf("irq_level=%d next_delay=%f\n",
            irq_level,
-           (double)(expire_time - current_time) / get_ticks_per_sec());
+           (double)(expire_time - current_time) / NANOSECONDS_PER_SECOND);
 #endif
     s->next_transition_time = expire_time;
     if (expire_time != -1)
diff --git a/qemu/hw/timer/i8254_common.c b/qemu/hw/timer/i8254_common.c
index 07345f668..e18299a48 100644
--- a/qemu/hw/timer/i8254_common.c
+++ b/qemu/hw/timer/i8254_common.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -46,7 +47,7 @@ int pit_get_out(PITChannelState *s, int64_t current_time)
     int out;
 
     d = muldiv64(current_time - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch (s->mode) {
     default:
     case 0:
@@ -80,7 +81,7 @@ int64_t pit_get_next_transition_time(PITChannelState *s, int64_t current_time)
     int period2;
 
     d = muldiv64(current_time - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch (s->mode) {
     default:
     case 0:
@@ -120,7 +121,7 @@ int64_t pit_get_next_transition_time(PITChannelState *s, int64_t current_time)
         break;
     }
     /* convert to timer units */
-    next_time = s->count_load_time + muldiv64(next_time, get_ticks_per_sec(),
+    next_time = s->count_load_time + muldiv64(next_time, NANOSECONDS_PER_SECOND,
                                               PIT_FREQ);
     /* fix potential rounding problems */
     /* XXX: better solution: use a clock at PIT_FREQ Hz */
diff --git a/qemu/hw/timer/imx_epit.c b/qemu/hw/timer/imx_epit.c
index ffefc22f4..f5836e21f 100644
--- a/qemu/hw/timer/imx_epit.c
+++ b/qemu/hw/timer/imx_epit.c
@@ -5,25 +5,29 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/misc/imx_ccm.h"
 #include "qemu/main-loop.h"
 
-#define TYPE_IMX_EPIT "imx.epit"
+#ifndef DEBUG_IMX_EPIT
+#define DEBUG_IMX_EPIT 0
+#endif
 
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_EPIT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_EPIT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_epit_reg_name(uint32_t reg)
 {
@@ -43,76 +47,17 @@ static char const *imx_epit_reg_name(uint32_t reg)
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-    do { fprintf(stderr, "%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_EPIT(obj) \
-        OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
-
-/*
- * EPIT: Enhanced periodic interrupt timer
- */
-
-#define CR_EN       (1 << 0)
-#define CR_ENMOD    (1 << 1)
-#define CR_OCIEN    (1 << 2)
-#define CR_RLD      (1 << 3)
-#define CR_PRESCALE_SHIFT (4)
-#define CR_PRESCALE_MASK  (0xfff)
-#define CR_SWR      (1 << 16)
-#define CR_IOVW     (1 << 17)
-#define CR_DBGEN    (1 << 18)
-#define CR_WAITEN   (1 << 19)
-#define CR_DOZEN    (1 << 20)
-#define CR_STOPEN   (1 << 21)
-#define CR_CLKSRC_SHIFT (24)
-#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
-
-#define EPIT_TIMER_MAX  0XFFFFFFFFUL
-
 /*
  * Exact clock frequencies vary from board to board.
  * These are typical.
  */
 static const IMXClk imx_epit_clocks[] =  {
-    0,        /* 00 disabled */
-    IPG,      /* 01 ipg_clk, ~532MHz */
-    IPG,      /* 10 ipg_clk_highfreq */
-    CLK_32k,  /* 11 ipg_clk_32k -- ~32kHz */
+    CLK_NONE,      /* 00 disabled */
+    CLK_IPG,       /* 01 ipg_clk, ~532MHz */
+    CLK_IPG_HIGH,  /* 10 ipg_clk_highfreq */
+    CLK_32k,       /* 11 ipg_clk_32k -- ~32kHz */
 };
 
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer_reload;
-    ptimer_state *timer_cmp;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t sr;
-    uint32_t lr;
-    uint32_t cmp;
-    uint32_t cnt;
-
-    uint32_t freq;
-    qemu_irq irq;
-} IMXEPITState;
-
 /*
  * Update interrupt status
  */
@@ -129,20 +74,18 @@ static void imx_epit_set_freq(IMXEPITState *s)
 {
     uint32_t clksrc;
     uint32_t prescaler;
-    uint32_t freq;
 
     clksrc = extract32(s->cr, CR_CLKSRC_SHIFT, 2);
     prescaler = 1 + extract32(s->cr, CR_PRESCALE_SHIFT, 12);
 
-    freq = imx_clock_frequency(s->ccm, imx_epit_clocks[clksrc]) / prescaler;
+    s->freq = imx_ccm_get_clock_frequency(s->ccm,
+                                imx_epit_clocks[clksrc]) / prescaler;
 
-    s->freq = freq;
+    DPRINTF("Setting ptimer frequency to %u\n", s->freq);
 
-    DPRINTF("Setting ptimer frequency to %u\n", freq);
-
-    if (freq) {
-        ptimer_set_freq(s->timer_reload, freq);
-        ptimer_set_freq(s->timer_cmp, freq);
+    if (s->freq) {
+        ptimer_set_freq(s->timer_reload, s->freq);
+        ptimer_set_freq(s->timer_cmp, s->freq);
     }
 }
 
@@ -174,18 +117,17 @@ static void imx_epit_reset(DeviceState *dev)
 
 static uint32_t imx_epit_update_count(IMXEPITState *s)
 {
-     s->cnt = ptimer_get_count(s->timer_reload);
+    s->cnt = ptimer_get_count(s->timer_reload);
 
-     return s->cnt;
+    return s->cnt;
 }
 
 static uint64_t imx_epit_read(void *opaque, hwaddr offset, unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -208,11 +150,12 @@ static uint64_t imx_epit_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -237,12 +180,12 @@ static void imx_epit_write(void *opaque, hwaddr offset, uint64_t value,
                            unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
-    uint32_t reg = offset >> 2;
     uint64_t oldcr;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(reg), (uint32_t)value);
+    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(offset >> 2),
+            (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* CR */
 
         oldcr = s->cr;
@@ -318,7 +261,8 @@ static void imx_epit_write(void *opaque, hwaddr offset, uint64_t value,
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
 
         break;
     }
@@ -333,24 +277,14 @@ static void imx_epit_cmp(void *opaque)
     imx_epit_update_int(s);
 }
 
-void imx_timerp_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXEPITState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_EPIT, addr, irq);
-    pp = IMX_EPIT(dev);
-    pp->ccm = ccm;
-}
-
 static const MemoryRegionOps imx_epit_ops = {
-  .read = imx_epit_read,
-  .write = imx_epit_write,
-  .endianness = DEVICE_NATIVE_ENDIAN,
+    .read = imx_epit_read,
+    .write = imx_epit_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
 static const VMStateDescription vmstate_imx_timer_epit = {
-    .name = "imx.epit",
+    .name = TYPE_IMX_EPIT,
     .version_id = 2,
     .minimum_version_id = 2,
     .fields = (VMStateField[]) {
diff --git a/qemu/hw/timer/imx_gpt.c b/qemu/hw/timer/imx_gpt.c
index 3b3101084..ab2e213a1 100644
--- a/qemu/hw/timer/imx_gpt.c
+++ b/qemu/hw/timer/imx_gpt.c
@@ -5,28 +5,29 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/misc/imx_ccm.h"
 #include "qemu/main-loop.h"
 
-#define TYPE_IMX_GPT "imx.gpt"
+#ifndef DEBUG_IMX_GPT
+#define DEBUG_IMX_GPT 0
+#endif
 
-/*
- * Define to 1 for debug messages
- */
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_gpt_reg_name(uint32_t reg)
 {
@@ -56,94 +57,8 @@ static char const *imx_gpt_reg_name(uint32_t reg)
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-          do { printf("%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_GPT(obj) \
-        OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
-/*
- * GPT : General purpose timer
- *
- * This timer counts up continuously while it is enabled, resetting itself
- * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
- * reaches the value of one of the ocrX (in periodic mode).
- */
-
-#define GPT_TIMER_MAX  0XFFFFFFFFUL
-
-/* Control register.  Not all of these bits have any effect (yet) */
-#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
-#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
-#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
-#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
-#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
-#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
-#define GPT_CR_CLKSRC_SHIFT (6)
-#define GPT_CR_CLKSRC_MASK  (0x7)
-
-#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
-#define GPT_CR_SWR    (1 << 15) /* Software Reset */
-#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
-#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
-#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
-#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
-#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
-#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
-#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
-#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
-
-#define GPT_SR_OF1  (1 << 0)
-#define GPT_SR_OF2  (1 << 1)
-#define GPT_SR_OF3  (1 << 2)
-#define GPT_SR_ROV  (1 << 5)
-
-#define GPT_IR_OF1IE  (1 << 0)
-#define GPT_IR_OF2IE  (1 << 1)
-#define GPT_IR_OF3IE  (1 << 2)
-#define GPT_IR_ROVIE  (1 << 5)
-
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t pr;
-    uint32_t sr;
-    uint32_t ir;
-    uint32_t ocr1;
-    uint32_t ocr2;
-    uint32_t ocr3;
-    uint32_t icr1;
-    uint32_t icr2;
-    uint32_t cnt;
-
-    uint32_t next_timeout;
-    uint32_t next_int;
-
-    uint32_t freq;
-
-    qemu_irq irq;
-} IMXGPTState;
-
 static const VMStateDescription vmstate_imx_timer_gpt = {
-    .name = "imx.gpt",
+    .name = TYPE_IMX_GPT,
     .version_id = 3,
     .minimum_version_id = 3,
     .fields = (VMStateField[]) {
@@ -166,27 +81,27 @@ static const VMStateDescription vmstate_imx_timer_gpt = {
 };
 
 static const IMXClk imx_gpt_clocks[] = {
-    NOCLK,    /* 000 No clock source */
-    IPG,      /* 001 ipg_clk, 532MHz*/
-    IPG,      /* 010 ipg_clk_highfreq */
-    NOCLK,    /* 011 not defined */
-    CLK_32k,  /* 100 ipg_clk_32k */
-    NOCLK,    /* 101 not defined */
-    NOCLK,    /* 110 not defined */
-    NOCLK,    /* 111 not defined */
+    CLK_NONE,      /* 000 No clock source */
+    CLK_IPG,       /* 001 ipg_clk, 532MHz*/
+    CLK_IPG_HIGH,  /* 010 ipg_clk_highfreq */
+    CLK_NONE,      /* 011 not defined */
+    CLK_32k,       /* 100 ipg_clk_32k */
+    CLK_NONE,      /* 101 not defined */
+    CLK_NONE,      /* 110 not defined */
+    CLK_NONE,      /* 111 not defined */
 };
 
 static void imx_gpt_set_freq(IMXGPTState *s)
 {
     uint32_t clksrc = extract32(s->cr, GPT_CR_CLKSRC_SHIFT, 3);
-    uint32_t freq = imx_clock_frequency(s->ccm, imx_gpt_clocks[clksrc])
-                                                / (1 + s->pr);
-    s->freq = freq;
 
-    DPRINTF("Setting clksrc %d to frequency %d\n", clksrc, freq);
+    s->freq = imx_ccm_get_clock_frequency(s->ccm,
+                                imx_gpt_clocks[clksrc]) / (1 + s->pr);
+
+    DPRINTF("Setting clksrc %d to frequency %d\n", clksrc, s->freq);
 
-    if (freq) {
-        ptimer_set_freq(s->timer, freq);
+    if (s->freq) {
+        ptimer_set_freq(s->timer, s->freq);
     }
 }
 
@@ -207,7 +122,7 @@ static uint32_t imx_gpt_update_count(IMXGPTState *s)
 }
 
 static inline uint32_t imx_gpt_find_limit(uint32_t count, uint32_t reg,
-                                             uint32_t timeout)
+                                          uint32_t timeout)
 {
     if ((count < reg) && (timeout > reg)) {
         timeout = reg;
@@ -219,7 +134,7 @@ static inline uint32_t imx_gpt_find_limit(uint32_t count, uint32_t reg,
 static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
 {
     uint32_t timeout = GPT_TIMER_MAX;
-    uint32_t count = 0;
+    uint32_t count;
     long long limit;
 
     if (!(s->cr & GPT_CR_EN)) {
@@ -227,20 +142,23 @@ static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
         return;
     }
 
-    if (event) {
-        /* This is a timer event  */
+    /* update the count */
+    count = imx_gpt_update_count(s);
 
-        if ((s->cr & GPT_CR_FRR)  && (s->next_timeout != GPT_TIMER_MAX)) {
-            /*
-             * if we are in free running mode and we have not reached
-             * the GPT_TIMER_MAX limit, then update the count
+    if (event) {
+        /*
+         * This is an event (the ptimer reached 0 and stopped), and the
+         * timer counter is now equal to s->next_timeout.
+         */
+        if (!(s->cr & GPT_CR_FRR) && (count == s->ocr1)) {
+            /* We are in restart mode and we crossed the compare channel 1
+             * value. We need to reset the counter to 0.
              */
-            count = imx_gpt_update_count(s);
+            count = s->cnt = s->next_timeout = 0;
+        } else if (count == GPT_TIMER_MAX) {
+            /* We reached GPT_TIMER_MAX so we need to rollover */
+            count = s->cnt = s->next_timeout = 0;
         }
-    } else {
-        /* not a timer event, then just update the count */
-
-        count = imx_gpt_update_count(s);
     }
 
     /* now, find the next timeout related to count */
@@ -298,9 +216,8 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -330,12 +247,14 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     case 7: /* input Capture Register 1 */
-        qemu_log_mask(LOG_UNIMP, "icr1 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr1 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr1;
         break;
 
     case 8: /* input Capture Register 2 */
-        qemu_log_mask(LOG_UNIMP, "icr2 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr2 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr2;
         break;
 
@@ -345,11 +264,12 @@ static uint64_t imx_gpt_read(void *opaque, hwaddr offset, unsigned size)
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -396,12 +316,11 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t oldreg;
-    uint32_t reg = offset >> 2;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(reg),
+    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(offset >> 2),
             (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0:
         oldreg = s->cr;
         s->cr = value & ~0x7c14;
@@ -477,7 +396,8 @@ static void imx_gpt_write(void *opaque, hwaddr offset, uint64_t value,
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 }
@@ -522,16 +442,6 @@ static void imx_gpt_realize(DeviceState *dev, Error **errp)
     s->timer = ptimer_init(bh);
 }
 
-void imx_timerg_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXGPTState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_GPT, addr, irq);
-    pp = IMX_GPT(dev);
-    pp->ccm = ccm;
-}
-
 static void imx_gpt_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
diff --git a/qemu/hw/timer/lm32_timer.c b/qemu/hw/timer/lm32_timer.c
index d2ab1e74b..3198355aa 100644
--- a/qemu/hw/timer/lm32_timer.c
+++ b/qemu/hw/timer/lm32_timer.c
@@ -21,6 +21,7 @@
  *   http://www.latticesemi.com/documents/mico32timer.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/qemu/hw/timer/m48t59.c b/qemu/hw/timer/m48t59.c
index 8ab683dda..e46ca8839 100644
--- a/qemu/hw/timer/m48t59.c
+++ b/qemu/hw/timer/m48t59.c
@@ -22,13 +22,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/timer/m48t59.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "hw/isa/isa.h"
 #include "exec/address-spaces.h"
+#include "qemu/bcd.h"
 
 //#define DEBUG_NVRAM
 
@@ -590,10 +593,8 @@ static void nvram_writel (void *opaque, hwaddr addr, uint32_t value)
 static uint32_t nvram_readb (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
-    uint32_t retval;
 
-    retval = m48t59_read(NVRAM, addr);
-    return retval;
+    return m48t59_read(NVRAM, addr);
 }
 
 static uint32_t nvram_readw (void *opaque, hwaddr addr)
diff --git a/qemu/hw/timer/mc146818rtc.c b/qemu/hw/timer/mc146818rtc.c
index a9f0efd5e..2ac0fd3e4 100644
--- a/qemu/hw/timer/mc146818rtc.c
+++ b/qemu/hw/timer/mc146818rtc.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "config-target.h"
+#include "qemu/cutils.h"
+#include "qemu/bcd.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
@@ -105,8 +109,8 @@ static uint64_t get_guest_rtc_ns(RTCState *s)
     uint64_t guest_rtc;
     uint64_t guest_clock = qemu_clock_get_ns(rtc_clock);
 
-    guest_rtc = s->base_rtc * NANOSECONDS_PER_SECOND
-                 + guest_clock - s->last_update + s->offset;
+    guest_rtc = s->base_rtc * NANOSECONDS_PER_SECOND +
+        guest_clock - s->last_update + s->offset;
     return guest_rtc;
 }
 
@@ -119,7 +123,7 @@ static void rtc_coalesced_timer_update(RTCState *s)
         /* divide each RTC interval to 2 - 8 smaller intervals */
         int c = MIN(s->irq_coalesced, 7) + 1; 
         int64_t next_clock = qemu_clock_get_ns(rtc_clock) +
-            muldiv64(s->period / c, get_ticks_per_sec(), RTC_CLOCK_RATE);
+            muldiv64(s->period / c, NANOSECONDS_PER_SECOND, RTC_CLOCK_RATE);
         timer_mod(s->coalesced_timer, next_clock);
     }
 }
@@ -165,10 +169,12 @@ static void periodic_timer_update(RTCState *s, int64_t current_time)
         s->period = period;
 #endif
         /* compute 32 khz clock */
-        cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, get_ticks_per_sec());
+        cur_clock =
+            muldiv64(current_time, RTC_CLOCK_RATE, NANOSECONDS_PER_SECOND);
+
         next_irq_clock = (cur_clock & ~(period - 1)) + period;
-        s->next_periodic_time =
-            muldiv64(next_irq_clock, get_ticks_per_sec(), RTC_CLOCK_RATE) + 1;
+        s->next_periodic_time = muldiv64(next_irq_clock, NANOSECONDS_PER_SECOND,
+                                         RTC_CLOCK_RATE) + 1;
         timer_mod(s->periodic_timer, s->next_periodic_time);
     } else {
 #ifdef TARGET_I386
diff --git a/qemu/hw/timer/milkymist-sysctl.c b/qemu/hw/timer/milkymist-sysctl.c
index 30535a4e3..5f2948037 100644
--- a/qemu/hw/timer/milkymist-sysctl.c
+++ b/qemu/hw/timer/milkymist-sysctl.c
@@ -21,6 +21,7 @@
  *   http://www.milkymist.org/socdoc/sysctl.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/hw/timer/omap_gptimer.c b/qemu/hw/timer/omap_gptimer.c
index b8c8c0137..3a4386304 100644
--- a/qemu/hw/timer/omap_gptimer.c
+++ b/qemu/hw/timer/omap_gptimer.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -401,7 +402,7 @@ static void omap_gp_timer_write(void *opaque, hwaddr addr,
         if (s->trigger == gpt_trigger_none)
             omap_gp_timer_out(s, s->scpwm);
         /* TODO: make sure this doesn't overflow 32-bits */
-        s->ticks_per_sec = get_ticks_per_sec() << (s->pre ? s->ptv + 1 : 0);
+        s->ticks_per_sec = NANOSECONDS_PER_SECOND << (s->pre ? s->ptv + 1 : 0);
         omap_gp_timer_update(s);
         break;
 
@@ -468,8 +469,7 @@ static const MemoryRegionOps omap_gp_timer_ops = {
 struct omap_gp_timer_s *omap_gp_timer_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_gp_timer_s *s = (struct omap_gp_timer_s *)
-            g_malloc0(sizeof(struct omap_gp_timer_s));
+    struct omap_gp_timer_s *s = g_new0(struct omap_gp_timer_s, 1);
 
     s->ta = ta;
     s->irq = irq;
diff --git a/qemu/hw/timer/omap_synctimer.c b/qemu/hw/timer/omap_synctimer.c
index 8e50488d1..9ee651979 100644
--- a/qemu/hw/timer/omap_synctimer.c
+++ b/qemu/hw/timer/omap_synctimer.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -28,7 +29,8 @@ struct omap_synctimer_s {
 
 /* 32-kHz Sync Timer of the OMAP2 */
 static uint32_t omap_synctimer_read(struct omap_synctimer_s *s) {
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 0x8000, get_ticks_per_sec());
+    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 0x8000,
+                    NANOSECONDS_PER_SECOND);
 }
 
 void omap_synctimer_reset(struct omap_synctimer_s *s)
diff --git a/qemu/hw/timer/pl031.c b/qemu/hw/timer/pl031.c
index 34d9b44e7..38e0cb5ad 100644
--- a/qemu/hw/timer/pl031.c
+++ b/qemu/hw/timer/pl031.c
@@ -11,9 +11,11 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_PL031
 
@@ -79,7 +81,7 @@ static void pl031_interrupt(void * opaque)
 static uint32_t pl031_get_count(PL031State *s)
 {
     int64_t now = qemu_clock_get_ns(rtc_clock);
-    return s->tick_offset + now / get_ticks_per_sec();
+    return s->tick_offset + now / NANOSECONDS_PER_SECOND;
 }
 
 static void pl031_set_alarm(PL031State *s)
@@ -95,7 +97,7 @@ static void pl031_set_alarm(PL031State *s)
         pl031_interrupt(s);
     } else {
         int64_t now = qemu_clock_get_ns(rtc_clock);
-        timer_mod(s->timer, now + (int64_t)ticks * get_ticks_per_sec());
+        timer_mod(s->timer, now + (int64_t)ticks * NANOSECONDS_PER_SECOND);
     }
 }
 
@@ -191,21 +193,21 @@ static const MemoryRegionOps pl031_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int pl031_init(SysBusDevice *dev)
+static void pl031_init(Object *obj)
 {
-    PL031State *s = PL031(dev);
+    PL031State *s = PL031(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     struct tm tm;
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pl031_ops, s, "pl031", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &pl031_ops, s, "pl031", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
 
     sysbus_init_irq(dev, &s->irq);
     qemu_get_timedate(&tm, 0);
     s->tick_offset = mktimegm(&tm) -
-        qemu_clock_get_ns(rtc_clock) / get_ticks_per_sec();
+        qemu_clock_get_ns(rtc_clock) / NANOSECONDS_PER_SECOND;
 
     s->timer = timer_new_ns(rtc_clock, pl031_interrupt, s);
-    return 0;
 }
 
 static void pl031_pre_save(void *opaque)
@@ -215,7 +217,7 @@ static void pl031_pre_save(void *opaque)
     /* tick_offset is base_time - rtc_clock base time.  Instead, we want to
      * store the base time relative to the QEMU_CLOCK_VIRTUAL for backwards-compatibility.  */
     int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset_vmstate = s->tick_offset + delta / get_ticks_per_sec();
+    s->tick_offset_vmstate = s->tick_offset + delta / NANOSECONDS_PER_SECOND;
 }
 
 static int pl031_post_load(void *opaque, int version_id)
@@ -223,7 +225,7 @@ static int pl031_post_load(void *opaque, int version_id)
     PL031State *s = opaque;
 
     int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset = s->tick_offset_vmstate - delta / get_ticks_per_sec();
+    s->tick_offset = s->tick_offset_vmstate - delta / NANOSECONDS_PER_SECOND;
     pl031_set_alarm(s);
     return 0;
 }
@@ -248,9 +250,7 @@ static const VMStateDescription vmstate_pl031 = {
 static void pl031_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pl031_init;
     dc->vmsd = &vmstate_pl031;
 }
 
@@ -258,6 +258,7 @@ static const TypeInfo pl031_info = {
     .name          = TYPE_PL031,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PL031State),
+    .instance_init = pl031_init,
     .class_init    = pl031_class_init,
 };
 
diff --git a/qemu/hw/timer/puv3_ost.c b/qemu/hw/timer/puv3_ost.c
index fa9eefd92..93650b799 100644
--- a/qemu/hw/timer/puv3_ost.c
+++ b/qemu/hw/timer/puv3_ost.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/main-loop.h"
diff --git a/qemu/hw/timer/pxa2xx_timer.c b/qemu/hw/timer/pxa2xx_timer.c
index 130e9dc34..59002b407 100644
--- a/qemu/hw/timer/pxa2xx_timer.c
+++ b/qemu/hw/timer/pxa2xx_timer.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
@@ -118,11 +119,11 @@ static void pxa2xx_timer_update(void *opaque, uint64_t now_qemu)
     uint64_t new_qemu;
 
     now_vm = s->clock +
-            muldiv64(now_qemu - s->lastload, s->freq, get_ticks_per_sec());
+            muldiv64(now_qemu - s->lastload, s->freq, NANOSECONDS_PER_SECOND);
 
     for (i = 0; i < 4; i ++) {
         new_qemu = now_qemu + muldiv64((uint32_t) (s->timer[i].value - now_vm),
-                        get_ticks_per_sec(), s->freq);
+                        NANOSECONDS_PER_SECOND, s->freq);
         timer_mod(s->timer[i].qtimer, new_qemu);
     }
 }
@@ -147,10 +148,10 @@ static void pxa2xx_timer_update4(void *opaque, uint64_t now_qemu, int n)
 
     now_vm = s->tm4[counter].clock + muldiv64(now_qemu -
                     s->tm4[counter].lastload,
-                    s->tm4[counter].freq, get_ticks_per_sec());
+                    s->tm4[counter].freq, NANOSECONDS_PER_SECOND);
 
     new_qemu = now_qemu + muldiv64((uint32_t) (s->tm4[n].tm.value - now_vm),
-                    get_ticks_per_sec(), s->tm4[counter].freq);
+                    NANOSECONDS_PER_SECOND, s->tm4[counter].freq);
     timer_mod(s->tm4[n].tm.qtimer, new_qemu);
 }
 
@@ -189,7 +190,7 @@ static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
         return s->tm4[tm].tm.value;
     case OSCR:
         return s->clock + muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                        s->lastload, s->freq, get_ticks_per_sec());
+                        s->lastload, s->freq, NANOSECONDS_PER_SECOND);
     case OSCR11: tm ++;
         /* fall through */
     case OSCR10: tm ++;
@@ -213,15 +214,17 @@ static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
                 s->snapshot = s->tm4[tm - 1].clock + muldiv64(
                                 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
                                 s->tm4[tm - 1].lastload,
-                                s->tm4[tm - 1].freq, get_ticks_per_sec());
+                                s->tm4[tm - 1].freq, NANOSECONDS_PER_SECOND);
             else
                 s->snapshot = s->tm4[tm - 1].clock;
         }
 
         if (!s->tm4[tm].freq)
             return s->tm4[tm].clock;
-        return s->tm4[tm].clock + muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                        s->tm4[tm].lastload, s->tm4[tm].freq, get_ticks_per_sec());
+        return s->tm4[tm].clock +
+            muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
+                     s->tm4[tm].lastload, s->tm4[tm].freq,
+                     NANOSECONDS_PER_SECOND);
     case OIER:
         return s->irq_enabled;
     case OSSR:	/* Status register */
@@ -432,10 +435,10 @@ static int pxa25x_timer_post_load(void *opaque, int version_id)
     return 0;
 }
 
-static int pxa2xx_timer_init(SysBusDevice *dev)
+static void pxa2xx_timer_init(Object *obj)
 {
-    PXA2xxTimerInfo *s = PXA2XX_TIMER(dev);
-    int i;
+    PXA2xxTimerInfo *s = PXA2XX_TIMER(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     s->irq_enabled = 0;
     s->oldclock = 0;
@@ -443,16 +446,28 @@ static int pxa2xx_timer_init(SysBusDevice *dev)
     s->lastload = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     s->reset3 = 0;
 
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_timer_ops, s,
+                          "pxa2xx-timer", 0x00001000);
+    sysbus_init_mmio(dev, &s->iomem);
+}
+
+static void pxa2xx_timer_realize(DeviceState *dev, Error **errp)
+{
+    PXA2xxTimerInfo *s = PXA2XX_TIMER(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    int i;
+
     for (i = 0; i < 4; i ++) {
         s->timer[i].value = 0;
-        sysbus_init_irq(dev, &s->timer[i].irq);
+        sysbus_init_irq(sbd, &s->timer[i].irq);
         s->timer[i].info = s;
         s->timer[i].num = i;
         s->timer[i].qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                        pxa2xx_timer_tick, &s->timer[i]);
+                                          pxa2xx_timer_tick, &s->timer[i]);
     }
+
     if (s->flags & (1 << PXA2XX_TIMER_HAVE_TM4)) {
-        sysbus_init_irq(dev, &s->irq4);
+        sysbus_init_irq(sbd, &s->irq4);
 
         for (i = 0; i < 8; i ++) {
             s->tm4[i].tm.value = 0;
@@ -461,15 +476,9 @@ static int pxa2xx_timer_init(SysBusDevice *dev)
             s->tm4[i].freq = 0;
             s->tm4[i].control = 0x0;
             s->tm4[i].tm.qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                        pxa2xx_timer_tick4, &s->tm4[i]);
+                                               pxa2xx_timer_tick4, &s->tm4[i]);
         }
     }
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_timer_ops, s,
-                          "pxa2xx-timer", 0x00001000);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_pxa2xx_timer0_regs = {
@@ -572,9 +581,8 @@ static const TypeInfo pxa27x_timer_dev_info = {
 static void pxa2xx_timer_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(oc);
 
-    sdc->init = pxa2xx_timer_init;
+    dc->realize  = pxa2xx_timer_realize;
     dc->vmsd = &vmstate_pxa2xx_timer_regs;
 }
 
@@ -582,6 +590,7 @@ static const TypeInfo pxa2xx_timer_type_info = {
     .name          = TYPE_PXA2XX_TIMER,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
+    .instance_init = pxa2xx_timer_init,
     .abstract      = true,
     .class_init    = pxa2xx_timer_class_init,
 };
diff --git a/qemu/hw/timer/sh_timer.c b/qemu/hw/timer/sh_timer.c
index 07f0670b5..255b2fc91 100644
--- a/qemu/hw/timer/sh_timer.c
+++ b/qemu/hw/timer/sh_timer.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/timer/slavio_timer.c b/qemu/hw/timer/slavio_timer.c
index 45d97e667..fb3e08bed 100644
--- a/qemu/hw/timer/slavio_timer.c
+++ b/qemu/hw/timer/slavio_timer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
diff --git a/qemu/hw/timer/stm32f2xx_timer.c b/qemu/hw/timer/stm32f2xx_timer.c
index ecadf9df8..55dacbbe3 100644
--- a/qemu/hw/timer/stm32f2xx_timer.c
+++ b/qemu/hw/timer/stm32f2xx_timer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/timer/stm32f2xx_timer.h"
 
 #ifndef STM_TIMER_ERR_DEBUG
diff --git a/qemu/hw/timer/twl92230.c b/qemu/hw/timer/twl92230.c
index 7ded4ba2a..7ba4e9a7c 100644
--- a/qemu/hw/timer/twl92230.c
+++ b/qemu/hw/timer/twl92230.c
@@ -19,11 +19,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/i2c/i2c.h"
 #include "sysemu/sysemu.h"
 #include "ui/console.h"
+#include "qemu/bcd.h"
 
 #define VERBOSE 1
 
diff --git a/qemu/hw/timer/xilinx_timer.c b/qemu/hw/timer/xilinx_timer.c
index cdb335517..2ea970dc9 100644
--- a/qemu/hw/timer/xilinx_timer.c
+++ b/qemu/hw/timer/xilinx_timer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
diff --git a/qemu/hw/tpm/tpm_passthrough.c b/qemu/hw/tpm/tpm_passthrough.c
index 79a8f98af..e88c0d20b 100644
--- a/qemu/hw/tpm/tpm_passthrough.c
+++ b/qemu/hw/tpm/tpm_passthrough.c
@@ -22,10 +22,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
-#include <dirent.h>
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "sysemu/tpm_backend.h"
@@ -85,12 +83,37 @@ static void tpm_passthrough_cancel_cmd(TPMBackend *tb);
 
 static int tpm_passthrough_unix_write(int fd, const uint8_t *buf, uint32_t len)
 {
-    return send_all(fd, buf, len);
+    int ret, remain;
+
+    remain = len;
+    while (remain > 0) {
+        ret = write(fd, buf, remain);
+        if (ret < 0) {
+            if (errno != EINTR && errno != EAGAIN) {
+                return -1;
+            }
+        } else if (ret == 0) {
+            break;
+        } else {
+            buf += ret;
+            remain -= ret;
+        }
+    }
+    return len - remain;
 }
 
 static int tpm_passthrough_unix_read(int fd, uint8_t *buf, uint32_t len)
 {
-    return recv_all(fd, buf, len, true);
+    int ret;
+ reread:
+    ret = read(fd, buf, len);
+    if (ret < 0) {
+        if (errno != EINTR && errno != EAGAIN) {
+            return -1;
+        }
+        goto reread;
+    }
+    return ret;
 }
 
 static uint32_t tpm_passthrough_get_size_from_buffer(const uint8_t *buf)
diff --git a/qemu/hw/tpm/tpm_tis.c b/qemu/hw/tpm/tpm_tis.c
index 0806b5f82..381e7266e 100644
--- a/qemu/hw/tpm/tpm_tis.c
+++ b/qemu/hw/tpm/tpm_tis.c
@@ -22,6 +22,7 @@
  * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/tpm_backend.h"
 #include "tpm_int.h"
 #include "sysemu/block-backend.h"
@@ -30,6 +31,7 @@
 #include "hw/i386/pc.h"
 #include "hw/pci/pci_ids.h"
 #include "tpm_tis.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 #include "sysemu/tpm_backend.h"
@@ -141,7 +143,7 @@
 
 #define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS1_3 \
     (TPM_TIS_IFACE_ID_INTERFACE_TIS1_3 | \
-     (~0 << 4)/* all of it is don't care */)
+     (~0u << 4)/* all of it is don't care */)
 
 /* if backend was a TPM 2.0: */
 #define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS2_0 \
@@ -1051,7 +1053,7 @@ static void tpm_tis_realizefn(DeviceState *dev, Error **errp)
 
     if (tis->irq_num > 15) {
         error_setg(errp, "tpm_tis: IRQ %d for TPM TIS is outside valid range "
-                   "of 0 to 15.\n", tis->irq_num);
+                   "of 0 to 15", tis->irq_num);
         return;
     }
 
diff --git a/qemu/hw/tpm/tpm_util.c b/qemu/hw/tpm/tpm_util.c
index 4ace5852e..7b3542972 100644
--- a/qemu/hw/tpm/tpm_util.c
+++ b/qemu/hw/tpm/tpm_util.c
@@ -19,6 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
+#include "qemu/osdep.h"
 #include "tpm_util.h"
 #include "tpm_int.h"
 
diff --git a/qemu/hw/tricore/tricore_testboard.c b/qemu/hw/tricore/tricore_testboard.c
index a059a20a3..8d3520f5b 100644
--- a/qemu/hw/tricore/tricore_testboard.c
+++ b/qemu/hw/tricore/tricore_testboard.c
@@ -18,6 +18,10 @@
  */
 
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "net/net.h"
@@ -44,7 +48,7 @@ static void tricore_load_kernel(CPUTriCoreState *env)
     kernel_size = load_elf(tricoretb_binfo.kernel_filename, NULL,
                            NULL, (uint64_t *)&entry, NULL,
                            NULL, 0,
-                           ELF_MACHINE, 1);
+                           EM_TRICORE, 1, 0);
     if (kernel_size <= 0) {
         error_report("qemu: no kernel file '%s'",
                 tricoretb_binfo.kernel_filename);
@@ -76,17 +80,23 @@ static void tricore_testboard_init(MachineState *machine, int board_id)
         exit(1);
     }
     env = &cpu->env;
-    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024, &error_abort);
+    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_cram);
-    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024, &error_abort);
+    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_dram);
-    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_cram);
-    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_dram);
-    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024, &error_abort);
+    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_data);
-    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024, &error_abort);
+    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_text);
 
     memory_region_add_subregion(sysmem, 0x80000000, ext_cram);
@@ -109,16 +119,11 @@ static void tricoreboard_init(MachineState *machine)
     tricore_testboard_init(machine, 0x183);
 }
 
-static QEMUMachine ttb_machine = {
-    .name = "tricore_testboard",
-    .desc = "a minimal TriCore board",
-    .init = tricoreboard_init,
-    .is_default = 0,
-};
-
-static void tricore_testboard_machine_init(void)
+static void ttb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ttb_machine);
+    mc->desc = "a minimal TriCore board";
+    mc->init = tricoreboard_init;
+    mc->is_default = 0;
 }
 
-machine_init(tricore_testboard_machine_init);
+DEFINE_MACHINE("tricore_testboard", ttb_machine_init)
diff --git a/qemu/hw/unicore32/puv3.c b/qemu/hw/unicore32/puv3.c
index 703e29d6d..31cd17101 100644
--- a/qemu/hw/unicore32/puv3.c
+++ b/qemu/hw/unicore32/puv3.c
@@ -9,6 +9,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "elf.h"
@@ -17,6 +21,7 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "hw/i386/pc.h"
+#include "qemu/error-report.h"
 #include "sysemu/qtest.h"
 
 #undef DEBUG_PUV3
@@ -75,7 +80,7 @@ static void puv3_board_init(CPUUniCore32State *env, ram_addr_t ram_size)
 
     /* SDRAM at address zero.  */
     memory_region_init_ram(ram_memory, NULL, "puv3.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_memory);
     memory_region_add_subregion(get_system_memory(), 0, ram_memory);
 }
@@ -95,7 +100,8 @@ static void puv3_load_kernel(const char *kernel_filename)
     size = load_image_targphys(kernel_filename, KERNEL_LOAD_ADDR,
             KERNEL_MAX_SIZE);
     if (size < 0) {
-        hw_error("Load kernel error: '%s'\n", kernel_filename);
+        error_report("Load kernel error: '%s'", kernel_filename);
+        exit(1);
     }
 
     /* cheat curses that we have a graphic console, only under ocd console */
@@ -112,7 +118,8 @@ static void puv3_init(MachineState *machine)
     UniCore32CPU *cpu;
 
     if (initrd_filename) {
-        hw_error("Please use kernel built-in initramdisk.\n");
+        error_report("Please use kernel built-in initramdisk");
+        exit(1);
     }
 
     if (!cpu_model) {
@@ -121,7 +128,8 @@ static void puv3_init(MachineState *machine)
 
     cpu = uc32_cpu_init(cpu_model);
     if (!cpu) {
-        hw_error("Unable to find CPU definition\n");
+        error_report("Unable to find CPU definition");
+        exit(1);
     }
     env = &cpu->env;
 
@@ -130,16 +138,11 @@ static void puv3_init(MachineState *machine)
     puv3_load_kernel(kernel_filename);
 }
 
-static QEMUMachine puv3_machine = {
-    .name = "puv3",
-    .desc = "PKUnity Version-3 based on UniCore32",
-    .init = puv3_init,
-    .is_default = 1,
-};
-
-static void puv3_machine_init(void)
+static void puv3_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&puv3_machine);
+    mc->desc = "PKUnity Version-3 based on UniCore32";
+    mc->init = puv3_init;
+    mc->is_default = 1;
 }
 
-machine_init(puv3_machine_init)
+DEFINE_MACHINE("puv3", puv3_machine_init)
diff --git a/qemu/hw/usb/Makefile.objs b/qemu/hw/usb/Makefile.objs
index 7443e386b..2717027d3 100644
--- a/qemu/hw/usb/Makefile.objs
+++ b/qemu/hw/usb/Makefile.objs
@@ -10,6 +10,8 @@ common-obj-$(CONFIG_USB_EHCI_SYSBUS) += hcd-ehci-sysbus.o
 common-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
 common-obj-$(CONFIG_USB_MUSB) += hcd-musb.o
 
+obj-$(CONFIG_TUSB6010) += tusb6010.o
+
 # emulated usb devices
 common-obj-$(CONFIG_USB) += dev-hub.o
 common-obj-$(CONFIG_USB) += dev-hid.o
@@ -23,9 +25,8 @@ common-obj-$(CONFIG_USB_BLUETOOTH)    += dev-bluetooth.o
 
 ifeq ($(CONFIG_USB_SMARTCARD),y)
 common-obj-y                          += dev-smartcard-reader.o
-common-obj-y                          += ccid-card-passthru.o
-common-obj-$(CONFIG_SMARTCARD_NSS)    += ccid-card-emulated.o
-ccid-card-emulated.o-cflags := -I$(SRC_PATH)/libcacard
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-passthru.o
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-emulated.o
 endif
 
 ifeq ($(CONFIG_POSIX),y)
diff --git a/qemu/hw/usb/bus.c b/qemu/hw/usb/bus.c
index 5f39e1e3a..16c3461d9 100644
--- a/qemu/hw/usb/bus.c
+++ b/qemu/hw/usb/bus.c
@@ -1,10 +1,13 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
 #include "trace.h"
+#include "qemu/cutils.h"
 
 static void usb_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent);
 
@@ -329,9 +332,9 @@ static USBDevice *usb_try_create_simple(USBBus *bus, const char *name,
     }
     object_property_set_bool(OBJECT(dev), true, "realized", &err);
     if (err) {
-        error_setg(errp, "Failed to initialize USB device '%s': %s",
-                   name, error_get_pretty(err));
-        error_free(err);
+        error_propagate(errp, err);
+        error_prepend(errp, "Failed to initialize USB device '%s': ",
+                      name);
         object_unparent(OBJECT(dev));
         return NULL;
     }
@@ -655,9 +658,12 @@ void hmp_info_usb(Monitor *mon, const QDict *qdict)
             dev = port->dev;
             if (!dev)
                 continue;
-            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, Product %s\n",
-                           bus->busnr, dev->addr, port->path, usb_speed(dev->speed),
-                           dev->product_desc);
+            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, "
+                           "Product %s%s%s\n",
+                           bus->busnr, dev->addr, port->path,
+                           usb_speed(dev->speed), dev->product_desc,
+                           dev->qdev.id ? ", ID: " : "",
+                           dev->qdev.id ?: "");
         }
     }
 }
@@ -722,9 +728,8 @@ USBDevice *usbdevice_create(const char *cmdline)
     }
     object_property_set_bool(OBJECT(dev), true, "realized", &err);
     if (err) {
-        error_report("Failed to initialize USB device '%s': %s",
-                     f->name, error_get_pretty(err));
-        error_free(err);
+        error_reportf_err(err, "Failed to initialize USB device '%s': ",
+                          f->name);
         object_unparent(OBJECT(dev));
         return NULL;
     }
diff --git a/qemu/hw/usb/ccid-card-emulated.c b/qemu/hw/usb/ccid-card-emulated.c
index 72329ed7d..3213f9f8a 100644
--- a/qemu/hw/usb/ccid-card-emulated.c
+++ b/qemu/hw/usb/ccid-card-emulated.c
@@ -26,6 +26,7 @@
  * the db parameter.
  */
 
+#include "qemu/osdep.h"
 #include <eventt.h>
 #include <vevent.h>
 #include <vreader.h>
@@ -42,7 +43,10 @@ do {\
     } \
 } while (0)
 
-#define EMULATED_DEV_NAME "ccid-card-emulated"
+
+#define TYPE_EMULATED_CCID "ccid-card-emulated"
+#define EMULATED_CCID_CARD(obj) \
+    OBJECT_CHECK(EmulatedState, (obj), TYPE_EMULATED_CCID)
 
 #define BACKEND_NSS_EMULATED_NAME "nss-emulated"
 #define BACKEND_CERTIFICATES_NAME "certificates"
@@ -133,7 +137,7 @@ struct EmulatedState {
 static void emulated_apdu_from_guest(CCIDCardState *base,
     const uint8_t *apdu, uint32_t len)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent) + len);
 
     assert(event);
@@ -150,7 +154,7 @@ static void emulated_apdu_from_guest(CCIDCardState *base,
 
 static const uint8_t *emulated_get_atr(CCIDCardState *base, uint32_t *len)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
 
     *len = card->atr_length;
     return card->atr;
@@ -166,7 +170,7 @@ static void emulated_push_event(EmulatedState *card, EmulEvent *event)
 
 static void emulated_push_type(EmulatedState *card, uint32_t type)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.gen.type = type;
@@ -175,7 +179,7 @@ static void emulated_push_type(EmulatedState *card, uint32_t type)
 
 static void emulated_push_error(EmulatedState *card, uint64_t code)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.error.type = EMUL_ERROR;
@@ -403,7 +407,7 @@ static int init_event_notifier(EmulatedState *card)
         DPRINTF(card, 2, "event notifier creation failed\n");
         return -1;
     }
-    event_notifier_set_handler(&card->notifier, card_event_handler);
+    event_notifier_set_handler(&card->notifier, false, card_event_handler);
     return 0;
 }
 
@@ -478,7 +482,7 @@ static uint32_t parse_enumeration(char *str,
 
 static int emulated_initfn(CCIDCardState *base)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     VCardEmulError ret;
     const EnumTable *ptable;
 
@@ -514,26 +518,26 @@ static int emulated_initfn(CCIDCardState *base)
             ret = emulated_initialize_vcard_from_certificates(card);
         } else {
             printf("%s: you must provide all three certs for"
-                   " certificates backend\n", EMULATED_DEV_NAME);
+                   " certificates backend\n", TYPE_EMULATED_CCID);
             return -1;
         }
     } else {
         if (card->backend != BACKEND_NSS_EMULATED) {
             printf("%s: bad backend specified. The options are:\n%s (default),"
-                " %s.\n", EMULATED_DEV_NAME, BACKEND_NSS_EMULATED_NAME,
+                " %s.\n", TYPE_EMULATED_CCID, BACKEND_NSS_EMULATED_NAME,
                 BACKEND_CERTIFICATES_NAME);
             return -1;
         }
         if (card->cert1 != NULL || card->cert2 != NULL || card->cert3 != NULL) {
             printf("%s: unexpected cert parameters to nss emulated backend\n",
-                   EMULATED_DEV_NAME);
+                   TYPE_EMULATED_CCID);
             return -1;
         }
         /* default to mirroring the local hardware readers */
         ret = wrap_vcard_emul_init(NULL);
     }
     if (ret != VCARD_EMUL_OK) {
-        printf("%s: failed to initialize vcard\n", EMULATED_DEV_NAME);
+        printf("%s: failed to initialize vcard\n", TYPE_EMULATED_CCID);
         return -1;
     }
     qemu_thread_create(&card->event_thread_id, "ccid/event", event_thread,
@@ -545,7 +549,7 @@ static int emulated_initfn(CCIDCardState *base)
 
 static int emulated_exitfn(CCIDCardState *base)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     VEvent *vevent = vevent_new(VEVENT_LAST, NULL, NULL);
 
     vevent_queue_vevent(vevent); /* stop vevent thread */
@@ -588,7 +592,7 @@ static void emulated_class_initfn(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo emulated_card_info = {
-    .name          = EMULATED_DEV_NAME,
+    .name          = TYPE_EMULATED_CCID,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(EmulatedState),
     .class_init    = emulated_class_initfn,
diff --git a/qemu/hw/usb/ccid-card-passthru.c b/qemu/hw/usb/ccid-card-passthru.c
index 85a4fc3e5..c0e90e501 100644
--- a/qemu/hw/usb/ccid-card-passthru.c
+++ b/qemu/hw/usb/ccid-card-passthru.c
@@ -8,11 +8,12 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "ccid.h"
-#include "libcacard/vscard_common.h"
+#include "cacard/vscard_common.h"
 
 #define DPRINTF(card, lvl, fmt, ...)                    \
 do {                                                    \
@@ -38,8 +39,6 @@ static const uint8_t DEFAULT_ATR[] = {
  0x13, 0x08
 };
 
-
-#define PASSTHRU_DEV_NAME "ccid-card-passthru"
 #define VSCARD_IN_SIZE 65536
 
 /* maximum size of ATR - from 7816-3 */
@@ -58,6 +57,10 @@ struct PassthruState {
     uint8_t  debug;
 };
 
+#define TYPE_CCID_PASSTHRU "ccid-card-passthru"
+#define PASSTHRU_CCID_CARD(obj) \
+    OBJECT_CHECK(PassthruState, (obj), TYPE_CCID_PASSTHRU)
+
 /*
  * VSCard protocol over chardev
  * This code should not depend on the card type.
@@ -316,7 +319,7 @@ static void ccid_card_vscard_event(void *opaque, int event)
 static void passthru_apdu_from_guest(
     CCIDCardState *base, const uint8_t *apdu, uint32_t len)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     if (!card->cs) {
         printf("ccid-passthru: no chardev, discarding apdu length %d\n", len);
@@ -327,7 +330,7 @@ static void passthru_apdu_from_guest(
 
 static const uint8_t *passthru_get_atr(CCIDCardState *base, uint32_t *len)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     *len = card->atr_length;
     return card->atr;
@@ -335,7 +338,7 @@ static const uint8_t *passthru_get_atr(CCIDCardState *base, uint32_t *len)
 
 static int passthru_initfn(CCIDCardState *base)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     card->vscard_in_pos = 0;
     card->vscard_in_hdr = 0;
@@ -399,7 +402,7 @@ static void passthru_class_initfn(ObjectClass *klass, void *data)
 }
 
 static const TypeInfo passthru_card_info = {
-    .name          = PASSTHRU_DEV_NAME,
+    .name          = TYPE_CCID_PASSTHRU,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(PassthruState),
     .class_init    = passthru_class_initfn,
diff --git a/qemu/hw/usb/combined-packet.c b/qemu/hw/usb/combined-packet.c
index ad77705f8..48cac87f6 100644
--- a/qemu/hw/usb/combined-packet.c
+++ b/qemu/hw/usb/combined-packet.c
@@ -19,6 +19,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "qemu/iov.h"
diff --git a/qemu/hw/usb/core.c b/qemu/hw/usb/core.c
index d0025db60..45fa00c51 100644
--- a/qemu/hw/usb/core.c
+++ b/qemu/hw/usb/core.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "qemu/iov.h"
@@ -128,9 +129,16 @@ static void do_token_setup(USBDevice *s, USBPacket *p)
     }
 
     usb_packet_copy(p, s->setup_buf, p->iov.size);
+    s->setup_index = 0;
     p->actual_length = 0;
     s->setup_len   = (s->setup_buf[7] << 8) | s->setup_buf[6];
-    s->setup_index = 0;
+    if (s->setup_len > sizeof(s->data_buf)) {
+        fprintf(stderr,
+                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
+                s->setup_len, sizeof(s->data_buf));
+        p->status = USB_RET_STALL;
+        return;
+    }
 
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
@@ -151,13 +159,6 @@ static void do_token_setup(USBDevice *s, USBPacket *p)
         }
         s->setup_state = SETUP_STATE_DATA;
     } else {
-        if (s->setup_len > sizeof(s->data_buf)) {
-            fprintf(stderr,
-                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
-                s->setup_len, sizeof(s->data_buf));
-            p->status = USB_RET_STALL;
-            return;
-        }
         if (s->setup_len == 0)
             s->setup_state = SETUP_STATE_ACK;
         else
@@ -176,7 +177,7 @@ static void do_token_in(USBDevice *s, USBPacket *p)
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
     index   = (s->setup_buf[5] << 8) | s->setup_buf[4];
- 
+
     switch(s->setup_state) {
     case SETUP_STATE_ACK:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
diff --git a/qemu/hw/usb/desc-msos.c b/qemu/hw/usb/desc-msos.c
index 32c3600df..365291981 100644
--- a/qemu/hw/usb/desc-msos.c
+++ b/qemu/hw/usb/desc-msos.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 
diff --git a/qemu/hw/usb/desc.c b/qemu/hw/usb/desc.c
index b82c397ef..adb026e43 100644
--- a/qemu/hw/usb/desc.c
+++ b/qemu/hw/usb/desc.c
@@ -1,4 +1,4 @@
-#include <ctype.h>
+#include "qemu/osdep.h"
 
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
diff --git a/qemu/hw/usb/desc.h b/qemu/hw/usb/desc.h
index 8e8db03a0..4d81c68e0 100644
--- a/qemu/hw/usb/desc.h
+++ b/qemu/hw/usb/desc.h
@@ -1,7 +1,6 @@
 #ifndef QEMU_HW_USB_DESC_H
 #define QEMU_HW_USB_DESC_H
 
-#include <inttypes.h>
 #include <wchar.h>
 
 /* binary representation */
diff --git a/qemu/hw/usb/dev-audio.c b/qemu/hw/usb/dev-audio.c
index f092bb849..87cab0a3d 100644
--- a/qemu/hw/usb/dev-audio.c
+++ b/qemu/hw/usb/dev-audio.c
@@ -29,6 +29,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -664,7 +665,7 @@ static const VMStateDescription vmstate_usb_audio = {
 static Property usb_audio_properties[] = {
     DEFINE_PROP_UINT32("debug", USBAudioState, debug, 0),
     DEFINE_PROP_UINT32("buffer", USBAudioState, buffer,
-                       8 * USBAUDIO_PACKET_SIZE),
+                       32 * USBAUDIO_PACKET_SIZE),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/qemu/hw/usb/dev-bluetooth.c b/qemu/hw/usb/dev-bluetooth.c
index b19ec76b0..91a4a0b8b 100644
--- a/qemu/hw/usb/dev-bluetooth.c
+++ b/qemu/hw/usb/dev-bluetooth.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/dev-hid.c b/qemu/hw/usb/dev-hid.c
index 2e7dcd96c..24d05f76f 100644
--- a/qemu/hw/usb/dev-hid.c
+++ b/qemu/hw/usb/dev-hid.c
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/input/hid.h"
 
diff --git a/qemu/hw/usb/dev-hub.c b/qemu/hw/usb/dev-hub.c
index c8c685550..a33f21cb3 100644
--- a/qemu/hw/usb/dev-hub.c
+++ b/qemu/hw/usb/dev-hub.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "trace.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/dev-mtp.c b/qemu/hw/usb/dev-mtp.c
index 809b1cb11..bda84a64b 100644
--- a/qemu/hw/usb/dev-mtp.c
+++ b/qemu/hw/usb/dev-mtp.c
@@ -9,12 +9,17 @@
  * This code is licensed under the GPL v2 or later.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <wchar.h>
 #include <dirent.h>
-#include <unistd.h>
 
-#include <sys/stat.h>
 #include <sys/statvfs.h>
+#ifdef CONFIG_INOTIFY1
+#include <sys/inotify.h>
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#endif
 
 #include "qemu-common.h"
 #include "qemu/iov.h"
@@ -62,6 +67,11 @@ enum mtp_code {
     /* format codes */
     FMT_UNDEFINED_OBJECT           = 0x3000,
     FMT_ASSOCIATION                = 0x3001,
+
+    /* event codes */
+    EVT_OBJ_ADDED                  = 0x4002,
+    EVT_OBJ_REMOVED                = 0x4003,
+    EVT_OBJ_INFO_CHANGED           = 0x4007,
 };
 
 typedef struct {
@@ -84,6 +94,17 @@ enum {
     EP_EVENT,
 };
 
+#ifdef CONFIG_INOTIFY1
+typedef struct MTPMonEntry MTPMonEntry;
+
+struct MTPMonEntry {
+    uint32_t event;
+    uint32_t handle;
+
+    QTAILQ_ENTRY(MTPMonEntry) next;
+};
+#endif
+
 struct MTPControl {
     uint16_t     code;
     uint32_t     trans;
@@ -108,9 +129,14 @@ struct MTPObject {
     char         *name;
     char         *path;
     struct stat  stat;
+#ifdef CONFIG_INOTIFY1
+    /* inotify watch cookie */
+    int          watchfd;
+#endif
     MTPObject    *parent;
-    MTPObject    **children;
     uint32_t     nchildren;
+    QLIST_HEAD(, MTPObject) children;
+    QLIST_ENTRY(MTPObject) list;
     bool         have_children;
     QTAILQ_ENTRY(MTPObject) next;
 };
@@ -128,6 +154,11 @@ struct MTPState {
     uint32_t     next_handle;
 
     QTAILQ_HEAD(, MTPObject) objects;
+#ifdef CONFIG_INOTIFY1
+    /* inotify descriptor */
+    int          inotifyfd;
+    QTAILQ_HEAD(events, MTPMonEntry) events;
+#endif
 };
 
 #define TYPE_USB_MTP "usb-mtp"
@@ -183,7 +214,7 @@ static const USBDescIface desc_iface_full = {
         },{
             .bEndpointAddress      = USB_DIR_IN | EP_EVENT,
             .bmAttributes          = USB_ENDPOINT_XFER_INT,
-            .wMaxPacketSize        = 8,
+            .wMaxPacketSize        = 64,
             .bInterval             = 0x0a,
         },
     }
@@ -225,7 +256,7 @@ static const USBDescIface desc_iface_high = {
         },{
             .bEndpointAddress      = USB_DIR_IN | EP_EVENT,
             .bmAttributes          = USB_ENDPOINT_XFER_INT,
-            .wMaxPacketSize        = 8,
+            .wMaxPacketSize        = 64,
             .bInterval             = 0x0a,
         },
     }
@@ -317,15 +348,24 @@ ignore:
 
 static void usb_mtp_object_free(MTPState *s, MTPObject *o)
 {
-    int i;
+    MTPObject *iter;
+
+    if (!o) {
+        return;
+    }
 
     trace_usb_mtp_object_free(s->dev.addr, o->handle, o->path);
 
     QTAILQ_REMOVE(&s->objects, o, next);
-    for (i = 0; i < o->nchildren; i++) {
-        usb_mtp_object_free(s, o->children[i]);
+    if (o->parent) {
+        QLIST_REMOVE(o, list);
+        o->parent->nchildren--;
+    }
+
+    while (!QLIST_EMPTY(&o->children)) {
+        iter = QLIST_FIRST(&o->children);
+        usb_mtp_object_free(s, iter);
     }
-    g_free(o->children);
     g_free(o->name);
     g_free(o->path);
     g_free(o);
@@ -343,6 +383,203 @@ static MTPObject *usb_mtp_object_lookup(MTPState *s, uint32_t handle)
     return NULL;
 }
 
+static MTPObject *usb_mtp_add_child(MTPState *s, MTPObject *o,
+                                    char *name)
+{
+    MTPObject *child =
+        usb_mtp_object_alloc(s, s->next_handle++, o, name);
+
+    if (child) {
+        trace_usb_mtp_add_child(s->dev.addr, child->handle, child->path);
+        QLIST_INSERT_HEAD(&o->children, child, list);
+        o->nchildren++;
+
+        if (child->format == FMT_ASSOCIATION) {
+            QLIST_INIT(&child->children);
+        }
+    }
+
+    return child;
+}
+
+#ifdef CONFIG_INOTIFY1
+static MTPObject *usb_mtp_object_lookup_name(MTPObject *parent,
+                                             char *name, int len)
+{
+    MTPObject *iter;
+
+    QLIST_FOREACH(iter, &parent->children, list) {
+        if (strncmp(iter->name, name, len) == 0) {
+            return iter;
+        }
+    }
+
+    return NULL;
+}
+
+static MTPObject *usb_mtp_object_lookup_wd(MTPState *s, int wd)
+{
+    MTPObject *iter;
+
+    QTAILQ_FOREACH(iter, &s->objects, next) {
+        if (iter->watchfd == wd) {
+            return iter;
+        }
+    }
+
+    return NULL;
+}
+
+static void inotify_watchfn(void *arg)
+{
+    MTPState *s = arg;
+    ssize_t bytes;
+    /* From the man page: atleast one event can be read */
+    int pos;
+    char buf[sizeof(struct inotify_event) + NAME_MAX + 1];
+
+    for (;;) {
+        bytes = read(s->inotifyfd, buf, sizeof(buf));
+        pos = 0;
+
+        if (bytes <= 0) {
+            /* Better luck next time */
+            return;
+        }
+
+        /*
+         * TODO: Ignore initiator initiated events.
+         * For now we are good because the store is RO
+         */
+        while (bytes > 0) {
+            char *p = buf + pos;
+            struct inotify_event *event = (struct inotify_event *)p;
+            int watchfd = 0;
+            uint32_t mask = event->mask & (IN_CREATE | IN_DELETE |
+                                           IN_MODIFY | IN_IGNORED);
+            MTPObject *parent = usb_mtp_object_lookup_wd(s, event->wd);
+            MTPMonEntry *entry = NULL;
+            MTPObject *o;
+
+            pos = pos + sizeof(struct inotify_event) + event->len;
+            bytes = bytes - pos;
+
+            if (!parent) {
+                continue;
+            }
+
+            switch (mask) {
+            case IN_CREATE:
+                if (usb_mtp_object_lookup_name
+                    (parent, event->name, event->len)) {
+                    /* Duplicate create event */
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = s->next_handle;
+                entry->event = EVT_OBJ_ADDED;
+                o = usb_mtp_add_child(s, parent, event->name);
+                if (!o) {
+                    g_free(entry);
+                    continue;
+                }
+                o->watchfd = watchfd;
+                trace_usb_mtp_inotify_event(s->dev.addr, event->name,
+                                            event->mask, "Obj Added");
+                break;
+
+            case IN_DELETE:
+                /*
+                 * The kernel issues a IN_IGNORED event
+                 * when a dir containing a watchpoint is
+                 * deleted, so we don't have to delete the
+                 * watchpoint
+                 */
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                if (!o) {
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = o->handle;
+                entry->event = EVT_OBJ_REMOVED;
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj Deleted");
+                usb_mtp_object_free(s, o);
+                break;
+
+            case IN_MODIFY:
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                if (!o) {
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = o->handle;
+                entry->event = EVT_OBJ_INFO_CHANGED;
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj Modified");
+                break;
+
+            case IN_IGNORED:
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj ignored");
+                break;
+
+            default:
+                fprintf(stderr, "usb-mtp: failed to parse inotify event\n");
+                continue;
+            }
+
+            if (entry) {
+                QTAILQ_INSERT_HEAD(&s->events, entry, next);
+            }
+        }
+    }
+}
+
+static int usb_mtp_inotify_init(MTPState *s)
+{
+    int fd;
+
+    fd = inotify_init1(IN_NONBLOCK);
+    if (fd == -1) {
+        return 1;
+    }
+
+    QTAILQ_INIT(&s->events);
+    s->inotifyfd = fd;
+
+    qemu_set_fd_handler(fd, inotify_watchfn, NULL, s);
+
+    return 0;
+}
+
+static void usb_mtp_inotify_cleanup(MTPState *s)
+{
+    MTPMonEntry *e, *p;
+
+    if (!s->inotifyfd) {
+        return;
+    }
+
+    qemu_set_fd_handler(s->inotifyfd, NULL, NULL, s);
+    close(s->inotifyfd);
+
+    QTAILQ_FOREACH_SAFE(e, &s->events, next, p) {
+        QTAILQ_REMOVE(&s->events, e, next);
+        g_free(e);
+    }
+}
+
+static int usb_mtp_add_watch(int inotifyfd, char *path)
+{
+    uint32_t mask = IN_CREATE | IN_DELETE | IN_MODIFY |
+        IN_ISDIR;
+
+    return inotify_add_watch(inotifyfd, path, mask);
+}
+#endif
+
 static void usb_mtp_object_readdir(MTPState *s, MTPObject *o)
 {
     struct dirent *entry;
@@ -357,16 +594,18 @@ static void usb_mtp_object_readdir(MTPState *s, MTPObject *o)
     if (!dir) {
         return;
     }
+#ifdef CONFIG_INOTIFY1
+    int watchfd = usb_mtp_add_watch(s->inotifyfd, o->path);
+    if (watchfd == -1) {
+        fprintf(stderr, "usb-mtp: failed to add watch for %s\n", o->path);
+    } else {
+        trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                    0, "Watch Added");
+        o->watchfd = watchfd;
+    }
+#endif
     while ((entry = readdir(dir)) != NULL) {
-        if ((o->nchildren % 32) == 0) {
-            o->children = g_realloc(o->children,
-                                    (o->nchildren + 32) * sizeof(MTPObject *));
-        }
-        o->children[o->nchildren] =
-            usb_mtp_object_alloc(s, s->next_handle++, o, entry->d_name);
-        if (o->children[o->nchildren] != NULL) {
-            o->nchildren++;
-        }
+        usb_mtp_add_child(s, o, entry->d_name);
     }
     closedir(dir);
 }
@@ -480,7 +719,7 @@ static void usb_mtp_add_wstr(MTPData *data, const wchar_t *str)
 static void usb_mtp_add_str(MTPData *data, const char *str)
 {
     uint32_t len = strlen(str)+1;
-    wchar_t wstr[len];
+    wchar_t *wstr = g_new(wchar_t, len);
     size_t ret;
 
     ret = mbstowcs(wstr, str, len);
@@ -489,6 +728,8 @@ static void usb_mtp_add_str(MTPData *data, const char *str)
     } else {
         usb_mtp_add_wstr(data, wstr);
     }
+
+    g_free(wstr);
 }
 
 static void usb_mtp_add_time(MTPData *data, time_t time)
@@ -618,13 +859,15 @@ static MTPData *usb_mtp_get_object_handles(MTPState *s, MTPControl *c,
                                            MTPObject *o)
 {
     MTPData *d = usb_mtp_data_alloc(c);
-    uint32_t i, handles[o->nchildren];
+    uint32_t i = 0, handles[o->nchildren];
+    MTPObject *iter;
 
     trace_usb_mtp_op_get_object_handles(s->dev.addr, o->handle, o->path);
 
-    for (i = 0; i < o->nchildren; i++) {
-        handles[i] = o->children[i]->handle;
+    QLIST_FOREACH(iter, &o->children, list) {
+        handles[i++] = iter->handle;
     }
+    assert(i == o->nchildren);
     usb_mtp_add_u32_array(d, o->nchildren, handles);
 
     return d;
@@ -755,11 +998,19 @@ static void usb_mtp_command(MTPState *s, MTPControl *c)
         trace_usb_mtp_op_open_session(s->dev.addr);
         s->session = c->argv[0];
         usb_mtp_object_alloc(s, s->next_handle++, NULL, s->root);
+#ifdef CONFIG_INOTIFY1
+        if (usb_mtp_inotify_init(s)) {
+            fprintf(stderr, "usb-mtp: file monitoring init failed\n");
+        }
+#endif
         break;
     case CMD_CLOSE_SESSION:
         trace_usb_mtp_op_close_session(s->dev.addr);
         s->session = 0;
         s->next_handle = 0;
+#ifdef CONFIG_INOTIFY1
+        usb_mtp_inotify_cleanup(s);
+#endif
         usb_mtp_object_free(s, QTAILQ_FIRST(&s->objects));
         assert(QTAILQ_EMPTY(&s->objects));
         break;
@@ -885,6 +1136,10 @@ static void usb_mtp_handle_reset(USBDevice *dev)
 
     trace_usb_mtp_reset(s->dev.addr);
 
+#ifdef CONFIG_INOTIFY1
+    usb_mtp_inotify_cleanup(s);
+#endif
+    usb_mtp_object_free(s, QTAILQ_FIRST(&s->objects));
     s->session = 0;
     usb_mtp_data_free(s->data_in);
     s->data_in = NULL;
@@ -1044,6 +1299,31 @@ static void usb_mtp_handle_data(USBDevice *dev, USBPacket *p)
         }
         break;
     case EP_EVENT:
+#ifdef CONFIG_INOTIFY1
+        if (!QTAILQ_EMPTY(&s->events)) {
+            struct MTPMonEntry *e = QTAILQ_LAST(&s->events, events);
+            uint32_t handle;
+            int len = sizeof(container) + sizeof(uint32_t);
+
+            if (p->iov.size < len) {
+                trace_usb_mtp_stall(s->dev.addr,
+                                    "packet too small to send event");
+                p->status = USB_RET_STALL;
+                return;
+            }
+
+            QTAILQ_REMOVE(&s->events, e, next);
+            container.length = cpu_to_le32(len);
+            container.type = cpu_to_le32(TYPE_EVENT);
+            container.code = cpu_to_le16(e->event);
+            container.trans = 0; /* no trans specific events */
+            handle = cpu_to_le32(e->handle);
+            usb_packet_copy(p, &container, sizeof(container));
+            usb_packet_copy(p, &handle, sizeof(uint32_t));
+            g_free(e);
+            return;
+        }
+#endif
         p->status = USB_RET_NAK;
         return;
     default:
diff --git a/qemu/hw/usb/dev-network.c b/qemu/hw/usb/dev-network.c
index 7800ceea5..74306b58e 100644
--- a/qemu/hw/usb/dev-network.c
+++ b/qemu/hw/usb/dev-network.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -32,6 +34,7 @@
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
 #include "qemu/iov.h"
+#include "qemu/cutils.h"
 
 /*#define TRAFFIC_DEBUG*/
 /* Thanks to NetChip Technologies for donating this product ID.
@@ -653,7 +656,8 @@ typedef struct USBNetState {
 
 static int is_rndis(USBNetState *s)
 {
-    return s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE;
+    return s->dev.config ?
+            s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE : 0;
 }
 
 static int ndis_query(USBNetState *s, uint32_t oid,
@@ -914,8 +918,9 @@ static int rndis_query_response(USBNetState *s,
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     infobuflen = ndis_query(s, le32_to_cpu(buf->OID),
                             bufoffs + (uint8_t *) buf, buflen, infobuf,
@@ -960,8 +965,9 @@ static int rndis_set_response(USBNetState *s,
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     ret = ndis_set(s, le32_to_cpu(buf->OID),
                     bufoffs + (uint8_t *) buf, buflen);
@@ -1211,8 +1217,9 @@ static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     if (le32_to_cpu(msg->MessageType) == RNDIS_PACKET_MSG) {
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
-        if (offs + size <= len)
+        if (offs < len && size < len && offs + size <= len) {
             qemu_send_packet(qemu_get_queue(s->nic), s->out_buf + offs, size);
+        }
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
diff --git a/qemu/hw/usb/dev-serial.c b/qemu/hw/usb/dev-serial.c
index a6a66008e..ba8538e60 100644
--- a/qemu/hw/usb/dev-serial.c
+++ b/qemu/hw/usb/dev-serial.c
@@ -8,7 +8,10 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
diff --git a/qemu/hw/usb/dev-smartcard-reader.c b/qemu/hw/usb/dev-smartcard-reader.c
index 8952efffa..af4b85135 100644
--- a/qemu/hw/usb/dev-smartcard-reader.c
+++ b/qemu/hw/usb/dev-smartcard-reader.c
@@ -34,6 +34,8 @@
  *  Not sure which messages trigger this.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/dev-storage.c b/qemu/hw/usb/dev-storage.c
index 9a4e7dc0c..248a58045 100644
--- a/qemu/hw/usb/dev-storage.c
+++ b/qemu/hw/usb/dev-storage.c
@@ -7,6 +7,8 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
@@ -20,6 +22,7 @@
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "qapi/visitor.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_MSD
 
@@ -613,20 +616,22 @@ static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
         return;
     }
 
-    bdrv_add_key(blk_bs(blk), NULL, &err);
-    if (err) {
-        if (monitor_cur_is_qmp()) {
-            error_propagate(errp, err);
-            return;
-        }
-        error_free(err);
-        err = NULL;
-        if (cur_mon) {
-            monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
-                                        usb_msd_password_cb, s);
-            s->dev.auto_attach = 0;
-        } else {
-            autostart = 0;
+    if (blk_bs(blk)) {
+        bdrv_add_key(blk_bs(blk), NULL, &err);
+        if (err) {
+            if (monitor_cur_is_qmp()) {
+                error_propagate(errp, err);
+                return;
+            }
+            error_free(err);
+            err = NULL;
+            if (cur_mon) {
+                monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
+                                            usb_msd_password_cb, s);
+                s->dev.auto_attach = 0;
+            } else {
+                autostart = 0;
+            }
         }
     }
 
@@ -778,24 +783,24 @@ static void usb_msd_class_initfn_storage(ObjectClass *klass, void *data)
     dc->props = msd_properties;
 }
 
-static void usb_msd_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void usb_msd_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(obj);
     MSDState *s = USB_STORAGE_DEV(dev);
 
-    visit_type_int32(v, &s->conf.bootindex, name, errp);
+    visit_type_int32(v, name, &s->conf.bootindex, errp);
 }
 
-static void usb_msd_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void usb_msd_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(obj);
     MSDState *s = USB_STORAGE_DEV(dev);
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
diff --git a/qemu/hw/usb/dev-uas.c b/qemu/hw/usb/dev-uas.c
index 38b26c586..0678b1b05 100644
--- a/qemu/hw/usb/dev-uas.c
+++ b/qemu/hw/usb/dev-uas.c
@@ -9,6 +9,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
diff --git a/qemu/hw/usb/dev-wacom.c b/qemu/hw/usb/dev-wacom.c
index c2450e729..c4702dbba 100644
--- a/qemu/hw/usb/dev-wacom.c
+++ b/qemu/hw/usb/dev-wacom.c
@@ -25,6 +25,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/hcd-ehci-pci.c b/qemu/hw/usb/hcd-ehci-pci.c
index 7afa5f9d6..56577051e 100644
--- a/qemu/hw/usb/hcd-ehci-pci.c
+++ b/qemu/hw/usb/hcd-ehci-pci.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/usb/hcd-ehci.h"
 #include "qemu/range.h"
 
@@ -95,10 +96,8 @@ static void usb_ehci_pci_exit(PCIDevice *dev)
 
     usb_ehci_unrealize(s, DEVICE(dev), NULL);
 
-    if (s->irq) {
-        g_free(s->irq);
-        s->irq = NULL;
-    }
+    g_free(s->irq);
+    s->irq = NULL;
 }
 
 static void usb_ehci_pci_reset(DeviceState *dev)
diff --git a/qemu/hw/usb/hcd-ehci-sysbus.c b/qemu/hw/usb/hcd-ehci-sysbus.c
index cd1cc142a..6c20604d0 100644
--- a/qemu/hw/usb/hcd-ehci-sysbus.c
+++ b/qemu/hw/usb/hcd-ehci-sysbus.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/usb/hcd-ehci.h"
 
 static const VMStateDescription vmstate_ehci_sysbus = {
diff --git a/qemu/hw/usb/hcd-ehci.c b/qemu/hw/usb/hcd-ehci.c
index 64a54c6e8..43a8f7abc 100644
--- a/qemu/hw/usb/hcd-ehci.c
+++ b/qemu/hw/usb/hcd-ehci.c
@@ -27,6 +27,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/usb/ehci-regs.h"
 #include "hw/usb/hcd-ehci.h"
 #include "trace.h"
@@ -726,7 +728,7 @@ static void ehci_detach(USBPort *port)
     ehci_queues_rip_device(s, port->dev, 0);
     ehci_queues_rip_device(s, port->dev, 1);
 
-    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED);
+    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED|PORTSC_SUSPEND);
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
@@ -865,6 +867,7 @@ void ehci_reset(void *opaque)
     s->usbsts = USBSTS_HALT;
     s->usbsts_pending = 0;
     s->usbsts_frindex = 0;
+    ehci_update_irq(s);
 
     s->astate = EST_INACTIVE;
     s->pstate = EST_INACTIVE;
@@ -893,6 +896,11 @@ static uint64_t ehci_caps_read(void *ptr, hwaddr addr,
     return s->caps[addr];
 }
 
+static void ehci_caps_write(void *ptr, hwaddr addr,
+                             uint64_t val, unsigned size)
+{
+}
+
 static uint64_t ehci_opreg_read(void *ptr, hwaddr addr,
                                 unsigned size)
 {
@@ -1404,21 +1412,23 @@ static int ehci_process_itd(EHCIState *ehci,
         if (itd->transact[i] & ITD_XACT_ACTIVE) {
             pg   = get_field(itd->transact[i], ITD_XACT_PGSEL);
             off  = itd->transact[i] & ITD_XACT_OFFSET_MASK;
-            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
-            ptr2 = (itd->bufptr[pg+1] & ITD_BUFPTR_MASK);
             len  = get_field(itd->transact[i], ITD_XACT_LENGTH);
 
             if (len > max * mult) {
                 len = max * mult;
             }
-
-            if (len > BUFF_SIZE) {
+            if (len > BUFF_SIZE || pg > 6) {
                 return -1;
             }
 
+            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
             qemu_sglist_init(&ehci->isgl, ehci->device, 2, ehci->as);
             if (off + len > 4096) {
                 /* transfer crosses page border */
+                if (pg == 6) {
+                    return -1;  /* avoid page pg + 1 */
+                }
+                ptr2 = (itd->bufptr[pg + 1] & ITD_BUFPTR_MASK);
                 uint32_t len2 = off + len - 4096;
                 uint32_t len1 = len - len2;
                 qemu_sglist_add(&ehci->isgl, ptr1 + off, len1);
@@ -2000,6 +2010,7 @@ static int ehci_state_writeback(EHCIQueue *q)
 static void ehci_advance_state(EHCIState *ehci, int async)
 {
     EHCIQueue *q = NULL;
+    int itd_count = 0;
     int again;
 
     do {
@@ -2024,10 +2035,12 @@ static void ehci_advance_state(EHCIState *ehci, int async)
 
         case EST_FETCHITD:
             again = ehci_state_fetchitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_FETCHSITD:
             again = ehci_state_fetchsitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_ADVANCEQUEUE:
@@ -2076,7 +2089,8 @@ static void ehci_advance_state(EHCIState *ehci, int async)
             break;
         }
 
-        if (again < 0) {
+        if (again < 0 || itd_count > 16) {
+            /* TODO: notify guest (raise HSE irq?) */
             fprintf(stderr, "processing error - resetting ehci HC\n");
             ehci_reset(ehci);
             again = 0;
@@ -2298,10 +2312,11 @@ static void ehci_frame_timer(void *opaque)
         /* If we've raised int, we speed up the timer, so that we quickly
          * notice any new packets queued up in response */
         if (ehci->int_req_by_async && (ehci->usbsts & USBSTS_INT)) {
-            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 4);
+            expire_time = t_now +
+                NANOSECONDS_PER_SECOND / (FRAME_TIMER_FREQ * 4);
             ehci->int_req_by_async = false;
         } else {
-            expire_time = t_now + (get_ticks_per_sec()
+            expire_time = t_now + (NANOSECONDS_PER_SECOND
                                * (ehci->async_stepdown+1) / FRAME_TIMER_FREQ);
         }
         timer_mod(ehci->frame_timer, expire_time);
@@ -2310,6 +2325,7 @@ static void ehci_frame_timer(void *opaque)
 
 static const MemoryRegionOps ehci_mmio_caps_ops = {
     .read = ehci_caps_read,
+    .write = ehci_caps_write,
     .valid.min_access_size = 1,
     .valid.max_access_size = 4,
     .impl.min_access_size = 1,
diff --git a/qemu/hw/usb/hcd-musb.c b/qemu/hw/usb/hcd-musb.c
index 61cc87894..27d9d0bd8 100644
--- a/qemu/hw/usb/hcd-musb.c
+++ b/qemu/hw/usb/hcd-musb.c
@@ -20,6 +20,7 @@
  *
  * Only host-mode and non-DMA accesses are currently supported.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
@@ -563,7 +564,7 @@ static void musb_schedule_cb(USBPort *port, USBPacket *packey)
         ep->intv_timer[dir] = timer_new_ns(QEMU_CLOCK_VIRTUAL, musb_cb_tick, ep);
 
     timer_mod(ep->intv_timer[dir], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(timeout, get_ticks_per_sec(), 8000));
+                   muldiv64(timeout, NANOSECONDS_PER_SECOND, 8000));
 }
 
 static int musb_timeout(int ttype, int speed, int val)
diff --git a/qemu/hw/usb/hcd-ohci.c b/qemu/hw/usb/hcd-ohci.c
index 7d6581806..ffab561cf 100644
--- a/qemu/hw/usb/hcd-ohci.c
+++ b/qemu/hw/usb/hcd-ohci.c
@@ -25,7 +25,9 @@
  *  o BIOS work to boot from USB storage
 */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
 #include "hw/pci/pci.h"
@@ -439,15 +441,37 @@ static void ohci_stop_endpoints(OHCIState *ohci)
     }
 }
 
-/* Reset the controller */
-static void ohci_reset(void *opaque)
+static void ohci_roothub_reset(OHCIState *ohci)
 {
-    OHCIState *ohci = opaque;
     OHCIPort *port;
     int i;
 
     ohci_bus_stop(ohci);
-    ohci->ctl = 0;
+    ohci->rhdesc_a = OHCI_RHA_NPS | ohci->num_ports;
+    ohci->rhdesc_b = 0x0; /* Impl. specific */
+    ohci->rhstatus = 0;
+
+    for (i = 0; i < ohci->num_ports; i++) {
+        port = &ohci->rhport[i];
+        port->ctrl = 0;
+        if (port->port.dev && port->port.dev->attached) {
+            usb_port_reset(&port->port);
+        }
+    }
+    if (ohci->async_td) {
+        usb_cancel_packet(&ohci->usb_packet);
+        ohci->async_td = 0;
+    }
+    ohci_stop_endpoints(ohci);
+}
+
+/* Reset the controller */
+static void ohci_soft_reset(OHCIState *ohci)
+{
+    trace_usb_ohci_reset(ohci->name);
+
+    ohci_bus_stop(ohci);
+    ohci->ctl = (ohci->ctl & OHCI_CTL_IR) | OHCI_USB_SUSPEND;
     ohci->old_ctl = 0;
     ohci->status = 0;
     ohci->intr_status = 0;
@@ -470,25 +494,13 @@ static void ohci_reset(void *opaque)
     ohci->frame_number = 0;
     ohci->pstart = 0;
     ohci->lst = OHCI_LS_THRESH;
+}
 
-    ohci->rhdesc_a = OHCI_RHA_NPS | ohci->num_ports;
-    ohci->rhdesc_b = 0x0; /* Impl. specific */
-    ohci->rhstatus = 0;
-
-    for (i = 0; i < ohci->num_ports; i++)
-      {
-        port = &ohci->rhport[i];
-        port->ctrl = 0;
-        if (port->port.dev && port->port.dev->attached) {
-            usb_port_reset(&port->port);
-        }
-      }
-    if (ohci->async_td) {
-        usb_cancel_packet(&ohci->usb_packet);
-        ohci->async_td = 0;
-    }
-    ohci_stop_endpoints(ohci);
-    trace_usb_ohci_reset(ohci->name);
+static void ohci_hard_reset(OHCIState *ohci)
+{
+    ohci_soft_reset(ohci);
+    ohci->ctl = 0;
+    ohci_roothub_reset(ohci);
 }
 
 /* Get an array of dwords from main memory */
@@ -1231,11 +1243,16 @@ static int ohci_service_ed_list(OHCIState *ohci, uint32_t head, int completion)
     return active;
 }
 
-/* Generate a SOF event, and set a timer for EOF */
-static void ohci_sof(OHCIState *ohci)
+/* set a timer for EOF */
+static void ohci_eof_timer(OHCIState *ohci)
 {
     ohci->sof_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     timer_mod(ohci->eof_timer, ohci->sof_time + usb_frame_time);
+}
+/* Set a timer for EOF and generate a SOF event */
+static void ohci_sof(OHCIState *ohci)
+{
+    ohci_eof_timer(ohci);
     ohci_set_interrupt(ohci, OHCI_INTR_SF);
 }
 
@@ -1331,19 +1348,14 @@ static void ohci_frame_boundary(void *opaque)
  */
 static int ohci_bus_start(OHCIState *ohci)
 {
-    ohci->eof_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                    ohci_frame_boundary,
-                    ohci);
-
-    if (ohci->eof_timer == NULL) {
-        trace_usb_ohci_bus_eof_timer_failed(ohci->name);
-        ohci_die(ohci);
-        return 0;
-    }
-
     trace_usb_ohci_start(ohci->name);
 
-    ohci_sof(ohci);
+    /* Delay the first SOF event by one frame time as
+     * linux driver is not ready to receive it and
+     * can meet some race conditions
+     */
+
+    ohci_eof_timer(ohci);
 
     return 1;
 }
@@ -1352,11 +1364,7 @@ static int ohci_bus_start(OHCIState *ohci)
 static void ohci_bus_stop(OHCIState *ohci)
 {
     trace_usb_ohci_stop(ohci->name);
-    if (ohci->eof_timer) {
-        timer_del(ohci->eof_timer);
-        timer_free(ohci->eof_timer);
-    }
-    ohci->eof_timer = NULL;
+    timer_del(ohci->eof_timer);
 }
 
 /* Sets a flag in a port status register but only set it if the port is
@@ -1436,12 +1444,15 @@ static void ohci_set_ctl(OHCIState *ohci, uint32_t val)
         break;
     case OHCI_USB_SUSPEND:
         ohci_bus_stop(ohci);
+        /* clear pending SF otherwise linux driver loops in ohci_irq() */
+        ohci->intr_status &= ~OHCI_INTR_SF;
+        ohci_intr_update(ohci);
         break;
     case OHCI_USB_RESUME:
         trace_usb_ohci_resume(ohci->name);
         break;
     case OHCI_USB_RESET:
-        ohci_reset(ohci);
+        ohci_roothub_reset(ohci);
         break;
     }
 }
@@ -1704,7 +1715,7 @@ static void ohci_mem_write(void *opaque,
         ohci->status |= val;
 
         if (ohci->status & OHCI_STATUS_HCR)
-            ohci_reset(ohci);
+            ohci_soft_reset(ohci);
         break;
 
     case 3: /* HcInterruptStatus */
@@ -1783,7 +1794,7 @@ static void ohci_mem_write(void *opaque,
     case 25: /* HcHReset */
         ohci->hreset = val & ~OHCI_HRESET_FSBIR;
         if (val & OHCI_HRESET_FSBIR)
-            ohci_reset(ohci);
+            ohci_hard_reset(ohci);
         break;
 
     case 26: /* HcHInterruptEnable */
@@ -1839,12 +1850,12 @@ static void usb_ohci_init(OHCIState *ohci, DeviceState *dev,
 
     if (usb_frame_time == 0) {
 #ifdef OHCI_TIME_WARP
-        usb_frame_time = get_ticks_per_sec();
-        usb_bit_time = muldiv64(1, get_ticks_per_sec(), USB_HZ/1000);
+        usb_frame_time = NANOSECONDS_PER_SECOND;
+        usb_bit_time = NANOSECONDS_PER_SECOND / (USB_HZ / 1000);
 #else
-        usb_frame_time = muldiv64(1, get_ticks_per_sec(), 1000);
-        if (get_ticks_per_sec() >= USB_HZ) {
-            usb_bit_time = muldiv64(1, get_ticks_per_sec(), USB_HZ);
+        usb_frame_time = NANOSECONDS_PER_SECOND / 1000;
+        if (NANOSECONDS_PER_SECOND >= USB_HZ) {
+            usb_bit_time = NANOSECONDS_PER_SECOND / USB_HZ;
         } else {
             usb_bit_time = 1;
         }
@@ -1883,6 +1894,9 @@ static void usb_ohci_init(OHCIState *ohci, DeviceState *dev,
     usb_packet_init(&ohci->usb_packet);
 
     ohci->async_td = 0;
+
+    ohci->eof_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                   ohci_frame_boundary, ohci);
 }
 
 #define TYPE_PCI_OHCI "pci-ohci"
@@ -1952,6 +1966,9 @@ static void usb_ohci_exit(PCIDevice *dev)
     if (!ohci->masterbus) {
         usb_bus_release(&s->bus);
     }
+
+    timer_del(s->eof_timer);
+    timer_free(s->eof_timer);
 }
 
 static void usb_ohci_reset_pci(DeviceState *d)
@@ -1960,7 +1977,7 @@ static void usb_ohci_reset_pci(DeviceState *d)
     OHCIPCIState *ohci = PCI_OHCI(dev);
     OHCIState *s = &ohci->state;
 
-    ohci_reset(s);
+    ohci_hard_reset(s);
 }
 
 #define TYPE_SYSBUS_OHCI "sysbus-ohci"
@@ -1993,7 +2010,7 @@ static void usb_ohci_reset_sysbus(DeviceState *dev)
     OHCISysBusState *s = SYSBUS_OHCI(dev);
     OHCIState *ohci = &s->ohci;
 
-    ohci_reset(ohci);
+    ohci_hard_reset(ohci);
 }
 
 static Property ohci_pci_properties[] = {
@@ -2017,23 +2034,13 @@ static bool ohci_eof_timer_needed(void *opaque)
 {
     OHCIState *ohci = opaque;
 
-    return ohci->eof_timer != NULL;
-}
-
-static int ohci_eof_timer_pre_load(void *opaque)
-{
-    OHCIState *ohci = opaque;
-
-    ohci_bus_start(ohci);
-
-    return 0;
+    return timer_pending(ohci->eof_timer);
 }
 
 static const VMStateDescription vmstate_ohci_eof_timer = {
     .name = "ohci-core/eof-timer",
     .version_id = 1,
     .minimum_version_id = 1,
-    .pre_load = ohci_eof_timer_pre_load,
     .needed = ohci_eof_timer_needed,
     .fields = (VMStateField[]) {
         VMSTATE_TIMER_PTR(eof_timer, OHCIState),
diff --git a/qemu/hw/usb/hcd-uhci.c b/qemu/hw/usb/hcd-uhci.c
index 3f0ed6268..ca72a80f2 100644
--- a/qemu/hw/usb/hcd-uhci.c
+++ b/qemu/hw/usb/hcd-uhci.c
@@ -25,10 +25,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/usb/uhci-regs.h"
 #include "hw/pci/pci.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/iov.h"
 #include "sysemu/dma.h"
@@ -401,7 +403,7 @@ static int uhci_post_load(void *opaque, int version_id)
 
     if (version_id < 2) {
         s->expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-            (get_ticks_per_sec() / FRAME_TIMER_FREQ);
+            (NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ);
     }
     return 0;
 }
@@ -443,7 +445,7 @@ static void uhci_port_write(void *opaque, hwaddr addr,
             /* start frame processing */
             trace_usb_uhci_schedule_start();
             s->expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                (get_ticks_per_sec() / FRAME_TIMER_FREQ);
+                (NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ);
             timer_mod(s->frame_timer, s->expire_time);
             s->status &= ~UHCI_STS_HCHALTED;
         } else if (!(val & UHCI_CMD_RS)) {
@@ -772,8 +774,9 @@ static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
     bool spd;
     bool queuing = (q != NULL);
     uint8_t pid = td->token & 0xff;
-    UHCIAsync *async = uhci_async_find_td(s, td_addr);
+    UHCIAsync *async;
 
+    async = uhci_async_find_td(s, td_addr);
     if (async) {
         if (uhci_queue_verify(async->queue, qh_addr, td, td_addr, queuing)) {
             assert(q == NULL || q == async->queue);
@@ -812,6 +815,19 @@ static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
         return TD_RESULT_NEXT_QH;
     }
 
+    switch (pid) {
+    case USB_TOKEN_OUT:
+    case USB_TOKEN_SETUP:
+    case USB_TOKEN_IN:
+        break;
+    default:
+        /* invalid pid : frame interrupted */
+        s->status |= UHCI_STS_HCPERR;
+        s->cmd &= ~UHCI_CMD_RS;
+        uhci_update_irq(s);
+        return TD_RESULT_STOP_FRAME;
+    }
+
     if (async) {
         if (queuing) {
             /* we are busy filling the queue, we are not prepared
@@ -879,11 +895,7 @@ static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
         break;
 
     default:
-        /* invalid pid : frame interrupted */
-        uhci_async_free(async);
-        s->status |= UHCI_STS_HCPERR;
-        uhci_update_irq(s);
-        return TD_RESULT_STOP_FRAME;
+        abort(); /* Never to execute */
     }
 
     if (async->packet.status == USB_RET_ASYNC) {
@@ -1119,7 +1131,7 @@ static void uhci_frame_timer(void *opaque)
     UHCIState *s = opaque;
     uint64_t t_now, t_last_run;
     int i, frames;
-    const uint64_t frame_t = get_ticks_per_sec() / FRAME_TIMER_FREQ;
+    const uint64_t frame_t = NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ;
 
     s->completions_only = false;
     qemu_bh_cancel(s->bh);
diff --git a/qemu/hw/usb/hcd-xhci.c b/qemu/hw/usb/hcd-xhci.c
index c673bed4c..bcde8a2f4 100644
--- a/qemu/hw/usb/hcd-xhci.c
+++ b/qemu/hw/usb/hcd-xhci.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
@@ -697,11 +698,13 @@ static inline void xhci_dma_write_u32s(XHCIState *xhci, dma_addr_t addr,
                                        uint32_t *buf, size_t len)
 {
     int i;
-    uint32_t tmp[len / sizeof(uint32_t)];
+    uint32_t tmp[5];
+    uint32_t n = len / sizeof(uint32_t);
 
     assert((len % sizeof(uint32_t)) == 0);
+    assert(n <= ARRAY_SIZE(tmp));
 
-    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+    for (i = 0; i < n; i++) {
         tmp[i] = cpu_to_le32(buf[i]);
     }
     pci_dma_write(PCI_DEVICE(xhci), addr, tmp, len);
@@ -1453,9 +1456,7 @@ static int xhci_ep_nuke_one_xfer(XHCITransfer *t, TRBCCode report)
         t->running_retry = 0;
         killed = 1;
     }
-    if (t->trbs) {
-        g_free(t->trbs);
-    }
+    g_free(t->trbs);
 
     t->trbs = NULL;
     t->trb_count = t->trb_alloced = 0;
@@ -2190,7 +2191,7 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
             xfer->trbs = NULL;
         }
         if (!xfer->trbs) {
-            xfer->trbs = g_malloc(sizeof(XHCITRB) * length);
+            xfer->trbs = g_new(XHCITRB, length);
             xfer->trb_alloced = length;
         }
         xfer->trb_count = length;
diff --git a/qemu/hw/usb/host-legacy.c b/qemu/hw/usb/host-legacy.c
index 422ed9a65..3b57e21b5 100644
--- a/qemu/hw/usb/host-legacy.c
+++ b/qemu/hw/usb/host-legacy.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/host.h"
diff --git a/qemu/hw/usb/host-libusb.c b/qemu/hw/usb/host-libusb.c
index 11429f5e7..6458a9448 100644
--- a/qemu/hw/usb/host-libusb.c
+++ b/qemu/hw/usb/host-libusb.c
@@ -33,9 +33,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <poll.h>
 #include <libusb.h>
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 #include "qemu/error-report.h"
@@ -451,6 +453,7 @@ static void usb_host_req_complete_iso(struct libusb_transfer *transfer)
     }
     if (xfer->ring->ep->pid == USB_TOKEN_IN) {
         QTAILQ_INSERT_TAIL(&xfer->ring->copy, xfer, next);
+        usb_wakeup(xfer->ring->ep, 0);
     } else {
         QTAILQ_INSERT_TAIL(&xfer->ring->unused, xfer, next);
     }
@@ -1239,7 +1242,7 @@ static void usb_host_handle_control(USBDevice *udev, USBPacket *p,
 
     /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
      * to work redirected to a not superspeed capable hcd */
-    if (udev->speed == USB_SPEED_SUPER &&
+    if ((udev->speedmask & USB_SPEED_MASK_SUPER) &&
         !(udev->port->speedmask & USB_SPEED_MASK_SUPER) &&
         request == 0x8006 && value == 0x100 && index == 0) {
         r->usb3ep0quirk = true;
@@ -1429,7 +1432,7 @@ static void usb_host_free_streams(USBDevice *udev, USBEndpoint **eps,
  * still present in the first place.  Attemping to contine where we
  * left off is impossible.
  *
- * What we are going to to to here is emulate a surprise removal of
+ * What we are going to do here is emulate a surprise removal of
  * the usb device passed through, then kick host scan so the device
  * will get re-attached (and re-initialized by the guest) in case it
  * is still present.
diff --git a/qemu/hw/usb/host-stub.c b/qemu/hw/usb/host-stub.c
index 2eaaa8341..6ba65a1f6 100644
--- a/qemu/hw/usb/host-stub.c
+++ b/qemu/hw/usb/host-stub.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/libhw.c b/qemu/hw/usb/libhw.c
index 8df11c461..73cdf0c97 100644
--- a/qemu/hw/usb/libhw.c
+++ b/qemu/hw/usb/libhw.c
@@ -19,6 +19,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
diff --git a/qemu/hw/usb/quirks.c b/qemu/hw/usb/quirks.c
index a761a9603..38a9c5634 100644
--- a/qemu/hw/usb/quirks.c
+++ b/qemu/hw/usb/quirks.c
@@ -12,6 +12,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "quirks.h"
 #include "hw/usb.h"
 
diff --git a/qemu/hw/usb/redirect.c b/qemu/hw/usb/redirect.c
index 25df25fd0..8d8054037 100644
--- a/qemu/hw/usb/redirect.c
+++ b/qemu/hw/usb/redirect.c
@@ -25,6 +25,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
@@ -33,14 +35,14 @@
 #include "qemu/iov.h"
 #include "sysemu/char.h"
 
-#include <dirent.h>
-#include <sys/ioctl.h>
-#include <signal.h>
 #include <usbredirparser.h>
 #include <usbredirfilter.h>
 
 #include "hw/usb.h"
 
+/* ERROR is defined below. Remove any previous definition. */
+#undef ERROR
+
 #define MAX_ENDPOINTS 32
 #define NO_INTERFACE_INFO 255 /* Valid interface_count always <= 32 */
 #define EP2I(ep_address) (((ep_address & 0x80) >> 3) | (ep_address & 0x0f))
@@ -324,7 +326,7 @@ static void packet_id_queue_add(struct PacketIdQueue *q, uint64_t id)
 
     DPRINTF("adding packet id %"PRIu64" to %s queue\n", id, q->name);
 
-    e = g_malloc0(sizeof(struct PacketIdQueueEntry));
+    e = g_new0(struct PacketIdQueueEntry, 1);
     e->id = id;
     QTAILQ_INSERT_TAIL(&q->head, e, next);
     q->size++;
@@ -448,7 +450,7 @@ static USBPacket *usbredir_find_packet_by_id(USBRedirDevice *dev,
     return p;
 }
 
-static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
+static int bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
     uint8_t status, uint8_t ep, void *free_on_destroy)
 {
     struct buf_packet *bufp;
@@ -465,12 +467,12 @@ static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
         if (dev->endpoint[EP2I(ep)].bufpq_size >
                 dev->endpoint[EP2I(ep)].bufpq_target_size) {
             free(data);
-            return;
+            return -1;
         }
         dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
     }
 
-    bufp = g_malloc(sizeof(struct buf_packet));
+    bufp = g_new(struct buf_packet, 1);
     bufp->data   = data;
     bufp->len    = len;
     bufp->offset = 0;
@@ -478,6 +480,7 @@ static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
     bufp->free_on_destroy = free_on_destroy;
     QTAILQ_INSERT_TAIL(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size++;
+    return 0;
 }
 
 static void bufp_free(USBRedirDevice *dev, struct buf_packet *bufp,
@@ -2083,13 +2086,17 @@ static void usbredir_buffered_bulk_packet(void *priv, uint64_t id,
     status = usb_redir_success;
     free_on_destroy = NULL;
     for (i = 0; i < data_len; i += len) {
+        int r;
         if (len >= (data_len - i)) {
             len = data_len - i;
             status = buffered_bulk_packet->status;
             free_on_destroy = data;
         }
         /* bufp_alloc also adds the packet to the ep queue */
-        bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+        r = bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+        if (r) {
+            break;
+        }
     }
 
     if (dev->endpoint[EP2I(ep)].pending_async_packet) {
@@ -2236,7 +2243,7 @@ static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
 
     endp->bufpq_size = qemu_get_be32(f);
     for (i = 0; i < endp->bufpq_size; i++) {
-        bufp = g_malloc(sizeof(struct buf_packet));
+        bufp = g_new(struct buf_packet, 1);
         bufp->len = qemu_get_be32(f);
         bufp->status = qemu_get_be32(f);
         bufp->offset = 0;
diff --git a/qemu/hw/timer/tusb6010.c b/qemu/hw/usb/tusb6010.c
index 459c748e1..8f593a6fd 100644
--- a/qemu/hw/timer/tusb6010.c
+++ b/qemu/hw/usb/tusb6010.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
@@ -515,7 +516,7 @@ static void tusb_async_writew(void *opaque, hwaddr addr,
         if (value & TUSB_DEV_OTG_TIMER_ENABLE)
             timer_mod(s->otg_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
                             muldiv64(TUSB_DEV_OTG_TIMER_VAL(value),
-                                     get_ticks_per_sec(), TUSB_DEVCLOCK));
+                                     NANOSECONDS_PER_SECOND, TUSB_DEVCLOCK));
         else
             timer_del(s->otg_timer);
         break;
@@ -725,8 +726,8 @@ static void tusb6010_power(TUSBState *s, int on)
         /* Pull the interrupt down after TUSB6010 comes up.  */
         s->intr_ok = 0;
         tusb_intr_update(s);
-        timer_mod(s->pwr_timer,
-                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + get_ticks_per_sec() / 2);
+        timer_mod(s->pwr_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  NANOSECONDS_PER_SECOND / 2);
     }
 }
 
diff --git a/qemu/hw/vfio/Makefile.objs b/qemu/hw/vfio/Makefile.objs
index d540c9d14..ceddbb8f9 100644
--- a/qemu/hw/vfio/Makefile.objs
+++ b/qemu/hw/vfio/Makefile.objs
@@ -1,6 +1,7 @@
 ifeq ($(CONFIG_LINUX), y)
 obj-$(CONFIG_SOFTMMU) += common.o
-obj-$(CONFIG_PCI) += pci.o
+obj-$(CONFIG_PCI) += pci.o pci-quirks.o
 obj-$(CONFIG_SOFTMMU) += platform.o
 obj-$(CONFIG_SOFTMMU) += calxeda-xgmac.o
+obj-$(CONFIG_SOFTMMU) += amd-xgbe.o
 endif
diff --git a/qemu/hw/vfio/amd-xgbe.c b/qemu/hw/vfio/amd-xgbe.c
new file mode 100644
index 000000000..2c60310cf
--- /dev/null
+++ b/qemu/hw/vfio/amd-xgbe.c
@@ -0,0 +1,56 @@
+/*
+ * AMD XGBE VFIO device
+ *
+ * Copyright Linaro Limited, 2015
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-amd-xgbe.h"
+
+static void amd_xgbe_realize(DeviceState *dev, Error **errp)
+{
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
+    VFIOAmdXgbeDeviceClass *k = VFIO_AMD_XGBE_DEVICE_GET_CLASS(dev);
+
+    vdev->compat = g_strdup("amd,xgbe-seattle-v1a");
+
+    k->parent_realize(dev, errp);
+}
+
+static const VMStateDescription vfio_platform_amd_xgbe_vmstate = {
+    .name = TYPE_VFIO_AMD_XGBE,
+    .unmigratable = 1,
+};
+
+static void vfio_amd_xgbe_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VFIOAmdXgbeDeviceClass *vcxc =
+        VFIO_AMD_XGBE_DEVICE_CLASS(klass);
+    vcxc->parent_realize = dc->realize;
+    dc->realize = amd_xgbe_realize;
+    dc->desc = "VFIO AMD XGBE";
+    dc->vmsd = &vfio_platform_amd_xgbe_vmstate;
+}
+
+static const TypeInfo vfio_amd_xgbe_dev_info = {
+    .name = TYPE_VFIO_AMD_XGBE,
+    .parent = TYPE_VFIO_PLATFORM,
+    .instance_size = sizeof(VFIOAmdXgbeDevice),
+    .class_init = vfio_amd_xgbe_class_init,
+    .class_size = sizeof(VFIOAmdXgbeDeviceClass),
+};
+
+static void register_amd_xgbe_dev_type(void)
+{
+    type_register_static(&vfio_amd_xgbe_dev_info);
+}
+
+type_init(register_amd_xgbe_dev_type)
diff --git a/qemu/hw/vfio/calxeda-xgmac.c b/qemu/hw/vfio/calxeda-xgmac.c
index eb914f0d0..bb15d588e 100644
--- a/qemu/hw/vfio/calxeda-xgmac.c
+++ b/qemu/hw/vfio/calxeda-xgmac.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/vfio/vfio-calxeda-xgmac.h"
 
 static void calxeda_xgmac_realize(DeviceState *dev, Error **errp)
diff --git a/qemu/hw/vfio/common.c b/qemu/hw/vfio/common.c
index 85ee9b005..f27db36fb 100644
--- a/qemu/hw/vfio/common.c
+++ b/qemu/hw/vfio/common.c
@@ -18,6 +18,7 @@
  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  */
 
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <linux/vfio.h>
@@ -226,7 +227,7 @@ static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
     /*
      * Try the mapping, if it fails with EBUSY, unmap the region and try
      * again.  This shouldn't be necessary, but we sometimes see it in
-     * the the VGA ROM space.
+     * the VGA ROM space.
      */
     if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
         (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
@@ -312,13 +313,17 @@ out:
     rcu_read_unlock();
 }
 
+static hwaddr vfio_container_granularity(VFIOContainer *container)
+{
+    return (hwaddr)1 << ctz64(container->iova_pgsizes);
+}
+
 static void vfio_listener_region_add(MemoryListener *listener,
                                      MemoryRegionSection *section)
 {
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
     hwaddr iova, end;
-    Int128 llend;
+    Int128 llend, llsize;
     void *vaddr;
     int ret;
 
@@ -344,14 +349,22 @@ static void vfio_listener_region_add(MemoryListener *listener,
     if (int128_ge(int128_make64(iova), llend)) {
         return;
     }
+    end = int128_get64(int128_sub(llend, int128_one()));
+
+    if ((iova < container->min_iova) || (end > container->max_iova)) {
+        error_report("vfio: IOMMU container %p can't map guest IOVA region"
+                     " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx,
+                     container, iova, end);
+        ret = -EFAULT;
+        goto fail;
+    }
 
     memory_region_ref(section->mr);
 
     if (memory_region_is_iommu(section->mr)) {
         VFIOGuestIOMMU *giommu;
 
-        trace_vfio_listener_region_add_iommu(iova,
-                    int128_get64(int128_sub(llend, int128_one())));
+        trace_vfio_listener_region_add_iommu(iova, end);
         /*
          * FIXME: We should do some checking to see if the
          * capabilities of the host VFIO IOMMU are adequate to model
@@ -362,65 +375,60 @@ static void vfio_listener_region_add(MemoryListener *listener,
          * would be the right place to wire that up (tell the KVM
          * device emulation the VFIO iommu handles to use).
          */
-        /*
-         * This assumes that the guest IOMMU is empty of
-         * mappings at this point.
-         *
-         * One way of doing this is:
-         * 1. Avoid sharing IOMMUs between emulated devices or different
-         * IOMMU groups.
-         * 2. Implement VFIO_IOMMU_ENABLE in the host kernel to fail if
-         * there are some mappings in IOMMU.
-         *
-         * VFIO on SPAPR does that. Other IOMMU models may do that different,
-         * they must make sure there are no existing mappings or
-         * loop through existing mappings to map them into VFIO.
-         */
         giommu = g_malloc0(sizeof(*giommu));
         giommu->iommu = section->mr;
         giommu->container = container;
         giommu->n.notify = vfio_iommu_map_notify;
         QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
+
         memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
+        memory_region_iommu_replay(giommu->iommu, &giommu->n,
+                                   vfio_container_granularity(container),
+                                   false);
 
         return;
     }
 
     /* Here we assume that memory_region_is_ram(section->mr)==true */
 
-    end = int128_get64(llend);
     vaddr = memory_region_get_ram_ptr(section->mr) +
             section->offset_within_region +
             (iova - section->offset_within_address_space);
 
-    trace_vfio_listener_region_add_ram(iova, end - 1, vaddr);
+    trace_vfio_listener_region_add_ram(iova, end, vaddr);
+
+    llsize = int128_sub(llend, int128_make64(iova));
 
-    ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
+    ret = vfio_dma_map(container, iova, int128_get64(llsize),
+                       vaddr, section->readonly);
     if (ret) {
         error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
                      "0x%"HWADDR_PRIx", %p) = %d (%m)",
-                     container, iova, end - iova, vaddr, ret);
+                     container, iova, int128_get64(llsize), vaddr, ret);
+        goto fail;
+    }
 
-        /*
-         * On the initfn path, store the first error in the container so we
-         * can gracefully fail.  Runtime, there's not much we can do other
-         * than throw a hardware error.
-         */
-        if (!container->iommu_data.type1.initialized) {
-            if (!container->iommu_data.type1.error) {
-                container->iommu_data.type1.error = ret;
-            }
-        } else {
-            hw_error("vfio: DMA mapping failed, unable to continue");
+    return;
+
+fail:
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    if (!container->initialized) {
+        if (!container->error) {
+            container->error = ret;
         }
+    } else {
+        hw_error("vfio: DMA mapping failed, unable to continue");
     }
 }
 
 static void vfio_listener_region_del(MemoryListener *listener,
                                      MemoryRegionSection *section)
 {
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
     hwaddr iova, end;
     int ret;
 
@@ -485,49 +493,165 @@ static const MemoryListener vfio_memory_listener = {
 
 static void vfio_listener_release(VFIOContainer *container)
 {
-    memory_listener_unregister(&container->iommu_data.type1.listener);
+    memory_listener_unregister(&container->listener);
 }
 
-int vfio_mmap_region(Object *obj, VFIORegion *region,
-                     MemoryRegion *mem, MemoryRegion *submem,
-                     void **map, size_t size, off_t offset,
-                     const char *name)
+int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
+                      int index, const char *name)
 {
-    int ret = 0;
-    VFIODevice *vbasedev = region->vbasedev;
+    struct vfio_region_info *info;
+    int ret;
+
+    ret = vfio_get_region_info(vbasedev, index, &info);
+    if (ret) {
+        return ret;
+    }
 
-    if (vbasedev->allow_mmap && size && region->flags &
-        VFIO_REGION_INFO_FLAG_MMAP) {
-        int prot = 0;
+    region->vbasedev = vbasedev;
+    region->flags = info->flags;
+    region->size = info->size;
+    region->fd_offset = info->offset;
+    region->nr = index;
 
-        if (region->flags & VFIO_REGION_INFO_FLAG_READ) {
-            prot |= PROT_READ;
+    if (region->size) {
+        region->mem = g_new0(MemoryRegion, 1);
+        memory_region_init_io(region->mem, obj, &vfio_region_ops,
+                              region, name, region->size);
+
+        if (!vbasedev->no_mmap &&
+            region->flags & VFIO_REGION_INFO_FLAG_MMAP &&
+            !(region->size & ~qemu_real_host_page_mask)) {
+
+            region->nr_mmaps = 1;
+            region->mmaps = g_new0(VFIOMmap, region->nr_mmaps);
+
+            region->mmaps[0].offset = 0;
+            region->mmaps[0].size = region->size;
         }
+    }
+
+    g_free(info);
+
+    trace_vfio_region_setup(vbasedev->name, index, name,
+                            region->flags, region->fd_offset, region->size);
+    return 0;
+}
+
+int vfio_region_mmap(VFIORegion *region)
+{
+    int i, prot = 0;
+    char *name;
+
+    if (!region->mem) {
+        return 0;
+    }
 
-        if (region->flags & VFIO_REGION_INFO_FLAG_WRITE) {
-            prot |= PROT_WRITE;
+    prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0;
+    prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0;
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot,
+                                     MAP_SHARED, region->vbasedev->fd,
+                                     region->fd_offset +
+                                     region->mmaps[i].offset);
+        if (region->mmaps[i].mmap == MAP_FAILED) {
+            int ret = -errno;
+
+            trace_vfio_region_mmap_fault(memory_region_name(region->mem), i,
+                                         region->fd_offset +
+                                         region->mmaps[i].offset,
+                                         region->fd_offset +
+                                         region->mmaps[i].offset +
+                                         region->mmaps[i].size - 1, ret);
+
+            region->mmaps[i].mmap = NULL;
+
+            for (i--; i >= 0; i--) {
+                memory_region_del_subregion(region->mem, &region->mmaps[i].mem);
+                munmap(region->mmaps[i].mmap, region->mmaps[i].size);
+                object_unparent(OBJECT(&region->mmaps[i].mem));
+                region->mmaps[i].mmap = NULL;
+            }
+
+            return ret;
         }
 
-        *map = mmap(NULL, size, prot, MAP_SHARED,
-                    vbasedev->fd,
-                    region->fd_offset + offset);
-        if (*map == MAP_FAILED) {
-            *map = NULL;
-            ret = -errno;
-            goto empty_region;
+        name = g_strdup_printf("%s mmaps[%d]",
+                               memory_region_name(region->mem), i);
+        memory_region_init_ram_ptr(&region->mmaps[i].mem,
+                                   memory_region_owner(region->mem),
+                                   name, region->mmaps[i].size,
+                                   region->mmaps[i].mmap);
+        g_free(name);
+        memory_region_set_skip_dump(&region->mmaps[i].mem);
+        memory_region_add_subregion(region->mem, region->mmaps[i].offset,
+                                    &region->mmaps[i].mem);
+
+        trace_vfio_region_mmap(memory_region_name(&region->mmaps[i].mem),
+                               region->mmaps[i].offset,
+                               region->mmaps[i].offset +
+                               region->mmaps[i].size - 1);
+    }
+
+    return 0;
+}
+
+void vfio_region_exit(VFIORegion *region)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
+    }
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            memory_region_del_subregion(region->mem, &region->mmaps[i].mem);
         }
+    }
 
-        memory_region_init_ram_ptr(submem, obj, name, size, *map);
-        memory_region_set_skip_dump(submem);
-    } else {
-empty_region:
-        /* Create a zero sized sub-region to make cleanup easy. */
-        memory_region_init(submem, obj, name, 0);
+    trace_vfio_region_exit(region->vbasedev->name, region->nr);
+}
+
+void vfio_region_finalize(VFIORegion *region)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
     }
 
-    memory_region_add_subregion(mem, offset, submem);
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            munmap(region->mmaps[i].mmap, region->mmaps[i].size);
+            object_unparent(OBJECT(&region->mmaps[i].mem));
+        }
+    }
 
-    return ret;
+    object_unparent(OBJECT(region->mem));
+
+    g_free(region->mem);
+    g_free(region->mmaps);
+
+    trace_vfio_region_finalize(region->vbasedev->name, region->nr);
+}
+
+void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
+    }
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            memory_region_set_enabled(&region->mmaps[i].mem, enabled);
+        }
+    }
+
+    trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem),
+                                        enabled);
 }
 
 void vfio_reset_handler(void *opaque)
@@ -668,6 +792,7 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
     if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ||
         ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) {
         bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU);
+        struct vfio_iommu_type1_info info;
 
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
@@ -684,21 +809,27 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
             goto free_container_exit;
         }
 
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 container->space->as);
-
-        if (container->iommu_data.type1.error) {
-            ret = container->iommu_data.type1.error;
-            error_report("vfio: memory listener initialization failed for container");
-            goto listener_release_exit;
+        /*
+         * FIXME: This assumes that a Type1 IOMMU can map any 64-bit
+         * IOVA whatsoever.  That's not actually true, but the current
+         * kernel interface doesn't tell us what it can map, and the
+         * existing Type1 IOMMUs generally support any IOVA we're
+         * going to actually try in practice.
+         */
+        container->min_iova = 0;
+        container->max_iova = (hwaddr)-1;
+
+        /* Assume just 4K IOVA page size */
+        container->iova_pgsizes = 0x1000;
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info);
+        /* Ignore errors */
+        if ((ret == 0) && (info.flags & VFIO_IOMMU_INFO_PGSIZES)) {
+            container->iova_pgsizes = info.iova_pgsizes;
         }
-
-        container->iommu_data.type1.initialized = true;
-
     } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) {
+        struct vfio_iommu_spapr_tce_info info;
+
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
             error_report("vfio: failed to set group container: %m");
@@ -724,18 +855,41 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
             goto free_container_exit;
         }
 
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 container->space->as);
+        /*
+         * This only considers the host IOMMU's 32-bit window.  At
+         * some point we need to add support for the optional 64-bit
+         * window and dynamic windows
+         */
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
+        if (ret) {
+            error_report("vfio: VFIO_IOMMU_SPAPR_TCE_GET_INFO failed: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+        container->min_iova = info.dma32_window_start;
+        container->max_iova = container->min_iova + info.dma32_window_size - 1;
 
+        /* Assume just 4K IOVA pages for now */
+        container->iova_pgsizes = 0x1000;
     } else {
         error_report("vfio: No available IOMMU models");
         ret = -EINVAL;
         goto free_container_exit;
     }
 
+    container->listener = vfio_memory_listener;
+
+    memory_listener_register(&container->listener, container->space->as);
+
+    if (container->error) {
+        ret = container->error;
+        error_report("vfio: memory listener initialization failed for container");
+        goto listener_release_exit;
+    }
+
+    container->initialized = true;
+
     QLIST_INIT(&container->group_list);
     QLIST_INSERT_HEAD(&space->containers, container, next);
 
@@ -774,9 +928,7 @@ static void vfio_disconnect_container(VFIOGroup *group)
         VFIOAddressSpace *space = container->space;
         VFIOGuestIOMMU *giommu, *tmp;
 
-        if (container->iommu_data.release) {
-            container->iommu_data.release(container);
-        }
+        vfio_listener_release(container);
         QLIST_REMOVE(container, next);
 
         QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) {
@@ -926,47 +1078,115 @@ void vfio_put_base_device(VFIODevice *vbasedev)
     close(vbasedev->fd);
 }
 
-static int vfio_container_do_ioctl(AddressSpace *as, int32_t groupid,
-                                   int req, void *param)
+int vfio_get_region_info(VFIODevice *vbasedev, int index,
+                         struct vfio_region_info **info)
 {
-    VFIOGroup *group;
-    VFIOContainer *container;
-    int ret = -1;
+    size_t argsz = sizeof(struct vfio_region_info);
 
-    group = vfio_get_group(groupid, as);
-    if (!group) {
-        error_report("vfio: group %d not registered", groupid);
-        return ret;
+    *info = g_malloc0(argsz);
+
+    (*info)->index = index;
+    (*info)->argsz = argsz;
+
+    if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) {
+        g_free(*info);
+        return -errno;
     }
 
-    container = group->container;
-    if (group->container) {
-        ret = ioctl(container->fd, req, param);
-        if (ret < 0) {
-            error_report("vfio: failed to ioctl %d to container: ret=%d, %s",
-                         _IOC_NR(req) - VFIO_BASE, ret, strerror(errno));
-        }
+    return 0;
+}
+
+/*
+ * Interfaces for IBM EEH (Enhanced Error Handling)
+ */
+static bool vfio_eeh_container_ok(VFIOContainer *container)
+{
+    /*
+     * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO
+     * implementation is broken if there are multiple groups in a
+     * container.  The hardware works in units of Partitionable
+     * Endpoints (== IOMMU groups) and the EEH operations naively
+     * iterate across all groups in the container, without any logic
+     * to make sure the groups have their state synchronized.  For
+     * certain operations (ENABLE) that might be ok, until an error
+     * occurs, but for others (GET_STATE) it's clearly broken.
+     */
+
+    /*
+     * XXX Once fixed kernels exist, test for them here
+     */
+
+    if (QLIST_EMPTY(&container->group_list)) {
+        return false;
     }
 
-    vfio_put_group(group);
+    if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) {
+        return false;
+    }
 
-    return ret;
+    return true;
 }
 
-int vfio_container_ioctl(AddressSpace *as, int32_t groupid,
-                         int req, void *param)
+static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op)
 {
-    /* We allow only certain ioctls to the container */
-    switch (req) {
-    case VFIO_CHECK_EXTENSION:
-    case VFIO_IOMMU_SPAPR_TCE_GET_INFO:
-    case VFIO_EEH_PE_OP:
-        break;
-    default:
-        /* Return an error on unknown requests */
-        error_report("vfio: unsupported ioctl %X", req);
-        return -1;
+    struct vfio_eeh_pe_op pe_op = {
+        .argsz = sizeof(pe_op),
+        .op = op,
+    };
+    int ret;
+
+    if (!vfio_eeh_container_ok(container)) {
+        error_report("vfio/eeh: EEH_PE_OP 0x%x: "
+                     "kernel requires a container with exactly one group", op);
+        return -EPERM;
+    }
+
+    ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op);
+    if (ret < 0) {
+        error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op);
+        return -errno;
+    }
+
+    return 0;
+}
+
+static VFIOContainer *vfio_eeh_as_container(AddressSpace *as)
+{
+    VFIOAddressSpace *space = vfio_get_address_space(as);
+    VFIOContainer *container = NULL;
+
+    if (QLIST_EMPTY(&space->containers)) {
+        /* No containers to act on */
+        goto out;
     }
 
-    return vfio_container_do_ioctl(as, groupid, req, param);
+    container = QLIST_FIRST(&space->containers);
+
+    if (QLIST_NEXT(container, next)) {
+        /* We don't yet have logic to synchronize EEH state across
+         * multiple containers */
+        container = NULL;
+        goto out;
+    }
+
+out:
+    vfio_put_address_space(space);
+    return container;
+}
+
+bool vfio_eeh_as_ok(AddressSpace *as)
+{
+    VFIOContainer *container = vfio_eeh_as_container(as);
+
+    return (container != NULL) && vfio_eeh_container_ok(container);
+}
+
+int vfio_eeh_as_op(AddressSpace *as, uint32_t op)
+{
+    VFIOContainer *container = vfio_eeh_as_container(as);
+
+    if (!container) {
+        return -ENODEV;
+    }
+    return vfio_eeh_container_op(container, op);
 }
diff --git a/qemu/hw/vfio/pci-quirks.c b/qemu/hw/vfio/pci-quirks.c
new file mode 100644
index 000000000..49ecf1172
--- /dev/null
+++ b/qemu/hw/vfio/pci-quirks.c
@@ -0,0 +1,1205 @@
+/*
+ * device quirks for PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "pci.h"
+#include "trace.h"
+#include "qemu/range.h"
+
+/* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */
+static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device)
+{
+    return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) &&
+           (device == PCI_ANY_ID || device == vdev->device_id);
+}
+
+static bool vfio_is_vga(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
+
+    return class == PCI_CLASS_DISPLAY_VGA;
+}
+
+/*
+ * List of device ids/vendor ids for which to disable
+ * option rom loading. This avoids the guest hangs during rom
+ * execution as noticed with the BCM 57810 card for lack of a
+ * more better way to handle such issues.
+ * The  user can still override by specifying a romfile or
+ * rombar=1.
+ * Please see https://bugs.launchpad.net/qemu/+bug/1284874
+ * for an analysis of the 57810 card hang. When adding
+ * a new vendor id/device id combination below, please also add
+ * your card/environment details and information that could
+ * help in debugging to the bug tracking this issue
+ */
+static const struct {
+    uint32_t vendor;
+    uint32_t device;
+} romblacklist[] = {
+    { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */
+};
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0 ; i < ARRAY_SIZE(romblacklist); i++) {
+        if (vfio_pci_is(vdev, romblacklist[i].vendor, romblacklist[i].device)) {
+            trace_vfio_quirk_rom_blacklisted(vdev->vbasedev.name,
+                                             romblacklist[i].vendor,
+                                             romblacklist[i].device);
+            return true;
+        }
+    }
+    return false;
+}
+
+/*
+ * Device specific region quirks (mostly backdoors to PCI config space)
+ */
+
+/*
+ * The generic window quirks operate on an address and data register,
+ * vfio_generic_window_address_quirk handles the address register and
+ * vfio_generic_window_data_quirk handles the data register.  These ops
+ * pass reads and writes through to hardware until a value matching the
+ * stored address match/mask is written.  When this occurs, the data
+ * register access emulated PCI config space for the device rather than
+ * passing through accesses.  This enables devices where PCI config space
+ * is accessible behind a window register to maintain the virtualization
+ * provided through vfio.
+ */
+typedef struct VFIOConfigWindowMatch {
+    uint32_t match;
+    uint32_t mask;
+} VFIOConfigWindowMatch;
+
+typedef struct VFIOConfigWindowQuirk {
+    struct VFIOPCIDevice *vdev;
+
+    uint32_t address_val;
+
+    uint32_t address_offset;
+    uint32_t data_offset;
+
+    bool window_enabled;
+    uint8_t bar;
+
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+
+    uint32_t nr_matches;
+    VFIOConfigWindowMatch matches[];
+} VFIOConfigWindowQuirk;
+
+static uint64_t vfio_generic_window_quirk_address_read(void *opaque,
+                                                       hwaddr addr,
+                                                       unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    return vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->address_offset, size);
+}
+
+static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    int i;
+
+    window->window_enabled = false;
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->address_offset, data, size);
+
+    for (i = 0; i < window->nr_matches; i++) {
+        if ((data & ~window->matches[i].mask) == window->matches[i].match) {
+            window->window_enabled = true;
+            window->address_val = data & window->matches[i].mask;
+            trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name,
+                                    memory_region_name(window->addr_mem), data);
+            break;
+        }
+    }
+}
+
+static const MemoryRegionOps vfio_generic_window_address_quirk = {
+    .read = vfio_generic_window_quirk_address_read,
+    .write = vfio_generic_window_quirk_address_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_generic_window_quirk_data_read(void *opaque,
+                                                    hwaddr addr, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    uint64_t data;
+
+    /* Always read data reg, discard if window enabled */
+    data = vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->data_offset, size);
+
+    if (window->window_enabled) {
+        data = vfio_pci_read_config(&vdev->pdev, window->address_val, size);
+        trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+    }
+
+    return data;
+}
+
+static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr,
+                                                 uint64_t data, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    if (window->window_enabled) {
+        vfio_pci_write_config(&vdev->pdev, window->address_val, data, size);
+        trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+        return;
+    }
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->data_offset, data, size);
+}
+
+static const MemoryRegionOps vfio_generic_window_data_quirk = {
+    .read = vfio_generic_window_quirk_data_read,
+    .write = vfio_generic_window_quirk_data_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * The generic mirror quirk handles devices which expose PCI config space
+ * through a region within a BAR.  When enabled, reads and writes are
+ * redirected through to emulated PCI config space.  XXX if PCI config space
+ * used memory regions, this could just be an alias.
+ */
+typedef struct VFIOConfigMirrorQuirk {
+    struct VFIOPCIDevice *vdev;
+    uint32_t offset;
+    uint8_t bar;
+    MemoryRegion *mem;
+} VFIOConfigMirrorQuirk;
+
+static uint64_t vfio_generic_quirk_mirror_read(void *opaque,
+                                               hwaddr addr, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    uint64_t data;
+
+    /* Read and discard in case the hardware cares */
+    (void)vfio_region_read(&vdev->bars[mirror->bar].region,
+                           addr + mirror->offset, size);
+
+    data = vfio_pci_read_config(&vdev->pdev, addr, size);
+    trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name,
+                                         memory_region_name(mirror->mem),
+                                         addr, data);
+    return data;
+}
+
+static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr,
+                                            uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+
+    vfio_pci_write_config(&vdev->pdev, addr, data, size);
+    trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name,
+                                          memory_region_name(mirror->mem),
+                                          addr, data);
+}
+
+static const MemoryRegionOps vfio_generic_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_generic_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/* Is range1 fully contained within range2?  */
+static bool vfio_range_contained(uint64_t first1, uint64_t len1,
+                                 uint64_t first2, uint64_t len2) {
+    return (first1 >= first2 && first1 + len1 <= first2 + len2);
+}
+
+#define PCI_VENDOR_ID_ATI               0x1002
+
+/*
+ * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
+ * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
+ * BAR4 (older cards like the X550 used BAR1, but we don't care to support
+ * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
+ * I/O port BAR address.  Originally this was coded to return the virtual BAR
+ * address only if the physical register read returns the actual BAR address,
+ * but users have reported greater success if we return the virtual address
+ * unconditionally.
+ */
+static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
+                                        hwaddr addr, unsigned size)
+{
+    VFIOPCIDevice *vdev = opaque;
+    uint64_t data = vfio_pci_read_config(&vdev->pdev,
+                                         PCI_BASE_ADDRESS_4 + 1, size);
+
+    trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data);
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_ati_3c3_quirk = {
+    .read = vfio_ati_3c3_quirk_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+
+    /*
+     * As long as the BAR is >= 256 bytes it will be aligned such that the
+     * lower byte is always zero.  Filter out anything else, if it exists.
+     */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+
+    memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev,
+                          "vfio-ati-3c3-quirk", 1);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                3 /* offset 3 bytes from 0x3c0 */, quirk->mem);
+
+    QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI
+ * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
+ * the MMIO space directly, but a window to this space is provided through
+ * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
+ * data register.  When the address is programmed to a range of 0x4000-0x4fff
+ * PCI configuration space is available.  Experimentation seems to indicate
+ * that read-only may be provided by hardware.
+ */
+static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigWindowQuirk *window;
+
+    /* This windows doesn't seem to be used except by legacy VGA code */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 4) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    window = quirk->data = g_malloc0(sizeof(*window) +
+                                     sizeof(VFIOConfigWindowMatch));
+    window->vdev = vdev;
+    window->address_offset = 0;
+    window->data_offset = 4;
+    window->nr_matches = 1;
+    window->matches[0].match = 0x4000;
+    window->matches[0].mask = vdev->config_size - 1;
+    window->bar = nr;
+    window->addr_mem = &quirk->mem[0];
+    window->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-ati-bar4-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-ati-bar4-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Trap the BAR2 MMIO mirror to config space as well.
+ */
+static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    /* Only enable on newer devices where BAR2 is 64bit */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 2 || !vdev->bars[2].mem64) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x4000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_generic_mirror_quirk, mirror,
+                          "vfio-ati-bar2-4000-quirk", PCI_CONFIG_SPACE_SIZE);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Older ATI/AMD cards like the X550 have a similar window to that above.
+ * I/O port BAR1 provides a window to a mirror of PCI config space located
+ * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
+ * note it for future reference.
+ */
+
+#define PCI_VENDOR_ID_NVIDIA                    0x10de
+
+/*
+ * Nvidia has several different methods to get to config space, the
+ * nouveu project has several of these documented here:
+ * https://github.com/pathscale/envytools/tree/master/hwdocs
+ *
+ * The first quirk is actually not documented in envytools and is found
+ * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
+ * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
+ * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
+ * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
+ * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
+ * is written for a write to 0x3d4.  The BAR0 offset is then accessible
+ * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
+ * that use the I/O port BAR5 window but it doesn't hurt to leave it.
+ */
+typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State;
+static const char *nv3d0_states[] = { "NONE", "SELECT",
+                                      "WINDOW", "READ", "WRITE" };
+
+typedef struct VFIONvidia3d0Quirk {
+    VFIOPCIDevice *vdev;
+    VFIONvidia3d0State state;
+    uint32_t offset;
+} VFIONvidia3d0Quirk;
+
+static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+
+    quirk->state = NONE;
+
+    return vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                         addr + 0x14, size);
+}
+
+static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    switch (data) {
+    case 0x338:
+        if (old_state == NONE) {
+            quirk->state = SELECT;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x538:
+        if (old_state == WINDOW) {
+            quirk->state = READ;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x738:
+        if (old_state == WINDOW) {
+            quirk->state = WRITE;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    }
+
+    vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x14, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d4_quirk = {
+    .read = vfio_nvidia_3d4_quirk_read,
+    .write = vfio_nvidia_3d4_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+    uint64_t data = vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                                  addr + 0x10, size);
+
+    quirk->state = NONE;
+
+    if (old_state == READ &&
+        (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+        uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+        data = vfio_pci_read_config(&vdev->pdev, offset, size);
+        trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name,
+                                         offset, size, data);
+    }
+
+    return data;
+}
+
+static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    if (old_state == SELECT) {
+        quirk->offset = (uint32_t)data;
+        quirk->state = WINDOW;
+        trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                          nv3d0_states[quirk->state]);
+    } else if (old_state == WRITE) {
+        if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+            uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+            vfio_pci_write_config(&vdev->pdev, offset, data, size);
+            trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name,
+                                              offset, data, size);
+            return;
+        }
+    }
+
+    vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x10, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
+    .read = vfio_nvidia_3d0_quirk_read,
+    .write = vfio_nvidia_3d0_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    VFIONvidia3d0Quirk *data;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->bars[1].region.size) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->data = data = g_malloc0(sizeof(*data));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    data->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk,
+                          data, "vfio-nvidia-3d4-quirk", 2);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk,
+                          data, "vfio-nvidia-3d0-quirk", 2);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]);
+
+    QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * The second quirk is documented in envytools.  The I/O port BAR5 is just
+ * a set of address/data ports to the MMIO BARs.  The BAR we care about is
+ * again BAR0.  This backdoor is apparently a bit newer than the one above
+ * so we need to not only trap 256 bytes @0x1800, but all of PCI config
+ * space, including extended space is available at the 4k @0x88000.
+ */
+typedef struct VFIONvidiaBAR5Quirk {
+    uint32_t master;
+    uint32_t enable;
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+    bool enabled;
+    VFIOConfigWindowQuirk window; /* last for match data */
+} VFIONvidiaBAR5Quirk;
+
+static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5)
+{
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) {
+        return;
+    }
+
+    bar5->enabled = !bar5->enabled;
+    trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name,
+                                       bar5->enabled ?  "Enable" : "Disable");
+    memory_region_set_enabled(bar5->addr_mem, bar5->enabled);
+    memory_region_set_enabled(bar5->data_mem, bar5->enabled);
+}
+
+static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr, size);
+}
+
+static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr, data, size);
+
+    bar5->master = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = {
+    .read = vfio_nvidia_bar5_quirk_master_read,
+    .write = vfio_nvidia_bar5_quirk_master_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr + 4, size);
+}
+
+static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr + 4, data, size);
+
+    bar5->enable = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = {
+    .read = vfio_nvidia_bar5_quirk_enable_read,
+    .write = vfio_nvidia_bar5_quirk_enable_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIONvidiaBAR5Quirk *bar5;
+    VFIOConfigWindowQuirk *window;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->has_vga || nr != 5) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 4);
+    quirk->nr_mem = 4;
+    bar5 = quirk->data = g_malloc0(sizeof(*bar5) +
+                                   (sizeof(VFIOConfigWindowMatch) * 2));
+    window = &bar5->window;
+
+    window->vdev = vdev;
+    window->address_offset = 0x8;
+    window->data_offset = 0xc;
+    window->nr_matches = 2;
+    window->matches[0].match = 0x1800;
+    window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1;
+    window->matches[1].match = 0x88000;
+    window->matches[1].mask = vdev->config_size - 1;
+    window->bar = nr;
+    window->addr_mem = bar5->addr_mem = &quirk->mem[0];
+    window->data_mem = bar5->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-nvidia-bar5-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+    memory_region_set_enabled(window->addr_mem, false);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-nvidia-bar5-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+    memory_region_set_enabled(window->data_mem, false);
+
+    memory_region_init_io(&quirk->mem[2], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_master, bar5,
+                          "vfio-nvidia-bar5-master-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0, &quirk->mem[2], 1);
+
+    memory_region_init_io(&quirk->mem[3], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_enable, bar5,
+                          "vfio-nvidia-bar5-enable-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        4, &quirk->mem[3], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Finally, BAR0 itself.  We want to redirect any accesses to either
+ * 0x1800 or 0x88000 through the PCI config space access functions.
+ */
+static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr,
+                                           uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    PCIDevice *pdev = &vdev->pdev;
+
+    vfio_generic_quirk_mirror_write(opaque, addr, data, size);
+
+    /*
+     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
+     * MSI capability ID register.  Both the ID and next register are
+     * read-only, so we allow writes covering either of those to real hw.
+     */
+    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
+        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
+        vfio_region_write(&vdev->bars[mirror->bar].region,
+                          addr + mirror->offset, data, size);
+        trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name);
+    }
+}
+
+static const MemoryRegionOps vfio_nvidia_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_nvidia_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vfio_is_vga(vdev) || nr != 0) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x88000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_nvidia_mirror_quirk, mirror,
+                          "vfio-nvidia-bar0-88000-mirror-quirk",
+                          vdev->config_size);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    /* The 0x1800 offset mirror only seems to get used by legacy VGA */
+    if (vdev->has_vga) {
+        quirk = g_malloc0(sizeof(*quirk));
+        mirror = quirk->data = g_malloc0(sizeof(*mirror));
+        mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+        quirk->nr_mem = 1;
+        mirror->vdev = vdev;
+        mirror->offset = 0x1800;
+        mirror->bar = nr;
+
+        memory_region_init_io(mirror->mem, OBJECT(vdev),
+                              &vfio_nvidia_mirror_quirk, mirror,
+                              "vfio-nvidia-bar0-1800-mirror-quirk",
+                              PCI_CONFIG_SPACE_SIZE);
+        memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                            mirror->offset, mirror->mem, 1);
+
+        QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+    }
+
+    trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * TODO - Some Nvidia devices provide config access to their companion HDA
+ * device and even to their parent bridge via these config space mirrors.
+ * Add quirks for those regions.
+ */
+
+#define PCI_VENDOR_ID_REALTEK 0x10ec
+
+/*
+ * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
+ * offset 0x70 there is a dword data register, offset 0x74 is a dword address
+ * register.  According to the Linux r8169 driver, the MSI-X table is addressed
+ * when the "type" portion of the address register is set to 0x1.  This appears
+ * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
+ * "address latched" indicator.  Bits 12:15 are a mask field, which we can
+ * ignore because the MSI-X table should always be accessed as a dword (full
+ * mask).  Bits 0:11 is offset within the type.
+ *
+ * Example trace:
+ *
+ * Read from MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
+ *
+ * Write 0xfee00000 to MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
+ */
+typedef struct VFIOrtl8168Quirk {
+    VFIOPCIDevice *vdev;
+    uint32_t addr;
+    uint32_t data;
+    bool enabled;
+} VFIOrtl8168Quirk;
+
+static uint64_t vfio_rtl8168_quirk_address_read(void *opaque,
+                                                hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled) {
+        data = rtl->addr ^ 0x80000000U; /* latch/complete */
+        trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr,
+                                             uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->enabled = false;
+
+    if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */
+        rtl->enabled = true;
+        rtl->addr = (uint32_t)data;
+
+        if (data & 0x80000000U) { /* Do write */
+            if (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
+                hwaddr offset = data & 0xfff;
+                uint64_t val = rtl->data;
+
+                trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name,
+                                                    (uint16_t)offset, val);
+
+                /* Write to the proper guest MSI-X table instead */
+                memory_region_dispatch_write(&vdev->pdev.msix_table_mmio,
+                                             offset, val, size,
+                                             MEMTXATTRS_UNSPECIFIED);
+            }
+            return; /* Do not write guest MSI-X data to hardware */
+        }
+    }
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_address_quirk = {
+    .read = vfio_rtl8168_quirk_address_read,
+    .write = vfio_rtl8168_quirk_address_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_rtl8168_quirk_data_read(void *opaque,
+                                             hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
+        hwaddr offset = rtl->addr & 0xfff;
+        memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset,
+                                    &data, size, MEMTXATTRS_UNSPECIFIED);
+        trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr,
+                                          uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->data = (uint32_t)data;
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_data_quirk = {
+    .read = vfio_rtl8168_quirk_data_read,
+    .write = vfio_rtl8168_quirk_data_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOrtl8168Quirk *rtl;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    quirk->data = rtl = g_malloc0(sizeof(*rtl));
+    rtl->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev),
+                          &vfio_rtl_address_quirk, rtl,
+                          "vfio-rtl8168-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0x74, &quirk->mem[0], 1);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev),
+                          &vfio_rtl_data_quirk, rtl,
+                          "vfio-rtl8168-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0x70, &quirk->mem[1], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Common quirk probe entry points.
+ */
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev)
+{
+    vfio_vga_probe_ati_3c3_quirk(vdev);
+    vfio_vga_probe_nvidia_3d0_quirk(vdev);
+}
+
+void vfio_vga_quirk_exit(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+        QLIST_FOREACH(quirk, &vdev->vga->region[i].quirks, next) {
+            for (j = 0; j < quirk->nr_mem; j++) {
+                memory_region_del_subregion(&vdev->vga->region[i].mem,
+                                            &quirk->mem[j]);
+            }
+        }
+    }
+}
+
+void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev)
+{
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+        while (!QLIST_EMPTY(&vdev->vga->region[i].quirks)) {
+            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga->region[i].quirks);
+            QLIST_REMOVE(quirk, next);
+            for (j = 0; j < quirk->nr_mem; j++) {
+                object_unparent(OBJECT(&quirk->mem[j]));
+            }
+            g_free(quirk->mem);
+            g_free(quirk->data);
+            g_free(quirk);
+        }
+    }
+}
+
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr)
+{
+    vfio_probe_ati_bar4_quirk(vdev, nr);
+    vfio_probe_ati_bar2_quirk(vdev, nr);
+    vfio_probe_nvidia_bar5_quirk(vdev, nr);
+    vfio_probe_nvidia_bar0_quirk(vdev, nr);
+    vfio_probe_rtl8168_bar2_quirk(vdev, nr);
+}
+
+void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    VFIOQuirk *quirk;
+    int i;
+
+    QLIST_FOREACH(quirk, &bar->quirks, next) {
+        for (i = 0; i < quirk->nr_mem; i++) {
+            memory_region_del_subregion(bar->region.mem, &quirk->mem[i]);
+        }
+    }
+}
+
+void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    int i;
+
+    while (!QLIST_EMPTY(&bar->quirks)) {
+        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
+        QLIST_REMOVE(quirk, next);
+        for (i = 0; i < quirk->nr_mem; i++) {
+            object_unparent(OBJECT(&quirk->mem[i]));
+        }
+        g_free(quirk->mem);
+        g_free(quirk->data);
+        g_free(quirk);
+    }
+}
+
+/*
+ * Reset quirks
+ */
+
+/*
+ * AMD Radeon PCI config reset, based on Linux:
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running()
+ *   drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc()
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock()
+ * IDs: include/drm/drm_pciids.h
+ * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0
+ *
+ * Bonaire and Hawaii GPUs do not respond to a bus reset.  This is a bug in the
+ * hardware that should be fixed on future ASICs.  The symptom of this is that
+ * once the accerlated driver loads, Windows guests will bsod on subsequent
+ * attmpts to load the driver, such as after VM reset or shutdown/restart.  To
+ * work around this, we do an AMD specific PCI config reset, followed by an SMC
+ * reset.  The PCI config reset only works if SMC firmware is running, so we
+ * have a dependency on the state of the device as to whether this reset will
+ * be effective.  There are still cases where we won't be able to kick the
+ * device into working, but this greatly improves the usability overall.  The
+ * config reset magic is relatively common on AMD GPUs, but the setup and SMC
+ * poking is largely ASIC specific.
+ */
+static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev)
+{
+    uint32_t clk, pc_c;
+
+    /*
+     * Registers 200h and 204h are index and data registers for accessing
+     * indirect configuration registers within the device.
+     */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4);
+    pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+
+    return (!(clk & 1) && (0x20100 <= pc_c));
+}
+
+/*
+ * The scope of a config reset is controlled by a mode bit in the misc register
+ * and a fuse, exposed as a bit in another register.  The fuse is the default
+ * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula
+ * scope = !(misc ^ fuse), where the resulting scope is defined the same as
+ * the fuse.  A truth table therefore tells us that if misc == fuse, we need
+ * to flip the value of the bit in the misc register.
+ */
+static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev)
+{
+    uint32_t misc, fuse;
+    bool a, b;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4);
+    fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    b = fuse & 64;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4);
+    misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    a = misc & 2;
+
+    if (a == b) {
+        vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4);
+        vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */
+    }
+}
+
+static int vfio_radeon_reset(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    int i, ret = 0;
+    uint32_t data;
+
+    /* Defer to a kernel implemented reset */
+    if (vdev->vbasedev.reset_works) {
+        trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name);
+        return -ENODEV;
+    }
+
+    /* Enable only memory BAR access */
+    vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2);
+
+    /* Reset only works if SMC firmware is loaded and running */
+    if (!vfio_radeon_smc_is_running(vdev)) {
+        ret = -EINVAL;
+        trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Make sure only the GFX function is reset */
+    vfio_radeon_set_gfx_only_reset(vdev);
+
+    /* AMD PCI config reset */
+    vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4);
+    usleep(100);
+
+    /* Read back the memory size to make sure we're out of reset */
+    for (i = 0; i < 100000; i++) {
+        if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) {
+            goto reset_smc;
+        }
+        usleep(1);
+    }
+
+    trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name);
+
+reset_smc:
+    /* Reset SMC */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    /* Disable SMC clock */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name);
+
+out:
+    /* Restore PCI command register */
+    vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2);
+
+    return ret;
+}
+
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev)
+{
+    switch (vdev->vendor_id) {
+    case 0x1002:
+        switch (vdev->device_id) {
+        /* Bonaire */
+        case 0x6649: /* Bonaire [FirePro W5100] */
+        case 0x6650:
+        case 0x6651:
+        case 0x6658: /* Bonaire XTX [Radeon R7 260X] */
+        case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */
+        case 0x665d: /* Bonaire [Radeon R7 200 Series] */
+        /* Hawaii */
+        case 0x67A0: /* Hawaii XT GL [FirePro W9100] */
+        case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */
+        case 0x67A2:
+        case 0x67A8:
+        case 0x67A9:
+        case 0x67AA:
+        case 0x67B0: /* Hawaii XT [Radeon R9 290X] */
+        case 0x67B1: /* Hawaii PRO [Radeon R9 290] */
+        case 0x67B8:
+        case 0x67B9:
+        case 0x67BA:
+        case 0x67BE:
+            vdev->resetfn = vfio_radeon_reset;
+            trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name);
+            break;
+        }
+        break;
+    }
+}
diff --git a/qemu/hw/vfio/pci.c b/qemu/hw/vfio/pci.c
index 4023d8e82..d091d8cf0 100644
--- a/qemu/hw/vfio/pci.c
+++ b/qemu/hw/vfio/pci.c
@@ -18,187 +18,24 @@
  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  */
 
-#include <dirent.h>
+#include "qemu/osdep.h"
 #include <linux/vfio.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 
-#include "config.h"
-#include "exec/address-spaces.h"
-#include "exec/memory.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
-#include "hw/pci/pci.h"
-#include "qemu-common.h"
+#include "hw/pci/pci_bridge.h"
 #include "qemu/error-report.h"
-#include "qemu/event_notifier.h"
-#include "qemu/queue.h"
 #include "qemu/range.h"
 #include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
+#include "pci.h"
 #include "trace.h"
-#include "hw/vfio/vfio.h"
-#include "hw/vfio/vfio-common.h"
-
-struct VFIOPCIDevice;
-
-typedef struct VFIOQuirk {
-    MemoryRegion mem;
-    struct VFIOPCIDevice *vdev;
-    QLIST_ENTRY(VFIOQuirk) next;
-    struct {
-        uint32_t base_offset:TARGET_PAGE_BITS;
-        uint32_t address_offset:TARGET_PAGE_BITS;
-        uint32_t address_size:3;
-        uint32_t bar:3;
-
-        uint32_t address_match;
-        uint32_t address_mask;
-
-        uint32_t address_val:TARGET_PAGE_BITS;
-        uint32_t data_offset:TARGET_PAGE_BITS;
-        uint32_t data_size:3;
-
-        uint8_t flags;
-        uint8_t read_flags;
-        uint8_t write_flags;
-    } data;
-} VFIOQuirk;
-
-typedef struct VFIOBAR {
-    VFIORegion region;
-    bool ioport;
-    bool mem64;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOBAR;
-
-typedef struct VFIOVGARegion {
-    MemoryRegion mem;
-    off_t offset;
-    int nr;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOVGARegion;
-
-typedef struct VFIOVGA {
-    off_t fd_offset;
-    int fd;
-    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
-} VFIOVGA;
-
-typedef struct VFIOINTx {
-    bool pending; /* interrupt pending */
-    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
-    uint8_t pin; /* which pin to pull for qemu_set_irq */
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
-    PCIINTxRoute route; /* routing info for QEMU bypass */
-    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
-    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
-} VFIOINTx;
-
-typedef struct VFIOMSIVector {
-    /*
-     * Two interrupt paths are configured per vector.  The first, is only used
-     * for interrupts injected via QEMU.  This is typically the non-accel path,
-     * but may also be used when we want QEMU to handle masking and pending
-     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
-     * but requires masking at the device.  virq is used to track the MSI route
-     * through KVM, thus kvm_interrupt is only available when virq is set to a
-     * valid (>= 0) value.
-     */
-    EventNotifier interrupt;
-    EventNotifier kvm_interrupt;
-    struct VFIOPCIDevice *vdev; /* back pointer to device */
-    int virq;
-    bool use;
-} VFIOMSIVector;
-
-enum {
-    VFIO_INT_NONE = 0,
-    VFIO_INT_INTx = 1,
-    VFIO_INT_MSI  = 2,
-    VFIO_INT_MSIX = 3,
-};
-
-/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
-typedef struct VFIOMSIXInfo {
-    uint8_t table_bar;
-    uint8_t pba_bar;
-    uint16_t entries;
-    uint32_t table_offset;
-    uint32_t pba_offset;
-    MemoryRegion mmap_mem;
-    void *mmap;
-} VFIOMSIXInfo;
-
-typedef struct VFIOPCIDevice {
-    PCIDevice pdev;
-    VFIODevice vbasedev;
-    VFIOINTx intx;
-    unsigned int config_size;
-    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
-    off_t config_offset; /* Offset of config space region within device fd */
-    unsigned int rom_size;
-    off_t rom_offset; /* Offset of ROM region within device fd */
-    void *rom;
-    int msi_cap_size;
-    VFIOMSIVector *msi_vectors;
-    VFIOMSIXInfo *msix;
-    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
-    int interrupt; /* Current interrupt type */
-    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
-    VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
-    PCIHostDeviceAddress host;
-    EventNotifier err_notifier;
-    EventNotifier req_notifier;
-    int (*resetfn)(struct VFIOPCIDevice *);
-    uint32_t features;
-#define VFIO_FEATURE_ENABLE_VGA_BIT 0
-#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
-#define VFIO_FEATURE_ENABLE_REQ_BIT 1
-#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
-    int32_t bootindex;
-    uint8_t pm_cap;
-    bool has_vga;
-    bool pci_aer;
-    bool req_enabled;
-    bool has_flr;
-    bool has_pm_reset;
-    bool rom_read_failed;
-} VFIOPCIDevice;
-
-typedef struct VFIORomBlacklistEntry {
-    uint16_t vendor_id;
-    uint16_t device_id;
-} VFIORomBlacklistEntry;
-
-/*
- * List of device ids/vendor ids for which to disable
- * option rom loading. This avoids the guest hangs during rom
- * execution as noticed with the BCM 57810 card for lack of a
- * more better way to handle such issues.
- * The  user can still override by specifying a romfile or
- * rombar=1.
- * Please see https://bugs.launchpad.net/qemu/+bug/1284874
- * for an analysis of the 57810 card hang. When adding
- * a new vendor id/device id combination below, please also add
- * your card/environment details and information that could
- * help in debugging to the bug tracking this issue
- */
-static const VFIORomBlacklistEntry romblacklist[] = {
-    /* Broadcom BCM 57810 */
-    { 0x14e4, 0x168e }
-};
 
 #define MSIX_CAP_LENGTH 12
 
 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len);
 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
 
 /*
@@ -248,7 +85,7 @@ static void vfio_intx_interrupt(void *opaque)
     }
 }
 
-static void vfio_eoi(VFIODevice *vbasedev)
+static void vfio_intx_eoi(VFIODevice *vbasedev)
 {
     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
 
@@ -256,14 +93,14 @@ static void vfio_eoi(VFIODevice *vbasedev)
         return;
     }
 
-    trace_vfio_eoi(vbasedev->name);
+    trace_vfio_intx_eoi(vbasedev->name);
 
     vdev->intx.pending = false;
     pci_irq_deassert(&vdev->pdev);
     vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
 }
 
-static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
+static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev)
 {
 #ifdef CONFIG_KVM
     struct kvm_irqfd irqfd = {
@@ -275,7 +112,7 @@ static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
     int ret, argsz;
     int32_t *pfd;
 
-    if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() ||
+    if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
         vdev->intx.route.mode != PCI_INTX_ENABLED ||
         !kvm_resamplefds_enabled()) {
         return;
@@ -325,7 +162,7 @@ static void vfio_enable_intx_kvm(VFIOPCIDevice *vdev)
 
     vdev->intx.kvm_accel = true;
 
-    trace_vfio_enable_intx_kvm(vdev->vbasedev.name);
+    trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
 
     return;
 
@@ -340,7 +177,7 @@ fail:
 #endif
 }
 
-static void vfio_disable_intx_kvm(VFIOPCIDevice *vdev)
+static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
 {
 #ifdef CONFIG_KVM
     struct kvm_irqfd irqfd = {
@@ -377,11 +214,11 @@ static void vfio_disable_intx_kvm(VFIOPCIDevice *vdev)
     /* If we've missed an event, let it re-fire through QEMU */
     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
 
-    trace_vfio_disable_intx_kvm(vdev->vbasedev.name);
+    trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
 #endif
 }
 
-static void vfio_update_irq(PCIDevice *pdev)
+static void vfio_intx_update(PCIDevice *pdev)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     PCIINTxRoute route;
@@ -396,10 +233,10 @@ static void vfio_update_irq(PCIDevice *pdev)
         return; /* Nothing changed */
     }
 
-    trace_vfio_update_irq(vdev->vbasedev.name,
-                          vdev->intx.route.irq, route.irq);
+    trace_vfio_intx_update(vdev->vbasedev.name,
+                           vdev->intx.route.irq, route.irq);
 
-    vfio_disable_intx_kvm(vdev);
+    vfio_intx_disable_kvm(vdev);
 
     vdev->intx.route = route;
 
@@ -407,13 +244,13 @@ static void vfio_update_irq(PCIDevice *pdev)
         return;
     }
 
-    vfio_enable_intx_kvm(vdev);
+    vfio_intx_enable_kvm(vdev);
 
     /* Re-enable the interrupt in cased we missed an EOI */
-    vfio_eoi(&vdev->vbasedev);
+    vfio_intx_eoi(&vdev->vbasedev);
 }
 
-static int vfio_enable_intx(VFIOPCIDevice *vdev)
+static int vfio_intx_enable(VFIOPCIDevice *vdev)
 {
     uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
     int ret, argsz;
@@ -468,21 +305,21 @@ static int vfio_enable_intx(VFIOPCIDevice *vdev)
         return -errno;
     }
 
-    vfio_enable_intx_kvm(vdev);
+    vfio_intx_enable_kvm(vdev);
 
     vdev->interrupt = VFIO_INT_INTx;
 
-    trace_vfio_enable_intx(vdev->vbasedev.name);
+    trace_vfio_intx_enable(vdev->vbasedev.name);
 
     return 0;
 }
 
-static void vfio_disable_intx(VFIOPCIDevice *vdev)
+static void vfio_intx_disable(VFIOPCIDevice *vdev)
 {
     int fd;
 
     timer_del(vdev->intx.mmap_timer);
-    vfio_disable_intx_kvm(vdev);
+    vfio_intx_disable_kvm(vdev);
     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
     vdev->intx.pending = false;
     pci_irq_deassert(&vdev->pdev);
@@ -494,7 +331,7 @@ static void vfio_disable_intx(VFIOPCIDevice *vdev)
 
     vdev->interrupt = VFIO_INT_NONE;
 
-    trace_vfio_disable_intx(vdev->vbasedev.name);
+    trace_vfio_intx_disable(vdev->vbasedev.name);
 }
 
 /*
@@ -504,33 +341,35 @@ static void vfio_msi_interrupt(void *opaque)
 {
     VFIOMSIVector *vector = opaque;
     VFIOPCIDevice *vdev = vector->vdev;
+    MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
+    void (*notify)(PCIDevice *dev, unsigned vector);
+    MSIMessage msg;
     int nr = vector - vdev->msi_vectors;
 
     if (!event_notifier_test_and_clear(&vector->interrupt)) {
         return;
     }
 
-#ifdef DEBUG_VFIO
-    MSIMessage msg;
-
     if (vdev->interrupt == VFIO_INT_MSIX) {
-        msg = msix_get_message(&vdev->pdev, nr);
+        get_msg = msix_get_message;
+        notify = msix_notify;
+
+        /* A masked vector firing needs to use the PBA, enable it */
+        if (msix_is_masked(&vdev->pdev, nr)) {
+            set_bit(nr, vdev->msix->pending);
+            memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
+            trace_vfio_msix_pba_enable(vdev->vbasedev.name);
+        }
     } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msg = msi_get_message(&vdev->pdev, nr);
+        get_msg = msi_get_message;
+        notify = msi_notify;
     } else {
         abort();
     }
 
+    msg = get_msg(&vdev->pdev, nr);
     trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
-#endif
-
-    if (vdev->interrupt == VFIO_INT_MSIX) {
-        msix_notify(&vdev->pdev, nr);
-    } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msi_notify(&vdev->pdev, nr);
-    } else {
-        error_report("vfio: MSI interrupt receieved, but not enabled?");
-    }
+    notify(&vdev->pdev, nr);
 }
 
 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
@@ -577,13 +416,12 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
     return ret;
 }
 
-static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
-                                  bool msix)
+static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
+                                  MSIMessage *msg, bool msix)
 {
     int virq;
 
-    if ((msix && !VFIO_ALLOW_KVM_MSIX) ||
-        (!msix && !VFIO_ALLOW_KVM_MSI) || !msg) {
+    if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi) || !msg) {
         return;
     }
 
@@ -591,7 +429,7 @@ static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
         return;
     }
 
-    virq = kvm_irqchip_add_msi_route(kvm_state, *msg);
+    virq = kvm_irqchip_add_msi_route(kvm_state, *msg, &vdev->pdev);
     if (virq < 0) {
         event_notifier_cleanup(&vector->kvm_interrupt);
         return;
@@ -616,9 +454,10 @@ static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
     event_notifier_cleanup(&vector->kvm_interrupt);
 }
 
-static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg)
+static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
+                                     PCIDevice *pdev)
 {
-    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
+    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
 }
 
 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
@@ -653,10 +492,10 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         if (!msg) {
             vfio_remove_kvm_msi_virq(vector);
         } else {
-            vfio_update_kvm_msi_virq(vector, *msg);
+            vfio_update_kvm_msi_virq(vector, *msg, pdev);
         }
     } else {
-        vfio_add_kvm_msi_virq(vector, msg, true);
+        vfio_add_kvm_msi_virq(vdev, vector, msg, true);
     }
 
     /*
@@ -700,6 +539,14 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
         }
     }
 
+    /* Disable PBA emulation when nothing more is pending. */
+    clear_bit(nr, vdev->msix->pending);
+    if (find_first_bit(vdev->msix->pending,
+                       vdev->nr_vectors) == vdev->nr_vectors) {
+        memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
+        trace_vfio_msix_pba_disable(vdev->vbasedev.name);
+    }
+
     return 0;
 }
 
@@ -748,11 +595,11 @@ static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
     }
 }
 
-static void vfio_enable_msix(VFIOPCIDevice *vdev)
+static void vfio_msix_enable(VFIOPCIDevice *vdev)
 {
     vfio_disable_interrupts(vdev);
 
-    vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
 
     vdev->interrupt = VFIO_INT_MSIX;
 
@@ -777,10 +624,10 @@ static void vfio_enable_msix(VFIOPCIDevice *vdev)
         error_report("vfio: msix_set_vector_notifiers failed");
     }
 
-    trace_vfio_enable_msix(vdev->vbasedev.name);
+    trace_vfio_msix_enable(vdev->vbasedev.name);
 }
 
-static void vfio_enable_msi(VFIOPCIDevice *vdev)
+static void vfio_msi_enable(VFIOPCIDevice *vdev)
 {
     int ret, i;
 
@@ -788,7 +635,7 @@ static void vfio_enable_msi(VFIOPCIDevice *vdev)
 
     vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
 retry:
-    vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
 
     for (i = 0; i < vdev->nr_vectors; i++) {
         VFIOMSIVector *vector = &vdev->msi_vectors[i];
@@ -809,7 +656,7 @@ retry:
          * Attempt to enable route through KVM irqchip,
          * default to userspace handling if unavailable.
          */
-        vfio_add_kvm_msi_virq(vector, &msg, false);
+        vfio_add_kvm_msi_virq(vdev, vector, &msg, false);
     }
 
     /* Set interrupt type prior to possible interrupts */
@@ -853,10 +700,10 @@ retry:
         return;
     }
 
-    trace_vfio_enable_msi(vdev->vbasedev.name, vdev->nr_vectors);
+    trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
 }
 
-static void vfio_disable_msi_common(VFIOPCIDevice *vdev)
+static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
 {
     int i;
 
@@ -877,10 +724,10 @@ static void vfio_disable_msi_common(VFIOPCIDevice *vdev)
     vdev->nr_vectors = 0;
     vdev->interrupt = VFIO_INT_NONE;
 
-    vfio_enable_intx(vdev);
+    vfio_intx_enable(vdev);
 }
 
-static void vfio_disable_msix(VFIOPCIDevice *vdev)
+static void vfio_msix_disable(VFIOPCIDevice *vdev)
 {
     int i;
 
@@ -901,17 +748,20 @@ static void vfio_disable_msix(VFIOPCIDevice *vdev)
         vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
     }
 
-    vfio_disable_msi_common(vdev);
+    vfio_msi_disable_common(vdev);
+
+    memset(vdev->msix->pending, 0,
+           BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
 
-    trace_vfio_disable_msix(vdev->vbasedev.name);
+    trace_vfio_msix_disable(vdev->vbasedev.name);
 }
 
-static void vfio_disable_msi(VFIOPCIDevice *vdev)
+static void vfio_msi_disable(VFIOPCIDevice *vdev)
 {
     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
-    vfio_disable_msi_common(vdev);
+    vfio_msi_disable_common(vdev);
 
-    trace_vfio_disable_msi(vdev->vbasedev.name);
+    trace_vfio_msi_disable(vdev->vbasedev.name);
 }
 
 static void vfio_update_msi(VFIOPCIDevice *vdev)
@@ -927,31 +777,31 @@ static void vfio_update_msi(VFIOPCIDevice *vdev)
         }
 
         msg = msi_get_message(&vdev->pdev, i);
-        vfio_update_kvm_msi_virq(vector, msg);
+        vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
     }
 }
 
 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
 {
-    struct vfio_region_info reg_info = {
-        .argsz = sizeof(reg_info),
-        .index = VFIO_PCI_ROM_REGION_INDEX
-    };
+    struct vfio_region_info *reg_info;
     uint64_t size;
     off_t off = 0;
     ssize_t bytes;
 
-    if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
+    if (vfio_get_region_info(&vdev->vbasedev,
+                             VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
         error_report("vfio: Error getting ROM info: %m");
         return;
     }
 
-    trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info.size,
-                            (unsigned long)reg_info.offset,
-                            (unsigned long)reg_info.flags);
+    trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
+                            (unsigned long)reg_info->offset,
+                            (unsigned long)reg_info->flags);
+
+    vdev->rom_size = size = reg_info->size;
+    vdev->rom_offset = reg_info->offset;
 
-    vdev->rom_size = size = reg_info.size;
-    vdev->rom_offset = reg_info.offset;
+    g_free(reg_info);
 
     if (!vdev->rom_size) {
         vdev->rom_read_failed = true;
@@ -982,6 +832,36 @@ static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
             break;
         }
     }
+
+    /*
+     * Test the ROM signature against our device, if the vendor is correct
+     * but the device ID doesn't match, store the correct device ID and
+     * recompute the checksum.  Intel IGD devices need this and are known
+     * to have bogus checksums so we can't simply adjust the checksum.
+     */
+    if (pci_get_word(vdev->rom) == 0xaa55 &&
+        pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
+        !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
+        uint16_t vid, did;
+
+        vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
+        did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
+
+        if (vid == vdev->vendor_id && did != vdev->device_id) {
+            int i;
+            uint8_t csum, *data = vdev->rom;
+
+            pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
+                         vdev->device_id);
+            data[6] = 0;
+
+            for (csum = 0, i = 0; i < vdev->rom_size; i++) {
+                csum += data[i];
+            }
+
+            data[6] = -csum;
+        }
+    }
 }
 
 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
@@ -1034,43 +914,19 @@ static const MemoryRegionOps vfio_rom_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t vendor_id, device_id;
-    int count = 0;
-
-    vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
-    while (count < ARRAY_SIZE(romblacklist)) {
-        if (romblacklist[count].vendor_id == vendor_id &&
-            romblacklist[count].device_id == device_id) {
-                return true;
-        }
-        count++;
-    }
-
-    return false;
-}
-
 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
 {
     uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
     off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
     DeviceState *dev = DEVICE(vdev);
-    char name[32];
+    char *name;
     int fd = vdev->vbasedev.fd;
 
     if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
         /* Since pci handles romfile, just print a message and return */
         if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
-            error_printf("Warning : Device at %04x:%02x:%02x.%x "
-                         "is known to cause system instability issues during "
-                         "option rom execution. "
-                         "Proceeding anyway since user specified romfile\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
+            error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified romfile\n",
+                         vdev->vbasedev.name);
         }
         return;
     }
@@ -1083,9 +939,7 @@ static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
         pwrite(fd, &size, 4, offset) != 4 ||
         pread(fd, &size, 4, offset) != 4 ||
         pwrite(fd, &orig, 4, offset) != 4) {
-        error_report("%s(%04x:%02x:%02x.%x) failed: %m",
-                     __func__, vdev->host.domain, vdev->host.bus,
-                     vdev->host.slot, vdev->host.function);
+        error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
         return;
     }
 
@@ -1097,32 +951,22 @@ static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
 
     if (vfio_blacklist_opt_rom(vdev)) {
         if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
-            error_printf("Warning : Device at %04x:%02x:%02x.%x "
-                         "is known to cause system instability issues during "
-                         "option rom execution. "
-                         "Proceeding anyway since user specified non zero value for "
-                         "rombar\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
+            error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified non zero value for rombar\n",
+                         vdev->vbasedev.name);
         } else {
-            error_printf("Warning : Rom loading for device at "
-                         "%04x:%02x:%02x.%x has been disabled due to "
-                         "system instability issues. "
-                         "Specify rombar=1 or romfile to force\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
+            error_printf("Warning : Rom loading for device at %s has been disabled due to system instability issues. Specify rombar=1 or romfile to force\n",
+                         vdev->vbasedev.name);
             return;
         }
     }
 
     trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
 
-    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function);
+    name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
 
     memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
                           &vfio_rom_ops, vdev, name, size);
+    g_free(name);
 
     pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
                      PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
@@ -1131,7 +975,7 @@ static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
     vdev->rom_read_failed = false;
 }
 
-static void vfio_vga_write(void *opaque, hwaddr addr,
+void vfio_vga_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     VFIOVGARegion *region = opaque;
@@ -1167,7 +1011,7 @@ static void vfio_vga_write(void *opaque, hwaddr addr,
     trace_vfio_vga_write(region->offset + addr, data, size);
 }
 
-static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
 {
     VFIOVGARegion *region = opaque;
     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
@@ -1213,858 +1057,9 @@ static const MemoryRegionOps vfio_vga_ops = {
 };
 
 /*
- * Device specific quirks
- */
-
-/* Is range1 fully contained within range2?  */
-static bool vfio_range_contained(uint64_t first1, uint64_t len1,
-                                 uint64_t first2, uint64_t len2) {
-    return (first1 >= first2 && first1 + len1 <= first2 + len2);
-}
-
-static bool vfio_flags_enabled(uint8_t flags, uint8_t mask)
-{
-    return (mask && (flags & mask) == mask);
-}
-
-static uint64_t vfio_generic_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev,
-                                    quirk->data.address_val + offset, size);
-
-        trace_vfio_generic_window_quirk_read(memory_region_name(&quirk->mem),
-                                             vdev->vbasedev.name,
-                                             quirk->data.bar,
-                                             addr, size, data);
-    } else {
-        data = vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                                addr + quirk->data.base_offset, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    if (ranges_overlap(addr, size,
-                       quirk->data.address_offset, quirk->data.address_size)) {
-
-        if (addr != quirk->data.address_offset) {
-            hw_error("%s: offset write into address window: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        if ((data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags |= quirk->data.write_flags |
-                                 quirk->data.read_flags;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        } else {
-            quirk->data.flags &= ~(quirk->data.write_flags |
-                                   quirk->data.read_flags);
-        }
-    }
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev,
-                              quirk->data.address_val + offset, data, size);
-        trace_vfio_generic_window_quirk_write(memory_region_name(&quirk->mem),
-                                              vdev->vbasedev.name,
-                                              quirk->data.bar,
-                                              addr, data, size);
-        return;
-    }
-
-    vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_generic_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_generic_window_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static uint64_t vfio_generic_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev, addr - offset, size);
-
-        trace_vfio_generic_quirk_read(memory_region_name(&quirk->mem),
-                                      vdev->vbasedev.name, quirk->data.bar,
-                                      addr + base, size, data);
-    } else {
-        data = vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                                addr + base, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_quirk_write(void *opaque, hwaddr addr,
-                                     uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev, addr - offset, data, size);
-
-        trace_vfio_generic_quirk_write(memory_region_name(&quirk->mem),
-                                       vdev->vbasedev.name, quirk->data.bar,
-                                       addr + base, data, size);
-    } else {
-        vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                          addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_generic_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_generic_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-#define PCI_VENDOR_ID_ATI               0x1002
-
-/*
- * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
- * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
- * BAR4 (older cards like the X550 used BAR1, but we don't care to support
- * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
- * I/O port BAR address.  Originally this was coded to return the virtual BAR
- * address only if the physical register read returns the actual BAR address,
- * but users have reported greater success if we return the virtual address
- * unconditionally.
- */
-static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    uint64_t data = vfio_pci_read_config(&vdev->pdev,
-                                         PCI_BASE_ADDRESS_0 + (4 * 4) + 1,
-                                         size);
-    trace_vfio_ati_3c3_quirk_read(data);
-
-    return data;
-}
-
-static const MemoryRegionOps vfio_ati_3c3_quirk = {
-    .read = vfio_ati_3c3_quirk_read,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    /*
-     * As long as the BAR is >= 256 bytes it will be aligned such that the
-     * lower byte is always zero.  Filter out anything else, if it exists.
-     */
-    if (!vdev->bars[4].ioport || vdev->bars[4].region.size < 256) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk,
-                          "vfio-ati-3c3-quirk", 1);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                3 /* offset 3 bytes from 0x3c0 */, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    trace_vfio_vga_probe_ati_3c3_quirk(vdev->vbasedev.name);
-}
-
-/*
- * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
- * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
- * the MMIO space directly, but a window to this space is provided through
- * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
- * data register.  When the address is programmed to a range of 0x4000-0x4fff
- * PCI configuration space is available.  Experimentation seems to indicate
- * that only read-only access is provided, but we drop writes when the window
- * is enabled to config space nonetheless.
- */
-static void vfio_probe_ati_bar4_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 4 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.address_size = 4;
-    quirk->data.data_offset = 4;
-    quirk->data.data_size = 4;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-    quirk->data.read_flags = quirk->data.write_flags = 1;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_generic_window_quirk, quirk,
-                          "vfio-ati-bar4-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.base_offset, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_ati_bar4_window_quirk(vdev->vbasedev.name);
-}
-
-#define PCI_VENDOR_ID_REALTEK 0x10ec
-
-/*
- * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
- * offset 0x70 there is a dword data register, offset 0x74 is a dword address
- * register.  According to the Linux r8169 driver, the MSI-X table is addressed
- * when the "type" portion of the address register is set to 0x1.  This appears
- * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
- * "address latched" indicator.  Bits 12:15 are a mask field, which we can
- * ignore because the MSI-X table should always be accessed as a dword (full
- * mask).  Bits 0:11 is offset within the type.
- *
- * Example trace:
- *
- * Read from MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
- *
- * Write 0xfee00000 to MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
- */
-
-static uint64_t vfio_rtl8168_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if (quirk->data.flags) {
-            trace_vfio_rtl8168_window_quirk_read_fake(
-                    memory_region_name(&quirk->mem),
-                    vdev->vbasedev.name);
-
-            return quirk->data.address_match ^ 0x80000000U;
-        }
-        break;
-    case 0: /* data */
-        if (quirk->data.flags) {
-            uint64_t val;
-
-            trace_vfio_rtl8168_window_quirk_read_table(
-                    memory_region_name(&quirk->mem),
-                    vdev->vbasedev.name);
-
-            if (!(vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
-                return 0;
-            }
-
-            memory_region_dispatch_read(&vdev->pdev.msix_table_mmio,
-                                        (hwaddr)(quirk->data.address_match
-                                                 & 0xfff),
-                                        &val,
-                                        size,
-                                        MEMTXATTRS_UNSPECIFIED);
-            return val;
-        }
-    }
-
-    trace_vfio_rtl8168_window_quirk_read_direct(memory_region_name(&quirk->mem),
-                                                vdev->vbasedev.name);
-
-    return vfio_region_read(&vdev->bars[quirk->data.bar].region,
-                            addr + 0x70, size);
-}
-
-static void vfio_rtl8168_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if ((data & 0x7fff0000) == 0x10000) {
-            if (data & 0x80000000U &&
-                vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
-
-                trace_vfio_rtl8168_window_quirk_write_table(
-                        memory_region_name(&quirk->mem),
-                        vdev->vbasedev.name);
-
-                memory_region_dispatch_write(&vdev->pdev.msix_table_mmio,
-                                             (hwaddr)(data & 0xfff),
-                                             (uint64_t)quirk->data.address_mask,
-                                             size, MEMTXATTRS_UNSPECIFIED);
-            }
-
-            quirk->data.flags = 1;
-            quirk->data.address_match = data;
-
-            return;
-        }
-        quirk->data.flags = 0;
-        break;
-    case 0: /* data */
-        quirk->data.address_mask = data;
-        break;
-    }
-
-    trace_vfio_rtl8168_window_quirk_write_direct(
-            memory_region_name(&quirk->mem),
-            vdev->vbasedev.name);
-
-    vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                      addr + 0x70, data, size);
-}
-
-static const MemoryRegionOps vfio_rtl8168_window_quirk = {
-    .read = vfio_rtl8168_window_quirk_read,
-    .write = vfio_rtl8168_window_quirk_write,
-    .valid = {
-        .min_access_size = 4,
-        .max_access_size = 4,
-        .unaligned = false,
-    },
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_rtl8168_bar2_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_REALTEK ||
-        pci_get_word(pdev->config + PCI_DEVICE_ID) != 0x8168 || nr != 2) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_rtl8168_window_quirk,
-                          quirk, "vfio-rtl8168-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                                        0x70, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_rtl8168_bar2_window_quirk(vdev->vbasedev.name);
-}
-/*
- * Trap the BAR2 MMIO window to config space as well.
- */
-static void vfio_probe_ati_bar2_4000_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    /* Only enable on newer devices where BAR2 is 64bit */
-    if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-ati-bar2-4000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_ati_bar2_4000_quirk(vdev->vbasedev.name);
-}
-
-/*
- * Older ATI/AMD cards like the X550 have a similar window to that above.
- * I/O port BAR1 provides a window to a mirror of PCI config space located
- * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
- * note it for future reference.
- */
-
-#define PCI_VENDOR_ID_NVIDIA                    0x10de
-
-/*
- * Nvidia has several different methods to get to config space, the
- * nouveu project has several of these documented here:
- * https://github.com/pathscale/envytools/tree/master/hwdocs
- *
- * The first quirk is actually not documented in envytools and is found
- * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
- * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
- * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
- * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
- * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
- * is written for a write to 0x3d4.  The BAR0 offset is then accessible
- * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
- * that use the I/O port BAR5 window but it doesn't hurt to leave it.
- */
-enum {
-    NV_3D0_NONE = 0,
-    NV_3D0_SELECT,
-    NV_3D0_WINDOW,
-    NV_3D0_READ,
-    NV_3D0_WRITE,
-};
-
-static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
-                                           hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                                  addr + quirk->data.base_offset, size);
-
-    if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) {
-        data = vfio_pci_read_config(pdev, quirk->data.address_val, size);
-        trace_vfio_nvidia_3d0_quirk_read(size, data);
-    }
-
-    quirk->data.flags = NV_3D0_NONE;
-
-    return data;
-}
-
-static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
-                                        uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-
-    switch (quirk->data.flags) {
-    case NV_3D0_NONE:
-        if (addr == quirk->data.address_offset && data == 0x338) {
-            quirk->data.flags = NV_3D0_SELECT;
-        }
-        break;
-    case NV_3D0_SELECT:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset &&
-            (data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags = NV_3D0_WINDOW;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        }
-        break;
-    case NV_3D0_WINDOW:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.address_offset) {
-            if (data == 0x538) {
-                quirk->data.flags = NV_3D0_READ;
-            } else if (data == 0x738) {
-                quirk->data.flags = NV_3D0_WRITE;
-            }
-        }
-        break;
-    case NV_3D0_WRITE:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset) {
-            vfio_pci_write_config(pdev, quirk->data.address_val, data, size);
-            trace_vfio_nvidia_3d0_quirk_write(data, size);
-            return;
-        }
-        break;
-    }
-
-    vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
-    .read = vfio_nvidia_3d0_quirk_read,
-    .write = vfio_nvidia_3d0_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA ||
-        !vdev->bars[1].region.size) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.base_offset = 0x10;
-    quirk->data.address_offset = 4;
-    quirk->data.address_size = 2;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.data_offset = 0;
-    quirk->data.data_size = 4;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk,
-                          quirk, "vfio-nvidia-3d0-quirk", 6);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                quirk->data.base_offset, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    trace_vfio_vga_probe_nvidia_3d0_quirk(vdev->vbasedev.name);
-}
-
-/*
- * The second quirk is documented in envytools.  The I/O port BAR5 is just
- * a set of address/data ports to the MMIO BARs.  The BAR we care about is
- * again BAR0.  This backdoor is apparently a bit newer than the one above
- * so we need to not only trap 256 bytes @0x1800, but all of PCI config
- * space, including extended space is available at the 4k @0x88000.
- */
-enum {
-    NV_BAR5_ADDRESS = 0x1,
-    NV_BAR5_ENABLE = 0x2,
-    NV_BAR5_MASTER = 0x4,
-    NV_BAR5_VALID = 0x7,
-};
-
-static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr,
-                                                uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-
-    switch (addr) {
-    case 0x0:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_MASTER;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_MASTER;
-        }
-        break;
-    case 0x4:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_ENABLE;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_ENABLE;
-        }
-        break;
-    case 0x8:
-        if (quirk->data.flags & NV_BAR5_MASTER) {
-            if ((data & ~0xfff) == 0x88000) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xfff;
-            } else if ((data & ~0xff) == 0x1800) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xff;
-            } else {
-                quirk->data.flags &= ~NV_BAR5_ADDRESS;
-            }
-        }
-        break;
-    }
-
-    vfio_generic_window_quirk_write(opaque, addr, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_nvidia_bar5_window_quirk_write,
-    .valid.min_access_size = 4,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_nvidia_bar5_window_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 5 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID;
-    quirk->data.address_offset = 0x8;
-    quirk->data.address_size = 0; /* actually 4, but avoids generic code */
-    quirk->data.data_offset = 0xc;
-    quirk->data.data_size = 4;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_nvidia_bar5_window_quirk, quirk,
-                          "vfio-nvidia-bar5-window-quirk", 16);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                                        0, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar5_window_quirk(vdev->vbasedev.name);
-}
-
-static void vfio_nvidia_88000_quirk_write(void *opaque, hwaddr addr,
-                                          uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIOPCIDevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-
-    vfio_generic_quirk_write(opaque, addr, data, size);
-
-    /*
-     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
-     * MSI capability ID register.  Both the ID and next register are
-     * read-only, so we allow writes covering either of those to real hw.
-     * NB - only fixed for the 0x88000 MMIO window.
-     */
-    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
-        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
-        vfio_region_write(&vdev->bars[quirk->data.bar].region,
-                          addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_nvidia_88000_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_nvidia_88000_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-/*
- * Finally, BAR0 itself.  We want to redirect any accesses to either
- * 0x1800 or 0x88000 through the PCI config space access functions.
- *
- * NB - quirk at a page granularity or else they don't seem to work when
- *      BARs are mmap'd
- *
- * Here's offset 0x88000...
- */
-static void vfio_probe_nvidia_bar0_88000_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-    uint16_t vendor, class;
-
-    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
-
-    if (nr != 0 || vendor != PCI_VENDOR_ID_NVIDIA ||
-        class != PCI_CLASS_DISPLAY_VGA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x88000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_88000_quirk,
-                          quirk, "vfio-nvidia-bar0-88000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar0_88000_quirk(vdev->vbasedev.name);
-}
-
-/*
- * And here's the same for BAR0 offset 0x1800...
- */
-static void vfio_probe_nvidia_bar0_1800_quirk(VFIOPCIDevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 0 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    /* Log the chipset ID */
-    trace_vfio_probe_nvidia_bar0_1800_quirk_id(
-            (unsigned int)(vfio_region_read(&vdev->bars[0].region, 0, 4) >> 20)
-            & 0xff);
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-nvidia-bar0-1800-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].region.mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    trace_vfio_probe_nvidia_bar0_1800_quirk(vdev->vbasedev.name);
-}
-
-/*
- * TODO - Some Nvidia devices provide config access to their companion HDA
- * device and even to their parent bridge via these config space mirrors.
- * Add quirks for those regions.
- */
-
-/*
- * Common quirk probe entry points.
- */
-static void vfio_vga_quirk_setup(VFIOPCIDevice *vdev)
-{
-    vfio_vga_probe_ati_3c3_quirk(vdev);
-    vfio_vga_probe_nvidia_3d0_quirk(vdev);
-}
-
-static void vfio_vga_quirk_teardown(VFIOPCIDevice *vdev)
-{
-    VFIOQuirk *quirk;
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
-        QLIST_FOREACH(quirk, &vdev->vga.region[i].quirks, next) {
-            memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
-        }
-    }
-}
-
-static void vfio_vga_quirk_free(VFIOPCIDevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
-        while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
-            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
-            object_unparent(OBJECT(&quirk->mem));
-            QLIST_REMOVE(quirk, next);
-            g_free(quirk);
-        }
-    }
-}
-
-static void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr)
-{
-    vfio_probe_ati_bar4_window_quirk(vdev, nr);
-    vfio_probe_ati_bar2_4000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
-    vfio_probe_rtl8168_bar2_window_quirk(vdev, nr);
-}
-
-static void vfio_bar_quirk_teardown(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-    VFIOQuirk *quirk;
-
-    QLIST_FOREACH(quirk, &bar->quirks, next) {
-        memory_region_del_subregion(&bar->region.mem, &quirk->mem);
-    }
-}
-
-static void vfio_bar_quirk_free(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-
-    while (!QLIST_EMPTY(&bar->quirks)) {
-        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
-        object_unparent(OBJECT(&quirk->mem));
-        QLIST_REMOVE(quirk, next);
-        g_free(quirk);
-    }
-}
-
-/*
  * PCI config space
  */
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
@@ -2082,9 +1077,8 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
         ret = pread(vdev->vbasedev.fd, &phys_val, len,
                     vdev->config_offset + addr);
         if (ret != len) {
-            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
-                         __func__, vdev->host.domain, vdev->host.bus,
-                         vdev->host.slot, vdev->host.function, addr, len);
+            error_report("%s(%s, 0x%x, 0x%x) failed: %m",
+                         __func__, vdev->vbasedev.name, addr, len);
             return -errno;
         }
         phys_val = le32_to_cpu(phys_val);
@@ -2097,8 +1091,8 @@ static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
     return val;
 }
 
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len)
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     uint32_t val_le = cpu_to_le32(val);
@@ -2108,9 +1102,8 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
     /* Write everything to VFIO, let it filter out what we can't write */
     if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
                 != len) {
-        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
-                     __func__, vdev->host.domain, vdev->host.bus,
-                     vdev->host.slot, vdev->host.function, addr, val, len);
+        error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
+                     __func__, vdev->vbasedev.name, addr, val, len);
     }
 
     /* MSI/MSI-X Enabling/Disabling */
@@ -2124,11 +1117,11 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
 
         if (!was_enabled) {
             if (is_enabled) {
-                vfio_enable_msi(vdev);
+                vfio_msi_enable(vdev);
             }
         } else {
             if (!is_enabled) {
-                vfio_disable_msi(vdev);
+                vfio_msi_disable(vdev);
             } else {
                 vfio_update_msi(vdev);
             }
@@ -2142,9 +1135,9 @@ static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
         is_enabled = msix_enabled(pdev);
 
         if (!was_enabled && is_enabled) {
-            vfio_enable_msix(vdev);
+            vfio_msix_enable(vdev);
         } else if (was_enabled && !is_enabled) {
-            vfio_disable_msix(vdev);
+            vfio_msix_disable(vdev);
         }
     } else {
         /* Write everything to QEMU to keep emulated bits correct */
@@ -2163,17 +1156,17 @@ static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
      * disable MSI/X and then cleanup by disabling INTx.
      */
     if (vdev->interrupt == VFIO_INT_MSIX) {
-        vfio_disable_msix(vdev);
+        vfio_msix_disable(vdev);
     } else if (vdev->interrupt == VFIO_INT_MSI) {
-        vfio_disable_msi(vdev);
+        vfio_msi_disable(vdev);
     }
 
     if (vdev->interrupt == VFIO_INT_INTx) {
-        vfio_disable_intx(vdev);
+        vfio_intx_disable(vdev);
     }
 }
 
-static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
+static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos)
 {
     uint16_t ctrl;
     bool msi_64bit, msi_maskbit;
@@ -2189,7 +1182,7 @@ static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
     msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
     entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
 
-    trace_vfio_setup_msi(vdev->vbasedev.name, pos);
+    trace_vfio_msi_setup(vdev->vbasedev.name, pos);
 
     ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
     if (ret < 0) {
@@ -2204,6 +1197,74 @@ static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
     return 0;
 }
 
+static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
+{
+    off_t start, end;
+    VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
+
+    /*
+     * We expect to find a single mmap covering the whole BAR, anything else
+     * means it's either unsupported or already setup.
+     */
+    if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
+        region->size != region->mmaps[0].size) {
+        return;
+    }
+
+    /* MSI-X table start and end aligned to host page size */
+    start = vdev->msix->table_offset & qemu_real_host_page_mask;
+    end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
+                               (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
+
+    /*
+     * Does the MSI-X table cover the beginning of the BAR?  The whole BAR?
+     * NB - Host page size is necessarily a power of two and so is the PCI
+     * BAR (not counting EA yet), therefore if we have host page aligned
+     * @start and @end, then any remainder of the BAR before or after those
+     * must be at least host page sized and therefore mmap'able.
+     */
+    if (!start) {
+        if (end >= region->size) {
+            region->nr_mmaps = 0;
+            g_free(region->mmaps);
+            region->mmaps = NULL;
+            trace_vfio_msix_fixup(vdev->vbasedev.name,
+                                  vdev->msix->table_bar, 0, 0);
+        } else {
+            region->mmaps[0].offset = end;
+            region->mmaps[0].size = region->size - end;
+            trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+        }
+
+    /* Maybe it's aligned at the end of the BAR */
+    } else if (end >= region->size) {
+        region->mmaps[0].size = start;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+
+    /* Otherwise it must split the BAR */
+    } else {
+        region->nr_mmaps = 2;
+        region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
+
+        memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
+
+        region->mmaps[0].size = start;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+
+        region->mmaps[1].offset = end;
+        region->mmaps[1].size = region->size - end;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[1].offset,
+                              region->mmaps[1].offset + region->mmaps[1].size);
+    }
+}
+
 /*
  * We don't have any control over how pci_add_capability() inserts
  * capabilities into the chain.  In order to setup MSI-X we need a
@@ -2212,12 +1273,13 @@ static int vfio_setup_msi(VFIOPCIDevice *vdev, int pos)
  * need to first look for where the MSI-X table lives.  So we
  * unfortunately split MSI-X setup across two functions.
  */
-static int vfio_early_setup_msix(VFIOPCIDevice *vdev)
+static int vfio_msix_early_setup(VFIOPCIDevice *vdev)
 {
     uint8_t pos;
     uint16_t ctrl;
     uint32_t table, pba;
     int fd = vdev->vbasedev.fd;
+    VFIOMSIXInfo *msix;
 
     pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
     if (!pos) {
@@ -2225,7 +1287,7 @@ static int vfio_early_setup_msix(VFIOPCIDevice *vdev)
     }
 
     if (pread(fd, &ctrl, sizeof(ctrl),
-              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+              vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
         return -errno;
     }
 
@@ -2243,56 +1305,55 @@ static int vfio_early_setup_msix(VFIOPCIDevice *vdev)
     table = le32_to_cpu(table);
     pba = le32_to_cpu(pba);
 
-    vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
-    vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+    msix = g_malloc0(sizeof(*msix));
+    msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
 
     /*
      * Test the size of the pba_offset variable and catch if it extends outside
      * of the specified BAR. If it is the case, we need to apply a hardware
      * specific quirk if the device is known or we have a broken configuration.
      */
-    if (vdev->msix->pba_offset >=
-        vdev->bars[vdev->msix->pba_bar].region.size) {
-
-        PCIDevice *pdev = &vdev->pdev;
-        uint16_t vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-        uint16_t device = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
+    if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
         /*
          * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
          * adapters. The T5 hardware returns an incorrect value of 0x8000 for
          * the VF PBA offset while the BAR itself is only 8k. The correct value
          * is 0x1000, so we hard code that here.
          */
-        if (vendor == PCI_VENDOR_ID_CHELSIO && (device & 0xff00) == 0x5800) {
-            vdev->msix->pba_offset = 0x1000;
+        if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
+            (vdev->device_id & 0xff00) == 0x5800) {
+            msix->pba_offset = 0x1000;
         } else {
             error_report("vfio: Hardware reports invalid configuration, "
                          "MSIX PBA outside of specified BAR");
+            g_free(msix);
             return -EINVAL;
         }
     }
 
-    trace_vfio_early_setup_msix(vdev->vbasedev.name, pos,
-                                vdev->msix->table_bar,
-                                vdev->msix->table_offset,
-                                vdev->msix->entries);
+    trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
+                                msix->table_offset, msix->entries);
+    vdev->msix = msix;
+
+    vfio_pci_fixup_msix_region(vdev);
 
     return 0;
 }
 
-static int vfio_setup_msix(VFIOPCIDevice *vdev, int pos)
+static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos)
 {
     int ret;
 
+    vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
+                                    sizeof(unsigned long));
     ret = msix_init(&vdev->pdev, vdev->msix->entries,
-                    &vdev->bars[vdev->msix->table_bar].region.mem,
+                    vdev->bars[vdev->msix->table_bar].region.mem,
                     vdev->msix->table_bar, vdev->msix->table_offset,
-                    &vdev->bars[vdev->msix->pba_bar].region.mem,
+                    vdev->bars[vdev->msix->pba_bar].region.mem,
                     vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
     if (ret < 0) {
         if (ret == -ENOTSUP) {
@@ -2302,6 +1363,24 @@ static int vfio_setup_msix(VFIOPCIDevice *vdev, int pos)
         return ret;
     }
 
+    /*
+     * The PCI spec suggests that devices provide additional alignment for
+     * MSI-X structures and avoid overlapping non-MSI-X related registers.
+     * For an assigned device, this hopefully means that emulation of MSI-X
+     * structures does not affect the performance of the device.  If devices
+     * fail to provide that alignment, a significant performance penalty may
+     * result, for instance Mellanox MT27500 VFs:
+     * http://www.spinics.net/lists/kvm/msg125881.html
+     *
+     * The PBA is simply not that important for such a serious regression and
+     * most drivers do not appear to look at it.  The solution for this is to
+     * disable the PBA MemoryRegion unless it's being used.  We disable it
+     * here and only enable it if a masked vector fires through QEMU.  As the
+     * vector-use notifier is called, which occurs on unmask, we test whether
+     * PBA emulation is needed and again disable if not.
+     */
+    memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
+
     return 0;
 }
 
@@ -2311,8 +1390,9 @@ static void vfio_teardown_msi(VFIOPCIDevice *vdev)
 
     if (vdev->msix) {
         msix_uninit(&vdev->pdev,
-                    &vdev->bars[vdev->msix->table_bar].region.mem,
-                    &vdev->bars[vdev->msix->pba_bar].region.mem);
+                    vdev->bars[vdev->msix->table_bar].region.mem,
+                    vdev->bars[vdev->msix->pba_bar].region.mem);
+        g_free(vdev->msix->pending);
     }
 }
 
@@ -2324,71 +1404,23 @@ static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
     int i;
 
     for (i = 0; i < PCI_ROM_SLOT; i++) {
-        VFIOBAR *bar = &vdev->bars[i];
-
-        if (!bar->region.size) {
-            continue;
-        }
-
-        memory_region_set_enabled(&bar->region.mmap_mem, enabled);
-        if (vdev->msix && vdev->msix->table_bar == i) {
-            memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
-        }
+        vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
     }
 }
 
-static void vfio_unregister_bar(VFIOPCIDevice *vdev, int nr)
+static void vfio_bar_setup(VFIOPCIDevice *vdev, int nr)
 {
     VFIOBAR *bar = &vdev->bars[nr];
 
-    if (!bar->region.size) {
-        return;
-    }
-
-    vfio_bar_quirk_teardown(vdev, nr);
-
-    memory_region_del_subregion(&bar->region.mem, &bar->region.mmap_mem);
-
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        memory_region_del_subregion(&bar->region.mem, &vdev->msix->mmap_mem);
-    }
-}
-
-static void vfio_unmap_bar(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-
-    if (!bar->region.size) {
-        return;
-    }
-
-    vfio_bar_quirk_free(vdev, nr);
-
-    munmap(bar->region.mmap, memory_region_size(&bar->region.mmap_mem));
-
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
-    }
-}
-
-static void vfio_map_bar(VFIOPCIDevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-    uint64_t size = bar->region.size;
-    char name[64];
     uint32_t pci_bar;
     uint8_t type;
     int ret;
 
     /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
-    if (!size) {
+    if (!bar->region.size) {
         return;
     }
 
-    snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function, nr);
-
     /* Determine what type of BAR this is for registration */
     ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
                 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
@@ -2403,102 +1435,78 @@ static void vfio_map_bar(VFIOPCIDevice *vdev, int nr)
     type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
                                     ~PCI_BASE_ADDRESS_MEM_MASK);
 
-    /* A "slow" read/write mapping underlies all BARs */
-    memory_region_init_io(&bar->region.mem, OBJECT(vdev), &vfio_region_ops,
-                          bar, name, size);
-    pci_register_bar(&vdev->pdev, nr, type, &bar->region.mem);
-
-    /*
-     * We can't mmap areas overlapping the MSIX vector table, so we
-     * potentially insert a direct-mapped subregion before and after it.
-     */
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        size = vdev->msix->table_offset & qemu_real_host_page_mask;
-    }
-
-    strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
-    if (vfio_mmap_region(OBJECT(vdev), &bar->region, &bar->region.mem,
-                      &bar->region.mmap_mem, &bar->region.mmap,
-                      size, 0, name)) {
-        error_report("%s unsupported. Performance may be slow", name);
-    }
-
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        uint64_t start;
-
-        start = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
-                                     (vdev->msix->entries *
-                                      PCI_MSIX_ENTRY_SIZE));
-
-        size = start < bar->region.size ? bar->region.size - start : 0;
-        strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
-        /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
-        if (vfio_mmap_region(OBJECT(vdev), &bar->region, &bar->region.mem,
-                          &vdev->msix->mmap_mem,
-                          &vdev->msix->mmap, size, start, name)) {
-            error_report("%s unsupported. Performance may be slow", name);
-        }
+    if (vfio_region_mmap(&bar->region)) {
+        error_report("Failed to mmap %s BAR %d. Performance may be slow",
+                     vdev->vbasedev.name, nr);
     }
 
     vfio_bar_quirk_setup(vdev, nr);
+
+    pci_register_bar(&vdev->pdev, nr, type, bar->region.mem);
 }
 
-static void vfio_map_bars(VFIOPCIDevice *vdev)
+static void vfio_bars_setup(VFIOPCIDevice *vdev)
 {
     int i;
 
     for (i = 0; i < PCI_ROM_SLOT; i++) {
-        vfio_map_bar(vdev, i);
+        vfio_bar_setup(vdev, i);
     }
 
-    if (vdev->has_vga) {
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
+    if (vdev->vga) {
+        memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
                               OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_MEM],
+                              &vdev->vga->region[QEMU_PCI_VGA_MEM],
                               "vfio-vga-mmio@0xa0000",
                               QEMU_PCI_VGA_MEM_SIZE);
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
+        memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
                               OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_IO_LO],
+                              &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
                               "vfio-vga-io@0x3b0",
                               QEMU_PCI_VGA_IO_LO_SIZE);
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
+        memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
                               OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_IO_HI],
+                              &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
                               "vfio-vga-io@0x3c0",
                               QEMU_PCI_VGA_IO_HI_SIZE);
 
-        pci_register_vga(&vdev->pdev, &vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
-                         &vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
-                         &vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
+        pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
+                         &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
+                         &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
         vfio_vga_quirk_setup(vdev);
     }
 }
 
-static void vfio_unregister_bars(VFIOPCIDevice *vdev)
+static void vfio_bars_exit(VFIOPCIDevice *vdev)
 {
     int i;
 
     for (i = 0; i < PCI_ROM_SLOT; i++) {
-        vfio_unregister_bar(vdev, i);
+        vfio_bar_quirk_exit(vdev, i);
+        vfio_region_exit(&vdev->bars[i].region);
     }
 
-    if (vdev->has_vga) {
-        vfio_vga_quirk_teardown(vdev);
+    if (vdev->vga) {
         pci_unregister_vga(&vdev->pdev);
+        vfio_vga_quirk_exit(vdev);
     }
 }
 
-static void vfio_unmap_bars(VFIOPCIDevice *vdev)
+static void vfio_bars_finalize(VFIOPCIDevice *vdev)
 {
     int i;
 
     for (i = 0; i < PCI_ROM_SLOT; i++) {
-        vfio_unmap_bar(vdev, i);
+        vfio_bar_quirk_finalize(vdev, i);
+        vfio_region_finalize(&vdev->bars[i].region);
     }
 
-    if (vdev->has_vga) {
-        vfio_vga_quirk_free(vdev);
+    if (vdev->vga) {
+        vfio_vga_quirk_finalize(vdev);
+        for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+            object_unparent(OBJECT(&vdev->vga->region[i].mem));
+        }
+        g_free(vdev->vga);
     }
 }
 
@@ -2507,10 +1515,11 @@ static void vfio_unmap_bars(VFIOPCIDevice *vdev)
  */
 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
 {
-    uint8_t tmp, next = 0xff;
+    uint8_t tmp;
+    uint16_t next = PCI_CONFIG_SPACE_SIZE;
 
     for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
-         tmp = pdev->config[tmp + 1]) {
+         tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
         if (tmp > pos && tmp < next) {
             next = tmp;
         }
@@ -2563,13 +1572,38 @@ static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size)
     }
 
     if (!pci_bus_is_express(vdev->pdev.bus)) {
+        PCIBus *bus = vdev->pdev.bus;
+        PCIDevice *bridge;
+
         /*
-         * Use express capability as-is on PCI bus.  It doesn't make much
-         * sense to even expose, but some drivers (ex. tg3) depend on it
-         * and guests don't seem to be particular about it.  We'll need
-         * to revist this or force express devices to express buses if we
-         * ever expose an IOMMU to the guest.
+         * Traditionally PCI device assignment exposes the PCIe capability
+         * as-is on non-express buses.  The reason being that some drivers
+         * simply assume that it's there, for example tg3.  However when
+         * we're running on a native PCIe machine type, like Q35, we need
+         * to hide the PCIe capability.  The reason for this is twofold;
+         * first Windows guests get a Code 10 error when the PCIe capability
+         * is exposed in this configuration.  Therefore express devices won't
+         * work at all unless they're attached to express buses in the VM.
+         * Second, a native PCIe machine introduces the possibility of fine
+         * granularity IOMMUs supporting both translation and isolation.
+         * Guest code to discover the IOMMU visibility of a device, such as
+         * IOMMU grouping code on Linux, is very aware of device types and
+         * valid transitions between bus types.  An express device on a non-
+         * express bus is not a valid combination on bare metal systems.
+         *
+         * Drivers that require a PCIe capability to make the device
+         * functional are simply going to need to have their devices placed
+         * on a PCIe bus in the VM.
          */
+        while (!pci_bus_is_root(bus)) {
+            bridge = pci_bridge_get_device(bus);
+            bus = bridge->bus;
+        }
+
+        if (pci_bus_is_express(bus)) {
+            return 0;
+        }
+
     } else if (pci_bus_is_root(vdev->pdev.bus)) {
         /*
          * On a Root Complex bus Endpoints become Root Complex Integrated
@@ -2674,7 +1708,7 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
     int ret;
 
     cap_id = pdev->config[pos];
-    next = pdev->config[pos + 1];
+    next = pdev->config[pos + PCI_CAP_LIST_NEXT];
 
     /*
      * If it becomes important to configure capabilities to their actual
@@ -2688,7 +1722,7 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
      * pci_add_capability always inserts the new capability at the head
      * of the chain.  Therefore to end up with a chain that matches the
      * physical device, we insert from the end by making this recursive.
-     * This is also why we pre-caclulate size above as cached config space
+     * This is also why we pre-calculate size above as cached config space
      * will be changed as we unwind the stack.
      */
     if (next) {
@@ -2704,18 +1738,18 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
     }
 
     /* Use emulated next pointer to allow dropping caps */
-    pci_set_byte(vdev->emulated_config_bits + pos + 1, 0xff);
+    pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
 
     switch (cap_id) {
     case PCI_CAP_ID_MSI:
-        ret = vfio_setup_msi(vdev, pos);
+        ret = vfio_msi_setup(vdev, pos);
         break;
     case PCI_CAP_ID_EXP:
         vfio_check_pcie_flr(vdev, pos);
         ret = vfio_setup_pcie_cap(vdev, pos, size);
         break;
     case PCI_CAP_ID_MSIX:
-        ret = vfio_setup_msix(vdev, pos);
+        ret = vfio_msix_setup(vdev, pos);
         break;
     case PCI_CAP_ID_PM:
         vfio_check_pm_reset(vdev, pos);
@@ -2732,9 +1766,8 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
     }
 
     if (ret < 0) {
-        error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
-                     "0x%x[0x%x]@0x%x: %d", vdev->host.domain,
-                     vdev->host.bus, vdev->host.slot, vdev->host.function,
+        error_report("vfio: %s Error adding PCI capability "
+                     "0x%x[0x%x]@0x%x: %d", vdev->vbasedev.name,
                      cap_id, size, pos, ret);
         return ret;
     }
@@ -2793,14 +1826,17 @@ static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
 
 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
 {
-    vfio_enable_intx(vdev);
+    vfio_intx_enable(vdev);
 }
 
-static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
-                                PCIHostDeviceAddress *host2)
+static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
 {
-    return (host1->domain == host2->domain && host1->bus == host2->bus &&
-            host1->slot == host2->slot && host1->function == host2->function);
+    char tmp[13];
+
+    sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
+            addr->bus, addr->slot, addr->function);
+
+    return (strcmp(tmp, name) == 0);
 }
 
 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
@@ -2825,9 +1861,8 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
     if (ret && errno != ENOSPC) {
         ret = -errno;
         if (!vdev->has_pm_reset) {
-            error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
-                         "no available reset mechanism.", vdev->host.domain,
-                         vdev->host.bus, vdev->host.slot, vdev->host.function);
+            error_report("vfio: Cannot reset device %s, "
+                         "no available reset mechanism.", vdev->vbasedev.name);
         }
         goto out_single;
     }
@@ -2860,7 +1895,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
         trace_vfio_pci_hot_reset_dep_devices(host.domain,
                 host.bus, host.slot, host.function, devices[i].group_id);
 
-        if (vfio_pci_host_match(&host, &vdev->host)) {
+        if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
             continue;
         }
 
@@ -2886,7 +1921,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
                 continue;
             }
             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
-            if (vfio_pci_host_match(&host, &tmp->host)) {
+            if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
                 if (single) {
                     ret = -EINVAL;
                     goto out_single;
@@ -2948,7 +1983,7 @@ out:
         host.slot = PCI_SLOT(devices[i].devfn);
         host.function = PCI_FUNC(devices[i].devfn);
 
-        if (vfio_pci_host_match(&host, &vdev->host)) {
+        if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
             continue;
         }
 
@@ -2967,7 +2002,7 @@ out:
                 continue;
             }
             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
-            if (vfio_pci_host_match(&host, &tmp->host)) {
+            if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
                 vfio_pci_post_reset(tmp);
                 break;
             }
@@ -3017,13 +2052,59 @@ static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
 static VFIODeviceOps vfio_pci_ops = {
     .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
     .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
-    .vfio_eoi = vfio_eoi,
+    .vfio_eoi = vfio_intx_eoi,
 };
 
+int vfio_populate_vga(VFIOPCIDevice *vdev)
+{
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    struct vfio_region_info *reg_info;
+    int ret;
+
+    if (vbasedev->num_regions > VFIO_PCI_VGA_REGION_INDEX) {
+        ret = vfio_get_region_info(vbasedev,
+                                   VFIO_PCI_VGA_REGION_INDEX, &reg_info);
+        if (ret) {
+            return ret;
+        }
+
+        if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
+            !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
+            reg_info->size < 0xbffff + 1) {
+            error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
+                         (unsigned long)reg_info->flags,
+                         (unsigned long)reg_info->size);
+            g_free(reg_info);
+            return -EINVAL;
+        }
+
+        vdev->vga = g_new0(VFIOVGA, 1);
+
+        vdev->vga->fd_offset = reg_info->offset;
+        vdev->vga->fd = vdev->vbasedev.fd;
+
+        g_free(reg_info);
+
+        vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
+        vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
+        QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
+
+        vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
+        vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
+        QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
+
+        vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
+        vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
+        QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
+    }
+
+    return 0;
+}
+
 static int vfio_populate_device(VFIOPCIDevice *vdev)
 {
     VFIODevice *vbasedev = &vdev->vbasedev;
-    struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
+    struct vfio_region_info *reg_info;
     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
     int i, ret = -1;
 
@@ -3045,85 +2126,47 @@ static int vfio_populate_device(VFIOPCIDevice *vdev)
     }
 
     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        reg_info.index = i;
+        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+
+        ret = vfio_region_setup(OBJECT(vdev), vbasedev,
+                                &vdev->bars[i].region, i, name);
+        g_free(name);
 
-        ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
         if (ret) {
             error_report("vfio: Error getting region %d info: %m", i);
             goto error;
         }
 
-        trace_vfio_populate_device_region(vbasedev->name, i,
-                                          (unsigned long)reg_info.size,
-                                          (unsigned long)reg_info.offset,
-                                          (unsigned long)reg_info.flags);
-
-        vdev->bars[i].region.vbasedev = vbasedev;
-        vdev->bars[i].region.flags = reg_info.flags;
-        vdev->bars[i].region.size = reg_info.size;
-        vdev->bars[i].region.fd_offset = reg_info.offset;
-        vdev->bars[i].region.nr = i;
         QLIST_INIT(&vdev->bars[i].quirks);
     }
 
-    reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
-
-    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+    ret = vfio_get_region_info(vbasedev,
+                               VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
     if (ret) {
         error_report("vfio: Error getting config info: %m");
         goto error;
     }
 
     trace_vfio_populate_device_config(vdev->vbasedev.name,
-                                      (unsigned long)reg_info.size,
-                                      (unsigned long)reg_info.offset,
-                                      (unsigned long)reg_info.flags);
+                                      (unsigned long)reg_info->size,
+                                      (unsigned long)reg_info->offset,
+                                      (unsigned long)reg_info->flags);
 
-    vdev->config_size = reg_info.size;
+    vdev->config_size = reg_info->size;
     if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
         vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
     }
-    vdev->config_offset = reg_info.offset;
+    vdev->config_offset = reg_info->offset;
 
-    if ((vdev->features & VFIO_FEATURE_ENABLE_VGA) &&
-        vbasedev->num_regions > VFIO_PCI_VGA_REGION_INDEX) {
-        struct vfio_region_info vga_info = {
-            .argsz = sizeof(vga_info),
-            .index = VFIO_PCI_VGA_REGION_INDEX,
-         };
+    g_free(reg_info);
 
-        ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_REGION_INFO, &vga_info);
+    if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
+        ret = vfio_populate_vga(vdev);
         if (ret) {
             error_report(
                 "vfio: Device does not support requested feature x-vga");
             goto error;
         }
-
-        if (!(vga_info.flags & VFIO_REGION_INFO_FLAG_READ) ||
-            !(vga_info.flags & VFIO_REGION_INFO_FLAG_WRITE) ||
-            vga_info.size < 0xbffff + 1) {
-            error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
-                         (unsigned long)vga_info.flags,
-                         (unsigned long)vga_info.size);
-            goto error;
-        }
-
-        vdev->vga.fd_offset = vga_info.offset;
-        vdev->vga.fd = vdev->vbasedev.fd;
-
-        vdev->vga.region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_MEM].quirks);
-
-        vdev->vga.region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].quirks);
-
-        vdev->vga.region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks);
-
-        vdev->has_vga = true;
     }
 
     irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
@@ -3148,11 +2191,8 @@ error:
 static void vfio_put_device(VFIOPCIDevice *vdev)
 {
     g_free(vdev->vbasedev.name);
-    if (vdev->msix) {
-        object_unparent(OBJECT(&vdev->msix->mmap_mem));
-        g_free(vdev->msix);
-        vdev->msix = NULL;
-    }
+    g_free(vdev->msix);
+
     vfio_put_base_device(&vdev->vbasedev);
 }
 
@@ -3173,10 +2213,7 @@ static void vfio_err_notifier_handler(void *opaque)
      * guest to contain the error.
      */
 
-    error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected.  "
-                 "Please collect any data possible and then kill the guest",
-                 __func__, vdev->host.domain, vdev->host.bus,
-                 vdev->host.slot, vdev->host.function);
+    error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
 
     vm_stop(RUN_STATE_INTERNAL_ERROR);
 }
@@ -3352,203 +2389,48 @@ static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
     vdev->req_enabled = false;
 }
 
-/*
- * AMD Radeon PCI config reset, based on Linux:
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running()
- *   drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc()
- *   drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock()
- * IDs: include/drm/drm_pciids.h
- * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0
- *
- * Bonaire and Hawaii GPUs do not respond to a bus reset.  This is a bug in the
- * hardware that should be fixed on future ASICs.  The symptom of this is that
- * once the accerlated driver loads, Windows guests will bsod on subsequent
- * attmpts to load the driver, such as after VM reset or shutdown/restart.  To
- * work around this, we do an AMD specific PCI config reset, followed by an SMC
- * reset.  The PCI config reset only works if SMC firmware is running, so we
- * have a dependency on the state of the device as to whether this reset will
- * be effective.  There are still cases where we won't be able to kick the
- * device into working, but this greatly improves the usability overall.  The
- * config reset magic is relatively common on AMD GPUs, but the setup and SMC
- * poking is largely ASIC specific.
- */
-static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev)
-{
-    uint32_t clk, pc_c;
-
-    /*
-     * Registers 200h and 204h are index and data registers for acessing
-     * indirect configuration registers within the device.
-     */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
-    clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4);
-    pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-
-    return (!(clk & 1) && (0x20100 <= pc_c));
-}
-
-/*
- * The scope of a config reset is controlled by a mode bit in the misc register
- * and a fuse, exposed as a bit in another register.  The fuse is the default
- * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula
- * scope = !(misc ^ fuse), where the resulting scope is defined the same as
- * the fuse.  A truth table therefore tells us that if misc == fuse, we need
- * to flip the value of the bit in the misc register.
- */
-static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev)
-{
-    uint32_t misc, fuse;
-    bool a, b;
-
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4);
-    fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    b = fuse & 64;
-
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4);
-    misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    a = misc & 2;
-
-    if (a == b) {
-        vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4);
-        vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */
-    }
-}
-
-static int vfio_radeon_reset(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    int i, ret = 0;
-    uint32_t data;
-
-    /* Defer to a kernel implemented reset */
-    if (vdev->vbasedev.reset_works) {
-        return -ENODEV;
-    }
-
-    /* Enable only memory BAR access */
-    vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2);
-
-    /* Reset only works if SMC firmware is loaded and running */
-    if (!vfio_radeon_smc_is_running(vdev)) {
-        ret = -EINVAL;
-        goto out;
-    }
-
-    /* Make sure only the GFX function is reset */
-    vfio_radeon_set_gfx_only_reset(vdev);
-
-    /* AMD PCI config reset */
-    vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4);
-    usleep(100);
-
-    /* Read back the memory size to make sure we're out of reset */
-    for (i = 0; i < 100000; i++) {
-        if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) {
-            break;
-        }
-        usleep(1);
-    }
-
-    /* Reset SMC */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4);
-    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    data |= 1;
-    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
-
-    /* Disable SMC clock */
-    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
-    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
-    data |= 1;
-    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
-
-out:
-    /* Restore PCI command register */
-    vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2);
-
-    return ret;
-}
-
-static void vfio_setup_resetfn(VFIOPCIDevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t vendor, device;
-
-    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    device = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
-    switch (vendor) {
-    case 0x1002:
-        switch (device) {
-        /* Bonaire */
-        case 0x6649: /* Bonaire [FirePro W5100] */
-        case 0x6650:
-        case 0x6651:
-        case 0x6658: /* Bonaire XTX [Radeon R7 260X] */
-        case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */
-        case 0x665d: /* Bonaire [Radeon R7 200 Series] */
-        /* Hawaii */
-        case 0x67A0: /* Hawaii XT GL [FirePro W9100] */
-        case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */
-        case 0x67A2:
-        case 0x67A8:
-        case 0x67A9:
-        case 0x67AA:
-        case 0x67B0: /* Hawaii XT [Radeon R9 290X] */
-        case 0x67B1: /* Hawaii PRO [Radeon R9 290] */
-        case 0x67B8:
-        case 0x67B9:
-        case 0x67BA:
-        case 0x67BE:
-            vdev->resetfn = vfio_radeon_reset;
-            break;
-        }
-        break;
-    }
-}
-
 static int vfio_initfn(PCIDevice *pdev)
 {
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
     VFIODevice *vbasedev_iter;
     VFIOGroup *group;
-    char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
+    char *tmp, group_path[PATH_MAX], *group_name;
     ssize_t len;
     struct stat st;
     int groupid;
     int ret;
 
-    /* Check that the host device exists */
-    snprintf(path, sizeof(path),
-             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function);
-    if (stat(path, &st) < 0) {
-        error_report("vfio: error: no such host device: %s", path);
+    if (!vdev->vbasedev.sysfsdev) {
+        vdev->vbasedev.sysfsdev =
+            g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
+                            vdev->host.domain, vdev->host.bus,
+                            vdev->host.slot, vdev->host.function);
+    }
+
+    if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
+        error_report("vfio: error: no such host device: %s",
+                     vdev->vbasedev.sysfsdev);
         return -errno;
     }
 
+    vdev->vbasedev.name = g_strdup(basename(vdev->vbasedev.sysfsdev));
     vdev->vbasedev.ops = &vfio_pci_ops;
-
     vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
-    vdev->vbasedev.name = g_strdup_printf("%04x:%02x:%02x.%01x",
-                                          vdev->host.domain, vdev->host.bus,
-                                          vdev->host.slot, vdev->host.function);
 
-    strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
+    tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
+    len = readlink(tmp, group_path, sizeof(group_path));
+    g_free(tmp);
 
-    len = readlink(path, iommu_group_path, sizeof(path));
-    if (len <= 0 || len >= sizeof(path)) {
+    if (len <= 0 || len >= sizeof(group_path)) {
         error_report("vfio: error no iommu_group for device");
         return len < 0 ? -errno : -ENAMETOOLONG;
     }
 
-    iommu_group_path[len] = 0;
-    group_name = basename(iommu_group_path);
+    group_path[len] = 0;
 
+    group_name = basename(group_path);
     if (sscanf(group_name, "%d", &groupid) != 1) {
-        error_report("vfio: error reading %s: %m", path);
+        error_report("vfio: error reading %s: %m", group_path);
         return -errno;
     }
 
@@ -3560,21 +2442,18 @@ static int vfio_initfn(PCIDevice *pdev)
         return -ENOENT;
     }
 
-    snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-
     QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
         if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
-            error_report("vfio: error: device %s is already attached", path);
+            error_report("vfio: error: device %s is already attached",
+                         vdev->vbasedev.name);
             vfio_put_group(group);
             return -EBUSY;
         }
     }
 
-    ret = vfio_get_device(group, path, &vdev->vbasedev);
+    ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev);
     if (ret) {
-        error_report("vfio: failed to get device %s", path);
+        error_report("vfio: failed to get device %s", vdev->vbasedev.name);
         vfio_put_group(group);
         return ret;
     }
@@ -3600,6 +2479,54 @@ static int vfio_initfn(PCIDevice *pdev)
     /* QEMU can choose to expose the ROM or not */
     memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
 
+    /*
+     * The PCI spec reserves vendor ID 0xffff as an invalid value.  The
+     * device ID is managed by the vendor and need only be a 16-bit value.
+     * Allow any 16-bit value for subsystem so they can be hidden or changed.
+     */
+    if (vdev->vendor_id != PCI_ANY_ID) {
+        if (vdev->vendor_id >= 0xffff) {
+            error_report("vfio: Invalid PCI vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
+        trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
+    } else {
+        vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
+    }
+
+    if (vdev->device_id != PCI_ANY_ID) {
+        if (vdev->device_id > 0xffff) {
+            error_report("vfio: Invalid PCI device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
+        trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
+    } else {
+        vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
+    }
+
+    if (vdev->sub_vendor_id != PCI_ANY_ID) {
+        if (vdev->sub_vendor_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
+                               vdev->sub_vendor_id, ~0);
+        trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
+                                              vdev->sub_vendor_id);
+    }
+
+    if (vdev->sub_device_id != PCI_ANY_ID) {
+        if (vdev->sub_device_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
+        trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
+                                              vdev->sub_device_id);
+    }
+
     /* QEMU can change multi-function devices to single function, or reverse */
     vdev->emulated_config_bits[PCI_HEADER_TYPE] =
                                               PCI_HEADER_TYPE_MULTI_FUNCTION;
@@ -3621,12 +2548,12 @@ static int vfio_initfn(PCIDevice *pdev)
 
     vfio_pci_size_rom(vdev);
 
-    ret = vfio_early_setup_msix(vdev);
+    ret = vfio_msix_early_setup(vdev);
     if (ret) {
         return ret;
     }
 
-    vfio_map_bars(vdev);
+    vfio_bars_setup(vdev);
 
     ret = vfio_add_capabilities(vdev);
     if (ret) {
@@ -3647,8 +2574,8 @@ static int vfio_initfn(PCIDevice *pdev)
     if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
         vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
                                                   vfio_intx_mmap_enable, vdev);
-        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
-        ret = vfio_enable_intx(vdev);
+        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
+        ret = vfio_intx_enable(vdev);
         if (ret) {
             goto out_teardown;
         }
@@ -3656,14 +2583,14 @@ static int vfio_initfn(PCIDevice *pdev)
 
     vfio_register_err_notifier(vdev);
     vfio_register_req_notifier(vdev);
-    vfio_setup_resetfn(vdev);
+    vfio_setup_resetfn_quirk(vdev);
 
     return 0;
 
 out_teardown:
     pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
     vfio_teardown_msi(vdev);
-    vfio_unregister_bars(vdev);
+    vfio_bars_exit(vdev);
     return ret;
 }
 
@@ -3673,7 +2600,7 @@ static void vfio_instance_finalize(Object *obj)
     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pci_dev);
     VFIOGroup *group = vdev->vbasedev.group;
 
-    vfio_unmap_bars(vdev);
+    vfio_bars_finalize(vdev);
     g_free(vdev->emulated_config_bits);
     g_free(vdev->rom);
     vfio_put_device(vdev);
@@ -3692,7 +2619,7 @@ static void vfio_exitfn(PCIDevice *pdev)
         timer_free(vdev->intx.mmap_timer);
     }
     vfio_teardown_msi(vdev);
-    vfio_unregister_bars(vdev);
+    vfio_bars_exit(vdev);
 }
 
 static void vfio_pci_reset(DeviceState *dev)
@@ -3743,13 +2670,23 @@ static void vfio_instance_init(Object *obj)
 
 static Property vfio_pci_dev_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
+    DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
                        intx.mmap_timeout, 1100),
     DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
                     VFIO_FEATURE_ENABLE_VGA_BIT, false),
     DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
                     VFIO_FEATURE_ENABLE_REQ_BIT, true),
-    DEFINE_PROP_BOOL("x-mmap", VFIOPCIDevice, vbasedev.allow_mmap, true),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
+    DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
+    DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
+                       sub_vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
+                       sub_device_id, PCI_ANY_ID),
     /*
      * TODO - support passed fds... is this necessary?
      * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
diff --git a/qemu/hw/vfio/pci.h b/qemu/hw/vfio/pci.h
new file mode 100644
index 000000000..3976f6854
--- /dev/null
+++ b/qemu/hw/vfio/pci.h
@@ -0,0 +1,162 @@
+/*
+ * vfio based device assignment support - PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+#ifndef HW_VFIO_VFIO_PCI_H
+#define HW_VFIO_VFIO_PCI_H
+
+#include "qemu-common.h"
+#include "exec/memory.h"
+#include "hw/pci/pci.h"
+#include "hw/vfio/vfio-common.h"
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+
+#define PCI_ANY_ID (~0)
+
+struct VFIOPCIDevice;
+
+typedef struct VFIOQuirk {
+    QLIST_ENTRY(VFIOQuirk) next;
+    void *data;
+    int nr_mem;
+    MemoryRegion *mem;
+} VFIOQuirk;
+
+typedef struct VFIOBAR {
+    VFIORegion region;
+    bool ioport;
+    bool mem64;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOBAR;
+
+typedef struct VFIOVGARegion {
+    MemoryRegion mem;
+    off_t offset;
+    int nr;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOVGARegion;
+
+typedef struct VFIOVGA {
+    off_t fd_offset;
+    int fd;
+    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
+} VFIOVGA;
+
+typedef struct VFIOINTx {
+    bool pending; /* interrupt pending */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+    uint8_t pin; /* which pin to pull for qemu_set_irq */
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    PCIINTxRoute route; /* routing info for QEMU bypass */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+typedef struct VFIOMSIVector {
+    /*
+     * Two interrupt paths are configured per vector.  The first, is only used
+     * for interrupts injected via QEMU.  This is typically the non-accel path,
+     * but may also be used when we want QEMU to handle masking and pending
+     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
+     * but requires masking at the device.  virq is used to track the MSI route
+     * through KVM, thus kvm_interrupt is only available when virq is set to a
+     * valid (>= 0) value.
+     */
+    EventNotifier interrupt;
+    EventNotifier kvm_interrupt;
+    struct VFIOPCIDevice *vdev; /* back pointer to device */
+    int virq;
+    bool use;
+} VFIOMSIVector;
+
+enum {
+    VFIO_INT_NONE = 0,
+    VFIO_INT_INTx = 1,
+    VFIO_INT_MSI  = 2,
+    VFIO_INT_MSIX = 3,
+};
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+    MemoryRegion mmap_mem;
+    void *mmap;
+    unsigned long *pending;
+} VFIOMSIXInfo;
+
+typedef struct VFIOPCIDevice {
+    PCIDevice pdev;
+    VFIODevice vbasedev;
+    VFIOINTx intx;
+    unsigned int config_size;
+    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
+    off_t config_offset; /* Offset of config space region within device fd */
+    unsigned int rom_size;
+    off_t rom_offset; /* Offset of ROM region within device fd */
+    void *rom;
+    int msi_cap_size;
+    VFIOMSIVector *msi_vectors;
+    VFIOMSIXInfo *msix;
+    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+    int interrupt; /* Current interrupt type */
+    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+    VFIOVGA *vga; /* 0xa0000, 0x3b0, 0x3c0 */
+    PCIHostDeviceAddress host;
+    EventNotifier err_notifier;
+    EventNotifier req_notifier;
+    int (*resetfn)(struct VFIOPCIDevice *);
+    uint32_t vendor_id;
+    uint32_t device_id;
+    uint32_t sub_vendor_id;
+    uint32_t sub_device_id;
+    uint32_t features;
+#define VFIO_FEATURE_ENABLE_VGA_BIT 0
+#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
+#define VFIO_FEATURE_ENABLE_REQ_BIT 1
+#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
+    int32_t bootindex;
+    uint8_t pm_cap;
+    bool has_vga;
+    bool pci_aer;
+    bool req_enabled;
+    bool has_flr;
+    bool has_pm_reset;
+    bool rom_read_failed;
+    bool no_kvm_intx;
+    bool no_kvm_msi;
+    bool no_kvm_msix;
+} VFIOPCIDevice;
+
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len);
+
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size);
+void vfio_vga_write(void *opaque, hwaddr addr, uint64_t data, unsigned size);
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_exit(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev);
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr);
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev);
+
+int vfio_populate_vga(VFIOPCIDevice *vdev);
+
+#endif /* HW_VFIO_VFIO_PCI_H */
diff --git a/qemu/hw/vfio/platform.c b/qemu/hw/vfio/platform.c
index 60365d127..1798a00a3 100644
--- a/qemu/hw/vfio/platform.c
+++ b/qemu/hw/vfio/platform.c
@@ -14,6 +14,8 @@
  *  Copyright Red Hat, Inc. 2012
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
 #include <linux/vfio.h>
 
@@ -32,6 +34,11 @@
  * Functions used whatever the injection method
  */
 
+static inline bool vfio_irq_is_automasked(VFIOINTp *intp)
+{
+    return intp->flags & VFIO_IRQ_INFO_AUTOMASKED;
+}
+
 /**
  * vfio_init_intp - allocate, initialize the IRQ struct pointer
  * and add it into the list of IRQs
@@ -57,18 +64,25 @@ static VFIOINTp *vfio_init_intp(VFIODevice *vbasedev,
     sysbus_init_irq(sbdev, &intp->qemuirq);
 
     /* Get an eventfd for trigger */
-    ret = event_notifier_init(&intp->interrupt, 0);
+    intp->interrupt = g_malloc0(sizeof(EventNotifier));
+    ret = event_notifier_init(intp->interrupt, 0);
     if (ret) {
+        g_free(intp->interrupt);
         g_free(intp);
         error_report("vfio: Error: trigger event_notifier_init failed ");
         return NULL;
     }
-    /* Get an eventfd for resample/unmask */
-    ret = event_notifier_init(&intp->unmask, 0);
-    if (ret) {
-        g_free(intp);
-        error_report("vfio: Error: resamplefd event_notifier_init failed");
-        return NULL;
+    if (vfio_irq_is_automasked(intp)) {
+        /* Get an eventfd for resample/unmask */
+        intp->unmask = g_malloc0(sizeof(EventNotifier));
+        ret = event_notifier_init(intp->unmask, 0);
+        if (ret) {
+            g_free(intp->interrupt);
+            g_free(intp->unmask);
+            g_free(intp);
+            error_report("vfio: Error: resamplefd event_notifier_init failed");
+            return NULL;
+        }
     }
 
     QLIST_INSERT_HEAD(&vdev->intp_list, intp, next);
@@ -100,7 +114,7 @@ static int vfio_set_trigger_eventfd(VFIOINTp *intp,
     irq_set->start = 0;
     irq_set->count = 1;
     pfd = (int32_t *)&irq_set->data;
-    *pfd = event_notifier_get_fd(&intp->interrupt);
+    *pfd = event_notifier_get_fd(intp->interrupt);
     qemu_set_fd_handler(*pfd, (IOHandler *)handler, NULL, intp);
     ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
@@ -130,12 +144,8 @@ static void vfio_mmap_set_enabled(VFIOPlatformDevice *vdev, bool enabled)
 {
     int i;
 
-    trace_vfio_platform_mmap_set_enabled(enabled);
-
     for (i = 0; i < vdev->vbasedev.num_regions; i++) {
-        VFIORegion *region = vdev->regions[i];
-
-        memory_region_set_enabled(&region->mmap_mem, enabled);
+        vfio_region_mmaps_set_enabled(vdev->regions[i], enabled);
     }
 }
 
@@ -182,7 +192,7 @@ static void vfio_intp_mmap_enable(void *opaque)
 static void vfio_intp_inject_pending_lockheld(VFIOINTp *intp)
 {
     trace_vfio_platform_intp_inject_pending_lockheld(intp->pin,
-                              event_notifier_get_fd(&intp->interrupt));
+                              event_notifier_get_fd(intp->interrupt));
 
     intp->state = VFIO_IRQ_ACTIVE;
 
@@ -224,18 +234,18 @@ static void vfio_intp_interrupt(VFIOINTp *intp)
         trace_vfio_intp_interrupt_set_pending(intp->pin);
         QSIMPLEQ_INSERT_TAIL(&vdev->pending_intp_queue,
                              intp, pqnext);
-        ret = event_notifier_test_and_clear(&intp->interrupt);
+        ret = event_notifier_test_and_clear(intp->interrupt);
         qemu_mutex_unlock(&vdev->intp_mutex);
         return;
     }
 
     trace_vfio_platform_intp_interrupt(intp->pin,
-                              event_notifier_get_fd(&intp->interrupt));
+                              event_notifier_get_fd(intp->interrupt));
 
-    ret = event_notifier_test_and_clear(&intp->interrupt);
+    ret = event_notifier_test_and_clear(intp->interrupt);
     if (!ret) {
-        error_report("Error when clearing fd=%d (ret = %d)\n",
-                     event_notifier_get_fd(&intp->interrupt), ret);
+        error_report("Error when clearing fd=%d (ret = %d)",
+                     event_notifier_get_fd(intp->interrupt), ret);
     }
 
     intp->state = VFIO_IRQ_ACTIVE;
@@ -283,13 +293,13 @@ static void vfio_platform_eoi(VFIODevice *vbasedev)
     QLIST_FOREACH(intp, &vdev->intp_list, next) {
         if (intp->state == VFIO_IRQ_ACTIVE) {
             trace_vfio_platform_eoi(intp->pin,
-                                event_notifier_get_fd(&intp->interrupt));
+                                event_notifier_get_fd(intp->interrupt));
             intp->state = VFIO_IRQ_INACTIVE;
 
             /* deassert the virtual IRQ */
             qemu_set_irq(intp->qemuirq, 0);
 
-            if (intp->flags & VFIO_IRQ_INFO_AUTOMASKED) {
+            if (vfio_irq_is_automasked(intp)) {
                 /* unmasks the physical level-sensitive IRQ */
                 vfio_unmask_single_irqindex(vbasedev, intp->pin);
             }
@@ -310,18 +320,29 @@ static void vfio_platform_eoi(VFIODevice *vbasedev)
 /**
  * vfio_start_eventfd_injection - starts the virtual IRQ injection using
  * user-side handled eventfds
- * @intp: the IRQ struct pointer
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
  */
 
-static int vfio_start_eventfd_injection(VFIOINTp *intp)
+static void vfio_start_eventfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 {
     int ret;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIOINTp *intp;
+
+    QLIST_FOREACH(intp, &vdev->intp_list, next) {
+        if (intp->qemuirq == irq) {
+            break;
+        }
+    }
+    assert(intp);
 
     ret = vfio_set_trigger_eventfd(intp, vfio_intp_interrupt);
     if (ret) {
-        error_report("vfio: Error: Failed to pass IRQ fd to the driver: %m");
+        error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                     intp->pin);
+        abort();
     }
-    return ret;
 }
 
 /*
@@ -349,7 +370,7 @@ static int vfio_set_resample_eventfd(VFIOINTp *intp)
     irq_set->start = 0;
     irq_set->count = 1;
     pfd = (int32_t *)&irq_set->data;
-    *pfd = event_notifier_get_fd(&intp->unmask);
+    *pfd = event_notifier_get_fd(intp->unmask);
     qemu_set_fd_handler(*pfd, NULL, NULL, NULL);
     ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
     g_free(irq_set);
@@ -359,6 +380,15 @@ static int vfio_set_resample_eventfd(VFIOINTp *intp)
     return ret;
 }
 
+/**
+ * vfio_start_irqfd_injection - starts the virtual IRQ injection using
+ * irqfd
+ *
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
+ *
+ * In case the irqfd setup fails, we fallback to userspace handled eventfd
+ */
 static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 {
     VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
@@ -366,7 +396,7 @@ static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
 
     if (!kvm_irqfds_enabled() || !kvm_resamplefds_enabled() ||
         !vdev->irqfd_allowed) {
-        return;
+        goto fail_irqfd;
     }
 
     QLIST_FOREACH(intp, &vdev->intp_list, next) {
@@ -376,39 +406,36 @@ static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
     }
     assert(intp);
 
-    /* Get to a known interrupt state */
-    qemu_set_fd_handler(event_notifier_get_fd(&intp->interrupt),
-                        NULL, NULL, vdev);
-
-    vfio_mask_single_irqindex(&vdev->vbasedev, intp->pin);
-    qemu_set_irq(intp->qemuirq, 0);
-
-    if (kvm_irqchip_add_irqfd_notifier(kvm_state, &intp->interrupt,
-                                   &intp->unmask, irq) < 0) {
+    if (kvm_irqchip_add_irqfd_notifier(kvm_state, intp->interrupt,
+                                   intp->unmask, irq) < 0) {
         goto fail_irqfd;
     }
 
     if (vfio_set_trigger_eventfd(intp, NULL) < 0) {
         goto fail_vfio;
     }
-    if (vfio_set_resample_eventfd(intp) < 0) {
-        goto fail_vfio;
+    if (vfio_irq_is_automasked(intp)) {
+        if (vfio_set_resample_eventfd(intp) < 0) {
+            goto fail_vfio;
+        }
+        trace_vfio_platform_start_level_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt),
+                                    event_notifier_get_fd(intp->unmask));
+    } else {
+        trace_vfio_platform_start_edge_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt));
     }
 
-    /* Let's resume injection with irqfd setup */
-    vfio_unmask_single_irqindex(&vdev->vbasedev, intp->pin);
-
     intp->kvm_accel = true;
 
-    trace_vfio_platform_start_irqfd_injection(intp->pin,
-                                     event_notifier_get_fd(&intp->interrupt),
-                                     event_notifier_get_fd(&intp->unmask));
     return;
 fail_vfio:
-    kvm_irqchip_remove_irqfd_notifier(kvm_state, &intp->interrupt, irq);
+    kvm_irqchip_remove_irqfd_notifier(kvm_state, intp->interrupt, irq);
+    error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                 intp->pin);
+    abort();
 fail_irqfd:
-    vfio_start_eventfd_injection(intp);
-    vfio_unmask_single_irqindex(&vdev->vbasedev, intp->pin);
+    vfio_start_eventfd_injection(sbdev, irq);
     return;
 }
 
@@ -446,28 +473,16 @@ static int vfio_populate_device(VFIODevice *vbasedev)
     vdev->regions = g_new0(VFIORegion *, vbasedev->num_regions);
 
     for (i = 0; i < vbasedev->num_regions; i++) {
-        struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
-        VFIORegion *ptr;
+        char *name = g_strdup_printf("VFIO %s region %d\n", vbasedev->name, i);
 
-        vdev->regions[i] = g_malloc0(sizeof(VFIORegion));
-        ptr = vdev->regions[i];
-        reg_info.index = i;
-        ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+        vdev->regions[i] = g_new0(VFIORegion, 1);
+        ret = vfio_region_setup(OBJECT(vdev), vbasedev,
+                                vdev->regions[i], i, name);
+        g_free(name);
         if (ret) {
             error_report("vfio: Error getting region %d info: %m", i);
             goto reg_error;
         }
-        ptr->flags = reg_info.flags;
-        ptr->size = reg_info.size;
-        ptr->fd_offset = reg_info.offset;
-        ptr->nr = i;
-        ptr->vbasedev = vbasedev;
-
-        trace_vfio_platform_populate_regions(ptr->nr,
-                            (unsigned long)ptr->flags,
-                            (unsigned long)ptr->size,
-                            ptr->vbasedev->fd,
-                            (unsigned long)ptr->fd_offset);
     }
 
     vdev->mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
@@ -504,6 +519,9 @@ irq_err:
     }
 reg_error:
     for (i = 0; i < vbasedev->num_regions; i++) {
+        if (vdev->regions[i]) {
+            vfio_region_finalize(vdev->regions[i]);
+        }
         g_free(vdev->regions[i]);
     }
     g_free(vdev->regions);
@@ -530,38 +548,45 @@ static int vfio_base_device_init(VFIODevice *vbasedev)
 {
     VFIOGroup *group;
     VFIODevice *vbasedev_iter;
-    char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
+    char *tmp, group_path[PATH_MAX], *group_name;
     ssize_t len;
     struct stat st;
     int groupid;
     int ret;
 
-    /* name must be set prior to the call */
-    if (!vbasedev->name || strchr(vbasedev->name, '/')) {
-        return -EINVAL;
-    }
+    /* @sysfsdev takes precedence over @host */
+    if (vbasedev->sysfsdev) {
+        g_free(vbasedev->name);
+        vbasedev->name = g_strdup(basename(vbasedev->sysfsdev));
+    } else {
+        if (!vbasedev->name || strchr(vbasedev->name, '/')) {
+            return -EINVAL;
+        }
 
-    /* Check that the host device exists */
-    g_snprintf(path, sizeof(path), "/sys/bus/platform/devices/%s/",
-               vbasedev->name);
+        vbasedev->sysfsdev = g_strdup_printf("/sys/bus/platform/devices/%s",
+                                             vbasedev->name);
+    }
 
-    if (stat(path, &st) < 0) {
-        error_report("vfio: error: no such host device: %s", path);
+    if (stat(vbasedev->sysfsdev, &st) < 0) {
+        error_report("vfio: error: no such host device: %s",
+                     vbasedev->sysfsdev);
         return -errno;
     }
 
-    g_strlcat(path, "iommu_group", sizeof(path));
-    len = readlink(path, iommu_group_path, sizeof(iommu_group_path));
-    if (len < 0 || len >= sizeof(iommu_group_path)) {
+    tmp = g_strdup_printf("%s/iommu_group", vbasedev->sysfsdev);
+    len = readlink(tmp, group_path, sizeof(group_path));
+    g_free(tmp);
+
+    if (len < 0 || len >= sizeof(group_path)) {
         error_report("vfio: error no iommu_group for device");
         return len < 0 ? -errno : -ENAMETOOLONG;
     }
 
-    iommu_group_path[len] = 0;
-    group_name = basename(iommu_group_path);
+    group_path[len] = 0;
 
+    group_name = basename(group_path);
     if (sscanf(group_name, "%d", &groupid) != 1) {
-        error_report("vfio: error reading %s: %m", path);
+        error_report("vfio: error reading %s: %m", group_path);
         return -errno;
     }
 
@@ -573,25 +598,24 @@ static int vfio_base_device_init(VFIODevice *vbasedev)
         return -ENOENT;
     }
 
-    g_snprintf(path, sizeof(path), "%s", vbasedev->name);
-
     QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
         if (strcmp(vbasedev_iter->name, vbasedev->name) == 0) {
-            error_report("vfio: error: device %s is already attached", path);
+            error_report("vfio: error: device %s is already attached",
+                         vbasedev->name);
             vfio_put_group(group);
             return -EBUSY;
         }
     }
-    ret = vfio_get_device(group, path, vbasedev);
+    ret = vfio_get_device(group, vbasedev->name, vbasedev);
     if (ret) {
-        error_report("vfio: failed to get device %s", path);
+        error_report("vfio: failed to get device %s", vbasedev->name);
         vfio_put_group(group);
         return ret;
     }
 
     ret = vfio_populate_device(vbasedev);
     if (ret) {
-        error_report("vfio: failed to populate device %s", path);
+        error_report("vfio: failed to populate device %s", vbasedev->name);
         vfio_put_group(group);
     }
 
@@ -599,41 +623,6 @@ static int vfio_base_device_init(VFIODevice *vbasedev)
 }
 
 /**
- * vfio_map_region - initialize the 2 memory regions for a given
- * MMIO region index
- * @vdev: the VFIO platform device handle
- * @nr: the index of the region
- *
- * Init the top memory region and the mmapped memory region beneath
- * VFIOPlatformDevice is used since VFIODevice is not a QOM Object
- * and could not be passed to memory region functions
-*/
-static void vfio_map_region(VFIOPlatformDevice *vdev, int nr)
-{
-    VFIORegion *region = vdev->regions[nr];
-    uint64_t size = region->size;
-    char name[64];
-
-    if (!size) {
-        return;
-    }
-
-    g_snprintf(name, sizeof(name), "VFIO %s region %d",
-               vdev->vbasedev.name, nr);
-
-    /* A "slow" read/write mapping underlies all regions */
-    memory_region_init_io(&region->mem, OBJECT(vdev), &vfio_region_ops,
-                          region, name, size);
-
-    g_strlcat(name, " mmap", sizeof(name));
-
-    if (vfio_mmap_region(OBJECT(vdev), region, &region->mem,
-                         &region->mmap_mem, &region->mmap, size, 0, name)) {
-        error_report("%s unsupported. Performance may be slow", name);
-    }
-}
-
-/**
  * vfio_platform_realize  - the device realize function
  * @dev: device state pointer
  * @errp: error
@@ -646,13 +635,14 @@ static void vfio_platform_realize(DeviceState *dev, Error **errp)
     VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
     SysBusDevice *sbdev = SYS_BUS_DEVICE(dev);
     VFIODevice *vbasedev = &vdev->vbasedev;
-    VFIOINTp *intp;
     int i, ret;
 
     vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
     vbasedev->ops = &vfio_platform_ops;
 
-    trace_vfio_platform_realize(vbasedev->name, vdev->compat);
+    trace_vfio_platform_realize(vbasedev->sysfsdev ?
+                                vbasedev->sysfsdev : vbasedev->name,
+                                vdev->compat);
 
     ret = vfio_base_device_init(vbasedev);
     if (ret) {
@@ -662,12 +652,11 @@ static void vfio_platform_realize(DeviceState *dev, Error **errp)
     }
 
     for (i = 0; i < vbasedev->num_regions; i++) {
-        vfio_map_region(vdev, i);
-        sysbus_init_mmio(sbdev, &vdev->regions[i]->mem);
-    }
-
-    QLIST_FOREACH(intp, &vdev->intp_list, next) {
-        vfio_start_eventfd_injection(intp);
+        if (vfio_region_mmap(vdev->regions[i])) {
+            error_report("%s mmap unsupported. Performance may be slow",
+                         memory_region_name(vdev->regions[i]->mem));
+        }
+        sysbus_init_mmio(sbdev, vdev->regions[i]->mem);
     }
 }
 
@@ -678,7 +667,8 @@ static const VMStateDescription vfio_platform_vmstate = {
 
 static Property vfio_platform_dev_properties[] = {
     DEFINE_PROP_STRING("host", VFIOPlatformDevice, vbasedev.name),
-    DEFINE_PROP_BOOL("x-mmap", VFIOPlatformDevice, vbasedev.allow_mmap, true),
+    DEFINE_PROP_STRING("sysfsdev", VFIOPlatformDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPlatformDevice, vbasedev.no_mmap, false),
     DEFINE_PROP_UINT32("mmap-timeout-ms", VFIOPlatformDevice,
                        mmap_timeout, 1100),
     DEFINE_PROP_BOOL("x-irqfd", VFIOPlatformDevice, irqfd_allowed, true),
diff --git a/qemu/hw/virtio/Makefile.objs b/qemu/hw/virtio/Makefile.objs
index 19b224a44..3e2b175da 100644
--- a/qemu/hw/virtio/Makefile.objs
+++ b/qemu/hw/virtio/Makefile.objs
@@ -2,7 +2,6 @@ common-obj-y += virtio-rng.o
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
 common-obj-y += virtio-bus.o
 common-obj-y += virtio-mmio.o
-obj-$(CONFIG_VIRTIO) += dataplane/
 
 obj-y += virtio.o virtio-balloon.o 
 obj-$(CONFIG_LINUX) += vhost.o vhost-backend.o vhost-user.o
diff --git a/qemu/hw/virtio/dataplane/Makefile.objs b/qemu/hw/virtio/dataplane/Makefile.objs
deleted file mode 100644
index 753a9cab4..000000000
--- a/qemu/hw/virtio/dataplane/Makefile.objs
+++ /dev/null
@@ -1 +0,0 @@
-obj-y += vring.o
diff --git a/qemu/hw/virtio/dataplane/vring.c b/qemu/hw/virtio/dataplane/vring.c
deleted file mode 100644
index 07fd69c69..000000000
--- a/qemu/hw/virtio/dataplane/vring.c
+++ /dev/null
@@ -1,453 +0,0 @@
-/* Copyright 2012 Red Hat, Inc.
- * Copyright IBM, Corp. 2012
- *
- * Based on Linux 2.6.39 vhost code:
- * Copyright (C) 2009 Red Hat, Inc.
- * Copyright (C) 2006 Rusty Russell IBM Corporation
- *
- * Author: Michael S. Tsirkin <mst@redhat.com>
- *         Stefan Hajnoczi <stefanha@redhat.com>
- *
- * Inspiration, some code, and most witty comments come from
- * Documentation/virtual/lguest/lguest.c, by Rusty Russell
- *
- * This work is licensed under the terms of the GNU GPL, version 2.
- */
-
-#include "trace.h"
-#include "hw/hw.h"
-#include "exec/memory.h"
-#include "exec/address-spaces.h"
-#include "hw/virtio/virtio-access.h"
-#include "hw/virtio/dataplane/vring.h"
-#include "hw/virtio/dataplane/vring-accessors.h"
-#include "qemu/error-report.h"
-
-/* vring_map can be coupled with vring_unmap or (if you still have the
- * value returned in *mr) memory_region_unref.
- */
-static void *vring_map(MemoryRegion **mr, hwaddr phys, hwaddr len,
-                       bool is_write)
-{
-    MemoryRegionSection section = memory_region_find(get_system_memory(), phys, len);
-
-    if (!section.mr || int128_get64(section.size) < len) {
-        goto out;
-    }
-    if (is_write && section.readonly) {
-        goto out;
-    }
-    if (!memory_region_is_ram(section.mr)) {
-        goto out;
-    }
-
-    /* Ignore regions with dirty logging, we cannot mark them dirty */
-    if (memory_region_get_dirty_log_mask(section.mr)) {
-        goto out;
-    }
-
-    *mr = section.mr;
-    return memory_region_get_ram_ptr(section.mr) + section.offset_within_region;
-
-out:
-    memory_region_unref(section.mr);
-    *mr = NULL;
-    return NULL;
-}
-
-static void vring_unmap(void *buffer, bool is_write)
-{
-    ram_addr_t addr;
-    MemoryRegion *mr;
-
-    mr = qemu_ram_addr_from_host(buffer, &addr);
-    memory_region_unref(mr);
-}
-
-/* Map the guest's vring to host memory */
-bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
-{
-    hwaddr vring_addr = virtio_queue_get_ring_addr(vdev, n);
-    hwaddr vring_size = virtio_queue_get_ring_size(vdev, n);
-    void *vring_ptr;
-
-    vring->broken = false;
-
-    vring_ptr = vring_map(&vring->mr, vring_addr, vring_size, true);
-    if (!vring_ptr) {
-        error_report("Failed to map vring "
-                     "addr %#" HWADDR_PRIx " size %" HWADDR_PRIu,
-                     vring_addr, vring_size);
-        vring->broken = true;
-        return false;
-    }
-
-    vring_init(&vring->vr, virtio_queue_get_num(vdev, n), vring_ptr, 4096);
-
-    vring->last_avail_idx = virtio_queue_get_last_avail_idx(vdev, n);
-    vring->last_used_idx = vring_get_used_idx(vdev, vring);
-    vring->signalled_used = 0;
-    vring->signalled_used_valid = false;
-
-    trace_vring_setup(virtio_queue_get_ring_addr(vdev, n),
-                      vring->vr.desc, vring->vr.avail, vring->vr.used);
-    return true;
-}
-
-void vring_teardown(Vring *vring, VirtIODevice *vdev, int n)
-{
-    virtio_queue_set_last_avail_idx(vdev, n, vring->last_avail_idx);
-    virtio_queue_invalidate_signalled_used(vdev, n);
-
-    memory_region_unref(vring->mr);
-}
-
-/* Disable guest->host notifies */
-void vring_disable_notification(VirtIODevice *vdev, Vring *vring)
-{
-    if (!virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
-        vring_set_used_flags(vdev, vring, VRING_USED_F_NO_NOTIFY);
-    }
-}
-
-/* Enable guest->host notifies
- *
- * Return true if the vring is empty, false if there are more requests.
- */
-bool vring_enable_notification(VirtIODevice *vdev, Vring *vring)
-{
-    if (virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
-    } else {
-        vring_clear_used_flags(vdev, vring, VRING_USED_F_NO_NOTIFY);
-    }
-    smp_mb(); /* ensure update is seen before reading avail_idx */
-    return !vring_more_avail(vdev, vring);
-}
-
-/* This is stolen from linux/drivers/vhost/vhost.c:vhost_notify() */
-bool vring_should_notify(VirtIODevice *vdev, Vring *vring)
-{
-    uint16_t old, new;
-    bool v;
-    /* Flush out used index updates. This is paired
-     * with the barrier that the Guest executes when enabling
-     * interrupts. */
-    smp_mb();
-
-    if (virtio_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
-        unlikely(!vring_more_avail(vdev, vring))) {
-        return true;
-    }
-
-    if (!virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
-        return !(vring_get_avail_flags(vdev, vring) &
-                 VRING_AVAIL_F_NO_INTERRUPT);
-    }
-    old = vring->signalled_used;
-    v = vring->signalled_used_valid;
-    new = vring->signalled_used = vring->last_used_idx;
-    vring->signalled_used_valid = true;
-
-    if (unlikely(!v)) {
-        return true;
-    }
-
-    return vring_need_event(virtio_tswap16(vdev, vring_used_event(&vring->vr)),
-                            new, old);
-}
-
-
-static int get_desc(Vring *vring, VirtQueueElement *elem,
-                    struct vring_desc *desc)
-{
-    unsigned *num;
-    struct iovec *iov;
-    hwaddr *addr;
-    MemoryRegion *mr;
-
-    if (desc->flags & VRING_DESC_F_WRITE) {
-        num = &elem->in_num;
-        iov = &elem->in_sg[*num];
-        addr = &elem->in_addr[*num];
-    } else {
-        num = &elem->out_num;
-        iov = &elem->out_sg[*num];
-        addr = &elem->out_addr[*num];
-
-        /* If it's an output descriptor, they're all supposed
-         * to come before any input descriptors. */
-        if (unlikely(elem->in_num)) {
-            error_report("Descriptor has out after in");
-            return -EFAULT;
-        }
-    }
-
-    /* Stop for now if there are not enough iovecs available. */
-    if (*num >= VIRTQUEUE_MAX_SIZE) {
-        error_report("Invalid SG num: %u", *num);
-        return -EFAULT;
-    }
-
-    /* TODO handle non-contiguous memory across region boundaries */
-    iov->iov_base = vring_map(&mr, desc->addr, desc->len,
-                              desc->flags & VRING_DESC_F_WRITE);
-    if (!iov->iov_base) {
-        error_report("Failed to map descriptor addr %#" PRIx64 " len %u",
-                     (uint64_t)desc->addr, desc->len);
-        return -EFAULT;
-    }
-
-    /* The MemoryRegion is looked up again and unref'ed later, leave the
-     * ref in place.  */
-    iov->iov_len = desc->len;
-    *addr = desc->addr;
-    *num += 1;
-    return 0;
-}
-
-static void copy_in_vring_desc(VirtIODevice *vdev,
-                               const struct vring_desc *guest,
-                               struct vring_desc *host)
-{
-    host->addr = virtio_ldq_p(vdev, &guest->addr);
-    host->len = virtio_ldl_p(vdev, &guest->len);
-    host->flags = virtio_lduw_p(vdev, &guest->flags);
-    host->next = virtio_lduw_p(vdev, &guest->next);
-}
-
-/* This is stolen from linux/drivers/vhost/vhost.c. */
-static int get_indirect(VirtIODevice *vdev, Vring *vring,
-                        VirtQueueElement *elem, struct vring_desc *indirect)
-{
-    struct vring_desc desc;
-    unsigned int i = 0, count, found = 0;
-    int ret;
-
-    /* Sanity check */
-    if (unlikely(indirect->len % sizeof(desc))) {
-        error_report("Invalid length in indirect descriptor: "
-                     "len %#x not multiple of %#zx",
-                     indirect->len, sizeof(desc));
-        vring->broken = true;
-        return -EFAULT;
-    }
-
-    count = indirect->len / sizeof(desc);
-    /* Buffers are chained via a 16 bit next field, so
-     * we can have at most 2^16 of these. */
-    if (unlikely(count > USHRT_MAX + 1)) {
-        error_report("Indirect buffer length too big: %d", indirect->len);
-        vring->broken = true;
-        return -EFAULT;
-    }
-
-    do {
-        struct vring_desc *desc_ptr;
-        MemoryRegion *mr;
-
-        /* Translate indirect descriptor */
-        desc_ptr = vring_map(&mr,
-                             indirect->addr + found * sizeof(desc),
-                             sizeof(desc), false);
-        if (!desc_ptr) {
-            error_report("Failed to map indirect descriptor "
-                         "addr %#" PRIx64 " len %zu",
-                         (uint64_t)indirect->addr + found * sizeof(desc),
-                         sizeof(desc));
-            vring->broken = true;
-            return -EFAULT;
-        }
-        copy_in_vring_desc(vdev, desc_ptr, &desc);
-        memory_region_unref(mr);
-
-        /* Ensure descriptor has been loaded before accessing fields */
-        barrier(); /* read_barrier_depends(); */
-
-        if (unlikely(++found > count)) {
-            error_report("Loop detected: last one at %u "
-                         "indirect size %u", i, count);
-            vring->broken = true;
-            return -EFAULT;
-        }
-
-        if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
-            error_report("Nested indirect descriptor");
-            vring->broken = true;
-            return -EFAULT;
-        }
-
-        ret = get_desc(vring, elem, &desc);
-        if (ret < 0) {
-            vring->broken |= (ret == -EFAULT);
-            return ret;
-        }
-        i = desc.next;
-    } while (desc.flags & VRING_DESC_F_NEXT);
-    return 0;
-}
-
-static void vring_unmap_element(VirtQueueElement *elem)
-{
-    int i;
-
-    /* This assumes that the iovecs, if changed, are never moved past
-     * the end of the valid area.  This is true if iovec manipulations
-     * are done with iov_discard_front and iov_discard_back.
-     */
-    for (i = 0; i < elem->out_num; i++) {
-        vring_unmap(elem->out_sg[i].iov_base, false);
-    }
-
-    for (i = 0; i < elem->in_num; i++) {
-        vring_unmap(elem->in_sg[i].iov_base, true);
-    }
-}
-
-/* This looks in the virtqueue and for the first available buffer, and converts
- * it to an iovec for convenient access.  Since descriptors consist of some
- * number of output then some number of input descriptors, it's actually two
- * iovecs, but we pack them into one and note how many of each there were.
- *
- * This function returns the descriptor number found, or vq->num (which is
- * never a valid descriptor number) if none was found.  A negative code is
- * returned on error.
- *
- * Stolen from linux/drivers/vhost/vhost.c.
- */
-int vring_pop(VirtIODevice *vdev, Vring *vring,
-              VirtQueueElement *elem)
-{
-    struct vring_desc desc;
-    unsigned int i, head, found = 0, num = vring->vr.num;
-    uint16_t avail_idx, last_avail_idx;
-    int ret;
-
-    /* Initialize elem so it can be safely unmapped */
-    elem->in_num = elem->out_num = 0;
-
-    /* If there was a fatal error then refuse operation */
-    if (vring->broken) {
-        ret = -EFAULT;
-        goto out;
-    }
-
-    /* Check it isn't doing very strange things with descriptor numbers. */
-    last_avail_idx = vring->last_avail_idx;
-    avail_idx = vring_get_avail_idx(vdev, vring);
-    barrier(); /* load indices now and not again later */
-
-    if (unlikely((uint16_t)(avail_idx - last_avail_idx) > num)) {
-        error_report("Guest moved used index from %u to %u",
-                     last_avail_idx, avail_idx);
-        ret = -EFAULT;
-        goto out;
-    }
-
-    /* If there's nothing new since last we looked. */
-    if (avail_idx == last_avail_idx) {
-        ret = -EAGAIN;
-        goto out;
-    }
-
-    /* Only get avail ring entries after they have been exposed by guest. */
-    smp_rmb();
-
-    /* Grab the next descriptor number they're advertising, and increment
-     * the index we've seen. */
-    head = vring_get_avail_ring(vdev, vring, last_avail_idx % num);
-
-    elem->index = head;
-
-    /* If their number is silly, that's an error. */
-    if (unlikely(head >= num)) {
-        error_report("Guest says index %u > %u is available", head, num);
-        ret = -EFAULT;
-        goto out;
-    }
-
-    i = head;
-    do {
-        if (unlikely(i >= num)) {
-            error_report("Desc index is %u > %u, head = %u", i, num, head);
-            ret = -EFAULT;
-            goto out;
-        }
-        if (unlikely(++found > num)) {
-            error_report("Loop detected: last one at %u vq size %u head %u",
-                         i, num, head);
-            ret = -EFAULT;
-            goto out;
-        }
-        copy_in_vring_desc(vdev, &vring->vr.desc[i], &desc);
-
-        /* Ensure descriptor is loaded before accessing fields */
-        barrier();
-
-        if (desc.flags & VRING_DESC_F_INDIRECT) {
-            ret = get_indirect(vdev, vring, elem, &desc);
-            if (ret < 0) {
-                goto out;
-            }
-            continue;
-        }
-
-        ret = get_desc(vring, elem, &desc);
-        if (ret < 0) {
-            goto out;
-        }
-
-        i = desc.next;
-    } while (desc.flags & VRING_DESC_F_NEXT);
-
-    /* On success, increment avail index. */
-    vring->last_avail_idx++;
-    if (virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(&vring->vr) =
-            virtio_tswap16(vdev, vring->last_avail_idx);
-    }
-
-    return head;
-
-out:
-    assert(ret < 0);
-    if (ret == -EFAULT) {
-        vring->broken = true;
-    }
-    vring_unmap_element(elem);
-    return ret;
-}
-
-/* After we've used one of their buffers, we tell them about it.
- *
- * Stolen from linux/drivers/vhost/vhost.c.
- */
-void vring_push(VirtIODevice *vdev, Vring *vring, VirtQueueElement *elem,
-                int len)
-{
-    unsigned int head = elem->index;
-    uint16_t new;
-
-    vring_unmap_element(elem);
-
-    /* Don't touch vring if a fatal error occurred */
-    if (vring->broken) {
-        return;
-    }
-
-    /* The virtqueue contains a ring of used buffers.  Get a pointer to the
-     * next entry in that used ring. */
-    vring_set_used_ring_id(vdev, vring, vring->last_used_idx % vring->vr.num,
-                           head);
-    vring_set_used_ring_len(vdev, vring, vring->last_used_idx % vring->vr.num,
-                            len);
-
-    /* Make sure buffer is written before we update index. */
-    smp_wmb();
-
-    new = ++vring->last_used_idx;
-    vring_set_used_idx(vdev, vring, new);
-    if (unlikely((int16_t)(new - vring->signalled_used) < (uint16_t)1)) {
-        vring->signalled_used_valid = false;
-    }
-}
diff --git a/qemu/hw/virtio/vhost-backend.c b/qemu/hw/virtio/vhost-backend.c
index 4d68a2765..b35890289 100644
--- a/qemu/hw/virtio/vhost-backend.c
+++ b/qemu/hw/virtio/vhost-backend.c
@@ -8,9 +8,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
 #include "qemu/error-report.h"
+#include "linux/vhost.h"
 
 #include <sys/ioctl.h>
 
@@ -42,11 +44,152 @@ static int vhost_kernel_cleanup(struct vhost_dev *dev)
     return close(fd);
 }
 
+static int vhost_kernel_memslots_limit(struct vhost_dev *dev)
+{
+    int limit = 64;
+    char *s;
+
+    if (g_file_get_contents("/sys/module/vhost/parameters/max_mem_regions",
+                            &s, NULL, NULL)) {
+        uint64_t val = g_ascii_strtoull(s, NULL, 10);
+        if (!((val == G_MAXUINT64 || !val) && errno)) {
+            return val;
+        }
+        error_report("ignoring invalid max_mem_regions value in vhost module:"
+                     " %s", s);
+    }
+    return limit;
+}
+
+static int vhost_kernel_net_set_backend(struct vhost_dev *dev,
+                                        struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_NET_SET_BACKEND, file);
+}
+
+static int vhost_kernel_scsi_set_endpoint(struct vhost_dev *dev,
+                                          struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_SET_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_clear_endpoint(struct vhost_dev *dev,
+                                            struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_CLEAR_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
+}
+
+static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log)
+{
+    return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
+}
+
+static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
+                                      struct vhost_memory *mem)
+{
+    return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
+}
+
+static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
+}
+
+static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
+}
+
+static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
+}
+
+static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
+}
+
+static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
+static int vhost_kernel_set_features(struct vhost_dev *dev,
+                                     uint64_t features)
+{
+    return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
+}
+
+static int vhost_kernel_get_features(struct vhost_dev *dev,
+                                     uint64_t *features)
+{
+    return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
+}
+
+static int vhost_kernel_set_owner(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
+}
+
+static int vhost_kernel_reset_device(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_RESET_OWNER, NULL);
+}
+
+static int vhost_kernel_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
 static const VhostOps kernel_ops = {
         .backend_type = VHOST_BACKEND_TYPE_KERNEL,
-        .vhost_call = vhost_kernel_call,
         .vhost_backend_init = vhost_kernel_init,
-        .vhost_backend_cleanup = vhost_kernel_cleanup
+        .vhost_backend_cleanup = vhost_kernel_cleanup,
+        .vhost_backend_memslots_limit = vhost_kernel_memslots_limit,
+        .vhost_net_set_backend = vhost_kernel_net_set_backend,
+        .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint,
+        .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint,
+        .vhost_scsi_get_abi_version = vhost_kernel_scsi_get_abi_version,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_mem_table = vhost_kernel_set_mem_table,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_endian = vhost_kernel_set_vring_endian,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_set_features = vhost_kernel_set_features,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_reset_device = vhost_kernel_reset_device,
+        .vhost_get_vq_index = vhost_kernel_get_vq_index,
 };
 
 int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
diff --git a/qemu/hw/virtio/vhost-user.c b/qemu/hw/virtio/vhost-user.c
index e7ab8293d..5914e8510 100644
--- a/qemu/hw/virtio/vhost-user.c
+++ b/qemu/hw/virtio/vhost-user.c
@@ -8,22 +8,35 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
 #include "sysemu/char.h"
 #include "sysemu/kvm.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "exec/ram_addr.h"
+#include "migration/migration.h"
 
-#include <fcntl.h>
-#include <unistd.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <linux/vhost.h>
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
 
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
@@ -41,6 +54,11 @@ typedef enum VhostUserRequest {
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -57,6 +75,11 @@ typedef struct VhostUserMemory {
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -71,7 +94,8 @@ typedef struct VhostUserMsg {
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -89,37 +113,6 @@ static bool ioeventfd_enabled(void)
     return kvm_enabled() && kvm_eventfds_enabled();
 }
 
-static unsigned long int ioctl_to_vhost_user_request[VHOST_USER_MAX] = {
-    -1,                     /* VHOST_USER_NONE */
-    VHOST_GET_FEATURES,     /* VHOST_USER_GET_FEATURES */
-    VHOST_SET_FEATURES,     /* VHOST_USER_SET_FEATURES */
-    VHOST_SET_OWNER,        /* VHOST_USER_SET_OWNER */
-    VHOST_RESET_OWNER,      /* VHOST_USER_RESET_OWNER */
-    VHOST_SET_MEM_TABLE,    /* VHOST_USER_SET_MEM_TABLE */
-    VHOST_SET_LOG_BASE,     /* VHOST_USER_SET_LOG_BASE */
-    VHOST_SET_LOG_FD,       /* VHOST_USER_SET_LOG_FD */
-    VHOST_SET_VRING_NUM,    /* VHOST_USER_SET_VRING_NUM */
-    VHOST_SET_VRING_ADDR,   /* VHOST_USER_SET_VRING_ADDR */
-    VHOST_SET_VRING_BASE,   /* VHOST_USER_SET_VRING_BASE */
-    VHOST_GET_VRING_BASE,   /* VHOST_USER_GET_VRING_BASE */
-    VHOST_SET_VRING_KICK,   /* VHOST_USER_SET_VRING_KICK */
-    VHOST_SET_VRING_CALL,   /* VHOST_USER_SET_VRING_CALL */
-    VHOST_SET_VRING_ERR     /* VHOST_USER_SET_VRING_ERR */
-};
-
-static VhostUserRequest vhost_user_request_translate(unsigned long int request)
-{
-    VhostUserRequest idx;
-
-    for (idx = 0; idx < VHOST_USER_MAX; idx++) {
-        if (ioctl_to_vhost_user_request[idx] == request) {
-            break;
-        }
-    }
-
-    return (idx == VHOST_USER_MAX) ? VHOST_USER_NONE : idx;
-}
-
 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 {
     CharDriverState *chr = dev->opaque;
@@ -128,8 +121,8 @@ static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 
     r = qemu_chr_fe_read_all(chr, p, size);
     if (r != size) {
-        error_report("Failed to read msg header. Read %d instead of %d.", r,
-                size);
+        error_report("Failed to read msg header. Read %d instead of %d."
+                     " Original request %d.", r, size, msg->request);
         goto fail;
     }
 
@@ -166,12 +159,35 @@ fail:
     return -1;
 }
 
+static bool vhost_user_one_time_request(VhostUserRequest request)
+{
+    switch (request) {
+    case VHOST_USER_SET_OWNER:
+    case VHOST_USER_RESET_OWNER:
+    case VHOST_USER_SET_MEM_TABLE:
+    case VHOST_USER_GET_QUEUE_NUM:
+        return true;
+    default:
+        return false;
+    }
+}
+
+/* most non-init callers ignore the error */
 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
                             int *fds, int fd_num)
 {
     CharDriverState *chr = dev->opaque;
     int size = VHOST_USER_HDR_SIZE + msg->size;
 
+    /*
+     * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
+     * we just need send it once in the first time. For later such
+     * request, we just ignore it.
+     */
+    if (vhost_user_one_time_request(msg->request) && dev->vq_index != 0) {
+        return 0;
+    }
+
     if (fd_num) {
         qemu_chr_fe_set_msgfds(chr, fds, fd_num);
     }
@@ -180,157 +196,364 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
             0 : -1;
 }
 
-static int vhost_user_call(struct vhost_dev *dev, unsigned long int request,
-        void *arg)
+static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                   struct vhost_log *log)
 {
-    VhostUserMsg msg;
-    VhostUserRequest msg_request;
-    struct vhost_vring_file *file = 0;
-    int need_reply = 0;
     int fds[VHOST_MEMORY_MAX_NREGIONS];
-    int i, fd;
     size_t fd_num = 0;
+    bool shmfd = virtio_has_feature(dev->protocol_features,
+                                    VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_LOG_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.log.mmap_size = log->size * sizeof(*(log->log)),
+        .payload.log.mmap_offset = 0,
+        .size = sizeof(msg.payload.log),
+    };
 
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+    if (shmfd && log->fd != -1) {
+        fds[fd_num++] = log->fd;
+    }
 
-    msg_request = vhost_user_request_translate(request);
-    msg.request = msg_request;
-    msg.flags = VHOST_USER_VERSION;
-    msg.size = 0;
+    vhost_user_write(dev, &msg, fds, fd_num);
 
-    switch (request) {
-    case VHOST_GET_FEATURES:
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_FEATURES:
-    case VHOST_SET_LOG_BASE:
-        msg.u64 = *((__u64 *) arg);
-        msg.size = sizeof(m.u64);
-        break;
-
-    case VHOST_SET_OWNER:
-    case VHOST_RESET_OWNER:
-        break;
-
-    case VHOST_SET_MEM_TABLE:
-        for (i = 0; i < dev->mem->nregions; ++i) {
-            struct vhost_memory_region *reg = dev->mem->regions + i;
-            ram_addr_t ram_addr;
-
-            assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
-            qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr, &ram_addr);
-            fd = qemu_get_ram_fd(ram_addr);
-            if (fd > 0) {
-                msg.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
-                msg.memory.regions[fd_num].memory_size  = reg->memory_size;
-                msg.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
-                msg.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
-                    (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
-                assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
-                fds[fd_num++] = fd;
-            }
+    if (shmfd) {
+        msg.size = 0;
+        if (vhost_user_read(dev, &msg) < 0) {
+            return 0;
         }
 
-        msg.memory.nregions = fd_num;
-
-        if (!fd_num) {
-            error_report("Failed initializing vhost-user memory map, "
-                    "consider using -object memory-backend-file share=on");
+        if (msg.request != VHOST_USER_SET_LOG_BASE) {
+            error_report("Received unexpected msg type. "
+                         "Expected %d received %d",
+                         VHOST_USER_SET_LOG_BASE, msg.request);
             return -1;
         }
+    }
 
-        msg.size = sizeof(m.memory.nregions);
-        msg.size += sizeof(m.memory.padding);
-        msg.size += fd_num * sizeof(VhostUserMemoryRegion);
-
-        break;
-
-    case VHOST_SET_LOG_FD:
-        fds[fd_num++] = *((int *) arg);
-        break;
-
-    case VHOST_SET_VRING_NUM:
-    case VHOST_SET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        break;
-
-    case VHOST_GET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_VRING_ADDR:
-        memcpy(&msg.addr, arg, sizeof(struct vhost_vring_addr));
-        msg.size = sizeof(m.addr);
-        break;
-
-    case VHOST_SET_VRING_KICK:
-    case VHOST_SET_VRING_CALL:
-    case VHOST_SET_VRING_ERR:
-        file = arg;
-        msg.u64 = file->index & VHOST_USER_VRING_IDX_MASK;
-        msg.size = sizeof(m.u64);
-        if (ioeventfd_enabled() && file->fd > 0) {
-            fds[fd_num++] = file->fd;
-        } else {
-            msg.u64 |= VHOST_USER_VRING_NOFD_MASK;
+    return 0;
+}
+
+static int vhost_user_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int i, fd;
+    size_t fd_num = 0;
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_MEM_TABLE,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    for (i = 0; i < dev->mem->nregions; ++i) {
+        struct vhost_memory_region *reg = dev->mem->regions + i;
+        ram_addr_t ram_addr;
+
+        assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
+        qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr,
+                                &ram_addr);
+        fd = qemu_get_ram_fd(ram_addr);
+        if (fd > 0) {
+            msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
+            msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
+            msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
+            msg.payload.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
+                (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
+            assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
+            fds[fd_num++] = fd;
         }
-        break;
-    default:
-        error_report("vhost-user trying to send unhandled ioctl");
+    }
+
+    msg.payload.memory.nregions = fd_num;
+
+    if (!fd_num) {
+        error_report("Failed initializing vhost-user memory map, "
+                     "consider using -object memory-backend-file share=on");
         return -1;
-        break;
     }
 
-    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+    msg.size = sizeof(msg.payload.memory.nregions);
+    msg.size += sizeof(msg.payload.memory.padding);
+    msg.size += fd_num * sizeof(VhostUserMemoryRegion);
+
+    vhost_user_write(dev, &msg, fds, fd_num);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_addr(struct vhost_dev *dev,
+                                     struct vhost_vring_addr *addr)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_VRING_ADDR,
+        .flags = VHOST_USER_VERSION,
+        .payload.addr = *addr,
+        .size = sizeof(msg.payload.addr),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_endian(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    error_report("vhost-user trying to send unhandled ioctl");
+    return -1;
+}
+
+static int vhost_set_vring(struct vhost_dev *dev,
+                           unsigned long int request,
+                           struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_num(struct vhost_dev *dev,
+                                    struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
+}
+
+static int vhost_user_set_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
+}
+
+static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
+{
+    int i;
+
+    if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        return -1;
+    }
+
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num   = enable,
+        };
+
+        vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
+    }
+
+    return 0;
+}
+
+static int vhost_user_get_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_GET_VRING_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    if (vhost_user_read(dev, &msg) < 0) {
         return 0;
     }
 
-    if (need_reply) {
-        if (vhost_user_read(dev, &msg) < 0) {
-            return 0;
-        }
+    if (msg.request != VHOST_USER_GET_VRING_BASE) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     VHOST_USER_GET_VRING_BASE, msg.request);
+        return -1;
+    }
 
-        if (msg_request != msg.request) {
-            error_report("Received unexpected msg type."
-                    " Expected %d received %d", msg_request, msg.request);
-            return -1;
-        }
+    if (msg.size != sizeof(msg.payload.state)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
 
-        switch (msg_request) {
-        case VHOST_USER_GET_FEATURES:
-            if (msg.size != sizeof(m.u64)) {
-                error_report("Received bad msg size.");
-                return -1;
-            }
-            *((__u64 *) arg) = msg.u64;
-            break;
-        case VHOST_USER_GET_VRING_BASE:
-            if (msg.size != sizeof(m.state)) {
-                error_report("Received bad msg size.");
-                return -1;
-            }
-            memcpy(arg, &msg.state, sizeof(struct vhost_vring_state));
-            break;
-        default:
-            error_report("Received unexpected msg type.");
-            return -1;
-            break;
-        }
+    *ring = msg.payload.state;
+
+    return 0;
+}
+
+static int vhost_set_vring_file(struct vhost_dev *dev,
+                                VhostUserRequest request,
+                                struct vhost_vring_file *file)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    size_t fd_num = 0;
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    if (ioeventfd_enabled() && file->fd > 0) {
+        fds[fd_num++] = file->fd;
+    } else {
+        msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
     }
 
+    vhost_user_write(dev, &msg, fds, fd_num);
+
+    return 0;
+}
+
+static int vhost_user_set_vring_kick(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
+}
+
+static int vhost_user_set_vring_call(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
+}
+
+static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = u64,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
+}
+
+static int vhost_user_set_protocol_features(struct vhost_dev *dev,
+                                            uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
+}
+
+static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
+        return 0;
+    }
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return 0;
+    }
+
+    if (msg.request != request) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     request, msg.request);
+        return -1;
+    }
+
+    if (msg.size != sizeof(msg.payload.u64)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
+
+    *u64 = msg.payload.u64;
+
+    return 0;
+}
+
+static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
+{
+    return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
+}
+
+static int vhost_user_set_owner(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
+    return 0;
+}
+
+static int vhost_user_reset_device(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_RESET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    vhost_user_write(dev, &msg, NULL, 0);
+
     return 0;
 }
 
 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
 {
+    uint64_t features;
+    int err;
+
     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 
     dev->opaque = opaque;
 
+    err = vhost_user_get_features(dev, &features);
+    if (err < 0) {
+        return err;
+    }
+
+    if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+
+        err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
+                                 &features);
+        if (err < 0) {
+            return err;
+        }
+
+        dev->protocol_features = features & VHOST_USER_PROTOCOL_FEATURE_MASK;
+        err = vhost_user_set_protocol_features(dev, dev->protocol_features);
+        if (err < 0) {
+            return err;
+        }
+
+        /* query the max queues we support if backend supports Multiple Queue */
+        if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
+            err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
+                                     &dev->max_queues);
+            if (err < 0) {
+                return err;
+            }
+        }
+    }
+
+    if (dev->migration_blocker == NULL &&
+        !virtio_has_feature(dev->protocol_features,
+                            VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
+        error_setg(&dev->migration_blocker,
+                   "Migration disabled: vhost-user backend lacks "
+                   "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
+    }
+
     return 0;
 }
 
@@ -343,9 +566,92 @@ static int vhost_user_cleanup(struct vhost_dev *dev)
     return 0;
 }
 
+static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx;
+}
+
+static int vhost_user_memslots_limit(struct vhost_dev *dev)
+{
+    return VHOST_MEMORY_MAX_NREGIONS;
+}
+
+static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
+{
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    return virtio_has_feature(dev->protocol_features,
+                              VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+}
+
+static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
+{
+    VhostUserMsg msg = { 0 };
+    int err;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    /* If guest supports GUEST_ANNOUNCE do nothing */
+    if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
+        return 0;
+    }
+
+    /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
+    if (virtio_has_feature(dev->protocol_features,
+                           VHOST_USER_PROTOCOL_F_RARP)) {
+        msg.request = VHOST_USER_SEND_RARP;
+        msg.flags = VHOST_USER_VERSION;
+        memcpy((char *)&msg.payload.u64, mac_addr, 6);
+        msg.size = sizeof(msg.payload.u64);
+
+        err = vhost_user_write(dev, &msg, NULL, 0);
+        return err;
+    }
+    return -1;
+}
+
+static bool vhost_user_can_merge(struct vhost_dev *dev,
+                                 uint64_t start1, uint64_t size1,
+                                 uint64_t start2, uint64_t size2)
+{
+    ram_addr_t ram_addr;
+    int mfd, rfd;
+    MemoryRegion *mr;
+
+    mr = qemu_ram_addr_from_host((void *)(uintptr_t)start1, &ram_addr);
+    assert(mr);
+    mfd = qemu_get_ram_fd(ram_addr);
+
+    mr = qemu_ram_addr_from_host((void *)(uintptr_t)start2, &ram_addr);
+    assert(mr);
+    rfd = qemu_get_ram_fd(ram_addr);
+
+    return mfd == rfd;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
-        .vhost_call = vhost_user_call,
         .vhost_backend_init = vhost_user_init,
-        .vhost_backend_cleanup = vhost_user_cleanup
-        };
+        .vhost_backend_cleanup = vhost_user_cleanup,
+        .vhost_backend_memslots_limit = vhost_user_memslots_limit,
+        .vhost_set_log_base = vhost_user_set_log_base,
+        .vhost_set_mem_table = vhost_user_set_mem_table,
+        .vhost_set_vring_addr = vhost_user_set_vring_addr,
+        .vhost_set_vring_endian = vhost_user_set_vring_endian,
+        .vhost_set_vring_num = vhost_user_set_vring_num,
+        .vhost_set_vring_base = vhost_user_set_vring_base,
+        .vhost_get_vring_base = vhost_user_get_vring_base,
+        .vhost_set_vring_kick = vhost_user_set_vring_kick,
+        .vhost_set_vring_call = vhost_user_set_vring_call,
+        .vhost_set_features = vhost_user_set_features,
+        .vhost_get_features = vhost_user_get_features,
+        .vhost_set_owner = vhost_user_set_owner,
+        .vhost_reset_device = vhost_user_reset_device,
+        .vhost_get_vq_index = vhost_user_get_vq_index,
+        .vhost_set_vring_enable = vhost_user_set_vring_enable,
+        .vhost_requires_shm_log = vhost_user_requires_shm_log,
+        .vhost_migration_done = vhost_user_migration_done,
+        .vhost_backend_can_merge = vhost_user_can_merge,
+};
diff --git a/qemu/hw/virtio/vhost.c b/qemu/hw/virtio/vhost.c
index 2712c6fc0..440071815 100644
--- a/qemu/hw/virtio/vhost.c
+++ b/qemu/hw/virtio/vhost.c
@@ -13,11 +13,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/virtio/vhost.h"
 #include "hw/hw.h"
 #include "qemu/atomic.h"
 #include "qemu/range.h"
 #include "qemu/error-report.h"
+#include "qemu/memfd.h"
 #include <linux/vhost.h>
 #include "exec/address-spaces.h"
 #include "hw/virtio/virtio-bus.h"
@@ -25,6 +28,23 @@
 #include "migration/migration.h"
 
 static struct vhost_log *vhost_log;
+static struct vhost_log *vhost_log_shm;
+
+static unsigned int used_memslots;
+static QLIST_HEAD(, vhost_dev) vhost_devices =
+    QLIST_HEAD_INITIALIZER(vhost_devices);
+
+bool vhost_has_free_slot(void)
+{
+    unsigned int slots_limit = ~0U;
+    struct vhost_dev *hdev;
+
+    QLIST_FOREACH(hdev, &vhost_devices, entry) {
+        unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
+        slots_limit = MIN(slots_limit, r);
+    }
+    return slots_limit > used_memslots;
+}
 
 static void vhost_dev_sync_region(struct vhost_dev *dev,
                                   MemoryRegionSection *section,
@@ -241,6 +261,13 @@ static void vhost_dev_assign_memory(struct vhost_dev *dev,
             continue;
         }
 
+        if (dev->vhost_ops->vhost_backend_can_merge &&
+            !dev->vhost_ops->vhost_backend_can_merge(dev, uaddr, size,
+                                                     reg->userspace_addr,
+                                                     reg->memory_size)) {
+            continue;
+        }
+
         if (merged) {
             --to;
             assert(to >= 0);
@@ -286,25 +313,46 @@ static uint64_t vhost_get_log_size(struct vhost_dev *dev)
     }
     return log_size;
 }
-static struct vhost_log *vhost_log_alloc(uint64_t size)
+
+static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
 {
-    struct vhost_log *log = g_malloc0(sizeof *log + size * sizeof(*(log->log)));
+    struct vhost_log *log;
+    uint64_t logsize = size * sizeof(*(log->log));
+    int fd = -1;
+
+    log = g_new0(struct vhost_log, 1);
+    if (share) {
+        log->log = qemu_memfd_alloc("vhost-log", logsize,
+                                    F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
+                                    &fd);
+        memset(log->log, 0, logsize);
+    } else {
+        log->log = g_malloc0(logsize);
+    }
 
     log->size = size;
     log->refcnt = 1;
+    log->fd = fd;
 
     return log;
 }
 
-static struct vhost_log *vhost_log_get(uint64_t size)
+static struct vhost_log *vhost_log_get(uint64_t size, bool share)
 {
-    if (!vhost_log || vhost_log->size != size) {
-        vhost_log = vhost_log_alloc(size);
+    struct vhost_log *log = share ? vhost_log_shm : vhost_log;
+
+    if (!log || log->size != size) {
+        log = vhost_log_alloc(size, share);
+        if (share) {
+            vhost_log_shm = log;
+        } else {
+            vhost_log = log;
+        }
     } else {
-        ++vhost_log->refcnt;
+        ++log->refcnt;
     }
 
-    return vhost_log;
+    return log;
 }
 
 static void vhost_log_put(struct vhost_dev *dev, bool sync)
@@ -321,20 +369,35 @@ static void vhost_log_put(struct vhost_dev *dev, bool sync)
         if (dev->log_size && sync) {
             vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
         }
+
         if (vhost_log == log) {
+            g_free(log->log);
             vhost_log = NULL;
+        } else if (vhost_log_shm == log) {
+            qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
+                            log->fd);
+            vhost_log_shm = NULL;
         }
+
         g_free(log);
     }
 }
 
-static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
+static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
 {
-    struct vhost_log *log = vhost_log_get(size);
+    return dev->vhost_ops->vhost_requires_shm_log &&
+           dev->vhost_ops->vhost_requires_shm_log(dev);
+}
+
+static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
+{
+    struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
     uint64_t log_base = (uintptr_t)log->log;
     int r;
 
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_LOG_BASE, &log_base);
+    /* inform backend of log switching, this must be done before
+       releasing the current log, to ensure no logging is lost */
+    r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
     assert(r >= 0);
     vhost_log_put(dev, true);
     dev->log = log;
@@ -457,6 +520,7 @@ static void vhost_set_memory(MemoryListener *listener,
     dev->mem_changed_start_addr = MIN(dev->mem_changed_start_addr, start_addr);
     dev->mem_changed_end_addr = MAX(dev->mem_changed_end_addr, start_addr + size - 1);
     dev->memory_changed = true;
+    used_memslots = dev->mem->nregions;
 }
 
 static bool vhost_section(MemoryRegionSection *section)
@@ -500,7 +564,7 @@ static void vhost_commit(MemoryListener *listener)
     }
 
     if (!dev->log_enabled) {
-        r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
+        r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
         assert(r >= 0);
         dev->memory_changed = false;
         return;
@@ -513,7 +577,7 @@ static void vhost_commit(MemoryListener *listener)
     if (dev->log_size < log_size) {
         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
+    r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
     assert(r >= 0);
     /* To log less, can only decrease log size after table update. */
     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
@@ -581,7 +645,7 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
-    int r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_ADDR, &addr);
+    int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
         return -errno;
     }
@@ -595,19 +659,20 @@ static int vhost_dev_set_features(struct vhost_dev *dev, bool enable_log)
     if (enable_log) {
         features |= 0x1ULL << VHOST_F_LOG_ALL;
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_FEATURES, &features);
+    r = dev->vhost_ops->vhost_set_features(dev, features);
     return r < 0 ? -errno : 0;
 }
 
 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 {
-    int r, t, i;
+    int r, t, i, idx;
     r = vhost_dev_set_features(dev, enable_log);
     if (r < 0) {
         goto err_features;
     }
     for (i = 0; i < dev->nvqs; ++i) {
-        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
                                      enable_log);
         if (r < 0) {
             goto err_vq;
@@ -616,7 +681,8 @@ static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
     return 0;
 err_vq:
     for (; i >= 0; --i) {
-        t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        t = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
                                      dev->log_enabled);
         assert(t >= 0);
     }
@@ -691,6 +757,27 @@ static void vhost_log_stop(MemoryListener *listener,
     /* FIXME: implement */
 }
 
+/* The vhost driver natively knows how to handle the vrings of non
+ * cross-endian legacy devices and modern devices. Only legacy devices
+ * exposed to a bi-endian guest may require the vhost driver to use a
+ * specific endianness.
+ */
+static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
+{
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return false;
+    }
+#ifdef TARGET_IS_BIENDIAN
+#ifdef HOST_WORDS_BIGENDIAN
+    return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
+#else
+    return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
+#endif
+#else
+    return false;
+#endif
+}
+
 static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
                                                    bool is_big_endian,
                                                    int vhost_vq_index)
@@ -700,7 +787,7 @@ static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
         .num = is_big_endian
     };
 
-    if (!dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_ENDIAN, &s)) {
+    if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
         return 0;
     }
 
@@ -719,7 +806,7 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
 {
     hwaddr s, l, a;
     int r;
-    int vhost_vq_index = idx - dev->vq_index;
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_file file = {
         .index = vhost_vq_index
     };
@@ -728,22 +815,20 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
     };
     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 
     vq->num = state.num = virtio_queue_get_num(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_NUM, &state);
+    r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
     if (r) {
         return -errno;
     }
 
     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_BASE, &state);
+    r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
     if (r) {
         return -errno;
     }
 
-    if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1) &&
-        virtio_legacy_is_cross_endian(vdev)) {
+    if (vhost_needs_vring_endian(vdev)) {
         r = vhost_virtqueue_set_vring_endian_legacy(dev,
                                                     virtio_is_big_endian(vdev),
                                                     vhost_vq_index);
@@ -789,7 +874,7 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
     }
 
     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_KICK, &file);
+    r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
     if (r) {
         r = -errno;
         goto fail_kick;
@@ -798,6 +883,14 @@ static int vhost_virtqueue_start(struct vhost_dev *dev,
     /* Clear and discard previous events if any. */
     event_notifier_test_and_clear(&vq->masked_notifier);
 
+    /* Init vring in unmasked state, unless guest_notifier_mask
+     * will do it later.
+     */
+    if (!vdev->use_guest_notifier_mask) {
+        /* TODO: check and handle errors. */
+        vhost_virtqueue_mask(dev, vdev, idx, false);
+    }
+
     return 0;
 
 fail_kick:
@@ -822,13 +915,13 @@ static void vhost_virtqueue_stop(struct vhost_dev *dev,
                                     struct vhost_virtqueue *vq,
                                     unsigned idx)
 {
-    int vhost_vq_index = idx - dev->vq_index;
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_state state = {
         .index = vhost_vq_index,
     };
     int r;
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_GET_VRING_BASE, &state);
+
+    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
         fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
         fflush(stderr);
@@ -839,8 +932,7 @@ static void vhost_virtqueue_stop(struct vhost_dev *dev,
     /* In the cross-endian case, we need to reset the vring endianness to
      * native as legacy devices expect so by default.
      */
-    if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1) &&
-        virtio_legacy_is_cross_endian(vdev)) {
+    if (vhost_needs_vring_endian(vdev)) {
         r = vhost_virtqueue_set_vring_endian_legacy(dev,
                                                     !virtio_is_big_endian(vdev),
                                                     vhost_vq_index);
@@ -875,8 +967,9 @@ static void vhost_eventfd_del(MemoryListener *listener,
 static int vhost_virtqueue_init(struct vhost_dev *dev,
                                 struct vhost_virtqueue *vq, int n)
 {
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
     struct vhost_vring_file file = {
-        .index = n,
+        .index = vhost_vq_index,
     };
     int r = event_notifier_init(&vq->masked_notifier, 0);
     if (r < 0) {
@@ -884,7 +977,7 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
     }
 
     file.fd = event_notifier_get_fd(&vq->masked_notifier);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_CALL, &file);
+    r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
     if (r) {
         r = -errno;
         goto fail_call;
@@ -906,6 +999,8 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
     uint64_t features;
     int i, r;
 
+    hdev->migration_blocker = NULL;
+
     if (vhost_set_backend_type(hdev, backend_type) < 0) {
         close((uintptr_t)opaque);
         return -1;
@@ -916,18 +1011,26 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
         return -errno;
     }
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_OWNER, NULL);
+    if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
+        fprintf(stderr, "vhost backend memory slots limit is less"
+                " than current number of present memory slots\n");
+        close((uintptr_t)opaque);
+        return -1;
+    }
+    QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
+
+    r = hdev->vhost_ops->vhost_set_owner(hdev);
     if (r < 0) {
         goto fail;
     }
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_GET_FEATURES, &features);
+    r = hdev->vhost_ops->vhost_get_features(hdev, &features);
     if (r < 0) {
         goto fail;
     }
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
+        r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
         if (r < 0) {
             goto fail_vq;
         }
@@ -949,12 +1052,21 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
         .eventfd_del = vhost_eventfd_del,
         .priority = 10
     };
-    hdev->migration_blocker = NULL;
-    if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
-        error_setg(&hdev->migration_blocker,
-                   "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+
+    if (hdev->migration_blocker == NULL) {
+        if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+        } else if (!qemu_memfd_check()) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: failed to allocate shared memory");
+        }
+    }
+
+    if (hdev->migration_blocker != NULL) {
         migrate_add_blocker(hdev->migration_blocker);
     }
+
     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
     hdev->n_mem_sections = 0;
     hdev->mem_sections = NULL;
@@ -972,6 +1084,7 @@ fail_vq:
 fail:
     r = -errno;
     hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    QLIST_REMOVE(hdev, entry);
     return r;
 }
 
@@ -989,6 +1102,7 @@ void vhost_dev_cleanup(struct vhost_dev *hdev)
     g_free(hdev->mem);
     g_free(hdev->mem_sections);
     hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    QLIST_REMOVE(hdev, entry);
 }
 
 /* Stop processing guest IO notifications in qemu.
@@ -1066,18 +1180,17 @@ void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
 {
     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
     int r, index = n - hdev->vq_index;
+    struct vhost_vring_file file;
 
-    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
-
-    struct vhost_vring_file file = {
-        .index = index
-    };
     if (mask) {
+        assert(vdev->use_guest_notifier_mask);
         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
     } else {
         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_VRING_CALL, &file);
+
+    file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
+    r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
     assert(r >= 0);
 }
 
@@ -1119,7 +1232,7 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
     if (r < 0) {
         goto fail_features;
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_MEM_TABLE, hdev->mem);
+    r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
     if (r < 0) {
         r = -errno;
         goto fail_mem;
@@ -1138,10 +1251,12 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         uint64_t log_base;
 
         hdev->log_size = vhost_get_log_size(hdev);
-        hdev->log = vhost_log_get(hdev->log_size);
+        hdev->log = vhost_log_get(hdev->log_size,
+                                  vhost_dev_log_is_shared(hdev));
         log_base = (uintptr_t)hdev->log->log;
-        r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_LOG_BASE,
-                                        hdev->log_size ? &log_base : NULL);
+        r = hdev->vhost_ops->vhost_set_log_base(hdev,
+                                                hdev->log_size ? log_base : 0,
+                                                hdev->log);
         if (r < 0) {
             r = -errno;
             goto fail_log;
diff --git a/qemu/hw/virtio/virtio-balloon.c b/qemu/hw/virtio/virtio-balloon.c
index 3577b7af9..9dbe68179 100644
--- a/qemu/hw/virtio/virtio-balloon.c
+++ b/qemu/hw/virtio/virtio-balloon.c
@@ -13,12 +13,12 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
 #include "hw/virtio/virtio.h"
 #include "hw/i386/pc.h"
-#include "cpu.h"
 #include "sysemu/balloon.h"
 #include "hw/virtio/virtio-balloon.h"
 #include "sysemu/kvm.h"
@@ -34,12 +34,16 @@
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 
+#define BALLOON_PAGE_SIZE  (1 << VIRTIO_BALLOON_PFN_SHIFT)
+
 static void balloon_page(void *addr, int deflate)
 {
 #if defined(__linux__)
-    if (!kvm_enabled() || kvm_has_sync_mmu())
-        qemu_madvise(addr, TARGET_PAGE_SIZE,
+    if (!qemu_balloon_is_inhibited() && (!kvm_enabled() ||
+                                         kvm_has_sync_mmu())) {
+        qemu_madvise(addr, BALLOON_PAGE_SIZE,
                 deflate ? QEMU_MADV_WILLNEED : QEMU_MADV_DONTNEED);
+    }
 #endif
 }
 
@@ -50,6 +54,7 @@ static const char *balloon_stat_names[] = {
    [VIRTIO_BALLOON_S_MINFLT] = "stat-minor-faults",
    [VIRTIO_BALLOON_S_MEMFREE] = "stat-free-memory",
    [VIRTIO_BALLOON_S_MEMTOT] = "stat-total-memory",
+   [VIRTIO_BALLOON_S_AVAIL] = "stat-available-memory",
    [VIRTIO_BALLOON_S_NR] = NULL
 };
 
@@ -70,7 +75,7 @@ static inline void reset_stats(VirtIOBalloon *dev)
 static bool balloon_stats_supported(const VirtIOBalloon *s)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
-    return virtio_has_feature(vdev, VIRTIO_BALLOON_F_STATS_VQ);
+    return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_STATS_VQ);
 }
 
 static bool balloon_stats_enabled(const VirtIOBalloon *s)
@@ -98,39 +103,43 @@ static void balloon_stats_poll_cb(void *opaque)
     VirtIOBalloon *s = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
-    if (!balloon_stats_supported(s)) {
+    if (s->stats_vq_elem == NULL || !balloon_stats_supported(s)) {
         /* re-schedule */
         balloon_stats_change_timer(s, s->stats_poll_interval);
         return;
     }
 
-    virtqueue_push(s->svq, &s->stats_vq_elem, s->stats_vq_offset);
+    virtqueue_push(s->svq, s->stats_vq_elem, s->stats_vq_offset);
     virtio_notify(vdev, s->svq);
+    g_free(s->stats_vq_elem);
+    s->stats_vq_elem = NULL;
 }
 
-static void balloon_stats_get_all(Object *obj, struct Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void balloon_stats_get_all(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     Error *err = NULL;
     VirtIOBalloon *s = opaque;
     int i;
 
-    visit_start_struct(v, NULL, "guest-stats", name, 0, &err);
+    visit_start_struct(v, name, NULL, 0, &err);
     if (err) {
         goto out;
     }
-    visit_type_int(v, &s->stats_last_update, "last-update", &err);
+    visit_type_int(v, "last-update", &s->stats_last_update, &err);
     if (err) {
         goto out_end;
     }
 
-    visit_start_struct(v, NULL, NULL, "stats", 0, &err);
+    visit_start_struct(v, "stats", NULL, 0, &err);
     if (err) {
         goto out_end;
     }
-    for (i = 0; !err && i < VIRTIO_BALLOON_S_NR; i++) {
-        visit_type_int64(v, (int64_t *) &s->stats[i], balloon_stat_names[i],
-                         &err);
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; i++) {
+        visit_type_uint64(v, balloon_stat_names[i], &s->stats[i], &err);
+        if (err) {
+            break;
+        }
     }
     error_propagate(errp, err);
     err = NULL;
@@ -144,23 +153,23 @@ out:
     error_propagate(errp, err);
 }
 
-static void balloon_stats_get_poll_interval(Object *obj, struct Visitor *v,
-                                            void *opaque, const char *name,
+static void balloon_stats_get_poll_interval(Object *obj, Visitor *v,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     VirtIOBalloon *s = opaque;
-    visit_type_int(v, &s->stats_poll_interval, name, errp);
+    visit_type_int(v, name, &s->stats_poll_interval, errp);
 }
 
-static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
-                                            void *opaque, const char *name,
+static void balloon_stats_set_poll_interval(Object *obj, Visitor *v,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     VirtIOBalloon *s = opaque;
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -203,14 +212,18 @@ static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
 static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     MemoryRegionSection section;
 
-    while (virtqueue_pop(vq, &elem)) {
+    for (;;) {
         size_t offset = 0;
         uint32_t pfn;
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            return;
+        }
 
-        while (iov_to_buf(elem.out_sg, elem.out_num, offset, &pfn, 4) == 4) {
+        while (iov_to_buf(elem->out_sg, elem->out_num, offset, &pfn, 4) == 4) {
             ram_addr_t pa;
             ram_addr_t addr;
             int p = virtio_ldl_p(vdev, &pfn);
@@ -233,23 +246,34 @@ static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
             memory_region_unref(section.mr);
         }
 
-        virtqueue_push(vq, &elem, offset);
+        virtqueue_push(vq, elem, offset);
         virtio_notify(vdev, vq);
+        g_free(elem);
     }
 }
 
 static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
-    VirtQueueElement *elem = &s->stats_vq_elem;
+    VirtQueueElement *elem;
     VirtIOBalloonStat stat;
     size_t offset = 0;
     qemu_timeval tv;
 
-    if (!virtqueue_pop(vq, elem)) {
+    elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+    if (!elem) {
         goto out;
     }
 
+    if (s->stats_vq_elem != NULL) {
+        /* This should never happen if the driver follows the spec. */
+        virtqueue_push(vq, s->stats_vq_elem, 0);
+        virtio_notify(vdev, vq);
+        g_free(s->stats_vq_elem);
+    }
+
+    s->stats_vq_elem = elem;
+
     /* Initialize the stats to get rid of any stale values.  This is only
      * needed to handle the case where a guest supports fewer stats than it
      * used to (ie. it has booted into an old kernel).
@@ -292,6 +316,39 @@ static void virtio_balloon_get_config(VirtIODevice *vdev, uint8_t *config_data)
     memcpy(config_data, &config, sizeof(struct virtio_balloon_config));
 }
 
+static int build_dimm_list(Object *obj, void *opaque)
+{
+    GSList **list = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
+        DeviceState *dev = DEVICE(obj);
+        if (dev->realized) { /* only realized DIMMs matter */
+            *list = g_slist_prepend(*list, dev);
+        }
+    }
+
+    object_child_foreach(obj, build_dimm_list, opaque);
+    return 0;
+}
+
+static ram_addr_t get_current_ram_size(void)
+{
+    GSList *list = NULL, *item;
+    ram_addr_t size = ram_size;
+
+    build_dimm_list(qdev_get_machine(), &list);
+    for (item = list; item; item = g_slist_next(item)) {
+        Object *obj = OBJECT(item->data);
+        if (!strcmp(object_get_typename(obj), TYPE_PC_DIMM)) {
+            size += object_property_get_int(obj, PC_DIMM_SIZE_PROP,
+                                            &error_abort);
+        }
+    }
+    g_slist_free(list);
+
+    return size;
+}
+
 static void virtio_balloon_set_config(VirtIODevice *vdev,
                                       const uint8_t *config_data)
 {
@@ -370,6 +427,10 @@ static int virtio_balloon_load_device(VirtIODevice *vdev, QEMUFile *f,
 
     s->num_pages = qemu_get_be32(f);
     s->actual = qemu_get_be32(f);
+
+    if (balloon_stats_enabled(s)) {
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+    }
     return 0;
 }
 
@@ -412,6 +473,16 @@ static void virtio_balloon_device_unrealize(DeviceState *dev, Error **errp)
     virtio_cleanup(vdev);
 }
 
+static void virtio_balloon_device_reset(VirtIODevice *vdev)
+{
+    VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
+
+    if (s->stats_vq_elem != NULL) {
+        g_free(s->stats_vq_elem);
+        s->stats_vq_elem = NULL;
+    }
+}
+
 static void virtio_balloon_instance_init(Object *obj)
 {
     VirtIOBalloon *s = VIRTIO_BALLOON(obj);
@@ -440,6 +511,7 @@ static void virtio_balloon_class_init(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     vdc->realize = virtio_balloon_device_realize;
     vdc->unrealize = virtio_balloon_device_unrealize;
+    vdc->reset = virtio_balloon_device_reset;
     vdc->get_config = virtio_balloon_get_config;
     vdc->set_config = virtio_balloon_set_config;
     vdc->get_features = virtio_balloon_get_features;
diff --git a/qemu/hw/virtio/virtio-bus.c b/qemu/hw/virtio/virtio-bus.c
index febda76b9..574f0e23f 100644
--- a/qemu/hw/virtio/virtio-bus.c
+++ b/qemu/hw/virtio/virtio-bus.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/error-report.h"
 #include "hw/qdev.h"
@@ -56,6 +57,9 @@ void virtio_bus_device_plugged(VirtIODevice *vdev, Error **errp)
     assert(vdc->get_features != NULL);
     vdev->host_features = vdc->get_features(vdev, vdev->host_features,
                                             errp);
+    if (klass->post_plugged != NULL) {
+        klass->post_plugged(qbus->parent, errp);
+    }
 }
 
 /* Reset the virtio_bus */
diff --git a/qemu/hw/virtio/virtio-mmio.c b/qemu/hw/virtio/virtio-mmio.c
index 18660b07b..d4cd91f8c 100644
--- a/qemu/hw/virtio/virtio-mmio.c
+++ b/qemu/hw/virtio/virtio-mmio.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/virtio/virtio.h"
 #include "qemu/host-utils.h"
diff --git a/qemu/hw/virtio/virtio-pci.c b/qemu/hw/virtio/virtio-pci.c
index c024161f5..bfedbbf17 100644
--- a/qemu/hw/virtio/virtio-pci.c
+++ b/qemu/hw/virtio/virtio-pci.c
@@ -15,7 +15,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "standard-headers/linux/virtio_pci.h"
 #include "hw/virtio/virtio.h"
@@ -26,6 +26,7 @@
 #include "hw/virtio/virtio-balloon.h"
 #include "hw/virtio/virtio-input.h"
 #include "hw/pci/pci.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
@@ -47,6 +48,7 @@
 
 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtIOPCIProxy *dev);
+static void virtio_pci_reset(DeviceState *qdev);
 
 /* virtio device */
 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
@@ -86,6 +88,129 @@ static void virtio_pci_save_config(DeviceState *d, QEMUFile *f)
         qemu_put_be16(f, vdev->config_vector);
 }
 
+static void virtio_pci_load_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    vq->num = qemu_get_be16(f);
+    vq->enabled = qemu_get_be16(f);
+    vq->desc[0] = qemu_get_be32(f);
+    vq->desc[1] = qemu_get_be32(f);
+    vq->avail[0] = qemu_get_be32(f);
+    vq->avail[1] = qemu_get_be32(f);
+    vq->used[0] = qemu_get_be32(f);
+    vq->used[1] = qemu_get_be32(f);
+}
+
+static bool virtio_pci_has_extra_state(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
+}
+
+static int get_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    proxy->dfselect = qemu_get_be32(f);
+    proxy->gfselect = qemu_get_be32(f);
+    proxy->guest_features[0] = qemu_get_be32(f);
+    proxy->guest_features[1] = qemu_get_be32(f);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_load_modern_queue_state(&proxy->vqs[i], f);
+    }
+
+    return 0;
+}
+
+static void virtio_pci_save_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    qemu_put_be16(f, vq->num);
+    qemu_put_be16(f, vq->enabled);
+    qemu_put_be32(f, vq->desc[0]);
+    qemu_put_be32(f, vq->desc[1]);
+    qemu_put_be32(f, vq->avail[0]);
+    qemu_put_be32(f, vq->avail[1]);
+    qemu_put_be32(f, vq->used[0]);
+    qemu_put_be32(f, vq->used[1]);
+}
+
+static void put_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    qemu_put_be32(f, proxy->dfselect);
+    qemu_put_be32(f, proxy->gfselect);
+    qemu_put_be32(f, proxy->guest_features[0]);
+    qemu_put_be32(f, proxy->guest_features[1]);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_save_modern_queue_state(&proxy->vqs[i], f);
+    }
+}
+
+static const VMStateInfo vmstate_info_virtio_pci_modern_state = {
+    .name = "virtqueue_state",
+    .get = get_virtio_pci_modern_state,
+    .put = put_virtio_pci_modern_state,
+};
+
+static bool virtio_pci_modern_state_needed(void *opaque)
+{
+    VirtIOPCIProxy *proxy = opaque;
+
+    return !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+}
+
+static const VMStateDescription vmstate_virtio_pci_modern_state = {
+    .name = "virtio_pci/modern_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_pci_modern_state_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "modern_state",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_virtio_pci_modern_state,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_pci = {
+    .name = "virtio_pci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_virtio_pci_modern_state,
+        NULL
+    }
+};
+
+static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
+}
+
+static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
+}
+
 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 {
     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
@@ -133,6 +258,7 @@ static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)
     if (vector != VIRTIO_NO_VECTOR) {
         return msix_vector_use(&proxy->pci_dev, vector);
     }
+
     return 0;
 }
 
@@ -146,7 +272,10 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
     bool legacy = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_LEGACY);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool fast_mmio = kvm_ioeventfd_any_length_enabled();
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
     MemoryRegion *modern_mr = &proxy->notify.mr;
+    MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
     MemoryRegion *legacy_mr = &proxy->bar;
     hwaddr modern_addr = QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
                          virtio_get_queue_index(vq);
@@ -162,8 +291,17 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
         }
         virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
         if (modern) {
-            memory_region_add_eventfd(modern_mr, modern_addr, 2,
-                                      true, n, notifier);
+            if (fast_mmio) {
+                memory_region_add_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_add_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_add_eventfd(modern_notify_mr, 0, 2,
+                                              true, n, notifier);
+            }
         }
         if (legacy) {
             memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
@@ -171,8 +309,17 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
         }
     } else {
         if (modern) {
-            memory_region_del_eventfd(modern_mr, modern_addr, 2,
-                                      true, n, notifier);
+            if (fast_mmio) {
+                memory_region_del_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_del_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_del_eventfd(modern_notify_mr, 0, 2,
+                                          true, n, notifier);
+            }
         }
         if (legacy) {
             memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
@@ -259,9 +406,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     case VIRTIO_PCI_QUEUE_PFN:
         pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
         if (pa == 0) {
-            virtio_pci_stop_ioeventfd(proxy);
-            virtio_reset(vdev);
-            msix_unuse_all_vectors(&proxy->pci_dev);
+            virtio_pci_reset(DEVICE(proxy));
         }
         else
             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
@@ -287,8 +432,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         }
 
         if (vdev->status == 0) {
-            virtio_reset(vdev);
-            msix_unuse_all_vectors(&proxy->pci_dev);
+            virtio_pci_reset(DEVICE(proxy));
         }
 
         /* Linux before 2.6.34 drives the device without enabling
@@ -590,7 +734,7 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
     int ret;
 
     if (irqfd->users == 0) {
-        ret = kvm_irqchip_add_msi_route(kvm_state, msg);
+        ret = kvm_irqchip_add_msi_route(kvm_state, msg, &proxy->pci_dev);
         if (ret < 0) {
             return ret;
         }
@@ -661,7 +805,7 @@ static int kvm_virtio_pci_vector_use(VirtIOPCIProxy *proxy, int nvqs)
         /* If guest supports masking, set up irqfd now.
          * Otherwise, delay until unmasked in the frontend.
          */
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
             if (ret < 0) {
                 kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -677,7 +821,7 @@ undo:
         if (vector >= msix_nr_vectors_allocated(dev)) {
             continue;
         }
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
         }
         kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -704,7 +848,7 @@ static void kvm_virtio_pci_vector_release(VirtIOPCIProxy *proxy, int nvqs)
         /* If guest supports masking, clean up irqfd now.
          * Otherwise, it was cleaned when masked in the frontend.
          */
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
         }
         kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -726,7 +870,8 @@ static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
     if (proxy->vector_irqfd) {
         irqfd = &proxy->vector_irqfd[vector];
         if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
-            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg);
+            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
+                                               &proxy->pci_dev);
             if (ret < 0) {
                 return ret;
             }
@@ -736,7 +881,7 @@ static int virtio_pci_vq_vector_unmask(VirtIOPCIProxy *proxy,
     /* If guest supports masking, irqfd is already setup, unmask it.
      * Otherwise, set it up now.
      */
-    if (k->guest_notifier_mask) {
+    if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
         k->guest_notifier_mask(vdev, queue_no, false);
         /* Test after unmasking to avoid losing events. */
         if (k->guest_notifier_pending &&
@@ -759,7 +904,7 @@ static void virtio_pci_vq_vector_mask(VirtIOPCIProxy *proxy,
     /* If guest supports masking, keep irqfd but mask it.
      * Otherwise, clean it up now.
      */ 
-    if (k->guest_notifier_mask) {
+    if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
         k->guest_notifier_mask(vdev, queue_no, true);
     } else {
         kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
@@ -876,7 +1021,9 @@ static int virtio_pci_set_guest_notifier(DeviceState *d, int n, bool assign,
         event_notifier_cleanup(notifier);
     }
 
-    if (!msix_enabled(&proxy->pci_dev) && vdc->guest_notifier_mask) {
+    if (!msix_enabled(&proxy->pci_dev) &&
+        vdev->use_guest_notifier_mask &&
+        vdc->guest_notifier_mask) {
         vdc->guest_notifier_mask(vdev, n, !assign);
     }
 
@@ -1205,8 +1352,7 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr,
         }
 
         if (vdev->status == 0) {
-            virtio_reset(vdev);
-            msix_unuse_all_vectors(&proxy->pci_dev);
+            virtio_pci_reset(DEVICE(proxy));
         }
 
         break;
@@ -1238,6 +1384,7 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr,
                        proxy->vqs[vdev->queue_sel].avail[0],
                        ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
                        proxy->vqs[vdev->queue_sel].used[0]);
+        proxy->vqs[vdev->queue_sel].enabled = 1;
         break;
     case VIRTIO_PCI_COMMON_Q_DESCLO:
         proxy->vqs[vdev->queue_sel].desc[0] = val;
@@ -1280,6 +1427,17 @@ static void virtio_pci_notify_write(void *opaque, hwaddr addr,
     }
 }
 
+static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
+                                        uint64_t val, unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    unsigned queue = val;
+
+    if (queue < VIRTIO_QUEUE_MAX) {
+        virtio_queue_notify(vdev, queue);
+    }
+}
+
 static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
@@ -1373,6 +1531,16 @@ static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy)
         },
         .endianness = DEVICE_LITTLE_ENDIAN,
     };
+    static const MemoryRegionOps notify_pio_ops = {
+        .read = virtio_pci_notify_read,
+        .write = virtio_pci_notify_write_pio,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+
 
     memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
                           &common_ops,
@@ -1397,30 +1565,60 @@ static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy)
                           virtio_bus_get_device(&proxy->bus),
                           "virtio-pci-notify",
                           proxy->notify.size);
+
+    memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
+                          &notify_pio_ops,
+                          virtio_bus_get_device(&proxy->bus),
+                          "virtio-pci-notify-pio",
+                          proxy->notify.size);
 }
 
 static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
                                          VirtIOPCIRegion *region,
-                                         struct virtio_pci_cap *cap)
+                                         struct virtio_pci_cap *cap,
+                                         MemoryRegion *mr,
+                                         uint8_t bar)
 {
-    memory_region_add_subregion(&proxy->modern_bar,
-                                region->offset,
-                                &region->mr);
+    memory_region_add_subregion(mr, region->offset, &region->mr);
 
     cap->cfg_type = region->type;
-    cap->bar = proxy->modern_mem_bar;
+    cap->bar = bar;
     cap->offset = cpu_to_le32(region->offset);
     cap->length = cpu_to_le32(region->size);
     virtio_pci_add_mem_cap(proxy, cap);
+
+}
+
+static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
+                                             VirtIOPCIRegion *region,
+                                             struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->modern_bar, proxy->modern_mem_bar);
 }
 
-static void virtio_pci_modern_region_unmap(VirtIOPCIProxy *proxy,
-                                           VirtIOPCIRegion *region)
+static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
+                                            VirtIOPCIRegion *region,
+                                            struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->io_bar, proxy->modern_io_bar);
+}
+
+static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
+                                               VirtIOPCIRegion *region)
 {
     memory_region_del_subregion(&proxy->modern_bar,
                                 &region->mr);
 }
 
+static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
+                                              VirtIOPCIRegion *region)
+{
+    memory_region_del_subregion(&proxy->io_bar,
+                                &region->mr);
+}
+
 /* This is called by virtio-bus just after the device is plugged. */
 static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
 {
@@ -1428,6 +1626,7 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
     VirtioBusState *bus = &proxy->bus;
     bool legacy = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_LEGACY);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
     uint8_t *config;
     uint32_t size;
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
@@ -1466,16 +1665,31 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
             .cap.cap_len = sizeof cfg,
             .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
         };
-        struct virtio_pci_cfg_cap *cfg_mask;
+        struct virtio_pci_notify_cap notify_pio = {
+            .cap.cap_len = sizeof notify,
+            .notify_off_multiplier = cpu_to_le32(0x0),
+        };
 
-        /* TODO: add io access for speed */
+        struct virtio_pci_cfg_cap *cfg_mask;
 
         virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
         virtio_pci_modern_regions_init(proxy);
-        virtio_pci_modern_region_map(proxy, &proxy->common, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->isr, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->device, &cap);
-        virtio_pci_modern_region_map(proxy, &proxy->notify, &notify.cap);
+
+        virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
+
+        if (modern_pio) {
+            memory_region_init(&proxy->io_bar, OBJECT(proxy),
+                               "virtio-pci-io", 0x4);
+
+            pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar,
+                             PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
+
+            virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
+                                            &notify_pio.cap);
+        }
 
         pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar,
                          PCI_BASE_ADDRESS_SPACE_MEMORY |
@@ -1491,12 +1705,17 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
         pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
     }
 
-    if (proxy->nvectors &&
-        msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
-                                proxy->msix_bar)) {
-        error_report("unable to init msix vectors to %" PRIu32,
-                     proxy->nvectors);
-        proxy->nvectors = 0;
+    if (proxy->nvectors) {
+        int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
+                                          proxy->msix_bar);
+        if (err) {
+            /* Notice when a system that supports MSIx can't initialize it.  */
+            if (err != -ENOTSUP) {
+                error_report("unable to init msix vectors to %" PRIu32,
+                             proxy->nvectors);
+            }
+            proxy->nvectors = 0;
+        }
     }
 
     proxy->pci_dev.config_write = virtio_write_config;
@@ -1505,9 +1724,7 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
     if (legacy) {
         size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
             + virtio_bus_get_vdev_config_len(bus);
-        if (size & (size - 1)) {
-            size = 1 << qemu_fls(size);
-        }
+        size = pow2ceil(size);
 
         memory_region_init_io(&proxy->bar, OBJECT(proxy),
                               &virtio_pci_config_ops,
@@ -1528,14 +1745,18 @@ static void virtio_pci_device_unplugged(DeviceState *d)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
     bool modern = !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 
     virtio_pci_stop_ioeventfd(proxy);
 
     if (modern) {
-        virtio_pci_modern_region_unmap(proxy, &proxy->common);
-        virtio_pci_modern_region_unmap(proxy, &proxy->isr);
-        virtio_pci_modern_region_unmap(proxy, &proxy->device);
-        virtio_pci_modern_region_unmap(proxy, &proxy->notify);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
+        if (modern_pio) {
+            virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
+        }
     }
 }
 
@@ -1555,6 +1776,7 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
      */
     proxy->legacy_io_bar  = 0;
     proxy->msix_bar       = 1;
+    proxy->modern_io_bar  = 2;
     proxy->modern_mem_bar = 4;
 
     proxy->common.offset = 0x0;
@@ -1574,6 +1796,10 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
         QEMU_VIRTIO_PCI_QUEUE_MEM_MULT * VIRTIO_QUEUE_MAX;
     proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
 
+    proxy->notify_pio.offset = 0x0;
+    proxy->notify_pio.size = 0x4;
+    proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
+
     /* subclasses can enforce modern, so do this unconditionally */
     memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
                        2 * QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
@@ -1588,6 +1814,29 @@ static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
 
     address_space_init(&proxy->modern_as, &proxy->modern_cfg, "virtio-pci-cfg-as");
 
+    if (pci_is_express(pci_dev) && pci_bus_is_express(pci_dev->bus) &&
+        !pci_bus_is_root(pci_dev->bus)) {
+        int pos;
+
+        pos = pcie_endpoint_cap_init(pci_dev, 0);
+        assert(pos > 0);
+
+        pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0, PCI_PM_SIZEOF);
+        assert(pos > 0);
+
+        /*
+         * Indicates that this function complies with revision 1.2 of the
+         * PCI Power Management Interface Specification.
+         */
+        pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
+    } else {
+        /*
+         * make future invocations of pci_is_express() return false
+         * and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
+         */
+        pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+    }
+
     virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
     if (k->realize) {
         k->realize(proxy, errp);
@@ -1606,9 +1855,15 @@ static void virtio_pci_reset(DeviceState *qdev)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
     VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    int i;
+
     virtio_pci_stop_ioeventfd(proxy);
     virtio_bus_reset(bus);
     msix_unuse_all_vectors(&proxy->pci_dev);
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        proxy->vqs[i].enabled = 0;
+    }
 }
 
 static Property virtio_pci_properties[] = {
@@ -1618,13 +1873,34 @@ static Property virtio_pci_properties[] = {
                     VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT, false),
     DEFINE_PROP_BIT("disable-modern", VirtIOPCIProxy, flags,
                     VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT, true),
+    DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
+    DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
+{
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
+    PCIDevice *pci_dev = &proxy->pci_dev;
+
+    if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
+        !(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_MODERN)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    vpciklass->parent_dc_realize(qdev, errp);
+}
+
 static void virtio_pci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
 
     dc->props = virtio_pci_properties;
     k->realize = virtio_pci_realize;
@@ -1632,6 +1908,8 @@ static void virtio_pci_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     k->revision = VIRTIO_PCI_ABI_VERSION;
     k->class_id = PCI_CLASS_OTHERS;
+    vpciklass->parent_dc_realize = dc->realize;
+    dc->realize = virtio_pci_dc_realize;
     dc->reset = virtio_pci_reset;
 }
 
@@ -2009,10 +2287,6 @@ static const TypeInfo virtio_net_pci_info = {
 
 /* virtio-rng-pci */
 
-static Property virtio_rng_pci_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void virtio_rng_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIORngPCI *vrng = VIRTIO_RNG_PCI(vpci_dev);
@@ -2039,7 +2313,6 @@ static void virtio_rng_pci_class_init(ObjectClass *klass, void *data)
 
     k->realize = virtio_rng_pci_realize;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    dc->props = virtio_rng_pci_properties;
 
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_RNG;
@@ -2136,14 +2409,6 @@ static void virtio_tablet_initfn(Object *obj)
                                 TYPE_VIRTIO_TABLET);
 }
 
-static void virtio_host_initfn(Object *obj)
-{
-    VirtIOInputHostPCI *dev = VIRTIO_INPUT_HOST_PCI(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_INPUT_HOST);
-}
-
 static const TypeInfo virtio_input_pci_info = {
     .name          = TYPE_VIRTIO_INPUT_PCI,
     .parent        = TYPE_VIRTIO_PCI,
@@ -2182,12 +2447,22 @@ static const TypeInfo virtio_tablet_pci_info = {
     .instance_init = virtio_tablet_initfn,
 };
 
+#ifdef CONFIG_LINUX
+static void virtio_host_initfn(Object *obj)
+{
+    VirtIOInputHostPCI *dev = VIRTIO_INPUT_HOST_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_INPUT_HOST);
+}
+
 static const TypeInfo virtio_host_pci_info = {
     .name          = TYPE_VIRTIO_INPUT_HOST_PCI,
     .parent        = TYPE_VIRTIO_INPUT_PCI,
     .instance_size = sizeof(VirtIOInputHostPCI),
     .instance_init = virtio_host_initfn,
 };
+#endif
 
 /* virtio-pci-bus */
 
@@ -2211,6 +2486,9 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
     k->load_config = virtio_pci_load_config;
     k->save_queue = virtio_pci_save_queue;
     k->load_queue = virtio_pci_load_queue;
+    k->save_extra_state = virtio_pci_save_extra_state;
+    k->load_extra_state = virtio_pci_load_extra_state;
+    k->has_extra_state = virtio_pci_has_extra_state;
     k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
     k->set_host_notifier = virtio_pci_set_host_notifier;
     k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
@@ -2235,7 +2513,9 @@ static void virtio_pci_register_types(void)
     type_register_static(&virtio_keyboard_pci_info);
     type_register_static(&virtio_mouse_pci_info);
     type_register_static(&virtio_tablet_pci_info);
+#ifdef CONFIG_LINUX
     type_register_static(&virtio_host_pci_info);
+#endif
     type_register_static(&virtio_pci_bus_info);
     type_register_static(&virtio_pci_info);
 #ifdef CONFIG_VIRTFS
diff --git a/qemu/hw/virtio/virtio-pci.h b/qemu/hw/virtio/virtio-pci.h
index b6c442f52..e4548c2f9 100644
--- a/qemu/hw/virtio/virtio-pci.h
+++ b/qemu/hw/virtio/virtio-pci.h
@@ -23,7 +23,6 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/virtio-balloon.h"
 #include "hw/virtio/virtio-bus.h"
-#include "hw/virtio/virtio-9p.h"
 #include "hw/virtio/virtio-input.h"
 #include "hw/virtio/virtio-gpu.h"
 #ifdef CONFIG_VIRTFS
@@ -59,21 +58,35 @@ typedef struct VirtioBusClass VirtioPCIBusClass;
 #define VIRTIO_PCI_BUS_CLASS(klass) \
         OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
+enum {
+    VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT,
+    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT,
+    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT,
+    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT,
+};
+
 /* Need to activate work-arounds for buggy guests at vmstate load. */
-#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT  0
 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION \
     (1 << VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT)
 
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
-#define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
 
 /* virtio version flags */
-#define VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT 2
-#define VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT 3
 #define VIRTIO_PCI_FLAG_DISABLE_LEGACY (1 << VIRTIO_PCI_FLAG_DISABLE_LEGACY_BIT)
 #define VIRTIO_PCI_FLAG_DISABLE_MODERN (1 << VIRTIO_PCI_FLAG_DISABLE_MODERN_BIT)
+#define VIRTIO_PCI_FLAG_DISABLE_PCIE (1 << VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT)
+
+/* migrate extra state */
+#define VIRTIO_PCI_FLAG_MIGRATE_EXTRA (1 << VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT)
+
+/* have pio notification for modern device ? */
+#define VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY \
+    (1 << VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT)
 
 typedef struct {
     MSIMessage msg;
@@ -94,6 +107,7 @@ typedef struct {
 
 typedef struct VirtioPCIClass {
     PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
     void (*realize)(VirtIOPCIProxy *vpci_dev, Error **errp);
 } VirtioPCIClass;
 
@@ -104,6 +118,14 @@ typedef struct VirtIOPCIRegion {
     uint32_t type;
 } VirtIOPCIRegion;
 
+typedef struct VirtIOPCIQueue {
+  uint16_t num;
+  bool enabled;
+  uint32_t desc[2];
+  uint32_t avail[2];
+  uint32_t used[2];
+} VirtIOPCIQueue;
+
 struct VirtIOPCIProxy {
     PCIDevice pci_dev;
     MemoryRegion bar;
@@ -111,11 +133,14 @@ struct VirtIOPCIProxy {
     VirtIOPCIRegion isr;
     VirtIOPCIRegion device;
     VirtIOPCIRegion notify;
+    VirtIOPCIRegion notify_pio;
     MemoryRegion modern_bar;
+    MemoryRegion io_bar;
     MemoryRegion modern_cfg;
     AddressSpace modern_as;
     uint32_t legacy_io_bar;
     uint32_t msix_bar;
+    uint32_t modern_io_bar;
     uint32_t modern_mem_bar;
     int config_cap;
     uint32_t flags;
@@ -124,13 +149,7 @@ struct VirtIOPCIProxy {
     uint32_t dfselect;
     uint32_t gfselect;
     uint32_t guest_features[2];
-    struct {
-        uint16_t num;
-        bool enabled;
-        uint32_t desc[2];
-        uint32_t avail[2];
-        uint32_t used[2];
-    } vqs[VIRTIO_QUEUE_MAX];
+    VirtIOPCIQueue vqs[VIRTIO_QUEUE_MAX];
 
     bool ioeventfd_disabled;
     bool ioeventfd_started;
@@ -226,7 +245,7 @@ struct VirtIONetPCI {
 
 typedef struct V9fsPCIState {
     VirtIOPCIProxy parent_obj;
-    V9fsState vdev;
+    V9fsVirtioState vdev;
 } V9fsPCIState;
 
 #endif
@@ -267,6 +286,8 @@ struct VirtIOInputHIDPCI {
     VirtIOInputHID vdev;
 };
 
+#ifdef CONFIG_LINUX
+
 #define TYPE_VIRTIO_INPUT_HOST_PCI "virtio-input-host-pci"
 #define VIRTIO_INPUT_HOST_PCI(obj) \
         OBJECT_CHECK(VirtIOInputHostPCI, (obj), TYPE_VIRTIO_INPUT_HOST_PCI)
@@ -276,6 +297,8 @@ struct VirtIOInputHostPCI {
     VirtIOInputHost vdev;
 };
 
+#endif
+
 /*
  * virtio-gpu-pci: This extends VirtioPCIProxy.
  */
diff --git a/qemu/hw/virtio/virtio-rng.c b/qemu/hw/virtio/virtio-rng.c
index 97d154191..6b991a764 100644
--- a/qemu/hw/virtio/virtio-rng.c
+++ b/qemu/hw/virtio/virtio-rng.c
@@ -9,6 +9,8 @@
  * top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
 #include "hw/qdev.h"
 #include "hw/virtio/virtio.h"
@@ -43,7 +45,7 @@ static void chr_read(void *opaque, const void *buf, size_t size)
 {
     VirtIORNG *vrng = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(vrng);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     size_t len;
     int offset;
 
@@ -55,17 +57,26 @@ static void chr_read(void *opaque, const void *buf, size_t size)
 
     offset = 0;
     while (offset < size) {
-        if (!virtqueue_pop(vrng->vq, &elem)) {
+        elem = virtqueue_pop(vrng->vq, sizeof(VirtQueueElement));
+        if (!elem) {
             break;
         }
-        len = iov_from_buf(elem.in_sg, elem.in_num,
+        len = iov_from_buf(elem->in_sg, elem->in_num,
                            0, buf + offset, size - offset);
         offset += len;
 
-        virtqueue_push(vrng->vq, &elem, len);
+        virtqueue_push(vrng->vq, elem, len);
         trace_virtio_rng_pushed(vrng, len);
+        g_free(elem);
     }
     virtio_notify(vdev, vrng->vq);
+
+    if (!virtio_queue_empty(vrng->vq)) {
+        /* If we didn't drain the queue, call virtio_rng_process
+         * to take care of asking for more data as appropriate.
+         */
+        virtio_rng_process(vrng);
+    }
 }
 
 static void virtio_rng_process(VirtIORNG *vrng)
diff --git a/qemu/hw/virtio/virtio.c b/qemu/hw/virtio/virtio.c
index 788b556a7..30ede3d1c 100644
--- a/qemu/hw/virtio/virtio.c
+++ b/qemu/hw/virtio/virtio.c
@@ -11,8 +11,10 @@
  *
  */
 
-#include <inttypes.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "trace.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -60,6 +62,7 @@ typedef struct VRingUsed
 typedef struct VRing
 {
     unsigned int num;
+    unsigned int num_default;
     unsigned int align;
     hwaddr desc;
     hwaddr avail;
@@ -69,7 +72,15 @@ typedef struct VRing
 struct VirtQueue
 {
     VRing vring;
+
+    /* Next head to pop */
     uint16_t last_avail_idx;
+
+    /* Last avail_idx read from VQ. */
+    uint16_t shadow_avail_idx;
+
+    uint16_t used_idx;
+
     /* Last used index value we have signalled on */
     uint16_t signalled_used;
 
@@ -85,6 +96,7 @@ struct VirtQueue
 
     uint16_t vector;
     void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
+    void (*handle_aio_output)(VirtIODevice *vdev, VirtQueue *vq);
     VirtIODevice *vdev;
     EventNotifier guest_notifier;
     EventNotifier host_notifier;
@@ -106,35 +118,15 @@ void virtio_queue_update_rings(VirtIODevice *vdev, int n)
                               vring->align);
 }
 
-static inline uint64_t vring_desc_addr(VirtIODevice *vdev, hwaddr desc_pa,
-                                       int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
-    return virtio_ldq_phys(vdev, pa);
-}
-
-static inline uint32_t vring_desc_len(VirtIODevice *vdev, hwaddr desc_pa, int i)
+static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
+                            hwaddr desc_pa, int i)
 {
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
-    return virtio_ldl_phys(vdev, pa);
-}
-
-static inline uint16_t vring_desc_flags(VirtIODevice *vdev, hwaddr desc_pa,
-                                        int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
-    return virtio_lduw_phys(vdev, pa);
-}
-
-static inline uint16_t vring_desc_next(VirtIODevice *vdev, hwaddr desc_pa,
-                                       int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
-    return virtio_lduw_phys(vdev, pa);
+    address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
+                       MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
+    virtio_tswap64s(vdev, &desc->addr);
+    virtio_tswap32s(vdev, &desc->len);
+    virtio_tswap16s(vdev, &desc->flags);
+    virtio_tswap16s(vdev, &desc->next);
 }
 
 static inline uint16_t vring_avail_flags(VirtQueue *vq)
@@ -148,7 +140,8 @@ static inline uint16_t vring_avail_idx(VirtQueue *vq)
 {
     hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, idx);
-    return virtio_lduw_phys(vq->vdev, pa);
+    vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
+    return vq->shadow_avail_idx;
 }
 
 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
@@ -163,18 +156,15 @@ static inline uint16_t vring_get_used_event(VirtQueue *vq)
     return vring_avail_ring(vq, vq->vring.num);
 }
 
-static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
-{
-    hwaddr pa;
-    pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
-    virtio_stl_phys(vq->vdev, pa, val);
-}
-
-static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
+static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
+                                    int i)
 {
     hwaddr pa;
-    pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
-    virtio_stl_phys(vq->vdev, pa, val);
+    virtio_tswap32s(vq->vdev, &uelem->id);
+    virtio_tswap32s(vq->vdev, &uelem->len);
+    pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
+    address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
+                       (void *)uelem, sizeof(VRingUsedElem));
 }
 
 static uint16_t vring_used_idx(VirtQueue *vq)
@@ -189,6 +179,7 @@ static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
     hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     virtio_stw_phys(vq->vdev, pa, val);
+    vq->used_idx = val;
 }
 
 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
@@ -220,7 +211,7 @@ static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 {
     vq->notification = enable;
-    if (virtio_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
+    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
         vring_set_avail_event(vq, vring_avail_idx(vq));
     } else if (enable) {
         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
@@ -238,19 +229,23 @@ int virtio_queue_ready(VirtQueue *vq)
     return vq->vring.avail != 0;
 }
 
+/* Fetch avail_idx from VQ memory only when we really need to know if
+ * guest has added some buffers. */
 int virtio_queue_empty(VirtQueue *vq)
 {
+    if (vq->shadow_avail_idx != vq->last_avail_idx) {
+        return 0;
+    }
+
     return vring_avail_idx(vq) == vq->last_avail_idx;
 }
 
-void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
-                    unsigned int len, unsigned int idx)
+static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
+                               unsigned int len)
 {
     unsigned int offset;
     int i;
 
-    trace_virtqueue_fill(vq, elem, len, idx);
-
     offset = 0;
     for (i = 0; i < elem->in_num; i++) {
         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
@@ -266,12 +261,29 @@ void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
                                   elem->out_sg[i].iov_len,
                                   0, elem->out_sg[i].iov_len);
+}
 
-    idx = (idx + vring_used_idx(vq)) % vq->vring.num;
+void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
+                       unsigned int len)
+{
+    vq->last_avail_idx--;
+    virtqueue_unmap_sg(vq, elem, len);
+}
 
-    /* Get a pointer to the next entry in the used ring. */
-    vring_used_ring_id(vq, idx, elem->index);
-    vring_used_ring_len(vq, idx, len);
+void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
+                    unsigned int len, unsigned int idx)
+{
+    VRingUsedElem uelem;
+
+    trace_virtqueue_fill(vq, elem, len, idx);
+
+    virtqueue_unmap_sg(vq, elem, len);
+
+    idx = (idx + vq->used_idx) % vq->vring.num;
+
+    uelem.id = elem->index;
+    uelem.len = len;
+    vring_used_write(vq, &uelem, idx);
 }
 
 void virtqueue_flush(VirtQueue *vq, unsigned int count)
@@ -280,7 +292,7 @@ void virtqueue_flush(VirtQueue *vq, unsigned int count)
     /* Make sure buffer is written before we update index. */
     smp_wmb();
     trace_virtqueue_flush(vq, count);
-    old = vring_used_idx(vq);
+    old = vq->used_idx;
     new = old + count;
     vring_used_idx_set(vq, new);
     vq->inuse -= count;
@@ -302,7 +314,7 @@ static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
     /* Check it isn't doing very strange things with descriptor numbers. */
     if (num_heads > vq->vring.num) {
         error_report("Guest moved used index from %u to %u",
-                     idx, vring_avail_idx(vq));
+                     idx, vq->shadow_avail_idx);
         exit(1);
     }
     /* On success, callers read a descriptor at vq->last_avail_idx.
@@ -331,18 +343,18 @@ static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
     return head;
 }
 
-static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa,
-                                    unsigned int i, unsigned int max)
+static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
+                                         hwaddr desc_pa, unsigned int max)
 {
     unsigned int next;
 
     /* If this descriptor says it doesn't chain, we're done. */
-    if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) {
+    if (!(desc->flags & VRING_DESC_F_NEXT)) {
         return max;
     }
 
     /* Check they're not leading us off end of descriptors. */
-    next = vring_desc_next(vdev, desc_pa, i);
+    next = desc->next;
     /* Make sure compiler knows to grab that: we don't want it changing! */
     smp_wmb();
 
@@ -351,6 +363,7 @@ static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa,
         exit(1);
     }
 
+    vring_desc_read(vdev, desc, desc_pa, next);
     return next;
 }
 
@@ -367,6 +380,7 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
     while (virtqueue_num_heads(vq, idx)) {
         VirtIODevice *vdev = vq->vdev;
         unsigned int max, num_bufs, indirect = 0;
+        VRingDesc desc;
         hwaddr desc_pa;
         int i;
 
@@ -374,9 +388,10 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
         num_bufs = total_bufs;
         i = virtqueue_get_head(vq, idx++);
         desc_pa = vq->vring.desc;
+        vring_desc_read(vdev, &desc, desc_pa, i);
 
-        if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
-            if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+        if (desc.flags & VRING_DESC_F_INDIRECT) {
+            if (desc.len % sizeof(VRingDesc)) {
                 error_report("Invalid size for indirect buffer table");
                 exit(1);
             }
@@ -389,9 +404,10 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
 
             /* loop over the indirect descriptor table */
             indirect = 1;
-            max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
-            desc_pa = vring_desc_addr(vdev, desc_pa, i);
+            max = desc.len / sizeof(VRingDesc);
+            desc_pa = desc.addr;
             num_bufs = i = 0;
+            vring_desc_read(vdev, &desc, desc_pa, i);
         }
 
         do {
@@ -401,15 +417,15 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
                 exit(1);
             }
 
-            if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
-                in_total += vring_desc_len(vdev, desc_pa, i);
+            if (desc.flags & VRING_DESC_F_WRITE) {
+                in_total += desc.len;
             } else {
-                out_total += vring_desc_len(vdev, desc_pa, i);
+                out_total += desc.len;
             }
             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
                 goto done;
             }
-        } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
+        } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 
         if (!indirect)
             total_bufs = num_bufs;
@@ -434,98 +450,256 @@ int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
     return in_bytes <= in_total && out_bytes <= out_total;
 }
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write)
+static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
+                               unsigned int max_num_sg, bool is_write,
+                               hwaddr pa, size_t sz)
+{
+    unsigned num_sg = *p_num_sg;
+    assert(num_sg <= max_num_sg);
+
+    while (sz) {
+        hwaddr len = sz;
+
+        if (num_sg == max_num_sg) {
+            error_report("virtio: too many write descriptors in indirect table");
+            exit(1);
+        }
+
+        iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
+        iov[num_sg].iov_len = len;
+        addr[num_sg] = pa;
+
+        sz -= len;
+        pa += len;
+        num_sg++;
+    }
+    *p_num_sg = num_sg;
+}
+
+static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
+                                unsigned int *num_sg, unsigned int max_size,
+                                int is_write)
 {
     unsigned int i;
     hwaddr len;
 
-    if (num_sg > VIRTQUEUE_MAX_SIZE) {
-        error_report("virtio: map attempt out of bounds: %zd > %d",
-                     num_sg, VIRTQUEUE_MAX_SIZE);
-        exit(1);
-    }
+    /* Note: this function MUST validate input, some callers
+     * are passing in num_sg values received over the network.
+     */
+    /* TODO: teach all callers that this can fail, and return failure instead
+     * of asserting here.
+     * When we do, we might be able to re-enable NDEBUG below.
+     */
+#ifdef NDEBUG
+#error building with NDEBUG is not supported
+#endif
+    assert(*num_sg <= max_size);
 
-    for (i = 0; i < num_sg; i++) {
+    for (i = 0; i < *num_sg; i++) {
         len = sg[i].iov_len;
         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
-        if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
+        if (!sg[i].iov_base) {
             error_report("virtio: error trying to map MMIO memory");
             exit(1);
         }
+        if (len != sg[i].iov_len) {
+            error_report("virtio: unexpected memory split");
+            exit(1);
+        }
     }
 }
 
-int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
+void virtqueue_map(VirtQueueElement *elem)
+{
+    virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
+                        VIRTQUEUE_MAX_SIZE, 1);
+    virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
+                        VIRTQUEUE_MAX_SIZE, 0);
+}
+
+void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
+{
+    VirtQueueElement *elem;
+    size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
+    size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
+    size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
+    size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
+    size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
+    size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
+
+    assert(sz >= sizeof(VirtQueueElement));
+    elem = g_malloc(out_sg_end);
+    elem->out_num = out_num;
+    elem->in_num = in_num;
+    elem->in_addr = (void *)elem + in_addr_ofs;
+    elem->out_addr = (void *)elem + out_addr_ofs;
+    elem->in_sg = (void *)elem + in_sg_ofs;
+    elem->out_sg = (void *)elem + out_sg_ofs;
+    return elem;
+}
+
+void *virtqueue_pop(VirtQueue *vq, size_t sz)
 {
     unsigned int i, head, max;
     hwaddr desc_pa = vq->vring.desc;
     VirtIODevice *vdev = vq->vdev;
+    VirtQueueElement *elem;
+    unsigned out_num, in_num;
+    hwaddr addr[VIRTQUEUE_MAX_SIZE];
+    struct iovec iov[VIRTQUEUE_MAX_SIZE];
+    VRingDesc desc;
 
-    if (!virtqueue_num_heads(vq, vq->last_avail_idx))
-        return 0;
+    if (virtio_queue_empty(vq)) {
+        return NULL;
+    }
+    /* Needed after virtio_queue_empty(), see comment in
+     * virtqueue_num_heads(). */
+    smp_rmb();
 
     /* When we start there are none of either input nor output. */
-    elem->out_num = elem->in_num = 0;
+    out_num = in_num = 0;
 
     max = vq->vring.num;
 
     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
-    if (virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
         vring_set_avail_event(vq, vq->last_avail_idx);
     }
 
-    if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
-        if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+    vring_desc_read(vdev, &desc, desc_pa, i);
+    if (desc.flags & VRING_DESC_F_INDIRECT) {
+        if (desc.len % sizeof(VRingDesc)) {
             error_report("Invalid size for indirect buffer table");
             exit(1);
         }
 
         /* loop over the indirect descriptor table */
-        max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
-        desc_pa = vring_desc_addr(vdev, desc_pa, i);
+        max = desc.len / sizeof(VRingDesc);
+        desc_pa = desc.addr;
         i = 0;
+        vring_desc_read(vdev, &desc, desc_pa, i);
     }
 
     /* Collect all the descriptors */
     do {
-        struct iovec *sg;
-
-        if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
-            if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
-                error_report("Too many write descriptors in indirect table");
-                exit(1);
-            }
-            elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i);
-            sg = &elem->in_sg[elem->in_num++];
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
+                               VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
         } else {
-            if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
-                error_report("Too many read descriptors in indirect table");
+            if (in_num) {
+                error_report("Incorrect order for descriptors");
                 exit(1);
             }
-            elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i);
-            sg = &elem->out_sg[elem->out_num++];
+            virtqueue_map_desc(&out_num, addr, iov,
+                               VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
         }
 
-        sg->iov_len = vring_desc_len(vdev, desc_pa, i);
-
         /* If we've got too many, that implies a descriptor loop. */
-        if ((elem->in_num + elem->out_num) > max) {
+        if ((in_num + out_num) > max) {
             error_report("Looped descriptor");
             exit(1);
         }
-    } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
-
-    /* Now map what we have collected */
-    virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
-    virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
+    } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 
+    /* Now copy what we have collected and mapped */
+    elem = virtqueue_alloc_element(sz, out_num, in_num);
     elem->index = head;
+    for (i = 0; i < out_num; i++) {
+        elem->out_addr[i] = addr[i];
+        elem->out_sg[i] = iov[i];
+    }
+    for (i = 0; i < in_num; i++) {
+        elem->in_addr[i] = addr[out_num + i];
+        elem->in_sg[i] = iov[out_num + i];
+    }
 
     vq->inuse++;
 
     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
-    return elem->in_num + elem->out_num;
+    return elem;
+}
+
+/* Reading and writing a structure directly to QEMUFile is *awful*, but
+ * it is what QEMU has always done by mistake.  We can change it sooner
+ * or later by bumping the version number of the affected vm states.
+ * In the meanwhile, since the in-memory layout of VirtQueueElement
+ * has changed, we need to marshal to and from the layout that was
+ * used before the change.
+ */
+typedef struct VirtQueueElementOld {
+    unsigned int index;
+    unsigned int out_num;
+    unsigned int in_num;
+    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
+    struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
+    struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
+} VirtQueueElementOld;
+
+void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
+{
+    VirtQueueElement *elem;
+    VirtQueueElementOld data;
+    int i;
+
+    qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
+
+    elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
+    elem->index = data.index;
+
+    for (i = 0; i < elem->in_num; i++) {
+        elem->in_addr[i] = data.in_addr[i];
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        elem->out_addr[i] = data.out_addr[i];
+    }
+
+    for (i = 0; i < elem->in_num; i++) {
+        /* Base is overwritten by virtqueue_map.  */
+        elem->in_sg[i].iov_base = 0;
+        elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        /* Base is overwritten by virtqueue_map.  */
+        elem->out_sg[i].iov_base = 0;
+        elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
+    }
+
+    virtqueue_map(elem);
+    return elem;
+}
+
+void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
+{
+    VirtQueueElementOld data;
+    int i;
+
+    memset(&data, 0, sizeof(data));
+    data.index = elem->index;
+    data.in_num = elem->in_num;
+    data.out_num = elem->out_num;
+
+    for (i = 0; i < elem->in_num; i++) {
+        data.in_addr[i] = elem->in_addr[i];
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        data.out_addr[i] = elem->out_addr[i];
+    }
+
+    for (i = 0; i < elem->in_num; i++) {
+        /* Base is overwritten by virtqueue_map when loading.  Do not
+         * save it, as it would leak the QEMU address space layout.  */
+        data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        /* Do not save iov_base as above.  */
+        data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
+    }
+    qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 }
 
 /* virtio device */
@@ -560,7 +734,7 @@ int virtio_set_status(VirtIODevice *vdev, uint8_t val)
     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
     trace_virtio_set_status(vdev, val);
 
-    if (virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
             val & VIRTIO_CONFIG_S_FEATURES_OK) {
             int ret = virtio_validate_features(vdev);
@@ -629,10 +803,13 @@ void virtio_reset(void *opaque)
         vdev->vq[i].vring.avail = 0;
         vdev->vq[i].vring.used = 0;
         vdev->vq[i].last_avail_idx = 0;
+        vdev->vq[i].shadow_avail_idx = 0;
+        vdev->vq[i].used_idx = 0;
         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
         vdev->vq[i].signalled_used = 0;
         vdev->vq[i].signalled_used_valid = false;
         vdev->vq[i].notification = true;
+        vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
     }
 }
 
@@ -898,7 +1075,7 @@ void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 
     /* virtio-1 compliant devices cannot change the alignment */
-    if (virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
         error_report("tried to modify queue alignment for virtio-1 device");
         return;
     }
@@ -912,7 +1089,17 @@ void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
     virtio_queue_update_rings(vdev, n);
 }
 
-void virtio_queue_notify_vq(VirtQueue *vq)
+static void virtio_queue_notify_aio_vq(VirtQueue *vq)
+{
+    if (vq->vring.desc && vq->handle_aio_output) {
+        VirtIODevice *vdev = vq->vdev;
+
+        trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
+        vq->handle_aio_output(vdev, vq);
+    }
+}
+
+static void virtio_queue_notify_vq(VirtQueue *vq)
 {
     if (vq->vring.desc && vq->handle_output) {
         VirtIODevice *vdev = vq->vdev;
@@ -964,8 +1151,10 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
         abort();
 
     vdev->vq[i].vring.num = queue_size;
+    vdev->vq[i].vring.num_default = queue_size;
     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
     vdev->vq[i].handle_output = handle_output;
+    vdev->vq[i].handle_aio_output = NULL;
 
     return &vdev->vq[i];
 }
@@ -977,6 +1166,7 @@ void virtio_del_queue(VirtIODevice *vdev, int n)
     }
 
     vdev->vq[n].vring.num = 0;
+    vdev->vq[n].vring.num_default = 0;
 }
 
 void virtio_irq(VirtQueue *vq)
@@ -986,32 +1176,32 @@ void virtio_irq(VirtQueue *vq)
     virtio_notify_vector(vq->vdev, vq->vector);
 }
 
-static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
+bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
 {
     uint16_t old, new;
     bool v;
     /* We need to expose used array entries before checking used event. */
     smp_mb();
     /* Always notify when queue is empty (when feature acknowledge) */
-    if (virtio_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
-        !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
+        !vq->inuse && virtio_queue_empty(vq)) {
         return true;
     }
 
-    if (!virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
     }
 
     v = vq->signalled_used_valid;
     vq->signalled_used_valid = true;
     old = vq->signalled_used;
-    new = vq->signalled_used = vring_used_idx(vq);
+    new = vq->signalled_used = vq->used_idx;
     return !v || vring_need_event(vring_get_used_event(vq), new, old);
 }
 
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
 {
-    if (!vring_notify(vdev, vq)) {
+    if (!virtio_should_notify(vdev, vq)) {
         return;
     }
 
@@ -1035,7 +1225,7 @@ static bool virtio_device_endian_needed(void *opaque)
     VirtIODevice *vdev = opaque;
 
     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
-    if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
         return vdev->device_endian != virtio_default_endian();
     }
     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
@@ -1056,33 +1246,38 @@ static bool virtio_virtqueue_needed(void *opaque)
     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
 }
 
-static void put_virtqueue_state(QEMUFile *f, void *pv, size_t size)
+static bool virtio_ringsize_needed(void *opaque)
 {
-    VirtIODevice *vdev = pv;
+    VirtIODevice *vdev = opaque;
     int i;
 
     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
-        qemu_put_be64(f, vdev->vq[i].vring.avail);
-        qemu_put_be64(f, vdev->vq[i].vring.used);
+        if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
+            return true;
+        }
     }
+    return false;
 }
 
-static int get_virtqueue_state(QEMUFile *f, void *pv, size_t size)
+static bool virtio_extra_state_needed(void *opaque)
 {
-    VirtIODevice *vdev = pv;
-    int i;
+    VirtIODevice *vdev = opaque;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 
-    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
-        vdev->vq[i].vring.avail = qemu_get_be64(f);
-        vdev->vq[i].vring.used = qemu_get_be64(f);
-    }
-    return 0;
+    return k->has_extra_state &&
+        k->has_extra_state(qbus->parent);
 }
 
-static VMStateInfo vmstate_info_virtqueue = {
+static const VMStateDescription vmstate_virtqueue = {
     .name = "virtqueue_state",
-    .get = get_virtqueue_state,
-    .put = put_virtqueue_state,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(vring.avail, struct VirtQueue),
+        VMSTATE_UINT64(vring.used, struct VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
 };
 
 static const VMStateDescription vmstate_virtio_virtqueues = {
@@ -1091,12 +1286,74 @@ static const VMStateDescription vmstate_virtio_virtqueues = {
     .minimum_version_id = 1,
     .needed = &virtio_virtqueue_needed,
     .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
+                      VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_ringsize = {
+    .name = "ringsize_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(vring.num_default, struct VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_ringsize = {
+    .name = "virtio/ringsize",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_ringsize_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
+                      VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int get_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (!k->load_extra_state) {
+        return -1;
+    } else {
+        return k->load_extra_state(qbus->parent, f);
+    }
+}
+
+static void put_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    k->save_extra_state(qbus->parent, f);
+}
+
+static const VMStateInfo vmstate_info_extra_state = {
+    .name = "virtqueue_extra_state",
+    .get = get_extra_state,
+    .put = put_extra_state,
+};
+
+static const VMStateDescription vmstate_virtio_extra_state = {
+    .name = "virtio/extra_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_extra_state_needed,
+    .fields = (VMStateField[]) {
         {
-            .name         = "virtqueues",
+            .name         = "extra_state",
             .version_id   = 0,
             .field_exists = NULL,
             .size         = 0,
-            .info         = &vmstate_info_virtqueue,
+            .info         = &vmstate_info_extra_state,
             .flags        = VMS_SINGLE,
             .offset       = 0,
         },
@@ -1138,6 +1395,8 @@ static const VMStateDescription vmstate_virtio = {
         &vmstate_virtio_device_endian,
         &vmstate_virtio_64bit_features,
         &vmstate_virtio_virtqueues,
+        &vmstate_virtio_ringsize,
+        &vmstate_virtio_extra_state,
         NULL
     }
 };
@@ -1264,7 +1523,7 @@ int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
     num = qemu_get_be32(f);
 
     if (num > VIRTIO_QUEUE_MAX) {
-        error_report("Invalid number of PCI queues: 0x%x", num);
+        error_report("Invalid number of virtqueues: 0x%x", num);
         return -1;
     }
 
@@ -1348,6 +1607,8 @@ int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
                              vdev->vq[i].last_avail_idx, nheads);
                 return -1;
             }
+            vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
+            vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
         }
     }
 
@@ -1430,6 +1691,7 @@ void virtio_init(VirtIODevice *vdev, const char *name,
     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
                                                      vdev);
     vdev->device_endian = virtio_default_endian();
+    vdev->use_guest_notifier_mask = true;
 }
 
 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
@@ -1460,7 +1722,7 @@ hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingAvail, ring) +
-        sizeof(uint64_t) * vdev->vq[n].vring.num;
+        sizeof(uint16_t) * vdev->vq[n].vring.num;
 }
 
 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
@@ -1483,6 +1745,7 @@ uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
 {
     vdev->vq[n].last_avail_idx = idx;
+    vdev->vq[n].shadow_avail_idx = idx;
 }
 
 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
@@ -1512,10 +1775,10 @@ void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                 bool with_irqfd)
 {
     if (assign && !with_irqfd) {
-        event_notifier_set_handler(&vq->guest_notifier,
+        event_notifier_set_handler(&vq->guest_notifier, false,
                                    virtio_queue_guest_notifier_read);
     } else {
-        event_notifier_set_handler(&vq->guest_notifier, NULL);
+        event_notifier_set_handler(&vq->guest_notifier, false, NULL);
     }
     if (!assign) {
         /* Test and clear notifier before closing it,
@@ -1529,6 +1792,31 @@ EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
     return &vq->guest_notifier;
 }
 
+static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
+{
+    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
+    if (event_notifier_test_and_clear(n)) {
+        virtio_queue_notify_aio_vq(vq);
+    }
+}
+
+void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
+                                                void (*handle_output)(VirtIODevice *,
+                                                                      VirtQueue *))
+{
+    if (handle_output) {
+        vq->handle_aio_output = handle_output;
+        aio_set_event_notifier(ctx, &vq->host_notifier, true,
+                               virtio_queue_host_notifier_aio_read);
+    } else {
+        aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
+        /* Test and clear notifier before after disabling event,
+         * in case poll callback didn't have time to run. */
+        virtio_queue_host_notifier_aio_read(&vq->host_notifier);
+        vq->handle_aio_output = NULL;
+    }
+}
+
 static void virtio_queue_host_notifier_read(EventNotifier *n)
 {
     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
@@ -1541,10 +1829,10 @@ void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                bool set_handler)
 {
     if (assign && set_handler) {
-        event_notifier_set_handler(&vq->host_notifier,
+        event_notifier_set_handler(&vq->host_notifier, true,
                                    virtio_queue_host_notifier_read);
     } else {
-        event_notifier_set_handler(&vq->host_notifier, NULL);
+        event_notifier_set_handler(&vq->host_notifier, true, NULL);
     }
     if (!assign) {
         /* Test and clear notifier before after disabling event,
diff --git a/qemu/hw/watchdog/watchdog.c b/qemu/hw/watchdog/watchdog.c
index 8d4b0eeeb..bbf3646ba 100644
--- a/qemu/hw/watchdog/watchdog.c
+++ b/qemu/hw/watchdog/watchdog.c
@@ -19,7 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/queue.h"
@@ -28,15 +28,7 @@
 #include "sysemu/watchdog.h"
 #include "qapi-event.h"
 #include "hw/nmi.h"
-
-/* Possible values for action parameter. */
-#define WDT_RESET        1	/* Hard reset. */
-#define WDT_SHUTDOWN     2	/* Shutdown. */
-#define WDT_POWEROFF     3	/* Quit. */
-#define WDT_PAUSE        4	/* Pause. */
-#define WDT_DEBUG        5	/* Prints a message and continues running. */
-#define WDT_NONE         6	/* Do nothing. */
-#define WDT_NMI          7	/* Inject nmi into the guest */
+#include "qemu/help_option.h"
 
 static int watchdog_action = WDT_RESET;
 static QLIST_HEAD(watchdog_list, WatchdogTimerModel) watchdog_list;
@@ -105,6 +97,11 @@ int select_watchdog_action(const char *p)
     return 0;
 }
 
+int get_watchdog_action(void)
+{
+    return watchdog_action;
+}
+
 /* This actually performs the "action" once a watchdog has expired,
  * ie. reboot, shutdown, exit, etc.
  */
diff --git a/qemu/hw/watchdog/wdt_diag288.c b/qemu/hw/watchdog/wdt_diag288.c
index 2a885a447..f54a35a0e 100644
--- a/qemu/hw/watchdog/wdt_diag288.c
+++ b/qemu/hw/watchdog/wdt_diag288.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/watchdog.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
@@ -50,8 +51,19 @@ static void diag288_reset(void *opaque)
 static void diag288_timer_expired(void *dev)
 {
     qemu_log_mask(CPU_LOG_RESET, "Watchdog timer expired.\n");
+    /* Reset the watchdog only if the guest gets notified about
+     * expiry. watchdog_perform_action() may temporarily relinquish
+     * the BQL; reset before triggering the action to avoid races with
+     * diag288 instructions. */
+    switch (get_watchdog_action()) {
+    case WDT_DEBUG:
+    case WDT_NONE:
+    case WDT_PAUSE:
+        break;
+    default:
+        wdt_diag288_reset(dev);
+    }
     watchdog_perform_action();
-    wdt_diag288_reset(dev);
 }
 
 static int wdt_diag288_handle_timer(DIAG288State *diag288,
@@ -67,7 +79,7 @@ static int wdt_diag288_handle_timer(DIAG288State *diag288,
         }
         timer_mod(diag288->timer,
                   qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                  timeout * get_ticks_per_sec());
+                  timeout * NANOSECONDS_PER_SECOND);
         break;
     case WDT_DIAG288_CANCEL:
         if (!diag288->enabled) {
diff --git a/qemu/hw/watchdog/wdt_i6300esb.c b/qemu/hw/watchdog/wdt_i6300esb.c
index cfa2b1be1..a83d95121 100644
--- a/qemu/hw/watchdog/wdt_i6300esb.c
+++ b/qemu/hw/watchdog/wdt_i6300esb.c
@@ -19,7 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
@@ -129,14 +129,9 @@ static void i6300esb_restart_timer(I6300State *d, int stage)
     else
         timeout <<= 5;
 
-    /* Get the timeout in units of ticks_per_sec.
-     *
-     * ticks_per_sec is typically 10^9 == 0x3B9ACA00 (30 bits), with
-     * 20 bits of user supplied preload, and 15 bits of scale, the
-     * multiply here can exceed 64-bits, before we divide by 33MHz, so
-     * we use a higher-precision intermediate result.
-     */
-    timeout = muldiv64(get_ticks_per_sec(), timeout, 33000000);
+    /* Get the timeout in nanoseconds. */
+
+    timeout = timeout * 30; /* on a PCI bus, 1 tick is 30 ns*/
 
     i6300esb_debug("stage %d, timeout %" PRIi64 "\n", d->stage, timeout);
 
diff --git a/qemu/hw/watchdog/wdt_ib700.c b/qemu/hw/watchdog/wdt_ib700.c
index 0917a713d..532afe89e 100644
--- a/qemu/hw/watchdog/wdt_ib700.c
+++ b/qemu/hw/watchdog/wdt_ib700.c
@@ -19,6 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "sysemu/watchdog.h"
@@ -63,7 +64,7 @@ static void ib700_write_enable_reg(void *vp, uint32_t addr, uint32_t data)
 
     ib700_debug("addr = %x, data = %x\n", addr, data);
 
-    timeout = (int64_t) time_map[data & 0xF] * get_ticks_per_sec();
+    timeout = (int64_t) time_map[data & 0xF] * NANOSECONDS_PER_SECOND;
     timer_mod(s->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + timeout);
 }
 
diff --git a/qemu/hw/xen/Makefile.objs b/qemu/hw/xen/Makefile.objs
index a0ca0aa3d..d3670940b 100644
--- a/qemu/hw/xen/Makefile.objs
+++ b/qemu/hw/xen/Makefile.objs
@@ -2,4 +2,4 @@
 common-obj-$(CONFIG_XEN_BACKEND) += xen_backend.o xen_devconfig.o
 
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
-obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
+obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_graphics.o xen_pt_msi.o
diff --git a/qemu/hw/xen/xen-host-pci-device.c b/qemu/hw/xen/xen-host-pci-device.c
index 743b37b99..eed8cc88e 100644
--- a/qemu/hw/xen/xen-host-pci-device.c
+++ b/qemu/hw/xen/xen-host-pci-device.c
@@ -6,7 +6,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "xen-host-pci-device.h"
 
 #define XEN_HOST_PCI_MAX_EXT_CAP \
@@ -31,25 +34,20 @@
 #define IORESOURCE_PREFETCH     0x00001000      /* No side effects */
 #define IORESOURCE_MEM_64       0x00100000
 
-static int xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
-                                   const char *name, char *buf, ssize_t size)
+static void xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
+                                    const char *name, char *buf, ssize_t size)
 {
     int rc;
 
     rc = snprintf(buf, size, "/sys/bus/pci/devices/%04x:%02x:%02x.%d/%s",
                   d->domain, d->bus, d->dev, d->func, name);
-
-    if (rc >= size || rc < 0) {
-        /* The ouput is truncated or an other error is encountered */
-        return -ENODEV;
-    }
-    return 0;
+    assert(rc >= 0 && rc < size);
 }
 
 
 /* This size should be enough to read the first 7 lines of a resource file */
 #define XEN_HOST_PCI_RESOURCE_BUFFER_SIZE 400
-static int xen_host_pci_get_resource(XenHostPCIDevice *d)
+static void xen_host_pci_get_resource(XenHostPCIDevice *d, Error **errp)
 {
     int i, rc, fd;
     char path[PATH_MAX];
@@ -58,25 +56,22 @@ static int xen_host_pci_get_resource(XenHostPCIDevice *d)
     char *endptr, *s;
     uint8_t type;
 
-    rc = xen_host_pci_sysfs_path(d, "resource", path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
+    xen_host_pci_sysfs_path(d, "resource", path, sizeof(path));
+
     fd = open(path, O_RDONLY);
     if (fd == -1) {
-        XEN_HOST_PCI_LOG("Error: Can't open %s: %s\n", path, strerror(errno));
-        return -errno;
+        error_setg_file_open(errp, errno, path);
+        return;
     }
 
     do {
-        rc = read(fd, &buf, sizeof (buf) - 1);
+        rc = read(fd, &buf, sizeof(buf) - 1);
         if (rc < 0 && errno != EINTR) {
-            rc = -errno;
+            error_setg_errno(errp, errno, "read err");
             goto out;
         }
     } while (rc < 0);
     buf[rc] = 0;
-    rc = 0;
 
     s = buf;
     for (i = 0; i < PCI_NUM_REGIONS; i++) {
@@ -129,70 +124,69 @@ static int xen_host_pci_get_resource(XenHostPCIDevice *d)
             d->rom.bus_flags = flags & IORESOURCE_BITS;
         }
     }
+
     if (i != PCI_NUM_REGIONS) {
-        /* Invalid format or input to short */
-        rc = -ENODEV;
+        error_setg(errp, "Invalid format or input too short: %s", buf);
     }
 
 out:
     close(fd);
-    return rc;
 }
 
 /* This size should be enough to read a long from a file */
 #define XEN_HOST_PCI_GET_VALUE_BUFFER_SIZE 22
-static int xen_host_pci_get_value(XenHostPCIDevice *d, const char *name,
-                                  unsigned int *pvalue, int base)
+static void xen_host_pci_get_value(XenHostPCIDevice *d, const char *name,
+                                   unsigned int *pvalue, int base, Error **errp)
 {
     char path[PATH_MAX];
     char buf[XEN_HOST_PCI_GET_VALUE_BUFFER_SIZE];
     int fd, rc;
     unsigned long value;
-    char *endptr;
+    const char *endptr;
+
+    xen_host_pci_sysfs_path(d, name, path, sizeof(path));
 
-    rc = xen_host_pci_sysfs_path(d, name, path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
     fd = open(path, O_RDONLY);
     if (fd == -1) {
-        XEN_HOST_PCI_LOG("Error: Can't open %s: %s\n", path, strerror(errno));
-        return -errno;
+        error_setg_file_open(errp, errno, path);
+        return;
     }
+
     do {
-        rc = read(fd, &buf, sizeof (buf) - 1);
+        rc = read(fd, &buf, sizeof(buf) - 1);
         if (rc < 0 && errno != EINTR) {
-            rc = -errno;
+            error_setg_errno(errp, errno, "read err");
             goto out;
         }
     } while (rc < 0);
+
     buf[rc] = 0;
-    value = strtol(buf, &endptr, base);
-    if (endptr == buf || *endptr != '\n') {
-        rc = -1;
-    } else if ((value == LONG_MIN || value == LONG_MAX) && errno == ERANGE) {
-        rc = -errno;
-    } else {
-        rc = 0;
+    rc = qemu_strtoul(buf, &endptr, base, &value);
+    if (!rc) {
+        assert(value <= UINT_MAX);
         *pvalue = value;
+    } else {
+        error_setg_errno(errp, -rc, "failed to parse value '%s'", buf);
     }
+
 out:
     close(fd);
-    return rc;
 }
 
-static inline int xen_host_pci_get_hex_value(XenHostPCIDevice *d,
-                                             const char *name,
-                                             unsigned int *pvalue)
+static inline void xen_host_pci_get_hex_value(XenHostPCIDevice *d,
+                                              const char *name,
+                                              unsigned int *pvalue,
+                                              Error **errp)
 {
-    return xen_host_pci_get_value(d, name, pvalue, 16);
+    xen_host_pci_get_value(d, name, pvalue, 16, errp);
 }
 
-static inline int xen_host_pci_get_dec_value(XenHostPCIDevice *d,
-                                             const char *name,
-                                             unsigned int *pvalue)
+static inline void xen_host_pci_get_dec_value(XenHostPCIDevice *d,
+                                              const char *name,
+                                              unsigned int *pvalue,
+                                              Error **errp)
 {
-    return xen_host_pci_get_value(d, name, pvalue, 10);
+    xen_host_pci_get_value(d, name, pvalue, 10, errp);
 }
 
 static bool xen_host_pci_dev_is_virtfn(XenHostPCIDevice *d)
@@ -200,26 +194,21 @@ static bool xen_host_pci_dev_is_virtfn(XenHostPCIDevice *d)
     char path[PATH_MAX];
     struct stat buf;
 
-    if (xen_host_pci_sysfs_path(d, "physfn", path, sizeof (path))) {
-        return false;
-    }
+    xen_host_pci_sysfs_path(d, "physfn", path, sizeof(path));
+
     return !stat(path, &buf);
 }
 
-static int xen_host_pci_config_open(XenHostPCIDevice *d)
+static void xen_host_pci_config_open(XenHostPCIDevice *d, Error **errp)
 {
     char path[PATH_MAX];
-    int rc;
 
-    rc = xen_host_pci_sysfs_path(d, "config", path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
+    xen_host_pci_sysfs_path(d, "config", path, sizeof(path));
+
     d->config_fd = open(path, O_RDWR);
-    if (d->config_fd < 0) {
-        return -errno;
+    if (d->config_fd == -1) {
+        error_setg_file_open(errp, errno, path);
     }
-    return 0;
 }
 
 static int xen_host_pci_config_read(XenHostPCIDevice *d,
@@ -341,11 +330,12 @@ int xen_host_pci_find_ext_cap_offset(XenHostPCIDevice *d, uint32_t cap)
     return -1;
 }
 
-int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
-                            uint8_t bus, uint8_t dev, uint8_t func)
+void xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
+                             uint8_t bus, uint8_t dev, uint8_t func,
+                             Error **errp)
 {
     unsigned int v;
-    int rc = 0;
+    Error *err = NULL;
 
     d->config_fd = -1;
     d->domain = domain;
@@ -353,38 +343,56 @@ int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
     d->dev = dev;
     d->func = func;
 
-    rc = xen_host_pci_config_open(d);
-    if (rc) {
+    xen_host_pci_config_open(d, &err);
+    if (err) {
         goto error;
     }
-    rc = xen_host_pci_get_resource(d);
-    if (rc) {
+
+    xen_host_pci_get_resource(d, &err);
+    if (err) {
         goto error;
     }
-    rc = xen_host_pci_get_hex_value(d, "vendor", &v);
-    if (rc) {
+
+    xen_host_pci_get_hex_value(d, "vendor", &v, &err);
+    if (err) {
         goto error;
     }
     d->vendor_id = v;
-    rc = xen_host_pci_get_hex_value(d, "device", &v);
-    if (rc) {
+
+    xen_host_pci_get_hex_value(d, "device", &v, &err);
+    if (err) {
         goto error;
     }
     d->device_id = v;
-    rc = xen_host_pci_get_dec_value(d, "irq", &v);
-    if (rc) {
+
+    xen_host_pci_get_dec_value(d, "irq", &v, &err);
+    if (err) {
         goto error;
     }
     d->irq = v;
+
+    xen_host_pci_get_hex_value(d, "class", &v, &err);
+    if (err) {
+        goto error;
+    }
+    d->class_code = v;
+
     d->is_virtfn = xen_host_pci_dev_is_virtfn(d);
 
-    return 0;
+    return;
+
 error:
+    error_propagate(errp, err);
+
     if (d->config_fd >= 0) {
         close(d->config_fd);
         d->config_fd = -1;
     }
-    return rc;
+}
+
+bool xen_host_pci_device_closed(XenHostPCIDevice *d)
+{
+    return d->config_fd == -1;
 }
 
 void xen_host_pci_device_put(XenHostPCIDevice *d)
diff --git a/qemu/hw/xen/xen-host-pci-device.h b/qemu/hw/xen/xen-host-pci-device.h
index c2486f0c1..6acf36e13 100644
--- a/qemu/hw/xen/xen-host-pci-device.h
+++ b/qemu/hw/xen/xen-host-pci-device.h
@@ -25,6 +25,7 @@ typedef struct XenHostPCIDevice {
 
     uint16_t vendor_id;
     uint16_t device_id;
+    uint32_t class_code;
     int irq;
 
     XenHostPCIIORegion io_regions[PCI_NUM_REGIONS - 1];
@@ -35,9 +36,11 @@ typedef struct XenHostPCIDevice {
     int config_fd;
 } XenHostPCIDevice;
 
-int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
-                            uint8_t bus, uint8_t dev, uint8_t func);
+void xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
+                             uint8_t bus, uint8_t dev, uint8_t func,
+                             Error **errp);
 void xen_host_pci_device_put(XenHostPCIDevice *pci_dev);
+bool xen_host_pci_device_closed(XenHostPCIDevice *d);
 
 int xen_host_pci_get_byte(XenHostPCIDevice *d, int pos, uint8_t *p);
 int xen_host_pci_get_word(XenHostPCIDevice *d, int pos, uint16_t *p);
diff --git a/qemu/hw/xen/xen_backend.c b/qemu/hw/xen/xen_backend.c
index 2510e2e4f..60575ad38 100644
--- a/qemu/hw/xen/xen_backend.c
+++ b/qemu/hw/xen/xen_backend.c
@@ -22,15 +22,7 @@
  * TODO: add some xenbus / xenstore concepts overview here.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <inttypes.h>
-#include <sys/types.h>
-#include <sys/stat.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 #include <sys/signal.h>
 
@@ -44,7 +36,8 @@
 /* ------------------------------------------------------------- */
 
 /* public */
-XenXC xen_xc = XC_HANDLER_INITIAL_VALUE;
+xc_interface *xen_xc = NULL;
+xenforeignmemory_handle *xen_fmem = NULL;
 struct xs_handle *xenstore = NULL;
 const char *xen_protocol;
 
@@ -243,24 +236,24 @@ static struct XenDevice *xen_be_get_xendev(const char *type, int dom, int dev,
     xendev->debug      = debug;
     xendev->local_port = -1;
 
-    xendev->evtchndev = xen_xc_evtchn_open(NULL, 0);
-    if (xendev->evtchndev == XC_HANDLER_INITIAL_VALUE) {
+    xendev->evtchndev = xenevtchn_open(NULL, 0);
+    if (xendev->evtchndev == NULL) {
         xen_be_printf(NULL, 0, "can't open evtchn device\n");
         g_free(xendev);
         return NULL;
     }
-    fcntl(xc_evtchn_fd(xendev->evtchndev), F_SETFD, FD_CLOEXEC);
+    fcntl(xenevtchn_fd(xendev->evtchndev), F_SETFD, FD_CLOEXEC);
 
     if (ops->flags & DEVOPS_FLAG_NEED_GNTDEV) {
-        xendev->gnttabdev = xen_xc_gnttab_open(NULL, 0);
-        if (xendev->gnttabdev == XC_HANDLER_INITIAL_VALUE) {
+        xendev->gnttabdev = xengnttab_open(NULL, 0);
+        if (xendev->gnttabdev == NULL) {
             xen_be_printf(NULL, 0, "can't open gnttab device\n");
-            xc_evtchn_close(xendev->evtchndev);
+            xenevtchn_close(xendev->evtchndev);
             g_free(xendev);
             return NULL;
         }
     } else {
-        xendev->gnttabdev = XC_HANDLER_INITIAL_VALUE;
+        xendev->gnttabdev = NULL;
     }
 
     QTAILQ_INSERT_TAIL(&xendevs, xendev, next);
@@ -306,11 +299,11 @@ static struct XenDevice *xen_be_del_xendev(int dom, int dev)
             g_free(xendev->fe);
         }
 
-        if (xendev->evtchndev != XC_HANDLER_INITIAL_VALUE) {
-            xc_evtchn_close(xendev->evtchndev);
+        if (xendev->evtchndev != NULL) {
+            xenevtchn_close(xendev->evtchndev);
         }
-        if (xendev->gnttabdev != XC_HANDLER_INITIAL_VALUE) {
-            xc_gnttab_close(xendev->gnttabdev);
+        if (xendev->gnttabdev != NULL) {
+            xengnttab_close(xendev->gnttabdev);
         }
 
         QTAILQ_REMOVE(&xendevs, xendev, next);
@@ -691,13 +684,14 @@ static void xen_be_evtchn_event(void *opaque)
     struct XenDevice *xendev = opaque;
     evtchn_port_t port;
 
-    port = xc_evtchn_pending(xendev->evtchndev);
+    port = xenevtchn_pending(xendev->evtchndev);
     if (port != xendev->local_port) {
-        xen_be_printf(xendev, 0, "xc_evtchn_pending returned %d (expected %d)\n",
+        xen_be_printf(xendev, 0,
+                      "xenevtchn_pending returned %d (expected %d)\n",
                       port, xendev->local_port);
         return;
     }
-    xc_evtchn_unmask(xendev->evtchndev, port);
+    xenevtchn_unmask(xendev->evtchndev, port);
 
     if (xendev->ops->event) {
         xendev->ops->event(xendev);
@@ -716,7 +710,7 @@ int xen_be_init(void)
 
     qemu_set_fd_handler(xs_fileno(xenstore), xenstore_update, NULL, NULL);
 
-    if (xen_xc == XC_HANDLER_INITIAL_VALUE) {
+    if (xen_xc == NULL || xen_fmem == NULL) {
         /* Check if xen_init() have been called */
         goto err;
     }
@@ -740,14 +734,14 @@ int xen_be_bind_evtchn(struct XenDevice *xendev)
     if (xendev->local_port != -1) {
         return 0;
     }
-    xendev->local_port = xc_evtchn_bind_interdomain
+    xendev->local_port = xenevtchn_bind_interdomain
         (xendev->evtchndev, xendev->dom, xendev->remote_port);
     if (xendev->local_port == -1) {
-        xen_be_printf(xendev, 0, "xc_evtchn_bind_interdomain failed\n");
+        xen_be_printf(xendev, 0, "xenevtchn_bind_interdomain failed\n");
         return -1;
     }
     xen_be_printf(xendev, 2, "bind evtchn port %d\n", xendev->local_port);
-    qemu_set_fd_handler(xc_evtchn_fd(xendev->evtchndev),
+    qemu_set_fd_handler(xenevtchn_fd(xendev->evtchndev),
                         xen_be_evtchn_event, NULL, xendev);
     return 0;
 }
@@ -757,15 +751,15 @@ void xen_be_unbind_evtchn(struct XenDevice *xendev)
     if (xendev->local_port == -1) {
         return;
     }
-    qemu_set_fd_handler(xc_evtchn_fd(xendev->evtchndev), NULL, NULL, NULL);
-    xc_evtchn_unbind(xendev->evtchndev, xendev->local_port);
+    qemu_set_fd_handler(xenevtchn_fd(xendev->evtchndev), NULL, NULL, NULL);
+    xenevtchn_unbind(xendev->evtchndev, xendev->local_port);
     xen_be_printf(xendev, 2, "unbind evtchn port %d\n", xendev->local_port);
     xendev->local_port = -1;
 }
 
 int xen_be_send_notify(struct XenDevice *xendev)
 {
-    return xc_evtchn_notify(xendev->evtchndev, xendev->local_port);
+    return xenevtchn_notify(xendev->evtchndev, xendev->local_port);
 }
 
 /*
diff --git a/qemu/hw/xen/xen_devconfig.c b/qemu/hw/xen/xen_devconfig.c
index e138dbbec..1f30fe4f5 100644
--- a/qemu/hw/xen/xen_devconfig.c
+++ b/qemu/hw/xen/xen_devconfig.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
diff --git a/qemu/hw/xen/xen_pt.c b/qemu/hw/xen/xen_pt.c
index ed5fcaec0..f593b046e 100644
--- a/qemu/hw/xen/xen_pt.c
+++ b/qemu/hw/xen/xen_pt.c
@@ -52,10 +52,13 @@
  *         - Set entry->pirq to '-1'.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
 
 #include "hw/pci/pci.h"
 #include "hw/xen/xen.h"
+#include "hw/i386/pc.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
 #include "qemu/range.h"
@@ -378,7 +381,7 @@ static void xen_pt_pci_write_config(PCIDevice *d, uint32_t addr,
         }
     }
 
-    /* need to shift back before passing them to xen_host_pci_device */
+    /* need to shift back before passing them to xen_host_pci_set_block. */
     val >>= (addr & 3) << 3;
 
     memory_region_transaction_commit();
@@ -406,7 +409,7 @@ out:
                                     (uint8_t *)&val + index, len);
 
         if (rc < 0) {
-            XEN_PT_ERR(d, "pci_write_block failed. return value: %d.\n", rc);
+            XEN_PT_ERR(d, "xen_host_pci_set_block failed. return value: %d.\n", rc);
         }
     }
 }
@@ -502,6 +505,7 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s, uint16_t *cmd)
                    d->rom.size, d->rom.base_addr);
     }
 
+    xen_pt_register_vga_regions(d);
     return 0;
 }
 
@@ -683,15 +687,89 @@ static const MemoryListener xen_pt_io_listener = {
     .priority = 10,
 };
 
+static void
+xen_igd_passthrough_isa_bridge_create(XenPCIPassthroughState *s,
+                                      XenHostPCIDevice *dev)
+{
+    uint16_t gpu_dev_id;
+    PCIDevice *d = &s->dev;
+
+    gpu_dev_id = dev->device_id;
+    igd_passthrough_isa_bridge_create(d->bus, gpu_dev_id);
+}
+
+/* destroy. */
+static void xen_pt_destroy(PCIDevice *d) {
+
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
+    XenHostPCIDevice *host_dev = &s->real_device;
+    uint8_t machine_irq = s->machine_irq;
+    uint8_t intx;
+    int rc;
+
+    if (machine_irq && !xen_host_pci_device_closed(&s->real_device)) {
+        intx = xen_pt_pci_intx(s);
+        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
+                                     PT_IRQ_TYPE_PCI,
+                                     pci_bus_num(d->bus),
+                                     PCI_SLOT(s->dev.devfn),
+                                     intx,
+                                     0 /* isa_irq */);
+        if (rc < 0) {
+            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
+                       " (machine irq: %i, err: %d)"
+                       " But bravely continuing on..\n",
+                       'a' + intx, machine_irq, errno);
+        }
+    }
+
+    /* N.B. xen_pt_config_delete takes care of freeing them. */
+    if (s->msi) {
+        xen_pt_msi_disable(s);
+    }
+    if (s->msix) {
+        xen_pt_msix_disable(s);
+    }
+
+    if (machine_irq) {
+        xen_pt_mapped_machine_irq[machine_irq]--;
+
+        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
+            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
+
+            if (rc < 0) {
+                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (err: %d)"
+                           " But bravely continuing on..\n",
+                           machine_irq, errno);
+            }
+        }
+        s->machine_irq = 0;
+    }
+
+    /* delete all emulated config registers */
+    xen_pt_config_delete(s);
+
+    xen_pt_unregister_vga_regions(host_dev);
+
+    if (s->listener_set) {
+        memory_listener_unregister(&s->memory_listener);
+        memory_listener_unregister(&s->io_listener);
+        s->listener_set = false;
+    }
+    if (!xen_host_pci_device_closed(&s->real_device)) {
+        xen_host_pci_device_put(&s->real_device);
+    }
+}
 /* init */
 
-static int xen_pt_initfn(PCIDevice *d)
+static void xen_pt_realize(PCIDevice *d, Error **errp)
 {
     XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
-    int rc = 0;
-    uint8_t machine_irq = 0;
+    int i, rc = 0;
+    uint8_t machine_irq = 0, scratch;
     uint16_t cmd = 0;
     int pirq = XEN_PT_UNASSIGNED_PIRQ;
+    Error *err = NULL;
 
     /* register real device */
     XEN_PT_LOG(d, "Assigning real physical device %02x:%02x.%d"
@@ -699,12 +777,14 @@ static int xen_pt_initfn(PCIDevice *d)
                s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function,
                s->dev.devfn);
 
-    rc = xen_host_pci_device_get(&s->real_device,
-                                 s->hostaddr.domain, s->hostaddr.bus,
-                                 s->hostaddr.slot, s->hostaddr.function);
-    if (rc) {
-        XEN_PT_ERR(d, "Failed to \"open\" the real pci device. rc: %i\n", rc);
-        return -1;
+    xen_host_pci_device_get(&s->real_device,
+                            s->hostaddr.domain, s->hostaddr.bus,
+                            s->hostaddr.slot, s->hostaddr.function,
+                            &err);
+    if (err) {
+        error_append_hint(&err, "Failed to \"open\" the real pci device");
+        error_propagate(errp, err);
+        return;
     }
 
     s->is_virtfn = s->real_device.is_virtfn;
@@ -715,37 +795,62 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
     /* Initialize virtualized PCI configuration (Extended 256 Bytes) */
-    if (xen_host_pci_get_block(&s->real_device, 0, d->config,
-                               PCI_CONFIG_SPACE_SIZE) == -1) {
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
-    }
+    memset(d->config, 0, PCI_CONFIG_SPACE_SIZE);
 
     s->memory_listener = xen_pt_memory_listener;
     s->io_listener = xen_pt_io_listener;
 
+    /* Setup VGA bios for passthrough GFX */
+    if ((s->real_device.domain == 0) && (s->real_device.bus == 0) &&
+        (s->real_device.dev == 2) && (s->real_device.func == 0)) {
+        if (!is_igd_vga_passthrough(&s->real_device)) {
+            error_setg(errp, "Need to enable igd-passthru if you're trying"
+                    " to passthrough IGD GFX");
+            xen_host_pci_device_put(&s->real_device);
+            return;
+        }
+
+        xen_pt_setup_vga(s, &s->real_device, &err);
+        if (err) {
+            error_append_hint(&err, "Setup VGA BIOS of passthrough"
+                    " GFX failed");
+            error_propagate(errp, err);
+            xen_host_pci_device_put(&s->real_device);
+            return;
+        }
+
+        /* Register ISA bridge for passthrough GFX. */
+        xen_igd_passthrough_isa_bridge_create(s, &s->real_device);
+    }
+
     /* Handle real device's MMIO/PIO BARs */
     xen_pt_register_regions(s, &cmd);
 
     /* reinitialize each config register to be emulated */
-    if (xen_pt_config_init(s)) {
-        XEN_PT_ERR(d, "PCI Config space initialisation failed.\n");
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
+    xen_pt_config_init(s, &err);
+    if (err) {
+        error_append_hint(&err, "PCI Config space initialisation failed");
+        error_report_err(err);
+        rc = -1;
+        goto err_out;
     }
 
     /* Bind interrupt */
-    if (!s->dev.config[PCI_INTERRUPT_PIN]) {
-        XEN_PT_LOG(d, "no pin interrupt\n");
+    rc = xen_host_pci_get_byte(&s->real_device, PCI_INTERRUPT_PIN, &scratch);
+    if (rc) {
+        error_setg_errno(errp, errno, "Failed to read PCI_INTERRUPT_PIN");
+        goto err_out;
+    }
+    if (!scratch) {
+        error_setg(errp, "no pin interrupt");
         goto out;
     }
 
     machine_irq = s->real_device.irq;
     rc = xc_physdev_map_pirq(xen_xc, xen_domid, machine_irq, &pirq);
-
     if (rc < 0) {
-        XEN_PT_ERR(d, "Mapping machine irq %u to pirq %i failed, (err: %d)\n",
-                   machine_irq, pirq, errno);
+        error_setg_errno(errp, errno, "Mapping machine irq %u to"
+                         " pirq %i failed", machine_irq, pirq);
 
         /* Disable PCI intx assertion (turn on bit10 of devctl) */
         cmd |= PCI_COMMAND_INTX_DISABLE;
@@ -766,8 +871,8 @@ static int xen_pt_initfn(PCIDevice *d)
                                        PCI_SLOT(d->devfn),
                                        e_intx);
         if (rc < 0) {
-            XEN_PT_ERR(d, "Binding of interrupt %i failed! (err: %d)\n",
-                       e_intx, errno);
+            error_setg_errno(errp, errno, "Binding of interrupt %u failed",
+                             e_intx);
 
             /* Disable PCI intx assertion (turn on bit10 of devctl) */
             cmd |= PCI_COMMAND_INTX_DISABLE;
@@ -775,8 +880,8 @@ static int xen_pt_initfn(PCIDevice *d)
 
             if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
                 if (xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq)) {
-                    XEN_PT_ERR(d, "Unmapping of machine interrupt %i failed!"
-                               " (err: %d)\n", machine_irq, errno);
+                    error_setg_errno(errp, errno, "Unmapping of machine"
+                            " interrupt %u failed", machine_irq);
                 }
             }
             s->machine_irq = 0;
@@ -785,69 +890,45 @@ static int xen_pt_initfn(PCIDevice *d)
 
 out:
     if (cmd) {
-        xen_host_pci_set_word(&s->real_device, PCI_COMMAND,
-                              pci_get_word(d->config + PCI_COMMAND) | cmd);
+        uint16_t val;
+
+        rc = xen_host_pci_get_word(&s->real_device, PCI_COMMAND, &val);
+        if (rc) {
+            error_setg_errno(errp, errno, "Failed to read PCI_COMMAND");
+            goto err_out;
+        } else {
+            val |= cmd;
+            rc = xen_host_pci_set_word(&s->real_device, PCI_COMMAND, val);
+            if (rc) {
+                error_setg_errno(errp, errno, "Failed to write PCI_COMMAND"
+                                 " val = 0x%x", val);
+                goto err_out;
+            }
+        }
     }
 
     memory_listener_register(&s->memory_listener, &s->dev.bus_master_as);
     memory_listener_register(&s->io_listener, &address_space_io);
+    s->listener_set = true;
     XEN_PT_LOG(d,
-               "Real physical device %02x:%02x.%d registered successfully!\n",
+               "Real physical device %02x:%02x.%d registered successfully\n",
                s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
 
-    return 0;
-}
-
-static void xen_pt_unregister_device(PCIDevice *d)
-{
-    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
-    uint8_t machine_irq = s->machine_irq;
-    uint8_t intx = xen_pt_pci_intx(s);
-    int rc;
-
-    if (machine_irq) {
-        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
-                                     PT_IRQ_TYPE_PCI,
-                                     pci_bus_num(d->bus),
-                                     PCI_SLOT(s->dev.devfn),
-                                     intx,
-                                     0 /* isa_irq */);
-        if (rc < 0) {
-            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
-                       " (machine irq: %i, err: %d)"
-                       " But bravely continuing on..\n",
-                       'a' + intx, machine_irq, errno);
-        }
-    }
-
-    if (s->msi) {
-        xen_pt_msi_disable(s);
-    }
-    if (s->msix) {
-        xen_pt_msix_disable(s);
-    }
-
-    if (machine_irq) {
-        xen_pt_mapped_machine_irq[machine_irq]--;
+    return;
 
-        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
-            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
-
-            if (rc < 0) {
-                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (err: %d)"
-                           " But bravely continuing on..\n",
-                           machine_irq, errno);
-            }
-        }
+err_out:
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        object_unparent(OBJECT(&s->bar[i]));
     }
+    object_unparent(OBJECT(&s->rom));
 
-    /* delete all emulated config registers */
-    xen_pt_config_delete(s);
-
-    memory_listener_unregister(&s->memory_listener);
-    memory_listener_unregister(&s->io_listener);
+    xen_pt_destroy(d);
+    assert(rc);
+}
 
-    xen_host_pci_device_put(&s->real_device);
+static void xen_pt_unregister_device(PCIDevice *d)
+{
+    xen_pt_destroy(d);
 }
 
 static Property xen_pci_passthrough_properties[] = {
@@ -861,7 +942,7 @@ static void xen_pci_passthrough_class_init(ObjectClass *klass, void *data)
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_pt_initfn;
+    k->realize = xen_pt_realize;
     k->exit = xen_pt_unregister_device;
     k->config_read = xen_pt_pci_read_config;
     k->config_write = xen_pt_pci_write_config;
@@ -870,10 +951,18 @@ static void xen_pci_passthrough_class_init(ObjectClass *klass, void *data)
     dc->props = xen_pci_passthrough_properties;
 };
 
+static void xen_pci_passthrough_finalize(Object *obj)
+{
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(obj);
+
+    xen_pt_msix_delete(s);
+}
+
 static const TypeInfo xen_pci_passthrough_info = {
     .name = TYPE_XEN_PT_DEVICE,
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XenPCIPassthroughState),
+    .instance_finalize = xen_pci_passthrough_finalize,
     .class_init = xen_pci_passthrough_class_init,
 };
 
diff --git a/qemu/hw/xen/xen_pt.h b/qemu/hw/xen/xen_pt.h
index 393f36ccb..c2f8e1fc2 100644
--- a/qemu/hw/xen/xen_pt.h
+++ b/qemu/hw/xen/xen_pt.h
@@ -40,6 +40,9 @@ typedef struct XenPCIPassthroughState XenPCIPassthroughState;
 #define XEN_PT_DEVICE(obj) \
     OBJECT_CHECK(XenPCIPassthroughState, (obj), TYPE_XEN_PT_DEVICE)
 
+uint32_t igd_read_opregion(XenPCIPassthroughState *s);
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val);
+
 /* function type for config reg */
 typedef int (*xen_pt_conf_reg_init)
     (XenPCIPassthroughState *, XenPTRegInfo *, uint32_t real_offset,
@@ -66,8 +69,9 @@ typedef int (*xen_pt_conf_byte_read)
 #define XEN_PT_BAR_ALLF 0xFFFFFFFF
 #define XEN_PT_BAR_UNMAPPED (-1)
 
-#define PCI_CAP_MAX 48
+#define XEN_PCI_CAP_MAX 48
 
+#define XEN_PCI_INTEL_OPREGION 0xfc
 
 typedef enum {
     XEN_PT_GRP_TYPE_HARDWIRED = 0,  /* 0 Hardwired reg group */
@@ -109,6 +113,8 @@ struct XenPTRegInfo {
     uint32_t res_mask;
     /* reg read only field mask (ON:RO/ROS, OFF:other) */
     uint32_t ro_mask;
+    /* reg read/write-1-clear field mask (ON:RW1C/RW1CS, OFF:other) */
+    uint32_t rw1c_mask;
     /* reg emulate field mask (ON:emu, OFF:passthrough) */
     uint32_t emu_mask;
     xen_pt_conf_reg_init init;
@@ -134,7 +140,11 @@ struct XenPTRegInfo {
 struct XenPTReg {
     QLIST_ENTRY(XenPTReg) entries;
     XenPTRegInfo *reg;
-    uint32_t data; /* emulated value */
+    union {
+        uint8_t *byte;
+        uint16_t *half_word;
+        uint32_t *word;
+    } ptr; /* pointer to dev.config. */
 };
 
 typedef const struct XenPTRegGroupInfo XenPTRegGroupInfo;
@@ -179,13 +189,13 @@ typedef struct XenPTMSIXEntry {
     int pirq;
     uint64_t addr;
     uint32_t data;
-    uint32_t vector_ctrl;
+    uint32_t latch[4];
     bool updated; /* indicate whether MSI ADDR or DATA is updated */
-    bool warned;  /* avoid issuing (bogus) warning more than once */
 } XenPTMSIXEntry;
 typedef struct XenPTMSIX {
     uint32_t ctrl_offset;
     bool enabled;
+    bool maskall;
     int total_entries;
     int bar_index;
     uint64_t table_base;
@@ -217,9 +227,10 @@ struct XenPCIPassthroughState {
 
     MemoryListener memory_listener;
     MemoryListener io_listener;
+    bool listener_set;
 };
 
-int xen_pt_config_init(XenPCIPassthroughState *s);
+void xen_pt_config_init(XenPCIPassthroughState *s, Error **errp);
 void xen_pt_config_delete(XenPCIPassthroughState *s);
 XenPTRegGroup *xen_pt_find_reg_grp(XenPCIPassthroughState *s, uint32_t address);
 XenPTReg *xen_pt_find_reg(XenPTRegGroup *reg_grp, uint32_t address);
@@ -282,6 +293,7 @@ static inline uint8_t xen_pt_pci_intx(XenPCIPassthroughState *s)
                    " value=%i, acceptable range is 1 - 4\n", r_val);
         r_val = 0;
     } else {
+        /* Note that if s.real_device.config_fd is closed we make 0xff. */
         r_val -= 1;
     }
 
@@ -289,13 +301,13 @@ static inline uint8_t xen_pt_pci_intx(XenPCIPassthroughState *s)
 }
 
 /* MSI/MSI-X */
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool en);
 int xen_pt_msi_setup(XenPCIPassthroughState *s);
 int xen_pt_msi_update(XenPCIPassthroughState *d);
 void xen_pt_msi_disable(XenPCIPassthroughState *s);
 
 int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base);
 void xen_pt_msix_delete(XenPCIPassthroughState *s);
+void xen_pt_msix_unmap(XenPCIPassthroughState *s);
 int xen_pt_msix_update(XenPCIPassthroughState *s);
 int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index);
 void xen_pt_msix_disable(XenPCIPassthroughState *s);
@@ -305,5 +317,19 @@ static inline bool xen_pt_has_msix_mapping(XenPCIPassthroughState *s, int bar)
     return s->msix && s->msix->bar_index == bar;
 }
 
-
+extern void *pci_assign_dev_load_option_rom(PCIDevice *dev,
+                                            struct Object *owner, int *size,
+                                            unsigned int domain,
+                                            unsigned int bus, unsigned int slot,
+                                            unsigned int function);
+extern bool has_igd_gfx_passthru;
+static inline bool is_igd_vga_passthrough(XenHostPCIDevice *dev)
+{
+    return (has_igd_gfx_passthru
+            && ((dev->class_code >> 0x8) == PCI_CLASS_DISPLAY_VGA));
+}
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev);
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev);
+void xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev,
+                     Error **errp);
 #endif /* !XEN_PT_H */
diff --git a/qemu/hw/xen/xen_pt_config_init.c b/qemu/hw/xen/xen_pt_config_init.c
index dd37be38a..9869ffda0 100644
--- a/qemu/hw/xen/xen_pt_config_init.c
+++ b/qemu/hw/xen/xen_pt_config_init.c
@@ -12,6 +12,8 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
@@ -128,10 +130,11 @@ static int xen_pt_byte_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t valid_emu_mask = 0;
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* emulate byte register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -140,10 +143,11 @@ static int xen_pt_word_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t valid_emu_mask = 0;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* emulate word register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -152,10 +156,11 @@ static int xen_pt_long_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t valid_emu_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* emulate long register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -169,13 +174,15 @@ static int xen_pt_byte_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t writable_mask = 0;
     uint8_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -186,13 +193,15 @@ static int xen_pt_word_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -203,13 +212,15 @@ static int xen_pt_long_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
     uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -255,7 +266,7 @@ static int xen_pt_status_reg_init(XenPCIPassthroughState *s,
         reg_entry = xen_pt_find_reg(reg_grp_entry, PCI_CAPABILITY_LIST);
         if (reg_entry) {
             /* check Capabilities Pointer register */
-            if (reg_entry->data) {
+            if (*reg_entry->ptr.half_word) {
                 reg_field |= PCI_STATUS_CAP_LIST;
             } else {
                 reg_field &= ~PCI_STATUS_CAP_LIST;
@@ -301,10 +312,11 @@ static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     if (*val & PCI_COMMAND_INTX_DISABLE) {
@@ -447,7 +459,7 @@ static int xen_pt_bar_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* emulate BAR */
     valid_emu_mask = bar_emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *cfg_entry->ptr.word, ~valid_emu_mask);
 
     return 0;
 }
@@ -464,6 +476,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     uint32_t bar_ro_mask = 0;
     uint32_t r_size = 0;
     int index = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     index = xen_pt_bar_offset_to_index(reg->offset);
     if (index < 0 || index >= PCI_NUM_REGIONS) {
@@ -500,7 +513,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* modify emulate register */
     writable_mask = bar_emu_mask & ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* check whether we need to update the virtual region address or not */
     switch (s->bases[index].bar_flag) {
@@ -533,6 +546,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     pcibus_t r_size = 0;
     uint32_t bar_ro_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     r_size = d->io_regions[PCI_ROM_SLOT].size;
     base = &s->bases[PCI_ROM_SLOT];
@@ -544,7 +558,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -552,6 +566,22 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
+static int xen_pt_intel_opregion_read(XenPCIPassthroughState *s,
+                                      XenPTReg *cfg_entry,
+                                      uint32_t *value, uint32_t valid_mask)
+{
+    *value = igd_read_opregion(s);
+    return 0;
+}
+
+static int xen_pt_intel_opregion_write(XenPCIPassthroughState *s,
+                                       XenPTReg *cfg_entry, uint32_t *value,
+                                       uint32_t dev_value, uint32_t valid_mask)
+{
+    igd_write_opregion(s, *value);
+    return 0;
+}
+
 /* Header Type0 reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_header0[] = {
     /* Vendor ID reg */
@@ -608,6 +638,7 @@ static XenPTRegInfo xen_pt_emu_reg_header0[] = {
         .init_val   = 0x0000,
         .res_mask   = 0x0007,
         .ro_mask    = 0x06F8,
+        .rw1c_mask  = 0xF900,
         .emu_mask   = 0x0010,
         .init       = xen_pt_status_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
@@ -800,15 +831,21 @@ static XenPTRegInfo xen_pt_emu_reg_vendor[] = {
 static inline uint8_t get_capability_version(XenPCIPassthroughState *s,
                                              uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return flags & PCI_EXP_FLAGS_VERS;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return flag & PCI_EXP_FLAGS_VERS;
 }
 
 static inline uint8_t get_device_type(XenPCIPassthroughState *s,
                                       uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return (flags & PCI_EXP_FLAGS_TYPE) >> 4;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return (flag & PCI_EXP_FLAGS_TYPE) >> 4;
 }
 
 /* initialize Link Control register */
@@ -857,8 +894,14 @@ static int xen_pt_linkctrl2_reg_init(XenPCIPassthroughState *s,
         reg_field = XEN_PT_INVALID_REG;
     } else {
         /* set Supported Link Speed */
-        uint8_t lnkcap = pci_get_byte(s->dev.config + real_offset - reg->offset
-                                      + PCI_EXP_LNKCAP);
+        uint8_t lnkcap;
+        int rc;
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   real_offset - reg->offset + PCI_EXP_LNKCAP,
+                                   &lnkcap);
+        if (rc) {
+            return rc;
+        }
         reg_field |= PCI_EXP_LNKCAP_SLS & lnkcap;
     }
 
@@ -907,6 +950,7 @@ static XenPTRegInfo xen_pt_emu_reg_pcie[] = {
         .size       = 2,
         .res_mask   = 0xFFC0,
         .ro_mask    = 0x0030,
+        .rw1c_mask  = 0x000F,
         .init       = xen_pt_common_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_word_reg_write,
@@ -927,6 +971,7 @@ static XenPTRegInfo xen_pt_emu_reg_pcie[] = {
         .offset     = PCI_EXP_LNKSTA,
         .size       = 2,
         .ro_mask    = 0x3FFF,
+        .rw1c_mask  = 0xC000,
         .init       = xen_pt_common_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_word_reg_write,
@@ -963,26 +1008,6 @@ static XenPTRegInfo xen_pt_emu_reg_pcie[] = {
  * Power Management Capability
  */
 
-/* write Power Management Control/Status register */
-static int xen_pt_pmcsr_reg_write(XenPCIPassthroughState *s,
-                                  XenPTReg *cfg_entry, uint16_t *val,
-                                  uint16_t dev_value, uint16_t valid_mask)
-{
-    XenPTRegInfo *reg = cfg_entry->reg;
-    uint16_t writable_mask = 0;
-    uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
-
-    /* modify emulate register */
-    writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-
-    /* create value for writing to I/O device register */
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~PCI_PM_CTRL_PME_STATUS,
-                              throughable_mask);
-
-    return 0;
-}
-
 /* Power Management Capability reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pm[] = {
     /* Next Pointer reg */
@@ -1013,11 +1038,12 @@ static XenPTRegInfo xen_pt_emu_reg_pm[] = {
         .size       = 2,
         .init_val   = 0x0008,
         .res_mask   = 0x00F0,
-        .ro_mask    = 0xE10C,
+        .ro_mask    = 0x610C,
+        .rw1c_mask  = 0x8000,
         .emu_mask   = 0x810B,
         .init       = xen_pt_common_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
-        .u.w.write  = xen_pt_pmcsr_reg_write,
+        .u.w.write  = xen_pt_word_reg_write,
     },
     {
         .size = 0,
@@ -1039,13 +1065,15 @@ static int xen_pt_msgctrl_reg_init(XenPCIPassthroughState *s,
                                    XenPTRegInfo *reg, uint32_t real_offset,
                                    uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
     XenPTMSI *msi = s->msi;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSI_FLAGS_ENABLE) {
         XEN_PT_LOG(&s->dev, "MSI already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
@@ -1067,6 +1095,7 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* Currently no support for multi-vector */
     if (*val & PCI_MSI_FLAGS_QSIZE) {
@@ -1075,8 +1104,8 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    msi->flags |= cfg_entry->data & ~PCI_MSI_FLAGS_ENABLE;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    msi->flags |= *data & ~PCI_MSI_FLAGS_ENABLE;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -1086,7 +1115,7 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
         /* setup MSI pirq for the first time */
         if (!msi->initialized) {
             /* Init physical one */
-            XEN_PT_LOG(&s->dev, "setup MSI\n");
+            XEN_PT_LOG(&s->dev, "setup MSI (register: %x).\n", *val);
             if (xen_pt_msi_setup(s)) {
                 /* We do not broadcast the error to the framework code, so
                  * that MSI errors are contained in MSI emulation code and
@@ -1094,12 +1123,12 @@ static int xen_pt_msgctrl_reg_write(XenPCIPassthroughState *s,
                  * Guest MSI would be actually not working.
                  */
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not map MSI.\n");
+                XEN_PT_WARN(&s->dev, "Can not map MSI (register: %x)!\n", *val);
                 return 0;
             }
             if (xen_pt_msi_update(s)) {
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not bind MSI\n");
+                XEN_PT_WARN(&s->dev, "Can not bind MSI (register: %x)!\n", *val);
                 return 0;
             }
             msi->initialized = true;
@@ -1190,18 +1219,19 @@ static int xen_pt_msgaddr32_reg_write(XenPCIPassthroughState *s,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    s->msi->addr_lo = cfg_entry->data;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    s->msi->addr_lo = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1216,7 +1246,8 @@ static int xen_pt_msgaddr64_reg_write(XenPCIPassthroughState *s,
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* check whether the type is 64 bit or not */
     if (!(s->msi->flags & PCI_MSI_FLAGS_64BIT)) {
@@ -1227,15 +1258,15 @@ static int xen_pt_msgaddr64_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    s->msi->addr_hi = cfg_entry->data;
+    s->msi->addr_hi = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1254,8 +1285,9 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
     XenPTRegInfo *reg = cfg_entry->reg;
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
-    uint16_t old_data = cfg_entry->data;
+    uint16_t old_data = *cfg_entry->ptr.half_word;
     uint32_t offset = reg->offset;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* check the offset whether matches the type or not */
     if (!xen_pt_msi_check_type(offset, msi->flags, DATA)) {
@@ -1266,15 +1298,15 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    msi->data = cfg_entry->data;
+    msi->data = *data;
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_data) {
+    if (*data != old_data) {
         if (msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1411,14 +1443,16 @@ static int xen_pt_msixctrl_reg_init(XenPCIPassthroughState *s,
                                     XenPTRegInfo *reg, uint32_t real_offset,
                                     uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSIX_FLAGS_ENABLE) {
-        XEN_PT_LOG(d, "MSIX already enabled, disabling it first\n");
+        XEN_PT_LOG(&s->dev, "MSIX already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
                               reg_field & ~PCI_MSIX_FLAGS_ENABLE);
     }
@@ -1436,10 +1470,11 @@ static int xen_pt_msixctrl_reg_write(XenPCIPassthroughState *s,
     uint16_t writable_mask = 0;
     uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     int debug_msix_enabled_old;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
@@ -1452,6 +1487,8 @@ static int xen_pt_msixctrl_reg_write(XenPCIPassthroughState *s,
         xen_pt_msix_disable(s);
     }
 
+    s->msix->maskall = *val & PCI_MSIX_FLAGS_MASKALL;
+
     debug_msix_enabled_old = s->msix->enabled;
     s->msix->enabled = !!(*val & PCI_MSIX_FLAGS_ENABLE);
     if (s->msix->enabled != debug_msix_enabled_old) {
@@ -1492,6 +1529,19 @@ static XenPTRegInfo xen_pt_emu_reg_msix[] = {
     },
 };
 
+static XenPTRegInfo xen_pt_emu_reg_igd_opregion[] = {
+    /* Intel IGFX OpRegion reg */
+    {
+        .offset     = 0x0,
+        .size       = 4,
+        .init_val   = 0,
+        .u.dw.read   = xen_pt_intel_opregion_read,
+        .u.dw.write  = xen_pt_intel_opregion_write,
+    },
+    {
+        .size = 0,
+    },
+};
 
 /****************************
  * Capabilities
@@ -1511,8 +1561,7 @@ static int xen_pt_vendor_size_init(XenPCIPassthroughState *s,
                                    const XenPTRegGroupInfo *grp_reg,
                                    uint32_t base_offset, uint8_t *size)
 {
-    *size = pci_get_byte(s->dev.config + base_offset + 0x02);
-    return 0;
+    return xen_host_pci_get_byte(&s->real_device, base_offset + 0x02, size);
 }
 /* get PCI Express Capability Structure register group size */
 static int xen_pt_pcie_size_init(XenPCIPassthroughState *s,
@@ -1591,12 +1640,15 @@ static int xen_pt_msi_size_init(XenPCIPassthroughState *s,
                                 const XenPTRegGroupInfo *grp_reg,
                                 uint32_t base_offset, uint8_t *size)
 {
-    PCIDevice *d = &s->dev;
     uint16_t msg_ctrl = 0;
     uint8_t msi_size = 0xa;
+    int rc;
 
-    msg_ctrl = pci_get_word(d->config + (base_offset + PCI_MSI_FLAGS));
-
+    rc = xen_host_pci_get_word(&s->real_device, base_offset + PCI_MSI_FLAGS,
+                               &msg_ctrl);
+    if (rc) {
+        return rc;
+    }
     /* check if 64-bit address is capable of per-vector masking */
     if (msg_ctrl & PCI_MSI_FLAGS_64BIT) {
         msi_size += 4;
@@ -1729,6 +1781,14 @@ static const XenPTRegGroupInfo xen_pt_emu_reg_grps[] = {
         .size_init   = xen_pt_msix_size_init,
         .emu_regs = xen_pt_emu_reg_msix,
     },
+    /* Intel IGD Opregion group */
+    {
+        .grp_id      = XEN_PCI_INTEL_OPREGION,
+        .grp_type    = XEN_PT_GRP_TYPE_EMU,
+        .grp_size    = 0x4,
+        .size_init   = xen_pt_reg_grp_size_init,
+        .emu_regs    = xen_pt_emu_reg_igd_opregion,
+    },
     {
         .grp_size = 0,
     },
@@ -1739,11 +1799,14 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
                                XenPTRegInfo *reg, uint32_t real_offset,
                                uint32_t *data)
 {
-    int i;
-    uint8_t *config = s->dev.config;
-    uint32_t reg_field = pci_get_byte(config + real_offset);
+    int i, rc;
+    uint8_t reg_field;
     uint8_t cap_id = 0;
 
+    rc = xen_host_pci_get_byte(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     /* find capability offset */
     while (reg_field) {
         for (i = 0; xen_pt_emu_reg_grps[i].grp_size != 0; i++) {
@@ -1752,7 +1815,13 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
                 continue;
             }
 
-            cap_id = pci_get_byte(config + reg_field + PCI_CAP_LIST_ID);
+            rc = xen_host_pci_get_byte(&s->real_device,
+                                       reg_field + PCI_CAP_LIST_ID, &cap_id);
+            if (rc) {
+                XEN_PT_ERR(&s->dev, "Failed to read capability @0x%x (rc:%d)\n",
+                           reg_field + PCI_CAP_LIST_ID, rc);
+                return rc;
+            }
             if (xen_pt_emu_reg_grps[i].grp_id == cap_id) {
                 if (xen_pt_emu_reg_grps[i].grp_type == XEN_PT_GRP_TYPE_EMU) {
                     goto out;
@@ -1763,7 +1832,11 @@ static int xen_pt_ptr_reg_init(XenPCIPassthroughState *s,
         }
 
         /* next capability */
-        reg_field = pci_get_byte(config + reg_field + PCI_CAP_LIST_NEXT);
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   reg_field + PCI_CAP_LIST_NEXT, &reg_field);
+        if (rc) {
+            return rc;
+        }
     }
 
 out:
@@ -1779,7 +1852,7 @@ out:
 static uint8_t find_cap_offset(XenPCIPassthroughState *s, uint8_t cap)
 {
     uint8_t id;
-    unsigned max_cap = PCI_CAP_MAX;
+    unsigned max_cap = XEN_PCI_CAP_MAX;
     uint8_t pos = PCI_CAPABILITY_LIST;
     uint8_t status = 0;
 
@@ -1816,8 +1889,9 @@ static uint8_t find_cap_offset(XenPCIPassthroughState *s, uint8_t cap)
     return 0;
 }
 
-static int xen_pt_config_reg_init(XenPCIPassthroughState *s,
-                                  XenPTRegGroup *reg_grp, XenPTRegInfo *reg)
+static void xen_pt_config_reg_init(XenPCIPassthroughState *s,
+                                   XenPTRegGroup *reg_grp, XenPTRegInfo *reg,
+                                   Error **errp)
 {
     XenPTReg *reg_entry;
     uint32_t data = 0;
@@ -1827,30 +1901,94 @@ static int xen_pt_config_reg_init(XenPCIPassthroughState *s,
     reg_entry->reg = reg;
 
     if (reg->init) {
+        uint32_t host_mask, size_mask;
+        unsigned int offset;
+        uint32_t val;
+
         /* initialize emulate register */
         rc = reg->init(s, reg_entry->reg,
                        reg_grp->base_offset + reg->offset, &data);
         if (rc < 0) {
             g_free(reg_entry);
-            return rc;
+            error_setg(errp, "Init emulate register fail");
+            return;
         }
         if (data == XEN_PT_INVALID_REG) {
             /* free unused BAR register entry */
             g_free(reg_entry);
-            return 0;
+            return;
+        }
+        /* Sync up the data to dev.config */
+        offset = reg_grp->base_offset + reg->offset;
+        size_mask = 0xFFFFFFFF >> ((4 - reg->size) << 3);
+
+        switch (reg->size) {
+        case 1: rc = xen_host_pci_get_byte(&s->real_device, offset, (uint8_t *)&val);
+                break;
+        case 2: rc = xen_host_pci_get_word(&s->real_device, offset, (uint16_t *)&val);
+                break;
+        case 4: rc = xen_host_pci_get_long(&s->real_device, offset, &val);
+                break;
+        default: abort();
+        }
+        if (rc) {
+            /* Serious issues when we cannot read the host values! */
+            g_free(reg_entry);
+            error_setg(errp, "Cannot read host values");
+            return;
+        }
+        /* Set bits in emu_mask are the ones we emulate. The dev.config shall
+         * contain the emulated view of the guest - therefore we flip the mask
+         * to mask out the host values (which dev.config initially has) . */
+        host_mask = size_mask & ~reg->emu_mask;
+
+        if ((data & host_mask) != (val & host_mask)) {
+            uint32_t new_val;
+
+            /* Mask out host (including past size). */
+            new_val = val & host_mask;
+            /* Merge emulated ones (excluding the non-emulated ones). */
+            new_val |= data & host_mask;
+            /* Leave intact host and emulated values past the size - even though
+             * we do not care as we write per reg->size granularity, but for the
+             * logging below lets have the proper value. */
+            new_val |= ((val | data)) & ~size_mask;
+            XEN_PT_LOG(&s->dev,"Offset 0x%04x mismatch! Emulated=0x%04x, host=0x%04x, syncing to 0x%04x.\n",
+                       offset, data, val, new_val);
+            val = new_val;
+        } else
+            val = data;
+
+        if (val & ~size_mask) {
+            error_setg(errp, "Offset 0x%04x:0x%04x expands past"
+                    " register size (%d)", offset, val, reg->size);
+            g_free(reg_entry);
+            return;
+        }
+        /* This could be just pci_set_long as we don't modify the bits
+         * past reg->size, but in case this routine is run in parallel or the
+         * init value is larger, we do not want to over-write registers. */
+        switch (reg->size) {
+        case 1: pci_set_byte(s->dev.config + offset, (uint8_t)val);
+                break;
+        case 2: pci_set_word(s->dev.config + offset, (uint16_t)val);
+                break;
+        case 4: pci_set_long(s->dev.config + offset, val);
+                break;
+        default: abort();
         }
-        /* set register value */
-        reg_entry->data = data;
+        /* set register value pointer to the data. */
+        reg_entry->ptr.byte = s->dev.config + offset;
+
     }
     /* list add register entry */
     QLIST_INSERT_HEAD(&reg_grp->reg_tbl_list, reg_entry, entries);
-
-    return 0;
 }
 
-int xen_pt_config_init(XenPCIPassthroughState *s)
+void xen_pt_config_init(XenPCIPassthroughState *s, Error **errp)
 {
     int i, rc;
+    Error *err = NULL;
 
     QLIST_INIT(&s->reg_grps);
 
@@ -1858,7 +1996,8 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
         uint32_t reg_grp_offset = 0;
         XenPTRegGroup *reg_grp_entry = NULL;
 
-        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF) {
+        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF
+            && xen_pt_emu_reg_grps[i].grp_id != XEN_PCI_INTEL_OPREGION) {
             if (xen_pt_hide_dev_cap(&s->real_device,
                                     xen_pt_emu_reg_grps[i].grp_id)) {
                 continue;
@@ -1871,6 +2010,15 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
             }
         }
 
+        /*
+         * By default we will trap up to 0x40 in the cfg space.
+         * If an intel device is pass through we need to trap 0xfc,
+         * therefore the size should be 0xff.
+         */
+        if (xen_pt_emu_reg_grps[i].grp_id == XEN_PCI_INTEL_OPREGION) {
+            reg_grp_offset = XEN_PCI_INTEL_OPREGION;
+        }
+
         reg_grp_entry = g_new0(XenPTRegGroup, 1);
         QLIST_INIT(&reg_grp_entry->reg_tbl_list);
         QLIST_INSERT_HEAD(&s->reg_grps, reg_grp_entry, entries);
@@ -1883,8 +2031,12 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
                                                   reg_grp_offset,
                                                   &reg_grp_entry->size);
             if (rc < 0) {
+                error_setg(&err, "Failed to initialize %d/%zu, type = 0x%x,"
+                           " rc: %d", i, ARRAY_SIZE(xen_pt_emu_reg_grps),
+                           xen_pt_emu_reg_grps[i].grp_type, rc);
+                error_propagate(errp, err);
                 xen_pt_config_delete(s);
-                return rc;
+                return;
             }
         }
 
@@ -1892,20 +2044,24 @@ int xen_pt_config_init(XenPCIPassthroughState *s)
             if (xen_pt_emu_reg_grps[i].emu_regs) {
                 int j = 0;
                 XenPTRegInfo *regs = xen_pt_emu_reg_grps[i].emu_regs;
+
                 /* initialize capability register */
                 for (j = 0; regs->size != 0; j++, regs++) {
-                    /* initialize capability register */
-                    rc = xen_pt_config_reg_init(s, reg_grp_entry, regs);
-                    if (rc < 0) {
+                    xen_pt_config_reg_init(s, reg_grp_entry, regs, &err);
+                    if (err) {
+                        error_append_hint(&err, "Failed to initialize %d/%zu"
+                                " reg 0x%x in grp_type = 0x%x (%d/%zu)",
+                                j, ARRAY_SIZE(xen_pt_emu_reg_grps[i].emu_regs),
+                                regs->offset, xen_pt_emu_reg_grps[i].grp_type,
+                                i, ARRAY_SIZE(xen_pt_emu_reg_grps));
+                        error_propagate(errp, err);
                         xen_pt_config_delete(s);
-                        return rc;
+                        return;
                     }
                 }
             }
         }
     }
-
-    return 0;
 }
 
 /* delete all emulate register */
@@ -1916,11 +2072,9 @@ void xen_pt_config_delete(XenPCIPassthroughState *s)
 
     /* free MSI/MSI-X info table */
     if (s->msix) {
-        xen_pt_msix_delete(s);
-    }
-    if (s->msi) {
-        g_free(s->msi);
+        xen_pt_msix_unmap(s);
     }
+    g_free(s->msi);
 
     /* free all register group entry */
     QLIST_FOREACH_SAFE(reg_group, &s->reg_grps, entries, next_grp) {
diff --git a/qemu/hw/xen/xen_pt_graphics.c b/qemu/hw/xen/xen_pt_graphics.c
new file mode 100644
index 000000000..0f4c8d77e
--- /dev/null
+++ b/qemu/hw/xen/xen_pt_graphics.c
@@ -0,0 +1,275 @@
+/*
+ * graphics passthrough
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "xen_pt.h"
+#include "xen-host-pci-device.h"
+#include "hw/xen/xen_backend.h"
+
+static unsigned long igd_guest_opregion;
+static unsigned long igd_host_opregion;
+
+#define XEN_PCI_INTEL_OPREGION_MASK 0xfff
+
+typedef struct VGARegion {
+    int type;           /* Memory or port I/O */
+    uint64_t guest_base_addr;
+    uint64_t machine_base_addr;
+    uint64_t size;    /* size of the region */
+    int rc;
+} VGARegion;
+
+#define IORESOURCE_IO           0x00000100
+#define IORESOURCE_MEM          0x00000200
+
+static struct VGARegion vga_args[] = {
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3B0,
+        .machine_base_addr = 0x3B0,
+        .size = 0xC,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3C0,
+        .machine_base_addr = 0x3C0,
+        .size = 0x20,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_MEM,
+        .guest_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .machine_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .size = 0x20,
+        .rc = -1,
+    },
+};
+
+/*
+ * register VGA resources for the domain with assigned gfx
+ */
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s mapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * unregister VGA resources for the domain with assigned gfx
+ */
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+    int ret = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s unmapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    if (igd_guest_opregion) {
+        ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+                (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+                (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                3,
+                DPCI_REMOVE_MAPPING);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void *get_vgabios(XenPCIPassthroughState *s, int *size,
+                       XenHostPCIDevice *dev)
+{
+    return pci_assign_dev_load_option_rom(&s->dev, OBJECT(&s->dev), size,
+                                          dev->domain, dev->bus,
+                                          dev->dev, dev->func);
+}
+
+/* Refer to Seabios. */
+struct rom_header {
+    uint16_t signature;
+    uint8_t size;
+    uint8_t initVector[4];
+    uint8_t reserved[17];
+    uint16_t pcioffset;
+    uint16_t pnpoffset;
+} __attribute__((packed));
+
+struct pci_data {
+    uint32_t signature;
+    uint16_t vendor;
+    uint16_t device;
+    uint16_t vitaldata;
+    uint16_t dlen;
+    uint8_t drevision;
+    uint8_t class_lo;
+    uint16_t class_hi;
+    uint16_t ilen;
+    uint16_t irevision;
+    uint8_t type;
+    uint8_t indicator;
+    uint16_t reserved;
+} __attribute__((packed));
+
+void xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev,
+                     Error **errp)
+{
+    unsigned char *bios = NULL;
+    struct rom_header *rom;
+    int bios_size;
+    char *c = NULL;
+    char checksum = 0;
+    uint32_t len = 0;
+    struct pci_data *pd = NULL;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        error_setg(errp, "Need to enable igd-passthrough");
+        return;
+    }
+
+    bios = get_vgabios(s, &bios_size, dev);
+    if (!bios) {
+        error_setg(errp, "VGA: Can't get VBIOS");
+        return;
+    }
+
+    /* Currently we fixed this address as a primary. */
+    rom = (struct rom_header *)bios;
+    pd = (void *)(bios + (unsigned char)rom->pcioffset);
+
+    /* We may need to fixup Device Identification. */
+    if (pd->device != s->real_device.device_id) {
+        pd->device = s->real_device.device_id;
+
+        len = rom->size * 512;
+        /* Then adjust the bios checksum */
+        for (c = (char *)bios; c < ((char *)bios + len); c++) {
+            checksum += *c;
+        }
+        if (checksum) {
+            bios[len - 1] -= checksum;
+            XEN_PT_LOG(&s->dev, "vga bios checksum is adjusted %x!\n",
+                       checksum);
+        }
+    }
+
+    /* Currently we fixed this address as a primary for legacy BIOS. */
+    cpu_physical_memory_rw(0xc0000, bios, bios_size, 1);
+}
+
+uint32_t igd_read_opregion(XenPCIPassthroughState *s)
+{
+    uint32_t val = 0;
+
+    if (!igd_guest_opregion) {
+        return val;
+    }
+
+    val = igd_guest_opregion;
+
+    XEN_PT_LOG(&s->dev, "Read opregion val=%x\n", val);
+    return val;
+}
+
+#define XEN_PCI_INTEL_OPREGION_PAGES 0x3
+#define XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED 0x1
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val)
+{
+    int ret;
+
+    if (igd_guest_opregion) {
+        XEN_PT_LOG(&s->dev, "opregion register already been set, ignoring %x\n",
+                   val);
+        return;
+    }
+
+    /* We just work with LE. */
+    xen_host_pci_get_block(&s->real_device, XEN_PCI_INTEL_OPREGION,
+            (uint8_t *)&igd_host_opregion, 4);
+    igd_guest_opregion = (unsigned long)(val & ~XEN_PCI_INTEL_OPREGION_MASK)
+                            | (igd_host_opregion & XEN_PCI_INTEL_OPREGION_MASK);
+
+    ret = xc_domain_iomem_permission(xen_xc, xen_domid,
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't enable to access IGD host opregion:"
+                    " 0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT)),
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+            (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            DPCI_ADD_MAPPING);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't map IGD host opregion:0x%lx to"
+                    " guest opregion:0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    XEN_PT_LOG(&s->dev, "Map OpRegion: 0x%lx -> 0x%lx\n",
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+}
diff --git a/qemu/hw/xen/xen_pt_msi.c b/qemu/hw/xen/xen_pt_msi.c
index 263e0514a..9a16f2bff 100644
--- a/qemu/hw/xen/xen_pt_msi.c
+++ b/qemu/hw/xen/xen_pt_msi.c
@@ -9,6 +9,7 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 
 #include "hw/xen/xen_backend.h"
@@ -25,6 +26,7 @@
 #define XEN_PT_GFLAGSSHIFT_DELIV_MODE     12
 #define XEN_PT_GFLAGSSHIFT_TRG_MODE       15
 
+#define latch(fld) latch[PCI_MSIX_ENTRY_##fld / sizeof(uint32_t)]
 
 /*
  * Helpers
@@ -75,19 +77,29 @@ static int msi_msix_enable(XenPCIPassthroughState *s,
                            bool enable)
 {
     uint16_t val = 0;
+    int rc;
 
     if (!address) {
         return -1;
     }
 
-    xen_host_pci_get_word(&s->real_device, address, &val);
+    rc = xen_host_pci_get_word(&s->real_device, address, &val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+        return rc;
+    }
     if (enable) {
         val |= flag;
     } else {
         val &= ~flag;
     }
-    xen_host_pci_set_word(&s->real_device, address, val);
-    return 0;
+    rc = xen_host_pci_set_word(&s->real_device, address, val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+    }
+    return rc;
 }
 
 static int msi_msix_setup(XenPCIPassthroughState *s,
@@ -103,9 +115,7 @@ static int msi_msix_setup(XenPCIPassthroughState *s,
 
     assert((!is_msix && msix_entry == 0) || is_msix);
 
-    if (gvec == 0) {
-        /* if gvec is 0, the guest is asking for a particular pirq that
-         * is passed as dest_id */
+    if (xen_is_pirq_msi(data)) {
         *ppirq = msi_ext_dest_id(addr >> 32) | msi_dest_id(addr);
         if (!*ppirq) {
             /* this probably identifies an misconfiguration of the guest,
@@ -220,7 +230,7 @@ static int msi_msix_disable(XenPCIPassthroughState *s,
  * MSI virtualization functions
  */
 
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
+static int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
 {
     XEN_PT_LOG(&s->dev, "%s MSI.\n", enable ? "enabling" : "disabling");
 
@@ -276,7 +286,7 @@ void xen_pt_msi_disable(XenPCIPassthroughState *s)
         return;
     }
 
-    xen_pt_msi_set_enable(s, false);
+    (void)xen_pt_msi_set_enable(s, false);
 
     msi_msix_disable(s, msi_addr64(msi), msi->data, msi->pirq, false,
                      msi->initialized);
@@ -304,7 +314,8 @@ static int msix_set_enable(XenPCIPassthroughState *s, bool enabled)
                            enabled);
 }
 
-static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr)
+static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr,
+                                  uint32_t vec_ctrl)
 {
     XenPTMSIXEntry *entry = NULL;
     int pirq;
@@ -322,6 +333,19 @@ static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr)
 
     pirq = entry->pirq;
 
+    /*
+     * Update the entry addr and data to the latest values only when the
+     * entry is masked or they are all masked, as required by the spec.
+     * Addr and data changes while the MSI-X entry is unmasked get deferred
+     * until the next masked -> unmasked transition.
+     */
+    if (pirq == XEN_PT_UNASSIGNED_PIRQ || s->msix->maskall ||
+        (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
+        entry->addr = entry->latch(LOWER_ADDR) |
+                      ((uint64_t)entry->latch(UPPER_ADDR) << 32);
+        entry->data = entry->latch(DATA);
+    }
+
     rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr,
                         entry->pirq == XEN_PT_UNASSIGNED_PIRQ);
     if (rc) {
@@ -347,7 +371,7 @@ int xen_pt_msix_update(XenPCIPassthroughState *s)
     int i;
 
     for (i = 0; i < msix->total_entries; i++) {
-        xen_pt_msix_update_one(s, i);
+        xen_pt_msix_update_one(s, i, msix->msix_entry[i].latch(VECTOR_CTRL));
     }
 
     return 0;
@@ -396,36 +420,14 @@ int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index)
 
 static uint32_t get_entry_value(XenPTMSIXEntry *e, int offset)
 {
-    switch (offset) {
-    case PCI_MSIX_ENTRY_LOWER_ADDR:
-        return e->addr & UINT32_MAX;
-    case PCI_MSIX_ENTRY_UPPER_ADDR:
-        return e->addr >> 32;
-    case PCI_MSIX_ENTRY_DATA:
-        return e->data;
-    case PCI_MSIX_ENTRY_VECTOR_CTRL:
-        return e->vector_ctrl;
-    default:
-        return 0;
-    }
+    assert(!(offset % sizeof(*e->latch)));
+    return e->latch[offset / sizeof(*e->latch)];
 }
 
 static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val)
 {
-    switch (offset) {
-    case PCI_MSIX_ENTRY_LOWER_ADDR:
-        e->addr = (e->addr & ((uint64_t)UINT32_MAX << 32)) | val;
-        break;
-    case PCI_MSIX_ENTRY_UPPER_ADDR:
-        e->addr = (uint64_t)val << 32 | (e->addr & UINT32_MAX);
-        break;
-    case PCI_MSIX_ENTRY_DATA:
-        e->data = val;
-        break;
-    case PCI_MSIX_ENTRY_VECTOR_CTRL:
-        e->vector_ctrl = val;
-        break;
-    }
+    assert(!(offset % sizeof(*e->latch)));
+    e->latch[offset / sizeof(*e->latch)] = val;
 }
 
 static void pci_msix_write(void *opaque, hwaddr addr,
@@ -444,39 +446,26 @@ static void pci_msix_write(void *opaque, hwaddr addr,
     offset = addr % PCI_MSIX_ENTRY_SIZE;
 
     if (offset != PCI_MSIX_ENTRY_VECTOR_CTRL) {
-        const volatile uint32_t *vec_ctrl;
-
         if (get_entry_value(entry, offset) == val
             && entry->pirq != XEN_PT_UNASSIGNED_PIRQ) {
             return;
         }
 
+        entry->updated = true;
+    } else if (msix->enabled && entry->updated &&
+               !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
+        const volatile uint32_t *vec_ctrl;
+
         /*
          * If Xen intercepts the mask bit access, entry->vec_ctrl may not be
          * up-to-date. Read from hardware directly.
          */
         vec_ctrl = s->msix->phys_iomem_base + entry_nr * PCI_MSIX_ENTRY_SIZE
             + PCI_MSIX_ENTRY_VECTOR_CTRL;
-
-        if (msix->enabled && !(*vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
-            if (!entry->warned) {
-                entry->warned = true;
-                XEN_PT_ERR(&s->dev, "Can't update msix entry %d since MSI-X is"
-                           " already enabled.\n", entry_nr);
-            }
-            return;
-        }
-
-        entry->updated = true;
+        xen_pt_msix_update_one(s, entry_nr, *vec_ctrl);
     }
 
     set_entry_value(entry, offset, val);
-
-    if (offset == PCI_MSIX_ENTRY_VECTOR_CTRL) {
-        if (msix->enabled && !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
-            xen_pt_msix_update_one(s, entry_nr);
-        }
-    }
 }
 
 static uint64_t pci_msix_read(void *opaque, hwaddr addr,
@@ -502,6 +491,12 @@ static uint64_t pci_msix_read(void *opaque, hwaddr addr,
     }
 }
 
+static bool pci_msix_accepts(void *opaque, hwaddr addr,
+                             unsigned size, bool is_write)
+{
+    return !(addr & (size - 1));
+}
+
 static const MemoryRegionOps pci_msix_ops = {
     .read = pci_msix_read,
     .write = pci_msix_write,
@@ -510,7 +505,13 @@ static const MemoryRegionOps pci_msix_ops = {
         .min_access_size = 4,
         .max_access_size = 4,
         .unaligned = false,
+        .accepts = pci_msix_accepts
     },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false
+    }
 };
 
 int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base)
@@ -600,7 +601,7 @@ error_out:
     return rc;
 }
 
-void xen_pt_msix_delete(XenPCIPassthroughState *s)
+void xen_pt_msix_unmap(XenPCIPassthroughState *s)
 {
     XenPTMSIX *msix = s->msix;
 
@@ -617,6 +618,17 @@ void xen_pt_msix_delete(XenPCIPassthroughState *s)
     }
 
     memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio);
+}
+
+void xen_pt_msix_delete(XenPCIPassthroughState *s)
+{
+    XenPTMSIX *msix = s->msix;
+
+    if (!msix) {
+        return;
+    }
+
+    object_unparent(OBJECT(&msix->mmio));
 
     g_free(s->msix);
     s->msix = NULL;
diff --git a/qemu/hw/xenpv/Makefile.objs b/qemu/hw/xenpv/Makefile.objs
index 49f6e9e3c..bbf5873fd 100644
--- a/qemu/hw/xenpv/Makefile.objs
+++ b/qemu/hw/xenpv/Makefile.objs
@@ -1,2 +1,4 @@
 # Xen PV machine support
-obj-$(CONFIG_XEN) += xen_domainbuild.o xen_machine_pv.o
+obj-$(CONFIG_XEN) += xen_machine_pv.o
+# Xen PV machine builder support
+obj-$(CONFIG_XEN_PV_DOMAIN_BUILD) += xen_domainbuild.o
diff --git a/qemu/hw/xenpv/xen_domainbuild.c b/qemu/hw/xenpv/xen_domainbuild.c
index c0ab7537d..5a9f5ac80 100644
--- a/qemu/hw/xenpv/xen_domainbuild.c
+++ b/qemu/hw/xenpv/xen_domainbuild.c
@@ -1,4 +1,4 @@
-#include <signal.h>
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_domainbuild.h"
 #include "qemu/timer.h"
@@ -174,12 +174,15 @@ static int xen_domain_watcher(void)
     for (i = 3; i < n; i++) {
         if (i == fd[0])
             continue;
-        if (i == xc_fd(xen_xc)) {
-            continue;
-        }
         close(i);
     }
 
+    /*
+     * Reopen xc interface, since the original is unsafe after fork
+     * and was closed above.
+     */
+    xen_xc = xc_interface_open(0, 0, 0);
+
     /* ignore term signals */
     signal(SIGINT,  SIG_IGN);
     signal(SIGTERM, SIG_IGN);
@@ -234,7 +237,7 @@ int xen_domain_build_pv(const char *kernel, const char *ramdisk,
     int rc;
 
     memcpy(uuid, qemu_uuid, sizeof(uuid));
-    rc = xc_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
+    rc = xen_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
     if (rc < 0) {
         fprintf(stderr, "xen: xc_domain_create() failed\n");
         goto err;
diff --git a/qemu/hw/xenpv/xen_machine_pv.c b/qemu/hw/xenpv/xen_machine_pv.c
index 2e545d241..fc1353599 100644
--- a/qemu/hw/xenpv/xen_machine_pv.c
+++ b/qemu/hw/xenpv/xen_machine_pv.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/xen/xen_backend.h"
@@ -30,9 +31,6 @@
 
 static void xen_init_pv(MachineState *machine)
 {
-    const char *kernel_filename = machine->kernel_filename;
-    const char *kernel_cmdline = machine->kernel_cmdline;
-    const char *initrd_filename = machine->initrd_filename;
     DriveInfo *dinfo;
     int i;
 
@@ -46,17 +44,27 @@ static void xen_init_pv(MachineState *machine)
     case XEN_ATTACH:
         /* nothing to do, xend handles everything */
         break;
-    case XEN_CREATE:
+#ifdef CONFIG_XEN_PV_DOMAIN_BUILD
+    case XEN_CREATE: {
+        const char *kernel_filename = machine->kernel_filename;
+        const char *kernel_cmdline = machine->kernel_cmdline;
+        const char *initrd_filename = machine->initrd_filename;
         if (xen_domain_build_pv(kernel_filename, initrd_filename,
                                 kernel_cmdline) < 0) {
             fprintf(stderr, "xen pv domain creation failed\n");
             exit(1);
         }
         break;
+    }
+#endif
     case XEN_EMULATE:
         fprintf(stderr, "xen emulation not implemented (yet)\n");
         exit(1);
         break;
+    default:
+        fprintf(stderr, "unhandled xen_mode %d\n", xen_mode);
+        exit(1);
+        break;
     }
 
     xen_be_register("console", &xen_console_ops);
@@ -93,17 +101,12 @@ static void xen_init_pv(MachineState *machine)
     xen_init_display(xen_domid);
 }
 
-static QEMUMachine xenpv_machine = {
-    .name = "xenpv",
-    .desc = "Xen Para-virtualized PC",
-    .init = xen_init_pv,
-    .max_cpus = 1,
-    .default_machine_opts = "accel=xen",
-};
-
-static void xenpv_machine_init(void)
+static void xenpv_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xenpv_machine);
+    mc->desc = "Xen Para-virtualized PC";
+    mc->init = xen_init_pv;
+    mc->max_cpus = 1;
+    mc->default_machine_opts = "accel=xen";
 }
 
-machine_init(xenpv_machine_init);
+DEFINE_MACHINE("xenpv", xenpv_machine_init)
diff --git a/qemu/hw/xtensa/pic_cpu.c b/qemu/hw/xtensa/pic_cpu.c
index 18825d19f..c835bd009 100644
--- a/qemu/hw/xtensa/pic_cpu.c
+++ b/qemu/hw/xtensa/pic_cpu.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/log.h"
 #include "qemu/timer.h"
diff --git a/qemu/hw/xtensa/sim.c b/qemu/hw/xtensa/sim.c
index 328d20975..5e9400426 100644
--- a/qemu/hw/xtensa/sim.c
+++ b/qemu/hw/xtensa/sim.c
@@ -25,6 +25,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -79,12 +83,12 @@ static void xtensa_sim_init(MachineState *machine)
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
     rom = g_malloc(sizeof(*rom));
-    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0xfe000000, rom);
 
@@ -93,10 +97,10 @@ static void xtensa_sim_init(MachineState *machine)
         uint64_t elf_lowaddr;
 #ifdef TARGET_WORDS_BIGENDIAN
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 1, EM_XTENSA, 0, 0);
 #else
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 0, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 0, EM_XTENSA, 0, 0);
 #endif
         if (success > 0) {
             env->pc = elf_entry;
@@ -104,17 +108,12 @@ static void xtensa_sim_init(MachineState *machine)
     }
 }
 
-static QEMUMachine xtensa_sim_machine = {
-    .name = "sim",
-    .desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .is_default = true,
-    .init = xtensa_sim_init,
-    .max_cpus = 4,
-};
-
-static void xtensa_sim_machine_init(void)
+static void xtensa_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xtensa_sim_machine);
+    mc->desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->is_default = true;
+    mc->init = xtensa_sim_init;
+    mc->max_cpus = 4;
 }
 
-machine_init(xtensa_sim_machine_init);
+DEFINE_MACHINE("sim", xtensa_sim_machine_init)
diff --git a/qemu/hw/xtensa/xtfpga.c b/qemu/hw/xtensa/xtfpga.c
index ab4d0e412..2d117369a 100644
--- a/qemu/hw/xtensa/xtfpga.c
+++ b/qemu/hw/xtensa/xtfpga.c
@@ -25,6 +25,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -143,11 +147,34 @@ static void lx60_net_init(MemoryRegion *address_space,
             sysbus_mmio_get_region(s, 1));
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384, &error_abort);
+    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384,
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space, buffers, ram);
 }
 
+static pflash_t *xtfpga_flash_init(MemoryRegion *address_space,
+                                   const LxBoardDesc *board,
+                                   DriveInfo *dinfo, int be)
+{
+    SysBusDevice *s;
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+
+    qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                        &error_abort);
+    qdev_prop_set_uint32(dev, "num-blocks",
+                         board->flash_size / board->flash_sector_size);
+    qdev_prop_set_uint64(dev, "sector-length", board->flash_sector_size);
+    qdev_prop_set_uint8(dev, "width", 4);
+    qdev_prop_set_bit(dev, "big-endian", be);
+    qdev_prop_set_string(dev, "name", "lx60.io.flash");
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    memory_region_add_subregion(address_space, board->flash_base,
+                                sysbus_mmio_get_region(s, 0));
+    return OBJECT_CHECK(pflash_t, (dev), "cfi.pflash01");
+}
+
 static uint64_t translate_phys_addr(void *opaque, uint64_t addr)
 {
     XtensaCPU *cpu = opaque;
@@ -223,7 +250,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
     ram = g_malloc(sizeof(*ram));
     memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -246,16 +273,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     if (dinfo) {
-        flash = pflash_cfi01_register(board->flash_base,
-                NULL, "lx60.io.flash", board->flash_size,
-                blk_by_legacy_dinfo(dinfo),
-                board->flash_sector_size,
-                board->flash_size / board->flash_sector_size,
-                4, 0x0000, 0x0000, 0x0000, 0x0000, be);
-        if (flash == NULL) {
-            error_report("unable to mount pflash");
-            exit(EXIT_FAILURE);
-        }
+        flash = xtfpga_flash_init(system_io, board, dinfo, be);
     }
 
     /* Use presence of kernel file name as 'boot from SRAM' switch. */
@@ -275,7 +293,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 
         rom = g_malloc(sizeof(*rom));
         memory_region_init_ram(rom, NULL, "lx60.sram", board->sram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(rom);
         memory_region_add_subregion(system_memory, 0xfe000000, rom);
 
@@ -340,7 +358,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
         uint64_t elf_entry;
         uint64_t elf_lowaddr;
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, be, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, be, EM_XTENSA, 0, 0);
         if (success > 0) {
             entry_point = elf_entry;
         } else {
@@ -385,7 +403,7 @@ static void lx_init(const LxBoardDesc *board, MachineState *machine)
 static void xtensa_lx60_init(MachineState *machine)
 {
     static const LxBoardDesc lx60_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x00400000,
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
@@ -396,7 +414,7 @@ static void xtensa_lx60_init(MachineState *machine)
 static void xtensa_lx200_init(MachineState *machine)
 {
     static const LxBoardDesc lx200_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
@@ -407,7 +425,7 @@ static void xtensa_lx200_init(MachineState *machine)
 static void xtensa_ml605_init(MachineState *machine)
 {
     static const LxBoardDesc ml605_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
@@ -418,7 +436,7 @@ static void xtensa_ml605_init(MachineState *machine)
 static void xtensa_kc705_init(MachineState *machine)
 {
     static const LxBoardDesc kc705_board = {
-        .flash_base = 0xf0000000,
+        .flash_base = 0x00000000,
         .flash_size = 0x08000000,
         .flash_boot_base = 0x06000000,
         .flash_sector_size = 0x20000,
@@ -427,40 +445,72 @@ static void xtensa_kc705_init(MachineState *machine)
     lx_init(&kc705_board, machine);
 }
 
-static QEMUMachine xtensa_lx60_machine = {
-    .name = "lx60",
-    .desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx60_init,
-    .max_cpus = 4,
+static void xtensa_lx60_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx60_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx60_type = {
+    .name = MACHINE_TYPE_NAME("lx60"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx60_class_init,
 };
 
-static QEMUMachine xtensa_lx200_machine = {
-    .name = "lx200",
-    .desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx200_init,
-    .max_cpus = 4,
+static void xtensa_lx200_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx200_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx200_type = {
+    .name = MACHINE_TYPE_NAME("lx200"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx200_class_init,
 };
 
-static QEMUMachine xtensa_ml605_machine = {
-    .name = "ml605",
-    .desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_ml605_init,
-    .max_cpus = 4,
+static void xtensa_ml605_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_ml605_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_ml605_type = {
+    .name = MACHINE_TYPE_NAME("ml605"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_ml605_class_init,
 };
 
-static QEMUMachine xtensa_kc705_machine = {
-    .name = "kc705",
-    .desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_kc705_init,
-    .max_cpus = 4,
+static void xtensa_kc705_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_kc705_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_kc705_type = {
+    .name = MACHINE_TYPE_NAME("kc705"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_kc705_class_init,
 };
 
 static void xtensa_lx_machines_init(void)
 {
-    qemu_register_machine(&xtensa_lx60_machine);
-    qemu_register_machine(&xtensa_lx200_machine);
-    qemu_register_machine(&xtensa_ml605_machine);
-    qemu_register_machine(&xtensa_kc705_machine);
+    type_register_static(&xtensa_lx60_type);
+    type_register_static(&xtensa_lx200_type);
+    type_register_static(&xtensa_ml605_type);
+    type_register_static(&xtensa_kc705_type);
 }
 
-machine_init(xtensa_lx_machines_init);
+type_init(xtensa_lx_machines_init)
diff --git a/qemu/include/block/accounting.h b/qemu/include/block/accounting.h
index 4c406cff7..20891639d 100644
--- a/qemu/include/block/accounting.h
+++ b/qemu/include/block/accounting.h
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -24,9 +25,9 @@
 #ifndef BLOCK_ACCOUNTING_H
 #define BLOCK_ACCOUNTING_H
 
-#include <stdint.h>
+#include "qemu/timed-average.h"
 
-#include "qemu/typedefs.h"
+typedef struct BlockAcctTimedStats BlockAcctTimedStats;
 
 enum BlockAcctType {
     BLOCK_ACCT_READ,
@@ -35,12 +36,23 @@ enum BlockAcctType {
     BLOCK_MAX_IOTYPE,
 };
 
+struct BlockAcctTimedStats {
+    TimedAverage latency[BLOCK_MAX_IOTYPE];
+    unsigned interval_length; /* in seconds */
+    QSLIST_ENTRY(BlockAcctTimedStats) entries;
+};
+
 typedef struct BlockAcctStats {
     uint64_t nr_bytes[BLOCK_MAX_IOTYPE];
     uint64_t nr_ops[BLOCK_MAX_IOTYPE];
+    uint64_t invalid_ops[BLOCK_MAX_IOTYPE];
+    uint64_t failed_ops[BLOCK_MAX_IOTYPE];
     uint64_t total_time_ns[BLOCK_MAX_IOTYPE];
     uint64_t merged[BLOCK_MAX_IOTYPE];
-    uint64_t wr_highest_sector;
+    int64_t last_access_time_ns;
+    QSLIST_HEAD(, BlockAcctTimedStats) intervals;
+    bool account_invalid;
+    bool account_failed;
 } BlockAcctStats;
 
 typedef struct BlockAcctCookie {
@@ -49,12 +61,21 @@ typedef struct BlockAcctCookie {
     enum BlockAcctType type;
 } BlockAcctCookie;
 
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed);
+void block_acct_cleanup(BlockAcctStats *stats);
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length);
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s);
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type);
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie);
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors);
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie);
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type);
 void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
                            int num_requests);
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats);
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type);
 
 #endif
diff --git a/qemu/include/block/aio.h b/qemu/include/block/aio.h
index 400b1b002..88a64eeb3 100644
--- a/qemu/include/block/aio.h
+++ b/qemu/include/block/aio.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_AIO_H
 #define QEMU_AIO_H
 
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/event_notifier.h"
@@ -122,6 +121,13 @@ struct AioContext {
 
     /* TimerLists for calling timers - one per clock type */
     QEMUTimerListGroup tlg;
+
+    int external_disable_cnt;
+
+    /* epoll(7) state used when built with CONFIG_EPOLL */
+    int epollfd;
+    bool epoll_enabled;
+    bool epoll_available;
 };
 
 /**
@@ -207,6 +213,11 @@ void aio_notify(AioContext *ctx);
 void aio_notify_accept(AioContext *ctx);
 
 /**
+ * aio_bh_call: Executes callback function of the specified BH.
+ */
+void aio_bh_call(QEMUBH *bh);
+
+/**
  * aio_bh_poll: Poll bottom halves for an AioContext.
  *
  * These are internal functions used by the QEMU main loop.
@@ -299,6 +310,7 @@ bool aio_poll(AioContext *ctx, bool blocking);
  */
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque);
@@ -312,6 +324,7 @@ void aio_set_fd_handler(AioContext *ctx,
  */
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read);
 
 /* Return a GSource that lets the main loop poll the file descriptors attached
@@ -373,4 +386,59 @@ static inline void aio_timer_init(AioContext *ctx,
  */
 int64_t aio_compute_timeout(AioContext *ctx);
 
+/**
+ * aio_disable_external:
+ * @ctx: the aio context
+ *
+ * Disable the further processing of external clients.
+ */
+static inline void aio_disable_external(AioContext *ctx)
+{
+    atomic_inc(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_enable_external:
+ * @ctx: the aio context
+ *
+ * Enable the processing of external clients.
+ */
+static inline void aio_enable_external(AioContext *ctx)
+{
+    assert(ctx->external_disable_cnt > 0);
+    atomic_dec(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_external_disabled:
+ * @ctx: the aio context
+ *
+ * Return true if the external clients are disabled.
+ */
+static inline bool aio_external_disabled(AioContext *ctx)
+{
+    return atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_node_check:
+ * @ctx: the aio context
+ * @is_external: Whether or not the checked node is an external event source.
+ *
+ * Check if the node's is_external flag is okay to be polled by the ctx at this
+ * moment. True means green light.
+ */
+static inline bool aio_node_check(AioContext *ctx, bool is_external)
+{
+    return !is_external || !atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_context_setup:
+ * @ctx: the aio context
+ *
+ * Initialize the aio context.
+ */
+void aio_context_setup(AioContext *ctx, Error **errp);
+
 #endif
diff --git a/qemu/include/block/block.h b/qemu/include/block/block.h
index 37916f720..3a731377d 100644
--- a/qemu/include/block/block.h
+++ b/qemu/include/block/block.h
@@ -2,18 +2,21 @@
 #define BLOCK_H
 
 #include "block/aio.h"
-#include "qemu-common.h"
+#include "qemu/iov.h"
 #include "qemu/option.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/accounting.h"
+#include "block/dirty-bitmap.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
+#include "qemu/hbitmap.h"
 
 /* block.c */
 typedef struct BlockDriver BlockDriver;
 typedef struct BlockJob BlockJob;
 typedef struct BdrvChild BdrvChild;
 typedef struct BdrvChildRole BdrvChildRole;
+typedef struct BlockJobTxn BlockJobTxn;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
@@ -23,7 +26,7 @@ typedef struct BlockDriverInfo {
     bool is_dirty;
     /*
      * True if unallocated blocks read back as zeroes. This is equivalent
-     * to the the LBPRZ flag in the SCSI logical block provisioning page.
+     * to the LBPRZ flag in the SCSI logical block provisioning page.
      */
     bool unallocated_blocks_are_zero;
     /*
@@ -51,15 +54,17 @@ typedef struct BlockFragInfo {
 } BlockFragInfo;
 
 typedef enum {
-    BDRV_REQ_COPY_ON_READ = 0x1,
-    BDRV_REQ_ZERO_WRITE   = 0x2,
+    BDRV_REQ_COPY_ON_READ       = 0x1,
+    BDRV_REQ_ZERO_WRITE         = 0x2,
     /* The BDRV_REQ_MAY_UNMAP flag is used to indicate that the block driver
      * is allowed to optimize a write zeroes request by unmapping (discarding)
      * blocks if it is guaranteed that the result will read back as
      * zeroes. The flag is only passed to the driver if the block device is
      * opened with BDRV_O_UNMAP.
      */
-    BDRV_REQ_MAY_UNMAP    = 0x4,
+    BDRV_REQ_MAY_UNMAP          = 0x4,
+    BDRV_REQ_NO_SERIALISING     = 0x8,
+    BDRV_REQ_FUA                = 0x10,
 } BdrvRequestFlags;
 
 typedef struct BlockSizes {
@@ -77,20 +82,20 @@ typedef struct HDGeometry {
 #define BDRV_O_SNAPSHOT    0x0008 /* open the file read only and save writes in a snapshot */
 #define BDRV_O_TEMPORARY   0x0010 /* delete the file after use */
 #define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
-#define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
 #define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
 #define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
 #define BDRV_O_NO_FLUSH    0x0200 /* disable flushing on this disk */
 #define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
-#define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
+#define BDRV_O_INACTIVE    0x0800  /* consistency hint for migration handoff */
 #define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
 #define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
 #define BDRV_O_UNMAP       0x4000  /* execute guest UNMAP/TRIM operations */
 #define BDRV_O_PROTOCOL    0x8000  /* if no block driver is explicitly given:
                                       select an appropriate protocol driver,
                                       ignoring the format layer */
+#define BDRV_O_NO_IO       0x10000 /* don't initialize for I/O */
 
-#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
+#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_NO_FLUSH)
 
 
 /* Option names of options parsed by the block layer */
@@ -109,9 +114,10 @@ typedef struct HDGeometry {
 
 /*
  * Allocation status flags
- * BDRV_BLOCK_DATA: data is read from bs->file or another file
+ * BDRV_BLOCK_DATA: data is read from a file returned by bdrv_get_block_status.
  * BDRV_BLOCK_ZERO: sectors read as zero
- * BDRV_BLOCK_OFFSET_VALID: sector stored in bs->file as raw data
+ * BDRV_BLOCK_OFFSET_VALID: sector stored as raw data in a file returned by
+ *                          bdrv_get_block_status.
  * BDRV_BLOCK_ALLOCATED: the content of the block is determined by this
  *                       layer (as opposed to the backing file)
  * BDRV_BLOCK_RAW: used internally to indicate that the request
@@ -147,6 +153,8 @@ typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
 typedef struct BDRVReopenState {
     BlockDriverState *bs;
     int flags;
+    QDict *options;
+    QDict *explicit_options;
     void *opaque;
 } BDRVReopenState;
 
@@ -165,18 +173,14 @@ typedef enum BlockOpType {
     BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
     BLOCK_OP_TYPE_INTERNAL_SNAPSHOT,
     BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
-    BLOCK_OP_TYPE_MIRROR,
+    BLOCK_OP_TYPE_MIRROR_SOURCE,
+    BLOCK_OP_TYPE_MIRROR_TARGET,
     BLOCK_OP_TYPE_RESIZE,
     BLOCK_OP_TYPE_STREAM,
     BLOCK_OP_TYPE_REPLACE,
     BLOCK_OP_TYPE_MAX,
 } BlockOpType;
 
-void bdrv_iostatus_enable(BlockDriverState *bs);
-void bdrv_iostatus_reset(BlockDriverState *bs);
-void bdrv_iostatus_disable(BlockDriverState *bs);
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
 void bdrv_info_print(Monitor *mon, const QObject *data);
 void bdrv_info(Monitor *mon, QObject **ret_data);
 void bdrv_stats_print(Monitor *mon, const QObject *data);
@@ -189,51 +193,43 @@ void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group);
 
 void bdrv_init(void);
 void bdrv_init_with_whitelist(void);
+bool bdrv_uses_whitelist(void);
 BlockDriver *bdrv_find_protocol(const char *filename,
                                 bool allow_protocol_prefix,
                                 Error **errp);
 BlockDriver *bdrv_find_format(const char *format_name);
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool readonly);
 int bdrv_create(BlockDriver *drv, const char* filename,
                 QemuOpts *opts, Error **errp);
 int bdrv_create_file(const char *filename, QemuOpts *opts, Error **errp);
 BlockDriverState *bdrv_new_root(void);
 BlockDriverState *bdrv_new(void);
-void bdrv_make_anon(BlockDriverState *bs);
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
-int bdrv_parse_cache_flags(const char *mode, int *flags);
+void bdrv_replace_in_backing_chain(BlockDriverState *old,
+                                   BlockDriverState *new);
+
+int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough);
 int bdrv_parse_discard_flags(const char *mode, int *flags);
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key,
-                    BlockDriverState* parent, const BdrvChildRole *child_role,
-                    bool allow_none, Error **errp);
 BdrvChild *bdrv_open_child(const char *filename,
                            QDict *options, const char *bdref_key,
                            BlockDriverState* parent,
                            const BdrvChildRole *child_role,
                            bool allow_none, Error **errp);
 void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd);
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp);
-int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp);
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options,
+                           const char *bdref_key, Error **errp);
 int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp);
+              const char *reference, QDict *options, int flags, Error **errp);
 BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags);
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags);
 int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
 int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
 int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
                         BlockReopenQueue *queue, Error **errp);
 void bdrv_reopen_commit(BDRVReopenState *reopen_state);
 void bdrv_reopen_abort(BDRVReopenState *reopen_state);
-void bdrv_close(BlockDriverState *bs);
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
 int bdrv_read(BlockDriverState *bs, int64_t sector_num,
               uint8_t *buf, int nb_sectors);
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors);
 int bdrv_write(BlockDriverState *bs, int64_t sector_num,
                const uint8_t *buf, int nb_sectors);
 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
@@ -253,6 +249,8 @@ int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
     int nb_sectors, QEMUIOVector *qiov);
 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
     int nb_sectors, QEMUIOVector *qiov);
 /*
@@ -274,7 +272,6 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs);
 void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp);
 int bdrv_commit(BlockDriverState *bs);
-int bdrv_commit_all(void);
 int bdrv_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt);
 void bdrv_register(BlockDriver *bdrv);
@@ -307,9 +304,9 @@ int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix);
  * block driver; total_work_size may change during the course of the amendment
  * operation */
 typedef void BlockDriverAmendStatusCB(BlockDriverState *bs, int64_t offset,
-                                      int64_t total_work_size);
+                                      int64_t total_work_size, void *opaque);
 int bdrv_amend_options(BlockDriverState *bs_new, QemuOpts *opts,
-                       BlockDriverAmendStatusCB *status_cb);
+                       BlockDriverAmendStatusCB *status_cb, void *cb_opaque);
 
 /* external snapshots */
 bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -317,11 +314,10 @@ bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
 bool bdrv_is_first_non_filter(BlockDriverState *candidate);
 
 /* check if a named node can be replaced when doing drive-mirror */
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp);
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp);
 
 /* async block I/O */
-typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
-                                     int sector_num);
 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
                            QEMUIOVector *iov, int nb_sectors,
                            BlockCompletionFunc *cb, void *opaque);
@@ -338,10 +334,18 @@ void bdrv_aio_cancel_async(BlockAIOCB *acb);
 
 typedef struct BlockRequest {
     /* Fields to be filled by multiwrite caller */
-    int64_t sector;
-    int nb_sectors;
-    int flags;
-    QEMUIOVector *qiov;
+    union {
+        struct {
+            int64_t sector;
+            int nb_sectors;
+            int flags;
+            QEMUIOVector *qiov;
+        };
+        struct {
+            int req;
+            void *buf;
+        };
+    };
     BlockCompletionFunc *cb;
     void *opaque;
 
@@ -361,13 +365,14 @@ BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
 /* Invalidate any cached metadata used by image formats */
 void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp);
 void bdrv_invalidate_cache_all(Error **errp);
+int bdrv_inactivate_all(void);
 
 /* Ensure contents are flushed to disk.  */
 int bdrv_flush(BlockDriverState *bs);
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
-int bdrv_flush_all(void);
 void bdrv_close_all(void);
 void bdrv_drain(BlockDriverState *bs);
+void coroutine_fn bdrv_co_drain(BlockDriverState *bs);
 void bdrv_drain_all(void);
 
 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
@@ -377,27 +382,21 @@ int bdrv_has_zero_init(BlockDriverState *bs);
 bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs);
 bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs);
 int64_t bdrv_get_block_status(BlockDriverState *bs, int64_t sector_num,
-                              int nb_sectors, int *pnum);
+                              int nb_sectors, int *pnum,
+                              BlockDriverState **file);
 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
                                     BlockDriverState *base,
                                     int64_t sector_num,
-                                    int nb_sectors, int *pnum);
+                                    int nb_sectors, int *pnum,
+                                    BlockDriverState **file);
 int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
                       int *pnum);
 int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
                             int64_t sector_num, int nb_sectors, int *pnum);
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error);
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error);
 int bdrv_is_read_only(BlockDriverState *bs);
 int bdrv_is_sg(BlockDriverState *bs);
-int bdrv_enable_write_cache(BlockDriverState *bs);
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce);
-int bdrv_is_inserted(BlockDriverState *bs);
+bool bdrv_is_inserted(BlockDriverState *bs);
 int bdrv_media_changed(BlockDriverState *bs);
 void bdrv_lock_medium(BlockDriverState *bs, bool locked);
 void bdrv_eject(BlockDriverState *bs, bool eject_flag);
@@ -410,6 +409,7 @@ BlockDriverState *bdrv_lookup_bs(const char *device,
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base);
 BlockDriverState *bdrv_next_node(BlockDriverState *bs);
 BlockDriverState *bdrv_next(BlockDriverState *bs);
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs);
 int bdrv_is_encrypted(BlockDriverState *bs);
 int bdrv_key_required(BlockDriverState *bs);
 int bdrv_set_key(BlockDriverState *bs, const char *key);
@@ -464,50 +464,12 @@ void bdrv_img_create(const char *filename, const char *fmt,
 size_t bdrv_min_mem_align(BlockDriverState *bs);
 /* Returns optimal alignment in bytes for bounce buffer */
 size_t bdrv_opt_mem_align(BlockDriverState *bs);
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
 void *qemu_blockalign0(BlockDriverState *bs, size_t size);
 void *qemu_try_blockalign(BlockDriverState *bs, size_t size);
 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size);
 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
 
-struct HBitmapIter;
-typedef struct BdrvDirtyBitmap BdrvDirtyBitmap;
-BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
-                                          uint32_t granularity,
-                                          const char *name,
-                                          Error **errp);
-int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
-                                       BdrvDirtyBitmap *bitmap,
-                                       Error **errp);
-BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
-                                            BdrvDirtyBitmap *bitmap,
-                                            Error **errp);
-BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
-                                           BdrvDirtyBitmap *bitmap,
-                                           Error **errp);
-BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
-                                        const char *name);
-void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap);
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
-void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
-void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
-BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs);
-uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs);
-uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap);
-bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap);
-bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap);
-DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap);
-int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, int64_t sector);
-void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-                           int64_t cur_sector, int nr_sectors);
-void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
-                             int64_t cur_sector, int nr_sectors);
-void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap);
-void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, struct HBitmapIter *hbi);
-void bdrv_set_dirty_iter(struct HBitmapIter *hbi, int64_t offset);
-int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
-
 void bdrv_enable_copy_on_read(BlockDriverState *bs);
 void bdrv_disable_copy_on_read(BlockDriverState *bs);
 
@@ -522,68 +484,14 @@ void bdrv_op_block_all(BlockDriverState *bs, Error *reason);
 void bdrv_op_unblock_all(BlockDriverState *bs, Error *reason);
 bool bdrv_op_blocker_is_empty(BlockDriverState *bs);
 
-typedef enum {
-    BLKDBG_L1_UPDATE,
-
-    BLKDBG_L1_GROW_ALLOC_TABLE,
-    BLKDBG_L1_GROW_WRITE_TABLE,
-    BLKDBG_L1_GROW_ACTIVATE_TABLE,
-
-    BLKDBG_L2_LOAD,
-    BLKDBG_L2_UPDATE,
-    BLKDBG_L2_UPDATE_COMPRESSED,
-    BLKDBG_L2_ALLOC_COW_READ,
-    BLKDBG_L2_ALLOC_WRITE,
-
-    BLKDBG_READ_AIO,
-    BLKDBG_READ_BACKING_AIO,
-    BLKDBG_READ_COMPRESSED,
-
-    BLKDBG_WRITE_AIO,
-    BLKDBG_WRITE_COMPRESSED,
-
-    BLKDBG_VMSTATE_LOAD,
-    BLKDBG_VMSTATE_SAVE,
-
-    BLKDBG_COW_READ,
-    BLKDBG_COW_WRITE,
-
-    BLKDBG_REFTABLE_LOAD,
-    BLKDBG_REFTABLE_GROW,
-    BLKDBG_REFTABLE_UPDATE,
-
-    BLKDBG_REFBLOCK_LOAD,
-    BLKDBG_REFBLOCK_UPDATE,
-    BLKDBG_REFBLOCK_UPDATE_PART,
-    BLKDBG_REFBLOCK_ALLOC,
-    BLKDBG_REFBLOCK_ALLOC_HOOKUP,
-    BLKDBG_REFBLOCK_ALLOC_WRITE,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE,
-    BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE,
+#define BLKDBG_EVENT(child, evt) \
+    do { \
+        if (child) { \
+            bdrv_debug_event(child->bs, evt); \
+        } \
+    } while (0)
 
-    BLKDBG_CLUSTER_ALLOC,
-    BLKDBG_CLUSTER_ALLOC_BYTES,
-    BLKDBG_CLUSTER_FREE,
-
-    BLKDBG_FLUSH_TO_OS,
-    BLKDBG_FLUSH_TO_DISK,
-
-    BLKDBG_PWRITEV_RMW_HEAD,
-    BLKDBG_PWRITEV_RMW_AFTER_HEAD,
-    BLKDBG_PWRITEV_RMW_TAIL,
-    BLKDBG_PWRITEV_RMW_AFTER_TAIL,
-    BLKDBG_PWRITEV,
-    BLKDBG_PWRITEV_ZERO,
-    BLKDBG_PWRITEV_DONE,
-
-    BLKDBG_EMPTY_IMAGE_PREPARE,
-
-    BLKDBG_EVENT_MAX,
-} BlkDebugEvent;
-
-#define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
+void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event);
 
 int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
                            const char *tag);
@@ -614,6 +522,23 @@ void bdrv_io_plug(BlockDriverState *bs);
 void bdrv_io_unplug(BlockDriverState *bs);
 void bdrv_flush_io_queue(BlockDriverState *bs);
 
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs);
+/**
+ * bdrv_drained_begin:
+ *
+ * Begin a quiesced section for exclusive access to the BDS, by disabling
+ * external request sources including NBD server and device model. Note that
+ * this doesn't block timers or coroutines from submitting more requests, which
+ * means block_job_pause is still necessary.
+ *
+ * This function can be recursive.
+ */
+void bdrv_drained_begin(BlockDriverState *bs);
+
+/**
+ * bdrv_drained_end:
+ *
+ * End a quiescent section started by bdrv_drained_begin().
+ */
+void bdrv_drained_end(BlockDriverState *bs);
 
 #endif
diff --git a/qemu/include/block/block_int.h b/qemu/include/block/block_int.h
index 14ad4c334..10d87595b 100644
--- a/qemu/include/block/block_int.h
+++ b/qemu/include/block/block_int.h
@@ -26,9 +26,10 @@
 
 #include "block/accounting.h"
 #include "block/block.h"
+#include "block/throttle-groups.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/timer.h"
 #include "qapi-types.h"
 #include "qemu/hbitmap.h"
@@ -59,11 +60,19 @@
 
 #define BLOCK_PROBE_BUF_SIZE        512
 
+enum BdrvTrackedRequestType {
+    BDRV_TRACKED_READ,
+    BDRV_TRACKED_WRITE,
+    BDRV_TRACKED_FLUSH,
+    BDRV_TRACKED_IOCTL,
+    BDRV_TRACKED_DISCARD,
+};
+
 typedef struct BdrvTrackedRequest {
     BlockDriverState *bs;
     int64_t offset;
     unsigned int bytes;
-    bool is_write;
+    enum BdrvTrackedRequestType type;
 
     bool serialising;
     int64_t overlap_offset;
@@ -112,6 +121,7 @@ struct BlockDriver {
                                BlockReopenQueue *queue, Error **errp);
     void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
     void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
+    void (*bdrv_join_options)(QDict *options, QDict *old_options);
 
     int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp);
@@ -122,12 +132,11 @@ struct BlockDriver {
     int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
                       const uint8_t *buf, int nb_sectors);
     void (*bdrv_close)(BlockDriverState *bs);
-    void (*bdrv_rebind)(BlockDriverState *bs);
     int (*bdrv_create)(const char *filename, QemuOpts *opts, Error **errp);
     int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
     int (*bdrv_make_empty)(BlockDriverState *bs);
 
-    void (*bdrv_refresh_filename)(BlockDriverState *bs);
+    void (*bdrv_refresh_filename)(BlockDriverState *bs, QDict *options);
 
     /* aio */
     BlockAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
@@ -146,6 +155,11 @@ struct BlockDriver {
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
     int coroutine_fn (*bdrv_co_writev)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+    int coroutine_fn (*bdrv_co_writev_flags)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int flags);
+
+    int supported_write_flags;
+
     /*
      * Efficiently zero a region of the disk image.  Typically an image format
      * would use a compact metadata representation to implement this.  This
@@ -157,12 +171,21 @@ struct BlockDriver {
     int coroutine_fn (*bdrv_co_discard)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors);
     int64_t coroutine_fn (*bdrv_co_get_block_status)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum);
+        int64_t sector_num, int nb_sectors, int *pnum,
+        BlockDriverState **file);
 
     /*
      * Invalidate any cached meta-data.
      */
     void (*bdrv_invalidate_cache)(BlockDriverState *bs, Error **errp);
+    int (*bdrv_inactivate)(BlockDriverState *bs);
+
+    /*
+     * Flushes all data for all layers by calling bdrv_co_flush for underlying
+     * layers, if needed. This function is needed for deterministic
+     * synchronization of the flush finishing callback.
+     */
+    int coroutine_fn (*bdrv_co_flush)(BlockDriverState *bs);
 
     /*
      * Flushes all data that was already written to the OS all the way down to
@@ -213,13 +236,12 @@ struct BlockDriver {
         const char *backing_file, const char *backing_fmt);
 
     /* removable device specific */
-    int (*bdrv_is_inserted)(BlockDriverState *bs);
+    bool (*bdrv_is_inserted)(BlockDriverState *bs);
     int (*bdrv_media_changed)(BlockDriverState *bs);
     void (*bdrv_eject)(BlockDriverState *bs, bool eject_flag);
     void (*bdrv_lock_medium)(BlockDriverState *bs, bool locked);
 
     /* to control generic scsi devices */
-    int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
     BlockAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque);
@@ -235,9 +257,10 @@ struct BlockDriver {
         BdrvCheckMode fix);
 
     int (*bdrv_amend_options)(BlockDriverState *bs, QemuOpts *opts,
-                              BlockDriverAmendStatusCB *status_cb);
+                              BlockDriverAmendStatusCB *status_cb,
+                              void *cb_opaque);
 
-    void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
+    void (*bdrv_debug_event)(BlockDriverState *bs, BlkdebugEvent event);
 
     /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
     int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
@@ -288,6 +311,12 @@ struct BlockDriver {
      */
     int (*bdrv_probe_geometry)(BlockDriverState *bs, HDGeometry *geo);
 
+    /**
+     * Drain and stop any internal sources of requests in the driver, and
+     * remain so until next I/O callback (e.g. bdrv_co_writev) is called.
+     */
+    void (*bdrv_drain)(BlockDriverState *bs);
+
     QLIST_ENTRY(BlockDriver) list;
 };
 
@@ -315,6 +344,9 @@ typedef struct BlockLimits {
 
     /* memory alignment for bounce buffer */
     size_t opt_mem_alignment;
+
+    /* maximum number of iovec elements */
+    int max_iov;
 } BlockLimits;
 
 typedef struct BdrvOpBlocker BdrvOpBlocker;
@@ -329,7 +361,8 @@ typedef struct BdrvAioNotifier {
 } BdrvAioNotifier;
 
 struct BdrvChildRole {
-    int (*inherit_flags)(int parent_flags);
+    void (*inherit_options)(int *child_flags, QDict *child_options,
+                            int parent_flags, QDict *parent_options);
 };
 
 extern const BdrvChildRole child_file;
@@ -337,8 +370,10 @@ extern const BdrvChildRole child_format;
 
 struct BdrvChild {
     BlockDriverState *bs;
+    char *name;
     const BdrvChildRole *role;
     QLIST_ENTRY(BdrvChild) next;
+    QLIST_ENTRY(BdrvChild) next_parent;
 };
 
 /*
@@ -378,11 +413,8 @@ struct BlockDriverState {
     QDict *full_open_options;
     char exact_filename[PATH_MAX];
 
-    BlockDriverState *backing_hd;
-    BdrvChild *backing_child;
-    BlockDriverState *file;
-
-    NotifierList close_notifiers;
+    BdrvChild *backing;
+    BdrvChild *file;
 
     /* Callback before write request is processed */
     NotifierWithReturnList before_write_notifiers;
@@ -390,7 +422,10 @@ struct BlockDriverState {
     /* number of in-flight serialising requests */
     unsigned int serialising_in_flight;
 
-    /* I/O throttling */
+    /* I/O throttling.
+     * throttle_state tells us if this BDS has I/O limits configured.
+     * io_limits_enabled tells us if they are currently being
+     * enforced, but it can be temporarily set to false */
     CoQueue      throttled_reqs[2];
     bool         io_limits_enabled;
     /* The following fields are protected by the ThrottleGroup lock.
@@ -400,8 +435,8 @@ struct BlockDriverState {
     unsigned       pending_reqs[2];
     QLIST_ENTRY(BlockDriverState) round_robin;
 
-    /* I/O stats (display with "info blockstats"). */
-    BlockAcctStats stats;
+    /* Offset after the highest byte written to */
+    uint64_t wr_highest_offset;
 
     /* I/O Limits */
     BlockLimits bl;
@@ -412,24 +447,14 @@ struct BlockDriverState {
     /* Alignment requirement for offset/length of I/O requests */
     unsigned int request_alignment;
 
-    /* the block size for which the guest device expects atomicity */
-    int guest_block_size;
-
-    /* do we need to tell the quest if we have a volatile write cache? */
-    int enable_write_cache;
-
-    /* NOTE: the following infos are only hints for real hardware
-       drivers. They are not used by the block driver */
-    BlockdevOnError on_read_error, on_write_error;
-    bool iostatus_enabled;
-    BlockDeviceIoStatus iostatus;
-
     /* the following member gives a name to every node on the bs graph. */
     char node_name[32];
     /* element of the list of named nodes building the graph */
     QTAILQ_ENTRY(BlockDriverState) node_list;
-    /* element of the list of "drives" the guest sees */
-    QTAILQ_ENTRY(BlockDriverState) device_list;
+    /* element of the list of all BlockDriverStates (all_bdrv_states) */
+    QTAILQ_ENTRY(BlockDriverState) bs_list;
+    /* element of the list of monitor-owned BDS */
+    QTAILQ_ENTRY(BlockDriverState) monitor_list;
     QLIST_HEAD(, BdrvDirtyBitmap) dirty_bitmaps;
     int refcnt;
 
@@ -446,8 +471,10 @@ struct BlockDriverState {
      * parent node of this node. */
     BlockDriverState *inherits_from;
     QLIST_HEAD(, BdrvChild) children;
+    QLIST_HEAD(, BdrvChild) parents;
 
     QDict *options;
+    QDict *explicit_options;
     BlockdevDetectZeroesOptions detect_zeroes;
 
     /* The error object in use for blocking operations on backing_hd */
@@ -456,8 +483,24 @@ struct BlockDriverState {
     /* threshold limit for writes, in bytes. "High water mark". */
     uint64_t write_threshold_offset;
     NotifierWithReturn write_threshold_notifier;
+
+    int quiesce_counter;
+};
+
+struct BlockBackendRootState {
+    int open_flags;
+    bool read_only;
+    BlockdevDetectZeroesOptions detect_zeroes;
+
+    char *throttle_group;
+    ThrottleState *throttle_state;
 };
 
+static inline BlockDriverState *backing_bs(BlockDriverState *bs)
+{
+    return bs->backing ? bs->backing->bs : NULL;
+}
+
 
 /* Essential block drivers which must always be statically linked into qemu, and
  * which therefore can be accessed without using bdrv_find_format() */
@@ -474,6 +517,13 @@ extern BlockDriver bdrv_qcow2;
  */
 void bdrv_setup_io_funcs(BlockDriver *bdrv);
 
+int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+
 int get_tmp_filename(char *filename, int size);
 BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
                             const char *filename);
@@ -496,7 +546,7 @@ void bdrv_add_before_write_notifier(BlockDriverState *bs,
  *
  * May be called from .bdrv_detach_aio_context() to detach children from the
  * current #AioContext.  This is only needed by block drivers that manage their
- * own children.  Both ->file and ->backing_hd are automatically handled and
+ * own children.  Both ->file and ->backing are automatically handled and
  * block drivers should not call this function on them explicitly.
  */
 void bdrv_detach_aio_context(BlockDriverState *bs);
@@ -506,7 +556,7 @@ void bdrv_detach_aio_context(BlockDriverState *bs);
  *
  * May be called from .bdrv_attach_aio_context() to attach children to the new
  * #AioContext.  This is only needed by block drivers that manage their own
- * children.  Both ->file and ->backing_hd are automatically handled and block
+ * children.  Both ->file and ->backing are automatically handled and block
  * drivers should not call this function on them explicitly.
  */
 void bdrv_attach_aio_context(BlockDriverState *bs,
@@ -643,6 +693,7 @@ void mirror_start(BlockDriverState *bs, BlockDriverState *target,
  * @on_target_error: The action to take upon error writing to the target.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
+ * @txn: Transaction that this job is part of (may be NULL).
  *
  * Start a backup operation on @bs.  Clusters in @bs are written to @target
  * until the job is cancelled or manually completed.
@@ -653,15 +704,29 @@ void backup_start(BlockDriverState *bs, BlockDriverState *target,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp);
+                  BlockJobTxn *txn, Error **errp);
+
+void hmp_drive_add_node(Monitor *mon, const char *optstr);
+
+BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs,
+                                  const char *child_name,
+                                  const BdrvChildRole *child_role);
+void bdrv_root_unref_child(BdrvChild *child);
 
 void blk_dev_change_media_cb(BlockBackend *blk, bool load);
 bool blk_dev_has_removable_media(BlockBackend *blk);
+bool blk_dev_has_tray(BlockBackend *blk);
 void blk_dev_eject_request(BlockBackend *blk, bool force);
 bool blk_dev_is_tray_open(BlockBackend *blk);
 bool blk_dev_is_medium_locked(BlockBackend *blk);
 void blk_dev_resize_cb(BlockBackend *blk);
 
 void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+bool bdrv_requests_pending(BlockDriverState *bs);
+
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in);
+
+void blockdev_close_all_bdrv_states(void);
 
 #endif /* BLOCK_INT_H */
diff --git a/qemu/include/block/blockjob.h b/qemu/include/block/blockjob.h
index dd9d5e6aa..8bedc4936 100644
--- a/qemu/include/block/blockjob.h
+++ b/qemu/include/block/blockjob.h
@@ -50,6 +50,26 @@ typedef struct BlockJobDriver {
      * manually.
      */
     void (*complete)(BlockJob *job, Error **errp);
+
+    /**
+     * If the callback is not NULL, it will be invoked when all the jobs
+     * belonging to the same transaction complete; or upon this job's
+     * completion if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*commit)(BlockJob *job);
+
+    /**
+     * If the callback is not NULL, it will be invoked when any job in the
+     * same transaction fails; or upon this job's failure (due to error or
+     * cancellation) if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*abort)(BlockJob *job);
 } BlockJobDriver;
 
 /**
@@ -65,6 +85,14 @@ struct BlockJob {
     BlockDriverState *bs;
 
     /**
+     * The ID of the block job. Currently the BlockBackend name of the BDS
+     * owning the job at the time when the job is started.
+     *
+     * TODO Decouple block job IDs from BlockBackend names
+     */
+    char *id;
+
+    /**
      * The coroutine that executes the job.  If not NULL, it is
      * reentered when busy is false and the job is cancelled.
      */
@@ -102,6 +130,11 @@ struct BlockJob {
      */
     bool ready;
 
+    /**
+     * Set to true when the job has deferred work to the main loop.
+     */
+    bool deferred_to_main_loop;
+
     /** Status that is published by the query-block-jobs QMP API */
     BlockDeviceIoStatus iostatus;
 
@@ -122,6 +155,21 @@ struct BlockJob {
 
     /** The opaque value that is passed to the completion function.  */
     void *opaque;
+
+    /** Reference count of the block job */
+    int refcnt;
+
+    /* True if this job has reported completion by calling block_job_completed.
+     */
+    bool completed;
+
+    /* ret code passed to block_job_completed.
+     */
+    int ret;
+
+    /** Non-NULL if this job is part of a transaction */
+    BlockJobTxn *txn;
+    QLIST_ENTRY(BlockJob) txn_list;
 };
 
 /**
@@ -166,12 +214,21 @@ void block_job_sleep_ns(BlockJob *job, QEMUClockType type, int64_t ns);
 void block_job_yield(BlockJob *job);
 
 /**
- * block_job_release:
+ * block_job_ref:
  * @bs: The block device.
  *
- * Release job resources when an error occurred or job completed.
+ * Grab a reference to the block job. Should be paired with block_job_unref.
  */
-void block_job_release(BlockDriverState *bs);
+void block_job_ref(BlockJob *job);
+
+/**
+ * block_job_unref:
+ * @bs: The block device.
+ *
+ * Release reference to the block job and release resources if it is the last
+ * reference.
+ */
+void block_job_unref(BlockJob *job);
 
 /**
  * block_job_completed:
@@ -356,4 +413,39 @@ void block_job_defer_to_main_loop(BlockJob *job,
                                   BlockJobDeferToMainLoopFn *fn,
                                   void *opaque);
 
+/**
+ * block_job_txn_new:
+ *
+ * Allocate and return a new block job transaction.  Jobs can be added to the
+ * transaction using block_job_txn_add_job().
+ *
+ * The transaction is automatically freed when the last job completes or is
+ * cancelled.
+ *
+ * All jobs in the transaction either complete successfully or fail/cancel as a
+ * group.  Jobs wait for each other before completing.  Cancelling one job
+ * cancels all jobs in the transaction.
+ */
+BlockJobTxn *block_job_txn_new(void);
+
+/**
+ * block_job_txn_unref:
+ *
+ * Release a reference that was previously acquired with block_job_txn_add_job
+ * or block_job_txn_new. If it's the last reference to the object, it will be
+ * freed.
+ */
+void block_job_txn_unref(BlockJobTxn *txn);
+
+/**
+ * block_job_txn_add_job:
+ * @txn: The transaction (may be NULL)
+ * @job: Job to add to the transaction
+ *
+ * Add @job to the transaction.  The @job must not already be in a transaction.
+ * The caller must call either block_job_txn_unref() or block_job_completed()
+ * to release the reference that is automatically grabbed here.
+ */
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job);
+
 #endif
diff --git a/qemu/include/block/dirty-bitmap.h b/qemu/include/block/dirty-bitmap.h
new file mode 100644
index 000000000..80afe603f
--- /dev/null
+++ b/qemu/include/block/dirty-bitmap.h
@@ -0,0 +1,44 @@
+#ifndef BLOCK_DIRTY_BITMAP_H
+#define BLOCK_DIRTY_BITMAP_H
+
+#include "qemu-common.h"
+#include "qemu/hbitmap.h"
+
+BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
+                                          uint32_t granularity,
+                                          const char *name,
+                                          Error **errp);
+int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
+                                       BdrvDirtyBitmap *bitmap,
+                                       Error **errp);
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
+                                            BdrvDirtyBitmap *bitmap,
+                                            Error **errp);
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
+                                           BdrvDirtyBitmap *bitmap,
+                                           Error **errp);
+BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
+                                        const char *name);
+void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap);
+void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
+void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
+void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
+void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
+BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs);
+uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs);
+uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap);
+bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap);
+bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap);
+DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap);
+int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
+                   int64_t sector);
+void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                           int64_t cur_sector, int nr_sectors);
+void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                             int64_t cur_sector, int nr_sectors);
+void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, struct HBitmapIter *hbi);
+void bdrv_set_dirty_iter(struct HBitmapIter *hbi, int64_t offset);
+int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
+void bdrv_dirty_bitmap_truncate(BlockDriverState *bs);
+
+#endif
diff --git a/qemu/include/block/nbd.h b/qemu/include/block/nbd.h
index 65f409d80..b86a97698 100644
--- a/qemu/include/block/nbd.h
+++ b/qemu/include/block/nbd.h
@@ -19,10 +19,11 @@
 #ifndef NBD_H
 #define NBD_H
 
-#include <sys/types.h>
 
 #include "qemu-common.h"
 #include "qemu/option.h"
+#include "io/channel-socket.h"
+#include "crypto/tlscreds.h"
 
 struct nbd_request {
     uint32_t magic;
@@ -55,7 +56,10 @@ struct nbd_reply {
 #define NBD_REP_ACK             (1)             /* Data sending finished. */
 #define NBD_REP_SERVER          (2)             /* Export description. */
 #define NBD_REP_ERR_UNSUP       ((UINT32_C(1) << 31) | 1) /* Unknown option. */
+#define NBD_REP_ERR_POLICY      ((UINT32_C(1) << 31) | 2) /* Server denied */
 #define NBD_REP_ERR_INVALID     ((UINT32_C(1) << 31) | 3) /* Invalid length. */
+#define NBD_REP_ERR_TLS_REQD    ((UINT32_C(1) << 31) | 5) /* TLS required */
+
 
 #define NBD_CMD_MASK_COMMAND	0x0000ffff
 #define NBD_CMD_FLAG_FUA	(1 << 16)
@@ -73,12 +77,19 @@ enum {
 /* Maximum size of a single READ/WRITE data buffer */
 #define NBD_MAX_BUFFER_SIZE (32 * 1024 * 1024)
 
-ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read);
-int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
+ssize_t nbd_wr_syncv(QIOChannel *ioc,
+                     struct iovec *iov,
+                     size_t niov,
+                     size_t offset,
+                     size_t length,
+                     bool do_read);
+int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
                           off_t *size, Error **errp);
-int nbd_init(int fd, int csock, uint32_t flags, off_t size);
-ssize_t nbd_send_request(int csock, struct nbd_request *request);
-ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply);
+int nbd_init(int fd, QIOChannelSocket *sioc, uint32_t flags, off_t size);
+ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request);
+ssize_t nbd_receive_reply(QIOChannel *ioc, struct nbd_reply *reply);
 int nbd_client(int fd);
 int nbd_disconnect(int fd);
 
@@ -98,8 +109,11 @@ NBDExport *nbd_export_find(const char *name);
 void nbd_export_set_name(NBDExport *exp, const char *name);
 void nbd_export_close_all(void);
 
-NBDClient *nbd_client_new(NBDExport *exp, int csock,
-                          void (*close)(NBDClient *));
+void nbd_client_new(NBDExport *exp,
+                    QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
+                    void (*close)(NBDClient *));
 void nbd_client_get(NBDClient *client);
 void nbd_client_put(NBDClient *client);
 
diff --git a/qemu/include/block/qapi.h b/qemu/include/block/qapi.h
index 327549d91..82ba4b63a 100644
--- a/qemu/include/block/qapi.h
+++ b/qemu/include/block/qapi.h
@@ -29,7 +29,8 @@
 #include "block/block.h"
 #include "block/snapshot.h"
 
-BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs, Error **errp);
+BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
+                                        BlockDriverState *bs, Error **errp);
 int bdrv_query_snapshot_info_list(BlockDriverState *bs,
                                   SnapshotInfoList **p_list,
                                   Error **errp);
diff --git a/qemu/include/block/scsi.h b/qemu/include/block/scsi.h
index edde960d1..a311341e6 100644
--- a/qemu/include/block/scsi.h
+++ b/qemu/include/block/scsi.h
@@ -229,7 +229,7 @@ const char *scsi_command_name(uint8_t cmd);
 #define MODE_PAGE_TO_PROTECT                  0x1d
 #define MODE_PAGE_CAPABILITIES                0x2a
 #define MODE_PAGE_ALLS                        0x3f
-/* Not in Mt. Fuji, but in ATAPI 2.6 -- depricated now in favor
+/* Not in Mt. Fuji, but in ATAPI 2.6 -- deprecated now in favor
  * of MODE_PAGE_SENSE_POWER */
 #define MODE_PAGE_CDROM                       0x0d
 
diff --git a/qemu/include/block/snapshot.h b/qemu/include/block/snapshot.h
index 770d9bbc8..e5c055311 100644
--- a/qemu/include/block/snapshot.h
+++ b/qemu/include/block/snapshot.h
@@ -26,7 +26,6 @@
 #define SNAPSHOT_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 #include "qemu/option.h"
 
 
@@ -63,9 +62,9 @@ int bdrv_snapshot_delete(BlockDriverState *bs,
                          const char *snapshot_id,
                          const char *name,
                          Error **errp);
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp);
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp);
 int bdrv_snapshot_list(BlockDriverState *bs,
                        QEMUSnapshotInfo **psn_info);
 int bdrv_snapshot_load_tmp(BlockDriverState *bs,
@@ -75,4 +74,22 @@ int bdrv_snapshot_load_tmp(BlockDriverState *bs,
 int bdrv_snapshot_load_tmp_by_id_or_name(BlockDriverState *bs,
                                          const char *id_or_name,
                                          Error **errp);
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs);
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bsd_bs,
+                             Error **err);
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bsd_bs);
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs);
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs);
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void);
+
 #endif
diff --git a/qemu/include/block/throttle-groups.h b/qemu/include/block/throttle-groups.h
index fab113f6d..aba28f30b 100644
--- a/qemu/include/block/throttle-groups.h
+++ b/qemu/include/block/throttle-groups.h
@@ -30,6 +30,9 @@
 
 const char *throttle_group_get_name(BlockDriverState *bs);
 
+ThrottleState *throttle_group_incref(const char *name);
+void throttle_group_unref(ThrottleState *ts);
+
 void throttle_group_config(BlockDriverState *bs, ThrottleConfig *cfg);
 void throttle_group_get_config(BlockDriverState *bs, ThrottleConfig *cfg);
 
@@ -40,7 +43,4 @@ void coroutine_fn throttle_group_co_io_limits_intercept(BlockDriverState *bs,
                                                         unsigned int bytes,
                                                         bool is_write);
 
-void throttle_group_lock(BlockDriverState *bs);
-void throttle_group_unlock(BlockDriverState *bs);
-
 #endif
diff --git a/qemu/include/block/write-threshold.h b/qemu/include/block/write-threshold.h
index f1b899cd5..234d2193e 100644
--- a/qemu/include/block/write-threshold.h
+++ b/qemu/include/block/write-threshold.h
@@ -12,9 +12,6 @@
 #ifndef BLOCK_WRITE_THRESHOLD_H
 #define BLOCK_WRITE_THRESHOLD_H
 
-#include <stdint.h>
-
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 
 /*
diff --git a/qemu/include/config.h b/qemu/include/config.h
deleted file mode 100644
index e20f78696..000000000
--- a/qemu/include/config.h
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "config-host.h"
-#include "config-target.h"
diff --git a/qemu/include/crypto/afsplit.h b/qemu/include/crypto/afsplit.h
new file mode 100644
index 000000000..4cc4ca4b3
--- /dev/null
+++ b/qemu/include/crypto/afsplit.h
@@ -0,0 +1,135 @@
+/*
+ * QEMU Crypto anti forensic information splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_AFSPLIT_H__
+#define QCRYPTO_AFSPLIT_H__
+
+#include "crypto/hash.h"
+
+/**
+ * This module implements the anti-forensic splitter that is specified
+ * as part of the LUKS format:
+ *
+ *   http://clemens.endorphin.org/cryptography
+ *   http://clemens.endorphin.org/TKS1-draft.pdf
+ *
+ * The core idea is to take a short piece of data (key material)
+ * and process it to expand it to a much larger piece of data.
+ * The expansion process is reversible, to obtain the original
+ * short data. The key property of the expansion is that if any
+ * byte in the larger data set is changed / missing, it should be
+ * impossible to recreate the original short data.
+ *
+ * <example>
+ *    <title>Creating a large split key for storage</title>
+ *    <programlisting>
+ * size_t nkey = 32;
+ * uint32_t stripes = 32768; // To produce a 1 MB split key
+ * uint8_t *masterkey = ....a 32-byte AES key...
+ * uint8_t *splitkey;
+ *
+ * splitkey = g_new0(uint8_t, nkey * stripes);
+ *
+ * if (qcrypto_afsplit_encode(QCRYPTO_HASH_ALG_SHA256,
+ *                            nkey, stripes,
+ *                            masterkey, splitkey, errp) < 0) {
+ *     g_free(splitkey);
+ *     g_free(masterkey);
+ *     return -1;
+ * }
+ *
+ * ...store splitkey somewhere...
+ *
+ * g_free(splitkey);
+ * g_free(masterkey);
+ *    </programlisting>
+ * </example>
+ *
+ * <example>
+ *    <title>Retrieving a master key from storage</title>
+ *    <programlisting>
+ * size_t nkey = 32;
+ * uint32_t stripes = 32768; // To produce a 1 MB split key
+ * uint8_t *masterkey;
+ * uint8_t *splitkey = .... read in 1 MB of data...
+ *
+ * masterkey = g_new0(uint8_t, nkey);
+ *
+ * if (qcrypto_afsplit_decode(QCRYPTO_HASH_ALG_SHA256,
+ *                            nkey, stripes,
+ *                            splitkey, masterkey, errp) < 0) {
+ *     g_free(splitkey);
+ *     g_free(masterkey);
+ *     return -1;
+ * }
+ *
+ * ..decrypt data with masterkey...
+ *
+ * g_free(splitkey);
+ * g_free(masterkey);
+ *    </programlisting>
+ * </example>
+ */
+
+/**
+ * qcrypto_afsplit_encode:
+ * @hash: the hash algorithm to use for data expansion
+ * @blocklen: the size of @in in bytes
+ * @stripes: the number of times to expand @in in size
+ * @in: the master key to be expanded in size
+ * @out: preallocated buffer to hold the split key
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Split the data in @in, which is @blocklen bytes in
+ * size, to form a larger piece of data @out, which is
+ * @blocklen * @stripes bytes in size.
+ *
+ * Returns: 0 on success, -1 on error;
+ */
+int qcrypto_afsplit_encode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp);
+
+/**
+ * qcrypto_afsplit_decode:
+ * @hash: the hash algorithm to use for data compression
+ * @blocklen: the size of @out in bytes
+ * @stripes: the number of times to decrease @in in size
+ * @in: the split key to be recombined
+ * @out: preallocated buffer to hold the master key
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Join the data in @in, which is @blocklen * @stripes
+ * bytes in size, to form the original small piece of
+ * data @out, which is @blocklen bytes in size.
+ *
+ * Returns: 0 on success, -1 on error;
+ */
+int qcrypto_afsplit_decode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp);
+
+#endif /* QCRYPTO_AFSPLIT_H__ */
diff --git a/qemu/include/crypto/block.h b/qemu/include/crypto/block.h
new file mode 100644
index 000000000..a21e11ff8
--- /dev/null
+++ b/qemu/include/crypto/block.h
@@ -0,0 +1,232 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_H__
+#define QCRYPTO_BLOCK_H__
+
+#include "crypto/cipher.h"
+#include "crypto/ivgen.h"
+
+typedef struct QCryptoBlock QCryptoBlock;
+
+/* See also QCryptoBlockFormat, QCryptoBlockCreateOptions
+ * and QCryptoBlockOpenOptions in qapi/crypto.json */
+
+typedef ssize_t (*QCryptoBlockReadFunc)(QCryptoBlock *block,
+                                        size_t offset,
+                                        uint8_t *buf,
+                                        size_t buflen,
+                                        Error **errp,
+                                        void *opaque);
+
+typedef ssize_t (*QCryptoBlockInitFunc)(QCryptoBlock *block,
+                                        size_t headerlen,
+                                        Error **errp,
+                                        void *opaque);
+
+typedef ssize_t (*QCryptoBlockWriteFunc)(QCryptoBlock *block,
+                                         size_t offset,
+                                         const uint8_t *buf,
+                                         size_t buflen,
+                                         Error **errp,
+                                         void *opaque);
+
+/**
+ * qcrypto_block_has_format:
+ * @format: the encryption format
+ * @buf: the data from head of the volume
+ * @len: the length of @buf in bytes
+ *
+ * Given @len bytes of data from the head of a storage volume
+ * in @buf, probe to determine if the volume has the encryption
+ * format specified in @format.
+ *
+ * Returns: true if the data in @buf matches @format
+ */
+bool qcrypto_block_has_format(QCryptoBlockFormat format,
+                              const uint8_t *buf,
+                              size_t buflen);
+
+typedef enum {
+    QCRYPTO_BLOCK_OPEN_NO_IO = (1 << 0),
+} QCryptoBlockOpenFlags;
+
+/**
+ * qcrypto_block_open:
+ * @options: the encryption options
+ * @readfunc: callback for reading data from the volume
+ * @opaque: data to pass to @readfunc
+ * @flags: bitmask of QCryptoBlockOpenFlags values
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new block encryption object for an existing
+ * storage volume encrypted with format identified by
+ * the parameters in @options.
+ *
+ * This will use @readfunc to initialize the encryption
+ * context based on the volume header(s), extracting the
+ * master key(s) as required.
+ *
+ * If @flags contains QCRYPTO_BLOCK_OPEN_NO_IO then
+ * the open process will be optimized to skip any parts
+ * that are only required to perform I/O. In particular
+ * this would usually avoid the need to decrypt any
+ * master keys. The only thing that can be done with
+ * the resulting QCryptoBlock object would be to query
+ * metadata such as the payload offset. There will be
+ * no cipher or ivgen objects available.
+ *
+ * If any part of initializing the encryption context
+ * fails an error will be returned. This could be due
+ * to the volume being in the wrong format, a cipher
+ * or IV generator algorithm that is not supported,
+ * or incorrect passphrases.
+ *
+ * Returns: a block encryption format, or NULL on error
+ */
+QCryptoBlock *qcrypto_block_open(QCryptoBlockOpenOptions *options,
+                                 QCryptoBlockReadFunc readfunc,
+                                 void *opaque,
+                                 unsigned int flags,
+                                 Error **errp);
+
+/**
+ * qcrypto_block_create:
+ * @format: the encryption format
+ * @initfunc: callback for initializing volume header
+ * @writefunc: callback for writing data to the volume header
+ * @opaque: data to pass to @initfunc and @writefunc
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new block encryption object for initializing
+ * a storage volume to be encrypted with format identified
+ * by the parameters in @options.
+ *
+ * This method will allocate space for a new volume header
+ * using @initfunc and then write header data using @writefunc,
+ * generating new master keys, etc as required. Any existing
+ * data present on the volume will be irrevocably destroyed.
+ *
+ * If any part of initializing the encryption context
+ * fails an error will be returned. This could be due
+ * to the volume being in the wrong format, a cipher
+ * or IV generator algorithm that is not supported,
+ * or incorrect passphrases.
+ *
+ * Returns: a block encryption format, or NULL on error
+ */
+QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
+                                   QCryptoBlockInitFunc initfunc,
+                                   QCryptoBlockWriteFunc writefunc,
+                                   void *opaque,
+                                   Error **errp);
+
+/**
+ * @qcrypto_block_decrypt:
+ * @block: the block encryption object
+ * @startsector: the sector from which @buf was read
+ * @buf: the buffer to decrypt
+ * @len: the length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Decrypt @len bytes of cipher text in @buf, writing
+ * plain text back into @buf
+ *
+ * Returns 0 on success, -1 on failure
+ */
+int qcrypto_block_decrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp);
+
+/**
+ * @qcrypto_block_encrypt:
+ * @block: the block encryption object
+ * @startsector: the sector to which @buf will be written
+ * @buf: the buffer to decrypt
+ * @len: the length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Encrypt @len bytes of plain text in @buf, writing
+ * cipher text back into @buf
+ *
+ * Returns 0 on success, -1 on failure
+ */
+int qcrypto_block_encrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp);
+
+/**
+ * qcrypto_block_get_cipher:
+ * @block: the block encryption object
+ *
+ * Get the cipher to use for payload encryption
+ *
+ * Returns: the cipher object
+ */
+QCryptoCipher *qcrypto_block_get_cipher(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_get_ivgen:
+ * @block: the block encryption object
+ *
+ * Get the initialization vector generator to use for
+ * payload encryption
+ *
+ * Returns: the IV generator object
+ */
+QCryptoIVGen *qcrypto_block_get_ivgen(QCryptoBlock *block);
+
+
+/**
+ * qcrypto_block_get_kdf_hash:
+ * @block: the block encryption object
+ *
+ * Get the hash algorithm used with the key derivation
+ * function
+ *
+ * Returns: the hash algorithm
+ */
+QCryptoHashAlgorithm qcrypto_block_get_kdf_hash(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_get_payload_offset:
+ * @block: the block encryption object
+ *
+ * Get the offset to the payload indicated by the
+ * encryption header, in bytes.
+ *
+ * Returns: the payload offset in bytes
+ */
+uint64_t qcrypto_block_get_payload_offset(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_free:
+ * @block: the block encryption object
+ *
+ * Release all resources associated with the encryption
+ * object
+ */
+void qcrypto_block_free(QCryptoBlock *block);
+
+#endif /* QCRYPTO_BLOCK_H__ */
diff --git a/qemu/include/crypto/cipher.h b/qemu/include/crypto/cipher.h
index b4d714f26..d770c4835 100644
--- a/qemu/include/crypto/cipher.h
+++ b/qemu/include/crypto/cipher.h
@@ -21,26 +21,12 @@
 #ifndef QCRYPTO_CIPHER_H__
 #define QCRYPTO_CIPHER_H__
 
-#include "qemu-common.h"
-#include "qapi/error.h"
+#include "qapi-types.h"
 
 typedef struct QCryptoCipher QCryptoCipher;
 
-typedef enum {
-    QCRYPTO_CIPHER_ALG_AES_128,
-    QCRYPTO_CIPHER_ALG_AES_192,
-    QCRYPTO_CIPHER_ALG_AES_256,
-    QCRYPTO_CIPHER_ALG_DES_RFB, /* A stupid variant on DES for VNC */
-
-    QCRYPTO_CIPHER_ALG_LAST
-} QCryptoCipherAlgorithm;
-
-typedef enum {
-    QCRYPTO_CIPHER_MODE_ECB,
-    QCRYPTO_CIPHER_MODE_CBC,
-
-    QCRYPTO_CIPHER_MODE_LAST
-} QCryptoCipherMode;
+/* See also "QCryptoCipherAlgorithm" and "QCryptoCipherMode"
+ * enums defined in qapi/crypto.json */
 
 /**
  * QCryptoCipher:
@@ -107,6 +93,43 @@ struct QCryptoCipher {
  */
 bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg);
 
+/**
+ * qcrypto_cipher_get_block_len:
+ * @alg: the cipher algorithm
+ *
+ * Get the required data block size in bytes. When
+ * encrypting data, it must be a multiple of the
+ * block size.
+ *
+ * Returns: the block size in bytes
+ */
+size_t qcrypto_cipher_get_block_len(QCryptoCipherAlgorithm alg);
+
+
+/**
+ * qcrypto_cipher_get_key_len:
+ * @alg: the cipher algorithm
+ *
+ * Get the required key size in bytes.
+ *
+ * Returns: the key size in bytes
+ */
+size_t qcrypto_cipher_get_key_len(QCryptoCipherAlgorithm alg);
+
+
+/**
+ * qcrypto_cipher_get_iv_len:
+ * @alg: the cipher algorithm
+ * @mode: the cipher mode
+ *
+ * Get the required initialization vector size
+ * in bytes, if one is required.
+ *
+ * Returns: the IV size in bytes, or 0 if no IV is permitted
+ */
+size_t qcrypto_cipher_get_iv_len(QCryptoCipherAlgorithm alg,
+                                 QCryptoCipherMode mode);
+
 
 /**
  * qcrypto_cipher_new:
@@ -114,7 +137,7 @@ bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg);
  * @mode: the cipher usage mode
  * @key: the private key bytes
  * @nkey: the length of @key
- * @errp: pointer to an uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Creates a new cipher object for encrypting/decrypting
  * data with the algorithm @alg in the usage mode @mode.
@@ -150,7 +173,7 @@ void qcrypto_cipher_free(QCryptoCipher *cipher);
  * @in: buffer holding the plain text input data
  * @out: buffer to fill with the cipher text output data
  * @len: the length of @in and @out buffers
- * @errp: pointer to an uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Encrypts the plain text stored in @in, filling
  * @out with the resulting ciphered text. Both the
@@ -172,7 +195,7 @@ int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
  * @in: buffer holding the cipher text input data
  * @out: buffer to fill with the plain text output data
  * @len: the length of @in and @out buffers
- * @errp: pointer to an uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Decrypts the cipher text stored in @in, filling
  * @out with the resulting plain text. Both the
@@ -192,7 +215,7 @@ int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
  * @cipher: the cipher object
  * @iv: the initialization vector bytes
  * @niv: the length of @iv
- * @errpr: pointer to an uninitialized error object
+ * @errpr: pointer to a NULL-initialized error object
  *
  * If the @cipher object is setup to use a mode that requires
  * initialization vectors, this sets the initialization vector
diff --git a/qemu/include/crypto/hash.h b/qemu/include/crypto/hash.h
index b5acbf638..f38caed66 100644
--- a/qemu/include/crypto/hash.h
+++ b/qemu/include/crypto/hash.h
@@ -21,17 +21,9 @@
 #ifndef QCRYPTO_HASH_H__
 #define QCRYPTO_HASH_H__
 
-#include "qemu-common.h"
-#include "qapi/error.h"
-
-typedef enum {
-    QCRYPTO_HASH_ALG_MD5,
-    QCRYPTO_HASH_ALG_SHA1,
-    QCRYPTO_HASH_ALG_SHA256,
-
-    QCRYPTO_HASH_ALG_LAST
-} QCryptoHashAlgorithm;
+#include "qapi-types.h"
 
+/* See also "QCryptoHashAlgorithm" defined in qapi/crypto.json */
 
 /**
  * qcrypto_hash_supports:
@@ -44,6 +36,17 @@ typedef enum {
  */
 gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg);
 
+
+/**
+ * qcrypto_hash_digest_len:
+ * @alg: the hash algorithm
+ *
+ * Determine the size of the hash digest in bytes
+ *
+ * Returns: the digest length in bytes
+ */
+size_t qcrypto_hash_digest_len(QCryptoHashAlgorithm alg);
+
 /**
  * qcrypto_hash_bytesv:
  * @alg: the hash algorithm
@@ -51,7 +54,7 @@ gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg);
  * @niov: the length of @iov
  * @result: pointer to hold output hash
  * @resultlen: pointer to hold length of @result
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory regions
  * present in @iov. The @result pointer will be
@@ -76,7 +79,7 @@ int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
  * @len: the length of @buf
  * @result: pointer to hold output hash
  * @resultlen: pointer to hold length of @result
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory region
  * @buf of length @len. The @result pointer will be
@@ -100,7 +103,7 @@ int qcrypto_hash_bytes(QCryptoHashAlgorithm alg,
  * @iov: the array of memory regions to hash
  * @niov: the length of @iov
  * @digest: pointer to hold output hash
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory regions
  * present in @iov. The @digest pointer will be
@@ -123,7 +126,7 @@ int qcrypto_hash_digestv(QCryptoHashAlgorithm alg,
  * @buf: the memory region to hash
  * @len: the length of @buf
  * @digest: pointer to hold output hash
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory region
  * @buf of length @len. The @digest pointer will be
@@ -146,7 +149,7 @@ int qcrypto_hash_digest(QCryptoHashAlgorithm alg,
  * @iov: the array of memory regions to hash
  * @niov: the length of @iov
  * @base64: pointer to hold output hash
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory regions
  * present in @iov. The @base64 pointer will be
@@ -169,7 +172,7 @@ int qcrypto_hash_base64v(QCryptoHashAlgorithm alg,
  * @buf: the memory region to hash
  * @len: the length of @buf
  * @base64: pointer to hold output hash
- * @errp: pointer to uninitialized error object
+ * @errp: pointer to a NULL-initialized error object
  *
  * Computes the hash across all the memory region
  * @buf of length @len. The @base64 pointer will be
diff --git a/qemu/include/crypto/init.h b/qemu/include/crypto/init.h
index 5fc510c4f..2513ed098 100644
--- a/qemu/include/crypto/init.h
+++ b/qemu/include/crypto/init.h
@@ -21,9 +21,6 @@
 #ifndef QCRYPTO_INIT_H__
 #define QCRYPTO_INIT_H__
 
-#include "qemu-common.h"
-#include "qapi/error.h"
-
 int qcrypto_init(Error **errp);
 
 #endif /* QCRYPTO_INIT_H__ */
diff --git a/qemu/include/crypto/ivgen.h b/qemu/include/crypto/ivgen.h
new file mode 100644
index 000000000..09cdb6fcd
--- /dev/null
+++ b/qemu/include/crypto/ivgen.h
@@ -0,0 +1,206 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_IVGEN_H__
+#define QCRYPTO_IVGEN_H__
+
+#include "crypto/cipher.h"
+#include "crypto/hash.h"
+
+/**
+ * This module provides a framework for generating initialization
+ * vectors for block encryption schemes using chained cipher modes
+ * CBC. The principle is that each disk sector is assigned a unique
+ * initialization vector for use for encryption of data in that
+ * sector.
+ *
+ * <example>
+ *   <title>Encrypting block data with initialiation vectors</title>
+ *   <programlisting>
+ * uint8_t *data = ....data to encrypt...
+ * size_t ndata = XXX;
+ * uint8_t *key = ....some encryption key...
+ * size_t nkey = XXX;
+ * uint8_t *iv;
+ * size_t niv;
+ * size_t sector = 0;
+ *
+ * g_assert((ndata % 512) == 0);
+ *
+ * QCryptoIVGen *ivgen = qcrypto_ivgen_new(QCRYPTO_IVGEN_ALG_ESSIV,
+ *                                         QCRYPTO_CIPHER_ALG_AES_128,
+ *                                         QCRYPTO_HASH_ALG_SHA256,
+ *                                         key, nkey, errp);
+ * if (!ivgen) {
+ *    return -1;
+ * }
+ *
+ * QCryptoCipher *cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+ *                                            QCRYPTO_CIPHER_MODE_CBC,
+ *                                            key, nkey, errp);
+ * if (!cipher) {
+ *     goto error;
+ * }
+ *
+ * niv =  qcrypto_cipher_get_iv_len(QCRYPTO_CIPHER_ALG_AES_128,
+ *                                  QCRYPTO_CIPHER_MODE_CBC);
+ * iv = g_new0(uint8_t, niv);
+ *
+ *
+ * while (ndata) {
+ *     if (qcrypto_ivgen_calculate(ivgen, sector, iv, niv, errp) < 0) {
+ *         goto error;
+ *     }
+ *     if (qcrypto_cipher_setiv(cipher, iv, niv, errp) < 0) {
+ *         goto error;
+ *     }
+ *     if (qcrypto_cipher_encrypt(cipher,
+ *                                data + (sector * 512),
+ *                                data + (sector * 512),
+ *                                512, errp) < 0) {
+ *         goto error;
+ *     }
+ *     sector++;
+ *     ndata -= 512;
+ * }
+ *
+ * g_free(iv);
+ * qcrypto_ivgen_free(ivgen);
+ * qcrypto_cipher_free(cipher);
+ * return 0;
+ *
+ *error:
+ * g_free(iv);
+ * qcrypto_ivgen_free(ivgen);
+ * qcrypto_cipher_free(cipher);
+ * return -1;
+ *   </programlisting>
+ * </example>
+ */
+
+typedef struct QCryptoIVGen QCryptoIVGen;
+
+/* See also QCryptoIVGenAlgorithm enum in qapi/crypto.json */
+
+
+/**
+ * qcrypto_ivgen_new:
+ * @alg: the initialization vector generation algorithm
+ * @cipheralg: the cipher algorithm or 0
+ * @hash: the hash algorithm or 0
+ * @key: the encryption key or NULL
+ * @nkey: the size of @key in bytes
+ *
+ * Create a new initialization vector generator that uses
+ * the algorithm @alg. Whether the remaining parameters
+ * are required or not depends on the choice of @alg
+ * requested.
+ *
+ * - QCRYPTO_IVGEN_ALG_PLAIN
+ *
+ * The IVs are generated by the 32-bit truncated sector
+ * number. This should never be used for block devices
+ * that are larger than 2^32 sectors in size.
+ * All the other parameters are unused.
+ *
+ * - QCRYPTO_IVGEN_ALG_PLAIN64
+ *
+ * The IVs are generated by the 64-bit sector number.
+ * All the other parameters are unused.
+ *
+ * - QCRYPTO_IVGEN_ALG_ESSIV:
+ *
+ * The IVs are generated by encrypting the 64-bit sector
+ * number with a hash of an encryption key. The @cipheralg,
+ * @hash, @key and @nkey parameters are all required.
+ *
+ * Returns: a new IV generator, or NULL on error
+ */
+QCryptoIVGen *qcrypto_ivgen_new(QCryptoIVGenAlgorithm alg,
+                                QCryptoCipherAlgorithm cipheralg,
+                                QCryptoHashAlgorithm hash,
+                                const uint8_t *key, size_t nkey,
+                                Error **errp);
+
+/**
+ * qcrypto_ivgen_calculate:
+ * @ivgen: the IV generator object
+ * @sector: the 64-bit sector number
+ * @iv: a pre-allocated buffer to hold the generated IV
+ * @niv: the number of bytes in @iv
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Calculate a new initialiation vector for the data
+ * to be stored in sector @sector. The IV will be
+ * written into the buffer @iv of size @niv.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_ivgen_calculate(QCryptoIVGen *ivgen,
+                            uint64_t sector,
+                            uint8_t *iv, size_t niv,
+                            Error **errp);
+
+
+/**
+ * qcrypto_ivgen_get_algorithm:
+ * @ivgen: the IV generator object
+ *
+ * Get the algorithm used by this IV generator
+ *
+ * Returns: the IV generator algorithm
+ */
+QCryptoIVGenAlgorithm qcrypto_ivgen_get_algorithm(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_get_cipher:
+ * @ivgen: the IV generator object
+ *
+ * Get the cipher algorithm used by this IV generator (if
+ * applicable)
+ *
+ * Returns: the cipher algorithm
+ */
+QCryptoCipherAlgorithm qcrypto_ivgen_get_cipher(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_get_hash:
+ * @ivgen: the IV generator object
+ *
+ * Get the hash algorithm used by this IV generator (if
+ * applicable)
+ *
+ * Returns: the hash algorithm
+ */
+QCryptoHashAlgorithm qcrypto_ivgen_get_hash(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_free:
+ * @ivgen: the IV generator object
+ *
+ * Release all resources associated with @ivgen, or a no-op
+ * if @ivgen is NULL
+ */
+void qcrypto_ivgen_free(QCryptoIVGen *ivgen);
+
+#endif /* QCRYPTO_IVGEN_H__ */
diff --git a/qemu/include/crypto/pbkdf.h b/qemu/include/crypto/pbkdf.h
new file mode 100644
index 000000000..58a1fe62a
--- /dev/null
+++ b/qemu/include/crypto/pbkdf.h
@@ -0,0 +1,152 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_PBKDF_H__
+#define QCRYPTO_PBKDF_H__
+
+#include "crypto/hash.h"
+
+/**
+ * This module provides an interface to the PBKDF2 algorithm
+ *
+ *   https://en.wikipedia.org/wiki/PBKDF2
+ *
+ * <example>
+ *   <title>Generating an AES encryption key from a user password</title>
+ *   <programlisting>
+ * #include "crypto/cipher.h"
+ * #include "crypto/random.h"
+ * #include "crypto/pbkdf.h"
+ *
+ * ....
+ *
+ * char *password = "a-typical-awful-user-password";
+ * size_t nkey = qcrypto_cipher_get_key_len(QCRYPTO_CIPHER_ALG_AES_128);
+ * uint8_t *salt = g_new0(uint8_t, nkey);
+ * uint8_t *key = g_new0(uint8_t, nkey);
+ * int iterations;
+ * QCryptoCipher *cipher;
+ *
+ * if (qcrypto_random_bytes(salt, nkey, errp) < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * iterations = qcrypto_pbkdf2_count_iters(QCRYPTO_HASH_ALG_SHA256,
+ *                                         (const uint8_t *)password,
+ *                                         strlen(password),
+ *                                         salt, nkey, errp);
+ * if (iterations < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * if (qcrypto_pbkdf2(QCRYPTO_HASH_ALG_SHA256,
+ *                    (const uint8_t *)password, strlen(password),
+ *                    salt, nkey, iterations, key, nkey, errp) < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * g_free(salt);
+ *
+ * cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+ *                             QCRYPTO_CIPHER_MODE_ECB,
+ *                             key, nkey, errp);
+ * g_free(key);
+ *
+ * ....encrypt some data...
+ *
+ * qcrypto_cipher_free(cipher);
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+/**
+ * qcrypto_pbkdf2_supports:
+ * @hash: the hash algorithm
+ *
+ * Determine if the current build supports the PBKDF2 algorithm
+ * in combination with the hash @hash.
+ *
+ * Returns true if supported, false otherwise
+ */
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash);
+
+
+/**
+ * qcrypto_pbkdf2:
+ * @hash: the hash algorithm to use
+ * @key: the user password / key
+ * @nkey: the length of @key in bytes
+ * @salt: a random salt
+ * @nsalt: length of @salt in bytes
+ * @iterations: the number of iterations to compute
+ * @out: pointer to pre-allocated buffer to hold output
+ * @nout: length of @out in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Apply the PBKDF2 algorithm to derive an encryption
+ * key from a user password provided in @key. The
+ * @salt parameter is used to perturb the algorithm.
+ * The @iterations count determines how many times
+ * the hashing process is run, which influences how
+ * hard it is to crack the key. The number of @iterations
+ * should be large enough such that the algorithm takes
+ * 1 second or longer to derive a key. The derived key
+ * will be stored in the preallocated buffer @out.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp);
+
+/**
+ * qcrypto_pbkdf2_count_iters:
+ * @hash: the hash algorithm to use
+ * @key: the user password / key
+ * @nkey: the length of @key in bytes
+ * @salt: a random salt
+ * @nsalt: length of @salt in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Time the PBKDF2 algorithm to determine how many
+ * iterations are required to derive an encryption
+ * key from a user password provided in @key in 1
+ * second of compute time. The result of this can
+ * be used as a the @iterations parameter of a later
+ * call to qcrypto_pbkdf2().
+ *
+ * Returns: number of iterations in 1 second, -1 on error
+ */
+int qcrypto_pbkdf2_count_iters(QCryptoHashAlgorithm hash,
+                               const uint8_t *key, size_t nkey,
+                               const uint8_t *salt, size_t nsalt,
+                               Error **errp);
+
+#endif /* QCRYPTO_PBKDF_H__ */
diff --git a/qemu/include/crypto/random.h b/qemu/include/crypto/random.h
new file mode 100644
index 000000000..b3021c4ce
--- /dev/null
+++ b/qemu/include/crypto/random.h
@@ -0,0 +1,44 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_RANDOM_H__
+#define QCRYPTO_RANDOM_H__
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+
+
+/**
+ * qcrypto_random_bytes:
+ * @buf: the buffer to fill
+ * @buflen: length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Fill @buf with @buflen bytes of cryptographically strong
+ * random data
+ *
+ * Returns 0 on sucess, -1 on error
+ */
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp);
+
+
+#endif /* QCRYPTO_RANDOM_H__ */
diff --git a/qemu/include/crypto/secret.h b/qemu/include/crypto/secret.h
new file mode 100644
index 000000000..b7392c6ba
--- /dev/null
+++ b/qemu/include/crypto/secret.h
@@ -0,0 +1,146 @@
+/*
+ * QEMU crypto secret support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_SECRET_H__
+#define QCRYPTO_SECRET_H__
+
+#include "qom/object.h"
+
+#define TYPE_QCRYPTO_SECRET "secret"
+#define QCRYPTO_SECRET(obj)                  \
+    OBJECT_CHECK(QCryptoSecret, (obj), TYPE_QCRYPTO_SECRET)
+
+typedef struct QCryptoSecret QCryptoSecret;
+typedef struct QCryptoSecretClass QCryptoSecretClass;
+
+/**
+ * QCryptoSecret:
+ *
+ * The QCryptoSecret object provides storage of secrets,
+ * which may be user passwords, encryption keys or any
+ * other kind of sensitive data that is represented as
+ * a sequence of bytes.
+ *
+ * The sensitive data associated with the secret can
+ * be provided directly via the 'data' property, or
+ * indirectly via the 'file' property. In the latter
+ * case there is support for file descriptor passing
+ * via the usual /dev/fdset/NN syntax that QEMU uses.
+ *
+ * The data for a secret can be provided in two formats,
+ * either as a UTF-8 string (the default), or as base64
+ * encoded 8-bit binary data. The latter is appropriate
+ * for raw encryption keys, while the former is appropriate
+ * for user entered passwords.
+ *
+ * The data may be optionally encrypted with AES-256-CBC,
+ * and the decryption key provided by another
+ * QCryptoSecret instance identified by the 'keyid'
+ * property. When passing sensitive data directly
+ * via the 'data' property it is strongly recommended
+ * to use the AES encryption facility to prevent the
+ * sensitive data being exposed in the process listing
+ * or system log files.
+ *
+ * Providing data directly, insecurely (suitable for
+ * ad hoc developer testing only)
+ *
+ *  $QEMU -object secret,id=sec0,data=letmein
+ *
+ * Providing data indirectly:
+ *
+ *  # printf "letmein" > password.txt
+ *  # $QEMU \
+ *      -object secret,id=sec0,file=password.txt
+ *
+ * Using a master encryption key with data.
+ *
+ * The master key needs to be created as 32 secure
+ * random bytes (optionally base64 encoded)
+ *
+ *  # openssl rand -base64 32 > key.b64
+ *  # KEY=$(base64 -d key.b64 | hexdump  -v -e '/1 "%02X"')
+ *
+ * Each secret to be encrypted needs to have a random
+ * initialization vector generated. These do not need
+ * to be kept secret
+ *
+ *  # openssl rand -base64 16 > iv.b64
+ *  # IV=$(base64 -d iv.b64 | hexdump  -v -e '/1 "%02X"')
+ *
+ * A secret to be defined can now be encrypted
+ *
+ *  # SECRET=$(printf "letmein" |
+ *             openssl enc -aes-256-cbc -a -K $KEY -iv $IV)
+ *
+ * When launching QEMU, create a master secret pointing
+ * to key.b64 and specify that to be used to decrypt
+ * the user password
+ *
+ *  # $QEMU \
+ *      -object secret,id=secmaster0,format=base64,file=key.b64 \
+ *      -object secret,id=sec0,keyid=secmaster0,format=base64,\
+ *          data=$SECRET,iv=$(<iv.b64)
+ *
+ * When encrypting, the data can still be provided via an
+ * external file, in which case it is possible to use either
+ * raw binary data, or base64 encoded. This example uses
+ * raw format
+ *
+ *  # printf "letmein" |
+ *       openssl enc -aes-256-cbc -K $KEY -iv $IV -o pw.aes
+ *  # $QEMU \
+ *      -object secret,id=secmaster0,format=base64,file=key.b64 \
+ *      -object secret,id=sec0,keyid=secmaster0,\
+ *          file=pw.aes,iv=$(<iv.b64)
+ *
+ * Note that the ciphertext can be in either raw or base64
+ * format, as indicated by the 'format' parameter, but the
+ * plaintext resulting from decryption is expected to always
+ * be in raw format.
+ */
+
+struct QCryptoSecret {
+    Object parent_obj;
+    uint8_t *rawdata;
+    size_t rawlen;
+    QCryptoSecretFormat format;
+    char *data;
+    char *file;
+    char *keyid;
+    char *iv;
+};
+
+
+struct QCryptoSecretClass {
+    ObjectClass parent_class;
+};
+
+
+extern int qcrypto_secret_lookup(const char *secretid,
+                                 uint8_t **data,
+                                 size_t *datalen,
+                                 Error **errp);
+extern char *qcrypto_secret_lookup_as_utf8(const char *secretid,
+                                           Error **errp);
+extern char *qcrypto_secret_lookup_as_base64(const char *secretid,
+                                             Error **errp);
+
+#endif /* QCRYPTO_SECRET_H__ */
diff --git a/qemu/include/crypto/tlscreds.h b/qemu/include/crypto/tlscreds.h
new file mode 100644
index 000000000..8e2babd53
--- /dev/null
+++ b/qemu/include/crypto/tlscreds.h
@@ -0,0 +1,66 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_H__
+#define QCRYPTO_TLSCRED_H__
+
+#include "qom/object.h"
+
+#ifdef CONFIG_GNUTLS
+#include <gnutls/gnutls.h>
+#endif
+
+#define TYPE_QCRYPTO_TLS_CREDS "tls-creds"
+#define QCRYPTO_TLS_CREDS(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCreds, (obj), TYPE_QCRYPTO_TLS_CREDS)
+
+typedef struct QCryptoTLSCreds QCryptoTLSCreds;
+typedef struct QCryptoTLSCredsClass QCryptoTLSCredsClass;
+
+#define QCRYPTO_TLS_CREDS_DH_PARAMS "dh-params.pem"
+
+
+/**
+ * QCryptoTLSCreds:
+ *
+ * The QCryptoTLSCreds object is an abstract base for different
+ * types of TLS handshake credentials. Most commonly the
+ * QCryptoTLSCredsX509 subclass will be used to provide x509
+ * certificate credentials.
+ */
+
+struct QCryptoTLSCreds {
+    Object parent_obj;
+    char *dir;
+    QCryptoTLSCredsEndpoint endpoint;
+#ifdef CONFIG_GNUTLS
+    gnutls_dh_params_t dh_params;
+#endif
+    bool verifyPeer;
+};
+
+
+struct QCryptoTLSCredsClass {
+    ObjectClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_H__ */
+
diff --git a/qemu/include/crypto/tlscredsanon.h b/qemu/include/crypto/tlscredsanon.h
new file mode 100644
index 000000000..d3976b84b
--- /dev/null
+++ b/qemu/include/crypto/tlscredsanon.h
@@ -0,0 +1,112 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_ANON_H__
+#define QCRYPTO_TLSCRED_ANON_H__
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_ANON "tls-creds-anon"
+#define QCRYPTO_TLS_CREDS_ANON(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsAnon, (obj), TYPE_QCRYPTO_TLS_CREDS_ANON)
+
+
+typedef struct QCryptoTLSCredsAnon QCryptoTLSCredsAnon;
+typedef struct QCryptoTLSCredsAnonClass QCryptoTLSCredsAnonClass;
+
+/**
+ * QCryptoTLSCredsAnon:
+ *
+ * The QCryptoTLSCredsAnon object provides a representation
+ * of anonymous credentials used perform a TLS handshake.
+ * This is primarily provided for backwards compatibility and
+ * its use is discouraged as it has poor security characteristics
+ * due to lacking MITM attack protection amongst other problems.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_ANON,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-anon",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-anon,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+
+struct QCryptoTLSCredsAnon {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    union {
+        gnutls_anon_server_credentials_t server;
+        gnutls_anon_client_credentials_t client;
+    } data;
+#endif
+};
+
+
+struct QCryptoTLSCredsAnonClass {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_H__ */
+
diff --git a/qemu/include/crypto/tlscredsx509.h b/qemu/include/crypto/tlscredsx509.h
new file mode 100644
index 000000000..25796d7de
--- /dev/null
+++ b/qemu/include/crypto/tlscredsx509.h
@@ -0,0 +1,114 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCRED_X509_H__
+#define QCRYPTO_TLSCRED_X509_H__
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_X509 "tls-creds-x509"
+#define QCRYPTO_TLS_CREDS_X509(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsX509, (obj), TYPE_QCRYPTO_TLS_CREDS_X509)
+
+typedef struct QCryptoTLSCredsX509 QCryptoTLSCredsX509;
+typedef struct QCryptoTLSCredsX509Class QCryptoTLSCredsX509Class;
+
+#define QCRYPTO_TLS_CREDS_X509_CA_CERT "ca-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CA_CRL "ca-crl.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_KEY "server-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_CERT "server-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_KEY "client-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_CERT "client-cert.pem"
+
+
+/**
+ * QCryptoTLSCredsX509:
+ *
+ * The QCryptoTLSCredsX509 object provides a representation
+ * of x509 credentials used to perform a TLS handshake.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_X509,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-x509",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-x509,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+struct QCryptoTLSCredsX509 {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    gnutls_certificate_credentials_t data;
+#endif
+    bool sanityCheck;
+    char *passwordid;
+};
+
+
+struct QCryptoTLSCredsX509Class {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCRED_X509_H__ */
+
diff --git a/qemu/include/crypto/tlssession.h b/qemu/include/crypto/tlssession.h
new file mode 100644
index 000000000..c1bad9e4f
--- /dev/null
+++ b/qemu/include/crypto/tlssession.h
@@ -0,0 +1,322 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLS_SESSION_H__
+#define QCRYPTO_TLS_SESSION_H__
+
+#include "crypto/tlscreds.h"
+
+/**
+ * QCryptoTLSSession:
+ *
+ * The QCryptoTLSSession object encapsulates the
+ * logic to integrate with a TLS providing library such
+ * as GNUTLS, to setup and run TLS sessions.
+ *
+ * The API is designed such that it has no assumption about
+ * the type of transport it is running over. It may be a
+ * traditional TCP socket, or something else entirely. The
+ * only requirement is a full-duplex stream of some kind.
+ *
+ * <example>
+ *   <title>Using TLS session objects</title>
+ *   <programlisting>
+ * static ssize_t mysock_send(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return write(*fd, buf, len);
+ * }
+ *
+ * static ssize_t mysock_recv(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return read(*fd, buf, len);
+ * }
+ *
+ * static int mysock_run_tls(int sockfd,
+ *                           QCryptoTLSCreds *creds,
+ *                           Error *errp)
+ * {
+ *    QCryptoTLSSession *sess;
+ *
+ *    sess = qcrypto_tls_session_new(creds,
+ *                                   "vnc.example.com",
+ *                                   NULL,
+ *                                   QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+ *                                   errp);
+ *    if (sess == NULL) {
+ *       return -1;
+ *    }
+ *
+ *    qcrypto_tls_session_set_callbacks(sess,
+ *                                      mysock_send,
+ *                                      mysock_recv
+ *                                      GINT_TO_POINTER(fd));
+ *
+ *    while (1) {
+ *       if (qcrypto_tls_session_handshake(sess, errp) < 0) {
+ *           qcrypto_tls_session_free(sess);
+ *           return -1;
+ *       }
+ *
+ *       switch(qcrypto_tls_session_get_handshake_status(sess)) {
+ *       case QCRYPTO_TLS_HANDSHAKE_COMPLETE:
+ *           if (qcrypto_tls_session_check_credentials(sess, errp) < )) {
+ *               qcrypto_tls_session_free(sess);
+ *               return -1;
+ *           }
+ *           goto done;
+ *       case QCRYPTO_TLS_HANDSHAKE_RECVING:
+ *           ...wait for GIO_IN event on fd...
+ *           break;
+ *       case QCRYPTO_TLS_HANDSHAKE_SENDING:
+ *           ...wait for GIO_OUT event on fd...
+ *           break;
+ *       }
+ *    }
+ *   done:
+ *
+ *    ....send/recv payload data on sess...
+ *
+ *    qcrypto_tls_session_free(sess):
+ * }
+ *   </programlisting>
+ * </example>
+ */
+
+typedef struct QCryptoTLSSession QCryptoTLSSession;
+
+
+/**
+ * qcrypto_tls_session_new:
+ * @creds: pointer to a TLS credentials object
+ * @hostname: optional hostname to validate
+ * @aclname: optional ACL to validate peer credentials against
+ * @endpoint: role of the TLS session, client or server
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS session object that will be used to
+ * negotiate a TLS session over an arbitrary data channel.
+ * The session object can operate as either the server or
+ * client, according to the value of the @endpoint argument.
+ *
+ * For clients, the @hostname parameter should hold the full
+ * unmodified hostname as requested by the user. This will
+ * be used to verify the against the hostname reported in
+ * the server's credentials (aka x509 certificate).
+ *
+ * The @aclname parameter (optionally) specifies the name
+ * of an access control list that will be used to validate
+ * the peer's credentials. For x509 credentials, the ACL
+ * will be matched against the CommonName shown in the peer's
+ * certificate. If the session is acting as a server, setting
+ * an ACL will require that the client provide a validate
+ * x509 client certificate.
+ *
+ * After creating the session object, the I/O callbacks
+ * must be set using the qcrypto_tls_session_set_callbacks()
+ * method. A TLS handshake sequence must then be completed
+ * using qcrypto_tls_session_handshake(), before payload
+ * data is permitted to be sent/received.
+ *
+ * The session object must be released by calling
+ * qcrypto_tls_session_free() when no longer required
+ *
+ * Returns: a TLS session object, or NULL on error.
+ */
+QCryptoTLSSession *qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                                           const char *hostname,
+                                           const char *aclname,
+                                           QCryptoTLSCredsEndpoint endpoint,
+                                           Error **errp);
+
+/**
+ * qcrypto_tls_session_free:
+ * @sess: the TLS session object
+ *
+ * Release all memory associated with the TLS session
+ * object previously allocated by qcrypto_tls_session_new()
+ */
+void qcrypto_tls_session_free(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_check_credentials:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Validate the peer's credentials after a successful
+ * TLS handshake. It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns 0 if the credentials validated, -1 on error
+ */
+int qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess,
+                                          Error **errp);
+
+typedef ssize_t (*QCryptoTLSSessionWriteFunc)(const char *buf,
+                                              size_t len,
+                                              void *opaque);
+typedef ssize_t (*QCryptoTLSSessionReadFunc)(char *buf,
+                                             size_t len,
+                                             void *opaque);
+
+/**
+ * qcrypto_tls_session_set_callbacks:
+ * @sess: the TLS session object
+ * @writeFunc: callback for sending data
+ * @readFunc: callback to receiving data
+ * @opaque: data to pass to callbacks
+ *
+ * Sets the callback functions that are to be used for sending
+ * and receiving data on the underlying data channel. Typically
+ * the callbacks to write/read to/from a TCP socket, but there
+ * is no assumption made about the type of channel used.
+ *
+ * The @writeFunc callback will be passed the encrypted
+ * data to send to the remote peer.
+ *
+ * The @readFunc callback will be passed a pointer to fill
+ * with encrypted data received from the remote peer
+ */
+void qcrypto_tls_session_set_callbacks(QCryptoTLSSession *sess,
+                                       QCryptoTLSSessionWriteFunc writeFunc,
+                                       QCryptoTLSSessionReadFunc readFunc,
+                                       void *opaque);
+
+/**
+ * qcrypto_tls_session_write:
+ * @sess: the TLS session object
+ * @buf: the plain text to send
+ * @len: the length of @buf
+ *
+ * Encrypt @len bytes of the data in @buf and send
+ * it to the remote peer using the callback previously
+ * registered with qcrypto_tls_session_set_callbacks()
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes sent, or -1 on error
+ */
+ssize_t qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                                  const char *buf,
+                                  size_t len);
+
+/**
+ * qcrypto_tls_session_read:
+ * @sess: the TLS session object
+ * @buf: to fill with plain text received
+ * @len: the length of @buf
+ *
+ * Receive up to @len bytes of data from the remote peer
+ * using the callback previously registered with
+ * qcrypto_tls_session_set_callbacks(), decrypt it and
+ * store it in @buf.
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes received, or -1 on error
+ */
+ssize_t qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                                 char *buf,
+                                 size_t len);
+
+/**
+ * qcrypto_tls_session_handshake:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Start, or continue, a TLS handshake sequence. If
+ * the underlying data channel is non-blocking, then
+ * this method may return control before the handshake
+ * is complete. On non-blocking channels the
+ * qcrypto_tls_session_get_handshake_status() method
+ * should be used to determine whether the handshake
+ * has completed, or is waiting to send or receive
+ * data. In the latter cases, the caller should setup
+ * an event loop watch and call this method again
+ * once the underlying data channel is ready to read
+ * or write again
+ */
+int qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                                  Error **errp);
+
+typedef enum {
+    QCRYPTO_TLS_HANDSHAKE_COMPLETE,
+    QCRYPTO_TLS_HANDSHAKE_SENDING,
+    QCRYPTO_TLS_HANDSHAKE_RECVING,
+} QCryptoTLSSessionHandshakeStatus;
+
+/**
+ * qcrypto_tls_session_get_handshake_status:
+ * @sess: the TLS session object
+ *
+ * Check the status of the TLS handshake. This
+ * is used with non-blocking data channels to
+ * determine whether the handshake is waiting
+ * to send or receive further data to/from the
+ * remote peer.
+ *
+ * Once this returns QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ * it is permitted to send/receive payload data on
+ * the channel
+ */
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_get_key_size:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Check the size of the data channel encryption key
+ *
+ * Returns: the length in bytes of the encryption key
+ * or -1 on error
+ */
+int qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                     Error **errp);
+
+/**
+ * qcrypto_tls_session_get_peer_name:
+ * @sess: the TLS session object
+ *
+ * Get the identified name of the remote peer. If the
+ * TLS session was negotiated using x509 certificate
+ * credentials, this will return the CommonName from
+ * the peer's certificate. If no identified name is
+ * available it will return NULL.
+ *
+ * The returned data must be released with g_free()
+ * when no longer required.
+ *
+ * Returns: the peer's name or NULL.
+ */
+char *qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess);
+
+#endif /* QCRYPTO_TLS_SESSION_H__ */
diff --git a/qemu/include/crypto/xts.h b/qemu/include/crypto/xts.h
new file mode 100644
index 000000000..c2924d8ba
--- /dev/null
+++ b/qemu/include/crypto/xts.h
@@ -0,0 +1,86 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+
+#ifndef QCRYPTO_XTS_H_
+#define QCRYPTO_XTS_H_
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+
+
+#define XTS_BLOCK_SIZE 16
+
+typedef void xts_cipher_func(const void *ctx,
+                             size_t length,
+                             uint8_t *dst,
+                             const uint8_t *src);
+
+/**
+ * xts_decrypt:
+ * @datactx: the cipher context for data decryption
+ * @tweakctx: the cipher context for tweak decryption
+ * @encfunc: the cipher function for encryption
+ * @decfunc: the cipher function for decryption
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ * @length: the length of @dst and @src
+ * @dst: buffer to hold the decrypted plaintext
+ * @src: buffer providing the ciphertext
+ *
+ * Decrypts @src into @dst
+ */
+void xts_decrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src);
+
+/**
+ * xts_decrypt:
+ * @datactx: the cipher context for data encryption
+ * @tweakctx: the cipher context for tweak encryption
+ * @encfunc: the cipher function for encryption
+ * @decfunc: the cipher function for decryption
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ * @length: the length of @dst and @src
+ * @dst: buffer to hold the encrypted ciphertext
+ * @src: buffer providing the plaintext
+ *
+ * Decrypts @src into @dst
+ */
+void xts_encrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src);
+
+
+#endif /* QCRYPTO_XTS_H_ */
diff --git a/qemu/include/elf.h b/qemu/include/elf.h
index 4afd474d5..28d448bbc 100644
--- a/qemu/include/elf.h
+++ b/qemu/include/elf.h
@@ -1,7 +1,6 @@
 #ifndef _QEMU_ELF_H
 #define _QEMU_ELF_H
 
-#include <inttypes.h>
 
 /* 32-bit ELF base types. */
 typedef uint32_t Elf32_Addr;
@@ -133,6 +132,11 @@ typedef int64_t  Elf64_Sxword;
 
 #define EM_AARCH64  183
 
+#define EM_TILEGX   191 /* TILE-Gx */
+
+#define EM_MOXIE           223     /* Moxie processor family */
+#define EM_MOXIE_OLD       0xFEED
+
 /* This is the info that is needed to parse the dynamic section of the file */
 #define DT_NULL		0
 #define DT_NEEDED	1
@@ -1466,6 +1470,11 @@ typedef struct elf64_shdr {
 #define NT_PPC_VMX       0x100          /* PowerPC Altivec/VMX registers */
 #define NT_PPC_SPE       0x101          /* PowerPC SPE/EVR registers */
 #define NT_PPC_VSX       0x102          /* PowerPC VSX registers */
+#define NT_ARM_VFP      0x400           /* ARM VFP/NEON registers */
+#define NT_ARM_TLS      0x401           /* ARM TLS register */
+#define NT_ARM_HW_BREAK 0x402           /* ARM hardware breakpoint registers */
+#define NT_ARM_HW_WATCH 0x403           /* ARM hardware watchpoint registers */
+#define NT_ARM_SYSTEM_CALL      0x404   /* ARM system call number */
 
 
 /* Note header in a PT_NOTE section */
diff --git a/qemu/include/exec/cpu-all.h b/qemu/include/exec/cpu-all.h
index ea6a9a667..08e5093d0 100644
--- a/qemu/include/exec/cpu-all.h
+++ b/qemu/include/exec/cpu-all.h
@@ -154,24 +154,16 @@ static inline void tswap64s(uint64_t *s)
 /* MMU memory access macros */
 
 #if defined(CONFIG_USER_ONLY)
-#include <assert.h>
 #include "exec/user/abitypes.h"
 
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
  */
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long guest_base;
 extern int have_guest_base;
 extern unsigned long reserved_va;
-#define GUEST_BASE guest_base
-#define RESERVED_VA reserved_va
-#else
-#define GUEST_BASE 0ul
-#define RESERVED_VA 0ul
-#endif
 
-#define GUEST_ADDR_MAX (RESERVED_VA ? RESERVED_VA : \
+#define GUEST_ADDR_MAX (reserved_va ? reserved_va : \
                                     (1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
 #endif
 
@@ -181,11 +173,13 @@ extern unsigned long reserved_va;
 #define TARGET_PAGE_MASK ~(TARGET_PAGE_SIZE - 1)
 #define TARGET_PAGE_ALIGN(addr) (((addr) + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK)
 
-/* ??? These should be the larger of uintptr_t and target_ulong.  */
+/* Using intptr_t ensures that qemu_*_page_mask is sign-extended even
+ * when intptr_t is 32-bit and we are aligning a long long.
+ */
 extern uintptr_t qemu_real_host_page_size;
-extern uintptr_t qemu_real_host_page_mask;
+extern intptr_t qemu_real_host_page_mask;
 extern uintptr_t qemu_host_page_size;
-extern uintptr_t qemu_host_page_mask;
+extern intptr_t qemu_host_page_mask;
 
 #define HOST_PAGE_ALIGN(addr) (((addr) + qemu_host_page_size - 1) & qemu_host_page_mask)
 #define REAL_HOST_PAGE_ALIGN(addr) (((addr) + qemu_real_host_page_size - 1) & \
@@ -273,44 +267,6 @@ CPUArchState *cpu_copy(CPUArchState *env);
 
 #if !defined(CONFIG_USER_ONLY)
 
-/* memory API */
-
-typedef struct RAMBlock RAMBlock;
-
-struct RAMBlock {
-    struct rcu_head rcu;
-    struct MemoryRegion *mr;
-    uint8_t *host;
-    ram_addr_t offset;
-    ram_addr_t used_length;
-    ram_addr_t max_length;
-    void (*resized)(const char*, uint64_t length, void *host);
-    uint32_t flags;
-    /* Protected by iothread lock.  */
-    char idstr[256];
-    /* RCU-enabled, writes protected by the ramlist lock */
-    QLIST_ENTRY(RAMBlock) next;
-    int fd;
-};
-
-static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
-{
-    assert(offset < block->used_length);
-    assert(block->host);
-    return (char *)block->host + offset;
-}
-
-typedef struct RAMList {
-    QemuMutex mutex;
-    /* Protected by the iothread lock.  */
-    unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
-    RAMBlock *mru_block;
-    /* RCU-enabled, writes protected by the ramlist lock. */
-    QLIST_HEAD(, RAMBlock) blocks;
-    uint32_t version;
-} RAMList;
-extern RAMList ram_list;
-
 /* Flags stored in the low bits of the TLB virtual address.  These are
    defined so that fast path ram access is all zeros.  */
 /* Zero if TLB entry is valid.  */
@@ -323,9 +279,6 @@ extern RAMList ram_list;
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf);
-ram_addr_t last_ram_offset(void);
-void qemu_mutex_lock_ramlist(void);
-void qemu_mutex_unlock_ramlist(void);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
diff --git a/qemu/include/exec/cpu-common.h b/qemu/include/exec/cpu-common.h
index 9fb1d541d..9e839e50c 100644
--- a/qemu/include/exec/cpu-common.h
+++ b/qemu/include/exec/cpu-common.h
@@ -14,7 +14,6 @@
 #include "qemu/bswap.h"
 #include "qemu/queue.h"
 #include "qemu/fprintf-fn.h"
-#include "qemu/typedefs.h"
 
 /**
  * CPUListState:
@@ -54,7 +53,6 @@ typedef uintptr_t ram_addr_t;
 #endif
 
 extern ram_addr_t ram_size;
-ram_addr_t get_current_ram_size(void);
 
 /* memory API */
 
@@ -64,8 +62,12 @@ typedef uint32_t CPUReadMemoryFunc(void *opaque, hwaddr addr);
 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
 /* This should not be used by devices.  */
 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
+RAMBlock *qemu_ram_block_by_name(const char *name);
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *ram_addr, ram_addr_t *offset);
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
 void qemu_ram_unset_idstr(ram_addr_t addr);
+const char *qemu_ram_get_idstr(RAMBlock *rb);
 
 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write);
diff --git a/qemu/include/exec/cpu-defs.h b/qemu/include/exec/cpu-defs.h
index 98b9cff31..854e7e356 100644
--- a/qemu/include/exec/cpu-defs.h
+++ b/qemu/include/exec/cpu-defs.h
@@ -23,9 +23,6 @@
 #error cpu.h included from common code
 #endif
 
-#include "config.h"
-#include <inttypes.h>
-#include "qemu/osdep.h"
 #include "qemu/queue.h"
 #include "tcg-target.h"
 #ifndef CONFIG_USER_ONLY
@@ -105,17 +102,18 @@ typedef struct CPUTLBEntry {
        bit 3                      : indicates that the entry is invalid
        bit 2..0                   : zero
     */
-    target_ulong addr_read;
-    target_ulong addr_write;
-    target_ulong addr_code;
-    /* Addend to virtual address to get host address.  IO accesses
-       use the corresponding iotlb value.  */
-    uintptr_t addend;
-    /* padding to get a power of two size */
-    uint8_t dummy[(1 << CPU_TLB_ENTRY_BITS) -
-                  (sizeof(target_ulong) * 3 +
-                   ((-sizeof(target_ulong) * 3) & (sizeof(uintptr_t) - 1)) +
-                   sizeof(uintptr_t))];
+    union {
+        struct {
+            target_ulong addr_read;
+            target_ulong addr_write;
+            target_ulong addr_code;
+            /* Addend to virtual address to get host address.  IO accesses
+               use the corresponding iotlb value.  */
+            uintptr_t addend;
+        };
+        /* padding to get a power of two size */
+        uint8_t dummy[1 << CPU_TLB_ENTRY_BITS];
+    };
 } CPUTLBEntry;
 
 QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS));
diff --git a/qemu/include/exec/cpu_ldst.h b/qemu/include/exec/cpu_ldst.h
index 1239c60f2..b573df53b 100644
--- a/qemu/include/exec/cpu_ldst.h
+++ b/qemu/include/exec/cpu_ldst.h
@@ -49,20 +49,20 @@
 
 #if defined(CONFIG_USER_ONLY)
 /* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
-#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + GUEST_BASE))
+#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + guest_base))
 
 #if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
 #define h2g_valid(x) 1
 #else
 #define h2g_valid(x) ({ \
-    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __guest = (unsigned long)(x) - guest_base; \
     (__guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS)) && \
-    (!RESERVED_VA || (__guest < RESERVED_VA)); \
+    (!reserved_va || (__guest < reserved_va)); \
 })
 #endif
 
 #define h2g_nocheck(x) ({ \
-    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __ret = (unsigned long)(x) - guest_base; \
     (abi_ulong)__ret; \
 })
 
@@ -113,25 +113,6 @@
 /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
 #include "tcg.h"
 
-uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
-void helper_stb_mmu(CPUArchState *env, target_ulong addr,
-                    uint8_t val, int mmu_idx);
-void helper_stw_mmu(CPUArchState *env, target_ulong addr,
-                    uint16_t val, int mmu_idx);
-void helper_stl_mmu(CPUArchState *env, target_ulong addr,
-                    uint32_t val, int mmu_idx);
-void helper_stq_mmu(CPUArchState *env, target_ulong addr,
-                    uint64_t val, int mmu_idx);
-
-uint8_t helper_ldb_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
 #ifdef MMU_MODE0_SUFFIX
 #define CPU_MMU_INDEX 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
@@ -363,7 +344,7 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 #endif /* (NB_MMU_MODES > 12) */
 
 /* these access are slower, they must be as rare as possible */
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, false))
 #define MEMSUFFIX _data
 #define DATA_SIZE 1
 #include "exec/cpu_ldst_template.h"
@@ -379,7 +360,7 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 #undef CPU_MMU_INDEX
 #undef MEMSUFFIX
 
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, true))
 #define MEMSUFFIX _code
 #define SOFTMMU_CODE_ACCESS
 
diff --git a/qemu/include/exec/cpu_ldst_template.h b/qemu/include/exec/cpu_ldst_template.h
index 95ab7504e..3091c0003 100644
--- a/qemu/include/exec/cpu_ldst_template.h
+++ b/qemu/include/exec/cpu_ldst_template.h
@@ -27,20 +27,24 @@
 #define SUFFIX q
 #define USUFFIX q
 #define DATA_TYPE uint64_t
+#define SHIFT 3
 #elif DATA_SIZE == 4
 #define SUFFIX l
 #define USUFFIX l
 #define DATA_TYPE uint32_t
+#define SHIFT 2
 #elif DATA_SIZE == 2
 #define SUFFIX w
 #define USUFFIX uw
 #define DATA_TYPE uint16_t
 #define DATA_STYPE int16_t
+#define SHIFT 1
 #elif DATA_SIZE == 1
 #define SUFFIX b
 #define USUFFIX ub
 #define DATA_TYPE uint8_t
 #define DATA_STYPE int8_t
+#define SHIFT 0
 #else
 #error unsupported data size
 #endif
@@ -54,27 +58,36 @@
 #ifdef SOFTMMU_CODE_ACCESS
 #define ADDR_READ addr_code
 #define MMUSUFFIX _cmmu
+#define URETSUFFIX SUFFIX
+#define SRETSUFFIX SUFFIX
 #else
 #define ADDR_READ addr_read
 #define MMUSUFFIX _mmu
+#define URETSUFFIX USUFFIX
+#define SRETSUFFIX glue(s, SUFFIX)
 #endif
 
 /* generic load/store macros */
 
 static inline RES_TYPE
-glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int page_index;
     RES_TYPE res;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr,
+                                                            oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr);
@@ -82,27 +95,43 @@ glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
     return res;
 }
 
+static inline RES_TYPE
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
+
 #if DATA_SIZE <= 2
 static inline int
-glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int res, page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = (DATA_STYPE)glue(glue(helper_ld, SUFFIX),
-                               MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX),
+                               MMUSUFFIX)(env, addr, oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
 }
+
+static inline int
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
 #endif
 
 #ifndef SOFTMMU_CODE_ACCESS
@@ -110,25 +139,36 @@ glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 /* generic store macro */
 
 static inline void
-glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
-                                      RES_TYPE v)
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                 target_ulong ptr,
+                                                 RES_TYPE v, uintptr_t retaddr)
 {
     int page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        glue(glue(helper_st, SUFFIX), MMUSUFFIX)(env, addr, v, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
+                                                     retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v);
     }
 }
 
+static inline void
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
+{
+    glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(env, ptr, v, 0);
+}
+
 #endif /* !SOFTMMU_CODE_ACCESS */
 
 #undef RES_TYPE
@@ -139,3 +179,6 @@ glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
 #undef DATA_SIZE
 #undef MMUSUFFIX
 #undef ADDR_READ
+#undef URETSUFFIX
+#undef SRETSUFFIX
+#undef SHIFT
diff --git a/qemu/include/exec/cpu_ldst_useronly_template.h b/qemu/include/exec/cpu_ldst_useronly_template.h
index b3b865fae..040b14743 100644
--- a/qemu/include/exec/cpu_ldst_useronly_template.h
+++ b/qemu/include/exec/cpu_ldst_useronly_template.h
@@ -56,12 +56,28 @@ glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
     return glue(glue(ld, USUFFIX), _p)(g2h(ptr));
 }
 
+static inline RES_TYPE
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(env, ptr);
+}
+
 #if DATA_SIZE <= 2
 static inline int
 glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     return glue(glue(lds, SUFFIX), _p)(g2h(ptr));
 }
+
+static inline int
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(env, ptr);
+}
 #endif
 
 #ifndef CODE_ACCESS
@@ -71,6 +87,15 @@ glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
 {
     glue(glue(st, SUFFIX), _p)(g2h(ptr), v);
 }
+
+static inline void
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  RES_TYPE v,
+                                                  uintptr_t retaddr)
+{
+    glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(env, ptr, v);
+}
 #endif
 
 #undef RES_TYPE
diff --git a/qemu/include/exec/cputlb.h b/qemu/include/exec/cputlb.h
index 360815e1b..d454c005b 100644
--- a/qemu/include/exec/cputlb.h
+++ b/qemu/include/exec/cputlb.h
@@ -25,23 +25,7 @@ void tlb_protect_code(ram_addr_t ram_addr);
 void tlb_unprotect_code(ram_addr_t ram_addr);
 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                            uintptr_t length);
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
 extern int tlb_flush_count;
 
-/* exec.c */
-void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
-
-MemoryRegionSection *
-address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr, hwaddr *xlat,
-                                  hwaddr *plen);
-hwaddr memory_region_section_get_iotlb(CPUState *cpu,
-                                       MemoryRegionSection *section,
-                                       target_ulong vaddr,
-                                       hwaddr paddr, hwaddr xlat,
-                                       int prot,
-                                       target_ulong *address);
-bool memory_region_is_unassigned(MemoryRegion *mr);
-
 #endif
 #endif
diff --git a/qemu/include/exec/exec-all.h b/qemu/include/exec/exec-all.h
index a6fce04f6..736209505 100644
--- a/qemu/include/exec/exec-all.h
+++ b/qemu/include/exec/exec-all.h
@@ -62,26 +62,16 @@ typedef struct TranslationBlock TranslationBlock;
 #define OPC_BUF_SIZE 640
 #define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
 
-/* Maximum size a TCG op can expand to.  This is complicated because a
-   single op may require several host instructions and register reloads.
-   For now take a wild guess at 192 bytes, which should allow at least
-   a couple of fixup instructions per argument.  */
-#define TCG_MAX_OP_SIZE 192
-
 #define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
 
 #include "qemu/log.h"
 
 void gen_intermediate_code(CPUArchState *env, struct TranslationBlock *tb);
-void gen_intermediate_code_pc(CPUArchState *env, struct TranslationBlock *tb);
 void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
-                          int pc_pos);
+                          target_ulong *data);
 
 void cpu_gen_init(void);
-int cpu_gen_code(CPUArchState *env, struct TranslationBlock *tb,
-                 int *gen_code_size_ptr);
 bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc);
-void page_size_init(void);
 
 void QEMU_NORETURN cpu_resume_from_signal(CPUState *cpu, void *puc);
 void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
@@ -90,20 +80,113 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
                               int cflags);
 void cpu_exec_init(CPUState *cpu, Error **errp);
 void QEMU_NORETURN cpu_loop_exit(CPUState *cpu);
+void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc);
 
 #if !defined(CONFIG_USER_ONLY)
-bool qemu_in_vcpu_thread(void);
-void cpu_reload_memory_map(CPUState *cpu);
-void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as);
+void cpu_reloading_memory_map(void);
+/**
+ * cpu_address_space_init:
+ * @cpu: CPU to add this address space to
+ * @as: address space to add
+ * @asidx: integer index of this address space
+ *
+ * Add the specified address space to the CPU's cpu_ases list.
+ * The address space added with @asidx 0 is the one used for the
+ * convenience pointer cpu->as.
+ * The target-specific code which registers ASes is responsible
+ * for defining what semantics address space 0, 1, 2, etc have.
+ *
+ * Before the first call to this function, the caller must set
+ * cpu->num_ases to the total number of address spaces it needs
+ * to support.
+ *
+ * Note that with KVM only one address space is supported.
+ */
+void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx);
+/**
+ * cpu_get_address_space:
+ * @cpu: CPU to get address space from
+ * @asidx: index identifying which address space to get
+ *
+ * Return the requested address space of this CPU. @asidx
+ * specifies which address space to read.
+ */
+AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx);
 /* cputlb.c */
+/**
+ * tlb_flush_page:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ *
+ * Flush one page from the TLB of the specified CPU, for all
+ * MMU indexes.
+ */
 void tlb_flush_page(CPUState *cpu, target_ulong addr);
+/**
+ * tlb_flush:
+ * @cpu: CPU whose TLB should be flushed
+ * @flush_global: ignored
+ *
+ * Flush the entire TLB for the specified CPU.
+ * The flush_global flag is in theory an indicator of whether the whole
+ * TLB should be flushed, or only those entries not marked global.
+ * In practice QEMU does not implement any global/not global flag for
+ * TLB entries, and the argument is ignored.
+ */
 void tlb_flush(CPUState *cpu, int flush_global);
-void tlb_set_page(CPUState *cpu, target_ulong vaddr,
-                  hwaddr paddr, int prot,
-                  int mmu_idx, target_ulong size);
+/**
+ * tlb_flush_page_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush one page from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...);
+/**
+ * tlb_flush_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush all entries from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_by_mmuidx(CPUState *cpu, ...);
+/**
+ * tlb_set_page_with_attrs:
+ * @cpu: CPU to add this TLB entry for
+ * @vaddr: virtual address of page to add entry for
+ * @paddr: physical address of the page
+ * @attrs: memory transaction attributes
+ * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits)
+ * @mmu_idx: MMU index to insert TLB entry for
+ * @size: size of the page in bytes
+ *
+ * Add an entry to this CPU's TLB (a mapping from virtual address
+ * @vaddr to physical address @paddr) with the specified memory
+ * transaction attributes. This is generally called by the target CPU
+ * specific code after it has been called through the tlb_fill()
+ * entry point and performed a successful page table walk to find
+ * the physical address and attributes for the virtual address
+ * which provoked the TLB miss.
+ *
+ * At most one entry for a given virtual address is permitted. Only a
+ * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only
+ * used by tlb_flush_page.
+ */
 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
                              hwaddr paddr, MemTxAttrs attrs,
                              int prot, int mmu_idx, target_ulong size);
+/* tlb_set_page:
+ *
+ * This function is equivalent to calling tlb_set_page_with_attrs()
+ * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided
+ * as a convenience for CPUs which don't use memory transaction attributes.
+ */
+void tlb_set_page(CPUState *cpu, target_ulong vaddr,
+                  hwaddr paddr, int prot,
+                  int mmu_idx, target_ulong size);
 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr);
 void probe_write(CPUArchState *env, target_ulong addr, int mmu_idx,
                  uintptr_t retaddr);
@@ -115,6 +198,15 @@ static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)
 static inline void tlb_flush(CPUState *cpu, int flush_global)
 {
 }
+
+static inline void tlb_flush_page_by_mmuidx(CPUState *cpu,
+                                            target_ulong addr, ...)
+{
+}
+
+static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+}
 #endif
 
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
@@ -122,13 +214,14 @@ static inline void tlb_flush(CPUState *cpu, int flush_global)
 #define CODE_GEN_PHYS_HASH_BITS     15
 #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
 
-/* estimated block size for TB allocation */
-/* XXX: use a per code average code fragment size and modulate it
-   according to the host CPU */
+/* Estimated block size for TB allocation.  */
+/* ??? The following is based on a 2015 survey of x86_64 host output.
+   Better would seem to be some sort of dynamically sized TB array,
+   adapting to the block sizes actually being produced.  */
 #if defined(CONFIG_SOFTMMU)
-#define CODE_GEN_AVG_BLOCK_SIZE 128
+#define CODE_GEN_AVG_BLOCK_SIZE 400
 #else
-#define CODE_GEN_AVG_BLOCK_SIZE 64
+#define CODE_GEN_AVG_BLOCK_SIZE 150
 #endif
 
 #if defined(__arm__) || defined(_ARCH_PPC) \
@@ -151,10 +244,14 @@ struct TranslationBlock {
 #define CF_LAST_IO     0x8000 /* Last insn may be an IO access.  */
 #define CF_NOCACHE     0x10000 /* To be freed after execution */
 #define CF_USE_ICOUNT  0x20000
+#define CF_IGNORE_ICOUNT 0x40000 /* Do not generate icount code */
 
     void *tc_ptr;    /* pointer to the translated code */
+    uint8_t *tc_search;  /* pointer to search data */
     /* next matching tb for physical address. */
     struct TranslationBlock *phys_hash_next;
+    /* original tb when cflags has CF_NOCACHE */
+    struct TranslationBlock *orig_tb;
     /* first and second physical page containing code. The lower bit
        of the pointer tells the index in page_next[] */
     struct TranslationBlock *page_next[2];
@@ -176,7 +273,7 @@ struct TranslationBlock {
     struct TranslationBlock *jmp_first;
 };
 
-#include "exec/spinlock.h"
+#include "qemu/thread.h"
 
 typedef struct TBContext TBContext;
 
@@ -186,7 +283,7 @@ struct TBContext {
     TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
     int nb_tbs;
     /* any access to the tbs or the page table must use this lock */
-    spinlock_t tb_lock;
+    QemuMutex tb_lock;
 
     /* statistics */
     int tb_flush_count;
@@ -282,6 +379,11 @@ static inline void tb_add_jump(TranslationBlock *tb, int n,
 {
     /* NOTE: this test is only needed for thread safety */
     if (!tb->jmp_next[n]) {
+        qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc,
+                               "Linking TBs %p [" TARGET_FMT_lx
+                               "] index %d -> %p [" TARGET_FMT_lx "]\n",
+                               tb->tc_ptr, tb->pc, n,
+                               tb_next->tc_ptr, tb_next->pc);
         /* patch the native jump address */
         tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr);
 
@@ -308,20 +410,14 @@ extern uintptr_t tci_tb_ptr;
    to indicate the compressed mode; subtracting two works around that.  It
    is also the case that there are no host isas that contain a call insn
    smaller than 4 bytes, so we don't worry about special-casing this.  */
-#if defined(CONFIG_TCG_INTERPRETER)
-# define GETPC_ADJ   0
-#else
-# define GETPC_ADJ   2
-#endif
+#define GETPC_ADJ   2
 
 #define GETPC()  (GETRA() - GETPC_ADJ)
 
 #if !defined(CONFIG_USER_ONLY)
 
-void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
-
 struct MemoryRegion *iotlb_to_region(CPUState *cpu,
-                                     hwaddr index);
+                                     hwaddr index, MemTxAttrs attrs);
 
 void tlb_fill(CPUState *cpu, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr);
@@ -329,43 +425,44 @@ void tlb_fill(CPUState *cpu, target_ulong addr, int is_write, int mmu_idx,
 #endif
 
 #if defined(CONFIG_USER_ONLY)
+void mmap_lock(void);
+void mmap_unlock(void);
+
 static inline tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 {
     return addr;
 }
 #else
+static inline void mmap_lock(void) {}
+static inline void mmap_unlock(void) {}
+
 /* cputlb.c */
 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr);
+
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length);
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr);
+
+/* exec.c */
+void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
+
+MemoryRegionSection *
+address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
+                                  hwaddr *xlat, hwaddr *plen);
+hwaddr memory_region_section_get_iotlb(CPUState *cpu,
+                                       MemoryRegionSection *section,
+                                       target_ulong vaddr,
+                                       hwaddr paddr, hwaddr xlat,
+                                       int prot,
+                                       target_ulong *address);
+bool memory_region_is_unassigned(MemoryRegion *mr);
+
 #endif
 
 /* vl.c */
 extern int singlestep;
 
-/* cpu-exec.c */
-extern volatile sig_atomic_t exit_request;
+/* cpu-exec.c, accessed with atomic_mb_read/atomic_mb_set */
+extern CPUState *tcg_current_cpu;
+extern bool exit_request;
 
-/**
- * cpu_can_do_io:
- * @cpu: The CPU for which to check IO.
- *
- * Deterministic execution requires that IO only be performed on the last
- * instruction of a TB so that interrupts take effect immediately.
- *
- * Returns: %true if memory-mapped IO is safe, %false otherwise.
- */
-static inline bool cpu_can_do_io(CPUState *cpu)
-{
-    if (!use_icount) {
-        return true;
-    }
-    /* If not executing code then assume we are ok.  */
-    if (cpu->current_tb == NULL) {
-        return true;
-    }
-    return cpu->can_do_io != 0;
-}
-
-#if !defined(CONFIG_USER_ONLY)
-void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
-#endif
 #endif
diff --git a/qemu/include/exec/gdbstub.h b/qemu/include/exec/gdbstub.h
index 05f57c243..d9e8cf771 100644
--- a/qemu/include/exec/gdbstub.h
+++ b/qemu/include/exec/gdbstub.h
@@ -14,7 +14,34 @@
 typedef void (*gdb_syscall_complete_cb)(CPUState *cpu,
                                         target_ulong ret, target_ulong err);
 
+/**
+ * gdb_do_syscall:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * ...: list of arguments to interpolate into @fmt
+ *
+ * Send a GDB syscall request. This function will return immediately;
+ * the callback function will be called later when the remote system
+ * call has completed.
+ *
+ * @fmt should be in the 'call-id,parameter,parameter...' format documented
+ * for the F request packet in the GDB remote protocol. A limited set of
+ * printf-style format specifiers is supported:
+ *   %x  - target_ulong argument printed in hex
+ *   %lx - 64-bit argument printed in hex
+ *   %s  - string pointer (target_ulong) and length (int) pair
+ */
 void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...);
+/**
+ * gdb_do_syscallv:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * @va: arguments to interpolate into @fmt
+ *
+ * As gdb_do_syscall, but taking a va_list rather than a variable
+ * argument list.
+ */
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va);
 int use_gdb_syscalls(void);
 void gdb_set_stop_cpu(CPUState *cpu);
 void gdb_exit(CPUArchState *, int);
diff --git a/qemu/include/exec/helper-head.h b/qemu/include/exec/helper-head.h
index b009ccb11..ec790432d 100644
--- a/qemu/include/exec/helper-head.h
+++ b/qemu/include/exec/helper-head.h
@@ -18,7 +18,6 @@
 #ifndef DEF_HELPER_H
 #define DEF_HELPER_H 1
 
-#include "qemu/osdep.h"
 
 #define HELPER(name) glue(helper_, name)
 
diff --git a/qemu/include/exec/log.h b/qemu/include/exec/log.h
new file mode 100644
index 000000000..ba1c9b568
--- /dev/null
+++ b/qemu/include/exec/log.h
@@ -0,0 +1,60 @@
+#ifndef QEMU_EXEC_LOG_H
+#define QEMU_EXEC_LOG_H
+
+#include "qemu/log.h"
+#include "qom/cpu.h"
+#include "disas/disas.h"
+
+/* cpu_dump_state() logging functions: */
+/**
+ * log_cpu_state:
+ * @cpu: The CPU whose state is to be logged.
+ * @flags: Flags what to log.
+ *
+ * Logs the output of cpu_dump_state().
+ */
+static inline void log_cpu_state(CPUState *cpu, int flags)
+{
+    if (qemu_log_enabled()) {
+        cpu_dump_state(cpu, qemu_logfile, fprintf, flags);
+    }
+}
+
+/**
+ * log_cpu_state_mask:
+ * @mask: Mask when to log.
+ * @cpu: The CPU whose state is to be logged.
+ * @flags: Flags what to log.
+ *
+ * Logs the output of cpu_dump_state() if loglevel includes @mask.
+ */
+static inline void log_cpu_state_mask(int mask, CPUState *cpu, int flags)
+{
+    if (qemu_loglevel & mask) {
+        log_cpu_state(cpu, flags);
+    }
+}
+
+#ifdef NEED_CPU_H
+/* disas() and target_disas() to qemu_logfile: */
+static inline void log_target_disas(CPUState *cpu, target_ulong start,
+                                    target_ulong len, int flags)
+{
+    target_disas(qemu_logfile, cpu, start, len, flags);
+}
+
+static inline void log_disas(void *code, unsigned long size)
+{
+    disas(qemu_logfile, code, size);
+}
+
+#if defined(CONFIG_USER_ONLY)
+/* page_dump() output to the log file: */
+static inline void log_page_dump(void)
+{
+    page_dump(qemu_logfile);
+}
+#endif
+#endif
+
+#endif
diff --git a/qemu/include/exec/memattrs.h b/qemu/include/exec/memattrs.h
index f8537a8d9..e60106184 100644
--- a/qemu/include/exec/memattrs.h
+++ b/qemu/include/exec/memattrs.h
@@ -35,8 +35,8 @@ typedef struct MemTxAttrs {
     unsigned int secure:1;
     /* Memory access is usermode (unprivileged) */
     unsigned int user:1;
-    /* Stream ID (for MSI for example) */
-    unsigned int stream_id:16;
+    /* Requester ID (for MSI for example) */
+    unsigned int requester_id:16;
 } MemTxAttrs;
 
 /* Bus masters which don't specify any attributes will get this,
diff --git a/qemu/include/exec/memory.h b/qemu/include/exec/memory.h
index 94d20eae0..e2a3e9953 100644
--- a/qemu/include/exec/memory.h
+++ b/qemu/include/exec/memory.h
@@ -21,8 +21,6 @@
 #define DIRTY_MEMORY_MIGRATION 2
 #define DIRTY_MEMORY_NUM       3        /* num of dirty bits */
 
-#include <stdint.h>
-#include <stdbool.h>
 #include "exec/cpu-common.h"
 #ifndef CONFIG_USER_ONLY
 #include "exec/hwaddr.h"
@@ -31,7 +29,6 @@
 #include "qemu/queue.h"
 #include "qemu/int128.h"
 #include "qemu/notify.h"
-#include "qapi/error.h"
 #include "qom/object.h"
 #include "qemu/rcu.h"
 
@@ -159,27 +156,33 @@ typedef struct MemoryRegionIoeventfd MemoryRegionIoeventfd;
 
 struct MemoryRegion {
     Object parent_obj;
+
     /* All fields are private - violators will be prosecuted */
-    const MemoryRegionOps *ops;
+
+    /* The following fields should fit in a cache line */
+    bool romd_mode;
+    bool ram;
+    bool subpage;
+    bool readonly; /* For RAM regions */
+    bool rom_device;
+    bool flush_coalesced_mmio;
+    bool global_locking;
+    uint8_t dirty_log_mask;
+    RAMBlock *ram_block;
+    Object *owner;
     const MemoryRegionIOMMUOps *iommu_ops;
+
+    const MemoryRegionOps *ops;
     void *opaque;
     MemoryRegion *container;
     Int128 size;
     hwaddr addr;
     void (*destructor)(MemoryRegion *mr);
-    ram_addr_t ram_addr;
     uint64_t align;
-    bool subpage;
     bool terminates;
-    bool romd_mode;
-    bool ram;
     bool skip_dump;
-    bool readonly; /* For RAM regions */
     bool enabled;
-    bool rom_device;
     bool warning_printed; /* For reservations */
-    bool flush_coalesced_mmio;
-    bool global_locking;
     uint8_t vga_logging_count;
     MemoryRegion *alias;
     hwaddr alias_offset;
@@ -189,7 +192,6 @@ struct MemoryRegion {
     QTAILQ_ENTRY(MemoryRegion) subregions_link;
     QTAILQ_HEAD(coalesced_ranges, CoalescedMemoryRange) coalesced;
     const char *name;
-    uint8_t dirty_log_mask;
     unsigned ioeventfd_nb;
     MemoryRegionIoeventfd *ioeventfds;
     NotifierList iommu_notify;
@@ -236,6 +238,8 @@ struct AddressSpace {
     struct rcu_head rcu;
     char *name;
     MemoryRegion *root;
+    int ref_count;
+    bool malloced;
 
     /* Accessed via RCU.  */
     struct FlatView *current_map;
@@ -324,7 +328,7 @@ void memory_region_unref(MemoryRegion *mr);
  * @owner: the object that tracks the region's reference count
  * @ops: a structure containing read and write callbacks to be used when
  *       I/O is performed on the region.
- * @opaque: passed to to the read and write callbacks of the @ops structure.
+ * @opaque: passed to the read and write callbacks of the @ops structure.
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
@@ -437,6 +441,9 @@ void memory_region_init_alias(MemoryRegion *mr,
  * memory_region_init_rom_device:  Initialize a ROM memory region.  Writes are
  *                                 handled via callbacks.
  *
+ * If NULL callbacks pointer is given, then I/O space is not supposed to be
+ * handled by QEMU itself. Any access via the memory API will cause an abort().
+ *
  * @mr: the #MemoryRegion to be initialized.
  * @owner: the object that tracks the region's reference count
  * @ops: callbacks for write access handling.
@@ -459,16 +466,21 @@ void memory_region_init_rom_device(MemoryRegion *mr,
  * A reservation region primariy serves debugging purposes.  It claims I/O
  * space that is not supposed to be handled by QEMU itself.  Any access via
  * the memory API will cause an abort().
+ * This function is deprecated. Use memory_region_init_io() with NULL
+ * callbacks instead.
  *
  * @mr: the #MemoryRegion to be initialized
  * @owner: the object that tracks the region's reference count
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    struct Object *owner,
+static inline void memory_region_init_reservation(MemoryRegion *mr,
+                                    Object *owner,
                                     const char *name,
-                                    uint64_t size);
+                                    uint64_t size)
+{
+    memory_region_init_io(mr, owner, NULL, mr, name, size);
+}
 
 /**
  * memory_region_init_iommu: Initialize a memory region that translates
@@ -510,7 +522,10 @@ uint64_t memory_region_size(MemoryRegion *mr);
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_ram(MemoryRegion *mr);
+static inline bool memory_region_is_ram(MemoryRegion *mr)
+{
+    return mr->ram;
+}
 
 /**
  * memory_region_is_skip_dump: check whether a memory region should not be
@@ -550,7 +565,11 @@ static inline bool memory_region_is_romd(MemoryRegion *mr)
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_iommu(MemoryRegion *mr);
+static inline bool memory_region_is_iommu(MemoryRegion *mr)
+{
+    return mr->iommu_ops;
+}
+
 
 /**
  * memory_region_notify_iommu: notify a change in an IOMMU translation entry.
@@ -575,6 +594,19 @@ void memory_region_notify_iommu(MemoryRegion *mr,
 void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n);
 
 /**
+ * memory_region_iommu_replay: replay existing IOMMU translations to
+ * a notifier
+ *
+ * @mr: the memory region to observe
+ * @n: the notifier to which to replay iommu mappings
+ * @granularity: Minimum page granularity to replay notifications for
+ * @is_write: Whether to treat the replay as a translate "write"
+ *     through the iommu
+ */
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n,
+                                hwaddr granularity, bool is_write);
+
+/**
  * memory_region_unregister_iommu_notifier: unregister a notifier for
  * changes to IOMMU translation entries.
  *
@@ -619,7 +651,11 @@ uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr);
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_rom(MemoryRegion *mr);
+static inline bool memory_region_is_rom(MemoryRegion *mr)
+{
+    return mr->ram && mr->readonly;
+}
+
 
 /**
  * memory_region_get_fd: Get a file descriptor backing a RAM memory region.
@@ -635,8 +671,13 @@ int memory_region_get_fd(MemoryRegion *mr);
  * memory_region_get_ram_ptr: Get a pointer into a RAM memory region.
  *
  * Returns a host pointer to a RAM memory region (created with
- * memory_region_init_ram() or memory_region_init_ram_ptr()).  Use with
- * care.
+ * memory_region_init_ram() or memory_region_init_ram_ptr()).
+ *
+ * Use with care; by the time this function returns, the returned pointer is
+ * not protected by RCU anymore.  If the caller is not within an RCU critical
+ * section and does not hold the iothread lock, it must have other means of
+ * protecting the pointer, such as a reference to the region that includes
+ * the incoming ram_addr_t.
  *
  * @mr: the memory region being queried.
  */
@@ -935,9 +976,6 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr,
 /**
  * memory_region_get_ram_addr: Get the ram address associated with a memory
  *                             region
- *
- * DO NOT USE THIS FUNCTION.  This is a temporary workaround while the Xen
- * code is being reworked.
  */
 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr);
 
@@ -1138,12 +1176,28 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
  * address_space_init: initializes an address space
  *
  * @as: an uninitialized #AddressSpace
- * @root: a #MemoryRegion that routes addesses for the address space
+ * @root: a #MemoryRegion that routes addresses for the address space
  * @name: an address space name.  The name is only used for debugging
  *        output.
  */
 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name);
 
+/**
+ * address_space_init_shareable: return an address space for a memory region,
+ *                               creating it if it does not already exist
+ *
+ * @root: a #MemoryRegion that routes addresses for the address space
+ * @name: an address space name.  The name is only used for debugging
+ *        output.
+ *
+ * This function will return a pointer to an existing AddressSpace
+ * which was initialized with the specified MemoryRegion, or it will
+ * create and initialize one if it does not already exist. The ASes
+ * are reference-counted, so the memory will be freed automatically
+ * when the AddressSpace is destroyed via address_space_destroy.
+ */
+AddressSpace *address_space_init_shareable(MemoryRegion *root,
+                                           const char *name);
 
 /**
  * address_space_destroy: destroy an address space
@@ -1189,23 +1243,7 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
                                 MemTxAttrs attrs,
                                 const uint8_t *buf, int len);
 
-/**
- * address_space_read: read from an address space.
- *
- * Return a MemTxResult indicating whether the operation succeeded
- * or failed (eg unassigned memory, device rejected the transaction,
- * IOMMU fault).
- *
- * @as: #AddressSpace to be accessed
- * @addr: address within that address space
- * @attrs: memory transaction attributes
- * @buf: buffer with the data transferred
- */
-MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
-                               uint8_t *buf, int len);
-
-/**
- * address_space_ld*: load from an address space
+/* address_space_ld*: load from an address space
  * address_space_st*: store to an address space
  *
  * These functions perform a load or store of the byte, word,
@@ -1335,6 +1373,66 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
                          int is_write, hwaddr access_len);
 
 
+/* Internal functions, part of the implementation of address_space_read.  */
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+					MemoryRegion *mr);
+MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
+                                    MemTxAttrs attrs, uint8_t *buf, int len);
+void *qemu_get_ram_ptr(RAMBlock *ram_block, ram_addr_t addr);
+
+static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
+{
+    if (is_write) {
+        return memory_region_is_ram(mr) && !mr->readonly;
+    } else {
+        return memory_region_is_ram(mr) || memory_region_is_romd(mr);
+    }
+}
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * Return a MemTxResult indicating whether the operation succeeded
+ * or failed (eg unassigned memory, device rejected the transaction,
+ * IOMMU fault).
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @attrs: memory transaction attributes
+ * @buf: buffer with the data transferred
+ */
+static inline __attribute__((__always_inline__))
+MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                               uint8_t *buf, int len)
+{
+    MemTxResult result = MEMTX_OK;
+    hwaddr l, addr1;
+    void *ptr;
+    MemoryRegion *mr;
+
+    if (__builtin_constant_p(len)) {
+        if (len) {
+            rcu_read_lock();
+            l = len;
+            mr = address_space_translate(as, addr, &addr1, &l, false);
+            if (len == l && memory_access_is_direct(mr, false)) {
+                addr1 += memory_region_get_ram_addr(mr);
+                ptr = qemu_get_ram_ptr(mr->ram_block, addr1);
+                memcpy(buf, ptr, len);
+            } else {
+                result = address_space_read_continue(as, addr, attrs, buf, len,
+                                                     addr1, l, mr);
+            }
+            rcu_read_unlock();
+        }
+    } else {
+        result = address_space_read_full(as, addr, attrs, buf, len);
+    }
+    return result;
+}
+
 #endif
 
 #endif
diff --git a/qemu/include/exec/ram_addr.h b/qemu/include/exec/ram_addr.h
index c113f2114..5adf7a4fc 100644
--- a/qemu/include/exec/ram_addr.h
+++ b/qemu/include/exec/ram_addr.h
@@ -22,22 +22,93 @@
 #ifndef CONFIG_USER_ONLY
 #include "hw/xen/xen.h"
 
-ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
-                                    bool share, const char *mem_path,
-                                    Error **errp);
-ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr, Error **errp);
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
-ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
-                                     void (*resized)(const char*,
-                                                     uint64_t length,
-                                                     void *host),
-                                     MemoryRegion *mr, Error **errp);
+struct RAMBlock {
+    struct rcu_head rcu;
+    struct MemoryRegion *mr;
+    uint8_t *host;
+    ram_addr_t offset;
+    ram_addr_t used_length;
+    ram_addr_t max_length;
+    void (*resized)(const char*, uint64_t length, void *host);
+    uint32_t flags;
+    /* Protected by iothread lock.  */
+    char idstr[256];
+    /* RCU-enabled, writes protected by the ramlist lock */
+    QLIST_ENTRY(RAMBlock) next;
+    int fd;
+};
+
+static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
+{
+    return (b && b->host && offset < b->used_length) ? true : false;
+}
+
+static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
+{
+    assert(offset_in_ramblock(block, offset));
+    return (char *)block->host + offset;
+}
+
+/* The dirty memory bitmap is split into fixed-size blocks to allow growth
+ * under RCU.  The bitmap for a block can be accessed as follows:
+ *
+ *   rcu_read_lock();
+ *
+ *   DirtyMemoryBlocks *blocks =
+ *       atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
+ *
+ *   ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
+ *   unsigned long *block = blocks.blocks[idx];
+ *   ...access block bitmap...
+ *
+ *   rcu_read_unlock();
+ *
+ * Remember to check for the end of the block when accessing a range of
+ * addresses.  Move on to the next block if you reach the end.
+ *
+ * Organization into blocks allows dirty memory to grow (but not shrink) under
+ * RCU.  When adding new RAMBlocks requires the dirty memory to grow, a new
+ * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
+ * the same.  Other threads can safely access existing blocks while dirty
+ * memory is being grown.  When no threads are using the old DirtyMemoryBlocks
+ * anymore it is freed by RCU (but the underlying blocks stay because they are
+ * pointed to from the new DirtyMemoryBlocks).
+ */
+#define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
+typedef struct {
+    struct rcu_head rcu;
+    unsigned long *blocks[];
+} DirtyMemoryBlocks;
+
+typedef struct RAMList {
+    QemuMutex mutex;
+    RAMBlock *mru_block;
+    /* RCU-enabled, writes protected by the ramlist lock. */
+    QLIST_HEAD(, RAMBlock) blocks;
+    DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
+    uint32_t version;
+} RAMList;
+extern RAMList ram_list;
+
+ram_addr_t last_ram_offset(void);
+void qemu_mutex_lock_ramlist(void);
+void qemu_mutex_unlock_ramlist(void);
+
+RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
+                                   bool share, const char *mem_path,
+                                   Error **errp);
+RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+                                  MemoryRegion *mr, Error **errp);
+RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
+RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
+                                    void (*resized)(const char*,
+                                                    uint64_t length,
+                                                    void *host),
+                                    MemoryRegion *mr, Error **errp);
 int qemu_get_ram_fd(ram_addr_t addr);
+void qemu_set_ram_fd(ram_addr_t addr, int fd);
 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
-void *qemu_get_ram_ptr(ram_addr_t addr);
-void qemu_ram_free(ram_addr_t addr);
-void qemu_ram_free_from_ptr(ram_addr_t addr);
+void qemu_ram_free(RAMBlock *block);
 
 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp);
 
@@ -48,30 +119,82 @@ static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
                                                  ram_addr_t length,
                                                  unsigned client)
 {
-    unsigned long end, page, next;
+    DirtyMemoryBlocks *blocks;
+    unsigned long end, page;
+    unsigned long idx, offset, base;
+    bool dirty = false;
 
     assert(client < DIRTY_MEMORY_NUM);
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    next = find_next_bit(ram_list.dirty_memory[client], end, page);
 
-    return next < end;
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+        unsigned long num = next - base;
+        unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
+        if (found < num) {
+            dirty = true;
+            break;
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
+    }
+
+    rcu_read_unlock();
+
+    return dirty;
 }
 
 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
                                                  ram_addr_t length,
                                                  unsigned client)
 {
-    unsigned long end, page, next;
+    DirtyMemoryBlocks *blocks;
+    unsigned long end, page;
+    unsigned long idx, offset, base;
+    bool dirty = true;
 
     assert(client < DIRTY_MEMORY_NUM);
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    next = find_next_zero_bit(ram_list.dirty_memory[client], end, page);
 
-    return next >= end;
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+        unsigned long num = next - base;
+        unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
+        if (found < num) {
+            dirty = false;
+            break;
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
+    }
+
+    rcu_read_unlock();
+
+    return dirty;
 }
 
 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
@@ -113,28 +236,73 @@ static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
                                                       unsigned client)
 {
+    unsigned long page, idx, offset;
+    DirtyMemoryBlocks *blocks;
+
     assert(client < DIRTY_MEMORY_NUM);
-    set_bit_atomic(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]);
+
+    page = addr >> TARGET_PAGE_BITS;
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    set_bit_atomic(offset, blocks->blocks[idx]);
+
+    rcu_read_unlock();
 }
 
 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
                                                        ram_addr_t length,
                                                        uint8_t mask)
 {
+    DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
     unsigned long end, page;
-    unsigned long **d = ram_list.dirty_memory;
+    unsigned long idx, offset, base;
+    int i;
+
+    if (!mask && !xen_enabled()) {
+        return;
+    }
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
-        bitmap_set_atomic(d[DIRTY_MEMORY_MIGRATION], page, end - page);
-    }
-    if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
-        bitmap_set_atomic(d[DIRTY_MEMORY_VGA], page, end - page);
+
+    rcu_read_lock();
+
+    for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+        blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
     }
-    if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
-        bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page);
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+
+        if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
+                              offset, next - page);
+        }
+        if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
+                              offset, next - page);
+        }
+        if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
+                              offset, next - page);
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
     }
+
+    rcu_read_unlock();
+
     xen_modified_memory(start, length);
 }
 
@@ -154,21 +322,41 @@ static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
     /* start address is aligned at the start of a word? */
     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
         (hpratio == 1)) {
+        unsigned long **blocks[DIRTY_MEMORY_NUM];
+        unsigned long idx;
+        unsigned long offset;
         long k;
         long nr = BITS_TO_LONGS(pages);
 
+        idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
+        offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
+                          DIRTY_MEMORY_BLOCK_SIZE);
+
+        rcu_read_lock();
+
+        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+            blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
+        }
+
         for (k = 0; k < nr; k++) {
             if (bitmap[k]) {
                 unsigned long temp = leul_to_cpu(bitmap[k]);
-                unsigned long **d = ram_list.dirty_memory;
 
-                atomic_or(&d[DIRTY_MEMORY_MIGRATION][page + k], temp);
-                atomic_or(&d[DIRTY_MEMORY_VGA][page + k], temp);
+                atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
+                atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
                 if (tcg_enabled()) {
-                    atomic_or(&d[DIRTY_MEMORY_CODE][page + k], temp);
+                    atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
                 }
             }
+
+            if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
+                offset = 0;
+                idx++;
+            }
         }
+
+        rcu_read_unlock();
+
         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
     } else {
         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
@@ -220,18 +408,33 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
         int k;
         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
-        unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION];
+        unsigned long * const *src;
+        unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
+                                        DIRTY_MEMORY_BLOCK_SIZE);
+
+        rcu_read_lock();
+
+        src = atomic_rcu_read(
+                &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 
         for (k = page; k < page + nr; k++) {
-            if (src[k]) {
-                unsigned long bits = atomic_xchg(&src[k], 0);
+            if (src[idx][offset]) {
+                unsigned long bits = atomic_xchg(&src[idx][offset], 0);
                 unsigned long new_dirty;
                 new_dirty = ~dest[k];
                 dest[k] |= bits;
                 new_dirty &= bits;
                 num_dirty += ctpopl(new_dirty);
             }
+
+            if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
+                offset = 0;
+                idx++;
+            }
         }
+
+        rcu_read_unlock();
     } else {
         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
             if (cpu_physical_memory_test_and_clear_dirty(
@@ -249,5 +452,6 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
     return num_dirty;
 }
 
+void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 #endif
 #endif
diff --git a/qemu/include/exec/softmmu-semi.h b/qemu/include/exec/softmmu-semi.h
index 1819cc249..3a58c3f08 100644
--- a/qemu/include/exec/softmmu-semi.h
+++ b/qemu/include/exec/softmmu-semi.h
@@ -9,6 +9,14 @@
 #ifndef SOFTMMU_SEMI_H
 #define SOFTMMU_SEMI_H 1
 
+static inline uint64_t softmmu_tget64(CPUArchState *env, target_ulong addr)
+{
+    uint64_t val;
+
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 0);
+    return tswap64(val);
+}
+
 static inline uint32_t softmmu_tget32(CPUArchState *env, target_ulong addr)
 {
     uint32_t val;
@@ -16,6 +24,7 @@ static inline uint32_t softmmu_tget32(CPUArchState *env, target_ulong addr)
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 0);
     return tswap32(val);
 }
+
 static inline uint32_t softmmu_tget8(CPUArchState *env, target_ulong addr)
 {
     uint8_t val;
@@ -24,16 +33,25 @@ static inline uint32_t softmmu_tget8(CPUArchState *env, target_ulong addr)
     return val;
 }
 
+#define get_user_u64(arg, p) ({ arg = softmmu_tget64(env, p); 0; })
 #define get_user_u32(arg, p) ({ arg = softmmu_tget32(env, p) ; 0; })
 #define get_user_u8(arg, p) ({ arg = softmmu_tget8(env, p) ; 0; })
 #define get_user_ual(arg, p) get_user_u32(arg, p)
 
+static inline void softmmu_tput64(CPUArchState *env,
+                                  target_ulong addr, uint64_t val)
+{
+    val = tswap64(val);
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 1);
+}
+
 static inline void softmmu_tput32(CPUArchState *env,
                                   target_ulong addr, uint32_t val)
 {
     val = tswap32(val);
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 1);
 }
+#define put_user_u64(arg, p) ({ softmmu_tput64(env, p, arg) ; 0; })
 #define put_user_u32(arg, p) ({ softmmu_tput32(env, p, arg) ; 0; })
 #define put_user_ual(arg, p) put_user_u32(arg, p)
 
diff --git a/qemu/include/exec/spinlock.h b/qemu/include/exec/spinlock.h
deleted file mode 100644
index a72edda1d..000000000
--- a/qemu/include/exec/spinlock.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-
-/* configure guarantees us that we have pthreads on any host except
- * mingw32, which doesn't support any of the user-only targets.
- * So we can simply assume we have pthread mutexes here.
- */
-#if defined(CONFIG_USER_ONLY)
-
-#include <pthread.h>
-#define spin_lock pthread_mutex_lock
-#define spin_unlock pthread_mutex_unlock
-#define spinlock_t pthread_mutex_t
-#define SPIN_LOCK_UNLOCKED PTHREAD_MUTEX_INITIALIZER
-
-#else
-
-/* Empty implementations, on the theory that system mode emulation
- * is single-threaded. This means that these functions should only
- * be used from code run in the TCG cpu thread, and cannot protect
- * data structures which might also be accessed from the IO thread
- * or from signal handlers.
- */
-typedef int spinlock_t;
-#define SPIN_LOCK_UNLOCKED 0
-
-static inline void spin_lock(spinlock_t *lock)
-{
-}
-
-static inline void spin_unlock(spinlock_t *lock)
-{
-}
-
-#endif
diff --git a/qemu/include/exec/user/thunk.h b/qemu/include/exec/user/thunk.h
index 3b6746272..ad1d60266 100644
--- a/qemu/include/exec/user/thunk.h
+++ b/qemu/include/exec/user/thunk.h
@@ -19,7 +19,6 @@
 #ifndef THUNK_H
 #define THUNK_H
 
-#include <inttypes.h>
 #include "cpu.h"
 
 /* types enums definitions */
diff --git a/qemu/include/fpu/softfloat.h b/qemu/include/fpu/softfloat.h
index ded34eb00..c93706253 100644
--- a/qemu/include/fpu/softfloat.h
+++ b/qemu/include/fpu/softfloat.h
@@ -86,25 +86,12 @@ this code that are retained.
 #include <sunmath.h>
 #endif
 
-#include <inttypes.h>
-#include "config-host.h"
-#include "qemu/osdep.h"
 
-/*----------------------------------------------------------------------------
-| Each of the following `typedef's defines the most convenient type that holds
-| integers of at least as many bits as specified.  For example, `uint8' should
-| be the most convenient type that can hold unsigned integers of as many as
-| 8 bits.  The `flag' type must be able to hold either a 0 or 1.  For most
-| implementations of C, `flag', `uint8', and `int8' should all be `typedef'ed
-| to the same as `int'.
-*----------------------------------------------------------------------------*/
+/* This 'flag' type must be able to hold at least 0 and 1. It should
+ * probably be replaced with 'bool' but the uses would need to be audited
+ * to check that they weren't accidentally relying on it being a larger type.
+ */
 typedef uint8_t flag;
-typedef uint8_t uint8;
-typedef int8_t int8;
-typedef unsigned int uint32;
-typedef signed int int32;
-typedef uint64_t uint64;
-typedef int64_t int64;
 
 #define LIT64( a ) a##LL
 
@@ -282,7 +269,7 @@ static inline flag get_default_nan_mode(float_status *status)
 | Routine to raise any or all of the software IEC/IEEE floating-point
 | exception flags.
 *----------------------------------------------------------------------------*/
-void float_raise(int8 flags, float_status *status);
+void float_raise(int8_t flags, float_status *status);
 
 /*----------------------------------------------------------------------------
 | If `a' is denormal and we are in flush-to-zero mode then set the
@@ -372,18 +359,18 @@ extern const float16 float16_default_nan;
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE single-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int_fast16_t float32_to_int16(float32, float_status *status);
-uint_fast16_t float32_to_uint16(float32, float_status *status);
-int_fast16_t float32_to_int16_round_to_zero(float32, float_status *status);
-uint_fast16_t float32_to_uint16_round_to_zero(float32, float_status *status);
-int32 float32_to_int32(float32, float_status *status);
-int32 float32_to_int32_round_to_zero(float32, float_status *status);
-uint32 float32_to_uint32(float32, float_status *status);
-uint32 float32_to_uint32_round_to_zero(float32, float_status *status);
-int64 float32_to_int64(float32, float_status *status);
-uint64 float32_to_uint64(float32, float_status *status);
-uint64 float32_to_uint64_round_to_zero(float32, float_status *status);
-int64 float32_to_int64_round_to_zero(float32, float_status *status);
+int16_t float32_to_int16(float32, float_status *status);
+uint16_t float32_to_uint16(float32, float_status *status);
+int16_t float32_to_int16_round_to_zero(float32, float_status *status);
+uint16_t float32_to_uint16_round_to_zero(float32, float_status *status);
+int32_t float32_to_int32(float32, float_status *status);
+int32_t float32_to_int32_round_to_zero(float32, float_status *status);
+uint32_t float32_to_uint32(float32, float_status *status);
+uint32_t float32_to_uint32_round_to_zero(float32, float_status *status);
+int64_t float32_to_int64(float32, float_status *status);
+uint64_t float32_to_uint64(float32, float_status *status);
+uint64_t float32_to_uint64_round_to_zero(float32, float_status *status);
+int64_t float32_to_int64_round_to_zero(float32, float_status *status);
 float64 float32_to_float64(float32, float_status *status);
 floatx80 float32_to_floatx80(float32, float_status *status);
 float128 float32_to_float128(float32, float_status *status);
@@ -484,18 +471,18 @@ extern const float32 float32_default_nan;
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE double-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int_fast16_t float64_to_int16(float64, float_status *status);
-uint_fast16_t float64_to_uint16(float64, float_status *status);
-int_fast16_t float64_to_int16_round_to_zero(float64, float_status *status);
-uint_fast16_t float64_to_uint16_round_to_zero(float64, float_status *status);
-int32 float64_to_int32(float64, float_status *status);
-int32 float64_to_int32_round_to_zero(float64, float_status *status);
-uint32 float64_to_uint32(float64, float_status *status);
-uint32 float64_to_uint32_round_to_zero(float64, float_status *status);
-int64 float64_to_int64(float64, float_status *status);
-int64 float64_to_int64_round_to_zero(float64, float_status *status);
-uint64 float64_to_uint64(float64 a, float_status *status);
-uint64 float64_to_uint64_round_to_zero(float64 a, float_status *status);
+int16_t float64_to_int16(float64, float_status *status);
+uint16_t float64_to_uint16(float64, float_status *status);
+int16_t float64_to_int16_round_to_zero(float64, float_status *status);
+uint16_t float64_to_uint16_round_to_zero(float64, float_status *status);
+int32_t float64_to_int32(float64, float_status *status);
+int32_t float64_to_int32_round_to_zero(float64, float_status *status);
+uint32_t float64_to_uint32(float64, float_status *status);
+uint32_t float64_to_uint32_round_to_zero(float64, float_status *status);
+int64_t float64_to_int64(float64, float_status *status);
+int64_t float64_to_int64_round_to_zero(float64, float_status *status);
+uint64_t float64_to_uint64(float64 a, float_status *status);
+uint64_t float64_to_uint64_round_to_zero(float64 a, float_status *status);
 float32 float64_to_float32(float64, float_status *status);
 floatx80 float64_to_floatx80(float64, float_status *status);
 float128 float64_to_float128(float64, float_status *status);
@@ -596,10 +583,10 @@ extern const float64 float64_default_nan;
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE extended double-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int32 floatx80_to_int32(floatx80, float_status *status);
-int32 floatx80_to_int32_round_to_zero(floatx80, float_status *status);
-int64 floatx80_to_int64(floatx80, float_status *status);
-int64 floatx80_to_int64_round_to_zero(floatx80, float_status *status);
+int32_t floatx80_to_int32(floatx80, float_status *status);
+int32_t floatx80_to_int32_round_to_zero(floatx80, float_status *status);
+int64_t floatx80_to_int64(floatx80, float_status *status);
+int64_t floatx80_to_int64_round_to_zero(floatx80, float_status *status);
 float32 floatx80_to_float32(floatx80, float_status *status);
 float64 floatx80_to_float64(floatx80, float_status *status);
 float128 floatx80_to_float128(floatx80, float_status *status);
@@ -681,10 +668,10 @@ extern const floatx80 floatx80_default_nan;
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE quadruple-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int32 float128_to_int32(float128, float_status *status);
-int32 float128_to_int32_round_to_zero(float128, float_status *status);
-int64 float128_to_int64(float128, float_status *status);
-int64 float128_to_int64_round_to_zero(float128, float_status *status);
+int32_t float128_to_int32(float128, float_status *status);
+int32_t float128_to_int32_round_to_zero(float128, float_status *status);
+int64_t float128_to_int64(float128, float_status *status);
+int64_t float128_to_int64_round_to_zero(float128, float_status *status);
 float32 float128_to_float32(float128, float_status *status);
 float64 float128_to_float64(float128, float_status *status);
 floatx80 float128_to_floatx80(float128, float_status *status);
diff --git a/qemu/include/glib-compat.h b/qemu/include/glib-compat.h
index 318e00036..03d8b1267 100644
--- a/qemu/include/glib-compat.h
+++ b/qemu/include/glib-compat.h
@@ -165,4 +165,73 @@ static inline GThread *g_thread_new(const char *name,
 #define CompatGCond GCond
 #endif /* glib 2.31 */
 
+#if !GLIB_CHECK_VERSION(2, 32, 0)
+/* Beware, function returns gboolean since 2.39.2, see GLib commit 9101915 */
+static inline void g_hash_table_add(GHashTable *hash_table, gpointer key)
+{
+    g_hash_table_replace(hash_table, key, key);
+}
+#endif
+
+#ifndef g_assert_true
+#define g_assert_true(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY(expr)) {                                                  \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be TRUE");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_false
+#define g_assert_false(expr)                                                   \
+    do {                                                                       \
+        if (G_LIKELY(!(expr))) {                                               \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be FALSE");                \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_null
+#define g_assert_null(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY((expr) == NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be NULL");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_nonnull
+#define g_assert_nonnull(expr)                                                 \
+    do {                                                                       \
+        if (G_LIKELY((expr) != NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should not be NULL");             \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_cmpmem
+#define g_assert_cmpmem(m1, l1, m2, l2)                                        \
+    do {                                                                       \
+        gconstpointer __m1 = m1, __m2 = m2;                                    \
+        int __l1 = l1, __l2 = l2;                                              \
+        if (__l1 != __l2) {                                                    \
+            g_assertion_message_cmpnum(                                        \
+                G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,                   \
+                #l1 " (len(" #m1 ")) == " #l2 " (len(" #m2 "))", __l1, "==",   \
+                __l2, 'i');                                                    \
+        } else if (memcmp(__m1, __m2, __l1) != 0) {                            \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "assertion failed (" #m1 " == " #m2 ")");      \
+        }                                                                      \
+    } while (0)
+#endif
+
 #endif
diff --git a/qemu/include/hw/acpi/acpi-defs.h b/qemu/include/hw/acpi/acpi-defs.h
index 2b431e624..c7a03d43b 100644
--- a/qemu/include/hw/acpi/acpi-defs.h
+++ b/qemu/include/hw/acpi/acpi-defs.h
@@ -384,6 +384,15 @@ struct AcpiMadtGenericMsiFrame {
 
 typedef struct AcpiMadtGenericMsiFrame AcpiMadtGenericMsiFrame;
 
+struct AcpiMadtGenericRedistributor {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint64_t base_address;
+    uint32_t range_length;
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericRedistributor AcpiMadtGenericRedistributor;
+
 /*
  * Generic Timer Description Table (GTDT)
  */
diff --git a/qemu/include/hw/acpi/acpi.h b/qemu/include/hw/acpi/acpi.h
index b20bd55a6..e0978c8b9 100644
--- a/qemu/include/hw/acpi/acpi.h
+++ b/qemu/include/hw/acpi/acpi.h
@@ -19,8 +19,6 @@
  * <http://www.gnu.org/licenses/>.
  */
 
-#include "qapi/error.h"
-#include "qemu/typedefs.h"
 #include "qemu/notify.h"
 #include "qemu/option.h"
 #include "exec/memory.h"
@@ -156,7 +154,7 @@ void acpi_pm_tmr_reset(ACPIREGS *ar);
 static inline int64_t acpi_pm_tmr_get_clock(void)
 {
     return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), PM_TIMER_FREQUENCY,
-                    get_ticks_per_sec());
+                    NANOSECONDS_PER_SECOND);
 }
 
 /* PM1a_EVT: piix and ich9 don't implement PM1b. */
@@ -196,4 +194,11 @@ unsigned acpi_table_len(void *current);
 void acpi_table_add(const QemuOpts *opts, Error **errp);
 void acpi_table_add_builtin(const QemuOpts *opts, Error **errp);
 
+typedef struct AcpiSlicOem AcpiSlicOem;
+struct AcpiSlicOem {
+  char *id;
+  char *table_id;
+};
+int acpi_get_slic_oem(AcpiSlicOem *oem);
+
 #endif /* !QEMU_HW_ACPI_H */
diff --git a/qemu/include/hw/acpi/aml-build.h b/qemu/include/hw/acpi/aml-build.h
index e3afa1367..2c994b351 100644
--- a/qemu/include/hw/acpi/aml-build.h
+++ b/qemu/include/hw/acpi/aml-build.h
@@ -1,15 +1,12 @@
 #ifndef HW_ACPI_GEN_UTILS_H
 #define HW_ACPI_GEN_UTILS_H
 
-#include <stdint.h>
 #include <glib.h>
-#include "qemu/compiler.h"
 #include "hw/acpi/acpi-defs.h"
 
 /* Reserve RAM space for tables: add another order of magnitude. */
 #define ACPI_BUILD_TABLE_MAX_SIZE         0x200000
 
-#define ACPI_BUILD_APPNAME  "Bochs"
 #define ACPI_BUILD_APPNAME6 "BOCHS "
 #define ACPI_BUILD_APPNAME4 "BXPC"
 
@@ -17,6 +14,8 @@
 #define ACPI_BUILD_RSDP_FILE "etc/acpi/rsdp"
 #define ACPI_BUILD_TPMLOG_FILE "etc/tpm/log"
 
+#define AML_NOTIFY_METHOD "NTFY"
+
 typedef enum {
     AML_NO_OPCODE = 0,/* has only data */
     AML_OPCODE,       /* has opcode optionally followed by data */
@@ -36,6 +35,24 @@ struct Aml {
 typedef struct Aml Aml;
 
 typedef enum {
+    AML_COMPATIBILITY = 0,
+    AML_TYPEA = 1,
+    AML_TYPEB = 2,
+    AML_TYPEF = 3,
+} AmlDmaType;
+
+typedef enum {
+    AML_NOTBUSMASTER = 0,
+    AML_BUSMASTER = 1,
+} AmlDmaBusMaster;
+
+typedef enum {
+    AML_TRANSFER8 = 0,
+    AML_TRANSFER8_16 = 1,
+    AML_TRANSFER16 = 2,
+} AmlTransferSize;
+
+typedef enum {
     AML_DECODE10 = 0,
     AML_DECODE16 = 1,
 } AmlIODecode;
@@ -50,6 +67,11 @@ typedef enum {
 } AmlAccessType;
 
 typedef enum {
+    AML_NOLOCK = 0,
+    AML_LOCK = 1,
+} AmlLockRule;
+
+typedef enum {
     AML_PRESERVE = 0,
     AML_WRITE_AS_ONES = 1,
     AML_WRITE_AS_ZEROS = 2,
@@ -58,6 +80,7 @@ typedef enum {
 typedef enum {
     AML_SYSTEM_MEMORY = 0X00,
     AML_SYSTEM_IO = 0X01,
+    AML_PCI_CONFIG = 0X02,
 } AmlRegionSpace;
 
 typedef enum {
@@ -149,6 +172,32 @@ typedef enum {
     AML_SHARED_AND_WAKE = 3,
 } AmlShared;
 
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: MethodFlags */
+typedef enum {
+    AML_NOTSERIALIZED = 0,
+    AML_SERIALIZED = 1,
+} AmlSerializeFlag;
+
+/*
+ * ACPI 5.0: Table 6-189 GPIO Connection Descriptor Definition
+ * GPIO Connection Type
+ */
+typedef enum {
+    AML_INTERRUPT_CONNECTION = 0,
+    AML_IO_CONNECTION = 1,
+} AmlGpioConnectionType;
+
+/*
+ * ACPI 5.0: Table 6-189 GPIO Connection Descriptor Definition
+ * _PPI field definition
+ */
+typedef enum {
+    AML_PULL_DEFAULT = 0,
+    AML_PULL_UP = 1,
+    AML_PULL_DOWN = 2,
+    AML_PULL_NONE = 3,
+} AmlPinConfig;
+
 typedef
 struct AcpiBuildTables {
     GArray *table_data;
@@ -199,30 +248,44 @@ Aml *aml_name_decl(const char *name, Aml *val);
 Aml *aml_return(Aml *val);
 Aml *aml_int(const uint64_t val);
 Aml *aml_arg(int pos);
+Aml *aml_to_integer(Aml *arg);
+Aml *aml_to_hexstring(Aml *src, Aml *dst);
+Aml *aml_to_buffer(Aml *src, Aml *dst);
 Aml *aml_store(Aml *val, Aml *target);
-Aml *aml_and(Aml *arg1, Aml *arg2);
-Aml *aml_or(Aml *arg1, Aml *arg2);
+Aml *aml_and(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_or(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_lor(Aml *arg1, Aml *arg2);
 Aml *aml_shiftleft(Aml *arg1, Aml *count);
-Aml *aml_shiftright(Aml *arg1, Aml *count);
+Aml *aml_shiftright(Aml *arg1, Aml *count, Aml *dst);
 Aml *aml_lless(Aml *arg1, Aml *arg2);
-Aml *aml_add(Aml *arg1, Aml *arg2);
+Aml *aml_add(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_subtract(Aml *arg1, Aml *arg2, Aml *dst);
 Aml *aml_increment(Aml *arg);
+Aml *aml_decrement(Aml *arg);
 Aml *aml_index(Aml *arg1, Aml *idx);
 Aml *aml_notify(Aml *arg1, Aml *arg2);
+Aml *aml_call0(const char *method);
 Aml *aml_call1(const char *method, Aml *arg1);
 Aml *aml_call2(const char *method, Aml *arg1, Aml *arg2);
 Aml *aml_call3(const char *method, Aml *arg1, Aml *arg2, Aml *arg3);
 Aml *aml_call4(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4);
+Aml *aml_gpio_int(AmlConsumerAndProducer con_and_pro,
+                  AmlLevelAndEdge edge_level,
+                  AmlActiveHighAndLow active_level, AmlShared shared,
+                  AmlPinConfig pin_config, uint16_t debounce_timeout,
+                  const uint32_t pin_list[], uint32_t pin_count,
+                  const char *resource_source_name,
+                  const uint8_t *vendor_data, uint16_t vendor_data_len);
 Aml *aml_memory32_fixed(uint32_t addr, uint32_t size,
                         AmlReadAndWrite read_and_write);
 Aml *aml_interrupt(AmlConsumerAndProducer con_and_pro,
                    AmlLevelAndEdge level_and_edge,
                    AmlActiveHighAndLow high_and_low, AmlShared shared,
-                   uint32_t irq);
+                   uint32_t *irq_list, uint8_t irq_count);
 Aml *aml_io(AmlIODecode dec, uint16_t min_base, uint16_t max_base,
             uint8_t aln, uint8_t len);
 Aml *aml_operation_region(const char *name, AmlRegionSpace rs,
-                          uint32_t offset, uint32_t len);
+                          Aml *offset, uint32_t len);
 Aml *aml_irq_no_flags(uint8_t irq);
 Aml *aml_named_field(const char *name, unsigned length);
 Aml *aml_reserved_field(unsigned length);
@@ -230,6 +293,8 @@ Aml *aml_local(int num);
 Aml *aml_string(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
 Aml *aml_lnot(Aml *arg);
 Aml *aml_equal(Aml *arg1, Aml *arg2);
+Aml *aml_lgreater(Aml *arg1, Aml *arg2);
+Aml *aml_lgreater_equal(Aml *arg1, Aml *arg2);
 Aml *aml_processor(uint8_t proc_id, uint32_t pblk_addr, uint8_t pblk_len,
                    const char *name_format, ...) GCC_FMT_ATTR(4, 5);
 Aml *aml_eisaid(const char *str);
@@ -259,32 +324,52 @@ Aml *aml_qword_memory(AmlDecode dec, AmlMinFixed min_fixed,
                       uint64_t addr_gran, uint64_t addr_min,
                       uint64_t addr_max, uint64_t addr_trans,
                       uint64_t len);
+Aml *aml_dma(AmlDmaType typ, AmlDmaBusMaster bm, AmlTransferSize sz,
+             uint8_t channel);
+Aml *aml_sleep(uint64_t msec);
 
 /* Block AML object primitives */
 Aml *aml_scope(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
 Aml *aml_device(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
-Aml *aml_method(const char *name, int arg_count);
+Aml *aml_method(const char *name, int arg_count, AmlSerializeFlag sflag);
 Aml *aml_if(Aml *predicate);
 Aml *aml_else(void);
 Aml *aml_while(Aml *predicate);
 Aml *aml_package(uint8_t num_elements);
 Aml *aml_buffer(int buffer_size, uint8_t *byte_list);
 Aml *aml_resource_template(void);
-Aml *aml_field(const char *name, AmlAccessType type, AmlUpdateRule rule);
+Aml *aml_field(const char *name, AmlAccessType type, AmlLockRule lock,
+               AmlUpdateRule rule);
+Aml *aml_mutex(const char *name, uint8_t sync_level);
+Aml *aml_acquire(Aml *mutex, uint16_t timeout);
+Aml *aml_release(Aml *mutex);
+Aml *aml_alias(const char *source_object, const char *alias_object);
+Aml *aml_create_field(Aml *srcbuf, Aml *bit_index, Aml *num_bits,
+                      const char *name);
 Aml *aml_create_dword_field(Aml *srcbuf, Aml *index, const char *name);
+Aml *aml_create_qword_field(Aml *srcbuf, Aml *index, const char *name);
 Aml *aml_varpackage(uint32_t num_elements);
 Aml *aml_touuid(const char *uuid);
 Aml *aml_unicode(const char *str);
+Aml *aml_derefof(Aml *arg);
+Aml *aml_sizeof(Aml *arg);
+Aml *aml_concatenate(Aml *source1, Aml *source2, Aml *target);
 
 void
 build_header(GArray *linker, GArray *table_data,
-             AcpiTableHeader *h, const char *sig, int len, uint8_t rev);
+             AcpiTableHeader *h, const char *sig, int len, uint8_t rev,
+             const char *oem_id, const char *oem_table_id);
 void *acpi_data_push(GArray *table_data, unsigned size);
 unsigned acpi_data_len(GArray *table);
 void acpi_add_table(GArray *table_offsets, GArray *table_data);
 void acpi_build_tables_init(AcpiBuildTables *tables);
 void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre);
 void
-build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets);
+build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets,
+           const char *oem_id, const char *oem_table_id);
+
+int
+build_append_named_dword(GArray *array, const char *name_format, ...)
+GCC_FMT_ATTR(2, 3);
 
 #endif
diff --git a/qemu/include/hw/acpi/bios-linker-loader.h b/qemu/include/hw/acpi/bios-linker-loader.h
index 498c0af77..82f1af643 100644
--- a/qemu/include/hw/acpi/bios-linker-loader.h
+++ b/qemu/include/hw/acpi/bios-linker-loader.h
@@ -2,8 +2,6 @@
 #define BIOS_LINKER_LOADER_H
 
 #include <glib.h>
-#include <stdbool.h>
-#include <inttypes.h>
 
 GArray *bios_linker_loader_init(void);
 
@@ -13,7 +11,7 @@ void bios_linker_loader_alloc(GArray *linker,
                               bool alloc_fseg);
 
 void bios_linker_loader_add_checksum(GArray *linker, const char *file,
-                                     void *table,
+                                     GArray *table,
                                      void *start, unsigned size,
                                      uint8_t *checksum);
 
diff --git a/qemu/include/hw/acpi/cpu_hotplug.h b/qemu/include/hw/acpi/cpu_hotplug.h
index f6d358def..f22640e38 100644
--- a/qemu/include/hw/acpi/cpu_hotplug.h
+++ b/qemu/include/hw/acpi/cpu_hotplug.h
@@ -14,6 +14,7 @@
 
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/pc-hotplug.h"
+#include "hw/acpi/aml-build.h"
 
 typedef struct AcpiCpuHotplug {
     MemoryRegion io;
@@ -25,4 +26,13 @@ void acpi_cpu_plug_cb(ACPIREGS *ar, qemu_irq irq,
 
 void acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
                            AcpiCpuHotplug *gpe_cpu, uint16_t base);
+
+#define CPU_EJECT_METHOD "CPEJ"
+#define CPU_MAT_METHOD "CPMA"
+#define CPU_ON_BITMAP "CPON"
+#define CPU_STATUS_METHOD "CPST"
+#define CPU_STATUS_MAP "PRS"
+#define CPU_SCAN_METHOD "PRSC"
+
+void build_cpu_hotplug_aml(Aml *ctx);
 #endif
diff --git a/qemu/include/hw/acpi/ich9.h b/qemu/include/hw/acpi/ich9.h
index 345fd8d92..63fa19814 100644
--- a/qemu/include/hw/acpi/ich9.h
+++ b/qemu/include/hw/acpi/ich9.h
@@ -62,7 +62,6 @@ typedef struct ICH9LPCPMRegs {
 
 void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
                   bool smm_enabled,
-                  bool enable_tco,
                   qemu_irq sci_irq);
 
 void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base);
diff --git a/qemu/include/hw/acpi/memory_hotplug.h b/qemu/include/hw/acpi/memory_hotplug.h
index 1342adb08..3a646b12e 100644
--- a/qemu/include/hw/acpi/memory_hotplug.h
+++ b/qemu/include/hw/acpi/memory_hotplug.h
@@ -4,6 +4,7 @@
 #include "hw/qdev-core.h"
 #include "hw/acpi/acpi.h"
 #include "migration/vmstate.h"
+#include "hw/acpi/aml-build.h"
 
 /**
  * MemStatus:
@@ -45,4 +46,12 @@ extern const VMStateDescription vmstate_memory_hotplug;
                    vmstate_memory_hotplug, MemHotplugState)
 
 void acpi_memory_ospm_status(MemHotplugState *mem_st, ACPIOSTInfoList ***list);
+
+#define MEMORY_HOTPLUG_DEVICE        "MHPD"
+#define MEMORY_SLOT_SCAN_METHOD      "MSCN"
+#define MEMORY_HOTPLUG_HANDLER_PATH "\\_SB.PCI0." \
+     MEMORY_HOTPLUG_DEVICE "." MEMORY_SLOT_SCAN_METHOD
+
+void build_memory_hotplug_aml(Aml *ctx, uint32_t nr_mem,
+                              uint16_t io_base, uint16_t io_len);
 #endif
diff --git a/qemu/include/hw/acpi/pc-hotplug.h b/qemu/include/hw/acpi/pc-hotplug.h
index 77b156900..6a8d268f8 100644
--- a/qemu/include/hw/acpi/pc-hotplug.h
+++ b/qemu/include/hw/acpi/pc-hotplug.h
@@ -32,28 +32,26 @@
 #define ACPI_MEMORY_HOTPLUG_IO_LEN 24
 #define ACPI_MEMORY_HOTPLUG_BASE 0x0a00
 
-#define MEMORY_HOTPLUG_DEVICE        MHPD
-#define MEMORY_SLOTS_NUMBER          MDNR
-#define MEMORY_HOTPLUG_IO_REGION     HPMR
-#define MEMORY_SLOT_ADDR_LOW         MRBL
-#define MEMORY_SLOT_ADDR_HIGH        MRBH
-#define MEMORY_SLOT_SIZE_LOW         MRLL
-#define MEMORY_SLOT_SIZE_HIGH        MRLH
-#define MEMORY_SLOT_PROXIMITY        MPX
-#define MEMORY_SLOT_ENABLED          MES
-#define MEMORY_SLOT_INSERT_EVENT     MINS
-#define MEMORY_SLOT_REMOVE_EVENT     MRMV
-#define MEMORY_SLOT_EJECT            MEJ
-#define MEMORY_SLOT_SLECTOR          MSEL
-#define MEMORY_SLOT_OST_EVENT        MOEV
-#define MEMORY_SLOT_OST_STATUS       MOSC
-#define MEMORY_SLOT_LOCK             MLCK
-#define MEMORY_SLOT_STATUS_METHOD    MRST
-#define MEMORY_SLOT_CRS_METHOD       MCRS
-#define MEMORY_SLOT_OST_METHOD       MOST
-#define MEMORY_SLOT_PROXIMITY_METHOD MPXM
-#define MEMORY_SLOT_EJECT_METHOD     MEJ0
-#define MEMORY_SLOT_NOTIFY_METHOD    MTFY
-#define MEMORY_SLOT_SCAN_METHOD      MSCN
+#define MEMORY_SLOTS_NUMBER          "MDNR"
+#define MEMORY_HOTPLUG_IO_REGION     "HPMR"
+#define MEMORY_SLOT_ADDR_LOW         "MRBL"
+#define MEMORY_SLOT_ADDR_HIGH        "MRBH"
+#define MEMORY_SLOT_SIZE_LOW         "MRLL"
+#define MEMORY_SLOT_SIZE_HIGH        "MRLH"
+#define MEMORY_SLOT_PROXIMITY        "MPX"
+#define MEMORY_SLOT_ENABLED          "MES"
+#define MEMORY_SLOT_INSERT_EVENT     "MINS"
+#define MEMORY_SLOT_REMOVE_EVENT     "MRMV"
+#define MEMORY_SLOT_EJECT            "MEJ"
+#define MEMORY_SLOT_SLECTOR          "MSEL"
+#define MEMORY_SLOT_OST_EVENT        "MOEV"
+#define MEMORY_SLOT_OST_STATUS       "MOSC"
+#define MEMORY_SLOT_LOCK             "MLCK"
+#define MEMORY_SLOT_STATUS_METHOD    "MRST"
+#define MEMORY_SLOT_CRS_METHOD       "MCRS"
+#define MEMORY_SLOT_OST_METHOD       "MOST"
+#define MEMORY_SLOT_PROXIMITY_METHOD "MPXM"
+#define MEMORY_SLOT_EJECT_METHOD     "MEJ0"
+#define MEMORY_SLOT_NOTIFY_METHOD    "MTFY"
 
 #endif
diff --git a/qemu/include/hw/acpi/pcihp.h b/qemu/include/hw/acpi/pcihp.h
index f3526d4aa..79a43923e 100644
--- a/qemu/include/hw/acpi/pcihp.h
+++ b/qemu/include/hw/acpi/pcihp.h
@@ -27,8 +27,6 @@
 #ifndef HW_ACPI_PCIHP_H
 #define HW_ACPI_PCIHP_H
 
-#include <inttypes.h>
-#include <qemu/typedefs.h>
 #include "hw/acpi/acpi.h"
 #include "migration/vmstate.h"
 
diff --git a/qemu/include/hw/acpi/piix4.h b/qemu/include/hw/acpi/piix4.h
index 65e6fd7aa..26c2370e3 100644
--- a/qemu/include/hw/acpi/piix4.h
+++ b/qemu/include/hw/acpi/piix4.h
@@ -1,8 +1,6 @@
 #ifndef HW_ACPI_PIIX4_H
 #define HW_ACPI_PIIX4_H
 
-#include "qemu/typedefs.h"
-
 Object *piix4_pm_find(void);
 
 #endif
diff --git a/qemu/include/hw/acpi/tco.h b/qemu/include/hw/acpi/tco.h
index c63afc8ca..52ad767dd 100644
--- a/qemu/include/hw/acpi/tco.h
+++ b/qemu/include/hw/acpi/tco.h
@@ -9,7 +9,6 @@
 #ifndef HW_ACPI_TCO_H
 #define HW_ACPI_TCO_H
 
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 
 /* As per ICH9 spec, the internal timer has an error of ~0.6s on every tick */
diff --git a/qemu/include/hw/arm/allwinner-a10.h b/qemu/include/hw/arm/allwinner-a10.h
index 01a189bcd..6b32a99e2 100644
--- a/qemu/include/hw/arm/allwinner-a10.h
+++ b/qemu/include/hw/arm/allwinner-a10.h
@@ -7,6 +7,8 @@
 #include "hw/timer/allwinner-a10-pit.h"
 #include "hw/intc/allwinner-a10-pic.h"
 #include "hw/net/allwinner_emac.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
 
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
@@ -16,6 +18,7 @@
 #define AW_A10_PIT_REG_BASE     0x01c20c00
 #define AW_A10_UART0_REG_BASE   0x01c28000
 #define AW_A10_EMAC_BASE        0x01c0b000
+#define AW_A10_SATA_BASE        0x01c18000
 
 #define AW_A10_SDRAM_BASE       0x40000000
 
@@ -32,6 +35,7 @@ typedef struct AwA10State {
     AwA10PITState timer;
     AwA10PICState intc;
     AwEmacState emac;
+    AllwinnerAHCIState sata;
 } AwA10State;
 
 #define ALLWINNER_H_
diff --git a/qemu/include/hw/arm/arm.h b/qemu/include/hw/arm/arm.h
index 4dcd4f9b6..b2517f9a4 100644
--- a/qemu/include/hw/arm/arm.h
+++ b/qemu/include/hw/arm/arm.h
@@ -16,8 +16,15 @@
 #include "qemu/notify.h"
 #include "cpu.h"
 
+typedef enum {
+    ARM_ENDIANNESS_UNKNOWN = 0,
+    ARM_ENDIANNESS_LE,
+    ARM_ENDIANNESS_BE8,
+    ARM_ENDIANNESS_BE32,
+} arm_endianness;
+
 /* armv7m.c */
-qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model);
 
 /*
@@ -87,6 +94,24 @@ struct arm_boot_info {
      * -pflash. It also implies that fw_cfg_find() will succeed.
      */
     bool firmware_loaded;
+
+    /* Address at which board specific loader/setup code exists. If enabled,
+     * this code-blob will run before anything else. It must return to the
+     * caller via the link register. There is no stack set up. Enabled by
+     * defining write_board_setup, which is responsible for loading the blob
+     * to the specified address.
+     */
+    hwaddr board_setup_addr;
+    void (*write_board_setup)(ARMCPU *cpu,
+                              const struct arm_boot_info *info);
+
+    /* If set, the board specific loader/setup blob will be run from secure
+     * mode, regardless of secure_boot. The blob becomes responsible for
+     * changing to non-secure state if implementing a non-secure boot
+     */
+    bool secure_board_setup;
+
+    arm_endianness endianness;
 };
 
 /**
@@ -106,6 +131,11 @@ struct arm_boot_info {
  */
 void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info);
 
+/* Write a secure board setup routine with a dummy handler for SMCs */
+void arm_write_secure_board_setup_dummy_smc(ARMCPU *cpu,
+                                            const struct arm_boot_info *info,
+                                            hwaddr mvbar_addr);
+
 /* Multiplication factor to convert from system clock ticks to qemu timer
    ticks.  */
 extern int system_clock_scale;
diff --git a/qemu/include/hw/arm/ast2400.h b/qemu/include/hw/arm/ast2400.h
new file mode 100644
index 000000000..f16a1ed25
--- /dev/null
+++ b/qemu/include/hw/arm/ast2400.h
@@ -0,0 +1,35 @@
+/*
+ * ASPEED AST2400 SoC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef AST2400_H
+#define AST2400_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/aspeed_vic.h"
+#include "hw/timer/aspeed_timer.h"
+
+typedef struct AST2400State {
+    /*< private >*/
+    DeviceState parent;
+
+    /*< public >*/
+    ARMCPU *cpu;
+    MemoryRegion iomem;
+    AspeedVICState vic;
+    AspeedTimerCtrlState timerctrl;
+} AST2400State;
+
+#define TYPE_AST2400 "ast2400"
+#define AST2400(obj) OBJECT_CHECK(AST2400State, (obj), TYPE_AST2400)
+
+#define AST2400_SDRAM_BASE       0x40000000
+
+#endif /* AST2400_H */
diff --git a/qemu/include/hw/arm/bcm2835_peripherals.h b/qemu/include/hw/arm/bcm2835_peripherals.h
new file mode 100644
index 000000000..e12ae3721
--- /dev/null
+++ b/qemu/include/hw/arm/bcm2835_peripherals.h
@@ -0,0 +1,48 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_PERIPHERALS_H
+#define BCM2835_PERIPHERALS_H
+
+#include "qemu-common.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
+#include "hw/char/bcm2835_aux.h"
+#include "hw/display/bcm2835_fb.h"
+#include "hw/dma/bcm2835_dma.h"
+#include "hw/intc/bcm2835_ic.h"
+#include "hw/misc/bcm2835_property.h"
+#include "hw/misc/bcm2835_mbox.h"
+#include "hw/sd/sdhci.h"
+
+#define TYPE_BCM2835_PERIPHERALS "bcm2835-peripherals"
+#define BCM2835_PERIPHERALS(obj) \
+    OBJECT_CHECK(BCM2835PeripheralState, (obj), TYPE_BCM2835_PERIPHERALS)
+
+typedef struct BCM2835PeripheralState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion peri_mr, peri_mr_alias, gpu_bus_mr, mbox_mr;
+    MemoryRegion ram_alias[4];
+    qemu_irq irq, fiq;
+
+    SysBusDevice *uart0;
+    BCM2835AuxState aux;
+    BCM2835FBState fb;
+    BCM2835DMAState dma;
+    BCM2835ICState ic;
+    BCM2835PropertyState property;
+    BCM2835MboxState mboxes;
+    SDHCIState sdhci;
+} BCM2835PeripheralState;
+
+#endif /* BCM2835_PERIPHERALS_H */
diff --git a/qemu/include/hw/arm/bcm2836.h b/qemu/include/hw/arm/bcm2836.h
new file mode 100644
index 000000000..76de1996a
--- /dev/null
+++ b/qemu/include/hw/arm/bcm2836.h
@@ -0,0 +1,35 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2836_H
+#define BCM2836_H
+
+#include "hw/arm/arm.h"
+#include "hw/arm/bcm2835_peripherals.h"
+#include "hw/intc/bcm2836_control.h"
+
+#define TYPE_BCM2836 "bcm2836"
+#define BCM2836(obj) OBJECT_CHECK(BCM2836State, (obj), TYPE_BCM2836)
+
+#define BCM2836_NCPUS 4
+
+typedef struct BCM2836State {
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
+
+    uint32_t enabled_cpus;
+
+    ARMCPU cpus[BCM2836_NCPUS];
+    BCM2836ControlState control;
+    BCM2835PeripheralState peripherals;
+} BCM2836State;
+
+#endif /* BCM2836_H */
diff --git a/qemu/include/hw/arm/fsl-imx25.h b/qemu/include/hw/arm/fsl-imx25.h
new file mode 100644
index 000000000..d0e8e9d95
--- /dev/null
+++ b/qemu/include/hw/arm/fsl-imx25.h
@@ -0,0 +1,249 @@
+/*
+ * Freescale i.MX25 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX25_H
+#define FSL_IMX25_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx25_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/net/imx_fec.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX25 "fsl,imx25"
+#define FSL_IMX25(obj) OBJECT_CHECK(FslIMX25State, (obj), TYPE_FSL_IMX25)
+
+#define FSL_IMX25_NUM_UARTS 5
+#define FSL_IMX25_NUM_GPTS 4
+#define FSL_IMX25_NUM_EPITS 2
+#define FSL_IMX25_NUM_I2CS 3
+#define FSL_IMX25_NUM_GPIOS 4
+
+typedef struct FslIMX25State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMX25CCMState  ccm;
+    IMXSerialState uart[FSL_IMX25_NUM_UARTS];
+    IMXGPTState    gpt[FSL_IMX25_NUM_GPTS];
+    IMXEPITState   epit[FSL_IMX25_NUM_EPITS];
+    IMXFECState    fec;
+    IMXI2CState    i2c[FSL_IMX25_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX25_NUM_GPIOS];
+    MemoryRegion   rom[2];
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX25State;
+
+/**
+ * i.MX25 memory map
+ ****************************************************************
+ * 0x0000_0000 0x0000_3FFF 16 Kbytes    ROM (36 Kbytes)
+ * 0x0000_4000 0x0040_3FFF 4 Mbytes     Reserved
+ * 0x0040_4000 0x0040_8FFF 20 Kbytes    ROM (36 Kbytes)
+ * 0x0040_9000 0x0FFF_FFFF 252 Mbytes (minus 36 Kbytes) Reserved
+ * 0x1000_0000 0x1FFF_FFFF 256 Mbytes   Reserved
+ * 0x2000_0000 0x2FFF_FFFF 256 Mbytes   Reserved
+ * 0x3000_0000 0x3FFF_FFFF 256 Mbytes   Reserved
+ * 0x4000_0000 0x43EF_FFFF 63 Mbytes    Reserved
+ * 0x43F0_0000 0x43F0_3FFF 16 Kbytes    AIPS A control registers
+ * 0x43F0_4000 0x43F0_7FFF 16 Kbytes    ARM926 platform MAX
+ * 0x43F0_8000 0x43F0_BFFF 16 Kbytes    ARM926 platform CLKCTL
+ * 0x43F0_C000 0x43F0_FFFF 16 Kbytes    ARM926 platform ETB registers
+ * 0x43F1_0000 0x43F1_3FFF 16 Kbytes    ARM926 platform ETB memory
+ * 0x43F1_4000 0x43F1_7FFF 16 Kbytes    ARM926 platform AAPE registers
+ * 0x43F1_8000 0x43F7_FFFF 416 Kbytes   Reserved
+ * 0x43F8_0000 0x43F8_3FFF 16 Kbytes    I2C-1
+ * 0x43F8_4000 0x43F8_7FFF 16 Kbytes    I2C-3
+ * 0x43F8_8000 0x43F8_BFFF 16 Kbytes    CAN-1
+ * 0x43F8_C000 0x43F8_FFFF 16 Kbytes    CAN-2
+ * 0x43F9_0000 0x43F9_3FFF 16 Kbytes    UART-1
+ * 0x43F9_4000 0x43F9_7FFF 16 Kbytes    UART-2
+ * 0x43F9_8000 0x43F9_BFFF 16 Kbytes    I2C-2
+ * 0x43F9_C000 0x43F9_FFFF 16 Kbytes    1-Wire
+ * 0x43FA_0000 0x43FA_3FFF 16 Kbytes    ATA (CPU side)
+ * 0x43FA_4000 0x43FA_7FFF 16 Kbytes    CSPI-1
+ * 0x43FA_8000 0x43FA_BFFF 16 Kbytes    KPP
+ * 0x43FA_C000 0x43FA_FFFF 16 Kbytes    IOMUXC
+ * 0x43FB_0000 0x43FB_3FFF 16 Kbytes    AUDMUX
+ * 0x43FB_4000 0x43FB_7FFF 16 Kbytes    Reserved
+ * 0x43FB_8000 0x43FB_BFFF 16 Kbytes    ECT (IP BUS A)
+ * 0x43FB_C000 0x43FB_FFFF 16 Kbytes    ECT (IP BUS B)
+ * 0x43FC_0000 0x43FF_FFFF 256 Kbytes   Reserved AIPS A off-platform slots
+ * 0x4400_0000 0x4FFF_FFFF 192 Mbytes   Reserved
+ * 0x5000_0000 0x5000_3FFF 16 Kbytes    SPBA base address
+ * 0x5000_4000 0x5000_7FFF 16 Kbytes    CSPI-3
+ * 0x5000_8000 0x5000_BFFF 16 Kbytes    UART-4
+ * 0x5000_C000 0x5000_FFFF 16 Kbytes    UART-3
+ * 0x5001_0000 0x5001_3FFF 16 Kbytes    CSPI-2
+ * 0x5001_4000 0x5001_7FFF 16 Kbytes    SSI-2
+ * 0x5001_C000 0x5001_FFFF 16 Kbytes    Reserved
+ * 0x5002_0000 0x5002_3FFF 16 Kbytes    ATA
+ * 0x5002_4000 0x5002_7FFF 16 Kbytes    SIM-1
+ * 0x5002_8000 0x5002_BFFF 16 Kbytes    SIM-2
+ * 0x5002_C000 0x5002_FFFF 16 Kbytes    UART-5
+ * 0x5003_0000 0x5003_3FFF 16 Kbytes    TSC
+ * 0x5003_4000 0x5003_7FFF 16 Kbytes    SSI-1
+ * 0x5003_8000 0x5003_BFFF 16 Kbytes    FEC
+ * 0x5003_C000 0x5003_FFFF 16 Kbytes    SPBA registers
+ * 0x5004_0000 0x51FF_FFFF 32 Mbytes (minus 256 Kbytes)
+ * 0x5200_0000 0x53EF_FFFF 31 Mbytes    Reserved
+ * 0x53F0_0000 0x53F0_3FFF 16 Kbytes    AIPS B control registers
+ * 0x53F0_4000 0x53F7_FFFF 496 Kbytes   Reserved
+ * 0x53F8_0000 0x53F8_3FFF 16 Kbytes    CCM
+ * 0x53F8_4000 0x53F8_7FFF 16 Kbytes    GPT-4
+ * 0x53F8_8000 0x53F8_BFFF 16 Kbytes    GPT-3
+ * 0x53F8_C000 0x53F8_FFFF 16 Kbytes    GPT-2
+ * 0x53F9_0000 0x53F9_3FFF 16 Kbytes    GPT-1
+ * 0x53F9_4000 0x53F9_7FFF 16 Kbytes    EPIT-1
+ * 0x53F9_8000 0x53F9_BFFF 16 Kbytes    EPIT-2
+ * 0x53F9_C000 0x53F9_FFFF 16 Kbytes    GPIO-4
+ * 0x53FA_0000 0x53FA_3FFF 16 Kbytes    PWM-2
+ * 0x53FA_4000 0x53FA_7FFF 16 Kbytes    GPIO-3
+ * 0x53FA_8000 0x53FA_BFFF 16 Kbytes    PWM-3
+ * 0x53FA_C000 0x53FA_FFFF 16 Kbytes    SCC
+ * 0x53FB_0000 0x53FB_3FFF 16 Kbytes    RNGB
+ * 0x53FB_4000 0x53FB_7FFF 16 Kbytes    eSDHC-1
+ * 0x53FB_8000 0x53FB_BFFF 16 Kbytes    eSDHC-2
+ * 0x53FB_C000 0x53FB_FFFF 16 Kbytes    LCDC
+ * 0x53FC_0000 0x53FC_3FFF 16 Kbytes    SLCDC
+ * 0x53FC_4000 0x53FC_7FFF 16 Kbytes    Reserved
+ * 0x53FC_8000 0x53FC_BFFF 16 Kbytes    PWM-4
+ * 0x53FC_C000 0x53FC_FFFF 16 Kbytes    GPIO-1
+ * 0x53FD_0000 0x53FD_3FFF 16 Kbytes    GPIO-2
+ * 0x53FD_4000 0x53FD_7FFF 16 Kbytes    SDMA
+ * 0x53FD_8000 0x53FD_BFFF 16 Kbytes    Reserved
+ * 0x53FD_C000 0x53FD_FFFF 16 Kbytes    WDOG
+ * 0x53FE_0000 0x53FE_3FFF 16 Kbytes    PWM-1
+ * 0x53FE_4000 0x53FE_7FFF 16 Kbytes    Reserved
+ * 0x53FE_8000 0x53FE_BFFF 16 Kbytes    Reserved
+ * 0x53FE_C000 0x53FE_FFFF 16 Kbytes    RTICv3
+ * 0x53FF_0000 0x53FF_3FFF 16 Kbytes    IIM
+ * 0x53FF_4000 0x53FF_7FFF 16 Kbytes    USB
+ * 0x53FF_8000 0x53FF_BFFF 16 Kbytes    CSI
+ * 0x53FF_C000 0x53FF_FFFF 16 Kbytes    DryIce
+ * 0x5400_0000 0x5FFF_FFFF 192 Mbytes   Reserved (aliased AIPS B slots)
+ * 0x6000_0000 0x67FF_FFFF 128 Mbytes   ARM926 platform ROMPATCH
+ * 0x6800_0000 0x6FFF_FFFF 128 Mbytes   ARM926 platform ASIC
+ * 0x7000_0000 0x77FF_FFFF 128 Mbytes   Reserved
+ * 0x7800_0000 0x7801_FFFF 128 Kbytes   RAM
+ * 0x7802_0000 0x7FFF_FFFF 128 Mbytes (minus 128 Kbytes)
+ * 0x8000_0000 0x8FFF_FFFF 256 Mbytes   SDRAM bank 0
+ * 0x9000_0000 0x9FFF_FFFF 256 Mbytes   SDRAM bank 1
+ * 0xA000_0000 0xA7FF_FFFF 128 Mbytes   WEIM CS0 (flash 128) 1
+ * 0xA800_0000 0xAFFF_FFFF 128 Mbytes   WEIM CS1 (flash 64) 1
+ * 0xB000_0000 0xB1FF_FFFF 32 Mbytes    WEIM CS2 (SRAM)
+ * 0xB200_0000 0xB3FF_FFFF 32 Mbytes    WEIM CS3 (SRAM)
+ * 0xB400_0000 0xB5FF_FFFF 32 Mbytes    WEIM CS4
+ * 0xB600_0000 0xB7FF_FFFF 32 Mbytes    Reserved
+ * 0xB800_0000 0xB800_0FFF 4 Kbytes     Reserved
+ * 0xB800_1000 0xB800_1FFF 4 Kbytes     SDRAM control registers
+ * 0xB800_2000 0xB800_2FFF 4 Kbytes     WEIM control registers
+ * 0xB800_3000 0xB800_3FFF 4 Kbytes     M3IF control registers
+ * 0xB800_4000 0xB800_4FFF 4 Kbytes     EMI control registers
+ * 0xB800_5000 0xBAFF_FFFF 32 Mbytes (minus 20 Kbytes)
+ * 0xBB00_0000 0xBB00_0FFF 4 Kbytes     NAND flash main area buffer
+ * 0xBB00_1000 0xBB00_11FF 512 B        NAND flash spare area buffer
+ * 0xBB00_1200 0xBB00_1DFF 3 Kbytes     Reserved
+ * 0xBB00_1E00 0xBB00_1FFF 512 B        NAND flash control regisers
+ * 0xBB01_2000 0xBFFF_FFFF 96 Mbytes (minus 8 Kbytes) Reserved
+ * 0xC000_0000 0xFFFF_FFFF 1024 Mbytes  Reserved
+ */
+
+#define FSL_IMX25_ROM0_ADDR     0x00000000
+#define FSL_IMX25_ROM0_SIZE     0x4000
+#define FSL_IMX25_ROM1_ADDR     0x00404000
+#define FSL_IMX25_ROM1_SIZE     0x4000
+#define FSL_IMX25_I2C1_ADDR     0x43F80000
+#define FSL_IMX25_I2C1_SIZE     0x4000
+#define FSL_IMX25_I2C3_ADDR     0x43F84000
+#define FSL_IMX25_I2C3_SIZE     0x4000
+#define FSL_IMX25_UART1_ADDR    0x43F90000
+#define FSL_IMX25_UART1_SIZE    0x4000
+#define FSL_IMX25_UART2_ADDR    0x43F94000
+#define FSL_IMX25_UART2_SIZE    0x4000
+#define FSL_IMX25_I2C2_ADDR     0x43F98000
+#define FSL_IMX25_I2C2_SIZE     0x4000
+#define FSL_IMX25_UART4_ADDR    0x50008000
+#define FSL_IMX25_UART4_SIZE    0x4000
+#define FSL_IMX25_UART3_ADDR    0x5000C000
+#define FSL_IMX25_UART3_SIZE    0x4000
+#define FSL_IMX25_UART5_ADDR    0x5002C000
+#define FSL_IMX25_UART5_SIZE    0x4000
+#define FSL_IMX25_FEC_ADDR      0x50038000
+#define FSL_IMX25_FEC_SIZE      0x4000
+#define FSL_IMX25_CCM_ADDR      0x53F80000
+#define FSL_IMX25_CCM_SIZE      0x4000
+#define FSL_IMX25_GPT4_ADDR     0x53F84000
+#define FSL_IMX25_GPT4_SIZE     0x4000
+#define FSL_IMX25_GPT3_ADDR     0x53F88000
+#define FSL_IMX25_GPT3_SIZE     0x4000
+#define FSL_IMX25_GPT2_ADDR     0x53F8C000
+#define FSL_IMX25_GPT2_SIZE     0x4000
+#define FSL_IMX25_GPT1_ADDR     0x53F90000
+#define FSL_IMX25_GPT1_SIZE     0x4000
+#define FSL_IMX25_EPIT1_ADDR    0x53F94000
+#define FSL_IMX25_EPIT1_SIZE    0x4000
+#define FSL_IMX25_EPIT2_ADDR    0x53F98000
+#define FSL_IMX25_EPIT2_SIZE    0x4000
+#define FSL_IMX25_GPIO4_ADDR    0x53F9C000
+#define FSL_IMX25_GPIO4_SIZE    0x4000
+#define FSL_IMX25_GPIO3_ADDR    0x53FA4000
+#define FSL_IMX25_GPIO3_SIZE    0x4000
+#define FSL_IMX25_GPIO1_ADDR    0x53FCC000
+#define FSL_IMX25_GPIO1_SIZE    0x4000
+#define FSL_IMX25_GPIO2_ADDR    0x53FD0000
+#define FSL_IMX25_GPIO2_SIZE    0x4000
+#define FSL_IMX25_AVIC_ADDR     0x68000000
+#define FSL_IMX25_AVIC_SIZE     0x4000
+#define FSL_IMX25_IRAM_ADDR     0x78000000
+#define FSL_IMX25_IRAM_SIZE     0x20000
+#define FSL_IMX25_IRAM_ALIAS_ADDR     0x78020000
+#define FSL_IMX25_IRAM_ALIAS_SIZE     0x7FE0000
+#define FSL_IMX25_SDRAM0_ADDR   0x80000000
+#define FSL_IMX25_SDRAM0_SIZE   0x10000000
+#define FSL_IMX25_SDRAM1_ADDR   0x90000000
+#define FSL_IMX25_SDRAM1_SIZE   0x10000000
+
+#define FSL_IMX25_UART1_IRQ     45
+#define FSL_IMX25_UART2_IRQ     32
+#define FSL_IMX25_UART3_IRQ     18
+#define FSL_IMX25_UART4_IRQ     5
+#define FSL_IMX25_UART5_IRQ     40
+#define FSL_IMX25_GPT1_IRQ      54
+#define FSL_IMX25_GPT2_IRQ      53
+#define FSL_IMX25_GPT3_IRQ      29
+#define FSL_IMX25_GPT4_IRQ      1
+#define FSL_IMX25_EPIT1_IRQ     28
+#define FSL_IMX25_EPIT2_IRQ     27
+#define FSL_IMX25_FEC_IRQ       57
+#define FSL_IMX25_I2C1_IRQ      3
+#define FSL_IMX25_I2C2_IRQ      4
+#define FSL_IMX25_I2C3_IRQ      10
+#define FSL_IMX25_GPIO1_IRQ     52
+#define FSL_IMX25_GPIO2_IRQ     51
+#define FSL_IMX25_GPIO3_IRQ     16
+#define FSL_IMX25_GPIO4_IRQ     23
+
+#endif /* FSL_IMX25_H */
diff --git a/qemu/include/hw/arm/fsl-imx31.h b/qemu/include/hw/arm/fsl-imx31.h
new file mode 100644
index 000000000..d408abbba
--- /dev/null
+++ b/qemu/include/hw/arm/fsl-imx31.h
@@ -0,0 +1,122 @@
+/*
+ * Freescale i.MX31 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX31_H
+#define FSL_IMX31_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx31_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX31 "fsl,imx31"
+#define FSL_IMX31(obj) OBJECT_CHECK(FslIMX31State, (obj), TYPE_FSL_IMX31)
+
+#define FSL_IMX31_NUM_UARTS 2
+#define FSL_IMX31_NUM_EPITS 2
+#define FSL_IMX31_NUM_I2CS 3
+#define FSL_IMX31_NUM_GPIOS 3
+
+typedef struct FslIMX31State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMX31CCMState  ccm;
+    IMXSerialState uart[FSL_IMX31_NUM_UARTS];
+    IMXGPTState    gpt;
+    IMXEPITState   epit[FSL_IMX31_NUM_EPITS];
+    IMXI2CState    i2c[FSL_IMX31_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX31_NUM_GPIOS];
+    MemoryRegion   secure_rom;
+    MemoryRegion   rom;
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX31State;
+
+#define FSL_IMX31_SECURE_ROM_ADDR       0x00000000
+#define FSL_IMX31_SECURE_ROM_SIZE       0x4000
+#define FSL_IMX31_ROM_ADDR              0x00404000
+#define FSL_IMX31_ROM_SIZE              0x4000
+#define FSL_IMX31_IRAM_ALIAS_ADDR       0x10000000
+#define FSL_IMX31_IRAM_ALIAS_SIZE       0xFFC0000
+#define FSL_IMX31_IRAM_ADDR             0x1FFFC000
+#define FSL_IMX31_IRAM_SIZE             0x4000
+#define FSL_IMX31_I2C1_ADDR             0x43F80000
+#define FSL_IMX31_I2C1_SIZE             0x4000
+#define FSL_IMX31_I2C3_ADDR             0x43F84000
+#define FSL_IMX31_I2C3_SIZE             0x4000
+#define FSL_IMX31_UART1_ADDR            0x43F90000
+#define FSL_IMX31_UART1_SIZE            0x4000
+#define FSL_IMX31_UART2_ADDR            0x43F94000
+#define FSL_IMX31_UART2_SIZE            0x4000
+#define FSL_IMX31_I2C2_ADDR             0x43F98000
+#define FSL_IMX31_I2C2_SIZE             0x4000
+#define FSL_IMX31_CCM_ADDR              0x53F80000
+#define FSL_IMX31_CCM_SIZE              0x4000
+#define FSL_IMX31_GPT_ADDR              0x53F90000
+#define FSL_IMX31_GPT_SIZE              0x4000
+#define FSL_IMX31_EPIT1_ADDR            0x53F94000
+#define FSL_IMX31_EPIT1_SIZE            0x4000
+#define FSL_IMX31_EPIT2_ADDR            0x53F98000
+#define FSL_IMX31_EPIT2_SIZE            0x4000
+#define FSL_IMX31_GPIO3_ADDR            0x53FA4000
+#define FSL_IMX31_GPIO3_SIZE            0x4000
+#define FSL_IMX31_GPIO1_ADDR            0x53FCC000
+#define FSL_IMX31_GPIO1_SIZE            0x4000
+#define FSL_IMX31_GPIO2_ADDR            0x53FD0000
+#define FSL_IMX31_GPIO2_SIZE            0x4000
+#define FSL_IMX31_AVIC_ADDR             0x68000000
+#define FSL_IMX31_AVIC_SIZE             0x100
+#define FSL_IMX31_SDRAM0_ADDR           0x80000000
+#define FSL_IMX31_SDRAM0_SIZE           0x10000000
+#define FSL_IMX31_SDRAM1_ADDR           0x90000000
+#define FSL_IMX31_SDRAM1_SIZE           0x10000000
+#define FSL_IMX31_FLASH0_ADDR           0xA0000000
+#define FSL_IMX31_FLASH0_SIZE           0x8000000
+#define FSL_IMX31_FLASH1_ADDR           0xA8000000
+#define FSL_IMX31_FLASH1_SIZE           0x8000000
+#define FSL_IMX31_CS2_ADDR              0xB0000000
+#define FSL_IMX31_CS2_SIZE              0x2000000
+#define FSL_IMX31_CS3_ADDR              0xB2000000
+#define FSL_IMX31_CS3_SIZE              0x2000000
+#define FSL_IMX31_CS4_ADDR              0xB4000000
+#define FSL_IMX31_CS4_SIZE              0x2000000
+#define FSL_IMX31_CS5_ADDR              0xB6000000
+#define FSL_IMX31_CS5_SIZE              0x2000000
+#define FSL_IMX31_NAND_ADDR             0xB8000000
+#define FSL_IMX31_NAND_SIZE             0x1000
+
+#define FSL_IMX31_EPIT2_IRQ             27
+#define FSL_IMX31_EPIT1_IRQ             28
+#define FSL_IMX31_GPT_IRQ               29
+#define FSL_IMX31_UART2_IRQ             32
+#define FSL_IMX31_UART1_IRQ             45
+#define FSL_IMX31_I2C1_IRQ              10
+#define FSL_IMX31_I2C2_IRQ              4
+#define FSL_IMX31_I2C3_IRQ              3
+#define FSL_IMX31_GPIO1_IRQ             52
+#define FSL_IMX31_GPIO2_IRQ             51
+#define FSL_IMX31_GPIO3_IRQ             56
+
+#endif /* FSL_IMX31_H */
diff --git a/qemu/include/hw/arm/imx.h b/qemu/include/hw/arm/imx.h
deleted file mode 100644
index ea9e09327..000000000
--- a/qemu/include/hw/arm/imx.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * i.MX31 emulation
- *
- * Copyright (C) 2012 Peter Chubb
- * NICTA
- *
- * This code is released under the GPL, version 2.0 or later
- * See the file `../COPYING' for details.
- */
-
-#ifndef IMX_H
-#define IMX_H
-
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq);
-
-typedef enum  {
-    NOCLK,
-    MCU,
-    HSP,
-    IPG,
-    CLK_32k
-} IMXClk;
-
-uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
-
-void imx_timerp_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-void imx_timerg_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-
-
-#endif /* IMX_H */
diff --git a/qemu/include/hw/arm/linux-boot-if.h b/qemu/include/hw/arm/linux-boot-if.h
new file mode 100644
index 000000000..aba4479a1
--- /dev/null
+++ b/qemu/include/hw/arm/linux-boot-if.h
@@ -0,0 +1,43 @@
+/*
+ * hw/arm/linux-boot-if.h : interface for devices which need to behave
+ * specially for direct boot of an ARM Linux kernel
+ */
+
+#ifndef HW_ARM_LINUX_BOOT_IF_H
+#define HW_ARM_LINUX_BOOT_IF_H
+
+#include "qom/object.h"
+
+#define TYPE_ARM_LINUX_BOOT_IF "arm-linux-boot-if"
+#define ARM_LINUX_BOOT_IF_CLASS(klass) \
+    OBJECT_CLASS_CHECK(ARMLinuxBootIfClass, (klass), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ARMLinuxBootIfClass, (obj), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF(obj) \
+    INTERFACE_CHECK(ARMLinuxBootIf, (obj), TYPE_ARM_LINUX_BOOT_IF)
+
+typedef struct ARMLinuxBootIf {
+    /*< private >*/
+    Object parent_obj;
+} ARMLinuxBootIf;
+
+typedef struct ARMLinuxBootIfClass {
+    /*< private >*/
+    InterfaceClass parent_class;
+
+    /*< public >*/
+    /** arm_linux_init: configure the device for a direct boot
+     * of an ARM Linux kernel (so that device reset puts it into
+     * the state the kernel expects after firmware initialization,
+     * rather than the true hardware reset state). This callback is
+     * called once after machine construction is complete (before the
+     * first system reset).
+     *
+     * @obj: the object implementing this interface
+     * @secure_boot: true if we are booting Secure, false for NonSecure
+     * (or for a CPU which doesn't support TrustZone)
+     */
+    void (*arm_linux_init)(ARMLinuxBootIf *obj, bool secure_boot);
+} ARMLinuxBootIfClass;
+
+#endif
diff --git a/qemu/include/hw/arm/raspi_platform.h b/qemu/include/hw/arm/raspi_platform.h
new file mode 100644
index 000000000..6467e88ae
--- /dev/null
+++ b/qemu/include/hw/arm/raspi_platform.h
@@ -0,0 +1,128 @@
+/*
+ * bcm2708 aka bcm2835/2836 aka Raspberry Pi/Pi2 SoC platform defines
+ *
+ * These definitions are derived from those in Raspbian Linux at
+ * arch/arm/mach-{bcm2708,bcm2709}/include/mach/platform.h
+ * where they carry the following notice:
+ *
+ * Copyright (C) 2010 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#define MCORE_OFFSET            0x0000   /* Fake frame buffer device
+                                          * (the multicore sync block) */
+#define IC0_OFFSET              0x2000
+#define ST_OFFSET               0x3000   /* System Timer */
+#define MPHI_OFFSET             0x6000   /* Message-based Parallel Host Intf. */
+#define DMA_OFFSET              0x7000   /* DMA controller, channels 0-14 */
+#define ARM_OFFSET              0xB000   /* BCM2708 ARM control block */
+#define ARMCTRL_OFFSET          (ARM_OFFSET + 0x000)
+#define ARMCTRL_IC_OFFSET       (ARM_OFFSET + 0x200) /* Interrupt controller */
+#define ARMCTRL_TIMER0_1_OFFSET (ARM_OFFSET + 0x400) /* Timer 0 and 1 */
+#define ARMCTRL_0_SBM_OFFSET    (ARM_OFFSET + 0x800) /* User 0 (ARM) Semaphores
+                                                      * Doorbells & Mailboxes */
+#define PM_OFFSET               0x100000 /* Power Management, Reset controller
+                                          * and Watchdog registers */
+#define PCM_CLOCK_OFFSET        0x101098
+#define RNG_OFFSET              0x104000
+#define GPIO_OFFSET             0x200000
+#define UART0_OFFSET            0x201000
+#define MMCI0_OFFSET            0x202000
+#define I2S_OFFSET              0x203000
+#define SPI0_OFFSET             0x204000
+#define BSC0_OFFSET             0x205000 /* BSC0 I2C/TWI */
+#define UART1_OFFSET            0x215000
+#define EMMC_OFFSET             0x300000
+#define SMI_OFFSET              0x600000
+#define BSC1_OFFSET             0x804000 /* BSC1 I2C/TWI */
+#define USB_OFFSET              0x980000 /* DTC_OTG USB controller */
+#define DMA15_OFFSET            0xE05000 /* DMA controller, channel 15 */
+
+/* GPU interrupts */
+#define INTERRUPT_TIMER0               0
+#define INTERRUPT_TIMER1               1
+#define INTERRUPT_TIMER2               2
+#define INTERRUPT_TIMER3               3
+#define INTERRUPT_CODEC0               4
+#define INTERRUPT_CODEC1               5
+#define INTERRUPT_CODEC2               6
+#define INTERRUPT_JPEG                 7
+#define INTERRUPT_ISP                  8
+#define INTERRUPT_USB                  9
+#define INTERRUPT_3D                   10
+#define INTERRUPT_TRANSPOSER           11
+#define INTERRUPT_MULTICORESYNC0       12
+#define INTERRUPT_MULTICORESYNC1       13
+#define INTERRUPT_MULTICORESYNC2       14
+#define INTERRUPT_MULTICORESYNC3       15
+#define INTERRUPT_DMA0                 16
+#define INTERRUPT_DMA1                 17
+#define INTERRUPT_DMA2                 18
+#define INTERRUPT_DMA3                 19
+#define INTERRUPT_DMA4                 20
+#define INTERRUPT_DMA5                 21
+#define INTERRUPT_DMA6                 22
+#define INTERRUPT_DMA7                 23
+#define INTERRUPT_DMA8                 24
+#define INTERRUPT_DMA9                 25
+#define INTERRUPT_DMA10                26
+#define INTERRUPT_DMA11                27
+#define INTERRUPT_DMA12                28
+#define INTERRUPT_AUX                  29
+#define INTERRUPT_ARM                  30
+#define INTERRUPT_VPUDMA               31
+#define INTERRUPT_HOSTPORT             32
+#define INTERRUPT_VIDEOSCALER          33
+#define INTERRUPT_CCP2TX               34
+#define INTERRUPT_SDC                  35
+#define INTERRUPT_DSI0                 36
+#define INTERRUPT_AVE                  37
+#define INTERRUPT_CAM0                 38
+#define INTERRUPT_CAM1                 39
+#define INTERRUPT_HDMI0                40
+#define INTERRUPT_HDMI1                41
+#define INTERRUPT_PIXELVALVE1          42
+#define INTERRUPT_I2CSPISLV            43
+#define INTERRUPT_DSI1                 44
+#define INTERRUPT_PWA0                 45
+#define INTERRUPT_PWA1                 46
+#define INTERRUPT_CPR                  47
+#define INTERRUPT_SMI                  48
+#define INTERRUPT_GPIO0                49
+#define INTERRUPT_GPIO1                50
+#define INTERRUPT_GPIO2                51
+#define INTERRUPT_GPIO3                52
+#define INTERRUPT_I2C                  53
+#define INTERRUPT_SPI                  54
+#define INTERRUPT_I2SPCM               55
+#define INTERRUPT_SDIO                 56
+#define INTERRUPT_UART                 57
+#define INTERRUPT_SLIMBUS              58
+#define INTERRUPT_VEC                  59
+#define INTERRUPT_CPG                  60
+#define INTERRUPT_RNG                  61
+#define INTERRUPT_ARASANSDIO           62
+#define INTERRUPT_AVSPMON              63
+
+/* ARM CPU IRQs use a private number space */
+#define INTERRUPT_ARM_TIMER            0
+#define INTERRUPT_ARM_MAILBOX          1
+#define INTERRUPT_ARM_DOORBELL_0       2
+#define INTERRUPT_ARM_DOORBELL_1       3
+#define INTERRUPT_VPU0_HALTED          4
+#define INTERRUPT_VPU1_HALTED          5
+#define INTERRUPT_ILLEGAL_TYPE0        6
+#define INTERRUPT_ILLEGAL_TYPE1        7
diff --git a/qemu/include/hw/arm/virt-acpi-build.h b/qemu/include/hw/arm/virt-acpi-build.h
index 04f174d52..7d3700ebf 100644
--- a/qemu/include/hw/arm/virt-acpi-build.h
+++ b/qemu/include/hw/arm/virt-acpi-build.h
@@ -23,7 +23,6 @@
 #include "qemu-common.h"
 #include "hw/arm/virt.h"
 
-#define VIRT_ACPI_CPU_ID_LIMIT 8
 #define ACPI_GICC_ENABLED 1
 
 typedef struct VirtGuestInfo {
@@ -31,6 +30,8 @@ typedef struct VirtGuestInfo {
     FWCfgState *fw_cfg;
     const MemMapEntry *memmap;
     const int *irqmap;
+    bool use_highmem;
+    int gic_version;
 } VirtGuestInfo;
 
 
diff --git a/qemu/include/hw/arm/virt.h b/qemu/include/hw/arm/virt.h
index d22fd8e50..ecd858960 100644
--- a/qemu/include/hw/arm/virt.h
+++ b/qemu/include/hw/arm/virt.h
@@ -46,6 +46,9 @@ enum {
     VIRT_CPUPERIPHS,
     VIRT_GIC_DIST,
     VIRT_GIC_CPU,
+    VIRT_GIC_V2M,
+    VIRT_GIC_ITS,
+    VIRT_GIC_REDIST,
     VIRT_UART,
     VIRT_MMIO,
     VIRT_RTC,
@@ -54,8 +57,11 @@ enum {
     VIRT_PCIE_MMIO,
     VIRT_PCIE_PIO,
     VIRT_PCIE_ECAM,
-    VIRT_GIC_V2M,
     VIRT_PLATFORM_BUS,
+    VIRT_PCIE_MMIO_HIGH,
+    VIRT_GPIO,
+    VIRT_SECURE_UART,
+    VIRT_SECURE_MEM,
 };
 
 typedef struct MemMapEntry {
diff --git a/qemu/include/hw/arm/xlnx-zynqmp.h b/qemu/include/hw/arm/xlnx-zynqmp.h
index c379632f2..2332596b4 100644
--- a/qemu/include/hw/arm/xlnx-zynqmp.h
+++ b/qemu/include/hw/arm/xlnx-zynqmp.h
@@ -22,6 +22,10 @@
 #include "hw/intc/arm_gic.h"
 #include "hw/net/cadence_gem.h"
 #include "hw/char/cadence_uart.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
+#include "hw/sd/sdhci.h"
+#include "hw/ssi/xilinx_spips.h"
 
 #define TYPE_XLNX_ZYNQMP "xlnx,zynqmp"
 #define XLNX_ZYNQMP(obj) OBJECT_CHECK(XlnxZynqMPState, (obj), \
@@ -31,6 +35,12 @@
 #define XLNX_ZYNQMP_NUM_RPU_CPUS 2
 #define XLNX_ZYNQMP_NUM_GEMS 4
 #define XLNX_ZYNQMP_NUM_UARTS 2
+#define XLNX_ZYNQMP_NUM_SDHCI 2
+#define XLNX_ZYNQMP_NUM_SPIS 2
+
+#define XLNX_ZYNQMP_NUM_OCM_BANKS 4
+#define XLNX_ZYNQMP_OCM_RAM_0_ADDRESS 0xFFFC0000
+#define XLNX_ZYNQMP_OCM_RAM_SIZE 0x10000
 
 #define XLNX_ZYNQMP_GIC_REGIONS 2
 
@@ -40,9 +50,17 @@
  * number of memory region aliases.
  */
 
-#define XLNX_ZYNQMP_GIC_REGION_SIZE 0x4000
+#define XLNX_ZYNQMP_GIC_REGION_SIZE 0x1000
 #define XLNX_ZYNQMP_GIC_ALIASES     (0x10000 / XLNX_ZYNQMP_GIC_REGION_SIZE - 1)
 
+#define XLNX_ZYNQMP_MAX_LOW_RAM_SIZE    0x80000000ull
+
+#define XLNX_ZYNQMP_MAX_HIGH_RAM_SIZE   0x800000000ull
+#define XLNX_ZYNQMP_HIGH_RAM_START      0x800000000ull
+
+#define XLNX_ZYNQMP_MAX_RAM_SIZE (XLNX_ZYNQMP_MAX_LOW_RAM_SIZE + \
+                                  XLNX_ZYNQMP_MAX_HIGH_RAM_SIZE)
+
 typedef struct XlnxZynqMPState {
     /*< private >*/
     DeviceState parent_obj;
@@ -52,8 +70,17 @@ typedef struct XlnxZynqMPState {
     ARMCPU rpu_cpu[XLNX_ZYNQMP_NUM_RPU_CPUS];
     GICState gic;
     MemoryRegion gic_mr[XLNX_ZYNQMP_GIC_REGIONS][XLNX_ZYNQMP_GIC_ALIASES];
+
+    MemoryRegion ocm_ram[XLNX_ZYNQMP_NUM_OCM_BANKS];
+
+    MemoryRegion *ddr_ram;
+    MemoryRegion ddr_ram_low, ddr_ram_high;
+
     CadenceGEMState gem[XLNX_ZYNQMP_NUM_GEMS];
     CadenceUARTState uart[XLNX_ZYNQMP_NUM_UARTS];
+    SysbusAHCIState sata;
+    SDHCIState sdhci[XLNX_ZYNQMP_NUM_SDHCI];
+    XilinxSPIPS spi[XLNX_ZYNQMP_NUM_SPIS];
 
     char *boot_cpu;
     ARMCPU *boot_cpu_ptr;
diff --git a/qemu/include/hw/block/block.h b/qemu/include/hw/block/block.h
index 8d7c4b413..984660efd 100644
--- a/qemu/include/hw/block/block.h
+++ b/qemu/include/hw/block/block.h
@@ -12,7 +12,6 @@
 #define HW_BLOCK_COMMON_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 /* Configuration */
 
diff --git a/qemu/include/hw/block/fdc.h b/qemu/include/hw/block/fdc.h
index d48b2f860..1749dabf2 100644
--- a/qemu/include/hw/block/fdc.h
+++ b/qemu/include/hw/block/fdc.h
@@ -6,13 +6,6 @@
 /* fdc.c */
 #define MAX_FD 2
 
-typedef enum FDriveType {
-    FDRIVE_DRV_144  = 0x00,   /* 1.44 MB 3"5 drive      */
-    FDRIVE_DRV_288  = 0x01,   /* 2.88 MB 3"5 drive      */
-    FDRIVE_DRV_120  = 0x02,   /* 1.2  MB 5"25 drive     */
-    FDRIVE_DRV_NONE = 0x03,   /* No drive connected     */
-} FDriveType;
-
 #define TYPE_ISA_FDC "isa-fdc"
 
 ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds);
@@ -21,6 +14,8 @@ void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
 void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc);
 
-FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
+FloppyDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
+void isa_fdc_get_drive_max_chs(FloppyDriveType type,
+                               uint8_t *maxc, uint8_t *maxh, uint8_t *maxs);
 
 #endif
diff --git a/qemu/include/hw/boards.h b/qemu/include/hw/boards.h
index 2aec9cbb1..8d4fe56b5 100644
--- a/qemu/include/hw/boards.h
+++ b/qemu/include/hw/boards.h
@@ -3,43 +3,23 @@
 #ifndef HW_BOARDS_H
 #define HW_BOARDS_H
 
-#include "qemu/typedefs.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/accel.h"
 #include "hw/qdev.h"
 #include "qom/object.h"
-
-
-typedef void QEMUMachineInitFunc(MachineState *ms);
-
-typedef void QEMUMachineResetFunc(void);
-
-typedef void QEMUMachineHotAddCPUFunc(const int64_t id, Error **errp);
-
-typedef int QEMUMachineGetKvmtypeFunc(const char *arg);
-
-struct QEMUMachine {
-    const char *name;
-    const char *desc;
-    QEMUMachineInitFunc *init;
-    QEMUMachineGetKvmtypeFunc *kvm_type;
-    BlockInterfaceType block_default_type;
-    int max_cpus;
-    unsigned int
-        no_sdcard:1,
-        has_dynamic_sysbus:1;
-    int is_default;
-    const char *default_machine_opts;
-    const char *default_boot_order;
-};
+#include "qom/cpu.h"
 
 void memory_region_allocate_system_memory(MemoryRegion *mr, Object *owner,
                                           const char *name,
                                           uint64_t ram_size);
 
-int qemu_register_machine(QEMUMachine *m);
-
 #define TYPE_MACHINE_SUFFIX "-machine"
+
+/* Machine class name that needs to be used for class-name-based machine
+ * type lookup to work.
+ */
+#define MACHINE_TYPE_NAME(machinename) (machinename TYPE_MACHINE_SUFFIX)
+
 #define TYPE_MACHINE "machine"
 #undef MACHINE  /* BSD defines it and QEMU does not use it */
 #define MACHINE(obj) \
@@ -53,17 +33,36 @@ MachineClass *find_default_machine(void);
 extern MachineState *current_machine;
 
 bool machine_usb(MachineState *machine);
-bool machine_iommu(MachineState *machine);
 bool machine_kernel_irqchip_allowed(MachineState *machine);
 bool machine_kernel_irqchip_required(MachineState *machine);
+bool machine_kernel_irqchip_split(MachineState *machine);
 int machine_kvm_shadow_mem(MachineState *machine);
 int machine_phandle_start(MachineState *machine);
 bool machine_dump_guest_core(MachineState *machine);
 bool machine_mem_merge(MachineState *machine);
 
 /**
+ * CPUArchId:
+ * @arch_id - architecture-dependent CPU ID of present or possible CPU
+ * @cpu - pointer to corresponding CPU object if it's present on NULL otherwise
+ */
+typedef struct {
+    uint64_t arch_id;
+    struct CPUState *cpu;
+} CPUArchId;
+
+/**
+ * CPUArchIdList:
+ * @len - number of @CPUArchId items in @cpus array
+ * @cpus - array of present or possible CPUs for current machine configuration
+ */
+typedef struct {
+    int len;
+    CPUArchId cpus[0];
+} CPUArchIdList;
+
+/**
  * MachineClass:
- * @qemu_machine: #QEMUMachine
  * @get_hotplug_handler: this function is called during bus-less
  *    device hotplug. If defined it returns pointer to an instance
  *    of HotplugHandler object, which handles hotplug operation
@@ -73,6 +72,15 @@ bool machine_mem_merge(MachineState *machine);
  *    used to provide @cpu_index to socket number mapping, allowing
  *    a machine to group CPU threads belonging to the same socket/package
  *    Returns: socket number given cpu_index belongs to.
+ * @hw_version:
+ *    Value of QEMU_VERSION when the machine was added to QEMU.
+ *    Set only by old machines because they need to keep
+ *    compatibility on code that exposed QEMU_VERSION to guests in
+ *    the past (and now use qemu_hw_version()).
+ * @possible_cpu_arch_ids:
+ *    Returns an array of @CPUArchId architecture-dependent CPU IDs
+ *    which includes CPU IDs for present and possible to hotplug CPUs.
+ *    Caller is responsible for freeing returned list.
  */
 struct MachineClass {
     /*< private >*/
@@ -100,7 +108,8 @@ struct MachineClass {
         no_cdrom:1,
         no_sdcard:1,
         has_dynamic_sysbus:1,
-        no_tco:1;
+        pci_allow_0_address:1,
+        legacy_fw_cfg_order:1;
     int is_default;
     const char *default_machine_opts;
     const char *default_boot_order;
@@ -108,10 +117,13 @@ struct MachineClass {
     GlobalProperty *compat_props;
     const char *hw_version;
     ram_addr_t default_ram_size;
+    bool option_rom_has_mr;
+    bool rom_file_has_mr;
 
     HotplugHandler *(*get_hotplug_handler)(MachineState *machine,
                                            DeviceState *dev);
     unsigned (*cpu_index_to_socket_id)(unsigned cpu_index);
+    CPUArchIdList *(*possible_cpu_arch_ids)(MachineState *machine);
 };
 
 /**
@@ -127,6 +139,7 @@ struct MachineState {
     char *accel;
     bool kernel_irqchip_allowed;
     bool kernel_irqchip_required;
+    bool kernel_irqchip_split;
     int kvm_shadow_mem;
     char *dtb;
     char *dumpdtb;
@@ -136,9 +149,11 @@ struct MachineState {
     bool mem_merge;
     bool usb;
     bool usb_disabled;
+    bool igd_gfx_passthru;
     char *firmware;
     bool iommu;
     bool suppress_vmdesc;
+    bool enforce_config_section;
 
     ram_addr_t ram_size;
     ram_addr_t maxram_size;
@@ -151,4 +166,30 @@ struct MachineState {
     AccelState *accelerator;
 };
 
+#define DEFINE_MACHINE(namestr, machine_initfn) \
+    static void machine_initfn##_class_init(ObjectClass *oc, void *data) \
+    { \
+        MachineClass *mc = MACHINE_CLASS(oc); \
+        machine_initfn(mc); \
+    } \
+    static const TypeInfo machine_initfn##_typeinfo = { \
+        .name       = MACHINE_TYPE_NAME(namestr), \
+        .parent     = TYPE_MACHINE, \
+        .class_init = machine_initfn##_class_init, \
+    }; \
+    static void machine_initfn##_register_types(void) \
+    { \
+        type_register_static(&machine_initfn##_typeinfo); \
+    } \
+    type_init(machine_initfn##_register_types)
+
+#define SET_MACHINE_COMPAT(m, COMPAT) \
+    do {                              \
+        static GlobalProperty props[] = {       \
+            COMPAT                              \
+            { /* end of list */ }               \
+        };                                      \
+        (m)->compat_props = props;              \
+    } while (0)
+
 #endif
diff --git a/qemu/include/hw/bt.h b/qemu/include/hw/bt.h
index cb2a7e657..c7c7909a3 100644
--- a/qemu/include/hw/bt.h
+++ b/qemu/include/hw/bt.h
@@ -504,7 +504,6 @@ typedef struct {
 
 #define OCF_CREATE_CONN_CANCEL		0x0008
 typedef struct {
-    uint8_t	status;
     bdaddr_t	bdaddr;
 } QEMU_PACKED create_conn_cancel_cp;
 #define CREATE_CONN_CANCEL_CP_SIZE 6
@@ -1266,13 +1265,13 @@ typedef struct {
     uint8_t	status;
     uint16_t	handle;
 } QEMU_PACKED reset_failed_contact_counter_rp;
-#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 4
+#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 3
 
 #define OCF_READ_LINK_QUALITY		0x0003
 typedef struct {
     uint16_t	handle;
 } QEMU_PACKED read_link_quality_cp;
-#define READ_LINK_QUALITY_CP_SIZE 4
+#define READ_LINK_QUALITY_CP_SIZE 2
 
 typedef struct {
     uint8_t	status;
@@ -1332,7 +1331,7 @@ typedef struct {
     uint8_t	dev_class[3];
     uint16_t	clock_offset;
 } QEMU_PACKED inquiry_info;
-#define INQUIRY_INFO_SIZE 14
+#define INQUIRY_INFO_SIZE 15
 
 #define EVT_CONN_COMPLETE		0x03
 typedef struct {
@@ -1381,7 +1380,7 @@ typedef struct {
     uint16_t	handle;
     uint8_t	encrypt;
 } QEMU_PACKED evt_encrypt_change;
-#define EVT_ENCRYPT_CHANGE_SIZE 5
+#define EVT_ENCRYPT_CHANGE_SIZE 4
 
 #define EVT_CHANGE_CONN_LINK_KEY_COMPLETE	0x09
 typedef struct {
@@ -1629,18 +1628,6 @@ typedef struct {
 } QEMU_PACKED evt_sniff_subrate;
 #define EVT_SNIFF_SUBRATE_SIZE 11
 
-#define EVT_EXTENDED_INQUIRY_RESULT	0x2F
-typedef struct {
-    bdaddr_t	bdaddr;
-    uint8_t	pscan_rep_mode;
-    uint8_t	pscan_period_mode;
-    uint8_t	dev_class[3];
-    uint16_t	clock_offset;
-    int8_t	rssi;
-    uint8_t	data[240];
-} QEMU_PACKED extended_inquiry_info;
-#define EXTENDED_INQUIRY_INFO_SIZE 254
-
 #define EVT_TESTING			0xFE
 
 #define EVT_VENDOR			0xFF
diff --git a/qemu/include/hw/char/bcm2835_aux.h b/qemu/include/hw/char/bcm2835_aux.h
new file mode 100644
index 000000000..42f0ee7a9
--- /dev/null
+++ b/qemu/include/hw/char/bcm2835_aux.h
@@ -0,0 +1,33 @@
+/*
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_AUX_H
+#define BCM2835_AUX_H
+
+#include "hw/sysbus.h"
+#include "sysemu/char.h"
+
+#define TYPE_BCM2835_AUX "bcm2835-aux"
+#define BCM2835_AUX(obj) OBJECT_CHECK(BCM2835AuxState, (obj), TYPE_BCM2835_AUX)
+
+#define BCM2835_AUX_RX_FIFO_LEN 8
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion iomem;
+    CharDriverState *chr;
+    qemu_irq irq;
+
+    uint8_t read_fifo[BCM2835_AUX_RX_FIFO_LEN];
+    uint8_t read_pos, read_count;
+    uint8_t ier, iir;
+} BCM2835AuxState;
+
+#endif
diff --git a/qemu/include/hw/char/digic-uart.h b/qemu/include/hw/char/digic-uart.h
index ef83a3059..7b3f14537 100644
--- a/qemu/include/hw/char/digic-uart.h
+++ b/qemu/include/hw/char/digic-uart.h
@@ -19,7 +19,6 @@
 #define HW_CHAR_DIGIC_UART_H
 
 #include "hw/sysbus.h"
-#include "qemu/typedefs.h"
 
 #define TYPE_DIGIC_UART "digic-uart"
 #define DIGIC_UART(obj) \
diff --git a/qemu/include/hw/char/imx_serial.h b/qemu/include/hw/char/imx_serial.h
new file mode 100644
index 000000000..6cd75c0ba
--- /dev/null
+++ b/qemu/include/hw/char/imx_serial.h
@@ -0,0 +1,102 @@
+/*
+ * Device model for i.MX UART
+ *
+ * Copyright (c) 2008 OKL
+ * Originally Written by Hans Jiang
+ * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_SERIAL_H
+#define IMX_SERIAL_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_SERIAL "imx.serial"
+#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
+
+#define URXD_CHARRDY    (1<<15)   /* character read is valid */
+#define URXD_ERR        (1<<14)   /* Character has error */
+#define URXD_BRK        (1<<11)   /* Break received */
+
+#define USR1_PARTYER    (1<<15)   /* Parity Error */
+#define USR1_RTSS       (1<<14)   /* RTS pin status */
+#define USR1_TRDY       (1<<13)   /* Tx ready */
+#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
+#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
+#define USR1_FRAMERR    (1<<10)   /* Framing error  */
+#define USR1_RRDY       (1<<9)    /* receiver ready */
+#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
+#define USR1_DTRD       (1<<7)    /* DTR changed */
+#define USR1_RXDS       (1<<6)    /* Receiver is idle */
+#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
+#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
+
+#define USR2_ADET       (1<<15)   /* Autobaud complete */
+#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
+#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
+#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
+#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
+#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
+#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
+#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
+#define USR2_WAKE       (1<<7)    /* Start bit detected */
+#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
+#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
+#define USR2_RTSF       (1<<4)    /* RTS transition */
+#define USR2_TXDC       (1<<3)    /* Transmission complete */
+#define USR2_BRCD       (1<<2)    /* Break condition detected */
+#define USR2_ORE        (1<<1)    /* Overrun error */
+#define USR2_RDR        (1<<0)    /* Receive data ready */
+
+#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
+#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
+#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
+#define UCR1_UARTEN     (1<<0)    /* UART Enable */
+
+#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
+#define UCR2_RXEN       (1<<1)    /* Receiver enable */
+#define UCR2_SRST       (1<<0)    /* Reset complete */
+
+#define UTS1_TXEMPTY    (1<<6)
+#define UTS1_RXEMPTY    (1<<5)
+#define UTS1_TXFULL     (1<<4)
+#define UTS1_RXFULL     (1<<3)
+
+typedef struct IMXSerialState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    int32_t readbuff;
+
+    uint32_t usr1;
+    uint32_t usr2;
+    uint32_t ucr1;
+    uint32_t ucr2;
+    uint32_t uts1;
+
+    /*
+     * The registers below are implemented just so that the
+     * guest OS sees what it has written
+     */
+    uint32_t onems;
+    uint32_t ufcr;
+    uint32_t ubmr;
+    uint32_t ubrc;
+    uint32_t ucr3;
+
+    qemu_irq irq;
+    CharDriverState *chr;
+} IMXSerialState;
+
+#endif
diff --git a/qemu/include/hw/compat.h b/qemu/include/hw/compat.h
index 94c8097d0..a5dbbf898 100644
--- a/qemu/include/hw/compat.h
+++ b/qemu/include/hw/compat.h
@@ -1,61 +1,112 @@
 #ifndef HW_COMPAT_H
 #define HW_COMPAT_H
 
+#define HW_COMPAT_2_5 \
+    {\
+        .driver   = "isa-fdc",\
+        .property = "fallback",\
+        .value    = "144",\
+    },{\
+        .driver   = "pvscsi",\
+        .property = "x-old-pci-configuration",\
+        .value    = "on",\
+    },{\
+        .driver   = "pvscsi",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "vmxnet3",\
+        .property = "x-old-msi-offsets",\
+        .value    = "on",\
+    },{\
+        .driver   = "vmxnet3",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },
+
+#define HW_COMPAT_2_4 \
+    {\
+        .driver   = "virtio-blk-device",\
+        .property = "scsi",\
+        .value    = "true",\
+    },{\
+        .driver   = "e1000",\
+        .property = "extra_mac_registers",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "migrate-extra",\
+        .value    = "off",\
+    },{\
+        .driver   = "fw_cfg_mem",\
+        .property = "dma_enabled",\
+        .value    = "off",\
+    },{\
+        .driver   = "fw_cfg_io",\
+        .property = "dma_enabled",\
+        .value    = "off",\
+    },
+
 #define HW_COMPAT_2_3 \
-        {\
-            .driver   = "virtio-blk-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-balloon-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-serial-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-9p-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-rng-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },
+    {\
+        .driver   = "virtio-blk-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-balloon-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-serial-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-9p-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-rng-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },
 
 #define HW_COMPAT_2_2 \
-        /* empty */
+    /* empty */
 
 #define HW_COMPAT_2_1 \
-        {\
-            .driver   = "intel-hda",\
-            .property = "old_msi_addr",\
-            .value    = "on",\
-        },{\
-            .driver   = "VGA",\
-            .property = "qemu-extended-regs",\
-            .value    = "off",\
-        },{\
-            .driver   = "secondary-vga",\
-            .property = "qemu-extended-regs",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-scsi-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "usb-mouse",\
-            .property = "usb_version",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "usb-kbd",\
-            .property = "usb_version",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "virtio-pci",\
-            .property = "virtio-pci-bus-master-bug-migration",\
-            .value    = "on",\
-        },
+    {\
+        .driver   = "intel-hda",\
+        .property = "old_msi_addr",\
+        .value    = "on",\
+    },{\
+        .driver   = "VGA",\
+        .property = "qemu-extended-regs",\
+        .value    = "off",\
+    },{\
+        .driver   = "secondary-vga",\
+        .property = "qemu-extended-regs",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-scsi-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "usb-mouse",\
+        .property = "usb_version",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "usb-kbd",\
+        .property = "usb_version",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "virtio-pci-bus-master-bug-migration",\
+        .value    = "on",\
+    },
 
 #endif /* HW_COMPAT_H */
diff --git a/qemu/include/hw/cpu/icc_bus.h b/qemu/include/hw/cpu/icc_bus.h
deleted file mode 100644
index 98a979fa1..000000000
--- a/qemu/include/hw/cpu/icc_bus.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* icc_bus.h
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#ifndef ICC_BUS_H
-#define ICC_BUS_H
-
-#include "exec/memory.h"
-#include "hw/qdev-core.h"
-
-#define TYPE_ICC_BUS "icc-bus"
-
-#ifndef CONFIG_USER_ONLY
-
-/**
- * ICCBus:
- *
- * ICC bus
- */
-typedef struct ICCBus {
-    /*< private >*/
-    BusState parent_obj;
-    /*< public >*/
-
-    MemoryRegion *apic_address_space;
-} ICCBus;
-
-#define ICC_BUS(obj) OBJECT_CHECK(ICCBus, (obj), TYPE_ICC_BUS)
-
-/**
- * ICCDevice:
- *
- * ICC device
- */
-typedef struct ICCDevice {
-    /*< private >*/
-    DeviceState qdev;
-    /*< public >*/
-} ICCDevice;
-
-/**
- * ICCDeviceClass:
- * @init: Initialization callback for derived classes.
- *
- * ICC device class
- */
-typedef struct ICCDeviceClass {
-    /*< private >*/
-    DeviceClass parent_class;
-    /*< public >*/
-
-    DeviceRealize realize;
-} ICCDeviceClass;
-
-#define TYPE_ICC_DEVICE "icc-device"
-#define ICC_DEVICE(obj) OBJECT_CHECK(ICCDevice, (obj), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(ICCDeviceClass, (klass), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(ICCDeviceClass, (obj), TYPE_ICC_DEVICE)
-
-#define TYPE_ICC_BRIDGE "icc-bridge"
-
-#endif /* CONFIG_USER_ONLY */
-#endif
diff --git a/qemu/include/hw/display/bcm2835_fb.h b/qemu/include/hw/display/bcm2835_fb.h
new file mode 100644
index 000000000..9a12d7afa
--- /dev/null
+++ b/qemu/include/hw/display/bcm2835_fb.h
@@ -0,0 +1,47 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_FB_H
+#define BCM2835_FB_H
+
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "ui/console.h"
+
+#define TYPE_BCM2835_FB "bcm2835-fb"
+#define BCM2835_FB(obj) OBJECT_CHECK(BCM2835FBState, (obj), TYPE_BCM2835_FB)
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    uint32_t vcram_base, vcram_size;
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+    MemoryRegion iomem;
+    MemoryRegionSection fbsection;
+    QemuConsole *con;
+    qemu_irq mbox_irq;
+
+    bool lock, invalidate, pending;
+    uint32_t xres, yres;
+    uint32_t xres_virtual, yres_virtual;
+    uint32_t xoffset, yoffset;
+    uint32_t bpp;
+    uint32_t base, pitch, size;
+    uint32_t pixo, alpha;
+} BCM2835FBState;
+
+void bcm2835_fb_reconfigure(BCM2835FBState *s, uint32_t *xres, uint32_t *yres,
+                            uint32_t *xoffset, uint32_t *yoffset, uint32_t *bpp,
+                            uint32_t *pixo, uint32_t *alpha);
+
+#endif
diff --git a/qemu/include/hw/dma/bcm2835_dma.h b/qemu/include/hw/dma/bcm2835_dma.h
new file mode 100644
index 000000000..75312e2e1
--- /dev/null
+++ b/qemu/include/hw/dma/bcm2835_dma.h
@@ -0,0 +1,47 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_DMA_H
+#define BCM2835_DMA_H
+
+#include "qemu-common.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
+
+typedef struct {
+    uint32_t cs;
+    uint32_t conblk_ad;
+    uint32_t ti;
+    uint32_t source_ad;
+    uint32_t dest_ad;
+    uint32_t txfr_len;
+    uint32_t stride;
+    uint32_t nextconbk;
+    uint32_t debug;
+
+    qemu_irq irq;
+} BCM2835DMAChan;
+
+#define TYPE_BCM2835_DMA "bcm2835-dma"
+#define BCM2835_DMA(obj) \
+        OBJECT_CHECK(BCM2835DMAState, (obj), TYPE_BCM2835_DMA)
+
+#define BCM2835_DMA_NCHANS 16
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion iomem0, iomem15;
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+
+    BCM2835DMAChan chan[BCM2835_DMA_NCHANS];
+    uint32_t int_status;
+    uint32_t enable;
+} BCM2835DMAState;
+
+#endif
diff --git a/qemu/include/hw/elf_ops.h b/qemu/include/hw/elf_ops.h
index bd7196814..f510e7ec2 100644
--- a/qemu/include/hw/elf_ops.h
+++ b/qemu/include/hw/elf_ops.h
@@ -263,7 +263,7 @@ static int glue(load_elf, SZ)(const char *name, int fd,
                               void *translate_opaque,
                               int must_swab, uint64_t *pentry,
                               uint64_t *lowaddr, uint64_t *highaddr,
-                              int elf_machine, int clear_lsb)
+                              int elf_machine, int clear_lsb, int data_swab)
 {
     struct elfhdr ehdr;
     struct elf_phdr *phdr = NULL, *ph;
@@ -282,25 +282,36 @@ static int glue(load_elf, SZ)(const char *name, int fd,
 
     switch (elf_machine) {
         case EM_PPC64:
-            if (EM_PPC64 != ehdr.e_machine)
-                if (EM_PPC != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_PPC64) {
+                if (ehdr.e_machine != EM_PPC) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_X86_64:
-            if (EM_X86_64 != ehdr.e_machine)
-                if (EM_386 != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_X86_64) {
+                if (ehdr.e_machine != EM_386) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_MICROBLAZE:
-            if (EM_MICROBLAZE != ehdr.e_machine)
-                if (EM_MICROBLAZE_OLD != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_MICROBLAZE) {
+                if (ehdr.e_machine != EM_MICROBLAZE_OLD) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
+            break;
+        case EM_MOXIE:
+            if (ehdr.e_machine != EM_MOXIE) {
+                if (ehdr.e_machine != EM_MOXIE_OLD) {
+                    ret = ELF_LOAD_WRONG_ARCH;
+                    goto fail;
+                }
+            }
             break;
         default:
             if (elf_machine != ehdr.e_machine) {
@@ -355,6 +366,26 @@ static int glue(load_elf, SZ)(const char *name, int fd,
                 addr = ph->p_paddr;
             }
 
+            if (data_swab) {
+                int j;
+                for (j = 0; j < file_size; j += (1 << data_swab)) {
+                    uint8_t *dp = data + j;
+                    switch (data_swab) {
+                    case (1):
+                        *(uint16_t *)dp = bswap16(*(uint16_t *)dp);
+                        break;
+                    case (2):
+                        *(uint32_t *)dp = bswap32(*(uint32_t *)dp);
+                        break;
+                    case (3):
+                        *(uint64_t *)dp = bswap64(*(uint64_t *)dp);
+                        break;
+                    default:
+                        g_assert_not_reached();
+                    }
+                }
+            }
+
             /* the entry pointer in the ELF header is a virtual
              * address, if the text segments paddr and vaddr differ
              * we need to adjust the entry */
diff --git a/qemu/include/hw/gpio/imx_gpio.h b/qemu/include/hw/gpio/imx_gpio.h
new file mode 100644
index 000000000..b15a09fbc
--- /dev/null
+++ b/qemu/include/hw/gpio/imx_gpio.h
@@ -0,0 +1,63 @@
+/*
+ * i.MX processors GPIO registers definition.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __IMX_GPIO_H_
+#define __IMX_GPIO_H_
+
+#include <hw/sysbus.h>
+
+#define TYPE_IMX_GPIO "imx.gpio"
+#define IMX_GPIO(obj) OBJECT_CHECK(IMXGPIOState, (obj), TYPE_IMX_GPIO)
+
+#define IMX_GPIO_MEM_SIZE 0x20
+
+/* i.MX GPIO memory map */
+#define DR_ADDR             0x00 /* DATA REGISTER */
+#define GDIR_ADDR           0x04 /* DIRECTION REGISTER */
+#define PSR_ADDR            0x08 /* PAD STATUS REGISTER */
+#define ICR1_ADDR           0x0c /* INTERRUPT CONFIGURATION REGISTER 1 */
+#define ICR2_ADDR           0x10 /* INTERRUPT CONFIGURATION REGISTER 2 */
+#define IMR_ADDR            0x14 /* INTERRUPT MASK REGISTER */
+#define ISR_ADDR            0x18 /* INTERRUPT STATUS REGISTER */
+#define EDGE_SEL_ADDR       0x1c /* EDGE SEL REGISTER */
+
+#define IMX_GPIO_PIN_COUNT 32
+
+typedef struct IMXGPIOState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t dr;
+    uint32_t gdir;
+    uint32_t psr;
+    uint64_t icr;
+    uint32_t imr;
+    uint32_t isr;
+    bool has_edge_sel;
+    uint32_t edge_sel;
+    bool has_upper_pin_irq;
+
+    qemu_irq irq[2];
+    qemu_irq output[IMX_GPIO_PIN_COUNT];
+} IMXGPIOState;
+
+#endif /* __IMX_GPIO_H_ */
diff --git a/qemu/include/hw/hotplug.h b/qemu/include/hw/hotplug.h
index 2db025d0b..da1d0e4ab 100644
--- a/qemu/include/hw/hotplug.h
+++ b/qemu/include/hw/hotplug.h
@@ -13,7 +13,6 @@
 #define HOTPLUG_H
 
 #include "qom/object.h"
-#include "qemu/typedefs.h"
 
 #define TYPE_HOTPLUG_HANDLER "hotplug-handler"
 
diff --git a/qemu/include/hw/hw.h b/qemu/include/hw/hw.h
index c78adae06..2cb69d5f5 100644
--- a/qemu/include/hw/hw.h
+++ b/qemu/include/hw/hw.h
@@ -2,7 +2,6 @@
 #ifndef QEMU_HW_H
 #define QEMU_HW_H
 
-#include "qemu-common.h"
 
 #if !defined(CONFIG_USER_ONLY) && !defined(NEED_CPU_H)
 #include "exec/cpu-common.h"
@@ -13,6 +12,7 @@
 #include "block/aio.h"
 #include "migration/vmstate.h"
 #include "qemu/log.h"
+#include "qemu/module.h"
 
 #ifdef NEED_CPU_H
 #if TARGET_LONG_BITS == 64
@@ -41,6 +41,8 @@ typedef void QEMUResetHandler(void *opaque);
 void qemu_register_reset(QEMUResetHandler *func, void *opaque);
 void qemu_unregister_reset(QEMUResetHandler *func, void *opaque);
 
+void QEMU_NORETURN hw_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
+
 #ifdef NEED_CPU_H
 #if TARGET_LONG_BITS == 64
 #define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
@@ -49,6 +51,7 @@ void qemu_unregister_reset(QEMUResetHandler *func, void *opaque);
     VMSTATE_UINT64_EQUAL_V(_f, _s, _v)
 #define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
     VMSTATE_UINT64_ARRAY_V(_f, _s, _n, _v)
+#define vmstate_info_uinttl vmstate_info_uint64
 #else
 #define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
     VMSTATE_UINT32_V(_f, _s, _v)
@@ -56,6 +59,7 @@ void qemu_unregister_reset(QEMUResetHandler *func, void *opaque);
     VMSTATE_UINT32_EQUAL_V(_f, _s, _v)
 #define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
     VMSTATE_UINT32_ARRAY_V(_f, _s, _n, _v)
+#define vmstate_info_uinttl vmstate_info_uint32
 #endif
 #define VMSTATE_UINTTL(_f, _s)                                        \
     VMSTATE_UINTTL_V(_f, _s, 0)
diff --git a/qemu/include/hw/i2c/imx_i2c.h b/qemu/include/hw/i2c/imx_i2c.h
new file mode 100644
index 000000000..e2ee8eaee
--- /dev/null
+++ b/qemu/include/hw/i2c/imx_i2c.h
@@ -0,0 +1,87 @@
+/*
+ *  i.MX I2C Bus Serial Interface registers definition
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef __IMX_I2C_H_
+#define __IMX_I2C_H_
+
+#include <hw/sysbus.h>
+
+#define TYPE_IMX_I2C "imx.i2c"
+#define IMX_I2C(obj) OBJECT_CHECK(IMXI2CState, (obj), TYPE_IMX_I2C)
+
+#define IMX_I2C_MEM_SIZE           0x14
+
+/* i.MX I2C memory map */
+#define IADR_ADDR                  0x00  /* address register */
+#define IFDR_ADDR                  0x04  /* frequency divider register */
+#define I2CR_ADDR                  0x08  /* control register */
+#define I2SR_ADDR                  0x0c  /* status register */
+#define I2DR_ADDR                  0x10  /* data register */
+
+#define IADR_MASK                  0xFE
+#define IADR_RESET                 0
+
+#define IFDR_MASK                  0x3F
+#define IFDR_RESET                 0
+
+#define I2CR_IEN                   (1 << 7)
+#define I2CR_IIEN                  (1 << 6)
+#define I2CR_MSTA                  (1 << 5)
+#define I2CR_MTX                   (1 << 4)
+#define I2CR_TXAK                  (1 << 3)
+#define I2CR_RSTA                  (1 << 2)
+#define I2CR_MASK                  0xFC
+#define I2CR_RESET                 0
+
+#define I2SR_ICF                   (1 << 7)
+#define I2SR_IAAF                  (1 << 6)
+#define I2SR_IBB                   (1 << 5)
+#define I2SR_IAL                   (1 << 4)
+#define I2SR_SRW                   (1 << 2)
+#define I2SR_IIF                   (1 << 1)
+#define I2SR_RXAK                  (1 << 0)
+#define I2SR_MASK                  0xE9
+#define I2SR_RESET                 0x81
+
+#define I2DR_MASK                  0xFF
+#define I2DR_RESET                 0
+
+#define ADDR_RESET                 0xFF00
+
+typedef struct IMXI2CState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    I2CBus *bus;
+    qemu_irq irq;
+
+    uint16_t  address;
+
+    uint16_t iadr;
+    uint16_t ifdr;
+    uint16_t i2cr;
+    uint16_t i2sr;
+    uint16_t i2dr_read;
+    uint16_t i2dr_write;
+} IMXI2CState;
+
+#endif /* __IMX_I2C_H_ */
diff --git a/qemu/include/hw/i386/apic_internal.h b/qemu/include/hw/i386/apic_internal.h
index dc7a89d98..74fe935e8 100644
--- a/qemu/include/hw/i386/apic_internal.h
+++ b/qemu/include/hw/i386/apic_internal.h
@@ -20,8 +20,8 @@
 #ifndef QEMU_APIC_INTERNAL_H
 #define QEMU_APIC_INTERNAL_H
 
+#include "cpu.h"
 #include "exec/memory.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/timer.h"
 
 /* APIC Local Vector Table */
@@ -49,17 +49,73 @@
 #define APIC_TRIGGER_EDGE               0
 #define APIC_TRIGGER_LEVEL              1
 
-#define APIC_LVT_TIMER_PERIODIC         (1<<17)
-#define APIC_LVT_MASKED                 (1<<16)
-#define APIC_LVT_LEVEL_TRIGGER          (1<<15)
-#define APIC_LVT_REMOTE_IRR             (1<<14)
-#define APIC_INPUT_POLARITY             (1<<13)
-#define APIC_SEND_PENDING               (1<<12)
-
-#define ESR_ILLEGAL_ADDRESS (1 << 7)
-
-#define APIC_SV_DIRECTED_IO             (1<<12)
-#define APIC_SV_ENABLE                  (1<<8)
+#define APIC_VECTOR_MASK                0xff
+#define APIC_DCR_MASK                   0xf
+
+#define APIC_LVT_TIMER_SHIFT            17
+#define APIC_LVT_MASKED_SHIFT           16
+#define APIC_LVT_LEVEL_TRIGGER_SHIFT    15
+#define APIC_LVT_REMOTE_IRR_SHIFT       14
+#define APIC_LVT_INT_POLARITY_SHIFT     13
+#define APIC_LVT_DELIV_STS_SHIFT        12
+#define APIC_LVT_DELIV_MOD_SHIFT        8
+
+#define APIC_LVT_TIMER_TSCDEADLINE      (2 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_TIMER_PERIODIC         (1 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_MASKED                 (1 << APIC_LVT_MASKED_SHIFT)
+#define APIC_LVT_LEVEL_TRIGGER          (1 << APIC_LVT_LEVEL_TRIGGER_SHIFT)
+#define APIC_LVT_REMOTE_IRR             (1 << APIC_LVT_REMOTE_IRR_SHIFT)
+#define APIC_LVT_INT_POLARITY           (1 << APIC_LVT_INT_POLARITY_SHIFT)
+#define APIC_LVT_DELIV_STS              (1 << APIC_LVT_DELIV_STS_SHIFT)
+#define APIC_LVT_DELIV_MOD              (7 << APIC_LVT_DELIV_MOD_SHIFT)
+
+#define APIC_ESR_ILL_ADDRESS_SHIFT      7
+#define APIC_ESR_RECV_ILL_VECT_SHIFT    6
+#define APIC_ESR_SEND_ILL_VECT_SHIFT    5
+#define APIC_ESR_RECV_ACCEPT_SHIFT      3
+#define APIC_ESR_SEND_ACCEPT_SHIFT      2
+#define APIC_ESR_RECV_CHECK_SUM_SHIFT   1
+
+#define APIC_ESR_ILLEGAL_ADDRESS        (1 << APIC_ESR_ILL_ADDRESS_SHIFT)
+#define APIC_ESR_RECV_ILLEGAL_VECT      (1 << APIC_ESR_RECV_ILL_VECT_SHIFT)
+#define APIC_ESR_SEND_ILLEGAL_VECT      (1 << APIC_ESR_SEND_ILL_VECT_SHIFT)
+#define APIC_ESR_RECV_ACCEPT            (1 << APIC_ESR_RECV_ACCEPT_SHIFT)
+#define APIC_ESR_SEND_ACCEPT            (1 << APIC_ESR_SEND_ACCEPT_SHIFT)
+#define APIC_ESR_RECV_CHECK_SUM         (1 << APIC_ESR_RECV_CHECK_SUM_SHIFT)
+#define APIC_ESR_SEND_CHECK_SUM         1
+
+#define APIC_ICR_DEST_SHIFT             24
+#define APIC_ICR_DEST_SHORT_SHIFT       18
+#define APIC_ICR_TRIGGER_MOD_SHIFT      15
+#define APIC_ICR_LEVEL_SHIFT            14
+#define APIC_ICR_DELIV_STS_SHIFT        12
+#define APIC_ICR_DEST_MOD_SHIFT         11
+#define APIC_ICR_DELIV_MOD_SHIFT        8
+
+#define APIC_ICR_DEST_SHORT             (3 << APIC_ICR_DEST_SHORT_SHIFT)
+#define APIC_ICR_TRIGGER_MOD            (1 << APIC_ICR_TRIGGER_MOD_SHIFT)
+#define APIC_ICR_LEVEL                  (1 << APIC_ICR_LEVEL_SHIFT)
+#define APIC_ICR_DELIV_STS              (1 << APIC_ICR_DELIV_STS_SHIFT)
+#define APIC_ICR_DEST_MOD               (1 << APIC_ICR_DEST_MOD_SHIFT)
+#define APIC_ICR_DELIV_MOD              (7 << APIC_ICR_DELIV_MOD_SHIFT)
+
+#define APIC_PR_CLASS_SHIFT             4
+#define APIC_PR_SUB_CLASS               0xf
+
+#define APIC_LOGDEST_XAPIC_SHIFT        4
+#define APIC_LOGDEST_XAPIC_ID           0xf
+
+#define APIC_LOGDEST_X2APIC_SHIFT       16
+#define APIC_LOGDEST_X2APIC_ID          0xffff
+
+#define APIC_SPURIO_FOCUS_SHIFT         9
+#define APIC_SPURIO_ENABLED_SHIFT       8
+
+#define APIC_SPURIO_FOCUS               (1 << APIC_SPURIO_FOCUS_SHIFT)
+#define APIC_SPURIO_ENABLED             (1 << APIC_SPURIO_ENABLED_SHIFT)
+
+#define APIC_SV_DIRECTED_IO             (1 << 12)
+#define APIC_SV_ENABLE                  (1 << 8)
 
 #define VAPIC_ENABLE_BIT                0
 #define VAPIC_ENABLE_MASK               (1 << VAPIC_ENABLE_BIT)
@@ -78,7 +134,7 @@ typedef struct APICCommonState APICCommonState;
 
 typedef struct APICCommonClass
 {
-    ICCDeviceClass parent_class;
+    DeviceClass parent_class;
 
     DeviceRealize realize;
     void (*set_base)(APICCommonState *s, uint64_t val);
@@ -93,7 +149,9 @@ typedef struct APICCommonClass
 } APICCommonClass;
 
 struct APICCommonState {
-    ICCDevice busdev;
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
 
     MemoryRegion io_memory;
     X86CPU *cpu;
@@ -146,4 +204,22 @@ void apic_enable_vapic(DeviceState *d, hwaddr paddr);
 void vapic_report_tpr_access(DeviceState *dev, CPUState *cpu, target_ulong ip,
                              TPRAccess access);
 
+int apic_get_ppr(APICCommonState *s);
+
+static inline void apic_set_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    tab[i] |= mask;
+}
+
+static inline int apic_get_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    return !!(tab[i] & mask);
+}
+
 #endif /* !QEMU_APIC_INTERNAL_H */
diff --git a/qemu/include/hw/i386/ich9.h b/qemu/include/hw/i386/ich9.h
index b9d2b04b6..d04dcdcfb 100644
--- a/qemu/include/hw/i386/ich9.h
+++ b/qemu/include/hw/i386/ich9.h
@@ -17,13 +17,13 @@
 void ich9_lpc_set_irq(void *opaque, int irq_num, int level);
 int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx);
 PCIINTxRoute ich9_route_intx_pin_to_irq(void *opaque, int pirq_pin);
-void ich9_lpc_pm_init(PCIDevice *pci_lpc, bool smm_enabled, bool enable_tco);
+void ich9_lpc_pm_init(PCIDevice *pci_lpc, bool smm_enabled);
 I2CBus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base);
 
 void ich9_generate_smi(void);
 void ich9_generate_nmi(void);
 
-#define ICH9_CC_SIZE                            (16 * 1024)     /* 16KB */
+#define ICH9_CC_SIZE (16 * 1024) /* 16KB. Chipset configuration registers */
 
 #define TYPE_ICH9_LPC_DEVICE "ICH9-LPC"
 #define ICH9_LPC_DEVICE(obj) \
@@ -65,7 +65,7 @@ typedef struct ICH9LPCState {
 
     /* isa bus */
     ISABus *isa_bus;
-    MemoryRegion rbca_mem;
+    MemoryRegion rcrb_mem; /* root complex register block */
     Notifier machine_ready;
 
     qemu_irq *pic;
diff --git a/qemu/include/hw/i386/intel_iommu.h b/qemu/include/hw/i386/intel_iommu.h
index e321ee4fb..b024ffa72 100644
--- a/qemu/include/hw/i386/intel_iommu.h
+++ b/qemu/include/hw/i386/intel_iommu.h
@@ -49,6 +49,7 @@ typedef struct VTDContextCacheEntry VTDContextCacheEntry;
 typedef struct IntelIOMMUState IntelIOMMUState;
 typedef struct VTDAddressSpace VTDAddressSpace;
 typedef struct VTDIOTLBEntry VTDIOTLBEntry;
+typedef struct VTDBus VTDBus;
 
 /* Context-Entry */
 struct VTDContextEntry {
@@ -65,7 +66,7 @@ struct VTDContextCacheEntry {
 };
 
 struct VTDAddressSpace {
-    uint8_t bus_num;
+    PCIBus *bus;
     uint8_t devfn;
     AddressSpace as;
     MemoryRegion iommu;
@@ -73,10 +74,16 @@ struct VTDAddressSpace {
     VTDContextCacheEntry context_cache_entry;
 };
 
+struct VTDBus {
+    PCIBus* bus;		/* A reference to the bus to provide translation for */
+    VTDAddressSpace *dev_as[0];	/* A table of VTDAddressSpace objects indexed by devfn */
+};
+
 struct VTDIOTLBEntry {
     uint64_t gfn;
     uint16_t domain_id;
     uint64_t slpte;
+    uint64_t mask;
     bool read_flags;
     bool write_flags;
 };
@@ -114,7 +121,13 @@ struct IntelIOMMUState {
     GHashTable *iotlb;              /* IOTLB */
 
     MemoryRegionIOMMUOps iommu_ops;
-    VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];
+    GHashTable *vtd_as_by_busptr;   /* VTDBus objects indexed by PCIBus* reference */
+    VTDBus *vtd_as_by_bus_num[VTD_PCI_BUS_MAX]; /* VTDBus objects indexed by bus number */
 };
 
+/* Find the VTD Address space associated with the given bus pointer,
+ * create a new one if none exists
+ */
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn);
+
 #endif
diff --git a/qemu/include/hw/i386/ioapic_internal.h b/qemu/include/hw/i386/ioapic_internal.h
index 3be335218..797ed4730 100644
--- a/qemu/include/hw/i386/ioapic_internal.h
+++ b/qemu/include/hw/i386/ioapic_internal.h
@@ -40,7 +40,12 @@
 #define IOAPIC_LVT_DELIV_MODE_SHIFT     8
 
 #define IOAPIC_LVT_MASKED               (1 << IOAPIC_LVT_MASKED_SHIFT)
+#define IOAPIC_LVT_TRIGGER_MODE         (1 << IOAPIC_LVT_TRIGGER_MODE_SHIFT)
 #define IOAPIC_LVT_REMOTE_IRR           (1 << IOAPIC_LVT_REMOTE_IRR_SHIFT)
+#define IOAPIC_LVT_POLARITY             (1 << IOAPIC_LVT_POLARITY_SHIFT)
+#define IOAPIC_LVT_DELIV_STATUS         (1 << IOAPIC_LVT_DELIV_STATUS_SHIFT)
+#define IOAPIC_LVT_DEST_MODE            (1 << IOAPIC_LVT_DEST_MODE_SHIFT)
+#define IOAPIC_LVT_DELIV_MODE           (7 << IOAPIC_LVT_DELIV_MODE_SHIFT)
 
 #define IOAPIC_TRIGGER_EDGE             0
 #define IOAPIC_TRIGGER_LEVEL            1
@@ -100,4 +105,6 @@ struct IOAPICCommonState {
 
 void ioapic_reset_common(DeviceState *dev);
 
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s);
+
 #endif /* !QEMU_IOAPIC_INTERNAL_H */
diff --git a/qemu/include/hw/i386/pc.h b/qemu/include/hw/i386/pc.h
index 954203d22..96f0b66c7 100644
--- a/qemu/include/hw/i386/pc.h
+++ b/qemu/include/hw/i386/pc.h
@@ -16,29 +16,62 @@
 #include "hw/boards.h"
 #include "hw/compat.h"
 #include "hw/mem/pc-dimm.h"
+#include "hw/mem/nvdimm.h"
 
 #define HPET_INTCAP "hpet-intcap"
 
+#ifdef CONFIG_KVM
+#define kvm_pit_in_kernel() \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_pic_in_kernel()  \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_ioapic_in_kernel() \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#else
+#define kvm_pit_in_kernel()      0
+#define kvm_pic_in_kernel()      0
+#define kvm_ioapic_in_kernel()   0
+#endif
+
 /**
  * PCMachineState:
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
  */
 struct PCMachineState {
     /*< private >*/
     MachineState parent_obj;
 
     /* <public> */
+
+    /* State for other subsystems/APIs: */
     MemoryHotplugState hotplug_memory;
+    Notifier machine_done;
 
+    /* Pointers to devices and objects: */
     HotplugHandler *acpi_dev;
     ISADevice *rtc;
+    PCIBus *bus;
+    FWCfgState *fw_cfg;
 
+    /* Configuration options: */
     uint64_t max_ram_below_4g;
     OnOffAuto vmport;
     OnOffAuto smm;
-    bool enforce_aligned_dimm;
+
+    AcpiNVDIMMState acpi_nvdimm_state;
+
+    /* RAM information (sizes, addresses, configuration): */
+    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+
+    /* CPU and apic information: */
+    bool apic_xrupt_override;
+    unsigned apic_id_limit;
+    CPUArchIdList *possible_cpus;
+
+    /* NUMA information: */
+    uint64_t numa_nodes;
+    uint64_t *node_mem;
+    uint64_t *node_cpu;
 };
 
 #define PC_MACHINE_ACPI_DEVICE_PROP "acpi-device"
@@ -46,23 +79,64 @@ struct PCMachineState {
 #define PC_MACHINE_MAX_RAM_BELOW_4G "max-ram-below-4g"
 #define PC_MACHINE_VMPORT           "vmport"
 #define PC_MACHINE_SMM              "smm"
-#define PC_MACHINE_ENFORCE_ALIGNED_DIMM "enforce-aligned-dimm"
+#define PC_MACHINE_NVDIMM           "nvdimm"
 
 /**
  * PCMachineClass:
+ *
+ * Methods:
+ *
  * @get_hotplug_handler: pointer to parent class callback @get_hotplug_handler
+ *
+ * Compat fields:
+ *
+ * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
+ *                        backend's alignment value if provided
+ * @acpi_data_size: Size of the chunk of memory at the top of RAM
+ *                  for the BIOS ACPI tables and other BIOS
+ *                  datastructures.
+ * @gigabyte_align: Make sure that guest addresses aligned at
+ *                  1Gbyte boundaries get mapped to host
+ *                  addresses aligned at 1Gbyte boundaries. This
+ *                  way we can use 1GByte pages in the host.
+ *
  */
 struct PCMachineClass {
     /*< private >*/
     MachineClass parent_class;
 
     /*< public >*/
+
+    /* Methods: */
     HotplugHandler *(*get_hotplug_handler)(MachineState *machine,
                                            DeviceState *dev);
-};
 
-typedef struct PCMachineState PCMachineState;
-typedef struct PCMachineClass PCMachineClass;
+    /* Device configuration: */
+    bool pci_enabled;
+    bool kvmclock_enabled;
+
+    /* Compat options: */
+
+    /* ACPI compat: */
+    bool has_acpi_build;
+    bool rsdp_in_ram;
+    int legacy_acpi_table_size;
+    unsigned acpi_data_size;
+
+    /* SMBIOS compat: */
+    bool smbios_defaults;
+    bool smbios_legacy_mode;
+    bool smbios_uuid_encoded;
+
+    /* RAM / address space compat: */
+    bool gigabyte_align;
+    bool has_reserved_memory;
+    bool enforce_aligned_dimm;
+    bool broken_reserved_end;
+
+    /* TSC rate migration: */
+    bool save_tsc_khz;
+};
 
 #define TYPE_PC_MACHINE "generic-pc-machine"
 #define PC_MACHINE(obj) \
@@ -90,21 +164,6 @@ typedef struct PcPciInfo {
 #define ACPI_PM_PROP_GPE0_BLK_LEN "gpe0_blk_len"
 #define ACPI_PM_PROP_TCO_ENABLED "enable_tco"
 
-struct PcGuestInfo {
-    bool isapc_ram_fw;
-    hwaddr ram_size, ram_size_below_4g;
-    unsigned apic_id_limit;
-    bool apic_xrupt_override;
-    uint64_t numa_nodes;
-    uint64_t *node_mem;
-    uint64_t *node_cpu;
-    FWCfgState *fw_cfg;
-    int legacy_acpi_table_size;
-    bool has_acpi_build;
-    bool has_reserved_memory;
-    bool rsdp_in_ram;
-};
-
 /* parallel.c */
 
 void parallel_hds_isa_init(ISABus *bus, int n);
@@ -123,6 +182,11 @@ int pic_get_output(DeviceState *d);
 void hmp_info_pic(Monitor *mon, const QDict *qdict);
 void hmp_info_irq(Monitor *mon, const QDict *qdict);
 
+/* ioapic.c */
+
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict);
+void ioapic_dump_state(Monitor *mon, const QDict *qdict);
+
 /* Global System Interrupts */
 
 #define GSI_NUM_PINS IOAPIC_NUM_PINS
@@ -162,14 +226,11 @@ bool pc_machine_is_smm_enabled(PCMachineState *pcms);
 void pc_register_ferr_irq(qemu_irq irq);
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level);
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge);
+void pc_cpus_init(PCMachineState *pcms);
 void pc_hot_add_cpu(const int64_t id, Error **errp);
 void pc_acpi_init(const char *default_dsdt);
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size);
-
-void pc_set_legacy_acpi_data_size(void);
+void pc_guest_info_init(PCMachineState *pcms);
 
 #define PCI_HOST_PROP_PCI_HOLE_START   "pci-hole-start"
 #define PCI_HOST_PROP_PCI_HOLE_END     "pci-hole-end"
@@ -182,28 +243,20 @@ void pc_set_legacy_acpi_data_size(void);
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
                             MemoryRegion *pci_address_space);
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
-                           PcGuestInfo *guest_info);
-FWCfgState *pc_memory_init(MachineState *machine,
-                           MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
-                           MemoryRegion *rom_memory,
-                           MemoryRegion **ram_memory,
-                           PcGuestInfo *guest_info);
+void xen_load_linux(PCMachineState *pcms);
+void pc_memory_init(PCMachineState *pcms,
+                    MemoryRegion *system_memory,
+                    MemoryRegion *rom_memory,
+                    MemoryRegion **ram_memory);
 qemu_irq pc_allocate_cpu_irq(void);
 DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus);
 void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
                           bool create_fdctrl,
                           bool no_vmport,
-                          uint32 hpet_irqs);
+                          uint32_t hpet_irqs);
 void pc_init_ne2k_isa(ISABus *bus, NICInfo *nd);
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
+void pc_cmos_init(PCMachineState *pcms,
                   BusState *ide0, BusState *ide1,
                   ISADevice *s);
 void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
@@ -213,6 +266,11 @@ typedef void (*cpu_set_smm_t)(int smm, void *arg);
 
 void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name);
 
+ISADevice *pc_find_fdc0(void);
+int cmos_get_fd_drive_type(FloppyDriveType fd0);
+
+#define FW_CFG_IO_BASE     0x510
+
 /* acpi_piix.c */
 
 I2CBus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
@@ -227,7 +285,13 @@ extern int no_hpet;
 struct PCII440FXState;
 typedef struct PCII440FXState PCII440FXState;
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
+#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
+#define TYPE_I440FX_PCI_DEVICE "i440FX"
+
+#define TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE "igd-passthrough-i440FX"
+
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state, int *piix_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
@@ -292,316 +356,489 @@ int e820_add_entry(uint64_t, uint64_t, uint32_t);
 int e820_get_num_entries(void);
 bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
 
+#define PC_COMPAT_2_5 \
+    HW_COMPAT_2_5
+
+#define PC_COMPAT_2_4 \
+    PC_COMPAT_2_5 \
+    HW_COMPAT_2_4 \
+    {\
+        .driver   = "Haswell-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Haswell-noTSX-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Broadwell-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Broadwell-noTSX-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "host" "-" TYPE_X86_CPU,\
+        .property = "host-cache-info",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = TYPE_X86_CPU,\
+        .property = "check",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "sse4a",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "popcnt",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu32" "-" TYPE_X86_CPU,\
+        .property = "popcnt",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G2" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G3" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G4" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G5" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },
+
+
 #define PC_COMPAT_2_3 \
-        HW_COMPAT_2_3 \
-        {\
-            .driver   = TYPE_X86_CPU,\
-            .property = "arat",\
-            .value    = "off",\
-        },{\
-            .driver   = "qemu64" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(4),\
-        },{\
-            .driver   = "kvm64" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(5),\
-        },{\
-            .driver   = "pentium3" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "n270" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(5),\
-        },{\
-            .driver   = "Conroe" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(4),\
-        },{\
-            .driver   = "Penryn" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(4),\
-        },{\
-            .driver   = "Nehalem" "-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(4),\
-        },{\
-            .driver   = "n270" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Penryn" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Conroe" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Nehalem" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Westmere" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "SandyBridge" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "IvyBridge" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Haswell" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Haswell-noTSX" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Broadwell" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },{\
-            .driver   = "Broadwell-noTSX" "-" TYPE_X86_CPU,\
-            .property = "xlevel",\
-            .value    = stringify(0x8000000a),\
-        },
+    PC_COMPAT_2_4 \
+    HW_COMPAT_2_3 \
+    {\
+        .driver   = TYPE_X86_CPU,\
+        .property = "arat",\
+        .value    = "off",\
+    },{\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "kvm64" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(5),\
+    },{\
+        .driver   = "pentium3" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "n270" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(5),\
+    },{\
+        .driver   = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "n270" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "SandyBridge" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "IvyBridge" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Haswell-noTSX" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Broadwell-noTSX" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },
 
 #define PC_COMPAT_2_2 \
-        PC_COMPAT_2_3 \
-        HW_COMPAT_2_2
+    PC_COMPAT_2_3 \
+    HW_COMPAT_2_2 \
+    {\
+        .driver = "kvm64" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "kvm32" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "SandyBridge" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G1" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G2" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G3" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G4" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G5" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "f16c",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "rdrand",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "f16c",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "rdrand",\
+        .value = "off",\
+    },
 
 #define PC_COMPAT_2_1 \
-        PC_COMPAT_2_2 \
-        HW_COMPAT_2_1
+    PC_COMPAT_2_2 \
+    HW_COMPAT_2_1 \
+    {\
+        .driver = "coreduo" "-" TYPE_X86_CPU,\
+        .property = "vmx",\
+        .value = "on",\
+    },\
+    {\
+        .driver = "core2duo" "-" TYPE_X86_CPU,\
+        .property = "vmx",\
+        .value = "on",\
+    },
 
 #define PC_COMPAT_2_0 \
-        PC_COMPAT_2_1 \
-        {\
-            .driver   = "virtio-scsi-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "PIIX4_PM",\
-            .property = "memory-hotplug-support",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "apic",\
-            .property = "version",\
-            .value    = stringify(0x11),\
-        },\
-        {\
-            .driver   = "nec-usb-xhci",\
-            .property = "superspeed-ports-first",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "nec-usb-xhci",\
-            .property = "force-pcie-endcap",\
-            .value    = "on",\
-        },\
-        {\
-            .driver   = "pci-serial",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "pci-serial-2x",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "pci-serial-4x",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "virtio-net-pci",\
-            .property = "guest_announce",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "ICH9-LPC",\
-            .property = "memory-hotplug-support",\
-            .value    = "off",\
-        },{\
-            .driver   = "xio3130-downstream",\
-            .property = COMPAT_PROP_PCP,\
-            .value    = "off",\
-        },{\
-            .driver   = "ioh3420",\
-            .property = COMPAT_PROP_PCP,\
-            .value    = "off",\
-        },
+    PC_COMPAT_2_1 \
+    {\
+        .driver   = "virtio-scsi-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "PIIX4_PM",\
+        .property = "memory-hotplug-support",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "apic",\
+        .property = "version",\
+        .value    = stringify(0x11),\
+    },\
+    {\
+        .driver   = "nec-usb-xhci",\
+        .property = "superspeed-ports-first",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "nec-usb-xhci",\
+        .property = "force-pcie-endcap",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "pci-serial",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "pci-serial-2x",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "pci-serial-4x",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "virtio-net-pci",\
+        .property = "guest_announce",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "ICH9-LPC",\
+        .property = "memory-hotplug-support",\
+        .value    = "off",\
+    },{\
+        .driver   = "xio3130-downstream",\
+        .property = COMPAT_PROP_PCP,\
+        .value    = "off",\
+    },{\
+        .driver   = "ioh3420",\
+        .property = COMPAT_PROP_PCP,\
+        .value    = "off",\
+    },
 
 #define PC_COMPAT_1_7 \
-        PC_COMPAT_2_0 \
-        {\
-            .driver   = TYPE_USB_DEVICE,\
-            .property = "msos-desc",\
-            .value    = "no",\
-        },\
-        {\
-            .driver   = "PIIX4_PM",\
-            .property = "acpi-pci-hotplug-with-bridge-support",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "hpet",\
-            .property = HPET_INTCAP,\
-            .value    = stringify(4),\
-        },
+    PC_COMPAT_2_0 \
+    {\
+        .driver   = TYPE_USB_DEVICE,\
+        .property = "msos-desc",\
+        .value    = "no",\
+    },\
+    {\
+        .driver   = "PIIX4_PM",\
+        .property = "acpi-pci-hotplug-with-bridge-support",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "hpet",\
+        .property = HPET_INTCAP,\
+        .value    = stringify(4),\
+    },
 
 #define PC_COMPAT_1_6 \
-        PC_COMPAT_1_7 \
-        {\
-            .driver   = "e1000",\
-            .property = "mitigation",\
-            .value    = "off",\
-        },{\
-            .driver   = "qemu64-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "qemu32-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(3),\
-        },{\
-            .driver   = "i440FX-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "q35-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(1),\
-        },
+    PC_COMPAT_1_7 \
+    {\
+        .driver   = "e1000",\
+        .property = "mitigation",\
+        .value    = "off",\
+    },{\
+        .driver   = "qemu64-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "qemu32-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(3),\
+    },{\
+        .driver   = "i440FX-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "q35-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(1),\
+    },
 
 #define PC_COMPAT_1_5 \
-        PC_COMPAT_1_6 \
-        {\
-            .driver   = "Conroe-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Conroe-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Penryn-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Penryn-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Nehalem-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Nehalem-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "virtio-net-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver = TYPE_X86_CPU,\
-            .property = "pmu",\
-            .value = "on",\
-        },{\
-            .driver   = "i440FX-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "q35-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(0),\
-        },
+    PC_COMPAT_1_6 \
+    {\
+        .driver   = "Conroe-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Conroe-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Penryn-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Penryn-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Nehalem-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Nehalem-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "virtio-net-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver = TYPE_X86_CPU,\
+        .property = "pmu",\
+        .value = "on",\
+    },{\
+        .driver   = "i440FX-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "q35-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(0),\
+    },
 
 #define PC_COMPAT_1_4 \
-        PC_COMPAT_1_5 \
-        {\
-            .driver   = "scsi-hd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "scsi-cd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "scsi-disk",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "ide-hd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "ide-cd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "ide-drive",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "virtio-blk-pci",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "virtio-serial-pci",\
-            .property = "vectors",\
-            /* DEV_NVECTORS_UNSPECIFIED as a uint32_t string */\
-            .value    = stringify(0xFFFFFFFF),\
-        },{ \
-            .driver   = "virtio-net-pci", \
-            .property = "ctrl_guest_offloads", \
-            .value    = "off", \
-        },{\
-            .driver   = "e1000",\
-            .property = "romfile",\
-            .value    = "pxe-e1000.rom",\
-        },{\
-            .driver   = "ne2k_pci",\
-            .property = "romfile",\
-            .value    = "pxe-ne2k_pci.rom",\
-        },{\
-            .driver   = "pcnet",\
-            .property = "romfile",\
-            .value    = "pxe-pcnet.rom",\
-        },{\
-            .driver   = "rtl8139",\
-            .property = "romfile",\
-            .value    = "pxe-rtl8139.rom",\
-        },{\
-            .driver   = "virtio-net-pci",\
-            .property = "romfile",\
-            .value    = "pxe-virtio.rom",\
-        },{\
-            .driver   = "486-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(0),\
-        },
-
-static inline void pc_common_machine_options(MachineClass *m)
-{
-    m->default_boot_order = "cad";
-}
-
-static inline void pc_default_machine_options(MachineClass *m)
-{
-    pc_common_machine_options(m);
-    m->hot_add_cpu = pc_hot_add_cpu;
-    m->max_cpus = 255;
-}
+    PC_COMPAT_1_5 \
+    {\
+        .driver   = "scsi-hd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "scsi-cd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "scsi-disk",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-hd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-cd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-drive",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "virtio-blk-pci",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "virtio-serial-pci",\
+        .property = "vectors",\
+        /* DEV_NVECTORS_UNSPECIFIED as a uint32_t string */\
+        .value    = stringify(0xFFFFFFFF),\
+    },{ \
+        .driver   = "virtio-net-pci", \
+        .property = "ctrl_guest_offloads", \
+        .value    = "off", \
+    },{\
+        .driver   = "e1000",\
+        .property = "romfile",\
+        .value    = "pxe-e1000.rom",\
+    },{\
+        .driver   = "ne2k_pci",\
+        .property = "romfile",\
+        .value    = "pxe-ne2k_pci.rom",\
+    },{\
+        .driver   = "pcnet",\
+        .property = "romfile",\
+        .value    = "pxe-pcnet.rom",\
+    },{\
+        .driver   = "rtl8139",\
+        .property = "romfile",\
+        .value    = "pxe-rtl8139.rom",\
+    },{\
+        .driver   = "virtio-net-pci",\
+        .property = "romfile",\
+        .value    = "pxe-virtio.rom",\
+    },{\
+        .driver   = "486-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver = "n270" "-" TYPE_X86_CPU,\
+        .property = "movbe",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "pclmulqdq",\
+        .value = "off",\
+    },
 
 #define DEFINE_PC_MACHINE(suffix, namestr, initfn, optsfn) \
     static void pc_machine_##suffix##_class_init(ObjectClass *oc, void *data) \
@@ -620,14 +857,7 @@ static inline void pc_default_machine_options(MachineClass *m)
     { \
         type_register(&pc_machine_type_##suffix); \
     } \
-    machine_init(pc_machine_init_##suffix)
-
-#define SET_MACHINE_COMPAT(m, COMPAT) do { \
-    static GlobalProperty props[] = { \
-        COMPAT \
-        { /* end of list */ } \
-    }; \
-    (m)->compat_props = props; \
-} while (0)
+    type_init(pc_machine_init_##suffix)
 
+extern void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id);
 #endif
diff --git a/qemu/include/hw/i386/topology.h b/qemu/include/hw/i386/topology.h
index 9c6f3a937..fc9557239 100644
--- a/qemu/include/hw/i386/topology.h
+++ b/qemu/include/hw/i386/topology.h
@@ -38,8 +38,6 @@
  *  CPUID Fn8000_0008_ECX[ApicIdCoreIdSize[3:0]] is set to apicid_core_width().
  */
 
-#include <stdint.h>
-#include <string.h>
 
 #include "qemu/bitops.h"
 
@@ -47,6 +45,12 @@
  */
 typedef uint32_t apic_id_t;
 
+typedef struct X86CPUTopoInfo {
+    unsigned pkg_id;
+    unsigned core_id;
+    unsigned smt_id;
+} X86CPUTopoInfo;
+
 /* Return the bit width needed for 'count' IDs
  */
 static unsigned apicid_bitwidth_for_count(unsigned count)
@@ -92,13 +96,11 @@ static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
  */
 static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
                                              unsigned nr_threads,
-                                             unsigned pkg_id,
-                                             unsigned core_id,
-                                             unsigned smt_id)
+                                             const X86CPUTopoInfo *topo)
 {
-    return (pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
-           (core_id << apicid_core_offset(nr_cores, nr_threads)) |
-           smt_id;
+    return (topo->pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
+           (topo->core_id << apicid_core_offset(nr_cores, nr_threads)) |
+           topo->smt_id;
 }
 
 /* Calculate thread/core/package IDs for a specific topology,
@@ -107,14 +109,12 @@ static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
 static inline void x86_topo_ids_from_idx(unsigned nr_cores,
                                          unsigned nr_threads,
                                          unsigned cpu_index,
-                                         unsigned *pkg_id,
-                                         unsigned *core_id,
-                                         unsigned *smt_id)
+                                         X86CPUTopoInfo *topo)
 {
     unsigned core_index = cpu_index / nr_threads;
-    *smt_id = cpu_index % nr_threads;
-    *core_id = core_index % nr_cores;
-    *pkg_id = core_index / nr_cores;
+    topo->smt_id = cpu_index % nr_threads;
+    topo->core_id = core_index % nr_cores;
+    topo->pkg_id = core_index / nr_cores;
 }
 
 /* Make APIC ID for the CPU 'cpu_index'
@@ -125,10 +125,9 @@ static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
                                                 unsigned nr_threads,
                                                 unsigned cpu_index)
 {
-    unsigned pkg_id, core_id, smt_id;
-    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index,
-                          &pkg_id, &core_id, &smt_id);
-    return apicid_from_topo_ids(nr_cores, nr_threads, pkg_id, core_id, smt_id);
+    X86CPUTopoInfo topo;
+    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index, &topo);
+    return apicid_from_topo_ids(nr_cores, nr_threads, &topo);
 }
 
 #endif /* HW_I386_TOPOLOGY_H */
diff --git a/qemu/include/hw/input/adb.h b/qemu/include/hw/input/adb.h
index bdfccd404..db51d0380 100644
--- a/qemu/include/hw/input/adb.h
+++ b/qemu/include/hw/input/adb.h
@@ -79,7 +79,7 @@ struct ADBBusState {
 
 int adb_request(ADBBusState *s, uint8_t *buf_out,
                 const uint8_t *buf, int len);
-int adb_poll(ADBBusState *s, uint8_t *buf_out);
+int adb_poll(ADBBusState *s, uint8_t *buf_out, uint16_t poll_mask);
 
 #define TYPE_ADB_KEYBOARD "adb-keyboard"
 #define TYPE_ADB_MOUSE "adb-mouse"
diff --git a/qemu/include/hw/intc/arm_gic_common.h b/qemu/include/hw/intc/arm_gic_common.h
index 899db3d7a..f4c349a2e 100644
--- a/qemu/include/hw/intc/arm_gic_common.h
+++ b/qemu/include/hw/intc/arm_gic_common.h
@@ -68,7 +68,6 @@ typedef struct GICState {
     uint8_t irq_target[GIC_MAXIRQ];
     uint8_t priority1[GIC_INTERNAL][GIC_NCPU];
     uint8_t priority2[GIC_MAXIRQ - GIC_INTERNAL];
-    uint16_t last_active[GIC_MAXIRQ][GIC_NCPU];
     /* For each SGI on the target CPU, we store 8 bits
      * indicating which source CPUs have made this SGI
      * pending on the target CPU. These correspond to
@@ -78,7 +77,6 @@ typedef struct GICState {
     uint8_t sgi_pending[GIC_NR_SGIS][GIC_NCPU];
 
     uint16_t priority_mask[GIC_NCPU];
-    uint16_t running_irq[GIC_NCPU];
     uint16_t running_priority[GIC_NCPU];
     uint16_t current_pending[GIC_NCPU];
 
@@ -96,16 +94,9 @@ typedef struct GICState {
      * If an interrupt for preemption level X is active, then
      *   APRn[X mod 32] == 0b1,  where n = X / 32
      * otherwise the bit is clear.
-     *
-     * TODO: rewrite the interrupt acknowlege/complete routines to use
-     * the APR registers to track the necessary information to update
-     * s->running_priority[] on interrupt completion (ie completely remove
-     * last_active[][] and running_irq[]). This will be necessary if we ever
-     * want to support TCG<->KVM migration, or TCG guests which can
-     * do power management involving powering down and restarting
-     * the GIC.
      */
     uint32_t apr[GIC_NR_APRS][GIC_NCPU];
+    uint32_t nsapr[GIC_NR_APRS][GIC_NCPU];
 
     uint32_t num_cpu;
 
@@ -118,7 +109,9 @@ typedef struct GICState {
     uint32_t num_irq;
     uint32_t revision;
     bool security_extn;
+    bool irq_reset_nonsecure; /* configure IRQs as group 1 (NS) on reset? */
     int dev_fd; /* kvm device fd if backed by kvm vgic support */
+    Error *migration_blocker;
 } GICState;
 
 #define TYPE_ARM_GIC_COMMON "arm_gic_common"
@@ -138,4 +131,7 @@ typedef struct ARMGICCommonClass {
     void (*post_load)(GICState *s);
 } ARMGICCommonClass;
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops);
+
 #endif
diff --git a/qemu/include/hw/intc/arm_gicv3_common.h b/qemu/include/hw/intc/arm_gicv3_common.h
new file mode 100644
index 000000000..c2fd8da4e
--- /dev/null
+++ b/qemu/include/hw/intc/arm_gicv3_common.h
@@ -0,0 +1,68 @@
+/*
+ * ARM GIC support
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Written by Peter Maydell
+ * Extended to 64 cores by Shlomo Pongratz
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HW_ARM_GICV3_COMMON_H
+#define HW_ARM_GICV3_COMMON_H
+
+#include "hw/sysbus.h"
+#include "hw/intc/arm_gic_common.h"
+
+typedef struct GICv3State {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    qemu_irq *parent_irq;
+    qemu_irq *parent_fiq;
+
+    MemoryRegion iomem_dist; /* Distributor */
+    MemoryRegion iomem_redist; /* Redistributors */
+
+    uint32_t num_cpu;
+    uint32_t num_irq;
+    uint32_t revision;
+    bool security_extn;
+
+    int dev_fd; /* kvm device fd if backed by kvm vgic support */
+} GICv3State;
+
+#define TYPE_ARM_GICV3_COMMON "arm-gicv3-common"
+#define ARM_GICV3_COMMON(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_CLASS(klass) \
+     OBJECT_CLASS_CHECK(ARMGICv3CommonClass, (klass), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(ARMGICv3CommonClass, (obj), TYPE_ARM_GICV3_COMMON)
+
+typedef struct ARMGICv3CommonClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    void (*pre_save)(GICv3State *s);
+    void (*post_load)(GICv3State *s);
+} ARMGICv3CommonClass;
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops);
+
+#endif
diff --git a/qemu/include/hw/intc/aspeed_vic.h b/qemu/include/hw/intc/aspeed_vic.h
new file mode 100644
index 000000000..107ff17c3
--- /dev/null
+++ b/qemu/include/hw/intc/aspeed_vic.h
@@ -0,0 +1,48 @@
+/*
+ * ASPEED Interrupt Controller (New)
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Need to add SVIC and CVIC support
+ */
+#ifndef ASPEED_VIC_H
+#define ASPEED_VIC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_ASPEED_VIC "aspeed.vic"
+#define ASPEED_VIC(obj) OBJECT_CHECK(AspeedVICState, (obj), TYPE_ASPEED_VIC)
+
+#define ASPEED_VIC_NR_IRQS 51
+
+typedef struct AspeedVICState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    qemu_irq irq;
+    qemu_irq fiq;
+
+    uint64_t level;
+    uint64_t raw;
+    uint64_t select;
+    uint64_t enable;
+    uint64_t trigger;
+
+    /* 0=edge, 1=level */
+    uint64_t sense;
+
+    /* 0=single-edge, 1=dual-edge */
+    uint64_t dual_edge;
+
+    /* 0=low-sensitive/falling-edge, 1=high-sensitive/rising-edge */
+    uint64_t event;
+} AspeedVICState;
+
+#endif /* ASPEED_VIC_H */
diff --git a/qemu/include/hw/intc/bcm2835_ic.h b/qemu/include/hw/intc/bcm2835_ic.h
new file mode 100644
index 000000000..fb75fa006
--- /dev/null
+++ b/qemu/include/hw/intc/bcm2835_ic.h
@@ -0,0 +1,33 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_IC_H
+#define BCM2835_IC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_BCM2835_IC "bcm2835-ic"
+#define BCM2835_IC(obj) OBJECT_CHECK(BCM2835ICState, (obj), TYPE_BCM2835_IC)
+
+#define BCM2835_IC_GPU_IRQ "gpu-irq"
+#define BCM2835_IC_ARM_IRQ "arm-irq"
+
+typedef struct BCM2835ICState {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion iomem;
+    qemu_irq irq;
+    qemu_irq fiq;
+
+    /* 64 GPU IRQs + 8 ARM IRQs = 72 total (GPU first) */
+    uint64_t gpu_irq_level, gpu_irq_enable;
+    uint8_t arm_irq_level, arm_irq_enable;
+    bool fiq_enable;
+    uint8_t fiq_select;
+} BCM2835ICState;
+
+#endif
diff --git a/qemu/include/hw/intc/bcm2836_control.h b/qemu/include/hw/intc/bcm2836_control.h
new file mode 100644
index 000000000..613f3c418
--- /dev/null
+++ b/qemu/include/hw/intc/bcm2836_control.h
@@ -0,0 +1,51 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2836_CONTROL_H
+#define BCM2836_CONTROL_H
+
+#include "hw/sysbus.h"
+
+/* 4 mailboxes per core, for 16 total */
+#define BCM2836_NCORES 4
+#define BCM2836_MBPERCORE 4
+
+#define TYPE_BCM2836_CONTROL "bcm2836-control"
+#define BCM2836_CONTROL(obj) \
+    OBJECT_CHECK(BCM2836ControlState, (obj), TYPE_BCM2836_CONTROL)
+
+typedef struct BCM2836ControlState {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+    MemoryRegion iomem;
+
+    /* mailbox state */
+    uint32_t mailboxes[BCM2836_NCORES * BCM2836_MBPERCORE];
+
+    /* interrupt routing/control registers */
+    uint8_t route_gpu_irq, route_gpu_fiq;
+    uint32_t timercontrol[BCM2836_NCORES];
+    uint32_t mailboxcontrol[BCM2836_NCORES];
+
+    /* interrupt status regs (derived from input pins; not visible to user) */
+    bool gpu_irq, gpu_fiq;
+    uint8_t timerirqs[BCM2836_NCORES];
+
+    /* interrupt source registers, post-routing (also input-derived; visible) */
+    uint32_t irqsrc[BCM2836_NCORES];
+    uint32_t fiqsrc[BCM2836_NCORES];
+
+    /* outputs to CPU cores */
+    qemu_irq irq[BCM2836_NCORES];
+    qemu_irq fiq[BCM2836_NCORES];
+} BCM2836ControlState;
+
+#endif
diff --git a/qemu/include/hw/intc/imx_avic.h b/qemu/include/hw/intc/imx_avic.h
new file mode 100644
index 000000000..1b8076901
--- /dev/null
+++ b/qemu/include/hw/intc/imx_avic.h
@@ -0,0 +1,55 @@
+/*
+ * i.MX31 Vectored Interrupt Controller
+ *
+ * Note this is NOT the PL192 provided by ARM, but
+ * a custom implementation by Freescale.
+ *
+ * Copyright (c) 2008 OKL
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ *
+ * TODO: implement vectors.
+ */
+#ifndef IMX_AVIC_H
+#define IMX_AVIC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_AVIC "imx.avic"
+#define IMX_AVIC(obj) OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
+
+#define IMX_AVIC_NUM_IRQS 64
+
+/* Interrupt Control Bits */
+#define ABFLAG (1<<25)
+#define ABFEN  (1<<24)
+#define NIDIS  (1<<22) /* Normal Interrupt disable */
+#define FIDIS  (1<<21) /* Fast interrupt disable */
+#define NIAD   (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
+#define FIAD   (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
+#define NM     (1<<18) /* Normal interrupt mode */
+
+#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
+#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
+
+typedef struct IMXAVICState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    uint64_t pending;
+    uint64_t enabled;
+    uint64_t is_fiq;
+    uint32_t intcntl;
+    uint32_t intmask;
+    qemu_irq irq;
+    qemu_irq fiq;
+    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
+} IMXAVICState;
+
+#endif /* IMX_AVIC_H */
diff --git a/qemu/include/hw/ipmi/ipmi.h b/qemu/include/hw/ipmi/ipmi.h
new file mode 100644
index 000000000..74a2b5af9
--- /dev/null
+++ b/qemu/include/hw/ipmi/ipmi.h
@@ -0,0 +1,258 @@
+/*
+ * IPMI base class
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_IPMI_H
+#define HW_IPMI_H
+
+#include "exec/memory.h"
+#include "qemu-common.h"
+#include "hw/qdev.h"
+
+#define MAX_IPMI_MSG_SIZE 300
+
+enum ipmi_op {
+    IPMI_RESET_CHASSIS,
+    IPMI_POWEROFF_CHASSIS,
+    IPMI_POWERON_CHASSIS,
+    IPMI_POWERCYCLE_CHASSIS,
+    IPMI_PULSE_DIAG_IRQ,
+    IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP,
+    IPMI_SEND_NMI
+};
+
+#define IPMI_CC_INVALID_CMD                              0xc1
+#define IPMI_CC_COMMAND_INVALID_FOR_LUN                  0xc2
+#define IPMI_CC_TIMEOUT                                  0xc3
+#define IPMI_CC_OUT_OF_SPACE                             0xc4
+#define IPMI_CC_INVALID_RESERVATION                      0xc5
+#define IPMI_CC_REQUEST_DATA_TRUNCATED                   0xc6
+#define IPMI_CC_REQUEST_DATA_LENGTH_INVALID              0xc7
+#define IPMI_CC_PARM_OUT_OF_RANGE                        0xc9
+#define IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES              0xca
+#define IPMI_CC_REQ_ENTRY_NOT_PRESENT                    0xcb
+#define IPMI_CC_INVALID_DATA_FIELD                       0xcc
+#define IPMI_CC_BMC_INIT_IN_PROGRESS                     0xd2
+#define IPMI_CC_COMMAND_NOT_SUPPORTED                    0xd5
+
+#define IPMI_NETFN_APP                0x06
+
+#define IPMI_DEBUG 1
+
+/* Specified in the SMBIOS spec. */
+#define IPMI_SMBIOS_KCS         0x01
+#define IPMI_SMBIOS_SMIC        0x02
+#define IPMI_SMBIOS_BT          0x03
+#define IPMI_SMBIOS_SSIF        0x04
+
+/* IPMI Interface types (KCS, SMIC, BT) are prefixed with this */
+#define TYPE_IPMI_INTERFACE_PREFIX "ipmi-interface-"
+
+/*
+ * An IPMI Interface, the interface for talking between the target
+ * and the BMC.
+ */
+#define TYPE_IPMI_INTERFACE "ipmi-interface"
+#define IPMI_INTERFACE(obj) \
+     INTERFACE_CHECK(IPMIInterface, (obj), TYPE_IPMI_INTERFACE)
+#define IPMI_INTERFACE_CLASS(class) \
+     OBJECT_CLASS_CHECK(IPMIInterfaceClass, (class), TYPE_IPMI_INTERFACE)
+#define IPMI_INTERFACE_GET_CLASS(class) \
+     OBJECT_GET_CLASS(IPMIInterfaceClass, (class), TYPE_IPMI_INTERFACE)
+
+typedef struct IPMIInterface {
+    Object parent;
+} IPMIInterface;
+
+typedef struct IPMIInterfaceClass {
+    InterfaceClass parent;
+
+    void (*init)(struct IPMIInterface *s, Error **errp);
+
+    /*
+     * Perform various operations on the hardware.  If checkonly is
+     * true, it will return if the operation can be performed, but it
+     * will not do the operation.
+     */
+    int (*do_hw_op)(struct IPMIInterface *s, enum ipmi_op op, int checkonly);
+
+    /*
+     * Enable/disable irqs on the interface when the BMC requests this.
+     */
+    void (*set_irq_enable)(struct IPMIInterface *s, int val);
+
+    /*
+     * Handle an event that occurred on the interface, generally the.
+     * target writing to a register.
+     */
+    void (*handle_if_event)(struct IPMIInterface *s);
+
+    /*
+     * The interfaces use this to perform certain ops
+     */
+    void (*set_atn)(struct IPMIInterface *s, int val, int irq);
+
+    /*
+     * Got an IPMI warm/cold reset.
+     */
+    void (*reset)(struct IPMIInterface *s, bool is_cold);
+
+    /*
+     * Handle a response from the bmc.
+     */
+    void (*handle_rsp)(struct IPMIInterface *s, uint8_t msg_id,
+                       unsigned char *rsp, unsigned int rsp_len);
+
+    /*
+     * Set by the owner to hold the backend data for the interface.
+     */
+    void *(*get_backend_data)(struct IPMIInterface *s);
+} IPMIInterfaceClass;
+
+/*
+ * Define a BMC simulator (or perhaps a connection to a real BMC)
+ */
+#define TYPE_IPMI_BMC "ipmi-bmc"
+#define IPMI_BMC(obj) \
+     OBJECT_CHECK(IPMIBmc, (obj), TYPE_IPMI_BMC)
+#define IPMI_BMC_CLASS(obj_class) \
+     OBJECT_CLASS_CHECK(IPMIBmcClass, (obj_class), TYPE_IPMI_BMC)
+#define IPMI_BMC_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IPMIBmcClass, (obj), TYPE_IPMI_BMC)
+
+typedef struct IPMIBmc {
+    DeviceState parent;
+
+    uint8_t slave_addr;
+
+    IPMIInterface *intf;
+} IPMIBmc;
+
+typedef struct IPMIBmcClass {
+    DeviceClass parent;
+
+    /* Called when the system resets to report to the bmc. */
+    void (*handle_reset)(struct IPMIBmc *s);
+
+    /*
+     * Handle a command to the bmc.
+     */
+    void (*handle_command)(struct IPMIBmc *s,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           unsigned int max_cmd_len,
+                           uint8_t msg_id);
+} IPMIBmcClass;
+
+/*
+ * Add a link property to obj that points to a BMC.
+ */
+void ipmi_bmc_find_and_link(Object *obj, Object **bmc);
+
+/*
+ * Used for transferring information to interfaces that add
+ * entries to firmware tables.
+ */
+typedef struct IPMIFwInfo {
+    const char *interface_name;
+    int interface_type;
+    uint8_t ipmi_spec_major_revision;
+    uint8_t ipmi_spec_minor_revision;
+    uint8_t i2c_slave_address;
+    uint32_t uuid;
+
+    uint64_t base_address;
+    uint64_t register_length;
+    uint8_t register_spacing;
+    enum {
+        IPMI_MEMSPACE_IO,
+        IPMI_MEMSPACE_MEM32,
+        IPMI_MEMSPACE_MEM64,
+        IPMI_MEMSPACE_SMBUS
+    } memspace;
+
+    int interrupt_number;
+    enum {
+        IPMI_LEVEL_IRQ,
+        IPMI_EDGE_IRQ
+    } irq_type;
+
+    const char *acpi_parent;
+} IPMIFwInfo;
+
+void ipmi_add_fwinfo(IPMIFwInfo *info, Error **errp);
+IPMIFwInfo *ipmi_first_fwinfo(void);
+IPMIFwInfo *ipmi_next_fwinfo(IPMIFwInfo *current);
+
+#ifdef IPMI_DEBUG
+#define ipmi_debug(fs, ...) \
+    fprintf(stderr, "IPMI (%s): " fs, __func__, ##__VA_ARGS__)
+#else
+#define ipmi_debug(fs, ...)
+#endif
+
+struct ipmi_sdr_header {
+    uint8_t  rec_id[2];
+    uint8_t  sdr_version;               /* 0x51 */
+    uint8_t  rec_type;
+    uint8_t  rec_length;
+};
+#define IPMI_SDR_HEADER_SIZE     sizeof(struct ipmi_sdr_header)
+
+#define ipmi_sdr_recid(sdr) ((sdr)->rec_id[0] | ((sdr)->rec_id[1] << 8))
+#define ipmi_sdr_length(sdr) ((sdr)->rec_length + IPMI_SDR_HEADER_SIZE)
+
+/*
+ * 43.2 SDR Type 02h. Compact Sensor Record
+ */
+#define IPMI_SDR_COMPACT_TYPE    2
+
+struct ipmi_sdr_compact {
+    struct ipmi_sdr_header header;
+
+    uint8_t  sensor_owner_id;
+    uint8_t  sensor_owner_lun;
+    uint8_t  sensor_owner_number;       /* byte 8 */
+    uint8_t  entity_id;
+    uint8_t  entity_instance;
+    uint8_t  sensor_init;
+    uint8_t  sensor_caps;
+    uint8_t  sensor_type;
+    uint8_t  reading_type;
+    uint8_t  assert_mask[2];            /* byte 16 */
+    uint8_t  deassert_mask[2];
+    uint8_t  discrete_mask[2];
+    uint8_t  sensor_unit1;
+    uint8_t  sensor_unit2;
+    uint8_t  sensor_unit3;
+    uint8_t  sensor_direction[2];       /* byte 24 */
+    uint8_t  positive_threshold;
+    uint8_t  negative_threshold;
+    uint8_t  reserved[3];
+    uint8_t  oem;
+    uint8_t  id_str_len;                /* byte 32 */
+    uint8_t  id_string[16];
+};
+
+typedef uint8_t ipmi_sdr_compact_buffer[sizeof(struct ipmi_sdr_compact)];
+
+#endif
diff --git a/qemu/include/hw/isa/apm.h b/qemu/include/hw/isa/apm.h
index 3edea5f62..4839ff1df 100644
--- a/qemu/include/hw/isa/apm.h
+++ b/qemu/include/hw/isa/apm.h
@@ -1,7 +1,6 @@
 #ifndef APM_H
 #define APM_H
 
-#include <stdint.h>
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "exec/memory.h"
diff --git a/qemu/include/hw/isa/i8257.h b/qemu/include/hw/isa/i8257.h
new file mode 100644
index 000000000..8c44d3628
--- /dev/null
+++ b/qemu/include/hw/isa/i8257.h
@@ -0,0 +1,42 @@
+#ifndef HW_I8257_H
+#define HW_I8257_H
+
+#define TYPE_I8257 "i8257"
+
+typedef struct I8257Regs {
+    int now[2];
+    uint16_t base[2];
+    uint8_t mode;
+    uint8_t page;
+    uint8_t pageh;
+    uint8_t dack;
+    uint8_t eop;
+    IsaDmaTransferHandler transfer_handler;
+    void *opaque;
+} I8257Regs;
+
+typedef struct I8257State {
+    /* <private> */
+    ISADevice parent_obj;
+
+    /* <public> */
+    int32_t base;
+    int32_t page_base;
+    int32_t pageh_base;
+    int32_t dshift;
+
+    uint8_t status;
+    uint8_t command;
+    uint8_t mask;
+    uint8_t flip_flop;
+    I8257Regs regs[4];
+    MemoryRegion channel_io;
+    MemoryRegion cont_io;
+
+    QEMUBH *dma_bh;
+    bool dma_bh_scheduled;
+    int running;
+} I8257State;
+
+#endif
+
diff --git a/qemu/include/hw/isa/isa.h b/qemu/include/hw/isa/isa.h
index f21ceaafc..ffb2ea7cd 100644
--- a/qemu/include/hw/isa/isa.h
+++ b/qemu/include/hw/isa/isa.h
@@ -34,6 +34,44 @@ static inline uint16_t applesmc_port(void)
     return 0;
 }
 
+#define TYPE_ISADMA "isa-dma"
+
+#define ISADMA_CLASS(klass) \
+    OBJECT_CLASS_CHECK(IsaDmaClass, (klass), TYPE_ISADMA)
+#define ISADMA_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(IsaDmaClass, (obj), TYPE_ISADMA)
+#define ISADMA(obj) \
+    INTERFACE_CHECK(IsaDma, (obj), TYPE_ISADMA)
+
+struct IsaDma {
+    Object parent;
+};
+
+typedef enum {
+    ISADMA_TRANSFER_VERIFY,
+    ISADMA_TRANSFER_READ,
+    ISADMA_TRANSFER_WRITE,
+    ISADMA_TRANSFER_ILLEGAL,
+} IsaDmaTransferMode;
+
+typedef int (*IsaDmaTransferHandler)(void *opaque, int nchan, int pos,
+                                     int size);
+
+typedef struct IsaDmaClass {
+    InterfaceClass parent;
+
+    IsaDmaTransferMode (*get_transfer_mode)(IsaDma *obj, int nchan);
+    bool (*has_autoinitialization)(IsaDma *obj, int nchan);
+    int (*read_memory)(IsaDma *obj, int nchan, void *buf, int pos, int len);
+    int (*write_memory)(IsaDma *obj, int nchan, void *buf, int pos, int len);
+    void (*hold_DREQ)(IsaDma *obj, int nchan);
+    void (*release_DREQ)(IsaDma *obj, int nchan);
+    void (*schedule)(IsaDma *obj);
+    void (*register_channel)(IsaDma *obj, int nchan,
+                             IsaDmaTransferHandler transfer_handler,
+                             void *opaque);
+} IsaDmaClass;
+
 typedef struct ISADeviceClass {
     DeviceClass parent_class;
 } ISADeviceClass;
@@ -46,6 +84,7 @@ struct ISABus {
     MemoryRegion *address_space;
     MemoryRegion *address_space_io;
     qemu_irq *irqs;
+    IsaDma *dma[2];
 };
 
 struct ISADevice {
@@ -59,10 +98,12 @@ struct ISADevice {
 };
 
 ISABus *isa_bus_new(DeviceState *dev, MemoryRegion *address_space,
-                    MemoryRegion *address_space_io);
+                    MemoryRegion *address_space_io, Error **errp);
 void isa_bus_irqs(ISABus *bus, qemu_irq *irqs);
 qemu_irq isa_get_irq(ISADevice *dev, int isairq);
 void isa_init_irq(ISADevice *dev, qemu_irq *p, int isairq);
+void isa_bus_dma(ISABus *bus, IsaDma *dma8, IsaDma *dma16);
+IsaDma *isa_get_dma(ISABus *bus, int nchan);
 MemoryRegion *isa_address_space(ISADevice *dev);
 MemoryRegion *isa_address_space_io(ISADevice *dev);
 ISADevice *isa_create(ISABus *bus, const char *name);
@@ -106,15 +147,6 @@ static inline ISABus *isa_bus_from_device(ISADevice *d)
     return ISA_BUS(qdev_get_parent_bus(DEVICE(d)));
 }
 
-/* dma.c */
-int DMA_get_channel_mode (int nchan);
-int DMA_read_memory (int nchan, void *buf, int pos, int size);
-int DMA_write_memory (int nchan, void *buf, int pos, int size);
-void DMA_hold_DREQ (int nchan);
-void DMA_release_DREQ (int nchan);
-void DMA_schedule(int nchan);
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit);
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque);
+/* i8257.c */
+void DMA_init(ISABus *bus, int high_page_enable);
 #endif
diff --git a/qemu/include/hw/loader.h b/qemu/include/hw/loader.h
index f7b43ab62..4879b63a2 100644
--- a/qemu/include/hw/loader.h
+++ b/qemu/include/hw/loader.h
@@ -16,6 +16,18 @@ int load_image(const char *filename, uint8_t *addr); /* deprecated */
 ssize_t load_image_size(const char *filename, void *addr, size_t size);
 int load_image_targphys(const char *filename, hwaddr,
                         uint64_t max_sz);
+/**
+ * load_image_mr: load an image into a memory region
+ * @filename: Path to the image file
+ * @mr: Memory Region to load into
+ *
+ * Load the specified file into the memory region.
+ * The file loaded is registered as a ROM, so its contents will be
+ * reinstated whenever the system is reset.
+ * If the file is larger than the memory region's size the call will fail.
+ * Returns -1 on failure, or the size of the file.
+ */
+int load_image_mr(const char *filename, MemoryRegion *mr);
 
 /* This is the limit on the maximum uncompressed image size that
  * load_image_gzipped_buffer() and load_image_gzipped() will read. It prevents
@@ -32,10 +44,49 @@ int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz);
 #define ELF_LOAD_WRONG_ARCH   -3
 #define ELF_LOAD_WRONG_ENDIAN -4
 const char *load_elf_strerror(int error);
+
+/** load_elf:
+ * @filename: Path of ELF file
+ * @translate_fn: optional function to translate load addresses
+ * @translate_opaque: opaque data passed to @translate_fn
+ * @pentry: Populated with program entry point. Ignored if NULL.
+ * @lowaddr: Populated with lowest loaded address. Ignored if NULL.
+ * @highaddr: Populated with highest loaded address. Ignored if NULL.
+ * @bigendian: Expected ELF endianness. 0 for LE otherwise BE
+ * @elf_machine: Expected ELF machine type
+ * @clear_lsb: Set to mask off LSB of addresses (Some architectures use
+ *             this for non-address data)
+ * @data_swab: Set to order of byte swapping for data. 0 for no swap, 1
+ *             for swapping bytes within halfwords, 2 for bytes within
+ *             words and 3 for within doublewords.
+ *
+ * Load an ELF file's contents to the emulated system's address space.
+ * Clients may optionally specify a callback to perform address
+ * translations. @pentry, @lowaddr and @highaddr are optional pointers
+ * which will be populated with various load information. @bigendian and
+ * @elf_machine give the expected endianness and machine for the ELF the
+ * load will fail if the target ELF does not match. Some architectures
+ * have some architecture-specific behaviours that come into effect when
+ * their particular values for @elf_machine are set.
+ */
+
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
              uint64_t *highaddr, int big_endian, int elf_machine,
-             int clear_lsb);
+             int clear_lsb, int data_swab);
+
+/** load_elf_hdr:
+ * @filename: Path of ELF file
+ * @hdr: Buffer to populate with header data. Header data will not be
+ * filled if set to NULL.
+ * @is64: Set to true if the ELF is 64bit. Ignored if set to NULL
+ * @errp: Populated with an error in failure cases
+ *
+ * Inspect an ELF file's header. Read its full header contents into a
+ * buffer and/or determine if the ELF is 64bit.
+ */
+void load_elf_hdr(const char *filename, void *hdr, bool *is64, Error **errp);
+
 int load_aout(const char *filename, hwaddr addr, int max_sz,
               int bswap_needed, hwaddr target_page_size);
 int load_uimage(const char *filename, hwaddr *ep,
@@ -67,7 +118,7 @@ extern bool rom_file_has_mr;
 
 int rom_add_file(const char *file, const char *fw_dir,
                  hwaddr addr, int32_t bootindex,
-                 bool option_rom);
+                 bool option_rom, MemoryRegion *mr);
 MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len,
                            size_t max_len, hwaddr addr,
                            const char *fw_file_name,
@@ -77,14 +128,18 @@ int rom_add_elf_program(const char *name, void *data, size_t datasize,
                         size_t romsize, hwaddr addr);
 int rom_check_and_register_reset(void);
 void rom_set_fw(FWCfgState *f);
+void rom_set_order_override(int order);
+void rom_reset_order_override(void);
 int rom_copy(uint8_t *dest, hwaddr addr, size_t size);
 void *rom_ptr(hwaddr addr);
 void hmp_info_roms(Monitor *mon, const QDict *qdict);
 
 #define rom_add_file_fixed(_f, _a, _i)          \
-    rom_add_file(_f, NULL, _a, _i, false)
+    rom_add_file(_f, NULL, _a, _i, false, NULL)
 #define rom_add_blob_fixed(_f, _b, _l, _a)      \
     rom_add_blob(_f, _b, _l, _l, _a, NULL, NULL, NULL)
+#define rom_add_file_mr(_f, _mr, _i)            \
+    rom_add_file(_f, NULL, 0, _i, false, _mr)
 
 #define PC_ROM_MIN_VGA     0xc0000
 #define PC_ROM_MIN_OPTION  0xc8000
diff --git a/qemu/include/hw/mem/nvdimm.h b/qemu/include/hw/mem/nvdimm.h
new file mode 100644
index 000000000..517de9c36
--- /dev/null
+++ b/qemu/include/hw/mem/nvdimm.h
@@ -0,0 +1,62 @@
+/*
+ * Non-Volatile Dual In-line Memory Module Virtualization Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * NVDIMM specifications and some documents can be found at:
+ * NVDIMM ACPI device and NFIT are introduced in ACPI 6:
+ *      http://www.uefi.org/sites/default/files/resources/ACPI_6.0.pdf
+ * NVDIMM Namespace specification:
+ *      http://pmem.io/documents/NVDIMM_Namespace_Spec.pdf
+ * DSM Interface Example:
+ *      http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf
+ * Driver Writer's Guide:
+ *      http://pmem.io/documents/NVDIMM_Driver_Writers_Guide.pdf
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_NVDIMM_H
+#define QEMU_NVDIMM_H
+
+#include "hw/mem/pc-dimm.h"
+
+#define NVDIMM_DEBUG 0
+#define nvdimm_debug(fmt, ...)                                \
+    do {                                                      \
+        if (NVDIMM_DEBUG) {                                   \
+            fprintf(stderr, "nvdimm: " fmt, ## __VA_ARGS__);  \
+        }                                                     \
+    } while (0)
+
+#define TYPE_NVDIMM             "nvdimm"
+
+#define NVDIMM_DSM_MEM_FILE     "etc/acpi/nvdimm-mem"
+
+/*
+ * 32 bits IO port starting from 0x0a18 in guest is reserved for
+ * NVDIMM ACPI emulation.
+ */
+#define NVDIMM_ACPI_IO_BASE     0x0a18
+#define NVDIMM_ACPI_IO_LEN      4
+
+struct AcpiNVDIMMState {
+    /* detect if NVDIMM support is enabled. */
+    bool is_enabled;
+
+    /* the data of the fw_cfg file NVDIMM_DSM_MEM_FILE. */
+    GArray *dsm_mem;
+    /* the IO region used by OSPM to transfer control to QEMU. */
+    MemoryRegion io_mr;
+};
+typedef struct AcpiNVDIMMState AcpiNVDIMMState;
+
+void nvdimm_init_acpi_state(AcpiNVDIMMState *state, MemoryRegion *io,
+                            FWCfgState *fw_cfg, Object *owner);
+void nvdimm_build_acpi(GArray *table_offsets, GArray *table_data,
+                       GArray *linker);
+#endif
diff --git a/qemu/include/hw/mem/pc-dimm.h b/qemu/include/hw/mem/pc-dimm.h
index d83bf30ea..218dfb0ed 100644
--- a/qemu/include/hw/mem/pc-dimm.h
+++ b/qemu/include/hw/mem/pc-dimm.h
@@ -77,7 +77,7 @@ typedef struct PCDIMMDeviceClass {
  * @mr: hotplug memory address space container
  */
 typedef struct MemoryHotplugState {
-    ram_addr_t base;
+    hwaddr base;
     MemoryRegion mr;
 } MemoryHotplugState;
 
diff --git a/qemu/include/hw/mips/cps.h b/qemu/include/hw/mips/cps.h
new file mode 100644
index 000000000..4dbae9c8c
--- /dev/null
+++ b/qemu/include/hw/mips/cps.h
@@ -0,0 +1,46 @@
+/*
+ * Coherent Processing System emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_CPS_H
+#define MIPS_CPS_H
+
+#include "hw/sysbus.h"
+#include "hw/misc/mips_cmgcr.h"
+#include "hw/misc/mips_cpc.h"
+#include "hw/misc/mips_itu.h"
+
+#define TYPE_MIPS_CPS "mips-cps"
+#define MIPS_CPS(obj) OBJECT_CHECK(MIPSCPSState, (obj), TYPE_MIPS_CPS)
+
+typedef struct MIPSCPSState {
+    SysBusDevice parent_obj;
+
+    uint32_t num_vp;
+    uint32_t num_irq;
+    char *cpu_model;
+
+    MemoryRegion container;
+    MIPSGCRState gcr;
+    MIPSCPCState cpc;
+    MIPSITUState itu;
+} MIPSCPSState;
+
+qemu_irq get_cps_irq(MIPSCPSState *cps, int pin_number);
+
+#endif
diff --git a/qemu/include/hw/misc/bcm2835_mbox.h b/qemu/include/hw/misc/bcm2835_mbox.h
new file mode 100644
index 000000000..f4e9ff9ef
--- /dev/null
+++ b/qemu/include/hw/misc/bcm2835_mbox.h
@@ -0,0 +1,38 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_MBOX_H
+#define BCM2835_MBOX_H
+
+#include "bcm2835_mbox_defs.h"
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+
+#define TYPE_BCM2835_MBOX "bcm2835-mbox"
+#define BCM2835_MBOX(obj) \
+        OBJECT_CHECK(BCM2835MboxState, (obj), TYPE_BCM2835_MBOX)
+
+typedef struct {
+    uint32_t reg[MBOX_SIZE];
+    uint32_t count;
+    uint32_t status;
+    uint32_t config;
+} BCM2835Mbox;
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+    MemoryRegion *mbox_mr;
+    AddressSpace mbox_as;
+    MemoryRegion iomem;
+    qemu_irq arm_irq;
+
+    bool mbox_irq_disabled;
+    bool available[MBOX_CHAN_COUNT];
+    BCM2835Mbox mbox[2];
+} BCM2835MboxState;
+
+#endif
diff --git a/qemu/include/hw/misc/bcm2835_mbox_defs.h b/qemu/include/hw/misc/bcm2835_mbox_defs.h
new file mode 100644
index 000000000..a18e520b2
--- /dev/null
+++ b/qemu/include/hw/misc/bcm2835_mbox_defs.h
@@ -0,0 +1,27 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_MBOX_DEFS_H
+#define BCM2835_MBOX_DEFS_H
+
+/* Constants shared with the ARM identifying separate mailbox channels */
+#define MBOX_CHAN_POWER    0 /* for use by the power management interface */
+#define MBOX_CHAN_FB       1 /* for use by the frame buffer */
+#define MBOX_CHAN_VCHIQ    3 /* for use by the VCHIQ interface */
+#define MBOX_CHAN_PROPERTY 8 /* for use by the property channel */
+#define MBOX_CHAN_COUNT    9
+
+#define MBOX_SIZE          32
+#define MBOX_INVALID_DATA  0x0f
+
+/* Layout of the private address space used for communication between
+ * the mbox device emulation, and child devices: each channel occupies
+ * 16 bytes of address space, but only two registers are presently defined.
+ */
+#define MBOX_AS_CHAN_SHIFT 4
+#define MBOX_AS_DATA       0 /* request / response data (RW at offset 0) */
+#define MBOX_AS_PENDING    4 /* pending response status (RO at offset 4) */
+
+#endif /* BCM2835_MBOX_DEFS_H */
diff --git a/qemu/include/hw/misc/bcm2835_property.h b/qemu/include/hw/misc/bcm2835_property.h
new file mode 100644
index 000000000..edcab603c
--- /dev/null
+++ b/qemu/include/hw/misc/bcm2835_property.h
@@ -0,0 +1,35 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_PROPERTY_H
+#define BCM2835_PROPERTY_H
+
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "net/net.h"
+#include "hw/display/bcm2835_fb.h"
+
+#define TYPE_BCM2835_PROPERTY "bcm2835-property"
+#define BCM2835_PROPERTY(obj) \
+        OBJECT_CHECK(BCM2835PropertyState, (obj), TYPE_BCM2835_PROPERTY)
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+    MemoryRegion iomem;
+    qemu_irq mbox_irq;
+    BCM2835FBState *fbdev;
+
+    MACAddr macaddr;
+    uint32_t board_rev;
+    uint32_t addr;
+    bool pending;
+} BCM2835PropertyState;
+
+#endif
diff --git a/qemu/include/hw/misc/imx25_ccm.h b/qemu/include/hw/misc/imx25_ccm.h
new file mode 100644
index 000000000..296321c61
--- /dev/null
+++ b/qemu/include/hw/misc/imx25_ccm.h
@@ -0,0 +1,79 @@
+/*
+ * IMX25 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX25_CCM_H
+#define IMX25_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+
+#define IMX25_CCM_MPCTL_REG  0
+#define IMX25_CCM_UPCTL_REG  1
+#define IMX25_CCM_CCTL_REG   2
+#define IMX25_CCM_CGCR0_REG  3
+#define IMX25_CCM_CGCR1_REG  4
+#define IMX25_CCM_CGCR2_REG  5
+#define IMX25_CCM_PCDR0_REG  6
+#define IMX25_CCM_PCDR1_REG  7
+#define IMX25_CCM_PCDR2_REG  8
+#define IMX25_CCM_PCDR3_REG  9
+#define IMX25_CCM_RCSR_REG   10
+#define IMX25_CCM_CRDR_REG   11
+#define IMX25_CCM_DCVR0_REG  12
+#define IMX25_CCM_DCVR1_REG  13
+#define IMX25_CCM_DCVR2_REG  14
+#define IMX25_CCM_DCVR3_REG  15
+#define IMX25_CCM_LTR0_REG   16
+#define IMX25_CCM_LTR1_REG   17
+#define IMX25_CCM_LTR2_REG   18
+#define IMX25_CCM_LTR3_REG   19
+#define IMX25_CCM_LTBR0_REG  20
+#define IMX25_CCM_LTBR1_REG  21
+#define IMX25_CCM_PMCR0_REG  22
+#define IMX25_CCM_PMCR1_REG  23
+#define IMX25_CCM_PMCR2_REG  24
+#define IMX25_CCM_MCR_REG    25
+#define IMX25_CCM_LPIMR0_REG 26
+#define IMX25_CCM_LPIMR1_REG 27
+#define IMX25_CCM_MAX_REG    28
+
+/* CCTL */
+#define CCTL_ARM_CLK_DIV_SHIFT (30)
+#define CCTL_ARM_CLK_DIV_MASK  (0x3)
+#define CCTL_AHB_CLK_DIV_SHIFT (28)
+#define CCTL_AHB_CLK_DIV_MASK  (0x3)
+#define CCTL_MPLL_BYPASS_SHIFT (22)
+#define CCTL_MPLL_BYPASS_MASK  (0x1)
+#define CCTL_USB_DIV_SHIFT     (16)
+#define CCTL_USB_DIV_MASK      (0x3F)
+#define CCTL_ARM_SRC_SHIFT     (13)
+#define CCTL_ARM_SRC_MASK      (0x1)
+#define CCTL_UPLL_DIS_SHIFT    (23)
+#define CCTL_UPLL_DIS_MASK     (0x1)
+
+#define EXTRACT(value, name) (((value) >> CCTL_##name##_SHIFT) \
+                              & CCTL_##name##_MASK)
+#define INSERT(value, name) (((value) & CCTL_##name##_MASK) << \
+                             CCTL_##name##_SHIFT)
+
+#define TYPE_IMX25_CCM "imx25.ccm"
+#define IMX25_CCM(obj) OBJECT_CHECK(IMX25CCMState, (obj), TYPE_IMX25_CCM)
+
+typedef struct IMX25CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t reg[IMX25_CCM_MAX_REG];
+
+} IMX25CCMState;
+
+#endif /* IMX25_CCM_H */
diff --git a/qemu/include/hw/misc/imx31_ccm.h b/qemu/include/hw/misc/imx31_ccm.h
new file mode 100644
index 000000000..c376fad14
--- /dev/null
+++ b/qemu/include/hw/misc/imx31_ccm.h
@@ -0,0 +1,88 @@
+/*
+ * IMX31 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX31_CCM_H
+#define IMX31_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+
+#define IMX31_CCM_CCMR_REG  0
+#define IMX31_CCM_PDR0_REG  1
+#define IMX31_CCM_PDR1_REG  2
+#define IMX31_CCM_RCSR_REG  3
+#define IMX31_CCM_MPCTL_REG 4
+#define IMX31_CCM_UPCTL_REG 5
+#define IMX31_CCM_SPCTL_REG 6
+#define IMX31_CCM_COSR_REG  7
+#define IMX31_CCM_CGR0_REG  8
+#define IMX31_CCM_CGR1_REG  9
+#define IMX31_CCM_CGR2_REG  10
+#define IMX31_CCM_WIMR_REG  11
+#define IMX31_CCM_LDC_REG   12
+#define IMX31_CCM_DCVR0_REG 13
+#define IMX31_CCM_DCVR1_REG 14
+#define IMX31_CCM_DCVR2_REG 15
+#define IMX31_CCM_DCVR3_REG 16
+#define IMX31_CCM_LTR0_REG  17
+#define IMX31_CCM_LTR1_REG  18
+#define IMX31_CCM_LTR2_REG  19
+#define IMX31_CCM_LTR3_REG  20
+#define IMX31_CCM_LTBR0_REG 21
+#define IMX31_CCM_LTBR1_REG 22
+#define IMX31_CCM_PMCR0_REG 23
+#define IMX31_CCM_PMCR1_REG 24
+#define IMX31_CCM_PDR2_REG  25
+#define IMX31_CCM_MAX_REG   26
+
+/* CCMR */
+#define CCMR_FPME    (1<<0)
+#define CCMR_MPE     (1<<3)
+#define CCMR_MDS     (1<<7)
+#define CCMR_FPMF    (1<<26)
+#define CCMR_PRCS    (3<<1)
+
+#define PMCR0_DFSUP1 (1<<31)
+
+/* PDR0 */
+#define PDR0_MCU_PODF_SHIFT (0)
+#define PDR0_MCU_PODF_MASK (0x7)
+#define PDR0_MAX_PODF_SHIFT (3)
+#define PDR0_MAX_PODF_MASK (0x7)
+#define PDR0_IPG_PODF_SHIFT (6)
+#define PDR0_IPG_PODF_MASK (0x3)
+#define PDR0_NFC_PODF_SHIFT (8)
+#define PDR0_NFC_PODF_MASK (0x7)
+#define PDR0_HSP_PODF_SHIFT (11)
+#define PDR0_HSP_PODF_MASK (0x7)
+#define PDR0_PER_PODF_SHIFT (16)
+#define PDR0_PER_PODF_MASK (0x1f)
+#define PDR0_CSI_PODF_SHIFT (23)
+#define PDR0_CSI_PODF_MASK (0x1ff)
+
+#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
+                              & PDR0_##name##_PODF_MASK)
+#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
+                             PDR0_##name##_PODF_SHIFT)
+
+#define TYPE_IMX31_CCM "imx31.ccm"
+#define IMX31_CCM(obj) OBJECT_CHECK(IMX31CCMState, (obj), TYPE_IMX31_CCM)
+
+typedef struct IMX31CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t reg[IMX31_CCM_MAX_REG];
+
+} IMX31CCMState;
+
+#endif /* IMX31_CCM_H */
diff --git a/qemu/include/hw/misc/imx6_ccm.h b/qemu/include/hw/misc/imx6_ccm.h
new file mode 100644
index 000000000..80505809b
--- /dev/null
+++ b/qemu/include/hw/misc/imx6_ccm.h
@@ -0,0 +1,197 @@
+/*
+ * IMX6 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX6_CCM_H
+#define IMX6_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+#include "qemu/bitops.h"
+
+#define CCM_CCR 0
+#define CCM_CCDR 1
+#define CCM_CSR 2
+#define CCM_CCSR 3
+#define CCM_CACRR 4
+#define CCM_CBCDR 5
+#define CCM_CBCMR 6
+#define CCM_CSCMR1 7
+#define CCM_CSCMR2 8
+#define CCM_CSCDR1 9
+#define CCM_CS1CDR 10
+#define CCM_CS2CDR 11
+#define CCM_CDCDR 12
+#define CCM_CHSCCDR 13
+#define CCM_CSCDR2 14
+#define CCM_CSCDR3 15
+#define CCM_CDHIPR 18
+#define CCM_CTOR 20
+#define CCM_CLPCR 21
+#define CCM_CISR 22
+#define CCM_CIMR 23
+#define CCM_CCOSR 24
+#define CCM_CGPR 25
+#define CCM_CCGR0 26
+#define CCM_CCGR1 27
+#define CCM_CCGR2 28
+#define CCM_CCGR3 29
+#define CCM_CCGR4 30
+#define CCM_CCGR5 31
+#define CCM_CCGR6 32
+#define CCM_CMEOR 34
+#define CCM_MAX 35
+
+#define CCM_ANALOG_PLL_ARM 0
+#define CCM_ANALOG_PLL_ARM_SET 1
+#define CCM_ANALOG_PLL_ARM_CLR 2
+#define CCM_ANALOG_PLL_ARM_TOG 3
+#define CCM_ANALOG_PLL_USB1 4
+#define CCM_ANALOG_PLL_USB1_SET 5
+#define CCM_ANALOG_PLL_USB1_CLR 6
+#define CCM_ANALOG_PLL_USB1_TOG 7
+#define CCM_ANALOG_PLL_USB2 8
+#define CCM_ANALOG_PLL_USB2_SET 9
+#define CCM_ANALOG_PLL_USB2_CLR 10
+#define CCM_ANALOG_PLL_USB2_TOG 11
+#define CCM_ANALOG_PLL_SYS 12
+#define CCM_ANALOG_PLL_SYS_SET 13
+#define CCM_ANALOG_PLL_SYS_CLR 14
+#define CCM_ANALOG_PLL_SYS_TOG 15
+#define CCM_ANALOG_PLL_SYS_SS 16
+#define CCM_ANALOG_PLL_SYS_NUM 20
+#define CCM_ANALOG_PLL_SYS_DENOM 24
+#define CCM_ANALOG_PLL_AUDIO 28
+#define CCM_ANALOG_PLL_AUDIO_SET 29
+#define CCM_ANALOG_PLL_AUDIO_CLR 30
+#define CCM_ANALOG_PLL_AUDIO_TOG 31
+#define CCM_ANALOG_PLL_AUDIO_NUM 32
+#define CCM_ANALOG_PLL_AUDIO_DENOM 36
+#define CCM_ANALOG_PLL_VIDEO 40
+#define CCM_ANALOG_PLL_VIDEO_SET 41
+#define CCM_ANALOG_PLL_VIDEO_CLR 42
+#define CCM_ANALOG_PLL_VIDEO_TOG 44
+#define CCM_ANALOG_PLL_VIDEO_NUM 46
+#define CCM_ANALOG_PLL_VIDEO_DENOM 48
+#define CCM_ANALOG_PLL_MLB 52
+#define CCM_ANALOG_PLL_MLB_SET 53
+#define CCM_ANALOG_PLL_MLB_CLR 54
+#define CCM_ANALOG_PLL_MLB_TOG 55
+#define CCM_ANALOG_PLL_ENET 56
+#define CCM_ANALOG_PLL_ENET_SET 57
+#define CCM_ANALOG_PLL_ENET_CLR 58
+#define CCM_ANALOG_PLL_ENET_TOG 59
+#define CCM_ANALOG_PFD_480 60
+#define CCM_ANALOG_PFD_480_SET 61
+#define CCM_ANALOG_PFD_480_CLR 62
+#define CCM_ANALOG_PFD_480_TOG 63
+#define CCM_ANALOG_PFD_528 64
+#define CCM_ANALOG_PFD_528_SET 65
+#define CCM_ANALOG_PFD_528_CLR 66
+#define CCM_ANALOG_PFD_528_TOG 67
+
+/* PMU registers */
+#define PMU_REG_1P1 68
+#define PMU_REG_3P0 72
+#define PMU_REG_2P5 76
+#define PMU_REG_CORE 80
+
+#define CCM_ANALOG_MISC0 84
+#define PMU_MISC0 84
+#define CCM_ANALOG_MISC0_SET 85
+#define CCM_ANALOG_MISC0_CLR 86
+#define CCM_ANALOG_MISC0_TOG 87
+
+#define PMU_MISC1 88
+#define PMU_MISC1_SET 89
+#define PMU_MISC1_CLR 90
+#define PMU_MISC1_TOG 91
+
+#define CCM_ANALOG_MISC2 92
+#define PMU_MISC2 92
+#define CCM_ANALOG_MISC2_SET 93
+#define CCM_ANALOG_MISC2_CLR 94
+#define CCM_ANALOG_MISC2_TOG 95
+
+#define USB_ANALOG_USB1_VBUS_DETECT 104
+#define USB_ANALOG_USB1_VBUS_DETECT_SET 105
+#define USB_ANALOG_USB1_VBUS_DETECT_CLR 106
+#define USB_ANALOG_USB1_VBUS_DETECT_TOG 107
+#define USB_ANALOG_USB1_CHRG_DETECT 108
+#define USB_ANALOG_USB1_CHRG_DETECT_SET 109
+#define USB_ANALOG_USB1_CHRG_DETECT_CLR 110
+#define USB_ANALOG_USB1_CHRG_DETECT_TOG 111
+#define USB_ANALOG_USB1_VBUS_DETECT_STAT 112
+#define USB_ANALOG_USB1_CHRG_DETECT_STAT 116
+#define USB_ANALOG_USB1_MISC 124
+#define USB_ANALOG_USB1_MISC_SET 125
+#define USB_ANALOG_USB1_MISC_CLR 126
+#define USB_ANALOG_USB1_MISC_TOG 127
+#define USB_ANALOG_USB2_VBUS_DETECT 128
+#define USB_ANALOG_USB2_VBUS_DETECT_SET 129
+#define USB_ANALOG_USB2_VBUS_DETECT_CLR 130
+#define USB_ANALOG_USB2_VBUS_DETECT_TOG 131
+#define USB_ANALOG_USB2_CHRG_DETECT 132
+#define USB_ANALOG_USB2_CHRG_DETECT_SET 133
+#define USB_ANALOG_USB2_CHRG_DETECT_CLR 134
+#define USB_ANALOG_USB2_CHRG_DETECT_TOG 135
+#define USB_ANALOG_USB2_VBUS_DETECT_STAT 136
+#define USB_ANALOG_USB2_CHRG_DETECT_STAT 140
+#define USB_ANALOG_USB2_MISC 148
+#define USB_ANALOG_USB2_MISC_SET 149
+#define USB_ANALOG_USB2_MISC_CLR 150
+#define USB_ANALOG_USB2_MISC_TOG 151
+#define USB_ANALOG_DIGPROG 152
+#define CCM_ANALOG_MAX 153
+
+/* CCM_CBCMR */
+#define PRE_PERIPH_CLK_SEL_SHIFT  (18)
+#define PRE_PERIPH_CLK_SEL_LENGTH (2)
+
+/* CCM_CBCDR */
+#define AHB_PODF_SHIFT           (10)
+#define AHB_PODF_LENGTH          (3)
+#define IPG_PODF_SHIFT           (8)
+#define IPG_PODF_LENGTH          (2)
+
+/* CCM_CSCMR1 */
+#define PERCLK_PODF_SHIFT        (0)
+#define PERCLK_PODF_LENGTH       (6)
+
+/* CCM_ANALOG_PFD_528 */
+#define PFD0_FRAC_SHIFT          (0)
+#define PFD0_FRAC_LENGTH         (6)
+#define PFD2_FRAC_SHIFT          (16)
+#define PFD2_FRAC_LENGTH         (6)
+
+/* CCM_ANALOG_PLL_SYS */
+#define DIV_SELECT_SHIFT         (0)
+#define DIV_SELECT_LENGTH        (1)
+
+#define CCM_ANALOG_PLL_LOCK      (1 << 31);
+
+#define EXTRACT(value, name) extract32(value, name##_SHIFT, name##_LENGTH)
+
+#define TYPE_IMX6_CCM "imx6.ccm"
+#define IMX6_CCM(obj) OBJECT_CHECK(IMX6CCMState, (obj), TYPE_IMX6_CCM)
+
+typedef struct IMX6CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion container;
+    MemoryRegion ioccm;
+    MemoryRegion ioanalog;
+
+    uint32_t ccm[CCM_MAX];
+    uint32_t analog[CCM_ANALOG_MAX];
+
+} IMX6CCMState;
+
+#endif /* IMX6_CCM_H */
diff --git a/qemu/include/hw/misc/imx_ccm.h b/qemu/include/hw/misc/imx_ccm.h
new file mode 100644
index 000000000..48a7afad5
--- /dev/null
+++ b/qemu/include/hw/misc/imx_ccm.h
@@ -0,0 +1,64 @@
+/*
+ * IMX Clock Control Module base class
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX_CCM_H
+#define IMX_CCM_H
+
+#include "hw/sysbus.h"
+
+#define CKIL_FREQ 32768 /* nominal 32khz clock */
+
+/* PLL control registers */
+#define PD(v) (((v) >> 26) & 0xf)
+#define MFD(v) (((v) >> 16) & 0x3ff)
+#define MFI(v) (((v) >> 10) & 0xf);
+#define MFN(v) ((v) & 0x3ff)
+
+#define PLL_PD(x)               (((x) & 0xf) << 26)
+#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
+#define PLL_MFI(x)              (((x) & 0xf) << 10)
+#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
+
+#define TYPE_IMX_CCM "imx.ccm"
+#define IMX_CCM(obj) \
+     OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
+#define IMX_CCM_CLASS(klass) \
+     OBJECT_CLASS_CHECK(IMXCCMClass, (klass), TYPE_IMX_CCM)
+#define IMX_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IMXCCMClass, (obj), TYPE_IMX_CCM)
+
+typedef struct IMXCCMState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+
+} IMXCCMState;
+
+typedef enum  {
+    CLK_NONE,
+    CLK_IPG,
+    CLK_IPG_HIGH,
+    CLK_32k
+} IMXClk;
+
+typedef struct IMXCCMClass {
+    /* <private> */
+    SysBusDeviceClass parent_class;
+
+    /* <public> */
+    uint32_t (*get_clock_frequency)(IMXCCMState *s, IMXClk clk);
+} IMXCCMClass;
+
+uint32_t imx_ccm_calc_pll(uint32_t pllreg, uint32_t base_freq);
+
+uint32_t imx_ccm_get_clock_frequency(IMXCCMState *s, IMXClk clock);
+
+#endif /* IMX_CCM_H */
diff --git a/qemu/include/hw/misc/ivshmem.h b/qemu/include/hw/misc/ivshmem.h
new file mode 100644
index 000000000..433ef53d7
--- /dev/null
+++ b/qemu/include/hw/misc/ivshmem.h
@@ -0,0 +1,25 @@
+
+/*
+ * Inter-VM Shared Memory PCI device.
+ *
+ * Author:
+ *      Cam Macdonell <cam@cs.ualberta.ca>
+ *
+ * Based On: cirrus_vga.c
+ *          Copyright (c) 2004 Fabrice Bellard
+ *          Copyright (c) 2004 Makoto Suzuki (suzu)
+ *
+ *      and rtl8139.c
+ *          Copyright (c) 2006 Igor Kovalenko
+ *
+ * This code is licensed under the GNU GPL v2.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#ifndef IVSHMEM_H
+#define IVSHMEM_H
+
+#define IVSHMEM_PROTOCOL_VERSION 0
+
+#endif /* IVSHMEM_H */
diff --git a/qemu/include/hw/misc/mips_cmgcr.h b/qemu/include/hw/misc/mips_cmgcr.h
new file mode 100644
index 000000000..cc60eefa5
--- /dev/null
+++ b/qemu/include/hw/misc/mips_cmgcr.h
@@ -0,0 +1,59 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2015 Imagination Technologies
+ *
+ */
+
+#ifndef _MIPS_GCR_H
+#define _MIPS_GCR_H
+
+#define TYPE_MIPS_GCR "mips-gcr"
+#define MIPS_GCR(obj) OBJECT_CHECK(MIPSGCRState, (obj), TYPE_MIPS_GCR)
+
+#define GCR_BASE_ADDR           0x1fbf8000ULL
+#define GCR_ADDRSPACE_SZ        0x8000
+
+/* Offsets to register blocks */
+#define MIPS_GCB_OFS        0x0000 /* Global Control Block */
+#define MIPS_CLCB_OFS       0x2000 /* Core Local Control Block */
+#define MIPS_COCB_OFS       0x4000 /* Core Other Control Block */
+#define MIPS_GDB_OFS        0x6000 /* Global Debug Block */
+
+/* Global Control Block Register Map */
+#define GCR_CONFIG_OFS      0x0000
+#define GCR_BASE_OFS        0x0008
+#define GCR_REV_OFS         0x0030
+#define GCR_CPC_BASE_OFS    0x0088
+#define GCR_CPC_STATUS_OFS  0x00F0
+#define GCR_L2_CONFIG_OFS   0x0130
+
+/* Core Local and Core Other Block Register Map */
+#define GCR_CL_CONFIG_OFS   0x0010
+#define GCR_CL_OTHER_OFS    0x0018
+
+/* GCR_L2_CONFIG register fields */
+#define GCR_L2_CONFIG_BYPASS_SHF    20
+#define GCR_L2_CONFIG_BYPASS_MSK    ((0x1ULL) << GCR_L2_CONFIG_BYPASS_SHF)
+
+/* GCR_CPC_BASE register fields */
+#define GCR_CPC_BASE_CPCEN_MSK   1
+#define GCR_CPC_BASE_CPCBASE_MSK 0xFFFFFFFF8000ULL
+#define GCR_CPC_BASE_MSK (GCR_CPC_BASE_CPCEN_MSK | GCR_CPC_BASE_CPCBASE_MSK)
+
+typedef struct MIPSGCRState MIPSGCRState;
+struct MIPSGCRState {
+    SysBusDevice parent_obj;
+
+    int32_t gcr_rev;
+    int32_t num_vps;
+    hwaddr gcr_base;
+    MemoryRegion iomem;
+    MemoryRegion *cpc_mr;
+
+    uint64_t cpc_base;
+};
+
+#endif /* _MIPS_GCR_H */
diff --git a/qemu/include/hw/misc/mips_cpc.h b/qemu/include/hw/misc/mips_cpc.h
new file mode 100644
index 000000000..72c834e03
--- /dev/null
+++ b/qemu/include/hw/misc/mips_cpc.h
@@ -0,0 +1,47 @@
+/*
+ * Cluster Power Controller emulation
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_CPC_H
+#define MIPS_CPC_H
+
+#define CPC_ADDRSPACE_SZ    0x6000
+
+/* CPC blocks offsets relative to base address */
+#define CPC_CL_BASE_OFS     0x2000
+#define CPC_CO_BASE_OFS     0x4000
+
+/* CPC register offsets relative to block offsets */
+#define CPC_VP_STOP_OFS     0x20
+#define CPC_VP_RUN_OFS      0x28
+#define CPC_VP_RUNNING_OFS  0x30
+
+#define TYPE_MIPS_CPC "mips-cpc"
+#define MIPS_CPC(obj) OBJECT_CHECK(MIPSCPCState, (obj), TYPE_MIPS_CPC)
+
+typedef struct MIPSCPCState {
+    SysBusDevice parent_obj;
+
+    uint32_t num_vp;
+    uint64_t vp_start_running; /* VPs running from restart */
+
+    MemoryRegion mr;
+    uint64_t vp_running; /* Indicates which VPs are in the run state */
+} MIPSCPCState;
+
+#endif /* MIPS_CPC_H */
diff --git a/qemu/include/hw/misc/mips_itu.h b/qemu/include/hw/misc/mips_itu.h
new file mode 100644
index 000000000..b3a453203
--- /dev/null
+++ b/qemu/include/hw/misc/mips_itu.h
@@ -0,0 +1,72 @@
+/*
+ * Inter-Thread Communication Unit emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_ITU_H
+#define MIPS_ITU_H
+
+#define TYPE_MIPS_ITU "mips-itu"
+#define MIPS_ITU(obj) OBJECT_CHECK(MIPSITUState, (obj), TYPE_MIPS_ITU)
+
+#define ITC_CELL_DEPTH_SHIFT 2
+#define ITC_CELL_DEPTH (1u << ITC_CELL_DEPTH_SHIFT)
+
+typedef struct ITCStorageCell {
+    struct {
+        uint8_t FIFODepth; /* Log2 of the cell depth */
+        uint8_t FIFOPtr; /* Number of elements in a FIFO cell */
+        uint8_t FIFO; /* 1 - FIFO cell, 0 - Semaphore cell */
+        uint8_t T; /* Trap Bit */
+        uint8_t F; /* Full Bit */
+        uint8_t E; /* Empty Bit */
+    } tag;
+
+    /* Index of the oldest element in the queue */
+    uint8_t fifo_out;
+
+    /* Circular buffer for FIFO. Semaphore cells use index 0 only */
+    uint64_t data[ITC_CELL_DEPTH];
+
+    /* Bitmap tracking blocked threads on the cell.
+       TODO: support >64 threads ? */
+    uint64_t blocked_threads;
+} ITCStorageCell;
+
+#define ITC_ADDRESSMAP_NUM 2
+
+typedef struct MIPSITUState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    int32_t num_fifo;
+    int32_t num_semaphores;
+
+    /* ITC Storage */
+    ITCStorageCell *cell;
+    MemoryRegion storage_io;
+
+    /* ITC Configuration Tags */
+    uint64_t ITCAddressMap[ITC_ADDRESSMAP_NUM];
+    MemoryRegion tag_io;
+} MIPSITUState;
+
+/* Get ITC Configuration Tag memory region. */
+MemoryRegion *mips_itu_get_tag_region(MIPSITUState *itu);
+
+#endif /* MIPS_ITU_H */
diff --git a/qemu/include/hw/misc/zynq-xadc.h b/qemu/include/hw/misc/zynq-xadc.h
new file mode 100644
index 000000000..f1a410a37
--- /dev/null
+++ b/qemu/include/hw/misc/zynq-xadc.h
@@ -0,0 +1,46 @@
+/*
+ * Device model for Zynq ADC controller
+ *
+ * Copyright (c) 2015 Guenter Roeck <linux@roeck-us.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ZYNQ_XADC_H
+#define ZYNQ_XADC_H
+
+#include "hw/sysbus.h"
+
+#define ZYNQ_XADC_MMIO_SIZE     0x0020
+#define ZYNQ_XADC_NUM_IO_REGS   (ZYNQ_XADC_MMIO_SIZE / 4)
+#define ZYNQ_XADC_NUM_ADC_REGS  128
+#define ZYNQ_XADC_FIFO_DEPTH    15
+
+#define TYPE_ZYNQ_XADC          "xlnx,zynq-xadc"
+#define ZYNQ_XADC(obj) \
+    OBJECT_CHECK(ZynqXADCState, (obj), TYPE_ZYNQ_XADC)
+
+typedef struct ZynqXADCState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t regs[ZYNQ_XADC_NUM_IO_REGS];
+    uint16_t xadc_regs[ZYNQ_XADC_NUM_ADC_REGS];
+    uint16_t xadc_read_reg_previous;
+    uint16_t xadc_dfifo[ZYNQ_XADC_FIFO_DEPTH];
+    uint16_t xadc_dfifo_entries;
+
+    struct IRQState *qemu_irq;
+
+} ZynqXADCState;
+
+#endif /* ZYNQ_XADC_H */
diff --git a/qemu/include/hw/net/imx_fec.h b/qemu/include/hw/net/imx_fec.h
new file mode 100644
index 000000000..cbf86509e
--- /dev/null
+++ b/qemu/include/hw/net/imx_fec.h
@@ -0,0 +1,113 @@
+/*
+ * i.MX Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_FEC_H
+#define IMX_FEC_H
+
+#define TYPE_IMX_FEC "imx.fec"
+#define IMX_FEC(obj) OBJECT_CHECK(IMXFECState, (obj), TYPE_IMX_FEC)
+
+#include "hw/sysbus.h"
+#include "net/net.h"
+
+#define FEC_MAX_FRAME_SIZE 2032
+
+#define FEC_INT_HB      (1 << 31)
+#define FEC_INT_BABR    (1 << 30)
+#define FEC_INT_BABT    (1 << 29)
+#define FEC_INT_GRA     (1 << 28)
+#define FEC_INT_TXF     (1 << 27)
+#define FEC_INT_TXB     (1 << 26)
+#define FEC_INT_RXF     (1 << 25)
+#define FEC_INT_RXB     (1 << 24)
+#define FEC_INT_MII     (1 << 23)
+#define FEC_INT_EBERR   (1 << 22)
+#define FEC_INT_LC      (1 << 21)
+#define FEC_INT_RL      (1 << 20)
+#define FEC_INT_UN      (1 << 19)
+
+#define FEC_EN      2
+#define FEC_RESET   1
+
+/* Buffer Descriptor.  */
+typedef struct {
+    uint16_t length;
+    uint16_t flags;
+    uint32_t data;
+} IMXFECBufDesc;
+
+#define FEC_BD_R    (1 << 15)
+#define FEC_BD_E    (1 << 15)
+#define FEC_BD_O1   (1 << 14)
+#define FEC_BD_W    (1 << 13)
+#define FEC_BD_O2   (1 << 12)
+#define FEC_BD_L    (1 << 11)
+#define FEC_BD_TC   (1 << 10)
+#define FEC_BD_ABC  (1 << 9)
+#define FEC_BD_M    (1 << 8)
+#define FEC_BD_BC   (1 << 7)
+#define FEC_BD_MC   (1 << 6)
+#define FEC_BD_LG   (1 << 5)
+#define FEC_BD_NO   (1 << 4)
+#define FEC_BD_CR   (1 << 2)
+#define FEC_BD_OV   (1 << 1)
+#define FEC_BD_TR   (1 << 0)
+
+typedef struct IMXFECState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    NICState *nic;
+    NICConf conf;
+    qemu_irq irq;
+    MemoryRegion iomem;
+
+    uint32_t irq_state;
+    uint32_t eir;
+    uint32_t eimr;
+    uint32_t rx_enabled;
+    uint32_t rx_descriptor;
+    uint32_t tx_descriptor;
+    uint32_t ecr;
+    uint32_t mmfr;
+    uint32_t mscr;
+    uint32_t mibc;
+    uint32_t rcr;
+    uint32_t tcr;
+    uint32_t tfwr;
+    uint32_t frsr;
+    uint32_t erdsr;
+    uint32_t etdsr;
+    uint32_t emrbr;
+    uint32_t miigsk_cfgr;
+    uint32_t miigsk_enr;
+
+    uint32_t phy_status;
+    uint32_t phy_control;
+    uint32_t phy_advertise;
+    uint32_t phy_int;
+    uint32_t phy_int_mask;
+} IMXFECState;
+
+#endif
diff --git a/qemu/include/hw/nvram/fw_cfg.h b/qemu/include/hw/nvram/fw_cfg.h
index e60d3ca21..d00811258 100644
--- a/qemu/include/hw/nvram/fw_cfg.h
+++ b/qemu/include/hw/nvram/fw_cfg.h
@@ -1,54 +1,9 @@
 #ifndef FW_CFG_H
 #define FW_CFG_H
 
-#ifndef NO_QEMU_PROTOS
-#include <stdint.h>
-#include <stddef.h>
-
 #include "exec/hwaddr.h"
-#include "qemu/typedefs.h"
-#endif
+#include "hw/nvram/fw_cfg_keys.h"
 
-#define FW_CFG_SIGNATURE        0x00
-#define FW_CFG_ID               0x01
-#define FW_CFG_UUID             0x02
-#define FW_CFG_RAM_SIZE         0x03
-#define FW_CFG_NOGRAPHIC        0x04
-#define FW_CFG_NB_CPUS          0x05
-#define FW_CFG_MACHINE_ID       0x06
-#define FW_CFG_KERNEL_ADDR      0x07
-#define FW_CFG_KERNEL_SIZE      0x08
-#define FW_CFG_KERNEL_CMDLINE   0x09
-#define FW_CFG_INITRD_ADDR      0x0a
-#define FW_CFG_INITRD_SIZE      0x0b
-#define FW_CFG_BOOT_DEVICE      0x0c
-#define FW_CFG_NUMA             0x0d
-#define FW_CFG_BOOT_MENU        0x0e
-#define FW_CFG_MAX_CPUS         0x0f
-#define FW_CFG_KERNEL_ENTRY     0x10
-#define FW_CFG_KERNEL_DATA      0x11
-#define FW_CFG_INITRD_DATA      0x12
-#define FW_CFG_CMDLINE_ADDR     0x13
-#define FW_CFG_CMDLINE_SIZE     0x14
-#define FW_CFG_CMDLINE_DATA     0x15
-#define FW_CFG_SETUP_ADDR       0x16
-#define FW_CFG_SETUP_SIZE       0x17
-#define FW_CFG_SETUP_DATA       0x18
-#define FW_CFG_FILE_DIR         0x19
-
-#define FW_CFG_FILE_FIRST       0x20
-#define FW_CFG_FILE_SLOTS       0x10
-#define FW_CFG_MAX_ENTRY        (FW_CFG_FILE_FIRST+FW_CFG_FILE_SLOTS)
-
-#define FW_CFG_WRITE_CHANNEL    0x4000
-#define FW_CFG_ARCH_LOCAL       0x8000
-#define FW_CFG_ENTRY_MASK       ~(FW_CFG_WRITE_CHANNEL | FW_CFG_ARCH_LOCAL)
-
-#define FW_CFG_INVALID          0xffff
-
-#define FW_CFG_MAX_FILE_PATH    56
-
-#ifndef NO_QEMU_PROTOS
 typedef struct FWCfgFile {
     uint32_t  size;        /* file size */
     uint16_t  select;      /* write this to 0x510 to read it */
@@ -56,34 +11,176 @@ typedef struct FWCfgFile {
     char      name[FW_CFG_MAX_FILE_PATH];
 } FWCfgFile;
 
+#define FW_CFG_ORDER_OVERRIDE_VGA    70
+#define FW_CFG_ORDER_OVERRIDE_NIC    80
+#define FW_CFG_ORDER_OVERRIDE_USER   100
+#define FW_CFG_ORDER_OVERRIDE_DEVICE 110
+
+void fw_cfg_set_order_override(FWCfgState *fw_cfg, int order);
+void fw_cfg_reset_order_override(FWCfgState *fw_cfg);
+
 typedef struct FWCfgFiles {
     uint32_t  count;
     FWCfgFile f[];
 } FWCfgFiles;
 
-typedef void (*FWCfgCallback)(void *opaque, uint8_t *data);
-typedef void (*FWCfgReadCallback)(void *opaque, uint32_t offset);
+/* Control as first field allows for different structures selected by this
+ * field, which might be useful in the future
+ */
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} QEMU_PACKED FWCfgDmaAccess;
 
+typedef void (*FWCfgReadCallback)(void *opaque);
+
+/**
+ * fw_cfg_add_bytes:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new fw_cfg item, available by selecting the given key, as a raw
+ * "blob" of the given size. The data referenced by the starting pointer
+ * is only linked, NOT copied, into the data structure of the fw_cfg device.
+ */
 void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len);
+
+/**
+ * fw_cfg_add_string:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: NUL-terminated ascii string
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the provided string,
+ * including its NUL terminator.
+ */
 void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value);
+
+/**
+ * fw_cfg_add_i16:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 16-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 16-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
+
+/**
+ * fw_cfg_modify_i16:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 16-bit integer
+ *
+ * Replace the fw_cfg item available by selecting the given key. The new
+ * data will consist of a dynamically allocated copy of the given 16-bit
+ * value, converted to little-endian representation. The data being replaced,
+ * assumed to have been dynamically allocated during an earlier call to
+ * either fw_cfg_add_i16() or fw_cfg_modify_i16(), is freed before returning.
+ */
 void fw_cfg_modify_i16(FWCfgState *s, uint16_t key, uint16_t value);
+
+/**
+ * fw_cfg_add_i32:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 32-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 32-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
+
+/**
+ * fw_cfg_add_i64:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 64-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 64-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
+
+/**
+ * fw_cfg_add_file:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new NAMED fw_cfg item as a raw "blob" of the given size. The data
+ * referenced by the starting pointer is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ * The next available (unused) selector key starting at FW_CFG_FILE_FIRST
+ * will be used; also, a new entry will be added to the file directory
+ * structure residing at key value FW_CFG_FILE_DIR, containing the item name,
+ * data size, and assigned selector key value.
+ */
 void fw_cfg_add_file(FWCfgState *s, const char *filename, void *data,
                      size_t len);
+
+/**
+ * fw_cfg_add_file_callback:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @callback: callback function
+ * @callback_opaque: argument to be passed into callback function
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new NAMED fw_cfg item as a raw "blob" of the given size. The data
+ * referenced by the starting pointer is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ * The next available (unused) selector key starting at FW_CFG_FILE_FIRST
+ * will be used; also, a new entry will be added to the file directory
+ * structure residing at key value FW_CFG_FILE_DIR, containing the item name,
+ * data size, and assigned selector key value.
+ * Additionally, set a callback function (and argument) to be called each
+ * time this item is selected (by having its selector key either written to
+ * the fw_cfg control register, or passed to QEMU in FWCfgDmaAccess.control
+ * with FW_CFG_DMA_CTL_SELECT).
+ */
 void fw_cfg_add_file_callback(FWCfgState *s, const char *filename,
                               FWCfgReadCallback callback, void *callback_opaque,
                               void *data, size_t len);
+
+/**
+ * fw_cfg_modify_file:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Replace a NAMED fw_cfg item. If an existing item is found, its callback
+ * information will be cleared, and a pointer to its data will be returned
+ * to the caller, so that it may be freed if necessary. If an existing item
+ * is not found, this call defaults to fw_cfg_add_file(), and NULL is
+ * returned to the caller.
+ * In either case, the new item data is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ *
+ * Returns: pointer to old item's data, or NULL if old item does not exist.
+ */
 void *fw_cfg_modify_file(FWCfgState *s, const char *filename, void *data,
                          size_t len);
+
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as);
 FWCfgState *fw_cfg_init_io(uint32_t iobase);
 FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr);
-FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr, hwaddr data_addr,
-                                 uint32_t data_width);
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as);
 
 FWCfgState *fw_cfg_find(void);
 
-#endif /* NO_QEMU_PROTOS */
-
 #endif
diff --git a/qemu/include/hw/nvram/fw_cfg_keys.h b/qemu/include/hw/nvram/fw_cfg_keys.h
new file mode 100644
index 000000000..0f3e87188
--- /dev/null
+++ b/qemu/include/hw/nvram/fw_cfg_keys.h
@@ -0,0 +1,46 @@
+#ifndef FW_CFG_KEYS_H
+#define FW_CFG_KEYS_H
+
+#define FW_CFG_SIGNATURE        0x00
+#define FW_CFG_ID               0x01
+#define FW_CFG_UUID             0x02
+#define FW_CFG_RAM_SIZE         0x03
+#define FW_CFG_NOGRAPHIC        0x04
+#define FW_CFG_NB_CPUS          0x05
+#define FW_CFG_MACHINE_ID       0x06
+#define FW_CFG_KERNEL_ADDR      0x07
+#define FW_CFG_KERNEL_SIZE      0x08
+#define FW_CFG_KERNEL_CMDLINE   0x09
+#define FW_CFG_INITRD_ADDR      0x0a
+#define FW_CFG_INITRD_SIZE      0x0b
+#define FW_CFG_BOOT_DEVICE      0x0c
+#define FW_CFG_NUMA             0x0d
+#define FW_CFG_BOOT_MENU        0x0e
+#define FW_CFG_MAX_CPUS         0x0f
+#define FW_CFG_KERNEL_ENTRY     0x10
+#define FW_CFG_KERNEL_DATA      0x11
+#define FW_CFG_INITRD_DATA      0x12
+#define FW_CFG_CMDLINE_ADDR     0x13
+#define FW_CFG_CMDLINE_SIZE     0x14
+#define FW_CFG_CMDLINE_DATA     0x15
+#define FW_CFG_SETUP_ADDR       0x16
+#define FW_CFG_SETUP_SIZE       0x17
+#define FW_CFG_SETUP_DATA       0x18
+#define FW_CFG_FILE_DIR         0x19
+
+#define FW_CFG_FILE_FIRST       0x20
+#define FW_CFG_FILE_SLOTS       0x10
+#define FW_CFG_MAX_ENTRY        (FW_CFG_FILE_FIRST + FW_CFG_FILE_SLOTS)
+
+#define FW_CFG_WRITE_CHANNEL    0x4000
+#define FW_CFG_ARCH_LOCAL       0x8000
+#define FW_CFG_ENTRY_MASK       (~(FW_CFG_WRITE_CHANNEL | FW_CFG_ARCH_LOCAL))
+
+#define FW_CFG_INVALID          0xffff
+
+/* width in bytes of fw_cfg control register */
+#define FW_CFG_CTL_SIZE         0x02
+
+#define FW_CFG_MAX_FILE_PATH    56
+
+#endif
diff --git a/qemu/include/hw/pci-host/q35.h b/qemu/include/hw/pci-host/q35.h
index dbe6dc05b..c5c073dde 100644
--- a/qemu/include/hw/pci-host/q35.h
+++ b/qemu/include/hw/pci-host/q35.h
@@ -59,7 +59,6 @@ typedef struct MCHPCIState {
     ram_addr_t below_4g_mem_size;
     ram_addr_t above_4g_mem_size;
     uint64_t pci_hole64_size;
-    PcGuestInfo *guest_info;
     uint32_t short_root_bus;
     IntelIOMMUState *iommu;
 } MCHPCIState;
diff --git a/qemu/include/hw/pci-host/spapr.h b/qemu/include/hw/pci-host/spapr.h
index 5322b560e..03ee00640 100644
--- a/qemu/include/hw/pci-host/spapr.h
+++ b/qemu/include/hw/pci-host/spapr.h
@@ -28,32 +28,11 @@
 #include "hw/ppc/xics.h"
 
 #define TYPE_SPAPR_PCI_HOST_BRIDGE "spapr-pci-host-bridge"
-#define TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE "spapr-pci-vfio-host-bridge"
 
 #define SPAPR_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(sPAPRPHBState, (obj), TYPE_SPAPR_PCI_HOST_BRIDGE)
 
-#define SPAPR_PCI_VFIO_HOST_BRIDGE(obj) \
-    OBJECT_CHECK(sPAPRPHBVFIOState, (obj), TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE)
-
-#define SPAPR_PCI_HOST_BRIDGE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(sPAPRPHBClass, (klass), TYPE_SPAPR_PCI_HOST_BRIDGE)
-#define SPAPR_PCI_HOST_BRIDGE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(sPAPRPHBClass, (obj), TYPE_SPAPR_PCI_HOST_BRIDGE)
-
-typedef struct sPAPRPHBClass sPAPRPHBClass;
 typedef struct sPAPRPHBState sPAPRPHBState;
-typedef struct sPAPRPHBVFIOState sPAPRPHBVFIOState;
-
-struct sPAPRPHBClass {
-    PCIHostBridgeClass parent_class;
-
-    void (*finish_realize)(sPAPRPHBState *sphb, Error **errp);
-    int (*eeh_set_option)(sPAPRPHBState *sphb, unsigned int addr, int option);
-    int (*eeh_get_state)(sPAPRPHBState *sphb, int *state);
-    int (*eeh_reset)(sPAPRPHBState *sphb, int option);
-    int (*eeh_configure)(sPAPRPHBState *sphb);
-};
 
 typedef struct spapr_pci_msi {
     uint32_t first_irq;
@@ -78,6 +57,7 @@ struct sPAPRPHBState {
     MemoryRegion memwindow, iowindow, msiwindow;
 
     uint32_t dma_liobn;
+    hwaddr dma_win_addr, dma_win_size;
     AddressSpace iommu_as;
     MemoryRegion iommu_root;
 
@@ -93,12 +73,6 @@ struct sPAPRPHBState {
     QLIST_ENTRY(sPAPRPHBState) list;
 };
 
-struct sPAPRPHBVFIOState {
-    sPAPRPHBState phb;
-
-    int32_t iommugroupid;
-};
-
 #define SPAPR_PCI_MAX_INDEX          255
 
 #define SPAPR_PCI_BASE_BUID          0x800000020000000ULL
@@ -115,8 +89,6 @@ struct sPAPRPHBVFIOState {
 
 #define SPAPR_PCI_MSI_WINDOW         0x40000000000ULL
 
-#define SPAPR_PCI_DMA32_SIZE         0x40000000
-
 static inline qemu_irq spapr_phb_lsi_qirq(struct sPAPRPHBState *phb, int pin)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
@@ -138,4 +110,41 @@ sPAPRPHBState *spapr_pci_find_phb(sPAPRMachineState *spapr, uint64_t buid);
 PCIDevice *spapr_pci_find_dev(sPAPRMachineState *spapr, uint64_t buid,
                               uint32_t config_addr);
 
+/* VFIO EEH hooks */
+#ifdef CONFIG_LINUX
+bool spapr_phb_eeh_available(sPAPRPHBState *sphb);
+int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                  unsigned int addr, int option);
+int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state);
+int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option);
+int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb);
+void spapr_phb_vfio_reset(DeviceState *qdev);
+#else
+static inline bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
+{
+    return false;
+}
+static inline int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                                unsigned int addr, int option)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb,
+                                               int *state)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline void spapr_phb_vfio_reset(DeviceState *qdev)
+{
+}
+#endif
+
 #endif /* __HW_SPAPR_PCI_H__ */
diff --git a/qemu/include/hw/pci/msi.h b/qemu/include/hw/pci/msi.h
index 50e452bd0..8124908ab 100644
--- a/qemu/include/hw/pci/msi.h
+++ b/qemu/include/hw/pci/msi.h
@@ -29,7 +29,7 @@ struct MSIMessage {
     uint32_t data;
 };
 
-extern bool msi_supported;
+extern bool msi_nonbroken;
 
 void msi_set_message(PCIDevice *dev, MSIMessage msg);
 MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector);
diff --git a/qemu/include/hw/pci/msix.h b/qemu/include/hw/pci/msix.h
index 954d82b35..72e5f931c 100644
--- a/qemu/include/hw/pci/msix.h
+++ b/qemu/include/hw/pci/msix.h
@@ -46,12 +46,16 @@ void msix_unset_vector_notifiers(PCIDevice *dev);
 
 extern const VMStateDescription vmstate_msix;
 
-#define VMSTATE_MSIX(_field, _state) {                               \
-    .name       = (stringify(_field)),                               \
-    .size       = sizeof(PCIDevice),                                 \
-    .vmsd       = &vmstate_msix,                                     \
-    .flags      = VMS_STRUCT,                                        \
-    .offset     = vmstate_offset_value(_state, _field, PCIDevice),   \
+#define VMSTATE_MSIX_TEST(_field, _state, _test) {                   \
+    .name         = (stringify(_field)),                             \
+    .size         = sizeof(PCIDevice),                               \
+    .vmsd         = &vmstate_msix,                                   \
+    .flags        = VMS_STRUCT,                                      \
+    .offset       = vmstate_offset_value(_state, _field, PCIDevice), \
+    .field_exists = (_test)                                          \
 }
 
+#define VMSTATE_MSIX(_f, _s)                                         \
+    VMSTATE_MSIX_TEST(_f, _s, NULL)
+
 #endif
diff --git a/qemu/include/hw/pci/pci-assign.h b/qemu/include/hw/pci/pci-assign.h
new file mode 100644
index 000000000..55f42c56f
--- /dev/null
+++ b/qemu/include/hw/pci/pci-assign.h
@@ -0,0 +1,27 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Just split from hw/i386/kvm/pci-assign.c.
+ */
+#ifndef PCI_ASSIGN_H
+#define PCI_ASSIGN_H
+
+#include "hw/pci/pci.h"
+
+//#define DEVICE_ASSIGNMENT_DEBUG
+
+#ifdef DEVICE_ASSIGNMENT_DEBUG
+#define DEBUG(fmt, ...)                                       \
+    do {                                                      \
+        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
+    } while (0)
+#else
+#define DEBUG(fmt, ...)
+#endif
+
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function);
+#endif /* PCI_ASSIGN_H */
diff --git a/qemu/include/hw/pci/pci.h b/qemu/include/hw/pci/pci.h
index 551cb3d60..ef6ba51f6 100644
--- a/qemu/include/hw/pci/pci.h
+++ b/qemu/include/hw/pci/pci.h
@@ -1,12 +1,9 @@
 #ifndef QEMU_PCI_H
 #define QEMU_PCI_H
 
-#include "qemu-common.h"
-
 #include "hw/qdev.h"
 #include "exec/memory.h"
 #include "sysemu/dma.h"
-#include "qapi/error.h"
 
 /* PCI includes legacy ISA access.  */
 #include "hw/isa/isa.h"
@@ -93,10 +90,20 @@
 #define PCI_DEVICE_ID_REDHAT_PCIE_HOST   0x0008
 #define PCI_DEVICE_ID_REDHAT_PXB         0x0009
 #define PCI_DEVICE_ID_REDHAT_BRIDGE_SEAT 0x000a
+#define PCI_DEVICE_ID_REDHAT_PXB_PCIE    0x000b
 #define PCI_DEVICE_ID_REDHAT_QXL         0x0100
 
 #define FMT_PCIBUS                      PRIx64
 
+typedef uint64_t pcibus_t;
+
+struct PCIHostDeviceAddress {
+    unsigned int domain;
+    unsigned int bus;
+    unsigned int slot;
+    unsigned int function;
+};
+
 typedef void PCIConfigWriteFunc(PCIDevice *pci_dev,
                                 uint32_t address, uint32_t data, int len);
 typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev,
@@ -397,6 +404,7 @@ void pci_for_each_bus_depth_first(PCIBus *bus,
                                   void *(*begin)(PCIBus *bus, void *parent_state),
                                   void (*end)(PCIBus *bus, void *state),
                                   void *parent_state);
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev);
 
 /* Use this wrapper when specific scan order is not required. */
 static inline
@@ -677,6 +685,11 @@ static inline uint32_t pci_config_size(const PCIDevice *d)
     return pci_is_express(d) ? PCIE_CONFIG_SPACE_SIZE : PCI_CONFIG_SPACE_SIZE;
 }
 
+static inline uint16_t pci_requester_id(PCIDevice *dev)
+{
+    return (pci_bus_num(dev->bus) << 8) | dev->devfn;
+}
+
 /* DMA access functions */
 static inline AddressSpace *pci_get_address_space(PCIDevice *dev)
 {
diff --git a/qemu/include/hw/pci/pci_bridge.h b/qemu/include/hw/pci/pci_bridge.h
index 93b621cef..ed4aff6cd 100644
--- a/qemu/include/hw/pci/pci_bridge.h
+++ b/qemu/include/hw/pci/pci_bridge.h
@@ -48,7 +48,7 @@ void pci_bridge_disable_base_limit(PCIDevice *dev);
 void pci_bridge_reset_reg(PCIDevice *dev);
 void pci_bridge_reset(DeviceState *qdev);
 
-int pci_bridge_initfn(PCIDevice *pci_dev, const char *typename);
+void pci_bridge_initfn(PCIDevice *pci_dev, const char *typename);
 void pci_bridge_exitfn(PCIDevice *pci_dev);
 
 
diff --git a/qemu/include/hw/pci/pci_ids.h b/qemu/include/hw/pci/pci_ids.h
index d98e6c915..db85afa03 100644
--- a/qemu/include/hw/pci/pci_ids.h
+++ b/qemu/include/hw/pci/pci_ids.h
@@ -64,6 +64,7 @@
 #define PCI_VENDOR_ID_LSI_LOGIC          0x1000
 #define PCI_DEVICE_ID_LSI_53C810         0x0001
 #define PCI_DEVICE_ID_LSI_53C895A        0x0012
+#define PCI_DEVICE_ID_LSI_SAS1068        0x0054
 #define PCI_DEVICE_ID_LSI_SAS1078        0x0060
 #define PCI_DEVICE_ID_LSI_SAS0079        0x0079
 
diff --git a/qemu/include/hw/pci/pcie_aer.h b/qemu/include/hw/pci/pcie_aer.h
index 2fb83882b..c2ee4e2bd 100644
--- a/qemu/include/hw/pci/pcie_aer.h
+++ b/qemu/include/hw/pci/pcie_aer.h
@@ -87,7 +87,7 @@ struct PCIEAERErr {
 
 extern const VMStateDescription vmstate_pcie_aer_log;
 
-int pcie_aer_init(PCIDevice *dev, uint16_t offset);
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size);
 void pcie_aer_exit(PCIDevice *dev);
 void pcie_aer_write_config(PCIDevice *dev,
                            uint32_t addr, uint32_t val, int len);
@@ -102,5 +102,6 @@ void pcie_aer_root_write_config(PCIDevice *dev,
 
 /* error injection */
 int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err);
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg);
 
 #endif /* QEMU_PCIE_AER_H */
diff --git a/qemu/include/hw/pci/shpc.h b/qemu/include/hw/pci/shpc.h
index 2c871b947..b2085543d 100644
--- a/qemu/include/hw/pci/shpc.h
+++ b/qemu/include/hw/pci/shpc.h
@@ -4,7 +4,6 @@
 #include "qemu-common.h"
 #include "exec/memory.h"
 #include "migration/vmstate.h"
-#include "qapi/error.h"
 #include "hw/hotplug.h"
 #include "hw/pci/pci.h"
 
diff --git a/qemu/include/hw/ppc/mac_dbdma.h b/qemu/include/hw/ppc/mac_dbdma.h
index c6870212e..0cce4e8bb 100644
--- a/qemu/include/hw/ppc/mac_dbdma.h
+++ b/qemu/include/hw/ppc/mac_dbdma.h
@@ -23,6 +23,7 @@
 #define HW_MAC_DBDMA_H 1
 
 #include "exec/memory.h"
+#include "qemu/iov.h"
 
 typedef struct DBDMA_io DBDMA_io;
 
diff --git a/qemu/include/hw/ppc/spapr.h b/qemu/include/hw/ppc/spapr.h
index 91a61abbc..815d5eec4 100644
--- a/qemu/include/hw/ppc/spapr.h
+++ b/qemu/include/hw/ppc/spapr.h
@@ -5,6 +5,7 @@
 #include "hw/boards.h"
 #include "hw/ppc/xics.h"
 #include "hw/ppc/spapr_drc.h"
+#include "hw/mem/pc-dimm.h"
 
 struct VIOsPAPRBus;
 struct sPAPRPHBState;
@@ -34,6 +35,8 @@ struct sPAPRMachineClass {
     MachineClass parent_class;
 
     /*< public >*/
+    bool dr_lmb_enabled;       /* enable dynamic-reconfig/hotplug of LMBs */
+    bool use_ohci_by_default;  /* use USB-OHCI instead of XHCI */
 };
 
 /**
@@ -69,13 +72,13 @@ struct sPAPRMachineState {
     int htab_save_index;
     bool htab_first_pass;
     int htab_fd;
-    bool htab_fd_stale;
 
     /* RTAS state */
     QTAILQ_HEAD(, sPAPRConfigureConnectorState) ccs_list;
 
     /*< public >*/
     char *kvm_type;
+    MemoryHotplugState hotplug_memory;
 };
 
 #define H_SUCCESS         0
@@ -201,11 +204,6 @@ struct sPAPRMachineState {
 #define H_SET_MODE_ENDIAN_BIG    0
 #define H_SET_MODE_ENDIAN_LITTLE 1
 
-/* Flags for H_SET_MODE_RESOURCE_ADDR_TRANS_MODE */
-#define H_SET_MODE_ADDR_TRANS_NONE                  0
-#define H_SET_MODE_ADDR_TRANS_0001_8000             2
-#define H_SET_MODE_ADDR_TRANS_C000_0000_0000_4000   3
-
 /* VASI States */
 #define H_VASI_INVALID    0
 #define H_VASI_ENABLED    1
@@ -331,6 +329,7 @@ struct sPAPRMachineState {
 #define H_SET_MPP               0x2D0
 #define H_GET_MPP               0x2D4
 #define H_XIRR_X                0x2FC
+#define H_RANDOM                0x300
 #define H_SET_MODE              0x31C
 #define MAX_HCALL_OPCODE        H_SET_MODE
 
@@ -353,15 +352,10 @@ typedef struct sPAPRDeviceTreeUpdateHeader {
     uint32_t version_id;
 } sPAPRDeviceTreeUpdateHeader;
 
-/*#define DEBUG_SPAPR_HCALLS*/
-
-#ifdef DEBUG_SPAPR_HCALLS
-#define hcall_dprintf(fmt, ...) \
-    do { fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); } while (0)
-#else
 #define hcall_dprintf(fmt, ...) \
-    do { } while (0)
-#endif
+    do { \
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: " fmt, __func__, ## __VA_ARGS__); \
+    } while (0)
 
 typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm,
                                        target_ulong opcode,
@@ -408,13 +402,15 @@ int spapr_allocate_irq_block(int num, bool lsi, bool msi);
 #define RTAS_SLOT_PERM_ERR_LOG           2
 
 /* RTAS return codes */
-#define RTAS_OUT_SUCCESS            0
-#define RTAS_OUT_NO_ERRORS_FOUND    1
-#define RTAS_OUT_HW_ERROR           -1
-#define RTAS_OUT_BUSY               -2
-#define RTAS_OUT_PARAM_ERROR        -3
-#define RTAS_OUT_NOT_SUPPORTED      -3
-#define RTAS_OUT_NOT_AUTHORIZED     -9002
+#define RTAS_OUT_SUCCESS                        0
+#define RTAS_OUT_NO_ERRORS_FOUND                1
+#define RTAS_OUT_HW_ERROR                       -1
+#define RTAS_OUT_BUSY                           -2
+#define RTAS_OUT_PARAM_ERROR                    -3
+#define RTAS_OUT_NOT_SUPPORTED                  -3
+#define RTAS_OUT_NO_SUCH_INDICATOR              -3
+#define RTAS_OUT_NOT_AUTHORIZED                 -9002
+#define RTAS_OUT_SYSPARM_PARAM_ERROR            -9999
 
 /* RTAS tokens */
 #define RTAS_TOKEN_BASE      0x2000
@@ -494,28 +490,14 @@ static inline uint32_t rtas_ld(target_ulong phys, int n)
     return ldl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n));
 }
 
-static inline void rtas_st(target_ulong phys, int n, uint32_t val)
-{
-    stl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n), val);
-}
-
-static inline void rtas_st_buffer_direct(target_ulong phys,
-                                         target_ulong phys_len,
-                                         uint8_t *buffer, uint16_t buffer_len)
+static inline uint64_t rtas_ldq(target_ulong phys, int n)
 {
-    cpu_physical_memory_write(ppc64_phys_to_real(phys), buffer,
-                              MIN(buffer_len, phys_len));
+    return (uint64_t)rtas_ld(phys, n) << 32 | rtas_ld(phys, n + 1);
 }
 
-static inline void rtas_st_buffer(target_ulong phys, target_ulong phys_len,
-                                  uint8_t *buffer, uint16_t buffer_len)
+static inline void rtas_st(target_ulong phys, int n, uint32_t val)
 {
-    if (phys_len < 2) {
-        return;
-    }
-    stw_be_phys(&address_space_memory,
-                ppc64_phys_to_real(phys), buffer_len);
-    rtas_st_buffer_direct(phys + 2, phys_len - 2, buffer, buffer_len);
+    stl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n), val);
 }
 
 typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm,
@@ -558,7 +540,7 @@ struct sPAPRTCETable {
     uint32_t page_shift;
     uint64_t *table;
     bool bypass;
-    bool vfio_accel;
+    bool need_vfio;
     int fd;
     MemoryRegion iommu;
     struct VIOsPAPRDevice *vdev; /* for @bypass migration compatibility only */
@@ -577,20 +559,27 @@ struct sPAPREventLogEntry {
 void spapr_events_init(sPAPRMachineState *sm);
 void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
 int spapr_h_cas_compose_response(sPAPRMachineState *sm,
-                                 target_ulong addr, target_ulong size);
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update);
 sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
                                    uint64_t bus_offset,
                                    uint32_t page_shift,
                                    uint32_t nb_table,
-                                   bool vfio_accel);
+                                   bool need_vfio);
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio);
+
 MemoryRegion *spapr_tce_get_iommu(sPAPRTCETable *tcet);
 int spapr_dma_dt(void *fdt, int node_off, const char *propname,
                  uint32_t liobn, uint64_t window, uint32_t size);
 int spapr_tcet_dma_dt(void *fdt, int node_off, const char *propname,
                       sPAPRTCETable *tcet);
 void spapr_pci_switch_vga(bool big_endian);
-void spapr_hotplug_req_add_event(sPAPRDRConnector *drc);
-void spapr_hotplug_req_remove_event(sPAPRDRConnector *drc);
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count);
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count);
 
 /* rtas-configure-connector state */
 struct sPAPRConfigureConnectorState {
@@ -603,10 +592,35 @@ struct sPAPRConfigureConnectorState {
 void spapr_ccs_reset_hook(void *opaque);
 
 #define TYPE_SPAPR_RTC "spapr-rtc"
+#define TYPE_SPAPR_RNG "spapr-rng"
 
 void spapr_rtc_read(DeviceState *dev, struct tm *tm, uint32_t *ns);
 int spapr_rtc_import_offset(DeviceState *dev, int64_t legacy_offset);
 
+int spapr_rng_populate_dt(void *fdt);
+
 #define SPAPR_MEMORY_BLOCK_SIZE (1 << 28) /* 256MB */
 
+/*
+ * This defines the maximum number of DIMM slots we can have for sPAPR
+ * guest. This is not defined by sPAPR but we are defining it to 32 slots
+ * based on default number of slots provided by PowerPC kernel.
+ */
+#define SPAPR_MAX_RAM_SLOTS     32
+
+/* 1GB alignment for hotplug memory region */
+#define SPAPR_HOTPLUG_MEM_ALIGN (1ULL << 30)
+
+/*
+ * Number of 32 bit words in each LMB list entry in ibm,dynamic-memory
+ * property under ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_DR_LMB_LIST_ENTRY_SIZE 6
+
+/*
+ * This flag value defines the LMB as assigned in ibm,dynamic-memory
+ * property under ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_LMB_FLAGS_ASSIGNED 0x00000008
+
 #endif /* !defined (__HW_SPAPR_H__) */
diff --git a/qemu/include/hw/ppc/spapr_drc.h b/qemu/include/hw/ppc/spapr_drc.h
index 60cda35ed..fa21ba044 100644
--- a/qemu/include/hw/ppc/spapr_drc.h
+++ b/qemu/include/hw/ppc/spapr_drc.h
@@ -119,13 +119,14 @@ typedef enum {
 } sPAPRDREntitySense;
 
 typedef enum {
-    SPAPR_DR_CC_RESPONSE_NEXT_SIB       = 1, /* currently unused */
-    SPAPR_DR_CC_RESPONSE_NEXT_CHILD     = 2,
-    SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY  = 3,
-    SPAPR_DR_CC_RESPONSE_PREV_PARENT    = 4,
-    SPAPR_DR_CC_RESPONSE_SUCCESS        = 0,
-    SPAPR_DR_CC_RESPONSE_ERROR          = -1,
-    SPAPR_DR_CC_RESPONSE_CONTINUE       = -2,
+    SPAPR_DR_CC_RESPONSE_NEXT_SIB         = 1, /* currently unused */
+    SPAPR_DR_CC_RESPONSE_NEXT_CHILD       = 2,
+    SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY    = 3,
+    SPAPR_DR_CC_RESPONSE_PREV_PARENT      = 4,
+    SPAPR_DR_CC_RESPONSE_SUCCESS          = 0,
+    SPAPR_DR_CC_RESPONSE_ERROR            = -1,
+    SPAPR_DR_CC_RESPONSE_CONTINUE         = -2,
+    SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE = -9003,
 } sPAPRDRCCResponse;
 
 typedef void (spapr_drc_detach_cb)(DeviceState *d, void *opaque);
@@ -150,6 +151,7 @@ typedef struct sPAPRDRConnector {
     bool configured;
 
     bool awaiting_release;
+    bool signalled;
 
     /* device pointer, via link property */
     DeviceState *dev;
@@ -164,17 +166,17 @@ typedef struct sPAPRDRConnectorClass {
     /*< public >*/
 
     /* accessors for guest-visible (generally via RTAS) DR state */
-    int (*set_isolation_state)(sPAPRDRConnector *drc,
-                               sPAPRDRIsolationState state);
-    int (*set_indicator_state)(sPAPRDRConnector *drc,
-                               sPAPRDRIndicatorState state);
-    int (*set_allocation_state)(sPAPRDRConnector *drc,
-                                sPAPRDRAllocationState state);
+    uint32_t (*set_isolation_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state);
+    uint32_t (*set_indicator_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state);
+    uint32_t (*set_allocation_state)(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state);
     uint32_t (*get_index)(sPAPRDRConnector *drc);
     uint32_t (*get_type)(sPAPRDRConnector *drc);
     const char *(*get_name)(sPAPRDRConnector *drc);
 
-    sPAPRDREntitySense (*entity_sense)(sPAPRDRConnector *drc);
+    uint32_t (*entity_sense)(sPAPRDRConnector *drc, sPAPRDREntitySense *state);
 
     /* QEMU interfaces for managing FDT/configure-connector */
     const void *(*get_fdt)(sPAPRDRConnector *drc, int *fdt_start_offset);
@@ -187,6 +189,7 @@ typedef struct sPAPRDRConnectorClass {
                    spapr_drc_detach_cb *detach_cb,
                    void *detach_cb_opaque, Error **errp);
     bool (*release_pending)(sPAPRDRConnector *drc);
+    void (*set_signalled)(sPAPRDRConnector *drc);
 } sPAPRDRConnectorClass;
 
 sPAPRDRConnector *spapr_dr_connector_new(Object *owner,
diff --git a/qemu/include/hw/ppc/spapr_vio.h b/qemu/include/hw/ppc/spapr_vio.h
index 2299a5405..c9733e755 100644
--- a/qemu/include/hw/ppc/spapr_vio.h
+++ b/qemu/include/hw/ppc/spapr_vio.h
@@ -34,7 +34,7 @@
 #define TYPE_SPAPR_VIO_BUS "spapr-vio-bus"
 #define SPAPR_VIO_BUS(obj) OBJECT_CHECK(VIOsPAPRBus, (obj), TYPE_SPAPR_VIO_BUS)
 
-struct VIOsPAPRDevice;
+#define TYPE_SPAPR_VIO_BRIDGE "spapr-vio-bridge"
 
 typedef struct VIOsPAPR_CRQ {
     uint64_t qladdr;
diff --git a/qemu/include/hw/ppc/xics.h b/qemu/include/hw/ppc/xics.h
index 355a96623..f60b06ae8 100644
--- a/qemu/include/hw/ppc/xics.h
+++ b/qemu/include/hw/ppc/xics.h
@@ -161,8 +161,9 @@ struct ICSIRQState {
 
 qemu_irq xics_get_qirq(XICSState *icp, int irq);
 void xics_set_irq_type(XICSState *icp, int irq, bool lsi);
-int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi);
-int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align);
+int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi, Error **errp);
+int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align,
+                     Error **errp);
 void xics_free(XICSState *icp, int irq, int num);
 
 void xics_cpu_setup(XICSState *icp, PowerPCCPU *cpu);
diff --git a/qemu/include/hw/qdev-core.h b/qemu/include/hw/qdev-core.h
index 038b54d94..1ce02b20d 100644
--- a/qemu/include/hw/qdev-core.h
+++ b/qemu/include/hw/qdev-core.h
@@ -3,11 +3,9 @@
 
 #include "qemu/queue.h"
 #include "qemu/option.h"
-#include "qemu/typedefs.h"
 #include "qemu/bitmap.h"
 #include "qom/object.h"
 #include "hw/irq.h"
-#include "qapi/error.h"
 #include "hw/hotplug.h"
 
 enum {
@@ -114,6 +112,19 @@ typedef struct DeviceClass {
      * TODO remove once we're there
      */
     bool cannot_instantiate_with_device_add_yet;
+    /*
+     * Does this device model survive object_unref(object_new(TNAME))?
+     * All device models should, and this flag shouldn't exist.  Some
+     * devices crash in object_new(), some crash or hang in
+     * object_unref().  Makes introspecting properties with
+     * qmp_device_list_properties() dangerous.  Bad, because it's used
+     * by -device FOO,help.  This flag serves to protect that code.
+     * It should never be set without a comment explaining why it is
+     * set.
+     * TODO remove once we're there
+     */
+    bool cannot_destroy_with_object_finalize_yet;
+
     bool hotpluggable;
 
     /* callbacks */
@@ -224,9 +235,9 @@ struct BusState {
 struct Property {
     const char   *name;
     PropertyInfo *info;
-    int          offset;
+    ptrdiff_t    offset;
     uint8_t      bitnr;
-    qtype_code   qtype;
+    QType        qtype;
     int64_t      defval;
     int          arrayoffset;
     PropertyInfo *arrayinfo;
@@ -324,6 +335,7 @@ int qdev_walk_children(DeviceState *dev,
                        void *opaque);
 
 void qdev_reset_all(DeviceState *dev);
+void qdev_reset_all_fn(void *opaque);
 
 /**
  * @qbus_reset_all:
diff --git a/qemu/include/hw/qdev-properties.h b/qemu/include/hw/qdev-properties.h
index 77538a8ca..0586cacce 100644
--- a/qemu/include/hw/qdev-properties.h
+++ b/qemu/include/hw/qdev-properties.h
@@ -18,8 +18,10 @@ extern PropertyInfo qdev_prop_string;
 extern PropertyInfo qdev_prop_chr;
 extern PropertyInfo qdev_prop_ptr;
 extern PropertyInfo qdev_prop_macaddr;
+extern PropertyInfo qdev_prop_on_off_auto;
 extern PropertyInfo qdev_prop_losttickpolicy;
 extern PropertyInfo qdev_prop_bios_chs_trans;
+extern PropertyInfo qdev_prop_fdc_drive_type;
 extern PropertyInfo qdev_prop_drive;
 extern PropertyInfo qdev_prop_netdev;
 extern PropertyInfo qdev_prop_vlan;
@@ -154,6 +156,8 @@ extern PropertyInfo qdev_prop_arraylen;
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_ON_OFF_AUTO(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_on_off_auto, OnOffAuto)
 #define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
     DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
                         LostTickPolicy)
@@ -180,8 +184,6 @@ void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *valu
 void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
 void qdev_prop_set_drive(DeviceState *dev, const char *name,
                          BlockBackend *value, Error **errp);
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
-                                BlockBackend *value);
 void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
 void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
 /* FIXME: Remove opaque pointer properties.  */
diff --git a/qemu/include/hw/s390x/event-facility.h b/qemu/include/hw/s390x/event-facility.h
index 6a062b668..dd8881838 100644
--- a/qemu/include/hw/s390x/event-facility.h
+++ b/qemu/include/hw/s390x/event-facility.h
@@ -47,6 +47,7 @@
      OBJECT_GET_CLASS(SCLPEventClass, (obj), TYPE_SCLP_EVENT)
 
 #define TYPE_SCLP_CPU_HOTPLUG "sclp-cpu-hotplug"
+#define TYPE_SCLP_QUIESCE "sclpquiesce"
 
 typedef struct WriteEventMask {
     SCCBHeader h;
@@ -146,8 +147,10 @@ typedef struct WriteEventData {
 
 typedef struct ReadEventData {
     SCCBHeader h;
-    EventBufferHeader ebh;
-    uint32_t mask;
+    union {
+        uint32_t mask;
+        EventBufferHeader ebh;
+    };
 } QEMU_PACKED ReadEventData;
 
 typedef struct SCLPEvent {
@@ -186,11 +189,8 @@ typedef struct SCLPEventClass {
      OBJECT_GET_CLASS(SCLPEventFacilityClass, (obj), \
                       TYPE_SCLP_EVENT_FACILITY)
 
-typedef struct SCLPEventFacility SCLPEventFacility;
-
 typedef struct SCLPEventFacilityClass {
-    DeviceClass parent_class;
-    int (*init)(SCLPEventFacility *ef);
+    SysBusDeviceClass parent_class;
     void (*command_handler)(SCLPEventFacility *ef, SCCB *sccb, uint64_t code);
     bool (*event_pending)(SCLPEventFacility *ef);
 } SCLPEventFacilityClass;
diff --git a/qemu/include/hw/s390x/s390-virtio-ccw.h b/qemu/include/hw/s390x/s390-virtio-ccw.h
new file mode 100644
index 000000000..ab08332fe
--- /dev/null
+++ b/qemu/include/hw/s390x/s390-virtio-ccw.h
@@ -0,0 +1,40 @@
+/*
+ * virtio ccw machine definitions
+ *
+ * Copyright 2012, 2016 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390X_S390_VIRTIO_CCW_H
+#define HW_S390X_S390_VIRTIO_CCW_H
+
+#include "hw/boards.h"
+
+#define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
+
+#define S390_CCW_MACHINE(obj) \
+    OBJECT_CHECK(S390CcwMachineState, (obj), TYPE_S390_CCW_MACHINE)
+
+#define S390_MACHINE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390CcwMachineClass, (klass), TYPE_S390_CCW_MACHINE)
+
+typedef struct S390CcwMachineState {
+    /*< private >*/
+    MachineState parent_obj;
+
+    /*< public >*/
+    bool aes_key_wrap;
+    bool dea_key_wrap;
+} S390CcwMachineState;
+
+typedef struct S390CcwMachineClass {
+    /*< private >*/
+    MachineClass parent_class;
+
+    /*< public >*/
+} S390CcwMachineClass;
+
+#endif
diff --git a/qemu/include/hw/s390x/sclp.h b/qemu/include/hw/s390x/sclp.h
index e8a64e25b..b0c71b555 100644
--- a/qemu/include/hw/s390x/sclp.h
+++ b/qemu/include/hw/s390x/sclp.h
@@ -163,6 +163,39 @@ typedef struct SCCB {
     char data[SCCB_DATA_LEN];
  } QEMU_PACKED SCCB;
 
+#define TYPE_SCLP "sclp"
+#define SCLP(obj) OBJECT_CHECK(SCLPDevice, (obj), TYPE_SCLP)
+#define SCLP_CLASS(oc) OBJECT_CLASS_CHECK(SCLPDeviceClass, (oc), TYPE_SCLP)
+#define SCLP_GET_CLASS(obj) OBJECT_GET_CLASS(SCLPDeviceClass, (obj), TYPE_SCLP)
+
+typedef struct SCLPEventFacility SCLPEventFacility;
+
+typedef struct SCLPDevice {
+    /* private */
+    DeviceState parent_obj;
+    SCLPEventFacility *event_facility;
+    int increment_size;
+
+    /* public */
+} SCLPDevice;
+
+typedef struct SCLPDeviceClass {
+    /* private */
+    DeviceClass parent_class;
+    void (*read_SCP_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element0_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element1_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*attach_storage_element)(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element);
+    void (*assign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*unassign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_cpu_info)(SCLPDevice *sclp, SCCB *sccb);
+
+    /* public */
+    void (*execute)(SCLPDevice *sclp, SCCB *sccb, uint32_t code);
+    void (*service_interrupt)(SCLPDevice *sclp, uint32_t sccb);
+} SCLPDeviceClass;
+
 typedef struct sclpMemoryHotplugDev sclpMemoryHotplugDev;
 
 #define TYPE_SCLP_MEMORY_HOTPLUG_DEV "sclp-memory-hotplug-dev"
diff --git a/qemu/include/hw/s390x/storage-keys.h b/qemu/include/hw/s390x/storage-keys.h
new file mode 100644
index 000000000..72b850cb1
--- /dev/null
+++ b/qemu/include/hw/s390x/storage-keys.h
@@ -0,0 +1,60 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef __S390_STORAGE_KEYS_H
+#define __S390_STORAGE_KEYS_H
+
+#include <hw/qdev.h>
+#include "monitor/monitor.h"
+
+#define TYPE_S390_SKEYS "s390-skeys"
+#define S390_SKEYS(obj) \
+    OBJECT_CHECK(S390SKeysState, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysState {
+    DeviceState parent_obj;
+    bool migration_enabled;
+
+} S390SKeysState;
+
+#define S390_SKEYS_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390SKeysClass, (klass), TYPE_S390_SKEYS)
+#define S390_SKEYS_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390SKeysClass, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysClass {
+    DeviceClass parent_class;
+    int (*skeys_enabled)(S390SKeysState *ks);
+    int (*get_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+    int (*set_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+} S390SKeysClass;
+
+#define TYPE_KVM_S390_SKEYS "s390-skeys-kvm"
+#define TYPE_QEMU_S390_SKEYS "s390-skeys-qemu"
+#define QEMU_S390_SKEYS(obj) \
+    OBJECT_CHECK(QEMUS390SKeysState, (obj), TYPE_QEMU_S390_SKEYS)
+
+typedef struct QEMUS390SKeysState {
+    S390SKeysState parent_obj;
+    uint8_t *keydata;
+    uint32_t key_count;
+} QEMUS390SKeysState;
+
+void s390_skeys_init(void);
+
+S390SKeysState *s390_get_skeys_device(void);
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict);
+void hmp_info_skeys(Monitor *mon, const QDict *qdict);
+
+#endif /* __S390_STORAGE_KEYS_H */
diff --git a/qemu/include/hw/scsi/scsi.h b/qemu/include/hw/scsi/scsi.h
index cdaf0f8eb..8acd3fa99 100644
--- a/qemu/include/hw/scsi/scsi.h
+++ b/qemu/include/hw/scsi/scsi.h
@@ -2,7 +2,6 @@
 #define QEMU_HW_SCSI_H
 
 #include "hw/qdev.h"
-#include "qemu/typedefs.h"
 #include "hw/block/block.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
@@ -108,6 +107,8 @@ struct SCSIDevice
     int blocksize;
     int type;
     uint64_t max_lba;
+    uint64_t wwn;
+    uint64_t port_wwn;
 };
 
 extern const VMStateDescription vmstate_scsi_device;
@@ -250,7 +251,6 @@ SCSIRequest *scsi_req_alloc(const SCSIReqOps *reqops, SCSIDevice *d,
 SCSIRequest *scsi_req_new(SCSIDevice *d, uint32_t tag, uint32_t lun,
                           uint8_t *buf, void *hba_private);
 int32_t scsi_req_enqueue(SCSIRequest *req);
-void scsi_req_free(SCSIRequest *req);
 SCSIRequest *scsi_req_ref(SCSIRequest *req);
 void scsi_req_unref(SCSIRequest *req);
 
@@ -272,6 +272,7 @@ void scsi_device_purge_requests(SCSIDevice *sdev, SCSISense sense);
 void scsi_device_set_ua(SCSIDevice *sdev, SCSISense sense);
 void scsi_device_report_change(SCSIDevice *dev, SCSISense sense);
 void scsi_device_unit_attention_reported(SCSIDevice *dev);
+void scsi_generic_read_device_identification(SCSIDevice *dev);
 int scsi_device_get_sense(SCSIDevice *dev, uint8_t *buf, int len, bool fixed);
 SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int target, int lun);
 
diff --git a/qemu/include/hw/sd.h b/qemu/include/hw/sd/sd.h
index 79adb5bb4..d5d273a44 100644
--- a/qemu/include/hw/sd.h
+++ b/qemu/include/hw/sd/sd.h
@@ -67,7 +67,51 @@ typedef struct {
 } SDRequest;
 
 typedef struct SDState SDState;
+typedef struct SDBus SDBus;
 
+#define TYPE_SD_CARD "sd-card"
+#define SD_CARD(obj) OBJECT_CHECK(SDState, (obj), TYPE_SD_CARD)
+#define SD_CARD_CLASS(klass) \
+    OBJECT_CLASS_CHECK(SDCardClass, (klass), TYPE_SD_CARD)
+#define SD_CARD_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(SDCardClass, (obj), TYPE_SD_CARD)
+
+typedef struct {
+    /*< private >*/
+    DeviceClass parent_class;
+    /*< public >*/
+
+    int (*do_command)(SDState *sd, SDRequest *req, uint8_t *response);
+    void (*write_data)(SDState *sd, uint8_t value);
+    uint8_t (*read_data)(SDState *sd);
+    bool (*data_ready)(SDState *sd);
+    void (*enable)(SDState *sd, bool enable);
+    bool (*get_inserted)(SDState *sd);
+    bool (*get_readonly)(SDState *sd);
+} SDCardClass;
+
+#define TYPE_SD_BUS "sd-bus"
+#define SD_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_SD_BUS)
+#define SD_BUS_CLASS(klass) OBJECT_CLASS_CHECK(SDBusClass, (klass), TYPE_SD_BUS)
+#define SD_BUS_GET_CLASS(obj) OBJECT_GET_CLASS(SDBusClass, (obj), TYPE_SD_BUS)
+
+struct SDBus {
+    BusState qbus;
+};
+
+typedef struct {
+    /*< private >*/
+    BusClass parent_class;
+    /*< public >*/
+
+    /* These methods are called by the SD device to notify the controller
+     * when the card insertion or readonly status changes
+     */
+    void (*set_inserted)(DeviceState *dev, bool inserted);
+    void (*set_readonly)(DeviceState *dev, bool readonly);
+} SDBusClass;
+
+/* Legacy functions to be used only by non-qdevified callers */
 SDState *sd_init(BlockBackend *bs, bool is_spi);
 int sd_do_command(SDState *sd, SDRequest *req,
                   uint8_t *response);
@@ -75,6 +119,27 @@ void sd_write_data(SDState *sd, uint8_t value);
 uint8_t sd_read_data(SDState *sd);
 void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert);
 bool sd_data_ready(SDState *sd);
+/* sd_enable should not be used -- it is only used on the nseries boards,
+ * where it is part of a broken implementation of the MMC card slot switch
+ * (there should be two card slots which are multiplexed to a single MMC
+ * controller, but instead we model it with one card and controller and
+ * disable the card when the second slot is selected, so it looks like the
+ * second slot is always empty).
+ */
 void sd_enable(SDState *sd, bool enable);
 
+/* Functions to be used by qdevified callers (working via
+ * an SDBus rather than directly with SDState)
+ */
+int sdbus_do_command(SDBus *sd, SDRequest *req, uint8_t *response);
+void sdbus_write_data(SDBus *sd, uint8_t value);
+uint8_t sdbus_read_data(SDBus *sd);
+bool sdbus_data_ready(SDBus *sd);
+bool sdbus_get_inserted(SDBus *sd);
+bool sdbus_get_readonly(SDBus *sd);
+
+/* Functions to be used by SD devices to report back to qdevified controllers */
+void sdbus_set_inserted(SDBus *sd, bool inserted);
+void sdbus_set_readonly(SDBus *sd, bool inserted);
+
 #endif	/* __hw_sd_h */
diff --git a/qemu/include/hw/sd/sdhci.h b/qemu/include/hw/sd/sdhci.h
new file mode 100644
index 000000000..0f0c3f1e6
--- /dev/null
+++ b/qemu/include/hw/sd/sdhci.h
@@ -0,0 +1,95 @@
+/*
+ * SD Association Host Standard Specification v2.0 controller emulation
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ * Mitsyanko Igor <i.mitsyanko@samsung.com>
+ * Peter A.G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ *
+ * Based on MMC controller for Samsung S5PC1xx-based board emulation
+ * by Alexey Merkulov and Vladimir Monakhov.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU _General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SDHCI_H
+#define SDHCI_H
+
+#include "qemu-common.h"
+#include "hw/block/block.h"
+#include "hw/pci/pci.h"
+#include "hw/sysbus.h"
+#include "hw/sd/sd.h"
+
+/* SD/MMC host controller state */
+typedef struct SDHCIState {
+    union {
+        PCIDevice pcidev;
+        SysBusDevice busdev;
+    };
+    SDBus sdbus;
+    MemoryRegion iomem;
+
+    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
+    QEMUTimer *transfer_timer;
+    qemu_irq eject_cb;
+    qemu_irq ro_cb;
+    qemu_irq irq;
+
+    uint32_t sdmasysad;    /* SDMA System Address register */
+    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
+    uint16_t blkcnt;       /* Blocks count for current transfer */
+    uint32_t argument;     /* Command Argument Register */
+    uint16_t trnmod;       /* Transfer Mode Setting Register */
+    uint16_t cmdreg;       /* Command Register */
+    uint32_t rspreg[4];    /* Response Registers 0-3 */
+    uint32_t prnsts;       /* Present State Register */
+    uint8_t  hostctl;      /* Host Control Register */
+    uint8_t  pwrcon;       /* Power control Register */
+    uint8_t  blkgap;       /* Block Gap Control Register */
+    uint8_t  wakcon;       /* WakeUp Control Register */
+    uint16_t clkcon;       /* Clock control Register */
+    uint8_t  timeoutcon;   /* Timeout Control Register */
+    uint8_t  admaerr;      /* ADMA Error Status Register */
+    uint16_t norintsts;    /* Normal Interrupt Status Register */
+    uint16_t errintsts;    /* Error Interrupt Status Register */
+    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
+    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
+    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
+    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
+    uint16_t acmd12errsts; /* Auto CMD12 error status register */
+    uint64_t admasysaddr;  /* ADMA System Address Register */
+
+    uint32_t capareg;      /* Capabilities Register */
+    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
+    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
+    uint32_t buf_maxsz;
+    uint16_t data_count;   /* current element in FIFO buffer */
+    uint8_t  stopped_state;/* Current SDHC state */
+    bool     pending_insert_quirk;/* Quirk for Raspberry Pi card insert int */
+    bool     pending_insert_state;
+    /* Buffer Data Port Register - virtual access point to R and W buffers */
+    /* Software Reset Register - always reads as 0 */
+    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
+    /* Force Event Error Interrupt Register- write only */
+    /* RO Host Controller Version Register always reads as 0x2401 */
+} SDHCIState;
+
+#define TYPE_PCI_SDHCI "sdhci-pci"
+#define PCI_SDHCI(obj) OBJECT_CHECK(SDHCIState, (obj), TYPE_PCI_SDHCI)
+
+#define TYPE_SYSBUS_SDHCI "generic-sdhci"
+#define SYSBUS_SDHCI(obj)                               \
+     OBJECT_CHECK(SDHCIState, (obj), TYPE_SYSBUS_SDHCI)
+
+#endif /* SDHCI_H */
diff --git a/qemu/include/hw/i386/smbios.h b/qemu/include/hw/smbios/smbios.h
index d2850bed2..76ccf7098 100644
--- a/qemu/include/hw/i386/smbios.h
+++ b/qemu/include/hw/smbios/smbios.h
@@ -17,23 +17,33 @@
 
 #define SMBIOS_MAX_TYPE 127
 
-void smbios_entry_add(QemuOpts *opts);
-void smbios_set_cpuid(uint32_t version, uint32_t features);
-void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode,
-                         bool uuid_encoded);
-uint8_t *smbios_get_table_legacy(size_t *length);
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
-                       uint8_t **anchor, size_t *anchor_len);
+/* memory area description, used by type 19 table */
+struct smbios_phys_mem_area {
+    uint64_t address;
+    uint64_t length;
+};
 
 /*
  * SMBIOS spec defined tables
  */
+typedef enum SmbiosEntryPointType {
+    SMBIOS_ENTRY_POINT_21,
+    SMBIOS_ENTRY_POINT_30,
+} SmbiosEntryPointType;
 
-/* SMBIOS entry point (anchor).
- * BIOS must place this at a 16-bit-aligned address between 0xf0000 and 0xfffff.
+/* SMBIOS Entry Point
+ * There are two types of entry points defined in the SMBIOS specification
+ * (see below). BIOS must place the entry point(s) at a 16-bit-aligned
+ * address between 0xf0000 and 0xfffff. Note that either entry point type
+ * can be used in a 64-bit target system, except that SMBIOS 2.1 entry point
+ * only allows the SMBIOS struct table to reside below 4GB address space.
  */
-struct smbios_entry_point {
+
+/* SMBIOS 2.1 (32-bit) Entry Point
+ *  - introduced since SMBIOS 2.1
+ *  - supports structure table below 4GB only
+ */
+struct smbios_21_entry_point {
     uint8_t anchor_string[4];
     uint8_t checksum;
     uint8_t length;
@@ -50,6 +60,28 @@ struct smbios_entry_point {
     uint8_t smbios_bcd_revision;
 } QEMU_PACKED;
 
+/* SMBIOS 3.0 (64-bit) Entry Point
+ *  - introduced since SMBIOS 3.0
+ *  - supports structure table at 64-bit address space
+ */
+struct smbios_30_entry_point {
+    uint8_t anchor_string[5];
+    uint8_t checksum;
+    uint8_t length;
+    uint8_t smbios_major_version;
+    uint8_t smbios_minor_version;
+    uint8_t smbios_doc_rev;
+    uint8_t entry_point_revision;
+    uint8_t reserved;
+    uint32_t structure_table_max_size;
+    uint64_t structure_table_address;
+} QEMU_PACKED;
+
+typedef union {
+    struct smbios_21_entry_point ep21;
+    struct smbios_30_entry_point ep30;
+} QEMU_PACKED SmbiosEntryPoint;
+
 /* This goes at the beginning of every SMBIOS structure. */
 struct smbios_structure_header {
     uint8_t type;
@@ -224,4 +256,14 @@ struct smbios_type_127 {
     struct smbios_structure_header header;
 } QEMU_PACKED;
 
+void smbios_entry_add(QemuOpts *opts);
+void smbios_set_cpuid(uint32_t version, uint32_t features);
+void smbios_set_defaults(const char *manufacturer, const char *product,
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded, SmbiosEntryPointType ep_type);
+uint8_t *smbios_get_table_legacy(size_t *length);
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
+                       uint8_t **anchor, size_t *anchor_len);
 #endif /*QEMU_SMBIOS_H */
diff --git a/qemu/include/hw/ssi.h b/qemu/include/hw/ssi/ssi.h
index df0f83851..4a0a53903 100644
--- a/qemu/include/hw/ssi.h
+++ b/qemu/include/hw/ssi/ssi.h
@@ -14,6 +14,8 @@
 #include "hw/qdev.h"
 
 typedef struct SSISlave SSISlave;
+typedef struct SSISlaveClass SSISlaveClass;
+typedef enum SSICSMode SSICSMode;
 
 #define TYPE_SSI_SLAVE "ssi-slave"
 #define SSI_SLAVE(obj) \
@@ -25,14 +27,14 @@ typedef struct SSISlave SSISlave;
 
 #define SSI_GPIO_CS "ssi-gpio-cs"
 
-typedef enum {
+enum SSICSMode {
     SSI_CS_NONE = 0,
     SSI_CS_LOW,
     SSI_CS_HIGH,
-} SSICSMode;
+};
 
 /* Slave devices.  */
-typedef struct SSISlaveClass {
+struct SSISlaveClass {
     DeviceClass parent_class;
 
     int (*init)(SSISlave *dev);
@@ -55,7 +57,7 @@ typedef struct SSISlaveClass {
      * always be called for the device for every txrx access to the parent bus
      */
     uint32_t (*transfer_raw)(SSISlave *dev, uint32_t val);
-} SSISlaveClass;
+};
 
 struct SSISlave {
     DeviceState parent_obj;
diff --git a/qemu/include/hw/ssi/xilinx_spips.h b/qemu/include/hw/ssi/xilinx_spips.h
new file mode 100644
index 000000000..dbb9eefba
--- /dev/null
+++ b/qemu/include/hw/ssi/xilinx_spips.h
@@ -0,0 +1,72 @@
+/*
+ * Header file for the Xilinx Zynq SPI controller
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef XLNX_SPIPS_H
+#define XLNX_SPIPS_H
+
+#include "hw/ssi/ssi.h"
+#include "qemu/fifo8.h"
+
+typedef struct XilinxSPIPS XilinxSPIPS;
+
+#define XLNX_SPIPS_R_MAX        (0x100 / 4)
+
+struct XilinxSPIPS {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    MemoryRegion mmlqspi;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    uint8_t num_busses;
+
+    uint8_t snoop_state;
+    qemu_irq *cs_lines;
+    SSIBus **spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint8_t num_txrx_bytes;
+
+    uint32_t regs[XLNX_SPIPS_R_MAX];
+};
+
+#define TYPE_XILINX_SPIPS "xlnx.ps7-spi"
+#define TYPE_XILINX_QSPIPS "xlnx.ps7-qspi"
+
+#define XILINX_SPIPS(obj) \
+     OBJECT_CHECK(XilinxSPIPS, (obj), TYPE_XILINX_SPIPS)
+#define XILINX_SPIPS_CLASS(klass) \
+     OBJECT_CLASS_CHECK(XilinxSPIPSClass, (klass), TYPE_XILINX_SPIPS)
+#define XILINX_SPIPS_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(XilinxSPIPSClass, (obj), TYPE_XILINX_SPIPS)
+
+#define XILINX_QSPIPS(obj) \
+     OBJECT_CHECK(XilinxQSPIPS, (obj), TYPE_XILINX_QSPIPS)
+
+#endif /* XLNX_SPIPS_H */
diff --git a/qemu/include/hw/timer/aspeed_timer.h b/qemu/include/hw/timer/aspeed_timer.h
new file mode 100644
index 000000000..44dc2f89d
--- /dev/null
+++ b/qemu/include/hw/timer/aspeed_timer.h
@@ -0,0 +1,59 @@
+/*
+ *  ASPEED AST2400 Timer
+ *
+ *  Andrew Jeffery <andrew@aj.id.au>
+ *
+ *  Copyright (C) 2016 IBM Corp.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#ifndef ASPEED_TIMER_H
+#define ASPEED_TIMER_H
+
+#include "hw/ptimer.h"
+
+#define ASPEED_TIMER(obj) \
+    OBJECT_CHECK(AspeedTimerCtrlState, (obj), TYPE_ASPEED_TIMER);
+#define TYPE_ASPEED_TIMER "aspeed.timer"
+#define ASPEED_TIMER_NR_TIMERS 8
+
+typedef struct AspeedTimer {
+    qemu_irq irq;
+
+    uint8_t id;
+
+    /**
+     * Track the line level as the ASPEED timers implement edge triggered
+     * interrupts, signalling with both the rising and falling edge.
+     */
+    int32_t level;
+    ptimer_state *timer;
+    uint32_t reload;
+    uint32_t match[2];
+} AspeedTimer;
+
+typedef struct AspeedTimerCtrlState {
+    /*< private >*/
+    SysBusDevice parent;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t ctrl;
+    uint32_t ctrl2;
+    AspeedTimer timers[ASPEED_TIMER_NR_TIMERS];
+} AspeedTimerCtrlState;
+
+#endif /* ASPEED_TIMER_H */
diff --git a/qemu/include/hw/timer/digic-timer.h b/qemu/include/hw/timer/digic-timer.h
index ae913482c..d9e67fe29 100644
--- a/qemu/include/hw/timer/digic-timer.h
+++ b/qemu/include/hw/timer/digic-timer.h
@@ -19,7 +19,6 @@
 #define HW_TIMER_DIGIC_TIMER_H
 
 #include "hw/sysbus.h"
-#include "qemu/typedefs.h"
 #include "hw/ptimer.h"
 
 #define TYPE_DIGIC_TIMER "digic-timer"
diff --git a/qemu/include/hw/timer/hpet.h b/qemu/include/hw/timer/hpet.h
index 773953be7..f38bcfecd 100644
--- a/qemu/include/hw/timer/hpet.h
+++ b/qemu/include/hw/timer/hpet.h
@@ -16,9 +16,10 @@
 #include "qom/object.h"
 
 #define HPET_BASE               0xfed00000
-#define HPET_CLK_PERIOD         10000000ULL /* 10000000 femtoseconds == 10ns*/
+#define HPET_LEN                0x400
+#define HPET_CLK_PERIOD         10 /* 10 ns*/
 
-#define FS_PER_NS 1000000
+#define FS_PER_NS 1000000       /* 1000000 femtoseconds == 1 ns */
 #define HPET_MIN_TIMERS         3
 #define HPET_MAX_TIMERS         32
 
diff --git a/qemu/include/hw/timer/imx_epit.h b/qemu/include/hw/timer/imx_epit.h
new file mode 100644
index 000000000..0730ac35e
--- /dev/null
+++ b/qemu/include/hw/timer/imx_epit.h
@@ -0,0 +1,80 @@
+/*
+ * i.MX EPIT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_EPIT_H
+#define IMX_EPIT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+#include "hw/misc/imx_ccm.h"
+
+/*
+ * EPIT: Enhanced periodic interrupt timer
+ */
+
+#define CR_EN       (1 << 0)
+#define CR_ENMOD    (1 << 1)
+#define CR_OCIEN    (1 << 2)
+#define CR_RLD      (1 << 3)
+#define CR_PRESCALE_SHIFT (4)
+#define CR_PRESCALE_MASK  (0xfff)
+#define CR_SWR      (1 << 16)
+#define CR_IOVW     (1 << 17)
+#define CR_DBGEN    (1 << 18)
+#define CR_WAITEN   (1 << 19)
+#define CR_DOZEN    (1 << 20)
+#define CR_STOPEN   (1 << 21)
+#define CR_CLKSRC_SHIFT (24)
+#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
+
+#define EPIT_TIMER_MAX  0XFFFFFFFFUL
+
+#define TYPE_IMX_EPIT "imx.epit"
+#define IMX_EPIT(obj) OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
+
+typedef struct IMXEPITState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer_reload;
+    ptimer_state *timer_cmp;
+    MemoryRegion  iomem;
+    IMXCCMState  *ccm;
+
+    uint32_t cr;
+    uint32_t sr;
+    uint32_t lr;
+    uint32_t cmp;
+    uint32_t cnt;
+
+    uint32_t freq;
+    qemu_irq irq;
+} IMXEPITState;
+
+#endif /* IMX_EPIT_H */
diff --git a/qemu/include/hw/timer/imx_gpt.h b/qemu/include/hw/timer/imx_gpt.h
new file mode 100644
index 000000000..461adbe53
--- /dev/null
+++ b/qemu/include/hw/timer/imx_gpt.h
@@ -0,0 +1,108 @@
+/*
+ * i.MX GPT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_GPT_H
+#define IMX_GPT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+#include "hw/misc/imx_ccm.h"
+
+/*
+ * GPT : General purpose timer
+ *
+ * This timer counts up continuously while it is enabled, resetting itself
+ * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
+ * reaches the value of one of the ocrX (in periodic mode).
+ */
+
+#define GPT_TIMER_MAX  0XFFFFFFFFUL
+
+/* Control register.  Not all of these bits have any effect (yet) */
+#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
+#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
+#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
+#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
+#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
+#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
+#define GPT_CR_CLKSRC_SHIFT (6)
+#define GPT_CR_CLKSRC_MASK  (0x7)
+
+#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
+#define GPT_CR_SWR    (1 << 15) /* Software Reset */
+#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
+#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
+#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
+#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
+#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
+#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
+#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
+#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
+
+#define GPT_SR_OF1  (1 << 0)
+#define GPT_SR_OF2  (1 << 1)
+#define GPT_SR_OF3  (1 << 2)
+#define GPT_SR_ROV  (1 << 5)
+
+#define GPT_IR_OF1IE  (1 << 0)
+#define GPT_IR_OF2IE  (1 << 1)
+#define GPT_IR_OF3IE  (1 << 2)
+#define GPT_IR_ROVIE  (1 << 5)
+
+#define TYPE_IMX_GPT "imx.gpt"
+#define IMX_GPT(obj) OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
+
+typedef struct IMXGPTState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer;
+    MemoryRegion  iomem;
+    IMXCCMState  *ccm;
+
+    uint32_t cr;
+    uint32_t pr;
+    uint32_t sr;
+    uint32_t ir;
+    uint32_t ocr1;
+    uint32_t ocr2;
+    uint32_t ocr3;
+    uint32_t icr1;
+    uint32_t icr2;
+    uint32_t cnt;
+
+    uint32_t next_timeout;
+    uint32_t next_int;
+
+    uint32_t freq;
+
+    qemu_irq irq;
+} IMXGPTState;
+
+#endif /* IMX_GPT_H */
diff --git a/qemu/include/hw/usb.h b/qemu/include/hw/usb.h
index c8b6e7b57..163fe0490 100644
--- a/qemu/include/hw/usb.h
+++ b/qemu/include/hw/usb.h
@@ -26,6 +26,7 @@
  */
 
 #include "hw/qdev.h"
+#include "qemu/iov.h"
 #include "qemu/queue.h"
 
 /* Constants related to the USB / PCI interaction */
diff --git a/qemu/include/hw/vfio/vfio-amd-xgbe.h b/qemu/include/hw/vfio/vfio-amd-xgbe.h
new file mode 100644
index 000000000..9fff65e99
--- /dev/null
+++ b/qemu/include/hw/vfio/vfio-amd-xgbe.h
@@ -0,0 +1,51 @@
+/*
+ * VFIO AMD XGBE device
+ *
+ * Copyright Linaro Limited, 2015
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VFIO_VFIO_AMD_XGBE_H
+#define HW_VFIO_VFIO_AMD_XGBE_H
+
+#include "hw/vfio/vfio-platform.h"
+
+#define TYPE_VFIO_AMD_XGBE "vfio-amd-xgbe"
+
+/**
+ * This device exposes:
+ * - 5 MMIO regions: MAC, PCS, SerDes Rx/Tx regs,
+     SerDes Integration Registers 1/2 & 2/2
+ * - 2 level sensitive IRQs and optional DMA channel IRQs
+ */
+struct VFIOAmdXgbeDevice {
+    VFIOPlatformDevice vdev;
+};
+
+typedef struct VFIOAmdXgbeDevice VFIOAmdXgbeDevice;
+
+struct VFIOAmdXgbeDeviceClass {
+    /*< private >*/
+    VFIOPlatformDeviceClass parent_class;
+    /*< public >*/
+    DeviceRealize parent_realize;
+};
+
+typedef struct VFIOAmdXgbeDeviceClass VFIOAmdXgbeDeviceClass;
+
+#define VFIO_AMD_XGBE_DEVICE(obj) \
+     OBJECT_CHECK(VFIOAmdXgbeDevice, (obj), TYPE_VFIO_AMD_XGBE)
+#define VFIO_AMD_XGBE_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(VFIOAmdXgbeDeviceClass, (klass), \
+                        TYPE_VFIO_AMD_XGBE)
+#define VFIO_AMD_XGBE_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(VFIOAmdXgbeDeviceClass, (obj), \
+                      TYPE_VFIO_AMD_XGBE)
+
+#endif
diff --git a/qemu/include/hw/vfio/vfio-common.h b/qemu/include/hw/vfio/vfio-common.h
index 59a321d47..eb0e1b034 100644
--- a/qemu/include/hw/vfio/vfio-common.h
+++ b/qemu/include/hw/vfio/vfio-common.h
@@ -25,6 +25,9 @@
 #include "exec/memory.h"
 #include "qemu/queue.h"
 #include "qemu/notify.h"
+#ifdef CONFIG_LINUX
+#include <linux/vfio.h>
+#endif
 
 /*#define DEBUG_VFIO*/
 #ifdef DEBUG_VFIO
@@ -35,24 +38,26 @@
     do { } while (0)
 #endif
 
-/* Extra debugging, trap acceleration paths for more logging */
-#define VFIO_ALLOW_KVM_INTX 1
-#define VFIO_ALLOW_KVM_MSI 1
-#define VFIO_ALLOW_KVM_MSIX 1
-
 enum {
     VFIO_DEVICE_TYPE_PCI = 0,
     VFIO_DEVICE_TYPE_PLATFORM = 1,
 };
 
+typedef struct VFIOMmap {
+    MemoryRegion mem;
+    void *mmap;
+    off_t offset;
+    size_t size;
+} VFIOMmap;
+
 typedef struct VFIORegion {
     struct VFIODevice *vbasedev;
     off_t fd_offset; /* offset of region within device fd */
-    MemoryRegion mem; /* slow, read/write access */
-    MemoryRegion mmap_mem; /* direct mapped access */
-    void *mmap;
+    MemoryRegion *mem; /* slow, read/write access */
     size_t size;
     uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
+    uint32_t nr_mmaps;
+    VFIOMmap *mmaps;
     uint8_t nr; /* cache the region number for debug */
 } VFIORegion;
 
@@ -64,22 +69,19 @@ typedef struct VFIOAddressSpace {
 
 struct VFIOGroup;
 
-typedef struct VFIOType1 {
-    MemoryListener listener;
-    int error;
-    bool initialized;
-} VFIOType1;
-
 typedef struct VFIOContainer {
     VFIOAddressSpace *space;
     int fd; /* /dev/vfio/vfio, empowered by the attached groups */
-    struct {
-        /* enable abstraction to support various iommu backends */
-        union {
-            VFIOType1 type1;
-        };
-        void (*release)(struct VFIOContainer *);
-    } iommu_data;
+    MemoryListener listener;
+    int error;
+    bool initialized;
+    /*
+     * This assumes the host IOMMU can support only a single
+     * contiguous IOVA window.  We may need to generalize that in
+     * future
+     */
+    hwaddr min_iova, max_iova;
+    uint64_t iova_pgsizes;
     QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
     QLIST_HEAD(, VFIOGroup) group_list;
     QLIST_ENTRY(VFIOContainer) next;
@@ -97,12 +99,13 @@ typedef struct VFIODeviceOps VFIODeviceOps;
 typedef struct VFIODevice {
     QLIST_ENTRY(VFIODevice) next;
     struct VFIOGroup *group;
+    char *sysfsdev;
     char *name;
     int fd;
     int type;
     bool reset_works;
     bool needs_reset;
-    bool allow_mmap;
+    bool no_mmap;
     VFIODeviceOps *ops;
     unsigned int num_irqs;
     unsigned int num_regions;
@@ -132,10 +135,12 @@ void vfio_region_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size);
 uint64_t vfio_region_read(void *opaque,
                           hwaddr addr, unsigned size);
-int vfio_mmap_region(Object *vdev, VFIORegion *region,
-                     MemoryRegion *mem, MemoryRegion *submem,
-                     void **map, size_t size, off_t offset,
-                     const char *name);
+int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
+                      int index, const char *name);
+int vfio_region_mmap(VFIORegion *region);
+void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled);
+void vfio_region_exit(VFIORegion *region);
+void vfio_region_finalize(VFIORegion *region);
 void vfio_reset_handler(void *opaque);
 VFIOGroup *vfio_get_group(int groupid, AddressSpace *as);
 void vfio_put_group(VFIOGroup *group);
@@ -146,4 +151,8 @@ extern const MemoryRegionOps vfio_region_ops;
 extern QLIST_HEAD(vfio_group_head, VFIOGroup) vfio_group_list;
 extern QLIST_HEAD(vfio_as_head, VFIOAddressSpace) vfio_address_spaces;
 
+#ifdef CONFIG_LINUX
+int vfio_get_region_info(VFIODevice *vbasedev, int index,
+                         struct vfio_region_info **info);
+#endif
 #endif /* !HW_VFIO_VFIO_COMMON_H */
diff --git a/qemu/include/hw/vfio/vfio-platform.h b/qemu/include/hw/vfio/vfio-platform.h
index c5cf1d79f..b468f80b1 100644
--- a/qemu/include/hw/vfio/vfio-platform.h
+++ b/qemu/include/hw/vfio/vfio-platform.h
@@ -34,8 +34,8 @@ enum {
 typedef struct VFIOINTp {
     QLIST_ENTRY(VFIOINTp) next; /* entry for IRQ list */
     QSIMPLEQ_ENTRY(VFIOINTp) pqnext; /* entry for pending IRQ queue */
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    EventNotifier *interrupt; /* eventfd triggered on interrupt */
+    EventNotifier *unmask; /* eventfd for unmask on QEMU bypass */
     qemu_irq qemuirq;
     struct VFIOPlatformDevice *vdev; /* back pointer to device */
     int state; /* inactive, pending, active */
diff --git a/qemu/include/hw/vfio/vfio.h b/qemu/include/hw/vfio/vfio.h
index 0b26cd8e1..f27d59922 100644
--- a/qemu/include/hw/vfio/vfio.h
+++ b/qemu/include/hw/vfio/vfio.h
@@ -1,9 +1,7 @@
 #ifndef VFIO_API_H
 #define VFIO_API_H
 
-#include "qemu/typedefs.h"
-
-extern int vfio_container_ioctl(AddressSpace *as, int32_t groupid,
-                                int req, void *param);
+bool vfio_eeh_as_ok(AddressSpace *as);
+int vfio_eeh_as_op(AddressSpace *as, uint32_t op);
 
 #endif
diff --git a/qemu/include/hw/virtio/dataplane/vring-accessors.h b/qemu/include/hw/virtio/dataplane/vring-accessors.h
deleted file mode 100644
index 815c19b6e..000000000
--- a/qemu/include/hw/virtio/dataplane/vring-accessors.h
+++ /dev/null
@@ -1,75 +0,0 @@
-#ifndef VRING_ACCESSORS_H
-#define VRING_ACCESSORS_H
-
-#include "standard-headers/linux/virtio_ring.h"
-#include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-access.h"
-
-static inline uint16_t vring_get_used_idx(VirtIODevice *vdev, Vring *vring)
-{
-    return virtio_tswap16(vdev, vring->vr.used->idx);
-}
-
-static inline void vring_set_used_idx(VirtIODevice *vdev, Vring *vring,
-                                      uint16_t idx)
-{
-    vring->vr.used->idx = virtio_tswap16(vdev, idx);
-}
-
-static inline uint16_t vring_get_avail_idx(VirtIODevice *vdev, Vring *vring)
-{
-    return virtio_tswap16(vdev, vring->vr.avail->idx);
-}
-
-static inline uint16_t vring_get_avail_ring(VirtIODevice *vdev, Vring *vring,
-                                            int i)
-{
-    return virtio_tswap16(vdev, vring->vr.avail->ring[i]);
-}
-
-static inline void vring_set_used_ring_id(VirtIODevice *vdev, Vring *vring,
-                                          int i, uint32_t id)
-{
-    vring->vr.used->ring[i].id = virtio_tswap32(vdev, id);
-}
-
-static inline void vring_set_used_ring_len(VirtIODevice *vdev, Vring *vring,
-                                          int i, uint32_t len)
-{
-    vring->vr.used->ring[i].len = virtio_tswap32(vdev, len);
-}
-
-static inline uint16_t vring_get_used_flags(VirtIODevice *vdev, Vring *vring)
-{
-    return virtio_tswap16(vdev, vring->vr.used->flags);
-}
-
-static inline uint16_t vring_get_avail_flags(VirtIODevice *vdev, Vring *vring)
-{
-    return virtio_tswap16(vdev, vring->vr.avail->flags);
-}
-
-static inline void vring_set_used_flags(VirtIODevice *vdev, Vring *vring,
-                                        uint16_t flags)
-{
-    vring->vr.used->flags |= virtio_tswap16(vdev, flags);
-}
-
-static inline void vring_clear_used_flags(VirtIODevice *vdev, Vring *vring,
-                                          uint16_t flags)
-{
-    vring->vr.used->flags &= virtio_tswap16(vdev, ~flags);
-}
-
-static inline unsigned int vring_get_num(Vring *vring)
-{
-    return vring->vr.num;
-}
-
-/* Are there more descriptors available? */
-static inline bool vring_more_avail(VirtIODevice *vdev, Vring *vring)
-{
-    return vring_get_avail_idx(vdev, vring) != vring->last_avail_idx;
-}
-
-#endif
diff --git a/qemu/include/hw/virtio/dataplane/vring.h b/qemu/include/hw/virtio/dataplane/vring.h
deleted file mode 100644
index 8d97db9e2..000000000
--- a/qemu/include/hw/virtio/dataplane/vring.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Copyright 2012 Red Hat, Inc. and/or its affiliates
- * Copyright IBM, Corp. 2012
- *
- * Based on Linux 2.6.39 vhost code:
- * Copyright (C) 2009 Red Hat, Inc.
- * Copyright (C) 2006 Rusty Russell IBM Corporation
- *
- * Author: Michael S. Tsirkin <mst@redhat.com>
- *         Stefan Hajnoczi <stefanha@redhat.com>
- *
- * Inspiration, some code, and most witty comments come from
- * Documentation/virtual/lguest/lguest.c, by Rusty Russell
- *
- * This work is licensed under the terms of the GNU GPL, version 2.
- */
-
-#ifndef VRING_H
-#define VRING_H
-
-#include "qemu-common.h"
-#include "standard-headers/linux/virtio_ring.h"
-#include "hw/virtio/virtio.h"
-
-typedef struct {
-    MemoryRegion *mr;               /* memory region containing the vring */
-    struct vring vr;                /* virtqueue vring mapped to host memory */
-    uint16_t last_avail_idx;        /* last processed avail ring index */
-    uint16_t last_used_idx;         /* last processed used ring index */
-    uint16_t signalled_used;        /* EVENT_IDX state */
-    bool signalled_used_valid;
-    bool broken;                    /* was there a fatal error? */
-} Vring;
-
-/* Fail future vring_pop() and vring_push() calls until reset */
-static inline void vring_set_broken(Vring *vring)
-{
-    vring->broken = true;
-}
-
-bool vring_setup(Vring *vring, VirtIODevice *vdev, int n);
-void vring_teardown(Vring *vring, VirtIODevice *vdev, int n);
-void vring_disable_notification(VirtIODevice *vdev, Vring *vring);
-bool vring_enable_notification(VirtIODevice *vdev, Vring *vring);
-bool vring_should_notify(VirtIODevice *vdev, Vring *vring);
-int vring_pop(VirtIODevice *vdev, Vring *vring, VirtQueueElement *elem);
-void vring_push(VirtIODevice *vdev, Vring *vring, VirtQueueElement *elem,
-                int len);
-
-#endif /* VRING_H */
diff --git a/qemu/include/hw/virtio/vhost-backend.h b/qemu/include/hw/virtio/vhost-backend.h
index e472f2971..95fcc9667 100644
--- a/qemu/include/hw/virtio/vhost-backend.h
+++ b/qemu/include/hw/virtio/vhost-backend.h
@@ -11,6 +11,7 @@
 #ifndef VHOST_BACKEND_H_
 #define VHOST_BACKEND_H_
 
+
 typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
@@ -19,17 +20,86 @@ typedef enum VhostBackendType {
 } VhostBackendType;
 
 struct vhost_dev;
+struct vhost_log;
+struct vhost_memory;
+struct vhost_vring_file;
+struct vhost_vring_state;
+struct vhost_vring_addr;
+struct vhost_scsi_target;
 
-typedef int (*vhost_call)(struct vhost_dev *dev, unsigned long int request,
-             void *arg);
 typedef int (*vhost_backend_init)(struct vhost_dev *dev, void *opaque);
 typedef int (*vhost_backend_cleanup)(struct vhost_dev *dev);
+typedef int (*vhost_backend_memslots_limit)(struct vhost_dev *dev);
+
+typedef int (*vhost_net_set_backend_op)(struct vhost_dev *dev,
+                                struct vhost_vring_file *file);
+typedef int (*vhost_scsi_set_endpoint_op)(struct vhost_dev *dev,
+                                  struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_clear_endpoint_op)(struct vhost_dev *dev,
+                                    struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_get_abi_version_op)(struct vhost_dev *dev,
+                                             int *version);
+typedef int (*vhost_set_log_base_op)(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log);
+typedef int (*vhost_set_mem_table_op)(struct vhost_dev *dev,
+                                      struct vhost_memory *mem);
+typedef int (*vhost_set_vring_addr_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr);
+typedef int (*vhost_set_vring_endian_op)(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_num_op)(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_get_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_kick_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_vring_call_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_features_op)(struct vhost_dev *dev,
+                                     uint64_t features);
+typedef int (*vhost_get_features_op)(struct vhost_dev *dev,
+                                     uint64_t *features);
+typedef int (*vhost_set_owner_op)(struct vhost_dev *dev);
+typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
+typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
+typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
+                                         int enable);
+typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
+typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
+                                       char *mac_addr);
+typedef bool (*vhost_backend_can_merge_op)(struct vhost_dev *dev,
+                                           uint64_t start1, uint64_t size1,
+                                           uint64_t start2, uint64_t size2);
 
 typedef struct VhostOps {
     VhostBackendType backend_type;
-    vhost_call vhost_call;
     vhost_backend_init vhost_backend_init;
     vhost_backend_cleanup vhost_backend_cleanup;
+    vhost_backend_memslots_limit vhost_backend_memslots_limit;
+    vhost_net_set_backend_op vhost_net_set_backend;
+    vhost_scsi_set_endpoint_op vhost_scsi_set_endpoint;
+    vhost_scsi_clear_endpoint_op vhost_scsi_clear_endpoint;
+    vhost_scsi_get_abi_version_op vhost_scsi_get_abi_version;
+    vhost_set_log_base_op vhost_set_log_base;
+    vhost_set_mem_table_op vhost_set_mem_table;
+    vhost_set_vring_addr_op vhost_set_vring_addr;
+    vhost_set_vring_endian_op vhost_set_vring_endian;
+    vhost_set_vring_num_op vhost_set_vring_num;
+    vhost_set_vring_base_op vhost_set_vring_base;
+    vhost_get_vring_base_op vhost_get_vring_base;
+    vhost_set_vring_kick_op vhost_set_vring_kick;
+    vhost_set_vring_call_op vhost_set_vring_call;
+    vhost_set_features_op vhost_set_features;
+    vhost_get_features_op vhost_get_features;
+    vhost_set_owner_op vhost_set_owner;
+    vhost_reset_device_op vhost_reset_device;
+    vhost_get_vq_index_op vhost_get_vq_index;
+    vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_requires_shm_log_op vhost_requires_shm_log;
+    vhost_migration_done_op vhost_migration_done;
+    vhost_backend_can_merge_op vhost_backend_can_merge;
 } VhostOps;
 
 extern const VhostOps user_ops;
diff --git a/qemu/include/hw/virtio/vhost-scsi.h b/qemu/include/hw/virtio/vhost-scsi.h
index 701bfee61..9fd63df12 100644
--- a/qemu/include/hw/virtio/vhost-scsi.h
+++ b/qemu/include/hw/virtio/vhost-scsi.h
@@ -19,37 +19,12 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/vhost.h"
 
-/*
- * Used by QEMU userspace to ensure a consistent vhost-scsi ABI.
- *
- * ABI Rev 0: July 2012 version starting point for v3.6-rc merge candidate +
- *            RFC-v2 vhost-scsi userspace.  Add GET_ABI_VERSION ioctl usage
- * ABI Rev 1: January 2013. Ignore vhost_tpgt filed in struct vhost_scsi_target.
- * 	      All the targets under vhost_wwpn can be seen and used by guest.
- */
-
-#define VHOST_SCSI_ABI_VERSION 1
-
-/* TODO #include <linux/vhost.h> properly */
-/* For VHOST_SCSI_SET_ENDPOINT/VHOST_SCSI_CLEAR_ENDPOINT ioctl */
-struct vhost_scsi_target {
-    int abi_version;
-    char vhost_wwpn[224];
-    unsigned short vhost_tpgt;
-    unsigned short reserved;
-};
-
 enum vhost_scsi_vq_list {
     VHOST_SCSI_VQ_CONTROL = 0,
     VHOST_SCSI_VQ_EVENT = 1,
     VHOST_SCSI_VQ_NUM_FIXED = 2,
 };
 
-#define VHOST_VIRTIO 0xAF
-#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target)
-#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
-#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
-
 #define TYPE_VHOST_SCSI "vhost-scsi"
 #define VHOST_SCSI(obj) \
         OBJECT_CHECK(VHostSCSI, (obj), TYPE_VHOST_SCSI)
diff --git a/qemu/include/hw/virtio/vhost.h b/qemu/include/hw/virtio/vhost.h
index dd510509e..b60d7585b 100644
--- a/qemu/include/hw/virtio/vhost.h
+++ b/qemu/include/hw/virtio/vhost.h
@@ -31,7 +31,8 @@ typedef unsigned long vhost_log_chunk_t;
 struct vhost_log {
     unsigned long long size;
     int refcnt;
-    vhost_log_chunk_t log[0];
+    int fd;
+    vhost_log_chunk_t *log;
 };
 
 struct vhost_memory;
@@ -44,12 +45,14 @@ struct vhost_dev {
     int nvqs;
     /* the first virtqueue which would be used by this vhost dev */
     int vq_index;
-    unsigned long long features;
-    unsigned long long acked_features;
-    unsigned long long backend_features;
+    uint64_t features;
+    uint64_t acked_features;
+    uint64_t backend_features;
+    uint64_t protocol_features;
+    uint64_t max_queues;
     bool started;
     bool log_enabled;
-    unsigned long long log_size;
+    uint64_t log_size;
     Error *migration_blocker;
     bool memory_changed;
     hwaddr mem_changed_start_addr;
@@ -57,12 +60,12 @@ struct vhost_dev {
     const VhostOps *vhost_ops;
     void *opaque;
     struct vhost_log *log;
+    QLIST_ENTRY(vhost_dev) entry;
 };
 
 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
                    VhostBackendType backend_type);
 void vhost_dev_cleanup(struct vhost_dev *hdev);
-bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev);
 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
@@ -81,4 +84,5 @@ uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
                             uint64_t features);
 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
                         uint64_t features);
+bool vhost_has_free_slot(void);
 #endif
diff --git a/qemu/include/hw/virtio/virtio-9p.h b/qemu/include/hw/virtio/virtio-9p.h
deleted file mode 100644
index 65789db13..000000000
--- a/qemu/include/hw/virtio/virtio-9p.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Virtio 9p
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_VIRTIO_9P_DEVICE_H
-#define QEMU_VIRTIO_9P_DEVICE_H
-
-typedef struct V9fsConf
-{
-    /* tag name for the device */
-    char *tag;
-    char *fsdev_id;
-} V9fsConf;
-
-#endif
diff --git a/qemu/include/hw/virtio/virtio-access.h b/qemu/include/hw/virtio/virtio-access.h
index 1ec1dfdb6..8dc84f520 100644
--- a/qemu/include/hw/virtio/virtio-access.h
+++ b/qemu/include/hw/virtio/virtio-access.h
@@ -19,32 +19,19 @@
 
 static inline bool virtio_access_is_big_endian(VirtIODevice *vdev)
 {
-    if (virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
-        /* Devices conforming to VIRTIO 1.0 or later are always LE. */
-        return false;
-    }
 #if defined(TARGET_IS_BIENDIAN)
     return virtio_is_big_endian(vdev);
 #elif defined(TARGET_WORDS_BIGENDIAN)
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        /* Devices conforming to VIRTIO 1.0 or later are always LE. */
+        return false;
+    }
     return true;
 #else
     return false;
 #endif
 }
 
-static inline bool virtio_legacy_is_cross_endian(VirtIODevice *vdev)
-{
-#ifdef TARGET_IS_BIENDIAN
-#ifdef HOST_WORDS_BIGENDIAN
-    return !virtio_is_big_endian(vdev);
-#else
-    return virtio_is_big_endian(vdev);
-#endif
-#else
-    return false;
-#endif
-}
-
 static inline uint16_t virtio_lduw_phys(VirtIODevice *vdev, hwaddr pa)
 {
     if (virtio_access_is_big_endian(vdev)) {
@@ -143,15 +130,6 @@ static inline uint64_t virtio_ldq_p(VirtIODevice *vdev, const void *ptr)
     }
 }
 
-static inline bool virtio_needs_swap(VirtIODevice *vdev)
-{
-#ifdef HOST_WORDS_BIGENDIAN
-    return virtio_access_is_big_endian(vdev) ? false : true;
-#else
-    return virtio_access_is_big_endian(vdev) ? true : false;
-#endif
-}
-
 static inline uint16_t virtio_tswap16(VirtIODevice *vdev, uint16_t s)
 {
 #ifdef HOST_WORDS_BIGENDIAN
diff --git a/qemu/include/hw/virtio/virtio-balloon.h b/qemu/include/hw/virtio/virtio-balloon.h
index 09c2ce4dc..35f62ac11 100644
--- a/qemu/include/hw/virtio/virtio-balloon.h
+++ b/qemu/include/hw/virtio/virtio-balloon.h
@@ -37,7 +37,7 @@ typedef struct VirtIOBalloon {
     uint32_t num_pages;
     uint32_t actual;
     uint64_t stats[VIRTIO_BALLOON_S_NR];
-    VirtQueueElement stats_vq_elem;
+    VirtQueueElement *stats_vq_elem;
     size_t stats_vq_offset;
     QEMUTimer *stats_timer;
     int64_t stats_last_update;
diff --git a/qemu/include/hw/virtio/virtio-blk.h b/qemu/include/hw/virtio/virtio-blk.h
index 6bf5905c5..8f2b05651 100644
--- a/qemu/include/hw/virtio/virtio-blk.h
+++ b/qemu/include/hw/virtio/virtio-blk.h
@@ -37,7 +37,6 @@ struct VirtIOBlkConf
     char *serial;
     uint32_t scsi;
     uint32_t config_wce;
-    uint32_t data_plane;
     uint32_t request_merging;
 };
 
@@ -54,16 +53,15 @@ typedef struct VirtIOBlock {
     unsigned short sector_mask;
     bool original_wce;
     VMChangeStateEntry *change;
-    /* Function to push to vq and notify guest */
-    void (*complete_request)(struct VirtIOBlockReq *req, unsigned char status);
-    Notifier migration_state_notifier;
+    bool dataplane_disabled;
+    bool dataplane_started;
     struct VirtIOBlockDataPlane *dataplane;
 } VirtIOBlock;
 
 typedef struct VirtIOBlockReq {
+    VirtQueueElement elem;
     int64_t sector_num;
     VirtIOBlock *dev;
-    VirtQueueElement elem;
     struct virtio_blk_inhdr *in;
     struct virtio_blk_outhdr out;
     QEMUIOVector qiov;
@@ -81,12 +79,13 @@ typedef struct MultiReqBuffer {
     bool is_write;
 } MultiReqBuffer;
 
-VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s);
-
+void virtio_blk_init_request(VirtIOBlock *s, VirtIOBlockReq *req);
 void virtio_blk_free_request(VirtIOBlockReq *req);
 
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb);
 
 void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb);
 
+void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq);
+
 #endif
diff --git a/qemu/include/hw/virtio/virtio-bus.h b/qemu/include/hw/virtio/virtio-bus.h
index 8811415fa..3f2c1363d 100644
--- a/qemu/include/hw/virtio/virtio-bus.h
+++ b/qemu/include/hw/virtio/virtio-bus.h
@@ -44,9 +44,12 @@ typedef struct VirtioBusClass {
     void (*notify)(DeviceState *d, uint16_t vector);
     void (*save_config)(DeviceState *d, QEMUFile *f);
     void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    void (*save_extra_state)(DeviceState *d, QEMUFile *f);
     int (*load_config)(DeviceState *d, QEMUFile *f);
     int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
     int (*load_done)(DeviceState *d, QEMUFile *f);
+    int (*load_extra_state)(DeviceState *d, QEMUFile *f);
+    bool (*has_extra_state)(DeviceState *d);
     bool (*query_guest_notifiers)(DeviceState *d);
     int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assign);
     int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
@@ -57,6 +60,11 @@ typedef struct VirtioBusClass {
      */
     void (*device_plugged)(DeviceState *d, Error **errp);
     /*
+     * Re-evaluate setup after feature bits have been validated
+     * by the device backend.
+     */
+    void (*post_plugged)(DeviceState *d, Error **errp);
+    /*
      * transport independent exit function.
      * This is called by virtio-bus just before the device is unplugged.
      */
diff --git a/qemu/include/hw/virtio/virtio-gpu.h b/qemu/include/hw/virtio/virtio-gpu.h
index 889676147..13b0ab084 100644
--- a/qemu/include/hw/virtio/virtio-gpu.h
+++ b/qemu/include/hw/virtio/virtio-gpu.h
@@ -56,8 +56,19 @@ struct virtio_gpu_requested_state {
     int x, y;
 };
 
+enum virtio_gpu_conf_flags {
+    VIRTIO_GPU_FLAG_VIRGL_ENABLED = 1,
+    VIRTIO_GPU_FLAG_STATS_ENABLED,
+};
+
+#define virtio_gpu_virgl_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED))
+#define virtio_gpu_stats_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_STATS_ENABLED))
+
 struct virtio_gpu_conf {
     uint32_t max_outputs;
+    uint32_t flags;
 };
 
 struct virtio_gpu_ctrl_command {
@@ -65,6 +76,7 @@ struct virtio_gpu_ctrl_command {
     VirtQueue *vq;
     struct virtio_gpu_ctrl_hdr cmd_hdr;
     uint32_t error;
+    bool waiting;
     bool finished;
     QTAILQ_ENTRY(virtio_gpu_ctrl_command) next;
 };
@@ -83,6 +95,7 @@ typedef struct VirtIOGPU {
     DeviceState *qdev;
 
     QTAILQ_HEAD(, virtio_gpu_simple_resource) reslist;
+    QTAILQ_HEAD(, virtio_gpu_ctrl_command) cmdq;
     QTAILQ_HEAD(, virtio_gpu_ctrl_command) fenceq;
 
     struct virtio_gpu_scanout scanout[VIRTIO_GPU_MAX_SCANOUT];
@@ -92,11 +105,14 @@ typedef struct VirtIOGPU {
     int enabled_output_bitmask;
     struct virtio_gpu_config virtio_config;
 
+    bool use_virgl_renderer;
+    bool renderer_inited;
+    bool renderer_blocked;
     QEMUTimer *fence_poll;
     QEMUTimer *print_stats;
 
+    uint32_t inflight;
     struct {
-        uint32_t inflight;
         uint32_t max_inflight;
         uint32_t requests;
         uint32_t req_3d;
@@ -138,5 +154,13 @@ int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
                                   struct virtio_gpu_ctrl_command *cmd,
                                   struct iovec **iov);
 void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count);
+void virtio_gpu_process_cmdq(VirtIOGPU *g);
+
+/* virtio-gpu-3d.c */
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd);
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g);
+void virtio_gpu_virgl_reset(VirtIOGPU *g);
+int virtio_gpu_virgl_init(VirtIOGPU *g);
 
 #endif
diff --git a/qemu/include/hw/virtio/virtio-input.h b/qemu/include/hw/virtio/virtio-input.h
index af1c207ab..bddbd4b28 100644
--- a/qemu/include/hw/virtio/virtio-input.h
+++ b/qemu/include/hw/virtio/virtio-input.h
@@ -13,20 +13,6 @@ typedef struct virtio_input_absinfo virtio_input_absinfo;
 typedef struct virtio_input_config virtio_input_config;
 typedef struct virtio_input_event virtio_input_event;
 
-#if defined(HOST_WORDS_BIGENDIAN)
-# define const_le32(_x)                          \
-    (((_x & 0x000000ffU) << 24) |                \
-     ((_x & 0x0000ff00U) <<  8) |                \
-     ((_x & 0x00ff0000U) >>  8) |                \
-     ((_x & 0xff000000U) >> 24))
-# define const_le16(_x)                          \
-    (((_x & 0x00ff) << 8) |                      \
-     ((_x & 0xff00) >> 8))
-#else
-# define const_le32(_x) (_x)
-# define const_le16(_x) (_x)
-#endif
-
 /* ----------------------------------------------------------------- */
 /* qemu internals                                                    */
 
@@ -111,6 +97,9 @@ struct VirtIOInputHost {
 void virtio_input_send(VirtIOInput *vinput, virtio_input_event *event);
 void virtio_input_init_config(VirtIOInput *vinput,
                               virtio_input_config *config);
+virtio_input_config *virtio_input_find_config(VirtIOInput *vinput,
+                                              uint8_t select,
+                                              uint8_t subsel);
 void virtio_input_add_config(VirtIOInput *vinput,
                              virtio_input_config *config);
 void virtio_input_idstr_config(VirtIOInput *vinput,
diff --git a/qemu/include/hw/virtio/virtio-net.h b/qemu/include/hw/virtio/virtio-net.h
index 60b11d5c2..0cabdb682 100644
--- a/qemu/include/hw/virtio/virtio-net.h
+++ b/qemu/include/hw/virtio/virtio-net.h
@@ -47,8 +47,7 @@ typedef struct VirtIONetQueue {
     QEMUBH *tx_bh;
     int tx_waiting;
     struct {
-        VirtQueueElement elem;
-        ssize_t len;
+        VirtQueueElement *elem;
     } async_tx;
     struct VirtIONet *n;
 } VirtIONetQueue;
@@ -95,6 +94,7 @@ typedef struct VirtIONet {
     uint64_t curr_guest_offloads;
     QEMUTimer *announce_timer;
     int announce_counter;
+    bool needs_vnet_hdr_swap;
 } VirtIONet;
 
 void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
diff --git a/qemu/include/hw/virtio/virtio-scsi.h b/qemu/include/hw/virtio/virtio-scsi.h
index 088fe9f4b..ba2f5ce07 100644
--- a/qemu/include/hw/virtio/virtio-scsi.h
+++ b/qemu/include/hw/virtio/virtio-scsi.h
@@ -22,7 +22,6 @@
 #include "hw/pci/pci.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/iothread.h"
-#include "hw/virtio/dataplane/vring.h"
 
 #define TYPE_VIRTIO_SCSI_COMMON "virtio-scsi-common"
 #define VIRTIO_SCSI_COMMON(obj) \
@@ -58,13 +57,6 @@ struct VirtIOSCSIConf {
 
 struct VirtIOSCSI;
 
-typedef struct {
-    struct VirtIOSCSI *parent;
-    Vring vring;
-    EventNotifier host_notifier;
-    EventNotifier guest_notifier;
-} VirtIOSCSIVring;
-
 typedef struct VirtIOSCSICommon {
     VirtIODevice parent_obj;
     VirtIOSCSIConf conf;
@@ -76,6 +68,13 @@ typedef struct VirtIOSCSICommon {
     VirtQueue **cmd_vqs;
 } VirtIOSCSICommon;
 
+typedef struct VirtIOSCSIBlkChangeNotifier {
+    Notifier n;
+    struct VirtIOSCSI *s;
+    SCSIDevice *sd;
+    QTAILQ_ENTRY(VirtIOSCSIBlkChangeNotifier) next;
+} VirtIOSCSIBlkChangeNotifier;
+
 typedef struct VirtIOSCSI {
     VirtIOSCSICommon parent_obj;
 
@@ -86,37 +85,29 @@ typedef struct VirtIOSCSI {
     /* Fields for dataplane below */
     AioContext *ctx; /* one iothread per virtio-scsi-pci for now */
 
-    /* Vring is used instead of vq in dataplane code, because of the underlying
-     * memory layer thread safety */
-    VirtIOSCSIVring *ctrl_vring;
-    VirtIOSCSIVring *event_vring;
-    VirtIOSCSIVring **cmd_vrings;
+    QTAILQ_HEAD(, VirtIOSCSIBlkChangeNotifier) insert_notifiers;
+    QTAILQ_HEAD(, VirtIOSCSIBlkChangeNotifier) remove_notifiers;
+
     bool dataplane_started;
     bool dataplane_starting;
     bool dataplane_stopping;
-    bool dataplane_disabled;
     bool dataplane_fenced;
     Error *blocker;
-    Notifier migration_state_notifier;
     uint32_t host_features;
 } VirtIOSCSI;
 
 typedef struct VirtIOSCSIReq {
+    /* Note:
+     * - fields up to resp_iov are initialized by virtio_scsi_init_req;
+     * - fields starting at vring are zeroed by virtio_scsi_init_req.
+     * */
+    VirtQueueElement elem;
+
     VirtIOSCSI *dev;
     VirtQueue *vq;
     QEMUSGList qsgl;
     QEMUIOVector resp_iov;
 
-    /* Note:
-     * - fields before elem are initialized by virtio_scsi_init_req;
-     * - elem is uninitialized at the time of allocation.
-     * - fields after elem are zeroed by virtio_scsi_init_req.
-     * */
-
-    VirtQueueElement elem;
-    /* Set by dataplane code. */
-    VirtIOSCSIVring *vring;
-
     union {
         /* Used for two-stage request submission */
         QTAILQ_ENTRY(VirtIOSCSIReq) next;
@@ -148,10 +139,10 @@ void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
                                 HandleOutput cmd);
 
 void virtio_scsi_common_unrealize(DeviceState *dev, Error **errp);
-void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req);
-bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req);
-void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req);
-VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq, VirtIOSCSIReq *req);
 void virtio_scsi_free_req(VirtIOSCSIReq *req);
 void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
                             uint32_t event, uint32_t reason);
@@ -159,8 +150,6 @@ void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
 void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread);
 void virtio_scsi_dataplane_start(VirtIOSCSI *s);
 void virtio_scsi_dataplane_stop(VirtIOSCSI *s);
-void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req);
-VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
-                                         VirtIOSCSIVring *vring);
+void virtio_scsi_dataplane_notify(VirtIODevice *vdev, VirtIOSCSIReq *req);
 
 #endif /* _QEMU_VIRTIO_SCSI_H */
diff --git a/qemu/include/hw/virtio/virtio-serial.h b/qemu/include/hw/virtio/virtio-serial.h
index 527d0bf62..12a55a19e 100644
--- a/qemu/include/hw/virtio/virtio-serial.h
+++ b/qemu/include/hw/virtio/virtio-serial.h
@@ -122,7 +122,7 @@ struct VirtIOSerialPort {
      * element popped and continue consuming it once the backend
      * becomes writable again.
      */
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
 
     /*
      * The index and the offset into the iov buffer that was popped in
diff --git a/qemu/include/hw/virtio/virtio.h b/qemu/include/hw/virtio/virtio.h
index cccae89d8..6a37065c2 100644
--- a/qemu/include/hw/virtio/virtio.h
+++ b/qemu/include/hw/virtio/virtio.h
@@ -46,10 +46,10 @@ typedef struct VirtQueueElement
     unsigned int index;
     unsigned int out_num;
     unsigned int in_num;
-    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
-    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
-    struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
-    struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
+    hwaddr *in_addr;
+    hwaddr *out_addr;
+    struct iovec *in_sg;
+    struct iovec *out_sg;
 } VirtQueueElement;
 
 #define VIRTIO_QUEUE_MAX 1024
@@ -90,6 +90,7 @@ struct VirtIODevice
     VMChangeStateEntry *vmstate;
     char *bus_name;
     uint8_t device_endian;
+    bool use_guest_notifier_mask;
     QLIST_HEAD(, VirtQueue) *vector_queues;
 };
 
@@ -143,21 +144,26 @@ VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
 
 void virtio_del_queue(VirtIODevice *vdev, int n);
 
+void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num);
 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len);
 void virtqueue_flush(VirtQueue *vq, unsigned int count);
+void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
+                       unsigned int len);
 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len, unsigned int idx);
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write);
-int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
+void virtqueue_map(VirtQueueElement *elem);
+void *virtqueue_pop(VirtQueue *vq, size_t sz);
+void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz);
+void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem);
 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
                           unsigned int out_bytes);
 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
                                unsigned int *out_bytes,
                                unsigned max_in_bytes, unsigned max_out_bytes);
 
+bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq);
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
 
 void virtio_save(VirtIODevice *vdev, QEMUFile *f);
@@ -244,7 +250,9 @@ void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                bool set_handler);
-void virtio_queue_notify_vq(VirtQueue *vq);
+void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
+                                                void (*fn)(VirtIODevice *,
+                                                           VirtQueue *));
 void virtio_irq(VirtQueue *vq);
 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector);
 VirtQueue *virtio_vector_next_queue(VirtQueue *vq);
@@ -261,26 +269,27 @@ static inline void virtio_clear_feature(uint64_t *features, unsigned int fbit)
     *features &= ~(1ULL << fbit);
 }
 
-static inline bool __virtio_has_feature(uint64_t features, unsigned int fbit)
+static inline bool virtio_has_feature(uint64_t features, unsigned int fbit)
 {
     assert(fbit < 64);
     return !!(features & (1ULL << fbit));
 }
 
-static inline bool virtio_has_feature(VirtIODevice *vdev, unsigned int fbit)
+static inline bool virtio_vdev_has_feature(VirtIODevice *vdev,
+                                           unsigned int fbit)
 {
-    return __virtio_has_feature(vdev->guest_features, fbit);
+    return virtio_has_feature(vdev->guest_features, fbit);
 }
 
 static inline bool virtio_host_has_feature(VirtIODevice *vdev,
                                            unsigned int fbit)
 {
-    return __virtio_has_feature(vdev->host_features, fbit);
+    return virtio_has_feature(vdev->host_features, fbit);
 }
 
 static inline bool virtio_is_big_endian(VirtIODevice *vdev)
 {
-    if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
         assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
         return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
     }
diff --git a/qemu/include/hw/xen/xen.h b/qemu/include/hw/xen/xen.h
index 4356af456..6eb815aac 100644
--- a/qemu/include/hw/xen/xen.h
+++ b/qemu/include/hw/xen/xen.h
@@ -6,7 +6,6 @@
  *   must not depend on any xen headers being present in
  *   /usr/include/xen, so it can be included unconditionally.
  */
-#include <inttypes.h>
 
 #include "hw/irq.h"
 #include "qemu-common.h"
@@ -32,23 +31,19 @@ int xen_pci_slot_get_pirq(PCIDevice *pci_dev, int irq_num);
 void xen_piix3_set_irq(void *opaque, int irq_num, int level);
 void xen_piix_pci_write_config_client(uint32_t address, uint32_t val, int len);
 void xen_hvm_inject_msi(uint64_t addr, uint32_t data);
+int xen_is_pirq_msi(uint32_t msi_data);
 
 qemu_irq *xen_interrupt_controller_init(void);
 
 void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 
 #if defined(NEED_CPU_H) && !defined(CONFIG_USER_ONLY)
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory);
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory);
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
-                   struct MemoryRegion *mr);
+                   struct MemoryRegion *mr, Error **errp);
 void xen_modified_memory(ram_addr_t start, ram_addr_t length);
 #endif
 
 void xen_register_framebuffer(struct MemoryRegion *mr);
 
-#if defined(CONFIG_XEN) && CONFIG_XEN_CTRL_INTERFACE_VERSION < 400
-#  define HVM_MAX_VCPUS 32
-#endif
-
 #endif /* QEMU_HW_XEN_H */
diff --git a/qemu/include/hw/xen/xen_backend.h b/qemu/include/hw/xen/xen_backend.h
index 3b4125e39..c839eeb48 100644
--- a/qemu/include/hw/xen/xen_backend.h
+++ b/qemu/include/hw/xen/xen_backend.h
@@ -46,8 +46,8 @@ struct XenDevice {
     int                remote_port;
     int                local_port;
 
-    XenEvtchn          evtchndev;
-    XenGnttab          gnttabdev;
+    xenevtchn_handle   *evtchndev;
+    xengnttab_handle   *gnttabdev;
 
     struct XenDevOps   *ops;
     QTAILQ_ENTRY(XenDevice) next;
@@ -56,7 +56,8 @@ struct XenDevice {
 /* ------------------------------------------------------------- */
 
 /* variables */
-extern XenXC xen_xc;
+extern xc_interface *xen_xc;
+extern xenforeignmemory_handle *xen_fmem;
 extern struct xs_handle *xenstore;
 extern const char *xen_protocol;
 
diff --git a/qemu/include/hw/xen/xen_common.h b/qemu/include/hw/xen/xen_common.h
index ed5fd3e1a..bd65e6705 100644
--- a/qemu/include/hw/xen/xen_common.h
+++ b/qemu/include/hw/xen/xen_common.h
@@ -1,17 +1,19 @@
 #ifndef QEMU_HW_XEN_COMMON_H
 #define QEMU_HW_XEN_COMMON_H 1
 
-#include "config-host.h"
 
-#include <stddef.h>
-#include <inttypes.h>
+
+/*
+ * If we have new enough libxenctrl then we do not want/need these compat
+ * interfaces, despite what the user supplied cflags might say. They
+ * must be undefined before including xenctrl.h
+ */
+#undef XC_WANT_COMPAT_EVTCHN_API
+#undef XC_WANT_COMPAT_GNTTAB_API
+#undef XC_WANT_COMPAT_MAP_FOREIGN_API
 
 #include <xenctrl.h>
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 420
-#  include <xs.h>
-#else
-#  include <xenstore.h>
-#endif
+#include <xenstore.h>
 #include <xen/io/xenbus.h>
 
 #include "hw/hw.h"
@@ -21,144 +23,54 @@
 #include "trace.h"
 
 /*
- * We don't support Xen prior to 3.3.0.
+ * We don't support Xen prior to 4.2.0.
  */
 
-/* Xen before 4.0 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 400
-static inline void *xc_map_foreign_bulk(int xc_handle, uint32_t dom, int prot,
-                                        xen_pfn_t *arr, int *err,
-                                        unsigned int num)
-{
-    return xc_map_foreign_batch(xc_handle, dom, prot, arr, num);
-}
-#endif
-
-
-/* Xen before 4.1 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 410
-
-typedef int XenXC;
-typedef int XenEvtchn;
-typedef int XenGnttab;
-
-#  define XC_INTERFACE_FMT "%i"
-#  define XC_HANDLER_INITIAL_VALUE    -1
-
-static inline XenEvtchn xen_xc_evtchn_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_evtchn_open();
-}
-
-static inline XenGnttab xen_xc_gnttab_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_gnttab_open();
-}
-
-static inline XenXC xen_xc_interface_open(void *logger, void *dombuild_logger,
-                                          unsigned int open_flags)
-{
-    return xc_interface_open();
-}
-
-static inline int xc_fd(int xen_xc)
-{
-    return xen_xc;
-}
-
-
-static inline int xc_domain_populate_physmap_exact
-    (XenXC xc_handle, uint32_t domid, unsigned long nr_extents,
-     unsigned int extent_order, unsigned int mem_flags, xen_pfn_t *extent_start)
-{
-    return xc_domain_memory_populate_physmap
-        (xc_handle, domid, nr_extents, extent_order, mem_flags, extent_start);
-}
-
-static inline int xc_domain_add_to_physmap(int xc_handle, uint32_t domid,
-                                           unsigned int space, unsigned long idx,
-                                           xen_pfn_t gpfn)
-{
-    struct xen_add_to_physmap xatp = {
-        .domid = domid,
-        .space = space,
-        .idx = idx,
-        .gpfn = gpfn,
-    };
-
-    return xc_memory_op(xc_handle, XENMEM_add_to_physmap, &xatp);
-}
-
-static inline struct xs_handle *xs_open(unsigned long flags)
-{
-    return xs_daemon_open();
-}
-
-static inline void xs_close(struct xs_handle *xsh)
-{
-    if (xsh != NULL) {
-        xs_daemon_close(xsh);
-    }
-}
+/* Xen 4.2 thru 4.6 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 471
 
+typedef xc_interface xenforeignmemory_handle;
+typedef xc_evtchn xenevtchn_handle;
+typedef xc_gnttab xengnttab_handle;
 
-/* Xen 4.1 */
-#else
+#define xenevtchn_open(l, f) xc_evtchn_open(l, f);
+#define xenevtchn_close(h) xc_evtchn_close(h)
+#define xenevtchn_fd(h) xc_evtchn_fd(h)
+#define xenevtchn_pending(h) xc_evtchn_pending(h)
+#define xenevtchn_notify(h, p) xc_evtchn_notify(h, p)
+#define xenevtchn_bind_interdomain(h, d, p) xc_evtchn_bind_interdomain(h, d, p)
+#define xenevtchn_unmask(h, p) xc_evtchn_unmask(h, p)
+#define xenevtchn_unbind(h, p) xc_evtchn_unbind(h, p)
 
-typedef xc_interface *XenXC;
-typedef xc_evtchn *XenEvtchn;
-typedef xc_gnttab *XenGnttab;
+#define xengnttab_open(l, f) xc_gnttab_open(l, f)
+#define xengnttab_close(h) xc_gnttab_close(h)
+#define xengnttab_set_max_grants(h, n) xc_gnttab_set_max_grants(h, n)
+#define xengnttab_map_grant_ref(h, d, r, p) xc_gnttab_map_grant_ref(h, d, r, p)
+#define xengnttab_unmap(h, a, n) xc_gnttab_munmap(h, a, n)
+#define xengnttab_map_grant_refs(h, c, d, r, p) \
+    xc_gnttab_map_grant_refs(h, c, d, r, p)
 
-#  define XC_INTERFACE_FMT "%p"
-#  define XC_HANDLER_INITIAL_VALUE    NULL
+#define xenforeignmemory_open(l, f) xen_xc
 
-static inline XenEvtchn xen_xc_evtchn_open(void *logger,
-                                           unsigned int open_flags)
+static inline void *xenforeignmemory_map(xc_interface *h, uint32_t dom,
+                                         int prot, size_t pages,
+                                         const xen_pfn_t arr[/*pages*/],
+                                         int err[/*pages*/])
 {
-    return xc_evtchn_open(logger, open_flags);
+    if (err)
+        return xc_map_foreign_bulk(h, dom, prot, arr, err, pages);
+    else
+        return xc_map_foreign_pages(h, dom, prot, arr, pages);
 }
 
-static inline XenGnttab xen_xc_gnttab_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_gnttab_open(logger, open_flags);
-}
+#define xenforeignmemory_unmap(h, p, s) munmap(p, s * XC_PAGE_SIZE)
 
-static inline XenXC xen_xc_interface_open(void *logger, void *dombuild_logger,
-                                          unsigned int open_flags)
-{
-    return xc_interface_open(logger, dombuild_logger, open_flags);
-}
+#else /* CONFIG_XEN_CTRL_INTERFACE_VERSION >= 471 */
 
-/* FIXME There is now way to have the xen fd */
-static inline int xc_fd(xc_interface *xen_xc)
-{
-    return -1;
-}
-#endif
+#include <xenevtchn.h>
+#include <xengnttab.h>
+#include <xenforeignmemory.h>
 
-/* Xen before 4.2 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 420
-static inline int xen_xc_hvm_inject_msi(XenXC xen_xc, domid_t dom,
-        uint64_t addr, uint32_t data)
-{
-    return -ENOSYS;
-}
-/* The followings are only to compile op_discard related code on older
- * Xen releases. */
-#define BLKIF_OP_DISCARD 5
-struct blkif_request_discard {
-    uint64_t nr_sectors;
-    uint64_t sector_number;
-};
-#else
-static inline int xen_xc_hvm_inject_msi(XenXC xen_xc, domid_t dom,
-        uint64_t addr, uint32_t data)
-{
-    return xc_hvm_inject_msi(xen_xc, dom, addr, data);
-}
 #endif
 
 void destroy_hvm_domain(bool reboot);
@@ -167,7 +79,7 @@ void destroy_hvm_domain(bool reboot);
 void xen_shutdown_fatal_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 
 #ifdef HVM_PARAM_VMPORT_REGS_PFN
-static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
+static inline int xen_get_vmport_regs_pfn(xc_interface *xc, domid_t dom,
                                           xen_pfn_t *vmport_regs_pfn)
 {
     int rc;
@@ -179,13 +91,22 @@ static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
     return rc;
 }
 #else
-static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
+static inline int xen_get_vmport_regs_pfn(xc_interface *xc, domid_t dom,
                                           xen_pfn_t *vmport_regs_pfn)
 {
     return -ENOSYS;
 }
 #endif
 
+/* Xen before 4.6 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+
+#ifndef HVM_IOREQSRV_BUFIOREQ_ATOMIC
+#define HVM_IOREQSRV_BUFIOREQ_ATOMIC 2
+#endif
+
+#endif
+
 /* Xen before 4.5 */
 #if CONFIG_XEN_CTRL_INTERFACE_VERSION < 450
 
@@ -197,54 +118,54 @@ static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
 
 typedef uint16_t ioservid_t;
 
-static inline void xen_map_memory_section(XenXC xc, domid_t dom,
+static inline void xen_map_memory_section(xc_interface *xc, domid_t dom,
                                           ioservid_t ioservid,
                                           MemoryRegionSection *section)
 {
 }
 
-static inline void xen_unmap_memory_section(XenXC xc, domid_t dom,
+static inline void xen_unmap_memory_section(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid,
                                             MemoryRegionSection *section)
 {
 }
 
-static inline void xen_map_io_section(XenXC xc, domid_t dom,
+static inline void xen_map_io_section(xc_interface *xc, domid_t dom,
                                       ioservid_t ioservid,
                                       MemoryRegionSection *section)
 {
 }
 
-static inline void xen_unmap_io_section(XenXC xc, domid_t dom,
+static inline void xen_unmap_io_section(xc_interface *xc, domid_t dom,
                                         ioservid_t ioservid,
                                         MemoryRegionSection *section)
 {
 }
 
-static inline void xen_map_pcidev(XenXC xc, domid_t dom,
+static inline void xen_map_pcidev(xc_interface *xc, domid_t dom,
                                   ioservid_t ioservid,
                                   PCIDevice *pci_dev)
 {
 }
 
-static inline void xen_unmap_pcidev(XenXC xc, domid_t dom,
+static inline void xen_unmap_pcidev(xc_interface *xc, domid_t dom,
                                     ioservid_t ioservid,
                                     PCIDevice *pci_dev)
 {
 }
 
-static inline int xen_create_ioreq_server(XenXC xc, domid_t dom,
+static inline int xen_create_ioreq_server(xc_interface *xc, domid_t dom,
                                           ioservid_t *ioservid)
 {
     return 0;
 }
 
-static inline void xen_destroy_ioreq_server(XenXC xc, domid_t dom,
+static inline void xen_destroy_ioreq_server(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid)
 {
 }
 
-static inline int xen_get_ioreq_server_info(XenXC xc, domid_t dom,
+static inline int xen_get_ioreq_server_info(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid,
                                             xen_pfn_t *ioreq_pfn,
                                             xen_pfn_t *bufioreq_pfn,
@@ -281,7 +202,7 @@ static inline int xen_get_ioreq_server_info(XenXC xc, domid_t dom,
     return 0;
 }
 
-static inline int xen_set_ioreq_server_state(XenXC xc, domid_t dom,
+static inline int xen_set_ioreq_server_state(xc_interface *xc, domid_t dom,
                                              ioservid_t ioservid,
                                              bool enable)
 {
@@ -291,7 +212,7 @@ static inline int xen_set_ioreq_server_state(XenXC xc, domid_t dom,
 /* Xen 4.5 */
 #else
 
-static inline void xen_map_memory_section(XenXC xc, domid_t dom,
+static inline void xen_map_memory_section(xc_interface *xc, domid_t dom,
                                           ioservid_t ioservid,
                                           MemoryRegionSection *section)
 {
@@ -304,7 +225,7 @@ static inline void xen_map_memory_section(XenXC xc, domid_t dom,
                                         start_addr, end_addr);
 }
 
-static inline void xen_unmap_memory_section(XenXC xc, domid_t dom,
+static inline void xen_unmap_memory_section(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid,
                                             MemoryRegionSection *section)
 {
@@ -317,7 +238,7 @@ static inline void xen_unmap_memory_section(XenXC xc, domid_t dom,
                                             start_addr, end_addr);
 }
 
-static inline void xen_map_io_section(XenXC xc, domid_t dom,
+static inline void xen_map_io_section(xc_interface *xc, domid_t dom,
                                       ioservid_t ioservid,
                                       MemoryRegionSection *section)
 {
@@ -330,7 +251,7 @@ static inline void xen_map_io_section(XenXC xc, domid_t dom,
                                         start_addr, end_addr);
 }
 
-static inline void xen_unmap_io_section(XenXC xc, domid_t dom,
+static inline void xen_unmap_io_section(xc_interface *xc, domid_t dom,
                                         ioservid_t ioservid,
                                         MemoryRegionSection *section)
 {
@@ -343,7 +264,7 @@ static inline void xen_unmap_io_section(XenXC xc, domid_t dom,
                                             start_addr, end_addr);
 }
 
-static inline void xen_map_pcidev(XenXC xc, domid_t dom,
+static inline void xen_map_pcidev(xc_interface *xc, domid_t dom,
                                   ioservid_t ioservid,
                                   PCIDevice *pci_dev)
 {
@@ -355,7 +276,7 @@ static inline void xen_map_pcidev(XenXC xc, domid_t dom,
                                       PCI_FUNC(pci_dev->devfn));
 }
 
-static inline void xen_unmap_pcidev(XenXC xc, domid_t dom,
+static inline void xen_unmap_pcidev(xc_interface *xc, domid_t dom,
                                     ioservid_t ioservid,
                                     PCIDevice *pci_dev)
 {
@@ -367,10 +288,11 @@ static inline void xen_unmap_pcidev(XenXC xc, domid_t dom,
                                           PCI_FUNC(pci_dev->devfn));
 }
 
-static inline int xen_create_ioreq_server(XenXC xc, domid_t dom,
+static inline int xen_create_ioreq_server(xc_interface *xc, domid_t dom,
                                           ioservid_t *ioservid)
 {
-    int rc = xc_hvm_create_ioreq_server(xc, dom, 1, ioservid);
+    int rc = xc_hvm_create_ioreq_server(xc, dom, HVM_IOREQSRV_BUFIOREQ_ATOMIC,
+                                        ioservid);
 
     if (rc == 0) {
         trace_xen_ioreq_server_create(*ioservid);
@@ -379,14 +301,14 @@ static inline int xen_create_ioreq_server(XenXC xc, domid_t dom,
     return rc;
 }
 
-static inline void xen_destroy_ioreq_server(XenXC xc, domid_t dom,
+static inline void xen_destroy_ioreq_server(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid)
 {
     trace_xen_ioreq_server_destroy(ioservid);
     xc_hvm_destroy_ioreq_server(xc, dom, ioservid);
 }
 
-static inline int xen_get_ioreq_server_info(XenXC xc, domid_t dom,
+static inline int xen_get_ioreq_server_info(xc_interface *xc, domid_t dom,
                                             ioservid_t ioservid,
                                             xen_pfn_t *ioreq_pfn,
                                             xen_pfn_t *bufioreq_pfn,
@@ -397,7 +319,7 @@ static inline int xen_get_ioreq_server_info(XenXC xc, domid_t dom,
                                         bufioreq_evtchn);
 }
 
-static inline int xen_set_ioreq_server_state(XenXC xc, domid_t dom,
+static inline int xen_set_ioreq_server_state(xc_interface *xc, domid_t dom,
                                              ioservid_t ioservid,
                                              bool enable)
 {
@@ -407,4 +329,44 @@ static inline int xen_set_ioreq_server_state(XenXC xc, domid_t dom,
 
 #endif
 
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+static inline int xen_xc_domain_add_to_physmap(xc_interface *xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    return xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+}
+#else
+static inline int xen_xc_domain_add_to_physmap(xc_interface *xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    /* In Xen 4.6 rc is -1 and errno contains the error value. */
+    int rc = xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+    if (rc == -1)
+        return errno;
+    return rc;
+}
+#endif
+
+#ifdef CONFIG_XEN_PV_DOMAIN_BUILD
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 470
+static inline int xen_domain_create(xc_interface *xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid);
+}
+#else
+static inline int xen_domain_create(xc_interface *xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid, NULL);
+}
+#endif
+#endif
+
 #endif /* QEMU_HW_XEN_COMMON_H */
diff --git a/qemu/include/io/channel-buffer.h b/qemu/include/io/channel-buffer.h
new file mode 100644
index 000000000..65c498b2c
--- /dev/null
+++ b/qemu/include/io/channel-buffer.h
@@ -0,0 +1,60 @@
+/*
+ * QEMU I/O channels memory buffer driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_BUFFER_H__
+#define QIO_CHANNEL_BUFFER_H__
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_BUFFER "qio-channel-buffer"
+#define QIO_CHANNEL_BUFFER(obj)                                     \
+    OBJECT_CHECK(QIOChannelBuffer, (obj), TYPE_QIO_CHANNEL_BUFFER)
+
+typedef struct QIOChannelBuffer QIOChannelBuffer;
+
+/**
+ * QIOChannelBuffer:
+ *
+ * The QIOChannelBuffer object provides a channel implementation
+ * that is able to perform I/O to/from a memory buffer.
+ *
+ */
+
+struct QIOChannelBuffer {
+    QIOChannel parent;
+    size_t capacity; /* Total allocated memory */
+    size_t usage;    /* Current size of data */
+    size_t offset;   /* Offset for future I/O ops */
+    uint8_t *data;
+};
+
+
+/**
+ * qio_channel_buffer_new:
+ * @capacity: the initial buffer capacity to allocate
+ *
+ * Allocate a new buffer which is initially empty
+ *
+ * Returns: the new channel object
+ */
+QIOChannelBuffer *
+qio_channel_buffer_new(size_t capacity);
+
+#endif /* QIO_CHANNEL_BUFFER_H__ */
diff --git a/qemu/include/io/channel-command.h b/qemu/include/io/channel-command.h
new file mode 100644
index 000000000..cfc177e78
--- /dev/null
+++ b/qemu/include/io/channel-command.h
@@ -0,0 +1,91 @@
+/*
+ * QEMU I/O channels external command driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_COMMAND_H__
+#define QIO_CHANNEL_COMMAND_H__
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_COMMAND "qio-channel-command"
+#define QIO_CHANNEL_COMMAND(obj)                                     \
+    OBJECT_CHECK(QIOChannelCommand, (obj), TYPE_QIO_CHANNEL_COMMAND)
+
+typedef struct QIOChannelCommand QIOChannelCommand;
+
+
+/**
+ * QIOChannelCommand:
+ *
+ * The QIOChannelCommand class provides a channel implementation
+ * that can transport data with an externally running command
+ * via its stdio streams.
+ */
+
+struct QIOChannelCommand {
+    QIOChannel parent;
+    int writefd;
+    int readfd;
+    pid_t pid;
+};
+
+
+/**
+ * qio_channel_command_new_pid:
+ * @writefd: the FD connected to the command's stdin
+ * @readfd: the FD connected to the command's stdout
+ * @pid: the PID of the running child command
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O with the
+ * previously spawned command identified by @pid.
+ * The two file descriptors provide the connection
+ * to command's stdio streams, either one or which
+ * may be -1 to indicate that stream is not open.
+ *
+ * The channel will take ownership of the process
+ * @pid and will kill it when closing the channel.
+ * Similarly it will take responsibility for
+ * closing the file descriptors @writefd and @readfd.
+ *
+ * Returns: the command channel object, or NULL on error
+ */
+QIOChannelCommand *
+qio_channel_command_new_pid(int writefd,
+                            int readfd,
+                            pid_t pid);
+
+/**
+ * qio_channel_command_new_spawn:
+ * @argv: the NULL terminated list of command arguments
+ * @flags: the I/O mode, one of O_RDONLY, O_WRONLY, O_RDWR
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O with the
+ * command to be spawned with arguments @argv.
+ *
+ * Returns: the command channel object, or NULL on error
+ */
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp);
+
+
+#endif /* QIO_CHANNEL_COMMAND_H__ */
diff --git a/qemu/include/io/channel-file.h b/qemu/include/io/channel-file.h
new file mode 100644
index 000000000..308e6d44d
--- /dev/null
+++ b/qemu/include/io/channel-file.h
@@ -0,0 +1,93 @@
+/*
+ * QEMU I/O channels files driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_FILE_H__
+#define QIO_CHANNEL_FILE_H__
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_FILE "qio-channel-file"
+#define QIO_CHANNEL_FILE(obj)                                     \
+    OBJECT_CHECK(QIOChannelFile, (obj), TYPE_QIO_CHANNEL_FILE)
+
+typedef struct QIOChannelFile QIOChannelFile;
+
+/**
+ * QIOChannelFile:
+ *
+ * The QIOChannelFile object provides a channel implementation
+ * that is able to perform I/O on block devices, character
+ * devices, FIFOs, pipes and plain files. While it is technically
+ * able to work on sockets too on the UNIX platform, this is not
+ * portable to Windows and lacks some extra sockets specific
+ * functionality. So the QIOChannelSocket object is recommended
+ * for that use case.
+ *
+ */
+
+struct QIOChannelFile {
+    QIOChannel parent;
+    int fd;
+};
+
+
+/**
+ * qio_channel_file_new_fd:
+ * @fd: the file descriptor
+ *
+ * Create a new IO channel object for a file represented
+ * by the @fd parameter. @fd can be associated with a
+ * block device, character device, fifo, pipe, or a
+ * regular file. For sockets, the QIOChannelSocket class
+ * should be used instead, as this provides greater
+ * functionality and cross platform portability.
+ *
+ * The channel will own the passed in file descriptor
+ * and will take responsibility for closing it, so the
+ * caller must not close it. If appropriate the caller
+ * should dup() its FD before opening the channel.
+ *
+ * Returns: the new channel object
+ */
+QIOChannelFile *
+qio_channel_file_new_fd(int fd);
+
+/**
+ * qio_channel_file_new_path:
+ * @fd: the file descriptor
+ * @flags: the open flags (O_RDONLY|O_WRONLY|O_RDWR, etc)
+ * @mode: the file creation mode if O_WRONLY is set in @flags
+ * @errp: pointer to initialized error object
+ *
+ * Create a new IO channel object for a file represented
+ * by the @path parameter. @path can point to any
+ * type of file on which sequential I/O can be
+ * performed, whether it be a plain file, character
+ * device or block device.
+ *
+ * Returns: the new channel object
+ */
+QIOChannelFile *
+qio_channel_file_new_path(const char *path,
+                          int flags,
+                          mode_t mode,
+                          Error **errp);
+
+#endif /* QIO_CHANNEL_FILE_H__ */
diff --git a/qemu/include/io/channel-socket.h b/qemu/include/io/channel-socket.h
new file mode 100644
index 000000000..70d06b40d
--- /dev/null
+++ b/qemu/include/io/channel-socket.h
@@ -0,0 +1,251 @@
+/*
+ * QEMU I/O channels sockets driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_SOCKET_H__
+#define QIO_CHANNEL_SOCKET_H__
+
+#include "io/channel.h"
+#include "io/task.h"
+#include "qemu/sockets.h"
+
+#define TYPE_QIO_CHANNEL_SOCKET "qio-channel-socket"
+#define QIO_CHANNEL_SOCKET(obj)                                     \
+    OBJECT_CHECK(QIOChannelSocket, (obj), TYPE_QIO_CHANNEL_SOCKET)
+
+typedef struct QIOChannelSocket QIOChannelSocket;
+
+/**
+ * QIOChannelSocket:
+ *
+ * The QIOChannelSocket class provides a channel implementation
+ * that can transport data over a UNIX socket or TCP socket.
+ * Beyond the core channel API, it also provides functionality
+ * for accepting client connections, tuning some socket
+ * parameters and getting socket address strings.
+ */
+
+struct QIOChannelSocket {
+    QIOChannel parent;
+    int fd;
+    struct sockaddr_storage localAddr;
+    socklen_t localAddrLen;
+    struct sockaddr_storage remoteAddr;
+    socklen_t remoteAddrLen;
+};
+
+
+/**
+ * qio_channel_socket_new:
+ *
+ * Create a channel for performing I/O on a socket
+ * connection, that is initially closed. After
+ * creating the socket, it must be setup as a client
+ * connection or server.
+ *
+ * Returns: the socket channel object
+ */
+QIOChannelSocket *
+qio_channel_socket_new(void);
+
+/**
+ * qio_channel_socket_new_fd:
+ * @fd: the socket file descriptor
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O on the socket
+ * connection represented by the file descriptor @fd.
+ *
+ * Returns: the socket channel object, or NULL on error
+ */
+QIOChannelSocket *
+qio_channel_socket_new_fd(int fd,
+                          Error **errp);
+
+
+/**
+ * qio_channel_socket_connect_sync:
+ * @ioc: the socket channel object
+ * @addr: the address to connect to
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to connect to the address @addr. This method
+ * will run in the foreground so the caller will not regain
+ * execution control until the connection is established or
+ * an error occurs.
+ */
+int qio_channel_socket_connect_sync(QIOChannelSocket *ioc,
+                                    SocketAddress *addr,
+                                    Error **errp);
+
+/**
+ * qio_channel_socket_connect_async:
+ * @ioc: the socket channel object
+ * @addr: the address to connect to
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to connect to the address @addr. This method
+ * will run in the background so the caller will regain
+ * execution control immediately. The function @callback
+ * will be invoked on completion or failure. The @addr
+ * parameter will be copied, so may be freed as soon
+ * as this function returns without waiting for completion.
+ */
+void qio_channel_socket_connect_async(QIOChannelSocket *ioc,
+                                      SocketAddress *addr,
+                                      QIOTaskFunc callback,
+                                      gpointer opaque,
+                                      GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_listen_sync:
+ * @ioc: the socket channel object
+ * @addr: the address to listen to
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to listen to the address @addr. This method
+ * will run in the foreground so the caller will not regain
+ * execution control until the connection is established or
+ * an error occurs.
+ */
+int qio_channel_socket_listen_sync(QIOChannelSocket *ioc,
+                                   SocketAddress *addr,
+                                   Error **errp);
+
+/**
+ * qio_channel_socket_listen_async:
+ * @ioc: the socket channel object
+ * @addr: the address to listen to
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to listen to the address @addr. This method
+ * will run in the background so the caller will regain
+ * execution control immediately. The function @callback
+ * will be invoked on completion or failure. The @addr
+ * parameter will be copied, so may be freed as soon
+ * as this function returns without waiting for completion.
+ */
+void qio_channel_socket_listen_async(QIOChannelSocket *ioc,
+                                     SocketAddress *addr,
+                                     QIOTaskFunc callback,
+                                     gpointer opaque,
+                                     GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_dgram_sync:
+ * @ioc: the socket channel object
+ * @localAddr: the address to local bind address
+ * @remoteAddr: the address to remote peer address
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to initialize a datagram socket bound to
+ * @localAddr and communicating with peer @remoteAddr.
+ * This method will run in the foreground so the caller
+ * will not regain execution control until the socket
+ * is established or an error occurs.
+ */
+int qio_channel_socket_dgram_sync(QIOChannelSocket *ioc,
+                                  SocketAddress *localAddr,
+                                  SocketAddress *remoteAddr,
+                                  Error **errp);
+
+/**
+ * qio_channel_socket_dgram_async:
+ * @ioc: the socket channel object
+ * @localAddr: the address to local bind address
+ * @remoteAddr: the address to remote peer address
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to initialize a datagram socket bound to
+ * @localAddr and communicating with peer @remoteAddr.
+ * This method will run in the background so the caller
+ * will regain execution control immediately. The function
+ * @callback will be invoked on completion or failure.
+ * The @localAddr and @remoteAddr parameters will be copied,
+ * so may be freed as soon as this function returns without
+ * waiting for completion.
+ */
+void qio_channel_socket_dgram_async(QIOChannelSocket *ioc,
+                                    SocketAddress *localAddr,
+                                    SocketAddress *remoteAddr,
+                                    QIOTaskFunc callback,
+                                    gpointer opaque,
+                                    GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_get_local_address:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+SocketAddress *
+qio_channel_socket_get_local_address(QIOChannelSocket *ioc,
+                                     Error **errp);
+
+/**
+ * qio_channel_socket_get_remote_address:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *
+qio_channel_socket_get_remote_address(QIOChannelSocket *ioc,
+                                      Error **errp);
+
+
+/**
+ * qio_channel_socket_accept:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * If the socket represents a server, then this accepts
+ * a new client connection. The returned channel will
+ * represent the connected client socket.
+ *
+ * Returns: the new client channel, or NULL on error
+ */
+QIOChannelSocket *
+qio_channel_socket_accept(QIOChannelSocket *ioc,
+                          Error **errp);
+
+
+#endif /* QIO_CHANNEL_SOCKET_H__ */
diff --git a/qemu/include/io/channel-tls.h b/qemu/include/io/channel-tls.h
new file mode 100644
index 000000000..322eccbaa
--- /dev/null
+++ b/qemu/include/io/channel-tls.h
@@ -0,0 +1,142 @@
+/*
+ * QEMU I/O channels TLS driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_TLS_H__
+#define QIO_CHANNEL_TLS_H__
+
+#include "io/channel.h"
+#include "io/task.h"
+#include "crypto/tlssession.h"
+
+#define TYPE_QIO_CHANNEL_TLS "qio-channel-tls"
+#define QIO_CHANNEL_TLS(obj)                                     \
+    OBJECT_CHECK(QIOChannelTLS, (obj), TYPE_QIO_CHANNEL_TLS)
+
+typedef struct QIOChannelTLS QIOChannelTLS;
+
+/**
+ * QIOChannelTLS
+ *
+ * The QIOChannelTLS class provides a channel wrapper which
+ * can transparently run the TLS encryption protocol. It is
+ * usually used over a TCP socket, but there is actually no
+ * technical restriction on which type of master channel is
+ * used as the transport.
+ *
+ * This channel object is capable of running as either a
+ * TLS server or TLS client.
+ */
+
+struct QIOChannelTLS {
+    QIOChannel parent;
+    QIOChannel *master;
+    QCryptoTLSSession *session;
+};
+
+/**
+ * qio_channel_tls_new_server:
+ * @master: the underlying channel object
+ * @creds: the credentials to use for TLS handshake
+ * @aclname: the access control list for validating clients
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS channel that runs the server side of
+ * a TLS session. The TLS session handshake will use the
+ * credentials provided in @creds. If the @aclname parameter
+ * is non-NULL, then the client will have to provide
+ * credentials (ie a x509 client certificate) which will
+ * then be validated against the ACL.
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_tls_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new TLS channel object and not the original
+ * master channel
+ *
+ * Returns: the new TLS channel object, or NULL
+ */
+QIOChannelTLS *
+qio_channel_tls_new_server(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *aclname,
+                           Error **errp);
+
+/**
+ * qio_channel_tls_new_client:
+ * @master: the underlying channel object
+ * @creds: the credentials to use for TLS handshake
+ * @hostname: the user specified server hostname
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS channel that runs the client side of
+ * a TLS session. The TLS session handshake will use the
+ * credentials provided in @creds. The @hostname parameter
+ * should provide the user specified hostname of the server
+ * and will be validated against the server's credentials
+ * (ie CommonName of the x509 certificate)
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_tls_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new TLS channel object and not the original
+ * master channel
+ *
+ * Returns: the new TLS channel object, or NULL
+ */
+QIOChannelTLS *
+qio_channel_tls_new_client(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *hostname,
+                           Error **errp);
+
+/**
+ * qio_channel_tls_handshake:
+ * @ioc: the TLS channel object
+ * @func: the callback to invoke when completed
+ * @opaque: opaque data to pass to @func
+ * @destroy: optional callback to free @opaque
+ *
+ * Perform the TLS session handshake. This method
+ * will return immediately and the handshake will
+ * continue in the background, provided the main
+ * loop is running. When the handshake is complete,
+ * or fails, the @func callback will be invoked.
+ */
+void qio_channel_tls_handshake(QIOChannelTLS *ioc,
+                               QIOTaskFunc func,
+                               gpointer opaque,
+                               GDestroyNotify destroy);
+
+/**
+ * qio_channel_tls_get_session:
+ * @ioc: the TLS channel object
+ *
+ * Get the TLS session used by the channel.
+ *
+ * Returns: the TLS session
+ */
+QCryptoTLSSession *
+qio_channel_tls_get_session(QIOChannelTLS *ioc);
+
+#endif /* QIO_CHANNEL_TLS_H__ */
diff --git a/qemu/include/io/channel-util.h b/qemu/include/io/channel-util.h
new file mode 100644
index 000000000..c93af8288
--- /dev/null
+++ b/qemu/include/io/channel-util.h
@@ -0,0 +1,52 @@
+/*
+ * QEMU I/O channels utility APIs
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_UTIL_H__
+#define QIO_CHANNEL_UTIL_H__
+
+#include "io/channel.h"
+
+/*
+ * This module provides helper functions that are useful when dealing
+ * with QIOChannel objects
+ */
+
+
+/**
+ * qio_channel_new_fd:
+ * @fd: the file descriptor
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O on the file
+ * descriptor @fd. The particular subclass of QIOChannel
+ * that is returned will depend on what underlying object
+ * the file descriptor is associated with. It may be either
+ * a QIOChannelSocket or a QIOChannelFile instance. Upon
+ * success, the returned QIOChannel instance will own
+ * the @fd file descriptor, and take responsibility for
+ * closing it when no longer required. On failure, the
+ * caller is responsible for closing @fd.
+ *
+ * Returns: the channel object, or NULL on error
+ */
+QIOChannel *qio_channel_new_fd(int fd,
+                               Error **errp);
+
+#endif /* QIO_CHANNEL_UTIL_H__ */
diff --git a/qemu/include/io/channel-watch.h b/qemu/include/io/channel-watch.h
new file mode 100644
index 000000000..76d764223
--- /dev/null
+++ b/qemu/include/io/channel-watch.h
@@ -0,0 +1,90 @@
+/*
+ * QEMU I/O channels watch helper APIs
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_WATCH_H__
+#define QIO_CHANNEL_WATCH_H__
+
+#include "io/channel.h"
+
+/*
+ * This module provides helper functions that will be needed by
+ * the various QIOChannel implementations, for creating watches
+ * on file descriptors / sockets
+ */
+
+/**
+ * qio_channel_create_fd_watch:
+ * @ioc: the channel object
+ * @fd: the file descriptor
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the file descriptor @fd for the
+ * I/O conditions in @condition. This is able
+ * monitor block devices, character devices,
+ * pipes but not plain files or, on Win32, sockets.
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_fd_watch(QIOChannel *ioc,
+                                     int fd,
+                                     GIOCondition condition);
+
+/**
+ * qio_channel_create_socket_watch:
+ * @ioc: the channel object
+ * @fd: the file descriptor
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the file descriptor @fd for the
+ * I/O conditions in @condition. This is equivalent
+ * to qio_channel_create_fd_watch on POSIX systems
+ * but not on Windows.
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int fd,
+                                         GIOCondition condition);
+
+/**
+ * qio_channel_create_fd_pair_watch:
+ * @ioc: the channel object
+ * @fdread: the file descriptor for reading
+ * @fdwrite: the file descriptor for writing
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the pair of file descriptors @fdread
+ * and @fdwrite for the I/O conditions in @condition.
+ * This is intended for monitoring unidirectional
+ * file descriptors such as pipes, where a pair
+ * of descriptors is required for bidirectional
+ * I/O
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_fd_pair_watch(QIOChannel *ioc,
+                                          int fdread,
+                                          int fdwrite,
+                                          GIOCondition condition);
+
+#endif /* QIO_CHANNEL_WATCH_H__ */
diff --git a/qemu/include/io/channel-websock.h b/qemu/include/io/channel-websock.h
new file mode 100644
index 000000000..0dc21cc56
--- /dev/null
+++ b/qemu/include/io/channel-websock.h
@@ -0,0 +1,108 @@
+/*
+ * QEMU I/O channels driver websockets
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_WEBSOCK_H__
+#define QIO_CHANNEL_WEBSOCK_H__
+
+#include "io/channel.h"
+#include "qemu/buffer.h"
+#include "io/task.h"
+
+#define TYPE_QIO_CHANNEL_WEBSOCK "qio-channel-websock"
+#define QIO_CHANNEL_WEBSOCK(obj)                                     \
+    OBJECT_CHECK(QIOChannelWebsock, (obj), TYPE_QIO_CHANNEL_WEBSOCK)
+
+typedef struct QIOChannelWebsock QIOChannelWebsock;
+typedef union QIOChannelWebsockMask QIOChannelWebsockMask;
+
+union QIOChannelWebsockMask {
+    char c[4];
+    uint32_t u;
+};
+
+/**
+ * QIOChannelWebsock
+ *
+ * The QIOChannelWebsock class provides a channel wrapper which
+ * can transparently run the HTTP websockets protocol. This is
+ * usually used over a TCP socket, but there is actually no
+ * technical restriction on which type of master channel is
+ * used as the transport.
+ *
+ * This channel object is currently only capable of running as
+ * a websocket server and is a pretty crude implementation
+ * of it, not supporting the full websockets protocol feature
+ * set. It is sufficient to use with a simple websockets
+ * client for encapsulating VNC for noVNC in-browser client.
+ */
+
+struct QIOChannelWebsock {
+    QIOChannel parent;
+    QIOChannel *master;
+    Buffer encinput;
+    Buffer encoutput;
+    Buffer rawinput;
+    Buffer rawoutput;
+    size_t payload_remain;
+    QIOChannelWebsockMask mask;
+    guint io_tag;
+    Error *io_err;
+    gboolean io_eof;
+};
+
+/**
+ * qio_channel_websock_new_server:
+ * @master: the underlying channel object
+ *
+ * Create a new websockets channel that runs the server
+ * side of the protocol.
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_websock_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new websocket channel object and not the original
+ * master channel
+ *
+ * Returns: the new websockets channel object
+ */
+QIOChannelWebsock *
+qio_channel_websock_new_server(QIOChannel *master);
+
+/**
+ * qio_channel_websock_handshake:
+ * @ioc: the websocket channel object
+ * @func: the callback to invoke when completed
+ * @opaque: opaque data to pass to @func
+ * @destroy: optional callback to free @opaque
+ *
+ * Perform the websocket handshake. This method
+ * will return immediately and the handshake will
+ * continue in the background, provided the main
+ * loop is running. When the handshake is complete,
+ * or fails, the @func callback will be invoked.
+ */
+void qio_channel_websock_handshake(QIOChannelWebsock *ioc,
+                                   QIOTaskFunc func,
+                                   gpointer opaque,
+                                   GDestroyNotify destroy);
+
+#endif /* QIO_CHANNEL_WEBSOCK_H__ */
diff --git a/qemu/include/io/channel.h b/qemu/include/io/channel.h
new file mode 100644
index 000000000..d37acd29e
--- /dev/null
+++ b/qemu/include/io/channel.h
@@ -0,0 +1,504 @@
+/*
+ * QEMU I/O channels
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_H__
+#define QIO_CHANNEL_H__
+
+#include "qemu-common.h"
+#include "qom/object.h"
+
+#define TYPE_QIO_CHANNEL "qio-channel"
+#define QIO_CHANNEL(obj)                                    \
+    OBJECT_CHECK(QIOChannel, (obj), TYPE_QIO_CHANNEL)
+#define QIO_CHANNEL_CLASS(klass)                                    \
+    OBJECT_CLASS_CHECK(QIOChannelClass, klass, TYPE_QIO_CHANNEL)
+#define QIO_CHANNEL_GET_CLASS(obj)                                  \
+    OBJECT_GET_CLASS(QIOChannelClass, obj, TYPE_QIO_CHANNEL)
+
+typedef struct QIOChannel QIOChannel;
+typedef struct QIOChannelClass QIOChannelClass;
+
+#define QIO_CHANNEL_ERR_BLOCK -2
+
+typedef enum QIOChannelFeature QIOChannelFeature;
+
+enum QIOChannelFeature {
+    QIO_CHANNEL_FEATURE_FD_PASS  = (1 << 0),
+    QIO_CHANNEL_FEATURE_SHUTDOWN = (1 << 1),
+};
+
+
+typedef enum QIOChannelShutdown QIOChannelShutdown;
+
+enum QIOChannelShutdown {
+    QIO_CHANNEL_SHUTDOWN_BOTH,
+    QIO_CHANNEL_SHUTDOWN_READ,
+    QIO_CHANNEL_SHUTDOWN_WRITE,
+};
+
+typedef gboolean (*QIOChannelFunc)(QIOChannel *ioc,
+                                   GIOCondition condition,
+                                   gpointer data);
+
+/**
+ * QIOChannel:
+ *
+ * The QIOChannel defines the core API for a generic I/O channel
+ * class hierarchy. It is inspired by GIOChannel, but has the
+ * following differences
+ *
+ *  - Use QOM to properly support arbitrary subclassing
+ *  - Support use of iovecs for efficient I/O with multiple blocks
+ *  - None of the character set translation, binary data exclusively
+ *  - Direct support for QEMU Error object reporting
+ *  - File descriptor passing
+ *
+ * This base class is abstract so cannot be instantiated. There
+ * will be subclasses for dealing with sockets, files, and higher
+ * level protocols such as TLS, WebSocket, etc.
+ */
+
+struct QIOChannel {
+    Object parent;
+    unsigned int features; /* bitmask of QIOChannelFeatures */
+#ifdef _WIN32
+    HANDLE event; /* For use with GSource on Win32 */
+#endif
+};
+
+/**
+ * QIOChannelClass:
+ *
+ * This class defines the contract that all subclasses
+ * must follow to provide specific channel implementations.
+ * The first five callbacks are mandatory to support, others
+ * provide additional optional features.
+ *
+ * Consult the corresponding public API docs for a description
+ * of the semantics of each callback
+ */
+struct QIOChannelClass {
+    ObjectClass parent;
+
+    /* Mandatory callbacks */
+    ssize_t (*io_writev)(QIOChannel *ioc,
+                         const struct iovec *iov,
+                         size_t niov,
+                         int *fds,
+                         size_t nfds,
+                         Error **errp);
+    ssize_t (*io_readv)(QIOChannel *ioc,
+                        const struct iovec *iov,
+                        size_t niov,
+                        int **fds,
+                        size_t *nfds,
+                        Error **errp);
+    int (*io_close)(QIOChannel *ioc,
+                    Error **errp);
+    GSource * (*io_create_watch)(QIOChannel *ioc,
+                                 GIOCondition condition);
+    int (*io_set_blocking)(QIOChannel *ioc,
+                           bool enabled,
+                           Error **errp);
+
+    /* Optional callbacks */
+    int (*io_shutdown)(QIOChannel *ioc,
+                       QIOChannelShutdown how,
+                       Error **errp);
+    void (*io_set_cork)(QIOChannel *ioc,
+                        bool enabled);
+    void (*io_set_delay)(QIOChannel *ioc,
+                         bool enabled);
+    off_t (*io_seek)(QIOChannel *ioc,
+                     off_t offset,
+                     int whence,
+                     Error **errp);
+};
+
+/* General I/O handling functions */
+
+/**
+ * qio_channel_has_feature:
+ * @ioc: the channel object
+ * @feature: the feature to check support of
+ *
+ * Determine whether the channel implementation supports
+ * the optional feature named in @feature.
+ *
+ * Returns: true if supported, false otherwise.
+ */
+bool qio_channel_has_feature(QIOChannel *ioc,
+                             QIOChannelFeature feature);
+
+/**
+ * qio_channel_readv_full:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to read data into
+ * @niov: the length of the @iov array
+ * @fds: pointer to an array that will received file handles
+ * @nfds: pointer filled with number of elements in @fds on return
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Read data from the IO channel, storing it in the
+ * memory regions referenced by @iov. Each element
+ * in the @iov will be fully populated with data
+ * before the next one is used. The @niov parameter
+ * specifies the total number of elements in @iov.
+ *
+ * It is not required for all @iov to be filled with
+ * data. If the channel is in blocking mode, at least
+ * one byte of data will be read, but no more is
+ * guaranteed. If the channel is non-blocking and no
+ * data is available, it will return QIO_CHANNEL_ERR_BLOCK
+ *
+ * If the channel has passed any file descriptors,
+ * the @fds array pointer will be allocated and
+ * the elements filled with the received file
+ * descriptors. The @nfds pointer will be updated
+ * to indicate the size of the @fds array that
+ * was allocated. It is the callers responsibility
+ * to call close() on each file descriptor and to
+ * call g_free() on the array pointer in @fds.
+ *
+ * It is an error to pass a non-NULL @fds parameter
+ * unless qio_channel_has_feature() returns a true
+ * value for the QIO_CHANNEL_FEATURE_FD_PASS constant.
+ *
+ * Returns: the number of bytes read, or -1 on error,
+ * or QIO_CHANNEL_ERR_BLOCK if no data is available
+ * and the channel is non-blocking
+ */
+ssize_t qio_channel_readv_full(QIOChannel *ioc,
+                               const struct iovec *iov,
+                               size_t niov,
+                               int **fds,
+                               size_t *nfds,
+                               Error **errp);
+
+
+/**
+ * qio_channel_writev_full:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to write data from
+ * @niov: the length of the @iov array
+ * @fds: an array of file handles to send
+ * @nfds: number of file handles in @fds
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Write data to the IO channel, reading it from the
+ * memory regions referenced by @iov. Each element
+ * in the @iov will be fully sent, before the next
+ * one is used. The @niov parameter specifies the
+ * total number of elements in @iov.
+ *
+ * It is not required for all @iov data to be fully
+ * sent. If the channel is in blocking mode, at least
+ * one byte of data will be sent, but no more is
+ * guaranteed. If the channel is non-blocking and no
+ * data can be sent, it will return QIO_CHANNEL_ERR_BLOCK
+ *
+ * If there are file descriptors to send, the @fds
+ * array should be non-NULL and provide the handles.
+ * All file descriptors will be sent if at least one
+ * byte of data was sent.
+ *
+ * It is an error to pass a non-NULL @fds parameter
+ * unless qio_channel_has_feature() returns a true
+ * value for the QIO_CHANNEL_FEATURE_FD_PASS constant.
+ *
+ * Returns: the number of bytes sent, or -1 on error,
+ * or QIO_CHANNEL_ERR_BLOCK if no data is can be sent
+ * and the channel is non-blocking
+ */
+ssize_t qio_channel_writev_full(QIOChannel *ioc,
+                                const struct iovec *iov,
+                                size_t niov,
+                                int *fds,
+                                size_t nfds,
+                                Error **errp);
+
+/**
+ * qio_channel_readv:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to read data into
+ * @niov: the length of the @iov array
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_readv_full() but does not support
+ * receiving of file handles.
+ */
+ssize_t qio_channel_readv(QIOChannel *ioc,
+                          const struct iovec *iov,
+                          size_t niov,
+                          Error **errp);
+
+/**
+ * qio_channel_writev:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to write data from
+ * @niov: the length of the @iov array
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_writev_full() but does not support
+ * sending of file handles.
+ */
+ssize_t qio_channel_writev(QIOChannel *ioc,
+                           const struct iovec *iov,
+                           size_t niov,
+                           Error **errp);
+
+/**
+ * qio_channel_readv:
+ * @ioc: the channel object
+ * @buf: the memory region to read data into
+ * @buflen: the length of @buf
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_readv_full() but does not support
+ * receiving of file handles, and only supports reading into
+ * a single memory region.
+ */
+ssize_t qio_channel_read(QIOChannel *ioc,
+                         char *buf,
+                         size_t buflen,
+                         Error **errp);
+
+/**
+ * qio_channel_writev:
+ * @ioc: the channel object
+ * @buf: the memory regions to send data from
+ * @buflen: the length of @buf
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_writev_full() but does not support
+ * sending of file handles, and only supports writing from a
+ * single memory region.
+ */
+ssize_t qio_channel_write(QIOChannel *ioc,
+                          const char *buf,
+                          size_t buflen,
+                          Error **errp);
+
+/**
+ * qio_channel_set_blocking:
+ * @ioc: the channel object
+ * @enabled: the blocking flag state
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * If @enabled is true, then the channel is put into
+ * blocking mode, otherwise it will be non-blocking.
+ *
+ * In non-blocking mode, read/write operations may
+ * return QIO_CHANNEL_ERR_BLOCK if they would otherwise
+ * block on I/O
+ */
+int qio_channel_set_blocking(QIOChannel *ioc,
+                             bool enabled,
+                             Error **errp);
+
+/**
+ * qio_channel_close:
+ * @ioc: the channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Close the channel, flushing any pending I/O
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qio_channel_close(QIOChannel *ioc,
+                      Error **errp);
+
+/**
+ * qio_channel_shutdown:
+ * @ioc: the channel object
+ * @how: the direction to shutdown
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Shutdowns transmission and/or receiving of data
+ * without closing the underlying transport.
+ *
+ * Not all implementations will support this facility,
+ * so may report an error. To avoid errors, the
+ * caller may check for the feature flag
+ * QIO_CHANNEL_FEATURE_SHUTDOWN prior to calling
+ * this method.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qio_channel_shutdown(QIOChannel *ioc,
+                         QIOChannelShutdown how,
+                         Error **errp);
+
+/**
+ * qio_channel_set_delay:
+ * @ioc: the channel object
+ * @enabled: the new flag state
+ *
+ * Controls whether the underlying transport is
+ * permitted to delay writes in order to merge
+ * small packets. If @enabled is true, then the
+ * writes may be delayed in order to opportunistically
+ * merge small packets into larger ones. If @enabled
+ * is false, writes are dispatched immediately with
+ * no delay.
+ *
+ * When @enabled is false, applications may wish to
+ * use the qio_channel_set_cork() method to explicitly
+ * control write merging.
+ *
+ * On channels which are backed by a socket, this
+ * API corresponds to the inverse of TCP_NODELAY flag,
+ * controlling whether the Nagle algorithm is active.
+ *
+ * This setting is merely a hint, so implementations are
+ * free to ignore this without it being considered an
+ * error.
+ */
+void qio_channel_set_delay(QIOChannel *ioc,
+                           bool enabled);
+
+/**
+ * qio_channel_set_cork:
+ * @ioc: the channel object
+ * @enabled: the new flag state
+ *
+ * Controls whether the underlying transport is
+ * permitted to dispatch data that is written.
+ * If @enabled is true, then any data written will
+ * be queued in local buffers until @enabled is
+ * set to false once again.
+ *
+ * This feature is typically used when the automatic
+ * write coalescing facility is disabled via the
+ * qio_channel_set_delay() method.
+ *
+ * On channels which are backed by a socket, this
+ * API corresponds to the TCP_CORK flag.
+ *
+ * This setting is merely a hint, so implementations are
+ * free to ignore this without it being considered an
+ * error.
+ */
+void qio_channel_set_cork(QIOChannel *ioc,
+                          bool enabled);
+
+
+/**
+ * qio_channel_seek:
+ * @ioc: the channel object
+ * @offset: the position to seek to, relative to @whence
+ * @whence: one of the (POSIX) SEEK_* constants listed below
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Moves the current I/O position within the channel
+ * @ioc, to be @offset. The value of @offset is
+ * interpreted relative to @whence:
+ *
+ * SEEK_SET - the position is set to @offset bytes
+ * SEEK_CUR - the position is moved by @offset bytes
+ * SEEK_END - the position is set to end of the file plus @offset bytes
+ *
+ * Not all implementations will support this facility,
+ * so may report an error.
+ *
+ * Returns: the new position on success, (off_t)-1 on failure
+ */
+off_t qio_channel_io_seek(QIOChannel *ioc,
+                          off_t offset,
+                          int whence,
+                          Error **errp);
+
+
+/**
+ * qio_channel_create_watch:
+ * @ioc: the channel object
+ * @condition: the I/O condition to monitor
+ *
+ * Create a new main loop source that is used to watch
+ * for the I/O condition @condition. Typically the
+ * qio_channel_add_watch() method would be used instead
+ * of this, since it directly attaches a callback to
+ * the source
+ *
+ * Returns: the new main loop source.
+ */
+GSource *qio_channel_create_watch(QIOChannel *ioc,
+                                  GIOCondition condition);
+
+/**
+ * qio_channel_add_watch:
+ * @ioc: the channel object
+ * @condition: the I/O condition to monitor
+ * @func: callback to invoke when the source becomes ready
+ * @user_data: opaque data to pass to @func
+ * @notify: callback to free @user_data
+ *
+ * Create a new main loop source that is used to watch
+ * for the I/O condition @condition. The callback @func
+ * will be registered against the source, to be invoked
+ * when the source becomes ready. The optional @user_data
+ * will be passed to @func when it is invoked. The @notify
+ * callback will be used to free @user_data when the
+ * watch is deleted
+ *
+ * The returned source ID can be used with g_source_remove()
+ * to remove and free the source when no longer required.
+ * Alternatively the @func callback can return a FALSE
+ * value.
+ *
+ * Returns: the source ID
+ */
+guint qio_channel_add_watch(QIOChannel *ioc,
+                            GIOCondition condition,
+                            QIOChannelFunc func,
+                            gpointer user_data,
+                            GDestroyNotify notify);
+
+
+/**
+ * qio_channel_yield:
+ * @ioc: the channel object
+ * @condition: the I/O condition to wait for
+ *
+ * Yields execution from the current coroutine until
+ * the condition indicated by @condition becomes
+ * available.
+ *
+ * This must only be called from coroutine context
+ */
+void qio_channel_yield(QIOChannel *ioc,
+                       GIOCondition condition);
+
+/**
+ * qio_channel_wait:
+ * @ioc: the channel object
+ * @condition: the I/O condition to wait for
+ *
+ * Block execution from the current thread until
+ * the condition indicated by @condition becomes
+ * available.
+ *
+ * This will enter a nested event loop to perform
+ * the wait.
+ */
+void qio_channel_wait(QIOChannel *ioc,
+                      GIOCondition condition);
+
+#endif /* QIO_CHANNEL_H__ */
diff --git a/qemu/include/io/task.h b/qemu/include/io/task.h
new file mode 100644
index 000000000..2e69d8a47
--- /dev/null
+++ b/qemu/include/io/task.h
@@ -0,0 +1,255 @@
+/*
+ * QEMU I/O task
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_TASK_H__
+#define QIO_TASK_H__
+
+#include "qemu-common.h"
+#include "qom/object.h"
+
+typedef struct QIOTask QIOTask;
+
+typedef void (*QIOTaskFunc)(Object *source,
+                            Error *err,
+                            gpointer opaque);
+
+typedef int (*QIOTaskWorker)(QIOTask *task,
+                             Error **errp,
+                             gpointer opaque);
+
+/**
+ * QIOTask:
+ *
+ * The QIOTask object provides a simple mechanism for reporting
+ * success / failure of long running background operations.
+ *
+ * A object on which the operation is to be performed could have
+ * a public API which accepts a task callback:
+ *
+ * <example>
+ *   <title>Task callback function signature</title>
+ *   <programlisting>
+ *  void myobject_operation(QMyObject *obj,
+ *                          QIOTaskFunc *func,
+ *                          gpointer opaque,
+ *                          GDestroyNotify *notify);
+ *   </programlisting>
+ * </example>
+ *
+ * The 'func' parameter is the callback to be invoked, and 'opaque'
+ * is data to pass to it. The optional 'notify' function is used
+ * to free 'opaque' when no longer needed.
+ *
+ * Now, lets say the implementation of this method wants to set
+ * a timer to run once a second checking for completion of some
+ * activity. It would do something like
+ *
+ * <example>
+ *   <title>Task callback function implementation</title>
+ *   <programlisting>
+ *    void myobject_operation(QMyObject *obj,
+ *                            QIOTaskFunc *func,
+ *                            gpointer opaque,
+ *                            GDestroyNotify *notify)
+ *    {
+ *      QIOTask *task;
+ *
+ *      task = qio_task_new(OBJECT(obj), func, opaque, notify);
+ *
+ *      g_timeout_add_full(G_PRIORITY_DEFAULT,
+ *                         1000,
+ *                         myobject_operation_timer,
+ *                         task,
+ *                         NULL);
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ * It could equally have setup a watch on a file descriptor or
+ * created a background thread, or something else entirely.
+ * Notice that the source object is passed to the task, and
+ * QIOTask will hold a reference on that. This ensure that
+ * the QMyObject instance cannot be garbage collected while
+ * the async task is still in progress.
+ *
+ * In this case, myobject_operation_timer will fire after
+ * 3 secs and do
+ *
+ * <example>
+ *   <title>Task timer function</title>
+ *   <programlisting>
+ *   gboolean myobject_operation_timer(gpointer opaque)
+ *   {
+ *      QIOTask *task = QIO_TASK(opaque);
+ *      Error *err;*
+ *
+ *      ...check something important...
+ *       if (err) {
+ *           qio_task_abort(task, err);
+ *           error_free(task);
+ *           return FALSE;
+ *       } else if (...work is completed ...) {
+ *           qio_task_complete(task);
+ *           return FALSE;
+ *       }
+ *       ...carry on polling ...
+ *       return TRUE;
+ *   }
+ *   </programlisting>
+ * </example>
+ *
+ * Once this function returns false, object_unref will be called
+ * automatically on the task causing it to be released and the
+ * ref on QMyObject dropped too.
+ *
+ * The QIOTask module can also be used to perform operations
+ * in a background thread context, while still reporting the
+ * results in the main event thread. This allows code which
+ * cannot easily be rewritten to be asychronous (such as DNS
+ * lookups) to be easily run non-blocking. Reporting the
+ * results in the main thread context means that the caller
+ * typically does not need to be concerned about thread
+ * safety wrt the QEMU global mutex.
+ *
+ * For example, the socket_listen() method will block the caller
+ * while DNS lookups take place if given a name, instead of IP
+ * address. The C library often do not provide a practical async
+ * DNS API, so the to get non-blocking DNS lookups in a portable
+ * manner requires use of a thread. So achieve a non-blocking
+ * socket listen using QIOTask would require:
+ *
+ * <example>
+ *    static int myobject_listen_worker(QIOTask *task,
+ *                                      Error **errp,
+ *                                      gpointer opaque)
+ *    {
+ *       QMyObject obj = QMY_OBJECT(qio_task_get_source(task));
+ *       SocketAddress *addr = opaque;
+ *
+ *       obj->fd = socket_listen(addr, errp);
+ *       if (obj->fd < 0) {
+ *          return -1;
+ *       }
+ *       return 0;
+ *    }
+ *
+ *    void myobject_listen_async(QMyObject *obj,
+ *                               SocketAddress *addr,
+ *                               QIOTaskFunc *func,
+ *                               gpointer opaque,
+ *                               GDestroyNotify *notify)
+ *    {
+ *      QIOTask *task;
+ *      SocketAddress *addrCopy;
+ *
+ *      qapi_copy_SocketAddress(&addrCopy, addr);
+ *      task = qio_task_new(OBJECT(obj), func, opaque, notify);
+ *
+ *      qio_task_run_in_thread(task, myobject_listen_worker,
+ *                             addrCopy,
+ *                             qapi_free_SocketAddress);
+ *    }
+ * </example>
+ *
+ * NB, The 'func' callback passed into myobject_listen_async
+ * will be invoked from the main event thread, despite the
+ * actual operation being performed in a different thread.
+ */
+
+/**
+ * qio_task_new:
+ * @source: the object on which the operation is invoked
+ * @func: the callback to invoke when the task completes
+ * @opaque: opaque data to pass to @func when invoked
+ * @destroy: optional callback to free @opaque
+ *
+ * Creates a new task struct to track completion of a
+ * background operation running on the object @source.
+ * When the operation completes or fails, the callback
+ * @func will be invoked. The callback can access the
+ * 'err' attribute in the task object to determine if
+ * the operation was successful or not.
+ *
+ * The returned task will be released when one of
+ * qio_task_abort() or qio_task_complete() are invoked.
+ *
+ * Returns: the task struct
+ */
+QIOTask *qio_task_new(Object *source,
+                      QIOTaskFunc func,
+                      gpointer opaque,
+                      GDestroyNotify destroy);
+
+/**
+ * qio_task_run_in_thread:
+ * @task: the task struct
+ * @worker: the function to invoke in a thread
+ * @opaque: opaque data to pass to @worker
+ * @destroy: function to free @opaque
+ *
+ * Run a task in a background thread. If @worker
+ * returns 0 it will call qio_task_complete() in
+ * the main event thread context. If @worker
+ * returns -1 it will call qio_task_abort() in
+ * the main event thread context.
+ */
+void qio_task_run_in_thread(QIOTask *task,
+                            QIOTaskWorker worker,
+                            gpointer opaque,
+                            GDestroyNotify destroy);
+
+/**
+ * qio_task_complete:
+ * @task: the task struct
+ *
+ * Mark the operation as succesfully completed
+ * and free the memory for @task.
+ */
+void qio_task_complete(QIOTask *task);
+
+/**
+ * qio_task_abort:
+ * @task: the task struct
+ * @err: the error to record for the operation
+ *
+ * Mark the operation as failed, with @err providing
+ * details about the failure. The @err may be freed
+ * afer the function returns, as the notification
+ * callback is invoked synchronously. The @task will
+ * be freed when this call completes.
+ */
+void qio_task_abort(QIOTask *task,
+                    Error *err);
+
+
+/**
+ * qio_task_get_source:
+ * @task: the task struct
+ *
+ * Get the source object associated with the background
+ * task. This returns a new reference to the object,
+ * which the caller must released with object_unref()
+ * when no longer required.
+ *
+ * Returns: the source object
+ */
+Object *qio_task_get_source(QIOTask *task);
+
+#endif /* QIO_TASK_H__ */
diff --git a/qemu/include/libdecnumber/dconfig.h b/qemu/include/libdecnumber/dconfig.h
index 2f0455a06..0f7dccef1 100644
--- a/qemu/include/libdecnumber/dconfig.h
+++ b/qemu/include/libdecnumber/dconfig.h
@@ -27,7 +27,6 @@
    Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.  */
 
-#include "config-host.h"
 
 #if defined(HOST_WORDS_BIGENDIAN)
 #define WORDS_BIGENDIAN 1
diff --git a/qemu/include/libdecnumber/decContext.h b/qemu/include/libdecnumber/decContext.h
index c3e46f405..01365e231 100644
--- a/qemu/include/libdecnumber/decContext.h
+++ b/qemu/include/libdecnumber/decContext.h
@@ -56,9 +56,6 @@
   #define DECCFULLNAME "Decimal Context Descriptor"   /* Verbose name */
   #define DECCAUTHOR   "Mike Cowlishaw"		      /* Who to blame */
 
-  #include <stdint.h>
-  #include <stdio.h>		   /* for printf, etc.		      */
-  #include <signal.h>		   /* for traps			      */
 
   /* Extended flags setting -- set this to 0 to use only IEEE flags   */
   #define DECEXTFLAG 1		   /* 1=enable extended flags	      */
diff --git a/qemu/include/libdecnumber/decNumberLocal.h b/qemu/include/libdecnumber/decNumberLocal.h
index 71ed77bf2..94fb51292 100644
--- a/qemu/include/libdecnumber/decNumberLocal.h
+++ b/qemu/include/libdecnumber/decNumberLocal.h
@@ -42,8 +42,6 @@
   #define DECVERSION	"decNumber 3.53" /* Package Version [16 max.] */
   #define DECNLAUTHOR	"Mike Cowlishaw"	      /* Who to blame */
 
-  #include <stdlib.h>	      /* for abs			      */
-  #include <string.h>	      /* for memset, strcpy		      */
   #include "libdecnumber/dconfig.h"
 
   /* Conditional code flag -- set this to match hardware platform     */
diff --git a/qemu/include/migration/migration.h b/qemu/include/migration/migration.h
index 83346210b..ac2c12c2a 100644
--- a/qemu/include/migration/migration.h
+++ b/qemu/include/migration/migration.h
@@ -18,7 +18,6 @@
 #include "qemu-common.h"
 #include "qemu/thread.h"
 #include "qemu/notify.h"
-#include "qapi/error.h"
 #include "migration/vmstate.h"
 #include "qapi-types.h"
 #include "exec/cpu-common.h"
@@ -35,6 +34,7 @@
 #define QEMU_VM_SUBSECTION           0x05
 #define QEMU_VM_VMDESCRIPTION        0x06
 #define QEMU_VM_CONFIGURATION        0x07
+#define QEMU_VM_COMMAND              0x08
 #define QEMU_VM_SECTION_FOOTER       0x7e
 
 struct MigrationParams {
@@ -42,14 +42,71 @@ struct MigrationParams {
     bool shared;
 };
 
-typedef struct MigrationState MigrationState;
+/* Messages sent on the return path from destination to source */
+enum mig_rp_message_type {
+    MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
+    MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
+    MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
+
+    MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
+    MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
+
+    MIG_RP_MSG_MAX
+};
 
 typedef QLIST_HEAD(, LoadStateEntry) LoadStateEntry_Head;
 
+/* The current postcopy state is read/set by postcopy_state_get/set
+ * which update it atomically.
+ * The state is updated as postcopy messages are received, and
+ * in general only one thread should be writing to the state at any one
+ * time, initially the main thread and then the listen thread;
+ * Corner cases are where either thread finishes early and/or errors.
+ * The state is checked as messages are received to ensure that
+ * the source is sending us messages in the correct order.
+ * The state is also used by the RAM reception code to know if it
+ * has to place pages atomically, and the cleanup code at the end of
+ * the main thread to know if it has to delay cleanup until the end
+ * of postcopy.
+ */
+typedef enum {
+    POSTCOPY_INCOMING_NONE = 0,  /* Initial state - no postcopy */
+    POSTCOPY_INCOMING_ADVISE,
+    POSTCOPY_INCOMING_DISCARD,
+    POSTCOPY_INCOMING_LISTENING,
+    POSTCOPY_INCOMING_RUNNING,
+    POSTCOPY_INCOMING_END
+} PostcopyState;
+
 /* State for the incoming migration */
 struct MigrationIncomingState {
-    QEMUFile *file;
+    QEMUFile *from_src_file;
+
+    /*
+     * Free at the start of the main state load, set as the main thread finishes
+     * loading state.
+     */
+    QemuEvent main_thread_load_event;
+
+    bool           have_fault_thread;
+    QemuThread     fault_thread;
+    QemuSemaphore  fault_thread_sem;
 
+    bool           have_listen_thread;
+    QemuThread     listen_thread;
+    QemuSemaphore  listen_thread_sem;
+
+    /* For the kernel to send us notifications */
+    int       userfault_fd;
+    /* To tell the fault_thread to quit */
+    int       userfault_quit_fd;
+    QEMUFile *to_src_file;
+    QemuMutex rp_mutex;    /* We send replies from multiple threads */
+    void     *postcopy_tmp_page;
+
+    QEMUBH *bh;
+
+    int state;
     /* See savevm.c */
     LoadStateEntry_Head loadvm_handlers;
 };
@@ -58,6 +115,18 @@ MigrationIncomingState *migration_incoming_get_current(void);
 MigrationIncomingState *migration_incoming_state_new(QEMUFile *f);
 void migration_incoming_state_destroy(void);
 
+/*
+ * An outstanding page request, on the source, having been received
+ * and queued
+ */
+struct MigrationSrcPageRequest {
+    RAMBlock *rb;
+    hwaddr    offset;
+    hwaddr    len;
+
+    QSIMPLEQ_ENTRY(MigrationSrcPageRequest) next_req;
+};
+
 struct MigrationState
 {
     int64_t bandwidth_limit;
@@ -65,23 +134,47 @@ struct MigrationState
     size_t xfer_limit;
     QemuThread thread;
     QEMUBH *cleanup_bh;
-    QEMUFile *file;
-    int parameters[MIGRATION_PARAMETER_MAX];
+    QEMUFile *to_dst_file;
+    int parameters[MIGRATION_PARAMETER__MAX];
 
     int state;
     MigrationParams params;
+
+    /* State related to return path */
+    struct {
+        QEMUFile     *from_dst_file;
+        QemuThread    rp_thread;
+        bool          error;
+    } rp_state;
+
     double mbps;
     int64_t total_time;
     int64_t downtime;
     int64_t expected_downtime;
     int64_t dirty_pages_rate;
     int64_t dirty_bytes_rate;
-    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
+    bool enabled_capabilities[MIGRATION_CAPABILITY__MAX];
     int64_t xbzrle_cache_size;
     int64_t setup_time;
     int64_t dirty_sync_count;
+
+    /* Flag set once the migration has been asked to enter postcopy */
+    bool start_postcopy;
+    /* Flag set after postcopy has sent the device state */
+    bool postcopy_after_devices;
+
+    /* Flag set once the migration thread is running (and needs joining) */
+    bool migration_thread_running;
+
+    /* Queue of outstanding page requests from the destination */
+    QemuMutex src_page_req_mutex;
+    QSIMPLEQ_HEAD(src_page_requests, MigrationSrcPageRequest) src_page_requests;
+    /* The RAMBlock used in the last src_page_request */
+    RAMBlock *last_req_rb;
 };
 
+void migrate_set_state(int *state, int old_state, int new_state);
+
 void process_incoming_migration(QEMUFile *f);
 
 void qemu_start_incoming_migration(const char *uri, Error **errp);
@@ -116,9 +209,14 @@ int migrate_fd_close(MigrationState *s);
 
 void add_migration_state_change_notifier(Notifier *notify);
 void remove_migration_state_change_notifier(Notifier *notify);
+MigrationState *migrate_init(const MigrationParams *params);
 bool migration_in_setup(MigrationState *);
 bool migration_has_finished(MigrationState *);
 bool migration_has_failed(MigrationState *);
+/* True if outgoing migration has entered postcopy phase */
+bool migration_in_postcopy(MigrationState *);
+/* ...and after the device transmission */
+bool migration_in_postcopy_after_devices(MigrationState *);
 MigrationState *migrate_get_current(void);
 
 void migrate_compress_threads_create(void);
@@ -145,6 +243,13 @@ uint64_t xbzrle_mig_pages_cache_miss(void);
 double xbzrle_mig_cache_miss_rate(void);
 
 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size);
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected);
+/* For outgoing discard bitmap */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms);
+/* For incoming postcopy discard */
+int ram_discard_range(MigrationIncomingState *mis, const char *block_name,
+                      uint64_t start, size_t length);
+int ram_postcopy_incoming_init(MigrationIncomingState *mis);
 
 /**
  * @migrate_add_blocker - prevent migration from proceeding
@@ -160,6 +265,7 @@ void migrate_add_blocker(Error *reason);
  */
 void migrate_del_blocker(Error *reason);
 
+bool migrate_postcopy_ram(void);
 bool migrate_zero_blocks(void);
 
 bool migrate_auto_converge(void);
@@ -179,6 +285,17 @@ int migrate_compress_threads(void);
 int migrate_decompress_threads(void);
 bool migrate_use_events(void);
 
+/* Sending on the return path - generic and then for each message type */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data);
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char* rbname,
+                              ram_addr_t start, size_t len);
+
 void ram_control_before_iterate(QEMUFile *f, uint64_t flags);
 void ram_control_after_iterate(QEMUFile *f, uint64_t flags);
 void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data);
@@ -204,4 +321,12 @@ void global_state_set_optional(void);
 void savevm_skip_configuration(void);
 int global_state_store(void);
 void global_state_store_running(void);
+
+void flush_page_queue(MigrationState *ms);
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len);
+
+PostcopyState postcopy_state_get(void);
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state);
 #endif
diff --git a/qemu/include/migration/postcopy-ram.h b/qemu/include/migration/postcopy-ram.h
new file mode 100644
index 000000000..b6a7491f2
--- /dev/null
+++ b/qemu/include/migration/postcopy-ram.h
@@ -0,0 +1,99 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#ifndef QEMU_POSTCOPY_RAM_H
+#define QEMU_POSTCOPY_RAM_H
+
+/* Return true if the host supports everything we need to do postcopy-ram */
+bool postcopy_ram_supported_by_host(void);
+
+/*
+ * Make all of RAM sensitive to accesses to areas that haven't yet been written
+ * and wire up anything necessary to deal with it.
+ */
+int postcopy_ram_enable_notify(MigrationIncomingState *mis);
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from ram.c's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages);
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis);
+
+/*
+ * Discard the contents of 'length' bytes from 'start'
+ * We can assume that if we've been called postcopy_ram_hosttest returned true
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length);
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis);
+
+/*
+ * Called at the start of each RAMBlock by the bitmap code.
+ * 'offset' is the bitmap offset of the named RAMBlock in the migration
+ * bitmap.
+ * Returns a new PDS
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name);
+
+/*
+ * Called by the bitmap code for each chunk to discard.
+ * May send a discard message, may just leave it queued to
+ * be sent later.
+ * @start,@length: a range of pages in the migration bitmap in the
+ *  RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                 unsigned long start, unsigned long length);
+
+/*
+ * Called at the end of each RAMBlock by the bitmap code.
+ * Sends any outstanding discard messages, frees the PDS.
+ */
+void postcopy_discard_send_finish(MigrationState *ms,
+                                  PostcopyDiscardState *pds);
+
+/*
+ * Place a page (from) at (host) efficiently
+ *    There are restrictions on how 'from' must be mapped, in general best
+ *    to use other postcopy_ routines to allocate.
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from);
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host);
+
+/*
+ * Allocate a page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * Returns: Pointer to allocated page
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis);
+
+#endif
diff --git a/qemu/include/migration/qemu-file.h b/qemu/include/migration/qemu-file.h
index ea49f33fa..3f6b4ed58 100644
--- a/qemu/include/migration/qemu-file.h
+++ b/qemu/include/migration/qemu-file.h
@@ -25,21 +25,20 @@
 #define QEMU_FILE_H 1
 #include "exec/cpu-common.h"
 
-#include <stdint.h>
 
 /* This function writes a chunk of data to a file at the given position.
  * The pos argument can be ignored if the file is only being used for
  * streaming.  The handler should try to write all of the data it can.
  */
-typedef int (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
-                                    int64_t pos, int size);
+typedef ssize_t (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
+                                        int64_t pos, size_t size);
 
 /* Read a chunk of data from a file at the given position.  The pos argument
  * can be ignored if the file is only be used for streaming.  The number of
  * bytes actually read should be returned.
  */
-typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
-                                    int64_t pos, int size);
+typedef ssize_t (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
+                                        int64_t pos, size_t size);
 
 /* Close a file
  *
@@ -89,6 +88,11 @@ typedef size_t (QEMURamSaveFunc)(QEMUFile *f, void *opaque,
                                uint64_t *bytes_sent);
 
 /*
+ * Return a QEMUFile for comms in the opposite direction
+ */
+typedef QEMUFile *(QEMURetPathFunc)(void *opaque);
+
+/*
  * Stop any read or write (depending on flags) on the underlying
  * transport on the QEMUFile.
  * Existing blocking reads/writes must be woken
@@ -106,6 +110,7 @@ typedef struct QEMUFileOps {
     QEMURamHookFunc *after_ram_iterate;
     QEMURamHookFunc *hook_ram_load;
     QEMURamSaveFunc *save_page;
+    QEMURetPathFunc *get_return_path;
     QEMUFileShutdownFunc *shut_down;
 } QEMUFileOps;
 
@@ -126,13 +131,13 @@ int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
 int64_t qemu_ftell_fast(QEMUFile *f);
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size);
 void qemu_put_byte(QEMUFile *f, int v);
 /*
  * put_buffer without copying the buffer.
  * The buffer should be available till it is sent asynchronously.
  */
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size);
 bool qemu_file_mode_is_not_valid(const char *mode);
 bool qemu_file_is_writable(QEMUFile *f);
 
@@ -161,11 +166,13 @@ static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
 void qemu_put_be16(QEMUFile *f, unsigned int v);
 void qemu_put_be32(QEMUFile *f, unsigned int v);
 void qemu_put_be64(QEMUFile *f, uint64_t v);
-int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset);
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size);
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset);
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size);
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size);
 ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
                                   int level);
 int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src);
+
 /*
  * Note that you can only peek continuous bytes from where the current pointer
  * is; you aren't guaranteed to be able to peak to +n bytes unless you've
@@ -194,7 +201,9 @@ int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
 void qemu_file_set_error(QEMUFile *f, int ret);
 int qemu_file_shutdown(QEMUFile *f);
+QEMUFile *qemu_file_get_return_path(QEMUFile *f);
 void qemu_fflush(QEMUFile *f);
+void qemu_file_set_blocking(QEMUFile *f, bool block);
 
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
@@ -237,7 +246,7 @@ static inline void qemu_get_8s(QEMUFile *f, uint8_t *pv)
 }
 
 // Signed versions for type safety
-static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, int size)
+static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, size_t size)
 {
     qemu_put_buffer(f, (const uint8_t *)buf, size);
 }
diff --git a/qemu/include/migration/vmstate.h b/qemu/include/migration/vmstate.h
index 2e5a97dec..84ee355ce 100644
--- a/qemu/include/migration/vmstate.h
+++ b/qemu/include/migration/vmstate.h
@@ -39,8 +39,9 @@ typedef struct SaveVMHandlers {
     void (*set_params)(const MigrationParams *params, void * opaque);
     SaveStateHandler *save_state;
 
-    void (*cancel)(void *opaque);
-    int (*save_live_complete)(QEMUFile *f, void *opaque);
+    void (*cleanup)(void *opaque);
+    int (*save_live_complete_postcopy)(QEMUFile *f, void *opaque);
+    int (*save_live_complete_precopy)(QEMUFile *f, void *opaque);
 
     /* This runs both outside and inside the iothread lock.  */
     bool (*is_active)(void *opaque);
@@ -54,8 +55,9 @@ typedef struct SaveVMHandlers {
 
     /* This runs outside the iothread lock!  */
     int (*save_live_setup)(QEMUFile *f, void *opaque);
-    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
-
+    void (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size,
+                              uint64_t *non_postcopiable_pending,
+                              uint64_t *postcopiable_pending);
     LoadStateHandler *load_state;
 } SaveVMHandlers;
 
@@ -86,20 +88,101 @@ struct VMStateInfo {
 };
 
 enum VMStateFlags {
+    /* Ignored */
     VMS_SINGLE           = 0x001,
+
+    /* The struct member at opaque + VMStateField.offset is a pointer
+     * to the actual field (e.g. struct a { uint8_t *b;
+     * }). Dereference the pointer before using it as basis for
+     * further pointer arithmetic (see e.g. VMS_ARRAY). Does not
+     * affect the meaning of VMStateField.num_offset or
+     * VMStateField.size_offset; see VMS_VARRAY* and VMS_VBUFFER for
+     * those. */
     VMS_POINTER          = 0x002,
+
+    /* The field is an array of fixed size. VMStateField.num contains
+     * the number of entries in the array. The size of each entry is
+     * given by VMStateField.size and / or opaque +
+     * VMStateField.size_offset; see VMS_VBUFFER and
+     * VMS_MULTIPLY. Each array entry will be processed individually
+     * (VMStateField.info.get()/put() if VMS_STRUCT is not set,
+     * recursion into VMStateField.vmsd if VMS_STRUCT is set). May not
+     * be combined with VMS_VARRAY*. */
     VMS_ARRAY            = 0x004,
+
+    /* The field is itself a struct, containing one or more
+     * fields. Recurse into VMStateField.vmsd. Most useful in
+     * combination with VMS_ARRAY / VMS_VARRAY*, recursing into each
+     * array entry. */
     VMS_STRUCT           = 0x008,
-    VMS_VARRAY_INT32     = 0x010,  /* Array with size in int32_t field*/
-    VMS_BUFFER           = 0x020,  /* static sized buffer */
+
+    /* The field is an array of variable size. The int32_t at opaque +
+     * VMStateField.num_offset contains the number of entries in the
+     * array. See the VMS_ARRAY description regarding array handling
+     * in general. May not be combined with VMS_ARRAY or any other
+     * VMS_VARRAY*. */
+    VMS_VARRAY_INT32     = 0x010,
+
+    /* Ignored */
+    VMS_BUFFER           = 0x020,
+
+    /* The field is a (fixed-size or variable-size) array of pointers
+     * (e.g. struct a { uint8_t *b[]; }). Dereference each array entry
+     * before using it. Note: Does not imply any one of VMS_ARRAY /
+     * VMS_VARRAY*; these need to be set explicitly. */
     VMS_ARRAY_OF_POINTER = 0x040,
-    VMS_VARRAY_UINT16    = 0x080,  /* Array with size in uint16_t field */
-    VMS_VBUFFER          = 0x100,  /* Buffer with size in int32_t field */
-    VMS_MULTIPLY         = 0x200,  /* multiply "size" field by field_size */
-    VMS_VARRAY_UINT8     = 0x400,  /* Array with size in uint8_t field*/
-    VMS_VARRAY_UINT32    = 0x800,  /* Array with size in uint32_t field*/
-    VMS_MUST_EXIST       = 0x1000, /* Field must exist in input */
-    VMS_ALLOC            = 0x2000, /* Alloc a buffer on the destination */
+
+    /* The field is an array of variable size. The uint16_t at opaque
+     * + VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT16    = 0x080,
+
+    /* The size of the individual entries (a single array entry if
+     * VMS_ARRAY or any of VMS_VARRAY* are set, or the field itself if
+     * neither is set) is variable (i.e. not known at compile-time),
+     * but the same for all entries. Use the int32_t at opaque +
+     * VMStateField.size_offset (subject to VMS_MULTIPLY) to determine
+     * the size of each (and every) entry. */
+    VMS_VBUFFER          = 0x100,
+
+    /* Multiply the entry size given by the int32_t at opaque +
+     * VMStateField.size_offset (see VMS_VBUFFER description) with
+     * VMStateField.size to determine the number of bytes to be
+     * allocated. Only valid in combination with VMS_VBUFFER. */
+    VMS_MULTIPLY         = 0x200,
+
+    /* The field is an array of variable size. The uint8_t at opaque +
+     * VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT8     = 0x400,
+
+    /* The field is an array of variable size. The uint32_t at opaque
+     * + VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT32    = 0x800,
+
+    /* Fail loading the serialised VM state if this field is missing
+     * from the input. */
+    VMS_MUST_EXIST       = 0x1000,
+
+    /* When loading serialised VM state, allocate memory for the
+     * (entire) field. Only valid in combination with
+     * VMS_POINTER. Note: Not all combinations with other flags are
+     * currently supported, e.g. VMS_ALLOC|VMS_ARRAY_OF_POINTER won't
+     * cause the individual entries to be allocated. */
+    VMS_ALLOC            = 0x2000,
+
+    /* Multiply the number of entries given by the integer at opaque +
+     * VMStateField.num_offset (see VMS_VARRAY*) with VMStateField.num
+     * to determine the number of entries in the array. Only valid in
+     * combination with one of VMS_VARRAY*. */
+    VMS_MULTIPLY_ELEMENTS = 0x4000,
 };
 
 typedef struct {
@@ -154,6 +237,7 @@ extern const VMStateInfo vmstate_info_uint32;
 extern const VMStateInfo vmstate_info_uint64;
 
 extern const VMStateInfo vmstate_info_float64;
+extern const VMStateInfo vmstate_info_cpudouble;
 
 extern const VMStateInfo vmstate_info_timer;
 extern const VMStateInfo vmstate_info_buffer;
@@ -243,6 +327,16 @@ extern const VMStateInfo vmstate_info_bitmap;
     .offset     = vmstate_offset_2darray(_state, _field, _type, _n1, _n2),  \
 }
 
+#define VMSTATE_VARRAY_MULTIPLY(_field, _state, _field_num, _multiply, _info, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t),\
+    .num        = (_multiply),                                       \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_UINT32|VMS_MULTIPLY_ELEMENTS,           \
+    .offset     = offsetof(_state, _field),                          \
+}
+
 #define VMSTATE_ARRAY_TEST(_field, _state, _num, _test, _info, _type) {\
     .name         = (stringify(_field)),                              \
     .field_exists = (_test),                                          \
@@ -382,6 +476,19 @@ extern const VMStateInfo vmstate_info_bitmap;
     .offset     = offsetof(_state, _field),                          \
 }
 
+/* a variable length array (i.e. _type *_field) but we know the
+ * length
+ */
+#define VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(_field, _state, _num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num          = (_num),                                          \
+    .version_id = (_version),                                        \
+    .vmsd       = &(_vmsd),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_STRUCT|VMS_ARRAY|VMS_POINTER,                  \
+    .offset     = offsetof(_state, _field),                          \
+}
+
 #define VMSTATE_STRUCT_VARRAY_POINTER_INT32(_field, _state, _field_num, _vmsd, _type) { \
     .name       = (stringify(_field)),                               \
     .version_id = 0,                                                 \
@@ -754,9 +861,6 @@ extern const VMStateInfo vmstate_info_bitmap;
 #define VMSTATE_UINT32_SUB_ARRAY(_f, _s, _start, _num)                \
     VMSTATE_SUB_ARRAY(_f, _s, _start, _num, 0, vmstate_info_uint32, uint32_t)
 
-#define VMSTATE_UINT32_ARRAY(_f, _s, _n)                              \
-    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, 0)
-
 #define VMSTATE_INT64_ARRAY_V(_f, _s, _n, _v)                         \
     VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int64, int64_t)
 
@@ -769,6 +873,12 @@ extern const VMStateInfo vmstate_info_bitmap;
 #define VMSTATE_FLOAT64_ARRAY(_f, _s, _n)                             \
     VMSTATE_FLOAT64_ARRAY_V(_f, _s, _n, 0)
 
+#define VMSTATE_CPUDOUBLE_ARRAY_V(_f, _s, _n, _v)                     \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_cpudouble, CPU_DoubleU)
+
+#define VMSTATE_CPUDOUBLE_ARRAY(_f, _s, _n)                           \
+    VMSTATE_CPUDOUBLE_ARRAY_V(_f, _s, _n, 0)
+
 #define VMSTATE_BUFFER_V(_f, _s, _v)                                  \
     VMSTATE_STATIC_BUFFER(_f, _s, _v, NULL, 0, sizeof(typeof_field(_s, _f)))
 
diff --git a/qemu/ui/vnc-tls.h b/qemu/include/monitor/hmp-target.h
index f9829c782..bc2c9c04d 100644
--- a/qemu/ui/vnc-tls.h
+++ b/qemu/include/monitor/hmp-target.h
@@ -1,9 +1,7 @@
 /*
- * QEMU VNC display driver. TLS helpers
+ * QEMU monitor
  *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
+ * Copyright (c) 2003-2004 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,47 +21,29 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#ifndef MONITOR_COMMON_H
+#define MONITOR_COMMON_H
 
+#define MD_TLONG 0
+#define MD_I32   1
 
-#ifndef __QEMU_VNC_TLS_H__
-#define __QEMU_VNC_TLS_H__
-
-#include <gnutls/gnutls.h>
-#include <gnutls/x509.h>
-
-#include "qemu/acl.h"
-
-typedef struct VncDisplayTLS VncDisplayTLS;
-typedef struct VncStateTLS VncStateTLS;
-
-/* Server state */
-struct VncDisplayTLS {
-    int x509verify; /* Non-zero if server requests & validates client cert */
-    qemu_acl *acl;
-
-    /* Paths to x509 certs/keys */
-    char *x509cacert;
-    char *x509cacrl;
-    char *x509cert;
-    char *x509key;
+struct MonitorDef {
+    const char *name;
+    int offset;
+    target_long (*get_value)(const struct MonitorDef *md, int val);
+    int type;
 };
 
-/* Per client state */
-struct VncStateTLS {
-    gnutls_session_t session;
-
-    /* Client's Distinguished Name from the x509 cert */
-    char *dname;
-};
-
-int vnc_tls_client_setup(VncState *vs, int x509Creds);
-void vnc_tls_client_cleanup(VncState *vs);
-
-int vnc_tls_validate_certificate(VncState *vs);
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-			       const char *path);
+const MonitorDef *target_monitor_defs(void);
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
 
+CPUArchState *mon_get_cpu_env(void);
+CPUState *mon_get_cpu(void);
 
-#endif /* __QEMU_VNC_TLS_H__ */
+void hmp_info_mem(Monitor *mon, const QDict *qdict);
+void hmp_info_tlb(Monitor *mon, const QDict *qdict);
+void hmp_mce(Monitor *mon, const QDict *qdict);
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict);
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict);
 
+#endif /* MONITOR_COMMON */
diff --git a/qemu/include/monitor/monitor.h b/qemu/include/monitor/monitor.h
index 9aff47e69..aa0f37320 100644
--- a/qemu/include/monitor/monitor.h
+++ b/qemu/include/monitor/monitor.h
@@ -34,6 +34,7 @@ int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp);
 void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
     GCC_FMT_ATTR(2, 0);
 void monitor_printf(Monitor *mon, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
+int monitor_fprintf(FILE *stream, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
 void monitor_flush(Monitor *mon);
 int monitor_set_cpu(int cpu_index);
 int monitor_get_cpu_index(void);
@@ -42,12 +43,6 @@ void monitor_read_command(Monitor *mon, int show_prompt);
 int monitor_read_password(Monitor *mon, ReadLineFunc *readline_func,
                           void *opaque);
 
-void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp);
-void qmp_qom_get(QDict *qdict, QObject **ret, Error **errp);
-void qmp_object_add(QDict *qdict, QObject **ret, Error **errp);
-void object_add(const char *type, const char *id, const QDict *qdict,
-                Visitor *v, Error **errp);
-
 AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
                                 bool has_opaque, const char *opaque,
                                 Error **errp);
diff --git a/qemu/include/monitor/qdev.h b/qemu/include/monitor/qdev.h
index 50ec2ffc1..c4b8a0514 100644
--- a/qemu/include/monitor/qdev.h
+++ b/qemu/include/monitor/qdev.h
@@ -2,7 +2,6 @@
 #define QEMU_QDEV_MONITOR_H
 
 #include "hw/qdev-core.h"
-#include "qemu/typedefs.h"
 
 /*** monitor commands ***/
 
diff --git a/qemu/include/net/checksum.h b/qemu/include/net/checksum.h
index 2d7a363d4..7de1acb79 100644
--- a/qemu/include/net/checksum.h
+++ b/qemu/include/net/checksum.h
@@ -18,7 +18,6 @@
 #ifndef QEMU_NET_CHECKSUM_H
 #define QEMU_NET_CHECKSUM_H
 
-#include <stdint.h>
 struct iovec;
 
 uint32_t net_checksum_add_cont(int len, uint8_t *buf, int seq);
diff --git a/qemu/include/net/eth.h b/qemu/include/net/eth.h
index b3273b823..18d0be3b1 100644
--- a/qemu/include/net/eth.h
+++ b/qemu/include/net/eth.h
@@ -26,12 +26,11 @@
 #ifndef QEMU_ETH_H
 #define QEMU_ETH_H
 
-#include <sys/types.h>
-#include <string.h>
 #include "qemu/bswap.h"
 #include "qemu/iov.h"
 
 #define ETH_ALEN 6
+#define ETH_HLEN 14
 
 struct eth_header {
     uint8_t  h_dest[ETH_ALEN];   /* destination eth addr */
@@ -170,7 +169,8 @@ struct tcp_hdr {
 #define IP_HEADER_VERSION(ip)     \
     ((ip->ip_ver_len >> 4)&0xf)
 
-#define ETH_P_IP                  (0x0800)
+#define ETH_P_IP                  (0x0800)      /* Internet Protocol packet  */
+#define ETH_P_ARP                 (0x0806)      /* Address Resolution packet */
 #define ETH_P_IPV6                (0x86dd)
 #define ETH_P_VLAN                (0x8100)
 #define ETH_P_DVLAN               (0x88a8)
diff --git a/qemu/include/net/filter.h b/qemu/include/net/filter.h
new file mode 100644
index 000000000..0c4a2ea6c
--- /dev/null
+++ b/qemu/include/net/filter.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_NET_FILTER_H
+#define QEMU_NET_FILTER_H
+
+#include "qom/object.h"
+#include "qemu-common.h"
+#include "net/queue.h"
+
+#define TYPE_NETFILTER "netfilter"
+#define NETFILTER(obj) \
+    OBJECT_CHECK(NetFilterState, (obj), TYPE_NETFILTER)
+#define NETFILTER_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(NetFilterClass, (obj), TYPE_NETFILTER)
+#define NETFILTER_CLASS(klass) \
+    OBJECT_CLASS_CHECK(NetFilterClass, (klass), TYPE_NETFILTER)
+
+typedef void (FilterSetup) (NetFilterState *nf, Error **errp);
+typedef void (FilterCleanup) (NetFilterState *nf);
+/*
+ * Return:
+ *   0: finished handling the packet, we should continue
+ *   size: filter stolen this packet, we stop pass this packet further
+ */
+typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
+                                   NetClientState *sender,
+                                   unsigned flags,
+                                   const struct iovec *iov,
+                                   int iovcnt,
+                                   NetPacketSent *sent_cb);
+
+typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
+
+typedef struct NetFilterClass {
+    ObjectClass parent_class;
+
+    /* optional */
+    FilterSetup *setup;
+    FilterCleanup *cleanup;
+    FilterStatusChanged *status_changed;
+    /* mandatory */
+    FilterReceiveIOV *receive_iov;
+} NetFilterClass;
+
+
+struct NetFilterState {
+    /* private */
+    Object parent;
+
+    /* protected */
+    char *netdev_id;
+    NetClientState *netdev;
+    NetFilterDirection direction;
+    bool on;
+    QTAILQ_ENTRY(NetFilterState) next;
+};
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
+
+/* pass the packet to the next filter */
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque);
+
+#endif /* QEMU_NET_FILTER_H */
diff --git a/qemu/include/net/net.h b/qemu/include/net/net.h
index 6a6cbef24..73e4c466e 100644
--- a/qemu/include/net/net.h
+++ b/qemu/include/net/net.h
@@ -92,6 +92,7 @@ struct NetClientState {
     NetClientDestructor *destructor;
     unsigned int queue_index;
     unsigned rxfilter_notify_enabled:1;
+    QTAILQ_HEAD(NetFilterHead, NetFilterState) filters;
 };
 
 typedef struct NICState {
@@ -151,11 +152,6 @@ void qemu_check_nic_model(NICInfo *nd, const char *model);
 int qemu_find_nic_model(NICInfo *nd, const char * const *models,
                         const char *default_model);
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque);
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                             unsigned flags,
                             const struct iovec *iov,
diff --git a/qemu/include/net/queue.h b/qemu/include/net/queue.h
index fc02b3391..5469fdbea 100644
--- a/qemu/include/net/queue.h
+++ b/qemu/include/net/queue.h
@@ -34,7 +34,25 @@ typedef void (NetPacketSent) (NetClientState *sender, ssize_t ret);
 #define QEMU_NET_PACKET_FLAG_NONE  0
 #define QEMU_NET_PACKET_FLAG_RAW  (1<<0)
 
-NetQueue *qemu_new_net_queue(void *opaque);
+/* Returns:
+ *   >0 - success
+ *    0 - queue packet for future redelivery
+ *   <0 - failure (discard packet)
+ */
+typedef ssize_t (NetQueueDeliverFunc)(NetClientState *sender,
+                                      unsigned flags,
+                                      const struct iovec *iov,
+                                      int iovcnt,
+                                      void *opaque);
+
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque);
+
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
 
 void qemu_del_net_queue(NetQueue *queue);
 
diff --git a/qemu/include/net/vhost_net.h b/qemu/include/net/vhost_net.h
index 840d4b16e..3389b410d 100644
--- a/qemu/include/net/vhost_net.h
+++ b/qemu/include/net/vhost_net.h
@@ -13,6 +13,7 @@ typedef struct VhostNetOptions {
     void *opaque;
 } VhostNetOptions;
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net);
 struct vhost_net *vhost_net_init(VhostNetOptions *options);
 
 int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, int total_queues);
@@ -26,5 +27,8 @@ void vhost_net_ack_features(VHostNetState *net, uint64_t features);
 bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
 void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
                               int idx, bool mask);
+int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
+
+int vhost_set_vring_enable(NetClientState * nc, int enable);
 #endif
diff --git a/qemu/include/qapi/error.h b/qemu/include/qapi/error.h
index f44c45183..11be2327c 100644
--- a/qemu/include/qapi/error.h
+++ b/qemu/include/qapi/error.h
@@ -2,107 +2,301 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
  */
+
+/*
+ * Error reporting system loosely patterned after Glib's GError.
+ *
+ * Create an error:
+ *     error_setg(&err, "situation normal, all fouled up");
+ *
+ * Create an error and add additional explanation:
+ *     error_setg(&err, "invalid quark");
+ *     error_append_hint(&err, "Valid quarks are up, down, strange, "
+ *                       "charm, top, bottom.\n");
+ *
+ * Do *not* contract this to
+ *     error_setg(&err, "invalid quark\n"
+ *                "Valid quarks are up, down, strange, charm, top, bottom.");
+ *
+ * Report an error to the current monitor if we have one, else stderr:
+ *     error_report_err(err);
+ * This frees the error object.
+ *
+ * Likewise, but with additional text prepended:
+ *     error_reportf_err(err, "Could not frobnicate '%s': ", name);
+ *
+ * Report an error somewhere else:
+ *     const char *msg = error_get_pretty(err);
+ *     do with msg what needs to be done...
+ *     error_free(err);
+ * Note that this loses hints added with error_append_hint().
+ *
+ * Handle an error without reporting it (just for completeness):
+ *     error_free(err);
+ *
+ * Assert that an expected error occurred, but clean it up without
+ * reporting it (primarily useful in testsuites):
+ *     error_free_or_abort(&err);
+ *
+ * Pass an existing error to the caller:
+ *     error_propagate(errp, err);
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Pass an existing error to the caller with the message modified:
+ *     error_propagate(errp, err);
+ *     error_prepend(errp, "Could not frobnicate '%s': ", name);
+ *
+ * Create a new error and pass it to the caller:
+ *     error_setg(errp, "situation normal, all fouled up");
+ *
+ * Call a function and receive an error from it:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *     }
+ *
+ * Call a function ignoring errors:
+ *     foo(arg, NULL);
+ *
+ * Call a function aborting on errors:
+ *     foo(arg, &error_abort);
+ *
+ * Call a function treating errors as fatal:
+ *     foo(arg, &error_fatal);
+ *
+ * Receive an error and pass it on to the caller:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *         error_propagate(errp, err);
+ *     }
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Do *not* "optimize" this to
+ *     foo(arg, errp);
+ *     if (*errp) { // WRONG!
+ *         handle the error...
+ *     }
+ * because errp may be NULL!
+ *
+ * But when all you do with the error is pass it on, please use
+ *     foo(arg, errp);
+ * for readability.
+ *
+ * Receive and accumulate multiple errors (first one wins):
+ *     Error *err = NULL, *local_err = NULL;
+ *     foo(arg, &err);
+ *     bar(arg, &local_err);
+ *     error_propagate(&err, local_err);
+ *     if (err) {
+ *         handle the error...
+ *     }
+ *
+ * Do *not* "optimize" this to
+ *     foo(arg, &err);
+ *     bar(arg, &err); // WRONG!
+ *     if (err) {
+ *         handle the error...
+ *     }
+ * because this may pass a non-null err to bar().
+ */
+
 #ifndef ERROR_H
 #define ERROR_H
 
-#include "qemu/compiler.h"
 #include "qapi-types.h"
-#include <stdbool.h>
 
-/**
- * A class representing internal errors within QEMU.  An error has a ErrorClass
- * code and a human message.
+/*
+ * Overall category of an error.
+ * Based on the qapi type QapiErrorClass, but reproduced here for nicer
+ * enum names.
  */
-typedef struct Error Error;
+typedef enum ErrorClass {
+    ERROR_CLASS_GENERIC_ERROR = QAPI_ERROR_CLASS_GENERICERROR,
+    ERROR_CLASS_COMMAND_NOT_FOUND = QAPI_ERROR_CLASS_COMMANDNOTFOUND,
+    ERROR_CLASS_DEVICE_ENCRYPTED = QAPI_ERROR_CLASS_DEVICEENCRYPTED,
+    ERROR_CLASS_DEVICE_NOT_ACTIVE = QAPI_ERROR_CLASS_DEVICENOTACTIVE,
+    ERROR_CLASS_DEVICE_NOT_FOUND = QAPI_ERROR_CLASS_DEVICENOTFOUND,
+    ERROR_CLASS_KVM_MISSING_CAP = QAPI_ERROR_CLASS_KVMMISSINGCAP,
+} ErrorClass;
 
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message.  This function is not meant to be used outside
- * of QEMU.
+/*
+ * Get @err's human-readable error message.
  */
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
-    GCC_FMT_ATTR(3, 4);
+const char *error_get_pretty(Error *err);
 
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a strerror() string if
- * @os_error is not zero.
+/*
+ * Get @err's error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
  */
-void error_set_errno(Error **errp, int os_error, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+ErrorClass error_get_class(const Error *err);
 
-#ifdef _WIN32
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a g_win32_error_message() string if
- * @win32_err is not zero.
+/*
+ * Create a new error object and assign it to *@errp.
+ * If @errp is NULL, the error is ignored.  Don't bother creating one
+ * then.
+ * If @errp is &error_abort, print a suitable message and abort().
+ * If @errp is &error_fatal, print a suitable message and exit(1).
+ * If @errp is anything else, *@errp must be NULL.
+ * The new error's class is ERROR_CLASS_GENERIC_ERROR, and its
+ * human-readable error message is made from printf-style @fmt, ...
+ * The resulting message should be a single phrase, with no newline or
+ * trailing punctuation.
+ * Please don't error_setg(&error_fatal, ...), use error_report() and
+ * exit(), because that's more obvious.
+ * Likewise, don't error_setg(&error_abort, ...), use assert().
  */
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
-#endif
+#define error_setg(errp, fmt, ...)                              \
+    error_setg_internal((errp), __FILE__, __LINE__, __func__,   \
+                        (fmt), ## __VA_ARGS__)
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+    GCC_FMT_ATTR(5, 6);
 
-/**
- * Same as error_set(), but sets a generic error
+/*
+ * Just like error_setg(), with @os_error info added to the message.
+ * If @os_error is non-zero, ": " + strerror(os_error) is appended to
+ * the human-readable error message.
  */
-#define error_setg(errp, fmt, ...) \
-    error_set(errp, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
-#define error_setg_errno(errp, os_error, fmt, ...) \
-    error_set_errno(errp, os_error, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
+#define error_setg_errno(errp, os_error, fmt, ...)                      \
+    error_setg_errno_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (os_error), (fmt), ## __VA_ARGS__)
+void error_setg_errno_internal(Error **errp,
+                               const char *fname, int line, const char *func,
+                               int os_error, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
+
 #ifdef _WIN32
-#define error_setg_win32(errp, win32_err, fmt, ...) \
-    error_set_win32(errp, win32_err, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
+/*
+ * Just like error_setg(), with @win32_error info added to the message.
+ * If @win32_error is non-zero, ": " + g_win32_error_message(win32_err)
+ * is appended to the human-readable error message.
+ */
+#define error_setg_win32(errp, win32_err, fmt, ...)                     \
+    error_setg_win32_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (win32_err), (fmt), ## __VA_ARGS__)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 #endif
 
-/**
- * Helper for open() errors
+/*
+ * Propagate error object (if any) from @local_err to @dst_errp.
+ * If @local_err is NULL, do nothing (because there's nothing to
+ * propagate).
+ * Else, if @dst_errp is NULL, errors are being ignored.  Free the
+ * error object.
+ * Else, if @dst_errp is &error_abort, print a suitable message and
+ * abort().
+ * Else, if @dst_errp is &error_fatal, print a suitable message and
+ * exit(1).
+ * Else, if @dst_errp already contains an error, ignore this one: free
+ * the error object.
+ * Else, move the error object from @local_err to *@dst_errp.
+ * On return, @local_err is invalid.
+ * Please don't error_propagate(&error_fatal, ...), use
+ * error_report_err() and exit(), because that's more obvious.
  */
-void error_setg_file_open(Error **errp, int os_errno, const char *filename);
+void error_propagate(Error **dst_errp, Error *local_err);
 
 /*
- * Get the error class of an error object.
+ * Prepend some text to @errp's human-readable error message.
+ * The text is made by formatting @fmt, @ap like vprintf().
  */
-ErrorClass error_get_class(const Error *err);
+void error_vprepend(Error **errp, const char *fmt, va_list ap);
 
-/**
- * Returns an exact copy of the error passed as an argument.
+/*
+ * Prepend some text to @errp's human-readable error message.
+ * The text is made by formatting @fmt, ... like printf().
  */
-Error *error_copy(const Error *err);
+void error_prepend(Error **errp, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
 
-/**
- * Get a human readable representation of an error object.
+/*
+ * Append a printf-style human-readable explanation to an existing error.
+ * @errp may be NULL, but not &error_fatal or &error_abort.
+ * Trivially the case if you call it only after error_setg() or
+ * error_propagate().
+ * May be called multiple times.  The resulting hint should end with a
+ * newline.
  */
-const char *error_get_pretty(Error *err);
+void error_append_hint(Error **errp, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
 
-/**
- * Convenience function to error_report() and free an error object.
+/*
+ * Convenience function to report open() failure.
  */
-void error_report_err(Error *);
+#define error_setg_file_open(errp, os_errno, filename)                  \
+    error_setg_file_open_internal((errp), __FILE__, __LINE__, __func__, \
+                                  (os_errno), (filename))
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename);
 
-/**
- * Propagate an error to an indirect pointer to an error.  This function will
- * always transfer ownership of the error reference and handles the case where
- * dst_err is NULL correctly.  Errors after the first are discarded.
+/*
+ * Return an exact copy of @err.
  */
-void error_propagate(Error **dst_errp, Error *local_err);
+Error *error_copy(const Error *err);
 
-/**
- * Free an error object.
+/*
+ * Free @err.
+ * @err may be NULL.
  */
 void error_free(Error *err);
 
-/**
- * If passed to error_set and friends, abort().
+/*
+ * Convenience function to assert that *@errp is set, then silently free it.
+ */
+void error_free_or_abort(Error **errp);
+
+/*
+ * Convenience function to error_report() and free @err.
  */
+void error_report_err(Error *err);
 
+/*
+ * Convenience function to error_prepend(), error_report() and free @err.
+ */
+void error_reportf_err(Error *err, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+/*
+ * Just like error_setg(), except you get to specify the error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
+ */
+#define error_set(errp, err_class, fmt, ...)                    \
+    error_set_internal((errp), __FILE__, __LINE__, __func__,    \
+                       (err_class), (fmt), ## __VA_ARGS__)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
+
+/*
+ * Special error destination to abort on error.
+ * See error_setg() and error_propagate() for details.
+ */
 extern Error *error_abort;
 
+/*
+ * Special error destination to exit(1) on error.
+ * See error_setg() and error_propagate() for details.
+ */
+extern Error *error_fatal;
+
 #endif
diff --git a/qemu/include/qapi/qmp-event.h b/qemu/include/qapi/qmp-event.h
index 8a8ffb571..40fe3cbc1 100644
--- a/qemu/include/qapi/qmp-event.h
+++ b/qemu/include/qapi/qmp-event.h
@@ -14,7 +14,6 @@
 #ifndef QMP_EVENT_H
 #define QMP_EVENT_H
 
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 typedef void (*QMPEventFuncEmit)(unsigned event, QDict *dict, Error **errp);
diff --git a/qemu/include/qapi/qmp/dispatch.h b/qemu/include/qapi/qmp/dispatch.h
index e389697f1..495520994 100644
--- a/qemu/include/qapi/qmp/dispatch.h
+++ b/qemu/include/qapi/qmp/dispatch.h
@@ -16,7 +16,6 @@
 
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qdict.h"
-#include "qapi/error.h"
 
 typedef void (QmpCommandFunc)(QDict *, QObject **, Error **);
 
diff --git a/qemu/include/qapi/qmp/json-lexer.h b/qemu/include/qapi/qmp/json-lexer.h
index cdff0460a..afee7828c 100644
--- a/qemu/include/qapi/qmp/json-lexer.h
+++ b/qemu/include/qapi/qmp/json-lexer.h
@@ -14,11 +14,15 @@
 #ifndef QEMU_JSON_LEXER_H
 #define QEMU_JSON_LEXER_H
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
 
 typedef enum json_token_type {
-    JSON_OPERATOR = 100,
+    JSON_MIN = 100,
+    JSON_LCURLY = JSON_MIN,
+    JSON_RCURLY,
+    JSON_LSQUARE,
+    JSON_RSQUARE,
+    JSON_COLON,
+    JSON_COMMA,
     JSON_INTEGER,
     JSON_FLOAT,
     JSON_KEYWORD,
@@ -30,13 +34,14 @@ typedef enum json_token_type {
 
 typedef struct JSONLexer JSONLexer;
 
-typedef void (JSONLexerEmitter)(JSONLexer *, QString *, JSONTokenType, int x, int y);
+typedef void (JSONLexerEmitter)(JSONLexer *, GString *,
+                                JSONTokenType, int x, int y);
 
 struct JSONLexer
 {
     JSONLexerEmitter *emit;
     int state;
-    QString *token;
+    GString *token;
     int x, y;
 };
 
diff --git a/qemu/include/qapi/qmp/json-parser.h b/qemu/include/qapi/qmp/json-parser.h
index 44d88f346..9987f8ca8 100644
--- a/qemu/include/qapi/qmp/json-parser.h
+++ b/qemu/include/qapi/qmp/json-parser.h
@@ -16,9 +16,8 @@
 
 #include "qemu-common.h"
 #include "qapi/qmp/qlist.h"
-#include "qapi/error.h"
 
-QObject *json_parser_parse(QList *tokens, va_list *ap);
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
+QObject *json_parser_parse(GQueue *tokens, va_list *ap);
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
 
 #endif
diff --git a/qemu/include/qapi/qmp/json-streamer.h b/qemu/include/qapi/qmp/json-streamer.h
index 823f7d7fa..00d8a23af 100644
--- a/qemu/include/qapi/qmp/json-streamer.h
+++ b/qemu/include/qapi/qmp/json-streamer.h
@@ -14,21 +14,27 @@
 #ifndef QEMU_JSON_STREAMER_H
 #define QEMU_JSON_STREAMER_H
 
-#include "qapi/qmp/qlist.h"
 #include "qapi/qmp/json-lexer.h"
 
+typedef struct JSONToken {
+    int type;
+    int x;
+    int y;
+    char str[];
+} JSONToken;
+
 typedef struct JSONMessageParser
 {
-    void (*emit)(struct JSONMessageParser *parser, QList *tokens);
+    void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
     JSONLexer lexer;
     int brace_count;
     int bracket_count;
-    QList *tokens;
+    GQueue *tokens;
     uint64_t token_size;
 } JSONMessageParser;
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *));
+                              void (*func)(JSONMessageParser *, GQueue *));
 
 int json_message_parser_feed(JSONMessageParser *parser,
                              const char *buffer, size_t size);
diff --git a/qemu/include/qapi/qmp/qbool.h b/qemu/include/qapi/qmp/qbool.h
index 4aa6be3b3..a41111c30 100644
--- a/qemu/include/qapi/qmp/qbool.h
+++ b/qemu/include/qapi/qmp/qbool.h
@@ -14,16 +14,16 @@
 #ifndef QBOOL_H
 #define QBOOL_H
 
-#include <stdbool.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QBool {
-    QObject_HEAD;
+    QObject base;
     bool value;
 } QBool;
 
 QBool *qbool_from_bool(bool value);
 bool qbool_get_bool(const QBool *qb);
 QBool *qobject_to_qbool(const QObject *obj);
+void qbool_destroy_obj(QObject *obj);
 
 #endif /* QBOOL_H */
diff --git a/qemu/include/qapi/qmp/qdict.h b/qemu/include/qapi/qmp/qdict.h
index a37f4c156..71b8eb041 100644
--- a/qemu/include/qapi/qmp/qdict.h
+++ b/qemu/include/qapi/qmp/qdict.h
@@ -16,8 +16,6 @@
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qlist.h"
 #include "qemu/queue.h"
-#include <stdbool.h>
-#include <stdint.h>
 
 #define QDICT_BUCKET_MAX 512
 
@@ -28,7 +26,7 @@ typedef struct QDictEntry {
 } QDictEntry;
 
 typedef struct QDict {
-    QObject_HEAD;
+    QObject base;
     size_t size;
     QLIST_HEAD(,QDictEntry) table[QDICT_BUCKET_MAX];
 } QDict;
@@ -48,6 +46,7 @@ void qdict_iter(const QDict *qdict,
                 void *opaque);
 const QDictEntry *qdict_first(const QDict *qdict);
 const QDictEntry *qdict_next(const QDict *qdict, const QDictEntry *entry);
+void qdict_destroy_obj(QObject *obj);
 
 /* Helper to qdict_put_obj(), accepts any object */
 #define qdict_put(qdict, key, obj) \
diff --git a/qemu/include/qapi/qmp/qerror.h b/qemu/include/qapi/qmp/qerror.h
index 842b27ae1..d08652aaa 100644
--- a/qemu/include/qapi/qmp/qerror.h
+++ b/qemu/include/qapi/qmp/qerror.h
@@ -100,10 +100,10 @@
 #define QERR_UNDEFINED_ERROR \
     "An undefined error has occurred"
 
-#define QERR_UNKNOWN_BLOCK_FORMAT_FEATURE \
-    "'%s' uses a %s feature which is not supported by this qemu version: %s"
-
 #define QERR_UNSUPPORTED \
     "this feature or command is not currently supported"
 
+#define QERR_REPLAY_NOT_SUPPORTED \
+    "Record/replay feature is not supported for '%s'"
+
 #endif /* QERROR_H */
diff --git a/qemu/include/qapi/qmp/qfloat.h b/qemu/include/qapi/qmp/qfloat.h
index a8658443d..b5d15836b 100644
--- a/qemu/include/qapi/qmp/qfloat.h
+++ b/qemu/include/qapi/qmp/qfloat.h
@@ -14,16 +14,16 @@
 #ifndef QFLOAT_H
 #define QFLOAT_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QFloat {
-    QObject_HEAD;
+    QObject base;
     double value;
 } QFloat;
 
 QFloat *qfloat_from_double(double value);
 double qfloat_get_double(const QFloat *qi);
 QFloat *qobject_to_qfloat(const QObject *obj);
+void qfloat_destroy_obj(QObject *obj);
 
 #endif /* QFLOAT_H */
diff --git a/qemu/include/qapi/qmp/qint.h b/qemu/include/qapi/qmp/qint.h
index 48a41b0f2..3aaff768d 100644
--- a/qemu/include/qapi/qmp/qint.h
+++ b/qemu/include/qapi/qmp/qint.h
@@ -13,16 +13,16 @@
 #ifndef QINT_H
 #define QINT_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QInt {
-    QObject_HEAD;
+    QObject base;
     int64_t value;
 } QInt;
 
 QInt *qint_from_int(int64_t value);
 int64_t qint_get_int(const QInt *qi);
 QInt *qobject_to_qint(const QObject *obj);
+void qint_destroy_obj(QObject *obj);
 
 #endif /* QINT_H */
diff --git a/qemu/include/qapi/qmp/qjson.h b/qemu/include/qapi/qmp/qjson.h
index ee4d31a46..02b1f2ce3 100644
--- a/qemu/include/qapi/qmp/qjson.h
+++ b/qemu/include/qapi/qmp/qjson.h
@@ -14,8 +14,6 @@
 #ifndef QJSON_H
 #define QJSON_H
 
-#include <stdarg.h>
-#include "qemu/compiler.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qstring.h"
 
diff --git a/qemu/include/qapi/qmp/qlist.h b/qemu/include/qapi/qmp/qlist.h
index 6cc4831df..a84117ecb 100644
--- a/qemu/include/qapi/qmp/qlist.h
+++ b/qemu/include/qapi/qmp/qlist.h
@@ -22,7 +22,7 @@ typedef struct QListEntry {
 } QListEntry;
 
 typedef struct QList {
-    QObject_HEAD;
+    QObject base;
     QTAILQ_HEAD(,QListEntry) head;
 } QList;
 
@@ -49,6 +49,7 @@ QObject *qlist_peek(QList *qlist);
 int qlist_empty(const QList *qlist);
 size_t qlist_size(const QList *qlist);
 QList *qobject_to_qlist(const QObject *obj);
+void qlist_destroy_obj(QObject *obj);
 
 static inline const QListEntry *qlist_first(const QList *qlist)
 {
diff --git a/qemu/include/qapi/qmp/qobject.h b/qemu/include/qapi/qmp/qobject.h
index 260d2ed3c..b8ddbca40 100644
--- a/qemu/include/qapi/qmp/qobject.h
+++ b/qemu/include/qapi/qmp/qobject.h
@@ -32,36 +32,12 @@
 #ifndef QOBJECT_H
 #define QOBJECT_H
 
-#include <stddef.h>
-#include <assert.h>
+#include "qapi-types.h"
 
-typedef enum {
-    QTYPE_NONE,    /* sentinel value, no QObject has this type code */
-    QTYPE_QNULL,
-    QTYPE_QINT,
-    QTYPE_QSTRING,
-    QTYPE_QDICT,
-    QTYPE_QLIST,
-    QTYPE_QFLOAT,
-    QTYPE_QBOOL,
-    QTYPE_MAX,
-} qtype_code;
-
-struct QObject;
-
-typedef struct QType {
-    qtype_code code;
-    void (*destroy)(struct QObject *);
-} QType;
-
-typedef struct QObject {
-    const QType *type;
+struct QObject {
+    QType type;
     size_t refcnt;
-} QObject;
-
-/* Objects definitions must include this */
-#define QObject_HEAD  \
-    QObject base
+};
 
 /* Get the 'base' part of an object */
 #define QOBJECT(obj) (&(obj)->base)
@@ -75,9 +51,12 @@ typedef struct QObject {
     qobject_decref(obj ? QOBJECT(obj) : NULL)
 
 /* Initialize an object to default values */
-#define QOBJECT_INIT(obj, qtype_type)   \
-    obj->base.refcnt = 1;               \
-    obj->base.type   = qtype_type
+static inline void qobject_init(QObject *obj, QType type)
+{
+    assert(QTYPE_NONE < type && type < QTYPE__MAX);
+    obj->refcnt = 1;
+    obj->type = type;
+}
 
 /**
  * qobject_incref(): Increment QObject's reference count
@@ -89,25 +68,29 @@ static inline void qobject_incref(QObject *obj)
 }
 
 /**
+ * qobject_destroy(): Free resources used by the object
+ */
+void qobject_destroy(QObject *obj);
+
+/**
  * qobject_decref(): Decrement QObject's reference count, deallocate
  * when it reaches zero
  */
 static inline void qobject_decref(QObject *obj)
 {
+    assert(!obj || obj->refcnt);
     if (obj && --obj->refcnt == 0) {
-        assert(obj->type != NULL);
-        assert(obj->type->destroy != NULL);
-        obj->type->destroy(obj);
+        qobject_destroy(obj);
     }
 }
 
 /**
  * qobject_type(): Return the QObject's type
  */
-static inline qtype_code qobject_type(const QObject *obj)
+static inline QType qobject_type(const QObject *obj)
 {
-    assert(obj->type != NULL);
-    return obj->type->code;
+    assert(QTYPE_NONE < obj->type && obj->type < QTYPE__MAX);
+    return obj->type;
 }
 
 extern QObject qnull_;
diff --git a/qemu/include/qapi/qmp/qstring.h b/qemu/include/qapi/qmp/qstring.h
index 1bc366610..10076b7c8 100644
--- a/qemu/include/qapi/qmp/qstring.h
+++ b/qemu/include/qapi/qmp/qstring.h
@@ -13,11 +13,10 @@
 #ifndef QSTRING_H
 #define QSTRING_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QString {
-    QObject_HEAD;
+    QObject base;
     char *string;
     size_t length;
     size_t capacity;
@@ -32,5 +31,6 @@ void qstring_append_int(QString *qstring, int64_t value);
 void qstring_append(QString *qstring, const char *str);
 void qstring_append_chr(QString *qstring, int c);
 QString *qobject_to_qstring(const QObject *obj);
+void qstring_destroy_obj(QObject *obj);
 
 #endif /* QSTRING_H */
diff --git a/qemu/include/qapi/visitor-impl.h b/qemu/include/qapi/visitor-impl.h
index f4a2f746c..2bd8f292b 100644
--- a/qemu/include/qapi/visitor-impl.h
+++ b/qemu/include/qapi/visitor-impl.h
@@ -1,7 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor implementations
  *
- * Copyright (C) 2012 Red Hat, Inc.
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  *
  * Author: Paolo Bonizni <pbonzini@redhat.com>
  *
@@ -12,56 +12,57 @@
 #ifndef QAPI_VISITOR_IMPL_H
 #define QAPI_VISITOR_IMPL_H
 
-#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 struct Visitor
 {
     /* Must be set */
-    void (*start_struct)(Visitor *v, void **obj, const char *kind,
-                         const char *name, size_t size, Error **errp);
+    void (*start_struct)(Visitor *v, const char *name, void **obj,
+                         size_t size, Error **errp);
     void (*end_struct)(Visitor *v, Error **errp);
 
-    void (*start_implicit_struct)(Visitor *v, void **obj, size_t size,
-                                  Error **errp);
-    void (*end_implicit_struct)(Visitor *v, Error **errp);
-
     void (*start_list)(Visitor *v, const char *name, Error **errp);
-    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
-    void (*end_list)(Visitor *v, Error **errp);
+    /* Must be set */
+    GenericList *(*next_list)(Visitor *v, GenericList **list, size_t size);
+    /* Must be set */
+    void (*end_list)(Visitor *v);
 
-    void (*type_enum)(Visitor *v, int *obj, const char * const strings[],
-                      const char *kind, const char *name, Error **errp);
-    void (*get_next_type)(Visitor *v, int *kind, const int *qobjects,
-                          const char *name, Error **errp);
+    /* Optional, needed for input and dealloc visitors.  */
+    void (*start_alternate)(Visitor *v, const char *name,
+                            GenericAlternate **obj, size_t size,
+                            bool promote_int, Error **errp);
 
-    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
-    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
-    void (*type_number)(Visitor *v, double *obj, const char *name,
-                        Error **errp);
+    /* Optional, needed for dealloc visitor.  */
+    void (*end_alternate)(Visitor *v);
 
-    /* May be NULL */
-    void (*optional)(Visitor *v, bool *present, const char *name,
+    /* Must be set. */
+    void (*type_enum)(Visitor *v, const char *name, int *obj,
+                      const char *const strings[], Error **errp);
+
+    /* Must be set. */
+    void (*type_int64)(Visitor *v, const char *name, int64_t *obj,
+                       Error **errp);
+    /* Must be set. */
+    void (*type_uint64)(Visitor *v, const char *name, uint64_t *obj,
+                        Error **errp);
+    /* Optional; fallback is type_uint64().  */
+    void (*type_size)(Visitor *v, const char *name, uint64_t *obj,
+                      Error **errp);
+    /* Must be set. */
+    void (*type_bool)(Visitor *v, const char *name, bool *obj, Error **errp);
+    void (*type_str)(Visitor *v, const char *name, char **obj, Error **errp);
+    void (*type_number)(Visitor *v, const char *name, double *obj,
+                        Error **errp);
+    void (*type_any)(Visitor *v, const char *name, QObject **obj,
                      Error **errp);
 
-    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
-    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
-    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
-    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
-    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    bool (*start_union)(Visitor *v, bool data_present, Error **errp);
-    void (*end_union)(Visitor *v, bool data_present, Error **errp);
+    /* May be NULL; most useful for input visitors. */
+    void (*optional)(Visitor *v, const char *name, bool *present);
 };
 
-void input_type_enum(Visitor *v, int *obj, const char * const strings[],
-                     const char *kind, const char *name, Error **errp);
-void output_type_enum(Visitor *v, int *obj, const char * const strings[],
-                      const char *kind, const char *name, Error **errp);
+void input_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp);
+void output_type_enum(Visitor *v, const char *name, int *obj,
+                      const char *const strings[], Error **errp);
 
 #endif
diff --git a/qemu/include/qapi/visitor.h b/qemu/include/qapi/visitor.h
index 00ba104cd..9a8d0105f 100644
--- a/qemu/include/qapi/visitor.h
+++ b/qemu/include/qapi/visitor.h
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor Classes
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -13,52 +14,94 @@
 #ifndef QAPI_VISITOR_CORE_H
 #define QAPI_VISITOR_CORE_H
 
-#include "qemu/typedefs.h"
 #include "qapi/qmp/qobject.h"
-#include "qapi/error.h"
-#include <stdlib.h>
 
-typedef struct GenericList
-{
-    union {
-        void *value;
-        uint64_t padding;
-    };
+/* This struct is layout-compatible with all other *List structs
+ * created by the qapi generator.  It is used as a typical
+ * singly-linked list. */
+typedef struct GenericList {
     struct GenericList *next;
+    char padding[];
 } GenericList;
 
-void visit_start_handle(Visitor *v, void **obj, const char *kind,
-                        const char *name, Error **errp);
-void visit_end_handle(Visitor *v, Error **errp);
-void visit_start_struct(Visitor *v, void **obj, const char *kind,
-                        const char *name, size_t size, Error **errp);
+/* This struct is layout-compatible with all Alternate types
+ * created by the qapi generator. */
+typedef struct GenericAlternate {
+    QType type;
+    char padding[];
+} GenericAlternate;
+
+void visit_start_struct(Visitor *v, const char *name, void **obj,
+                        size_t size, Error **errp);
 void visit_end_struct(Visitor *v, Error **errp);
-void visit_start_implicit_struct(Visitor *v, void **obj, size_t size,
-                                 Error **errp);
-void visit_end_implicit_struct(Visitor *v, Error **errp);
+
 void visit_start_list(Visitor *v, const char *name, Error **errp);
-GenericList *visit_next_list(Visitor *v, GenericList **list, Error **errp);
-void visit_end_list(Visitor *v, Error **errp);
-void visit_optional(Visitor *v, bool *present, const char *name,
-                    Error **errp);
-void visit_get_next_type(Visitor *v, int *obj, const int *qtypes,
-                         const char *name, Error **errp);
-void visit_type_enum(Visitor *v, int *obj, const char * const strings[],
-                     const char *kind, const char *name, Error **errp);
-void visit_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp);
-void visit_type_uint8(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-void visit_type_uint16(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-void visit_type_uint32(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-void visit_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-void visit_type_int8(Visitor *v, int8_t *obj, const char *name, Error **errp);
-void visit_type_int16(Visitor *v, int16_t *obj, const char *name, Error **errp);
-void visit_type_int32(Visitor *v, int32_t *obj, const char *name, Error **errp);
-void visit_type_int64(Visitor *v, int64_t *obj, const char *name, Error **errp);
-void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp);
-void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp);
-void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp);
-bool visit_start_union(Visitor *v, bool data_present, Error **errp);
-void visit_end_union(Visitor *v, bool data_present, Error **errp);
+GenericList *visit_next_list(Visitor *v, GenericList **list, size_t size);
+void visit_end_list(Visitor *v);
+
+/*
+ * Start the visit of an alternate @obj with the given @size.
+ *
+ * @name specifies the relationship to the containing struct (ignored
+ * for a top level visit, the name of the key if this alternate is
+ * part of an object, or NULL if this alternate is part of a list).
+ *
+ * @obj must not be NULL. Input visitors will allocate @obj and
+ * determine the qtype of the next thing to be visited, stored in
+ * (*@obj)->type.  Other visitors will leave @obj unchanged.
+ *
+ * If @promote_int, treat integers as QTYPE_FLOAT.
+ *
+ * If successful, this must be paired with visit_end_alternate(), even
+ * if visiting the contents of the alternate fails.
+ */
+void visit_start_alternate(Visitor *v, const char *name,
+                           GenericAlternate **obj, size_t size,
+                           bool promote_int, Error **errp);
+
+/*
+ * Finish visiting an alternate type.
+ *
+ * Must be called after a successful visit_start_alternate(), even if
+ * an error occurred in the meantime.
+ *
+ * TODO: Should all the visit_end_* interfaces take obj parameter, so
+ * that dealloc visitor need not track what was passed in visit_start?
+ */
+void visit_end_alternate(Visitor *v);
+
+/**
+ * Check if an optional member @name of an object needs visiting.
+ * For input visitors, set *@present according to whether the
+ * corresponding visit_type_*() needs calling; for other visitors,
+ * leave *@present unchanged.  Return *@present for convenience.
+ */
+bool visit_optional(Visitor *v, const char *name, bool *present);
+
+void visit_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp);
+void visit_type_int(Visitor *v, const char *name, int64_t *obj, Error **errp);
+void visit_type_uint8(Visitor *v, const char *name, uint8_t *obj,
+                      Error **errp);
+void visit_type_uint16(Visitor *v, const char *name, uint16_t *obj,
+                       Error **errp);
+void visit_type_uint32(Visitor *v, const char *name, uint32_t *obj,
+                       Error **errp);
+void visit_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                       Error **errp);
+void visit_type_int8(Visitor *v, const char *name, int8_t *obj, Error **errp);
+void visit_type_int16(Visitor *v, const char *name, int16_t *obj,
+                      Error **errp);
+void visit_type_int32(Visitor *v, const char *name, int32_t *obj,
+                      Error **errp);
+void visit_type_int64(Visitor *v, const char *name, int64_t *obj,
+                      Error **errp);
+void visit_type_size(Visitor *v, const char *name, uint64_t *obj,
+                     Error **errp);
+void visit_type_bool(Visitor *v, const char *name, bool *obj, Error **errp);
+void visit_type_str(Visitor *v, const char *name, char **obj, Error **errp);
+void visit_type_number(Visitor *v, const char *name, double *obj,
+                       Error **errp);
+void visit_type_any(Visitor *v, const char *name, QObject **obj, Error **errp);
 
 #endif
diff --git a/qemu/include/qemu-common.h b/qemu/include/qemu-common.h
index fb3da6ca2..163bcbb86 100644
--- a/qemu/include/qemu-common.h
+++ b/qemu/include/qemu-common.h
@@ -12,9 +12,6 @@
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
-#include "qemu/compiler.h"
-#include "config-host.h"
-#include "qemu/typedefs.h"
 #include "qemu/fprintf-fn.h"
 
 #if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
@@ -23,85 +20,8 @@
 
 #define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
 
-/* we put basic includes here to avoid repeating them in device drivers */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <string.h>
-#include <strings.h>
-#include <inttypes.h>
-#include <limits.h>
-#include <time.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <assert.h>
-#include <signal.h>
-#include "glib-compat.h"
 #include "qemu/option.h"
-
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
-
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
-#ifndef O_LARGEFILE
-#define O_LARGEFILE 0
-#endif
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-#ifndef ENOMEDIUM
-#define ENOMEDIUM ENODEV
-#endif
-#if !defined(ENOTSUP)
-#define ENOTSUP 4096
-#endif
-#if !defined(ECANCELED)
-#define ECANCELED 4097
-#endif
-#if !defined(EMEDIUMTYPE)
-#define EMEDIUMTYPE 4098
-#endif
-#ifndef TIME_MAX
-#define TIME_MAX LONG_MAX
-#endif
-
-/* HOST_LONG_BITS is the size of a native pointer in bits. */
-#if UINTPTR_MAX == UINT32_MAX
-# define HOST_LONG_BITS 32
-#elif UINTPTR_MAX == UINT64_MAX
-# define HOST_LONG_BITS 64
-#else
-# error Unknown pointer size
-#endif
-
-#ifdef _WIN32
-#define fsync _commit
-#if !defined(lseek)
-# define lseek _lseeki64
-#endif
-int qemu_ftruncate64(int, int64_t);
-#if !defined(ftruncate)
-# define ftruncate qemu_ftruncate64
-#endif
-
-static inline char *realpath(const char *path, char *resolved_path)
-{
-    _fullpath(resolved_path, path, _MAX_PATH);
-    return resolved_path;
-}
-#endif
+#include "qemu/host-utils.h"
 
 void cpu_ticks_init(void);
 
@@ -114,7 +34,6 @@ extern int64_t max_delay;
 extern int64_t max_advance;
 void dump_drift_info(FILE *f, fprintf_function cpu_fprintf);
 
-#include "qemu/osdep.h"
 #include "qemu/bswap.h"
 
 /* FIXME: Remove NEED_CPU_H.  */
@@ -130,189 +49,6 @@ int qemu_main(int argc, char **argv, char **envp);
 void qemu_get_timedate(struct tm *tm, int offset);
 int qemu_timedate_diff(struct tm *tm);
 
-/**
- * is_help_option:
- * @s: string to test
- *
- * Check whether @s is one of the standard strings which indicate
- * that the user is asking for a list of the valid values for a
- * command option like -cpu or -M. The current accepted strings
- * are 'help' and '?'. '?' is deprecated (it is a shell wildcard
- * which makes it annoying to use in a reliable way) but provided
- * for backwards compatibility.
- *
- * Returns: true if @s is a request for a list.
- */
-static inline bool is_help_option(const char *s)
-{
-    return !strcmp(s, "?") || !strcmp(s, "help");
-}
-
-/* util/cutils.c */
-/**
- * pstrcpy:
- * @buf: buffer to copy string into
- * @buf_size: size of @buf in bytes
- * @str: string to copy
- *
- * Copy @str into @buf, including the trailing NUL, but do not
- * write more than @buf_size bytes. The resulting buffer is
- * always NUL terminated (even if the source string was too long).
- * If @buf_size is zero or negative then no bytes are copied.
- *
- * This function is similar to strncpy(), but avoids two of that
- * function's problems:
- *  * if @str fits in the buffer, pstrcpy() does not zero-fill the
- *    remaining space at the end of @buf
- *  * if @str is too long, pstrcpy() will copy the first @buf_size-1
- *    bytes and then add a NUL
- */
-void pstrcpy(char *buf, int buf_size, const char *str);
-/**
- * strpadcpy:
- * @buf: buffer to copy string into
- * @buf_size: size of @buf in bytes
- * @str: string to copy
- * @pad: character to pad the remainder of @buf with
- *
- * Copy @str into @buf (but *not* its trailing NUL!), and then pad the
- * rest of the buffer with the @pad character. If @str is too large
- * for the buffer then it is truncated, so that @buf contains the
- * first @buf_size characters of @str, with no terminator.
- */
-void strpadcpy(char *buf, int buf_size, const char *str, char pad);
-/**
- * pstrcat:
- * @buf: buffer containing existing string
- * @buf_size: size of @buf in bytes
- * @s: string to concatenate to @buf
- *
- * Append a copy of @s to the string already in @buf, but do not
- * allow the buffer to overflow. If the existing contents of @buf
- * plus @str would total more than @buf_size bytes, then write
- * as much of @str as will fit followed by a NUL terminator.
- *
- * @buf must already contain a NUL-terminated string, or the
- * behaviour is undefined.
- *
- * Returns: @buf.
- */
-char *pstrcat(char *buf, int buf_size, const char *s);
-/**
- * strstart:
- * @str: string to test
- * @val: prefix string to look for
- * @ptr: NULL, or pointer to be written to indicate start of
- *       the remainder of the string
- *
- * Test whether @str starts with the prefix @val.
- * If it does (including the degenerate case where @str and @val
- * are equal) then return true. If @ptr is not NULL then a
- * pointer to the first character following the prefix is written
- * to it. If @val is not a prefix of @str then return false (and
- * @ptr is not written to).
- *
- * Returns: true if @str starts with prefix @val, false otherwise.
- */
-int strstart(const char *str, const char *val, const char **ptr);
-/**
- * stristart:
- * @str: string to test
- * @val: prefix string to look for
- * @ptr: NULL, or pointer to be written to indicate start of
- *       the remainder of the string
- *
- * Test whether @str starts with the case-insensitive prefix @val.
- * This function behaves identically to strstart(), except that the
- * comparison is made after calling qemu_toupper() on each pair of
- * characters.
- *
- * Returns: true if @str starts with case-insensitive prefix @val,
- *          false otherwise.
- */
-int stristart(const char *str, const char *val, const char **ptr);
-/**
- * qemu_strnlen:
- * @s: string
- * @max_len: maximum number of bytes in @s to scan
- *
- * Return the length of the string @s, like strlen(), but do not
- * examine more than @max_len bytes of the memory pointed to by @s.
- * If no NUL terminator is found within @max_len bytes, then return
- * @max_len instead.
- *
- * This function has the same behaviour as the POSIX strnlen()
- * function.
- *
- * Returns: length of @s in bytes, or @max_len, whichever is smaller.
- */
-int qemu_strnlen(const char *s, int max_len);
-/**
- * qemu_strsep:
- * @input: pointer to string to parse
- * @delim: string containing delimiter characters to search for
- *
- * Locate the first occurrence of any character in @delim within
- * the string referenced by @input, and replace it with a NUL.
- * The location of the next character after the delimiter character
- * is stored into @input.
- * If the end of the string was reached without finding a delimiter
- * character, then NULL is stored into @input.
- * If @input points to a NULL pointer on entry, return NULL.
- * The return value is always the original value of *@input (and
- * so now points to a NUL-terminated string corresponding to the
- * part of the input up to the first delimiter).
- *
- * This function has the same behaviour as the BSD strsep() function.
- *
- * Returns: the pointer originally in @input.
- */
-char *qemu_strsep(char **input, const char *delim);
-time_t mktimegm(struct tm *tm);
-int qemu_fls(int i);
-int qemu_fdatasync(int fd);
-int fcntl_setfl(int fd, int flag);
-int qemu_parse_fd(const char *param);
-
-int parse_uint(const char *s, unsigned long long *value, char **endptr,
-               int base);
-int parse_uint_full(const char *s, unsigned long long *value, int base);
-
-/*
- * strtosz() suffixes used to specify the default treatment of an
- * argument passed to strtosz() without an explicit suffix.
- * These should be defined using upper case characters in the range
- * A-Z, as strtosz() will use qemu_toupper() on the given argument
- * prior to comparison.
- */
-#define STRTOSZ_DEFSUFFIX_EB	'E'
-#define STRTOSZ_DEFSUFFIX_PB	'P'
-#define STRTOSZ_DEFSUFFIX_TB	'T'
-#define STRTOSZ_DEFSUFFIX_GB	'G'
-#define STRTOSZ_DEFSUFFIX_MB	'M'
-#define STRTOSZ_DEFSUFFIX_KB	'K'
-#define STRTOSZ_DEFSUFFIX_B	'B'
-int64_t strtosz(const char *nptr, char **end);
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix);
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
-                            const char default_suffix, int64_t unit);
-#define K_BYTE     (1ULL << 10)
-#define M_BYTE     (1ULL << 20)
-#define G_BYTE     (1ULL << 30)
-#define T_BYTE     (1ULL << 40)
-#define P_BYTE     (1ULL << 50)
-#define E_BYTE     (1ULL << 60)
-
-/* used to print char* safely */
-#define STR_OR_NULL(str) ((str) ? (str) : "null")
-
-/* id.c */
-bool id_wellformed(const char *id);
-
-/* path.c */
-void init_paths(const char *prefix);
-const char *path(const char *pathname);
-
 #define qemu_isalnum(c)		isalnum((unsigned char)(c))
 #define qemu_isalpha(c)		isalpha((unsigned char)(c))
 #define qemu_iscntrl(c)		iscntrl((unsigned char)(c))
@@ -359,37 +95,11 @@ int qemu_openpty_raw(int *aslave, char *pty_name);
     sendto(sockfd, buf, len, flags, destaddr, addrlen)
 #endif
 
-/* Error handling.  */
-
-void QEMU_NORETURN hw_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
-
-struct ParallelIOArg {
-    void *buffer;
-    int count;
-};
-
-typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size);
-
-typedef uint64_t pcibus_t;
-
-typedef struct PCIHostDeviceAddress {
-    unsigned int domain;
-    unsigned int bus;
-    unsigned int slot;
-    unsigned int function;
-} PCIHostDeviceAddress;
-
 void tcg_exec_init(unsigned long tb_size);
 bool tcg_enabled(void);
 
 void cpu_exec_init_all(void);
 
-/* CPU save/load.  */
-#ifdef CPU_SAVE_VERSION
-void cpu_save(QEMUFile *f, void *opaque);
-int cpu_load(QEMUFile *f, void *opaque, int version_id);
-#endif
-
 /* Unblock cpu */
 void qemu_cpu_kick_self(void);
 
@@ -426,36 +136,6 @@ ssize_t qemu_co_send_recv(int sockfd, void *buf, size_t bytes, bool do_send);
 #define qemu_co_send(sockfd, buf, bytes) \
   qemu_co_send_recv(sockfd, buf, bytes, true)
 
-typedef struct QEMUIOVector {
-    struct iovec *iov;
-    int niov;
-    int nalloc;
-    size_t size;
-} QEMUIOVector;
-
-void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
-void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
-void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
-void qemu_iovec_concat(QEMUIOVector *dst,
-                       QEMUIOVector *src, size_t soffset, size_t sbytes);
-size_t qemu_iovec_concat_iov(QEMUIOVector *dst,
-                             struct iovec *src_iov, unsigned int src_cnt,
-                             size_t soffset, size_t sbytes);
-bool qemu_iovec_is_zero(QEMUIOVector *qiov);
-void qemu_iovec_destroy(QEMUIOVector *qiov);
-void qemu_iovec_reset(QEMUIOVector *qiov);
-size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
-                         void *buf, size_t bytes);
-size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
-                           const void *buf, size_t bytes);
-size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
-                         int fillc, size_t bytes);
-ssize_t qemu_iovec_compare(QEMUIOVector *a, QEMUIOVector *b);
-void qemu_iovec_clone(QEMUIOVector *dest, const QEMUIOVector *src, void *buf);
-void qemu_iovec_discard_back(QEMUIOVector *qiov, size_t bytes);
-
-bool buffer_is_zero(const void *buf, size_t len);
-
 void qemu_progress_init(int enabled, float min_skip);
 void qemu_progress_end(void);
 void qemu_progress_print(float delta, int max);
@@ -470,133 +150,24 @@ void os_setup_early_signal_handling(void);
 char *os_find_datadir(void);
 void os_parse_cmd_args(int index, const char *optarg);
 
-/* Convert a byte between binary and BCD.  */
-static inline uint8_t to_bcd(uint8_t val)
-{
-    return ((val / 10) << 4) | (val % 10);
-}
-
-static inline uint8_t from_bcd(uint8_t val)
-{
-    return ((val >> 4) * 10) + (val & 0x0f);
-}
-
-/* compute with 96 bit intermediate result: (a*b)/c */
-#ifdef CONFIG_INT128
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    return (__int128_t)a * b / c;
-}
-#else
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    union {
-        uint64_t ll;
-        struct {
-#ifdef HOST_WORDS_BIGENDIAN
-            uint32_t high, low;
-#else
-            uint32_t low, high;
-#endif
-        } l;
-    } u, res;
-    uint64_t rl, rh;
-
-    u.ll = a;
-    rl = (uint64_t)u.l.low * (uint64_t)b;
-    rh = (uint64_t)u.l.high * (uint64_t)b;
-    rh += (rl >> 32);
-    res.l.high = rh / c;
-    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
-    return res.ll;
-}
-#endif
-
-/* Round number down to multiple */
-#define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
-
-/* Round number up to multiple */
-#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
-
-static inline bool is_power_of_2(uint64_t value)
-{
-    if (!value) {
-        return 0;
-    }
-
-    return !(value & (value - 1));
-}
-
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value);
-
-/* round up to the nearest power of 2 (0 if overflow) */
-uint64_t pow2ceil(uint64_t value);
-
 #include "qemu/module.h"
 
 /*
- * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
- * Input is limited to 14-bit numbers
- */
-
-int uleb128_encode_small(uint8_t *out, uint32_t n);
-int uleb128_decode_small(const uint8_t *in, uint32_t *n);
-
-/* unicode.c */
-int mod_utf8_codepoint(const char *s, size_t n, char **end);
-
-/*
  * Hexdump a buffer to a file. An optional string prefix is added to every line
  */
 
 void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size);
 
-/* vector definitions */
-#ifdef __ALTIVEC__
-#include <altivec.h>
-/* The altivec.h header says we're allowed to undef these for
- * C++ compatibility.  Here we don't care about C++, but we
- * undef them anyway to avoid namespace pollution.
- */
-#undef vector
-#undef pixel
-#undef bool
-#define VECTYPE        __vector unsigned char
-#define SPLAT(p)       vec_splat(vec_ld(0, p), 0)
-#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
-#define VEC_OR(v1, v2) ((v1) | (v2))
-/* altivec.h may redefine the bool macro as vector type.
- * Reset it to POSIX semantics. */
-#define bool _Bool
-#elif defined __SSE2__
-#include <emmintrin.h>
-#define VECTYPE        __m128i
-#define SPLAT(p)       _mm_set1_epi8(*(p))
-#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
-#define VEC_OR(v1, v2) (_mm_or_si128(v1, v2))
-#else
-#define VECTYPE        unsigned long
-#define SPLAT(p)       (*(p) * (~0UL / 255))
-#define ALL_EQ(v1, v2) ((v1) == (v2))
-#define VEC_OR(v1, v2) ((v1) | (v2))
-#endif
-
-#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
-static inline bool
-can_use_buffer_find_nonzero_offset(const void *buf, size_t len)
-{
-    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
-                   * sizeof(VECTYPE)) == 0
-            && ((uintptr_t) buf) % sizeof(VECTYPE) == 0);
-}
-size_t buffer_find_nonzero_offset(const void *buf, size_t len);
-
 /*
  * helper to parse debug environment variables
  */
 int parse_debug_env(const char *name, int max, int initial);
 
 const char *qemu_ether_ntoa(const MACAddr *mac);
+void page_size_init(void);
+
+/* returns non-zero if dump is in progress, otherwise zero is
+ * returned. */
+bool dump_in_progress(void);
 
 #endif
diff --git a/qemu/include/qemu/atomic.h b/qemu/include/qemu/atomic.h
index bd2c07534..5bc4d6cc4 100644
--- a/qemu/include/qemu/atomic.h
+++ b/qemu/include/qemu/atomic.h
@@ -8,19 +8,148 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
+ * See docs/atomics.txt for discussion about the guarantees each
+ * atomic primitive is meant to provide.
  */
 
 #ifndef __QEMU_ATOMIC_H
 #define __QEMU_ATOMIC_H 1
 
-#include "qemu/compiler.h"
 
-/* For C11 atomic ops */
 
 /* Compiler barrier */
 #define barrier()   ({ asm volatile("" ::: "memory"); (void)0; })
 
-#ifndef __ATOMIC_RELAXED
+#ifdef __ATOMIC_RELAXED
+/* For C11 atomic ops */
+
+/* Manual memory barriers
+ *
+ *__atomic_thread_fence does not include a compiler barrier; instead,
+ * the barrier is part of __atomic_load/__atomic_store's "volatile-like"
+ * semantics. If smp_wmb() is a no-op, absence of the barrier means that
+ * the compiler is free to reorder stores on each side of the barrier.
+ * Add one here, and similarly in smp_rmb() and smp_read_barrier_depends().
+ */
+
+#define smp_mb()    ({ barrier(); __atomic_thread_fence(__ATOMIC_SEQ_CST); barrier(); })
+#define smp_wmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_RELEASE); barrier(); })
+#define smp_rmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_ACQUIRE); barrier(); })
+
+#define smp_read_barrier_depends() ({ barrier(); __atomic_thread_fence(__ATOMIC_CONSUME); barrier(); })
+
+/* Weak atomic operations prevent the compiler moving other
+ * loads/stores past the atomic operation load/store. However there is
+ * no explicit memory barrier for the processor.
+ */
+#define atomic_read(ptr)                              \
+    ({                                                \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val;                                \
+     __atomic_load(ptr, &_val, __ATOMIC_RELAXED);     \
+    _val;                                             \
+    })
+
+#define atomic_set(ptr, i)  do {                      \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val = (i);                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELAXED);     \
+} while(0)
+
+/* Atomic RCU operations imply weak memory barriers */
+
+#define atomic_rcu_read(ptr)                          \
+    ({                                                \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val;                                \
+    __atomic_load(ptr, &_val, __ATOMIC_CONSUME);      \
+    _val;                                             \
+    })
+
+#define atomic_rcu_set(ptr, i) do {                   \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val = (i);                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELEASE);     \
+} while(0)
+
+/* atomic_mb_read/set semantics map Java volatile variables. They are
+ * less expensive on some platforms (notably POWER & ARMv7) than fully
+ * sequentially consistent operations.
+ *
+ * As long as they are used as paired operations they are safe to
+ * use. See docs/atomic.txt for more discussion.
+ */
+
+#if defined(_ARCH_PPC)
+#define atomic_mb_read(ptr)                             \
+    ({                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val;                                  \
+     __atomic_load(ptr, &_val, __ATOMIC_RELAXED);       \
+     smp_rmb();                                         \
+    _val;                                               \
+    })
+
+#define atomic_mb_set(ptr, i)  do {                     \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val = (i);                            \
+    smp_wmb();                                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELAXED);       \
+    smp_mb();                                           \
+} while(0)
+#else
+#define atomic_mb_read(ptr)                             \
+    ({                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val;                                  \
+    __atomic_load(ptr, &_val, __ATOMIC_SEQ_CST);        \
+    _val;                                               \
+    })
+
+#define atomic_mb_set(ptr, i)  do {                     \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val = (i);                            \
+    __atomic_store(ptr, &_val, __ATOMIC_SEQ_CST);       \
+} while(0)
+#endif
+
+
+/* All the remaining operations are fully sequentially consistent */
+
+#define atomic_xchg(ptr, i)    ({                           \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));       \
+    typeof(*ptr) _new = (i), _old;                          \
+    __atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
+    _old;                                                   \
+})
+
+/* Returns the eventual value, failed or not */
+#define atomic_cmpxchg(ptr, old, new)                                   \
+    ({                                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));                   \
+    typeof(*ptr) _old = (old), _new = (new);                            \
+    __atomic_compare_exchange(ptr, &_old, &_new, false,                 \
+                              __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);      \
+    _old;                                                               \
+    })
+
+/* Provide shorter names for GCC atomic builtins, return old value */
+#define atomic_fetch_inc(ptr)  __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
+#define atomic_fetch_dec(ptr)  __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST)
+#define atomic_fetch_add(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_or(ptr, n)  __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)
+
+/* And even shorter names that return void.  */
+#define atomic_inc(ptr)    ((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST))
+#define atomic_dec(ptr)    ((void) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST))
+#define atomic_add(ptr, n) ((void) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_sub(ptr, n) ((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_and(ptr, n) ((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_or(ptr, n)  ((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST))
+
+#else /* __ATOMIC_RELAXED */
 
 /*
  * We use GCC builtin if it's available, as that can use mfence on
@@ -85,8 +214,6 @@
 
 #endif /* _ARCH_PPC */
 
-#endif /* C11 atomics */
-
 /*
  * For (host) platforms we don't have explicit barrier definitions
  * for, we use the gcc __sync_synchronize() primitive to generate a
@@ -98,42 +225,22 @@
 #endif
 
 #ifndef smp_wmb
-#ifdef __ATOMIC_RELEASE
-/* __atomic_thread_fence does not include a compiler barrier; instead,
- * the barrier is part of __atomic_load/__atomic_store's "volatile-like"
- * semantics. If smp_wmb() is a no-op, absence of the barrier means that
- * the compiler is free to reorder stores on each side of the barrier.
- * Add one here, and similarly in smp_rmb() and smp_read_barrier_depends().
- */
-#define smp_wmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_RELEASE); barrier(); })
-#else
 #define smp_wmb()   __sync_synchronize()
 #endif
-#endif
 
 #ifndef smp_rmb
-#ifdef __ATOMIC_ACQUIRE
-#define smp_rmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_ACQUIRE); barrier(); })
-#else
 #define smp_rmb()   __sync_synchronize()
 #endif
-#endif
 
 #ifndef smp_read_barrier_depends
-#ifdef __ATOMIC_CONSUME
-#define smp_read_barrier_depends()   ({ barrier(); __atomic_thread_fence(__ATOMIC_CONSUME); barrier(); })
-#else
 #define smp_read_barrier_depends()   barrier()
 #endif
-#endif
 
-#ifndef atomic_read
+/* These will only be atomic if the processor does the fetch or store
+ * in a single issue memory operation
+ */
 #define atomic_read(ptr)       (*(__typeof__(*ptr) volatile*) (ptr))
-#endif
-
-#ifndef atomic_set
 #define atomic_set(ptr, i)     ((*(__typeof__(*ptr) volatile*) (ptr)) = (i))
-#endif
 
 /**
  * atomic_rcu_read - reads a RCU-protected pointer to a local variable
@@ -146,30 +253,18 @@
  * Inserts memory barriers on architectures that require them (currently only
  * Alpha) and documents which pointers are protected by RCU.
  *
- * Unless the __ATOMIC_CONSUME memory order is available, atomic_rcu_read also
- * includes a compiler barrier to ensure that value-speculative optimizations
- * (e.g. VSS: Value Speculation Scheduling) does not perform the data read
- * before the pointer read by speculating the value of the pointer.  On new
- * enough compilers, atomic_load takes care of such concern about
- * dependency-breaking optimizations.
+ * atomic_rcu_read also includes a compiler barrier to ensure that
+ * value-speculative optimizations (e.g. VSS: Value Speculation
+ * Scheduling) does not perform the data read before the pointer read
+ * by speculating the value of the pointer.
  *
  * Should match atomic_rcu_set(), atomic_xchg(), atomic_cmpxchg().
  */
-#ifndef atomic_rcu_read
-#ifdef __ATOMIC_CONSUME
-#define atomic_rcu_read(ptr)    ({                \
-    typeof(*ptr) _val;                            \
-     __atomic_load(ptr, &_val, __ATOMIC_CONSUME); \
-    _val;                                         \
-})
-#else
 #define atomic_rcu_read(ptr)    ({                \
     typeof(*ptr) _val = atomic_read(ptr);         \
     smp_read_barrier_depends();                   \
     _val;                                         \
 })
-#endif
-#endif
 
 /**
  * atomic_rcu_set - assigns (publicizes) a pointer to a new data structure
@@ -182,19 +277,10 @@
  *
  * Should match atomic_rcu_read().
  */
-#ifndef atomic_rcu_set
-#ifdef __ATOMIC_RELEASE
-#define atomic_rcu_set(ptr, i)  do {              \
-    typeof(*ptr) _val = (i);                      \
-    __atomic_store(ptr, &_val, __ATOMIC_RELEASE); \
-} while(0)
-#else
 #define atomic_rcu_set(ptr, i)  do {              \
     smp_wmb();                                    \
     atomic_set(ptr, i);                           \
 } while (0)
-#endif
-#endif
 
 /* These have the same semantics as Java volatile variables.
  * See http://gee.cs.oswego.edu/dl/jmm/cookbook.html:
@@ -218,13 +304,11 @@
  * (see docs/atomics.txt), and I'm not sure that __ATOMIC_ACQ_REL is enough.
  * Just always use the barriers manually by the rules above.
  */
-#ifndef atomic_mb_read
 #define atomic_mb_read(ptr)    ({           \
     typeof(*ptr) _val = atomic_read(ptr);   \
     smp_rmb();                              \
     _val;                                   \
 })
-#endif
 
 #ifndef atomic_mb_set
 #define atomic_mb_set(ptr, i)  do {         \
@@ -237,12 +321,6 @@
 #ifndef atomic_xchg
 #if defined(__clang__)
 #define atomic_xchg(ptr, i)    __sync_swap(ptr, i)
-#elif defined(__ATOMIC_SEQ_CST)
-#define atomic_xchg(ptr, i)    ({                           \
-    typeof(*ptr) _new = (i), _old;                          \
-    __atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
-    _old;                                                   \
-})
 #else
 /* __sync_lock_test_and_set() is documented to be an acquire barrier only.  */
 #define atomic_xchg(ptr, i)    (smp_mb(), __sync_lock_test_and_set(ptr, i))
@@ -266,4 +344,5 @@
 #define atomic_and(ptr, n)     ((void) __sync_fetch_and_and(ptr, n))
 #define atomic_or(ptr, n)      ((void) __sync_fetch_and_or(ptr, n))
 
-#endif
+#endif /* __ATOMIC_RELAXED */
+#endif /* __QEMU_ATOMIC_H */
diff --git a/qemu/include/qemu/base64.h b/qemu/include/qemu/base64.h
new file mode 100644
index 000000000..793708dc3
--- /dev/null
+++ b/qemu/include/qemu/base64.h
@@ -0,0 +1,58 @@
+/*
+ * QEMU base64 helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QEMU_BASE64_H__
+#define QEMU_BASE64_H__
+
+#include "qemu-common.h"
+
+
+/**
+ * qbase64_decode:
+ * @input: the (possibly) base64 encoded text
+ * @in_len: length of @input or -1 if NUL terminated
+ * @out_len: filled with length of decoded data
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to decode the (possibly) base64 encoded
+ * text provided in @input. If the @input text may
+ * contain embedded NUL characters, or may not be
+ * NUL terminated, then @in_len must be set to the
+ * known size of the @input buffer.
+ *
+ * Note that embedded NULs, or lack of a NUL terminator
+ * are considered invalid base64 data and errors
+ * will be reported to this effect.
+ *
+ * If decoding is successful, the decoded data will
+ * be returned and @out_len set to indicate the
+ * number of bytes in the decoded data. The caller
+ * must use g_free() to free the returned data when
+ * it is no longer required.
+ *
+ * Returns: the decoded data or NULL
+ */
+uint8_t *qbase64_decode(const char *input,
+                        size_t in_len,
+                        size_t *out_len,
+                        Error **errp);
+
+
+#endif /* QEMU_BUFFER_H__ */
diff --git a/qemu/include/qemu/bcd.h b/qemu/include/qemu/bcd.h
new file mode 100644
index 000000000..b4c9b64b8
--- /dev/null
+++ b/qemu/include/qemu/bcd.h
@@ -0,0 +1,15 @@
+#ifndef QEMU_BCD_H
+#define QEMU_BCD_H 1
+
+/* Convert a byte between binary and BCD.  */
+static inline uint8_t to_bcd(uint8_t val)
+{
+    return ((val / 10) << 4) | (val % 10);
+}
+
+static inline uint8_t from_bcd(uint8_t val)
+{
+    return ((val >> 4) * 10) + (val & 0x0f);
+}
+
+#endif
diff --git a/qemu/include/qemu/bitmap.h b/qemu/include/qemu/bitmap.h
index 86dd9cd5f..0e33fa5d9 100644
--- a/qemu/include/qemu/bitmap.h
+++ b/qemu/include/qemu/bitmap.h
@@ -13,10 +13,7 @@
 #define BITMAP_H
 
 #include <glib.h>
-#include <string.h>
-#include <stdlib.h>
 
-#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 /*
diff --git a/qemu/include/qemu/bitops.h b/qemu/include/qemu/bitops.h
index 816422515..755fdd129 100644
--- a/qemu/include/qemu/bitops.h
+++ b/qemu/include/qemu/bitops.h
@@ -12,8 +12,6 @@
 #ifndef BITOPS_H
 #define BITOPS_H
 
-#include <stdint.h>
-#include <assert.h>
 
 #include "host-utils.h"
 #include "atomic.h"
diff --git a/qemu/include/qemu/bswap.h b/qemu/include/qemu/bswap.h
index 07d88de74..ce3c42e4d 100644
--- a/qemu/include/qemu/bswap.h
+++ b/qemu/include/qemu/bswap.h
@@ -1,15 +1,10 @@
 #ifndef BSWAP_H
 #define BSWAP_H
 
-#include "config-host.h"
-#include <inttypes.h>
-#include <limits.h>
-#include <string.h>
 #include "fpu/softfloat.h"
 
 #ifdef CONFIG_MACHINE_BSWAP_H
 # include <sys/endian.h>
-# include <sys/types.h>
 # include <machine/bswap.h>
 #elif defined(__FreeBSD__)
 # include <sys/endian.h>
@@ -130,6 +125,25 @@ static inline uint32_t qemu_bswap_len(uint32_t value, int len)
     return bswap32(value) >> (32 - 8 * len);
 }
 
+/*
+ * Same as cpu_to_le{16,23}, except that gcc will figure the result is
+ * a compile-time constant if you pass in a constant.  So this can be
+ * used to initialize static variables.
+ */
+#if defined(HOST_WORDS_BIGENDIAN)
+# define const_le32(_x)                          \
+    ((((_x) & 0x000000ffU) << 24) |              \
+     (((_x) & 0x0000ff00U) <<  8) |              \
+     (((_x) & 0x00ff0000U) >>  8) |              \
+     (((_x) & 0xff000000U) >> 24))
+# define const_le16(_x)                          \
+    ((((_x) & 0x00ff) << 8) |                    \
+     (((_x) & 0xff00) >> 8))
+#else
+# define const_le32(_x) (_x)
+# define const_le16(_x) (_x)
+#endif
+
 /* Unions for reinterpreting between floats and integers.  */
 
 typedef union {
@@ -424,11 +438,9 @@ static inline void stfq_be_p(void *ptr, float64 v)
 
 static inline unsigned long leul_to_cpu(unsigned long v)
 {
-    /* In order to break an include loop between here and
-       qemu-common.h, don't rely on HOST_LONG_BITS.  */
-#if ULONG_MAX == UINT32_MAX
+#if HOST_LONG_BITS == 32
     return le_bswap(v, 32);
-#elif ULONG_MAX == UINT64_MAX
+#elif HOST_LONG_BITS == 64
     return le_bswap(v, 64);
 #else
 # error Unknown sizeof long
diff --git a/qemu/include/qemu/buffer.h b/qemu/include/qemu/buffer.h
new file mode 100644
index 000000000..dead9b77e
--- /dev/null
+++ b/qemu/include/qemu/buffer.h
@@ -0,0 +1,161 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QEMU_BUFFER_H__
+#define QEMU_BUFFER_H__
+
+#include "qemu-common.h"
+
+typedef struct Buffer Buffer;
+
+/**
+ * Buffer:
+ *
+ * The Buffer object provides a simple dynamically resizing
+ * array, with separate tracking of capacity and usage. This
+ * is typically useful when buffering I/O or processing data.
+ */
+
+struct Buffer {
+    char *name;
+    size_t capacity;
+    size_t offset;
+    uint64_t avg_size;
+    uint8_t *buffer;
+};
+
+/**
+ * buffer_init:
+ * @buffer: the buffer object
+ * @name: buffer name
+ *
+ * Optionally attach a name to the buffer, to make it easier
+ * to identify in debug traces.
+ */
+void buffer_init(Buffer *buffer, const char *name, ...)
+        GCC_FMT_ATTR(2, 3);
+
+/**
+ * buffer_shrink:
+ * @buffer: the buffer object
+ *
+ * Try to shrink the buffer.  Checks current buffer capacity and size
+ * and reduces capacity in case only a fraction of the buffer is
+ * actually used.
+ */
+void buffer_shrink(Buffer *buffer);
+
+/**
+ * buffer_reserve:
+ * @buffer: the buffer object
+ * @len: the minimum required free space
+ *
+ * Ensure that the buffer has space allocated for at least
+ * @len bytes. If the current buffer is too small, it will
+ * be reallocated, possibly to a larger size than requested.
+ */
+void buffer_reserve(Buffer *buffer, size_t len);
+
+/**
+ * buffer_reset:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero, but do
+ * not free / reallocate the memory buffer
+ */
+void buffer_reset(Buffer *buffer);
+
+/**
+ * buffer_free:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero and also
+ * free the internal memory buffer
+ */
+void buffer_free(Buffer *buffer);
+
+/**
+ * buffer_append:
+ * @buffer: the buffer object
+ * @data: the data block to append
+ * @len: the length of @data in bytes
+ *
+ * Append the contents of @data to the end of the buffer.
+ * The caller must ensure that the buffer has sufficient
+ * free space for @len bytes, typically by calling the
+ * buffer_reserve() method prior to appending.
+ */
+void buffer_append(Buffer *buffer, const void *data, size_t len);
+
+/**
+ * buffer_advance:
+ * @buffer: the buffer object
+ * @len: the number of bytes to skip
+ *
+ * Remove @len bytes of data from the head of the buffer.
+ * The internal buffer will not be reallocated, so will
+ * have at least @len bytes of free space after this
+ * call completes
+ */
+void buffer_advance(Buffer *buffer, size_t len);
+
+/**
+ * buffer_end:
+ * @buffer: the buffer object
+ *
+ * Get a pointer to the tail end of the internal buffer
+ * The returned pointer is only valid until the next
+ * call to buffer_reserve().
+ *
+ * Returns: the tail of the buffer
+ */
+uint8_t *buffer_end(Buffer *buffer);
+
+/**
+ * buffer_empty:
+ * @buffer: the buffer object
+ *
+ * Determine if the buffer contains any current data
+ *
+ * Returns: true if the buffer holds data, false otherwise
+ */
+gboolean buffer_empty(Buffer *buffer);
+
+/**
+ * buffer_move_empty:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, without copying data.  'to' buffer must be empty.
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move_empty(Buffer *to, Buffer *from);
+
+/**
+ * buffer_move:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, copying data (unless 'to' buffer happens to be empty).
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move(Buffer *to, Buffer *from);
+
+#endif /* QEMU_BUFFER_H__ */
diff --git a/qemu/include/qemu/compatfd.h b/qemu/include/qemu/compatfd.h
index fc3791520..aa12ee936 100644
--- a/qemu/include/qemu/compatfd.h
+++ b/qemu/include/qemu/compatfd.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_COMPATFD_H
 #define QEMU_COMPATFD_H
 
-#include <signal.h>
 
 struct qemu_signalfd_siginfo {
     uint32_t ssi_signo;   /* Signal number */
diff --git a/qemu/include/qemu/compiler.h b/qemu/include/qemu/compiler.h
index df9dd514f..8f1cc7ba6 100644
--- a/qemu/include/qemu/compiler.h
+++ b/qemu/include/qemu/compiler.h
@@ -3,7 +3,6 @@
 #ifndef COMPILER_H
 #define COMPILER_H
 
-#include "config-host.h"
 
 /*----------------------------------------------------------------------------
 | The macro QEMU_GNUC_PREREQ tests for minimum version of the GNU C compiler.
@@ -42,10 +41,43 @@
 # define QEMU_PACKED __attribute__((packed))
 #endif
 
-#define cat(x,y) x ## y
-#define cat2(x,y) cat(x,y)
+#ifndef glue
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+#endif
+
+#ifndef likely
+#if __GNUC__ < 3
+#define __builtin_expect(x, n) (x)
+#endif
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x)   __builtin_expect(!!(x), 0)
+#endif
+
+#ifndef container_of
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+#endif
+
+/* Convert from a base type to a parent type, with compile time checking.  */
+#ifdef __GNUC__
+#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
+    char __attribute__((unused)) offset_must_be_zero[ \
+        -offsetof(type, field)]; \
+    container_of(dev, type, field);}))
+#else
+#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+#endif
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
 #define QEMU_BUILD_BUG_ON(x) \
-    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+    typedef char glue(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
 
 #if defined __GNUC__
 # if !QEMU_GNUC_PREREQ(4, 4)
diff --git a/qemu/include/qemu/config-file.h b/qemu/include/qemu/config-file.h
index d4ba20e04..3b8ecb095 100644
--- a/qemu/include/qemu/config-file.h
+++ b/qemu/include/qemu/config-file.h
@@ -1,9 +1,7 @@
 #ifndef QEMU_CONFIG_H
 #define QEMU_CONFIG_H
 
-#include <stdio.h>
 #include "qemu/option.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 QemuOptsList *qemu_find_opts(const char *group);
diff --git a/qemu/include/block/coroutine.h b/qemu/include/qemu/coroutine.h
index 20c027a7f..305fe76c2 100644
--- a/qemu/include/block/coroutine.h
+++ b/qemu/include/qemu/coroutine.h
@@ -15,8 +15,6 @@
 #ifndef QEMU_COROUTINE_H
 #define QEMU_COROUTINE_H
 
-#include <stdbool.h>
-#include "qemu/typedefs.h"
 #include "qemu/queue.h"
 #include "qemu/timer.h"
 
diff --git a/qemu/include/block/coroutine_int.h b/qemu/include/qemu/coroutine_int.h
index 9aa1aae5d..42d683840 100644
--- a/qemu/include/block/coroutine_int.h
+++ b/qemu/include/qemu/coroutine_int.h
@@ -26,7 +26,7 @@
 #define QEMU_COROUTINE_INT_H
 
 #include "qemu/queue.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 typedef enum {
     COROUTINE_YIELD = 1,
diff --git a/qemu/include/qemu/cutils.h b/qemu/include/qemu/cutils.h
new file mode 100644
index 000000000..db7adadcf
--- /dev/null
+++ b/qemu/include/qemu/cutils.h
@@ -0,0 +1,183 @@
+#ifndef QEMU_CUTILS_H
+#define QEMU_CUTILS_H 1
+
+#include "qemu/fprintf-fn.h"
+
+/**
+ * pstrcpy:
+ * @buf: buffer to copy string into
+ * @buf_size: size of @buf in bytes
+ * @str: string to copy
+ *
+ * Copy @str into @buf, including the trailing NUL, but do not
+ * write more than @buf_size bytes. The resulting buffer is
+ * always NUL terminated (even if the source string was too long).
+ * If @buf_size is zero or negative then no bytes are copied.
+ *
+ * This function is similar to strncpy(), but avoids two of that
+ * function's problems:
+ *  * if @str fits in the buffer, pstrcpy() does not zero-fill the
+ *    remaining space at the end of @buf
+ *  * if @str is too long, pstrcpy() will copy the first @buf_size-1
+ *    bytes and then add a NUL
+ */
+void pstrcpy(char *buf, int buf_size, const char *str);
+/**
+ * strpadcpy:
+ * @buf: buffer to copy string into
+ * @buf_size: size of @buf in bytes
+ * @str: string to copy
+ * @pad: character to pad the remainder of @buf with
+ *
+ * Copy @str into @buf (but *not* its trailing NUL!), and then pad the
+ * rest of the buffer with the @pad character. If @str is too large
+ * for the buffer then it is truncated, so that @buf contains the
+ * first @buf_size characters of @str, with no terminator.
+ */
+void strpadcpy(char *buf, int buf_size, const char *str, char pad);
+/**
+ * pstrcat:
+ * @buf: buffer containing existing string
+ * @buf_size: size of @buf in bytes
+ * @s: string to concatenate to @buf
+ *
+ * Append a copy of @s to the string already in @buf, but do not
+ * allow the buffer to overflow. If the existing contents of @buf
+ * plus @str would total more than @buf_size bytes, then write
+ * as much of @str as will fit followed by a NUL terminator.
+ *
+ * @buf must already contain a NUL-terminated string, or the
+ * behaviour is undefined.
+ *
+ * Returns: @buf.
+ */
+char *pstrcat(char *buf, int buf_size, const char *s);
+/**
+ * strstart:
+ * @str: string to test
+ * @val: prefix string to look for
+ * @ptr: NULL, or pointer to be written to indicate start of
+ *       the remainder of the string
+ *
+ * Test whether @str starts with the prefix @val.
+ * If it does (including the degenerate case where @str and @val
+ * are equal) then return true. If @ptr is not NULL then a
+ * pointer to the first character following the prefix is written
+ * to it. If @val is not a prefix of @str then return false (and
+ * @ptr is not written to).
+ *
+ * Returns: true if @str starts with prefix @val, false otherwise.
+ */
+int strstart(const char *str, const char *val, const char **ptr);
+/**
+ * stristart:
+ * @str: string to test
+ * @val: prefix string to look for
+ * @ptr: NULL, or pointer to be written to indicate start of
+ *       the remainder of the string
+ *
+ * Test whether @str starts with the case-insensitive prefix @val.
+ * This function behaves identically to strstart(), except that the
+ * comparison is made after calling qemu_toupper() on each pair of
+ * characters.
+ *
+ * Returns: true if @str starts with case-insensitive prefix @val,
+ *          false otherwise.
+ */
+int stristart(const char *str, const char *val, const char **ptr);
+/**
+ * qemu_strnlen:
+ * @s: string
+ * @max_len: maximum number of bytes in @s to scan
+ *
+ * Return the length of the string @s, like strlen(), but do not
+ * examine more than @max_len bytes of the memory pointed to by @s.
+ * If no NUL terminator is found within @max_len bytes, then return
+ * @max_len instead.
+ *
+ * This function has the same behaviour as the POSIX strnlen()
+ * function.
+ *
+ * Returns: length of @s in bytes, or @max_len, whichever is smaller.
+ */
+int qemu_strnlen(const char *s, int max_len);
+/**
+ * qemu_strsep:
+ * @input: pointer to string to parse
+ * @delim: string containing delimiter characters to search for
+ *
+ * Locate the first occurrence of any character in @delim within
+ * the string referenced by @input, and replace it with a NUL.
+ * The location of the next character after the delimiter character
+ * is stored into @input.
+ * If the end of the string was reached without finding a delimiter
+ * character, then NULL is stored into @input.
+ * If @input points to a NULL pointer on entry, return NULL.
+ * The return value is always the original value of *@input (and
+ * so now points to a NUL-terminated string corresponding to the
+ * part of the input up to the first delimiter).
+ *
+ * This function has the same behaviour as the BSD strsep() function.
+ *
+ * Returns: the pointer originally in @input.
+ */
+char *qemu_strsep(char **input, const char *delim);
+time_t mktimegm(struct tm *tm);
+int qemu_fdatasync(int fd);
+int fcntl_setfl(int fd, int flag);
+int qemu_parse_fd(const char *param);
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result);
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result);
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result);
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result);
+
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base);
+int parse_uint_full(const char *s, unsigned long long *value, int base);
+
+/*
+ * qemu_strtosz() suffixes used to specify the default treatment of an
+ * argument passed to qemu_strtosz() without an explicit suffix.
+ * These should be defined using upper case characters in the range
+ * A-Z, as qemu_strtosz() will use qemu_toupper() on the given argument
+ * prior to comparison.
+ */
+#define QEMU_STRTOSZ_DEFSUFFIX_EB 'E'
+#define QEMU_STRTOSZ_DEFSUFFIX_PB 'P'
+#define QEMU_STRTOSZ_DEFSUFFIX_TB 'T'
+#define QEMU_STRTOSZ_DEFSUFFIX_GB 'G'
+#define QEMU_STRTOSZ_DEFSUFFIX_MB 'M'
+#define QEMU_STRTOSZ_DEFSUFFIX_KB 'K'
+#define QEMU_STRTOSZ_DEFSUFFIX_B 'B'
+int64_t qemu_strtosz(const char *nptr, char **end);
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix);
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
+                            const char default_suffix, int64_t unit);
+#define K_BYTE     (1ULL << 10)
+#define M_BYTE     (1ULL << 20)
+#define G_BYTE     (1ULL << 30)
+#define T_BYTE     (1ULL << 40)
+#define P_BYTE     (1ULL << 50)
+#define E_BYTE     (1ULL << 60)
+
+/* used to print char* safely */
+#define STR_OR_NULL(str) ((str) ? (str) : "null")
+
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len);
+size_t buffer_find_nonzero_offset(const void *buf, size_t len);
+bool buffer_is_zero(const void *buf, size_t len);
+
+/*
+ * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
+ * Input is limited to 14-bit numbers
+ */
+
+int uleb128_encode_small(uint8_t *out, uint32_t n);
+int uleb128_decode_small(const uint8_t *in, uint32_t *n);
+
+#endif
diff --git a/qemu/include/qemu/error-report.h b/qemu/include/qemu/error-report.h
index 7ab235590..7a2a363fb 100644
--- a/qemu/include/qemu/error-report.h
+++ b/qemu/include/qemu/error-report.h
@@ -13,9 +13,6 @@
 #ifndef QEMU_ERROR_H
 #define QEMU_ERROR_H
 
-#include <stdarg.h>
-#include <stdbool.h>
-#include "qemu/compiler.h"
 
 typedef struct Location {
     /* all members are private to qemu-error.c */
diff --git a/qemu/include/qemu/event_notifier.h b/qemu/include/qemu/event_notifier.h
index 88b57af7c..e326990db 100644
--- a/qemu/include/qemu/event_notifier.h
+++ b/qemu/include/qemu/event_notifier.h
@@ -34,11 +34,13 @@ int event_notifier_init(EventNotifier *, int active);
 void event_notifier_cleanup(EventNotifier *);
 int event_notifier_set(EventNotifier *);
 int event_notifier_test_and_clear(EventNotifier *);
-int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
+int event_notifier_set_handler(EventNotifier *,
+                               bool is_external,
+                               EventNotifierHandler *);
 
 #ifdef CONFIG_POSIX
 void event_notifier_init_fd(EventNotifier *, int fd);
-int event_notifier_get_fd(EventNotifier *);
+int event_notifier_get_fd(const EventNotifier *);
 #else
 HANDLE event_notifier_get_handle(EventNotifier *);
 #endif
diff --git a/qemu/include/qemu/fprintf-fn.h b/qemu/include/qemu/fprintf-fn.h
index 9ddc90f1c..b6bad35b1 100644
--- a/qemu/include/qemu/fprintf-fn.h
+++ b/qemu/include/qemu/fprintf-fn.h
@@ -8,8 +8,6 @@
 #ifndef QEMU_FPRINTF_FN_H
 #define QEMU_FPRINTF_FN_H 1
 
-#include "qemu/compiler.h"
-#include <stdio.h>
 
 typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
     GCC_FMT_ATTR(2, 3);
diff --git a/qemu/include/qemu/hbitmap.h b/qemu/include/qemu/hbitmap.h
index bb94a00c5..e29188c0a 100644
--- a/qemu/include/qemu/hbitmap.h
+++ b/qemu/include/qemu/hbitmap.h
@@ -12,9 +12,6 @@
 #ifndef HBITMAP_H
 #define HBITMAP_H 1
 
-#include <limits.h>
-#include <stdint.h>
-#include <stdbool.h>
 #include "bitops.h"
 #include "host-utils.h"
 
diff --git a/qemu/include/qemu/help_option.h b/qemu/include/qemu/help_option.h
new file mode 100644
index 000000000..e39a66e77
--- /dev/null
+++ b/qemu/include/qemu/help_option.h
@@ -0,0 +1,22 @@
+#ifndef QEMU_HELP_OPTION_H
+#define QEMU_HELP_OPTION_H 1
+
+/**
+ * is_help_option:
+ * @s: string to test
+ *
+ * Check whether @s is one of the standard strings which indicate
+ * that the user is asking for a list of the valid values for a
+ * command option like -cpu or -M. The current accepted strings
+ * are 'help' and '?'. '?' is deprecated (it is a shell wildcard
+ * which makes it annoying to use in a reliable way) but provided
+ * for backwards compatibility.
+ *
+ * Returns: true if @s is a request for a list.
+ */
+static inline bool is_help_option(const char *s)
+{
+    return !strcmp(s, "?") || !strcmp(s, "help");
+}
+
+#endif
diff --git a/qemu/include/qemu/host-utils.h b/qemu/include/qemu/host-utils.h
index d4f21c947..1cdae0d0e 100644
--- a/qemu/include/qemu/host-utils.h
+++ b/qemu/include/qemu/host-utils.h
@@ -25,8 +25,7 @@
 #ifndef HOST_UTILS_H
 #define HOST_UTILS_H 1
 
-#include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
-#include <limits.h>
+#include "qemu/bswap.h"
 
 #ifdef CONFIG_INT128
 static inline void mulu64(uint64_t *plow, uint64_t *phigh,
@@ -45,6 +44,12 @@ static inline void muls64(uint64_t *plow, uint64_t *phigh,
     *phigh = r >> 64;
 }
 
+/* compute with 96 bit intermediate result: (a*b)/c */
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    return (__int128_t)a * b / c;
+}
+
 static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
 {
     if (divisor == 0) {
@@ -75,6 +80,29 @@ void muls64(uint64_t *phigh, uint64_t *plow, int64_t a, int64_t b);
 void mulu64(uint64_t *phigh, uint64_t *plow, uint64_t a, uint64_t b);
 int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor);
 int divs128(int64_t *plow, int64_t *phigh, int64_t divisor);
+
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    union {
+        uint64_t ll;
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint32_t high, low;
+#else
+            uint32_t low, high;
+#endif
+        } l;
+    } u, res;
+    uint64_t rl, rh;
+
+    u.ll = a;
+    rl = (uint64_t)u.l.low * (uint64_t)b;
+    rh = (uint64_t)u.l.high * (uint64_t)b;
+    rh += (rl >> 32);
+    res.l.high = rh / c;
+    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
+    return res.ll;
+}
 #endif
 
 /**
@@ -361,6 +389,80 @@ static inline int ctpop64(uint64_t val)
 #endif
 }
 
+/**
+ * revbit8 - reverse the bits in an 8-bit value.
+ * @x: The value to modify.
+ */
+static inline uint8_t revbit8(uint8_t x)
+{
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0) >> 4)
+      | ((x & 0x0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88) >> 3)
+      | ((x & 0x44) >> 1)
+      | ((x & 0x22) << 1)
+      | ((x & 0x11) << 3);
+    return x;
+}
+
+/**
+ * revbit16 - reverse the bits in a 16-bit value.
+ * @x: The value to modify.
+ */
+static inline uint16_t revbit16(uint16_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap16(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0) >> 4)
+      | ((x & 0x0f0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888) >> 3)
+      | ((x & 0x4444) >> 1)
+      | ((x & 0x2222) << 1)
+      | ((x & 0x1111) << 3);
+    return x;
+}
+
+/**
+ * revbit32 - reverse the bits in a 32-bit value.
+ * @x: The value to modify.
+ */
+static inline uint32_t revbit32(uint32_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap32(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0u) >> 4)
+      | ((x & 0x0f0f0f0fu) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88888888u) >> 3)
+      | ((x & 0x44444444u) >> 1)
+      | ((x & 0x22222222u) << 1)
+      | ((x & 0x11111111u) << 3);
+    return x;
+}
+
+/**
+ * revbit64 - reverse the bits in a 64-bit value.
+ * @x: The value to modify.
+ */
+static inline uint64_t revbit64(uint64_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap64(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0f0f0f0f0ull) >> 4)
+      | ((x & 0x0f0f0f0f0f0f0f0full) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888888888888888ull) >> 3)
+      | ((x & 0x4444444444444444ull) >> 1)
+      | ((x & 0x2222222222222222ull) << 1)
+      | ((x & 0x1111111111111111ull) << 3);
+    return x;
+}
+
 /* Host type specific sizes of these routines.  */
 
 #if ULONG_MAX == UINT32_MAX
@@ -369,14 +471,48 @@ static inline int ctpop64(uint64_t val)
 # define clol   clo32
 # define ctol   cto32
 # define ctpopl ctpop32
+# define revbitl revbit32
 #elif ULONG_MAX == UINT64_MAX
 # define clzl   clz64
 # define ctzl   ctz64
 # define clol   clo64
 # define ctol   cto64
 # define ctpopl ctpop64
+# define revbitl revbit64
 #else
 # error Unknown sizeof long
 #endif
 
+static inline bool is_power_of_2(uint64_t value)
+{
+    if (!value) {
+        return 0;
+    }
+
+    return !(value & (value - 1));
+}
+
+/* round down to the nearest power of 2*/
+static inline int64_t pow2floor(int64_t value)
+{
+    if (!is_power_of_2(value)) {
+        value = 0x8000000000000000ULL >> clz64(value);
+    }
+    return value;
+}
+
+/* round up to the nearest power of 2 (0 if overflow) */
+static inline uint64_t pow2ceil(uint64_t value)
+{
+    uint8_t nlz = clz64(value);
+
+    if (is_power_of_2(value)) {
+        return value;
+    }
+    if (!nlz) {
+        return 0;
+    }
+    return 1ULL << (64 - nlz);
+}
+
 #endif
diff --git a/qemu/include/qemu/id.h b/qemu/include/qemu/id.h
new file mode 100644
index 000000000..7d90335af
--- /dev/null
+++ b/qemu/include/qemu/id.h
@@ -0,0 +1,13 @@
+#ifndef QEMU_ID_H
+#define QEMU_ID_H 1
+
+typedef enum IdSubSystems {
+    ID_QDEV,
+    ID_BLOCK,
+    ID_MAX      /* last element, used as array size */
+} IdSubSystems;
+
+char *id_generate(IdSubSystems id);
+bool id_wellformed(const char *id);
+
+#endif
diff --git a/qemu/include/qemu/int128.h b/qemu/include/qemu/int128.h
index fb782aadd..c5988813d 100644
--- a/qemu/include/qemu/int128.h
+++ b/qemu/include/qemu/int128.h
@@ -1,9 +1,6 @@
 #ifndef INT128_H
 #define INT128_H
 
-#include <assert.h>
-#include <stdint.h>
-#include <stdbool.h>
 
 typedef struct Int128 Int128;
 
diff --git a/qemu/include/qemu/iov.h b/qemu/include/qemu/iov.h
index 569b2c2a2..bd9fd55b0 100644
--- a/qemu/include/qemu/iov.h
+++ b/qemu/include/qemu/iov.h
@@ -14,8 +14,6 @@
 #ifndef IOV_H
 #define IOV_H
 
-#include "qemu-common.h"
-
 /**
  * count and return data size, in bytes, of an iovec
  * starting at `iov' of `iov_cnt' number of elements.
@@ -39,10 +37,36 @@ size_t iov_size(const struct iovec *iov, const unsigned int iov_cnt);
  * such "large" value is -1 (sinice size_t is unsigned),
  * so specifying `-1' as `bytes' means 'up to the end of iovec'.
  */
-size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
-                    size_t offset, const void *buf, size_t bytes);
-size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
-                  size_t offset, void *buf, size_t bytes);
+size_t iov_from_buf_full(const struct iovec *iov, unsigned int iov_cnt,
+                         size_t offset, const void *buf, size_t bytes);
+size_t iov_to_buf_full(const struct iovec *iov, const unsigned int iov_cnt,
+		       size_t offset, void *buf, size_t bytes);
+
+static inline size_t
+iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
+             size_t offset, const void *buf, size_t bytes)
+{
+    if (__builtin_constant_p(bytes) && iov_cnt &&
+        offset <= iov[0].iov_len && bytes <= iov[0].iov_len - offset) {
+        memcpy(iov[0].iov_base + offset, buf, bytes);
+        return bytes;
+    } else {
+        return iov_from_buf_full(iov, iov_cnt, offset, buf, bytes);
+    }
+}
+
+static inline size_t
+iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
+           size_t offset, void *buf, size_t bytes)
+{
+    if (__builtin_constant_p(bytes) && iov_cnt &&
+        offset <= iov[0].iov_len && bytes <= iov[0].iov_len - offset) {
+        memcpy(buf, iov[0].iov_base + offset, bytes);
+        return bytes;
+    } else {
+        return iov_to_buf_full(iov, iov_cnt, offset, buf, bytes);
+    }
+}
 
 /**
  * Set data bytes pointed out by iovec `iov' of size `iov_cnt' elements,
@@ -112,4 +136,32 @@ size_t iov_discard_front(struct iovec **iov, unsigned int *iov_cnt,
 size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
                         size_t bytes);
 
+typedef struct QEMUIOVector {
+    struct iovec *iov;
+    int niov;
+    int nalloc;
+    size_t size;
+} QEMUIOVector;
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes);
+size_t qemu_iovec_concat_iov(QEMUIOVector *dst,
+                             struct iovec *src_iov, unsigned int src_cnt,
+                             size_t soffset, size_t sbytes);
+bool qemu_iovec_is_zero(QEMUIOVector *qiov);
+void qemu_iovec_destroy(QEMUIOVector *qiov);
+void qemu_iovec_reset(QEMUIOVector *qiov);
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes);
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes);
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes);
+ssize_t qemu_iovec_compare(QEMUIOVector *a, QEMUIOVector *b);
+void qemu_iovec_clone(QEMUIOVector *dest, const QEMUIOVector *src, void *buf);
+void qemu_iovec_discard_back(QEMUIOVector *qiov, size_t bytes);
+
 #endif
diff --git a/qemu/include/qemu/log.h b/qemu/include/qemu/log.h
index f880e66db..c52f136ac 100644
--- a/qemu/include/qemu/log.h
+++ b/qemu/include/qemu/log.h
@@ -1,14 +1,6 @@
 #ifndef QEMU_LOG_H
 #define QEMU_LOG_H
 
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include "qemu/compiler.h"
-#include "qom/cpu.h"
-#ifdef NEED_CPU_H
-#include "disas/disas.h"
-#endif
 
 /* Private global variables, don't use */
 extern FILE *qemu_logfile;
@@ -28,6 +20,13 @@ static inline bool qemu_log_enabled(void)
     return qemu_logfile != NULL;
 }
 
+/* Returns true if qemu_log() will write somewhere else than stderr
+ */
+static inline bool qemu_log_separate(void)
+{
+    return qemu_logfile != NULL && qemu_logfile != stderr;
+}
+
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
 #define CPU_LOG_TB_IN_ASM  (1 << 1)
 #define CPU_LOG_TB_OP      (1 << 2)
@@ -35,12 +34,14 @@ static inline bool qemu_log_enabled(void)
 #define CPU_LOG_INT        (1 << 4)
 #define CPU_LOG_EXEC       (1 << 5)
 #define CPU_LOG_PCALL      (1 << 6)
-#define CPU_LOG_IOPORT     (1 << 7)
 #define CPU_LOG_TB_CPU     (1 << 8)
 #define CPU_LOG_RESET      (1 << 9)
 #define LOG_UNIMP          (1 << 10)
 #define LOG_GUEST_ERROR    (1 << 11)
 #define CPU_LOG_MMU        (1 << 12)
+#define CPU_LOG_TB_NOCHAIN (1 << 13)
+#define CPU_LOG_PAGE       (1 << 14)
+#define LOG_TRACE          (1 << 15)
 
 /* Returns true if a bit is set in the current loglevel mask
  */
@@ -65,91 +66,35 @@ qemu_log_vprintf(const char *fmt, va_list va)
     }
 }
 
-/* log only if a bit is set on the current loglevel mask
- */
-void GCC_FMT_ATTR(2, 3) qemu_log_mask(int mask, const char *fmt, ...);
-
-
-/* Special cases: */
-
-/* cpu_dump_state() logging functions: */
-/**
- * log_cpu_state:
- * @cpu: The CPU whose state is to be logged.
- * @flags: Flags what to log.
- *
- * Logs the output of cpu_dump_state().
+/* log only if a bit is set on the current loglevel mask:
+ * @mask: bit to check in the mask
+ * @fmt: printf-style format string
+ * @args: optional arguments for format string
  */
-static inline void log_cpu_state(CPUState *cpu, int flags)
-{
-    if (qemu_log_enabled()) {
-        cpu_dump_state(cpu, qemu_logfile, fprintf, flags);
-    }
-}
+#define qemu_log_mask(MASK, FMT, ...)                   \
+    do {                                                \
+        if (unlikely(qemu_loglevel_mask(MASK))) {       \
+            qemu_log(FMT, ## __VA_ARGS__);              \
+        }                                               \
+    } while (0)
 
-/**
- * log_cpu_state_mask:
- * @mask: Mask when to log.
- * @cpu: The CPU whose state is to be logged.
- * @flags: Flags what to log.
- *
- * Logs the output of cpu_dump_state() if loglevel includes @mask.
+/* log only if a bit is set on the current loglevel mask
+ * and we are in the address range we care about:
+ * @mask: bit to check in the mask
+ * @addr: address to check in dfilter
+ * @fmt: printf-style format string
+ * @args: optional arguments for format string
  */
-static inline void log_cpu_state_mask(int mask, CPUState *cpu, int flags)
-{
-    if (qemu_loglevel & mask) {
-        log_cpu_state(cpu, flags);
-    }
-}
-
-#ifdef NEED_CPU_H
-/* disas() and target_disas() to qemu_logfile: */
-static inline void log_target_disas(CPUState *cpu, target_ulong start,
-                                    target_ulong len, int flags)
-{
-    target_disas(qemu_logfile, cpu, start, len, flags);
-}
-
-static inline void log_disas(void *code, unsigned long size)
-{
-    disas(qemu_logfile, code, size);
-}
-
-#if defined(CONFIG_USER_ONLY)
-/* page_dump() output to the log file: */
-static inline void log_page_dump(void)
-{
-    page_dump(qemu_logfile);
-}
-#endif
-#endif
-
+#define qemu_log_mask_and_addr(MASK, ADDR, FMT, ...)    \
+    do {                                                \
+        if (unlikely(qemu_loglevel_mask(MASK)) &&       \
+                     qemu_log_in_addr_range(ADDR)) {    \
+            qemu_log(FMT, ## __VA_ARGS__);              \
+        }                                               \
+    } while (0)
 
 /* Maintenance: */
 
-/* fflush() the log file */
-static inline void qemu_log_flush(void)
-{
-    fflush(qemu_logfile);
-}
-
-/* Close the log file */
-static inline void qemu_log_close(void)
-{
-    if (qemu_logfile) {
-        if (qemu_logfile != stderr) {
-            fclose(qemu_logfile);
-        }
-        qemu_logfile = NULL;
-    }
-}
-
-/* Set up a new log file */
-static inline void qemu_log_set_file(FILE *f)
-{
-    qemu_logfile = f;
-}
-
 /* define log items */
 typedef struct QEMULogItem {
     int mask;
@@ -175,6 +120,8 @@ static inline void qemu_set_log(int log_flags)
 }
 
 void qemu_set_log_filename(const char *filename);
+void qemu_set_dfilter_ranges(const char *ranges);
+bool qemu_log_in_addr_range(uint64_t addr);
 int qemu_str_to_log_mask(const char *str);
 
 /* Print a usage message listing all the valid logging categories
@@ -182,4 +129,9 @@ int qemu_str_to_log_mask(const char *str);
  */
 void qemu_print_log_usage(FILE *f);
 
+/* fflush() the log file */
+void qemu_log_flush(void);
+/* Close the log file */
+void qemu_log_close(void);
+
 #endif
diff --git a/qemu/include/qemu/main-loop.h b/qemu/include/qemu/main-loop.h
index bc18ca30e..19b5de3dd 100644
--- a/qemu/include/qemu/main-loop.h
+++ b/qemu/include/qemu/main-loop.h
@@ -203,6 +203,8 @@ void qemu_set_fd_handler(int fd,
                          IOHandler *fd_write,
                          void *opaque);
 
+GSource *iohandler_get_g_source(void);
+AioContext *iohandler_get_aio_context(void);
 #ifdef CONFIG_POSIX
 /**
  * qemu_add_child_watch: Register a child process for reaping.
@@ -265,8 +267,6 @@ void qemu_mutex_unlock_iothread(void);
 /* internal interfaces */
 
 void qemu_fd_register(int fd);
-void qemu_iohandler_fill(GArray *pollfds);
-void qemu_iohandler_poll(GArray *pollfds, int rc);
 
 QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule_idle(QEMUBH *bh);
diff --git a/qemu/include/qemu/memfd.h b/qemu/include/qemu/memfd.h
new file mode 100644
index 000000000..745a8c501
--- /dev/null
+++ b/qemu/include/qemu/memfd.h
@@ -0,0 +1,24 @@
+#ifndef QEMU_MEMFD_H
+#define QEMU_MEMFD_H
+
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#define F_LINUX_SPECIFIC_BASE 1024
+#endif
+
+#ifndef F_ADD_SEALS
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
+
+#define F_SEAL_SEAL     0x0001  /* prevent further seals from being set */
+#define F_SEAL_SHRINK   0x0002  /* prevent file from shrinking */
+#define F_SEAL_GROW     0x0004  /* prevent file from growing */
+#define F_SEAL_WRITE    0x0008  /* prevent writes */
+#endif
+
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd);
+void qemu_memfd_free(void *ptr, size_t size, int fd);
+bool qemu_memfd_check(void);
+
+#endif /* QEMU_MEMFD_H */
diff --git a/qemu/include/qemu/mmap-alloc.h b/qemu/include/qemu/mmap-alloc.h
new file mode 100644
index 000000000..0899b2f01
--- /dev/null
+++ b/qemu/include/qemu/mmap-alloc.h
@@ -0,0 +1,12 @@
+#ifndef QEMU_MMAP_ALLOC
+#define QEMU_MMAP_ALLOC
+
+#include "qemu-common.h"
+
+size_t qemu_fd_getpagesize(int fd);
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared);
+
+void qemu_ram_munmap(void *ptr, size_t size);
+
+#endif
diff --git a/qemu/include/qemu/module.h b/qemu/include/qemu/module.h
index 72d94984a..237070844 100644
--- a/qemu/include/qemu/module.h
+++ b/qemu/include/qemu/module.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_MODULE_H
 #define QEMU_MODULE_H
 
-#include "qemu/osdep.h"
 
 #define DSO_STAMP_FUN         glue(qemu_stamp, CONFIG_STAMP)
 #define DSO_STAMP_FUN_STR     stringify(DSO_STAMP_FUN)
@@ -42,14 +41,14 @@ static void __attribute__((constructor)) do_qemu_init_ ## function(void)    \
 
 typedef enum {
     MODULE_INIT_BLOCK,
-    MODULE_INIT_MACHINE,
+    MODULE_INIT_OPTS,
     MODULE_INIT_QAPI,
     MODULE_INIT_QOM,
     MODULE_INIT_MAX
 } module_init_type;
 
 #define block_init(function) module_init(function, MODULE_INIT_BLOCK)
-#define machine_init(function) module_init(function, MODULE_INIT_MACHINE)
+#define opts_init(function) module_init(function, MODULE_INIT_OPTS)
 #define qapi_init(function) module_init(function, MODULE_INIT_QAPI)
 #define type_init(function) module_init(function, MODULE_INIT_QOM)
 
diff --git a/qemu/include/qemu/option.h b/qemu/include/qemu/option.h
index 57e51c962..8542d2dfd 100644
--- a/qemu/include/qemu/option.h
+++ b/qemu/include/qemu/option.h
@@ -26,9 +26,7 @@
 #ifndef QEMU_OPTIONS_H
 #define QEMU_OPTIONS_H
 
-#include <stdint.h>
 #include "qemu/queue.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 const char *get_opt_name(char *buf, int buf_size, const char *p, char delim);
@@ -44,10 +42,6 @@ void parse_option_size(const char *name, const char *value,
 bool has_help_option(const char *param);
 bool is_valid_option_list(const char *param);
 
-typedef struct QemuOpt QemuOpt;
-typedef struct QemuOpts QemuOpts;
-typedef struct QemuOptsList QemuOptsList;
-
 enum QemuOptType {
     QEMU_OPT_STRING = 0,  /* no parsing (use string as-is)                        */
     QEMU_OPT_BOOL,        /* on/off                                               */
diff --git a/qemu/include/qemu/osdep.h b/qemu/include/qemu/osdep.h
index 324736426..408783f53 100644
--- a/qemu/include/qemu/osdep.h
+++ b/qemu/include/qemu/osdep.h
@@ -1,12 +1,88 @@
+/*
+ * OS includes and handling of OS dependencies
+ *
+ * This header exists to pull in some common system headers that
+ * most code in QEMU will want, and to fix up some possible issues with
+ * it (missing defines, Windows weirdness, and so on).
+ *
+ * To avoid getting into possible circular include dependencies, this
+ * file should not include any other QEMU headers, with the exceptions
+ * of config-host.h, config-target.h, qemu/compiler.h,
+ * sysemu/os-posix.h, sysemu/os-win32.h, glib-compat.h and
+ * qemu/typedefs.h, all of which are doing a similar job to this file
+ * and are under similar constraints.
+ *
+ * This header also contains prototypes for functions defined in
+ * os-*.c and util/oslib-*.c; those would probably be better split
+ * out into separate header files.
+ *
+ * In an ideal world this header would contain only:
+ *  (1) things which everybody needs
+ *  (2) things without which code would work on most platforms but
+ *      fail to compile or misbehave on a minority of host OSes
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
 #ifndef QEMU_OSDEP_H
 #define QEMU_OSDEP_H
 
 #include "config-host.h"
+#ifdef NEED_CPU_H
+#include "config-target.h"
+#endif
+#include "qemu/compiler.h"
+
+/* Older versions of C++ don't get definitions of various macros from
+ * stdlib.h unless we define these macros before first inclusion of
+ * that system header.
+ */
+#ifndef __STDC_CONSTANT_MACROS
+#define __STDC_CONSTANT_MACROS
+#endif
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+/* The following block of code temporarily renames the daemon() function so the
+ * compiler does not see the warning associated with it in stdlib.h on OSX
+ */
+#ifdef __APPLE__
+#define daemon qemu_fake_daemon_function
+#include <stdlib.h>
+#undef daemon
+extern int daemon(int, int);
+#endif
+
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <inttypes.h>
+#include <limits.h>
+/* Put unistd.h before time.h as that triggers localtime_r/gmtime_r
+ * function availability on recentish Mingw-w64 platforms. */
+#include <unistd.h>
+#include <time.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <assert.h>
+/* setjmp must be declared before sysemu/os-win32.h
+ * because it is redefined there. */
+#include <setjmp.h>
+#include <signal.h>
+
 #ifdef __OpenBSD__
 #include <sys/signal.h>
 #endif
@@ -18,50 +94,51 @@
 #define WEXITSTATUS(x) (x)
 #endif
 
-#include <sys/time.h>
-
-#if defined(CONFIG_SOLARIS) && CONFIG_SOLARIS_VERSION < 10
-/* [u]int_fast*_t not in <sys/int_types.h> */
-typedef unsigned char           uint_fast8_t;
-typedef unsigned int            uint_fast16_t;
-typedef signed int              int_fast16_t;
+#ifdef _WIN32
+#include "sysemu/os-win32.h"
 #endif
 
-#ifndef glue
-#define xglue(x, y) x ## y
-#define glue(x, y) xglue(x, y)
-#define stringify(s)	tostring(s)
-#define tostring(s)	#s
+#ifdef CONFIG_POSIX
+#include "sysemu/os-posix.h"
 #endif
 
-#ifndef likely
-#if __GNUC__ < 3
-#define __builtin_expect(x, n) (x)
-#endif
+#include "glib-compat.h"
+#include "qemu/typedefs.h"
 
-#define likely(x)   __builtin_expect(!!(x), 1)
-#define unlikely(x)   __builtin_expect(!!(x), 0)
+#ifndef O_LARGEFILE
+#define O_LARGEFILE 0
 #endif
-
-#ifndef container_of
-#define container_of(ptr, type, member) ({                      \
-        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
-        (type *) ((char *) __mptr - offsetof(type, member));})
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+#ifndef ENOMEDIUM
+#define ENOMEDIUM ENODEV
+#endif
+#if !defined(ENOTSUP)
+#define ENOTSUP 4096
+#endif
+#if !defined(ECANCELED)
+#define ECANCELED 4097
+#endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
+#ifndef TIME_MAX
+#define TIME_MAX LONG_MAX
 #endif
 
-/* Convert from a base type to a parent type, with compile time checking.  */
-#ifdef __GNUC__
-#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
-    char __attribute__((unused)) offset_must_be_zero[ \
-        -offsetof(type, field)]; \
-    container_of(dev, type, field);}))
+/* HOST_LONG_BITS is the size of a native pointer in bits. */
+#if UINTPTR_MAX == UINT32_MAX
+# define HOST_LONG_BITS 32
+#elif UINTPTR_MAX == UINT64_MAX
+# define HOST_LONG_BITS 64
 #else
-#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+# error Unknown pointer size
 #endif
 
-#define typeof_field(type, field) typeof(((type *)0)->field)
-#define type_check(t1,t2) ((t1*)0 - (t2*)0)
-
 #ifndef MIN
 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
 #endif
@@ -75,6 +152,12 @@ typedef signed int              int_fast16_t;
 #define MIN_NON_ZERO(a, b) (((a) != 0 && (a) < (b)) ? (a) : (b))
 #endif
 
+/* Round number down to multiple */
+#define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
+
+/* Round number up to multiple */
+#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
+
 #ifndef ROUND_UP
 #define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
 #endif
@@ -87,20 +170,6 @@ typedef signed int              int_fast16_t;
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 #endif
 
-#ifndef always_inline
-#if !((__GNUC__ < 3) || defined(__APPLE__))
-#ifdef __OPTIMIZE__
-#undef inline
-#define inline __attribute__ (( always_inline )) __inline__
-#endif
-#endif
-#else
-#undef inline
-#define inline always_inline
-#endif
-
-#define qemu_printf printf
-
 int qemu_daemon(int nochdir, int noclose);
 void *qemu_try_memalign(size_t alignment, size_t size);
 void *qemu_memalign(size_t alignment, size_t size);
@@ -112,6 +181,8 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 
 #if defined(CONFIG_MADVISE)
 
+#include <sys/mman.h>
+
 #define QEMU_MADV_WILLNEED  MADV_WILLNEED
 #define QEMU_MADV_DONTNEED  MADV_DONTNEED
 #ifdef MADV_DONTFORK
@@ -144,6 +215,11 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #else
 #define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
 #endif
+#ifdef MADV_NOHUGEPAGE
+#define QEMU_MADV_NOHUGEPAGE MADV_NOHUGEPAGE
+#else
+#define QEMU_MADV_NOHUGEPAGE QEMU_MADV_INVALID
+#endif
 
 #elif defined(CONFIG_POSIX_MADVISE)
 
@@ -155,6 +231,7 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #else /* no-op */
 
@@ -166,6 +243,7 @@ void qemu_anon_ram_free(void *ptr, size_t size);
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #endif
 
@@ -220,8 +298,12 @@ static inline void qemu_timersub(const struct timeval *val1,
 
 void qemu_set_cloexec(int fd);
 
-void qemu_set_version(const char *);
-const char *qemu_get_version(void);
+/* QEMU "hardware version" setting. Used to replace code that exposed
+ * QEMU_VERSION to guests in the past and need to keep compatibilty.
+ * Do not use qemu_hw_version() in new code.
+ */
+void qemu_set_hw_version(const char *);
+const char *qemu_hw_version(void);
 
 void fips_set_state(bool requested);
 bool fips_get_state(void);
@@ -261,4 +343,18 @@ void os_mem_prealloc(int fd, char *area, size_t sz);
 
 int qemu_read_password(char *buf, int buf_size);
 
+/**
+ * qemu_fork:
+ *
+ * A version of fork that avoids signal handler race
+ * conditions that can lead to child process getting
+ * signals that are otherwise only expected by the
+ * parent. It also resets all signal handlers to the
+ * default settings.
+ *
+ * Returns 0 to child process, pid number to parent
+ * or -1 on failure.
+ */
+pid_t qemu_fork(Error **errp);
+
 #endif
diff --git a/qemu/include/qemu/path.h b/qemu/include/qemu/path.h
new file mode 100644
index 000000000..ed5fee086
--- /dev/null
+++ b/qemu/include/qemu/path.h
@@ -0,0 +1,7 @@
+#ifndef QEMU_PATH_H
+#define QEMU_PATH_H 1
+
+void init_paths(const char *prefix);
+const char *path(const char *pathname);
+
+#endif
diff --git a/qemu/include/qemu/queue.h b/qemu/include/qemu/queue.h
index a8d3cb8e6..f781aa20a 100644
--- a/qemu/include/qemu/queue.h
+++ b/qemu/include/qemu/queue.h
@@ -117,12 +117,6 @@ struct {                                                                \
         }                                                               \
 } while (/*CONSTCOND*/0)
 
-#define QLIST_FIX_HEAD_PTR(head, field) do {                            \
-        if ((head)->lh_first != NULL) {                                 \
-            (head)->lh_first->field.le_prev = &(head)->lh_first;        \
-        }                                                               \
-} while (/*CONSTCOND*/0)
-
 #define QLIST_INSERT_AFTER(listelm, elm, field) do {                    \
         if (((elm)->field.le_next = (listelm)->field.le_next) != NULL)  \
                 (listelm)->field.le_next->field.le_prev =               \
diff --git a/qemu/include/qemu/range.h b/qemu/include/qemu/range.h
index cfa021fd4..c903eb574 100644
--- a/qemu/include/qemu/range.h
+++ b/qemu/include/qemu/range.h
@@ -1,8 +1,6 @@
 #ifndef QEMU_RANGE_H
 #define QEMU_RANGE_H
 
-#include <inttypes.h>
-#include <qemu/typedefs.h>
 #include "qemu/queue.h"
 
 /*
diff --git a/qemu/include/qemu/rcu.h b/qemu/include/qemu/rcu.h
index 7df1e8662..56d3a682a 100644
--- a/qemu/include/qemu/rcu.h
+++ b/qemu/include/qemu/rcu.h
@@ -23,15 +23,8 @@
  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  */
 
-#include <stdlib.h>
-#include <assert.h>
-#include <limits.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <stdbool.h>
 #include <glib.h>
 
-#include "qemu/compiler.h"
 #include "qemu/thread.h"
 #include "qemu/queue.h"
 #include "qemu/atomic.h"
@@ -71,7 +64,7 @@ struct rcu_reader_data {
     /* Data used by reader only */
     unsigned depth;
 
-    /* Data used for registry, protected by rcu_gp_lock */
+    /* Data used for registry, protected by rcu_registry_lock */
     QLIST_ENTRY(rcu_reader_data) node;
 };
 
@@ -88,10 +81,6 @@ static inline void rcu_read_lock(void)
 
     ctr = atomic_read(&rcu_gp_ctr);
     atomic_xchg(&p_rcu_reader->ctr, ctr);
-    if (atomic_read(&p_rcu_reader->waiting)) {
-        atomic_set(&p_rcu_reader->waiting, false);
-        qemu_event_set(&rcu_gp_event);
-    }
 }
 
 static inline void rcu_read_unlock(void)
@@ -104,7 +93,7 @@ static inline void rcu_read_unlock(void)
     }
 
     atomic_xchg(&p_rcu_reader->ctr, 0);
-    if (atomic_read(&p_rcu_reader->waiting)) {
+    if (unlikely(atomic_read(&p_rcu_reader->waiting))) {
         atomic_set(&p_rcu_reader->waiting, false);
         qemu_event_set(&rcu_gp_event);
     }
diff --git a/qemu/include/qemu/seqlock.h b/qemu/include/qemu/seqlock.h
index 3ff118a1a..70b01fd60 100644
--- a/qemu/include/qemu/seqlock.h
+++ b/qemu/include/qemu/seqlock.h
@@ -55,18 +55,18 @@ static inline void seqlock_write_unlock(QemuSeqLock *sl)
 static inline unsigned seqlock_read_begin(QemuSeqLock *sl)
 {
     /* Always fail if a write is in progress.  */
-    unsigned ret = sl->sequence & ~1;
+    unsigned ret = atomic_read(&sl->sequence);
 
     /* Read sequence before reading other fields.  */
     smp_rmb();
-    return ret;
+    return ret & ~1;
 }
 
-static int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
+static inline int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
 {
     /* Read other fields before reading final sequence.  */
     smp_rmb();
-    return unlikely(sl->sequence != start);
+    return unlikely(atomic_read(&sl->sequence) != start);
 }
 
 #endif
diff --git a/qemu/include/qemu/sockets.h b/qemu/include/qemu/sockets.h
index c174b5cbd..1bd92180f 100644
--- a/qemu/include/qemu/sockets.h
+++ b/qemu/include/qemu/sockets.h
@@ -3,35 +3,13 @@
 #define QEMU_SOCKET_H
 
 #ifdef _WIN32
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-
-#define socket_error() WSAGetLastError()
 
 int inet_aton(const char *cp, struct in_addr *ia);
 
-#else
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <sys/un.h>
-
-#define socket_error() errno
-#define closesocket(s) close(s)
-
 #endif /* !_WIN32 */
 
-#include "qemu/option.h"
-#include "qapi/error.h"
 #include "qapi-types.h"
 
-extern QemuOptsList socket_optslist;
-
 /* misc helpers */
 int qemu_socket(int domain, int type, int protocol);
 int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
@@ -40,8 +18,6 @@ int socket_set_nodelay(int fd);
 void qemu_set_block(int fd);
 void qemu_set_nonblock(int fd);
 int socket_set_fast_reuse(int fd);
-int send_all(int fd, const void *buf, int len1);
-int recv_all(int fd, void *buf, int len1, bool single_read);
 
 #ifdef WIN32
 /* Windows has different names for the same constants with the same values */
@@ -53,26 +29,19 @@ int recv_all(int fd, void *buf, int len1, bool single_read);
 /* callback function for nonblocking connect
  * valid fd on success, negative error code on failure
  */
-typedef void NonBlockingConnectHandler(int fd, Error *errp, void *opaque);
+typedef void NonBlockingConnectHandler(int fd, Error *err, void *opaque);
 
 InetSocketAddress *inet_parse(const char *str, Error **errp);
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
 int inet_listen(const char *str, char *ostr, int olen,
                 int socktype, int port_offset, Error **errp);
-int inet_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque);
 int inet_connect(const char *str, Error **errp);
 int inet_nonblocking_connect(const char *str,
                              NonBlockingConnectHandler *callback,
                              void *opaque, Error **errp);
 
-int inet_dgram_opts(QemuOpts *opts, Error **errp);
 NetworkAddressFamily inet_netfamily(int family);
 
-int unix_listen_opts(QemuOpts *opts, Error **errp);
 int unix_listen(const char *path, char *ostr, int olen, Error **errp);
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque);
 int unix_connect(const char *path, Error **errp);
 int unix_nonblocking_connect(const char *str,
                              NonBlockingConnectHandler *callback,
@@ -88,4 +57,57 @@ int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp);
 int parse_host_port(struct sockaddr_in *saddr, const char *str);
 int socket_init(void);
 
+/**
+ * socket_sockaddr_to_address:
+ * @sa: socket address struct
+ * @salen: size of @sa struct
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *
+socket_sockaddr_to_address(struct sockaddr_storage *sa,
+                           socklen_t salen,
+                           Error **errp);
+
+/**
+ * socket_local_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_local_address(int fd, Error **errp);
+
+/**
+ * socket_remote_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the remote socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_remote_address(int fd, Error **errp);
+
+
+void qapi_copy_SocketAddress(SocketAddress **p_dest,
+                             SocketAddress *src);
+
 #endif /* QEMU_SOCKET_H */
diff --git a/qemu/include/qemu/thread-win32.h b/qemu/include/qemu/thread-win32.h
index 3d58081be..385ff5f76 100644
--- a/qemu/include/qemu/thread-win32.h
+++ b/qemu/include/qemu/thread-win32.h
@@ -18,6 +18,7 @@ struct QemuSemaphore {
 };
 
 struct QemuEvent {
+    int value;
     HANDLE event;
 };
 
diff --git a/qemu/include/qemu/thread.h b/qemu/include/qemu/thread.h
index 5114ec8e7..bdae6dfdb 100644
--- a/qemu/include/qemu/thread.h
+++ b/qemu/include/qemu/thread.h
@@ -1,8 +1,6 @@
 #ifndef __QEMU_THREAD_H
 #define __QEMU_THREAD_H 1
 
-#include <inttypes.h>
-#include <stdbool.h>
 
 typedef struct QemuMutex QemuMutex;
 typedef struct QemuCond QemuCond;
diff --git a/qemu/include/qemu/throttle.h b/qemu/include/qemu/throttle.h
index 995b2d595..910965760 100644
--- a/qemu/include/qemu/throttle.h
+++ b/qemu/include/qemu/throttle.h
@@ -2,7 +2,7 @@
  * QEMU throttling infrastructure
  *
  * Copyright (C) Nodalink, EURL. 2013-2014
- * Copyright (C) Igalia, S.L. 2015
+ * Copyright (C) Igalia, S.L. 2015-2016
  *
  * Authors:
  *   Benoît Canet <benoit.canet@nodalink.com>
@@ -25,10 +25,11 @@
 #ifndef THROTTLE_H
 #define THROTTLE_H
 
-#include <stdint.h>
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
+#define THROTTLE_VALUE_MAX 1000000000000000LL
+
 typedef enum {
     THROTTLE_BPS_TOTAL,
     THROTTLE_BPS_READ,
@@ -40,16 +41,47 @@ typedef enum {
 } BucketType;
 
 /*
- * The max parameter of the leaky bucket throttling algorithm can be used to
- * allow the guest to do bursts.
- * The max value is a pool of I/O that the guest can use without being throttled
- * at all. Throttling is triggered once this pool is empty.
+ * This module implements I/O limits using the leaky bucket
+ * algorithm. The code is independent of the I/O units, but it is
+ * currently used for bytes per second and operations per second.
+ *
+ * Three parameters can be set by the user:
+ *
+ * - avg: the desired I/O limits in units per second.
+ * - max: the limit during bursts, also in units per second.
+ * - burst_length: the maximum length of the burst period, in seconds.
+ *
+ * Here's how it works:
+ *
+ * - The bucket level (number of performed I/O units) is kept in
+ *   bkt.level and leaks at a rate of bkt.avg units per second.
+ *
+ * - The size of the bucket is bkt.max * bkt.burst_length. Once the
+ *   bucket is full no more I/O is performed until the bucket leaks
+ *   again. This is what makes the I/O rate bkt.avg.
+ *
+ * - The bkt.avg rate does not apply until the bucket is full,
+ *   allowing the user to do bursts until then. The I/O limit during
+ *   bursts is bkt.max. To enforce this limit we keep an additional
+ *   bucket in bkt.burst_length that leaks at a rate of bkt.max units
+ *   per second.
+ *
+ * - Because of all of the above, the user can perform I/O at a
+ *   maximum of bkt.max units per second for at most bkt.burst_length
+ *   seconds in a row. After that the bucket will be full and the I/O
+ *   rate will go down to bkt.avg.
+ *
+ * - Since the bucket always leaks at a rate of bkt.avg, this also
+ *   determines how much the user needs to wait before being able to
+ *   do bursts again.
  */
 
 typedef struct LeakyBucket {
     double  avg;              /* average goal in units per second */
     double  max;              /* leaky bucket max burst in units */
     double  level;            /* bucket level in units */
+    double  burst_level;      /* bucket level in units (for computing bursts) */
+    unsigned burst_length;    /* max length of the burst period, in seconds */
 } LeakyBucket;
 
 /* The following structure is used to configure a ThrottleState
@@ -82,12 +114,6 @@ void throttle_leak_bucket(LeakyBucket *bkt, int64_t delta);
 
 int64_t throttle_compute_wait(LeakyBucket *bkt);
 
-/* expose timer computation function for unit tests */
-bool throttle_compute_timer(ThrottleState *ts,
-                            bool is_write,
-                            int64_t now,
-                            int64_t *next_timestamp);
-
 /* init/destroy cycle */
 void throttle_init(ThrottleState *ts);
 
@@ -110,9 +136,7 @@ bool throttle_timers_are_initialized(ThrottleTimers *tt);
 /* configuration */
 bool throttle_enabled(ThrottleConfig *cfg);
 
-bool throttle_conflicting(ThrottleConfig *cfg);
-
-bool throttle_is_valid(ThrottleConfig *cfg);
+bool throttle_is_valid(ThrottleConfig *cfg, Error **errp);
 
 void throttle_config(ThrottleState *ts,
                      ThrottleTimers *tt,
@@ -120,6 +144,8 @@ void throttle_config(ThrottleState *ts,
 
 void throttle_get_config(ThrottleState *ts, ThrottleConfig *cfg);
 
+void throttle_config_init(ThrottleConfig *cfg);
+
 /* usage */
 bool throttle_schedule_timer(ThrottleState *ts,
                              ThrottleTimers *tt,
diff --git a/qemu/include/qemu/timed-average.h b/qemu/include/qemu/timed-average.h
new file mode 100644
index 000000000..08245e7a1
--- /dev/null
+++ b/qemu/include/qemu/timed-average.h
@@ -0,0 +1,63 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TIMED_AVERAGE_H
+#define TIMED_AVERAGE_H
+
+
+#include "qemu/timer.h"
+
+typedef struct TimedAverageWindow TimedAverageWindow;
+typedef struct TimedAverage TimedAverage;
+
+/* All fields of both structures are private */
+
+struct TimedAverageWindow {
+    uint64_t      min;             /* minimum value accounted in the window */
+    uint64_t      max;             /* maximum value accounted in the window */
+    uint64_t      sum;             /* sum of all values */
+    uint64_t      count;           /* number of values */
+    int64_t       expiration;      /* the end of the current window in ns */
+};
+
+struct TimedAverage {
+    uint64_t           period;     /* period in nanoseconds */
+    TimedAverageWindow windows[2]; /* two overlapping windows of with
+                                    * an offset of period / 2 between them */
+    unsigned           current;    /* the current window index: it's also the
+                                    * oldest window index */
+    QEMUClockType      clock_type; /* the clock used */
+};
+
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period);
+
+void timed_average_account(TimedAverage *ta, uint64_t value);
+
+uint64_t timed_average_min(TimedAverage *ta);
+uint64_t timed_average_avg(TimedAverage *ta);
+uint64_t timed_average_max(TimedAverage *ta);
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed);
+
+#endif
diff --git a/qemu/include/qemu/timer.h b/qemu/include/qemu/timer.h
index 5923d600f..471969a24 100644
--- a/qemu/include/qemu/timer.h
+++ b/qemu/include/qemu/timer.h
@@ -1,9 +1,9 @@
 #ifndef QEMU_TIMER_H
 #define QEMU_TIMER_H
 
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
+#include "qemu/host-utils.h"
 
 #define NANOSECONDS_PER_SECOND 1000000000LL
 
@@ -209,12 +209,11 @@ void qemu_clock_notify(QEMUClockType type);
 void qemu_clock_enable(QEMUClockType type, bool enabled);
 
 /**
- * qemu_clock_warp:
- * @type: the clock type
+ * qemu_start_warp_timer:
  *
- * Warp a clock to a new value
+ * Starts a timer for virtual clock update
  */
-void qemu_clock_warp(QEMUClockType type);
+void qemu_start_warp_timer(void);
 
 /**
  * qemu_clock_register_reset_notifier:
@@ -784,18 +783,13 @@ void cpu_enable_ticks(void);
 /* Caller must hold BQL */
 void cpu_disable_ticks(void);
 
-static inline int64_t get_ticks_per_sec(void)
-{
-    return 1000000000LL;
-}
-
 static inline int64_t get_max_clock_jump(void)
 {
     /* This should be small enough to prevent excessive interrupts from being
      * generated by the RTC on clock jumps, but large enough to avoid frequent
      * unnecessary resets in idle VMs.
      */
-    return 60 * get_ticks_per_sec();
+    return 60 * NANOSECONDS_PER_SECOND;
 }
 
 /*
@@ -821,7 +815,7 @@ static inline int64_t get_clock(void)
 {
     LARGE_INTEGER ti;
     QueryPerformanceCounter(&ti);
-    return muldiv64(ti.QuadPart, get_ticks_per_sec(), clock_freq);
+    return muldiv64(ti.QuadPart, NANOSECONDS_PER_SECOND, clock_freq);
 }
 
 #else
@@ -856,7 +850,7 @@ int64_t cpu_icount_to_ns(int64_t icount);
 
 #if defined(_ARCH_PPC)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t retval;
 #ifdef _ARCH_PPC64
@@ -882,7 +876,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__i386__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("rdtsc" : "=A" (val));
@@ -891,7 +885,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__x86_64__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint32_t low,high;
     int64_t val;
@@ -904,7 +898,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__hppa__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int val;
     asm volatile ("mfctl %%cr16, %0" : "=r"(val));
@@ -913,7 +907,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__ia64)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("mov %0 = ar.itc" : "=r"(val) :: "memory");
@@ -922,7 +916,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__s390__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile("stck 0(%1)" : "=m" (val) : "a" (&val) : "cc");
@@ -931,7 +925,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__sparc__)
 
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
 #if defined(_LP64)
     uint64_t        rval;
@@ -969,7 +963,7 @@ static inline int64_t cpu_get_real_ticks (void)
                               : "=r" (value));          \
     }
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     /* On kernels >= 2.6.25 rdhwr <reg>, $2 and $3 are emulated */
     uint32_t count;
@@ -985,7 +979,7 @@ static inline int64_t cpu_get_real_ticks(void)
 
 #elif defined(__alpha__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint64_t cc;
     uint32_t cur, ofs;
@@ -1000,7 +994,7 @@ static inline int64_t cpu_get_real_ticks(void)
 /* The host CPU doesn't have an easily accessible cycle counter.
    Just return a monotonically increasing value.  This will be
    totally wrong, but hopefully better than nothing.  */
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
     static int64_t ticks = 0;
     return ticks++;
diff --git a/qemu/include/qemu/tls.h b/qemu/include/qemu/tls.h
deleted file mode 100644
index b92ea9d7d..000000000
--- a/qemu/include/qemu/tls.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Abstraction layer for defining and using TLS variables
- *
- * Copyright (c) 2011 Red Hat, Inc
- * Copyright (c) 2011 Linaro Limited
- *
- * Authors:
- *  Paolo Bonzini <pbonzini@redhat.com>
- *  Peter Maydell <peter.maydell@linaro.org>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef QEMU_TLS_H
-#define QEMU_TLS_H
-
-/* Per-thread variables. Note that we only have implementations
- * which are really thread-local on Linux; the dummy implementations
- * define plain global variables.
- *
- * This means that for the moment use should be restricted to
- * per-VCPU variables, which are OK because:
- *  - the only -user mode supporting multiple VCPU threads is linux-user
- *  - TCG system mode is single-threaded regarding VCPUs
- *  - KVM system mode is multi-threaded but limited to Linux
- *
- * TODO: proper implementations via Win32 .tls sections and
- * POSIX pthread_getspecific.
- */
-#ifdef __linux__
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __thread __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#else
-/* Dummy implementations which define plain global variables */
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#endif
-
-#endif
diff --git a/qemu/include/qemu/typedefs.h b/qemu/include/qemu/typedefs.h
index 6fdcbcd52..1dcf6f5d5 100644
--- a/qemu/include/qemu/typedefs.h
+++ b/qemu/include/qemu/typedefs.h
@@ -3,25 +3,30 @@
 
 /* A load of opaque types so that device init declarations don't have to
    pull in all the real definitions.  */
-struct Monitor;
 
 /* Please keep this list in alphabetical order */
 typedef struct AdapterInfo AdapterInfo;
 typedef struct AddressSpace AddressSpace;
 typedef struct AioContext AioContext;
+typedef struct AllwinnerAHCIState AllwinnerAHCIState;
 typedef struct AudioState AudioState;
+typedef struct BdrvDirtyBitmap BdrvDirtyBitmap;
 typedef struct BlockBackend BlockBackend;
+typedef struct BlockBackendRootState BlockBackendRootState;
 typedef struct BlockDriverState BlockDriverState;
 typedef struct BusClass BusClass;
 typedef struct BusState BusState;
 typedef struct CharDriverState CharDriverState;
 typedef struct CompatProperty CompatProperty;
-typedef struct DeviceState DeviceState;
+typedef struct CPUAddressSpace CPUAddressSpace;
+typedef struct CPUState CPUState;
 typedef struct DeviceListener DeviceListener;
+typedef struct DeviceState DeviceState;
 typedef struct DisplayChangeListener DisplayChangeListener;
 typedef struct DisplayState DisplayState;
 typedef struct DisplaySurface DisplaySurface;
 typedef struct DriveInfo DriveInfo;
+typedef struct Error Error;
 typedef struct EventNotifier EventNotifier;
 typedef struct FWCfgIoState FWCfgIoState;
 typedef struct FWCfgMemState FWCfgMemState;
@@ -31,6 +36,7 @@ typedef struct I2CBus I2CBus;
 typedef struct I2SCodec I2SCodec;
 typedef struct ISABus ISABus;
 typedef struct ISADevice ISADevice;
+typedef struct IsaDma IsaDma;
 typedef struct LoadStateEntry LoadStateEntry;
 typedef struct MACAddr MACAddr;
 typedef struct MachineClass MachineClass;
@@ -41,10 +47,13 @@ typedef struct MemoryRegion MemoryRegion;
 typedef struct MemoryRegionSection MemoryRegionSection;
 typedef struct MigrationIncomingState MigrationIncomingState;
 typedef struct MigrationParams MigrationParams;
+typedef struct MigrationState MigrationState;
 typedef struct Monitor Monitor;
+typedef struct MonitorDef MonitorDef;
 typedef struct MouseTransformInfo MouseTransformInfo;
 typedef struct MSIMessage MSIMessage;
 typedef struct NetClientState NetClientState;
+typedef struct NetFilterState NetFilterState;
 typedef struct NICInfo NICInfo;
 typedef struct PcGuestInfo PcGuestInfo;
 typedef struct PCIBridge PCIBridge;
@@ -57,19 +66,27 @@ typedef struct PCIEPort PCIEPort;
 typedef struct PCIESlot PCIESlot;
 typedef struct PCIExpressDevice PCIExpressDevice;
 typedef struct PCIExpressHost PCIExpressHost;
+typedef struct PCIHostDeviceAddress PCIHostDeviceAddress;
 typedef struct PCIHostState PCIHostState;
+typedef struct PCMachineClass PCMachineClass;
+typedef struct PCMachineState PCMachineState;
 typedef struct PCMCIACardState PCMCIACardState;
 typedef struct PixelFormat PixelFormat;
-typedef struct PropertyInfo PropertyInfo;
+typedef struct PostcopyDiscardState PostcopyDiscardState;
 typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
 typedef struct QEMUBH QEMUBH;
 typedef struct QemuConsole QemuConsole;
 typedef struct QEMUFile QEMUFile;
-typedef struct QEMUMachine QEMUMachine;
+typedef struct QemuOpt QemuOpt;
+typedef struct QemuOpts QemuOpts;
+typedef struct QemuOptsList QemuOptsList;
 typedef struct QEMUSGList QEMUSGList;
 typedef struct QEMUSizedBuffer QEMUSizedBuffer;
-typedef struct QEMUTimerListGroup QEMUTimerListGroup;
 typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUTimerListGroup QEMUTimerListGroup;
+typedef struct QObject QObject;
+typedef struct RAMBlock RAMBlock;
 typedef struct Range Range;
 typedef struct SerialState SerialState;
 typedef struct SHPCDevice SHPCDevice;
diff --git a/qemu/include/qemu/unicode.h b/qemu/include/qemu/unicode.h
new file mode 100644
index 000000000..d8731652d
--- /dev/null
+++ b/qemu/include/qemu/unicode.h
@@ -0,0 +1,6 @@
+#ifndef QEMU_UNICODE_H
+#define QEMU_UNICODE_H 1
+
+int mod_utf8_codepoint(const char *s, size_t n, char **end);
+
+#endif
diff --git a/qemu/include/qemu/xattr.h b/qemu/include/qemu/xattr.h
index f910d96ea..83cf98cbd 100644
--- a/qemu/include/qemu/xattr.h
+++ b/qemu/include/qemu/xattr.h
@@ -18,7 +18,6 @@
  * in /usr/include/sys, and don't have ENOATTR.
  */
 
-#include "config-host.h"
 
 #ifdef CONFIG_LIBATTR
 #  include <attr/xattr.h>
diff --git a/qemu/include/qom/cpu.h b/qemu/include/qom/cpu.h
index 20aabc9cb..b7a10f791 100644
--- a/qemu/include/qom/cpu.h
+++ b/qemu/include/qom/cpu.h
@@ -20,16 +20,12 @@
 #ifndef QEMU_CPU_H
 #define QEMU_CPU_H
 
-#include <signal.h>
-#include <setjmp.h>
 #include "hw/qdev-core.h"
 #include "disas/bfd.h"
 #include "exec/hwaddr.h"
 #include "exec/memattrs.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
-#include "qemu/tls.h"
-#include "qemu/typedefs.h"
 
 typedef int (*WriteCoreDumpFunction)(const void *buf, size_t size,
                                      void *opaque);
@@ -64,7 +60,7 @@ typedef uint64_t vaddr;
 #define CPU_CLASS(class) OBJECT_CLASS_CHECK(CPUClass, (class), TYPE_CPU)
 #define CPU_GET_CLASS(obj) OBJECT_GET_CLASS(CPUClass, (obj), TYPE_CPU)
 
-typedef struct CPUState CPUState;
+typedef struct CPUWatchpoint CPUWatchpoint;
 
 typedef void (*CPUUnassignedAccess)(CPUState *cpu, hwaddr addr,
                                     bool is_write, bool is_exec, int opaque,
@@ -99,8 +95,16 @@ struct TranslationBlock;
  * #TranslationBlock.
  * @handle_mmu_fault: Callback for handling an MMU fault.
  * @get_phys_page_debug: Callback for obtaining a physical address.
+ * @get_phys_page_attrs_debug: Callback for obtaining a physical address and the
+ *       associated memory transaction attributes to use for the access.
+ *       CPUs which use memory transaction attributes should implement this
+ *       instead of get_phys_page_debug.
+ * @asidx_from_attrs: Callback to return the CPU AddressSpace to use for
+ *       a memory access with the specified memory transaction attributes.
  * @gdb_read_register: Callback for letting GDB read a register.
  * @gdb_write_register: Callback for letting GDB write a register.
+ * @debug_check_watchpoint: Callback: return true if the architectural
+ *       watchpoint whose address has matched should really fire.
  * @debug_excp_handler: Callback for handling debug exceptions.
  * @write_elf64_note: Callback for writing a CPU-specific ELF note to a
  * 64-bit VM coredump.
@@ -115,6 +119,8 @@ struct TranslationBlock;
  * @gdb_core_xml_file: File name for core registers GDB XML description.
  * @gdb_stop_before_watchpoint: Indicates whether GDB expects the CPU to stop
  *           before the insn which triggers a watchpoint rather than after it.
+ * @gdb_arch_name: Optional callback that returns the architecture name known
+ * to GDB. The caller must free the returned string with g_free.
  * @cpu_exec_enter: Callback for cpu_exec preparation.
  * @cpu_exec_exit: Callback for cpu_exec cleanup.
  * @cpu_exec_interrupt: Callback for processing interrupts in cpu_exec.
@@ -153,8 +159,12 @@ typedef struct CPUClass {
     int (*handle_mmu_fault)(CPUState *cpu, vaddr address, int rw,
                             int mmu_index);
     hwaddr (*get_phys_page_debug)(CPUState *cpu, vaddr addr);
+    hwaddr (*get_phys_page_attrs_debug)(CPUState *cpu, vaddr addr,
+                                        MemTxAttrs *attrs);
+    int (*asidx_from_attrs)(CPUState *cpu, MemTxAttrs attrs);
     int (*gdb_read_register)(CPUState *cpu, uint8_t *buf, int reg);
     int (*gdb_write_register)(CPUState *cpu, uint8_t *buf, int reg);
+    bool (*debug_check_watchpoint)(CPUState *cpu, CPUWatchpoint *wp);
     void (*debug_excp_handler)(CPUState *cpu);
 
     int (*write_elf64_note)(WriteCoreDumpFunction f, CPUState *cpu,
@@ -169,6 +179,7 @@ typedef struct CPUClass {
     const struct VMStateDescription *vmsd;
     int gdb_num_core_regs;
     const char *gdb_core_xml_file;
+    gchar * (*gdb_arch_name)(CPUState *cpu);
     bool gdb_stop_before_watchpoint;
 
     void (*cpu_exec_enter)(CPUState *cpu);
@@ -196,14 +207,14 @@ typedef struct CPUBreakpoint {
     QTAILQ_ENTRY(CPUBreakpoint) entry;
 } CPUBreakpoint;
 
-typedef struct CPUWatchpoint {
+struct CPUWatchpoint {
     vaddr vaddr;
     vaddr len;
     vaddr hitaddr;
     MemTxAttrs hitattrs;
     int flags; /* BP_* */
     QTAILQ_ENTRY(CPUWatchpoint) entry;
-} CPUWatchpoint;
+};
 
 struct KVMState;
 struct kvm_run;
@@ -224,6 +235,7 @@ struct kvm_run;
  * @halted: Nonzero if the CPU is in suspended state.
  * @stop: Indicates a pending stop request.
  * @stopped: Indicates the CPU has been artificially stopped.
+ * @crash_occurred: Indicates the OS reported a crash (panic) for this CPU
  * @tcg_exit_req: Set to force TCG to stop executing linked TBs for this
  *           CPU and return to its top level loop.
  * @singlestep_enabled: Flags for single-stepping.
@@ -231,7 +243,14 @@ struct kvm_run;
  * @icount_decr: Number of cycles left, with interrupt flag in high bit.
  * This allows a single read-compare-cbranch-write sequence to test
  * for both decrementer underflow and exceptions.
- * @can_do_io: Nonzero if memory-mapped IO is safe.
+ * @can_do_io: Nonzero if memory-mapped IO is safe. Deterministic execution
+ * requires that IO only be performed on the last instruction of a TB
+ * so that interrupts take effect immediately.
+ * @cpu_ases: Pointer to array of CPUAddressSpaces (which define the
+ *            AddressSpaces this CPU has)
+ * @num_ases: number of CPUAddressSpaces in @cpu_ases
+ * @as: Pointer to the first AddressSpace, for the convenience of targets which
+ *      only have a single AddressSpace
  * @env_ptr: Pointer to subclass-specific CPUArchState field.
  * @current_tb: Currently executing TB.
  * @gdb_regs: Additional GDB registers.
@@ -242,6 +261,8 @@ struct kvm_run;
  * @mem_io_pc: Host Program Counter at which the memory was accessed.
  * @mem_io_vaddr: Target virtual address at which the memory was accessed.
  * @kvm_fd: vCPU file descriptor for KVM.
+ * @work_mutex: Lock to prevent multiple access to queued_work_*.
+ * @queued_work_first: First asynchronous work pending.
  *
  * State of one CPU core or thread.
  */
@@ -262,20 +283,24 @@ struct CPUState {
     uint32_t host_tid;
     bool running;
     struct QemuCond *halt_cond;
-    struct qemu_work_item *queued_work_first, *queued_work_last;
     bool thread_kicked;
     bool created;
     bool stop;
     bool stopped;
-    volatile sig_atomic_t exit_request;
+    bool crash_occurred;
+    bool exit_request;
     uint32_t interrupt_request;
     int singlestep_enabled;
     int64_t icount_extra;
     sigjmp_buf jmp_env;
 
+    QemuMutex work_mutex;
+    struct qemu_work_item *queued_work_first, *queued_work_last;
+
+    CPUAddressSpace *cpu_ases;
+    int num_ases;
     AddressSpace *as;
-    struct AddressSpaceDispatch *memory_dispatch;
-    MemoryListener *tcg_as_listener;
+    MemoryRegion *memory;
 
     void *env_ptr; /* CPUArchState */
     struct TranslationBlock *current_tb;
@@ -314,11 +339,16 @@ struct CPUState {
     uint32_t can_do_io;
     int32_t exception_index; /* used by m68k TCG */
 
+    /* Used to keep track of an outstanding cpu throttle thread for migration
+     * autoconverge
+     */
+    bool throttle_thread_scheduled;
+
     /* Note that this is accessed at the start of every TB via a negative
        offset from AREG0.  Leave this field at the end so as to make the
        (absolute value) offset as small as possible.  This reduces code
        size, especially for hosts without large memory offsets.  */
-    volatile sig_atomic_t tcg_exit_req;
+    uint32_t tcg_exit_req;
 };
 
 QTAILQ_HEAD(CPUTailQ, CPUState);
@@ -331,8 +361,7 @@ extern struct CPUTailQ cpus;
     QTAILQ_FOREACH_REVERSE(cpu, &cpus, CPUTailQ, node)
 #define first_cpu QTAILQ_FIRST(&cpus)
 
-DECLARE_TLS(CPUState *, current_cpu);
-#define current_cpu tls_var(current_cpu)
+extern __thread CPUState *current_cpu;
 
 /**
  * cpu_paging_enabled:
@@ -429,6 +458,32 @@ void cpu_dump_statistics(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
 
 #ifndef CONFIG_USER_ONLY
 /**
+ * cpu_get_phys_page_attrs_debug:
+ * @cpu: The CPU to obtain the physical page address for.
+ * @addr: The virtual address.
+ * @attrs: Updated on return with the memory transaction attributes to use
+ *         for this access.
+ *
+ * Obtains the physical page corresponding to a virtual one, together
+ * with the corresponding memory transaction attributes to use for the access.
+ * Use it only for debugging because no protection checks are done.
+ *
+ * Returns: Corresponding physical page address or -1 if no page found.
+ */
+static inline hwaddr cpu_get_phys_page_attrs_debug(CPUState *cpu, vaddr addr,
+                                                   MemTxAttrs *attrs)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->get_phys_page_attrs_debug) {
+        return cc->get_phys_page_attrs_debug(cpu, addr, attrs);
+    }
+    /* Fallback for CPUs which don't implement the _attrs_ hook */
+    *attrs = MEMTXATTRS_UNSPECIFIED;
+    return cc->get_phys_page_debug(cpu, addr);
+}
+
+/**
  * cpu_get_phys_page_debug:
  * @cpu: The CPU to obtain the physical page address for.
  * @addr: The virtual address.
@@ -440,9 +495,26 @@ void cpu_dump_statistics(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
  */
 static inline hwaddr cpu_get_phys_page_debug(CPUState *cpu, vaddr addr)
 {
+    MemTxAttrs attrs = {};
+
+    return cpu_get_phys_page_attrs_debug(cpu, addr, &attrs);
+}
+
+/** cpu_asidx_from_attrs:
+ * @cpu: CPU
+ * @attrs: memory transaction attributes
+ *
+ * Returns the address space index specifying the CPU AddressSpace
+ * to use for a memory access with the given transaction attributes.
+ */
+static inline int cpu_asidx_from_attrs(CPUState *cpu, MemTxAttrs attrs)
+{
     CPUClass *cc = CPU_GET_CLASS(cpu);
 
-    return cc->get_phys_page_debug(cpu, addr);
+    if (cc->asidx_from_attrs) {
+        return cc->asidx_from_attrs(cpu, attrs);
+    }
+    return 0;
 }
 #endif
 
@@ -559,6 +631,43 @@ CPUState *qemu_get_cpu(int index);
  */
 bool cpu_exists(int64_t id);
 
+/**
+ * cpu_throttle_set:
+ * @new_throttle_pct: Percent of sleep time. Valid range is 1 to 99.
+ *
+ * Throttles all vcpus by forcing them to sleep for the given percentage of
+ * time. A throttle_percentage of 25 corresponds to a 75% duty cycle roughly.
+ * (example: 10ms sleep for every 30ms awake).
+ *
+ * cpu_throttle_set can be called as needed to adjust new_throttle_pct.
+ * Once the throttling starts, it will remain in effect until cpu_throttle_stop
+ * is called.
+ */
+void cpu_throttle_set(int new_throttle_pct);
+
+/**
+ * cpu_throttle_stop:
+ *
+ * Stops the vcpu throttling started by cpu_throttle_set.
+ */
+void cpu_throttle_stop(void);
+
+/**
+ * cpu_throttle_active:
+ *
+ * Returns: %true if the vcpus are currently being throttled, %false otherwise.
+ */
+bool cpu_throttle_active(void);
+
+/**
+ * cpu_throttle_get_percentage:
+ *
+ * Returns the vcpu throttle percentage. See cpu_throttle_set for details.
+ *
+ * Returns: The throttle percentage in range 1 to 99.
+ */
+int cpu_throttle_get_percentage(void);
+
 #ifndef CONFIG_USER_ONLY
 
 typedef void (*CPUInterruptHandler)(CPUState *, int);
@@ -673,6 +782,7 @@ void cpu_single_step(CPUState *cpu, int enabled);
 /* 0x08 currently unused */
 #define BP_GDB                0x10
 #define BP_CPU                0x20
+#define BP_ANY                (BP_GDB | BP_CPU)
 #define BP_WATCHPOINT_HIT_READ 0x40
 #define BP_WATCHPOINT_HIT_WRITE 0x80
 #define BP_WATCHPOINT_HIT (BP_WATCHPOINT_HIT_READ | BP_WATCHPOINT_HIT_WRITE)
@@ -683,6 +793,21 @@ int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags);
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint);
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask);
 
+/* Return true if PC matches an installed breakpoint.  */
+static inline bool cpu_breakpoint_test(CPUState *cpu, vaddr pc, int mask)
+{
+    CPUBreakpoint *bp;
+
+    if (unlikely(!QTAILQ_EMPTY(&cpu->breakpoints))) {
+        QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
+            if (bp->pc == pc && (bp->flags & mask)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint);
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr,
diff --git a/qemu/include/qom/object.h b/qemu/include/qom/object.h
index 807978eec..21bb5ff14 100644
--- a/qemu/include/qom/object.h
+++ b/qemu/include/qom/object.h
@@ -15,12 +15,8 @@
 #define QEMU_OBJECT_H
 
 #include <glib.h>
-#include <stdint.h>
-#include <stdbool.h>
+#include "qapi-types.h"
 #include "qemu/queue.h"
-#include "qapi/error.h"
-
-struct Visitor;
 
 struct TypeImpl;
 typedef struct TypeImpl *Type;
@@ -291,16 +287,16 @@ typedef struct InterfaceInfo InterfaceInfo;
  * ObjectPropertyAccessor:
  * @obj: the object that owns the property
  * @v: the visitor that contains the property data
- * @opaque: the object property opaque
  * @name: the name of the property
+ * @opaque: the object property opaque
  * @errp: a pointer to an Error that is filled if getting/setting fails.
  *
  * Called when trying to get/set a property.
  */
 typedef void (ObjectPropertyAccessor)(Object *obj,
-                                      struct Visitor *v,
-                                      void *opaque,
+                                      Visitor *v,
                                       const char *name,
+                                      void *opaque,
                                       Error **errp);
 
 /**
@@ -344,8 +340,6 @@ typedef struct ObjectProperty
     ObjectPropertyResolve *resolve;
     ObjectPropertyRelease *release;
     void *opaque;
-
-    QTAILQ_ENTRY(ObjectProperty) node;
 } ObjectProperty;
 
 /**
@@ -383,6 +377,8 @@ struct ObjectClass
     const char *class_cast_cache[OBJECT_CLASS_CAST_CACHE];
 
     ObjectUnparent *unparent;
+
+    GHashTable *properties;
 };
 
 /**
@@ -396,16 +392,13 @@ struct ObjectClass
  * As a result, #Object contains a reference to the objects type as its
  * first member.  This allows identification of the real type of the object at
  * run time.
- *
- * #Object also contains a list of #Interfaces that this object
- * implements.
  */
 struct Object
 {
     /*< private >*/
     ObjectClass *class;
     ObjectFree *free;
-    QTAILQ_HEAD(, ObjectProperty) properties;
+    GHashTable *properties;
     uint32_t ref;
     Object *parent;
 };
@@ -510,16 +503,16 @@ struct TypeInfo
 
 /**
  * OBJECT_CLASS_CHECK:
- * @class: The C type to use for the return value.
- * @obj: A derivative of @type to cast.
- * @name: the QOM typename of @class.
+ * @class_type: The C type to use for the return value.
+ * @class: A derivative class of @class_type to cast.
+ * @name: the QOM typename of @class_type.
  *
  * A type safe version of @object_class_dynamic_cast_assert.  This macro is
  * typically wrapped by each type to perform type safe casts of a class to a
  * specific class type.
  */
-#define OBJECT_CLASS_CHECK(class, obj, name) \
-    ((class *)object_class_dynamic_cast_assert(OBJECT_CLASS(obj), (name), \
+#define OBJECT_CLASS_CHECK(class_type, class, name) \
+    ((class_type *)object_class_dynamic_cast_assert(OBJECT_CLASS(class), (name), \
                                                __FILE__, __LINE__, __func__))
 
 /**
@@ -949,6 +942,13 @@ ObjectProperty *object_property_add(Object *obj, const char *name,
 
 void object_property_del(Object *obj, const char *name, Error **errp);
 
+ObjectProperty *object_class_property_add(ObjectClass *klass, const char *name,
+                                          const char *type,
+                                          ObjectPropertyAccessor *get,
+                                          ObjectPropertyAccessor *set,
+                                          ObjectPropertyRelease *release,
+                                          void *opaque, Error **errp);
+
 /**
  * object_property_find:
  * @obj: the object
@@ -959,6 +959,55 @@ void object_property_del(Object *obj, const char *name, Error **errp);
  */
 ObjectProperty *object_property_find(Object *obj, const char *name,
                                      Error **errp);
+ObjectProperty *object_class_property_find(ObjectClass *klass, const char *name,
+                                           Error **errp);
+
+typedef struct ObjectPropertyIterator {
+    ObjectClass *nextclass;
+    GHashTableIter iter;
+} ObjectPropertyIterator;
+
+/**
+ * object_property_iter_init:
+ * @obj: the object
+ *
+ * Initializes an iterator for traversing all properties
+ * registered against an object instance, its class and all parent classes.
+ *
+ * It is forbidden to modify the property list while iterating,
+ * whether removing or adding properties.
+ *
+ * Typical usage pattern would be
+ *
+ * <example>
+ *   <title>Using object property iterators</title>
+ *   <programlisting>
+ *   ObjectProperty *prop;
+ *   ObjectPropertyIterator iter;
+ *
+ *   object_property_iter_init(&iter, obj);
+ *   while ((prop = object_property_iter_next(&iter))) {
+ *     ... do something with prop ...
+ *   }
+ *   </programlisting>
+ * </example>
+ */
+void object_property_iter_init(ObjectPropertyIterator *iter,
+                               Object *obj);
+
+/**
+ * object_property_iter_next:
+ * @iter: the iterator instance
+ *
+ * Return the next available property. If no further properties
+ * are available, a %NULL value will be returned and the @iter
+ * pointer should not be used again after this point without
+ * re-initializing it.
+ *
+ * Returns: the next property, or %NULL when all properties
+ * have been traversed.
+ */
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter);
 
 void object_unparent(Object *obj);
 
@@ -972,7 +1021,7 @@ void object_unparent(Object *obj);
  *
  * Reads a property from a object.
  */
-void object_property_get(Object *obj, struct Visitor *v, const char *name,
+void object_property_get(Object *obj, Visitor *v, const char *name,
                          Error **errp);
 
 /**
@@ -1063,7 +1112,7 @@ void object_property_set_int(Object *obj, int64_t value,
  * @name: the name of the property
  * @errp: returns an error if this function fails
  *
- * Returns: the value of the property, converted to an integer, or NULL if
+ * Returns: the value of the property, converted to an integer, or negative if
  * an error occurs (including when the property value is not an integer).
  */
 int64_t object_property_get_int(Object *obj, const char *name,
@@ -1108,7 +1157,7 @@ void object_property_get_uint16List(Object *obj, const char *name,
  *
  * Writes a property to a object.
  */
-void object_property_set(Object *obj, struct Visitor *v, const char *name,
+void object_property_set(Object *obj, Visitor *v, const char *name,
                          Error **errp);
 
 /**
@@ -1327,6 +1376,12 @@ void object_property_add_str(Object *obj, const char *name,
                              void (*set)(Object *, const char *, Error **),
                              Error **errp);
 
+void object_class_property_add_str(ObjectClass *klass, const char *name,
+                                   char *(*get)(Object *, Error **),
+                                   void (*set)(Object *, const char *,
+                                               Error **),
+                                   Error **errp);
+
 /**
  * object_property_add_bool:
  * @obj: the object to add a property to
@@ -1343,6 +1398,11 @@ void object_property_add_bool(Object *obj, const char *name,
                               void (*set)(Object *, bool, Error **),
                               Error **errp);
 
+void object_class_property_add_bool(ObjectClass *klass, const char *name,
+                                    bool (*get)(Object *, Error **),
+                                    void (*set)(Object *, bool, Error **),
+                                    Error **errp);
+
 /**
  * object_property_add_enum:
  * @obj: the object to add a property to
@@ -1362,6 +1422,13 @@ void object_property_add_enum(Object *obj, const char *name,
                               void (*set)(Object *, int, Error **),
                               Error **errp);
 
+void object_class_property_add_enum(ObjectClass *klass, const char *name,
+                                    const char *typename,
+                                    const char * const *strings,
+                                    int (*get)(Object *, Error **),
+                                    void (*set)(Object *, int, Error **),
+                                    Error **errp);
+
 /**
  * object_property_add_tm:
  * @obj: the object to add a property to
@@ -1376,6 +1443,10 @@ void object_property_add_tm(Object *obj, const char *name,
                             void (*get)(Object *, struct tm *, Error **),
                             Error **errp);
 
+void object_class_property_add_tm(ObjectClass *klass, const char *name,
+                                  void (*get)(Object *, struct tm *, Error **),
+                                  Error **errp);
+
 /**
  * object_property_add_uint8_ptr:
  * @obj: the object to add a property to
@@ -1388,6 +1459,8 @@ void object_property_add_tm(Object *obj, const char *name,
  */
 void object_property_add_uint8_ptr(Object *obj, const char *name,
                                    const uint8_t *v, Error **errp);
+void object_class_property_add_uint8_ptr(ObjectClass *klass, const char *name,
+                                         const uint8_t *v, Error **errp);
 
 /**
  * object_property_add_uint16_ptr:
@@ -1401,6 +1474,8 @@ void object_property_add_uint8_ptr(Object *obj, const char *name,
  */
 void object_property_add_uint16_ptr(Object *obj, const char *name,
                                     const uint16_t *v, Error **errp);
+void object_class_property_add_uint16_ptr(ObjectClass *klass, const char *name,
+                                          const uint16_t *v, Error **errp);
 
 /**
  * object_property_add_uint32_ptr:
@@ -1414,6 +1489,8 @@ void object_property_add_uint16_ptr(Object *obj, const char *name,
  */
 void object_property_add_uint32_ptr(Object *obj, const char *name,
                                     const uint32_t *v, Error **errp);
+void object_class_property_add_uint32_ptr(ObjectClass *klass, const char *name,
+                                          const uint32_t *v, Error **errp);
 
 /**
  * object_property_add_uint64_ptr:
@@ -1427,6 +1504,8 @@ void object_property_add_uint32_ptr(Object *obj, const char *name,
  */
 void object_property_add_uint64_ptr(Object *obj, const char *name,
                                     const uint64_t *v, Error **Errp);
+void object_class_property_add_uint64_ptr(ObjectClass *klass, const char *name,
+                                          const uint64_t *v, Error **Errp);
 
 /**
  * object_property_add_alias:
@@ -1478,6 +1557,9 @@ void object_property_add_const_link(Object *obj, const char *name,
  */
 void object_property_set_description(Object *obj, const char *name,
                                      const char *description, Error **errp);
+void object_class_property_set_description(ObjectClass *klass, const char *name,
+                                           const char *description,
+                                           Error **errp);
 
 /**
  * object_child_foreach:
@@ -1488,12 +1570,33 @@ void object_property_set_description(Object *obj, const char *name,
  * Call @fn passing each child of @obj and @opaque to it, until @fn returns
  * non-zero.
  *
+ * It is forbidden to add or remove children from @obj from the @fn
+ * callback.
+ *
  * Returns: The last value returned by @fn, or 0 if there is no child.
  */
 int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
                          void *opaque);
 
 /**
+ * object_child_foreach_recursive:
+ * @obj: the object whose children will be navigated
+ * @fn: the iterator function to be called
+ * @opaque: an opaque value that will be passed to the iterator
+ *
+ * Call @fn passing each child of @obj and @opaque to it, until @fn returns
+ * non-zero. Calls recursively, all child nodes of @obj will also be passed
+ * all the way down to the leaf nodes of the tree. Depth first ordering.
+ *
+ * It is forbidden to add or remove children from @obj (or its
+ * child nodes) from the @fn callback.
+ *
+ * Returns: The last value returned by @fn, or 0 if there is no child.
+ */
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque);
+/**
  * container_get:
  * @root: root of the #path, e.g., object_get_root()
  * @path: path to the container
diff --git a/qemu/include/qom/object_interfaces.h b/qemu/include/qom/object_interfaces.h
index 283ae0db4..8b17f4def 100644
--- a/qemu/include/qom/object_interfaces.h
+++ b/qemu/include/qom/object_interfaces.h
@@ -2,6 +2,8 @@
 #define OBJECT_INTERFACES_H
 
 #include "qom/object.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/visitor.h"
 
 #define TYPE_USER_CREATABLE "user-creatable"
 
@@ -72,4 +74,95 @@ void user_creatable_complete(Object *obj, Error **errp);
  * from implements USER_CREATABLE interface.
  */
 bool user_creatable_can_be_deleted(UserCreatable *uc, Error **errp);
+
+/**
+ * user_creatable_add:
+ * @qdict: the object definition
+ * @v: the visitor
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object whose type
+ * is defined in @qdict by the 'qom-type' field, placing it
+ * in the object composition tree with name provided by the
+ * 'id' field. The remaining fields in @qdict are used to
+ * initialize the object properties.
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add(const QDict *qdict,
+                           Visitor *v, Error **errp);
+
+/**
+ * user_creatable_add_type:
+ * @type: the object type name
+ * @id: the unique ID for the object
+ * @qdict: the object properties
+ * @v: the visitor
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object @type, placing
+ * it in the object composition tree with name @id, initializing
+ * it with properties from @qdict
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add_type(const char *type, const char *id,
+                                const QDict *qdict,
+                                Visitor *v, Error **errp);
+
+/**
+ * user_creatable_add_opts:
+ * @opts: the object definition
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object whose type
+ * is defined in @opts by the 'qom-type' option, placing it
+ * in the object composition tree with name provided by the
+ * 'id' field. The remaining options in @opts are used to
+ * initialize the object properties.
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add_opts(QemuOpts *opts, Error **errp);
+
+
+/**
+ * user_creatable_add_opts_predicate:
+ * @type: the QOM type to be added
+ *
+ * A callback function to determine whether an object
+ * of type @type should be created. Instances of this
+ * callback should be passed to user_creatable_add_opts_foreach
+ */
+typedef bool (*user_creatable_add_opts_predicate)(const char *type);
+
+/**
+ * user_creatable_add_opts_foreach:
+ * @opaque: a user_creatable_add_opts_predicate callback or NULL
+ * @opts: options to create
+ * @errp: unused
+ *
+ * An iterator callback to be used in conjunction with
+ * the qemu_opts_foreach() method for creating a list of
+ * objects from a set of QemuOpts
+ *
+ * The @opaque parameter can be passed a user_creatable_add_opts_predicate
+ * callback to filter which types of object are created during iteration.
+ * When it fails, report the error.
+ *
+ * Returns: 0 on success, -1 when an error was reported.
+ */
+int user_creatable_add_opts_foreach(void *opaque,
+                                    QemuOpts *opts, Error **errp);
+
+/**
+ * user_creatable_del:
+ * @id: the unique ID for the object
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Delete an instance of the user creatable object identified
+ * by @id.
+ */
+void user_creatable_del(const char *id, Error **errp);
+
 #endif
diff --git a/qemu/include/standard-headers/asm-x86/hyperv.h b/qemu/include/standard-headers/asm-x86/hyperv.h
new file mode 100644
index 000000000..47b38fb81
--- /dev/null
+++ b/qemu/include/standard-headers/asm-x86/hyperv.h
@@ -0,0 +1,366 @@
+#ifndef _ASM_X86_HYPERV_H
+#define _ASM_X86_HYPERV_H
+
+#include "standard-headers/linux/types.h"
+
+/*
+ * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
+ * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
+ */
+#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
+#define HYPERV_CPUID_INTERFACE			0x40000001
+#define HYPERV_CPUID_VERSION			0x40000002
+#define HYPERV_CPUID_FEATURES			0x40000003
+#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
+#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
+
+#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
+#define HYPERV_CPUID_MIN			0x40000005
+#define HYPERV_CPUID_MAX			0x4000ffff
+
+/*
+ * Feature identification. EAX indicates which features are available
+ * to the partition based upon the current partition privileges.
+ */
+
+/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
+#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
+/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
+#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
+/* Partition reference TSC MSR is available */
+#define HV_X64_MSR_REFERENCE_TSC_AVAILABLE              (1 << 9)
+
+/* A partition's reference time stamp counter (TSC) page */
+#define HV_X64_MSR_REFERENCE_TSC		0x40000021
+
+/*
+ * There is a single feature flag that signifies the presence of the MSR
+ * that can be used to retrieve both the local APIC Timer frequency as
+ * well as the TSC frequency.
+ */
+
+/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
+#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
+
+/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
+#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
+
+/*
+ * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
+ * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
+ */
+#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
+/*
+ * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
+ * HV_X64_MSR_STIMER3_COUNT) available
+ */
+#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
+/*
+ * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
+ * are available
+ */
+#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
+/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
+#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
+/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
+#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
+/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
+#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
+ /*
+  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
+  */
+#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
+
+/*
+ * Feature identification: EBX indicates which flags were specified at
+ * partition creation. The format is the same as the partition creation
+ * flag structure defined in section Partition Creation Flags.
+ */
+#define HV_X64_CREATE_PARTITIONS		(1 << 0)
+#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
+#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
+#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
+#define HV_X64_POST_MESSAGES			(1 << 4)
+#define HV_X64_SIGNAL_EVENTS			(1 << 5)
+#define HV_X64_CREATE_PORT			(1 << 6)
+#define HV_X64_CONNECT_PORT			(1 << 7)
+#define HV_X64_ACCESS_STATS			(1 << 8)
+#define HV_X64_DEBUGGING			(1 << 11)
+#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
+#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
+
+/*
+ * Feature identification. EDX indicates which miscellaneous features
+ * are available to the partition.
+ */
+/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
+#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
+/* Guest debugging support is available */
+#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
+/* Performance Monitor support is available*/
+#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
+/* Support for physical CPU dynamic partitioning events is available*/
+#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
+/*
+ * Support for passing hypercall input parameter block via XMM
+ * registers is available
+ */
+#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
+/* Support for a virtual guest idle state is available */
+#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
+/* Guest crash data handler available */
+#define HV_X64_GUEST_CRASH_MSR_AVAILABLE		(1 << 10)
+
+/*
+ * Implementation recommendations. Indicates which behaviors the hypervisor
+ * recommends the OS implement for optimal performance.
+ */
+ /*
+  * Recommend using hypercall for address space switches rather
+  * than MOV to CR3 instruction
+  */
+#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
+/* Recommend using hypercall for local TLB flushes rather
+ * than INVLPG or MOV to CR3 instructions */
+#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
+/*
+ * Recommend using hypercall for remote TLB flushes rather
+ * than inter-processor interrupts
+ */
+#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
+/*
+ * Recommend using MSRs for accessing APIC registers
+ * EOI, ICR and TPR rather than their memory-mapped counterparts
+ */
+#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
+/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
+#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
+/*
+ * Recommend using relaxed timing for this partition. If used,
+ * the VM should disable any watchdog timeouts that rely on the
+ * timely delivery of external interrupts
+ */
+#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
+
+/* MSR used to identify the guest OS. */
+#define HV_X64_MSR_GUEST_OS_ID			0x40000000
+
+/* MSR used to setup pages used to communicate with the hypervisor. */
+#define HV_X64_MSR_HYPERCALL			0x40000001
+
+/* MSR used to provide vcpu index */
+#define HV_X64_MSR_VP_INDEX			0x40000002
+
+/* MSR used to reset the guest OS. */
+#define HV_X64_MSR_RESET			0x40000003
+
+/* MSR used to provide vcpu runtime in 100ns units */
+#define HV_X64_MSR_VP_RUNTIME			0x40000010
+
+/* MSR used to read the per-partition time reference counter */
+#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
+
+/* MSR used to retrieve the TSC frequency */
+#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
+
+/* MSR used to retrieve the local APIC timer frequency */
+#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
+
+/* Define the virtual APIC registers */
+#define HV_X64_MSR_EOI				0x40000070
+#define HV_X64_MSR_ICR				0x40000071
+#define HV_X64_MSR_TPR				0x40000072
+#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
+
+/* Define synthetic interrupt controller model specific registers. */
+#define HV_X64_MSR_SCONTROL			0x40000080
+#define HV_X64_MSR_SVERSION			0x40000081
+#define HV_X64_MSR_SIEFP			0x40000082
+#define HV_X64_MSR_SIMP				0x40000083
+#define HV_X64_MSR_EOM				0x40000084
+#define HV_X64_MSR_SINT0			0x40000090
+#define HV_X64_MSR_SINT1			0x40000091
+#define HV_X64_MSR_SINT2			0x40000092
+#define HV_X64_MSR_SINT3			0x40000093
+#define HV_X64_MSR_SINT4			0x40000094
+#define HV_X64_MSR_SINT5			0x40000095
+#define HV_X64_MSR_SINT6			0x40000096
+#define HV_X64_MSR_SINT7			0x40000097
+#define HV_X64_MSR_SINT8			0x40000098
+#define HV_X64_MSR_SINT9			0x40000099
+#define HV_X64_MSR_SINT10			0x4000009A
+#define HV_X64_MSR_SINT11			0x4000009B
+#define HV_X64_MSR_SINT12			0x4000009C
+#define HV_X64_MSR_SINT13			0x4000009D
+#define HV_X64_MSR_SINT14			0x4000009E
+#define HV_X64_MSR_SINT15			0x4000009F
+
+/*
+ * Synthetic Timer MSRs. Four timers per vcpu.
+ */
+#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
+#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
+#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
+#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
+#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
+#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
+#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
+#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
+
+/* Hyper-V guest crash notification MSR's */
+#define HV_X64_MSR_CRASH_P0			0x40000100
+#define HV_X64_MSR_CRASH_P1			0x40000101
+#define HV_X64_MSR_CRASH_P2			0x40000102
+#define HV_X64_MSR_CRASH_P3			0x40000103
+#define HV_X64_MSR_CRASH_P4			0x40000104
+#define HV_X64_MSR_CRASH_CTL			0x40000105
+#define HV_X64_MSR_CRASH_CTL_NOTIFY		(1ULL << 63)
+#define HV_X64_MSR_CRASH_PARAMS		\
+		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
+
+#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
+
+/* Declare the various hypercall operations. */
+#define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
+#define HVCALL_POST_MESSAGE			0x005c
+#define HVCALL_SIGNAL_EVENT			0x005d
+
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
+
+#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
+#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
+
+#define HV_PROCESSOR_POWER_STATE_C0		0
+#define HV_PROCESSOR_POWER_STATE_C1		1
+#define HV_PROCESSOR_POWER_STATE_C2		2
+#define HV_PROCESSOR_POWER_STATE_C3		3
+
+/* hypercall status code */
+#define HV_STATUS_SUCCESS			0
+#define HV_STATUS_INVALID_HYPERCALL_CODE	2
+#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
+#define HV_STATUS_INVALID_ALIGNMENT		4
+#define HV_STATUS_INSUFFICIENT_MEMORY		11
+#define HV_STATUS_INVALID_CONNECTION_ID		18
+#define HV_STATUS_INSUFFICIENT_BUFFERS		19
+
+typedef struct _HV_REFERENCE_TSC_PAGE {
+	uint32_t tsc_sequence;
+	uint32_t res1;
+	uint64_t tsc_scale;
+	int64_t tsc_offset;
+} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
+
+/* Define the number of synthetic interrupt sources. */
+#define HV_SYNIC_SINT_COUNT		(16)
+/* Define the expected SynIC version. */
+#define HV_SYNIC_VERSION_1		(0x1)
+
+#define HV_SYNIC_CONTROL_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SIMP_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SIEFP_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SINT_MASKED		(1ULL << 16)
+#define HV_SYNIC_SINT_AUTO_EOI		(1ULL << 17)
+#define HV_SYNIC_SINT_VECTOR_MASK	(0xFF)
+
+#define HV_SYNIC_STIMER_COUNT		(4)
+
+/* Define synthetic interrupt controller message constants. */
+#define HV_MESSAGE_SIZE			(256)
+#define HV_MESSAGE_PAYLOAD_BYTE_COUNT	(240)
+#define HV_MESSAGE_PAYLOAD_QWORD_COUNT	(30)
+
+/* Define hypervisor message types. */
+enum hv_message_type {
+	HVMSG_NONE			= 0x00000000,
+
+	/* Memory access messages. */
+	HVMSG_UNMAPPED_GPA		= 0x80000000,
+	HVMSG_GPA_INTERCEPT		= 0x80000001,
+
+	/* Timer notification messages. */
+	HVMSG_TIMER_EXPIRED			= 0x80000010,
+
+	/* Error messages. */
+	HVMSG_INVALID_VP_REGISTER_VALUE	= 0x80000020,
+	HVMSG_UNRECOVERABLE_EXCEPTION	= 0x80000021,
+	HVMSG_UNSUPPORTED_FEATURE		= 0x80000022,
+
+	/* Trace buffer complete messages. */
+	HVMSG_EVENTLOG_BUFFERCOMPLETE	= 0x80000040,
+
+	/* Platform-specific processor intercept messages. */
+	HVMSG_X64_IOPORT_INTERCEPT		= 0x80010000,
+	HVMSG_X64_MSR_INTERCEPT		= 0x80010001,
+	HVMSG_X64_CPUID_INTERCEPT		= 0x80010002,
+	HVMSG_X64_EXCEPTION_INTERCEPT	= 0x80010003,
+	HVMSG_X64_APIC_EOI			= 0x80010004,
+	HVMSG_X64_LEGACY_FP_ERROR		= 0x80010005
+};
+
+/* Define synthetic interrupt controller message flags. */
+union hv_message_flags {
+	uint8_t asu8;
+	struct {
+		uint8_t msg_pending:1;
+		uint8_t reserved:7;
+	};
+};
+
+/* Define port identifier type. */
+union hv_port_id {
+	uint32_t asu32;
+	struct {
+		uint32_t id:24;
+		uint32_t reserved:8;
+	} u;
+};
+
+/* Define synthetic interrupt controller message header. */
+struct hv_message_header {
+	uint32_t message_type;
+	uint8_t payload_size;
+	union hv_message_flags message_flags;
+	uint8_t reserved[2];
+	union {
+		uint64_t sender;
+		union hv_port_id port;
+	};
+};
+
+/* Define synthetic interrupt controller message format. */
+struct hv_message {
+	struct hv_message_header header;
+	union {
+		uint64_t payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
+	} u;
+};
+
+/* Define the synthetic interrupt message page layout. */
+struct hv_message_page {
+	struct hv_message sint_message[HV_SYNIC_SINT_COUNT];
+};
+
+/* Define timer message payload structure. */
+struct hv_timer_message_payload {
+	uint32_t timer_index;
+	uint32_t reserved;
+	uint64_t expiration_time;	/* When the timer expired */
+	uint64_t delivery_time;	/* When the message was delivered */
+};
+
+#define HV_STIMER_ENABLE		(1ULL << 0)
+#define HV_STIMER_PERIODIC		(1ULL << 1)
+#define HV_STIMER_LAZY			(1ULL << 2)
+#define HV_STIMER_AUTOENABLE		(1ULL << 3)
+#define HV_STIMER_SINT(config)		(uint8_t)(((config) >> 16) & 0x0F)
+
+#endif
diff --git a/qemu/include/standard-headers/linux/input-event-codes.h b/qemu/include/standard-headers/linux/input-event-codes.h
new file mode 100644
index 000000000..354f0decf
--- /dev/null
+++ b/qemu/include/standard-headers/linux/input-event-codes.h
@@ -0,0 +1,805 @@
+/*
+ * Input event codes
+ *
+ *    *** IMPORTANT ***
+ * This file is not only included from C-code but also from devicetree source
+ * files. As such this file MUST only contain comments and defines.
+ *
+ * Copyright (c) 1999-2002 Vojtech Pavlik
+ * Copyright (c) 2015 Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+#ifndef _INPUT_EVENT_CODES_H
+#define _INPUT_EVENT_CODES_H
+
+/*
+ * Device properties and quirks
+ */
+
+#define INPUT_PROP_POINTER		0x00	/* needs a pointer */
+#define INPUT_PROP_DIRECT		0x01	/* direct input devices */
+#define INPUT_PROP_BUTTONPAD		0x02	/* has button(s) under pad */
+#define INPUT_PROP_SEMI_MT		0x03	/* touch rectangle only */
+#define INPUT_PROP_TOPBUTTONPAD		0x04	/* softbuttons at top of pad */
+#define INPUT_PROP_POINTING_STICK	0x05	/* is a pointing stick */
+#define INPUT_PROP_ACCELEROMETER	0x06	/* has accelerometer */
+
+#define INPUT_PROP_MAX			0x1f
+#define INPUT_PROP_CNT			(INPUT_PROP_MAX + 1)
+
+/*
+ * Event types
+ */
+
+#define EV_SYN			0x00
+#define EV_KEY			0x01
+#define EV_REL			0x02
+#define EV_ABS			0x03
+#define EV_MSC			0x04
+#define EV_SW			0x05
+#define EV_LED			0x11
+#define EV_SND			0x12
+#define EV_REP			0x14
+#define EV_FF			0x15
+#define EV_PWR			0x16
+#define EV_FF_STATUS		0x17
+#define EV_MAX			0x1f
+#define EV_CNT			(EV_MAX+1)
+
+/*
+ * Synchronization events.
+ */
+
+#define SYN_REPORT		0
+#define SYN_CONFIG		1
+#define SYN_MT_REPORT		2
+#define SYN_DROPPED		3
+#define SYN_MAX			0xf
+#define SYN_CNT			(SYN_MAX+1)
+
+/*
+ * Keys and buttons
+ *
+ * Most of the keys/buttons are modeled after USB HUT 1.12
+ * (see http://www.usb.org/developers/hidpage).
+ * Abbreviations in the comments:
+ * AC - Application Control
+ * AL - Application Launch Button
+ * SC - System Control
+ */
+
+#define KEY_RESERVED		0
+#define KEY_ESC			1
+#define KEY_1			2
+#define KEY_2			3
+#define KEY_3			4
+#define KEY_4			5
+#define KEY_5			6
+#define KEY_6			7
+#define KEY_7			8
+#define KEY_8			9
+#define KEY_9			10
+#define KEY_0			11
+#define KEY_MINUS		12
+#define KEY_EQUAL		13
+#define KEY_BACKSPACE		14
+#define KEY_TAB			15
+#define KEY_Q			16
+#define KEY_W			17
+#define KEY_E			18
+#define KEY_R			19
+#define KEY_T			20
+#define KEY_Y			21
+#define KEY_U			22
+#define KEY_I			23
+#define KEY_O			24
+#define KEY_P			25
+#define KEY_LEFTBRACE		26
+#define KEY_RIGHTBRACE		27
+#define KEY_ENTER		28
+#define KEY_LEFTCTRL		29
+#define KEY_A			30
+#define KEY_S			31
+#define KEY_D			32
+#define KEY_F			33
+#define KEY_G			34
+#define KEY_H			35
+#define KEY_J			36
+#define KEY_K			37
+#define KEY_L			38
+#define KEY_SEMICOLON		39
+#define KEY_APOSTROPHE		40
+#define KEY_GRAVE		41
+#define KEY_LEFTSHIFT		42
+#define KEY_BACKSLASH		43
+#define KEY_Z			44
+#define KEY_X			45
+#define KEY_C			46
+#define KEY_V			47
+#define KEY_B			48
+#define KEY_N			49
+#define KEY_M			50
+#define KEY_COMMA		51
+#define KEY_DOT			52
+#define KEY_SLASH		53
+#define KEY_RIGHTSHIFT		54
+#define KEY_KPASTERISK		55
+#define KEY_LEFTALT		56
+#define KEY_SPACE		57
+#define KEY_CAPSLOCK		58
+#define KEY_F1			59
+#define KEY_F2			60
+#define KEY_F3			61
+#define KEY_F4			62
+#define KEY_F5			63
+#define KEY_F6			64
+#define KEY_F7			65
+#define KEY_F8			66
+#define KEY_F9			67
+#define KEY_F10			68
+#define KEY_NUMLOCK		69
+#define KEY_SCROLLLOCK		70
+#define KEY_KP7			71
+#define KEY_KP8			72
+#define KEY_KP9			73
+#define KEY_KPMINUS		74
+#define KEY_KP4			75
+#define KEY_KP5			76
+#define KEY_KP6			77
+#define KEY_KPPLUS		78
+#define KEY_KP1			79
+#define KEY_KP2			80
+#define KEY_KP3			81
+#define KEY_KP0			82
+#define KEY_KPDOT		83
+
+#define KEY_ZENKAKUHANKAKU	85
+#define KEY_102ND		86
+#define KEY_F11			87
+#define KEY_F12			88
+#define KEY_RO			89
+#define KEY_KATAKANA		90
+#define KEY_HIRAGANA		91
+#define KEY_HENKAN		92
+#define KEY_KATAKANAHIRAGANA	93
+#define KEY_MUHENKAN		94
+#define KEY_KPJPCOMMA		95
+#define KEY_KPENTER		96
+#define KEY_RIGHTCTRL		97
+#define KEY_KPSLASH		98
+#define KEY_SYSRQ		99
+#define KEY_RIGHTALT		100
+#define KEY_LINEFEED		101
+#define KEY_HOME		102
+#define KEY_UP			103
+#define KEY_PAGEUP		104
+#define KEY_LEFT		105
+#define KEY_RIGHT		106
+#define KEY_END			107
+#define KEY_DOWN		108
+#define KEY_PAGEDOWN		109
+#define KEY_INSERT		110
+#define KEY_DELETE		111
+#define KEY_MACRO		112
+#define KEY_MUTE		113
+#define KEY_VOLUMEDOWN		114
+#define KEY_VOLUMEUP		115
+#define KEY_POWER		116	/* SC System Power Down */
+#define KEY_KPEQUAL		117
+#define KEY_KPPLUSMINUS		118
+#define KEY_PAUSE		119
+#define KEY_SCALE		120	/* AL Compiz Scale (Expose) */
+
+#define KEY_KPCOMMA		121
+#define KEY_HANGEUL		122
+#define KEY_HANGUEL		KEY_HANGEUL
+#define KEY_HANJA		123
+#define KEY_YEN			124
+#define KEY_LEFTMETA		125
+#define KEY_RIGHTMETA		126
+#define KEY_COMPOSE		127
+
+#define KEY_STOP		128	/* AC Stop */
+#define KEY_AGAIN		129
+#define KEY_PROPS		130	/* AC Properties */
+#define KEY_UNDO		131	/* AC Undo */
+#define KEY_FRONT		132
+#define KEY_COPY		133	/* AC Copy */
+#define KEY_OPEN		134	/* AC Open */
+#define KEY_PASTE		135	/* AC Paste */
+#define KEY_FIND		136	/* AC Search */
+#define KEY_CUT			137	/* AC Cut */
+#define KEY_HELP		138	/* AL Integrated Help Center */
+#define KEY_MENU		139	/* Menu (show menu) */
+#define KEY_CALC		140	/* AL Calculator */
+#define KEY_SETUP		141
+#define KEY_SLEEP		142	/* SC System Sleep */
+#define KEY_WAKEUP		143	/* System Wake Up */
+#define KEY_FILE		144	/* AL Local Machine Browser */
+#define KEY_SENDFILE		145
+#define KEY_DELETEFILE		146
+#define KEY_XFER		147
+#define KEY_PROG1		148
+#define KEY_PROG2		149
+#define KEY_WWW			150	/* AL Internet Browser */
+#define KEY_MSDOS		151
+#define KEY_COFFEE		152	/* AL Terminal Lock/Screensaver */
+#define KEY_SCREENLOCK		KEY_COFFEE
+#define KEY_ROTATE_DISPLAY	153	/* Display orientation for e.g. tablets */
+#define KEY_DIRECTION		KEY_ROTATE_DISPLAY
+#define KEY_CYCLEWINDOWS	154
+#define KEY_MAIL		155
+#define KEY_BOOKMARKS		156	/* AC Bookmarks */
+#define KEY_COMPUTER		157
+#define KEY_BACK		158	/* AC Back */
+#define KEY_FORWARD		159	/* AC Forward */
+#define KEY_CLOSECD		160
+#define KEY_EJECTCD		161
+#define KEY_EJECTCLOSECD	162
+#define KEY_NEXTSONG		163
+#define KEY_PLAYPAUSE		164
+#define KEY_PREVIOUSSONG	165
+#define KEY_STOPCD		166
+#define KEY_RECORD		167
+#define KEY_REWIND		168
+#define KEY_PHONE		169	/* Media Select Telephone */
+#define KEY_ISO			170
+#define KEY_CONFIG		171	/* AL Consumer Control Configuration */
+#define KEY_HOMEPAGE		172	/* AC Home */
+#define KEY_REFRESH		173	/* AC Refresh */
+#define KEY_EXIT		174	/* AC Exit */
+#define KEY_MOVE		175
+#define KEY_EDIT		176
+#define KEY_SCROLLUP		177
+#define KEY_SCROLLDOWN		178
+#define KEY_KPLEFTPAREN		179
+#define KEY_KPRIGHTPAREN	180
+#define KEY_NEW			181	/* AC New */
+#define KEY_REDO		182	/* AC Redo/Repeat */
+
+#define KEY_F13			183
+#define KEY_F14			184
+#define KEY_F15			185
+#define KEY_F16			186
+#define KEY_F17			187
+#define KEY_F18			188
+#define KEY_F19			189
+#define KEY_F20			190
+#define KEY_F21			191
+#define KEY_F22			192
+#define KEY_F23			193
+#define KEY_F24			194
+
+#define KEY_PLAYCD		200
+#define KEY_PAUSECD		201
+#define KEY_PROG3		202
+#define KEY_PROG4		203
+#define KEY_DASHBOARD		204	/* AL Dashboard */
+#define KEY_SUSPEND		205
+#define KEY_CLOSE		206	/* AC Close */
+#define KEY_PLAY		207
+#define KEY_FASTFORWARD		208
+#define KEY_BASSBOOST		209
+#define KEY_PRINT		210	/* AC Print */
+#define KEY_HP			211
+#define KEY_CAMERA		212
+#define KEY_SOUND		213
+#define KEY_QUESTION		214
+#define KEY_EMAIL		215
+#define KEY_CHAT		216
+#define KEY_SEARCH		217
+#define KEY_CONNECT		218
+#define KEY_FINANCE		219	/* AL Checkbook/Finance */
+#define KEY_SPORT		220
+#define KEY_SHOP		221
+#define KEY_ALTERASE		222
+#define KEY_CANCEL		223	/* AC Cancel */
+#define KEY_BRIGHTNESSDOWN	224
+#define KEY_BRIGHTNESSUP	225
+#define KEY_MEDIA		226
+
+#define KEY_SWITCHVIDEOMODE	227	/* Cycle between available video
+					   outputs (Monitor/LCD/TV-out/etc) */
+#define KEY_KBDILLUMTOGGLE	228
+#define KEY_KBDILLUMDOWN	229
+#define KEY_KBDILLUMUP		230
+
+#define KEY_SEND		231	/* AC Send */
+#define KEY_REPLY		232	/* AC Reply */
+#define KEY_FORWARDMAIL		233	/* AC Forward Msg */
+#define KEY_SAVE		234	/* AC Save */
+#define KEY_DOCUMENTS		235
+
+#define KEY_BATTERY		236
+
+#define KEY_BLUETOOTH		237
+#define KEY_WLAN		238
+#define KEY_UWB			239
+
+#define KEY_UNKNOWN		240
+
+#define KEY_VIDEO_NEXT		241	/* drive next video source */
+#define KEY_VIDEO_PREV		242	/* drive previous video source */
+#define KEY_BRIGHTNESS_CYCLE	243	/* brightness up, after max is min */
+#define KEY_BRIGHTNESS_AUTO	244	/* Set Auto Brightness: manual
+					  brightness control is off,
+					  rely on ambient */
+#define KEY_BRIGHTNESS_ZERO	KEY_BRIGHTNESS_AUTO
+#define KEY_DISPLAY_OFF		245	/* display device to off state */
+
+#define KEY_WWAN		246	/* Wireless WAN (LTE, UMTS, GSM, etc.) */
+#define KEY_WIMAX		KEY_WWAN
+#define KEY_RFKILL		247	/* Key that controls all radios */
+
+#define KEY_MICMUTE		248	/* Mute / unmute the microphone */
+
+/* Code 255 is reserved for special needs of AT keyboard driver */
+
+#define BTN_MISC		0x100
+#define BTN_0			0x100
+#define BTN_1			0x101
+#define BTN_2			0x102
+#define BTN_3			0x103
+#define BTN_4			0x104
+#define BTN_5			0x105
+#define BTN_6			0x106
+#define BTN_7			0x107
+#define BTN_8			0x108
+#define BTN_9			0x109
+
+#define BTN_MOUSE		0x110
+#define BTN_LEFT		0x110
+#define BTN_RIGHT		0x111
+#define BTN_MIDDLE		0x112
+#define BTN_SIDE		0x113
+#define BTN_EXTRA		0x114
+#define BTN_FORWARD		0x115
+#define BTN_BACK		0x116
+#define BTN_TASK		0x117
+
+#define BTN_JOYSTICK		0x120
+#define BTN_TRIGGER		0x120
+#define BTN_THUMB		0x121
+#define BTN_THUMB2		0x122
+#define BTN_TOP			0x123
+#define BTN_TOP2		0x124
+#define BTN_PINKIE		0x125
+#define BTN_BASE		0x126
+#define BTN_BASE2		0x127
+#define BTN_BASE3		0x128
+#define BTN_BASE4		0x129
+#define BTN_BASE5		0x12a
+#define BTN_BASE6		0x12b
+#define BTN_DEAD		0x12f
+
+#define BTN_GAMEPAD		0x130
+#define BTN_SOUTH		0x130
+#define BTN_A			BTN_SOUTH
+#define BTN_EAST		0x131
+#define BTN_B			BTN_EAST
+#define BTN_C			0x132
+#define BTN_NORTH		0x133
+#define BTN_X			BTN_NORTH
+#define BTN_WEST		0x134
+#define BTN_Y			BTN_WEST
+#define BTN_Z			0x135
+#define BTN_TL			0x136
+#define BTN_TR			0x137
+#define BTN_TL2			0x138
+#define BTN_TR2			0x139
+#define BTN_SELECT		0x13a
+#define BTN_START		0x13b
+#define BTN_MODE		0x13c
+#define BTN_THUMBL		0x13d
+#define BTN_THUMBR		0x13e
+
+#define BTN_DIGI		0x140
+#define BTN_TOOL_PEN		0x140
+#define BTN_TOOL_RUBBER		0x141
+#define BTN_TOOL_BRUSH		0x142
+#define BTN_TOOL_PENCIL		0x143
+#define BTN_TOOL_AIRBRUSH	0x144
+#define BTN_TOOL_FINGER		0x145
+#define BTN_TOOL_MOUSE		0x146
+#define BTN_TOOL_LENS		0x147
+#define BTN_TOOL_QUINTTAP	0x148	/* Five fingers on trackpad */
+#define BTN_TOUCH		0x14a
+#define BTN_STYLUS		0x14b
+#define BTN_STYLUS2		0x14c
+#define BTN_TOOL_DOUBLETAP	0x14d
+#define BTN_TOOL_TRIPLETAP	0x14e
+#define BTN_TOOL_QUADTAP	0x14f	/* Four fingers on trackpad */
+
+#define BTN_WHEEL		0x150
+#define BTN_GEAR_DOWN		0x150
+#define BTN_GEAR_UP		0x151
+
+#define KEY_OK			0x160
+#define KEY_SELECT		0x161
+#define KEY_GOTO		0x162
+#define KEY_CLEAR		0x163
+#define KEY_POWER2		0x164
+#define KEY_OPTION		0x165
+#define KEY_INFO		0x166	/* AL OEM Features/Tips/Tutorial */
+#define KEY_TIME		0x167
+#define KEY_VENDOR		0x168
+#define KEY_ARCHIVE		0x169
+#define KEY_PROGRAM		0x16a	/* Media Select Program Guide */
+#define KEY_CHANNEL		0x16b
+#define KEY_FAVORITES		0x16c
+#define KEY_EPG			0x16d
+#define KEY_PVR			0x16e	/* Media Select Home */
+#define KEY_MHP			0x16f
+#define KEY_LANGUAGE		0x170
+#define KEY_TITLE		0x171
+#define KEY_SUBTITLE		0x172
+#define KEY_ANGLE		0x173
+#define KEY_ZOOM		0x174
+#define KEY_MODE		0x175
+#define KEY_KEYBOARD		0x176
+#define KEY_SCREEN		0x177
+#define KEY_PC			0x178	/* Media Select Computer */
+#define KEY_TV			0x179	/* Media Select TV */
+#define KEY_TV2			0x17a	/* Media Select Cable */
+#define KEY_VCR			0x17b	/* Media Select VCR */
+#define KEY_VCR2		0x17c	/* VCR Plus */
+#define KEY_SAT			0x17d	/* Media Select Satellite */
+#define KEY_SAT2		0x17e
+#define KEY_CD			0x17f	/* Media Select CD */
+#define KEY_TAPE		0x180	/* Media Select Tape */
+#define KEY_RADIO		0x181
+#define KEY_TUNER		0x182	/* Media Select Tuner */
+#define KEY_PLAYER		0x183
+#define KEY_TEXT		0x184
+#define KEY_DVD			0x185	/* Media Select DVD */
+#define KEY_AUX			0x186
+#define KEY_MP3			0x187
+#define KEY_AUDIO		0x188	/* AL Audio Browser */
+#define KEY_VIDEO		0x189	/* AL Movie Browser */
+#define KEY_DIRECTORY		0x18a
+#define KEY_LIST		0x18b
+#define KEY_MEMO		0x18c	/* Media Select Messages */
+#define KEY_CALENDAR		0x18d
+#define KEY_RED			0x18e
+#define KEY_GREEN		0x18f
+#define KEY_YELLOW		0x190
+#define KEY_BLUE		0x191
+#define KEY_CHANNELUP		0x192	/* Channel Increment */
+#define KEY_CHANNELDOWN		0x193	/* Channel Decrement */
+#define KEY_FIRST		0x194
+#define KEY_LAST		0x195	/* Recall Last */
+#define KEY_AB			0x196
+#define KEY_NEXT		0x197
+#define KEY_RESTART		0x198
+#define KEY_SLOW		0x199
+#define KEY_SHUFFLE		0x19a
+#define KEY_BREAK		0x19b
+#define KEY_PREVIOUS		0x19c
+#define KEY_DIGITS		0x19d
+#define KEY_TEEN		0x19e
+#define KEY_TWEN		0x19f
+#define KEY_VIDEOPHONE		0x1a0	/* Media Select Video Phone */
+#define KEY_GAMES		0x1a1	/* Media Select Games */
+#define KEY_ZOOMIN		0x1a2	/* AC Zoom In */
+#define KEY_ZOOMOUT		0x1a3	/* AC Zoom Out */
+#define KEY_ZOOMRESET		0x1a4	/* AC Zoom */
+#define KEY_WORDPROCESSOR	0x1a5	/* AL Word Processor */
+#define KEY_EDITOR		0x1a6	/* AL Text Editor */
+#define KEY_SPREADSHEET		0x1a7	/* AL Spreadsheet */
+#define KEY_GRAPHICSEDITOR	0x1a8	/* AL Graphics Editor */
+#define KEY_PRESENTATION	0x1a9	/* AL Presentation App */
+#define KEY_DATABASE		0x1aa	/* AL Database App */
+#define KEY_NEWS		0x1ab	/* AL Newsreader */
+#define KEY_VOICEMAIL		0x1ac	/* AL Voicemail */
+#define KEY_ADDRESSBOOK		0x1ad	/* AL Contacts/Address Book */
+#define KEY_MESSENGER		0x1ae	/* AL Instant Messaging */
+#define KEY_DISPLAYTOGGLE	0x1af	/* Turn display (LCD) on and off */
+#define KEY_BRIGHTNESS_TOGGLE	KEY_DISPLAYTOGGLE
+#define KEY_SPELLCHECK		0x1b0   /* AL Spell Check */
+#define KEY_LOGOFF		0x1b1   /* AL Logoff */
+
+#define KEY_DOLLAR		0x1b2
+#define KEY_EURO		0x1b3
+
+#define KEY_FRAMEBACK		0x1b4	/* Consumer - transport controls */
+#define KEY_FRAMEFORWARD	0x1b5
+#define KEY_CONTEXT_MENU	0x1b6	/* GenDesc - system context menu */
+#define KEY_MEDIA_REPEAT	0x1b7	/* Consumer - transport control */
+#define KEY_10CHANNELSUP	0x1b8	/* 10 channels up (10+) */
+#define KEY_10CHANNELSDOWN	0x1b9	/* 10 channels down (10-) */
+#define KEY_IMAGES		0x1ba	/* AL Image Browser */
+
+#define KEY_DEL_EOL		0x1c0
+#define KEY_DEL_EOS		0x1c1
+#define KEY_INS_LINE		0x1c2
+#define KEY_DEL_LINE		0x1c3
+
+#define KEY_FN			0x1d0
+#define KEY_FN_ESC		0x1d1
+#define KEY_FN_F1		0x1d2
+#define KEY_FN_F2		0x1d3
+#define KEY_FN_F3		0x1d4
+#define KEY_FN_F4		0x1d5
+#define KEY_FN_F5		0x1d6
+#define KEY_FN_F6		0x1d7
+#define KEY_FN_F7		0x1d8
+#define KEY_FN_F8		0x1d9
+#define KEY_FN_F9		0x1da
+#define KEY_FN_F10		0x1db
+#define KEY_FN_F11		0x1dc
+#define KEY_FN_F12		0x1dd
+#define KEY_FN_1		0x1de
+#define KEY_FN_2		0x1df
+#define KEY_FN_D		0x1e0
+#define KEY_FN_E		0x1e1
+#define KEY_FN_F		0x1e2
+#define KEY_FN_S		0x1e3
+#define KEY_FN_B		0x1e4
+
+#define KEY_BRL_DOT1		0x1f1
+#define KEY_BRL_DOT2		0x1f2
+#define KEY_BRL_DOT3		0x1f3
+#define KEY_BRL_DOT4		0x1f4
+#define KEY_BRL_DOT5		0x1f5
+#define KEY_BRL_DOT6		0x1f6
+#define KEY_BRL_DOT7		0x1f7
+#define KEY_BRL_DOT8		0x1f8
+#define KEY_BRL_DOT9		0x1f9
+#define KEY_BRL_DOT10		0x1fa
+
+#define KEY_NUMERIC_0		0x200	/* used by phones, remote controls, */
+#define KEY_NUMERIC_1		0x201	/* and other keypads */
+#define KEY_NUMERIC_2		0x202
+#define KEY_NUMERIC_3		0x203
+#define KEY_NUMERIC_4		0x204
+#define KEY_NUMERIC_5		0x205
+#define KEY_NUMERIC_6		0x206
+#define KEY_NUMERIC_7		0x207
+#define KEY_NUMERIC_8		0x208
+#define KEY_NUMERIC_9		0x209
+#define KEY_NUMERIC_STAR	0x20a
+#define KEY_NUMERIC_POUND	0x20b
+#define KEY_NUMERIC_A		0x20c	/* Phone key A - HUT Telephony 0xb9 */
+#define KEY_NUMERIC_B		0x20d
+#define KEY_NUMERIC_C		0x20e
+#define KEY_NUMERIC_D		0x20f
+
+#define KEY_CAMERA_FOCUS	0x210
+#define KEY_WPS_BUTTON		0x211	/* WiFi Protected Setup key */
+
+#define KEY_TOUCHPAD_TOGGLE	0x212	/* Request switch touchpad on or off */
+#define KEY_TOUCHPAD_ON		0x213
+#define KEY_TOUCHPAD_OFF	0x214
+
+#define KEY_CAMERA_ZOOMIN	0x215
+#define KEY_CAMERA_ZOOMOUT	0x216
+#define KEY_CAMERA_UP		0x217
+#define KEY_CAMERA_DOWN		0x218
+#define KEY_CAMERA_LEFT		0x219
+#define KEY_CAMERA_RIGHT	0x21a
+
+#define KEY_ATTENDANT_ON	0x21b
+#define KEY_ATTENDANT_OFF	0x21c
+#define KEY_ATTENDANT_TOGGLE	0x21d	/* Attendant call on or off */
+#define KEY_LIGHTS_TOGGLE	0x21e	/* Reading light on or off */
+
+#define BTN_DPAD_UP		0x220
+#define BTN_DPAD_DOWN		0x221
+#define BTN_DPAD_LEFT		0x222
+#define BTN_DPAD_RIGHT		0x223
+
+#define KEY_ALS_TOGGLE		0x230	/* Ambient light sensor */
+
+#define KEY_BUTTONCONFIG		0x240	/* AL Button Configuration */
+#define KEY_TASKMANAGER		0x241	/* AL Task/Project Manager */
+#define KEY_JOURNAL		0x242	/* AL Log/Journal/Timecard */
+#define KEY_CONTROLPANEL		0x243	/* AL Control Panel */
+#define KEY_APPSELECT		0x244	/* AL Select Task/Application */
+#define KEY_SCREENSAVER		0x245	/* AL Screen Saver */
+#define KEY_VOICECOMMAND		0x246	/* Listening Voice Command */
+
+#define KEY_BRIGHTNESS_MIN		0x250	/* Set Brightness to Minimum */
+#define KEY_BRIGHTNESS_MAX		0x251	/* Set Brightness to Maximum */
+
+#define KEY_KBDINPUTASSIST_PREV		0x260
+#define KEY_KBDINPUTASSIST_NEXT		0x261
+#define KEY_KBDINPUTASSIST_PREVGROUP		0x262
+#define KEY_KBDINPUTASSIST_NEXTGROUP		0x263
+#define KEY_KBDINPUTASSIST_ACCEPT		0x264
+#define KEY_KBDINPUTASSIST_CANCEL		0x265
+
+#define BTN_TRIGGER_HAPPY		0x2c0
+#define BTN_TRIGGER_HAPPY1		0x2c0
+#define BTN_TRIGGER_HAPPY2		0x2c1
+#define BTN_TRIGGER_HAPPY3		0x2c2
+#define BTN_TRIGGER_HAPPY4		0x2c3
+#define BTN_TRIGGER_HAPPY5		0x2c4
+#define BTN_TRIGGER_HAPPY6		0x2c5
+#define BTN_TRIGGER_HAPPY7		0x2c6
+#define BTN_TRIGGER_HAPPY8		0x2c7
+#define BTN_TRIGGER_HAPPY9		0x2c8
+#define BTN_TRIGGER_HAPPY10		0x2c9
+#define BTN_TRIGGER_HAPPY11		0x2ca
+#define BTN_TRIGGER_HAPPY12		0x2cb
+#define BTN_TRIGGER_HAPPY13		0x2cc
+#define BTN_TRIGGER_HAPPY14		0x2cd
+#define BTN_TRIGGER_HAPPY15		0x2ce
+#define BTN_TRIGGER_HAPPY16		0x2cf
+#define BTN_TRIGGER_HAPPY17		0x2d0
+#define BTN_TRIGGER_HAPPY18		0x2d1
+#define BTN_TRIGGER_HAPPY19		0x2d2
+#define BTN_TRIGGER_HAPPY20		0x2d3
+#define BTN_TRIGGER_HAPPY21		0x2d4
+#define BTN_TRIGGER_HAPPY22		0x2d5
+#define BTN_TRIGGER_HAPPY23		0x2d6
+#define BTN_TRIGGER_HAPPY24		0x2d7
+#define BTN_TRIGGER_HAPPY25		0x2d8
+#define BTN_TRIGGER_HAPPY26		0x2d9
+#define BTN_TRIGGER_HAPPY27		0x2da
+#define BTN_TRIGGER_HAPPY28		0x2db
+#define BTN_TRIGGER_HAPPY29		0x2dc
+#define BTN_TRIGGER_HAPPY30		0x2dd
+#define BTN_TRIGGER_HAPPY31		0x2de
+#define BTN_TRIGGER_HAPPY32		0x2df
+#define BTN_TRIGGER_HAPPY33		0x2e0
+#define BTN_TRIGGER_HAPPY34		0x2e1
+#define BTN_TRIGGER_HAPPY35		0x2e2
+#define BTN_TRIGGER_HAPPY36		0x2e3
+#define BTN_TRIGGER_HAPPY37		0x2e4
+#define BTN_TRIGGER_HAPPY38		0x2e5
+#define BTN_TRIGGER_HAPPY39		0x2e6
+#define BTN_TRIGGER_HAPPY40		0x2e7
+
+/* We avoid low common keys in module aliases so they don't get huge. */
+#define KEY_MIN_INTERESTING	KEY_MUTE
+#define KEY_MAX			0x2ff
+#define KEY_CNT			(KEY_MAX+1)
+
+/*
+ * Relative axes
+ */
+
+#define REL_X			0x00
+#define REL_Y			0x01
+#define REL_Z			0x02
+#define REL_RX			0x03
+#define REL_RY			0x04
+#define REL_RZ			0x05
+#define REL_HWHEEL		0x06
+#define REL_DIAL		0x07
+#define REL_WHEEL		0x08
+#define REL_MISC		0x09
+#define REL_MAX			0x0f
+#define REL_CNT			(REL_MAX+1)
+
+/*
+ * Absolute axes
+ */
+
+#define ABS_X			0x00
+#define ABS_Y			0x01
+#define ABS_Z			0x02
+#define ABS_RX			0x03
+#define ABS_RY			0x04
+#define ABS_RZ			0x05
+#define ABS_THROTTLE		0x06
+#define ABS_RUDDER		0x07
+#define ABS_WHEEL		0x08
+#define ABS_GAS			0x09
+#define ABS_BRAKE		0x0a
+#define ABS_HAT0X		0x10
+#define ABS_HAT0Y		0x11
+#define ABS_HAT1X		0x12
+#define ABS_HAT1Y		0x13
+#define ABS_HAT2X		0x14
+#define ABS_HAT2Y		0x15
+#define ABS_HAT3X		0x16
+#define ABS_HAT3Y		0x17
+#define ABS_PRESSURE		0x18
+#define ABS_DISTANCE		0x19
+#define ABS_TILT_X		0x1a
+#define ABS_TILT_Y		0x1b
+#define ABS_TOOL_WIDTH		0x1c
+
+#define ABS_VOLUME		0x20
+
+#define ABS_MISC		0x28
+
+#define ABS_MT_SLOT		0x2f	/* MT slot being modified */
+#define ABS_MT_TOUCH_MAJOR	0x30	/* Major axis of touching ellipse */
+#define ABS_MT_TOUCH_MINOR	0x31	/* Minor axis (omit if circular) */
+#define ABS_MT_WIDTH_MAJOR	0x32	/* Major axis of approaching ellipse */
+#define ABS_MT_WIDTH_MINOR	0x33	/* Minor axis (omit if circular) */
+#define ABS_MT_ORIENTATION	0x34	/* Ellipse orientation */
+#define ABS_MT_POSITION_X	0x35	/* Center X touch position */
+#define ABS_MT_POSITION_Y	0x36	/* Center Y touch position */
+#define ABS_MT_TOOL_TYPE	0x37	/* Type of touching device */
+#define ABS_MT_BLOB_ID		0x38	/* Group a set of packets as a blob */
+#define ABS_MT_TRACKING_ID	0x39	/* Unique ID of initiated contact */
+#define ABS_MT_PRESSURE		0x3a	/* Pressure on contact area */
+#define ABS_MT_DISTANCE		0x3b	/* Contact hover distance */
+#define ABS_MT_TOOL_X		0x3c	/* Center X tool position */
+#define ABS_MT_TOOL_Y		0x3d	/* Center Y tool position */
+
+
+#define ABS_MAX			0x3f
+#define ABS_CNT			(ABS_MAX+1)
+
+/*
+ * Switch events
+ */
+
+#define SW_LID			0x00  /* set = lid shut */
+#define SW_TABLET_MODE		0x01  /* set = tablet mode */
+#define SW_HEADPHONE_INSERT	0x02  /* set = inserted */
+#define SW_RFKILL_ALL		0x03  /* rfkill master switch, type "any"
+					 set = radio enabled */
+#define SW_RADIO		SW_RFKILL_ALL	/* deprecated */
+#define SW_MICROPHONE_INSERT	0x04  /* set = inserted */
+#define SW_DOCK			0x05  /* set = plugged into dock */
+#define SW_LINEOUT_INSERT	0x06  /* set = inserted */
+#define SW_JACK_PHYSICAL_INSERT 0x07  /* set = mechanical switch set */
+#define SW_VIDEOOUT_INSERT	0x08  /* set = inserted */
+#define SW_CAMERA_LENS_COVER	0x09  /* set = lens covered */
+#define SW_KEYPAD_SLIDE		0x0a  /* set = keypad slide out */
+#define SW_FRONT_PROXIMITY	0x0b  /* set = front proximity sensor active */
+#define SW_ROTATE_LOCK		0x0c  /* set = rotate locked/disabled */
+#define SW_LINEIN_INSERT	0x0d  /* set = inserted */
+#define SW_MUTE_DEVICE		0x0e  /* set = device disabled */
+#define SW_MAX_			0x0f
+#define SW_CNT			(SW_MAX_+1)
+
+/*
+ * Misc events
+ */
+
+#define MSC_SERIAL		0x00
+#define MSC_PULSELED		0x01
+#define MSC_GESTURE		0x02
+#define MSC_RAW			0x03
+#define MSC_SCAN		0x04
+#define MSC_TIMESTAMP		0x05
+#define MSC_MAX			0x07
+#define MSC_CNT			(MSC_MAX+1)
+
+/*
+ * LEDs
+ */
+
+#define LED_NUML		0x00
+#define LED_CAPSL		0x01
+#define LED_SCROLLL		0x02
+#define LED_COMPOSE		0x03
+#define LED_KANA		0x04
+#define LED_SLEEP		0x05
+#define LED_SUSPEND		0x06
+#define LED_MUTE		0x07
+#define LED_MISC		0x08
+#define LED_MAIL		0x09
+#define LED_CHARGING		0x0a
+#define LED_MAX			0x0f
+#define LED_CNT			(LED_MAX+1)
+
+/*
+ * Autorepeat values
+ */
+
+#define REP_DELAY		0x00
+#define REP_PERIOD		0x01
+#define REP_MAX			0x01
+#define REP_CNT			(REP_MAX+1)
+
+/*
+ * Sounds
+ */
+
+#define SND_CLICK		0x00
+#define SND_BELL		0x01
+#define SND_TONE		0x02
+#define SND_MAX			0x07
+#define SND_CNT			(SND_MAX+1)
+
+#endif
diff --git a/qemu/include/standard-headers/linux/input.h b/qemu/include/standard-headers/linux/input.h
index b003c6705..a52b2025b 100644
--- a/qemu/include/standard-headers/linux/input.h
+++ b/qemu/include/standard-headers/linux/input.h
@@ -13,6 +13,7 @@
 #include <sys/types.h>
 #include "standard-headers/linux/types.h"
 
+#include "standard-headers/linux/input-event-codes.h"
 
 /*
  * The event structure itself
@@ -94,6 +95,12 @@ struct input_keymap_entry {
 	uint8_t  scancode[32];
 };
 
+struct input_mask {
+	uint32_t type;
+	uint32_t codes_size;
+	uint64_t codes_ptr;
+};
+
 #define EVIOCGVERSION		_IOR('E', 0x01, int)			/* get driver version */
 #define EVIOCGID		_IOR('E', 0x02, struct input_id)	/* get device ID */
 #define EVIOCGREP		_IOR('E', 0x03, unsigned int[2])	/* get repeat settings */
@@ -144,801 +151,68 @@ struct input_keymap_entry {
 #define EVIOCGABS(abs)		_IOR('E', 0x40 + (abs), struct input_absinfo)	/* get abs value/limits */
 #define EVIOCSABS(abs)		_IOW('E', 0xc0 + (abs), struct input_absinfo)	/* set abs value/limits */
 
-#define EVIOCSFF		_IOC(_IOC_WRITE, 'E', 0x80, sizeof(struct ff_effect))	/* send a force effect to a force feedback device */
+#define EVIOCSFF		_IOW('E', 0x80, struct ff_effect)	/* send a force effect to a force feedback device */
 #define EVIOCRMFF		_IOW('E', 0x81, int)			/* Erase a force effect */
 #define EVIOCGEFFECTS		_IOR('E', 0x84, int)			/* Report number of effects playable at the same time */
 
 #define EVIOCGRAB		_IOW('E', 0x90, int)			/* Grab/Release device */
 #define EVIOCREVOKE		_IOW('E', 0x91, int)			/* Revoke device access */
 
-#define EVIOCSCLOCKID		_IOW('E', 0xa0, int)			/* Set clockid to be used for timestamps */
-
-/*
- * Device properties and quirks
- */
-
-#define INPUT_PROP_POINTER		0x00	/* needs a pointer */
-#define INPUT_PROP_DIRECT		0x01	/* direct input devices */
-#define INPUT_PROP_BUTTONPAD		0x02	/* has button(s) under pad */
-#define INPUT_PROP_SEMI_MT		0x03	/* touch rectangle only */
-#define INPUT_PROP_TOPBUTTONPAD		0x04	/* softbuttons at top of pad */
-#define INPUT_PROP_POINTING_STICK	0x05	/* is a pointing stick */
-#define INPUT_PROP_ACCELEROMETER	0x06	/* has accelerometer */
-
-#define INPUT_PROP_MAX			0x1f
-#define INPUT_PROP_CNT			(INPUT_PROP_MAX + 1)
-
-/*
- * Event types
- */
-
-#define EV_SYN			0x00
-#define EV_KEY			0x01
-#define EV_REL			0x02
-#define EV_ABS			0x03
-#define EV_MSC			0x04
-#define EV_SW			0x05
-#define EV_LED			0x11
-#define EV_SND			0x12
-#define EV_REP			0x14
-#define EV_FF			0x15
-#define EV_PWR			0x16
-#define EV_FF_STATUS		0x17
-#define EV_MAX			0x1f
-#define EV_CNT			(EV_MAX+1)
-
-/*
- * Synchronization events.
- */
-
-#define SYN_REPORT		0
-#define SYN_CONFIG		1
-#define SYN_MT_REPORT		2
-#define SYN_DROPPED		3
-#define SYN_MAX			0xf
-#define SYN_CNT			(SYN_MAX+1)
-
-/*
- * Keys and buttons
+/**
+ * EVIOCGMASK - Retrieve current event mask
  *
- * Most of the keys/buttons are modeled after USB HUT 1.12
- * (see http://www.usb.org/developers/hidpage).
- * Abbreviations in the comments:
- * AC - Application Control
- * AL - Application Launch Button
- * SC - System Control
- */
-
-#define KEY_RESERVED		0
-#define KEY_ESC			1
-#define KEY_1			2
-#define KEY_2			3
-#define KEY_3			4
-#define KEY_4			5
-#define KEY_5			6
-#define KEY_6			7
-#define KEY_7			8
-#define KEY_8			9
-#define KEY_9			10
-#define KEY_0			11
-#define KEY_MINUS		12
-#define KEY_EQUAL		13
-#define KEY_BACKSPACE		14
-#define KEY_TAB			15
-#define KEY_Q			16
-#define KEY_W			17
-#define KEY_E			18
-#define KEY_R			19
-#define KEY_T			20
-#define KEY_Y			21
-#define KEY_U			22
-#define KEY_I			23
-#define KEY_O			24
-#define KEY_P			25
-#define KEY_LEFTBRACE		26
-#define KEY_RIGHTBRACE		27
-#define KEY_ENTER		28
-#define KEY_LEFTCTRL		29
-#define KEY_A			30
-#define KEY_S			31
-#define KEY_D			32
-#define KEY_F			33
-#define KEY_G			34
-#define KEY_H			35
-#define KEY_J			36
-#define KEY_K			37
-#define KEY_L			38
-#define KEY_SEMICOLON		39
-#define KEY_APOSTROPHE		40
-#define KEY_GRAVE		41
-#define KEY_LEFTSHIFT		42
-#define KEY_BACKSLASH		43
-#define KEY_Z			44
-#define KEY_X			45
-#define KEY_C			46
-#define KEY_V			47
-#define KEY_B			48
-#define KEY_N			49
-#define KEY_M			50
-#define KEY_COMMA		51
-#define KEY_DOT			52
-#define KEY_SLASH		53
-#define KEY_RIGHTSHIFT		54
-#define KEY_KPASTERISK		55
-#define KEY_LEFTALT		56
-#define KEY_SPACE		57
-#define KEY_CAPSLOCK		58
-#define KEY_F1			59
-#define KEY_F2			60
-#define KEY_F3			61
-#define KEY_F4			62
-#define KEY_F5			63
-#define KEY_F6			64
-#define KEY_F7			65
-#define KEY_F8			66
-#define KEY_F9			67
-#define KEY_F10			68
-#define KEY_NUMLOCK		69
-#define KEY_SCROLLLOCK		70
-#define KEY_KP7			71
-#define KEY_KP8			72
-#define KEY_KP9			73
-#define KEY_KPMINUS		74
-#define KEY_KP4			75
-#define KEY_KP5			76
-#define KEY_KP6			77
-#define KEY_KPPLUS		78
-#define KEY_KP1			79
-#define KEY_KP2			80
-#define KEY_KP3			81
-#define KEY_KP0			82
-#define KEY_KPDOT		83
-
-#define KEY_ZENKAKUHANKAKU	85
-#define KEY_102ND		86
-#define KEY_F11			87
-#define KEY_F12			88
-#define KEY_RO			89
-#define KEY_KATAKANA		90
-#define KEY_HIRAGANA		91
-#define KEY_HENKAN		92
-#define KEY_KATAKANAHIRAGANA	93
-#define KEY_MUHENKAN		94
-#define KEY_KPJPCOMMA		95
-#define KEY_KPENTER		96
-#define KEY_RIGHTCTRL		97
-#define KEY_KPSLASH		98
-#define KEY_SYSRQ		99
-#define KEY_RIGHTALT		100
-#define KEY_LINEFEED		101
-#define KEY_HOME		102
-#define KEY_UP			103
-#define KEY_PAGEUP		104
-#define KEY_LEFT		105
-#define KEY_RIGHT		106
-#define KEY_END			107
-#define KEY_DOWN		108
-#define KEY_PAGEDOWN		109
-#define KEY_INSERT		110
-#define KEY_DELETE		111
-#define KEY_MACRO		112
-#define KEY_MUTE		113
-#define KEY_VOLUMEDOWN		114
-#define KEY_VOLUMEUP		115
-#define KEY_POWER		116	/* SC System Power Down */
-#define KEY_KPEQUAL		117
-#define KEY_KPPLUSMINUS		118
-#define KEY_PAUSE		119
-#define KEY_SCALE		120	/* AL Compiz Scale (Expose) */
-
-#define KEY_KPCOMMA		121
-#define KEY_HANGEUL		122
-#define KEY_HANGUEL		KEY_HANGEUL
-#define KEY_HANJA		123
-#define KEY_YEN			124
-#define KEY_LEFTMETA		125
-#define KEY_RIGHTMETA		126
-#define KEY_COMPOSE		127
-
-#define KEY_STOP		128	/* AC Stop */
-#define KEY_AGAIN		129
-#define KEY_PROPS		130	/* AC Properties */
-#define KEY_UNDO		131	/* AC Undo */
-#define KEY_FRONT		132
-#define KEY_COPY		133	/* AC Copy */
-#define KEY_OPEN		134	/* AC Open */
-#define KEY_PASTE		135	/* AC Paste */
-#define KEY_FIND		136	/* AC Search */
-#define KEY_CUT			137	/* AC Cut */
-#define KEY_HELP		138	/* AL Integrated Help Center */
-#define KEY_MENU		139	/* Menu (show menu) */
-#define KEY_CALC		140	/* AL Calculator */
-#define KEY_SETUP		141
-#define KEY_SLEEP		142	/* SC System Sleep */
-#define KEY_WAKEUP		143	/* System Wake Up */
-#define KEY_FILE		144	/* AL Local Machine Browser */
-#define KEY_SENDFILE		145
-#define KEY_DELETEFILE		146
-#define KEY_XFER		147
-#define KEY_PROG1		148
-#define KEY_PROG2		149
-#define KEY_WWW			150	/* AL Internet Browser */
-#define KEY_MSDOS		151
-#define KEY_COFFEE		152	/* AL Terminal Lock/Screensaver */
-#define KEY_SCREENLOCK		KEY_COFFEE
-#define KEY_ROTATE_DISPLAY	153	/* Display orientation for e.g. tablets */
-#define KEY_DIRECTION		KEY_ROTATE_DISPLAY
-#define KEY_CYCLEWINDOWS	154
-#define KEY_MAIL		155
-#define KEY_BOOKMARKS		156	/* AC Bookmarks */
-#define KEY_COMPUTER		157
-#define KEY_BACK		158	/* AC Back */
-#define KEY_FORWARD		159	/* AC Forward */
-#define KEY_CLOSECD		160
-#define KEY_EJECTCD		161
-#define KEY_EJECTCLOSECD	162
-#define KEY_NEXTSONG		163
-#define KEY_PLAYPAUSE		164
-#define KEY_PREVIOUSSONG	165
-#define KEY_STOPCD		166
-#define KEY_RECORD		167
-#define KEY_REWIND		168
-#define KEY_PHONE		169	/* Media Select Telephone */
-#define KEY_ISO			170
-#define KEY_CONFIG		171	/* AL Consumer Control Configuration */
-#define KEY_HOMEPAGE		172	/* AC Home */
-#define KEY_REFRESH		173	/* AC Refresh */
-#define KEY_EXIT		174	/* AC Exit */
-#define KEY_MOVE		175
-#define KEY_EDIT		176
-#define KEY_SCROLLUP		177
-#define KEY_SCROLLDOWN		178
-#define KEY_KPLEFTPAREN		179
-#define KEY_KPRIGHTPAREN	180
-#define KEY_NEW			181	/* AC New */
-#define KEY_REDO		182	/* AC Redo/Repeat */
-
-#define KEY_F13			183
-#define KEY_F14			184
-#define KEY_F15			185
-#define KEY_F16			186
-#define KEY_F17			187
-#define KEY_F18			188
-#define KEY_F19			189
-#define KEY_F20			190
-#define KEY_F21			191
-#define KEY_F22			192
-#define KEY_F23			193
-#define KEY_F24			194
-
-#define KEY_PLAYCD		200
-#define KEY_PAUSECD		201
-#define KEY_PROG3		202
-#define KEY_PROG4		203
-#define KEY_DASHBOARD		204	/* AL Dashboard */
-#define KEY_SUSPEND		205
-#define KEY_CLOSE		206	/* AC Close */
-#define KEY_PLAY		207
-#define KEY_FASTFORWARD		208
-#define KEY_BASSBOOST		209
-#define KEY_PRINT		210	/* AC Print */
-#define KEY_HP			211
-#define KEY_CAMERA		212
-#define KEY_SOUND		213
-#define KEY_QUESTION		214
-#define KEY_EMAIL		215
-#define KEY_CHAT		216
-#define KEY_SEARCH		217
-#define KEY_CONNECT		218
-#define KEY_FINANCE		219	/* AL Checkbook/Finance */
-#define KEY_SPORT		220
-#define KEY_SHOP		221
-#define KEY_ALTERASE		222
-#define KEY_CANCEL		223	/* AC Cancel */
-#define KEY_BRIGHTNESSDOWN	224
-#define KEY_BRIGHTNESSUP	225
-#define KEY_MEDIA		226
-
-#define KEY_SWITCHVIDEOMODE	227	/* Cycle between available video
-					   outputs (Monitor/LCD/TV-out/etc) */
-#define KEY_KBDILLUMTOGGLE	228
-#define KEY_KBDILLUMDOWN	229
-#define KEY_KBDILLUMUP		230
-
-#define KEY_SEND		231	/* AC Send */
-#define KEY_REPLY		232	/* AC Reply */
-#define KEY_FORWARDMAIL		233	/* AC Forward Msg */
-#define KEY_SAVE		234	/* AC Save */
-#define KEY_DOCUMENTS		235
-
-#define KEY_BATTERY		236
-
-#define KEY_BLUETOOTH		237
-#define KEY_WLAN		238
-#define KEY_UWB			239
-
-#define KEY_UNKNOWN		240
-
-#define KEY_VIDEO_NEXT		241	/* drive next video source */
-#define KEY_VIDEO_PREV		242	/* drive previous video source */
-#define KEY_BRIGHTNESS_CYCLE	243	/* brightness up, after max is min */
-#define KEY_BRIGHTNESS_AUTO	244	/* Set Auto Brightness: manual
-					  brightness control is off,
-					  rely on ambient */
-#define KEY_BRIGHTNESS_ZERO	KEY_BRIGHTNESS_AUTO
-#define KEY_DISPLAY_OFF		245	/* display device to off state */
-
-#define KEY_WWAN		246	/* Wireless WAN (LTE, UMTS, GSM, etc.) */
-#define KEY_WIMAX		KEY_WWAN
-#define KEY_RFKILL		247	/* Key that controls all radios */
-
-#define KEY_MICMUTE		248	/* Mute / unmute the microphone */
-
-/* Code 255 is reserved for special needs of AT keyboard driver */
-
-#define BTN_MISC		0x100
-#define BTN_0			0x100
-#define BTN_1			0x101
-#define BTN_2			0x102
-#define BTN_3			0x103
-#define BTN_4			0x104
-#define BTN_5			0x105
-#define BTN_6			0x106
-#define BTN_7			0x107
-#define BTN_8			0x108
-#define BTN_9			0x109
-
-#define BTN_MOUSE		0x110
-#define BTN_LEFT		0x110
-#define BTN_RIGHT		0x111
-#define BTN_MIDDLE		0x112
-#define BTN_SIDE		0x113
-#define BTN_EXTRA		0x114
-#define BTN_FORWARD		0x115
-#define BTN_BACK		0x116
-#define BTN_TASK		0x117
-
-#define BTN_JOYSTICK		0x120
-#define BTN_TRIGGER		0x120
-#define BTN_THUMB		0x121
-#define BTN_THUMB2		0x122
-#define BTN_TOP			0x123
-#define BTN_TOP2		0x124
-#define BTN_PINKIE		0x125
-#define BTN_BASE		0x126
-#define BTN_BASE2		0x127
-#define BTN_BASE3		0x128
-#define BTN_BASE4		0x129
-#define BTN_BASE5		0x12a
-#define BTN_BASE6		0x12b
-#define BTN_DEAD		0x12f
-
-#define BTN_GAMEPAD		0x130
-#define BTN_SOUTH		0x130
-#define BTN_A			BTN_SOUTH
-#define BTN_EAST		0x131
-#define BTN_B			BTN_EAST
-#define BTN_C			0x132
-#define BTN_NORTH		0x133
-#define BTN_X			BTN_NORTH
-#define BTN_WEST		0x134
-#define BTN_Y			BTN_WEST
-#define BTN_Z			0x135
-#define BTN_TL			0x136
-#define BTN_TR			0x137
-#define BTN_TL2			0x138
-#define BTN_TR2			0x139
-#define BTN_SELECT		0x13a
-#define BTN_START		0x13b
-#define BTN_MODE		0x13c
-#define BTN_THUMBL		0x13d
-#define BTN_THUMBR		0x13e
-
-#define BTN_DIGI		0x140
-#define BTN_TOOL_PEN		0x140
-#define BTN_TOOL_RUBBER		0x141
-#define BTN_TOOL_BRUSH		0x142
-#define BTN_TOOL_PENCIL		0x143
-#define BTN_TOOL_AIRBRUSH	0x144
-#define BTN_TOOL_FINGER		0x145
-#define BTN_TOOL_MOUSE		0x146
-#define BTN_TOOL_LENS		0x147
-#define BTN_TOOL_QUINTTAP	0x148	/* Five fingers on trackpad */
-#define BTN_TOUCH		0x14a
-#define BTN_STYLUS		0x14b
-#define BTN_STYLUS2		0x14c
-#define BTN_TOOL_DOUBLETAP	0x14d
-#define BTN_TOOL_TRIPLETAP	0x14e
-#define BTN_TOOL_QUADTAP	0x14f	/* Four fingers on trackpad */
-
-#define BTN_WHEEL		0x150
-#define BTN_GEAR_DOWN		0x150
-#define BTN_GEAR_UP		0x151
-
-#define KEY_OK			0x160
-#define KEY_SELECT		0x161
-#define KEY_GOTO		0x162
-#define KEY_CLEAR		0x163
-#define KEY_POWER2		0x164
-#define KEY_OPTION		0x165
-#define KEY_INFO		0x166	/* AL OEM Features/Tips/Tutorial */
-#define KEY_TIME		0x167
-#define KEY_VENDOR		0x168
-#define KEY_ARCHIVE		0x169
-#define KEY_PROGRAM		0x16a	/* Media Select Program Guide */
-#define KEY_CHANNEL		0x16b
-#define KEY_FAVORITES		0x16c
-#define KEY_EPG			0x16d
-#define KEY_PVR			0x16e	/* Media Select Home */
-#define KEY_MHP			0x16f
-#define KEY_LANGUAGE		0x170
-#define KEY_TITLE		0x171
-#define KEY_SUBTITLE		0x172
-#define KEY_ANGLE		0x173
-#define KEY_ZOOM		0x174
-#define KEY_MODE		0x175
-#define KEY_KEYBOARD		0x176
-#define KEY_SCREEN		0x177
-#define KEY_PC			0x178	/* Media Select Computer */
-#define KEY_TV			0x179	/* Media Select TV */
-#define KEY_TV2			0x17a	/* Media Select Cable */
-#define KEY_VCR			0x17b	/* Media Select VCR */
-#define KEY_VCR2		0x17c	/* VCR Plus */
-#define KEY_SAT			0x17d	/* Media Select Satellite */
-#define KEY_SAT2		0x17e
-#define KEY_CD			0x17f	/* Media Select CD */
-#define KEY_TAPE		0x180	/* Media Select Tape */
-#define KEY_RADIO		0x181
-#define KEY_TUNER		0x182	/* Media Select Tuner */
-#define KEY_PLAYER		0x183
-#define KEY_TEXT		0x184
-#define KEY_DVD			0x185	/* Media Select DVD */
-#define KEY_AUX			0x186
-#define KEY_MP3			0x187
-#define KEY_AUDIO		0x188	/* AL Audio Browser */
-#define KEY_VIDEO		0x189	/* AL Movie Browser */
-#define KEY_DIRECTORY		0x18a
-#define KEY_LIST		0x18b
-#define KEY_MEMO		0x18c	/* Media Select Messages */
-#define KEY_CALENDAR		0x18d
-#define KEY_RED			0x18e
-#define KEY_GREEN		0x18f
-#define KEY_YELLOW		0x190
-#define KEY_BLUE		0x191
-#define KEY_CHANNELUP		0x192	/* Channel Increment */
-#define KEY_CHANNELDOWN		0x193	/* Channel Decrement */
-#define KEY_FIRST		0x194
-#define KEY_LAST		0x195	/* Recall Last */
-#define KEY_AB			0x196
-#define KEY_NEXT		0x197
-#define KEY_RESTART		0x198
-#define KEY_SLOW		0x199
-#define KEY_SHUFFLE		0x19a
-#define KEY_BREAK		0x19b
-#define KEY_PREVIOUS		0x19c
-#define KEY_DIGITS		0x19d
-#define KEY_TEEN		0x19e
-#define KEY_TWEN		0x19f
-#define KEY_VIDEOPHONE		0x1a0	/* Media Select Video Phone */
-#define KEY_GAMES		0x1a1	/* Media Select Games */
-#define KEY_ZOOMIN		0x1a2	/* AC Zoom In */
-#define KEY_ZOOMOUT		0x1a3	/* AC Zoom Out */
-#define KEY_ZOOMRESET		0x1a4	/* AC Zoom */
-#define KEY_WORDPROCESSOR	0x1a5	/* AL Word Processor */
-#define KEY_EDITOR		0x1a6	/* AL Text Editor */
-#define KEY_SPREADSHEET		0x1a7	/* AL Spreadsheet */
-#define KEY_GRAPHICSEDITOR	0x1a8	/* AL Graphics Editor */
-#define KEY_PRESENTATION	0x1a9	/* AL Presentation App */
-#define KEY_DATABASE		0x1aa	/* AL Database App */
-#define KEY_NEWS		0x1ab	/* AL Newsreader */
-#define KEY_VOICEMAIL		0x1ac	/* AL Voicemail */
-#define KEY_ADDRESSBOOK		0x1ad	/* AL Contacts/Address Book */
-#define KEY_MESSENGER		0x1ae	/* AL Instant Messaging */
-#define KEY_DISPLAYTOGGLE	0x1af	/* Turn display (LCD) on and off */
-#define KEY_BRIGHTNESS_TOGGLE	KEY_DISPLAYTOGGLE
-#define KEY_SPELLCHECK		0x1b0   /* AL Spell Check */
-#define KEY_LOGOFF		0x1b1   /* AL Logoff */
-
-#define KEY_DOLLAR		0x1b2
-#define KEY_EURO		0x1b3
-
-#define KEY_FRAMEBACK		0x1b4	/* Consumer - transport controls */
-#define KEY_FRAMEFORWARD	0x1b5
-#define KEY_CONTEXT_MENU	0x1b6	/* GenDesc - system context menu */
-#define KEY_MEDIA_REPEAT	0x1b7	/* Consumer - transport control */
-#define KEY_10CHANNELSUP	0x1b8	/* 10 channels up (10+) */
-#define KEY_10CHANNELSDOWN	0x1b9	/* 10 channels down (10-) */
-#define KEY_IMAGES		0x1ba	/* AL Image Browser */
-
-#define KEY_DEL_EOL		0x1c0
-#define KEY_DEL_EOS		0x1c1
-#define KEY_INS_LINE		0x1c2
-#define KEY_DEL_LINE		0x1c3
-
-#define KEY_FN			0x1d0
-#define KEY_FN_ESC		0x1d1
-#define KEY_FN_F1		0x1d2
-#define KEY_FN_F2		0x1d3
-#define KEY_FN_F3		0x1d4
-#define KEY_FN_F4		0x1d5
-#define KEY_FN_F5		0x1d6
-#define KEY_FN_F6		0x1d7
-#define KEY_FN_F7		0x1d8
-#define KEY_FN_F8		0x1d9
-#define KEY_FN_F9		0x1da
-#define KEY_FN_F10		0x1db
-#define KEY_FN_F11		0x1dc
-#define KEY_FN_F12		0x1dd
-#define KEY_FN_1		0x1de
-#define KEY_FN_2		0x1df
-#define KEY_FN_D		0x1e0
-#define KEY_FN_E		0x1e1
-#define KEY_FN_F		0x1e2
-#define KEY_FN_S		0x1e3
-#define KEY_FN_B		0x1e4
-
-#define KEY_BRL_DOT1		0x1f1
-#define KEY_BRL_DOT2		0x1f2
-#define KEY_BRL_DOT3		0x1f3
-#define KEY_BRL_DOT4		0x1f4
-#define KEY_BRL_DOT5		0x1f5
-#define KEY_BRL_DOT6		0x1f6
-#define KEY_BRL_DOT7		0x1f7
-#define KEY_BRL_DOT8		0x1f8
-#define KEY_BRL_DOT9		0x1f9
-#define KEY_BRL_DOT10		0x1fa
-
-#define KEY_NUMERIC_0		0x200	/* used by phones, remote controls, */
-#define KEY_NUMERIC_1		0x201	/* and other keypads */
-#define KEY_NUMERIC_2		0x202
-#define KEY_NUMERIC_3		0x203
-#define KEY_NUMERIC_4		0x204
-#define KEY_NUMERIC_5		0x205
-#define KEY_NUMERIC_6		0x206
-#define KEY_NUMERIC_7		0x207
-#define KEY_NUMERIC_8		0x208
-#define KEY_NUMERIC_9		0x209
-#define KEY_NUMERIC_STAR	0x20a
-#define KEY_NUMERIC_POUND	0x20b
-#define KEY_NUMERIC_A		0x20c	/* Phone key A - HUT Telephony 0xb9 */
-#define KEY_NUMERIC_B		0x20d
-#define KEY_NUMERIC_C		0x20e
-#define KEY_NUMERIC_D		0x20f
-
-#define KEY_CAMERA_FOCUS	0x210
-#define KEY_WPS_BUTTON		0x211	/* WiFi Protected Setup key */
-
-#define KEY_TOUCHPAD_TOGGLE	0x212	/* Request switch touchpad on or off */
-#define KEY_TOUCHPAD_ON		0x213
-#define KEY_TOUCHPAD_OFF	0x214
-
-#define KEY_CAMERA_ZOOMIN	0x215
-#define KEY_CAMERA_ZOOMOUT	0x216
-#define KEY_CAMERA_UP		0x217
-#define KEY_CAMERA_DOWN		0x218
-#define KEY_CAMERA_LEFT		0x219
-#define KEY_CAMERA_RIGHT	0x21a
-
-#define KEY_ATTENDANT_ON	0x21b
-#define KEY_ATTENDANT_OFF	0x21c
-#define KEY_ATTENDANT_TOGGLE	0x21d	/* Attendant call on or off */
-#define KEY_LIGHTS_TOGGLE	0x21e	/* Reading light on or off */
-
-#define BTN_DPAD_UP		0x220
-#define BTN_DPAD_DOWN		0x221
-#define BTN_DPAD_LEFT		0x222
-#define BTN_DPAD_RIGHT		0x223
-
-#define KEY_ALS_TOGGLE		0x230	/* Ambient light sensor */
-
-#define KEY_BUTTONCONFIG		0x240	/* AL Button Configuration */
-#define KEY_TASKMANAGER		0x241	/* AL Task/Project Manager */
-#define KEY_JOURNAL		0x242	/* AL Log/Journal/Timecard */
-#define KEY_CONTROLPANEL		0x243	/* AL Control Panel */
-#define KEY_APPSELECT		0x244	/* AL Select Task/Application */
-#define KEY_SCREENSAVER		0x245	/* AL Screen Saver */
-#define KEY_VOICECOMMAND		0x246	/* Listening Voice Command */
-
-#define KEY_BRIGHTNESS_MIN		0x250	/* Set Brightness to Minimum */
-#define KEY_BRIGHTNESS_MAX		0x251	/* Set Brightness to Maximum */
-
-#define KEY_KBDINPUTASSIST_PREV		0x260
-#define KEY_KBDINPUTASSIST_NEXT		0x261
-#define KEY_KBDINPUTASSIST_PREVGROUP		0x262
-#define KEY_KBDINPUTASSIST_NEXTGROUP		0x263
-#define KEY_KBDINPUTASSIST_ACCEPT		0x264
-#define KEY_KBDINPUTASSIST_CANCEL		0x265
-
-#define BTN_TRIGGER_HAPPY		0x2c0
-#define BTN_TRIGGER_HAPPY1		0x2c0
-#define BTN_TRIGGER_HAPPY2		0x2c1
-#define BTN_TRIGGER_HAPPY3		0x2c2
-#define BTN_TRIGGER_HAPPY4		0x2c3
-#define BTN_TRIGGER_HAPPY5		0x2c4
-#define BTN_TRIGGER_HAPPY6		0x2c5
-#define BTN_TRIGGER_HAPPY7		0x2c6
-#define BTN_TRIGGER_HAPPY8		0x2c7
-#define BTN_TRIGGER_HAPPY9		0x2c8
-#define BTN_TRIGGER_HAPPY10		0x2c9
-#define BTN_TRIGGER_HAPPY11		0x2ca
-#define BTN_TRIGGER_HAPPY12		0x2cb
-#define BTN_TRIGGER_HAPPY13		0x2cc
-#define BTN_TRIGGER_HAPPY14		0x2cd
-#define BTN_TRIGGER_HAPPY15		0x2ce
-#define BTN_TRIGGER_HAPPY16		0x2cf
-#define BTN_TRIGGER_HAPPY17		0x2d0
-#define BTN_TRIGGER_HAPPY18		0x2d1
-#define BTN_TRIGGER_HAPPY19		0x2d2
-#define BTN_TRIGGER_HAPPY20		0x2d3
-#define BTN_TRIGGER_HAPPY21		0x2d4
-#define BTN_TRIGGER_HAPPY22		0x2d5
-#define BTN_TRIGGER_HAPPY23		0x2d6
-#define BTN_TRIGGER_HAPPY24		0x2d7
-#define BTN_TRIGGER_HAPPY25		0x2d8
-#define BTN_TRIGGER_HAPPY26		0x2d9
-#define BTN_TRIGGER_HAPPY27		0x2da
-#define BTN_TRIGGER_HAPPY28		0x2db
-#define BTN_TRIGGER_HAPPY29		0x2dc
-#define BTN_TRIGGER_HAPPY30		0x2dd
-#define BTN_TRIGGER_HAPPY31		0x2de
-#define BTN_TRIGGER_HAPPY32		0x2df
-#define BTN_TRIGGER_HAPPY33		0x2e0
-#define BTN_TRIGGER_HAPPY34		0x2e1
-#define BTN_TRIGGER_HAPPY35		0x2e2
-#define BTN_TRIGGER_HAPPY36		0x2e3
-#define BTN_TRIGGER_HAPPY37		0x2e4
-#define BTN_TRIGGER_HAPPY38		0x2e5
-#define BTN_TRIGGER_HAPPY39		0x2e6
-#define BTN_TRIGGER_HAPPY40		0x2e7
-
-/* We avoid low common keys in module aliases so they don't get huge. */
-#define KEY_MIN_INTERESTING	KEY_MUTE
-#define KEY_MAX			0x2ff
-#define KEY_CNT			(KEY_MAX+1)
-
-/*
- * Relative axes
- */
-
-#define REL_X			0x00
-#define REL_Y			0x01
-#define REL_Z			0x02
-#define REL_RX			0x03
-#define REL_RY			0x04
-#define REL_RZ			0x05
-#define REL_HWHEEL		0x06
-#define REL_DIAL		0x07
-#define REL_WHEEL		0x08
-#define REL_MISC		0x09
-#define REL_MAX			0x0f
-#define REL_CNT			(REL_MAX+1)
-
-/*
- * Absolute axes
- */
-
-#define ABS_X			0x00
-#define ABS_Y			0x01
-#define ABS_Z			0x02
-#define ABS_RX			0x03
-#define ABS_RY			0x04
-#define ABS_RZ			0x05
-#define ABS_THROTTLE		0x06
-#define ABS_RUDDER		0x07
-#define ABS_WHEEL		0x08
-#define ABS_GAS			0x09
-#define ABS_BRAKE		0x0a
-#define ABS_HAT0X		0x10
-#define ABS_HAT0Y		0x11
-#define ABS_HAT1X		0x12
-#define ABS_HAT1Y		0x13
-#define ABS_HAT2X		0x14
-#define ABS_HAT2Y		0x15
-#define ABS_HAT3X		0x16
-#define ABS_HAT3Y		0x17
-#define ABS_PRESSURE		0x18
-#define ABS_DISTANCE		0x19
-#define ABS_TILT_X		0x1a
-#define ABS_TILT_Y		0x1b
-#define ABS_TOOL_WIDTH		0x1c
-
-#define ABS_VOLUME		0x20
-
-#define ABS_MISC		0x28
-
-#define ABS_MT_SLOT		0x2f	/* MT slot being modified */
-#define ABS_MT_TOUCH_MAJOR	0x30	/* Major axis of touching ellipse */
-#define ABS_MT_TOUCH_MINOR	0x31	/* Minor axis (omit if circular) */
-#define ABS_MT_WIDTH_MAJOR	0x32	/* Major axis of approaching ellipse */
-#define ABS_MT_WIDTH_MINOR	0x33	/* Minor axis (omit if circular) */
-#define ABS_MT_ORIENTATION	0x34	/* Ellipse orientation */
-#define ABS_MT_POSITION_X	0x35	/* Center X touch position */
-#define ABS_MT_POSITION_Y	0x36	/* Center Y touch position */
-#define ABS_MT_TOOL_TYPE	0x37	/* Type of touching device */
-#define ABS_MT_BLOB_ID		0x38	/* Group a set of packets as a blob */
-#define ABS_MT_TRACKING_ID	0x39	/* Unique ID of initiated contact */
-#define ABS_MT_PRESSURE		0x3a	/* Pressure on contact area */
-#define ABS_MT_DISTANCE		0x3b	/* Contact hover distance */
-#define ABS_MT_TOOL_X		0x3c	/* Center X tool position */
-#define ABS_MT_TOOL_Y		0x3d	/* Center Y tool position */
-
-
-#define ABS_MAX			0x3f
-#define ABS_CNT			(ABS_MAX+1)
-
-/*
- * Switch events
- */
-
-#define SW_LID			0x00  /* set = lid shut */
-#define SW_TABLET_MODE		0x01  /* set = tablet mode */
-#define SW_HEADPHONE_INSERT	0x02  /* set = inserted */
-#define SW_RFKILL_ALL		0x03  /* rfkill master switch, type "any"
-					 set = radio enabled */
-#define SW_RADIO		SW_RFKILL_ALL	/* deprecated */
-#define SW_MICROPHONE_INSERT	0x04  /* set = inserted */
-#define SW_DOCK			0x05  /* set = plugged into dock */
-#define SW_LINEOUT_INSERT	0x06  /* set = inserted */
-#define SW_JACK_PHYSICAL_INSERT 0x07  /* set = mechanical switch set */
-#define SW_VIDEOOUT_INSERT	0x08  /* set = inserted */
-#define SW_CAMERA_LENS_COVER	0x09  /* set = lens covered */
-#define SW_KEYPAD_SLIDE		0x0a  /* set = keypad slide out */
-#define SW_FRONT_PROXIMITY	0x0b  /* set = front proximity sensor active */
-#define SW_ROTATE_LOCK		0x0c  /* set = rotate locked/disabled */
-#define SW_LINEIN_INSERT	0x0d  /* set = inserted */
-#define SW_MUTE_DEVICE		0x0e  /* set = device disabled */
-#define SW_MAX			0x0f
-#define SW_CNT			(SW_MAX+1)
-
-/*
- * Misc events
- */
-
-#define MSC_SERIAL		0x00
-#define MSC_PULSELED		0x01
-#define MSC_GESTURE		0x02
-#define MSC_RAW			0x03
-#define MSC_SCAN		0x04
-#define MSC_TIMESTAMP		0x05
-#define MSC_MAX			0x07
-#define MSC_CNT			(MSC_MAX+1)
-
-/*
- * LEDs
- */
-
-#define LED_NUML		0x00
-#define LED_CAPSL		0x01
-#define LED_SCROLLL		0x02
-#define LED_COMPOSE		0x03
-#define LED_KANA		0x04
-#define LED_SLEEP		0x05
-#define LED_SUSPEND		0x06
-#define LED_MUTE		0x07
-#define LED_MISC		0x08
-#define LED_MAIL		0x09
-#define LED_CHARGING		0x0a
-#define LED_MAX			0x0f
-#define LED_CNT			(LED_MAX+1)
-
-/*
- * Autorepeat values
+ * This ioctl allows user to retrieve the current event mask for specific
+ * event type. The argument must be of type "struct input_mask" and
+ * specifies the event type to query, the address of the receive buffer and
+ * the size of the receive buffer.
+ *
+ * The event mask is a per-client mask that specifies which events are
+ * forwarded to the client. Each event code is represented by a single bit
+ * in the event mask. If the bit is set, the event is passed to the client
+ * normally. Otherwise, the event is filtered and will never be queued on
+ * the client's receive buffer.
+ *
+ * Event masks do not affect global state of the input device. They only
+ * affect the file descriptor they are applied to.
+ *
+ * The default event mask for a client has all bits set, i.e. all events
+ * are forwarded to the client. If the kernel is queried for an unknown
+ * event type or if the receive buffer is larger than the number of
+ * event codes known to the kernel, the kernel returns all zeroes for those
+ * codes.
+ *
+ * At maximum, codes_size bytes are copied.
+ *
+ * This ioctl may fail with ENODEV in case the file is revoked, EFAULT
+ * if the receive-buffer points to invalid memory, or EINVAL if the kernel
+ * does not implement the ioctl.
  */
+#define EVIOCGMASK		_IOR('E', 0x92, struct input_mask)	/* Get event-masks */
 
-#define REP_DELAY		0x00
-#define REP_PERIOD		0x01
-#define REP_MAX			0x01
-#define REP_CNT			(REP_MAX+1)
-
-/*
- * Sounds
+/**
+ * EVIOCSMASK - Set event mask
+ *
+ * This ioctl is the counterpart to EVIOCGMASK. Instead of receiving the
+ * current event mask, this changes the client's event mask for a specific
+ * type.  See EVIOCGMASK for a description of event-masks and the
+ * argument-type.
+ *
+ * This ioctl provides full forward compatibility. If the passed event type
+ * is unknown to the kernel, or if the number of event codes specified in
+ * the mask is bigger than what is known to the kernel, the ioctl is still
+ * accepted and applied. However, any unknown codes are left untouched and
+ * stay cleared. That means, the kernel always filters unknown codes
+ * regardless of what the client requests.  If the new mask doesn't cover
+ * all known event-codes, all remaining codes are automatically cleared and
+ * thus filtered.
+ *
+ * This ioctl may fail with ENODEV in case the file is revoked. EFAULT is
+ * returned if the receive-buffer points to invalid memory. EINVAL is returned
+ * if the kernel does not implement the ioctl.
  */
+#define EVIOCSMASK		_IOW('E', 0x93, struct input_mask)	/* Set event-masks */
 
-#define SND_CLICK		0x00
-#define SND_BELL		0x01
-#define SND_TONE		0x02
-#define SND_MAX			0x07
-#define SND_CNT			(SND_MAX+1)
+#define EVIOCSCLOCKID		_IOW('E', 0xa0, int)			/* Set clockid to be used for timestamps */
 
 /*
  * IDs.
@@ -969,6 +243,7 @@ struct input_keymap_entry {
 #define BUS_GSC			0x1A
 #define BUS_ATARI		0x1B
 #define BUS_SPI			0x1C
+#define BUS_RMI			0x1D
 
 /*
  * MT_TOOL types
@@ -1197,6 +472,14 @@ struct ff_effect {
 #define FF_GAIN		0x60
 #define FF_AUTOCENTER	0x61
 
+/*
+ * ff->playback(effect_id = FF_GAIN) is the first effect_id to
+ * cause a collision with another ff method, in this case ff->set_gain().
+ * Therefore the greatest safe value for effect_id is FF_GAIN - 1,
+ * and thus the total number of effects should never exceed FF_GAIN.
+ */
+#define FF_MAX_EFFECTS	FF_GAIN
+
 #define FF_MAX		0x7f
 #define FF_CNT		(FF_MAX+1)
 
diff --git a/qemu/include/standard-headers/linux/pci_regs.h b/qemu/include/standard-headers/linux/pci_regs.h
index 57e8c80c3..1becea86c 100644
--- a/qemu/include/standard-headers/linux/pci_regs.h
+++ b/qemu/include/standard-headers/linux/pci_regs.h
@@ -13,10 +13,10 @@
  *	PCI to PCI Bridge Specification
  *	PCI System Design Guide
  *
- * 	For hypertransport information, please consult the following manuals
- * 	from http://www.hypertransport.org
+ *	For HyperTransport information, please consult the following manuals
+ *	from http://www.hypertransport.org
  *
- *	The Hypertransport I/O Link Specification
+ *	The HyperTransport I/O Link Specification
  */
 
 #ifndef LINUX_PCI_REGS_H
@@ -26,6 +26,7 @@
  * Under PCI, each device has 256 bytes of configuration address space,
  * of which the first 64 bytes are standardized as follows:
  */
+#define PCI_STD_HEADER_SIZEOF	64
 #define PCI_VENDOR_ID		0x00	/* 16 bits */
 #define PCI_DEVICE_ID		0x02	/* 16 bits */
 #define PCI_COMMAND		0x04	/* 16 bits */
@@ -36,7 +37,7 @@
 #define  PCI_COMMAND_INVALIDATE	0x10	/* Use memory write and invalidate */
 #define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
 #define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
-#define  PCI_COMMAND_WAIT 	0x80	/* Enable address/data stepping */
+#define  PCI_COMMAND_WAIT	0x80	/* Enable address/data stepping */
 #define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
 #define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
 #define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
@@ -44,7 +45,7 @@
 #define PCI_STATUS		0x06	/* 16 bits */
 #define  PCI_STATUS_INTERRUPT	0x08	/* Interrupt status */
 #define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
-#define  PCI_STATUS_66MHZ	0x20	/* Support 66 Mhz PCI 2.1 bus */
+#define  PCI_STATUS_66MHZ	0x20	/* Support 66 MHz PCI 2.1 bus */
 #define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
 #define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
 #define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
@@ -125,7 +126,8 @@
 #define  PCI_IO_RANGE_TYPE_MASK	0x0fUL	/* I/O bridging type */
 #define  PCI_IO_RANGE_TYPE_16	0x00
 #define  PCI_IO_RANGE_TYPE_32	0x01
-#define  PCI_IO_RANGE_MASK	(~0x0fUL)
+#define  PCI_IO_RANGE_MASK	(~0x0fUL) /* Standard 4K I/O windows */
+#define  PCI_IO_1K_RANGE_MASK	(~0x03UL) /* Intel 1K I/O windows */
 #define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
 #define PCI_MEMORY_BASE		0x20	/* Memory range behind */
 #define PCI_MEMORY_LIMIT	0x22
@@ -203,16 +205,19 @@
 #define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
 #define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
 #define  PCI_CAP_ID_HT		0x08	/* HyperTransport */
-#define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific */
+#define  PCI_CAP_ID_VNDR	0x09	/* Vendor-Specific */
 #define  PCI_CAP_ID_DBG		0x0A	/* Debug port */
 #define  PCI_CAP_ID_CCRC	0x0B	/* CompactPCI Central Resource Control */
-#define  PCI_CAP_ID_SHPC 	0x0C	/* PCI Standard Hot-Plug Controller */
+#define  PCI_CAP_ID_SHPC	0x0C	/* PCI Standard Hot-Plug Controller */
 #define  PCI_CAP_ID_SSVID	0x0D	/* Bridge subsystem vendor/device ID */
 #define  PCI_CAP_ID_AGP3	0x0E	/* AGP Target PCI-PCI bridge */
-#define  PCI_CAP_ID_EXP 	0x10	/* PCI Express */
+#define  PCI_CAP_ID_SECDEV	0x0F	/* Secure Device */
+#define  PCI_CAP_ID_EXP		0x10	/* PCI Express */
 #define  PCI_CAP_ID_MSIX	0x11	/* MSI-X */
-#define  PCI_CAP_ID_SATA	0x12	/* Serial ATA */
+#define  PCI_CAP_ID_SATA	0x12	/* SATA Data/Index Conf. */
 #define  PCI_CAP_ID_AF		0x13	/* PCI Advanced Features */
+#define  PCI_CAP_ID_EA		0x14	/* PCI Enhanced Allocation */
+#define  PCI_CAP_ID_MAX		PCI_CAP_ID_EA
 #define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
 #define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
 #define PCI_CAP_SIZEOF		4
@@ -264,8 +269,8 @@
 #define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
 #define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
 #define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
-#define  PCI_AGP_COMMAND_64BIT	0x0020 	/* Allow processing of 64-bit addresses */
-#define  PCI_AGP_COMMAND_FW	0x0010 	/* Force FW transfers */
+#define  PCI_AGP_COMMAND_64BIT	0x0020	/* Allow processing of 64-bit addresses */
+#define  PCI_AGP_COMMAND_FW	0x0010	/* Force FW transfers */
 #define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
 #define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
 #define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
@@ -277,6 +282,7 @@
 #define  PCI_VPD_ADDR_MASK	0x7fff	/* Address mask */
 #define  PCI_VPD_ADDR_F		0x8000	/* Write 0, 1 indicates completion */
 #define PCI_VPD_DATA		4	/* 32-bits of data returned here */
+#define PCI_CAP_VPD_SIZEOF	8
 
 /* Slot Identification */
 
@@ -287,32 +293,37 @@
 
 /* Message Signalled Interrupts registers */
 
-#define PCI_MSI_FLAGS		2	/* Various flags */
-#define  PCI_MSI_FLAGS_64BIT	0x80	/* 64-bit addresses allowed */
-#define  PCI_MSI_FLAGS_QSIZE	0x70	/* Message queue size configured */
-#define  PCI_MSI_FLAGS_QMASK	0x0e	/* Maximum queue size available */
-#define  PCI_MSI_FLAGS_ENABLE	0x01	/* MSI feature enabled */
-#define  PCI_MSI_FLAGS_MASKBIT	0x100	/* 64-bit mask bits allowed */
+#define PCI_MSI_FLAGS		2	/* Message Control */
+#define  PCI_MSI_FLAGS_ENABLE	0x0001	/* MSI feature enabled */
+#define  PCI_MSI_FLAGS_QMASK	0x000e	/* Maximum queue size available */
+#define  PCI_MSI_FLAGS_QSIZE	0x0070	/* Message queue size configured */
+#define  PCI_MSI_FLAGS_64BIT	0x0080	/* 64-bit addresses allowed */
+#define  PCI_MSI_FLAGS_MASKBIT	0x0100	/* Per-vector masking capable */
 #define PCI_MSI_RFU		3	/* Rest of capability flags */
 #define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
 #define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
 #define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
 #define PCI_MSI_MASK_32		12	/* Mask bits register for 32-bit devices */
-#define PCI_MSI_PENDING_32	16	/* Pending bits register for 32-bit devices */
+#define PCI_MSI_PENDING_32	16	/* Pending intrs for 32-bit devices */
 #define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
 #define PCI_MSI_MASK_64		16	/* Mask bits register for 64-bit devices */
-#define PCI_MSI_PENDING_64	20	/* Pending bits register for 32-bit devices */
+#define PCI_MSI_PENDING_64	20	/* Pending intrs for 64-bit devices */
 
 /* MSI-X registers */
-#define PCI_MSIX_FLAGS		2
-#define  PCI_MSIX_FLAGS_QSIZE	0x7FF
-#define  PCI_MSIX_FLAGS_ENABLE	(1 << 15)
-#define  PCI_MSIX_FLAGS_MASKALL	(1 << 14)
-#define PCI_MSIX_TABLE		4
-#define PCI_MSIX_PBA		8
-#define  PCI_MSIX_FLAGS_BIRMASK	(7 << 0)
-
-/* MSI-X entry's format */
+#define PCI_MSIX_FLAGS		2	/* Message Control */
+#define  PCI_MSIX_FLAGS_QSIZE	0x07FF	/* Table size */
+#define  PCI_MSIX_FLAGS_MASKALL	0x4000	/* Mask all vectors for this function */
+#define  PCI_MSIX_FLAGS_ENABLE	0x8000	/* MSI-X enable */
+#define PCI_MSIX_TABLE		4	/* Table offset */
+#define  PCI_MSIX_TABLE_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_TABLE_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_PBA		8	/* Pending Bit Array offset */
+#define  PCI_MSIX_PBA_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_PBA_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_FLAGS_BIRMASK	PCI_MSIX_PBA_BIR /* deprecated */
+#define PCI_CAP_MSIX_SIZEOF	12	/* size of MSIX registers */
+
+/* MSI-X Table entry format */
 #define PCI_MSIX_ENTRY_SIZE		16
 #define  PCI_MSIX_ENTRY_LOWER_ADDR	0
 #define  PCI_MSIX_ENTRY_UPPER_ADDR	4
@@ -341,8 +352,49 @@
 #define  PCI_AF_CTRL_FLR	0x01
 #define PCI_AF_STATUS		5
 #define  PCI_AF_STATUS_TP	0x01
-
-/* PCI-X registers */
+#define PCI_CAP_AF_SIZEOF	6	/* size of AF registers */
+
+/* PCI Enhanced Allocation registers */
+
+#define PCI_EA_NUM_ENT		2	/* Number of Capability Entries */
+#define  PCI_EA_NUM_ENT_MASK	0x3f	/* Num Entries Mask */
+#define PCI_EA_FIRST_ENT	4	/* First EA Entry in List */
+#define PCI_EA_FIRST_ENT_BRIDGE	8	/* First EA Entry for Bridges */
+#define  PCI_EA_ES		0x00000007 /* Entry Size */
+#define  PCI_EA_BEI		0x000000f0 /* BAR Equivalent Indicator */
+/* 0-5 map to BARs 0-5 respectively */
+#define   PCI_EA_BEI_BAR0		0
+#define   PCI_EA_BEI_BAR5		5
+#define   PCI_EA_BEI_BRIDGE		6	/* Resource behind bridge */
+#define   PCI_EA_BEI_ENI		7	/* Equivalent Not Indicated */
+#define   PCI_EA_BEI_ROM		8	/* Expansion ROM */
+/* 9-14 map to VF BARs 0-5 respectively */
+#define   PCI_EA_BEI_VF_BAR0		9
+#define   PCI_EA_BEI_VF_BAR5		14
+#define   PCI_EA_BEI_RESERVED		15	/* Reserved - Treat like ENI */
+#define  PCI_EA_PP		0x0000ff00	/* Primary Properties */
+#define  PCI_EA_SP		0x00ff0000	/* Secondary Properties */
+#define   PCI_EA_P_MEM			0x00	/* Non-Prefetch Memory */
+#define   PCI_EA_P_MEM_PREFETCH		0x01	/* Prefetchable Memory */
+#define   PCI_EA_P_IO			0x02	/* I/O Space */
+#define   PCI_EA_P_VF_MEM_PREFETCH	0x03	/* VF Prefetchable Memory */
+#define   PCI_EA_P_VF_MEM		0x04	/* VF Non-Prefetch Memory */
+#define   PCI_EA_P_BRIDGE_MEM		0x05	/* Bridge Non-Prefetch Memory */
+#define   PCI_EA_P_BRIDGE_MEM_PREFETCH	0x06	/* Bridge Prefetchable Memory */
+#define   PCI_EA_P_BRIDGE_IO		0x07	/* Bridge I/O Space */
+/* 0x08-0xfc reserved */
+#define   PCI_EA_P_MEM_RESERVED		0xfd	/* Reserved Memory */
+#define   PCI_EA_P_IO_RESERVED		0xfe	/* Reserved I/O Space */
+#define   PCI_EA_P_UNAVAILABLE		0xff	/* Entry Unavailable */
+#define  PCI_EA_WRITABLE	0x40000000	/* Writable: 1 = RW, 0 = HwInit */
+#define  PCI_EA_ENABLE		0x80000000	/* Enable for this entry */
+#define PCI_EA_BASE		4		/* Base Address Offset */
+#define PCI_EA_MAX_OFFSET	8		/* MaxOffset (resource length) */
+/* bit 0 is reserved */
+#define  PCI_EA_IS_64		0x00000002	/* 64-bit field flag */
+#define  PCI_EA_FIELD_MASK	0xfffffffc	/* For Base & Max Offset */
+
+/* PCI-X registers (Type 0 (non-bridge) devices) */
 
 #define PCI_X_CMD		2	/* Modes & Features */
 #define  PCI_X_CMD_DPERR_E	0x0001	/* Data Parity Error Recovery Enable */
@@ -362,7 +414,7 @@
 #define  PCI_X_CMD_SPLIT_16	0x0060	/* Max 16 */
 #define  PCI_X_CMD_SPLIT_32	0x0070	/* Max 32 */
 #define  PCI_X_CMD_MAX_SPLIT	0x0070	/* Max Outstanding Split Transactions */
-#define  PCI_X_CMD_VERSION(x) 	(((x) >> 12) & 3) /* Version */
+#define  PCI_X_CMD_VERSION(x)	(((x) >> 12) & 3) /* Version */
 #define PCI_X_STATUS		4	/* PCI-X capabilities */
 #define  PCI_X_STATUS_DEVFN	0x000000ff	/* A copy of devfn */
 #define  PCI_X_STATUS_BUS	0x0000ff00	/* A copy of bus nr */
@@ -377,11 +429,28 @@
 #define  PCI_X_STATUS_SPL_ERR	0x20000000	/* Rcvd Split Completion Error Msg */
 #define  PCI_X_STATUS_266MHZ	0x40000000	/* 266 MHz capable */
 #define  PCI_X_STATUS_533MHZ	0x80000000	/* 533 MHz capable */
+#define PCI_X_ECC_CSR		8	/* ECC control and status */
+#define PCI_CAP_PCIX_SIZEOF_V0	8	/* size of registers for Version 0 */
+#define PCI_CAP_PCIX_SIZEOF_V1	24	/* size for Version 1 */
+#define PCI_CAP_PCIX_SIZEOF_V2	PCI_CAP_PCIX_SIZEOF_V1	/* Same for v2 */
+
+/* PCI-X registers (Type 1 (bridge) devices) */
+
+#define PCI_X_BRIDGE_SSTATUS	2	/* Secondary Status */
+#define  PCI_X_SSTATUS_64BIT	0x0001	/* Secondary AD interface is 64 bits */
+#define  PCI_X_SSTATUS_133MHZ	0x0002	/* 133 MHz capable */
+#define  PCI_X_SSTATUS_FREQ	0x03c0	/* Secondary Bus Mode and Frequency */
+#define  PCI_X_SSTATUS_VERS	0x3000	/* PCI-X Capability Version */
+#define  PCI_X_SSTATUS_V1	0x1000	/* Mode 2, not Mode 1 */
+#define  PCI_X_SSTATUS_V2	0x2000	/* Mode 1 or Modes 1 and 2 */
+#define  PCI_X_SSTATUS_266MHZ	0x4000	/* 266 MHz capable */
+#define  PCI_X_SSTATUS_533MHZ	0x8000	/* 533 MHz capable */
+#define PCI_X_BRIDGE_STATUS	4	/* Bridge Status */
 
 /* PCI Bridge Subsystem ID registers */
 
-#define PCI_SSVID_VENDOR_ID     4	/* PCI-Bridge subsystem vendor id register */
-#define PCI_SSVID_DEVICE_ID     6	/* PCI-Bridge subsystem device id register */
+#define PCI_SSVID_VENDOR_ID     4	/* PCI Bridge subsystem vendor ID */
+#define PCI_SSVID_DEVICE_ID     6	/* PCI Bridge subsystem device ID */
 
 /* PCI Express capability registers */
 
@@ -393,24 +462,24 @@
 #define  PCI_EXP_TYPE_ROOT_PORT 0x4	/* Root Port */
 #define  PCI_EXP_TYPE_UPSTREAM	0x5	/* Upstream Port */
 #define  PCI_EXP_TYPE_DOWNSTREAM 0x6	/* Downstream Port */
-#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCI/PCI-X Bridge */
-#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8   /* PCI/PCI-X to PCIE Bridge */
+#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCIe to PCI/PCI-X Bridge */
+#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8	/* PCI/PCI-X to PCIe Bridge */
 #define  PCI_EXP_TYPE_RC_END	0x9	/* Root Complex Integrated Endpoint */
-#define  PCI_EXP_TYPE_RC_EC     0xa     /* Root Complex Event Collector */
+#define  PCI_EXP_TYPE_RC_EC	0xa	/* Root Complex Event Collector */
 #define PCI_EXP_FLAGS_SLOT	0x0100	/* Slot implemented */
 #define PCI_EXP_FLAGS_IRQ	0x3e00	/* Interrupt message number */
 #define PCI_EXP_DEVCAP		4	/* Device capabilities */
-#define  PCI_EXP_DEVCAP_PAYLOAD	0x07	/* Max_Payload_Size */
-#define  PCI_EXP_DEVCAP_PHANTOM	0x18	/* Phantom functions */
-#define  PCI_EXP_DEVCAP_EXT_TAG	0x20	/* Extended tags */
-#define  PCI_EXP_DEVCAP_L0S	0x1c0	/* L0s Acceptable Latency */
-#define  PCI_EXP_DEVCAP_L1	0xe00	/* L1 Acceptable Latency */
-#define  PCI_EXP_DEVCAP_ATN_BUT	0x1000	/* Attention Button Present */
-#define  PCI_EXP_DEVCAP_ATN_IND	0x2000	/* Attention Indicator Present */
-#define  PCI_EXP_DEVCAP_PWR_IND	0x4000	/* Power Indicator Present */
-#define  PCI_EXP_DEVCAP_RBER	0x8000	/* Role-Based Error Reporting */
-#define  PCI_EXP_DEVCAP_PWR_VAL	0x3fc0000 /* Slot Power Limit Value */
-#define  PCI_EXP_DEVCAP_PWR_SCL	0xc000000 /* Slot Power Limit Scale */
+#define  PCI_EXP_DEVCAP_PAYLOAD	0x00000007 /* Max_Payload_Size */
+#define  PCI_EXP_DEVCAP_PHANTOM	0x00000018 /* Phantom functions */
+#define  PCI_EXP_DEVCAP_EXT_TAG	0x00000020 /* Extended tags */
+#define  PCI_EXP_DEVCAP_L0S	0x000001c0 /* L0s Acceptable Latency */
+#define  PCI_EXP_DEVCAP_L1	0x00000e00 /* L1 Acceptable Latency */
+#define  PCI_EXP_DEVCAP_ATN_BUT	0x00001000 /* Attention Button Present */
+#define  PCI_EXP_DEVCAP_ATN_IND	0x00002000 /* Attention Indicator Present */
+#define  PCI_EXP_DEVCAP_PWR_IND	0x00004000 /* Power Indicator Present */
+#define  PCI_EXP_DEVCAP_RBER	0x00008000 /* Role-Based Error Reporting */
+#define  PCI_EXP_DEVCAP_PWR_VAL	0x03fc0000 /* Slot Power Limit Value */
+#define  PCI_EXP_DEVCAP_PWR_SCL	0x0c000000 /* Slot Power Limit Scale */
 #define  PCI_EXP_DEVCAP_FLR     0x10000000 /* Function Level Reset */
 #define PCI_EXP_DEVCTL		8	/* Device Control */
 #define  PCI_EXP_DEVCTL_CERE	0x0001	/* Correctable Error Reporting En. */
@@ -424,47 +493,61 @@
 #define  PCI_EXP_DEVCTL_AUX_PME	0x0400	/* Auxiliary Power PM Enable */
 #define  PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800  /* Enable No Snoop */
 #define  PCI_EXP_DEVCTL_READRQ	0x7000	/* Max_Read_Request_Size */
+#define  PCI_EXP_DEVCTL_READRQ_128B  0x0000 /* 128 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_256B  0x1000 /* 256 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_512B  0x2000 /* 512 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_1024B 0x3000 /* 1024 Bytes */
 #define  PCI_EXP_DEVCTL_BCR_FLR 0x8000  /* Bridge Configuration Retry / FLR */
 #define PCI_EXP_DEVSTA		10	/* Device Status */
-#define  PCI_EXP_DEVSTA_CED	0x01	/* Correctable Error Detected */
-#define  PCI_EXP_DEVSTA_NFED	0x02	/* Non-Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_FED	0x04	/* Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_URD	0x08	/* Unsupported Request Detected */
-#define  PCI_EXP_DEVSTA_AUXPD	0x10	/* AUX Power Detected */
-#define  PCI_EXP_DEVSTA_TRPND	0x20	/* Transactions Pending */
+#define  PCI_EXP_DEVSTA_CED	0x0001	/* Correctable Error Detected */
+#define  PCI_EXP_DEVSTA_NFED	0x0002	/* Non-Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_FED	0x0004	/* Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_URD	0x0008	/* Unsupported Request Detected */
+#define  PCI_EXP_DEVSTA_AUXPD	0x0010	/* AUX Power Detected */
+#define  PCI_EXP_DEVSTA_TRPND	0x0020	/* Transactions Pending */
 #define PCI_EXP_LNKCAP		12	/* Link Capabilities */
 #define  PCI_EXP_LNKCAP_SLS	0x0000000f /* Supported Link Speeds */
+#define  PCI_EXP_LNKCAP_SLS_2_5GB 0x00000001 /* LNKCAP2 SLS Vector bit 0 */
+#define  PCI_EXP_LNKCAP_SLS_5_0GB 0x00000002 /* LNKCAP2 SLS Vector bit 1 */
 #define  PCI_EXP_LNKCAP_MLW	0x000003f0 /* Maximum Link Width */
 #define  PCI_EXP_LNKCAP_ASPMS	0x00000c00 /* ASPM Support */
 #define  PCI_EXP_LNKCAP_L0SEL	0x00007000 /* L0s Exit Latency */
 #define  PCI_EXP_LNKCAP_L1EL	0x00038000 /* L1 Exit Latency */
-#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* L1 Clock Power Management */
+#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* Clock Power Management */
 #define  PCI_EXP_LNKCAP_SDERC	0x00080000 /* Surprise Down Error Reporting Capable */
 #define  PCI_EXP_LNKCAP_DLLLARC	0x00100000 /* Data Link Layer Link Active Reporting Capable */
 #define  PCI_EXP_LNKCAP_LBNC	0x00200000 /* Link Bandwidth Notification Capability */
 #define  PCI_EXP_LNKCAP_PN	0xff000000 /* Port Number */
 #define PCI_EXP_LNKCTL		16	/* Link Control */
 #define  PCI_EXP_LNKCTL_ASPMC	0x0003	/* ASPM Control */
+#define  PCI_EXP_LNKCTL_ASPM_L0S 0x0001	/* L0s Enable */
+#define  PCI_EXP_LNKCTL_ASPM_L1  0x0002	/* L1 Enable */
 #define  PCI_EXP_LNKCTL_RCB	0x0008	/* Read Completion Boundary */
 #define  PCI_EXP_LNKCTL_LD	0x0010	/* Link Disable */
 #define  PCI_EXP_LNKCTL_RL	0x0020	/* Retrain Link */
 #define  PCI_EXP_LNKCTL_CCC	0x0040	/* Common Clock Configuration */
 #define  PCI_EXP_LNKCTL_ES	0x0080	/* Extended Synch */
-#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x100	/* Enable clkreq */
+#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x0100 /* Enable clkreq */
 #define  PCI_EXP_LNKCTL_HAWD	0x0200	/* Hardware Autonomous Width Disable */
 #define  PCI_EXP_LNKCTL_LBMIE	0x0400	/* Link Bandwidth Management Interrupt Enable */
-#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Lnk Autonomous Bandwidth Interrupt Enable */
+#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Link Autonomous Bandwidth Interrupt Enable */
 #define PCI_EXP_LNKSTA		18	/* Link Status */
 #define  PCI_EXP_LNKSTA_CLS	0x000f	/* Current Link Speed */
-#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x01	/* Current Link Speed 2.5GT/s */
-#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x02	/* Current Link Speed 5.0GT/s */
-#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Nogotiated Link Width */
+#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x0001 /* Current Link Speed 2.5GT/s */
+#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x0002 /* Current Link Speed 5.0GT/s */
+#define  PCI_EXP_LNKSTA_CLS_8_0GB 0x0003 /* Current Link Speed 8.0GT/s */
+#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Negotiated Link Width */
+#define  PCI_EXP_LNKSTA_NLW_X1	0x0010	/* Current Link Width x1 */
+#define  PCI_EXP_LNKSTA_NLW_X2	0x0020	/* Current Link Width x2 */
+#define  PCI_EXP_LNKSTA_NLW_X4	0x0040	/* Current Link Width x4 */
+#define  PCI_EXP_LNKSTA_NLW_X8	0x0080	/* Current Link Width x8 */
 #define  PCI_EXP_LNKSTA_NLW_SHIFT 4	/* start of NLW mask in link status */
 #define  PCI_EXP_LNKSTA_LT	0x0800	/* Link Training */
 #define  PCI_EXP_LNKSTA_SLC	0x1000	/* Slot Clock Configuration */
 #define  PCI_EXP_LNKSTA_DLLLA	0x2000	/* Data Link Layer Link Active */
 #define  PCI_EXP_LNKSTA_LBMS	0x4000	/* Link Bandwidth Management Status */
 #define  PCI_EXP_LNKSTA_LABS	0x8000	/* Link Autonomous Bandwidth Status */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V1	20	/* v1 endpoints end here */
 #define PCI_EXP_SLTCAP		20	/* Slot Capabilities */
 #define  PCI_EXP_SLTCAP_ABP	0x00000001 /* Attention Button Present */
 #define  PCI_EXP_SLTCAP_PCP	0x00000002 /* Power Controller Present */
@@ -486,8 +569,16 @@
 #define  PCI_EXP_SLTCTL_CCIE	0x0010	/* Command Completed Interrupt Enable */
 #define  PCI_EXP_SLTCTL_HPIE	0x0020	/* Hot-Plug Interrupt Enable */
 #define  PCI_EXP_SLTCTL_AIC	0x00c0	/* Attention Indicator Control */
+#define  PCI_EXP_SLTCTL_ATTN_IND_ON    0x0040 /* Attention Indicator on */
+#define  PCI_EXP_SLTCTL_ATTN_IND_BLINK 0x0080 /* Attention Indicator blinking */
+#define  PCI_EXP_SLTCTL_ATTN_IND_OFF   0x00c0 /* Attention Indicator off */
 #define  PCI_EXP_SLTCTL_PIC	0x0300	/* Power Indicator Control */
+#define  PCI_EXP_SLTCTL_PWR_IND_ON     0x0100 /* Power Indicator on */
+#define  PCI_EXP_SLTCTL_PWR_IND_BLINK  0x0200 /* Power Indicator blinking */
+#define  PCI_EXP_SLTCTL_PWR_IND_OFF    0x0300 /* Power Indicator off */
 #define  PCI_EXP_SLTCTL_PCC	0x0400	/* Power Controller Control */
+#define  PCI_EXP_SLTCTL_PWR_ON         0x0000 /* Power On */
+#define  PCI_EXP_SLTCTL_PWR_OFF        0x0400 /* Power Off */
 #define  PCI_EXP_SLTCTL_EIC	0x0800	/* Electromechanical Interlock Control */
 #define  PCI_EXP_SLTCTL_DLLSCE	0x1000	/* Data Link Layer State Changed Enable */
 #define PCI_EXP_SLTSTA		26	/* Slot Status */
@@ -501,52 +592,94 @@
 #define  PCI_EXP_SLTSTA_EIS	0x0080	/* Electromechanical Interlock Status */
 #define  PCI_EXP_SLTSTA_DLLSC	0x0100	/* Data Link Layer State Changed */
 #define PCI_EXP_RTCTL		28	/* Root Control */
-#define  PCI_EXP_RTCTL_SECEE	0x01	/* System Error on Correctable Error */
-#define  PCI_EXP_RTCTL_SENFEE	0x02	/* System Error on Non-Fatal Error */
-#define  PCI_EXP_RTCTL_SEFEE	0x04	/* System Error on Fatal Error */
-#define  PCI_EXP_RTCTL_PMEIE	0x08	/* PME Interrupt Enable */
-#define  PCI_EXP_RTCTL_CRSSVE	0x10	/* CRS Software Visibility Enable */
+#define  PCI_EXP_RTCTL_SECEE	0x0001	/* System Error on Correctable Error */
+#define  PCI_EXP_RTCTL_SENFEE	0x0002	/* System Error on Non-Fatal Error */
+#define  PCI_EXP_RTCTL_SEFEE	0x0004	/* System Error on Fatal Error */
+#define  PCI_EXP_RTCTL_PMEIE	0x0008	/* PME Interrupt Enable */
+#define  PCI_EXP_RTCTL_CRSSVE	0x0010	/* CRS Software Visibility Enable */
 #define PCI_EXP_RTCAP		30	/* Root Capabilities */
+#define  PCI_EXP_RTCAP_CRSVIS	0x0001	/* CRS Software Visibility capability */
 #define PCI_EXP_RTSTA		32	/* Root Status */
-#define PCI_EXP_RTSTA_PME	0x10000 /* PME status */
-#define PCI_EXP_RTSTA_PENDING	0x20000 /* PME pending */
+#define PCI_EXP_RTSTA_PME	0x00010000 /* PME status */
+#define PCI_EXP_RTSTA_PENDING	0x00020000 /* PME pending */
+/*
+ * The Device Capabilities 2, Device Status 2, Device Control 2,
+ * Link Capabilities 2, Link Status 2, Link Control 2,
+ * Slot Capabilities 2, Slot Status 2, and Slot Control 2 registers
+ * are only present on devices with PCIe Capability version 2.
+ * Use pcie_capability_read_word() and similar interfaces to use them
+ * safely.
+ */
 #define PCI_EXP_DEVCAP2		36	/* Device Capabilities 2 */
-#define  PCI_EXP_DEVCAP2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_DEVCAP2_LTR	0x800	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MASK	0xc0000 /* OBFF support mechanism */
-#define  PCI_EXP_OBFF_MSG	0x40000 /* New message signaling */
-#define  PCI_EXP_OBFF_WAKE	0x80000 /* Re-use WAKE# for OBFF */
+#define  PCI_EXP_DEVCAP2_ARI		0x00000020 /* Alternative Routing-ID */
+#define  PCI_EXP_DEVCAP2_LTR		0x00000800 /* Latency tolerance reporting */
+#define  PCI_EXP_DEVCAP2_OBFF_MASK	0x000c0000 /* OBFF support mechanism */
+#define  PCI_EXP_DEVCAP2_OBFF_MSG	0x00040000 /* New message signaling */
+#define  PCI_EXP_DEVCAP2_OBFF_WAKE	0x00080000 /* Re-use WAKE# for OBFF */
 #define PCI_EXP_DEVCTL2		40	/* Device Control 2 */
-#define  PCI_EXP_DEVCTL2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_IDO_REQ_EN	0x100	/* ID-based ordering request enable */
-#define  PCI_EXP_IDO_CMP_EN	0x200	/* ID-based ordering completion enable */
-#define  PCI_EXP_LTR_EN		0x400	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MSGA_EN	0x2000	/* OBFF enable with Message type A */
-#define  PCI_EXP_OBFF_MSGB_EN	0x4000	/* OBFF enable with Message type B */
-#define  PCI_EXP_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
+#define  PCI_EXP_DEVCTL2_COMP_TIMEOUT	0x000f	/* Completion Timeout Value */
+#define  PCI_EXP_DEVCTL2_ARI		0x0020	/* Alternative Routing-ID */
+#define  PCI_EXP_DEVCTL2_IDO_REQ_EN	0x0100	/* Allow IDO for requests */
+#define  PCI_EXP_DEVCTL2_IDO_CMP_EN	0x0200	/* Allow IDO for completions */
+#define  PCI_EXP_DEVCTL2_LTR_EN		0x0400	/* Enable LTR mechanism */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGA_EN	0x2000	/* Enable OBFF Message type A */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGB_EN	0x4000	/* Enable OBFF Message type B */
+#define  PCI_EXP_DEVCTL2_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
+#define PCI_EXP_DEVSTA2		42	/* Device Status 2 */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2	44	/* v2 endpoints end here */
+#define PCI_EXP_LNKCAP2		44	/* Link Capabilities 2 */
+#define  PCI_EXP_LNKCAP2_SLS_2_5GB	0x00000002 /* Supported Speed 2.5GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_5_0GB	0x00000004 /* Supported Speed 5.0GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_8_0GB	0x00000008 /* Supported Speed 8.0GT/s */
+#define  PCI_EXP_LNKCAP2_CROSSLINK	0x00000100 /* Crosslink supported */
 #define PCI_EXP_LNKCTL2		48	/* Link Control 2 */
+#define PCI_EXP_LNKSTA2		50	/* Link Status 2 */
+#define PCI_EXP_SLTCAP2		52	/* Slot Capabilities 2 */
 #define PCI_EXP_SLTCTL2		56	/* Slot Control 2 */
+#define PCI_EXP_SLTSTA2		58	/* Slot Status 2 */
 
 /* Extended Capabilities (PCI-X 2.0 and Express) */
 #define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
 #define PCI_EXT_CAP_VER(header)		((header >> 16) & 0xf)
 #define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
 
-#define PCI_EXT_CAP_ID_ERR	1
-#define PCI_EXT_CAP_ID_VC	2
-#define PCI_EXT_CAP_ID_DSN	3
-#define PCI_EXT_CAP_ID_PWR	4
-#define PCI_EXT_CAP_ID_VNDR	11
-#define PCI_EXT_CAP_ID_ACS	13
-#define PCI_EXT_CAP_ID_ARI	14
-#define PCI_EXT_CAP_ID_ATS	15
-#define PCI_EXT_CAP_ID_SRIOV	16
-#define PCI_EXT_CAP_ID_LTR	24
+#define PCI_EXT_CAP_ID_ERR	0x01	/* Advanced Error Reporting */
+#define PCI_EXT_CAP_ID_VC	0x02	/* Virtual Channel Capability */
+#define PCI_EXT_CAP_ID_DSN	0x03	/* Device Serial Number */
+#define PCI_EXT_CAP_ID_PWR	0x04	/* Power Budgeting */
+#define PCI_EXT_CAP_ID_RCLD	0x05	/* Root Complex Link Declaration */
+#define PCI_EXT_CAP_ID_RCILC	0x06	/* Root Complex Internal Link Control */
+#define PCI_EXT_CAP_ID_RCEC	0x07	/* Root Complex Event Collector */
+#define PCI_EXT_CAP_ID_MFVC	0x08	/* Multi-Function VC Capability */
+#define PCI_EXT_CAP_ID_VC9	0x09	/* same as _VC */
+#define PCI_EXT_CAP_ID_RCRB	0x0A	/* Root Complex RB? */
+#define PCI_EXT_CAP_ID_VNDR	0x0B	/* Vendor-Specific */
+#define PCI_EXT_CAP_ID_CAC	0x0C	/* Config Access - obsolete */
+#define PCI_EXT_CAP_ID_ACS	0x0D	/* Access Control Services */
+#define PCI_EXT_CAP_ID_ARI	0x0E	/* Alternate Routing ID */
+#define PCI_EXT_CAP_ID_ATS	0x0F	/* Address Translation Services */
+#define PCI_EXT_CAP_ID_SRIOV	0x10	/* Single Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MRIOV	0x11	/* Multi Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MCAST	0x12	/* Multicast */
+#define PCI_EXT_CAP_ID_PRI	0x13	/* Page Request Interface */
+#define PCI_EXT_CAP_ID_AMD_XXX	0x14	/* Reserved for AMD */
+#define PCI_EXT_CAP_ID_REBAR	0x15	/* Resizable BAR */
+#define PCI_EXT_CAP_ID_DPA	0x16	/* Dynamic Power Allocation */
+#define PCI_EXT_CAP_ID_TPH	0x17	/* TPH Requester */
+#define PCI_EXT_CAP_ID_LTR	0x18	/* Latency Tolerance Reporting */
+#define PCI_EXT_CAP_ID_SECPCI	0x19	/* Secondary PCIe Capability */
+#define PCI_EXT_CAP_ID_PMUX	0x1A	/* Protocol Multiplexing */
+#define PCI_EXT_CAP_ID_PASID	0x1B	/* Process Address Space ID */
+#define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_PASID
+
+#define PCI_EXT_CAP_DSN_SIZEOF	12
+#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
 
 /* Advanced Error Reporting */
 #define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
-#define  PCI_ERR_UNC_TRAIN	0x00000001	/* Training */
+#define  PCI_ERR_UNC_UND	0x00000001	/* Undefined */
 #define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
+#define  PCI_ERR_UNC_SURPDN	0x00000020	/* Surprise Down */
 #define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
 #define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
 #define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
@@ -556,6 +689,11 @@
 #define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
 #define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
 #define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
+#define  PCI_ERR_UNC_ACSV	0x00200000	/* ACS Violation */
+#define  PCI_ERR_UNC_INTN	0x00400000	/* internal error */
+#define  PCI_ERR_UNC_MCBTLP	0x00800000	/* MC blocked TLP */
+#define  PCI_ERR_UNC_ATOMEG	0x01000000	/* Atomic egress blocked */
+#define  PCI_ERR_UNC_TLPPRE	0x02000000	/* TLP prefix blocked */
 #define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
 	/* Same bits as above */
 #define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
@@ -566,6 +704,9 @@
 #define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
 #define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
 #define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
+#define  PCI_ERR_COR_ADV_NFAT	0x00002000	/* Advisory Non-Fatal */
+#define  PCI_ERR_COR_INTERNAL	0x00004000	/* Corrected Internal */
+#define  PCI_ERR_COR_LOG_OVER	0x00008000	/* Header Log Overflow */
 #define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
 	/* Same bits as above */
 #define PCI_ERR_CAP		24	/* Advanced Error Capabilities */
@@ -586,9 +727,9 @@
 #define PCI_ERR_ROOT_COR_RCV		0x00000001	/* ERR_COR Received */
 /* Multi ERR_COR Received */
 #define PCI_ERR_ROOT_MULTI_COR_RCV	0x00000002
-/* ERR_FATAL/NONFATAL Recevied */
+/* ERR_FATAL/NONFATAL Received */
 #define PCI_ERR_ROOT_UNCOR_RCV		0x00000004
-/* Multi ERR_FATAL/NONFATAL Recevied */
+/* Multi ERR_FATAL/NONFATAL Received */
 #define PCI_ERR_ROOT_MULTI_UNCOR_RCV	0x00000008
 #define PCI_ERR_ROOT_FIRST_FATAL	0x00000010	/* First Fatal */
 #define PCI_ERR_ROOT_NONFATAL_RCV	0x00000020	/* Non-Fatal Received */
@@ -596,13 +737,36 @@
 #define PCI_ERR_ROOT_ERR_SRC	52	/* Error Source Identification */
 
 /* Virtual Channel */
-#define PCI_VC_PORT_REG1	4
-#define PCI_VC_PORT_REG2	8
+#define PCI_VC_PORT_CAP1	4
+#define  PCI_VC_CAP1_EVCC	0x00000007	/* extended VC count */
+#define  PCI_VC_CAP1_LPEVCC	0x00000070	/* low prio extended VC count */
+#define  PCI_VC_CAP1_ARB_SIZE	0x00000c00
+#define PCI_VC_PORT_CAP2	8
+#define  PCI_VC_CAP2_32_PHASE		0x00000002
+#define  PCI_VC_CAP2_64_PHASE		0x00000004
+#define  PCI_VC_CAP2_128_PHASE		0x00000008
+#define  PCI_VC_CAP2_ARB_OFF		0xff000000
 #define PCI_VC_PORT_CTRL	12
+#define  PCI_VC_PORT_CTRL_LOAD_TABLE	0x00000001
 #define PCI_VC_PORT_STATUS	14
+#define  PCI_VC_PORT_STATUS_TABLE	0x00000001
 #define PCI_VC_RES_CAP		16
+#define  PCI_VC_RES_CAP_32_PHASE	0x00000002
+#define  PCI_VC_RES_CAP_64_PHASE	0x00000004
+#define  PCI_VC_RES_CAP_128_PHASE	0x00000008
+#define  PCI_VC_RES_CAP_128_PHASE_TB	0x00000010
+#define  PCI_VC_RES_CAP_256_PHASE	0x00000020
+#define  PCI_VC_RES_CAP_ARB_OFF		0xff000000
 #define PCI_VC_RES_CTRL		20
+#define  PCI_VC_RES_CTRL_LOAD_TABLE	0x00010000
+#define  PCI_VC_RES_CTRL_ARB_SELECT	0x000e0000
+#define  PCI_VC_RES_CTRL_ID		0x07000000
+#define  PCI_VC_RES_CTRL_ENABLE		0x80000000
 #define PCI_VC_RES_STATUS	26
+#define  PCI_VC_RES_STATUS_TABLE	0x00000001
+#define  PCI_VC_RES_STATUS_NEGO		0x00000002
+#define PCI_CAP_VC_BASE_SIZEOF		0x10
+#define PCI_CAP_VC_PER_VC_SIZEOF	0x0C
 
 /* Power Budgeting */
 #define PCI_PWR_DSR		4	/* Data Select Register */
@@ -615,9 +779,16 @@
 #define  PCI_PWR_DATA_RAIL(x)	(((x) >> 18) & 7)   /* Power Rail */
 #define PCI_PWR_CAP		12	/* Capability */
 #define  PCI_PWR_CAP_BUDGET(x)	((x) & 1)	/* Included in system budget */
+#define PCI_EXT_CAP_PWR_SIZEOF	16
+
+/* Vendor-Specific (VSEC, PCI_EXT_CAP_ID_VNDR) */
+#define PCI_VNDR_HEADER		4	/* Vendor-Specific Header */
+#define  PCI_VNDR_HEADER_ID(x)	((x) & 0xffff)
+#define  PCI_VNDR_HEADER_REV(x)	(((x) >> 16) & 0xf)
+#define  PCI_VNDR_HEADER_LEN(x)	(((x) >> 20) & 0xfff)
 
 /*
- * Hypertransport sub capability types
+ * HyperTransport sub capability types
  *
  * Unfortunately there are both 3 bit and 5 bit capability types defined
  * in the HT spec, catering for that is a little messy. You probably don't
@@ -645,8 +816,10 @@
 #define HT_CAPTYPE_DIRECT_ROUTE	0xB0	/* Direct routing configuration */
 #define HT_CAPTYPE_VCSET	0xB8	/* Virtual Channel configuration */
 #define HT_CAPTYPE_ERROR_RETRY	0xC0	/* Retry on error configuration */
-#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 hypertransport configuration */
-#define HT_CAPTYPE_PM		0xE0	/* Hypertransport powermanagement configuration */
+#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 HyperTransport configuration */
+#define HT_CAPTYPE_PM		0xE0	/* HyperTransport power management configuration */
+#define HT_CAP_SIZEOF_LONG	28	/* slave & primary */
+#define HT_CAP_SIZEOF_SHORT	24	/* host & secondary */
 
 /* Alternative Routing-ID Interpretation */
 #define PCI_ARI_CAP		0x04	/* ARI Capability Register */
@@ -657,6 +830,7 @@
 #define  PCI_ARI_CTRL_MFVC	0x0001	/* MFVC Function Groups Enable */
 #define  PCI_ARI_CTRL_ACS	0x0002	/* ACS Function Groups Enable */
 #define  PCI_ARI_CTRL_FG(x)	(((x) >> 4) & 7) /* Function Group */
+#define PCI_EXT_CAP_ARI_SIZEOF	8
 
 /* Address Translation Service */
 #define PCI_ATS_CAP		0x04	/* ATS Capability Register */
@@ -666,6 +840,29 @@
 #define  PCI_ATS_CTRL_ENABLE	0x8000	/* ATS Enable */
 #define  PCI_ATS_CTRL_STU(x)	((x) & 0x1f)	/* Smallest Translation Unit */
 #define  PCI_ATS_MIN_STU	12	/* shift of minimum STU block */
+#define PCI_EXT_CAP_ATS_SIZEOF	8
+
+/* Page Request Interface */
+#define PCI_PRI_CTRL		0x04	/* PRI control register */
+#define  PCI_PRI_CTRL_ENABLE	0x01	/* Enable */
+#define  PCI_PRI_CTRL_RESET	0x02	/* Reset */
+#define PCI_PRI_STATUS		0x06	/* PRI status register */
+#define  PCI_PRI_STATUS_RF	0x001	/* Response Failure */
+#define  PCI_PRI_STATUS_UPRGI	0x002	/* Unexpected PRG index */
+#define  PCI_PRI_STATUS_STOPPED	0x100	/* PRI Stopped */
+#define PCI_PRI_MAX_REQ		0x08	/* PRI max reqs supported */
+#define PCI_PRI_ALLOC_REQ	0x0c	/* PRI max reqs allowed */
+#define PCI_EXT_CAP_PRI_SIZEOF	16
+
+/* Process Address Space ID */
+#define PCI_PASID_CAP		0x04    /* PASID feature register */
+#define  PCI_PASID_CAP_EXEC	0x02	/* Exec permissions Supported */
+#define  PCI_PASID_CAP_PRIV	0x04	/* Privilege Mode Supported */
+#define PCI_PASID_CTRL		0x06    /* PASID control register */
+#define  PCI_PASID_CTRL_ENABLE	0x01	/* Enable bit */
+#define  PCI_PASID_CTRL_EXEC	0x02	/* Exec permissions Enable */
+#define  PCI_PASID_CTRL_PRIV	0x04	/* Privilege Mode Enable */
+#define PCI_EXT_CAP_PASID_SIZEOF	8
 
 /* Single Root I/O Virtualization */
 #define PCI_SRIOV_CAP		0x04	/* SR-IOV Capabilities */
@@ -697,12 +894,14 @@
 #define  PCI_SRIOV_VFM_MI	0x1	/* Dormant.MigrateIn */
 #define  PCI_SRIOV_VFM_MO	0x2	/* Active.MigrateOut */
 #define  PCI_SRIOV_VFM_AV	0x3	/* Active.Available */
+#define PCI_EXT_CAP_SRIOV_SIZEOF 64
 
 #define PCI_LTR_MAX_SNOOP_LAT	0x4
 #define PCI_LTR_MAX_NOSNOOP_LAT	0x6
 #define  PCI_LTR_VALUE_MASK	0x000003ff
 #define  PCI_LTR_SCALE_MASK	0x00001c00
 #define  PCI_LTR_SCALE_SHIFT	10
+#define PCI_EXT_CAP_LTR_SIZEOF	8
 
 /* Access Control Service */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
@@ -713,7 +912,38 @@
 #define  PCI_ACS_UF		0x10	/* Upstream Forwarding */
 #define  PCI_ACS_EC		0x20	/* P2P Egress Control */
 #define  PCI_ACS_DT		0x40	/* Direct Translated P2P */
+#define PCI_ACS_EGRESS_BITS	0x05	/* ACS Egress Control Vector Size */
 #define PCI_ACS_CTRL		0x06	/* ACS Control Register */
 #define PCI_ACS_EGRESS_CTL_V	0x08	/* ACS Egress Control Vector */
 
+#define PCI_VSEC_HDR		4	/* extended cap - vendor-specific */
+#define  PCI_VSEC_HDR_LEN_SHIFT	20	/* shift for length field */
+
+/* SATA capability */
+#define PCI_SATA_REGS		4	/* SATA REGs specifier */
+#define  PCI_SATA_REGS_MASK	0xF	/* location - BAR#/inline */
+#define  PCI_SATA_REGS_INLINE	0xF	/* REGS in config space */
+#define PCI_SATA_SIZEOF_SHORT	8
+#define PCI_SATA_SIZEOF_LONG	16
+
+/* Resizable BARs */
+#define PCI_REBAR_CTRL		8	/* control register */
+#define  PCI_REBAR_CTRL_NBAR_MASK	(7 << 5)	/* mask for # bars */
+#define  PCI_REBAR_CTRL_NBAR_SHIFT	5	/* shift for # bars */
+
+/* Dynamic Power Allocation */
+#define PCI_DPA_CAP		4	/* capability register */
+#define  PCI_DPA_CAP_SUBSTATE_MASK	0x1F	/* # substates - 1 */
+#define PCI_DPA_BASE_SIZEOF	16	/* size with 0 substates */
+
+/* TPH Requester */
+#define PCI_TPH_CAP		4	/* capability register */
+#define  PCI_TPH_CAP_LOC_MASK	0x600	/* location mask */
+#define   PCI_TPH_LOC_NONE	0x000	/* no location */
+#define   PCI_TPH_LOC_CAP	0x200	/* in capability */
+#define   PCI_TPH_LOC_MSIX	0x400	/* in MSI-X */
+#define PCI_TPH_CAP_ST_MASK	0x07FF0000	/* st table mask */
+#define PCI_TPH_CAP_ST_SHIFT	16	/* st table shift */
+#define PCI_TPH_BASE_SIZEOF	12	/* size with no st table */
+
 #endif /* LINUX_PCI_REGS_H */
diff --git a/qemu/include/standard-headers/linux/types.h b/qemu/include/standard-headers/linux/types.h
index 0526c2b87..9dbbc73e4 100644
--- a/qemu/include/standard-headers/linux/types.h
+++ b/qemu/include/standard-headers/linux/types.h
@@ -1,2 +1,3 @@
-#include <stdint.h>
-#include "qemu/compiler.h"
+/* For QEMU all types are already defined via osdep.h, so this
+ * header does not need to do anything.
+ */
diff --git a/qemu/include/standard-headers/linux/virtio_balloon.h b/qemu/include/standard-headers/linux/virtio_balloon.h
index 2e2a6dcf3..9d06ccd06 100644
--- a/qemu/include/standard-headers/linux/virtio_balloon.h
+++ b/qemu/include/standard-headers/linux/virtio_balloon.h
@@ -51,7 +51,8 @@ struct virtio_balloon_config {
 #define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
 #define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
 #define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
-#define VIRTIO_BALLOON_S_NR       6
+#define VIRTIO_BALLOON_S_AVAIL    6   /* Available memory as in /proc */
+#define VIRTIO_BALLOON_S_NR       7
 
 /*
  * Memory statistics structure.
diff --git a/qemu/include/standard-headers/linux/virtio_blk.h b/qemu/include/standard-headers/linux/virtio_blk.h
index cd601f406..ab16ec5fd 100644
--- a/qemu/include/standard-headers/linux/virtio_blk.h
+++ b/qemu/include/standard-headers/linux/virtio_blk.h
@@ -43,10 +43,10 @@
 #ifndef VIRTIO_BLK_NO_LEGACY
 #define VIRTIO_BLK_F_BARRIER	0	/* Does host support barriers? */
 #define VIRTIO_BLK_F_SCSI	7	/* Supports scsi command passthru */
-#define VIRTIO_BLK_F_WCE	9	/* Writeback mode enabled after reset */
+#define VIRTIO_BLK_F_FLUSH	9	/* Flush command supported */
 #define VIRTIO_BLK_F_CONFIG_WCE	11	/* Writeback mode available in config */
-/* Old (deprecated) name for VIRTIO_BLK_F_WCE. */
-#define VIRTIO_BLK_F_FLUSH VIRTIO_BLK_F_WCE
+/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
+#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
 #endif /* !VIRTIO_BLK_NO_LEGACY */
 
 #define VIRTIO_BLK_ID_BYTES	20	/* ID string length */
diff --git a/qemu/include/standard-headers/linux/virtio_gpu.h b/qemu/include/standard-headers/linux/virtio_gpu.h
index 72ef815f5..c1c8f0751 100644
--- a/qemu/include/standard-headers/linux/virtio_gpu.h
+++ b/qemu/include/standard-headers/linux/virtio_gpu.h
@@ -40,6 +40,8 @@
 
 #include "standard-headers/linux/types.h"
 
+#define VIRTIO_GPU_F_VIRGL 0
+
 enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_UNDEFINED = 0,
 
@@ -52,6 +54,18 @@ enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D,
 	VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING,
 	VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING,
+	VIRTIO_GPU_CMD_GET_CAPSET_INFO,
+	VIRTIO_GPU_CMD_GET_CAPSET,
+
+	/* 3d commands */
+	VIRTIO_GPU_CMD_CTX_CREATE = 0x0200,
+	VIRTIO_GPU_CMD_CTX_DESTROY,
+	VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE,
+	VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE,
+	VIRTIO_GPU_CMD_RESOURCE_CREATE_3D,
+	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D,
+	VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D,
+	VIRTIO_GPU_CMD_SUBMIT_3D,
 
 	/* cursor commands */
 	VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
@@ -60,6 +74,8 @@ enum virtio_gpu_ctrl_type {
 	/* success responses */
 	VIRTIO_GPU_RESP_OK_NODATA = 0x1100,
 	VIRTIO_GPU_RESP_OK_DISPLAY_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET,
 
 	/* error responses */
 	VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
@@ -180,13 +196,107 @@ struct virtio_gpu_resp_display_info {
 	} pmodes[VIRTIO_GPU_MAX_SCANOUTS];
 };
 
+/* data passed in the control vq, 3d related */
+
+struct virtio_gpu_box {
+	uint32_t x, y, z;
+	uint32_t w, h, d;
+};
+
+/* VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D, VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D */
+struct virtio_gpu_transfer_host_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_box box;
+	uint64_t offset;
+	uint32_t resource_id;
+	uint32_t level;
+	uint32_t stride;
+	uint32_t layer_stride;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_CREATE_3D */
+#define VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP (1 << 0)
+struct virtio_gpu_resource_create_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t target;
+	uint32_t format;
+	uint32_t bind;
+	uint32_t width;
+	uint32_t height;
+	uint32_t depth;
+	uint32_t array_size;
+	uint32_t last_level;
+	uint32_t nr_samples;
+	uint32_t flags;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_CTX_CREATE */
+struct virtio_gpu_ctx_create {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t nlen;
+	uint32_t padding;
+	char debug_name[64];
+};
+
+/* VIRTIO_GPU_CMD_CTX_DESTROY */
+struct virtio_gpu_ctx_destroy {
+	struct virtio_gpu_ctrl_hdr hdr;
+};
+
+/* VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE, VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE */
+struct virtio_gpu_ctx_resource {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_SUBMIT_3D */
+struct virtio_gpu_cmd_submit {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t size;
+	uint32_t padding;
+};
+
+#define VIRTIO_GPU_CAPSET_VIRGL 1
+
+/* VIRTIO_GPU_CMD_GET_CAPSET_INFO */
+struct virtio_gpu_get_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_index;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET_INFO */
+struct virtio_gpu_resp_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_max_version;
+	uint32_t capset_max_size;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_GET_CAPSET */
+struct virtio_gpu_get_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_version;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET */
+struct virtio_gpu_resp_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint8_t capset_data[];
+};
+
 #define VIRTIO_GPU_EVENT_DISPLAY (1 << 0)
 
 struct virtio_gpu_config {
 	uint32_t events_read;
 	uint32_t events_clear;
 	uint32_t num_scanouts;
-	uint32_t reserved;
+	uint32_t num_capsets;
 };
 
 /* simple formats for fbcon/X use */
diff --git a/qemu/include/standard-headers/linux/virtio_ring.h b/qemu/include/standard-headers/linux/virtio_ring.h
index 6fe276faf..023c6db04 100644
--- a/qemu/include/standard-headers/linux/virtio_ring.h
+++ b/qemu/include/standard-headers/linux/virtio_ring.h
@@ -31,6 +31,7 @@
  * SUCH DAMAGE.
  *
  * Copyright Rusty Russell IBM Corporation 2007. */
+#include <stdint.h>
 #include "standard-headers/linux/types.h"
 #include "standard-headers/linux/virtio_types.h"
 
@@ -143,7 +144,7 @@ static inline void vring_init(struct vring *vr, unsigned int num, void *p,
 	vr->num = num;
 	vr->desc = p;
 	vr->avail = p + num*sizeof(struct vring_desc);
-	vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + sizeof(__virtio16)
+	vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16)
 		+ align-1) & ~(align - 1));
 }
 
diff --git a/qemu/include/sysemu/accel.h b/qemu/include/sysemu/accel.h
index 997720f36..a74b2faf5 100644
--- a/qemu/include/sysemu/accel.h
+++ b/qemu/include/sysemu/accel.h
@@ -23,7 +23,6 @@
 #ifndef HW_ACCEL_H
 #define HW_ACCEL_H
 
-#include "qemu/typedefs.h"
 #include "qom/object.h"
 
 typedef struct AccelState {
diff --git a/qemu/include/sysemu/balloon.h b/qemu/include/sysemu/balloon.h
index 17fe30070..3f976b49e 100644
--- a/qemu/include/sysemu/balloon.h
+++ b/qemu/include/sysemu/balloon.h
@@ -22,5 +22,7 @@ typedef void (QEMUBalloonStatus)(void *opaque, BalloonInfo *info);
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
 			     QEMUBalloonStatus *stat_func, void *opaque);
 void qemu_remove_balloon_handler(void *opaque);
+bool qemu_balloon_is_inhibited(void);
+void qemu_balloon_inhibit(bool state);
 
 #endif
diff --git a/qemu/include/sysemu/block-backend.h b/qemu/include/sysemu/block-backend.h
index 8fc960fcb..c62b6fe96 100644
--- a/qemu/include/sysemu/block-backend.h
+++ b/qemu/include/sysemu/block-backend.h
@@ -13,8 +13,7 @@
 #ifndef BLOCK_BACKEND_H
 #define BLOCK_BACKEND_H
 
-#include "qemu/typedefs.h"
-#include "qapi/error.h"
+#include "qemu/iov.h"
 
 /*
  * TODO Have to include block/block.h for a bunch of block layer
@@ -60,22 +59,32 @@ typedef struct BlockDevOps {
     void (*resize_cb)(void *opaque);
 } BlockDevOps;
 
-BlockBackend *blk_new(const char *name, Error **errp);
-BlockBackend *blk_new_with_bs(const char *name, Error **errp);
-BlockBackend *blk_new_open(const char *name, const char *filename,
-                           const char *reference, QDict *options, int flags,
-                           Error **errp);
+BlockBackend *blk_new(Error **errp);
+BlockBackend *blk_new_with_bs(Error **errp);
+BlockBackend *blk_new_open(const char *filename, const char *reference,
+                           QDict *options, int flags, Error **errp);
+int blk_get_refcnt(BlockBackend *blk);
 void blk_ref(BlockBackend *blk);
 void blk_unref(BlockBackend *blk);
+void blk_remove_all_bs(void);
 const char *blk_name(BlockBackend *blk);
 BlockBackend *blk_by_name(const char *name);
 BlockBackend *blk_next(BlockBackend *blk);
+BlockDriverState *blk_next_root_bs(BlockDriverState *bs);
+bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp);
+void monitor_remove_blk(BlockBackend *blk);
 
 BlockDriverState *blk_bs(BlockBackend *blk);
+void blk_remove_bs(BlockBackend *blk);
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs);
 
-void blk_hide_on_behalf_of_hmp_drive_del(BlockBackend *blk);
-
+void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow);
 void blk_iostatus_enable(BlockBackend *blk);
+bool blk_iostatus_is_enabled(const BlockBackend *blk);
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk);
+void blk_iostatus_disable(BlockBackend *blk);
+void blk_iostatus_reset(BlockBackend *blk);
+void blk_iostatus_set_err(BlockBackend *blk, int error);
 int blk_attach_dev(BlockBackend *blk, void *dev);
 void blk_attach_dev_nofail(BlockBackend *blk, void *dev);
 void blk_detach_dev(BlockBackend *blk, void *dev);
@@ -118,8 +127,11 @@ int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors);
 int blk_co_flush(BlockBackend *blk);
 int blk_flush(BlockBackend *blk);
 int blk_flush_all(void);
+int blk_commit_all(void);
 void blk_drain(BlockBackend *blk);
 void blk_drain_all(void);
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error);
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read);
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error);
@@ -130,12 +142,15 @@ int blk_is_sg(BlockBackend *blk);
 int blk_enable_write_cache(BlockBackend *blk);
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce);
 void blk_invalidate_cache(BlockBackend *blk, Error **errp);
-int blk_is_inserted(BlockBackend *blk);
+bool blk_is_inserted(BlockBackend *blk);
+bool blk_is_available(BlockBackend *blk);
 void blk_lock_medium(BlockBackend *blk, bool locked);
 void blk_eject(BlockBackend *blk, bool eject_flag);
 int blk_get_flags(BlockBackend *blk);
 int blk_get_max_transfer_length(BlockBackend *blk);
+int blk_get_max_iov(BlockBackend *blk);
 void blk_set_guest_block_size(BlockBackend *blk, int align);
+void *blk_try_blockalign(BlockBackend *blk, size_t size);
 void *blk_blockalign(BlockBackend *blk, size_t size);
 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp);
 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason);
@@ -151,10 +166,15 @@ void blk_remove_aio_context_notifier(BlockBackend *blk,
                                                                   void *),
                                      void (*detach_aio_context)(void *),
                                      void *opaque);
-void blk_add_close_notifier(BlockBackend *blk, Notifier *notify);
+void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify);
+void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify);
 void blk_io_plug(BlockBackend *blk);
 void blk_io_unplug(BlockBackend *blk);
 BlockAcctStats *blk_get_stats(BlockBackend *blk);
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk);
+void blk_update_root_state(BlockBackend *blk);
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs);
+int blk_get_open_flags_from_root_state(BlockBackend *blk);
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
                   BlockCompletionFunc *cb, void *opaque);
@@ -169,5 +189,8 @@ int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
 int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size);
 int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz);
 int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo);
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret);
 
 #endif
diff --git a/qemu/include/sysemu/blockdev.h b/qemu/include/sysemu/blockdev.h
index 310415025..16432f350 100644
--- a/qemu/include/sysemu/blockdev.h
+++ b/qemu/include/sysemu/blockdev.h
@@ -11,7 +11,6 @@
 #define BLOCKDEV_H
 
 #include "block/block.h"
-#include "qapi/error.h"
 #include "qemu/queue.h"
 
 void blockdev_mark_auto_del(BlockBackend *blk);
@@ -20,7 +19,7 @@ void blockdev_auto_del(BlockBackend *blk);
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
     /*
-     * IF_IDE must be zero, because we want QEMUMachine member
+     * IF_IDE must be zero, because we want MachineClass member
      * block_default_type to default-initialize to IF_IDE
      */
     IF_IDE = 0,
@@ -63,8 +62,6 @@ DriveInfo *drive_new(QemuOpts *arg, BlockInterfaceType block_default_type);
 
 /* device-hotplug */
 
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp);
 void hmp_commit(Monitor *mon, const QDict *qdict);
 void hmp_drive_del(Monitor *mon, const QDict *qdict);
 #endif
diff --git a/qemu/include/sysemu/char.h b/qemu/include/sysemu/char.h
index 832b7fead..307fd8fde 100644
--- a/qemu/include/sysemu/char.h
+++ b/qemu/include/sysemu/char.h
@@ -41,6 +41,11 @@ typedef struct {
 #define CHR_IOCTL_PP_EPP_WRITE       11
 #define CHR_IOCTL_PP_DATA_DIR        12
 
+struct ParallelIOArg {
+    void *buffer;
+    int count;
+};
+
 #define CHR_IOCTL_SERIAL_SET_TIOCM   13
 #define CHR_IOCTL_SERIAL_GET_TIOCM   14
 
@@ -77,6 +82,7 @@ struct CharDriverState {
     void *opaque;
     char *label;
     char *filename;
+    int logfd;
     int be_open;
     int fe_open;
     int explicit_fe_open;
@@ -85,17 +91,20 @@ struct CharDriverState {
     int is_mux;
     guint fd_in_tag;
     QemuOpts *opts;
+    bool replay;
     QTAILQ_ENTRY(CharDriverState) next;
 };
 
 /**
- * @qemu_chr_alloc:
+ * qemu_chr_alloc:
+ * @backend: the common backend config
+ * @errp: pointer to a NULL-initialized error object
  *
  * Allocate and initialize a new CharDriverState.
  *
- * Returns: a newly allocated CharDriverState.
+ * Returns: a newly allocated CharDriverState, or NULL on error.
  */
-CharDriverState *qemu_chr_alloc(void);
+CharDriverState *qemu_chr_alloc(ChardevCommon *backend, Error **errp);
 
 /**
  * @qemu_chr_new_from_opts:
@@ -112,6 +121,16 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
                                     Error **errp);
 
 /**
+ * @qemu_chr_parse_common:
+ *
+ * Parse the common options available to all character backends.
+ *
+ * @opts the options that still need parsing
+ * @backend a new backend
+ */
+void qemu_chr_parse_common(QemuOpts *opts, ChardevCommon *backend);
+
+/**
  * @qemu_chr_new:
  *
  * Create a new character backend from a URI.
@@ -126,13 +145,37 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename,
                               void (*init)(struct CharDriverState *s));
 
 /**
+ * @qemu_chr_new_noreplay:
+ *
+ * Create a new character backend from a URI.
+ * Character device communications are not written
+ * into the replay log.
+ *
+ * @label the name of the backend
+ * @filename the URI
+ * @init not sure..
+ *
+ * Returns: a new character backend
+ */
+CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
+                                       void (*init)(struct CharDriverState *s));
+
+/**
  * @qemu_chr_delete:
  *
- * Destroy a character backend.
+ * Destroy a character backend and remove it from the list of
+ * identified character backends.
  */
 void qemu_chr_delete(CharDriverState *chr);
 
 /**
+ * @qemu_chr_free:
+ *
+ * Destroy a character backend.
+ */
+void qemu_chr_free(CharDriverState *chr);
+
+/**
  * @qemu_chr_fe_set_echo:
  *
  * Ask the backend to override its normal echo setting.  This only really
@@ -320,6 +363,15 @@ int qemu_chr_be_can_write(CharDriverState *s);
  */
 void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len);
 
+/**
+ * @qemu_chr_be_write_impl:
+ *
+ * Implementation of back end writing. Used by replay module.
+ *
+ * @buf a buffer to receive data from the front end
+ * @len the number of bytes to receive from the front end
+ */
+void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len);
 
 /**
  * @qemu_chr_be_event:
@@ -345,27 +397,16 @@ bool chr_is_ringbuf(const CharDriverState *chr);
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename);
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp));
-
-/* add an eventfd to the qemu devices that are polled */
-CharDriverState *qemu_chr_open_eventfd(int eventfd);
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp));
 
 extern int term_escape_char;
 
 CharDriverState *qemu_char_get_next_serial(void);
 
-/* msmouse */
-CharDriverState *qemu_chr_open_msmouse(void);
-
-/* testdev.c */
-CharDriverState *chr_testdev_init(void);
-
-/* baum.c */
-CharDriverState *chr_baum_init(void);
-
 /* console.c */
-typedef CharDriverState *(VcHandler)(ChardevVC *vc);
-
+typedef CharDriverState *(VcHandler)(ChardevVC *vc, Error **errp);
 void register_vc_handler(VcHandler *handler);
-CharDriverState *vc_init(ChardevVC *vc);
+
 #endif
diff --git a/qemu/include/sysemu/cpus.h b/qemu/include/sysemu/cpus.h
index 3f162a9e0..3d1e5ba1e 100644
--- a/qemu/include/sysemu/cpus.h
+++ b/qemu/include/sysemu/cpus.h
@@ -2,6 +2,7 @@
 #define QEMU_CPUS_H
 
 /* cpus.c */
+bool qemu_in_vcpu_thread(void);
 void qemu_init_cpu_loop(void);
 void resume_all_vcpus(void);
 void pause_all_vcpus(void);
@@ -10,7 +11,6 @@ void cpu_stop_current(void);
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);
-void cpu_clean_all_dirty(void);
 
 void qtest_clock_warp(int64_t dest);
 
diff --git a/qemu/include/sysemu/device_tree.h b/qemu/include/sysemu/device_tree.h
index 359e14304..705650aad 100644
--- a/qemu/include/sysemu/device_tree.h
+++ b/qemu/include/sysemu/device_tree.h
@@ -16,6 +16,32 @@
 
 void *create_device_tree(int *sizep);
 void *load_device_tree(const char *filename_path, int *sizep);
+#ifdef CONFIG_LINUX
+/**
+ * load_device_tree_from_sysfs: reads the device tree information in the
+ * /proc/device-tree directory and return the corresponding binary blob
+ * buffer pointer. Asserts in case of error.
+ */
+void *load_device_tree_from_sysfs(void);
+#endif
+
+/**
+ * qemu_fdt_node_path: return the paths of nodes matching a given
+ * name and compat string
+ * @fdt: pointer to the dt blob
+ * @name: node name
+ * @compat: compatibility string
+ * @errp: handle to an error object
+ *
+ * returns a newly allocated NULL-terminated array of node paths.
+ * Use g_strfreev() to free it. If one or more nodes were found, the
+ * array contains the path of each node and the last element equals to
+ * NULL. If there is no error but no matching node was found, the
+ * returned array contains a single element equal to NULL. If an error
+ * was encountered when parsing the blob, the function returns NULL
+ */
+char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
+                          Error **errp);
 
 int qemu_fdt_setprop(void *fdt, const char *node_path,
                      const char *property, const void *val, int size);
@@ -28,10 +54,33 @@ int qemu_fdt_setprop_string(void *fdt, const char *node_path,
 int qemu_fdt_setprop_phandle(void *fdt, const char *node_path,
                              const char *property,
                              const char *target_node_path);
+/**
+ * qemu_fdt_getprop: retrieve the value of a given property
+ * @fdt: pointer to the device tree blob
+ * @node_path: node path
+ * @property: name of the property to find
+ * @lenp: fdt error if any or length of the property on success
+ * @errp: handle to an error object
+ *
+ * returns a pointer to the property on success and NULL on failure
+ */
 const void *qemu_fdt_getprop(void *fdt, const char *node_path,
-                             const char *property, int *lenp);
+                             const char *property, int *lenp,
+                             Error **errp);
+/**
+ * qemu_fdt_getprop_cell: retrieve the value of a given 4 byte property
+ * @fdt: pointer to the device tree blob
+ * @node_path: node path
+ * @property: name of the property to find
+ * @lenp: fdt error if any or -EINVAL if the property size is different from
+ *        4 bytes, or 4 (expected length of the property) upon success.
+ * @errp: handle to an error object
+ *
+ * returns the property value on success
+ */
 uint32_t qemu_fdt_getprop_cell(void *fdt, const char *node_path,
-                               const char *property);
+                               const char *property, int *lenp,
+                               Error **errp);
 uint32_t qemu_fdt_get_phandle(void *fdt, const char *path);
 uint32_t qemu_fdt_alloc_phandle(void *fdt);
 int qemu_fdt_nop_node(void *fdt, const char *node_path);
diff --git a/qemu/include/sysemu/dma.h b/qemu/include/sysemu/dma.h
index efa8b9993..b0fbb9bb3 100644
--- a/qemu/include/sysemu/dma.h
+++ b/qemu/include/sysemu/dma.h
@@ -10,7 +10,6 @@
 #ifndef DMA_H
 #define DMA_H
 
-#include <stdio.h>
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
diff --git a/qemu/include/sysemu/dump-arch.h b/qemu/include/sysemu/dump-arch.h
index 9c95cede3..e25b02e99 100644
--- a/qemu/include/sysemu/dump-arch.h
+++ b/qemu/include/sysemu/dump-arch.h
@@ -15,9 +15,12 @@
 #define DUMP_ARCH_H
 
 typedef struct ArchDumpInfo {
-    int d_machine;  /* Architecture */
-    int d_endian;   /* ELFDATA2LSB or ELFDATA2MSB */
-    int d_class;    /* ELFCLASS32 or ELFCLASS64 */
+    int d_machine;           /* Architecture */
+    int d_endian;            /* ELFDATA2LSB or ELFDATA2MSB */
+    int d_class;             /* ELFCLASS32 or ELFCLASS64 */
+    uint32_t page_size;      /* The target's page size. If it's variable and
+                              * unknown, then this should be the maximum. */
+    uint64_t phys_base;      /* The target's physmem base. */
 } ArchDumpInfo;
 
 struct GuestPhysBlockList; /* memory_mapping.h */
diff --git a/qemu/include/sysemu/dump.h b/qemu/include/sysemu/dump.h
index 7e4ec5c7d..ef931be46 100644
--- a/qemu/include/sysemu/dump.h
+++ b/qemu/include/sysemu/dump.h
@@ -20,12 +20,9 @@
 #define VERSION_FLAT_HEADER         (1)    /* version of flattened format */
 #define END_FLAG_FLAT_HEADER        (-1)
 
+#ifndef ARCH_PFN_OFFSET
 #define ARCH_PFN_OFFSET             (0)
-
-#define paddr_to_pfn(X) \
-    (((unsigned long long)(X) >> TARGET_PAGE_BITS) - ARCH_PFN_OFFSET)
-#define pfn_to_paddr(X) \
-    (((unsigned long long)(X) + ARCH_PFN_OFFSET) << TARGET_PAGE_BITS)
+#endif
 
 /*
  * flag for compressed format
@@ -36,15 +33,12 @@
 
 #define KDUMP_SIGNATURE             "KDUMP   "
 #define SIG_LEN                     (sizeof(KDUMP_SIGNATURE) - 1)
-#define PHYS_BASE                   (0)
 #define DUMP_LEVEL                  (1)
 #define DISKDUMP_HEADER_BLOCKS      (1)
-#define BUFSIZE_BITMAP              (TARGET_PAGE_SIZE)
-#define PFN_BUFBITMAP               (CHAR_BIT * BUFSIZE_BITMAP)
-#define BUFSIZE_DATA_CACHE          (TARGET_PAGE_SIZE * 4)
 
 #include "sysemu/dump-arch.h"
 #include "sysemu/memory_mapping.h"
+#include "qapi-types.h"
 
 typedef struct QEMU_PACKED MakedumpfileHeader {
     char signature[16];     /* = "makedumpfile" */
@@ -183,6 +177,20 @@ typedef struct DumpState {
     off_t offset_page;          /* offset of page part in vmcore */
     size_t num_dumpable;        /* number of page that can be dumped */
     uint32_t flag_compress;     /* indicate the compression format */
+    DumpStatus status;          /* current dump status */
+
+    bool has_format;              /* whether format is provided */
+    DumpGuestMemoryFormat format; /* valid only if has_format == true */
+    QemuThread dump_thread;       /* thread for detached dump */
+
+    int64_t total_size;          /* total memory size (in bytes) to
+                                  * be dumped. When filter is
+                                  * enabled, this will only count
+                                  * those to be written. */
+    int64_t written_size;        /* written memory size (in bytes),
+                                  * this could be used to calculate
+                                  * how much work we have
+                                  * finished. */
 } DumpState;
 
 uint16_t cpu_to_dump16(DumpState *s, uint16_t val);
diff --git a/qemu/include/sysemu/hostmem.h b/qemu/include/sysemu/hostmem.h
index 1ce439415..a19801d20 100644
--- a/qemu/include/sysemu/hostmem.h
+++ b/qemu/include/sysemu/hostmem.h
@@ -14,7 +14,6 @@
 
 #include "sysemu/sysemu.h" /* for MAX_NODES */
 #include "qom/object.h"
-#include "qapi/error.h"
 #include "exec/memory.h"
 #include "qemu/option.h"
 #include "qemu/bitmap.h"
diff --git a/qemu/include/sysemu/kvm.h b/qemu/include/sysemu/kvm.h
index 983e99e1e..0e18f15c9 100644
--- a/qemu/include/sysemu/kvm.h
+++ b/qemu/include/sysemu/kvm.h
@@ -14,8 +14,6 @@
 #ifndef QEMU_KVM_H
 #define QEMU_KVM_H
 
-#include <errno.h>
-#include "config-host.h"
 #include "qemu/queue.h"
 #include "qom/cpu.h"
 #include "exec/memattrs.h"
@@ -43,6 +41,7 @@
 
 extern bool kvm_allowed;
 extern bool kvm_kernel_irqchip;
+extern bool kvm_split_irqchip;
 extern bool kvm_async_interrupts_allowed;
 extern bool kvm_halt_in_kernel_allowed;
 extern bool kvm_eventfds_allowed;
@@ -52,6 +51,8 @@ extern bool kvm_msi_via_irqfd_allowed;
 extern bool kvm_gsi_routing_allowed;
 extern bool kvm_gsi_direct_mapping;
 extern bool kvm_readonly_mem_allowed;
+extern bool kvm_direct_msi_allowed;
+extern bool kvm_ioeventfd_any_length_allowed;
 
 #if defined CONFIG_KVM || !defined NEED_CPU_H
 #define kvm_enabled()           (kvm_allowed)
@@ -69,6 +70,16 @@ extern bool kvm_readonly_mem_allowed;
 #define kvm_irqchip_in_kernel() (kvm_kernel_irqchip)
 
 /**
+ * kvm_irqchip_is_split:
+ *
+ * Returns: true if the user asked us to split the irqchip
+ * implementation between user and kernel space. The details are
+ * architecture and machine specific. On PC, it means that the PIC,
+ * IOAPIC, and PIT are in user space while the LAPIC is in the kernel.
+ */
+#define kvm_irqchip_is_split() (kvm_split_irqchip)
+
+/**
  * kvm_async_interrupts_enabled:
  *
  * Returns: true if we can deliver interrupts to KVM
@@ -145,9 +156,23 @@ extern bool kvm_readonly_mem_allowed;
  */
 #define kvm_readonly_mem_enabled() (kvm_readonly_mem_allowed)
 
+/**
+ * kvm_direct_msi_enabled:
+ *
+ * Returns: true if KVM allows direct MSI injection.
+ */
+#define kvm_direct_msi_enabled() (kvm_direct_msi_allowed)
+
+/**
+ * kvm_ioeventfd_any_length_enabled:
+ * Returns: true if KVM allows any length io eventfd.
+ */
+#define kvm_ioeventfd_any_length_enabled() (kvm_ioeventfd_any_length_allowed)
+
 #else
 #define kvm_enabled()           (0)
 #define kvm_irqchip_in_kernel() (false)
+#define kvm_irqchip_is_split() (false)
 #define kvm_async_interrupts_enabled() (false)
 #define kvm_halt_in_kernel() (false)
 #define kvm_eventfds_enabled() (false)
@@ -157,6 +182,8 @@ extern bool kvm_readonly_mem_allowed;
 #define kvm_gsi_routing_allowed() (false)
 #define kvm_gsi_direct_mapping() (false)
 #define kvm_readonly_mem_enabled() (false)
+#define kvm_direct_msi_enabled() (false)
+#define kvm_ioeventfd_any_length_enabled() (false)
 #endif
 
 struct kvm_run;
@@ -182,8 +209,6 @@ int kvm_has_sync_mmu(void);
 int kvm_has_vcpu_events(void);
 int kvm_has_robust_singlestep(void);
 int kvm_has_debugregs(void);
-int kvm_has_xsave(void);
-int kvm_has_xcrs(void);
 int kvm_has_pit_state2(void);
 int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
@@ -210,6 +235,10 @@ int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset);
 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_on_sigbus(int code, void *addr);
 
+/* interface with exec.c */
+
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
+
 /* internal API */
 
 int kvm_ioctl(KVMState *s, int type, ...);
@@ -240,6 +269,32 @@ int kvm_device_ioctl(int fd, int type, ...);
 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr);
 
 /**
+ * kvm_device_check_attr - check for existence of a specific device attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to query for
+ *
+ * Returns: 1 if the attribute exists
+ *          0 if the attribute either does not exist or if the vm device
+ *            interface is unavailable
+ */
+int kvm_device_check_attr(int fd, uint32_t group, uint64_t attr);
+
+/**
+ * kvm_device_access - set or get value of a specific vm attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to set or get
+ * @val: pointer to a storage area for the value
+ * @write: true for set and false for get operation
+ *
+ * This function is not allowed to fail. Use kvm_device_check_attr()
+ * in order to check for the availability of optional attributes.
+ */
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write);
+
+/**
  * kvm_create_device - create a KVM device for the device control API
  * @KVMState: The KVMState pointer
  * @type: The KVM device type (see Documentation/virtual/kvm/devices in the
@@ -251,6 +306,15 @@ int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr);
  */
 int kvm_create_device(KVMState *s, uint64_t type, bool test);
 
+/**
+ * kvm_device_supported - probe whether KVM supports specific device
+ *
+ * @vmfd: The fd handler for VM
+ * @type: type of device
+ *
+ * @return: true if supported, otherwise false.
+ */
+bool kvm_device_supported(int vmfd, uint64_t type);
 
 /* Arch specific hooks */
 
@@ -261,6 +325,8 @@ MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run);
 
 int kvm_arch_handle_exit(CPUState *cpu, struct kvm_run *run);
 
+int kvm_arch_handle_ioapic_eoi(CPUState *cpu, struct kvm_run *run);
+
 int kvm_arch_process_async_events(CPUState *cpu);
 
 int kvm_arch_get_registers(CPUState *cpu);
@@ -287,7 +353,7 @@ int kvm_arch_on_sigbus(int code, void *addr);
 void kvm_arch_init_irq_routing(KVMState *s);
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data);
+                             uint64_t address, uint32_t data, PCIDevice *dev);
 
 int kvm_arch_msi_data_to_gsi(uint32_t data);
 
@@ -380,7 +446,6 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
-void kvm_cpu_clean_state(CPUState *cpu);
 
 /* generic hooks - to be moved/refactored once there are more users */
 
@@ -405,18 +470,13 @@ static inline void cpu_synchronize_post_init(CPUState *cpu)
     }
 }
 
-static inline void cpu_clean_state(CPUState *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_cpu_clean_state(cpu);
-    }
-}
-
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev);
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint);
 
 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
                                        EventNotifier *rn, int virq);
@@ -434,6 +494,7 @@ void kvm_init_irq_routing(KVMState *s);
 /**
  * kvm_arch_irqchip_create:
  * @KVMState: The KVMState pointer
+ * @MachineState: The MachineState pointer
  *
  * Allow architectures to create an in-kernel irq chip themselves.
  *
@@ -441,7 +502,7 @@ void kvm_init_irq_routing(KVMState *s);
  *            0: irq chip was not created
  *          > 0: irq chip was created
  */
-int kvm_arch_irqchip_create(KVMState *s);
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s);
 
 /**
  * kvm_set_one_reg - set a register value in KVM via KVM_SET_ONE_REG ioctl
diff --git a/qemu/include/sysemu/memory_mapping.h b/qemu/include/sysemu/memory_mapping.h
index a75d59a55..706152d53 100644
--- a/qemu/include/sysemu/memory_mapping.h
+++ b/qemu/include/sysemu/memory_mapping.h
@@ -15,7 +15,7 @@
 #define MEMORY_MAPPING_H
 
 #include "qemu/queue.h"
-#include "qemu/typedefs.h"
+#include "exec/memory.h"
 
 typedef struct GuestPhysBlock {
     /* visible to guest, reflects PCI hole, etc */
@@ -27,6 +27,9 @@ typedef struct GuestPhysBlock {
     /* points into host memory */
     uint8_t *host_addr;
 
+    /* points to the MemoryRegion that this block belongs to */
+    MemoryRegion *mr;
+
     QTAILQ_ENTRY(GuestPhysBlock) next;
 } GuestPhysBlock;
 
diff --git a/qemu/include/sysemu/numa.h b/qemu/include/sysemu/numa.h
index a6392bc50..bb184c9cf 100644
--- a/qemu/include/sysemu/numa.h
+++ b/qemu/include/sysemu/numa.h
@@ -1,7 +1,6 @@
 #ifndef SYSEMU_NUMA_H
 #define SYSEMU_NUMA_H
 
-#include <stdint.h>
 #include "qemu/bitmap.h"
 #include "qemu/option.h"
 #include "sysemu/sysemu.h"
diff --git a/qemu/include/sysemu/os-posix.h b/qemu/include/sysemu/os-posix.h
index f1315213f..07e3e5ae9 100644
--- a/qemu/include/sysemu/os-posix.h
+++ b/qemu/include/sysemu/os-posix.h
@@ -26,7 +26,12 @@
 #ifndef QEMU_OS_POSIX_H
 #define QEMU_OS_POSIX_H
 
-#include <sys/time.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/un.h>
 
 void os_set_line_buffering(void);
 void os_set_proc_name(const char *s);
@@ -35,6 +40,9 @@ void os_daemonize(void);
 void os_setup_post(void);
 int os_mlock(void);
 
+#define closesocket(s) close(s)
+#define ioctlsocket(s, r, v) ioctl(s, r, v)
+
 typedef struct timeval qemu_timeval;
 #define qemu_gettimeofday(tp) gettimeofday(tp, NULL)
 
diff --git a/qemu/include/sysemu/os-win32.h b/qemu/include/sysemu/os-win32.h
index 4035c4fe5..17aad3b20 100644
--- a/qemu/include/sysemu/os-win32.h
+++ b/qemu/include/sysemu/os-win32.h
@@ -26,34 +26,9 @@
 #ifndef QEMU_OS_WIN32_H
 #define QEMU_OS_WIN32_H
 
-#include <windows.h>
 #include <winsock2.h>
-
-/* Workaround for older versions of MinGW. */
-#ifndef ECONNREFUSED
-# define ECONNREFUSED WSAECONNREFUSED
-#endif
-#ifndef EINPROGRESS
-# define EINPROGRESS  WSAEINPROGRESS
-#endif
-#ifndef EHOSTUNREACH
-# define EHOSTUNREACH WSAEHOSTUNREACH
-#endif
-#ifndef EINTR
-# define EINTR        WSAEINTR
-#endif
-#ifndef EINPROGRESS
-# define EINPROGRESS  WSAEINPROGRESS
-#endif
-#ifndef ENETUNREACH
-# define ENETUNREACH  WSAENETUNREACH
-#endif
-#ifndef ENOTCONN
-# define ENOTCONN     WSAENOTCONN
-#endif
-#ifndef EWOULDBLOCK
-# define EWOULDBLOCK  WSAEWOULDBLOCK
-#endif
+#include <windows.h>
+#include <ws2tcpip.h>
 
 #if defined(_WIN64)
 /* On w64, setjmp is implemented by _setjmp which needs a second parameter.
@@ -73,11 +48,12 @@
 #define siglongjmp(env, val) longjmp(env, val)
 
 /* Missing POSIX functions. Don't use MinGW-w64 macros. */
+#ifndef CONFIG_LOCALTIME_R
 #undef gmtime_r
 struct tm *gmtime_r(const time_t *timep, struct tm *result);
 #undef localtime_r
 struct tm *localtime_r(const time_t *timep, struct tm *result);
-
+#endif /* CONFIG_LOCALTIME_R */
 
 static inline void os_setup_signal_handling(void) {}
 static inline void os_daemonize(void) {}
@@ -85,7 +61,7 @@ static inline void os_setup_post(void) {}
 void os_set_line_buffering(void);
 static inline void os_set_proc_name(const char *dummy) {}
 
-size_t getpagesize(void);
+int getpagesize(void);
 
 #if !defined(EPROTONOSUPPORT)
 # define EPROTONOSUPPORT EINVAL
@@ -109,4 +85,100 @@ static inline int os_mlock(void)
     return -ENOSYS;
 }
 
+#define fsync _commit
+
+#if !defined(lseek)
+# define lseek _lseeki64
+#endif
+
+int qemu_ftruncate64(int, int64_t);
+
+#if !defined(ftruncate)
+# define ftruncate qemu_ftruncate64
+#endif
+
+static inline char *realpath(const char *path, char *resolved_path)
+{
+    _fullpath(resolved_path, path, _MAX_PATH);
+    return resolved_path;
+}
+
+
+/* We wrap all the sockets functions so that we can
+ * set errno based on WSAGetLastError()
+ */
+
+#undef connect
+#define connect qemu_connect_wrap
+int qemu_connect_wrap(int sockfd, const struct sockaddr *addr,
+                      socklen_t addrlen);
+
+#undef listen
+#define listen qemu_listen_wrap
+int qemu_listen_wrap(int sockfd, int backlog);
+
+#undef bind
+#define bind qemu_bind_wrap
+int qemu_bind_wrap(int sockfd, const struct sockaddr *addr,
+                   socklen_t addrlen);
+
+#undef socket
+#define socket qemu_socket_wrap
+int qemu_socket_wrap(int domain, int type, int protocol);
+
+#undef accept
+#define accept qemu_accept_wrap
+int qemu_accept_wrap(int sockfd, struct sockaddr *addr,
+                     socklen_t *addrlen);
+
+#undef shutdown
+#define shutdown qemu_shutdown_wrap
+int qemu_shutdown_wrap(int sockfd, int how);
+
+#undef ioctlsocket
+#define ioctlsocket qemu_ioctlsocket_wrap
+int qemu_ioctlsocket_wrap(int fd, int req, void *val);
+
+#undef closesocket
+#define closesocket qemu_closesocket_wrap
+int qemu_closesocket_wrap(int fd);
+
+#undef getsockopt
+#define getsockopt qemu_getsockopt_wrap
+int qemu_getsockopt_wrap(int sockfd, int level, int optname,
+                         void *optval, socklen_t *optlen);
+
+#undef setsockopt
+#define setsockopt qemu_setsockopt_wrap
+int qemu_setsockopt_wrap(int sockfd, int level, int optname,
+                         const void *optval, socklen_t optlen);
+
+#undef getpeername
+#define getpeername qemu_getpeername_wrap
+int qemu_getpeername_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen);
+
+#undef getsockname
+#define getsockname qemu_getsockname_wrap
+int qemu_getsockname_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen);
+
+#undef send
+#define send qemu_send_wrap
+ssize_t qemu_send_wrap(int sockfd, const void *buf, size_t len, int flags);
+
+#undef sendto
+#define sendto qemu_sendto_wrap
+ssize_t qemu_sendto_wrap(int sockfd, const void *buf, size_t len, int flags,
+                         const struct sockaddr *addr, socklen_t addrlen);
+
+#undef recv
+#define recv qemu_recv_wrap
+ssize_t qemu_recv_wrap(int sockfd, void *buf, size_t len, int flags);
+
+#undef recvfrom
+#define recvfrom qemu_recvfrom_wrap
+ssize_t qemu_recvfrom_wrap(int sockfd, void *buf, size_t len, int flags,
+                           struct sockaddr *addr, socklen_t *addrlen);
+
 #endif
diff --git a/qemu/include/sysemu/qtest.h b/qemu/include/sysemu/qtest.h
index 05473b75a..70aa40aa7 100644
--- a/qemu/include/sysemu/qtest.h
+++ b/qemu/include/sysemu/qtest.h
@@ -15,7 +15,6 @@
 #define QTEST_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 extern bool qtest_allowed;
 
diff --git a/qemu/include/sysemu/replay.h b/qemu/include/sysemu/replay.h
new file mode 100644
index 000000000..0a88393d2
--- /dev/null
+++ b/qemu/include/sysemu/replay.h
@@ -0,0 +1,136 @@
+#ifndef REPLAY_H
+#define REPLAY_H
+
+/*
+ * replay.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qapi-types.h"
+
+/* replay clock kinds */
+enum ReplayClockKind {
+    /* host_clock */
+    REPLAY_CLOCK_HOST,
+    /* virtual_rt_clock */
+    REPLAY_CLOCK_VIRTUAL_RT,
+    REPLAY_CLOCK_COUNT
+};
+typedef enum ReplayClockKind ReplayClockKind;
+
+/* IDs of the checkpoints */
+enum ReplayCheckpoint {
+    CHECKPOINT_CLOCK_WARP_START,
+    CHECKPOINT_CLOCK_WARP_ACCOUNT,
+    CHECKPOINT_RESET_REQUESTED,
+    CHECKPOINT_SUSPEND_REQUESTED,
+    CHECKPOINT_CLOCK_VIRTUAL,
+    CHECKPOINT_CLOCK_HOST,
+    CHECKPOINT_CLOCK_VIRTUAL_RT,
+    CHECKPOINT_INIT,
+    CHECKPOINT_RESET,
+    CHECKPOINT_COUNT
+};
+typedef enum ReplayCheckpoint ReplayCheckpoint;
+
+extern ReplayMode replay_mode;
+
+/* Replay process control functions */
+
+/*! Enables recording or saving event log with specified parameters */
+void replay_configure(struct QemuOpts *opts);
+/*! Initializes timers used for snapshotting and enables events recording */
+void replay_start(void);
+/*! Closes replay log file and frees other resources. */
+void replay_finish(void);
+/*! Adds replay blocker with the specified error description */
+void replay_add_blocker(Error *reason);
+
+/* Processing the instructions */
+
+/*! Returns number of executed instructions. */
+uint64_t replay_get_current_step(void);
+/*! Returns number of instructions to execute in replay mode. */
+int replay_get_instructions(void);
+/*! Updates instructions counter in replay mode. */
+void replay_account_executed_instructions(void);
+
+/* Interrupts and exceptions */
+
+/*! Called by exception handler to write or read
+    exception processing events. */
+bool replay_exception(void);
+/*! Used to determine that exception is pending.
+    Does not proceed to the next event in the log. */
+bool replay_has_exception(void);
+/*! Called by interrupt handlers to write or read
+    interrupt processing events.
+    \return true if interrupt should be processed */
+bool replay_interrupt(void);
+/*! Tries to read interrupt event from the file.
+    Returns true, when interrupt request is pending */
+bool replay_has_interrupt(void);
+
+/* Processing clocks and other time sources */
+
+/*! Save the specified clock */
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock);
+/*! Read the specified clock from the log or return cached data */
+int64_t replay_read_clock(ReplayClockKind kind);
+/*! Saves or reads the clock depending on the current replay mode. */
+#define REPLAY_CLOCK(clock, value)                                      \
+    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
+        : replay_mode == REPLAY_MODE_RECORD                             \
+            ? replay_save_clock((clock), (value))                       \
+        : (value))
+
+/* Events */
+
+/*! Called when qemu shutdown is requested. */
+void replay_shutdown_request(void);
+/*! Should be called at check points in the execution.
+    These check points are skipped, if they were not met.
+    Saves checkpoint in the SAVE mode and validates in the PLAY mode.
+    Returns 0 in PLAY mode if checkpoint was not found.
+    Returns 1 in all other cases. */
+bool replay_checkpoint(ReplayCheckpoint checkpoint);
+
+/* Asynchronous events queue */
+
+/*! Disables storing events in the queue */
+void replay_disable_events(void);
+/*! Returns true when saving events is enabled */
+bool replay_events_enabled(void);
+/*! Adds bottom half event to the queue */
+void replay_bh_schedule_event(QEMUBH *bh);
+/*! Adds input event to the queue */
+void replay_input_event(QemuConsole *src, InputEvent *evt);
+/*! Adds input sync event to the queue */
+void replay_input_sync_event(void);
+/*! Adds block layer event to the queue */
+void replay_block_event(QEMUBH *bh, uint64_t id);
+
+/* Character device */
+
+/*! Registers char driver to save it's events */
+void replay_register_char_driver(struct CharDriverState *chr);
+/*! Saves write to char device event to the log */
+void replay_chr_be_write(struct CharDriverState *s, uint8_t *buf, int len);
+/*! Writes char write return value to the replay log. */
+void replay_char_write_event_save(int res, int offset);
+/*! Reads char write return value from the replay log. */
+void replay_char_write_event_load(int *res, int *offset);
+/*! Reads information about read_all character event. */
+int replay_char_read_all_load(uint8_t *buf);
+/*! Writes character read_all error code into the replay log. */
+void replay_char_read_all_save_error(int res);
+/*! Writes character read_all execution result into the replay log. */
+void replay_char_read_all_save_buf(uint8_t *buf, int offset);
+
+#endif
diff --git a/qemu/include/sysemu/rng.h b/qemu/include/sysemu/rng.h
index 0a27c9b88..45629c4c5 100644
--- a/qemu/include/sysemu/rng.h
+++ b/qemu/include/sysemu/rng.h
@@ -15,7 +15,6 @@
 
 #include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 #define TYPE_RNG_BACKEND "rng-backend"
 #define RNG_BACKEND(obj) \
@@ -25,6 +24,7 @@
 #define RNG_BACKEND_CLASS(klass) \
     OBJECT_CLASS_CHECK(RngBackendClass, (klass), TYPE_RNG_BACKEND)
 
+typedef struct RngRequest RngRequest;
 typedef struct RngBackendClass RngBackendClass;
 typedef struct RngBackend RngBackend;
 
@@ -32,13 +32,21 @@ typedef void (EntropyReceiveFunc)(void *opaque,
                                   const void *data,
                                   size_t size);
 
+struct RngRequest
+{
+    EntropyReceiveFunc *receive_entropy;
+    uint8_t *data;
+    void *opaque;
+    size_t offset;
+    size_t size;
+    QSIMPLEQ_ENTRY(RngRequest) next;
+};
+
 struct RngBackendClass
 {
     ObjectClass parent_class;
 
-    void (*request_entropy)(RngBackend *s, size_t size,
-                            EntropyReceiveFunc *receive_entropy, void *opaque);
-    void (*cancel_requests)(RngBackend *s);
+    void (*request_entropy)(RngBackend *s, RngRequest *req);
 
     void (*opened)(RngBackend *s, Error **errp);
 };
@@ -49,8 +57,10 @@ struct RngBackend
 
     /*< protected >*/
     bool opened;
+    QSIMPLEQ_HEAD(requests, RngRequest) requests;
 };
 
+
 /**
  * rng_backend_request_entropy:
  * @s: the backend to request entropy from
@@ -71,12 +81,13 @@ void rng_backend_request_entropy(RngBackend *s, size_t size,
                                  void *opaque);
 
 /**
- * rng_backend_cancel_requests:
- * @s: the backend to cancel all pending requests in
+ * rng_backend_free_request:
+ * @s: the backend that created the request
+ * @req: the request to finalize
  *
- * Cancels all pending requests submitted by @rng_backend_request_entropy.  This
- * should be used by a device during reset or in preparation for live migration
- * to stop tracking any request.
+ * Used by child rng backend classes to finalize requests once they've been
+ * processed. The request is removed from the list of active requests and
+ * deleted.
  */
-void rng_backend_cancel_requests(RngBackend *s);
+void rng_backend_finalize_request(RngBackend *s, RngRequest *req);
 #endif
diff --git a/qemu/include/sysemu/seccomp.h b/qemu/include/sysemu/seccomp.h
index 1189fa241..cfc06008c 100644
--- a/qemu/include/sysemu/seccomp.h
+++ b/qemu/include/sysemu/seccomp.h
@@ -16,7 +16,6 @@
 #define QEMU_SECCOMP_H
 
 #include <seccomp.h>
-#include "qemu/osdep.h"
 
 int seccomp_start(void);
 #endif
diff --git a/qemu/include/sysemu/sysemu.h b/qemu/include/sysemu/sysemu.h
index 44570d17e..38fb3cad3 100644
--- a/qemu/include/sysemu/sysemu.h
+++ b/qemu/include/sysemu/sysemu.h
@@ -2,7 +2,6 @@
 #define SYSEMU_H
 /* Misc. things related to the system emulator.  */
 
-#include "qemu/typedefs.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
 #include "qemu/timer.h"
@@ -69,6 +68,8 @@ int qemu_reset_requested_get(void);
 void qemu_system_killed(int signal, pid_t pid);
 void qemu_devices_reset(void);
 void qemu_system_reset(bool report);
+void qemu_system_guest_panicked(void);
+size_t qemu_target_page_bits(void);
 
 void qemu_add_exit_notifier(Notifier *notify);
 void qemu_remove_exit_notifier(Notifier *notify);
@@ -82,14 +83,52 @@ void hmp_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
+/* Subcommands for QEMU_VM_COMMAND */
+enum qemu_vm_cmd {
+    MIG_CMD_INVALID = 0,   /* Must be 0 */
+    MIG_CMD_OPEN_RETURN_PATH,  /* Tell the dest to open the Return path */
+    MIG_CMD_PING,              /* Request a PONG on the RP */
+
+    MIG_CMD_POSTCOPY_ADVISE,       /* Prior to any page transfers, just
+                                      warn we might want to do PC */
+    MIG_CMD_POSTCOPY_LISTEN,       /* Start listening for incoming
+                                      pages as it's running. */
+    MIG_CMD_POSTCOPY_RUN,          /* Start execution */
+
+    MIG_CMD_POSTCOPY_RAM_DISCARD,  /* A list of pages to discard that
+                                      were previously sent during
+                                      precopy but are dirty. */
+    MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
+    MIG_CMD_MAX
+};
+
+#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24)
+
 bool qemu_savevm_state_blocked(Error **errp);
 void qemu_savevm_state_begin(QEMUFile *f,
                              const MigrationParams *params);
 void qemu_savevm_state_header(QEMUFile *f);
-int qemu_savevm_state_iterate(QEMUFile *f);
-void qemu_savevm_state_complete(QEMUFile *f);
-void qemu_savevm_state_cancel(void);
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy);
+void qemu_savevm_state_cleanup(void);
+void qemu_savevm_state_complete_postcopy(QEMUFile *f);
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only);
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable);
+void qemu_savevm_command_send(QEMUFile *f, enum qemu_vm_cmd command,
+                              uint16_t len, uint8_t *data);
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value);
+void qemu_savevm_send_open_return_path(QEMUFile *f);
+int qemu_savevm_send_packaged(QEMUFile *f, const QEMUSizedBuffer *qsb);
+void qemu_savevm_send_postcopy_advise(QEMUFile *f);
+void qemu_savevm_send_postcopy_listen(QEMUFile *f);
+void qemu_savevm_send_postcopy_run(QEMUFile *f);
+
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list);
+
 int qemu_loadvm_state(QEMUFile *f);
 
 typedef enum DisplayType
@@ -132,6 +171,7 @@ extern int boot_menu;
 extern bool boot_strict;
 extern uint8_t *boot_splash_filedata;
 extern size_t boot_splash_filedata_size;
+extern bool enable_mlock;
 extern uint8_t qemu_extra_params_fw[2];
 extern QEMUClockType rtc_clock;
 extern const char *mem_path;
@@ -194,7 +234,7 @@ void device_add_bootindex_property(Object *obj, int32_t *bootindex,
 void restore_boot_order(void *opaque);
 void validate_bootdevices(const char *devices, Error **errp);
 
-/* handler to set the boot_device order for a specific type of QEMUMachine */
+/* handler to set the boot_device order for a specific type of MachineClass */
 typedef void QEMUBootSetHandler(void *opaque, const char *boot_order,
                                 Error **errp);
 void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque);
diff --git a/qemu/include/sysemu/tpm_backend.h b/qemu/include/sysemu/tpm_backend.h
index 0a366be0f..e3ec80020 100644
--- a/qemu/include/sysemu/tpm_backend.h
+++ b/qemu/include/sysemu/tpm_backend.h
@@ -15,7 +15,6 @@
 
 #include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
 #include "qapi-types.h"
 #include "qemu/option.h"
 #include "sysemu/tpm.h"
diff --git a/qemu/include/sysemu/watchdog.h b/qemu/include/sysemu/watchdog.h
index 3e9a97068..72a4da07a 100644
--- a/qemu/include/sysemu/watchdog.h
+++ b/qemu/include/sysemu/watchdog.h
@@ -24,6 +24,15 @@
 
 #include "qemu/queue.h"
 
+/* Possible values for action parameter. */
+#define WDT_RESET        1      /* Hard reset. */
+#define WDT_SHUTDOWN     2      /* Shutdown. */
+#define WDT_POWEROFF     3      /* Quit. */
+#define WDT_PAUSE        4      /* Pause. */
+#define WDT_DEBUG        5      /* Prints a message and continues running. */
+#define WDT_NONE         6      /* Do nothing. */
+#define WDT_NMI          7      /* Inject nmi into the guest. */
+
 struct WatchdogTimerModel {
     QLIST_ENTRY(WatchdogTimerModel) entry;
 
@@ -37,6 +46,7 @@ typedef struct WatchdogTimerModel WatchdogTimerModel;
 /* in hw/watchdog.c */
 int select_watchdog(const char *p);
 int select_watchdog_action(const char *action);
+int get_watchdog_action(void);
 void watchdog_add_model(WatchdogTimerModel *model);
 void watchdog_perform_action(void);
 
diff --git a/qemu/include/sysemu/xen-mapcache.h b/qemu/include/sysemu/xen-mapcache.h
index c59804060..c849489fb 100644
--- a/qemu/include/sysemu/xen-mapcache.h
+++ b/qemu/include/sysemu/xen-mapcache.h
@@ -9,7 +9,6 @@
 #ifndef XEN_MAPCACHE_H
 #define XEN_MAPCACHE_H
 
-#include <stdlib.h>
 
 typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr start_addr,
                                                      ram_addr_t size,
diff --git a/qemu/include/ui/console.h b/qemu/include/ui/console.h
index 047a2b464..d5a88d93e 100644
--- a/qemu/include/ui/console.h
+++ b/qemu/include/ui/console.h
@@ -5,9 +5,7 @@
 #include "qom/object.h"
 #include "qapi/qmp/qdict.h"
 #include "qemu/notify.h"
-#include "qemu/typedefs.h"
 #include "qapi-types.h"
-#include "qapi/error.h"
 
 #ifdef CONFIG_OPENGL
 # include <epoxy/gl.h>
@@ -30,6 +28,21 @@
 #define GUI_REFRESH_INTERVAL_DEFAULT    30
 #define GUI_REFRESH_INTERVAL_IDLE     3000
 
+/* Color number is match to standard vga palette */
+enum qemu_color_names {
+    QEMU_COLOR_BLACK   = 0,
+    QEMU_COLOR_BLUE    = 1,
+    QEMU_COLOR_GREEN   = 2,
+    QEMU_COLOR_CYAN    = 3,
+    QEMU_COLOR_RED     = 4,
+    QEMU_COLOR_MAGENTA = 5,
+    QEMU_COLOR_YELLOW  = 6,
+    QEMU_COLOR_WHITE   = 7
+};
+/* Convert to curses char attributes */
+#define ATTR2CHTYPE(c, fg, bg, bold) \
+    ((bold) << 21 | (bg) << 11 | (fg) << 8 | (c))
+
 typedef void QEMUPutKBDEvent(void *opaque, int keycode);
 typedef void QEMUPutLEDEvent(void *opaque, int ledstate);
 typedef void QEMUPutMouseEvent(void *opaque, int dx, int dy, int dz, int buttons_state);
@@ -157,6 +170,14 @@ void cursor_set_mono(QEMUCursor *c,
 void cursor_get_mono_image(QEMUCursor *c, int foreground, uint8_t *mask);
 void cursor_get_mono_mask(QEMUCursor *c, int transparent, uint8_t *mask);
 
+typedef void *QEMUGLContext;
+typedef struct QEMUGLParams QEMUGLParams;
+
+struct QEMUGLParams {
+    int major_ver;
+    int minor_ver;
+};
+
 typedef struct DisplayChangeListenerOps {
     const char *dpy_name;
 
@@ -183,6 +204,21 @@ typedef struct DisplayChangeListenerOps {
                           int x, int y, int on);
     void (*dpy_cursor_define)(DisplayChangeListener *dcl,
                               QEMUCursor *cursor);
+
+    QEMUGLContext (*dpy_gl_ctx_create)(DisplayChangeListener *dcl,
+                                       QEMUGLParams *params);
+    void (*dpy_gl_ctx_destroy)(DisplayChangeListener *dcl,
+                               QEMUGLContext ctx);
+    int (*dpy_gl_ctx_make_current)(DisplayChangeListener *dcl,
+                                   QEMUGLContext ctx);
+    QEMUGLContext (*dpy_gl_ctx_get_current)(DisplayChangeListener *dcl);
+
+    void (*dpy_gl_scanout)(DisplayChangeListener *dcl,
+                           uint32_t backing_id, bool backing_y_0_top,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+    void (*dpy_gl_update)(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
 } DisplayChangeListenerOps;
 
 struct DisplayChangeListener {
@@ -198,6 +234,7 @@ DisplayState *init_displaystate(void);
 DisplaySurface *qemu_create_displaysurface_from(int width, int height,
                                                 pixman_format_code_t format,
                                                 int linesize, uint8_t *data);
+DisplaySurface *qemu_create_displaysurface_pixman(pixman_image_t *image);
 DisplaySurface *qemu_create_displaysurface_guestmem(int width, int height,
                                                     pixman_format_code_t format,
                                                     int linesize,
@@ -244,6 +281,20 @@ bool dpy_cursor_define_supported(QemuConsole *con);
 bool dpy_gfx_check_format(QemuConsole *con,
                           pixman_format_code_t format);
 
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                QEMUGLParams *params);
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx);
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx);
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con);
+
+bool console_has_gl(QemuConsole *con);
+
 static inline int surface_stride(DisplaySurface *s)
 {
     return pixman_image_get_stride(s->image);
@@ -284,13 +335,23 @@ static inline pixman_format_code_t surface_format(DisplaySurface *s)
 #ifdef CONFIG_CURSES
 #include <curses.h>
 typedef chtype console_ch_t;
+extern chtype vga_to_curses[];
 #else
 typedef unsigned long console_ch_t;
 #endif
 static inline void console_write_ch(console_ch_t *dest, uint32_t ch)
 {
-    if (!(ch & 0xff))
+    uint8_t c = ch;
+#ifdef CONFIG_CURSES
+    if (vga_to_curses[c]) {
+        ch &= ~(console_ch_t)0xff;
+        ch |= vga_to_curses[c];
+    }
+#else
+    if (c == '\0') {
         ch |= ' ';
+    }
+#endif
     *dest = ch;
 }
 
@@ -300,6 +361,7 @@ typedef struct GraphicHwOps {
     void (*text_update)(void *opaque, console_ch_t *text);
     void (*update_interval)(void *opaque, uint64_t interval);
     int (*ui_info)(void *opaque, uint32_t head, QemuUIInfo *info);
+    void (*gl_block)(void *opaque, bool block);
 } GraphicHwOps;
 
 QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
@@ -312,9 +374,12 @@ void graphic_console_set_hwops(QemuConsole *con,
 void graphic_hw_update(QemuConsole *con);
 void graphic_hw_invalidate(QemuConsole *con);
 void graphic_hw_text_update(QemuConsole *con, console_ch_t *chardata);
+void graphic_hw_gl_block(QemuConsole *con, bool block);
 
 QemuConsole *qemu_console_lookup_by_index(unsigned int index);
 QemuConsole *qemu_console_lookup_by_device(DeviceState *dev, uint32_t head);
+QemuConsole *qemu_console_lookup_by_device_name(const char *device_id,
+                                                uint32_t head, Error **errp);
 bool qemu_console_is_visible(QemuConsole *con);
 bool qemu_console_is_graphic(QemuConsole *con);
 bool qemu_console_is_fixedsize(QemuConsole *con);
@@ -386,7 +451,7 @@ static inline int vnc_display_pw_expire(const char *id, time_t expires)
 void curses_display_init(DisplayState *ds, int full_screen);
 
 /* input.c */
-int index_from_key(const char *key);
+int index_from_key(const char *key, size_t key_length);
 
 /* gtk.c */
 void early_gtk_display_init(int opengl);
diff --git a/qemu/include/ui/egl-context.h b/qemu/include/ui/egl-context.h
new file mode 100644
index 000000000..f004ce11a
--- /dev/null
+++ b/qemu/include/ui/egl-context.h
@@ -0,0 +1,14 @@
+#ifndef EGL_CONTEXT_H
+#define EGL_CONTEXT_H
+
+#include "ui/console.h"
+#include "ui/egl-helpers.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params);
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx);
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx);
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl);
+
+#endif /* EGL_CONTEXT_H */
diff --git a/qemu/include/ui/egl-helpers.h b/qemu/include/ui/egl-helpers.h
index 5ad5dc308..03fcf4bba 100644
--- a/qemu/include/ui/egl-helpers.h
+++ b/qemu/include/ui/egl-helpers.h
@@ -3,14 +3,26 @@
 
 #include <epoxy/gl.h>
 #include <epoxy/egl.h>
+#include <gbm.h>
 
 extern EGLDisplay *qemu_egl_display;
 extern EGLConfig qemu_egl_config;
 
+#ifdef CONFIG_OPENGL_DMABUF
+
+extern int qemu_egl_rn_fd;
+extern struct gbm_device *qemu_egl_rn_gbm_dev;
+extern EGLContext qemu_egl_rn_ctx;
+
+int qemu_egl_rendernode_open(void);
+int egl_rendernode_init(void);
+int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc);
+
+#endif
+
 EGLSurface qemu_egl_init_surface_x11(EGLContext ectx, Window win);
 
 int qemu_egl_init_dpy(EGLNativeDisplayType dpy, bool gles, bool debug);
 EGLContext qemu_egl_init_ctx(void);
-bool qemu_egl_has_ext(const char *haystack, const char *needle);
 
 #endif /* EGL_HELPERS_H */
diff --git a/qemu/include/ui/gtk.h b/qemu/include/ui/gtk.h
index ee6dffd30..2bf60f3ec 100644
--- a/qemu/include/ui/gtk.h
+++ b/qemu/include/ui/gtk.h
@@ -1,10 +1,6 @@
 #ifndef UI_GTK_H
 #define UI_GTK_H
 
-#ifdef _WIN32
-# define _WIN32_WINNT 0x0601 /* needed to get definition of MAPVK_VK_TO_VSC */
-#endif
-
 #ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
 /* Work around an -Wstrict-prototypes warning in GTK headers */
 #pragma GCC diagnostic push
@@ -24,6 +20,7 @@
 
 #if defined(CONFIG_OPENGL)
 #include "ui/egl-helpers.h"
+#include "ui/egl-context.h"
 #endif
 
 /* Compatibility define to let us build on both Gtk2 and Gtk3 */
@@ -50,6 +47,11 @@ typedef struct VirtualGfxConsole {
     EGLContext ectx;
     EGLSurface esurface;
     int glupdates;
+    int x, y, w, h;
+    GLuint tex_id;
+    GLuint fbo_id;
+    bool y0_top;
+    bool scanout_mode;
 #endif
 } VirtualGfxConsole;
 
@@ -59,6 +61,7 @@ typedef struct VirtualVteConsole {
     GtkWidget *scrollbar;
     GtkWidget *terminal;
     CharDriverState *chr;
+    bool echo;
 } VirtualVteConsole;
 #endif
 
@@ -94,6 +97,39 @@ void gd_egl_update(DisplayChangeListener *dcl,
 void gd_egl_refresh(DisplayChangeListener *dcl);
 void gd_egl_switch(DisplayChangeListener *dcl,
                    DisplaySurface *surface);
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params);
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h);
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
 void gtk_egl_init(void);
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx);
+
+/* ui/gtk-gl-area.c */
+void gd_gl_area_init(VirtualConsole *vc);
+void gd_gl_area_draw(VirtualConsole *vc);
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                       int x, int y, int w, int h);
+void gd_gl_area_refresh(DisplayChangeListener *dcl);
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface);
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params);
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl,
+                                QEMUGLContext ctx);
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h);
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                              uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void gtk_gl_area_init(void);
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl);
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx);
 
 #endif /* UI_GTK_H */
diff --git a/qemu/include/ui/input.h b/qemu/include/ui/input.h
index 5d5ac0066..102d8a334 100644
--- a/qemu/include/ui/input.h
+++ b/qemu/include/ui/input.h
@@ -33,7 +33,9 @@ void qemu_input_handler_bind(QemuInputHandlerState *s,
                              const char *device_id, int head,
                              Error **errp);
 void qemu_input_event_send(QemuConsole *src, InputEvent *evt);
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt);
 void qemu_input_event_sync(void);
+void qemu_input_event_sync_impl(void);
 
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down);
 void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down);
@@ -63,4 +65,6 @@ void qemu_input_check_mode_change(void);
 void qemu_add_mouse_mode_change_notifier(Notifier *notify);
 void qemu_remove_mouse_mode_change_notifier(Notifier *notify);
 
+int input_linux_init(void *opaque, QemuOpts *opts, Error **errp);
+
 #endif /* INPUT_H */
diff --git a/qemu/include/ui/qemu-pixman.h b/qemu/include/ui/qemu-pixman.h
index e34c4effc..4a67e0123 100644
--- a/qemu/include/ui/qemu-pixman.h
+++ b/qemu/include/ui/qemu-pixman.h
@@ -16,8 +16,6 @@
 #pragma GCC diagnostic pop
 #endif
 
-#include "qemu/typedefs.h"
-
 /*
  * pixman image formats are defined to be native endian,
  * that means host byte order on qemu.  So we go define
diff --git a/qemu/include/ui/qemu-spice.h b/qemu/include/ui/qemu-spice.h
index 0dff4229f..aa2436355 100644
--- a/qemu/include/ui/qemu-spice.h
+++ b/qemu/include/ui/qemu-spice.h
@@ -18,12 +18,11 @@
 #ifndef QEMU_SPICE_H
 #define QEMU_SPICE_H
 
-#include "config-host.h"
+#include "qapi/error.h"
 
 #ifdef CONFIG_SPICE
 
 #include <spice.h>
-
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 
@@ -43,9 +42,7 @@ int qemu_spice_set_pw_expire(time_t expires);
 int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
                             const char *subject);
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type);
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name);
 void qemu_spice_register_ports(void);
 #else
 static inline CharDriverState *qemu_chr_open_spice_port(const char *name)
diff --git a/qemu/include/ui/sdl2.h b/qemu/include/ui/sdl2.h
index 2fdad8f30..3f0b57bb1 100644
--- a/qemu/include/ui/sdl2.h
+++ b/qemu/include/ui/sdl2.h
@@ -15,12 +15,19 @@ struct sdl2_console {
     SDL_Renderer *real_renderer;
     int idx;
     int last_vm_running; /* per console for caption reasons */
-    int x, y;
+    int x, y, w, h;
     int hidden;
     int opengl;
     int updates;
+    int idle_counter;
     SDL_GLContext winctx;
+#ifdef CONFIG_OPENGL
     ConsoleGLState *gls;
+    GLuint tex_id;
+    GLuint fbo_id;
+    bool y0_top;
+    bool scanout_mode;
+#endif
 };
 
 void sdl2_window_create(struct sdl2_console *scon);
@@ -48,4 +55,18 @@ void sdl2_gl_switch(DisplayChangeListener *dcl,
 void sdl2_gl_refresh(DisplayChangeListener *dcl);
 void sdl2_gl_redraw(struct sdl2_console *scon);
 
+QEMUGLContext sdl2_gl_create_context(DisplayChangeListener *dcl,
+                                     QEMUGLParams *params);
+void sdl2_gl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx);
+int sdl2_gl_make_context_current(DisplayChangeListener *dcl,
+                                 QEMUGLContext ctx);
+QEMUGLContext sdl2_gl_get_current_context(DisplayChangeListener *dcl);
+
+void sdl2_gl_scanout(DisplayChangeListener *dcl,
+                     uint32_t backing_id, bool backing_y_0_top,
+                     uint32_t x, uint32_t y,
+                     uint32_t w, uint32_t h);
+void sdl2_gl_scanout_flush(DisplayChangeListener *dcl,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
 #endif /* SDL2_H */
diff --git a/qemu/include/ui/shader.h b/qemu/include/ui/shader.h
index 8509596ac..f7d86188b 100644
--- a/qemu/include/ui/shader.h
+++ b/qemu/include/ui/shader.h
@@ -3,7 +3,9 @@
 
 #include <epoxy/gl.h>
 
-void qemu_gl_run_texture_blit(GLint texture_blit_prog);
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog);
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao);
 
 GLuint qemu_gl_create_compile_shader(GLenum type, const GLchar *src);
 GLuint qemu_gl_create_link_program(GLuint vert, GLuint frag);
diff --git a/qemu/include/ui/spice-display.h b/qemu/include/ui/spice-display.h
index b25328a6b..30ccfe3da 100644
--- a/qemu/include/ui/spice-display.h
+++ b/qemu/include/ui/spice-display.h
@@ -24,6 +24,14 @@
 #include "ui/console.h"
 #include "sysemu/sysemu.h"
 
+#if defined(CONFIG_OPENGL_DMABUF)
+# if SPICE_SERVER_VERSION >= 0x000d01 /* release 0.13.1 */
+#  define HAVE_SPICE_GL 1
+#  include "ui/egl-helpers.h"
+#  include "ui/egl-context.h"
+# endif
+#endif
+
 #define NUM_MEMSLOTS 8
 #define MEMSLOT_GENERATION_BITS 8
 #define MEMSLOT_SLOT_BITS 8
@@ -50,6 +58,7 @@ enum {
     QXL_COOKIE_TYPE_IO,
     QXL_COOKIE_TYPE_RENDER_UPDATE_AREA,
     QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
+    QXL_COOKIE_TYPE_GL_DRAW_DONE,
 };
 
 typedef struct QXLCookie {
@@ -104,6 +113,13 @@ struct SimpleSpiceDisplay {
     QEMUCursor *cursor;
     int mouse_x, mouse_y;
     QEMUBH *cursor_bh;
+
+#ifdef HAVE_SPICE_GL
+    /* opengl rendering */
+    QEMUBH *gl_unblock_bh;
+    QEMUTimer *gl_unblock_timer;
+    int dmabuf_fd;
+#endif
 };
 
 struct SimpleSpiceUpdate {
diff --git a/qemu/io/Makefile.objs b/qemu/io/Makefile.objs
new file mode 100644
index 000000000..9d8337d89
--- /dev/null
+++ b/qemu/io/Makefile.objs
@@ -0,0 +1,10 @@
+io-obj-y = channel.o
+io-obj-y += channel-buffer.o
+io-obj-y += channel-command.o
+io-obj-y += channel-file.o
+io-obj-y += channel-socket.o
+io-obj-y += channel-tls.o
+io-obj-y += channel-watch.o
+io-obj-y += channel-websock.o
+io-obj-y += channel-util.o
+io-obj-y += task.o
diff --git a/qemu/io/channel-buffer.c b/qemu/io/channel-buffer.c
new file mode 100644
index 000000000..3e5117bf2
--- /dev/null
+++ b/qemu/io/channel-buffer.c
@@ -0,0 +1,249 @@
+/*
+ * QEMU I/O channels memory buffer driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-buffer.h"
+#include "io/channel-watch.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+QIOChannelBuffer *
+qio_channel_buffer_new(size_t capacity)
+{
+    QIOChannelBuffer *ioc;
+
+    ioc = QIO_CHANNEL_BUFFER(object_new(TYPE_QIO_CHANNEL_BUFFER));
+
+    if (capacity) {
+        ioc->data = g_new0(uint8_t, capacity);
+        ioc->capacity = capacity;
+    }
+
+    return ioc;
+}
+
+
+static void qio_channel_buffer_finalize(Object *obj)
+{
+    QIOChannelBuffer *ioc = QIO_CHANNEL_BUFFER(obj);
+    g_free(ioc->data);
+    ioc->capacity = ioc->usage = ioc->offset = 0;
+}
+
+
+static ssize_t qio_channel_buffer_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    ssize_t ret = 0;
+    size_t i;
+
+    for (i = 0; i < niov; i++) {
+        size_t want = iov[i].iov_len;
+        if (bioc->offset >= bioc->usage) {
+            break;
+        }
+        if ((bioc->offset + want) > bioc->usage)  {
+            want = bioc->usage - bioc->offset;
+        }
+        memcpy(iov[i].iov_base, bioc->data + bioc->offset, want);
+        ret += want;
+        bioc->offset += want;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_buffer_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    ssize_t ret = 0;
+    size_t i;
+    size_t towrite = 0;
+
+    for (i = 0; i < niov; i++) {
+        towrite += iov[i].iov_len;
+    }
+
+    if ((bioc->offset + towrite) > bioc->capacity) {
+        bioc->capacity = bioc->offset + towrite;
+        bioc->data = g_realloc(bioc->data, bioc->capacity);
+    }
+
+    if (bioc->offset > bioc->usage) {
+        memset(bioc->data, 0, bioc->offset - bioc->usage);
+        bioc->usage = bioc->offset;
+    }
+
+    for (i = 0; i < niov; i++) {
+        memcpy(bioc->data + bioc->usage,
+               iov[i].iov_base,
+               iov[i].iov_len);
+        bioc->usage += iov[i].iov_len;
+        bioc->offset += iov[i].iov_len;
+        ret += iov[i].iov_len;
+    }
+
+    return ret;
+}
+
+static int qio_channel_buffer_set_blocking(QIOChannel *ioc G_GNUC_UNUSED,
+                                           bool enabled G_GNUC_UNUSED,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    return 0;
+}
+
+
+static off_t qio_channel_buffer_seek(QIOChannel *ioc,
+                                     off_t offset,
+                                     int whence,
+                                     Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+
+    bioc->offset = offset;
+
+    return offset;
+}
+
+
+static int qio_channel_buffer_close(QIOChannel *ioc,
+                                    Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+
+    g_free(bioc->data);
+    bioc->capacity = bioc->usage = bioc->offset = 0;
+
+    return 0;
+}
+
+
+typedef struct QIOChannelBufferSource QIOChannelBufferSource;
+struct QIOChannelBufferSource {
+    GSource parent;
+    QIOChannelBuffer *bioc;
+    GIOCondition condition;
+};
+
+static gboolean
+qio_channel_buffer_source_prepare(GSource *source,
+                                  gint *timeout)
+{
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    *timeout = -1;
+
+    return (G_IO_IN | G_IO_OUT) & bsource->condition;
+}
+
+static gboolean
+qio_channel_buffer_source_check(GSource *source)
+{
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    return (G_IO_IN | G_IO_OUT) & bsource->condition;
+}
+
+static gboolean
+qio_channel_buffer_source_dispatch(GSource *source,
+                                   GSourceFunc callback,
+                                   gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    return (*func)(QIO_CHANNEL(bsource->bioc),
+                   ((G_IO_IN | G_IO_OUT) & bsource->condition),
+                   user_data);
+}
+
+static void
+qio_channel_buffer_source_finalize(GSource *source)
+{
+    QIOChannelBufferSource *ssource = (QIOChannelBufferSource *)source;
+
+    object_unref(OBJECT(ssource->bioc));
+}
+
+GSourceFuncs qio_channel_buffer_source_funcs = {
+    qio_channel_buffer_source_prepare,
+    qio_channel_buffer_source_check,
+    qio_channel_buffer_source_dispatch,
+    qio_channel_buffer_source_finalize
+};
+
+static GSource *qio_channel_buffer_create_watch(QIOChannel *ioc,
+                                                GIOCondition condition)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    QIOChannelBufferSource *ssource;
+    GSource *source;
+
+    source = g_source_new(&qio_channel_buffer_source_funcs,
+                          sizeof(QIOChannelBufferSource));
+    ssource = (QIOChannelBufferSource *)source;
+
+    ssource->bioc = bioc;
+    object_ref(OBJECT(bioc));
+
+    ssource->condition = condition;
+
+    return source;
+}
+
+
+static void qio_channel_buffer_class_init(ObjectClass *klass,
+                                          void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_buffer_writev;
+    ioc_klass->io_readv = qio_channel_buffer_readv;
+    ioc_klass->io_set_blocking = qio_channel_buffer_set_blocking;
+    ioc_klass->io_seek = qio_channel_buffer_seek;
+    ioc_klass->io_close = qio_channel_buffer_close;
+    ioc_klass->io_create_watch = qio_channel_buffer_create_watch;
+}
+
+static const TypeInfo qio_channel_buffer_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_BUFFER,
+    .instance_size = sizeof(QIOChannelBuffer),
+    .instance_finalize = qio_channel_buffer_finalize,
+    .class_init = qio_channel_buffer_class_init,
+};
+
+static void qio_channel_buffer_register_types(void)
+{
+    type_register_static(&qio_channel_buffer_info);
+}
+
+type_init(qio_channel_buffer_register_types);
diff --git a/qemu/io/channel-command.c b/qemu/io/channel-command.c
new file mode 100644
index 000000000..ad25313be
--- /dev/null
+++ b/qemu/io/channel-command.c
@@ -0,0 +1,368 @@
+/*
+ * QEMU I/O channels external command driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-command.h"
+#include "io/channel-watch.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+
+QIOChannelCommand *
+qio_channel_command_new_pid(int writefd,
+                            int readfd,
+                            pid_t pid)
+{
+    QIOChannelCommand *ioc;
+
+    ioc = QIO_CHANNEL_COMMAND(object_new(TYPE_QIO_CHANNEL_COMMAND));
+
+    ioc->readfd = readfd;
+    ioc->writefd = writefd;
+    ioc->pid = pid;
+
+    trace_qio_channel_command_new_pid(ioc, writefd, readfd, pid);
+    return ioc;
+}
+
+
+#ifndef WIN32
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp)
+{
+    pid_t pid = -1;
+    int stdinfd[2] = { -1, -1 };
+    int stdoutfd[2] = { -1, -1 };
+    int devnull = -1;
+    bool stdinnull = false, stdoutnull = false;
+    QIOChannelCommand *ioc;
+
+    flags = flags & O_ACCMODE;
+
+    if (flags == O_RDONLY) {
+        stdinnull = true;
+    }
+    if (flags == O_WRONLY) {
+        stdoutnull = true;
+    }
+
+    if (stdinnull || stdoutnull) {
+        devnull = open("/dev/null", O_RDWR);
+        if (devnull < 0) {
+            error_setg_errno(errp, errno,
+                             "Unable to open /dev/null");
+            goto error;
+        }
+    }
+
+    if ((!stdinnull && pipe(stdinfd) < 0) ||
+        (!stdoutnull && pipe(stdoutfd) < 0)) {
+        error_setg_errno(errp, errno,
+                         "Unable to open pipe");
+        goto error;
+    }
+
+    pid = qemu_fork(errp);
+    if (pid < 0) {
+        goto error;
+    }
+
+    if (pid == 0) { /* child */
+        dup2(stdinnull ? devnull : stdinfd[0], STDIN_FILENO);
+        dup2(stdoutnull ? devnull : stdoutfd[1], STDOUT_FILENO);
+        /* Leave stderr connected to qemu's stderr */
+
+        if (!stdinnull) {
+            close(stdinfd[0]);
+            close(stdinfd[1]);
+        }
+        if (!stdoutnull) {
+            close(stdoutfd[0]);
+            close(stdoutfd[1]);
+        }
+        if (devnull != -1) {
+            close(devnull);
+        }
+
+        execv(argv[0], (char * const *)argv);
+        _exit(1);
+    }
+
+    if (!stdinnull) {
+        close(stdinfd[0]);
+    }
+    if (!stdoutnull) {
+        close(stdoutfd[1]);
+    }
+
+    ioc = qio_channel_command_new_pid(stdinnull ? devnull : stdinfd[1],
+                                      stdoutnull ? devnull : stdoutfd[0],
+                                      pid);
+    trace_qio_channel_command_new_spawn(ioc, argv[0], flags);
+    return ioc;
+
+ error:
+    if (devnull != -1) {
+        close(devnull);
+    }
+    if (stdinfd[0] != -1) {
+        close(stdinfd[0]);
+    }
+    if (stdinfd[1] != -1) {
+        close(stdinfd[1]);
+    }
+    if (stdoutfd[0] != -1) {
+        close(stdoutfd[0]);
+    }
+    if (stdoutfd[1] != -1) {
+        close(stdoutfd[1]);
+    }
+    return NULL;
+}
+
+#else /* WIN32 */
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp)
+{
+    error_setg_errno(errp, ENOSYS,
+                     "Command spawn not supported on this platform");
+    return NULL;
+}
+#endif /* WIN32 */
+
+#ifndef WIN32
+static int qio_channel_command_abort(QIOChannelCommand *ioc,
+                                     Error **errp)
+{
+    pid_t ret;
+    int status;
+    int step = 0;
+
+    /* See if intermediate process has exited; if not, try a nice
+     * SIGTERM followed by a more severe SIGKILL.
+     */
+ rewait:
+    trace_qio_channel_command_abort(ioc, ioc->pid);
+    ret = waitpid(ioc->pid, &status, WNOHANG);
+    trace_qio_channel_command_wait(ioc, ioc->pid, ret, status);
+    if (ret == (pid_t)-1) {
+        if (errno == EINTR) {
+            goto rewait;
+        } else {
+            error_setg_errno(errp, errno,
+                             "Cannot wait on pid %llu",
+                             (unsigned long long)ioc->pid);
+            return -1;
+        }
+    } else if (ret == 0) {
+        if (step == 0) {
+            kill(ioc->pid, SIGTERM);
+        } else if (step == 1) {
+            kill(ioc->pid, SIGKILL);
+        } else {
+            error_setg(errp,
+                       "Process %llu refused to die",
+                       (unsigned long long)ioc->pid);
+            return -1;
+        }
+        step++;
+        usleep(10 * 1000);
+        goto rewait;
+    }
+
+    return 0;
+}
+#endif /* ! WIN32 */
+
+
+static void qio_channel_command_init(Object *obj)
+{
+    QIOChannelCommand *ioc = QIO_CHANNEL_COMMAND(obj);
+    ioc->readfd = -1;
+    ioc->writefd = -1;
+    ioc->pid = -1;
+}
+
+static void qio_channel_command_finalize(Object *obj)
+{
+    QIOChannelCommand *ioc = QIO_CHANNEL_COMMAND(obj);
+    if (ioc->readfd != -1) {
+        close(ioc->readfd);
+    }
+    if (ioc->writefd != -1 &&
+        ioc->writefd != ioc->readfd) {
+        close(ioc->writefd);
+    }
+    ioc->writefd = ioc->readfd = -1;
+    if (ioc->pid > 0) {
+#ifndef WIN32
+        qio_channel_command_abort(ioc, NULL);
+#endif
+    }
+}
+
+
+static ssize_t qio_channel_command_readv(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int **fds,
+                                         size_t *nfds,
+                                         Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = readv(cioc->readfd, iov, niov);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from command");
+        return -1;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_command_writev(QIOChannel *ioc,
+                                          const struct iovec *iov,
+                                          size_t niov,
+                                          int *fds,
+                                          size_t nfds,
+                                          Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = writev(cioc->writefd, iov, niov);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to write to command");
+        return -1;
+    }
+    return ret;
+}
+
+static int qio_channel_command_set_blocking(QIOChannel *ioc,
+                                            bool enabled,
+                                            Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+
+    if (enabled) {
+        qemu_set_block(cioc->writefd);
+        qemu_set_block(cioc->readfd);
+    } else {
+        qemu_set_nonblock(cioc->writefd);
+        qemu_set_nonblock(cioc->readfd);
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_command_close(QIOChannel *ioc,
+                                     Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    int rv = 0;
+
+    /* We close FDs before killing, because that
+     * gives a better chance of clean shutdown
+     */
+    if (cioc->readfd != -1 &&
+        close(cioc->readfd) < 0) {
+        rv = -1;
+    }
+    if (cioc->writefd != -1 &&
+        cioc->writefd != cioc->readfd &&
+        close(cioc->writefd) < 0) {
+        rv = -1;
+    }
+    cioc->writefd = cioc->readfd = -1;
+#ifndef WIN32
+    if (qio_channel_command_abort(cioc, errp) < 0) {
+        return -1;
+    }
+#endif
+    if (rv < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to close command");
+    }
+    return rv;
+}
+
+
+static GSource *qio_channel_command_create_watch(QIOChannel *ioc,
+                                                 GIOCondition condition)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    return qio_channel_create_fd_pair_watch(ioc,
+                                            cioc->readfd,
+                                            cioc->writefd,
+                                            condition);
+}
+
+
+static void qio_channel_command_class_init(ObjectClass *klass,
+                                           void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_command_writev;
+    ioc_klass->io_readv = qio_channel_command_readv;
+    ioc_klass->io_set_blocking = qio_channel_command_set_blocking;
+    ioc_klass->io_close = qio_channel_command_close;
+    ioc_klass->io_create_watch = qio_channel_command_create_watch;
+}
+
+static const TypeInfo qio_channel_command_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_COMMAND,
+    .instance_size = sizeof(QIOChannelCommand),
+    .instance_init = qio_channel_command_init,
+    .instance_finalize = qio_channel_command_finalize,
+    .class_init = qio_channel_command_class_init,
+};
+
+static void qio_channel_command_register_types(void)
+{
+    type_register_static(&qio_channel_command_info);
+}
+
+type_init(qio_channel_command_register_types);
diff --git a/qemu/io/channel-file.c b/qemu/io/channel-file.c
new file mode 100644
index 000000000..e1da2435e
--- /dev/null
+++ b/qemu/io/channel-file.c
@@ -0,0 +1,225 @@
+/*
+ * QEMU I/O channels files driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-file.h"
+#include "io/channel-watch.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+QIOChannelFile *
+qio_channel_file_new_fd(int fd)
+{
+    QIOChannelFile *ioc;
+
+    ioc = QIO_CHANNEL_FILE(object_new(TYPE_QIO_CHANNEL_FILE));
+
+    ioc->fd = fd;
+
+    trace_qio_channel_file_new_fd(ioc, fd);
+
+    return ioc;
+}
+
+
+QIOChannelFile *
+qio_channel_file_new_path(const char *path,
+                          int flags,
+                          mode_t mode,
+                          Error **errp)
+{
+    QIOChannelFile *ioc;
+
+    ioc = QIO_CHANNEL_FILE(object_new(TYPE_QIO_CHANNEL_FILE));
+
+    if (flags & O_WRONLY) {
+        ioc->fd = open(path, flags, mode);
+    } else {
+        ioc->fd = open(path, flags);
+    }
+    if (ioc->fd < 0) {
+        object_unref(OBJECT(ioc));
+        error_setg_errno(errp, errno,
+                         "Unable to open %s", path);
+        return NULL;
+    }
+
+    trace_qio_channel_file_new_path(ioc, path, flags, mode, ioc->fd);
+
+    return ioc;
+}
+
+
+static void qio_channel_file_init(Object *obj)
+{
+    QIOChannelFile *ioc = QIO_CHANNEL_FILE(obj);
+    ioc->fd = -1;
+}
+
+static void qio_channel_file_finalize(Object *obj)
+{
+    QIOChannelFile *ioc = QIO_CHANNEL_FILE(obj);
+    if (ioc->fd != -1) {
+        close(ioc->fd);
+        ioc->fd = -1;
+    }
+}
+
+
+static ssize_t qio_channel_file_readv(QIOChannel *ioc,
+                                      const struct iovec *iov,
+                                      size_t niov,
+                                      int **fds,
+                                      size_t *nfds,
+                                      Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = readv(fioc->fd, iov, niov);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from file");
+        return -1;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_file_writev(QIOChannel *ioc,
+                                       const struct iovec *iov,
+                                       size_t niov,
+                                       int *fds,
+                                       size_t nfds,
+                                       Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = writev(fioc->fd, iov, niov);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno,
+                         "Unable to write to file");
+        return -1;
+    }
+    return ret;
+}
+
+static int qio_channel_file_set_blocking(QIOChannel *ioc,
+                                         bool enabled,
+                                         Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+
+    if (enabled) {
+        qemu_set_block(fioc->fd);
+    } else {
+        qemu_set_nonblock(fioc->fd);
+    }
+    return 0;
+}
+
+
+static off_t qio_channel_file_seek(QIOChannel *ioc,
+                                   off_t offset,
+                                   int whence,
+                                   Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    off_t ret;
+
+    ret = lseek(fioc->fd, offset, whence);
+    if (ret == (off_t)-1) {
+        error_setg_errno(errp, errno,
+                         "Unable to seek to offset %lld whence %d in file",
+                         (long long int)offset, whence);
+        return -1;
+    }
+    return ret;
+}
+
+
+static int qio_channel_file_close(QIOChannel *ioc,
+                                  Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+
+    if (close(fioc->fd) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to close file");
+        return -1;
+    }
+    return 0;
+}
+
+
+static GSource *qio_channel_file_create_watch(QIOChannel *ioc,
+                                              GIOCondition condition)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    return qio_channel_create_fd_watch(ioc,
+                                       fioc->fd,
+                                       condition);
+}
+
+static void qio_channel_file_class_init(ObjectClass *klass,
+                                        void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_file_writev;
+    ioc_klass->io_readv = qio_channel_file_readv;
+    ioc_klass->io_set_blocking = qio_channel_file_set_blocking;
+    ioc_klass->io_seek = qio_channel_file_seek;
+    ioc_klass->io_close = qio_channel_file_close;
+    ioc_klass->io_create_watch = qio_channel_file_create_watch;
+}
+
+static const TypeInfo qio_channel_file_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_FILE,
+    .instance_size = sizeof(QIOChannelFile),
+    .instance_init = qio_channel_file_init,
+    .instance_finalize = qio_channel_file_finalize,
+    .class_init = qio_channel_file_class_init,
+};
+
+static void qio_channel_file_register_types(void)
+{
+    type_register_static(&qio_channel_file_info);
+}
+
+type_init(qio_channel_file_register_types);
diff --git a/qemu/io/channel-socket.c b/qemu/io/channel-socket.c
new file mode 100644
index 000000000..ca8bc20b1
--- /dev/null
+++ b/qemu/io/channel-socket.c
@@ -0,0 +1,772 @@
+/*
+ * QEMU I/O channels sockets driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "io/channel-socket.h"
+#include "io/channel-watch.h"
+#include "trace.h"
+
+#define SOCKET_MAX_FDS 16
+
+SocketAddress *
+qio_channel_socket_get_local_address(QIOChannelSocket *ioc,
+                                     Error **errp)
+{
+    return socket_sockaddr_to_address(&ioc->localAddr,
+                                      ioc->localAddrLen,
+                                      errp);
+}
+
+SocketAddress *
+qio_channel_socket_get_remote_address(QIOChannelSocket *ioc,
+                                      Error **errp)
+{
+    return socket_sockaddr_to_address(&ioc->remoteAddr,
+                                      ioc->remoteAddrLen,
+                                      errp);
+}
+
+QIOChannelSocket *
+qio_channel_socket_new(void)
+{
+    QIOChannelSocket *sioc;
+    QIOChannel *ioc;
+
+    sioc = QIO_CHANNEL_SOCKET(object_new(TYPE_QIO_CHANNEL_SOCKET));
+    sioc->fd = -1;
+
+    ioc = QIO_CHANNEL(sioc);
+    ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+
+#ifdef WIN32
+    ioc->event = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+
+    trace_qio_channel_socket_new(sioc);
+
+    return sioc;
+}
+
+
+static int
+qio_channel_socket_set_fd(QIOChannelSocket *sioc,
+                          int fd,
+                          Error **errp)
+{
+    if (sioc->fd != -1) {
+        error_setg(errp, "Socket is already open");
+        return -1;
+    }
+
+    sioc->fd = fd;
+    sioc->remoteAddrLen = sizeof(sioc->remoteAddr);
+    sioc->localAddrLen = sizeof(sioc->localAddr);
+
+
+    if (getpeername(fd, (struct sockaddr *)&sioc->remoteAddr,
+                    &sioc->remoteAddrLen) < 0) {
+        if (errno == ENOTCONN) {
+            memset(&sioc->remoteAddr, 0, sizeof(sioc->remoteAddr));
+            sioc->remoteAddrLen = sizeof(sioc->remoteAddr);
+        } else {
+            error_setg_errno(errp, errno,
+                             "Unable to query remote socket address");
+            goto error;
+        }
+    }
+
+    if (getsockname(fd, (struct sockaddr *)&sioc->localAddr,
+                    &sioc->localAddrLen) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to query local socket address");
+        goto error;
+    }
+
+#ifndef WIN32
+    if (sioc->localAddr.ss_family == AF_UNIX) {
+        QIOChannel *ioc = QIO_CHANNEL(sioc);
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_FD_PASS);
+    }
+#endif /* WIN32 */
+
+    return 0;
+
+ error:
+    sioc->fd = -1; /* Let the caller close FD on failure */
+    return -1;
+}
+
+QIOChannelSocket *
+qio_channel_socket_new_fd(int fd,
+                          Error **errp)
+{
+    QIOChannelSocket *ioc;
+
+    ioc = qio_channel_socket_new();
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        object_unref(OBJECT(ioc));
+        return NULL;
+    }
+
+    trace_qio_channel_socket_new_fd(ioc, fd);
+
+    return ioc;
+}
+
+
+int qio_channel_socket_connect_sync(QIOChannelSocket *ioc,
+                                    SocketAddress *addr,
+                                    Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_connect_sync(ioc, addr);
+    fd = socket_connect(addr, errp, NULL, NULL);
+    if (fd < 0) {
+        trace_qio_channel_socket_connect_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_connect_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_socket_connect_worker(QIOTask *task,
+                                             Error **errp,
+                                             gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    SocketAddress *addr = opaque;
+    int ret;
+
+    ret = qio_channel_socket_connect_sync(ioc,
+                                          addr,
+                                          errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_connect_async(QIOChannelSocket *ioc,
+                                      SocketAddress *addr,
+                                      QIOTaskFunc callback,
+                                      gpointer opaque,
+                                      GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    SocketAddress *addrCopy;
+
+    qapi_copy_SocketAddress(&addrCopy, addr);
+
+    /* socket_connect() does a non-blocking connect(), but it
+     * still blocks in DNS lookups, so we must use a thread */
+    trace_qio_channel_socket_connect_async(ioc, addr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_connect_worker,
+                           addrCopy,
+                           (GDestroyNotify)qapi_free_SocketAddress);
+}
+
+
+int qio_channel_socket_listen_sync(QIOChannelSocket *ioc,
+                                   SocketAddress *addr,
+                                   Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_listen_sync(ioc, addr);
+    fd = socket_listen(addr, errp);
+    if (fd < 0) {
+        trace_qio_channel_socket_listen_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_listen_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_socket_listen_worker(QIOTask *task,
+                                            Error **errp,
+                                            gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    SocketAddress *addr = opaque;
+    int ret;
+
+    ret = qio_channel_socket_listen_sync(ioc,
+                                         addr,
+                                         errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_listen_async(QIOChannelSocket *ioc,
+                                     SocketAddress *addr,
+                                     QIOTaskFunc callback,
+                                     gpointer opaque,
+                                     GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    SocketAddress *addrCopy;
+
+    qapi_copy_SocketAddress(&addrCopy, addr);
+
+    /* socket_listen() blocks in DNS lookups, so we must use a thread */
+    trace_qio_channel_socket_listen_async(ioc, addr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_listen_worker,
+                           addrCopy,
+                           (GDestroyNotify)qapi_free_SocketAddress);
+}
+
+
+int qio_channel_socket_dgram_sync(QIOChannelSocket *ioc,
+                                  SocketAddress *localAddr,
+                                  SocketAddress *remoteAddr,
+                                  Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_dgram_sync(ioc, localAddr, remoteAddr);
+    fd = socket_dgram(remoteAddr, localAddr, errp);
+    if (fd < 0) {
+        trace_qio_channel_socket_dgram_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_dgram_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+struct QIOChannelSocketDGramWorkerData {
+    SocketAddress *localAddr;
+    SocketAddress *remoteAddr;
+};
+
+
+static void qio_channel_socket_dgram_worker_free(gpointer opaque)
+{
+    struct QIOChannelSocketDGramWorkerData *data = opaque;
+    qapi_free_SocketAddress(data->localAddr);
+    qapi_free_SocketAddress(data->remoteAddr);
+    g_free(data);
+}
+
+static int qio_channel_socket_dgram_worker(QIOTask *task,
+                                           Error **errp,
+                                           gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    struct QIOChannelSocketDGramWorkerData *data = opaque;
+    int ret;
+
+    /* socket_dgram() blocks in DNS lookups, so we must use a thread */
+    ret = qio_channel_socket_dgram_sync(ioc,
+                                        data->localAddr,
+                                        data->remoteAddr,
+                                        errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_dgram_async(QIOChannelSocket *ioc,
+                                    SocketAddress *localAddr,
+                                    SocketAddress *remoteAddr,
+                                    QIOTaskFunc callback,
+                                    gpointer opaque,
+                                    GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    struct QIOChannelSocketDGramWorkerData *data = g_new0(
+        struct QIOChannelSocketDGramWorkerData, 1);
+
+    qapi_copy_SocketAddress(&data->localAddr, localAddr);
+    qapi_copy_SocketAddress(&data->remoteAddr, remoteAddr);
+
+    trace_qio_channel_socket_dgram_async(ioc, localAddr, remoteAddr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_dgram_worker,
+                           data,
+                           qio_channel_socket_dgram_worker_free);
+}
+
+
+QIOChannelSocket *
+qio_channel_socket_accept(QIOChannelSocket *ioc,
+                          Error **errp)
+{
+    QIOChannelSocket *cioc;
+
+    cioc = QIO_CHANNEL_SOCKET(object_new(TYPE_QIO_CHANNEL_SOCKET));
+    cioc->fd = -1;
+    cioc->remoteAddrLen = sizeof(ioc->remoteAddr);
+    cioc->localAddrLen = sizeof(ioc->localAddr);
+
+#ifdef WIN32
+    QIO_CHANNEL(cioc)->event = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+
+
+ retry:
+    trace_qio_channel_socket_accept(ioc);
+    cioc->fd = qemu_accept(ioc->fd, (struct sockaddr *)&cioc->remoteAddr,
+                           &cioc->remoteAddrLen);
+    if (cioc->fd < 0) {
+        trace_qio_channel_socket_accept_fail(ioc);
+        if (errno == EINTR) {
+            goto retry;
+        }
+        goto error;
+    }
+
+    if (getsockname(cioc->fd, (struct sockaddr *)&cioc->localAddr,
+                    &cioc->localAddrLen) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to query local socket address");
+        goto error;
+    }
+
+#ifndef WIN32
+    if (cioc->localAddr.ss_family == AF_UNIX) {
+        QIO_CHANNEL(cioc)->features |= (1 << QIO_CHANNEL_FEATURE_FD_PASS);
+    }
+#endif /* WIN32 */
+
+    trace_qio_channel_socket_accept_complete(ioc, cioc, cioc->fd);
+    return cioc;
+
+ error:
+    object_unref(OBJECT(cioc));
+    return NULL;
+}
+
+static void qio_channel_socket_init(Object *obj)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(obj);
+    ioc->fd = -1;
+}
+
+static void qio_channel_socket_finalize(Object *obj)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(obj);
+    if (ioc->fd != -1) {
+#ifdef WIN32
+        WSAEventSelect(ioc->fd, NULL, 0);
+#endif
+        closesocket(ioc->fd);
+        ioc->fd = -1;
+    }
+}
+
+
+#ifndef WIN32
+static void qio_channel_socket_copy_fds(struct msghdr *msg,
+                                        int **fds, size_t *nfds)
+{
+    struct cmsghdr *cmsg;
+
+    *nfds = 0;
+    *fds = NULL;
+
+    for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+        int fd_size, i;
+        int gotfds;
+
+        if (cmsg->cmsg_len < CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+
+        fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+
+        if (!fd_size) {
+            continue;
+        }
+
+        gotfds = fd_size / sizeof(int);
+        *fds = g_renew(int, *fds, *nfds + gotfds);
+        memcpy(*fds + *nfds, CMSG_DATA(cmsg), fd_size);
+
+        for (i = 0; i < gotfds; i++) {
+            int fd = (*fds)[*nfds + i];
+            if (fd < 0) {
+                continue;
+            }
+
+            /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
+            qemu_set_block(fd);
+
+#ifndef MSG_CMSG_CLOEXEC
+            qemu_set_cloexec(fd);
+#endif
+        }
+        *nfds += gotfds;
+    }
+}
+
+
+static ssize_t qio_channel_socket_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t ret;
+    struct msghdr msg = { NULL, };
+    char control[CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS)];
+    int sflags = 0;
+
+    memset(control, 0, CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS));
+
+#ifdef MSG_CMSG_CLOEXEC
+    sflags |= MSG_CMSG_CLOEXEC;
+#endif
+
+    msg.msg_iov = (struct iovec *)iov;
+    msg.msg_iovlen = niov;
+    if (fds && nfds) {
+        msg.msg_control = control;
+        msg.msg_controllen = sizeof(control);
+    }
+
+ retry:
+    ret = recvmsg(sioc->fd, &msg, sflags);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from socket");
+        return -1;
+    }
+
+    if (fds && nfds) {
+        qio_channel_socket_copy_fds(&msg, fds, nfds);
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_socket_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t ret;
+    struct msghdr msg = { NULL, };
+    char control[CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS)];
+    size_t fdsize = sizeof(int) * nfds;
+    struct cmsghdr *cmsg;
+
+    memset(control, 0, CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS));
+
+    msg.msg_iov = (struct iovec *)iov;
+    msg.msg_iovlen = niov;
+
+    if (nfds) {
+        if (nfds > SOCKET_MAX_FDS) {
+            error_setg_errno(errp, EINVAL,
+                             "Only %d FDs can be sent, got %zu",
+                             SOCKET_MAX_FDS, nfds);
+            return -1;
+        }
+
+        msg.msg_control = control;
+        msg.msg_controllen = CMSG_SPACE(sizeof(int) * nfds);
+
+        cmsg = CMSG_FIRSTHDR(&msg);
+        cmsg->cmsg_len = CMSG_LEN(fdsize);
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type = SCM_RIGHTS;
+        memcpy(CMSG_DATA(cmsg), fds, fdsize);
+    }
+
+ retry:
+    ret = sendmsg(sioc->fd, &msg, 0);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno,
+                         "Unable to write to socket");
+        return -1;
+    }
+    return ret;
+}
+#else /* WIN32 */
+static ssize_t qio_channel_socket_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t done = 0;
+    ssize_t i;
+
+    for (i = 0; i < niov; i++) {
+        ssize_t ret;
+    retry:
+        ret = recv(sioc->fd,
+                   iov[i].iov_base,
+                   iov[i].iov_len,
+                   0);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            } else if (errno == EINTR) {
+                goto retry;
+            } else {
+                error_setg_errno(errp, errno,
+                                 "Unable to read from socket");
+                return -1;
+            }
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            return done;
+        }
+    }
+
+    return done;
+}
+
+static ssize_t qio_channel_socket_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t done = 0;
+    ssize_t i;
+
+    for (i = 0; i < niov; i++) {
+        ssize_t ret;
+    retry:
+        ret = send(sioc->fd,
+                   iov[i].iov_base,
+                   iov[i].iov_len,
+                   0);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            } else if (errno == EINTR) {
+                goto retry;
+            } else {
+                error_setg_errno(errp, errno,
+                                 "Unable to write to socket");
+                return -1;
+            }
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            return done;
+        }
+    }
+
+    return done;
+}
+#endif /* WIN32 */
+
+static int
+qio_channel_socket_set_blocking(QIOChannel *ioc,
+                                bool enabled,
+                                Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+
+    if (enabled) {
+        qemu_set_block(sioc->fd);
+    } else {
+        qemu_set_nonblock(sioc->fd);
+#ifdef WIN32
+        WSAEventSelect(sioc->fd, ioc->event,
+                       FD_READ | FD_ACCEPT | FD_CLOSE |
+                       FD_CONNECT | FD_WRITE | FD_OOB);
+#endif
+    }
+    return 0;
+}
+
+
+static void
+qio_channel_socket_set_delay(QIOChannel *ioc,
+                             bool enabled)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int v = enabled ? 0 : 1;
+
+    qemu_setsockopt(sioc->fd,
+                    IPPROTO_TCP, TCP_NODELAY,
+                    &v, sizeof(v));
+}
+
+
+static void
+qio_channel_socket_set_cork(QIOChannel *ioc,
+                            bool enabled)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int v = enabled ? 1 : 0;
+
+    socket_set_cork(sioc->fd, v);
+}
+
+
+static int
+qio_channel_socket_close(QIOChannel *ioc,
+                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+
+    if (sioc->fd != -1) {
+#ifdef WIN32
+        WSAEventSelect(sioc->fd, NULL, 0);
+#endif
+        if (closesocket(sioc->fd) < 0) {
+            sioc->fd = -1;
+            error_setg_errno(errp, errno,
+                             "Unable to close socket");
+            return -1;
+        }
+        sioc->fd = -1;
+    }
+    return 0;
+}
+
+static int
+qio_channel_socket_shutdown(QIOChannel *ioc,
+                            QIOChannelShutdown how,
+                            Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int sockhow;
+
+    switch (how) {
+    case QIO_CHANNEL_SHUTDOWN_READ:
+        sockhow = SHUT_RD;
+        break;
+    case QIO_CHANNEL_SHUTDOWN_WRITE:
+        sockhow = SHUT_WR;
+        break;
+    case QIO_CHANNEL_SHUTDOWN_BOTH:
+    default:
+        sockhow = SHUT_RDWR;
+        break;
+    }
+
+    if (shutdown(sioc->fd, sockhow) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to shutdown socket");
+        return -1;
+    }
+    return 0;
+}
+
+static GSource *qio_channel_socket_create_watch(QIOChannel *ioc,
+                                                GIOCondition condition)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    return qio_channel_create_socket_watch(ioc,
+                                           sioc->fd,
+                                           condition);
+}
+
+static void qio_channel_socket_class_init(ObjectClass *klass,
+                                          void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_socket_writev;
+    ioc_klass->io_readv = qio_channel_socket_readv;
+    ioc_klass->io_set_blocking = qio_channel_socket_set_blocking;
+    ioc_klass->io_close = qio_channel_socket_close;
+    ioc_klass->io_shutdown = qio_channel_socket_shutdown;
+    ioc_klass->io_set_cork = qio_channel_socket_set_cork;
+    ioc_klass->io_set_delay = qio_channel_socket_set_delay;
+    ioc_klass->io_create_watch = qio_channel_socket_create_watch;
+}
+
+static const TypeInfo qio_channel_socket_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_SOCKET,
+    .instance_size = sizeof(QIOChannelSocket),
+    .instance_init = qio_channel_socket_init,
+    .instance_finalize = qio_channel_socket_finalize,
+    .class_init = qio_channel_socket_class_init,
+};
+
+static void qio_channel_socket_register_types(void)
+{
+    type_register_static(&qio_channel_socket_info);
+}
+
+type_init(qio_channel_socket_register_types);
diff --git a/qemu/io/channel-tls.c b/qemu/io/channel-tls.c
new file mode 100644
index 000000000..9a8525c81
--- /dev/null
+++ b/qemu/io/channel-tls.c
@@ -0,0 +1,395 @@
+/*
+ * QEMU I/O channels TLS driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "io/channel-tls.h"
+#include "trace.h"
+
+
+static ssize_t qio_channel_tls_write_handler(const char *buf,
+                                             size_t len,
+                                             void *opaque)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(opaque);
+    ssize_t ret;
+
+    ret = qio_channel_write(tioc->master, buf, len, NULL);
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        return -1;
+    } else if (ret < 0) {
+        errno = EIO;
+        return -1;
+    }
+    return ret;
+}
+
+static ssize_t qio_channel_tls_read_handler(char *buf,
+                                            size_t len,
+                                            void *opaque)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(opaque);
+    ssize_t ret;
+
+    ret = qio_channel_read(tioc->master, buf, len, NULL);
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        return -1;
+    } else if (ret < 0) {
+        errno = EIO;
+        return -1;
+    }
+    return ret;
+}
+
+
+QIOChannelTLS *
+qio_channel_tls_new_server(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *aclname,
+                           Error **errp)
+{
+    QIOChannelTLS *ioc;
+
+    ioc = QIO_CHANNEL_TLS(object_new(TYPE_QIO_CHANNEL_TLS));
+
+    ioc->master = master;
+    object_ref(OBJECT(master));
+
+    ioc->session = qcrypto_tls_session_new(
+        creds,
+        NULL,
+        aclname,
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        errp);
+    if (!ioc->session) {
+        goto error;
+    }
+
+    qcrypto_tls_session_set_callbacks(
+        ioc->session,
+        qio_channel_tls_write_handler,
+        qio_channel_tls_read_handler,
+        ioc);
+
+    trace_qio_channel_tls_new_server(ioc, master, creds, aclname);
+    return ioc;
+
+ error:
+    object_unref(OBJECT(ioc));
+    return NULL;
+}
+
+QIOChannelTLS *
+qio_channel_tls_new_client(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *hostname,
+                           Error **errp)
+{
+    QIOChannelTLS *tioc;
+    QIOChannel *ioc;
+
+    tioc = QIO_CHANNEL_TLS(object_new(TYPE_QIO_CHANNEL_TLS));
+    ioc = QIO_CHANNEL(tioc);
+
+    tioc->master = master;
+    if (master->features & (1 << QIO_CHANNEL_FEATURE_SHUTDOWN)) {
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+    }
+    object_ref(OBJECT(master));
+
+    tioc->session = qcrypto_tls_session_new(
+        creds,
+        hostname,
+        NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        errp);
+    if (!tioc->session) {
+        goto error;
+    }
+
+    qcrypto_tls_session_set_callbacks(
+        tioc->session,
+        qio_channel_tls_write_handler,
+        qio_channel_tls_read_handler,
+        tioc);
+
+    trace_qio_channel_tls_new_client(tioc, master, creds, hostname);
+    return tioc;
+
+ error:
+    object_unref(OBJECT(tioc));
+    return NULL;
+}
+
+
+static gboolean qio_channel_tls_handshake_io(QIOChannel *ioc,
+                                             GIOCondition condition,
+                                             gpointer user_data);
+
+static void qio_channel_tls_handshake_task(QIOChannelTLS *ioc,
+                                           QIOTask *task)
+{
+    Error *err = NULL;
+    QCryptoTLSSessionHandshakeStatus status;
+
+    if (qcrypto_tls_session_handshake(ioc->session, &err) < 0) {
+        trace_qio_channel_tls_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        goto cleanup;
+    }
+
+    status = qcrypto_tls_session_get_handshake_status(ioc->session);
+    if (status == QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+        trace_qio_channel_tls_handshake_complete(ioc);
+        if (qcrypto_tls_session_check_credentials(ioc->session,
+                                                  &err) < 0) {
+            trace_qio_channel_tls_credentials_deny(ioc);
+            qio_task_abort(task, err);
+            goto cleanup;
+        }
+        trace_qio_channel_tls_credentials_allow(ioc);
+        qio_task_complete(task);
+    } else {
+        GIOCondition condition;
+        if (status == QCRYPTO_TLS_HANDSHAKE_SENDING) {
+            condition = G_IO_OUT;
+        } else {
+            condition = G_IO_IN;
+        }
+
+        trace_qio_channel_tls_handshake_pending(ioc, status);
+        qio_channel_add_watch(ioc->master,
+                              condition,
+                              qio_channel_tls_handshake_io,
+                              task,
+                              NULL);
+    }
+
+ cleanup:
+    error_free(err);
+}
+
+
+static gboolean qio_channel_tls_handshake_io(QIOChannel *ioc,
+                                             GIOCondition condition,
+                                             gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(
+        qio_task_get_source(task));
+
+    qio_channel_tls_handshake_task(
+       tioc, task);
+
+    object_unref(OBJECT(tioc));
+
+    return FALSE;
+}
+
+void qio_channel_tls_handshake(QIOChannelTLS *ioc,
+                               QIOTaskFunc func,
+                               gpointer opaque,
+                               GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = qio_task_new(OBJECT(ioc),
+                        func, opaque, destroy);
+
+    trace_qio_channel_tls_handshake_start(ioc);
+    qio_channel_tls_handshake_task(ioc, task);
+}
+
+
+static void qio_channel_tls_init(Object *obj G_GNUC_UNUSED)
+{
+}
+
+
+static void qio_channel_tls_finalize(Object *obj)
+{
+    QIOChannelTLS *ioc = QIO_CHANNEL_TLS(obj);
+
+    object_unref(OBJECT(ioc->master));
+    qcrypto_tls_session_free(ioc->session);
+}
+
+
+static ssize_t qio_channel_tls_readv(QIOChannel *ioc,
+                                     const struct iovec *iov,
+                                     size_t niov,
+                                     int **fds,
+                                     size_t *nfds,
+                                     Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+    size_t i;
+    ssize_t got = 0;
+
+    for (i = 0 ; i < niov ; i++) {
+        ssize_t ret = qcrypto_tls_session_read(tioc->session,
+                                               iov[i].iov_base,
+                                               iov[i].iov_len);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (got) {
+                    return got;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            }
+
+            error_setg_errno(errp, errno,
+                             "Cannot read from TLS channel");
+            return -1;
+        }
+        got += ret;
+        if (ret < iov[i].iov_len) {
+            break;
+        }
+    }
+    return got;
+}
+
+
+static ssize_t qio_channel_tls_writev(QIOChannel *ioc,
+                                      const struct iovec *iov,
+                                      size_t niov,
+                                      int *fds,
+                                      size_t nfds,
+                                      Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+    size_t i;
+    ssize_t done = 0;
+
+    for (i = 0 ; i < niov ; i++) {
+        ssize_t ret = qcrypto_tls_session_write(tioc->session,
+                                                iov[i].iov_base,
+                                                iov[i].iov_len);
+        if (ret <= 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            }
+
+            error_setg_errno(errp, errno,
+                             "Cannot write to TLS channel");
+            return -1;
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            break;
+        }
+    }
+    return done;
+}
+
+static int qio_channel_tls_set_blocking(QIOChannel *ioc,
+                                        bool enabled,
+                                        Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_set_blocking(tioc->master, enabled, errp);
+}
+
+static void qio_channel_tls_set_delay(QIOChannel *ioc,
+                                      bool enabled)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    qio_channel_set_delay(tioc->master, enabled);
+}
+
+static void qio_channel_tls_set_cork(QIOChannel *ioc,
+                                     bool enabled)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    qio_channel_set_cork(tioc->master, enabled);
+}
+
+static int qio_channel_tls_shutdown(QIOChannel *ioc,
+                                    QIOChannelShutdown how,
+                                    Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_shutdown(tioc->master, how, errp);
+}
+
+static int qio_channel_tls_close(QIOChannel *ioc,
+                                 Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_close(tioc->master, errp);
+}
+
+static GSource *qio_channel_tls_create_watch(QIOChannel *ioc,
+                                             GIOCondition condition)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_create_watch(tioc->master, condition);
+}
+
+QCryptoTLSSession *
+qio_channel_tls_get_session(QIOChannelTLS *ioc)
+{
+    return ioc->session;
+}
+
+static void qio_channel_tls_class_init(ObjectClass *klass,
+                                       void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_tls_writev;
+    ioc_klass->io_readv = qio_channel_tls_readv;
+    ioc_klass->io_set_blocking = qio_channel_tls_set_blocking;
+    ioc_klass->io_set_delay = qio_channel_tls_set_delay;
+    ioc_klass->io_set_cork = qio_channel_tls_set_cork;
+    ioc_klass->io_close = qio_channel_tls_close;
+    ioc_klass->io_shutdown = qio_channel_tls_shutdown;
+    ioc_klass->io_create_watch = qio_channel_tls_create_watch;
+}
+
+static const TypeInfo qio_channel_tls_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_TLS,
+    .instance_size = sizeof(QIOChannelTLS),
+    .instance_init = qio_channel_tls_init,
+    .instance_finalize = qio_channel_tls_finalize,
+    .class_init = qio_channel_tls_class_init,
+};
+
+static void qio_channel_tls_register_types(void)
+{
+    type_register_static(&qio_channel_tls_info);
+}
+
+type_init(qio_channel_tls_register_types);
diff --git a/qemu/io/channel-util.c b/qemu/io/channel-util.c
new file mode 100644
index 000000000..0fb4bd083
--- /dev/null
+++ b/qemu/io/channel-util.c
@@ -0,0 +1,51 @@
+/*
+ * QEMU I/O channels utility APIs
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-util.h"
+#include "io/channel-file.h"
+#include "io/channel-socket.h"
+
+
+static bool fd_is_socket(int fd)
+{
+    int optval;
+    socklen_t optlen;
+    optlen = sizeof(optval);
+    return qemu_getsockopt(fd,
+                           SOL_SOCKET,
+                           SO_TYPE,
+                           (char *)&optval,
+                           &optlen) == 0;
+}
+
+
+QIOChannel *qio_channel_new_fd(int fd,
+                               Error **errp)
+{
+    QIOChannel *ioc;
+
+    if (fd_is_socket(fd)) {
+        ioc = QIO_CHANNEL(qio_channel_socket_new_fd(fd, errp));
+    } else {
+        ioc = QIO_CHANNEL(qio_channel_file_new_fd(fd));
+    }
+    return ioc;
+}
diff --git a/qemu/io/channel-watch.c b/qemu/io/channel-watch.c
new file mode 100644
index 000000000..cf1cdff89
--- /dev/null
+++ b/qemu/io/channel-watch.c
@@ -0,0 +1,347 @@
+/*
+ * QEMU I/O channels watch helper APIs
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-watch.h"
+
+typedef struct QIOChannelFDSource QIOChannelFDSource;
+struct QIOChannelFDSource {
+    GSource parent;
+    GPollFD fd;
+    QIOChannel *ioc;
+    GIOCondition condition;
+};
+
+
+#ifdef CONFIG_WIN32
+typedef struct QIOChannelSocketSource QIOChannelSocketSource;
+struct QIOChannelSocketSource {
+    GSource parent;
+    GPollFD fd;
+    QIOChannel *ioc;
+    SOCKET socket;
+    int revents;
+    GIOCondition condition;
+};
+
+#endif
+
+
+typedef struct QIOChannelFDPairSource QIOChannelFDPairSource;
+struct QIOChannelFDPairSource {
+    GSource parent;
+    GPollFD fdread;
+    GPollFD fdwrite;
+    QIOChannel *ioc;
+    GIOCondition condition;
+};
+
+
+static gboolean
+qio_channel_fd_source_prepare(GSource *source G_GNUC_UNUSED,
+                              gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+static gboolean
+qio_channel_fd_source_check(GSource *source)
+{
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    return ssource->fd.revents & ssource->condition;
+}
+
+
+static gboolean
+qio_channel_fd_source_dispatch(GSource *source,
+                               GSourceFunc callback,
+                               gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    return (*func)(ssource->ioc,
+                   ssource->fd.revents & ssource->condition,
+                   user_data);
+}
+
+
+static void
+qio_channel_fd_source_finalize(GSource *source)
+{
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+#ifdef CONFIG_WIN32
+static gboolean
+qio_channel_socket_source_prepare(GSource *source G_GNUC_UNUSED,
+                                  gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+/*
+ * NB, this impl only works when the socket is in non-blocking
+ * mode on Win32
+ */
+static gboolean
+qio_channel_socket_source_check(GSource *source)
+{
+    static struct timeval tv0;
+
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+    WSANETWORKEVENTS ev;
+    fd_set rfds, wfds, xfds;
+
+    if (!ssource->condition) {
+        return 0;
+    }
+
+    WSAEnumNetworkEvents(ssource->socket, ssource->ioc->event, &ev);
+
+    FD_ZERO(&rfds);
+    FD_ZERO(&wfds);
+    FD_ZERO(&xfds);
+    if (ssource->condition & G_IO_IN) {
+        FD_SET((SOCKET)ssource->socket, &rfds);
+    }
+    if (ssource->condition & G_IO_OUT) {
+        FD_SET((SOCKET)ssource->socket, &wfds);
+    }
+    if (ssource->condition & G_IO_PRI) {
+        FD_SET((SOCKET)ssource->socket, &xfds);
+    }
+    ssource->revents = 0;
+    if (select(0, &rfds, &wfds, &xfds, &tv0) == 0) {
+        return 0;
+    }
+
+    if (FD_ISSET(ssource->socket, &rfds)) {
+        ssource->revents |= G_IO_IN;
+    }
+    if (FD_ISSET(ssource->socket, &wfds)) {
+        ssource->revents |= G_IO_OUT;
+    }
+    if (FD_ISSET(ssource->socket, &xfds)) {
+        ssource->revents |= G_IO_PRI;
+    }
+
+    return ssource->revents;
+}
+
+
+static gboolean
+qio_channel_socket_source_dispatch(GSource *source,
+                                   GSourceFunc callback,
+                                   gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+
+    return (*func)(ssource->ioc, ssource->revents, user_data);
+}
+
+
+static void
+qio_channel_socket_source_finalize(GSource *source)
+{
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+GSourceFuncs qio_channel_socket_source_funcs = {
+    qio_channel_socket_source_prepare,
+    qio_channel_socket_source_check,
+    qio_channel_socket_source_dispatch,
+    qio_channel_socket_source_finalize
+};
+#endif
+
+
+static gboolean
+qio_channel_fd_pair_source_prepare(GSource *source G_GNUC_UNUSED,
+                                   gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+static gboolean
+qio_channel_fd_pair_source_check(GSource *source)
+{
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+    GIOCondition poll_condition = ssource->fdread.revents |
+        ssource->fdwrite.revents;
+
+    return poll_condition & ssource->condition;
+}
+
+
+static gboolean
+qio_channel_fd_pair_source_dispatch(GSource *source,
+                                    GSourceFunc callback,
+                                    gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+    GIOCondition poll_condition = ssource->fdread.revents |
+        ssource->fdwrite.revents;
+
+    return (*func)(ssource->ioc,
+                   poll_condition & ssource->condition,
+                   user_data);
+}
+
+
+static void
+qio_channel_fd_pair_source_finalize(GSource *source)
+{
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+GSourceFuncs qio_channel_fd_source_funcs = {
+    qio_channel_fd_source_prepare,
+    qio_channel_fd_source_check,
+    qio_channel_fd_source_dispatch,
+    qio_channel_fd_source_finalize
+};
+
+
+GSourceFuncs qio_channel_fd_pair_source_funcs = {
+    qio_channel_fd_pair_source_prepare,
+    qio_channel_fd_pair_source_check,
+    qio_channel_fd_pair_source_dispatch,
+    qio_channel_fd_pair_source_finalize
+};
+
+
+GSource *qio_channel_create_fd_watch(QIOChannel *ioc,
+                                     int fd,
+                                     GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelFDSource *ssource;
+
+    source = g_source_new(&qio_channel_fd_source_funcs,
+                          sizeof(QIOChannelFDSource));
+    ssource = (QIOChannelFDSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+
+#ifdef CONFIG_WIN32
+    ssource->fd.fd = (gint64)_get_osfhandle(fd);
+#else
+    ssource->fd.fd = fd;
+#endif
+    ssource->fd.events = condition;
+
+    g_source_add_poll(source, &ssource->fd);
+
+    return source;
+}
+
+#ifdef CONFIG_WIN32
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int socket,
+                                         GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelSocketSource *ssource;
+
+    source = g_source_new(&qio_channel_socket_source_funcs,
+                          sizeof(QIOChannelSocketSource));
+    ssource = (QIOChannelSocketSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+    ssource->socket = socket;
+    ssource->revents = 0;
+
+    ssource->fd.fd = (gintptr)ioc->event;
+    ssource->fd.events = G_IO_IN;
+
+    g_source_add_poll(source, &ssource->fd);
+
+    return source;
+}
+#else
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int socket,
+                                         GIOCondition condition)
+{
+    return qio_channel_create_fd_watch(ioc, socket, condition);
+}
+#endif
+
+GSource *qio_channel_create_fd_pair_watch(QIOChannel *ioc,
+                                          int fdread,
+                                          int fdwrite,
+                                          GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelFDPairSource *ssource;
+
+    source = g_source_new(&qio_channel_fd_pair_source_funcs,
+                          sizeof(QIOChannelFDPairSource));
+    ssource = (QIOChannelFDPairSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+
+#ifdef CONFIG_WIN32
+    ssource->fdread.fd = (gint64)_get_osfhandle(fdread);
+    ssource->fdwrite.fd = (gint64)_get_osfhandle(fdwrite);
+#else
+    ssource->fdread.fd = fdread;
+    ssource->fdwrite.fd = fdwrite;
+#endif
+
+    ssource->fdread.events = condition & G_IO_IN;
+    ssource->fdwrite.events = condition & G_IO_OUT;
+
+    g_source_add_poll(source, &ssource->fdread);
+    g_source_add_poll(source, &ssource->fdwrite);
+
+    return source;
+}
diff --git a/qemu/io/channel-websock.c b/qemu/io/channel-websock.c
new file mode 100644
index 000000000..708178779
--- /dev/null
+++ b/qemu/io/channel-websock.c
@@ -0,0 +1,964 @@
+/*
+ * QEMU I/O channels driver websockets
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "io/channel-websock.h"
+#include "crypto/hash.h"
+#include "trace.h"
+
+
+/* Max amount to allow in rawinput/rawoutput buffers */
+#define QIO_CHANNEL_WEBSOCK_MAX_BUFFER 8192
+
+#define QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN 24
+#define QIO_CHANNEL_WEBSOCK_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+#define QIO_CHANNEL_WEBSOCK_GUID_LEN strlen(QIO_CHANNEL_WEBSOCK_GUID)
+
+#define QIO_CHANNEL_WEBSOCK_HEADER_PROTOCOL "Sec-WebSocket-Protocol"
+#define QIO_CHANNEL_WEBSOCK_HEADER_VERSION "Sec-WebSocket-Version"
+#define QIO_CHANNEL_WEBSOCK_HEADER_KEY "Sec-WebSocket-Key"
+
+#define QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY "binary"
+
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_RESPONSE  \
+    "HTTP/1.1 101 Switching Protocols\r\n"      \
+    "Upgrade: websocket\r\n"                    \
+    "Connection: Upgrade\r\n"                   \
+    "Sec-WebSocket-Accept: %s\r\n"              \
+    "Sec-WebSocket-Protocol: binary\r\n"        \
+    "\r\n"
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM "\r\n"
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_END "\r\n\r\n"
+#define QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION "13"
+
+/* The websockets packet header is variable length
+ * depending on the size of the payload... */
+
+/* ...length when using 7-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT 6
+/* ...length when using 16-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT 8
+/* ...length when using 64-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT 14
+
+/* Length of the optional data mask field in header */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_MASK 4
+
+/* Maximum length that can fit in 7-bit payload size */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_7_BIT 126
+/* Maximum length that can fit in 16-bit payload size */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_16_BIT 65536
+
+/* Magic 7-bit length to indicate use of 16-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT 126
+/* Magic 7-bit length to indicate use of 64-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT 127
+
+/* Bitmasks & shifts for accessing header fields */
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_FIN 0x80
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE 0x0f
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_HAS_MASK 0x80
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_PAYLOAD_LEN 0x7f
+#define QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN 7
+#define QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_HAS_MASK 7
+
+typedef struct QIOChannelWebsockHeader QIOChannelWebsockHeader;
+
+struct QEMU_PACKED QIOChannelWebsockHeader {
+    unsigned char b0;
+    unsigned char b1;
+    union {
+        struct QEMU_PACKED {
+            uint16_t l16;
+            QIOChannelWebsockMask m16;
+        } s16;
+        struct QEMU_PACKED {
+            uint64_t l64;
+            QIOChannelWebsockMask m64;
+        } s64;
+        QIOChannelWebsockMask m;
+    } u;
+};
+
+enum {
+    QIO_CHANNEL_WEBSOCK_OPCODE_CONTINUATION = 0x0,
+    QIO_CHANNEL_WEBSOCK_OPCODE_TEXT_FRAME = 0x1,
+    QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME = 0x2,
+    QIO_CHANNEL_WEBSOCK_OPCODE_CLOSE = 0x8,
+    QIO_CHANNEL_WEBSOCK_OPCODE_PING = 0x9,
+    QIO_CHANNEL_WEBSOCK_OPCODE_PONG = 0xA
+};
+
+static char *qio_channel_websock_handshake_entry(const char *handshake,
+                                                 size_t handshake_len,
+                                                 const char *name)
+{
+    char *begin, *end, *ret = NULL;
+    char *line = g_strdup_printf("%s%s: ",
+                                 QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM,
+                                 name);
+    begin = g_strstr_len(handshake, handshake_len, line);
+    if (begin != NULL) {
+        begin += strlen(line);
+        end = g_strstr_len(begin, handshake_len - (begin - handshake),
+                QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM);
+        if (end != NULL) {
+            ret = g_strndup(begin, end - begin);
+        }
+    }
+    g_free(line);
+    return ret;
+}
+
+
+static int qio_channel_websock_handshake_send_response(QIOChannelWebsock *ioc,
+                                                       const char *key,
+                                                       Error **errp)
+{
+    char combined_key[QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+                      QIO_CHANNEL_WEBSOCK_GUID_LEN + 1];
+    char *accept = NULL, *response = NULL;
+    size_t responselen;
+
+    g_strlcpy(combined_key, key, QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN + 1);
+    g_strlcat(combined_key, QIO_CHANNEL_WEBSOCK_GUID,
+              QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+              QIO_CHANNEL_WEBSOCK_GUID_LEN + 1);
+
+    /* hash and encode it */
+    if (qcrypto_hash_base64(QCRYPTO_HASH_ALG_SHA1,
+                            combined_key,
+                            QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+                            QIO_CHANNEL_WEBSOCK_GUID_LEN,
+                            &accept,
+                            errp) < 0) {
+        return -1;
+    }
+
+    response = g_strdup_printf(QIO_CHANNEL_WEBSOCK_HANDSHAKE_RESPONSE, accept);
+    responselen = strlen(response);
+    buffer_reserve(&ioc->encoutput, responselen);
+    buffer_append(&ioc->encoutput, response, responselen);
+
+    g_free(accept);
+    g_free(response);
+
+    return 0;
+}
+
+static int qio_channel_websock_handshake_process(QIOChannelWebsock *ioc,
+                                                 const char *line,
+                                                 size_t size,
+                                                 Error **errp)
+{
+    int ret = -1;
+    char *protocols = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_PROTOCOL);
+    char *version = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_VERSION);
+    char *key = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_KEY);
+
+    if (!protocols) {
+        error_setg(errp, "Missing websocket protocol header data");
+        goto cleanup;
+    }
+
+    if (!version) {
+        error_setg(errp, "Missing websocket version header data");
+        goto cleanup;
+    }
+
+    if (!key) {
+        error_setg(errp, "Missing websocket key header data");
+        goto cleanup;
+    }
+
+    if (!g_strrstr(protocols, QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY)) {
+        error_setg(errp, "No '%s' protocol is supported by client '%s'",
+                   QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY, protocols);
+        goto cleanup;
+    }
+
+    if (!g_str_equal(version, QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION)) {
+        error_setg(errp, "Version '%s' is not supported by client '%s'",
+                   QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION, version);
+        goto cleanup;
+    }
+
+    if (strlen(key) != QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN) {
+        error_setg(errp, "Key length '%zu' was not as expected '%d'",
+                   strlen(key), QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN);
+        goto cleanup;
+    }
+
+    ret = qio_channel_websock_handshake_send_response(ioc, key, errp);
+
+ cleanup:
+    g_free(protocols);
+    g_free(version);
+    g_free(key);
+    return ret;
+}
+
+static int qio_channel_websock_handshake_read(QIOChannelWebsock *ioc,
+                                              Error **errp)
+{
+    char *handshake_end;
+    ssize_t ret;
+    /* Typical HTTP headers from novnc are 512 bytes, so limiting
+     * total header size to 4096 is easily enough. */
+    size_t want = 4096 - ioc->encinput.offset;
+    buffer_reserve(&ioc->encinput, want);
+    ret = qio_channel_read(ioc->master,
+                           (char *)buffer_end(&ioc->encinput), want, errp);
+    if (ret < 0) {
+        return -1;
+    }
+    ioc->encinput.offset += ret;
+
+    handshake_end = g_strstr_len((char *)ioc->encinput.buffer,
+                                 ioc->encinput.offset,
+                                 QIO_CHANNEL_WEBSOCK_HANDSHAKE_END);
+    if (!handshake_end) {
+        if (ioc->encinput.offset >= 4096) {
+            error_setg(errp,
+                       "End of headers not found in first 4096 bytes");
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    if (qio_channel_websock_handshake_process(ioc,
+                                              (char *)ioc->encinput.buffer,
+                                              ioc->encinput.offset,
+                                              errp) < 0) {
+        return -1;
+    }
+
+    buffer_advance(&ioc->encinput,
+                   handshake_end - (char *)ioc->encinput.buffer +
+                   strlen(QIO_CHANNEL_WEBSOCK_HANDSHAKE_END));
+    return 1;
+}
+
+static gboolean qio_channel_websock_handshake_send(QIOChannel *ioc,
+                                                   GIOCondition condition,
+                                                   gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(
+        qio_task_get_source(task));
+    Error *err = NULL;
+    ssize_t ret;
+
+    ret = qio_channel_write(wioc->master,
+                            (char *)wioc->encoutput.buffer,
+                            wioc->encoutput.offset,
+                            &err);
+
+    if (ret < 0) {
+        trace_qio_channel_websock_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        error_free(err);
+        return FALSE;
+    }
+
+    buffer_advance(&wioc->encoutput, ret);
+    if (wioc->encoutput.offset == 0) {
+        trace_qio_channel_websock_handshake_complete(ioc);
+        qio_task_complete(task);
+        return FALSE;
+    }
+    trace_qio_channel_websock_handshake_pending(ioc, G_IO_OUT);
+    return TRUE;
+}
+
+static gboolean qio_channel_websock_handshake_io(QIOChannel *ioc,
+                                                 GIOCondition condition,
+                                                 gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(
+        qio_task_get_source(task));
+    Error *err = NULL;
+    int ret;
+
+    ret = qio_channel_websock_handshake_read(wioc, &err);
+    if (ret < 0) {
+        trace_qio_channel_websock_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        error_free(err);
+        return FALSE;
+    }
+    if (ret == 0) {
+        trace_qio_channel_websock_handshake_pending(ioc, G_IO_IN);
+        /* need more data still */
+        return TRUE;
+    }
+
+    object_ref(OBJECT(task));
+    trace_qio_channel_websock_handshake_reply(ioc);
+    qio_channel_add_watch(
+        wioc->master,
+        G_IO_OUT,
+        qio_channel_websock_handshake_send,
+        task,
+        (GDestroyNotify)object_unref);
+    return FALSE;
+}
+
+
+static void qio_channel_websock_encode(QIOChannelWebsock *ioc)
+{
+    size_t header_size;
+    union {
+        char buf[QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT];
+        QIOChannelWebsockHeader ws;
+    } header;
+
+    if (!ioc->rawoutput.offset) {
+        return;
+    }
+
+    header.ws.b0 = (1 << QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN) |
+        (QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME &
+         QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE);
+    if (ioc->rawoutput.offset <
+        QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_7_BIT) {
+        header.ws.b1 = (uint8_t)ioc->rawoutput.offset;
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT;
+    } else if (ioc->rawoutput.offset <
+               QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_16_BIT) {
+        header.ws.b1 = QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT;
+        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)ioc->rawoutput.offset);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT;
+    } else {
+        header.ws.b1 = QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT;
+        header.ws.u.s64.l64 = cpu_to_be64(ioc->rawoutput.offset);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT;
+    }
+    header_size -= QIO_CHANNEL_WEBSOCK_HEADER_LEN_MASK;
+
+    buffer_reserve(&ioc->encoutput, header_size + ioc->rawoutput.offset);
+    buffer_append(&ioc->encoutput, header.buf, header_size);
+    buffer_append(&ioc->encoutput, ioc->rawoutput.buffer,
+                  ioc->rawoutput.offset);
+    buffer_reset(&ioc->rawoutput);
+}
+
+
+static ssize_t qio_channel_websock_decode_header(QIOChannelWebsock *ioc,
+                                                 Error **errp)
+{
+    unsigned char opcode, fin, has_mask;
+    size_t header_size;
+    size_t payload_len;
+    QIOChannelWebsockHeader *header =
+        (QIOChannelWebsockHeader *)ioc->encinput.buffer;
+
+    if (ioc->payload_remain) {
+        error_setg(errp,
+                   "Decoding header but %zu bytes of payload remain",
+                   ioc->payload_remain);
+        return -1;
+    }
+    if (ioc->encinput.offset < QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT) {
+        /* header not complete */
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    fin = (header->b0 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_FIN) >>
+        QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN;
+    opcode = header->b0 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE;
+    has_mask = (header->b1 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_HAS_MASK) >>
+        QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_HAS_MASK;
+    payload_len = header->b1 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_PAYLOAD_LEN;
+
+    if (opcode == QIO_CHANNEL_WEBSOCK_OPCODE_CLOSE) {
+        /* disconnect */
+        return 0;
+    }
+
+    /* Websocket frame sanity check:
+     * * Websocket fragmentation is not supported.
+     * * All  websockets frames sent by a client have to be masked.
+     * * Only binary encoding is supported.
+     */
+    if (!fin) {
+        error_setg(errp, "websocket fragmentation is not supported");
+        return -1;
+    }
+    if (!has_mask) {
+        error_setg(errp, "websocket frames must be masked");
+        return -1;
+    }
+    if (opcode != QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME) {
+        error_setg(errp, "only binary websocket frames are supported");
+        return -1;
+    }
+
+    if (payload_len < QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT) {
+        ioc->payload_remain = payload_len;
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT;
+        ioc->mask = header->u.m;
+    } else if (payload_len == QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT &&
+               ioc->encinput.offset >= QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT) {
+        ioc->payload_remain = be16_to_cpu(header->u.s16.l16);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT;
+        ioc->mask = header->u.s16.m16;
+    } else if (payload_len == QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT &&
+               ioc->encinput.offset >= QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT) {
+        ioc->payload_remain = be64_to_cpu(header->u.s64.l64);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT;
+        ioc->mask = header->u.s64.m64;
+    } else {
+        /* header not complete */
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    buffer_advance(&ioc->encinput, header_size);
+    return 1;
+}
+
+
+static ssize_t qio_channel_websock_decode_payload(QIOChannelWebsock *ioc,
+                                                  Error **errp)
+{
+    size_t i;
+    size_t payload_len;
+    uint32_t *payload32;
+
+    if (!ioc->payload_remain) {
+        error_setg(errp,
+                   "Decoding payload but no bytes of payload remain");
+        return -1;
+    }
+
+    /* If we aren't at the end of the payload, then drop
+     * off the last bytes, so we're always multiple of 4
+     * for purpose of unmasking, except at end of payload
+     */
+    if (ioc->encinput.offset < ioc->payload_remain) {
+        payload_len = ioc->encinput.offset - (ioc->encinput.offset % 4);
+    } else {
+        payload_len = ioc->payload_remain;
+    }
+    if (payload_len == 0) {
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    ioc->payload_remain -= payload_len;
+
+    /* unmask frame */
+    /* process 1 frame (32 bit op) */
+    payload32 = (uint32_t *)ioc->encinput.buffer;
+    for (i = 0; i < payload_len / 4; i++) {
+        payload32[i] ^= ioc->mask.u;
+    }
+    /* process the remaining bytes (if any) */
+    for (i *= 4; i < payload_len; i++) {
+        ioc->encinput.buffer[i] ^= ioc->mask.c[i % 4];
+    }
+
+    buffer_reserve(&ioc->rawinput, payload_len);
+    buffer_append(&ioc->rawinput, ioc->encinput.buffer, payload_len);
+    buffer_advance(&ioc->encinput, payload_len);
+    return payload_len;
+}
+
+
+QIOChannelWebsock *
+qio_channel_websock_new_server(QIOChannel *master)
+{
+    QIOChannelWebsock *wioc;
+    QIOChannel *ioc;
+
+    wioc = QIO_CHANNEL_WEBSOCK(object_new(TYPE_QIO_CHANNEL_WEBSOCK));
+    ioc = QIO_CHANNEL(wioc);
+
+    wioc->master = master;
+    if (master->features & (1 << QIO_CHANNEL_FEATURE_SHUTDOWN)) {
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+    }
+    object_ref(OBJECT(master));
+
+    trace_qio_channel_websock_new_server(wioc, master);
+    return wioc;
+}
+
+void qio_channel_websock_handshake(QIOChannelWebsock *ioc,
+                                   QIOTaskFunc func,
+                                   gpointer opaque,
+                                   GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = qio_task_new(OBJECT(ioc),
+                        func,
+                        opaque,
+                        destroy);
+
+    trace_qio_channel_websock_handshake_start(ioc);
+    trace_qio_channel_websock_handshake_pending(ioc, G_IO_IN);
+    qio_channel_add_watch(ioc->master,
+                          G_IO_IN,
+                          qio_channel_websock_handshake_io,
+                          task,
+                          NULL);
+}
+
+
+static void qio_channel_websock_finalize(Object *obj)
+{
+    QIOChannelWebsock *ioc = QIO_CHANNEL_WEBSOCK(obj);
+
+    buffer_free(&ioc->encinput);
+    buffer_free(&ioc->encoutput);
+    buffer_free(&ioc->rawinput);
+    buffer_free(&ioc->rawoutput);
+    object_unref(OBJECT(ioc->master));
+    if (ioc->io_tag) {
+        g_source_remove(ioc->io_tag);
+    }
+    if (ioc->io_err) {
+        error_free(ioc->io_err);
+    }
+}
+
+
+static ssize_t qio_channel_websock_read_wire(QIOChannelWebsock *ioc,
+                                             Error **errp)
+{
+    ssize_t ret;
+
+    if (ioc->encinput.offset < 4096) {
+        size_t want = 4096 - ioc->encinput.offset;
+
+        buffer_reserve(&ioc->encinput, want);
+        ret = qio_channel_read(ioc->master,
+                               (char *)ioc->encinput.buffer +
+                               ioc->encinput.offset,
+                               want,
+                               errp);
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret == 0 &&
+            ioc->encinput.offset == 0) {
+            return 0;
+        }
+        ioc->encinput.offset += ret;
+    }
+
+    if (ioc->payload_remain == 0) {
+        ret = qio_channel_websock_decode_header(ioc, errp);
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret == 0) {
+            return 0;
+        }
+    }
+
+    ret = qio_channel_websock_decode_payload(ioc, errp);
+    if (ret < 0) {
+        return ret;
+    }
+    return ret;
+}
+
+
+static ssize_t qio_channel_websock_write_wire(QIOChannelWebsock *ioc,
+                                              Error **errp)
+{
+    ssize_t ret;
+    ssize_t done = 0;
+    qio_channel_websock_encode(ioc);
+
+    while (ioc->encoutput.offset > 0) {
+        ret = qio_channel_write(ioc->master,
+                                (char *)ioc->encoutput.buffer,
+                                ioc->encoutput.offset,
+                                errp);
+        if (ret < 0) {
+            if (ret == QIO_CHANNEL_ERR_BLOCK &&
+                done > 0) {
+                return done;
+            } else {
+                return ret;
+            }
+        }
+        buffer_advance(&ioc->encoutput, ret);
+        done += ret;
+    }
+    return done;
+}
+
+
+static void qio_channel_websock_flush_free(gpointer user_data)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(user_data);
+    object_unref(OBJECT(wioc));
+}
+
+static void qio_channel_websock_set_watch(QIOChannelWebsock *ioc);
+
+static gboolean qio_channel_websock_flush(QIOChannel *ioc,
+                                          GIOCondition condition,
+                                          gpointer user_data)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(user_data);
+    ssize_t ret;
+
+    if (condition & G_IO_OUT) {
+        ret = qio_channel_websock_write_wire(wioc, &wioc->io_err);
+        if (ret < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (condition & G_IO_IN) {
+        ret = qio_channel_websock_read_wire(wioc, &wioc->io_err);
+        if (ret < 0) {
+            goto cleanup;
+        }
+        if (ret == 0) {
+            wioc->io_eof = TRUE;
+        }
+    }
+
+ cleanup:
+    qio_channel_websock_set_watch(wioc);
+    return FALSE;
+}
+
+
+static void qio_channel_websock_unset_watch(QIOChannelWebsock *ioc)
+{
+    if (ioc->io_tag) {
+        g_source_remove(ioc->io_tag);
+        ioc->io_tag = 0;
+    }
+}
+
+static void qio_channel_websock_set_watch(QIOChannelWebsock *ioc)
+{
+    GIOCondition cond = 0;
+
+    qio_channel_websock_unset_watch(ioc);
+
+    if (ioc->io_err) {
+        return;
+    }
+
+    if (ioc->encoutput.offset) {
+        cond |= G_IO_OUT;
+    }
+    if (ioc->encinput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER &&
+        !ioc->io_eof) {
+        cond |= G_IO_IN;
+    }
+
+    if (cond) {
+        object_ref(OBJECT(ioc));
+        ioc->io_tag =
+            qio_channel_add_watch(ioc->master,
+                                  cond,
+                                  qio_channel_websock_flush,
+                                  ioc,
+                                  qio_channel_websock_flush_free);
+    }
+}
+
+
+static ssize_t qio_channel_websock_readv(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int **fds,
+                                         size_t *nfds,
+                                         Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    size_t i;
+    ssize_t got = 0;
+    ssize_t ret;
+
+    if (wioc->io_err) {
+        *errp = error_copy(wioc->io_err);
+        return -1;
+    }
+
+    if (!wioc->rawinput.offset) {
+        ret = qio_channel_websock_read_wire(QIO_CHANNEL_WEBSOCK(ioc), errp);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    for (i = 0 ; i < niov ; i++) {
+        size_t want = iov[i].iov_len;
+        if (want > (wioc->rawinput.offset - got)) {
+            want = (wioc->rawinput.offset - got);
+        }
+
+        memcpy(iov[i].iov_base,
+               wioc->rawinput.buffer + got,
+               want);
+        got += want;
+
+        if (want < iov[i].iov_len) {
+            break;
+        }
+    }
+
+    buffer_advance(&wioc->rawinput, got);
+    qio_channel_websock_set_watch(wioc);
+    return got;
+}
+
+
+static ssize_t qio_channel_websock_writev(QIOChannel *ioc,
+                                          const struct iovec *iov,
+                                          size_t niov,
+                                          int *fds,
+                                          size_t nfds,
+                                          Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    size_t i;
+    ssize_t done = 0;
+    ssize_t ret;
+
+    if (wioc->io_err) {
+        *errp = error_copy(wioc->io_err);
+        return -1;
+    }
+
+    if (wioc->io_eof) {
+        error_setg(errp, "%s", "Broken pipe");
+        return -1;
+    }
+
+    for (i = 0; i < niov; i++) {
+        size_t want = iov[i].iov_len;
+        if ((want + wioc->rawoutput.offset) > QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+            want = (QIO_CHANNEL_WEBSOCK_MAX_BUFFER - wioc->rawoutput.offset);
+        }
+        if (want == 0) {
+            goto done;
+        }
+
+        buffer_reserve(&wioc->rawoutput, want);
+        buffer_append(&wioc->rawoutput, iov[i].iov_base, want);
+        done += want;
+        if (want < iov[i].iov_len) {
+            break;
+        }
+    }
+
+ done:
+    ret = qio_channel_websock_write_wire(wioc, errp);
+    if (ret < 0 &&
+        ret != QIO_CHANNEL_ERR_BLOCK) {
+        qio_channel_websock_unset_watch(wioc);
+        return -1;
+    }
+
+    qio_channel_websock_set_watch(wioc);
+
+    if (done == 0) {
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    return done;
+}
+
+static int qio_channel_websock_set_blocking(QIOChannel *ioc,
+                                            bool enabled,
+                                            Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_blocking(wioc->master, enabled, errp);
+    return 0;
+}
+
+static void qio_channel_websock_set_delay(QIOChannel *ioc,
+                                          bool enabled)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_delay(tioc->master, enabled);
+}
+
+static void qio_channel_websock_set_cork(QIOChannel *ioc,
+                                         bool enabled)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_cork(tioc->master, enabled);
+}
+
+static int qio_channel_websock_shutdown(QIOChannel *ioc,
+                                        QIOChannelShutdown how,
+                                        Error **errp)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    return qio_channel_shutdown(tioc->master, how, errp);
+}
+
+static int qio_channel_websock_close(QIOChannel *ioc,
+                                     Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    return qio_channel_close(wioc->master, errp);
+}
+
+typedef struct QIOChannelWebsockSource QIOChannelWebsockSource;
+struct QIOChannelWebsockSource {
+    GSource parent;
+    QIOChannelWebsock *wioc;
+    GIOCondition condition;
+};
+
+static gboolean
+qio_channel_websock_source_prepare(GSource *source,
+                                   gint *timeout)
+{
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+    *timeout = -1;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return cond & wsource->condition;
+}
+
+static gboolean
+qio_channel_websock_source_check(GSource *source)
+{
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return cond & wsource->condition;
+}
+
+static gboolean
+qio_channel_websock_source_dispatch(GSource *source,
+                                    GSourceFunc callback,
+                                    gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return (*func)(QIO_CHANNEL(wsource->wioc),
+                   (cond & wsource->condition),
+                   user_data);
+}
+
+static void
+qio_channel_websock_source_finalize(GSource *source)
+{
+    QIOChannelWebsockSource *ssource = (QIOChannelWebsockSource *)source;
+
+    object_unref(OBJECT(ssource->wioc));
+}
+
+GSourceFuncs qio_channel_websock_source_funcs = {
+    qio_channel_websock_source_prepare,
+    qio_channel_websock_source_check,
+    qio_channel_websock_source_dispatch,
+    qio_channel_websock_source_finalize
+};
+
+static GSource *qio_channel_websock_create_watch(QIOChannel *ioc,
+                                                 GIOCondition condition)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    QIOChannelWebsockSource *ssource;
+    GSource *source;
+
+    source = g_source_new(&qio_channel_websock_source_funcs,
+                          sizeof(QIOChannelWebsockSource));
+    ssource = (QIOChannelWebsockSource *)source;
+
+    ssource->wioc = wioc;
+    object_ref(OBJECT(wioc));
+
+    ssource->condition = condition;
+
+    qio_channel_websock_set_watch(wioc);
+    return source;
+}
+
+static void qio_channel_websock_class_init(ObjectClass *klass,
+                                           void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_websock_writev;
+    ioc_klass->io_readv = qio_channel_websock_readv;
+    ioc_klass->io_set_blocking = qio_channel_websock_set_blocking;
+    ioc_klass->io_set_cork = qio_channel_websock_set_cork;
+    ioc_klass->io_set_delay = qio_channel_websock_set_delay;
+    ioc_klass->io_close = qio_channel_websock_close;
+    ioc_klass->io_shutdown = qio_channel_websock_shutdown;
+    ioc_klass->io_create_watch = qio_channel_websock_create_watch;
+}
+
+static const TypeInfo qio_channel_websock_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_WEBSOCK,
+    .instance_size = sizeof(QIOChannelWebsock),
+    .instance_finalize = qio_channel_websock_finalize,
+    .class_init = qio_channel_websock_class_init,
+};
+
+static void qio_channel_websock_register_types(void)
+{
+    type_register_static(&qio_channel_websock_info);
+}
+
+type_init(qio_channel_websock_register_types);
diff --git a/qemu/io/channel.c b/qemu/io/channel.c
new file mode 100644
index 000000000..692eb179b
--- /dev/null
+++ b/qemu/io/channel.c
@@ -0,0 +1,307 @@
+/*
+ * QEMU I/O channels
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel.h"
+#include "qapi/error.h"
+#include "qemu/coroutine.h"
+
+bool qio_channel_has_feature(QIOChannel *ioc,
+                             QIOChannelFeature feature)
+{
+    return ioc->features & (1 << feature);
+}
+
+
+ssize_t qio_channel_readv_full(QIOChannel *ioc,
+                               const struct iovec *iov,
+                               size_t niov,
+                               int **fds,
+                               size_t *nfds,
+                               Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if ((fds || nfds) &&
+        !(ioc->features & (1 << QIO_CHANNEL_FEATURE_FD_PASS))) {
+        error_setg_errno(errp, EINVAL,
+                         "Channel does not support file descriptor passing");
+        return -1;
+    }
+
+    return klass->io_readv(ioc, iov, niov, fds, nfds, errp);
+}
+
+
+ssize_t qio_channel_writev_full(QIOChannel *ioc,
+                                const struct iovec *iov,
+                                size_t niov,
+                                int *fds,
+                                size_t nfds,
+                                Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if ((fds || nfds) &&
+        !(ioc->features & (1 << QIO_CHANNEL_FEATURE_FD_PASS))) {
+        error_setg_errno(errp, EINVAL,
+                         "Channel does not support file descriptor passing");
+        return -1;
+    }
+
+    return klass->io_writev(ioc, iov, niov, fds, nfds, errp);
+}
+
+
+ssize_t qio_channel_readv(QIOChannel *ioc,
+                          const struct iovec *iov,
+                          size_t niov,
+                          Error **errp)
+{
+    return qio_channel_readv_full(ioc, iov, niov, NULL, NULL, errp);
+}
+
+
+ssize_t qio_channel_writev(QIOChannel *ioc,
+                           const struct iovec *iov,
+                           size_t niov,
+                           Error **errp)
+{
+    return qio_channel_writev_full(ioc, iov, niov, NULL, 0, errp);
+}
+
+
+ssize_t qio_channel_read(QIOChannel *ioc,
+                         char *buf,
+                         size_t buflen,
+                         Error **errp)
+{
+    struct iovec iov = { .iov_base = buf, .iov_len = buflen };
+    return qio_channel_readv_full(ioc, &iov, 1, NULL, NULL, errp);
+}
+
+
+ssize_t qio_channel_write(QIOChannel *ioc,
+                          const char *buf,
+                          size_t buflen,
+                          Error **errp)
+{
+    struct iovec iov = { .iov_base = (char *)buf, .iov_len = buflen };
+    return qio_channel_writev_full(ioc, &iov, 1, NULL, 0, errp);
+}
+
+
+int qio_channel_set_blocking(QIOChannel *ioc,
+                              bool enabled,
+                              Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_set_blocking(ioc, enabled, errp);
+}
+
+
+int qio_channel_close(QIOChannel *ioc,
+                      Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_close(ioc, errp);
+}
+
+
+GSource *qio_channel_create_watch(QIOChannel *ioc,
+                                  GIOCondition condition)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_create_watch(ioc, condition);
+}
+
+
+guint qio_channel_add_watch(QIOChannel *ioc,
+                            GIOCondition condition,
+                            QIOChannelFunc func,
+                            gpointer user_data,
+                            GDestroyNotify notify)
+{
+    GSource *source;
+    guint id;
+
+    source = qio_channel_create_watch(ioc, condition);
+
+    g_source_set_callback(source, (GSourceFunc)func, user_data, notify);
+
+    id = g_source_attach(source, NULL);
+    g_source_unref(source);
+
+    return id;
+}
+
+
+int qio_channel_shutdown(QIOChannel *ioc,
+                         QIOChannelShutdown how,
+                         Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (!klass->io_shutdown) {
+        error_setg(errp, "Data path shutdown not supported");
+        return -1;
+    }
+
+    return klass->io_shutdown(ioc, how, errp);
+}
+
+
+void qio_channel_set_delay(QIOChannel *ioc,
+                           bool enabled)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (klass->io_set_delay) {
+        klass->io_set_delay(ioc, enabled);
+    }
+}
+
+
+void qio_channel_set_cork(QIOChannel *ioc,
+                          bool enabled)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (klass->io_set_cork) {
+        klass->io_set_cork(ioc, enabled);
+    }
+}
+
+
+off_t qio_channel_io_seek(QIOChannel *ioc,
+                          off_t offset,
+                          int whence,
+                          Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (!klass->io_seek) {
+        error_setg(errp, "Channel does not support random access");
+        return -1;
+    }
+
+    return klass->io_seek(ioc, offset, whence, errp);
+}
+
+
+typedef struct QIOChannelYieldData QIOChannelYieldData;
+struct QIOChannelYieldData {
+    QIOChannel *ioc;
+    Coroutine *co;
+};
+
+
+static gboolean qio_channel_yield_enter(QIOChannel *ioc,
+                                        GIOCondition condition,
+                                        gpointer opaque)
+{
+    QIOChannelYieldData *data = opaque;
+    qemu_coroutine_enter(data->co, NULL);
+    return FALSE;
+}
+
+
+void coroutine_fn qio_channel_yield(QIOChannel *ioc,
+                                    GIOCondition condition)
+{
+    QIOChannelYieldData data;
+
+    assert(qemu_in_coroutine());
+    data.ioc = ioc;
+    data.co = qemu_coroutine_self();
+    qio_channel_add_watch(ioc,
+                          condition,
+                          qio_channel_yield_enter,
+                          &data,
+                          NULL);
+    qemu_coroutine_yield();
+}
+
+
+static gboolean qio_channel_wait_complete(QIOChannel *ioc,
+                                          GIOCondition condition,
+                                          gpointer opaque)
+{
+    GMainLoop *loop = opaque;
+
+    g_main_loop_quit(loop);
+    return FALSE;
+}
+
+
+void qio_channel_wait(QIOChannel *ioc,
+                      GIOCondition condition)
+{
+    GMainContext *ctxt = g_main_context_new();
+    GMainLoop *loop = g_main_loop_new(ctxt, TRUE);
+    GSource *source;
+
+    source = qio_channel_create_watch(ioc, condition);
+
+    g_source_set_callback(source,
+                          (GSourceFunc)qio_channel_wait_complete,
+                          loop,
+                          NULL);
+
+    g_source_attach(source, ctxt);
+
+    g_main_loop_run(loop);
+
+    g_source_unref(source);
+    g_main_loop_unref(loop);
+    g_main_context_unref(ctxt);
+}
+
+
+#ifdef _WIN32
+static void qio_channel_finalize(Object *obj)
+{
+    QIOChannel *ioc = QIO_CHANNEL(obj);
+
+    if (ioc->event) {
+        CloseHandle(ioc->event);
+    }
+}
+#endif
+
+static const TypeInfo qio_channel_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QIO_CHANNEL,
+    .instance_size = sizeof(QIOChannel),
+#ifdef _WIN32
+    .instance_finalize = qio_channel_finalize,
+#endif
+    .abstract = true,
+    .class_size = sizeof(QIOChannelClass),
+};
+
+
+static void qio_channel_register_types(void)
+{
+    type_register_static(&qio_channel_info);
+}
+
+
+type_init(qio_channel_register_types);
diff --git a/qemu/io/task.c b/qemu/io/task.c
new file mode 100644
index 000000000..c7f97a9b1
--- /dev/null
+++ b/qemu/io/task.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU I/O task
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/task.h"
+#include "qapi/error.h"
+#include "qemu/thread.h"
+#include "trace.h"
+
+struct QIOTask {
+    Object *source;
+    QIOTaskFunc func;
+    gpointer opaque;
+    GDestroyNotify destroy;
+};
+
+
+QIOTask *qio_task_new(Object *source,
+                      QIOTaskFunc func,
+                      gpointer opaque,
+                      GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = g_new0(QIOTask, 1);
+
+    task->source = source;
+    object_ref(source);
+    task->func = func;
+    task->opaque = opaque;
+    task->destroy = destroy;
+
+    trace_qio_task_new(task, source, func, opaque);
+
+    return task;
+}
+
+static void qio_task_free(QIOTask *task)
+{
+    if (task->destroy) {
+        task->destroy(task->opaque);
+    }
+    object_unref(task->source);
+
+    g_free(task);
+}
+
+
+struct QIOTaskThreadData {
+    QIOTask *task;
+    QIOTaskWorker worker;
+    gpointer opaque;
+    GDestroyNotify destroy;
+    Error *err;
+    int ret;
+};
+
+
+static gboolean gio_task_thread_result(gpointer opaque)
+{
+    struct QIOTaskThreadData *data = opaque;
+
+    trace_qio_task_thread_result(data->task);
+    if (data->ret == 0) {
+        qio_task_complete(data->task);
+    } else {
+        qio_task_abort(data->task, data->err);
+    }
+
+    error_free(data->err);
+    if (data->destroy) {
+        data->destroy(data->opaque);
+    }
+
+    g_free(data);
+
+    return FALSE;
+}
+
+
+static gpointer qio_task_thread_worker(gpointer opaque)
+{
+    struct QIOTaskThreadData *data = opaque;
+
+    trace_qio_task_thread_run(data->task);
+    data->ret = data->worker(data->task, &data->err, data->opaque);
+    if (data->ret < 0 && data->err == NULL) {
+        error_setg(&data->err, "Task worker failed but did not set an error");
+    }
+
+    /* We're running in the background thread, and must only
+     * ever report the task results in the main event loop
+     * thread. So we schedule an idle callback to report
+     * the worker results
+     */
+    trace_qio_task_thread_exit(data->task);
+    g_idle_add(gio_task_thread_result, data);
+    return NULL;
+}
+
+
+void qio_task_run_in_thread(QIOTask *task,
+                            QIOTaskWorker worker,
+                            gpointer opaque,
+                            GDestroyNotify destroy)
+{
+    struct QIOTaskThreadData *data = g_new0(struct QIOTaskThreadData, 1);
+    QemuThread thread;
+
+    data->task = task;
+    data->worker = worker;
+    data->opaque = opaque;
+    data->destroy = destroy;
+
+    trace_qio_task_thread_start(task, worker, opaque);
+    qemu_thread_create(&thread,
+                       "io-task-worker",
+                       qio_task_thread_worker,
+                       data,
+                       QEMU_THREAD_DETACHED);
+}
+
+
+void qio_task_complete(QIOTask *task)
+{
+    task->func(task->source, NULL, task->opaque);
+    trace_qio_task_complete(task);
+    qio_task_free(task);
+}
+
+void qio_task_abort(QIOTask *task,
+                    Error *err)
+{
+    task->func(task->source, err, task->opaque);
+    trace_qio_task_abort(task);
+    qio_task_free(task);
+}
+
+
+Object *qio_task_get_source(QIOTask *task)
+{
+    object_ref(task->source);
+    return task->source;
+}
diff --git a/qemu/iohandler.c b/qemu/iohandler.c
index 826f713e9..f2fc8a9bd 100644
--- a/qemu/iohandler.c
+++ b/qemu/iohandler.c
@@ -22,7 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "block/aio.h"
@@ -32,111 +33,37 @@
 #include <sys/wait.h>
 #endif
 
-typedef struct IOHandlerRecord {
-    IOHandler *fd_read;
-    IOHandler *fd_write;
-    void *opaque;
-    QLIST_ENTRY(IOHandlerRecord) next;
-    int fd;
-    int pollfds_idx;
-    bool deleted;
-} IOHandlerRecord;
+/* This context runs on top of main loop. We can't reuse qemu_aio_context
+ * because iohandlers mustn't be polled by aio_poll(qemu_aio_context). */
+static AioContext *iohandler_ctx;
 
-static QLIST_HEAD(, IOHandlerRecord) io_handlers =
-    QLIST_HEAD_INITIALIZER(io_handlers);
-
-void qemu_set_fd_handler(int fd,
-                         IOHandler *fd_read,
-                         IOHandler *fd_write,
-                         void *opaque)
+static void iohandler_init(void)
 {
-    IOHandlerRecord *ioh;
-
-    assert(fd >= 0);
-
-    if (!fd_read && !fd_write) {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd) {
-                ioh->deleted = 1;
-                break;
-            }
-        }
-    } else {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd)
-                goto found;
-        }
-        ioh = g_malloc0(sizeof(IOHandlerRecord));
-        QLIST_INSERT_HEAD(&io_handlers, ioh, next);
-    found:
-        ioh->fd = fd;
-        ioh->fd_read = fd_read;
-        ioh->fd_write = fd_write;
-        ioh->opaque = opaque;
-        ioh->pollfds_idx = -1;
-        ioh->deleted = 0;
-        qemu_notify_event();
+    if (!iohandler_ctx) {
+        iohandler_ctx = aio_context_new(&error_abort);
     }
 }
 
-void qemu_iohandler_fill(GArray *pollfds)
+AioContext *iohandler_get_aio_context(void)
 {
-    IOHandlerRecord *ioh;
-
-    QLIST_FOREACH(ioh, &io_handlers, next) {
-        int events = 0;
+    iohandler_init();
+    return iohandler_ctx;
+}
 
-        if (ioh->deleted)
-            continue;
-        if (ioh->fd_read) {
-            events |= G_IO_IN | G_IO_HUP | G_IO_ERR;
-        }
-        if (ioh->fd_write) {
-            events |= G_IO_OUT | G_IO_ERR;
-        }
-        if (events) {
-            GPollFD pfd = {
-                .fd = ioh->fd,
-                .events = events,
-            };
-            ioh->pollfds_idx = pollfds->len;
-            g_array_append_val(pollfds, pfd);
-        } else {
-            ioh->pollfds_idx = -1;
-        }
-    }
+GSource *iohandler_get_g_source(void)
+{
+    iohandler_init();
+    return aio_get_g_source(iohandler_ctx);
 }
 
-void qemu_iohandler_poll(GArray *pollfds, int ret)
+void qemu_set_fd_handler(int fd,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque)
 {
-    if (ret > 0) {
-        IOHandlerRecord *pioh, *ioh;
-
-        QLIST_FOREACH_SAFE(ioh, &io_handlers, next, pioh) {
-            int revents = 0;
-
-            if (!ioh->deleted && ioh->pollfds_idx != -1) {
-                GPollFD *pfd = &g_array_index(pollfds, GPollFD,
-                                              ioh->pollfds_idx);
-                revents = pfd->revents;
-            }
-
-            if (!ioh->deleted && ioh->fd_read &&
-                (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR))) {
-                ioh->fd_read(ioh->opaque);
-            }
-            if (!ioh->deleted && ioh->fd_write &&
-                (revents & (G_IO_OUT | G_IO_ERR))) {
-                ioh->fd_write(ioh->opaque);
-            }
-
-            /* Do this last in case read/write handlers marked it for deletion */
-            if (ioh->deleted) {
-                QLIST_REMOVE(ioh, next);
-                g_free(ioh);
-            }
-        }
-    }
+    iohandler_init();
+    aio_set_fd_handler(iohandler_ctx, fd, false,
+                       fd_read, fd_write, opaque);
 }
 
 /* reaping of zombies.  right now we're not passing the status to
diff --git a/qemu/ioport.c b/qemu/ioport.c
index e39093edb..7a84d5444 100644
--- a/qemu/ioport.c
+++ b/qemu/ioport.c
@@ -25,19 +25,12 @@
  * splitted out ioport related stuffs from vl.c.
  */
 
+#include "qemu/osdep.h"
 #include "exec/ioport.h"
 #include "trace.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 
-//#define DEBUG_IOPORT
-
-#ifdef DEBUG_IOPORT
-#  define LOG_IOPORT(...) qemu_log_mask(CPU_LOG_IOPORT, ## __VA_ARGS__)
-#else
-#  define LOG_IOPORT(...) do { } while (0)
-#endif
-
 typedef struct MemoryRegionPortioList {
     MemoryRegion mr;
     void *portio_opaque;
@@ -62,8 +55,7 @@ const MemoryRegionOps unassigned_io_ops = {
 
 void cpu_outb(pio_addr_t addr, uint8_t val)
 {
-    LOG_IOPORT("outb: %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'b', val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         &val, 1);
 }
@@ -72,8 +64,7 @@ void cpu_outw(pio_addr_t addr, uint16_t val)
 {
     uint8_t buf[2];
 
-    LOG_IOPORT("outw: %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'w', val);
     stw_p(buf, val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         buf, 2);
@@ -83,8 +74,7 @@ void cpu_outl(pio_addr_t addr, uint32_t val)
 {
     uint8_t buf[4];
 
-    LOG_IOPORT("outl: %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'l', val);
     stl_p(buf, val);
     address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                         buf, 4);
@@ -96,8 +86,7 @@ uint8_t cpu_inb(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
                        &val, 1);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inb : %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
+    trace_cpu_in(addr, 'b', val);
     return val;
 }
 
@@ -108,8 +97,7 @@ uint16_t cpu_inw(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 2);
     val = lduw_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inw : %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
+    trace_cpu_in(addr, 'w', val);
     return val;
 }
 
@@ -120,8 +108,7 @@ uint32_t cpu_inl(pio_addr_t addr)
 
     address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 4);
     val = ldl_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inl : %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
+    trace_cpu_in(addr, 'l', val);
     return val;
 }
 
diff --git a/qemu/iothread.c b/qemu/iothread.c
index da6ce7b30..f183d380e 100644
--- a/qemu/iothread.c
+++ b/qemu/iothread.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "qom/object_interfaces.h"
 #include "qemu/module.h"
@@ -72,6 +73,7 @@ static void iothread_complete(UserCreatable *obj, Error **errp)
 {
     Error *local_error = NULL;
     IOThread *iothread = IOTHREAD(obj);
+    char *name, *thread_name;
 
     iothread->stopping = false;
     iothread->thread_id = -1;
@@ -87,8 +89,12 @@ static void iothread_complete(UserCreatable *obj, Error **errp)
     /* This assumes we are called from a thread with useful CPU affinity for us
      * to inherit.
      */
-    qemu_thread_create(&iothread->thread, "iothread", iothread_run,
+    name = object_get_canonical_path_component(OBJECT(obj));
+    thread_name = g_strdup_printf("IO %s", name);
+    qemu_thread_create(&iothread->thread, thread_name, iothread_run,
                        iothread, QEMU_THREAD_JOINABLE);
+    g_free(thread_name);
+    g_free(name);
 
     /* Wait for initialization to complete */
     qemu_mutex_lock(&iothread->init_done_lock);
diff --git a/qemu/kvm-all.c b/qemu/kvm-all.c
index 06e06f2b3..e7b66df19 100644
--- a/qemu/kvm-all.c
+++ b/qemu/kvm-all.c
@@ -13,10 +13,9 @@
  *
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
-#include <stdarg.h>
 
 #include <linux/kvm.h>
 
@@ -24,6 +23,7 @@
 #include "qemu/atomic.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/s390x/adapter.h"
@@ -44,8 +44,10 @@
 #include <sys/eventfd.h>
 #endif
 
-/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
-#define PAGE_SIZE TARGET_PAGE_SIZE
+/* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
+ * need to use the real host PAGE_SIZE, as that's what KVM will use.
+ */
+#define PAGE_SIZE getpagesize()
 
 //#define DEBUG_KVM
 
@@ -76,8 +78,6 @@ struct KVMState
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
-    int pit_state2;
-    int xsave, xcrs;
     int many_ioeventfds;
     int intx_set_mask;
     /* The man page (and posix) say ioctl numbers are signed int, but
@@ -89,16 +89,16 @@ struct KVMState
 #ifdef KVM_CAP_IRQ_ROUTING
     struct kvm_irq_routing *irq_routes;
     int nr_allocated_irq_routes;
-    uint32_t *used_gsi_bitmap;
+    unsigned long *used_gsi_bitmap;
     unsigned int gsi_count;
     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
-    bool direct_msi;
 #endif
     KVMMemoryListener memory_listener;
 };
 
 KVMState *kvm_state;
 bool kvm_kernel_irqchip;
+bool kvm_split_irqchip;
 bool kvm_async_interrupts_allowed;
 bool kvm_halt_in_kernel_allowed;
 bool kvm_eventfds_allowed;
@@ -110,6 +110,8 @@ bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
 bool kvm_vm_attributes_allowed;
+bool kvm_direct_msi_allowed;
+bool kvm_ioeventfd_any_length_allowed;
 
 static const KVMCapabilityInfo kvm_required_capabilites[] = {
     KVM_CAP_INFO(USER_MEMORY),
@@ -364,7 +366,8 @@ static void kvm_log_stop(MemoryListener *listener,
 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
                                          unsigned long *bitmap)
 {
-    ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
+    ram_addr_t start = section->offset_within_region +
+                       memory_region_get_ram_addr(section->mr);
     ram_addr_t pages = int128_get64(section->size) / getpagesize();
 
     cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
@@ -641,15 +644,15 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
     /* kvm works in page size chunks, but the function may be called
        with sub-page size and unaligned start address. Pad the start
        address to next and truncate size to previous page boundary. */
-    delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
-    delta &= ~TARGET_PAGE_MASK;
+    delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
+    delta &= ~qemu_real_host_page_mask;
     if (delta > size) {
         return;
     }
     start_addr += delta;
     size -= delta;
-    size &= TARGET_PAGE_MASK;
-    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
+    size &= qemu_real_host_page_mask;
+    if (!size || (start_addr & ~qemu_real_host_page_mask)) {
         return;
     }
 
@@ -948,12 +951,12 @@ typedef struct KVMMSIRoute {
 
 static void set_gsi(KVMState *s, unsigned int gsi)
 {
-    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
+    set_bit(gsi, s->used_gsi_bitmap);
 }
 
 static void clear_gsi(KVMState *s, unsigned int gsi)
 {
-    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
+    clear_bit(gsi, s->used_gsi_bitmap);
 }
 
 void kvm_init_irq_routing(KVMState *s)
@@ -962,23 +965,15 @@ void kvm_init_irq_routing(KVMState *s)
 
     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
     if (gsi_count > 0) {
-        unsigned int gsi_bits, i;
-
         /* Round up so we can search ints using ffs */
-        gsi_bits = ALIGN(gsi_count, 32);
-        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
+        s->used_gsi_bitmap = bitmap_new(gsi_count);
         s->gsi_count = gsi_count;
-
-        /* Mark any over-allocated bits as already in use */
-        for (i = gsi_count; i < gsi_bits; i++) {
-            set_gsi(s, i);
-        }
     }
 
     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
     s->nr_allocated_irq_routes = 0;
 
-    if (!s->direct_msi) {
+    if (!kvm_direct_msi_allowed) {
         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
             QTAILQ_INIT(&s->msi_hashtab[i]);
         }
@@ -1102,9 +1097,7 @@ static void kvm_flush_dynamic_msi_routes(KVMState *s)
 
 static int kvm_irqchip_get_virq(KVMState *s)
 {
-    uint32_t *word = s->used_gsi_bitmap;
-    int max_words = ALIGN(s->gsi_count, 32) / 32;
-    int i, zeroes;
+    int next_virq;
 
     /*
      * PIC and IOAPIC share the first 16 GSI numbers, thus the available
@@ -1112,21 +1105,17 @@ static int kvm_irqchip_get_virq(KVMState *s)
      * number can succeed even though a new route entry cannot be added.
      * When this happens, flush dynamic MSI entries to free IRQ route entries.
      */
-    if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) {
+    if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
         kvm_flush_dynamic_msi_routes(s);
     }
 
     /* Return the lowest unused GSI in the bitmap */
-    for (i = 0; i < max_words; i++) {
-        zeroes = ctz32(~word[i]);
-        if (zeroes == 32) {
-            continue;
-        }
-
-        return zeroes + i * 32;
+    next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
+    if (next_virq >= s->gsi_count) {
+        return -ENOSPC;
+    } else {
+        return next_virq;
     }
-    return -ENOSPC;
-
 }
 
 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
@@ -1149,7 +1138,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
     struct kvm_msi msi;
     KVMMSIRoute *route;
 
-    if (s->direct_msi) {
+    if (kvm_direct_msi_allowed) {
         msi.address_lo = (uint32_t)msg.address;
         msi.address_hi = msg.address >> 32;
         msi.data = le32_to_cpu(msg.data);
@@ -1188,7 +1177,7 @@ int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
     return kvm_set_irq(s, route->kroute.gsi, 1);
 }
 
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
     int virq;
@@ -1212,7 +1201,7 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
-    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
         kvm_irqchip_release_virq(s, virq);
         return -EINVAL;
     }
@@ -1223,7 +1212,8 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     return virq;
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
 
@@ -1241,7 +1231,7 @@ int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
-    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data)) {
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
         return -EINVAL;
     }
 
@@ -1293,6 +1283,33 @@ int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
     kroute.u.adapter.adapter_id = adapter->adapter_id;
 
     kvm_add_routing_entry(s, &kroute);
+
+    return virq;
+}
+
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
+{
+    struct kvm_irq_routing_entry kroute = {};
+    int virq;
+
+    if (!kvm_gsi_routing_enabled()) {
+        return -ENOSYS;
+    }
+    if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) {
+        return -ENOSYS;
+    }
+    virq = kvm_irqchip_get_virq(s);
+    if (virq < 0) {
+        return virq;
+    }
+
+    kroute.gsi = virq;
+    kroute.type = KVM_IRQ_ROUTING_HV_SINT;
+    kroute.flags = 0;
+    kroute.u.hv_sint.vcpu = vcpu;
+    kroute.u.hv_sint.sint = sint;
+
+    kvm_add_routing_entry(s, &kroute);
     kvm_irqchip_commit_routes(s);
 
     return virq;
@@ -1323,6 +1340,11 @@ int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
     return -ENOSYS;
 }
 
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
+{
+    return -ENOSYS;
+}
+
 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     abort();
@@ -1395,9 +1417,14 @@ static void kvm_irqchip_create(MachineState *machine, KVMState *s)
 
     /* First probe and see if there's a arch-specific hook to create the
      * in-kernel irqchip for us */
-    ret = kvm_arch_irqchip_create(s);
+    ret = kvm_arch_irqchip_create(machine, s);
     if (ret == 0) {
-        ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
+        if (machine_kernel_irqchip_split(machine)) {
+            perror("Split IRQ chip mode not supported.");
+            exit(1);
+        } else {
+            ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
+        }
     }
     if (ret < 0) {
         fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
@@ -1462,7 +1489,6 @@ static int kvm_init(MachineState *ms)
      * page size for the system though.
      */
     assert(TARGET_PAGE_SIZE <= getpagesize());
-    page_size_init();
 
     s->sigmask_len = 8;
 
@@ -1585,20 +1611,8 @@ static int kvm_init(MachineState *ms)
     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 #endif
 
-#ifdef KVM_CAP_XSAVE
-    s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
-#endif
-
-#ifdef KVM_CAP_XCRS
-    s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
-#endif
-
-#ifdef KVM_CAP_PIT_STATE2
-    s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
-#endif
-
 #ifdef KVM_CAP_IRQ_ROUTING
-    s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+    kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
 #endif
 
     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
@@ -1625,6 +1639,9 @@ static int kvm_init(MachineState *ms)
     kvm_vm_attributes_allowed =
         (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
 
+    kvm_ioeventfd_any_length_allowed =
+        (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
+
     ret = kvm_arch_init(ms, s);
     if (ret < 0) {
         goto err;
@@ -1636,8 +1653,10 @@ static int kvm_init(MachineState *ms)
 
     kvm_state = s;
 
-    s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
-    s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
+    if (kvm_eventfds_allowed) {
+        s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
+        s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
+    }
     s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
     s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
 
@@ -1779,11 +1798,6 @@ void kvm_cpu_synchronize_post_init(CPUState *cpu)
     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
 }
 
-void kvm_cpu_clean_state(CPUState *cpu)
-{
-    cpu->kvm_vcpu_dirty = false;
-}
-
 int kvm_cpu_exec(CPUState *cpu)
 {
     struct kvm_run *run = cpu->kvm_run;
@@ -1890,6 +1904,12 @@ int kvm_cpu_exec(CPUState *cpu)
                 qemu_system_reset_request();
                 ret = EXCP_INTERRUPT;
                 break;
+            case KVM_SYSTEM_EVENT_CRASH:
+                qemu_mutex_lock_iothread();
+                qemu_system_guest_panicked();
+                qemu_mutex_unlock_iothread();
+                ret = 0;
+                break;
             default:
                 DPRINTF("kvm_arch_handle_exit\n");
                 ret = kvm_arch_handle_exit(cpu, run);
@@ -2003,6 +2023,39 @@ int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
     return ret ? 0 : 1;
 }
 
+int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
+{
+    struct kvm_device_attr attribute = {
+        .group = group,
+        .attr = attr,
+        .flags = 0,
+    };
+
+    return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
+}
+
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write)
+{
+    struct kvm_device_attr kvmattr;
+    int err;
+
+    kvmattr.flags = 0;
+    kvmattr.group = group;
+    kvmattr.attr = attr;
+    kvmattr.addr = (uintptr_t)val;
+
+    err = kvm_device_ioctl(fd,
+                           write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
+                           &kvmattr);
+    if (err < 0) {
+        error_report("KVM_%s_DEVICE_ATTR failed: %s",
+                     write ? "SET" : "GET", strerror(-err));
+        error_printf("Group %d attr 0x%016" PRIx64, group, attr);
+        abort();
+    }
+}
+
 int kvm_has_sync_mmu(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
@@ -2023,21 +2076,6 @@ int kvm_has_debugregs(void)
     return kvm_state->debugregs;
 }
 
-int kvm_has_xsave(void)
-{
-    return kvm_state->xsave;
-}
-
-int kvm_has_xcrs(void)
-{
-    return kvm_state->xcrs;
-}
-
-int kvm_has_pit_state2(void)
-{
-    return kvm_state->pit_state2;
-}
-
 int kvm_has_many_ioeventfds(void)
 {
     if (!kvm_enabled()) {
@@ -2301,6 +2339,21 @@ int kvm_create_device(KVMState *s, uint64_t type, bool test)
     return test ? 0 : create_dev.fd;
 }
 
+bool kvm_device_supported(int vmfd, uint64_t type)
+{
+    struct kvm_create_device create_dev = {
+        .type = type,
+        .fd = -1,
+        .flags = KVM_CREATE_DEVICE_TEST,
+    };
+
+    if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
+        return false;
+    }
+
+    return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
+}
+
 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
 {
     struct kvm_one_reg reg;
@@ -2310,7 +2363,7 @@ int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
     reg.addr = (uintptr_t) source;
     r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
     if (r) {
-        trace_kvm_failed_reg_set(id, strerror(r));
+        trace_kvm_failed_reg_set(id, strerror(-r));
     }
     return r;
 }
@@ -2324,7 +2377,7 @@ int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
     reg.addr = (uintptr_t) target;
     r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
     if (r) {
-        trace_kvm_failed_reg_get(id, strerror(r));
+        trace_kvm_failed_reg_get(id, strerror(-r));
     }
     return r;
 }
diff --git a/qemu/kvm-stub.c b/qemu/kvm-stub.c
index d9ad624ee..b962b2483 100644
--- a/qemu/kvm-stub.c
+++ b/qemu/kvm-stub.c
@@ -10,6 +10,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "cpu.h"
@@ -30,6 +31,7 @@ bool kvm_gsi_routing_allowed;
 bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
+bool kvm_ioeventfd_any_length_allowed;
 
 int kvm_init_vcpu(CPUState *cpu)
 {
@@ -67,11 +69,6 @@ int kvm_has_many_ioeventfds(void)
     return 0;
 }
 
-int kvm_has_pit_state2(void)
-{
-    return 0;
-}
-
 void kvm_setup_guest_memory(void *start, size_t size)
 {
 }
@@ -115,7 +112,7 @@ int kvm_on_sigbus(int code, void *addr)
 }
 
 #ifndef CONFIG_USER_ONLY
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg, PCIDevice *dev)
 {
     return -ENOSYS;
 }
@@ -128,7 +125,8 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
 {
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     return -ENOSYS;
 }
diff --git a/qemu/libcacard/Makefile b/qemu/libcacard/Makefile
deleted file mode 100644
index b5eddff11..000000000
--- a/qemu/libcacard/Makefile
+++ /dev/null
@@ -1,45 +0,0 @@
-libcacard_includedir=$(includedir)/cacard
-
-TOOLS += vscclient$(EXESUF)
-
-# objects linked into a shared library, built with libtool with -fPIC if required
-libcacard-obj-y = $(libcacard-y)
-libcacard-lobj-y=$(patsubst %.o,%.lo,$(libcacard-obj-y))
-
-# libtool will build the .o files, too
-$(libcacard-obj-y): | $(libcacard-lobj-y)
-
-all: libcacard.la libcacard.pc
-
-vscclient$(EXESUF): libcacard/vscclient.o libcacard.la
-	$(call LINK,$^)
-
-#########################################################################
-# Rules for building libcacard standalone library
-
-libcacard.la: LDFLAGS += -rpath $(libdir) -no-undefined \
-	-export-symbols $(SRC_PATH)/libcacard/libcacard.syms
-# Prevent libcacard.so linking against the entire world of 3rd party libs
-libcacard.la: LIBS =
-libcacard.la: $(libcacard-lobj-y)
-	$(call LINK,$^)
-
-libcacard.pc: $(SRC_PATH)/libcacard/libcacard.pc.in
-	$(call quiet-command,sed -e 's|@LIBDIR@|$(libdir)|' \
-		-e 's|@INCLUDEDIR@|$(libcacard_includedir)|' \
-	    -e 's|@VERSION@|$(shell cat $(SRC_PATH)/VERSION)|' \
-		-e 's|@PREFIX@|$(prefix)|' $< > libcacard.pc,\
-	"  GEN   $@")
-
-.PHONY: install-libcacard
-
-install: install-libcacard
-install-libcacard: libcacard.pc libcacard.la
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)"
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)/pkgconfig"
-	$(INSTALL_DIR) "$(DESTDIR)$(libcacard_includedir)"
-	$(INSTALL_LIB) libcacard.la "$(DESTDIR)$(libdir)"
-	$(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
-	for inc in $(SRC_PATH)/libcacard/*.h; do \
-		$(INSTALL_DATA) $$inc "$(DESTDIR)$(libcacard_includedir)"; \
-	done
diff --git a/qemu/libcacard/cac.c b/qemu/libcacard/cac.c
deleted file mode 100644
index bc84534f9..000000000
--- a/qemu/libcacard/cac.c
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * implement the applets for the CAC card.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-#include <stdbool.h>
-
-#include "cac.h"
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/* private data for PKI applets */
-typedef struct CACPKIAppletDataStruct {
-    unsigned char *cert;
-    int cert_len;
-    unsigned char *cert_buffer;
-    int cert_buffer_len;
-    unsigned char *sign_buffer;
-    int sign_buffer_len;
-    VCardKey *key;
-} CACPKIAppletData;
-
-/*
- * CAC applet private data
- */
-struct VCardAppletPrivateStruct {
-    union {
-        CACPKIAppletData pki_data;
-        void *reserved;
-    } u;
-};
-
-/*
- * handle all the APDU's that are common to all CAC applets
- */
-static VCardStatus
-cac_common_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    int ef;
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x02) {
-            /* let the 7816 code handle applet switches */
-            ret = VCARD_NEXT;
-            break;
-        }
-        /* handle file id setting */
-        if (apdu->a_Lc != 2) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_DATA_INVALID);
-            ret = VCARD_DONE;
-            break;
-        }
-        /* CAC 1.0 only supports ef = 0 */
-        ef = apdu->a_body[0] | (apdu->a_body[1] << 8);
-        if (ef != 0) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-            ret = VCARD_DONE;
-            break;
-        }
-        *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-        ret = VCARD_DONE;
-        break;
-    case VCARD7816_INS_GET_RESPONSE:
-    case VCARD7816_INS_VERIFY:
-        /* let the 7816 code handle these */
-        ret = VCARD_NEXT;
-        break;
-    case CAC_GET_PROPERTIES:
-    case CAC_GET_ACR:
-        /* skip these for now, this will probably be needed */
-        *response = vcard_make_response(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-        ret = VCARD_DONE;
-        break;
-    default:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    }
-    return ret;
-}
-
-/*
- *  reset the inter call state between applet selects
- */
-static VCardStatus
-cac_applet_pki_reset(VCard *card, int channel)
-{
-    VCardAppletPrivate *applet_private;
-    CACPKIAppletData *pki_applet;
-    applet_private = vcard_get_current_applet_private(card, channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    pki_applet->cert_buffer = NULL;
-    g_free(pki_applet->sign_buffer);
-    pki_applet->sign_buffer = NULL;
-    pki_applet->cert_buffer_len = 0;
-    pki_applet->sign_buffer_len = 0;
-    return VCARD_DONE;
-}
-
-static VCardStatus
-cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
-                            VCardResponse **response)
-{
-    CACPKIAppletData *pki_applet;
-    VCardAppletPrivate *applet_private;
-    int size, next;
-    unsigned char *sign_buffer;
-    bool retain_sign_buffer = FALSE;
-    vcard_7816_status_t status;
-    VCardStatus ret = VCARD_FAIL;
-
-    applet_private = vcard_get_current_applet_private(card, apdu->a_channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_GET_CERTIFICATE:
-        if ((apdu->a_p2 != 0) || (apdu->a_p1 != 0)) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        assert(pki_applet->cert != NULL);
-        size = apdu->a_Le;
-        if (pki_applet->cert_buffer == NULL) {
-            pki_applet->cert_buffer = pki_applet->cert;
-            pki_applet->cert_buffer_len = pki_applet->cert_len;
-        }
-        size = MIN(size, pki_applet->cert_buffer_len);
-        next = MIN(255, pki_applet->cert_buffer_len - size);
-        *response = vcard_response_new_bytes(
-                        card, pki_applet->cert_buffer, size,
-                        apdu->a_Le, next ?
-                        VCARD7816_SW1_WARNING_CHANGE :
-                        VCARD7816_SW1_SUCCESS,
-                        next);
-        pki_applet->cert_buffer += size;
-        pki_applet->cert_buffer_len -= size;
-        if ((*response == NULL) || (next == 0)) {
-            pki_applet->cert_buffer = NULL;
-        }
-        if (*response == NULL) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_SIGN_DECRYPT:
-        if (apdu->a_p2 != 0) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        size = apdu->a_Lc;
-
-        sign_buffer = g_realloc(pki_applet->sign_buffer,
-                                pki_applet->sign_buffer_len + size);
-        memcpy(sign_buffer+pki_applet->sign_buffer_len, apdu->a_body, size);
-        size += pki_applet->sign_buffer_len;
-        switch (apdu->a_p1) {
-        case  0x80:
-            /* p1 == 0x80 means we haven't yet sent the whole buffer, wait for
-             * the rest */
-            pki_applet->sign_buffer = sign_buffer;
-            pki_applet->sign_buffer_len = size;
-            *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-            retain_sign_buffer = TRUE;
-            break;
-        case 0x00:
-            /* we now have the whole buffer, do the operation, result will be
-             * in the sign_buffer */
-            status = vcard_emul_rsa_op(card, pki_applet->key,
-                                       sign_buffer, size);
-            if (status != VCARD7816_STATUS_SUCCESS) {
-                *response = vcard_make_response(status);
-                break;
-            }
-            *response = vcard_response_new(card, sign_buffer, size, apdu->a_Le,
-                                                     VCARD7816_STATUS_SUCCESS);
-            if (*response == NULL) {
-                *response = vcard_make_response(
-                                VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-            }
-            break;
-        default:
-           *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        if (!retain_sign_buffer) {
-            g_free(sign_buffer);
-            pki_applet->sign_buffer = NULL;
-            pki_applet->sign_buffer_len = 0;
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-static VCardStatus
-cac_applet_id_process_apdu(VCard *card, VCardAPDU *apdu,
-                           VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-/*
- * TODO: if we ever want to support general CAC middleware, we will need to
- * implement the various containers.
- */
-static VCardStatus
-cac_applet_container_process_apdu(VCard *card, VCardAPDU *apdu,
-                                  VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_READ_BUFFER:
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-/*
- * utilities for creating and destroying the private applet data
- */
-static void
-cac_delete_pki_applet_private(VCardAppletPrivate *applet_private)
-{
-    CACPKIAppletData *pki_applet_data;
-
-    if (applet_private == NULL) {
-        return;
-    }
-    pki_applet_data = &(applet_private->u.pki_data);
-    g_free(pki_applet_data->cert);
-    g_free(pki_applet_data->sign_buffer);
-    if (pki_applet_data->key != NULL) {
-        vcard_emul_delete_key(pki_applet_data->key);
-    }
-    g_free(applet_private);
-}
-
-static VCardAppletPrivate *
-cac_new_pki_applet_private(const unsigned char *cert,
-                           int cert_len, VCardKey *key)
-{
-    CACPKIAppletData *pki_applet_data;
-    VCardAppletPrivate *applet_private;
-
-    applet_private = g_new0(VCardAppletPrivate, 1);
-    pki_applet_data = &(applet_private->u.pki_data);
-    pki_applet_data->cert = (unsigned char *)g_malloc(cert_len+1);
-    /*
-     * if we want to support compression, then we simply change the 0 to a 1
-     * and compress the cert data with libz
-     */
-    pki_applet_data->cert[0] = 0; /* not compressed */
-    memcpy(&pki_applet_data->cert[1], cert, cert_len);
-    pki_applet_data->cert_len = cert_len+1;
-
-    pki_applet_data->key = key;
-    return applet_private;
-}
-
-
-/*
- * create a new cac applet which links to a given cert
- */
-static VCardApplet *
-cac_new_pki_applet(int i, const unsigned char *cert,
-                   int cert_len, VCardKey *key)
-{
-    VCardAppletPrivate *applet_private;
-    VCardApplet *applet;
-    unsigned char pki_aid[] = { 0xa0, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00 };
-    int pki_aid_len = sizeof(pki_aid);
-
-    pki_aid[pki_aid_len-1] = i;
-
-    applet_private = cac_new_pki_applet_private(cert, cert_len, key);
-    if (applet_private == NULL) {
-        goto failure;
-    }
-    applet = vcard_new_applet(cac_applet_pki_process_apdu, cac_applet_pki_reset,
-                              pki_aid, pki_aid_len);
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_set_applet_private(applet, applet_private,
-                             cac_delete_pki_applet_private);
-    applet_private = NULL;
-
-    return applet;
-
-failure:
-    if (applet_private != NULL) {
-        cac_delete_pki_applet_private(applet_private);
-    }
-    return NULL;
-}
-
-
-static unsigned char cac_default_container_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00 };
-static unsigned char cac_id_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x79, 0x03, 0x00 };
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus
-cac_card_init(VReader *reader, VCard *card,
-              const char *params,
-              unsigned char * const *cert,
-              int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count)
-{
-    int i;
-    VCardApplet *applet;
-
-    /* CAC Cards are VM Cards */
-    vcard_set_type(card, VCARD_VM);
-
-    /* create one PKI applet for each cert */
-    for (i = 0; i < cert_count; i++) {
-        applet = cac_new_pki_applet(i, cert[i], cert_len[i], key[i]);
-        if (applet == NULL) {
-            goto failure;
-        }
-        vcard_add_applet(card, applet);
-    }
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_container_process_apdu,
-                              NULL, cac_default_container_aid,
-                              sizeof(cac_default_container_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_id_process_apdu,
-                              NULL, cac_id_aid,
-                              sizeof(cac_id_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-    return VCARD_DONE;
-
-failure:
-    return VCARD_FAIL;
-}
-
diff --git a/qemu/libcacard/cac.h b/qemu/libcacard/cac.h
deleted file mode 100644
index d24a2a846..000000000
--- a/qemu/libcacard/cac.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * defines the entry point for the cac card. Only used by cac.c anc
- * vcard_emul_type.c
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CAC_H
-#define CAC_H 1
-#include "vcard.h"
-#include "vreader.h"
-
-#define CAC_GET_PROPERTIES  0x56
-#define CAC_GET_ACR         0x4c
-#define CAC_READ_BUFFER     0x52
-#define CAC_UPDATE_BUFFER   0x58
-#define CAC_SIGN_DECRYPT    0x42
-#define CAC_GET_CERTIFICATE 0x36
-
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus cac_card_init(VReader *reader, VCard *card, const char *params,
-              unsigned char * const *cert, int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count);
-
-/* not yet implemented */
-VCardStatus cac_is_cac_card(VReader *reader);
-#endif
diff --git a/qemu/libcacard/card_7816.c b/qemu/libcacard/card_7816.c
deleted file mode 100644
index 22fd334d1..000000000
--- a/qemu/libcacard/card_7816.c
+++ /dev/null
@@ -1,757 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/*
- * set the status bytes based on the status word
- */
-static void
-vcard_response_set_status(VCardResponse *response, vcard_7816_status_t status)
-{
-    unsigned char sw1, sw2;
-    response->b_status = status; /* make sure the status and swX representations
-                                  * are consistent */
-    sw1 = (status >> 8) & 0xff;
-    sw2 = status & 0xff;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * set the status bytes in a response buffer
- */
-static void
-vcard_response_set_status_bytes(VCardResponse *response,
-                               unsigned char sw1, unsigned char sw2)
-{
-    response->b_status = sw1 << 8 | sw2;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * allocate a VCardResponse structure, plus space for the data buffer, and
- * set up everything but the resonse bytes.
- */
-VCardResponse *
-vcard_response_new_data(unsigned char *buf, int len)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = g_malloc(len + 2);
-    memcpy(new_response->b_data, buf, len);
-    new_response->b_total_len = len+2;
-    new_response->b_len = len;
-    new_response->b_type = VCARD_MALLOC;
-    return new_response;
-}
-
-static VCardResponse *
-vcard_init_buffer_response(VCard *card, unsigned char *buf, int len)
-{
-    VCardResponse *response;
-    VCardBufferResponse *buffer_response;
-
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response) {
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-    }
-    buffer_response = vcard_buffer_response_new(buf, len);
-    if (buffer_response == NULL) {
-        return NULL;
-    }
-    response = vcard_response_new_status_bytes(VCARD7816_SW1_RESPONSE_BYTES,
-                                               len > 255 ? 0 : len);
-    if (response == NULL) {
-        return NULL;
-    }
-    vcard_set_buffer_response(card, buffer_response);
-    return response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new(VCard *card, unsigned char *buf,
-                   int len, int Le, vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new_bytes(VCard *card, unsigned char *buf, int len, int Le,
-                         unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-/*
- * get a new Response buffer that only has a status.
- */
-static VCardResponse *
-vcard_response_new_status(vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * same as above, but specify the status as separate bytes
- */
-VCardResponse *
-vcard_response_new_status_bytes(unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-
-/*
- * free the response buffer. The Buffer has a type to handle the buffer
- * allocated in other ways than through malloc.
- */
-void
-vcard_response_delete(VCardResponse *response)
-{
-    if (response == NULL) {
-        return;
-    }
-    switch (response->b_type) {
-    case VCARD_MALLOC:
-        /* everything was malloc'ed */
-        g_free(response->b_data);
-        g_free(response);
-        break;
-    case VCARD_MALLOC_DATA:
-        /* only the data buffer was malloc'ed */
-        g_free(response->b_data);
-        break;
-    case VCARD_MALLOC_STRUCT:
-        /* only the structure was malloc'ed */
-        g_free(response);
-        break;
-    case VCARD_STATIC:
-        break;
-    }
-}
-
-/*
- * decode the class bit and set our generic type field, channel, and
- * secure messaging values.
- */
-static vcard_7816_status_t
-vcard_apdu_set_class(VCardAPDU *apdu) {
-    apdu->a_channel = 0;
-    apdu->a_secure_messaging = 0;
-    apdu->a_type = apdu->a_cla & 0xf0;
-    apdu->a_gen_type = VCARD_7816_ISO;
-
-    /* parse the class  tables 8 & 9 of the 7816-4 Part 4 spec */
-    switch (apdu->a_type) {
-        /* we only support the basic types */
-    case 0x00:
-    case 0x80:
-    case 0x90:
-    case 0xa0:
-        apdu->a_channel = apdu->a_cla & 3;
-        apdu->a_secure_messaging = apdu->a_cla & 0xe;
-        break;
-    case 0xb0:
-    case 0xc0:
-        break;
-
-    case 0x10:
-    case 0x20:
-    case 0x30:
-    case 0x40:
-    case 0x50:
-    case 0x60:
-    case 0x70:
-        /* Reserved for future use */
-        apdu->a_gen_type = VCARD_7816_RFU;
-        break;
-    case 0xd0:
-    case 0xe0:
-    case 0xf0:
-    default:
-        apdu->a_gen_type =
-            (apdu->a_cla == 0xff) ? VCARD_7816_PTS : VCARD_7816_PROPRIETARY;
-        break;
-    }
-    return VCARD7816_STATUS_SUCCESS;
-}
-
-/*
- * set the Le and Lc fields according to table 5 of the
- * 7816-4 part 4 spec
- */
-static vcard_7816_status_t
-vcard_apdu_set_length(VCardAPDU *apdu)
-{
-    int L, Le;
-
-    /* process according to table 5 of the 7816-4 Part 4 spec.
-     * variable names match the variables in the spec */
-    L = apdu->a_len-4; /* fixed APDU header */
-    apdu->a_Lc = 0;
-    apdu->a_Le = 0;
-    apdu->a_body = NULL;
-    switch (L) {
-    case 0:
-        /* 1 minimal apdu */
-        return VCARD7816_STATUS_SUCCESS;
-    case 1:
-        /* 2S only return values apdu */
-        /*   zero maps to 256 here */
-        apdu->a_Le = apdu->a_header->ah_Le ?
-                         apdu->a_header->ah_Le : 256;
-        return VCARD7816_STATUS_SUCCESS;
-    default:
-        /* if the ah_Le byte is zero and we have more than
-         * 1 byte in the header, then we must be using extended Le and Lc.
-         * process the extended now. */
-        if (apdu->a_header->ah_Le == 0) {
-            if (L < 3) {
-                /* coding error, need at least 3 bytes */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* calculate the first extended value. Could be either Le or Lc */
-            Le = (apdu->a_header->ah_body[0] << 8)
-               || apdu->a_header->ah_body[1];
-            if (L == 3) {
-                /* 2E extended, return data only */
-                /*   zero maps to 65536 */
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (Le == 0) {
-                /* reserved for future use, probably for next time we need
-                 * to extend the lengths */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* we know that the first extended value is Lc now */
-            apdu->a_Lc = Le;
-            apdu->a_body = &apdu->a_header->ah_body[2];
-            if (L == Le+3) {
-                /* 3E extended, only body parameters */
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (L == Le+5) {
-                /* 4E extended, parameters and return data */
-                Le = (apdu->a_data[apdu->a_len-2] << 8)
-                   || apdu->a_data[apdu->a_len-1];
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        }
-        /* not extended */
-        apdu->a_Lc = apdu->a_header->ah_Le;
-        apdu->a_body = &apdu->a_header->ah_body[0];
-        if (L ==  apdu->a_Lc + 1) {
-            /* 3S only body parameters */
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        if (L ==  apdu->a_Lc + 2) {
-            /* 4S parameters and return data */
-            Le = apdu->a_data[apdu->a_len-1];
-            apdu->a_Le = Le ?  Le : 256;
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        break;
-    }
-    return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-}
-
-/*
- * create a new APDU from a raw set of bytes. This will decode all the
- * above fields. users of VCARDAPDU's can then depend on the already decoded
- * values.
- */
-VCardAPDU *
-vcard_apdu_new(unsigned char *raw_apdu, int len, vcard_7816_status_t *status)
-{
-    VCardAPDU *new_apdu;
-
-    *status = VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    if (len < 4) {
-        *status = VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        return NULL;
-    }
-
-    new_apdu = g_new(VCardAPDU, 1);
-    new_apdu->a_data = g_memdup(raw_apdu, len);
-    new_apdu->a_len = len;
-    *status = vcard_apdu_set_class(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        return NULL;
-    }
-    *status = vcard_apdu_set_length(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        new_apdu = NULL;
-    }
-    return new_apdu;
-}
-
-void
-vcard_apdu_delete(VCardAPDU *apdu)
-{
-    if (apdu == NULL) {
-        return;
-    }
-    g_free(apdu->a_data);
-    g_free(apdu);
-}
-
-
-/*
- * declare response buffers for all the 7816 defined error codes
- */
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_SUCCESS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_RET_CORUPT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FILE_FILLED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_LENGTH)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NO_EF)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FILE_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_INS_CODE_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_GENERAL)
-
-/*
- * return a single response code. This function cannot fail. It will always
- * return a response.
- */
-VCardResponse *
-vcard_make_response(vcard_7816_status_t status)
-{
-    VCardResponse *response;
-
-    switch (status) {
-    /* known 7816 response codes */
-    case VCARD7816_STATUS_SUCCESS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_SUCCESS);
-    case VCARD7816_STATUS_WARNING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING);
-    case VCARD7816_STATUS_WARNING_RET_CORUPT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_RET_CORUPT);
-    case VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE);
-    case VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED);
-    case VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID);
-    case VCARD7816_STATUS_WARNING_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_CHANGE);
-    case VCARD7816_STATUS_WARNING_FILE_FILLED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FILE_FILLED);
-    case VCARD7816_STATUS_EXC_ERROR:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR);
-    case VCARD7816_STATUS_EXC_ERROR_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_CHANGE);
-    case VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-    case VCARD7816_STATUS_ERROR_WRONG_LENGTH:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_LENGTH);
-    case VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE);
-    case VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED);
-    case VCARD7816_STATUS_ERROR_DATA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_INVALID);
-    case VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_DATA_NO_EF:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NO_EF);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA);
-    case VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_FILE_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE);
-    case VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_P1_P2_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-    case VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_DATA_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2);
-    case VCARD7816_STATUS_ERROR_INS_CODE_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_INS_CODE_INVALID);
-    case VCARD7816_STATUS_ERROR_CLA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_INVALID);
-    case VCARD7816_STATUS_ERROR_GENERAL:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_GENERAL);
-    default:
-        /* we don't know this status code, create a response buffer to
-         * hold it */
-        response = vcard_response_new_status(status);
-        if (response == NULL) {
-            /* couldn't allocate the buffer, return memmory error */
-            return VCARD_RESPONSE_GET_STATIC(
-                        VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        return response;
-    }
-}
-
-/*
- * Add File card support here if you need it.
- */
-static VCardStatus
-vcard7816_file_system_process_apdu(VCard *card, VCardAPDU *apdu,
-                                   VCardResponse **response)
-{
-    /* TODO: if we want to support a virtual file system card, we do it here.
-     * It would probably be a pkcs #15 card type */
-    *response = vcard_make_response(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
-
-/*
- * VM card (including java cards)
- */
-static VCardStatus
-vcard7816_vm_process_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    int bytes_to_copy, next_byte_count, count;
-    VCardApplet *current_applet;
-    VCardBufferResponse *buffer_response;
-    vcard_7816_status_t status;
-
-    /* parse the class first */
-    if (apdu->a_gen_type !=  VCARD_7816_ISO) {
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* use a switch so that if we need to support secure channel stuff later,
-     * we know where to put it */
-    switch (apdu->a_secure_messaging) {
-    case 0x0: /* no SM */
-        break;
-    case 0x4: /* proprietary SM */
-    case 0x8: /* header not authenticated */
-    case 0xc: /* header authenticated */
-    default:
-        /* for now, don't try to support secure channel stuff in the
-         * virtual card. */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* now parse the instruction */
-    switch (apdu->a_ins) {
-    case  VCARD7816_INS_MANAGE_CHANNEL: /* secure channel op */
-    case  VCARD7816_INS_EXTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_GET_CHALLENGE: /* secure channel op */
-    case  VCARD7816_INS_INTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_ERASE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_BINARY: /* applet control op */
-    case  VCARD7816_INS_WRITE_BINARY: /* applet control op */
-    case  VCARD7816_INS_UPDATE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_RECORD: /* file op */
-    case  VCARD7816_INS_WRITE_RECORD: /* file op */
-    case  VCARD7816_INS_UPDATE_RECORD: /* file op */
-    case  VCARD7816_INS_APPEND_RECORD: /* file op */
-    case  VCARD7816_INS_ENVELOPE:
-    case  VCARD7816_INS_PUT_DATA:
-        *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    case  VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x04) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-            break;
-        }
-
-        /* side effect, deselect the current applet if no applet has been found
-         * */
-        current_applet = vcard_find_applet(card, apdu->a_body, apdu->a_Lc);
-        vcard_select_applet(card, apdu->a_channel, current_applet);
-        if (current_applet) {
-            unsigned char *aid;
-            int aid_len;
-            aid = vcard_applet_get_aid(current_applet, &aid_len);
-            *response = vcard_response_new(card, aid, aid_len, apdu->a_Le,
-                                          VCARD7816_STATUS_SUCCESS);
-        } else {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-        }
-        break;
-
-    case  VCARD7816_INS_VERIFY:
-        if ((apdu->a_p1 != 0x00) || (apdu->a_p2 != 0x00)) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-        } else {
-            if (apdu->a_Lc == 0) {
-                /* handle pin count if possible */
-                count = vcard_emul_get_login_count(card);
-                if (count < 0) {
-                    *response = vcard_make_response(
-                                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-                } else {
-                    if (count > 0xf) {
-                        count = 0xf;
-                    }
-                    *response = vcard_response_new_status_bytes(
-                                                VCARD7816_SW1_WARNING_CHANGE,
-                                                                0xc0 | count);
-                    if (*response == NULL) {
-                        *response = vcard_make_response(
-                                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-                    }
-                }
-            } else {
-                    status = vcard_emul_login(card, apdu->a_body, apdu->a_Lc);
-                *response = vcard_make_response(status);
-            }
-        }
-        break;
-
-    case VCARD7816_INS_GET_RESPONSE:
-        buffer_response = vcard_get_buffer_response(card);
-        if (!buffer_response) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-            /* handle error */
-            break;
-        }
-        bytes_to_copy = MIN(buffer_response->len, apdu->a_Le);
-        next_byte_count = MIN(256, buffer_response->len - bytes_to_copy);
-        *response = vcard_response_new_bytes(
-                        card, buffer_response->current, bytes_to_copy,
-                        apdu->a_Le,
-                        next_byte_count ?
-                        VCARD7816_SW1_RESPONSE_BYTES : VCARD7816_SW1_SUCCESS,
-                        next_byte_count);
-        buffer_response->current += bytes_to_copy;
-        buffer_response->len -= bytes_to_copy;
-        if (*response == NULL || (next_byte_count == 0)) {
-            vcard_set_buffer_response(card, NULL);
-            vcard_buffer_response_delete(buffer_response);
-        }
-        if (*response == NULL) {
-            *response =
-                vcard_make_response(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        break;
-
-    case VCARD7816_INS_GET_DATA:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    default:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-    }
-
-    /* response should have been set somewhere */
-    assert(*response != NULL);
-    return VCARD_DONE;
-}
-
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location.
- */
-VCardStatus
-vcard_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    VCardStatus status;
-    VCardBufferResponse *buffer_response;
-
-    /* first handle any PTS commands, which aren't really APDU's */
-    if (apdu->a_type == VCARD_7816_PTS) {
-        /* the PTS responses aren't really responses either */
-        *response = vcard_response_new_data(apdu->a_data, apdu->a_len);
-        /* PTS responses have no status bytes */
-        (*response)->b_total_len = (*response)->b_len;
-        return VCARD_DONE;
-    }
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response && apdu->a_ins != VCARD7816_INS_GET_RESPONSE) {
-        /* clear out buffer_response, return an error */
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-        *response = vcard_make_response(VCARD7816_STATUS_EXC_ERROR);
-        return VCARD_DONE;
-    }
-
-    status = vcard_process_applet_apdu(card, apdu, response);
-    if (status != VCARD_NEXT) {
-        return status;
-    }
-    switch (vcard_get_type(card)) {
-    case VCARD_FILE_SYSTEM:
-        return vcard7816_file_system_process_apdu(card, apdu, response);
-    case VCARD_VM:
-        return vcard7816_vm_process_apdu(card, apdu, response);
-    case VCARD_DIRECT:
-        /* if we are type direct, then the applet should handle everything */
-        assert(!"VCARD_DIRECT: applet failure");
-        break;
-    }
-    *response =
-        vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
diff --git a/qemu/libcacard/card_7816.h b/qemu/libcacard/card_7816.h
deleted file mode 100644
index 4a01993d2..000000000
--- a/qemu/libcacard/card_7816.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816_H
-#define CARD_7816_H  1
-
-#include "card_7816t.h"
-#include "vcardt.h"
-
-/*
- * constructors for VCardResponse's
- */
-/* response from a return buffer and a status */
-VCardResponse *vcard_response_new(VCard *card, unsigned char *buf, int len,
-                                  int Le, vcard_7816_status_t status);
-/* response from a return buffer and status bytes */
-VCardResponse *vcard_response_new_bytes(VCard *card, unsigned char *buf,
-                                        int len, int Le,
-                                        unsigned char sw1, unsigned char sw2);
-/* response from just status bytes */
-VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                               unsigned char sw2);
-/* response from just status: NOTE this cannot fail, it will always return a
- * valid response, if it can't allocate memory, the response will be
- * VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE */
-VCardResponse *vcard_make_response(vcard_7816_status_t status);
-
-/* create a raw response (status has already been encoded */
-VCardResponse *vcard_response_new_data(unsigned char *buf, int len);
-
-
-
-
-/*
- * destructor for VCardResponse.
- *  Can be called with a NULL response
- */
-void vcard_response_delete(VCardResponse *response);
-
-/*
- * constructor for VCardAPDU
- */
-VCardAPDU *vcard_apdu_new(unsigned char *raw_apdu, int len,
-                          unsigned short *status);
-
-/*
- * destructor for VCardAPDU
- *  Can be called with a NULL apdu
- */
-void vcard_apdu_delete(VCardAPDU *apdu);
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location. Always returns a valid response.
- */
-VCardStatus vcard_process_apdu(VCard *card, VCardAPDU *apdu,
-                               VCardResponse **response);
-
-#endif
diff --git a/qemu/libcacard/card_7816t.h b/qemu/libcacard/card_7816t.h
deleted file mode 100644
index 8eef0ce42..000000000
--- a/qemu/libcacard/card_7816t.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816T_H
-#define CARD_7816T_H 1
-
-typedef unsigned short vcard_7816_status_t;
-
-struct VCardResponseStruct {
-    unsigned char *b_data;
-    vcard_7816_status_t b_status;
-    unsigned char b_sw1;
-    unsigned char b_sw2;
-    int b_len;
-    int b_total_len;
-    enum VCardResponseBufferType {
-        VCARD_MALLOC,
-        VCARD_MALLOC_DATA,
-        VCARD_MALLOC_STRUCT,
-        VCARD_STATIC
-    } b_type;
-};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS(stat) \
-static const VCardResponse VCardResponse##stat = \
-        {(unsigned char *)&VCardResponse##stat.b_sw1, (stat), ((stat) >> 8), \
-         ((stat) & 0xff), 0, 2, VCARD_STATIC};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS_BYTES(sw1, sw2) \
-static const VCardResponse VCARDResponse##sw1 = \
-        {(unsigned char *)&VCardResponse##name.b_sw1, ((sw1) << 8 | (sw2)), \
-         (sw1), (sw2), 0, 2, VCARD_STATIC};
-
-/* cast away the const, callers need may need to 'free' the
- * result, and const implies that they don't */
-#define VCARD_RESPONSE_GET_STATIC(name) \
-        ((VCardResponse *)(&VCardResponse##name))
-
-typedef enum {
-    VCARD_7816_ISO,
-    VCARD_7816_RFU,
-    VCARD_7816_PTS,
-    VCARD_7816_PROPRIETARY
-} VCardAPDUType;
-
-
-/*
- * 7816 header. All APDU's have this header.
- * They must be laid out in this order.
- */
-struct VCardAPDUHeader {
-    unsigned char ah_cla;
-    unsigned char ah_ins;
-    unsigned char ah_p1;
-    unsigned char ah_p2;
-    unsigned char ah_Le;
-    unsigned char ah_body[1]; /* indefinate length */
-};
-
-/*
- * 7816 APDU structure. The raw bytes are stored in the union and can be
- * accessed directly through u.data (which is aliased as a_data).
- *
- * Names of the fields match the 7816 documentation.
- */
-struct VCardAPDUStruct {
-    int a_len;                /* length of the whole buffer, including header */
-    int a_Lc;                 /* 7816 Lc (parameter length) value */
-    int a_Le;                 /* 7816 Le (expected result length) value */
-    unsigned char *a_body;    /* pointer to the parameter */
-    int a_channel;            /* decoded channel */
-    int a_secure_messaging;   /* decoded secure messaging type */
-    int a_type;               /* decoded type from cla (top nibble of class) */
-    VCardAPDUType a_gen_type; /* generic type (7816, PROPRIETARY, RFU, etc) */
-    union {
-        struct VCardAPDUHeader *header;
-        unsigned char   *data;
-    } u;
-/* give the subfields a unified look */
-#define a_header u.header
-#define a_data u.data
-#define a_cla a_header->ah_cla /* class */
-#define a_ins a_header->ah_ins /* instruction */
-#define a_p1 a_header->ah_p1   /* parameter 1 */
-#define a_p2 a_header->ah_p2   /* parameter 2 */
-};
-
-/* 7816 status codes */
-#define VCARD7816_STATUS_SUCCESS                              0x9000
-#define VCARD7816_STATUS_WARNING                              0x6200
-#define VCARD7816_STATUS_WARNING_RET_CORUPT                   0x6281
-#define VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE            0x6282
-#define VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED        0x6283
-#define VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID           0x6284
-#define VCARD7816_STATUS_WARNING_CHANGE                       0x6300
-#define VCARD7816_STATUS_WARNING_FILE_FILLED                  0x6381
-#define VCARD7816_STATUS_EXC_ERROR                            0x6400
-#define VCARD7816_STATUS_EXC_ERROR_CHANGE                     0x6500
-#define VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE             0x6581
-#define VCARD7816_STATUS_ERROR_WRONG_LENGTH                   0x6700
-#define VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED              0x6800
-#define VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED          0x6881
-#define VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED           0x6882
-#define VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED          0x6900
-#define VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE 0x6981
-#define VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED         0x6982
-#define VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED         0x6983
-#define VCARD7816_STATUS_ERROR_DATA_INVALID                   0x6984
-#define VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED        0x6985
-#define VCARD7816_STATUS_ERROR_DATA_NO_EF                     0x6986
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING              0x6987
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT            0x6988
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS               0x6a00
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA       0x6a80
-#define VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED         0x6a81
-#define VCARD7816_STATUS_ERROR_FILE_NOT_FOUND                 0x6a82
-#define VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND               0x6a83
-#define VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE              0x6a84
-#define VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT            0x6a85
-#define VCARD7816_STATUS_ERROR_P1_P2_INCORRECT                0x6a86
-#define VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT          0x6a87
-#define VCARD7816_STATUS_ERROR_DATA_NOT_FOUND                 0x6a88
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2             0x6b00
-#define VCARD7816_STATUS_ERROR_INS_CODE_INVALID               0x6d00
-#define VCARD7816_STATUS_ERROR_CLA_INVALID                    0x6e00
-#define VCARD7816_STATUS_ERROR_GENERAL                        0x6f00
-/* 7816 sw1 codes */
-#define VCARD7816_SW1_SUCCESS               0x90
-#define VCARD7816_SW1_RESPONSE_BYTES        0x61
-#define VCARD7816_SW1_WARNING               0x62
-#define VCARD7816_SW1_WARNING_CHANGE        0x63
-#define VCARD7816_SW1_EXC_ERROR             0x64
-#define VCARD7816_SW1_EXC_ERROR_CHANGE      0x65
-#define VCARD7816_SW1_ERROR_WRONG_LENGTH    0x67
-#define VCARD7816_SW1_CLA_ERROR             0x68
-#define VCARD7816_SW1_COMMAND_ERROR         0x69
-#define VCARD7816_SW1_P1_P2_ERROR           0x6a
-#define VCARD7816_SW1_LE_ERROR              0x6c
-#define VCARD7816_SW1_INS_ERROR             0x6d
-#define VCARD7816_SW1_CLA_NOT_SUPPORTED     0x6e
-
-/* 7816 Instructions */
-#define VCARD7816_INS_MANAGE_CHANNEL        0x70
-#define VCARD7816_INS_EXTERNAL_AUTHENTICATE 0x82
-#define VCARD7816_INS_GET_CHALLENGE         0x84
-#define VCARD7816_INS_INTERNAL_AUTHENTICATE 0x88
-#define VCARD7816_INS_ERASE_BINARY          0x0e
-#define VCARD7816_INS_READ_BINARY           0xb0
-#define VCARD7816_INS_WRITE_BINARY          0xd0
-#define VCARD7816_INS_UPDATE_BINARY         0xd6
-#define VCARD7816_INS_READ_RECORD           0xb2
-#define VCARD7816_INS_WRITE_RECORD          0xd2
-#define VCARD7816_INS_UPDATE_RECORD         0xdc
-#define VCARD7816_INS_APPEND_RECORD         0xe2
-#define VCARD7816_INS_ENVELOPE              0xc2
-#define VCARD7816_INS_PUT_DATA              0xda
-#define VCARD7816_INS_GET_DATA              0xca
-#define VCARD7816_INS_SELECT_FILE           0xa4
-#define VCARD7816_INS_VERIFY                0x20
-#define VCARD7816_INS_GET_RESPONSE          0xc0
-
-#endif
diff --git a/qemu/libcacard/event.c b/qemu/libcacard/event.c
deleted file mode 100644
index 63f4057fe..000000000
--- a/qemu/libcacard/event.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * event queue implementation.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include "vcard.h"
-#include "vreader.h"
-#include "vevent.h"
-
-VEvent *
-vevent_new(VEventType type, VReader *reader, VCard *card)
-{
-    VEvent *new_vevent;
-
-    new_vevent = g_new(VEvent, 1);
-    new_vevent->next = NULL;
-    new_vevent->type = type;
-    new_vevent->reader = vreader_reference(reader);
-    new_vevent->card = vcard_reference(card);
-
-    return new_vevent;
-}
-
-void
-vevent_delete(VEvent *vevent)
-{
-    if (vevent == NULL) {
-        return;
-    }
-    vreader_free(vevent->reader);
-    vcard_free(vevent->card);
-    g_free(vevent);
-}
-
-/*
- * VEvent queue management
- */
-
-static VEvent *vevent_queue_head;
-static VEvent *vevent_queue_tail;
-static CompatGMutex vevent_queue_lock;
-static CompatGCond vevent_queue_condition;
-
-void vevent_queue_init(void)
-{
-    vevent_queue_head = vevent_queue_tail = NULL;
-}
-
-void
-vevent_queue_vevent(VEvent *vevent)
-{
-    vevent->next = NULL;
-    g_mutex_lock(&vevent_queue_lock);
-    if (vevent_queue_head) {
-        assert(vevent_queue_tail);
-        vevent_queue_tail->next = vevent;
-    } else {
-        vevent_queue_head = vevent;
-    }
-    vevent_queue_tail = vevent;
-    g_cond_signal(&vevent_queue_condition);
-    g_mutex_unlock(&vevent_queue_lock);
-}
-
-/* must have lock */
-static VEvent *
-vevent_dequeue_vevent(void)
-{
-    VEvent *vevent = NULL;
-    if (vevent_queue_head) {
-        vevent = vevent_queue_head;
-        vevent_queue_head = vevent->next;
-        vevent->next = NULL;
-    }
-    return vevent;
-}
-
-VEvent *vevent_wait_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    while ((vevent = vevent_dequeue_vevent()) == NULL) {
-        g_cond_wait(&vevent_queue_condition, &vevent_queue_lock);
-    }
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
-VEvent *vevent_get_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    vevent = vevent_dequeue_vevent();
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
diff --git a/qemu/libcacard/eventt.h b/qemu/libcacard/eventt.h
deleted file mode 100644
index 0dc7bd468..000000000
--- a/qemu/libcacard/eventt.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef EVENTT_H
-#define EVENTT_H 1
-#include "vreadert.h"
-#include "vcardt.h"
-
-typedef struct VEventStruct VEvent;
-
-typedef enum {
-    VEVENT_READER_INSERT,
-    VEVENT_READER_REMOVE,
-    VEVENT_CARD_INSERT,
-    VEVENT_CARD_REMOVE,
-    VEVENT_LAST,
-} VEventType;
-
-struct VEventStruct {
-    VEvent *next;
-    VEventType type;
-    VReader *reader;
-    VCard *card;
-};
-#endif
-
-
diff --git a/qemu/libcacard/libcacard.pc.in b/qemu/libcacard/libcacard.pc.in
deleted file mode 100644
index 4b60023ce..000000000
--- a/qemu/libcacard/libcacard.pc.in
+++ /dev/null
@@ -1,13 +0,0 @@
-prefix=@PREFIX@
-exec_prefix=${prefix}
-libdir=@LIBDIR@
-includedir=@INCLUDEDIR@
-
-Name: cacard
-Description: CA Card library
-Version: @VERSION@
-
-Requires.private: nss glib-2.0
-Libs: -L${libdir} -lcacard
-Libs.private:
-Cflags: -I${includedir}
diff --git a/qemu/libcacard/libcacard.syms b/qemu/libcacard/libcacard.syms
deleted file mode 100644
index 1697515a7..000000000
--- a/qemu/libcacard/libcacard.syms
+++ /dev/null
@@ -1,77 +0,0 @@
-cac_card_init
-cac_is_cac_card
-vcard_add_applet
-vcard_apdu_delete
-vcard_apdu_new
-vcard_applet_get_aid
-vcard_buffer_response_delete
-vcard_buffer_response_new
-vcard_delete_applet
-vcard_emul_delete_key
-vcard_emul_force_card_insert
-vcard_emul_force_card_remove
-vcard_emul_get_atr
-vcard_emul_get_login_count
-vcard_emul_init
-vcard_emul_login
-vcard_emul_options
-vcard_emul_replay_insertion_events
-vcard_emul_reset
-vcard_emul_rsa_op
-vcard_emul_type_from_string
-vcard_emul_type_select
-vcard_emul_usage
-vcard_find_applet
-vcard_free
-vcard_get_atr
-vcard_get_buffer_response
-vcard_get_current_applet_private
-vcard_get_private
-vcard_get_type
-vcard_init
-vcard_make_response
-vcard_new
-vcard_new_applet
-vcard_process_apdu
-vcard_process_applet_apdu
-vcard_reference
-vcard_reset
-vcard_response_delete
-vcard_response_new
-vcard_response_new_bytes
-vcard_response_new_data
-vcard_response_new_status_bytes
-vcard_select_applet
-vcard_set_applet_private
-vcard_set_atr_func
-vcard_set_buffer_response
-vcard_set_type
-vevent_delete
-vevent_get_next_vevent
-vevent_new
-vevent_queue_init
-vevent_queue_vevent
-vevent_wait_next_vevent
-vreader_add_reader
-vreader_card_is_present
-vreader_free
-vreader_get_id
-vreader_get_name
-vreader_get_private
-vreader_get_reader_by_id
-vreader_get_reader_by_name
-vreader_get_reader_list
-vreader_init
-vreader_insert_card
-vreader_list_delete
-vreader_list_get_first
-vreader_list_get_next
-vreader_list_get_reader
-vreader_new
-vreader_power_off
-vreader_power_on
-vreader_queue_card_event
-vreader_reference
-vreader_remove_reader
-vreader_set_id
-vreader_xfr_bytes
diff --git a/qemu/libcacard/link_test.c b/qemu/libcacard/link_test.c
deleted file mode 100644
index 6f67a23d9..000000000
--- a/qemu/libcacard/link_test.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <stdio.h>
-#include "vcard.h"
-
-VCardStatus cac_card_init(const char *flags, VCard *card,
-                const unsigned char *cert[],
-                int cert_len[], VCardKey *key[] /* adopt the keys*/,
-                int cert_count);
-/*
- * this will crash... just test the linkage right now
- */
-
-main(int argc, char **argv)
-{
-    VCard *card; /* no constructor yet */
-    cac_card_init("", card, NULL, 0, NULL, 0);
-}
-
diff --git a/qemu/libcacard/vcard.c b/qemu/libcacard/vcard.c
deleted file mode 100644
index 1a87208f3..000000000
--- a/qemu/libcacard/vcard.c
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * implement the Java card standard.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816t.h"
-
-struct VCardAppletStruct {
-    VCardApplet   *next;
-    VCardProcessAPDU process_apdu;
-    VCardResetApplet reset_applet;
-    unsigned char *aid;
-    int aid_len;
-    void *applet_private;
-    VCardAppletPrivateFree applet_private_free;
-};
-
-struct VCardStruct {
-    int reference_count;
-    VCardApplet *applet_list;
-    VCardApplet *current_applet[MAX_CHANNEL];
-    VCardBufferResponse *vcard_buffer_response;
-    VCardType type;
-    VCardEmul *vcard_private;
-    VCardEmulFree vcard_private_free;
-    VCardGetAtr vcard_get_atr;
-};
-
-VCardBufferResponse *
-vcard_buffer_response_new(unsigned char *buffer, int size)
-{
-    VCardBufferResponse *new_buffer;
-
-    new_buffer = g_new(VCardBufferResponse, 1);
-    new_buffer->buffer = (unsigned char *)g_memdup(buffer, size);
-    new_buffer->buffer_len = size;
-    new_buffer->current = new_buffer->buffer;
-    new_buffer->len = size;
-    return new_buffer;
-}
-
-void
-vcard_buffer_response_delete(VCardBufferResponse *buffer_response)
-{
-    if (buffer_response == NULL) {
-        return;
-    }
-    g_free(buffer_response->buffer);
-    g_free(buffer_response);
-}
-
-
-/*
- * clean up state after a reset
- */
-void
-vcard_reset(VCard *card, VCardPower power)
-{
-    int i;
-    VCardApplet *applet = NULL;
-
-    if (card->type ==  VCARD_DIRECT) {
-        /* select the last applet */
-        VCardApplet *current_applet = NULL;
-        for (current_applet = card->applet_list; current_applet;
-                                       current_applet = current_applet->next) {
-            applet = current_applet;
-        }
-    }
-    for (i = 0; i < MAX_CHANNEL; i++) {
-        card->current_applet[i] = applet;
-    }
-    if (card->vcard_buffer_response) {
-        vcard_buffer_response_delete(card->vcard_buffer_response);
-        card->vcard_buffer_response = NULL;
-    }
-    vcard_emul_reset(card, power);
-    if (applet) {
-        applet->reset_applet(card, 0);
-    }
-}
-
-/* applet utilities */
-
-/*
- * applet utilities
- */
-/* constructor */
-VCardApplet *
-vcard_new_applet(VCardProcessAPDU applet_process_function,
-                 VCardResetApplet applet_reset_function,
-                 unsigned char *aid, int aid_len)
-{
-    VCardApplet *applet;
-
-    applet = g_new0(VCardApplet, 1);
-    applet->process_apdu = applet_process_function;
-    applet->reset_applet = applet_reset_function;
-
-    applet->aid = g_memdup(aid, aid_len);
-    applet->aid_len = aid_len;
-    return applet;
-}
-
-/* destructor */
-void
-vcard_delete_applet(VCardApplet *applet)
-{
-    if (applet == NULL) {
-        return;
-    }
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    g_free(applet->aid);
-    g_free(applet);
-}
-
-/* accessor */
-void
-vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *private,
-                         VCardAppletPrivateFree private_free)
-{
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    applet->applet_private = private;
-    applet->applet_private_free = private_free;
-}
-
-VCard *
-vcard_new(VCardEmul *private, VCardEmulFree private_free)
-{
-    VCard *new_card;
-
-    new_card = g_new0(VCard, 1);
-    new_card->type = VCARD_VM;
-    new_card->vcard_private = private;
-    new_card->vcard_private_free = private_free;
-    new_card->reference_count = 1;
-    return new_card;
-}
-
-VCard *
-vcard_reference(VCard *vcard)
-{
-    if (vcard == NULL) {
-        return NULL;
-    }
-    vcard->reference_count++;
-    return vcard;
-}
-
-void
-vcard_free(VCard *vcard)
-{
-    VCardApplet *current_applet;
-    VCardApplet *next_applet;
-
-    if (vcard == NULL) {
-        return;
-    }
-    vcard->reference_count--;
-    if (vcard->reference_count != 0) {
-        return;
-    }
-    if (vcard->vcard_private_free) {
-        (*vcard->vcard_private_free)(vcard->vcard_private);
-    }
-    for (current_applet = vcard->applet_list; current_applet;
-                                        current_applet = next_applet) {
-        next_applet = current_applet->next;
-        vcard_delete_applet(current_applet);
-    }
-    vcard_buffer_response_delete(vcard->vcard_buffer_response);
-    g_free(vcard);
-}
-
-void
-vcard_get_atr(VCard *vcard, unsigned char *atr, int *atr_len)
-{
-    if (vcard->vcard_get_atr) {
-        (*vcard->vcard_get_atr)(vcard, atr, atr_len);
-        return;
-    }
-    vcard_emul_get_atr(vcard, atr, atr_len);
-}
-
-void
-vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr)
-{
-    card->vcard_get_atr = vcard_get_atr;
-}
-
-
-VCardStatus
-vcard_add_applet(VCard *card, VCardApplet *applet)
-{
-    applet->next = card->applet_list;
-    card->applet_list = applet;
-    /* if our card-type is direct, always call the applet */
-    if (card->type ==  VCARD_DIRECT) {
-        int i;
-
-        for (i = 0; i < MAX_CHANNEL; i++) {
-            card->current_applet[i] = applet;
-        }
-    }
-    return VCARD_DONE;
-}
-
-/*
- * manage applets
- */
-VCardApplet *
-vcard_find_applet(VCard *card, unsigned char *aid, int aid_len)
-{
-    VCardApplet *current_applet;
-
-    for (current_applet = card->applet_list; current_applet;
-                                        current_applet = current_applet->next) {
-        if (current_applet->aid_len != aid_len) {
-            continue;
-        }
-        if (memcmp(current_applet->aid, aid, aid_len) == 0) {
-            break;
-        }
-    }
-    return current_applet;
-}
-
-unsigned char *
-vcard_applet_get_aid(VCardApplet *applet, int *aid_len)
-{
-    if (applet == NULL) {
-        return NULL;
-    }
-    *aid_len = applet->aid_len;
-    return applet->aid;
-}
-
-
-void
-vcard_select_applet(VCard *card, int channel, VCardApplet *applet)
-{
-    assert(channel < MAX_CHANNEL);
-
-    /* If using an emulated card, make sure to log out of any already logged in
-     * session. */
-    vcard_emul_logout(card);
-
-    card->current_applet[channel] = applet;
-    /* reset the applet */
-    if (applet && applet->reset_applet) {
-        applet->reset_applet(card, channel);
-    }
-}
-
-VCardAppletPrivate *
-vcard_get_current_applet_private(VCard *card, int channel)
-{
-    VCardApplet *applet = card->current_applet[channel];
-
-    if (applet == NULL) {
-        return NULL;
-    }
-    return applet->applet_private;
-}
-
-VCardStatus
-vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    if (card->current_applet[apdu->a_channel]) {
-        return card->current_applet[apdu->a_channel]->process_apdu(
-                                                        card, apdu, response);
-    }
-    return VCARD_NEXT;
-}
-
-/*
- * Accessor functions
- */
-/* accessor functions for the response buffer */
-VCardBufferResponse *
-vcard_get_buffer_response(VCard *card)
-{
-    return card->vcard_buffer_response;
-}
-
-void
-vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer)
-{
-    card->vcard_buffer_response = buffer;
-}
-
-
-/* accessor functions for the type */
-VCardType
-vcard_get_type(VCard *card)
-{
-    return card->type;
-}
-
-void
-vcard_set_type(VCard *card, VCardType type)
-{
-    card->type = type;
-}
-
-/* accessor for private data */
-VCardEmul *
-vcard_get_private(VCard *vcard)
-{
-    return vcard->vcard_private;
-}
-
diff --git a/qemu/libcacard/vcard.h b/qemu/libcacard/vcard.h
deleted file mode 100644
index 47dc70382..000000000
--- a/qemu/libcacard/vcard.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARD_H
-#define VCARD_H 1
-
-#include "vcardt.h"
-
-/*
- * response buffer constructors and destructors.
- *
- * response buffers are used when we need to return more data than will fit in
- * a normal APDU response (nominally 254 bytes).
- */
-VCardBufferResponse *vcard_buffer_response_new(unsigned char *buffer, int size);
-void vcard_buffer_response_delete(VCardBufferResponse *buffer_response);
-
-
-/*
- * clean up state on reset
- */
-void vcard_reset(VCard *card, VCardPower power);
-
-/*
- * applet utilities
- */
-/*
- * Constructor for a VCardApplet
- */
-VCardApplet *vcard_new_applet(VCardProcessAPDU applet_process_function,
-                              VCardResetApplet applet_reset_function,
-                              unsigned char *aid, int aid_len);
-
-/*
- * destructor for a VCardApplet
- *  Can be called with a NULL applet
- */
-void vcard_delete_applet(VCardApplet *applet);
-
-/* accessor - set the card type specific private data */
-void vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *_private,
-                              VCardAppletPrivateFree private_free);
-
-/* set type of vcard */
-void vcard_set_type(VCard *card, VCardType type);
-
-/*
- * utilities interacting with the current applet
- */
-/* add a new applet to a card */
-VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-/* find the applet on the card with the given aid */
-VCardApplet *vcard_find_applet(VCard *card, unsigned char *aid, int aid_len);
-/* set the following applet to be current on the given channel */
-void vcard_select_applet(VCard *card, int channel, VCardApplet *applet);
-/* get the card type specific private data on the given channel */
-VCardAppletPrivate *vcard_get_current_applet_private(VCard *card, int channel);
-/* fetch the applet's id */
-unsigned char *vcard_applet_get_aid(VCardApplet *applet, int *aid_len);
-
-/* process the apdu for the current selected applet/file */
-VCardStatus vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                                      VCardResponse **response);
-/*
- * VCard utilities
- */
-/* constructor */
-VCard *vcard_new(VCardEmul *_private, VCardEmulFree private_free);
-/* get a reference */
-VCard *vcard_reference(VCard *);
-/* destructor (reference counted) */
-void vcard_free(VCard *);
-/* get the atr from the card */
-void vcard_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-void vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr);
-
-/* accessor functions for the response buffer */
-VCardBufferResponse *vcard_get_buffer_response(VCard *card);
-void vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer);
-/* accessor functions for the type */
-VCardType vcard_get_type(VCard *card);
-/* get the private data */
-VCardEmul *vcard_get_private(VCard *card);
-
-#endif
diff --git a/qemu/libcacard/vcard_emul.h b/qemu/libcacard/vcard_emul.h
deleted file mode 100644
index f09ee98dc..000000000
--- a/qemu/libcacard/vcard_emul.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_H
-#define VCARD_EMUL_H 1
-
-#include "card_7816t.h"
-#include "vcard.h"
-#include "vcard_emul_type.h"
-
-/*
- * types
- */
-typedef enum {
-    VCARD_EMUL_OK = 0,
-    VCARD_EMUL_FAIL,
-    /* return values by vcard_emul_init */
-    VCARD_EMUL_INIT_ALREADY_INITED,
-} VCardEmulError;
-
-/* options are emul specific. call card_emul_parse_args to change a string
- * To an options struct */
-typedef struct VCardEmulOptionsStruct VCardEmulOptions;
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int vcard_emul_get_login_count(VCard *card);
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t vcard_emul_login(VCard *card, unsigned char *pin,
-                                     int pin_len);
-void vcard_emul_logout(VCard *card);
-
-/*
- * key functions
- */
-/* delete a key */
-void vcard_emul_delete_key(VCardKey *key);
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                  unsigned char *buffer, int buffer_size);
-
-void vcard_emul_reset(VCard *card, VCardPower power);
-void vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError vcard_emul_force_card_insert(VReader *vreader);
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-VCardEmulOptions *vcard_emul_options(const char *args);
-VCardEmulError vcard_emul_init(const VCardEmulOptions *options);
-void vcard_emul_replay_insertion_events(void);
-void vcard_emul_usage(void);
-#endif
diff --git a/qemu/libcacard/vcard_emul_nss.c b/qemu/libcacard/vcard_emul_nss.c
deleted file mode 100644
index d9761eedc..000000000
--- a/qemu/libcacard/vcard_emul_nss.c
+++ /dev/null
@@ -1,1274 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-/*
- * NSS headers
- */
-
-/* avoid including prototypes.h that redefines uint32 */
-#define NO_NSPR_10_SUPPORT
-
-#include <nss.h>
-#include <pk11pub.h>
-#include <cert.h>
-#include <key.h>
-#include <secmod.h>
-#include <prthread.h>
-#include <secerr.h>
-
-#include "glib-compat.h"
-
-#include "vcard.h"
-#include "card_7816t.h"
-#include "vcard_emul.h"
-#include "vreader.h"
-#include "vevent.h"
-
-#include "vcardt_internal.h"
-
-
-typedef enum {
-    VCardEmulUnknown = -1,
-    VCardEmulFalse = 0,
-    VCardEmulTrue = 1
-} VCardEmulTriState;
-
-struct VCardKeyStruct {
-    CERTCertificate *cert;
-    PK11SlotInfo *slot;
-    SECKEYPrivateKey *key;
-    VCardEmulTriState failedX509;
-};
-
-
-typedef struct VirtualReaderOptionsStruct VirtualReaderOptions;
-
-struct VReaderEmulStruct {
-    PK11SlotInfo *slot;
-    VCardEmulType default_type;
-    char *type_params;
-    PRBool present;
-    int     series;
-    VCard *saved_vcard;
-};
-
-/*
- *  NSS Specific options
- */
-struct VirtualReaderOptionsStruct {
-    char *name;
-    char *vname;
-    VCardEmulType card_type;
-    char *type_params;
-    char **cert_name;
-    int cert_count;
-};
-
-struct VCardEmulOptionsStruct {
-    void *nss_db;
-    VirtualReaderOptions *vreader;
-    int vreader_count;
-    VCardEmulType hw_card_type;
-    const char *hw_type_params;
-    PRBool use_hw;
-};
-
-static int nss_emul_init;
-
-/* if we have more that just the slot, define
- * VCardEmulStruct here */
-
-/*
- * allocate the set of arrays for certs, cert_len, key
- */
-static void
-vcard_emul_alloc_arrays(unsigned char ***certsp, int **cert_lenp,
-                        VCardKey ***keysp, int cert_count)
-{
-    *certsp = g_new(unsigned char *, cert_count);
-    *cert_lenp = g_new(int, cert_count);
-    *keysp = g_new(VCardKey *, cert_count);
-}
-
-/*
- * Emulator specific card information
- */
-typedef struct CardEmulCardStruct CardEmulPrivate;
-
-static VCardEmul *
-vcard_emul_new_card(PK11SlotInfo *slot)
-{
-    PK11_ReferenceSlot(slot);
-    /* currently we don't need anything other than the slot */
-    return (VCardEmul *)slot;
-}
-
-static void
-vcard_emul_delete_card(VCardEmul *vcard_emul)
-{
-    PK11SlotInfo *slot = (PK11SlotInfo *)vcard_emul;
-    if (slot == NULL) {
-        return;
-    }
-    PK11_FreeSlot(slot);
-}
-
-static PK11SlotInfo *
-vcard_emul_card_get_slot(VCard *card)
-{
-    /* note, the card is holding the reference, no need to get another one */
-    return (PK11SlotInfo *)vcard_get_private(card);
-}
-
-
-/*
- * key functions
- */
-/* private constructure */
-static VCardKey *
-vcard_emul_make_key(PK11SlotInfo *slot, CERTCertificate *cert)
-{
-    VCardKey *key;
-
-    key = g_new(VCardKey, 1);
-    key->slot = PK11_ReferenceSlot(slot);
-    key->cert = CERT_DupCertificate(cert);
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    /* NOTE: the cert is a temp cert, not necessarily the cert in the token,
-     * use the DER version of this function */
-    key->key = PK11_FindKeyByDERCert(slot, cert, NULL);
-    key->failedX509 = VCardEmulUnknown;
-    return key;
-}
-
-/* destructor */
-void
-vcard_emul_delete_key(VCardKey *key)
-{
-    if (!nss_emul_init || (key == NULL)) {
-        return;
-    }
-    if (key->key) {
-        SECKEY_DestroyPrivateKey(key->key);
-        key->key = NULL;
-    }
-    if (key->cert) {
-        CERT_DestroyCertificate(key->cert);
-    }
-    if (key->slot) {
-        PK11_FreeSlot(key->slot);
-    }
-}
-
-/*
- * grab the nss key from a VCardKey. If it doesn't exist, try to look it up
- */
-static SECKEYPrivateKey *
-vcard_emul_get_nss_key(VCardKey *key)
-{
-    if (key->key) {
-        return key->key;
-    }
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    key->key = PK11_FindPrivateKeyFromCert(key->slot, key->cert, NULL);
-    return key->key;
-}
-
-/*
- * Map NSS errors to 7816 errors
- */
-static vcard_7816_status_t
-vcard_emul_map_error(int error)
-{
-    switch (error) {
-    case SEC_ERROR_TOKEN_NOT_LOGGED_IN:
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    case SEC_ERROR_BAD_DATA:
-    case SEC_ERROR_OUTPUT_LEN:
-    case SEC_ERROR_INPUT_LEN:
-    case SEC_ERROR_INVALID_ARGS:
-    case SEC_ERROR_INVALID_ALGORITHM:
-    case SEC_ERROR_NO_KEY:
-    case SEC_ERROR_INVALID_KEY:
-    case SEC_ERROR_DECRYPTION_DISALLOWED:
-        return VCARD7816_STATUS_ERROR_DATA_INVALID;
-    case SEC_ERROR_NO_MEMORY:
-        return VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    }
-    return VCARD7816_STATUS_EXC_ERROR_CHANGE;
-}
-
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t
-vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                  unsigned char *buffer, int buffer_size)
-{
-    SECKEYPrivateKey *priv_key;
-    unsigned signature_len;
-    PK11SlotInfo *slot;
-    SECStatus rv;
-    unsigned char buf[2048];
-    unsigned char *bp = NULL;
-    int pad_len;
-    vcard_7816_status_t ret = VCARD7816_STATUS_SUCCESS;
-
-    if ((!nss_emul_init) || (key == NULL)) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    priv_key = vcard_emul_get_nss_key(key);
-    if (priv_key == NULL) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-
-    /*
-     * this is only true of the rsa signature
-     */
-    signature_len = PK11_SignatureLen(priv_key);
-    if (buffer_size != signature_len) {
-        return  VCARD7816_STATUS_ERROR_DATA_INVALID;
-    }
-    /* be able to handle larger keys if necessariy */
-    bp = &buf[0];
-    if (sizeof(buf) < signature_len) {
-        bp = g_malloc(signature_len);
-    }
-
-    /*
-     * do the raw operations. Some tokens claim to do CKM_RSA_X_509, but then
-     * choke when they try to do the actual operations. Try to detect
-     * those cases and treat them as if the token didn't claim support for
-     * X_509.
-     */
-    if (key->failedX509 != VCardEmulTrue
-                              && PK11_DoesMechanism(slot, CKM_RSA_X_509)) {
-        rv = PK11_PrivDecryptRaw(priv_key, bp, &signature_len, signature_len,
-                                 buffer, buffer_size);
-        if (rv == SECSuccess) {
-            assert(buffer_size == signature_len);
-            memcpy(buffer, bp, signature_len);
-            key->failedX509 = VCardEmulFalse;
-            goto cleanup;
-        }
-        /*
-         * we've had a successful X509 operation, this failure must be
-         * somethine else
-         */
-        if (key->failedX509 == VCardEmulFalse) {
-            ret = vcard_emul_map_error(PORT_GetError());
-            goto cleanup;
-        }
-        /*
-         * key->failedX509 must be Unknown at this point, try the
-         * non-x_509 case
-         */
-    }
-    /* token does not support CKM_RSA_X509, emulate that with CKM_RSA_PKCS */
-    /* is this a PKCS #1 formatted signature? */
-    if ((buffer[0] == 0) && (buffer[1] == 1)) {
-        int i;
-
-        for (i = 2; i < buffer_size; i++) {
-            /* rsa signature pad */
-            if (buffer[i] != 0xff) {
-                break;
-            }
-        }
-        if ((i < buffer_size) && (buffer[i] == 0)) {
-            /* yes, we have a properly formatted PKCS #1 signature */
-            /*
-             * NOTE: even if we accidentally got an encrypt buffer, which
-             * through sheer luck started with 00, 01, ff, 00, it won't matter
-             * because the resulting Sign operation will effectively decrypt
-             * the real buffer.
-             */
-            SECItem signature;
-            SECItem hash;
-
-            i++;
-            hash.data = &buffer[i];
-            hash.len = buffer_size - i;
-            signature.data = bp;
-            signature.len = signature_len;
-            rv = PK11_Sign(priv_key,  &signature, &hash);
-            if (rv != SECSuccess) {
-                ret = vcard_emul_map_error(PORT_GetError());
-                goto cleanup;
-            }
-            assert(buffer_size == signature.len);
-            memcpy(buffer, bp, signature.len);
-            /*
-             * we got here because either the X509 attempt failed, or the
-             * token couldn't do the X509 operation, in either case stay
-             * with the PKCS version for future operations on this key
-             */
-            key->failedX509 = VCardEmulTrue;
-            goto cleanup;
-        }
-    }
-    pad_len = buffer_size - signature_len;
-    assert(pad_len < 4);
-    /*
-     * OK now we've decrypted the payload, package it up in PKCS #1 for the
-     * upper layer.
-     */
-    buffer[0] = 0;
-    buffer[1] = 2; /* RSA_encrypt  */
-    pad_len -= 3; /* format is 0 || 2 || pad || 0 || data */
-    /*
-     * padding for PKCS #1 encrypted data is a string of random bytes. The
-     * random butes protect against potential decryption attacks against RSA.
-     * Since PrivDecrypt has already stripped those bytes, we can't reconstruct
-     * them. This shouldn't matter to the upper level code which should just
-     * strip this code out anyway, so We'll pad with a constant 3.
-     */
-    memset(&buffer[2], 0x03, pad_len);
-    pad_len += 2; /* index to the end of the pad */
-    buffer[pad_len] = 0;
-    pad_len++; /* index to the start of the data */
-    memcpy(&buffer[pad_len], bp, signature_len);
-    /*
-     * we got here because either the X509 attempt failed, or the
-     * token couldn't do the X509 operation, in either case stay
-     * with the PKCS version for future operations on this key
-     */
-    key->failedX509 = VCardEmulTrue;
-cleanup:
-    if (bp != buf) {
-        g_free(bp);
-    }
-    return ret;
-}
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int
-vcard_emul_get_login_count(VCard *card)
-{
-    return -1;
-}
-
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t
-vcard_emul_login(VCard *card, unsigned char *pin, int pin_len)
-{
-    PK11SlotInfo *slot;
-    unsigned char *pin_string;
-    int i;
-    SECStatus rv;
-
-    if (!nss_emul_init) {
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-     /* We depend on the PKCS #11 module internal login state here because we
-      * create a separate process to handle each guest instance. If we needed
-      * to handle multiple guests from one process, then we would need to keep
-      * a lot of extra state in our card structure
-      * */
-    pin_string = g_malloc(pin_len+1);
-    memcpy(pin_string, pin, pin_len);
-    pin_string[pin_len] = 0;
-
-    /* handle CAC expanded pins correctly */
-    for (i = pin_len-1; i >= 0 && (pin_string[i] == 0xff); i--) {
-        pin_string[i] = 0;
-    }
-
-    rv = PK11_Authenticate(slot, PR_FALSE, pin_string);
-    memset(pin_string, 0, pin_len);  /* don't let the pin hang around in memory
-                                        to be snooped */
-    g_free(pin_string);
-    if (rv == SECSuccess) {
-        return VCARD7816_STATUS_SUCCESS;
-    }
-    /* map the error from port get error */
-    return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-}
-
-void
-vcard_emul_logout(VCard *card)
-{
-    PK11SlotInfo *slot;
-
-    if (!nss_emul_init) {
-        return;
-    }
-
-    slot = vcard_emul_card_get_slot(card);
-    if (PK11_IsLoggedIn(slot, NULL)) {
-        PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
-    }
-}
-
-void
-vcard_emul_reset(VCard *card, VCardPower power)
-{
-    /*
-     * if we reset the card (either power on or power off), we lose our login
-     * state
-     */
-    vcard_emul_logout(card);
-
-    /* TODO: we may also need to send insertion/removal events? */
-}
-
-static VReader *
-vcard_emul_find_vreader_from_slot(PK11SlotInfo *slot)
-{
-    VReaderList *reader_list = vreader_get_reader_list();
-    VReaderListEntry *current_entry;
-
-    if (reader_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(reader_list); current_entry;
-                        current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderEmul *reader_emul = vreader_get_private(reader);
-        if (reader_emul->slot == slot) {
-            vreader_list_delete(reader_list);
-            return reader;
-        }
-        vreader_free(reader);
-    }
-
-    vreader_list_delete(reader_list);
-    return NULL;
-}
-
-/*
- * create a new reader emul
- */
-static VReaderEmul *
-vreader_emul_new(PK11SlotInfo *slot, VCardEmulType type, const char *params)
-{
-    VReaderEmul *new_reader_emul;
-
-    new_reader_emul = g_new(VReaderEmul, 1);
-
-    new_reader_emul->slot = PK11_ReferenceSlot(slot);
-    new_reader_emul->default_type = type;
-    new_reader_emul->type_params = g_strdup(params);
-    new_reader_emul->present = PR_FALSE;
-    new_reader_emul->series = 0;
-    new_reader_emul->saved_vcard = NULL;
-    return new_reader_emul;
-}
-
-static void
-vreader_emul_delete(VReaderEmul *vreader_emul)
-{
-    if (vreader_emul == NULL) {
-        return;
-    }
-    if (vreader_emul->slot) {
-        PK11_FreeSlot(vreader_emul->slot);
-    }
-    g_free(vreader_emul->type_params);
-    g_free(vreader_emul);
-}
-
-/*
- *  TODO: move this to emulater non-specific file
- */
-static VCardEmulType
-vcard_emul_get_type(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->default_type != VCARD_EMUL_NONE) {
-        return vreader_emul->default_type;
-    }
-
-    return vcard_emul_type_select(vreader);
-}
-/*
- *  TODO: move this to emulater non-specific file
- */
-static const char *
-vcard_emul_get_type_params(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->type_params) {
-        return vreader_emul->type_params;
-    }
-
-    return "";
-}
-
-/* pull the slot out of the reader private data */
-static PK11SlotInfo *
-vcard_emul_reader_get_slot(VReader *vreader)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul == NULL) {
-        return NULL;
-    }
-    return vreader_emul->slot;
-}
-
-/*
- *  Card ATR's map to physical cards. vcard_alloc_atr will set appropriate
- *  historical bytes for any software emulated card. The remaining bytes can be
- *  used to indicate the actual emulator
- */
-static unsigned char *nss_atr;
-static int nss_atr_len;
-
-void
-vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len)
-{
-    int len;
-    assert(atr != NULL);
-
-    if (nss_atr == NULL) {
-        nss_atr = vcard_alloc_atr("NSS", &nss_atr_len);
-    }
-    len = MIN(nss_atr_len, *atr_len);
-    memcpy(atr, nss_atr, len);
-    *atr_len = len;
-}
-
-/*
- * create a new card from certs and keys
- */
-static VCard *
-vcard_emul_make_card(VReader *reader,
-                     unsigned char * const *certs, int *cert_len,
-                     VCardKey *keys[], int cert_count)
-{
-    VCardEmul *vcard_emul;
-    VCard *vcard;
-    PK11SlotInfo *slot;
-    VCardEmulType type;
-    const char *params;
-
-    type = vcard_emul_get_type(reader);
-
-    /* ignore the inserted card */
-    if (type == VCARD_EMUL_NONE) {
-        return NULL;
-    }
-    slot = vcard_emul_reader_get_slot(reader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    params = vcard_emul_get_type_params(reader);
-    /* params these can be NULL */
-
-    vcard_emul = vcard_emul_new_card(slot);
-    if (vcard_emul == NULL) {
-        return NULL;
-    }
-    vcard = vcard_new(vcard_emul, vcard_emul_delete_card);
-    if (vcard == NULL) {
-        vcard_emul_delete_card(vcard_emul);
-        return NULL;
-    }
-    vcard_init(reader, vcard, type, params, certs, cert_len, keys, cert_count);
-    return vcard;
-}
-
-
-/*
- * 'clone' a physical card as a virtual card
- */
-static VCard *
-vcard_emul_mirror_card(VReader *vreader)
-{
-    /*
-     * lookup certs using the C_FindObjects. The Stan Cert handle won't give
-     * us the real certs until we log in.
-     */
-    PK11GenericObject *firstObj, *thisObj;
-    int cert_count;
-    unsigned char **certs;
-    int *cert_len;
-    VCardKey **keys;
-    PK11SlotInfo *slot;
-    VCard *card;
-
-    slot = vcard_emul_reader_get_slot(vreader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    firstObj = PK11_FindGenericObjects(slot, CKO_CERTIFICATE);
-    if (firstObj == NULL) {
-        return NULL;
-    }
-
-    /* count the certs */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        cert_count++;
-    }
-
-    /* allocate the arrays */
-    vcard_emul_alloc_arrays(&certs, &cert_len, &keys, cert_count);
-
-    /* fill in the arrays */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        SECItem derCert;
-        CERTCertificate *cert;
-        SECStatus rv;
-
-        rv = PK11_ReadRawAttribute(PK11_TypeGeneric, thisObj,
-                                   CKA_VALUE, &derCert);
-        if (rv != SECSuccess) {
-            continue;
-        }
-        /* create floating temp cert. This gives us a cert structure even if
-         * the token isn't logged in */
-        cert = CERT_NewTempCertificate(CERT_GetDefaultCertDB(), &derCert,
-                                       NULL, PR_FALSE, PR_TRUE);
-        SECITEM_FreeItem(&derCert, PR_FALSE);
-        if (cert == NULL) {
-            continue;
-        }
-
-        certs[cert_count] = cert->derCert.data;
-        cert_len[cert_count] = cert->derCert.len;
-        keys[cert_count] = vcard_emul_make_key(slot, cert);
-        cert_count++;
-        CERT_DestroyCertificate(cert); /* key obj still has a reference */
-    }
-
-    /* now create the card */
-    card = vcard_emul_make_card(vreader, certs, cert_len, keys, cert_count);
-    g_free(certs);
-    g_free(cert_len);
-    g_free(keys);
-
-    return card;
-}
-
-static VCardEmulType default_card_type = VCARD_EMUL_NONE;
-static const char *default_type_params = "";
-
-/*
- * This thread looks for card and reader insertions and puts events on the
- * event queue
- */
-static void
-vcard_emul_event_thread(void *arg)
-{
-    PK11SlotInfo *slot;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-    SECMODModule *module = (SECMODModule *)arg;
-
-    do {
-        /*
-         * XXX - the latency value doesn't matter one bit. you only get no
-         * blocking (flags |= CKF_DONT_BLOCK) or PKCS11_WAIT_LATENCY (==500),
-         * hard coded in coolkey.  And it isn't coolkey's fault - the timeout
-         * value we pass get's dropped on the floor before C_WaitForSlotEvent
-         * is called.
-         */
-        slot = SECMOD_WaitForAnyTokenEvent(module, 0, 500);
-        if (slot == NULL) {
-            /* this could be just a no event indication */
-            if (PORT_GetError() == SEC_ERROR_NO_EVENT) {
-                continue;
-            }
-            break;
-        }
-        vreader = vcard_emul_find_vreader_from_slot(slot);
-        if (vreader == NULL) {
-            /* new vreader */
-            vreader_emul = vreader_emul_new(slot, default_card_type,
-                                            default_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            PK11_FreeSlot(slot);
-            slot = NULL;
-            vreader_add_reader(vreader);
-            vreader_free(vreader);
-            continue;
-        }
-        /* card remove/insert */
-        vreader_emul = vreader_get_private(vreader);
-        if (PK11_IsPresent(slot)) {
-            int series = PK11_GetSlotSeries(slot);
-            if (series != vreader_emul->series) {
-                if (vreader_emul->present) {
-                    vreader_insert_card(vreader, NULL);
-                }
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_free(vcard);
-            }
-            vreader_emul->series = series;
-            vreader_emul->present = 1;
-            vreader_free(vreader);
-            PK11_FreeSlot(slot);
-            continue;
-        }
-        if (vreader_emul->present) {
-            vreader_insert_card(vreader, NULL);
-        }
-        vreader_emul->series = 0;
-        vreader_emul->present = 0;
-        PK11_FreeSlot(slot);
-        vreader_free(vreader);
-    } while (1);
-}
-
-/* if the card is inserted when we start up, make sure our state is correct */
-static void
-vcard_emul_init_series(VReader *vreader, VCard *vcard)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    PK11SlotInfo *slot = vreader_emul->slot;
-
-    vreader_emul->present = PK11_IsPresent(slot);
-    vreader_emul->series = PK11_GetSlotSeries(slot);
-    if (vreader_emul->present == 0) {
-        vreader_insert_card(vreader, NULL);
-    }
-}
-
-/*
- * each module has a separate wait call, create a thread for each module that
- * we are using.
- */
-static void
-vcard_emul_new_event_thread(SECMODModule *module)
-{
-    PR_CreateThread(PR_SYSTEM_THREAD, vcard_emul_event_thread,
-                     module, PR_PRIORITY_HIGH, PR_GLOBAL_THREAD,
-                     PR_UNJOINABLE_THREAD, 0);
-}
-
-static const VCardEmulOptions default_options = {
-    .nss_db = NULL,
-    .vreader = NULL,
-    .vreader_count = 0,
-    .hw_card_type = VCARD_EMUL_CAC,
-    .hw_type_params = "",
-    .use_hw = PR_TRUE
-};
-
-
-/*
- *  NSS needs the app to supply a password prompt. In our case the only time
- *  the password is supplied is as part of the Login APDU. The actual password
- *  is passed in the pw_arg in that case. In all other cases pw_arg should be
- *  NULL.
- */
-static char *
-vcard_emul_get_password(PK11SlotInfo *slot, PRBool retries, void *pw_arg)
-{
-    /* if it didn't work the first time, don't keep trying */
-    if (retries) {
-        return NULL;
-    }
-    /* we are looking up a password when we don't have one in hand */
-    if (pw_arg == NULL) {
-        return NULL;
-    }
-    /* TODO: we really should verify that were are using the right slot */
-    return PORT_Strdup(pw_arg);
-}
-
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError
-vcard_emul_force_card_remove(VReader *vreader)
-{
-    if (!nss_emul_init || (vreader_card_is_present(vreader) != VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-
-    /* OK, remove it */
-    vreader_insert_card(vreader, NULL);
-    return VCARD_EMUL_OK;
-}
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError
-vcard_emul_force_card_insert(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-
-    if (!nss_emul_init || (vreader_card_is_present(vreader) == VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-    vreader_emul = vreader_get_private(vreader);
-
-    /* if it's a softcard, get the saved vcard from the reader emul structure */
-    if (vreader_emul->saved_vcard) {
-        vcard = vcard_reference(vreader_emul->saved_vcard);
-    } else {
-        /* it must be a physical card, rebuild it */
-        if (!PK11_IsPresent(vreader_emul->slot)) {
-            /* physical card has been removed, not way to reinsert it */
-            return VCARD_EMUL_FAIL;
-        }
-        vcard = vcard_emul_mirror_card(vreader);
-    }
-    vreader_insert_card(vreader, vcard);
-    vcard_free(vcard);
-
-    return VCARD_EMUL_OK;
-}
-
-
-static PRBool
-module_has_removable_hw_slots(SECMODModule *mod)
-{
-    int i;
-    PRBool ret = PR_FALSE;
-    SECMODListLock *moduleLock = SECMOD_GetDefaultModuleListLock();
-
-    if (!moduleLock) {
-        PORT_SetError(SEC_ERROR_NOT_INITIALIZED);
-        return ret;
-    }
-    SECMOD_GetReadLock(moduleLock);
-    for (i = 0; i < mod->slotCount; i++) {
-        PK11SlotInfo *slot = mod->slots[i];
-        if (PK11_IsRemovable(slot) && PK11_IsHW(slot)) {
-            ret = PR_TRUE;
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(moduleLock);
-    return ret;
-}
-
-/* Previously we returned FAIL if no readers found. This makes
- * no sense when using hardware, since there may be no readers connected
- * at the time vcard_emul_init is called, but they will be properly
- * recognized later. So Instead return FAIL only if no_hw==1 and no
- * vcards can be created (indicates error with certificates provided
- * or db), or if any other higher level error (NSS error, missing coolkey). */
-static int vcard_emul_init_called;
-
-VCardEmulError
-vcard_emul_init(const VCardEmulOptions *options)
-{
-    SECStatus rv;
-    PRBool has_readers = PR_FALSE;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    SECMODListLock *module_lock;
-    SECMODModuleList *module_list;
-    SECMODModuleList *mlp;
-    int i;
-
-    if (vcard_emul_init_called) {
-        return VCARD_EMUL_INIT_ALREADY_INITED;
-    }
-    vcard_emul_init_called = 1;
-    vreader_init();
-    vevent_queue_init();
-
-    if (options == NULL) {
-        options = &default_options;
-    }
-
-    /* first initialize NSS */
-    if (options->nss_db) {
-        rv = NSS_Init(options->nss_db);
-    } else {
-        gchar *path;
-#ifndef _WIN32
-        path = g_strdup("/etc/pki/nssdb");
-#else
-        if (g_get_system_config_dirs() == NULL ||
-            g_get_system_config_dirs()[0] == NULL) {
-            return VCARD_EMUL_FAIL;
-        }
-
-        path = g_build_filename(
-            g_get_system_config_dirs()[0], "pki", "nssdb", NULL);
-#endif
-
-        rv = NSS_Init(path);
-        g_free(path);
-    }
-    if (rv != SECSuccess) {
-        return VCARD_EMUL_FAIL;
-    }
-    /* Set password callback function */
-    PK11_SetPasswordFunc(vcard_emul_get_password);
-
-    /* set up soft cards emulated by software certs rather than physical cards
-     * */
-    for (i = 0; i < options->vreader_count; i++) {
-        int j;
-        int cert_count;
-        unsigned char **certs;
-        int *cert_len;
-        VCardKey **keys;
-        PK11SlotInfo *slot;
-
-        slot = PK11_FindSlotByName(options->vreader[i].name);
-        if (slot == NULL) {
-            continue;
-        }
-        vreader_emul = vreader_emul_new(slot, options->vreader[i].card_type,
-                                        options->vreader[i].type_params);
-        vreader = vreader_new(options->vreader[i].vname, vreader_emul,
-                              vreader_emul_delete);
-        vreader_add_reader(vreader);
-
-        vcard_emul_alloc_arrays(&certs, &cert_len, &keys,
-                                options->vreader[i].cert_count);
-
-        cert_count = 0;
-        for (j = 0; j < options->vreader[i].cert_count; j++) {
-            /* we should have a better way of identifying certs than by
-             * nickname here */
-            CERTCertificate *cert = PK11_FindCertFromNickname(
-                                        options->vreader[i].cert_name[j],
-                                        NULL);
-            if (cert == NULL) {
-                continue;
-            }
-            certs[cert_count] = cert->derCert.data;
-            cert_len[cert_count] = cert->derCert.len;
-            keys[cert_count] = vcard_emul_make_key(slot, cert);
-            /* this is safe because the key is still holding a cert reference */
-            CERT_DestroyCertificate(cert);
-            cert_count++;
-        }
-        if (cert_count) {
-            VCard *vcard = vcard_emul_make_card(vreader, certs, cert_len,
-                                                keys, cert_count);
-            vreader_insert_card(vreader, vcard);
-            vcard_emul_init_series(vreader, vcard);
-            /* allow insertion and removal of soft cards */
-            vreader_emul->saved_vcard = vcard_reference(vcard);
-            vcard_free(vcard);
-            vreader_free(vreader);
-            has_readers = PR_TRUE;
-        }
-        g_free(certs);
-        g_free(cert_len);
-        g_free(keys);
-    }
-
-    /* if we aren't suppose to use hw, skip looking up hardware tokens */
-    if (!options->use_hw) {
-        nss_emul_init = has_readers;
-        return has_readers ? VCARD_EMUL_OK : VCARD_EMUL_FAIL;
-    }
-
-    /* make sure we have some PKCS #11 module loaded */
-    module_lock = SECMOD_GetDefaultModuleListLock();
-    module_list = SECMOD_GetDefaultModuleList();
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-        if (module_has_removable_hw_slots(module)) {
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-
-    /* now examine all the slots, finding which should be readers */
-    /* We should control this with options. For now we mirror out any
-     * removable hardware slot */
-    default_card_type = options->hw_card_type;
-    default_type_params = g_strdup(options->hw_type_params);
-
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-
-        /* Ignore the internal module */
-        if (module == NULL || module == SECMOD_GetInternalModule()) {
-            continue;
-        }
-
-        for (i = 0; i < module->slotCount; i++) {
-            PK11SlotInfo *slot = module->slots[i];
-
-            /* only map removable HW slots */
-            if (slot == NULL || !PK11_IsRemovable(slot) || !PK11_IsHW(slot)) {
-                continue;
-            }
-            if (strcmp("E-Gate 0 0", PK11_GetSlotName(slot)) == 0) {
-                /*
-                 * coolkey <= 1.1.0-20 emulates this reader if it can't find
-                 * any hardware readers. This causes problems, warn user of
-                 * problems.
-                 */
-                fprintf(stderr, "known bad coolkey version - see "
-                        "https://bugzilla.redhat.com/show_bug.cgi?id=802435\n");
-                continue;
-            }
-            vreader_emul = vreader_emul_new(slot, options->hw_card_type,
-                                            options->hw_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            vreader_add_reader(vreader);
-
-            if (PK11_IsPresent(slot)) {
-                VCard *vcard;
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_emul_init_series(vreader, vcard);
-                vcard_free(vcard);
-            }
-        }
-        vcard_emul_new_event_thread(module);
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-    nss_emul_init = PR_TRUE;
-
-    return VCARD_EMUL_OK;
-}
-
-/* Recreate card insert events for all readers (user should
- * deduce implied reader insert. perhaps do a reader insert as well?)
- */
-void
-vcard_emul_replay_insertion_events(void)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    VReaderList *list = vreader_get_reader_list();
-
-    for (current_entry = vreader_list_get_first(list); current_entry;
-            current_entry = next_entry) {
-        VReader *vreader = vreader_list_get_reader(current_entry);
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_queue_card_event(vreader);
-    }
-
-    vreader_list_delete(list);
-}
-
-/*
- *  Silly little functions to help parsing our argument string
- */
-static int
-count_tokens(const char *str, char token, char token_end)
-{
-    int count = 0;
-
-    for (; *str; str++) {
-        if (*str == token) {
-            count++;
-        }
-        if (*str == token_end) {
-            break;
-        }
-    }
-    return count;
-}
-
-static const char *
-strip(const char *str)
-{
-    for (; *str && isspace(*str); str++) {
-    }
-    return str;
-}
-
-static const char *
-find_blank(const char *str)
-{
-    for (; *str && !isspace(*str); str++) {
-    }
-    return str;
-}
-
-
-/*
- *  We really want to use some existing argument parsing library here. That
- *  would give us a consistent look */
-static VCardEmulOptions options;
-#define READER_STEP 4
-
-/* Expects "args" to be at the beginning of a token (ie right after the ','
- * ending the previous token), and puts the next token start in "token",
- * and its length in "token_length". "token" will not be nul-terminated.
- * After calling the macro, "args" will be advanced to the beginning of
- * the next token.
- * This macro may call continue or break.
- */
-#define NEXT_TOKEN(token) \
-            (token) = args; \
-            args = strpbrk(args, ",)"); \
-            if (*args == 0) { \
-                break; \
-            } \
-            if (*args == ')') { \
-                args++; \
-                continue; \
-            } \
-            (token##_length) = args - (token); \
-            args = strip(args+1);
-
-VCardEmulOptions *
-vcard_emul_options(const char *args)
-{
-    int reader_count = 0;
-    VCardEmulOptions *opts;
-
-    /* Allow the future use of allocating the options structure on the fly */
-    memcpy(&options, &default_options, sizeof(options));
-    opts = &options;
-
-    do {
-        args = strip(args); /* strip off the leading spaces */
-        if (*args == ',') {
-            continue;
-        }
-        /* soft=(slot_name,virt_name,emul_type,emul_flags,cert_1, (no eol)
-         *       cert_2,cert_3...) */
-        if (strncmp(args, "soft=", 5) == 0) {
-            const char *name;
-            size_t name_length;
-            const char *vname;
-            size_t vname_length;
-            const char *type_params;
-            size_t type_params_length;
-            char type_str[100];
-            VCardEmulType type;
-            int count, i;
-            VirtualReaderOptions *vreaderOpt;
-
-            args = strip(args + 5);
-            if (*args != '(') {
-                continue;
-            }
-            args = strip(args+1);
-
-            NEXT_TOKEN(name)
-            NEXT_TOKEN(vname)
-            NEXT_TOKEN(type_params)
-            type_params_length = MIN(type_params_length, sizeof(type_str)-1);
-            memcpy(type_str, type_params, type_params_length);
-            type_str[type_params_length] = '\0';
-            type = vcard_emul_type_from_string(type_str);
-
-            NEXT_TOKEN(type_params)
-
-            if (*args == 0) {
-                break;
-            }
-
-            if (opts->vreader_count >= reader_count) {
-                reader_count += READER_STEP;
-                opts->vreader = g_renew(VirtualReaderOptions, opts->vreader,
-                                        reader_count);
-            }
-            vreaderOpt = &opts->vreader[opts->vreader_count];
-            vreaderOpt->name = g_strndup(name, name_length);
-            vreaderOpt->vname = g_strndup(vname, vname_length);
-            vreaderOpt->card_type = type;
-            vreaderOpt->type_params =
-                g_strndup(type_params, type_params_length);
-            count = count_tokens(args, ',', ')') + 1;
-            vreaderOpt->cert_count = count;
-            vreaderOpt->cert_name = g_new(char *, count);
-            for (i = 0; i < count; i++) {
-                const char *cert = args;
-                args = strpbrk(args, ",)");
-                vreaderOpt->cert_name[i] = g_strndup(cert, args - cert);
-                args = strip(args+1);
-            }
-            if (*args == ')') {
-                args++;
-            }
-            opts->vreader_count++;
-        /* use_hw= */
-        } else if (strncmp(args, "use_hw=", 7) == 0) {
-            args = strip(args+7);
-            if (*args == '0' || *args == 'N' || *args == 'n' || *args == 'F') {
-                opts->use_hw = PR_FALSE;
-            } else {
-                opts->use_hw = PR_TRUE;
-            }
-            args = find_blank(args);
-        /* hw_type= */
-        } else if (strncmp(args, "hw_type=", 8) == 0) {
-            args = strip(args+8);
-            opts->hw_card_type = vcard_emul_type_from_string(args);
-            args = find_blank(args);
-        /* hw_params= */
-        } else if (strncmp(args, "hw_params=", 10) == 0) {
-            const char *params;
-            args = strip(args+10);
-            params = args;
-            args = find_blank(args);
-            opts->hw_type_params = g_strndup(params, args-params);
-        /* db="/data/base/path" */
-        } else if (strncmp(args, "db=", 3) == 0) {
-            const char *db;
-            args = strip(args+3);
-            if (*args != '"') {
-                continue;
-            }
-            args++;
-            db = args;
-            args = strpbrk(args, "\"\n");
-            opts->nss_db = g_strndup(db, args-db);
-            if (*args != 0) {
-                args++;
-            }
-        } else {
-            args = find_blank(args);
-        }
-    } while (*args != 0);
-
-    return opts;
-}
-
-void
-vcard_emul_usage(void)
-{
-   fprintf(stderr,
-"emul args: comma separated list of the following arguments\n"
-" db={nss_database}               (default sql:/etc/pki/nssdb)\n"
-" use_hw=[yes|no]                 (default yes)\n"
-" hw_type={card_type_to_emulate}  (default CAC)\n"
-" hw_param={param_for_card}       (default \"\")\n"
-" soft=({slot_name},{vreader_name},{card_type_to_emulate},{params_for_card},\n"
-"       {cert1},{cert2},{cert3}    (default none)\n"
-"\n"
-"  {nss_database}          The location of the NSS cert & key database\n"
-"  {card_type_to_emulate}  What card interface to present to the guest\n"
-"  {param_for_card}        Card interface specific parameters\n"
-"  {slot_name}             NSS slot that contains the certs\n"
-"  {vreader_name}          Virtual reader name to present to the guest\n"
-"  {certN}                 Nickname of the certificate n on the virtual card\n"
-"\n"
-"These parameters come as a single string separated by blanks or newlines."
-"\n"
-"Unless use_hw is set to no, all tokens that look like removable hardware\n"
-"tokens will be presented to the guest using the emulator specified by\n"
-"hw_type, and parameters of hw_param.\n"
-"\n"
-"If more one or more soft= parameters are specified, these readers will be\n"
-"presented to the guest\n");
-}
diff --git a/qemu/libcacard/vcard_emul_type.c b/qemu/libcacard/vcard_emul_type.c
deleted file mode 100644
index 59a145820..000000000
--- a/qemu/libcacard/vcard_emul_type.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- *  This file contains utility functions which abstract the different card
- *  types.  The goal is that new card types can easily be added by simply
- *  changing this file and vcard_emul_type.h. It is currently not a requirement
- *  to dynamically add new card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <strings.h>
-#include "vcardt.h"
-#include "vcard_emul_type.h"
-#include "cac.h"
-
-VCardStatus vcard_init(VReader *vreader, VCard *vcard,
-                       VCardEmulType type, const char *params,
-                       unsigned char *const *cert, int cert_len[],
-                       VCardKey *key[], int cert_count)
-{
-    switch (type) {
-    case VCARD_EMUL_NONE:
-        break;
-    case VCARD_EMUL_CAC:
-        return cac_card_init(vreader, vcard, params,
-                             cert, cert_len, key,  cert_count);
-    /* add new ones here */
-    default:
-        break;
-    }
-    return VCARD_FAIL;
-}
-
-VCardEmulType vcard_emul_type_select(VReader *vreader)
-{
-#ifdef notdef
-    /* since there is only one emulator no need to call this function */
-    if (cac_is_cac_card(vreader) == VCARD_DONE) {
-        return VCARD_EMUL_CAC;
-    }
-#endif
-    /* return the default */
-    return VCARD_EMUL_CAC;
-}
-
-VCardEmulType vcard_emul_type_from_string(const char *type_string)
-{
-     if (strcasecmp(type_string, "CAC") == 0) {
-        return VCARD_EMUL_CAC;
-     }
-#ifdef USE_PASSTHRU
-     if (strcasecmp(type_string, "PASSTHRU") == 0) {
-        return VCARD_EMUL_PASSTHRU;
-     }
-#endif
-     return VCARD_EMUL_NONE;
-}
diff --git a/qemu/libcacard/vcard_emul_type.h b/qemu/libcacard/vcard_emul_type.h
deleted file mode 100644
index 0242f40eb..000000000
--- a/qemu/libcacard/vcard_emul_type.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *  This header file abstracts the different card types. The goal is new card
- *  types can easily be added by simply changing this file and
- *  vcard_emul_type.c. It is currently not a requirement to dynamically add new
- *  card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_TYPE_H
-#define VCARD_EMUL_TYPE_H 1
-#include "vcardt.h"
-#include "vreadert.h"
-
-/*
- * types
- */
-typedef enum {
-     VCARD_EMUL_NONE = 0,
-     VCARD_EMUL_CAC,
-     VCARD_EMUL_PASSTHRU
-} VCardEmulType;
-
-/* functions used by the rest of the emulator */
-VCardStatus vcard_init(VReader *vreader, VCard *vcard, VCardEmulType type,
-                       const char *params, unsigned char * const *cert,
-                       int cert_len[], VCardKey *key[], int cert_count);
-VCardEmulType vcard_emul_type_select(VReader *vreader);
-VCardEmulType vcard_emul_type_from_string(const char *type_string);
-
-#endif
diff --git a/qemu/libcacard/vcardt.c b/qemu/libcacard/vcardt.c
deleted file mode 100644
index c67de2f3c..000000000
--- a/qemu/libcacard/vcardt.c
+++ /dev/null
@@ -1,40 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <glib.h>
-
-#include "vcardt.h"
-
-#include "vcardt_internal.h"
-
-/* create an ATR with appropriate historical bytes */
-#define ATR_TS_DIRECT_CONVENTION 0x3b
-#define ATR_TA_PRESENT 0x10
-#define ATR_TB_PRESENT 0x20
-#define ATR_TC_PRESENT 0x40
-#define ATR_TD_PRESENT 0x80
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len)
-{
-    int postfix_len;
-    const char prefix[] = "VCARD_";
-    const char default_postfix[] = "DEFAULT";
-    const int prefix_len = sizeof(prefix) - 1;
-    int total_len;
-    unsigned char *atr;
-
-    if (postfix == NULL) {
-        postfix = default_postfix;
-    }
-    postfix_len = strlen(postfix);
-    total_len = 3 + prefix_len + postfix_len;
-    atr = g_malloc(total_len);
-    atr[0] = ATR_TS_DIRECT_CONVENTION;
-    atr[1] = ATR_TD_PRESENT + prefix_len + postfix_len;
-    atr[2] = 0x00;
-    memcpy(&atr[3], prefix, prefix_len);
-    memcpy(&atr[3 + prefix_len], postfix, postfix_len);
-    if (atr_len) {
-        *atr_len = total_len;
-    }
-    return atr;
-}
diff --git a/qemu/libcacard/vcardt.h b/qemu/libcacard/vcardt.h
deleted file mode 100644
index 795e26530..000000000
--- a/qemu/libcacard/vcardt.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARDT_H
-#define VCARDT_H 1
-
-/*
- * these should come from some common spice header file
- */
-#include <assert.h>
-#ifndef MIN
-#define MIN(x, y) ((x) > (y) ? (y) : (x))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
-#endif
-
-typedef struct VCardStruct VCard;
-typedef struct VCardAPDUStruct VCardAPDU;
-typedef struct VCardResponseStruct VCardResponse;
-typedef struct VCardBufferResponseStruct VCardBufferResponse;
-typedef struct VCardAppletStruct VCardApplet;
-typedef struct VCardAppletPrivateStruct VCardAppletPrivate;
-typedef struct VCardKeyStruct VCardKey;  /* opaque */
-typedef struct VCardEmulStruct VCardEmul;
-
-#define MAX_CHANNEL 4
-
-typedef enum {
-    VCARD_DONE,
-    VCARD_NEXT,
-    VCARD_FAIL
-} VCardStatus;
-
-typedef enum {
-    VCARD_FILE_SYSTEM,
-    VCARD_VM,
-    VCARD_DIRECT
-} VCardType;
-
-typedef enum {
-    VCARD_POWER_ON,
-    VCARD_POWER_OFF
-} VCardPower;
-
-typedef VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                        VCardResponse **response);
-typedef VCardStatus (*VCardResetApplet)(VCard *card, int channel);
-typedef void (*VCardAppletPrivateFree) (VCardAppletPrivate *);
-typedef void (*VCardEmulFree) (VCardEmul *);
-typedef void (*VCardGetAtr) (VCard *, unsigned char *atr, int *atr_len);
-
-struct VCardBufferResponseStruct {
-    unsigned char *buffer;
-    int buffer_len;
-    unsigned char *current;
-    int len;
-};
-
-#endif
diff --git a/qemu/libcacard/vcardt_internal.h b/qemu/libcacard/vcardt_internal.h
deleted file mode 100644
index e5c8d2dd3..000000000
--- a/qemu/libcacard/vcardt_internal.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef VCARDT_INTERNAL_H
-#define VCARDT_INTERNAL_H
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len);
-
-#endif
diff --git a/qemu/libcacard/vevent.h b/qemu/libcacard/vevent.h
deleted file mode 100644
index 38c3482c3..000000000
--- a/qemu/libcacard/vevent.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef EVENT_H
-#define EVENT_H 1
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-VEvent *vevent_new(VEventType type, VReader *reader, VCard *card);
-void vevent_delete(VEvent *);
-
-/*
- * VEvent queueing services
- */
-void vevent_queue_vevent(VEvent *);
-void vevent_queue_init(void);
-
-/*
- *  VEvent dequeing services
- */
-VEvent *vevent_wait_next_vevent(void);
-VEvent *vevent_get_next_vevent(void);
-
-
-#endif
diff --git a/qemu/libcacard/vreader.c b/qemu/libcacard/vreader.c
deleted file mode 100644
index 9725f46a7..000000000
--- a/qemu/libcacard/vreader.c
+++ /dev/null
@@ -1,578 +0,0 @@
-/*
- * emulate the reader
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifdef G_LOG_DOMAIN
-#undef G_LOG_DOMAIN
-#endif
-#define G_LOG_DOMAIN "libcacard"
-
-#include "glib-compat.h"
-
-#include <string.h>
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-#include "vreader.h"
-#include "vevent.h"
-#include "cac.h" /* just for debugging defines */
-
-#define LIBCACARD_LOG_DOMAIN "libcacard"
-
-struct VReaderStruct {
-    int    reference_count;
-    VCard *card;
-    char *name;
-    vreader_id_t id;
-    CompatGMutex lock;
-    VReaderEmul  *reader_private;
-    VReaderEmulFree reader_private_free;
-};
-
-/*
- * Debug helpers
- */
-
-static const char *
-apdu_ins_to_string(int ins)
-{
-    switch (ins) {
-    case VCARD7816_INS_MANAGE_CHANNEL:
-        return "manage channel";
-    case VCARD7816_INS_EXTERNAL_AUTHENTICATE:
-        return "external authenticate";
-    case VCARD7816_INS_GET_CHALLENGE:
-        return "get challenge";
-    case VCARD7816_INS_INTERNAL_AUTHENTICATE:
-        return "internal authenticate";
-    case VCARD7816_INS_ERASE_BINARY:
-        return "erase binary";
-    case VCARD7816_INS_READ_BINARY:
-        return "read binary";
-    case VCARD7816_INS_WRITE_BINARY:
-        return "write binary";
-    case VCARD7816_INS_UPDATE_BINARY:
-        return "update binary";
-    case VCARD7816_INS_READ_RECORD:
-        return "read record";
-    case VCARD7816_INS_WRITE_RECORD:
-        return "write record";
-    case VCARD7816_INS_UPDATE_RECORD:
-        return "update record";
-    case VCARD7816_INS_APPEND_RECORD:
-        return "append record";
-    case VCARD7816_INS_ENVELOPE:
-        return "envelope";
-    case VCARD7816_INS_PUT_DATA:
-        return "put data";
-    case VCARD7816_INS_GET_DATA:
-        return "get data";
-    case VCARD7816_INS_SELECT_FILE:
-        return "select file";
-    case VCARD7816_INS_VERIFY:
-        return "verify";
-    case VCARD7816_INS_GET_RESPONSE:
-        return "get response";
-    case CAC_GET_PROPERTIES:
-        return "get properties";
-    case CAC_GET_ACR:
-        return "get acr";
-    case CAC_READ_BUFFER:
-        return "read buffer";
-    case CAC_UPDATE_BUFFER:
-        return "update buffer";
-    case CAC_SIGN_DECRYPT:
-        return "sign decrypt";
-    case CAC_GET_CERTIFICATE:
-        return "get certificate";
-    }
-    return "unknown";
-}
-
-/* manage locking */
-static inline void
-vreader_lock(VReader *reader)
-{
-    g_mutex_lock(&reader->lock);
-}
-
-static inline void
-vreader_unlock(VReader *reader)
-{
-    g_mutex_unlock(&reader->lock);
-}
-
-/*
- * vreader constructor
- */
-VReader *
-vreader_new(const char *name, VReaderEmul *private,
-            VReaderEmulFree private_free)
-{
-    VReader *reader;
-
-    reader = g_new(VReader, 1);
-    g_mutex_init(&reader->lock);
-    reader->reference_count = 1;
-    reader->name = g_strdup(name);
-    reader->card = NULL;
-    reader->id = (vreader_id_t)-1;
-    reader->reader_private = private;
-    reader->reader_private_free = private_free;
-    return reader;
-}
-
-/* get a reference */
-VReader*
-vreader_reference(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    vreader_lock(reader);
-    reader->reference_count++;
-    vreader_unlock(reader);
-    return reader;
-}
-
-/* free a reference */
-void
-vreader_free(VReader *reader)
-{
-    if (reader == NULL) {
-        return;
-    }
-    vreader_lock(reader);
-    if (reader->reference_count-- > 1) {
-        vreader_unlock(reader);
-        return;
-    }
-    vreader_unlock(reader);
-    g_mutex_clear(&reader->lock);
-    if (reader->card) {
-        vcard_free(reader->card);
-    }
-    g_free(reader->name);
-    if (reader->reader_private_free) {
-        reader->reader_private_free(reader->reader_private);
-    }
-    g_free(reader);
-}
-
-static VCard *
-vreader_get_card(VReader *reader)
-{
-    VCard *card;
-
-    vreader_lock(reader);
-    card = vcard_reference(reader->card);
-    vreader_unlock(reader);
-    return card;
-}
-
-VReaderStatus
-vreader_card_is_present(VReader *reader)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    vcard_free(card);
-    return VREADER_OK;
-}
-
-vreader_id_t
-vreader_get_id(VReader *reader)
-{
-    if (reader == NULL) {
-        return (vreader_id_t)-1;
-    }
-    return reader->id;
-}
-
-VReaderStatus
-vreader_set_id(VReader *reader, vreader_id_t id)
-{
-    if (reader == NULL) {
-        return VREADER_NO_CARD;
-    }
-    reader->id = id;
-    return VREADER_OK;
-}
-
-const char *
-vreader_get_name(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    return reader->name;
-}
-
-VReaderEmul *
-vreader_get_private(VReader *reader)
-{
-    return reader->reader_private;
-}
-
-static VReaderStatus
-vreader_reset(VReader *reader, VCardPower power, unsigned char *atr, int *len)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    /*
-     * clean up our state
-     */
-    vcard_reset(card, power);
-    if (atr) {
-        vcard_get_atr(card, atr, len);
-    }
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-VReaderStatus
-vreader_power_on(VReader *reader, unsigned char *atr, int *len)
-{
-    return vreader_reset(reader, VCARD_POWER_ON, atr, len);
-}
-
-VReaderStatus
-vreader_power_off(VReader *reader)
-{
-    return vreader_reset(reader, VCARD_POWER_OFF, NULL, 0);
-}
-
-
-VReaderStatus
-vreader_xfr_bytes(VReader *reader,
-                  unsigned char *send_buf, int send_buf_len,
-                  unsigned char *receive_buf, int *receive_buf_len)
-{
-    VCardAPDU *apdu;
-    VCardResponse *response = NULL;
-    VCardStatus card_status;
-    unsigned short status;
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-
-    apdu = vcard_apdu_new(send_buf, send_buf_len, &status);
-    if (apdu == NULL) {
-        response = vcard_make_response(status);
-        card_status = VCARD_DONE;
-    } else {
-        g_debug("%s: CLS=0x%x,INS=0x%x,P1=0x%x,P2=0x%x,Lc=%d,Le=%d %s",
-              __func__, apdu->a_cla, apdu->a_ins, apdu->a_p1, apdu->a_p2,
-              apdu->a_Lc, apdu->a_Le, apdu_ins_to_string(apdu->a_ins));
-        card_status = vcard_process_apdu(card, apdu, &response);
-        if (response) {
-            g_debug("%s: status=%d sw1=0x%x sw2=0x%x len=%d (total=%d)",
-                  __func__, response->b_status, response->b_sw1,
-                  response->b_sw2, response->b_len, response->b_total_len);
-        }
-    }
-    assert(card_status == VCARD_DONE && response);
-    int size = MIN(*receive_buf_len, response->b_total_len);
-    memcpy(receive_buf, response->b_data, size);
-    *receive_buf_len = size;
-    vcard_response_delete(response);
-    vcard_apdu_delete(apdu);
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-struct VReaderListStruct {
-    VReaderListEntry *head;
-    VReaderListEntry *tail;
-};
-
-struct VReaderListEntryStruct {
-    VReaderListEntry *next;
-    VReaderListEntry *prev;
-    VReader *reader;
-};
-
-
-static VReaderListEntry *
-vreader_list_entry_new(VReader *reader)
-{
-    VReaderListEntry *new_reader_list_entry;
-
-    new_reader_list_entry = g_new0(VReaderListEntry, 1);
-    new_reader_list_entry->reader = vreader_reference(reader);
-    return new_reader_list_entry;
-}
-
-static void
-vreader_list_entry_delete(VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    vreader_free(entry->reader);
-    g_free(entry);
-}
-
-
-static VReaderList *
-vreader_list_new(void)
-{
-    VReaderList *new_reader_list;
-
-    new_reader_list = g_new0(VReaderList, 1);
-    return new_reader_list;
-}
-
-void
-vreader_list_delete(VReaderList *list)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = next_entry) {
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_list_entry_delete(current_entry);
-    }
-    g_free(list);
-}
-
-
-VReaderListEntry *
-vreader_list_get_first(VReaderList *list)
-{
-    return list ? list->head : NULL;
-}
-
-VReaderListEntry *
-vreader_list_get_next(VReaderListEntry *current)
-{
-    return current ? current->next : NULL;
-}
-
-VReader *
-vreader_list_get_reader(VReaderListEntry *entry)
-{
-    return entry ? vreader_reference(entry->reader) : NULL;
-}
-
-static void
-vreader_queue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    entry->next = NULL;
-    entry->prev = list->tail;
-    if (list->head) {
-        list->tail->next = entry;
-    } else {
-        list->head = entry;
-    }
-    list->tail = entry;
-}
-
-static void
-vreader_dequeue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    if (entry->next == NULL) {
-        list->tail = entry->prev;
-    } else if (entry->prev == NULL) {
-        list->head = entry->next;
-    } else {
-        entry->prev->next = entry->next;
-        entry->next->prev = entry->prev;
-    }
-    if ((list->tail == NULL) || (list->head == NULL)) {
-        list->head = list->tail = NULL;
-    }
-    entry->next = entry->prev = NULL;
-}
-
-static VReaderList *vreader_list;
-static CompatGMutex vreader_list_mutex;
-
-static void
-vreader_list_init(void)
-{
-    vreader_list = vreader_list_new();
-}
-
-static void
-vreader_list_lock(void)
-{
-    g_mutex_lock(&vreader_list_mutex);
-}
-
-static void
-vreader_list_unlock(void)
-{
-    g_mutex_unlock(&vreader_list_mutex);
-}
-
-static VReaderList *
-vreader_copy_list(VReaderList *list)
-{
-    VReaderList *new_list;
-    VReaderListEntry *current_entry;
-
-    new_list = vreader_list_new();
-    if (new_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderListEntry *new_entry = vreader_list_entry_new(reader);
-
-        vreader_free(reader);
-        vreader_queue(new_list, new_entry);
-    }
-    return new_list;
-}
-
-VReaderList *
-vreader_get_reader_list(void)
-{
-    VReaderList *new_reader_list;
-
-    vreader_list_lock();
-    new_reader_list = vreader_copy_list(vreader_list);
-    vreader_list_unlock();
-    return new_reader_list;
-}
-
-VReader *
-vreader_get_reader_by_id(vreader_id_t id)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    if (id == (vreader_id_t) -1) {
-        return NULL;
-    }
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (creader->id == id) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-VReader *
-vreader_get_reader_by_name(const char *name)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (strcmp(creader->name, name) == 0) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-/* called from card_emul to initialize the readers */
-VReaderStatus
-vreader_add_reader(VReader *reader)
-{
-    VReaderListEntry *reader_entry;
-
-    reader_entry = vreader_list_entry_new(reader);
-    if (reader_entry == NULL) {
-        return VREADER_OUT_OF_MEMORY;
-    }
-    vreader_list_lock();
-    vreader_queue(vreader_list, reader_entry);
-    vreader_list_unlock();
-    vevent_queue_vevent(vevent_new(VEVENT_READER_INSERT, reader, NULL));
-    return VREADER_OK;
-}
-
-
-VReaderStatus
-vreader_remove_reader(VReader *reader)
-{
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        if (current_entry->reader == reader) {
-            break;
-        }
-    }
-    vreader_dequeue(vreader_list, current_entry);
-    vreader_list_unlock();
-    vreader_list_entry_delete(current_entry);
-    vevent_queue_vevent(vevent_new(VEVENT_READER_REMOVE, reader, NULL));
-    return VREADER_OK;
-}
-
-/*
- * Generate VEVENT_CARD_INSERT or VEVENT_CARD_REMOVE based on vreader
- * state. Separated from vreader_insert_card to allow replaying events
- * for a given state.
- */
-void
-vreader_queue_card_event(VReader *reader)
-{
-    vevent_queue_vevent(vevent_new(
-        reader->card ? VEVENT_CARD_INSERT : VEVENT_CARD_REMOVE, reader,
-        reader->card));
-}
-
-/*
- * insert/remove a new card. for removal, card == NULL
- */
-VReaderStatus
-vreader_insert_card(VReader *reader, VCard *card)
-{
-    vreader_lock(reader);
-    if (reader->card) {
-        /* decrement reference count */
-        vcard_free(reader->card);
-        reader->card = NULL;
-    }
-    reader->card = vcard_reference(card);
-    vreader_unlock(reader);
-    vreader_queue_card_event(reader);
-    return VREADER_OK;
-}
-
-/*
- * initialize all the static reader structures
- */
-void
-vreader_init(void)
-{
-    vreader_list_init();
-}
-
diff --git a/qemu/libcacard/vreader.h b/qemu/libcacard/vreader.h
deleted file mode 100644
index ec2042136..000000000
--- a/qemu/libcacard/vreader.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADER_H
-#define VREADER_H 1
-
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-/*
- * calls for reader front end
- */
-VReaderStatus vreader_power_on(VReader *reader, unsigned char *atr, int *len);
-VReaderStatus vreader_power_off(VReader *reader);
-VReaderStatus vreader_xfr_bytes(VReader *reader, unsigned char *send_buf,
-                                int send_buf_len, unsigned char *receive_buf,
-                                int *receive_buf_len);
-
-/* constructor */
-VReader *vreader_new(const char *readerName, VReaderEmul *emul_private,
-                     VReaderEmulFree private_free);
-/* get a new reference to a reader */
-VReader *vreader_reference(VReader *reader);
-/* "destructor" (readers are reference counted) */
-void vreader_free(VReader *reader);
-
-/* accessors */
-VReaderEmul *vreader_get_private(VReader *);
-VReaderStatus vreader_card_is_present(VReader *reader);
-void vreader_queue_card_event(VReader *reader);
-const char *vreader_get_name(VReader *reader);
-vreader_id_t vreader_get_id(VReader *reader);
-VReaderStatus vreader_set_id(VReader *reader, vreader_id_t id);
-
-/* list operations */
-VReaderList *vreader_get_reader_list(void);
-void vreader_list_delete(VReaderList *list);
-VReader *vreader_list_get_reader(VReaderListEntry *entry);
-VReaderListEntry *vreader_list_get_first(VReaderList *list);
-VReaderListEntry *vreader_list_get_next(VReaderListEntry *list);
-VReader *vreader_get_reader_by_id(vreader_id_t id);
-VReader *vreader_get_reader_by_name(const char *name);
-
-/*
- * list tools for vcard_emul
- */
-void vreader_init(void);
-VReaderStatus vreader_add_reader(VReader *reader);
-VReaderStatus vreader_remove_reader(VReader *reader);
-VReaderStatus vreader_insert_card(VReader *reader, VCard *card);
-
-#endif
diff --git a/qemu/libcacard/vreadert.h b/qemu/libcacard/vreadert.h
deleted file mode 100644
index f97e0a79e..000000000
--- a/qemu/libcacard/vreadert.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADERT_H
-#define VREADERT_H 1
-
-typedef enum {
-    VREADER_OK = 0,
-    VREADER_NO_CARD,
-    VREADER_OUT_OF_MEMORY
-} VReaderStatus;
-
-typedef unsigned int vreader_id_t;
-typedef struct VReaderStruct VReader;
-typedef struct VReaderListStruct VReaderList;
-typedef struct VReaderListEntryStruct VReaderListEntry;
-
-typedef struct VReaderEmulStruct VReaderEmul;
-typedef void (*VReaderEmulFree)(VReaderEmul *);
-
-#endif
-
diff --git a/qemu/libcacard/vscard_common.h b/qemu/libcacard/vscard_common.h
deleted file mode 100644
index 08f68e4dd..000000000
--- a/qemu/libcacard/vscard_common.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/* Virtual Smart Card protocol definition
- *
- * This protocol is between a host using virtual smart card readers,
- * and a client providing the smart cards, perhaps by emulating them or by
- * access to real cards.
- *
- * Definitions for this protocol:
- *  Host   - user of the card
- *  Client - owner of the card
- *
- * The current implementation passes the raw APDU's from 7816 and additionally
- * contains messages to setup and teardown readers, handle insertion and
- * removal of cards, negotiate the protocol via capabilities and provide
- * for error responses.
- *
- * Copyright (c) 2011 Red Hat.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VSCARD_COMMON_H
-#define VSCARD_COMMON_H
-
-#include <stdint.h>
-
-#define VERSION_MAJOR_BITS 11
-#define VERSION_MIDDLE_BITS 11
-#define VERSION_MINOR_BITS 10
-
-#define MAKE_VERSION(major, middle, minor) \
-     ((major  << (VERSION_MINOR_BITS + VERSION_MIDDLE_BITS)) \
-      | (middle <<  VERSION_MINOR_BITS) \
-      | (minor))
-
-/*
- * IMPORTANT NOTE on VERSION
- *
- * The version below MUST be changed whenever a change in this file is made.
- *
- * The last digit, the minor, is for bug fix changes only.
- *
- * The middle digit is for backward / forward compatible changes, updates
- * to the existing messages, addition of fields.
- *
- * The major digit is for a breaking change of protocol, presumably
- * something that cannot be accommodated with the existing protocol.
- */
-
-#define VSCARD_VERSION MAKE_VERSION(0, 0, 2)
-
-typedef enum VSCMsgType {
-    VSC_Init = 1,
-    VSC_Error,
-    VSC_ReaderAdd,
-    VSC_ReaderRemove,
-    VSC_ATR,
-    VSC_CardRemove,
-    VSC_APDU,
-    VSC_Flush,
-    VSC_FlushComplete
-} VSCMsgType;
-
-typedef enum VSCErrorCode {
-    VSC_SUCCESS = 0,
-    VSC_GENERAL_ERROR = 1,
-    VSC_CANNOT_ADD_MORE_READERS,
-    VSC_CARD_ALREAY_INSERTED,
-} VSCErrorCode;
-
-#define VSCARD_UNDEFINED_READER_ID  0xffffffff
-#define VSCARD_MINIMAL_READER_ID    0
-
-#define VSCARD_MAGIC (*(uint32_t *)"VSCD")
-
-/*
- * Header
- * Each message starts with the header.
- * type - message type
- * reader_id - used by messages that are reader specific
- * length - length of payload (not including header, i.e. zero for
- *  messages containing empty payloads)
- */
-typedef struct VSCMsgHeader {
-    uint32_t   type;
-    uint32_t   reader_id;
-    uint32_t   length;
-    uint8_t    data[0];
-} VSCMsgHeader;
-
-/*
- * VSCMsgInit               Client <-> Host
- * Client sends it on connection, with its own capabilities.
- * Host replies with VSCMsgInit filling in its capabilities.
- *
- * It is not meant to be used for negotiation, i.e. sending more then
- * once from any side, but could be used for that in the future.
- */
-typedef struct VSCMsgInit {
-    uint32_t   magic;
-    uint32_t   version;
-    uint32_t   capabilities[1]; /* receiver must check length,
-                                   array may grow in the future*/
-} VSCMsgInit;
-
-/*
- * VSCMsgError              Client <-> Host
- * This message is a response to any of:
- *  Reader Add
- *  Reader Remove
- *  Card Remove
- * If the operation was successful then VSC_SUCCESS
- * is returned, other wise a specific error code.
- */
-typedef struct VSCMsgError {
-    uint32_t   code;
-} VSCMsgError;
-
-/*
- * VSCMsgReaderAdd          Client -> Host
- * Host replies with allocated reader id in VSCMsgError with code==SUCCESS.
- *
- * name - name of the reader on client side, UTF-8 encoded. Only used
- *  for client presentation (may be translated to the device presented to the
- *  guest), protocol wise only reader_id is important.
- */
-typedef struct VSCMsgReaderAdd {
-    uint8_t    name[0];
-} VSCMsgReaderAdd;
-
-/*
- * VSCMsgReaderRemove       Client -> Host
- * The client's reader has been removed.
- */
-typedef struct VSCMsgReaderRemove {
-} VSCMsgReaderRemove;
-
-/*
- * VSCMsgATR                Client -> Host
- * Answer to reset. Sent for card insertion or card reset. The reset/insertion
- * happens on the client side, they do not require any action from the host.
- */
-typedef struct VSCMsgATR {
-    uint8_t     atr[0];
-} VSCMsgATR;
-
-/*
- * VSCMsgCardRemove         Client -> Host
- * The client card has been removed.
- */
-typedef struct VSCMsgCardRemove {
-} VSCMsgCardRemove;
-
-/*
- * VSCMsgAPDU               Client <-> Host
- * Main reason of existence. Transfer a single APDU in either direction.
- */
-typedef struct VSCMsgAPDU {
-    uint8_t    data[0];
-} VSCMsgAPDU;
-
-/*
- * VSCMsgFlush               Host -> Client
- * Request client to send a FlushComplete message when it is done
- * servicing all outstanding APDUs
- */
-typedef struct VSCMsgFlush {
-} VSCMsgFlush;
-
-/*
- * VSCMsgFlush               Client -> Host
- * Client response to Flush after all APDUs have been processed and
- * responses sent.
- */
-typedef struct VSCMsgFlushComplete {
-} VSCMsgFlushComplete;
-
-#endif /* VSCARD_COMMON_H */
diff --git a/qemu/libcacard/vscclient.c b/qemu/libcacard/vscclient.c
deleted file mode 100644
index 065268443..000000000
--- a/qemu/libcacard/vscclient.c
+++ /dev/null
@@ -1,785 +0,0 @@
-/*
- * Tester for VSCARD protocol, client side.
- *
- * Can be used with ccid-card-passthru.
- *
- * Copyright (c) 2011 Red Hat.
- * Written by Alon Levy.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#ifndef _WIN32
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netdb.h>
-#include <unistd.h>
-#define closesocket(x) close(x)
-#else
-#include <getopt.h>
-#endif
-
-#include "glib-compat.h"
-
-#include "vscard_common.h"
-
-#include "vreader.h"
-#include "vcard_emul.h"
-#include "vevent.h"
-
-static int verbose;
-
-static void
-print_byte_array(
-    uint8_t *arrBytes,
-    unsigned int nSize
-) {
-    int i;
-    for (i = 0; i < nSize; i++) {
-        printf("%02X ", arrBytes[i]);
-    }
-    printf("\n");
-}
-
-static void
-print_usage(void) {
-    printf("vscclient [-c <certname> .. -e <emul_args> -d <level>%s] "
-            "<host> <port>\n",
-#ifdef USE_PASSTHRU
-    " -p");
-    printf(" -p use passthrough mode\n");
-#else
-   "");
-#endif
-    vcard_emul_usage();
-}
-
-static GIOChannel *channel_socket;
-static GByteArray *socket_to_send;
-static CompatGMutex socket_to_send_lock;
-static guint socket_tag;
-
-static void
-update_socket_watch(void);
-
-static gboolean
-do_socket_send(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    gsize bw;
-    GError *err = NULL;
-
-    g_return_val_if_fail(socket_to_send->len != 0, FALSE);
-    g_return_val_if_fail(condition & G_IO_OUT, FALSE);
-
-    g_io_channel_write_chars(channel_socket,
-        (gchar *)socket_to_send->data, socket_to_send->len, &bw, &err);
-    if (err != NULL) {
-        g_error("Error while sending socket %s", err->message);
-        return FALSE;
-    }
-    g_byte_array_remove_range(socket_to_send, 0, bw);
-
-    if (socket_to_send->len == 0) {
-        update_socket_watch();
-        return FALSE;
-    }
-    return TRUE;
-}
-
-static gboolean
-socket_prepare_sending(gpointer user_data)
-{
-    update_socket_watch();
-
-    return FALSE;
-}
-
-static int
-send_msg(
-    VSCMsgType type,
-    uint32_t reader_id,
-    const void *msg,
-    unsigned int length
-) {
-    VSCMsgHeader mhHeader;
-
-    g_mutex_lock(&socket_to_send_lock);
-
-    if (verbose > 10) {
-        printf("sending type=%d id=%u, len =%u (0x%x)\n",
-               type, reader_id, length, length);
-    }
-
-    mhHeader.type = htonl(type);
-    mhHeader.reader_id = 0;
-    mhHeader.length = htonl(length);
-    g_byte_array_append(socket_to_send, (guint8 *)&mhHeader, sizeof(mhHeader));
-    g_byte_array_append(socket_to_send, (guint8 *)msg, length);
-    g_idle_add(socket_prepare_sending, NULL);
-
-    g_mutex_unlock(&socket_to_send_lock);
-
-    return 0;
-}
-
-static VReader *pending_reader;
-static CompatGMutex pending_reader_lock;
-static CompatGCond pending_reader_condition;
-
-#define MAX_ATR_LEN 40
-static gpointer
-event_thread(gpointer arg)
-{
-    unsigned char atr[MAX_ATR_LEN];
-    int atr_len;
-    VEvent *event;
-    unsigned int reader_id;
-
-
-    while (1) {
-        const char *reader_name;
-
-        event = vevent_wait_next_vevent();
-        if (event == NULL) {
-            break;
-        }
-        reader_id = vreader_get_id(event->reader);
-        if (reader_id == VSCARD_UNDEFINED_READER_ID &&
-            event->type != VEVENT_READER_INSERT) {
-            /* ignore events from readers qemu has rejected */
-            /* if qemu is still deciding on this reader, wait to see if need to
-             * forward this event */
-            g_mutex_lock(&pending_reader_lock);
-            if (!pending_reader || (pending_reader != event->reader)) {
-                /* wasn't for a pending reader, this reader has already been
-                 * rejected by qemu */
-                g_mutex_unlock(&pending_reader_lock);
-                vevent_delete(event);
-                continue;
-            }
-            /* this reader hasn't been told its status from qemu yet, wait for
-             * that status */
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            g_mutex_unlock(&pending_reader_lock);
-            /* now recheck the id */
-            reader_id = vreader_get_id(event->reader);
-            if (reader_id == VSCARD_UNDEFINED_READER_ID) {
-                /* this reader was rejected */
-                vevent_delete(event);
-                continue;
-            }
-            /* reader was accepted, now forward the event */
-        }
-        switch (event->type) {
-        case VEVENT_READER_INSERT:
-            /* tell qemu to insert a new CCID reader */
-            /* wait until qemu has responded to our first reader insert
-             * before we send a second. That way we won't confuse the responses
-             * */
-            g_mutex_lock(&pending_reader_lock);
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            pending_reader = vreader_reference(event->reader);
-            g_mutex_unlock(&pending_reader_lock);
-            reader_name = vreader_get_name(event->reader);
-            if (verbose > 10) {
-                printf(" READER INSERT: %s\n", reader_name);
-            }
-            send_msg(VSC_ReaderAdd,
-                reader_id, /* currerntly VSCARD_UNDEFINED_READER_ID */
-                NULL, 0 /* TODO reader_name, strlen(reader_name) */);
-            break;
-        case VEVENT_READER_REMOVE:
-            /* future, tell qemu that an old CCID reader has been removed */
-            if (verbose > 10) {
-                printf(" READER REMOVE: %u\n", reader_id);
-            }
-            send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            break;
-        case VEVENT_CARD_INSERT:
-            /* get the ATR (intended as a response to a power on from the
-             * reader */
-            atr_len = MAX_ATR_LEN;
-            vreader_power_on(event->reader, atr, &atr_len);
-            /* ATR call functions as a Card Insert event */
-            if (verbose > 10) {
-                printf(" CARD INSERT %u: ", reader_id);
-                print_byte_array(atr, atr_len);
-            }
-            send_msg(VSC_ATR, reader_id, atr, atr_len);
-            break;
-        case VEVENT_CARD_REMOVE:
-            /* Card removed */
-            if (verbose > 10) {
-                printf(" CARD REMOVE %u:\n", reader_id);
-            }
-            send_msg(VSC_CardRemove, reader_id, NULL, 0);
-            break;
-        default:
-            break;
-        }
-        vevent_delete(event);
-    }
-    return NULL;
-}
-
-
-static unsigned int
-get_id_from_string(char *string, unsigned int default_id)
-{
-    unsigned int id = atoi(string);
-
-    /* don't accidentally swith to zero because no numbers have been supplied */
-    if ((id == 0) && *string != '0') {
-        return default_id;
-    }
-    return id;
-}
-
-static int
-on_host_init(VSCMsgHeader *mhHeader, VSCMsgInit *incoming)
-{
-    uint32_t *capabilities = (incoming->capabilities);
-    int num_capabilities =
-        1 + ((mhHeader->length - sizeof(VSCMsgInit)) / sizeof(uint32_t));
-    int i;
-
-    incoming->version = ntohl(incoming->version);
-    if (incoming->version != VSCARD_VERSION) {
-        if (verbose > 0) {
-            printf("warning: host has version %d, we have %d\n",
-                verbose, VSCARD_VERSION);
-        }
-    }
-    if (incoming->magic != VSCARD_MAGIC) {
-        printf("unexpected magic: got %d, expected %d\n",
-            incoming->magic, VSCARD_MAGIC);
-        return -1;
-    }
-    for (i = 0 ; i < num_capabilities; ++i) {
-        capabilities[i] = ntohl(capabilities[i]);
-    }
-    /* Future: check capabilities */
-    /* remove whatever reader might be left in qemu,
-     * in case of an unclean previous exit. */
-    send_msg(VSC_ReaderRemove, VSCARD_MINIMAL_READER_ID, NULL, 0);
-    /* launch the event_thread. This will trigger reader adds for all the
-     * existing readers */
-    g_thread_new("vsc/event", event_thread, NULL);
-    return 0;
-}
-
-
-enum {
-    STATE_HEADER,
-    STATE_MESSAGE,
-};
-
-#define APDUBufSize 270
-
-static gboolean
-do_socket_read(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    int rv;
-    int dwSendLength;
-    int dwRecvLength;
-    uint8_t pbRecvBuffer[APDUBufSize];
-    static uint8_t pbSendBuffer[APDUBufSize];
-    VReaderStatus reader_status;
-    VReader *reader = NULL;
-    static VSCMsgHeader mhHeader;
-    VSCMsgError *error_msg;
-    GError *err = NULL;
-
-    static gchar *buf;
-    static gsize br, to_read;
-    static int state = STATE_HEADER;
-
-    if (state == STATE_HEADER && to_read == 0) {
-        buf = (gchar *)&mhHeader;
-        to_read = sizeof(mhHeader);
-    }
-
-    if (to_read > 0) {
-        g_io_channel_read_chars(source, (gchar *)buf, to_read, &br, &err);
-        if (err != NULL) {
-            g_error("error while reading: %s", err->message);
-        }
-        buf += br;
-        to_read -= br;
-        if (to_read != 0) {
-            return TRUE;
-        }
-    }
-
-    if (state == STATE_HEADER) {
-        mhHeader.type = ntohl(mhHeader.type);
-        mhHeader.reader_id = ntohl(mhHeader.reader_id);
-        mhHeader.length = ntohl(mhHeader.length);
-        if (verbose) {
-            printf("Header: type=%d, reader_id=%u length=%d (0x%x)\n",
-                   mhHeader.type, mhHeader.reader_id, mhHeader.length,
-                   mhHeader.length);
-        }
-        switch (mhHeader.type) {
-        case VSC_APDU:
-        case VSC_Flush:
-        case VSC_Error:
-        case VSC_Init:
-            buf = (gchar *)pbSendBuffer;
-            to_read = mhHeader.length;
-            state = STATE_MESSAGE;
-            return TRUE;
-        default:
-            fprintf(stderr, "Unexpected message of type 0x%X\n", mhHeader.type);
-            return FALSE;
-        }
-    }
-
-    if (state == STATE_MESSAGE) {
-        switch (mhHeader.type) {
-        case VSC_APDU:
-            if (verbose) {
-                printf(" recv APDU: ");
-                print_byte_array(pbSendBuffer, mhHeader.length);
-            }
-            /* Transmit received APDU */
-            dwSendLength = mhHeader.length;
-            dwRecvLength = sizeof(pbRecvBuffer);
-            reader = vreader_get_reader_by_id(mhHeader.reader_id);
-            reader_status = vreader_xfr_bytes(reader,
-                                              pbSendBuffer, dwSendLength,
-                                              pbRecvBuffer, &dwRecvLength);
-            if (reader_status == VREADER_OK) {
-                mhHeader.length = dwRecvLength;
-                if (verbose) {
-                    printf(" send response: ");
-                    print_byte_array(pbRecvBuffer, mhHeader.length);
-                }
-                send_msg(VSC_APDU, mhHeader.reader_id,
-                         pbRecvBuffer, dwRecvLength);
-            } else {
-                rv = reader_status; /* warning: not meaningful */
-                send_msg(VSC_Error, mhHeader.reader_id, &rv, sizeof(uint32_t));
-            }
-            vreader_free(reader);
-            reader = NULL; /* we've freed it, don't use it by accident
-                              again */
-            break;
-        case VSC_Flush:
-            /* TODO: actually flush */
-            send_msg(VSC_FlushComplete, mhHeader.reader_id, NULL, 0);
-            break;
-        case VSC_Error:
-            error_msg = (VSCMsgError *) pbSendBuffer;
-            if (error_msg->code == VSC_SUCCESS) {
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    vreader_set_id(pending_reader, mhHeader.reader_id);
-                    vreader_free(pending_reader);
-                    pending_reader = NULL;
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-                break;
-            }
-            printf("warning: qemu refused to add reader\n");
-            if (error_msg->code == VSC_CANNOT_ADD_MORE_READERS) {
-                /* clear pending reader, qemu can't handle any more */
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    pending_reader = NULL;
-                    /* make sure the event loop doesn't hang */
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-            }
-            break;
-        case VSC_Init:
-            if (on_host_init(&mhHeader, (VSCMsgInit *)pbSendBuffer) < 0) {
-                return FALSE;
-            }
-            break;
-        default:
-            g_assert_not_reached();
-            return FALSE;
-        }
-
-        state = STATE_HEADER;
-    }
-
-
-    return TRUE;
-}
-
-static gboolean
-do_socket(GIOChannel *source,
-          GIOCondition condition,
-          gpointer data)
-{
-    /* not sure if two watches work well with a single win32 sources */
-    if (condition & G_IO_OUT) {
-        if (!do_socket_send(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    if (condition & G_IO_IN) {
-        if (!do_socket_read(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    return TRUE;
-}
-
-static void
-update_socket_watch(void)
-{
-    gboolean out = socket_to_send->len > 0;
-
-    if (socket_tag != 0) {
-        g_source_remove(socket_tag);
-    }
-
-    socket_tag = g_io_add_watch(channel_socket,
-        G_IO_IN | (out ? G_IO_OUT : 0), do_socket, NULL);
-}
-
-static gboolean
-do_command(GIOChannel *source,
-           GIOCondition condition,
-           gpointer data)
-{
-    char *string;
-    VCardEmulError error;
-    static unsigned int default_reader_id;
-    unsigned int reader_id;
-    VReader *reader = NULL;
-    GError *err = NULL;
-
-    g_assert(condition & G_IO_IN);
-
-    reader_id = default_reader_id;
-    g_io_channel_read_line(source, &string, NULL, NULL, &err);
-    if (err != NULL) {
-        g_error("Error while reading command: %s", err->message);
-    }
-
-    if (string != NULL) {
-        if (strncmp(string, "exit", 4) == 0) {
-            /* remove all the readers */
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                /* be nice and signal card removal first (qemu probably should
-                 * do this itself) */
-                if (vreader_card_is_present(reader) == VREADER_OK) {
-                    send_msg(VSC_CardRemove, reader_id, NULL, 0);
-                }
-                send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            }
-            exit(0);
-        } else if (strncmp(string, "insert", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_insert(reader);
-                printf("insert %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "remove", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_remove(reader);
-                printf("remove %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "select", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7],
-                                               VSCARD_UNDEFINED_READER_ID);
-            }
-            if (reader_id != VSCARD_UNDEFINED_READER_ID) {
-                reader = vreader_get_reader_by_id(reader_id);
-            }
-            if (reader) {
-                printf("Selecting reader %u, %s\n", reader_id,
-                        vreader_get_name(reader));
-                default_reader_id = reader_id;
-            } else {
-                printf("Reader with id %u not found\n", reader_id);
-            }
-        } else if (strncmp(string, "debug", 5) == 0) {
-            if (string[5] == ' ') {
-                verbose = get_id_from_string(&string[6], 0);
-            }
-            printf("debug level = %d\n", verbose);
-        } else if (strncmp(string, "list", 4) == 0) {
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                printf("%3u %s %s\n", reader_id,
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            printf("Inactive Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id != -1) {
-                    continue;
-                }
-
-                printf("INA %s %s\n",
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            vreader_list_delete(list);
-        } else if (*string != 0) {
-            printf("valid commands:\n");
-            printf("insert [reader_id]\n");
-            printf("remove [reader_id]\n");
-            printf("select reader_id\n");
-            printf("list\n");
-            printf("debug [level]\n");
-            printf("exit\n");
-        }
-    }
-    vreader_free(reader);
-    printf("> ");
-    fflush(stdout);
-
-    return TRUE;
-}
-
-
-/* just for ease of parsing command line arguments. */
-#define MAX_CERTS 100
-
-static int
-connect_to_qemu(
-    const char *host,
-    const char *port
-) {
-    struct addrinfo hints;
-    struct addrinfo *server = NULL;
-    int ret, sock;
-
-    sock = socket(AF_INET, SOCK_STREAM, 0);
-    if (sock < 0) {
-        /* Error */
-        fprintf(stderr, "Error opening socket!\n");
-        return -1;
-    }
-
-    memset(&hints, 0, sizeof(struct addrinfo));
-    hints.ai_family = AF_UNSPEC;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_flags = 0;
-    hints.ai_protocol = 0;          /* Any protocol */
-
-    ret = getaddrinfo(host, port, &hints, &server);
-
-    if (ret != 0) {
-        /* Error */
-        fprintf(stderr, "getaddrinfo failed\n");
-        goto cleanup_socket;
-    }
-
-    if (connect(sock, server->ai_addr, server->ai_addrlen) < 0) {
-        /* Error */
-        fprintf(stderr, "Could not connect\n");
-        goto cleanup_socket;
-    }
-    if (verbose) {
-        printf("Connected (sizeof Header=%zd)!\n", sizeof(VSCMsgHeader));
-    }
-
-    freeaddrinfo(server);
-    return sock;
-
-cleanup_socket:
-    if (server) {
-        freeaddrinfo(server);
-    }
-    closesocket(sock);
-    return -1;
-}
-
-int
-main(
-    int argc,
-    char *argv[]
-) {
-    GMainLoop *loop;
-    GIOChannel *channel_stdin;
-    char *qemu_host;
-    char *qemu_port;
-
-    VCardEmulOptions *command_line_options = NULL;
-
-    char *cert_names[MAX_CERTS];
-    char *emul_args = NULL;
-    int cert_count = 0;
-    int c, sock;
-
-#ifdef _WIN32
-    WSADATA Data;
-
-    if (WSAStartup(MAKEWORD(2, 2), &Data) != 0) {
-        c = WSAGetLastError();
-        fprintf(stderr, "WSAStartup: %d\n", c);
-        return 1;
-    }
-#endif
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-    if (!g_thread_supported()) {
-         g_thread_init(NULL);
-    }
-#endif
-
-    while ((c = getopt(argc, argv, "c:e:pd:")) != -1) {
-        switch (c) {
-        case 'c':
-            if (cert_count >= MAX_CERTS) {
-                printf("too many certificates (max = %d)\n", MAX_CERTS);
-                exit(5);
-            }
-            cert_names[cert_count++] = optarg;
-            break;
-        case 'e':
-            emul_args = optarg;
-            break;
-        case 'p':
-            print_usage();
-            exit(4);
-            break;
-        case 'd':
-            verbose = get_id_from_string(optarg, 1);
-            break;
-        }
-    }
-
-    if (argc - optind != 2) {
-        print_usage();
-        exit(4);
-    }
-
-    if (cert_count > 0) {
-        char *new_args;
-        int len, i;
-        /* if we've given some -c options, we clearly we want do so some
-         * software emulation.  add that emulation now. this is NSS Emulator
-         * specific */
-        if (emul_args == NULL) {
-            emul_args = (char *)"db=\"/etc/pki/nssdb\"";
-        }
-#define SOFT_STRING ",soft=(,Virtual Reader,CAC,,"
-             /* 2 == close paren & null */
-        len = strlen(emul_args) + strlen(SOFT_STRING) + 2;
-        for (i = 0; i < cert_count; i++) {
-            len += strlen(cert_names[i])+1; /* 1 == comma */
-        }
-        new_args = g_malloc(len);
-        strcpy(new_args, emul_args);
-        strcat(new_args, SOFT_STRING);
-        for (i = 0; i < cert_count; i++) {
-            strcat(new_args, cert_names[i]);
-            strcat(new_args, ",");
-        }
-        strcat(new_args, ")");
-        emul_args = new_args;
-    }
-    if (emul_args) {
-        command_line_options = vcard_emul_options(emul_args);
-    }
-
-    qemu_host = g_strdup(argv[argc - 2]);
-    qemu_port = g_strdup(argv[argc - 1]);
-    sock = connect_to_qemu(qemu_host, qemu_port);
-    if (sock == -1) {
-        fprintf(stderr, "error opening socket, exiting.\n");
-        exit(5);
-    }
-
-    socket_to_send = g_byte_array_new();
-    vcard_emul_init(command_line_options);
-    loop = g_main_loop_new(NULL, TRUE);
-
-    printf("> ");
-    fflush(stdout);
-
-#ifdef _WIN32
-    channel_stdin = g_io_channel_win32_new_fd(STDIN_FILENO);
-#else
-    channel_stdin = g_io_channel_unix_new(STDIN_FILENO);
-#endif
-    g_io_add_watch(channel_stdin, G_IO_IN, do_command, NULL);
-#ifdef _WIN32
-    channel_socket = g_io_channel_win32_new_socket(sock);
-#else
-    channel_socket = g_io_channel_unix_new(sock);
-#endif
-    g_io_channel_set_encoding(channel_socket, NULL, NULL);
-    /* we buffer ourself for thread safety reasons */
-    g_io_channel_set_buffered(channel_socket, FALSE);
-
-    /* Send init message, Host responds (and then we send reader attachments) */
-    VSCMsgInit init = {
-        .version = htonl(VSCARD_VERSION),
-        .magic = VSCARD_MAGIC,
-        .capabilities = {0}
-    };
-    send_msg(VSC_Init, 0, &init, sizeof(init));
-
-    g_main_loop_run(loop);
-    g_main_loop_unref(loop);
-
-    g_io_channel_unref(channel_stdin);
-    g_io_channel_unref(channel_socket);
-    g_byte_array_free(socket_to_send, TRUE);
-
-    closesocket(sock);
-    return 0;
-}
diff --git a/qemu/libdecnumber/decContext.c b/qemu/libdecnumber/decContext.c
index 8b6ae21be..7d97a65ac 100644
--- a/qemu/libdecnumber/decContext.c
+++ b/qemu/libdecnumber/decContext.c
@@ -35,8 +35,7 @@
 /* context structures.						      */
 /* ------------------------------------------------------------------ */
 
-#include <string.h>	      /* for strcmp */
-#include <stdio.h>	      /* for printf if DECCHECK */
+#include "qemu/osdep.h"
 #include "libdecnumber/dconfig.h"
 #include "libdecnumber/decContext.h"
 #include "libdecnumber/decNumberLocal.h"
@@ -186,7 +185,7 @@ uInt decContextGetStatus(decContext *context) {
 /*  newstatus is the source for the bits to be restored		      */
 /*  mask indicates the bits to be restored (the status bit that	      */
 /*    corresponds to each 1 bit in the mask is set to the value of    */
-/*    the correspnding bit in newstatus)			      */
+/*    the corresponding bit in newstatus)			      */
 /*  returns context						      */
 /*								      */
 /* No error is possible.					      */
diff --git a/qemu/libdecnumber/decNumber.c b/qemu/libdecnumber/decNumber.c
index 58211e7af..c9e7807f8 100644
--- a/qemu/libdecnumber/decNumber.c
+++ b/qemu/libdecnumber/decNumber.c
@@ -166,10 +166,7 @@
 /*	**  -- raise to the power				      */
 /* ------------------------------------------------------------------ */
 
-#include <stdlib.h>		   /* for malloc, free, etc. */
-#include <stdio.h>		   /* for printf [if needed] */
-#include <string.h>		   /* for strcpy */
-#include <ctype.h>		   /* for lower */
+#include "qemu/osdep.h"
 #include "libdecnumber/dconfig.h"
 #include "libdecnumber/decNumber.h"
 #include "libdecnumber/decNumberLocal.h"
@@ -2238,7 +2235,7 @@ decNumber * decNumberPower(decNumber *res, const decNumber *lhs,
       /* if a negative power the constant 1 is needed, and if not subset */
       /* invert the lhs now rather than inverting the result later */
       if (decNumberIsNegative(rhs)) {	/* was a **-n [hence digits>0] */
-	decNumber *inv=invbuff;		/* asssume use fixed buffer */
+	decNumber *inv=invbuff;		/* assume use fixed buffer */
 	decNumberCopy(&dnOne, dac);	/* dnOne=1;  [needed now or later] */
 	#if DECSUBSET
 	if (set->extended) {		/* need to calculate 1/lhs */
@@ -3849,7 +3846,7 @@ static void decToString(const decNumber *dn, char *string, Flag eng) {
 /*								      */
 /* Addition, especially x=x+1, is speed-critical.		      */
 /* The static buffer is larger than might be expected to allow for    */
-/* calls from higher-level funtions (notable exp).		      */
+/* calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decAddOp(decNumber *res, const decNumber *lhs,
 			    const decNumber *rhs, decContext *set,
@@ -4263,7 +4260,7 @@ static decNumber * decAddOp(decNumber *res, const decNumber *lhs,
 /* long subtractions.  These are acc and var1 respectively.	      */
 /* var1 is a copy of the lhs coefficient, var2 is the rhs coefficient.*/
 /* The static buffers may be larger than might be expected to allow   */
-/* for calls from higher-level funtions (notable exp).		      */
+/* for calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decDivideOp(decNumber *res,
 			       const decNumber *lhs, const decNumber *rhs,
@@ -5254,7 +5251,7 @@ static decNumber * decMultiplyOp(decNumber *res, const decNumber *lhs,
 /*    exp(-x) where x can be the tiniest number (Ntiny).	      */
 /*								      */
 /* 2. Normalizing x to be <=0.1 (instead of <=1) reduces loop	      */
-/*    iterations by appoximately a third with additional (although    */
+/*    iterations by approximately a third with additional (although    */
 /*    diminishing) returns as the range is reduced to even smaller    */
 /*    fractions.  However, h (the power of 10 used to correct the     */
 /*    result at the end, see below) must be kept <=8 as otherwise     */
diff --git a/qemu/libdecnumber/dpd/decimal128.c b/qemu/libdecnumber/dpd/decimal128.c
index 7551b7caa..ca4764e54 100644
--- a/qemu/libdecnumber/dpd/decimal128.c
+++ b/qemu/libdecnumber/dpd/decimal128.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS 34      /* make decNumbers with space for 34 */
diff --git a/qemu/libdecnumber/dpd/decimal32.c b/qemu/libdecnumber/dpd/decimal32.c
index 095ab7565..53f29789d 100644
--- a/qemu/libdecnumber/dpd/decimal32.c
+++ b/qemu/libdecnumber/dpd/decimal32.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS  7      /* make decNumbers with space for 7 */
diff --git a/qemu/libdecnumber/dpd/decimal64.c b/qemu/libdecnumber/dpd/decimal64.c
index 8256084e9..481617641 100644
--- a/qemu/libdecnumber/dpd/decimal64.c
+++ b/qemu/libdecnumber/dpd/decimal64.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS 16      /* make decNumbers with space for 16 */
diff --git a/qemu/linux-headers/asm-arm/unistd.h b/qemu/linux-headers/asm-arm/unistd.h
new file mode 100644
index 000000000..3f6f72792
--- /dev/null
+++ b/qemu/linux-headers/asm-arm/unistd.h
@@ -0,0 +1,450 @@
+/*
+ *  arch/arm/include/asm/unistd.h
+ *
+ *  Copyright (C) 2001-2005 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Please forward _all_ changes to this file to rmk@arm.linux.org.uk,
+ * no matter what the change is.  Thanks!
+ */
+#ifndef __ASM_ARM_UNISTD_H
+#define __ASM_ARM_UNISTD_H
+
+#define __NR_OABI_SYSCALL_BASE	0x900000
+
+#if defined(__thumb__) || defined(__ARM_EABI__)
+#define __NR_SYSCALL_BASE	0
+#else
+#define __NR_SYSCALL_BASE	__NR_OABI_SYSCALL_BASE
+#endif
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_restart_syscall		(__NR_SYSCALL_BASE+  0)
+#define __NR_exit			(__NR_SYSCALL_BASE+  1)
+#define __NR_fork			(__NR_SYSCALL_BASE+  2)
+#define __NR_read			(__NR_SYSCALL_BASE+  3)
+#define __NR_write			(__NR_SYSCALL_BASE+  4)
+#define __NR_open			(__NR_SYSCALL_BASE+  5)
+#define __NR_close			(__NR_SYSCALL_BASE+  6)
+					/* 7 was sys_waitpid */
+#define __NR_creat			(__NR_SYSCALL_BASE+  8)
+#define __NR_link			(__NR_SYSCALL_BASE+  9)
+#define __NR_unlink			(__NR_SYSCALL_BASE+ 10)
+#define __NR_execve			(__NR_SYSCALL_BASE+ 11)
+#define __NR_chdir			(__NR_SYSCALL_BASE+ 12)
+#define __NR_time			(__NR_SYSCALL_BASE+ 13)
+#define __NR_mknod			(__NR_SYSCALL_BASE+ 14)
+#define __NR_chmod			(__NR_SYSCALL_BASE+ 15)
+#define __NR_lchown			(__NR_SYSCALL_BASE+ 16)
+					/* 17 was sys_break */
+					/* 18 was sys_stat */
+#define __NR_lseek			(__NR_SYSCALL_BASE+ 19)
+#define __NR_getpid			(__NR_SYSCALL_BASE+ 20)
+#define __NR_mount			(__NR_SYSCALL_BASE+ 21)
+#define __NR_umount			(__NR_SYSCALL_BASE+ 22)
+#define __NR_setuid			(__NR_SYSCALL_BASE+ 23)
+#define __NR_getuid			(__NR_SYSCALL_BASE+ 24)
+#define __NR_stime			(__NR_SYSCALL_BASE+ 25)
+#define __NR_ptrace			(__NR_SYSCALL_BASE+ 26)
+#define __NR_alarm			(__NR_SYSCALL_BASE+ 27)
+					/* 28 was sys_fstat */
+#define __NR_pause			(__NR_SYSCALL_BASE+ 29)
+#define __NR_utime			(__NR_SYSCALL_BASE+ 30)
+					/* 31 was sys_stty */
+					/* 32 was sys_gtty */
+#define __NR_access			(__NR_SYSCALL_BASE+ 33)
+#define __NR_nice			(__NR_SYSCALL_BASE+ 34)
+					/* 35 was sys_ftime */
+#define __NR_sync			(__NR_SYSCALL_BASE+ 36)
+#define __NR_kill			(__NR_SYSCALL_BASE+ 37)
+#define __NR_rename			(__NR_SYSCALL_BASE+ 38)
+#define __NR_mkdir			(__NR_SYSCALL_BASE+ 39)
+#define __NR_rmdir			(__NR_SYSCALL_BASE+ 40)
+#define __NR_dup			(__NR_SYSCALL_BASE+ 41)
+#define __NR_pipe			(__NR_SYSCALL_BASE+ 42)
+#define __NR_times			(__NR_SYSCALL_BASE+ 43)
+					/* 44 was sys_prof */
+#define __NR_brk			(__NR_SYSCALL_BASE+ 45)
+#define __NR_setgid			(__NR_SYSCALL_BASE+ 46)
+#define __NR_getgid			(__NR_SYSCALL_BASE+ 47)
+					/* 48 was sys_signal */
+#define __NR_geteuid			(__NR_SYSCALL_BASE+ 49)
+#define __NR_getegid			(__NR_SYSCALL_BASE+ 50)
+#define __NR_acct			(__NR_SYSCALL_BASE+ 51)
+#define __NR_umount2			(__NR_SYSCALL_BASE+ 52)
+					/* 53 was sys_lock */
+#define __NR_ioctl			(__NR_SYSCALL_BASE+ 54)
+#define __NR_fcntl			(__NR_SYSCALL_BASE+ 55)
+					/* 56 was sys_mpx */
+#define __NR_setpgid			(__NR_SYSCALL_BASE+ 57)
+					/* 58 was sys_ulimit */
+					/* 59 was sys_olduname */
+#define __NR_umask			(__NR_SYSCALL_BASE+ 60)
+#define __NR_chroot			(__NR_SYSCALL_BASE+ 61)
+#define __NR_ustat			(__NR_SYSCALL_BASE+ 62)
+#define __NR_dup2			(__NR_SYSCALL_BASE+ 63)
+#define __NR_getppid			(__NR_SYSCALL_BASE+ 64)
+#define __NR_getpgrp			(__NR_SYSCALL_BASE+ 65)
+#define __NR_setsid			(__NR_SYSCALL_BASE+ 66)
+#define __NR_sigaction			(__NR_SYSCALL_BASE+ 67)
+					/* 68 was sys_sgetmask */
+					/* 69 was sys_ssetmask */
+#define __NR_setreuid			(__NR_SYSCALL_BASE+ 70)
+#define __NR_setregid			(__NR_SYSCALL_BASE+ 71)
+#define __NR_sigsuspend			(__NR_SYSCALL_BASE+ 72)
+#define __NR_sigpending			(__NR_SYSCALL_BASE+ 73)
+#define __NR_sethostname		(__NR_SYSCALL_BASE+ 74)
+#define __NR_setrlimit			(__NR_SYSCALL_BASE+ 75)
+#define __NR_getrlimit			(__NR_SYSCALL_BASE+ 76)	/* Back compat 2GB limited rlimit */
+#define __NR_getrusage			(__NR_SYSCALL_BASE+ 77)
+#define __NR_gettimeofday		(__NR_SYSCALL_BASE+ 78)
+#define __NR_settimeofday		(__NR_SYSCALL_BASE+ 79)
+#define __NR_getgroups			(__NR_SYSCALL_BASE+ 80)
+#define __NR_setgroups			(__NR_SYSCALL_BASE+ 81)
+#define __NR_select			(__NR_SYSCALL_BASE+ 82)
+#define __NR_symlink			(__NR_SYSCALL_BASE+ 83)
+					/* 84 was sys_lstat */
+#define __NR_readlink			(__NR_SYSCALL_BASE+ 85)
+#define __NR_uselib			(__NR_SYSCALL_BASE+ 86)
+#define __NR_swapon			(__NR_SYSCALL_BASE+ 87)
+#define __NR_reboot			(__NR_SYSCALL_BASE+ 88)
+#define __NR_readdir			(__NR_SYSCALL_BASE+ 89)
+#define __NR_mmap			(__NR_SYSCALL_BASE+ 90)
+#define __NR_munmap			(__NR_SYSCALL_BASE+ 91)
+#define __NR_truncate			(__NR_SYSCALL_BASE+ 92)
+#define __NR_ftruncate			(__NR_SYSCALL_BASE+ 93)
+#define __NR_fchmod			(__NR_SYSCALL_BASE+ 94)
+#define __NR_fchown			(__NR_SYSCALL_BASE+ 95)
+#define __NR_getpriority		(__NR_SYSCALL_BASE+ 96)
+#define __NR_setpriority		(__NR_SYSCALL_BASE+ 97)
+					/* 98 was sys_profil */
+#define __NR_statfs			(__NR_SYSCALL_BASE+ 99)
+#define __NR_fstatfs			(__NR_SYSCALL_BASE+100)
+					/* 101 was sys_ioperm */
+#define __NR_socketcall			(__NR_SYSCALL_BASE+102)
+#define __NR_syslog			(__NR_SYSCALL_BASE+103)
+#define __NR_setitimer			(__NR_SYSCALL_BASE+104)
+#define __NR_getitimer			(__NR_SYSCALL_BASE+105)
+#define __NR_stat			(__NR_SYSCALL_BASE+106)
+#define __NR_lstat			(__NR_SYSCALL_BASE+107)
+#define __NR_fstat			(__NR_SYSCALL_BASE+108)
+					/* 109 was sys_uname */
+					/* 110 was sys_iopl */
+#define __NR_vhangup			(__NR_SYSCALL_BASE+111)
+					/* 112 was sys_idle */
+#define __NR_syscall			(__NR_SYSCALL_BASE+113) /* syscall to call a syscall! */
+#define __NR_wait4			(__NR_SYSCALL_BASE+114)
+#define __NR_swapoff			(__NR_SYSCALL_BASE+115)
+#define __NR_sysinfo			(__NR_SYSCALL_BASE+116)
+#define __NR_ipc			(__NR_SYSCALL_BASE+117)
+#define __NR_fsync			(__NR_SYSCALL_BASE+118)
+#define __NR_sigreturn			(__NR_SYSCALL_BASE+119)
+#define __NR_clone			(__NR_SYSCALL_BASE+120)
+#define __NR_setdomainname		(__NR_SYSCALL_BASE+121)
+#define __NR_uname			(__NR_SYSCALL_BASE+122)
+					/* 123 was sys_modify_ldt */
+#define __NR_adjtimex			(__NR_SYSCALL_BASE+124)
+#define __NR_mprotect			(__NR_SYSCALL_BASE+125)
+#define __NR_sigprocmask		(__NR_SYSCALL_BASE+126)
+					/* 127 was sys_create_module */
+#define __NR_init_module		(__NR_SYSCALL_BASE+128)
+#define __NR_delete_module		(__NR_SYSCALL_BASE+129)
+					/* 130 was sys_get_kernel_syms */
+#define __NR_quotactl			(__NR_SYSCALL_BASE+131)
+#define __NR_getpgid			(__NR_SYSCALL_BASE+132)
+#define __NR_fchdir			(__NR_SYSCALL_BASE+133)
+#define __NR_bdflush			(__NR_SYSCALL_BASE+134)
+#define __NR_sysfs			(__NR_SYSCALL_BASE+135)
+#define __NR_personality		(__NR_SYSCALL_BASE+136)
+					/* 137 was sys_afs_syscall */
+#define __NR_setfsuid			(__NR_SYSCALL_BASE+138)
+#define __NR_setfsgid			(__NR_SYSCALL_BASE+139)
+#define __NR__llseek			(__NR_SYSCALL_BASE+140)
+#define __NR_getdents			(__NR_SYSCALL_BASE+141)
+#define __NR__newselect			(__NR_SYSCALL_BASE+142)
+#define __NR_flock			(__NR_SYSCALL_BASE+143)
+#define __NR_msync			(__NR_SYSCALL_BASE+144)
+#define __NR_readv			(__NR_SYSCALL_BASE+145)
+#define __NR_writev			(__NR_SYSCALL_BASE+146)
+#define __NR_getsid			(__NR_SYSCALL_BASE+147)
+#define __NR_fdatasync			(__NR_SYSCALL_BASE+148)
+#define __NR__sysctl			(__NR_SYSCALL_BASE+149)
+#define __NR_mlock			(__NR_SYSCALL_BASE+150)
+#define __NR_munlock			(__NR_SYSCALL_BASE+151)
+#define __NR_mlockall			(__NR_SYSCALL_BASE+152)
+#define __NR_munlockall			(__NR_SYSCALL_BASE+153)
+#define __NR_sched_setparam		(__NR_SYSCALL_BASE+154)
+#define __NR_sched_getparam		(__NR_SYSCALL_BASE+155)
+#define __NR_sched_setscheduler		(__NR_SYSCALL_BASE+156)
+#define __NR_sched_getscheduler		(__NR_SYSCALL_BASE+157)
+#define __NR_sched_yield		(__NR_SYSCALL_BASE+158)
+#define __NR_sched_get_priority_max	(__NR_SYSCALL_BASE+159)
+#define __NR_sched_get_priority_min	(__NR_SYSCALL_BASE+160)
+#define __NR_sched_rr_get_interval	(__NR_SYSCALL_BASE+161)
+#define __NR_nanosleep			(__NR_SYSCALL_BASE+162)
+#define __NR_mremap			(__NR_SYSCALL_BASE+163)
+#define __NR_setresuid			(__NR_SYSCALL_BASE+164)
+#define __NR_getresuid			(__NR_SYSCALL_BASE+165)
+					/* 166 was sys_vm86 */
+					/* 167 was sys_query_module */
+#define __NR_poll			(__NR_SYSCALL_BASE+168)
+#define __NR_nfsservctl			(__NR_SYSCALL_BASE+169)
+#define __NR_setresgid			(__NR_SYSCALL_BASE+170)
+#define __NR_getresgid			(__NR_SYSCALL_BASE+171)
+#define __NR_prctl			(__NR_SYSCALL_BASE+172)
+#define __NR_rt_sigreturn		(__NR_SYSCALL_BASE+173)
+#define __NR_rt_sigaction		(__NR_SYSCALL_BASE+174)
+#define __NR_rt_sigprocmask		(__NR_SYSCALL_BASE+175)
+#define __NR_rt_sigpending		(__NR_SYSCALL_BASE+176)
+#define __NR_rt_sigtimedwait		(__NR_SYSCALL_BASE+177)
+#define __NR_rt_sigqueueinfo		(__NR_SYSCALL_BASE+178)
+#define __NR_rt_sigsuspend		(__NR_SYSCALL_BASE+179)
+#define __NR_pread64			(__NR_SYSCALL_BASE+180)
+#define __NR_pwrite64			(__NR_SYSCALL_BASE+181)
+#define __NR_chown			(__NR_SYSCALL_BASE+182)
+#define __NR_getcwd			(__NR_SYSCALL_BASE+183)
+#define __NR_capget			(__NR_SYSCALL_BASE+184)
+#define __NR_capset			(__NR_SYSCALL_BASE+185)
+#define __NR_sigaltstack		(__NR_SYSCALL_BASE+186)
+#define __NR_sendfile			(__NR_SYSCALL_BASE+187)
+					/* 188 reserved */
+					/* 189 reserved */
+#define __NR_vfork			(__NR_SYSCALL_BASE+190)
+#define __NR_ugetrlimit			(__NR_SYSCALL_BASE+191)	/* SuS compliant getrlimit */
+#define __NR_mmap2			(__NR_SYSCALL_BASE+192)
+#define __NR_truncate64			(__NR_SYSCALL_BASE+193)
+#define __NR_ftruncate64		(__NR_SYSCALL_BASE+194)
+#define __NR_stat64			(__NR_SYSCALL_BASE+195)
+#define __NR_lstat64			(__NR_SYSCALL_BASE+196)
+#define __NR_fstat64			(__NR_SYSCALL_BASE+197)
+#define __NR_lchown32			(__NR_SYSCALL_BASE+198)
+#define __NR_getuid32			(__NR_SYSCALL_BASE+199)
+#define __NR_getgid32			(__NR_SYSCALL_BASE+200)
+#define __NR_geteuid32			(__NR_SYSCALL_BASE+201)
+#define __NR_getegid32			(__NR_SYSCALL_BASE+202)
+#define __NR_setreuid32			(__NR_SYSCALL_BASE+203)
+#define __NR_setregid32			(__NR_SYSCALL_BASE+204)
+#define __NR_getgroups32		(__NR_SYSCALL_BASE+205)
+#define __NR_setgroups32		(__NR_SYSCALL_BASE+206)
+#define __NR_fchown32			(__NR_SYSCALL_BASE+207)
+#define __NR_setresuid32		(__NR_SYSCALL_BASE+208)
+#define __NR_getresuid32		(__NR_SYSCALL_BASE+209)
+#define __NR_setresgid32		(__NR_SYSCALL_BASE+210)
+#define __NR_getresgid32		(__NR_SYSCALL_BASE+211)
+#define __NR_chown32			(__NR_SYSCALL_BASE+212)
+#define __NR_setuid32			(__NR_SYSCALL_BASE+213)
+#define __NR_setgid32			(__NR_SYSCALL_BASE+214)
+#define __NR_setfsuid32			(__NR_SYSCALL_BASE+215)
+#define __NR_setfsgid32			(__NR_SYSCALL_BASE+216)
+#define __NR_getdents64			(__NR_SYSCALL_BASE+217)
+#define __NR_pivot_root			(__NR_SYSCALL_BASE+218)
+#define __NR_mincore			(__NR_SYSCALL_BASE+219)
+#define __NR_madvise			(__NR_SYSCALL_BASE+220)
+#define __NR_fcntl64			(__NR_SYSCALL_BASE+221)
+					/* 222 for tux */
+					/* 223 is unused */
+#define __NR_gettid			(__NR_SYSCALL_BASE+224)
+#define __NR_readahead			(__NR_SYSCALL_BASE+225)
+#define __NR_setxattr			(__NR_SYSCALL_BASE+226)
+#define __NR_lsetxattr			(__NR_SYSCALL_BASE+227)
+#define __NR_fsetxattr			(__NR_SYSCALL_BASE+228)
+#define __NR_getxattr			(__NR_SYSCALL_BASE+229)
+#define __NR_lgetxattr			(__NR_SYSCALL_BASE+230)
+#define __NR_fgetxattr			(__NR_SYSCALL_BASE+231)
+#define __NR_listxattr			(__NR_SYSCALL_BASE+232)
+#define __NR_llistxattr			(__NR_SYSCALL_BASE+233)
+#define __NR_flistxattr			(__NR_SYSCALL_BASE+234)
+#define __NR_removexattr		(__NR_SYSCALL_BASE+235)
+#define __NR_lremovexattr		(__NR_SYSCALL_BASE+236)
+#define __NR_fremovexattr		(__NR_SYSCALL_BASE+237)
+#define __NR_tkill			(__NR_SYSCALL_BASE+238)
+#define __NR_sendfile64			(__NR_SYSCALL_BASE+239)
+#define __NR_futex			(__NR_SYSCALL_BASE+240)
+#define __NR_sched_setaffinity		(__NR_SYSCALL_BASE+241)
+#define __NR_sched_getaffinity		(__NR_SYSCALL_BASE+242)
+#define __NR_io_setup			(__NR_SYSCALL_BASE+243)
+#define __NR_io_destroy			(__NR_SYSCALL_BASE+244)
+#define __NR_io_getevents		(__NR_SYSCALL_BASE+245)
+#define __NR_io_submit			(__NR_SYSCALL_BASE+246)
+#define __NR_io_cancel			(__NR_SYSCALL_BASE+247)
+#define __NR_exit_group			(__NR_SYSCALL_BASE+248)
+#define __NR_lookup_dcookie		(__NR_SYSCALL_BASE+249)
+#define __NR_epoll_create		(__NR_SYSCALL_BASE+250)
+#define __NR_epoll_ctl			(__NR_SYSCALL_BASE+251)
+#define __NR_epoll_wait			(__NR_SYSCALL_BASE+252)
+#define __NR_remap_file_pages		(__NR_SYSCALL_BASE+253)
+					/* 254 for set_thread_area */
+					/* 255 for get_thread_area */
+#define __NR_set_tid_address		(__NR_SYSCALL_BASE+256)
+#define __NR_timer_create		(__NR_SYSCALL_BASE+257)
+#define __NR_timer_settime		(__NR_SYSCALL_BASE+258)
+#define __NR_timer_gettime		(__NR_SYSCALL_BASE+259)
+#define __NR_timer_getoverrun		(__NR_SYSCALL_BASE+260)
+#define __NR_timer_delete		(__NR_SYSCALL_BASE+261)
+#define __NR_clock_settime		(__NR_SYSCALL_BASE+262)
+#define __NR_clock_gettime		(__NR_SYSCALL_BASE+263)
+#define __NR_clock_getres		(__NR_SYSCALL_BASE+264)
+#define __NR_clock_nanosleep		(__NR_SYSCALL_BASE+265)
+#define __NR_statfs64			(__NR_SYSCALL_BASE+266)
+#define __NR_fstatfs64			(__NR_SYSCALL_BASE+267)
+#define __NR_tgkill			(__NR_SYSCALL_BASE+268)
+#define __NR_utimes			(__NR_SYSCALL_BASE+269)
+#define __NR_arm_fadvise64_64		(__NR_SYSCALL_BASE+270)
+#define __NR_pciconfig_iobase		(__NR_SYSCALL_BASE+271)
+#define __NR_pciconfig_read		(__NR_SYSCALL_BASE+272)
+#define __NR_pciconfig_write		(__NR_SYSCALL_BASE+273)
+#define __NR_mq_open			(__NR_SYSCALL_BASE+274)
+#define __NR_mq_unlink			(__NR_SYSCALL_BASE+275)
+#define __NR_mq_timedsend		(__NR_SYSCALL_BASE+276)
+#define __NR_mq_timedreceive		(__NR_SYSCALL_BASE+277)
+#define __NR_mq_notify			(__NR_SYSCALL_BASE+278)
+#define __NR_mq_getsetattr		(__NR_SYSCALL_BASE+279)
+#define __NR_waitid			(__NR_SYSCALL_BASE+280)
+#define __NR_socket			(__NR_SYSCALL_BASE+281)
+#define __NR_bind			(__NR_SYSCALL_BASE+282)
+#define __NR_connect			(__NR_SYSCALL_BASE+283)
+#define __NR_listen			(__NR_SYSCALL_BASE+284)
+#define __NR_accept			(__NR_SYSCALL_BASE+285)
+#define __NR_getsockname		(__NR_SYSCALL_BASE+286)
+#define __NR_getpeername		(__NR_SYSCALL_BASE+287)
+#define __NR_socketpair			(__NR_SYSCALL_BASE+288)
+#define __NR_send			(__NR_SYSCALL_BASE+289)
+#define __NR_sendto			(__NR_SYSCALL_BASE+290)
+#define __NR_recv			(__NR_SYSCALL_BASE+291)
+#define __NR_recvfrom			(__NR_SYSCALL_BASE+292)
+#define __NR_shutdown			(__NR_SYSCALL_BASE+293)
+#define __NR_setsockopt			(__NR_SYSCALL_BASE+294)
+#define __NR_getsockopt			(__NR_SYSCALL_BASE+295)
+#define __NR_sendmsg			(__NR_SYSCALL_BASE+296)
+#define __NR_recvmsg			(__NR_SYSCALL_BASE+297)
+#define __NR_semop			(__NR_SYSCALL_BASE+298)
+#define __NR_semget			(__NR_SYSCALL_BASE+299)
+#define __NR_semctl			(__NR_SYSCALL_BASE+300)
+#define __NR_msgsnd			(__NR_SYSCALL_BASE+301)
+#define __NR_msgrcv			(__NR_SYSCALL_BASE+302)
+#define __NR_msgget			(__NR_SYSCALL_BASE+303)
+#define __NR_msgctl			(__NR_SYSCALL_BASE+304)
+#define __NR_shmat			(__NR_SYSCALL_BASE+305)
+#define __NR_shmdt			(__NR_SYSCALL_BASE+306)
+#define __NR_shmget			(__NR_SYSCALL_BASE+307)
+#define __NR_shmctl			(__NR_SYSCALL_BASE+308)
+#define __NR_add_key			(__NR_SYSCALL_BASE+309)
+#define __NR_request_key		(__NR_SYSCALL_BASE+310)
+#define __NR_keyctl			(__NR_SYSCALL_BASE+311)
+#define __NR_semtimedop			(__NR_SYSCALL_BASE+312)
+#define __NR_vserver			(__NR_SYSCALL_BASE+313)
+#define __NR_ioprio_set			(__NR_SYSCALL_BASE+314)
+#define __NR_ioprio_get			(__NR_SYSCALL_BASE+315)
+#define __NR_inotify_init		(__NR_SYSCALL_BASE+316)
+#define __NR_inotify_add_watch		(__NR_SYSCALL_BASE+317)
+#define __NR_inotify_rm_watch		(__NR_SYSCALL_BASE+318)
+#define __NR_mbind			(__NR_SYSCALL_BASE+319)
+#define __NR_get_mempolicy		(__NR_SYSCALL_BASE+320)
+#define __NR_set_mempolicy		(__NR_SYSCALL_BASE+321)
+#define __NR_openat			(__NR_SYSCALL_BASE+322)
+#define __NR_mkdirat			(__NR_SYSCALL_BASE+323)
+#define __NR_mknodat			(__NR_SYSCALL_BASE+324)
+#define __NR_fchownat			(__NR_SYSCALL_BASE+325)
+#define __NR_futimesat			(__NR_SYSCALL_BASE+326)
+#define __NR_fstatat64			(__NR_SYSCALL_BASE+327)
+#define __NR_unlinkat			(__NR_SYSCALL_BASE+328)
+#define __NR_renameat			(__NR_SYSCALL_BASE+329)
+#define __NR_linkat			(__NR_SYSCALL_BASE+330)
+#define __NR_symlinkat			(__NR_SYSCALL_BASE+331)
+#define __NR_readlinkat			(__NR_SYSCALL_BASE+332)
+#define __NR_fchmodat			(__NR_SYSCALL_BASE+333)
+#define __NR_faccessat			(__NR_SYSCALL_BASE+334)
+#define __NR_pselect6			(__NR_SYSCALL_BASE+335)
+#define __NR_ppoll			(__NR_SYSCALL_BASE+336)
+#define __NR_unshare			(__NR_SYSCALL_BASE+337)
+#define __NR_set_robust_list		(__NR_SYSCALL_BASE+338)
+#define __NR_get_robust_list		(__NR_SYSCALL_BASE+339)
+#define __NR_splice			(__NR_SYSCALL_BASE+340)
+#define __NR_arm_sync_file_range	(__NR_SYSCALL_BASE+341)
+#define __NR_sync_file_range2		__NR_arm_sync_file_range
+#define __NR_tee			(__NR_SYSCALL_BASE+342)
+#define __NR_vmsplice			(__NR_SYSCALL_BASE+343)
+#define __NR_move_pages			(__NR_SYSCALL_BASE+344)
+#define __NR_getcpu			(__NR_SYSCALL_BASE+345)
+#define __NR_epoll_pwait		(__NR_SYSCALL_BASE+346)
+#define __NR_kexec_load			(__NR_SYSCALL_BASE+347)
+#define __NR_utimensat			(__NR_SYSCALL_BASE+348)
+#define __NR_signalfd			(__NR_SYSCALL_BASE+349)
+#define __NR_timerfd_create		(__NR_SYSCALL_BASE+350)
+#define __NR_eventfd			(__NR_SYSCALL_BASE+351)
+#define __NR_fallocate			(__NR_SYSCALL_BASE+352)
+#define __NR_timerfd_settime		(__NR_SYSCALL_BASE+353)
+#define __NR_timerfd_gettime		(__NR_SYSCALL_BASE+354)
+#define __NR_signalfd4			(__NR_SYSCALL_BASE+355)
+#define __NR_eventfd2			(__NR_SYSCALL_BASE+356)
+#define __NR_epoll_create1		(__NR_SYSCALL_BASE+357)
+#define __NR_dup3			(__NR_SYSCALL_BASE+358)
+#define __NR_pipe2			(__NR_SYSCALL_BASE+359)
+#define __NR_inotify_init1		(__NR_SYSCALL_BASE+360)
+#define __NR_preadv			(__NR_SYSCALL_BASE+361)
+#define __NR_pwritev			(__NR_SYSCALL_BASE+362)
+#define __NR_rt_tgsigqueueinfo		(__NR_SYSCALL_BASE+363)
+#define __NR_perf_event_open		(__NR_SYSCALL_BASE+364)
+#define __NR_recvmmsg			(__NR_SYSCALL_BASE+365)
+#define __NR_accept4			(__NR_SYSCALL_BASE+366)
+#define __NR_fanotify_init		(__NR_SYSCALL_BASE+367)
+#define __NR_fanotify_mark		(__NR_SYSCALL_BASE+368)
+#define __NR_prlimit64			(__NR_SYSCALL_BASE+369)
+#define __NR_name_to_handle_at		(__NR_SYSCALL_BASE+370)
+#define __NR_open_by_handle_at		(__NR_SYSCALL_BASE+371)
+#define __NR_clock_adjtime		(__NR_SYSCALL_BASE+372)
+#define __NR_syncfs			(__NR_SYSCALL_BASE+373)
+#define __NR_sendmmsg			(__NR_SYSCALL_BASE+374)
+#define __NR_setns			(__NR_SYSCALL_BASE+375)
+#define __NR_process_vm_readv		(__NR_SYSCALL_BASE+376)
+#define __NR_process_vm_writev		(__NR_SYSCALL_BASE+377)
+#define __NR_kcmp			(__NR_SYSCALL_BASE+378)
+#define __NR_finit_module		(__NR_SYSCALL_BASE+379)
+#define __NR_sched_setattr		(__NR_SYSCALL_BASE+380)
+#define __NR_sched_getattr		(__NR_SYSCALL_BASE+381)
+#define __NR_renameat2			(__NR_SYSCALL_BASE+382)
+#define __NR_seccomp			(__NR_SYSCALL_BASE+383)
+#define __NR_getrandom			(__NR_SYSCALL_BASE+384)
+#define __NR_memfd_create		(__NR_SYSCALL_BASE+385)
+#define __NR_bpf			(__NR_SYSCALL_BASE+386)
+#define __NR_execveat			(__NR_SYSCALL_BASE+387)
+#define __NR_userfaultfd		(__NR_SYSCALL_BASE+388)
+#define __NR_membarrier			(__NR_SYSCALL_BASE+389)
+#define __NR_mlock2			(__NR_SYSCALL_BASE+390)
+#define __NR_copy_file_range		(__NR_SYSCALL_BASE+391)
+
+/*
+ * The following SWIs are ARM private.
+ */
+#define __ARM_NR_BASE			(__NR_SYSCALL_BASE+0x0f0000)
+#define __ARM_NR_breakpoint		(__ARM_NR_BASE+1)
+#define __ARM_NR_cacheflush		(__ARM_NR_BASE+2)
+#define __ARM_NR_usr26			(__ARM_NR_BASE+3)
+#define __ARM_NR_usr32			(__ARM_NR_BASE+4)
+#define __ARM_NR_set_tls		(__ARM_NR_BASE+5)
+
+/*
+ * The following syscalls are obsolete and no longer available for EABI.
+ */
+#if defined(__ARM_EABI__)
+#undef __NR_time
+#undef __NR_umount
+#undef __NR_stime
+#undef __NR_alarm
+#undef __NR_utime
+#undef __NR_getrlimit
+#undef __NR_select
+#undef __NR_readdir
+#undef __NR_mmap
+#undef __NR_socketcall
+#undef __NR_syscall
+#undef __NR_ipc
+#endif
+
+#endif /* __ASM_ARM_UNISTD_H */
diff --git a/qemu/linux-headers/asm-arm64/kvm.h b/qemu/linux-headers/asm-arm64/kvm.h
index c8abf257c..7d82d1f9d 100644
--- a/qemu/linux-headers/asm-arm64/kvm.h
+++ b/qemu/linux-headers/asm-arm64/kvm.h
@@ -32,7 +32,7 @@
 
 #ifndef __ASSEMBLY__
 #include <linux/psci.h>
-#include <asm/types.h>
+#include <linux/types.h>
 #include <asm/ptrace.h>
 
 #define __KVM_HAVE_GUEST_DEBUG
@@ -53,14 +53,20 @@ struct kvm_regs {
 	struct user_fpsimd_state fp_regs;
 };
 
-/* Supported Processor Types */
+/*
+ * Supported CPU Targets - Adding a new target type is not recommended,
+ * unless there are some special registers not supported by the
+ * genericv8 syreg table.
+ */
 #define KVM_ARM_TARGET_AEM_V8		0
 #define KVM_ARM_TARGET_FOUNDATION_V8	1
 #define KVM_ARM_TARGET_CORTEX_A57	2
 #define KVM_ARM_TARGET_XGENE_POTENZA	3
 #define KVM_ARM_TARGET_CORTEX_A53	4
+/* Generic ARM v8 target */
+#define KVM_ARM_TARGET_GENERIC_V8	5
 
-#define KVM_ARM_NUM_TARGETS		5
+#define KVM_ARM_NUM_TARGETS		6
 
 /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
 #define KVM_ARM_DEVICE_TYPE_SHIFT	0
@@ -88,6 +94,7 @@ struct kvm_regs {
 #define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
 #define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
 #define KVM_ARM_VCPU_PSCI_0_2		2 /* CPU uses PSCI v0.2 */
+#define KVM_ARM_VCPU_PMU_V3		3 /* Support guest PMUv3 */
 
 struct kvm_vcpu_init {
 	__u32 target;
@@ -100,12 +107,39 @@ struct kvm_sregs {
 struct kvm_fpu {
 };
 
+/*
+ * See v8 ARM ARM D7.3: Debug Registers
+ *
+ * The architectural limit is 16 debug registers of each type although
+ * in practice there are usually less (see ID_AA64DFR0_EL1).
+ *
+ * Although the control registers are architecturally defined as 32
+ * bits wide we use a 64 bit structure here to keep parity with
+ * KVM_GET/SET_ONE_REG behaviour which treats all system registers as
+ * 64 bit values. It also allows for the possibility of the
+ * architecture expanding the control registers without having to
+ * change the userspace ABI.
+ */
+#define KVM_ARM_MAX_DBG_REGS 16
 struct kvm_guest_debug_arch {
+	__u64 dbg_bcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_bvr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wvr[KVM_ARM_MAX_DBG_REGS];
 };
 
 struct kvm_debug_exit_arch {
+	__u32 hsr;
+	__u64 far;	/* used for watchpoints */
 };
 
+/*
+ * Architecture specific defines for kvm_guest_debug->control
+ */
+
+#define KVM_GUESTDBG_USE_SW_BP		(1 << 16)
+#define KVM_GUESTDBG_USE_HW		(1 << 17)
+
 struct kvm_sync_regs {
 };
 
@@ -171,6 +205,11 @@ struct kvm_arch_memory_slot {
 #define KVM_DEV_ARM_VGIC_GRP_CTRL	4
 #define   KVM_DEV_ARM_VGIC_CTRL_INIT	0
 
+/* Device Control API on vcpu fd */
+#define KVM_ARM_VCPU_PMU_V3_CTRL	0
+#define   KVM_ARM_VCPU_PMU_V3_IRQ	0
+#define   KVM_ARM_VCPU_PMU_V3_INIT	1
+
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
 #define KVM_ARM_IRQ_TYPE_MASK		0xff
diff --git a/qemu/linux-headers/asm-arm64/unistd.h b/qemu/linux-headers/asm-arm64/unistd.h
new file mode 100644
index 000000000..1caadc24e
--- /dev/null
+++ b/qemu/linux-headers/asm-arm64/unistd.h
@@ -0,0 +1,16 @@
+/*
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <asm-generic/unistd.h>
diff --git a/qemu/linux-headers/asm-mips/unistd.h b/qemu/linux-headers/asm-mips/unistd.h
new file mode 100644
index 000000000..2a2020938
--- /dev/null
+++ b/qemu/linux-headers/asm-mips/unistd.h
@@ -0,0 +1,1063 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1995, 96, 97, 98, 99, 2000 by Ralf Baechle
+ * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
+ *
+ * Changed system calls macros _syscall5 - _syscall7 to push args 5 to 7 onto
+ * the stack. Robin Farine for ACN S.A, Copyright (C) 1996 by ACN S.A
+ */
+#ifndef _ASM_UNISTD_H
+#define _ASM_UNISTD_H
+
+#include <asm/sgidefs.h>
+
+#if _MIPS_SIM == _MIPS_SIM_ABI32
+
+/*
+ * Linux o32 style syscalls are in the range from 4000 to 4999.
+ */
+#define __NR_Linux			4000
+#define __NR_syscall			(__NR_Linux +	0)
+#define __NR_exit			(__NR_Linux +	1)
+#define __NR_fork			(__NR_Linux +	2)
+#define __NR_read			(__NR_Linux +	3)
+#define __NR_write			(__NR_Linux +	4)
+#define __NR_open			(__NR_Linux +	5)
+#define __NR_close			(__NR_Linux +	6)
+#define __NR_waitpid			(__NR_Linux +	7)
+#define __NR_creat			(__NR_Linux +	8)
+#define __NR_link			(__NR_Linux +	9)
+#define __NR_unlink			(__NR_Linux +  10)
+#define __NR_execve			(__NR_Linux +  11)
+#define __NR_chdir			(__NR_Linux +  12)
+#define __NR_time			(__NR_Linux +  13)
+#define __NR_mknod			(__NR_Linux +  14)
+#define __NR_chmod			(__NR_Linux +  15)
+#define __NR_lchown			(__NR_Linux +  16)
+#define __NR_break			(__NR_Linux +  17)
+#define __NR_unused18			(__NR_Linux +  18)
+#define __NR_lseek			(__NR_Linux +  19)
+#define __NR_getpid			(__NR_Linux +  20)
+#define __NR_mount			(__NR_Linux +  21)
+#define __NR_umount			(__NR_Linux +  22)
+#define __NR_setuid			(__NR_Linux +  23)
+#define __NR_getuid			(__NR_Linux +  24)
+#define __NR_stime			(__NR_Linux +  25)
+#define __NR_ptrace			(__NR_Linux +  26)
+#define __NR_alarm			(__NR_Linux +  27)
+#define __NR_unused28			(__NR_Linux +  28)
+#define __NR_pause			(__NR_Linux +  29)
+#define __NR_utime			(__NR_Linux +  30)
+#define __NR_stty			(__NR_Linux +  31)
+#define __NR_gtty			(__NR_Linux +  32)
+#define __NR_access			(__NR_Linux +  33)
+#define __NR_nice			(__NR_Linux +  34)
+#define __NR_ftime			(__NR_Linux +  35)
+#define __NR_sync			(__NR_Linux +  36)
+#define __NR_kill			(__NR_Linux +  37)
+#define __NR_rename			(__NR_Linux +  38)
+#define __NR_mkdir			(__NR_Linux +  39)
+#define __NR_rmdir			(__NR_Linux +  40)
+#define __NR_dup			(__NR_Linux +  41)
+#define __NR_pipe			(__NR_Linux +  42)
+#define __NR_times			(__NR_Linux +  43)
+#define __NR_prof			(__NR_Linux +  44)
+#define __NR_brk			(__NR_Linux +  45)
+#define __NR_setgid			(__NR_Linux +  46)
+#define __NR_getgid			(__NR_Linux +  47)
+#define __NR_signal			(__NR_Linux +  48)
+#define __NR_geteuid			(__NR_Linux +  49)
+#define __NR_getegid			(__NR_Linux +  50)
+#define __NR_acct			(__NR_Linux +  51)
+#define __NR_umount2			(__NR_Linux +  52)
+#define __NR_lock			(__NR_Linux +  53)
+#define __NR_ioctl			(__NR_Linux +  54)
+#define __NR_fcntl			(__NR_Linux +  55)
+#define __NR_mpx			(__NR_Linux +  56)
+#define __NR_setpgid			(__NR_Linux +  57)
+#define __NR_ulimit			(__NR_Linux +  58)
+#define __NR_unused59			(__NR_Linux +  59)
+#define __NR_umask			(__NR_Linux +  60)
+#define __NR_chroot			(__NR_Linux +  61)
+#define __NR_ustat			(__NR_Linux +  62)
+#define __NR_dup2			(__NR_Linux +  63)
+#define __NR_getppid			(__NR_Linux +  64)
+#define __NR_getpgrp			(__NR_Linux +  65)
+#define __NR_setsid			(__NR_Linux +  66)
+#define __NR_sigaction			(__NR_Linux +  67)
+#define __NR_sgetmask			(__NR_Linux +  68)
+#define __NR_ssetmask			(__NR_Linux +  69)
+#define __NR_setreuid			(__NR_Linux +  70)
+#define __NR_setregid			(__NR_Linux +  71)
+#define __NR_sigsuspend			(__NR_Linux +  72)
+#define __NR_sigpending			(__NR_Linux +  73)
+#define __NR_sethostname		(__NR_Linux +  74)
+#define __NR_setrlimit			(__NR_Linux +  75)
+#define __NR_getrlimit			(__NR_Linux +  76)
+#define __NR_getrusage			(__NR_Linux +  77)
+#define __NR_gettimeofday		(__NR_Linux +  78)
+#define __NR_settimeofday		(__NR_Linux +  79)
+#define __NR_getgroups			(__NR_Linux +  80)
+#define __NR_setgroups			(__NR_Linux +  81)
+#define __NR_reserved82			(__NR_Linux +  82)
+#define __NR_symlink			(__NR_Linux +  83)
+#define __NR_unused84			(__NR_Linux +  84)
+#define __NR_readlink			(__NR_Linux +  85)
+#define __NR_uselib			(__NR_Linux +  86)
+#define __NR_swapon			(__NR_Linux +  87)
+#define __NR_reboot			(__NR_Linux +  88)
+#define __NR_readdir			(__NR_Linux +  89)
+#define __NR_mmap			(__NR_Linux +  90)
+#define __NR_munmap			(__NR_Linux +  91)
+#define __NR_truncate			(__NR_Linux +  92)
+#define __NR_ftruncate			(__NR_Linux +  93)
+#define __NR_fchmod			(__NR_Linux +  94)
+#define __NR_fchown			(__NR_Linux +  95)
+#define __NR_getpriority		(__NR_Linux +  96)
+#define __NR_setpriority		(__NR_Linux +  97)
+#define __NR_profil			(__NR_Linux +  98)
+#define __NR_statfs			(__NR_Linux +  99)
+#define __NR_fstatfs			(__NR_Linux + 100)
+#define __NR_ioperm			(__NR_Linux + 101)
+#define __NR_socketcall			(__NR_Linux + 102)
+#define __NR_syslog			(__NR_Linux + 103)
+#define __NR_setitimer			(__NR_Linux + 104)
+#define __NR_getitimer			(__NR_Linux + 105)
+#define __NR_stat			(__NR_Linux + 106)
+#define __NR_lstat			(__NR_Linux + 107)
+#define __NR_fstat			(__NR_Linux + 108)
+#define __NR_unused109			(__NR_Linux + 109)
+#define __NR_iopl			(__NR_Linux + 110)
+#define __NR_vhangup			(__NR_Linux + 111)
+#define __NR_idle			(__NR_Linux + 112)
+#define __NR_vm86			(__NR_Linux + 113)
+#define __NR_wait4			(__NR_Linux + 114)
+#define __NR_swapoff			(__NR_Linux + 115)
+#define __NR_sysinfo			(__NR_Linux + 116)
+#define __NR_ipc			(__NR_Linux + 117)
+#define __NR_fsync			(__NR_Linux + 118)
+#define __NR_sigreturn			(__NR_Linux + 119)
+#define __NR_clone			(__NR_Linux + 120)
+#define __NR_setdomainname		(__NR_Linux + 121)
+#define __NR_uname			(__NR_Linux + 122)
+#define __NR_modify_ldt			(__NR_Linux + 123)
+#define __NR_adjtimex			(__NR_Linux + 124)
+#define __NR_mprotect			(__NR_Linux + 125)
+#define __NR_sigprocmask		(__NR_Linux + 126)
+#define __NR_create_module		(__NR_Linux + 127)
+#define __NR_init_module		(__NR_Linux + 128)
+#define __NR_delete_module		(__NR_Linux + 129)
+#define __NR_get_kernel_syms		(__NR_Linux + 130)
+#define __NR_quotactl			(__NR_Linux + 131)
+#define __NR_getpgid			(__NR_Linux + 132)
+#define __NR_fchdir			(__NR_Linux + 133)
+#define __NR_bdflush			(__NR_Linux + 134)
+#define __NR_sysfs			(__NR_Linux + 135)
+#define __NR_personality		(__NR_Linux + 136)
+#define __NR_afs_syscall		(__NR_Linux + 137) /* Syscall for Andrew File System */
+#define __NR_setfsuid			(__NR_Linux + 138)
+#define __NR_setfsgid			(__NR_Linux + 139)
+#define __NR__llseek			(__NR_Linux + 140)
+#define __NR_getdents			(__NR_Linux + 141)
+#define __NR__newselect			(__NR_Linux + 142)
+#define __NR_flock			(__NR_Linux + 143)
+#define __NR_msync			(__NR_Linux + 144)
+#define __NR_readv			(__NR_Linux + 145)
+#define __NR_writev			(__NR_Linux + 146)
+#define __NR_cacheflush			(__NR_Linux + 147)
+#define __NR_cachectl			(__NR_Linux + 148)
+#define __NR_sysmips			(__NR_Linux + 149)
+#define __NR_unused150			(__NR_Linux + 150)
+#define __NR_getsid			(__NR_Linux + 151)
+#define __NR_fdatasync			(__NR_Linux + 152)
+#define __NR__sysctl			(__NR_Linux + 153)
+#define __NR_mlock			(__NR_Linux + 154)
+#define __NR_munlock			(__NR_Linux + 155)
+#define __NR_mlockall			(__NR_Linux + 156)
+#define __NR_munlockall			(__NR_Linux + 157)
+#define __NR_sched_setparam		(__NR_Linux + 158)
+#define __NR_sched_getparam		(__NR_Linux + 159)
+#define __NR_sched_setscheduler		(__NR_Linux + 160)
+#define __NR_sched_getscheduler		(__NR_Linux + 161)
+#define __NR_sched_yield		(__NR_Linux + 162)
+#define __NR_sched_get_priority_max	(__NR_Linux + 163)
+#define __NR_sched_get_priority_min	(__NR_Linux + 164)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 165)
+#define __NR_nanosleep			(__NR_Linux + 166)
+#define __NR_mremap			(__NR_Linux + 167)
+#define __NR_accept			(__NR_Linux + 168)
+#define __NR_bind			(__NR_Linux + 169)
+#define __NR_connect			(__NR_Linux + 170)
+#define __NR_getpeername		(__NR_Linux + 171)
+#define __NR_getsockname		(__NR_Linux + 172)
+#define __NR_getsockopt			(__NR_Linux + 173)
+#define __NR_listen			(__NR_Linux + 174)
+#define __NR_recv			(__NR_Linux + 175)
+#define __NR_recvfrom			(__NR_Linux + 176)
+#define __NR_recvmsg			(__NR_Linux + 177)
+#define __NR_send			(__NR_Linux + 178)
+#define __NR_sendmsg			(__NR_Linux + 179)
+#define __NR_sendto			(__NR_Linux + 180)
+#define __NR_setsockopt			(__NR_Linux + 181)
+#define __NR_shutdown			(__NR_Linux + 182)
+#define __NR_socket			(__NR_Linux + 183)
+#define __NR_socketpair			(__NR_Linux + 184)
+#define __NR_setresuid			(__NR_Linux + 185)
+#define __NR_getresuid			(__NR_Linux + 186)
+#define __NR_query_module		(__NR_Linux + 187)
+#define __NR_poll			(__NR_Linux + 188)
+#define __NR_nfsservctl			(__NR_Linux + 189)
+#define __NR_setresgid			(__NR_Linux + 190)
+#define __NR_getresgid			(__NR_Linux + 191)
+#define __NR_prctl			(__NR_Linux + 192)
+#define __NR_rt_sigreturn		(__NR_Linux + 193)
+#define __NR_rt_sigaction		(__NR_Linux + 194)
+#define __NR_rt_sigprocmask		(__NR_Linux + 195)
+#define __NR_rt_sigpending		(__NR_Linux + 196)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 197)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 198)
+#define __NR_rt_sigsuspend		(__NR_Linux + 199)
+#define __NR_pread64			(__NR_Linux + 200)
+#define __NR_pwrite64			(__NR_Linux + 201)
+#define __NR_chown			(__NR_Linux + 202)
+#define __NR_getcwd			(__NR_Linux + 203)
+#define __NR_capget			(__NR_Linux + 204)
+#define __NR_capset			(__NR_Linux + 205)
+#define __NR_sigaltstack		(__NR_Linux + 206)
+#define __NR_sendfile			(__NR_Linux + 207)
+#define __NR_getpmsg			(__NR_Linux + 208)
+#define __NR_putpmsg			(__NR_Linux + 209)
+#define __NR_mmap2			(__NR_Linux + 210)
+#define __NR_truncate64			(__NR_Linux + 211)
+#define __NR_ftruncate64		(__NR_Linux + 212)
+#define __NR_stat64			(__NR_Linux + 213)
+#define __NR_lstat64			(__NR_Linux + 214)
+#define __NR_fstat64			(__NR_Linux + 215)
+#define __NR_pivot_root			(__NR_Linux + 216)
+#define __NR_mincore			(__NR_Linux + 217)
+#define __NR_madvise			(__NR_Linux + 218)
+#define __NR_getdents64			(__NR_Linux + 219)
+#define __NR_fcntl64			(__NR_Linux + 220)
+#define __NR_reserved221		(__NR_Linux + 221)
+#define __NR_gettid			(__NR_Linux + 222)
+#define __NR_readahead			(__NR_Linux + 223)
+#define __NR_setxattr			(__NR_Linux + 224)
+#define __NR_lsetxattr			(__NR_Linux + 225)
+#define __NR_fsetxattr			(__NR_Linux + 226)
+#define __NR_getxattr			(__NR_Linux + 227)
+#define __NR_lgetxattr			(__NR_Linux + 228)
+#define __NR_fgetxattr			(__NR_Linux + 229)
+#define __NR_listxattr			(__NR_Linux + 230)
+#define __NR_llistxattr			(__NR_Linux + 231)
+#define __NR_flistxattr			(__NR_Linux + 232)
+#define __NR_removexattr		(__NR_Linux + 233)
+#define __NR_lremovexattr		(__NR_Linux + 234)
+#define __NR_fremovexattr		(__NR_Linux + 235)
+#define __NR_tkill			(__NR_Linux + 236)
+#define __NR_sendfile64			(__NR_Linux + 237)
+#define __NR_futex			(__NR_Linux + 238)
+#define __NR_sched_setaffinity		(__NR_Linux + 239)
+#define __NR_sched_getaffinity		(__NR_Linux + 240)
+#define __NR_io_setup			(__NR_Linux + 241)
+#define __NR_io_destroy			(__NR_Linux + 242)
+#define __NR_io_getevents		(__NR_Linux + 243)
+#define __NR_io_submit			(__NR_Linux + 244)
+#define __NR_io_cancel			(__NR_Linux + 245)
+#define __NR_exit_group			(__NR_Linux + 246)
+#define __NR_lookup_dcookie		(__NR_Linux + 247)
+#define __NR_epoll_create		(__NR_Linux + 248)
+#define __NR_epoll_ctl			(__NR_Linux + 249)
+#define __NR_epoll_wait			(__NR_Linux + 250)
+#define __NR_remap_file_pages		(__NR_Linux + 251)
+#define __NR_set_tid_address		(__NR_Linux + 252)
+#define __NR_restart_syscall		(__NR_Linux + 253)
+#define __NR_fadvise64			(__NR_Linux + 254)
+#define __NR_statfs64			(__NR_Linux + 255)
+#define __NR_fstatfs64			(__NR_Linux + 256)
+#define __NR_timer_create		(__NR_Linux + 257)
+#define __NR_timer_settime		(__NR_Linux + 258)
+#define __NR_timer_gettime		(__NR_Linux + 259)
+#define __NR_timer_getoverrun		(__NR_Linux + 260)
+#define __NR_timer_delete		(__NR_Linux + 261)
+#define __NR_clock_settime		(__NR_Linux + 262)
+#define __NR_clock_gettime		(__NR_Linux + 263)
+#define __NR_clock_getres		(__NR_Linux + 264)
+#define __NR_clock_nanosleep		(__NR_Linux + 265)
+#define __NR_tgkill			(__NR_Linux + 266)
+#define __NR_utimes			(__NR_Linux + 267)
+#define __NR_mbind			(__NR_Linux + 268)
+#define __NR_get_mempolicy		(__NR_Linux + 269)
+#define __NR_set_mempolicy		(__NR_Linux + 270)
+#define __NR_mq_open			(__NR_Linux + 271)
+#define __NR_mq_unlink			(__NR_Linux + 272)
+#define __NR_mq_timedsend		(__NR_Linux + 273)
+#define __NR_mq_timedreceive		(__NR_Linux + 274)
+#define __NR_mq_notify			(__NR_Linux + 275)
+#define __NR_mq_getsetattr		(__NR_Linux + 276)
+#define __NR_vserver			(__NR_Linux + 277)
+#define __NR_waitid			(__NR_Linux + 278)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 279) */
+#define __NR_add_key			(__NR_Linux + 280)
+#define __NR_request_key		(__NR_Linux + 281)
+#define __NR_keyctl			(__NR_Linux + 282)
+#define __NR_set_thread_area		(__NR_Linux + 283)
+#define __NR_inotify_init		(__NR_Linux + 284)
+#define __NR_inotify_add_watch		(__NR_Linux + 285)
+#define __NR_inotify_rm_watch		(__NR_Linux + 286)
+#define __NR_migrate_pages		(__NR_Linux + 287)
+#define __NR_openat			(__NR_Linux + 288)
+#define __NR_mkdirat			(__NR_Linux + 289)
+#define __NR_mknodat			(__NR_Linux + 290)
+#define __NR_fchownat			(__NR_Linux + 291)
+#define __NR_futimesat			(__NR_Linux + 292)
+#define __NR_fstatat64			(__NR_Linux + 293)
+#define __NR_unlinkat			(__NR_Linux + 294)
+#define __NR_renameat			(__NR_Linux + 295)
+#define __NR_linkat			(__NR_Linux + 296)
+#define __NR_symlinkat			(__NR_Linux + 297)
+#define __NR_readlinkat			(__NR_Linux + 298)
+#define __NR_fchmodat			(__NR_Linux + 299)
+#define __NR_faccessat			(__NR_Linux + 300)
+#define __NR_pselect6			(__NR_Linux + 301)
+#define __NR_ppoll			(__NR_Linux + 302)
+#define __NR_unshare			(__NR_Linux + 303)
+#define __NR_splice			(__NR_Linux + 304)
+#define __NR_sync_file_range		(__NR_Linux + 305)
+#define __NR_tee			(__NR_Linux + 306)
+#define __NR_vmsplice			(__NR_Linux + 307)
+#define __NR_move_pages			(__NR_Linux + 308)
+#define __NR_set_robust_list		(__NR_Linux + 309)
+#define __NR_get_robust_list		(__NR_Linux + 310)
+#define __NR_kexec_load			(__NR_Linux + 311)
+#define __NR_getcpu			(__NR_Linux + 312)
+#define __NR_epoll_pwait		(__NR_Linux + 313)
+#define __NR_ioprio_set			(__NR_Linux + 314)
+#define __NR_ioprio_get			(__NR_Linux + 315)
+#define __NR_utimensat			(__NR_Linux + 316)
+#define __NR_signalfd			(__NR_Linux + 317)
+#define __NR_timerfd			(__NR_Linux + 318)
+#define __NR_eventfd			(__NR_Linux + 319)
+#define __NR_fallocate			(__NR_Linux + 320)
+#define __NR_timerfd_create		(__NR_Linux + 321)
+#define __NR_timerfd_gettime		(__NR_Linux + 322)
+#define __NR_timerfd_settime		(__NR_Linux + 323)
+#define __NR_signalfd4			(__NR_Linux + 324)
+#define __NR_eventfd2			(__NR_Linux + 325)
+#define __NR_epoll_create1		(__NR_Linux + 326)
+#define __NR_dup3			(__NR_Linux + 327)
+#define __NR_pipe2			(__NR_Linux + 328)
+#define __NR_inotify_init1		(__NR_Linux + 329)
+#define __NR_preadv			(__NR_Linux + 330)
+#define __NR_pwritev			(__NR_Linux + 331)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 332)
+#define __NR_perf_event_open		(__NR_Linux + 333)
+#define __NR_accept4			(__NR_Linux + 334)
+#define __NR_recvmmsg			(__NR_Linux + 335)
+#define __NR_fanotify_init		(__NR_Linux + 336)
+#define __NR_fanotify_mark		(__NR_Linux + 337)
+#define __NR_prlimit64			(__NR_Linux + 338)
+#define __NR_name_to_handle_at		(__NR_Linux + 339)
+#define __NR_open_by_handle_at		(__NR_Linux + 340)
+#define __NR_clock_adjtime		(__NR_Linux + 341)
+#define __NR_syncfs			(__NR_Linux + 342)
+#define __NR_sendmmsg			(__NR_Linux + 343)
+#define __NR_setns			(__NR_Linux + 344)
+#define __NR_process_vm_readv		(__NR_Linux + 345)
+#define __NR_process_vm_writev		(__NR_Linux + 346)
+#define __NR_kcmp			(__NR_Linux + 347)
+#define __NR_finit_module		(__NR_Linux + 348)
+#define __NR_sched_setattr		(__NR_Linux + 349)
+#define __NR_sched_getattr		(__NR_Linux + 350)
+#define __NR_renameat2			(__NR_Linux + 351)
+#define __NR_seccomp			(__NR_Linux + 352)
+#define __NR_getrandom			(__NR_Linux + 353)
+#define __NR_memfd_create		(__NR_Linux + 354)
+#define __NR_bpf			(__NR_Linux + 355)
+#define __NR_execveat			(__NR_Linux + 356)
+
+/*
+ * Offset of the last Linux o32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		356
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
+
+#define __NR_O32_Linux			4000
+#define __NR_O32_Linux_syscalls		356
+
+#if _MIPS_SIM == _MIPS_SIM_ABI64
+
+/*
+ * Linux 64-bit syscalls are in the range from 5000 to 5999.
+ */
+#define __NR_Linux			5000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_set_tid_address		(__NR_Linux + 212)
+#define __NR_restart_syscall		(__NR_Linux + 213)
+#define __NR_semtimedop			(__NR_Linux + 214)
+#define __NR_fadvise64			(__NR_Linux + 215)
+#define __NR_timer_create		(__NR_Linux + 216)
+#define __NR_timer_settime		(__NR_Linux + 217)
+#define __NR_timer_gettime		(__NR_Linux + 218)
+#define __NR_timer_getoverrun		(__NR_Linux + 219)
+#define __NR_timer_delete		(__NR_Linux + 220)
+#define __NR_clock_settime		(__NR_Linux + 221)
+#define __NR_clock_gettime		(__NR_Linux + 222)
+#define __NR_clock_getres		(__NR_Linux + 223)
+#define __NR_clock_nanosleep		(__NR_Linux + 224)
+#define __NR_tgkill			(__NR_Linux + 225)
+#define __NR_utimes			(__NR_Linux + 226)
+#define __NR_mbind			(__NR_Linux + 227)
+#define __NR_get_mempolicy		(__NR_Linux + 228)
+#define __NR_set_mempolicy		(__NR_Linux + 229)
+#define __NR_mq_open			(__NR_Linux + 230)
+#define __NR_mq_unlink			(__NR_Linux + 231)
+#define __NR_mq_timedsend		(__NR_Linux + 232)
+#define __NR_mq_timedreceive		(__NR_Linux + 233)
+#define __NR_mq_notify			(__NR_Linux + 234)
+#define __NR_mq_getsetattr		(__NR_Linux + 235)
+#define __NR_vserver			(__NR_Linux + 236)
+#define __NR_waitid			(__NR_Linux + 237)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 238) */
+#define __NR_add_key			(__NR_Linux + 239)
+#define __NR_request_key		(__NR_Linux + 240)
+#define __NR_keyctl			(__NR_Linux + 241)
+#define __NR_set_thread_area		(__NR_Linux + 242)
+#define __NR_inotify_init		(__NR_Linux + 243)
+#define __NR_inotify_add_watch		(__NR_Linux + 244)
+#define __NR_inotify_rm_watch		(__NR_Linux + 245)
+#define __NR_migrate_pages		(__NR_Linux + 246)
+#define __NR_openat			(__NR_Linux + 247)
+#define __NR_mkdirat			(__NR_Linux + 248)
+#define __NR_mknodat			(__NR_Linux + 249)
+#define __NR_fchownat			(__NR_Linux + 250)
+#define __NR_futimesat			(__NR_Linux + 251)
+#define __NR_newfstatat			(__NR_Linux + 252)
+#define __NR_unlinkat			(__NR_Linux + 253)
+#define __NR_renameat			(__NR_Linux + 254)
+#define __NR_linkat			(__NR_Linux + 255)
+#define __NR_symlinkat			(__NR_Linux + 256)
+#define __NR_readlinkat			(__NR_Linux + 257)
+#define __NR_fchmodat			(__NR_Linux + 258)
+#define __NR_faccessat			(__NR_Linux + 259)
+#define __NR_pselect6			(__NR_Linux + 260)
+#define __NR_ppoll			(__NR_Linux + 261)
+#define __NR_unshare			(__NR_Linux + 262)
+#define __NR_splice			(__NR_Linux + 263)
+#define __NR_sync_file_range		(__NR_Linux + 264)
+#define __NR_tee			(__NR_Linux + 265)
+#define __NR_vmsplice			(__NR_Linux + 266)
+#define __NR_move_pages			(__NR_Linux + 267)
+#define __NR_set_robust_list		(__NR_Linux + 268)
+#define __NR_get_robust_list		(__NR_Linux + 269)
+#define __NR_kexec_load			(__NR_Linux + 270)
+#define __NR_getcpu			(__NR_Linux + 271)
+#define __NR_epoll_pwait		(__NR_Linux + 272)
+#define __NR_ioprio_set			(__NR_Linux + 273)
+#define __NR_ioprio_get			(__NR_Linux + 274)
+#define __NR_utimensat			(__NR_Linux + 275)
+#define __NR_signalfd			(__NR_Linux + 276)
+#define __NR_timerfd			(__NR_Linux + 277)
+#define __NR_eventfd			(__NR_Linux + 278)
+#define __NR_fallocate			(__NR_Linux + 279)
+#define __NR_timerfd_create		(__NR_Linux + 280)
+#define __NR_timerfd_gettime		(__NR_Linux + 281)
+#define __NR_timerfd_settime		(__NR_Linux + 282)
+#define __NR_signalfd4			(__NR_Linux + 283)
+#define __NR_eventfd2			(__NR_Linux + 284)
+#define __NR_epoll_create1		(__NR_Linux + 285)
+#define __NR_dup3			(__NR_Linux + 286)
+#define __NR_pipe2			(__NR_Linux + 287)
+#define __NR_inotify_init1		(__NR_Linux + 288)
+#define __NR_preadv			(__NR_Linux + 289)
+#define __NR_pwritev			(__NR_Linux + 290)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 291)
+#define __NR_perf_event_open		(__NR_Linux + 292)
+#define __NR_accept4			(__NR_Linux + 293)
+#define __NR_recvmmsg			(__NR_Linux + 294)
+#define __NR_fanotify_init		(__NR_Linux + 295)
+#define __NR_fanotify_mark		(__NR_Linux + 296)
+#define __NR_prlimit64			(__NR_Linux + 297)
+#define __NR_name_to_handle_at		(__NR_Linux + 298)
+#define __NR_open_by_handle_at		(__NR_Linux + 299)
+#define __NR_clock_adjtime		(__NR_Linux + 300)
+#define __NR_syncfs			(__NR_Linux + 301)
+#define __NR_sendmmsg			(__NR_Linux + 302)
+#define __NR_setns			(__NR_Linux + 303)
+#define __NR_process_vm_readv		(__NR_Linux + 304)
+#define __NR_process_vm_writev		(__NR_Linux + 305)
+#define __NR_kcmp			(__NR_Linux + 306)
+#define __NR_finit_module		(__NR_Linux + 307)
+#define __NR_getdents64			(__NR_Linux + 308)
+#define __NR_sched_setattr		(__NR_Linux + 309)
+#define __NR_sched_getattr		(__NR_Linux + 310)
+#define __NR_renameat2			(__NR_Linux + 311)
+#define __NR_seccomp			(__NR_Linux + 312)
+#define __NR_getrandom			(__NR_Linux + 313)
+#define __NR_memfd_create		(__NR_Linux + 314)
+#define __NR_bpf			(__NR_Linux + 315)
+#define __NR_execveat			(__NR_Linux + 316)
+
+/*
+ * Offset of the last Linux 64-bit flavoured syscall
+ */
+#define __NR_Linux_syscalls		316
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI64 */
+
+#define __NR_64_Linux			5000
+#define __NR_64_Linux_syscalls		316
+
+#if _MIPS_SIM == _MIPS_SIM_NABI32
+
+/*
+ * Linux N32 syscalls are in the range from 6000 to 6999.
+ */
+#define __NR_Linux			6000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_fcntl64			(__NR_Linux + 212)
+#define __NR_set_tid_address		(__NR_Linux + 213)
+#define __NR_restart_syscall		(__NR_Linux + 214)
+#define __NR_semtimedop			(__NR_Linux + 215)
+#define __NR_fadvise64			(__NR_Linux + 216)
+#define __NR_statfs64			(__NR_Linux + 217)
+#define __NR_fstatfs64			(__NR_Linux + 218)
+#define __NR_sendfile64			(__NR_Linux + 219)
+#define __NR_timer_create		(__NR_Linux + 220)
+#define __NR_timer_settime		(__NR_Linux + 221)
+#define __NR_timer_gettime		(__NR_Linux + 222)
+#define __NR_timer_getoverrun		(__NR_Linux + 223)
+#define __NR_timer_delete		(__NR_Linux + 224)
+#define __NR_clock_settime		(__NR_Linux + 225)
+#define __NR_clock_gettime		(__NR_Linux + 226)
+#define __NR_clock_getres		(__NR_Linux + 227)
+#define __NR_clock_nanosleep		(__NR_Linux + 228)
+#define __NR_tgkill			(__NR_Linux + 229)
+#define __NR_utimes			(__NR_Linux + 230)
+#define __NR_mbind			(__NR_Linux + 231)
+#define __NR_get_mempolicy		(__NR_Linux + 232)
+#define __NR_set_mempolicy		(__NR_Linux + 233)
+#define __NR_mq_open			(__NR_Linux + 234)
+#define __NR_mq_unlink			(__NR_Linux + 235)
+#define __NR_mq_timedsend		(__NR_Linux + 236)
+#define __NR_mq_timedreceive		(__NR_Linux + 237)
+#define __NR_mq_notify			(__NR_Linux + 238)
+#define __NR_mq_getsetattr		(__NR_Linux + 239)
+#define __NR_vserver			(__NR_Linux + 240)
+#define __NR_waitid			(__NR_Linux + 241)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 242) */
+#define __NR_add_key			(__NR_Linux + 243)
+#define __NR_request_key		(__NR_Linux + 244)
+#define __NR_keyctl			(__NR_Linux + 245)
+#define __NR_set_thread_area		(__NR_Linux + 246)
+#define __NR_inotify_init		(__NR_Linux + 247)
+#define __NR_inotify_add_watch		(__NR_Linux + 248)
+#define __NR_inotify_rm_watch		(__NR_Linux + 249)
+#define __NR_migrate_pages		(__NR_Linux + 250)
+#define __NR_openat			(__NR_Linux + 251)
+#define __NR_mkdirat			(__NR_Linux + 252)
+#define __NR_mknodat			(__NR_Linux + 253)
+#define __NR_fchownat			(__NR_Linux + 254)
+#define __NR_futimesat			(__NR_Linux + 255)
+#define __NR_newfstatat			(__NR_Linux + 256)
+#define __NR_unlinkat			(__NR_Linux + 257)
+#define __NR_renameat			(__NR_Linux + 258)
+#define __NR_linkat			(__NR_Linux + 259)
+#define __NR_symlinkat			(__NR_Linux + 260)
+#define __NR_readlinkat			(__NR_Linux + 261)
+#define __NR_fchmodat			(__NR_Linux + 262)
+#define __NR_faccessat			(__NR_Linux + 263)
+#define __NR_pselect6			(__NR_Linux + 264)
+#define __NR_ppoll			(__NR_Linux + 265)
+#define __NR_unshare			(__NR_Linux + 266)
+#define __NR_splice			(__NR_Linux + 267)
+#define __NR_sync_file_range		(__NR_Linux + 268)
+#define __NR_tee			(__NR_Linux + 269)
+#define __NR_vmsplice			(__NR_Linux + 270)
+#define __NR_move_pages			(__NR_Linux + 271)
+#define __NR_set_robust_list		(__NR_Linux + 272)
+#define __NR_get_robust_list		(__NR_Linux + 273)
+#define __NR_kexec_load			(__NR_Linux + 274)
+#define __NR_getcpu			(__NR_Linux + 275)
+#define __NR_epoll_pwait		(__NR_Linux + 276)
+#define __NR_ioprio_set			(__NR_Linux + 277)
+#define __NR_ioprio_get			(__NR_Linux + 278)
+#define __NR_utimensat			(__NR_Linux + 279)
+#define __NR_signalfd			(__NR_Linux + 280)
+#define __NR_timerfd			(__NR_Linux + 281)
+#define __NR_eventfd			(__NR_Linux + 282)
+#define __NR_fallocate			(__NR_Linux + 283)
+#define __NR_timerfd_create		(__NR_Linux + 284)
+#define __NR_timerfd_gettime		(__NR_Linux + 285)
+#define __NR_timerfd_settime		(__NR_Linux + 286)
+#define __NR_signalfd4			(__NR_Linux + 287)
+#define __NR_eventfd2			(__NR_Linux + 288)
+#define __NR_epoll_create1		(__NR_Linux + 289)
+#define __NR_dup3			(__NR_Linux + 290)
+#define __NR_pipe2			(__NR_Linux + 291)
+#define __NR_inotify_init1		(__NR_Linux + 292)
+#define __NR_preadv			(__NR_Linux + 293)
+#define __NR_pwritev			(__NR_Linux + 294)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 295)
+#define __NR_perf_event_open		(__NR_Linux + 296)
+#define __NR_accept4			(__NR_Linux + 297)
+#define __NR_recvmmsg			(__NR_Linux + 298)
+#define __NR_getdents64			(__NR_Linux + 299)
+#define __NR_fanotify_init		(__NR_Linux + 300)
+#define __NR_fanotify_mark		(__NR_Linux + 301)
+#define __NR_prlimit64			(__NR_Linux + 302)
+#define __NR_name_to_handle_at		(__NR_Linux + 303)
+#define __NR_open_by_handle_at		(__NR_Linux + 304)
+#define __NR_clock_adjtime		(__NR_Linux + 305)
+#define __NR_syncfs			(__NR_Linux + 306)
+#define __NR_sendmmsg			(__NR_Linux + 307)
+#define __NR_setns			(__NR_Linux + 308)
+#define __NR_process_vm_readv		(__NR_Linux + 309)
+#define __NR_process_vm_writev		(__NR_Linux + 310)
+#define __NR_kcmp			(__NR_Linux + 311)
+#define __NR_finit_module		(__NR_Linux + 312)
+#define __NR_sched_setattr		(__NR_Linux + 313)
+#define __NR_sched_getattr		(__NR_Linux + 314)
+#define __NR_renameat2			(__NR_Linux + 315)
+#define __NR_seccomp			(__NR_Linux + 316)
+#define __NR_getrandom			(__NR_Linux + 317)
+#define __NR_memfd_create		(__NR_Linux + 318)
+#define __NR_bpf			(__NR_Linux + 319)
+#define __NR_execveat			(__NR_Linux + 320)
+
+/*
+ * Offset of the last N32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		320
+
+#endif /* _MIPS_SIM == _MIPS_SIM_NABI32 */
+
+#define __NR_N32_Linux			6000
+#define __NR_N32_Linux_syscalls		320
+
+#endif /* _ASM_UNISTD_H */
diff --git a/qemu/linux-headers/asm-powerpc/epapr_hcalls.h b/qemu/linux-headers/asm-powerpc/epapr_hcalls.h
index 06f724786..33b3f89f5 100644
--- a/qemu/linux-headers/asm-powerpc/epapr_hcalls.h
+++ b/qemu/linux-headers/asm-powerpc/epapr_hcalls.h
@@ -78,7 +78,7 @@
 #define EV_SUCCESS		0
 #define EV_EPERM		1	/* Operation not permitted */
 #define EV_ENOENT		2	/*  Entry Not Found */
-#define EV_EIO			3	/* I/O error occured */
+#define EV_EIO			3	/* I/O error occurred */
 #define EV_EAGAIN		4	/* The operation had insufficient
 					 * resources to complete and should be
 					 * retried
@@ -89,7 +89,7 @@
 #define EV_ENODEV		7	/* No such device */
 #define EV_EINVAL		8	/* An argument supplied to the hcall
 					   was out of range or invalid */
-#define EV_INTERNAL		9	/* An internal error occured */
+#define EV_INTERNAL		9	/* An internal error occurred */
 #define EV_CONFIG		10	/* A configuration error was detected */
 #define EV_INVALID_STATE	11	/* The object is in an invalid state */
 #define EV_UNIMPLEMENTED	12	/* Unimplemented hypercall */
diff --git a/qemu/linux-headers/asm-powerpc/kvm.h b/qemu/linux-headers/asm-powerpc/kvm.h
index ab4d4732c..c93cf35ce 100644
--- a/qemu/linux-headers/asm-powerpc/kvm.h
+++ b/qemu/linux-headers/asm-powerpc/kvm.h
@@ -333,6 +333,15 @@ struct kvm_create_spapr_tce {
 	__u32 window_size;
 };
 
+/* for KVM_CAP_SPAPR_TCE_64 */
+struct kvm_create_spapr_tce_64 {
+	__u64 liobn;
+	__u32 page_shift;
+	__u32 flags;
+	__u64 offset;	/* in pages */
+	__u64 size;	/* in pages */
+};
+
 /* for KVM_ALLOCATE_RMA */
 struct kvm_allocate_rma {
 	__u64 rma_size;
diff --git a/qemu/linux-headers/asm-powerpc/unistd.h b/qemu/linux-headers/asm-powerpc/unistd.h
new file mode 100644
index 000000000..cd92d982f
--- /dev/null
+++ b/qemu/linux-headers/asm-powerpc/unistd.h
@@ -0,0 +1,394 @@
+/*
+ * This file contains the system call numbers.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#ifndef _ASM_POWERPC_UNISTD_H_
+#define _ASM_POWERPC_UNISTD_H_
+
+
+#define __NR_restart_syscall	  0
+#define __NR_exit		  1
+#define __NR_fork		  2
+#define __NR_read		  3
+#define __NR_write		  4
+#define __NR_open		  5
+#define __NR_close		  6
+#define __NR_waitpid		  7
+#define __NR_creat		  8
+#define __NR_link		  9
+#define __NR_unlink		 10
+#define __NR_execve		 11
+#define __NR_chdir		 12
+#define __NR_time		 13
+#define __NR_mknod		 14
+#define __NR_chmod		 15
+#define __NR_lchown		 16
+#define __NR_break		 17
+#define __NR_oldstat		 18
+#define __NR_lseek		 19
+#define __NR_getpid		 20
+#define __NR_mount		 21
+#define __NR_umount		 22
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_ptrace		 26
+#define __NR_alarm		 27
+#define __NR_oldfstat		 28
+#define __NR_pause		 29
+#define __NR_utime		 30
+#define __NR_stty		 31
+#define __NR_gtty		 32
+#define __NR_access		 33
+#define __NR_nice		 34
+#define __NR_ftime		 35
+#define __NR_sync		 36
+#define __NR_kill		 37
+#define __NR_rename		 38
+#define __NR_mkdir		 39
+#define __NR_rmdir		 40
+#define __NR_dup		 41
+#define __NR_pipe		 42
+#define __NR_times		 43
+#define __NR_prof		 44
+#define __NR_brk		 45
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_signal		 48
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_acct		 51
+#define __NR_umount2		 52
+#define __NR_lock		 53
+#define __NR_ioctl		 54
+#define __NR_fcntl		 55
+#define __NR_mpx		 56
+#define __NR_setpgid		 57
+#define __NR_ulimit		 58
+#define __NR_oldolduname	 59
+#define __NR_umask		 60
+#define __NR_chroot		 61
+#define __NR_ustat		 62
+#define __NR_dup2		 63
+#define __NR_getppid		 64
+#define __NR_getpgrp		 65
+#define __NR_setsid		 66
+#define __NR_sigaction		 67
+#define __NR_sgetmask		 68
+#define __NR_ssetmask		 69
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_sigsuspend		 72
+#define __NR_sigpending		 73
+#define __NR_sethostname	 74
+#define __NR_setrlimit		 75
+#define __NR_getrlimit		 76
+#define __NR_getrusage		 77
+#define __NR_gettimeofday	 78
+#define __NR_settimeofday	 79
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_select		 82
+#define __NR_symlink		 83
+#define __NR_oldlstat		 84
+#define __NR_readlink		 85
+#define __NR_uselib		 86
+#define __NR_swapon		 87
+#define __NR_reboot		 88
+#define __NR_readdir		 89
+#define __NR_mmap		 90
+#define __NR_munmap		 91
+#define __NR_truncate		 92
+#define __NR_ftruncate		 93
+#define __NR_fchmod		 94
+#define __NR_fchown		 95
+#define __NR_getpriority	 96
+#define __NR_setpriority	 97
+#define __NR_profil		 98
+#define __NR_statfs		 99
+#define __NR_fstatfs		100
+#define __NR_ioperm		101
+#define __NR_socketcall		102
+#define __NR_syslog		103
+#define __NR_setitimer		104
+#define __NR_getitimer		105
+#define __NR_stat		106
+#define __NR_lstat		107
+#define __NR_fstat		108
+#define __NR_olduname		109
+#define __NR_iopl		110
+#define __NR_vhangup		111
+#define __NR_idle		112
+#define __NR_vm86		113
+#define __NR_wait4		114
+#define __NR_swapoff		115
+#define __NR_sysinfo		116
+#define __NR_ipc		117
+#define __NR_fsync		118
+#define __NR_sigreturn		119
+#define __NR_clone		120
+#define __NR_setdomainname	121
+#define __NR_uname		122
+#define __NR_modify_ldt		123
+#define __NR_adjtimex		124
+#define __NR_mprotect		125
+#define __NR_sigprocmask	126
+#define __NR_create_module	127
+#define __NR_init_module	128
+#define __NR_delete_module	129
+#define __NR_get_kernel_syms	130
+#define __NR_quotactl		131
+#define __NR_getpgid		132
+#define __NR_fchdir		133
+#define __NR_bdflush		134
+#define __NR_sysfs		135
+#define __NR_personality	136
+#define __NR_afs_syscall	137 /* Syscall for Andrew File System */
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR_getdents		141
+#define __NR__newselect		142
+#define __NR_flock		143
+#define __NR_msync		144
+#define __NR_readv		145
+#define __NR_writev		146
+#define __NR_getsid		147
+#define __NR_fdatasync		148
+#define __NR__sysctl		149
+#define __NR_mlock		150
+#define __NR_munlock		151
+#define __NR_mlockall		152
+#define __NR_munlockall		153
+#define __NR_sched_setparam		154
+#define __NR_sched_getparam		155
+#define __NR_sched_setscheduler		156
+#define __NR_sched_getscheduler		157
+#define __NR_sched_yield		158
+#define __NR_sched_get_priority_max	159
+#define __NR_sched_get_priority_min	160
+#define __NR_sched_rr_get_interval	161
+#define __NR_nanosleep		162
+#define __NR_mremap		163
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_query_module	166
+#define __NR_poll		167
+#define __NR_nfsservctl		168
+#define __NR_setresgid		169
+#define __NR_getresgid		170
+#define __NR_prctl		171
+#define __NR_rt_sigreturn	172
+#define __NR_rt_sigaction	173
+#define __NR_rt_sigprocmask	174
+#define __NR_rt_sigpending	175
+#define __NR_rt_sigtimedwait	176
+#define __NR_rt_sigqueueinfo	177
+#define __NR_rt_sigsuspend	178
+#define __NR_pread64		179
+#define __NR_pwrite64		180
+#define __NR_chown		181
+#define __NR_getcwd		182
+#define __NR_capget		183
+#define __NR_capset		184
+#define __NR_sigaltstack	185
+#define __NR_sendfile		186
+#define __NR_getpmsg		187	/* some people actually want streams */
+#define __NR_putpmsg		188	/* some people actually want streams */
+#define __NR_vfork		189
+#define __NR_ugetrlimit		190	/* SuS compliant getrlimit */
+#define __NR_readahead		191
+#ifndef __powerpc64__			/* these are 32-bit only */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#endif
+#define __NR_pciconfig_read	198
+#define __NR_pciconfig_write	199
+#define __NR_pciconfig_iobase	200
+#define __NR_multiplexer	201
+#define __NR_getdents64		202
+#define __NR_pivot_root		203
+#ifndef __powerpc64__
+#define __NR_fcntl64		204
+#endif
+#define __NR_madvise		205
+#define __NR_mincore		206
+#define __NR_gettid		207
+#define __NR_tkill		208
+#define __NR_setxattr		209
+#define __NR_lsetxattr		210
+#define __NR_fsetxattr		211
+#define __NR_getxattr		212
+#define __NR_lgetxattr		213
+#define __NR_fgetxattr		214
+#define __NR_listxattr		215
+#define __NR_llistxattr		216
+#define __NR_flistxattr		217
+#define __NR_removexattr	218
+#define __NR_lremovexattr	219
+#define __NR_fremovexattr	220
+#define __NR_futex		221
+#define __NR_sched_setaffinity	222
+#define __NR_sched_getaffinity	223
+/* 224 currently unused */
+#define __NR_tuxcall		225
+#ifndef __powerpc64__
+#define __NR_sendfile64		226
+#endif
+#define __NR_io_setup		227
+#define __NR_io_destroy		228
+#define __NR_io_getevents	229
+#define __NR_io_submit		230
+#define __NR_io_cancel		231
+#define __NR_set_tid_address	232
+#define __NR_fadvise64		233
+#define __NR_exit_group		234
+#define __NR_lookup_dcookie	235
+#define __NR_epoll_create	236
+#define __NR_epoll_ctl		237
+#define __NR_epoll_wait		238
+#define __NR_remap_file_pages	239
+#define __NR_timer_create	240
+#define __NR_timer_settime	241
+#define __NR_timer_gettime	242
+#define __NR_timer_getoverrun	243
+#define __NR_timer_delete	244
+#define __NR_clock_settime	245
+#define __NR_clock_gettime	246
+#define __NR_clock_getres	247
+#define __NR_clock_nanosleep	248
+#define __NR_swapcontext	249
+#define __NR_tgkill		250
+#define __NR_utimes		251
+#define __NR_statfs64		252
+#define __NR_fstatfs64		253
+#ifndef __powerpc64__
+#define __NR_fadvise64_64	254
+#endif
+#define __NR_rtas		255
+#define __NR_sys_debug_setcontext 256
+/* Number 257 is reserved for vserver */
+#define __NR_migrate_pages	258
+#define __NR_mbind		259
+#define __NR_get_mempolicy	260
+#define __NR_set_mempolicy	261
+#define __NR_mq_open		262
+#define __NR_mq_unlink		263
+#define __NR_mq_timedsend	264
+#define __NR_mq_timedreceive	265
+#define __NR_mq_notify		266
+#define __NR_mq_getsetattr	267
+#define __NR_kexec_load		268
+#define __NR_add_key		269
+#define __NR_request_key	270
+#define __NR_keyctl		271
+#define __NR_waitid		272
+#define __NR_ioprio_set		273
+#define __NR_ioprio_get		274
+#define __NR_inotify_init	275
+#define __NR_inotify_add_watch	276
+#define __NR_inotify_rm_watch	277
+#define __NR_spu_run		278
+#define __NR_spu_create		279
+#define __NR_pselect6		280
+#define __NR_ppoll		281
+#define __NR_unshare		282
+#define __NR_splice		283
+#define __NR_tee		284
+#define __NR_vmsplice		285
+#define __NR_openat		286
+#define __NR_mkdirat		287
+#define __NR_mknodat		288
+#define __NR_fchownat		289
+#define __NR_futimesat		290
+#ifdef __powerpc64__
+#define __NR_newfstatat		291
+#else
+#define __NR_fstatat64		291
+#endif
+#define __NR_unlinkat		292
+#define __NR_renameat		293
+#define __NR_linkat		294
+#define __NR_symlinkat		295
+#define __NR_readlinkat		296
+#define __NR_fchmodat		297
+#define __NR_faccessat		298
+#define __NR_get_robust_list	299
+#define __NR_set_robust_list	300
+#define __NR_move_pages		301
+#define __NR_getcpu		302
+#define __NR_epoll_pwait	303
+#define __NR_utimensat		304
+#define __NR_signalfd		305
+#define __NR_timerfd_create	306
+#define __NR_eventfd		307
+#define __NR_sync_file_range2	308
+#define __NR_fallocate		309
+#define __NR_subpage_prot	310
+#define __NR_timerfd_settime	311
+#define __NR_timerfd_gettime	312
+#define __NR_signalfd4		313
+#define __NR_eventfd2		314
+#define __NR_epoll_create1	315
+#define __NR_dup3		316
+#define __NR_pipe2		317
+#define __NR_inotify_init1	318
+#define __NR_perf_event_open	319
+#define __NR_preadv		320
+#define __NR_pwritev		321
+#define __NR_rt_tgsigqueueinfo	322
+#define __NR_fanotify_init	323
+#define __NR_fanotify_mark	324
+#define __NR_prlimit64		325
+#define __NR_socket		326
+#define __NR_bind		327
+#define __NR_connect		328
+#define __NR_listen		329
+#define __NR_accept		330
+#define __NR_getsockname	331
+#define __NR_getpeername	332
+#define __NR_socketpair		333
+#define __NR_send		334
+#define __NR_sendto		335
+#define __NR_recv		336
+#define __NR_recvfrom		337
+#define __NR_shutdown		338
+#define __NR_setsockopt		339
+#define __NR_getsockopt		340
+#define __NR_sendmsg		341
+#define __NR_recvmsg		342
+#define __NR_recvmmsg		343
+#define __NR_accept4		344
+#define __NR_name_to_handle_at	345
+#define __NR_open_by_handle_at	346
+#define __NR_clock_adjtime	347
+#define __NR_syncfs		348
+#define __NR_sendmmsg		349
+#define __NR_setns		350
+#define __NR_process_vm_readv	351
+#define __NR_process_vm_writev	352
+#define __NR_finit_module	353
+#define __NR_kcmp		354
+#define __NR_sched_setattr	355
+#define __NR_sched_getattr	356
+#define __NR_renameat2		357
+#define __NR_seccomp		358
+#define __NR_getrandom		359
+#define __NR_memfd_create	360
+#define __NR_bpf		361
+#define __NR_execveat		362
+#define __NR_switch_endian	363
+#define __NR_userfaultfd	364
+#define __NR_membarrier		365
+#define __NR_mlock2		378
+#define __NR_copy_file_range	379
+
+#endif /* _ASM_POWERPC_UNISTD_H_ */
diff --git a/qemu/linux-headers/asm-s390/kvm.h b/qemu/linux-headers/asm-s390/kvm.h
index 512d8f1d4..a59499be0 100644
--- a/qemu/linux-headers/asm-s390/kvm.h
+++ b/qemu/linux-headers/asm-s390/kvm.h
@@ -66,6 +66,8 @@ struct kvm_s390_io_adapter_req {
 #define KVM_S390_VM_MEM_CLR_CMMA	1
 #define KVM_S390_VM_MEM_LIMIT_SIZE	2
 
+#define KVM_S390_NO_MEM_LIMIT		U64_MAX
+
 /* kvm attributes for KVM_S390_VM_TOD */
 #define KVM_S390_VM_TOD_LOW		0
 #define KVM_S390_VM_TOD_HIGH		1
@@ -151,6 +153,8 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_ARCH0  (1UL << 4)
 #define KVM_SYNC_PFAULT (1UL << 5)
 #define KVM_SYNC_VRS    (1UL << 6)
+#define KVM_SYNC_RICCB  (1UL << 7)
+#define KVM_SYNC_FPRS   (1UL << 8)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
@@ -165,9 +169,14 @@ struct kvm_sync_regs {
 	__u64 pft;	/* pfault token [PFAULT] */
 	__u64 pfs;	/* pfault select [PFAULT] */
 	__u64 pfc;	/* pfault compare [PFAULT] */
-	__u64 vrs[32][2];	/* vector registers */
+	union {
+		__u64 vrs[32][2];	/* vector registers (KVM_SYNC_VRS) */
+		__u64 fprs[16];		/* fp registers (KVM_SYNC_FPRS) */
+	};
 	__u8  reserved[512];	/* for future vector expansion */
-	__u32 fpc;	/* only valid with vector registers */
+	__u32 fpc;		/* valid on KVM_SYNC_VRS or KVM_SYNC_FPRS */
+	__u8 padding[52];	/* riccb needs to be 64byte aligned */
+	__u8 riccb[64];		/* runtime instrumentation controls block */
 };
 
 #define KVM_REG_S390_TODPR	(KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
diff --git a/qemu/linux-headers/asm-s390/unistd.h b/qemu/linux-headers/asm-s390/unistd.h
new file mode 100644
index 000000000..885837ed5
--- /dev/null
+++ b/qemu/linux-headers/asm-s390/unistd.h
@@ -0,0 +1,406 @@
+/*
+ *  S390 version
+ *
+ *  Derived from "include/asm-i386/unistd.h"
+ */
+
+#ifndef _ASM_S390_UNISTD_H_
+#define _ASM_S390_UNISTD_H_
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_exit                 1
+#define __NR_fork                 2
+#define __NR_read                 3
+#define __NR_write                4
+#define __NR_open                 5
+#define __NR_close                6
+#define __NR_restart_syscall	  7
+#define __NR_creat                8
+#define __NR_link                 9
+#define __NR_unlink              10
+#define __NR_execve              11
+#define __NR_chdir               12
+#define __NR_mknod               14
+#define __NR_chmod               15
+#define __NR_lseek               19
+#define __NR_getpid              20
+#define __NR_mount               21
+#define __NR_umount              22
+#define __NR_ptrace              26
+#define __NR_alarm               27
+#define __NR_pause               29
+#define __NR_utime               30
+#define __NR_access              33
+#define __NR_nice                34
+#define __NR_sync                36
+#define __NR_kill                37
+#define __NR_rename              38
+#define __NR_mkdir               39
+#define __NR_rmdir               40
+#define __NR_dup                 41
+#define __NR_pipe                42
+#define __NR_times               43
+#define __NR_brk                 45
+#define __NR_signal              48
+#define __NR_acct                51
+#define __NR_umount2             52
+#define __NR_ioctl               54
+#define __NR_fcntl               55
+#define __NR_setpgid             57
+#define __NR_umask               60
+#define __NR_chroot              61
+#define __NR_ustat               62
+#define __NR_dup2                63
+#define __NR_getppid             64
+#define __NR_getpgrp             65
+#define __NR_setsid              66
+#define __NR_sigaction           67
+#define __NR_sigsuspend          72
+#define __NR_sigpending          73
+#define __NR_sethostname         74
+#define __NR_setrlimit           75
+#define __NR_getrusage           77
+#define __NR_gettimeofday        78
+#define __NR_settimeofday        79
+#define __NR_symlink             83
+#define __NR_readlink            85
+#define __NR_uselib              86
+#define __NR_swapon              87
+#define __NR_reboot              88
+#define __NR_readdir             89
+#define __NR_mmap                90
+#define __NR_munmap              91
+#define __NR_truncate            92
+#define __NR_ftruncate           93
+#define __NR_fchmod              94
+#define __NR_getpriority         96
+#define __NR_setpriority         97
+#define __NR_statfs              99
+#define __NR_fstatfs            100
+#define __NR_socketcall         102
+#define __NR_syslog             103
+#define __NR_setitimer          104
+#define __NR_getitimer          105
+#define __NR_stat               106
+#define __NR_lstat              107
+#define __NR_fstat              108
+#define __NR_lookup_dcookie     110
+#define __NR_vhangup            111
+#define __NR_idle               112
+#define __NR_wait4              114
+#define __NR_swapoff            115
+#define __NR_sysinfo            116
+#define __NR_ipc                117
+#define __NR_fsync              118
+#define __NR_sigreturn          119
+#define __NR_clone              120
+#define __NR_setdomainname      121
+#define __NR_uname              122
+#define __NR_adjtimex           124
+#define __NR_mprotect           125
+#define __NR_sigprocmask        126
+#define __NR_create_module      127
+#define __NR_init_module        128
+#define __NR_delete_module      129
+#define __NR_get_kernel_syms    130
+#define __NR_quotactl           131
+#define __NR_getpgid            132
+#define __NR_fchdir             133
+#define __NR_bdflush            134
+#define __NR_sysfs              135
+#define __NR_personality        136
+#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
+#define __NR_getdents           141
+#define __NR_flock              143
+#define __NR_msync              144
+#define __NR_readv              145
+#define __NR_writev             146
+#define __NR_getsid             147
+#define __NR_fdatasync          148
+#define __NR__sysctl            149
+#define __NR_mlock              150
+#define __NR_munlock            151
+#define __NR_mlockall           152
+#define __NR_munlockall         153
+#define __NR_sched_setparam             154
+#define __NR_sched_getparam             155
+#define __NR_sched_setscheduler         156
+#define __NR_sched_getscheduler         157
+#define __NR_sched_yield                158
+#define __NR_sched_get_priority_max     159
+#define __NR_sched_get_priority_min     160
+#define __NR_sched_rr_get_interval      161
+#define __NR_nanosleep          162
+#define __NR_mremap             163
+#define __NR_query_module       167
+#define __NR_poll               168
+#define __NR_nfsservctl         169
+#define __NR_prctl              172
+#define __NR_rt_sigreturn       173
+#define __NR_rt_sigaction       174
+#define __NR_rt_sigprocmask     175
+#define __NR_rt_sigpending      176
+#define __NR_rt_sigtimedwait    177
+#define __NR_rt_sigqueueinfo    178
+#define __NR_rt_sigsuspend      179
+#define __NR_pread64            180
+#define __NR_pwrite64           181
+#define __NR_getcwd             183
+#define __NR_capget             184
+#define __NR_capset             185
+#define __NR_sigaltstack        186
+#define __NR_sendfile           187
+#define __NR_getpmsg		188
+#define __NR_putpmsg		189
+#define __NR_vfork		190
+#define __NR_pivot_root         217
+#define __NR_mincore            218
+#define __NR_madvise            219
+#define __NR_getdents64		220
+#define __NR_readahead		222
+#define __NR_setxattr		224
+#define __NR_lsetxattr		225
+#define __NR_fsetxattr		226
+#define __NR_getxattr		227
+#define __NR_lgetxattr		228
+#define __NR_fgetxattr		229
+#define __NR_listxattr		230
+#define __NR_llistxattr		231
+#define __NR_flistxattr		232
+#define __NR_removexattr	233
+#define __NR_lremovexattr	234
+#define __NR_fremovexattr	235
+#define __NR_gettid		236
+#define __NR_tkill		237
+#define __NR_futex		238
+#define __NR_sched_setaffinity	239
+#define __NR_sched_getaffinity	240
+#define __NR_tgkill		241
+/* Number 242 is reserved for tux */
+#define __NR_io_setup		243
+#define __NR_io_destroy		244
+#define __NR_io_getevents	245
+#define __NR_io_submit		246
+#define __NR_io_cancel		247
+#define __NR_exit_group		248
+#define __NR_epoll_create	249
+#define __NR_epoll_ctl		250
+#define __NR_epoll_wait		251
+#define __NR_set_tid_address	252
+#define __NR_fadvise64		253
+#define __NR_timer_create	254
+#define __NR_timer_settime	255
+#define __NR_timer_gettime	256
+#define __NR_timer_getoverrun	257
+#define __NR_timer_delete	258
+#define __NR_clock_settime	259
+#define __NR_clock_gettime	260
+#define __NR_clock_getres	261
+#define __NR_clock_nanosleep	262
+/* Number 263 is reserved for vserver */
+#define __NR_statfs64		265
+#define __NR_fstatfs64		266
+#define __NR_remap_file_pages	267
+#define __NR_mbind		268
+#define __NR_get_mempolicy	269
+#define __NR_set_mempolicy	270
+#define __NR_mq_open		271
+#define __NR_mq_unlink		272
+#define __NR_mq_timedsend	273
+#define __NR_mq_timedreceive	274
+#define __NR_mq_notify		275
+#define __NR_mq_getsetattr	276
+#define __NR_kexec_load		277
+#define __NR_add_key		278
+#define __NR_request_key	279
+#define __NR_keyctl		280
+#define __NR_waitid		281
+#define __NR_ioprio_set		282
+#define __NR_ioprio_get		283
+#define __NR_inotify_init	284
+#define __NR_inotify_add_watch	285
+#define __NR_inotify_rm_watch	286
+#define __NR_migrate_pages	287
+#define __NR_openat		288
+#define __NR_mkdirat		289
+#define __NR_mknodat		290
+#define __NR_fchownat		291
+#define __NR_futimesat		292
+#define __NR_unlinkat		294
+#define __NR_renameat		295
+#define __NR_linkat		296
+#define __NR_symlinkat		297
+#define __NR_readlinkat		298
+#define __NR_fchmodat		299
+#define __NR_faccessat		300
+#define __NR_pselect6		301
+#define __NR_ppoll		302
+#define __NR_unshare		303
+#define __NR_set_robust_list	304
+#define __NR_get_robust_list	305
+#define __NR_splice		306
+#define __NR_sync_file_range	307
+#define __NR_tee		308
+#define __NR_vmsplice		309
+#define __NR_move_pages		310
+#define __NR_getcpu		311
+#define __NR_epoll_pwait	312
+#define __NR_utimes		313
+#define __NR_fallocate		314
+#define __NR_utimensat		315
+#define __NR_signalfd		316
+#define __NR_timerfd		317
+#define __NR_eventfd		318
+#define __NR_timerfd_create	319
+#define __NR_timerfd_settime	320
+#define __NR_timerfd_gettime	321
+#define __NR_signalfd4		322
+#define __NR_eventfd2		323
+#define __NR_inotify_init1	324
+#define __NR_pipe2		325
+#define __NR_dup3		326
+#define __NR_epoll_create1	327
+#define	__NR_preadv		328
+#define	__NR_pwritev		329
+#define __NR_rt_tgsigqueueinfo	330
+#define __NR_perf_event_open	331
+#define __NR_fanotify_init	332
+#define __NR_fanotify_mark	333
+#define __NR_prlimit64		334
+#define __NR_name_to_handle_at	335
+#define __NR_open_by_handle_at	336
+#define __NR_clock_adjtime	337
+#define __NR_syncfs		338
+#define __NR_setns		339
+#define __NR_process_vm_readv	340
+#define __NR_process_vm_writev	341
+#define __NR_s390_runtime_instr 342
+#define __NR_kcmp		343
+#define __NR_finit_module	344
+#define __NR_sched_setattr	345
+#define __NR_sched_getattr	346
+#define __NR_renameat2		347
+#define __NR_seccomp		348
+#define __NR_getrandom		349
+#define __NR_memfd_create	350
+#define __NR_bpf		351
+#define __NR_s390_pci_mmio_write	352
+#define __NR_s390_pci_mmio_read		353
+#define __NR_execveat		354
+#define __NR_userfaultfd	355
+#define __NR_membarrier		356
+#define __NR_recvmmsg		357
+#define __NR_sendmmsg		358
+#define __NR_socket		359
+#define __NR_socketpair		360
+#define __NR_bind		361
+#define __NR_connect		362
+#define __NR_listen		363
+#define __NR_accept4		364
+#define __NR_getsockopt		365
+#define __NR_setsockopt		366
+#define __NR_getsockname	367
+#define __NR_getpeername	368
+#define __NR_sendto		369
+#define __NR_sendmsg		370
+#define __NR_recvfrom		371
+#define __NR_recvmsg		372
+#define __NR_shutdown		373
+#define __NR_mlock2		374
+#define __NR_copy_file_range	375
+#define NR_syscalls 376
+
+/* 
+ * There are some system calls that are not present on 64 bit, some
+ * have a different name although they do the same (e.g. __NR_chown32
+ * is __NR_chown on 64 bit).
+ */
+#ifndef __s390x__
+
+#define __NR_time		 13
+#define __NR_lchown		 16
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_getrlimit		 76
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_fchown		 95
+#define __NR_ioperm		101
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR__newselect 	142
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_setresgid		170
+#define __NR_getresgid		171
+#define __NR_chown		182
+#define __NR_ugetrlimit		191	/* SuS compliant getrlimit */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#define __NR_lchown32		198
+#define __NR_getuid32		199
+#define __NR_getgid32		200
+#define __NR_geteuid32		201
+#define __NR_getegid32		202
+#define __NR_setreuid32		203
+#define __NR_setregid32		204
+#define __NR_getgroups32	205
+#define __NR_setgroups32	206
+#define __NR_fchown32		207
+#define __NR_setresuid32	208
+#define __NR_getresuid32	209
+#define __NR_setresgid32	210
+#define __NR_getresgid32	211
+#define __NR_chown32		212
+#define __NR_setuid32		213
+#define __NR_setgid32		214
+#define __NR_setfsuid32		215
+#define __NR_setfsgid32		216
+#define __NR_fcntl64		221
+#define __NR_sendfile64		223
+#define __NR_fadvise64_64	264
+#define __NR_fstatat64		293
+
+#else
+
+#define __NR_select		142
+#define __NR_getrlimit		191	/* SuS compliant getrlimit */
+#define __NR_lchown  		198
+#define __NR_getuid  		199
+#define __NR_getgid  		200
+#define __NR_geteuid  		201
+#define __NR_getegid  		202
+#define __NR_setreuid  		203
+#define __NR_setregid  		204
+#define __NR_getgroups  	205
+#define __NR_setgroups  	206
+#define __NR_fchown  		207
+#define __NR_setresuid  	208
+#define __NR_getresuid  	209
+#define __NR_setresgid  	210
+#define __NR_getresgid  	211
+#define __NR_chown  		212
+#define __NR_setuid  		213
+#define __NR_setgid  		214
+#define __NR_setfsuid  		215
+#define __NR_setfsgid  		216
+#define __NR_newfstatat		293
+
+#endif
+
+#endif /* _ASM_S390_UNISTD_H_ */
diff --git a/qemu/linux-headers/asm-x86/hyperv.h b/qemu/linux-headers/asm-x86/hyperv.h
index 8fba544e9..01af4d859 100644
--- a/qemu/linux-headers/asm-x86/hyperv.h
+++ b/qemu/linux-headers/asm-x86/hyperv.h
@@ -1,250 +1 @@
-#ifndef _ASM_X86_HYPERV_H
-#define _ASM_X86_HYPERV_H
-
-#include <linux/types.h>
-
-/*
- * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
- * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
- */
-#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
-#define HYPERV_CPUID_INTERFACE			0x40000001
-#define HYPERV_CPUID_VERSION			0x40000002
-#define HYPERV_CPUID_FEATURES			0x40000003
-#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
-#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
-
-#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
-#define HYPERV_CPUID_MIN			0x40000005
-#define HYPERV_CPUID_MAX			0x4000ffff
-
-/*
- * Feature identification. EAX indicates which features are available
- * to the partition based upon the current partition privileges.
- */
-
-/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
-#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
-/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
-#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
-
-/* A partition's reference time stamp counter (TSC) page */
-#define HV_X64_MSR_REFERENCE_TSC		0x40000021
-
-/*
- * There is a single feature flag that signifies the presence of the MSR
- * that can be used to retrieve both the local APIC Timer frequency as
- * well as the TSC frequency.
- */
-
-/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
-#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
-
-/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
-#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
-
-/*
- * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
- * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
- */
-#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
-/*
- * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
- * HV_X64_MSR_STIMER3_COUNT) available
- */
-#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
-/*
- * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
- * are available
- */
-#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
-/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
-#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
-/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
-#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
-/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
-#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
- /*
-  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
-  */
-#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
-
-/*
- * Feature identification: EBX indicates which flags were specified at
- * partition creation. The format is the same as the partition creation
- * flag structure defined in section Partition Creation Flags.
- */
-#define HV_X64_CREATE_PARTITIONS		(1 << 0)
-#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
-#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
-#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
-#define HV_X64_POST_MESSAGES			(1 << 4)
-#define HV_X64_SIGNAL_EVENTS			(1 << 5)
-#define HV_X64_CREATE_PORT			(1 << 6)
-#define HV_X64_CONNECT_PORT			(1 << 7)
-#define HV_X64_ACCESS_STATS			(1 << 8)
-#define HV_X64_DEBUGGING			(1 << 11)
-#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
-#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
-
-/*
- * Feature identification. EDX indicates which miscellaneous features
- * are available to the partition.
- */
-/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
-#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
-/* Guest debugging support is available */
-#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
-/* Performance Monitor support is available*/
-#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
-/* Support for physical CPU dynamic partitioning events is available*/
-#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
-/*
- * Support for passing hypercall input parameter block via XMM
- * registers is available
- */
-#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
-/* Support for a virtual guest idle state is available */
-#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
-
-/*
- * Implementation recommendations. Indicates which behaviors the hypervisor
- * recommends the OS implement for optimal performance.
- */
- /*
-  * Recommend using hypercall for address space switches rather
-  * than MOV to CR3 instruction
-  */
-#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
-/* Recommend using hypercall for local TLB flushes rather
- * than INVLPG or MOV to CR3 instructions */
-#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
-/*
- * Recommend using hypercall for remote TLB flushes rather
- * than inter-processor interrupts
- */
-#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
-/*
- * Recommend using MSRs for accessing APIC registers
- * EOI, ICR and TPR rather than their memory-mapped counterparts
- */
-#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
-/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
-#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
-/*
- * Recommend using relaxed timing for this partition. If used,
- * the VM should disable any watchdog timeouts that rely on the
- * timely delivery of external interrupts
- */
-#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
-
-/* MSR used to identify the guest OS. */
-#define HV_X64_MSR_GUEST_OS_ID			0x40000000
-
-/* MSR used to setup pages used to communicate with the hypervisor. */
-#define HV_X64_MSR_HYPERCALL			0x40000001
-
-/* MSR used to provide vcpu index */
-#define HV_X64_MSR_VP_INDEX			0x40000002
-
-/* MSR used to read the per-partition time reference counter */
-#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
-
-/* MSR used to retrieve the TSC frequency */
-#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
-
-/* MSR used to retrieve the local APIC timer frequency */
-#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
-
-/* Define the virtual APIC registers */
-#define HV_X64_MSR_EOI				0x40000070
-#define HV_X64_MSR_ICR				0x40000071
-#define HV_X64_MSR_TPR				0x40000072
-#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
-
-/* Define synthetic interrupt controller model specific registers. */
-#define HV_X64_MSR_SCONTROL			0x40000080
-#define HV_X64_MSR_SVERSION			0x40000081
-#define HV_X64_MSR_SIEFP			0x40000082
-#define HV_X64_MSR_SIMP				0x40000083
-#define HV_X64_MSR_EOM				0x40000084
-#define HV_X64_MSR_SINT0			0x40000090
-#define HV_X64_MSR_SINT1			0x40000091
-#define HV_X64_MSR_SINT2			0x40000092
-#define HV_X64_MSR_SINT3			0x40000093
-#define HV_X64_MSR_SINT4			0x40000094
-#define HV_X64_MSR_SINT5			0x40000095
-#define HV_X64_MSR_SINT6			0x40000096
-#define HV_X64_MSR_SINT7			0x40000097
-#define HV_X64_MSR_SINT8			0x40000098
-#define HV_X64_MSR_SINT9			0x40000099
-#define HV_X64_MSR_SINT10			0x4000009A
-#define HV_X64_MSR_SINT11			0x4000009B
-#define HV_X64_MSR_SINT12			0x4000009C
-#define HV_X64_MSR_SINT13			0x4000009D
-#define HV_X64_MSR_SINT14			0x4000009E
-#define HV_X64_MSR_SINT15			0x4000009F
-
-/*
- * Synthetic Timer MSRs. Four timers per vcpu.
- */
-#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
-#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
-#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
-#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
-#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
-#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
-#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
-#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
-
-/* Hyper-V guest crash notification MSR's */
-#define HV_X64_MSR_CRASH_P0			0x40000100
-#define HV_X64_MSR_CRASH_P1			0x40000101
-#define HV_X64_MSR_CRASH_P2			0x40000102
-#define HV_X64_MSR_CRASH_P3			0x40000103
-#define HV_X64_MSR_CRASH_P4			0x40000104
-#define HV_X64_MSR_CRASH_CTL			0x40000105
-#define HV_X64_MSR_CRASH_CTL_NOTIFY		(1ULL << 63)
-#define HV_X64_MSR_CRASH_PARAMS		\
-		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
-
-#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
-
-/* Declare the various hypercall operations. */
-#define HV_X64_HV_NOTIFY_LONG_SPIN_WAIT		0x0008
-
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
-
-#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
-#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
-
-#define HV_PROCESSOR_POWER_STATE_C0		0
-#define HV_PROCESSOR_POWER_STATE_C1		1
-#define HV_PROCESSOR_POWER_STATE_C2		2
-#define HV_PROCESSOR_POWER_STATE_C3		3
-
-/* hypercall status code */
-#define HV_STATUS_SUCCESS			0
-#define HV_STATUS_INVALID_HYPERCALL_CODE	2
-#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
-#define HV_STATUS_INVALID_ALIGNMENT		4
-#define HV_STATUS_INSUFFICIENT_MEMORY		11
-#define HV_STATUS_INVALID_CONNECTION_ID		18
-#define HV_STATUS_INSUFFICIENT_BUFFERS		19
-
-typedef struct _HV_REFERENCE_TSC_PAGE {
-	__u32 tsc_sequence;
-	__u32 res1;
-	__u64 tsc_scale;
-	__s64 tsc_offset;
-} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
-
-#endif
+#include "standard-headers/asm-x86/hyperv.h"
diff --git a/qemu/linux-headers/asm-x86/kvm.h b/qemu/linux-headers/asm-x86/kvm.h
index a4ae82eb8..cd54147cb 100644
--- a/qemu/linux-headers/asm-x86/kvm.h
+++ b/qemu/linux-headers/asm-x86/kvm.h
@@ -354,7 +354,7 @@ struct kvm_xcrs {
 struct kvm_sync_regs {
 };
 
-#define KVM_QUIRK_LINT0_REENABLED	(1 << 0)
-#define KVM_QUIRK_CD_NW_CLEARED		(1 << 1)
+#define KVM_X86_QUIRK_LINT0_REENABLED	(1 << 0)
+#define KVM_X86_QUIRK_CD_NW_CLEARED	(1 << 1)
 
 #endif /* _ASM_X86_KVM_H */
diff --git a/qemu/linux-headers/asm-x86/unistd.h b/qemu/linux-headers/asm-x86/unistd.h
new file mode 100644
index 000000000..1f99b1284
--- /dev/null
+++ b/qemu/linux-headers/asm-x86/unistd.h
@@ -0,0 +1,15 @@
+#ifndef _ASM_X86_UNISTD_H
+#define _ASM_X86_UNISTD_H
+
+/* x32 syscall flag bit */
+#define __X32_SYSCALL_BIT	0x40000000
+
+# ifdef __i386__
+#  include <asm/unistd_32.h>
+# elif defined(__ILP32__)
+#  include <asm/unistd_x32.h>
+# else
+#  include <asm/unistd_64.h>
+# endif
+
+#endif /* _ASM_X86_UNISTD_H */
diff --git a/qemu/linux-headers/asm-x86/unistd_32.h b/qemu/linux-headers/asm-x86/unistd_32.h
new file mode 100644
index 000000000..abeaf40d3
--- /dev/null
+++ b/qemu/linux-headers/asm-x86/unistd_32.h
@@ -0,0 +1,381 @@
+#ifndef _ASM_X86_UNISTD_32_H
+#define _ASM_X86_UNISTD_32_H 1
+
+#define __NR_restart_syscall 0
+#define __NR_exit 1
+#define __NR_fork 2
+#define __NR_read 3
+#define __NR_write 4
+#define __NR_open 5
+#define __NR_close 6
+#define __NR_waitpid 7
+#define __NR_creat 8
+#define __NR_link 9
+#define __NR_unlink 10
+#define __NR_execve 11
+#define __NR_chdir 12
+#define __NR_time 13
+#define __NR_mknod 14
+#define __NR_chmod 15
+#define __NR_lchown 16
+#define __NR_break 17
+#define __NR_oldstat 18
+#define __NR_lseek 19
+#define __NR_getpid 20
+#define __NR_mount 21
+#define __NR_umount 22
+#define __NR_setuid 23
+#define __NR_getuid 24
+#define __NR_stime 25
+#define __NR_ptrace 26
+#define __NR_alarm 27
+#define __NR_oldfstat 28
+#define __NR_pause 29
+#define __NR_utime 30
+#define __NR_stty 31
+#define __NR_gtty 32
+#define __NR_access 33
+#define __NR_nice 34
+#define __NR_ftime 35
+#define __NR_sync 36
+#define __NR_kill 37
+#define __NR_rename 38
+#define __NR_mkdir 39
+#define __NR_rmdir 40
+#define __NR_dup 41
+#define __NR_pipe 42
+#define __NR_times 43
+#define __NR_prof 44
+#define __NR_brk 45
+#define __NR_setgid 46
+#define __NR_getgid 47
+#define __NR_signal 48
+#define __NR_geteuid 49
+#define __NR_getegid 50
+#define __NR_acct 51
+#define __NR_umount2 52
+#define __NR_lock 53
+#define __NR_ioctl 54
+#define __NR_fcntl 55
+#define __NR_mpx 56
+#define __NR_setpgid 57
+#define __NR_ulimit 58
+#define __NR_oldolduname 59
+#define __NR_umask 60
+#define __NR_chroot 61
+#define __NR_ustat 62
+#define __NR_dup2 63
+#define __NR_getppid 64
+#define __NR_getpgrp 65
+#define __NR_setsid 66
+#define __NR_sigaction 67
+#define __NR_sgetmask 68
+#define __NR_ssetmask 69
+#define __NR_setreuid 70
+#define __NR_setregid 71
+#define __NR_sigsuspend 72
+#define __NR_sigpending 73
+#define __NR_sethostname 74
+#define __NR_setrlimit 75
+#define __NR_getrlimit 76
+#define __NR_getrusage 77
+#define __NR_gettimeofday 78
+#define __NR_settimeofday 79
+#define __NR_getgroups 80
+#define __NR_setgroups 81
+#define __NR_select 82
+#define __NR_symlink 83
+#define __NR_oldlstat 84
+#define __NR_readlink 85
+#define __NR_uselib 86
+#define __NR_swapon 87
+#define __NR_reboot 88
+#define __NR_readdir 89
+#define __NR_mmap 90
+#define __NR_munmap 91
+#define __NR_truncate 92
+#define __NR_ftruncate 93
+#define __NR_fchmod 94
+#define __NR_fchown 95
+#define __NR_getpriority 96
+#define __NR_setpriority 97
+#define __NR_profil 98
+#define __NR_statfs 99
+#define __NR_fstatfs 100
+#define __NR_ioperm 101
+#define __NR_socketcall 102
+#define __NR_syslog 103
+#define __NR_setitimer 104
+#define __NR_getitimer 105
+#define __NR_stat 106
+#define __NR_lstat 107
+#define __NR_fstat 108
+#define __NR_olduname 109
+#define __NR_iopl 110
+#define __NR_vhangup 111
+#define __NR_idle 112
+#define __NR_vm86old 113
+#define __NR_wait4 114
+#define __NR_swapoff 115
+#define __NR_sysinfo 116
+#define __NR_ipc 117
+#define __NR_fsync 118
+#define __NR_sigreturn 119
+#define __NR_clone 120
+#define __NR_setdomainname 121
+#define __NR_uname 122
+#define __NR_modify_ldt 123
+#define __NR_adjtimex 124
+#define __NR_mprotect 125
+#define __NR_sigprocmask 126
+#define __NR_create_module 127
+#define __NR_init_module 128
+#define __NR_delete_module 129
+#define __NR_get_kernel_syms 130
+#define __NR_quotactl 131
+#define __NR_getpgid 132
+#define __NR_fchdir 133
+#define __NR_bdflush 134
+#define __NR_sysfs 135
+#define __NR_personality 136
+#define __NR_afs_syscall 137
+#define __NR_setfsuid 138
+#define __NR_setfsgid 139
+#define __NR__llseek 140
+#define __NR_getdents 141
+#define __NR__newselect 142
+#define __NR_flock 143
+#define __NR_msync 144
+#define __NR_readv 145
+#define __NR_writev 146
+#define __NR_getsid 147
+#define __NR_fdatasync 148
+#define __NR__sysctl 149
+#define __NR_mlock 150
+#define __NR_munlock 151
+#define __NR_mlockall 152
+#define __NR_munlockall 153
+#define __NR_sched_setparam 154
+#define __NR_sched_getparam 155
+#define __NR_sched_setscheduler 156
+#define __NR_sched_getscheduler 157
+#define __NR_sched_yield 158
+#define __NR_sched_get_priority_max 159
+#define __NR_sched_get_priority_min 160
+#define __NR_sched_rr_get_interval 161
+#define __NR_nanosleep 162
+#define __NR_mremap 163
+#define __NR_setresuid 164
+#define __NR_getresuid 165
+#define __NR_vm86 166
+#define __NR_query_module 167
+#define __NR_poll 168
+#define __NR_nfsservctl 169
+#define __NR_setresgid 170
+#define __NR_getresgid 171
+#define __NR_prctl 172
+#define __NR_rt_sigreturn 173
+#define __NR_rt_sigaction 174
+#define __NR_rt_sigprocmask 175
+#define __NR_rt_sigpending 176
+#define __NR_rt_sigtimedwait 177
+#define __NR_rt_sigqueueinfo 178
+#define __NR_rt_sigsuspend 179
+#define __NR_pread64 180
+#define __NR_pwrite64 181
+#define __NR_chown 182
+#define __NR_getcwd 183
+#define __NR_capget 184
+#define __NR_capset 185
+#define __NR_sigaltstack 186
+#define __NR_sendfile 187
+#define __NR_getpmsg 188
+#define __NR_putpmsg 189
+#define __NR_vfork 190
+#define __NR_ugetrlimit 191
+#define __NR_mmap2 192
+#define __NR_truncate64 193
+#define __NR_ftruncate64 194
+#define __NR_stat64 195
+#define __NR_lstat64 196
+#define __NR_fstat64 197
+#define __NR_lchown32 198
+#define __NR_getuid32 199
+#define __NR_getgid32 200
+#define __NR_geteuid32 201
+#define __NR_getegid32 202
+#define __NR_setreuid32 203
+#define __NR_setregid32 204
+#define __NR_getgroups32 205
+#define __NR_setgroups32 206
+#define __NR_fchown32 207
+#define __NR_setresuid32 208
+#define __NR_getresuid32 209
+#define __NR_setresgid32 210
+#define __NR_getresgid32 211
+#define __NR_chown32 212
+#define __NR_setuid32 213
+#define __NR_setgid32 214
+#define __NR_setfsuid32 215
+#define __NR_setfsgid32 216
+#define __NR_pivot_root 217
+#define __NR_mincore 218
+#define __NR_madvise 219
+#define __NR_getdents64 220
+#define __NR_fcntl64 221
+#define __NR_gettid 224
+#define __NR_readahead 225
+#define __NR_setxattr 226
+#define __NR_lsetxattr 227
+#define __NR_fsetxattr 228
+#define __NR_getxattr 229
+#define __NR_lgetxattr 230
+#define __NR_fgetxattr 231
+#define __NR_listxattr 232
+#define __NR_llistxattr 233
+#define __NR_flistxattr 234
+#define __NR_removexattr 235
+#define __NR_lremovexattr 236
+#define __NR_fremovexattr 237
+#define __NR_tkill 238
+#define __NR_sendfile64 239
+#define __NR_futex 240
+#define __NR_sched_setaffinity 241
+#define __NR_sched_getaffinity 242
+#define __NR_set_thread_area 243
+#define __NR_get_thread_area 244
+#define __NR_io_setup 245
+#define __NR_io_destroy 246
+#define __NR_io_getevents 247
+#define __NR_io_submit 248
+#define __NR_io_cancel 249
+#define __NR_fadvise64 250
+#define __NR_exit_group 252
+#define __NR_lookup_dcookie 253
+#define __NR_epoll_create 254
+#define __NR_epoll_ctl 255
+#define __NR_epoll_wait 256
+#define __NR_remap_file_pages 257
+#define __NR_set_tid_address 258
+#define __NR_timer_create 259
+#define __NR_timer_settime 260
+#define __NR_timer_gettime 261
+#define __NR_timer_getoverrun 262
+#define __NR_timer_delete 263
+#define __NR_clock_settime 264
+#define __NR_clock_gettime 265
+#define __NR_clock_getres 266
+#define __NR_clock_nanosleep 267
+#define __NR_statfs64 268
+#define __NR_fstatfs64 269
+#define __NR_tgkill 270
+#define __NR_utimes 271
+#define __NR_fadvise64_64 272
+#define __NR_vserver 273
+#define __NR_mbind 274
+#define __NR_get_mempolicy 275
+#define __NR_set_mempolicy 276
+#define __NR_mq_open 277
+#define __NR_mq_unlink 278
+#define __NR_mq_timedsend 279
+#define __NR_mq_timedreceive 280
+#define __NR_mq_notify 281
+#define __NR_mq_getsetattr 282
+#define __NR_kexec_load 283
+#define __NR_waitid 284
+#define __NR_add_key 286
+#define __NR_request_key 287
+#define __NR_keyctl 288
+#define __NR_ioprio_set 289
+#define __NR_ioprio_get 290
+#define __NR_inotify_init 291
+#define __NR_inotify_add_watch 292
+#define __NR_inotify_rm_watch 293
+#define __NR_migrate_pages 294
+#define __NR_openat 295
+#define __NR_mkdirat 296
+#define __NR_mknodat 297
+#define __NR_fchownat 298
+#define __NR_futimesat 299
+#define __NR_fstatat64 300
+#define __NR_unlinkat 301
+#define __NR_renameat 302
+#define __NR_linkat 303
+#define __NR_symlinkat 304
+#define __NR_readlinkat 305
+#define __NR_fchmodat 306
+#define __NR_faccessat 307
+#define __NR_pselect6 308
+#define __NR_ppoll 309
+#define __NR_unshare 310
+#define __NR_set_robust_list 311
+#define __NR_get_robust_list 312
+#define __NR_splice 313
+#define __NR_sync_file_range 314
+#define __NR_tee 315
+#define __NR_vmsplice 316
+#define __NR_move_pages 317
+#define __NR_getcpu 318
+#define __NR_epoll_pwait 319
+#define __NR_utimensat 320
+#define __NR_signalfd 321
+#define __NR_timerfd_create 322
+#define __NR_eventfd 323
+#define __NR_fallocate 324
+#define __NR_timerfd_settime 325
+#define __NR_timerfd_gettime 326
+#define __NR_signalfd4 327
+#define __NR_eventfd2 328
+#define __NR_epoll_create1 329
+#define __NR_dup3 330
+#define __NR_pipe2 331
+#define __NR_inotify_init1 332
+#define __NR_preadv 333
+#define __NR_pwritev 334
+#define __NR_rt_tgsigqueueinfo 335
+#define __NR_perf_event_open 336
+#define __NR_recvmmsg 337
+#define __NR_fanotify_init 338
+#define __NR_fanotify_mark 339
+#define __NR_prlimit64 340
+#define __NR_name_to_handle_at 341
+#define __NR_open_by_handle_at 342
+#define __NR_clock_adjtime 343
+#define __NR_syncfs 344
+#define __NR_sendmmsg 345
+#define __NR_setns 346
+#define __NR_process_vm_readv 347
+#define __NR_process_vm_writev 348
+#define __NR_kcmp 349
+#define __NR_finit_module 350
+#define __NR_sched_setattr 351
+#define __NR_sched_getattr 352
+#define __NR_renameat2 353
+#define __NR_seccomp 354
+#define __NR_getrandom 355
+#define __NR_memfd_create 356
+#define __NR_bpf 357
+#define __NR_execveat 358
+#define __NR_socket 359
+#define __NR_socketpair 360
+#define __NR_bind 361
+#define __NR_connect 362
+#define __NR_listen 363
+#define __NR_accept4 364
+#define __NR_getsockopt 365
+#define __NR_setsockopt 366
+#define __NR_getsockname 367
+#define __NR_getpeername 368
+#define __NR_sendto 369
+#define __NR_sendmsg 370
+#define __NR_recvfrom 371
+#define __NR_recvmsg 372
+#define __NR_shutdown 373
+#define __NR_userfaultfd 374
+#define __NR_membarrier 375
+#define __NR_mlock2 376
+#define __NR_copy_file_range 377
+#define __NR_preadv2 378
+#define __NR_pwritev2 379
+
+#endif /* _ASM_X86_UNISTD_32_H */
diff --git a/qemu/linux-headers/asm-x86/unistd_64.h b/qemu/linux-headers/asm-x86/unistd_64.h
new file mode 100644
index 000000000..73c3d1f66
--- /dev/null
+++ b/qemu/linux-headers/asm-x86/unistd_64.h
@@ -0,0 +1,334 @@
+#ifndef _ASM_X86_UNISTD_64_H
+#define _ASM_X86_UNISTD_64_H 1
+
+#define __NR_read 0
+#define __NR_write 1
+#define __NR_open 2
+#define __NR_close 3
+#define __NR_stat 4
+#define __NR_fstat 5
+#define __NR_lstat 6
+#define __NR_poll 7
+#define __NR_lseek 8
+#define __NR_mmap 9
+#define __NR_mprotect 10
+#define __NR_munmap 11
+#define __NR_brk 12
+#define __NR_rt_sigaction 13
+#define __NR_rt_sigprocmask 14
+#define __NR_rt_sigreturn 15
+#define __NR_ioctl 16
+#define __NR_pread64 17
+#define __NR_pwrite64 18
+#define __NR_readv 19
+#define __NR_writev 20
+#define __NR_access 21
+#define __NR_pipe 22
+#define __NR_select 23
+#define __NR_sched_yield 24
+#define __NR_mremap 25
+#define __NR_msync 26
+#define __NR_mincore 27
+#define __NR_madvise 28
+#define __NR_shmget 29
+#define __NR_shmat 30
+#define __NR_shmctl 31
+#define __NR_dup 32
+#define __NR_dup2 33
+#define __NR_pause 34
+#define __NR_nanosleep 35
+#define __NR_getitimer 36
+#define __NR_alarm 37
+#define __NR_setitimer 38
+#define __NR_getpid 39
+#define __NR_sendfile 40
+#define __NR_socket 41
+#define __NR_connect 42
+#define __NR_accept 43
+#define __NR_sendto 44
+#define __NR_recvfrom 45
+#define __NR_sendmsg 46
+#define __NR_recvmsg 47
+#define __NR_shutdown 48
+#define __NR_bind 49
+#define __NR_listen 50
+#define __NR_getsockname 51
+#define __NR_getpeername 52
+#define __NR_socketpair 53
+#define __NR_setsockopt 54
+#define __NR_getsockopt 55
+#define __NR_clone 56
+#define __NR_fork 57
+#define __NR_vfork 58
+#define __NR_execve 59
+#define __NR_exit 60
+#define __NR_wait4 61
+#define __NR_kill 62
+#define __NR_uname 63
+#define __NR_semget 64
+#define __NR_semop 65
+#define __NR_semctl 66
+#define __NR_shmdt 67
+#define __NR_msgget 68
+#define __NR_msgsnd 69
+#define __NR_msgrcv 70
+#define __NR_msgctl 71
+#define __NR_fcntl 72
+#define __NR_flock 73
+#define __NR_fsync 74
+#define __NR_fdatasync 75
+#define __NR_truncate 76
+#define __NR_ftruncate 77
+#define __NR_getdents 78
+#define __NR_getcwd 79
+#define __NR_chdir 80
+#define __NR_fchdir 81
+#define __NR_rename 82
+#define __NR_mkdir 83
+#define __NR_rmdir 84
+#define __NR_creat 85
+#define __NR_link 86
+#define __NR_unlink 87
+#define __NR_symlink 88
+#define __NR_readlink 89
+#define __NR_chmod 90
+#define __NR_fchmod 91
+#define __NR_chown 92
+#define __NR_fchown 93
+#define __NR_lchown 94
+#define __NR_umask 95
+#define __NR_gettimeofday 96
+#define __NR_getrlimit 97
+#define __NR_getrusage 98
+#define __NR_sysinfo 99
+#define __NR_times 100
+#define __NR_ptrace 101
+#define __NR_getuid 102
+#define __NR_syslog 103
+#define __NR_getgid 104
+#define __NR_setuid 105
+#define __NR_setgid 106
+#define __NR_geteuid 107
+#define __NR_getegid 108
+#define __NR_setpgid 109
+#define __NR_getppid 110
+#define __NR_getpgrp 111
+#define __NR_setsid 112
+#define __NR_setreuid 113
+#define __NR_setregid 114
+#define __NR_getgroups 115
+#define __NR_setgroups 116
+#define __NR_setresuid 117
+#define __NR_getresuid 118
+#define __NR_setresgid 119
+#define __NR_getresgid 120
+#define __NR_getpgid 121
+#define __NR_setfsuid 122
+#define __NR_setfsgid 123
+#define __NR_getsid 124
+#define __NR_capget 125
+#define __NR_capset 126
+#define __NR_rt_sigpending 127
+#define __NR_rt_sigtimedwait 128
+#define __NR_rt_sigqueueinfo 129
+#define __NR_rt_sigsuspend 130
+#define __NR_sigaltstack 131
+#define __NR_utime 132
+#define __NR_mknod 133
+#define __NR_uselib 134
+#define __NR_personality 135
+#define __NR_ustat 136
+#define __NR_statfs 137
+#define __NR_fstatfs 138
+#define __NR_sysfs 139
+#define __NR_getpriority 140
+#define __NR_setpriority 141
+#define __NR_sched_setparam 142
+#define __NR_sched_getparam 143
+#define __NR_sched_setscheduler 144
+#define __NR_sched_getscheduler 145
+#define __NR_sched_get_priority_max 146
+#define __NR_sched_get_priority_min 147
+#define __NR_sched_rr_get_interval 148
+#define __NR_mlock 149
+#define __NR_munlock 150
+#define __NR_mlockall 151
+#define __NR_munlockall 152
+#define __NR_vhangup 153
+#define __NR_modify_ldt 154
+#define __NR_pivot_root 155
+#define __NR__sysctl 156
+#define __NR_prctl 157
+#define __NR_arch_prctl 158
+#define __NR_adjtimex 159
+#define __NR_setrlimit 160
+#define __NR_chroot 161
+#define __NR_sync 162
+#define __NR_acct 163
+#define __NR_settimeofday 164
+#define __NR_mount 165
+#define __NR_umount2 166
+#define __NR_swapon 167
+#define __NR_swapoff 168
+#define __NR_reboot 169
+#define __NR_sethostname 170
+#define __NR_setdomainname 171
+#define __NR_iopl 172
+#define __NR_ioperm 173
+#define __NR_create_module 174
+#define __NR_init_module 175
+#define __NR_delete_module 176
+#define __NR_get_kernel_syms 177
+#define __NR_query_module 178
+#define __NR_quotactl 179
+#define __NR_nfsservctl 180
+#define __NR_getpmsg 181
+#define __NR_putpmsg 182
+#define __NR_afs_syscall 183
+#define __NR_tuxcall 184
+#define __NR_security 185
+#define __NR_gettid 186
+#define __NR_readahead 187
+#define __NR_setxattr 188
+#define __NR_lsetxattr 189
+#define __NR_fsetxattr 190
+#define __NR_getxattr 191
+#define __NR_lgetxattr 192
+#define __NR_fgetxattr 193
+#define __NR_listxattr 194
+#define __NR_llistxattr 195
+#define __NR_flistxattr 196
+#define __NR_removexattr 197
+#define __NR_lremovexattr 198
+#define __NR_fremovexattr 199
+#define __NR_tkill 200
+#define __NR_time 201
+#define __NR_futex 202
+#define __NR_sched_setaffinity 203
+#define __NR_sched_getaffinity 204
+#define __NR_set_thread_area 205
+#define __NR_io_setup 206
+#define __NR_io_destroy 207
+#define __NR_io_getevents 208
+#define __NR_io_submit 209
+#define __NR_io_cancel 210
+#define __NR_get_thread_area 211
+#define __NR_lookup_dcookie 212
+#define __NR_epoll_create 213
+#define __NR_epoll_ctl_old 214
+#define __NR_epoll_wait_old 215
+#define __NR_remap_file_pages 216
+#define __NR_getdents64 217
+#define __NR_set_tid_address 218
+#define __NR_restart_syscall 219
+#define __NR_semtimedop 220
+#define __NR_fadvise64 221
+#define __NR_timer_create 222
+#define __NR_timer_settime 223
+#define __NR_timer_gettime 224
+#define __NR_timer_getoverrun 225
+#define __NR_timer_delete 226
+#define __NR_clock_settime 227
+#define __NR_clock_gettime 228
+#define __NR_clock_getres 229
+#define __NR_clock_nanosleep 230
+#define __NR_exit_group 231
+#define __NR_epoll_wait 232
+#define __NR_epoll_ctl 233
+#define __NR_tgkill 234
+#define __NR_utimes 235
+#define __NR_vserver 236
+#define __NR_mbind 237
+#define __NR_set_mempolicy 238
+#define __NR_get_mempolicy 239
+#define __NR_mq_open 240
+#define __NR_mq_unlink 241
+#define __NR_mq_timedsend 242
+#define __NR_mq_timedreceive 243
+#define __NR_mq_notify 244
+#define __NR_mq_getsetattr 245
+#define __NR_kexec_load 246
+#define __NR_waitid 247
+#define __NR_add_key 248
+#define __NR_request_key 249
+#define __NR_keyctl 250
+#define __NR_ioprio_set 251
+#define __NR_ioprio_get 252
+#define __NR_inotify_init 253
+#define __NR_inotify_add_watch 254
+#define __NR_inotify_rm_watch 255
+#define __NR_migrate_pages 256
+#define __NR_openat 257
+#define __NR_mkdirat 258
+#define __NR_mknodat 259
+#define __NR_fchownat 260
+#define __NR_futimesat 261
+#define __NR_newfstatat 262
+#define __NR_unlinkat 263
+#define __NR_renameat 264
+#define __NR_linkat 265
+#define __NR_symlinkat 266
+#define __NR_readlinkat 267
+#define __NR_fchmodat 268
+#define __NR_faccessat 269
+#define __NR_pselect6 270
+#define __NR_ppoll 271
+#define __NR_unshare 272
+#define __NR_set_robust_list 273
+#define __NR_get_robust_list 274
+#define __NR_splice 275
+#define __NR_tee 276
+#define __NR_sync_file_range 277
+#define __NR_vmsplice 278
+#define __NR_move_pages 279
+#define __NR_utimensat 280
+#define __NR_epoll_pwait 281
+#define __NR_signalfd 282
+#define __NR_timerfd_create 283
+#define __NR_eventfd 284
+#define __NR_fallocate 285
+#define __NR_timerfd_settime 286
+#define __NR_timerfd_gettime 287
+#define __NR_accept4 288
+#define __NR_signalfd4 289
+#define __NR_eventfd2 290
+#define __NR_epoll_create1 291
+#define __NR_dup3 292
+#define __NR_pipe2 293
+#define __NR_inotify_init1 294
+#define __NR_preadv 295
+#define __NR_pwritev 296
+#define __NR_rt_tgsigqueueinfo 297
+#define __NR_perf_event_open 298
+#define __NR_recvmmsg 299
+#define __NR_fanotify_init 300
+#define __NR_fanotify_mark 301
+#define __NR_prlimit64 302
+#define __NR_name_to_handle_at 303
+#define __NR_open_by_handle_at 304
+#define __NR_clock_adjtime 305
+#define __NR_syncfs 306
+#define __NR_sendmmsg 307
+#define __NR_setns 308
+#define __NR_getcpu 309
+#define __NR_process_vm_readv 310
+#define __NR_process_vm_writev 311
+#define __NR_kcmp 312
+#define __NR_finit_module 313
+#define __NR_sched_setattr 314
+#define __NR_sched_getattr 315
+#define __NR_renameat2 316
+#define __NR_seccomp 317
+#define __NR_getrandom 318
+#define __NR_memfd_create 319
+#define __NR_kexec_file_load 320
+#define __NR_bpf 321
+#define __NR_execveat 322
+#define __NR_userfaultfd 323
+#define __NR_membarrier 324
+#define __NR_mlock2 325
+#define __NR_copy_file_range 326
+#define __NR_preadv2 327
+#define __NR_pwritev2 328
+
+#endif /* _ASM_X86_UNISTD_64_H */
diff --git a/qemu/linux-headers/asm-x86/unistd_x32.h b/qemu/linux-headers/asm-x86/unistd_x32.h
new file mode 100644
index 000000000..8f77ee868
--- /dev/null
+++ b/qemu/linux-headers/asm-x86/unistd_x32.h
@@ -0,0 +1,321 @@
+#ifndef _ASM_X86_UNISTD_X32_H
+#define _ASM_X86_UNISTD_X32_H 1
+
+#define __NR_read (__X32_SYSCALL_BIT + 0)
+#define __NR_write (__X32_SYSCALL_BIT + 1)
+#define __NR_open (__X32_SYSCALL_BIT + 2)
+#define __NR_close (__X32_SYSCALL_BIT + 3)
+#define __NR_stat (__X32_SYSCALL_BIT + 4)
+#define __NR_fstat (__X32_SYSCALL_BIT + 5)
+#define __NR_lstat (__X32_SYSCALL_BIT + 6)
+#define __NR_poll (__X32_SYSCALL_BIT + 7)
+#define __NR_lseek (__X32_SYSCALL_BIT + 8)
+#define __NR_mmap (__X32_SYSCALL_BIT + 9)
+#define __NR_mprotect (__X32_SYSCALL_BIT + 10)
+#define __NR_munmap (__X32_SYSCALL_BIT + 11)
+#define __NR_brk (__X32_SYSCALL_BIT + 12)
+#define __NR_rt_sigprocmask (__X32_SYSCALL_BIT + 14)
+#define __NR_pread64 (__X32_SYSCALL_BIT + 17)
+#define __NR_pwrite64 (__X32_SYSCALL_BIT + 18)
+#define __NR_access (__X32_SYSCALL_BIT + 21)
+#define __NR_pipe (__X32_SYSCALL_BIT + 22)
+#define __NR_select (__X32_SYSCALL_BIT + 23)
+#define __NR_sched_yield (__X32_SYSCALL_BIT + 24)
+#define __NR_mremap (__X32_SYSCALL_BIT + 25)
+#define __NR_msync (__X32_SYSCALL_BIT + 26)
+#define __NR_mincore (__X32_SYSCALL_BIT + 27)
+#define __NR_madvise (__X32_SYSCALL_BIT + 28)
+#define __NR_shmget (__X32_SYSCALL_BIT + 29)
+#define __NR_shmat (__X32_SYSCALL_BIT + 30)
+#define __NR_shmctl (__X32_SYSCALL_BIT + 31)
+#define __NR_dup (__X32_SYSCALL_BIT + 32)
+#define __NR_dup2 (__X32_SYSCALL_BIT + 33)
+#define __NR_pause (__X32_SYSCALL_BIT + 34)
+#define __NR_nanosleep (__X32_SYSCALL_BIT + 35)
+#define __NR_getitimer (__X32_SYSCALL_BIT + 36)
+#define __NR_alarm (__X32_SYSCALL_BIT + 37)
+#define __NR_setitimer (__X32_SYSCALL_BIT + 38)
+#define __NR_getpid (__X32_SYSCALL_BIT + 39)
+#define __NR_sendfile (__X32_SYSCALL_BIT + 40)
+#define __NR_socket (__X32_SYSCALL_BIT + 41)
+#define __NR_connect (__X32_SYSCALL_BIT + 42)
+#define __NR_accept (__X32_SYSCALL_BIT + 43)
+#define __NR_sendto (__X32_SYSCALL_BIT + 44)
+#define __NR_shutdown (__X32_SYSCALL_BIT + 48)
+#define __NR_bind (__X32_SYSCALL_BIT + 49)
+#define __NR_listen (__X32_SYSCALL_BIT + 50)
+#define __NR_getsockname (__X32_SYSCALL_BIT + 51)
+#define __NR_getpeername (__X32_SYSCALL_BIT + 52)
+#define __NR_socketpair (__X32_SYSCALL_BIT + 53)
+#define __NR_clone (__X32_SYSCALL_BIT + 56)
+#define __NR_fork (__X32_SYSCALL_BIT + 57)
+#define __NR_vfork (__X32_SYSCALL_BIT + 58)
+#define __NR_exit (__X32_SYSCALL_BIT + 60)
+#define __NR_wait4 (__X32_SYSCALL_BIT + 61)
+#define __NR_kill (__X32_SYSCALL_BIT + 62)
+#define __NR_uname (__X32_SYSCALL_BIT + 63)
+#define __NR_semget (__X32_SYSCALL_BIT + 64)
+#define __NR_semop (__X32_SYSCALL_BIT + 65)
+#define __NR_semctl (__X32_SYSCALL_BIT + 66)
+#define __NR_shmdt (__X32_SYSCALL_BIT + 67)
+#define __NR_msgget (__X32_SYSCALL_BIT + 68)
+#define __NR_msgsnd (__X32_SYSCALL_BIT + 69)
+#define __NR_msgrcv (__X32_SYSCALL_BIT + 70)
+#define __NR_msgctl (__X32_SYSCALL_BIT + 71)
+#define __NR_fcntl (__X32_SYSCALL_BIT + 72)
+#define __NR_flock (__X32_SYSCALL_BIT + 73)
+#define __NR_fsync (__X32_SYSCALL_BIT + 74)
+#define __NR_fdatasync (__X32_SYSCALL_BIT + 75)
+#define __NR_truncate (__X32_SYSCALL_BIT + 76)
+#define __NR_ftruncate (__X32_SYSCALL_BIT + 77)
+#define __NR_getdents (__X32_SYSCALL_BIT + 78)
+#define __NR_getcwd (__X32_SYSCALL_BIT + 79)
+#define __NR_chdir (__X32_SYSCALL_BIT + 80)
+#define __NR_fchdir (__X32_SYSCALL_BIT + 81)
+#define __NR_rename (__X32_SYSCALL_BIT + 82)
+#define __NR_mkdir (__X32_SYSCALL_BIT + 83)
+#define __NR_rmdir (__X32_SYSCALL_BIT + 84)
+#define __NR_creat (__X32_SYSCALL_BIT + 85)
+#define __NR_link (__X32_SYSCALL_BIT + 86)
+#define __NR_unlink (__X32_SYSCALL_BIT + 87)
+#define __NR_symlink (__X32_SYSCALL_BIT + 88)
+#define __NR_readlink (__X32_SYSCALL_BIT + 89)
+#define __NR_chmod (__X32_SYSCALL_BIT + 90)
+#define __NR_fchmod (__X32_SYSCALL_BIT + 91)
+#define __NR_chown (__X32_SYSCALL_BIT + 92)
+#define __NR_fchown (__X32_SYSCALL_BIT + 93)
+#define __NR_lchown (__X32_SYSCALL_BIT + 94)
+#define __NR_umask (__X32_SYSCALL_BIT + 95)
+#define __NR_gettimeofday (__X32_SYSCALL_BIT + 96)
+#define __NR_getrlimit (__X32_SYSCALL_BIT + 97)
+#define __NR_getrusage (__X32_SYSCALL_BIT + 98)
+#define __NR_sysinfo (__X32_SYSCALL_BIT + 99)
+#define __NR_times (__X32_SYSCALL_BIT + 100)
+#define __NR_getuid (__X32_SYSCALL_BIT + 102)
+#define __NR_syslog (__X32_SYSCALL_BIT + 103)
+#define __NR_getgid (__X32_SYSCALL_BIT + 104)
+#define __NR_setuid (__X32_SYSCALL_BIT + 105)
+#define __NR_setgid (__X32_SYSCALL_BIT + 106)
+#define __NR_geteuid (__X32_SYSCALL_BIT + 107)
+#define __NR_getegid (__X32_SYSCALL_BIT + 108)
+#define __NR_setpgid (__X32_SYSCALL_BIT + 109)
+#define __NR_getppid (__X32_SYSCALL_BIT + 110)
+#define __NR_getpgrp (__X32_SYSCALL_BIT + 111)
+#define __NR_setsid (__X32_SYSCALL_BIT + 112)
+#define __NR_setreuid (__X32_SYSCALL_BIT + 113)
+#define __NR_setregid (__X32_SYSCALL_BIT + 114)
+#define __NR_getgroups (__X32_SYSCALL_BIT + 115)
+#define __NR_setgroups (__X32_SYSCALL_BIT + 116)
+#define __NR_setresuid (__X32_SYSCALL_BIT + 117)
+#define __NR_getresuid (__X32_SYSCALL_BIT + 118)
+#define __NR_setresgid (__X32_SYSCALL_BIT + 119)
+#define __NR_getresgid (__X32_SYSCALL_BIT + 120)
+#define __NR_getpgid (__X32_SYSCALL_BIT + 121)
+#define __NR_setfsuid (__X32_SYSCALL_BIT + 122)
+#define __NR_setfsgid (__X32_SYSCALL_BIT + 123)
+#define __NR_getsid (__X32_SYSCALL_BIT + 124)
+#define __NR_capget (__X32_SYSCALL_BIT + 125)
+#define __NR_capset (__X32_SYSCALL_BIT + 126)
+#define __NR_rt_sigsuspend (__X32_SYSCALL_BIT + 130)
+#define __NR_utime (__X32_SYSCALL_BIT + 132)
+#define __NR_mknod (__X32_SYSCALL_BIT + 133)
+#define __NR_personality (__X32_SYSCALL_BIT + 135)
+#define __NR_ustat (__X32_SYSCALL_BIT + 136)
+#define __NR_statfs (__X32_SYSCALL_BIT + 137)
+#define __NR_fstatfs (__X32_SYSCALL_BIT + 138)
+#define __NR_sysfs (__X32_SYSCALL_BIT + 139)
+#define __NR_getpriority (__X32_SYSCALL_BIT + 140)
+#define __NR_setpriority (__X32_SYSCALL_BIT + 141)
+#define __NR_sched_setparam (__X32_SYSCALL_BIT + 142)
+#define __NR_sched_getparam (__X32_SYSCALL_BIT + 143)
+#define __NR_sched_setscheduler (__X32_SYSCALL_BIT + 144)
+#define __NR_sched_getscheduler (__X32_SYSCALL_BIT + 145)
+#define __NR_sched_get_priority_max (__X32_SYSCALL_BIT + 146)
+#define __NR_sched_get_priority_min (__X32_SYSCALL_BIT + 147)
+#define __NR_sched_rr_get_interval (__X32_SYSCALL_BIT + 148)
+#define __NR_mlock (__X32_SYSCALL_BIT + 149)
+#define __NR_munlock (__X32_SYSCALL_BIT + 150)
+#define __NR_mlockall (__X32_SYSCALL_BIT + 151)
+#define __NR_munlockall (__X32_SYSCALL_BIT + 152)
+#define __NR_vhangup (__X32_SYSCALL_BIT + 153)
+#define __NR_modify_ldt (__X32_SYSCALL_BIT + 154)
+#define __NR_pivot_root (__X32_SYSCALL_BIT + 155)
+#define __NR_prctl (__X32_SYSCALL_BIT + 157)
+#define __NR_arch_prctl (__X32_SYSCALL_BIT + 158)
+#define __NR_adjtimex (__X32_SYSCALL_BIT + 159)
+#define __NR_setrlimit (__X32_SYSCALL_BIT + 160)
+#define __NR_chroot (__X32_SYSCALL_BIT + 161)
+#define __NR_sync (__X32_SYSCALL_BIT + 162)
+#define __NR_acct (__X32_SYSCALL_BIT + 163)
+#define __NR_settimeofday (__X32_SYSCALL_BIT + 164)
+#define __NR_mount (__X32_SYSCALL_BIT + 165)
+#define __NR_umount2 (__X32_SYSCALL_BIT + 166)
+#define __NR_swapon (__X32_SYSCALL_BIT + 167)
+#define __NR_swapoff (__X32_SYSCALL_BIT + 168)
+#define __NR_reboot (__X32_SYSCALL_BIT + 169)
+#define __NR_sethostname (__X32_SYSCALL_BIT + 170)
+#define __NR_setdomainname (__X32_SYSCALL_BIT + 171)
+#define __NR_iopl (__X32_SYSCALL_BIT + 172)
+#define __NR_ioperm (__X32_SYSCALL_BIT + 173)
+#define __NR_init_module (__X32_SYSCALL_BIT + 175)
+#define __NR_delete_module (__X32_SYSCALL_BIT + 176)
+#define __NR_quotactl (__X32_SYSCALL_BIT + 179)
+#define __NR_getpmsg (__X32_SYSCALL_BIT + 181)
+#define __NR_putpmsg (__X32_SYSCALL_BIT + 182)
+#define __NR_afs_syscall (__X32_SYSCALL_BIT + 183)
+#define __NR_tuxcall (__X32_SYSCALL_BIT + 184)
+#define __NR_security (__X32_SYSCALL_BIT + 185)
+#define __NR_gettid (__X32_SYSCALL_BIT + 186)
+#define __NR_readahead (__X32_SYSCALL_BIT + 187)
+#define __NR_setxattr (__X32_SYSCALL_BIT + 188)
+#define __NR_lsetxattr (__X32_SYSCALL_BIT + 189)
+#define __NR_fsetxattr (__X32_SYSCALL_BIT + 190)
+#define __NR_getxattr (__X32_SYSCALL_BIT + 191)
+#define __NR_lgetxattr (__X32_SYSCALL_BIT + 192)
+#define __NR_fgetxattr (__X32_SYSCALL_BIT + 193)
+#define __NR_listxattr (__X32_SYSCALL_BIT + 194)
+#define __NR_llistxattr (__X32_SYSCALL_BIT + 195)
+#define __NR_flistxattr (__X32_SYSCALL_BIT + 196)
+#define __NR_removexattr (__X32_SYSCALL_BIT + 197)
+#define __NR_lremovexattr (__X32_SYSCALL_BIT + 198)
+#define __NR_fremovexattr (__X32_SYSCALL_BIT + 199)
+#define __NR_tkill (__X32_SYSCALL_BIT + 200)
+#define __NR_time (__X32_SYSCALL_BIT + 201)
+#define __NR_futex (__X32_SYSCALL_BIT + 202)
+#define __NR_sched_setaffinity (__X32_SYSCALL_BIT + 203)
+#define __NR_sched_getaffinity (__X32_SYSCALL_BIT + 204)
+#define __NR_io_destroy (__X32_SYSCALL_BIT + 207)
+#define __NR_io_getevents (__X32_SYSCALL_BIT + 208)
+#define __NR_io_cancel (__X32_SYSCALL_BIT + 210)
+#define __NR_lookup_dcookie (__X32_SYSCALL_BIT + 212)
+#define __NR_epoll_create (__X32_SYSCALL_BIT + 213)
+#define __NR_remap_file_pages (__X32_SYSCALL_BIT + 216)
+#define __NR_getdents64 (__X32_SYSCALL_BIT + 217)
+#define __NR_set_tid_address (__X32_SYSCALL_BIT + 218)
+#define __NR_restart_syscall (__X32_SYSCALL_BIT + 219)
+#define __NR_semtimedop (__X32_SYSCALL_BIT + 220)
+#define __NR_fadvise64 (__X32_SYSCALL_BIT + 221)
+#define __NR_timer_settime (__X32_SYSCALL_BIT + 223)
+#define __NR_timer_gettime (__X32_SYSCALL_BIT + 224)
+#define __NR_timer_getoverrun (__X32_SYSCALL_BIT + 225)
+#define __NR_timer_delete (__X32_SYSCALL_BIT + 226)
+#define __NR_clock_settime (__X32_SYSCALL_BIT + 227)
+#define __NR_clock_gettime (__X32_SYSCALL_BIT + 228)
+#define __NR_clock_getres (__X32_SYSCALL_BIT + 229)
+#define __NR_clock_nanosleep (__X32_SYSCALL_BIT + 230)
+#define __NR_exit_group (__X32_SYSCALL_BIT + 231)
+#define __NR_epoll_wait (__X32_SYSCALL_BIT + 232)
+#define __NR_epoll_ctl (__X32_SYSCALL_BIT + 233)
+#define __NR_tgkill (__X32_SYSCALL_BIT + 234)
+#define __NR_utimes (__X32_SYSCALL_BIT + 235)
+#define __NR_mbind (__X32_SYSCALL_BIT + 237)
+#define __NR_set_mempolicy (__X32_SYSCALL_BIT + 238)
+#define __NR_get_mempolicy (__X32_SYSCALL_BIT + 239)
+#define __NR_mq_open (__X32_SYSCALL_BIT + 240)
+#define __NR_mq_unlink (__X32_SYSCALL_BIT + 241)
+#define __NR_mq_timedsend (__X32_SYSCALL_BIT + 242)
+#define __NR_mq_timedreceive (__X32_SYSCALL_BIT + 243)
+#define __NR_mq_getsetattr (__X32_SYSCALL_BIT + 245)
+#define __NR_add_key (__X32_SYSCALL_BIT + 248)
+#define __NR_request_key (__X32_SYSCALL_BIT + 249)
+#define __NR_keyctl (__X32_SYSCALL_BIT + 250)
+#define __NR_ioprio_set (__X32_SYSCALL_BIT + 251)
+#define __NR_ioprio_get (__X32_SYSCALL_BIT + 252)
+#define __NR_inotify_init (__X32_SYSCALL_BIT + 253)
+#define __NR_inotify_add_watch (__X32_SYSCALL_BIT + 254)
+#define __NR_inotify_rm_watch (__X32_SYSCALL_BIT + 255)
+#define __NR_migrate_pages (__X32_SYSCALL_BIT + 256)
+#define __NR_openat (__X32_SYSCALL_BIT + 257)
+#define __NR_mkdirat (__X32_SYSCALL_BIT + 258)
+#define __NR_mknodat (__X32_SYSCALL_BIT + 259)
+#define __NR_fchownat (__X32_SYSCALL_BIT + 260)
+#define __NR_futimesat (__X32_SYSCALL_BIT + 261)
+#define __NR_newfstatat (__X32_SYSCALL_BIT + 262)
+#define __NR_unlinkat (__X32_SYSCALL_BIT + 263)
+#define __NR_renameat (__X32_SYSCALL_BIT + 264)
+#define __NR_linkat (__X32_SYSCALL_BIT + 265)
+#define __NR_symlinkat (__X32_SYSCALL_BIT + 266)
+#define __NR_readlinkat (__X32_SYSCALL_BIT + 267)
+#define __NR_fchmodat (__X32_SYSCALL_BIT + 268)
+#define __NR_faccessat (__X32_SYSCALL_BIT + 269)
+#define __NR_pselect6 (__X32_SYSCALL_BIT + 270)
+#define __NR_ppoll (__X32_SYSCALL_BIT + 271)
+#define __NR_unshare (__X32_SYSCALL_BIT + 272)
+#define __NR_splice (__X32_SYSCALL_BIT + 275)
+#define __NR_tee (__X32_SYSCALL_BIT + 276)
+#define __NR_sync_file_range (__X32_SYSCALL_BIT + 277)
+#define __NR_utimensat (__X32_SYSCALL_BIT + 280)
+#define __NR_epoll_pwait (__X32_SYSCALL_BIT + 281)
+#define __NR_signalfd (__X32_SYSCALL_BIT + 282)
+#define __NR_timerfd_create (__X32_SYSCALL_BIT + 283)
+#define __NR_eventfd (__X32_SYSCALL_BIT + 284)
+#define __NR_fallocate (__X32_SYSCALL_BIT + 285)
+#define __NR_timerfd_settime (__X32_SYSCALL_BIT + 286)
+#define __NR_timerfd_gettime (__X32_SYSCALL_BIT + 287)
+#define __NR_accept4 (__X32_SYSCALL_BIT + 288)
+#define __NR_signalfd4 (__X32_SYSCALL_BIT + 289)
+#define __NR_eventfd2 (__X32_SYSCALL_BIT + 290)
+#define __NR_epoll_create1 (__X32_SYSCALL_BIT + 291)
+#define __NR_dup3 (__X32_SYSCALL_BIT + 292)
+#define __NR_pipe2 (__X32_SYSCALL_BIT + 293)
+#define __NR_inotify_init1 (__X32_SYSCALL_BIT + 294)
+#define __NR_perf_event_open (__X32_SYSCALL_BIT + 298)
+#define __NR_fanotify_init (__X32_SYSCALL_BIT + 300)
+#define __NR_fanotify_mark (__X32_SYSCALL_BIT + 301)
+#define __NR_prlimit64 (__X32_SYSCALL_BIT + 302)
+#define __NR_name_to_handle_at (__X32_SYSCALL_BIT + 303)
+#define __NR_open_by_handle_at (__X32_SYSCALL_BIT + 304)
+#define __NR_clock_adjtime (__X32_SYSCALL_BIT + 305)
+#define __NR_syncfs (__X32_SYSCALL_BIT + 306)
+#define __NR_setns (__X32_SYSCALL_BIT + 308)
+#define __NR_getcpu (__X32_SYSCALL_BIT + 309)
+#define __NR_kcmp (__X32_SYSCALL_BIT + 312)
+#define __NR_finit_module (__X32_SYSCALL_BIT + 313)
+#define __NR_sched_setattr (__X32_SYSCALL_BIT + 314)
+#define __NR_sched_getattr (__X32_SYSCALL_BIT + 315)
+#define __NR_renameat2 (__X32_SYSCALL_BIT + 316)
+#define __NR_seccomp (__X32_SYSCALL_BIT + 317)
+#define __NR_getrandom (__X32_SYSCALL_BIT + 318)
+#define __NR_memfd_create (__X32_SYSCALL_BIT + 319)
+#define __NR_kexec_file_load (__X32_SYSCALL_BIT + 320)
+#define __NR_bpf (__X32_SYSCALL_BIT + 321)
+#define __NR_userfaultfd (__X32_SYSCALL_BIT + 323)
+#define __NR_membarrier (__X32_SYSCALL_BIT + 324)
+#define __NR_mlock2 (__X32_SYSCALL_BIT + 325)
+#define __NR_copy_file_range (__X32_SYSCALL_BIT + 326)
+#define __NR_rt_sigaction (__X32_SYSCALL_BIT + 512)
+#define __NR_rt_sigreturn (__X32_SYSCALL_BIT + 513)
+#define __NR_ioctl (__X32_SYSCALL_BIT + 514)
+#define __NR_readv (__X32_SYSCALL_BIT + 515)
+#define __NR_writev (__X32_SYSCALL_BIT + 516)
+#define __NR_recvfrom (__X32_SYSCALL_BIT + 517)
+#define __NR_sendmsg (__X32_SYSCALL_BIT + 518)
+#define __NR_recvmsg (__X32_SYSCALL_BIT + 519)
+#define __NR_execve (__X32_SYSCALL_BIT + 520)
+#define __NR_ptrace (__X32_SYSCALL_BIT + 521)
+#define __NR_rt_sigpending (__X32_SYSCALL_BIT + 522)
+#define __NR_rt_sigtimedwait (__X32_SYSCALL_BIT + 523)
+#define __NR_rt_sigqueueinfo (__X32_SYSCALL_BIT + 524)
+#define __NR_sigaltstack (__X32_SYSCALL_BIT + 525)
+#define __NR_timer_create (__X32_SYSCALL_BIT + 526)
+#define __NR_mq_notify (__X32_SYSCALL_BIT + 527)
+#define __NR_kexec_load (__X32_SYSCALL_BIT + 528)
+#define __NR_waitid (__X32_SYSCALL_BIT + 529)
+#define __NR_set_robust_list (__X32_SYSCALL_BIT + 530)
+#define __NR_get_robust_list (__X32_SYSCALL_BIT + 531)
+#define __NR_vmsplice (__X32_SYSCALL_BIT + 532)
+#define __NR_move_pages (__X32_SYSCALL_BIT + 533)
+#define __NR_preadv (__X32_SYSCALL_BIT + 534)
+#define __NR_pwritev (__X32_SYSCALL_BIT + 535)
+#define __NR_rt_tgsigqueueinfo (__X32_SYSCALL_BIT + 536)
+#define __NR_recvmmsg (__X32_SYSCALL_BIT + 537)
+#define __NR_sendmmsg (__X32_SYSCALL_BIT + 538)
+#define __NR_process_vm_readv (__X32_SYSCALL_BIT + 539)
+#define __NR_process_vm_writev (__X32_SYSCALL_BIT + 540)
+#define __NR_setsockopt (__X32_SYSCALL_BIT + 541)
+#define __NR_getsockopt (__X32_SYSCALL_BIT + 542)
+#define __NR_io_setup (__X32_SYSCALL_BIT + 543)
+#define __NR_io_submit (__X32_SYSCALL_BIT + 544)
+#define __NR_execveat (__X32_SYSCALL_BIT + 545)
+
+#endif /* _ASM_X86_UNISTD_X32_H */
diff --git a/qemu/linux-headers/linux/kvm.h b/qemu/linux-headers/linux/kvm.h
index 3bac8736d..3bae71a87 100644
--- a/qemu/linux-headers/linux/kvm.h
+++ b/qemu/linux-headers/linux/kvm.h
@@ -154,6 +154,26 @@ struct kvm_s390_skeys {
 	__u32 flags;
 	__u32 reserved[9];
 };
+
+struct kvm_hyperv_exit {
+#define KVM_EXIT_HYPERV_SYNIC          1
+#define KVM_EXIT_HYPERV_HCALL          2
+	__u32 type;
+	union {
+		struct {
+			__u32 msr;
+			__u64 control;
+			__u64 evt_page;
+			__u64 msg_page;
+		} synic;
+		struct {
+			__u64 input;
+			__u64 result;
+			__u64 params[2];
+		} hcall;
+	} u;
+};
+
 #define KVM_S390_GET_SKEYS_NONE   1
 #define KVM_S390_SKEYS_MAX        1048576
 
@@ -183,6 +203,8 @@ struct kvm_s390_skeys {
 #define KVM_EXIT_EPR              23
 #define KVM_EXIT_SYSTEM_EVENT     24
 #define KVM_EXIT_S390_STSI        25
+#define KVM_EXIT_IOAPIC_EOI       26
+#define KVM_EXIT_HYPERV           27
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -237,6 +259,7 @@ struct kvm_run {
 			__u32 count;
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
+		/* KVM_EXIT_DEBUG */
 		struct {
 			struct kvm_debug_exit_arch arch;
 		} debug;
@@ -285,6 +308,7 @@ struct kvm_run {
 			__u32 data;
 			__u8  is_write;
 		} dcr;
+		/* KVM_EXIT_INTERNAL_ERROR */
 		struct {
 			__u32 suberror;
 			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
@@ -295,6 +319,7 @@ struct kvm_run {
 		struct {
 			__u64 gprs[32];
 		} osi;
+		/* KVM_EXIT_PAPR_HCALL */
 		struct {
 			__u64 nr;
 			__u64 ret;
@@ -317,6 +342,7 @@ struct kvm_run {
 		struct {
 #define KVM_SYSTEM_EVENT_SHUTDOWN       1
 #define KVM_SYSTEM_EVENT_RESET          2
+#define KVM_SYSTEM_EVENT_CRASH          3
 			__u32 type;
 			__u64 flags;
 		} system_event;
@@ -329,6 +355,12 @@ struct kvm_run {
 			__u8 sel1;
 			__u16 sel2;
 		} s390_stsi;
+		/* KVM_EXIT_IOAPIC_EOI */
+		struct {
+			__u8 vector;
+		} eoi;
+		/* KVM_EXIT_HYPERV */
+		struct kvm_hyperv_exit hyperv;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -481,6 +513,7 @@ struct kvm_s390_psw {
 	 ((ai) << 26))
 #define KVM_S390_INT_IO_MIN		0x00000000u
 #define KVM_S390_INT_IO_MAX		0xfffdffffu
+#define KVM_S390_INT_IO_AI_MASK		0x04000000u
 
 
 struct kvm_s390_interrupt {
@@ -514,7 +547,13 @@ struct kvm_s390_pgm_info {
 	__u8 exc_access_id;
 	__u8 per_access_id;
 	__u8 op_access_id;
-	__u8 pad[3];
+#define KVM_S390_PGM_FLAGS_ILC_VALID	0x01
+#define KVM_S390_PGM_FLAGS_ILC_0	0x02
+#define KVM_S390_PGM_FLAGS_ILC_1	0x04
+#define KVM_S390_PGM_FLAGS_ILC_MASK	0x06
+#define KVM_S390_PGM_FLAGS_NO_REWIND	0x08
+	__u8 flags;
+	__u8 pad[2];
 };
 
 struct kvm_s390_prefix_info {
@@ -817,6 +856,15 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_DISABLE_QUIRKS 116
 #define KVM_CAP_X86_SMM 117
 #define KVM_CAP_MULTI_ADDRESS_SPACE 118
+#define KVM_CAP_GUEST_DEBUG_HW_BPS 119
+#define KVM_CAP_GUEST_DEBUG_HW_WPS 120
+#define KVM_CAP_SPLIT_IRQCHIP 121
+#define KVM_CAP_IOEVENTFD_ANY_LENGTH 122
+#define KVM_CAP_HYPERV_SYNIC 123
+#define KVM_CAP_S390_RI 124
+#define KVM_CAP_SPAPR_TCE_64 125
+#define KVM_CAP_ARM_PMU_V3 126
+#define KVM_CAP_VCPU_ATTRIBUTES 127
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -840,10 +888,16 @@ struct kvm_irq_routing_s390_adapter {
 	__u32 adapter_id;
 };
 
+struct kvm_irq_routing_hv_sint {
+	__u32 vcpu;
+	__u32 sint;
+};
+
 /* gsi routing entry types */
 #define KVM_IRQ_ROUTING_IRQCHIP 1
 #define KVM_IRQ_ROUTING_MSI 2
 #define KVM_IRQ_ROUTING_S390_ADAPTER 3
+#define KVM_IRQ_ROUTING_HV_SINT 4
 
 struct kvm_irq_routing_entry {
 	__u32 gsi;
@@ -854,6 +908,7 @@ struct kvm_irq_routing_entry {
 		struct kvm_irq_routing_irqchip irqchip;
 		struct kvm_irq_routing_msi msi;
 		struct kvm_irq_routing_s390_adapter adapter;
+		struct kvm_irq_routing_hv_sint hv_sint;
 		__u32 pad[8];
 	} u;
 };
@@ -1102,6 +1157,8 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_PPC_ALLOC_HTAB */
 #define KVM_PPC_ALLOCATE_HTAB	  _IOWR(KVMIO, 0xa7, __u32)
 #define KVM_CREATE_SPAPR_TCE	  _IOW(KVMIO,  0xa8, struct kvm_create_spapr_tce)
+#define KVM_CREATE_SPAPR_TCE_64	  _IOW(KVMIO,  0xa8, \
+				       struct kvm_create_spapr_tce_64)
 /* Available with KVM_CAP_RMA */
 #define KVM_ALLOCATE_RMA	  _IOR(KVMIO,  0xa9, struct kvm_allocate_rma)
 /* Available with KVM_CAP_PPC_HTAB_FD */
diff --git a/qemu/linux-headers/linux/psci.h b/qemu/linux-headers/linux/psci.h
index 5a7676307..08d443f7c 100644
--- a/qemu/linux-headers/linux/psci.h
+++ b/qemu/linux-headers/linux/psci.h
@@ -46,6 +46,11 @@
 #define PSCI_0_2_FN64_MIGRATE			PSCI_0_2_FN64(5)
 #define PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU	PSCI_0_2_FN64(7)
 
+#define PSCI_1_0_FN_PSCI_FEATURES		PSCI_0_2_FN(10)
+#define PSCI_1_0_FN_SYSTEM_SUSPEND		PSCI_0_2_FN(14)
+
+#define PSCI_1_0_FN64_SYSTEM_SUSPEND		PSCI_0_2_FN64(14)
+
 /* PSCI v0.2 power state encoding for CPU_SUSPEND function */
 #define PSCI_0_2_POWER_STATE_ID_MASK		0xffff
 #define PSCI_0_2_POWER_STATE_ID_SHIFT		0
@@ -56,6 +61,13 @@
 #define PSCI_0_2_POWER_STATE_AFFL_MASK		\
 				(0x3 << PSCI_0_2_POWER_STATE_AFFL_SHIFT)
 
+/* PSCI extended power state encoding for CPU_SUSPEND function */
+#define PSCI_1_0_EXT_POWER_STATE_ID_MASK	0xfffffff
+#define PSCI_1_0_EXT_POWER_STATE_ID_SHIFT	0
+#define PSCI_1_0_EXT_POWER_STATE_TYPE_SHIFT	30
+#define PSCI_1_0_EXT_POWER_STATE_TYPE_MASK	\
+				(0x1 << PSCI_1_0_EXT_POWER_STATE_TYPE_SHIFT)
+
 /* PSCI v0.2 affinity level state returned by AFFINITY_INFO */
 #define PSCI_0_2_AFFINITY_LEVEL_ON		0
 #define PSCI_0_2_AFFINITY_LEVEL_OFF		1
@@ -76,6 +88,11 @@
 #define PSCI_VERSION_MINOR(ver)			\
 		((ver) & PSCI_VERSION_MINOR_MASK)
 
+/* PSCI features decoding (>=1.0) */
+#define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT	1
+#define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_MASK	\
+			(0x1 << PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT)
+
 /* PSCI return values (inclusive of all PSCI versions) */
 #define PSCI_RET_SUCCESS			0
 #define PSCI_RET_NOT_SUPPORTED			-1
@@ -86,5 +103,6 @@
 #define PSCI_RET_INTERNAL_FAILURE		-6
 #define PSCI_RET_NOT_PRESENT			-7
 #define PSCI_RET_DISABLED			-8
+#define PSCI_RET_INVALID_ADDRESS		-9
 
 #endif /* _LINUX_PSCI_H */
diff --git a/qemu/linux-headers/linux/userfaultfd.h b/qemu/linux-headers/linux/userfaultfd.h
new file mode 100644
index 000000000..19e845324
--- /dev/null
+++ b/qemu/linux-headers/linux/userfaultfd.h
@@ -0,0 +1,167 @@
+/*
+ *  include/linux/userfaultfd.h
+ *
+ *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
+ *  Copyright (C) 2015  Red Hat, Inc.
+ *
+ */
+
+#ifndef _LINUX_USERFAULTFD_H
+#define _LINUX_USERFAULTFD_H
+
+#include <linux/types.h>
+
+#define UFFD_API ((__u64)0xAA)
+/*
+ * After implementing the respective features it will become:
+ * #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \
+ *			      UFFD_FEATURE_EVENT_FORK)
+ */
+#define UFFD_API_FEATURES (0)
+#define UFFD_API_IOCTLS				\
+	((__u64)1 << _UFFDIO_REGISTER |		\
+	 (__u64)1 << _UFFDIO_UNREGISTER |	\
+	 (__u64)1 << _UFFDIO_API)
+#define UFFD_API_RANGE_IOCTLS			\
+	((__u64)1 << _UFFDIO_WAKE |		\
+	 (__u64)1 << _UFFDIO_COPY |		\
+	 (__u64)1 << _UFFDIO_ZEROPAGE)
+
+/*
+ * Valid ioctl command number range with this API is from 0x00 to
+ * 0x3F.  UFFDIO_API is the fixed number, everything else can be
+ * changed by implementing a different UFFD_API. If sticking to the
+ * same UFFD_API more ioctl can be added and userland will be aware of
+ * which ioctl the running kernel implements through the ioctl command
+ * bitmask written by the UFFDIO_API.
+ */
+#define _UFFDIO_REGISTER		(0x00)
+#define _UFFDIO_UNREGISTER		(0x01)
+#define _UFFDIO_WAKE			(0x02)
+#define _UFFDIO_COPY			(0x03)
+#define _UFFDIO_ZEROPAGE		(0x04)
+#define _UFFDIO_API			(0x3F)
+
+/* userfaultfd ioctl ids */
+#define UFFDIO 0xAA
+#define UFFDIO_API		_IOWR(UFFDIO, _UFFDIO_API,	\
+				      struct uffdio_api)
+#define UFFDIO_REGISTER		_IOWR(UFFDIO, _UFFDIO_REGISTER, \
+				      struct uffdio_register)
+#define UFFDIO_UNREGISTER	_IOR(UFFDIO, _UFFDIO_UNREGISTER,	\
+				     struct uffdio_range)
+#define UFFDIO_WAKE		_IOR(UFFDIO, _UFFDIO_WAKE,	\
+				     struct uffdio_range)
+#define UFFDIO_COPY		_IOWR(UFFDIO, _UFFDIO_COPY,	\
+				      struct uffdio_copy)
+#define UFFDIO_ZEROPAGE		_IOWR(UFFDIO, _UFFDIO_ZEROPAGE,	\
+				      struct uffdio_zeropage)
+
+/* read() structure */
+struct uffd_msg {
+	__u8	event;
+
+	__u8	reserved1;
+	__u16	reserved2;
+	__u32	reserved3;
+
+	union {
+		struct {
+			__u64	flags;
+			__u64	address;
+		} pagefault;
+
+		struct {
+			/* unused reserved fields */
+			__u64	reserved1;
+			__u64	reserved2;
+			__u64	reserved3;
+		} reserved;
+	} arg;
+} __attribute__((packed));
+
+/*
+ * Start at 0x12 and not at 0 to be more strict against bugs.
+ */
+#define UFFD_EVENT_PAGEFAULT	0x12
+#if 0 /* not available yet */
+#define UFFD_EVENT_FORK		0x13
+#endif
+
+/* flags for UFFD_EVENT_PAGEFAULT */
+#define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
+#define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
+
+struct uffdio_api {
+	/* userland asks for an API number and the features to enable */
+	__u64 api;
+	/*
+	 * Kernel answers below with the all available features for
+	 * the API, this notifies userland of which events and/or
+	 * which flags for each event are enabled in the current
+	 * kernel.
+	 *
+	 * Note: UFFD_EVENT_PAGEFAULT and UFFD_PAGEFAULT_FLAG_WRITE
+	 * are to be considered implicitly always enabled in all kernels as
+	 * long as the uffdio_api.api requested matches UFFD_API.
+	 */
+#if 0 /* not available yet */
+#define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
+#define UFFD_FEATURE_EVENT_FORK			(1<<1)
+#endif
+	__u64 features;
+
+	__u64 ioctls;
+};
+
+struct uffdio_range {
+	__u64 start;
+	__u64 len;
+};
+
+struct uffdio_register {
+	struct uffdio_range range;
+#define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
+#define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
+	__u64 mode;
+
+	/*
+	 * kernel answers which ioctl commands are available for the
+	 * range, keep at the end as the last 8 bytes aren't read.
+	 */
+	__u64 ioctls;
+};
+
+struct uffdio_copy {
+	__u64 dst;
+	__u64 src;
+	__u64 len;
+	/*
+	 * There will be a wrprotection flag later that allows to map
+	 * pages wrprotected on the fly. And such a flag will be
+	 * available if the wrprotection ioctl are implemented for the
+	 * range according to the uffdio_register.ioctls.
+	 */
+#define UFFDIO_COPY_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "copy" is written by the ioctl and must be at the end: the
+	 * copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 copy;
+};
+
+struct uffdio_zeropage {
+	struct uffdio_range range;
+#define UFFDIO_ZEROPAGE_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "zeropage" is written by the ioctl and must be at the end:
+	 * the copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 zeropage;
+};
+
+#endif /* _LINUX_USERFAULTFD_H */
diff --git a/qemu/linux-headers/linux/vfio.h b/qemu/linux-headers/linux/vfio.h
index aa276bce3..759b850a3 100644
--- a/qemu/linux-headers/linux/vfio.h
+++ b/qemu/linux-headers/linux/vfio.h
@@ -39,6 +39,13 @@
 #define VFIO_SPAPR_TCE_v2_IOMMU		7
 
 /*
+ * The No-IOMMU IOMMU offers no translation or isolation for devices and
+ * supports no ioctls outside of VFIO_CHECK_EXTENSION.  Use of VFIO's No-IOMMU
+ * code will taint the host kernel and should be used with extreme caution.
+ */
+#define VFIO_NOIOMMU_IOMMU		8
+
+/*
  * The IOCTL interface is designed for extensibility by embedding the
  * structure length (argsz) and flags into structures passed between
  * kernel and userspace.  We therefore use the _IO() macro for these
@@ -52,6 +59,33 @@
 #define VFIO_TYPE	(';')
 #define VFIO_BASE	100
 
+/*
+ * For extension of INFO ioctls, VFIO makes use of a capability chain
+ * designed after PCI/e capabilities.  A flag bit indicates whether
+ * this capability chain is supported and a field defined in the fixed
+ * structure defines the offset of the first capability in the chain.
+ * This field is only valid when the corresponding bit in the flags
+ * bitmap is set.  This offset field is relative to the start of the
+ * INFO buffer, as is the next field within each capability header.
+ * The id within the header is a shared address space per INFO ioctl,
+ * while the version field is specific to the capability id.  The
+ * contents following the header are specific to the capability id.
+ */
+struct vfio_info_cap_header {
+	__u16	id;		/* Identifies capability */
+	__u16	version;	/* Version specific to the capability ID */
+	__u32	next;		/* Offset of next capability */
+};
+
+/*
+ * Callers of INFO ioctls passing insufficiently sized buffers will see
+ * the capability chain flag bit set, a zero value for the first capability
+ * offset (if available within the provided argsz), and argsz will be
+ * updated to report the necessary buffer size.  For compatibility, the
+ * INFO ioctl will not report error in this case, but the capability chain
+ * will not be available.
+ */
+
 /* -------- IOCTLs for VFIO file descriptor (/dev/vfio/vfio) -------- */
 
 /**
@@ -187,13 +221,73 @@ struct vfio_region_info {
 #define VFIO_REGION_INFO_FLAG_READ	(1 << 0) /* Region supports read */
 #define VFIO_REGION_INFO_FLAG_WRITE	(1 << 1) /* Region supports write */
 #define VFIO_REGION_INFO_FLAG_MMAP	(1 << 2) /* Region supports mmap */
+#define VFIO_REGION_INFO_FLAG_CAPS	(1 << 3) /* Info supports caps */
 	__u32	index;		/* Region index */
-	__u32	resv;		/* Reserved for alignment */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 	__u64	size;		/* Region size (bytes) */
 	__u64	offset;		/* Region offset from start of device fd */
 };
 #define VFIO_DEVICE_GET_REGION_INFO	_IO(VFIO_TYPE, VFIO_BASE + 8)
 
+/*
+ * The sparse mmap capability allows finer granularity of specifying areas
+ * within a region with mmap support.  When specified, the user should only
+ * mmap the offset ranges specified by the areas array.  mmaps outside of the
+ * areas specified may fail (such as the range covering a PCI MSI-X table) or
+ * may result in improper device behavior.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_REGION_INFO_CAP_SPARSE_MMAP	1
+
+struct vfio_region_sparse_mmap_area {
+	__u64	offset;	/* Offset of mmap'able area within region */
+	__u64	size;	/* Size of mmap'able area */
+};
+
+struct vfio_region_info_cap_sparse_mmap {
+	struct vfio_info_cap_header header;
+	__u32	nr_areas;
+	__u32	reserved;
+	struct vfio_region_sparse_mmap_area areas[];
+};
+
+/*
+ * The device specific type capability allows regions unique to a specific
+ * device or class of devices to be exposed.  This helps solve the problem for
+ * vfio bus drivers of defining which region indexes correspond to which region
+ * on the device, without needing to resort to static indexes, as done by
+ * vfio-pci.  For instance, if we were to go back in time, we might remove
+ * VFIO_PCI_VGA_REGION_INDEX and let vfio-pci simply define that all indexes
+ * greater than or equal to VFIO_PCI_NUM_REGIONS are device specific and we'd
+ * make a "VGA" device specific type to describe the VGA access space.  This
+ * means that non-VGA devices wouldn't need to waste this index, and thus the
+ * address space associated with it due to implementation of device file
+ * descriptor offsets in vfio-pci.
+ *
+ * The current implementation is now part of the user ABI, so we can't use this
+ * for VGA, but there are other upcoming use cases, such as opregions for Intel
+ * IGD devices and framebuffers for vGPU devices.  We missed VGA, but we'll
+ * use this for future additions.
+ *
+ * The structure below defines version 1 of this capability.
+ */
+#define VFIO_REGION_INFO_CAP_TYPE	2
+
+struct vfio_region_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;	/* global per bus driver */
+	__u32 subtype;	/* type specific */
+};
+
+#define VFIO_REGION_TYPE_PCI_VENDOR_TYPE	(1 << 31)
+#define VFIO_REGION_TYPE_PCI_VENDOR_MASK	(0xffff)
+
+/* 8086 Vendor sub-types */
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION	(1)
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG	(2)
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG	(3)
+
 /**
  * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9,
  *				    struct vfio_irq_info)
@@ -329,7 +423,8 @@ enum {
 	 * between described ranges are unimplemented.
 	 */
 	VFIO_PCI_VGA_REGION_INDEX,
-	VFIO_PCI_NUM_REGIONS
+	VFIO_PCI_NUM_REGIONS = 9 /* Fixed user ABI, region indexes >=9 use */
+				 /* device specific cap to define content. */
 };
 
 enum {
@@ -568,8 +663,10 @@ struct vfio_iommu_spapr_tce_create {
 	__u32 flags;
 	/* in */
 	__u32 page_shift;
+	__u32 __resv1;
 	__u64 window_size;
 	__u32 levels;
+	__u32 __resv2;
 	/* out */
 	__u64 start_addr;
 };
diff --git a/qemu/linux-headers/linux/vhost.h b/qemu/linux-headers/linux/vhost.h
index ead86db91..571294cea 100644
--- a/qemu/linux-headers/linux/vhost.h
+++ b/qemu/linux-headers/linux/vhost.h
@@ -126,6 +126,12 @@ struct vhost_memory {
 #define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
 /* Set eventfd to signal an error */
 #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
+/* Set busy loop timeout (in us) */
+#define VHOST_SET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x23,	\
+					 struct vhost_vring_state)
+/* Get busy loop timeout (in us) */
+#define VHOST_GET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x24,	\
+					 struct vhost_vring_state)
 
 /* VHOST_NET specific defines */
 
diff --git a/qemu/linux-user/aarch64/syscall_nr.h b/qemu/linux-user/aarch64/syscall_nr.h
index 743255db0..59511d855 100644
--- a/qemu/linux-user/aarch64/syscall_nr.h
+++ b/qemu/linux-user/aarch64/syscall_nr.h
@@ -262,62 +262,16 @@
 #define TARGET_NR_process_vm_writev 271
 #define TARGET_NR_kcmp 272
 #define TARGET_NR_finit_module 273
-#define TARGET_NR_open 1024
-#define TARGET_NR_link 1025
-#define TARGET_NR_unlink 1026
-#define TARGET_NR_mknod 1027
-#define TARGET_NR_chmod 1028
-#define TARGET_NR_chown 1029
-#define TARGET_NR_mkdir 1030
-#define TARGET_NR_rmdir 1031
-#define TARGET_NR_lchown 1032
-#define TARGET_NR_access 1033
-#define TARGET_NR_rename 1034
-#define TARGET_NR_readlink 1035
-#define TARGET_NR_symlink 1036
-#define TARGET_NR_utimes 1037
-#define TARGET_NR_stat 1038
-#define TARGET_NR_lstat 1039
-#define TARGET_NR_pipe 1040
-#define TARGET_NR_dup2 1041
-#define TARGET_NR_epoll_create 1042
-#define TARGET_NR_inotify_init 1043
-#define TARGET_NR_eventfd 1044
-#define TARGET_NR_signalfd 1045
-#define TARGET_NR_sendfile64 1046
-#define TARGET_NR_ftruncate64 1047
-#define TARGET_NR_truncate64 1048
-#define TARGET_NR_stat64 1049
-#define TARGET_NR_lstat64 1050
-#define TARGET_NR_fstat64 1051
-#define TARGET_NR_fcntl64 1052
-/* #define TARGET_NR_fadvise64 1053 */
-#define TARGET_NR_newfstatat 1054
-#define TARGET_NR_fstatfs64 1055
-#define TARGET_NR_statfs64 1056
-#define TARGET_NR_lseek64 1057
-#define TARGET_NR_mmap64 1058
-#define TARGET_NR_alarm 1059
-#define TARGET_NR_getpgrp 1060
-#define TARGET_NR_pause 1061
-#define TARGET_NR_time 1062
-#define TARGET_NR_utime 1063
-#define TARGET_NR_creat 1064
-#define TARGET_NR_getdents 1065
-#define TARGET_NR_futimesat 1066
-#define TARGET_NR_select 1067
-#define TARGET_NR_poll 1068
-#define TARGET_NR_epoll_wait 1069
-#define TARGET_NR_ustat 1070
-#define TARGET_NR_vfork 1071
-#define TARGET_NR_oldwait4 1072
-#define TARGET_NR_recv 1073
-#define TARGET_NR_send 1074
-#define TARGET_NR_bdflush 1075
-#define TARGET_NR_umount 1076
-#define TARGET_NR_uselib 1077
-#define TARGET_NR__sysctl 1078
-#define TARGET_NR_fork 1079
-#define TARGET_NR_syscalls (__NR_fork+1)
+#define TARGET_NR_sched_setattr 274
+#define TARGET_NR_sched_getattr 275
+#define TARGET_NR_renameat2 276
+#define TARGET_NR_seccomp 277
+#define TARGET_NR_getrandom 278
+#define TARGET_NR_memfd_create 279
+#define TARGET_NR_bpf 280
+#define TARGET_NR_execveat 281
+#define TARGET_NR_userfaultfd 282
+#define TARGET_NR_membarrier 283
+#define TARGET_NR_mlock2 284
+#define TARGET_NR_copy_file_range 285
 
-#define TARGET_NR_sigreturn 1999
diff --git a/qemu/linux-user/aarch64/syscall.h b/qemu/linux-user/aarch64/target_syscall.h
index dc72a15c5..f45801804 100644
--- a/qemu/linux-user/aarch64/syscall.h
+++ b/qemu/linux-user/aarch64/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
     uint64_t        regs[31];
     uint64_t        sp;
@@ -11,3 +14,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ       2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/alpha/syscall_nr.h b/qemu/linux-user/alpha/syscall_nr.h
index dde8d5c6a..00e14bb6b 100644
--- a/qemu/linux-user/alpha/syscall_nr.h
+++ b/qemu/linux-user/alpha/syscall_nr.h
@@ -444,3 +444,9 @@
 #define TARGET_NR_process_vm_writev             505
 #define TARGET_NR_kcmp                          506
 #define TARGET_NR_finit_module                  507
+#define TARGET_NR_sched_setattr                 508
+#define TARGET_NR_sched_getattr                 509
+#define TARGET_NR_renameat2                     510
+#define TARGET_NR_getrandom                     511
+#define TARGET_NR_memfd_create                  512
+#define TARGET_NR_execveat                      513
diff --git a/qemu/linux-user/alpha/syscall.h b/qemu/linux-user/alpha/target_syscall.h
index 245cff254..3db4b16f6 100644
--- a/qemu/linux-user/alpha/syscall.h
+++ b/qemu/linux-user/alpha/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 /* default linux values for the selectors */
 #define __USER_DS	(1)
 
@@ -255,3 +258,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ              4096
 #define TARGET_MLOCKALL_MCL_CURRENT     0x2000
 #define TARGET_MLOCKALL_MCL_FUTURE      0x4000
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/arm/nwfpe/double_cpdo.c b/qemu/linux-user/arm/nwfpe/double_cpdo.c
index 41c28f309..1cef38085 100644
--- a/qemu/linux-user/arm/nwfpe/double_cpdo.c
+++ b/qemu/linux-user/arm/nwfpe/double_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/nwfpe/extended_cpdo.c b/qemu/linux-user/arm/nwfpe/extended_cpdo.c
index 48eca3b4a..1c8a41229 100644
--- a/qemu/linux-user/arm/nwfpe/extended_cpdo.c
+++ b/qemu/linux-user/arm/nwfpe/extended_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/nwfpe/fpa11.c b/qemu/linux-user/arm/nwfpe/fpa11.c
index eebd93fc0..441e3b1cf 100644
--- a/qemu/linux-user/arm/nwfpe/fpa11.c
+++ b/qemu/linux-user/arm/nwfpe/fpa11.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 
 #include "fpopcode.h"
@@ -27,7 +28,6 @@
 
 //#include <asm/system.h>
 
-#include <stdio.h>
 
 FPA11* qemufpa = NULL;
 CPUARMState* user_registers;
diff --git a/qemu/linux-user/arm/nwfpe/fpa11.h b/qemu/linux-user/arm/nwfpe/fpa11.h
index bb9ac6554..0b072843d 100644
--- a/qemu/linux-user/arm/nwfpe/fpa11.h
+++ b/qemu/linux-user/arm/nwfpe/fpa11.h
@@ -21,9 +21,6 @@
 #ifndef __FPA11_H__
 #define __FPA11_H__
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
 
 #include <cpu.h>
 
@@ -108,7 +105,7 @@ static inline void writeRegister(unsigned int x, unsigned int y)
 
 static inline void writeConditionCodes(unsigned int x)
 {
-        cpsr_write(user_registers,x,CPSR_NZCV);
+    cpsr_write(user_registers, x, CPSR_NZCV, CPSRWriteByInstr);
 }
 
 #define ARM_REG_PC 15
diff --git a/qemu/linux-user/arm/nwfpe/fpa11_cpdo.c b/qemu/linux-user/arm/nwfpe/fpa11_cpdo.c
index 5f4a6a413..94ac98aef 100644
--- a/qemu/linux-user/arm/nwfpe/fpa11_cpdo.c
+++ b/qemu/linux-user/arm/nwfpe/fpa11_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpopcode.h"
 
diff --git a/qemu/linux-user/arm/nwfpe/fpa11_cpdt.c b/qemu/linux-user/arm/nwfpe/fpa11_cpdt.c
index 007a3d650..c32b0c2fa 100644
--- a/qemu/linux-user/arm/nwfpe/fpa11_cpdt.c
+++ b/qemu/linux-user/arm/nwfpe/fpa11_cpdt.c
@@ -19,6 +19,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/nwfpe/fpa11_cprt.c b/qemu/linux-user/arm/nwfpe/fpa11_cprt.c
index 7be93fa54..04dc2ebb0 100644
--- a/qemu/linux-user/arm/nwfpe/fpa11_cprt.c
+++ b/qemu/linux-user/arm/nwfpe/fpa11_cprt.c
@@ -19,6 +19,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/nwfpe/fpopcode.c b/qemu/linux-user/arm/nwfpe/fpopcode.c
index 0ada30c6b..6784256d2 100644
--- a/qemu/linux-user/arm/nwfpe/fpopcode.c
+++ b/qemu/linux-user/arm/nwfpe/fpopcode.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/nwfpe/single_cpdo.c b/qemu/linux-user/arm/nwfpe/single_cpdo.c
index 2bfb359eb..21e177bae 100644
--- a/qemu/linux-user/arm/nwfpe/single_cpdo.c
+++ b/qemu/linux-user/arm/nwfpe/single_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/qemu/linux-user/arm/syscall_nr.h b/qemu/linux-user/arm/syscall_nr.h
index 53552beab..cc9089ccd 100644
--- a/qemu/linux-user/arm/syscall_nr.h
+++ b/qemu/linux-user/arm/syscall_nr.h
@@ -384,3 +384,15 @@
 #define TARGET_NR_process_vm_writev            (377)
 #define TARGET_NR_kcmp                         (378)
 #define TARGET_NR_finit_module                 (379)
+
+#define TARGET_NR_sched_setattr                (380)
+#define TARGET_NR_sched_getattr                (381)
+#define TARGET_NR_renameat2                    (382)
+#define TARGET_NR_seccomp                      (383)
+#define TARGET_NR_getrandom                    (384)
+#define TARGET_NR_memfd_create                 (385)
+#define TARGET_NR_bpf                          (386)
+#define TARGET_NR_execveat                     (387)
+#define TARGET_NR_userfaultfd                  (388)
+#define TARGET_NR_membarrier                   (389)
+#define TARGET_NR_mlock2                       (390)
diff --git a/qemu/linux-user/arm/syscall.h b/qemu/linux-user/arm/target_syscall.h
index 3844a9611..ea863db0b 100644
--- a/qemu/linux-user/arm/syscall.h
+++ b/qemu/linux-user/arm/target_syscall.h
@@ -1,3 +1,5 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
 
 /* this struct defines the way the registers are stored on the
    stack during a system call. */
@@ -48,3 +50,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/cris/syscall_nr.h b/qemu/linux-user/cris/syscall_nr.h
index 694bd02fa..44f0b645b 100644
--- a/qemu/linux-user/cris/syscall_nr.h
+++ b/qemu/linux-user/cris/syscall_nr.h
@@ -336,3 +336,27 @@
 #define TARGET_NR_preadv             333
 #define TARGET_NR_pwritev            334
 #define TARGET_NR_setns              335
+#define TARGET_NR_name_to_handle_at  336
+#define TARGET_NR_open_by_handle_at  337
+#define TARGET_NR_rt_tgsigqueueinfo  338
+#define TARGET_NR_perf_event_open    339
+#define TARGET_NR_recvmmsg           340
+#define TARGET_NR_accept4            341
+#define TARGET_NR_fanotify_init      342
+#define TARGET_NR_fanotify_mark      343
+#define TARGET_NR_prlimit64          344
+#define TARGET_NR_clock_adjtime      345
+#define TARGET_NR_syncfs             346
+#define TARGET_NR_sendmmsg           347
+#define TARGET_NR_process_vm_readv   348
+#define TARGET_NR_process_vm_writev  349
+#define TARGET_NR_kcmp               350
+#define TARGET_NR_finit_module       351
+#define TARGET_NR_sched_setattr      352
+#define TARGET_NR_sched_getattr      353
+#define TARGET_NR_renameat2          354
+#define TARGET_NR_seccomp            355
+#define TARGET_NR_getrandom          356
+#define TARGET_NR_memfd_create       357
+#define TARGET_NR_bpf                358
+#define TARGET_NR_execveat           359
diff --git a/qemu/linux-user/cris/syscall.h b/qemu/linux-user/cris/target_syscall.h
index 2957b0d6a..2957b0d6a 100644
--- a/qemu/linux-user/cris/syscall.h
+++ b/qemu/linux-user/cris/target_syscall.h
diff --git a/qemu/linux-user/elfload.c b/qemu/linux-user/elfload.c
index 17883686f..e47caff7a 100644
--- a/qemu/linux-user/elfload.c
+++ b/qemu/linux-user/elfload.c
@@ -1,20 +1,13 @@
 /* This is the Linux kernel elf-loading code, ported into user space */
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #include <sys/param.h>
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
 #include <sys/mman.h>
 #include <sys/resource.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
 
 #include "qemu.h"
 #include "disas/disas.h"
+#include "qemu/path.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
@@ -145,7 +138,6 @@ static uint32_t get_elf_hwcap(void)
 
 #ifdef TARGET_X86_64
 #define ELF_START_MMAP 0x2aaaaab000ULL
-#define elf_check_arch(x) ( ((x) == ELF_ARCH) )
 
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_X86_64
@@ -273,9 +265,7 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUX86State *en
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_ARM
 #define ELF_CLASS       ELFCLASS32
 
 static inline void init_thread(struct target_pt_regs *regs,
@@ -481,9 +471,7 @@ static uint32_t get_elf_hwcap2(void)
 /* 64 bit ARM definitions */
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_AARCH64
 #define ELF_CLASS       ELFCLASS64
 #define ELF_PLATFORM    "aarch64"
 
@@ -556,8 +544,6 @@ static uint32_t get_elf_hwcap(void)
 
 #define ELF_START_MMAP          0x80000000
 
-#define elf_check_arch(x)       ((x) == EM_UNICORE32)
-
 #define ELF_CLASS               ELFCLASS32
 #define ELF_DATA                ELFDATA2LSB
 #define ELF_ARCH                EM_UNICORE32
@@ -666,7 +652,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 #define ELF_START_MMAP 0x80000000
 #define ELF_HWCAP  (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | HWCAP_SPARC_SWAP \
                     | HWCAP_SPARC_MULDIV)
-#define elf_check_arch(x) ( (x) == EM_SPARC )
 
 #define ELF_CLASS   ELFCLASS32
 #define ELF_ARCH    EM_SPARC
@@ -686,6 +671,7 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #ifdef TARGET_PPC
 
+#define ELF_MACHINE    PPC_ELF_MACHINE
 #define ELF_START_MMAP 0x80000000
 
 #if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
@@ -696,8 +682,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #else
 
-#define elf_check_arch(x) ( (x) == EM_PPC )
-
 #define ELF_CLASS       ELFCLASS32
 
 #endif
@@ -875,8 +859,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUPPCState *en
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_MIPS )
-
 #ifdef TARGET_MIPS64
 #define ELF_CLASS   ELFCLASS64
 #else
@@ -985,8 +967,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUMBState *env
 
 #define ELF_START_MMAP 0x08000000
 
-#define elf_check_arch(x) ((x) == EM_OPENRISC)
-
 #define ELF_ARCH EM_OPENRISC
 #define ELF_CLASS ELFCLASS32
 #define ELF_DATA  ELFDATA2MSB
@@ -1026,8 +1006,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs,
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_SH )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_SH
 
@@ -1110,8 +1088,6 @@ static uint32_t get_elf_hwcap(void)
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_CRIS )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_CRIS
 
@@ -1129,8 +1105,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_68K )
-
 #define ELF_CLASS       ELFCLASS32
 #define ELF_ARCH        EM_68K
 
@@ -1182,8 +1156,6 @@ static void elf_core_copy_regs(target_elf_gregset_t *regs, const CPUM68KState *e
 
 #define ELF_START_MMAP (0x30000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_ALPHA
 
@@ -1203,8 +1175,6 @@ static inline void init_thread(struct target_pt_regs *regs,
 
 #define ELF_START_MMAP (0x20000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS	ELFCLASS64
 #define ELF_DATA	ELFDATA2MSB
 #define ELF_ARCH	EM_S390
@@ -1218,10 +1188,41 @@ static inline void init_thread(struct target_pt_regs *regs, struct image_info *i
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+/* 42 bits real used address, a half for user mode */
+#define ELF_START_MMAP (0x00000020000000000ULL)
+
+#define elf_check_arch(x) ((x) == EM_TILEGX)
+
+#define ELF_CLASS   ELFCLASS64
+#define ELF_DATA    ELFDATA2LSB
+#define ELF_ARCH    EM_TILEGX
+
+static inline void init_thread(struct target_pt_regs *regs,
+                               struct image_info *infop)
+{
+    regs->pc = infop->entry;
+    regs->sp = infop->start_stack;
+
+}
+
+#define ELF_EXEC_PAGESIZE        65536 /* TILE-Gx page size is 64KB */
+
+#endif /* TARGET_TILEGX */
+
 #ifndef ELF_PLATFORM
 #define ELF_PLATFORM (NULL)
 #endif
 
+#ifndef ELF_MACHINE
+#define ELF_MACHINE ELF_ARCH
+#endif
+
+#ifndef elf_check_arch
+#define elf_check_arch(x) ((x) == ELF_ARCH)
+#endif
+
 #ifndef ELF_HWCAP
 #define ELF_HWCAP 0
 #endif
@@ -1365,66 +1366,69 @@ static bool elf_check_ehdr(struct elfhdr *ehdr)
  * to be put directly into the top of new user memory.
  *
  */
-static abi_ulong copy_elf_strings(int argc,char ** argv, void **page,
-                                  abi_ulong p)
+static abi_ulong copy_elf_strings(int argc, char **argv, char *scratch,
+                                  abi_ulong p, abi_ulong stack_limit)
 {
-    char *tmp, *tmp1, *pag = NULL;
-    int len, offset = 0;
+    char *tmp;
+    int len, offset;
+    abi_ulong top = p;
 
     if (!p) {
         return 0;       /* bullet-proofing */
     }
+
+    offset = ((p - 1) % TARGET_PAGE_SIZE) + 1;
+
     while (argc-- > 0) {
         tmp = argv[argc];
         if (!tmp) {
             fprintf(stderr, "VFS: argc is wrong");
             exit(-1);
         }
-        tmp1 = tmp;
-        while (*tmp++);
-        len = tmp - tmp1;
-        if (p < len) {  /* this shouldn't happen - 128kB */
+        len = strlen(tmp) + 1;
+        tmp += len;
+
+        if (len > (p - stack_limit)) {
             return 0;
         }
         while (len) {
-            --p; --tmp; --len;
-            if (--offset < 0) {
-                offset = p % TARGET_PAGE_SIZE;
-                pag = (char *)page[p/TARGET_PAGE_SIZE];
-                if (!pag) {
-                    pag = g_try_malloc0(TARGET_PAGE_SIZE);
-                    page[p/TARGET_PAGE_SIZE] = pag;
-                    if (!pag)
-                        return 0;
-                }
-            }
-            if (len == 0 || offset == 0) {
-                *(pag + offset) = *tmp;
-            }
-            else {
-                int bytes_to_copy = (len > offset) ? offset : len;
-                tmp -= bytes_to_copy;
-                p -= bytes_to_copy;
-                offset -= bytes_to_copy;
-                len -= bytes_to_copy;
-                memcpy_fromfs(pag + offset, tmp, bytes_to_copy + 1);
+            int bytes_to_copy = (len > offset) ? offset : len;
+            tmp -= bytes_to_copy;
+            p -= bytes_to_copy;
+            offset -= bytes_to_copy;
+            len -= bytes_to_copy;
+
+            memcpy_fromfs(scratch + offset, tmp, bytes_to_copy);
+
+            if (offset == 0) {
+                memcpy_to_target(p, scratch, top - p);
+                top = p;
+                offset = TARGET_PAGE_SIZE;
             }
         }
     }
+    if (offset) {
+        memcpy_to_target(p, scratch + offset, top - p);
+    }
+
     return p;
 }
 
-static abi_ulong setup_arg_pages(abi_ulong p, struct linux_binprm *bprm,
+/* Older linux kernels provide up to MAX_ARG_PAGES (default: 32) of
+ * argument/environment space. Newer kernels (>2.6.33) allow more,
+ * dependent on stack size, but guarantee at least 32 pages for
+ * backwards compatibility.
+ */
+#define STACK_LOWER_LIMIT (32 * TARGET_PAGE_SIZE)
+
+static abi_ulong setup_arg_pages(struct linux_binprm *bprm,
                                  struct image_info *info)
 {
-    abi_ulong stack_base, size, error, guard;
-    int i;
+    abi_ulong size, error, guard;
 
-    /* Create enough stack to hold everything.  If we don't use
-       it for args, we'll use it for something else.  */
     size = guest_stack_size;
-    if (size < MAX_ARG_PAGES*TARGET_PAGE_SIZE) {
-        size = MAX_ARG_PAGES*TARGET_PAGE_SIZE;
+    if (size < STACK_LOWER_LIMIT) {
+        size = STACK_LOWER_LIMIT;
     }
     guard = TARGET_PAGE_SIZE;
     if (guard < qemu_real_host_page_size) {
@@ -1442,19 +1446,8 @@ static abi_ulong setup_arg_pages(abi_ulong p, struct linux_binprm *bprm,
     target_mprotect(error, guard, PROT_NONE);
 
     info->stack_limit = error + guard;
-    stack_base = info->stack_limit + size - MAX_ARG_PAGES*TARGET_PAGE_SIZE;
-    p += stack_base;
-
-    for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
-        if (bprm->page[i]) {
-            info->rss++;
-            /* FIXME - check return value of memcpy_to_target() for failure */
-            memcpy_to_target(stack_base, bprm->page[i], TARGET_PAGE_SIZE);
-            g_free(bprm->page[i]);
-        }
-        stack_base += TARGET_PAGE_SIZE;
-    }
-    return p;
+
+    return info->stack_limit + size - sizeof(void *);
 }
 
 /* Map and zero the bss.  We need to explicitly zero any fractional pages
@@ -1478,8 +1471,7 @@ static void zero_bss(abi_ulong elf_bss, abi_ulong last_bss, int prot)
 
     host_start = (uintptr_t) g2h(elf_bss);
     host_end = (uintptr_t) g2h(last_bss);
-    host_map_start = (host_start + qemu_real_host_page_size - 1);
-    host_map_start &= -qemu_real_host_page_size;
+    host_map_start = REAL_HOST_PAGE_ALIGN(host_start);
 
     if (host_map_start < host_end) {
         void *p = mmap((void *)host_map_start, host_end - host_map_start,
@@ -1744,7 +1736,7 @@ unsigned long init_guest_space(unsigned long host_start,
         }
     }
 
-    qemu_log("Reserved 0x%lx bytes of guest address space\n", host_size);
+    qemu_log_mask(CPU_LOG_PAGE, "Reserved 0x%lx bytes of guest address space\n", host_size);
 
     return real_start;
 }
@@ -1756,7 +1748,6 @@ static void probe_guest_base(const char *image_name,
      * it explicitly, and set guest_base appropriately.
      * In case of error we will print a suitable message and exit.
      */
-#if defined(CONFIG_USE_GUEST_BASE)
     const char *errmsg;
     if (!have_guest_base && !reserved_va) {
         unsigned long host_start, real_start, host_size;
@@ -1786,16 +1777,15 @@ static void probe_guest_base(const char *image_name,
         }
         guest_base = real_start - loaddr;
 
-        qemu_log("Relocating guest address space from 0x"
-                 TARGET_ABI_FMT_lx " to 0x%lx\n",
-                 loaddr, real_start);
+        qemu_log_mask(CPU_LOG_PAGE, "Relocating guest address space from 0x"
+                      TARGET_ABI_FMT_lx " to 0x%lx\n",
+                      loaddr, real_start);
     }
     return;
 
 exit_errmsg:
     fprintf(stderr, "%s: %s\n", image_name, errmsg);
     exit(-1);
-#endif
 }
 
 
@@ -2198,10 +2188,9 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)
     struct image_info interp_info;
     struct elfhdr elf_ex;
     char *elf_interpreter = NULL;
+    char *scratch;
 
     info->start_mmap = (abi_ulong)ELF_START_MMAP;
-    info->mmap = 0;
-    info->rss = 0;
 
     load_elf_image(bprm->filename, bprm->fd, info,
                    &elf_interpreter, bprm->buf);
@@ -2211,18 +2200,24 @@ int load_elf_binary(struct linux_binprm *bprm, struct image_info *info)
        when we load the interpreter.  */
     elf_ex = *(struct elfhdr *)bprm->buf;
 
-    bprm->p = copy_elf_strings(1, &bprm->filename, bprm->page, bprm->p);
-    bprm->p = copy_elf_strings(bprm->envc,bprm->envp,bprm->page,bprm->p);
-    bprm->p = copy_elf_strings(bprm->argc,bprm->argv,bprm->page,bprm->p);
+    /* Do this so that we can load the interpreter, if need be.  We will
+       change some of these later */
+    bprm->p = setup_arg_pages(bprm, info);
+
+    scratch = g_new0(char, TARGET_PAGE_SIZE);
+    bprm->p = copy_elf_strings(1, &bprm->filename, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->envc, bprm->envp, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->argc, bprm->argv, scratch,
+                               bprm->p, info->stack_limit);
+    g_free(scratch);
+
     if (!bprm->p) {
         fprintf(stderr, "%s: %s\n", bprm->filename, strerror(E2BIG));
         exit(-1);
     }
 
-    /* Do this so that we can load the interpreter, if need be.  We will
-       change some of these later */
-    bprm->p = setup_arg_pages(bprm->p, bprm, info);
-
     if (elf_interpreter) {
         load_elf_interp(elf_interpreter, &interp_info, bprm->buf);
 
@@ -2850,7 +2845,7 @@ static int fill_note_info(struct elf_note_info *info,
     TaskState *ts = (TaskState *)cpu->opaque;
     int i;
 
-    info->notes = g_malloc0(NUMNOTES * sizeof (struct memelfnote));
+    info->notes = g_new0(struct memelfnote, NUMNOTES);
     if (info->notes == NULL)
         return (-ENOMEM);
     info->prstatus = g_malloc0(sizeof (*info->prstatus));
@@ -3016,7 +3011,7 @@ static int elf_core_dump(int signr, const CPUArchState *env)
     if (dump_write(fd, &elf, sizeof (elf)) != 0)
         goto out;
 
-    /* fill in in-memory version of notes */
+    /* fill in the in-memory version of notes */
     if (fill_note_info(&info, signr, env) < 0)
         goto out;
 
diff --git a/qemu/linux-user/flatload.c b/qemu/linux-user/flatload.c
index 566a7a87a..f9139c399 100644
--- a/qemu/linux-user/flatload.c
+++ b/qemu/linux-user/flatload.c
@@ -33,15 +33,11 @@
 
 /****************************************************************************/
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
-#include <unistd.h>
 
 #include "qemu.h"
 #include "flat.h"
-#define ntohl(x) be32_to_cpu(x)
 #include <target_flat.h>
 
 //#define DEBUG
@@ -707,7 +703,7 @@ static int load_flat_shared_library(int id, struct lib_info *libs)
 int load_flt_binary(struct linux_binprm *bprm, struct image_info *info)
 {
     struct lib_info libinfo[MAX_SHARED_LIBS];
-    abi_ulong p = bprm->p;
+    abi_ulong p;
     abi_ulong stack_len;
     abi_ulong start_addr;
     abi_ulong sp;
diff --git a/qemu/linux-user/i386/syscall_nr.h b/qemu/linux-user/i386/syscall_nr.h
index c8f730219..bc1bc233e 100644
--- a/qemu/linux-user/i386/syscall_nr.h
+++ b/qemu/linux-user/i386/syscall_nr.h
@@ -324,7 +324,7 @@
 #define TARGET_NR_epoll_pwait	319
 #define TARGET_NR_utimensat		320
 #define TARGET_NR_signalfd		321
-#define TARGET_NR_timerfd		322
+#define TARGET_NR_timerfd_create	322
 #define TARGET_NR_eventfd		323
 #define TARGET_NR_fallocate		324
 #define TARGET_NR_timerfd_settime	325
@@ -353,3 +353,30 @@
 #define TARGET_NR_process_vm_writev     348
 #define TARGET_NR_kcmp                  349
 #define TARGET_NR_finit_module          350
+#define TARGET_NR_sched_setattr         351
+#define TARGET_NR_sched_getattr         352
+#define TARGET_NR_renameat2             353
+#define TARGET_NR_seccomp               354
+#define TARGET_NR_getrandom             355
+#define TARGET_NR_memfd_create          356
+#define TARGET_NR_bpf                   357
+#define TARGET_NR_execveat              358
+#define TARGET_NR_socket                359
+#define TARGET_NR_socketpair            360
+#define TARGET_NR_bind                  361
+#define TARGET_NR_connect               362
+#define TARGET_NR_listen                363
+#define TARGET_NR_accept4               364
+#define TARGET_NR_getsockopt            365
+#define TARGET_NR_setsockopt            366
+#define TARGET_NR_getsockname           367
+#define TARGET_NR_getpeername           368
+#define TARGET_NR_sendto                369
+#define TARGET_NR_sendmsg               370
+#define TARGET_NR_recvfrom              371
+#define TARGET_NR_recvmsg               372
+#define TARGET_NR_shutdown              373
+#define TARGET_NR_userfaultfd           374
+#define TARGET_NR_membarrier            375
+#define TARGET_NR_mlock2                376
+#define TARGET_NR_copy_file_range       377
diff --git a/qemu/linux-user/i386/syscall.h b/qemu/linux-user/i386/target_syscall.h
index 906aaac0b..0ac84dc02 100644
--- a/qemu/linux-user/i386/syscall.h
+++ b/qemu/linux-user/i386/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 /* default linux values for the selectors */
 #define __USER_CS	(0x23)
 #define __USER_DS	(0x2B)
@@ -150,3 +153,5 @@ struct target_vm86plus_struct {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/linuxload.c b/qemu/linux-user/linuxload.c
index 506e837ae..6f0d6054c 100644
--- a/qemu/linux-user/linuxload.c
+++ b/qemu/linux-user/linuxload.c
@@ -1,12 +1,6 @@
 /* Code for loading Linux executables.  Mostly linux kernel code.  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
@@ -135,10 +129,7 @@ int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
              struct linux_binprm *bprm)
 {
     int retval;
-    int i;
 
-    bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
-    memset(bprm->page, 0, sizeof(bprm->page));
     bprm->fd = fdexec;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
@@ -172,9 +163,5 @@ int loader_exec(int fdexec, const char *filename, char **argv, char **envp,
         return retval;
     }
 
-    /* Something went wrong, return the inode and free the argument pages*/
-    for (i=0 ; i<MAX_ARG_PAGES ; i++) {
-        g_free(bprm->page[i]);
-    }
     return(retval);
 }
diff --git a/qemu/linux-user/m68k-sim.c b/qemu/linux-user/m68k-sim.c
index 1994e4000..34d332d8b 100644
--- a/qemu/linux-user/m68k-sim.c
+++ b/qemu/linux-user/m68k-sim.c
@@ -17,14 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
diff --git a/qemu/linux-user/m68k/syscall_nr.h b/qemu/linux-user/m68k/syscall_nr.h
index 25f8521ec..4b50fb29b 100644
--- a/qemu/linux-user/m68k/syscall_nr.h
+++ b/qemu/linux-user/m68k/syscall_nr.h
@@ -317,7 +317,7 @@
 #define TARGET_NR_epoll_pwait	315
 #define TARGET_NR_utimensat		316
 #define TARGET_NR_signalfd		317
-#define TARGET_NR_timerfd		318
+#define TARGET_NR_timerfd_create	318
 #define TARGET_NR_eventfd		319
 #define TARGET_NR_fallocate		320
 #define TARGET_NR_timerfd_settime	321
@@ -349,3 +349,30 @@
 #define TARGET_NR_process_vm_writev     346
 #define TARGET_NR_kcmp                  347
 #define TARGET_NR_finit_module          348
+#define TARGET_NR_sched_setattr         349
+#define TARGET_NR_sched_getattr         350
+#define TARGET_NR_renameat2             351
+#define TARGET_NR_getrandom             352
+#define TARGET_NR_memfd_create          353
+#define TARGET_NR_bpf                   354
+#define TARGET_NR_execveat              355
+#define TARGET_NR_socket                356
+#define TARGET_NR_socketpair            357
+#define TARGET_NR_bind                  358
+#define TARGET_NR_connect               359
+#define TARGET_NR_listen                360
+#define TARGET_NR_accept4               361
+#define TARGET_NR_getsockopt            362
+#define TARGET_NR_setsockopt            363
+#define TARGET_NR_getsockname           364
+#define TARGET_NR_getpeername           365
+#define TARGET_NR_sendto                366
+#define TARGET_NR_sendmsg               367
+#define TARGET_NR_recvfrom              368
+#define TARGET_NR_recvmsg               369
+#define TARGET_NR_shutdown              370
+#define TARGET_NR_recvmmsg              371
+#define TARGET_NR_sendmmsg              372
+#define TARGET_NR_userfaultfd           373
+#define TARGET_NR_membarrier            374
+#define TARGET_NR_mlock2                375
diff --git a/qemu/linux-user/m68k/syscall.h b/qemu/linux-user/m68k/target_syscall.h
index 9218493a4..97a4cc0cb 100644
--- a/qemu/linux-user/m68k/syscall.h
+++ b/qemu/linux-user/m68k/target_syscall.h
@@ -1,3 +1,5 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
 
 /* this struct defines the way the registers are stored on the
    stack during a system call. */
@@ -23,3 +25,5 @@ struct target_pt_regs {
 #define TARGET_MLOCKALL_MCL_FUTURE  2
 
 void do_m68k_simcall(CPUM68KState *, int);
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/main.c b/qemu/linux-user/main.c
index fdee98135..5f3ec9747 100644
--- a/qemu/linux-user/main.c
+++ b/qemu/linux-user/main.c
@@ -16,36 +16,45 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <sys/resource.h>
 
 #include "qemu.h"
-#include "qemu-common.h"
+#include "qemu/path.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
 #include "cpu.h"
 #include "tcg.h"
 #include "qemu/timer.h"
 #include "qemu/envlist.h"
 #include "elf.h"
+#include "exec/log.h"
 
 char *exec_path;
 
 int singlestep;
-const char *filename;
-const char *argv0;
-int gdbstub_port;
-envlist_t *envlist;
+static const char *filename;
+static const char *argv0;
+static int gdbstub_port;
+static envlist_t *envlist;
 static const char *cpu_model;
 unsigned long mmap_min_addr;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long guest_base;
 int have_guest_base;
+
+#define EXCP_DUMP(env, fmt, ...)                                        \
+do {                                                                    \
+    CPUState *cs = ENV_GET_CPU(env);                                    \
+    fprintf(stderr, fmt , ## __VA_ARGS__);                              \
+    cpu_dump_state(cs, stderr, fprintf, 0);                             \
+    if (qemu_log_separate()) {                                          \
+        qemu_log(fmt, ## __VA_ARGS__);                                  \
+        log_cpu_state(cs, 0);                                           \
+    }                                                                   \
+} while (0)
+
 #if (TARGET_LONG_BITS == 32) && (HOST_LONG_BITS == 64)
 /*
  * When running 32-on-64 we should make sure we can fit all of the possible
@@ -63,9 +72,8 @@ unsigned long reserved_va = 0xf7000000;
 #else
 unsigned long reserved_va;
 #endif
-#endif
 
-static void usage(void);
+static void usage(int exitcode);
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -107,7 +115,7 @@ static int pending_cpus;
 /* Make sure everything is in a consistent state for calling fork().  */
 void fork_start(void)
 {
-    pthread_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
     pthread_mutex_lock(&exclusive_lock);
     mmap_fork_start();
 }
@@ -129,11 +137,11 @@ void fork_end(int child)
         pthread_mutex_init(&cpu_list_mutex, NULL);
         pthread_cond_init(&exclusive_cond, NULL);
         pthread_cond_init(&exclusive_resume, NULL);
-        pthread_mutex_init(&tcg_ctx.tb_ctx.tb_lock, NULL);
+        qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
         gdbserver_fork(thread_cpu);
     } else {
         pthread_mutex_unlock(&exclusive_lock);
-        pthread_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
     }
 }
 
@@ -217,7 +225,7 @@ void cpu_list_unlock(void)
 
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -418,8 +426,8 @@ void cpu_loop(CPUX86State *env)
             break;
         default:
             pc = env->segs[R_CS].base + env->eip;
-            fprintf(stderr, "qemu: 0x%08lx: unhandled CPU exception 0x%x - aborting\n",
-                    (long)pc, trapnr);
+            EXCP_DUMP(env, "qemu: 0x%08lx: unhandled CPU exception 0x%x - aborting\n",
+                      (long)pc, trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -429,22 +437,54 @@ void cpu_loop(CPUX86State *env)
 
 #ifdef TARGET_ARM
 
-#define get_user_code_u32(x, gaddr, doswap)             \
+#define get_user_code_u32(x, gaddr, env)                \
     ({ abi_long __r = get_user_u32((x), (gaddr));       \
-        if (!__r && (doswap)) {                         \
+        if (!__r && bswap_code(arm_sctlr_b(env))) {     \
             (x) = bswap32(x);                           \
         }                                               \
         __r;                                            \
     })
 
-#define get_user_code_u16(x, gaddr, doswap)             \
+#define get_user_code_u16(x, gaddr, env)                \
     ({ abi_long __r = get_user_u16((x), (gaddr));       \
-        if (!__r && (doswap)) {                         \
+        if (!__r && bswap_code(arm_sctlr_b(env))) {     \
             (x) = bswap16(x);                           \
         }                                               \
         __r;                                            \
     })
 
+#define get_user_data_u32(x, gaddr, env)                \
+    ({ abi_long __r = get_user_u32((x), (gaddr));       \
+        if (!__r && arm_cpu_bswap_data(env)) {          \
+            (x) = bswap32(x);                           \
+        }                                               \
+        __r;                                            \
+    })
+
+#define get_user_data_u16(x, gaddr, env)                \
+    ({ abi_long __r = get_user_u16((x), (gaddr));       \
+        if (!__r && arm_cpu_bswap_data(env)) {          \
+            (x) = bswap16(x);                           \
+        }                                               \
+        __r;                                            \
+    })
+
+#define put_user_data_u32(x, gaddr, env)                \
+    ({ typeof(x) __x = (x);                             \
+        if (arm_cpu_bswap_data(env)) {                  \
+            __x = bswap32(__x);                         \
+        }                                               \
+        put_user_u32(__x, (gaddr));                     \
+    })
+
+#define put_user_data_u16(x, gaddr, env)                \
+    ({ typeof(x) __x = (x);                             \
+        if (arm_cpu_bswap_data(env)) {                  \
+            __x = bswap16(__x);                         \
+        }                                               \
+        put_user_u16(__x, (gaddr));                     \
+    })
+
 #ifdef TARGET_ABI32
 /* Commpage handling -- there is no commpage for AArch64 */
 
@@ -507,7 +547,7 @@ static void arm_kernel_cmpxchg64_helper(CPUARMState *env)
         env->regs[0] = -1;
         cpsr &= ~CPSR_C;
     }
-    cpsr_write(env, cpsr, CPSR_C);
+    cpsr_write(env, cpsr, CPSR_C, CPSRWriteByInstr);
     end_exclusive();
     return;
 
@@ -556,7 +596,7 @@ do_kernel_trap(CPUARMState *env)
             env->regs[0] = -1;
             cpsr &= ~CPSR_C;
         }
-        cpsr_write(env, cpsr, CPSR_C);
+        cpsr_write(env, cpsr, CPSR_C, CPSRWriteByInstr);
         end_exclusive();
         break;
     case 0xffff0fe0: /* __kernel_get_tls */
@@ -604,11 +644,11 @@ static int do_strex(CPUARMState *env)
         segv = get_user_u8(val, addr);
         break;
     case 1:
-        segv = get_user_u16(val, addr);
+        segv = get_user_data_u16(val, addr, env);
         break;
     case 2:
     case 3:
-        segv = get_user_u32(val, addr);
+        segv = get_user_data_u32(val, addr, env);
         break;
     default:
         abort();
@@ -619,12 +659,16 @@ static int do_strex(CPUARMState *env)
     }
     if (size == 3) {
         uint32_t valhi;
-        segv = get_user_u32(valhi, addr + 4);
+        segv = get_user_data_u32(valhi, addr + 4, env);
         if (segv) {
             env->exception.vaddress = addr + 4;
             goto done;
         }
-        val = deposit64(val, 32, 32, valhi);
+        if (arm_cpu_bswap_data(env)) {
+            val = deposit64((uint64_t)valhi, 32, 32, val);
+        } else {
+            val = deposit64(val, 32, 32, valhi);
+        }
     }
     if (val != env->exclusive_val) {
         goto fail;
@@ -636,11 +680,11 @@ static int do_strex(CPUARMState *env)
         segv = put_user_u8(val, addr);
         break;
     case 1:
-        segv = put_user_u16(val, addr);
+        segv = put_user_data_u16(val, addr, env);
         break;
     case 2:
     case 3:
-        segv = put_user_u32(val, addr);
+        segv = put_user_data_u32(val, addr, env);
         break;
     }
     if (segv) {
@@ -649,7 +693,7 @@ static int do_strex(CPUARMState *env)
     }
     if (size == 3) {
         val = env->regs[(env->exclusive_info >> 12) & 0xf];
-        segv = put_user_u32(val, addr + 4);
+        segv = put_user_data_u32(val, addr + 4, env);
         if (segv) {
             env->exception.vaddress = addr + 4;
             goto done;
@@ -686,7 +730,7 @@ void cpu_loop(CPUARMState *env)
                 /* we handle the FPU emulation here, as Linux */
                 /* we get the opcode */
                 /* FIXME - what to do if get_user() fails? */
-                get_user_code_u32(opcode, env->regs[15], env->bswap_code);
+                get_user_code_u32(opcode, env->regs[15], env);
 
                 rc = EmulateAll(opcode, &ts->fpa, env);
                 if (rc == 0) { /* illegal instruction */
@@ -756,25 +800,23 @@ void cpu_loop(CPUARMState *env)
                 if (trapnr == EXCP_BKPT) {
                     if (env->thumb) {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u16(insn, env->regs[15], env->bswap_code);
+                        get_user_code_u16(insn, env->regs[15], env);
                         n = insn & 0xff;
                         env->regs[15] += 2;
                     } else {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u32(insn, env->regs[15], env->bswap_code);
+                        get_user_code_u32(insn, env->regs[15], env);
                         n = (insn & 0xf) | ((insn >> 4) & 0xff0);
                         env->regs[15] += 4;
                     }
                 } else {
                     if (env->thumb) {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u16(insn, env->regs[15] - 2,
-                                          env->bswap_code);
+                        get_user_code_u16(insn, env->regs[15] - 2, env);
                         n = insn & 0xff;
                     } else {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u32(insn, env->regs[15] - 4,
-                                          env->bswap_code);
+                        get_user_code_u32(insn, env->regs[15] - 4, env);
                         n = insn & 0xffffff;
                     }
                 }
@@ -865,11 +907,12 @@ void cpu_loop(CPUARMState *env)
             if (do_kernel_trap(env))
               goto error;
             break;
+        case EXCP_YIELD:
+            /* nothing to do here for user-mode, just resume guest code */
+            break;
         default:
         error:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -1054,10 +1097,14 @@ void cpu_loop(CPUARMState *env)
                 queue_signal(env, info.si_signo, &info);
             }
             break;
+        case EXCP_SEMIHOST:
+            env->xregs[0] = do_arm_semihosting(env);
+            break;
+        case EXCP_YIELD:
+            /* nothing to do here for user-mode, just resume guest code */
+            break;
         default:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -1147,8 +1194,7 @@ void cpu_loop(CPUUniCore32State *env)
     }
 
 error:
-    fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
-    cpu_dump_state(cs, stderr, fprintf, 0);
+    EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
     abort();
 }
 #endif
@@ -1413,7 +1459,7 @@ void cpu_loop (CPUSPARCState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -1424,7 +1470,7 @@ void cpu_loop (CPUSPARCState *env)
 #ifdef TARGET_PPC
 static inline uint64_t cpu_ppc_get_tb(CPUPPCState *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 uint64_t cpu_ppc_load_tbl(CPUPPCState *env)
@@ -1466,17 +1512,6 @@ int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val)
     return -1;
 }
 
-#define EXCP_DUMP(env, fmt, ...)                                        \
-do {                                                                    \
-    CPUState *cs = ENV_GET_CPU(env);                                    \
-    fprintf(stderr, fmt , ## __VA_ARGS__);                              \
-    cpu_dump_state(cs, stderr, fprintf, 0);                             \
-    qemu_log(fmt, ## __VA_ARGS__);                                      \
-    if (qemu_log_enabled()) {                                           \
-        log_cpu_state(cs, 0);                                           \
-    }                                                                   \
-} while (0)
-
 static int do_store_exclusive(CPUPPCState *env)
 {
     target_ulong addr;
@@ -1649,7 +1684,7 @@ void cpu_loop(CPUPPCState *env)
             info.si_signo = TARGET_SIGBUS;
             info.si_errno = 0;
             info.si_code = TARGET_BUS_ADRALN;
-            info._sifields._sigfault._addr = env->nip - 4;
+            info._sifields._sigfault._addr = env->nip;
             queue_signal(env, info.si_signo, &info);
             break;
         case POWERPC_EXCP_PROGRAM:  /* Program exception                     */
@@ -2635,9 +2670,7 @@ done_syscall:
             break;
         default:
 error:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -2660,11 +2693,11 @@ void cpu_loop(CPUOpenRISCState *env)
 
         switch (trapnr) {
         case EXCP_RESET:
-            qemu_log("\nReset request, exit, pc is %#x\n", env->pc);
-            exit(1);
+            qemu_log_mask(CPU_LOG_INT, "\nReset request, exit, pc is %#x\n", env->pc);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_BUSERR:
-            qemu_log("\nBus error, exit, pc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nBus error, exit, pc is %#x\n", env->pc);
             gdbsig = TARGET_SIGBUS;
             break;
         case EXCP_DPF:
@@ -2673,25 +2706,25 @@ void cpu_loop(CPUOpenRISCState *env)
             gdbsig = TARGET_SIGSEGV;
             break;
         case EXCP_TICK:
-            qemu_log("\nTick time interrupt pc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nTick time interrupt pc is %#x\n", env->pc);
             break;
         case EXCP_ALIGN:
-            qemu_log("\nAlignment pc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nAlignment pc is %#x\n", env->pc);
             gdbsig = TARGET_SIGBUS;
             break;
         case EXCP_ILLEGAL:
-            qemu_log("\nIllegal instructionpc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nIllegal instructionpc is %#x\n", env->pc);
             gdbsig = TARGET_SIGILL;
             break;
         case EXCP_INT:
-            qemu_log("\nExternal interruptpc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nExternal interruptpc is %#x\n", env->pc);
             break;
         case EXCP_DTLBMISS:
         case EXCP_ITLBMISS:
-            qemu_log("\nTLB miss\n");
+            qemu_log_mask(CPU_LOG_INT, "\nTLB miss\n");
             break;
         case EXCP_RANGE:
-            qemu_log("\nRange\n");
+            qemu_log_mask(CPU_LOG_INT, "\nRange\n");
             gdbsig = TARGET_SIGSEGV;
             break;
         case EXCP_SYSCALL:
@@ -2706,26 +2739,25 @@ void cpu_loop(CPUOpenRISCState *env)
                                       env->gpr[8], 0, 0);
             break;
         case EXCP_FPE:
-            qemu_log("\nFloating point error\n");
+            qemu_log_mask(CPU_LOG_INT, "\nFloating point error\n");
             break;
         case EXCP_TRAP:
-            qemu_log("\nTrap\n");
+            qemu_log_mask(CPU_LOG_INT, "\nTrap\n");
             gdbsig = TARGET_SIGTRAP;
             break;
         case EXCP_NR:
-            qemu_log("\nNR\n");
+            qemu_log_mask(CPU_LOG_INT, "\nNR\n");
             break;
         default:
-            qemu_log("\nqemu: unhandled CPU exception %#x - aborting\n",
+            EXCP_DUMP(env, "\nqemu: unhandled CPU exception %#x - aborting\n",
                      trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
             gdbsig = TARGET_SIGILL;
             break;
         }
         if (gdbsig) {
             gdb_handlesig(cs, gdbsig);
             if (gdbsig != TARGET_SIGTRAP) {
-                exit(1);
+                exit(EXIT_FAILURE);
             }
         }
 
@@ -2790,7 +2822,7 @@ void cpu_loop(CPUSH4State *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2851,7 +2883,7 @@ void cpu_loop(CPUCRISState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2932,7 +2964,7 @@ void cpu_loop(CPUMBState *env)
                     printf ("Unhandled hw-exception: 0x%x\n",
                             env->sregs[SR_ESR] & ESR_EC_MASK);
                     cpu_dump_state(cs, stderr, fprintf, 0);
-                    exit (1);
+                    exit(EXIT_FAILURE);
                     break;
             }
             break;
@@ -2953,7 +2985,7 @@ void cpu_loop(CPUMBState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3046,9 +3078,7 @@ void cpu_loop(CPUM68KState *env)
             }
             break;
         default:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -3122,17 +3152,17 @@ void cpu_loop(CPUAlphaState *env)
         switch (trapnr) {
         case EXCP_RESET:
             fprintf(stderr, "Reset requested. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MCHK:
             fprintf(stderr, "Machine check exception. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_SMP_INTERRUPT:
         case EXCP_CLK_INTERRUPT:
         case EXCP_DEV_INTERRUPT:
             fprintf(stderr, "External interrupt. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MMFAULT:
             env->lock_addr = -1;
@@ -3282,7 +3312,7 @@ void cpu_loop(CPUAlphaState *env)
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3386,7 +3416,7 @@ void cpu_loop(CPUS390XState *env)
             default:
                 fprintf(stderr, "Unhandled program exception: %#x\n", n);
                 cpu_dump_state(cs, stderr, fprintf, 0);
-                exit(1);
+                exit(EXIT_FAILURE);
             }
             break;
 
@@ -3403,7 +3433,7 @@ void cpu_loop(CPUS390XState *env)
         default:
             fprintf(stderr, "Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3411,6 +3441,252 @@ void cpu_loop(CPUS390XState *env)
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+static void gen_sigill_reg(CPUTLGState *env)
+{
+    target_siginfo_t info;
+
+    info.si_signo = TARGET_SIGILL;
+    info.si_errno = 0;
+    info.si_code = TARGET_ILL_PRVREG;
+    info._sifields._sigfault._addr = env->pc;
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void do_signal(CPUTLGState *env, int signo, int sigcode)
+{
+    target_siginfo_t info;
+
+    info.si_signo = signo;
+    info.si_errno = 0;
+    info._sifields._sigfault._addr = env->pc;
+
+    if (signo == TARGET_SIGSEGV) {
+        /* The passed in sigcode is a dummy; check for a page mapping
+           and pass either MAPERR or ACCERR.  */
+        target_ulong addr = env->excaddr;
+        info._sifields._sigfault._addr = addr;
+        if (page_check_range(addr, 1, PAGE_VALID) < 0) {
+            sigcode = TARGET_SEGV_MAPERR;
+        } else {
+            sigcode = TARGET_SEGV_ACCERR;
+        }
+    }
+    info.si_code = sigcode;
+
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void gen_sigsegv_maperr(CPUTLGState *env, target_ulong addr)
+{
+    env->excaddr = addr;
+    do_signal(env, TARGET_SIGSEGV, 0);
+}
+
+static void set_regval(CPUTLGState *env, uint8_t reg, uint64_t val)
+{
+    if (unlikely(reg >= TILEGX_R_COUNT)) {
+        switch (reg) {
+        case TILEGX_R_SN:
+        case TILEGX_R_ZERO:
+            return;
+        case TILEGX_R_IDN0:
+        case TILEGX_R_IDN1:
+        case TILEGX_R_UDN0:
+        case TILEGX_R_UDN1:
+        case TILEGX_R_UDN2:
+        case TILEGX_R_UDN3:
+            gen_sigill_reg(env);
+            return;
+        default:
+            g_assert_not_reached();
+        }
+    }
+    env->regs[reg] = val;
+}
+
+/*
+ * Compare the 8-byte contents of the CmpValue SPR with the 8-byte value in
+ * memory at the address held in the first source register. If the values are
+ * not equal, then no memory operation is performed. If the values are equal,
+ * the 8-byte quantity from the second source register is written into memory
+ * at the address held in the first source register. In either case, the result
+ * of the instruction is the value read from memory. The compare and write to
+ * memory are atomic and thus can be used for synchronization purposes. This
+ * instruction only operates for addresses aligned to a 8-byte boundary.
+ * Unaligned memory access causes an Unaligned Data Reference interrupt.
+ *
+ * Functional Description (64-bit)
+ *       uint64_t memVal = memoryReadDoubleWord (rf[SrcA]);
+ *       rf[Dest] = memVal;
+ *       if (memVal == SPR[CmpValueSPR])
+ *           memoryWriteDoubleWord (rf[SrcA], rf[SrcB]);
+ *
+ * Functional Description (32-bit)
+ *       uint64_t memVal = signExtend32 (memoryReadWord (rf[SrcA]));
+ *       rf[Dest] = memVal;
+ *       if (memVal == signExtend32 (SPR[CmpValueSPR]))
+ *           memoryWriteWord (rf[SrcA], rf[SrcB]);
+ *
+ *
+ * This function also processes exch and exch4 which need not process SPR.
+ */
+static void do_exch(CPUTLGState *env, bool quad, bool cmp)
+{
+    target_ulong addr;
+    target_long val, sprval;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    if (cmp) {
+        if (quad) {
+            sprval = env->spregs[TILEGX_SPR_CMPEXCH];
+        } else {
+            sprval = sextract64(env->spregs[TILEGX_SPR_CMPEXCH], 0, 32);
+        }
+    }
+
+    if (!cmp || val == sprval) {
+        target_long valb = env->atomic_srcb;
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+static void do_fetch(CPUTLGState *env, int trapnr, bool quad)
+{
+    int8_t write = 1;
+    target_ulong addr;
+    target_long val, valb;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    valb = env->atomic_srcb;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    switch (trapnr) {
+    case TILEGX_EXCP_OPCODE_FETCHADD:
+    case TILEGX_EXCP_OPCODE_FETCHADD4:
+        valb += val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        valb += val;
+        if (valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        valb += val;
+        if ((int32_t)valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHAND:
+    case TILEGX_EXCP_OPCODE_FETCHAND4:
+        valb &= val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHOR:
+    case TILEGX_EXCP_OPCODE_FETCHOR4:
+        valb |= val;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (write) {
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+void cpu_loop(CPUTLGState *env)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+    int trapnr;
+
+    while (1) {
+        cpu_exec_start(cs);
+        trapnr = cpu_tilegx_exec(cs);
+        cpu_exec_end(cs);
+        switch (trapnr) {
+        case TILEGX_EXCP_SYSCALL:
+            env->regs[TILEGX_R_RE] = do_syscall(env, env->regs[TILEGX_R_NR],
+                                                env->regs[0], env->regs[1],
+                                                env->regs[2], env->regs[3],
+                                                env->regs[4], env->regs[5],
+                                                env->regs[6], env->regs[7]);
+            env->regs[TILEGX_R_ERR] = TILEGX_IS_ERRNO(env->regs[TILEGX_R_RE])
+                                                      ? - env->regs[TILEGX_R_RE]
+                                                      : 0;
+            break;
+        case TILEGX_EXCP_OPCODE_EXCH:
+            do_exch(env, true, false);
+            break;
+        case TILEGX_EXCP_OPCODE_EXCH4:
+            do_exch(env, false, false);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH:
+            do_exch(env, true, true);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH4:
+            do_exch(env, false, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        case TILEGX_EXCP_OPCODE_FETCHAND:
+        case TILEGX_EXCP_OPCODE_FETCHOR:
+            do_fetch(env, trapnr, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD4:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        case TILEGX_EXCP_OPCODE_FETCHAND4:
+        case TILEGX_EXCP_OPCODE_FETCHOR4:
+            do_fetch(env, trapnr, false);
+            break;
+        case TILEGX_EXCP_SIGNAL:
+            do_signal(env, env->signo, env->sigcode);
+            break;
+        case TILEGX_EXCP_REG_IDN_ACCESS:
+        case TILEGX_EXCP_REG_UDN_ACCESS:
+            gen_sigill_reg(env);
+            break;
+        default:
+            fprintf(stderr, "trapnr is %d[0x%x].\n", trapnr, trapnr);
+            g_assert_not_reached();
+        }
+        process_pending_signals(env);
+    }
+}
+
+#endif
+
 THREAD CPUState *thread_cpu;
 
 void task_settid(TaskState *ts)
@@ -3472,7 +3748,7 @@ CPUArchState *cpu_copy(CPUArchState *env)
 
 static void handle_arg_help(const char *arg)
 {
-    usage();
+    usage(EXIT_SUCCESS);
 }
 
 static void handle_arg_log(const char *arg)
@@ -3482,7 +3758,7 @@ static void handle_arg_log(const char *arg)
     mask = qemu_str_to_log_mask(arg);
     if (!mask) {
         qemu_print_log_usage(stdout);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     qemu_set_log(mask);
 }
@@ -3498,7 +3774,7 @@ static void handle_arg_set_env(const char *arg)
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_setenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3510,7 +3786,7 @@ static void handle_arg_unset_env(const char *arg)
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_unsetenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3526,7 +3802,7 @@ static void handle_arg_stack_size(const char *arg)
     char *p;
     guest_stack_size = strtoul(arg, &p, 0);
     if (guest_stack_size == 0) {
-        usage();
+        usage(EXIT_FAILURE);
     }
 
     if (*p == 'M') {
@@ -3547,7 +3823,7 @@ static void handle_arg_pagesize(const char *arg)
     if (qemu_host_page_size == 0 ||
         (qemu_host_page_size & (qemu_host_page_size - 1)) != 0) {
         fprintf(stderr, "page size must be a power of two\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
@@ -3557,7 +3833,7 @@ static void handle_arg_randseed(const char *arg)
 
     if (parse_uint_full(arg, &seed, 0) != 0 || seed > UINT_MAX) {
         fprintf(stderr, "Invalid seed number: %s\n", arg);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     srand(seed);
 }
@@ -3580,11 +3856,10 @@ static void handle_arg_cpu(const char *arg)
 #if defined(cpu_list)
         cpu_list(stdout, &fprintf);
 #endif
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
-#if defined(CONFIG_USE_GUEST_BASE)
 static void handle_arg_guest_base(const char *arg)
 {
     guest_base = strtol(arg, NULL, 0);
@@ -3618,15 +3893,14 @@ static void handle_arg_reserved_va(const char *arg)
 #endif
             ) {
             fprintf(stderr, "Reserved virtual address too big\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
     }
     if (*p) {
         fprintf(stderr, "Unrecognised -R size suffix '%s'\n", p);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
-#endif
 
 static void handle_arg_singlestep(const char *arg)
 {
@@ -3642,7 +3916,7 @@ static void handle_arg_version(const char *arg)
 {
     printf("qemu-" TARGET_NAME " version " QEMU_VERSION QEMU_PKGVERSION
            ", Copyright (c) 2003-2008 Fabrice Bellard\n");
-    exit(0);
+    exit(EXIT_SUCCESS);
 }
 
 struct qemu_argument {
@@ -3657,6 +3931,8 @@ struct qemu_argument {
 static const struct qemu_argument arg_table[] = {
     {"h",          "",                 false, handle_arg_help,
      "",           "print this help"},
+    {"help",       "",                 false, handle_arg_help,
+     "",           ""},
     {"g",          "QEMU_GDB",         true,  handle_arg_gdb,
      "port",       "wait gdb connection to 'port'"},
     {"L",          "QEMU_LD_PREFIX",   true,  handle_arg_ld_prefix,
@@ -3673,12 +3949,10 @@ static const struct qemu_argument arg_table[] = {
      "argv0",      "forces target process argv[0] to be 'argv0'"},
     {"r",          "QEMU_UNAME",       true,  handle_arg_uname,
      "uname",      "set qemu uname release string to 'uname'"},
-#if defined(CONFIG_USE_GUEST_BASE)
     {"B",          "QEMU_GUEST_BASE",  true,  handle_arg_guest_base,
      "address",    "set guest_base address to 'address'"},
     {"R",          "QEMU_RESERVED_VA", true,  handle_arg_reserved_va,
      "size",       "reserve 'size' bytes for guest virtual address space"},
-#endif
     {"d",          "QEMU_LOG",         true,  handle_arg_log,
      "item[,...]", "enable logging of specified items "
      "(use '-d help' for a list of items)"},
@@ -3697,7 +3971,7 @@ static const struct qemu_argument arg_table[] = {
     {NULL, NULL, false, NULL, NULL, NULL}
 };
 
-static void usage(void)
+static void usage(int exitcode)
 {
     const struct qemu_argument *arginfo;
     int maxarglen;
@@ -3764,7 +4038,7 @@ static void usage(void)
            "Note that if you provide several changes to a single variable\n"
            "the last change will stay in effect.\n");
 
-    exit(1);
+    exit(exitcode);
 }
 
 static int parse_args(int argc, char **argv)
@@ -3798,12 +4072,18 @@ static int parse_args(int argc, char **argv)
         if (!strcmp(r, "-")) {
             break;
         }
+        /* Treat --foo the same as -foo.  */
+        if (r[0] == '-') {
+            r++;
+        }
 
         for (arginfo = arg_table; arginfo->handle_opt != NULL; arginfo++) {
             if (!strcmp(r, arginfo->argv)) {
                 if (arginfo->has_arg) {
                     if (optind >= argc) {
-                        usage();
+                        (void) fprintf(stderr,
+                            "qemu: missing argument for option '%s'\n", r);
+                        exit(EXIT_FAILURE);
                     }
                     arginfo->handle_opt(argv[optind]);
                     optind++;
@@ -3816,12 +4096,14 @@ static int parse_args(int argc, char **argv)
 
         /* no option matched the current argv */
         if (arginfo->handle_opt == NULL) {
-            usage();
+            (void) fprintf(stderr, "qemu: unknown option '%s'\n", r);
+            exit(EXIT_FAILURE);
         }
     }
 
     if (optind >= argc) {
-        usage();
+        (void) fprintf(stderr, "qemu: no user program specified\n");
+        exit(EXIT_FAILURE);
     }
 
     filename = argv[optind];
@@ -3850,7 +4132,7 @@ int main(int argc, char **argv, char **envp)
 
     if ((envlist = envlist_create()) == NULL) {
         (void) fprintf(stderr, "Unable to allocate envlist\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 
     /* add current environment into the list */
@@ -3920,7 +4202,7 @@ int main(int argc, char **argv, char **envp)
         cpu_model = "or1200";
 #elif defined(TARGET_PPC)
 # ifdef TARGET_PPC64
-        cpu_model = "POWER7";
+        cpu_model = "POWER8";
 # else
         cpu_model = "750";
 # endif
@@ -3936,7 +4218,7 @@ int main(int argc, char **argv, char **envp)
     cpu = cpu_init(cpu_model);
     if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     env = cpu->env_ptr;
     cpu_reset(cpu);
@@ -3954,7 +4236,6 @@ int main(int argc, char **argv, char **envp)
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -3969,14 +4250,13 @@ int main(int argc, char **argv, char **envp)
                     "space for use as guest address space (check your virtual "
                     "memory ulimit setting or reserve less using -R option)\n",
                     reserved_va);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         if (reserved_va) {
             mmap_next_start = reserved_va;
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /*
      * Read in mmap_min_addr kernel parameter.  This value is used
@@ -3990,7 +4270,7 @@ int main(int argc, char **argv, char **envp)
             unsigned long tmp;
             if (fscanf(fp, "%lu", &tmp) == 1) {
                 mmap_min_addr = tmp;
-                qemu_log("host mmap_min_addr=0x%lx\n", mmap_min_addr);
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n", mmap_min_addr);
             }
             fclose(fp);
         }
@@ -4003,7 +4283,7 @@ int main(int argc, char **argv, char **envp)
     target_argv = calloc(target_argc + 1, sizeof (char *));
     if (target_argv == NULL) {
 	(void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-	exit(1);
+	exit(EXIT_FAILURE);
     }
 
     /*
@@ -4019,7 +4299,7 @@ int main(int argc, char **argv, char **envp)
     }
     target_argv[target_argc] = NULL;
 
-    ts = g_malloc0 (sizeof(TaskState));
+    ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
     ts->info = info;
@@ -4032,7 +4312,7 @@ int main(int argc, char **argv, char **envp)
         execfd = open(filename, O_RDONLY);
         if (execfd < 0) {
             printf("Error while loading %s: %s\n", filename, strerror(errno));
-            _exit(1);
+            _exit(EXIT_FAILURE);
         }
     }
 
@@ -4040,7 +4320,7 @@ int main(int argc, char **argv, char **envp)
         info, &bprm);
     if (ret != 0) {
         printf("Error while loading %s: %s\n", filename, strerror(-ret));
-        _exit(1);
+        _exit(EXIT_FAILURE);
     }
 
     for (wrk = target_environ; *wrk; wrk++) {
@@ -4049,10 +4329,8 @@ int main(int argc, char **argv, char **envp)
 
     free(target_environ);
 
-    if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
+    if (qemu_loglevel_mask(CPU_LOG_PAGE)) {
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -4072,12 +4350,10 @@ int main(int argc, char **argv, char **envp)
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
 #if defined(TARGET_I386)
     env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK;
@@ -4090,7 +4366,7 @@ int main(int argc, char **argv, char **envp)
     /* enable 64 bit mode if possible */
     if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM)) {
         fprintf(stderr, "The selected x86 CPU does not support 64 bit mode\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     env->cr[4] |= CR4_PAE_MASK;
     env->efer |= MSR_EFER_LMA | MSR_EFER_LME;
@@ -4200,7 +4476,7 @@ int main(int argc, char **argv, char **envp)
         if (!(arm_feature(env, ARM_FEATURE_AARCH64))) {
             fprintf(stderr,
                     "The selected ARM CPU does not support 64 bit mode\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         for (i = 0; i < 31; i++) {
@@ -4212,15 +4488,21 @@ int main(int argc, char **argv, char **envp)
 #elif defined(TARGET_ARM)
     {
         int i;
-        cpsr_write(env, regs->uregs[16], 0xffffffff);
+        cpsr_write(env, regs->uregs[16], CPSR_USER | CPSR_EXEC,
+                   CPSRWriteByInstr);
         for(i = 0; i < 16; i++) {
             env->regs[i] = regs->uregs[i];
         }
+#ifdef TARGET_WORDS_BIGENDIAN
         /* Enable BE8.  */
         if (EF_ARM_EABI_VERSION(info->elf_flags) >= EF_ARM_EABI_VER4
             && (info->elf_flags & EF_ARM_BE8)) {
-            env->bswap_code = 1;
+            env->uncached_cpsr |= CPSR_E;
+            env->cp15.sctlr_el[1] |= SCTLR_E0E;
+        } else {
+            env->cp15.sctlr_el[1] |= SCTLR_B;
         }
+#endif
     }
 #elif defined(TARGET_UNICORE32)
     {
@@ -4386,6 +4668,17 @@ int main(int argc, char **argv, char **envp)
             env->psw.mask = regs->psw.mask;
             env->psw.addr = regs->psw.addr;
     }
+#elif defined(TARGET_TILEGX)
+    {
+        int i;
+        for (i = 0; i < TILEGX_R_COUNT; i++) {
+            env->regs[i] = regs->regs[i];
+        }
+        for (i = 0; i < TILEGX_SPR_COUNT; i++) {
+            env->spregs[i] = 0;
+        }
+        env->pc = regs->pc;
+    }
 #else
 #error unsupported target CPU
 #endif
@@ -4401,7 +4694,7 @@ int main(int argc, char **argv, char **envp)
         if (gdbserver_start(gdbstub_port) < 0) {
             fprintf(stderr, "qemu: could not open gdbserver on port %d\n",
                     gdbstub_port);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         gdb_handlesig(cpu, 0);
     }
diff --git a/qemu/linux-user/microblaze/syscall_nr.h b/qemu/linux-user/microblaze/syscall_nr.h
index 6f530f9d1..0704449ba 100644
--- a/qemu/linux-user/microblaze/syscall_nr.h
+++ b/qemu/linux-user/microblaze/syscall_nr.h
@@ -382,3 +382,11 @@
 #define TARGET_NR_process_vm_writev     378
 #define TARGET_NR_kcmp                  379
 #define TARGET_NR_finit_module          380
+#define TARGET_NR_sched_setattr         381
+#define TARGET_NR_sched_getattr         382
+#define TARGET_NR_renameat2             383
+#define TARGET_NR_seccomp               384
+#define TARGET_NR_getrandom             385
+#define TARGET_NR_memfd_create          386
+#define TARGET_NR_bpf                   387
+#define TARGET_NR_execveat              388
diff --git a/qemu/linux-user/microblaze/syscall.h b/qemu/linux-user/microblaze/target_syscall.h
index 3c1ed27c0..3c1ed27c0 100644
--- a/qemu/linux-user/microblaze/syscall.h
+++ b/qemu/linux-user/microblaze/target_syscall.h
diff --git a/qemu/linux-user/mips/syscall_nr.h b/qemu/linux-user/mips/syscall_nr.h
index 2d1a13ee2..6819f865e 100644
--- a/qemu/linux-user/mips/syscall_nr.h
+++ b/qemu/linux-user/mips/syscall_nr.h
@@ -351,3 +351,15 @@
 #define TARGET_NR_process_vm_writev     (TARGET_NR_Linux + 346)
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 347)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 348)
+
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 349)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 350)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 351)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 352)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 353)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 354)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 355)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 356)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 357)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 358)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 359)
diff --git a/qemu/linux-user/mips/syscall.h b/qemu/linux-user/mips/target_syscall.h
index 35ca23b16..68db160e5 100644
--- a/qemu/linux-user/mips/syscall.h
+++ b/qemu/linux-user/mips/target_syscall.h
@@ -1,3 +1,5 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
 
 /* this struct defines the way the registers are stored on the
    stack during a system call. */
@@ -231,3 +233,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/mips64/syscall_nr.h b/qemu/linux-user/mips64/syscall_nr.h
index 004232a8a..746cc267e 100644
--- a/qemu/linux-user/mips64/syscall_nr.h
+++ b/qemu/linux-user/mips64/syscall_nr.h
@@ -316,6 +316,18 @@
 #define TARGET_NR_process_vm_writev     (TARGET_NR_Linux + 310)
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 311)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 312)
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 313)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 314)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 315)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 316)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 317)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 318)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 319)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 320)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 321)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 322)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 323)
+
 #else
 /*
  * Linux 64-bit syscalls are in the range from 5000 to 5999.
@@ -630,4 +642,15 @@
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 306)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 307)
 #define TARGET_NR_getdents64            (TARGET_NR_Linux + 308)
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 309)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 310)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 311)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 312)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 313)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 314)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 315)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 316)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 317)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 318)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 319)
 #endif
diff --git a/qemu/linux-user/mips64/syscall.h b/qemu/linux-user/mips64/target_syscall.h
index 6733107dd..0e0c2d232 100644
--- a/qemu/linux-user/mips64/syscall.h
+++ b/qemu/linux-user/mips64/target_syscall.h
@@ -1,3 +1,5 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
 
 /* this struct defines the way the registers are stored on the
    stack during a system call. */
@@ -228,3 +230,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ      2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/mmap.c b/qemu/linux-user/mmap.c
index 78e1b2df4..3519147bc 100644
--- a/qemu/linux-user/mmap.c
+++ b/qemu/linux-user/mmap.c
@@ -16,14 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/stat.h>
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 #include <linux/mman.h>
 #include <linux/unistd.h>
@@ -186,10 +179,12 @@ static int mmap_frag(abi_ulong real_start,
         if (prot_new != (prot1 | PROT_WRITE))
             mprotect(host_start, qemu_host_page_size, prot_new);
     } else {
-        /* just update the protection */
         if (prot_new != prot1) {
             mprotect(host_start, qemu_host_page_size, prot_new);
         }
+        if (prot_new & PROT_WRITE) {
+            memset(g2h(start), 0, end - start);
+        }
     }
     return 0;
 }
@@ -206,7 +201,6 @@ abi_ulong mmap_next_start = TASK_UNMAPPED_BASE;
 
 unsigned long last_brk;
 
-#ifdef CONFIG_USE_GUEST_BASE
 /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
    of guest address space.  */
 static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
@@ -216,14 +210,14 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
     int prot;
     int looped = 0;
 
-    if (size > RESERVED_VA) {
+    if (size > reserved_va) {
         return (abi_ulong)-1;
     }
 
     size = HOST_PAGE_ALIGN(size);
     end_addr = start + size;
-    if (end_addr > RESERVED_VA) {
-        end_addr = RESERVED_VA;
+    if (end_addr > reserved_va) {
+        end_addr = reserved_va;
     }
     addr = end_addr - qemu_host_page_size;
 
@@ -232,7 +226,7 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
             if (looped) {
                 return (abi_ulong)-1;
             }
-            end_addr = RESERVED_VA;
+            end_addr = reserved_va;
             addr = end_addr - qemu_host_page_size;
             looped = 1;
             continue;
@@ -253,7 +247,6 @@ static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
 
     return addr;
 }
-#endif
 
 /*
  * Find and reserve a free memory area of size 'size'. The search
@@ -276,11 +269,9 @@ abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size)
 
     size = HOST_PAGE_ALIGN(size);
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (RESERVED_VA) {
+    if (reserved_va) {
         return mmap_find_vma_reserved(start, size);
     }
-#endif
 
     addr = start;
     wrapped = repeat = 0;
@@ -448,9 +439,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
            /* If so, truncate the file map at eof aligned with 
               the hosts real pagesize. Additional anonymous maps
               will be created beyond EOF.  */
-           len = (sb.st_size - offset);
-           len += qemu_real_host_page_size - 1;
-           len &= ~(qemu_real_host_page_size - 1);
+           len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset);
        }
     }
 
@@ -518,10 +507,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
                 goto fail;
             if (!(prot & PROT_WRITE)) {
                 ret = target_mprotect(start, len, prot);
-                if (ret != 0) {
-                    start = ret;
-                    goto the_end;
-                }
+                assert(ret == 0);
             }
             goto the_end;
         }
@@ -545,7 +531,7 @@ abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
         /* handle the end of the mapping */
         if (end < real_end) {
             ret = mmap_frag(real_end - qemu_host_page_size,
-                            real_end - qemu_host_page_size, real_end,
+                            real_end - qemu_host_page_size, end,
                             prot, flags, fd,
                             offset + real_end - qemu_host_page_size - start);
             if (ret == -1)
@@ -671,7 +657,7 @@ int target_munmap(abi_ulong start, abi_ulong len)
     ret = 0;
     /* unmap what we can */
     if (real_start < real_end) {
-        if (RESERVED_VA) {
+        if (reserved_va) {
             mmap_reserve(real_start, real_end - real_start);
         } else {
             ret = munmap(g2h(real_start), real_end - real_start);
@@ -701,7 +687,7 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                                      flags,
                                      g2h(new_addr));
 
-        if (RESERVED_VA && host_addr != MAP_FAILED) {
+        if (reserved_va && host_addr != MAP_FAILED) {
             /* If new and old addresses overlap then the above mremap will
                already have failed with EINVAL.  */
             mmap_reserve(old_addr, old_size);
@@ -719,13 +705,13 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
                                          old_size, new_size,
                                          flags | MREMAP_FIXED,
                                          g2h(mmap_start));
-            if ( RESERVED_VA ) {
+            if (reserved_va) {
                 mmap_reserve(old_addr, old_size);
             }
         }
     } else {
         int prot = 0;
-        if (RESERVED_VA && old_size < new_size) {
+        if (reserved_va && old_size < new_size) {
             abi_ulong addr;
             for (addr = old_addr + old_size;
                  addr < old_addr + new_size;
@@ -735,7 +721,7 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
         }
         if (prot == 0) {
             host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
-            if (host_addr != MAP_FAILED && RESERVED_VA && old_size > new_size) {
+            if (host_addr != MAP_FAILED && reserved_va && old_size > new_size) {
                 mmap_reserve(old_addr + old_size, new_size - old_size);
             }
         } else {
diff --git a/qemu/linux-user/openrisc/syscall_nr.h b/qemu/linux-user/openrisc/syscall_nr.h
index 4c386eae9..6b1c7d265 100644
--- a/qemu/linux-user/openrisc/syscall_nr.h
+++ b/qemu/linux-user/openrisc/syscall_nr.h
@@ -382,9 +382,18 @@
 #define TARGET_NR_process_vm_writev 271
 #define TARGET_NR_kcmp 272
 #define TARGET_NR_finit_module 273
-
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls 274
+#define TARGET_NR_sched_setattr 274
+#define TARGET_NR_sched_getattr 275
+#define TARGET_NR_renameat2 276
+#define TARGET_NR_seccomp 277
+#define TARGET_NR_getrandom 278
+#define TARGET_NR_memfd_create 279
+#define TARGET_NR_bpf 280
+#define TARGET_NR_execveat 281
+#define TARGET_NR_userfaultfd 282
+#define TARGET_NR_membarrier 283
+#define TARGET_NR_mlock2 284
+#define TARGET_NR_copy_file_range 285
 
 /*
  * All syscalls below here should go away really,
@@ -411,9 +420,6 @@
 #define TARGET_NR_3264_stat 1038
 #define TARGET_NR_3264_lstat 1039
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_3264_lstat+1)
-
 #define TARGET_NR_pipe 1040
 #define TARGET_NR_dup2 1041
 #define TARGET_NR_epoll_create 1042
@@ -421,10 +427,6 @@
 #define TARGET_NR_eventfd 1044
 #define TARGET_NR_signalfd 1045
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_signalfd+1)
-
-
 #define TARGET_NR_sendfile 1046
 #define TARGET_NR_ftruncate 1047
 #define TARGET_NR_truncate 1048
@@ -441,9 +443,6 @@
 #define TARGET_NR_lseek 1057
 #define TARGET_NR_mmap 1058
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_mmap+1)
-
 #define TARGET_NR_alarm 1059
 #define __ARCH_WANT_SYS_ALARM
 #define TARGET_NR_getpgrp 1060
@@ -477,9 +476,6 @@
 
 #define TARGET_NR_fork 1079
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_fork+1)
-
 
 /*
  * 32 bit systems traditionally used different
diff --git a/qemu/linux-user/openrisc/syscall.h b/qemu/linux-user/openrisc/target_syscall.h
index 8ac03656d..19aeffc95 100644
--- a/qemu/linux-user/openrisc/syscall.h
+++ b/qemu/linux-user/openrisc/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
     union {
         struct {
@@ -27,3 +30,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/ppc/syscall_nr.h b/qemu/linux-user/ppc/syscall_nr.h
index 1e1736e11..46ed8a68c 100644
--- a/qemu/linux-user/ppc/syscall_nr.h
+++ b/qemu/linux-user/ppc/syscall_nr.h
@@ -319,7 +319,7 @@
 #define TARGET_NR_epoll_pwait	303
 #define TARGET_NR_utimensat		304
 #define TARGET_NR_signalfd		305
-#define TARGET_NR_timerfd		306
+#define TARGET_NR_timerfd_create	306
 #define TARGET_NR_eventfd		307
 #define TARGET_NR_sync_file_range2	308
 #define TARGET_NR_fallocate		309
@@ -368,3 +368,27 @@
 #define TARGET_NR_process_vm_writev     352
 #define TARGET_NR_finit_module          353
 #define TARGET_NR_kcmp                  354
+#define TARGET_NR_sched_setattr         355
+#define TARGET_NR_sched_getattr         356
+#define TARGET_NR_renameat2             357
+#define TARGET_NR_seccomp               358
+#define TARGET_NR_getrandom             359
+#define TARGET_NR_memfd_create          360
+#define TARGET_NR_bpf                   361
+#define TARGET_NR_execveat              362
+#define TARGET_NR_switch_endian         363
+#define TARGET_NR_userfaultfd           364
+#define TARGET_NR_membarrier            365
+#define TARGET_NR_semop                 366
+#define TARGET_NR_semget                367
+#define TARGET_NR_semctl                368
+#define TARGET_NR_semtimedop            369
+#define TARGET_NR_msgsnd                370
+#define TARGET_NR_msgrcv                371
+#define TARGET_NR_msgget                372
+#define TARGET_NR_msgctl                373
+#define TARGET_NR_shmat                 374
+#define TARGET_NR_shmdt                 375
+#define TARGET_NR_shmget                376
+#define TARGET_NR_shmctl                377
+#define TARGET_NR_mlock2                378
diff --git a/qemu/linux-user/ppc/syscall.h b/qemu/linux-user/ppc/target_syscall.h
index 0daf5cd2d..35cab5946 100644
--- a/qemu/linux-user/ppc/syscall.h
+++ b/qemu/linux-user/ppc/target_syscall.h
@@ -17,6 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 /* XXX: ABSOLUTELY BUGGY:
  * for now, this is quite just a cut-and-paste from i386 target...
  */
@@ -73,3 +76,5 @@ struct target_revectored_struct {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 0x2000
 #define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/qemu.h b/qemu/linux-user/qemu.h
index 8012cc2f5..26b0ba273 100644
--- a/qemu/linux-user/qemu.h
+++ b/qemu/linux-user/qemu.h
@@ -1,22 +1,19 @@
 #ifndef QEMU_H
 #define QEMU_H
 
-#include <signal.h>
-#include <string.h>
 
 #include "cpu.h"
 #include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
-#include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
 #include "exec/user/abitypes.h"
 
 #include "exec/user/thunk.h"
 #include "syscall_defs.h"
-#include "syscall.h"
+#include "target_syscall.h"
 #include "exec/gdbstub.h"
 #include "qemu/queue.h"
 
@@ -36,8 +33,6 @@ struct image_info {
         abi_ulong       start_brk;
         abi_ulong       brk;
         abi_ulong       start_mmap;
-        abi_ulong       mmap;
-        abi_ulong       rss;
         abi_ulong       start_stack;
         abi_ulong       stack_limit;
         abi_ulong       entry;
@@ -145,12 +140,6 @@ extern const char *qemu_uname_release;
 extern unsigned long mmap_min_addr;
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
-/*
- * MAX_ARG_PAGES defines the number of pages allocated for arguments
- * and envelope for the new program. 32 should suffice, this gives
- * a maximum env+arg of 128kB w/4KB pages!
- */
-#define MAX_ARG_PAGES 33
 
 /* Read a good amount of data initially, to hopefully get all the
    program headers loaded.  */
@@ -162,7 +151,6 @@ extern unsigned long mmap_min_addr;
  */
 struct linux_binprm {
         char buf[BPRM_BUF_SIZE] __attribute__((aligned));
-        void *page[MAX_ARG_PAGES];
         abi_ulong p;
 	int fd;
         int e_uid, e_gid;
@@ -261,8 +249,6 @@ abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
 extern abi_ulong mmap_next_start;
-void mmap_lock(void);
-void mmap_unlock(void);
 abi_ulong mmap_find_vma(abi_ulong, abi_ulong);
 void cpu_list_lock(void);
 void cpu_list_unlock(void);
diff --git a/qemu/linux-user/s390x/syscall_nr.h b/qemu/linux-user/s390x/syscall_nr.h
index 7c0b8b2ea..1a66c5561 100644
--- a/qemu/linux-user/s390x/syscall_nr.h
+++ b/qemu/linux-user/s390x/syscall_nr.h
@@ -271,6 +271,36 @@
 #define TARGET_NR_s390_runtime_instr    342
 #define TARGET_NR_kcmp                  343
 #define TARGET_NR_finit_module          344
+#define TARGET_NR_sched_setattr         345
+#define TARGET_NR_sched_getattr         346
+#define TARGET_NR_renameat2             347
+#define TARGET_NR_seccomp               348
+#define TARGET_NR_getrandom             349
+#define TARGET_NR_memfd_create          350
+#define TARGET_NR_bpf                   351
+#define TARGET_NR_s390_pci_mmio_write   352
+#define TARGET_NR_s390_pci_mmio_read    353
+#define TARGET_NR_execveat              354
+#define TARGET_NR_userfaultfd           355
+#define TARGET_NR_membarrier            356
+#define TARGET_NR_recvmmsg              357
+#define TARGET_NR_sendmmsg              358
+#define TARGET_NR_socket                359
+#define TARGET_NR_socketpair            360
+#define TARGET_NR_bind                  361
+#define TARGET_NR_connect               362
+#define TARGET_NR_listen                363
+#define TARGET_NR_accept4               364
+#define TARGET_NR_getsockopt            365
+#define TARGET_NR_setsockopt            366
+#define TARGET_NR_getsockname           367
+#define TARGET_NR_getpeername           368
+#define TARGET_NR_sendto                369
+#define TARGET_NR_sendmsg               370
+#define TARGET_NR_recvfrom              371
+#define TARGET_NR_recvmsg               372
+#define TARGET_NR_shutdown              373
+#define TARGET_NR_mlock2                374
 
 /*
  * There are some system calls that are not present on 64 bit, some
diff --git a/qemu/linux-user/s390x/syscall.h b/qemu/linux-user/s390x/target_syscall.h
index 35f170af2..02061efc7 100644
--- a/qemu/linux-user/s390x/syscall.h
+++ b/qemu/linux-user/s390x/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 /* this typedef defines how a Program Status Word looks like */
 typedef struct {
     abi_ulong mask;
@@ -27,3 +30,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ        2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/sh4/syscall_nr.h b/qemu/linux-user/sh4/syscall_nr.h
index bdf8742c6..50099846d 100644
--- a/qemu/linux-user/sh4/syscall_nr.h
+++ b/qemu/linux-user/sh4/syscall_nr.h
@@ -323,7 +323,7 @@
 #define TARGET_NR_epoll_pwait	319
 #define TARGET_NR_utimensat		320
 #define TARGET_NR_signalfd		321
-#define TARGET_NR_timerfd		322
+#define TARGET_NR_timerfd_create	322
 #define TARGET_NR_eventfd		323
 #define TARGET_NR_fallocate		324
 #define TARGET_NR_timerfd_settime	325
diff --git a/qemu/linux-user/sh4/syscall.h b/qemu/linux-user/sh4/target_syscall.h
index 7aa4f239c..9f3381bc9 100644
--- a/qemu/linux-user/sh4/syscall.h
+++ b/qemu/linux-user/sh4/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
         unsigned long regs[16];
         unsigned long pc;
@@ -15,3 +18,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/signal.c b/qemu/linux-user/signal.c
index 9d4cef409..96e86c0a2 100644
--- a/qemu/linux-user/signal.c
+++ b/qemu/linux-user/signal.c
@@ -16,21 +16,14 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <errno.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 #include <sys/ucontext.h>
 #include <sys/resource.h>
 
 #include "qemu.h"
 #include "qemu-common.h"
 #include "target_signal.h"
-
-//#define DEBUG_SIGNAL
+#include "trace.h"
 
 static struct target_sigaltstack target_sigaltstack_used = {
     .ss_sp = 0,
@@ -445,7 +438,9 @@ static void QEMU_NORETURN force_sig(int target_sig)
     TaskState *ts = (TaskState *)cpu->opaque;
     int host_sig, core_dumped = 0;
     struct sigaction act;
+
     host_sig = target_to_host_signal(target_sig);
+    trace_user_force_sig(env, target_sig, host_sig);
     gdb_signalled(env, target_sig);
 
     /* dump core if supported by target binary format */
@@ -500,10 +495,7 @@ int queue_signal(CPUArchState *env, int sig, target_siginfo_t *info)
     abi_ulong handler;
     int queue;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "queue_signal: sig=%d\n",
-            sig);
-#endif
+    trace_user_queue_signal(env, sig);
     k = &ts->sigtab[sig - 1];
     queue = gdb_queuesig ();
     handler = sigact_table[sig - 1]._sa_handler;
@@ -588,9 +580,7 @@ static void host_signal_handler(int host_signum, siginfo_t *info,
     sig = host_to_target_signal(host_signum);
     if (sig < 1 || sig > TARGET_NSIG)
         return;
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "qemu: got signal %d\n", sig);
-#endif
+    trace_user_host_signal(env, host_signum, sig);
     host_to_target_siginfo_noswap(&tinfo, info);
     if (queue_signal(env, sig, &tinfo) == 1) {
         /* interrupt the virtual CPU as soon as possible */
@@ -683,10 +673,6 @@ int do_sigaction(int sig, const struct target_sigaction *act,
     if (sig < 1 || sig > TARGET_NSIG || sig == TARGET_SIGKILL || sig == TARGET_SIGSTOP)
         return -EINVAL;
     k = &sigact_table[sig - 1];
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "sigaction sig=%d act=0x%p, oact=0x%p\n",
-            sig, act, oact);
-#endif
     if (oact) {
         __put_user(k->_sa_handler, &oact->_sa_handler);
         __put_user(k->sa_flags, &oact->sa_flags);
@@ -910,6 +896,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 	int i;
 
 	frame_addr = get_sigframe(ka, env, sizeof(*frame));
+        trace_user_setup_frame(env, frame_addr);
 
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 		goto give_sigsegv;
@@ -971,6 +958,7 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
 	int i;
 
 	frame_addr = get_sigframe(ka, env, sizeof(*frame));
+        trace_user_setup_rt_frame(env, frame_addr);
 
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 		goto give_sigsegv;
@@ -1082,9 +1070,7 @@ long do_sigreturn(CPUX86State *env)
     sigset_t set;
     int eax, i;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
     /* set blocked signals */
@@ -1116,6 +1102,7 @@ long do_rt_sigreturn(CPUX86State *env)
 	int eax;
 
         frame_addr = env->regs[R_ESP] - 4;
+        trace_user_do_rt_sigreturn(env, frame_addr);
         if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
                 goto badframe;
         target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
@@ -1319,6 +1306,7 @@ static void target_setup_frame(int usig, struct target_sigaction *ka,
     abi_ulong frame_addr, return_addr;
 
     frame_addr = get_sigframe(ka, env);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -1378,6 +1366,7 @@ long do_rt_sigreturn(CPUARMState *env)
     struct target_rt_sigframe *frame = NULL;
     abi_ulong frame_addr = env->xregs[31];
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (frame_addr & 15) {
         goto badframe;
     }
@@ -1547,82 +1536,84 @@ static void
 setup_sigcontext(struct target_sigcontext *sc, /*struct _fpstate *fpstate,*/
                  CPUARMState *env, abi_ulong mask)
 {
-	__put_user(env->regs[0], &sc->arm_r0);
-	__put_user(env->regs[1], &sc->arm_r1);
-	__put_user(env->regs[2], &sc->arm_r2);
-	__put_user(env->regs[3], &sc->arm_r3);
-	__put_user(env->regs[4], &sc->arm_r4);
-	__put_user(env->regs[5], &sc->arm_r5);
-	__put_user(env->regs[6], &sc->arm_r6);
-	__put_user(env->regs[7], &sc->arm_r7);
-	__put_user(env->regs[8], &sc->arm_r8);
-	__put_user(env->regs[9], &sc->arm_r9);
-	__put_user(env->regs[10], &sc->arm_r10);
-	__put_user(env->regs[11], &sc->arm_fp);
-	__put_user(env->regs[12], &sc->arm_ip);
-	__put_user(env->regs[13], &sc->arm_sp);
-	__put_user(env->regs[14], &sc->arm_lr);
-	__put_user(env->regs[15], &sc->arm_pc);
+    __put_user(env->regs[0], &sc->arm_r0);
+    __put_user(env->regs[1], &sc->arm_r1);
+    __put_user(env->regs[2], &sc->arm_r2);
+    __put_user(env->regs[3], &sc->arm_r3);
+    __put_user(env->regs[4], &sc->arm_r4);
+    __put_user(env->regs[5], &sc->arm_r5);
+    __put_user(env->regs[6], &sc->arm_r6);
+    __put_user(env->regs[7], &sc->arm_r7);
+    __put_user(env->regs[8], &sc->arm_r8);
+    __put_user(env->regs[9], &sc->arm_r9);
+    __put_user(env->regs[10], &sc->arm_r10);
+    __put_user(env->regs[11], &sc->arm_fp);
+    __put_user(env->regs[12], &sc->arm_ip);
+    __put_user(env->regs[13], &sc->arm_sp);
+    __put_user(env->regs[14], &sc->arm_lr);
+    __put_user(env->regs[15], &sc->arm_pc);
 #ifdef TARGET_CONFIG_CPU_32
-	__put_user(cpsr_read(env), &sc->arm_cpsr);
+    __put_user(cpsr_read(env), &sc->arm_cpsr);
 #endif
 
-	__put_user(/* current->thread.trap_no */ 0, &sc->trap_no);
-	__put_user(/* current->thread.error_code */ 0, &sc->error_code);
-	__put_user(/* current->thread.address */ 0, &sc->fault_address);
-	__put_user(mask, &sc->oldmask);
+    __put_user(/* current->thread.trap_no */ 0, &sc->trap_no);
+    __put_user(/* current->thread.error_code */ 0, &sc->error_code);
+    __put_user(/* current->thread.address */ 0, &sc->fault_address);
+    __put_user(mask, &sc->oldmask);
 }
 
 static inline abi_ulong
 get_sigframe(struct target_sigaction *ka, CPUARMState *regs, int framesize)
 {
-	unsigned long sp = regs->regs[13];
+    unsigned long sp = regs->regs[13];
 
-	/*
-	 * This is the X/Open sanctioned signal stack switching.
-	 */
-	if ((ka->sa_flags & TARGET_SA_ONSTACK) && !sas_ss_flags(sp))
-            sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
-	/*
-	 * ATPCS B01 mandates 8-byte alignment
-	 */
-	return (sp - framesize) & ~7;
+    /*
+     * This is the X/Open sanctioned signal stack switching.
+     */
+    if ((ka->sa_flags & TARGET_SA_ONSTACK) && !sas_ss_flags(sp)) {
+        sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    }
+    /*
+     * ATPCS B01 mandates 8-byte alignment
+     */
+    return (sp - framesize) & ~7;
 }
 
 static void
 setup_return(CPUARMState *env, struct target_sigaction *ka,
 	     abi_ulong *rc, abi_ulong frame_addr, int usig, abi_ulong rc_addr)
 {
-	abi_ulong handler = ka->_sa_handler;
-	abi_ulong retcode;
-	int thumb = handler & 1;
-	uint32_t cpsr = cpsr_read(env);
+    abi_ulong handler = ka->_sa_handler;
+    abi_ulong retcode;
+    int thumb = handler & 1;
+    uint32_t cpsr = cpsr_read(env);
 
-	cpsr &= ~CPSR_IT;
-	if (thumb) {
-		cpsr |= CPSR_T;
-	} else {
-		cpsr &= ~CPSR_T;
-	}
+    cpsr &= ~CPSR_IT;
+    if (thumb) {
+        cpsr |= CPSR_T;
+    } else {
+        cpsr &= ~CPSR_T;
+    }
 
-	if (ka->sa_flags & TARGET_SA_RESTORER) {
-		retcode = ka->sa_restorer;
-	} else {
-		unsigned int idx = thumb;
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
+        retcode = ka->sa_restorer;
+    } else {
+        unsigned int idx = thumb;
 
-		if (ka->sa_flags & TARGET_SA_SIGINFO)
-			idx += 2;
+        if (ka->sa_flags & TARGET_SA_SIGINFO) {
+            idx += 2;
+        }
 
         __put_user(retcodes[idx], rc);
 
-		retcode = rc_addr + thumb;
-	}
+        retcode = rc_addr + thumb;
+    }
 
-	env->regs[0] = usig;
-	env->regs[13] = frame_addr;
-	env->regs[14] = retcode;
-	env->regs[15] = handler & (thumb ? ~1 : ~3);
-	cpsr_write(env, cpsr, 0xffffffff);
+    env->regs[0] = usig;
+    env->regs[13] = frame_addr;
+    env->regs[14] = retcode;
+    env->regs[15] = handler & (thumb ? ~1 : ~3);
+    cpsr_write(env, cpsr, CPSR_IT | CPSR_T, CPSRWriteByInstr);
 }
 
 static abi_ulong *setup_sigframe_v2_vfp(abi_ulong *regspace, CPUARMState *env)
@@ -1704,6 +1695,7 @@ static void setup_frame_v1(int usig, struct target_sigaction *ka,
 	abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
 	int i;
 
+        trace_user_setup_frame(regs, frame_addr);
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 		return;
 
@@ -1725,6 +1717,7 @@ static void setup_frame_v2(int usig, struct target_sigaction *ka,
 	struct sigframe_v2 *frame;
 	abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
 
+        trace_user_setup_frame(regs, frame_addr);
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 		return;
 
@@ -1757,6 +1750,7 @@ static void setup_rt_frame_v1(int usig, struct target_sigaction *ka,
 	int i;
         abi_ulong info_addr, uc_addr;
 
+        trace_user_setup_rt_frame(env, frame_addr);
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
             return /* 1 */;
 
@@ -1797,6 +1791,7 @@ static void setup_rt_frame_v2(int usig, struct target_sigaction *ka,
 	abi_ulong frame_addr = get_sigframe(ka, env, sizeof(*frame));
         abi_ulong info_addr, uc_addr;
 
+        trace_user_setup_rt_frame(env, frame_addr);
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
             return /* 1 */;
 
@@ -1850,7 +1845,7 @@ restore_sigcontext(CPUARMState *env, struct target_sigcontext *sc)
     __get_user(env->regs[15], &sc->arm_pc);
 #ifdef TARGET_CONFIG_CPU_32
     __get_user(cpsr, &sc->arm_cpsr);
-        cpsr_write(env, cpsr, CPSR_USER | CPSR_EXEC);
+    cpsr_write(env, cpsr, CPSR_USER | CPSR_EXEC, CPSRWriteByInstr);
 #endif
 
 	err |= !valid_user_regs(env);
@@ -1872,6 +1867,7 @@ static long do_sigreturn_v1(CPUARMState *env)
 	 * not, then the user is trying to mess with us.
 	 */
         frame_addr = env->regs[13];
+        trace_user_do_sigreturn(env, frame_addr);
         if (frame_addr & 7) {
             goto badframe;
         }
@@ -2008,6 +2004,7 @@ static long do_sigreturn_v2(CPUARMState *env)
 	 * not, then the user is trying to mess with us.
 	 */
         frame_addr = env->regs[13];
+        trace_user_do_sigreturn(env, frame_addr);
         if (frame_addr & 7) {
             goto badframe;
         }
@@ -2048,6 +2045,7 @@ static long do_rt_sigreturn_v1(CPUARMState *env)
 	 * not, then the user is trying to mess with us.
 	 */
         frame_addr = env->regs[13];
+        trace_user_do_rt_sigreturn(env, frame_addr);
         if (frame_addr & 7) {
             goto badframe;
         }
@@ -2089,6 +2087,7 @@ static long do_rt_sigreturn_v2(CPUARMState *env)
 	 * not, then the user is trying to mess with us.
 	 */
         frame_addr = env->regs[13];
+        trace_user_do_rt_sigreturn(env, frame_addr);
         if (frame_addr & 7) {
             goto badframe;
         }
@@ -2284,13 +2283,13 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
         sigframe_size = NF_ALIGNEDSZ;
 	sf_addr = get_sigframe(ka, env, sigframe_size);
+        trace_user_setup_frame(env, sf_addr);
 
         sf = lock_user(VERIFY_WRITE, sf_addr, 
                        sizeof(struct target_signal_frame), 0);
         if (!sf)
 		goto sigsegv;
                 
-	//fprintf(stderr, "sf: %x pc %x fp %x sp %x\n", sf, env->pc, env->regwptr[UREG_FP], env->regwptr[UREG_SP]);
 #if 0
 	if (invalid_frame_pointer(sf, sigframe_size))
 		goto sigill_and_return;
@@ -2357,7 +2356,6 @@ sigill_and_return:
 	force_sig(TARGET_SIGILL);
 #endif
 sigsegv:
-	//fprintf(stderr, "force_sig\n");
         unlock_user(sf, sf_addr, sizeof(struct target_signal_frame));
 	force_sig(TARGET_SIGSEGV);
 }
@@ -2379,13 +2377,9 @@ long do_sigreturn(CPUSPARCState *env)
         int err=0, i;
 
         sf_addr = env->regwptr[UREG_FP];
+        trace_user_do_sigreturn(env, sf_addr);
         if (!lock_user_struct(VERIFY_READ, sf, sf_addr, 1))
                 goto segv_and_exit;
-#if 0
-	fprintf(stderr, "sigreturn\n");
-	fprintf(stderr, "sf: %x pc %x fp %x sp %x\n", sf, env->pc, env->regwptr[UREG_FP], env->regwptr[UREG_SP]);
-#endif
-	//cpu_dump_state(env, stderr, fprintf, 0);
 
         /* 1. Make sure we are not getting garbage from the user */
 
@@ -2444,6 +2438,7 @@ segv_and_exit:
 
 long do_rt_sigreturn(CPUSPARCState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -2903,6 +2898,7 @@ static void setup_frame(int sig, struct target_sigaction * ka,
     int i;
 
     frame_addr = get_sigframe(ka, regs, sizeof(*frame));
+    trace_user_setup_frame(regs, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -2949,10 +2945,8 @@ long do_sigreturn(CPUMIPSState *regs)
     target_sigset_t target_set;
     int i;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
     frame_addr = regs->active_tc.gpr[29];
+    trace_user_do_sigreturn(regs, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
    	goto badframe;
 
@@ -2999,6 +2993,7 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -3056,10 +3051,8 @@ long do_rt_sigreturn(CPUMIPSState *env)
     abi_ulong frame_addr;
     sigset_t blocked;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_rt_sigreturn\n");
-#endif
     frame_addr = env->active_tc.gpr[29];
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
    	goto badframe;
 
@@ -3214,9 +3207,9 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     struct target_sigframe *frame;
     abi_ulong frame_addr;
     int i;
-    int err = 0;
 
     frame_addr = get_sigframe(ka, regs->gregs[15], sizeof(*frame));
+    trace_user_setup_frame(regs, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -3232,15 +3225,14 @@ static void setup_frame(int sig, struct target_sigaction *ka,
         regs->pr = (unsigned long) ka->sa_restorer;
     } else {
         /* Generate return code (system call to sigreturn) */
+        abi_ulong retcode_addr = frame_addr +
+                                 offsetof(struct target_sigframe, retcode);
         __put_user(MOVW(2), &frame->retcode[0]);
         __put_user(TRAP_NOARG, &frame->retcode[1]);
         __put_user((TARGET_NR_sigreturn), &frame->retcode[2]);
-        regs->pr = (unsigned long) frame->retcode;
+        regs->pr = (unsigned long) retcode_addr;
     }
 
-    if (err)
-        goto give_sigsegv;
-
     /* Set up registers for signal handler */
     regs->gregs[15] = frame_addr;
     regs->gregs[4] = sig; /* Arg for signal handler */
@@ -3263,9 +3255,9 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     struct target_rt_sigframe *frame;
     abi_ulong frame_addr;
     int i;
-    int err = 0;
 
     frame_addr = get_sigframe(ka, regs->gregs[15], sizeof(*frame));
+    trace_user_setup_rt_frame(regs, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -3292,15 +3284,14 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
         regs->pr = (unsigned long) ka->sa_restorer;
     } else {
         /* Generate return code (system call to sigreturn) */
+        abi_ulong retcode_addr = frame_addr +
+                                 offsetof(struct target_rt_sigframe, retcode);
         __put_user(MOVW(2), &frame->retcode[0]);
         __put_user(TRAP_NOARG, &frame->retcode[1]);
         __put_user((TARGET_NR_rt_sigreturn), &frame->retcode[2]);
-        regs->pr = (unsigned long) frame->retcode;
+        regs->pr = (unsigned long) retcode_addr;
     }
 
-    if (err)
-        goto give_sigsegv;
-
     /* Set up registers for signal handler */
     regs->gregs[15] = frame_addr;
     regs->gregs[4] = sig; /* Arg for signal handler */
@@ -3326,10 +3317,8 @@ long do_sigreturn(CPUSH4State *regs)
     int i;
     int err = 0;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
     frame_addr = regs->gregs[15];
+    trace_user_do_sigreturn(regs, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
    	goto badframe;
 
@@ -3362,10 +3351,8 @@ long do_rt_sigreturn(CPUSH4State *regs)
     sigset_t blocked;
     target_ulong r0;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_rt_sigreturn\n");
-#endif
     frame_addr = regs->gregs[15];
+    trace_user_do_rt_sigreturn(regs, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
    	goto badframe;
 
@@ -3515,6 +3502,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
         goto badframe;
 
@@ -3580,6 +3568,7 @@ long do_sigreturn(CPUMBState *env)
     int i;
 
     frame_addr = env->regs[R_SP];
+    trace_user_do_sigreturn(env, frame_addr);
     /* Make sure the guest isn't playing games.  */
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
         goto badframe;
@@ -3605,6 +3594,7 @@ long do_sigreturn(CPUMBState *env)
 
 long do_rt_sigreturn(CPUMBState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "Microblaze do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -3694,6 +3684,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 	int i;
 
 	frame_addr = get_sigframe(env, sizeof *frame);
+        trace_user_setup_frame(env, frame_addr);
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 		goto badframe;
 
@@ -3747,6 +3738,7 @@ long do_sigreturn(CPUCRISState *env)
 	int i;
 
 	frame_addr = env->regs[R_SP];
+        trace_user_do_sigreturn(env, frame_addr);
 	/* Make sure the guest isn't playing games.  */
 	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
 		goto badframe;
@@ -3768,6 +3760,7 @@ long do_sigreturn(CPUCRISState *env)
 
 long do_rt_sigreturn(CPUCRISState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "CRIS do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -3901,12 +3894,6 @@ static inline abi_ulong get_sigframe(struct target_sigaction *ka,
     return sp;
 }
 
-static void setup_frame(int sig, struct target_sigaction *ka,
-                        target_sigset_t *set, CPUOpenRISCState *env)
-{
-    qemu_log("Not implement.\n");
-}
-
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_siginfo_t *info,
                            target_sigset_t *set, CPUOpenRISCState *env)
@@ -3918,6 +3905,7 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     abi_ulong info_addr, uc_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -3979,14 +3967,15 @@ give_sigsegv:
 
 long do_sigreturn(CPUOpenRISCState *env)
 {
-
-    qemu_log("do_sigreturn: not implemented\n");
+    trace_user_do_sigreturn(env, 0);
+    fprintf(stderr, "do_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
 
 long do_rt_sigreturn(CPUOpenRISCState *env)
 {
-    qemu_log("do_rt_sigreturn: not implemented\n");
+    trace_user_do_rt_sigreturn(env, 0);
+    fprintf(stderr, "do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
 /* TARGET_OPENRISC */
@@ -4109,13 +4098,11 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     abi_ulong frame_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
             goto give_sigsegv;
     }
 
-    qemu_log("%s: 1\n", __FUNCTION__);
     __put_user(set->sig[0], &frame->sc.oldmask[0]);
 
     save_sigregs(env, &frame->sregs);
@@ -4156,7 +4143,6 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     return;
 
 give_sigsegv:
-    qemu_log("%s: give_sigsegv\n", __FUNCTION__);
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4169,13 +4155,11 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     abi_ulong frame_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
 
-    qemu_log("%s: 1\n", __FUNCTION__);
     tswap_siginfo(&frame->info, info);
 
     /* Create the ucontext.  */
@@ -4214,7 +4198,6 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     return;
 
 give_sigsegv:
-    qemu_log("%s: give_sigsegv\n", __FUNCTION__);
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4229,9 +4212,8 @@ restore_sigregs(CPUS390XState *env, target_sigregs *sc)
     }
 
     __get_user(env->psw.mask, &sc->regs.psw.mask);
-    qemu_log("%s: sc->regs.psw.addr 0x%llx env->psw.addr 0x%llx\n",
-             __FUNCTION__, (unsigned long long)sc->regs.psw.addr,
-             (unsigned long long)env->psw.addr);
+    trace_user_s390x_restore_sigregs(env, (unsigned long long)sc->regs.psw.addr,
+                                     (unsigned long long)env->psw.addr);
     __get_user(env->psw.addr, &sc->regs.psw.addr);
     /* FIXME: 31-bit -> | PSW_ADDR_AMODE */
 
@@ -4249,11 +4231,10 @@ long do_sigreturn(CPUS390XState *env)
 {
     sigframe *frame;
     abi_ulong frame_addr = env->regs[15];
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
     target_sigset_t target_set;
     sigset_t set;
 
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
@@ -4278,10 +4259,9 @@ long do_rt_sigreturn(CPUS390XState *env)
 {
     rt_sigframe *frame;
     abi_ulong frame_addr = env->regs[15];
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
     sigset_t set;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
@@ -4666,6 +4646,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 #endif
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
         goto sigsegv;
     sc = &frame->sctx;
@@ -4730,7 +4711,6 @@ static void setup_frame(int sig, struct target_sigaction *ka,
 
 sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
-    qemu_log("segfaulting from setup_frame\n");
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4826,7 +4806,6 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
 
 sigsegv:
     unlock_user_struct(rt_sf, rt_sf_addr, 1);
-    qemu_log("segfaulting from setup_rt_frame\n");
     force_sig(TARGET_SIGSEGV);
 
 }
@@ -4864,7 +4843,6 @@ long do_sigreturn(CPUPPCState *env)
 sigsegv:
     unlock_user_struct(sr, sr_addr, 1);
     unlock_user_struct(sc, sc_addr, 1);
-    qemu_log("segfaulting from do_sigreturn\n");
     force_sig(TARGET_SIGSEGV);
     return 0;
 }
@@ -4920,7 +4898,6 @@ long do_rt_sigreturn(CPUPPCState *env)
 
 sigsegv:
     unlock_user_struct(rt_sf, rt_sf_addr, 1);
-    qemu_log("segfaulting from do_rt_sigreturn\n");
     force_sig(TARGET_SIGSEGV);
     return 0;
 }
@@ -5044,6 +5021,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -5160,6 +5138,7 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
 	goto give_sigsegv;
 
@@ -5227,6 +5206,7 @@ long do_sigreturn(CPUM68KState *env)
     sigset_t set;
     int d0, i;
 
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
 
@@ -5261,6 +5241,7 @@ long do_rt_sigreturn(CPUM68KState *env)
     sigset_t set;
     int d0;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
 
@@ -5400,6 +5381,7 @@ static void setup_frame(int sig, struct target_sigaction *ka,
     int err = 0;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -5444,6 +5426,7 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
     int i, err = 0;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -5522,6 +5505,7 @@ long do_rt_sigreturn(CPUAlphaState *env)
     struct target_rt_sigframe *frame;
     sigset_t set;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
@@ -5544,6 +5528,165 @@ long do_rt_sigreturn(CPUAlphaState *env)
     force_sig(TARGET_SIGSEGV);
 }
 
+#elif defined(TARGET_TILEGX)
+
+struct target_sigcontext {
+    union {
+        /* General-purpose registers.  */
+        abi_ulong gregs[56];
+        struct {
+            abi_ulong __gregs[53];
+            abi_ulong tp;        /* Aliases gregs[TREG_TP].  */
+            abi_ulong sp;        /* Aliases gregs[TREG_SP].  */
+            abi_ulong lr;        /* Aliases gregs[TREG_LR].  */
+        };
+    };
+    abi_ulong pc;        /* Program counter.  */
+    abi_ulong ics;       /* In Interrupt Critical Section?  */
+    abi_ulong faultnum;  /* Fault number.  */
+    abi_ulong pad[5];
+};
+
+struct target_ucontext {
+    abi_ulong tuc_flags;
+    abi_ulong tuc_link;
+    target_stack_t tuc_stack;
+    struct target_sigcontext tuc_mcontext;
+    target_sigset_t tuc_sigmask;   /* mask last for extensibility */
+};
+
+struct target_rt_sigframe {
+    unsigned char save_area[16]; /* caller save area */
+    struct target_siginfo info;
+    struct target_ucontext uc;
+};
+
+static void setup_sigcontext(struct target_sigcontext *sc,
+                             CPUArchState *env, int signo)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __put_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __put_user(env->pc, &sc->pc);
+    __put_user(0, &sc->ics);
+    __put_user(signo, &sc->faultnum);
+}
+
+static void restore_sigcontext(CPUTLGState *env, struct target_sigcontext *sc)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __get_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __get_user(env->pc, &sc->pc);
+}
+
+static abi_ulong get_sigframe(struct target_sigaction *ka, CPUArchState *env,
+                              size_t frame_size)
+{
+    unsigned long sp = env->regs[TILEGX_R_SP];
+
+    if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) {
+        return -1UL;
+    }
+
+    if ((ka->sa_flags & SA_ONSTACK) && !sas_ss_flags(sp)) {
+        sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    }
+
+    sp -= frame_size;
+    sp &= -16UL;
+    return sp;
+}
+
+static void setup_rt_frame(int sig, struct target_sigaction *ka,
+                           target_siginfo_t *info,
+                           target_sigset_t *set, CPUArchState *env)
+{
+    abi_ulong frame_addr;
+    struct target_rt_sigframe *frame;
+    unsigned long restorer;
+
+    frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
+
+    /* Always write at least the signal number for the stack backtracer. */
+    if (ka->sa_flags & TARGET_SA_SIGINFO) {
+        /* At sigreturn time, restore the callee-save registers too. */
+        tswap_siginfo(&frame->info, info);
+        /* regs->flags |= PT_FLAGS_RESTORE_REGS; FIXME: we can skip it? */
+    } else {
+        __put_user(info->si_signo, &frame->info.si_signo);
+    }
+
+    /* Create the ucontext.  */
+    __put_user(0, &frame->uc.tuc_flags);
+    __put_user(0, &frame->uc.tuc_link);
+    __put_user(target_sigaltstack_used.ss_sp, &frame->uc.tuc_stack.ss_sp);
+    __put_user(sas_ss_flags(env->regs[TILEGX_R_SP]),
+               &frame->uc.tuc_stack.ss_flags);
+    __put_user(target_sigaltstack_used.ss_size, &frame->uc.tuc_stack.ss_size);
+    setup_sigcontext(&frame->uc.tuc_mcontext, env, info->si_signo);
+
+    restorer = (unsigned long) do_rt_sigreturn;
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
+            restorer = (unsigned long) ka->sa_restorer;
+    }
+    env->pc = (unsigned long) ka->_sa_handler;
+    env->regs[TILEGX_R_SP] = (unsigned long) frame;
+    env->regs[TILEGX_R_LR] = restorer;
+    env->regs[0] = (unsigned long) sig;
+    env->regs[1] = (unsigned long) &frame->info;
+    env->regs[2] = (unsigned long) &frame->uc;
+    /* regs->flags |= PT_FLAGS_CALLER_SAVES; FIXME: we can skip it? */
+
+    unlock_user_struct(frame, frame_addr, 1);
+    return;
+
+give_sigsegv:
+    if (sig == TARGET_SIGSEGV) {
+        ka->_sa_handler = TARGET_SIG_DFL;
+    }
+    force_sig(TARGET_SIGSEGV /* , current */);
+}
+
+long do_rt_sigreturn(CPUTLGState *env)
+{
+    abi_ulong frame_addr = env->regs[TILEGX_R_SP];
+    struct target_rt_sigframe *frame;
+    sigset_t set;
+
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
+    target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
+    do_sigprocmask(SIG_SETMASK, &set, NULL);
+
+    restore_sigcontext(env, &frame->uc.tuc_mcontext);
+    if (do_sigaltstack(frame_addr + offsetof(struct target_rt_sigframe,
+                                             uc.tuc_stack),
+                       0, env->regs[TILEGX_R_SP]) == -EFAULT) {
+        goto badframe;
+    }
+
+    unlock_user_struct(frame, frame_addr, 0);
+    return env->regs[TILEGX_R_RE];
+
+
+ badframe:
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV);
+}
+
 #else
 
 static void setup_frame(int sig, struct target_sigaction *ka,
@@ -5600,9 +5743,7 @@ void process_pending_signals(CPUArchState *cpu_env)
     return;
 
  handle_signal:
-#ifdef DEBUG_SIGNAL
-    fprintf(stderr, "qemu: process signal %d\n", sig);
-#endif
+    trace_user_handle_signal(cpu_env, sig);
     /* dequeue signal */
     q = k->first;
     k->first = q->next;
@@ -5663,7 +5804,8 @@ void process_pending_signals(CPUArchState *cpu_env)
         }
 #endif
         /* prepare the stack frame of the virtual CPU */
-#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64)
+#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
         /* These targets do not have traditional signals.  */
         setup_rt_frame(sig, sa, &q->info, &target_old_set, cpu_env);
 #else
diff --git a/qemu/linux-user/sparc/syscall_nr.h b/qemu/linux-user/sparc/syscall_nr.h
index 181cd3265..732b1052a 100644
--- a/qemu/linux-user/sparc/syscall_nr.h
+++ b/qemu/linux-user/sparc/syscall_nr.h
@@ -278,7 +278,7 @@
 #define TARGET_NR_epoll_pwait	309
 #define TARGET_NR_utimensat		310
 #define TARGET_NR_signalfd		311
-#define TARGET_NR_timerfd		312
+#define TARGET_NR_timerfd_create	312
 #define TARGET_NR_eventfd		313
 #define TARGET_NR_fallocate		314
 #define TARGET_NR_timerfd_settime	315
@@ -309,3 +309,17 @@
 #define TARGET_NR_kern_features         340
 #define TARGET_NR_kcmp                  341
 #define TARGET_NR_finit_module          342
+#define TARGET_NR_sched_setattr         343
+#define TARGET_NR_sched_getattr         344
+#define TARGET_NR_renameat2             345
+#define TARGET_NR_seccomp               346
+#define TARGET_NR_getrandom             347
+#define TARGET_NR_memfd_create          348
+#define TARGET_NR_bpf                   349
+#define TARGET_NR_execveat              350
+#define TARGET_NR_membarrier            351
+#define TARGET_NR_userfaultfd           352
+#define TARGET_NR_bind                  353
+#define TARGET_NR_listen                354
+#define TARGET_NR_setsockopt            355
+#define TARGET_NR_mlock2                356
diff --git a/qemu/linux-user/sparc/syscall.h b/qemu/linux-user/sparc/target_syscall.h
index 58573b92e..a73fa6dae 100644
--- a/qemu/linux-user/sparc/syscall.h
+++ b/qemu/linux-user/sparc/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
 	abi_ulong psr;
 	abi_ulong pc;
@@ -18,3 +21,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ      4096
 #define TARGET_MLOCKALL_MCL_CURRENT 0x2000
 #define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/sparc64/syscall_nr.h b/qemu/linux-user/sparc64/syscall_nr.h
index 34a984cc8..2b49ead26 100644
--- a/qemu/linux-user/sparc64/syscall_nr.h
+++ b/qemu/linux-user/sparc64/syscall_nr.h
@@ -341,3 +341,17 @@
 #define TARGET_NR_kern_features         340
 #define TARGET_NR_kcmp                  341
 #define TARGET_NR_finit_module          342
+#define TARGET_NR_sched_setattr         343
+#define TARGET_NR_sched_getattr         344
+#define TARGET_NR_renameat2             345
+#define TARGET_NR_seccomp               346
+#define TARGET_NR_getrandom             347
+#define TARGET_NR_memfd_create          348
+#define TARGET_NR_bpf                   349
+#define TARGET_NR_execveat              350
+#define TARGET_NR_membarrier            351
+#define TARGET_NR_userfaultfd           352
+#define TARGET_NR_bind                  353
+#define TARGET_NR_listen                354
+#define TARGET_NR_setsockopt            355
+#define TARGET_NR_mlock2                356
diff --git a/qemu/linux-user/sparc64/syscall.h b/qemu/linux-user/sparc64/target_syscall.h
index 8398d3f46..eb827fcac 100644
--- a/qemu/linux-user/sparc64/syscall.h
+++ b/qemu/linux-user/sparc64/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 struct target_pt_regs {
 	abi_ulong u_regs[16];
 	abi_ulong tstate;
@@ -19,3 +22,5 @@ struct target_pt_regs {
 #define TARGET_MINSIGSTKSZ      4096
 #define TARGET_MLOCKALL_MCL_CURRENT 0x2000
 #define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/linux-user/strace.c b/qemu/linux-user/strace.c
index ea6c1d24e..0810c85fb 100644
--- a/qemu/linux-user/strace.c
+++ b/qemu/linux-user/strace.c
@@ -1,13 +1,11 @@
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <sys/ipc.h>
 #include <sys/msg.h>
 #include <sys/sem.h>
 #include <sys/shm.h>
 #include <sys/select.h>
-#include <sys/types.h>
 #include <sys/mount.h>
 #include <sys/mman.h>
-#include <unistd.h>
 #include <sched.h>
 #include "qemu.h"
 
diff --git a/qemu/linux-user/syscall.c b/qemu/linux-user/syscall.c
index f62c69894..032d33886 100644
--- a/qemu/linux-user/syscall.c
+++ b/qemu/linux-user/syscall.c
@@ -17,24 +17,15 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #define _ATFILE_SOURCE
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 #include <elf.h>
 #include <endian.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <limits.h>
 #include <grp.h>
-#include <sys/types.h>
 #include <sys/ipc.h>
 #include <sys/msg.h>
 #include <sys/wait.h>
-#include <sys/time.h>
-#include <sys/stat.h>
 #include <sys/mount.h>
 #include <sys/file.h>
 #include <sys/fsuid.h>
@@ -44,7 +35,6 @@
 #include <sys/mman.h>
 #include <sys/swap.h>
 #include <linux/capability.h>
-#include <signal.h>
 #include <sched.h>
 #ifdef __ia64__
 int __clone2(int (*fn)(void *), void *child_stack_base,
@@ -60,6 +50,7 @@ int __clone2(int (*fn)(void *), void *child_stack_base,
 #include <sys/statfs.h>
 #include <utime.h>
 #include <sys/sysinfo.h>
+#include <sys/signalfd.h>
 //#include <sys/user.h>
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
@@ -213,7 +204,7 @@ static int gettid(void) {
     return -ENOSYS;
 }
 #endif
-#ifdef __NR_getdents
+#if defined(TARGET_NR_getdents) && defined(__NR_getdents)
 _syscall3(int, sys_getdents, uint, fd, struct linux_dirent *, dirp, uint, count);
 #endif
 #if !defined(__NR_getdents) || \
@@ -260,6 +251,9 @@ _syscall2(int, ioprio_get, int, which, int, who)
 #if defined(TARGET_NR_ioprio_set) && defined(__NR_ioprio_set)
 _syscall3(int, ioprio_set, int, which, int, who, int, ioprio)
 #endif
+#if defined(TARGET_NR_getrandom) && defined(__NR_getrandom)
+_syscall3(int, getrandom, void *, buf, size_t, buflen, unsigned int, flags)
+#endif
 
 static bitmask_transtbl fcntl_flags_tbl[] = {
   { TARGET_O_ACCMODE,   TARGET_O_WRONLY,    O_ACCMODE,   O_WRONLY,    },
@@ -294,6 +288,64 @@ static bitmask_transtbl fcntl_flags_tbl[] = {
   { 0, 0, 0, 0 }
 };
 
+typedef abi_long (*TargetFdDataFunc)(void *, size_t);
+typedef abi_long (*TargetFdAddrFunc)(void *, abi_ulong, socklen_t);
+typedef struct TargetFdTrans {
+    TargetFdDataFunc host_to_target_data;
+    TargetFdDataFunc target_to_host_data;
+    TargetFdAddrFunc target_to_host_addr;
+} TargetFdTrans;
+
+static TargetFdTrans **target_fd_trans;
+
+static unsigned int target_fd_max;
+
+static TargetFdDataFunc fd_trans_host_to_target_data(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max && target_fd_trans[fd]) {
+        return target_fd_trans[fd]->host_to_target_data;
+    }
+    return NULL;
+}
+
+static TargetFdAddrFunc fd_trans_target_to_host_addr(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max && target_fd_trans[fd]) {
+        return target_fd_trans[fd]->target_to_host_addr;
+    }
+    return NULL;
+}
+
+static void fd_trans_register(int fd, TargetFdTrans *trans)
+{
+    unsigned int oldmax;
+
+    if (fd >= target_fd_max) {
+        oldmax = target_fd_max;
+        target_fd_max = ((fd >> 6) + 1) << 6; /* by slice of 64 entries */
+        target_fd_trans = g_renew(TargetFdTrans *,
+                                  target_fd_trans, target_fd_max);
+        memset((void *)(target_fd_trans + oldmax), 0,
+               (target_fd_max - oldmax) * sizeof(TargetFdTrans *));
+    }
+    target_fd_trans[fd] = trans;
+}
+
+static void fd_trans_unregister(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max) {
+        target_fd_trans[fd] = NULL;
+    }
+}
+
+static void fd_trans_dup(int oldfd, int newfd)
+{
+    fd_trans_unregister(newfd);
+    if (oldfd < target_fd_max && target_fd_trans[oldfd]) {
+        fd_trans_register(newfd, target_fd_trans[oldfd]);
+    }
+}
+
 static int sys_getcwd1(char *buf, size_t size)
 {
   if (getcwd(buf, size) == NULL) {
@@ -457,6 +509,7 @@ static uint16_t target_to_host_errno_table[ERRNO_TABLE_SIZE] = {
  * minus the errnos that are not actually generic to all archs.
  */
 static uint16_t host_to_target_errno_table[ERRNO_TABLE_SIZE] = {
+    [EAGAIN]		= TARGET_EAGAIN,
     [EIDRM]		= TARGET_EIDRM,
     [ECHRNG]		= TARGET_ECHRNG,
     [EL2NSYNC]		= TARGET_EL2NSYNC,
@@ -1112,7 +1165,7 @@ static inline abi_long target_to_host_ip_mreq(struct ip_mreqn *mreqn,
     return 0;
 }
 
-static inline abi_long target_to_host_sockaddr(struct sockaddr *addr,
+static inline abi_long target_to_host_sockaddr(int fd, struct sockaddr *addr,
                                                abi_ulong target_addr,
                                                socklen_t len)
 {
@@ -1120,6 +1173,10 @@ static inline abi_long target_to_host_sockaddr(struct sockaddr *addr,
     sa_family_t sa_family;
     struct target_sockaddr *target_saddr;
 
+    if (fd_trans_target_to_host_addr(fd)) {
+        return fd_trans_target_to_host_addr(fd)(addr, target_addr, len);
+    }
+
     target_saddr = lock_user(VERIFY_READ, target_addr, len, 1);
     if (!target_saddr)
         return -TARGET_EFAULT;
@@ -1181,7 +1238,7 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
     
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1189,6 +1246,7 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1247,7 +1305,8 @@ static inline abi_long target_to_host_cmsg(struct msghdr *msgh,
         }
 
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, 0);
  the_end:
@@ -1261,7 +1320,7 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
 
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1269,6 +1328,7 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1382,14 +1442,15 @@ static inline abi_long host_to_target_cmsg(struct target_msghdr *target_msgh,
         }
 
         target_cmsg->cmsg_len = tswapal(tgt_len);
-        tgt_space = TARGET_CMSG_SPACE(tgt_len);
+        tgt_space = TARGET_CMSG_SPACE(len);
         if (msg_controllen < tgt_space) {
             tgt_space = msg_controllen;
         }
         msg_controllen -= tgt_space;
         space += tgt_space;
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, space);
  the_end:
@@ -1554,7 +1615,7 @@ set_timeout:
                 }
 
                 fprog.len = tswap16(tfprog->len);
-                filter = malloc(fprog.len * sizeof(*filter));
+                filter = g_try_new(struct sock_filter, fprog.len);
                 if (filter == NULL) {
                     unlock_user_struct(tfilter, tfprog->filter, 1);
                     unlock_user_struct(tfprog, optval_addr, 1);
@@ -1570,7 +1631,7 @@ set_timeout:
 
                 ret = get_errno(setsockopt(sockfd, SOL_SOCKET,
                                 SO_ATTACH_FILTER, &fprog, sizeof(fprog)));
-                free(filter);
+                g_free(filter);
 
                 unlock_user_struct(tfilter, tfprog->filter, 1);
                 unlock_user_struct(tfprog, optval_addr, 1);
@@ -1591,7 +1652,8 @@ set_timeout:
 		addr_ifname = alloca(IFNAMSIZ);
 		memcpy(addr_ifname, dev_ifname, optlen);
 		addr_ifname[optlen] = 0;
-		ret = get_errno(setsockopt(sockfd, level, optname, addr_ifname, optlen));
+		ret = get_errno(setsockopt(sockfd, SOL_SOCKET, optname,
+                                           addr_ifname, optlen));
 		unlock_user (dev_ifname, optval_addr, 0);
 		return ret;
 	}
@@ -1881,7 +1943,7 @@ static struct iovec *lock_iovec(int type, abi_ulong target_addr,
         return NULL;
     }
 
-    vec = calloc(count, sizeof(struct iovec));
+    vec = g_try_new0(struct iovec, count);
     if (vec == NULL) {
         errno = ENOMEM;
         return NULL;
@@ -1945,7 +2007,7 @@ static struct iovec *lock_iovec(int type, abi_ulong target_addr,
     }
     unlock_user(target_vec, target_addr, 0);
  fail2:
-    free(vec);
+    g_free(vec);
     errno = err;
     return NULL;
 }
@@ -1970,7 +2032,7 @@ static void unlock_iovec(struct iovec *vec, abi_ulong target_addr,
         unlock_user(target_vec, target_addr, 0);
     }
 
-    free(vec);
+    g_free(vec);
 }
 
 static inline int target_to_host_sock_type(int *type)
@@ -2022,6 +2084,30 @@ static int sock_flags_fixup(int fd, int target_type)
     return fd;
 }
 
+static abi_long packet_target_to_host_sockaddr(void *host_addr,
+                                               abi_ulong target_addr,
+                                               socklen_t len)
+{
+    struct sockaddr *addr = host_addr;
+    struct target_sockaddr *target_saddr;
+
+    target_saddr = lock_user(VERIFY_READ, target_addr, len, 1);
+    if (!target_saddr) {
+        return -TARGET_EFAULT;
+    }
+
+    memcpy(addr, target_saddr, len);
+    addr->sa_family = tswap16(target_saddr->sa_family);
+    /* spkt_protocol is big-endian */
+
+    unlock_user(target_saddr, target_addr, 0);
+    return 0;
+}
+
+static TargetFdTrans target_packet_trans = {
+    .target_to_host_addr = packet_target_to_host_sockaddr,
+};
+
 /* do_socket() Must return target values and target errnos. */
 static abi_long do_socket(int domain, int type, int protocol)
 {
@@ -2035,9 +2121,21 @@ static abi_long do_socket(int domain, int type, int protocol)
 
     if (domain == PF_NETLINK)
         return -TARGET_EAFNOSUPPORT;
+
+    if (domain == AF_PACKET ||
+        (domain == AF_INET && type == SOCK_PACKET)) {
+        protocol = tswap16(protocol);
+    }
+
     ret = get_errno(socket(domain, type, protocol));
     if (ret >= 0) {
         ret = sock_flags_fixup(ret, target_type);
+        if (type == SOCK_PACKET) {
+            /* Manage an obsolete case :
+             * if socket type is SOCK_PACKET, bind by name
+             */
+            fd_trans_register(ret, &target_packet_trans);
+        }
     }
     return ret;
 }
@@ -2055,7 +2153,7 @@ static abi_long do_bind(int sockfd, abi_ulong target_addr,
 
     addr = alloca(addrlen+1);
 
-    ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+    ret = target_to_host_sockaddr(sockfd, addr, target_addr, addrlen);
     if (ret)
         return ret;
 
@@ -2075,7 +2173,7 @@ static abi_long do_connect(int sockfd, abi_ulong target_addr,
 
     addr = alloca(addrlen+1);
 
-    ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+    ret = target_to_host_sockaddr(sockfd, addr, target_addr, addrlen);
     if (ret)
         return ret;
 
@@ -2095,8 +2193,9 @@ static abi_long do_sendrecvmsg_locked(int fd, struct target_msghdr *msgp,
     if (msgp->msg_name) {
         msg.msg_namelen = tswap32(msgp->msg_namelen);
         msg.msg_name = alloca(msg.msg_namelen+1);
-        ret = target_to_host_sockaddr(msg.msg_name, tswapal(msgp->msg_name),
-                                msg.msg_namelen);
+        ret = target_to_host_sockaddr(fd, msg.msg_name,
+                                      tswapal(msgp->msg_name),
+                                      msg.msg_namelen);
         if (ret) {
             goto out2;
         }
@@ -2166,7 +2265,6 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
     return ret;
 }
 
-#ifdef TARGET_NR_sendmmsg
 /* We don't rely on the C library to have sendmmsg/recvmmsg support,
  * so it might not have this *mmsg-specific flag either.
  */
@@ -2213,7 +2311,6 @@ static abi_long do_sendrecvmmsg(int fd, abi_ulong target_msgvec,
     }
     return ret;
 }
-#endif
 
 /* If we don't have a system accept4() then just call accept.
  * The callsites to do_accept4() will ensure that they don't
@@ -2358,7 +2455,7 @@ static abi_long do_sendto(int fd, abi_ulong msg, size_t len, int flags,
         return -TARGET_EFAULT;
     if (target_addr) {
         addr = alloca(addrlen+1);
-        ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+        ret = target_to_host_sockaddr(fd, addr, target_addr, addrlen);
         if (ret) {
             unlock_user(host_msg, msg, 0);
             return ret;
@@ -2436,6 +2533,8 @@ static abi_long do_socketcall(int num, abi_ulong vptr)
         [SOCKOP_shutdown] = 2,    /* sockfd, how */
         [SOCKOP_sendmsg] = 3,     /* sockfd, msg, flags */
         [SOCKOP_recvmsg] = 3,     /* sockfd, msg, flags */
+        [SOCKOP_sendmmsg] = 4,    /* sockfd, msgvec, vlen, flags */
+        [SOCKOP_recvmmsg] = 4,    /* sockfd, msgvec, vlen, flags */
         [SOCKOP_setsockopt] = 5,  /* sockfd, level, optname, optval, optlen */
         [SOCKOP_getsockopt] = 5,  /* sockfd, level, optname, optval, optlen */
     };
@@ -2486,6 +2585,10 @@ static abi_long do_socketcall(int num, abi_ulong vptr)
         return do_sendrecvmsg(a[0], a[1], a[2], 1);
     case SOCKOP_recvmsg: /* sockfd, msg, flags */
         return do_sendrecvmsg(a[0], a[1], a[2], 0);
+    case SOCKOP_sendmmsg: /* sockfd, msgvec, vlen, flags */
+        return do_sendrecvmmsg(a[0], a[1], a[2], a[3], 1);
+    case SOCKOP_recvmmsg: /* sockfd, msgvec, vlen, flags */
+        return do_sendrecvmmsg(a[0], a[1], a[2], a[3], 0);
     case SOCKOP_setsockopt: /* sockfd, level, optname, optval, optlen */
         return do_setsockopt(a[0], a[1], a[2], a[3], a[4]);
     case SOCKOP_getsockopt: /* sockfd, level, optname, optval, optlen */
@@ -2500,8 +2603,9 @@ static abi_long do_socketcall(int num, abi_ulong vptr)
 #define N_SHM_REGIONS	32
 
 static struct shm_region {
-    abi_ulong	start;
-    abi_ulong	size;
+    abi_ulong start;
+    abi_ulong size;
+    bool in_use;
 } shm_regions[N_SHM_REGIONS];
 
 struct target_semid_ds
@@ -2672,14 +2776,14 @@ static inline abi_long target_to_host_semarray(int semid, unsigned short **host_
 
     nsems = semid_ds.sem_nsems;
 
-    *host_array = malloc(nsems*sizeof(unsigned short));
+    *host_array = g_try_new(unsigned short, nsems);
     if (!*host_array) {
         return -TARGET_ENOMEM;
     }
     array = lock_user(VERIFY_READ, target_addr,
                       nsems*sizeof(unsigned short), 1);
     if (!array) {
-        free(*host_array);
+        g_free(*host_array);
         return -TARGET_EFAULT;
     }
 
@@ -2716,15 +2820,16 @@ static inline abi_long host_to_target_semarray(int semid, abi_ulong target_addr,
     for(i=0; i<nsems; i++) {
         __put_user((*host_array)[i], &array[i]);
     }
-    free(*host_array);
+    g_free(*host_array);
     unlock_user(array, target_addr, 1);
 
     return 0;
 }
 
 static inline abi_long do_semctl(int semid, int semnum, int cmd,
-                                 union target_semun target_su)
+                                 abi_ulong target_arg)
 {
+    union target_semun target_su = { .buf = target_arg };
     union semun arg;
     struct semid_ds dsarg;
     unsigned short *array = NULL;
@@ -2975,7 +3080,7 @@ static inline abi_long do_msgsnd(int msqid, abi_long msgp,
 
     if (!lock_user_struct(VERIFY_READ, target_mb, msgp, 0))
         return -TARGET_EFAULT;
-    host_mb = malloc(msgsz+sizeof(long));
+    host_mb = g_try_malloc(msgsz + sizeof(long));
     if (!host_mb) {
         unlock_user_struct(target_mb, msgp, 0);
         return -TARGET_ENOMEM;
@@ -2983,7 +3088,7 @@ static inline abi_long do_msgsnd(int msqid, abi_long msgp,
     host_mb->mtype = (abi_long) tswapal(target_mb->mtype);
     memcpy(host_mb->mtext, target_mb->mtext, msgsz);
     ret = get_errno(msgsnd(msqid, host_mb, msgsz, msgflg));
-    free(host_mb);
+    g_free(host_mb);
     unlock_user_struct(target_mb, msgp, 0);
 
     return ret;
@@ -3192,7 +3297,8 @@ static inline abi_ulong do_shmat(int shmid, abi_ulong shmaddr, int shmflg)
                    ((shmflg & SHM_RDONLY)? 0 : PAGE_WRITE));
 
     for (i = 0; i < N_SHM_REGIONS; i++) {
-        if (shm_regions[i].start == 0) {
+        if (!shm_regions[i].in_use) {
+            shm_regions[i].in_use = true;
             shm_regions[i].start = raddr;
             shm_regions[i].size = shm_info.shm_segsz;
             break;
@@ -3209,8 +3315,8 @@ static inline abi_long do_shmdt(abi_ulong shmaddr)
     int i;
 
     for (i = 0; i < N_SHM_REGIONS; ++i) {
-        if (shm_regions[i].start == shmaddr) {
-            shm_regions[i].start = 0;
+        if (shm_regions[i].in_use && shm_regions[i].start == shmaddr) {
+            shm_regions[i].in_use = false;
             page_set_flags(shmaddr, shmaddr + shm_regions[i].size, 0);
             break;
         }
@@ -3246,8 +3352,7 @@ static abi_long do_ipc(unsigned int call, abi_long first,
          * ptr argument. */
         abi_ulong atptr;
         get_user_ual(atptr, ptr);
-        ret = do_semctl(first, second, third,
-                (union target_semun) atptr);
+        ret = do_semctl(first, second, third, atptr);
         break;
     }
 
@@ -3411,7 +3516,7 @@ static abi_long do_ioctl_fs_ioc_fiemap(const IOCTLEntry *ie, uint8_t *buf_temp,
         /* We can't fit all the extents into the fixed size buffer.
          * Allocate one that is large enough and use it instead.
          */
-        fm = malloc(outbufsz);
+        fm = g_try_malloc(outbufsz);
         if (!fm) {
             return -TARGET_ENOMEM;
         }
@@ -3446,7 +3551,7 @@ static abi_long do_ioctl_fs_ioc_fiemap(const IOCTLEntry *ie, uint8_t *buf_temp,
         }
     }
     if (free_fm) {
-        free(fm);
+        g_free(fm);
     }
     return ret;
 }
@@ -3939,7 +4044,6 @@ static abi_long do_ioctl(int fd, int cmd, abi_long arg)
         break;
     case TYPE_PTRVOID:
     case TYPE_INT:
-        /* int argment */
         ret = get_errno(ioctl(fd, ie->host_cmd, arg));
         break;
     case TYPE_PTR:
@@ -4513,6 +4617,7 @@ static void *clone_func(void *arg)
     CPUState *cpu;
     TaskState *ts;
 
+    rcu_register_thread();
     env = info->env;
     cpu = ENV_GET_CPU(env);
     thread_cpu = cpu;
@@ -4561,7 +4666,7 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
         new_thread_info info;
         pthread_attr_t attr;
 
-        ts = g_malloc0(sizeof(TaskState));
+        ts = g_new0(TaskState, 1);
         init_task_state(ts);
         /* we create a new CPU instance. */
         new_env = cpu_copy(env);
@@ -4622,8 +4727,9 @@ static int do_fork(CPUArchState *env, unsigned int flags, abi_ulong newsp,
         pthread_mutex_unlock(&clone_lock);
     } else {
         /* if no CLONE_VM, we consider it is a fork */
-        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0)
-            return -EINVAL;
+        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0) {
+            return -TARGET_EINVAL;
+        }
         fork_start();
         ret = fork();
         if (ret == 0) {
@@ -5130,7 +5236,6 @@ static inline int target_to_host_mlockall_arg(int arg)
 }
 #endif
 
-#if defined(TARGET_NR_stat64) || defined(TARGET_NR_newfstatat)
 static inline abi_long host_to_target_stat64(void *cpu_env,
                                              abi_ulong target_addr,
                                              struct stat *host_st)
@@ -5193,7 +5298,6 @@ static inline abi_long host_to_target_stat64(void *cpu_env,
 
     return 0;
 }
-#endif
 
 /* ??? Using host futex calls even when target atomic operations
    are not really atomic probably breaks things.  However implementing
@@ -5246,6 +5350,179 @@ static int do_futex(target_ulong uaddr, int op, int val, target_ulong timeout,
         return -TARGET_ENOSYS;
     }
 }
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_name_to_handle_at(abi_long dirfd, abi_long pathname,
+                                     abi_long handle, abi_long mount_id,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    int mid = 0;
+    abi_long ret;
+    char *name;
+    unsigned int size, total_size;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    name = lock_user_string(pathname);
+    if (!name) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_WRITE, handle, total_size, 0);
+    if (!target_fh) {
+        unlock_user(name, pathname, 0);
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_malloc0(total_size);
+    fh->handle_bytes = size;
+
+    ret = get_errno(name_to_handle_at(dirfd, path(name), fh, &mid, flags));
+    unlock_user(name, pathname, 0);
+
+    /* man name_to_handle_at(2):
+     * Other than the use of the handle_bytes field, the caller should treat
+     * the file_handle structure as an opaque data type
+     */
+
+    memcpy(target_fh, fh, total_size);
+    target_fh->handle_bytes = tswap32(fh->handle_bytes);
+    target_fh->handle_type = tswap32(fh->handle_type);
+    g_free(fh);
+    unlock_user(target_fh, handle, total_size);
+
+    if (put_user_s32(mid, mount_id)) {
+        return -TARGET_EFAULT;
+    }
+
+    return ret;
+
+}
+#endif
+
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_open_by_handle_at(abi_long mount_fd, abi_long handle,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    unsigned int size, total_size;
+    abi_long ret;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_READ, handle, total_size, 1);
+    if (!target_fh) {
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_memdup(target_fh, total_size);
+    fh->handle_bytes = size;
+    fh->handle_type = tswap32(target_fh->handle_type);
+
+    ret = get_errno(open_by_handle_at(mount_fd, fh,
+                    target_to_host_bitmask(flags, fcntl_flags_tbl)));
+
+    g_free(fh);
+
+    unlock_user(target_fh, handle, total_size);
+
+    return ret;
+}
+#endif
+
+#if defined(TARGET_NR_signalfd) || defined(TARGET_NR_signalfd4)
+
+/* signalfd siginfo conversion */
+
+static void
+host_to_target_signalfd_siginfo(struct signalfd_siginfo *tinfo,
+                                const struct signalfd_siginfo *info)
+{
+    int sig = host_to_target_signal(info->ssi_signo);
+
+    /* linux/signalfd.h defines a ssi_addr_lsb
+     * not defined in sys/signalfd.h but used by some kernels
+     */
+
+#ifdef BUS_MCEERR_AO
+    if (tinfo->ssi_signo == SIGBUS &&
+        (tinfo->ssi_code == BUS_MCEERR_AR ||
+         tinfo->ssi_code == BUS_MCEERR_AO)) {
+        uint16_t *ssi_addr_lsb = (uint16_t *)(&info->ssi_addr + 1);
+        uint16_t *tssi_addr_lsb = (uint16_t *)(&tinfo->ssi_addr + 1);
+        *tssi_addr_lsb = tswap16(*ssi_addr_lsb);
+    }
+#endif
+
+    tinfo->ssi_signo = tswap32(sig);
+    tinfo->ssi_errno = tswap32(tinfo->ssi_errno);
+    tinfo->ssi_code = tswap32(info->ssi_code);
+    tinfo->ssi_pid = tswap32(info->ssi_pid);
+    tinfo->ssi_uid = tswap32(info->ssi_uid);
+    tinfo->ssi_fd = tswap32(info->ssi_fd);
+    tinfo->ssi_tid = tswap32(info->ssi_tid);
+    tinfo->ssi_band = tswap32(info->ssi_band);
+    tinfo->ssi_overrun = tswap32(info->ssi_overrun);
+    tinfo->ssi_trapno = tswap32(info->ssi_trapno);
+    tinfo->ssi_status = tswap32(info->ssi_status);
+    tinfo->ssi_int = tswap32(info->ssi_int);
+    tinfo->ssi_ptr = tswap64(info->ssi_ptr);
+    tinfo->ssi_utime = tswap64(info->ssi_utime);
+    tinfo->ssi_stime = tswap64(info->ssi_stime);
+    tinfo->ssi_addr = tswap64(info->ssi_addr);
+}
+
+static abi_long host_to_target_data_signalfd(void *buf, size_t len)
+{
+    int i;
+
+    for (i = 0; i < len; i += sizeof(struct signalfd_siginfo)) {
+        host_to_target_signalfd_siginfo(buf + i, buf + i);
+    }
+
+    return len;
+}
+
+static TargetFdTrans target_signalfd_trans = {
+    .host_to_target_data = host_to_target_data_signalfd,
+};
+
+static abi_long do_signalfd4(int fd, abi_long mask, int flags)
+{
+    int host_flags;
+    target_sigset_t *target_mask;
+    sigset_t host_mask;
+    abi_long ret;
+
+    if (flags & ~(TARGET_O_NONBLOCK | TARGET_O_CLOEXEC)) {
+        return -TARGET_EINVAL;
+    }
+    if (!lock_user_struct(VERIFY_READ, target_mask, mask, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    target_to_host_sigset(&host_mask, target_mask);
+
+    host_flags = target_to_host_bitmask(flags, fcntl_flags_tbl);
+
+    ret = get_errno(signalfd(fd, &host_mask, host_flags));
+    if (ret >= 0) {
+        fd_trans_register(ret, &target_signalfd_trans);
+    }
+
+    unlock_user_struct(target_mask, mask, 0);
+
+    return ret;
+}
+#endif
 
 /* Map host to target signal numbers for the wait family of syscalls.
    Assume all other status bits are the same.  */
@@ -5614,6 +5891,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             thread_cpu = NULL;
             object_unref(OBJECT(cpu));
             g_free(ts);
+            rcu_unregister_thread();
             pthread_exit(NULL);
         }
 #ifdef TARGET_GPROF
@@ -5630,6 +5908,10 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
                 goto efault;
             ret = get_errno(read(arg1, p, arg3));
+            if (ret >= 0 &&
+                fd_trans_host_to_target_data(arg1)) {
+                ret = fd_trans_host_to_target_data(arg1)(p, ret);
+            }
             unlock_user(p, arg2, ret);
         }
         break;
@@ -5639,31 +5921,49 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(write(arg1, p, arg3));
         unlock_user(p, arg2, 0);
         break;
+#ifdef TARGET_NR_open
     case TARGET_NR_open:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(do_openat(cpu_env, AT_FDCWD, p,
                                   target_to_host_bitmask(arg2, fcntl_flags_tbl),
                                   arg3));
+        fd_trans_unregister(ret);
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_openat:
         if (!(p = lock_user_string(arg2)))
             goto efault;
         ret = get_errno(do_openat(cpu_env, arg1, p,
                                   target_to_host_bitmask(arg3, fcntl_flags_tbl),
                                   arg4));
+        fd_trans_unregister(ret);
         unlock_user(p, arg2, 0);
         break;
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_name_to_handle_at:
+        ret = do_name_to_handle_at(arg1, arg2, arg3, arg4, arg5);
+        break;
+#endif
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_open_by_handle_at:
+        ret = do_open_by_handle_at(arg1, arg2, arg3);
+        fd_trans_unregister(ret);
+        break;
+#endif
     case TARGET_NR_close:
+        fd_trans_unregister(arg1);
         ret = get_errno(close(arg1));
         break;
     case TARGET_NR_brk:
         ret = do_brk(arg1);
         break;
+#ifdef TARGET_NR_fork
     case TARGET_NR_fork:
         ret = get_errno(do_fork(cpu_env, SIGCHLD, 0, 0, 0, 0));
         break;
+#endif
 #ifdef TARGET_NR_waitpid
     case TARGET_NR_waitpid:
         {
@@ -5695,9 +5995,11 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(creat(p, arg2));
+        fd_trans_unregister(ret);
         unlock_user(p, arg1, 0);
         break;
 #endif
+#ifdef TARGET_NR_link
     case TARGET_NR_link:
         {
             void * p2;
@@ -5711,6 +6013,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_linkat)
     case TARGET_NR_linkat:
         {
@@ -5728,12 +6031,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_unlink
     case TARGET_NR_unlink:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(unlink(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_unlinkat)
     case TARGET_NR_unlinkat:
         if (!(p = lock_user_string(arg2)))
@@ -5799,12 +6104,6 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             }
             *q = NULL;
 
-            /* This case will not be caught by the host's execve() if its
-               page size is bigger than the target's. */
-            if (total_size > MAX_ARG_PAGES * TARGET_PAGE_SIZE) {
-                ret = -TARGET_E2BIG;
-                goto execve_end;
-            }
             if (!(p = lock_user_string(arg1)))
                 goto execve_efault;
             ret = get_errno(execve(p, argp, envp));
@@ -5850,12 +6149,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_mknod
     case TARGET_NR_mknod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mknod(p, arg2, arg3));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mknodat)
     case TARGET_NR_mknodat:
         if (!(p = lock_user_string(arg2)))
@@ -5864,12 +6165,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_chmod
     case TARGET_NR_chmod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chmod(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_break
     case TARGET_NR_break:
         goto unimplemented;
@@ -6004,6 +6307,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_utimes
     case TARGET_NR_utimes:
         {
             struct timeval *tvp, tv[2];
@@ -6022,6 +6326,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_futimesat)
     case TARGET_NR_futimesat:
         {
@@ -6050,12 +6355,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_gtty:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_access
     case TARGET_NR_access:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(access(path(p), arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_faccessat) && defined(__NR_faccessat)
     case TARGET_NR_faccessat:
         if (!(p = lock_user_string(arg2)))
@@ -6080,6 +6387,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_kill:
         ret = get_errno(kill(arg1, target_to_host_signal(arg2)));
         break;
+#ifdef TARGET_NR_rename
     case TARGET_NR_rename:
         {
             void *p2;
@@ -6093,6 +6401,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_renameat)
     case TARGET_NR_renameat:
         {
@@ -6108,12 +6417,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_mkdir
     case TARGET_NR_mkdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mkdir(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mkdirat)
     case TARGET_NR_mkdirat:
         if (!(p = lock_user_string(arg2)))
@@ -6122,18 +6433,25 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_rmdir
     case TARGET_NR_rmdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(rmdir(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_dup:
         ret = get_errno(dup(arg1));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, ret);
+        }
         break;
+#ifdef TARGET_NR_pipe
     case TARGET_NR_pipe:
         ret = do_pipe(cpu_env, arg1, 0, 0);
         break;
+#endif
 #ifdef TARGET_NR_pipe2
     case TARGET_NR_pipe2:
         ret = do_pipe(cpu_env, arg1,
@@ -6218,14 +6536,24 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(chroot(p));
         unlock_user(p, arg1, 0);
         break;
+#ifdef TARGET_NR_ustat
     case TARGET_NR_ustat:
         goto unimplemented;
+#endif
+#ifdef TARGET_NR_dup2
     case TARGET_NR_dup2:
         ret = get_errno(dup2(arg1, arg2));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, arg2);
+        }
         break;
+#endif
 #if defined(CONFIG_DUP3) && defined(TARGET_NR_dup3)
     case TARGET_NR_dup3:
         ret = get_errno(dup3(arg1, arg2, arg3));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, arg2);
+        }
         break;
 #endif
 #ifdef TARGET_NR_getppid /* not on alpha */
@@ -6233,9 +6561,11 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(getppid());
         break;
 #endif
+#ifdef TARGET_NR_getpgrp
     case TARGET_NR_getpgrp:
         ret = get_errno(getpgrp());
         break;
+#endif
     case TARGET_NR_setsid:
         ret = get_errno(setsid());
         break;
@@ -6811,6 +7141,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_symlink
     case TARGET_NR_symlink:
         {
             void *p2;
@@ -6824,6 +7155,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_symlinkat)
     case TARGET_NR_symlinkat:
         {
@@ -6843,6 +7175,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
     case TARGET_NR_oldlstat:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_readlink
     case TARGET_NR_readlink:
         {
             void *p2;
@@ -6873,6 +7206,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_readlinkat)
     case TARGET_NR_readlinkat:
         {
@@ -7212,9 +7546,20 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(shutdown(arg1, arg2));
         break;
 #endif
+#if defined(TARGET_NR_getrandom) && defined(__NR_getrandom)
+    case TARGET_NR_getrandom:
+        p = lock_user(VERIFY_WRITE, arg1, arg2, 0);
+        if (!p) {
+            goto efault;
+        }
+        ret = get_errno(getrandom(p, arg2, arg3));
+        unlock_user(p, arg1, ret);
+        break;
+#endif
 #ifdef TARGET_NR_socket
     case TARGET_NR_socket:
         ret = do_socket(arg1, arg2, arg3);
+        fd_trans_unregister(ret);
         break;
 #endif
 #ifdef TARGET_NR_socketpair
@@ -7272,22 +7617,28 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             }
         }
         break;
+#ifdef TARGET_NR_stat
     case TARGET_NR_stat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(stat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
+#ifdef TARGET_NR_lstat
     case TARGET_NR_lstat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(lstat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
     case TARGET_NR_fstat:
         {
             ret = get_errno(fstat(arg1, &st));
+#if defined(TARGET_NR_stat) || defined(TARGET_NR_lstat)
         do_stat:
+#endif
             if (!is_error(ret)) {
                 struct target_stat *target_st;
 
@@ -7411,7 +7762,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #endif
 #ifdef TARGET_NR_semctl
     case TARGET_NR_semctl:
-        ret = do_semctl(arg1, arg2, arg3, (union target_semun)(abi_ulong)arg4);
+        ret = do_semctl(arg1, arg2, arg3, arg4);
         break;
 #endif
 #ifdef TARGET_NR_msgctl
@@ -7575,6 +7926,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_getdents
     case TARGET_NR_getdents:
 #ifdef __NR_getdents
 #if TARGET_ABI_BITS == 32 && HOST_LONG_BITS == 64
@@ -7583,8 +7935,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             struct linux_dirent *dirp;
             abi_long count = arg3;
 
-	    dirp = malloc(count);
-	    if (!dirp) {
+            dirp = g_try_malloc(count);
+            if (!dirp) {
                 ret = -TARGET_ENOMEM;
                 goto fail;
             }
@@ -7620,7 +7972,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 		ret = count1;
                 unlock_user(target_dirp, arg2, ret);
             }
-	    free(dirp);
+            g_free(dirp);
         }
 #else
         {
@@ -7705,6 +8057,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
 #endif
         break;
+#endif /* TARGET_NR_getdents */
 #if defined(TARGET_NR_getdents64) && defined(__NR_getdents64)
     case TARGET_NR_getdents64:
         {
@@ -7752,14 +8105,20 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             struct pollfd *pfd;
             unsigned int i;
 
-            target_pfd = lock_user(VERIFY_WRITE, arg1, sizeof(struct target_pollfd) * nfds, 1);
-            if (!target_pfd)
-                goto efault;
+            pfd = NULL;
+            target_pfd = NULL;
+            if (nfds) {
+                target_pfd = lock_user(VERIFY_WRITE, arg1,
+                                       sizeof(struct target_pollfd) * nfds, 1);
+                if (!target_pfd) {
+                    goto efault;
+                }
 
-            pfd = alloca(sizeof(struct pollfd) * nfds);
-            for(i = 0; i < nfds; i++) {
-                pfd[i].fd = tswap32(target_pfd[i].fd);
-                pfd[i].events = tswap16(target_pfd[i].events);
+                pfd = alloca(sizeof(struct pollfd) * nfds);
+                for (i = 0; i < nfds; i++) {
+                    pfd[i].fd = tswap32(target_pfd[i].fd);
+                    pfd[i].events = tswap16(target_pfd[i].events);
+                }
             }
 
 # ifdef TARGET_NR_ppoll
@@ -7844,11 +8203,13 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(fdatasync(arg1));
         break;
 #endif
+#ifdef TARGET_NR__sysctl
     case TARGET_NR__sysctl:
         /* We don't implement this, but ENOTDIR is always a safe
            return value. */
         ret = -TARGET_ENOTDIR;
         break;
+#endif
     case TARGET_NR_sched_getaffinity:
         {
             unsigned int mask_size;
@@ -8150,14 +8511,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     }
     case TARGET_NR_sigaltstack:
-#if defined(TARGET_I386) || defined(TARGET_ARM) || defined(TARGET_MIPS) || \
-    defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_ALPHA) || \
-    defined(TARGET_M68K) || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
         ret = do_sigaltstack(arg1, arg2, get_sp_from_cpustate((CPUArchState *)cpu_env));
         break;
-#else
-        goto unimplemented;
-#endif
 
 #ifdef CONFIG_SENDFILE
     case TARGET_NR_sendfile:
@@ -8295,12 +8650,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             ret = host_to_target_stat64(cpu_env, arg3, &st);
         break;
 #endif
+#ifdef TARGET_NR_lchown
     case TARGET_NR_lchown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(lchown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_getuid
     case TARGET_NR_getuid:
         ret = get_errno(high2lowuid(getuid()));
@@ -8423,12 +8780,14 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         }
         break;
 #endif
+#ifdef TARGET_NR_chown
     case TARGET_NR_chown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_setuid:
         ret = get_errno(setuid(low2highuid(arg1)));
         break;
@@ -9454,6 +9813,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #if defined(TARGET_NR_eventfd)
     case TARGET_NR_eventfd:
         ret = get_errno(eventfd(arg1, 0));
+        fd_trans_unregister(ret);
         break;
 #endif
 #if defined(TARGET_NR_eventfd2)
@@ -9467,6 +9827,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             host_flags |= O_CLOEXEC;
         }
         ret = get_errno(eventfd(arg1, host_flags));
+        fd_trans_unregister(ret);
         break;
     }
 #endif
@@ -9509,6 +9870,16 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
 #endif
 #endif
+#if defined(TARGET_NR_signalfd4)
+    case TARGET_NR_signalfd4:
+        ret = do_signalfd4(arg1, arg2, arg4);
+        break;
+#endif
+#if defined(TARGET_NR_signalfd)
+    case TARGET_NR_signalfd:
+        ret = do_signalfd4(arg1, arg2, 0);
+        break;
+#endif
 #if defined(CONFIG_EPOLL)
 #if defined(TARGET_NR_epoll_create)
     case TARGET_NR_epoll_create:
@@ -9780,6 +10151,7 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             timer_t htimer = g_posix_timers[timerid];
             ret = get_errno(timer_getoverrun(htimer));
         }
+        fd_trans_unregister(ret);
         break;
     }
 #endif
diff --git a/qemu/linux-user/syscall_defs.h b/qemu/linux-user/syscall_defs.h
index edd5f3c80..9e2b3c200 100644
--- a/qemu/linux-user/syscall_defs.h
+++ b/qemu/linux-user/syscall_defs.h
@@ -28,6 +28,8 @@
 #define SOCKOP_sendmsg          16
 #define SOCKOP_recvmsg          17
 #define SOCKOP_accept4          18
+#define SOCKOP_recvmmsg         19
+#define SOCKOP_sendmmsg         20
 
 #define IPCOP_semop		1
 #define IPCOP_semget		2
@@ -64,8 +66,9 @@
 #endif
 
 #if defined(TARGET_I386) || defined(TARGET_ARM) || defined(TARGET_SH4) \
-    || defined(TARGET_M68K) || defined(TARGET_CRIS) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_M68K) || defined(TARGET_CRIS) \
+    || defined(TARGET_UNICORE32) || defined(TARGET_S390X) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 #define TARGET_IOC_SIZEBITS	14
 #define TARGET_IOC_DIRBITS	2
@@ -234,7 +237,8 @@ struct target_cmsghdr {
 };
 
 #define TARGET_CMSG_DATA(cmsg) ((unsigned char *) ((struct target_cmsghdr *) (cmsg) + 1))
-#define TARGET_CMSG_NXTHDR(mhdr, cmsg) __target_cmsg_nxthdr (mhdr, cmsg)
+#define TARGET_CMSG_NXTHDR(mhdr, cmsg, cmsg_start) \
+                               __target_cmsg_nxthdr(mhdr, cmsg, cmsg_start)
 #define TARGET_CMSG_ALIGN(len) (((len) + sizeof (abi_long) - 1) \
                                & (size_t) ~(sizeof (abi_long) - 1))
 #define TARGET_CMSG_SPACE(len) (TARGET_CMSG_ALIGN (len) \
@@ -242,17 +246,20 @@ struct target_cmsghdr {
 #define TARGET_CMSG_LEN(len)   (TARGET_CMSG_ALIGN (sizeof (struct target_cmsghdr)) + (len))
 
 static __inline__ struct target_cmsghdr *
-__target_cmsg_nxthdr (struct target_msghdr *__mhdr, struct target_cmsghdr *__cmsg)
+__target_cmsg_nxthdr(struct target_msghdr *__mhdr,
+                     struct target_cmsghdr *__cmsg,
+                     struct target_cmsghdr *__cmsg_start)
 {
   struct target_cmsghdr *__ptr;
 
   __ptr = (struct target_cmsghdr *)((unsigned char *) __cmsg
                                     + TARGET_CMSG_ALIGN (tswapal(__cmsg->cmsg_len)));
-  if ((unsigned long)((char *)(__ptr+1) - (char *)(size_t)tswapal(__mhdr->msg_control))
-      > tswapal(__mhdr->msg_controllen))
+  if ((unsigned long)((char *)(__ptr+1) - (char *)__cmsg_start)
+      > tswapal(__mhdr->msg_controllen)) {
     /* No more entries.  */
     return (struct target_cmsghdr *)0;
-  return __cmsg;
+  }
+  return __ptr;
 }
 
 struct target_mmsghdr {
@@ -365,7 +372,8 @@ int do_sigaction(int sig, const struct target_sigaction *act,
     || defined(TARGET_PPC) || defined(TARGET_MIPS) || defined(TARGET_SH4) \
     || defined(TARGET_M68K) || defined(TARGET_ALPHA) || defined(TARGET_CRIS) \
     || defined(TARGET_MICROBLAZE) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_S390X) || defined(TARGET_OPENRISC) \
+    || defined(TARGET_TILEGX)
 
 #if defined(TARGET_SPARC)
 #define TARGET_SA_NOCLDSTOP    8u
@@ -742,6 +750,10 @@ typedef struct target_siginfo {
 #define TARGET_ILL_PRVREG	(6)	/* privileged register */
 #define TARGET_ILL_COPROC	(7)	/* coprocessor error */
 #define TARGET_ILL_BADSTK	(8)	/* internal stack error */
+#ifdef TARGET_TILEGX
+#define TARGET_ILL_DBLFLT       (9)     /* double fault */
+#define TARGET_ILL_HARDWALL     (10)    /* user networks hardwall violation */
+#endif
 
 /*
  * SIGFPE si_codes
@@ -761,6 +773,7 @@ typedef struct target_siginfo {
  */
 #define TARGET_SEGV_MAPERR     (1)  /* address not mapped to object */
 #define TARGET_SEGV_ACCERR     (2)  /* invalid permissions for mapped object */
+#define TARGET_SEGV_BNDERR     (3)  /* failed address bound checks */
 
 /*
  * SIGBUS si_codes
@@ -768,12 +781,18 @@ typedef struct target_siginfo {
 #define TARGET_BUS_ADRALN       (1)	/* invalid address alignment */
 #define TARGET_BUS_ADRERR       (2)	/* non-existent physical address */
 #define TARGET_BUS_OBJERR       (3)	/* object specific hardware error */
+/* hardware memory error consumed on a machine check: action required */
+#define TARGET_BUS_MCEERR_AR    (4)
+/* hardware memory error detected in process but not consumed: action optional*/
+#define TARGET_BUS_MCEERR_AO    (5)
 
 /*
  * SIGTRAP si_codes
  */
 #define TARGET_TRAP_BRKPT	(1)	/* process breakpoint */
 #define TARGET_TRAP_TRACE	(2)	/* process trace trap */
+#define TARGET_TRAP_BRANCH      (3)     /* process taken branch trap */
+#define TARGET_TRAP_HWBKPT      (4)     /* hardware breakpoint/watchpoint */
 
 #endif /* defined(TARGET_I386) || defined(TARGET_ARM) */
 
@@ -969,7 +988,7 @@ struct target_pollfd {
                                            (struct cdrom_multisession) */
 #define TARGET_CDROM_GET_MCN		0x5311 /* Obtain the "Universal Product Code"
                                            if available (struct cdrom_mcn) */
-#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is depricated,
+#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is deprecated,
                                           but here anyway for compatibility */
 #define TARGET_CDROMRESET		0x5312 /* hard-reset the drive */
 #define TARGET_CDROMVOLREAD		0x5313 /* Get the drive's volume setting
@@ -1871,7 +1890,7 @@ struct target_stat {
     abi_ulong  target_st_ctime_nsec;
     unsigned int __unused[2];
 };
-#elif defined(TARGET_OPENRISC)
+#elif defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 /* These are the asm-generic versions of the stat and stat64 structures */
 
@@ -2264,7 +2283,9 @@ struct target_flock {
 struct target_flock64 {
 	short  l_type;
 	short  l_whence;
-#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) || defined(TARGET_SPARC) || defined(TARGET_HPPA) || defined (TARGET_MICROBLAZE)
+#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) \
+    || defined(TARGET_SPARC) || defined(TARGET_HPPA) \
+    || defined(TARGET_MICROBLAZE) || defined(TARGET_TILEGX)
         int __pad;
 #endif
 	unsigned long long l_start;
@@ -2289,7 +2310,7 @@ struct target_f_owner_ex {
 };
 
 /* soundcard defines */
-/* XXX: convert them all to arch indepedent entries */
+/* XXX: convert them all to arch independent entries */
 #define TARGET_SNDCTL_COPR_HALT           TARGET_IOWR('C',  7, int);
 #define TARGET_SNDCTL_COPR_LOAD           0xcfb04301
 #define TARGET_SNDCTL_COPR_RCODE          0xc0144303
@@ -2495,20 +2516,23 @@ struct target_mq_attr {
 #define FUTEX_CMD_MASK          ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)
 
 #ifdef CONFIG_EPOLL
+#if defined(TARGET_X86_64)
+#define TARGET_EPOLL_PACKED QEMU_PACKED
+#else
+#define TARGET_EPOLL_PACKED
+#endif
+
 typedef union target_epoll_data {
     abi_ulong ptr;
-    abi_ulong fd;
-    uint32_t u32;
-    uint64_t u64;
+    abi_int fd;
+    abi_uint u32;
+    abi_ullong u64;
 } target_epoll_data_t;
 
 struct target_epoll_event {
-    uint32_t events;
-#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_MIPS64)
-    uint32_t __pad;
-#endif
+    abi_uint events;
     target_epoll_data_t data;
-} QEMU_PACKED;
+} TARGET_EPOLL_PACKED;
 #endif
 struct target_rlimit64 {
     uint64_t rlim_cur;
diff --git a/qemu/linux-user/tilegx/syscall_nr.h b/qemu/linux-user/tilegx/syscall_nr.h
new file mode 100644
index 000000000..87fb72c55
--- /dev/null
+++ b/qemu/linux-user/tilegx/syscall_nr.h
@@ -0,0 +1,328 @@
+#ifndef TILEGX_SYSCALL_NR
+#define TILEGX_SYSCALL_NR
+
+/*
+ * Copy from linux kernel asm-generic/unistd.h, which tilegx uses.
+ */
+#define TARGET_NR_io_setup                      0
+#define TARGET_NR_io_destroy                    1
+#define TARGET_NR_io_submit                     2
+#define TARGET_NR_io_cancel                     3
+#define TARGET_NR_io_getevents                  4
+#define TARGET_NR_setxattr                      5
+#define TARGET_NR_lsetxattr                     6
+#define TARGET_NR_fsetxattr                     7
+#define TARGET_NR_getxattr                      8
+#define TARGET_NR_lgetxattr                     9
+#define TARGET_NR_fgetxattr                     10
+#define TARGET_NR_listxattr                     11
+#define TARGET_NR_llistxattr                    12
+#define TARGET_NR_flistxattr                    13
+#define TARGET_NR_removexattr                   14
+#define TARGET_NR_lremovexattr                  15
+#define TARGET_NR_fremovexattr                  16
+#define TARGET_NR_getcwd                        17
+#define TARGET_NR_lookup_dcookie                18
+#define TARGET_NR_eventfd2                      19
+#define TARGET_NR_epoll_create1                 20
+#define TARGET_NR_epoll_ctl                     21
+#define TARGET_NR_epoll_pwait                   22
+#define TARGET_NR_dup                           23
+#define TARGET_NR_dup3                          24
+#define TARGET_NR_fcntl                         25
+#define TARGET_NR_inotify_init1                 26
+#define TARGET_NR_inotify_add_watch             27
+#define TARGET_NR_inotify_rm_watch              28
+#define TARGET_NR_ioctl                         29
+#define TARGET_NR_ioprio_set                    30
+#define TARGET_NR_ioprio_get                    31
+#define TARGET_NR_flock                         32
+#define TARGET_NR_mknodat                       33
+#define TARGET_NR_mkdirat                       34
+#define TARGET_NR_unlinkat                      35
+#define TARGET_NR_symlinkat                     36
+#define TARGET_NR_linkat                        37
+#define TARGET_NR_renameat                      38
+#define TARGET_NR_umount2                       39
+#define TARGET_NR_mount                         40
+#define TARGET_NR_pivot_root                    41
+#define TARGET_NR_nfsservctl                    42
+#define TARGET_NR_statfs                        43
+#define TARGET_NR_fstatfs                       44
+#define TARGET_NR_truncate                      45
+#define TARGET_NR_ftruncate                     46
+#define TARGET_NR_fallocate                     47
+#define TARGET_NR_faccessat                     48
+#define TARGET_NR_chdir                         49
+#define TARGET_NR_fchdir                        50
+#define TARGET_NR_chroot                        51
+#define TARGET_NR_fchmod                        52
+#define TARGET_NR_fchmodat                      53
+#define TARGET_NR_fchownat                      54
+#define TARGET_NR_fchown                        55
+#define TARGET_NR_openat                        56
+#define TARGET_NR_close                         57
+#define TARGET_NR_vhangup                       58
+#define TARGET_NR_pipe2                         59
+#define TARGET_NR_quotactl                      60
+#define TARGET_NR_getdents64                    61
+#define TARGET_NR_lseek                         62
+#define TARGET_NR_read                          63
+#define TARGET_NR_write                         64
+#define TARGET_NR_readv                         65
+#define TARGET_NR_writev                        66
+#define TARGET_NR_pread64                       67
+#define TARGET_NR_pwrite64                      68
+#define TARGET_NR_preadv                        69
+#define TARGET_NR_pwritev                       70
+#define TARGET_NR_sendfile                      71
+#define TARGET_NR_pselect6                      72
+#define TARGET_NR_ppoll                         73
+#define TARGET_NR_signalfd4                     74
+#define TARGET_NR_vmsplice                      75
+#define TARGET_NR_splice                        76
+#define TARGET_NR_tee                           77
+#define TARGET_NR_readlinkat                    78
+#define TARGET_NR_fstatat64                     79 /* let syscall.c known */
+#define TARGET_NR_fstat                         80
+#define TARGET_NR_sync                          81
+#define TARGET_NR_fsync                         82
+#define TARGET_NR_fdatasync                     83
+#define TARGET_NR_sync_file_range               84 /* For tilegx, no range2 */
+#define TARGET_NR_timerfd_create                85
+#define TARGET_NR_timerfd_settime               86
+#define TARGET_NR_timerfd_gettime               87
+#define TARGET_NR_utimensat                     88
+#define TARGET_NR_acct                          89
+#define TARGET_NR_capget                        90
+#define TARGET_NR_capset                        91
+#define TARGET_NR_personality                   92
+#define TARGET_NR_exit                          93
+#define TARGET_NR_exit_group                    94
+#define TARGET_NR_waitid                        95
+#define TARGET_NR_set_tid_address               96
+#define TARGET_NR_unshare                       97
+#define TARGET_NR_futex                         98
+#define TARGET_NR_set_robust_list               99
+#define TARGET_NR_get_robust_list               100
+#define TARGET_NR_nanosleep                     101
+#define TARGET_NR_getitimer                     102
+#define TARGET_NR_setitimer                     103
+#define TARGET_NR_kexec_load                    104
+#define TARGET_NR_init_module                   105
+#define TARGET_NR_delete_module                 106
+#define TARGET_NR_timer_create                  107
+#define TARGET_NR_timer_gettime                 108
+#define TARGET_NR_timer_getoverrun              109
+#define TARGET_NR_timer_settime                 110
+#define TARGET_NR_timer_delete                  111
+#define TARGET_NR_clock_settime                 112
+#define TARGET_NR_clock_gettime                 113
+#define TARGET_NR_clock_getres                  114
+#define TARGET_NR_clock_nanosleep               115
+#define TARGET_NR_syslog                        116
+#define TARGET_NR_ptrace                        117
+#define TARGET_NR_sched_setparam                118
+#define TARGET_NR_sched_setscheduler            119
+#define TARGET_NR_sched_getscheduler            120
+#define TARGET_NR_sched_getparam                121
+#define TARGET_NR_sched_setaffinity             122
+#define TARGET_NR_sched_getaffinity             123
+#define TARGET_NR_sched_yield                   124
+#define TARGET_NR_sched_get_priority_max        125
+#define TARGET_NR_sched_get_priority_min        126
+#define TARGET_NR_sched_rr_get_interval         127
+#define TARGET_NR_restart_syscall               128
+#define TARGET_NR_kill                          129
+#define TARGET_NR_tkill                         130
+#define TARGET_NR_tgkill                        131
+#define TARGET_NR_sigaltstack                   132
+#define TARGET_NR_rt_sigsuspend                 133
+#define TARGET_NR_rt_sigaction                  134
+#define TARGET_NR_rt_sigprocmask                135
+#define TARGET_NR_rt_sigpending                 136
+#define TARGET_NR_rt_sigtimedwait               137
+#define TARGET_NR_rt_sigqueueinfo               138
+#define TARGET_NR_rt_sigreturn                  139
+#define TARGET_NR_setpriority                   140
+#define TARGET_NR_getpriority                   141
+#define TARGET_NR_reboot                        142
+#define TARGET_NR_setregid                      143
+#define TARGET_NR_setgid                        144
+#define TARGET_NR_setreuid                      145
+#define TARGET_NR_setuid                        146
+#define TARGET_NR_setresuid                     147
+#define TARGET_NR_getresuid                     148
+#define TARGET_NR_setresgid                     149
+#define TARGET_NR_getresgid                     150
+#define TARGET_NR_setfsuid                      151
+#define TARGET_NR_setfsgid                      152
+#define TARGET_NR_times                         153
+#define TARGET_NR_setpgid                       154
+#define TARGET_NR_getpgid                       155
+#define TARGET_NR_getsid                        156
+#define TARGET_NR_setsid                        157
+#define TARGET_NR_getgroups                     158
+#define TARGET_NR_setgroups                     159
+#define TARGET_NR_uname                         160
+#define TARGET_NR_sethostname                   161
+#define TARGET_NR_setdomainname                 162
+#define TARGET_NR_getrlimit                     163
+#define TARGET_NR_setrlimit                     164
+#define TARGET_NR_getrusage                     165
+#define TARGET_NR_umask                         166
+#define TARGET_NR_prctl                         167
+#define TARGET_NR_getcpu                        168
+#define TARGET_NR_gettimeofday                  169
+#define TARGET_NR_settimeofday                  170
+#define TARGET_NR_adjtimex                      171
+#define TARGET_NR_getpid                        172
+#define TARGET_NR_getppid                       173
+#define TARGET_NR_getuid                        174
+#define TARGET_NR_geteuid                       175
+#define TARGET_NR_getgid                        176
+#define TARGET_NR_getegid                       177
+#define TARGET_NR_gettid                        178
+#define TARGET_NR_sysinfo                       179
+#define TARGET_NR_mq_open                       180
+#define TARGET_NR_mq_unlink                     181
+#define TARGET_NR_mq_timedsend                  182
+#define TARGET_NR_mq_timedreceive               183
+#define TARGET_NR_mq_notify                     184
+#define TARGET_NR_mq_getsetattr                 185
+#define TARGET_NR_msgget                        186
+#define TARGET_NR_msgctl                        187
+#define TARGET_NR_msgrcv                        188
+#define TARGET_NR_msgsnd                        189
+#define TARGET_NR_semget                        190
+#define TARGET_NR_semctl                        191
+#define TARGET_NR_semtimedop                    192
+#define TARGET_NR_semop                         193
+#define TARGET_NR_shmget                        194
+#define TARGET_NR_shmctl                        195
+#define TARGET_NR_shmat                         196
+#define TARGET_NR_shmdt                         197
+#define TARGET_NR_socket                        198
+#define TARGET_NR_socketpair                    199
+#define TARGET_NR_bind                          200
+#define TARGET_NR_listen                        201
+#define TARGET_NR_accept                        202
+#define TARGET_NR_connect                       203
+#define TARGET_NR_getsockname                   204
+#define TARGET_NR_getpeername                   205
+#define TARGET_NR_sendto                        206
+#define TARGET_NR_recvfrom                      207
+#define TARGET_NR_setsockopt                    208
+#define TARGET_NR_getsockopt                    209
+#define TARGET_NR_shutdown                      210
+#define TARGET_NR_sendmsg                       211
+#define TARGET_NR_recvmsg                       212
+#define TARGET_NR_readahead                     213
+#define TARGET_NR_brk                           214
+#define TARGET_NR_munmap                        215
+#define TARGET_NR_mremap                        216
+#define TARGET_NR_add_key                       217
+#define TARGET_NR_request_key                   218
+#define TARGET_NR_keyctl                        219
+#define TARGET_NR_clone                         220
+#define TARGET_NR_execve                        221
+#define TARGET_NR_mmap                          222
+#define TARGET_NR_fadvise64                     223
+#define TARGET_NR_swapon                        224
+#define TARGET_NR_swapoff                       225
+#define TARGET_NR_mprotect                      226
+#define TARGET_NR_msync                         227
+#define TARGET_NR_mlock                         228
+#define TARGET_NR_munlock                       229
+#define TARGET_NR_mlockall                      230
+#define TARGET_NR_munlockall                    231
+#define TARGET_NR_mincore                       232
+#define TARGET_NR_madvise                       233
+#define TARGET_NR_remap_file_pages              234
+#define TARGET_NR_mbind                         235
+#define TARGET_NR_get_mempolicy                 236
+#define TARGET_NR_set_mempolicy                 237
+#define TARGET_NR_migrate_pages                 238
+#define TARGET_NR_move_pages                    239
+#define TARGET_NR_rt_tgsigqueueinfo             240
+#define TARGET_NR_perf_event_open               241
+#define TARGET_NR_accept4                       242
+#define TARGET_NR_recvmmsg                      243
+
+#define TARGET_NR_arch_specific_syscall         244
+#define TARGET_NR_cacheflush                    245  /* tilegx own syscall */
+
+#define TARGET_NR_wait4                         260
+#define TARGET_NR_prlimit64                     261
+#define TARGET_NR_fanotify_init                 262
+#define TARGET_NR_fanotify_mark                 263
+#define TARGET_NR_name_to_handle_at             264
+#define TARGET_NR_open_by_handle_at             265
+#define TARGET_NR_clock_adjtime                 266
+#define TARGET_NR_syncfs                        267
+#define TARGET_NR_setns                         268
+#define TARGET_NR_sendmmsg                      269
+#define TARGET_NR_process_vm_readv              270
+#define TARGET_NR_process_vm_writev             271
+#define TARGET_NR_kcmp                          272
+#define TARGET_NR_finit_module                  273
+#define TARGET_NR_sched_setattr                 274
+#define TARGET_NR_sched_getattr                 275
+#define TARGET_NR_renameat2                     276
+#define TARGET_NR_seccomp                       277
+#define TARGET_NR_getrandom                     278
+#define TARGET_NR_memfd_create                  279
+#define TARGET_NR_bpf                           280
+#define TARGET_NR_execveat                      281
+#define TARGET_NR_userfaultfd                   282
+#define TARGET_NR_membarrier                    283
+#define TARGET_NR_mlock2                        284
+#define TARGET_NR_copy_file_range               285
+
+#define TARGET_NR_open                          1024
+#define TARGET_NR_link                          1025
+#define TARGET_NR_unlink                        1026
+#define TARGET_NR_mknod                         1027
+#define TARGET_NR_chmod                         1028
+#define TARGET_NR_chown                         1029
+#define TARGET_NR_mkdir                         1030
+#define TARGET_NR_rmdir                         1031
+#define TARGET_NR_lchown                        1032
+#define TARGET_NR_access                        1033
+#define TARGET_NR_rename                        1034
+#define TARGET_NR_readlink                      1035
+#define TARGET_NR_symlink                       1036
+#define TARGET_NR_utimes                        1037
+#define TARGET_NR_stat64                        1038 /* let syscall.c known */
+#define TARGET_NR_lstat                         1039
+
+#define TARGET_NR_pipe                          1040
+#define TARGET_NR_dup2                          1041
+#define TARGET_NR_epoll_create                  1042
+#define TARGET_NR_inotify_init                  1043
+#define TARGET_NR_eventfd                       1044
+#define TARGET_NR_signalfd                      1045
+
+#define TARGET_NR_alarm                         1059
+#define TARGET_NR_getpgrp                       1060
+#define TARGET_NR_pause                         1061
+#define TARGET_NR_time                          1062
+#define TARGET_NR_utime                         1063
+#define TARGET_NR_creat                         1064
+#define TARGET_NR_getdents                      1065
+#define TARGET_NR_futimesat                     1066
+#define TARGET_NR_select                        1067
+#define TARGET_NR_poll                          1068
+#define TARGET_NR_epoll_wait                    1069
+#define TARGET_NR_ustat                         1070
+#define TARGET_NR_vfork                         1071
+#define TARGET_NR_oldwait4                      1072
+#define TARGET_NR_recv                          1073
+#define TARGET_NR_send                          1074
+#define TARGET_NR_bdflush                       1075
+#define TARGET_NR_umount                        1076
+#define TARGET_NR_uselib                        1077
+#define TARGET_NR__sysctl                       1078
+#define TARGET_NR_fork                          1079
+
+#endif
diff --git a/qemu/linux-user/tilegx/target_cpu.h b/qemu/linux-user/tilegx/target_cpu.h
new file mode 100644
index 000000000..c96e81d05
--- /dev/null
+++ b/qemu/linux-user/tilegx/target_cpu.h
@@ -0,0 +1,35 @@
+/*
+ * TILE-Gx specific CPU ABI and functions for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TARGET_CPU_H
+#define TARGET_CPU_H
+
+static inline void cpu_clone_regs(CPUTLGState *env, target_ulong newsp)
+{
+    if (newsp) {
+        env->regs[TILEGX_R_SP] = newsp;
+    }
+    env->regs[TILEGX_R_RE] = 0;
+}
+
+static inline void cpu_set_tls(CPUTLGState *env, target_ulong newtls)
+{
+    env->regs[TILEGX_R_TP] = newtls;
+}
+
+#endif
diff --git a/qemu/linux-user/tilegx/target_signal.h b/qemu/linux-user/tilegx/target_signal.h
new file mode 100644
index 000000000..b595f985c
--- /dev/null
+++ b/qemu/linux-user/tilegx/target_signal.h
@@ -0,0 +1,28 @@
+#ifndef TARGET_SIGNAL_H
+#define TARGET_SIGNAL_H
+
+#include "cpu.h"
+
+/* this struct defines a stack used during syscall handling */
+
+typedef struct target_sigaltstack {
+    abi_ulong ss_sp;
+    abi_int ss_flags;
+    abi_ulong ss_size;
+} target_stack_t;
+
+/*
+ * sigaltstack controls
+ */
+#define TARGET_SS_ONSTACK     1
+#define TARGET_SS_DISABLE     2
+
+#define TARGET_MINSIGSTKSZ    2048
+#define TARGET_SIGSTKSZ       8192
+
+static inline abi_ulong get_sp_from_cpustate(CPUTLGState *state)
+{
+    return state->regs[TILEGX_R_SP];
+}
+
+#endif /* TARGET_SIGNAL_H */
diff --git a/qemu/linux-user/tilegx/target_structs.h b/qemu/linux-user/tilegx/target_structs.h
new file mode 100644
index 000000000..7d3ff782f
--- /dev/null
+++ b/qemu/linux-user/tilegx/target_structs.h
@@ -0,0 +1,46 @@
+/*
+ * TILE-Gx specific structures for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TARGET_STRUCTS_H
+#define TARGET_STRUCTS_H
+
+struct target_ipc_perm {
+    abi_int __key;                      /* Key.  */
+    abi_uint uid;                       /* Owner's user ID.  */
+    abi_uint gid;                       /* Owner's group ID.  */
+    abi_uint cuid;                      /* Creator's user ID.  */
+    abi_uint cgid;                      /* Creator's group ID.  */
+    abi_uint mode;                      /* Read/write permission.  */
+    abi_ushort __seq;                   /* Sequence number.  */
+};
+
+struct target_shmid_ds {
+    struct target_ipc_perm shm_perm;    /* operation permission struct */
+    abi_long shm_segsz;                 /* size of segment in bytes */
+    abi_ulong shm_atime;                /* time of last shmat() */
+    abi_ulong shm_dtime;                /* time of last shmdt() */
+    abi_ulong shm_ctime;                /* time of last change by shmctl() */
+    abi_int shm_cpid;                   /* pid of creator */
+    abi_int shm_lpid;                   /* pid of last shmop */
+    abi_ushort shm_nattch;              /* number of current attaches */
+    abi_ushort shm_unused;              /* compatibility */
+    abi_ulong __unused4;
+    abi_ulong __unused5;
+};
+
+#endif
diff --git a/qemu/linux-user/tilegx/target_syscall.h b/qemu/linux-user/tilegx/target_syscall.h
new file mode 100644
index 000000000..a938d4e90
--- /dev/null
+++ b/qemu/linux-user/tilegx/target_syscall.h
@@ -0,0 +1,43 @@
+#ifndef TILEGX_SYSCALLS_H
+#define TILEGX_SYSCALLS_H
+
+#define UNAME_MACHINE "tilegx"
+#define UNAME_MINIMUM_RELEASE "3.19"
+
+#define MMAP_SHIFT TARGET_PAGE_BITS
+
+#define TILEGX_IS_ERRNO(ret) \
+                       ((ret) > 0xfffffffffffff000ULL) /* errno is 0 -- 4096 */
+
+typedef uint64_t tilegx_reg_t;
+
+struct target_pt_regs {
+
+    union {
+        /* Saved main processor registers; 56..63 are special. */
+        tilegx_reg_t regs[56];
+        struct {
+            tilegx_reg_t __regs[53];
+            tilegx_reg_t tp;    /* aliases regs[TREG_TP] */
+            tilegx_reg_t sp;    /* aliases regs[TREG_SP] */
+            tilegx_reg_t lr;    /* aliases regs[TREG_LR] */
+        };
+    };
+
+    /* Saved special registers. */
+    tilegx_reg_t pc;            /* stored in EX_CONTEXT_K_0 */
+    tilegx_reg_t ex1;           /* stored in EX_CONTEXT_K_1 (PL and ICS bit) */
+    tilegx_reg_t faultnum;      /* fault number (INT_SWINT_1 for syscall) */
+    tilegx_reg_t orig_r0;       /* r0 at syscall entry, else zero */
+    tilegx_reg_t flags;         /* flags (see below) */
+    tilegx_reg_t cmpexch;       /* value of CMPEXCH_VALUE SPR at interrupt */
+    tilegx_reg_t pad[2];
+};
+
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+/* For faultnum */
+#define TARGET_INT_SWINT_1            14
+
+#endif
diff --git a/qemu/linux-user/tilegx/termbits.h b/qemu/linux-user/tilegx/termbits.h
new file mode 100644
index 000000000..91ec23654
--- /dev/null
+++ b/qemu/linux-user/tilegx/termbits.h
@@ -0,0 +1,274 @@
+#ifndef TILEGX_TERMBITS_H
+#define TILEGX_TERMBITS_H
+
+/* From asm-generic/termbits.h, which is used by tilegx */
+
+#define TARGET_NCCS 19
+struct target_termios {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+};
+
+struct target_termios2 {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+    unsigned int c_ispeed;            /* input speed */
+    unsigned int c_ospeed;            /* output speed */
+};
+
+/* c_cc characters */
+#define TARGET_VINTR     0
+#define TARGET_VQUIT     1
+#define TARGET_VERASE    2
+#define TARGET_VKILL     3
+#define TARGET_VEOF      4
+#define TARGET_VTIME     5
+#define TARGET_VMIN      6
+#define TARGET_VSWTC     7
+#define TARGET_VSTART    8
+#define TARGET_VSTOP     9
+#define TARGET_VSUSP     10
+#define TARGET_VEOL      11
+#define TARGET_VREPRINT  12
+#define TARGET_VDISCARD  13
+#define TARGET_VWERASE   14
+#define TARGET_VLNEXT    15
+#define TARGET_VEOL2     16
+
+/* c_iflag bits */
+#define TARGET_IGNBRK    0000001
+#define TARGET_BRKINT    0000002
+#define TARGET_IGNPAR    0000004
+#define TARGET_PARMRK    0000010
+#define TARGET_INPCK     0000020
+#define TARGET_ISTRIP    0000040
+#define TARGET_INLCR     0000100
+#define TARGET_IGNCR     0000200
+#define TARGET_ICRNL     0000400
+#define TARGET_IUCLC     0001000
+#define TARGET_IXON      0002000
+#define TARGET_IXANY     0004000
+#define TARGET_IXOFF     0010000
+#define TARGET_IMAXBEL   0020000
+#define TARGET_IUTF8     0040000
+
+/* c_oflag bits */
+#define TARGET_OPOST     0000001
+#define TARGET_OLCUC     0000002
+#define TARGET_ONLCR     0000004
+#define TARGET_OCRNL     0000010
+#define TARGET_ONOCR     0000020
+#define TARGET_ONLRET    0000040
+#define TARGET_OFILL     0000100
+#define TARGET_OFDEL     0000200
+#define TARGET_NLDLY     0000400
+#define   TARGET_NL0     0000000
+#define   TARGET_NL1     0000400
+#define TARGET_CRDLY     0003000
+#define   TARGET_CR0     0000000
+#define   TARGET_CR1     0001000
+#define   TARGET_CR2     0002000
+#define   TARGET_CR3     0003000
+#define TARGET_TABDLY    0014000
+#define   TARGET_TAB0    0000000
+#define   TARGET_TAB1    0004000
+#define   TARGET_TAB2    0010000
+#define   TARGET_TAB3    0014000
+#define   TARGET_XTABS   0014000
+#define TARGET_BSDLY     0020000
+#define   TARGET_BS0     0000000
+#define   TARGET_BS1     0020000
+#define TARGET_VTDLY     0040000
+#define   TARGET_VT0     0000000
+#define   TARGET_VT1     0040000
+#define TARGET_FFDLY     0100000
+#define   TARGET_FF0     0000000
+#define   TARGET_FF1     0100000
+
+/* c_cflag bit meaning */
+#define TARGET_CBAUD     0010017
+#define  TARGET_B0       0000000        /* hang up */
+#define  TARGET_B50      0000001
+#define  TARGET_B75      0000002
+#define  TARGET_B110     0000003
+#define  TARGET_B134     0000004
+#define  TARGET_B150     0000005
+#define  TARGET_B200     0000006
+#define  TARGET_B300     0000007
+#define  TARGET_B600     0000010
+#define  TARGET_B1200    0000011
+#define  TARGET_B1800    0000012
+#define  TARGET_B2400    0000013
+#define  TARGET_B4800    0000014
+#define  TARGET_B9600    0000015
+#define  TARGET_B19200   0000016
+#define  TARGET_B38400   0000017
+#define TARGET_EXTA      TARGET_B19200
+#define TARGET_EXTB      TARGET_B38400
+#define TARGET_CSIZE     0000060
+#define   TARGET_CS5     0000000
+#define   TARGET_CS6     0000020
+#define   TARGET_CS7     0000040
+#define   TARGET_CS8     0000060
+#define TARGET_CSTOPB    0000100
+#define TARGET_CREAD     0000200
+#define TARGET_PARENB    0000400
+#define TARGET_PARODD    0001000
+#define TARGET_HUPCL     0002000
+#define TARGET_CLOCAL    0004000
+#define TARGET_CBAUDEX   0010000
+#define    TARGET_BOTHER 0010000
+#define    TARGET_B57600 0010001
+#define   TARGET_B115200 0010002
+#define   TARGET_B230400 0010003
+#define   TARGET_B460800 0010004
+#define   TARGET_B500000 0010005
+#define   TARGET_B576000 0010006
+#define   TARGET_B921600 0010007
+#define  TARGET_B1000000 0010010
+#define  TARGET_B1152000 0010011
+#define  TARGET_B1500000 0010012
+#define  TARGET_B2000000 0010013
+#define  TARGET_B2500000 0010014
+#define  TARGET_B3000000 0010015
+#define  TARGET_B3500000 0010016
+#define  TARGET_B4000000 0010017
+#define TARGET_CIBAUD    002003600000    /* input baud rate */
+#define TARGET_CMSPAR    010000000000    /* mark or space (stick) parity */
+#define TARGET_CRTSCTS   020000000000    /* flow control */
+
+#define TARGET_IBSHIFT   16        /* Shift from CBAUD to CIBAUD */
+
+/* c_lflag bits */
+#define TARGET_ISIG      0000001
+#define TARGET_ICANON    0000002
+#define TARGET_XCASE     0000004
+#define TARGET_ECHO      0000010
+#define TARGET_ECHOE     0000020
+#define TARGET_ECHOK     0000040
+#define TARGET_ECHONL    0000100
+#define TARGET_NOFLSH    0000200
+#define TARGET_TOSTOP    0000400
+#define TARGET_ECHOCTL   0001000
+#define TARGET_ECHOPRT   0002000
+#define TARGET_ECHOKE    0004000
+#define TARGET_FLUSHO    0010000
+#define TARGET_PENDIN    0040000
+#define TARGET_IEXTEN    0100000
+#define TARGET_EXTPROC   0200000
+
+/* tcflow() and TCXONC use these */
+#define TARGET_TCOOFF    0
+#define TARGET_TCOON     1
+#define TARGET_TCIOFF    2
+#define TARGET_TCION     3
+
+/* tcflush() and TCFLSH use these */
+#define TARGET_TCIFLUSH  0
+#define TARGET_TCOFLUSH  1
+#define TARGET_TCIOFLUSH 2
+
+/* tcsetattr uses these */
+#define TARGET_TCSANOW   0
+#define TARGET_TCSADRAIN 1
+#define TARGET_TCSAFLUSH 2
+
+/* From asm-generic/ioctls.h, which is used by tilegx */
+
+#define TARGET_TCGETS                   0x5401
+#define TARGET_TCSETS                   0x5402
+#define TARGET_TCSETSW                  0x5403
+#define TARGET_TCSETSF                  0x5404
+#define TARGET_TCGETA                   0x5405
+#define TARGET_TCSETA                   0x5406
+#define TARGET_TCSETAW                  0x5407
+#define TARGET_TCSETAF                  0x5408
+#define TARGET_TCSBRK                   0x5409
+#define TARGET_TCXONC                   0x540A
+#define TARGET_TCFLSH                   0x540B
+#define TARGET_TIOCEXCL                 0x540C
+#define TARGET_TIOCNXCL                 0x540D
+#define TARGET_TIOCSCTTY                0x540E
+#define TARGET_TIOCGPGRP                0x540F
+#define TARGET_TIOCSPGRP                0x5410
+#define TARGET_TIOCOUTQ                 0x5411
+#define TARGET_TIOCSTI                  0x5412
+#define TARGET_TIOCGWINSZ               0x5413
+#define TARGET_TIOCSWINSZ               0x5414
+#define TARGET_TIOCMGET                 0x5415
+#define TARGET_TIOCMBIS                 0x5416
+#define TARGET_TIOCMBIC                 0x5417
+#define TARGET_TIOCMSET                 0x5418
+#define TARGET_TIOCGSOFTCAR             0x5419
+#define TARGET_TIOCSSOFTCAR             0x541A
+#define TARGET_FIONREAD                 0x541B
+#define TARGET_TIOCINQ                  TARGET_FIONREAD
+#define TARGET_TIOCLINUX                0x541C
+#define TARGET_TIOCCONS                 0x541D
+#define TARGET_TIOCGSERIAL              0x541E
+#define TARGET_TIOCSSERIAL              0x541F
+#define TARGET_TIOCPKT                  0x5420
+#define TARGET_FIONBIO                  0x5421
+#define TARGET_TIOCNOTTY                0x5422
+#define TARGET_TIOCSETD                 0x5423
+#define TARGET_TIOCGETD                 0x5424
+#define TARGET_TCSBRKP                  0x5425
+#define TARGET_TIOCSBRK                 0x5427
+#define TARGET_TIOCCBRK                 0x5428
+#define TARGET_TIOCGSID                 0x5429
+#define TARGET_TCGETS2                  TARGET_IOR('T', 0x2A, struct termios2)
+#define TARGET_TCSETS2                  TARGET_IOW('T', 0x2B, struct termios2)
+#define TARGET_TCSETSW2                 TARGET_IOW('T', 0x2C, struct termios2)
+#define TARGET_TCSETSF2                 TARGET_IOW('T', 0x2D, struct termios2)
+#define TARGET_TIOCGRS485               0x542E
+#define TARGET_TIOCSRS485               0x542F
+#define TARGET_TIOCGPTN                 TARGET_IOR('T', 0x30, unsigned int)
+#define TARGET_TIOCSPTLCK               TARGET_IOW('T', 0x31, int)
+#define TARGET_TIOCGDEV                 TARGET_IOR('T', 0x32, unsigned int)
+#define TARGET_TCGETX                   0x5432
+#define TARGET_TCSETX                   0x5433
+#define TARGET_TCSETXF                  0x5434
+#define TARGET_TCSETXW                  0x5435
+#define TARGET_TIOCSIG                  TARGET_IOW('T', 0x36, int)
+#define TARGET_TIOCVHANGUP              0x5437
+#define TARGET_TIOCGPKT                 TARGET_IOR('T', 0x38, int)
+#define TARGET_TIOCGPTLCK               TARGET_IOR('T', 0x39, int)
+#define TARGET_TIOCGEXCL                TARGET_IOR('T', 0x40, int)
+
+#define TARGET_FIONCLEX                 0x5450
+#define TARGET_FIOCLEX                  0x5451
+#define TARGET_FIOASYNC                 0x5452
+#define TARGET_TIOCSERCONFIG            0x5453
+#define TARGET_TIOCSERGWILD             0x5454
+#define TARGET_TIOCSERSWILD             0x5455
+#define TARGET_TIOCGLCKTRMIOS           0x5456
+#define TARGET_TIOCSLCKTRMIOS           0x5457
+#define TARGET_TIOCSERGSTRUCT           0x5458
+#define TARGET_TIOCSERGETLSR            0x5459
+#define TARGET_TIOCSERGETMULTI          0x545A
+#define TARGET_TIOCSERSETMULTI          0x545B
+
+#define TARGET_TIOCMIWAIT               0x545C
+#define TARGET_TIOCGICOUNT              0x545D
+#define TARGET_FIOQSIZE                 0x5460
+
+#define TARGET_TIOCPKT_DATA             0
+#define TARGET_TIOCPKT_FLUSHREAD        1
+#define TARGET_TIOCPKT_FLUSHWRITE       2
+#define TARGET_TIOCPKT_STOP             4
+#define TARGET_TIOCPKT_START            8
+#define TARGET_TIOCPKT_NOSTOP           16
+#define TARGET_TIOCPKT_DOSTOP           32
+#define TARGET_TIOCPKT_IOCTL            64
+
+#define TARGET_TIOCSER_TEMT             0x01
+
+#endif
diff --git a/qemu/linux-user/uaccess.c b/qemu/linux-user/uaccess.c
index a4d108c2f..0a5c0b0b2 100644
--- a/qemu/linux-user/uaccess.c
+++ b/qemu/linux-user/uaccess.c
@@ -1,6 +1,6 @@
 /* User memory access */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "qemu.h"
 
diff --git a/qemu/linux-user/uname.c b/qemu/linux-user/uname.c
index 1e6560d72..313b79dba 100644
--- a/qemu/linux-user/uname.c
+++ b/qemu/linux-user/uname.c
@@ -17,7 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 //#include "qemu-common.h"
diff --git a/qemu/linux-user/unicore32/target_signal.h b/qemu/linux-user/unicore32/target_signal.h
index 8b255c455..7c442381a 100644
--- a/qemu/linux-user/unicore32/target_signal.h
+++ b/qemu/linux-user/unicore32/target_signal.h
@@ -21,6 +21,10 @@ typedef struct target_sigaltstack {
 #define TARGET_SS_ONSTACK               1
 #define TARGET_SS_DISABLE               2
 
-#define get_sp_from_cpustate(cpustate)  (cpustate->regs[29])
+static inline abi_ulong get_sp_from_cpustate(CPUUniCore32State *state)
+{
+    return state->regs[29];
+}
+
 
 #endif /* TARGET_SIGNAL_H */
diff --git a/qemu/linux-user/unicore32/syscall.h b/qemu/linux-user/unicore32/target_syscall.h
index 385a97562..385a97562 100644
--- a/qemu/linux-user/unicore32/syscall.h
+++ b/qemu/linux-user/unicore32/target_syscall.h
diff --git a/qemu/linux-user/vm86.c b/qemu/linux-user/vm86.c
index 22a4eb962..3829b9a67 100644
--- a/qemu/linux-user/vm86.c
+++ b/qemu/linux-user/vm86.c
@@ -16,12 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
diff --git a/qemu/linux-user/x86_64/syscall_nr.h b/qemu/linux-user/x86_64/syscall_nr.h
index 7c59e3a09..16397b3e8 100644
--- a/qemu/linux-user/x86_64/syscall_nr.h
+++ b/qemu/linux-user/x86_64/syscall_nr.h
@@ -281,7 +281,7 @@
 #define TARGET_NR_utimensat		280
 #define TARGET_NR_epoll_pwait	281
 #define TARGET_NR_signalfd		282
-#define TARGET_NR_timerfd		283
+#define TARGET_NR_timerfd_create	283
 #define TARGET_NR_eventfd		284
 #define TARGET_NR_fallocate		285
 #define TARGET_NR_timerfd_settime	286
@@ -312,3 +312,16 @@
 #define TARGET_NR_process_vm_writev     311
 #define TARGET_NR_kcmp                  312
 #define TARGET_NR_finit_module          313
+#define TARGET_NR_sched_setattr         314
+#define TARGET_NR_sched_getattr         315
+#define TARGET_NR_renameat2             316
+#define TARGET_NR_seccomp               317
+#define TARGET_NR_getrandom             318
+#define TARGET_NR_memfd_create          319
+#define TARGET_NR_kexec_file_load       320
+#define TARGET_NR_bpf                   321
+#define TARGET_NR_execveat              322
+#define TARGET_NR_userfaultfd           323
+#define TARGET_NR_membarrier            324
+#define TARGET_NR_mlock2                325
+#define TARGET_NR_copy_file_range       326
diff --git a/qemu/linux-user/x86_64/syscall.h b/qemu/linux-user/x86_64/target_syscall.h
index 88b3c3fe3..feecd32d5 100644
--- a/qemu/linux-user/x86_64/syscall.h
+++ b/qemu/linux-user/x86_64/target_syscall.h
@@ -1,3 +1,6 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
 #define __USER_CS	(0x33)
 #define __USER_DS	(0x2B)
 
@@ -100,3 +103,5 @@ struct target_msqid64_ds {
 #define TARGET_MINSIGSTKSZ 2048
 #define TARGET_MLOCKALL_MCL_CURRENT 1
 #define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif  /* TARGET_SYSCALL_H */
diff --git a/qemu/main-loop.c b/qemu/main-loop.c
index 39970437f..89a699419 100644
--- a/qemu/main-loop.c
+++ b/qemu/main-loop.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"	// struct in_addr needed for libslirp.h
 #include "sysemu/qtest.h"
@@ -161,6 +163,9 @@ int qemu_init_main_loop(Error **errp)
     src = aio_get_g_source(qemu_aio_context);
     g_source_attach(src, NULL);
     g_source_unref(src);
+    src = iohandler_get_g_source();
+    g_source_attach(src, NULL);
+    g_source_unref(src);
     return 0;
 }
 
@@ -227,7 +232,7 @@ static int os_host_main_loop_wait(int64_t timeout)
     if (!timeout && (spin_counter > MAX_MAIN_LOOP_SPIN)) {
         static bool notified;
 
-        if (!notified && !qtest_enabled()) {
+        if (!notified && !qtest_driver()) {
             fprintf(stderr,
                     "main-loop: WARNING: I/O thread spun for %d iterations\n",
                     MAX_MAIN_LOOP_SPIN);
@@ -487,7 +492,6 @@ int main_loop_wait(int nonblocking)
 #ifdef CONFIG_SLIRP
     slirp_pollfds_fill(gpollfds, &timeout);
 #endif
-    qemu_iohandler_fill(gpollfds);
 
     if (timeout == UINT32_MAX) {
         timeout_ns = -1;
@@ -500,11 +504,13 @@ int main_loop_wait(int nonblocking)
                                           &main_loop_tlg));
 
     ret = os_host_main_loop_wait(timeout_ns);
-    qemu_iohandler_poll(gpollfds, ret);
 #ifdef CONFIG_SLIRP
     slirp_pollfds_poll(gpollfds, (ret < 0));
 #endif
 
+    /* CPU thread can infinitely wait for event after
+       missing the warp */
+    qemu_start_warp_timer();
     qemu_clock_run_all_timers();
 
     return ret;
diff --git a/qemu/memory.c b/qemu/memory.c
index 4eb138a42..f76f85df9 100644
--- a/qemu/memory.c
+++ b/qemu/memory.c
@@ -13,17 +13,20 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "exec/ioport.h"
 #include "qapi/visitor.h"
 #include "qemu/bitops.h"
+#include "qemu/error-report.h"
 #include "qom/object.h"
 #include "trace.h"
-#include <assert.h>
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 
 //#define DEBUG_UNASSIGNED
@@ -370,6 +373,28 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
     }
 }
 
+static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
+{
+    MemoryRegion *root;
+    hwaddr abs_addr = offset;
+
+    abs_addr += mr->addr;
+    for (root = mr; root->container; ) {
+        root = root->container;
+        abs_addr += root->addr;
+    }
+
+    return abs_addr;
+}
+
+static int get_cpu_index(void)
+{
+    if (current_cpu) {
+        return current_cpu->cpu_index;
+    }
+    return -1;
+}
+
 static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr,
                                                        hwaddr addr,
                                                        uint64_t *value,
@@ -381,7 +406,17 @@ static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr,
     uint64_t tmp;
 
     tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr);
-    trace_memory_region_ops_read(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     *value |= (tmp & mask) << shift;
     return MEMTX_OK;
 }
@@ -397,7 +432,17 @@ static MemTxResult  memory_region_read_accessor(MemoryRegion *mr,
     uint64_t tmp;
 
     tmp = mr->ops->read(mr->opaque, addr, size);
-    trace_memory_region_ops_read(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     *value |= (tmp & mask) << shift;
     return MEMTX_OK;
 }
@@ -414,7 +459,17 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
     MemTxResult r;
 
     r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
-    trace_memory_region_ops_read(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     *value |= (tmp & mask) << shift;
     return r;
 }
@@ -430,7 +485,17 @@ static MemTxResult memory_region_oldmmio_write_accessor(MemoryRegion *mr,
     uint64_t tmp;
 
     tmp = (*value >> shift) & mask;
-    trace_memory_region_ops_write(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp);
     return MEMTX_OK;
 }
@@ -446,7 +511,17 @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
     uint64_t tmp;
 
     tmp = (*value >> shift) & mask;
-    trace_memory_region_ops_write(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     mr->ops->write(mr->opaque, addr, tmp, size);
     return MEMTX_OK;
 }
@@ -462,7 +537,17 @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
     uint64_t tmp;
 
     tmp = (*value >> shift) & mask;
-    trace_memory_region_ops_write(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
 }
 
@@ -856,17 +941,12 @@ static void memory_region_destructor_none(MemoryRegion *mr)
 
 static void memory_region_destructor_ram(MemoryRegion *mr)
 {
-    qemu_ram_free(mr->ram_addr);
-}
-
-static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr)
-{
-    qemu_ram_free_from_ptr(mr->ram_addr);
+    qemu_ram_free(mr->ram_block);
 }
 
 static void memory_region_destructor_rom_device(MemoryRegion *mr)
 {
-    qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK);
+    qemu_ram_free(mr->ram_block);
 }
 
 static bool memory_region_need_escape(char c)
@@ -908,20 +988,23 @@ void memory_region_init(MemoryRegion *mr,
                         const char *name,
                         uint64_t size)
 {
-    if (!owner) {
-        owner = container_get(qdev_get_machine(), "/unattached");
-    }
-
     object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
     mr->size = int128_make64(size);
     if (size == UINT64_MAX) {
         mr->size = int128_2_64();
     }
     mr->name = g_strdup(name);
+    mr->owner = owner;
+    mr->ram_block = NULL;
 
     if (name) {
         char *escaped_name = memory_region_escape_name(name);
         char *name_array = g_strdup_printf("%s[*]", escaped_name);
+
+        if (!owner) {
+            owner = container_get(qdev_get_machine(), "/unattached");
+        }
+
         object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
         object_unref(OBJECT(mr));
         g_free(name_array);
@@ -929,17 +1012,18 @@ void memory_region_init(MemoryRegion *mr,
     }
 }
 
-static void memory_region_get_addr(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = mr->addr;
 
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
-static void memory_region_get_container(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void memory_region_get_container(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     gchar *path = (gchar *)"";
@@ -947,7 +1031,7 @@ static void memory_region_get_container(Object *obj, Visitor *v, void *opaque,
     if (mr->container) {
         path = object_get_canonical_path(OBJECT(mr->container));
     }
-    visit_type_str(v, &path, name, errp);
+    visit_type_str(v, name, &path, errp);
     if (mr->container) {
         g_free(path);
     }
@@ -961,13 +1045,14 @@ static Object *memory_region_resolve_container(Object *obj, void *opaque,
     return OBJECT(mr->container);
 }
 
-static void memory_region_get_priority(Object *obj, Visitor *v, void *opaque,
-                                       const char *name, Error **errp)
+static void memory_region_get_priority(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     int32_t value = mr->priority;
 
-    visit_type_int32(v, &value, name, errp);
+    visit_type_int32(v, name, &value, errp);
 }
 
 static bool memory_region_get_may_overlap(Object *obj, Error **errp)
@@ -977,13 +1062,13 @@ static bool memory_region_get_may_overlap(Object *obj, Error **errp)
     return mr->may_overlap;
 }
 
-static void memory_region_get_size(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = memory_region_size(mr);
 
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 static void memory_region_initfn(Object *obj)
@@ -992,7 +1077,6 @@ static void memory_region_initfn(Object *obj)
     ObjectProperty *op;
 
     mr->ops = &unassigned_mem_ops;
-    mr->ram_addr = RAM_ADDR_INVALID;
     mr->enabled = true;
     mr->romd_mode = true;
     mr->global_locking = true;
@@ -1141,6 +1225,32 @@ MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
     return r;
 }
 
+/* Return true if an eventfd was signalled */
+static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
+                                                    hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size,
+                                                    MemTxAttrs attrs)
+{
+    MemoryRegionIoeventfd ioeventfd = {
+        .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
+        .data = data,
+    };
+    unsigned i;
+
+    for (i = 0; i < mr->ioeventfd_nb; i++) {
+        ioeventfd.match_data = mr->ioeventfds[i].match_data;
+        ioeventfd.e = mr->ioeventfds[i].e;
+
+        if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) {
+            event_notifier_set(ioeventfd.e);
+            return true;
+        }
+    }
+
+    return false;
+}
+
 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
                                          hwaddr addr,
                                          uint64_t data,
@@ -1154,6 +1264,11 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
 
     adjust_endianness(mr, &data, size);
 
+    if ((!kvm_eventfds_enabled()) &&
+        memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
+        return MEMTX_OK;
+    }
+
     if (mr->ops->write) {
         return access_with_adjusted_size(addr, &data, size,
                                          mr->ops->impl.min_access_size,
@@ -1182,7 +1297,7 @@ void memory_region_init_io(MemoryRegion *mr,
                            uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
-    mr->ops = ops;
+    mr->ops = ops ? ops : &unassigned_mem_ops;
     mr->opaque = opaque;
     mr->terminates = true;
 }
@@ -1197,7 +1312,7 @@ void memory_region_init_ram(MemoryRegion *mr,
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
+    mr->ram_block = qemu_ram_alloc(size, mr, errp);
     mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 }
 
@@ -1215,7 +1330,8 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr,
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc_resizeable(size, max_size, resized, mr, errp);
+    mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized,
+                                              mr, errp);
     mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 }
 
@@ -1232,7 +1348,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc_from_file(size, mr, share, path, errp);
+    mr->ram_block = qemu_ram_alloc_from_file(size, mr, share, path, errp);
     mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 }
 #endif
@@ -1246,12 +1362,12 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
-    mr->destructor = memory_region_destructor_ram_from_ptr;
+    mr->destructor = memory_region_destructor_ram;
     mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
-    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
+    mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
 }
 
 void memory_region_set_skip_dump(MemoryRegion *mr)
@@ -1285,7 +1401,7 @@ void memory_region_init_rom_device(MemoryRegion *mr,
     mr->terminates = true;
     mr->rom_device = true;
     mr->destructor = memory_region_destructor_rom_device;
-    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
+    mr->ram_block = qemu_ram_alloc(size, mr, errp);
 }
 
 void memory_region_init_iommu(MemoryRegion *mr,
@@ -1300,19 +1416,26 @@ void memory_region_init_iommu(MemoryRegion *mr,
     notifier_list_init(&mr->iommu_notify);
 }
 
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    Object *owner,
-                                    const char *name,
-                                    uint64_t size)
-{
-    memory_region_init_io(mr, owner, &unassigned_mem_ops, mr, name, size);
-}
-
 static void memory_region_finalize(Object *obj)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
 
-    assert(QTAILQ_EMPTY(&mr->subregions));
+    assert(!mr->container);
+
+    /* We know the region is not visible in any address space (it
+     * does not have a container and cannot be a root either because
+     * it has no references, so we can blindly clear mr->enabled.
+     * memory_region_set_enabled instead could trigger a transaction
+     * and cause an infinite loop.
+     */
+    mr->enabled = false;
+    memory_region_transaction_begin();
+    while (!QTAILQ_EMPTY(&mr->subregions)) {
+        MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
+        memory_region_del_subregion(mr, subregion);
+    }
+    memory_region_transaction_commit();
+
     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);
@@ -1334,24 +1457,18 @@ void memory_region_ref(MemoryRegion *mr)
      * The memory region is a child of its owner.  As long as the
      * owner doesn't call unparent itself on the memory region,
      * ref-ing the owner will also keep the memory region alive.
-     * Memory regions without an owner are supposed to never go away,
-     * but we still ref/unref them for debugging purposes.
+     * Memory regions without an owner are supposed to never go away;
+     * we do not ref/unref them because it slows down DMA sensibly.
      */
-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_ref(obj->parent);
-    } else {
-        object_ref(obj);
+    if (mr && mr->owner) {
+        object_ref(mr->owner);
     }
 }
 
 void memory_region_unref(MemoryRegion *mr)
 {
-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_unref(obj->parent);
-    } else {
-        object_unref(obj);
+    if (mr && mr->owner) {
+        object_unref(mr->owner);
     }
 }
 
@@ -1372,11 +1489,6 @@ const char *memory_region_name(const MemoryRegion *mr)
     return mr->name;
 }
 
-bool memory_region_is_ram(MemoryRegion *mr)
-{
-    return mr->ram;
-}
-
 bool memory_region_is_skip_dump(MemoryRegion *mr)
 {
     return mr->skip_dump;
@@ -1396,19 +1508,29 @@ bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)
     return memory_region_get_dirty_log_mask(mr) & (1 << client);
 }
 
-bool memory_region_is_rom(MemoryRegion *mr)
+void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)
 {
-    return mr->ram && mr->readonly;
+    notifier_list_add(&mr->iommu_notify, n);
 }
 
-bool memory_region_is_iommu(MemoryRegion *mr)
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n,
+                                hwaddr granularity, bool is_write)
 {
-    return mr->iommu_ops;
-}
+    hwaddr addr;
+    IOMMUTLBEntry iotlb;
 
-void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)
-{
-    notifier_list_add(&mr->iommu_notify, n);
+    for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
+        if (iotlb.perm != IOMMU_NONE) {
+            n->notify(n, &iotlb);
+        }
+
+        /* if (2^64 - MR size) < granularity, it's possible to get an
+         * infinite loop here.  This should catch such a wraparound */
+        if ((addr + granularity) < addr) {
+            break;
+        }
+    }
 }
 
 void memory_region_unregister_iommu_notifier(Notifier *n)
@@ -1444,24 +1566,26 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size, unsigned client)
 {
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
-    return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
+    assert(mr->ram_block);
+    return cpu_physical_memory_get_dirty(memory_region_get_ram_addr(mr) + addr,
+                                         size, client);
 }
 
 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size)
 {
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
-    cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size,
+    assert(mr->ram_block);
+    cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
+                                        size,
                                         memory_region_get_dirty_log_mask(mr));
 }
 
 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
                                         hwaddr size, unsigned client)
 {
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
-    return cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr,
-                                                    size, client);
+    assert(mr->ram_block);
+    return cpu_physical_memory_test_and_clear_dirty(
+                memory_region_get_ram_addr(mr) + addr, size, client);
 }
 
 
@@ -1504,9 +1628,9 @@ void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
                                hwaddr size, unsigned client)
 {
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
-    cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr, size,
-                                             client);
+    assert(mr->ram_block);
+    cpu_physical_memory_test_and_clear_dirty(
+        memory_region_get_ram_addr(mr) + addr, size, client);
 }
 
 int memory_region_get_fd(MemoryRegion *mr)
@@ -1515,27 +1639,39 @@ int memory_region_get_fd(MemoryRegion *mr)
         return memory_region_get_fd(mr->alias);
     }
 
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    assert(mr->ram_block);
 
-    return qemu_get_ram_fd(mr->ram_addr & TARGET_PAGE_MASK);
+    return qemu_get_ram_fd(memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK);
 }
 
 void *memory_region_get_ram_ptr(MemoryRegion *mr)
 {
-    if (mr->alias) {
-        return memory_region_get_ram_ptr(mr->alias) + mr->alias_offset;
+    void *ptr;
+    uint64_t offset = 0;
+
+    rcu_read_lock();
+    while (mr->alias) {
+        offset += mr->alias_offset;
+        mr = mr->alias;
     }
+    assert(mr->ram_block);
+    ptr = qemu_get_ram_ptr(mr->ram_block,
+                           memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK);
+    rcu_read_unlock();
 
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    return ptr + offset;
+}
 
-    return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK);
+ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
+{
+    return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
 }
 
 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
 {
-    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    assert(mr->ram_block);
 
-    qemu_ram_resize(mr->ram_addr, newsize, errp);
+    qemu_ram_resize(memory_region_get_ram_addr(mr), newsize, errp);
 }
 
 static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
@@ -1645,6 +1781,8 @@ void memory_region_clear_global_locking(MemoryRegion *mr)
     mr->global_locking = false;
 }
 
+static bool userspace_eventfd_warning;
+
 void memory_region_add_eventfd(MemoryRegion *mr,
                                hwaddr addr,
                                unsigned size,
@@ -1661,7 +1799,16 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
+                            userspace_eventfd_warning))) {
+        userspace_eventfd_warning = true;
+        error_report("Using eventfd without MMIO binding in KVM. "
+                     "Suboptimal performance expected");
+    }
+
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
@@ -1694,7 +1841,9 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
@@ -1858,11 +2007,6 @@ void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
     memory_region_transaction_commit();
 }
 
-ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
-{
-    return mr->ram_addr;
-}
-
 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
 {
     return mr->align;
@@ -2009,6 +2153,9 @@ static void listener_add_address_space(MemoryListener *listener,
         return;
     }
 
+    if (listener->begin) {
+        listener->begin(listener);
+    }
     if (global_dirty_log) {
         if (listener->log_global_start) {
             listener->log_global_start(listener);
@@ -2025,10 +2172,16 @@ static void listener_add_address_space(MemoryListener *listener,
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
+        if (fr->dirty_log_mask && listener->log_start) {
+            listener->log_start(listener, &section, 0, fr->dirty_log_mask);
+        }
         if (listener->region_add) {
             listener->region_add(listener, &section);
         }
     }
+    if (listener->commit) {
+        listener->commit(listener);
+    }
     flatview_unref(view);
 }
 
@@ -2065,7 +2218,9 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
 {
     memory_region_ref(root);
     memory_region_transaction_begin();
+    as->ref_count = 1;
     as->root = root;
+    as->malloced = false;
     as->current_map = g_new(FlatView, 1);
     flatview_init(as->current_map);
     as->ioeventfd_nb = 0;
@@ -2080,6 +2235,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
 static void do_address_space_destroy(AddressSpace *as)
 {
     MemoryListener *listener;
+    bool do_free = as->malloced;
 
     address_space_destroy_dispatch(as);
 
@@ -2091,12 +2247,36 @@ static void do_address_space_destroy(AddressSpace *as)
     g_free(as->name);
     g_free(as->ioeventfds);
     memory_region_unref(as->root);
+    if (do_free) {
+        g_free(as);
+    }
+}
+
+AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name)
+{
+    AddressSpace *as;
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (root == as->root && as->malloced) {
+            as->ref_count++;
+            return as;
+        }
+    }
+
+    as = g_malloc0(sizeof *as);
+    address_space_init(as, root, name);
+    as->malloced = true;
+    return as;
 }
 
 void address_space_destroy(AddressSpace *as)
 {
     MemoryRegion *root = as->root;
 
+    as->ref_count--;
+    if (as->ref_count) {
+        return;
+    }
     /* Flush out anything from MemoryListeners listening in on this */
     memory_region_transaction_begin();
     as->root = NULL;
diff --git a/qemu/memory_mapping.c b/qemu/memory_mapping.c
index 36d6b2604..2354b2b7f 100644
--- a/qemu/memory_mapping.c
+++ b/qemu/memory_mapping.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <glib.h>
 
 #include "qemu-common.h"
@@ -177,6 +179,7 @@ void guest_phys_blocks_free(GuestPhysBlockList *list)
 
     QTAILQ_FOREACH_SAFE(p, &list->head, next, q) {
         QTAILQ_REMOVE(&list->head, p, next);
+        memory_region_unref(p->mr);
         g_free(p);
     }
     list->num = 0;
@@ -240,6 +243,8 @@ static void guest_phys_blocks_region_add(MemoryListener *listener,
         block->target_start = target_start;
         block->target_end   = target_end;
         block->host_addr    = host_addr;
+        block->mr           = section->mr;
+        memory_region_ref(section->mr);
 
         QTAILQ_INSERT_TAIL(&g->list->head, block, next);
         ++g->list->num;
diff --git a/qemu/migration/Makefile.objs b/qemu/migration/Makefile.objs
index d929e969a..0cac6d707 100644
--- a/qemu/migration/Makefile.objs
+++ b/qemu/migration/Makefile.objs
@@ -1,7 +1,7 @@
 common-obj-y += migration.o tcp.o
 common-obj-y += vmstate.o
 common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o
-common-obj-y += xbzrle.o
+common-obj-y += xbzrle.o postcopy-ram.o
 
 common-obj-$(CONFIG_RDMA) += rdma.o
 common-obj-$(CONFIG_POSIX) += exec.o unix.o fd.o
diff --git a/qemu/migration/block.c b/qemu/migration/block.c
index 85496fd80..174331728 100644
--- a/qemu/migration/block.c
+++ b/qemu/migration/block.c
@@ -13,18 +13,20 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "hw/hw.h"
+#include "qemu/cutils.h"
 #include "qemu/queue.h"
 #include "qemu/timer.h"
 #include "migration/block.h"
 #include "migration/migration.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
-#include <assert.h>
 
 #define BLOCK_SIZE                       (1 << 20)
 #define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
@@ -36,6 +38,8 @@
 
 #define MAX_IS_ALLOCATED_SEARCH 65536
 
+#define MAX_INFLIGHT_IO 512
+
 //#define DEBUG_BLK_MIGRATION
 
 #ifdef DEBUG_BLK_MIGRATION
@@ -52,17 +56,25 @@ typedef struct BlkMigDevState {
     int shared_base;
     int64_t total_sectors;
     QSIMPLEQ_ENTRY(BlkMigDevState) entry;
+    Error *blocker;
 
     /* Only used by migration thread.  Does not need a lock.  */
     int bulk_completed;
     int64_t cur_sector;
     int64_t cur_dirty;
 
-    /* Protected by block migration lock.  */
+    /* Data in the aio_bitmap is protected by block migration lock.
+     * Allocation and free happen during setup and cleanup respectively.
+     */
     unsigned long *aio_bitmap;
+
+    /* Protected by block migration lock.  */
     int64_t completed_sectors;
+
+    /* During migration this is protected by iothread lock / AioContext.
+     * Allocation and free happen during setup and cleanup respectively.
+     */
     BdrvDirtyBitmap *dirty_bitmap;
-    Error *blocker;
 } BlkMigDevState;
 
 typedef struct BlkMigBlock {
@@ -98,7 +110,7 @@ typedef struct BlkMigState {
     int prev_progress;
     int bulk_completed;
 
-    /* Lock must be taken _inside_ the iothread lock.  */
+    /* Lock must be taken _inside_ the iothread lock and any AioContexts.  */
     QemuMutex lock;
 } BlkMigState;
 
@@ -262,11 +274,13 @@ static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
 
     if (bmds->shared_base) {
         qemu_mutex_lock_iothread();
+        aio_context_acquire(bdrv_get_aio_context(bs));
         while (cur_sector < total_sectors &&
                !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
                                   &nr_sectors)) {
             cur_sector += nr_sectors;
         }
+        aio_context_release(bdrv_get_aio_context(bs));
         qemu_mutex_unlock_iothread();
     }
 
@@ -300,11 +314,21 @@ static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
     block_mig_state.submitted++;
     blk_mig_unlock();
 
+    /* We do not know if bs is under the main thread (and thus does
+     * not acquire the AioContext when doing AIO) or rather under
+     * dataplane.  Thus acquire both the iothread mutex and the
+     * AioContext.
+     *
+     * This is ugly and will disappear when we make bdrv_* thread-safe,
+     * without the need to acquire the AioContext.
+     */
     qemu_mutex_lock_iothread();
+    aio_context_acquire(bdrv_get_aio_context(bmds->bs));
     blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
                                 nr_sectors, blk_mig_read_cb, blk);
 
     bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors);
+    aio_context_release(bdrv_get_aio_context(bmds->bs));
     qemu_mutex_unlock_iothread();
 
     bmds->cur_sector = cur_sector + nr_sectors;
@@ -319,8 +343,10 @@ static int set_dirty_tracking(void)
     int ret;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(bdrv_get_aio_context(bmds->bs));
         bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
                                                       NULL, NULL);
+        aio_context_release(bdrv_get_aio_context(bmds->bs));
         if (!bmds->dirty_bitmap) {
             ret = -errno;
             goto fail;
@@ -331,18 +357,24 @@ static int set_dirty_tracking(void)
 fail:
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
         if (bmds->dirty_bitmap) {
+            aio_context_acquire(bdrv_get_aio_context(bmds->bs));
             bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
+            aio_context_release(bdrv_get_aio_context(bmds->bs));
         }
     }
     return ret;
 }
 
+/* Called with iothread lock taken.  */
+
 static void unset_dirty_tracking(void)
 {
     BlkMigDevState *bmds;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(bdrv_get_aio_context(bmds->bs));
         bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
+        aio_context_release(bdrv_get_aio_context(bmds->bs));
     }
 }
 
@@ -442,7 +474,7 @@ static void blk_mig_reset_dirty_cursor(void)
     }
 }
 
-/* Called with iothread lock taken.  */
+/* Called with iothread lock and AioContext taken.  */
 
 static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
                                  int is_async)
@@ -525,7 +557,9 @@ static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
     int ret = 1;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(bdrv_get_aio_context(bmds->bs));
         ret = mig_save_device_dirty(f, bmds, is_async);
+        aio_context_release(bdrv_get_aio_context(bmds->bs));
         if (ret <= 0) {
             break;
         }
@@ -583,7 +617,9 @@ static int64_t get_remaining_dirty(void)
     int64_t dirty = 0;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(bdrv_get_aio_context(bmds->bs));
         dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
+        aio_context_release(bdrv_get_aio_context(bmds->bs));
     }
 
     return dirty << BDRV_SECTOR_BITS;
@@ -591,25 +627,32 @@ static int64_t get_remaining_dirty(void)
 
 /* Called with iothread lock taken.  */
 
-static void blk_mig_cleanup(void)
+static void block_migration_cleanup(void *opaque)
 {
     BlkMigDevState *bmds;
     BlkMigBlock *blk;
+    AioContext *ctx;
 
     bdrv_drain_all();
 
     unset_dirty_tracking();
 
-    blk_mig_lock();
     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
         bdrv_op_unblock_all(bmds->bs, bmds->blocker);
         error_free(bmds->blocker);
+
+        /* Save ctx, because bmds->bs can disappear during bdrv_unref.  */
+        ctx = bdrv_get_aio_context(bmds->bs);
+        aio_context_acquire(ctx);
         bdrv_unref(bmds->bs);
+        aio_context_release(ctx);
+
         g_free(bmds->aio_bitmap);
         g_free(bmds);
     }
 
+    blk_mig_lock();
     while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
         g_free(blk->buf);
@@ -618,11 +661,6 @@ static void blk_mig_cleanup(void)
     blk_mig_unlock();
 }
 
-static void block_migration_cancel(void *opaque)
-{
-    blk_mig_cleanup();
-}
-
 static int block_save_setup(QEMUFile *f, void *opaque)
 {
     int ret;
@@ -636,13 +674,12 @@ static int block_save_setup(QEMUFile *f, void *opaque)
     /* start track dirty blocks */
     ret = set_dirty_tracking();
 
+    qemu_mutex_unlock_iothread();
+
     if (ret) {
-        qemu_mutex_unlock_iothread();
         return ret;
     }
 
-    qemu_mutex_unlock_iothread();
-
     ret = flush_blks(f);
     blk_mig_reset_dirty_cursor();
     qemu_put_be64(f, BLK_MIG_FLAG_EOS);
@@ -670,7 +707,10 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
     blk_mig_lock();
     while ((block_mig_state.submitted +
             block_mig_state.read_done) * BLOCK_SIZE <
-           qemu_file_get_rate_limit(f)) {
+           qemu_file_get_rate_limit(f) &&
+           (block_mig_state.submitted +
+            block_mig_state.read_done) <
+           MAX_INFLIGHT_IO) {
         blk_mig_unlock();
         if (block_mig_state.bulk_completed == 0) {
             /* first finish the bulk phase */
@@ -753,26 +793,30 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     return 0;
 }
 
-static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                               uint64_t *non_postcopiable_pending,
+                               uint64_t *postcopiable_pending)
 {
     /* Estimate pending number of bytes to send */
     uint64_t pending;
 
     qemu_mutex_lock_iothread();
+    pending = get_remaining_dirty();
+    qemu_mutex_unlock_iothread();
+
     blk_mig_lock();
-    pending = get_remaining_dirty() +
-                       block_mig_state.submitted * BLOCK_SIZE +
-                       block_mig_state.read_done * BLOCK_SIZE;
+    pending += block_mig_state.submitted * BLOCK_SIZE +
+               block_mig_state.read_done * BLOCK_SIZE;
+    blk_mig_unlock();
 
     /* Report at least one block pending during bulk phase */
     if (pending <= max_size && !block_mig_state.bulk_completed) {
         pending = max_size + BLOCK_SIZE;
     }
-    blk_mig_unlock();
-    qemu_mutex_unlock_iothread();
 
     DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
-    return pending;
+    /* We don't do postcopy */
+    *non_postcopiable_pending += pending;
 }
 
 static int block_load(QEMUFile *f, void *opaque, int version_id)
@@ -783,6 +827,7 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
     int64_t addr;
     BlockDriverState *bs, *bs_prev = NULL;
     BlockBackend *blk;
+    Error *local_err = NULL;
     uint8_t *buf;
     int64_t total_sectors = 0;
     int nr_sectors;
@@ -807,6 +852,11 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
                 return -EINVAL;
             }
             bs = blk_bs(blk);
+            if (!bs) {
+                fprintf(stderr, "Block device %s has no medium\n",
+                        device_name);
+                return -EINVAL;
+            }
 
             if (bs != bs_prev) {
                 bs_prev = bs;
@@ -816,6 +866,12 @@ static int block_load(QEMUFile *f, void *opaque, int version_id)
                                  device_name);
                     return -EINVAL;
                 }
+
+                bdrv_invalidate_cache(bs, &local_err);
+                if (local_err) {
+                    error_report_err(local_err);
+                    return -EINVAL;
+                }
             }
 
             if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
@@ -876,10 +932,10 @@ static SaveVMHandlers savevm_block_handlers = {
     .set_params = block_set_params,
     .save_live_setup = block_save_setup,
     .save_live_iterate = block_save_iterate,
-    .save_live_complete = block_save_complete,
+    .save_live_complete_precopy = block_save_complete,
     .save_live_pending = block_save_pending,
     .load_state = block_load,
-    .cancel = block_migration_cancel,
+    .cleanup = block_migration_cleanup,
     .is_active = block_is_active,
 };
 
diff --git a/qemu/migration/exec.c b/qemu/migration/exec.c
index 8406d2bbd..559420969 100644
--- a/qemu/migration/exec.c
+++ b/qemu/migration/exec.c
@@ -15,13 +15,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/sockets.h"
 #include "qemu/main-loop.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "block/block.h"
-#include <sys/types.h>
 #include <sys/wait.h>
 
 //#define DEBUG_MIGRATION_EXEC
@@ -36,8 +37,8 @@
 
 void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp)
 {
-    s->file = qemu_popen_cmd(command, "w");
-    if (s->file == NULL) {
+    s->to_dst_file = qemu_popen_cmd(command, "w");
+    if (s->to_dst_file == NULL) {
         error_setg_errno(errp, errno, "failed to popen the migration target");
         return;
     }
diff --git a/qemu/migration/fd.c b/qemu/migration/fd.c
index 3e4bed0e0..3d788bb29 100644
--- a/qemu/migration/fd.c
+++ b/qemu/migration/fd.c
@@ -13,6 +13,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 #include "qemu/sockets.h"
@@ -50,9 +52,9 @@ void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **
     }
 
     if (fd_is_socket(fd)) {
-        s->file = qemu_fopen_socket(fd, "wb");
+        s->to_dst_file = qemu_fopen_socket(fd, "wb");
     } else {
-        s->file = qemu_fdopen(fd, "wb");
+        s->to_dst_file = qemu_fdopen(fd, "wb");
     }
 
     migrate_fd_connect(s);
diff --git a/qemu/migration/migration.c b/qemu/migration/migration.c
index 4ddb9adce..991313a86 100644
--- a/qemu/migration/migration.c
+++ b/qemu/migration/migration.c
@@ -13,7 +13,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "migration/migration.h"
@@ -21,16 +22,20 @@
 #include "sysemu/sysemu.h"
 #include "block/block.h"
 #include "qapi/qmp/qerror.h"
+#include "qapi/util.h"
 #include "qemu/sockets.h"
 #include "qemu/rcu.h"
 #include "migration/block.h"
+#include "migration/postcopy-ram.h"
 #include "qemu/thread.h"
 #include "qmp-commands.h"
 #include "trace.h"
-#include "qapi/util.h"
 #include "qapi-event.h"
+#include "qom/cpu.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
 
-#define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
+#define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
 
 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  * data. */
@@ -44,6 +49,9 @@
 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
+/* Define default autoconverge cpu throttle migration parameters */
+#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL 20
+#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT 10
 
 /* Migration XBZRLE default cache size */
 #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
@@ -53,6 +61,13 @@ static NotifierList migration_state_notifiers =
 
 static bool deferred_incoming;
 
+/*
+ * Current state of incoming postcopy; note this is not part of
+ * MigrationIncomingState since it's state is used during cleanup
+ * at the end as MIS is being freed.
+ */
+static PostcopyState incoming_postcopy_state;
+
 /* When we add fault tolerance, we could have several
    migrations at once.  For now we don't need to add
    dynamic creation of migration */
@@ -60,6 +75,7 @@ static bool deferred_incoming;
 /* For outgoing */
 MigrationState *migrate_get_current(void)
 {
+    static bool once;
     static MigrationState current_migration = {
         .state = MIGRATION_STATUS_NONE,
         .bandwidth_limit = MAX_THROTTLE,
@@ -71,8 +87,16 @@ MigrationState *migrate_get_current(void)
                 DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT,
         .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
                 DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT,
+        .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] =
+                DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL,
+        .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] =
+                DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT,
     };
 
+    if (!once) {
+        qemu_mutex_init(&current_migration.src_page_req_mutex);
+        once = true;
+    }
     return &current_migration;
 }
 
@@ -86,15 +110,19 @@ MigrationIncomingState *migration_incoming_get_current(void)
 
 MigrationIncomingState *migration_incoming_state_new(QEMUFile* f)
 {
-    mis_current = g_malloc0(sizeof(MigrationIncomingState));
-    mis_current->file = f;
+    mis_current = g_new0(MigrationIncomingState, 1);
+    mis_current->from_src_file = f;
+    mis_current->state = MIGRATION_STATUS_NONE;
     QLIST_INIT(&mis_current->loadvm_handlers);
+    qemu_mutex_init(&mis_current->rp_mutex);
+    qemu_event_init(&mis_current->main_thread_load_event, false);
 
     return mis_current;
 }
 
 void migration_incoming_state_destroy(void)
 {
+    qemu_event_destroy(&mis_current->main_thread_load_event);
     loadvm_free_handlers(mis_current);
     g_free(mis_current);
     mis_current = NULL;
@@ -176,7 +204,7 @@ static int global_state_post_load(void *opaque, int version_id)
     s->received = true;
     trace_migrate_global_state_post_load(runstate);
 
-    r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX,
+    r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE__MAX,
                                 -1, &local_err);
 
     if (r == -1) {
@@ -240,6 +268,35 @@ static void deferred_incoming_migration(Error **errp)
     deferred_incoming = true;
 }
 
+/* Request a range of pages from the source VM at the given
+ * start address.
+ *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
+ *           as the last request (a name must have been given previously)
+ *   Start: Address offset within the RB
+ *   Len: Length in bytes required - must be a multiple of pagesize
+ */
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
+                               ram_addr_t start, size_t len)
+{
+    uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname upto 256 */
+    size_t msglen = 12; /* start + len */
+
+    *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
+    *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
+
+    if (rbname) {
+        int rbname_len = strlen(rbname);
+        assert(rbname_len < 256);
+
+        bufc[msglen++] = rbname_len;
+        memcpy(bufc + msglen, rbname, rbname_len);
+        msglen += rbname_len;
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES_ID, msglen, bufc);
+    } else {
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES, msglen, bufc);
+    }
+}
+
 void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
     const char *p;
@@ -266,37 +323,27 @@ void qemu_start_incoming_migration(const char *uri, Error **errp)
     }
 }
 
-static void process_incoming_migration_co(void *opaque)
+static void process_incoming_migration_bh(void *opaque)
 {
-    QEMUFile *f = opaque;
     Error *local_err = NULL;
-    int ret;
-
-    migration_incoming_state_new(f);
-    migrate_generate_event(MIGRATION_STATUS_ACTIVE);
-    ret = qemu_loadvm_state(f);
-
-    qemu_fclose(f);
-    free_xbzrle_decoded_buf();
-    migration_incoming_state_destroy();
-
-    if (ret < 0) {
-        migrate_generate_event(MIGRATION_STATUS_FAILED);
-        error_report("load of migration failed: %s", strerror(-ret));
-        migrate_decompress_threads_join();
-        exit(EXIT_FAILURE);
-    }
-    migrate_generate_event(MIGRATION_STATUS_COMPLETED);
-    qemu_announce_self();
+    MigrationIncomingState *mis = opaque;
 
     /* Make sure all file formats flush their mutable metadata */
     bdrv_invalidate_cache_all(&local_err);
     if (local_err) {
+        migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
         error_report_err(local_err);
         migrate_decompress_threads_join();
         exit(EXIT_FAILURE);
     }
 
+    /*
+     * This must happen after all error conditions are dealt with and
+     * we're sure the VM is going to be running on this host.
+     */
+    qemu_announce_self();
+
     /* If global state section was not received or we are in running
        state, we need to obey autostart. Any other state is set with
        runstate_set. */
@@ -312,6 +359,64 @@ static void process_incoming_migration_co(void *opaque)
         runstate_set(global_state_get_runstate());
     }
     migrate_decompress_threads_join();
+    /*
+     * This must happen after any state changes since as soon as an external
+     * observer sees this event they might start to prod at the VM assuming
+     * it's ready to use.
+     */
+    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                      MIGRATION_STATUS_COMPLETED);
+    qemu_bh_delete(mis->bh);
+    migration_incoming_state_destroy();
+}
+
+static void process_incoming_migration_co(void *opaque)
+{
+    QEMUFile *f = opaque;
+    MigrationIncomingState *mis;
+    PostcopyState ps;
+    int ret;
+
+    mis = migration_incoming_state_new(f);
+    postcopy_state_set(POSTCOPY_INCOMING_NONE);
+    migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
+                      MIGRATION_STATUS_ACTIVE);
+    ret = qemu_loadvm_state(f);
+
+    ps = postcopy_state_get();
+    trace_process_incoming_migration_co_end(ret, ps);
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        if (ps == POSTCOPY_INCOMING_ADVISE) {
+            /*
+             * Where a migration had postcopy enabled (and thus went to advise)
+             * but managed to complete within the precopy period, we can use
+             * the normal exit.
+             */
+            postcopy_ram_incoming_cleanup(mis);
+        } else if (ret >= 0) {
+            /*
+             * Postcopy was started, cleanup should happen at the end of the
+             * postcopy thread.
+             */
+            trace_process_incoming_migration_co_postcopy_end_main();
+            return;
+        }
+        /* Else if something went wrong then just fall out of the normal exit */
+    }
+
+    qemu_fclose(f);
+    free_xbzrle_decoded_buf();
+
+    if (ret < 0) {
+        migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+        error_report("load of migration failed: %s", strerror(-ret));
+        migrate_decompress_threads_join();
+        exit(EXIT_FAILURE);
+    }
+
+    mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
+    qemu_bh_schedule(mis->bh);
 }
 
 void process_incoming_migration(QEMUFile *f)
@@ -325,6 +430,50 @@ void process_incoming_migration(QEMUFile *f)
     qemu_coroutine_enter(co, f);
 }
 
+/*
+ * Send a message on the return channel back to the source
+ * of the migration.
+ */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data)
+{
+    trace_migrate_send_rp_message((int)message_type, len);
+    qemu_mutex_lock(&mis->rp_mutex);
+    qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
+    qemu_put_be16(mis->to_src_file, len);
+    qemu_put_buffer(mis->to_src_file, data, len);
+    qemu_fflush(mis->to_src_file);
+    qemu_mutex_unlock(&mis->rp_mutex);
+}
+
+/*
+ * Send a 'SHUT' message on the return channel with the given value
+ * to indicate that we've finished with the RP.  Non-0 value indicates
+ * error.
+ */
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
+}
+
+/*
+ * Send a 'PONG' message on the return channel with the given value
+ * (normally in response to a 'PING')
+ */
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
+}
+
 /* amount of nanoseconds we are willing to wait for migration to be down.
  * the choice of nanoseconds is because it is the maximum resolution that
  * get_clock() can achieve. It is an internal measure. All user-visible
@@ -344,7 +493,7 @@ MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
     int i;
 
     caps = NULL; /* silence compiler warning */
-    for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+    for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
         if (head == NULL) {
             head = g_malloc0(sizeof(*caps));
             caps = head;
@@ -372,10 +521,32 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
             s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
     params->decompress_threads =
             s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+    params->x_cpu_throttle_initial =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL];
+    params->x_cpu_throttle_increment =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT];
 
     return params;
 }
 
+/*
+ * Return true if we're already in the middle of a migration
+ * (i.e. any of the active or setup states)
+ */
+static bool migration_is_setup_or_active(int state)
+{
+    switch (state) {
+    case MIGRATION_STATUS_ACTIVE:
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+    case MIGRATION_STATUS_SETUP:
+        return true;
+
+    default:
+        return false;
+
+    }
+}
+
 static void get_xbzrle_cache_stats(MigrationInfo *info)
 {
     if (migrate_use_xbzrle()) {
@@ -435,6 +606,45 @@ MigrationInfo *qmp_query_migrate(Error **errp)
             info->disk->total = blk_mig_bytes_total();
         }
 
+        if (cpu_throttle_active()) {
+            info->has_x_cpu_throttle_percentage = true;
+            info->x_cpu_throttle_percentage = cpu_throttle_get_percentage();
+        }
+
+        get_xbzrle_cache_stats(info);
+        break;
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+        /* Mostly the same as active; TODO add some postcopy stats */
+        info->has_status = true;
+        info->has_total_time = true;
+        info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
+            - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
+        info->has_setup_time = true;
+        info->setup_time = s->setup_time;
+
+        info->has_ram = true;
+        info->ram = g_malloc0(sizeof(*info->ram));
+        info->ram->transferred = ram_bytes_transferred();
+        info->ram->remaining = ram_bytes_remaining();
+        info->ram->total = ram_bytes_total();
+        info->ram->duplicate = dup_mig_pages_transferred();
+        info->ram->skipped = skipped_mig_pages_transferred();
+        info->ram->normal = norm_mig_pages_transferred();
+        info->ram->normal_bytes = norm_mig_bytes_transferred();
+        info->ram->dirty_pages_rate = s->dirty_pages_rate;
+        info->ram->mbps = s->mbps;
+        info->ram->dirty_sync_count = s->dirty_sync_count;
+
+        if (blk_mig_active()) {
+            info->has_disk = true;
+            info->disk = g_malloc0(sizeof(*info->disk));
+            info->disk->transferred = blk_mig_bytes_transferred();
+            info->disk->remaining = blk_mig_bytes_remaining();
+            info->disk->total = blk_mig_bytes_total();
+        }
+
         get_xbzrle_cache_stats(info);
         break;
     case MIGRATION_STATUS_COMPLETED:
@@ -478,8 +688,7 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
     MigrationState *s = migrate_get_current();
     MigrationCapabilityStatusList *cap;
 
-    if (s->state == MIGRATION_STATUS_ACTIVE ||
-        s->state == MIGRATION_STATUS_SETUP) {
+    if (migration_is_setup_or_active(s->state)) {
         error_setg(errp, QERR_MIGRATION_ACTIVE);
         return;
     }
@@ -487,6 +696,20 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
     for (cap = params; cap; cap = cap->next) {
         s->enabled_capabilities[cap->value->capability] = cap->value->state;
     }
+
+    if (migrate_postcopy_ram()) {
+        if (migrate_use_compression()) {
+            /* The decompression threads asynchronously write into RAM
+             * rather than use the atomic copies needed to avoid
+             * userfaulting.  It should be possible to fix the decompression
+             * threads for compatibility in future.
+             */
+            error_report("Postcopy is not currently compatible with "
+                         "compression");
+            s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM] =
+                false;
+        }
+    }
 }
 
 void qmp_migrate_set_parameters(bool has_compress_level,
@@ -494,7 +717,11 @@ void qmp_migrate_set_parameters(bool has_compress_level,
                                 bool has_compress_threads,
                                 int64_t compress_threads,
                                 bool has_decompress_threads,
-                                int64_t decompress_threads, Error **errp)
+                                int64_t decompress_threads,
+                                bool has_x_cpu_throttle_initial,
+                                int64_t x_cpu_throttle_initial,
+                                bool has_x_cpu_throttle_increment,
+                                int64_t x_cpu_throttle_increment, Error **errp)
 {
     MigrationState *s = migrate_get_current();
 
@@ -517,6 +744,18 @@ void qmp_migrate_set_parameters(bool has_compress_level,
                    "is invalid, it should be in the range of 1 to 255");
         return;
     }
+    if (has_x_cpu_throttle_initial &&
+            (x_cpu_throttle_initial < 1 || x_cpu_throttle_initial > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "x_cpu_throttle_initial",
+                   "an integer in the range of 1 to 99");
+    }
+    if (has_x_cpu_throttle_increment &&
+            (x_cpu_throttle_increment < 1 || x_cpu_throttle_increment > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "x_cpu_throttle_increment",
+                   "an integer in the range of 1 to 99");
+    }
 
     if (has_compress_level) {
         s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level;
@@ -528,13 +767,44 @@ void qmp_migrate_set_parameters(bool has_compress_level,
         s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
                                                     decompress_threads;
     }
+    if (has_x_cpu_throttle_initial) {
+        s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] =
+                                                    x_cpu_throttle_initial;
+    }
+
+    if (has_x_cpu_throttle_increment) {
+        s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] =
+                                                    x_cpu_throttle_increment;
+    }
+}
+
+void qmp_migrate_start_postcopy(Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+
+    if (!migrate_postcopy_ram()) {
+        error_setg(errp, "Enable postcopy with migrate_set_capability before"
+                         " the start of migration");
+        return;
+    }
+
+    if (s->state == MIGRATION_STATUS_NONE) {
+        error_setg(errp, "Postcopy must be started after migration has been"
+                         " started");
+        return;
+    }
+    /*
+     * we don't error if migration has finished since that would be racy
+     * with issuing this command.
+     */
+    atomic_set(&s->start_postcopy, true);
 }
 
 /* shared migration helpers */
 
-static void migrate_set_state(MigrationState *s, int old_state, int new_state)
+void migrate_set_state(int *state, int old_state, int new_state)
 {
-    if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) {
+    if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
         trace_migrate_set_state(new_state);
         migrate_generate_event(new_state);
     }
@@ -547,21 +817,27 @@ static void migrate_fd_cleanup(void *opaque)
     qemu_bh_delete(s->cleanup_bh);
     s->cleanup_bh = NULL;
 
-    if (s->file) {
+    flush_page_queue(s);
+
+    if (s->to_dst_file) {
         trace_migrate_fd_cleanup();
         qemu_mutex_unlock_iothread();
-        qemu_thread_join(&s->thread);
+        if (s->migration_thread_running) {
+            qemu_thread_join(&s->thread);
+            s->migration_thread_running = false;
+        }
         qemu_mutex_lock_iothread();
 
         migrate_compress_threads_join();
-        qemu_fclose(s->file);
-        s->file = NULL;
+        qemu_fclose(s->to_dst_file);
+        s->to_dst_file = NULL;
     }
 
-    assert(s->state != MIGRATION_STATUS_ACTIVE);
+    assert((s->state != MIGRATION_STATUS_ACTIVE) &&
+           (s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE));
 
     if (s->state == MIGRATION_STATUS_CANCELLING) {
-        migrate_set_state(s, MIGRATION_STATUS_CANCELLING,
+        migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
                           MIGRATION_STATUS_CANCELLED);
     }
 
@@ -571,24 +847,29 @@ static void migrate_fd_cleanup(void *opaque)
 void migrate_fd_error(MigrationState *s)
 {
     trace_migrate_fd_error();
-    assert(s->file == NULL);
-    migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED);
+    assert(s->to_dst_file == NULL);
+    migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                      MIGRATION_STATUS_FAILED);
     notifier_list_notify(&migration_state_notifiers, s);
 }
 
 static void migrate_fd_cancel(MigrationState *s)
 {
     int old_state ;
-    QEMUFile *f = migrate_get_current()->file;
+    QEMUFile *f = migrate_get_current()->to_dst_file;
     trace_migrate_fd_cancel();
 
+    if (s->rp_state.from_dst_file) {
+        /* shutdown the rp socket, so causing the rp thread to shutdown */
+        qemu_file_shutdown(s->rp_state.from_dst_file);
+    }
+
     do {
         old_state = s->state;
-        if (old_state != MIGRATION_STATUS_SETUP &&
-            old_state != MIGRATION_STATUS_ACTIVE) {
+        if (!migration_is_setup_or_active(old_state)) {
             break;
         }
-        migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING);
+        migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
     } while (s->state != MIGRATION_STATUS_CANCELLING);
 
     /*
@@ -629,34 +910,48 @@ bool migration_has_failed(MigrationState *s)
             s->state == MIGRATION_STATUS_FAILED);
 }
 
-static MigrationState *migrate_init(const MigrationParams *params)
+bool migration_in_postcopy(MigrationState *s)
 {
-    MigrationState *s = migrate_get_current();
-    int64_t bandwidth_limit = s->bandwidth_limit;
-    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
-    int64_t xbzrle_cache_size = s->xbzrle_cache_size;
-    int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL];
-    int compress_thread_count =
-            s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS];
-    int decompress_thread_count =
-            s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS];
+    return (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
+}
 
-    memcpy(enabled_capabilities, s->enabled_capabilities,
-           sizeof(enabled_capabilities));
+bool migration_in_postcopy_after_devices(MigrationState *s)
+{
+    return migration_in_postcopy(s) && s->postcopy_after_devices;
+}
 
-    memset(s, 0, sizeof(*s));
+MigrationState *migrate_init(const MigrationParams *params)
+{
+    MigrationState *s = migrate_get_current();
+
+    /*
+     * Reinitialise all migration state, except
+     * parameters/capabilities that the user set, and
+     * locks.
+     */
+    s->bytes_xfer = 0;
+    s->xfer_limit = 0;
+    s->cleanup_bh = 0;
+    s->to_dst_file = NULL;
+    s->state = MIGRATION_STATUS_NONE;
     s->params = *params;
-    memcpy(s->enabled_capabilities, enabled_capabilities,
-           sizeof(enabled_capabilities));
-    s->xbzrle_cache_size = xbzrle_cache_size;
-
-    s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level;
-    s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] =
-               compress_thread_count;
-    s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] =
-               decompress_thread_count;
-    s->bandwidth_limit = bandwidth_limit;
-    migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
+    s->rp_state.from_dst_file = NULL;
+    s->rp_state.error = false;
+    s->mbps = 0.0;
+    s->downtime = 0;
+    s->expected_downtime = 0;
+    s->dirty_pages_rate = 0;
+    s->dirty_bytes_rate = 0;
+    s->setup_time = 0;
+    s->dirty_sync_count = 0;
+    s->start_postcopy = false;
+    s->postcopy_after_devices = false;
+    s->migration_thread_running = false;
+    s->last_req_rb = NULL;
+
+    migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
+
+    QSIMPLEQ_INIT(&s->src_page_requests);
 
     s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     return s;
@@ -709,8 +1004,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     params.blk = has_blk && blk;
     params.shared = has_inc && inc;
 
-    if (s->state == MIGRATION_STATUS_ACTIVE ||
-        s->state == MIGRATION_STATUS_SETUP ||
+    if (migration_is_setup_or_active(s->state) ||
         s->state == MIGRATION_STATUS_CANCELLING) {
         error_setg(errp, QERR_MIGRATION_ACTIVE);
         return;
@@ -729,12 +1023,6 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
         return;
     }
 
-    /* We are starting a new migration, so we want to start in a clean
-       state.  This change is only needed if previous migration
-       failed/was cancelled.  We don't use migrate_set_state() because
-       we are setting the initial state, not changing it. */
-    s->state = MIGRATION_STATUS_NONE;
-
     s = migrate_init(&params);
 
     if (strstart(uri, "tcp:", &p)) {
@@ -754,7 +1042,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     } else {
         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
                    "a valid migration protocol");
-        migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED);
+        migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                          MIGRATION_STATUS_FAILED);
         return;
     }
 
@@ -817,8 +1106,9 @@ void qmp_migrate_set_speed(int64_t value, Error **errp)
 
     s = migrate_get_current();
     s->bandwidth_limit = value;
-    if (s->file) {
-        qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO);
+    if (s->to_dst_file) {
+        qemu_file_set_rate_limit(s->to_dst_file,
+                                 s->bandwidth_limit / XFER_LIMIT_RATIO);
     }
 }
 
@@ -829,6 +1119,15 @@ void qmp_migrate_set_downtime(double value, Error **errp)
     max_downtime = (uint64_t)value;
 }
 
+bool migrate_postcopy_ram(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
+}
+
 bool migrate_auto_converge(void)
 {
     MigrationState *s;
@@ -911,10 +1210,419 @@ int64_t migrate_xbzrle_cache_size(void)
 }
 
 /* migration thread support */
+/*
+ * Something bad happened to the RP stream, mark an error
+ * The caller shall print or trace something to indicate why
+ */
+static void mark_source_rp_bad(MigrationState *s)
+{
+    s->rp_state.error = true;
+}
+
+static struct rp_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} rp_cmd_args[] = {
+    [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
+    [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
+    [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
+    [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
+    [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
+    [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
+};
+
+/*
+ * Process a request for pages received on the return path,
+ * We're allowed to send more than requested (e.g. to round to our page size)
+ * and we don't need to send pages that have already been sent.
+ */
+static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
+                                       ram_addr_t start, size_t len)
+{
+    long our_host_ps = getpagesize();
 
+    trace_migrate_handle_rp_req_pages(rbname, start, len);
+
+    /*
+     * Since we currently insist on matching page sizes, just sanity check
+     * we're being asked for whole host pages.
+     */
+    if (start & (our_host_ps-1) ||
+       (len & (our_host_ps-1))) {
+        error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
+                     " len: %zd", __func__, start, len);
+        mark_source_rp_bad(ms);
+        return;
+    }
+
+    if (ram_save_queue_pages(ms, rbname, start, len)) {
+        mark_source_rp_bad(ms);
+    }
+}
+
+/*
+ * Handles messages sent on the return path towards the source VM
+ *
+ */
+static void *source_return_path_thread(void *opaque)
+{
+    MigrationState *ms = opaque;
+    QEMUFile *rp = ms->rp_state.from_dst_file;
+    uint16_t header_len, header_type;
+    uint8_t buf[512];
+    uint32_t tmp32, sibling_error;
+    ram_addr_t start = 0; /* =0 to silence warning */
+    size_t  len = 0, expected_len;
+    int res;
+
+    trace_source_return_path_thread_entry();
+    while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
+           migration_is_setup_or_active(ms->state)) {
+        trace_source_return_path_thread_loop_top();
+        header_type = qemu_get_be16(rp);
+        header_len = qemu_get_be16(rp);
+
+        if (header_type >= MIG_RP_MSG_MAX ||
+            header_type == MIG_RP_MSG_INVALID) {
+            error_report("RP: Received invalid message 0x%04x length 0x%04x",
+                    header_type, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        if ((rp_cmd_args[header_type].len != -1 &&
+            header_len != rp_cmd_args[header_type].len) ||
+            header_len > sizeof(buf)) {
+            error_report("RP: Received '%s' message (0x%04x) with"
+                    "incorrect length %d expecting %zu",
+                    rp_cmd_args[header_type].name, header_type, header_len,
+                    (size_t)rp_cmd_args[header_type].len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* We know we've got a valid header by this point */
+        res = qemu_get_buffer(rp, buf, header_len);
+        if (res != header_len) {
+            error_report("RP: Failed reading data for message 0x%04x"
+                         " read %d expected %d",
+                         header_type, res, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* OK, we have the message and the data */
+        switch (header_type) {
+        case MIG_RP_MSG_SHUT:
+            sibling_error = be32_to_cpup((uint32_t *)buf);
+            trace_source_return_path_thread_shut(sibling_error);
+            if (sibling_error) {
+                error_report("RP: Sibling indicated error %d", sibling_error);
+                mark_source_rp_bad(ms);
+            }
+            /*
+             * We'll let the main thread deal with closing the RP
+             * we could do a shutdown(2) on it, but we're the only user
+             * anyway, so there's nothing gained.
+             */
+            goto out;
+
+        case MIG_RP_MSG_PONG:
+            tmp32 = be32_to_cpup((uint32_t *)buf);
+            trace_source_return_path_thread_pong(tmp32);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES:
+            start = be64_to_cpup((uint64_t *)buf);
+            len = be32_to_cpup((uint32_t *)(buf + 8));
+            migrate_handle_rp_req_pages(ms, NULL, start, len);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES_ID:
+            expected_len = 12 + 1; /* header + termination */
+
+            if (header_len >= expected_len) {
+                start = be64_to_cpup((uint64_t *)buf);
+                len = be32_to_cpup((uint32_t *)(buf + 8));
+                /* Now we expect an idstr */
+                tmp32 = buf[12]; /* Length of the following idstr */
+                buf[13 + tmp32] = '\0';
+                expected_len += tmp32;
+            }
+            if (header_len != expected_len) {
+                error_report("RP: Req_Page_id with length %d expecting %zd",
+                        header_len, expected_len);
+                mark_source_rp_bad(ms);
+                goto out;
+            }
+            migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
+            break;
+
+        default:
+            break;
+        }
+    }
+    if (qemu_file_get_error(rp)) {
+        trace_source_return_path_thread_bad_end();
+        mark_source_rp_bad(ms);
+    }
+
+    trace_source_return_path_thread_end();
+out:
+    ms->rp_state.from_dst_file = NULL;
+    qemu_fclose(rp);
+    return NULL;
+}
+
+static int open_return_path_on_source(MigrationState *ms)
+{
+
+    ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
+    if (!ms->rp_state.from_dst_file) {
+        return -1;
+    }
+
+    trace_open_return_path_on_source();
+    qemu_thread_create(&ms->rp_state.rp_thread, "return path",
+                       source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
+
+    trace_open_return_path_on_source_continue();
+
+    return 0;
+}
+
+/* Returns 0 if the RP was ok, otherwise there was an error on the RP */
+static int await_return_path_close_on_source(MigrationState *ms)
+{
+    /*
+     * If this is a normal exit then the destination will send a SHUT and the
+     * rp_thread will exit, however if there's an error we need to cause
+     * it to exit.
+     */
+    if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
+        /*
+         * shutdown(2), if we have it, will cause it to unblock if it's stuck
+         * waiting for the destination.
+         */
+        qemu_file_shutdown(ms->rp_state.from_dst_file);
+        mark_source_rp_bad(ms);
+    }
+    trace_await_return_path_close_on_source_joining();
+    qemu_thread_join(&ms->rp_state.rp_thread);
+    trace_await_return_path_close_on_source_close();
+    return ms->rp_state.error;
+}
+
+/*
+ * Switch from normal iteration to postcopy
+ * Returns non-0 on error
+ */
+static int postcopy_start(MigrationState *ms, bool *old_vm_running)
+{
+    int ret;
+    const QEMUSizedBuffer *qsb;
+    int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
+                      MIGRATION_STATUS_POSTCOPY_ACTIVE);
+
+    trace_postcopy_start();
+    qemu_mutex_lock_iothread();
+    trace_postcopy_start_set_run();
+
+    qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+    *old_vm_running = runstate_is_running();
+    global_state_store();
+    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = bdrv_inactivate_all();
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /*
+     * Cause any non-postcopiable, but iterative devices to
+     * send out their final data.
+     */
+    qemu_savevm_state_complete_precopy(ms->to_dst_file, true);
+
+    /*
+     * in Finish migrate and with the io-lock held everything should
+     * be quiet, but we've potentially still got dirty pages and we
+     * need to tell the destination to throw any pages it's already received
+     * that are dirty
+     */
+    if (ram_postcopy_send_discard_bitmap(ms)) {
+        error_report("postcopy send discard bitmap failed");
+        goto fail;
+    }
+
+    /*
+     * send rest of state - note things that are doing postcopy
+     * will notice we're in POSTCOPY_ACTIVE and not actually
+     * wrap their state up here
+     */
+    qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
+    /* Ping just for debugging, helps line traces up */
+    qemu_savevm_send_ping(ms->to_dst_file, 2);
+
+    /*
+     * While loading the device state we may trigger page transfer
+     * requests and the fd must be free to process those, and thus
+     * the destination must read the whole device state off the fd before
+     * it starts processing it.  Unfortunately the ad-hoc migration format
+     * doesn't allow the destination to know the size to read without fully
+     * parsing it through each devices load-state code (especially the open
+     * coded devices that use get/put).
+     * So we wrap the device state up in a package with a length at the start;
+     * to do this we use a qemu_buf to hold the whole of the device state.
+     */
+    QEMUFile *fb = qemu_bufopen("w", NULL);
+    if (!fb) {
+        error_report("Failed to create buffered file");
+        goto fail;
+    }
+
+    /*
+     * Make sure the receiver can get incoming pages before we send the rest
+     * of the state
+     */
+    qemu_savevm_send_postcopy_listen(fb);
+
+    qemu_savevm_state_complete_precopy(fb, false);
+    qemu_savevm_send_ping(fb, 3);
+
+    qemu_savevm_send_postcopy_run(fb);
+
+    /* <><> end of stuff going into the package */
+    qsb = qemu_buf_get(fb);
+
+    /* Now send that blob */
+    if (qemu_savevm_send_packaged(ms->to_dst_file, qsb)) {
+        goto fail_closefb;
+    }
+    qemu_fclose(fb);
+
+    /* Send a notify to give a chance for anything that needs to happen
+     * at the transition to postcopy and after the device state; in particular
+     * spice needs to trigger a transition now
+     */
+    ms->postcopy_after_devices = true;
+    notifier_list_notify(&migration_state_notifiers, ms);
+
+    ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
+
+    qemu_mutex_unlock_iothread();
+
+    /*
+     * Although this ping is just for debug, it could potentially be
+     * used for getting a better measurement of downtime at the source.
+     */
+    qemu_savevm_send_ping(ms->to_dst_file, 4);
+
+    ret = qemu_file_get_error(ms->to_dst_file);
+    if (ret) {
+        error_report("postcopy_start: Migration stream errored");
+        migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                              MIGRATION_STATUS_FAILED);
+    }
+
+    return ret;
+
+fail_closefb:
+    qemu_fclose(fb);
+fail:
+    migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+    qemu_mutex_unlock_iothread();
+    return -1;
+}
+
+/**
+ * migration_completion: Used by migration_thread when there's not much left.
+ *   The caller 'breaks' the loop when this returns.
+ *
+ * @s: Current migration state
+ * @current_active_state: The migration state we expect to be in
+ * @*old_vm_running: Pointer to old_vm_running flag
+ * @*start_time: Pointer to time to update
+ */
+static void migration_completion(MigrationState *s, int current_active_state,
+                                 bool *old_vm_running,
+                                 int64_t *start_time)
+{
+    int ret;
+
+    if (s->state == MIGRATION_STATUS_ACTIVE) {
+        qemu_mutex_lock_iothread();
+        *start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+        *old_vm_running = runstate_is_running();
+        ret = global_state_store();
+
+        if (!ret) {
+            ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+            if (ret >= 0) {
+                ret = bdrv_inactivate_all();
+            }
+            if (ret >= 0) {
+                qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
+                qemu_savevm_state_complete_precopy(s->to_dst_file, false);
+            }
+        }
+        qemu_mutex_unlock_iothread();
+
+        if (ret < 0) {
+            goto fail;
+        }
+    } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
+        trace_migration_completion_postcopy_end();
+
+        qemu_savevm_state_complete_postcopy(s->to_dst_file);
+        trace_migration_completion_postcopy_end_after_complete();
+    }
+
+    /*
+     * If rp was opened we must clean up the thread before
+     * cleaning everything else up (since if there are no failures
+     * it will wait for the destination to send it's status in
+     * a SHUT command).
+     * Postcopy opens rp if enabled (even if it's not avtivated)
+     */
+    if (migrate_postcopy_ram()) {
+        int rp_error;
+        trace_migration_completion_postcopy_end_before_rp();
+        rp_error = await_return_path_close_on_source(s);
+        trace_migration_completion_postcopy_end_after_rp(rp_error);
+        if (rp_error) {
+            goto fail;
+        }
+    }
+
+    if (qemu_file_get_error(s->to_dst_file)) {
+        trace_migration_completion_file_err();
+        goto fail;
+    }
+
+    migrate_set_state(&s->state, current_active_state,
+                      MIGRATION_STATUS_COMPLETED);
+    return;
+
+fail:
+    migrate_set_state(&s->state, current_active_state,
+                      MIGRATION_STATUS_FAILED);
+}
+
+/*
+ * Master migration thread on the source VM.
+ * It drives the migration and pumps the data down the outgoing channel.
+ */
 static void *migration_thread(void *opaque)
 {
     MigrationState *s = opaque;
+    /* Used by the bandwidth calcs, updated later */
     int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
     int64_t initial_bytes = 0;
@@ -922,70 +1630,92 @@ static void *migration_thread(void *opaque)
     int64_t start_time = initial_time;
     int64_t end_time;
     bool old_vm_running = false;
+    bool entered_postcopy = false;
+    /* The active state we expect to be in; ACTIVE or POSTCOPY_ACTIVE */
+    enum MigrationStatus current_active_state = MIGRATION_STATUS_ACTIVE;
 
     rcu_register_thread();
 
-    qemu_savevm_state_header(s->file);
-    qemu_savevm_state_begin(s->file, &s->params);
+    qemu_savevm_state_header(s->to_dst_file);
+
+    if (migrate_postcopy_ram()) {
+        /* Now tell the dest that it should open its end so it can reply */
+        qemu_savevm_send_open_return_path(s->to_dst_file);
+
+        /* And do a ping that will make stuff easier to debug */
+        qemu_savevm_send_ping(s->to_dst_file, 1);
+
+        /*
+         * Tell the destination that we *might* want to do postcopy later;
+         * if the other end can't do postcopy it should fail now, nice and
+         * early.
+         */
+        qemu_savevm_send_postcopy_advise(s->to_dst_file);
+    }
+
+    qemu_savevm_state_begin(s->to_dst_file, &s->params);
 
     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
-    migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE);
+    current_active_state = MIGRATION_STATUS_ACTIVE;
+    migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                      MIGRATION_STATUS_ACTIVE);
+
+    trace_migration_thread_setup_complete();
 
-    while (s->state == MIGRATION_STATUS_ACTIVE) {
+    while (s->state == MIGRATION_STATUS_ACTIVE ||
+           s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
         int64_t current_time;
         uint64_t pending_size;
 
-        if (!qemu_file_rate_limit(s->file)) {
-            pending_size = qemu_savevm_state_pending(s->file, max_size);
-            trace_migrate_pending(pending_size, max_size);
+        if (!qemu_file_rate_limit(s->to_dst_file)) {
+            uint64_t pend_post, pend_nonpost;
+
+            qemu_savevm_state_pending(s->to_dst_file, max_size, &pend_nonpost,
+                                      &pend_post);
+            pending_size = pend_nonpost + pend_post;
+            trace_migrate_pending(pending_size, max_size,
+                                  pend_post, pend_nonpost);
             if (pending_size && pending_size >= max_size) {
-                qemu_savevm_state_iterate(s->file);
-            } else {
-                int ret;
-
-                qemu_mutex_lock_iothread();
-                start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-                qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-                old_vm_running = runstate_is_running();
-
-                ret = global_state_store();
-                if (!ret) {
-                    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
-                    if (ret >= 0) {
-                        qemu_file_set_rate_limit(s->file, INT64_MAX);
-                        qemu_savevm_state_complete(s->file);
-                    }
-                }
-                qemu_mutex_unlock_iothread();
+                /* Still a significant amount to transfer */
 
-                if (ret < 0) {
-                    migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                                      MIGRATION_STATUS_FAILED);
-                    break;
-                }
+                if (migrate_postcopy_ram() &&
+                    s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE &&
+                    pend_nonpost <= max_size &&
+                    atomic_read(&s->start_postcopy)) {
 
-                if (!qemu_file_get_error(s->file)) {
-                    migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
-                                      MIGRATION_STATUS_COMPLETED);
-                    break;
+                    if (!postcopy_start(s, &old_vm_running)) {
+                        current_active_state = MIGRATION_STATUS_POSTCOPY_ACTIVE;
+                        entered_postcopy = true;
+                    }
+
+                    continue;
                 }
+                /* Just another iteration step */
+                qemu_savevm_state_iterate(s->to_dst_file, entered_postcopy);
+            } else {
+                trace_migration_thread_low_pending(pending_size);
+                migration_completion(s, current_active_state,
+                                     &old_vm_running, &start_time);
+                break;
             }
         }
 
-        if (qemu_file_get_error(s->file)) {
-            migrate_set_state(s, MIGRATION_STATUS_ACTIVE,
+        if (qemu_file_get_error(s->to_dst_file)) {
+            migrate_set_state(&s->state, current_active_state,
                               MIGRATION_STATUS_FAILED);
+            trace_migration_thread_file_err();
             break;
         }
         current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
         if (current_time >= initial_time + BUFFER_DELAY) {
-            uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes;
+            uint64_t transferred_bytes = qemu_ftell(s->to_dst_file) -
+                                         initial_bytes;
             uint64_t time_spent = current_time - initial_time;
-            double bandwidth = transferred_bytes / time_spent;
+            double bandwidth = (double)transferred_bytes / time_spent;
             max_size = bandwidth * migrate_max_downtime() / 1000000;
 
-            s->mbps = time_spent ? (((double) transferred_bytes * 8.0) /
-                    ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1;
+            s->mbps = (((double) transferred_bytes * 8.0) /
+                    ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
 
             trace_migrate_transferred(transferred_bytes, time_spent,
                                       bandwidth, max_size);
@@ -995,31 +1725,36 @@ static void *migration_thread(void *opaque)
                 s->expected_downtime = s->dirty_bytes_rate / bandwidth;
             }
 
-            qemu_file_reset_rate_limit(s->file);
+            qemu_file_reset_rate_limit(s->to_dst_file);
             initial_time = current_time;
-            initial_bytes = qemu_ftell(s->file);
+            initial_bytes = qemu_ftell(s->to_dst_file);
         }
-        if (qemu_file_rate_limit(s->file)) {
+        if (qemu_file_rate_limit(s->to_dst_file)) {
             /* usleep expects microseconds */
             g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
         }
     }
 
+    trace_migration_thread_after_loop();
+    /* If we enabled cpu throttling for auto-converge, turn it off. */
+    cpu_throttle_stop();
     end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
 
     qemu_mutex_lock_iothread();
-    qemu_savevm_state_cancel();
+    qemu_savevm_state_cleanup();
     if (s->state == MIGRATION_STATUS_COMPLETED) {
-        uint64_t transferred_bytes = qemu_ftell(s->file);
+        uint64_t transferred_bytes = qemu_ftell(s->to_dst_file);
         s->total_time = end_time - s->total_time;
-        s->downtime = end_time - start_time;
+        if (!entered_postcopy) {
+            s->downtime = end_time - start_time;
+        }
         if (s->total_time) {
             s->mbps = (((double) transferred_bytes * 8.0) /
                        ((double) s->total_time)) / 1000;
         }
         runstate_set(RUN_STATE_POSTMIGRATE);
     } else {
-        if (old_vm_running) {
+        if (old_vm_running && !entered_postcopy) {
             vm_start();
         }
     }
@@ -1036,13 +1771,40 @@ void migrate_fd_connect(MigrationState *s)
     s->expected_downtime = max_downtime/1000000;
     s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s);
 
-    qemu_file_set_rate_limit(s->file,
+    qemu_file_set_rate_limit(s->to_dst_file,
                              s->bandwidth_limit / XFER_LIMIT_RATIO);
 
     /* Notify before starting migration thread */
     notifier_list_notify(&migration_state_notifiers, s);
 
+    /*
+     * Open the return path; currently for postcopy but other things might
+     * also want it.
+     */
+    if (migrate_postcopy_ram()) {
+        if (open_return_path_on_source(s)) {
+            error_report("Unable to open return-path for postcopy");
+            migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                              MIGRATION_STATUS_FAILED);
+            migrate_fd_cleanup(s);
+            return;
+        }
+    }
+
     migrate_compress_threads_create();
     qemu_thread_create(&s->thread, "migration", migration_thread, s,
                        QEMU_THREAD_JOINABLE);
+    s->migration_thread_running = true;
+}
+
+PostcopyState  postcopy_state_get(void)
+{
+    return atomic_mb_read(&incoming_postcopy_state);
 }
+
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state)
+{
+    return atomic_xchg(&incoming_postcopy_state, new_state);
+}
+
diff --git a/qemu/migration/postcopy-ram.c b/qemu/migration/postcopy-ram.c
new file mode 100644
index 000000000..fbd0064fc
--- /dev/null
+++ b/qemu/migration/postcopy-ram.c
@@ -0,0 +1,761 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+/*
+ * Postcopy is a migration technique where the execution flips from the
+ * source to the destination before all the data has been copied.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "migration/postcopy-ram.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/balloon.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+/* Arbitrary limit on size of each discard command,
+ * keeps them around ~200 bytes
+ */
+#define MAX_DISCARDS_PER_COMMAND 12
+
+struct PostcopyDiscardState {
+    const char *ramblock_name;
+    uint64_t offset; /* Bitmap entry for the 1st bit of this RAMBlock */
+    uint16_t cur_entry;
+    /*
+     * Start and length of a discard range (bytes)
+     */
+    uint64_t start_list[MAX_DISCARDS_PER_COMMAND];
+    uint64_t length_list[MAX_DISCARDS_PER_COMMAND];
+    unsigned int nsentwords;
+    unsigned int nsentcmds;
+};
+
+/* Postcopy needs to detect accesses to pages that haven't yet been copied
+ * across, and efficiently map new pages in, the techniques for doing this
+ * are target OS specific.
+ */
+#if defined(__linux__)
+
+#include <poll.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <asm/types.h> /* for __u64 */
+#endif
+
+#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD)
+#include <sys/eventfd.h>
+#include <linux/userfaultfd.h>
+
+static bool ufd_version_check(int ufd)
+{
+    struct uffdio_api api_struct;
+    uint64_t ioctl_mask;
+
+    api_struct.api = UFFD_API;
+    api_struct.features = 0;
+    if (ioctl(ufd, UFFDIO_API, &api_struct)) {
+        error_report("postcopy_ram_supported_by_host: UFFDIO_API failed: %s",
+                     strerror(errno));
+        return false;
+    }
+
+    ioctl_mask = (__u64)1 << _UFFDIO_REGISTER |
+                 (__u64)1 << _UFFDIO_UNREGISTER;
+    if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
+        error_report("Missing userfault features: %" PRIx64,
+                     (uint64_t)(~api_struct.ioctls & ioctl_mask));
+        return false;
+    }
+
+    return true;
+}
+
+/*
+ * Note: This has the side effect of munlock'ing all of RAM, that's
+ * normally fine since if the postcopy succeeds it gets turned back on at the
+ * end.
+ */
+bool postcopy_ram_supported_by_host(void)
+{
+    long pagesize = getpagesize();
+    int ufd = -1;
+    bool ret = false; /* Error unless we change it */
+    void *testarea = NULL;
+    struct uffdio_register reg_struct;
+    struct uffdio_range range_struct;
+    uint64_t feature_mask;
+
+    if ((1ul << qemu_target_page_bits()) > pagesize) {
+        error_report("Target page size bigger than host page size");
+        goto out;
+    }
+
+    ufd = syscall(__NR_userfaultfd, O_CLOEXEC);
+    if (ufd == -1) {
+        error_report("%s: userfaultfd not available: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+
+    /* Version and features check */
+    if (!ufd_version_check(ufd)) {
+        goto out;
+    }
+
+    /*
+     * userfault and mlock don't go together; we'll put it back later if
+     * it was enabled.
+     */
+    if (munlockall()) {
+        error_report("%s: munlockall: %s", __func__,  strerror(errno));
+        return -1;
+    }
+
+    /*
+     *  We need to check that the ops we need are supported on anon memory
+     *  To do that we need to register a chunk and see the flags that
+     *  are returned.
+     */
+    testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                                    MAP_ANONYMOUS, -1, 0);
+    if (testarea == MAP_FAILED) {
+        error_report("%s: Failed to map test area: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+    g_assert(((size_t)testarea & (pagesize-1)) == 0);
+
+    reg_struct.range.start = (uintptr_t)testarea;
+    reg_struct.range.len = pagesize;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    if (ioctl(ufd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    range_struct.start = (uintptr_t)testarea;
+    range_struct.len = pagesize;
+    if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s userfault unregister: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    feature_mask = (__u64)1 << _UFFDIO_WAKE |
+                   (__u64)1 << _UFFDIO_COPY |
+                   (__u64)1 << _UFFDIO_ZEROPAGE;
+    if ((reg_struct.ioctls & feature_mask) != feature_mask) {
+        error_report("Missing userfault map features: %" PRIx64,
+                     (uint64_t)(~reg_struct.ioctls & feature_mask));
+        goto out;
+    }
+
+    /* Success! */
+    ret = true;
+out:
+    if (testarea) {
+        munmap(testarea, pagesize);
+    }
+    if (ufd != -1) {
+        close(ufd);
+    }
+    return ret;
+}
+
+/**
+ * postcopy_ram_discard_range: Discard a range of memory.
+ * We can assume that if we've been called postcopy_ram_hosttest returned true.
+ *
+ * @mis: Current incoming migration state.
+ * @start, @length: range of memory to discard.
+ *
+ * returns: 0 on success.
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    trace_postcopy_ram_discard_range(start, length);
+    if (madvise(start, length, MADV_DONTNEED)) {
+        error_report("%s MADV_DONTNEED: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Setup an area of RAM so that it *can* be used for postcopy later; this
+ * must be done right at the start prior to pre-copy.
+ * opaque should be the MIS.
+ */
+static int init_range(const char *block_name, void *host_addr,
+                      ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+
+    trace_postcopy_init_range(block_name, host_addr, offset, length);
+
+    /*
+     * We need the whole of RAM to be truly empty for postcopy, so things
+     * like ROMs and any data tables built during init must be zero'd
+     * - we're going to get the copy from the source anyway.
+     * (Precopy will just overwrite this data, so doesn't need the discard)
+     */
+    if (postcopy_ram_discard_range(mis, host_addr, length)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of migration, undo the effects of init_range
+ * opaque should be the MIS.
+ */
+static int cleanup_range(const char *block_name, void *host_addr,
+                        ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_range range_struct;
+    trace_postcopy_cleanup_range(block_name, host_addr, offset, length);
+
+    /*
+     * We turned off hugepage for the precopy stage with postcopy enabled
+     * we can turn it back on now.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE);
+
+    /*
+     * We can also turn off userfault now since we should have all the
+     * pages.   It can be useful to leave it on to debug postcopy
+     * if you're not sure it's always getting every page.
+     */
+    range_struct.start = (uintptr_t)host_addr;
+    range_struct.len = length;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s: userfault unregister %s", __func__, strerror(errno));
+
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from arch_init's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    if (qemu_ram_foreach_block(init_range, mis)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    trace_postcopy_ram_incoming_cleanup_entry();
+
+    if (mis->have_fault_thread) {
+        uint64_t tmp64;
+
+        if (qemu_ram_foreach_block(cleanup_range, mis)) {
+            return -1;
+        }
+        /*
+         * Tell the fault_thread to exit, it's an eventfd that should
+         * currently be at 0, we're going to increment it to 1
+         */
+        tmp64 = 1;
+        if (write(mis->userfault_quit_fd, &tmp64, 8) == 8) {
+            trace_postcopy_ram_incoming_cleanup_join();
+            qemu_thread_join(&mis->fault_thread);
+        } else {
+            /* Not much we can do here, but may as well report it */
+            error_report("%s: incrementing userfault_quit_fd: %s", __func__,
+                         strerror(errno));
+        }
+        trace_postcopy_ram_incoming_cleanup_closeuf();
+        close(mis->userfault_fd);
+        close(mis->userfault_quit_fd);
+        mis->have_fault_thread = false;
+    }
+
+    qemu_balloon_inhibit(false);
+
+    if (enable_mlock) {
+        if (os_mlock() < 0) {
+            error_report("mlock: %s", strerror(errno));
+            /*
+             * It doesn't feel right to fail at this point, we have a valid
+             * VM state.
+             */
+        }
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_END);
+    migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
+
+    if (mis->postcopy_tmp_page) {
+        munmap(mis->postcopy_tmp_page, getpagesize());
+        mis->postcopy_tmp_page = NULL;
+    }
+    trace_postcopy_ram_incoming_cleanup_exit();
+    return 0;
+}
+
+/*
+ * Disable huge pages on an area
+ */
+static int nhp_range(const char *block_name, void *host_addr,
+                    ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    trace_postcopy_nhp_range(block_name, host_addr, offset, length);
+
+    /*
+     * Before we do discards we need to ensure those discards really
+     * do delete areas of the page, even if THP thinks a hugepage would
+     * be a good idea, so force hugepages off.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE);
+
+    return 0;
+}
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    if (qemu_ram_foreach_block(nhp_range, mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_DISCARD);
+
+    return 0;
+}
+
+/*
+ * Mark the given area of RAM as requiring notification to unwritten areas
+ * Used as a  callback on qemu_ram_foreach_block.
+ *   host_addr: Base of area to mark
+ *   offset: Offset in the whole ram arena
+ *   length: Length of the section
+ *   opaque: MigrationIncomingState pointer
+ * Returns 0 on success
+ */
+static int ram_block_enable_notify(const char *block_name, void *host_addr,
+                                   ram_addr_t offset, ram_addr_t length,
+                                   void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_register reg_struct;
+
+    reg_struct.range.start = (uintptr_t)host_addr;
+    reg_struct.range.len = length;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    /* Now tell our userfault_fd that it's responsible for this area */
+    if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Handle faults detected by the USERFAULT markings
+ */
+static void *postcopy_ram_fault_thread(void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffd_msg msg;
+    int ret;
+    size_t hostpagesize = getpagesize();
+    RAMBlock *rb = NULL;
+    RAMBlock *last_rb = NULL; /* last RAMBlock we sent part of */
+
+    trace_postcopy_ram_fault_thread_entry();
+    qemu_sem_post(&mis->fault_thread_sem);
+
+    while (true) {
+        ram_addr_t rb_offset;
+        ram_addr_t in_raspace;
+        struct pollfd pfd[2];
+
+        /*
+         * We're mainly waiting for the kernel to give us a faulting HVA,
+         * however we can be told to quit via userfault_quit_fd which is
+         * an eventfd
+         */
+        pfd[0].fd = mis->userfault_fd;
+        pfd[0].events = POLLIN;
+        pfd[0].revents = 0;
+        pfd[1].fd = mis->userfault_quit_fd;
+        pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */
+        pfd[1].revents = 0;
+
+        if (poll(pfd, 2, -1 /* Wait forever */) == -1) {
+            error_report("%s: userfault poll: %s", __func__, strerror(errno));
+            break;
+        }
+
+        if (pfd[1].revents) {
+            trace_postcopy_ram_fault_thread_quit();
+            break;
+        }
+
+        ret = read(mis->userfault_fd, &msg, sizeof(msg));
+        if (ret != sizeof(msg)) {
+            if (errno == EAGAIN) {
+                /*
+                 * if a wake up happens on the other thread just after
+                 * the poll, there is nothing to read.
+                 */
+                continue;
+            }
+            if (ret < 0) {
+                error_report("%s: Failed to read full userfault message: %s",
+                             __func__, strerror(errno));
+                break;
+            } else {
+                error_report("%s: Read %d bytes from userfaultfd expected %zd",
+                             __func__, ret, sizeof(msg));
+                break; /* Lost alignment, don't know what we'd read next */
+            }
+        }
+        if (msg.event != UFFD_EVENT_PAGEFAULT) {
+            error_report("%s: Read unexpected event %ud from userfaultfd",
+                         __func__, msg.event);
+            continue; /* It's not a page fault, shouldn't happen */
+        }
+
+        rb = qemu_ram_block_from_host(
+                 (void *)(uintptr_t)msg.arg.pagefault.address,
+                 true, &in_raspace, &rb_offset);
+        if (!rb) {
+            error_report("postcopy_ram_fault_thread: Fault outside guest: %"
+                         PRIx64, (uint64_t)msg.arg.pagefault.address);
+            break;
+        }
+
+        rb_offset &= ~(hostpagesize - 1);
+        trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address,
+                                                qemu_ram_get_idstr(rb),
+                                                rb_offset);
+
+        /*
+         * Send the request to the source - we want to request one
+         * of our host page sizes (which is >= TPS)
+         */
+        if (rb != last_rb) {
+            last_rb = rb;
+            migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb),
+                                     rb_offset, hostpagesize);
+        } else {
+            /* Save some space */
+            migrate_send_rp_req_pages(mis, NULL,
+                                     rb_offset, hostpagesize);
+        }
+    }
+    trace_postcopy_ram_fault_thread_exit();
+    return NULL;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    /* Open the fd for the kernel to give us userfaults */
+    mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
+    if (mis->userfault_fd == -1) {
+        error_report("%s: Failed to open userfault fd: %s", __func__,
+                     strerror(errno));
+        return -1;
+    }
+
+    /*
+     * Although the host check already tested the API, we need to
+     * do the check again as an ABI handshake on the new fd.
+     */
+    if (!ufd_version_check(mis->userfault_fd)) {
+        return -1;
+    }
+
+    /* Now an eventfd we use to tell the fault-thread to quit */
+    mis->userfault_quit_fd = eventfd(0, EFD_CLOEXEC);
+    if (mis->userfault_quit_fd == -1) {
+        error_report("%s: Opening userfault_quit_fd: %s", __func__,
+                     strerror(errno));
+        close(mis->userfault_fd);
+        return -1;
+    }
+
+    qemu_sem_init(&mis->fault_thread_sem, 0);
+    qemu_thread_create(&mis->fault_thread, "postcopy/fault",
+                       postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
+    qemu_sem_wait(&mis->fault_thread_sem);
+    qemu_sem_destroy(&mis->fault_thread_sem);
+    mis->have_fault_thread = true;
+
+    /* Mark so that we get notified of accesses to unwritten areas */
+    if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) {
+        return -1;
+    }
+
+    /*
+     * Ballooning can mark pages as absent while we're postcopying
+     * that would cause false userfaults.
+     */
+    qemu_balloon_inhibit(true);
+
+    trace_postcopy_ram_enable_notify();
+
+    return 0;
+}
+
+/*
+ * Place a host page (from) at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    struct uffdio_copy copy_struct;
+
+    copy_struct.dst = (uint64_t)(uintptr_t)host;
+    copy_struct.src = (uint64_t)(uintptr_t)from;
+    copy_struct.len = getpagesize();
+    copy_struct.mode = 0;
+
+    /* copy also acks to the kernel waking the stalled thread up
+     * TODO: We can inhibit that ack and only do it if it was requested
+     * which would be slightly cheaper, but we'd have to be careful
+     * of the order of updating our page state.
+     */
+    if (ioctl(mis->userfault_fd, UFFDIO_COPY, &copy_struct)) {
+        int e = errno;
+        error_report("%s: %s copy host: %p from: %p",
+                     __func__, strerror(e), host, from);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page(host);
+    return 0;
+}
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    struct uffdio_zeropage zero_struct;
+
+    zero_struct.range.start = (uint64_t)(uintptr_t)host;
+    zero_struct.range.len = getpagesize();
+    zero_struct.mode = 0;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_ZEROPAGE, &zero_struct)) {
+        int e = errno;
+        error_report("%s: %s zero host: %p",
+                     __func__, strerror(e), host);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page_zero(host);
+    return 0;
+}
+
+/*
+ * Returns a target page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * The same address is used repeatedly, postcopy_place_page just takes the
+ * backing page away.
+ * Returns: Pointer to allocated page
+ *
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    if (!mis->postcopy_tmp_page) {
+        mis->postcopy_tmp_page = mmap(NULL, getpagesize(),
+                             PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                             MAP_ANONYMOUS, -1, 0);
+        if (!mis->postcopy_tmp_page) {
+            error_report("%s: %s", __func__, strerror(errno));
+            return NULL;
+        }
+    }
+
+    return mis->postcopy_tmp_page;
+}
+
+#else
+/* No target OS support, stubs just fail */
+bool postcopy_ram_supported_by_host(void)
+{
+    error_report("%s: No OS support", __func__);
+    return false;
+}
+
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    error_report("postcopy_ram_incoming_init: No OS support");
+    return -1;
+}
+
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    assert(0);
+    return -1;
+}
+
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    assert(0);
+    return NULL;
+}
+
+#endif
+
+/* ------------------------------------------------------------------------- */
+
+/**
+ * postcopy_discard_send_init: Called at the start of each RAMBlock before
+ *   asking to discard individual ranges.
+ *
+ * @ms: The current migration state.
+ * @offset: the bitmap offset of the named RAMBlock in the migration
+ *   bitmap.
+ * @name: RAMBlock that discards will operate on.
+ *
+ * returns: a new PDS.
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name)
+{
+    PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState));
+
+    if (res) {
+        res->ramblock_name = name;
+        res->offset = offset;
+    }
+
+    return res;
+}
+
+/**
+ * postcopy_discard_send_range: Called by the bitmap code for each chunk to
+ *   discard. May send a discard message, may just leave it queued to
+ *   be sent later.
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ * @start,@length: a range of pages in the migration bitmap in the
+ *   RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                unsigned long start, unsigned long length)
+{
+    size_t tp_bits = qemu_target_page_bits();
+    /* Convert to byte offsets within the RAM block */
+    pds->start_list[pds->cur_entry] = (start - pds->offset) << tp_bits;
+    pds->length_list[pds->cur_entry] = length << tp_bits;
+    trace_postcopy_discard_send_range(pds->ramblock_name, start, length);
+    pds->cur_entry++;
+    pds->nsentwords++;
+
+    if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) {
+        /* Full set, ship it! */
+        qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
+                                              pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+        pds->cur_entry = 0;
+    }
+}
+
+/**
+ * postcopy_discard_send_finish: Called at the end of each RAMBlock by the
+ * bitmap code. Sends any outstanding discard messages, frees the PDS
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ */
+void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds)
+{
+    /* Anything unsent? */
+    if (pds->cur_entry) {
+        qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
+                                              pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+    }
+
+    trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords,
+                                       pds->nsentcmds);
+
+    g_free(pds);
+}
diff --git a/qemu/migration/qemu-file-buf.c b/qemu/migration/qemu-file-buf.c
index 2de9330ca..7b8e78e99 100644
--- a/qemu/migration/qemu-file-buf.c
+++ b/qemu/migration/qemu-file-buf.c
@@ -25,11 +25,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
@@ -372,7 +373,8 @@ typedef struct QEMUBuffer {
     bool qsb_allocated;
 } QEMUBuffer;
 
-static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                              size_t size)
 {
     QEMUBuffer *s = opaque;
     ssize_t len = qsb_get_length(s->qsb) - pos;
@@ -387,8 +389,8 @@ static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     return qsb_get_buffer(s->qsb, pos, len, buf);
 }
 
-static int buf_put_buffer(void *opaque, const uint8_t *buf,
-                          int64_t pos, int size)
+static ssize_t buf_put_buffer(void *opaque, const uint8_t *buf,
+                              int64_t pos, size_t size)
 {
     QEMUBuffer *s = opaque;
 
@@ -439,7 +441,7 @@ QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUBuffer));
+    s = g_new0(QEMUBuffer, 1);
     s->qsb = input;
 
     if (s->qsb == NULL) {
diff --git a/qemu/migration/qemu-file-stdio.c b/qemu/migration/qemu-file-stdio.c
index 285068b30..f402e8f70 100644
--- a/qemu/migration/qemu-file-stdio.c
+++ b/qemu/migration/qemu-file-stdio.c
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/qemu-file.h"
 
 typedef struct QEMUFileStdio {
@@ -37,11 +38,11 @@ static int stdio_get_fd(void *opaque)
     return fileno(s->stdio_file);
 }
 
-static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
-                            int size)
+static ssize_t stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     QEMUFileStdio *s = opaque;
-    int res;
+    size_t res;
 
     res = fwrite(buf, 1, size, s->stdio_file);
 
@@ -51,11 +52,12 @@ static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
     return res;
 }
 
-static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     QEMUFileStdio *s = opaque;
     FILE *fp = s->stdio_file;
-    int bytes;
+    ssize_t bytes;
 
     for (;;) {
         clearerr(fp);
@@ -143,7 +145,7 @@ QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileStdio));
+    s = g_new0(QEMUFileStdio, 1);
 
     s->stdio_file = stdio_file;
 
@@ -175,7 +177,7 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileStdio));
+    s = g_new0(QEMUFileStdio, 1);
 
     s->stdio_file = fopen(filename, mode);
     if (!s->stdio_file) {
diff --git a/qemu/migration/qemu-file-unix.c b/qemu/migration/qemu-file-unix.c
index bfbc0861a..4474e18ff 100644
--- a/qemu/migration/qemu-file-unix.c
+++ b/qemu/migration/qemu-file-unix.c
@@ -21,10 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
 
@@ -39,12 +41,42 @@ static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     QEMUFileSocket *s = opaque;
     ssize_t len;
     ssize_t size = iov_size(iov, iovcnt);
+    ssize_t offset = 0;
+    int     err;
 
-    len = iov_send(s->fd, iov, iovcnt, 0, size);
-    if (len < size) {
-        len = -socket_error();
-    }
-    return len;
+    while (size > 0) {
+        len = iov_send(s->fd, iov, iovcnt, offset, size);
+
+        if (len > 0) {
+            size -= len;
+            offset += len;
+        }
+
+        if (size > 0) {
+            if (errno != EAGAIN && errno != EWOULDBLOCK) {
+                error_report("socket_writev_buffer: Got err=%d for (%zu/%zu)",
+                             errno, (size_t)size, (size_t)len);
+                /*
+                 * If I've already sent some but only just got the error, I
+                 * could return the amount validly sent so far and wait for the
+                 * next call to report the error, but I'd rather flag the error
+                 * immediately.
+                 */
+                return -errno;
+            }
+
+            /* Emulate blocking */
+            GPollFD pfd;
+
+            pfd.fd = s->fd;
+            pfd.events = G_IO_OUT | G_IO_ERR;
+            pfd.revents = 0;
+            TFR(err = g_poll(&pfd, 1, -1 /* no timeout */));
+            /* Errors other than EINTR intentionally ignored */
+        }
+     }
+
+    return offset;
 }
 
 static int socket_get_fd(void *opaque)
@@ -54,7 +86,8 @@ static int socket_get_fd(void *opaque)
     return s->fd;
 }
 
-static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                 size_t size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
@@ -64,15 +97,15 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
         if (len != -1) {
             break;
         }
-        if (socket_error() == EAGAIN) {
+        if (errno == EAGAIN) {
             yield_until_fd_readable(s->fd);
-        } else if (socket_error() != EINTR) {
+        } else if (errno != EINTR) {
             break;
         }
     }
 
     if (len == -1) {
-        len = -socket_error();
+        len = -errno;
     }
     return len;
 }
@@ -96,6 +129,56 @@ static int socket_shutdown(void *opaque, bool rd, bool wr)
     }
 }
 
+static int socket_return_close(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+    /*
+     * Note: We don't close the socket, that should be done by the forward
+     * path.
+     */
+    g_free(s);
+    return 0;
+}
+
+static const QEMUFileOps socket_return_read_ops = {
+    .get_fd          = socket_get_fd,
+    .get_buffer      = socket_get_buffer,
+    .close           = socket_return_close,
+    .shut_down       = socket_shutdown,
+};
+
+static const QEMUFileOps socket_return_write_ops = {
+    .get_fd          = socket_get_fd,
+    .writev_buffer   = socket_writev_buffer,
+    .close           = socket_return_close,
+    .shut_down       = socket_shutdown,
+};
+
+/*
+ * Give a QEMUFile* off the same socket but data in the opposite
+ * direction.
+ */
+static QEMUFile *socket_get_return_path(void *opaque)
+{
+    QEMUFileSocket *forward = opaque;
+    QEMUFileSocket *reverse;
+
+    if (qemu_file_get_error(forward->file)) {
+        /* If the forward file is in error, don't try and open a return */
+        return NULL;
+    }
+
+    reverse = g_malloc0(sizeof(QEMUFileSocket));
+    reverse->fd = forward->fd;
+    /* I don't think there's a better way to tell which direction 'this' is */
+    if (forward->file->ops->get_buffer != NULL) {
+        /* being called from the read side, so we need to be able to write */
+        return qemu_fopen_ops(reverse, &socket_return_write_ops);
+    } else {
+        return qemu_fopen_ops(reverse, &socket_return_read_ops);
+    }
+}
+
 static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
                                   int64_t pos)
 {
@@ -138,7 +221,8 @@ static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     return total;
 }
 
-static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                              size_t size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
@@ -192,7 +276,7 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileSocket));
+    s = g_new0(QEMUFileSocket, 1);
     s->fd = fd;
 
     if (mode[0] == 'r') {
@@ -204,18 +288,19 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
 }
 
 static const QEMUFileOps socket_read_ops = {
-    .get_fd     = socket_get_fd,
-    .get_buffer = socket_get_buffer,
-    .close      = socket_close,
-    .shut_down  = socket_shutdown
-
+    .get_fd          = socket_get_fd,
+    .get_buffer      = socket_get_buffer,
+    .close           = socket_close,
+    .shut_down       = socket_shutdown,
+    .get_return_path = socket_get_return_path
 };
 
 static const QEMUFileOps socket_write_ops = {
-    .get_fd        = socket_get_fd,
-    .writev_buffer = socket_writev_buffer,
-    .close         = socket_close,
-    .shut_down     = socket_shutdown
+    .get_fd          = socket_get_fd,
+    .writev_buffer   = socket_writev_buffer,
+    .close           = socket_close,
+    .shut_down       = socket_shutdown,
+    .get_return_path = socket_get_return_path
 };
 
 QEMUFile *qemu_fopen_socket(int fd, const char *mode)
@@ -226,7 +311,7 @@ QEMUFile *qemu_fopen_socket(int fd, const char *mode)
         return NULL;
     }
 
-    s = g_malloc0(sizeof(QEMUFileSocket));
+    s = g_new0(QEMUFileSocket, 1);
     s->fd = fd;
     if (mode[0] == 'w') {
         qemu_set_block(s->fd);
diff --git a/qemu/migration/qemu-file.c b/qemu/migration/qemu-file.c
index 6bb3dc15c..6f4a1299b 100644
--- a/qemu/migration/qemu-file.c
+++ b/qemu/migration/qemu-file.c
@@ -21,12 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <zlib.h>
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "migration/qemu-file-internal.h"
@@ -44,6 +45,18 @@ int qemu_file_shutdown(QEMUFile *f)
     return f->ops->shut_down(f->opaque, true, true);
 }
 
+/*
+ * Result: QEMUFile* for a 'return path' for comms in the opposite direction
+ *         NULL if not available
+ */
+QEMUFile *qemu_file_get_return_path(QEMUFile *f)
+{
+    if (!f->ops->get_return_path) {
+        return NULL;
+    }
+    return f->ops->get_return_path(f->opaque);
+}
+
 bool qemu_file_mode_is_not_valid(const char *mode)
 {
     if (mode == NULL ||
@@ -60,7 +73,7 @@ QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
 {
     QEMUFile *f;
 
-    f = g_malloc0(sizeof(QEMUFile));
+    f = g_new0(QEMUFile, 1);
 
     f->opaque = opaque;
     f->ops = ops;
@@ -270,7 +283,7 @@ int qemu_fclose(QEMUFile *f)
     return ret;
 }
 
-static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size)
+static void add_to_iovec(QEMUFile *f, const uint8_t *buf, size_t size)
 {
     /* check for adjacent buffer and coalesce them */
     if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base +
@@ -286,7 +299,7 @@ static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size)
     }
 }
 
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size)
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size)
 {
     if (!f->ops->writev_buffer) {
         qemu_put_buffer(f, buf, size);
@@ -301,9 +314,9 @@ void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size)
     add_to_iovec(f, buf, size);
 }
 
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size)
 {
-    int l;
+    size_t l;
 
     if (f->last_error) {
         return;
@@ -363,10 +376,10 @@ void qemu_file_skip(QEMUFile *f, int size)
  * return as many as it managed to read (assuming blocking fd's which
  * all current QEMUFile are)
  */
-int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset)
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset)
 {
-    int pending;
-    int index;
+    ssize_t pending;
+    size_t index;
 
     assert(!qemu_file_is_writable(f));
     assert(offset < IO_BUF_SIZE);
@@ -411,13 +424,13 @@ int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset)
  * return as many as it managed to read (assuming blocking fd's which
  * all current QEMUFile are)
  */
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size)
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size)
 {
-    int pending = size;
-    int done = 0;
+    size_t pending = size;
+    size_t done = 0;
 
     while (pending > 0) {
-        int res;
+        size_t res;
         uint8_t *src;
 
         res = qemu_peek_buffer(f, &src, MIN(pending, IO_BUF_SIZE), 0);
@@ -434,6 +447,43 @@ int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size)
 }
 
 /*
+ * Read 'size' bytes of data from the file.
+ * 'size' can be larger than the internal buffer.
+ *
+ * The data:
+ *   may be held on an internal buffer (in which case *buf is updated
+ *     to point to it) that is valid until the next qemu_file operation.
+ * OR
+ *   will be copied to the *buf that was passed in.
+ *
+ * The code tries to avoid the copy if possible.
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ *
+ * Note: Since **buf may get changed, the caller should take care to
+ *       keep a pointer to the original buffer if it needs to deallocate it.
+ */
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size)
+{
+    if (size < IO_BUF_SIZE) {
+        size_t res;
+        uint8_t *src;
+
+        res = qemu_peek_buffer(f, &src, size, 0);
+
+        if (res == size) {
+            qemu_file_skip(f, res);
+            *buf = src;
+            return res;
+        }
+    }
+
+    return qemu_get_buffer(f, *buf, size);
+}
+
+/*
  * Peeks a single byte from the buffer; this isn't guaranteed to work if
  * offset leaves a gap after the previous read/peeked data.
  */
@@ -611,3 +661,18 @@ size_t qemu_get_counted_string(QEMUFile *f, char buf[256])
 
     return res == len ? res : 0;
 }
+
+/*
+ * Set the blocking state of the QEMUFile.
+ * Note: On some transports the OS only keeps a single blocking state for
+ *       both directions, and thus changing the blocking on the main
+ *       QEMUFile can also affect the return path.
+ */
+void qemu_file_set_blocking(QEMUFile *f, bool block)
+{
+    if (block) {
+        qemu_set_block(qemu_get_fd(f));
+    } else {
+        qemu_set_nonblock(qemu_get_fd(f));
+    }
+}
diff --git a/qemu/migration/ram.c b/qemu/migration/ram.c
index 6249f6e60..3f057388c 100644
--- a/qemu/migration/ram.c
+++ b/qemu/migration/ram.c
@@ -25,13 +25,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include <zlib.h>
+#include "qapi-event.h"
+#include "qemu/cutils.h"
 #include "qemu/bitops.h"
 #include "qemu/bitmap.h"
 #include "qemu/timer.h"
 #include "qemu/main-loop.h"
 #include "migration/migration.h"
+#include "migration/postcopy-ram.h"
 #include "exec/address-spaces.h"
 #include "migration/page_cache.h"
 #include "qemu/error-report.h"
@@ -47,9 +50,7 @@
     do { } while (0)
 #endif
 
-static bool mig_throttle_on;
 static int dirty_rate_high_cnt;
-static void check_guest_throttling(void);
 
 static uint64_t bitmap_sync_count;
 
@@ -221,12 +222,34 @@ static RAMBlock *last_seen_block;
 /* This is the last block from where we have sent data */
 static RAMBlock *last_sent_block;
 static ram_addr_t last_offset;
-static unsigned long *migration_bitmap;
 static QemuMutex migration_bitmap_mutex;
 static uint64_t migration_dirty_pages;
 static uint32_t last_version;
 static bool ram_bulk_stage;
 
+/* used by the search for pages to send */
+struct PageSearchStatus {
+    /* Current block being searched */
+    RAMBlock    *block;
+    /* Current offset to search from */
+    ram_addr_t   offset;
+    /* Set once we wrap around */
+    bool         complete_round;
+};
+typedef struct PageSearchStatus PageSearchStatus;
+
+static struct BitmapRcu {
+    struct rcu_head rcu;
+    /* Main migration bitmap */
+    unsigned long *bmap;
+    /* bitmap of pages that haven't been sent even once
+     * only maintained and used in postcopy at the moment
+     * where it's used to send the dirtymap at the start
+     * of the postcopy phase
+     */
+    unsigned long *unsentmap;
+} *migration_bitmap_rcu;
+
 struct CompressParam {
     bool start;
     bool done;
@@ -243,7 +266,7 @@ struct DecompressParam {
     QemuMutex mutex;
     QemuCond cond;
     void *des;
-    uint8 *compbuf;
+    uint8_t *compbuf;
     int len;
 };
 typedef struct DecompressParam DecompressParam;
@@ -264,7 +287,6 @@ static bool quit_comp_thread;
 static bool quit_decomp_thread;
 static DecompressParam *decomp_param;
 static QemuThread *decompress_threads;
-static uint8_t *compressed_data_buf;
 
 static int do_compress_ram_page(CompressParam *param);
 
@@ -396,6 +418,29 @@ static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset)
     return size;
 }
 
+/* Reduce amount of guest cpu execution to hopefully slow down memory writes.
+ * If guest dirty memory rate is reduced below the rate at which we can
+ * transfer pages to the destination then we should be able to complete
+ * migration. Some workloads dirty memory way too fast and will not effectively
+ * converge, even with auto-converge.
+ */
+static void mig_throttle_guest_down(void)
+{
+    MigrationState *s = migrate_get_current();
+    uint64_t pct_initial =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL];
+    uint64_t pct_icrement =
+            s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT];
+
+    /* We have not started throttling yet. Let's start it. */
+    if (!cpu_throttle_active()) {
+        cpu_throttle_set(pct_initial);
+    } else {
+        /* Throttling already on, just increase the rate */
+        cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
+    }
+}
+
 /* Update the xbzrle cache to reflect a page that's been sent as all 0.
  * The important thing is that a stale (not-yet-0'd) page be replaced
  * by the new data.
@@ -495,43 +540,60 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
     return 1;
 }
 
-/* Called with rcu_read_lock() to protect migration_bitmap */
+/* Called with rcu_read_lock() to protect migration_bitmap
+ * rb: The RAMBlock  to search for dirty pages in
+ * start: Start address (typically so we can continue from previous page)
+ * ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ *
+ * Returns: byte offset within memory region of the start of a dirty page
+ */
 static inline
-ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
-                                                 ram_addr_t start)
+ram_addr_t migration_bitmap_find_dirty(RAMBlock *rb,
+                                       ram_addr_t start,
+                                       ram_addr_t *ram_addr_abs)
 {
-    unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
+    unsigned long base = rb->offset >> TARGET_PAGE_BITS;
     unsigned long nr = base + (start >> TARGET_PAGE_BITS);
-    uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr));
-    unsigned long size = base + (mr_size >> TARGET_PAGE_BITS);
+    uint64_t rb_size = rb->used_length;
+    unsigned long size = base + (rb_size >> TARGET_PAGE_BITS);
     unsigned long *bitmap;
 
     unsigned long next;
 
-    bitmap = atomic_rcu_read(&migration_bitmap);
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
     if (ram_bulk_stage && nr > base) {
         next = nr + 1;
     } else {
         next = find_next_bit(bitmap, size, nr);
     }
 
-    if (next < size) {
-        clear_bit(next, bitmap);
+    *ram_addr_abs = next << TARGET_PAGE_BITS;
+    return (next - base) << TARGET_PAGE_BITS;
+}
+
+static inline bool migration_bitmap_clear_dirty(ram_addr_t addr)
+{
+    bool ret;
+    int nr = addr >> TARGET_PAGE_BITS;
+    unsigned long *bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+
+    ret = test_and_clear_bit(nr, bitmap);
+
+    if (ret) {
         migration_dirty_pages--;
     }
-    return (next - base) << TARGET_PAGE_BITS;
+    return ret;
 }
 
-/* Called with rcu_read_lock() to protect migration_bitmap */
 static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
 {
     unsigned long *bitmap;
-    bitmap = atomic_rcu_read(&migration_bitmap);
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
     migration_dirty_pages +=
         cpu_physical_memory_sync_dirty_bitmap(bitmap, start, length);
 }
 
-
 /* Fix me: there are too many global variables used in migration process. */
 static int64_t start_time;
 static int64_t bytes_xfer_prev;
@@ -548,7 +610,6 @@ static void migration_bitmap_sync_init(void)
     iterations_prev = 0;
 }
 
-/* Called with iothread lock held, to protect ram_list.dirty_memory[] */
 static void migration_bitmap_sync(void)
 {
     RAMBlock *block;
@@ -573,7 +634,7 @@ static void migration_bitmap_sync(void)
     qemu_mutex_lock(&migration_bitmap_mutex);
     rcu_read_lock();
     QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        migration_bitmap_sync_range(block->mr->ram_addr, block->used_length);
+        migration_bitmap_sync_range(block->offset, block->used_length);
     }
     rcu_read_unlock();
     qemu_mutex_unlock(&migration_bitmap_mutex);
@@ -589,21 +650,21 @@ static void migration_bitmap_sync(void)
             /* The following detection logic can be refined later. For now:
                Check to see if the dirtied bytes is 50% more than the approx.
                amount of bytes that just got transferred since the last time we
-               were in this routine. If that happens >N times (for now N==4)
-               we turn on the throttle down logic */
+               were in this routine. If that happens twice, start or increase
+               throttling */
             bytes_xfer_now = ram_bytes_transferred();
+
             if (s->dirty_pages_rate &&
                (num_dirty_pages_period * TARGET_PAGE_SIZE >
                    (bytes_xfer_now - bytes_xfer_prev)/2) &&
-               (dirty_rate_high_cnt++ > 4)) {
+               (dirty_rate_high_cnt++ >= 2)) {
                     trace_migration_throttle();
-                    mig_throttle_on = true;
                     dirty_rate_high_cnt = 0;
+                    mig_throttle_guest_down();
              }
              bytes_xfer_prev = bytes_xfer_now;
-        } else {
-             mig_throttle_on = false;
         }
+
         if (migrate_use_xbzrle()) {
             if (iterations_prev != acct_info.iterations) {
                 acct_info.xbzrle_cache_miss_rate =
@@ -621,6 +682,9 @@ static void migration_bitmap_sync(void)
         num_dirty_pages_period = 0;
     }
     s->dirty_sync_count = bitmap_sync_count;
+    if (migrate_use_events()) {
+        qapi_event_send_migration_pass(bitmap_sync_count, NULL);
+    }
 }
 
 /**
@@ -655,6 +719,9 @@ static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
  * ram_save_page: Send the given page to the stream
  *
  * Returns: Number of pages written.
+ *          < 0 - error
+ *          >=0 - Number of pages written - this might legally be 0
+ *                if xbzrle noticed the page was the same.
  *
  * @f: QEMUFile where to send the data
  * @block: block that contains the page we want to send
@@ -662,18 +729,19 @@ static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
  * @last_stage: if we are at the completion stage
  * @bytes_transferred: increase it with the number of transferred bytes
  */
-static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
+static int ram_save_page(QEMUFile *f, PageSearchStatus *pss,
                          bool last_stage, uint64_t *bytes_transferred)
 {
     int pages = -1;
     uint64_t bytes_xmit;
     ram_addr_t current_addr;
-    MemoryRegion *mr = block->mr;
     uint8_t *p;
     int ret;
     bool send_async = true;
+    RAMBlock *block = pss->block;
+    ram_addr_t offset = pss->offset;
 
-    p = memory_region_get_ram_ptr(mr) + offset;
+    p = block->host + offset;
 
     /* In doubt sent page as normal */
     bytes_xmit = 0;
@@ -744,7 +812,7 @@ static int do_compress_ram_page(CompressParam *param)
     RAMBlock *block = param->block;
     ram_addr_t offset = param->offset;
 
-    p = memory_region_get_ram_ptr(block->mr) + (offset & TARGET_PAGE_MASK);
+    p = block->host + (offset & TARGET_PAGE_MASK);
 
     bytes_sent = save_page_header(param->file, block, offset |
                                   RAM_SAVE_FLAG_COMPRESS_PAGE);
@@ -846,17 +914,18 @@ static int compress_page_with_multi_thread(QEMUFile *f, RAMBlock *block,
  * @last_stage: if we are at the completion stage
  * @bytes_transferred: increase it with the number of transferred bytes
  */
-static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
-                                    ram_addr_t offset, bool last_stage,
+static int ram_save_compressed_page(QEMUFile *f, PageSearchStatus *pss,
+                                    bool last_stage,
                                     uint64_t *bytes_transferred)
 {
     int pages = -1;
     uint64_t bytes_xmit;
-    MemoryRegion *mr = block->mr;
     uint8_t *p;
     int ret;
+    RAMBlock *block = pss->block;
+    ram_addr_t offset = pss->offset;
 
-    p = memory_region_get_ram_ptr(mr) + offset;
+    p = block->host + offset;
 
     bytes_xmit = 0;
     ret = ram_control_save_page(f, block->offset,
@@ -909,6 +978,340 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
     return pages;
 }
 
+/*
+ * Find the next dirty page and update any state associated with
+ * the search process.
+ *
+ * Returns: True if a page is found
+ *
+ * @f: Current migration stream.
+ * @pss: Data about the state of the current dirty page scan.
+ * @*again: Set to false if the search has scanned the whole of RAM
+ * *ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ */
+static bool find_dirty_block(QEMUFile *f, PageSearchStatus *pss,
+                             bool *again, ram_addr_t *ram_addr_abs)
+{
+    pss->offset = migration_bitmap_find_dirty(pss->block, pss->offset,
+                                              ram_addr_abs);
+    if (pss->complete_round && pss->block == last_seen_block &&
+        pss->offset >= last_offset) {
+        /*
+         * We've been once around the RAM and haven't found anything.
+         * Give up.
+         */
+        *again = false;
+        return false;
+    }
+    if (pss->offset >= pss->block->used_length) {
+        /* Didn't find anything in this RAM Block */
+        pss->offset = 0;
+        pss->block = QLIST_NEXT_RCU(pss->block, next);
+        if (!pss->block) {
+            /* Hit the end of the list */
+            pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
+            /* Flag that we've looped */
+            pss->complete_round = true;
+            ram_bulk_stage = false;
+            if (migrate_use_xbzrle()) {
+                /* If xbzrle is on, stop using the data compression at this
+                 * point. In theory, xbzrle can do better than compression.
+                 */
+                flush_compressed_data(f);
+                compression_switch = false;
+            }
+        }
+        /* Didn't find anything this time, but try again on the new block */
+        *again = true;
+        return false;
+    } else {
+        /* Can go around again, but... */
+        *again = true;
+        /* We've found something so probably don't need to */
+        return true;
+    }
+}
+
+/*
+ * Helper for 'get_queued_page' - gets a page off the queue
+ *      ms:      MigrationState in
+ * *offset:      Used to return the offset within the RAMBlock
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      block (or NULL if none available)
+ */
+static RAMBlock *unqueue_page(MigrationState *ms, ram_addr_t *offset,
+                              ram_addr_t *ram_addr_abs)
+{
+    RAMBlock *block = NULL;
+
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    if (!QSIMPLEQ_EMPTY(&ms->src_page_requests)) {
+        struct MigrationSrcPageRequest *entry =
+                                QSIMPLEQ_FIRST(&ms->src_page_requests);
+        block = entry->rb;
+        *offset = entry->offset;
+        *ram_addr_abs = (entry->offset + entry->rb->offset) &
+                        TARGET_PAGE_MASK;
+
+        if (entry->len > TARGET_PAGE_SIZE) {
+            entry->len -= TARGET_PAGE_SIZE;
+            entry->offset += TARGET_PAGE_SIZE;
+        } else {
+            memory_region_unref(block->mr);
+            QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+            g_free(entry);
+        }
+    }
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+
+    return block;
+}
+
+/*
+ * Unqueue a page from the queue fed by postcopy page requests; skips pages
+ * that are already sent (!dirty)
+ *
+ *      ms:      MigrationState in
+ *     pss:      PageSearchStatus structure updated with found block/offset
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      true if a queued page is found
+ */
+static bool get_queued_page(MigrationState *ms, PageSearchStatus *pss,
+                            ram_addr_t *ram_addr_abs)
+{
+    RAMBlock  *block;
+    ram_addr_t offset;
+    bool dirty;
+
+    do {
+        block = unqueue_page(ms, &offset, ram_addr_abs);
+        /*
+         * We're sending this page, and since it's postcopy nothing else
+         * will dirty it, and we must make sure it doesn't get sent again
+         * even if this queue request was received after the background
+         * search already sent it.
+         */
+        if (block) {
+            unsigned long *bitmap;
+            bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+            dirty = test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, bitmap);
+            if (!dirty) {
+                trace_get_queued_page_not_dirty(
+                    block->idstr, (uint64_t)offset,
+                    (uint64_t)*ram_addr_abs,
+                    test_bit(*ram_addr_abs >> TARGET_PAGE_BITS,
+                         atomic_rcu_read(&migration_bitmap_rcu)->unsentmap));
+            } else {
+                trace_get_queued_page(block->idstr,
+                                      (uint64_t)offset,
+                                      (uint64_t)*ram_addr_abs);
+            }
+        }
+
+    } while (block && !dirty);
+
+    if (block) {
+        /*
+         * As soon as we start servicing pages out of order, then we have
+         * to kill the bulk stage, since the bulk stage assumes
+         * in (migration_bitmap_find_and_reset_dirty) that every page is
+         * dirty, that's no longer true.
+         */
+        ram_bulk_stage = false;
+
+        /*
+         * We want the background search to continue from the queued page
+         * since the guest is likely to want other pages near to the page
+         * it just requested.
+         */
+        pss->block = block;
+        pss->offset = offset;
+    }
+
+    return !!block;
+}
+
+/**
+ * flush_page_queue: Flush any remaining pages in the ram request queue
+ *    it should be empty at the end anyway, but in error cases there may be
+ *    some left.
+ *
+ * ms: MigrationState
+ */
+void flush_page_queue(MigrationState *ms)
+{
+    struct MigrationSrcPageRequest *mspr, *next_mspr;
+    /* This queue generally should be empty - but in the case of a failed
+     * migration might have some droppings in.
+     */
+    rcu_read_lock();
+    QSIMPLEQ_FOREACH_SAFE(mspr, &ms->src_page_requests, next_req, next_mspr) {
+        memory_region_unref(mspr->rb->mr);
+        QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+        g_free(mspr);
+    }
+    rcu_read_unlock();
+}
+
+/**
+ * Queue the pages for transmission, e.g. a request from postcopy destination
+ *   ms: MigrationStatus in which the queue is held
+ *   rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
+ *   start: Offset from the start of the RAMBlock
+ *   len: Length (in bytes) to send
+ *   Return: 0 on success
+ */
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len)
+{
+    RAMBlock *ramblock;
+
+    rcu_read_lock();
+    if (!rbname) {
+        /* Reuse last RAMBlock */
+        ramblock = ms->last_req_rb;
+
+        if (!ramblock) {
+            /*
+             * Shouldn't happen, we can't reuse the last RAMBlock if
+             * it's the 1st request.
+             */
+            error_report("ram_save_queue_pages no previous block");
+            goto err;
+        }
+    } else {
+        ramblock = qemu_ram_block_by_name(rbname);
+
+        if (!ramblock) {
+            /* We shouldn't be asked for a non-existent RAMBlock */
+            error_report("ram_save_queue_pages no block '%s'", rbname);
+            goto err;
+        }
+        ms->last_req_rb = ramblock;
+    }
+    trace_ram_save_queue_pages(ramblock->idstr, start, len);
+    if (start+len > ramblock->used_length) {
+        error_report("%s request overrun start=" RAM_ADDR_FMT " len="
+                     RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
+                     __func__, start, len, ramblock->used_length);
+        goto err;
+    }
+
+    struct MigrationSrcPageRequest *new_entry =
+        g_malloc0(sizeof(struct MigrationSrcPageRequest));
+    new_entry->rb = ramblock;
+    new_entry->offset = start;
+    new_entry->len = len;
+
+    memory_region_ref(ramblock->mr);
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    QSIMPLEQ_INSERT_TAIL(&ms->src_page_requests, new_entry, next_req);
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+    rcu_read_unlock();
+
+    return 0;
+
+err:
+    rcu_read_unlock();
+    return -1;
+}
+
+/**
+ * ram_save_target_page: Save one target page
+ *
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page;
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ *
+ * Returns: Number of pages written.
+ */
+static int ram_save_target_page(MigrationState *ms, QEMUFile *f,
+                                PageSearchStatus *pss,
+                                bool last_stage,
+                                uint64_t *bytes_transferred,
+                                ram_addr_t dirty_ram_abs)
+{
+    int res = 0;
+
+    /* Check the pages is dirty and if it is send it */
+    if (migration_bitmap_clear_dirty(dirty_ram_abs)) {
+        unsigned long *unsentmap;
+        if (compression_switch && migrate_use_compression()) {
+            res = ram_save_compressed_page(f, pss,
+                                           last_stage,
+                                           bytes_transferred);
+        } else {
+            res = ram_save_page(f, pss, last_stage,
+                                bytes_transferred);
+        }
+
+        if (res < 0) {
+            return res;
+        }
+        unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+        if (unsentmap) {
+            clear_bit(dirty_ram_abs >> TARGET_PAGE_BITS, unsentmap);
+        }
+        /* Only update last_sent_block if a block was actually sent; xbzrle
+         * might have decided the page was identical so didn't bother writing
+         * to the stream.
+         */
+        if (res > 0) {
+            last_sent_block = pss->block;
+        }
+    }
+
+    return res;
+}
+
+/**
+ * ram_save_host_page: Starting at *offset send pages upto the end
+ *                     of the current host page.  It's valid for the initial
+ *                     offset to point into the middle of a host page
+ *                     in which case the remainder of the hostpage is sent.
+ *                     Only dirty target pages are sent.
+ *
+ * Returns: Number of pages written.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page; updated to last target page
+ *          sent
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ */
+static int ram_save_host_page(MigrationState *ms, QEMUFile *f,
+                              PageSearchStatus *pss,
+                              bool last_stage,
+                              uint64_t *bytes_transferred,
+                              ram_addr_t dirty_ram_abs)
+{
+    int tmppages, pages = 0;
+    do {
+        tmppages = ram_save_target_page(ms, f, pss, last_stage,
+                                        bytes_transferred, dirty_ram_abs);
+        if (tmppages < 0) {
+            return tmppages;
+        }
+
+        pages += tmppages;
+        pss->offset += TARGET_PAGE_SIZE;
+        dirty_ram_abs += TARGET_PAGE_SIZE;
+    } while (pss->offset & (qemu_host_page_size - 1));
+
+    /* The offset we leave with is the last one we looked at */
+    pss->offset -= TARGET_PAGE_SIZE;
+    return pages;
+}
+
 /**
  * ram_find_and_save_block: Finds a dirty page and sends it to f
  *
@@ -920,61 +1323,47 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block,
  * @f: QEMUFile where to send the data
  * @last_stage: if we are at the completion stage
  * @bytes_transferred: increase it with the number of transferred bytes
+ *
+ * On systems where host-page-size > target-page-size it will send all the
+ * pages in a host page that are dirty.
  */
 
 static int ram_find_and_save_block(QEMUFile *f, bool last_stage,
                                    uint64_t *bytes_transferred)
 {
-    RAMBlock *block = last_seen_block;
-    ram_addr_t offset = last_offset;
-    bool complete_round = false;
+    PageSearchStatus pss;
+    MigrationState *ms = migrate_get_current();
     int pages = 0;
-    MemoryRegion *mr;
+    bool again, found;
+    ram_addr_t dirty_ram_abs; /* Address of the start of the dirty page in
+                                 ram_addr_t space */
 
-    if (!block)
-        block = QLIST_FIRST_RCU(&ram_list.blocks);
+    pss.block = last_seen_block;
+    pss.offset = last_offset;
+    pss.complete_round = false;
 
-    while (true) {
-        mr = block->mr;
-        offset = migration_bitmap_find_and_reset_dirty(mr, offset);
-        if (complete_round && block == last_seen_block &&
-            offset >= last_offset) {
-            break;
+    if (!pss.block) {
+        pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
+    }
+
+    do {
+        again = true;
+        found = get_queued_page(ms, &pss, &dirty_ram_abs);
+
+        if (!found) {
+            /* priority queue empty, so just search for something dirty */
+            found = find_dirty_block(f, &pss, &again, &dirty_ram_abs);
         }
-        if (offset >= block->used_length) {
-            offset = 0;
-            block = QLIST_NEXT_RCU(block, next);
-            if (!block) {
-                block = QLIST_FIRST_RCU(&ram_list.blocks);
-                complete_round = true;
-                ram_bulk_stage = false;
-                if (migrate_use_xbzrle()) {
-                    /* If xbzrle is on, stop using the data compression at this
-                     * point. In theory, xbzrle can do better than compression.
-                     */
-                    flush_compressed_data(f);
-                    compression_switch = false;
-                }
-            }
-        } else {
-            if (compression_switch && migrate_use_compression()) {
-                pages = ram_save_compressed_page(f, block, offset, last_stage,
-                                                 bytes_transferred);
-            } else {
-                pages = ram_save_page(f, block, offset, last_stage,
-                                      bytes_transferred);
-            }
 
-            /* if page is unmodified, continue to the next */
-            if (pages > 0) {
-                last_sent_block = block;
-                break;
-            }
+        if (found) {
+            pages = ram_save_host_page(ms, f, &pss,
+                                       last_stage, bytes_transferred,
+                                       dirty_ram_abs);
         }
-    }
+    } while (!pages && again);
 
-    last_seen_block = block;
-    last_offset = offset;
+    last_seen_block = pss.block;
+    last_offset = pss.offset;
 
     return pages;
 }
@@ -1024,17 +1413,23 @@ void free_xbzrle_decoded_buf(void)
     xbzrle_decoded_buf = NULL;
 }
 
-static void migration_end(void)
+static void migration_bitmap_free(struct BitmapRcu *bmap)
+{
+    g_free(bmap->bmap);
+    g_free(bmap->unsentmap);
+    g_free(bmap);
+}
+
+static void ram_migration_cleanup(void *opaque)
 {
     /* caller have hold iothread lock or is in a bh, so there is
      * no writing race against this migration_bitmap
      */
-    unsigned long *bitmap = migration_bitmap;
-    atomic_rcu_set(&migration_bitmap, NULL);
+    struct BitmapRcu *bitmap = migration_bitmap_rcu;
+    atomic_rcu_set(&migration_bitmap_rcu, NULL);
     if (bitmap) {
         memory_global_dirty_log_stop();
-        synchronize_rcu();
-        g_free(bitmap);
+        call_rcu(bitmap, migration_bitmap_free, rcu);
     }
 
     XBZRLE_cache_lock();
@@ -1049,11 +1444,6 @@ static void migration_end(void)
     XBZRLE_cache_unlock();
 }
 
-static void ram_migration_cancel(void *opaque)
-{
-    migration_end();
-}
-
 static void reset_ram_globals(void)
 {
     last_seen_block = NULL;
@@ -1070,9 +1460,10 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
     /* called in qemu main thread, so there is
      * no writing race against this migration_bitmap
      */
-    if (migration_bitmap) {
-        unsigned long *old_bitmap = migration_bitmap, *bitmap;
-        bitmap = bitmap_new(new);
+    if (migration_bitmap_rcu) {
+        struct BitmapRcu *old_bitmap = migration_bitmap_rcu, *bitmap;
+        bitmap = g_new(struct BitmapRcu, 1);
+        bitmap->bmap = bitmap_new(new);
 
         /* prevent migration_bitmap content from being set bit
          * by migration_bitmap_sync_range() at the same time.
@@ -1080,16 +1471,410 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
          * at the same time.
          */
         qemu_mutex_lock(&migration_bitmap_mutex);
-        bitmap_copy(bitmap, old_bitmap, old);
-        bitmap_set(bitmap, old, new - old);
-        atomic_rcu_set(&migration_bitmap, bitmap);
+        bitmap_copy(bitmap->bmap, old_bitmap->bmap, old);
+        bitmap_set(bitmap->bmap, old, new - old);
+
+        /* We don't have a way to safely extend the sentmap
+         * with RCU; so mark it as missing, entry to postcopy
+         * will fail.
+         */
+        bitmap->unsentmap = NULL;
+
+        atomic_rcu_set(&migration_bitmap_rcu, bitmap);
         qemu_mutex_unlock(&migration_bitmap_mutex);
         migration_dirty_pages += new - old;
-        synchronize_rcu();
-        g_free(old_bitmap);
+        call_rcu(old_bitmap, migration_bitmap_free, rcu);
+    }
+}
+
+/*
+ * 'expected' is the value you expect the bitmap mostly to be full
+ * of; it won't bother printing lines that are all this value.
+ * If 'todump' is null the migration bitmap is dumped.
+ */
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected)
+{
+    int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    int64_t cur;
+    int64_t linelen = 128;
+    char linebuf[129];
+
+    if (!todump) {
+        todump = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    }
+
+    for (cur = 0; cur < ram_pages; cur += linelen) {
+        int64_t curb;
+        bool found = false;
+        /*
+         * Last line; catch the case where the line length
+         * is longer than remaining ram
+         */
+        if (cur + linelen > ram_pages) {
+            linelen = ram_pages - cur;
+        }
+        for (curb = 0; curb < linelen; curb++) {
+            bool thisbit = test_bit(cur + curb, todump);
+            linebuf[curb] = thisbit ? '1' : '.';
+            found = found || (thisbit != expected);
+        }
+        if (found) {
+            linebuf[curb] = '\0';
+            fprintf(stderr,  "0x%08" PRIx64 " : %s\n", cur, linebuf);
+        }
+    }
+}
+
+/* **** functions for postcopy ***** */
+
+/*
+ * Callback from postcopy_each_ram_send_discard for each RAMBlock
+ * Note: At this point the 'unsentmap' is the processed bitmap combined
+ *       with the dirtymap; so a '1' means it's either dirty or unsent.
+ * start,length: Indexes into the bitmap for the first bit
+ *            representing the named block and length in target-pages
+ */
+static int postcopy_send_discard_bm_ram(MigrationState *ms,
+                                        PostcopyDiscardState *pds,
+                                        unsigned long start,
+                                        unsigned long length)
+{
+    unsigned long end = start + length; /* one after the end */
+    unsigned long current;
+    unsigned long *unsentmap;
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    for (current = start; current < end; ) {
+        unsigned long one = find_next_bit(unsentmap, end, current);
+
+        if (one <= end) {
+            unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
+            unsigned long discard_length;
+
+            if (zero >= end) {
+                discard_length = end - one;
+            } else {
+                discard_length = zero - one;
+            }
+            postcopy_discard_send_range(ms, pds, one, discard_length);
+            current = one + discard_length;
+        } else {
+            current = one;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *   Calls postcopy_send_discard_bm_ram for each RAMBlock
+ *   passing it bitmap indexes and name.
+ * Returns: 0 on success
+ * (qemu_ram_foreach_block ends up passing unscaled lengths
+ *  which would mean postcopy code would have to deal with target page)
+ */
+static int postcopy_each_ram_send_discard(MigrationState *ms)
+{
+    struct RAMBlock *block;
+    int ret;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+        PostcopyDiscardState *pds = postcopy_discard_send_init(ms,
+                                                               first,
+                                                               block->idstr);
+
+        /*
+         * Postcopy sends chunks of bitmap over the wire, but it
+         * just needs indexes at this point, avoids it having
+         * target page specific code.
+         */
+        ret = postcopy_send_discard_bm_ram(ms, pds, first,
+                                    block->used_length >> TARGET_PAGE_BITS);
+        postcopy_discard_send_finish(ms, pds);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
+ *   the two bitmaps, that are similar, but one is inverted.
+ *
+ * We search for runs of target-pages that don't start or end on a
+ * host page boundary;
+ * unsent_pass=true: Cleans up partially unsent host pages by searching
+ *                 the unsentmap
+ * unsent_pass=false: Cleans up partially dirty host pages by searching
+ *                 the main migration bitmap
+ *
+ */
+static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
+                                          RAMBlock *block,
+                                          PostcopyDiscardState *pds)
+{
+    unsigned long *bitmap;
+    unsigned long *unsentmap;
+    unsigned int host_ratio = qemu_host_page_size / TARGET_PAGE_SIZE;
+    unsigned long first = block->offset >> TARGET_PAGE_BITS;
+    unsigned long len = block->used_length >> TARGET_PAGE_BITS;
+    unsigned long last = first + (len - 1);
+    unsigned long run_start;
+
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+
+    if (unsent_pass) {
+        /* Find a sent page */
+        run_start = find_next_zero_bit(unsentmap, last + 1, first);
+    } else {
+        /* Find a dirty page */
+        run_start = find_next_bit(bitmap, last + 1, first);
+    }
+
+    while (run_start <= last) {
+        bool do_fixup = false;
+        unsigned long fixup_start_addr;
+        unsigned long host_offset;
+
+        /*
+         * If the start of this run of pages is in the middle of a host
+         * page, then we need to fixup this host page.
+         */
+        host_offset = run_start % host_ratio;
+        if (host_offset) {
+            do_fixup = true;
+            run_start -= host_offset;
+            fixup_start_addr = run_start;
+            /* For the next pass */
+            run_start = run_start + host_ratio;
+        } else {
+            /* Find the end of this run */
+            unsigned long run_end;
+            if (unsent_pass) {
+                run_end = find_next_bit(unsentmap, last + 1, run_start + 1);
+            } else {
+                run_end = find_next_zero_bit(bitmap, last + 1, run_start + 1);
+            }
+            /*
+             * If the end isn't at the start of a host page, then the
+             * run doesn't finish at the end of a host page
+             * and we need to discard.
+             */
+            host_offset = run_end % host_ratio;
+            if (host_offset) {
+                do_fixup = true;
+                fixup_start_addr = run_end - host_offset;
+                /*
+                 * This host page has gone, the next loop iteration starts
+                 * from after the fixup
+                 */
+                run_start = fixup_start_addr + host_ratio;
+            } else {
+                /*
+                 * No discards on this iteration, next loop starts from
+                 * next sent/dirty page
+                 */
+                run_start = run_end + 1;
+            }
+        }
+
+        if (do_fixup) {
+            unsigned long page;
+
+            /* Tell the destination to discard this page */
+            if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
+                /* For the unsent_pass we:
+                 *     discard partially sent pages
+                 * For the !unsent_pass (dirty) we:
+                 *     discard partially dirty pages that were sent
+                 *     (any partially sent pages were already discarded
+                 *     by the previous unsent_pass)
+                 */
+                postcopy_discard_send_range(ms, pds, fixup_start_addr,
+                                            host_ratio);
+            }
+
+            /* Clean up the bitmap */
+            for (page = fixup_start_addr;
+                 page < fixup_start_addr + host_ratio; page++) {
+                /* All pages in this host page are now not sent */
+                set_bit(page, unsentmap);
+
+                /*
+                 * Remark them as dirty, updating the count for any pages
+                 * that weren't previously dirty.
+                 */
+                migration_dirty_pages += !test_and_set_bit(page, bitmap);
+            }
+        }
+
+        if (unsent_pass) {
+            /* Find the next sent page for the next iteration */
+            run_start = find_next_zero_bit(unsentmap, last + 1,
+                                           run_start);
+        } else {
+            /* Find the next dirty page for the next iteration */
+            run_start = find_next_bit(bitmap, last + 1, run_start);
+        }
+    }
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *
+ * Discard any partially sent host-page size chunks, mark any partially
+ * dirty host-page size chunks as all dirty.
+ *
+ * Returns: 0 on success
+ */
+static int postcopy_chunk_hostpages(MigrationState *ms)
+{
+    struct RAMBlock *block;
+
+    if (qemu_host_page_size == TARGET_PAGE_SIZE) {
+        /* Easy case - TPS==HPS - nothing to be done */
+        return 0;
+    }
+
+    /* Easiest way to make sure we don't resume in the middle of a host-page */
+    last_seen_block = NULL;
+    last_sent_block = NULL;
+    last_offset     = 0;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+
+        PostcopyDiscardState *pds =
+                         postcopy_discard_send_init(ms, first, block->idstr);
+
+        /* First pass: Discard all partially sent host pages */
+        postcopy_chunk_hostpages_pass(ms, true, block, pds);
+        /*
+         * Second pass: Ensure that all partially dirty host pages are made
+         * fully dirty.
+         */
+        postcopy_chunk_hostpages_pass(ms, false, block, pds);
+
+        postcopy_discard_send_finish(ms, pds);
+    } /* ram_list loop */
+
+    return 0;
+}
+
+/*
+ * Transmit the set of pages to be discarded after precopy to the target
+ * these are pages that:
+ *     a) Have been previously transmitted but are now dirty again
+ *     b) Pages that have never been transmitted, this ensures that
+ *        any pages on the destination that have been mapped by background
+ *        tasks get discarded (transparent huge pages is the specific concern)
+ * Hopefully this is pretty sparse
+ */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms)
+{
+    int ret;
+    unsigned long *bitmap, *unsentmap;
+
+    rcu_read_lock();
+
+    /* This should be our last sync, the src is now paused */
+    migration_bitmap_sync();
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    if (!unsentmap) {
+        /* We don't have a safe way to resize the sentmap, so
+         * if the bitmap was resized it will be NULL at this
+         * point.
+         */
+        error_report("migration ram resized during precopy phase");
+        rcu_read_unlock();
+        return -EINVAL;
+    }
+
+    /* Deal with TPS != HPS */
+    ret = postcopy_chunk_hostpages(ms);
+    if (ret) {
+        rcu_read_unlock();
+        return ret;
+    }
+
+    /*
+     * Update the unsentmap to be unsentmap = unsentmap | dirty
+     */
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    bitmap_or(unsentmap, unsentmap, bitmap,
+               last_ram_offset() >> TARGET_PAGE_BITS);
+
+
+    trace_ram_postcopy_send_discard_bitmap();
+#ifdef DEBUG_POSTCOPY
+    ram_debug_dump_bitmap(unsentmap, true);
+#endif
+
+    ret = postcopy_each_ram_send_discard(ms);
+    rcu_read_unlock();
+
+    return ret;
+}
+
+/*
+ * At the start of the postcopy phase of migration, any now-dirty
+ * precopied pages are discarded.
+ *
+ * start, length describe a byte address range within the RAMBlock
+ *
+ * Returns 0 on success.
+ */
+int ram_discard_range(MigrationIncomingState *mis,
+                      const char *block_name,
+                      uint64_t start, size_t length)
+{
+    int ret = -1;
+
+    rcu_read_lock();
+    RAMBlock *rb = qemu_ram_block_by_name(block_name);
+
+    if (!rb) {
+        error_report("ram_discard_range: Failed to find block '%s'",
+                     block_name);
+        goto err;
     }
+
+    uint8_t *host_startaddr = rb->host + start;
+
+    if ((uintptr_t)host_startaddr & (qemu_host_page_size - 1)) {
+        error_report("ram_discard_range: Unaligned start address: %p",
+                     host_startaddr);
+        goto err;
+    }
+
+    if ((start + length) <= rb->used_length) {
+        uint8_t *host_endaddr = host_startaddr + length;
+        if ((uintptr_t)host_endaddr & (qemu_host_page_size - 1)) {
+            error_report("ram_discard_range: Unaligned end address: %p",
+                         host_endaddr);
+            goto err;
+        }
+        ret = postcopy_ram_discard_range(mis, host_startaddr, length);
+    } else {
+        error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
+                     "/%zx/" RAM_ADDR_FMT")",
+                     block_name, start, length, rb->used_length);
+    }
+
+err:
+    rcu_read_unlock();
+
+    return ret;
 }
 
+
 /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
  * long-running RCU critical section.  When rcu-reclaims in the code
  * start to become numerous it will be necessary to reduce the
@@ -1101,7 +1886,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     RAMBlock *block;
     int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
 
-    mig_throttle_on = false;
     dirty_rate_high_cnt = 0;
     bitmap_sync_count = 0;
     migration_bitmap_sync_init();
@@ -1137,16 +1921,23 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
         acct_clear();
     }
 
-    /* iothread lock needed for ram_list.dirty_memory[] */
+    /* For memory_global_dirty_log_start below.  */
     qemu_mutex_lock_iothread();
+
     qemu_mutex_lock_ramlist();
     rcu_read_lock();
     bytes_transferred = 0;
     reset_ram_globals();
 
     ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
-    migration_bitmap = bitmap_new(ram_bitmap_pages);
-    bitmap_set(migration_bitmap, 0, ram_bitmap_pages);
+    migration_bitmap_rcu = g_new0(struct BitmapRcu, 1);
+    migration_bitmap_rcu->bmap = bitmap_new(ram_bitmap_pages);
+    bitmap_set(migration_bitmap_rcu->bmap, 0, ram_bitmap_pages);
+
+    if (migrate_postcopy_ram()) {
+        migration_bitmap_rcu->unsentmap = bitmap_new(ram_bitmap_pages);
+        bitmap_set(migration_bitmap_rcu->unsentmap, 0, ram_bitmap_pages);
+    }
 
     /*
      * Count the total number of pages used by ram blocks not including any
@@ -1206,7 +1997,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
         }
         pages_sent += pages;
         acct_info.iterations++;
-        check_guest_throttling();
+
         /* we want to check in the 1st loop, just in case it was the 1st time
            and we had to sync the dirty bitmap.
            qemu_get_clock_ns() is a bit expensive, so we only check each some
@@ -1247,7 +2038,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 {
     rcu_read_lock();
 
-    migration_bitmap_sync();
+    if (!migration_in_postcopy(migrate_get_current())) {
+        migration_bitmap_sync();
+    }
 
     ram_control_before_iterate(f, RAM_CONTROL_FINISH);
 
@@ -1274,13 +2067,16 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
     return 0;
 }
 
-static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
+static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                             uint64_t *non_postcopiable_pending,
+                             uint64_t *postcopiable_pending)
 {
     uint64_t remaining_size;
 
     remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
 
-    if (remaining_size < max_size) {
+    if (!migration_in_postcopy(migrate_get_current()) &&
+        remaining_size < max_size) {
         qemu_mutex_lock_iothread();
         rcu_read_lock();
         migration_bitmap_sync();
@@ -1288,17 +2084,21 @@ static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
         qemu_mutex_unlock_iothread();
         remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
     }
-    return remaining_size;
+
+    /* We can do postcopy, and all the data is postcopiable */
+    *postcopiable_pending += remaining_size;
 }
 
 static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
 {
     unsigned int xh_len;
     int xh_flags;
+    uint8_t *loaded_data;
 
     if (!xbzrle_decoded_buf) {
         xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE);
     }
+    loaded_data = xbzrle_decoded_buf;
 
     /* extract RLE header */
     xh_flags = qemu_get_byte(f);
@@ -1314,10 +2114,10 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
         return -1;
     }
     /* load data and decode */
-    qemu_get_buffer(f, xbzrle_decoded_buf, xh_len);
+    qemu_get_buffer_in_place(f, &loaded_data, xh_len);
 
     /* decode RLE */
-    if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host,
+    if (xbzrle_decode_buffer(loaded_data, xh_len, host,
                              TARGET_PAGE_SIZE) == -1) {
         error_report("Failed to load XBZRLE page - decode error!");
         return -1;
@@ -1329,36 +2129,48 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
 /* Must be called from within a rcu critical section.
  * Returns a pointer from within the RCU-protected ram_list.
  */
-static inline void *host_from_stream_offset(QEMUFile *f,
-                                            ram_addr_t offset,
-                                            int flags)
+/*
+ * Read a RAMBlock ID from the stream f.
+ *
+ * f: Stream to read from
+ * flags: Page flags (mostly to see if it's a continuation of previous block)
+ */
+static inline RAMBlock *ram_block_from_stream(QEMUFile *f,
+                                              int flags)
 {
     static RAMBlock *block = NULL;
     char id[256];
     uint8_t len;
 
     if (flags & RAM_SAVE_FLAG_CONTINUE) {
-        if (!block || block->max_length <= offset) {
+        if (!block) {
             error_report("Ack, bad migration stream!");
             return NULL;
         }
-
-        return memory_region_get_ram_ptr(block->mr) + offset;
+        return block;
     }
 
     len = qemu_get_byte(f);
     qemu_get_buffer(f, (uint8_t *)id, len);
     id[len] = 0;
 
-    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-        if (!strncmp(id, block->idstr, sizeof(id)) &&
-            block->max_length > offset) {
-            return memory_region_get_ram_ptr(block->mr) + offset;
-        }
+    block = qemu_ram_block_by_name(id);
+    if (!block) {
+        error_report("Can't find block %s", id);
+        return NULL;
     }
 
-    error_report("Can't find block %s!", id);
-    return NULL;
+    return block;
+}
+
+static inline void *host_from_ram_block_offset(RAMBlock *block,
+                                               ram_addr_t offset)
+{
+    if (!offset_in_ramblock(block, offset)) {
+        return NULL;
+    }
+
+    return block->host + offset;
 }
 
 /*
@@ -1406,7 +2218,6 @@ void migrate_decompress_threads_create(void)
     thread_count = migrate_decompress_threads();
     decompress_threads = g_new0(QemuThread, thread_count);
     decomp_param = g_new0(DecompressParam, thread_count);
-    compressed_data_buf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
     quit_decomp_thread = false;
     for (i = 0; i < thread_count; i++) {
         qemu_mutex_init(&decomp_param[i].mutex);
@@ -1437,13 +2248,11 @@ void migrate_decompress_threads_join(void)
     }
     g_free(decompress_threads);
     g_free(decomp_param);
-    g_free(compressed_data_buf);
     decompress_threads = NULL;
     decomp_param = NULL;
-    compressed_data_buf = NULL;
 }
 
-static void decompress_data_with_multi_threads(uint8_t *compbuf,
+static void decompress_data_with_multi_threads(QEMUFile *f,
                                                void *host, int len)
 {
     int idx, thread_count;
@@ -1452,7 +2261,7 @@ static void decompress_data_with_multi_threads(uint8_t *compbuf,
     while (true) {
         for (idx = 0; idx < thread_count; idx++) {
             if (!decomp_param[idx].start) {
-                memcpy(decomp_param[idx].compbuf, compbuf, len);
+                qemu_get_buffer(f, decomp_param[idx].compbuf, len);
                 decomp_param[idx].des = host;
                 decomp_param[idx].len = len;
                 start_decompression(&decomp_param[idx]);
@@ -1465,11 +2274,150 @@ static void decompress_data_with_multi_threads(uint8_t *compbuf,
     }
 }
 
+/*
+ * Allocate data structures etc needed by incoming migration with postcopy-ram
+ * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
+ */
+int ram_postcopy_incoming_init(MigrationIncomingState *mis)
+{
+    size_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    return postcopy_ram_incoming_init(mis, ram_pages);
+}
+
+/*
+ * Called in postcopy mode by ram_load().
+ * rcu_read_lock is taken prior to this being called.
+ */
+static int ram_load_postcopy(QEMUFile *f)
+{
+    int flags = 0, ret = 0;
+    bool place_needed = false;
+    bool matching_page_sizes = qemu_host_page_size == TARGET_PAGE_SIZE;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    /* Temporary page that is later 'placed' */
+    void *postcopy_host_page = postcopy_get_tmp_page(mis);
+    void *last_host = NULL;
+    bool all_zero = false;
+
+    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+        ram_addr_t addr;
+        void *host = NULL;
+        void *page_buffer = NULL;
+        void *place_source = NULL;
+        uint8_t ch;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        trace_ram_load_postcopy_loop((uint64_t)addr, flags);
+        place_needed = false;
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE)) {
+            RAMBlock *block = ram_block_from_stream(f, flags);
+
+            host = host_from_ram_block_offset(block, addr);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+            page_buffer = host;
+            /*
+             * Postcopy requires that we place whole host pages atomically.
+             * To make it atomic, the data is read into a temporary page
+             * that's moved into place later.
+             * The migration protocol uses,  possibly smaller, target-pages
+             * however the source ensures it always sends all the components
+             * of a host page in order.
+             */
+            page_buffer = postcopy_host_page +
+                          ((uintptr_t)host & ~qemu_host_page_mask);
+            /* If all TP are zero then we can optimise the place */
+            if (!((uintptr_t)host & ~qemu_host_page_mask)) {
+                all_zero = true;
+            } else {
+                /* not the 1st TP within the HP */
+                if (host != (last_host + TARGET_PAGE_SIZE)) {
+                    error_report("Non-sequential target page %p/%p",
+                                  host, last_host);
+                    ret = -EINVAL;
+                    break;
+                }
+            }
+
+
+            /*
+             * If it's the last part of a host page then we place the host
+             * page
+             */
+            place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
+                                     ~qemu_host_page_mask) == 0;
+            place_source = postcopy_host_page;
+        }
+        last_host = host;
+
+        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
+        case RAM_SAVE_FLAG_COMPRESS:
+            ch = qemu_get_byte(f);
+            memset(page_buffer, ch, TARGET_PAGE_SIZE);
+            if (ch) {
+                all_zero = false;
+            }
+            break;
+
+        case RAM_SAVE_FLAG_PAGE:
+            all_zero = false;
+            if (!place_needed || !matching_page_sizes) {
+                qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
+            } else {
+                /* Avoids the qemu_file copy during postcopy, which is
+                 * going to do a copy later; can only do it when we
+                 * do this read in one go (matching page sizes)
+                 */
+                qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
+                                         TARGET_PAGE_SIZE);
+            }
+            break;
+        case RAM_SAVE_FLAG_EOS:
+            /* normal exit */
+            break;
+        default:
+            error_report("Unknown combination of migration flags: %#x"
+                         " (postcopy mode)", flags);
+            ret = -EINVAL;
+        }
+
+        if (place_needed) {
+            /* This gets called at the last target page in the host page */
+            if (all_zero) {
+                ret = postcopy_place_page_zero(mis,
+                                               host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size);
+            } else {
+                ret = postcopy_place_page(mis, host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size,
+                                               place_source);
+            }
+        }
+        if (!ret) {
+            ret = qemu_file_get_error(f);
+        }
+    }
+
+    return ret;
+}
+
 static int ram_load(QEMUFile *f, void *opaque, int version_id)
 {
     int flags = 0, ret = 0;
     static uint64_t seq_iter;
     int len = 0;
+    /*
+     * If system is running in postcopy mode, page inserts to host memory must
+     * be atomic
+     */
+    bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING;
 
     seq_iter++;
 
@@ -1483,15 +2431,32 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
      * critical section.
      */
     rcu_read_lock();
-    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+
+    if (postcopy_running) {
+        ret = ram_load_postcopy(f);
+    }
+
+    while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
         ram_addr_t addr, total_ram_bytes;
-        void *host;
+        void *host = NULL;
         uint8_t ch;
 
         addr = qemu_get_be64(f);
         flags = addr & ~TARGET_PAGE_MASK;
         addr &= TARGET_PAGE_MASK;
 
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE |
+                     RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
+            RAMBlock *block = ram_block_from_stream(f, flags);
+
+            host = host_from_ram_block_offset(block, addr);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+        }
+
         switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
         case RAM_SAVE_FLAG_MEM_SIZE:
             /* Synchronize RAM block list */
@@ -1506,23 +2471,20 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 id[len] = 0;
                 length = qemu_get_be64(f);
 
-                QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
-                    if (!strncmp(id, block->idstr, sizeof(id))) {
-                        if (length != block->used_length) {
-                            Error *local_err = NULL;
+                block = qemu_ram_block_by_name(id);
+                if (block) {
+                    if (length != block->used_length) {
+                        Error *local_err = NULL;
 
-                            ret = qemu_ram_resize(block->offset, length, &local_err);
-                            if (local_err) {
-                                error_report_err(local_err);
-                            }
+                        ret = qemu_ram_resize(block->offset, length,
+                                              &local_err);
+                        if (local_err) {
+                            error_report_err(local_err);
                         }
-                        ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
-                                              block->idstr);
-                        break;
                     }
-                }
-
-                if (!block) {
+                    ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
+                                          block->idstr);
+                } else {
                     error_report("Unknown ramblock \"%s\", cannot "
                                  "accept migration", id);
                     ret = -EINVAL;
@@ -1531,49 +2493,27 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
                 total_ram_bytes -= length;
             }
             break;
+
         case RAM_SAVE_FLAG_COMPRESS:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             ch = qemu_get_byte(f);
             ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
             break;
+
         case RAM_SAVE_FLAG_PAGE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
             break;
-        case RAM_SAVE_FLAG_COMPRESS_PAGE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Invalid RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
 
+        case RAM_SAVE_FLAG_COMPRESS_PAGE:
             len = qemu_get_be32(f);
             if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
                 error_report("Invalid compressed data length: %d", len);
                 ret = -EINVAL;
                 break;
             }
-            qemu_get_buffer(f, compressed_data_buf, len);
-            decompress_data_with_multi_threads(compressed_data_buf, host, len);
+            decompress_data_with_multi_threads(f, host, len);
             break;
+
         case RAM_SAVE_FLAG_XBZRLE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
             if (load_xbzrle(f, addr, host) < 0) {
                 error_report("Failed to decompress XBZRLE page at "
                              RAM_ADDR_FMT, addr);
@@ -1607,10 +2547,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
 static SaveVMHandlers savevm_ram_handlers = {
     .save_live_setup = ram_save_setup,
     .save_live_iterate = ram_save_iterate,
-    .save_live_complete = ram_save_complete,
+    .save_live_complete_postcopy = ram_save_complete,
+    .save_live_complete_precopy = ram_save_complete,
     .save_live_pending = ram_save_pending,
     .load_state = ram_load,
-    .cancel = ram_migration_cancel,
+    .cleanup = ram_migration_cleanup,
 };
 
 void ram_mig_init(void)
@@ -1618,52 +2559,3 @@ void ram_mig_init(void)
     qemu_mutex_init(&XBZRLE.lock);
     register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
 }
-/* Stub function that's gets run on the vcpu when its brought out of the
-   VM to run inside qemu via async_run_on_cpu()*/
-
-static void mig_sleep_cpu(void *opq)
-{
-    qemu_mutex_unlock_iothread();
-    g_usleep(30*1000);
-    qemu_mutex_lock_iothread();
-}
-
-/* To reduce the dirty rate explicitly disallow the VCPUs from spending
-   much time in the VM. The migration thread will try to catchup.
-   Workload will experience a performance drop.
-*/
-static void mig_throttle_guest_down(void)
-{
-    CPUState *cpu;
-
-    qemu_mutex_lock_iothread();
-    CPU_FOREACH(cpu) {
-        async_run_on_cpu(cpu, mig_sleep_cpu, NULL);
-    }
-    qemu_mutex_unlock_iothread();
-}
-
-static void check_guest_throttling(void)
-{
-    static int64_t t0;
-    int64_t        t1;
-
-    if (!mig_throttle_on) {
-        return;
-    }
-
-    if (!t0)  {
-        t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        return;
-    }
-
-    t1 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-
-    /* If it has been more than 40 ms since the last time the guest
-     * was throttled then do it again.
-     */
-    if (40 < (t1-t0)/1000000) {
-        mig_throttle_guest_down();
-        t0 = t1;
-    }
-}
diff --git a/qemu/migration/rdma.c b/qemu/migration/rdma.c
index 74876fd7a..f6a9992b3 100644
--- a/qemu/migration/rdma.c
+++ b/qemu/migration/rdma.c
@@ -11,7 +11,10 @@
  * later.  See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
 #include "exec/cpu-common.h"
@@ -19,13 +22,10 @@
 #include "qemu/main-loop.h"
 #include "qemu/sockets.h"
 #include "qemu/bitmap.h"
-#include "block/coroutine.h"
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/coroutine.h"
 #include <sys/socket.h>
 #include <netdb.h>
 #include <arpa/inet.h>
-#include <string.h>
 #include <rdma/rdma_cma.h>
 #include "trace.h"
 
@@ -541,7 +541,7 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name,
     RDMALocalBlock *block;
     RDMALocalBlock *old = local->block;
 
-    local->block = g_malloc0(sizeof(RDMALocalBlock) * (local->nb_blocks + 1));
+    local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1);
 
     if (local->nb_blocks) {
         int x;
@@ -572,12 +572,12 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name,
     bitmap_clear(block->transit_bitmap, 0, block->nb_chunks);
     block->unregister_bitmap = bitmap_new(block->nb_chunks);
     bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks);
-    block->remote_keys = g_malloc0(block->nb_chunks * sizeof(uint32_t));
+    block->remote_keys = g_new0(uint32_t, block->nb_chunks);
 
     block->is_ram_block = local->init ? false : true;
 
     if (rdma->blockmap) {
-        g_hash_table_insert(rdma->blockmap, (void *) block_offset, block);
+        g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block);
     }
 
     trace_rdma_add_block(block_name, local->nb_blocks,
@@ -617,8 +617,8 @@ static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
     memset(local, 0, sizeof *local);
     qemu_ram_foreach_block(qemu_rdma_init_one_block, rdma);
     trace_qemu_rdma_init_ram_blocks(local->nb_blocks);
-    rdma->dest_blocks = (RDMADestBlock *) g_malloc0(sizeof(RDMADestBlock) *
-                        rdma->local_ram_blocks.nb_blocks);
+    rdma->dest_blocks = g_new0(RDMADestBlock,
+                               rdma->local_ram_blocks.nb_blocks);
     local->init = true;
     return 0;
 }
@@ -677,8 +677,7 @@ static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block)
 
     if (local->nb_blocks > 1) {
 
-        local->block = g_malloc0(sizeof(RDMALocalBlock) *
-                                    (local->nb_blocks - 1));
+        local->block = g_new0(RDMALocalBlock, local->nb_blocks - 1);
 
         if (block->index) {
             memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index);
@@ -778,7 +777,7 @@ static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
  *
  *  If the source VM connects with an IPv4 address without knowing that the
  *  destination has bound to '[::]' the migration will unconditionally fail
- *  unless the management software is explicitly listening on the the IPv4
+ *  unless the management software is explicitly listening on the IPv4
  *  address while using a RoCE-based device.
  *
  *  If the source VM connects with an IPv6 address, then we're OK because we can
@@ -1164,7 +1163,7 @@ static int qemu_rdma_register_and_get_keys(RDMAContext *rdma,
 
     /* allocate memory to store chunk MRs */
     if (!block->pmr) {
-        block->pmr = g_malloc0(block->nb_chunks * sizeof(struct ibv_mr *));
+        block->pmr = g_new0(struct ibv_mr *, block->nb_chunks);
     }
 
     /*
@@ -2494,7 +2493,7 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
     InetSocketAddress *addr;
 
     if (host_port) {
-        rdma = g_malloc0(sizeof(RDMAContext));
+        rdma = g_new0(RDMAContext, 1);
         rdma->current_index = -1;
         rdma->current_chunk = -1;
 
@@ -2519,8 +2518,8 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp)
  * SEND messages for control only.
  * VM's ram is handled with regular RDMA messages.
  */
-static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
-                                int64_t pos, int size)
+static ssize_t qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
+                                    int64_t pos, size_t size)
 {
     QEMUFileRDMA *r = opaque;
     QEMUFile *f = r->file;
@@ -2547,7 +2546,8 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
         r->len = MIN(remaining, RDMA_SEND_INCREMENT);
         remaining -= r->len;
 
-        head.len = r->len;
+        /* Guaranteed to fit due to RDMA_SEND_INCREMENT MIN above */
+        head.len = (uint32_t)r->len;
         head.type = RDMA_CONTROL_QEMU_FILE;
 
         ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL);
@@ -2564,7 +2564,7 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
 }
 
 static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
-                             int size, int idx)
+                             size_t size, int idx)
 {
     size_t len = 0;
 
@@ -2585,8 +2585,8 @@ static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
  * RDMA links don't use bytestreams, so we have to
  * return bytes to QEMUFile opportunistically.
  */
-static int qemu_rdma_get_buffer(void *opaque, uint8_t *buf,
-                                int64_t pos, int size)
+static ssize_t qemu_rdma_get_buffer(void *opaque, uint8_t *buf,
+                                    int64_t pos, size_t size)
 {
     QEMUFileRDMA *r = opaque;
     RDMAContext *rdma = r->rdma;
@@ -3399,7 +3399,7 @@ static void *qemu_fopen_rdma(RDMAContext *rdma, const char *mode)
         return NULL;
     }
 
-    r = g_malloc0(sizeof(QEMUFileRDMA));
+    r = g_new0(QEMUFileRDMA, 1);
     r->rdma = rdma;
 
     if (mode[0] == 'w') {
@@ -3506,7 +3506,7 @@ void rdma_start_outgoing_migration(void *opaque,
 
     trace_rdma_start_outgoing_migration_after_rdma_connect();
 
-    s->file = qemu_fopen_rdma(rdma, "wb");
+    s->to_dst_file = qemu_fopen_rdma(rdma, "wb");
     migrate_fd_connect(s);
     return;
 err:
diff --git a/qemu/migration/savevm.c b/qemu/migration/savevm.c
index a42874b10..16ba44379 100644
--- a/qemu/migration/savevm.c
+++ b/qemu/migration/savevm.c
@@ -26,8 +26,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "hw/boards.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
@@ -37,6 +36,7 @@
 #include "qemu/timer.h"
 #include "audio/audio.h"
 #include "migration/migration.h"
+#include "migration/postcopy-ram.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
@@ -45,10 +45,11 @@
 #include "exec/memory.h"
 #include "qmp-commands.h"
 #include "trace.h"
+#include "qemu/bitops.h"
 #include "qemu/iov.h"
 #include "block/snapshot.h"
 #include "block/qapi.h"
-
+#include "qemu/cutils.h"
 
 #ifndef ETH_P_RARP
 #define ETH_P_RARP 0x8035
@@ -57,8 +58,26 @@
 #define ARP_PTYPE_IP 0x0800
 #define ARP_OP_REQUEST_REV 0x3
 
+const unsigned int postcopy_ram_discard_version = 0;
+
 static bool skip_section_footers;
 
+static struct mig_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} mig_cmd_args[] = {
+    [MIG_CMD_INVALID]          = { .len = -1, .name = "INVALID" },
+    [MIG_CMD_OPEN_RETURN_PATH] = { .len =  0, .name = "OPEN_RETURN_PATH" },
+    [MIG_CMD_PING]             = { .len = sizeof(uint32_t), .name = "PING" },
+    [MIG_CMD_POSTCOPY_ADVISE]  = { .len = 16, .name = "POSTCOPY_ADVISE" },
+    [MIG_CMD_POSTCOPY_LISTEN]  = { .len =  0, .name = "POSTCOPY_LISTEN" },
+    [MIG_CMD_POSTCOPY_RUN]     = { .len =  0, .name = "POSTCOPY_RUN" },
+    [MIG_CMD_POSTCOPY_RAM_DISCARD] = {
+                                   .len = -1, .name = "POSTCOPY_RAM_DISCARD" },
+    [MIG_CMD_PACKAGED]         = { .len =  4, .name = "PACKAGED" },
+    [MIG_CMD_MAX]              = { .len = -1, .name = "MAX" },
+};
+
 static int announce_self_create(uint8_t *buf,
                                 uint8_t *mac_addr)
 {
@@ -138,14 +157,15 @@ static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
     return qiov.size;
 }
 
-static int block_put_buffer(void *opaque, const uint8_t *buf,
-                           int64_t pos, int size)
+static ssize_t block_put_buffer(void *opaque, const uint8_t *buf,
+                                int64_t pos, size_t size)
 {
     bdrv_save_vmstate(opaque, buf, pos, size);
     return size;
 }
 
-static int block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+static ssize_t block_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                size_t size)
 {
     return bdrv_load_vmstate(opaque, buf, pos, size);
 }
@@ -278,8 +298,8 @@ static int configuration_post_load(void *opaque, int version_id)
     const char *current_name = MACHINE_GET_CLASS(current_machine)->name;
 
     if (strncmp(state->name, current_name, state->len) != 0) {
-        error_report("Machine type received is '%s' and local is '%s'",
-                     state->name, current_name);
+        error_report("Machine type received is '%.*s' and local is '%s'",
+                     (int) state->len, state->name, current_name);
         return -EINVAL;
     }
     return 0;
@@ -480,7 +500,7 @@ int register_savevm_live(DeviceState *dev,
 {
     SaveStateEntry *se;
 
-    se = g_malloc0(sizeof(SaveStateEntry));
+    se = g_new0(SaveStateEntry, 1);
     se->version_id = version_id;
     se->section_id = savevm_state.global_section_id++;
     se->ops = ops;
@@ -498,7 +518,7 @@ int register_savevm_live(DeviceState *dev,
             pstrcat(se->idstr, sizeof(se->idstr), "/");
             g_free(id);
 
-            se->compat = g_malloc0(sizeof(CompatEntry));
+            se->compat = g_new0(CompatEntry, 1);
             pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), idstr);
             se->compat->instance_id = instance_id == -1 ?
                          calculate_compat_instance_id(idstr) : instance_id;
@@ -526,7 +546,7 @@ int register_savevm(DeviceState *dev,
                     LoadStateHandler *load_state,
                     void *opaque)
 {
-    SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers));
+    SaveVMHandlers *ops = g_new0(SaveVMHandlers, 1);
     ops->save_state = save_state;
     ops->load_state = load_state;
     return register_savevm_live(dev, idstr, instance_id, version_id,
@@ -551,9 +571,7 @@ void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque)
     QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
         if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) {
             QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
+            g_free(se->compat);
             g_free(se->ops);
             g_free(se);
         }
@@ -570,7 +588,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
     /* If this triggers, alias support can be dropped for the vmsd. */
     assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id);
 
-    se = g_malloc0(sizeof(SaveStateEntry));
+    se = g_new0(SaveStateEntry, 1);
     se->version_id = vmsd->version_id;
     se->section_id = savevm_state.global_section_id++;
     se->opaque = opaque;
@@ -584,7 +602,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
             pstrcat(se->idstr, sizeof(se->idstr), "/");
             g_free(id);
 
-            se->compat = g_malloc0(sizeof(CompatEntry));
+            se->compat = g_new0(CompatEntry, 1);
             pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name);
             se->compat->instance_id = instance_id == -1 ?
                          calculate_compat_instance_id(vmsd->name) : instance_id;
@@ -612,9 +630,7 @@ void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd,
     QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
         if (se->vmsd == vmsd && se->opaque == opaque) {
             QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
+            g_free(se->compat);
             g_free(se);
         }
     }
@@ -697,6 +713,156 @@ static void save_section_footer(QEMUFile *f, SaveStateEntry *se)
     }
 }
 
+/**
+ * qemu_savevm_command_send: Send a 'QEMU_VM_COMMAND' type element with the
+ *                           command and associated data.
+ *
+ * @f: File to send command on
+ * @command: Command type to send
+ * @len: Length of associated data
+ * @data: Data associated with command.
+ */
+void qemu_savevm_command_send(QEMUFile *f,
+                              enum qemu_vm_cmd command,
+                              uint16_t len,
+                              uint8_t *data)
+{
+    trace_savevm_command_send(command, len);
+    qemu_put_byte(f, QEMU_VM_COMMAND);
+    qemu_put_be16(f, (uint16_t)command);
+    qemu_put_be16(f, len);
+    qemu_put_buffer(f, data, len);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
+{
+    uint32_t buf;
+
+    trace_savevm_send_ping(value);
+    buf = cpu_to_be32(value);
+    qemu_savevm_command_send(f, MIG_CMD_PING, sizeof(value), (uint8_t *)&buf);
+}
+
+void qemu_savevm_send_open_return_path(QEMUFile *f)
+{
+    trace_savevm_send_open_return_path();
+    qemu_savevm_command_send(f, MIG_CMD_OPEN_RETURN_PATH, 0, NULL);
+}
+
+/* We have a buffer of data to send; we don't want that all to be loaded
+ * by the command itself, so the command contains just the length of the
+ * extra buffer that we then send straight after it.
+ * TODO: Must be a better way to organise that
+ *
+ * Returns:
+ *    0 on success
+ *    -ve on error
+ */
+int qemu_savevm_send_packaged(QEMUFile *f, const QEMUSizedBuffer *qsb)
+{
+    size_t cur_iov;
+    size_t len = qsb_get_length(qsb);
+    uint32_t tmp;
+
+    if (len > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("%s: Unreasonably large packaged state: %zu",
+                     __func__, len);
+        return -1;
+    }
+
+    tmp = cpu_to_be32(len);
+
+    trace_qemu_savevm_send_packaged();
+    qemu_savevm_command_send(f, MIG_CMD_PACKAGED, 4, (uint8_t *)&tmp);
+
+    /* all the data follows (concatinating the iov's) */
+    for (cur_iov = 0; cur_iov < qsb->n_iov; cur_iov++) {
+        /* The iov entries are partially filled */
+        size_t towrite = MIN(qsb->iov[cur_iov].iov_len, len);
+        len -= towrite;
+
+        if (!towrite) {
+            break;
+        }
+
+        qemu_put_buffer(f, qsb->iov[cur_iov].iov_base, towrite);
+    }
+
+    return 0;
+}
+
+/* Send prior to any postcopy transfer */
+void qemu_savevm_send_postcopy_advise(QEMUFile *f)
+{
+    uint64_t tmp[2];
+    tmp[0] = cpu_to_be64(getpagesize());
+    tmp[1] = cpu_to_be64(1ul << qemu_target_page_bits());
+
+    trace_qemu_savevm_send_postcopy_advise();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, 16, (uint8_t *)tmp);
+}
+
+/* Sent prior to starting the destination running in postcopy, discard pages
+ * that have already been sent but redirtied on the source.
+ * CMD_POSTCOPY_RAM_DISCARD consist of:
+ *      byte   version (0)
+ *      byte   Length of name field (not including 0)
+ *  n x byte   RAM block name
+ *      byte   0 terminator (just for safety)
+ *  n x        Byte ranges within the named RAMBlock
+ *      be64   Start of the range
+ *      be64   Length
+ *
+ *  name:  RAMBlock name that these entries are part of
+ *  len: Number of page entries
+ *  start_list: 'len' addresses
+ *  length_list: 'len' addresses
+ *
+ */
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list)
+{
+    uint8_t *buf;
+    uint16_t tmplen;
+    uint16_t t;
+    size_t name_len = strlen(name);
+
+    trace_qemu_savevm_send_postcopy_ram_discard(name, len);
+    assert(name_len < 256);
+    buf = g_malloc0(1 + 1 + name_len + 1 + (8 + 8) * len);
+    buf[0] = postcopy_ram_discard_version;
+    buf[1] = name_len;
+    memcpy(buf + 2, name, name_len);
+    tmplen = 2 + name_len;
+    buf[tmplen++] = '\0';
+
+    for (t = 0; t < len; t++) {
+        cpu_to_be64w((uint64_t *)(buf + tmplen), start_list[t]);
+        tmplen += 8;
+        cpu_to_be64w((uint64_t *)(buf + tmplen), length_list[t]);
+        tmplen += 8;
+    }
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RAM_DISCARD, tmplen, buf);
+    g_free(buf);
+}
+
+/* Get the destination into a state where it can receive postcopy data. */
+void qemu_savevm_send_postcopy_listen(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_listen();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_LISTEN, 0, NULL);
+}
+
+/* Kick the destination into running */
+void qemu_savevm_send_postcopy_run(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_run();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RUN, 0, NULL);
+}
+
 bool qemu_savevm_state_blocked(Error **errp)
 {
     SaveStateEntry *se;
@@ -711,11 +877,23 @@ bool qemu_savevm_state_blocked(Error **errp)
     return false;
 }
 
+static bool enforce_config_section(void)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    return machine->enforce_config_section;
+}
+
 void qemu_savevm_state_header(QEMUFile *f)
 {
     trace_savevm_state_header();
     qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
     qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
+        qemu_put_byte(f, QEMU_VM_CONFIGURATION);
+        vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0);
+    }
+
 }
 
 void qemu_savevm_state_begin(QEMUFile *f,
@@ -732,11 +910,6 @@ void qemu_savevm_state_begin(QEMUFile *f,
         se->ops->set_params(params, se->opaque);
     }
 
-    if (!savevm_state.skip_configuration) {
-        qemu_put_byte(f, QEMU_VM_CONFIGURATION);
-        vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0);
-    }
-
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
         if (!se->ops || !se->ops->save_live_setup) {
             continue;
@@ -763,7 +936,7 @@ void qemu_savevm_state_begin(QEMUFile *f,
  *   0 : We haven't finished, caller have to go again
  *   1 : We have finished, we can go to complete phase
  */
-int qemu_savevm_state_iterate(QEMUFile *f)
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy)
 {
     SaveStateEntry *se;
     int ret = 1;
@@ -778,6 +951,15 @@ int qemu_savevm_state_iterate(QEMUFile *f)
                 continue;
             }
         }
+        /*
+         * In the postcopy phase, any device that doesn't know how to
+         * do postcopy should have saved it's state in the _complete
+         * call that's already run, it might get confused if we call
+         * iterate afterwards.
+         */
+        if (postcopy && !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
         if (qemu_file_rate_limit(f)) {
             return 0;
         }
@@ -806,24 +988,69 @@ int qemu_savevm_state_iterate(QEMUFile *f)
 static bool should_send_vmdesc(void)
 {
     MachineState *machine = MACHINE(qdev_get_machine());
-    return !machine->suppress_vmdesc;
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
+    return !machine->suppress_vmdesc && !in_postcopy;
 }
 
-void qemu_savevm_state_complete(QEMUFile *f)
+/*
+ * Calls the save_live_complete_postcopy methods
+ * causing the last few pages to be sent immediately and doing any associated
+ * cleanup.
+ * Note postcopy also calls qemu_savevm_state_complete_precopy to complete
+ * all the other devices, but that happens at the point we switch to postcopy.
+ */
+void qemu_savevm_state_complete_postcopy(QEMUFile *f)
+{
+    SaveStateEntry *se;
+    int ret;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        trace_savevm_section_start(se->idstr, se->section_id);
+        /* Section type */
+        qemu_put_byte(f, QEMU_VM_SECTION_END);
+        qemu_put_be32(f, se->section_id);
+
+        ret = se->ops->save_live_complete_postcopy(f, se->opaque);
+        trace_savevm_section_end(se->idstr, se->section_id, ret);
+        save_section_footer(f, se);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+            return;
+        }
+    }
+
+    qemu_put_byte(f, QEMU_VM_EOF);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only)
 {
     QJSON *vmdesc;
     int vmdesc_len;
     SaveStateEntry *se;
     int ret;
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
 
-    trace_savevm_state_complete();
+    trace_savevm_state_complete_precopy();
 
     cpu_synchronize_all_states();
 
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
-        if (!se->ops || !se->ops->save_live_complete) {
+        if (!se->ops ||
+            (in_postcopy && se->ops->save_live_complete_postcopy) ||
+            (in_postcopy && !iterable_only) ||
+            !se->ops->save_live_complete_precopy) {
             continue;
         }
+
         if (se->ops && se->ops->is_active) {
             if (!se->ops->is_active(se->opaque)) {
                 continue;
@@ -833,7 +1060,7 @@ void qemu_savevm_state_complete(QEMUFile *f)
 
         save_section_header(f, se, QEMU_VM_SECTION_END);
 
-        ret = se->ops->save_live_complete(f, se->opaque);
+        ret = se->ops->save_live_complete_precopy(f, se->opaque);
         trace_savevm_section_end(se->idstr, se->section_id, ret);
         save_section_footer(f, se);
         if (ret < 0) {
@@ -842,6 +1069,10 @@ void qemu_savevm_state_complete(QEMUFile *f)
         }
     }
 
+    if (iterable_only) {
+        return;
+    }
+
     vmdesc = qjson_new();
     json_prop_int(vmdesc, "page_size", TARGET_PAGE_SIZE);
     json_start_array(vmdesc, "devices");
@@ -862,15 +1093,17 @@ void qemu_savevm_state_complete(QEMUFile *f)
         json_prop_int(vmdesc, "instance_id", se->instance_id);
 
         save_section_header(f, se, QEMU_VM_SECTION_FULL);
-
         vmstate_save(f, se, vmdesc);
-
-        json_end_object(vmdesc);
         trace_savevm_section_end(se->idstr, se->section_id, 0);
         save_section_footer(f, se);
+
+        json_end_object(vmdesc);
     }
 
-    qemu_put_byte(f, QEMU_VM_EOF);
+    if (!in_postcopy) {
+        /* Postcopy stream will still be going */
+        qemu_put_byte(f, QEMU_VM_EOF);
+    }
 
     json_end_array(vmdesc);
     qjson_finish(vmdesc);
@@ -886,10 +1119,19 @@ void qemu_savevm_state_complete(QEMUFile *f)
     qemu_fflush(f);
 }
 
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
+/* Give an estimate of the amount left to be transferred,
+ * the result is split into the amount for units that can and
+ * for units that can't do postcopy.
+ */
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable)
 {
     SaveStateEntry *se;
-    uint64_t ret = 0;
+
+    *res_non_postcopiable = 0;
+    *res_postcopiable = 0;
+
 
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
         if (!se->ops || !se->ops->save_live_pending) {
@@ -900,19 +1142,19 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
                 continue;
             }
         }
-        ret += se->ops->save_live_pending(f, se->opaque, max_size);
+        se->ops->save_live_pending(f, se->opaque, max_size,
+                                   res_non_postcopiable, res_postcopiable);
     }
-    return ret;
 }
 
-void qemu_savevm_state_cancel(void)
+void qemu_savevm_state_cleanup(void)
 {
     SaveStateEntry *se;
 
-    trace_savevm_state_cancel();
+    trace_savevm_state_cleanup();
     QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
-        if (se->ops && se->ops->cancel) {
-            se->ops->cancel(se->opaque);
+        if (se->ops && se->ops->cleanup) {
+            se->ops->cleanup(se->opaque);
         }
     }
 }
@@ -924,6 +1166,8 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
         .blk = 0,
         .shared = 0
     };
+    MigrationState *ms = migrate_init(&params);
+    ms->to_dst_file = f;
 
     if (qemu_savevm_state_blocked(errp)) {
         return -EINVAL;
@@ -935,17 +1179,17 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp)
     qemu_mutex_lock_iothread();
 
     while (qemu_file_get_error(f) == 0) {
-        if (qemu_savevm_state_iterate(f) > 0) {
+        if (qemu_savevm_state_iterate(f, false) > 0) {
             break;
         }
     }
 
     ret = qemu_file_get_error(f);
     if (ret == 0) {
-        qemu_savevm_state_complete(f);
+        qemu_savevm_state_complete_precopy(f, false);
         ret = qemu_file_get_error(f);
     }
-    qemu_savevm_state_cancel();
+    qemu_savevm_state_cleanup();
     if (ret != 0) {
         error_setg_errno(errp, -ret, "Error while writing VM state");
     }
@@ -1004,6 +1248,445 @@ static SaveStateEntry *find_se(const char *idstr, int instance_id)
     return NULL;
 }
 
+enum LoadVMExitCodes {
+    /* Allow a command to quit all layers of nested loadvm loops */
+    LOADVM_QUIT     =  1,
+};
+
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
+
+/* ------ incoming postcopy messages ------ */
+/* 'advise' arrives before any transfers just to tell us that a postcopy
+ * *might* happen - it might be skipped if precopy transferred everything
+ * quickly.
+ */
+static int loadvm_postcopy_handle_advise(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+    uint64_t remote_hps, remote_tps;
+
+    trace_loadvm_postcopy_handle_advise();
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        error_report("CMD_POSTCOPY_ADVISE in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    if (!postcopy_ram_supported_by_host()) {
+        return -1;
+    }
+
+    remote_hps = qemu_get_be64(mis->from_src_file);
+    if (remote_hps != getpagesize())  {
+        /*
+         * Some combinations of mismatch are probably possible but it gets
+         * a bit more complicated.  In particular we need to place whole
+         * host pages on the dest at once, and we need to ensure that we
+         * handle dirtying to make sure we never end up sending part of
+         * a hostpage on it's own.
+         */
+        error_report("Postcopy needs matching host page sizes (s=%d d=%d)",
+                     (int)remote_hps, getpagesize());
+        return -1;
+    }
+
+    remote_tps = qemu_get_be64(mis->from_src_file);
+    if (remote_tps != (1ul << qemu_target_page_bits())) {
+        /*
+         * Again, some differences could be dealt with, but for now keep it
+         * simple.
+         */
+        error_report("Postcopy needs matching target page sizes (s=%d d=%d)",
+                     (int)remote_tps, 1 << qemu_target_page_bits());
+        return -1;
+    }
+
+    if (ram_postcopy_incoming_init(mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+
+    return 0;
+}
+
+/* After postcopy we will be told to throw some pages away since they're
+ * dirty and will have to be demand fetched.  Must happen before CPU is
+ * started.
+ * There can be 0..many of these messages, each encoding multiple pages.
+ */
+static int loadvm_postcopy_ram_handle_discard(MigrationIncomingState *mis,
+                                              uint16_t len)
+{
+    int tmp;
+    char ramid[256];
+    PostcopyState ps = postcopy_state_get();
+
+    trace_loadvm_postcopy_ram_handle_discard();
+
+    switch (ps) {
+    case POSTCOPY_INCOMING_ADVISE:
+        /* 1st discard */
+        tmp = postcopy_ram_prepare_discard(mis);
+        if (tmp) {
+            return tmp;
+        }
+        break;
+
+    case POSTCOPY_INCOMING_DISCARD:
+        /* Expected state */
+        break;
+
+    default:
+        error_report("CMD_POSTCOPY_RAM_DISCARD in wrong postcopy state (%d)",
+                     ps);
+        return -1;
+    }
+    /* We're expecting a
+     *    Version (0)
+     *    a RAM ID string (length byte, name, 0 term)
+     *    then at least 1 16 byte chunk
+    */
+    if (len < (1 + 1 + 1 + 1 + 2 * 8)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != postcopy_ram_discard_version) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid version (%d)", tmp);
+        return -1;
+    }
+
+    if (!qemu_get_counted_string(mis->from_src_file, ramid)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD Failed to read RAMBlock ID");
+        return -1;
+    }
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != 0) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD missing nil (%d)", tmp);
+        return -1;
+    }
+
+    len -= 3 + strlen(ramid);
+    if (len % 16) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+    trace_loadvm_postcopy_ram_handle_discard_header(ramid, len);
+    while (len) {
+        uint64_t start_addr, block_length;
+        start_addr = qemu_get_be64(mis->from_src_file);
+        block_length = qemu_get_be64(mis->from_src_file);
+
+        len -= 16;
+        int ret = ram_discard_range(mis, ramid, start_addr,
+                                    block_length);
+        if (ret) {
+            return ret;
+        }
+    }
+    trace_loadvm_postcopy_ram_handle_discard_end();
+
+    return 0;
+}
+
+/*
+ * Triggered by a postcopy_listen command; this thread takes over reading
+ * the input stream, leaving the main thread free to carry on loading the rest
+ * of the device state (from RAM).
+ * (TODO:This could do with being in a postcopy file - but there again it's
+ * just another input loop, not that postcopy specific)
+ */
+static void *postcopy_ram_listen_thread(void *opaque)
+{
+    QEMUFile *f = opaque;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    int load_res;
+
+    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                                   MIGRATION_STATUS_POSTCOPY_ACTIVE);
+    qemu_sem_post(&mis->listen_thread_sem);
+    trace_postcopy_ram_listen_thread_start();
+
+    /*
+     * Because we're a thread and not a coroutine we can't yield
+     * in qemu_file, and thus we must be blocking now.
+     */
+    qemu_file_set_blocking(f, true);
+    load_res = qemu_loadvm_state_main(f, mis);
+    /* And non-blocking again so we don't block in any cleanup */
+    qemu_file_set_blocking(f, false);
+
+    trace_postcopy_ram_listen_thread_exit();
+    if (load_res < 0) {
+        error_report("%s: loadvm failed: %d", __func__, load_res);
+        qemu_file_set_error(f, load_res);
+        migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                                       MIGRATION_STATUS_FAILED);
+    } else {
+        /*
+         * This looks good, but it's possible that the device loading in the
+         * main thread hasn't finished yet, and so we might not be in 'RUN'
+         * state yet; wait for the end of the main thread.
+         */
+        qemu_event_wait(&mis->main_thread_load_event);
+    }
+    postcopy_ram_incoming_cleanup(mis);
+
+    if (load_res < 0) {
+        /*
+         * If something went wrong then we have a bad state so exit;
+         * depending how far we got it might be possible at this point
+         * to leave the guest running and fire MCEs for pages that never
+         * arrived as a desperate recovery step.
+         */
+        exit(EXIT_FAILURE);
+    }
+
+    migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                                   MIGRATION_STATUS_COMPLETED);
+    /*
+     * If everything has worked fine, then the main thread has waited
+     * for us to start, and we're the last use of the mis.
+     * (If something broke then qemu will have to exit anyway since it's
+     * got a bad migration state).
+     */
+    migration_incoming_state_destroy();
+
+
+    return NULL;
+}
+
+/* After this message we must be able to immediately receive postcopy data */
+static int loadvm_postcopy_handle_listen(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_LISTENING);
+    trace_loadvm_postcopy_handle_listen();
+    if (ps != POSTCOPY_INCOMING_ADVISE && ps != POSTCOPY_INCOMING_DISCARD) {
+        error_report("CMD_POSTCOPY_LISTEN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+    if (ps == POSTCOPY_INCOMING_ADVISE) {
+        /*
+         * A rare case, we entered listen without having to do any discards,
+         * so do the setup that's normally done at the time of the 1st discard.
+         */
+        postcopy_ram_prepare_discard(mis);
+    }
+
+    /*
+     * Sensitise RAM - can now generate requests for blocks that don't exist
+     * However, at this point the CPU shouldn't be running, and the IO
+     * shouldn't be doing anything yet so don't actually expect requests
+     */
+    if (postcopy_ram_enable_notify(mis)) {
+        return -1;
+    }
+
+    if (mis->have_listen_thread) {
+        error_report("CMD_POSTCOPY_RAM_LISTEN already has a listen thread");
+        return -1;
+    }
+
+    mis->have_listen_thread = true;
+    /* Start up the listening thread and wait for it to signal ready */
+    qemu_sem_init(&mis->listen_thread_sem, 0);
+    qemu_thread_create(&mis->listen_thread, "postcopy/listen",
+                       postcopy_ram_listen_thread, mis->from_src_file,
+                       QEMU_THREAD_DETACHED);
+    qemu_sem_wait(&mis->listen_thread_sem);
+    qemu_sem_destroy(&mis->listen_thread_sem);
+
+    return 0;
+}
+
+
+typedef struct {
+    QEMUBH *bh;
+} HandleRunBhData;
+
+static void loadvm_postcopy_handle_run_bh(void *opaque)
+{
+    Error *local_err = NULL;
+    HandleRunBhData *data = opaque;
+
+    /* TODO we should move all of this lot into postcopy_ram.c or a shared code
+     * in migration.c
+     */
+    cpu_synchronize_all_post_init();
+
+    qemu_announce_self();
+
+    /* Make sure all file formats flush their mutable metadata */
+    bdrv_invalidate_cache_all(&local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    }
+
+    trace_loadvm_postcopy_handle_run_cpu_sync();
+    cpu_synchronize_all_post_init();
+
+    trace_loadvm_postcopy_handle_run_vmstart();
+
+    if (autostart) {
+        /* Hold onto your hats, starting the CPU */
+        vm_start();
+    } else {
+        /* leave it paused and let management decide when to start the CPU */
+        runstate_set(RUN_STATE_PAUSED);
+    }
+
+    qemu_bh_delete(data->bh);
+    g_free(data);
+}
+
+/* After all discards we can start running and asking for pages */
+static int loadvm_postcopy_handle_run(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_RUNNING);
+    HandleRunBhData *data;
+
+    trace_loadvm_postcopy_handle_run();
+    if (ps != POSTCOPY_INCOMING_LISTENING) {
+        error_report("CMD_POSTCOPY_RUN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    data = g_new(HandleRunBhData, 1);
+    data->bh = qemu_bh_new(loadvm_postcopy_handle_run_bh, data);
+    qemu_bh_schedule(data->bh);
+
+    /* We need to finish reading the stream from the package
+     * and also stop reading anything more from the stream that loaded the
+     * package (since it's now being read by the listener thread).
+     * LOADVM_QUIT will quit all the layers of nested loadvm loops.
+     */
+    return LOADVM_QUIT;
+}
+
+/**
+ * Immediately following this command is a blob of data containing an embedded
+ * chunk of migration stream; read it and load it.
+ *
+ * @mis: Incoming state
+ * @length: Length of packaged data to read
+ *
+ * Returns: Negative values on error
+ *
+ */
+static int loadvm_handle_cmd_packaged(MigrationIncomingState *mis)
+{
+    int ret;
+    uint8_t *buffer;
+    uint32_t length;
+    QEMUSizedBuffer *qsb;
+
+    length = qemu_get_be32(mis->from_src_file);
+    trace_loadvm_handle_cmd_packaged(length);
+
+    if (length > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("Unreasonably large packaged state: %u", length);
+        return -1;
+    }
+    buffer = g_malloc0(length);
+    ret = qemu_get_buffer(mis->from_src_file, buffer, (int)length);
+    if (ret != length) {
+        g_free(buffer);
+        error_report("CMD_PACKAGED: Buffer receive fail ret=%d length=%d",
+                     ret, length);
+        return (ret < 0) ? ret : -EAGAIN;
+    }
+    trace_loadvm_handle_cmd_packaged_received(ret);
+
+    /* Setup a dummy QEMUFile that actually reads from the buffer */
+    qsb = qsb_create(buffer, length);
+    g_free(buffer); /* Because qsb_create copies */
+    if (!qsb) {
+        error_report("Unable to create qsb");
+    }
+    QEMUFile *packf = qemu_bufopen("r", qsb);
+
+    ret = qemu_loadvm_state_main(packf, mis);
+    trace_loadvm_handle_cmd_packaged_main(ret);
+    qemu_fclose(packf);
+    qsb_free(qsb);
+
+    return ret;
+}
+
+/*
+ * Process an incoming 'QEMU_VM_COMMAND'
+ * 0           just a normal return
+ * LOADVM_QUIT All good, but exit the loop
+ * <0          Error
+ */
+static int loadvm_process_command(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    uint16_t cmd;
+    uint16_t len;
+    uint32_t tmp32;
+
+    cmd = qemu_get_be16(f);
+    len = qemu_get_be16(f);
+
+    trace_loadvm_process_command(cmd, len);
+    if (cmd >= MIG_CMD_MAX || cmd == MIG_CMD_INVALID) {
+        error_report("MIG_CMD 0x%x unknown (len 0x%x)", cmd, len);
+        return -EINVAL;
+    }
+
+    if (mig_cmd_args[cmd].len != -1 && mig_cmd_args[cmd].len != len) {
+        error_report("%s received with bad length - expecting %zu, got %d",
+                     mig_cmd_args[cmd].name,
+                     (size_t)mig_cmd_args[cmd].len, len);
+        return -ERANGE;
+    }
+
+    switch (cmd) {
+    case MIG_CMD_OPEN_RETURN_PATH:
+        if (mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH called when RP already open");
+            /* Not really a problem, so don't give up */
+            return 0;
+        }
+        mis->to_src_file = qemu_file_get_return_path(f);
+        if (!mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH failed");
+            return -1;
+        }
+        break;
+
+    case MIG_CMD_PING:
+        tmp32 = qemu_get_be32(f);
+        trace_loadvm_process_command_ping(tmp32);
+        if (!mis->to_src_file) {
+            error_report("CMD_PING (0x%x) received with no return path",
+                         tmp32);
+            return -1;
+        }
+        migrate_send_rp_pong(mis, tmp32);
+        break;
+
+    case MIG_CMD_PACKAGED:
+        return loadvm_handle_cmd_packaged(mis);
+
+    case MIG_CMD_POSTCOPY_ADVISE:
+        return loadvm_postcopy_handle_advise(mis);
+
+    case MIG_CMD_POSTCOPY_LISTEN:
+        return loadvm_postcopy_handle_listen(mis);
+
+    case MIG_CMD_POSTCOPY_RUN:
+        return loadvm_postcopy_handle_run(mis);
+
+    case MIG_CMD_POSTCOPY_RAM_DISCARD:
+        return loadvm_postcopy_ram_handle_discard(mis, len);
+    }
+
+    return 0;
+}
+
 struct LoadStateEntry {
     QLIST_ENTRY(LoadStateEntry) entry;
     SaveStateEntry *se;
@@ -1056,14 +1739,141 @@ void loadvm_free_handlers(MigrationIncomingState *mis)
     }
 }
 
+static int
+qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint32_t instance_id, version_id, section_id;
+    SaveStateEntry *se;
+    LoadStateEntry *le;
+    char idstr[256];
+    int ret;
+
+    /* Read section start */
+    section_id = qemu_get_be32(f);
+    if (!qemu_get_counted_string(f, idstr)) {
+        error_report("Unable to read ID string for section %u",
+                     section_id);
+        return -EINVAL;
+    }
+    instance_id = qemu_get_be32(f);
+    version_id = qemu_get_be32(f);
+
+    trace_qemu_loadvm_state_section_startfull(section_id, idstr,
+            instance_id, version_id);
+    /* Find savevm section */
+    se = find_se(idstr, instance_id);
+    if (se == NULL) {
+        error_report("Unknown savevm section or instance '%s' %d",
+                     idstr, instance_id);
+        return -EINVAL;
+    }
+
+    /* Validate version */
+    if (version_id > se->version_id) {
+        error_report("savevm: unsupported version %d for '%s' v%d",
+                     version_id, idstr, se->version_id);
+        return -EINVAL;
+    }
+
+    /* Add entry */
+    le = g_malloc0(sizeof(*le));
+
+    le->se = se;
+    le->section_id = section_id;
+    le->version_id = version_id;
+    QLIST_INSERT_HEAD(&mis->loadvm_handlers, le, entry);
+
+    ret = vmstate_load(f, le->se, le->version_id);
+    if (ret < 0) {
+        error_report("error while loading state for instance 0x%x of"
+                     " device '%s'", instance_id, idstr);
+        return ret;
+    }
+    if (!check_section_footer(f, le)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int
+qemu_loadvm_section_part_end(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint32_t section_id;
+    LoadStateEntry *le;
+    int ret;
+
+    section_id = qemu_get_be32(f);
+
+    trace_qemu_loadvm_state_section_partend(section_id);
+    QLIST_FOREACH(le, &mis->loadvm_handlers, entry) {
+        if (le->section_id == section_id) {
+            break;
+        }
+    }
+    if (le == NULL) {
+        error_report("Unknown savevm section %d", section_id);
+        return -EINVAL;
+    }
+
+    ret = vmstate_load(f, le->se, le->version_id);
+    if (ret < 0) {
+        error_report("error while loading state section id %d(%s)",
+                     section_id, le->se->idstr);
+        return ret;
+    }
+    if (!check_section_footer(f, le)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint8_t section_type;
+    int ret;
+
+    while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
+
+        trace_qemu_loadvm_state_section(section_type);
+        switch (section_type) {
+        case QEMU_VM_SECTION_START:
+        case QEMU_VM_SECTION_FULL:
+            ret = qemu_loadvm_section_start_full(f, mis);
+            if (ret < 0) {
+                return ret;
+            }
+            break;
+        case QEMU_VM_SECTION_PART:
+        case QEMU_VM_SECTION_END:
+            ret = qemu_loadvm_section_part_end(f, mis);
+            if (ret < 0) {
+                return ret;
+            }
+            break;
+        case QEMU_VM_COMMAND:
+            ret = loadvm_process_command(f);
+            trace_qemu_loadvm_state_section_command(ret);
+            if ((ret < 0) || (ret & LOADVM_QUIT)) {
+                return ret;
+            }
+            break;
+        default:
+            error_report("Unknown savevm section type %d", section_type);
+            return -EINVAL;
+        }
+    }
+
+    return 0;
+}
+
 int qemu_loadvm_state(QEMUFile *f)
 {
     MigrationIncomingState *mis = migration_incoming_get_current();
     Error *local_err = NULL;
-    uint8_t section_type;
     unsigned int v;
     int ret;
-    int file_error_after_eof = -1;
 
     if (qemu_savevm_state_blocked(&local_err)) {
         error_report_err(local_err);
@@ -1086,7 +1896,7 @@ int qemu_loadvm_state(QEMUFile *f)
         return -ENOTSUP;
     }
 
-    if (!savevm_state.skip_configuration) {
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
         if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) {
             error_report("Configuration section missing");
             return -EINVAL;
@@ -1098,99 +1908,19 @@ int qemu_loadvm_state(QEMUFile *f)
         }
     }
 
-    while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
-        uint32_t instance_id, version_id, section_id;
-        SaveStateEntry *se;
-        LoadStateEntry *le;
-        char idstr[256];
-
-        trace_qemu_loadvm_state_section(section_type);
-        switch (section_type) {
-        case QEMU_VM_SECTION_START:
-        case QEMU_VM_SECTION_FULL:
-            /* Read section start */
-            section_id = qemu_get_be32(f);
-            if (!qemu_get_counted_string(f, idstr)) {
-                error_report("Unable to read ID string for section %u",
-                            section_id);
-                return -EINVAL;
-            }
-            instance_id = qemu_get_be32(f);
-            version_id = qemu_get_be32(f);
-
-            trace_qemu_loadvm_state_section_startfull(section_id, idstr,
-                                                      instance_id, version_id);
-            /* Find savevm section */
-            se = find_se(idstr, instance_id);
-            if (se == NULL) {
-                error_report("Unknown savevm section or instance '%s' %d",
-                             idstr, instance_id);
-                ret = -EINVAL;
-                goto out;
-            }
-
-            /* Validate version */
-            if (version_id > se->version_id) {
-                error_report("savevm: unsupported version %d for '%s' v%d",
-                             version_id, idstr, se->version_id);
-                ret = -EINVAL;
-                goto out;
-            }
-
-            /* Add entry */
-            le = g_malloc0(sizeof(*le));
-
-            le->se = se;
-            le->section_id = section_id;
-            le->version_id = version_id;
-            QLIST_INSERT_HEAD(&mis->loadvm_handlers, le, entry);
-
-            ret = vmstate_load(f, le->se, le->version_id);
-            if (ret < 0) {
-                error_report("error while loading state for instance 0x%x of"
-                             " device '%s'", instance_id, idstr);
-                goto out;
-            }
-            if (!check_section_footer(f, le)) {
-                ret = -EINVAL;
-                goto out;
-            }
-            break;
-        case QEMU_VM_SECTION_PART:
-        case QEMU_VM_SECTION_END:
-            section_id = qemu_get_be32(f);
+    ret = qemu_loadvm_state_main(f, mis);
+    qemu_event_set(&mis->main_thread_load_event);
 
-            trace_qemu_loadvm_state_section_partend(section_id);
-            QLIST_FOREACH(le, &mis->loadvm_handlers, entry) {
-                if (le->section_id == section_id) {
-                    break;
-                }
-            }
-            if (le == NULL) {
-                error_report("Unknown savevm section %d", section_id);
-                ret = -EINVAL;
-                goto out;
-            }
+    trace_qemu_loadvm_state_post_main(ret);
 
-            ret = vmstate_load(f, le->se, le->version_id);
-            if (ret < 0) {
-                error_report("error while loading state section id %d(%s)",
-                             section_id, le->se->idstr);
-                goto out;
-            }
-            if (!check_section_footer(f, le)) {
-                ret = -EINVAL;
-                goto out;
-            }
-            break;
-        default:
-            error_report("Unknown savevm section type %d", section_type);
-            ret = -EINVAL;
-            goto out;
-        }
+    if (mis->have_listen_thread) {
+        /* Listen thread still going, can't clean up yet */
+        return ret;
     }
 
-    file_error_after_eof = qemu_file_get_error(f);
+    if (ret == 0) {
+        ret = qemu_file_get_error(f);
+    }
 
     /*
      * Try to read in the VMDESC section as well, so that dumping tools that
@@ -1202,10 +1932,10 @@ int qemu_loadvm_state(QEMUFile *f)
      * We also mustn't read data that isn't there; some transports (RDMA)
      * will stall waiting for that data when the source has already closed.
      */
-    if (should_send_vmdesc()) {
+    if (ret == 0 && should_send_vmdesc()) {
         uint8_t *buf;
         uint32_t size;
-        section_type = qemu_get_byte(f);
+        uint8_t  section_type = qemu_get_byte(f);
 
         if (section_type != QEMU_VM_VMDESCRIPTION) {
             error_report("Expected vmdescription section, but got %d",
@@ -1229,57 +1959,9 @@ int qemu_loadvm_state(QEMUFile *f)
 
     cpu_synchronize_all_post_init();
 
-    ret = 0;
-
-out:
-    if (ret == 0) {
-        /* We may not have a VMDESC section, so ignore relative errors */
-        ret = file_error_after_eof;
-    }
-
     return ret;
 }
 
-static BlockDriverState *find_vmstate_bs(void)
-{
-    BlockDriverState *bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            return bs;
-        }
-    }
-    return NULL;
-}
-
-/*
- * Deletes snapshots of a given name in all opened images.
- */
-static int del_existing_snapshots(Monitor *mon, const char *name)
-{
-    BlockDriverState *bs;
-    QEMUSnapshotInfo sn1, *snapshot = &sn1;
-    Error *err = NULL;
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs) &&
-            bdrv_snapshot_find(bs, snapshot, name) >= 0) {
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-                return -1;
-            }
-        }
-    }
-
-    return 0;
-}
-
 void hmp_savevm(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
@@ -1292,27 +1974,28 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
     struct tm tm;
     const char *name = qdict_get_try_str(qdict, "name");
     Error *local_err = NULL;
+    AioContext *aio_context;
 
-    /* Verify if there is a device that doesn't support snapshots and is writable */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
+    if (!bdrv_all_can_snapshot(&bs)) {
+        monitor_printf(mon, "Device '%s' is writable but does not "
+                       "support snapshots.\n", bdrv_get_device_name(bs));
+        return;
+    }
 
-        if (!bdrv_can_snapshot(bs)) {
-            monitor_printf(mon, "Device '%s' is writable but does not support snapshots.\n",
-                               bdrv_get_device_name(bs));
-            return;
-        }
+    /* Delete old snapshots of the same name */
+    if (name && bdrv_all_delete_snapshot(name, &bs1, &local_err) < 0) {
+        error_reportf_err(local_err,
+                          "Error while deleting snapshot on device '%s': ",
+                          bdrv_get_device_name(bs1));
+        return;
     }
 
-    bs = find_vmstate_bs();
-    if (!bs) {
+    bs = bdrv_all_find_vmstate_bs();
+    if (bs == NULL) {
         monitor_printf(mon, "No block device can accept snapshots\n");
         return;
     }
+    aio_context = bdrv_get_aio_context(bs);
 
     saved_vm_running = runstate_is_running();
 
@@ -1323,6 +2006,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
     }
     vm_stop(RUN_STATE_SAVE_VM);
 
+    aio_context_acquire(aio_context);
+
     memset(sn, 0, sizeof(*sn));
 
     /* fill auxiliary fields */
@@ -1345,11 +2030,6 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
         strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
     }
 
-    /* Delete old snapshots of the same name */
-    if (name && del_existing_snapshots(mon, name) < 0) {
-        goto the_end;
-    }
-
     /* save the VM state */
     f = qemu_fopen_bdrv(bs, 1);
     if (!f) {
@@ -1360,27 +2040,18 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)
     vm_state_size = qemu_ftell(f);
     qemu_fclose(f);
     if (ret < 0) {
-        monitor_printf(mon, "%s\n", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         goto the_end;
     }
 
-    /* create the snapshots */
-
-    bs1 = NULL;
-    while ((bs1 = bdrv_next(bs1))) {
-        if (bdrv_can_snapshot(bs1)) {
-            /* Write VM state size only to the image that contains the state */
-            sn->vm_state_size = (bs == bs1 ? vm_state_size : 0);
-            ret = bdrv_snapshot_create(bs1, sn);
-            if (ret < 0) {
-                monitor_printf(mon, "Error while creating snapshot on '%s'\n",
-                               bdrv_get_device_name(bs1));
-            }
-        }
+    ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, &bs);
+    if (ret < 0) {
+        monitor_printf(mon, "Error while creating snapshot on '%s'\n",
+                       bdrv_get_device_name(bs));
     }
 
  the_end:
+    aio_context_release(aio_context);
     if (saved_vm_running) {
         vm_start();
     }
@@ -1419,15 +2090,31 @@ int load_vmstate(const char *name)
     QEMUSnapshotInfo sn;
     QEMUFile *f;
     int ret;
+    AioContext *aio_context;
+
+    if (!bdrv_all_can_snapshot(&bs)) {
+        error_report("Device '%s' is writable but does not support snapshots.",
+                     bdrv_get_device_name(bs));
+        return -ENOTSUP;
+    }
+    ret = bdrv_all_find_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Device '%s' does not have the requested snapshot '%s'",
+                     bdrv_get_device_name(bs), name);
+        return ret;
+    }
 
-    bs_vm_state = find_vmstate_bs();
+    bs_vm_state = bdrv_all_find_vmstate_bs();
     if (!bs_vm_state) {
         error_report("No block device supports snapshots");
         return -ENOTSUP;
     }
+    aio_context = bdrv_get_aio_context(bs_vm_state);
 
     /* Don't even try to load empty VM states */
+    aio_context_acquire(aio_context);
     ret = bdrv_snapshot_find(bs_vm_state, &sn, name);
+    aio_context_release(aio_context);
     if (ret < 0) {
         return ret;
     } else if (sn.vm_state_size == 0) {
@@ -1436,42 +2123,14 @@ int load_vmstate(const char *name)
         return -EINVAL;
     }
 
-    /* Verify if there is any device that doesn't support snapshots and is
-    writable and check if the requested snapshot is available too. */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
-
-        if (!bdrv_can_snapshot(bs)) {
-            error_report("Device '%s' is writable but does not support snapshots.",
-                               bdrv_get_device_name(bs));
-            return -ENOTSUP;
-        }
-
-        ret = bdrv_snapshot_find(bs, &sn, name);
-        if (ret < 0) {
-            error_report("Device '%s' does not have the requested snapshot '%s'",
-                           bdrv_get_device_name(bs), name);
-            return ret;
-        }
-    }
-
     /* Flush all IO requests so they don't interfere with the new state.  */
     bdrv_drain_all();
 
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            ret = bdrv_snapshot_goto(bs, name);
-            if (ret < 0) {
-                error_report("Error %d while activating snapshot '%s' on '%s'",
-                             ret, name, bdrv_get_device_name(bs));
-                return ret;
-            }
-        }
+    ret = bdrv_all_goto_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Error %d while activating snapshot '%s' on '%s'",
+                     ret, name, bdrv_get_device_name(bs));
+        return ret;
     }
 
     /* restore the VM state */
@@ -1483,9 +2142,12 @@ int load_vmstate(const char *name)
 
     qemu_system_reset(VMRESET_SILENT);
     migration_incoming_state_new(f);
-    ret = qemu_loadvm_state(f);
 
+    aio_context_acquire(aio_context);
+    ret = qemu_loadvm_state(f);
     qemu_fclose(f);
+    aio_context_release(aio_context);
+
     migration_incoming_state_destroy();
     if (ret < 0) {
         error_report("Error %d while loading VM state", ret);
@@ -1501,43 +2163,33 @@ void hmp_delvm(Monitor *mon, const QDict *qdict)
     Error *err;
     const char *name = qdict_get_str(qdict, "name");
 
-    if (!find_vmstate_bs()) {
-        monitor_printf(mon, "No block device supports snapshots\n");
-        return;
-    }
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            err = NULL;
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-            }
-        }
+    if (bdrv_all_delete_snapshot(name, &bs, &err) < 0) {
+        error_reportf_err(err,
+                          "Error while deleting snapshot on device '%s': ",
+                          bdrv_get_device_name(bs));
     }
 }
 
 void hmp_info_snapshots(Monitor *mon, const QDict *qdict)
 {
     BlockDriverState *bs, *bs1;
-    QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s;
-    int nb_sns, i, ret, available;
+    QEMUSnapshotInfo *sn_tab, *sn;
+    int nb_sns, i;
     int total;
     int *available_snapshots;
+    AioContext *aio_context;
 
-    bs = find_vmstate_bs();
+    bs = bdrv_all_find_vmstate_bs();
     if (!bs) {
         monitor_printf(mon, "No available block device supports snapshots\n");
         return;
     }
+    aio_context = bdrv_get_aio_context(bs);
 
+    aio_context_acquire(aio_context);
     nb_sns = bdrv_snapshot_list(bs, &sn_tab);
+    aio_context_release(aio_context);
+
     if (nb_sns < 0) {
         monitor_printf(mon, "bdrv_snapshot_list: error %d\n", nb_sns);
         return;
@@ -1548,24 +2200,10 @@ void hmp_info_snapshots(Monitor *mon, const QDict *qdict)
         return;
     }
 
-    available_snapshots = g_malloc0(sizeof(int) * nb_sns);
+    available_snapshots = g_new0(int, nb_sns);
     total = 0;
     for (i = 0; i < nb_sns; i++) {
-        sn = &sn_tab[i];
-        available = 1;
-        bs1 = NULL;
-
-        while ((bs1 = bdrv_next(bs1))) {
-            if (bdrv_can_snapshot(bs1) && bs1 != bs) {
-                ret = bdrv_snapshot_find(bs1, sn_info, sn->id_str);
-                if (ret < 0) {
-                    available = 0;
-                    break;
-                }
-            }
-        }
-
-        if (available) {
+        if (bdrv_all_find_snapshot(sn_tab[i].id_str, &bs1) == 0) {
             available_snapshots[total] = i;
             total++;
         }
diff --git a/qemu/migration/tcp.c b/qemu/migration/tcp.c
index ae891728e..e1fa7f8f1 100644
--- a/qemu/migration/tcp.c
+++ b/qemu/migration/tcp.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qemu/error-report.h"
@@ -39,11 +39,11 @@ static void tcp_wait_for_connect(int fd, Error *err, void *opaque)
 
     if (fd < 0) {
         DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
-        s->file = NULL;
+        s->to_dst_file = NULL;
         migrate_fd_error(s);
     } else {
         DPRINTF("migrate connect success\n");
-        s->file = qemu_fopen_socket(fd, "wb");
+        s->to_dst_file = qemu_fopen_socket(fd, "wb");
         migrate_fd_connect(s);
     }
 }
@@ -59,12 +59,11 @@ static void tcp_accept_incoming_migration(void *opaque)
     socklen_t addrlen = sizeof(addr);
     int s = (intptr_t)opaque;
     QEMUFile *f;
-    int c, err;
+    int c;
 
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
-        err = socket_error();
-    } while (c < 0 && err == EINTR);
+    } while (c < 0 && errno == EINTR);
     qemu_set_fd_handler(s, NULL, NULL, NULL);
     closesocket(s);
 
@@ -72,7 +71,7 @@ static void tcp_accept_incoming_migration(void *opaque)
 
     if (c < 0) {
         error_report("could not accept migration connection (%s)",
-                     strerror(err));
+                     strerror(errno));
         return;
     }
 
diff --git a/qemu/migration/unix.c b/qemu/migration/unix.c
index b591813eb..d9aac36b9 100644
--- a/qemu/migration/unix.c
+++ b/qemu/migration/unix.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qemu/error-report.h"
@@ -39,11 +39,11 @@ static void unix_wait_for_connect(int fd, Error *err, void *opaque)
 
     if (fd < 0) {
         DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
-        s->file = NULL;
+        s->to_dst_file = NULL;
         migrate_fd_error(s);
     } else {
         DPRINTF("migrate connect success\n");
-        s->file = qemu_fopen_socket(fd, "wb");
+        s->to_dst_file = qemu_fopen_socket(fd, "wb");
         migrate_fd_connect(s);
     }
 }
diff --git a/qemu/migration/vmstate.c b/qemu/migration/vmstate.c
index e8ccf22f6..bf3d5db30 100644
--- a/qemu/migration/vmstate.c
+++ b/qemu/migration/vmstate.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/migration.h"
 #include "migration/qemu-file.h"
@@ -28,6 +29,10 @@ static int vmstate_n_elems(void *opaque, VMStateField *field)
         n_elems = *(uint8_t *)(opaque+field->num_offset);
     }
 
+    if (field->flags & VMS_MULTIPLY_ELEMENTS) {
+        n_elems *= field->num;
+    }
+
     return n_elems;
 }
 
@@ -794,6 +799,29 @@ const VMStateInfo vmstate_info_float64 = {
     .put  = put_float64,
 };
 
+/* CPU_DoubleU type */
+
+static int get_cpudouble(QEMUFile *f, void *pv, size_t size)
+{
+    CPU_DoubleU *v = pv;
+    qemu_get_be32s(f, &v->l.upper);
+    qemu_get_be32s(f, &v->l.lower);
+    return 0;
+}
+
+static void put_cpudouble(QEMUFile *f, void *pv, size_t size)
+{
+    CPU_DoubleU *v = pv;
+    qemu_put_be32s(f, &v->l.upper);
+    qemu_put_be32s(f, &v->l.lower);
+}
+
+const VMStateInfo vmstate_info_cpudouble = {
+    .name = "CPU_Double_U",
+    .get  = get_cpudouble,
+    .put  = put_cpudouble,
+};
+
 /* uint8_t buffers */
 
 static int get_buffer(QEMUFile *f, void *pv, size_t size)
diff --git a/qemu/migration/xbzrle.c b/qemu/migration/xbzrle.c
index 8e220bf25..c85833925 100644
--- a/qemu/migration/xbzrle.c
+++ b/qemu/migration/xbzrle.c
@@ -10,7 +10,8 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "include/migration/migration.h"
 
 /*
diff --git a/qemu/module-common.c b/qemu/module-common.c
index 50c67500b..0a3cb8aea 100644
--- a/qemu/module-common.c
+++ b/qemu/module-common.c
@@ -1,4 +1,4 @@
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu/module.h"
 
 void qemu_module_dummy(void)
diff --git a/qemu/monitor.c b/qemu/monitor.c
index aeea2b5f9..d1c193013 100644
--- a/qemu/monitor.c
+++ b/qemu/monitor.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <dirent.h>
 #include "hw/hw.h"
 #include "monitor/qdev.h"
@@ -41,6 +42,7 @@
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "audio/audio.h"
 #include "disas/disas.h"
 #include "sysemu/balloon.h"
@@ -59,22 +61,24 @@
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
 #include <qom/object_interfaces.h>
-#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "trace/control.h"
+#include "monitor/hmp-target.h"
 #ifdef CONFIG_TRACE_SIMPLE
 #include "trace/simple.h"
 #endif
 #include "exec/memory.h"
-#include "exec/cpu_ldst.h"
 #include "qmp-commands.h"
 #include "hmp.h"
 #include "qemu/thread.h"
 #include "block/qapi.h"
 #include "qapi/qmp-event.h"
 #include "qapi-event.h"
+#include "qmp-introspect.h"
 #include "sysemu/block-backend.h"
+#include "sysemu/qtest.h"
+#include "qemu/cutils.h"
 
 /* for hmp_info_irq/pic */
 #if defined(TARGET_SPARC)
@@ -82,6 +86,10 @@
 #endif
 #include "hw/lm32/lm32_pic.h"
 
+#if defined(TARGET_S390X)
+#include "hw/s390x/storage-keys.h"
+#endif
+
 /*
  * Supported types:
  *
@@ -176,13 +184,16 @@ typedef struct {
  * instance.
  */
 typedef struct MonitorQAPIEventState {
-    QAPIEvent event;    /* Event being tracked */
-    int64_t rate;       /* Minimum time (in ns) between two events */
-    int64_t last;       /* QEMU_CLOCK_REALTIME value at last emission */
+    QAPIEvent event;    /* Throttling state for this event type and... */
+    QDict *data;        /* ... data, see qapi_event_throttle_equal() */
     QEMUTimer *timer;   /* Timer for handling delayed events */
-    QObject *data;      /* Event pending delayed dispatch */
+    QDict *qdict;       /* Delayed event (if any) */
 } MonitorQAPIEventState;
 
+typedef struct {
+    int64_t rate;       /* Minimum time (in ns) between two events */
+} MonitorQAPIEventConf;
+
 struct Monitor {
     CharDriverState *chr;
     int reset_seen;
@@ -224,6 +235,8 @@ static const mon_cmd_t qmp_cmds[];
 
 Monitor *cur_mon;
 
+static QEMUClockType event_clock_type = QEMU_CLOCK_REALTIME;
+
 static void monitor_command_cb(void *opaque, const char *cmdline,
                                void *readline_opaque);
 
@@ -367,8 +380,7 @@ void monitor_printf(Monitor *mon, const char *fmt, ...)
     va_end(ap);
 }
 
-static int GCC_FMT_ATTR(2, 3) monitor_fprintf(FILE *stream,
-                                              const char *fmt, ...)
+int monitor_fprintf(FILE *stream, const char *fmt, ...)
 {
     va_list ap;
     va_start(ap, fmt);
@@ -396,7 +408,7 @@ static QDict *build_qmp_error_dict(Error *err)
     QObject *obj;
 
     obj = qobject_from_jsonf("{ 'error': { 'class': %s, 'desc': %s } }",
-                             ErrorClass_lookup[error_get_class(err)],
+                             QapiErrorClass_lookup[error_get_class(err)],
                              error_get_pretty(err));
 
     return qobject_to_qdict(obj);
@@ -434,132 +446,174 @@ static void monitor_protocol_emitter(Monitor *mon, QObject *data,
 }
 
 
-static MonitorQAPIEventState monitor_qapi_event_state[QAPI_EVENT_MAX];
+static MonitorQAPIEventConf monitor_qapi_event_conf[QAPI_EVENT__MAX] = {
+    /* Limit guest-triggerable events to 1 per second */
+    [QAPI_EVENT_RTC_CHANGE]        = { 1000 * SCALE_MS },
+    [QAPI_EVENT_WATCHDOG]          = { 1000 * SCALE_MS },
+    [QAPI_EVENT_BALLOON_CHANGE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_REPORT_BAD] = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_FAILURE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_VSERPORT_CHANGE]   = { 1000 * SCALE_MS },
+};
+
+GHashTable *monitor_qapi_event_state;
 
 /*
  * Emits the event to every monitor instance, @event is only used for trace
  * Called with monitor_lock held.
  */
-static void monitor_qapi_event_emit(QAPIEvent event, QObject *data)
+static void monitor_qapi_event_emit(QAPIEvent event, QDict *qdict)
 {
     Monitor *mon;
 
-    trace_monitor_protocol_event_emit(event, data);
+    trace_monitor_protocol_event_emit(event, qdict);
     QLIST_FOREACH(mon, &mon_list, entry) {
         if (monitor_is_qmp(mon) && mon->qmp.in_command_mode) {
-            monitor_json_emitter(mon, data);
+            monitor_json_emitter(mon, QOBJECT(qdict));
         }
     }
 }
 
+static void monitor_qapi_event_handler(void *opaque);
+
 /*
  * Queue a new event for emission to Monitor instances,
  * applying any rate limiting if required.
  */
 static void
-monitor_qapi_event_queue(QAPIEvent event, QDict *data, Error **errp)
+monitor_qapi_event_queue(QAPIEvent event, QDict *qdict, Error **errp)
 {
+    MonitorQAPIEventConf *evconf;
     MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 
-    evstate = &(monitor_qapi_event_state[event]);
-    trace_monitor_protocol_event_queue(event,
-                                       data,
-                                       evstate->rate,
-                                       evstate->last,
-                                       now);
+    assert(event < QAPI_EVENT__MAX);
+    evconf = &monitor_qapi_event_conf[event];
+    trace_monitor_protocol_event_queue(event, qdict, evconf->rate);
 
-    /* Rate limit of 0 indicates no throttling */
     qemu_mutex_lock(&monitor_lock);
-    if (!evstate->rate) {
-        monitor_qapi_event_emit(event, QOBJECT(data));
-        evstate->last = now;
+
+    if (!evconf->rate) {
+        /* Unthrottled event */
+        monitor_qapi_event_emit(event, qdict);
     } else {
-        int64_t delta = now - evstate->last;
-        if (evstate->data ||
-            delta < evstate->rate) {
-            /* If there's an existing event pending, replace
-             * it with the new event, otherwise schedule a
-             * timer for delayed emission
+        QDict *data = qobject_to_qdict(qdict_get(qdict, "data"));
+        MonitorQAPIEventState key = { .event = event, .data = data };
+
+        evstate = g_hash_table_lookup(monitor_qapi_event_state, &key);
+        assert(!evstate || timer_pending(evstate->timer));
+
+        if (evstate) {
+            /*
+             * Timer is pending for (at least) evconf->rate ns after
+             * last send.  Store event for sending when timer fires,
+             * replacing a prior stored event if any.
              */
-            if (evstate->data) {
-                qobject_decref(evstate->data);
-            } else {
-                int64_t then = evstate->last + evstate->rate;
-                timer_mod_ns(evstate->timer, then);
-            }
-            evstate->data = QOBJECT(data);
-            qobject_incref(evstate->data);
+            QDECREF(evstate->qdict);
+            evstate->qdict = qdict;
+            QINCREF(evstate->qdict);
         } else {
-            monitor_qapi_event_emit(event, QOBJECT(data));
-            evstate->last = now;
+            /*
+             * Last send was (at least) evconf->rate ns ago.
+             * Send immediately, and arm the timer to call
+             * monitor_qapi_event_handler() in evconf->rate ns.  Any
+             * events arriving before then will be delayed until then.
+             */
+            int64_t now = qemu_clock_get_ns(event_clock_type);
+
+            monitor_qapi_event_emit(event, qdict);
+
+            evstate = g_new(MonitorQAPIEventState, 1);
+            evstate->event = event;
+            evstate->data = data;
+            QINCREF(evstate->data);
+            evstate->qdict = NULL;
+            evstate->timer = timer_new_ns(event_clock_type,
+                                          monitor_qapi_event_handler,
+                                          evstate);
+            g_hash_table_add(monitor_qapi_event_state, evstate);
+            timer_mod_ns(evstate->timer, now + evconf->rate);
         }
     }
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
 /*
- * The callback invoked by QemuTimer when a delayed
- * event is ready to be emitted
+ * This function runs evconf->rate ns after sending a throttled
+ * event.
+ * If another event has since been stored, send it.
  */
 static void monitor_qapi_event_handler(void *opaque)
 {
     MonitorQAPIEventState *evstate = opaque;
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    MonitorQAPIEventConf *evconf = &monitor_qapi_event_conf[evstate->event];
 
-    trace_monitor_protocol_event_handler(evstate->event,
-                                         evstate->data,
-                                         evstate->last,
-                                         now);
+    trace_monitor_protocol_event_handler(evstate->event, evstate->qdict);
     qemu_mutex_lock(&monitor_lock);
-    if (evstate->data) {
-        monitor_qapi_event_emit(evstate->event, evstate->data);
-        qobject_decref(evstate->data);
-        evstate->data = NULL;
+
+    if (evstate->qdict) {
+        int64_t now = qemu_clock_get_ns(event_clock_type);
+
+        monitor_qapi_event_emit(evstate->event, evstate->qdict);
+        QDECREF(evstate->qdict);
+        evstate->qdict = NULL;
+        timer_mod_ns(evstate->timer, now + evconf->rate);
+    } else {
+        g_hash_table_remove(monitor_qapi_event_state, evstate);
+        QDECREF(evstate->data);
+        timer_free(evstate->timer);
+        g_free(evstate);
     }
-    evstate->last = now;
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
-/*
- * @event: the event ID to be limited
- * @rate: the rate limit in milliseconds
- *
- * Sets a rate limit on a particular event, so no
- * more than 1 event will be emitted within @rate
- * milliseconds
- */
-static void
-monitor_qapi_event_throttle(QAPIEvent event, int64_t rate)
+static unsigned int qapi_event_throttle_hash(const void *key)
 {
-    MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
+    const MonitorQAPIEventState *evstate = key;
+    unsigned int hash = evstate->event * 255;
 
-    evstate = &(monitor_qapi_event_state[event]);
+    if (evstate->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        hash += g_str_hash(qdict_get_str(evstate->data, "id"));
+    }
 
-    trace_monitor_protocol_event_throttle(event, rate);
-    evstate->event = event;
-    assert(rate * SCALE_MS <= INT64_MAX);
-    evstate->rate = rate * SCALE_MS;
-    evstate->last = 0;
-    evstate->data = NULL;
-    evstate->timer = timer_new(QEMU_CLOCK_REALTIME,
-                               SCALE_MS,
-                               monitor_qapi_event_handler,
-                               evstate);
+    if (evstate->event == QAPI_EVENT_QUORUM_REPORT_BAD) {
+        hash += g_str_hash(qdict_get_str(evstate->data, "node-name"));
+    }
+
+    return hash;
+}
+
+static gboolean qapi_event_throttle_equal(const void *a, const void *b)
+{
+    const MonitorQAPIEventState *eva = a;
+    const MonitorQAPIEventState *evb = b;
+
+    if (eva->event != evb->event) {
+        return FALSE;
+    }
+
+    if (eva->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        return !strcmp(qdict_get_str(eva->data, "id"),
+                       qdict_get_str(evb->data, "id"));
+    }
+
+    if (eva->event == QAPI_EVENT_QUORUM_REPORT_BAD) {
+        return !strcmp(qdict_get_str(eva->data, "node-name"),
+                       qdict_get_str(evb->data, "node-name"));
+    }
+
+    return TRUE;
 }
 
 static void monitor_qapi_event_init(void)
 {
-    /* Limit guest-triggerable events to 1 per second */
-    monitor_qapi_event_throttle(QAPI_EVENT_RTC_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_WATCHDOG, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_BALLOON_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_REPORT_BAD, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_FAILURE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_VSERPORT_CHANGE, 1000);
+    if (qtest_enabled()) {
+        event_clock_type = QEMU_CLOCK_VIRTUAL;
+    }
 
+    monitor_qapi_event_state = g_hash_table_new(qapi_event_throttle_hash,
+                                                qapi_event_throttle_equal);
     qmp_event_set_func_emit(monitor_qapi_event_queue);
 }
 
@@ -678,7 +732,7 @@ static int get_str(char *buf, int buf_size, const char **pp)
                 case '\"':
                     break;
                 default:
-                    qemu_printf("unsupported escape code: '\\%c'\n", c);
+                    printf("unsupported escape code: '\\%c'\n", c);
                     goto fail;
                 }
                 if ((q - buf) < buf_size - 1) {
@@ -692,7 +746,7 @@ static int get_str(char *buf, int buf_size, const char **pp)
             }
         }
         if (*p != '\"') {
-            qemu_printf("unterminated string\n");
+            printf("unterminated string\n");
             goto fail;
         }
         p++;
@@ -910,7 +964,7 @@ EventInfoList *qmp_query_events(Error **errp)
     EventInfoList *info, *ev_list = NULL;
     QAPIEvent e;
 
-    for (e = 0 ; e < QAPI_EVENT_MAX ; e++) {
+    for (e = 0 ; e < QAPI_EVENT__MAX ; e++) {
         const char *event_name = QAPIEvent_lookup[e];
         assert(event_name != NULL);
         info = g_malloc0(sizeof(*info));
@@ -924,6 +978,21 @@ EventInfoList *qmp_query_events(Error **errp)
     return ev_list;
 }
 
+/*
+ * Minor hack: generated marshalling suppressed for this command
+ * ('gen': false in the schema) so we can parse the JSON string
+ * directly into QObject instead of first parsing it with
+ * visit_type_SchemaInfoList() into a SchemaInfoList, then marshal it
+ * to QObject with generated output marshallers, every time.  Instead,
+ * we do it in test-qmp-input-visitor.c, just to make sure
+ * qapi-introspect.py's output actually conforms to the schema.
+ */
+static void qmp_query_qmp_schema(QDict *qdict, QObject **ret_data,
+                                 Error **errp)
+{
+    *ret_data = qobject_from_json(qmp_schema_json);
+}
+
 /* set the current CPU defined by the user */
 int monitor_set_cpu(int cpu_index)
 {
@@ -937,7 +1006,7 @@ int monitor_set_cpu(int cpu_index)
     return 0;
 }
 
-static CPUState *mon_get_cpu(void)
+CPUState *mon_get_cpu(void)
 {
     if (!cur_mon->mon_cpu) {
         monitor_set_cpu(0);
@@ -946,7 +1015,7 @@ static CPUState *mon_get_cpu(void)
     return cur_mon->mon_cpu;
 }
 
-static CPUArchState *mon_get_cpu_env(void)
+CPUArchState *mon_get_cpu_env(void)
 {
     return mon_get_cpu()->env_ptr;
 }
@@ -1335,7 +1404,7 @@ static void hmp_mouse_move(Monitor *mon, const QDict *qdict)
 
 static void hmp_mouse_button(Monitor *mon, const QDict *qdict)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]      = MOUSE_EVENT_RBUTTON,
@@ -1413,449 +1482,12 @@ static void hmp_boot_set(Monitor *mon, const QDict *qdict)
 
     qemu_boot_set(bootdevice, &local_err);
     if (local_err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
     } else {
         monitor_printf(mon, "boot device list now set to %s\n", bootdevice);
     }
 }
 
-#if defined(TARGET_I386)
-static void print_pte(Monitor *mon, hwaddr addr,
-                      hwaddr pte,
-                      hwaddr mask)
-{
-#ifdef TARGET_X86_64
-    if (addr & (1ULL << 47)) {
-        addr |= -1LL << 48;
-    }
-#endif
-    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
-                   " %c%c%c%c%c%c%c%c%c\n",
-                   addr,
-                   pte & mask,
-                   pte & PG_NX_MASK ? 'X' : '-',
-                   pte & PG_GLOBAL_MASK ? 'G' : '-',
-                   pte & PG_PSE_MASK ? 'P' : '-',
-                   pte & PG_DIRTY_MASK ? 'D' : '-',
-                   pte & PG_ACCESSED_MASK ? 'A' : '-',
-                   pte & PG_PCD_MASK ? 'C' : '-',
-                   pte & PG_PWT_MASK ? 'T' : '-',
-                   pte & PG_USER_MASK ? 'U' : '-',
-                   pte & PG_RW_MASK ? 'W' : '-');
-}
-
-static void tlb_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    uint32_t pgd, pde, pte;
-
-    pgd = env->cr[3] & ~0xfff;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                /* 4M pages */
-                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    if (pte & PG_PRESENT_MASK) {
-                        print_pte(mon, (l1 << 22) + (l2 << 12),
-                                  pte & ~PG_PSE_MASK,
-                                  ~0xfff);
-                    }
-                }
-            }
-        }
-    }
-}
-
-static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        /* 2M pages with PAE, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
-                                  ~((hwaddr)(1 << 20) - 1));
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            if (pte & PG_PRESENT_MASK) {
-                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
-                                          + (l3 << 12),
-                                          pte & ~PG_PSE_MASK,
-                                          ~(hwaddr)0xfff);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-#ifdef TARGET_X86_64
-static void tlb_info_64(Monitor *mon, CPUArchState *env)
-{
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        /* 1G pages, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
-                                  0x3ffffc0000000ULL);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    /* 2M pages, CR4.PSE is ignored */
-                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
-                                              (l3 << 21), pde,
-                                              0x3ffffffe00000ULL);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            print_pte(mon, (l1 << 39) +
-                                                      (l2 << 30) +
-                                                      (l3 << 21) + (l4 << 12),
-                                                      pte & ~PG_PSE_MASK,
-                                                      0x3fffffffff000ULL);
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-#endif
-
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu_env();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            tlb_info_64(mon, env);
-        } else
-#endif
-        {
-            tlb_info_pae32(mon, env);
-        }
-    } else {
-        tlb_info_32(mon, env);
-    }
-}
-
-static void mem_print(Monitor *mon, hwaddr *pstart,
-                      int *plast_prot,
-                      hwaddr end, int prot)
-{
-    int prot1;
-    prot1 = *plast_prot;
-    if (prot != prot1) {
-        if (*pstart != -1) {
-            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
-                           TARGET_FMT_plx " %c%c%c\n",
-                           *pstart, end, end - *pstart,
-                           prot1 & PG_USER_MASK ? 'u' : '-',
-                           'r',
-                           prot1 & PG_RW_MASK ? 'w' : '-');
-        }
-        if (prot != 0)
-            *pstart = end;
-        else
-            *pstart = -1;
-        *plast_prot = prot;
-    }
-}
-
-static void mem_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    int prot, last_prot;
-    uint32_t pgd, pde, pte;
-    hwaddr start, end;
-
-    pgd = env->cr[3] & ~0xfff;
-    last_prot = 0;
-    start = -1;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        end = l1 << 22;
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                mem_print(mon, &start, &last_prot, end, prot);
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    end = (l1 << 22) + (l2 << 12);
-                    if (pte & PG_PRESENT_MASK) {
-                        prot = pte & pde &
-                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                    } else {
-                        prot = 0;
-                    }
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-static void mem_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    int prot, last_prot;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-    hwaddr start, end;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        end = l1 << 30;
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                end = (l1 << 30) + (l2 << 21);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                      PG_PRESENT_MASK);
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
-                            if (pte & PG_PRESENT_MASK) {
-                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
-                                                    PG_PRESENT_MASK);
-                            } else {
-                                prot = 0;
-                            }
-                            mem_print(mon, &start, &last_prot, end, prot);
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-
-#ifdef TARGET_X86_64
-static void mem_info_64(Monitor *mon, CPUArchState *env)
-{
-    int prot, last_prot;
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        end = l1 << 39;
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                end = (l1 << 39) + (l2 << 30);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
-                                       PG_PRESENT_MASK);
-                        prot &= pml4e;
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                                  PG_PRESENT_MASK);
-                                    prot &= pml4e & pdpe;
-                                    mem_print(mon, &start, &last_prot, end, prot);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        end = (l1 << 39) + (l2 << 30) +
-                                            (l3 << 21) + (l4 << 12);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
-                                                          PG_PRESENT_MASK);
-                                            prot &= pml4e & pdpe & pde;
-                                        } else {
-                                            prot = 0;
-                                        }
-                                        mem_print(mon, &start, &last_prot, end, prot);
-                                    }
-                                }
-                            } else {
-                                prot = 0;
-                                mem_print(mon, &start, &last_prot, end, prot);
-                            }
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
-}
-#endif
-
-static void hmp_info_mem(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu_env();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            mem_info_64(mon, env);
-        } else
-#endif
-        {
-            mem_info_pae32(mon, env);
-        }
-    } else {
-        mem_info_32(mon, env);
-    }
-}
-#endif
-
-#if defined(TARGET_SH4)
-
-static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
-{
-    monitor_printf(mon, " tlb%i:\t"
-                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
-                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
-                   "dirty=%hhu writethrough=%hhu\n",
-                   idx,
-                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
-                   tlb->v, tlb->sh, tlb->c, tlb->pr,
-                   tlb->d, tlb->wt);
-}
-
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    int i;
-
-    monitor_printf (mon, "ITLB:\n");
-    for (i = 0 ; i < ITLB_SIZE ; i++)
-        print_tlb (mon, i, &env->itlb[i]);
-    monitor_printf (mon, "UTLB:\n");
-    for (i = 0 ; i < UTLB_SIZE ; i++)
-        print_tlb (mon, i, &env->utlb[i]);
-}
-
-#endif
-
-#if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA)
-static void hmp_info_tlb(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env1 = mon_get_cpu_env();
-
-    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
-}
-#endif
-
 static void hmp_info_mtree(Monitor *mon, const QDict *qdict)
 {
     mtree_info((fprintf_function)monitor_printf, mon);
@@ -1892,9 +1524,9 @@ int64_t dev_time;
 static void hmp_info_profile(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "async time  %" PRId64 " (%0.3f)\n",
-                   dev_time, dev_time / (double)get_ticks_per_sec());
+                   dev_time, dev_time / (double)NANOSECONDS_PER_SECOND);
     monitor_printf(mon, "qemu time   %" PRId64 " (%0.3f)\n",
-                   tcg_time, tcg_time / (double)get_ticks_per_sec());
+                   tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND);
     tcg_time = 0;
     dev_time = 0;
 }
@@ -2066,31 +1698,6 @@ static void hmp_acl_remove(Monitor *mon, const QDict *qdict)
     }
 }
 
-#if defined(TARGET_I386)
-static void hmp_mce(Monitor *mon, const QDict *qdict)
-{
-    X86CPU *cpu;
-    CPUState *cs;
-    int cpu_index = qdict_get_int(qdict, "cpu_index");
-    int bank = qdict_get_int(qdict, "bank");
-    uint64_t status = qdict_get_int(qdict, "status");
-    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
-    uint64_t addr = qdict_get_int(qdict, "addr");
-    uint64_t misc = qdict_get_int(qdict, "misc");
-    int flags = MCE_INJECT_UNCOND_AO;
-
-    if (qdict_get_try_bool(qdict, "broadcast", false)) {
-        flags |= MCE_INJECT_BROADCAST;
-    }
-    cs = qemu_get_cpu(cpu_index);
-    if (cs != NULL) {
-        cpu = X86_CPU(cs);
-        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
-                           flags);
-    }
-}
-#endif
-
 void qmp_getfd(const char *fdname, Error **errp)
 {
     mon_fd_t *monfd;
@@ -2509,377 +2116,8 @@ int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp)
 
 /* Please update hmp-commands.hx when adding or changing commands */
 static mon_cmd_t info_cmds[] = {
-    {
-        .name       = "version",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the version of QEMU",
-        .mhandler.cmd = hmp_info_version,
-    },
-    {
-        .name       = "network",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the network state",
-        .mhandler.cmd = hmp_info_network,
-    },
-    {
-        .name       = "chardev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the character devices",
-        .mhandler.cmd = hmp_info_chardev,
-    },
-    {
-        .name       = "block",
-        .args_type  = "nodes:-n,verbose:-v,device:B?",
-        .params     = "[-n] [-v] [device]",
-        .help       = "show info of one block device or all block devices "
-                      "(-n: show named nodes; -v: show details)",
-        .mhandler.cmd = hmp_info_block,
-    },
-    {
-        .name       = "blockstats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show block device statistics",
-        .mhandler.cmd = hmp_info_blockstats,
-    },
-    {
-        .name       = "block-jobs",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show progress of ongoing block device operations",
-        .mhandler.cmd = hmp_info_block_jobs,
-    },
-    {
-        .name       = "registers",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the cpu registers",
-        .mhandler.cmd = hmp_info_registers,
-    },
-    {
-        .name       = "cpus",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show infos for each CPU",
-        .mhandler.cmd = hmp_info_cpus,
-    },
-    {
-        .name       = "history",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the command line history",
-        .mhandler.cmd = hmp_info_history,
-    },
-#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
-    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
-    {
-        .name       = "irq",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the interrupts statistics (if available)",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_hmp_info_irq,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_hmp_info_irq,
-#else
-        .mhandler.cmd = hmp_info_irq,
-#endif
-    },
-    {
-        .name       = "pic",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show i8259 (PIC) state",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_hmp_info_pic,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_hmp_info_pic,
-#else
-        .mhandler.cmd = hmp_info_pic,
-#endif
-    },
-#endif
-    {
-        .name       = "pci",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show PCI info",
-        .mhandler.cmd = hmp_info_pci,
-    },
-#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
-    defined(TARGET_PPC) || defined(TARGET_XTENSA)
-    {
-        .name       = "tlb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show virtual to physical memory mappings",
-        .mhandler.cmd = hmp_info_tlb,
-    },
-#endif
-#if defined(TARGET_I386)
-    {
-        .name       = "mem",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the active virtual memory mappings",
-        .mhandler.cmd = hmp_info_mem,
-    },
-#endif
-    {
-        .name       = "mtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory tree",
-        .mhandler.cmd = hmp_info_mtree,
-    },
-    {
-        .name       = "jit",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show dynamic compiler info",
-        .mhandler.cmd = hmp_info_jit,
-    },
-    {
-        .name       = "opcount",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show dynamic compiler opcode counters",
-        .mhandler.cmd = hmp_info_opcount,
-    },
-    {
-        .name       = "kvm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show KVM information",
-        .mhandler.cmd = hmp_info_kvm,
-    },
-    {
-        .name       = "numa",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show NUMA information",
-        .mhandler.cmd = hmp_info_numa,
-    },
-    {
-        .name       = "usb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show guest USB devices",
-        .mhandler.cmd = hmp_info_usb,
-    },
-    {
-        .name       = "usbhost",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show host USB devices",
-        .mhandler.cmd = hmp_info_usbhost,
-    },
-    {
-        .name       = "profile",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show profiling information",
-        .mhandler.cmd = hmp_info_profile,
-    },
-    {
-        .name       = "capture",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show capture information",
-        .mhandler.cmd = hmp_info_capture,
-    },
-    {
-        .name       = "snapshots",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the currently saved VM snapshots",
-        .mhandler.cmd = hmp_info_snapshots,
-    },
-    {
-        .name       = "status",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM status (running|paused)",
-        .mhandler.cmd = hmp_info_status,
-    },
-    {
-        .name       = "mice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show which guest mouse is receiving events",
-        .mhandler.cmd = hmp_info_mice,
-    },
-    {
-        .name       = "vnc",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the vnc server status",
-        .mhandler.cmd = hmp_info_vnc,
-    },
-#if defined(CONFIG_SPICE)
-    {
-        .name       = "spice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the spice server status",
-        .mhandler.cmd = hmp_info_spice,
-    },
-#endif
-    {
-        .name       = "name",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM name",
-        .mhandler.cmd = hmp_info_name,
-    },
-    {
-        .name       = "uuid",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM UUID",
-        .mhandler.cmd = hmp_info_uuid,
-    },
-    {
-        .name       = "cpustats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show CPU statistics",
-        .mhandler.cmd = hmp_info_cpustats,
-    },
-#if defined(CONFIG_SLIRP)
-    {
-        .name       = "usernet",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show user network stack connection states",
-        .mhandler.cmd = hmp_info_usernet,
-    },
-#endif
-    {
-        .name       = "migrate",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show migration status",
-        .mhandler.cmd = hmp_info_migrate,
-    },
-    {
-        .name       = "migrate_capabilities",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration capabilities",
-        .mhandler.cmd = hmp_info_migrate_capabilities,
-    },
-    {
-        .name       = "migrate_parameters",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration parameters",
-        .mhandler.cmd = hmp_info_migrate_parameters,
-    },
-    {
-        .name       = "migrate_cache_size",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration xbzrle cache size",
-        .mhandler.cmd = hmp_info_migrate_cache_size,
-    },
-    {
-        .name       = "balloon",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show balloon information",
-        .mhandler.cmd = hmp_info_balloon,
-    },
-    {
-        .name       = "qtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show device tree",
-        .mhandler.cmd = hmp_info_qtree,
-    },
-    {
-        .name       = "qdm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show qdev device model list",
-        .mhandler.cmd = hmp_info_qdm,
-    },
-    {
-        .name       = "qom-tree",
-        .args_type  = "path:s?",
-        .params     = "[path]",
-        .help       = "show QOM composition tree",
-        .mhandler.cmd = hmp_info_qom_tree,
-    },
-    {
-        .name       = "roms",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show roms",
-        .mhandler.cmd = hmp_info_roms,
-    },
-    {
-        .name       = "trace-events",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show available trace-events & their state",
-        .mhandler.cmd = hmp_info_trace_events,
-    },
-    {
-        .name       = "tpm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the TPM device",
-        .mhandler.cmd = hmp_info_tpm,
-    },
-    {
-        .name       = "memdev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory backends",
-        .mhandler.cmd = hmp_info_memdev,
-    },
-    {
-        .name       = "memory-devices",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory devices",
-        .mhandler.cmd = hmp_info_memory_devices,
-    },
-    {
-        .name       = "rocker",
-        .args_type  = "name:s",
-        .params     = "name",
-        .help       = "Show rocker switch",
-        .mhandler.cmd = hmp_rocker,
-    },
-    {
-        .name       = "rocker-ports",
-        .args_type  = "name:s",
-        .params     = "name",
-        .help       = "Show rocker ports",
-        .mhandler.cmd = hmp_rocker_ports,
-    },
-    {
-        .name       = "rocker-of-dpa-flows",
-        .args_type  = "name:s,tbl_id:i?",
-        .params     = "name [tbl_id]",
-        .help       = "Show rocker OF-DPA flow tables",
-        .mhandler.cmd = hmp_rocker_of_dpa_flows,
-    },
-    {
-        .name       = "rocker-of-dpa-groups",
-        .args_type  = "name:s,type:i?",
-        .params     = "name [type]",
-        .help       = "Show rocker OF-DPA groups",
-        .mhandler.cmd = hmp_rocker_of_dpa_groups,
-    },
-    {
-        .name       = NULL,
-    },
+#include "hmp-commands-info.h"
+    { NULL, NULL, },
 };
 
 /* mon_cmds and info_cmds would be sorted at runtime */
@@ -2898,394 +2136,6 @@ static const mon_cmd_t qmp_cmds[] = {
 static const char *pch;
 static sigjmp_buf expr_env;
 
-#define MD_TLONG 0
-#define MD_I32   1
-
-typedef struct MonitorDef {
-    const char *name;
-    int offset;
-    target_long (*get_value)(const struct MonitorDef *md, int val);
-    int type;
-} MonitorDef;
-
-#if defined(TARGET_I386)
-static target_long monitor_get_pc (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->eip + env->segs[R_CS].base;
-}
-#endif
-
-#if defined(TARGET_PPC)
-static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    unsigned int u;
-    int i;
-
-    u = 0;
-    for (i = 0; i < 8; i++)
-        u |= env->crf[i] << (32 - (4 * (i + 1)));
-
-    return u;
-}
-
-static target_long monitor_get_msr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->msr;
-}
-
-static target_long monitor_get_xer (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->xer;
-}
-
-static target_long monitor_get_decr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_decr(env);
-}
-
-static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_tbu(env);
-}
-
-static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return cpu_ppc_load_tbl(env);
-}
-#endif
-
-#if defined(TARGET_SPARC)
-#ifndef TARGET_SPARC64
-static target_long monitor_get_psr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-
-    return cpu_get_psr(env);
-}
-#endif
-
-static target_long monitor_get_reg(const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu_env();
-    return env->regwptr[val];
-}
-#endif
-
-static const MonitorDef monitor_defs[] = {
-#ifdef TARGET_I386
-
-#define SEG(name, seg) \
-    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
-    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
-    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
-
-    { "eax", offsetof(CPUX86State, regs[0]) },
-    { "ecx", offsetof(CPUX86State, regs[1]) },
-    { "edx", offsetof(CPUX86State, regs[2]) },
-    { "ebx", offsetof(CPUX86State, regs[3]) },
-    { "esp|sp", offsetof(CPUX86State, regs[4]) },
-    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
-    { "esi", offsetof(CPUX86State, regs[6]) },
-    { "edi", offsetof(CPUX86State, regs[7]) },
-#ifdef TARGET_X86_64
-    { "r8", offsetof(CPUX86State, regs[8]) },
-    { "r9", offsetof(CPUX86State, regs[9]) },
-    { "r10", offsetof(CPUX86State, regs[10]) },
-    { "r11", offsetof(CPUX86State, regs[11]) },
-    { "r12", offsetof(CPUX86State, regs[12]) },
-    { "r13", offsetof(CPUX86State, regs[13]) },
-    { "r14", offsetof(CPUX86State, regs[14]) },
-    { "r15", offsetof(CPUX86State, regs[15]) },
-#endif
-    { "eflags", offsetof(CPUX86State, eflags) },
-    { "eip", offsetof(CPUX86State, eip) },
-    SEG("cs", R_CS)
-    SEG("ds", R_DS)
-    SEG("es", R_ES)
-    SEG("ss", R_SS)
-    SEG("fs", R_FS)
-    SEG("gs", R_GS)
-    { "pc", 0, monitor_get_pc, },
-#elif defined(TARGET_PPC)
-    /* General purpose registers */
-    { "r0", offsetof(CPUPPCState, gpr[0]) },
-    { "r1", offsetof(CPUPPCState, gpr[1]) },
-    { "r2", offsetof(CPUPPCState, gpr[2]) },
-    { "r3", offsetof(CPUPPCState, gpr[3]) },
-    { "r4", offsetof(CPUPPCState, gpr[4]) },
-    { "r5", offsetof(CPUPPCState, gpr[5]) },
-    { "r6", offsetof(CPUPPCState, gpr[6]) },
-    { "r7", offsetof(CPUPPCState, gpr[7]) },
-    { "r8", offsetof(CPUPPCState, gpr[8]) },
-    { "r9", offsetof(CPUPPCState, gpr[9]) },
-    { "r10", offsetof(CPUPPCState, gpr[10]) },
-    { "r11", offsetof(CPUPPCState, gpr[11]) },
-    { "r12", offsetof(CPUPPCState, gpr[12]) },
-    { "r13", offsetof(CPUPPCState, gpr[13]) },
-    { "r14", offsetof(CPUPPCState, gpr[14]) },
-    { "r15", offsetof(CPUPPCState, gpr[15]) },
-    { "r16", offsetof(CPUPPCState, gpr[16]) },
-    { "r17", offsetof(CPUPPCState, gpr[17]) },
-    { "r18", offsetof(CPUPPCState, gpr[18]) },
-    { "r19", offsetof(CPUPPCState, gpr[19]) },
-    { "r20", offsetof(CPUPPCState, gpr[20]) },
-    { "r21", offsetof(CPUPPCState, gpr[21]) },
-    { "r22", offsetof(CPUPPCState, gpr[22]) },
-    { "r23", offsetof(CPUPPCState, gpr[23]) },
-    { "r24", offsetof(CPUPPCState, gpr[24]) },
-    { "r25", offsetof(CPUPPCState, gpr[25]) },
-    { "r26", offsetof(CPUPPCState, gpr[26]) },
-    { "r27", offsetof(CPUPPCState, gpr[27]) },
-    { "r28", offsetof(CPUPPCState, gpr[28]) },
-    { "r29", offsetof(CPUPPCState, gpr[29]) },
-    { "r30", offsetof(CPUPPCState, gpr[30]) },
-    { "r31", offsetof(CPUPPCState, gpr[31]) },
-    /* Floating point registers */
-    { "f0", offsetof(CPUPPCState, fpr[0]) },
-    { "f1", offsetof(CPUPPCState, fpr[1]) },
-    { "f2", offsetof(CPUPPCState, fpr[2]) },
-    { "f3", offsetof(CPUPPCState, fpr[3]) },
-    { "f4", offsetof(CPUPPCState, fpr[4]) },
-    { "f5", offsetof(CPUPPCState, fpr[5]) },
-    { "f6", offsetof(CPUPPCState, fpr[6]) },
-    { "f7", offsetof(CPUPPCState, fpr[7]) },
-    { "f8", offsetof(CPUPPCState, fpr[8]) },
-    { "f9", offsetof(CPUPPCState, fpr[9]) },
-    { "f10", offsetof(CPUPPCState, fpr[10]) },
-    { "f11", offsetof(CPUPPCState, fpr[11]) },
-    { "f12", offsetof(CPUPPCState, fpr[12]) },
-    { "f13", offsetof(CPUPPCState, fpr[13]) },
-    { "f14", offsetof(CPUPPCState, fpr[14]) },
-    { "f15", offsetof(CPUPPCState, fpr[15]) },
-    { "f16", offsetof(CPUPPCState, fpr[16]) },
-    { "f17", offsetof(CPUPPCState, fpr[17]) },
-    { "f18", offsetof(CPUPPCState, fpr[18]) },
-    { "f19", offsetof(CPUPPCState, fpr[19]) },
-    { "f20", offsetof(CPUPPCState, fpr[20]) },
-    { "f21", offsetof(CPUPPCState, fpr[21]) },
-    { "f22", offsetof(CPUPPCState, fpr[22]) },
-    { "f23", offsetof(CPUPPCState, fpr[23]) },
-    { "f24", offsetof(CPUPPCState, fpr[24]) },
-    { "f25", offsetof(CPUPPCState, fpr[25]) },
-    { "f26", offsetof(CPUPPCState, fpr[26]) },
-    { "f27", offsetof(CPUPPCState, fpr[27]) },
-    { "f28", offsetof(CPUPPCState, fpr[28]) },
-    { "f29", offsetof(CPUPPCState, fpr[29]) },
-    { "f30", offsetof(CPUPPCState, fpr[30]) },
-    { "f31", offsetof(CPUPPCState, fpr[31]) },
-    { "fpscr", offsetof(CPUPPCState, fpscr) },
-    /* Next instruction pointer */
-    { "nip|pc", offsetof(CPUPPCState, nip) },
-    { "lr", offsetof(CPUPPCState, lr) },
-    { "ctr", offsetof(CPUPPCState, ctr) },
-    { "decr", 0, &monitor_get_decr, },
-    { "ccr", 0, &monitor_get_ccr, },
-    /* Machine state register */
-    { "msr", 0, &monitor_get_msr, },
-    { "xer", 0, &monitor_get_xer, },
-    { "tbu", 0, &monitor_get_tbu, },
-    { "tbl", 0, &monitor_get_tbl, },
-    /* Segment registers */
-    { "sdr1", offsetof(CPUPPCState, spr[SPR_SDR1]) },
-    { "sr0", offsetof(CPUPPCState, sr[0]) },
-    { "sr1", offsetof(CPUPPCState, sr[1]) },
-    { "sr2", offsetof(CPUPPCState, sr[2]) },
-    { "sr3", offsetof(CPUPPCState, sr[3]) },
-    { "sr4", offsetof(CPUPPCState, sr[4]) },
-    { "sr5", offsetof(CPUPPCState, sr[5]) },
-    { "sr6", offsetof(CPUPPCState, sr[6]) },
-    { "sr7", offsetof(CPUPPCState, sr[7]) },
-    { "sr8", offsetof(CPUPPCState, sr[8]) },
-    { "sr9", offsetof(CPUPPCState, sr[9]) },
-    { "sr10", offsetof(CPUPPCState, sr[10]) },
-    { "sr11", offsetof(CPUPPCState, sr[11]) },
-    { "sr12", offsetof(CPUPPCState, sr[12]) },
-    { "sr13", offsetof(CPUPPCState, sr[13]) },
-    { "sr14", offsetof(CPUPPCState, sr[14]) },
-    { "sr15", offsetof(CPUPPCState, sr[15]) },
-    /* Too lazy to put BATs... */
-    { "pvr", offsetof(CPUPPCState, spr[SPR_PVR]) },
-
-    { "srr0", offsetof(CPUPPCState, spr[SPR_SRR0]) },
-    { "srr1", offsetof(CPUPPCState, spr[SPR_SRR1]) },
-    { "dar", offsetof(CPUPPCState, spr[SPR_DAR]) },
-    { "dsisr", offsetof(CPUPPCState, spr[SPR_DSISR]) },
-    { "cfar", offsetof(CPUPPCState, spr[SPR_CFAR]) },
-    { "sprg0", offsetof(CPUPPCState, spr[SPR_SPRG0]) },
-    { "sprg1", offsetof(CPUPPCState, spr[SPR_SPRG1]) },
-    { "sprg2", offsetof(CPUPPCState, spr[SPR_SPRG2]) },
-    { "sprg3", offsetof(CPUPPCState, spr[SPR_SPRG3]) },
-    { "sprg4", offsetof(CPUPPCState, spr[SPR_SPRG4]) },
-    { "sprg5", offsetof(CPUPPCState, spr[SPR_SPRG5]) },
-    { "sprg6", offsetof(CPUPPCState, spr[SPR_SPRG6]) },
-    { "sprg7", offsetof(CPUPPCState, spr[SPR_SPRG7]) },
-    { "pid", offsetof(CPUPPCState, spr[SPR_BOOKE_PID]) },
-    { "csrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR0]) },
-    { "csrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR1]) },
-    { "esr", offsetof(CPUPPCState, spr[SPR_BOOKE_ESR]) },
-    { "dear", offsetof(CPUPPCState, spr[SPR_BOOKE_DEAR]) },
-    { "mcsr", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSR]) },
-    { "tsr", offsetof(CPUPPCState, spr[SPR_BOOKE_TSR]) },
-    { "tcr", offsetof(CPUPPCState, spr[SPR_BOOKE_TCR]) },
-    { "vrsave", offsetof(CPUPPCState, spr[SPR_VRSAVE]) },
-    { "pir", offsetof(CPUPPCState, spr[SPR_BOOKE_PIR]) },
-    { "mcsrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR0]) },
-    { "mcsrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR1]) },
-    { "decar", offsetof(CPUPPCState, spr[SPR_BOOKE_DECAR]) },
-    { "ivpr", offsetof(CPUPPCState, spr[SPR_BOOKE_IVPR]) },
-    { "epcr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPCR]) },
-    { "sprg8", offsetof(CPUPPCState, spr[SPR_BOOKE_SPRG8]) },
-    { "ivor0", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR0]) },
-    { "ivor1", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR1]) },
-    { "ivor2", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR2]) },
-    { "ivor3", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR3]) },
-    { "ivor4", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR4]) },
-    { "ivor5", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR5]) },
-    { "ivor6", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR6]) },
-    { "ivor7", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR7]) },
-    { "ivor8", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR8]) },
-    { "ivor9", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR9]) },
-    { "ivor10", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR10]) },
-    { "ivor11", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR11]) },
-    { "ivor12", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR12]) },
-    { "ivor13", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR13]) },
-    { "ivor14", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR14]) },
-    { "ivor15", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR15]) },
-    { "ivor32", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR32]) },
-    { "ivor33", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR33]) },
-    { "ivor34", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR34]) },
-    { "ivor35", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR35]) },
-    { "ivor36", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR36]) },
-    { "ivor37", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR37]) },
-    { "mas0", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS0]) },
-    { "mas1", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS1]) },
-    { "mas2", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS2]) },
-    { "mas3", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS3]) },
-    { "mas4", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS4]) },
-    { "mas6", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS6]) },
-    { "mas7", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS7]) },
-    { "mmucfg", offsetof(CPUPPCState, spr[SPR_MMUCFG]) },
-    { "tlb0cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB0CFG]) },
-    { "tlb1cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB1CFG]) },
-    { "epr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPR]) },
-    { "eplc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPLC]) },
-    { "epsc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPSC]) },
-    { "svr", offsetof(CPUPPCState, spr[SPR_E500_SVR]) },
-    { "mcar", offsetof(CPUPPCState, spr[SPR_Exxx_MCAR]) },
-    { "pid1", offsetof(CPUPPCState, spr[SPR_BOOKE_PID1]) },
-    { "pid2", offsetof(CPUPPCState, spr[SPR_BOOKE_PID2]) },
-    { "hid0", offsetof(CPUPPCState, spr[SPR_HID0]) },
-
-#elif defined(TARGET_SPARC)
-    { "g0", offsetof(CPUSPARCState, gregs[0]) },
-    { "g1", offsetof(CPUSPARCState, gregs[1]) },
-    { "g2", offsetof(CPUSPARCState, gregs[2]) },
-    { "g3", offsetof(CPUSPARCState, gregs[3]) },
-    { "g4", offsetof(CPUSPARCState, gregs[4]) },
-    { "g5", offsetof(CPUSPARCState, gregs[5]) },
-    { "g6", offsetof(CPUSPARCState, gregs[6]) },
-    { "g7", offsetof(CPUSPARCState, gregs[7]) },
-    { "o0", 0, monitor_get_reg },
-    { "o1", 1, monitor_get_reg },
-    { "o2", 2, monitor_get_reg },
-    { "o3", 3, monitor_get_reg },
-    { "o4", 4, monitor_get_reg },
-    { "o5", 5, monitor_get_reg },
-    { "o6", 6, monitor_get_reg },
-    { "o7", 7, monitor_get_reg },
-    { "l0", 8, monitor_get_reg },
-    { "l1", 9, monitor_get_reg },
-    { "l2", 10, monitor_get_reg },
-    { "l3", 11, monitor_get_reg },
-    { "l4", 12, monitor_get_reg },
-    { "l5", 13, monitor_get_reg },
-    { "l6", 14, monitor_get_reg },
-    { "l7", 15, monitor_get_reg },
-    { "i0", 16, monitor_get_reg },
-    { "i1", 17, monitor_get_reg },
-    { "i2", 18, monitor_get_reg },
-    { "i3", 19, monitor_get_reg },
-    { "i4", 20, monitor_get_reg },
-    { "i5", 21, monitor_get_reg },
-    { "i6", 22, monitor_get_reg },
-    { "i7", 23, monitor_get_reg },
-    { "pc", offsetof(CPUSPARCState, pc) },
-    { "npc", offsetof(CPUSPARCState, npc) },
-    { "y", offsetof(CPUSPARCState, y) },
-#ifndef TARGET_SPARC64
-    { "psr", 0, &monitor_get_psr, },
-    { "wim", offsetof(CPUSPARCState, wim) },
-#endif
-    { "tbr", offsetof(CPUSPARCState, tbr) },
-    { "fsr", offsetof(CPUSPARCState, fsr) },
-    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
-    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
-    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
-    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
-    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
-    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
-    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
-    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
-    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
-    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
-    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
-    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
-    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
-    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
-    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
-    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
-    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
-    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
-    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
-    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
-    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
-    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
-    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
-    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
-    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
-    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
-    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
-    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
-    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
-    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
-    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
-    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
-#ifdef TARGET_SPARC64
-    { "f32", offsetof(CPUSPARCState, fpr[16]) },
-    { "f34", offsetof(CPUSPARCState, fpr[17]) },
-    { "f36", offsetof(CPUSPARCState, fpr[18]) },
-    { "f38", offsetof(CPUSPARCState, fpr[19]) },
-    { "f40", offsetof(CPUSPARCState, fpr[20]) },
-    { "f42", offsetof(CPUSPARCState, fpr[21]) },
-    { "f44", offsetof(CPUSPARCState, fpr[22]) },
-    { "f46", offsetof(CPUSPARCState, fpr[23]) },
-    { "f48", offsetof(CPUSPARCState, fpr[24]) },
-    { "f50", offsetof(CPUSPARCState, fpr[25]) },
-    { "f52", offsetof(CPUSPARCState, fpr[26]) },
-    { "f54", offsetof(CPUSPARCState, fpr[27]) },
-    { "f56", offsetof(CPUSPARCState, fpr[28]) },
-    { "f58", offsetof(CPUSPARCState, fpr[29]) },
-    { "f60", offsetof(CPUSPARCState, fpr[30]) },
-    { "f62", offsetof(CPUSPARCState, fpr[31]) },
-    { "asi", offsetof(CPUSPARCState, asi) },
-    { "pstate", offsetof(CPUSPARCState, pstate) },
-    { "cansave", offsetof(CPUSPARCState, cansave) },
-    { "canrestore", offsetof(CPUSPARCState, canrestore) },
-    { "otherwin", offsetof(CPUSPARCState, otherwin) },
-    { "wstate", offsetof(CPUSPARCState, wstate) },
-    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
-    { "fprs", offsetof(CPUSPARCState, fprs) },
-#endif
-#endif
-    { NULL },
-};
 
 static void GCC_FMT_ATTR(2, 3) QEMU_NORETURN
 expr_error(Monitor *mon, const char *fmt, ...)
@@ -3301,10 +2151,16 @@ expr_error(Monitor *mon, const char *fmt, ...)
 /* return 0 if OK, -1 if not found */
 static int get_monitor_def(target_long *pval, const char *name)
 {
-    const MonitorDef *md;
+    const MonitorDef *md = target_monitor_defs();
     void *ptr;
+    uint64_t tmp = 0;
+    int ret;
 
-    for(md = monitor_defs; md->name != NULL; md++) {
+    if (md == NULL) {
+        return -1;
+    }
+
+    for(; md->name != NULL; md++) {
         if (compare_cmd(name, md->name)) {
             if (md->get_value) {
                 *pval = md->get_value(md, md->offset);
@@ -3326,7 +2182,13 @@ static int get_monitor_def(target_long *pval, const char *name)
             return 0;
         }
     }
-    return -1;
+
+    ret = target_get_monitor_def(mon_get_cpu(), name, &tmp);
+    if (!ret) {
+        *pval = (target_long) tmp;
+    }
+
+    return ret;
 }
 
 static void next(void)
@@ -3891,7 +2753,7 @@ static QDict *monitor_parse_arguments(Monitor *mon,
                         break;
                     }
                 }
-                val = strtosz(p, &end);
+                val = qemu_strtosz(p, &end);
                 if (val < 0) {
                     monitor_printf(mon, "invalid size\n");
                     goto fail;
@@ -4372,7 +3234,7 @@ void sendkey_completion(ReadLineState *rs, int nb_args, const char *str)
     }
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+    for (i = 0; i < Q_KEY_CODE__MAX; i++) {
         if (!strncmp(str, QKeyCode_lookup[i], len)) {
             readline_add_completion(rs, QKeyCode_lookup[i]);
         }
@@ -4429,6 +3291,26 @@ void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str)
     }
 }
 
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str)
+{
+    size_t len;
+
+    len = strlen(str);
+    readline_set_completion_index(rs, len);
+    if (nb_args == 2) {
+        TraceEventID id;
+        for (id = 0; id < trace_event_count(); id++) {
+            const char *event_name = trace_event_get_name(trace_event_id(id));
+            if (!strncmp(str, event_name, len)) {
+                readline_add_completion(rs, event_name);
+            }
+        }
+    } else if (nb_args == 3) {
+        add_completion_option(rs, str, "on");
+        add_completion_option(rs, str, "off");
+    }
+}
+
 void watchdog_action_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     int i;
@@ -4451,7 +3333,7 @@ void migrate_set_capability_completion(ReadLineState *rs, int nb_args,
     readline_set_completion_index(rs, len);
     if (nb_args == 2) {
         int i;
-        for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+        for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
             const char *name = MigrationCapability_lookup[i];
             if (!strncmp(str, name, len)) {
                 readline_add_completion(rs, name);
@@ -4472,7 +3354,7 @@ void migrate_set_parameter_completion(ReadLineState *rs, int nb_args,
     readline_set_completion_index(rs, len);
     if (nb_args == 2) {
         int i;
-        for (i = 0; i < MIGRATION_PARAMETER_MAX; i++) {
+        for (i = 0; i < MIGRATION_PARAMETER__MAX; i++) {
             const char *name = MigrationParameter_lookup[i];
             if (!strncmp(str, name, len)) {
                 readline_add_completion(rs, name);
@@ -4551,13 +3433,18 @@ static void vm_completion(ReadLineState *rs, const char *str)
     readline_set_completion_index(rs, len);
     while ((bs = bdrv_next(bs))) {
         SnapshotInfoList *snapshots, *snapshot;
+        AioContext *ctx = bdrv_get_aio_context(bs);
+        bool ok = false;
 
-        if (!bdrv_can_snapshot(bs)) {
-            continue;
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            ok = bdrv_query_snapshot_info_list(bs, &snapshots, NULL) == 0;
         }
-        if (bdrv_query_snapshot_info_list(bs, &snapshots, NULL)) {
+        aio_context_release(ctx);
+        if (!ok) {
             continue;
         }
+
         snapshot = snapshots;
         while (snapshot) {
             char *completion = snapshot->value->name;
@@ -4598,7 +3485,7 @@ static void monitor_find_completion_by_table(Monitor *mon,
     int i;
     const char *ptype, *str, *name;
     const mon_cmd_t *cmd;
-    BlockDriverState *bs;
+    BlockBackend *blk = NULL;
 
     if (nb_args <= 1) {
         /* command completion */
@@ -4653,8 +3540,8 @@ static void monitor_find_completion_by_table(Monitor *mon,
         case 'B':
             /* block device name completion */
             readline_set_completion_index(mon->rs, strlen(str));
-            for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-                name = bdrv_get_device_name(bs);
+            while ((blk = blk_next(blk)) != NULL) {
+                name = blk_name(blk);
                 if (str[0] == '\0' ||
                     !strncmp(name, str, strlen(str))) {
                     readline_add_completion(mon->rs, name);
@@ -4979,7 +3866,7 @@ static QDict *qmp_check_input_obj(QObject *input_obj, Error **errp)
     return input_dict;
 }
 
-static void handle_qmp_command(JSONMessageParser *parser, QList *tokens)
+static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
 {
     Error *local_err = NULL;
     QObject *obj, *data;
@@ -5037,6 +3924,7 @@ static void handle_qmp_command(JSONMessageParser *parser, QList *tokens)
 err_out:
     monitor_protocol_emitter(mon, data, local_err);
     qobject_decref(data);
+    error_free(local_err);
     QDECREF(input);
     QDECREF(args);
 }
@@ -5102,7 +3990,7 @@ static QObject *get_qmp_greeting(void)
 {
     QObject *ver = NULL;
 
-    qmp_marshal_input_query_version(NULL, &ver, NULL);
+    qmp_marshal_query_version(NULL, &ver, NULL);
     return qobject_from_jsonf("{'QMP':{'version': %p,'capabilities': []}}",ver);
 }
 
@@ -5278,8 +4166,7 @@ static void bdrv_password_cb(void *opaque, const char *password,
 
     bdrv_add_key(bs, password, &local_err);
     if (local_err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         ret = -EPERM;
     }
     if (mon->password_completion_cb)
@@ -5320,6 +4207,10 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
         monitor_printf(mon, "Device not found %s\n", device);
         return -1;
     }
+    if (!blk_bs(blk)) {
+        monitor_printf(mon, "Device '%s' has no medium\n", device);
+        return -1;
+    }
 
     bdrv_add_key(blk_bs(blk), NULL, &err);
     if (err) {
@@ -5361,3 +4252,18 @@ void qmp_rtc_reset_reinjection(Error **errp)
     error_setg(errp, QERR_FEATURE_DISABLED, "rtc-reset-reinjection");
 }
 #endif
+
+#ifndef TARGET_S390X
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "dump-skeys");
+}
+#endif
+
+#ifndef TARGET_ARM
+GICCapabilityList *qmp_query_gic_capabilities(Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "query-gic-capabilities");
+    return NULL;
+}
+#endif
diff --git a/qemu/nbd/Makefile.objs b/qemu/nbd/Makefile.objs
new file mode 100644
index 000000000..eb3dd4461
--- /dev/null
+++ b/qemu/nbd/Makefile.objs
@@ -0,0 +1 @@
+block-obj-y += server.o client.o common.o
diff --git a/qemu/nbd/client.c b/qemu/nbd/client.c
new file mode 100644
index 000000000..48f2a21f3
--- /dev/null
+++ b/qemu/nbd/client.c
@@ -0,0 +1,745 @@
+/*
+ *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
+ *
+ *  Network Block Device Client Side
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
+
+static int nbd_errno_to_system_errno(int err)
+{
+    switch (err) {
+    case NBD_SUCCESS:
+        return 0;
+    case NBD_EPERM:
+        return EPERM;
+    case NBD_EIO:
+        return EIO;
+    case NBD_ENOMEM:
+        return ENOMEM;
+    case NBD_ENOSPC:
+        return ENOSPC;
+    case NBD_EINVAL:
+    default:
+        return EINVAL;
+    }
+}
+
+/* Definitions for opaque data types */
+
+static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
+
+/* That's all folks */
+
+/* Basic flow for negotiation
+
+   Server         Client
+   Negotiate
+
+   or
+
+   Server         Client
+   Negotiate #1
+                  Option
+   Negotiate #2
+
+   ----
+
+   followed by
+
+   Server         Client
+                  Request
+   Response
+                  Request
+   Response
+                  ...
+   ...
+                  Request (type == 2)
+
+*/
+
+
+/* If type represents success, return 1 without further action.
+ * If type represents an error reply, consume the rest of the packet on ioc.
+ * Then return 0 for unsupported (so the client can fall back to
+ * other approaches), or -1 with errp set for other errors.
+ */
+static int nbd_handle_reply_err(QIOChannel *ioc, uint32_t opt, uint32_t type,
+                                Error **errp)
+{
+    uint32_t len;
+    char *msg = NULL;
+    int result = -1;
+
+    if (!(type & (1 << 31))) {
+        return 1;
+    }
+
+    if (read_sync(ioc, &len, sizeof(len)) != sizeof(len)) {
+        error_setg(errp, "failed to read option length");
+        return -1;
+    }
+    len = be32_to_cpu(len);
+    if (len) {
+        if (len > NBD_MAX_BUFFER_SIZE) {
+            error_setg(errp, "server's error message is too long");
+            goto cleanup;
+        }
+        msg = g_malloc(len + 1);
+        if (read_sync(ioc, msg, len) != len) {
+            error_setg(errp, "failed to read option error message");
+            goto cleanup;
+        }
+        msg[len] = '\0';
+    }
+
+    switch (type) {
+    case NBD_REP_ERR_UNSUP:
+        TRACE("server doesn't understand request %d, attempting fallback",
+              opt);
+        result = 0;
+        goto cleanup;
+
+    case NBD_REP_ERR_POLICY:
+        error_setg(errp, "Denied by server for option %x", opt);
+        break;
+
+    case NBD_REP_ERR_INVALID:
+        error_setg(errp, "Invalid data length for option %x", opt);
+        break;
+
+    case NBD_REP_ERR_TLS_REQD:
+        error_setg(errp, "TLS negotiation required before option %x", opt);
+        break;
+
+    default:
+        error_setg(errp, "Unknown error code when asking for option %x", opt);
+        break;
+    }
+
+    if (msg) {
+        error_append_hint(errp, "%s\n", msg);
+    }
+
+ cleanup:
+    g_free(msg);
+    return result;
+}
+
+static int nbd_receive_list(QIOChannel *ioc, char **name, Error **errp)
+{
+    uint64_t magic;
+    uint32_t opt;
+    uint32_t type;
+    uint32_t len;
+    uint32_t namelen;
+    int error;
+
+    *name = NULL;
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read list option magic");
+        return -1;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option list magic");
+        return -1;
+    }
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read list option");
+        return -1;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_LIST) {
+        error_setg(errp, "Unexpected option type %x expected %x",
+                   opt, NBD_OPT_LIST);
+        return -1;
+    }
+
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read list option type");
+        return -1;
+    }
+    type = be32_to_cpu(type);
+    error = nbd_handle_reply_err(ioc, opt, type, errp);
+    if (error <= 0) {
+        return error;
+    }
+
+    if (read_sync(ioc, &len, sizeof(len)) != sizeof(len)) {
+        error_setg(errp, "failed to read option length");
+        return -1;
+    }
+    len = be32_to_cpu(len);
+
+    if (type == NBD_REP_ACK) {
+        if (len != 0) {
+            error_setg(errp, "length too long for option end");
+            return -1;
+        }
+    } else if (type == NBD_REP_SERVER) {
+        if (len < sizeof(namelen) || len > NBD_MAX_BUFFER_SIZE) {
+            error_setg(errp, "incorrect option length");
+            return -1;
+        }
+        if (read_sync(ioc, &namelen, sizeof(namelen)) != sizeof(namelen)) {
+            error_setg(errp, "failed to read option name length");
+            return -1;
+        }
+        namelen = be32_to_cpu(namelen);
+        len -= sizeof(namelen);
+        if (len < namelen) {
+            error_setg(errp, "incorrect option name length");
+            return -1;
+        }
+        if (namelen > 255) {
+            error_setg(errp, "export name length too long %d", namelen);
+            return -1;
+        }
+
+        *name = g_new0(char, namelen + 1);
+        if (read_sync(ioc, *name, namelen) != namelen) {
+            error_setg(errp, "failed to read export name");
+            g_free(*name);
+            *name = NULL;
+            return -1;
+        }
+        (*name)[namelen] = '\0';
+        len -= namelen;
+        if (len) {
+            char *buf = g_malloc(len + 1);
+            if (read_sync(ioc, buf, len) != len) {
+                error_setg(errp, "failed to read export description");
+                g_free(*name);
+                g_free(buf);
+                *name = NULL;
+                return -1;
+            }
+            buf[len] = '\0';
+            TRACE("Ignoring export description: %s", buf);
+            g_free(buf);
+        }
+    } else {
+        error_setg(errp, "Unexpected reply type %x expected %x",
+                   type, NBD_REP_SERVER);
+        return -1;
+    }
+    return 1;
+}
+
+
+static int nbd_receive_query_exports(QIOChannel *ioc,
+                                     const char *wantname,
+                                     Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_LIST);
+    uint32_t length = 0;
+    bool foundExport = false;
+
+    TRACE("Querying export list");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send list option magic");
+        return -1;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send list option number");
+        return -1;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send list option length");
+        return -1;
+    }
+
+    TRACE("Reading available export names");
+    while (1) {
+        char *name = NULL;
+        int ret = nbd_receive_list(ioc, &name, errp);
+
+        if (ret < 0) {
+            g_free(name);
+            name = NULL;
+            return -1;
+        }
+        if (ret == 0) {
+            /* Server doesn't support export listing, so
+             * we will just assume an export with our
+             * wanted name exists */
+            foundExport = true;
+            break;
+        }
+        if (name == NULL) {
+            TRACE("End of export name list");
+            break;
+        }
+        if (g_str_equal(name, wantname)) {
+            foundExport = true;
+            TRACE("Found desired export name '%s'", name);
+        } else {
+            TRACE("Ignored export name '%s'", name);
+        }
+        g_free(name);
+    }
+
+    if (!foundExport) {
+        error_setg(errp, "No export with name '%s' available", wantname);
+        return -1;
+    }
+
+    return 0;
+}
+
+static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,
+                                        QCryptoTLSCreds *tlscreds,
+                                        const char *hostname, Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_STARTTLS);
+    uint32_t length = 0;
+    uint32_t type;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Requesting TLS from server");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send option magic");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send option number");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send option length");
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server1");
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read option magic");
+        return NULL;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option magic");
+        return NULL;
+    }
+    TRACE("Getting TLS reply from server2");
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read option");
+        return NULL;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_STARTTLS) {
+        error_setg(errp, "Unexpected option type %x expected %x",
+                   opt, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read option type");
+        return NULL;
+    }
+    type = be32_to_cpu(type);
+    if (type != NBD_REP_ACK) {
+        error_setg(errp, "Server rejected request to start TLS %x",
+                   type);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "failed to read option length");
+        return NULL;
+    }
+    length = be32_to_cpu(length);
+    if (length != 0) {
+        error_setg(errp, "Start TLS reponse was not zero %x",
+                   length);
+        return NULL;
+    }
+
+    TRACE("TLS request approved, setting up TLS");
+    tioc = qio_channel_tls_new_client(ioc, tlscreds, hostname, errp);
+    if (!tioc) {
+        return NULL;
+    }
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    TRACE("Starting TLS hanshake");
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        error_propagate(errp, data.error);
+        object_unref(OBJECT(tioc));
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
+int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
+                          off_t *size, Error **errp)
+{
+    char buf[256];
+    uint64_t magic, s;
+    int rc;
+
+    TRACE("Receiving negotiation tlscreds=%p hostname=%s.",
+          tlscreds, hostname ? hostname : "<null>");
+
+    rc = -EINVAL;
+
+    if (outioc) {
+        *outioc = NULL;
+    }
+    if (tlscreds && !outioc) {
+        error_setg(errp, "Output I/O channel required for TLS");
+        goto fail;
+    }
+
+    if (read_sync(ioc, buf, 8) != 8) {
+        error_setg(errp, "Failed to read data");
+        goto fail;
+    }
+
+    buf[8] = '\0';
+    if (strlen(buf) == 0) {
+        error_setg(errp, "Server connection closed unexpectedly");
+        goto fail;
+    }
+
+    TRACE("Magic is %c%c%c%c%c%c%c%c",
+          qemu_isprint(buf[0]) ? buf[0] : '.',
+          qemu_isprint(buf[1]) ? buf[1] : '.',
+          qemu_isprint(buf[2]) ? buf[2] : '.',
+          qemu_isprint(buf[3]) ? buf[3] : '.',
+          qemu_isprint(buf[4]) ? buf[4] : '.',
+          qemu_isprint(buf[5]) ? buf[5] : '.',
+          qemu_isprint(buf[6]) ? buf[6] : '.',
+          qemu_isprint(buf[7]) ? buf[7] : '.');
+
+    if (memcmp(buf, "NBDMAGIC", 8) != 0) {
+        error_setg(errp, "Invalid magic received");
+        goto fail;
+    }
+
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to read magic");
+        goto fail;
+    }
+    magic = be64_to_cpu(magic);
+    TRACE("Magic is 0x%" PRIx64, magic);
+
+    if (magic == NBD_OPTS_MAGIC) {
+        uint32_t clientflags = 0;
+        uint32_t opt;
+        uint32_t namesize;
+        uint16_t globalflags;
+        uint16_t exportflags;
+        bool fixedNewStyle = false;
+
+        if (read_sync(ioc, &globalflags, sizeof(globalflags)) !=
+            sizeof(globalflags)) {
+            error_setg(errp, "Failed to read server flags");
+            goto fail;
+        }
+        globalflags = be16_to_cpu(globalflags);
+        *flags = globalflags << 16;
+        TRACE("Global flags are %x", globalflags);
+        if (globalflags & NBD_FLAG_FIXED_NEWSTYLE) {
+            fixedNewStyle = true;
+            TRACE("Server supports fixed new style");
+            clientflags |= NBD_FLAG_C_FIXED_NEWSTYLE;
+        }
+        /* client requested flags */
+        clientflags = cpu_to_be32(clientflags);
+        if (write_sync(ioc, &clientflags, sizeof(clientflags)) !=
+            sizeof(clientflags)) {
+            error_setg(errp, "Failed to send clientflags field");
+            goto fail;
+        }
+        if (tlscreds) {
+            if (fixedNewStyle) {
+                *outioc = nbd_receive_starttls(ioc, tlscreds, hostname, errp);
+                if (!*outioc) {
+                    goto fail;
+                }
+                ioc = *outioc;
+            } else {
+                error_setg(errp, "Server does not support STARTTLS");
+                goto fail;
+            }
+        }
+        if (!name) {
+            TRACE("Using default NBD export name \"\"");
+            name = "";
+        }
+        if (fixedNewStyle) {
+            /* Check our desired export is present in the
+             * server export list. Since NBD_OPT_EXPORT_NAME
+             * cannot return an error message, running this
+             * query gives us good error reporting if the
+             * server required TLS
+             */
+            if (nbd_receive_query_exports(ioc, name, errp) < 0) {
+                goto fail;
+            }
+        }
+        /* write the export name */
+        magic = cpu_to_be64(magic);
+        if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+            error_setg(errp, "Failed to send export name magic");
+            goto fail;
+        }
+        opt = cpu_to_be32(NBD_OPT_EXPORT_NAME);
+        if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+            error_setg(errp, "Failed to send export name option number");
+            goto fail;
+        }
+        namesize = cpu_to_be32(strlen(name));
+        if (write_sync(ioc, &namesize, sizeof(namesize)) !=
+            sizeof(namesize)) {
+            error_setg(errp, "Failed to send export name length");
+            goto fail;
+        }
+        if (write_sync(ioc, (char *)name, strlen(name)) != strlen(name)) {
+            error_setg(errp, "Failed to send export name");
+            goto fail;
+        }
+
+        if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
+            error_setg(errp, "Failed to read export length");
+            goto fail;
+        }
+        *size = be64_to_cpu(s);
+        TRACE("Size is %" PRIu64, *size);
+
+        if (read_sync(ioc, &exportflags, sizeof(exportflags)) !=
+            sizeof(exportflags)) {
+            error_setg(errp, "Failed to read export flags");
+            goto fail;
+        }
+        exportflags = be16_to_cpu(exportflags);
+        *flags |= exportflags;
+        TRACE("Export flags are %x", exportflags);
+    } else if (magic == NBD_CLIENT_MAGIC) {
+        if (name) {
+            error_setg(errp, "Server does not support export names");
+            goto fail;
+        }
+        if (tlscreds) {
+            error_setg(errp, "Server does not support STARTTLS");
+            goto fail;
+        }
+
+        if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
+            error_setg(errp, "Failed to read export length");
+            goto fail;
+        }
+        *size = be64_to_cpu(s);
+        TRACE("Size is %" PRIu64, *size);
+
+        if (read_sync(ioc, flags, sizeof(*flags)) != sizeof(*flags)) {
+            error_setg(errp, "Failed to read export flags");
+            goto fail;
+        }
+        *flags = be32_to_cpup(flags);
+    } else {
+        error_setg(errp, "Bad magic received");
+        goto fail;
+    }
+
+    if (read_sync(ioc, &buf, 124) != 124) {
+        error_setg(errp, "Failed to read reserved block");
+        goto fail;
+    }
+    rc = 0;
+
+fail:
+    return rc;
+}
+
+#ifdef __linux__
+int nbd_init(int fd, QIOChannelSocket *sioc, uint32_t flags, off_t size)
+{
+    TRACE("Setting NBD socket");
+
+    if (ioctl(fd, NBD_SET_SOCK, sioc->fd) < 0) {
+        int serrno = errno;
+        LOG("Failed to set NBD socket");
+        return -serrno;
+    }
+
+    TRACE("Setting block size to %lu", (unsigned long)BDRV_SECTOR_SIZE);
+
+    if (ioctl(fd, NBD_SET_BLKSIZE, (size_t)BDRV_SECTOR_SIZE) < 0) {
+        int serrno = errno;
+        LOG("Failed setting NBD block size");
+        return -serrno;
+    }
+
+    TRACE("Setting size to %zd block(s)", (size_t)(size / BDRV_SECTOR_SIZE));
+
+    if (ioctl(fd, NBD_SET_SIZE_BLOCKS, (size_t)(size / BDRV_SECTOR_SIZE)) < 0) {
+        int serrno = errno;
+        LOG("Failed setting size (in blocks)");
+        return -serrno;
+    }
+
+    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0) {
+        if (errno == ENOTTY) {
+            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
+            TRACE("Setting readonly attribute");
+
+            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+                int serrno = errno;
+                LOG("Failed setting read-only attribute");
+                return -serrno;
+            }
+        } else {
+            int serrno = errno;
+            LOG("Failed setting flags");
+            return -serrno;
+        }
+    }
+
+    TRACE("Negotiation ended");
+
+    return 0;
+}
+
+int nbd_client(int fd)
+{
+    int ret;
+    int serrno;
+
+    TRACE("Doing NBD loop");
+
+    ret = ioctl(fd, NBD_DO_IT);
+    if (ret < 0 && errno == EPIPE) {
+        /* NBD_DO_IT normally returns EPIPE when someone has disconnected
+         * the socket via NBD_DISCONNECT.  We do not want to return 1 in
+         * that case.
+         */
+        ret = 0;
+    }
+    serrno = errno;
+
+    TRACE("NBD loop returned %d: %s", ret, strerror(serrno));
+
+    TRACE("Clearing NBD queue");
+    ioctl(fd, NBD_CLEAR_QUE);
+
+    TRACE("Clearing NBD socket");
+    ioctl(fd, NBD_CLEAR_SOCK);
+
+    errno = serrno;
+    return ret;
+}
+#else
+int nbd_init(int fd, QIOChannelSocket *ioc, uint32_t flags, off_t size)
+{
+    return -ENOTSUP;
+}
+
+int nbd_client(int fd)
+{
+    return -ENOTSUP;
+}
+#endif
+
+ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request)
+{
+    uint8_t buf[NBD_REQUEST_SIZE];
+    ssize_t ret;
+
+    TRACE("Sending request to server: "
+          "{ .from = %" PRIu64", .len = %u, .handle = %" PRIu64", .type=%i}",
+          request->from, request->len, request->handle, request->type);
+
+    cpu_to_be32w((uint32_t*)buf, NBD_REQUEST_MAGIC);
+    cpu_to_be32w((uint32_t*)(buf + 4), request->type);
+    cpu_to_be64w((uint64_t*)(buf + 8), request->handle);
+    cpu_to_be64w((uint64_t*)(buf + 16), request->from);
+    cpu_to_be32w((uint32_t*)(buf + 24), request->len);
+
+    ret = write_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("writing to socket failed");
+        return -EINVAL;
+    }
+    return 0;
+}
+
+ssize_t nbd_receive_reply(QIOChannel *ioc, struct nbd_reply *reply)
+{
+    uint8_t buf[NBD_REPLY_SIZE];
+    uint32_t magic;
+    ssize_t ret;
+
+    ret = read_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("read failed");
+        return -EINVAL;
+    }
+
+    /* Reply
+       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
+       [ 4 ..  7]    error   (0 == no error)
+       [ 7 .. 15]    handle
+     */
+
+    magic = be32_to_cpup((uint32_t*)buf);
+    reply->error  = be32_to_cpup((uint32_t*)(buf + 4));
+    reply->handle = be64_to_cpup((uint64_t*)(buf + 8));
+
+    reply->error = nbd_errno_to_system_errno(reply->error);
+
+    TRACE("Got reply: "
+          "{ magic = 0x%x, .error = %d, handle = %" PRIu64" }",
+          magic, reply->error, reply->handle);
+
+    if (magic != NBD_REPLY_MAGIC) {
+        LOG("invalid magic (got 0x%x)", magic);
+        return -EINVAL;
+    }
+    return 0;
+}
+
diff --git a/qemu/nbd/common.c b/qemu/nbd/common.c
new file mode 100644
index 000000000..8ddb2dd2f
--- /dev/null
+++ b/qemu/nbd/common.c
@@ -0,0 +1,96 @@
+/*
+ *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
+ *
+ *  Network Block Device Common Code
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
+
+ssize_t nbd_wr_syncv(QIOChannel *ioc,
+                     struct iovec *iov,
+                     size_t niov,
+                     size_t offset,
+                     size_t length,
+                     bool do_read)
+{
+    ssize_t done = 0;
+    Error *local_err = NULL;
+    struct iovec *local_iov = g_new(struct iovec, niov);
+    struct iovec *local_iov_head = local_iov;
+    unsigned int nlocal_iov = niov;
+
+    nlocal_iov = iov_copy(local_iov, nlocal_iov,
+                          iov, niov,
+                          offset, length);
+
+    while (nlocal_iov > 0) {
+        ssize_t len;
+        if (do_read) {
+            len = qio_channel_readv(ioc, local_iov, nlocal_iov, &local_err);
+        } else {
+            len = qio_channel_writev(ioc, local_iov, nlocal_iov, &local_err);
+        }
+        if (len == QIO_CHANNEL_ERR_BLOCK) {
+            if (qemu_in_coroutine()) {
+                /* XXX figure out if we can create a variant on
+                 * qio_channel_yield() that works with AIO contexts
+                 * and consider using that in this branch */
+                qemu_coroutine_yield();
+            } else if (done) {
+                /* XXX this is needed by nbd_reply_ready.  */
+                qio_channel_wait(ioc,
+                                 do_read ? G_IO_IN : G_IO_OUT);
+            } else {
+                return -EAGAIN;
+            }
+            continue;
+        }
+        if (len < 0) {
+            TRACE("I/O error: %s", error_get_pretty(local_err));
+            error_free(local_err);
+            /* XXX handle Error objects */
+            done = -EIO;
+            goto cleanup;
+        }
+
+        if (do_read && len == 0) {
+            break;
+        }
+
+        iov_discard_front(&local_iov, &nlocal_iov, len);
+        done += len;
+    }
+
+ cleanup:
+    g_free(local_iov_head);
+    return done;
+}
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque)
+{
+    struct NBDTLSHandshakeData *data = opaque;
+
+    if (err) {
+        TRACE("TLS failed %s", error_get_pretty(err));
+        data->error = error_copy(err);
+    }
+    data->complete = true;
+    g_main_loop_quit(data->loop);
+}
diff --git a/qemu/nbd/nbd-internal.h b/qemu/nbd/nbd-internal.h
new file mode 100644
index 000000000..379153561
--- /dev/null
+++ b/qemu/nbd/nbd-internal.h
@@ -0,0 +1,124 @@
+/*
+ * NBD Internal Declarations
+ *
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef NBD_INTERNAL_H
+#define NBD_INTERNAL_H
+#include "block/nbd.h"
+#include "sysemu/block-backend.h"
+#include "io/channel-tls.h"
+
+#include "qemu/coroutine.h"
+#include "qemu/iov.h"
+
+#ifndef _WIN32
+#include <sys/ioctl.h>
+#endif
+#if defined(__sun__) || defined(__HAIKU__)
+#include <sys/ioccom.h>
+#endif
+
+#ifdef __linux__
+#include <linux/fs.h>
+#endif
+
+#include "qemu/queue.h"
+#include "qemu/main-loop.h"
+
+/* #define DEBUG_NBD */
+
+#ifdef DEBUG_NBD
+#define DEBUG_NBD_PRINT 1
+#else
+#define DEBUG_NBD_PRINT 0
+#endif
+
+#define TRACE(msg, ...) do { \
+    if (DEBUG_NBD_PRINT) { \
+        LOG(msg, ## __VA_ARGS__); \
+    } \
+} while (0)
+
+#define LOG(msg, ...) do { \
+    fprintf(stderr, "%s:%s():L%d: " msg "\n", \
+            __FILE__, __FUNCTION__, __LINE__, ## __VA_ARGS__); \
+} while (0)
+
+/* This is all part of the "official" NBD API.
+ *
+ * The most up-to-date documentation is available at:
+ * https://github.com/yoe/nbd/blob/master/doc/proto.txt
+ */
+
+#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
+#define NBD_REPLY_SIZE          (4 + 4 + 8)
+#define NBD_REQUEST_MAGIC       0x25609513
+#define NBD_REPLY_MAGIC         0x67446698
+#define NBD_OPTS_MAGIC          0x49484156454F5054LL
+#define NBD_CLIENT_MAGIC        0x0000420281861253LL
+#define NBD_REP_MAGIC           0x3e889045565a9LL
+
+#define NBD_SET_SOCK            _IO(0xab, 0)
+#define NBD_SET_BLKSIZE         _IO(0xab, 1)
+#define NBD_SET_SIZE            _IO(0xab, 2)
+#define NBD_DO_IT               _IO(0xab, 3)
+#define NBD_CLEAR_SOCK          _IO(0xab, 4)
+#define NBD_CLEAR_QUE           _IO(0xab, 5)
+#define NBD_PRINT_DEBUG         _IO(0xab, 6)
+#define NBD_SET_SIZE_BLOCKS     _IO(0xab, 7)
+#define NBD_DISCONNECT          _IO(0xab, 8)
+#define NBD_SET_TIMEOUT         _IO(0xab, 9)
+#define NBD_SET_FLAGS           _IO(0xab, 10)
+
+#define NBD_OPT_EXPORT_NAME     (1)
+#define NBD_OPT_ABORT           (2)
+#define NBD_OPT_LIST            (3)
+#define NBD_OPT_PEEK_EXPORT     (4)
+#define NBD_OPT_STARTTLS        (5)
+
+/* NBD errors are based on errno numbers, so there is a 1:1 mapping,
+ * but only a limited set of errno values is specified in the protocol.
+ * Everything else is squashed to EINVAL.
+ */
+#define NBD_SUCCESS    0
+#define NBD_EPERM      1
+#define NBD_EIO        5
+#define NBD_ENOMEM     12
+#define NBD_EINVAL     22
+#define NBD_ENOSPC     28
+
+static inline ssize_t read_sync(QIOChannel *ioc, void *buffer, size_t size)
+{
+    struct iovec iov = { .iov_base = buffer, .iov_len = size };
+    /* Sockets are kept in blocking mode in the negotiation phase.  After
+     * that, a non-readable socket simply means that another thread stole
+     * our request/reply.  Synchronization is done with recv_coroutine, so
+     * that this is coroutine-safe.
+     */
+    return nbd_wr_syncv(ioc, &iov, 1, 0, size, true);
+}
+
+static inline ssize_t write_sync(QIOChannel *ioc, void *buffer, size_t size)
+{
+    struct iovec iov = { .iov_base = buffer, .iov_len = size };
+
+    return nbd_wr_syncv(ioc, &iov, 1, 0, size, false);
+}
+
+struct NBDTLSHandshakeData {
+    GMainLoop *loop;
+    bool complete;
+    Error *error;
+};
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque);
+
+#endif
diff --git a/qemu/nbd.c b/qemu/nbd/server.c
index 06b501ba6..2184c64fe 100644
--- a/qemu/nbd.c
+++ b/qemu/nbd/server.c
@@ -1,7 +1,7 @@
 /*
  *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
  *
- *  Network Block Device
+ *  Network Block Device Server Side
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -16,86 +16,9 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "block/nbd.h"
-#include "sysemu/block-backend.h"
-
-#include "block/coroutine.h"
-
-#include <errno.h>
-#include <string.h>
-#ifndef _WIN32
-#include <sys/ioctl.h>
-#endif
-#if defined(__sun__) || defined(__HAIKU__)
-#include <sys/ioccom.h>
-#endif
-#include <ctype.h>
-#include <inttypes.h>
-
-#ifdef __linux__
-#include <linux/fs.h>
-#endif
-
-#include "qemu/sockets.h"
-#include "qemu/queue.h"
-#include "qemu/main-loop.h"
-
-//#define DEBUG_NBD
-
-#ifdef DEBUG_NBD
-#define TRACE(msg, ...) do { \
-    LOG(msg, ## __VA_ARGS__); \
-} while(0)
-#else
-#define TRACE(msg, ...) \
-    do { } while (0)
-#endif
-
-#define LOG(msg, ...) do { \
-    fprintf(stderr, "%s:%s():L%d: " msg "\n", \
-            __FILE__, __FUNCTION__, __LINE__, ## __VA_ARGS__); \
-} while(0)
-
-/* This is all part of the "official" NBD API.
- *
- * The most up-to-date documentation is available at:
- * https://github.com/yoe/nbd/blob/master/doc/proto.txt
- */
-
-#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
-#define NBD_REPLY_SIZE          (4 + 4 + 8)
-#define NBD_REQUEST_MAGIC       0x25609513
-#define NBD_REPLY_MAGIC         0x67446698
-#define NBD_OPTS_MAGIC          0x49484156454F5054LL
-#define NBD_CLIENT_MAGIC        0x0000420281861253LL
-#define NBD_REP_MAGIC           0x3e889045565a9LL
-
-#define NBD_SET_SOCK            _IO(0xab, 0)
-#define NBD_SET_BLKSIZE         _IO(0xab, 1)
-#define NBD_SET_SIZE            _IO(0xab, 2)
-#define NBD_DO_IT               _IO(0xab, 3)
-#define NBD_CLEAR_SOCK          _IO(0xab, 4)
-#define NBD_CLEAR_QUE           _IO(0xab, 5)
-#define NBD_PRINT_DEBUG         _IO(0xab, 6)
-#define NBD_SET_SIZE_BLOCKS     _IO(0xab, 7)
-#define NBD_DISCONNECT          _IO(0xab, 8)
-#define NBD_SET_TIMEOUT         _IO(0xab, 9)
-#define NBD_SET_FLAGS           _IO(0xab, 10)
-
-#define NBD_OPT_EXPORT_NAME     (1)
-#define NBD_OPT_ABORT           (2)
-#define NBD_OPT_LIST            (3)
-
-/* NBD errors are based on errno numbers, so there is a 1:1 mapping,
- * but only a limited set of errno values is specified in the protocol.
- * Everything else is squashed to EINVAL.
- */
-#define NBD_SUCCESS    0
-#define NBD_EPERM      1
-#define NBD_EIO        5
-#define NBD_ENOMEM     12
-#define NBD_EINVAL     22
-#define NBD_ENOSPC     28
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
 
 static int system_errno_to_nbd_errno(int err)
 {
@@ -103,6 +26,7 @@ static int system_errno_to_nbd_errno(int err)
     case 0:
         return NBD_SUCCESS;
     case EPERM:
+    case EROFS:
         return NBD_EPERM;
     case EIO:
         return NBD_EIO;
@@ -120,25 +44,6 @@ static int system_errno_to_nbd_errno(int err)
     }
 }
 
-static int nbd_errno_to_system_errno(int err)
-{
-    switch (err) {
-    case NBD_SUCCESS:
-        return 0;
-    case NBD_EPERM:
-        return EPERM;
-    case NBD_EIO:
-        return EIO;
-    case NBD_ENOMEM:
-        return ENOMEM;
-    case NBD_ENOSPC:
-        return ENOSPC;
-    case NBD_EINVAL:
-    default:
-        return EINVAL;
-    }
-}
-
 /* Definitions for opaque data types */
 
 typedef struct NBDRequest NBDRequest;
@@ -162,6 +67,8 @@ struct NBDExport {
     QTAILQ_ENTRY(NBDExport) next;
 
     AioContext *ctx;
+
+    Notifier eject_notifier;
 };
 
 static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
@@ -171,7 +78,10 @@ struct NBDClient {
     void (*close)(NBDClient *client);
 
     NBDExport *exp;
-    int sock;
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
+    QIOChannelSocket *sioc; /* The underlying data channel */
+    QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
 
     Coroutine *recv_coroutine;
 
@@ -191,68 +101,56 @@ static void nbd_set_handlers(NBDClient *client);
 static void nbd_unset_handlers(NBDClient *client);
 static void nbd_update_can_read(NBDClient *client);
 
-ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
+static gboolean nbd_negotiate_continue(QIOChannel *ioc,
+                                       GIOCondition condition,
+                                       void *opaque)
 {
-    size_t offset = 0;
-    int err;
-
-    if (qemu_in_coroutine()) {
-        if (do_read) {
-            return qemu_co_recv(fd, buffer, size);
-        } else {
-            return qemu_co_send(fd, buffer, size);
-        }
-    }
-
-    while (offset < size) {
-        ssize_t len;
-
-        if (do_read) {
-            len = qemu_recv(fd, buffer + offset, size - offset, 0);
-        } else {
-            len = send(fd, buffer + offset, size - offset, 0);
-        }
-
-        if (len < 0) {
-            err = socket_error();
-
-            /* recoverable error */
-            if (err == EINTR || (offset > 0 && (err == EAGAIN || err == EWOULDBLOCK))) {
-                continue;
-            }
-
-            /* unrecoverable error */
-            return -err;
-        }
-
-        /* eof */
-        if (len == 0) {
-            break;
-        }
+    qemu_coroutine_enter(opaque, NULL);
+    return TRUE;
+}
 
-        offset += len;
-    }
+static ssize_t nbd_negotiate_read(QIOChannel *ioc, void *buffer, size_t size)
+{
+    ssize_t ret;
+    guint watch;
+
+    assert(qemu_in_coroutine());
+    /* Negotiation are always in main loop. */
+    watch = qio_channel_add_watch(ioc,
+                                  G_IO_IN,
+                                  nbd_negotiate_continue,
+                                  qemu_coroutine_self(),
+                                  NULL);
+    ret = read_sync(ioc, buffer, size);
+    g_source_remove(watch);
+    return ret;
 
-    return offset;
 }
 
-static ssize_t read_sync(int fd, void *buffer, size_t size)
+static ssize_t nbd_negotiate_write(QIOChannel *ioc, void *buffer, size_t size)
 {
-    /* Sockets are kept in blocking mode in the negotiation phase.  After
-     * that, a non-readable socket simply means that another thread stole
-     * our request/reply.  Synchronization is done with recv_coroutine, so
-     * that this is coroutine-safe.
-     */
-    return nbd_wr_sync(fd, buffer, size, true);
+    ssize_t ret;
+    guint watch;
+
+    assert(qemu_in_coroutine());
+    /* Negotiation are always in main loop. */
+    watch = qio_channel_add_watch(ioc,
+                                  G_IO_OUT,
+                                  nbd_negotiate_continue,
+                                  qemu_coroutine_self(),
+                                  NULL);
+    ret = write_sync(ioc, buffer, size);
+    g_source_remove(watch);
+    return ret;
 }
 
-static ssize_t drop_sync(int fd, size_t size)
+static ssize_t nbd_negotiate_drop_sync(QIOChannel *ioc, size_t size)
 {
     ssize_t ret, dropped = size;
     uint8_t *buffer = g_malloc(MIN(65536, size));
 
     while (size > 0) {
-        ret = read_sync(fd, buffer, MIN(65536, size));
+        ret = nbd_negotiate_read(ioc, buffer, MIN(65536, size));
         if (ret < 0) {
             g_free(buffer);
             return ret;
@@ -266,16 +164,6 @@ static ssize_t drop_sync(int fd, size_t size)
     return dropped;
 }
 
-static ssize_t write_sync(int fd, void *buffer, size_t size)
-{
-    int ret;
-    do {
-        /* For writes, we do expect the socket to be writable.  */
-        ret = nbd_wr_sync(fd, buffer, size, false);
-    } while (ret == -EAGAIN);
-    return ret;
-}
-
 /* Basic flow for negotiation
 
    Server         Client
@@ -303,98 +191,100 @@ static ssize_t write_sync(int fd, void *buffer, size_t size)
 
 */
 
-static int nbd_send_rep(int csock, uint32_t type, uint32_t opt)
+static int nbd_negotiate_send_rep(QIOChannel *ioc, uint32_t type, uint32_t opt)
 {
     uint64_t magic;
     uint32_t len;
 
+    TRACE("Reply opt=%x type=%x", type, opt);
+
     magic = cpu_to_be64(NBD_REP_MAGIC);
-    if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
+    if (nbd_negotiate_write(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
         LOG("write failed (rep magic)");
         return -EINVAL;
     }
     opt = cpu_to_be32(opt);
-    if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
+    if (nbd_negotiate_write(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
         LOG("write failed (rep opt)");
         return -EINVAL;
     }
     type = cpu_to_be32(type);
-    if (write_sync(csock, &type, sizeof(type)) != sizeof(type)) {
+    if (nbd_negotiate_write(ioc, &type, sizeof(type)) != sizeof(type)) {
         LOG("write failed (rep type)");
         return -EINVAL;
     }
     len = cpu_to_be32(0);
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
         LOG("write failed (rep data length)");
         return -EINVAL;
     }
     return 0;
 }
 
-static int nbd_send_rep_list(int csock, NBDExport *exp)
+static int nbd_negotiate_send_rep_list(QIOChannel *ioc, NBDExport *exp)
 {
     uint64_t magic, name_len;
     uint32_t opt, type, len;
 
+    TRACE("Advertizing export name '%s'", exp->name ? exp->name : "");
     name_len = strlen(exp->name);
     magic = cpu_to_be64(NBD_REP_MAGIC);
-    if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
+    if (nbd_negotiate_write(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
         LOG("write failed (magic)");
         return -EINVAL;
      }
     opt = cpu_to_be32(NBD_OPT_LIST);
-    if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
+    if (nbd_negotiate_write(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
         LOG("write failed (opt)");
         return -EINVAL;
     }
     type = cpu_to_be32(NBD_REP_SERVER);
-    if (write_sync(csock, &type, sizeof(type)) != sizeof(type)) {
+    if (nbd_negotiate_write(ioc, &type, sizeof(type)) != sizeof(type)) {
         LOG("write failed (reply type)");
         return -EINVAL;
     }
     len = cpu_to_be32(name_len + sizeof(len));
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
         LOG("write failed (length)");
         return -EINVAL;
     }
     len = cpu_to_be32(name_len);
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
         LOG("write failed (length)");
         return -EINVAL;
     }
-    if (write_sync(csock, exp->name, name_len) != name_len) {
+    if (nbd_negotiate_write(ioc, exp->name, name_len) != name_len) {
         LOG("write failed (buffer)");
         return -EINVAL;
     }
     return 0;
 }
 
-static int nbd_handle_list(NBDClient *client, uint32_t length)
+static int nbd_negotiate_handle_list(NBDClient *client, uint32_t length)
 {
-    int csock;
     NBDExport *exp;
 
-    csock = client->sock;
     if (length) {
-        if (drop_sync(csock, length) != length) {
+        if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
             return -EIO;
         }
-        return nbd_send_rep(csock, NBD_REP_ERR_INVALID, NBD_OPT_LIST);
+        return nbd_negotiate_send_rep(client->ioc,
+                                      NBD_REP_ERR_INVALID, NBD_OPT_LIST);
     }
 
     /* For each export, send a NBD_REP_SERVER reply. */
     QTAILQ_FOREACH(exp, &exports, next) {
-        if (nbd_send_rep_list(csock, exp)) {
+        if (nbd_negotiate_send_rep_list(client->ioc, exp)) {
             return -EINVAL;
         }
     }
     /* Finish with a NBD_REP_ACK. */
-    return nbd_send_rep(csock, NBD_REP_ACK, NBD_OPT_LIST);
+    return nbd_negotiate_send_rep(client->ioc, NBD_REP_ACK, NBD_OPT_LIST);
 }
 
-static int nbd_handle_export_name(NBDClient *client, uint32_t length)
+static int nbd_negotiate_handle_export_name(NBDClient *client, uint32_t length)
 {
-    int rc = -EINVAL, csock = client->sock;
+    int rc = -EINVAL;
     char name[256];
 
     /* Client sends:
@@ -405,12 +295,14 @@ static int nbd_handle_export_name(NBDClient *client, uint32_t length)
         LOG("Bad length received");
         goto fail;
     }
-    if (read_sync(csock, name, length) != length) {
+    if (nbd_negotiate_read(client->ioc, name, length) != length) {
         LOG("read failed");
         goto fail;
     }
     name[length] = '\0';
 
+    TRACE("Client requested export '%s'", name);
+
     client->exp = nbd_export_find(name);
     if (!client->exp) {
         LOG("export not found");
@@ -424,10 +316,59 @@ fail:
     return rc;
 }
 
-static int nbd_receive_options(NBDClient *client)
+
+static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
+                                                 uint32_t length)
+{
+    QIOChannel *ioc;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Setting up TLS");
+    ioc = client->ioc;
+    if (length) {
+        if (nbd_negotiate_drop_sync(ioc, length) != length) {
+            return NULL;
+        }
+        nbd_negotiate_send_rep(ioc, NBD_REP_ERR_INVALID, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    nbd_negotiate_send_rep(client->ioc, NBD_REP_ACK, NBD_OPT_STARTTLS);
+
+    tioc = qio_channel_tls_new_server(ioc,
+                                      client->tlscreds,
+                                      client->tlsaclname,
+                                      NULL);
+    if (!tioc) {
+        return NULL;
+    }
+
+    TRACE("Starting TLS handshake");
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        object_unref(OBJECT(tioc));
+        error_free(data.error);
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
+static int nbd_negotiate_options(NBDClient *client)
 {
-    int csock = client->sock;
     uint32_t flags;
+    bool fixedNewstyle = false;
 
     /* Client sends:
         [ 0 ..   3]   client flags
@@ -443,23 +384,30 @@ static int nbd_receive_options(NBDClient *client)
         ...           Rest of request
     */
 
-    if (read_sync(csock, &flags, sizeof(flags)) != sizeof(flags)) {
+    if (nbd_negotiate_read(client->ioc, &flags, sizeof(flags)) !=
+        sizeof(flags)) {
         LOG("read failed");
         return -EIO;
     }
     TRACE("Checking client flags");
     be32_to_cpus(&flags);
-    if (flags != 0 && flags != NBD_FLAG_C_FIXED_NEWSTYLE) {
-        LOG("Bad client flags received");
+    if (flags & NBD_FLAG_C_FIXED_NEWSTYLE) {
+        TRACE("Support supports fixed newstyle handshake");
+        fixedNewstyle = true;
+        flags &= ~NBD_FLAG_C_FIXED_NEWSTYLE;
+    }
+    if (flags != 0) {
+        TRACE("Unknown client flags 0x%x received", flags);
         return -EIO;
     }
 
     while (1) {
         int ret;
-        uint32_t tmp, length;
+        uint32_t clientflags, length;
         uint64_t magic;
 
-        if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
+        if (nbd_negotiate_read(client->ioc, &magic, sizeof(magic)) !=
+            sizeof(magic)) {
             LOG("read failed");
             return -EINVAL;
         }
@@ -469,50 +417,122 @@ static int nbd_receive_options(NBDClient *client)
             return -EINVAL;
         }
 
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
+        if (nbd_negotiate_read(client->ioc, &clientflags,
+                               sizeof(clientflags)) != sizeof(clientflags)) {
             LOG("read failed");
             return -EINVAL;
         }
+        clientflags = be32_to_cpu(clientflags);
 
-        if (read_sync(csock, &length, sizeof(length)) != sizeof(length)) {
+        if (nbd_negotiate_read(client->ioc, &length, sizeof(length)) !=
+            sizeof(length)) {
             LOG("read failed");
             return -EINVAL;
         }
         length = be32_to_cpu(length);
 
-        TRACE("Checking option");
-        switch (be32_to_cpu(tmp)) {
-        case NBD_OPT_LIST:
-            ret = nbd_handle_list(client, length);
-            if (ret < 0) {
-                return ret;
+        TRACE("Checking option 0x%x", clientflags);
+        if (client->tlscreds &&
+            client->ioc == (QIOChannel *)client->sioc) {
+            QIOChannel *tioc;
+            if (!fixedNewstyle) {
+                TRACE("Unsupported option 0x%x", clientflags);
+                return -EINVAL;
+            }
+            switch (clientflags) {
+            case NBD_OPT_STARTTLS:
+                tioc = nbd_negotiate_handle_starttls(client, length);
+                if (!tioc) {
+                    return -EIO;
+                }
+                object_unref(OBJECT(client->ioc));
+                client->ioc = QIO_CHANNEL(tioc);
+                break;
+
+            case NBD_OPT_EXPORT_NAME:
+                /* No way to return an error to client, so drop connection */
+                TRACE("Option 0x%x not permitted before TLS", clientflags);
+                return -EINVAL;
+
+            default:
+                TRACE("Option 0x%x not permitted before TLS", clientflags);
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_TLS_REQD,
+                                       clientflags);
+                break;
+            }
+        } else if (fixedNewstyle) {
+            switch (clientflags) {
+            case NBD_OPT_LIST:
+                ret = nbd_negotiate_handle_list(client, length);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+
+            case NBD_OPT_ABORT:
+                return -EINVAL;
+
+            case NBD_OPT_EXPORT_NAME:
+                return nbd_negotiate_handle_export_name(client, length);
+
+            case NBD_OPT_STARTTLS:
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                if (client->tlscreds) {
+                    TRACE("TLS already enabled");
+                    nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_INVALID,
+                                           clientflags);
+                } else {
+                    TRACE("TLS not configured");
+                    nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_POLICY,
+                                           clientflags);
+                }
+                break;
+            default:
+                TRACE("Unsupported option 0x%x", clientflags);
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_UNSUP,
+                                       clientflags);
+                break;
+            }
+        } else {
+            /*
+             * If broken new-style we should drop the connection
+             * for anything except NBD_OPT_EXPORT_NAME
+             */
+            switch (clientflags) {
+            case NBD_OPT_EXPORT_NAME:
+                return nbd_negotiate_handle_export_name(client, length);
+
+            default:
+                TRACE("Unsupported option 0x%x", clientflags);
+                return -EINVAL;
             }
-            break;
-
-        case NBD_OPT_ABORT:
-            return -EINVAL;
-
-        case NBD_OPT_EXPORT_NAME:
-            return nbd_handle_export_name(client, length);
-
-        default:
-            tmp = be32_to_cpu(tmp);
-            LOG("Unsupported option 0x%x", tmp);
-            nbd_send_rep(client->sock, NBD_REP_ERR_UNSUP, tmp);
-            return -EINVAL;
         }
     }
 }
 
-static int nbd_send_negotiate(NBDClient *client)
+typedef struct {
+    NBDClient *client;
+    Coroutine *co;
+} NBDClientNewData;
+
+static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
 {
-    int csock = client->sock;
+    NBDClient *client = data->client;
     char buf[8 + 8 + 8 + 128];
     int rc;
     const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
                          NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
+    bool oldStyle;
 
-    /* Negotiation header without options:
+    /* Old style negotiation header without options
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
@@ -520,53 +540,59 @@ static int nbd_send_negotiate(NBDClient *client)
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
 
-       Negotiation header with options, part 1:
+       New style negotiation header with options
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
         [16 ..  17]   server flags (0)
-
-       part 2 (after options are sent):
+        ....options sent....
         [18 ..  25]   size
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
      */
 
-    qemu_set_block(csock);
+    qio_channel_set_blocking(client->ioc, false, NULL);
     rc = -EINVAL;
 
     TRACE("Beginning negotiation.");
     memset(buf, 0, sizeof(buf));
     memcpy(buf, "NBDMAGIC", 8);
-    if (client->exp) {
+
+    oldStyle = client->exp != NULL && !client->tlscreds;
+    if (oldStyle) {
         assert ((client->exp->nbdflags & ~65535) == 0);
-        cpu_to_be64w((uint64_t*)(buf + 8), NBD_CLIENT_MAGIC);
-        cpu_to_be64w((uint64_t*)(buf + 16), client->exp->size);
-        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
+        stq_be_p(buf + 8, NBD_CLIENT_MAGIC);
+        stq_be_p(buf + 16, client->exp->size);
+        stw_be_p(buf + 26, client->exp->nbdflags | myflags);
     } else {
-        cpu_to_be64w((uint64_t*)(buf + 8), NBD_OPTS_MAGIC);
-        cpu_to_be16w((uint16_t *)(buf + 16), NBD_FLAG_FIXED_NEWSTYLE);
+        stq_be_p(buf + 8, NBD_OPTS_MAGIC);
+        stw_be_p(buf + 16, NBD_FLAG_FIXED_NEWSTYLE);
     }
 
-    if (client->exp) {
-        if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
+    if (oldStyle) {
+        if (client->tlscreds) {
+            TRACE("TLS cannot be enabled with oldstyle protocol");
+            goto fail;
+        }
+        if (nbd_negotiate_write(client->ioc, buf, sizeof(buf)) != sizeof(buf)) {
             LOG("write failed");
             goto fail;
         }
     } else {
-        if (write_sync(csock, buf, 18) != 18) {
+        if (nbd_negotiate_write(client->ioc, buf, 18) != 18) {
             LOG("write failed");
             goto fail;
         }
-        rc = nbd_receive_options(client);
+        rc = nbd_negotiate_options(client);
         if (rc != 0) {
             LOG("option negotiation failed");
             goto fail;
         }
 
         assert ((client->exp->nbdflags & ~65535) == 0);
-        cpu_to_be64w((uint64_t*)(buf + 18), client->exp->size);
-        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
-        if (write_sync(csock, buf + 18, sizeof(buf) - 18) != sizeof(buf) - 18) {
+        stq_be_p(buf + 18, client->exp->size);
+        stw_be_p(buf + 26, client->exp->nbdflags | myflags);
+        if (nbd_negotiate_write(client->ioc, buf + 18, sizeof(buf) - 18) !=
+            sizeof(buf) - 18) {
             LOG("write failed");
             goto fail;
         }
@@ -575,192 +601,10 @@ static int nbd_send_negotiate(NBDClient *client)
     TRACE("Negotiation succeeded.");
     rc = 0;
 fail:
-    qemu_set_nonblock(csock);
-    return rc;
-}
-
-int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
-                          off_t *size, Error **errp)
-{
-    char buf[256];
-    uint64_t magic, s;
-    uint16_t tmp;
-    int rc;
-
-    TRACE("Receiving negotiation.");
-
-    rc = -EINVAL;
-
-    if (read_sync(csock, buf, 8) != 8) {
-        error_setg(errp, "Failed to read data");
-        goto fail;
-    }
-
-    buf[8] = '\0';
-    if (strlen(buf) == 0) {
-        error_setg(errp, "Server connection closed unexpectedly");
-        goto fail;
-    }
-
-    TRACE("Magic is %c%c%c%c%c%c%c%c",
-          qemu_isprint(buf[0]) ? buf[0] : '.',
-          qemu_isprint(buf[1]) ? buf[1] : '.',
-          qemu_isprint(buf[2]) ? buf[2] : '.',
-          qemu_isprint(buf[3]) ? buf[3] : '.',
-          qemu_isprint(buf[4]) ? buf[4] : '.',
-          qemu_isprint(buf[5]) ? buf[5] : '.',
-          qemu_isprint(buf[6]) ? buf[6] : '.',
-          qemu_isprint(buf[7]) ? buf[7] : '.');
-
-    if (memcmp(buf, "NBDMAGIC", 8) != 0) {
-        error_setg(errp, "Invalid magic received");
-        goto fail;
-    }
-
-    if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-        error_setg(errp, "Failed to read magic");
-        goto fail;
-    }
-    magic = be64_to_cpu(magic);
-    TRACE("Magic is 0x%" PRIx64, magic);
-
-    if (name) {
-        uint32_t reserved = 0;
-        uint32_t opt;
-        uint32_t namesize;
-
-        TRACE("Checking magic (opts_magic)");
-        if (magic != NBD_OPTS_MAGIC) {
-            if (magic == NBD_CLIENT_MAGIC) {
-                error_setg(errp, "Server does not support export names");
-            } else {
-                error_setg(errp, "Bad magic received");
-            }
-            goto fail;
-        }
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            error_setg(errp, "Failed to read server flags");
-            goto fail;
-        }
-        *flags = be16_to_cpu(tmp) << 16;
-        /* reserved for future use */
-        if (write_sync(csock, &reserved, sizeof(reserved)) !=
-            sizeof(reserved)) {
-            error_setg(errp, "Failed to read reserved field");
-            goto fail;
-        }
-        /* write the export name */
-        magic = cpu_to_be64(magic);
-        if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-            error_setg(errp, "Failed to send export name magic");
-            goto fail;
-        }
-        opt = cpu_to_be32(NBD_OPT_EXPORT_NAME);
-        if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
-            error_setg(errp, "Failed to send export name option number");
-            goto fail;
-        }
-        namesize = cpu_to_be32(strlen(name));
-        if (write_sync(csock, &namesize, sizeof(namesize)) !=
-            sizeof(namesize)) {
-            error_setg(errp, "Failed to send export name length");
-            goto fail;
-        }
-        if (write_sync(csock, (char*)name, strlen(name)) != strlen(name)) {
-            error_setg(errp, "Failed to send export name");
-            goto fail;
-        }
-    } else {
-        TRACE("Checking magic (cli_magic)");
-
-        if (magic != NBD_CLIENT_MAGIC) {
-            if (magic == NBD_OPTS_MAGIC) {
-                error_setg(errp, "Server requires an export name");
-            } else {
-                error_setg(errp, "Bad magic received");
-            }
-            goto fail;
-        }
-    }
-
-    if (read_sync(csock, &s, sizeof(s)) != sizeof(s)) {
-        error_setg(errp, "Failed to read export length");
-        goto fail;
-    }
-    *size = be64_to_cpu(s);
-    TRACE("Size is %" PRIu64, *size);
-
-    if (!name) {
-        if (read_sync(csock, flags, sizeof(*flags)) != sizeof(*flags)) {
-            error_setg(errp, "Failed to read export flags");
-            goto fail;
-        }
-        *flags = be32_to_cpup(flags);
-    } else {
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            error_setg(errp, "Failed to read export flags");
-            goto fail;
-        }
-        *flags |= be16_to_cpu(tmp);
-    }
-    if (read_sync(csock, &buf, 124) != 124) {
-        error_setg(errp, "Failed to read reserved block");
-        goto fail;
-    }
-    rc = 0;
-
-fail:
     return rc;
 }
 
 #ifdef __linux__
-int nbd_init(int fd, int csock, uint32_t flags, off_t size)
-{
-    TRACE("Setting NBD socket");
-
-    if (ioctl(fd, NBD_SET_SOCK, csock) < 0) {
-        int serrno = errno;
-        LOG("Failed to set NBD socket");
-        return -serrno;
-    }
-
-    TRACE("Setting block size to %lu", (unsigned long)BDRV_SECTOR_SIZE);
-
-    if (ioctl(fd, NBD_SET_BLKSIZE, (size_t)BDRV_SECTOR_SIZE) < 0) {
-        int serrno = errno;
-        LOG("Failed setting NBD block size");
-        return -serrno;
-    }
-
-    TRACE("Setting size to %zd block(s)", (size_t)(size / BDRV_SECTOR_SIZE));
-
-    if (ioctl(fd, NBD_SET_SIZE_BLOCKS, (size_t)(size / BDRV_SECTOR_SIZE)) < 0) {
-        int serrno = errno;
-        LOG("Failed setting size (in blocks)");
-        return -serrno;
-    }
-
-    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0) {
-        if (errno == ENOTTY) {
-            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
-            TRACE("Setting readonly attribute");
-
-            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
-                int serrno = errno;
-                LOG("Failed setting read-only attribute");
-                return -serrno;
-            }
-        } else {
-            int serrno = errno;
-            LOG("Failed setting flags");
-            return -serrno;
-        }
-    }
-
-    TRACE("Negotiation ended");
-
-    return 0;
-}
 
 int nbd_disconnect(int fd)
 {
@@ -770,85 +614,21 @@ int nbd_disconnect(int fd)
     return 0;
 }
 
-int nbd_client(int fd)
-{
-    int ret;
-    int serrno;
-
-    TRACE("Doing NBD loop");
-
-    ret = ioctl(fd, NBD_DO_IT);
-    if (ret < 0 && errno == EPIPE) {
-        /* NBD_DO_IT normally returns EPIPE when someone has disconnected
-         * the socket via NBD_DISCONNECT.  We do not want to return 1 in
-         * that case.
-         */
-        ret = 0;
-    }
-    serrno = errno;
-
-    TRACE("NBD loop returned %d: %s", ret, strerror(serrno));
-
-    TRACE("Clearing NBD queue");
-    ioctl(fd, NBD_CLEAR_QUE);
-
-    TRACE("Clearing NBD socket");
-    ioctl(fd, NBD_CLEAR_SOCK);
-
-    errno = serrno;
-    return ret;
-}
 #else
-int nbd_init(int fd, int csock, uint32_t flags, off_t size)
-{
-    return -ENOTSUP;
-}
 
 int nbd_disconnect(int fd)
 {
     return -ENOTSUP;
 }
-
-int nbd_client(int fd)
-{
-    return -ENOTSUP;
-}
 #endif
 
-ssize_t nbd_send_request(int csock, struct nbd_request *request)
-{
-    uint8_t buf[NBD_REQUEST_SIZE];
-    ssize_t ret;
-
-    cpu_to_be32w((uint32_t*)buf, NBD_REQUEST_MAGIC);
-    cpu_to_be32w((uint32_t*)(buf + 4), request->type);
-    cpu_to_be64w((uint64_t*)(buf + 8), request->handle);
-    cpu_to_be64w((uint64_t*)(buf + 16), request->from);
-    cpu_to_be32w((uint32_t*)(buf + 24), request->len);
-
-    TRACE("Sending request to client: "
-          "{ .from = %" PRIu64", .len = %u, .handle = %" PRIu64", .type=%i}",
-          request->from, request->len, request->handle, request->type);
-
-    ret = write_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("writing to socket failed");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static ssize_t nbd_receive_request(int csock, struct nbd_request *request)
+static ssize_t nbd_receive_request(QIOChannel *ioc, struct nbd_request *request)
 {
     uint8_t buf[NBD_REQUEST_SIZE];
     uint32_t magic;
     ssize_t ret;
 
-    ret = read_sync(csock, buf, sizeof(buf));
+    ret = read_sync(ioc, buf, sizeof(buf));
     if (ret < 0) {
         return ret;
     }
@@ -883,64 +663,26 @@ static ssize_t nbd_receive_request(int csock, struct nbd_request *request)
     return 0;
 }
 
-ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply)
-{
-    uint8_t buf[NBD_REPLY_SIZE];
-    uint32_t magic;
-    ssize_t ret;
-
-    ret = read_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("read failed");
-        return -EINVAL;
-    }
-
-    /* Reply
-       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
-       [ 4 ..  7]    error   (0 == no error)
-       [ 7 .. 15]    handle
-     */
-
-    magic = be32_to_cpup((uint32_t*)buf);
-    reply->error  = be32_to_cpup((uint32_t*)(buf + 4));
-    reply->handle = be64_to_cpup((uint64_t*)(buf + 8));
-
-    reply->error = nbd_errno_to_system_errno(reply->error);
-
-    TRACE("Got reply: "
-          "{ magic = 0x%x, .error = %d, handle = %" PRIu64" }",
-          magic, reply->error, reply->handle);
-
-    if (magic != NBD_REPLY_MAGIC) {
-        LOG("invalid magic (got 0x%x)", magic);
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
+static ssize_t nbd_send_reply(QIOChannel *ioc, struct nbd_reply *reply)
 {
     uint8_t buf[NBD_REPLY_SIZE];
     ssize_t ret;
 
     reply->error = system_errno_to_nbd_errno(reply->error);
 
+    TRACE("Sending response to client: { .error = %d, handle = %" PRIu64 " }",
+          reply->error, reply->handle);
+
     /* Reply
        [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
        [ 4 ..  7]    error   (0 == no error)
        [ 7 .. 15]    handle
      */
-    cpu_to_be32w((uint32_t*)buf, NBD_REPLY_MAGIC);
-    cpu_to_be32w((uint32_t*)(buf + 4), reply->error);
-    cpu_to_be64w((uint64_t*)(buf + 8), reply->handle);
+    stl_be_p(buf, NBD_REPLY_MAGIC);
+    stl_be_p(buf + 4, reply->error);
+    stq_be_p(buf + 8, reply->handle);
 
-    TRACE("Sending response to client");
-
-    ret = write_sync(csock, buf, sizeof(buf));
+    ret = write_sync(ioc, buf, sizeof(buf));
     if (ret < 0) {
         return ret;
     }
@@ -968,8 +710,12 @@ void nbd_client_put(NBDClient *client)
         assert(client->closing);
 
         nbd_unset_handlers(client);
-        close(client->sock);
-        client->sock = -1;
+        object_unref(OBJECT(client->sioc));
+        object_unref(OBJECT(client->ioc));
+        if (client->tlscreds) {
+            object_unref(OBJECT(client->tlscreds));
+        }
+        g_free(client->tlsaclname);
         if (client->exp) {
             QTAILQ_REMOVE(&client->exp->clients, client, next);
             nbd_export_put(client->exp);
@@ -989,7 +735,8 @@ static void client_close(NBDClient *client)
     /* Force requests to finish.  They will drop their own references,
      * then we'll close the socket and free the NBDClient.
      */
-    shutdown(client->sock, 2);
+    qio_channel_shutdown(client->ioc, QIO_CHANNEL_SHUTDOWN_BOTH,
+                         NULL);
 
     /* Also tell the client, so that they release their reference.  */
     if (client->close) {
@@ -1005,7 +752,7 @@ static NBDRequest *nbd_request_get(NBDClient *client)
     client->nb_requests++;
     nbd_update_can_read(client);
 
-    req = g_slice_new0(NBDRequest);
+    req = g_new0(NBDRequest, 1);
     nbd_client_get(client);
     req->client = client;
     return req;
@@ -1018,7 +765,7 @@ static void nbd_request_put(NBDRequest *req)
     if (req->data) {
         qemu_vfree(req->data);
     }
-    g_slice_free(NBDRequest, req);
+    g_free(req);
 
     client->nb_requests--;
     nbd_update_can_read(client);
@@ -1053,6 +800,12 @@ static void blk_aio_detach(void *opaque)
     exp->ctx = NULL;
 }
 
+static void nbd_eject_notifier(Notifier *n, void *data)
+{
+    NBDExport *exp = container_of(n, NBDExport, eject_notifier);
+    nbd_export_close(exp);
+}
+
 NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
                           uint32_t nbdflags, void (*close)(NBDExport *),
                           Error **errp)
@@ -1075,12 +828,18 @@ NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
     exp->ctx = blk_get_aio_context(blk);
     blk_ref(blk);
     blk_add_aio_context_notifier(blk, blk_aio_attached, blk_aio_detach, exp);
+
+    exp->eject_notifier.notify = nbd_eject_notifier;
+    blk_add_remove_bs_notifier(blk, &exp->eject_notifier);
+
     /*
      * NBD exports are used for non-shared storage migration.  Make sure
-     * that BDRV_O_INCOMING is cleared and the image is ready for write
+     * that BDRV_O_INACTIVE is cleared and the image is ready for write
      * access since the export could be available before migration handover.
      */
+    aio_context_acquire(exp->ctx);
     blk_invalidate_cache(blk, NULL);
+    aio_context_release(exp->ctx);
     return exp;
 
 fail:
@@ -1131,12 +890,6 @@ void nbd_export_close(NBDExport *exp)
     }
     nbd_export_set_name(exp, NULL);
     nbd_export_put(exp);
-    if (exp->blk) {
-        blk_remove_aio_context_notifier(exp->blk, blk_aio_attached,
-                                        blk_aio_detach, exp);
-        blk_unref(exp->blk);
-        exp->blk = NULL;
-    }
 }
 
 void nbd_export_get(NBDExport *exp)
@@ -1159,6 +912,14 @@ void nbd_export_put(NBDExport *exp)
             exp->close(exp);
         }
 
+        if (exp->blk) {
+            notifier_remove(&exp->eject_notifier);
+            blk_remove_aio_context_notifier(exp->blk, blk_aio_attached,
+                                            blk_aio_detach, exp);
+            blk_unref(exp->blk);
+            exp->blk = NULL;
+        }
+
         g_free(exp);
     }
 }
@@ -1181,25 +942,25 @@ static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
                                  int len)
 {
     NBDClient *client = req->client;
-    int csock = client->sock;
     ssize_t rc, ret;
 
+    g_assert(qemu_in_coroutine());
     qemu_co_mutex_lock(&client->send_lock);
     client->send_coroutine = qemu_coroutine_self();
     nbd_set_handlers(client);
 
     if (!len) {
-        rc = nbd_send_reply(csock, reply);
+        rc = nbd_send_reply(client->ioc, reply);
     } else {
-        socket_set_cork(csock, 1);
-        rc = nbd_send_reply(csock, reply);
+        qio_channel_set_cork(client->ioc, true);
+        rc = nbd_send_reply(client->ioc, reply);
         if (rc >= 0) {
-            ret = qemu_co_send(csock, req->data, len);
+            ret = write_sync(client->ioc, req->data, len);
             if (ret != len) {
                 rc = -EIO;
             }
         }
-        socket_set_cork(csock, 0);
+        qio_channel_set_cork(client->ioc, false);
     }
 
     client->send_coroutine = NULL;
@@ -1211,14 +972,14 @@ static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
 static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *request)
 {
     NBDClient *client = req->client;
-    int csock = client->sock;
     uint32_t command;
     ssize_t rc;
 
+    g_assert(qemu_in_coroutine());
     client->recv_coroutine = qemu_coroutine_self();
     nbd_update_can_read(client);
 
-    rc = nbd_receive_request(csock, request);
+    rc = nbd_receive_request(client->ioc, request);
     if (rc < 0) {
         if (rc != -EAGAIN) {
             rc = -EIO;
@@ -1226,13 +987,6 @@ static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *reque
         goto out;
     }
 
-    if (request->len > NBD_MAX_BUFFER_SIZE) {
-        LOG("len (%u) is larger than max len (%u)",
-            request->len, NBD_MAX_BUFFER_SIZE);
-        rc = -EINVAL;
-        goto out;
-    }
-
     if ((request->from + request->len) < request->from) {
         LOG("integer overflow detected! "
             "you're probably being attacked");
@@ -1244,12 +998,23 @@ static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *reque
 
     command = request->type & NBD_CMD_MASK_COMMAND;
     if (command == NBD_CMD_READ || command == NBD_CMD_WRITE) {
-        req->data = blk_blockalign(client->exp->blk, request->len);
+        if (request->len > NBD_MAX_BUFFER_SIZE) {
+            LOG("len (%u) is larger than max len (%u)",
+                request->len, NBD_MAX_BUFFER_SIZE);
+            rc = -EINVAL;
+            goto out;
+        }
+
+        req->data = blk_try_blockalign(client->exp->blk, request->len);
+        if (req->data == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
     }
     if (command == NBD_CMD_WRITE) {
         TRACE("Reading %u byte(s)", request->len);
 
-        if (qemu_co_recv(csock, req->data, request->len) != request->len) {
+        if (read_sync(client->ioc, req->data, request->len) != request->len) {
             LOG("reading from socket failed");
             rc = -EIO;
             goto out;
@@ -1305,6 +1070,14 @@ static void nbd_trip(void *opaque)
         goto invalid_request;
     }
 
+    if (client->closing) {
+        /*
+         * The client may be closed when we are blocked in
+         * nbd_co_receive_request()
+         */
+        goto done;
+    }
+
     switch (command) {
     case NBD_CMD_READ:
         TRACE("Request type is READ");
@@ -1318,9 +1091,8 @@ static void nbd_trip(void *opaque)
             }
         }
 
-        ret = blk_read(exp->blk,
-                       (request.from + exp->dev_offset) / BDRV_SECTOR_SIZE,
-                       req->data, request.len / BDRV_SECTOR_SIZE);
+        ret = blk_pread(exp->blk, request.from + exp->dev_offset,
+                        req->data, request.len);
         if (ret < 0) {
             LOG("reading from file failed");
             reply.error = -ret;
@@ -1342,9 +1114,8 @@ static void nbd_trip(void *opaque)
 
         TRACE("Writing to device");
 
-        ret = blk_write(exp->blk,
-                        (request.from + exp->dev_offset) / BDRV_SECTOR_SIZE,
-                        req->data, request.len / BDRV_SECTOR_SIZE);
+        ret = blk_pwrite(exp->blk, request.from + exp->dev_offset,
+                        req->data, request.len);
         if (ret < 0) {
             LOG("writing to file failed");
             reply.error = -ret;
@@ -1436,7 +1207,8 @@ static void nbd_restart_write(void *opaque)
 static void nbd_set_handlers(NBDClient *client)
 {
     if (client->exp && client->exp->ctx) {
-        aio_set_fd_handler(client->exp->ctx, client->sock,
+        aio_set_fd_handler(client->exp->ctx, client->sioc->fd,
+                           true,
                            client->can_read ? nbd_read : NULL,
                            client->send_coroutine ? nbd_restart_write : NULL,
                            client);
@@ -1446,7 +1218,8 @@ static void nbd_set_handlers(NBDClient *client)
 static void nbd_unset_handlers(NBDClient *client)
 {
     if (client->exp && client->exp->ctx) {
-        aio_set_fd_handler(client->exp->ctx, client->sock, NULL, NULL, NULL);
+        aio_set_fd_handler(client->exp->ctx, client->sioc->fd,
+                           true, NULL, NULL, NULL);
     }
 }
 
@@ -1464,26 +1237,54 @@ static void nbd_update_can_read(NBDClient *client)
     }
 }
 
-NBDClient *nbd_client_new(NBDExport *exp, int csock,
-                          void (*close)(NBDClient *))
+static coroutine_fn void nbd_co_client_start(void *opaque)
 {
-    NBDClient *client;
-    client = g_malloc0(sizeof(NBDClient));
-    client->refcount = 1;
-    client->exp = exp;
-    client->sock = csock;
-    client->can_read = true;
-    if (nbd_send_negotiate(client)) {
-        g_free(client);
-        return NULL;
+    NBDClientNewData *data = opaque;
+    NBDClient *client = data->client;
+    NBDExport *exp = client->exp;
+
+    if (exp) {
+        nbd_export_get(exp);
+    }
+    if (nbd_negotiate(data)) {
+        client_close(client);
+        goto out;
     }
-    client->close = close;
     qemu_co_mutex_init(&client->send_lock);
     nbd_set_handlers(client);
 
     if (exp) {
         QTAILQ_INSERT_TAIL(&exp->clients, client, next);
-        nbd_export_get(exp);
     }
-    return client;
+out:
+    g_free(data);
+}
+
+void nbd_client_new(NBDExport *exp,
+                    QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
+                    void (*close_fn)(NBDClient *))
+{
+    NBDClient *client;
+    NBDClientNewData *data = g_new(NBDClientNewData, 1);
+
+    client = g_malloc0(sizeof(NBDClient));
+    client->refcount = 1;
+    client->exp = exp;
+    client->tlscreds = tlscreds;
+    if (tlscreds) {
+        object_ref(OBJECT(client->tlscreds));
+    }
+    client->tlsaclname = g_strdup(tlsaclname);
+    client->sioc = sioc;
+    object_ref(OBJECT(client->sioc));
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    client->can_read = true;
+    client->close = close_fn;
+
+    data->client = client;
+    data->co = qemu_coroutine_create(nbd_co_client_start);
+    qemu_coroutine_enter(data->co, data);
 }
diff --git a/qemu/net/Makefile.objs b/qemu/net/Makefile.objs
index ec19cb31d..b7c22fddb 100644
--- a/qemu/net/Makefile.objs
+++ b/qemu/net/Makefile.objs
@@ -13,3 +13,6 @@ common-obj-$(CONFIG_HAIKU) += tap-haiku.o
 common-obj-$(CONFIG_SLIRP) += slirp.o
 common-obj-$(CONFIG_VDE) += vde.o
 common-obj-$(CONFIG_NETMAP) += netmap.o
+common-obj-y += filter.o
+common-obj-y += filter-buffer.o
+common-obj-y += filter-mirror.o
diff --git a/qemu/net/checksum.c b/qemu/net/checksum.c
index 14c08550e..d0fa424cc 100644
--- a/qemu/net/checksum.c
+++ b/qemu/net/checksum.c
@@ -15,6 +15,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "net/checksum.h"
 
@@ -59,6 +60,11 @@ void net_checksum_calculate(uint8_t *data, int length)
     int hlen, plen, proto, csum_offset;
     uint16_t csum;
 
+    /* Ensure data has complete L2 & L3 headers. */
+    if (length < 14 + 20) {
+        return;
+    }
+
     if ((data[14] & 0xf0) != 0x40)
 	return; /* not IPv4 */
     hlen  = (data[14] & 0x0f) * 4;
@@ -76,8 +82,9 @@ void net_checksum_calculate(uint8_t *data, int length)
 	return;
     }
 
-    if (plen < csum_offset+2)
-	return;
+    if (plen < csum_offset + 2 || 14 + hlen + plen > length) {
+        return;
+    }
 
     data[14+hlen+csum_offset]   = 0;
     data[14+hlen+csum_offset+1] = 0;
diff --git a/qemu/net/dump.c b/qemu/net/dump.c
index 02c8064be..41f7673ef 100644
--- a/qemu/net/dump.c
+++ b/qemu/net/dump.c
@@ -22,15 +22,18 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "clients.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
+#include "qemu/iov.h"
 #include "qemu/log.h"
 #include "qemu/timer.h"
-#include "hub.h"
+#include "qapi/visitor.h"
+#include "net/filter.h"
 
 typedef struct DumpState {
-    NetClientState nc;
     int64_t start_ts;
     int fd;
     int pcap_caplen;
@@ -57,28 +60,33 @@ struct pcap_sf_pkthdr {
     uint32_t len;
 };
 
-static ssize_t dump_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
     struct pcap_sf_pkthdr hdr;
     int64_t ts;
     int caplen;
+    size_t size = iov_size(iov, cnt);
+    struct iovec dumpiov[cnt + 1];
 
     /* Early return in case of previous error. */
     if (s->fd < 0) {
         return size;
     }
 
-    ts = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 1000000, get_ticks_per_sec());
+    ts = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL);
     caplen = size > s->pcap_caplen ? s->pcap_caplen : size;
 
     hdr.ts.tv_sec = ts / 1000000 + s->start_ts;
     hdr.ts.tv_usec = ts % 1000000;
     hdr.caplen = caplen;
     hdr.len = size;
-    if (write(s->fd, &hdr, sizeof(hdr)) != sizeof(hdr) ||
-        write(s->fd, buf, caplen) != caplen) {
-        qemu_log("-net dump write error - stop dump\n");
+
+    dumpiov[0].iov_base = &hdr;
+    dumpiov[0].iov_len = sizeof(hdr);
+    cnt = iov_copy(&dumpiov[1], cnt, iov, cnt, 0, caplen);
+
+    if (writev(s->fd, dumpiov, cnt + 1) != sizeof(hdr) + caplen) {
+        error_report("network dump write error - stopping dump");
         close(s->fd);
         s->fd = -1;
     }
@@ -86,27 +94,16 @@ static ssize_t dump_receive(NetClientState *nc, const uint8_t *buf, size_t size)
     return size;
 }
 
-static void dump_cleanup(NetClientState *nc)
+static void dump_cleanup(DumpState *s)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
-
     close(s->fd);
+    s->fd = -1;
 }
 
-static NetClientInfo net_dump_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_DUMP,
-    .size = sizeof(DumpState),
-    .receive = dump_receive,
-    .cleanup = dump_cleanup,
-};
-
-static int net_dump_init(NetClientState *peer, const char *device,
-                         const char *name, const char *filename, int len,
-                         Error **errp)
+static int net_dump_state_init(DumpState *s, const char *filename,
+                               int len, Error **errp)
 {
     struct pcap_file_hdr hdr;
-    NetClientState *nc;
-    DumpState *s;
     struct tm tm;
     int fd;
 
@@ -130,13 +127,6 @@ static int net_dump_init(NetClientState *peer, const char *device,
         return -1;
     }
 
-    nc = qemu_new_net_client(&net_dump_info, peer, device, name);
-
-    snprintf(nc->info_str, sizeof(nc->info_str),
-             "dump to %s (len=%d)", filename, len);
-
-    s = DO_UPCAST(DumpState, nc, nc);
-
     s->fd = fd;
     s->pcap_caplen = len;
 
@@ -146,16 +136,61 @@ static int net_dump_init(NetClientState *peer, const char *device,
     return 0;
 }
 
+/* Dumping via VLAN netclient */
+
+struct DumpNetClient {
+    NetClientState nc;
+    DumpState ds;
+};
+typedef struct DumpNetClient DumpNetClient;
+
+static ssize_t dumpclient_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = size
+    };
+
+    return dump_receive_iov(&dc->ds, &iov, 1);
+}
+
+static ssize_t dumpclient_receive_iov(NetClientState *nc,
+                                      const struct iovec *iov, int cnt)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    return dump_receive_iov(&dc->ds, iov, cnt);
+}
+
+static void dumpclient_cleanup(NetClientState *nc)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    dump_cleanup(&dc->ds);
+}
+
+static NetClientInfo net_dump_info = {
+    .type = NET_CLIENT_OPTIONS_KIND_DUMP,
+    .size = sizeof(DumpNetClient),
+    .receive = dumpclient_receive,
+    .receive_iov = dumpclient_receive_iov,
+    .cleanup = dumpclient_cleanup,
+};
+
 int net_init_dump(const NetClientOptions *opts, const char *name,
                   NetClientState *peer, Error **errp)
 {
-    int len;
+    int len, rc;
     const char *file;
     char def_file[128];
     const NetdevDumpOptions *dump;
+    NetClientState *nc;
+    DumpNetClient *dnc;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_DUMP);
-    dump = opts->dump;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_DUMP);
+    dump = opts->u.dump.data;
 
     assert(peer);
 
@@ -182,5 +217,148 @@ int net_init_dump(const NetClientOptions *opts, const char *name,
         len = 65536;
     }
 
-    return net_dump_init(peer, "dump", name, file, len, errp);
+    nc = qemu_new_net_client(&net_dump_info, peer, "dump", name);
+    snprintf(nc->info_str, sizeof(nc->info_str),
+             "dump to %s (len=%d)", file, len);
+
+    dnc = DO_UPCAST(DumpNetClient, nc, nc);
+    rc = net_dump_state_init(&dnc->ds, file, len, errp);
+    if (rc) {
+        qemu_del_net_client(nc);
+    }
+    return rc;
+}
+
+/* Dumping via filter */
+
+#define TYPE_FILTER_DUMP "filter-dump"
+
+#define FILTER_DUMP(obj) \
+    OBJECT_CHECK(NetFilterDumpState, (obj), TYPE_FILTER_DUMP)
+
+struct NetFilterDumpState {
+    NetFilterState nfs;
+    DumpState ds;
+    char *filename;
+    uint32_t maxlen;
+};
+typedef struct NetFilterDumpState NetFilterDumpState;
+
+static ssize_t filter_dump_receive_iov(NetFilterState *nf, NetClientState *sndr,
+                                       unsigned flags, const struct iovec *iov,
+                                       int iovcnt, NetPacketSent *sent_cb)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_receive_iov(&nfds->ds, iov, iovcnt);
+    return 0;
 }
+
+static void filter_dump_cleanup(NetFilterState *nf)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_cleanup(&nfds->ds);
+}
+
+static void filter_dump_setup(NetFilterState *nf, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    if (!nfds->filename) {
+        error_setg(errp, "dump filter needs 'file' property set!");
+        return;
+    }
+
+    net_dump_state_init(&nfds->ds, nfds->filename, nfds->maxlen, errp);
+}
+
+static void filter_dump_get_maxlen(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    uint32_t value = nfds->maxlen;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void filter_dump_set_maxlen(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (value == 0) {
+        error_setg(&local_err, "Property '%s.%s' doesn't take value '%u'",
+                   object_get_typename(obj), name, value);
+        goto out;
+    }
+    nfds->maxlen = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static char *file_dump_get_filename(Object *obj, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    return g_strdup(nfds->filename);
+}
+
+static void file_dump_set_filename(Object *obj, const char *value, Error **errp)
+{
+   NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+    nfds->filename = g_strdup(value);
+}
+
+static void filter_dump_instance_init(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    nfds->maxlen = 65536;
+
+    object_property_add(obj, "maxlen", "int", filter_dump_get_maxlen,
+                        filter_dump_set_maxlen, NULL, NULL, NULL);
+    object_property_add_str(obj, "file", file_dump_get_filename,
+                            file_dump_set_filename, NULL);
+}
+
+static void filter_dump_instance_finalize(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+}
+
+static void filter_dump_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_dump_setup;
+    nfc->cleanup = filter_dump_cleanup;
+    nfc->receive_iov = filter_dump_receive_iov;
+}
+
+static const TypeInfo filter_dump_info = {
+    .name = TYPE_FILTER_DUMP,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_dump_class_init,
+    .instance_init = filter_dump_instance_init,
+    .instance_finalize = filter_dump_instance_finalize,
+    .instance_size = sizeof(NetFilterDumpState),
+};
+
+static void filter_dump_register_types(void)
+{
+    type_register_static(&filter_dump_info);
+}
+
+type_init(filter_dump_register_types);
diff --git a/qemu/net/eth.c b/qemu/net/eth.c
index 7c61132cb..7e32d274c 100644
--- a/qemu/net/eth.c
+++ b/qemu/net/eth.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "net/eth.h"
 #include "net/checksum.h"
 #include "qemu-common.h"
diff --git a/qemu/net/filter-buffer.c b/qemu/net/filter-buffer.c
new file mode 100644
index 000000000..cc6bd9444
--- /dev/null
+++ b/qemu/net/filter-buffer.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "net/filter.h"
+#include "net/queue.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/timer.h"
+#include "qemu/iov.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi-visit.h"
+#include "qom/object.h"
+
+#define TYPE_FILTER_BUFFER "filter-buffer"
+
+#define FILTER_BUFFER(obj) \
+    OBJECT_CHECK(FilterBufferState, (obj), TYPE_FILTER_BUFFER)
+
+typedef struct FilterBufferState {
+    NetFilterState parent_obj;
+
+    NetQueue *incoming_queue;
+    uint32_t interval;
+    QEMUTimer release_timer;
+} FilterBufferState;
+
+static void filter_buffer_flush(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (!qemu_net_queue_flush(s->incoming_queue)) {
+        /* Unable to empty the queue, purge remaining packets */
+        qemu_net_queue_purge(s->incoming_queue, nf->netdev);
+    }
+}
+
+static void filter_buffer_release_timer(void *opaque)
+{
+    NetFilterState *nf = opaque;
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * Note: filter_buffer_flush() drops packets that can't be sent
+     * TODO: We should leave them queued.  But currently there's no way
+     * for the next filter or receiver to notify us that it can receive
+     * more packets.
+     */
+    filter_buffer_flush(nf);
+    /* Timer rearmed to fire again in s->interval microseconds. */
+    timer_mod(&s->release_timer,
+              qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+}
+
+/* filter APIs */
+static ssize_t filter_buffer_receive_iov(NetFilterState *nf,
+                                         NetClientState *sender,
+                                         unsigned flags,
+                                         const struct iovec *iov,
+                                         int iovcnt,
+                                         NetPacketSent *sent_cb)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We return size when buffer a packet, the sender will take it as
+     * a already sent packet, so sent_cb should not be called later.
+     *
+     * FIXME: Even if the guest can't receive packets for some reasons,
+     * the filter can still accept packets until its internal queue is full.
+     * For example:
+     *   For some reason, receiver could not receive more packets
+     * (.can_receive() returns zero). Without a filter, at most one packet
+     * will be queued in incoming queue and sender's poll will be disabled
+     * unit its sent_cb() was called. With a filter, it will keep receiving
+     * the packets without caring about the receiver. This is suboptimal.
+     * May need more thoughts (e.g keeping sent_cb).
+     */
+    qemu_net_queue_append_iov(s->incoming_queue, sender, flags,
+                              iov, iovcnt, NULL);
+    return iov_size(iov, iovcnt);
+}
+
+static void filter_buffer_cleanup(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (s->interval) {
+        timer_del(&s->release_timer);
+    }
+
+    /* flush packets */
+    if (s->incoming_queue) {
+        filter_buffer_flush(nf);
+        g_free(s->incoming_queue);
+    }
+}
+
+static void filter_buffer_setup_timer(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (s->interval) {
+        timer_init_us(&s->release_timer, QEMU_CLOCK_VIRTUAL,
+                      filter_buffer_release_timer, nf);
+        /* Timer armed to fire in s->interval microseconds. */
+        timer_mod(&s->release_timer,
+                  qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+    }
+}
+
+static void filter_buffer_setup(NetFilterState *nf, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We may want to accept zero interval when VM FT solutions like MC
+     * or COLO use this filter to release packets on demand.
+     */
+    if (!s->interval) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "interval",
+                   "a non-zero interval");
+        return;
+    }
+
+    s->incoming_queue = qemu_new_net_queue(qemu_netfilter_pass_to_next, nf);
+    filter_buffer_setup_timer(nf);
+}
+
+static void filter_buffer_status_changed(NetFilterState *nf, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (!nf->on) {
+        if (s->interval) {
+            timer_del(&s->release_timer);
+        }
+        filter_buffer_flush(nf);
+    } else {
+        filter_buffer_setup_timer(nf);
+    }
+}
+
+static void filter_buffer_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_buffer_setup;
+    nfc->cleanup = filter_buffer_cleanup;
+    nfc->receive_iov = filter_buffer_receive_iov;
+    nfc->status_changed = filter_buffer_status_changed;
+}
+
+static void filter_buffer_get_interval(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    uint32_t value = s->interval;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void filter_buffer_set_interval(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' requires a positive value",
+                   object_get_typename(obj), name);
+        goto out;
+    }
+    s->interval = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void filter_buffer_init(Object *obj)
+{
+    object_property_add(obj, "interval", "int",
+                        filter_buffer_get_interval,
+                        filter_buffer_set_interval, NULL, NULL, NULL);
+}
+
+static const TypeInfo filter_buffer_info = {
+    .name = TYPE_FILTER_BUFFER,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_buffer_class_init,
+    .instance_init = filter_buffer_init,
+    .instance_size = sizeof(FilterBufferState),
+};
+
+static void register_types(void)
+{
+    type_register_static(&filter_buffer_info);
+}
+
+type_init(register_types);
diff --git a/qemu/net/filter-mirror.c b/qemu/net/filter-mirror.c
new file mode 100644
index 000000000..c0c4dc60b
--- /dev/null
+++ b/qemu/net/filter-mirror.c
@@ -0,0 +1,427 @@
+/*
+ * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "net/filter.h"
+#include "net/net.h"
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi-visit.h"
+#include "qom/object.h"
+#include "qemu/main-loop.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+#include "sysemu/char.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+
+#define FILTER_MIRROR(obj) \
+    OBJECT_CHECK(MirrorState, (obj), TYPE_FILTER_MIRROR)
+
+#define FILTER_REDIRECTOR(obj) \
+    OBJECT_CHECK(MirrorState, (obj), TYPE_FILTER_REDIRECTOR)
+
+#define TYPE_FILTER_MIRROR "filter-mirror"
+#define TYPE_FILTER_REDIRECTOR "filter-redirector"
+#define REDIRECTOR_MAX_LEN NET_BUFSIZE
+
+typedef struct MirrorState {
+    NetFilterState parent_obj;
+    char *indev;
+    char *outdev;
+    CharDriverState *chr_in;
+    CharDriverState *chr_out;
+    int state; /* 0 = getting length, 1 = getting data */
+    unsigned int index;
+    unsigned int packet_len;
+    uint8_t buf[REDIRECTOR_MAX_LEN];
+} MirrorState;
+
+static int filter_mirror_send(CharDriverState *chr_out,
+                              const struct iovec *iov,
+                              int iovcnt)
+{
+    int ret = 0;
+    ssize_t size = 0;
+    uint32_t len =  0;
+    char *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (!size) {
+        return 0;
+    }
+
+    len = htonl(size);
+    ret = qemu_chr_fe_write_all(chr_out, (uint8_t *)&len, sizeof(len));
+    if (ret != sizeof(len)) {
+        goto err;
+    }
+
+    buf = g_malloc(size);
+    iov_to_buf(iov, iovcnt, 0, buf, size);
+    ret = qemu_chr_fe_write_all(chr_out, (uint8_t *)buf, size);
+    g_free(buf);
+    if (ret != size) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+    return ret < 0 ? ret : -EIO;
+}
+
+static void
+redirector_to_filter(NetFilterState *nf, const uint8_t *buf, int len)
+{
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = len,
+    };
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL ||
+        nf->direction == NET_FILTER_DIRECTION_TX) {
+        qemu_netfilter_pass_to_next(nf->netdev, 0, &iov, 1, nf);
+    }
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL ||
+        nf->direction == NET_FILTER_DIRECTION_RX) {
+        qemu_netfilter_pass_to_next(nf->netdev->peer, 0, &iov, 1, nf);
+     }
+}
+
+static int redirector_chr_can_read(void *opaque)
+{
+    return REDIRECTOR_MAX_LEN;
+}
+
+static void redirector_chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    NetFilterState *nf = opaque;
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+    unsigned int l;
+
+    while (size > 0) {
+        /* reassemble a packet from the network */
+        switch (s->state) { /* 0 = getting length, 1 = getting data */
+        case 0:
+            l = 4 - s->index;
+            if (l > size) {
+                l = size;
+            }
+            memcpy(s->buf + s->index, buf, l);
+            buf += l;
+            size -= l;
+            s->index += l;
+            if (s->index == 4) {
+                /* got length */
+                s->packet_len = ntohl(*(uint32_t *)s->buf);
+                s->index = 0;
+                s->state = 1;
+            }
+            break;
+        case 1:
+            l = s->packet_len - s->index;
+            if (l > size) {
+                l = size;
+            }
+            if (s->index + l <= sizeof(s->buf)) {
+                memcpy(s->buf + s->index, buf, l);
+            } else {
+                error_report("serious error: oversized packet received.");
+                s->index = s->state = 0;
+                qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+                return;
+            }
+
+            s->index += l;
+            buf += l;
+            size -= l;
+            if (s->index >= s->packet_len) {
+                s->index = 0;
+                s->state = 0;
+                redirector_to_filter(nf, s->buf, s->packet_len);
+            }
+            break;
+        }
+    }
+}
+
+static void redirector_chr_event(void *opaque, int event)
+{
+    NetFilterState *nf = opaque;
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    switch (event) {
+    case CHR_EVENT_CLOSED:
+        qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+        break;
+    default:
+        break;
+    }
+}
+
+static ssize_t filter_mirror_receive_iov(NetFilterState *nf,
+                                         NetClientState *sender,
+                                         unsigned flags,
+                                         const struct iovec *iov,
+                                         int iovcnt,
+                                         NetPacketSent *sent_cb)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+    int ret;
+
+    ret = filter_mirror_send(s->chr_out, iov, iovcnt);
+    if (ret) {
+        error_report("filter_mirror_send failed(%s)", strerror(-ret));
+    }
+
+    /*
+     * we don't hope this error interrupt the normal
+     * path of net packet, so we always return zero.
+     */
+    return 0;
+}
+
+static ssize_t filter_redirector_receive_iov(NetFilterState *nf,
+                                             NetClientState *sender,
+                                             unsigned flags,
+                                             const struct iovec *iov,
+                                             int iovcnt,
+                                             NetPacketSent *sent_cb)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+    int ret;
+
+    if (s->chr_out) {
+        ret = filter_mirror_send(s->chr_out, iov, iovcnt);
+        if (ret) {
+            error_report("filter_mirror_send failed(%s)", strerror(-ret));
+        }
+        return iov_size(iov, iovcnt);
+    } else {
+        return 0;
+    }
+}
+
+static void filter_mirror_cleanup(NetFilterState *nf)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+
+    if (s->chr_out) {
+        qemu_chr_fe_release(s->chr_out);
+    }
+}
+
+static void filter_redirector_cleanup(NetFilterState *nf)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    if (s->chr_in) {
+        qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+        qemu_chr_fe_release(s->chr_in);
+    }
+    if (s->chr_out) {
+        qemu_chr_fe_release(s->chr_out);
+    }
+}
+
+static void filter_mirror_setup(NetFilterState *nf, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+
+    if (!s->outdev) {
+        error_setg(errp, "filter filter mirror needs 'outdev' "
+                   "property set");
+        return;
+    }
+
+    s->chr_out = qemu_chr_find(s->outdev);
+    if (s->chr_out == NULL) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", s->outdev);
+        return;
+    }
+
+    if (qemu_chr_fe_claim(s->chr_out) != 0) {
+        error_setg(errp, QERR_DEVICE_IN_USE, s->outdev);
+        return;
+    }
+}
+
+static void filter_redirector_setup(NetFilterState *nf, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    if (!s->indev && !s->outdev) {
+        error_setg(errp, "filter redirector needs 'indev' or "
+                   "'outdev' at least one property set");
+        return;
+    } else if (s->indev && s->outdev) {
+        if (!strcmp(s->indev, s->outdev)) {
+            error_setg(errp, "'indev' and 'outdev' could not be same "
+                       "for filter redirector");
+            return;
+        }
+    }
+
+    s->state = s->index = 0;
+
+    if (s->indev) {
+        s->chr_in = qemu_chr_find(s->indev);
+        if (s->chr_in == NULL) {
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "IN Device '%s' not found", s->indev);
+            return;
+        }
+
+        qemu_chr_fe_claim_no_fail(s->chr_in);
+        qemu_chr_add_handlers(s->chr_in, redirector_chr_can_read,
+                              redirector_chr_read, redirector_chr_event, nf);
+    }
+
+    if (s->outdev) {
+        s->chr_out = qemu_chr_find(s->outdev);
+        if (s->chr_out == NULL) {
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "OUT Device '%s' not found", s->outdev);
+            return;
+        }
+        qemu_chr_fe_claim_no_fail(s->chr_out);
+    }
+}
+
+static void filter_mirror_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_mirror_setup;
+    nfc->cleanup = filter_mirror_cleanup;
+    nfc->receive_iov = filter_mirror_receive_iov;
+}
+
+static void filter_redirector_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_redirector_setup;
+    nfc->cleanup = filter_redirector_cleanup;
+    nfc->receive_iov = filter_redirector_receive_iov;
+}
+
+static char *filter_redirector_get_indev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    return g_strdup(s->indev);
+}
+
+static void
+filter_redirector_set_indev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->indev);
+    s->indev = g_strdup(value);
+}
+
+static char *filter_mirror_get_outdev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    return g_strdup(s->outdev);
+}
+
+static void
+filter_mirror_set_outdev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    g_free(s->outdev);
+    s->outdev = g_strdup(value);
+    if (!s->outdev) {
+        error_setg(errp, "filter filter mirror needs 'outdev' "
+                   "property set");
+        return;
+    }
+}
+
+static char *filter_redirector_get_outdev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    return g_strdup(s->outdev);
+}
+
+static void
+filter_redirector_set_outdev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->outdev);
+    s->outdev = g_strdup(value);
+}
+
+static void filter_mirror_init(Object *obj)
+{
+    object_property_add_str(obj, "outdev", filter_mirror_get_outdev,
+                            filter_mirror_set_outdev, NULL);
+}
+
+static void filter_redirector_init(Object *obj)
+{
+    object_property_add_str(obj, "indev", filter_redirector_get_indev,
+                            filter_redirector_set_indev, NULL);
+    object_property_add_str(obj, "outdev", filter_redirector_get_outdev,
+                            filter_redirector_set_outdev, NULL);
+}
+
+static void filter_mirror_fini(Object *obj)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    g_free(s->outdev);
+}
+
+static void filter_redirector_fini(Object *obj)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->indev);
+    g_free(s->outdev);
+}
+
+static const TypeInfo filter_redirector_info = {
+    .name = TYPE_FILTER_REDIRECTOR,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_redirector_class_init,
+    .instance_init = filter_redirector_init,
+    .instance_finalize = filter_redirector_fini,
+    .instance_size = sizeof(MirrorState),
+};
+
+static const TypeInfo filter_mirror_info = {
+    .name = TYPE_FILTER_MIRROR,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_mirror_class_init,
+    .instance_init = filter_mirror_init,
+    .instance_finalize = filter_mirror_fini,
+    .instance_size = sizeof(MirrorState),
+};
+
+static void register_types(void)
+{
+    type_register_static(&filter_mirror_info);
+    type_register_static(&filter_redirector_info);
+}
+
+type_init(register_types);
diff --git a/qemu/net/filter.c b/qemu/net/filter.c
new file mode 100644
index 000000000..8ac79f3b7
--- /dev/null
+++ b/qemu/net/filter.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+
+#include "net/filter.h"
+#include "net/net.h"
+#include "net/vhost_net.h"
+#include "qom/object_interfaces.h"
+#include "qemu/iov.h"
+
+static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
+{
+    return !nf->on;
+}
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
+{
+    if (qemu_can_skip_netfilter(nf)) {
+        return 0;
+    }
+    if (nf->direction == direction ||
+        nf->direction == NET_FILTER_DIRECTION_ALL) {
+        return NETFILTER_GET_CLASS(OBJECT(nf))->receive_iov(
+                                   nf, sender, flags, iov, iovcnt, sent_cb);
+    }
+
+    return 0;
+}
+
+static NetFilterState *netfilter_next(NetFilterState *nf,
+                                      NetFilterDirection dir)
+{
+    NetFilterState *next;
+
+    if (dir == NET_FILTER_DIRECTION_TX) {
+        /* forward walk through filters */
+        next = QTAILQ_NEXT(nf, next);
+    } else {
+        /* reverse order */
+        next = QTAILQ_PREV(nf, NetFilterHead, next);
+    }
+
+    return next;
+}
+
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque)
+{
+    int ret = 0;
+    int direction;
+    NetFilterState *nf = opaque;
+    NetFilterState *next = NULL;
+
+    if (!sender || !sender->peer) {
+        /* no receiver, or sender been deleted, no need to pass it further */
+        goto out;
+    }
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL) {
+        if (sender == nf->netdev) {
+            /* This packet is sent by netdev itself */
+            direction = NET_FILTER_DIRECTION_TX;
+        } else {
+            direction = NET_FILTER_DIRECTION_RX;
+        }
+    } else {
+        direction = nf->direction;
+    }
+
+    next = netfilter_next(nf, direction);
+    while (next) {
+        /*
+         * if qemu_netfilter_pass_to_next been called, means that
+         * the packet has been hold by filter and has already retured size
+         * to the sender, so sent_cb shouldn't be called later, just
+         * pass NULL to next.
+         */
+        ret = qemu_netfilter_receive(next, direction, sender, flags, iov,
+                                     iovcnt, NULL);
+        if (ret) {
+            return ret;
+        }
+        next = netfilter_next(next, direction);
+    }
+
+    /*
+     * We have gone through all filters, pass it to receiver.
+     * Do the valid check again incase sender or receiver been
+     * deleted while we go through filters.
+     */
+    if (sender && sender->peer) {
+        qemu_net_queue_send_iov(sender->peer->incoming_queue,
+                                sender, flags, iov, iovcnt, NULL);
+    }
+
+out:
+    /* no receiver, or sender been deleted */
+    return iov_size(iov, iovcnt);
+}
+
+static char *netfilter_get_netdev_id(Object *obj, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    return g_strdup(nf->netdev_id);
+}
+
+static void netfilter_set_netdev_id(Object *obj, const char *str, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    nf->netdev_id = g_strdup(str);
+}
+
+static int netfilter_get_direction(Object *obj, Error **errp G_GNUC_UNUSED)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    return nf->direction;
+}
+
+static void netfilter_set_direction(Object *obj, int direction, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    nf->direction = direction;
+}
+
+static char *netfilter_get_status(Object *obj, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    return nf->on ? g_strdup("on") : g_strdup("off");
+}
+
+static void netfilter_set_status(Object *obj, const char *str, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(obj);
+
+    if (strcmp(str, "on") && strcmp(str, "off")) {
+        error_setg(errp, "Invalid value for netfilter status, "
+                         "should be 'on' or 'off'");
+        return;
+    }
+    if (nf->on == !strcmp(str, "on")) {
+        return;
+    }
+    nf->on = !nf->on;
+    if (nf->netdev && nfc->status_changed) {
+        nfc->status_changed(nf, errp);
+    }
+}
+
+static void netfilter_init(Object *obj)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    nf->on = true;
+
+    object_property_add_str(obj, "netdev",
+                            netfilter_get_netdev_id, netfilter_set_netdev_id,
+                            NULL);
+    object_property_add_enum(obj, "queue", "NetFilterDirection",
+                             NetFilterDirection_lookup,
+                             netfilter_get_direction, netfilter_set_direction,
+                             NULL);
+    object_property_add_str(obj, "status",
+                            netfilter_get_status, netfilter_set_status,
+                            NULL);
+}
+
+static void netfilter_complete(UserCreatable *uc, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(uc);
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(uc);
+    int queues;
+    Error *local_err = NULL;
+
+    if (!nf->netdev_id) {
+        error_setg(errp, "Parameter 'netdev' is required");
+        return;
+    }
+
+    queues = qemu_find_net_clients_except(nf->netdev_id, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    if (queues < 1) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "netdev",
+                   "a network backend id");
+        return;
+    } else if (queues > 1) {
+        error_setg(errp, "multiqueue is not supported");
+        return;
+    }
+
+    if (get_vhost_net(ncs[0])) {
+        error_setg(errp, "Vhost is not supported");
+        return;
+    }
+
+    nf->netdev = ncs[0];
+
+    if (nfc->setup) {
+        nfc->setup(nf, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+    QTAILQ_INSERT_TAIL(&nf->netdev->filters, nf, next);
+}
+
+static void netfilter_finalize(Object *obj)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(obj);
+
+    if (nfc->cleanup) {
+        nfc->cleanup(nf);
+    }
+
+    if (nf->netdev && !QTAILQ_EMPTY(&nf->netdev->filters) &&
+        nf->next.tqe_prev) {
+        QTAILQ_REMOVE(&nf->netdev->filters, nf, next);
+    }
+    g_free(nf->netdev_id);
+}
+
+static void netfilter_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = netfilter_complete;
+}
+
+static const TypeInfo netfilter_info = {
+    .name = TYPE_NETFILTER,
+    .parent = TYPE_OBJECT,
+    .abstract = true,
+    .class_size = sizeof(NetFilterClass),
+    .class_init = netfilter_class_init,
+    .instance_size = sizeof(NetFilterState),
+    .instance_init = netfilter_init,
+    .instance_finalize = netfilter_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&netfilter_info);
+}
+
+type_init(register_types);
diff --git a/qemu/net/hub.c b/qemu/net/hub.c
index 3047f1276..6d90c6ee6 100644
--- a/qemu/net/hub.c
+++ b/qemu/net/hub.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "monitor/monitor.h"
 #include "net/net.h"
 #include "clients.h"
@@ -285,9 +286,9 @@ int net_init_hubport(const NetClientOptions *opts, const char *name,
 {
     const NetdevHubPortOptions *hubport;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_HUBPORT);
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_HUBPORT);
     assert(!peer);
-    hubport = opts->hubport;
+    hubport = opts->u.hubport.data;
 
     net_hub_add_port(hubport->hubid, name);
     return 0;
diff --git a/qemu/net/l2tpv3.c b/qemu/net/l2tpv3.c
index 4f9bceecc..5c668f737 100644
--- a/qemu/net/l2tpv3.c
+++ b/qemu/net/l2tpv3.c
@@ -23,9 +23,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <linux/ip.h>
 #include <netdb.h>
-#include "config-host.h"
 #include "net/net.h"
 #include "clients.h"
 #include "qemu-common.h"
@@ -325,7 +325,7 @@ static int l2tpv3_verify_header(NetL2TPV3State *s, uint8_t *buf)
         if (s->cookie_is_64) {
             cookie = ldq_be_p(buf + s->cookie_offset);
         } else {
-            cookie = ldl_be_p(buf + s->cookie_offset);
+            cookie = ldl_be_p(buf + s->cookie_offset) & 0xffffffffULL;
         }
         if (cookie != s->rx_cookie) {
             if (!s->header_mismatch) {
@@ -545,8 +545,8 @@ int net_init_l2tpv3(const NetClientOptions *opts,
     s->queue_tail = 0;
     s->header_mismatch = false;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_L2TPV3);
-    l2tpv3 = opts->l2tpv3;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_L2TPV3);
+    l2tpv3 = opts->u.l2tpv3.data;
 
     if (l2tpv3->has_ipv6 && l2tpv3->ipv6) {
         s->ipv6 = l2tpv3->ipv6;
diff --git a/qemu/net/net.c b/qemu/net/net.c
index 28a5597b8..0ad6217cb 100644
--- a/qemu/net/net.c
+++ b/qemu/net/net.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "net/net.h"
 #include "clients.h"
@@ -32,9 +32,11 @@
 
 #include "monitor/monitor.h"
 #include "qemu-common.h"
+#include "qemu/help_option.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
+#include "qemu/cutils.h"
 #include "qemu/config-file.h"
 #include "qmp-commands.h"
 #include "hw/qdev.h"
@@ -42,8 +44,9 @@
 #include "qemu/main-loop.h"
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
 #include "sysemu/sysemu.h"
+#include "net/filter.h"
+#include "qapi/string-output-visitor.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
@@ -78,34 +81,6 @@ int default_net = 1;
 /***********************************************************/
 /* network device redirectors */
 
-#if defined(DEBUG_NET)
-static void hex_dump(FILE *f, const uint8_t *buf, int size)
-{
-    int len, i, j, c;
-
-    for(i=0;i<size;i+=16) {
-        len = size - i;
-        if (len > 16)
-            len = 16;
-        fprintf(f, "%08x ", i);
-        for(j=0;j<16;j++) {
-            if (j < len)
-                fprintf(f, " %02x", buf[i+j]);
-            else
-                fprintf(f, "   ");
-        }
-        fprintf(f, " ");
-        for(j=0;j<len;j++) {
-            c = buf[i+j];
-            if (c < ' ' || c > '~')
-                c = '.';
-            fprintf(f, "%c", c);
-        }
-        fprintf(f, "\n");
-    }
-}
-#endif
-
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
     const char *p, *p1;
@@ -285,8 +260,9 @@ static void qemu_net_client_setup(NetClientState *nc,
     }
     QTAILQ_INSERT_TAIL(&net_clients, nc, next);
 
-    nc->incoming_queue = qemu_new_net_queue(nc);
+    nc->incoming_queue = qemu_new_net_queue(qemu_deliver_packet_iov, nc);
     nc->destructor = destructor;
+    QTAILQ_INIT(&nc->filters);
 }
 
 NetClientState *qemu_new_net_client(NetClientInfo *info,
@@ -384,6 +360,7 @@ void qemu_del_net_client(NetClientState *nc)
 {
     NetClientState *ncs[MAX_QUEUE_NUM];
     int queues, i;
+    NetFilterState *nf, *next;
 
     assert(nc->info->type != NET_CLIENT_OPTIONS_KIND_NIC);
 
@@ -395,6 +372,10 @@ void qemu_del_net_client(NetClientState *nc)
                                           MAX_QUEUE_NUM);
     assert(queues != 0);
 
+    QTAILQ_FOREACH_SAFE(nf, &nc->filters, next, next) {
+        object_unparent(OBJECT(nf));
+    }
+
     /* If there is a peer NIC, delete and cleanup client, but do not free. */
     if (nc->peer && nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
         NICState *nic = qemu_get_nic(nc->peer);
@@ -517,20 +498,28 @@ void qemu_set_vnet_hdr_len(NetClientState *nc, int len)
 
 int qemu_set_vnet_le(NetClientState *nc, bool is_le)
 {
+#ifdef HOST_WORDS_BIGENDIAN
     if (!nc || !nc->info->set_vnet_le) {
         return -ENOSYS;
     }
 
     return nc->info->set_vnet_le(nc, is_le);
+#else
+    return 0;
+#endif
 }
 
 int qemu_set_vnet_be(NetClientState *nc, bool is_be)
 {
+#ifdef HOST_WORDS_BIGENDIAN
+    return 0;
+#else
     if (!nc || !nc->info->set_vnet_be) {
         return -ENOSYS;
     }
 
     return nc->info->set_vnet_be(nc, is_be);
+#endif
 }
 
 int qemu_can_send_packet(NetClientState *sender)
@@ -554,36 +543,54 @@ int qemu_can_send_packet(NetClientState *sender)
     return 1;
 }
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque)
+static ssize_t filter_receive_iov(NetClientState *nc,
+                                  NetFilterDirection direction,
+                                  NetClientState *sender,
+                                  unsigned flags,
+                                  const struct iovec *iov,
+                                  int iovcnt,
+                                  NetPacketSent *sent_cb)
 {
-    NetClientState *nc = opaque;
-    ssize_t ret;
-
-    if (nc->link_down) {
-        return size;
-    }
+    ssize_t ret = 0;
+    NetFilterState *nf = NULL;
 
-    if (nc->receive_disabled) {
-        return 0;
-    }
-
-    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
-        ret = nc->info->receive_raw(nc, data, size);
+    if (direction == NET_FILTER_DIRECTION_TX) {
+        QTAILQ_FOREACH(nf, &nc->filters, next) {
+            ret = qemu_netfilter_receive(nf, direction, sender, flags, iov,
+                                         iovcnt, sent_cb);
+            if (ret) {
+                return ret;
+            }
+        }
     } else {
-        ret = nc->info->receive(nc, data, size);
-    }
-
-    if (ret == 0) {
-        nc->receive_disabled = 1;
+        QTAILQ_FOREACH_REVERSE(nf, &nc->filters, NetFilterHead, next) {
+            ret = qemu_netfilter_receive(nf, direction, sender, flags, iov,
+                                         iovcnt, sent_cb);
+            if (ret) {
+                return ret;
+            }
+        }
     }
 
     return ret;
 }
 
+static ssize_t filter_receive(NetClientState *nc,
+                              NetFilterDirection direction,
+                              NetClientState *sender,
+                              unsigned flags,
+                              const uint8_t *data,
+                              size_t size,
+                              NetPacketSent *sent_cb)
+{
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
+
+    return filter_receive_iov(nc, direction, sender, flags, &iov, 1, sent_cb);
+}
+
 void qemu_purge_queued_packets(NetClientState *nc)
 {
     if (!nc->peer) {
@@ -625,16 +632,30 @@ static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
                                                  NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
 #ifdef DEBUG_NET
     printf("qemu_send_packet_async:\n");
-    hex_dump(stdout, buf, size);
+    qemu_hexdump((const char *)buf, stdout, "net", size);
 #endif
 
     if (sender->link_down || !sender->peer) {
         return size;
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive(sender, NET_FILTER_DIRECTION_TX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive(sender->peer, NET_FILTER_DIRECTION_RX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send(queue, sender, flags, buf, size, sent_cb);
@@ -660,14 +681,30 @@ ssize_t qemu_send_packet_raw(NetClientState *nc, const uint8_t *buf, int size)
 }
 
 static ssize_t nc_sendv_compat(NetClientState *nc, const struct iovec *iov,
-                               int iovcnt)
+                               int iovcnt, unsigned flags)
 {
-    uint8_t buffer[NET_BUFSIZE];
+    uint8_t *buf = NULL;
+    uint8_t *buffer;
     size_t offset;
+    ssize_t ret;
 
-    offset = iov_to_buf(iov, iovcnt, 0, buffer, sizeof(buffer));
+    if (iovcnt == 1) {
+        buffer = iov[0].iov_base;
+        offset = iov[0].iov_len;
+    } else {
+        buf = g_new(uint8_t, NET_BUFSIZE);
+        buffer = buf;
+        offset = iov_to_buf(iov, iovcnt, 0, buf, NET_BUFSIZE);
+    }
 
-    return nc->info->receive(nc, buffer, offset);
+    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
+        ret = nc->info->receive_raw(nc, buffer, offset);
+    } else {
+        ret = nc->info->receive(nc, buffer, offset);
+    }
+
+    g_free(buf);
+    return ret;
 }
 
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
@@ -690,7 +727,7 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
     if (nc->info->receive_iov) {
         ret = nc->info->receive_iov(nc, iov, iovcnt);
     } else {
-        ret = nc_sendv_compat(nc, iov, iovcnt);
+        ret = nc_sendv_compat(nc, iov, iovcnt, flags);
     }
 
     if (ret == 0) {
@@ -705,11 +742,25 @@ ssize_t qemu_sendv_packet_async(NetClientState *sender,
                                 NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
     if (sender->link_down || !sender->peer) {
         return iov_size(iov, iovcnt);
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive_iov(sender, NET_FILTER_DIRECTION_TX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive_iov(sender->peer, NET_FILTER_DIRECTION_RX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send_iov(queue, sender,
@@ -820,8 +871,8 @@ static int net_init_nic(const NetClientOptions *opts, const char *name,
     NICInfo *nd;
     const NetLegacyNicOptions *nic;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_NIC);
-    nic = opts->nic;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_NIC);
+    nic = opts->u.nic.data;
 
     idx = nic_get_free_idx();
     if (idx == -1 || nb_nics >= MAX_NICS) {
@@ -881,7 +932,7 @@ static int net_init_nic(const NetClientOptions *opts, const char *name,
 }
 
 
-static int (* const net_client_init_fun[NET_CLIENT_OPTIONS_KIND_MAX])(
+static int (* const net_client_init_fun[NET_CLIENT_OPTIONS_KIND__MAX])(
     const NetClientOptions *opts,
     const char *name,
     NetClientState *peer, Error **errp) = {
@@ -922,9 +973,9 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         opts = netdev->opts;
         name = netdev->id;
 
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_DUMP ||
-            opts->kind == NET_CLIENT_OPTIONS_KIND_NIC ||
-            !net_client_init_fun[opts->kind]) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_DUMP ||
+            opts->type == NET_CLIENT_OPTIONS_KIND_NIC ||
+            !net_client_init_fun[opts->type]) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a netdev backend type");
             return -1;
@@ -935,16 +986,16 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         /* missing optional values have been initialized to "all bits zero" */
         name = net->has_id ? net->id : net->name;
 
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_NONE) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_NONE) {
             return 0; /* nothing to do */
         }
-        if (opts->kind == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
+        if (opts->type == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a net type");
             return -1;
         }
 
-        if (!net_client_init_fun[opts->kind]) {
+        if (!net_client_init_fun[opts->type]) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
                        "a net backend type (maybe it is not compiled "
                        "into this binary)");
@@ -952,17 +1003,17 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
         }
 
         /* Do not add to a vlan if it's a nic with a netdev= parameter. */
-        if (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC ||
-            !opts->nic->has_netdev) {
+        if (opts->type != NET_CLIENT_OPTIONS_KIND_NIC ||
+            !opts->u.nic.data->has_netdev) {
             peer = net_hub_add_port(net->has_vlan ? net->vlan : 0, NULL);
         }
     }
 
-    if (net_client_init_fun[opts->kind](opts, name, peer, errp) < 0) {
+    if (net_client_init_fun[opts->type](opts, name, peer, errp) < 0) {
         /* FIXME drop when all init functions store an Error */
         if (errp && !*errp) {
             error_setg(errp, QERR_DEVICE_INIT_FAILED,
-                       NetClientOptionsKind_lookup[opts->kind]);
+                       NetClientOptionsKind_lookup[opts->type]);
         }
         return -1;
     }
@@ -970,41 +1021,63 @@ static int net_client_init1(const void *object, int is_netdev, Error **errp)
 }
 
 
-static void net_visit(Visitor *v, int is_netdev, void **object, Error **errp)
-{
-    if (is_netdev) {
-        visit_type_Netdev(v, (Netdev **)object, NULL, errp);
-    } else {
-        visit_type_NetLegacy(v, (NetLegacy **)object, NULL, errp);
-    }
-}
-
-
 int net_client_init(QemuOpts *opts, int is_netdev, Error **errp)
 {
     void *object = NULL;
     Error *err = NULL;
     int ret = -1;
+    OptsVisitor *ov = opts_visitor_new(opts);
+    Visitor *v = opts_get_visitor(ov);
 
     {
-        OptsVisitor *ov = opts_visitor_new(opts);
+        /* Parse convenience option format ip6-net=fec0::0[/64] */
+        const char *ip6_net = qemu_opt_get(opts, "ipv6-net");
+
+        if (ip6_net) {
+            char buf[strlen(ip6_net) + 1];
+
+            if (get_str_sep(buf, sizeof(buf), &ip6_net, '/') < 0) {
+                /* Default 64bit prefix length.  */
+                qemu_opt_set(opts, "ipv6-prefix", ip6_net, &error_abort);
+                qemu_opt_set_number(opts, "ipv6-prefixlen", 64, &error_abort);
+            } else {
+                /* User-specified prefix length.  */
+                unsigned long len;
+                int err;
+
+                qemu_opt_set(opts, "ipv6-prefix", buf, &error_abort);
+                err = qemu_strtoul(ip6_net, NULL, 10, &len);
+
+                if (err) {
+                    error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                              "ipv6-prefix", "a number");
+                } else {
+                    qemu_opt_set_number(opts, "ipv6-prefixlen", len,
+                                        &error_abort);
+                }
+            }
+            qemu_opt_unset(opts, "ipv6-net");
+        }
+    }
 
-        net_visit(opts_get_visitor(ov), is_netdev, &object, &err);
-        opts_visitor_cleanup(ov);
+    if (is_netdev) {
+        visit_type_Netdev(v, NULL, (Netdev **)&object, &err);
+    } else {
+        visit_type_NetLegacy(v, NULL, (NetLegacy **)&object, &err);
     }
 
     if (!err) {
         ret = net_client_init1(object, is_netdev, &err);
     }
 
-    if (object) {
-        QapiDeallocVisitor *dv = qapi_dealloc_visitor_new();
-
-        net_visit(qapi_dealloc_get_visitor(dv), is_netdev, &object, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
+    if (is_netdev) {
+        qapi_free_Netdev(object);
+    } else {
+        qapi_free_NetLegacy(object);
     }
 
     error_propagate(errp, err);
+    opts_visitor_cleanup(ov);
     return ret;
 }
 
@@ -1123,12 +1196,49 @@ void qmp_netdev_del(const char *id, Error **errp)
     qemu_opts_del(opts);
 }
 
+static void netfilter_print_info(Monitor *mon, NetFilterState *nf)
+{
+    char *str;
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+    StringOutputVisitor *ov;
+
+    /* generate info str */
+    object_property_iter_init(&iter, OBJECT(nf));
+    while ((prop = object_property_iter_next(&iter))) {
+        if (!strcmp(prop->name, "type")) {
+            continue;
+        }
+        ov = string_output_visitor_new(false);
+        object_property_get(OBJECT(nf), string_output_get_visitor(ov),
+                            prop->name, NULL);
+        str = string_output_get_string(ov);
+        string_output_visitor_cleanup(ov);
+        monitor_printf(mon, ",%s=%s", prop->name, str);
+        g_free(str);
+    }
+    monitor_printf(mon, "\n");
+}
+
 void print_net_client(Monitor *mon, NetClientState *nc)
 {
+    NetFilterState *nf;
+
     monitor_printf(mon, "%s: index=%d,type=%s,%s\n", nc->name,
                    nc->queue_index,
                    NetClientOptionsKind_lookup[nc->info->type],
                    nc->info_str);
+    if (!QTAILQ_EMPTY(&nc->filters)) {
+        monitor_printf(mon, "filters:\n");
+    }
+    QTAILQ_FOREACH(nf, &nc->filters, next) {
+        char *path = object_get_canonical_path_component(OBJECT(nf));
+
+        monitor_printf(mon, "  - %s: type=%s", path,
+                       object_get_typename(OBJECT(nf)));
+        netfilter_print_info(mon, nf);
+        g_free(path);
+    }
 }
 
 RxFilterInfoList *qmp_query_rx_filter(bool has_name, const char *name,
@@ -1154,6 +1264,12 @@ RxFilterInfoList *qmp_query_rx_filter(bool has_name, const char *name,
             continue;
         }
 
+        /* only query information on queue 0 since the info is per nic,
+         * not per queue
+         */
+        if (nc->queue_index != 0)
+            continue;
+
         if (nc->info->query_rx_filter) {
             info = nc->info->query_rx_filter(nc);
             entry = g_malloc0(sizeof(*entry));
@@ -1216,7 +1332,7 @@ void qmp_set_link(const char *name, bool up, Error **errp)
     int queues, i;
 
     queues = qemu_find_net_clients_except(name, ncs,
-                                          NET_CLIENT_OPTIONS_KIND_MAX,
+                                          NET_CLIENT_OPTIONS_KIND__MAX,
                                           MAX_QUEUE_NUM);
 
     if (queues == 0) {
diff --git a/qemu/net/netmap.c b/qemu/net/netmap.c
index 508b82947..6cc0db5ee 100644
--- a/qemu/net/netmap.c
+++ b/qemu/net/netmap.c
@@ -23,11 +23,10 @@
  */
 
 
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <sys/mman.h>
-#include <stdint.h>
-#include <stdio.h>
 #define NETMAP_WITH_LIBS
 #include <net/netmap.h>
 #include <net/netmap_user.h>
@@ -37,23 +36,16 @@
 #include "clients.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
-
-/* Private netmap device info. */
-typedef struct NetmapPriv {
-    int                 fd;
-    size_t              memsize;
-    void                *mem;
-    struct netmap_if    *nifp;
-    struct netmap_ring  *rx;
-    struct netmap_ring  *tx;
-    char                fdname[PATH_MAX];        /* Normally "/dev/netmap". */
-    char                ifname[IFNAMSIZ];
-} NetmapPriv;
+#include "qemu/cutils.h"
 
 typedef struct NetmapState {
     NetClientState      nc;
-    NetmapPriv          me;
+    struct nm_desc      *nmd;
+    char                ifname[IFNAMSIZ];
+    struct netmap_ring  *tx;
+    struct netmap_ring  *rx;
     bool                read_poll;
     bool                write_poll;
     struct iovec        iov[IOV_MAX];
@@ -90,46 +82,23 @@ pkt_copy(const void *_src, void *_dst, int l)
  * Open a netmap device. We assume there is only one queue
  * (which is the case for the VALE bridge).
  */
-static int netmap_open(NetmapPriv *me)
+static struct nm_desc *netmap_open(const NetdevNetmapOptions *nm_opts,
+                                   Error **errp)
 {
-    int fd;
-    int err;
-    size_t l;
+    struct nm_desc *nmd;
     struct nmreq req;
 
-    me->fd = fd = open(me->fdname, O_RDWR);
-    if (fd < 0) {
-        error_report("Unable to open netmap device '%s' (%s)",
-                        me->fdname, strerror(errno));
-        return -1;
-    }
     memset(&req, 0, sizeof(req));
-    pstrcpy(req.nr_name, sizeof(req.nr_name), me->ifname);
-    req.nr_ringid = NETMAP_NO_TX_POLL;
-    req.nr_version = NETMAP_API;
-    err = ioctl(fd, NIOCREGIF, &req);
-    if (err) {
-        error_report("Unable to register %s: %s", me->ifname, strerror(errno));
-        goto error;
-    }
-    l = me->memsize = req.nr_memsize;
-
-    me->mem = mmap(0, l, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
-    if (me->mem == MAP_FAILED) {
-        error_report("Unable to mmap netmap shared memory: %s",
-                        strerror(errno));
-        me->mem = NULL;
-        goto error;
-    }
 
-    me->nifp = NETMAP_IF(me->mem, req.nr_offset);
-    me->tx = NETMAP_TXRING(me->nifp, 0);
-    me->rx = NETMAP_RXRING(me->nifp, 0);
-    return 0;
+    nmd = nm_open(nm_opts->ifname, &req, NETMAP_NO_TX_POLL,
+                  NULL);
+    if (nmd == NULL) {
+        error_setg_errno(errp, errno, "Failed to nm_open() %s",
+                         nm_opts->ifname);
+        return NULL;
+    }
 
-error:
-    close(me->fd);
-    return -1;
+    return nmd;
 }
 
 static void netmap_send(void *opaque);
@@ -138,7 +107,7 @@ static void netmap_writable(void *opaque);
 /* Set the event-loop handlers for the netmap backend. */
 static void netmap_update_fd_handler(NetmapState *s)
 {
-    qemu_set_fd_handler(s->me.fd,
+    qemu_set_fd_handler(s->nmd->fd,
                         s->read_poll ? netmap_send : NULL,
                         s->write_poll ? netmap_writable : NULL,
                         s);
@@ -190,7 +159,7 @@ static ssize_t netmap_receive(NetClientState *nc,
       const uint8_t *buf, size_t size)
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
-    struct netmap_ring *ring = s->me.tx;
+    struct netmap_ring *ring = s->tx;
     uint32_t i;
     uint32_t idx;
     uint8_t *dst;
@@ -220,7 +189,7 @@ static ssize_t netmap_receive(NetClientState *nc,
     ring->slot[i].flags = 0;
     pkt_copy(buf, dst, size);
     ring->cur = ring->head = nm_ring_next(ring, i);
-    ioctl(s->me.fd, NIOCTXSYNC, NULL);
+    ioctl(s->nmd->fd, NIOCTXSYNC, NULL);
 
     return size;
 }
@@ -229,7 +198,7 @@ static ssize_t netmap_receive_iov(NetClientState *nc,
                     const struct iovec *iov, int iovcnt)
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
-    struct netmap_ring *ring = s->me.tx;
+    struct netmap_ring *ring = s->tx;
     uint32_t last;
     uint32_t idx;
     uint8_t *dst;
@@ -286,7 +255,7 @@ static ssize_t netmap_receive_iov(NetClientState *nc,
     /* Now update ring->cur and ring->head. */
     ring->cur = ring->head = i;
 
-    ioctl(s->me.fd, NIOCTXSYNC, NULL);
+    ioctl(s->nmd->fd, NIOCTXSYNC, NULL);
 
     return iov_size(iov, iovcnt);
 }
@@ -303,7 +272,7 @@ static void netmap_send_completed(NetClientState *nc, ssize_t len)
 static void netmap_send(void *opaque)
 {
     NetmapState *s = opaque;
-    struct netmap_ring *ring = s->me.rx;
+    struct netmap_ring *ring = s->rx;
 
     /* Keep sending while there are available packets into the netmap
        RX ring and the forwarding path towards the peer is open. */
@@ -351,27 +320,52 @@ static void netmap_cleanup(NetClientState *nc)
     qemu_purge_queued_packets(nc);
 
     netmap_poll(nc, false);
-    munmap(s->me.mem, s->me.memsize);
-    close(s->me.fd);
-
-    s->me.fd = -1;
+    nm_close(s->nmd);
+    s->nmd = NULL;
 }
 
 /* Offloading manipulation support callbacks. */
-static bool netmap_has_ufo(NetClientState *nc)
+static int netmap_fd_set_vnet_hdr_len(NetmapState *s, int len)
 {
-    return true;
+    struct nmreq req;
+
+    /* Issue a NETMAP_BDG_VNET_HDR command to change the virtio-net header
+     * length for the netmap adapter associated to 's->ifname'.
+     */
+    memset(&req, 0, sizeof(req));
+    pstrcpy(req.nr_name, sizeof(req.nr_name), s->ifname);
+    req.nr_version = NETMAP_API;
+    req.nr_cmd = NETMAP_BDG_VNET_HDR;
+    req.nr_arg1 = len;
+
+    return ioctl(s->nmd->fd, NIOCREGIF, &req);
 }
 
-static bool netmap_has_vnet_hdr(NetClientState *nc)
+static bool netmap_has_vnet_hdr_len(NetClientState *nc, int len)
 {
+    NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
+    int prev_len = s->vnet_hdr_len;
+
+    /* Check that we can set the new length. */
+    if (netmap_fd_set_vnet_hdr_len(s, len)) {
+        return false;
+    }
+
+    /* Restore the previous length. */
+    if (netmap_fd_set_vnet_hdr_len(s, prev_len)) {
+        error_report("Failed to restore vnet-hdr length %d on %s: %s",
+                     prev_len, s->ifname, strerror(errno));
+        abort();
+    }
+
     return true;
 }
 
-static bool netmap_has_vnet_hdr_len(NetClientState *nc, int len)
+/* A netmap interface that supports virtio-net headers always
+ * supports UFO, so we use this callback also for the has_ufo hook. */
+static bool netmap_has_vnet_hdr(NetClientState *nc)
 {
-    return len == 0 || len == sizeof(struct virtio_net_hdr) ||
-                len == sizeof(struct virtio_net_hdr_mrg_rxbuf);
+    return netmap_has_vnet_hdr_len(nc, sizeof(struct virtio_net_hdr));
 }
 
 static void netmap_using_vnet_hdr(NetClientState *nc, bool enable)
@@ -382,20 +376,11 @@ static void netmap_set_vnet_hdr_len(NetClientState *nc, int len)
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
     int err;
-    struct nmreq req;
 
-    /* Issue a NETMAP_BDG_VNET_HDR command to change the virtio-net header
-     * length for the netmap adapter associated to 'me->ifname'.
-     */
-    memset(&req, 0, sizeof(req));
-    pstrcpy(req.nr_name, sizeof(req.nr_name), s->me.ifname);
-    req.nr_version = NETMAP_API;
-    req.nr_cmd = NETMAP_BDG_VNET_HDR;
-    req.nr_arg1 = len;
-    err = ioctl(s->me.fd, NIOCREGIF, &req);
+    err = netmap_fd_set_vnet_hdr_len(s, len);
     if (err) {
-        error_report("Unable to execute NETMAP_BDG_VNET_HDR on %s: %s",
-                     s->me.ifname, strerror(errno));
+        error_report("Unable to set vnet-hdr length %d on %s: %s",
+                     len, s->ifname, strerror(errno));
     } else {
         /* Keep track of the current length. */
         s->vnet_hdr_len = len;
@@ -408,8 +393,7 @@ static void netmap_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
 
     /* Setting a virtio-net header length greater than zero automatically
-     * enables the offloadings.
-     */
+     * enables the offloadings. */
     if (!s->vnet_hdr_len) {
         netmap_set_vnet_hdr_len(nc, sizeof(struct virtio_net_hdr));
     }
@@ -423,7 +407,7 @@ static NetClientInfo net_netmap_info = {
     .receive_iov = netmap_receive_iov,
     .poll = netmap_poll,
     .cleanup = netmap_cleanup,
-    .has_ufo = netmap_has_ufo,
+    .has_ufo = netmap_has_vnet_hdr,
     .has_vnet_hdr = netmap_has_vnet_hdr,
     .has_vnet_hdr_len = netmap_has_vnet_hdr_len,
     .using_vnet_hdr = netmap_using_vnet_hdr,
@@ -438,24 +422,25 @@ static NetClientInfo net_netmap_info = {
 int net_init_netmap(const NetClientOptions *opts,
                     const char *name, NetClientState *peer, Error **errp)
 {
-    /* FIXME error_setg(errp, ...) on failure */
-    const NetdevNetmapOptions *netmap_opts = opts->netmap;
+    const NetdevNetmapOptions *netmap_opts = opts->u.netmap.data;
+    struct nm_desc *nmd;
     NetClientState *nc;
-    NetmapPriv me;
+    Error *err = NULL;
     NetmapState *s;
 
-    pstrcpy(me.fdname, sizeof(me.fdname),
-        netmap_opts->has_devname ? netmap_opts->devname : "/dev/netmap");
-    /* Set default name for the port if not supplied. */
-    pstrcpy(me.ifname, sizeof(me.ifname), netmap_opts->ifname);
-    if (netmap_open(&me)) {
+    nmd = netmap_open(netmap_opts, &err);
+    if (err) {
+        error_propagate(errp, err);
         return -1;
     }
     /* Create the object. */
     nc = qemu_new_net_client(&net_netmap_info, peer, "netmap", name);
     s = DO_UPCAST(NetmapState, nc, nc);
-    s->me = me;
+    s->nmd = nmd;
+    s->tx = NETMAP_TXRING(nmd->nifp, 0);
+    s->rx = NETMAP_RXRING(nmd->nifp, 0);
     s->vnet_hdr_len = 0;
+    pstrcpy(s->ifname, sizeof(s->ifname), netmap_opts->ifname);
     netmap_read_poll(s, true); /* Initially only poll for reads. */
 
     return 0;
diff --git a/qemu/net/queue.c b/qemu/net/queue.c
index ebbe2bb93..9c32abdb8 100644
--- a/qemu/net/queue.c
+++ b/qemu/net/queue.c
@@ -21,6 +21,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "net/queue.h"
 #include "qemu/queue.h"
 #include "net/net.h"
@@ -52,13 +53,14 @@ struct NetQueue {
     void *opaque;
     uint32_t nq_maxlen;
     uint32_t nq_count;
+    NetQueueDeliverFunc *deliver;
 
     QTAILQ_HEAD(packets, NetPacket) packets;
 
     unsigned delivering : 1;
 };
 
-NetQueue *qemu_new_net_queue(void *opaque)
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque)
 {
     NetQueue *queue;
 
@@ -67,6 +69,7 @@ NetQueue *qemu_new_net_queue(void *opaque)
     queue->opaque = opaque;
     queue->nq_maxlen = 10000;
     queue->nq_count = 0;
+    queue->deliver = deliver;
 
     QTAILQ_INIT(&queue->packets);
 
@@ -110,12 +113,12 @@ static void qemu_net_queue_append(NetQueue *queue,
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
 }
 
-static void qemu_net_queue_append_iov(NetQueue *queue,
-                                      NetClientState *sender,
-                                      unsigned flags,
-                                      const struct iovec *iov,
-                                      int iovcnt,
-                                      NetPacketSent *sent_cb)
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
 {
     NetPacket *packet;
     size_t max_len = 0;
@@ -152,9 +155,13 @@ static ssize_t qemu_net_queue_deliver(NetQueue *queue,
                                       size_t size)
 {
     ssize_t ret = -1;
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet(sender, flags, data, size, queue->opaque);
+    ret = queue->deliver(sender, flags, &iov, 1, queue->opaque);
     queue->delivering = 0;
 
     return ret;
@@ -169,7 +176,7 @@ static ssize_t qemu_net_queue_deliver_iov(NetQueue *queue,
     ssize_t ret = -1;
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet_iov(sender, flags, iov, iovcnt, queue->opaque);
+    ret = queue->deliver(sender, flags, iov, iovcnt, queue->opaque);
     queue->delivering = 0;
 
     return ret;
diff --git a/qemu/net/slirp.c b/qemu/net/slirp.c
index 7657b38fd..31630f005 100644
--- a/qemu/net/slirp.c
+++ b/qemu/net/slirp.c
@@ -21,9 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "net/slirp.h"
 
-#include "config-host.h"
 
 #ifndef _WIN32
 #include <pwd.h>
@@ -36,7 +36,9 @@
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "slirp/libslirp.h"
+#include "slirp/ip6.h"
 #include "sysemu/char.h"
+#include "qemu/cutils.h"
 
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
@@ -134,11 +136,14 @@ static NetClientInfo net_slirp_info = {
 
 static int net_slirp_init(NetClientState *peer, const char *model,
                           const char *name, int restricted,
-                          const char *vnetwork, const char *vhost,
+                          bool ipv4, const char *vnetwork, const char *vhost,
+                          bool ipv6, const char *vprefix6, int vprefix6_len,
+                          const char *vhost6,
                           const char *vhostname, const char *tftp_export,
                           const char *bootfile, const char *vdhcp_start,
-                          const char *vnameserver, const char *smb_export,
-                          const char *vsmbserver, const char **dnssearch)
+                          const char *vnameserver, const char *vnameserver6,
+                          const char *smb_export, const char *vsmbserver,
+                          const char **dnssearch)
 {
     /* default settings according to historic slirp */
     struct in_addr net  = { .s_addr = htonl(0x0a000200) }; /* 10.0.2.0 */
@@ -146,6 +151,9 @@ static int net_slirp_init(NetClientState *peer, const char *model,
     struct in_addr host = { .s_addr = htonl(0x0a000202) }; /* 10.0.2.2 */
     struct in_addr dhcp = { .s_addr = htonl(0x0a00020f) }; /* 10.0.2.15 */
     struct in_addr dns  = { .s_addr = htonl(0x0a000203) }; /* 10.0.2.3 */
+    struct in6_addr ip6_prefix;
+    struct in6_addr ip6_host;
+    struct in6_addr ip6_dns;
 #ifndef _WIN32
     struct in_addr smbsrv = { .s_addr = 0 };
 #endif
@@ -157,6 +165,19 @@ static int net_slirp_init(NetClientState *peer, const char *model,
     char *end;
     struct slirp_config_str *config;
 
+    if (!ipv4 && (vnetwork || vhost || vnameserver)) {
+        return -1;
+    }
+
+    if (!ipv6 && (vprefix6 || vhost6 || vnameserver6)) {
+        return -1;
+    }
+
+    if (!ipv4 && !ipv6) {
+        /* It doesn't make sense to disable both */
+        return -1;
+    }
+
     if (!tftp_export) {
         tftp_export = legacy_tftp_prefix;
     }
@@ -235,6 +256,64 @@ static int net_slirp_init(NetClientState *peer, const char *model,
     }
 #endif
 
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+    /* No inet_pton helper before Vista... */
+    if (vprefix6) {
+        /* Unsupported */
+        return -1;
+    }
+    memset(&ip6_prefix, 0, sizeof(ip6_prefix));
+    ip6_prefix.s6_addr[0] = 0xfe;
+    ip6_prefix.s6_addr[1] = 0xc0;
+#else
+    if (!vprefix6) {
+        vprefix6 = "fec0::";
+    }
+    if (!inet_pton(AF_INET6, vprefix6, &ip6_prefix)) {
+        return -1;
+    }
+#endif
+
+    if (!vprefix6_len) {
+        vprefix6_len = 64;
+    }
+    if (vprefix6_len < 0 || vprefix6_len > 126) {
+        return -1;
+    }
+
+    if (vhost6) {
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+        return -1;
+#else
+        if (!inet_pton(AF_INET6, vhost6, &ip6_host)) {
+            return -1;
+        }
+        if (!in6_equal_net(&ip6_prefix, &ip6_host, vprefix6_len)) {
+            return -1;
+        }
+#endif
+    } else {
+        ip6_host = ip6_prefix;
+        ip6_host.s6_addr[15] |= 2;
+    }
+
+    if (vnameserver6) {
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+        return -1;
+#else
+        if (!inet_pton(AF_INET6, vnameserver6, &ip6_dns)) {
+            return -1;
+        }
+        if (!in6_equal_net(&ip6_prefix, &ip6_dns, vprefix6_len)) {
+            return -1;
+        }
+#endif
+    } else {
+        ip6_dns = ip6_prefix;
+        ip6_dns.s6_addr[15] |= 3;
+    }
+
+
     nc = qemu_new_net_client(&net_slirp_info, peer, model, name);
 
     snprintf(nc->info_str, sizeof(nc->info_str),
@@ -243,8 +322,10 @@ static int net_slirp_init(NetClientState *peer, const char *model,
 
     s = DO_UPCAST(SlirpState, nc, nc);
 
-    s->slirp = slirp_init(restricted, net, mask, host, vhostname,
-                          tftp_export, bootfile, dhcp, dns, dnssearch, s);
+    s->slirp = slirp_init(restricted, ipv4, net, mask, host,
+                          ipv6, ip6_prefix, vprefix6_len, ip6_host,
+                          vhostname, tftp_export, bootfile, dhcp,
+                          dns, ip6_dns, dnssearch, s);
     QTAILQ_INSERT_TAIL(&slirp_stacks, s, entry);
 
     for (config = slirp_configs; config; config = config->next) {
@@ -745,9 +826,19 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     int ret;
     const NetdevUserOptions *user;
     const char **dnssearch;
+    bool ipv4 = true, ipv6 = true;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_USER);
-    user = opts->user;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_USER);
+    user = opts->u.user.data;
+
+    if ((user->has_ipv6 && user->ipv6 && !user->has_ipv4) ||
+        (user->has_ipv4 && !user->ipv4)) {
+        ipv4 = 0;
+    }
+    if ((user->has_ipv4 && user->ipv4 && !user->has_ipv6) ||
+        (user->has_ipv6 && !user->ipv6)) {
+        ipv6 = 0;
+    }
 
     vnet = user->has_net ? g_strdup(user->net) :
            user->has_ip  ? g_strdup_printf("%s/24", user->ip) :
@@ -760,9 +851,12 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     net_init_slirp_configs(user->hostfwd, SLIRP_CFG_HOSTFWD);
     net_init_slirp_configs(user->guestfwd, 0);
 
-    ret = net_slirp_init(peer, "user", name, user->q_restrict, vnet,
-                         user->host, user->hostname, user->tftp,
-                         user->bootfile, user->dhcpstart, user->dns, user->smb,
+    ret = net_slirp_init(peer, "user", name, user->q_restrict,
+                         ipv4, vnet, user->host,
+                         ipv6, user->ipv6_prefix, user->ipv6_prefixlen,
+                         user->ipv6_host, user->hostname, user->tftp,
+                         user->bootfile, user->dhcpstart,
+                         user->dns, user->ipv6_dns, user->smb,
                          user->smbserver, dnssearch);
 
     while (slirp_configs) {
@@ -784,6 +878,9 @@ int net_slirp_parse_legacy(QemuOptsList *opts_list, const char *optarg, int *ret
         return 0;
     }
 
+    error_report("The '-net channel' option is deprecated. "
+                 "Please use '-netdev user,guestfwd=...' instead.");
+
     /* handle legacy -net channel,port:chr */
     optarg += strlen("channel,");
 
diff --git a/qemu/net/socket.c b/qemu/net/socket.c
index b1e3b1c8d..9fa2cd8d5 100644
--- a/qemu/net/socket.c
+++ b/qemu/net/socket.c
@@ -21,11 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "net/net.h"
 #include "clients.h"
 #include "monitor/monitor.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
@@ -145,15 +146,14 @@ static void net_socket_send_completed(NetClientState *nc, ssize_t len)
 static void net_socket_send(void *opaque)
 {
     NetSocketState *s = opaque;
-    int size, err;
+    int size;
     unsigned l;
     uint8_t buf1[NET_BUFSIZE];
     const uint8_t *buf;
 
     size = qemu_recv(s->fd, buf1, sizeof(buf1), 0);
     if (size < 0) {
-        err = socket_error();
-        if (err != EWOULDBLOCK)
+        if (errno != EWOULDBLOCK)
             goto eoc;
     } else if (size == 0) {
         /* end of connection */
@@ -566,7 +566,7 @@ static int net_socket_connect_init(NetClientState *peer,
                                    const char *host_str)
 {
     NetSocketState *s;
-    int fd, connected, ret, err;
+    int fd, connected, ret;
     struct sockaddr_in saddr;
 
     if (parse_host_port(&saddr, host_str) < 0)
@@ -583,14 +583,12 @@ static int net_socket_connect_init(NetClientState *peer,
     for(;;) {
         ret = connect(fd, (struct sockaddr *)&saddr, sizeof(saddr));
         if (ret < 0) {
-            err = socket_error();
-            if (err == EINTR || err == EWOULDBLOCK) {
-            } else if (err == EINPROGRESS) {
-                break;
-#ifdef _WIN32
-            } else if (err == WSAEALREADY || err == WSAEINVAL) {
+            if (errno == EINTR || errno == EWOULDBLOCK) {
+                /* continue */
+            } else if (errno == EINPROGRESS ||
+                       errno == EALREADY ||
+                       errno == EINVAL) {
                 break;
-#endif
             } else {
                 perror("connect");
                 closesocket(fd);
@@ -706,8 +704,8 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
     Error *err = NULL;
     const NetdevSocketOptions *sock;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_SOCKET);
-    sock = opts->socket;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_SOCKET);
+    sock = opts->u.socket.data;
 
     if (sock->has_fd + sock->has_listen + sock->has_connect + sock->has_mcast +
         sock->has_udp != 1) {
diff --git a/qemu/net/tap-aix.c b/qemu/net/tap-aix.c
index e84fc3913..0e6da6396 100644
--- a/qemu/net/tap-aix.c
+++ b/qemu/net/tap-aix.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
              int vnet_hdr_required, int mq_required, Error **errp)
diff --git a/qemu/net/tap-bsd.c b/qemu/net/tap-bsd.c
index 7028d9be9..c506ac31d 100644
--- a/qemu/net/tap-bsd.c
+++ b/qemu/net/tap-bsd.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
 
@@ -109,8 +111,7 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
 
 #define PATH_NET_TAP "/dev/tap"
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required, Error **errp)
+static int tap_open_clone(char *ifname, int ifname_size, Error **errp)
 {
     int fd, s, ret;
     struct ifreq ifr;
@@ -126,7 +127,8 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
     ret = ioctl(fd, TAPGIFNAME, (void *)&ifr);
     if (ret < 0) {
         error_setg_errno(errp, errno, "could not get tap interface name");
-        goto error;
+        close(fd);
+        return -1;
     }
 
     if (ifname[0] != '\0') {
@@ -135,19 +137,47 @@ int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
         if (s < 0) {
             error_setg_errno(errp, errno,
                              "could not open socket to set interface name");
-            goto error;
+            close(fd);
+            return -1;
         }
         ifr.ifr_data = ifname;
         ret = ioctl(s, SIOCSIFNAME, (void *)&ifr);
         close(s);
         if (ret < 0) {
             error_setg(errp, "could not set tap interface name");
-            goto error;
+            close(fd);
+            return -1;
         }
     } else {
         pstrcpy(ifname, ifname_size, ifr.ifr_name);
     }
 
+    return fd;
+}
+
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required, Error **errp)
+{
+    int fd = -1;
+
+    /* If the specified tap device already exists just use it. */
+    if (ifname[0] != '\0') {
+        char dname[100];
+        snprintf(dname, sizeof dname, "/dev/%s", ifname);
+        TFR(fd = open(dname, O_RDWR));
+        if (fd < 0 && errno != ENOENT) {
+            error_setg_errno(errp, errno, "could not open %s", dname);
+            return -1;
+        }
+    }
+
+    if (fd < 0) {
+        /* Tap device not specified or does not exist. */
+        if ((fd = tap_open_clone(ifname, ifname_size, errp)) < 0) {
+            return -1;
+        }
+    }
+
     if (*vnet_hdr) {
         /* BSD doesn't have IFF_VNET_HDR */
         *vnet_hdr = 0;
diff --git a/qemu/net/tap-haiku.c b/qemu/net/tap-haiku.c
index 2e738ec6a..b27e57e95 100644
--- a/qemu/net/tap-haiku.c
+++ b/qemu/net/tap-haiku.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
              int vnet_hdr_required, int mq_required, Error **errp)
diff --git a/qemu/net/tap-linux.c b/qemu/net/tap-linux.c
index 394f2a646..a503fa9c6 100644
--- a/qemu/net/tap-linux.c
+++ b/qemu/net/tap-linux.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 #include "tap-linux.h"
 #include "net/tap.h"
@@ -31,8 +32,9 @@
 #include <sys/ioctl.h>
 
 #include "sysemu/sysemu.h"
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 #define PATH_NET_TUN "/dev/net/tun"
 
@@ -211,7 +213,7 @@ int tap_fd_set_vnet_le(int fd, int is_le)
         return -errno;
     }
 
-    error_report("TUNSETVNETLE ioctl() failed: %s.\n", strerror(errno));
+    error_report("TUNSETVNETLE ioctl() failed: %s.", strerror(errno));
     abort();
 }
 
@@ -228,7 +230,7 @@ int tap_fd_set_vnet_be(int fd, int is_be)
         return -errno;
     }
 
-    error_report("TUNSETVNETBE ioctl() failed: %s.\n", strerror(errno));
+    error_report("TUNSETVNETBE ioctl() failed: %s.", strerror(errno));
     abort();
 }
 
diff --git a/qemu/net/tap-linux.h b/qemu/net/tap-linux.h
index 01dc6f8a2..1dc3a9f27 100644
--- a/qemu/net/tap-linux.h
+++ b/qemu/net/tap-linux.h
@@ -16,7 +16,6 @@
 #ifndef QEMU_TAP_LINUX_H
 #define QEMU_TAP_LINUX_H
 
-#include <stdint.h>
 #ifdef __linux__
 
 #include <linux/ioctl.h>
diff --git a/qemu/net/tap-solaris.c b/qemu/net/tap-solaris.c
index 0f60f78dd..a2a92356c 100644
--- a/qemu/net/tap-solaris.c
+++ b/qemu/net/tap-solaris.c
@@ -22,10 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
-#include <sys/stat.h>
 #include <sys/ethernet.h>
 #include <sys/sockio.h>
 #include <netinet/arp.h>
diff --git a/qemu/net/tap-win32.c b/qemu/net/tap-win32.c
index 625d53c64..f1e142ace 100644
--- a/qemu/net/tap-win32.c
+++ b/qemu/net/tap-win32.c
@@ -26,6 +26,7 @@
  *  distribution); if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 
 #include "qemu-common.h"
@@ -34,7 +35,6 @@
 #include "net/tap.h"            /* tap_has_ufo, ... */
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
-#include <stdio.h>
 #include <windows.h>
 #include <winioctl.h>
 
@@ -77,7 +77,12 @@
 
 //#define DEBUG_TAP_WIN32
 
-#define TUN_ASYNCHRONOUS_WRITES 1
+/* FIXME: The asynch write path appears to be broken at
+ * present. WriteFile() ignores the lpNumberOfBytesWritten parameter
+ * for overlapped writes, with the result we return zero bytes sent,
+ * and after handling a single packet, receive is disabled for this
+ * interface. */
+/* #define TUN_ASYNCHRONOUS_WRITES 1 */
 
 #define TUN_BUFFER_SIZE 1560
 #define TUN_MAX_BUFFER_COUNT 32
@@ -356,7 +361,8 @@ static int get_device_guid(
                 &len);
 
             if (status != ERROR_SUCCESS || name_type != REG_SZ) {
-                    return -1;
+                ++i;
+                continue;
             }
             else {
                 if (is_tap_win32_dev(enum_name)) {
@@ -460,27 +466,48 @@ static int tap_win32_write(tap_win32_overlapped_t *overlapped,
     BOOL result;
     DWORD error;
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
     result = GetOverlappedResult( overlapped->handle, &overlapped->write_overlapped,
                                   &write_size, FALSE);
 
     if (!result && GetLastError() == ERROR_IO_INCOMPLETE)
         WaitForSingleObject(overlapped->write_event, INFINITE);
+#endif
 
     result = WriteFile(overlapped->handle, buffer, size,
                        &write_size, &overlapped->write_overlapped);
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
+    /* FIXME: we can't sensibly set write_size here, without waiting
+     * for the IO to complete! Moreover, we can't return zero,
+     * because that will disable receive on this interface, and we
+     * also can't assume it will succeed and return the full size,
+     * because that will result in the buffer being reclaimed while
+     * the IO is in progress. */
+#error Async writes are broken. Please disable TUN_ASYNCHRONOUS_WRITES.
+#else /* !TUN_ASYNCHRONOUS_WRITES */
     if (!result) {
-        switch (error = GetLastError())
-        {
-        case ERROR_IO_PENDING:
-#ifndef TUN_ASYNCHRONOUS_WRITES
-            WaitForSingleObject(overlapped->write_event, INFINITE);
-#endif
-            break;
-        default:
-            return -1;
+        error = GetLastError();
+        if (error == ERROR_IO_PENDING) {
+            result = GetOverlappedResult(overlapped->handle,
+                                         &overlapped->write_overlapped,
+                                         &write_size, TRUE);
         }
     }
+#endif
+
+    if (!result) {
+#ifdef DEBUG_TAP_WIN32
+        LPTSTR msgbuf;
+        error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM,
+                      NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                      &msgbuf, 0, NULL);
+        fprintf(stderr, "Tap-Win32: Error WriteFile %d - %s\n", error, msgbuf);
+        LocalFree(msgbuf);
+#endif
+        return 0;
+    }
 
     return write_size;
 }
@@ -767,8 +794,8 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
     /* FIXME error_setg(errp, ...) on failure */
     const NetdevTapOptions *tap;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    tap = opts->u.tap.data;
 
     if (!tap->has_ifname) {
         error_report("tap: no interface name");
diff --git a/qemu/net/tap.c b/qemu/net/tap.c
index bd01590e8..740e8a261 100644
--- a/qemu/net/tap.c
+++ b/qemu/net/tap.c
@@ -23,12 +23,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 
-#include "config-host.h"
 
 #include <sys/ioctl.h>
-#include <sys/stat.h>
 #include <sys/wait.h>
 #include <sys/socket.h>
 #include <net/if.h>
@@ -37,7 +36,9 @@
 #include "clients.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 
 #include "net/tap.h"
@@ -565,8 +566,8 @@ int net_init_bridge(const NetClientOptions *opts, const char *name,
     TAPState *s;
     int fd, vnet_hdr;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_BRIDGE);
-    bridge = opts->bridge;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_BRIDGE);
+    bridge = opts->u.bridge.data;
 
     helper = bridge->has_helper ? bridge->helper : DEFAULT_BRIDGE_HELPER;
     br     = bridge->has_br     ? bridge->br     : DEFAULT_BRIDGE_INTERFACE;
@@ -663,7 +664,7 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
         options.backend_type = VHOST_BACKEND_TYPE_KERNEL;
         options.net_backend = &s->nc;
 
-        if (tap->has_vhostfd || tap->has_vhostfds) {
+        if (vhostfdname) {
             vhostfd = monitor_fd_param(cur_mon, vhostfdname, &err);
             if (vhostfd == -1) {
                 error_propagate(errp, err);
@@ -685,7 +686,7 @@ static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
                        "vhost-net requested but could not be initialized");
             return;
         }
-    } else if (tap->has_vhostfd || tap->has_vhostfds) {
+    } else if (vhostfdname) {
         error_setg(errp, "vhostfd= is not valid without vhost");
     }
 }
@@ -728,8 +729,8 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
     const char *vhostfdname;
     char ifname[128];
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    tap = opts->u.tap.data;
     queues = tap->has_queues ? tap->queues : 1;
     vhostfdname = tap->has_vhostfd ? tap->vhostfd : NULL;
 
diff --git a/qemu/net/util.c b/qemu/net/util.c
index 7e9507679..0b3dbfe5d 100644
--- a/qemu/net/util.c
+++ b/qemu/net/util.c
@@ -22,9 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "util.h"
-#include <errno.h>
-#include <stdlib.h>
 
 int net_parse_macaddr(uint8_t *macaddr, const char *p)
 {
diff --git a/qemu/net/util.h b/qemu/net/util.h
index 10c7da95f..60b73d372 100644
--- a/qemu/net/util.h
+++ b/qemu/net/util.h
@@ -25,7 +25,6 @@
 #ifndef QEMU_NET_UTIL_H
 #define QEMU_NET_UTIL_H
 
-#include <stdint.h>
 
 int net_parse_macaddr(uint8_t *macaddr, const char *p);
 
diff --git a/qemu/net/vde.c b/qemu/net/vde.c
index dacaa64b4..9427eaa16 100644
--- a/qemu/net/vde.c
+++ b/qemu/net/vde.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <libvdeplug.h>
 
@@ -115,8 +115,8 @@ int net_init_vde(const NetClientOptions *opts, const char *name,
     /* FIXME error_setg(errp, ...) on failure */
     const NetdevVdeOptions *vde;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VDE);
-    vde = opts->vde;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_VDE);
+    vde = opts->u.vde.data;
 
     /* missing optional values have been initialized to "all bits zero" */
     if (net_vde_init(peer, "vde", name, vde->sock, vde->port, vde->group,
diff --git a/qemu/net/vhost-user.c b/qemu/net/vhost-user.c
index 93dcecd66..1b9e73a2d 100644
--- a/qemu/net/vhost-user.c
+++ b/qemu/net/vhost-user.c
@@ -8,12 +8,15 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "clients.h"
 #include "net/vhost_net.h"
 #include "net/vhost-user.h"
 #include "sysemu/char.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qmp-commands.h"
+#include "trace.h"
 
 typedef struct VhostUserState {
     NetClientState nc;
@@ -24,7 +27,6 @@ typedef struct VhostUserState {
 typedef struct VhostUserChardevProps {
     bool is_socket;
     bool is_unix;
-    bool is_server;
 } VhostUserChardevProps;
 
 VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
@@ -39,37 +41,106 @@ static int vhost_user_running(VhostUserState *s)
     return (s->vhost_net) ? 1 : 0;
 }
 
-static int vhost_user_start(VhostUserState *s)
+static void vhost_user_stop(int queues, NetClientState *ncs[])
 {
-    VhostNetOptions options;
+    VhostUserState *s;
+    int i;
 
-    if (vhost_user_running(s)) {
-        return 0;
+    for (i = 0; i < queues; i++) {
+        assert (ncs[i]->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
+        if (!vhost_user_running(s)) {
+            continue;
+        }
+
+        if (s->vhost_net) {
+            vhost_net_cleanup(s->vhost_net);
+            s->vhost_net = NULL;
+        }
     }
+}
+
+static int vhost_user_start(int queues, NetClientState *ncs[])
+{
+    VhostNetOptions options;
+    VhostUserState *s;
+    int max_queues;
+    int i;
 
     options.backend_type = VHOST_BACKEND_TYPE_USER;
-    options.net_backend = &s->nc;
-    options.opaque = s->chr;
 
-    s->vhost_net = vhost_net_init(&options);
+    for (i = 0; i < queues; i++) {
+        assert (ncs[i]->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
+        if (vhost_user_running(s)) {
+            continue;
+        }
+
+        options.net_backend = ncs[i];
+        options.opaque      = s->chr;
+        s->vhost_net = vhost_net_init(&options);
+        if (!s->vhost_net) {
+            error_report("failed to init vhost_net for queue %d", i);
+            goto err;
+        }
+
+        if (i == 0) {
+            max_queues = vhost_net_get_max_queues(s->vhost_net);
+            if (queues > max_queues) {
+                error_report("you are asking more queues than supported: %d",
+                             max_queues);
+                goto err;
+            }
+        }
+    }
 
-    return vhost_user_running(s) ? 0 : -1;
+    return 0;
+
+err:
+    vhost_user_stop(i + 1, ncs);
+    return -1;
 }
 
-static void vhost_user_stop(VhostUserState *s)
+static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
 {
-    if (vhost_user_running(s)) {
-        vhost_net_cleanup(s->vhost_net);
+    /* In case of RARP (message size is 60) notify backup to send a fake RARP.
+       This fake RARP will be sent by backend only for guest
+       without GUEST_ANNOUNCE capability.
+     */
+    if (size == 60) {
+        VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
+        int r;
+        static int display_rarp_failure = 1;
+        char mac_addr[6];
+
+        /* extract guest mac address from the RARP message */
+        memcpy(mac_addr, &buf[6], 6);
+
+        r = vhost_net_notify_migration_done(s->vhost_net, mac_addr);
+
+        if ((r != 0) && (display_rarp_failure)) {
+            fprintf(stderr,
+                    "Vhost user backend fails to broadcast fake RARP\n");
+            fflush(stderr);
+            display_rarp_failure = 0;
+        }
     }
 
-    s->vhost_net = 0;
+    return size;
 }
 
 static void vhost_user_cleanup(NetClientState *nc)
 {
     VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
 
-    vhost_user_stop(s);
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
     qemu_purge_queued_packets(nc);
 }
 
@@ -90,64 +161,69 @@ static bool vhost_user_has_ufo(NetClientState *nc)
 static NetClientInfo net_vhost_user_info = {
         .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
         .size = sizeof(VhostUserState),
+        .receive = vhost_user_receive,
         .cleanup = vhost_user_cleanup,
         .has_vnet_hdr = vhost_user_has_vnet_hdr,
         .has_ufo = vhost_user_has_ufo,
 };
 
-static void net_vhost_link_down(VhostUserState *s, bool link_down)
-{
-    s->nc.link_down = link_down;
-
-    if (s->nc.peer) {
-        s->nc.peer->link_down = link_down;
-    }
-
-    if (s->nc.info->link_status_changed) {
-        s->nc.info->link_status_changed(&s->nc);
-    }
-
-    if (s->nc.peer && s->nc.peer->info->link_status_changed) {
-        s->nc.peer->info->link_status_changed(s->nc.peer);
-    }
-}
-
 static void net_vhost_user_event(void *opaque, int event)
 {
-    VhostUserState *s = opaque;
+    const char *name = opaque;
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    VhostUserState *s;
+    Error *err = NULL;
+    int queues;
+
+    queues = qemu_find_net_clients_except(name, ncs,
+                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          MAX_QUEUE_NUM);
+    assert(queues < MAX_QUEUE_NUM);
 
+    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
+    trace_vhost_user_event(s->chr->label, event);
     switch (event) {
     case CHR_EVENT_OPENED:
-        vhost_user_start(s);
-        net_vhost_link_down(s, false);
-        error_report("chardev \"%s\" went up", s->chr->label);
+        if (vhost_user_start(queues, ncs) < 0) {
+            exit(1);
+        }
+        qmp_set_link(name, true, &err);
         break;
     case CHR_EVENT_CLOSED:
-        net_vhost_link_down(s, true);
-        vhost_user_stop(s);
-        error_report("chardev \"%s\" went down", s->chr->label);
+        qmp_set_link(name, false, &err);
+        vhost_user_stop(queues, ncs);
         break;
     }
+
+    if (err) {
+        error_report_err(err);
+    }
 }
 
 static int net_vhost_user_init(NetClientState *peer, const char *device,
-                               const char *name, CharDriverState *chr)
+                               const char *name, CharDriverState *chr,
+                               int queues)
 {
     NetClientState *nc;
     VhostUserState *s;
+    int i;
 
-    nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
+    assert(name);
+    assert(queues > 0);
 
-    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user to %s",
-             chr->label);
+    for (i = 0; i < queues; i++) {
+        nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
 
-    s = DO_UPCAST(VhostUserState, nc, nc);
+        snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user%d to %s",
+                 i, chr->label);
 
-    /* We don't provide a receive callback */
-    s->nc.receive_disabled = 1;
-    s->chr = chr;
+        nc->queue_index = i;
 
-    qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
+        s = DO_UPCAST(VhostUserState, nc, nc);
+        s->chr = chr;
+    }
+
+    qemu_chr_add_handlers(chr, NULL, NULL, net_vhost_user_event, nc[0].name);
 
     return 0;
 }
@@ -163,7 +239,6 @@ static int net_vhost_chardev_opts(void *opaque,
     } else if (strcmp(name, "path") == 0) {
         props->is_unix = true;
     } else if (strcmp(name, "server") == 0) {
-        props->is_server = true;
     } else {
         error_setg(errp,
                    "vhost-user does not support a chardev with option %s=%s",
@@ -226,11 +301,12 @@ static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
 int net_init_vhost_user(const NetClientOptions *opts, const char *name,
                         NetClientState *peer, Error **errp)
 {
+    int queues;
     const NetdevVhostUserOptions *vhost_user_opts;
     CharDriverState *chr;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
-    vhost_user_opts = opts->vhost_user;
+    assert(opts->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+    vhost_user_opts = opts->u.vhost_user.data;
 
     chr = net_vhost_parse_chardev(vhost_user_opts, errp);
     if (!chr) {
@@ -243,6 +319,13 @@ int net_init_vhost_user(const NetClientOptions *opts, const char *name,
         return -1;
     }
 
+    queues = vhost_user_opts->has_queues ? vhost_user_opts->queues : 1;
+    if (queues < 1 || queues > MAX_QUEUE_NUM) {
+        error_setg(errp,
+                   "vhost-user number of queues must be in range [1, %d]",
+                   MAX_QUEUE_NUM);
+        return -1;
+    }
 
-    return net_vhost_user_init(peer, "vhost_user", name, chr);
+    return net_vhost_user_init(peer, "vhost_user", name, chr, queues);
 }
diff --git a/qemu/numa.c b/qemu/numa.c
index 402804bdf..572712ccf 100644
--- a/qemu/numa.c
+++ b/qemu/numa.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/numa.h"
 #include "exec/cpu-common.h"
 #include "qemu/bitmap.h"
@@ -30,7 +31,6 @@
 #include "include/exec/cpu-common.h" /* for RAM_ADDR_FMT */
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
 #include "hw/boards.h"
 #include "sysemu/hostmem.h"
 #include "qmp-commands.h"
@@ -218,7 +218,7 @@ static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
 
     {
         OptsVisitor *ov = opts_visitor_new(opts);
-        visit_type_NumaOptions(opts_get_visitor(ov), &object, NULL, &err);
+        visit_type_NumaOptions(opts_get_visitor(ov), NULL, &object, &err);
         opts_visitor_cleanup(ov);
     }
 
@@ -226,9 +226,9 @@ static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
         goto error;
     }
 
-    switch (object->kind) {
+    switch (object->type) {
     case NUMA_OPTIONS_KIND_NODE:
-        numa_node_parse(object->node, opts, &err);
+        numa_node_parse(object->u.node.data, opts, &err);
         if (err) {
             goto error;
         }
@@ -242,13 +242,7 @@ static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
 
 error:
     error_report_err(err);
-
-    if (object) {
-        QapiDeallocVisitor *dv = qapi_dealloc_visitor_new();
-        visit_type_NumaOptions(qapi_dealloc_get_visitor(dv),
-                               &object, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
-    }
+    qapi_free_NumaOptions(object);
 
     return -1;
 }
@@ -280,7 +274,7 @@ static void validate_numa_cpus(void)
             bitmap_and(seen_cpus, seen_cpus,
                        numa_info[i].node_cpu, MAX_CPUMASK_BITS);
             error_report("CPU(s) present in multiple NUMA nodes: %s",
-                         enumerate_cpus(seen_cpus, max_cpus));;
+                         enumerate_cpus(seen_cpus, max_cpus));
             exit(EXIT_FAILURE);
         }
         bitmap_or(seen_cpus, seen_cpus,
@@ -418,20 +412,23 @@ static void allocate_system_memory_nonnuma(MemoryRegion *mr, Object *owner,
         Error *err = NULL;
         memory_region_init_ram_from_file(mr, owner, name, ram_size, false,
                                          mem_path, &err);
-
-        /* Legacy behavior: if allocation failed, fall back to
-         * regular RAM allocation.
-         */
         if (err) {
             error_report_err(err);
-            memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
+            if (mem_prealloc) {
+                exit(1);
+            }
+
+            /* Legacy behavior: if allocation failed, fall back to
+             * regular RAM allocation.
+             */
+            memory_region_init_ram(mr, owner, name, ram_size, &error_fatal);
         }
 #else
         fprintf(stderr, "-mem-path not supported on this host\n");
         exit(1);
 #endif
     } else {
-        memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
+        memory_region_init_ram(mr, owner, name, ram_size, &error_fatal);
     }
     vmstate_register_ram_global(mr);
 }
@@ -450,17 +447,13 @@ void memory_region_allocate_system_memory(MemoryRegion *mr, Object *owner,
 
     memory_region_init(mr, owner, name, ram_size);
     for (i = 0; i < MAX_NODES; i++) {
-        Error *local_err = NULL;
         uint64_t size = numa_info[i].node_mem;
         HostMemoryBackend *backend = numa_info[i].node_memdev;
         if (!backend) {
             continue;
         }
-        MemoryRegion *seg = host_memory_backend_get_memory(backend, &local_err);
-        if (local_err) {
-            error_report_err(local_err);
-            exit(1);
-        }
+        MemoryRegion *seg = host_memory_backend_get_memory(backend,
+                                                           &error_fatal);
 
         if (memory_region_is_mapped(seg)) {
             char *path = object_get_canonical_path_component(OBJECT(backend));
@@ -487,9 +480,9 @@ static void numa_stat_memory_devices(uint64_t node_mem[])
         MemoryDeviceInfo *value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                node_mem[value->dimm->node] += value->dimm->size;
+                node_mem[value->u.dimm.data->node] += value->u.dimm.data->size;
                 break;
             default:
                 break;
@@ -517,7 +510,6 @@ static int query_memdev(Object *obj, void *opaque)
 {
     MemdevList **list = opaque;
     MemdevList *m = NULL;
-    Error *err = NULL;
 
     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
         m = g_malloc0(sizeof(*m));
@@ -525,72 +517,34 @@ static int query_memdev(Object *obj, void *opaque)
         m->value = g_malloc0(sizeof(*m->value));
 
         m->value->size = object_property_get_int(obj, "size",
-                                                 &err);
-        if (err) {
-            goto error;
-        }
-
+                                                 &error_abort);
         m->value->merge = object_property_get_bool(obj, "merge",
-                                                   &err);
-        if (err) {
-            goto error;
-        }
-
+                                                   &error_abort);
         m->value->dump = object_property_get_bool(obj, "dump",
-                                                  &err);
-        if (err) {
-            goto error;
-        }
-
+                                                  &error_abort);
         m->value->prealloc = object_property_get_bool(obj,
-                                                      "prealloc", &err);
-        if (err) {
-            goto error;
-        }
-
+                                                      "prealloc",
+                                                      &error_abort);
         m->value->policy = object_property_get_enum(obj,
                                                     "policy",
                                                     "HostMemPolicy",
-                                                    &err);
-        if (err) {
-            goto error;
-        }
-
+                                                    &error_abort);
         object_property_get_uint16List(obj, "host-nodes",
-                                       &m->value->host_nodes, &err);
-        if (err) {
-            goto error;
-        }
+                                       &m->value->host_nodes,
+                                       &error_abort);
 
         m->next = *list;
         *list = m;
     }
 
     return 0;
-error:
-    g_free(m->value);
-    g_free(m);
-
-    return -1;
 }
 
 MemdevList *qmp_query_memdev(Error **errp)
 {
-    Object *obj;
+    Object *obj = object_get_objects_root();
     MemdevList *list = NULL;
 
-    obj = object_get_objects_root();
-    if (obj == NULL) {
-        return NULL;
-    }
-
-    if (object_child_foreach(obj, query_memdev, &list) != 0) {
-        goto error;
-    }
-
+    object_child_foreach(obj, query_memdev, &list);
     return list;
-
-error:
-    qapi_free_MemdevList(list);
-    return NULL;
 }
diff --git a/qemu/os-posix.c b/qemu/os-posix.c
index e4da406f3..107fde38b 100644
--- a/qemu/os-posix.c
+++ b/qemu/os-posix.c
@@ -23,10 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/wait.h>
 /*needed for MAP_POPULATE before including qemu-options.h */
 #include <sys/mman.h>
@@ -35,11 +32,13 @@
 #include <libgen.h>
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "net/slirp.h"
 #include "qemu-options.h"
 #include "qemu/rcu.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+#include "qemu/cutils.h"
 
 #ifdef CONFIG_LINUX
 #include <sys/prctl.h>
@@ -139,6 +138,8 @@ void os_parse_cmd_args(int index, const char *optarg)
     switch (index) {
 #ifdef CONFIG_SLIRP
     case QEMU_OPTION_smb:
+        error_report("The -smb option is deprecated. "
+                     "Please use '-netdev user,smb=...' instead.");
         if (net_slirp_smb(optarg) < 0)
             exit(1);
         break;
@@ -276,7 +277,10 @@ void os_setup_post(void)
 
         dup2(fd, 0);
         dup2(fd, 1);
-        dup2(fd, 2);
+        /* In case -D is given do not redirect stderr to /dev/null */
+        if (!qemu_logfile) {
+            dup2(fd, 2);
+        }
 
         close(fd);
 
diff --git a/qemu/os-win32.c b/qemu/os-win32.c
index c0daf8e18..ae9857448 100644
--- a/qemu/os-win32.c
+++ b/qemu/os-win32.c
@@ -22,15 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <windows.h>
 #include <mmsystem.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "qemu-options.h"
 
diff --git a/qemu/page_cache.c b/qemu/page_cache.c
index cf8878d1d..cb8a69e96 100644
--- a/qemu/page_cache.c
+++ b/qemu/page_cache.c
@@ -12,14 +12,7 @@
  *
  */
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <stdbool.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qemu-common.h"
diff --git a/qemu/pc-bios/README b/qemu/pc-bios/README
index 05cf0421b..5a8a93193 100644
--- a/qemu/pc-bios/README
+++ b/qemu/pc-bios/README
@@ -17,7 +17,7 @@
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
   https://github.com/aik/SLOF, and the image currently in qemu is
-  built from git tag qemu-slof-20150429.
+  built from git tag qemu-slof-20160223.
 
 - sgabios (the Serial Graphics Adapter option ROM) provides a means for
   legacy x86 software to communicate with an attached serial console as
@@ -35,10 +35,6 @@
 	10ec:8139 -> pxe-rtl8139.rom
 	1af4:1000 -> pxe-virtio.rom
 
-- The S390 zipl loader is an addition to the official IBM s390-tools
-  package. That fork is maintained in its own git repository at:
-  git://repo.or.cz/s390-tools.git
-
 - The sources for the Alpha palcode image is available from:
   git://github.com/rth7680/qemu-palcode.git
 
diff --git a/qemu/pc-bios/bios-256k.bin b/qemu/pc-bios/bios-256k.bin
index f86adff12..e7a7e72e3 100644
--- a/qemu/pc-bios/bios-256k.bin
+++ b/qemu/pc-bios/bios-256k.bin
diff --git a/qemu/pc-bios/bios.bin b/qemu/pc-bios/bios.bin
index db835fb5f..b0ae502f6 100644
--- a/qemu/pc-bios/bios.bin
+++ b/qemu/pc-bios/bios.bin
diff --git a/qemu/pc-bios/efi-e1000.rom b/qemu/pc-bios/efi-e1000.rom
index 4e29d9d1a..4bc89a3d2 100644
--- a/qemu/pc-bios/efi-e1000.rom
+++ b/qemu/pc-bios/efi-e1000.rom
diff --git a/qemu/pc-bios/efi-eepro100.rom b/qemu/pc-bios/efi-eepro100.rom
index 2a92d6f32..85b7f9bc7 100644
--- a/qemu/pc-bios/efi-eepro100.rom
+++ b/qemu/pc-bios/efi-eepro100.rom
diff --git a/qemu/pc-bios/efi-ne2k_pci.rom b/qemu/pc-bios/efi-ne2k_pci.rom
index 636601700..ebafd8452 100644
--- a/qemu/pc-bios/efi-ne2k_pci.rom
+++ b/qemu/pc-bios/efi-ne2k_pci.rom
diff --git a/qemu/pc-bios/efi-pcnet.rom b/qemu/pc-bios/efi-pcnet.rom
index a61f586ee..6f1972375 100644
--- a/qemu/pc-bios/efi-pcnet.rom
+++ b/qemu/pc-bios/efi-pcnet.rom
diff --git a/qemu/pc-bios/efi-rtl8139.rom b/qemu/pc-bios/efi-rtl8139.rom
index c9c77ea7f..086551b76 100644
--- a/qemu/pc-bios/efi-rtl8139.rom
+++ b/qemu/pc-bios/efi-rtl8139.rom
diff --git a/qemu/pc-bios/efi-virtio.rom b/qemu/pc-bios/efi-virtio.rom
index eec279099..140c6806d 100644
--- a/qemu/pc-bios/efi-virtio.rom
+++ b/qemu/pc-bios/efi-virtio.rom
diff --git a/qemu/pc-bios/openbios-ppc b/qemu/pc-bios/openbios-ppc
index 540e45a74..ff980adb1 100644
--- a/qemu/pc-bios/openbios-ppc
+++ b/qemu/pc-bios/openbios-ppc
diff --git a/qemu/pc-bios/openbios-sparc32 b/qemu/pc-bios/openbios-sparc32
index 0da11883b..e744e898b 100644
--- a/qemu/pc-bios/openbios-sparc32
+++ b/qemu/pc-bios/openbios-sparc32
diff --git a/qemu/pc-bios/openbios-sparc64 b/qemu/pc-bios/openbios-sparc64
index 9bf3ce545..4d23be3ed 100644
--- a/qemu/pc-bios/openbios-sparc64
+++ b/qemu/pc-bios/openbios-sparc64
diff --git a/qemu/pc-bios/optionrom/optionrom.h b/qemu/pc-bios/optionrom/optionrom.h
index f1a9021ec..6c4c2c82f 100644
--- a/qemu/pc-bios/optionrom/optionrom.h
+++ b/qemu/pc-bios/optionrom/optionrom.h
@@ -19,8 +19,7 @@
  */
 
 
-#define NO_QEMU_PROTOS
-#include "../../include/hw/nvram/fw_cfg.h"
+#include "../../include/hw/nvram/fw_cfg_keys.h"
 
 #define BIOS_CFG_IOPORT_CFG	0x510
 #define BIOS_CFG_IOPORT_DATA	0x511
diff --git a/qemu/pc-bios/s390-ccw.img b/qemu/pc-bios/s390-ccw.img
index f64380a97..d3978ba05 100644
--- a/qemu/pc-bios/s390-ccw.img
+++ b/qemu/pc-bios/s390-ccw.img
diff --git a/qemu/pc-bios/s390-ccw/Makefile b/qemu/pc-bios/s390-ccw/Makefile
index 746603a31..4208cb429 100644
--- a/qemu/pc-bios/s390-ccw/Makefile
+++ b/qemu/pc-bios/s390-ccw/Makefile
@@ -9,8 +9,9 @@ $(call set-vpath, $(SRC_PATH)/pc-bios/s390-ccw)
 
 .PHONY : all clean build-all
 
-OBJECTS = start.o main.o bootmap.o sclp-ascii.o virtio.o
-CFLAGS += -fPIE -fno-stack-protector -ffreestanding -fno-delete-null-pointer-checks
+OBJECTS = start.o main.o bootmap.o sclp-ascii.o virtio.o virtio-scsi.o
+CFLAGS += -fPIE -fno-stack-protector -ffreestanding -march=z900
+CFLAGS += -fno-delete-null-pointer-checks -msoft-float
 LDFLAGS += -Wl,-pie -nostdlib
 
 build-all: s390-ccw.img
diff --git a/qemu/pc-bios/s390-ccw/bootmap.c b/qemu/pc-bios/s390-ccw/bootmap.c
index b678d5ebb..611102e3e 100644
--- a/qemu/pc-bios/s390-ccw/bootmap.c
+++ b/qemu/pc-bios/s390-ccw/bootmap.c
@@ -72,7 +72,7 @@ static void jump_to_IPL_code(uint64_t address)
     asm volatile("lghi 1,1\n\t"
                  "diag 1,1,0x308\n\t"
                  : : : "1", "memory");
-    virtio_panic("\n! IPL returns !\n");
+    panic("\n! IPL returns !\n");
 }
 
 /***********************************************************************
@@ -84,7 +84,7 @@ static const int max_bprs_entries = sizeof(_bprs) / sizeof(ExtEckdBlockPtr);
 
 static inline void verify_boot_info(BootInfo *bip)
 {
-    IPL_assert(magic_match(bip->magic, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(bip->magic, ZIPL_MAGIC), "No zIPL sig in BootInfo");
     IPL_assert(bip->version == BOOT_INFO_VERSION, "Wrong zIPL version");
     IPL_assert(bip->bp_type == BOOT_INFO_BP_TYPE_IPL, "DASD is not for IPL");
     IPL_assert(bip->dev_type == BOOT_INFO_DEV_TYPE_ECKD, "DASD is not ECKD");
@@ -315,6 +315,40 @@ static void print_eckd_msg(void)
     sclp_print(msg);
 }
 
+static void ipl_eckd(void)
+{
+    ScsiMbr *mbr = (void *)sec;
+    LDL_VTOC *vlbl = (void *)sec;
+
+    print_eckd_msg();
+
+    /* Grab the MBR again */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(0, mbr, "Cannot read block 0 on DASD");
+
+    if (magic_match(mbr->magic, IPL1_MAGIC)) {
+        ipl_eckd_cdl(); /* no return */
+    }
+
+    /* LDL/CMS? */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(2, vlbl, "Cannot read block 2");
+
+    if (magic_match(vlbl->magic, CMS1_MAGIC)) {
+        ipl_eckd_ldl(ECKD_CMS); /* no return */
+    }
+    if (magic_match(vlbl->magic, LNX1_MAGIC)) {
+        ipl_eckd_ldl(ECKD_LDL); /* no return */
+    }
+
+    ipl_eckd_ldl(ECKD_LDL_UNLABELED); /* it still may return */
+    /*
+     * Ok, it is not a LDL by any means.
+     * It still might be a CDL with zero record keys for IPL1 and IPL2
+     */
+    ipl_eckd_cdl();
+}
+
 /***********************************************************************
  * IPL a SCSI disk
  */
@@ -382,7 +416,7 @@ static void zipl_run(ScsiBlockPtr *pte)
     read_block(pte->blockno, tmp_sec, "Cannot read header");
     header = (ComponentHeader *)tmp_sec;
 
-    IPL_assert(magic_match(tmp_sec, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(tmp_sec, ZIPL_MAGIC), "No zIPL magic in header");
     IPL_assert(header->type == ZIPL_COMP_HEADER_IPL, "Bad header type");
 
     dputs("start loading images\n");
@@ -412,19 +446,29 @@ static void ipl_scsi(void)
     const int pte_len = sizeof(ScsiBlockPtr);
     ScsiBlockPtr *prog_table_entry;
 
-    /* The 0-th block (MBR) was already read into sec[] */
+    /* Grab the MBR */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(0, mbr, "Cannot read block 0");
+
+    if (!magic_match(mbr->magic, ZIPL_MAGIC)) {
+        return;
+    }
 
     sclp_print("Using SCSI scheme.\n");
+    debug_print_int("MBR Version", mbr->version_id);
+    IPL_check(mbr->version_id == 1,
+              "Unknown MBR layout version, assuming version 1");
     debug_print_int("program table", mbr->blockptr.blockno);
+    IPL_assert(mbr->blockptr.blockno, "No Program Table");
 
     /* Parse the program table */
     read_block(mbr->blockptr.blockno, sec,
                "Error reading Program Table");
 
-    IPL_assert(magic_match(sec, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(sec, ZIPL_MAGIC), "No zIPL magic in PT");
 
     ns_end = sec + virtio_get_block_size();
-    for (ns = (sec + pte_len); (ns + pte_len) < ns_end; ns++) {
+    for (ns = (sec + pte_len); (ns + pte_len) < ns_end; ns += pte_len) {
         prog_table_entry = (ScsiBlockPtr *)ns;
         if (!prog_table_entry->blockno) {
             break;
@@ -445,51 +489,258 @@ static void ipl_scsi(void)
 }
 
 /***********************************************************************
- * IPL starts here
+ * IPL El Torito ISO9660 image or DVD
  */
 
-void zipl_load(void)
+static bool is_iso_bc_entry_compatible(IsoBcSection *s)
 {
-    ScsiMbr *mbr = (void *)sec;
-    LDL_VTOC *vlbl = (void *)sec;
+    uint8_t *magic_sec = (uint8_t *)(sec + ISO_SECTOR_SIZE);
 
-    /* Grab the MBR */
-    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
-    read_block(0, mbr, "Cannot read block 0");
+    if (s->unused || !s->sector_count) {
+        return false;
+    }
+    read_iso_sector(bswap32(s->load_rba), magic_sec,
+                    "Failed to read image sector 0");
 
-    dputs("checking magic\n");
+    /* Checking bytes 8 - 32 for S390 Linux magic */
+    return !_memcmp(magic_sec + 8, linux_s390_magic, 24);
+}
+
+/* Location of the current sector of the directory */
+static uint32_t sec_loc[ISO9660_MAX_DIR_DEPTH];
+/* Offset in the current sector of the directory */
+static uint32_t sec_offset[ISO9660_MAX_DIR_DEPTH];
+/* Remained directory space in bytes */
+static uint32_t dir_rem[ISO9660_MAX_DIR_DEPTH];
 
-    if (magic_match(mbr->magic, ZIPL_MAGIC)) {
-        ipl_scsi(); /* no return */
+static inline uint32_t iso_get_file_size(uint32_t load_rba)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    IsoDirHdr *cur_record = &vd->vd.primary.rootdir;
+    uint8_t *temp = sec + ISO_SECTOR_SIZE;
+    int level = 0;
+
+    read_iso_sector(ISO_PRIMARY_VD_SECTOR, sec,
+                    "Failed to read ISO primary descriptor");
+    sec_loc[0] = iso_733_to_u32(cur_record->ext_loc);
+    dir_rem[0] = 0;
+    sec_offset[0] = 0;
+
+    while (level >= 0) {
+        IPL_assert(sec_offset[level] <= ISO_SECTOR_SIZE,
+                   "Directory tree structure violation");
+
+        cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+
+        if (sec_offset[level] == 0) {
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+            if (dir_rem[level] == 0) {
+                /* Skip self and parent records */
+                dir_rem[level] = iso_733_to_u32(cur_record->data_len) -
+                                 cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+
+                cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+                dir_rem[level] -= cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+                continue;
+            }
+        }
+
+        if (!cur_record->dr_len || sec_offset[level] == ISO_SECTOR_SIZE) {
+            /* Zero-padding and/or the end of current sector */
+            dir_rem[level] -= ISO_SECTOR_SIZE - sec_offset[level];
+            sec_offset[level] = 0;
+            sec_loc[level]++;
+        } else {
+            /* The directory record is valid */
+            if (load_rba == iso_733_to_u32(cur_record->ext_loc)) {
+                return iso_733_to_u32(cur_record->data_len);
+            }
+
+            dir_rem[level] -= cur_record->dr_len;
+            sec_offset[level] += cur_record->dr_len;
+
+            if (cur_record->file_flags & 0x2) {
+                /* Subdirectory */
+                if (level == ISO9660_MAX_DIR_DEPTH - 1) {
+                    sclp_print("ISO-9660 directory depth limit exceeded\n");
+                } else {
+                    level++;
+                    sec_loc[level] = iso_733_to_u32(cur_record->ext_loc);
+                    sec_offset[level] = 0;
+                    dir_rem[level] = 0;
+                    continue;
+                }
+            }
+        }
+
+        if (dir_rem[level] == 0) {
+            /* Nothing remaining */
+            level--;
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+        }
     }
 
-    /* We have failed to follow the SCSI scheme, so */
-    if (virtio_guessed_disk_nature()) {
-        sclp_print("Using guessed DASD geometry.\n");
-        virtio_assume_eckd();
+    return 0;
+}
+
+static void load_iso_bc_entry(IsoBcSection *load)
+{
+    IsoBcSection s = *load;
+    /*
+     * According to spec, extent for each file
+     * is padded and ISO_SECTOR_SIZE bytes aligned
+     */
+    uint32_t blks_to_load = bswap16(s.sector_count) >> ET_SECTOR_SHIFT;
+    uint32_t real_size = iso_get_file_size(bswap32(s.load_rba));
+
+    if (real_size) {
+        /* Round up blocks to load */
+        blks_to_load = (real_size + ISO_SECTOR_SIZE - 1) / ISO_SECTOR_SIZE;
+        sclp_print("ISO boot image size verified\n");
+    } else {
+        sclp_print("ISO boot image size could not be verified\n");
     }
-    print_eckd_msg();
-    if (magic_match(mbr->magic, IPL1_MAGIC)) {
-        ipl_eckd_cdl(); /* no return */
+
+    read_iso_boot_image(bswap32(s.load_rba),
+                        (void *)((uint64_t)bswap16(s.load_segment)),
+                        blks_to_load);
+
+    /* Trying to get PSW at zero address */
+    if (*((uint64_t *)0) & IPL_PSW_MASK) {
+        jump_to_IPL_code((*((uint64_t *)0)) & 0x7fffffff);
     }
 
-    /* LDL/CMS? */
-    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
-    read_block(2, vlbl, "Cannot read block 2");
+    /* Try default linux start address */
+    jump_to_IPL_code(KERN_IMAGE_START);
+}
 
-    if (magic_match(vlbl->magic, CMS1_MAGIC)) {
-        ipl_eckd_ldl(ECKD_CMS); /* no return */
+static uint32_t find_iso_bc(void)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    uint32_t block_num = ISO_PRIMARY_VD_SECTOR;
+
+    if (virtio_read_many(block_num++, sec, 1)) {
+        /* If primary vd cannot be read, there is no boot catalog */
+        return 0;
     }
-    if (magic_match(vlbl->magic, LNX1_MAGIC)) {
-        ipl_eckd_ldl(ECKD_LDL); /* no return */
+
+    while (is_iso_vd_valid(vd) && vd->type != VOL_DESC_TERMINATOR) {
+        if (vd->type == VOL_DESC_TYPE_BOOT) {
+            IsoVdElTorito *et = &vd->vd.boot;
+
+            if (!_memcmp(&et->el_torito[0], el_torito_magic, 32)) {
+                return bswap32(et->bc_offset);
+            }
+        }
+        read_iso_sector(block_num++, sec,
+                        "Failed to read ISO volume descriptor");
+    }
+
+    return 0;
+}
+
+static IsoBcSection *find_iso_bc_entry(void)
+{
+    IsoBcEntry *e = (IsoBcEntry *)sec;
+    uint32_t offset = find_iso_bc();
+    int i;
+
+    if (!offset) {
+        return NULL;
+    }
+
+    read_iso_sector(offset, sec, "Failed to read El Torito boot catalog");
+
+    if (!is_iso_bc_valid(e)) {
+        /* The validation entry is mandatory */
+        panic("No valid boot catalog found!\n");
+        return NULL;
     }
 
-    ipl_eckd_ldl(ECKD_LDL_UNLABELED); /* it still may return */
     /*
-     * Ok, it is not a LDL by any means.
-     * It still might be a CDL with zero record keys for IPL1 and IPL2
+     * Each entry has 32 bytes size, so one sector cannot contain > 64 entries.
+     * We consider only boot catalogs with no more than 64 entries.
      */
-    ipl_eckd_cdl();
+    for (i = 1; i < ISO_BC_ENTRY_PER_SECTOR; i++) {
+        if (e[i].id == ISO_BC_BOOTABLE_SECTION) {
+            if (is_iso_bc_entry_compatible(&e[i].body.sect)) {
+                return &e[i].body.sect;
+            }
+        }
+    }
+
+    panic("No suitable boot entry found on ISO-9660 media!\n");
+
+    return NULL;
+}
+
+static void ipl_iso_el_torito(void)
+{
+    IsoBcSection *s = find_iso_bc_entry();
+
+    if (s) {
+        load_iso_bc_entry(s);
+        /* no return */
+    }
+}
+
+/***********************************************************************
+ * Bus specific IPL sequences
+ */
+
+static void zipl_load_vblk(void)
+{
+    if (virtio_guessed_disk_nature()) {
+        virtio_assume_iso9660();
+    }
+    ipl_iso_el_torito();
+
+    if (virtio_guessed_disk_nature()) {
+        sclp_print("Using guessed DASD geometry.\n");
+        virtio_assume_eckd();
+    }
+    ipl_eckd();
+}
+
+static void zipl_load_vscsi(void)
+{
+    if (virtio_get_block_size() == VIRTIO_ISO_BLOCK_SIZE) {
+        /* Is it an ISO image in non-CD drive? */
+        ipl_iso_el_torito();
+    }
+
+    sclp_print("Using guessed DASD geometry.\n");
+    virtio_assume_eckd();
+    ipl_eckd();
+}
+
+/***********************************************************************
+ * IPL starts here
+ */
+
+void zipl_load(void)
+{
+    if (virtio_get_device()->is_cdrom) {
+        ipl_iso_el_torito();
+        panic("\n! Cannot IPL this ISO image !\n");
+    }
+
+    ipl_scsi();
+
+    switch (virtio_get_device_type()) {
+    case VIRTIO_ID_BLOCK:
+        zipl_load_vblk();
+        break;
+    case VIRTIO_ID_SCSI:
+        zipl_load_vscsi();
+        break;
+    default:
+        panic("\n! Unknown IPL device type !\n");
+    }
 
-    virtio_panic("\n* this can never happen *\n");
+    panic("\n* this can never happen *\n");
 }
diff --git a/qemu/pc-bios/s390-ccw/bootmap.h b/qemu/pc-bios/s390-ccw/bootmap.h
index ab132e357..bea168714 100644
--- a/qemu/pc-bios/s390-ccw/bootmap.h
+++ b/qemu/pc-bios/s390-ccw/bootmap.h
@@ -264,15 +264,6 @@ typedef enum {
 
 /* utility code below */
 
-static inline void IPL_assert(bool term, const char *message)
-{
-    if (!term) {
-        sclp_print("\n! ");
-        sclp_print(message);
-        virtio_panic(" !\n"); /* no return */
-    }
-}
-
 static const unsigned char ebc2asc[256] =
       /* 0123456789abcdef0123456789abcdef */
         "................................" /* 1F */
@@ -341,4 +332,210 @@ static inline bool magic_match(const void *data, const void *magic)
     return *((uint32_t *)data) == *((uint32_t *)magic);
 }
 
+static inline int _memcmp(const void *s1, const void *s2, size_t n)
+{
+    int i;
+    const uint8_t *p1 = s1, *p2 = s2;
+
+    for (i = 0; i < n; i++) {
+        if (p1[i] != p2[i]) {
+            return p1[i] > p2[i] ? 1 : -1;
+        }
+    }
+
+    return 0;
+}
+
+/* from include/qemu/bswap.h */
+
+/* El Torito is always little-endian */
+static inline uint16_t bswap16(uint16_t x)
+{
+    return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return ((x & 0x000000ffU) << 24) | ((x & 0x0000ff00U) <<  8) |
+           ((x & 0x00ff0000U) >>  8) | ((x & 0xff000000U) >> 24);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return ((x & 0x00000000000000ffULL) << 56) |
+           ((x & 0x000000000000ff00ULL) << 40) |
+           ((x & 0x0000000000ff0000ULL) << 24) |
+           ((x & 0x00000000ff000000ULL) <<  8) |
+           ((x & 0x000000ff00000000ULL) >>  8) |
+           ((x & 0x0000ff0000000000ULL) >> 24) |
+           ((x & 0x00ff000000000000ULL) >> 40) |
+           ((x & 0xff00000000000000ULL) >> 56);
+}
+
+static inline uint32_t iso_733_to_u32(uint64_t x)
+{
+    return (uint32_t)x;
+}
+
+#define ISO_SECTOR_SIZE 2048
+/* El Torito specifies boot image size in 512 byte blocks */
+#define ET_SECTOR_SHIFT 2
+#define KERN_IMAGE_START 0x010000UL
+#define PSW_MASK_64 0x0000000100000000ULL
+#define PSW_MASK_32 0x0000000080000000ULL
+#define IPL_PSW_MASK (PSW_MASK_32 | PSW_MASK_64)
+
+#define ISO_PRIMARY_VD_SECTOR 16
+
+static inline void read_iso_sector(uint32_t block_offset, void *buf,
+                                   const char *errmsg)
+{
+    IPL_assert(virtio_read_many(block_offset, buf, 1) == 0, errmsg);
+}
+
+static inline void read_iso_boot_image(uint32_t block_offset, void *load_addr,
+                                       uint32_t blks_to_load)
+{
+    IPL_assert(virtio_read_many(block_offset, load_addr, blks_to_load) == 0,
+               "Failed to read boot image!");
+}
+
+const uint8_t el_torito_magic[] = "EL TORITO SPECIFICATION"
+                                  "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
+
+#define ISO9660_MAX_DIR_DEPTH 8
+
+typedef struct IsoDirHdr {
+    uint8_t dr_len;
+    uint8_t ear_len;
+    uint64_t ext_loc;
+    uint64_t data_len;
+    uint8_t recording_datetime[7];
+    uint8_t file_flags;
+    uint8_t file_unit_size;
+    uint8_t gap_size;
+    uint32_t vol_seqnum;
+    uint8_t fileid_len;
+} __attribute__((packed)) IsoDirHdr;
+
+typedef struct IsoVdElTorito {
+    uint8_t el_torito[32]; /* must contain el_torito_magic value */
+    uint8_t unused0[32];
+    uint32_t bc_offset;
+    uint8_t unused1[1974];
+} __attribute__((packed)) IsoVdElTorito;
+
+typedef struct IsoVdPrimary {
+    uint8_t unused1;
+    uint8_t sys_id[32];
+    uint8_t vol_id[32];
+    uint8_t unused2[8];
+    uint64_t vol_space_size;
+    uint8_t unused3[32];
+    uint32_t vol_set_size;
+    uint32_t vol_seqnum;
+    uint32_t log_block_size;
+    uint64_t path_table_size;
+    uint32_t l_path_table;
+    uint32_t opt_l_path_table;
+    uint32_t m_path_table;
+    uint32_t opt_m_path_table;
+    IsoDirHdr rootdir;
+    uint8_t root_null;
+    uint8_t reserved2[1858];
+} __attribute__((packed)) IsoVdPrimary;
+
+typedef struct IsoVolDesc {
+    uint8_t type;
+    uint8_t ident[5];
+    uint8_t version;
+    union {
+        IsoVdElTorito boot;
+        IsoVdPrimary primary;
+    } vd;
+} __attribute__((packed)) IsoVolDesc;
+
+const uint8_t vol_desc_magic[] = "CD001";
+#define VOL_DESC_TYPE_BOOT 0
+#define VOL_DESC_TYPE_PRIMARY 1
+#define VOL_DESC_TYPE_SUPPLEMENT 2
+#define VOL_DESC_TYPE_PARTITION 3
+#define VOL_DESC_TERMINATOR 255
+
+static inline bool is_iso_vd_valid(IsoVolDesc *vd)
+{
+    return !_memcmp(&vd->ident[0], vol_desc_magic, 5) &&
+           vd->version == 0x1 &&
+           vd->type <= VOL_DESC_TYPE_PARTITION;
+}
+
+typedef struct IsoBcValid {
+    uint8_t platform_id;
+    uint16_t reserved;
+    uint8_t id[24];
+    uint16_t checksum;
+    uint8_t key[2];
+} __attribute__((packed)) IsoBcValid;
+
+typedef struct IsoBcSection {
+    uint8_t boot_type;
+    uint16_t load_segment;
+    uint8_t sys_type;
+    uint8_t unused;
+    uint16_t sector_count;
+    uint32_t load_rba;
+    uint8_t selection[20];
+} __attribute__((packed)) IsoBcSection;
+
+typedef struct IsoBcHdr {
+    uint8_t platform_id;
+    uint16_t sect_num;
+    uint8_t id[28];
+} __attribute__((packed)) IsoBcHdr;
+
+/*
+ * Match two CCWs located after PSW and eight filler bytes.
+ * From libmagic and arch/s390/kernel/head.S.
+ */
+const uint8_t linux_s390_magic[] = "\x02\x00\x00\x18\x60\x00\x00\x50\x02\x00"
+                                   "\x00\x68\x60\x00\x00\x50\x40\x40\x40\x40"
+                                   "\x40\x40\x40\x40";
+
+typedef struct IsoBcEntry {
+    uint8_t id;
+    union {
+        IsoBcValid valid; /* id == 0x01 */
+        IsoBcSection sect; /* id == 0x88 || id == 0x0 */
+        IsoBcHdr hdr; /* id == 0x90 || id == 0x91 */
+    } body;
+} __attribute__((packed)) IsoBcEntry;
+
+#define ISO_BC_ENTRY_PER_SECTOR (ISO_SECTOR_SIZE / sizeof(IsoBcEntry))
+#define ISO_BC_HDR_VALIDATION 0x01
+#define ISO_BC_BOOTABLE_SECTION 0x88
+#define ISO_BC_MAGIC_55 0x55
+#define ISO_BC_MAGIC_AA 0xaa
+#define ISO_BC_PLATFORM_X86 0x0
+#define ISO_BC_PLATFORM_PPC 0x1
+#define ISO_BC_PLATFORM_MAC 0x2
+
+static inline bool is_iso_bc_valid(IsoBcEntry *e)
+{
+    IsoBcValid *v = &e->body.valid;
+
+    if (e->id != ISO_BC_HDR_VALIDATION) {
+        return false;
+    }
+
+    if (v->platform_id != ISO_BC_PLATFORM_X86 &&
+        v->platform_id != ISO_BC_PLATFORM_PPC &&
+        v->platform_id != ISO_BC_PLATFORM_MAC) {
+        return false;
+    }
+
+    return v->key[0] == ISO_BC_MAGIC_55 &&
+           v->key[1] == ISO_BC_MAGIC_AA &&
+           v->reserved == 0x0;
+}
+
 #endif /* _PC_BIOS_S390_CCW_BOOTMAP_H */
diff --git a/qemu/pc-bios/s390-ccw/main.c b/qemu/pc-bios/s390-ccw/main.c
index 584d4a276..1c9e0791a 100644
--- a/qemu/pc-bios/s390-ccw/main.c
+++ b/qemu/pc-bios/s390-ccw/main.c
@@ -12,9 +12,8 @@
 #include "virtio.h"
 
 char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
-char ring_area[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
 uint64_t boot_value;
-static struct subchannel_id blk_schid = { .one = 1 };
+static SubChannelId blk_schid = { .one = 1 };
 
 /*
  * Priniciples of Operations (SA22-7832-09) chapter 17 requires that
@@ -23,7 +22,7 @@ static struct subchannel_id blk_schid = { .one = 1 };
  */
 void write_subsystem_identification(void)
 {
-    struct subchannel_id *schid = (struct subchannel_id *) 184;
+    SubChannelId *schid = (SubChannelId *) 184;
     uint32_t *zeroes = (uint32_t *) 188;
 
     *schid = blk_schid;
@@ -31,60 +30,72 @@ void write_subsystem_identification(void)
 }
 
 
-void virtio_panic(const char *string)
+void panic(const char *string)
 {
     sclp_print(string);
     disabled_wait();
     while (1) { }
 }
 
+static bool find_dev(Schib *schib, int dev_no)
+{
+    int i, r;
+
+    for (i = 0; i < 0x10000; i++) {
+        blk_schid.sch_no = i;
+        r = stsch_err(blk_schid, schib);
+        if ((r == 3) || (r == -EIO)) {
+            break;
+        }
+        if (!schib->pmcw.dnv) {
+            continue;
+        }
+        if (!virtio_is_supported(blk_schid)) {
+            continue;
+        }
+        if ((dev_no < 0) || (schib->pmcw.dev == dev_no)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static void virtio_setup(uint64_t dev_info)
 {
-    struct schib schib;
-    int i;
-    int r;
+    Schib schib;
+    int ssid;
     bool found = false;
-    bool check_devno = false;
-    uint16_t dev_no = -1;
+    uint16_t dev_no;
+
+    /*
+     * We unconditionally enable mss support. In every sane configuration,
+     * this will succeed; and even if it doesn't, stsch_err() can deal
+     * with the consequences.
+     */
+    enable_mss_facility();
 
     if (dev_info != -1) {
-        check_devno = true;
         dev_no = dev_info & 0xffff;
         debug_print_int("device no. ", dev_no);
         blk_schid.ssid = (dev_info >> 16) & 0x3;
-        if (blk_schid.ssid != 0) {
-            debug_print_int("ssid ", blk_schid.ssid);
-            if (enable_mss_facility() != 0) {
-                virtio_panic("Failed to enable mss facility\n");
-            }
-        }
-    }
-
-    for (i = 0; i < 0x10000; i++) {
-        blk_schid.sch_no = i;
-        r = stsch_err(blk_schid, &schib);
-        if (r == 3) {
-            break;
-        }
-        if (schib.pmcw.dnv) {
-            if (!check_devno || (schib.pmcw.dev == dev_no)) {
-                if (virtio_is_blk(blk_schid)) {
-                    found = true;
-                    break;
-                }
+        debug_print_int("ssid ", blk_schid.ssid);
+        found = find_dev(&schib, dev_no);
+    } else {
+        for (ssid = 0; ssid < 0x3; ssid++) {
+            blk_schid.ssid = ssid;
+            found = find_dev(&schib, -1);
+            if (found) {
+                break;
             }
         }
     }
 
-    if (!found) {
-        virtio_panic("No virtio-blk device found!\n");
-    }
+    IPL_assert(found, "No virtio device found");
 
-    virtio_setup_block(blk_schid);
+    virtio_setup_device(blk_schid);
 
-    if (!virtio_ipl_disk_is_valid()) {
-        virtio_panic("No valid hard disk detected.\n");
-    }
+    IPL_assert(virtio_ipl_disk_is_valid(), "No valid IPL device detected");
 }
 
 int main(void)
@@ -95,6 +106,6 @@ int main(void)
 
     zipl_load(); /* no return */
 
-    virtio_panic("Failed to load OS from hard disk\n");
+    panic("Failed to load OS from hard disk\n");
     return 0; /* make compiler happy */
 }
diff --git a/qemu/pc-bios/s390-ccw/s390-ccw.h b/qemu/pc-bios/s390-ccw/s390-ccw.h
index 5484c2a45..616d96738 100644
--- a/qemu/pc-bios/s390-ccw/s390-ccw.h
+++ b/qemu/pc-bios/s390-ccw/s390-ccw.h
@@ -45,15 +45,22 @@ typedef unsigned long long __u64;
 
 #include "cio.h"
 
+typedef struct irb Irb;
+typedef struct ccw1 Ccw1;
+typedef struct cmd_orb CmdOrb;
+typedef struct schib Schib;
+typedef struct chsc_area_sda ChscAreaSda;
+typedef struct senseid SenseId;
+typedef struct subchannel_id SubChannelId;
+
 /* start.s */
 void disabled_wait(void);
 void consume_sclp_int(void);
 
 /* main.c */
-void virtio_panic(const char *string);
+void panic(const char *string);
 void write_subsystem_identification(void);
 extern char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
-extern char ring_area[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
 extern uint64_t boot_value;
 
 /* sclp-ascii.c */
@@ -63,10 +70,11 @@ void sclp_setup(void);
 /* virtio.c */
 unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
                                  ulong subchan_id, void *load_addr);
-bool virtio_is_blk(struct subchannel_id schid);
-void virtio_setup_block(struct subchannel_id schid);
+bool virtio_is_supported(SubChannelId schid);
+void virtio_setup_device(SubChannelId schid);
 int virtio_read(ulong sector, void *load_addr);
 int enable_mss_facility(void);
+ulong get_second(void);
 
 /* bootmap.c */
 void zipl_load(void);
@@ -143,4 +151,42 @@ static inline void yield(void)
 
 #define MAX_SECTOR_SIZE 4096
 
+static inline void sleep(unsigned int seconds)
+{
+    ulong target = get_second() + seconds;
+
+    while (get_second() < target) {
+        yield();
+    }
+}
+
+static inline void *memcpy(void *s1, const void *s2, size_t n)
+{
+    uint8_t *p1 = s1;
+    const uint8_t *p2 = s2;
+
+    while (n--) {
+        p1[n] = p2[n];
+    }
+    return s1;
+}
+
+static inline void IPL_assert(bool term, const char *message)
+{
+    if (!term) {
+        sclp_print("\n! ");
+        sclp_print(message);
+        panic(" !\n"); /* no return */
+    }
+}
+
+static inline void IPL_check(bool term, const char *message)
+{
+    if (!term) {
+        sclp_print("\n! WARNING: ");
+        sclp_print(message);
+        sclp_print(" !\n");
+    }
+}
+
 #endif /* S390_CCW_H */
diff --git a/qemu/pc-bios/s390-ccw/scsi.h b/qemu/pc-bios/s390-ccw/scsi.h
new file mode 100644
index 000000000..fc830f7e5
--- /dev/null
+++ b/qemu/pc-bios/s390-ccw/scsi.h
@@ -0,0 +1,184 @@
+/*
+ * SCSI definitions for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef SCSI_H
+#define SCSI_H
+
+#include "s390-ccw.h"
+
+#define SCSI_DEFAULT_CDB_SIZE                   32
+#define SCSI_DEFAULT_SENSE_SIZE                 96
+
+#define CDB_STATUS_GOOD                         0
+#define CDB_STATUS_CHECK_CONDITION              0x02U
+#define CDB_STATUS_VALID(status)    (((status) & ~0x3eU) == 0)
+
+#define SCSI_SENSE_CODE_MASK                    0x7fU
+#define SCSI_SENSE_KEY_MASK                     0x0fU
+#define SCSI_SENSE_KEY_NO_SENSE                 0
+#define SCSI_SENSE_KEY_UNIT_ATTENTION           6
+
+union ScsiLun {
+    uint64_t v64;        /* numeric shortcut                             */
+    uint8_t  v8[8];      /* generic 8 bytes representation               */
+    uint16_t v16[4];     /* 4-level big-endian LUN as specified by SAM-2 */
+};
+typedef union ScsiLun ScsiLun;
+
+struct ScsiSense70 {
+    uint8_t b0;         /* b0 & 7f = resp code (0x70 or 0x71)   */
+    uint8_t b1, b2;     /* b2 & 0f = sense key                  */
+    uint8_t u1[1 * 4 + 1 + 1 * 4];   /* b7 = N - 7                  */
+    uint8_t additional_sense_code;              /* b12          */
+    uint8_t additional_sense_code_qualifier;    /* b13          */
+    uint8_t u2[1 + 3 + 0];           /* up to N (<=252) bytes       */
+} __attribute__((packed));
+typedef struct ScsiSense70 ScsiSense70;
+
+/* don't confuse with virtio-scsi response/status fields! */
+
+static inline uint8_t scsi_sense_response(const void *p)
+{
+    return ((const ScsiSense70 *)p)->b0 & SCSI_SENSE_CODE_MASK;
+}
+
+static inline uint8_t scsi_sense_key(const void *p)
+{
+    return ((const ScsiSense70 *)p)->b2 & SCSI_SENSE_KEY_MASK;
+}
+
+#define SCSI_INQ_RDT_CDROM                      0x05
+
+struct ScsiInquiryStd {
+    uint8_t peripheral_qdt; /* b0, use (b0 & 0x1f) to get SCSI_INQ_RDT  */
+    uint8_t b1;             /* Removable Media Bit = b1 & 0x80          */
+    uint8_t spc_version;    /* b2                                       */
+    uint8_t b3;             /* b3 & 0x0f == resp_data_fmt == 2, must!   */
+    uint8_t u1[1 + 1 + 1 + 1 + 8];  /* b4..b15 unused, b4 = (N - 1)     */
+    char prod_id[16];       /* "QEMU CD-ROM" is here                    */
+    uint8_t u2[4            /* b32..b35 unused, mandatory               */
+              + 8 + 12 + 1 + 1 + 8 * 2 + 22  /* b36..95 unused, optional*/
+              + 0];          /* b96..bN unused, vendor specific          */
+    /* byte N                                                           */
+}  __attribute__((packed));
+typedef struct ScsiInquiryStd ScsiInquiryStd;
+
+struct ScsiCdbInquiry {
+    uint8_t command;     /* b0, == 0x12         */
+    uint8_t b1;          /* b1, |= 0x01 (evpd)  */
+    uint8_t b2;          /* b2; if evpd==1      */
+    uint16_t alloc_len;  /* b3, b4              */
+    uint8_t control;     /* b5                  */
+}  __attribute__((packed));
+typedef struct ScsiCdbInquiry ScsiCdbInquiry;
+
+struct ScsiCdbRead10 {
+    uint8_t command;    /* =0x28    */
+    uint8_t b1;
+    uint32_t lba;
+    uint8_t b6;
+    uint16_t xfer_length;
+    uint8_t control;
+}  __attribute__((packed));
+typedef struct ScsiCdbRead10 ScsiCdbRead10;
+
+struct ScsiCdbTestUnitReady {
+    uint8_t command;    /* =0x00    */
+    uint8_t b1_b4[4];
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbTestUnitReady ScsiCdbTestUnitReady;
+
+struct ScsiCdbReportLuns {
+    uint8_t command;        /* =0xa0        */
+    uint8_t b1;
+    uint8_t select_report;  /* =0x02, "all" */
+    uint8_t b3_b5[3];
+    uint32_t alloc_len;
+    uint8_t b10;
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbReportLuns ScsiCdbReportLuns;
+
+struct ScsiLunReport {
+    uint32_t lun_list_len;
+    uint32_t b4_b7;
+    ScsiLun lun[1];   /* space for at least 1 lun must be allocated */
+} __attribute__((packed));
+typedef struct ScsiLunReport ScsiLunReport;
+
+struct ScsiCdbReadCapacity16 {
+    uint8_t command;        /* =0x9e = "service action in 16"       */
+    uint8_t service_action; /* 5 bits, =0x10 = "read capacity 16"   */
+    uint64_t b2_b9;
+    uint32_t alloc_len;
+    uint8_t b14;
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbReadCapacity16 ScsiCdbReadCapacity16;
+
+struct ScsiReadCapacity16Data {
+    uint64_t ret_lba;             /* get it, 0..7     */
+    uint32_t lb_len;              /* bytes, 8..11     */
+    uint8_t u1[2 + 1 * 2 + 16];   /* b12..b31, unused */
+} __attribute__((packed));
+typedef struct ScsiReadCapacity16Data ScsiReadCapacity16Data;
+
+static inline ScsiLun make_lun(uint16_t channel, uint16_t target, uint32_t lun)
+{
+    ScsiLun r = { .v64 = 0 };
+
+    /* See QEMU code to choose the way to handle LUNs.
+     *
+     * So, a valid LUN must have (always channel #0):
+     *  lun[0] == 1
+     *  lun[1] - target, any value
+     *  lun[2] == 0 or (LUN, MSB, 0x40 set, 0x80 clear)
+     *  lun[3] - LUN, LSB, any value
+     */
+    r.v8[0] = 1;
+    r.v8[1] = target & 0xffU;
+    r.v8[2] = (lun >> 8) & 0x3fU;
+    if (r.v8[2]) {
+        r.v8[2] |= 0x40;
+    }
+    r.v8[3] = lun & 0xffU;
+
+    return r;
+}
+
+static inline const char *scsi_cdb_status_msg(uint8_t status)
+{
+    static char err_msg[] = "STATUS=XX";
+    uint8_t v = status & 0x3eU;
+
+    fill_hex_val(err_msg + 7, &v, 1);
+    return err_msg;
+}
+
+static inline const char *scsi_cdb_asc_msg(const void *s)
+{
+    static char err_msg[] = "RSPN=XX KEY=XX CODE=XX QLFR=XX";
+    const ScsiSense70 *p = s;
+    uint8_t sr = scsi_sense_response(s);
+    uint8_t sk = scsi_sense_key(s);
+    uint8_t ac = p->additional_sense_code;
+    uint8_t cq = p->additional_sense_code_qualifier;
+
+    fill_hex_val(err_msg + 5, &sr, 1);
+    fill_hex_val(err_msg + 12, &sk, 1);
+    fill_hex_val(err_msg + 20, &ac, 1);
+    fill_hex_val(err_msg + 28, &cq, 1);
+
+    return err_msg;
+}
+
+#endif /* SCSI_H */
diff --git a/qemu/pc-bios/s390-ccw/virtio-scsi.c b/qemu/pc-bios/s390-ccw/virtio-scsi.c
new file mode 100644
index 000000000..3bb48e917
--- /dev/null
+++ b/qemu/pc-bios/s390-ccw/virtio-scsi.c
@@ -0,0 +1,342 @@
+/*
+ * Virtio-SCSI implementation for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "s390-ccw.h"
+#include "virtio.h"
+#include "scsi.h"
+#include "virtio-scsi.h"
+
+static ScsiDevice default_scsi_device;
+static VirtioScsiCmdReq req;
+static VirtioScsiCmdResp resp;
+
+static uint8_t scsi_inquiry_std_response[256];
+
+static inline void vs_assert(bool term, const char **msgs)
+{
+    if (!term) {
+        int i = 0;
+
+        sclp_print("\n! ");
+        while (msgs[i]) {
+            sclp_print(msgs[i++]);
+        }
+        panic(" !\n");
+    }
+}
+
+static void virtio_scsi_verify_response(VirtioScsiCmdResp *resp,
+                                        const char *title)
+{
+    const char *mr[] = {
+        title, ": response ", virtio_scsi_response_msg(resp), 0
+    };
+    const char *ms[] = {
+        title,
+        CDB_STATUS_VALID(resp->status) ? ": " : ": invalid ",
+        scsi_cdb_status_msg(resp->status),
+        resp->status == CDB_STATUS_CHECK_CONDITION ? " " : 0,
+        resp->sense_len ? scsi_cdb_asc_msg(resp->sense)
+                        : "no sense data",
+        scsi_sense_response(resp->sense)  == 0x70 ? ", sure" : "?",
+        0
+    };
+
+    vs_assert(resp->response == VIRTIO_SCSI_S_OK, mr);
+    vs_assert(resp->status == CDB_STATUS_GOOD, ms);
+}
+
+static void prepare_request(VDev *vdev, const void *cdb, int cdb_size,
+                            void *data, uint32_t data_size)
+{
+    const ScsiDevice *sdev = vdev->scsi_device;
+
+    memset(&req, 0, sizeof(req));
+    req.lun = make_lun(sdev->channel, sdev->target, sdev->lun);
+    memcpy(&req.cdb, cdb, cdb_size);
+
+    memset(&resp, 0, sizeof(resp));
+    resp.status = 0xff;     /* set invalid  */
+    resp.response = 0xff;   /*              */
+
+    if (data && data_size) {
+        memset(data, 0, data_size);
+    }
+}
+
+static inline void vs_io_assert(bool term, const char *msg)
+{
+    if (!term) {
+        virtio_scsi_verify_response(&resp, msg);
+    }
+}
+
+static void vs_run(const char *title, VirtioCmd *cmd, VDev *vdev,
+                   const void *cdb, int cdb_size,
+                   void *data, uint32_t data_size)
+{
+    prepare_request(vdev, cdb, cdb_size, data, data_size);
+    vs_io_assert(virtio_run(vdev, VR_REQUEST, cmd) == 0, title);
+}
+
+/* SCSI protocol implementation routines */
+
+static bool scsi_inquiry(VDev *vdev, void *data, uint32_t data_size)
+{
+    ScsiCdbInquiry cdb = {
+        .command = 0x12,
+        .alloc_len = data_size < 65535 ? data_size : 65535,
+    };
+    VirtioCmd inquiry[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("inquiry", inquiry, vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_test_unit_ready(VDev *vdev)
+{
+    ScsiCdbTestUnitReady cdb = {
+        .command = 0x00,
+    };
+    VirtioCmd test_unit_ready[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE },
+    };
+
+    prepare_request(vdev, &cdb, sizeof(cdb), 0, 0);
+    virtio_run(vdev, VR_REQUEST, test_unit_ready); /* ignore errors here */
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_report_luns(VDev *vdev, void *data, uint32_t data_size)
+{
+    ScsiCdbReportLuns cdb = {
+        .command = 0xa0,
+        .select_report = 0x02, /* REPORT ALL */
+        .alloc_len = data_size,
+    };
+    VirtioCmd report_luns[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("report luns", report_luns,
+           vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_read_10(VDev *vdev,
+                         ulong sector, int sectors, void *data)
+{
+    int f = vdev->blk_factor;
+    unsigned int data_size = sectors * virtio_get_block_size() * f;
+    ScsiCdbRead10 cdb = {
+        .command = 0x28,
+        .lba = sector * f,
+        .xfer_length = sectors * f,
+    };
+    VirtioCmd read_10[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size * f, VRING_DESC_F_WRITE },
+    };
+
+    debug_print_int("read_10  sector", sector);
+    debug_print_int("read_10 sectors", sectors);
+
+    vs_run("read(10)", read_10, vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_read_capacity(VDev *vdev,
+                               void *data, uint32_t data_size)
+{
+    ScsiCdbReadCapacity16 cdb = {
+        .command = 0x9e, /* SERVICE_ACTION_IN_16 */
+        .service_action = 0x10, /* SA_READ_CAPACITY */
+        .alloc_len = data_size,
+    };
+    VirtioCmd read_capacity_16[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("read capacity", read_capacity_16,
+           vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+/* virtio-scsi routines */
+
+static void virtio_scsi_locate_device(VDev *vdev)
+{
+    const uint16_t channel = 0; /* again, it's what QEMU does */
+    uint16_t target;
+    static uint8_t data[16 + 8 * 63];
+    ScsiLunReport *r = (void *) data;
+    ScsiDevice *sdev = vdev->scsi_device;
+    int i, luns;
+
+    /* QEMU has hardcoded channel #0 in many places.
+     * If this hardcoded value is ever changed, we'll need to add code for
+     * vdev->config.scsi.max_channel != 0 here.
+     */
+    debug_print_int("config.scsi.max_channel", vdev->config.scsi.max_channel);
+    debug_print_int("config.scsi.max_target ", vdev->config.scsi.max_target);
+    debug_print_int("config.scsi.max_lun    ", vdev->config.scsi.max_lun);
+
+    for (target = 0; target <= vdev->config.scsi.max_target; target++) {
+        sdev->channel = channel;
+        sdev->target = target; /* sdev->lun will be 0 here */
+        if (!scsi_report_luns(vdev, data, sizeof(data))) {
+            if (resp.response == VIRTIO_SCSI_S_BAD_TARGET) {
+                continue;
+            }
+            print_int("target", target);
+            virtio_scsi_verify_response(&resp, "SCSI cannot report LUNs");
+        }
+        if (r->lun_list_len == 0) {
+            print_int("no LUNs for target", target);
+            continue;
+        }
+        luns = r->lun_list_len / 8;
+        debug_print_int("LUNs reported", luns);
+        if (luns == 1) {
+            /* There is no ",lun=#" arg for -device or ",lun=0" given.
+             * Hence, the only LUN reported.
+             * Usually, it's 0.
+             */
+            sdev->lun = r->lun[0].v16[0]; /* it's returned this way */
+            debug_print_int("Have to use LUN", sdev->lun);
+            return; /* we have to use this device */
+        }
+        for (i = 0; i < luns; i++) {
+            if (r->lun[i].v64) {
+                /* Look for non-zero LUN - we have where to choose from */
+                sdev->lun = r->lun[i].v16[0];
+                debug_print_int("Will use LUN", sdev->lun);
+                return; /* we have found a device */
+            }
+        }
+    }
+    panic("\n! Cannot locate virtio-scsi device !\n");
+}
+
+int virtio_scsi_read_many(VDev *vdev,
+                          ulong sector, void *load_addr, int sec_num)
+{
+    if (!scsi_read_10(vdev, sector, sec_num, load_addr)) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:read_many");
+    }
+
+    return 0;
+}
+
+static bool virtio_scsi_inquiry_response_is_cdrom(void *data)
+{
+    const ScsiInquiryStd *response = data;
+    const int resp_data_fmt = response->b3 & 0x0f;
+    int i;
+
+    IPL_check(resp_data_fmt == 2, "Wrong INQUIRY response format");
+    if (resp_data_fmt != 2) {
+        return false; /* cannot decode */
+    }
+
+    if ((response->peripheral_qdt & 0x1f) == SCSI_INQ_RDT_CDROM) {
+        return true;
+    }
+
+    for (i = 0; i < sizeof(response->prod_id); i++) {
+        if (response->prod_id[i] != QEMU_CDROM_SIGNATURE[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static void scsi_parse_capacity_report(void *data,
+                                       uint64_t *last_lba, uint32_t *lb_len)
+{
+    ScsiReadCapacity16Data *p = data;
+
+    if (last_lba) {
+        *last_lba = p->ret_lba;
+    }
+
+    if (lb_len) {
+        *lb_len = p->lb_len;
+    }
+}
+
+void virtio_scsi_setup(VDev *vdev)
+{
+    int retry_test_unit_ready = 3;
+    uint8_t data[256];
+    uint32_t data_size = sizeof(data);
+
+    vdev->scsi_device = &default_scsi_device;
+    virtio_scsi_locate_device(vdev);
+
+    /* We have to "ping" the device before it becomes readable */
+    while (!scsi_test_unit_ready(vdev)) {
+
+        if (!virtio_scsi_response_ok(&resp)) {
+            uint8_t code = resp.sense[0] & SCSI_SENSE_CODE_MASK;
+            uint8_t sense_key = resp.sense[2] & SCSI_SENSE_KEY_MASK;
+
+            IPL_assert(resp.sense_len != 0, "virtio-scsi:setup: no SENSE data");
+
+            IPL_assert(retry_test_unit_ready && code == 0x70 &&
+                       sense_key == SCSI_SENSE_KEY_UNIT_ATTENTION,
+                       "virtio-scsi:setup: cannot retry");
+
+            /* retry on CHECK_CONDITION/UNIT_ATTENTION as it
+             * may not designate a real error, but it may be
+             * a result of device reset, etc.
+             */
+            retry_test_unit_ready--;
+            sleep(1);
+            continue;
+        }
+
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup");
+    }
+
+    /* read and cache SCSI INQUIRY response */
+    if (!scsi_inquiry(vdev, scsi_inquiry_std_response,
+                      sizeof(scsi_inquiry_std_response))) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup:inquiry");
+    }
+
+    if (virtio_scsi_inquiry_response_is_cdrom(scsi_inquiry_std_response)) {
+        sclp_print("SCSI CD-ROM detected.\n");
+        vdev->is_cdrom = true;
+        vdev->scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
+    }
+
+    if (!scsi_read_capacity(vdev, data, data_size)) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup:read_capacity");
+    }
+    scsi_parse_capacity_report(data, &vdev->scsi_last_block,
+                               (uint32_t *) &vdev->scsi_block_size);
+}
diff --git a/qemu/pc-bios/s390-ccw/virtio-scsi.h b/qemu/pc-bios/s390-ccw/virtio-scsi.h
new file mode 100644
index 000000000..f50b38b18
--- /dev/null
+++ b/qemu/pc-bios/s390-ccw/virtio-scsi.h
@@ -0,0 +1,72 @@
+/*
+ * Virtio-SCSI definitions for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef VIRTIO_SCSI_H
+#define VIRTIO_SCSI_H
+
+#include "s390-ccw.h"
+#include "virtio.h"
+#include "scsi.h"
+
+#define VIRTIO_SCSI_CDB_SIZE   SCSI_DEFAULT_CDB_SIZE
+#define VIRTIO_SCSI_SENSE_SIZE SCSI_DEFAULT_SENSE_SIZE
+
+/* command-specific response values */
+#define VIRTIO_SCSI_S_OK                     0x00
+#define VIRTIO_SCSI_S_BAD_TARGET             0x03
+
+#define QEMU_CDROM_SIGNATURE "QEMU CD-ROM     "
+
+enum virtio_scsi_vq_id {
+    VR_CONTROL  = 0,
+    VR_EVENT    = 1,
+    VR_REQUEST  = 2,
+};
+
+struct VirtioScsiCmdReq {
+    ScsiLun lun;
+    uint64_t id;
+    uint8_t task_attr;   /* = 0 = VIRTIO_SCSI_S_SIMPLE */
+    uint8_t prio;
+    uint8_t crn;         /* = 0 */
+    uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
+} __attribute__((packed));
+typedef struct VirtioScsiCmdReq VirtioScsiCmdReq;
+
+struct VirtioScsiCmdResp {
+        uint32_t sense_len;
+        uint32_t residual;
+        uint16_t status_qualifier;
+        uint8_t status;      /* first check for .response    */
+        uint8_t response;    /* then for .status             */
+        uint8_t sense[VIRTIO_SCSI_SENSE_SIZE];
+} __attribute__((packed));
+typedef struct VirtioScsiCmdResp VirtioScsiCmdResp;
+
+static inline const char *virtio_scsi_response_msg(const VirtioScsiCmdResp *r)
+{
+    static char err_msg[] = "VS RESP=XX";
+    uint8_t v = r->response;
+
+    fill_hex_val(err_msg + 8, &v, 1);
+    return err_msg;
+}
+
+static inline bool virtio_scsi_response_ok(const VirtioScsiCmdResp *r)
+{
+        return r->response == VIRTIO_SCSI_S_OK && r->status == CDB_STATUS_GOOD;
+}
+
+void virtio_scsi_setup(VDev *vdev);
+int virtio_scsi_read_many(VDev *vdev,
+                          ulong sector, void *load_addr, int sec_num);
+
+#endif /* VIRTIO_SCSI_H */
diff --git a/qemu/pc-bios/s390-ccw/virtio.c b/qemu/pc-bios/s390-ccw/virtio.c
index 57ff1b07e..1d34e8c1a 100644
--- a/qemu/pc-bios/s390-ccw/virtio.c
+++ b/qemu/pc-bios/s390-ccw/virtio.c
@@ -10,39 +10,68 @@
 
 #include "s390-ccw.h"
 #include "virtio.h"
+#include "virtio-scsi.h"
 
-static struct vring block;
+#define VRING_WAIT_REPLY_TIMEOUT 3
+
+static VRing block[VIRTIO_MAX_VQS];
+static char ring_area[VIRTIO_RING_SIZE * VIRTIO_MAX_VQS]
+                     __attribute__((__aligned__(PAGE_SIZE)));
 
 static char chsc_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE)));
 
+static VDev vdev = {
+    .nr_vqs = 1,
+    .vrings = block,
+    .cmd_vr_idx = 0,
+    .ring_area = ring_area,
+    .wait_reply_timeout = VRING_WAIT_REPLY_TIMEOUT,
+    .schid = { .one = 1 },
+    .scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE,
+    .blk_factor = 1,
+};
+
+VDev *virtio_get_device(void)
+{
+    return &vdev;
+}
+
+VirtioDevType virtio_get_device_type(void)
+{
+    return vdev.senseid.cu_model;
+}
+
+/* virtio spec v1.0 para 4.3.3.2 */
 static long kvm_hypercall(unsigned long nr, unsigned long param1,
-                          unsigned long param2)
+                          unsigned long param2, unsigned long param3)
 {
     register ulong r_nr asm("1") = nr;
     register ulong r_param1 asm("2") = param1;
     register ulong r_param2 asm("3") = param2;
+    register ulong r_param3 asm("4") = param3;
     register long retval asm("2");
 
     asm volatile ("diag 2,4,0x500"
                   : "=d" (retval)
-                  : "d" (r_nr), "0" (r_param1), "r"(r_param2)
+                  : "d" (r_nr), "0" (r_param1), "r"(r_param2), "d"(r_param3)
                   : "memory", "cc");
 
     return retval;
 }
 
-static void virtio_notify(struct subchannel_id schid)
+static long virtio_notify(SubChannelId schid, int vq_idx, long cookie)
 {
-    kvm_hypercall(KVM_S390_VIRTIO_CCW_NOTIFY, *(u32 *)&schid, 0);
+    return kvm_hypercall(KVM_S390_VIRTIO_CCW_NOTIFY, *(u32 *)&schid,
+                         vq_idx, cookie);
 }
 
 /***********************************************
  *             Virtio functions                *
  ***********************************************/
 
-static int drain_irqs(struct subchannel_id schid)
+static int drain_irqs(SubChannelId schid)
 {
-    struct irb irb = {};
+    Irb irb = {};
     int r = 0;
 
     while (1) {
@@ -59,17 +88,17 @@ static int drain_irqs(struct subchannel_id schid)
     }
 }
 
-static int run_ccw(struct subchannel_id schid, int cmd, void *ptr, int len)
+static int run_ccw(VDev *vdev, int cmd, void *ptr, int len)
 {
-    struct ccw1 ccw = {};
-    struct cmd_orb orb = {};
-    struct schib schib;
+    Ccw1 ccw = {};
+    CmdOrb orb = {};
+    Schib schib;
     int r;
 
     /* start command processing */
-    stsch_err(schid, &schib);
+    stsch_err(vdev->schid, &schib);
     schib.scsw.ctrl = SCSW_FCTL_START_FUNC;
-    msch(schid, &schib);
+    msch(vdev->schid, &schib);
 
     /* start subchannel command */
     orb.fmt = 1;
@@ -80,41 +109,29 @@ static int run_ccw(struct subchannel_id schid, int cmd, void *ptr, int len)
     ccw.cda = (long)ptr;
     ccw.count = len;
 
-    r = ssch(schid, &orb);
+    r = ssch(vdev->schid, &orb);
     /*
      * XXX Wait until device is done processing the CCW. For now we can
      *     assume that a simple tsch will have finished the CCW processing,
      *     but the architecture allows for asynchronous operation
      */
     if (!r) {
-        r = drain_irqs(schid);
+        r = drain_irqs(vdev->schid);
     }
     return r;
 }
 
-static void virtio_set_status(struct subchannel_id schid,
-                              unsigned long dev_addr)
+static void vring_init(VRing *vr, VqInfo *info)
 {
-    unsigned char status = dev_addr;
-    if (run_ccw(schid, CCW_CMD_WRITE_STATUS, &status, sizeof(status))) {
-        virtio_panic("Could not write status to host!\n");
-    }
-}
-
-static void virtio_reset(struct subchannel_id schid)
-{
-    run_ccw(schid, CCW_CMD_VDEV_RESET, NULL, 0);
-}
+    void *p = (void *) info->queue;
 
-static void vring_init(struct vring *vr, unsigned int num, void *p,
-                       unsigned long align)
-{
     debug_print_addr("init p", p);
-    vr->num = num;
+    vr->id = info->index;
+    vr->num = info->num;
     vr->desc = p;
-    vr->avail = p + num*sizeof(struct vring_desc);
-    vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + align-1)
-                & ~(align - 1));
+    vr->avail = p + info->num * sizeof(VRingDesc);
+    vr->used = (void *)(((unsigned long)&vr->avail->ring[info->num]
+               + info->align - 1) & ~(info->align - 1));
 
     /* Zero out all relevant field */
     vr->avail->flags = 0;
@@ -125,16 +142,18 @@ static void vring_init(struct vring *vr, unsigned int num, void *p,
     vr->used->idx = 0;
     vr->used_idx = 0;
     vr->next_idx = 0;
+    vr->cookie = 0;
 
     debug_print_addr("init vr", vr);
 }
 
-static void vring_notify(struct subchannel_id schid)
+static bool vring_notify(VRing *vr)
 {
-    virtio_notify(schid);
+    vr->cookie = virtio_notify(vr->schid, vr->id, vr->cookie);
+    return vr->cookie >= 0;
 }
 
-static void vring_send_buf(struct vring *vr, void *p, int len, int flags)
+static void vring_send_buf(VRing *vr, void *p, int len, int flags)
 {
     /* For follow-up chains we need to keep the first entry point */
     if (!(flags & VRING_HIDDEN_IS_CHAIN)) {
@@ -162,11 +181,26 @@ static u64 get_clock(void)
     return r;
 }
 
-static ulong get_second(void)
+ulong get_second(void)
 {
     return (get_clock() >> 12) / 1000000;
 }
 
+static int vr_poll(VRing *vr)
+{
+    if (vr->used->idx == vr->used_idx) {
+        vring_notify(vr);
+        yield();
+        return 0;
+    }
+
+    vr->used_idx = vr->used->idx;
+    vr->next_idx = 0;
+    vr->desc[0].len = 0;
+    vr->desc[0].flags = 0;
+    return 1; /* vr has been updated */
+}
+
 /*
  * Wait for the host to reply.
  *
@@ -174,67 +208,92 @@ static ulong get_second(void)
  *
  * Returns 0 on success, 1 on timeout.
  */
-static int vring_wait_reply(struct vring *vr, int timeout)
+static int vring_wait_reply(void)
 {
-    ulong target_second = get_second() + timeout;
-    struct subchannel_id schid = vr->schid;
-    int r = 0;
+    ulong target_second = get_second() + vdev.wait_reply_timeout;
+
+    /* Wait for any queue to be updated by the host */
+    do {
+        int i, r = 0;
 
-    /* Wait until the used index has moved. */
-    while (vr->used->idx == vr->used_idx) {
-        vring_notify(schid);
-        if (timeout && (get_second() >= target_second)) {
-            r = 1;
-            break;
+        for (i = 0; i < vdev.nr_vqs; i++) {
+            r += vr_poll(&vdev.vrings[i]);
         }
         yield();
-    }
+        if (r) {
+            return 0;
+        }
+    } while (!vdev.wait_reply_timeout || (get_second() < target_second));
 
-    vr->used_idx = vr->used->idx;
-    vr->next_idx = 0;
-    vr->desc[0].len = 0;
-    vr->desc[0].flags = 0;
+    return 1;
+}
 
-    return r;
+int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd)
+{
+    VRing *vr = &vdev->vrings[vqid];
+    int i = 0;
+
+    do {
+        vring_send_buf(vr, cmd[i].data, cmd[i].size,
+                       cmd[i].flags | (i ? VRING_HIDDEN_IS_CHAIN : 0));
+    } while (cmd[i++].flags & VRING_DESC_F_NEXT);
+
+    vring_wait_reply();
+    if (drain_irqs(vr->schid)) {
+        return -1;
+    }
+    return 0;
 }
 
 /***********************************************
  *               Virtio block                  *
  ***********************************************/
 
-int virtio_read_many(ulong sector, void *load_addr, int sec_num)
+static int virtio_blk_read_many(VDev *vdev,
+                                ulong sector, void *load_addr, int sec_num)
 {
-    struct virtio_blk_outhdr out_hdr;
+    VirtioBlkOuthdr out_hdr;
     u8 status;
-    int r;
+    VRing *vr = &vdev->vrings[vdev->cmd_vr_idx];
 
     /* Tell the host we want to read */
     out_hdr.type = VIRTIO_BLK_T_IN;
     out_hdr.ioprio = 99;
     out_hdr.sector = virtio_sector_adjust(sector);
 
-    vring_send_buf(&block, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
+    vring_send_buf(vr, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
 
     /* This is where we want to receive data */
-    vring_send_buf(&block, load_addr, virtio_get_block_size() * sec_num,
+    vring_send_buf(vr, load_addr, virtio_get_block_size() * sec_num,
                    VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN |
                    VRING_DESC_F_NEXT);
 
     /* status field */
-    vring_send_buf(&block, &status, sizeof(u8), VRING_DESC_F_WRITE |
-                   VRING_HIDDEN_IS_CHAIN);
+    vring_send_buf(vr, &status, sizeof(u8),
+                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN);
 
     /* Now we can tell the host to read */
-    vring_wait_reply(&block, 0);
+    vring_wait_reply();
 
-    r = drain_irqs(block.schid);
-    if (r) {
+    if (drain_irqs(vr->schid)) {
         /* Well, whatever status is supposed to contain... */
         status = 1;
     }
     return status;
 }
 
+int virtio_read_many(ulong sector, void *load_addr, int sec_num)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return virtio_blk_read_many(&vdev, sector, load_addr, sec_num);
+    case VIRTIO_ID_SCSI:
+        return virtio_scsi_read_many(&vdev, sector, load_addr, sec_num);
+    }
+    panic("\n! No readable IPL device !\n");
+    return -1;
+}
+
 unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
                                  ulong subchan_id, void *load_addr)
 {
@@ -251,7 +310,7 @@ unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
     sclp_print(".");
     status = virtio_read_many(sec, (void *)addr, sec_num);
     if (status) {
-        virtio_panic("I/O Error");
+        panic("I/O Error");
     }
     addr += sec_num * virtio_get_block_size();
 
@@ -263,42 +322,6 @@ int virtio_read(ulong sector, void *load_addr)
     return virtio_read_many(sector, load_addr, 1);
 }
 
-static VirtioBlkConfig blk_cfg = {};
-static bool guessed_disk_nature;
-
-bool virtio_guessed_disk_nature(void)
-{
-    return guessed_disk_nature;
-}
-
-void virtio_assume_scsi(void)
-{
-    guessed_disk_nature = true;
-    blk_cfg.blk_size = 512;
-    blk_cfg.physical_block_exp = 0;
-}
-
-void virtio_assume_eckd(void)
-{
-    guessed_disk_nature = true;
-    blk_cfg.blk_size = 4096;
-    blk_cfg.physical_block_exp = 0;
-
-    /* this must be here to calculate code segment position */
-    blk_cfg.geometry.heads = 15;
-    blk_cfg.geometry.sectors = 12;
-}
-
-bool virtio_disk_is_scsi(void)
-{
-    if (guessed_disk_nature) {
-        return (virtio_get_block_size()  == 512);
-    }
-    return (blk_cfg.geometry.heads == 255)
-        && (blk_cfg.geometry.sectors == 63)
-        && (virtio_get_block_size()  == 512);
-}
-
 /*
  * Other supported value pairs, if any, would need to be added here.
  * Note: head count is always 15.
@@ -318,16 +341,91 @@ static inline u8 virtio_eckd_sectors_for_block_size(int size)
     return 0;
 }
 
+VirtioGDN virtio_guessed_disk_nature(void)
+{
+    return vdev.guessed_disk_nature;
+}
+
+void virtio_assume_scsi(void)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev.config.blk.blk_size = VIRTIO_SCSI_BLOCK_SIZE;
+        vdev.config.blk.physical_block_exp = 0;
+        vdev.blk_factor = 1;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_iso9660(void)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev.config.blk.blk_size = VIRTIO_ISO_BLOCK_SIZE;
+        vdev.config.blk.physical_block_exp = 0;
+        vdev.blk_factor = VIRTIO_ISO_BLOCK_SIZE / VIRTIO_SECTOR_SIZE;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_eckd(void)
+{
+    vdev.guessed_disk_nature = VIRTIO_GDN_DASD;
+    vdev.blk_factor = 1;
+    vdev.config.blk.physical_block_exp = 0;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.config.blk.blk_size = 4096;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.config.blk.blk_size = vdev.scsi_block_size;
+        break;
+    }
+    vdev.config.blk.geometry.heads = 15;
+    vdev.config.blk.geometry.sectors =
+        virtio_eckd_sectors_for_block_size(vdev.config.blk.blk_size);
+}
+
+bool virtio_disk_is_scsi(void)
+{
+    if (vdev.guessed_disk_nature == VIRTIO_GDN_SCSI) {
+        return true;
+    }
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev.config.blk.geometry.heads == 255)
+            && (vdev.config.blk.geometry.sectors == 63)
+            && (virtio_get_block_size()  == VIRTIO_SCSI_BLOCK_SIZE);
+    case VIRTIO_ID_SCSI:
+        return true;
+    }
+    return false;
+}
+
 bool virtio_disk_is_eckd(void)
 {
     const int block_size = virtio_get_block_size();
 
-    if (guessed_disk_nature) {
-        return (block_size  == 4096);
+    if (vdev.guessed_disk_nature == VIRTIO_GDN_DASD) {
+        return true;
     }
-    return (blk_cfg.geometry.heads == 15)
-        && (blk_cfg.geometry.sectors ==
-            virtio_eckd_sectors_for_block_size(block_size));
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev.config.blk.geometry.heads == 15)
+            && (vdev.config.blk.geometry.sectors ==
+                virtio_eckd_sectors_for_block_size(block_size));
+    case VIRTIO_ID_SCSI:
+        return false;
+    }
+    return false;
 }
 
 bool virtio_ipl_disk_is_valid(void)
@@ -337,34 +435,80 @@ bool virtio_ipl_disk_is_valid(void)
 
 int virtio_get_block_size(void)
 {
-    return blk_cfg.blk_size << blk_cfg.physical_block_exp;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.blk_size << vdev.config.blk.physical_block_exp;
+    case VIRTIO_ID_SCSI:
+        return vdev.scsi_block_size;
+    }
+    return 0;
 }
 
 uint8_t virtio_get_heads(void)
 {
-    return blk_cfg.geometry.heads;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.geometry.heads;
+    case VIRTIO_ID_SCSI:
+        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev.config.blk.geometry.heads : 255;
+    }
+    return 0;
 }
 
 uint8_t virtio_get_sectors(void)
 {
-    return blk_cfg.geometry.sectors;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.geometry.sectors;
+    case VIRTIO_ID_SCSI:
+        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev.config.blk.geometry.sectors : 63;
+    }
+    return 0;
 }
 
 uint64_t virtio_get_blocks(void)
 {
-    return blk_cfg.capacity /
-           (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
+    const uint64_t factor = virtio_get_block_size() / VIRTIO_SECTOR_SIZE;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.capacity / factor;
+    case VIRTIO_ID_SCSI:
+        return vdev.scsi_last_block / factor;
+    }
+    return 0;
 }
 
-void virtio_setup_block(struct subchannel_id schid)
+static void virtio_setup_ccw(VDev *vdev)
 {
-    struct vq_info_block info;
-    struct vq_config_block config = {};
-
-    blk_cfg.blk_size = 0; /* mark "illegal" - setup started... */
-    guessed_disk_nature = false;
-
-    virtio_reset(schid);
+    int i, cfg_size = 0;
+    unsigned char status = VIRTIO_CONFIG_S_DRIVER_OK;
+
+    IPL_assert(virtio_is_supported(vdev->schid), "PE");
+    /* device ID has been established now */
+
+    vdev->config.blk.blk_size = 0; /* mark "illegal" - setup started... */
+    vdev->guessed_disk_nature = VIRTIO_GDN_NONE;
+
+    run_ccw(vdev, CCW_CMD_VDEV_RESET, NULL, 0);
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev->nr_vqs = 1;
+        vdev->cmd_vr_idx = 0;
+        cfg_size = sizeof(vdev->config.blk);
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev->nr_vqs = 3;
+        vdev->cmd_vr_idx = VR_REQUEST;
+        cfg_size = sizeof(vdev->config.scsi);
+        break;
+    default:
+        panic("Unsupported virtio device\n");
+    }
+    IPL_assert(run_ccw(vdev, CCW_CMD_READ_CONF, &vdev->config, cfg_size) == 0,
+               "Could not get block device configuration");
 
     /*
      * Skipping CCW_CMD_READ_FEAT. We're not doing anything fancy, and
@@ -372,54 +516,84 @@ void virtio_setup_block(struct subchannel_id schid)
      * expect it.
      */
 
-    config.index = 0;
-    if (run_ccw(schid, CCW_CMD_READ_VQ_CONF, &config, sizeof(config))) {
-        virtio_panic("Could not get block device VQ configuration\n");
-    }
-    if (run_ccw(schid, CCW_CMD_READ_CONF, &blk_cfg, sizeof(blk_cfg))) {
-        virtio_panic("Could not get block device configuration\n");
-    }
-    vring_init(&block, config.num, ring_area,
-               KVM_S390_VIRTIO_RING_ALIGN);
-
-    info.queue = (unsigned long long) ring_area;
-    info.align = KVM_S390_VIRTIO_RING_ALIGN;
-    info.index = 0;
-    info.num = config.num;
-    block.schid = schid;
-
-    if (!run_ccw(schid, CCW_CMD_SET_VQ, &info, sizeof(info))) {
-        virtio_set_status(schid, VIRTIO_CONFIG_S_DRIVER_OK);
+    for (i = 0; i < vdev->nr_vqs; i++) {
+        VqInfo info = {
+            .queue = (unsigned long long) ring_area + (i * VIRTIO_RING_SIZE),
+            .align = KVM_S390_VIRTIO_RING_ALIGN,
+            .index = i,
+            .num = 0,
+        };
+        VqConfig config = {
+            .index = i,
+            .num = 0,
+        };
+
+        IPL_assert(
+            run_ccw(vdev, CCW_CMD_READ_VQ_CONF, &config, sizeof(config)) == 0,
+            "Could not get block device VQ configuration");
+        info.num = config.num;
+        vring_init(&vdev->vrings[i], &info);
+        vdev->vrings[i].schid = vdev->schid;
+        IPL_assert(run_ccw(vdev, CCW_CMD_SET_VQ, &info, sizeof(info)) == 0,
+                   "Cannot set VQ info");
     }
+    IPL_assert(
+        run_ccw(vdev, CCW_CMD_WRITE_STATUS, &status, sizeof(status)) == 0,
+        "Could not write status to host");
+}
 
-    if (!virtio_ipl_disk_is_valid()) {
-        /* make sure all getters but blocksize return 0 for invalid IPL disk */
-        memset(&blk_cfg, 0, sizeof(blk_cfg));
-        virtio_assume_scsi();
+void virtio_setup_device(SubChannelId schid)
+{
+    vdev.schid = schid;
+    virtio_setup_ccw(&vdev);
+
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        sclp_print("Using virtio-blk.\n");
+        if (!virtio_ipl_disk_is_valid()) {
+            /* make sure all getters but blocksize return 0 for
+             * invalid IPL disk
+             */
+            memset(&vdev.config.blk, 0, sizeof(vdev.config.blk));
+            virtio_assume_scsi();
+        }
+        break;
+    case VIRTIO_ID_SCSI:
+        IPL_assert(vdev.config.scsi.sense_size == VIRTIO_SCSI_SENSE_SIZE,
+            "Config: sense size mismatch");
+        IPL_assert(vdev.config.scsi.cdb_size == VIRTIO_SCSI_CDB_SIZE,
+            "Config: CDB size mismatch");
+
+        sclp_print("Using virtio-scsi.\n");
+        virtio_scsi_setup(&vdev);
+        break;
+    default:
+        panic("\n! No IPL device available !\n");
     }
 }
 
-bool virtio_is_blk(struct subchannel_id schid)
+bool virtio_is_supported(SubChannelId schid)
 {
-    int r;
-    struct senseid senseid = {};
-
+    vdev.schid = schid;
+    memset(&vdev.senseid, 0, sizeof(vdev.senseid));
     /* run sense id command */
-    r = run_ccw(schid, CCW_CMD_SENSE_ID, &senseid, sizeof(senseid));
-    if (r) {
+    if (run_ccw(&vdev, CCW_CMD_SENSE_ID, &vdev.senseid, sizeof(vdev.senseid))) {
         return false;
     }
-    if ((senseid.cu_type != 0x3832) || (senseid.cu_model != VIRTIO_ID_BLOCK)) {
-        return false;
+    if (vdev.senseid.cu_type == 0x3832) {
+        switch (vdev.senseid.cu_model) {
+        case VIRTIO_ID_BLOCK:
+        case VIRTIO_ID_SCSI:
+            return true;
+        }
     }
-
-    return true;
+    return false;
 }
 
 int enable_mss_facility(void)
 {
     int ret;
-    struct chsc_area_sda *sda_area = (struct chsc_area_sda *) chsc_page;
+    ChscAreaSda *sda_area = (ChscAreaSda *) chsc_page;
 
     memset(sda_area, 0, PAGE_SIZE);
     sda_area->request.length = 0x0400;
diff --git a/qemu/pc-bios/s390-ccw/virtio.h b/qemu/pc-bios/s390-ccw/virtio.h
index c23466b8d..3c6e91510 100644
--- a/qemu/pc-bios/s390-ccw/virtio.h
+++ b/qemu/pc-bios/s390-ccw/virtio.h
@@ -23,49 +23,58 @@
 /* We've given up on this device. */
 #define VIRTIO_CONFIG_S_FAILED          0x80
 
-enum virtio_dev_type {
+enum VirtioDevType {
     VIRTIO_ID_NET = 1,
     VIRTIO_ID_BLOCK = 2,
     VIRTIO_ID_CONSOLE = 3,
     VIRTIO_ID_BALLOON = 5,
+    VIRTIO_ID_SCSI = 8,
 };
-
-struct virtio_dev_header {
-    enum virtio_dev_type type : 8;
-    u8  num_vq;
-    u8  feature_len;
-    u8  config_len;
-    u8  status;
-    u8  vqconfig[];
+typedef enum VirtioDevType VirtioDevType;
+
+struct VirtioDevHeader {
+    VirtioDevType type:8;
+    uint8_t num_vq;
+    uint8_t feature_len;
+    uint8_t config_len;
+    uint8_t status;
+    uint8_t vqconfig[];
 } __attribute__((packed));
+typedef struct VirtioDevHeader VirtioDevHeader;
 
-struct virtio_vqconfig {
-    u64 token;
-    u64 address;
-    u16 num;
-    u8  pad[6];
+struct VirtioVqConfig {
+    uint64_t token;
+    uint64_t address;
+    uint16_t num;
+    uint8_t pad[6];
 } __attribute__((packed));
+typedef struct VirtioVqConfig VirtioVqConfig;
 
-struct vq_info_block {
-    u64 queue;
-    u32 align;
-    u16 index;
-    u16 num;
+struct VqInfo {
+    uint64_t queue;
+    uint32_t align;
+    uint16_t index;
+    uint16_t num;
 } __attribute__((packed));
+typedef struct VqInfo VqInfo;
 
-struct vq_config_block {
-    u16 index;
-    u16 num;
+struct VqConfig {
+    uint16_t index;
+    uint16_t num;
 } __attribute__((packed));
+typedef struct VqConfig VqConfig;
 
-struct virtio_dev {
-    struct virtio_dev_header *header;
-    struct virtio_vqconfig *vqconfig;
+struct VirtioDev {
+    VirtioDevHeader *header;
+    VirtioVqConfig *vqconfig;
     char *host_features;
     char *guest_features;
     char *config;
 };
+typedef struct VirtioDev VirtioDev;
 
+#define VIRTIO_RING_SIZE            (PAGE_SIZE * 8)
+#define VIRTIO_MAX_VQS              3
 #define KVM_S390_VIRTIO_RING_ALIGN  4096
 
 #define VRING_USED_F_NO_NOTIFY  1
@@ -81,46 +90,53 @@ struct virtio_dev {
 #define VRING_HIDDEN_IS_CHAIN   256
 
 /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
-struct vring_desc {
+struct VRingDesc {
     /* Address (guest-physical). */
-    u64 addr;
+    uint64_t addr;
     /* Length. */
-    u32 len;
+    uint32_t len;
     /* The flags as indicated above. */
-    u16 flags;
+    uint16_t flags;
     /* We chain unused descriptors via this, too */
-    u16 next;
+    uint16_t next;
 } __attribute__((packed));
+typedef struct VRingDesc VRingDesc;
 
-struct vring_avail {
-    u16 flags;
-    u16 idx;
-    u16 ring[];
+struct VRingAvail {
+    uint16_t flags;
+    uint16_t idx;
+    uint16_t ring[];
 } __attribute__((packed));
+typedef struct VRingAvail VRingAvail;
 
-/* u32 is used here for ids for padding reasons. */
-struct vring_used_elem {
+/* uint32_t is used here for ids for padding reasons. */
+struct VRingUsedElem {
     /* Index of start of used descriptor chain. */
-    u32 id;
+    uint32_t id;
     /* Total length of the descriptor chain which was used (written to) */
-    u32 len;
+    uint32_t len;
 } __attribute__((packed));
+typedef struct VRingUsedElem VRingUsedElem;
 
-struct vring_used {
-    u16 flags;
-    u16 idx;
-    struct vring_used_elem ring[];
+struct VRingUsed {
+    uint16_t flags;
+    uint16_t idx;
+    VRingUsedElem ring[];
 } __attribute__((packed));
+typedef struct VRingUsed VRingUsed;
 
-struct vring {
+struct VRing {
     unsigned int num;
     int next_idx;
     int used_idx;
-    struct vring_desc *desc;
-    struct vring_avail *avail;
-    struct vring_used *used;
-    struct subchannel_id schid;
+    VRingDesc *desc;
+    VRingAvail *avail;
+    VRingUsed *used;
+    SubChannelId schid;
+    long cookie;
+    int id;
 };
+typedef struct VRing VRing;
 
 
 /***********************************************
@@ -152,41 +168,52 @@ struct vring {
 #define VIRTIO_BLK_T_BARRIER    0x80000000
 
 /* This is the first element of the read scatter-gather list. */
-struct virtio_blk_outhdr {
+struct VirtioBlkOuthdr {
         /* VIRTIO_BLK_T* */
-        u32 type;
+        uint32_t type;
         /* io priority. */
-        u32 ioprio;
+        uint32_t ioprio;
         /* Sector (ie. 512 byte offset) */
-        u64 sector;
+        uint64_t sector;
 };
+typedef struct VirtioBlkOuthdr VirtioBlkOuthdr;
 
-typedef struct VirtioBlkConfig {
-    u64 capacity; /* in 512-byte sectors */
-    u32 size_max; /* max segment size (if VIRTIO_BLK_F_SIZE_MAX) */
-    u32 seg_max;  /* max number of segments (if VIRTIO_BLK_F_SEG_MAX) */
+struct VirtioBlkConfig {
+    uint64_t capacity; /* in 512-byte sectors */
+    uint32_t size_max; /* max segment size (if VIRTIO_BLK_F_SIZE_MAX) */
+    uint32_t seg_max;  /* max number of segments (if VIRTIO_BLK_F_SEG_MAX) */
 
-    struct virtio_blk_geometry {
-        u16 cylinders;
-        u8 heads;
-        u8 sectors;
+    struct VirtioBlkGeometry {
+        uint16_t cylinders;
+        uint8_t heads;
+        uint8_t sectors;
     } geometry; /* (if VIRTIO_BLK_F_GEOMETRY) */
 
-    u32 blk_size; /* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
+    uint32_t blk_size; /* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
 
     /* the next 4 entries are guarded by VIRTIO_BLK_F_TOPOLOGY  */
-    u8 physical_block_exp; /* exponent for physical block per logical block */
-    u8 alignment_offset;   /* alignment offset in logical blocks */
-    u16 min_io_size;       /* min I/O size without performance penalty
+    uint8_t physical_block_exp; /* exponent for physical blk per logical blk */
+    uint8_t alignment_offset;   /* alignment offset in logical blocks */
+    uint16_t min_io_size;       /* min I/O size without performance penalty
                               in logical blocks */
-    u32 opt_io_size;       /* optimal sustained I/O size in logical blocks */
+    uint32_t opt_io_size;       /* optimal sustained I/O size in logical blks */
+
+    uint8_t wce; /* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
+} __attribute__((packed));
+typedef struct VirtioBlkConfig VirtioBlkConfig;
 
-    u8 wce; /* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
-} __attribute__((packed)) VirtioBlkConfig;
+enum guessed_disk_nature_type {
+    VIRTIO_GDN_NONE     = 0,
+    VIRTIO_GDN_DASD     = 1,
+    VIRTIO_GDN_CDROM    = 2,
+    VIRTIO_GDN_SCSI     = 3,
+};
+typedef enum guessed_disk_nature_type VirtioGDN;
 
-bool virtio_guessed_disk_nature(void);
+VirtioGDN virtio_guessed_disk_nature(void);
 void virtio_assume_scsi(void);
 void virtio_assume_eckd(void);
+void virtio_assume_iso9660(void);
 
 extern bool virtio_disk_is_scsi(void);
 extern bool virtio_disk_is_eckd(void);
@@ -198,15 +225,68 @@ extern uint64_t virtio_get_blocks(void);
 extern int virtio_read_many(ulong sector, void *load_addr, int sec_num);
 
 #define VIRTIO_SECTOR_SIZE 512
-
-static inline ulong virtio_eckd_sector_adjust(ulong sector)
-{
-     return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
-}
+#define VIRTIO_ISO_BLOCK_SIZE 2048
+#define VIRTIO_SCSI_BLOCK_SIZE 512
 
 static inline ulong virtio_sector_adjust(ulong sector)
 {
-    return virtio_disk_is_eckd() ? virtio_eckd_sector_adjust(sector) : sector;
+    return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
 }
 
+struct VirtioScsiConfig {
+    uint32_t num_queues;
+    uint32_t seg_max;
+    uint32_t max_sectors;
+    uint32_t cmd_per_lun;
+    uint32_t event_info_size;
+    uint32_t sense_size;
+    uint32_t cdb_size;
+    uint16_t max_channel;
+    uint16_t max_target;
+    uint32_t max_lun;
+} __attribute__((packed));
+typedef struct VirtioScsiConfig VirtioScsiConfig;
+
+struct ScsiDevice {
+    uint16_t channel;   /* Always 0 in QEMU     */
+    uint16_t target;    /* will be scanned over */
+    uint32_t lun;       /* will be reported     */
+};
+typedef struct ScsiDevice ScsiDevice;
+
+struct VDev {
+    int nr_vqs;
+    VRing *vrings;
+    int cmd_vr_idx;
+    void *ring_area;
+    long wait_reply_timeout;
+    VirtioGDN guessed_disk_nature;
+    SubChannelId schid;
+    SenseId senseid;
+    union {
+        VirtioBlkConfig blk;
+        VirtioScsiConfig scsi;
+    } config;
+    ScsiDevice *scsi_device;
+    bool is_cdrom;
+    int scsi_block_size;
+    int blk_factor;
+    uint64_t scsi_last_block;
+    uint32_t scsi_dev_cyls;
+    uint8_t scsi_dev_heads;
+};
+typedef struct VDev VDev;
+
+VDev *virtio_get_device(void);
+VirtioDevType virtio_get_device_type(void);
+
+struct VirtioCmd {
+    void *data;
+    int size;
+    int flags;
+};
+typedef struct VirtioCmd VirtioCmd;
+
+int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd);
+
 #endif /* VIRTIO_H */
diff --git a/qemu/pc-bios/s390-zipl.rom b/qemu/pc-bios/s390-zipl.rom
deleted file mode 100644
index 3115128ef..000000000
--- a/qemu/pc-bios/s390-zipl.rom
+++ /dev/null
diff --git a/qemu/pc-bios/slof.bin b/qemu/pc-bios/slof.bin
index 0398ac67b..f2384939e 100644
--- a/qemu/pc-bios/slof.bin
+++ b/qemu/pc-bios/slof.bin
diff --git a/qemu/pc-bios/u-boot.e500 b/qemu/pc-bios/u-boot.e500
index bdb2a5e6f..6e547de6f 100755
--- a/qemu/pc-bios/u-boot.e500
+++ b/qemu/pc-bios/u-boot.e500
diff --git a/qemu/pc-bios/vgabios-cirrus.bin b/qemu/pc-bios/vgabios-cirrus.bin
index dde850290..3f4bb30fe 100644
--- a/qemu/pc-bios/vgabios-cirrus.bin
+++ b/qemu/pc-bios/vgabios-cirrus.bin
diff --git a/qemu/pc-bios/vgabios-qxl.bin b/qemu/pc-bios/vgabios-qxl.bin
index 5c43bd222..38d31b6b5 100644
--- a/qemu/pc-bios/vgabios-qxl.bin
+++ b/qemu/pc-bios/vgabios-qxl.bin
diff --git a/qemu/pc-bios/vgabios-stdvga.bin b/qemu/pc-bios/vgabios-stdvga.bin
index b2dd8f91e..e469c107a 100644
--- a/qemu/pc-bios/vgabios-stdvga.bin
+++ b/qemu/pc-bios/vgabios-stdvga.bin
diff --git a/qemu/pc-bios/vgabios-virtio.bin b/qemu/pc-bios/vgabios-virtio.bin
index 03ac8a7d4..d42b02873 100644
--- a/qemu/pc-bios/vgabios-virtio.bin
+++ b/qemu/pc-bios/vgabios-virtio.bin
diff --git a/qemu/pc-bios/vgabios-vmware.bin b/qemu/pc-bios/vgabios-vmware.bin
index 15e21c2a2..26bc0b704 100644
--- a/qemu/pc-bios/vgabios-vmware.bin
+++ b/qemu/pc-bios/vgabios-vmware.bin
diff --git a/qemu/pc-bios/vgabios.bin b/qemu/pc-bios/vgabios.bin
index 84f1561d9..2d1a7c6c9 100644
--- a/qemu/pc-bios/vgabios.bin
+++ b/qemu/pc-bios/vgabios.bin
diff --git a/qemu/po/de_DE.po b/qemu/po/de_DE.po
index dcdcf229d..97ae65516 100644
--- a/qemu/po/de_DE.po
+++ b/qemu/po/de_DE.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-17 20:39+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2012-02-28 16:00+0100\n"
 "Last-Translator: Kevin Wolf <kwolf@redhat.com>\n"
 "Language-Team: Deutsch <de@li.org>\n"
@@ -16,70 +16,70 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n!=1);\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Strg+Alt+G drücken, um Eingabegeräte freizugeben"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Angehalten]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Angehalten"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Herunterfahren"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Beenden"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "_Vollbild"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "_Heranzoomen"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Wegzoomen"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "_Einpassen"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Auf _Fenstergröße skalieren"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Tastatur _automatisch einfangen"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Eingabegeräte einfangen"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Reiter anzeigen"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "Reiter abtrennen"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Maschine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Ansicht"
diff --git a/qemu/po/fr_FR.po b/qemu/po/fr_FR.po
index bbb5ef87b..e9fb0402d 100644
--- a/qemu/po/fr_FR.po
+++ b/qemu/po/fr_FR.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-28 23:14+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-28 23:25+0200\n"
 "Last-Translator: Aurelien Jarno <aurelien@aurel32.net>\n"
 "Language-Team: French <FR@li.org>\n"
@@ -17,70 +17,70 @@ msgstr ""
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr "- Appuyer sur Ctrl+Alt+G pour arrêter la capture"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [En pause]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Pause"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Réinitialiser"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Éteindre"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Quitter"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "Mode _plein écran"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "Zoom _avant"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Zoom arrière"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "Zoom _idéal"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Zoomer pour a_juster"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Capturer en _survolant"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Capturer les entrées"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Montrer les _onglets"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "_Détacher l'onglet"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Machine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Vue"
diff --git a/qemu/po/hu.po b/qemu/po/hu.po
index 401ed211a..1149adf26 100644
--- a/qemu/po/hu.po
+++ b/qemu/po/hu.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2013-05-06 20:42+0200\n"
 "Last-Translator: Ákos Kovács <akoskovacs@gmx.com>\n"
 "Language-Team: Hungarian <hu@li.org>\n"
@@ -15,47 +15,73 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Nyomj Ctrl+Alt+G-t a bemeneti eszközök elengedéséhez"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Megállítva]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Megállítás"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "Új_raindítás"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Leállítás"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1961
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Ablakmérethez _igazítás"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Automatikus _elfogás"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Bemeneti eszközök megragadása"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "_Fülek megjelenítése"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Gép"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Nézet"
 
diff --git a/qemu/po/it.po b/qemu/po/it.po
index e46fb3a8b..a2c8e558c 100644
--- a/qemu/po/it.po
+++ b/qemu/po/it.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-29 08:14+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-29 08:25+0200\n"
 "Last-Translator: Paolo Bonzini <pbonzini@redhat.com>\n"
 "Language-Team: Italian <it@li.org>\n"
@@ -16,66 +16,70 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Premere Ctrl+Alt+G per rilasciare l'input"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Pausa]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Pausa"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Spegni"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Esci"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "_Aumenta zoom"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Riduci zoom"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "A_nnulla zoom"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Adatta alla _finestra"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Cattura _automatica input"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Cattura input"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Mostra _tab"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "_Sposta in una nuova finestra"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Macchina virtuale"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Visualizza"
diff --git a/qemu/po/messages.po b/qemu/po/messages.po
index 26c76bce8..e8f9ccd38 100644
--- a/qemu/po/messages.po
+++ b/qemu/po/messages.po
@@ -5,9 +5,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: QEMU 1.5.50\n"
+"Project-Id-Version: QEMU 2.5.92\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -16,46 +16,70 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr ""
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr ""
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr ""
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr ""
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr ""
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+msgid "Zoom _In"
+msgstr ""
+
+#: ui/gtk.c:1961
+msgid "Zoom _Out"
+msgstr ""
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr ""
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr ""
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr ""
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr ""
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr ""
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr ""
diff --git a/qemu/po/tr.po b/qemu/po/tr.po
index d712ced9c..ec9912779 100644
--- a/qemu/po/tr.po
+++ b/qemu/po/tr.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2013-04-22 18:35+0300\n"
 "Last-Translator: Ozan Çağlayan <ozancag@gmail.com>\n"
 "Language-Team: Türkçe <>\n"
@@ -17,46 +17,72 @@ msgstr ""
 "Plural-Forms: nplurals=1; plural=0;\n"
 "X-Generator: Gtranslator 2.91.6\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Yakalamayı durdurmak için Ctrl+Alt+G tuşlarına basın"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Duraklatıldı]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Duraklat"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Sıfırla"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Kapat"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1961
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Yakınlaş ve Sığ_dır"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Ü_zerindeyken Yakala"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "Girdiyi _Yakala"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Se_kmeleri Göster"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Makine"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Görüntüle"
diff --git a/qemu/po/zh_CN.po b/qemu/po/zh_CN.po
index 2b1d42e97..b55b6f375 100644
--- a/qemu/po/zh_CN.po
+++ b/qemu/po/zh_CN.po
@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: QEMU 2.2\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-31 10:03+0800\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-31 10:00+0800\n"
 "Last-Translator: Fam Zheng <famz@redhat.com>\n"
 "Language-Team: Chinese <zh@li.org>\n"
@@ -17,70 +17,70 @@ msgstr ""
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - 按下 Ctrl+Alt+G 取消捕获"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [已暂停]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "暂停(_P)"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "重置(_R)"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "关闭电源(_D)"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "退出(_Q)"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "全屏(_F)"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "放大(_I)"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "缩小(_O)"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "最合适大小(_F)"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "缩放以适应大小(_F)"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "鼠标经过时捕获(_H)"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "捕获输入(_G)"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "显示标签页(_T)"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "分离标签页"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "虚拟机(_M)"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "视图(_V)"
diff --git a/qemu/qapi-schema.json b/qemu/qapi-schema.json
index 4342a08d3..54634c473 100644
--- a/qemu/qapi-schema.json
+++ b/qemu/qapi-schema.json
@@ -5,6 +5,9 @@
 # QAPI common definitions
 { 'include': 'qapi/common.json' }
 
+# QAPI crypto definitions
+{ 'include': 'qapi/crypto.json' }
+
 # QAPI block definitions
 { 'include': 'qapi/block.json' }
 
@@ -14,6 +17,9 @@
 # Tracing commands
 { 'include': 'qapi/trace.json' }
 
+# QAPI introspection
+{ 'include': 'qapi/introspect.json' }
+
 ##
 # @LostTickPolicy:
 #
@@ -289,8 +295,6 @@
 # @format: #optional data encoding (default 'utf8').
 #          - base64: data must be base64 encoded text.  Its binary
 #            decoding gets written.
-#            Bug: invalid base64 is currently not rejected.
-#            Whitespace *is* invalid.
 #          - utf8: data's UTF-8 encoding is written
 #          - data itself is always Unicode regardless of format, like
 #            any other string.
@@ -424,6 +428,8 @@
 #
 # @active: in the process of doing migration.
 #
+# @postcopy-active: like active, but now in postcopy mode. (since 2.5)
+#
 # @completed: migration is finished.
 #
 # @failed: some error occurred during migration process.
@@ -433,7 +439,7 @@
 ##
 { 'enum': 'MigrationStatus',
   'data': [ 'none', 'setup', 'cancelling', 'cancelled',
-            'active', 'completed', 'failed' ] }
+            'active', 'postcopy-active', 'completed', 'failed' ] }
 
 ##
 # @MigrationInfo
@@ -474,6 +480,10 @@
 #        may be expensive, but do not actually occur during the iterative
 #        migration rounds themselves. (since 1.6)
 #
+# @x-cpu-throttle-percentage: #optional percentage of time guest cpus are being
+#       throttled during auto-converge. This is only present when auto-converge
+#       has started throttling guest cpus. (Since 2.5)
+#
 # Since: 0.14.0
 ##
 { 'struct': 'MigrationInfo',
@@ -483,7 +493,8 @@
            '*total-time': 'int',
            '*expected-downtime': 'int',
            '*downtime': 'int',
-           '*setup-time': 'int'} }
+           '*setup-time': 'int',
+           '*x-cpu-throttle-percentage': 'int'} }
 
 ##
 # @query-migrate
@@ -529,11 +540,15 @@
 # @auto-converge: If enabled, QEMU will automatically throttle down the guest
 #          to speed up convergence of RAM migration. (since 1.6)
 #
+# @postcopy-ram: Start executing on the migration target before all of RAM has
+#          been migrated, pulling the remaining pages along as needed. NOTE: If
+#          the migration fails during postcopy the VM will fail.  (since 2.6)
+#
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
   'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
-           'compress', 'events'] }
+           'compress', 'events', 'postcopy-ram'] }
 
 ##
 # @MigrationCapabilityStatus
@@ -590,10 +605,18 @@
 #          compression, so set the decompress-threads to the number about 1/4
 #          of compress-threads is adequate.
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
 # Since: 2.4
 ##
 { 'enum': 'MigrationParameter',
-  'data': ['compress-level', 'compress-threads', 'decompress-threads'] }
+  'data': ['compress-level', 'compress-threads', 'decompress-threads',
+           'x-cpu-throttle-initial', 'x-cpu-throttle-increment'] }
 
 #
 # @migrate-set-parameters
@@ -606,12 +629,21 @@
 #
 # @decompress-threads: decompression thread count
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
 # Since: 2.4
 ##
 { 'command': 'migrate-set-parameters',
   'data': { '*compress-level': 'int',
             '*compress-threads': 'int',
-            '*decompress-threads': 'int'} }
+            '*decompress-threads': 'int',
+            '*x-cpu-throttle-initial': 'int',
+            '*x-cpu-throttle-increment': 'int'} }
 
 #
 # @MigrationParameters
@@ -622,12 +654,22 @@
 #
 # @decompress-threads: decompression thread count
 #
+# @x-cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                          when migration auto-converge is activated. The
+#                          default value is 20. (Since 2.5)
+#
+# @x-cpu-throttle-increment: throttle percentage increase each time
+#                            auto-converge detects that migration is not making
+#                            progress. The default value is 10. (Since 2.5)
+#
 # Since: 2.4
 ##
 { 'struct': 'MigrationParameters',
   'data': { 'compress-level': 'int',
             'compress-threads': 'int',
-            'decompress-threads': 'int'} }
+            'decompress-threads': 'int',
+            'x-cpu-throttle-initial': 'int',
+            'x-cpu-throttle-increment': 'int'} }
 ##
 # @query-migrate-parameters
 #
@@ -660,6 +702,16 @@
             '*tls-port': 'int', '*cert-subject': 'str' } }
 
 ##
+# @migrate-start-postcopy
+#
+# Followup to a migration command to switch the migration to postcopy mode.
+# The postcopy-ram capability must be set before the original migration
+# command.
+#
+# Since: 2.5
+{ 'command': 'migrate-start-postcopy' }
+
+##
 # @MouseInfo:
 #
 # Information about a mouse device.
@@ -690,43 +742,117 @@
 { 'command': 'query-mice', 'returns': ['MouseInfo'] }
 
 ##
+# @CpuInfoArch:
+#
+# An enumeration of cpu types that enable additional information during
+# @query-cpus.
+#
+# Since: 2.6
+##
+{ 'enum': 'CpuInfoArch',
+  'data': ['x86', 'sparc', 'ppc', 'mips', 'tricore', 'other' ] }
+
+##
 # @CpuInfo:
 #
 # Information about a virtual CPU
 #
 # @CPU: the index of the virtual CPU
 #
-# @current: this only exists for backwards compatible and should be ignored
+# @current: this only exists for backwards compatibility and should be ignored
 #
 # @halted: true if the virtual CPU is in the halt state.  Halt usually refers
 #          to a processor specific low power mode.
 #
 # @qom_path: path to the CPU object in the QOM tree (since 2.4)
 #
-# @pc: #optional If the target is i386 or x86_64, this is the 64-bit instruction
-#                pointer.
-#                If the target is Sparc, this is the PC component of the
-#                instruction pointer.
-#
-# @nip: #optional If the target is PPC, the instruction pointer
-#
-# @npc: #optional If the target is Sparc, the NPC component of the instruction
-#                 pointer
-#
-# @PC: #optional If the target is MIPS, the instruction pointer
-#
 # @thread_id: ID of the underlying host thread
 #
+# @arch: architecture of the cpu, which determines which additional fields
+#        will be listed (since 2.6)
+#
 # Since: 0.14.0
 #
 # Notes: @halted is a transient state that changes frequently.  By the time the
 #        data is sent to the client, the guest may no longer be halted.
 ##
-{ 'struct': 'CpuInfo',
-  'data': {'CPU': 'int', 'current': 'bool', 'halted': 'bool',
-           'qom_path': 'str',
-           '*pc': 'int', '*nip': 'int', '*npc': 'int', '*PC': 'int',
-           'thread_id': 'int'} }
+{ 'union': 'CpuInfo',
+  'base': {'CPU': 'int', 'current': 'bool', 'halted': 'bool',
+           'qom_path': 'str', 'thread_id': 'int', 'arch': 'CpuInfoArch' },
+  'discriminator': 'arch',
+  'data': { 'x86': 'CpuInfoX86',
+            'sparc': 'CpuInfoSPARC',
+            'ppc': 'CpuInfoPPC',
+            'mips': 'CpuInfoMIPS',
+            'tricore': 'CpuInfoTricore',
+            'other': 'CpuInfoOther' } }
+
+##
+# @CpuInfoX86:
+#
+# Additional information about a virtual i386 or x86_64 CPU
+#
+# @pc: the 64-bit instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoX86', 'data': { 'pc': 'int' } }
+
+##
+# @CpuInfoSPARC:
+#
+# Additional information about a virtual SPARC CPU
+#
+# @pc: the PC component of the instruction pointer
+#
+# @npc: the NPC component of the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoSPARC', 'data': { 'pc': 'int', 'npc': 'int' } }
+
+##
+# @CpuInfoPPC:
+#
+# Additional information about a virtual PPC CPU
+#
+# @nip: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoPPC', 'data': { 'nip': 'int' } }
+
+##
+# @CpuInfoMIPS:
+#
+# Additional information about a virtual MIPS CPU
+#
+# @PC: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoMIPS', 'data': { 'PC': 'int' } }
+
+##
+# @CpuInfoTricore:
+#
+# Additional information about a virtual Tricore CPU
+#
+# @PC: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoTricore', 'data': { 'PC': 'int' } }
+
+##
+# @CpuInfoOther:
+#
+# No additional information is available about the virtual CPU
+#
+# Since 2.6
+#
+##
+{ 'struct': 'CpuInfoOther', 'data': { } }
 
 ##
 # @query-cpus:
@@ -1482,6 +1608,26 @@
   'data': { } }
 
 ##
+# @ActionCompletionMode
+#
+# An enumeration of Transactional completion modes.
+#
+# @individual: Do not attempt to cancel any other Actions if any Actions fail
+#              after the Transaction request succeeds. All Actions that
+#              can complete successfully will do so without waiting on others.
+#              This is the default.
+#
+# @grouped: If any Action fails after the Transaction succeeds, cancel all
+#           Actions. Actions do not complete until all Actions are ready to
+#           complete. May be rejected by Actions that do not support this
+#           completion mode.
+#
+# Since: 2.5
+##
+{ 'enum': 'ActionCompletionMode',
+  'data': [ 'individual', 'grouped' ] }
+
+##
 # @TransactionAction
 #
 # A discriminated record of operations that can be performed with
@@ -1493,25 +1639,52 @@
 # abort since 1.6
 # blockdev-snapshot-internal-sync since 1.7
 # blockdev-backup since 2.3
+# blockdev-snapshot since 2.5
+# block-dirty-bitmap-add since 2.5
+# block-dirty-bitmap-clear since 2.5
 ##
 { 'union': 'TransactionAction',
   'data': {
-       'blockdev-snapshot-sync': 'BlockdevSnapshot',
+       'blockdev-snapshot': 'BlockdevSnapshot',
+       'blockdev-snapshot-sync': 'BlockdevSnapshotSync',
        'drive-backup': 'DriveBackup',
        'blockdev-backup': 'BlockdevBackup',
        'abort': 'Abort',
-       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal'
+       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal',
+       'block-dirty-bitmap-add': 'BlockDirtyBitmapAdd',
+       'block-dirty-bitmap-clear': 'BlockDirtyBitmap'
    } }
 
 ##
+# @TransactionProperties
+#
+# Optional arguments to modify the behavior of a Transaction.
+#
+# @completion-mode: #optional Controls how jobs launched asynchronously by
+#                   Actions will complete or fail as a group.
+#                   See @ActionCompletionMode for details.
+#
+# Since: 2.5
+##
+{ 'struct': 'TransactionProperties',
+  'data': {
+       '*completion-mode': 'ActionCompletionMode'
+  }
+}
+
+##
 # @transaction
 #
 # Executes a number of transactionable QMP commands atomically. If any
 # operation fails, then the entire set of actions will be abandoned and the
 # appropriate error returned.
 #
-#  List of:
-#  @TransactionAction: information needed for the respective operation
+# @actions: List of @TransactionAction;
+#           information needed for the respective operations.
+#
+# @properties: #optional structure of additional options to control the
+#              execution of the transaction. See @TransactionProperties
+#              for additional detail.
 #
 # Returns: nothing on success
 #          Errors depend on the operations of the transaction
@@ -1523,7 +1696,10 @@
 # Since 1.1
 ##
 { 'command': 'transaction',
-  'data': { 'actions': [ 'TransactionAction' ] } }
+  'data': { 'actions': [ 'TransactionAction' ],
+            '*properties': 'TransactionProperties'
+          }
+}
 
 ##
 # @human-monitor-command:
@@ -1632,13 +1808,10 @@
 #        1) A primitive type such as 'u8', 'u16', 'bool', 'str', or 'double'.
 #           These types are mapped to the appropriate JSON type.
 #
-#        2) A legacy type in the form 'legacy<subtype>' where subtype is the
-#           legacy qdev typename.  These types are always treated as strings.
-#
-#        3) A child type in the form 'child<subtype>' where subtype is a qdev
+#        2) A child type in the form 'child<subtype>' where subtype is a qdev
 #           device type name.  Child properties create the composition tree.
 #
-#        4) A link type in the form 'link<subtype>' where subtype is a qdev
+#        3) A link type in the form 'link<subtype>' where subtype is a qdev
 #           device type name.  Link properties form the device model graph.
 #
 # Since: 1.2
@@ -1689,17 +1862,16 @@
 #
 # @property: The property name to read
 #
-# Returns: The property value.  The type depends on the property type.  legacy<>
-#          properties are returned as #str.  child<> and link<> properties are
-#          returns as #str pathnames.  All integer property types (u8, u16, etc)
-#          are returned as #int.
+# Returns: The property value.  The type depends on the property
+#          type. child<> and link<> properties are returned as #str
+#          pathnames.  All integer property types (u8, u16, etc) are
+#          returned as #int.
 #
 # Since: 1.2
 ##
 { 'command': 'qom-get',
   'data': { 'path': 'str', 'property': 'str' },
-  'returns': '**',
-  'gen': false }
+  'returns': 'any' }
 
 ##
 # @qom-set:
@@ -1716,8 +1888,7 @@
 # Since: 1.2
 ##
 { 'command': 'qom-set',
-  'data': { 'path': 'str', 'property': 'str', 'value': '**' },
-  'gen': false }
+  'data': { 'path': 'str', 'property': 'str', 'value': 'any' } }
 
 ##
 # @set_password:
@@ -1809,8 +1980,10 @@
 #          device's password.  The behavior of reads and writes to the block
 #          device between when these calls are executed is undefined.
 #
-# Notes:  It is strongly recommended that this interface is not used especially
-#         for changing block devices.
+# Notes:  This interface is deprecated, and it is strongly recommended that you
+#         avoid using it.  For changing block devices, use
+#         blockdev-change-medium; for changing VNC parameters, use
+#         change-vnc-password.
 #
 # Since: 0.14.0
 ##
@@ -1950,7 +2123,7 @@
 #
 # Remove a device from a guest
 #
-# @id: the name of the device
+# @id: the name or QOM path of the device
 #
 # Returns: Nothing on success
 #          If @id is not a valid device, DeviceNotFound
@@ -1988,8 +2161,7 @@
 # @dump-guest-memory
 #
 # Dump guest's memory to vmcore. It is a synchronous operation that can take
-# very long depending on the amount of guest memory. This command is only
-# supported on i386 and x86_64.
+# very long depending on the amount of guest memory.
 #
 # @paging: if true, do paging to get guest's memory mapping. This allows
 #          using gdb to process the core file.
@@ -2005,6 +2177,7 @@
 #             2. The guest can be in real-mode even if paging is enabled. For
 #                example, the guest uses ACPI to sleep, and ACPI sleep state
 #                goes in real-mode
+#             3. Currently only supported on i386 and x86_64.
 #
 # @protocol: the filename or file descriptor of the vmcore. The supported
 #            protocols are:
@@ -2014,6 +2187,10 @@
 #            2. fd: the protocol starts with "fd:", and the following string
 #               is the fd's name.
 #
+# @detach: #optional if true, QMP will return immediately rather than
+#          waiting for the dump to finish. The user can track progress
+#          using "query-dump". (since 2.6).
+#
 # @begin: #optional if specified, the starting physical address.
 #
 # @length: #optional if specified, the memory size, in bytes. If you don't
@@ -2030,8 +2207,56 @@
 # Since: 1.2
 ##
 { 'command': 'dump-guest-memory',
-  'data': { 'paging': 'bool', 'protocol': 'str', '*begin': 'int',
-            '*length': 'int', '*format': 'DumpGuestMemoryFormat' } }
+  'data': { 'paging': 'bool', 'protocol': 'str', '*detach': 'bool',
+            '*begin': 'int', '*length': 'int',
+            '*format': 'DumpGuestMemoryFormat'} }
+
+##
+# @DumpStatus
+#
+# Describe the status of a long-running background guest memory dump.
+#
+# @none: no dump-guest-memory has started yet.
+#
+# @active: there is one dump running in background.
+#
+# @completed: the last dump has finished successfully.
+#
+# @failed: the last dump has failed.
+#
+# Since 2.6
+##
+{ 'enum': 'DumpStatus',
+  'data': [ 'none', 'active', 'completed', 'failed' ] }
+
+##
+# @DumpQueryResult
+#
+# The result format for 'query-dump'.
+#
+# @status: enum of @DumpStatus, which shows current dump status
+#
+# @completed: bytes written in latest dump (uncompressed)
+#
+# @total: total bytes to be written in latest dump (uncompressed)
+#
+# Since 2.6
+##
+{ 'struct': 'DumpQueryResult',
+  'data': { 'status': 'DumpStatus',
+            'completed': 'int',
+            'total': 'int' } }
+
+##
+# @query-dump
+#
+# Query latest dump status.
+#
+# Returns: A @DumpStatus object showing the dump status.
+#
+# Since: 2.6
+##
+{ 'command': 'query-dump', 'returns': 'DumpQueryResult' }
 
 ##
 # @DumpGuestMemoryCapability:
@@ -2058,6 +2283,20 @@
   'returns': 'DumpGuestMemoryCapability' }
 
 ##
+# @dump-skeys
+#
+# Dump guest's storage keys
+#
+# @filename: the path to the file to dump to
+#
+# This command is only supported on s390 architecture.
+#
+# Since: 2.5
+##
+{ 'command': 'dump-skeys',
+  'data': { 'filename': 'str' } }
+
+##
 # @netdev_add:
 #
 # Add a network backend.
@@ -2067,11 +2306,12 @@
 #
 # @id: the name of the new network backend
 #
-# @props: #optional a list of properties to be passed to the backend in
-#         the format 'name=value', like 'ifname=tap0,script=no'
+# Additional arguments depend on the type.
 #
-# Notes: The semantics of @props is not well defined.  Future commands will be
-#        introduced that provide stronger typing for backend creation.
+# TODO This command effectively bypasses QAPI completely due to its
+# "additional arguments" business.  It shouldn't have been added to
+# the schema in this form.  It should be qapified properly, or
+# replaced by a properly qapified command.
 #
 # Since: 0.14.0
 #
@@ -2079,8 +2319,8 @@
 #          If @type is not a valid network backend, DeviceNotFound
 ##
 { 'command': 'netdev_add',
-  'data': {'type': 'str', 'id': 'str', '*props': '**'},
-  'gen': false }
+  'data': {'type': 'str', 'id': 'str'},
+  'gen': false }                # so we can get the additional arguments
 
 ##
 # @netdev_del:
@@ -2113,8 +2353,7 @@
 # Since: 2.0
 ##
 { 'command': 'object-add',
-  'data': {'qom-type': 'str', 'id': 'str', '*props': '**'},
-  'gen': false }
+  'data': {'qom-type': 'str', 'id': 'str', '*props': 'any'} }
 
 ##
 # @object-del:
@@ -2186,9 +2425,18 @@
 #
 # @restrict: #optional isolate the guest from the host
 #
+# @ipv4: #optional whether to support IPv4, default true for enabled
+#        (since 2.6)
+#
+# @ipv6: #optional whether to support IPv6, default true for enabled
+#        (since 2.6)
+#
 # @ip: #optional legacy parameter, use net= instead
 #
-# @net: #optional IP address and optional netmask
+# @net: #optional IP network address that the guest will see, in the
+#       form addr[/netmask] The netmask is optional, and can be
+#       either in the form a.b.c.d or as a number of valid top-most
+#       bits. Default is 10.0.2.0/24.
 #
 # @host: #optional guest-visible address of the host
 #
@@ -2204,6 +2452,18 @@
 # @dnssearch: #optional list of DNS suffixes to search, passed as DHCP option
 #             to the guest
 #
+# @ipv6-prefix: #optional IPv6 network prefix (default is fec0::) (since
+#               2.6). The network prefix is given in the usual
+#               hexadecimal IPv6 address notation.
+#
+# @ipv6-prefixlen: #optional IPv6 network prefix length (default is 64)
+#                  (since 2.6)
+#
+# @ipv6-host: #optional guest-visible IPv6 address of the host (since 2.6)
+#
+# @ipv6-dns: #optional guest-visible IPv6 address of the virtual
+#            nameserver (since 2.6)
+#
 # @smb: #optional root directory of the built-in SMB server
 #
 # @smbserver: #optional IP address of the built-in SMB server
@@ -2219,6 +2479,8 @@
   'data': {
     '*hostname':  'str',
     '*restrict':  'bool',
+    '*ipv4':      'bool',
+    '*ipv6':      'bool',
     '*ip':        'str',
     '*net':       'str',
     '*host':      'str',
@@ -2227,6 +2489,10 @@
     '*dhcpstart': 'str',
     '*dns':       'str',
     '*dnssearch': ['String'],
+    '*ipv6-prefix':      'str',
+    '*ipv6-prefixlen':   'int',
+    '*ipv6-host':        'str',
+    '*ipv6-dns':         'str',
     '*smb':       'str',
     '*smbserver': 'str',
     '*hostfwd':   ['String'],
@@ -2466,12 +2732,16 @@
 #
 # @vhostforce: #optional vhost on for non-MSIX virtio guests (default: false).
 #
+# @queues: #optional number of queues to be created for multiqueue vhost-user
+#          (default: 1) (Since 2.5)
+#
 # Since 2.1
 ##
 { 'struct': 'NetdevVhostUserOptions',
   'data': {
     'chardev':        'str',
-    '*vhostforce':    'bool' } }
+    '*vhostforce':    'bool',
+    '*queues':        'int' } }
 
 ##
 # @NetClientOptions
@@ -2537,6 +2807,26 @@
     'opts': 'NetClientOptions' } }
 
 ##
+# @NetFilterDirection
+#
+# Indicates whether a netfilter is attached to a netdev's transmit queue or
+# receive queue or both.
+#
+# @all: the filter is attached both to the receive and the transmit
+#       queue of the netdev (default).
+#
+# @rx: the filter is attached to the receive queue of the netdev,
+#      where it will receive packets sent to the netdev.
+#
+# @tx: the filter is attached to the transmit queue of the netdev,
+#      where it will receive packets sent by the netdev.
+#
+# Since 2.5
+##
+{ 'enum': 'NetFilterDirection',
+  'data': [ 'all', 'rx', 'tx' ] }
+
+##
 # @InetSocketAddress
 #
 # Captures a socket address or address range in the Internet namespace.
@@ -2807,6 +3097,7 @@
 #
 # 'unmapped' and 'pause' since 2.0
 # 'ro' and 'kp_comma' since 2.4
+# 'kp_equals' and 'power' since 2.6
 ##
 { 'enum': 'QKeyCode',
   'data': [ 'unmapped',
@@ -2825,7 +3116,7 @@
             'left', 'up', 'down', 'right', 'insert', 'delete', 'stop', 'again',
             'props', 'undo', 'front', 'copy', 'open', 'paste', 'find', 'cut',
             'lf', 'help', 'meta_l', 'meta_r', 'compose', 'pause', 'ro',
-            'kp_comma' ] }
+            'kp_comma', 'kp_equals', 'power' ] }
 
 ##
 # @KeyValue
@@ -2874,6 +3165,21 @@
 ##
 { 'command': 'screendump', 'data': {'filename': 'str'} }
 
+
+##
+# @ChardevCommon:
+#
+# Configuration shared across all chardev backends
+#
+# @logfile: #optional The name of a logfile to save output
+# @logappend: #optional true to append instead of truncate
+#             (default to false to truncate)
+#
+# Since: 2.6
+##
+{ 'struct': 'ChardevCommon', 'data': { '*logfile': 'str',
+                                       '*logappend': 'bool' } }
+
 ##
 # @ChardevFile:
 #
@@ -2881,11 +3187,15 @@
 #
 # @in:  #optional The name of the input file
 # @out: The name of the output file
+# @append: #optional Open the file in append mode (default false to
+#          truncate) (Since 2.6)
 #
 # Since: 1.4
 ##
 { 'struct': 'ChardevFile', 'data': { '*in' : 'str',
-                                   'out' : 'str' } }
+                                   'out' : 'str',
+                                   '*append': 'bool' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevHostdev:
@@ -2898,7 +3208,8 @@
 #
 # Since: 1.4
 ##
-{ 'struct': 'ChardevHostdev', 'data': { 'device' : 'str' } }
+{ 'struct': 'ChardevHostdev', 'data': { 'device' : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevSocket:
@@ -2907,6 +3218,7 @@
 #
 # @addr: socket address to listen on (server=true)
 #        or connect to (server=false)
+# @tls-creds: #optional the ID of the TLS credentials object (since 2.6)
 # @server: #optional create server socket (default: true)
 # @wait: #optional wait for incoming connection on server
 #        sockets (default: false).
@@ -2921,11 +3233,13 @@
 # Since: 1.4
 ##
 { 'struct': 'ChardevSocket', 'data': { 'addr'       : 'SocketAddress',
+                                     '*tls-creds'  : 'str',
                                      '*server'    : 'bool',
                                      '*wait'      : 'bool',
                                      '*nodelay'   : 'bool',
                                      '*telnet'    : 'bool',
-                                     '*reconnect' : 'int' } }
+                                     '*reconnect' : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevUdp:
@@ -2938,7 +3252,8 @@
 # Since: 1.5
 ##
 { 'struct': 'ChardevUdp', 'data': { 'remote' : 'SocketAddress',
-                                  '*local' : 'SocketAddress' } }
+                                  '*local' : 'SocketAddress' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevMux:
@@ -2949,7 +3264,8 @@
 #
 # Since: 1.5
 ##
-{ 'struct': 'ChardevMux', 'data': { 'chardev' : 'str' } }
+{ 'struct': 'ChardevMux', 'data': { 'chardev' : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevStdio:
@@ -2962,7 +3278,9 @@
 #
 # Since: 1.5
 ##
-{ 'struct': 'ChardevStdio', 'data': { '*signal' : 'bool' } }
+{ 'struct': 'ChardevStdio', 'data': { '*signal' : 'bool' },
+  'base': 'ChardevCommon' }
+
 
 ##
 # @ChardevSpiceChannel:
@@ -2973,7 +3291,8 @@
 #
 # Since: 1.5
 ##
-{ 'struct': 'ChardevSpiceChannel', 'data': { 'type'  : 'str' } }
+{ 'struct': 'ChardevSpiceChannel', 'data': { 'type'  : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevSpicePort:
@@ -2984,7 +3303,8 @@
 #
 # Since: 1.5
 ##
-{ 'struct': 'ChardevSpicePort', 'data': { 'fqdn'  : 'str' } }
+{ 'struct': 'ChardevSpicePort', 'data': { 'fqdn'  : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevVC:
@@ -3001,7 +3321,8 @@
 { 'struct': 'ChardevVC', 'data': { '*width'  : 'int',
                                  '*height' : 'int',
                                  '*cols'   : 'int',
-                                 '*rows'   : 'int' } }
+                                 '*rows'   : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevRingbuf:
@@ -3012,7 +3333,8 @@
 #
 # Since: 1.5
 ##
-{ 'struct': 'ChardevRingbuf', 'data': { '*size'  : 'int' } }
+{ 'struct': 'ChardevRingbuf', 'data': { '*size'  : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevBackend:
@@ -3021,22 +3343,20 @@
 #
 # Since: 1.4 (testdev since 2.2)
 ##
-{ 'struct': 'ChardevDummy', 'data': { } }
-
 { 'union': 'ChardevBackend', 'data': { 'file'   : 'ChardevFile',
                                        'serial' : 'ChardevHostdev',
                                        'parallel': 'ChardevHostdev',
                                        'pipe'   : 'ChardevHostdev',
                                        'socket' : 'ChardevSocket',
                                        'udp'    : 'ChardevUdp',
-                                       'pty'    : 'ChardevDummy',
-                                       'null'   : 'ChardevDummy',
+                                       'pty'    : 'ChardevCommon',
+                                       'null'   : 'ChardevCommon',
                                        'mux'    : 'ChardevMux',
-                                       'msmouse': 'ChardevDummy',
-                                       'braille': 'ChardevDummy',
-                                       'testdev': 'ChardevDummy',
+                                       'msmouse': 'ChardevCommon',
+                                       'braille': 'ChardevCommon',
+                                       'testdev': 'ChardevCommon',
                                        'stdio'  : 'ChardevStdio',
-                                       'console': 'ChardevDummy',
+                                       'console': 'ChardevCommon',
                                        'spicevmc' : 'ChardevSpiceChannel',
                                        'spiceport' : 'ChardevSpicePort',
                                        'vc'     : 'ChardevVC',
@@ -3345,6 +3665,17 @@
             'features': 'int' } }
 
 ##
+# @DummyForceArrays
+#
+# Not used by QMP; hack to let us use X86CPUFeatureWordInfoList internally
+#
+# Since 2.5
+##
+{ 'struct': 'DummyForceArrays',
+  'data': { 'unused': ['X86CPUFeatureWordInfo'] } }
+
+
+##
 # @RxState:
 #
 # Packets receiving state
@@ -3431,17 +3762,17 @@
 # Since: 2.0
 ##
 { 'enum'  : 'InputButton',
-  'data'  : [ 'Left', 'Middle', 'Right', 'WheelUp', 'WheelDown' ] }
+  'data'  : [ 'left', 'middle', 'right', 'wheel-up', 'wheel-down' ] }
 
 ##
-# @InputButton
+# @InputAxis
 #
 # Position axis of a pointer input device (mouse, tablet).
 #
 # Since: 2.0
 ##
 { 'enum'  : 'InputAxis',
-  'data'  : [ 'X', 'Y' ] }
+  'data'  : [ 'x', 'y' ] }
 
 ##
 # @InputKeyEvent
@@ -3505,35 +3836,34 @@
               'abs'     : 'InputMoveEvent' } }
 
 ##
-# @x-input-send-event
+# @input-send-event
 #
 # Send input event(s) to guest.
 #
-# @console: #optional console to send event(s) to.
-#           This parameter can be used to send the input event to
-#           specific input devices in case (a) multiple input devices
-#           of the same kind are added to the virtual machine and (b)
-#           you have configured input routing (see docs/multiseat.txt)
-#           for those input devices.  If input routing is not
-#           configured this parameter has no effect.
-#           If @console is missing, only devices that aren't associated
-#           with a console are admissible.
-#           If @console is specified, it must exist, and both devices
-#           associated with that console and devices not associated with a
-#           console are admissible, but the former take precedence.
-
-#
+# @device: #optional display device to send event(s) to.
+# @head: #optional head to send event(s) to, in case the
+#        display device supports multiple scanouts.
 # @events: List of InputEvent union.
 #
 # Returns: Nothing on success.
 #
-# Since: 2.2
-#
-# Note: this command is experimental, and not a stable API.
+# The @display and @head parameters can be used to send the input
+# event to specific input devices in case (a) multiple input devices
+# of the same kind are added to the virtual machine and (b) you have
+# configured input routing (see docs/multiseat.txt) for those input
+# devices.  The parameters work exactly like the device and head
+# properties of input devices.  If @device is missing, only devices
+# that have no input routing config are admissible.  If @device is
+# specified, both input devices with and without input routing config
+# are admissible, but devices with input routing config take
+# precedence.
 #
+# Since: 2.6
 ##
-{ 'command': 'x-input-send-event',
-  'data': { '*console':'int', 'events': [ 'InputEvent' ] } }
+{ 'command': 'input-send-event',
+  'data': { '*device': 'str',
+            '*head'  : 'int',
+            'events' : [ 'InputEvent' ] } }
 
 ##
 # @NumaOptions
@@ -3794,3 +4124,57 @@
 
 # Rocker ethernet network switch
 { 'include': 'qapi/rocker.json' }
+
+##
+# ReplayMode:
+#
+# Mode of the replay subsystem.
+#
+# @none: normal execution mode. Replay or record are not enabled.
+#
+# @record: record mode. All non-deterministic data is written into the
+#          replay log.
+#
+# @play: replay mode. Non-deterministic data required for system execution
+#        is read from the log.
+#
+# Since: 2.5
+##
+{ 'enum': 'ReplayMode',
+  'data': [ 'none', 'record', 'play' ] }
+
+##
+# @GICCapability:
+#
+# The struct describes capability for a specific GIC (Generic
+# Interrupt Controller) version. These bits are not only decided by
+# QEMU/KVM software version, but also decided by the hardware that
+# the program is running upon.
+#
+# @version:  version of GIC to be described. Currently, only 2 and 3
+#            are supported.
+#
+# @emulated: whether current QEMU/hardware supports emulated GIC
+#            device in user space.
+#
+# @kernel:   whether current QEMU/hardware supports hardware
+#            accelerated GIC device in kernel.
+#
+# Since: 2.6
+##
+{ 'struct': 'GICCapability',
+  'data': { 'version': 'int',
+            'emulated': 'bool',
+            'kernel': 'bool' } }
+
+##
+# @query-gic-capabilities:
+#
+# This command is ARM-only. It will return a list of GICCapability
+# objects that describe its capability bits.
+#
+# Returns: a list of GICCapability objects.
+#
+# Since: 2.6
+##
+{ 'command': 'query-gic-capabilities', 'returns': ['GICCapability'] }
diff --git a/qemu/qapi/block-core.json b/qemu/qapi/block-core.json
index 7b2efb867..1d09079cc 100644
--- a/qemu/qapi/block-core.json
+++ b/qemu/qapi/block-core.json
@@ -186,6 +186,33 @@
            '*fragmented-clusters': 'int', '*compressed-clusters': 'int' } }
 
 ##
+# @MapEntry:
+#
+# Mapping information from a virtual block range to a host file range
+#
+# @start: the start byte of the mapped virtual range
+#
+# @length: the number of bytes of the mapped virtual range
+#
+# @data: whether the mapped range has data
+#
+# @zero: whether the virtual blocks are zeroed
+#
+# @depth: the depth of the mapping
+#
+# @offset: #optional the offset in file that the virtual sectors are mapped to
+#
+# @filename: #optional filename that is referred to by @offset
+#
+# Since: 2.6
+#
+##
+{ 'struct': 'MapEntry',
+  'data': {'start': 'int', 'length': 'int', 'data': 'bool',
+           'zero': 'bool', 'depth': 'int', '*offset': 'int',
+           '*filename': 'str' } }
+
+##
 # @BlockdevCacheInfo
 #
 # Cache mode information for a block device
@@ -215,10 +242,12 @@
 # @drv: the name of the block format used to open the backing device. As of
 #       0.14.0 this can be: 'blkdebug', 'bochs', 'cloop', 'cow', 'dmg',
 #       'file', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-#       'host_floppy', 'http', 'https', 'nbd', 'parallels', 'qcow',
+#       'http', 'https', 'luks', 'nbd', 'parallels', 'qcow',
 #       'qcow2', 'raw', 'tftp', 'vdi', 'vmdk', 'vpc', 'vvfat'
 #       2.2: 'archipelago' added, 'cow' dropped
 #       2.3: 'host_floppy' deprecated
+#       2.5: 'host_floppy' dropped
+#       2.6: 'luks' added
 #
 # @backing_file: #optional the name of the backing file (for copy-on-write)
 #
@@ -245,17 +274,41 @@
 #
 # @image: the info of image used (since: 1.6)
 #
-# @bps_max: #optional total max in bytes (Since 1.7)
+# @bps_max: #optional total throughput limit during bursts,
+#                     in bytes (Since 1.7)
+#
+# @bps_rd_max: #optional read throughput limit during bursts,
+#                        in bytes (Since 1.7)
+#
+# @bps_wr_max: #optional write throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @bps_rd_max: #optional read max in bytes (Since 1.7)
+# @iops_max: #optional total I/O operations per second during bursts,
+#                      in bytes (Since 1.7)
 #
-# @bps_wr_max: #optional write max in bytes (Since 1.7)
+# @iops_rd_max: #optional read I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
 #
-# @iops_max: #optional total I/O operations max (Since 1.7)
+# @iops_wr_max: #optional write I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
 #
-# @iops_rd_max: #optional read I/O operations max (Since 1.7)
+# @bps_max_length: #optional maximum length of the @bps_max burst
+#                            period, in seconds. (Since 2.6)
 #
-# @iops_wr_max: #optional write I/O operations max (Since 1.7)
+# @bps_rd_max_length: #optional maximum length of the @bps_rd_max
+#                               burst period, in seconds. (Since 2.6)
+#
+# @bps_wr_max_length: #optional maximum length of the @bps_wr_max
+#                               burst period, in seconds. (Since 2.6)
+#
+# @iops_max_length: #optional maximum length of the @iops burst
+#                             period, in seconds. (Since 2.6)
+#
+# @iops_rd_max_length: #optional maximum length of the @iops_rd_max
+#                                burst period, in seconds. (Since 2.6)
+#
+# @iops_wr_max_length: #optional maximum length of the @iops_wr_max
+#                                burst period, in seconds. (Since 2.6)
 #
 # @iops_size: #optional an I/O size in bytes (Since 1.7)
 #
@@ -280,6 +333,9 @@
             '*bps_max': 'int', '*bps_rd_max': 'int',
             '*bps_wr_max': 'int', '*iops_max': 'int',
             '*iops_rd_max': 'int', '*iops_wr_max': 'int',
+            '*bps_max_length': 'int', '*bps_rd_max_length': 'int',
+            '*bps_wr_max_length': 'int', '*iops_max_length': 'int',
+            '*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int',
             '*iops_size': 'int', '*group': 'str', 'cache': 'BlockdevCacheInfo',
             'write_threshold': 'int' } }
 
@@ -381,8 +437,8 @@
 # @locked: True if the guest has locked this device from having its media
 #          removed
 #
-# @tray_open: #optional True if the device has a tray and it is open
-#             (only present if removable is true)
+# @tray_open: #optional True if the device's tray is open
+#             (only present if it has a tray)
 #
 # @dirty-bitmaps: #optional dirty bitmaps information (only present if the
 #                 driver has one or more dirty bitmaps) (Since 2.0)
@@ -413,6 +469,59 @@
 ##
 { 'command': 'query-block', 'returns': ['BlockInfo'] }
 
+
+##
+# @BlockDeviceTimedStats:
+#
+# Statistics of a block device during a given interval of time.
+#
+# @interval_length: Interval used for calculating the statistics,
+#                   in seconds.
+#
+# @min_rd_latency_ns: Minimum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_wr_latency_ns: Minimum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_flush_latency_ns: Minimum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @max_rd_latency_ns: Maximum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_wr_latency_ns: Maximum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_flush_latency_ns: Maximum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_latency_ns: Average latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_wr_latency_ns: Average latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_flush_latency_ns: Average latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_queue_depth: Average number of pending read operations
+#                      in the defined interval.
+#
+# @avg_wr_queue_depth: Average number of pending write operations
+#                      in the defined interval.
+#
+# Since: 2.5
+##
+
+{ 'struct': 'BlockDeviceTimedStats',
+  'data': { 'interval_length': 'int', 'min_rd_latency_ns': 'int',
+            'max_rd_latency_ns': 'int', 'avg_rd_latency_ns': 'int',
+            'min_wr_latency_ns': 'int', 'max_wr_latency_ns': 'int',
+            'avg_wr_latency_ns': 'int', 'min_flush_latency_ns': 'int',
+            'max_flush_latency_ns': 'int', 'avg_flush_latency_ns': 'int',
+            'avg_rd_queue_depth': 'number', 'avg_wr_queue_depth': 'number' } }
+
 ##
 # @BlockDeviceStats:
 #
@@ -447,6 +556,37 @@
 # @wr_merged: Number of write requests that have been merged into another
 #             request (Since 2.3).
 #
+# @idle_time_ns: #optional Time since the last I/O operation, in
+#                nanoseconds. If the field is absent it means that
+#                there haven't been any operations yet (Since 2.5).
+#
+# @failed_rd_operations: The number of failed read operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_wr_operations: The number of failed write operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_flush_operations: The number of failed flush operations
+#                           performed by the device (Since 2.5)
+#
+# @invalid_rd_operations: The number of invalid read operations
+#                          performed by the device (Since 2.5)
+#
+# @invalid_wr_operations: The number of invalid write operations
+#                         performed by the device (Since 2.5)
+#
+# @invalid_flush_operations: The number of invalid flush operations
+#                            performed by the device (Since 2.5)
+#
+# @account_invalid: Whether invalid operations are included in the
+#                   last access statistics (Since 2.5)
+#
+# @account_failed: Whether failed operations are included in the
+#                  latency and last access statistics (Since 2.5)
+#
+# @timed_stats: Statistics specific to the set of previously defined
+#               intervals of time (Since 2.5)
+#
 # Since: 0.14.0
 ##
 { 'struct': 'BlockDeviceStats',
@@ -454,7 +594,12 @@
            'wr_operations': 'int', 'flush_operations': 'int',
            'flush_total_time_ns': 'int', 'wr_total_time_ns': 'int',
            'rd_total_time_ns': 'int', 'wr_highest_offset': 'int',
-           'rd_merged': 'int', 'wr_merged': 'int' } }
+           'rd_merged': 'int', 'wr_merged': 'int', '*idle_time_ns': 'int',
+           'failed_rd_operations': 'int', 'failed_wr_operations': 'int',
+           'failed_flush_operations': 'int', 'invalid_rd_operations': 'int',
+           'invalid_wr_operations': 'int', 'invalid_flush_operations': 'int',
+           'account_invalid': 'bool', 'account_failed': 'bool',
+           'timed_stats': ['BlockDeviceTimedStats'] } }
 
 ##
 # @BlockStats:
@@ -681,7 +826,7 @@
   'data': [ 'existing', 'absolute-paths' ] }
 
 ##
-# @BlockdevSnapshot
+# @BlockdevSnapshotSync
 #
 # Either @device or @node-name must be set but not both.
 #
@@ -698,12 +843,27 @@
 # @mode: #optional whether and how QEMU should create a new image, default is
 #        'absolute-paths'.
 ##
-{ 'struct': 'BlockdevSnapshot',
+{ 'struct': 'BlockdevSnapshotSync',
   'data': { '*device': 'str', '*node-name': 'str',
             'snapshot-file': 'str', '*snapshot-node-name': 'str',
             '*format': 'str', '*mode': 'NewImageMode' } }
 
 ##
+# @BlockdevSnapshot
+#
+# @node: device or node name that will have a snapshot created.
+#
+# @overlay: reference to the existing block device that will become
+#           the overlay of @node, as part of creating the snapshot.
+#           It must not have a current backing file (this can be
+#           achieved by passing "backing": "" to blockdev-add).
+#
+# Since 2.5
+##
+{ 'struct': 'BlockdevSnapshot',
+  'data': { 'node': 'str', 'overlay': 'str' } }
+
+##
 # @DriveBackup
 #
 # @device: the name of the device which should be copied.
@@ -789,7 +949,7 @@
 #
 # Generates a synchronous snapshot of a block device.
 #
-# For the arguments, see the documentation of BlockdevSnapshot.
+# For the arguments, see the documentation of BlockdevSnapshotSync.
 #
 # Returns: nothing on success
 #          If @device is not a valid block device, DeviceNotFound
@@ -797,6 +957,19 @@
 # Since 0.14.0
 ##
 { 'command': 'blockdev-snapshot-sync',
+  'data': 'BlockdevSnapshotSync' }
+
+
+##
+# @blockdev-snapshot
+#
+# Generates a snapshot of a block device.
+#
+# For the arguments, see the documentation of BlockdevSnapshot.
+#
+# Since 2.5
+##
+{ 'command': 'blockdev-snapshot',
   'data': 'BlockdevSnapshot' }
 
 ##
@@ -1066,6 +1239,54 @@
   'data': 'BlockDirtyBitmap' }
 
 ##
+# @blockdev-mirror
+#
+# Start mirroring a block device's writes to a new destination.
+#
+# @device: the name of the device whose writes should be mirrored.
+#
+# @target: the id or node-name of the block device to mirror to. This mustn't be
+#          attached to guest.
+#
+# @replaces: #optional with sync=full graph node name to be replaced by the new
+#            image when a whole image copy is done. This can be used to repair
+#            broken Quorum files.
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# @sync: what parts of the disk image should be copied to the destination
+#        (all the disk, only the sectors allocated in the topmost image, or
+#        only new I/O).
+#
+# @granularity: #optional granularity of the dirty bitmap, default is 64K
+#               if the image format doesn't have clusters, 4K if the clusters
+#               are smaller than that, else the cluster size.  Must be a
+#               power of 2 between 512 and 64M
+#
+# @buf-size: #optional maximum amount of data in flight from source to
+#            target
+#
+# @on-source-error: #optional the action to take on an error on the source,
+#                   default 'report'.  'stop' and 'enospc' can only be used
+#                   if the block device supports io-status (see BlockInfo).
+#
+# @on-target-error: #optional the action to take on an error on the target,
+#                   default 'report' (no limitations, since this applies to
+#                   a different block device than @device).
+#
+# Returns: nothing on success.
+#
+# Since 2.6
+##
+{ 'command': 'blockdev-mirror',
+  'data': { 'device': 'str', 'target': 'str',
+            '*replaces': 'str',
+            'sync': 'MirrorSyncMode',
+            '*speed': 'int', '*granularity': 'uint32',
+            '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
+            '*on-target-error': 'BlockdevOnError' } }
+
+##
 # @block_set_io_throttle:
 #
 # Change I/O throttle limits for a block drive.
@@ -1101,21 +1322,57 @@
 #
 # @iops: total I/O operations per second
 #
-# @ops_rd: read I/O operations per second
+# @iops_rd: read I/O operations per second
 #
 # @iops_wr: write I/O operations per second
 #
-# @bps_max: #optional total max in bytes (Since 1.7)
+# @bps_max: #optional total throughput limit during bursts,
+#                     in bytes (Since 1.7)
 #
-# @bps_rd_max: #optional read max in bytes (Since 1.7)
+# @bps_rd_max: #optional read throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @bps_wr_max: #optional write max in bytes (Since 1.7)
+# @bps_wr_max: #optional write throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @iops_max: #optional total I/O operations max (Since 1.7)
+# @iops_max: #optional total I/O operations per second during bursts,
+#                      in bytes (Since 1.7)
 #
-# @iops_rd_max: #optional read I/O operations max (Since 1.7)
+# @iops_rd_max: #optional read I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
 #
-# @iops_wr_max: #optional write I/O operations max (Since 1.7)
+# @iops_wr_max: #optional write I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
+#
+# @bps_max_length: #optional maximum length of the @bps_max burst
+#                            period, in seconds. It must only
+#                            be set if @bps_max is set as well.
+#                            Defaults to 1. (Since 2.6)
+#
+# @bps_rd_max_length: #optional maximum length of the @bps_rd_max
+#                               burst period, in seconds. It must only
+#                               be set if @bps_rd_max is set as well.
+#                               Defaults to 1. (Since 2.6)
+#
+# @bps_wr_max_length: #optional maximum length of the @bps_wr_max
+#                               burst period, in seconds. It must only
+#                               be set if @bps_wr_max is set as well.
+#                               Defaults to 1. (Since 2.6)
+#
+# @iops_max_length: #optional maximum length of the @iops burst
+#                             period, in seconds. It must only
+#                             be set if @iops_max is set as well.
+#                             Defaults to 1. (Since 2.6)
+#
+# @iops_rd_max_length: #optional maximum length of the @iops_rd_max
+#                                burst period, in seconds. It must only
+#                                be set if @iops_rd_max is set as well.
+#                                Defaults to 1. (Since 2.6)
+#
+# @iops_wr_max_length: #optional maximum length of the @iops_wr_max
+#                                burst period, in seconds. It must only
+#                                be set if @iops_wr_max is set as well.
+#                                Defaults to 1. (Since 2.6)
 #
 # @iops_size: #optional an I/O size in bytes (Since 1.7)
 #
@@ -1132,6 +1389,9 @@
             '*bps_max': 'int', '*bps_rd_max': 'int',
             '*bps_wr_max': 'int', '*iops_max': 'int',
             '*iops_rd_max': 'int', '*iops_wr_max': 'int',
+            '*bps_max_length': 'int', '*bps_rd_max_length': 'int',
+            '*bps_wr_max_length': 'int', '*iops_max_length': 'int',
+            '*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int',
             '*iops_size': 'int', '*group': 'str' } }
 
 ##
@@ -1355,7 +1615,6 @@
 #
 # Includes cache-related options for block devices
 #
-# @writeback:   #optional enables writeback mode for any caches (default: true)
 # @direct:      #optional enables use of O_DIRECT (bypass the host page cache;
 #               default: false)
 # @no-flush:    #optional ignore any flush requests for the device (default:
@@ -1364,8 +1623,7 @@
 # Since: 1.7
 ##
 { 'struct': 'BlockdevCacheOptions',
-  'data': { '*writeback': 'bool',
-            '*direct': 'bool',
+  'data': { '*direct': 'bool',
             '*no-flush': 'bool' } }
 
 ##
@@ -1373,56 +1631,18 @@
 #
 # Drivers that are supported in block device operations.
 #
-# @host_device, @host_cdrom, @host_floppy: Since 2.1
-# @host_floppy: deprecated since 2.3
+# @host_device, @host_cdrom: Since 2.1
 #
 # Since: 2.0
 ##
 { 'enum': 'BlockdevDriver',
   'data': [ 'archipelago', 'blkdebug', 'blkverify', 'bochs', 'cloop',
             'dmg', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-            'host_floppy', 'http', 'https', 'null-aio', 'null-co', 'parallels',
+            'http', 'https', 'luks', 'null-aio', 'null-co', 'parallels',
             'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'tftp', 'vdi', 'vhdx',
             'vmdk', 'vpc', 'vvfat' ] }
 
 ##
-# @BlockdevOptionsBase
-#
-# Options that are available for all block devices, independent of the block
-# driver.
-#
-# @driver:        block driver name
-# @id:            #optional id by which the new block device can be referred to.
-#                 This is a required option on the top level of blockdev-add, and
-#                 currently not allowed on any other level.
-# @node-name:     #optional the name of a block driver state node (Since 2.0)
-# @discard:       #optional discard-related options (default: ignore)
-# @cache:         #optional cache-related options
-# @aio:           #optional AIO backend (default: threads)
-# @rerror:        #optional how to handle read errors on the device
-#                 (default: report)
-# @werror:        #optional how to handle write errors on the device
-#                 (default: enospc)
-# @read-only:     #optional whether the block device should be read-only
-#                 (default: false)
-# @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
-#                 (default: off)
-#
-# Since: 1.7
-##
-{ 'struct': 'BlockdevOptionsBase',
-  'data': { 'driver': 'BlockdevDriver',
-            '*id': 'str',
-            '*node-name': 'str',
-            '*discard': 'BlockdevDiscardOptions',
-            '*cache': 'BlockdevCacheOptions',
-            '*aio': 'BlockdevAioOptions',
-            '*rerror': 'BlockdevOnError',
-            '*werror': 'BlockdevOnError',
-            '*read-only': 'bool',
-            '*detect-zeroes': 'BlockdevDetectZeroesOptions' } }
-
-##
 # @BlockdevOptionsFile
 #
 # Driver specific block device options for the file backend and similar
@@ -1485,6 +1705,22 @@
   'data': { 'file': 'BlockdevRef' } }
 
 ##
+# @BlockdevOptionsLUKS
+#
+# Driver specific block device options for LUKS.
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing
+#              the decryption key (since 2.6). Mandatory except when
+#              doing a metadata-only probe of the image.
+#
+# Since: 2.6
+##
+{ 'struct': 'BlockdevOptionsLUKS',
+  'base': 'BlockdevOptionsGenericFormat',
+  'data': { '*key-secret': 'str' } }
+
+
+##
 # @BlockdevOptionsGenericCOWFormat
 #
 # Driver specific block device options for image format that have no option
@@ -1592,6 +1828,10 @@
 # @refcount-cache-size:   #optional the maximum size of the refcount block cache
 #                         in bytes (since 2.2)
 #
+# @cache-clean-interval:  #optional clean unused entries in the L2 and refcount
+#                         caches. The interval is in seconds. The default value
+#                         is 0 and it disables this feature (since 2.5)
+#
 # Since: 1.7
 ##
 { 'struct': 'BlockdevOptionsQcow2',
@@ -1603,7 +1843,8 @@
             '*overlap-check': 'Qcow2OverlapChecks',
             '*cache-size': 'int',
             '*l2-cache-size': 'int',
-            '*refcount-cache-size': 'int' } }
+            '*refcount-cache-size': 'int',
+            '*cache-clean-interval': 'int' } }
 
 
 ##
@@ -1640,21 +1881,23 @@
 # @BlkdebugEvent
 #
 # Trigger events supported by blkdebug.
+#
+# Since: 2.0
 ##
-{ 'enum': 'BlkdebugEvent',
-  'data': [ 'l1_update', 'l1_grow.alloc_table', 'l1_grow.write_table',
-            'l1_grow.activate_table', 'l2_load', 'l2_update',
-            'l2_update_compressed', 'l2_alloc.cow_read', 'l2_alloc.write',
+{ 'enum': 'BlkdebugEvent', 'prefix': 'BLKDBG',
+  'data': [ 'l1_update', 'l1_grow_alloc_table', 'l1_grow_write_table',
+            'l1_grow_activate_table', 'l2_load', 'l2_update',
+            'l2_update_compressed', 'l2_alloc_cow_read', 'l2_alloc_write',
             'read_aio', 'read_backing_aio', 'read_compressed', 'write_aio',
             'write_compressed', 'vmstate_load', 'vmstate_save', 'cow_read',
             'cow_write', 'reftable_load', 'reftable_grow', 'reftable_update',
             'refblock_load', 'refblock_update', 'refblock_update_part',
-            'refblock_alloc', 'refblock_alloc.hookup', 'refblock_alloc.write',
-            'refblock_alloc.write_blocks', 'refblock_alloc.write_table',
-            'refblock_alloc.switch_table', 'cluster_alloc',
+            'refblock_alloc', 'refblock_alloc_hookup', 'refblock_alloc_write',
+            'refblock_alloc_write_blocks', 'refblock_alloc_write_table',
+            'refblock_alloc_switch_table', 'cluster_alloc',
             'cluster_alloc_bytes', 'cluster_free', 'flush_to_os',
-            'flush_to_disk', 'pwritev_rmw.head', 'pwritev_rmw.after_head',
-            'pwritev_rmw.tail', 'pwritev_rmw.after_tail', 'pwritev',
+            'flush_to_disk', 'pwritev_rmw_head', 'pwritev_rmw_after_head',
+            'pwritev_rmw_tail', 'pwritev_rmw_after_tail', 'pwritev',
             'pwritev_zero', 'pwritev_done', 'empty_image_prepare' ] }
 
 ##
@@ -1791,12 +2034,55 @@
 ##
 # @BlockdevOptions
 #
-# Options for creating a block device.
+# Options for creating a block device.  Many options are available for all
+# block devices, independent of the block driver:
+#
+# @driver:        block driver name
+# @id:            #optional id by which the new block device can be referred to.
+#                 This option is only allowed on the top level of blockdev-add.
+#                 A BlockBackend will be created by blockdev-add if and only if
+#                 this option is given.
+# @node-name:     #optional the name of a block driver state node (Since 2.0).
+#                 This option is required on the top level of blockdev-add if
+#                 the @id option is not given there.
+# @discard:       #optional discard-related options (default: ignore)
+# @cache:         #optional cache-related options
+# @aio:           #optional AIO backend (default: threads)
+# @rerror:        #optional how to handle read errors on the device
+#                 (default: report)
+# @werror:        #optional how to handle write errors on the device
+#                 (default: enospc)
+# @read-only:     #optional whether the block device should be read-only
+#                 (default: false)
+# @stats-account-invalid: #optional whether to include invalid
+#                         operations when computing last access statistics
+#                         (default: true) (Since 2.5)
+# @stats-account-failed: #optional whether to include failed
+#                         operations when computing latency and last
+#                         access statistics (default: true) (Since 2.5)
+# @stats-intervals: #optional list of intervals for collecting I/O
+#                   statistics, in seconds (default: none) (Since 2.5)
+# @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
+#                 (default: off)
+#
+# Remaining options are determined by the block driver.
 #
 # Since: 1.7
 ##
 { 'union': 'BlockdevOptions',
-  'base': 'BlockdevOptionsBase',
+  'base': { 'driver': 'BlockdevDriver',
+            '*id': 'str',
+            '*node-name': 'str',
+            '*discard': 'BlockdevDiscardOptions',
+            '*cache': 'BlockdevCacheOptions',
+            '*aio': 'BlockdevAioOptions',
+            '*rerror': 'BlockdevOnError',
+            '*werror': 'BlockdevOnError',
+            '*read-only': 'bool',
+            '*stats-account-invalid': 'bool',
+            '*stats-account-failed': 'bool',
+            '*stats-intervals': ['int'],
+            '*detect-zeroes': 'BlockdevDetectZeroesOptions' },
   'discriminator': 'driver',
   'data': {
       'archipelago':'BlockdevOptionsArchipelago',
@@ -1811,10 +2097,10 @@
 # TODO gluster: Wait for structured options
       'host_cdrom': 'BlockdevOptionsFile',
       'host_device':'BlockdevOptionsFile',
-      'host_floppy':'BlockdevOptionsFile',
       'http':       'BlockdevOptionsFile',
       'https':      'BlockdevOptionsFile',
 # TODO iscsi: Wait for structured options
+      'luks':       'BlockdevOptionsLUKS',
 # TODO nbd: Should take InetSocketAddress for 'host'?
 # TODO nfs: Wait for structured options
       'null-aio':   'BlockdevOptionsNull',
@@ -1855,11 +2141,13 @@
 ##
 # @blockdev-add:
 #
-# Creates a new block device.
+# Creates a new block device. If the @id option is given at the top level, a
+# BlockBackend will be created; otherwise, @node-name is mandatory at the top
+# level and no BlockBackend will be created.
 #
 # This command is still a work in progress.  It doesn't support all
-# block drivers, it lacks a matching blockdev-del, and more.  Stay
-# away from it unless you want to help with its development.
+# block drivers among other things.  Stay away from it unless you want
+# to help with its development.
 #
 # @options: block device options for the new device
 #
@@ -1867,6 +2155,165 @@
 ##
 { 'command': 'blockdev-add', 'data': { 'options': 'BlockdevOptions' } }
 
+##
+# @x-blockdev-del:
+#
+# Deletes a block device that has been added using blockdev-add.
+# The selected device can be either a block backend or a graph node.
+#
+# In the former case the backend will be destroyed, along with its
+# inserted medium if there's any. The command will fail if the backend
+# or its medium are in use.
+#
+# In the latter case the node will be destroyed. The command will fail
+# if the node is attached to a block backend or is otherwise being
+# used.
+#
+# One of @id or @node-name must be specified, but not both.
+#
+# This command is still a work in progress and is considered
+# experimental. Stay away from it unless you want to help with its
+# development.
+#
+# @id: #optional Name of the block backend device to delete.
+#
+# @node-name: #optional Name of the graph node to delete.
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-del', 'data': { '*id': 'str', '*node-name': 'str' } }
+
+##
+# @blockdev-open-tray:
+#
+# Opens a block device's tray. If there is a block driver state tree inserted as
+# a medium, it will become inaccessible to the guest (but it will remain
+# associated to the block device, so closing the tray will make it accessible
+# again).
+#
+# If the tray was already open before, this will be a no-op.
+#
+# Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+# which no such event will be generated, these include:
+# - if the guest has locked the tray, @force is false and the guest does not
+#   respond to the eject request
+# - if the BlockBackend denoted by @device does not have a guest device attached
+#   to it
+# - if the guest device does not have an actual tray
+#
+# @device: block device name
+#
+# @force:  #optional if false (the default), an eject request will be sent to
+#          the guest if it has locked the tray (and the tray will not be opened
+#          immediately); if true, the tray will be opened regardless of whether
+#          it is locked
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-open-tray',
+  'data': { 'device': 'str',
+            '*force': 'bool' } }
+
+##
+# @blockdev-close-tray:
+#
+# Closes a block device's tray. If there is a block driver state tree associated
+# with the block device (which is currently ejected), that tree will be loaded
+# as the medium.
+#
+# If the tray was already closed before, this will be a no-op.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-close-tray',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-remove-medium:
+#
+# Removes a medium (a block driver state tree) from a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device).
+#
+# If the tray is open and there is no medium inserted, this will be a no-op.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-remove-medium',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-insert-medium:
+#
+# Inserts a medium (a block driver state tree) into a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device) and there must be no medium inserted already.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device:    block device name
+#
+# @node-name: name of a node in the block driver state graph
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-insert-medium',
+  'data': { 'device': 'str',
+            'node-name': 'str'} }
+
+
+##
+# @BlockdevChangeReadOnlyMode:
+#
+# Specifies the new read-only mode of a block device subject to the
+# @blockdev-change-medium command.
+#
+# @retain:      Retains the current read-only mode
+#
+# @read-only:   Makes the device read-only
+#
+# @read-write:  Makes the device writable
+#
+# Since: 2.3
+##
+{ 'enum': 'BlockdevChangeReadOnlyMode',
+  'data': ['retain', 'read-only', 'read-write'] }
+
+
+##
+# @blockdev-change-medium:
+#
+# Changes the medium inserted into a block device by ejecting the current medium
+# and loading a new image file which is inserted as the new medium (this command
+# combines blockdev-open-tray, x-blockdev-remove-medium,
+# x-blockdev-insert-medium and blockdev-close-tray).
+#
+# @device:          block device name
+#
+# @filename:        filename of the new image to be loaded
+#
+# @format:          #optional, format to open the new image with (defaults to
+#                   the probed format)
+#
+# @read-only-mode:  #optional, change the read-only mode of the device; defaults
+#                   to 'retain'
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-change-medium',
+  'data': { 'device': 'str',
+            'filename': 'str',
+            '*format': 'str',
+            '*read-only-mode': 'BlockdevChangeReadOnlyMode' } }
+
 
 ##
 # @BlockErrorAction
diff --git a/qemu/qapi/block.json b/qemu/qapi/block.json
index aad645c4a..937337dce 100644
--- a/qemu/qapi/block.json
+++ b/qemu/qapi/block.json
@@ -6,7 +6,7 @@
 { 'include': 'block-core.json' }
 
 ##
-# BiosAtaTranslation:
+# @BiosAtaTranslation:
 #
 # Policy that BIOS should use to interpret cylinder/head/sector
 # addresses.  Note that Bochs BIOS and SeaBIOS will not actually
@@ -40,6 +40,22 @@
   'data': ['auto', 'none', 'lba', 'large', 'rechs']}
 
 ##
+# @FloppyDriveType
+#
+# Type of Floppy drive to be emulated by the Floppy Disk Controller.
+#
+# @144:  1.44MB 3.5" drive
+# @288:  2.88MB 3.5" drive
+# @120:  1.2MB 5.25" drive
+# @none: No drive connected
+# @auto: Automatically determined by inserted media at boot
+#
+# Since: 2.6
+##
+{ 'enum': 'FloppyDriveType',
+  'data': ['144', '288', '120', 'none', 'auto']}
+
+##
 # @BlockdevSnapshotInternal
 #
 # @device: the name of the device to generate the snapshot from
@@ -130,13 +146,15 @@
 # QEMU instance could refer to them as "nbd:HOST:PORT:exportname=NAME".
 #
 # @addr: Address on which to listen.
+# @tls-creds: (optional) ID of the TLS credentials object. Since 2.6
 #
 # Returns: error if the server is already running.
 #
 # Since: 1.3.0
 ##
 { 'command': 'nbd-server-start',
-  'data': { 'addr': 'SocketAddress' } }
+  'data': { 'addr': 'SocketAddress',
+            '*tls-creds': 'str'} }
 
 ##
 # @nbd-server-add:
@@ -178,3 +196,19 @@
 ##
 { 'event': 'DEVICE_TRAY_MOVED',
   'data': { 'device': 'str', 'tray-open': 'bool' } }
+
+##
+# @QuorumOpType
+#
+# An enumeration of the quorum operation types
+#
+# @read: read operation
+#
+# @write: write operation
+#
+# @flush: flush operation
+#
+# Since: 2.6
+##
+{ 'enum': 'QuorumOpType',
+  'data': [ 'read', 'write', 'flush' ] }
diff --git a/qemu/qapi/common.json b/qemu/qapi/common.json
index bad56bf68..9353a7b37 100644
--- a/qemu/qapi/common.json
+++ b/qemu/qapi/common.json
@@ -3,7 +3,7 @@
 # QAPI common definitions
 
 ##
-# @ErrorClass
+# @QapiErrorClass
 #
 # QEMU error classes
 #
@@ -24,7 +24,8 @@
 #
 # Since: 1.2
 ##
-{ 'enum': 'ErrorClass',
+{ 'enum': 'QapiErrorClass',
+  # Keep this in sync with ErrorClass in error.h
   'data': [ 'GenericError', 'CommandNotFound', 'DeviceEncrypted',
             'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap' ] }
 
@@ -114,3 +115,19 @@
 ##
 { 'enum': 'OnOffAuto',
   'data': [ 'auto', 'on', 'off' ] }
+
+##
+# @OnOffSplit
+#
+# An enumeration of three values: on, off, and split
+#
+# @on: Enabled
+#
+# @off: Disabled
+#
+# @split: Mixed
+#
+# Since: 2.6
+##
+{ 'enum': 'OnOffSplit',
+  'data': [ 'on', 'off', 'split' ] }
diff --git a/qemu/qapi/crypto.json b/qemu/qapi/crypto.json
new file mode 100644
index 000000000..760d0c057
--- /dev/null
+++ b/qemu/qapi/crypto.json
@@ -0,0 +1,222 @@
+# -*- Mode: Python -*-
+#
+# QAPI crypto definitions
+
+##
+# QCryptoTLSCredsEndpoint:
+#
+# The type of network endpoint that will be using the credentials.
+# Most types of credential require different setup / structures
+# depending on whether they will be used in a server versus a
+# client.
+#
+# @client: the network endpoint is acting as the client
+#
+# @server: the network endpoint is acting as the server
+#
+# Since: 2.5
+##
+{ 'enum': 'QCryptoTLSCredsEndpoint',
+  'prefix': 'QCRYPTO_TLS_CREDS_ENDPOINT',
+  'data': ['client', 'server']}
+
+
+##
+# QCryptoSecretFormat:
+#
+# The data format that the secret is provided in
+#
+# @raw: raw bytes. When encoded in JSON only valid UTF-8 sequences can be used
+# @base64: arbitrary base64 encoded binary data
+# Since: 2.6
+##
+{ 'enum': 'QCryptoSecretFormat',
+  'prefix': 'QCRYPTO_SECRET_FORMAT',
+  'data': ['raw', 'base64']}
+
+
+##
+# QCryptoHashAlgorithm:
+#
+# The supported algorithms for computing content digests
+#
+# @md5: MD5. Should not be used in any new code, legacy compat only
+# @sha1: SHA-1. Should not be used in any new code, legacy compat only
+# @sha256: SHA-256. Current recommended strong hash.
+# Since: 2.6
+##
+{ 'enum': 'QCryptoHashAlgorithm',
+  'prefix': 'QCRYPTO_HASH_ALG',
+  'data': ['md5', 'sha1', 'sha256']}
+
+
+##
+# QCryptoCipherAlgorithm:
+#
+# The supported algorithms for content encryption ciphers
+#
+# @aes-128: AES with 128 bit / 16 byte keys
+# @aes-192: AES with 192 bit / 24 byte keys
+# @aes-256: AES with 256 bit / 32 byte keys
+# @des-rfb: RFB specific variant of single DES. Do not use except in VNC.
+# @cast5-128: Cast5 with 128 bit / 16 byte keys
+# @serpent-128: Serpent with 128 bit / 16 byte keys
+# @serpent-192: Serpent with 192 bit / 24 byte keys
+# @serpent-256: Serpent with 256 bit / 32 byte keys
+# @twofish-128: Twofish with 128 bit / 16 byte keys
+# @twofish-192: Twofish with 192 bit / 24 byte keys
+# @twofish-256: Twofish with 256 bit / 32 byte keys
+# Since: 2.6
+##
+{ 'enum': 'QCryptoCipherAlgorithm',
+  'prefix': 'QCRYPTO_CIPHER_ALG',
+  'data': ['aes-128', 'aes-192', 'aes-256',
+           'des-rfb',
+           'cast5-128',
+           'serpent-128', 'serpent-192', 'serpent-256',
+           'twofish-128', 'twofish-192', 'twofish-256']}
+
+
+##
+# QCryptoCipherMode:
+#
+# The supported modes for content encryption ciphers
+#
+# @ecb: Electronic Code Book
+# @cbc: Cipher Block Chaining
+# @xts: XEX with tweaked code book and ciphertext stealing
+# Since: 2.6
+##
+{ 'enum': 'QCryptoCipherMode',
+  'prefix': 'QCRYPTO_CIPHER_MODE',
+  'data': ['ecb', 'cbc', 'xts']}
+
+
+##
+# QCryptoIVGenAlgorithm:
+#
+# The supported algorithms for generating initialization
+# vectors for full disk encryption. The 'plain' generator
+# should not be used for disks with sector numbers larger
+# than 2^32, except where compatibility with pre-existing
+# Linux dm-crypt volumes is required.
+#
+# @plain: 64-bit sector number truncated to 32-bits
+# @plain64: 64-bit sector number
+# @essiv: 64-bit sector number encrypted with a hash of the encryption key
+# Since: 2.6
+##
+{ 'enum': 'QCryptoIVGenAlgorithm',
+  'prefix': 'QCRYPTO_IVGEN_ALG',
+  'data': ['plain', 'plain64', 'essiv']}
+
+##
+# QCryptoBlockFormat:
+#
+# The supported full disk encryption formats
+#
+# @qcow: QCow/QCow2 built-in AES-CBC encryption. Use only
+#        for liberating data from old images.
+# @luks: LUKS encryption format. Recommended for new images
+#
+# Since: 2.6
+##
+{ 'enum': 'QCryptoBlockFormat',
+#  'prefix': 'QCRYPTO_BLOCK_FORMAT',
+  'data': ['qcow', 'luks']}
+
+##
+# QCryptoBlockOptionsBase:
+#
+# The common options that apply to all full disk
+# encryption formats
+#
+# @format: the encryption format
+#
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsBase',
+  'data': { 'format': 'QCryptoBlockFormat' }}
+
+##
+# QCryptoBlockOptionsQCow:
+#
+# The options that apply to QCow/QCow2 AES-CBC encryption format
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing the
+#              decryption key. Mandatory except when probing image for
+#              metadata only.
+#
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsQCow',
+  'data': { '*key-secret': 'str' }}
+
+##
+# QCryptoBlockOptionsLUKS:
+#
+# The options that apply to LUKS encryption format
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing the
+#              decryption key. Mandatory except when probing image for
+#              metadata only.
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsLUKS',
+  'data': { '*key-secret': 'str' }}
+
+
+##
+# QCryptoBlockCreateOptionsLUKS:
+#
+# The options that apply to LUKS encryption format initialization
+#
+# @cipher-alg: #optional the cipher algorithm for data encryption
+#              Currently defaults to 'aes'.
+# @cipher-mode: #optional the cipher mode for data encryption
+#               Currently defaults to 'cbc'
+# @ivgen-alg: #optional the initialization vector generator
+#             Currently defaults to 'essiv'
+# @ivgen-hash-alg: #optional the initialization vector generator hash
+#                  Currently defaults to 'sha256'
+# @hash-alg: #optional the master key hash algorithm
+#            Currently defaults to 'sha256'
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockCreateOptionsLUKS',
+  'base': 'QCryptoBlockOptionsLUKS',
+  'data': { '*cipher-alg': 'QCryptoCipherAlgorithm',
+            '*cipher-mode': 'QCryptoCipherMode',
+            '*ivgen-alg': 'QCryptoIVGenAlgorithm',
+            '*ivgen-hash-alg': 'QCryptoHashAlgorithm',
+            '*hash-alg': 'QCryptoHashAlgorithm'}}
+
+
+##
+# QCryptoBlockOpenOptions:
+#
+# The options that are available for all encryption formats
+# when opening an existing volume
+#
+# Since: 2.6
+##
+{ 'union': 'QCryptoBlockOpenOptions',
+  'base': 'QCryptoBlockOptionsBase',
+  'discriminator': 'format',
+  'data': { 'qcow': 'QCryptoBlockOptionsQCow',
+            'luks': 'QCryptoBlockOptionsLUKS' } }
+
+
+##
+# QCryptoBlockCreateOptions:
+#
+# The options that are available for all encryption formats
+# when initializing a new volume
+#
+# Since: 2.6
+##
+{ 'union': 'QCryptoBlockCreateOptions',
+  'base': 'QCryptoBlockOptionsBase',
+  'discriminator': 'format',
+  'data': { 'qcow': 'QCryptoBlockOptionsQCow',
+            'luks': 'QCryptoBlockCreateOptionsLUKS' } }
diff --git a/qemu/qapi/event.json b/qemu/qapi/event.json
index f0cef010f..8642052eb 100644
--- a/qemu/qapi/event.json
+++ b/qemu/qapi/event.json
@@ -255,6 +255,19 @@
   'data': {'status': 'MigrationStatus'}}
 
 ##
+# @MIGRATION_PASS
+#
+# Emitted from the source side of a migration at the start of each pass
+# (when it syncs the dirty bitmap)
+#
+# @pass: An incrementing count (starting at 1 on the first pass)
+#
+# Since: 2.6
+##
+{ 'event': 'MIGRATION_PASS',
+  'data': { 'pass': 'int' } }
+
+##
 # @ACPI_DEVICE_OST
 #
 # Emitted when guest executes ACPI _OST method.
@@ -312,6 +325,8 @@
 #
 # Emitted to report a corruption of a Quorum file
 #
+# @type: quorum operation type (Since 2.6)
+#
 # @error: #optional, error message. Only present on failure. This field
 #         contains a human-readable error message. There are no semantics other
 #         than that the block layer reported an error and clients should not
@@ -326,7 +341,7 @@
 # Since: 2.0
 ##
 { 'event': 'QUORUM_REPORT_BAD',
-  'data': { '*error': 'str', 'node-name': 'str',
+  'data': { 'type': 'QuorumOpType', '*error': 'str', 'node-name': 'str',
             'sector-num': 'int', 'sectors-count': 'int' } }
 
 ##
@@ -356,3 +371,19 @@
 ##
 { 'event': 'MEM_UNPLUG_ERROR',
   'data': { 'device': 'str', 'msg': 'str' } }
+
+##
+# @DUMP_COMPLETED
+#
+# Emitted when background dump has completed
+#
+# @result: DumpQueryResult type described in qapi-schema.json.
+#
+# @error: #optional human-readable error string that provides
+#         hint on why dump failed. Only presents on failure. The
+#         user should not try to interpret the error string.
+#
+# Since: 2.6
+##
+{ 'event': 'DUMP_COMPLETED' ,
+  'data': { 'result': 'DumpQueryResult', '*error': 'str' } }
diff --git a/qemu/qapi/introspect.json b/qemu/qapi/introspect.json
new file mode 100644
index 000000000..3fd81fb54
--- /dev/null
+++ b/qemu/qapi/introspect.json
@@ -0,0 +1,279 @@
+# -*- Mode: Python -*-
+#
+# QAPI/QMP introspection
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+##
+# @query-qmp-schema
+#
+# Command query-qmp-schema exposes the QMP wire ABI as an array of
+# SchemaInfo.  This lets QMP clients figure out what commands and
+# events are available in this QEMU, and their parameters and results.
+#
+# However, the SchemaInfo can't reflect all the rules and restrictions
+# that apply to QMP.  It's interface introspection (figuring out
+# what's there), not interface specification.  The specification is in
+# the QAPI schema.
+#
+# Furthermore, while we strive to keep the QMP wire format
+# backwards-compatible across qemu versions, the introspection output
+# is not guaranteed to have the same stability.  For example, one
+# version of qemu may list an object member as an optional
+# non-variant, while another lists the same member only through the
+# object's variants; or the type of a member may change from a generic
+# string into a specific enum or from one specific type into an
+# alternate that includes the original type alongside something else.
+#
+# Returns: array of @SchemaInfo, where each element describes an
+# entity in the ABI: command, event, type, ...
+#
+# The order of the various SchemaInfo is unspecified; however, all
+# names are guaranteed to be unique (no name will be duplicated with
+# different meta-types).
+#
+# Note: the QAPI schema is also used to help define *internal*
+# interfaces, by defining QAPI types.  These are not part of the QMP
+# wire ABI, and therefore not returned by this command.
+#
+# Since: 2.5
+##
+{ 'command': 'query-qmp-schema',
+  'returns': [ 'SchemaInfo' ],
+  'gen': false }                # just to simplify qmp_query_json()
+
+##
+# @SchemaMetaType
+#
+# This is a @SchemaInfo's meta type, i.e. the kind of entity it
+# describes.
+#
+# @builtin: a predefined type such as 'int' or 'bool'.
+#
+# @enum: an enumeration type
+#
+# @array: an array type
+#
+# @object: an object type (struct or union)
+#
+# @alternate: an alternate type
+#
+# @command: a QMP command
+#
+# @event: a QMP event
+#
+# Since: 2.5
+##
+{ 'enum': 'SchemaMetaType',
+  'data': [ 'builtin', 'enum', 'array', 'object', 'alternate',
+            'command', 'event' ] }
+
+##
+# @SchemaInfo
+#
+# @name: the entity's name, inherited from @base.
+#        Commands and events have the name defined in the QAPI schema.
+#        Unlike command and event names, type names are not part of
+#        the wire ABI.  Consequently, type names are meaningless
+#        strings here, although they are still guaranteed unique
+#        regardless of @meta-type.
+#
+# All references to other SchemaInfo are by name.
+#
+# @meta-type: the entity's meta type, inherited from @base.
+#
+# Additional members depend on the value of @meta-type.
+#
+# Since: 2.5
+##
+{ 'union': 'SchemaInfo',
+  'base': { 'name': 'str', 'meta-type': 'SchemaMetaType' },
+  'discriminator': 'meta-type',
+  'data': {
+      'builtin': 'SchemaInfoBuiltin',
+      'enum': 'SchemaInfoEnum',
+      'array': 'SchemaInfoArray',
+      'object': 'SchemaInfoObject',
+      'alternate': 'SchemaInfoAlternate',
+      'command': 'SchemaInfoCommand',
+      'event': 'SchemaInfoEvent' } }
+
+##
+# @SchemaInfoBuiltin
+#
+# Additional SchemaInfo members for meta-type 'builtin'.
+#
+# @json-type: the JSON type used for this type on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoBuiltin',
+  'data': { 'json-type': 'JSONType' } }
+
+##
+# @JSONType
+#
+# The four primitive and two structured types according to RFC 7159
+# section 1, plus 'int' (split off 'number'), plus the obvious top
+# type 'value'.
+#
+# Since: 2.5
+##
+{ 'enum': 'JSONType',
+  'data': [ 'string', 'number', 'int', 'boolean', 'null',
+            'object', 'array', 'value' ] }
+
+##
+# @SchemaInfoEnum
+#
+# Additional SchemaInfo members for meta-type 'enum'.
+#
+# @values: the enumeration type's values, in no particular order.
+#
+# Values of this type are JSON string on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEnum',
+  'data': { 'values': ['str'] } }
+
+##
+# @SchemaInfoArray
+#
+# Additional SchemaInfo members for meta-type 'array'.
+#
+# @element-type: the array type's element type.
+#
+# Values of this type are JSON array on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoArray',
+  'data': { 'element-type': 'str' } }
+
+##
+# @SchemaInfoObject
+#
+# Additional SchemaInfo members for meta-type 'object'.
+#
+# @members: the object type's (non-variant) members, in no particular order.
+#
+# @tag: #optional the name of the member serving as type tag.
+#       An element of @members with this name must exist.
+#
+# @variants: #optional variant members, i.e. additional members that
+#            depend on the type tag's value.  Present exactly when
+#            @tag is present.  The variants are in no particular order,
+#            and may even differ from the order of the values of the
+#            enum type of the @tag.
+#
+# Values of this type are JSON object on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObject',
+  'data': { 'members': [ 'SchemaInfoObjectMember' ],
+            '*tag': 'str',
+            '*variants': [ 'SchemaInfoObjectVariant' ] } }
+
+##
+# @SchemaInfoObjectMember
+#
+# An object member.
+#
+# @name: the member's name, as defined in the QAPI schema.
+#
+# @type: the name of the member's type.
+#
+# @default: #optional default when used as command parameter.
+#           If absent, the parameter is mandatory.
+#           If present, the value must be null.  The parameter is
+#           optional, and behavior when it's missing is not specified
+#           here.
+#           Future extension: if present and non-null, the parameter
+#           is optional, and defaults to this value.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectMember',
+  'data': { 'name': 'str', 'type': 'str', '*default': 'any' } }
+# @default's type must be null or match @type
+
+##
+# @SchemaInfoObjectVariant
+#
+# The variant members for a value of the type tag.
+#
+# @case: a value of the type tag.
+#
+# @type: the name of the object type that provides the variant members
+#        when the type tag has value @case.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectVariant',
+  'data': { 'case': 'str', 'type': 'str' } }
+
+##
+# @SchemaInfoAlternate
+#
+# Additional SchemaInfo members for meta-type 'alternate'.
+#
+# @members: the alternate type's members, in no particular order.
+#           The members' wire encoding is distinct, see
+#           docs/qapi-code-gen.txt section Alternate types.
+#
+# On the wire, this can be any of the members.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternate',
+  'data': { 'members': [ 'SchemaInfoAlternateMember' ] } }
+
+##
+# @SchemaInfoAlternateMember
+#
+# An alternate member.
+#
+# @type: the name of the member's type.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternateMember',
+  'data': { 'type': 'str' } }
+
+##
+# @SchemaInfoCommand
+#
+# Additional SchemaInfo members for meta-type 'command'.
+#
+# @arg-type: the name of the object type that provides the command's
+#            parameters.
+#
+# @ret-type: the name of the command's result type.
+#
+# TODO @success-response (currently irrelevant, because it's QGA, not QMP)
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoCommand',
+  'data': { 'arg-type': 'str', 'ret-type': 'str' } }
+
+##
+# @SchemaInfoEvent
+#
+# Additional SchemaInfo members for meta-type 'event'.
+#
+# @arg-type: the name of the object type that provides the event's
+#            parameters.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEvent',
+  'data': { 'arg-type': 'str' } }
diff --git a/qemu/qapi/opts-visitor.c b/qemu/qapi/opts-visitor.c
index 7ae33b311..602f2609c 100644
--- a/qemu/qapi/opts-visitor.c
+++ b/qemu/qapi/opts-visitor.c
@@ -1,7 +1,7 @@
 /*
  * Options Visitor
  *
- * Copyright Red Hat, Inc. 2012, 2013
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Laszlo Ersek <lersek@redhat.com>
  *
@@ -10,7 +10,9 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/opts-visitor.h"
 #include "qemu/queue.h"
@@ -89,6 +91,12 @@ struct OptsVisitor
 };
 
 
+static OptsVisitor *to_ov(Visitor *v)
+{
+    return container_of(v, OptsVisitor, visitor);
+}
+
+
 static void
 destroy_list(gpointer list)
 {
@@ -118,10 +126,10 @@ opts_visitor_insert(GHashTable *unprocessed_opts, const QemuOpt *opt)
 
 
 static void
-opts_start_struct(Visitor *v, void **obj, const char *kind,
-                  const char *name, size_t size, Error **errp)
+opts_start_struct(Visitor *v, const char *name, void **obj,
+                  size_t size, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     if (obj) {
@@ -150,17 +158,11 @@ opts_start_struct(Visitor *v, void **obj, const char *kind,
 }
 
 
-static gboolean
-ghr_true(gpointer ign_key, gpointer ign_value, gpointer ign_user_data)
-{
-    return TRUE;
-}
-
-
 static void
 opts_end_struct(Visitor *v, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
+    GHashTableIter iter;
     GQueue *any;
 
     if (--ov->depth > 0) {
@@ -168,8 +170,8 @@ opts_end_struct(Visitor *v, Error **errp)
     }
 
     /* we should have processed all (distinct) QemuOpt instances */
-    any = g_hash_table_find(ov->unprocessed_opts, &ghr_true, NULL);
-    if (any) {
+    g_hash_table_iter_init(&iter, ov->unprocessed_opts);
+    if (g_hash_table_iter_next(&iter, NULL, (void **)&any)) {
         const QemuOpt *first;
 
         first = g_queue_peek_head(any);
@@ -202,7 +204,7 @@ lookup_distinct(const OptsVisitor *ov, const char *name, Error **errp)
 static void
 opts_start_list(Visitor *v, const char *name, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
     /* we can't traverse a list in a list */
     assert(ov->list_mode == LM_NONE);
@@ -214,9 +216,9 @@ opts_start_list(Visitor *v, const char *name, Error **errp)
 
 
 static GenericList *
-opts_next_list(Visitor *v, GenericList **list, Error **errp)
+opts_next_list(Visitor *v, GenericList **list, size_t size)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     GenericList **link;
 
     switch (ov->list_mode) {
@@ -257,15 +259,15 @@ opts_next_list(Visitor *v, GenericList **list, Error **errp)
         abort();
     }
 
-    *link = g_malloc0(sizeof **link);
+    *link = g_malloc0(size);
     return *link;
 }
 
 
 static void
-opts_end_list(Visitor *v, Error **errp)
+opts_end_list(Visitor *v)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
     assert(ov->list_mode == LM_STARTED ||
            ov->list_mode == LM_IN_PROGRESS ||
@@ -305,9 +307,9 @@ processed(OptsVisitor *ov, const char *name)
 
 
 static void
-opts_type_str(Visitor *v, char **obj, const char *name, Error **errp)
+opts_type_str(Visitor *v, const char *name, char **obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     opt = lookup_scalar(ov, name, errp);
@@ -321,9 +323,9 @@ opts_type_str(Visitor *v, char **obj, const char *name, Error **errp)
 
 /* mimics qemu-option.c::parse_option_bool() */
 static void
-opts_type_bool(Visitor *v, bool *obj, const char *name, Error **errp)
+opts_type_bool(Visitor *v, const char *name, bool *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     opt = lookup_scalar(ov, name, errp);
@@ -354,9 +356,9 @@ opts_type_bool(Visitor *v, bool *obj, const char *name, Error **errp)
 
 
 static void
-opts_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp)
+opts_type_int64(Visitor *v, const char *name, int64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     const char *str;
     long long val;
@@ -410,9 +412,9 @@ opts_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp)
 
 
 static void
-opts_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+opts_type_uint64(Visitor *v, const char *name, uint64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     const char *str;
     unsigned long long val;
@@ -462,9 +464,9 @@ opts_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp)
 
 
 static void
-opts_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+opts_type_size(Visitor *v, const char *name, uint64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     int64_t val;
     char *endptr;
@@ -474,8 +476,8 @@ opts_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
         return;
     }
 
-    val = strtosz_suffix(opt->str ? opt->str : "", &endptr,
-                         STRTOSZ_DEFSUFFIX_B);
+    val = qemu_strtosz_suffix(opt->str ? opt->str : "", &endptr,
+                         QEMU_STRTOSZ_DEFSUFFIX_B);
     if (val < 0 || *endptr) {
         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
                    "a size value representible as a non-negative int64");
@@ -488,9 +490,9 @@ opts_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
 
 
 static void
-opts_optional(Visitor *v, bool *present, const char *name, Error **errp)
+opts_optional(Visitor *v, const char *name, bool *present)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
     /* we only support a single mandatory scalar field in a list node */
     assert(ov->list_mode == LM_NONE);
@@ -522,7 +524,7 @@ opts_visitor_new(const QemuOpts *opts)
      */
     ov->visitor.type_enum = &input_type_enum;
 
-    ov->visitor.type_int    = &opts_type_int;
+    ov->visitor.type_int64  = &opts_type_int64;
     ov->visitor.type_uint64 = &opts_type_uint64;
     ov->visitor.type_size   = &opts_type_size;
     ov->visitor.type_bool   = &opts_type_bool;
diff --git a/qemu/qapi/qapi-dealloc-visitor.c b/qemu/qapi/qapi-dealloc-visitor.c
index d7f92c5d6..69221794e 100644
--- a/qemu/qapi/qapi-dealloc-visitor.c
+++ b/qemu/qapi/qapi-dealloc-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Dealloc Visitor
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/dealloc-visitor.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
@@ -28,7 +30,6 @@ struct QapiDeallocVisitor
 {
     Visitor visitor;
     QTAILQ_HEAD(, StackEntry) stack;
-    bool is_list_head;
 };
 
 static QapiDeallocVisitor *to_qov(Visitor *v)
@@ -59,9 +60,8 @@ static void *qapi_dealloc_pop(QapiDeallocVisitor *qov)
     return value;
 }
 
-static void qapi_dealloc_start_struct(Visitor *v, void **obj, const char *kind,
-                                      const char *name, size_t unused,
-                                      Error **errp)
+static void qapi_dealloc_start_struct(Visitor *v, const char *name, void **obj,
+                                      size_t unused, Error **errp)
 {
     QapiDeallocVisitor *qov = to_qov(v);
     qapi_dealloc_push(qov, obj);
@@ -76,16 +76,15 @@ static void qapi_dealloc_end_struct(Visitor *v, Error **errp)
     }
 }
 
-static void qapi_dealloc_start_implicit_struct(Visitor *v,
-                                               void **obj,
-                                               size_t size,
-                                               Error **errp)
+static void qapi_dealloc_start_alternate(Visitor *v, const char *name,
+                                         GenericAlternate **obj, size_t size,
+                                         bool promote_int, Error **errp)
 {
     QapiDeallocVisitor *qov = to_qov(v);
     qapi_dealloc_push(qov, obj);
 }
 
-static void qapi_dealloc_end_implicit_struct(Visitor *v, Error **errp)
+static void qapi_dealloc_end_alternate(Visitor *v)
 {
     QapiDeallocVisitor *qov = to_qov(v);
     void **obj = qapi_dealloc_pop(qov);
@@ -101,7 +100,7 @@ static void qapi_dealloc_start_list(Visitor *v, const char *name, Error **errp)
 }
 
 static GenericList *qapi_dealloc_next_list(Visitor *v, GenericList **listp,
-                                           Error **errp)
+                                           size_t size)
 {
     GenericList *list = *listp;
     QapiDeallocVisitor *qov = to_qov(v);
@@ -121,14 +120,14 @@ static GenericList *qapi_dealloc_next_list(Visitor *v, GenericList **listp,
     return NULL;
 }
 
-static void qapi_dealloc_end_list(Visitor *v, Error **errp)
+static void qapi_dealloc_end_list(Visitor *v)
 {
     QapiDeallocVisitor *qov = to_qov(v);
     void *obj = qapi_dealloc_pop(qov);
     assert(obj == NULL); /* should've been list head tracker with no payload */
 }
 
-static void qapi_dealloc_type_str(Visitor *v, char **obj, const char *name,
+static void qapi_dealloc_type_str(Visitor *v, const char *name, char **obj,
                                   Error **errp)
 {
     if (obj) {
@@ -136,56 +135,37 @@ static void qapi_dealloc_type_str(Visitor *v, char **obj, const char *name,
     }
 }
 
-static void qapi_dealloc_type_int(Visitor *v, int64_t *obj, const char *name,
-                                  Error **errp)
+static void qapi_dealloc_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                    Error **errp)
 {
 }
 
-static void qapi_dealloc_type_bool(Visitor *v, bool *obj, const char *name,
-                                   Error **errp)
+static void qapi_dealloc_type_uint64(Visitor *v, const char *name,
+                                     uint64_t *obj, Error **errp)
 {
 }
 
-static void qapi_dealloc_type_number(Visitor *v, double *obj, const char *name,
-                                     Error **errp)
+static void qapi_dealloc_type_bool(Visitor *v, const char *name, bool *obj,
+                                   Error **errp)
 {
 }
 
-static void qapi_dealloc_type_size(Visitor *v, uint64_t *obj, const char *name,
-                                   Error **errp)
+static void qapi_dealloc_type_number(Visitor *v, const char *name, double *obj,
+                                     Error **errp)
 {
 }
 
-static void qapi_dealloc_type_enum(Visitor *v, int *obj,
-                                   const char * const strings[],
-                                   const char *kind, const char *name,
-                                   Error **errp)
+static void qapi_dealloc_type_anything(Visitor *v, const char *name,
+                                       QObject **obj, Error **errp)
 {
+    if (obj) {
+        qobject_decref(*obj);
+    }
 }
 
-/* If there's no data present, the dealloc visitor has nothing to free.
- * Thus, indicate to visitor code that the subsequent union fields can
- * be skipped. This is not an error condition, since the cleanup of the
- * rest of an object can continue unhindered, so leave errp unset in
- * these cases.
- *
- * NOTE: In cases where we're attempting to deallocate an object that
- * may have missing fields, the field indicating the union type may
- * be missing. In such a case, it's possible we don't have enough
- * information to differentiate data_present == false from a case where
- * data *is* present but happens to be a scalar with a value of 0.
- * This is okay, since in the case of the dealloc visitor there's no
- * work that needs to done in either situation.
- *
- * The current inability in QAPI code to more thoroughly verify a union
- * type in such cases will likely need to be addressed if we wish to
- * implement this interface for other types of visitors in the future,
- * however.
- */
-static bool qapi_dealloc_start_union(Visitor *v, bool data_present,
-                                     Error **errp)
+static void qapi_dealloc_type_enum(Visitor *v, const char *name, int *obj,
+                                   const char * const strings[], Error **errp)
 {
-    return data_present;
 }
 
 Visitor *qapi_dealloc_get_visitor(QapiDeallocVisitor *v)
@@ -206,18 +186,18 @@ QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
 
     v->visitor.start_struct = qapi_dealloc_start_struct;
     v->visitor.end_struct = qapi_dealloc_end_struct;
-    v->visitor.start_implicit_struct = qapi_dealloc_start_implicit_struct;
-    v->visitor.end_implicit_struct = qapi_dealloc_end_implicit_struct;
+    v->visitor.start_alternate = qapi_dealloc_start_alternate;
+    v->visitor.end_alternate = qapi_dealloc_end_alternate;
     v->visitor.start_list = qapi_dealloc_start_list;
     v->visitor.next_list = qapi_dealloc_next_list;
     v->visitor.end_list = qapi_dealloc_end_list;
     v->visitor.type_enum = qapi_dealloc_type_enum;
-    v->visitor.type_int = qapi_dealloc_type_int;
+    v->visitor.type_int64 = qapi_dealloc_type_int64;
+    v->visitor.type_uint64 = qapi_dealloc_type_uint64;
     v->visitor.type_bool = qapi_dealloc_type_bool;
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
-    v->visitor.type_size = qapi_dealloc_type_size;
-    v->visitor.start_union = qapi_dealloc_start_union;
+    v->visitor.type_any = qapi_dealloc_type_anything;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qemu/qapi/qapi-util.c b/qemu/qapi/qapi-util.c
index bcdc94d5a..818730a66 100644
--- a/qemu/qapi/qapi-util.c
+++ b/qemu/qapi/qapi-util.c
@@ -10,8 +10,9 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qapi/error.h"
+#include "qemu-common.h"
 #include "qapi/util.h"
 
 int qapi_enum_parse(const char * const lookup[], const char *buf,
diff --git a/qemu/qapi/qapi-visit-core.c b/qemu/qapi/qapi-visit-core.c
index 5a7c90050..fa680c999 100644
--- a/qemu/qapi/qapi-visit-core.c
+++ b/qemu/qapi/qapi-visit-core.c
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor Classes
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,16 +12,18 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/visitor.h"
 #include "qapi/visitor-impl.h"
 
-void visit_start_struct(Visitor *v, void **obj, const char *kind,
-                        const char *name, size_t size, Error **errp)
+void visit_start_struct(Visitor *v, const char *name, void **obj,
+                        size_t size, Error **errp)
 {
-    v->start_struct(v, obj, kind, name, size, errp);
+    v->start_struct(v, name, obj, size, errp);
 }
 
 void visit_end_struct(Visitor *v, Error **errp)
@@ -28,241 +31,185 @@ void visit_end_struct(Visitor *v, Error **errp)
     v->end_struct(v, errp);
 }
 
-void visit_start_implicit_struct(Visitor *v, void **obj, size_t size,
-                                 Error **errp)
-{
-    if (v->start_implicit_struct) {
-        v->start_implicit_struct(v, obj, size, errp);
-    }
-}
-
-void visit_end_implicit_struct(Visitor *v, Error **errp)
-{
-    if (v->end_implicit_struct) {
-        v->end_implicit_struct(v, errp);
-    }
-}
-
 void visit_start_list(Visitor *v, const char *name, Error **errp)
 {
     v->start_list(v, name, errp);
 }
 
-GenericList *visit_next_list(Visitor *v, GenericList **list, Error **errp)
+GenericList *visit_next_list(Visitor *v, GenericList **list, size_t size)
 {
-    return v->next_list(v, list, errp);
+    assert(list && size >= sizeof(GenericList));
+    return v->next_list(v, list, size);
 }
 
-void visit_end_list(Visitor *v, Error **errp)
+void visit_end_list(Visitor *v)
 {
-    v->end_list(v, errp);
+    v->end_list(v);
 }
 
-bool visit_start_union(Visitor *v, bool data_present, Error **errp)
+void visit_start_alternate(Visitor *v, const char *name,
+                           GenericAlternate **obj, size_t size,
+                           bool promote_int, Error **errp)
 {
-    if (v->start_union) {
-        return v->start_union(v, data_present, errp);
+    assert(obj && size >= sizeof(GenericAlternate));
+    if (v->start_alternate) {
+        v->start_alternate(v, name, obj, size, promote_int, errp);
     }
-    return true;
 }
 
-void visit_end_union(Visitor *v, bool data_present, Error **errp)
+void visit_end_alternate(Visitor *v)
 {
-    if (v->end_union) {
-        v->end_union(v, data_present, errp);
+    if (v->end_alternate) {
+        v->end_alternate(v);
     }
 }
 
-void visit_optional(Visitor *v, bool *present, const char *name,
-                    Error **errp)
+bool visit_optional(Visitor *v, const char *name, bool *present)
 {
     if (v->optional) {
-        v->optional(v, present, name, errp);
+        v->optional(v, name, present);
     }
+    return *present;
 }
 
-void visit_get_next_type(Visitor *v, int *obj, const int *qtypes,
-                         const char *name, Error **errp)
+void visit_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp)
 {
-    if (v->get_next_type) {
-        v->get_next_type(v, obj, qtypes, name, errp);
-    }
-}
-
-void visit_type_enum(Visitor *v, int *obj, const char * const strings[],
-                     const char *kind, const char *name, Error **errp)
-{
-    v->type_enum(v, obj, strings, kind, name, errp);
+    v->type_enum(v, name, obj, strings, errp);
 }
 
-void visit_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp)
+void visit_type_int(Visitor *v, const char *name, int64_t *obj, Error **errp)
 {
-    v->type_int(v, obj, name, errp);
+    v->type_int64(v, name, obj, errp);
 }
 
-void visit_type_uint8(Visitor *v, uint8_t *obj, const char *name, Error **errp)
+static void visit_type_uintN(Visitor *v, uint64_t *obj, const char *name,
+                             uint64_t max, const char *type, Error **errp)
 {
-    int64_t value;
-
-    if (v->type_uint8) {
-        v->type_uint8(v, obj, name, errp);
+    Error *err = NULL;
+    uint64_t value = *obj;
+
+    v->type_uint64(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else if (value > max) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   name ? name : "null", type);
     } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT8_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "uint8_t");
-            return;
-        }
         *obj = value;
     }
 }
 
-void visit_type_uint16(Visitor *v, uint16_t *obj, const char *name, Error **errp)
+void visit_type_uint8(Visitor *v, const char *name, uint8_t *obj,
+                      Error **errp)
 {
-    int64_t value;
-
-    if (v->type_uint16) {
-        v->type_uint16(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT16_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "uint16_t");
-            return;
-        }
-        *obj = value;
-    }
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT8_MAX, "uint8_t", errp);
+    *obj = value;
 }
 
-void visit_type_uint32(Visitor *v, uint32_t *obj, const char *name, Error **errp)
+void visit_type_uint16(Visitor *v, const char *name, uint16_t *obj,
+                       Error **errp)
 {
-    int64_t value;
-
-    if (v->type_uint32) {
-        v->type_uint32(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT32_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "uint32_t");
-            return;
-        }
-        *obj = value;
-    }
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT16_MAX, "uint16_t", errp);
+    *obj = value;
 }
 
-void visit_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+void visit_type_uint32(Visitor *v, const char *name, uint32_t *obj,
+                       Error **errp)
 {
-    int64_t value;
-
-    if (v->type_uint64) {
-        v->type_uint64(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        *obj = value;
-    }
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT32_MAX, "uint32_t", errp);
+    *obj = value;
 }
 
-void visit_type_int8(Visitor *v, int8_t *obj, const char *name, Error **errp)
+void visit_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                       Error **errp)
 {
-    int64_t value;
-
-    if (v->type_int8) {
-        v->type_int8(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT8_MIN || value > INT8_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "int8_t");
-            return;
-        }
-        *obj = value;
-    }
+    v->type_uint64(v, name, obj, errp);
 }
 
-void visit_type_int16(Visitor *v, int16_t *obj, const char *name, Error **errp)
+static void visit_type_intN(Visitor *v, int64_t *obj, const char *name,
+                            int64_t min, int64_t max, const char *type,
+                            Error **errp)
 {
-    int64_t value;
-
-    if (v->type_int16) {
-        v->type_int16(v, obj, name, errp);
+    Error *err = NULL;
+    int64_t value = *obj;
+
+    v->type_int64(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else if (value < min || value > max) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   name ? name : "null", type);
     } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT16_MIN || value > INT16_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "int16_t");
-            return;
-        }
         *obj = value;
     }
 }
 
-void visit_type_int32(Visitor *v, int32_t *obj, const char *name, Error **errp)
+void visit_type_int8(Visitor *v, const char *name, int8_t *obj, Error **errp)
 {
-    int64_t value;
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT8_MIN, INT8_MAX, "int8_t", errp);
+    *obj = value;
+}
 
-    if (v->type_int32) {
-        v->type_int32(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT32_MIN || value > INT32_MAX) {
-            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
-                       name ? name : "null", "int32_t");
-            return;
-        }
-        *obj = value;
-    }
+void visit_type_int16(Visitor *v, const char *name, int16_t *obj,
+                      Error **errp)
+{
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT16_MIN, INT16_MAX, "int16_t", errp);
+    *obj = value;
 }
 
-void visit_type_int64(Visitor *v, int64_t *obj, const char *name, Error **errp)
+void visit_type_int32(Visitor *v, const char *name, int32_t *obj,
+                      Error **errp)
 {
-    if (v->type_int64) {
-        v->type_int64(v, obj, name, errp);
-    } else {
-        v->type_int(v, obj, name, errp);
-    }
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT32_MIN, INT32_MAX, "int32_t", errp);
+    *obj = value;
 }
 
-void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+void visit_type_int64(Visitor *v, const char *name, int64_t *obj,
+                      Error **errp)
 {
-    int64_t value;
+    v->type_int64(v, name, obj, errp);
+}
 
+void visit_type_size(Visitor *v, const char *name, uint64_t *obj,
+                     Error **errp)
+{
     if (v->type_size) {
-        v->type_size(v, obj, name, errp);
-    } else if (v->type_uint64) {
-        v->type_uint64(v, obj, name, errp);
+        v->type_size(v, name, obj, errp);
     } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        *obj = value;
+        v->type_uint64(v, name, obj, errp);
     }
 }
 
-void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp)
+void visit_type_bool(Visitor *v, const char *name, bool *obj, Error **errp)
 {
-    v->type_bool(v, obj, name, errp);
+    v->type_bool(v, name, obj, errp);
 }
 
-void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp)
+void visit_type_str(Visitor *v, const char *name, char **obj, Error **errp)
 {
-    v->type_str(v, obj, name, errp);
+    v->type_str(v, name, obj, errp);
 }
 
-void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp)
+void visit_type_number(Visitor *v, const char *name, double *obj,
+                       Error **errp)
 {
-    v->type_number(v, obj, name, errp);
+    v->type_number(v, name, obj, errp);
 }
 
-void output_type_enum(Visitor *v, int *obj, const char * const strings[],
-                      const char *kind, const char *name,
-                      Error **errp)
+void visit_type_any(Visitor *v, const char *name, QObject **obj, Error **errp)
+{
+    v->type_any(v, name, obj, errp);
+}
+
+void output_type_enum(Visitor *v, const char *name, int *obj,
+                      const char *const strings[], Error **errp)
 {
     int i = 0;
     int value = *obj;
@@ -276,12 +223,11 @@ void output_type_enum(Visitor *v, int *obj, const char * const strings[],
     }
 
     enum_str = (char *)strings[value];
-    visit_type_str(v, &enum_str, name, errp);
+    visit_type_str(v, name, &enum_str, errp);
 }
 
-void input_type_enum(Visitor *v, int *obj, const char * const strings[],
-                     const char *kind, const char *name,
-                     Error **errp)
+void input_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp)
 {
     Error *local_err = NULL;
     int64_t value = 0;
@@ -289,7 +235,7 @@ void input_type_enum(Visitor *v, int *obj, const char * const strings[],
 
     assert(strings);
 
-    visit_type_str(v, &enum_str, name, &local_err);
+    visit_type_str(v, name, &enum_str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
diff --git a/qemu/qapi/qmp-dispatch.c b/qemu/qapi/qmp-dispatch.c
index 7bcc86080..510a1aead 100644
--- a/qemu/qapi/qmp-dispatch.c
+++ b/qemu/qapi/qmp-dispatch.c
@@ -11,11 +11,12 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qapi/qmp/types.h"
 #include "qapi/qmp/dispatch.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi-types.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 
 static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp)
@@ -114,7 +115,7 @@ static QObject *do_qmp_dispatch(QObject *request, Error **errp)
 QObject *qmp_build_error_object(Error *err)
 {
     return qobject_from_jsonf("{ 'class': %s, 'desc': %s }",
-                              ErrorClass_lookup[error_get_class(err)],
+                              QapiErrorClass_lookup[error_get_class(err)],
                               error_get_pretty(err));
 }
 
diff --git a/qemu/qapi/qmp-event.c b/qemu/qapi/qmp-event.c
index 0d1ce0bd1..8bba165bf 100644
--- a/qemu/qapi/qmp-event.c
+++ b/qemu/qapi/qmp-event.c
@@ -11,21 +11,13 @@
  *
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qapi/qmp-event.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
 
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
-
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
 static QMPEventFuncEmit qmp_emit;
 
 void qmp_event_set_func_emit(QMPEventFuncEmit emit)
diff --git a/qemu/qapi/qmp-input-visitor.c b/qemu/qapi/qmp-input-visitor.c
index e97b8a428..7cd1b777a 100644
--- a/qemu/qapi/qmp-input-visitor.c
+++ b/qemu/qapi/qmp-input-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Input Visitor
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,6 +12,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/visitor-impl.h"
 #include "qemu/queue.h"
@@ -88,12 +91,6 @@ static void qmp_input_push(QmpInputVisitor *qiv, QObject *obj, Error **errp)
     qiv->nb_stack++;
 }
 
-/** Only for qmp_input_pop. */
-static gboolean always_true(gpointer key, gpointer val, gpointer user_pkey)
-{
-    *(const char **)user_pkey = (const char *)key;
-    return TRUE;
-}
 
 static void qmp_input_pop(QmpInputVisitor *qiv, Error **errp)
 {
@@ -102,9 +99,11 @@ static void qmp_input_pop(QmpInputVisitor *qiv, Error **errp)
     if (qiv->strict) {
         GHashTable * const top_ht = qiv->stack[qiv->nb_stack - 1].h;
         if (top_ht) {
-            if (g_hash_table_size(top_ht)) {
-                const char *key;
-                g_hash_table_find(top_ht, always_true, &key);
+            GHashTableIter iter;
+            const char *key;
+
+            g_hash_table_iter_init(&iter, top_ht);
+            if (g_hash_table_iter_next(&iter, (void **)&key, NULL)) {
                 error_setg(errp, QERR_QMP_EXTRA_MEMBER, key);
             }
             g_hash_table_unref(top_ht);
@@ -114,8 +113,8 @@ static void qmp_input_pop(QmpInputVisitor *qiv, Error **errp)
     qiv->nb_stack--;
 }
 
-static void qmp_input_start_struct(Visitor *v, void **obj, const char *kind,
-                                   const char *name, size_t size, Error **errp)
+static void qmp_input_start_struct(Visitor *v, const char *name, void **obj,
+                                   size_t size, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
@@ -145,18 +144,6 @@ static void qmp_input_end_struct(Visitor *v, Error **errp)
     qmp_input_pop(qiv, errp);
 }
 
-static void qmp_input_start_implicit_struct(Visitor *v, void **obj,
-                                            size_t size, Error **errp)
-{
-    if (obj) {
-        *obj = g_malloc0(size);
-    }
-}
-
-static void qmp_input_end_implicit_struct(Visitor *v, Error **errp)
-{
-}
-
 static void qmp_input_start_list(Visitor *v, const char *name, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
@@ -172,7 +159,7 @@ static void qmp_input_start_list(Visitor *v, const char *name, Error **errp)
 }
 
 static GenericList *qmp_input_next_list(Visitor *v, GenericList **list,
-                                        Error **errp)
+                                        size_t size)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     GenericList *entry;
@@ -191,7 +178,7 @@ static GenericList *qmp_input_next_list(Visitor *v, GenericList **list,
         return NULL;
     }
 
-    entry = g_malloc0(sizeof(*entry));
+    entry = g_malloc0(size);
     if (first) {
         *list = entry;
     } else {
@@ -201,97 +188,132 @@ static GenericList *qmp_input_next_list(Visitor *v, GenericList **list,
     return entry;
 }
 
-static void qmp_input_end_list(Visitor *v, Error **errp)
+static void qmp_input_end_list(Visitor *v)
 {
     QmpInputVisitor *qiv = to_qiv(v);
 
-    qmp_input_pop(qiv, errp);
+    qmp_input_pop(qiv, &error_abort);
 }
 
-static void qmp_input_get_next_type(Visitor *v, int *kind, const int *qobjects,
-                                    const char *name, Error **errp)
+static void qmp_input_start_alternate(Visitor *v, const char *name,
+                                      GenericAlternate **obj, size_t size,
+                                      bool promote_int, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, false);
 
     if (!qobj) {
+        *obj = NULL;
         error_setg(errp, QERR_MISSING_PARAMETER, name ? name : "null");
         return;
     }
-    *kind = qobjects[qobject_type(qobj)];
+    *obj = g_malloc0(size);
+    (*obj)->type = qobject_type(qobj);
+    if (promote_int && (*obj)->type == QTYPE_QINT) {
+        (*obj)->type = QTYPE_QFLOAT;
+    }
 }
 
-static void qmp_input_type_int(Visitor *v, int64_t *obj, const char *name,
-                               Error **errp)
+static void qmp_input_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                 Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint = qobject_to_qint(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QINT) {
+    if (!qint) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "integer");
         return;
     }
 
-    *obj = qint_get_int(qobject_to_qint(qobj));
+    *obj = qint_get_int(qint);
 }
 
-static void qmp_input_type_bool(Visitor *v, bool *obj, const char *name,
+static void qmp_input_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                                  Error **errp)
+{
+    /* FIXME: qobject_to_qint mishandles values over INT64_MAX */
+    QmpInputVisitor *qiv = to_qiv(v);
+    QInt *qint = qobject_to_qint(qmp_input_get_object(qiv, name, true));
+
+    if (!qint) {
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "integer");
+        return;
+    }
+
+    *obj = qint_get_int(qint);
+}
+
+static void qmp_input_type_bool(Visitor *v, const char *name, bool *obj,
                                 Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QBool *qbool = qobject_to_qbool(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QBOOL) {
+    if (!qbool) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "boolean");
         return;
     }
 
-    *obj = qbool_get_bool(qobject_to_qbool(qobj));
+    *obj = qbool_get_bool(qbool);
 }
 
-static void qmp_input_type_str(Visitor *v, char **obj, const char *name,
+static void qmp_input_type_str(Visitor *v, const char *name, char **obj,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QString *qstr = qobject_to_qstring(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QSTRING) {
+    if (!qstr) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
                    "string");
         return;
     }
 
-    *obj = g_strdup(qstring_get_str(qobject_to_qstring(qobj)));
+    *obj = g_strdup(qstring_get_str(qstr));
 }
 
-static void qmp_input_type_number(Visitor *v, double *obj, const char *name,
+static void qmp_input_type_number(Visitor *v, const char *name, double *obj,
                                   Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint;
+    QFloat *qfloat;
 
-    if (!qobj || (qobject_type(qobj) != QTYPE_QFLOAT &&
-        qobject_type(qobj) != QTYPE_QINT)) {
-        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                   "number");
+    qint = qobject_to_qint(qobj);
+    if (qint) {
+        *obj = qint_get_int(qobject_to_qint(qobj));
         return;
     }
 
-    if (qobject_type(qobj) == QTYPE_QINT) {
-        *obj = qint_get_int(qobject_to_qint(qobj));
-    } else {
+    qfloat = qobject_to_qfloat(qobj);
+    if (qfloat) {
         *obj = qfloat_get_double(qobject_to_qfloat(qobj));
+        return;
     }
+
+    error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+               "number");
 }
 
-static void qmp_input_optional(Visitor *v, bool *present, const char *name,
+static void qmp_input_type_any(Visitor *v, const char *name, QObject **obj,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
 
+    qobject_incref(qobj);
+    *obj = qobj;
+}
+
+static void qmp_input_optional(Visitor *v, const char *name, bool *present)
+{
+    QmpInputVisitor *qiv = to_qiv(v);
+    QObject *qobj = qmp_input_get_object(qiv, name, true);
+
     if (!qobj) {
         *present = false;
         return;
@@ -319,18 +341,18 @@ QmpInputVisitor *qmp_input_visitor_new(QObject *obj)
 
     v->visitor.start_struct = qmp_input_start_struct;
     v->visitor.end_struct = qmp_input_end_struct;
-    v->visitor.start_implicit_struct = qmp_input_start_implicit_struct;
-    v->visitor.end_implicit_struct = qmp_input_end_implicit_struct;
     v->visitor.start_list = qmp_input_start_list;
     v->visitor.next_list = qmp_input_next_list;
     v->visitor.end_list = qmp_input_end_list;
+    v->visitor.start_alternate = qmp_input_start_alternate;
     v->visitor.type_enum = input_type_enum;
-    v->visitor.type_int = qmp_input_type_int;
+    v->visitor.type_int64 = qmp_input_type_int64;
+    v->visitor.type_uint64 = qmp_input_type_uint64;
     v->visitor.type_bool = qmp_input_type_bool;
     v->visitor.type_str = qmp_input_type_str;
     v->visitor.type_number = qmp_input_type_number;
+    v->visitor.type_any = qmp_input_type_any;
     v->visitor.optional = qmp_input_optional;
-    v->visitor.get_next_type = qmp_input_get_next_type;
 
     qmp_input_push(v, obj, NULL);
     qobject_incref(obj);
diff --git a/qemu/qapi/qmp-output-visitor.c b/qemu/qapi/qmp-output-visitor.c
index efc19d584..d44c67631 100644
--- a/qemu/qapi/qmp-output-visitor.c
+++ b/qemu/qapi/qmp-output-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI/QMP Command Registry
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/visitor-impl.h"
 #include "qemu/queue.h"
@@ -29,7 +31,8 @@ typedef QTAILQ_HEAD(QStack, QStackEntry) QStack;
 struct QmpOutputVisitor
 {
     Visitor visitor;
-    QStack stack;
+    QStack stack; /* Stack of containers that haven't yet been finished */
+    QObject *root; /* Root of the output visit */
 };
 
 #define qmp_output_add(qov, name, value) \
@@ -41,10 +44,13 @@ static QmpOutputVisitor *to_qov(Visitor *v)
     return container_of(v, QmpOutputVisitor, visitor);
 }
 
+/* Push @value onto the stack of current QObjects being built */
 static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value)
 {
     QStackEntry *e = g_malloc0(sizeof(*e));
 
+    assert(qov->root);
+    assert(value);
     e->value = value;
     if (qobject_type(e->value) == QTYPE_QLIST) {
         e->is_list_head = true;
@@ -52,63 +58,51 @@ static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value)
     QTAILQ_INSERT_HEAD(&qov->stack, e, node);
 }
 
+/* Pop a value off the stack of QObjects being built, and return it. */
 static QObject *qmp_output_pop(QmpOutputVisitor *qov)
 {
     QStackEntry *e = QTAILQ_FIRST(&qov->stack);
     QObject *value;
+
+    assert(e);
     QTAILQ_REMOVE(&qov->stack, e, node);
     value = e->value;
+    assert(value);
     g_free(e);
     return value;
 }
 
-static QObject *qmp_output_first(QmpOutputVisitor *qov)
-{
-    QStackEntry *e = QTAILQ_LAST(&qov->stack, QStack);
-
-    /* FIXME - find a better way to deal with NULL values */
-    if (!e) {
-        return NULL;
-    }
-
-    return e->value;
-}
-
-static QObject *qmp_output_last(QmpOutputVisitor *qov)
-{
-    QStackEntry *e = QTAILQ_FIRST(&qov->stack);
-    return e->value;
-}
-
+/* Add @value to the current QObject being built.
+ * If the stack is visiting a dictionary or list, @value is now owned
+ * by that container. Otherwise, @value is now the root.  */
 static void qmp_output_add_obj(QmpOutputVisitor *qov, const char *name,
                                QObject *value)
 {
-    QObject *cur;
-
-    if (QTAILQ_EMPTY(&qov->stack)) {
-        qmp_output_push_obj(qov, value);
-        return;
-    }
+    QStackEntry *e = QTAILQ_FIRST(&qov->stack);
+    QObject *cur = e ? e->value : NULL;
 
-    cur = qmp_output_last(qov);
-
-    switch (qobject_type(cur)) {
-    case QTYPE_QDICT:
-        qdict_put_obj(qobject_to_qdict(cur), name, value);
-        break;
-    case QTYPE_QLIST:
-        qlist_append_obj(qobject_to_qlist(cur), value);
-        break;
-    default:
-        qobject_decref(qmp_output_pop(qov));
-        qmp_output_push_obj(qov, value);
-        break;
+    if (!cur) {
+        /* FIXME we should require the user to reset the visitor, rather
+         * than throwing away the previous root */
+        qobject_decref(qov->root);
+        qov->root = value;
+    } else {
+        switch (qobject_type(cur)) {
+        case QTYPE_QDICT:
+            assert(name);
+            qdict_put_obj(qobject_to_qdict(cur), name, value);
+            break;
+        case QTYPE_QLIST:
+            qlist_append_obj(qobject_to_qlist(cur), value);
+            break;
+        default:
+            g_assert_not_reached();
+        }
     }
 }
 
-static void qmp_output_start_struct(Visitor *v, void **obj, const char *kind,
-                                    const char *name, size_t unused,
-                                    Error **errp)
+static void qmp_output_start_struct(Visitor *v, const char *name, void **obj,
+                                    size_t unused, Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     QDict *dict = qdict_new();
@@ -133,7 +127,7 @@ static void qmp_output_start_list(Visitor *v, const char *name, Error **errp)
 }
 
 static GenericList *qmp_output_next_list(Visitor *v, GenericList **listp,
-                                         Error **errp)
+                                         size_t size)
 {
     GenericList *list = *listp;
     QmpOutputVisitor *qov = to_qov(v);
@@ -148,27 +142,35 @@ static GenericList *qmp_output_next_list(Visitor *v, GenericList **listp,
     return list ? list->next : NULL;
 }
 
-static void qmp_output_end_list(Visitor *v, Error **errp)
+static void qmp_output_end_list(Visitor *v)
 {
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_pop(qov);
 }
 
-static void qmp_output_type_int(Visitor *v, int64_t *obj, const char *name,
-                                Error **errp)
+static void qmp_output_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                  Error **errp)
+{
+    QmpOutputVisitor *qov = to_qov(v);
+    qmp_output_add(qov, name, qint_from_int(*obj));
+}
+
+static void qmp_output_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                                   Error **errp)
 {
+    /* FIXME: QMP outputs values larger than INT64_MAX as negative */
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_add(qov, name, qint_from_int(*obj));
 }
 
-static void qmp_output_type_bool(Visitor *v, bool *obj, const char *name,
+static void qmp_output_type_bool(Visitor *v, const char *name, bool *obj,
                                  Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_add(qov, name, qbool_from_bool(*obj));
 }
 
-static void qmp_output_type_str(Visitor *v, char **obj, const char *name,
+static void qmp_output_type_str(Visitor *v, const char *name, char **obj,
                                 Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
@@ -179,18 +181,31 @@ static void qmp_output_type_str(Visitor *v, char **obj, const char *name,
     }
 }
 
-static void qmp_output_type_number(Visitor *v, double *obj, const char *name,
+static void qmp_output_type_number(Visitor *v, const char *name, double *obj,
                                    Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_add(qov, name, qfloat_from_double(*obj));
 }
 
+static void qmp_output_type_any(Visitor *v, const char *name, QObject **obj,
+                                Error **errp)
+{
+    QmpOutputVisitor *qov = to_qov(v);
+    qobject_incref(*obj);
+    qmp_output_add_obj(qov, name, *obj);
+}
+
+/* Finish building, and return the root object. Will not be NULL. */
 QObject *qmp_output_get_qobject(QmpOutputVisitor *qov)
 {
-    QObject *obj = qmp_output_first(qov);
+    /* FIXME: we should require that a visit occurred, and that it is
+     * complete (no starts without a matching end) */
+    QObject *obj = qov->root;
     if (obj) {
         qobject_incref(obj);
+    } else {
+        obj = qnull();
     }
     return obj;
 }
@@ -204,16 +219,12 @@ void qmp_output_visitor_cleanup(QmpOutputVisitor *v)
 {
     QStackEntry *e, *tmp;
 
-    /* The bottom QStackEntry, if any, owns the root QObject. See the
-     * qmp_output_push_obj() invocations in qmp_output_add_obj(). */
-    QObject *root = QTAILQ_EMPTY(&v->stack) ? NULL : qmp_output_first(v);
-
     QTAILQ_FOREACH_SAFE(e, &v->stack, node, tmp) {
         QTAILQ_REMOVE(&v->stack, e, node);
         g_free(e);
     }
 
-    qobject_decref(root);
+    qobject_decref(v->root);
     g_free(v);
 }
 
@@ -229,10 +240,12 @@ QmpOutputVisitor *qmp_output_visitor_new(void)
     v->visitor.next_list = qmp_output_next_list;
     v->visitor.end_list = qmp_output_end_list;
     v->visitor.type_enum = output_type_enum;
-    v->visitor.type_int = qmp_output_type_int;
+    v->visitor.type_int64 = qmp_output_type_int64;
+    v->visitor.type_uint64 = qmp_output_type_uint64;
     v->visitor.type_bool = qmp_output_type_bool;
     v->visitor.type_str = qmp_output_type_str;
     v->visitor.type_number = qmp_output_type_number;
+    v->visitor.type_any = qmp_output_type_any;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qemu/qapi/qmp-registry.c b/qemu/qapi/qmp-registry.c
index 3e4498a3f..4ebfbccd4 100644
--- a/qemu/qapi/qmp-registry.c
+++ b/qemu/qapi/qmp-registry.c
@@ -12,8 +12,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "qapi/qmp/dispatch.h"
 
 static QTAILQ_HEAD(QmpCommandList, QmpCommand) qmp_commands =
diff --git a/qemu/qapi/string-input-visitor.c b/qemu/qapi/string-input-visitor.c
index bbd6a5456..5ea2d77b5 100644
--- a/qemu/qapi/string-input-visitor.c
+++ b/qemu/qapi/string-input-visitor.c
@@ -1,7 +1,7 @@
 /*
  * String parsing visitor
  *
- * Copyright Red Hat, Inc. 2012
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Paolo Bonzini <pbonzini@redhat.com>
  *
@@ -10,6 +10,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qapi/string-input-visitor.h"
 #include "qapi/visitor-impl.h"
@@ -32,6 +34,11 @@ struct StringInputVisitor
     const char *string;
 };
 
+static StringInputVisitor *to_siv(Visitor *v)
+{
+    return container_of(v, StringInputVisitor, visitor);
+}
+
 static void free_range(void *range, void *dummy)
 {
     g_free(range);
@@ -120,7 +127,7 @@ error:
 static void
 start_list(Visitor *v, const char *name, Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     parse_str(siv, errp);
 
@@ -133,10 +140,9 @@ start_list(Visitor *v, const char *name, Error **errp)
     }
 }
 
-static GenericList *
-next_list(Visitor *v, GenericList **list, Error **errp)
+static GenericList *next_list(Visitor *v, GenericList **list, size_t size)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     GenericList **link;
     Range *r;
 
@@ -168,21 +174,20 @@ next_list(Visitor *v, GenericList **list, Error **errp)
         link = &(*list)->next;
     }
 
-    *link = g_malloc0(sizeof **link);
+    *link = g_malloc0(size);
     return *link;
 }
 
-static void
-end_list(Visitor *v, Error **errp)
+static void end_list(Visitor *v)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     siv->head = true;
 }
 
-static void parse_type_int(Visitor *v, int64_t *obj, const char *name,
-                           Error **errp)
+static void parse_type_int64(Visitor *v, const char *name, int64_t *obj,
+                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (!siv->string) {
         error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
@@ -217,14 +222,28 @@ static void parse_type_int(Visitor *v, int64_t *obj, const char *name,
     return;
 
 error:
-    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name,
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
                "an int64 value or range");
 }
 
-static void parse_type_size(Visitor *v, uint64_t *obj, const char *name,
+static void parse_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                              Error **errp)
+{
+    /* FIXME: parse_type_int64 mishandles values over INT64_MAX */
+    int64_t i;
+    Error *err = NULL;
+    parse_type_int64(v, name, &i, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else {
+        *obj = i;
+    }
+}
+
+static void parse_type_size(Visitor *v, const char *name, uint64_t *obj,
                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     Error *err = NULL;
     uint64_t val;
 
@@ -243,10 +262,10 @@ static void parse_type_size(Visitor *v, uint64_t *obj, const char *name,
     *obj = val;
 }
 
-static void parse_type_bool(Visitor *v, bool *obj, const char *name,
+static void parse_type_bool(Visitor *v, const char *name, bool *obj,
                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (siv->string) {
         if (!strcasecmp(siv->string, "on") ||
@@ -267,10 +286,10 @@ static void parse_type_bool(Visitor *v, bool *obj, const char *name,
                "boolean");
 }
 
-static void parse_type_str(Visitor *v, char **obj, const char *name,
+static void parse_type_str(Visitor *v, const char *name, char **obj,
                            Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     if (siv->string) {
         *obj = g_strdup(siv->string);
     } else {
@@ -279,10 +298,10 @@ static void parse_type_str(Visitor *v, char **obj, const char *name,
     }
 }
 
-static void parse_type_number(Visitor *v, double *obj, const char *name,
+static void parse_type_number(Visitor *v, const char *name, double *obj,
                               Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     char *endp = (char *) siv->string;
     double val;
 
@@ -299,10 +318,9 @@ static void parse_type_number(Visitor *v, double *obj, const char *name,
     *obj = val;
 }
 
-static void parse_optional(Visitor *v, bool *present, const char *name,
-                           Error **errp)
+static void parse_optional(Visitor *v, const char *name, bool *present)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (!siv->string) {
         *present = false;
@@ -331,7 +349,8 @@ StringInputVisitor *string_input_visitor_new(const char *str)
     v = g_malloc0(sizeof(*v));
 
     v->visitor.type_enum = input_type_enum;
-    v->visitor.type_int = parse_type_int;
+    v->visitor.type_int64 = parse_type_int64;
+    v->visitor.type_uint64 = parse_type_uint64;
     v->visitor.type_size = parse_type_size;
     v->visitor.type_bool = parse_type_bool;
     v->visitor.type_str = parse_type_str;
diff --git a/qemu/qapi/string-output-visitor.c b/qemu/qapi/string-output-visitor.c
index b86ce2cd0..c2e5c5b92 100644
--- a/qemu/qapi/string-output-visitor.c
+++ b/qemu/qapi/string-output-visitor.c
@@ -1,7 +1,7 @@
 /*
  * String printing Visitor
  *
- * Copyright Red Hat, Inc. 2012
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Paolo Bonzini <pbonzini@redhat.com>
  *
@@ -10,6 +10,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/string-output-visitor.h"
 #include "qapi/visitor-impl.h"
@@ -66,6 +67,11 @@ struct StringOutputVisitor
     GList *ranges;
 };
 
+static StringOutputVisitor *to_sov(Visitor *v)
+{
+    return container_of(v, StringOutputVisitor, visitor);
+}
+
 static void string_output_set(StringOutputVisitor *sov, char *string)
 {
     if (sov->string) {
@@ -116,10 +122,10 @@ static void format_string(StringOutputVisitor *sov, Range *r, bool next,
     }
 }
 
-static void print_type_int(Visitor *v, int64_t *obj, const char *name,
-                           Error **errp)
+static void print_type_int64(Visitor *v, const char *name, int64_t *obj,
+                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     GList *l;
 
     switch (sov->list_mode) {
@@ -192,10 +198,18 @@ static void print_type_int(Visitor *v, int64_t *obj, const char *name,
     }
 }
 
-static void print_type_size(Visitor *v, uint64_t *obj, const char *name,
-                           Error **errp)
+static void print_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    /* FIXME: print_type_int64 mishandles values over INT64_MAX */
+    int64_t i = *obj;
+    print_type_int64(v, name, &i, errp);
+}
+
+static void print_type_size(Visitor *v, const char *name, uint64_t *obj,
+                            Error **errp)
+{
+    StringOutputVisitor *sov = to_sov(v);
     static const char suffixes[] = { 'B', 'K', 'M', 'G', 'T', 'P', 'E' };
     uint64_t div, val;
     char *out;
@@ -223,17 +237,17 @@ static void print_type_size(Visitor *v, uint64_t *obj, const char *name,
     string_output_set(sov, out);
 }
 
-static void print_type_bool(Visitor *v, bool *obj, const char *name,
+static void print_type_bool(Visitor *v, const char *name, bool *obj,
                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     string_output_set(sov, g_strdup(*obj ? "true" : "false"));
 }
 
-static void print_type_str(Visitor *v, char **obj, const char *name,
+static void print_type_str(Visitor *v, const char *name, char **obj,
                            Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     char *out;
 
     if (sov->human) {
@@ -244,17 +258,17 @@ static void print_type_str(Visitor *v, char **obj, const char *name,
     string_output_set(sov, out);
 }
 
-static void print_type_number(Visitor *v, double *obj, const char *name,
+static void print_type_number(Visitor *v, const char *name, double *obj,
                               Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     string_output_set(sov, g_strdup_printf("%f", *obj));
 }
 
 static void
 start_list(Visitor *v, const char *name, Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
 
     /* we can't traverse a list in a list */
     assert(sov->list_mode == LM_NONE);
@@ -262,10 +276,9 @@ start_list(Visitor *v, const char *name, Error **errp)
     sov->head = true;
 }
 
-static GenericList *
-next_list(Visitor *v, GenericList **list, Error **errp)
+static GenericList *next_list(Visitor *v, GenericList **list, size_t size)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     GenericList *ret = NULL;
     if (*list) {
         if (sov->head) {
@@ -289,10 +302,9 @@ next_list(Visitor *v, GenericList **list, Error **errp)
     return ret;
 }
 
-static void
-end_list(Visitor *v, Error **errp)
+static void end_list(Visitor *v)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
 
     assert(sov->list_mode == LM_STARTED ||
            sov->list_mode == LM_END ||
@@ -340,7 +352,8 @@ StringOutputVisitor *string_output_visitor_new(bool human)
     v->string = g_string_new(NULL);
     v->human = human;
     v->visitor.type_enum = output_type_enum;
-    v->visitor.type_int = print_type_int;
+    v->visitor.type_int64 = print_type_int64;
+    v->visitor.type_uint64 = print_type_uint64;
     v->visitor.type_size = print_type_size;
     v->visitor.type_bool = print_type_bool;
     v->visitor.type_str = print_type_str;
diff --git a/qemu/qdev-monitor.c b/qemu/qdev-monitor.c
index f9e2d6258..e19617fa8 100644
--- a/qemu/qdev-monitor.c
+++ b/qemu/qdev-monitor.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "hw/sysbus.h"
 #include "monitor/monitor.h"
@@ -26,6 +27,7 @@
 #include "qapi/qmp/qerror.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qemu/help_option.h"
 
 /*
  * Aliases were a bad idea from the start.  Let's keep them
@@ -38,18 +40,39 @@ typedef struct QDevAlias
     uint32_t arch_mask;
 } QDevAlias;
 
+/* Please keep this table sorted by typename. */
 static const QDevAlias qdev_alias_table[] = {
-    { "virtio-blk-pci", "virtio-blk", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-net-pci", "virtio-net", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-serial-pci", "virtio-serial", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "e1000", "e1000-82540em" },
+    { "ich9-ahci", "ahci" },
+    { "kvm-pci-assign", "pci-assign" },
+    { "lsi53c895a", "lsi" },
+    { "virtio-9p-ccw", "virtio-9p", QEMU_ARCH_S390X },
+    { "virtio-9p-pci", "virtio-9p", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-balloon-ccw", "virtio-balloon", QEMU_ARCH_S390X },
     { "virtio-balloon-pci", "virtio-balloon",
             QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
     { "virtio-blk-ccw", "virtio-blk", QEMU_ARCH_S390X },
+    { "virtio-blk-pci", "virtio-blk", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-gpu-ccw", "virtio-gpu", QEMU_ARCH_S390X },
+    { "virtio-gpu-pci", "virtio-gpu", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-input-host-ccw", "virtio-input-host", QEMU_ARCH_S390X },
+    { "virtio-input-host-pci", "virtio-input-host",
+            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-keyboard-ccw", "virtio-keyboard", QEMU_ARCH_S390X },
+    { "virtio-keyboard-pci", "virtio-keyboard",
+            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-mouse-ccw", "virtio-mouse", QEMU_ARCH_S390X },
+    { "virtio-mouse-pci", "virtio-mouse", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
     { "virtio-net-ccw", "virtio-net", QEMU_ARCH_S390X },
+    { "virtio-net-pci", "virtio-net", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-rng-ccw", "virtio-rng", QEMU_ARCH_S390X },
+    { "virtio-rng-pci", "virtio-rng", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-scsi-ccw", "virtio-scsi", QEMU_ARCH_S390X },
+    { "virtio-scsi-pci", "virtio-scsi", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
     { "virtio-serial-ccw", "virtio-serial", QEMU_ARCH_S390X },
-    { "lsi53c895a", "lsi" },
-    { "ich9-ahci", "ahci" },
-    { "kvm-pci-assign", "pci-assign" },
+    { "virtio-serial-pci", "virtio-serial", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-tablet-ccw", "virtio-tablet", QEMU_ARCH_S390X },
+    { "virtio-tablet-pci", "virtio-tablet", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
     { }
 };
 
@@ -186,6 +209,7 @@ static DeviceClass *qdev_get_device_class(const char **driver, Error **errp)
 {
     ObjectClass *oc;
     DeviceClass *dc;
+    const char *original_name = *driver;
 
     oc = object_class_by_name(*driver);
     if (!oc) {
@@ -198,7 +222,12 @@ static DeviceClass *qdev_get_device_class(const char **driver, Error **errp)
     }
 
     if (!object_class_dynamic_cast(oc, TYPE_DEVICE)) {
-        error_setg(errp, "'%s' is not a valid device model name", *driver);
+        if (*driver != original_name) {
+            error_setg(errp, "'%s' (alias '%s') is not a valid device model"
+                       " name", original_name, *driver);
+        } else {
+            error_setg(errp, "'%s' is not a valid device model name", *driver);
+        }
         return NULL;
     }
 
@@ -237,9 +266,12 @@ int qdev_device_help(QemuOpts *opts)
         return 0;
     }
 
-    qdev_get_device_class(&driver, &local_err);
-    if (local_err) {
-        goto error;
+    if (!object_class_by_name(driver)) {
+        const char *typename = find_typename_by_alias(driver);
+
+        if (typename) {
+            driver = typename;
+        }
     }
 
     prop_list = qmp_device_list_properties(driver, &local_err);
@@ -262,8 +294,7 @@ int qdev_device_help(QemuOpts *opts)
     return 1;
 
 error:
-    error_printf("%s\n", error_get_pretty(local_err));
-    error_free(local_err);
+    error_report_err(local_err);
     return 1;
 }
 
@@ -289,37 +320,37 @@ static Object *qdev_get_peripheral_anon(void)
     return dev;
 }
 
-#if 0 /* conversion from qerror_report() to error_set() broke their use */
-static void qbus_list_bus(DeviceState *dev)
+static void qbus_list_bus(DeviceState *dev, Error **errp)
 {
     BusState *child;
     const char *sep = " ";
 
-    error_printf("child buses at \"%s\":",
-                 dev->id ? dev->id : object_get_typename(OBJECT(dev)));
+    error_append_hint(errp, "child buses at \"%s\":",
+                      dev->id ? dev->id : object_get_typename(OBJECT(dev)));
     QLIST_FOREACH(child, &dev->child_bus, sibling) {
-        error_printf("%s\"%s\"", sep, child->name);
+        error_append_hint(errp, "%s\"%s\"", sep, child->name);
         sep = ", ";
     }
-    error_printf("\n");
+    error_append_hint(errp, "\n");
 }
 
-static void qbus_list_dev(BusState *bus)
+static void qbus_list_dev(BusState *bus, Error **errp)
 {
     BusChild *kid;
     const char *sep = " ";
 
-    error_printf("devices at \"%s\":", bus->name);
+    error_append_hint(errp, "devices at \"%s\":", bus->name);
     QTAILQ_FOREACH(kid, &bus->children, sibling) {
         DeviceState *dev = kid->child;
-        error_printf("%s\"%s\"", sep, object_get_typename(OBJECT(dev)));
-        if (dev->id)
-            error_printf("/\"%s\"", dev->id);
+        error_append_hint(errp, "%s\"%s\"", sep,
+                          object_get_typename(OBJECT(dev)));
+        if (dev->id) {
+            error_append_hint(errp, "/\"%s\"", dev->id);
+        }
         sep = ", ";
     }
-    error_printf("\n");
+    error_append_hint(errp, "\n");
 }
-#endif
 
 static BusState *qbus_find_bus(DeviceState *dev, char *elem)
 {
@@ -461,11 +492,7 @@ static BusState *qbus_find(const char *path, Error **errp)
         if (!dev) {
             error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                       "Device '%s' not found", elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_dev(bus);
-            }
-#endif
+            qbus_list_dev(bus, errp);
             return NULL;
         }
 
@@ -483,11 +510,7 @@ static BusState *qbus_find(const char *path, Error **errp)
             if (dev->num_child_bus) {
                 error_setg(errp, "Device '%s' has multiple child buses",
                            elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-                if (!monitor_cur_is_qmp()) {
-                    qbus_list_bus(dev);
-                }
-#endif
+                qbus_list_bus(dev, errp);
             } else {
                 error_setg(errp, "Device '%s' has no child bus", elem);
             }
@@ -503,11 +526,7 @@ static BusState *qbus_find(const char *path, Error **errp)
         bus = qbus_find_bus(dev, elem);
         if (!bus) {
             error_setg(errp, "Bus '%s' not found", elem);
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_bus(dev);
-            }
-#endif
+            qbus_list_bus(dev, errp);
             return NULL;
         }
     }
@@ -785,12 +804,17 @@ void qmp_device_add(QDict *qdict, QObject **ret_data, Error **errp)
 void qmp_device_del(const char *id, Error **errp)
 {
     Object *obj;
-    char *root_path = object_get_canonical_path(qdev_get_peripheral());
-    char *path = g_strdup_printf("%s/%s", root_path, id);
 
-    g_free(root_path);
-    obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
-    g_free(path);
+    if (id[0] == '/') {
+        obj = object_resolve_path(id, NULL);
+    } else {
+        char *root_path = object_get_canonical_path(qdev_get_peripheral());
+        char *path = g_strdup_printf("%s/%s", root_path, id);
+
+        g_free(root_path);
+        obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
+        g_free(path);
+    }
 
     if (!obj) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
@@ -798,6 +822,11 @@ void qmp_device_del(const char *id, Error **errp)
         return;
     }
 
+    if (!object_dynamic_cast(obj, TYPE_DEVICE)) {
+        error_setg(errp, "%s is not a hotpluggable device", id);
+        return;
+    }
+
     qdev_unplug(DEVICE(obj), errp);
 }
 
diff --git a/qemu/qemu-bridge-helper.c b/qemu/qemu-bridge-helper.c
index 36eb3bcfd..830fb9e26 100644
--- a/qemu/qemu-bridge-helper.c
+++ b/qemu/qemu-bridge-helper.c
@@ -13,19 +13,10 @@
  *
  */
 
-#include "config-host.h"
-
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdbool.h>
-#include <ctype.h>
+#include "qemu/osdep.h"
+
 #include <glib.h>
 
-#include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/un.h>
diff --git a/qemu/qemu-char.c b/qemu/qemu-char.c
index d956f8db6..b597ee19c 100644
--- a/qemu/qemu-char.c
+++ b/qemu/qemu-char.c
@@ -21,9 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/char.h"
@@ -32,12 +35,12 @@
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi-visit.h"
+#include "qemu/base64.h"
+#include "io/channel-socket.h"
+#include "io/channel-file.h"
+#include "io/channel-tls.h"
+#include "sysemu/replay.h"
 
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
 #include <zlib.h>
 
 #ifndef _WIN32
@@ -51,11 +54,9 @@
 #include <netinet/in.h>
 #include <net/if.h>
 #include <arpa/inet.h>
-#include <dirent.h>
 #include <netdb.h>
 #include <sys/select.h>
 #ifdef CONFIG_BSD
-#include <sys/stat.h>
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <dev/ppbus/ppi.h>
 #include <dev/ppbus/ppbconf.h>
@@ -69,7 +70,6 @@
 #include <linux/parport.h>
 #endif
 #ifdef __sun__
-#include <sys/stat.h>
 #include <sys/ethernet.h>
 #include <sys/sockio.h>
 #include <netinet/arp.h>
@@ -88,64 +88,39 @@
 
 #define READ_BUF_LEN 4096
 #define READ_RETRIES 10
-#define CHR_MAX_FILENAME_SIZE 256
 #define TCP_MAX_FDS 16
 
 /***********************************************************/
 /* Socket address helpers */
-static void qapi_copy_SocketAddress(SocketAddress **p_dest,
-                                    SocketAddress *src)
-{
-    QmpOutputVisitor *qov;
-    QmpInputVisitor *qiv;
-    Visitor *ov, *iv;
-    QObject *obj;
-
-    *p_dest = NULL;
-
-    qov = qmp_output_visitor_new();
-    ov = qmp_output_get_visitor(qov);
-    visit_type_SocketAddress(ov, &src, NULL, &error_abort);
-    obj = qmp_output_get_qobject(qov);
-    qmp_output_visitor_cleanup(qov);
-    if (!obj) {
-        return;
-    }
 
-    qiv = qmp_input_visitor_new(obj);
-    iv = qmp_input_get_visitor(qiv);
-    visit_type_SocketAddress(iv, p_dest, NULL, &error_abort);
-    qmp_input_visitor_cleanup(qiv);
-    qobject_decref(obj);
-}
-
-static int SocketAddress_to_str(char *dest, int max_len,
-                                const char *prefix, SocketAddress *addr,
-                                bool is_listen, bool is_telnet)
+static char *SocketAddress_to_str(const char *prefix, SocketAddress *addr,
+                                  bool is_listen, bool is_telnet)
 {
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        return snprintf(dest, max_len, "%s%s:%s:%s%s", prefix,
-                        is_telnet ? "telnet" : "tcp", addr->inet->host,
-                        addr->inet->port, is_listen ? ",server" : "");
+        return g_strdup_printf("%s%s:%s:%s%s", prefix,
+                               is_telnet ? "telnet" : "tcp",
+                               addr->u.inet.data->host,
+                               addr->u.inet.data->port,
+                               is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_UNIX:
-        return snprintf(dest, max_len, "%sunix:%s%s", prefix,
-                        addr->q_unix->path, is_listen ? ",server" : "");
+        return g_strdup_printf("%sunix:%s%s", prefix,
+                               addr->u.q_unix.data->path,
+                               is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_FD:
-        return snprintf(dest, max_len, "%sfd:%s%s", prefix, addr->fd->str,
-                        is_listen ? ",server" : "");
+        return g_strdup_printf("%sfd:%s%s", prefix, addr->u.fd.data->str,
+                               is_listen ? ",server" : "");
         break;
     default:
         abort();
     }
 }
 
-static int sockaddr_to_str(char *dest, int max_len,
-                           struct sockaddr_storage *ss, socklen_t ss_len,
-                           struct sockaddr_storage *ps, socklen_t ps_len,
-                           bool is_listen, bool is_telnet)
+static char *sockaddr_to_str(struct sockaddr_storage *ss, socklen_t ss_len,
+                             struct sockaddr_storage *ps, socklen_t ps_len,
+                             bool is_listen, bool is_telnet)
 {
     char shost[NI_MAXHOST], sserv[NI_MAXSERV];
     char phost[NI_MAXHOST], pserv[NI_MAXSERV];
@@ -154,9 +129,9 @@ static int sockaddr_to_str(char *dest, int max_len,
     switch (ss->ss_family) {
 #ifndef _WIN32
     case AF_UNIX:
-        return snprintf(dest, max_len, "unix:%s%s",
-                        ((struct sockaddr_un *)(ss))->sun_path,
-                        is_listen ? ",server" : "");
+        return g_strdup_printf("unix:%s%s",
+                               ((struct sockaddr_un *)(ss))->sun_path,
+                               is_listen ? ",server" : "");
 #endif
     case AF_INET6:
         left  = "[";
@@ -167,14 +142,14 @@ static int sockaddr_to_str(char *dest, int max_len,
                     sserv, sizeof(sserv), NI_NUMERICHOST | NI_NUMERICSERV);
         getnameinfo((struct sockaddr *) ps, ps_len, phost, sizeof(phost),
                     pserv, sizeof(pserv), NI_NUMERICHOST | NI_NUMERICSERV);
-        return snprintf(dest, max_len, "%s:%s%s%s:%s%s <-> %s%s%s:%s",
-                        is_telnet ? "telnet" : "tcp",
-                        left, shost, right, sserv,
-                        is_listen ? ",server" : "",
-                        left, phost, right, pserv);
+        return g_strdup_printf("%s:%s%s%s:%s%s <-> %s%s%s:%s",
+                               is_telnet ? "telnet" : "tcp",
+                               left, shost, right, sserv,
+                               is_listen ? ",server" : "",
+                               left, phost, right, pserv);
 
     default:
-        return snprintf(dest, max_len, "unknown");
+        return g_strdup_printf("unknown");
     }
 }
 
@@ -184,10 +159,33 @@ static int sockaddr_to_str(char *dest, int max_len,
 static QTAILQ_HEAD(CharDriverStateHead, CharDriverState) chardevs =
     QTAILQ_HEAD_INITIALIZER(chardevs);
 
-CharDriverState *qemu_chr_alloc(void)
+static void qemu_chr_free_common(CharDriverState *chr);
+
+CharDriverState *qemu_chr_alloc(ChardevCommon *backend, Error **errp)
 {
     CharDriverState *chr = g_malloc0(sizeof(CharDriverState));
     qemu_mutex_init(&chr->chr_write_lock);
+
+    if (backend->has_logfile) {
+        int flags = O_WRONLY | O_CREAT;
+        if (backend->has_logappend &&
+            backend->logappend) {
+            flags |= O_APPEND;
+        } else {
+            flags |= O_TRUNC;
+        }
+        chr->logfd = qemu_open(backend->logfile, flags, 0666);
+        if (chr->logfd < 0) {
+            error_setg_errno(errp, errno,
+                             "Unable to open logfile %s",
+                             backend->logfile);
+            g_free(chr);
+            return NULL;
+        }
+    } else {
+        chr->logfd = -1;
+    }
+
     return chr;
 }
 
@@ -213,37 +211,107 @@ void qemu_chr_be_generic_open(CharDriverState *s)
     qemu_chr_be_event(s, CHR_EVENT_OPENED);
 }
 
+
+/* Not reporting errors from writing to logfile, as logs are
+ * defined to be "best effort" only */
+static void qemu_chr_fe_write_log(CharDriverState *s,
+                                  const uint8_t *buf, size_t len)
+{
+    size_t done = 0;
+    ssize_t ret;
+
+    if (s->logfd < 0) {
+        return;
+    }
+
+    while (done < len) {
+    retry:
+        ret = write(s->logfd, buf + done, len - done);
+        if (ret == -1 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
+
+        if (ret <= 0) {
+            return;
+        }
+        done += ret;
+    }
+}
+
+static int qemu_chr_fe_write_buffer(CharDriverState *s, const uint8_t *buf, int len, int *offset)
+{
+    int res = 0;
+    *offset = 0;
+
+    qemu_mutex_lock(&s->chr_write_lock);
+    while (*offset < len) {
+    retry:
+        res = s->chr_write(s, buf + *offset, len - *offset);
+        if (res < 0 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
+
+        if (res <= 0) {
+            break;
+        }
+
+        *offset += res;
+    }
+    if (*offset > 0) {
+        qemu_chr_fe_write_log(s, buf, *offset);
+    }
+    qemu_mutex_unlock(&s->chr_write_lock);
+
+    return res;
+}
+
 int qemu_chr_fe_write(CharDriverState *s, const uint8_t *buf, int len)
 {
     int ret;
 
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        int offset;
+        replay_char_write_event_load(&ret, &offset);
+        assert(offset <= len);
+        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
+        return ret;
+    }
+
     qemu_mutex_lock(&s->chr_write_lock);
     ret = s->chr_write(s, buf, len);
+
+    if (ret > 0) {
+        qemu_chr_fe_write_log(s, buf, ret);
+    }
+
     qemu_mutex_unlock(&s->chr_write_lock);
+    
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_write_event_save(ret, ret < 0 ? 0 : ret);
+    }
+    
     return ret;
 }
 
 int qemu_chr_fe_write_all(CharDriverState *s, const uint8_t *buf, int len)
 {
-    int offset = 0;
-    int res = 0;
+    int offset;
+    int res;
 
-    qemu_mutex_lock(&s->chr_write_lock);
-    while (offset < len) {
-        do {
-            res = s->chr_write(s, buf + offset, len - offset);
-            if (res == -1 && errno == EAGAIN) {
-                g_usleep(100);
-            }
-        } while (res == -1 && errno == EAGAIN);
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        replay_char_write_event_load(&res, &offset);
+        assert(offset <= len);
+        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
+        return res;
+    }
 
-        if (res <= 0) {
-            break;
-        }
+    res = qemu_chr_fe_write_buffer(s, buf, len, &offset);
 
-        offset += res;
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_write_event_save(res, offset);
     }
-    qemu_mutex_unlock(&s->chr_write_lock);
 
     if (res < 0) {
         return res;
@@ -259,20 +327,27 @@ int qemu_chr_fe_read_all(CharDriverState *s, uint8_t *buf, int len)
     if (!s->chr_sync_read) {
         return 0;
     }
+    
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        return replay_char_read_all_load(buf);
+    }
 
     while (offset < len) {
-        do {
-            res = s->chr_sync_read(s, buf + offset, len - offset);
-            if (res == -1 && errno == EAGAIN) {
-                g_usleep(100);
-            }
-        } while (res == -1 && errno == EAGAIN);
+    retry:
+        res = s->chr_sync_read(s, buf + offset, len - offset);
+        if (res == -1 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
 
         if (res == 0) {
             break;
         }
 
         if (res < 0) {
+            if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+                replay_char_read_all_save_error(res);
+            }
             return res;
         }
 
@@ -283,14 +358,22 @@ int qemu_chr_fe_read_all(CharDriverState *s, uint8_t *buf, int len)
         }
     }
 
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_read_all_save_buf(buf, offset);
+    }
     return offset;
 }
 
 int qemu_chr_fe_ioctl(CharDriverState *s, int cmd, void *arg)
 {
-    if (!s->chr_ioctl)
-        return -ENOTSUP;
-    return s->chr_ioctl(s, cmd, arg);
+    int res;
+    if (!s->chr_ioctl || s->replay) {
+        res = -ENOTSUP;
+    } else {
+        res = s->chr_ioctl(s, cmd, arg);
+    }
+
+    return res;
 }
 
 int qemu_chr_be_can_write(CharDriverState *s)
@@ -300,17 +383,35 @@ int qemu_chr_be_can_write(CharDriverState *s)
     return s->chr_can_read(s->handler_opaque);
 }
 
-void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len)
 {
     if (s->chr_read) {
         s->chr_read(s->handler_opaque, buf, len);
     }
 }
 
+void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    if (s->replay) {
+        if (replay_mode == REPLAY_MODE_PLAY) {
+            return;
+        }
+        replay_chr_be_write(s, buf, len);
+    } else {
+        qemu_chr_be_write_impl(s, buf, len);
+    }
+}
+
 int qemu_chr_fe_get_msgfd(CharDriverState *s)
 {
     int fd;
-    return (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
+    int res = (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
+    if (s->replay) {
+        fprintf(stderr,
+                "Replay: get msgfd is not supported for serial devices yet\n");
+        exit(1);
+    }
+    return res;
 }
 
 int qemu_chr_fe_get_msgfds(CharDriverState *s, int *fds, int len)
@@ -384,11 +485,18 @@ static int null_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
     return len;
 }
 
-static CharDriverState *qemu_chr_open_null(void)
+static CharDriverState *qemu_chr_open_null(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
     CharDriverState *chr;
+    ChardevCommon *common = backend->u.null.data;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     chr->chr_write = null_chr_write;
     chr->explicit_be_open = true;
     return chr;
@@ -524,7 +632,7 @@ static int mux_proc_byte(CharDriverState *chr, MuxDriver *d, int ch)
                  break;
             }
         case 's':
-            bdrv_commit_all();
+            blk_commit_all();
             break;
         case 'b':
             qemu_chr_be_event(chr, CHR_EVENT_BREAK);
@@ -680,13 +788,26 @@ static GSource *mux_chr_add_watch(CharDriverState *s, GIOCondition cond)
     return d->drv->chr_add_watch(d->drv, cond);
 }
 
-static CharDriverState *qemu_chr_open_mux(CharDriverState *drv)
+static CharDriverState *qemu_chr_open_mux(const char *id,
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret, Error **errp)
 {
-    CharDriverState *chr;
+    ChardevMux *mux = backend->u.mux.data;
+    CharDriverState *chr, *drv;
     MuxDriver *d;
+    ChardevCommon *common = qapi_ChardevMux_base(mux);
 
-    chr = qemu_chr_alloc();
-    d = g_malloc0(sizeof(MuxDriver));
+    drv = qemu_chr_find(mux->chardev);
+    if (drv == NULL) {
+        error_setg(errp, "mux: base chardev %s not found", mux->chardev);
+        return NULL;
+    }
+
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    d = g_new0(MuxDriver, 1);
 
     chr->opaque = d;
     d->drv = drv;
@@ -709,82 +830,11 @@ static CharDriverState *qemu_chr_open_mux(CharDriverState *drv)
 }
 
 
-#ifdef _WIN32
-int send_all(int fd, const void *buf, int len1)
-{
-    int ret, len;
-
-    len = len1;
-    while (len > 0) {
-        ret = send(fd, buf, len, 0);
-        if (ret < 0) {
-            errno = WSAGetLastError();
-            if (errno != WSAEWOULDBLOCK) {
-                return -1;
-            }
-        } else if (ret == 0) {
-            break;
-        } else {
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-#else
-
-int send_all(int fd, const void *_buf, int len1)
-{
-    int ret, len;
-    const uint8_t *buf = _buf;
-
-    len = len1;
-    while (len > 0) {
-        ret = write(fd, buf, len);
-        if (ret < 0) {
-            if (errno != EINTR && errno != EAGAIN)
-                return -1;
-        } else if (ret == 0) {
-            break;
-        } else {
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-int recv_all(int fd, void *_buf, int len1, bool single_read)
-{
-    int ret, len;
-    uint8_t *buf = _buf;
-
-    len = len1;
-    while ((len > 0) && (ret = read(fd, buf, len)) != 0) {
-        if (ret < 0) {
-            if (errno != EINTR && errno != EAGAIN) {
-                return -1;
-            }
-            continue;
-        } else {
-            if (single_read) {
-                return ret;
-            }
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-#endif /* !_WIN32 */
-
 typedef struct IOWatchPoll
 {
     GSource parent;
 
-    GIOChannel *channel;
+    QIOChannel *ioc;
     GSource *src;
 
     IOCanReadHandler *fd_can_read;
@@ -807,8 +857,8 @@ static gboolean io_watch_poll_prepare(GSource *source, gint *timeout_)
     }
 
     if (now_active) {
-        iwp->src = g_io_create_watch(iwp->channel,
-                                     G_IO_IN | G_IO_ERR | G_IO_HUP | G_IO_NVAL);
+        iwp->src = qio_channel_create_watch(
+            iwp->ioc, G_IO_IN | G_IO_ERR | G_IO_HUP | G_IO_NVAL);
         g_source_set_callback(iwp->src, iwp->fd_read, iwp->opaque, NULL);
         g_source_attach(iwp->src, NULL);
     } else {
@@ -854,9 +904,9 @@ static GSourceFuncs io_watch_poll_funcs = {
 };
 
 /* Can only be used for read */
-static guint io_add_watch_poll(GIOChannel *channel,
+static guint io_add_watch_poll(QIOChannel *ioc,
                                IOCanReadHandler *fd_can_read,
-                               GIOFunc fd_read,
+                               QIOChannelFunc fd_read,
                                gpointer user_data)
 {
     IOWatchPoll *iwp;
@@ -865,7 +915,7 @@ static guint io_add_watch_poll(GIOChannel *channel,
     iwp = (IOWatchPoll *) g_source_new(&io_watch_poll_funcs, sizeof(IOWatchPoll));
     iwp->fd_can_read = fd_can_read;
     iwp->opaque = user_data;
-    iwp->channel = channel;
+    iwp->ioc = ioc;
     iwp->fd_read = (GSourceFunc) fd_read;
     iwp->src = NULL;
 
@@ -901,79 +951,50 @@ static void remove_fd_in_watch(CharDriverState *chr)
     }
 }
 
-#ifndef _WIN32
-static GIOChannel *io_channel_from_fd(int fd)
-{
-    GIOChannel *chan;
 
-    if (fd == -1) {
-        return NULL;
-    }
-
-    chan = g_io_channel_unix_new(fd);
-
-    g_io_channel_set_encoding(chan, NULL, NULL);
-    g_io_channel_set_buffered(chan, FALSE);
-
-    return chan;
-}
-#endif
-
-static GIOChannel *io_channel_from_socket(int fd)
+static int io_channel_send_full(QIOChannel *ioc,
+                                const void *buf, size_t len,
+                                int *fds, size_t nfds)
 {
-    GIOChannel *chan;
+    size_t offset = 0;
 
-    if (fd == -1) {
-        return NULL;
-    }
+    while (offset < len) {
+        ssize_t ret = 0;
+        struct iovec iov = { .iov_base = (char *)buf + offset,
+                             .iov_len = len - offset };
+
+        ret = qio_channel_writev_full(
+            ioc, &iov, 1,
+            fds, nfds, NULL);
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (offset) {
+                return offset;
+            }
 
-#ifdef _WIN32
-    chan = g_io_channel_win32_new_socket(fd);
-#else
-    chan = g_io_channel_unix_new(fd);
-#endif
+            errno = EAGAIN;
+            return -1;
+        } else if (ret < 0) {
+            errno = EINVAL;
+            return -1;
+        }
 
-    g_io_channel_set_encoding(chan, NULL, NULL);
-    g_io_channel_set_buffered(chan, FALSE);
+        offset += ret;
+    }
 
-    return chan;
+    return offset;
 }
 
-static int io_channel_send(GIOChannel *fd, const void *buf, size_t len)
-{
-    size_t offset = 0;
-    GIOStatus status = G_IO_STATUS_NORMAL;
-
-    while (offset < len && status == G_IO_STATUS_NORMAL) {
-        gsize bytes_written = 0;
 
-        status = g_io_channel_write_chars(fd, buf + offset, len - offset,
-                                          &bytes_written, NULL);
-        offset += bytes_written;
-    }
-
-    if (offset > 0) {
-        return offset;
-    }
-    switch (status) {
-    case G_IO_STATUS_NORMAL:
-        g_assert(len == 0);
-        return 0;
-    case G_IO_STATUS_AGAIN:
-        errno = EAGAIN;
-        return -1;
-    default:
-        break;
-    }
-    errno = EINVAL;
-    return -1;
+#ifndef _WIN32
+static int io_channel_send(QIOChannel *ioc, const void *buf, size_t len)
+{
+    return io_channel_send_full(ioc, buf, len, NULL, 0);
 }
 
-#ifndef _WIN32
 
 typedef struct FDCharDriver {
     CharDriverState *chr;
-    GIOChannel *fd_in, *fd_out;
+    QIOChannel *ioc_in, *ioc_out;
     int max_size;
 } FDCharDriver;
 
@@ -982,17 +1003,16 @@ static int fd_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     FDCharDriver *s = chr->opaque;
     
-    return io_channel_send(s->fd_out, buf, len);
+    return io_channel_send(s->ioc_out, buf, len);
 }
 
-static gboolean fd_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean fd_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     FDCharDriver *s = chr->opaque;
     int len;
     uint8_t buf[READ_BUF_LEN];
-    GIOStatus status;
-    gsize bytes_read;
+    ssize_t ret;
 
     len = sizeof(buf);
     if (len > s->max_size) {
@@ -1002,15 +1022,15 @@ static gboolean fd_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
         return TRUE;
     }
 
-    status = g_io_channel_read_chars(chan, (gchar *)buf,
-                                     len, &bytes_read, NULL);
-    if (status == G_IO_STATUS_EOF) {
+    ret = qio_channel_read(
+        chan, (gchar *)buf, len, NULL);
+    if (ret == 0) {
         remove_fd_in_watch(chr);
         qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
         return FALSE;
     }
-    if (status == G_IO_STATUS_NORMAL) {
-        qemu_chr_be_write(chr, buf, bytes_read);
+    if (ret > 0) {
+        qemu_chr_be_write(chr, buf, ret);
     }
 
     return TRUE;
@@ -1028,7 +1048,7 @@ static int fd_chr_read_poll(void *opaque)
 static GSource *fd_chr_add_watch(CharDriverState *chr, GIOCondition cond)
 {
     FDCharDriver *s = chr->opaque;
-    return g_io_create_watch(s->fd_out, cond);
+    return qio_channel_create_watch(s->ioc_out, cond);
 }
 
 static void fd_chr_update_read_handler(CharDriverState *chr)
@@ -1036,8 +1056,9 @@ static void fd_chr_update_read_handler(CharDriverState *chr)
     FDCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->fd_in) {
-        chr->fd_in_tag = io_add_watch_poll(s->fd_in, fd_chr_read_poll,
+    if (s->ioc_in) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc_in,
+                                           fd_chr_read_poll,
                                            fd_chr_read, chr);
     }
 }
@@ -1047,11 +1068,11 @@ static void fd_chr_close(struct CharDriverState *chr)
     FDCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->fd_in) {
-        g_io_channel_unref(s->fd_in);
+    if (s->ioc_in) {
+        object_unref(OBJECT(s->ioc_in));
     }
-    if (s->fd_out) {
-        g_io_channel_unref(s->fd_out);
+    if (s->ioc_out) {
+        object_unref(OBJECT(s->ioc_out));
     }
 
     g_free(s);
@@ -1059,15 +1080,19 @@ static void fd_chr_close(struct CharDriverState *chr)
 }
 
 /* open a character device to a unix fd */
-static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
+static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out,
+                                         ChardevCommon *backend, Error **errp)
 {
     CharDriverState *chr;
     FDCharDriver *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(FDCharDriver));
-    s->fd_in = io_channel_from_fd(fd_in);
-    s->fd_out = io_channel_from_fd(fd_out);
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(FDCharDriver, 1);
+    s->ioc_in = QIO_CHANNEL(qio_channel_file_new_fd(fd_in));
+    s->ioc_out = QIO_CHANNEL(qio_channel_file_new_fd(fd_out));
     qemu_set_nonblock(fd_out);
     s->chr = chr;
     chr->opaque = s;
@@ -1079,22 +1104,25 @@ static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe.data;
     int fd_in, fd_out;
-    char filename_in[CHR_MAX_FILENAME_SIZE];
-    char filename_out[CHR_MAX_FILENAME_SIZE];
+    char *filename_in;
+    char *filename_out;
     const char *filename = opts->device;
+    ChardevCommon *common = qapi_ChardevHostdev_base(opts);
 
-    if (filename == NULL) {
-        fprintf(stderr, "chardev: pipe: no filename given\n");
-        return NULL;
-    }
 
-    snprintf(filename_in, CHR_MAX_FILENAME_SIZE, "%s.in", filename);
-    snprintf(filename_out, CHR_MAX_FILENAME_SIZE, "%s.out", filename);
+    filename_in = g_strdup_printf("%s.in", filename);
+    filename_out = g_strdup_printf("%s.out", filename);
     TFR(fd_in = qemu_open(filename_in, O_RDWR | O_BINARY));
     TFR(fd_out = qemu_open(filename_out, O_RDWR | O_BINARY));
+    g_free(filename_in);
+    g_free(filename_out);
     if (fd_in < 0 || fd_out < 0) {
 	if (fd_in >= 0)
 	    close(fd_in);
@@ -1102,10 +1130,11 @@ static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
 	    close(fd_out);
         TFR(fd_in = fd_out = qemu_open(filename, O_RDWR | O_BINARY));
         if (fd_in < 0) {
+            error_setg_file_open(errp, errno, filename);
             return NULL;
         }
     }
-    return qemu_chr_open_fd(fd_in, fd_out);
+    return qemu_chr_open_fd(fd_in, fd_out, common, errp);
 }
 
 /* init terminal so that we can grab keys */
@@ -1157,19 +1186,24 @@ static void qemu_chr_close_stdio(struct CharDriverState *chr)
     fd_chr_close(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
+    ChardevStdio *opts = backend->u.stdio.data;
     CharDriverState *chr;
     struct sigaction act;
+    ChardevCommon *common = qapi_ChardevStdio_base(opts);
 
     if (is_daemonized()) {
-        error_report("cannot use stdio with -daemonize");
+        error_setg(errp, "cannot use stdio with -daemonize");
         return NULL;
     }
 
     if (stdio_in_use) {
-        error_report("cannot use stdio by multiple character devices");
-        exit(1);
+        error_setg(errp, "cannot use stdio by multiple character devices");
+        return NULL;
     }
 
     stdio_in_use = true;
@@ -1182,7 +1216,7 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     act.sa_handler = term_stdio_handler;
     sigaction(SIGCONT, &act, NULL);
 
-    chr = qemu_chr_open_fd(0, 1);
+    chr = qemu_chr_open_fd(0, 1, common, errp);
     chr->chr_close = qemu_chr_close_stdio;
     chr->chr_set_echo = qemu_chr_set_echo_stdio;
     if (opts->has_signal) {
@@ -1197,10 +1231,11 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
     || defined(__GLIBC__)
 
-#define HAVE_CHARDEV_TTY 1
+#define HAVE_CHARDEV_SERIAL 1
+#define HAVE_CHARDEV_PTY 1
 
 typedef struct {
-    GIOChannel *fd;
+    QIOChannel *ioc;
     int read_bytes;
 
     /* Protected by the CharDriverState chr_write_lock.  */
@@ -1250,11 +1285,17 @@ static void pty_chr_update_read_handler_locked(CharDriverState *chr)
 {
     PtyCharDriver *s = chr->opaque;
     GPollFD pfd;
+    int rc;
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(s->ioc);
 
-    pfd.fd = g_io_channel_unix_get_fd(s->fd);
+    pfd.fd = fioc->fd;
     pfd.events = G_IO_OUT;
     pfd.revents = 0;
-    g_poll(&pfd, 1, 0);
+    do {
+        rc = g_poll(&pfd, 1, 0);
+    } while (rc == -1 && errno == EINTR);
+    assert(rc >= 0);
+
     if (pfd.revents & G_IO_HUP) {
         pty_chr_state(chr, 0);
     } else {
@@ -1281,7 +1322,7 @@ static int pty_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
             return 0;
         }
     }
-    return io_channel_send(s->fd, buf, len);
+    return io_channel_send(s->ioc, buf, len);
 }
 
 static GSource *pty_chr_add_watch(CharDriverState *chr, GIOCondition cond)
@@ -1290,7 +1331,7 @@ static GSource *pty_chr_add_watch(CharDriverState *chr, GIOCondition cond)
     if (!s->connected) {
         return NULL;
     }
-    return g_io_create_watch(s->fd, cond);
+    return qio_channel_create_watch(s->ioc, cond);
 }
 
 static int pty_chr_read_poll(void *opaque)
@@ -1302,13 +1343,13 @@ static int pty_chr_read_poll(void *opaque)
     return s->read_bytes;
 }
 
-static gboolean pty_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean pty_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     PtyCharDriver *s = chr->opaque;
-    gsize size, len;
+    gsize len;
     uint8_t buf[READ_BUF_LEN];
-    GIOStatus status;
+    ssize_t ret;
 
     len = sizeof(buf);
     if (len > s->read_bytes)
@@ -1316,13 +1357,13 @@ static gboolean pty_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
     if (len == 0) {
         return TRUE;
     }
-    status = g_io_channel_read_chars(s->fd, (gchar *)buf, len, &size, NULL);
-    if (status != G_IO_STATUS_NORMAL) {
+    ret = qio_channel_read(s->ioc, (char *)buf, len, NULL);
+    if (ret <= 0) {
         pty_chr_state(chr, 0);
         return FALSE;
     } else {
         pty_chr_state(chr, 1);
-        qemu_chr_be_write(chr, buf, size);
+        qemu_chr_be_write(chr, buf, ret);
     }
     return TRUE;
 }
@@ -1364,7 +1405,8 @@ static void pty_chr_state(CharDriverState *chr, int connected)
             s->open_tag = g_idle_add(qemu_chr_be_generic_open_func, chr);
         }
         if (!chr->fd_in_tag) {
-            chr->fd_in_tag = io_add_watch_poll(s->fd, pty_chr_read_poll,
+            chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                               pty_chr_read_poll,
                                                pty_chr_read, chr);
         }
     }
@@ -1373,13 +1415,10 @@ static void pty_chr_state(CharDriverState *chr, int connected)
 static void pty_chr_close(struct CharDriverState *chr)
 {
     PtyCharDriver *s = chr->opaque;
-    int fd;
 
     qemu_mutex_lock(&chr->chr_write_lock);
     pty_chr_state(chr, 0);
-    fd = g_io_channel_unix_get_fd(s->fd);
-    g_io_channel_unref(s->fd);
-    close(fd);
+    object_unref(OBJECT(s->ioc));
     if (s->timer_tag) {
         g_source_remove(s->timer_tag);
         s->timer_tag = 0;
@@ -1390,22 +1429,30 @@ static void pty_chr_close(struct CharDriverState *chr)
 }
 
 static CharDriverState *qemu_chr_open_pty(const char *id,
-                                          ChardevReturn *ret)
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret,
+                                          Error **errp)
 {
     CharDriverState *chr;
     PtyCharDriver *s;
     int master_fd, slave_fd;
     char pty_name[PATH_MAX];
+    ChardevCommon *common = backend->u.pty.data;
 
     master_fd = qemu_openpty_raw(&slave_fd, pty_name);
     if (master_fd < 0) {
+        error_setg_errno(errp, errno, "Failed to create PTY");
         return NULL;
     }
 
     close(slave_fd);
     qemu_set_nonblock(master_fd);
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        close(master_fd);
+        return NULL;
+    }
 
     chr->filename = g_strdup_printf("pty:%s", pty_name);
     ret->pty = g_strdup(pty_name);
@@ -1414,7 +1461,7 @@ static CharDriverState *qemu_chr_open_pty(const char *id,
     fprintf(stderr, "char device redirected to %s (label %s)\n",
             pty_name, id);
 
-    s = g_malloc0(sizeof(PtyCharDriver));
+    s = g_new0(PtyCharDriver, 1);
     chr->opaque = s;
     chr->chr_write = pty_chr_write;
     chr->chr_update_read_handler = pty_chr_update_read_handler;
@@ -1422,7 +1469,7 @@ static CharDriverState *qemu_chr_open_pty(const char *id,
     chr->chr_add_watch = pty_chr_add_watch;
     chr->explicit_be_open = true;
 
-    s->fd = io_channel_from_fd(master_fd);
+    s->ioc = QIO_CHANNEL(qio_channel_file_new_fd(master_fd));
     s->timer_tag = 0;
 
     return chr;
@@ -1546,12 +1593,13 @@ static void tty_serial_init(int fd, int speed,
 static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
 {
     FDCharDriver *s = chr->opaque;
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(s->ioc_in);
 
     switch(cmd) {
     case CHR_IOCTL_SERIAL_SET_PARAMS:
         {
             QEMUSerialSetParams *ssp = arg;
-            tty_serial_init(g_io_channel_unix_get_fd(s->fd_in),
+            tty_serial_init(fioc->fd,
                             ssp->speed, ssp->parity,
                             ssp->data_bits, ssp->stop_bits);
         }
@@ -1560,7 +1608,7 @@ static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
         {
             int enable = *(int *)arg;
             if (enable) {
-                tcsendbreak(g_io_channel_unix_get_fd(s->fd_in), 1);
+                tcsendbreak(fioc->fd, 1);
             }
         }
         break;
@@ -1568,7 +1616,7 @@ static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
         {
             int sarg = 0;
             int *targ = (int *)arg;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMGET, &sarg);
+            ioctl(fioc->fd, TIOCMGET, &sarg);
             *targ = 0;
             if (sarg & TIOCM_CTS)
                 *targ |= CHR_TIOCM_CTS;
@@ -1588,7 +1636,7 @@ static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
         {
             int sarg = *(int *)arg;
             int targ = 0;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMGET, &targ);
+            ioctl(fioc->fd, TIOCMGET, &targ);
             targ &= ~(CHR_TIOCM_CTS | CHR_TIOCM_CAR | CHR_TIOCM_DSR
                      | CHR_TIOCM_RI | CHR_TIOCM_DTR | CHR_TIOCM_RTS);
             if (sarg & CHR_TIOCM_CTS)
@@ -1603,7 +1651,7 @@ static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
                 targ |= TIOCM_DTR;
             if (sarg & CHR_TIOCM_RTS)
                 targ |= TIOCM_RTS;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMSET, &targ);
+            ioctl(fioc->fd, TIOCMSET, &targ);
         }
         break;
     default:
@@ -1614,26 +1662,17 @@ static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
 
 static void qemu_chr_close_tty(CharDriverState *chr)
 {
-    FDCharDriver *s = chr->opaque;
-    int fd = -1;
-
-    if (s) {
-        fd = g_io_channel_unix_get_fd(s->fd_in);
-    }
-
     fd_chr_close(chr);
-
-    if (fd >= 0) {
-        close(fd);
-    }
 }
 
-static CharDriverState *qemu_chr_open_tty_fd(int fd)
+static CharDriverState *qemu_chr_open_tty_fd(int fd,
+                                             ChardevCommon *backend,
+                                             Error **errp)
 {
     CharDriverState *chr;
 
     tty_serial_init(fd, 115200, 'N', 8, 1);
-    chr = qemu_chr_open_fd(fd, fd);
+    chr = qemu_chr_open_fd(fd, fd, backend, errp);
     chr->chr_ioctl = tty_serial_ioctl;
     chr->chr_close = qemu_chr_close_tty;
     return chr;
@@ -1753,25 +1792,32 @@ static void pp_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd,
+                                            ChardevCommon *backend,
+                                            Error **errp)
 {
     CharDriverState *chr;
     ParallelCharDriver *drv;
 
     if (ioctl(fd, PPCLAIM) < 0) {
+        error_setg_errno(errp, errno, "not a parallel port");
         close(fd);
         return NULL;
     }
 
-    drv = g_malloc0(sizeof(ParallelCharDriver));
-    drv->fd = fd;
-    drv->mode = IEEE1284_MODE_COMPAT;
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
 
-    chr = qemu_chr_alloc();
+    drv = g_new0(ParallelCharDriver, 1);
+    chr->opaque = drv;
     chr->chr_write = null_chr_write;
     chr->chr_ioctl = pp_ioctl;
     chr->chr_close = pp_close;
-    chr->opaque = drv;
+
+    drv->fd = fd;
+    drv->mode = IEEE1284_MODE_COMPAT;
 
     return chr;
 }
@@ -1818,11 +1864,16 @@ static int pp_ioctl(CharDriverState *chr, int cmd, void *arg)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd,
+                                            ChardevCommon *backend,
+                                            Error **errp)
 {
     CharDriverState *chr;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
     chr->opaque = (void *)(intptr_t)fd;
     chr->chr_write = null_chr_write;
     chr->chr_ioctl = pp_ioctl;
@@ -1833,6 +1884,8 @@ static CharDriverState *qemu_chr_open_pp_fd(int fd)
 
 #else /* _WIN32 */
 
+#define HAVE_CHARDEV_SERIAL 1
+
 typedef struct {
     int max_size;
     HANDLE hcom, hrecv, hsend;
@@ -1884,7 +1937,7 @@ static void win_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static int win_chr_init(CharDriverState *chr, const char *filename)
+static int win_chr_init(CharDriverState *chr, const char *filename, Error **errp)
 {
     WinCharState *s = chr->opaque;
     COMMCONFIG comcfg;
@@ -1895,25 +1948,25 @@ static int win_chr_init(CharDriverState *chr, const char *filename)
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
     s->hcom = CreateFile(filename, GENERIC_READ|GENERIC_WRITE, 0, NULL,
                       OPEN_EXISTING, FILE_FLAG_OVERLAPPED, 0);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateFile (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateFile (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
 
     if (!SetupComm(s->hcom, NRECVBUF, NSENDBUF)) {
-        fprintf(stderr, "Failed SetupComm\n");
+        error_setg(errp, "Failed SetupComm");
         goto fail;
     }
 
@@ -1924,23 +1977,23 @@ static int win_chr_init(CharDriverState *chr, const char *filename)
     CommConfigDialog(filename, NULL, &comcfg);
 
     if (!SetCommState(s->hcom, &comcfg.dcb)) {
-        fprintf(stderr, "Failed SetCommState\n");
+        error_setg(errp, "Failed SetCommState");
         goto fail;
     }
 
     if (!SetCommMask(s->hcom, EV_ERR)) {
-        fprintf(stderr, "Failed SetCommMask\n");
+        error_setg(errp, "Failed SetCommMask");
         goto fail;
     }
 
     cto.ReadIntervalTimeout = MAXDWORD;
     if (!SetCommTimeouts(s->hcom, &cto)) {
-        fprintf(stderr, "Failed SetCommTimeouts\n");
+        error_setg(errp, "Failed SetCommTimeouts");
         goto fail;
     }
 
     if (!ClearCommError(s->hcom, &err, &comstat)) {
-        fprintf(stderr, "Failed ClearCommError\n");
+        error_setg(errp, "Failed ClearCommError");
         goto fail;
     }
     qemu_add_polling_cb(win_chr_poll, chr);
@@ -2045,20 +2098,25 @@ static int win_chr_poll(void *opaque)
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_win_path(const char *filename)
+static CharDriverState *qemu_chr_open_win_path(const char *filename,
+                                               ChardevCommon *backend,
+                                               Error **errp)
 {
     CharDriverState *chr;
     WinCharState *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_init(chr, filename) < 0) {
+    if (win_chr_init(chr, filename, errp) < 0) {
         g_free(s);
-        g_free(chr);
+        qemu_chr_free_common(chr);
         return NULL;
     }
     return chr;
@@ -2080,34 +2138,36 @@ static int win_chr_pipe_poll(void *opaque)
     return 0;
 }
 
-static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
+static int win_chr_pipe_init(CharDriverState *chr, const char *filename,
+                             Error **errp)
 {
     WinCharState *s = chr->opaque;
     OVERLAPPED ov;
     int ret;
     DWORD size;
-    char openname[CHR_MAX_FILENAME_SIZE];
+    char *openname;
 
     s->fpipe = TRUE;
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
-    snprintf(openname, sizeof(openname), "\\\\.\\pipe\\%s", filename);
+    openname = g_strdup_printf("\\\\.\\pipe\\%s", filename);
     s->hcom = CreateNamedPipe(openname, PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
                               PIPE_TYPE_BYTE | PIPE_READMODE_BYTE |
                               PIPE_WAIT,
                               MAXCONNECT, NSENDBUF, NRECVBUF, NTIMEOUT, NULL);
+    g_free(openname);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateNamedPipe (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateNamedPipe (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
@@ -2116,13 +2176,13 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
     ov.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
     ret = ConnectNamedPipe(s->hcom, &ov);
     if (ret) {
-        fprintf(stderr, "Failed ConnectNamedPipe\n");
+        error_setg(errp, "Failed ConnectNamedPipe");
         goto fail;
     }
 
     ret = GetOverlappedResult(s->hcom, &ov, &size, TRUE);
     if (!ret) {
-        fprintf(stderr, "Failed GetOverlappedResult\n");
+        error_setg(errp, "Failed GetOverlappedResult");
         if (ov.hEvent) {
             CloseHandle(ov.hEvent);
             ov.hEvent = NULL;
@@ -2143,42 +2203,60 @@ static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
 }
 
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe.data;
     const char *filename = opts->device;
     CharDriverState *chr;
     WinCharState *s;
+    ChardevCommon *common = qapi_ChardevHostdev_base(opts);
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_pipe_init(chr, filename) < 0) {
+    if (win_chr_pipe_init(chr, filename, errp) < 0) {
         g_free(s);
-        g_free(chr);
+        qemu_chr_free_common(chr);
         return NULL;
     }
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_win_file(HANDLE fd_out)
+static CharDriverState *qemu_chr_open_win_file(HANDLE fd_out,
+                                               ChardevCommon *backend,
+                                               Error **errp)
 {
     CharDriverState *chr;
     WinCharState *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     s->hcom = fd_out;
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_win_con(void)
+static CharDriverState *qemu_chr_open_win_con(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
-    return qemu_chr_open_win_file(GetStdHandle(STD_OUTPUT_HANDLE));
+    ChardevCommon *common = backend->u.console.data;
+    return qemu_chr_open_win_file(GetStdHandle(STD_OUTPUT_HANDLE),
+                                  common, errp);
 }
 
 static int win_stdio_write(CharDriverState *chr, const uint8_t *buf, int len)
@@ -2317,20 +2395,27 @@ static void win_stdio_close(CharDriverState *chr)
     g_free(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
     CharDriverState   *chr;
     WinStdioCharState *stdio;
     DWORD              dwMode;
     int                is_console = 0;
+    ChardevCommon *common = qapi_ChardevStdio_base(backend->u.stdio.data);
 
-    chr   = qemu_chr_alloc();
-    stdio = g_malloc0(sizeof(WinStdioCharState));
+    chr   = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    stdio = g_new0(WinStdioCharState, 1);
 
     stdio->hStdIn = GetStdHandle(STD_INPUT_HANDLE);
     if (stdio->hStdIn == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "cannot open stdio: invalid handle\n");
-        exit(1);
+        error_setg(errp, "cannot open stdio: invalid handle");
+        return NULL;
     }
 
     is_console = GetConsoleMode(stdio->hStdIn, &dwMode) != 0;
@@ -2342,25 +2427,30 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     if (is_console) {
         if (qemu_add_wait_object(stdio->hStdIn,
                                  win_stdio_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err1;
         }
     } else {
         DWORD   dwId;
             
         stdio->hInputReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
         stdio->hInputDoneEvent  = CreateEvent(NULL, FALSE, FALSE, NULL);
-        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
-                                               chr, 0, &dwId);
-
-        if (stdio->hInputThread == INVALID_HANDLE_VALUE
-            || stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
+        if (stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
             || stdio->hInputDoneEvent == INVALID_HANDLE_VALUE) {
-            fprintf(stderr, "cannot create stdio thread or event\n");
-            exit(1);
+            error_setg(errp, "cannot create event");
+            goto err2;
         }
         if (qemu_add_wait_object(stdio->hInputReadyEvent,
                                  win_stdio_thread_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err2;
+        }
+        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
+                                               chr, 0, &dwId);
+
+        if (stdio->hInputThread == INVALID_HANDLE_VALUE) {
+            error_setg(errp, "cannot create stdio thread");
+            goto err3;
         }
     }
 
@@ -2378,6 +2468,15 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
     qemu_chr_fe_set_echo(chr, false);
 
     return chr;
+
+err3:
+    qemu_del_wait_object(stdio->hInputReadyEvent, NULL, NULL);
+err2:
+    CloseHandle(stdio->hInputReadyEvent);
+    CloseHandle(stdio->hInputDoneEvent);
+err1:
+    qemu_del_wait_object(stdio->hStdIn, NULL, NULL);
+    return NULL;
 }
 #endif /* !_WIN32 */
 
@@ -2386,8 +2485,7 @@ static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
 /* UDP Net console */
 
 typedef struct {
-    int fd;
-    GIOChannel *chan;
+    QIOChannel *ioc;
     uint8_t buf[READ_BUF_LEN];
     int bufcnt;
     int bufptr;
@@ -2398,17 +2496,9 @@ typedef struct {
 static int udp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     NetCharDriver *s = chr->opaque;
-    gsize bytes_written;
-    GIOStatus status;
 
-    status = g_io_channel_write_chars(s->chan, (const gchar *)buf, len, &bytes_written, NULL);
-    if (status == G_IO_STATUS_EOF) {
-        return 0;
-    } else if (status != G_IO_STATUS_NORMAL) {
-        return -1;
-    }
-
-    return bytes_written;
+    return qio_channel_write(
+        s->ioc, (const char *)buf, len, NULL);
 }
 
 static int udp_chr_read_poll(void *opaque)
@@ -2429,24 +2519,22 @@ static int udp_chr_read_poll(void *opaque)
     return s->max_size;
 }
 
-static gboolean udp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean udp_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     NetCharDriver *s = chr->opaque;
-    gsize bytes_read = 0;
-    GIOStatus status;
+    ssize_t ret;
 
     if (s->max_size == 0) {
         return TRUE;
     }
-    status = g_io_channel_read_chars(s->chan, (gchar *)s->buf, sizeof(s->buf),
-                                     &bytes_read, NULL);
-    s->bufcnt = bytes_read;
-    s->bufptr = s->bufcnt;
-    if (status != G_IO_STATUS_NORMAL) {
+    ret = qio_channel_read(
+        s->ioc, (char *)s->buf, sizeof(s->buf), NULL);
+    if (ret <= 0) {
         remove_fd_in_watch(chr);
         return FALSE;
     }
+    s->bufcnt = ret;
 
     s->bufptr = 0;
     while (s->max_size > 0 && s->bufptr < s->bufcnt) {
@@ -2463,8 +2551,9 @@ static void udp_chr_update_read_handler(CharDriverState *chr)
     NetCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, udp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           udp_chr_read_poll,
                                            udp_chr_read, chr);
     }
 }
@@ -2474,24 +2563,27 @@ static void udp_chr_close(CharDriverState *chr)
     NetCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        g_io_channel_unref(s->chan);
-        closesocket(s->fd);
+    if (s->ioc) {
+        object_unref(OBJECT(s->ioc));
     }
     g_free(s);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_udp_fd(int fd)
+static CharDriverState *qemu_chr_open_udp(QIOChannelSocket *sioc,
+                                          ChardevCommon *backend,
+                                          Error **errp)
 {
     CharDriverState *chr = NULL;
     NetCharDriver *s = NULL;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(NetCharDriver));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(NetCharDriver, 1);
 
-    s->fd = fd;
-    s->chan = io_channel_from_socket(s->fd);
+    s->ioc = QIO_CHANNEL(sioc);
     s->bufcnt = 0;
     s->bufptr = 0;
     chr->opaque = s;
@@ -2507,19 +2599,20 @@ static CharDriverState *qemu_chr_open_udp_fd(int fd)
 /* TCP Net console */
 
 typedef struct {
-
-    GIOChannel *chan, *listen_chan;
+    QIOChannel *ioc; /* Client I/O channel */
+    QIOChannelSocket *sioc; /* Client master channel */
+    QIOChannelSocket *listen_ioc;
     guint listen_tag;
-    int fd, listen_fd;
+    QCryptoTLSCreds *tls_creds;
     int connected;
     int max_size;
     int do_telnetopt;
     int do_nodelay;
     int is_unix;
     int *read_msgfds;
-    int read_msgfds_num;
+    size_t read_msgfds_num;
     int *write_msgfds;
-    int write_msgfds_num;
+    size_t write_msgfds_num;
 
     SocketAddress *addr;
     bool is_listen;
@@ -2553,68 +2646,27 @@ static void check_report_connect_error(CharDriverState *chr,
     qemu_chr_socket_restart_timer(chr);
 }
 
-static gboolean tcp_chr_accept(GIOChannel *chan, GIOCondition cond, void *opaque);
-
-#ifndef _WIN32
-static int unix_send_msgfds(CharDriverState *chr, const uint8_t *buf, int len)
-{
-    TCPCharDriver *s = chr->opaque;
-    struct msghdr msgh;
-    struct iovec iov;
-    int r;
-
-    size_t fd_size = s->write_msgfds_num * sizeof(int);
-    char control[CMSG_SPACE(fd_size)];
-    struct cmsghdr *cmsg;
-
-    memset(&msgh, 0, sizeof(msgh));
-    memset(control, 0, sizeof(control));
-
-    /* set the payload */
-    iov.iov_base = (uint8_t *) buf;
-    iov.iov_len = len;
-
-    msgh.msg_iov = &iov;
-    msgh.msg_iovlen = 1;
-
-    msgh.msg_control = control;
-    msgh.msg_controllen = sizeof(control);
-
-    cmsg = CMSG_FIRSTHDR(&msgh);
-
-    cmsg->cmsg_len = CMSG_LEN(fd_size);
-    cmsg->cmsg_level = SOL_SOCKET;
-    cmsg->cmsg_type = SCM_RIGHTS;
-    memcpy(CMSG_DATA(cmsg), s->write_msgfds, fd_size);
-
-    do {
-        r = sendmsg(s->fd, &msgh, 0);
-    } while (r < 0 && errno == EINTR);
-
-    /* free the written msgfds, no matter what */
-    if (s->write_msgfds_num) {
-        g_free(s->write_msgfds);
-        s->write_msgfds = 0;
-        s->write_msgfds_num = 0;
-    }
-
-    return r;
-}
-#endif
+static gboolean tcp_chr_accept(QIOChannel *chan,
+                               GIOCondition cond,
+                               void *opaque);
 
 /* Called with chr_write_lock held.  */
 static int tcp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     TCPCharDriver *s = chr->opaque;
     if (s->connected) {
-#ifndef _WIN32
-        if (s->is_unix && s->write_msgfds_num) {
-            return unix_send_msgfds(chr, buf, len);
-        } else
-#endif
-        {
-            return io_channel_send(s->chan, buf, len);
+        int ret =  io_channel_send_full(s->ioc, buf, len,
+                                        s->write_msgfds,
+                                        s->write_msgfds_num);
+
+        /* free the written msgfds, no matter what */
+        if (s->write_msgfds_num) {
+            g_free(s->write_msgfds);
+            s->write_msgfds = 0;
+            s->write_msgfds_num = 0;
         }
+
+        return ret;
     } else {
         /* XXX: indicate an error ? */
         return len;
@@ -2710,13 +2762,16 @@ static int tcp_set_msgfds(CharDriverState *chr, int *fds, int num)
 {
     TCPCharDriver *s = chr->opaque;
 
-    /* clear old pending fd array */
-    if (s->write_msgfds) {
-        g_free(s->write_msgfds);
+    if (!qio_channel_has_feature(s->ioc,
+                                 QIO_CHANNEL_FEATURE_FD_PASS)) {
+        return -1;
     }
+    /* clear old pending fd array */
+    g_free(s->write_msgfds);
+    s->write_msgfds = NULL;
 
     if (num) {
-        s->write_msgfds = g_malloc(num * sizeof(int));
+        s->write_msgfds = g_new(int, num);
         memcpy(s->write_msgfds, fds, num * sizeof(int));
     }
 
@@ -2725,27 +2780,33 @@ static int tcp_set_msgfds(CharDriverState *chr, int *fds, int num)
     return 0;
 }
 
-#ifndef _WIN32
-static void unix_process_msgfd(CharDriverState *chr, struct msghdr *msg)
+static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
 {
     TCPCharDriver *s = chr->opaque;
-    struct cmsghdr *cmsg;
-
-    for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
-        int fd_size, i;
-
-        if (cmsg->cmsg_len < CMSG_LEN(sizeof(int)) ||
-            cmsg->cmsg_level != SOL_SOCKET ||
-            cmsg->cmsg_type != SCM_RIGHTS) {
-            continue;
-        }
-
-        fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+    struct iovec iov = { .iov_base = buf, .iov_len = len };
+    int ret;
+    size_t i;
+    int *msgfds = NULL;
+    size_t msgfds_num = 0;
+
+    if (qio_channel_has_feature(s->ioc, QIO_CHANNEL_FEATURE_FD_PASS)) {
+        ret = qio_channel_readv_full(s->ioc, &iov, 1,
+                                     &msgfds, &msgfds_num,
+                                     NULL);
+    } else {
+        ret = qio_channel_readv_full(s->ioc, &iov, 1,
+                                     NULL, NULL,
+                                     NULL);
+    }
 
-        if (!fd_size) {
-            continue;
-        }
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        ret = -1;
+    } else if (ret == -1) {
+        errno = EIO;
+    }
 
+    if (msgfds_num) {
         /* close and clean read_msgfds */
         for (i = 0; i < s->read_msgfds_num; i++) {
             close(s->read_msgfds[i]);
@@ -2755,102 +2816,62 @@ static void unix_process_msgfd(CharDriverState *chr, struct msghdr *msg)
             g_free(s->read_msgfds);
         }
 
-        s->read_msgfds_num = fd_size / sizeof(int);
-        s->read_msgfds = g_malloc(fd_size);
-        memcpy(s->read_msgfds, CMSG_DATA(cmsg), fd_size);
-
-        for (i = 0; i < s->read_msgfds_num; i++) {
-            int fd = s->read_msgfds[i];
-            if (fd < 0) {
-                continue;
-            }
-
-            /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
-            qemu_set_block(fd);
-
-    #ifndef MSG_CMSG_CLOEXEC
-            qemu_set_cloexec(fd);
-    #endif
-        }
+        s->read_msgfds = msgfds;
+        s->read_msgfds_num = msgfds_num;
     }
-}
-
-static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
-{
-    TCPCharDriver *s = chr->opaque;
-    struct msghdr msg = { NULL, };
-    struct iovec iov[1];
-    union {
-        struct cmsghdr cmsg;
-        char control[CMSG_SPACE(sizeof(int) * TCP_MAX_FDS)];
-    } msg_control;
-    int flags = 0;
-    ssize_t ret;
 
-    iov[0].iov_base = buf;
-    iov[0].iov_len = len;
+    for (i = 0; i < s->read_msgfds_num; i++) {
+        int fd = s->read_msgfds[i];
+        if (fd < 0) {
+            continue;
+        }
 
-    msg.msg_iov = iov;
-    msg.msg_iovlen = 1;
-    msg.msg_control = &msg_control;
-    msg.msg_controllen = sizeof(msg_control);
+        /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
+        qemu_set_block(fd);
 
-#ifdef MSG_CMSG_CLOEXEC
-    flags |= MSG_CMSG_CLOEXEC;
+#ifndef MSG_CMSG_CLOEXEC
+        qemu_set_cloexec(fd);
 #endif
-    do {
-        ret = recvmsg(s->fd, &msg, flags);
-    } while (ret == -1 && errno == EINTR);
-
-    if (ret > 0 && s->is_unix) {
-        unix_process_msgfd(chr, &msg);
     }
 
     return ret;
 }
-#else
-static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
-{
-    TCPCharDriver *s = chr->opaque;
-    ssize_t ret;
-
-    do {
-        ret = qemu_recv(s->fd, buf, len, 0);
-    } while (ret == -1 && socket_error() == EINTR);
-
-    return ret;
-}
-#endif
 
 static GSource *tcp_chr_add_watch(CharDriverState *chr, GIOCondition cond)
 {
     TCPCharDriver *s = chr->opaque;
-    return g_io_create_watch(s->chan, cond);
+    return qio_channel_create_watch(s->ioc, cond);
 }
 
 static void tcp_chr_disconnect(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
 
+    if (!s->connected) {
+        return;
+    }
+
     s->connected = 0;
-    if (s->listen_chan) {
-        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN,
-                                       tcp_chr_accept, chr);
+    if (s->listen_ioc) {
+        s->listen_tag = qio_channel_add_watch(
+            QIO_CHANNEL(s->listen_ioc), G_IO_IN, tcp_chr_accept, chr, NULL);
     }
+    tcp_set_msgfds(chr, NULL, 0);
     remove_fd_in_watch(chr);
-    g_io_channel_unref(s->chan);
-    s->chan = NULL;
-    closesocket(s->fd);
-    s->fd = -1;
-    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
-                         "disconnected:", s->addr, s->is_listen, s->is_telnet);
+    object_unref(OBJECT(s->sioc));
+    s->sioc = NULL;
+    object_unref(OBJECT(s->ioc));
+    s->ioc = NULL;
+    g_free(chr->filename);
+    chr->filename = SocketAddress_to_str("disconnected:", s->addr,
+                                         s->is_listen, s->is_telnet);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
     if (s->reconnect_time) {
         qemu_chr_socket_restart_timer(chr);
     }
 }
 
-static gboolean tcp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean tcp_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
@@ -2864,9 +2885,7 @@ static gboolean tcp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
     if (len > s->max_size)
         len = s->max_size;
     size = tcp_chr_recv(chr, (void *)buf, len);
-    if (size == 0 ||
-        (size < 0 &&
-         socket_error() != EAGAIN && socket_error() != EWOULDBLOCK)) {
+    if (size == 0 || size == -1) {
         /* connection closed */
         tcp_chr_disconnect(chr);
     } else if (size > 0) {
@@ -2897,42 +2916,21 @@ static int tcp_chr_sync_read(CharDriverState *chr, const uint8_t *buf, int len)
     return size;
 }
 
-#ifndef _WIN32
-CharDriverState *qemu_chr_open_eventfd(int eventfd)
-{
-    CharDriverState *chr = qemu_chr_open_fd(eventfd, eventfd);
-
-    if (chr) {
-        chr->avail_connections = 1;
-    }
-
-    return chr;
-}
-#endif
-
 static void tcp_chr_connect(void *opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
-    struct sockaddr_storage ss, ps;
-    socklen_t ss_len = sizeof(ss), ps_len = sizeof(ps);
-
-    memset(&ss, 0, ss_len);
-    if (getsockname(s->fd, (struct sockaddr *) &ss, &ss_len) != 0) {
-        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
-                 "Error in getsockname: %s\n", strerror(errno));
-    } else if (getpeername(s->fd, (struct sockaddr *) &ps, &ps_len) != 0) {
-        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
-                 "Error in getpeername: %s\n", strerror(errno));
-    } else {
-        sockaddr_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
-                        &ss, ss_len, &ps, ps_len,
-                        s->is_listen, s->is_telnet);
-    }
+
+    g_free(chr->filename);
+    chr->filename = sockaddr_to_str(
+        &s->sioc->localAddr, s->sioc->localAddrLen,
+        &s->sioc->remoteAddr, s->sioc->remoteAddrLen,
+        s->is_listen, s->is_telnet);
 
     s->connected = 1;
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, tcp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           tcp_chr_read_poll,
                                            tcp_chr_read, chr);
     }
     qemu_chr_be_generic_open(chr);
@@ -2942,83 +2940,201 @@ static void tcp_chr_update_read_handler(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
 
+    if (!s->connected) {
+        return;
+    }
+
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, tcp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           tcp_chr_read_poll,
                                            tcp_chr_read, chr);
     }
 }
 
-#define IACSET(x,a,b,c) x[0] = a; x[1] = b; x[2] = c;
-static void tcp_chr_telnet_init(int fd)
+typedef struct {
+    CharDriverState *chr;
+    char buf[12];
+    size_t buflen;
+} TCPCharDriverTelnetInit;
+
+static gboolean tcp_chr_telnet_init_io(QIOChannel *ioc,
+                                       GIOCondition cond G_GNUC_UNUSED,
+                                       gpointer user_data)
 {
-    char buf[3];
-    /* Send the telnet negotion to put telnet in binary, no echo, single char mode */
-    IACSET(buf, 0xff, 0xfb, 0x01);  /* IAC WILL ECHO */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfb, 0x03);  /* IAC WILL Suppress go ahead */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfb, 0x00);  /* IAC WILL Binary */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfd, 0x00);  /* IAC DO Binary */
-    send(fd, (char *)buf, 3, 0);
+    TCPCharDriverTelnetInit *init = user_data;
+    ssize_t ret;
+
+    ret = qio_channel_write(ioc, init->buf, init->buflen, NULL);
+    if (ret < 0) {
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            ret = 0;
+        } else {
+            tcp_chr_disconnect(init->chr);
+            return FALSE;
+        }
+    }
+    init->buflen -= ret;
+
+    if (init->buflen == 0) {
+        tcp_chr_connect(init->chr);
+        return FALSE;
+    }
+
+    memmove(init->buf, init->buf + ret, init->buflen);
+
+    return TRUE;
 }
 
-static int tcp_chr_add_client(CharDriverState *chr, int fd)
+static void tcp_chr_telnet_init(CharDriverState *chr)
+{
+    TCPCharDriver *s = chr->opaque;
+    TCPCharDriverTelnetInit *init =
+        g_new0(TCPCharDriverTelnetInit, 1);
+    size_t n = 0;
+
+    init->chr = chr;
+    init->buflen = 12;
+
+#define IACSET(x, a, b, c)                      \
+    do {                                        \
+        x[n++] = a;                             \
+        x[n++] = b;                             \
+        x[n++] = c;                             \
+    } while (0)
+
+    /* Prep the telnet negotion to put telnet in binary,
+     * no echo, single char mode */
+    IACSET(init->buf, 0xff, 0xfb, 0x01);  /* IAC WILL ECHO */
+    IACSET(init->buf, 0xff, 0xfb, 0x03);  /* IAC WILL Suppress go ahead */
+    IACSET(init->buf, 0xff, 0xfb, 0x00);  /* IAC WILL Binary */
+    IACSET(init->buf, 0xff, 0xfd, 0x00);  /* IAC DO Binary */
+
+#undef IACSET
+
+    qio_channel_add_watch(
+        s->ioc, G_IO_OUT,
+        tcp_chr_telnet_init_io,
+        init, NULL);
+}
+
+
+static void tcp_chr_tls_handshake(Object *source,
+                                  Error *err,
+                                  gpointer user_data)
 {
+    CharDriverState *chr = user_data;
     TCPCharDriver *s = chr->opaque;
-    if (s->fd != -1)
+
+    if (err) {
+        tcp_chr_disconnect(chr);
+    } else {
+        if (s->do_telnetopt) {
+            tcp_chr_telnet_init(chr);
+        } else {
+            tcp_chr_connect(chr);
+        }
+    }
+}
+
+
+static void tcp_chr_tls_init(CharDriverState *chr)
+{
+    TCPCharDriver *s = chr->opaque;
+    QIOChannelTLS *tioc;
+    Error *err = NULL;
+
+    if (s->is_listen) {
+        tioc = qio_channel_tls_new_server(
+            s->ioc, s->tls_creds,
+            NULL, /* XXX Use an ACL */
+            &err);
+    } else {
+        tioc = qio_channel_tls_new_client(
+            s->ioc, s->tls_creds,
+            s->addr->u.inet.data->host,
+            &err);
+    }
+    if (tioc == NULL) {
+        error_free(err);
+        tcp_chr_disconnect(chr);
+    }
+    object_unref(OBJECT(s->ioc));
+    s->ioc = QIO_CHANNEL(tioc);
+
+    qio_channel_tls_handshake(tioc,
+                              tcp_chr_tls_handshake,
+                              chr,
+                              NULL);
+}
+
+
+static int tcp_chr_new_client(CharDriverState *chr, QIOChannelSocket *sioc)
+{
+    TCPCharDriver *s = chr->opaque;
+    if (s->ioc != NULL) {
 	return -1;
+    }
 
-    qemu_set_nonblock(fd);
-    if (s->do_nodelay)
-        socket_set_nodelay(fd);
-    s->fd = fd;
-    s->chan = io_channel_from_socket(fd);
+    s->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(sioc));
+    s->sioc = sioc;
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(s->ioc, false, NULL);
+
+    if (s->do_nodelay) {
+        qio_channel_set_delay(s->ioc, false);
+    }
     if (s->listen_tag) {
         g_source_remove(s->listen_tag);
         s->listen_tag = 0;
     }
-    tcp_chr_connect(chr);
+
+    if (s->tls_creds) {
+        tcp_chr_tls_init(chr);
+    } else {
+        if (s->do_telnetopt) {
+            tcp_chr_telnet_init(chr);
+        } else {
+            tcp_chr_connect(chr);
+        }
+    }
 
     return 0;
 }
 
-static gboolean tcp_chr_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
+
+static int tcp_chr_add_client(CharDriverState *chr, int fd)
+{
+    int ret;
+    QIOChannelSocket *sioc;
+
+    sioc = qio_channel_socket_new_fd(fd, NULL);
+    if (!sioc) {
+        return -1;
+    }
+    ret = tcp_chr_new_client(chr, sioc);
+    object_unref(OBJECT(sioc));
+    return ret;
+}
+
+static gboolean tcp_chr_accept(QIOChannel *channel,
+                               GIOCondition cond,
+                               void *opaque)
 {
     CharDriverState *chr = opaque;
-    TCPCharDriver *s = chr->opaque;
-    struct sockaddr_in saddr;
-#ifndef _WIN32
-    struct sockaddr_un uaddr;
-#endif
-    struct sockaddr *addr;
-    socklen_t len;
-    int fd;
+    QIOChannelSocket *sioc;
 
-    for(;;) {
-#ifndef _WIN32
-	if (s->is_unix) {
-	    len = sizeof(uaddr);
-	    addr = (struct sockaddr *)&uaddr;
-	} else
-#endif
-	{
-	    len = sizeof(saddr);
-	    addr = (struct sockaddr *)&saddr;
-	}
-        fd = qemu_accept(s->listen_fd, addr, &len);
-        if (fd < 0 && errno != EINTR) {
-            s->listen_tag = 0;
-            return FALSE;
-        } else if (fd >= 0) {
-            if (s->do_telnetopt)
-                tcp_chr_telnet_init(fd);
-            break;
-        }
+    sioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(channel),
+                                     NULL);
+    if (!sioc) {
+        return TRUE;
     }
-    if (tcp_chr_add_client(chr, fd) < 0)
-	close(fd);
+
+    tcp_chr_new_client(chr, sioc);
+
+    object_unref(OBJECT(sioc));
 
     return TRUE;
 }
@@ -3033,22 +3149,16 @@ static void tcp_chr_close(CharDriverState *chr)
         s->reconnect_timer = 0;
     }
     qapi_free_SocketAddress(s->addr);
-    if (s->fd >= 0) {
-        remove_fd_in_watch(chr);
-        if (s->chan) {
-            g_io_channel_unref(s->chan);
-        }
-        closesocket(s->fd);
+    remove_fd_in_watch(chr);
+    if (s->ioc) {
+        object_unref(OBJECT(s->ioc));
     }
-    if (s->listen_fd >= 0) {
-        if (s->listen_tag) {
-            g_source_remove(s->listen_tag);
-            s->listen_tag = 0;
-        }
-        if (s->listen_chan) {
-            g_io_channel_unref(s->listen_chan);
-        }
-        closesocket(s->listen_fd);
+    if (s->listen_tag) {
+        g_source_remove(s->listen_tag);
+        s->listen_tag = 0;
+    }
+    if (s->listen_ioc) {
+        object_unref(OBJECT(s->listen_ioc));
     }
     if (s->read_msgfds_num) {
         for (i = 0; i < s->read_msgfds_num; i++) {
@@ -3056,6 +3166,9 @@ static void tcp_chr_close(CharDriverState *chr)
         }
         g_free(s->read_msgfds);
     }
+    if (s->tls_creds) {
+        object_unref(OBJECT(s->tls_creds));
+    }
     if (s->write_msgfds_num) {
         g_free(s->write_msgfds);
     }
@@ -3063,58 +3176,24 @@ static void tcp_chr_close(CharDriverState *chr)
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static void qemu_chr_finish_socket_connection(CharDriverState *chr, int fd)
-{
-    TCPCharDriver *s = chr->opaque;
-
-    if (s->is_listen) {
-        s->listen_fd = fd;
-        s->listen_chan = io_channel_from_socket(s->listen_fd);
-        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN,
-                                       tcp_chr_accept, chr);
-    } else {
-        s->connected = 1;
-        s->fd = fd;
-        socket_set_nodelay(fd);
-        s->chan = io_channel_from_socket(s->fd);
-        tcp_chr_connect(chr);
-    }
-}
 
-static void qemu_chr_socket_connected(int fd, Error *err, void *opaque)
+static void qemu_chr_socket_connected(Object *src, Error *err, void *opaque)
 {
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(src);
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
 
-    if (fd < 0) {
+    if (err) {
         check_report_connect_error(chr, err);
+        object_unref(src);
         return;
     }
 
     s->connect_err_reported = false;
-    qemu_chr_finish_socket_connection(chr, fd);
+    tcp_chr_new_client(chr, sioc);
+    object_unref(OBJECT(sioc));
 }
 
-static bool qemu_chr_open_socket_fd(CharDriverState *chr, Error **errp)
-{
-    TCPCharDriver *s = chr->opaque;
-    int fd;
-
-    if (s->is_listen) {
-        fd = socket_listen(s->addr, errp);
-    } else if (s->reconnect_time) {
-        fd = socket_connect(s->addr, errp, qemu_chr_socket_connected, chr);
-        return fd >= 0;
-    } else {
-        fd = socket_connect(s->addr, errp, NULL, NULL);
-    }
-    if (fd < 0) {
-        return false;
-    }
-
-    qemu_chr_finish_socket_connection(chr, fd);
-    return true;
-}
 
 /*********************************************************/
 /* Ring buffer chardev */
@@ -3176,13 +3255,20 @@ static void ringbuf_chr_close(struct CharDriverState *chr)
     chr->opaque = NULL;
 }
 
-static CharDriverState *qemu_chr_open_ringbuf(ChardevRingbuf *opts,
+static CharDriverState *qemu_chr_open_ringbuf(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
                                               Error **errp)
 {
+    ChardevRingbuf *opts = backend->u.ringbuf.data;
+    ChardevCommon *common = qapi_ChardevRingbuf_base(opts);
     CharDriverState *chr;
     RingBufCharDriver *d;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     d = g_malloc(sizeof(*d));
 
     d->size = opts->has_size ? opts->size : 65536;
@@ -3205,7 +3291,7 @@ static CharDriverState *qemu_chr_open_ringbuf(ChardevRingbuf *opts,
 
 fail:
     g_free(d);
-    g_free(chr);
+    qemu_chr_free_common(chr);
     return NULL;
 }
 
@@ -3235,7 +3321,12 @@ void qmp_ringbuf_write(const char *device, const char *data,
     }
 
     if (has_format && (format == DATA_FORMAT_BASE64)) {
-        write_data = g_base64_decode(data, &write_count);
+        write_data = qbase64_decode(data, -1,
+                                    &write_count,
+                                    errp);
+        if (!write_data) {
+            return;
+        }
     } else {
         write_data = (uint8_t *)data;
         write_count = strlen(data);
@@ -3444,77 +3535,109 @@ fail:
     return NULL;
 }
 
+void qemu_chr_parse_common(QemuOpts *opts, ChardevCommon *backend)
+{
+    const char *logfile = qemu_opt_get(opts, "logfile");
+
+    backend->has_logfile = logfile != NULL;
+    backend->logfile = logfile ? g_strdup(logfile) : NULL;
+
+    backend->has_logappend = true;
+    backend->logappend = qemu_opt_get_bool(opts, "logappend", false);
+}
+
+
 static void qemu_chr_parse_file_out(QemuOpts *opts, ChardevBackend *backend,
                                     Error **errp)
 {
     const char *path = qemu_opt_get(opts, "path");
+    ChardevFile *file;
 
     if (path == NULL) {
         error_setg(errp, "chardev: file: no filename given");
         return;
     }
-    backend->file = g_new0(ChardevFile, 1);
-    backend->file->out = g_strdup(path);
+    file = backend->u.file.data = g_new0(ChardevFile, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevFile_base(file));
+    file->out = g_strdup(path);
+
+    file->has_append = true;
+    file->append = qemu_opt_get_bool(opts, "append", false);
 }
 
 static void qemu_chr_parse_stdio(QemuOpts *opts, ChardevBackend *backend,
                                  Error **errp)
 {
-    backend->stdio = g_new0(ChardevStdio, 1);
-    backend->stdio->has_signal = true;
-    backend->stdio->signal = qemu_opt_get_bool(opts, "signal", true);
+    ChardevStdio *stdio;
+
+    stdio = backend->u.stdio.data = g_new0(ChardevStdio, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevStdio_base(stdio));
+    stdio->has_signal = true;
+    stdio->signal = qemu_opt_get_bool(opts, "signal", true);
 }
 
+#ifdef HAVE_CHARDEV_SERIAL
 static void qemu_chr_parse_serial(QemuOpts *opts, ChardevBackend *backend,
                                   Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *serial;
 
     if (device == NULL) {
         error_setg(errp, "chardev: serial/tty: no device path given");
         return;
     }
-    backend->serial = g_new0(ChardevHostdev, 1);
-    backend->serial->device = g_strdup(device);
+    serial = backend->u.serial.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(serial));
+    serial->device = g_strdup(device);
 }
+#endif
 
+#ifdef HAVE_CHARDEV_PARPORT
 static void qemu_chr_parse_parallel(QemuOpts *opts, ChardevBackend *backend,
                                     Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *parallel;
 
     if (device == NULL) {
         error_setg(errp, "chardev: parallel: no device path given");
         return;
     }
-    backend->parallel = g_new0(ChardevHostdev, 1);
-    backend->parallel->device = g_strdup(device);
+    parallel = backend->u.parallel.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(parallel));
+    parallel->device = g_strdup(device);
 }
+#endif
 
 static void qemu_chr_parse_pipe(QemuOpts *opts, ChardevBackend *backend,
                                 Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *dev;
 
     if (device == NULL) {
         error_setg(errp, "chardev: pipe: no device path given");
         return;
     }
-    backend->pipe = g_new0(ChardevHostdev, 1);
-    backend->pipe->device = g_strdup(device);
+    dev = backend->u.pipe.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(dev));
+    dev->device = g_strdup(device);
 }
 
 static void qemu_chr_parse_ringbuf(QemuOpts *opts, ChardevBackend *backend,
                                    Error **errp)
 {
     int val;
+    ChardevRingbuf *ringbuf;
 
-    backend->ringbuf = g_new0(ChardevRingbuf, 1);
+    ringbuf = backend->u.ringbuf.data = g_new0(ChardevRingbuf, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevRingbuf_base(ringbuf));
 
     val = qemu_opt_get_size(opts, "size", 0);
     if (val != 0) {
-        backend->ringbuf->has_size = true;
-        backend->ringbuf->size = val;
+        ringbuf->has_size = true;
+        ringbuf->size = val;
     }
 }
 
@@ -3522,13 +3645,15 @@ static void qemu_chr_parse_mux(QemuOpts *opts, ChardevBackend *backend,
                                Error **errp)
 {
     const char *chardev = qemu_opt_get(opts, "chardev");
+    ChardevMux *mux;
 
     if (chardev == NULL) {
         error_setg(errp, "chardev: mux: no chardev given");
         return;
     }
-    backend->mux = g_new0(ChardevMux, 1);
-    backend->mux->chardev = g_strdup(chardev);
+    mux = backend->u.mux.data = g_new0(ChardevMux, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevMux_base(mux));
+    mux->chardev = g_strdup(chardev);
 }
 
 static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
@@ -3542,7 +3667,9 @@ static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
     const char *path = qemu_opt_get(opts, "path");
     const char *host = qemu_opt_get(opts, "host");
     const char *port = qemu_opt_get(opts, "port");
+    const char *tls_creds = qemu_opt_get(opts, "tls-creds");
     SocketAddress *addr;
+    ChardevSocket *sock;
 
     if (!path) {
         if (!host) {
@@ -3553,39 +3680,49 @@ static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
             error_setg(errp, "chardev: socket: no port given");
             return;
         }
+    } else {
+        if (tls_creds) {
+            error_setg(errp, "TLS can only be used over TCP socket");
+            return;
+        }
     }
 
-    backend->socket = g_new0(ChardevSocket, 1);
+    sock = backend->u.socket.data = g_new0(ChardevSocket, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSocket_base(sock));
 
-    backend->socket->has_nodelay = true;
-    backend->socket->nodelay = do_nodelay;
-    backend->socket->has_server = true;
-    backend->socket->server = is_listen;
-    backend->socket->has_telnet = true;
-    backend->socket->telnet = is_telnet;
-    backend->socket->has_wait = true;
-    backend->socket->wait = is_waitconnect;
-    backend->socket->has_reconnect = true;
-    backend->socket->reconnect = reconnect;
+    sock->has_nodelay = true;
+    sock->nodelay = do_nodelay;
+    sock->has_server = true;
+    sock->server = is_listen;
+    sock->has_telnet = true;
+    sock->telnet = is_telnet;
+    sock->has_wait = true;
+    sock->wait = is_waitconnect;
+    sock->has_reconnect = true;
+    sock->reconnect = reconnect;
+    sock->tls_creds = g_strdup(tls_creds);
 
     addr = g_new0(SocketAddress, 1);
     if (path) {
-        addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-        addr->q_unix = g_new0(UnixSocketAddress, 1);
-        addr->q_unix->path = g_strdup(path);
+        UnixSocketAddress *q_unix;
+        addr->type = SOCKET_ADDRESS_KIND_UNIX;
+        q_unix = addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        q_unix->path = g_strdup(path);
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(host);
-        addr->inet->port = g_strdup(port);
-        addr->inet->has_to = qemu_opt_get(opts, "to");
-        addr->inet->to = qemu_opt_get_number(opts, "to", 0);
-        addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-        addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-        addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-        addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
-    }
-    backend->socket->addr = addr;
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = g_new(InetSocketAddress, 1);
+        *addr->u.inet.data = (InetSocketAddress) {
+            .host = g_strdup(host),
+            .port = g_strdup(port),
+            .has_to = qemu_opt_get(opts, "to"),
+            .to = qemu_opt_get_number(opts, "to", 0),
+            .has_ipv4 = qemu_opt_get(opts, "ipv4"),
+            .ipv4 = qemu_opt_get_bool(opts, "ipv4", 0),
+            .has_ipv6 = qemu_opt_get(opts, "ipv6"),
+            .ipv6 = qemu_opt_get_bool(opts, "ipv6", 0),
+        };
+    }
+    sock->addr = addr;
 }
 
 static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
@@ -3597,6 +3734,7 @@ static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
     const char *localport = qemu_opt_get(opts, "localport");
     bool has_local = false;
     SocketAddress *addr;
+    ChardevUdp *udp;
 
     if (host == NULL || strlen(host) == 0) {
         host = "localhost";
@@ -3616,27 +3754,32 @@ static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
         has_local = true;
     }
 
-    backend->udp = g_new0(ChardevUdp, 1);
+    udp = backend->u.udp.data = g_new0(ChardevUdp, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevUdp_base(udp));
 
     addr = g_new0(SocketAddress, 1);
-    addr->kind = SOCKET_ADDRESS_KIND_INET;
-    addr->inet = g_new0(InetSocketAddress, 1);
-    addr->inet->host = g_strdup(host);
-    addr->inet->port = g_strdup(port);
-    addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-    addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-    addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-    addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
-    backend->udp->remote = addr;
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup(host),
+        .port = g_strdup(port),
+        .has_ipv4 = qemu_opt_get(opts, "ipv4"),
+        .ipv4 = qemu_opt_get_bool(opts, "ipv4", 0),
+        .has_ipv6 = qemu_opt_get(opts, "ipv6"),
+        .ipv6 = qemu_opt_get_bool(opts, "ipv6", 0),
+    };
+    udp->remote = addr;
 
     if (has_local) {
-        backend->udp->has_local = true;
+        udp->has_local = true;
         addr = g_new0(SocketAddress, 1);
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(localaddr);
-        addr->inet->port = g_strdup(localport);
-        backend->udp->local = addr;
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = g_new(InetSocketAddress, 1);
+        *addr->u.inet.data = (InetSocketAddress) {
+            .host = g_strdup(localaddr),
+            .port = g_strdup(localport),
+        };
+        udp->local = addr;
     }
 }
 
@@ -3644,12 +3787,16 @@ typedef struct CharDriver {
     const char *name;
     ChardevBackendKind kind;
     void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp);
+    CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                               ChardevReturn *ret, Error **errp);
 } CharDriver;
 
 static GSList *backends;
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp))
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp))
 {
     CharDriver *s;
 
@@ -3657,6 +3804,7 @@ void register_char_driver(const char *name, ChardevBackendKind kind,
     s->name = g_strdup(name);
     s->kind = kind;
     s->parse = parse;
+    s->create = create;
 
     backends = g_slist_append(backends, s);
 }
@@ -3704,14 +3852,19 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
     }
 
     chr = NULL;
-    backend->kind = cd->kind;
+    backend->type = cd->kind;
     if (cd->parse) {
         cd->parse(opts, backend, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             goto qapi_out;
         }
+    } else {
+        ChardevCommon *cc = g_new0(ChardevCommon, 1);
+        qemu_chr_parse_common(opts, cc);
+        backend->u.null.data = cc; /* Any ChardevCommon member would work */
     }
+
     ret = qmp_chardev_add(bid ? bid : id, backend, errp);
     if (!ret) {
         goto qapi_out;
@@ -3721,9 +3874,9 @@ CharDriverState *qemu_chr_new_from_opts(QemuOpts *opts,
         qapi_free_ChardevBackend(backend);
         qapi_free_ChardevReturn(ret);
         backend = g_new0(ChardevBackend, 1);
-        backend->mux = g_new0(ChardevMux, 1);
-        backend->kind = CHARDEV_BACKEND_KIND_MUX;
-        backend->mux->chardev = g_strdup(bid);
+        backend->u.mux.data = g_new0(ChardevMux, 1);
+        backend->type = CHARDEV_BACKEND_KIND_MUX;
+        backend->u.mux.data->chardev = g_strdup(bid);
         ret = qmp_chardev_add(id, backend, errp);
         if (!ret) {
             chr = qemu_chr_find(bid);
@@ -3747,7 +3900,8 @@ err:
     return NULL;
 }
 
-CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
+CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
+                                       void (*init)(struct CharDriverState *s))
 {
     const char *p;
     CharDriverState *chr;
@@ -3773,6 +3927,21 @@ CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*in
     return chr;
 }
 
+CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
+{
+    CharDriverState *chr;
+    chr = qemu_chr_new_noreplay(label, filename, init);
+    if (chr) {
+        chr->replay = replay_mode != REPLAY_MODE_NONE;
+        if (chr->replay && chr->chr_ioctl) {
+            fprintf(stderr,
+                    "Replay: ioctl is not supported for serial devices yet\n");
+        }
+        replay_register_char_driver(chr);
+    }
+    return chr;
+}
+
 void qemu_chr_fe_set_echo(struct CharDriverState *chr, bool echo)
 {
     if (chr->chr_set_echo) {
@@ -3843,18 +4012,32 @@ void qemu_chr_fe_release(CharDriverState *s)
     s->avail_connections++;
 }
 
-void qemu_chr_delete(CharDriverState *chr)
+static void qemu_chr_free_common(CharDriverState *chr)
 {
-    QTAILQ_REMOVE(&chardevs, chr, next);
-    if (chr->chr_close) {
-        chr->chr_close(chr);
-    }
     g_free(chr->filename);
     g_free(chr->label);
     qemu_opts_del(chr->opts);
+    if (chr->logfd != -1) {
+        close(chr->logfd);
+    }
+    qemu_mutex_destroy(&chr->chr_write_lock);
     g_free(chr);
 }
 
+void qemu_chr_free(CharDriverState *chr)
+{
+    if (chr->chr_close) {
+        chr->chr_close(chr);
+    }
+    qemu_chr_free_common(chr);
+}
+
+void qemu_chr_delete(CharDriverState *chr)
+{
+    QTAILQ_REMOVE(&chardevs, chr, next);
+    qemu_chr_free(chr);
+}
+
 ChardevInfoList *qmp_query_chardev(Error **errp)
 {
     ChardevInfoList *chr_list = NULL;
@@ -3969,6 +4152,9 @@ QemuOptsList qemu_chardev_opts = {
             .name = "telnet",
             .type = QEMU_OPT_BOOL,
         },{
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+        },{
             .name = "width",
             .type = QEMU_OPT_NUMBER,
         },{
@@ -3998,6 +4184,15 @@ QemuOptsList qemu_chardev_opts = {
         },{
             .name = "chardev",
             .type = QEMU_OPT_STRING,
+        },{
+            .name = "append",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "logfile",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "logappend",
+            .type = QEMU_OPT_BOOL,
         },
         { /* end of list */ }
     },
@@ -4005,8 +4200,13 @@ QemuOptsList qemu_chardev_opts = {
 
 #ifdef _WIN32
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file.data;
+    ChardevCommon *common = qapi_ChardevFile_base(file);
     HANDLE out;
 
     if (file->has_in) {
@@ -4020,20 +4220,17 @@ static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
         error_setg(errp, "open %s failed", file->out);
         return NULL;
     }
-    return qemu_chr_open_win_file(out);
+    return qemu_chr_open_win_file(out, common, errp);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-    return qemu_chr_open_win_path(serial->device);
-}
-
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
-                                                  Error **errp)
-{
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
+    ChardevHostdev *serial = backend->u.serial.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(serial);
+    return qemu_chr_open_win_path(serial->device, common, errp);
 }
 
 #else /* WIN32 */
@@ -4050,11 +4247,22 @@ static int qmp_chardev_open_file_source(char *src, int flags,
     return fd;
 }
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file.data;
+    ChardevCommon *common = qapi_ChardevFile_base(file);
     int flags, in = -1, out;
 
-    flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
+    flags = O_WRONLY | O_CREAT | O_BINARY;
+    if (file->has_append && file->append) {
+        flags |= O_APPEND;
+    } else {
+        flags |= O_TRUNC;
+    }
+
     out = qmp_chardev_open_file_source(file->out, flags, errp);
     if (out < 0) {
         return NULL;
@@ -4069,13 +4277,17 @@ static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
         }
     }
 
-    return qemu_chr_open_fd(in, out);
+    return qemu_chr_open_fd(in, out, common, errp);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+#ifdef HAVE_CHARDEV_SERIAL
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-#ifdef HAVE_CHARDEV_TTY
+    ChardevHostdev *serial = backend->u.serial.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(serial);
     int fd;
 
     fd = qmp_chardev_open_file_source(serial->device, O_RDWR, errp);
@@ -4083,45 +4295,35 @@ static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
         return NULL;
     }
     qemu_set_nonblock(fd);
-    return qemu_chr_open_tty_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'serial' not supported");
-    return NULL;
-#endif
+    return qemu_chr_open_tty_fd(fd, common, errp);
 }
+#endif
 
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
+#ifdef HAVE_CHARDEV_PARPORT
+static CharDriverState *qmp_chardev_open_parallel(const char *id,
+                                                  ChardevBackend *backend,
+                                                  ChardevReturn *ret,
                                                   Error **errp)
 {
-#ifdef HAVE_CHARDEV_PARPORT
+    ChardevHostdev *parallel = backend->u.parallel.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(parallel);
     int fd;
 
     fd = qmp_chardev_open_file_source(parallel->device, O_RDWR, errp);
     if (fd < 0) {
         return NULL;
     }
-    return qemu_chr_open_pp_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
-#endif
+    return qemu_chr_open_pp_fd(fd, common, errp);
 }
+#endif
 
 #endif /* WIN32 */
 
-static void socket_try_connect(CharDriverState *chr)
-{
-    Error *err = NULL;
-
-    if (!qemu_chr_open_socket_fd(chr, &err)) {
-        check_report_connect_error(chr, err);
-    }
-}
-
 static gboolean socket_reconnect_timeout(gpointer opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
+    QIOChannelSocket *sioc;
 
     s->reconnect_timer = 0;
 
@@ -4129,32 +4331,74 @@ static gboolean socket_reconnect_timeout(gpointer opaque)
         return false;
     }
 
-    socket_try_connect(chr);
+    sioc = qio_channel_socket_new();
+    qio_channel_socket_connect_async(sioc, s->addr,
+                                     qemu_chr_socket_connected,
+                                     chr, NULL);
 
     return false;
 }
 
-static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
+static CharDriverState *qmp_chardev_open_socket(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
     CharDriverState *chr;
     TCPCharDriver *s;
+    ChardevSocket *sock = backend->u.socket.data;
     SocketAddress *addr = sock->addr;
     bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
     bool is_listen      = sock->has_server  ? sock->server  : true;
     bool is_telnet      = sock->has_telnet  ? sock->telnet  : false;
     bool is_waitconnect = sock->has_wait    ? sock->wait    : false;
     int64_t reconnect   = sock->has_reconnect ? sock->reconnect : 0;
+    ChardevCommon *common = qapi_ChardevSocket_base(sock);
+    QIOChannelSocket *sioc = NULL;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(TCPCharDriver));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(TCPCharDriver, 1);
 
-    s->fd = -1;
-    s->listen_fd = -1;
-    s->is_unix = addr->kind == SOCKET_ADDRESS_KIND_UNIX;
+    s->is_unix = addr->type == SOCKET_ADDRESS_KIND_UNIX;
     s->is_listen = is_listen;
     s->is_telnet = is_telnet;
     s->do_nodelay = do_nodelay;
+    if (sock->tls_creds) {
+        Object *creds;
+        creds = object_resolve_path_component(
+            object_get_objects_root(), sock->tls_creds);
+        if (!creds) {
+            error_setg(errp, "No TLS credentials with id '%s'",
+                       sock->tls_creds);
+            goto error;
+        }
+        s->tls_creds = (QCryptoTLSCreds *)
+            object_dynamic_cast(creds,
+                                TYPE_QCRYPTO_TLS_CREDS);
+        if (!s->tls_creds) {
+            error_setg(errp, "Object with id '%s' is not TLS credentials",
+                       sock->tls_creds);
+            goto error;
+        }
+        object_ref(OBJECT(s->tls_creds));
+        if (is_listen) {
+            if (s->tls_creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+                error_setg(errp, "%s",
+                           "Expected TLS credentials for server endpoint");
+                goto error;
+            }
+        } else {
+            if (s->tls_creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+                error_setg(errp, "%s",
+                           "Expected TLS credentials for client endpoint");
+                goto error;
+            }
+        }
+    }
+
     qapi_copy_SocketAddress(&s->addr, sock->addr);
 
     chr->opaque = s;
@@ -4169,9 +4413,8 @@ static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
     /* be isn't opened until we get a connection */
     chr->explicit_be_open = true;
 
-    chr->filename = g_malloc(CHR_MAX_FILENAME_SIZE);
-    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE, "disconnected:",
-                         addr, is_listen, is_telnet);
+    chr->filename = SocketAddress_to_str("disconnected:",
+                                         addr, is_listen, is_telnet);
 
     if (is_listen) {
         if (is_telnet) {
@@ -4181,42 +4424,74 @@ static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
         s->reconnect_time = reconnect;
     }
 
+    sioc = qio_channel_socket_new();
     if (s->reconnect_time) {
-        socket_try_connect(chr);
-    } else if (!qemu_chr_open_socket_fd(chr, errp)) {
-        g_free(s);
-        g_free(chr->filename);
-        g_free(chr);
-        return NULL;
-    }
-
-    if (is_listen && is_waitconnect) {
-        fprintf(stderr, "QEMU waiting for connection on: %s\n",
-                chr->filename);
-        tcp_chr_accept(s->listen_chan, G_IO_IN, chr);
-        qemu_set_nonblock(s->listen_fd);
+        qio_channel_socket_connect_async(sioc, s->addr,
+                                         qemu_chr_socket_connected,
+                                         chr, NULL);
+    } else if (s->is_listen) {
+        if (qio_channel_socket_listen_sync(sioc, s->addr, errp) < 0) {
+            goto error;
+        }
+        s->listen_ioc = sioc;
+        if (is_waitconnect) {
+            fprintf(stderr, "QEMU waiting for connection on: %s\n",
+                    chr->filename);
+            tcp_chr_accept(QIO_CHANNEL(s->listen_ioc), G_IO_IN, chr);
+        }
+        qio_channel_set_blocking(QIO_CHANNEL(s->listen_ioc), false, NULL);
+        if (!s->ioc) {
+            s->listen_tag = qio_channel_add_watch(
+                QIO_CHANNEL(s->listen_ioc), G_IO_IN, tcp_chr_accept, chr, NULL);
+        }
+    } else {
+        if (qio_channel_socket_connect_sync(sioc, s->addr, errp) < 0) {
+            goto error;
+        }
+        tcp_chr_new_client(chr, sioc);
+        object_unref(OBJECT(sioc));
     }
 
     return chr;
+
+ error:
+    if (sioc) {
+        object_unref(OBJECT(sioc));
+    }
+    if (s->tls_creds) {
+        object_unref(OBJECT(s->tls_creds));
+    }
+    g_free(s);
+    qemu_chr_free_common(chr);
+    return NULL;
 }
 
-static CharDriverState *qmp_chardev_open_udp(ChardevUdp *udp,
+static CharDriverState *qmp_chardev_open_udp(const char *id,
+                                             ChardevBackend *backend,
+                                             ChardevReturn *ret,
                                              Error **errp)
 {
-    int fd;
+    ChardevUdp *udp = backend->u.udp.data;
+    ChardevCommon *common = qapi_ChardevUdp_base(udp);
+    QIOChannelSocket *sioc = qio_channel_socket_new();
 
-    fd = socket_dgram(udp->remote, udp->local, errp);
-    if (fd < 0) {
+    if (qio_channel_socket_dgram_sync(sioc,
+                                      udp->local, udp->remote,
+                                      errp) < 0) {
+        object_unref(OBJECT(sioc));
         return NULL;
     }
-    return qemu_chr_open_udp_fd(fd);
+    return qemu_chr_open_udp(sioc, common, errp);
 }
 
 ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
                                Error **errp)
 {
     ChardevReturn *ret = g_new0(ChardevReturn, 1);
-    CharDriverState *base, *chr = NULL;
+    CharDriverState *chr = NULL;
+    Error *local_err = NULL;
+    GSList *i;
+    CharDriver *cd;
 
     chr = qemu_chr_find(id);
     if (chr) {
@@ -4225,106 +4500,40 @@ ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
         return NULL;
     }
 
-    switch (backend->kind) {
-    case CHARDEV_BACKEND_KIND_FILE:
-        chr = qmp_chardev_open_file(backend->file, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_SERIAL:
-        chr = qmp_chardev_open_serial(backend->serial, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PARALLEL:
-        chr = qmp_chardev_open_parallel(backend->parallel, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PIPE:
-        chr = qemu_chr_open_pipe(backend->pipe);
-        break;
-    case CHARDEV_BACKEND_KIND_SOCKET:
-        chr = qmp_chardev_open_socket(backend->socket, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_UDP:
-        chr = qmp_chardev_open_udp(backend->udp, errp);
-        break;
-#ifdef HAVE_CHARDEV_TTY
-    case CHARDEV_BACKEND_KIND_PTY:
-        chr = qemu_chr_open_pty(id, ret);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_NULL:
-        chr = qemu_chr_open_null();
-        break;
-    case CHARDEV_BACKEND_KIND_MUX:
-        base = qemu_chr_find(backend->mux->chardev);
-        if (base == NULL) {
-            error_setg(errp, "mux: base chardev %s not found",
-                       backend->mux->chardev);
+    for (i = backends; i; i = i->next) {
+        cd = i->data;
+
+        if (cd->kind == backend->type) {
+            chr = cd->create(id, backend, ret, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                goto out_error;
+            }
             break;
         }
-        chr = qemu_chr_open_mux(base);
-        break;
-    case CHARDEV_BACKEND_KIND_MSMOUSE:
-        chr = qemu_chr_open_msmouse();
-        break;
-#ifdef CONFIG_BRLAPI
-    case CHARDEV_BACKEND_KIND_BRAILLE:
-        chr = chr_baum_init();
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_TESTDEV:
-        chr = chr_testdev_init();
-        break;
-    case CHARDEV_BACKEND_KIND_STDIO:
-        chr = qemu_chr_open_stdio(backend->stdio);
-        break;
-#ifdef _WIN32
-    case CHARDEV_BACKEND_KIND_CONSOLE:
-        chr = qemu_chr_open_win_con();
-        break;
-#endif
-#ifdef CONFIG_SPICE
-    case CHARDEV_BACKEND_KIND_SPICEVMC:
-        chr = qemu_chr_open_spice_vmc(backend->spicevmc->type);
-        break;
-    case CHARDEV_BACKEND_KIND_SPICEPORT:
-        chr = qemu_chr_open_spice_port(backend->spiceport->fqdn);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_VC:
-        chr = vc_init(backend->vc);
-        break;
-    case CHARDEV_BACKEND_KIND_RINGBUF:
-    case CHARDEV_BACKEND_KIND_MEMORY:
-        chr = qemu_chr_open_ringbuf(backend->ringbuf, errp);
-        break;
-    default:
-        error_setg(errp, "unknown chardev backend (%d)", backend->kind);
-        break;
     }
 
-    /*
-     * Character backend open hasn't been fully converted to the Error
-     * API.  Some opens fail without setting an error.  Set a generic
-     * error then.
-     * TODO full conversion to Error API
-     */
-    if (chr == NULL && errp && !*errp) {
-        error_setg(errp, "Failed to create chardev");
+    if (chr == NULL) {
+        assert(!i);
+        error_setg(errp, "chardev backend not available");
+        goto out_error;
     }
-    if (chr) {
-        chr->label = g_strdup(id);
-        chr->avail_connections =
-            (backend->kind == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
-        if (!chr->filename) {
-            chr->filename = g_strdup(ChardevBackendKind_lookup[backend->kind]);
-        }
-        if (!chr->explicit_be_open) {
-            qemu_chr_be_event(chr, CHR_EVENT_OPENED);
-        }
-        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
-        return ret;
-    } else {
-        g_free(ret);
-        return NULL;
+
+    chr->label = g_strdup(id);
+    chr->avail_connections =
+        (backend->type == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
+    if (!chr->filename) {
+        chr->filename = g_strdup(ChardevBackendKind_lookup[backend->type]);
+    }
+    if (!chr->explicit_be_open) {
+        qemu_chr_be_event(chr, CHR_EVENT_OPENED);
     }
+    QTAILQ_INSERT_TAIL(&chardevs, chr, next);
+    return ret;
+
+out_error:
+    g_free(ret);
+    return NULL;
 }
 
 void qmp_chardev_remove(const char *id, Error **errp)
@@ -4341,37 +4550,55 @@ void qmp_chardev_remove(const char *id, Error **errp)
         error_setg(errp, "Chardev '%s' is busy", id);
         return;
     }
+    if (chr->replay) {
+        error_setg(errp,
+            "Chardev '%s' cannot be unplugged in record/replay mode", id);
+        return;
+    }
     qemu_chr_delete(chr);
 }
 
 static void register_types(void)
 {
-    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL);
+    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL,
+                         qemu_chr_open_null);
     register_char_driver("socket", CHARDEV_BACKEND_KIND_SOCKET,
-                         qemu_chr_parse_socket);
-    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp);
+                         qemu_chr_parse_socket, qmp_chardev_open_socket);
+    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp,
+                         qmp_chardev_open_udp);
     register_char_driver("ringbuf", CHARDEV_BACKEND_KIND_RINGBUF,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     register_char_driver("file", CHARDEV_BACKEND_KIND_FILE,
-                         qemu_chr_parse_file_out);
+                         qemu_chr_parse_file_out, qmp_chardev_open_file);
     register_char_driver("stdio", CHARDEV_BACKEND_KIND_STDIO,
-                         qemu_chr_parse_stdio);
+                         qemu_chr_parse_stdio, qemu_chr_open_stdio);
+#if defined HAVE_CHARDEV_SERIAL
     register_char_driver("serial", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
     register_char_driver("tty", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
+#endif
+#ifdef HAVE_CHARDEV_PARPORT
     register_char_driver("parallel", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
     register_char_driver("parport", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
-    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL);
-    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
+#endif
+#ifdef HAVE_CHARDEV_PTY
+    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL,
+                         qemu_chr_open_pty);
+#endif
+#ifdef _WIN32
+    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL,
+                         qemu_chr_open_win_con);
+#endif
     register_char_driver("pipe", CHARDEV_BACKEND_KIND_PIPE,
-                         qemu_chr_parse_pipe);
-    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux);
+                         qemu_chr_parse_pipe, qemu_chr_open_pipe);
+    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux,
+                         qemu_chr_open_mux);
     /* Bug-compatibility: */
     register_char_driver("memory", CHARDEV_BACKEND_KIND_MEMORY,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     /* this must be done after machine init, since we register FEs with muxes
      * as part of realize functions like serial_isa_realizefn when -nographic
      * is specified
diff --git a/qemu/qemu-doc.texi b/qemu/qemu-doc.texi
index 94af8c0f3..79141d358 100644
--- a/qemu/qemu-doc.texi
+++ b/qemu/qemu-doc.texi
@@ -158,7 +158,8 @@ TODO (no longer available)
 * pcsys_introduction:: Introduction
 * pcsys_quickstart::   Quick Start
 * sec_invocation::     Invocation
-* pcsys_keys::         Keys
+* pcsys_keys::         Keys in the graphical frontends
+* mux_keys::           Keys in the character backend multiplexer
 * pcsys_monitor::      QEMU Monitor
 * disk_images::        Disk Images
 * pcsys_network::      Network emulation
@@ -195,6 +196,8 @@ PCI and ISA network adapters
 @item
 Serial ports
 @item
+IPMI BMC, either and internal or external one
+@item
 Creative SoundBlaster 16 sound card
 @item
 ENSONIQ AudioPCI ES1370 sound card
@@ -257,7 +260,7 @@ Linux should boot and give you a prompt.
 
 @example
 @c man begin SYNOPSIS
-usage: qemu-system-i386 [options] [@var{disk_image}]
+@command{qemu-system-i386} [@var{options}] [@var{disk_image}]
 @c man end
 @end example
 
@@ -270,7 +273,7 @@ targets do not need a disk image.
 @c man end
 
 @node pcsys_keys
-@section Keys
+@section Keys in the graphical frontends
 
 @c man begin OPTIONS
 
@@ -320,15 +323,23 @@ Toggle mouse and keyboard grab.
 In the virtual consoles, you can use @key{Ctrl-Up}, @key{Ctrl-Down},
 @key{Ctrl-PageUp} and @key{Ctrl-PageDown} to move in the back log.
 
-@kindex Ctrl-a h
-During emulation, if you are using the @option{-nographic} option, use
-@key{Ctrl-a h} to get terminal commands:
+@c man end
+
+@node mux_keys
+@section Keys in the character backend multiplexer
+
+@c man begin OPTIONS
+
+During emulation, if you are using a character backend multiplexer
+(which is the default if you are using @option{-nographic}) then
+several commands are available via an escape sequence. These
+key sequences all start with an escape character, which is @key{Ctrl-a}
+by default, but can be changed with @option{-echr}. The list below assumes
+you're using the default.
 
 @table @key
 @item Ctrl-a h
 @kindex Ctrl-a h
-@item Ctrl-a ?
-@kindex Ctrl-a ?
 Print this help
 @item Ctrl-a x
 @kindex Ctrl-a x
@@ -344,10 +355,11 @@ Toggle console timestamps
 Send break (magic sysrq in Linux)
 @item Ctrl-a c
 @kindex Ctrl-a c
-Switch between console and monitor
+Rotate between the frontends connected to the multiplexer (usually
+this switches between the monitor and the console)
 @item Ctrl-a Ctrl-a
-@kindex Ctrl-a a
-Send Ctrl-a
+@kindex Ctrl-a Ctrl-a
+Send the escape character to the frontend
 @end table
 @c man end
 
@@ -391,6 +403,8 @@ The following commands are available:
 
 @include qemu-monitor.texi
 
+@include qemu-monitor-info.texi
+
 @subsection Integer expressions
 
 The monitor understands integers expressions for every integer
@@ -412,6 +426,7 @@ snapshots.
 * vm_snapshots::              VM snapshots
 * qemu_img_invocation::       qemu-img Invocation
 * qemu_nbd_invocation::       qemu-nbd Invocation
+* qemu_ga_invocation::        qemu-ga Invocation
 * disk_images_formats::       Disk image file formats
 * host_drives::               Using host drives
 * disk_images_fat_images::    Virtual FAT disk images
@@ -505,6 +520,11 @@ state is not saved or restored properly (in particular USB).
 
 @include qemu-nbd.texi
 
+@node qemu_ga_invocation
+@subsection @code{qemu-ga} Invocation
+
+@include qemu-ga.texi
+
 @node disk_images_formats
 @subsection Disk image file formats
 
@@ -1227,9 +1247,9 @@ echo 100 100 > /proc/sys/net/ipv4/ping_group_range
 When using the built-in TFTP server, the router is also the TFTP
 server.
 
-When using the @option{-redir} option, TCP or UDP connections can be
-redirected from the host to the guest. It allows for example to
-redirect X11, telnet or SSH connections.
+When using the @option{'-netdev user,hostfwd=...'} option, TCP or UDP
+connections can be redirected from the host to the guest. It allows for
+example to redirect X11, telnet or SSH connections.
 
 @subsection Connecting VLANs between QEMU instances
 
@@ -1242,13 +1262,18 @@ basic example.
 
 @subsection Inter-VM Shared Memory device
 
-With KVM enabled on a Linux host, a shared memory device is available.  Guests
-map a POSIX shared memory region into the guest as a PCI device that enables
-zero-copy communication to the application level of the guests.  The basic
-syntax is:
+On Linux hosts, a shared memory device is available.  The basic syntax
+is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,shm=<shm name>]
+qemu-system-x86_64 -device ivshmem-plain,memdev=@var{hostmem}
+@end example
+
+where @var{hostmem} names a host memory backend.  For a POSIX shared
+memory backend, use something like
+
+@example
+-object memory-backend-file,size=1M,share,mem-path=/dev/shm/ivshmem,id=@var{hostmem}
 @end example
 
 If desired, interrupts can be sent between guest VMs accessing the same shared
@@ -1258,28 +1283,41 @@ is qemu.git/contrib/ivshmem-server.  An example syntax when using the shared
 memory server is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,chardev=<id>]
-                 [,msi=on][,ioeventfd=on][,vectors=n][,role=peer|master]
-qemu-system-i386 -chardev socket,path=<path>,id=<id>
+# First start the ivshmem server once and for all
+ivshmem-server -p @var{pidfile} -S @var{path} -m @var{shm-name} -l @var{shm-size} -n @var{vectors}
+
+# Then start your qemu instances with matching arguments
+qemu-system-x86_64 -device ivshmem-doorbell,vectors=@var{vectors},chardev=@var{id}
+                 -chardev socket,path=@var{path},id=@var{id}
 @end example
 
 When using the server, the guest will be assigned a VM ID (>=0) that allows guests
 using the same server to communicate via interrupts.  Guests can read their
-VM ID from a device register (see example code).  Since receiving the shared
-memory region from the server is asynchronous, there is a (small) chance the
-guest may boot before the shared memory is attached.  To allow an application
-to ensure shared memory is attached, the VM ID register will return -1 (an
-invalid VM ID) until the memory is attached.  Once the shared memory is
-attached, the VM ID will return the guest's valid VM ID.  With these semantics,
-the guest application can check to ensure the shared memory is attached to the
-guest before proceeding.
-
-The @option{role} argument can be set to either master or peer and will affect
-how the shared memory is migrated.  With @option{role=master}, the guest will
-copy the shared memory on migration to the destination host.  With
-@option{role=peer}, the guest will not be able to migrate with the device attached.
-With the @option{peer} case, the device should be detached and then reattached
-after migration using the PCI hotplug support.
+VM ID from a device register (see ivshmem-spec.txt).
+
+@subsubsection Migration with ivshmem
+
+With device property @option{master=on}, the guest will copy the shared
+memory on migration to the destination host.  With @option{master=off},
+the guest will not be able to migrate with the device attached.  In the
+latter case, the device should be detached and then reattached after
+migration using the PCI hotplug support.
+
+At most one of the devices sharing the same memory can be master.  The
+master must complete migration before you plug back the other devices.
+
+@subsubsection ivshmem and hugepages
+
+Instead of specifying the <shm size> using POSIX shm, you may specify
+a memory backend that has hugepage support:
+
+@example
+qemu-system-x86_64 -object memory-backend-file,size=1G,mem-path=/dev/hugepages/my-shmem-file,share,id=mb1
+                 -device ivshmem-plain,memdev=mb1
+@end example
+
+ivshmem-server also supports hugepages mount points with the
+@option{-m} memory path argument.
 
 @node direct_linux_boot
 @section Direct Linux Boot
@@ -1379,7 +1417,7 @@ no type is given, the HCI logic corresponds to @code{-bt hci,vlan=0}.
 This USB device implements the USB Transport Layer of HCI.  Example
 usage:
 @example
-qemu-system-i386 [...OPTIONS...] -usbdevice bt:hci,vlan=3 -bt device:keyboard,vlan=3
+@command{qemu-system-i386} [...@var{OPTIONS}...] @option{-usbdevice} bt:hci,vlan=3 @option{-bt} device:keyboard,vlan=3
 @end example
 @end table
 
@@ -1766,7 +1804,7 @@ Use @code{set architecture i8086} to dump 16 bit code. Then use
 
 Advanced debugging options:
 
-The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
+The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
 @table @code
 @item maintenance packet qqemu.sstepbits
 
@@ -1862,7 +1900,8 @@ correctly instructs QEMU to shutdown at the appropriate moment.
 
 @subsubsection Share a directory between Unix and Windows
 
-See @ref{sec_invocation} about the help of the option @option{-smb}.
+See @ref{sec_invocation} about the help of the option
+@option{'-netdev user,smb=...'}.
 
 @subsubsection Windows XP security problem
 
@@ -2728,7 +2767,7 @@ qemu-i386 /usr/local/qemu-i386/wine/bin/wine \
 @subsection Command line options
 
 @example
-usage: qemu-i386 [-h] [-d] [-L path] [-s size] [-cpu model] [-g port] [-B offset] [-R size] program [arguments...]
+@command{qemu-i386} [@option{-h]} [@option{-d]} [@option{-L} @var{path}] [@option{-s} @var{size}] [@option{-cpu} @var{model}] [@option{-g} @var{port}] [@option{-B} @var{offset}] [@option{-R} @var{size}] @var{program} [@var{arguments}...]
 @end example
 
 @table @option
@@ -2870,7 +2909,7 @@ qemu-sparc64 /bin/ls
 @subsection Command line options
 
 @example
-usage: qemu-sparc64 [-h] [-d] [-L path] [-s size] [-bsd type] program [arguments...]
+@command{qemu-sparc64} [@option{-h]} [@option{-d]} [@option{-L} @var{path}] [@option{-s} @var{size}] [@option{-bsd} @var{type}] @var{program} [@var{arguments}...]
 @end example
 
 @table @option
@@ -3021,9 +3060,62 @@ and all other qemu-system-@var{target}.exe compiled for Win32.
 @node Mac OS X
 @section Mac OS X
 
-The Mac OS X patches are not fully merged in QEMU, so you should look
-at the QEMU mailing list archive to have all the necessary
-information.
+System Requirements:
+@itemize
+@item Mac OS 10.5 or higher
+@item The clang compiler shipped with Xcode 4.2 or higher,
+or GCC 4.3 or higher
+@end itemize
+
+Additional Requirements (install in order):
+@enumerate
+@item libffi: @uref{https://sourceware.org/libffi/}
+@item gettext: @uref{http://www.gnu.org/software/gettext/}
+@item glib: @uref{http://ftp.gnome.org/pub/GNOME/sources/glib/}
+@item pkg-config: @uref{http://www.freedesktop.org/wiki/Software/pkg-config/}
+@item autoconf: @uref{http://www.gnu.org/software/autoconf/autoconf.html}
+@item automake: @uref{http://www.gnu.org/software/automake/}
+@item pixman: @uref{http://www.pixman.org/}
+@end enumerate
+
+* You may find it easiest to get these from a third-party packager
+such as Homebrew, Macports, or Fink.
+
+After downloading the QEMU source code, double-click it to expand it.
+
+Then configure and make QEMU:
+@example
+./configure
+make
+@end example
+
+If you have a recent version of Mac OS X (OSX 10.7 or better
+with Xcode 4.2 or better) we recommend building QEMU with the
+default compiler provided by Apple, for your version of Mac OS X
+(which will be 'clang'). The configure script will
+automatically pick this.
+
+Note: If after the configure step you see a message like this:
+@example
+ERROR: Your compiler does not support the __thread specifier for
+       Thread-Local Storage (TLS). Please upgrade to a version that does.
+@end example
+you may have to build your own version of gcc from source. Expect that to take
+several hours. More information can be found here:
+@uref{https://gcc.gnu.org/install/} @*
+
+These are some of the third party binaries of gcc available for download:
+@itemize
+@item Homebrew: @uref{http://brew.sh/}
+@item @uref{https://www.litebeam.net/gcc/gcc_472.pkg}
+@item @uref{http://www.macports.org/ports.php?by=name&substr=gcc}
+@end itemize
+
+You can have several versions of GCC on your system. To specify a certain version,
+use the --cc and --cxx options.
+@example
+./configure --cxx=<path of your c++ compiler> --cc=<path of your c compiler> <other options>
+@end example
 
 @node Make targets
 @section Make targets
diff --git a/qemu/qemu-ga.texi b/qemu/qemu-ga.texi
new file mode 100644
index 000000000..0e53bf6b2
--- /dev/null
+++ b/qemu/qemu-ga.texi
@@ -0,0 +1,137 @@
+@example
+@c man begin SYNOPSIS
+@command{qemu-ga} [@var{OPTIONS}]
+@c man end
+@end example
+
+@c man begin DESCRIPTION
+
+The QEMU Guest Agent is a daemon intended to be run within virtual
+machines. It allows the hypervisor host to perform various operations
+in the guest, such as:
+
+@itemize
+@item
+get information from the guest
+@item
+set the guest's system time
+@item
+read/write a file
+@item
+sync and freeze the filesystems
+@item
+suspend the guest
+@item
+reconfigure guest local processors
+@item
+set user's password
+@item
+...
+@end itemize
+
+qemu-ga will read a system configuration file on startup (located at
+q@file{/etc/qemu/qemu-ga.conf} by default), then parse remaining
+configuration options on the command line. For the same key, the last
+option wins, but the lists accumulate (see below for configuration
+file format).
+
+@c man end
+
+@c man begin OPTIONS
+@table @option
+@item -m, --method=@var{method}
+  Transport method: one of @samp{unix-listen}, @samp{virtio-serial}, or
+  @samp{isa-serial} (@samp{virtio-serial} is the default).
+
+@item -p, --path=@var{path}
+  Device/socket path (the default for virtio-serial is
+  @samp{/dev/virtio-ports/org.qemu.guest_agent.0},
+  the default for isa-serial is @samp{/dev/ttyS0})
+
+@item -l, --logfile=@var{path}
+  Set log file path (default is stderr).
+
+@item -f, --pidfile=@var{path}
+  Specify pid file (default is @samp{/var/run/qemu-ga.pid}).
+
+@item -F, --fsfreeze-hook=@var{path}
+  Enable fsfreeze hook. Accepts an optional argument that specifies
+  script to run on freeze/thaw. Script will be called with
+  'freeze'/'thaw' arguments accordingly (default is
+  @samp{/etc/qemu/fsfreeze-hook}). If using -F with an argument, do
+  not follow -F with a space (for example:
+  @samp{-F/var/run/fsfreezehook.sh}).
+
+@item -t, --statedir=@var{path}
+  Specify the directory to store state information (absolute paths only,
+  default is @samp{/var/run}).
+
+@item -v, --verbose
+  Log extra debugging information.
+
+@item -V, --version
+  Print version information and exit.
+
+@item -d, --daemon
+  Daemonize after startup (detach from terminal).
+
+@item -b, --blacklist=@var{list}
+  Comma-separated list of RPCs to disable (no spaces, @samp{?} to list
+  available RPCs).
+
+@item -D, --dump-conf
+  Dump the configuration in a format compatible with @file{qemu-ga.conf}
+  and exit.
+
+@item -h, --help
+  Display this help and exit.
+@end table
+
+@c man end
+
+@c man begin FILES
+
+The syntax of the @file{qemu-ga.conf} configuration file follows the
+Desktop Entry Specification, here is a quick summary: it consists of
+groups of key-value pairs, interspersed with comments.
+
+@example
+# qemu-ga configuration sample
+[general]
+daemonize = 0
+pidfile = /var/run/qemu-ga.pid
+verbose = 0
+method = virtio-serial
+path = /dev/virtio-ports/org.qemu.guest_agent.0
+statedir = /var/run
+@end example
+
+The list of keys follows the command line options:
+@table @option
+@item daemon= boolean
+@item method= string
+@item path= string
+@item logfile= string
+@item pidfile= string
+@item fsfreeze-hook= string
+@item statedir= string
+@item verbose= boolean
+@item blacklist= string list
+@end table
+
+@c man end
+
+@ignore
+
+@setfilename qemu-ga
+@settitle QEMU Guest Agent
+
+@c man begin AUTHOR
+Michael Roth <mdroth@linux.vnet.ibm.com>
+@c man end
+
+@c man begin SEEALSO
+qemu(1)
+@c man end
+
+@end ignore
diff --git a/qemu/qemu-img-cmds.hx b/qemu/qemu-img-cmds.hx
index 95677745f..e7cded6e2 100644
--- a/qemu/qemu-img-cmds.hx
+++ b/qemu/qemu-img-cmds.hx
@@ -10,68 +10,68 @@ STEXI
 ETEXI
 
 DEF("check", img_check,
-    "check [-q] [-f fmt] [--output=ofmt] [-r [leaks | all]] [-T src_cache] filename")
+    "check [-q] [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] [-r [leaks | all]] [-T src_cache] filename")
 STEXI
-@item check [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
+@item check [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
 ETEXI
 
 DEF("create", img_create,
-    "create [-q] [-f fmt] [-o options] filename [size]")
+    "create [-q] [--object objectdef] [--image-opts] [-f fmt] [-o options] filename [size]")
 STEXI
-@item create [-q] [-f @var{fmt}] [-o @var{options}] @var{filename} [@var{size}]
+@item create [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-o @var{options}] @var{filename} [@var{size}]
 ETEXI
 
 DEF("commit", img_commit,
-    "commit [-q] [-f fmt] [-t cache] [-b base] [-d] [-p] filename")
+    "commit [-q] [--object objectdef] [--image-opts] [-f fmt] [-t cache] [-b base] [-d] [-p] filename")
 STEXI
-@item commit [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
+@item commit [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
 ETEXI
 
 DEF("compare", img_compare,
-    "compare [-f fmt] [-F fmt] [-T src_cache] [-p] [-q] [-s] filename1 filename2")
+    "compare [--object objectdef] [--image-opts] [-f fmt] [-F fmt] [-T src_cache] [-p] [-q] [-s] filename1 filename2")
 STEXI
-@item compare [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-q] [-s] @var{filename1} @var{filename2}
+@item compare [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-q] [-s] @var{filename1} @var{filename2}
 ETEXI
 
 DEF("convert", img_convert,
-    "convert [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
+    "convert [--object objectdef] [--image-opts] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
 STEXI
-@item convert [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
+@item convert [--object @var{objectdef}] [--image-opts] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
 ETEXI
 
 DEF("info", img_info,
-    "info [-f fmt] [--output=ofmt] [--backing-chain] filename")
+    "info [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] [--backing-chain] filename")
 STEXI
-@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
+@item info [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 ETEXI
 
 DEF("map", img_map,
-    "map [-f fmt] [--output=ofmt] filename")
+    "map [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] filename")
 STEXI
-@item map [-f @var{fmt}] [--output=@var{ofmt}] @var{filename}
+@item map [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [--output=@var{ofmt}] @var{filename}
 ETEXI
 
 DEF("snapshot", img_snapshot,
-    "snapshot [-q] [-l | -a snapshot | -c snapshot | -d snapshot] filename")
+    "snapshot [--object objectdef] [--image-opts] [-q] [-l | -a snapshot | -c snapshot | -d snapshot] filename")
 STEXI
-@item snapshot [-q] [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}
+@item snapshot [--object @var{objectdef}] [--image-opts] [-q] [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}
 ETEXI
 
 DEF("rebase", img_rebase,
-    "rebase [-q] [-f fmt] [-t cache] [-T src_cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
+    "rebase [--object objectdef] [--image-opts] [-q] [-f fmt] [-t cache] [-T src_cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
 STEXI
-@item rebase [-q] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
+@item rebase [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
 ETEXI
 
 DEF("resize", img_resize,
-    "resize [-q] filename [+ | -]size")
+    "resize [--object objectdef] [--image-opts] [-q] filename [+ | -]size")
 STEXI
-@item resize [-q] @var{filename} [+ | -]@var{size}
+@item resize [--object @var{objectdef}] [--image-opts] [-q] @var{filename} [+ | -]@var{size}
 ETEXI
 
 DEF("amend", img_amend,
-    "amend [-p] [-q] [-f fmt] [-t cache] -o options filename")
+    "amend [--object objectdef] [--image-opts] [-p] [-q] [-f fmt] [-t cache] -o options filename")
 STEXI
-@item amend [-p] [-q] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
+@item amend [--object @var{objectdef}] [--image-opts] [-p] [-q] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
 @end table
 ETEXI
diff --git a/qemu/qemu-img.c b/qemu/qemu-img.c
index 75f4ee442..46f2a6def 100644
--- a/qemu/qemu-img.c
+++ b/qemu/qemu-img.c
@@ -21,19 +21,23 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
-#include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "qemu/config-file.h"
 #include "qemu/option.h"
 #include "qemu/error-report.h"
-#include "qemu/osdep.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
 #include "block/qapi.h"
+#include "crypto/init.h"
 #include <getopt.h>
 
 #define QEMU_IMG_VERSION "qemu-img version " QEMU_VERSION QEMU_PKGVERSION \
@@ -47,6 +51,8 @@ typedef struct img_cmd_t {
 enum {
     OPTION_OUTPUT = 256,
     OPTION_BACKING_CHAIN = 257,
+    OPTION_OBJECT = 258,
+    OPTION_IMAGE_OPTS = 259,
 };
 
 typedef enum OutputFormat {
@@ -54,8 +60,7 @@ typedef enum OutputFormat {
     OFORMAT_HUMAN,
 } OutputFormat;
 
-/* Default to cache=writeback as data integrity is not important for qemu-tcg. */
-#define BDRV_O_FLAGS BDRV_O_CACHE_WB
+/* Default to cache=writeback as data integrity is not important for qemu-img */
 #define BDRV_DEFAULT_CACHE "writeback"
 
 static void format_print(void *opaque, const char *name)
@@ -94,6 +99,10 @@ static void QEMU_NORETURN help(void)
            "\n"
            "Command parameters:\n"
            "  'filename' is a disk image filename\n"
+           "  'objectdef' is a QEMU user creatable object definition. See the qemu(1)\n"
+           "    manual page for a description of the object properties. The most common\n"
+           "    object type is a 'secret', which is used to supply passwords and/or\n"
+           "    encryption keys.\n"
            "  'fmt' is the disk image format. It is guessed automatically in most cases\n"
            "  'cache' is the cache mode used to write the output disk image, the valid\n"
            "    options are: 'none', 'writeback' (default, except for convert), 'writethrough',\n"
@@ -154,6 +163,24 @@ static void QEMU_NORETURN help(void)
     exit(EXIT_SUCCESS);
 }
 
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+static QemuOptsList qemu_source_opts = {
+    .name = "source",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_source_opts.head),
+    .desc = {
+        { }
+    },
+};
+
 static int GCC_FMT_ATTR(2, 3) qprintf(bool quiet, const char *fmt, ...)
 {
     int ret = 0;
@@ -196,13 +223,57 @@ static int print_block_option_help(const char *filename, const char *fmt)
     return 0;
 }
 
-static BlockBackend *img_open(const char *id, const char *filename,
-                              const char *fmt, int flags,
-                              bool require_io, bool quiet)
+
+static int img_open_password(BlockBackend *blk, const char *filename,
+                             int flags, bool quiet)
 {
-    BlockBackend *blk;
     BlockDriverState *bs;
     char password[256];
+
+    bs = blk_bs(blk);
+    if (bdrv_is_encrypted(bs) && bdrv_key_required(bs) &&
+        !(flags & BDRV_O_NO_IO)) {
+        qprintf(quiet, "Disk image '%s' is encrypted.\n", filename);
+        if (qemu_read_password(password, sizeof(password)) < 0) {
+            error_report("No password given");
+            return -1;
+        }
+        if (bdrv_set_key(bs, password) < 0) {
+            error_report("invalid password");
+            return -1;
+        }
+    }
+    return 0;
+}
+
+
+static BlockBackend *img_open_opts(const char *optstr,
+                                   QemuOpts *opts, int flags, bool writethrough,
+                                   bool quiet)
+{
+    QDict *options;
+    Error *local_err = NULL;
+    BlockBackend *blk;
+    options = qemu_opts_to_qdict(opts, NULL);
+    blk = blk_new_open(NULL, NULL, options, flags, &local_err);
+    if (!blk) {
+        error_reportf_err(local_err, "Could not open '%s': ", optstr);
+        return NULL;
+    }
+    blk_set_enable_write_cache(blk, !writethrough);
+
+    if (img_open_password(blk, optstr, flags, quiet) < 0) {
+        blk_unref(blk);
+        return NULL;
+    }
+    return blk;
+}
+
+static BlockBackend *img_open_file(const char *filename,
+                                   const char *fmt, int flags,
+                                   bool writethrough, bool quiet)
+{
+    BlockBackend *blk;
     Error *local_err = NULL;
     QDict *options = NULL;
 
@@ -211,32 +282,46 @@ static BlockBackend *img_open(const char *id, const char *filename,
         qdict_put(options, "driver", qstring_from_str(fmt));
     }
 
-    blk = blk_new_open(id, filename, NULL, options, flags, &local_err);
+    blk = blk_new_open(filename, NULL, options, flags, &local_err);
     if (!blk) {
-        error_report("Could not open '%s': %s", filename,
-                     error_get_pretty(local_err));
-        error_free(local_err);
-        goto fail;
+        error_reportf_err(local_err, "Could not open '%s': ", filename);
+        return NULL;
     }
+    blk_set_enable_write_cache(blk, !writethrough);
 
-    bs = blk_bs(blk);
-    if (bdrv_is_encrypted(bs) && require_io) {
-        qprintf(quiet, "Disk image '%s' is encrypted.\n", filename);
-        if (qemu_read_password(password, sizeof(password)) < 0) {
-            error_report("No password given");
-            goto fail;
-        }
-        if (bdrv_set_key(bs, password) < 0) {
-            error_report("invalid password");
-            goto fail;
-        }
+    if (img_open_password(blk, filename, flags, quiet) < 0) {
+        blk_unref(blk);
+        return NULL;
     }
     return blk;
-fail:
-    blk_unref(blk);
-    return NULL;
 }
 
+
+static BlockBackend *img_open(bool image_opts,
+                              const char *filename,
+                              const char *fmt, int flags, bool writethrough,
+                              bool quiet)
+{
+    BlockBackend *blk;
+    if (image_opts) {
+        QemuOpts *opts;
+        if (fmt) {
+            error_report("--image-opts and --format are mutually exclusive");
+            return NULL;
+        }
+        opts = qemu_opts_parse_noisily(qemu_find_opts("source"),
+                                       filename, true);
+        if (!opts) {
+            return NULL;
+        }
+        blk = img_open_opts(filename, opts, flags, writethrough, quiet);
+    } else {
+        blk = img_open_file(filename, fmt, flags, writethrough, quiet);
+    }
+    return blk;
+}
+
+
 static int add_old_style_options(const char *fmt, QemuOpts *opts,
                                  const char *base_filename,
                                  const char *base_fmt)
@@ -277,7 +362,13 @@ static int img_create(int argc, char **argv)
     bool quiet = false;
 
     for(;;) {
-        c = getopt(argc, argv, "F:b:f:he6o:q");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "F:b:f:he6o:q",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -319,6 +410,14 @@ static int img_create(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                goto fail;
+            }
+        }   break;
         }
     }
 
@@ -334,11 +433,18 @@ static int img_create(int argc, char **argv)
     }
     optind++;
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        goto fail;
+    }
+
     /* Get image size, if specified */
     if (optind < argc) {
         int64_t sval;
         char *end;
-        sval = strtosz_suffix(argv[optind++], &end, STRTOSZ_DEFSUFFIX_B);
+        sval = qemu_strtosz_suffix(argv[optind++], &end,
+                                   QEMU_STRTOSZ_DEFSUFFIX_B);
         if (sval < 0 || *end) {
             if (sval == -ERANGE) {
                 error_report("Image size must be less than 8 EiB!");
@@ -357,10 +463,9 @@ static int img_create(int argc, char **argv)
     }
 
     bdrv_img_create(filename, fmt, base_filename, base_fmt,
-                    options, img_size, BDRV_O_FLAGS, &local_err, quiet);
+                    options, img_size, 0, &local_err, quiet);
     if (local_err) {
-        error_report("%s: %s", filename, error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "%s: ", filename);
         goto fail;
     }
 
@@ -378,8 +483,8 @@ static void dump_json_image_check(ImageCheck *check, bool quiet)
     QString *str;
     QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageCheck(qmp_output_get_visitor(ov),
-                          &check, NULL, &local_err);
+    visit_type_ImageCheck(qmp_output_get_visitor(ov), NULL, &check,
+                          &local_err);
     obj = qmp_output_get_qobject(ov);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
@@ -488,13 +593,16 @@ static int img_check(int argc, char **argv)
     BlockBackend *blk;
     BlockDriverState *bs;
     int fix = 0;
-    int flags = BDRV_O_FLAGS | BDRV_O_CHECK;
+    int flags = BDRV_O_CHECK;
+    bool writethrough;
     ImageCheck *check;
     bool quiet = false;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
     cache = BDRV_DEFAULT_CACHE;
+
     for(;;) {
         int option_index = 0;
         static const struct option long_options[] = {
@@ -502,6 +610,8 @@ static int img_check(int argc, char **argv)
             {"format", required_argument, 0, 'f'},
             {"repair", required_argument, 0, 'r'},
             {"output", required_argument, 0, OPTION_OUTPUT},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "hf:r:T:q",
@@ -538,6 +648,17 @@ static int img_check(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -554,13 +675,19 @@ static int img_check(int argc, char **argv)
         return 1;
     }
 
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", cache);
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         return 1;
     }
@@ -644,9 +771,6 @@ static void common_block_job_cb(void *opaque, int ret)
     if (ret < 0) {
         error_setg_errno(cbi->errp, -ret, "Block job failed");
     }
-
-    /* Drop this block job's reference */
-    bdrv_unref(cbi->bs);
 }
 
 static void run_block_job(BlockJob *job, Error **errp)
@@ -655,7 +779,8 @@ static void run_block_job(BlockJob *job, Error **errp)
 
     do {
         aio_poll(aio_context, true);
-        qemu_progress_print((float)job->offset / job->len * 100.f, 0);
+        qemu_progress_print(job->len ?
+                            ((float)job->offset / job->len * 100.f) : 0.0f, 0);
     } while (!job->ready);
 
     block_job_complete_sync(job, errp);
@@ -672,14 +797,23 @@ static int img_commit(int argc, char **argv)
     BlockBackend *blk;
     BlockDriverState *bs, *base_bs;
     bool progress = false, quiet = false, drop = false;
+    bool writethrough;
     Error *local_err = NULL;
     CommonBlockJobCBInfo cbi;
+    bool image_opts = false;
 
     fmt = NULL;
     cache = BDRV_DEFAULT_CACHE;
     base = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:ht:b:dpq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:ht:b:dpq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -708,6 +842,17 @@ static int img_commit(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -721,14 +866,20 @@ static int img_commit(int argc, char **argv)
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     flags = BDRV_O_RDWR | BDRV_O_UNMAP;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         return 1;
     }
@@ -747,7 +898,7 @@ static int img_commit(int argc, char **argv)
         /* This is different from QMP, which by default uses the deepest file in
          * the backing chain (i.e., the very base); however, the traditional
          * behavior of qemu-img commit is using the immediate backing file. */
-        base_bs = bs->backing_hd;
+        base_bs = backing_bs(bs);
         if (!base_bs) {
             error_setg(&local_err, "Image does not have a backing file");
             goto done;
@@ -765,12 +916,12 @@ static int img_commit(int argc, char **argv)
         goto done;
     }
 
-    /* The block job will swap base_bs and bs (which is not what we really want
-     * here, but okay) and unref base_bs (after the swap, i.e., the old top
-     * image). In order to still be able to empty that top image afterwards,
-     * increment the reference counter here preemptively. */
+    /* When the block job completes, the BlockBackend reference will point to
+     * the old backing file. In order to avoid that the top image is already
+     * deleted, so we can still empty it afterwards, increment the reference
+     * counter here preemptively. */
     if (!drop) {
-        bdrv_ref(base_bs);
+        bdrv_ref(bs);
     }
 
     run_block_job(bs->job, &local_err);
@@ -778,8 +929,8 @@ static int img_commit(int argc, char **argv)
         goto unref_backing;
     }
 
-    if (!drop && base_bs->drv->bdrv_make_empty) {
-        ret = base_bs->drv->bdrv_make_empty(base_bs);
+    if (!drop && bs->drv->bdrv_make_empty) {
+        ret = bs->drv->bdrv_make_empty(bs);
         if (ret) {
             error_setg_errno(&local_err, -ret, "Could not empty %s",
                              filename);
@@ -789,7 +940,7 @@ static int img_commit(int argc, char **argv)
 
 unref_backing:
     if (!drop) {
-        bdrv_unref(base_bs);
+        bdrv_unref(bs);
     }
 
 done:
@@ -972,15 +1123,24 @@ static int img_compare(int argc, char **argv)
     int ret = 0; /* return value - 0 Ident, 1 Different, >1 Error */
     bool progress = false, quiet = false, strict = false;
     int flags;
+    bool writethrough;
     int64_t total_sectors;
     int64_t sector_num = 0;
     int64_t nb_sectors;
     int c, pnum;
     uint64_t progress_base;
+    bool image_opts = false;
 
     cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "hf:F:T:pqs");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:F:T:pqs",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -1007,6 +1167,18 @@ static int img_compare(int argc, char **argv)
         case 's':
             strict = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                ret = 2;
+                goto out4;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -1022,29 +1194,36 @@ static int img_compare(int argc, char **argv)
     filename1 = argv[optind++];
     filename2 = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        ret = 2;
+        goto out4;
+    }
+
     /* Initialize before goto out */
     qemu_progress_init(progress, 2.0);
 
-    flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    flags = 0;
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", cache);
         ret = 2;
         goto out3;
     }
 
-    blk1 = img_open("image_1", filename1, fmt1, flags, true, quiet);
+    blk1 = img_open(image_opts, filename1, fmt1, flags, writethrough, quiet);
     if (!blk1) {
         ret = 2;
         goto out3;
     }
-    bs1 = blk_bs(blk1);
 
-    blk2 = img_open("image_2", filename2, fmt2, flags, true, quiet);
+    blk2 = img_open(image_opts, filename2, fmt2, flags, writethrough, quiet);
     if (!blk2) {
         ret = 2;
         goto out2;
     }
+    bs1 = blk_bs(blk1);
     bs2 = blk_bs(blk2);
 
     buf1 = blk_blockalign(blk1, IO_BUF_SIZE);
@@ -1075,28 +1254,52 @@ static int img_compare(int argc, char **argv)
     }
 
     for (;;) {
+        int64_t status1, status2;
+        BlockDriverState *file;
+
         nb_sectors = sectors_to_process(total_sectors, sector_num);
         if (nb_sectors <= 0) {
             break;
         }
-        allocated1 = bdrv_is_allocated_above(bs1, NULL, sector_num, nb_sectors,
-                                             &pnum1);
-        if (allocated1 < 0) {
+        status1 = bdrv_get_block_status_above(bs1, NULL, sector_num,
+                                              total_sectors1 - sector_num,
+                                              &pnum1, &file);
+        if (status1 < 0) {
             ret = 3;
             error_report("Sector allocation test failed for %s", filename1);
             goto out;
         }
+        allocated1 = status1 & BDRV_BLOCK_ALLOCATED;
 
-        allocated2 = bdrv_is_allocated_above(bs2, NULL, sector_num, nb_sectors,
-                                             &pnum2);
-        if (allocated2 < 0) {
+        status2 = bdrv_get_block_status_above(bs2, NULL, sector_num,
+                                              total_sectors2 - sector_num,
+                                              &pnum2, &file);
+        if (status2 < 0) {
             ret = 3;
             error_report("Sector allocation test failed for %s", filename2);
             goto out;
         }
-        nb_sectors = MIN(pnum1, pnum2);
+        allocated2 = status2 & BDRV_BLOCK_ALLOCATED;
+        if (pnum1) {
+            nb_sectors = MIN(nb_sectors, pnum1);
+        }
+        if (pnum2) {
+            nb_sectors = MIN(nb_sectors, pnum2);
+        }
 
-        if (allocated1 == allocated2) {
+        if (strict) {
+            if ((status1 & ~BDRV_BLOCK_OFFSET_MASK) !=
+                (status2 & ~BDRV_BLOCK_OFFSET_MASK)) {
+                ret = 1;
+                qprintf(quiet, "Strict mode: Offset %" PRId64
+                        " block status mismatch!\n",
+                        sectors_to_bytes(sector_num));
+                goto out;
+            }
+        }
+        if ((status1 & BDRV_BLOCK_ZERO) && (status2 & BDRV_BLOCK_ZERO)) {
+            nb_sectors = MIN(pnum1, pnum2);
+        } else if (allocated1 == allocated2) {
             if (allocated1) {
                 ret = blk_read(blk1, sector_num, buf1, nb_sectors);
                 if (ret < 0) {
@@ -1124,13 +1327,6 @@ static int img_compare(int argc, char **argv)
                 }
             }
         } else {
-            if (strict) {
-                ret = 1;
-                qprintf(quiet, "Strict mode: Offset %" PRId64
-                        " allocation mismatch!\n",
-                        sectors_to_bytes(sector_num));
-                goto out;
-            }
 
             if (allocated1) {
                 ret = check_empty_sectors(blk1, sector_num, nb_sectors,
@@ -1212,6 +1408,7 @@ out2:
     blk_unref(blk1);
 out3:
     qemu_progress_end();
+out4:
     return ret;
 }
 
@@ -1260,9 +1457,10 @@ static int convert_iteration_sectors(ImgConvertState *s, int64_t sector_num)
     n = MIN(s->total_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
 
     if (s->sector_next_status <= sector_num) {
+        BlockDriverState *file;
         ret = bdrv_get_block_status(blk_bs(s->src[s->src_cur]),
                                     sector_num - s->src_cur_offset,
-                                    n, &n);
+                                    n, &n, &file);
         if (ret < 0) {
             return ret;
         }
@@ -1311,10 +1509,6 @@ static int convert_read(ImgConvertState *s, int64_t sector_num, int nb_sectors,
     int n;
     int ret;
 
-    if (s->status == BLK_ZERO || s->status == BLK_BACKING_FILE) {
-        return 0;
-    }
-
     assert(nb_sectors <= s->buf_sectors);
     while (nb_sectors > 0) {
         BlockBackend *blk;
@@ -1452,7 +1646,8 @@ static int convert_do_copy(ImgConvertState *s)
             ret = n;
             goto fail;
         }
-        if (s->status == BLK_DATA) {
+        if (s->status == BLK_DATA || (!s->min_sparse && s->status == BLK_ZERO))
+        {
             s->allocated_sectors += n;
         }
         sector_num += n;
@@ -1472,17 +1667,24 @@ static int convert_do_copy(ImgConvertState *s)
             ret = n;
             goto fail;
         }
-        if (s->status == BLK_DATA) {
+        if (s->status == BLK_DATA || (!s->min_sparse && s->status == BLK_ZERO))
+        {
             allocated_done += n;
             qemu_progress_print(100.0 * allocated_done / s->allocated_sectors,
                                 0);
         }
 
-        ret = convert_read(s, sector_num, n, buf);
-        if (ret < 0) {
-            error_report("error while reading sector %" PRId64
-                         ": %s", sector_num, strerror(-ret));
-            goto fail;
+        if (s->status == BLK_DATA) {
+            ret = convert_read(s, sector_num, n, buf);
+            if (ret < 0) {
+                error_report("error while reading sector %" PRId64
+                             ": %s", sector_num, strerror(-ret));
+                goto fail;
+            }
+        } else if (!s->min_sparse && s->status == BLK_ZERO) {
+            n = MIN(n, s->buf_sectors);
+            memset(buf, 0, n * BDRV_SECTOR_SIZE);
+            s->status = BLK_DATA;
         }
 
         ret = convert_write(s, sector_num, n, buf);
@@ -1514,6 +1716,7 @@ static int img_convert(int argc, char **argv)
     int c, bs_n, bs_i, compress, cluster_sectors, skip_create;
     int64_t ret = 0;
     int progress = 0, flags, src_flags;
+    bool writethrough, src_writethrough;
     const char *fmt, *out_fmt, *cache, *src_cache, *out_baseimg, *out_filename;
     BlockDriver *drv, *proto_drv;
     BlockBackend **blk = NULL, *out_blk = NULL;
@@ -1532,6 +1735,7 @@ static int img_convert(int argc, char **argv)
     Error *local_err = NULL;
     QemuOpts *sn_opts = NULL;
     ImgConvertState state;
+    bool image_opts = false;
 
     fmt = NULL;
     out_fmt = "raw";
@@ -1541,7 +1745,14 @@ static int img_convert(int argc, char **argv)
     compress = 0;
     skip_create = 0;
     for(;;) {
-        c = getopt(argc, argv, "hf:O:B:ce6o:s:l:S:pt:T:qn");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:O:B:ce6o:s:l:S:pt:T:qn",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -1607,7 +1818,7 @@ static int img_convert(int argc, char **argv)
         {
             int64_t sval;
             char *end;
-            sval = strtosz_suffix(optarg, &end, STRTOSZ_DEFSUFFIX_B);
+            sval = qemu_strtosz_suffix(optarg, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
             if (sval < 0 || *end) {
                 error_report("Invalid minimum zero buffer size for sparse output specified");
                 ret = -1;
@@ -1632,16 +1843,31 @@ static int img_convert(int argc, char **argv)
         case 'n':
             skip_create = 1;
             break;
+        case OPTION_OBJECT:
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                goto fail_getopt;
+            }
+            break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        goto fail_getopt;
+    }
+
     /* Initialize before goto out */
     if (quiet) {
         progress = 0;
     }
     qemu_progress_init(progress, 1.0);
 
-
     bs_n = argc - optind - 1;
     out_filename = bs_n >= 1 ? argv[argc - 1] : NULL;
 
@@ -1662,8 +1888,8 @@ static int img_convert(int argc, char **argv)
         goto out;
     }
 
-    src_flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    src_flags = 0;
+    ret = bdrv_parse_cache_mode(src_cache, &src_flags, &src_writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", src_cache);
         goto out;
@@ -1677,11 +1903,8 @@ static int img_convert(int argc, char **argv)
 
     total_sectors = 0;
     for (bs_i = 0; bs_i < bs_n; bs_i++) {
-        char *id = bs_n > 1 ? g_strdup_printf("source_%d", bs_i)
-                            : g_strdup("source");
-        blk[bs_i] = img_open(id, argv[optind + bs_i], fmt, src_flags,
-                             true, quiet);
-        g_free(id);
+        blk[bs_i] = img_open(image_opts, argv[optind + bs_i],
+                             fmt, src_flags, src_writethrough, quiet);
         if (!blk[bs_i]) {
             ret = -1;
             goto out;
@@ -1712,9 +1935,7 @@ static int img_convert(int argc, char **argv)
         bdrv_snapshot_load_tmp_by_id_or_name(bs[0], snapshot_name, &local_err);
     }
     if (local_err) {
-        error_report("Failed to load snapshot: %s",
-                     error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "Failed to load snapshot: ");
         ret = -1;
         goto out;
     }
@@ -1810,21 +2031,25 @@ static int img_convert(int argc, char **argv)
         /* Create the new image */
         ret = bdrv_create(drv, out_filename, opts, &local_err);
         if (ret < 0) {
-            error_report("%s: error while converting %s: %s",
-                         out_filename, out_fmt, error_get_pretty(local_err));
-            error_free(local_err);
+            error_reportf_err(local_err, "%s: error while converting %s: ",
+                              out_filename, out_fmt);
             goto out;
         }
     }
 
     flags = min_sparse ? (BDRV_O_RDWR | BDRV_O_UNMAP) : BDRV_O_RDWR;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    out_blk = img_open("target", out_filename, out_fmt, flags, true, quiet);
+    /* XXX we should allow --image-opts to trigger use of
+     * img_open() here, but then we have trouble with
+     * the bdrv_create() call which takes different params.
+     * Not critical right now, so fix can wait...
+     */
+    out_blk = img_open_file(out_filename, out_fmt, flags, writethrough, quiet);
     if (!out_blk) {
         ret = -1;
         goto out;
@@ -1931,8 +2156,8 @@ static void dump_json_image_info_list(ImageInfoList *list)
     QString *str;
     QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageInfoList(qmp_output_get_visitor(ov),
-                             &list, NULL, &local_err);
+    visit_type_ImageInfoList(qmp_output_get_visitor(ov), NULL, &list,
+                             &local_err);
     obj = qmp_output_get_qobject(ov);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
@@ -1948,8 +2173,7 @@ static void dump_json_image_info(ImageInfo *info)
     QString *str;
     QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageInfo(qmp_output_get_visitor(ov),
-                         &info, NULL, &local_err);
+    visit_type_ImageInfo(qmp_output_get_visitor(ov), NULL, &info, &local_err);
     obj = qmp_output_get_qobject(ov);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
@@ -1991,7 +2215,8 @@ static gboolean str_equal_func(gconstpointer a, gconstpointer b)
  * image file.  If there was an error a message will have been printed to
  * stderr.
  */
-static ImageInfoList *collect_image_info_list(const char *filename,
+static ImageInfoList *collect_image_info_list(bool image_opts,
+                                              const char *filename,
                                               const char *fmt,
                                               bool chain)
 {
@@ -2015,8 +2240,8 @@ static ImageInfoList *collect_image_info_list(const char *filename,
         }
         g_hash_table_insert(filenames, (gpointer)filename, NULL);
 
-        blk = img_open("image", filename, fmt,
-                       BDRV_O_FLAGS | BDRV_O_NO_BACKING, false, false);
+        blk = img_open(image_opts, filename, fmt,
+                       BDRV_O_NO_BACKING | BDRV_O_NO_IO, false, false);
         if (!blk) {
             goto err;
         }
@@ -2041,7 +2266,10 @@ static ImageInfoList *collect_image_info_list(const char *filename,
             if (info->has_full_backing_filename) {
                 filename = info->full_backing_filename;
             } else if (info->has_backing_filename) {
-                filename = info->backing_filename;
+                error_report("Could not determine absolute backing filename,"
+                             " but backing filename '%s' present",
+                             info->backing_filename);
+                goto err;
             }
             if (info->has_backing_filename_format) {
                 fmt = info->backing_filename_format;
@@ -2064,6 +2292,7 @@ static int img_info(int argc, char **argv)
     bool chain = false;
     const char *filename, *fmt, *output;
     ImageInfoList *list;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
@@ -2074,6 +2303,8 @@ static int img_info(int argc, char **argv)
             {"format", required_argument, 0, 'f'},
             {"output", required_argument, 0, OPTION_OUTPUT},
             {"backing-chain", no_argument, 0, OPTION_BACKING_CHAIN},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "f:h",
@@ -2095,6 +2326,17 @@ static int img_info(int argc, char **argv)
         case OPTION_BACKING_CHAIN:
             chain = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2111,7 +2353,13 @@ static int img_info(int argc, char **argv)
         return 1;
     }
 
-    list = collect_image_info_list(filename, fmt, chain);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    list = collect_image_info_list(image_opts, filename, fmt, chain);
     if (!list) {
         return 1;
     }
@@ -2133,47 +2381,37 @@ static int img_info(int argc, char **argv)
     return 0;
 }
 
-
-typedef struct MapEntry {
-    int flags;
-    int depth;
-    int64_t start;
-    int64_t length;
-    int64_t offset;
-    BlockDriverState *bs;
-} MapEntry;
-
 static void dump_map_entry(OutputFormat output_format, MapEntry *e,
                            MapEntry *next)
 {
     switch (output_format) {
     case OFORMAT_HUMAN:
-        if ((e->flags & BDRV_BLOCK_DATA) &&
-            !(e->flags & BDRV_BLOCK_OFFSET_VALID)) {
+        if (e->data && !e->has_offset) {
             error_report("File contains external, encrypted or compressed clusters.");
             exit(1);
         }
-        if ((e->flags & (BDRV_BLOCK_DATA|BDRV_BLOCK_ZERO)) == BDRV_BLOCK_DATA) {
+        if (e->data && !e->zero) {
             printf("%#-16"PRIx64"%#-16"PRIx64"%#-16"PRIx64"%s\n",
-                   e->start, e->length, e->offset, e->bs->filename);
+                   e->start, e->length,
+                   e->has_offset ? e->offset : 0,
+                   e->has_filename ? e->filename : "");
         }
         /* This format ignores the distinction between 0, ZERO and ZERO|DATA.
          * Modify the flags here to allow more coalescing.
          */
-        if (next &&
-            (next->flags & (BDRV_BLOCK_DATA|BDRV_BLOCK_ZERO)) != BDRV_BLOCK_DATA) {
-            next->flags &= ~BDRV_BLOCK_DATA;
-            next->flags |= BDRV_BLOCK_ZERO;
+        if (next && (!next->data || next->zero)) {
+            next->data = false;
+            next->zero = true;
         }
         break;
     case OFORMAT_JSON:
-        printf("%s{ \"start\": %"PRId64", \"length\": %"PRId64", \"depth\": %d,"
-               " \"zero\": %s, \"data\": %s",
+        printf("%s{ \"start\": %"PRId64", \"length\": %"PRId64","
+               " \"depth\": %"PRId64", \"zero\": %s, \"data\": %s",
                (e->start == 0 ? "[" : ",\n"),
                e->start, e->length, e->depth,
-               (e->flags & BDRV_BLOCK_ZERO) ? "true" : "false",
-               (e->flags & BDRV_BLOCK_DATA) ? "true" : "false");
-        if (e->flags & BDRV_BLOCK_OFFSET_VALID) {
+               e->zero ? "true" : "false",
+               e->data ? "true" : "false");
+        if (e->has_offset) {
             printf(", \"offset\": %"PRId64"", e->offset);
         }
         putchar('}');
@@ -2190,6 +2428,8 @@ static int get_block_status(BlockDriverState *bs, int64_t sector_num,
 {
     int64_t ret;
     int depth;
+    BlockDriverState *file;
+    bool has_offset;
 
     /* As an optimization, we could cache the current range of unallocated
      * clusters in each file of the chain, and avoid querying the same
@@ -2198,7 +2438,8 @@ static int get_block_status(BlockDriverState *bs, int64_t sector_num,
 
     depth = 0;
     for (;;) {
-        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &nb_sectors);
+        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &nb_sectors,
+                                    &file);
         if (ret < 0) {
             return ret;
         }
@@ -2206,7 +2447,7 @@ static int get_block_status(BlockDriverState *bs, int64_t sector_num,
         if (ret & (BDRV_BLOCK_ZERO|BDRV_BLOCK_DATA)) {
             break;
         }
-        bs = bs->backing_hd;
+        bs = backing_bs(bs);
         if (bs == NULL) {
             ret = 0;
             break;
@@ -2215,15 +2456,44 @@ static int get_block_status(BlockDriverState *bs, int64_t sector_num,
         depth++;
     }
 
-    e->start = sector_num * BDRV_SECTOR_SIZE;
-    e->length = nb_sectors * BDRV_SECTOR_SIZE;
-    e->flags = ret & ~BDRV_BLOCK_OFFSET_MASK;
-    e->offset = ret & BDRV_BLOCK_OFFSET_MASK;
-    e->depth = depth;
-    e->bs = bs;
+    has_offset = !!(ret & BDRV_BLOCK_OFFSET_VALID);
+
+    *e = (MapEntry) {
+        .start = sector_num * BDRV_SECTOR_SIZE,
+        .length = nb_sectors * BDRV_SECTOR_SIZE,
+        .data = !!(ret & BDRV_BLOCK_DATA),
+        .zero = !!(ret & BDRV_BLOCK_ZERO),
+        .offset = ret & BDRV_BLOCK_OFFSET_MASK,
+        .has_offset = has_offset,
+        .depth = depth,
+        .has_filename = file && has_offset,
+        .filename = file && has_offset ? file->filename : NULL,
+    };
+
     return 0;
 }
 
+static inline bool entry_mergeable(const MapEntry *curr, const MapEntry *next)
+{
+    if (curr->length == 0) {
+        return false;
+    }
+    if (curr->zero != next->zero ||
+        curr->data != next->data ||
+        curr->depth != next->depth ||
+        curr->has_filename != next->has_filename ||
+        curr->has_offset != next->has_offset) {
+        return false;
+    }
+    if (curr->has_filename && strcmp(curr->filename, next->filename)) {
+        return false;
+    }
+    if (curr->has_offset && curr->offset + curr->length != next->offset) {
+        return false;
+    }
+    return true;
+}
+
 static int img_map(int argc, char **argv)
 {
     int c;
@@ -2234,6 +2504,7 @@ static int img_map(int argc, char **argv)
     int64_t length;
     MapEntry curr = { .length = 0 }, next;
     int ret = 0;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
@@ -2243,6 +2514,8 @@ static int img_map(int argc, char **argv)
             {"help", no_argument, 0, 'h'},
             {"format", required_argument, 0, 'f'},
             {"output", required_argument, 0, OPTION_OUTPUT},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "f:h",
@@ -2261,6 +2534,17 @@ static int img_map(int argc, char **argv)
         case OPTION_OUTPUT:
             output = optarg;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2277,7 +2561,13 @@ static int img_map(int argc, char **argv)
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, BDRV_O_FLAGS, true, false);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    blk = img_open(image_opts, filename, fmt, 0, false, false);
     if (!blk) {
         return 1;
     }
@@ -2305,10 +2595,7 @@ static int img_map(int argc, char **argv)
             goto out;
         }
 
-        if (curr.length != 0 && curr.flags == next.flags &&
-            curr.depth == next.depth &&
-            ((curr.flags & BDRV_BLOCK_OFFSET_VALID) == 0 ||
-             curr.offset + curr.length == next.offset)) {
+        if (entry_mergeable(&curr, &next)) {
             curr.length += next.length;
             continue;
         }
@@ -2342,11 +2629,19 @@ static int img_snapshot(int argc, char **argv)
     qemu_timeval tv;
     bool quiet = false;
     Error *err = NULL;
+    bool image_opts = false;
 
-    bdrv_oflags = BDRV_O_FLAGS | BDRV_O_RDWR;
+    bdrv_oflags = BDRV_O_RDWR;
     /* Parse commandline parameters */
     for(;;) {
-        c = getopt(argc, argv, "la:c:d:hq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "la:c:d:hq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2390,6 +2685,17 @@ static int img_snapshot(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -2398,8 +2704,14 @@ static int img_snapshot(int argc, char **argv)
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     /* Open the image */
-    blk = img_open("image", filename, NULL, bdrv_oflags, true, quiet);
+    blk = img_open(image_opts, filename, NULL, bdrv_oflags, false, quiet);
     if (!blk) {
         return 1;
     }
@@ -2437,9 +2749,8 @@ static int img_snapshot(int argc, char **argv)
     case SNAPSHOT_DELETE:
         bdrv_snapshot_delete_by_id_or_name(bs, snapshot_name, &err);
         if (err) {
-            error_report("Could not delete snapshot '%s': (%s)",
-                         snapshot_name, error_get_pretty(err));
-            error_free(err);
+            error_reportf_err(err, "Could not delete snapshot '%s': ",
+                              snapshot_name);
             ret = 1;
         }
         break;
@@ -2456,14 +2767,18 @@ static int img_snapshot(int argc, char **argv)
 static int img_rebase(int argc, char **argv)
 {
     BlockBackend *blk = NULL, *blk_old_backing = NULL, *blk_new_backing = NULL;
+    uint8_t *buf_old = NULL;
+    uint8_t *buf_new = NULL;
     BlockDriverState *bs = NULL;
     char *filename;
     const char *fmt, *cache, *src_cache, *out_basefmt, *out_baseimg;
     int c, flags, src_flags, ret;
+    bool writethrough, src_writethrough;
     int unsafe = 0;
     int progress = 0;
     bool quiet = false;
     Error *local_err = NULL;
+    bool image_opts = false;
 
     /* Parse commandline parameters */
     fmt = NULL;
@@ -2472,7 +2787,14 @@ static int img_rebase(int argc, char **argv)
     out_baseimg = NULL;
     out_basefmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "hf:F:b:upt:T:q");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:F:b:upt:T:q",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2505,6 +2827,17 @@ static int img_rebase(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -2520,30 +2853,40 @@ static int img_rebase(int argc, char **argv)
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     qemu_progress_init(progress, 2.0);
     qemu_progress_print(0, 100);
 
     flags = BDRV_O_RDWR | (unsafe ? BDRV_O_NO_BACKING : 0);
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    src_flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    src_flags = 0;
+    ret = bdrv_parse_cache_mode(src_cache, &src_flags, &src_writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", src_cache);
         goto out;
     }
 
+    /* The source files are opened read-only, don't care about WCE */
+    assert((src_flags & BDRV_O_RDWR) == 0);
+    (void) src_writethrough;
+
     /*
      * Open the images.
      *
      * Ignore the old backing file for unsafe rebase in case we want to correct
      * the reference to a renamed or moved backing file.
      */
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         ret = -1;
         goto out;
@@ -2569,12 +2912,12 @@ static int img_rebase(int argc, char **argv)
         }
 
         bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
-        blk_old_backing = blk_new_open("old_backing", backing_name, NULL,
+        blk_old_backing = blk_new_open(backing_name, NULL,
                                        options, src_flags, &local_err);
         if (!blk_old_backing) {
-            error_report("Could not open old backing file '%s': %s",
-                         backing_name, error_get_pretty(local_err));
-            error_free(local_err);
+            error_reportf_err(local_err,
+                              "Could not open old backing file '%s': ",
+                              backing_name);
             goto out;
         }
 
@@ -2586,12 +2929,12 @@ static int img_rebase(int argc, char **argv)
                 options = NULL;
             }
 
-            blk_new_backing = blk_new_open("new_backing", out_baseimg, NULL,
+            blk_new_backing = blk_new_open(out_baseimg, NULL,
                                            options, src_flags, &local_err);
             if (!blk_new_backing) {
-                error_report("Could not open new backing file '%s': %s",
-                             out_baseimg, error_get_pretty(local_err));
-                error_free(local_err);
+                error_reportf_err(local_err,
+                                  "Could not open new backing file '%s': ",
+                                  out_baseimg);
                 goto out;
             }
         }
@@ -2612,8 +2955,6 @@ static int img_rebase(int argc, char **argv)
         int64_t new_backing_num_sectors = 0;
         uint64_t sector;
         int n;
-        uint8_t * buf_old;
-        uint8_t * buf_new;
         float local_progress = 0;
 
         buf_old = blk_blockalign(blk, IO_BUF_SIZE);
@@ -2725,9 +3066,6 @@ static int img_rebase(int argc, char **argv)
             }
             qemu_progress_print(local_progress, 100);
         }
-
-        qemu_vfree(buf_old);
-        qemu_vfree(buf_new);
     }
 
     /*
@@ -2763,6 +3101,8 @@ out:
         blk_unref(blk_old_backing);
         blk_unref(blk_new_backing);
     }
+    qemu_vfree(buf_old);
+    qemu_vfree(buf_new);
 
     blk_unref(blk);
     if (ret) {
@@ -2780,6 +3120,7 @@ static int img_resize(int argc, char **argv)
     bool quiet = false;
     BlockBackend *blk = NULL;
     QemuOpts *param;
+
     static QemuOptsList resize_options = {
         .name = "resize_options",
         .head = QTAILQ_HEAD_INITIALIZER(resize_options.head),
@@ -2793,6 +3134,7 @@ static int img_resize(int argc, char **argv)
             }
         },
     };
+    bool image_opts = false;
 
     /* Remove size from argv manually so that negative numbers are not treated
      * as options by getopt. */
@@ -2806,7 +3148,14 @@ static int img_resize(int argc, char **argv)
     /* Parse getopt arguments */
     fmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:hq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:hq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2821,6 +3170,17 @@ static int img_resize(int argc, char **argv)
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2828,6 +3188,12 @@ static int img_resize(int argc, char **argv)
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     /* Choose grow, shrink, or absolute resize mode */
     switch (size[0]) {
     case '+':
@@ -2855,8 +3221,8 @@ static int img_resize(int argc, char **argv)
     n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0);
     qemu_opts_del(param);
 
-    blk = img_open("image", filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR,
-                   true, quiet);
+    blk = img_open(image_opts, filename, fmt,
+                   BDRV_O_RDWR, false, quiet);
     if (!blk) {
         ret = -1;
         goto out;
@@ -2897,7 +3263,8 @@ out:
 }
 
 static void amend_status_cb(BlockDriverState *bs,
-                            int64_t offset, int64_t total_work_size)
+                            int64_t offset, int64_t total_work_size,
+                            void *opaque)
 {
     qemu_progress_print(100.f * offset / total_work_size, 0);
 }
@@ -2911,13 +3278,22 @@ static int img_amend(int argc, char **argv)
     QemuOpts *opts = NULL;
     const char *fmt = NULL, *filename, *cache;
     int flags;
+    bool writethrough;
     bool quiet = false, progress = false;
     BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
+    bool image_opts = false;
 
     cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "ho:f:t:pq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "ho:f:t:pq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2931,7 +3307,7 @@ static int img_amend(int argc, char **argv)
                 if (!is_valid_option_list(optarg)) {
                     error_report("Invalid option list: %s", optarg);
                     ret = -1;
-                    goto out;
+                    goto out_no_progress;
                 }
                 if (!options) {
                     options = g_strdup(optarg);
@@ -2953,6 +3329,17 @@ static int img_amend(int argc, char **argv)
             case 'q':
                 quiet = true;
                 break;
+            case OPTION_OBJECT:
+                opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                               optarg, true);
+                if (!opts) {
+                    ret = -1;
+                    goto out_no_progress;
+                }
+                break;
+            case OPTION_IMAGE_OPTS:
+                image_opts = true;
+                break;
         }
     }
 
@@ -2960,6 +3347,13 @@ static int img_amend(int argc, char **argv)
         error_exit("Must specify options (-o)");
     }
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        ret = -1;
+        goto out_no_progress;
+    }
+
     if (quiet) {
         progress = false;
     }
@@ -2979,14 +3373,14 @@ static int img_amend(int argc, char **argv)
         goto out;
     }
 
-    flags = BDRV_O_FLAGS | BDRV_O_RDWR;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    flags = BDRV_O_RDWR;
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         ret = -1;
         goto out;
@@ -3021,7 +3415,7 @@ static int img_amend(int argc, char **argv)
 
     /* In case the driver does not call amend_status_cb() */
     qemu_progress_print(0.f, 0);
-    ret = bdrv_amend_options(bs, opts, &amend_status_cb);
+    ret = bdrv_amend_options(bs, opts, &amend_status_cb, NULL);
     qemu_progress_print(100.f, 0);
     if (ret < 0) {
         error_report("Error while amending options: %s", strerror(-ret));
@@ -3031,6 +3425,7 @@ static int img_amend(int argc, char **argv)
 out:
     qemu_progress_end();
 
+out_no_progress:
     blk_unref(blk);
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
@@ -3075,12 +3470,21 @@ int main(int argc, char **argv)
         exit(EXIT_FAILURE);
     }
 
+    if (qcrypto_init(&local_error) < 0) {
+        error_reportf_err(local_error, "cannot initialize crypto: ");
+        exit(1);
+    }
+
+    module_call_init(MODULE_INIT_QOM);
     bdrv_init();
     if (argc < 2) {
         error_exit("Not enough arguments");
     }
     cmdname = argv[1];
 
+    qemu_add_opts(&qemu_object_opts);
+    qemu_add_opts(&qemu_source_opts);
+
     /* find the command */
     for (cmd = img_cmds; cmd->name != NULL; cmd++) {
         if (!strcmp(cmdname, cmd->name)) {
diff --git a/qemu/qemu-img.texi b/qemu/qemu-img.texi
index 0a1ab3598..afaebdd40 100644
--- a/qemu/qemu-img.texi
+++ b/qemu/qemu-img.texi
@@ -1,6 +1,6 @@
 @example
 @c man begin SYNOPSIS
-usage: qemu-img command [command options]
+@command{qemu-img} @var{command} [@var{command} @var{options}]
 @c man end
 @end example
 
@@ -24,6 +24,20 @@ Command parameters:
 @table @var
 @item filename
  is a disk image filename
+
+@item --object @var{objectdef}
+
+is a QEMU user creatable object definition. See the @code{qemu(1)} manual
+page for a description of the object properties. The most common object
+type is a @code{secret}, which is used to supply passwords and/or encryption
+keys.
+
+@item --image-opts
+
+Indicates that the @var{filename} parameter is to be interpreted as a
+full option string, not a plain filename. This parameter is mutually
+exclusive with the @var{-f} and @var{-F} parameters.
+
 @item fmt
 is the disk image format. It is guessed automatically in most cases. See below
 for a description of the supported disk formats.
@@ -103,7 +117,7 @@ First image format
 @item -F
 Second image format
 @item -s
-Strict mode - fail on on different image size or sector allocation
+Strict mode - fail on different image size or sector allocation
 @end table
 
 Parameters to convert subcommand:
diff --git a/qemu/qemu-io-cmds.c b/qemu/qemu-io-cmds.c
index 53477e1e1..e34f77711 100644
--- a/qemu/qemu-io-cmds.c
+++ b/qemu/qemu-io-cmds.c
@@ -8,6 +8,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-io.h"
 #include "sysemu/block-backend.h"
 #include "block/block.h"
@@ -17,6 +19,7 @@
 #include "qemu/main-loop.h"
 #include "qemu/timer.h"
 #include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
 #define CMD_NOFILE_OK   0x01
 
@@ -136,7 +139,29 @@ static char **breakline(char *input, int *count)
 static int64_t cvtnum(const char *s)
 {
     char *end;
-    return strtosz_suffix(s, &end, STRTOSZ_DEFSUFFIX_B);
+    int64_t ret;
+
+    ret = qemu_strtosz_suffix(s, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
+    if (*end != '\0') {
+        /* Detritus at the end of the string */
+        return -EINVAL;
+    }
+    return ret;
+}
+
+static void print_cvtnum_err(int64_t rc, const char *arg)
+{
+    switch (rc) {
+    case -EINVAL:
+        printf("Parsing error: non-numeric argument,"
+               " or extraneous/unrecognized suffix -- %s\n", arg);
+        break;
+    case -ERANGE:
+        printf("Parsing error: argument too large -- %s\n", arg);
+        break;
+    default:
+        printf("Parsing error: %s\n", arg);
+    }
 }
 
 #define EXABYTES(x)     ((long long)(x) << 60)
@@ -294,9 +319,10 @@ static void qemu_io_free(void *p)
     qemu_vfree(p);
 }
 
-static void dump_buffer(const void *buffer, int64_t offset, int len)
+static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
 {
-    int i, j;
+    uint64_t i;
+    int j;
     const uint8_t *p;
 
     for (i = 0, p = buffer; i < len; i += 16) {
@@ -319,7 +345,7 @@ static void dump_buffer(const void *buffer, int64_t offset, int len)
 }
 
 static void print_report(const char *op, struct timeval *t, int64_t offset,
-                         int count, int total, int cnt, int Cflag)
+                         int64_t count, int64_t total, int cnt, int Cflag)
 {
     char s1[64], s2[64], ts[64];
 
@@ -327,12 +353,12 @@ static void print_report(const char *op, struct timeval *t, int64_t offset,
     if (!Cflag) {
         cvtstr((double)total, s1, sizeof(s1));
         cvtstr(tdiv((double)total, *t), s2, sizeof(s2));
-        printf("%s %d/%d bytes at offset %" PRId64 "\n",
+        printf("%s %"PRId64"/%"PRId64" bytes at offset %" PRId64 "\n",
                op, total, count, offset);
         printf("%s, %d ops; %s (%s/sec and %.4f ops/sec)\n",
                s1, cnt, ts, s2, tdiv((double)cnt, *t));
     } else {/* bytes,ops,time,bytes/sec,ops/sec */
-        printf("%d,%d,%s,%.3f,%.3f\n",
+        printf("%"PRId64",%d,%s,%.3f,%.3f\n",
             total, cnt, ts,
             tdiv((double)total, *t),
             tdiv((double)cnt, *t));
@@ -359,13 +385,13 @@ create_iovec(BlockBackend *blk, QEMUIOVector *qiov, char **argv, int nr_iov,
 
         len = cvtnum(arg);
         if (len < 0) {
-            printf("non-numeric length argument -- %s\n", arg);
+            print_cvtnum_err(len, arg);
             goto fail;
         }
 
         /* should be SIZE_T_MAX, but that doesn't exist */
         if (len > INT_MAX) {
-            printf("too large length argument -- %s\n", arg);
+            printf("Argument '%s' exceeds maximum size %d\n", arg, INT_MAX);
             goto fail;
         }
 
@@ -393,11 +419,15 @@ fail:
     return buf;
 }
 
-static int do_read(BlockBackend *blk, char *buf, int64_t offset, int count,
-                   int *total)
+static int do_read(BlockBackend *blk, char *buf, int64_t offset, int64_t count,
+                   int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_read(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -406,11 +436,15 @@ static int do_read(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_write(BlockBackend *blk, char *buf, int64_t offset, int count,
-                    int *total)
+static int do_write(BlockBackend *blk, char *buf, int64_t offset, int64_t count,
+                    int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_write(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -419,9 +453,13 @@ static int do_write(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_pread(BlockBackend *blk, char *buf, int64_t offset, int count,
-                    int *total)
+static int do_pread(BlockBackend *blk, char *buf, int64_t offset,
+                    int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_pread(blk, offset, (uint8_t *)buf, count);
     if (*total < 0) {
         return *total;
@@ -429,9 +467,13 @@ static int do_pread(BlockBackend *blk, char *buf, int64_t offset, int count,
     return 1;
 }
 
-static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset, int count,
-                     int *total)
+static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset,
+                     int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_pwrite(blk, offset, (uint8_t *)buf, count);
     if (*total < 0) {
         return *total;
@@ -442,8 +484,8 @@ static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset, int count,
 typedef struct {
     BlockBackend *blk;
     int64_t offset;
-    int count;
-    int *total;
+    int64_t count;
+    int64_t *total;
     int ret;
     bool done;
 } CoWriteZeroes;
@@ -463,8 +505,8 @@ static void coroutine_fn co_write_zeroes_entry(void *opaque)
     *data->total = data->count;
 }
 
-static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
-                              int *total)
+static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int64_t count,
+                              int64_t *total)
 {
     Coroutine *co;
     CoWriteZeroes data = {
@@ -475,6 +517,10 @@ static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
         .done   = false,
     };
 
+    if (count >> BDRV_SECTOR_BITS > INT_MAX) {
+        return -ERANGE;
+    }
+
     co = qemu_coroutine_create(co_write_zeroes_entry);
     qemu_coroutine_enter(co, &data);
     while (!data.done) {
@@ -488,10 +534,14 @@ static int do_co_write_zeroes(BlockBackend *blk, int64_t offset, int count,
 }
 
 static int do_write_compressed(BlockBackend *blk, char *buf, int64_t offset,
-                               int count, int *total)
+                               int64_t count, int64_t *total)
 {
     int ret;
 
+    if (count >> 9 > INT_MAX) {
+        return -ERANGE;
+    }
+
     ret = blk_write_compressed(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
@@ -501,8 +551,12 @@ static int do_write_compressed(BlockBackend *blk, char *buf, int64_t offset,
 }
 
 static int do_load_vmstate(BlockBackend *blk, char *buf, int64_t offset,
-                           int count, int *total)
+                           int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_load_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
@@ -511,8 +565,12 @@ static int do_load_vmstate(BlockBackend *blk, char *buf, int64_t offset,
 }
 
 static int do_save_vmstate(BlockBackend *blk, char *buf, int64_t offset,
-                           int count, int *total)
+                           int64_t count, int64_t *total)
 {
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
     *total = blk_save_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
@@ -642,10 +700,11 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
     int c, cnt;
     char *buf;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
-    int pattern = 0, pattern_offset = 0, pattern_count = 0;
+    int64_t total = 0;
+    int pattern = 0;
+    int64_t pattern_offset = 0, pattern_count = 0;
 
     while ((c = getopt(argc, argv, "bCl:pP:qs:v")) != -1) {
         switch (c) {
@@ -659,7 +718,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             lflag = 1;
             pattern_count = cvtnum(optarg);
             if (pattern_count < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_count, optarg);
                 return 0;
             }
             break;
@@ -680,7 +739,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             sflag = 1;
             pattern_offset = cvtnum(optarg);
             if (pattern_offset < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_offset, optarg);
                 return 0;
             }
             break;
@@ -703,14 +762,18 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
@@ -734,7 +797,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
             return 0;
         }
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
@@ -762,7 +825,7 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
         memset(cmp_buf, pattern, pattern_count);
         if (memcmp(buf + pattern_offset, cmp_buf, pattern_count)) {
             printf("Pattern verification failed at offset %"
-                   PRId64 ", %d bytes\n",
+                   PRId64 ", %"PRId64" bytes\n",
                    offset + pattern_offset, pattern_count);
         }
         g_free(cmp_buf);
@@ -861,7 +924,7 @@ static int readv_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
@@ -957,9 +1020,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     int c, cnt;
     char *buf = NULL;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
+    int64_t total = 0;
     int pattern = 0xcd;
 
     while ((c = getopt(argc, argv, "bcCpP:qz")) != -1) {
@@ -1010,14 +1073,18 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
@@ -1029,7 +1096,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         }
 
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
@@ -1142,7 +1209,7 @@ static int writev_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
@@ -1269,7 +1336,7 @@ static int multiwrite_f(BlockBackend *blk, int argc, char **argv)
         /* Read the offset of the request */
         offset = cvtnum(argv[optind]);
         if (offset < 0) {
-            printf("non-numeric offset argument -- %s\n", argv[optind]);
+            print_cvtnum_err(offset, argv[optind]);
             goto out;
         }
         optind++;
@@ -1349,6 +1416,7 @@ struct aio_ctx {
     int vflag;
     int Cflag;
     int Pflag;
+    int zflag;
     BlockAcctCookie acct;
     int pattern;
     struct timeval t1;
@@ -1364,6 +1432,7 @@ static void aio_write_done(void *opaque, int ret)
 
     if (ret < 0) {
         printf("aio_write failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
@@ -1378,8 +1447,10 @@ static void aio_write_done(void *opaque, int ret)
     print_report("wrote", &t2, ctx->offset, ctx->qiov.size,
                  ctx->qiov.size, 1, ctx->Cflag);
 out:
-    qemu_io_free(ctx->buf);
-    qemu_iovec_destroy(&ctx->qiov);
+    if (!ctx->zflag) {
+        qemu_io_free(ctx->buf);
+        qemu_iovec_destroy(&ctx->qiov);
+    }
     g_free(ctx);
 }
 
@@ -1392,6 +1463,7 @@ static void aio_read_done(void *opaque, int ret)
 
     if (ret < 0) {
         printf("readv failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
@@ -1496,7 +1568,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
 
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
@@ -1505,6 +1577,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
     if (ctx->offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
                ctx->offset);
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
         g_free(ctx);
         return 0;
     }
@@ -1512,6 +1585,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
     nr_iov = argc - optind;
     ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov, 0xab);
     if (ctx->buf == NULL) {
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
         g_free(ctx);
         return 0;
     }
@@ -1541,6 +1615,7 @@ static void aio_write_help(void)
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
+" -z, -- write zeroes using blk_aio_write_zeroes\n"
 "\n");
 }
 
@@ -1551,7 +1626,7 @@ static const cmdinfo_t aio_write_cmd = {
     .cfunc      = aio_write_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-Cq] [-P pattern ] off len [len..]",
+    .args       = "[-Cqz] [-P pattern ] off len [len..]",
     .oneline    = "asynchronously writes a number of bytes",
     .help       = aio_write_help,
 };
@@ -1563,7 +1638,7 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
     struct aio_ctx *ctx = g_new0(struct aio_ctx, 1);
 
     ctx->blk = blk;
-    while ((c = getopt(argc, argv, "CqP:")) != -1) {
+    while ((c = getopt(argc, argv, "CqP:z")) != -1) {
         switch (c) {
         case 'C':
             ctx->Cflag = 1;
@@ -1578,6 +1653,9 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
                 return 0;
             }
             break;
+        case 'z':
+            ctx->zflag = 1;
+            break;
         default:
             g_free(ctx);
             return qemuio_command_usage(&aio_write_cmd);
@@ -1589,9 +1667,21 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
         return qemuio_command_usage(&aio_write_cmd);
     }
 
+    if (ctx->zflag && optind != argc - 2) {
+        printf("-z supports only a single length parameter\n");
+        g_free(ctx);
+        return 0;
+    }
+
+    if (ctx->zflag && ctx->Pflag) {
+        printf("-z and -P cannot be specified at the same time\n");
+        g_free(ctx);
+        return 0;
+    }
+
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
@@ -1600,28 +1690,47 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
     if (ctx->offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
                ctx->offset);
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
         g_free(ctx);
         return 0;
     }
 
-    nr_iov = argc - optind;
-    ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov, pattern);
-    if (ctx->buf == NULL) {
-        g_free(ctx);
-        return 0;
-    }
+    if (ctx->zflag) {
+        int64_t count = cvtnum(argv[optind]);
+        if (count < 0) {
+            print_cvtnum_err(count, argv[optind]);
+            return 0;
+        }
 
-    gettimeofday(&ctx->t1, NULL);
-    block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
-                     BLOCK_ACCT_WRITE);
-    blk_aio_writev(blk, ctx->offset >> 9, &ctx->qiov,
-                   ctx->qiov.size >> 9, aio_write_done, ctx);
+        ctx->qiov.size = count;
+        blk_aio_write_zeroes(blk, ctx->offset >> 9, count >> 9, 0,
+                             aio_write_done, ctx);
+    } else {
+        nr_iov = argc - optind;
+        ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov,
+                                pattern);
+        if (ctx->buf == NULL) {
+            block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
+            g_free(ctx);
+            return 0;
+        }
+
+        gettimeofday(&ctx->t1, NULL);
+        block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
+                         BLOCK_ACCT_WRITE);
+
+        blk_aio_writev(blk, ctx->offset >> 9, &ctx->qiov,
+                       ctx->qiov.size >> 9, aio_write_done, ctx);
+    }
     return 0;
 }
 
 static int aio_flush_f(BlockBackend *blk, int argc, char **argv)
 {
+    BlockAcctCookie cookie;
+    block_acct_start(blk_get_stats(blk), &cookie, 0, BLOCK_ACCT_FLUSH);
     blk_drain_all();
+    block_acct_done(blk_get_stats(blk), &cookie);
     return 0;
 }
 
@@ -1651,7 +1760,7 @@ static int truncate_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric truncate argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     }
 
@@ -1777,8 +1886,7 @@ static int discard_f(BlockBackend *blk, int argc, char **argv)
     struct timeval t1, t2;
     int Cflag = 0, qflag = 0;
     int c, ret;
-    int64_t offset;
-    int count;
+    int64_t offset, count;
 
     while ((c = getopt(argc, argv, "Cq")) != -1) {
         switch (c) {
@@ -1799,14 +1907,19 @@ static int discard_f(BlockBackend *blk, int argc, char **argv)
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count >> BDRV_SECTOR_BITS > INT_MAX) {
+        printf("length cannot exceed %"PRIu64", given %s\n",
+               (uint64_t)INT_MAX << BDRV_SECTOR_BITS,
+               argv[optind]);
         return 0;
     }
 
@@ -1833,15 +1946,14 @@ out:
 static int alloc_f(BlockBackend *blk, int argc, char **argv)
 {
     BlockDriverState *bs = blk_bs(blk);
-    int64_t offset, sector_num;
-    int nb_sectors, remaining;
+    int64_t offset, sector_num, nb_sectors, remaining;
     char s1[64];
-    int num, sum_alloc;
-    int ret;
+    int num, ret;
+    int64_t sum_alloc;
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric offset argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     } else if (offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
@@ -1852,7 +1964,11 @@ static int alloc_f(BlockBackend *blk, int argc, char **argv)
     if (argc == 3) {
         nb_sectors = cvtnum(argv[2]);
         if (nb_sectors < 0) {
-            printf("non-numeric length argument -- %s\n", argv[2]);
+            print_cvtnum_err(nb_sectors, argv[2]);
+            return 0;
+        } else if (nb_sectors > INT_MAX) {
+            printf("length argument cannot exceed %d, given %s\n",
+                   INT_MAX, argv[2]);
             return 0;
         }
     } else {
@@ -1881,7 +1997,7 @@ static int alloc_f(BlockBackend *blk, int argc, char **argv)
 
     cvtstr(offset, s1, sizeof(s1));
 
-    printf("%d/%d sectors allocated at offset %s\n",
+    printf("%"PRId64"/%"PRId64" sectors allocated at offset %s\n",
            sum_alloc, nb_sectors, s1);
     return 0;
 }
@@ -1979,6 +2095,106 @@ static const cmdinfo_t map_cmd = {
        .oneline        = "prints the allocated areas of a file",
 };
 
+static void reopen_help(void)
+{
+    printf(
+"\n"
+" Changes the open options of an already opened image\n"
+"\n"
+" Example:\n"
+" 'reopen -o lazy-refcounts=on' - activates lazy refcount writeback on a qcow2 image\n"
+"\n"
+" -r, -- Reopen the image read-only\n"
+" -c, -- Change the cache mode to the given value\n"
+" -o, -- Changes block driver options (cf. 'open' command)\n"
+"\n");
+}
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv);
+
+static QemuOptsList reopen_opts = {
+    .name = "reopen",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(reopen_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static const cmdinfo_t reopen_cmd = {
+       .name           = "reopen",
+       .argmin         = 0,
+       .argmax         = -1,
+       .cfunc          = reopen_f,
+       .args           = "[-r] [-c cache] [-o options]",
+       .oneline        = "reopens an image with new options",
+       .help           = reopen_help,
+};
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv)
+{
+    BlockDriverState *bs = blk_bs(blk);
+    QemuOpts *qopts;
+    QDict *opts;
+    int c;
+    int flags = bs->open_flags;
+    bool writethrough = !blk_enable_write_cache(blk);
+
+    BlockReopenQueue *brq;
+    Error *local_err = NULL;
+
+    while ((c = getopt(argc, argv, "c:o:r")) != -1) {
+        switch (c) {
+        case 'c':
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) < 0) {
+                error_report("Invalid cache option: %s", optarg);
+                return 0;
+            }
+            break;
+        case 'o':
+            if (!qemu_opts_parse_noisily(&reopen_opts, optarg, 0)) {
+                qemu_opts_reset(&reopen_opts);
+                return 0;
+            }
+            break;
+        case 'r':
+            flags &= ~BDRV_O_RDWR;
+            break;
+        default:
+            qemu_opts_reset(&reopen_opts);
+            return qemuio_command_usage(&reopen_cmd);
+        }
+    }
+
+    if (optind != argc) {
+        qemu_opts_reset(&reopen_opts);
+        return qemuio_command_usage(&reopen_cmd);
+    }
+
+    if (writethrough != blk_enable_write_cache(blk) &&
+        blk_get_attached_dev(blk))
+    {
+        error_report("Cannot change cache.writeback: Device attached");
+        qemu_opts_reset(&reopen_opts);
+        return 0;
+    }
+
+    qopts = qemu_opts_find(&reopen_opts, NULL);
+    opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
+    qemu_opts_reset(&reopen_opts);
+
+    brq = bdrv_reopen_queue(NULL, bs, opts, flags);
+    bdrv_reopen_multiple(brq, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    } else {
+        blk_set_enable_write_cache(blk, !writethrough);
+    }
+
+    return 0;
+}
+
 static int break_f(BlockBackend *blk, int argc, char **argv)
 {
     int ret;
@@ -2102,9 +2318,13 @@ static const cmdinfo_t sigraise_cmd = {
 
 static int sigraise_f(BlockBackend *blk, int argc, char **argv)
 {
-    int sig = cvtnum(argv[1]);
+    int64_t sig = cvtnum(argv[1]);
     if (sig < 0) {
-        printf("non-numeric signal number argument -- %s\n", argv[1]);
+        print_cvtnum_err(sig, argv[1]);
+        return 0;
+    } else if (sig > NSIG) {
+        printf("signal argument '%s' is too large to be a valid signal\n",
+               argv[1]);
         return 0;
     }
 
@@ -2266,6 +2486,7 @@ static void __attribute((constructor)) init_qemuio_commands(void)
     qemuio_add_command(&discard_cmd);
     qemuio_add_command(&alloc_cmd);
     qemuio_add_command(&map_cmd);
+    qemuio_add_command(&reopen_cmd);
     qemuio_add_command(&break_cmd);
     qemuio_add_command(&remove_break_cmd);
     qemuio_add_command(&resume_cmd);
diff --git a/qemu/qemu-io.c b/qemu/qemu-io.c
index f1e3a6713..0598251e7 100644
--- a/qemu/qemu-io.c
+++ b/qemu/qemu-io.c
@@ -7,13 +7,11 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <sys/time.h>
-#include <sys/types.h>
-#include <stdarg.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <getopt.h>
 #include <libgen.h>
 
+#include "qapi/error.h"
 #include "qemu-io.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
@@ -21,9 +19,11 @@
 #include "qemu/config-file.h"
 #include "qemu/readline.h"
 #include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "trace/control.h"
+#include "crypto/init.h"
 
 #define CMD_NOFILE_OK   0x01
 
@@ -34,6 +34,7 @@ static BlockBackend *qemuio_blk;
 /* qemu-io commands passed using -c */
 static int ncmdline;
 static char **cmdline;
+static bool imageOpts;
 
 static ReadLineState *readline_state;
 
@@ -51,28 +52,26 @@ static const cmdinfo_t close_cmd = {
     .oneline    = "close the current open file",
 };
 
-static int openfile(char *name, int flags, QDict *opts)
+static int openfile(char *name, int flags, bool writethrough, QDict *opts)
 {
     Error *local_err = NULL;
     BlockDriverState *bs;
 
     if (qemuio_blk) {
-        fprintf(stderr, "file open already, try 'help close'\n");
+        error_report("file open already, try 'help close'");
         QDECREF(opts);
         return 1;
     }
 
-    qemuio_blk = blk_new_open("hda", name, NULL, opts, flags, &local_err);
+    qemuio_blk = blk_new_open(name, NULL, opts, flags, &local_err);
     if (!qemuio_blk) {
-        fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
-                name ? " device " : "", name ?: "",
-                error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "can't open%s%s: ",
+                          name ? " device " : "", name ?: "");
         return 1;
     }
 
     bs = blk_bs(qemuio_blk);
-    if (bdrv_is_encrypted(bs)) {
+    if (bdrv_is_encrypted(bs) && bdrv_key_required(bs)) {
         char password[256];
         printf("Disk image '%s' is encrypted.\n", name);
         if (qemu_read_password(password, sizeof(password)) < 0) {
@@ -85,6 +84,7 @@ static int openfile(char *name, int flags, QDict *opts)
         }
     }
 
+    blk_set_enable_write_cache(qemuio_blk, !writethrough);
 
     return 0;
 
@@ -139,6 +139,7 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
 {
     int flags = 0;
     int readonly = 0;
+    bool writethrough = true;
     int c;
     QemuOpts *qopts;
     QDict *opts;
@@ -149,14 +150,18 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
             flags |= BDRV_O_SNAPSHOT;
             break;
         case 'n':
-            flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+            flags |= BDRV_O_NOCACHE;
+            writethrough = false;
             break;
         case 'r':
             readonly = 1;
             break;
         case 'o':
+            if (imageOpts) {
+                printf("--image-opts and 'open -o' are mutually exclusive\n");
+                return 0;
+            }
             if (!qemu_opts_parse_noisily(&empty_opts, optarg, false)) {
-                printf("could not parse option list -- %s\n", optarg);
                 qemu_opts_reset(&empty_opts);
                 return 0;
             }
@@ -171,14 +176,22 @@ static int open_f(BlockBackend *blk, int argc, char **argv)
         flags |= BDRV_O_RDWR;
     }
 
+    if (imageOpts && (optind == argc - 1)) {
+        if (!qemu_opts_parse_noisily(&empty_opts, argv[optind], false)) {
+            qemu_opts_reset(&empty_opts);
+            return 0;
+        }
+        optind++;
+    }
+
     qopts = qemu_opts_find(&empty_opts, NULL);
     opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
     qemu_opts_reset(&empty_opts);
 
     if (optind == argc - 1) {
-        return openfile(argv[optind], flags, opts);
+        return openfile(argv[optind], flags, writethrough, opts);
     } else if (optind == argc) {
-        return openfile(NULL, flags, opts);
+        return openfile(NULL, flags, writethrough, opts);
     } else {
         QDECREF(opts);
         return qemuio_command_usage(&open_cmd);
@@ -206,6 +219,8 @@ static void usage(const char *name)
 "Usage: %s [-h] [-V] [-rsnm] [-f FMT] [-c STRING] ... [file]\n"
 "QEMU Disk exerciser\n"
 "\n"
+"  --object OBJECTDEF   define an object such as 'secret' for\n"
+"                       passwords and/or encryption keys\n"
 "  -c, --cmd STRING     execute command with its arguments\n"
 "                       from the given string\n"
 "  -f, --format FMT     specifies the block driver to use\n"
@@ -367,31 +382,60 @@ static void reenable_tty_echo(void)
     qemu_set_tty_echo(STDIN_FILENO, true);
 }
 
+enum {
+    OPTION_OBJECT = 256,
+    OPTION_IMAGE_OPTS = 257,
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+
+static QemuOptsList file_opts = {
+    .name = "file",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(file_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
 int main(int argc, char **argv)
 {
     int readonly = 0;
     const char *sopt = "hVc:d:f:rsnmgkt:T:";
     const struct option lopt[] = {
-        { "help", 0, NULL, 'h' },
-        { "version", 0, NULL, 'V' },
-        { "offset", 1, NULL, 'o' },
-        { "cmd", 1, NULL, 'c' },
-        { "format", 1, NULL, 'f' },
-        { "read-only", 0, NULL, 'r' },
-        { "snapshot", 0, NULL, 's' },
-        { "nocache", 0, NULL, 'n' },
-        { "misalign", 0, NULL, 'm' },
-        { "native-aio", 0, NULL, 'k' },
-        { "discard", 1, NULL, 'd' },
-        { "cache", 1, NULL, 't' },
-        { "trace", 1, NULL, 'T' },
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "offset", required_argument, NULL, 'o' },
+        { "cmd", required_argument, NULL, 'c' },
+        { "format", required_argument, NULL, 'f' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "snapshot", no_argument, NULL, 's' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "misalign", no_argument, NULL, 'm' },
+        { "native-aio", no_argument, NULL, 'k' },
+        { "discard", required_argument, NULL, 'd' },
+        { "cache", required_argument, NULL, 't' },
+        { "trace", required_argument, NULL, 'T' },
+        { "object", required_argument, NULL, OPTION_OBJECT },
+        { "image-opts", no_argument, NULL, OPTION_IMAGE_OPTS },
         { NULL, 0, NULL, 0 }
     };
     int c;
     int opt_index = 0;
     int flags = BDRV_O_UNMAP;
+    bool writethrough = true;
     Error *local_error = NULL;
     QDict *opts = NULL;
+    const char *format = NULL;
 
 #ifdef CONFIG_POSIX
     signal(SIGPIPE, SIG_IGN);
@@ -400,6 +444,13 @@ int main(int argc, char **argv)
     progname = basename(argv[0]);
     qemu_init_exec_dir(argv[0]);
 
+    if (qcrypto_init(&local_error) < 0) {
+        error_reportf_err(local_error, "cannot initialize crypto: ");
+        exit(1);
+    }
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_object_opts);
     bdrv_init();
 
     while ((c = getopt_long(argc, argv, sopt, lopt, &opt_index)) != -1) {
@@ -408,7 +459,8 @@ int main(int argc, char **argv)
             flags |= BDRV_O_SNAPSHOT;
             break;
         case 'n':
-            flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+            flags |= BDRV_O_NOCACHE;
+            writethrough = false;
             break;
         case 'd':
             if (bdrv_parse_discard_flags(optarg, &flags) < 0) {
@@ -417,10 +469,7 @@ int main(int argc, char **argv)
             }
             break;
         case 'f':
-            if (!opts) {
-                opts = qdict_new();
-            }
-            qdict_put(opts, "driver", qstring_from_str(optarg));
+            format = optarg;
             break;
         case 'c':
             add_user_command(optarg);
@@ -435,13 +484,13 @@ int main(int argc, char **argv)
             flags |= BDRV_O_NATIVE_AIO;
             break;
         case 't':
-            if (bdrv_parse_cache_flags(optarg, &flags) < 0) {
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) < 0) {
                 error_report("Invalid cache option: %s", optarg);
                 exit(1);
             }
             break;
         case 'T':
-            if (!trace_init_backends(optarg, NULL)) {
+            if (!trace_init_backends()) {
                 exit(1); /* error message will have been printed */
             }
             break;
@@ -451,6 +500,17 @@ int main(int argc, char **argv)
         case 'h':
             usage(progname);
             exit(0);
+        case OPTION_OBJECT: {
+            QemuOpts *qopts;
+            qopts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                            optarg, true);
+            if (!qopts) {
+                exit(1);
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            imageOpts = true;
+            break;
         default:
             usage(progname);
             exit(1);
@@ -462,11 +522,22 @@ int main(int argc, char **argv)
         exit(1);
     }
 
+    if (format && imageOpts) {
+        error_report("--image-opts and -f are mutually exclusive");
+        exit(1);
+    }
+
     if (qemu_init_main_loop(&local_error)) {
         error_report_err(local_error);
         exit(1);
     }
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        exit(1);
+    }
+
     /* initialize commands */
     qemuio_add_command(&quit_cmd);
     qemuio_add_command(&open_cmd);
@@ -487,7 +558,21 @@ int main(int argc, char **argv)
     }
 
     if ((argc - optind) == 1) {
-        openfile(argv[optind], flags, opts);
+        if (imageOpts) {
+            QemuOpts *qopts = NULL;
+            qopts = qemu_opts_parse_noisily(&file_opts, argv[optind], false);
+            if (!qopts) {
+                exit(1);
+            }
+            opts = qemu_opts_to_qdict(qopts, NULL);
+            openfile(NULL, flags, writethrough, opts);
+        } else {
+            if (format) {
+                opts = qdict_new();
+                qdict_put(opts, "driver", qstring_from_str(format));
+            }
+            openfile(argv[optind], flags, writethrough, opts);
+        }
     }
     command_loop();
 
diff --git a/qemu/qemu-log.c b/qemu/qemu-log.c
deleted file mode 100644
index 13f3813f6..000000000
--- a/qemu/qemu-log.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Logging support
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "qemu-common.h"
-#include "qemu/log.h"
-
-static char *logfilename;
-FILE *qemu_logfile;
-int qemu_loglevel;
-static int log_append = 0;
-
-void qemu_log(const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    if (qemu_logfile) {
-        vfprintf(qemu_logfile, fmt, ap);
-    }
-    va_end(ap);
-}
-
-void qemu_log_mask(int mask, const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    if ((qemu_loglevel & mask) && qemu_logfile) {
-        vfprintf(qemu_logfile, fmt, ap);
-    }
-    va_end(ap);
-}
-
-/* enable or disable low levels log */
-void do_qemu_set_log(int log_flags, bool use_own_buffers)
-{
-    qemu_loglevel = log_flags;
-    if (qemu_loglevel && !qemu_logfile) {
-        if (logfilename) {
-            qemu_logfile = fopen(logfilename, log_append ? "a" : "w");
-            if (!qemu_logfile) {
-                perror(logfilename);
-                _exit(1);
-            }
-        } else {
-            /* Default to stderr if no log file specified */
-            qemu_logfile = stderr;
-        }
-        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
-        if (use_own_buffers) {
-            static char logfile_buf[4096];
-
-            setvbuf(qemu_logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
-        } else {
-#if defined(_WIN32)
-            /* Win32 doesn't support line-buffering, so use unbuffered output. */
-            setvbuf(qemu_logfile, NULL, _IONBF, 0);
-#else
-            setvbuf(qemu_logfile, NULL, _IOLBF, 0);
-#endif
-            log_append = 1;
-        }
-    }
-    if (!qemu_loglevel && qemu_logfile) {
-        qemu_log_close();
-    }
-}
-
-void qemu_set_log_filename(const char *filename)
-{
-    g_free(logfilename);
-    logfilename = g_strdup(filename);
-    qemu_log_close();
-    qemu_set_log(qemu_loglevel);
-}
-
-const QEMULogItem qemu_log_items[] = {
-    { CPU_LOG_TB_OUT_ASM, "out_asm",
-      "show generated host assembly code for each compiled TB" },
-    { CPU_LOG_TB_IN_ASM, "in_asm",
-      "show target assembly code for each compiled TB" },
-    { CPU_LOG_TB_OP, "op",
-      "show micro ops for each compiled TB" },
-    { CPU_LOG_TB_OP_OPT, "op_opt",
-      "show micro ops (x86 only: before eflags optimization) and\n"
-      "after liveness analysis" },
-    { CPU_LOG_INT, "int",
-      "show interrupts/exceptions in short format" },
-    { CPU_LOG_EXEC, "exec",
-      "show trace before each executed TB (lots of logs)" },
-    { CPU_LOG_TB_CPU, "cpu",
-      "show CPU state before block translation" },
-    { CPU_LOG_MMU, "mmu",
-      "log MMU-related activities" },
-    { CPU_LOG_PCALL, "pcall",
-      "x86 only: show protected mode far calls/returns/exceptions" },
-    { CPU_LOG_RESET, "cpu_reset",
-      "show CPU state before CPU resets" },
-    { CPU_LOG_IOPORT, "ioport",
-      "show all i/o ports accesses" },
-    { LOG_UNIMP, "unimp",
-      "log unimplemented functionality" },
-    { LOG_GUEST_ERROR, "guest_errors",
-      "log when the guest OS does something invalid (eg accessing a\n"
-      "non-existent register)" },
-    { 0, NULL, NULL },
-};
-
-static int cmp1(const char *s1, int n, const char *s2)
-{
-    if (strlen(s2) != n) {
-        return 0;
-    }
-    return memcmp(s1, s2, n) == 0;
-}
-
-/* takes a comma separated list of log masks. Return 0 if error. */
-int qemu_str_to_log_mask(const char *str)
-{
-    const QEMULogItem *item;
-    int mask;
-    const char *p, *p1;
-
-    p = str;
-    mask = 0;
-    for (;;) {
-        p1 = strchr(p, ',');
-        if (!p1) {
-            p1 = p + strlen(p);
-        }
-        if (cmp1(p,p1-p,"all")) {
-            for (item = qemu_log_items; item->mask != 0; item++) {
-                mask |= item->mask;
-            }
-        } else {
-            for (item = qemu_log_items; item->mask != 0; item++) {
-                if (cmp1(p, p1 - p, item->name)) {
-                    goto found;
-                }
-            }
-            return 0;
-        }
-    found:
-        mask |= item->mask;
-        if (*p1 != ',') {
-            break;
-        }
-        p = p1 + 1;
-    }
-    return mask;
-}
-
-void qemu_print_log_usage(FILE *f)
-{
-    const QEMULogItem *item;
-    fprintf(f, "Log items (comma separated):\n");
-    for (item = qemu_log_items; item->mask != 0; item++) {
-        fprintf(f, "%-10s %s\n", item->name, item->help);
-    }
-}
diff --git a/qemu/qemu-nbd.c b/qemu/qemu-nbd.c
index 5106b802e..c55b40ffc 100644
--- a/qemu/qemu-nbd.c
+++ b/qemu/qemu-nbd.c
@@ -16,45 +16,48 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "block/nbd.h"
 #include "qemu/main-loop.h"
-#include "qemu/sockets.h"
 #include "qemu/error-report.h"
+#include "qemu/config-file.h"
 #include "block/snapshot.h"
 #include "qapi/util.h"
 #include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
+#include "io/channel-socket.h"
+#include "crypto/init.h"
 
-#include <stdarg.h>
-#include <stdio.h>
 #include <getopt.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <signal.h>
 #include <libgen.h>
 #include <pthread.h>
 
 #define SOCKET_PATH                "/var/lock/qemu-nbd-%s"
-#define QEMU_NBD_OPT_CACHE         1
-#define QEMU_NBD_OPT_AIO           2
-#define QEMU_NBD_OPT_DISCARD       3
-#define QEMU_NBD_OPT_DETECT_ZEROES 4
+#define QEMU_NBD_OPT_CACHE         256
+#define QEMU_NBD_OPT_AIO           257
+#define QEMU_NBD_OPT_DISCARD       258
+#define QEMU_NBD_OPT_DETECT_ZEROES 259
+#define QEMU_NBD_OPT_OBJECT        260
+#define QEMU_NBD_OPT_TLSCREDS      261
+#define QEMU_NBD_OPT_IMAGE_OPTS    262
 
 static NBDExport *exp;
+static bool newproto;
 static int verbose;
 static char *srcpath;
-static char *sockpath;
+static SocketAddress *saddr;
 static int persistent = 0;
 static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
 static int shared = 1;
 static int nb_fds;
-static int server_fd;
+static QIOChannelSocket *server_ioc;
+static int server_watch = -1;
+static QCryptoTLSCreds *tlscreds;
 
 static void usage(const char *name)
 {
@@ -73,11 +76,15 @@ static void usage(const char *name)
 "  -e, --shared=NUM          device can be shared by NUM clients (default '1')\n"
 "  -t, --persistent          don't exit on the last connection\n"
 "  -v, --verbose             display extra debugging information\n"
+"  -x, --export-name=NAME    expose export by name\n"
 "\n"
 "Exposing part of the image:\n"
 "  -o, --offset=OFFSET       offset into the image\n"
 "  -P, --partition=NUM       only expose partition NUM\n"
 "\n"
+"General purpose options:\n"
+"  --object type,id=ID,...   define an object such as 'secret' for providing\n"
+"                            passwords and/or encryption keys\n"
 #ifdef __linux__
 "Kernel NBD client support:\n"
 "  -c, --connect=DEV         connect FILE to the local NBD device DEV\n"
@@ -98,11 +105,10 @@ static void usage(const char *name)
 "                            '[ID_OR_NAME]'\n"
 "  -n, --nocache             disable host cache\n"
 "      --cache=MODE          set cache mode (none, writeback, ...)\n"
-#ifdef CONFIG_LINUX_AIO
 "      --aio=MODE            set AIO mode (native or threads)\n"
-#endif
 "      --discard=MODE        set discard mode (ignore, unmap)\n"
-"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, discard)\n"
+"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, unmap)\n"
+"      --image-opts          treat FILE as a full set of image options\n"
 "\n"
 "Report bugs to <qemu-devel@nongnu.org>\n"
     , name, NBD_DEFAULT_PORT, "DEVICE");
@@ -159,8 +165,8 @@ static int find_partition(BlockBackend *blk, int partition,
     int ret;
 
     if ((ret = blk_read(blk, 0, data, 1)) < 0) {
-        errno = -ret;
-        err(EXIT_FAILURE, "error while reading");
+        error_report("error while reading: %s", strerror(-ret));
+        exit(EXIT_FAILURE);
     }
 
     if (data[510] != 0x55 || data[511] != 0xaa) {
@@ -180,8 +186,8 @@ static int find_partition(BlockBackend *blk, int partition,
             int j;
 
             if ((ret = blk_read(blk, mbr[i].start_sector_abs, data1, 1)) < 0) {
-                errno = -ret;
-                err(EXIT_FAILURE, "error while reading");
+                error_report("error while reading: %s", strerror(-ret));
+                exit(EXIT_FAILURE);
             }
 
             for (j = 0; j < 4; j++) {
@@ -209,56 +215,10 @@ static int find_partition(BlockBackend *blk, int partition,
 
 static void termsig_handler(int signum)
 {
-    state = TERMINATE;
+    atomic_cmpxchg(&state, RUNNING, TERMINATE);
     qemu_notify_event();
 }
 
-static void combine_addr(char *buf, size_t len, const char* address,
-                         uint16_t port)
-{
-    /* If the address-part contains a colon, it's an IPv6 IP so needs [] */
-    if (strstr(address, ":")) {
-        snprintf(buf, len, "[%s]:%u", address, port);
-    } else {
-        snprintf(buf, len, "%s:%u", address, port);
-    }
-}
-
-static int tcp_socket_incoming(const char *address, uint16_t port)
-{
-    char address_and_port[128];
-    Error *local_err = NULL;
-
-    combine_addr(address_and_port, 128, address, port);
-    int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_incoming(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_listen(path, NULL, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_outgoing(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_connect(path, &local_err);
-
-    if (local_err != NULL) {
-        error_report_err(local_err);
-    }
-    return fd;
-}
 
 static void *show_parts(void *arg)
 {
@@ -282,22 +242,26 @@ static void *nbd_client_thread(void *arg)
     char *device = arg;
     off_t size;
     uint32_t nbdflags;
-    int fd, sock;
+    QIOChannelSocket *sioc;
+    int fd;
     int ret;
     pthread_t show_parts_thread;
     Error *local_error = NULL;
 
-    sock = unix_socket_outgoing(sockpath);
-    if (sock < 0) {
+    sioc = qio_channel_socket_new();
+    if (qio_channel_socket_connect_sync(sioc,
+                                        saddr,
+                                        &local_error) < 0) {
+        error_report_err(local_error);
         goto out;
     }
 
-    ret = nbd_receive_negotiate(sock, NULL, &nbdflags,
+    ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), NULL, &nbdflags,
+                                NULL, NULL, NULL,
                                 &size, &local_error);
     if (ret < 0) {
         if (local_error) {
-            fprintf(stderr, "%s\n", error_get_pretty(local_error));
-            error_free(local_error);
+            error_report_err(local_error);
         }
         goto out_socket;
     }
@@ -305,11 +269,11 @@ static void *nbd_client_thread(void *arg)
     fd = open(device, O_RDWR);
     if (fd < 0) {
         /* Linux-only, we can use %m in printf.  */
-        fprintf(stderr, "Failed to open %s: %m\n", device);
+        error_report("Failed to open %s: %m", device);
         goto out_socket;
     }
 
-    ret = nbd_init(fd, sock, nbdflags, size);
+    ret = nbd_init(fd, sioc, nbdflags, size);
     if (ret < 0) {
         goto out_fd;
     }
@@ -330,13 +294,14 @@ static void *nbd_client_thread(void *arg)
         goto out_fd;
     }
     close(fd);
+    object_unref(OBJECT(sioc));
     kill(getpid(), SIGTERM);
     return (void *) EXIT_SUCCESS;
 
 out_fd:
     close(fd);
 out_socket:
-    closesocket(sock);
+    object_unref(OBJECT(sioc));
 out:
     kill(getpid(), SIGTERM);
     return (void *) EXIT_FAILURE;
@@ -353,7 +318,7 @@ static void nbd_export_closed(NBDExport *exp)
     state = TERMINATED;
 }
 
-static void nbd_update_server_fd_handler(int fd);
+static void nbd_update_server_watch(void);
 
 static void nbd_client_closed(NBDClient *client)
 {
@@ -361,45 +326,130 @@ static void nbd_client_closed(NBDClient *client)
     if (nb_fds == 0 && !persistent && state == RUNNING) {
         state = TERMINATE;
     }
-    nbd_update_server_fd_handler(server_fd);
-    qemu_notify_event();
+    nbd_update_server_watch();
     nbd_client_put(client);
 }
 
-static void nbd_accept(void *opaque)
+static gboolean nbd_accept(QIOChannel *ioc, GIOCondition cond, gpointer opaque)
 {
-    struct sockaddr_in addr;
-    socklen_t addr_len = sizeof(addr);
+    QIOChannelSocket *cioc;
 
-    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    if (fd < 0) {
-        perror("accept");
-        return;
+    cioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc),
+                                     NULL);
+    if (!cioc) {
+        return TRUE;
     }
 
     if (state >= TERMINATE) {
-        close(fd);
-        return;
+        object_unref(OBJECT(cioc));
+        return TRUE;
     }
 
-    if (nbd_client_new(exp, fd, nbd_client_closed)) {
-        nb_fds++;
-        nbd_update_server_fd_handler(server_fd);
+    nb_fds++;
+    nbd_update_server_watch();
+    nbd_client_new(newproto ? NULL : exp, cioc,
+                   tlscreds, NULL, nbd_client_closed);
+    object_unref(OBJECT(cioc));
+
+    return TRUE;
+}
+
+static void nbd_update_server_watch(void)
+{
+    if (nbd_can_accept()) {
+        if (server_watch == -1) {
+            server_watch = qio_channel_add_watch(QIO_CHANNEL(server_ioc),
+                                                 G_IO_IN,
+                                                 nbd_accept,
+                                                 NULL, NULL);
+        }
     } else {
-        shutdown(fd, 2);
-        close(fd);
+        if (server_watch != -1) {
+            g_source_remove(server_watch);
+            server_watch = -1;
+        }
     }
 }
 
-static void nbd_update_server_fd_handler(int fd)
+
+static SocketAddress *nbd_build_socket_address(const char *sockpath,
+                                               const char *bindto,
+                                               const char *port)
 {
-    if (nbd_can_accept()) {
-        qemu_set_fd_handler(fd, nbd_accept, NULL, (void *)(uintptr_t)fd);
+    SocketAddress *saddr;
+
+    saddr = g_new0(SocketAddress, 1);
+    if (sockpath) {
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        saddr->u.q_unix.data->path = g_strdup(sockpath);
     } else {
-        qemu_set_fd_handler(fd, NULL, NULL, NULL);
+        InetSocketAddress *inet;
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+        inet->host = g_strdup(bindto);
+        if (port) {
+            inet->port = g_strdup(port);
+        } else  {
+            inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
+        }
     }
+
+    return saddr;
 }
 
+
+static QemuOptsList file_opts = {
+    .name = "file",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(file_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+
+
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
+    }
+
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a server endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
+}
+
+
 int main(int argc, char **argv)
 {
     BlockBackend *blk;
@@ -408,56 +458,60 @@ int main(int argc, char **argv)
     uint32_t nbdflags = 0;
     bool disconnect = false;
     const char *bindto = "0.0.0.0";
+    const char *port = NULL;
+    char *sockpath = NULL;
     char *device = NULL;
-    int port = NBD_DEFAULT_PORT;
     off_t fd_size;
     QemuOpts *sn_opts = NULL;
     const char *sn_id_or_name = NULL;
-    const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:";
+    const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:x:";
     struct option lopt[] = {
-        { "help", 0, NULL, 'h' },
-        { "version", 0, NULL, 'V' },
-        { "bind", 1, NULL, 'b' },
-        { "port", 1, NULL, 'p' },
-        { "socket", 1, NULL, 'k' },
-        { "offset", 1, NULL, 'o' },
-        { "read-only", 0, NULL, 'r' },
-        { "partition", 1, NULL, 'P' },
-        { "connect", 1, NULL, 'c' },
-        { "disconnect", 0, NULL, 'd' },
-        { "snapshot", 0, NULL, 's' },
-        { "load-snapshot", 1, NULL, 'l' },
-        { "nocache", 0, NULL, 'n' },
-        { "cache", 1, NULL, QEMU_NBD_OPT_CACHE },
-#ifdef CONFIG_LINUX_AIO
-        { "aio", 1, NULL, QEMU_NBD_OPT_AIO },
-#endif
-        { "discard", 1, NULL, QEMU_NBD_OPT_DISCARD },
-        { "detect-zeroes", 1, NULL, QEMU_NBD_OPT_DETECT_ZEROES },
-        { "shared", 1, NULL, 'e' },
-        { "format", 1, NULL, 'f' },
-        { "persistent", 0, NULL, 't' },
-        { "verbose", 0, NULL, 'v' },
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "bind", required_argument, NULL, 'b' },
+        { "port", required_argument, NULL, 'p' },
+        { "socket", required_argument, NULL, 'k' },
+        { "offset", required_argument, NULL, 'o' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "partition", required_argument, NULL, 'P' },
+        { "connect", required_argument, NULL, 'c' },
+        { "disconnect", no_argument, NULL, 'd' },
+        { "snapshot", no_argument, NULL, 's' },
+        { "load-snapshot", required_argument, NULL, 'l' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "cache", required_argument, NULL, QEMU_NBD_OPT_CACHE },
+        { "aio", required_argument, NULL, QEMU_NBD_OPT_AIO },
+        { "discard", required_argument, NULL, QEMU_NBD_OPT_DISCARD },
+        { "detect-zeroes", required_argument, NULL,
+          QEMU_NBD_OPT_DETECT_ZEROES },
+        { "shared", required_argument, NULL, 'e' },
+        { "format", required_argument, NULL, 'f' },
+        { "persistent", no_argument, NULL, 't' },
+        { "verbose", no_argument, NULL, 'v' },
+        { "object", required_argument, NULL, QEMU_NBD_OPT_OBJECT },
+        { "export-name", required_argument, NULL, 'x' },
+        { "tls-creds", required_argument, NULL, QEMU_NBD_OPT_TLSCREDS },
+        { "image-opts", no_argument, NULL, QEMU_NBD_OPT_IMAGE_OPTS },
         { NULL, 0, NULL, 0 }
     };
     int ch;
     int opt_ind = 0;
-    int li;
     char *end;
     int flags = BDRV_O_RDWR;
     int partition = -1;
     int ret = 0;
-    int fd;
     bool seen_cache = false;
     bool seen_discard = false;
-#ifdef CONFIG_LINUX_AIO
     bool seen_aio = false;
-#endif
     pthread_t client_thread;
     const char *fmt = NULL;
     Error *local_err = NULL;
     BlockdevDetectZeroesOptions detect_zeroes = BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
     QDict *options = NULL;
+    const char *export_name = NULL;
+    const char *tlscredsid = NULL;
+    bool imageOpts = false;
+    bool writethrough = true;
 
     /* The client thread uses SIGTERM to interrupt the server.  A signal
      * handler ensures that "qemu-nbd -v -c" exits with a nice status code.
@@ -466,6 +520,14 @@ int main(int argc, char **argv)
     memset(&sa_sigterm, 0, sizeof(sa_sigterm));
     sa_sigterm.sa_handler = termsig_handler;
     sigaction(SIGTERM, &sa_sigterm, NULL);
+
+    if (qcrypto_init(&local_err) < 0) {
+        error_reportf_err(local_err, "cannot initialize crypto: ");
+        exit(1);
+    }
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_object_opts);
     qemu_init_exec_dir(argv[0]);
 
     while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
@@ -478,17 +540,19 @@ int main(int argc, char **argv)
             /* fallthrough */
         case QEMU_NBD_OPT_CACHE:
             if (seen_cache) {
-                errx(EXIT_FAILURE, "-n and --cache can only be specified once");
+                error_report("-n and --cache can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_cache = true;
-            if (bdrv_parse_cache_flags(optarg, &flags) == -1) {
-                errx(EXIT_FAILURE, "Invalid cache mode `%s'", optarg);
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) == -1) {
+                error_report("Invalid cache mode `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
-#ifdef CONFIG_LINUX_AIO
         case QEMU_NBD_OPT_AIO:
             if (seen_aio) {
-                errx(EXIT_FAILURE, "--aio can only be specified once");
+                error_report("--aio can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_aio = true;
             if (!strcmp(optarg, "native")) {
@@ -496,56 +560,55 @@ int main(int argc, char **argv)
             } else if (!strcmp(optarg, "threads")) {
                 /* this is the default */
             } else {
-               errx(EXIT_FAILURE, "invalid aio mode `%s'", optarg);
+               error_report("invalid aio mode `%s'", optarg);
+               exit(EXIT_FAILURE);
             }
             break;
-#endif
         case QEMU_NBD_OPT_DISCARD:
             if (seen_discard) {
-                errx(EXIT_FAILURE, "--discard can only be specified once");
+                error_report("--discard can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_discard = true;
             if (bdrv_parse_discard_flags(optarg, &flags) == -1) {
-                errx(EXIT_FAILURE, "Invalid discard mode `%s'", optarg);
+                error_report("Invalid discard mode `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
         case QEMU_NBD_OPT_DETECT_ZEROES:
             detect_zeroes =
                 qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
                                 optarg,
-                                BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS__MAX,
                                 BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
                                 &local_err);
             if (local_err) {
-                errx(EXIT_FAILURE, "Failed to parse detect_zeroes mode: %s", 
-                     error_get_pretty(local_err));
+                error_reportf_err(local_err,
+                                  "Failed to parse detect_zeroes mode: ");
+                exit(EXIT_FAILURE);
             }
             if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
                 !(flags & BDRV_O_UNMAP)) {
-                errx(EXIT_FAILURE, "setting detect-zeroes to unmap is not allowed "
-                                   "without setting discard operation to unmap"); 
+                error_report("setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'b':
             bindto = optarg;
             break;
         case 'p':
-            li = strtol(optarg, &end, 0);
-            if (*end) {
-                errx(EXIT_FAILURE, "Invalid port `%s'", optarg);
-            }
-            if (li < 1 || li > 65535) {
-                errx(EXIT_FAILURE, "Port out of range `%s'", optarg);
-            }
-            port = (uint16_t)li;
+            port = optarg;
             break;
         case 'o':
                 dev_offset = strtoll (optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid offset `%s'", optarg);
+                error_report("Invalid offset `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (dev_offset < 0) {
-                errx(EXIT_FAILURE, "Offset must be positive `%s'", optarg);
+                error_report("Offset must be positive `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
         case 'l':
@@ -553,8 +616,9 @@ int main(int argc, char **argv)
                 sn_opts = qemu_opts_parse_noisily(&internal_snapshot_opts,
                                                   optarg, false);
                 if (!sn_opts) {
-                    errx(EXIT_FAILURE, "Failed in parsing snapshot param `%s'",
-                         optarg);
+                    error_report("Failed in parsing snapshot param `%s'",
+                                 optarg);
+                    exit(EXIT_FAILURE);
                 }
             } else {
                 sn_id_or_name = optarg;
@@ -567,16 +631,19 @@ int main(int argc, char **argv)
         case 'P':
             partition = strtol(optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid partition `%s'", optarg);
+                error_report("Invalid partition `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (partition < 1 || partition > 8) {
-                errx(EXIT_FAILURE, "Invalid partition %d", partition);
+                error_report("Invalid partition %d", partition);
+                exit(EXIT_FAILURE);
             }
             break;
         case 'k':
             sockpath = optarg;
             if (sockpath[0] != '/') {
-                errx(EXIT_FAILURE, "socket path must be absolute\n");
+                error_report("socket path must be absolute");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'd':
@@ -588,10 +655,12 @@ int main(int argc, char **argv)
         case 'e':
             shared = strtol(optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid shared device number '%s'", optarg);
+                error_report("Invalid shared device number '%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (shared < 1) {
-                errx(EXIT_FAILURE, "Shared device number must be greater than 0\n");
+                error_report("Shared device number must be greater than 0");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'f':
@@ -600,6 +669,9 @@ int main(int argc, char **argv)
         case 't':
             persistent = 1;
             break;
+        case 'x':
+            export_name = optarg;
+            break;
         case 'v':
             verbose = 1;
             break;
@@ -612,25 +684,69 @@ int main(int argc, char **argv)
             exit(0);
             break;
         case '?':
-            errx(EXIT_FAILURE, "Try `%s --help' for more information.",
-                 argv[0]);
+            error_report("Try `%s --help' for more information.", argv[0]);
+            exit(EXIT_FAILURE);
+        case QEMU_NBD_OPT_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                exit(EXIT_FAILURE);
+            }
+        }   break;
+        case QEMU_NBD_OPT_TLSCREDS:
+            tlscredsid = optarg;
+            break;
+        case QEMU_NBD_OPT_IMAGE_OPTS:
+            imageOpts = true;
+            break;
         }
     }
 
     if ((argc - optind) != 1) {
-        errx(EXIT_FAILURE, "Invalid number of argument.\n"
-             "Try `%s --help' for more information.",
-             argv[0]);
+        error_report("Invalid number of arguments");
+        error_printf("Try `%s --help' for more information.\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        exit(EXIT_FAILURE);
+    }
+
+    if (tlscredsid) {
+        if (sockpath) {
+            error_report("TLS is only supported with IPv4/IPv6");
+            exit(EXIT_FAILURE);
+        }
+        if (device) {
+            error_report("TLS is not supported with a host device");
+            exit(EXIT_FAILURE);
+        }
+        if (!export_name) {
+            /* Set the default NBD protocol export name, since
+             * we *must* use new style protocol for TLS */
+            export_name = "";
+        }
+        tlscreds = nbd_get_tls_creds(tlscredsid, &local_err);
+        if (local_err) {
+            error_report("Failed to get TLS creds %s",
+                         error_get_pretty(local_err));
+            exit(EXIT_FAILURE);
+        }
     }
 
     if (disconnect) {
-        fd = open(argv[optind], O_RDWR);
-        if (fd < 0) {
-            err(EXIT_FAILURE, "Cannot open %s", argv[optind]);
+        int nbdfd = open(argv[optind], O_RDWR);
+        if (nbdfd < 0) {
+            error_report("Cannot open %s: %s", argv[optind],
+                         strerror(errno));
+            exit(EXIT_FAILURE);
         }
-        nbd_disconnect(fd);
+        nbd_disconnect(nbdfd);
 
-        close(fd);
+        close(nbdfd);
 
         printf("%s disconnected\n", argv[optind]);
 
@@ -643,7 +759,9 @@ int main(int argc, char **argv)
         int ret;
 
         if (qemu_pipe(stderr_fd) < 0) {
-            err(EXIT_FAILURE, "Error setting up communication pipe");
+            error_report("Error setting up communication pipe: %s",
+                         strerror(errno));
+            exit(EXIT_FAILURE);
         }
 
         /* Now daemonize, but keep a communication channel open to
@@ -651,7 +769,8 @@ int main(int argc, char **argv)
          */
         pid = fork();
         if (pid < 0) {
-            err(EXIT_FAILURE, "Failed to fork");
+            error_report("Failed to fork: %s", strerror(errno));
+            exit(EXIT_FAILURE);
         } else if (pid == 0) {
             close(stderr_fd[0]);
             ret = qemu_daemon(1, 0);
@@ -659,7 +778,8 @@ int main(int argc, char **argv)
             /* Temporarily redirect stderr to the parent's pipe...  */
             dup2(stderr_fd[1], STDERR_FILENO);
             if (ret < 0) {
-                err(EXIT_FAILURE, "Failed to daemonize");
+                error_report("Failed to daemonize: %s", strerror(errno));
+                exit(EXIT_FAILURE);
             }
 
             /* ... close the descriptor we inherited and go on.  */
@@ -681,7 +801,9 @@ int main(int argc, char **argv)
                 }
             }
             if (ret < 0) {
-                err(EXIT_FAILURE, "Cannot read from daemon");
+                error_report("Cannot read from daemon: %s",
+                             strerror(errno));
+                exit(EXIT_FAILURE);
             }
 
             /* Usually the daemon should not print any message.
@@ -696,6 +818,8 @@ int main(int argc, char **argv)
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
+    saddr = nbd_build_socket_address(sockpath, bindto, port);
+
     if (qemu_init_main_loop(&local_err)) {
         error_report_err(local_err);
         exit(EXIT_FAILURE);
@@ -703,19 +827,38 @@ int main(int argc, char **argv)
     bdrv_init();
     atexit(bdrv_close_all);
 
-    if (fmt) {
-        options = qdict_new();
-        qdict_put(options, "driver", qstring_from_str(fmt));
+    srcpath = argv[optind];
+    if (imageOpts) {
+        QemuOpts *opts;
+        if (fmt) {
+            error_report("--image-opts and -f are mutually exclusive");
+            exit(EXIT_FAILURE);
+        }
+        opts = qemu_opts_parse_noisily(&file_opts, srcpath, true);
+        if (!opts) {
+            qemu_opts_reset(&file_opts);
+            exit(EXIT_FAILURE);
+        }
+        options = qemu_opts_to_qdict(opts, NULL);
+        qemu_opts_reset(&file_opts);
+        blk = blk_new_open(NULL, NULL, options, flags, &local_err);
+    } else {
+        if (fmt) {
+            options = qdict_new();
+            qdict_put(options, "driver", qstring_from_str(fmt));
+        }
+        blk = blk_new_open(srcpath, NULL, options, flags, &local_err);
     }
 
-    srcpath = argv[optind];
-    blk = blk_new_open("hda", srcpath, NULL, options, flags, &local_err);
     if (!blk) {
-        errx(EXIT_FAILURE, "Failed to blk_new_open '%s': %s", argv[optind],
-             error_get_pretty(local_err));
+        error_reportf_err(local_err, "Failed to blk_new_open '%s': ",
+                          argv[optind]);
+        exit(EXIT_FAILURE);
     }
     bs = blk_bs(blk);
 
+    blk_set_enable_write_cache(blk, !writethrough);
+
     if (sn_opts) {
         ret = bdrv_snapshot_load_tmp(bs,
                                      qemu_opt_get(sn_opts, SNAPSHOT_OPT_ID),
@@ -726,40 +869,42 @@ int main(int argc, char **argv)
                                                    &local_err);
     }
     if (ret < 0) {
-        errno = -ret;
-        err(EXIT_FAILURE,
-            "Failed to load snapshot: %s",
-            error_get_pretty(local_err));
+        error_reportf_err(local_err, "Failed to load snapshot: ");
+        exit(EXIT_FAILURE);
     }
 
     bs->detect_zeroes = detect_zeroes;
     fd_size = blk_getlength(blk);
     if (fd_size < 0) {
-        errx(EXIT_FAILURE, "Failed to determine the image length: %s",
-             strerror(-fd_size));
+        error_report("Failed to determine the image length: %s",
+                     strerror(-fd_size));
+        exit(EXIT_FAILURE);
     }
 
     if (partition != -1) {
         ret = find_partition(blk, partition, &dev_offset, &fd_size);
         if (ret < 0) {
-            errno = -ret;
-            err(EXIT_FAILURE, "Could not find partition %d", partition);
+            error_report("Could not find partition %d: %s", partition,
+                         strerror(-ret));
+            exit(EXIT_FAILURE);
         }
     }
 
     exp = nbd_export_new(blk, dev_offset, fd_size, nbdflags, nbd_export_closed,
                          &local_err);
     if (!exp) {
-        errx(EXIT_FAILURE, "%s", error_get_pretty(local_err));
+        error_report_err(local_err);
+        exit(EXIT_FAILURE);
     }
-
-    if (sockpath) {
-        fd = unix_socket_incoming(sockpath);
-    } else {
-        fd = tcp_socket_incoming(bindto, port);
+    if (export_name) {
+        nbd_export_set_name(exp, export_name);
+        newproto = true;
     }
 
-    if (fd < 0) {
+    server_ioc = qio_channel_socket_new();
+    if (qio_channel_socket_listen_sync(server_ioc, saddr, &local_err) < 0) {
+        object_unref(OBJECT(server_ioc));
+        error_report_err(local_err);
         return 1;
     }
 
@@ -768,21 +913,22 @@ int main(int argc, char **argv)
 
         ret = pthread_create(&client_thread, NULL, nbd_client_thread, device);
         if (ret != 0) {
-            errx(EXIT_FAILURE, "Failed to create client thread: %s",
-                 strerror(ret));
+            error_report("Failed to create client thread: %s", strerror(ret));
+            exit(EXIT_FAILURE);
         }
     } else {
         /* Shut up GCC warnings.  */
         memset(&client_thread, 0, sizeof(client_thread));
     }
 
-    server_fd = fd;
-    nbd_update_server_fd_handler(fd);
+    nbd_update_server_watch();
 
     /* now when the initialization is (almost) complete, chdir("/")
      * to free any busy filesystems */
     if (chdir("/") < 0) {
-        err(EXIT_FAILURE, "Could not chdir to root directory");
+        error_report("Could not chdir to root directory: %s",
+                     strerror(errno));
+        exit(EXIT_FAILURE);
     }
 
     state = RUNNING;
diff --git a/qemu/qemu-nbd.texi b/qemu/qemu-nbd.texi
index 46fd483eb..9f2334345 100644
--- a/qemu/qemu-nbd.texi
+++ b/qemu/qemu-nbd.texi
@@ -1,68 +1,97 @@
 @example
 @c man begin SYNOPSIS
-usage: qemu-nbd [OPTION]...  @var{filename}
+@command{qemu-nbd} [OPTION]... @var{filename}
+
+@command{qemu-nbd} @option{-d} @var{dev}
 @c man end
 @end example
 
 @c man begin DESCRIPTION
 
-Export QEMU disk image using NBD protocol.
+Export a QEMU disk image using the NBD protocol.
 
 @c man end
 
 @c man begin OPTIONS
+@var{filename} is a disk image filename, or a set of block
+driver options if @var{--image-opts} is specified.
+
+@var{dev} is an NBD device.
+
 @table @option
-@item @var{filename}
- is a disk image filename
+@item --object type,id=@var{id},...props...
+Define a new instance of the @var{type} object class identified by @var{id}.
+See the @code{qemu(1)} manual page for full details of the properties
+supported. The common object types that it makes sense to define are the
+@code{secret} object, which is used to supply passwords and/or encryption
+keys, and the @code{tls-creds} object, which is used to supply TLS
+credentials for the qemu-nbd server.
 @item -p, --port=@var{port}
-  port to listen on (default @samp{10809})
+The TCP port to listen on (default @samp{10809})
 @item -o, --offset=@var{offset}
-  offset into the image
+The offset into the image
 @item -b, --bind=@var{iface}
-  interface to bind to (default @samp{0.0.0.0})
+The interface to bind to (default @samp{0.0.0.0})
 @item -k, --socket=@var{path}
-  Use a unix socket with path @var{path}
-@item -f, --format=@var{format}
-  Set image format as @var{format}
+Use a unix socket with path @var{path}
+@item --image-opts
+Treat @var{filename} as a set of image options, instead of a plain
+filename. If this flag is specified, the @var{-f} flag should
+not be used, instead the '@code{format=}' option should be set.
+@item -f, --format=@var{fmt}
+Force the use of the block driver for format @var{fmt} instead of
+auto-detecting
 @item -r, --read-only
-  export read-only
+Export the disk as read-only
 @item -P, --partition=@var{num}
-  only expose partition @var{num}
+Only expose partition @var{num}
 @item -s, --snapshot
-  use @var{filename} as an external snapshot, create a temporary
-  file with backing_file=@var{filename}, redirect the write to
-  the temporary one
+Use @var{filename} as an external snapshot, create a temporary
+file with backing_file=@var{filename}, redirect the write to
+the temporary one
 @item -l, --load-snapshot=@var{snapshot_param}
-  load an internal snapshot inside @var{filename} and export it
-  as an read-only device, @var{snapshot_param} format is
-  'snapshot.id=[ID],snapshot.name=[NAME]' or '[ID_OR_NAME]'
+Load an internal snapshot inside @var{filename} and export it
+as an read-only device, @var{snapshot_param} format is
+'snapshot.id=[ID],snapshot.name=[NAME]' or '[ID_OR_NAME]'
 @item -n, --nocache
 @itemx --cache=@var{cache}
-  set cache mode to be used with the file.  See the documentation of
-  the emulator's @code{-drive cache=...} option for allowed values.
+The cache mode to be used with the file.  See the documentation of
+the emulator's @code{-drive cache=...} option for allowed values.
 @item --aio=@var{aio}
-  choose asynchronous I/O mode between @samp{threads} (the default)
-  and @samp{native} (Linux only).
+Set the asynchronous I/O mode between @samp{threads} (the default)
+and @samp{native} (Linux only).
 @item --discard=@var{discard}
-  toggles whether @dfn{discard} (also known as @dfn{trim} or @dfn{unmap})
-  requests are ignored or passed to the filesystem.  The default is no
-  (@samp{--discard=ignore}).
+Control whether @dfn{discard} (also known as @dfn{trim} or @dfn{unmap})
+requests are ignored or passed to the filesystem.  @var{discard} is one of
+@samp{ignore} (or @samp{off}), @samp{unmap} (or @samp{on}).  The default is
+@samp{ignore}.
+@item --detect-zeroes=@var{detect-zeroes}
+Control the automatic conversion of plain zero writes by the OS to
+driver-specific optimized zero write commands.  @var{detect-zeroes} is one of
+@samp{off}, @samp{on} or @samp{unmap}.  @samp{unmap}
+converts a zero write to an unmap operation and can only be used if
+@var{discard} is set to @samp{unmap}.  The default is @samp{off}.
 @item -c, --connect=@var{dev}
-  connect @var{filename} to NBD device @var{dev}
+Connect @var{filename} to NBD device @var{dev}
 @item -d, --disconnect
-  disconnect the specified device
+Disconnect the device @var{dev}
 @item -e, --shared=@var{num}
-  device can be shared by @var{num} clients (default @samp{1})
-@item -f, --format=@var{fmt}
-  force block driver for format @var{fmt} instead of auto-detecting
+Allow up to @var{num} clients to share the device (default @samp{1})
 @item -t, --persistent
-  don't exit on the last connection
+Don't exit on the last connection
+@item -x NAME, --export-name=NAME
+Set the NBD volume export name. This switches the server to use
+the new style NBD protocol negotiation
+@item --tls-creds=ID
+Enable mandatory TLS encryption for the server by setting the ID
+of the TLS credentials object previously created with the --object
+option.
 @item -v, --verbose
-  display extra debugging information
+Display extra debugging information
 @item -h, --help
-  display this help and exit
+Display this help and exit
 @item -V, --version
-  output version information and exit
+Display version information and exit
 @end table
 
 @c man end
@@ -79,7 +108,7 @@ warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 @c man end
 
 @c man begin SEEALSO
-qemu-img(1)
+qemu(1), qemu-img(1)
 @c man end
 
 @end ignore
diff --git a/qemu/qemu-options.hx b/qemu/qemu-options.hx
index 77f5853d5..6106520c5 100644
--- a/qemu/qemu-options.hx
+++ b/qemu/qemu-options.hx
@@ -33,14 +33,18 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
     "                supported accelerators are kvm, xen, tcg (default: tcg)\n"
     "                kernel_irqchip=on|off controls accelerated irqchip support\n"
+    "                kernel_irqchip=on|off|split controls accelerated irqchip support (default=off)\n"
     "                vmport=on|off|auto controls emulation of vmport (default: auto)\n"
     "                kvm_shadow_mem=size of KVM shadow MMU\n"
     "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
     "                mem-merge=on|off controls memory merge support (default: on)\n"
     "                iommu=on|off controls emulated Intel IOMMU (VT-d) support (default=off)\n"
+    "                igd-passthru=on|off controls IGD GFX passthrough support (default=off)\n"
     "                aes-key-wrap=on|off controls support for AES key wrapping (default=on)\n"
     "                dea-key-wrap=on|off controls support for DEA key wrapping (default=on)\n"
-    "                suppress-vmdesc=on|off disables self-describing migration (default=off)\n",
+    "                suppress-vmdesc=on|off disables self-describing migration (default=off)\n"
+    "                nvdimm=on|off controls NVDIMM support (default=off)\n"
+    "                enforce-config-section=on|off enforce configuration section migration (default=off)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
@@ -54,7 +58,9 @@ kvm, xen, or tcg can be available. By default, tcg is used. If there is more
 than one accelerator specified, the next one is used if the previous one fails
 to initialize.
 @item kernel_irqchip=on|off
-Enables in-kernel irqchip support for the chosen accelerator when available.
+Controls in-kernel irqchip support for the chosen accelerator when available.
+@item gfx_passthru=on|off
+Enables IGD GFX passthrough support for the chosen machine when available.
 @item vmport=on|off|auto
 Enables emulation of VMWare IO port, for vmmouse etc. auto says to select the
 value based on accel. For accel=xen the default is off otherwise the default
@@ -77,6 +83,8 @@ execution of AES cryptographic functions.  The default is on.
 Enables or disables DEA key wrapping support on s390-ccw hosts. This feature
 controls whether DEA wrapping keys will be created to allow
 execution of DEA cryptographic functions.  The default is on.
+@item nvdimm=on|off
+Enables or disables NVDIMM support. The default is off.
 @end table
 ETEXI
 
@@ -378,6 +386,58 @@ Add device @var{driver}.  @var{prop}=@var{value} sets driver
 properties.  Valid properties depend on the driver.  To get help on
 possible drivers and properties, use @code{-device help} and
 @code{-device @var{driver},help}.
+
+Some drivers are:
+@item -device ipmi-bmc-sim,id=@var{id}[,slave_addr=@var{val}]
+
+Add an IPMI BMC.  This is a simulation of a hardware management
+interface processor that normally sits on a system.  It provides
+a watchdog and the ability to reset and power control the system.
+You need to connect this to an IPMI interface to make it useful
+
+The IPMI slave address to use for the BMC.  The default is 0x20.
+This address is the BMC's address on the I2C network of management
+controllers.  If you don't know what this means, it is safe to ignore
+it.
+
+@item -device ipmi-bmc-extern,id=@var{id},chardev=@var{id}[,slave_addr=@var{val}]
+
+Add a connection to an external IPMI BMC simulator.  Instead of
+locally emulating the BMC like the above item, instead connect
+to an external entity that provides the IPMI services.
+
+A connection is made to an external BMC simulator.  If you do this, it
+is strongly recommended that you use the "reconnect=" chardev option
+to reconnect to the simulator if the connection is lost.  Note that if
+this is not used carefully, it can be a security issue, as the
+interface has the ability to send resets, NMIs, and power off the VM.
+It's best if QEMU makes a connection to an external simulator running
+on a secure port on localhost, so neither the simulator nor QEMU is
+exposed to any outside network.
+
+See the "lanserv/README.vm" file in the OpenIPMI library for more
+details on the external interface.
+
+@item -device isa-ipmi-kcs,bmc=@var{id}[,ioport=@var{val}][,irq=@var{val}]
+
+Add a KCS IPMI interafce on the ISA bus.  This also adds a
+corresponding ACPI and SMBIOS entries, if appropriate.
+
+@table @option
+@item bmc=@var{id}
+The BMC to connect to, one of ipmi-bmc-sim or ipmi-bmc-extern above.
+@item ioport=@var{val}
+Define the I/O address of the interface.  The default is 0xca0 for KCS.
+@item irq=@var{val}
+Define the interrupt to use.  The default is 5.  To disable interrupts,
+set this to 0.
+@end table
+
+@item -device isa-ipmi-bt,bmc=@var{id}[,ioport=@var{val}][,irq=@var{val}]
+
+Like the KCS interface, but defines a BT interface.  The default port is
+0xe4 and the default interrupt is 5.
+
 ETEXI
 
 DEF("name", HAS_ARG, QEMU_OPTION_name,
@@ -992,6 +1052,7 @@ DEF("spice", HAS_ARG, QEMU_OPTION_spice,
     "       [,streaming-video=[off|all|filter]][,disable-copy-paste]\n"
     "       [,disable-agent-file-xfer][,agent-mouse=[on|off]]\n"
     "       [,playback-compression=[on|off]][,seamless-migration=[on|off]]\n"
+    "       [,gl=[on|off]]\n"
     "   enable spice\n"
     "   at least one of {port, tls-port} is mandatory\n",
     QEMU_ARCH_ALL)
@@ -1083,6 +1144,9 @@ Enable/disable audio stream compression (using celt 0.5.1).  Default is on.
 @item seamless-migration=[on|off]
 Enable/disable spice seamless migration. Default is off.
 
+@item gl=[on|off]
+Enable/disable OpenGL context. Default is off.
+
 @end table
 ETEXI
 
@@ -1214,8 +1278,9 @@ By definition the Websocket port is 5700+@var{display}. If @var{host} is
 specified connections will only be allowed from this host.
 As an alternative the Websocket port could be specified by using
 @code{websocket}=@var{port}.
-TLS encryption for the Websocket connection is supported if the required
-certificates are specified with the VNC option @option{x509}.
+If no TLS credentials are provided, the websocket connection runs in
+unencrypted mode. If TLS credentials are provided, the websocket connection
+requires encrypted client connections.
 
 @item password
 
@@ -1236,6 +1301,20 @@ date and time).
 You can also use keywords "now" or "never" for the expiration time to
 allow <protocol> password to expire immediately or never expire.
 
+@item tls-creds=@var{ID}
+
+Provides the ID of a set of TLS credentials to use to secure the
+VNC server. They will apply to both the normal VNC server socket
+and the websocket socket (if enabled). Setting TLS credentials
+will cause the VNC server socket to enable the VeNCrypt auth
+mechanism.  The credentials should have been previously created
+using the @option{-object tls-creds} argument.
+
+The @option{tls-creds} parameter obsoletes the @option{tls},
+@option{x509}, and @option{x509verify} options, and as such
+it is not permitted to set both new and old type options at
+the same time.
+
 @item tls
 
 Require that client use TLS when communicating with the VNC server. This
@@ -1243,6 +1322,9 @@ uses anonymous TLS credentials so is susceptible to a man-in-the-middle
 attack. It is recommended that this option be combined with either the
 @option{x509} or @option{x509verify} options.
 
+This option is now deprecated in favor of using the @option{tls-creds}
+argument.
+
 @item x509=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1252,6 +1334,9 @@ to provide authentication of the client when this is used. The path following
 this option specifies where the x509 certificates are to be loaded from.
 See the @ref{vnc_security} section for details on generating certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item x509verify=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1265,6 +1350,9 @@ path following this option specifies where the x509 certificates are to
 be loaded from. See the @ref{vnc_security} section for details on generating
 certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item sasl
 
 Require that the client use SASL to authenticate with the VNC server.
@@ -1389,6 +1477,10 @@ ACPI headers (possible overridden by other options).
 For data=, only data
 portion of the table is used, all header information is specified in the
 command line.
+If a SLIC table is supplied to QEMU, then the SLIC's oem_id and oem_table_id
+fields will override the same in the RSDT and the FADT (a.k.a. FACP), in order
+to ensure the field matches required by the Microsoft SLIC spec and the ACPI
+spec.
 ETEXI
 
 DEF("smbios", HAS_ARG, QEMU_OPTION_smbios,
@@ -1412,7 +1504,7 @@ DEF("smbios", HAS_ARG, QEMU_OPTION_smbios,
     "-smbios type=17[,loc_pfx=str][,bank=str][,manufacturer=str][,serial=str]\n"
     "               [,asset=str][,part=str][,speed=%d]\n"
     "                specify SMBIOS type 17 fields\n",
-    QEMU_ARCH_I386)
+    QEMU_ARCH_I386 | QEMU_ARCH_ARM)
 STEXI
 @item -smbios file=@var{binary}
 @findex -smbios
@@ -1459,8 +1551,10 @@ DEF("smb", HAS_ARG, QEMU_OPTION_smb, "", QEMU_ARCH_ALL)
 
 DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
 #ifdef CONFIG_SLIRP
-    "-netdev user,id=str[,net=addr[/mask]][,host=addr][,restrict=on|off]\n"
-    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,dnssearch=domain][,tftp=dir]\n"
+    "-netdev user,id=str[,ipv4[=on|off]][,net=addr[/mask]][,host=addr]\n"
+    "         [,ipv6[=on|off]][,ipv6-net=addr[/int]][,ipv6-host=addr]\n"
+    "         [,restrict=on|off][,hostname=host][,dhcpstart=addr]\n"
+    "         [,dns=addr][,ipv6-dns=addr][,dnssearch=domain][,tftp=dir]\n"
     "         [,bootfile=f][,hostfwd=rule][,guestfwd=rule]"
 #ifndef _WIN32
                                              "[,smb=dir[,smbserver=addr]]\n"
@@ -1608,6 +1702,9 @@ Connect user mode stack to VLAN @var{n} (@var{n} = 0 is the default).
 @itemx name=@var{name}
 Assign symbolic name for use in monitor commands.
 
+@option{ipv4} and @option{ipv6} specify that either IPv4 or IPv6 must
+be enabled.  If neither is specified both protocols are enabled.
+
 @item net=@var{addr}[/@var{mask}]
 Set IP network address the guest will see. Optionally specify the netmask,
 either in the form a.b.c.d or as number of valid top-most bits. Default is
@@ -1617,6 +1714,16 @@ either in the form a.b.c.d or as number of valid top-most bits. Default is
 Specify the guest-visible address of the host. Default is the 2nd IP in the
 guest network, i.e. x.x.x.2.
 
+@item ipv6-net=@var{addr}[/@var{int}]
+Set IPv6 network address the guest will see (default is fec0::/64). The
+network prefix is given in the usual hexadecimal IPv6 address
+notation. The prefix size is optional, and is given as the number of
+valid top-most bits (default is 64).
+
+@item ipv6-host=@var{addr}
+Specify the guest-visible IPv6 address of the host. Default is the 2nd IPv6 in
+the guest network, i.e. xxxx::2.
+
 @item restrict=on|off
 If this option is enabled, the guest will be isolated, i.e. it will not be
 able to contact the host and no guest IP packets will be routed over the host
@@ -1634,6 +1741,11 @@ Specify the guest-visible address of the virtual nameserver. The address must
 be different from the host address. Default is the 3rd IP in the guest network,
 i.e. x.x.x.3.
 
+@item ipv6-dns=@var{addr}
+Specify the guest-visible address of the IPv6 virtual nameserver. The address
+must be different from the host address. Default is the 3rd IP in the guest
+network, i.e. xxxx::3.
+
 @item dnssearch=@var{domain}
 Provides an entry for the domain-search list sent by the built-in
 DHCP server. More than one domain suffix can be transmitted by specifying
@@ -1963,13 +2075,14 @@ The hubport netdev lets you connect a NIC to a QEMU "vlan" instead of a single
 netdev.  @code{-net} and @code{-device} with parameter @option{vlan} create the
 required hub automatically.
 
-@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off]
+@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off][,queues=n]
 
 Establish a vhost-user netdev, backed by a chardev @var{id}. The chardev should
 be a unix domain socket backed one. The vhost-user uses a specifically defined
 protocol to pass vhost ioctl replacement messages to an application on the other
 end of the socket. On non-MSIX guests, the feature can be forced with
-@var{vhostforce}.
+@var{vhostforce}. Use 'queues=@var{n}' to specify the number of queues to
+be created for multiqueue vhost-user.
 
 Example:
 @example
@@ -1984,6 +2097,7 @@ qemu -m 512 -object memory-backend-file,id=mem,size=512M,mem-path=/hugetlbfs,sha
 Dump network traffic on VLAN @var{n} to file @var{file} (@file{qemu-vlan0.pcap} by default).
 At most @var{len} bytes (64k by default) per packet are stored. The file format is
 libpcap, so it can be analyzed with tools such as tcpdump or Wireshark.
+Note: For devices created with '-netdev', use '-object filter-dump,...' instead.
 
 @item -net none
 Indicate that no network devices should be configured. It is used to
@@ -2004,40 +2118,43 @@ The general form of a character device option is:
 ETEXI
 
 DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
-    "-chardev null,id=id[,mux=on|off]\n"
+    "-chardev null,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
     "-chardev socket,id=id[,host=host],port=port[,to=to][,ipv4][,ipv6][,nodelay][,reconnect=seconds]\n"
-    "         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (tcp)\n"
-    "-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (unix)\n"
+    "         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off]\n"
+    "         [,logfile=PATH][,logappend=on|off][,tls-creds=ID] (tcp)\n"
+    "-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds]\n"
+    "         [,mux=on|off][,logfile=PATH][,logappend=on|off] (unix)\n"
     "-chardev udp,id=id[,host=host],port=port[,localaddr=localaddr]\n"
     "         [,localport=localport][,ipv4][,ipv6][,mux=on|off]\n"
-    "-chardev msmouse,id=id[,mux=on|off]\n"
+    "         [,logfile=PATH][,logappend=on|off]\n"
+    "-chardev msmouse,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
     "-chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]]\n"
-    "         [,mux=on|off]\n"
-    "-chardev ringbuf,id=id[,size=size]\n"
-    "-chardev file,id=id,path=path[,mux=on|off]\n"
-    "-chardev pipe,id=id,path=path[,mux=on|off]\n"
+    "         [,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev ringbuf,id=id[,size=size][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev file,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev pipe,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #ifdef _WIN32
-    "-chardev console,id=id[,mux=on|off]\n"
-    "-chardev serial,id=id,path=path[,mux=on|off]\n"
+    "-chardev console,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #else
-    "-chardev pty,id=id[,mux=on|off]\n"
-    "-chardev stdio,id=id[,mux=on|off][,signal=on|off]\n"
+    "-chardev pty,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev stdio,id=id[,mux=on|off][,signal=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #ifdef CONFIG_BRLAPI
-    "-chardev braille,id=id[,mux=on|off]\n"
+    "-chardev braille,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
         || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
-    "-chardev serial,id=id,path=path[,mux=on|off]\n"
-    "-chardev tty,id=id,path=path[,mux=on|off]\n"
+    "-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev tty,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
-    "-chardev parallel,id=id,path=path[,mux=on|off]\n"
-    "-chardev parport,id=id,path=path[,mux=on|off]\n"
+    "-chardev parallel,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev parport,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(CONFIG_SPICE)
-    "-chardev spicevmc,id=id,name=name[,debug=debug]\n"
-    "-chardev spiceport,id=id,name=name[,debug=debug]\n"
+    "-chardev spicevmc,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev spiceport,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
 #endif
     , QEMU_ARCH_ALL
 )
@@ -2070,16 +2187,62 @@ All devices must have an id, which can be any string up to 127 characters long.
 It is used to uniquely identify this device in other command line directives.
 
 A character device may be used in multiplexing mode by multiple front-ends.
-The key sequence of @key{Control-a} and @key{c} will rotate the input focus
-between attached front-ends. Specify @option{mux=on} to enable this mode.
+Specify @option{mux=on} to enable this mode.
+A multiplexer is a "1:N" device, and here the "1" end is your specified chardev
+backend, and the "N" end is the various parts of QEMU that can talk to a chardev.
+If you create a chardev with @option{id=myid} and @option{mux=on}, QEMU will
+create a multiplexer with your specified ID, and you can then configure multiple
+front ends to use that chardev ID for their input/output. Up to four different
+front ends can be connected to a single multiplexed chardev. (Without
+multiplexing enabled, a chardev can only be used by a single front end.)
+For instance you could use this to allow a single stdio chardev to be used by
+two serial ports and the QEMU monitor:
 
-Options to each backend are described below.
+@example
+-chardev stdio,mux=on,id=char0 \
+-mon chardev=char0,mode=readline,default \
+-serial chardev:char0 \
+-serial chardev:char0
+@end example
+
+You can have more than one multiplexer in a system configuration; for instance
+you could have a TCP port multiplexed between UART 0 and UART 1, and stdio
+multiplexed between the QEMU monitor and a parallel port:
+
+@example
+-chardev stdio,mux=on,id=char0 \
+-mon chardev=char0,mode=readline,default \
+-parallel chardev:char0 \
+-chardev tcp,...,mux=on,id=char1 \
+-serial chardev:char1 \
+-serial chardev:char1
+@end example
+
+When you're using a multiplexed character device, some escape sequences are
+interpreted in the input. @xref{mux_keys, Keys in the character backend
+multiplexer}.
+
+Note that some other command line options may implicitly create multiplexed
+character backends; for instance @option{-serial mon:stdio} creates a
+multiplexed stdio backend connected to the serial port and the QEMU monitor,
+and @option{-nographic} also multiplexes the console and the monitor to
+stdio.
+
+There is currently no support for multiplexing in the other direction
+(where a single QEMU front end takes input and output from multiple chardevs).
+
+Every backend supports the @option{logfile} option, which supplies the path
+to a file to record all data transmitted via the backend. The @option{logappend}
+option controls whether the log file will be truncated or appended to when
+opened.
+
+Further options to each backend are described below.
 
 @item -chardev null ,id=@var{id}
 A void device. This device will not emit any data, and will drop any data it
 receives. The null backend does not take any options.
 
-@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet] [,reconnect=@var{seconds}]
+@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet] [,reconnect=@var{seconds}] [,tls-creds=@var{id}]
 
 Create a two-way stream socket, which can be either a TCP or a unix socket. A
 unix socket will be created if @option{path} is specified. Behaviour is
@@ -2097,6 +2260,11 @@ escape sequences.
 the remote end goes away.  qemu will delay this many seconds and then attempt
 to reconnect.  Zero disables reconnecting, and is the default.
 
+@option{tls-creds} requests enablement of the TLS protocol for encryption,
+and specifies the id of the TLS credentials to use for the handshake. The
+credentials must be previously created with the @option{-object tls-creds}
+argument.
+
 TCP and unix socket options are given below:
 
 @table @option
@@ -2696,13 +2864,32 @@ ETEXI
 
 DEF("fw_cfg", HAS_ARG, QEMU_OPTION_fwcfg,
     "-fw_cfg [name=]<name>,file=<file>\n"
-    "                add named fw_cfg entry from file\n",
+    "                add named fw_cfg entry with contents from file\n"
+    "-fw_cfg [name=]<name>,string=<str>\n"
+    "                add named fw_cfg entry with contents from string\n",
     QEMU_ARCH_ALL)
 STEXI
+
 @item -fw_cfg [name=]@var{name},file=@var{file}
 @findex -fw_cfg
-Add named fw_cfg entry from file. @var{name} determines the name of
-the entry in the fw_cfg file directory exposed to the guest.
+Add named fw_cfg entry with contents from file @var{file}.
+
+@item -fw_cfg [name=]@var{name},string=@var{str}
+Add named fw_cfg entry with contents from string @var{str}.
+
+The terminating NUL character of the contents of @var{str} will not be
+included as part of the fw_cfg item data. To insert contents with
+embedded NUL characters, you have to use the @var{file} parameter.
+
+The fw_cfg entries are passed by QEMU through to the guest.
+
+Example:
+@example
+    -fw_cfg name=opt/com.mycompany/blob,file=./my_blob.bin
+@end example
+creates an fw_cfg entry named opt/com.mycompany/blob with contents
+from ./my_blob.bin.
+
 ETEXI
 
 DEF("serial", HAS_ARG, QEMU_OPTION_serial, \
@@ -2987,6 +3174,24 @@ STEXI
 Output log in @var{logfile} instead of to stderr
 ETEXI
 
+DEF("dfilter", HAS_ARG, QEMU_OPTION_DFILTER, \
+    "-dfilter range,..  filter debug output to range of addresses (useful for -d cpu,exec,etc..)\n",
+    QEMU_ARCH_ALL)
+STEXI
+@item -dfilter @var{range1}[,...]
+@findex -dfilter
+Filter debug output to that relevant to a range of target addresses. The filter
+spec can be either @var{start}+@var{size}, @var{start}-@var{size} or
+@var{start}..@var{end} where @var{start} @var{end} and @var{size} are the
+addresses and sizes required. For example:
+@example
+    -dfilter 0x8000..0x8fff,0xffffffc000080000+0x200,0xffffffc000060000-0x1000
+@end example
+Will dump output for any code in the 0x1000 sized block starting at 0x8000 and
+the 0x200 sized block starting at 0xffffffc000080000 and another 0x1000 sized
+block starting at 0xffffffc00005f000.
+ETEXI
+
 DEF("L", HAS_ARG, QEMU_OPTION_L, \
     "-L path         set the directory for the BIOS, VGA BIOS and keymaps\n",
     QEMU_ARCH_ALL)
@@ -3123,12 +3328,12 @@ re-inject them.
 ETEXI
 
 DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
-    "-icount [shift=N|auto][,align=on|off][,sleep=no]\n" \
+    "-icount [shift=N|auto][,align=on|off][,sleep=on|off,rr=record|replay,rrfile=<filename>]\n" \
     "                enable virtual instruction counter with 2^N clock ticks per\n" \
     "                instruction, enable aligning the host and virtual clocks\n" \
     "                or disable real time cpu sleeping\n", QEMU_ARCH_ALL)
 STEXI
-@item -icount [shift=@var{N}|auto]
+@item -icount [shift=@var{N}|auto][,rr=record|replay,rrfile=@var{filename}]
 @findex -icount
 Enable virtual instruction counter.  The virtual cpu will execute one
 instruction every 2^@var{N} ns of virtual time.  If @code{auto} is specified
@@ -3136,8 +3341,8 @@ then the virtual cpu speed will be automatically adjusted to keep virtual
 time within a few seconds of real time.
 
 When the virtual cpu is sleeping, the virtual time will advance at default
-speed unless @option{sleep=no} is specified.
-With @option{sleep=no}, the virtual time will jump to the next timer deadline
+speed unless @option{sleep=on|off} is specified.
+With @option{sleep=on|off}, the virtual time will jump to the next timer deadline
 instantly whenever the virtual cpu goes to sleep mode and will not advance
 if no timer is enabled. This behavior give deterministic execution times from
 the guest point of view.
@@ -3147,7 +3352,7 @@ provide cycle accurate emulation.  Modern CPUs contain superscalar out of
 order cores with complex cache hierarchies.  The number of instructions
 executed often has little or no correlation with actual performance.
 
-@option{align=on} will activate the delay algorithm which will try to
+@option{align=on} will activate the delay algorithm which will try
 to synchronise the host clock and the virtual clock. The goal is to
 have a guest running at the real frequency imposed by the shift option.
 Whenever the guest clock is behind the host clock and if
@@ -3157,6 +3362,10 @@ Currently this option does not work when @option{shift} is @code{auto}.
 Note: The sync algorithm will work for those shift values for which
 the guest clock runs ahead of the host clock. Typically this happens
 when the shift value is high (how high depends on the host machine).
+
+When @option{rr} option is specified deterministic record/replay is enabled.
+Replay log is written into @var{filename} file in record mode and
+read from this file in replay mode.
 ETEXI
 
 DEF("watchdog", HAS_ARG, QEMU_OPTION_watchdog, \
@@ -3437,7 +3646,7 @@ config files on @var{sysconfdir}, but won't make it skip the QEMU-provided confi
 files from @var{datadir}.
 ETEXI
 DEF("trace", HAS_ARG, QEMU_OPTION_trace,
-    "-trace [events=<file>][,file=<file>]\n"
+    "-trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
     "                specify tracing options\n",
     QEMU_ARCH_ALL)
 STEXI
@@ -3449,15 +3658,25 @@ HXCOMM HX does not support conditional compilation of text.
 Specify tracing options.
 
 @table @option
+@item [enable=]@var{pattern}
+Immediately enable events matching @var{pattern}.
+The file must contain one event name (as listed in the @file{trace-events} file)
+per line; globbing patterns are accepted too.  This option is only
+available if QEMU has been compiled with the @var{simple}, @var{stderr}
+or @var{ftrace} tracing backend.  To specify multiple events or patterns,
+specify the @option{-trace} option multiple times.
+
+Use @code{-trace help} to print a list of names of trace points.
+
 @item events=@var{file}
 Immediately enable events listed in @var{file}.
-The file must contain one event name (as listed in the @var{trace-events} file)
-per line.
-This option is only available if QEMU has been compiled with
-either @var{simple} or @var{stderr} tracing backend.
+The file must contain one event name (as listed in the @file{trace-events} file)
+per line; globbing patterns are accepted too.  This option is only
+available if QEMU has been compiled with the @var{simple}, @var{stderr} or
+@var{ftrace} tracing backend.
+
 @item file=@var{file}
 Log output traces to @var{file}.
-
 This option is only available if QEMU has been compiled with
 the @var{simple} tracing backend.
 @end table
@@ -3510,7 +3729,7 @@ DEF("dump-vmstate", HAS_ARG, QEMU_OPTION_dump_vmstate,
     "                Output vmstate information in JSON format to file.\n"
     "                Use the scripts/vmstate-static-checker.py file to\n"
     "                check for possible regressions in migration code\n"
-    "                by comparing two such vmstate dumps.",
+    "                by comparing two such vmstate dumps.\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -dump-vmstate @var{file}
@@ -3568,6 +3787,176 @@ the @option{virtio-rng} device. The @option{chardev} parameter is
 the unique ID of a character device backend that provides the connection
 to the RNG daemon.
 
+@item -object tls-creds-anon,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified, though this is a no-op for anonymous credentials.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+@item -object tls-creds-x509,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off},passwordid=@var{id}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified. With x509 certificates, this implies that the clients
+must be provided with valid client certificates too.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+For x509 certificate credentials the directory will contain further files
+providing the x509 certificates. The certificates must be stored
+in PEM format, in filenames @var{ca-cert.pem}, @var{ca-crl.pem} (optional),
+@var{server-cert.pem} (only servers), @var{server-key.pem} (only servers),
+@var{client-cert.pem} (only clients), and @var{client-key.pem} (only clients).
+
+For the @var{server-key.pem} and @var{client-key.pem} files which
+contain sensitive private keys, it is possible to use an encrypted
+version by providing the @var{passwordid} parameter. This provides
+the ID of a previously created @code{secret} object containing the
+password for decryption.
+
+@item -object filter-buffer,id=@var{id},netdev=@var{netdevid},interval=@var{t}[,queue=@var{all|rx|tx}][,status=@var{on|off}]
+
+Interval @var{t} can't be 0, this filter batches the packet delivery: all
+packets arriving in a given interval on netdev @var{netdevid} are delayed
+until the end of the interval. Interval is in microseconds.
+@option{status} is optional that indicate whether the netfilter is
+on (enabled) or off (disabled), the default status for netfilter will be 'on'.
+
+queue @var{all|rx|tx} is an option that can be applied to any netfilter.
+
+@option{all}: the filter is attached both to the receive and the transmit
+              queue of the netdev (default).
+
+@option{rx}: the filter is attached to the receive queue of the netdev,
+             where it will receive packets sent to the netdev.
+
+@option{tx}: the filter is attached to the transmit queue of the netdev,
+             where it will receive packets sent by the netdev.
+
+@item -object filter-mirror,id=@var{id},netdev=@var{netdevid},outdev=@var{chardevid}[,queue=@var{all|rx|tx}]
+
+filter-mirror on netdev @var{netdevid},mirror net packet to chardev
+@var{chardevid}
+
+@item -object filter-redirector,id=@var{id},netdev=@var{netdevid},indev=@var{chardevid},
+outdev=@var{chardevid}[,queue=@var{all|rx|tx}]
+
+filter-redirector on netdev @var{netdevid},redirect filter's net packet to chardev
+@var{chardevid},and redirect indev's packet to filter.
+Create a filter-redirector we need to differ outdev id from indev id, id can not
+be the same. we can just use indev or outdev, but at least one of indev or outdev
+need to be specified.
+
+@item -object filter-dump,id=@var{id},netdev=@var{dev},file=@var{filename}][,maxlen=@var{len}]
+
+Dump the network traffic on netdev @var{dev} to the file specified by
+@var{filename}. At most @var{len} bytes (64k by default) per packet are stored.
+The file format is libpcap, so it can be analyzed with tools such as tcpdump
+or Wireshark.
+
+@item -object secret,id=@var{id},data=@var{string},format=@var{raw|base64}[,keyid=@var{secretid},iv=@var{string}]
+@item -object secret,id=@var{id},file=@var{filename},format=@var{raw|base64}[,keyid=@var{secretid},iv=@var{string}]
+
+Defines a secret to store a password, encryption key, or some other sensitive
+data. The sensitive data can either be passed directly via the @var{data}
+parameter, or indirectly via the @var{file} parameter. Using the @var{data}
+parameter is insecure unless the sensitive data is encrypted.
+
+The sensitive data can be provided in raw format (the default), or base64.
+When encoded as JSON, the raw format only supports valid UTF-8 characters,
+so base64 is recommended for sending binary data. QEMU will convert from
+which ever format is provided to the format it needs internally. eg, an
+RBD password can be provided in raw format, even though it will be base64
+encoded when passed onto the RBD sever.
+
+For added protection, it is possible to encrypt the data associated with
+a secret using the AES-256-CBC cipher. Use of encryption is indicated
+by providing the @var{keyid} and @var{iv} parameters. The @var{keyid}
+parameter provides the ID of a previously defined secret that contains
+the AES-256 decryption key. This key should be 32-bytes long and be
+base64 encoded. The @var{iv} parameter provides the random initialization
+vector used for encryption of this particular secret and should be a
+base64 encrypted string of the 16-byte IV.
+
+The simplest (insecure) usage is to provide the secret inline
+
+@example
+
+ # $QEMU -object secret,id=sec0,data=letmein,format=raw
+
+@end example
+
+The simplest secure usage is to provide the secret via a file
+
+ # echo -n "letmein" > mypasswd.txt
+ # $QEMU -object secret,id=sec0,file=mypasswd.txt,format=raw
+
+For greater security, AES-256-CBC should be used. To illustrate usage,
+consider the openssl command line tool which can encrypt the data. Note
+that when encrypting, the plaintext must be padded to the cipher block
+size (32 bytes) using the standard PKCS#5/6 compatible padding algorithm.
+
+First a master key needs to be created in base64 encoding:
+
+@example
+ # openssl rand -base64 32 > key.b64
+ # KEY=$(base64 -d key.b64 | hexdump  -v -e '/1 "%02X"')
+@end example
+
+Each secret to be encrypted needs to have a random initialization vector
+generated. These do not need to be kept secret
+
+@example
+ # openssl rand -base64 16 > iv.b64
+ # IV=$(base64 -d iv.b64 | hexdump  -v -e '/1 "%02X"')
+@end example
+
+The secret to be defined can now be encrypted, in this case we're
+telling openssl to base64 encode the result, but it could be left
+as raw bytes if desired.
+
+@example
+ # SECRET=$(echo -n "letmein" |
+            openssl enc -aes-256-cbc -a -K $KEY -iv $IV)
+@end example
+
+When launching QEMU, create a master secret pointing to @code{key.b64}
+and specify that to be used to decrypt the user password. Pass the
+contents of @code{iv.b64} to the second secret
+
+@example
+ # $QEMU \
+     -object secret,id=secmaster0,format=base64,file=key.b64 \
+     -object secret,id=sec0,keyid=secmaster0,format=base64,\
+         data=$SECRET,iv=$(<iv.b64)
+@end example
+
 @end table
 
 ETEXI
diff --git a/qemu/qemu-seccomp.c b/qemu/qemu-seccomp.c
index f9de0d339..cb569dc05 100644
--- a/qemu/qemu-seccomp.c
+++ b/qemu/qemu-seccomp.c
@@ -12,10 +12,20 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <seccomp.h>
 #include "sysemu/seccomp.h"
 
+/* For some architectures (notably ARM) cacheflush is not supported until
+ * libseccomp 2.2.3, but configure enforces that we are using a more recent
+ * version on those hosts, so it is OK for this check to be less strict.
+ */
+#if SCMP_VER_MAJOR >= 3
+  #define HAVE_CACHEFLUSH
+#elif SCMP_VER_MAJOR == 2 && SCMP_VER_MINOR >= 2
+  #define HAVE_CACHEFLUSH
+#endif
+
 struct QemuSeccompSyscall {
     int32_t num;
     uint8_t priority;
@@ -237,7 +247,12 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(fadvise64), 240 },
     { SCMP_SYS(inotify_init1), 240 },
     { SCMP_SYS(inotify_add_watch), 240 },
-    { SCMP_SYS(mbind), 240 }
+    { SCMP_SYS(mbind), 240 },
+    { SCMP_SYS(memfd_create), 240 },
+#ifdef HAVE_CACHEFLUSH
+    { SCMP_SYS(cacheflush), 240 },
+#endif
+    { SCMP_SYS(sysinfo), 240 },
 };
 
 int seccomp_start(void)
diff --git a/qemu/qemu-tech.texi b/qemu/qemu-tech.texi
index b6fcb2d61..bdb2285f4 100644
--- a/qemu/qemu-tech.texi
+++ b/qemu/qemu-tech.texi
@@ -320,7 +320,7 @@ SH4
 @node QEMU compared to other emulators
 @section QEMU compared to other emulators
 
-Like bochs [3], QEMU emulates an x86 CPU. But QEMU is much faster than
+Like bochs [1], QEMU emulates an x86 CPU. But QEMU is much faster than
 bochs as it uses dynamic compilation. Bochs is closely tied to x86 PC
 emulation while QEMU can emulate several processors.
 
@@ -333,25 +333,25 @@ than QEMU (in particular it does register allocation) but it is closely
 tied to an x86 host and target and has no support for precise exceptions
 and system emulation.
 
-EM86 [4] is the closest project to user space QEMU (and QEMU still uses
+EM86 [3] is the closest project to user space QEMU (and QEMU still uses
 some of its code, in particular the ELF file loader). EM86 was limited
 to an alpha host and used a proprietary and slow interpreter (the
-interpreter part of the FX!32 Digital Win32 code translator [5]).
+interpreter part of the FX!32 Digital Win32 code translator [4]).
 
-TWIN [6] is a Windows API emulator like Wine. It is less accurate than
-Wine but includes a protected mode x86 interpreter to launch x86 Windows
-executables. Such an approach has greater potential because most of the
-Windows API is executed natively but it is far more difficult to develop
-because all the data structures and function parameters exchanged
+TWIN from Willows Software was a Windows API emulator like Wine. It is less
+accurate than Wine but includes a protected mode x86 interpreter to launch
+x86 Windows executables. Such an approach has greater potential because most
+of the Windows API is executed natively but it is far more difficult to
+develop because all the data structures and function parameters exchanged
 between the API and the x86 code must be converted.
 
-User mode Linux [7] was the only solution before QEMU to launch a
+User mode Linux [5] was the only solution before QEMU to launch a
 Linux kernel as a process while not needing any host kernel
 patches. However, user mode Linux requires heavy kernel patches while
 QEMU accepts unpatched Linux kernels. The price to pay is that QEMU is
 slower.
 
-The Plex86 [8] PC virtualizer is done in the same spirit as the now
+The Plex86 [6] PC virtualizer is done in the same spirit as the now
 obsolete qemu-fast system emulator. It requires a patched Linux kernel
 to work (you cannot launch the same kernel on your PC), but the
 patches are really small. As it is a PC virtualizer (no emulation is
@@ -359,13 +359,13 @@ done except for some privileged instructions), it has the potential of
 being faster than QEMU. The downside is that a complicated (and
 potentially unsafe) host kernel patch is needed.
 
-The commercial PC Virtualizers (VMWare [9], VirtualPC [10], TwoOStwo
-[11]) are faster than QEMU, but they all need specific, proprietary
+The commercial PC Virtualizers (VMWare [7], VirtualPC [8]) are faster
+than QEMU (without virtualization), but they all need specific, proprietary
 and potentially unsafe host drivers. Moreover, they are unable to
 provide cycle exact simulation as an emulator can.
 
-VirtualBox [12], Xen [13] and KVM [14] are based on QEMU. QEMU-SystemC
-[15] uses QEMU to simulate a system where some hardware devices are
+VirtualBox [9], Xen [10] and KVM [11] are based on QEMU. QEMU-SystemC
+[12] uses QEMU to simulate a system where some hardware devices are
 developed in SystemC.
 
 @node Portable dynamic translation
@@ -385,7 +385,7 @@ ops (see @code{target-i386/translate.c}). Some optimizations can be
 performed at this stage, including liveness analysis and trivial
 constant expression evaluation. TCG ops are then implemented in the
 host CPU back end, also known as TCG target (see
-@code{tcg/i386/tcg-target.c}). For more information, please take a
+@code{tcg/i386/tcg-target.inc.c}). For more information, please take a
 look at @code{tcg/README}.
 
 @node Condition code optimisations
@@ -608,64 +608,51 @@ way, it can be relocated at load time.
 @table @asis
 
 @item [1]
-@url{http://citeseer.nj.nec.com/piumarta98optimizing.html}, Optimizing
-direct threaded code by selective inlining (1998) by Ian Piumarta, Fabio
-Riccardi.
+@url{http://bochs.sourceforge.net/}, the Bochs IA-32 Emulator Project,
+by Kevin Lawton et al.
 
 @item [2]
-@url{http://developer.kde.org/~sewardj/}, Valgrind, an open-source
-memory debugger for x86-GNU/Linux, by Julian Seward.
+@url{http://www.valgrind.org/}, Valgrind, an open-source memory debugger
+for GNU/Linux.
 
 @item [3]
-@url{http://bochs.sourceforge.net/}, the Bochs IA-32 Emulator Project,
-by Kevin Lawton et al.
+@url{http://ftp.dreamtime.org/pub/linux/Linux-Alpha/em86/v0.2/docs/em86.html},
+the EM86 x86 emulator on Alpha-Linux.
 
 @item [4]
-@url{http://www.cs.rose-hulman.edu/~donaldlf/em86/index.html}, the EM86
-x86 emulator on Alpha-Linux.
-
-@item [5]
 @url{http://www.usenix.org/publications/library/proceedings/usenix-nt97/@/full_papers/chernoff/chernoff.pdf},
 DIGITAL FX!32: Running 32-Bit x86 Applications on Alpha NT, by Anton
 Chernoff and Ray Hookway.
 
-@item [6]
-@url{http://www.willows.com/}, Windows API library emulation from
-Willows Software.
-
-@item [7]
+@item [5]
 @url{http://user-mode-linux.sourceforge.net/},
 The User-mode Linux Kernel.
 
-@item [8]
+@item [6]
 @url{http://www.plex86.org/},
 The new Plex86 project.
 
-@item [9]
+@item [7]
 @url{http://www.vmware.com/},
 The VMWare PC virtualizer.
 
-@item [10]
-@url{http://www.microsoft.com/windowsxp/virtualpc/},
+@item [8]
+@url{https://www.microsoft.com/download/details.aspx?id=3702},
 The VirtualPC PC virtualizer.
 
-@item [11]
-@url{http://www.twoostwo.org/},
-The TwoOStwo PC virtualizer.
-
-@item [12]
+@item [9]
 @url{http://virtualbox.org/},
 The VirtualBox PC virtualizer.
 
-@item [13]
+@item [10]
 @url{http://www.xen.org/},
 The Xen hypervisor.
 
-@item [14]
-@url{http://kvm.qumranet.com/kvmwiki/Front_Page},
+@item [11]
+@url{http://www.linux-kvm.org/},
 Kernel Based Virtual Machine (KVM).
 
-@item [15]
+@item [12]
 @url{http://www.greensocs.com/projects/QEMUSystemC},
 QEMU-SystemC, a hardware co-simulator.
 
diff --git a/qemu/qemu-timer.c b/qemu/qemu-timer.c
index 2463fe6f6..4441fe66f 100644
--- a/qemu/qemu-timer.c
+++ b/qemu/qemu-timer.c
@@ -22,8 +22,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/main-loop.h"
 #include "qemu/timer.h"
+#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
 
 #ifdef CONFIG_POSIX
 #include <pthread.h>
@@ -391,7 +394,9 @@ static bool timer_mod_ns_locked(QEMUTimerList *timer_list,
 static void timerlist_rearm(QEMUTimerList *timer_list)
 {
     /* Interrupt execution to force deadline recalculation.  */
-    qemu_clock_warp(timer_list->clock->type);
+    if (timer_list->clock->type == QEMU_CLOCK_VIRTUAL) {
+        qemu_start_warp_timer();
+    }
     timerlist_notify(timer_list);
 }
 
@@ -477,10 +482,31 @@ bool timerlist_run_timers(QEMUTimerList *timer_list)
     void *opaque;
 
     qemu_event_reset(&timer_list->timers_done_ev);
-    if (!timer_list->clock->enabled) {
+    if (!timer_list->clock->enabled || !timer_list->active_timers) {
         goto out;
     }
 
+    switch (timer_list->clock->type) {
+    case QEMU_CLOCK_REALTIME:
+        break;
+    default:
+    case QEMU_CLOCK_VIRTUAL:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_HOST:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_HOST)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_VIRTUAL_RT:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL_RT)) {
+            goto out;
+        }
+        break;
+    }
+
     current_time = qemu_clock_get_ns(timer_list->clock->type);
     for(;;) {
         qemu_mutex_lock(&timer_list->active_timers_lock);
@@ -544,11 +570,17 @@ int64_t timerlistgroup_deadline_ns(QEMUTimerListGroup *tlg)
 {
     int64_t deadline = -1;
     QEMUClockType type;
+    bool play = replay_mode == REPLAY_MODE_PLAY;
     for (type = 0; type < QEMU_CLOCK_MAX; type++) {
-        if (qemu_clock_use_for_deadline(tlg->tl[type]->clock->type)) {
-            deadline = qemu_soonest_timeout(deadline,
-                                            timerlist_deadline_ns(
-                                                tlg->tl[type]));
+        if (qemu_clock_use_for_deadline(type)) {
+            if (!play || type == QEMU_CLOCK_REALTIME) {
+                deadline = qemu_soonest_timeout(deadline,
+                                                timerlist_deadline_ns(tlg->tl[type]));
+            } else {
+                /* Read clock from the replay file and
+                   do not calculate the deadline, based on virtual clock. */
+                qemu_clock_get_ns(type);
+            }
         }
     }
     return deadline;
@@ -570,7 +602,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
             return cpu_get_clock();
         }
     case QEMU_CLOCK_HOST:
-        now = get_clock_realtime();
+        now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
         last = clock->last;
         clock->last = now;
         if (now < last || now > (last + get_max_clock_jump())) {
@@ -578,7 +610,7 @@ int64_t qemu_clock_get_ns(QEMUClockType type)
         }
         return now;
     case QEMU_CLOCK_VIRTUAL_RT:
-        return cpu_get_clock();
+        return REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT, cpu_get_clock());
     }
 }
 
diff --git a/qemu/qemu.nsi b/qemu/qemu.nsi
index cc5fafd57..a20f6ef35 100644
--- a/qemu/qemu.nsi
+++ b/qemu/qemu.nsi
@@ -139,6 +139,9 @@ Section "${PRODUCT} (required)"
 
     ; Write the uninstall keys for Windows
     WriteRegStr HKLM "${UNINST_KEY}" "DisplayName" "QEMU"
+!ifdef DISPLAYVERSION
+    WriteRegStr HKLM "${UNINST_KEY}" "DisplayVersion" "${DISPLAYVERSION}"
+!endif
     WriteRegStr HKLM "${UNINST_KEY}" "UninstallString" '"${UNINST_EXE}"'
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoModify" 1
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoRepair" 1
diff --git a/qemu/qga/channel-posix.c b/qemu/qga/channel-posix.c
index 8aad4fee9..63458c663 100644
--- a/qemu/qga/channel-posix.c
+++ b/qemu/qga/channel-posix.c
@@ -1,11 +1,7 @@
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <termios.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <string.h>
-#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h"
 #include "qga/channel.h"
 
@@ -217,25 +213,24 @@ GIOStatus ga_channel_write_all(GAChannel *c, const gchar *buf, gsize size)
     GIOStatus status = G_IO_STATUS_NORMAL;
 
     while (size) {
+        g_debug("sending data, count: %d", (int)size);
         status = g_io_channel_write_chars(c->client_channel, buf, size,
                                           &written, &err);
-        g_debug("sending data, count: %d", (int)size);
-        if (err != NULL) {
+        if (status == G_IO_STATUS_NORMAL) {
+            size -= written;
+            buf += written;
+        } else if (status != G_IO_STATUS_AGAIN) {
             g_warning("error writing to channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
+            return status;
         }
-        if (status != G_IO_STATUS_NORMAL) {
-            break;
-        }
-        size -= written;
     }
 
-    if (status == G_IO_STATUS_NORMAL) {
+    do {
         status = g_io_channel_flush(c->client_channel, &err);
-        if (err != NULL) {
-            g_warning("error flushing channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
-        }
+    } while (status == G_IO_STATUS_AGAIN);
+
+    if (status != G_IO_STATUS_NORMAL) {
+        g_warning("error flushing channel: %s", err->message);
     }
 
     return status;
@@ -249,7 +244,7 @@ GIOStatus ga_channel_read(GAChannel *c, gchar *buf, gsize size, gsize *count)
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     c->event_cb = cb;
     c->user_data = opaque;
 
diff --git a/qemu/qga/channel-win32.c b/qemu/qga/channel-win32.c
index 04fa5e4d1..bb5966124 100644
--- a/qemu/qga/channel-win32.c
+++ b/qemu/qga/channel-win32.c
@@ -1,9 +1,6 @@
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <windows.h>
-#include <errno.h>
 #include <io.h>
 #include "qga/guest-agent-core.h"
 #include "qga/channel.h"
@@ -269,7 +266,7 @@ static GIOStatus ga_channel_write(GAChannel *c, const char *buf, size_t size,
 GIOStatus ga_channel_write_all(GAChannel *c, const char *buf, size_t size)
 {
     GIOStatus status = G_IO_STATUS_NORMAL;
-    size_t count;
+    size_t count = 0;
 
     while (size) {
         status = ga_channel_write(c, buf, size, &count);
@@ -322,7 +319,7 @@ static gboolean ga_channel_open(GAChannel *c, GAChannelMethod method,
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     SECURITY_ATTRIBUTES sec_attrs;
 
     if (!ga_channel_open(c, method, path)) {
diff --git a/qemu/qga/commands-posix.c b/qemu/qga/commands-posix.c
index 675f4b4c6..2ae37255d 100644
--- a/qemu/qga/commands-posix.c
+++ b/qemu/qga/commands-posix.c
@@ -11,23 +11,19 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/wait.h>
-#include <unistd.h>
-#include <errno.h>
-#include <fcntl.h>
 #include <dirent.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <inttypes.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/queue.h"
 #include "qemu/host-utils.h"
+#include "qemu/sockets.h"
+#include "qemu/base64.h"
+#include "qemu/cutils.h"
 
 #ifndef CONFIG_HAS_ENVIRON
 #ifdef __APPLE__
@@ -215,15 +211,24 @@ void qmp_guest_set_time(bool has_time, int64_t time_ns, Error **errp)
     }
 }
 
+typedef enum {
+    RW_STATE_NEW,
+    RW_STATE_READING,
+    RW_STATE_WRITING,
+} RwState;
+
 typedef struct GuestFileHandle {
     uint64_t id;
     FILE *fh;
+    RwState state;
     QTAILQ_ENTRY(GuestFileHandle) next;
 } GuestFileHandle;
 
 static struct {
     QTAILQ_HEAD(, GuestFileHandle) filehandles;
-} guest_file_state;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
 
 static int64_t guest_file_handle_add(FILE *fh, Error **errp)
 {
@@ -235,7 +240,7 @@ static int64_t guest_file_handle_add(FILE *fh, Error **errp)
         return -1;
     }
 
-    gfh = g_malloc0(sizeof(GuestFileHandle));
+    gfh = g_new0(GuestFileHandle, 1);
     gfh->id = handle;
     gfh->fh = fh;
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
@@ -383,27 +388,6 @@ safe_open_or_create(const char *path, const char *mode, Error **errp)
     return NULL;
 }
 
-static int guest_file_toggle_flags(int fd, int flags, bool set, Error **err)
-{
-    int ret, old_flags;
-
-    old_flags = fcntl(fd, F_GETFL);
-    if (old_flags == -1) {
-        error_setg_errno(err, errno, QERR_QGA_COMMAND_FAILED,
-                         "failed to fetch filehandle flags");
-        return -1;
-    }
-
-    ret = fcntl(fd, F_SETFL, set ? (old_flags | flags) : (old_flags & ~flags));
-    if (ret == -1) {
-        error_setg_errno(err, errno, QERR_QGA_COMMAND_FAILED,
-                         "failed to set filehandle flags");
-        return -1;
-    }
-
-    return ret;
-}
-
 int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode,
                             Error **errp)
 {
@@ -424,10 +408,7 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode,
     /* set fd non-blocking to avoid common use cases (like reading from a
      * named pipe) from hanging the agent
      */
-    if (guest_file_toggle_flags(fileno(fh), O_NONBLOCK, true, errp) < 0) {
-        fclose(fh);
-        return -1;
-    }
+    qemu_set_nonblock(fileno(fh));
 
     handle = guest_file_handle_add(fh, errp);
     if (handle < 0) {
@@ -481,6 +462,17 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
     }
 
     fh = gfh->fh;
+
+    /* explicitly flush when switching from writing to reading */
+    if (gfh->state == RW_STATE_WRITING) {
+        int ret = fflush(fh);
+        if (ret == EOF) {
+            error_setg_errno(errp, errno, "failed to flush file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
     buf = g_malloc0(count+1);
     read_count = fread(buf, 1, count, fh);
     if (ferror(fh)) {
@@ -488,12 +480,13 @@ struct GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
         slog("guest-file-read failed, handle: %" PRId64, handle);
     } else {
         buf[read_count] = 0;
-        read_data = g_malloc0(sizeof(GuestFileRead));
+        read_data = g_new0(GuestFileRead, 1);
         read_data->count = read_count;
         read_data->eof = feof(fh);
         if (read_count) {
             read_data->buf_b64 = g_base64_encode(buf, read_count);
         }
+        gfh->state = RW_STATE_READING;
     }
     g_free(buf);
     clearerr(fh);
@@ -517,7 +510,20 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
     }
 
     fh = gfh->fh;
-    buf = g_base64_decode(buf_b64, &buf_len);
+
+    if (gfh->state == RW_STATE_READING) {
+        int ret = fseek(fh, 0, SEEK_CUR);
+        if (ret == -1) {
+            error_setg_errno(errp, errno, "failed to seek file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
+    buf = qbase64_decode(buf_b64, -1, &buf_len, errp);
+    if (!buf) {
+        return NULL;
+    }
 
     if (!has_count) {
         count = buf_len;
@@ -533,9 +539,10 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
         error_setg_errno(errp, errno, "failed to write to file");
         slog("guest-file-write failed, handle: %" PRId64, handle);
     } else {
-        write_data = g_malloc0(sizeof(GuestFileWrite));
+        write_data = g_new0(GuestFileWrite, 1);
         write_data->count = write_count;
         write_data->eof = feof(fh);
+        gfh->state = RW_STATE_WRITING;
     }
     g_free(buf);
     clearerr(fh);
@@ -544,25 +551,40 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
 }
 
 struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                          int64_t whence, Error **errp)
+                                          GuestFileWhence *whence_code,
+                                          Error **errp)
 {
     GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
     GuestFileSeek *seek_data = NULL;
     FILE *fh;
     int ret;
+    int whence;
+    Error *err = NULL;
 
     if (!gfh) {
         return NULL;
     }
 
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    whence = ga_parse_whence(whence_code, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
     fh = gfh->fh;
     ret = fseek(fh, offset, whence);
     if (ret == -1) {
         error_setg_errno(errp, errno, "failed to seek file");
+        if (errno == ESPIPE) {
+            /* file is non-seekable, stdio shouldn't be buffering anyways */
+            gfh->state = RW_STATE_NEW;
+        }
     } else {
         seek_data = g_new0(GuestFileSeek, 1);
         seek_data->position = ftell(fh);
         seek_data->eof = feof(fh);
+        gfh->state = RW_STATE_NEW;
     }
     clearerr(fh);
 
@@ -583,14 +605,11 @@ void qmp_guest_file_flush(int64_t handle, Error **errp)
     ret = fflush(fh);
     if (ret == EOF) {
         error_setg_errno(errp, errno, "failed to flush file");
+    } else {
+        gfh->state = RW_STATE_NEW;
     }
 }
 
-static void guest_file_init(void)
-{
-    QTAILQ_INIT(&guest_file_state.filehandles);
-}
-
 /* linux-specific implementations. avoid this if at all possible. */
 #if defined(__linux__)
 
@@ -678,7 +697,7 @@ static void build_fs_mount_list_from_mtab(FsMountList *mounts, Error **errp)
             continue;
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(ment->mnt_dir);
         mount->devtype = g_strdup(ment->mnt_type);
         mount->devmajor = devmajor;
@@ -757,7 +776,7 @@ static void build_fs_mount_list(FsMountList *mounts, Error **errp)
             }
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(line + dir_s);
         mount->devtype = g_strdup(dash + type_s);
         mount->devmajor = devmajor;
@@ -1935,7 +1954,10 @@ void qmp_guest_set_user_password(const char *username,
     char *chpasswddata = NULL;
     size_t chpasswdlen;
 
-    rawpasswddata = (char *)g_base64_decode(password, &rawpasswdlen);
+    rawpasswddata = (char *)qbase64_decode(password, -1, &rawpasswdlen, errp);
+    if (!rawpasswddata) {
+        return;
+    }
     rawpasswddata = g_renew(char, rawpasswddata, rawpasswdlen + 1);
     rawpasswddata[rawpasswdlen] = '\0';
 
@@ -2158,7 +2180,7 @@ static void transfer_memory_block(GuestMemoryBlock *mem_blk, bool sys2memblk,
 
         ga_read_sysfs_file(dirfd, "removable", &removable, 1, &local_err);
         if (local_err) {
-            /* if no 'removable' file, it does't support offline mem blk */
+            /* if no 'removable' file, it doesn't support offline mem blk */
             if (errno == ENOENT) {
                 error_free(local_err);
                 mem_blk->can_offline = false;
@@ -2213,8 +2235,14 @@ GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
 
     dp = opendir("/sys/devices/system/memory/");
     if (!dp) {
-        error_setg_errno(errp, errno, "Can't open directory"
-                         "\"/sys/devices/system/memory/\"\n");
+        /* it's ok if this happens to be a system that doesn't expose
+         * memory blocks via sysfs, but otherwise we should report
+         * an error
+         */
+        if (errno != ENOENT) {
+            error_setg_errno(errp, errno, "Can't open directory"
+                             "\"/sys/devices/system/memory/\"");
+        }
         return NULL;
     }
 
@@ -2454,7 +2482,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
@@ -2468,13 +2496,13 @@ GList *ga_command_blacklist_init(GList *blacklist)
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
 
 #if !defined(CONFIG_FSTRIM)
-    blacklist = g_list_append(blacklist, (char *)"guest-fstrim");
+    blacklist = g_list_append(blacklist, g_strdup("guest-fstrim"));
 #endif
 
     return blacklist;
@@ -2486,5 +2514,4 @@ void ga_command_state_init(GAState *s, GACommandState *cs)
 #if defined(CONFIG_FSFREEZE)
     ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
 #endif
-    ga_command_state_add(cs, guest_file_init, NULL);
 }
diff --git a/qemu/qga/commands-win32.c b/qemu/qga/commands-win32.c
index a7822d5ff..d76327f5a 100644
--- a/qemu/qga/commands-win32.c
+++ b/qemu/qga/commands-win32.c
@@ -11,11 +11,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <wtypes.h>
 #include <powrprof.h>
-#include <stdio.h>
-#include <string.h>
 #include <winsock2.h>
 #include <ws2tcpip.h>
 #include <iptypes.h>
@@ -26,12 +25,15 @@
 #include <setupapi.h>
 #include <initguid.h>
 #endif
+#include <lm.h>
+
 #include "qga/guest-agent-core.h"
 #include "qga/vss-win32.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/queue.h"
 #include "qemu/host-utils.h"
+#include "qemu/base64.h"
 
 #ifndef SHTDN_REASON_FLAG_PLANNED
 #define SHTDN_REASON_FLAG_PLANNED 0x80000000
@@ -53,8 +55,11 @@ typedef struct GuestFileHandle {
 
 static struct {
     QTAILQ_HEAD(, GuestFileHandle) filehandles;
-} guest_file_state;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
 
+#define FILE_GENERIC_APPEND (FILE_GENERIC_WRITE & ~FILE_WRITE_DATA)
 
 typedef struct OpenFlags {
     const char *forms;
@@ -62,20 +67,20 @@ typedef struct OpenFlags {
     DWORD creation_disposition;
 } OpenFlags;
 static OpenFlags guest_file_open_modes[] = {
-    {"r",   GENERIC_READ,               OPEN_EXISTING},
-    {"rb",  GENERIC_READ,               OPEN_EXISTING},
-    {"w",   GENERIC_WRITE,              CREATE_ALWAYS},
-    {"wb",  GENERIC_WRITE,              CREATE_ALWAYS},
-    {"a",   GENERIC_WRITE,              OPEN_ALWAYS  },
-    {"r+",  GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"rb+", GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"r+b", GENERIC_WRITE|GENERIC_READ, OPEN_EXISTING},
-    {"w+",  GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"wb+", GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"w+b", GENERIC_WRITE|GENERIC_READ, CREATE_ALWAYS},
-    {"a+",  GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  },
-    {"ab+", GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  },
-    {"a+b", GENERIC_WRITE|GENERIC_READ, OPEN_ALWAYS  }
+    {"r",   GENERIC_READ,                     OPEN_EXISTING},
+    {"rb",  GENERIC_READ,                     OPEN_EXISTING},
+    {"w",   GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"wb",  GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"a",   FILE_GENERIC_APPEND,              OPEN_ALWAYS  },
+    {"r+",  GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"rb+", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"r+b", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"w+",  GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"wb+", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"w+b", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"a+",  FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"ab+", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"a+b", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  }
 };
 
 static OpenFlags *find_open_flag(const char *mode_str)
@@ -104,7 +109,7 @@ static int64_t guest_file_handle_add(HANDLE fh, Error **errp)
     if (handle < 0) {
         return -1;
     }
-    gfh = g_malloc0(sizeof(GuestFileHandle));
+    gfh = g_new0(GuestFileHandle, 1);
     gfh->id = handle;
     gfh->fh = fh;
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
@@ -124,6 +129,28 @@ static GuestFileHandle *guest_file_handle_find(int64_t id, Error **errp)
     return NULL;
 }
 
+static void handle_set_nonblocking(HANDLE fh)
+{
+    DWORD file_type, pipe_state;
+    file_type = GetFileType(fh);
+    if (file_type != FILE_TYPE_PIPE) {
+        return;
+    }
+    /* If file_type == FILE_TYPE_PIPE, according to MSDN
+     * the specified file is socket or named pipe */
+    if (!GetNamedPipeHandleState(fh, &pipe_state, NULL,
+                                 NULL, NULL, NULL, 0)) {
+        return;
+    }
+    /* The fd is named pipe fd */
+    if (pipe_state & PIPE_NOWAIT) {
+        return;
+    }
+
+    pipe_state |= PIPE_NOWAIT;
+    SetNamedPipeHandleState(fh, &pipe_state, NULL, NULL);
+}
+
 int64_t qmp_guest_file_open(const char *path, bool has_mode,
                             const char *mode, Error **errp)
 {
@@ -154,9 +181,14 @@ int64_t qmp_guest_file_open(const char *path, bool has_mode,
         return -1;
     }
 
+    /* set fd non-blocking to avoid common use cases (like reading from a
+     * named pipe) from hanging the agent
+     */
+    handle_set_nonblocking(fh);
+
     fd = guest_file_handle_add(fh, errp);
     if (fd < 0) {
-        CloseHandle(&fh);
+        CloseHandle(fh);
         error_setg(errp, "failed to add handle to qmp handle table");
         return -1;
     }
@@ -296,7 +328,7 @@ GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
         slog("guest-file-read failed, handle %" PRId64, handle);
     } else {
         buf[read_count] = 0;
-        read_data = g_malloc0(sizeof(GuestFileRead));
+        read_data = g_new0(GuestFileRead, 1);
         read_data->count = (size_t)read_count;
         read_data->eof = read_count == 0;
 
@@ -325,7 +357,10 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
         return NULL;
     }
     fh = gfh->fh;
-    buf = g_base64_decode(buf_b64, &buf_len);
+    buf = qbase64_decode(buf_b64, -1, &buf_len, errp);
+    if (!buf) {
+        return NULL;
+    }
 
     if (!has_count) {
         count = buf_len;
@@ -340,7 +375,7 @@ GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
         error_setg_win32(errp, GetLastError(), "failed to write to file");
         slog("guest-file-write-failed, handle: %" PRId64, handle);
     } else {
-        write_data = g_malloc0(sizeof(GuestFileWrite));
+        write_data = g_new0(GuestFileWrite, 1);
         write_data->count = (size_t) write_count;
     }
 
@@ -350,7 +385,8 @@ done:
 }
 
 GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                   int64_t whence, Error **errp)
+                                   GuestFileWhence *whence_code,
+                                   Error **errp)
 {
     GuestFileHandle *gfh;
     GuestFileSeek *seek_data;
@@ -358,11 +394,21 @@ GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
     LARGE_INTEGER new_pos, off_pos;
     off_pos.QuadPart = offset;
     BOOL res;
+    int whence;
+    Error *err = NULL;
+
     gfh = guest_file_handle_find(handle, errp);
     if (!gfh) {
         return NULL;
     }
 
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    whence = ga_parse_whence(whence_code, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
     fh = gfh->fh;
     res = SetFilePointerEx(fh, off_pos, &new_pos, whence);
     if (!res) {
@@ -388,11 +434,6 @@ void qmp_guest_file_flush(int64_t handle, Error **errp)
     }
 }
 
-static void guest_file_init(void)
-{
-    QTAILQ_INIT(&guest_file_state.filehandles);
-}
-
 #ifdef CONFIG_QGA_NTDDSCSI
 
 static STORAGE_BUS_TYPE win2qemu[] = {
@@ -657,7 +698,7 @@ static GuestFilesystemInfo *build_guest_fsinfo(char *guid, Error **errp)
         fs->mountpoint = g_strndup(mnt_point, len);
     }
     fs->type = g_strdup(fs_name);
-    fs->disk = build_guest_disk_info(guid, errp);;
+    fs->disk = build_guest_disk_info(guid, errp);
 free:
     g_free(mnt_point);
     return fs;
@@ -863,7 +904,7 @@ static DWORD WINAPI do_suspend(LPVOID opaque)
 void qmp_guest_suspend_disk(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_DISK;
     check_suspend_mode(*mode, &local_err);
@@ -879,7 +920,7 @@ void qmp_guest_suspend_disk(Error **errp)
 void qmp_guest_suspend_ram(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_RAM;
     check_suspend_mode(*mode, &local_err);
@@ -1182,7 +1223,71 @@ void qmp_guest_set_time(bool has_time, int64_t time_ns, Error **errp)
 
 GuestLogicalProcessorList *qmp_guest_get_vcpus(Error **errp)
 {
-    error_setg(errp, QERR_UNSUPPORTED);
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION pslpi, ptr;
+    DWORD length;
+    GuestLogicalProcessorList *head, **link;
+    Error *local_err = NULL;
+    int64_t current;
+
+    ptr = pslpi = NULL;
+    length = 0;
+    current = 0;
+    head = NULL;
+    link = &head;
+
+    if ((GetLogicalProcessorInformation(pslpi, &length) == FALSE) &&
+        (GetLastError() == ERROR_INSUFFICIENT_BUFFER) &&
+        (length > sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION))) {
+        ptr = pslpi = g_malloc0(length);
+        if (GetLogicalProcessorInformation(pslpi, &length) == FALSE) {
+            error_setg(&local_err, "Failed to get processor information: %d",
+                       (int)GetLastError());
+        }
+    } else {
+        error_setg(&local_err,
+                   "Failed to get processor information buffer length: %d",
+                   (int)GetLastError());
+    }
+
+    while ((local_err == NULL) && (length > 0)) {
+        if (pslpi->Relationship == RelationProcessorCore) {
+            ULONG_PTR cpu_bits = pslpi->ProcessorMask;
+
+            while (cpu_bits > 0) {
+                if (!!(cpu_bits & 1)) {
+                    GuestLogicalProcessor *vcpu;
+                    GuestLogicalProcessorList *entry;
+
+                    vcpu = g_malloc0(sizeof *vcpu);
+                    vcpu->logical_id = current++;
+                    vcpu->online = true;
+                    vcpu->has_can_offline = false;
+
+                    entry = g_malloc0(sizeof *entry);
+                    entry->value = vcpu;
+
+                    *link = entry;
+                    link = &entry->next;
+                }
+                cpu_bits >>= 1;
+            }
+        }
+        length -= sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+        pslpi++; /* next entry */
+    }
+
+    g_free(ptr);
+
+    if (local_err == NULL) {
+        if (head != NULL) {
+            return head;
+        }
+        /* there's no guest with zero VCPUs */
+        error_setg(&local_err, "Guest reported zero VCPUs");
+    }
+
+    qapi_free_GuestLogicalProcessorList(head);
+    error_propagate(errp, local_err);
     return NULL;
 }
 
@@ -1192,12 +1297,103 @@ int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
     return -1;
 }
 
+static gchar *
+get_net_error_message(gint error)
+{
+    HMODULE module = NULL;
+    gchar *retval = NULL;
+    wchar_t *msg = NULL;
+    int flags;
+    size_t nchars;
+
+    flags = FORMAT_MESSAGE_ALLOCATE_BUFFER |
+        FORMAT_MESSAGE_IGNORE_INSERTS |
+        FORMAT_MESSAGE_FROM_SYSTEM;
+
+    if (error >= NERR_BASE && error <= MAX_NERR) {
+        module = LoadLibraryExW(L"netmsg.dll", NULL, LOAD_LIBRARY_AS_DATAFILE);
+
+        if (module != NULL) {
+            flags |= FORMAT_MESSAGE_FROM_HMODULE;
+        }
+    }
+
+    FormatMessageW(flags, module, error, 0, (LPWSTR)&msg, 0, NULL);
+
+    if (msg != NULL) {
+        nchars = wcslen(msg);
+
+        if (nchars >= 2 &&
+            msg[nchars - 1] == L'\n' &&
+            msg[nchars - 2] == L'\r') {
+            msg[nchars - 2] = L'\0';
+        }
+
+        retval = g_utf16_to_utf8(msg, -1, NULL, NULL, NULL);
+
+        LocalFree(msg);
+    }
+
+    if (module != NULL) {
+        FreeLibrary(module);
+    }
+
+    return retval;
+}
+
 void qmp_guest_set_user_password(const char *username,
                                  const char *password,
                                  bool crypted,
                                  Error **errp)
 {
-    error_setg(errp, QERR_UNSUPPORTED);
+    NET_API_STATUS nas;
+    char *rawpasswddata = NULL;
+    size_t rawpasswdlen;
+    wchar_t *user = NULL, *wpass = NULL;
+    USER_INFO_1003 pi1003 = { 0, };
+    GError *gerr = NULL;
+
+    if (crypted) {
+        error_setg(errp, QERR_UNSUPPORTED);
+        return;
+    }
+
+    rawpasswddata = (char *)qbase64_decode(password, -1, &rawpasswdlen, errp);
+    if (!rawpasswddata) {
+        return;
+    }
+    rawpasswddata = g_renew(char, rawpasswddata, rawpasswdlen + 1);
+    rawpasswddata[rawpasswdlen] = '\0';
+
+    user = g_utf8_to_utf16(username, -1, NULL, NULL, &gerr);
+    if (!user) {
+        goto done;
+    }
+
+    wpass = g_utf8_to_utf16(rawpasswddata, -1, NULL, NULL, &gerr);
+    if (!wpass) {
+        goto done;
+    }
+
+    pi1003.usri1003_password = wpass;
+    nas = NetUserSetInfo(NULL, user,
+                         1003, (LPBYTE)&pi1003,
+                         NULL);
+
+    if (nas != NERR_Success) {
+        gchar *msg = get_net_error_message(nas);
+        error_setg(errp, "failed to set password: %s", msg);
+        g_free(msg);
+    }
+
+done:
+    if (gerr) {
+        error_setg(errp, QERR_QGA_COMMAND_FAILED, gerr->message);
+        g_error_free(gerr);
+    }
+    g_free(user);
+    g_free(wpass);
+    g_free(rawpasswddata);
 }
 
 GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
@@ -1224,8 +1420,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
 {
     const char *list_unsupported[] = {
         "guest-suspend-hybrid",
-        "guest-get-vcpus", "guest-set-vcpus",
-        "guest-set-user-password",
+        "guest-set-vcpus",
         "guest-get-memory-blocks", "guest-set-memory-blocks",
         "guest-get-memory-block-size",
         "guest-fsfreeze-freeze-list",
@@ -1233,7 +1428,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
     char **p = (char **)list_unsupported;
 
     while (*p) {
-        blacklist = g_list_append(blacklist, *p++);
+        blacklist = g_list_append(blacklist, g_strdup(*p++));
     }
 
     if (!vss_init(true)) {
@@ -1244,7 +1439,7 @@ GList *ga_command_blacklist_init(GList *blacklist)
         p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 
@@ -1257,5 +1452,4 @@ void ga_command_state_init(GAState *s, GACommandState *cs)
     if (!vss_initialized()) {
         ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
     }
-    ga_command_state_add(cs, guest_file_init, NULL);
 }
diff --git a/qemu/qga/commands.c b/qemu/qga/commands.c
index 783496791..b653a460b 100644
--- a/qemu/qga/commands.c
+++ b/qemu/qga/commands.c
@@ -10,10 +10,18 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/base64.h"
+#include "qemu/cutils.h"
+
+/* Maximum captured guest-exec out_data/err_data - 16MB */
+#define GUEST_EXEC_MAX_OUTPUT (16*1024*1024)
+/* Allocation and I/O buffer for reading guest-exec out_data/err_data - 4KB */
+#define GUEST_EXEC_IO_SIZE (4*1024)
 
 /* Note: in some situations, like with the fsfreeze, logging may be
  * temporarilly disabled. if it is necessary that a command be able
@@ -51,12 +59,12 @@ static void qmp_command_info(QmpCommand *cmd, void *opaque)
     GuestAgentCommandInfo *cmd_info;
     GuestAgentCommandInfoList *cmd_info_list;
 
-    cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
+    cmd_info = g_new0(GuestAgentCommandInfo, 1);
     cmd_info->name = g_strdup(qmp_command_name(cmd));
     cmd_info->enabled = qmp_command_is_enabled(cmd);
     cmd_info->success_response = qmp_has_success_response(cmd);
 
-    cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
+    cmd_info_list = g_new0(GuestAgentCommandInfoList, 1);
     cmd_info_list->value = cmd_info;
     cmd_info_list->next = info->supported_commands;
     info->supported_commands = cmd_info_list;
@@ -64,9 +72,430 @@ static void qmp_command_info(QmpCommand *cmd, void *opaque)
 
 struct GuestAgentInfo *qmp_guest_info(Error **errp)
 {
-    GuestAgentInfo *info = g_malloc0(sizeof(GuestAgentInfo));
+    GuestAgentInfo *info = g_new0(GuestAgentInfo, 1);
 
     info->version = g_strdup(QEMU_VERSION);
     qmp_for_each_command(qmp_command_info, info);
     return info;
 }
+
+struct GuestExecIOData {
+    guchar *data;
+    gsize size;
+    gsize length;
+    gint closed;
+    bool truncated;
+    const char *name;
+};
+typedef struct GuestExecIOData GuestExecIOData;
+
+struct GuestExecInfo {
+    GPid pid;
+    int64_t pid_numeric;
+    gint status;
+    bool has_output;
+    gint finished;
+    GuestExecIOData in;
+    GuestExecIOData out;
+    GuestExecIOData err;
+    QTAILQ_ENTRY(GuestExecInfo) next;
+};
+typedef struct GuestExecInfo GuestExecInfo;
+
+static struct {
+    QTAILQ_HEAD(, GuestExecInfo) processes;
+} guest_exec_state = {
+    .processes = QTAILQ_HEAD_INITIALIZER(guest_exec_state.processes),
+};
+
+static int64_t gpid_to_int64(GPid pid)
+{
+#ifdef G_OS_WIN32
+    return GetProcessId(pid);
+#else
+    return (int64_t)pid;
+#endif
+}
+
+static GuestExecInfo *guest_exec_info_add(GPid pid)
+{
+    GuestExecInfo *gei;
+
+    gei = g_new0(GuestExecInfo, 1);
+    gei->pid = pid;
+    gei->pid_numeric = gpid_to_int64(pid);
+    QTAILQ_INSERT_TAIL(&guest_exec_state.processes, gei, next);
+
+    return gei;
+}
+
+static GuestExecInfo *guest_exec_info_find(int64_t pid_numeric)
+{
+    GuestExecInfo *gei;
+
+    QTAILQ_FOREACH(gei, &guest_exec_state.processes, next) {
+        if (gei->pid_numeric == pid_numeric) {
+            return gei;
+        }
+    }
+
+    return NULL;
+}
+
+GuestExecStatus *qmp_guest_exec_status(int64_t pid, Error **err)
+{
+    GuestExecInfo *gei;
+    GuestExecStatus *ges;
+
+    slog("guest-exec-status called, pid: %u", (uint32_t)pid);
+
+    gei = guest_exec_info_find(pid);
+    if (gei == NULL) {
+        error_setg(err, QERR_INVALID_PARAMETER, "pid");
+        return NULL;
+    }
+
+    ges = g_new0(GuestExecStatus, 1);
+
+    bool finished = g_atomic_int_get(&gei->finished);
+
+    /* need to wait till output channels are closed
+     * to be sure we captured all output at this point */
+    if (gei->has_output) {
+        finished = finished && g_atomic_int_get(&gei->out.closed);
+        finished = finished && g_atomic_int_get(&gei->err.closed);
+    }
+
+    ges->exited = finished;
+    if (finished) {
+        /* Glib has no portable way to parse exit status.
+         * On UNIX, we can get either exit code from normal termination
+         * or signal number.
+         * On Windows, it is either the same exit code or the exception
+         * value for an unhandled exception that caused the process
+         * to terminate.
+         * See MSDN for GetExitCodeProcess() and ntstatus.h for possible
+         * well-known codes, e.g. C0000005 ACCESS_DENIED - analog of SIGSEGV
+         * References:
+         *   https://msdn.microsoft.com/en-us/library/windows/desktop/ms683189(v=vs.85).aspx
+         *   https://msdn.microsoft.com/en-us/library/aa260331(v=vs.60).aspx
+         */
+#ifdef G_OS_WIN32
+        /* Additionally WIN32 does not provide any additional information
+         * on whetherthe child exited or terminated via signal.
+         * We use this simple range check to distingish application exit code
+         * (usually value less then 256) and unhandled exception code with
+         * ntstatus (always value greater then 0xC0000005). */
+        if ((uint32_t)gei->status < 0xC0000000U) {
+            ges->has_exitcode = true;
+            ges->exitcode = gei->status;
+        } else {
+            ges->has_signal = true;
+            ges->signal = gei->status;
+        }
+#else
+        if (WIFEXITED(gei->status)) {
+            ges->has_exitcode = true;
+            ges->exitcode = WEXITSTATUS(gei->status);
+        } else if (WIFSIGNALED(gei->status)) {
+            ges->has_signal = true;
+            ges->signal = WTERMSIG(gei->status);
+        }
+#endif
+        if (gei->out.length > 0) {
+            ges->has_out_data = true;
+            ges->out_data = g_base64_encode(gei->out.data, gei->out.length);
+            g_free(gei->out.data);
+            ges->has_out_truncated = gei->out.truncated;
+        }
+
+        if (gei->err.length > 0) {
+            ges->has_err_data = true;
+            ges->err_data = g_base64_encode(gei->err.data, gei->err.length);
+            g_free(gei->err.data);
+            ges->has_err_truncated = gei->err.truncated;
+        }
+
+        QTAILQ_REMOVE(&guest_exec_state.processes, gei, next);
+        g_free(gei);
+    }
+
+    return ges;
+}
+
+/* Get environment variables or arguments array for execve(). */
+static char **guest_exec_get_args(const strList *entry, bool log)
+{
+    const strList *it;
+    int count = 1, i = 0;  /* reserve for NULL terminator */
+    char **args;
+    char *str; /* for logging array of arguments */
+    size_t str_size = 1;
+
+    for (it = entry; it != NULL; it = it->next) {
+        count++;
+        str_size += 1 + strlen(it->value);
+    }
+
+    str = g_malloc(str_size);
+    *str = 0;
+    args = g_malloc(count * sizeof(char *));
+    for (it = entry; it != NULL; it = it->next) {
+        args[i++] = it->value;
+        pstrcat(str, str_size, it->value);
+        if (it->next) {
+            pstrcat(str, str_size, " ");
+        }
+    }
+    args[i] = NULL;
+
+    if (log) {
+        slog("guest-exec called: \"%s\"", str);
+    }
+    g_free(str);
+
+    return args;
+}
+
+static void guest_exec_child_watch(GPid pid, gint status, gpointer data)
+{
+    GuestExecInfo *gei = (GuestExecInfo *)data;
+
+    g_debug("guest_exec_child_watch called, pid: %d, status: %u",
+            (int32_t)gpid_to_int64(pid), (uint32_t)status);
+
+    gei->status = status;
+    gei->finished = true;
+
+    g_spawn_close_pid(pid);
+}
+
+/** Reset ignored signals back to default. */
+static void guest_exec_task_setup(gpointer data)
+{
+#if !defined(G_OS_WIN32)
+    struct sigaction sigact;
+
+    memset(&sigact, 0, sizeof(struct sigaction));
+    sigact.sa_handler = SIG_DFL;
+
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        slog("sigaction() failed to reset child process's SIGPIPE: %s",
+             strerror(errno));
+    }
+#endif
+}
+
+static gboolean guest_exec_input_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_written = 0;
+    GIOStatus status;
+    GError *gerr = NULL;
+
+    /* nothing left to write */
+    if (p->size == p->length) {
+        goto done;
+    }
+
+    status = g_io_channel_write_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_written, &gerr);
+
+    /* can be not 0 even if not G_IO_STATUS_NORMAL */
+    if (bytes_written != 0) {
+        p->length += bytes_written;
+    }
+
+    /* continue write, our callback will be called again */
+    if (status == G_IO_STATUS_NORMAL || status == G_IO_STATUS_AGAIN) {
+        return true;
+    }
+
+    if (gerr) {
+        g_warning("qga: i/o error writing to input_data channel: %s",
+                gerr->message);
+        g_error_free(gerr);
+    }
+
+done:
+    g_io_channel_shutdown(ch, true, NULL);
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    g_free(p->data);
+
+    return false;
+}
+
+static gboolean guest_exec_output_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_read;
+    GIOStatus gstatus;
+
+    if (cond == G_IO_HUP || cond == G_IO_ERR) {
+        goto close;
+    }
+
+    if (p->size == p->length) {
+        gpointer t = NULL;
+        if (!p->truncated && p->size < GUEST_EXEC_MAX_OUTPUT) {
+            t = g_try_realloc(p->data, p->size + GUEST_EXEC_IO_SIZE);
+        }
+        if (t == NULL) {
+            /* ignore truncated output */
+            gchar buf[GUEST_EXEC_IO_SIZE];
+
+            p->truncated = true;
+            gstatus = g_io_channel_read_chars(ch, buf, sizeof(buf),
+                                              &bytes_read, NULL);
+            if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+                goto close;
+            }
+
+            return true;
+        }
+        p->size += GUEST_EXEC_IO_SIZE;
+        p->data = t;
+    }
+
+    /* Calling read API once.
+     * On next available data our callback will be called again */
+    gstatus = g_io_channel_read_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_read, NULL);
+    if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+        goto close;
+    }
+
+    p->length += bytes_read;
+
+    return true;
+
+close:
+    g_io_channel_shutdown(ch, true, NULL);
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    return false;
+}
+
+GuestExec *qmp_guest_exec(const char *path,
+                       bool has_arg, strList *arg,
+                       bool has_env, strList *env,
+                       bool has_input_data, const char *input_data,
+                       bool has_capture_output, bool capture_output,
+                       Error **err)
+{
+    GPid pid;
+    GuestExec *ge = NULL;
+    GuestExecInfo *gei;
+    char **argv, **envp;
+    strList arglist;
+    gboolean ret;
+    GError *gerr = NULL;
+    gint in_fd, out_fd, err_fd;
+    GIOChannel *in_ch, *out_ch, *err_ch;
+    GSpawnFlags flags;
+    bool has_output = (has_capture_output && capture_output);
+    uint8_t *input = NULL;
+    size_t ninput = 0;
+
+    arglist.value = (char *)path;
+    arglist.next = has_arg ? arg : NULL;
+
+    if (has_input_data) {
+        input = qbase64_decode(input_data, -1, &ninput, err);
+        if (!input) {
+            return NULL;
+        }
+    }
+
+    argv = guest_exec_get_args(&arglist, true);
+    envp = has_env ? guest_exec_get_args(env, false) : NULL;
+
+    flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD;
+#if GLIB_CHECK_VERSION(2, 33, 2)
+    flags |= G_SPAWN_SEARCH_PATH_FROM_ENVP;
+#endif
+    if (!has_output) {
+        flags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL;
+    }
+
+    ret = g_spawn_async_with_pipes(NULL, argv, envp, flags,
+            guest_exec_task_setup, NULL, &pid, has_input_data ? &in_fd : NULL,
+            has_output ? &out_fd : NULL, has_output ? &err_fd : NULL, &gerr);
+    if (!ret) {
+        error_setg(err, QERR_QGA_COMMAND_FAILED, gerr->message);
+        g_error_free(gerr);
+        goto done;
+    }
+
+    ge = g_new0(GuestExec, 1);
+    ge->pid = gpid_to_int64(pid);
+
+    gei = guest_exec_info_add(pid);
+    gei->has_output = has_output;
+    g_child_watch_add(pid, guest_exec_child_watch, gei);
+
+    if (has_input_data) {
+        gei->in.data = input;
+        gei->in.size = ninput;
+#ifdef G_OS_WIN32
+        in_ch = g_io_channel_win32_new_fd(in_fd);
+#else
+        in_ch = g_io_channel_unix_new(in_fd);
+#endif
+        g_io_channel_set_encoding(in_ch, NULL, NULL);
+        g_io_channel_set_buffered(in_ch, false);
+        g_io_channel_set_flags(in_ch, G_IO_FLAG_NONBLOCK, NULL);
+        g_io_channel_set_close_on_unref(in_ch, true);
+        g_io_add_watch(in_ch, G_IO_OUT, guest_exec_input_watch, &gei->in);
+    }
+
+    if (has_output) {
+#ifdef G_OS_WIN32
+        out_ch = g_io_channel_win32_new_fd(out_fd);
+        err_ch = g_io_channel_win32_new_fd(err_fd);
+#else
+        out_ch = g_io_channel_unix_new(out_fd);
+        err_ch = g_io_channel_unix_new(err_fd);
+#endif
+        g_io_channel_set_encoding(out_ch, NULL, NULL);
+        g_io_channel_set_encoding(err_ch, NULL, NULL);
+        g_io_channel_set_buffered(out_ch, false);
+        g_io_channel_set_buffered(err_ch, false);
+        g_io_channel_set_close_on_unref(out_ch, true);
+        g_io_channel_set_close_on_unref(err_ch, true);
+        g_io_add_watch(out_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->out);
+        g_io_add_watch(err_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->err);
+    }
+
+done:
+    g_free(argv);
+    g_free(envp);
+
+    return ge;
+}
+
+/* Convert GuestFileWhence (either a raw integer or an enum value) into
+ * the guest's SEEK_ constants.  */
+int ga_parse_whence(GuestFileWhence *whence, Error **errp)
+{
+    /* Exploit the fact that we picked values to match QGA_SEEK_*. */
+    if (whence->type == QTYPE_QSTRING) {
+        whence->type = QTYPE_QINT;
+        whence->u.value = whence->u.name;
+    }
+    switch (whence->u.value) {
+    case QGA_SEEK_SET:
+        return SEEK_SET;
+    case QGA_SEEK_CUR:
+        return SEEK_CUR;
+    case QGA_SEEK_END:
+        return SEEK_END;
+    }
+    error_setg(errp, "invalid whence code %"PRId64, whence->u.value);
+    return -1;
+}
diff --git a/qemu/qga/guest-agent-command-state.c b/qemu/qga/guest-agent-command-state.c
index 969da2328..20b9b2222 100644
--- a/qemu/qga/guest-agent-command-state.c
+++ b/qemu/qga/guest-agent-command-state.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qga/guest-agent-core.h"
 
@@ -27,7 +28,7 @@ void ga_command_state_add(GACommandState *cs,
                           void (*init)(void),
                           void (*cleanup)(void))
 {
-    GACommandGroup *cg = g_malloc0(sizeof(GACommandGroup));
+    GACommandGroup *cg = g_new0(GACommandGroup, 1);
     cg->init = init;
     cg->cleanup = cleanup;
     cs->groups = g_slist_append(cs->groups, cg);
@@ -67,7 +68,7 @@ void ga_command_state_cleanup_all(GACommandState *cs)
 
 GACommandState *ga_command_state_new(void)
 {
-    GACommandState *cs = g_malloc0(sizeof(GACommandState));
+    GACommandState *cs = g_new0(GACommandState, 1);
     cs->groups = NULL;
     return cs;
 }
diff --git a/qemu/qga/guest-agent-core.h b/qemu/qga/guest-agent-core.h
index e92c6abaf..0a4951604 100644
--- a/qemu/qga/guest-agent-core.h
+++ b/qemu/qga/guest-agent-core.h
@@ -12,6 +12,7 @@
  */
 #include "qapi/qmp/dispatch.h"
 #include "qemu-common.h"
+#include "qga-qmp-commands.h"
 
 #define QGA_READ_COUNT_DEFAULT 4096
 
@@ -37,6 +38,7 @@ void ga_set_frozen(GAState *s);
 void ga_unset_frozen(GAState *s);
 const char *ga_fsfreeze_hook(GAState *s);
 int64_t ga_get_fd_handle(GAState *s, Error **errp);
+int ga_parse_whence(GuestFileWhence *whence, Error **errp);
 
 #ifndef _WIN32
 void reopen_fd_to_null(int fd);
diff --git a/qemu/qga/installer/qemu-ga.wxs b/qemu/qga/installer/qemu-ga.wxs
index 2c43f1b5a..fa2260caf 100644
--- a/qemu/qga/installer/qemu-ga.wxs
+++ b/qemu/qga/installer/qemu-ga.wxs
@@ -41,8 +41,8 @@
 
   <Product
     Name="QEMU guest agent"
-    Id="*"
-    UpgradeCode="{EB6B8302-C06E-4bec-ADAC-932C68A3A98D}"
+    Id="{DF9974AD-E41A-4304-81AD-69AA8F299766}"
+    UpgradeCode="{EB6B8302-C06E-4BEC-ADAC-932C68A3A98D}"
     Manufacturer="$(env.QEMU_GA_MANUFACTURER)"
     Version="$(env.QEMU_GA_VERSION)"
     Language="1033">
@@ -58,29 +58,15 @@
       />
     <Media Id="1" Cabinet="qemu_ga.$(env.QEMU_GA_VERSION).cab" EmbedCab="yes" />
     <Property Id="WHSLogo">1</Property>
-    <Property Id="PREVIOUSVERSIONSINSTALLED" />
-    <Upgrade Id="{EB6B8302-C06E-4bec-ADAC-932C68A3A98D}">
-      <UpgradeVersion
-        Minimum="1.0.0.0" Maximum="$(env.QEMU_GA_VERSION)"
-        Property="PREVIOUSVERSIONSINSTALLED"
-        IncludeMinimum="yes" IncludeMaximum="no" />
-    </Upgrade>
+    <MajorUpgrade
+      DowngradeErrorMessage="Error: A newer version of QEMU guest agent is already installed."
+      />
 
     <Directory Id="TARGETDIR" Name="SourceDir">
       <Directory Id="$(var.GaProgramFilesFolder)" Name="QEMU Guest Agent">
         <Directory Id="qemu_ga_directory" Name="Qemu-ga">
-          <Component Id="qemu_ga" Guid="{908B7199-DE2A-4dc6-A8D0-27A5AE444FEA}">
-            <File Id="qemu_ga.exe" Name="qemu-ga.exe" Source="../../qemu-ga.exe" KeyPath="yes" DiskId="1"/>
-            <?ifdef var.InstallVss ?>
-            <File Id="qga_vss.dll" Name="qga-vss.dll" Source="../vss-win32/qga-vss.dll" KeyPath="no" DiskId="1"/>
-            <File Id="qga_vss.tlb" Name="qga-vss.tlb" Source="../vss-win32/qga-vss.tlb" KeyPath="no" DiskId="1"/>
-            <?endif?>
-            <File Id="iconv.dll" Name="iconv.dll" Source="$(var.Mingw_bin)/iconv.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libgcc_arch_lib" Name="$(var.ArchLib)" Source="$(var.Mingw_bin)/$(var.ArchLib)" KeyPath="no" DiskId="1"/>
-            <File Id="libglib_2.0_0.dll" Name="libglib-2.0-0.dll" Source="$(var.Mingw_bin)/libglib-2.0-0.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libintl_8.dll" Name="libintl-8.dll" Source="$(var.Mingw_bin)/libintl-8.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libssp_0.dll" Name="libssp-0.dll" Source="$(var.Mingw_bin)/libssp-0.dll" KeyPath="no" DiskId="1"/>
-            <File Id="libwinpthread_1.dll" Name="libwinpthread-1.dll" Source="$(var.Mingw_bin)/libwinpthread-1.dll" KeyPath="no" DiskId="1"/>
+          <Component Id="qemu_ga" Guid="{908B7199-DE2A-4DC6-A8D0-27A5AE444FEA}">
+            <File Id="qemu_ga.exe" Name="qemu-ga.exe" Source="$(env.BUILD_DIR)/qemu-ga.exe" KeyPath="yes" DiskId="1"/>
             <ServiceInstall
               Id="ServiceInstaller"
               Type="ownProcess"
@@ -97,8 +83,49 @@
             </ServiceInstall>
             <ServiceControl Id="StartService" Start="install" Stop="both" Remove="uninstall" Name="QEMU-GA" Wait="no" />
           </Component>
-
-          <Component Id="registry_entries" Guid="d075d109-51ca-11e3-9f8b-000c29858960">
+          <?ifdef var.InstallVss?>
+          <Component Id="qga_vss_dll" Guid="{CB19C453-FABB-4BB1-ABAB-6B74F687BFBB}">
+            <File Id="qga_vss.dll" Name="qga-vss.dll" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="qga_vss_tlb" Guid="{D8D584B1-59C2-4FB7-A91F-636FF7BFA66E}">
+            <File Id="qga_vss.tlb" Name="qga-vss.tlb" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.tlb" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "32"?>
+          <Component Id="gspawn-helper-console" Guid="{446185B3-87BE-43D2-96B8-0FEFD9E8696D}">
+            <File Id="gspawn-win32-helper-console.exe" Name="gspawn-win32-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{CD67A5A3-2DB1-4DA1-A67A-8D71E797B466}">
+            <File Id="gspawn-win32-helper.exe" Name="gspawn-win32-helper.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "64"?>
+          <Component Id="gspawn-helper-console" Guid="{9E615A9F-349A-4992-A5C2-C10BAD173660}">
+            <File Id="gspawn-win64-helper-console.exe" Name="gspawn-win64-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{D201AD22-1846-4E4F-B6E1-C7A908ED2457}">
+            <File Id="gspawn-win64-helper.exe" Name="gspawn-win64-helper.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <Component Id="iconv" Guid="{35EE3558-D34B-4F0A-B8BD-430FF0775246}">
+            <File Id="iconv.dll" Name="iconv.dll" Source="$(var.Mingw_bin)/iconv.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libgcc_arch_lib" Guid="{ADD4D07D-4515-4AB6-AF3E-C904961B4BB0}">
+            <File Id="libgcc_arch_lib" Name="$(var.ArchLib)" Source="$(var.Mingw_bin)/$(var.ArchLib)" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libglib" Guid="{D31BFD83-2773-4B65-B45A-E0D2ADA58679}">
+            <File Id="libglib_2.0_0.dll" Name="libglib-2.0-0.dll" Source="$(var.Mingw_bin)/libglib-2.0-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libintl" Guid="{A641BC2D-A907-4A94-9149-F30ED430878F}">
+            <File Id="libintl_8.dll" Name="libintl-8.dll" Source="$(var.Mingw_bin)/libintl-8.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libssp" Guid="{7880087B-02B4-4EF6-A5D3-D18F8E3D90E1}">
+            <File Id="libssp_0.dll" Name="libssp-0.dll" Source="$(var.Mingw_bin)/libssp-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libwinpthread" Guid="{6C117C78-0F47-4B07-8F34-6BEE11643829}">
+            <File Id="libwinpthread_1.dll" Name="libwinpthread-1.dll" Source="$(var.Mingw_bin)/libwinpthread-1.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="registry_entries" Guid="{D075D109-51CA-11E3-9F8B-000C29858960}">
             <RegistryKey Root="HKLM"
                          Key="Software\$(env.QEMU_GA_MANUFACTURER)\$(env.QEMU_GA_DISTRO)\Tools\QemuGA">
               <RegistryValue Type="string" Name="ProductID" Value="fb0a0d66-c7fb-4e2e-a16b-c4a3bfe8d13b" />
@@ -110,10 +137,11 @@
     </Directory>
 
     <Property Id="cmd" Value="cmd.exe"/>
+    <Property Id="REINSTALLMODE" Value="amus"/>
 
-    <?ifdef var.InstallVss ?>
+    <?ifdef var.InstallVss?>
     <CustomAction Id="RegisterCom"
-             ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install'
+              ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install'
               Execute="deferred"
               Property="cmd"
               Impersonate="no"
@@ -126,19 +154,31 @@
               Property="cmd"
               Impersonate="no"
               Return="check"
-              ></CustomAction>
+              >
+    </CustomAction>
     <?endif?>
 
     <Feature Id="QEMUFeature" Title="QEMU Guest Agent" Level="1">
       <ComponentRef Id="qemu_ga" />
+      <?ifdef var.InstallVss?>
+      <ComponentRef Id="qga_vss_dll" />
+      <ComponentRef Id="qga_vss_tlb" />
+      <?endif?>
+      <ComponentRef Id="gspawn-helper-console" />
+      <ComponentRef Id="gspawn-helper" />
+      <ComponentRef Id="iconv" />
+      <ComponentRef Id="libgcc_arch_lib" />
+      <ComponentRef Id="libglib" />
+      <ComponentRef Id="libintl" />
+      <ComponentRef Id="libssp" />
+      <ComponentRef Id="libwinpthread" />
       <ComponentRef Id="registry_entries" />
     </Feature>
 
     <InstallExecuteSequence>
-      <RemoveExistingProducts Before="InstallInitialize" />
-      <?ifdef var.InstallVss ?>
-      <Custom Action="RegisterCom" After="InstallServices">NOT Installed</Custom>
+      <?ifdef var.InstallVss?>
       <Custom Action="UnRegisterCom" After="StopServices">Installed</Custom>
+      <Custom Action="RegisterCom" After="InstallServices">NOT REMOVE</Custom>
       <?endif?>
     </InstallExecuteSequence>
   </Product>
diff --git a/qemu/qga/main.c b/qemu/qga/main.c
index 791982ef0..c55278210 100644
--- a/qemu/qga/main.c
+++ b/qemu/qga/main.c
@@ -10,16 +10,13 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <getopt.h>
 #include <glib/gstdio.h>
 #ifndef _WIN32
 #include <syslog.h>
 #include <sys/wait.h>
-#include <sys/stat.h>
 #endif
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
@@ -27,11 +24,11 @@
 #include "qapi/qmp/qjson.h"
 #include "qga/guest-agent-core.h"
 #include "qemu/module.h"
-#include "signal.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/dispatch.h"
 #include "qga/channel.h"
 #include "qemu/bswap.h"
+#include "qemu/help_option.h"
 #ifdef _WIN32
 #include "qga/service-win32.h"
 #include "qga/vss-win32.h"
@@ -56,6 +53,7 @@
 #define QGA_FSFREEZE_HOOK_DEFAULT CONFIG_QEMU_CONFDIR "/fsfreeze-hook"
 #endif
 #define QGA_SENTINEL_BYTE 0xFF
+#define QGA_CONF_DEFAULT CONFIG_QEMU_CONFDIR G_DIR_SEPARATOR_S "qemu-ga.conf"
 
 static struct {
     const char *state_dir;
@@ -82,7 +80,7 @@ struct GAState {
     bool delimit_response;
     bool frozen;
     GList *blacklist;
-    const char *state_filepath_isfrozen;
+    char *state_filepath_isfrozen;
     struct {
         const char *log_filepath;
         const char *pid_filepath;
@@ -90,7 +88,7 @@ struct GAState {
 #ifdef CONFIG_FSFREEZE
     const char *fsfreeze_hook;
 #endif
-    const gchar *pstate_filepath;
+    gchar *pstate_filepath;
     GAPersistentState pstate;
 };
 
@@ -160,6 +158,12 @@ static gboolean register_signal_handlers(void)
         g_error("error configuring signal handler: %s", strerror(errno));
     }
 
+    sigact.sa_handler = SIG_IGN;
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        g_error("error configuring SIGPIPE signal handler: %s",
+                strerror(errno));
+    }
+
     return true;
 }
 
@@ -215,6 +219,8 @@ static void usage(const char *cmd)
 #endif
 "  -b, --blacklist   comma-separated list of RPCs to disable (no spaces, \"?\"\n"
 "                    to list available RPCs)\n"
+"  -D, --dump-conf   dump a qemu-ga config file based on current config\n"
+"                    options / command-line parameters to stdout\n"
 "  -h, --help        display this help and exit\n"
 "\n"
 "Report bugs to <mdroth@linux.vnet.ibm.com>\n"
@@ -561,10 +567,9 @@ static void process_command(GAState *s, QDict *req)
 }
 
 /* handle requests/control events coming in over the channel */
-static void process_event(JSONMessageParser *parser, QList *tokens)
+static void process_event(JSONMessageParser *parser, GQueue *tokens)
 {
     GAState *s = container_of(parser, GAState, parser);
-    QObject *obj;
     QDict *qdict;
     Error *err = NULL;
     int ret;
@@ -572,9 +577,9 @@ static void process_event(JSONMessageParser *parser, QList *tokens)
     g_assert(s && parser);
 
     g_debug("process_event: called");
-    obj = json_parser_parse_err(tokens, NULL, &err);
-    if (err || !obj || qobject_type(obj) != QTYPE_QDICT) {
-        qobject_decref(obj);
+    qdict = qobject_to_qdict(json_parser_parse_err(tokens, NULL, &err));
+    if (err || !qdict) {
+        QDECREF(qdict);
         qdict = qdict_new();
         if (!err) {
             g_warning("failed to parse event: unknown error");
@@ -584,12 +589,8 @@ static void process_event(JSONMessageParser *parser, QList *tokens)
         }
         qdict_put_obj(qdict, "error", qmp_build_error_object(err));
         error_free(err);
-    } else {
-        qdict = qobject_to_qdict(obj);
     }
 
-    g_assert(qdict);
-
     /* handle host->guest commands */
     if (qdict_haskey(qdict, "execute")) {
         process_command(s, qdict);
@@ -617,13 +618,7 @@ static gboolean channel_event_cb(GIOCondition condition, gpointer data)
     GAState *s = data;
     gchar buf[QGA_READ_COUNT_DEFAULT+1];
     gsize count;
-    GError *err = NULL;
     GIOStatus status = ga_channel_read(s->channel, buf, QGA_READ_COUNT_DEFAULT, &count);
-    if (err != NULL) {
-        g_warning("error reading channel: %s", err->message);
-        g_error_free(err);
-        return false;
-    }
     switch (status) {
     case G_IO_STATUS_ERROR:
         g_warning("error reading channel");
@@ -658,23 +653,6 @@ static gboolean channel_init(GAState *s, const gchar *method, const gchar *path)
 {
     GAChannelMethod channel_method;
 
-    if (method == NULL) {
-        method = "virtio-serial";
-    }
-
-    if (path == NULL) {
-        if (strcmp(method, "virtio-serial") == 0 ) {
-            /* try the default path for the virtio-serial port */
-            path = QGA_VIRTIO_PATH_DEFAULT;
-        } else if (strcmp(method, "isa-serial") == 0){
-            /* try the default path for the serial port - COM1 */
-            path = QGA_SERIAL_PATH_DEFAULT;
-        } else {
-            g_critical("must specify a path for this channel");
-            return false;
-        }
-    }
-
     if (strcmp(method, "virtio-serial") == 0) {
         s->virtio = true; /* virtio requires special handling in some cases */
         channel_method = GA_CHANNEL_VIRTIO_SERIAL;
@@ -921,22 +899,165 @@ static void ga_print_cmd(QmpCommand *cmd, void *opaque)
     printf("%s\n", qmp_command_name(cmd));
 }
 
-int main(int argc, char **argv)
+static GList *split_list(const gchar *str, const gchar *delim)
 {
-    const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
-    const char *method = NULL, *path = NULL;
-    const char *log_filepath = NULL;
-    const char *pid_filepath;
+    GList *list = NULL;
+    int i;
+    gchar **strv;
+
+    strv = g_strsplit(str, delim, -1);
+    for (i = 0; strv[i]; i++) {
+        list = g_list_prepend(list, strv[i]);
+    }
+    g_free(strv);
+
+    return list;
+}
+
+typedef struct GAConfig {
+    char *channel_path;
+    char *method;
+    char *log_filepath;
+    char *pid_filepath;
 #ifdef CONFIG_FSFREEZE
-    const char *fsfreeze_hook = NULL;
+    char *fsfreeze_hook;
 #endif
-    const char *state_dir;
+    char *state_dir;
 #ifdef _WIN32
-    const char *service = NULL;
+    const char *service;
+#endif
+    gchar *bliststr; /* blacklist may point to this string */
+    GList *blacklist;
+    int daemonize;
+    GLogLevelFlags log_level;
+    int dumpconf;
+} GAConfig;
+
+static void config_load(GAConfig *config)
+{
+    GError *gerr = NULL;
+    GKeyFile *keyfile;
+    const char *conf = g_getenv("QGA_CONF") ?: QGA_CONF_DEFAULT;
+
+    /* read system config */
+    keyfile = g_key_file_new();
+    if (!g_key_file_load_from_file(keyfile, conf, 0, &gerr)) {
+        goto end;
+    }
+    if (g_key_file_has_key(keyfile, "general", "daemon", NULL)) {
+        config->daemonize =
+            g_key_file_get_boolean(keyfile, "general", "daemon", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "method", NULL)) {
+        config->method =
+            g_key_file_get_string(keyfile, "general", "method", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "path", NULL)) {
+        config->channel_path =
+            g_key_file_get_string(keyfile, "general", "path", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "logfile", NULL)) {
+        config->log_filepath =
+            g_key_file_get_string(keyfile, "general", "logfile", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "pidfile", NULL)) {
+        config->pid_filepath =
+            g_key_file_get_string(keyfile, "general", "pidfile", &gerr);
+    }
+#ifdef CONFIG_FSFREEZE
+    if (g_key_file_has_key(keyfile, "general", "fsfreeze-hook", NULL)) {
+        config->fsfreeze_hook =
+            g_key_file_get_string(keyfile,
+                                  "general", "fsfreeze-hook", &gerr);
+    }
+#endif
+    if (g_key_file_has_key(keyfile, "general", "statedir", NULL)) {
+        config->state_dir =
+            g_key_file_get_string(keyfile, "general", "statedir", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "verbose", NULL) &&
+        g_key_file_get_boolean(keyfile, "general", "verbose", &gerr)) {
+        /* enable all log levels */
+        config->log_level = G_LOG_LEVEL_MASK;
+    }
+    if (g_key_file_has_key(keyfile, "general", "blacklist", NULL)) {
+        config->bliststr =
+            g_key_file_get_string(keyfile, "general", "blacklist", &gerr);
+        config->blacklist = g_list_concat(config->blacklist,
+                                          split_list(config->bliststr, ","));
+    }
+
+end:
+    g_key_file_free(keyfile);
+    if (gerr &&
+        !(gerr->domain == G_FILE_ERROR && gerr->code == G_FILE_ERROR_NOENT)) {
+        g_critical("error loading configuration from path: %s, %s",
+                   QGA_CONF_DEFAULT, gerr->message);
+        exit(EXIT_FAILURE);
+    }
+    g_clear_error(&gerr);
+}
+
+static gchar *list_join(GList *list, const gchar separator)
+{
+    GString *str = g_string_new("");
+
+    while (list) {
+        str = g_string_append(str, (gchar *)list->data);
+        list = g_list_next(list);
+        if (list) {
+            str = g_string_append_c(str, separator);
+        }
+    }
+
+    return g_string_free(str, FALSE);
+}
+
+static void config_dump(GAConfig *config)
+{
+    GError *error = NULL;
+    GKeyFile *keyfile;
+    gchar *tmp;
+
+    keyfile = g_key_file_new();
+    g_assert(keyfile);
+
+    g_key_file_set_boolean(keyfile, "general", "daemon", config->daemonize);
+    g_key_file_set_string(keyfile, "general", "method", config->method);
+    g_key_file_set_string(keyfile, "general", "path", config->channel_path);
+    if (config->log_filepath) {
+        g_key_file_set_string(keyfile, "general", "logfile",
+                              config->log_filepath);
+    }
+    g_key_file_set_string(keyfile, "general", "pidfile", config->pid_filepath);
+#ifdef CONFIG_FSFREEZE
+    if (config->fsfreeze_hook) {
+        g_key_file_set_string(keyfile, "general", "fsfreeze-hook",
+                              config->fsfreeze_hook);
+    }
 #endif
+    g_key_file_set_string(keyfile, "general", "statedir", config->state_dir);
+    g_key_file_set_boolean(keyfile, "general", "verbose",
+                           config->log_level == G_LOG_LEVEL_MASK);
+    tmp = list_join(config->blacklist, ',');
+    g_key_file_set_string(keyfile, "general", "blacklist", tmp);
+    g_free(tmp);
+
+    tmp = g_key_file_to_data(keyfile, NULL, &error);
+    printf("%s", tmp);
+
+    g_free(tmp);
+    g_key_file_free(keyfile);
+}
+
+static void config_parse(GAConfig *config, int argc, char **argv)
+{
+    const char *sopt = "hVvdm:p:l:f:F::b:s:t:D";
+    int opt_ind = 0, ch;
     const struct option lopt[] = {
         { "help", 0, NULL, 'h' },
         { "version", 0, NULL, 'V' },
+        { "dump-conf", 0, NULL, 'D' },
         { "logfile", 1, NULL, 'l' },
         { "pidfile", 1, NULL, 'f' },
 #ifdef CONFIG_FSFREEZE
@@ -953,141 +1074,113 @@ int main(int argc, char **argv)
         { "statedir", 1, NULL, 't' },
         { NULL, 0, NULL, 0 }
     };
-    int opt_ind = 0, ch, daemonize = 0, i, j, len;
-    GLogLevelFlags log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
-    GList *blacklist = NULL;
-    GAState *s;
-
-    module_call_init(MODULE_INIT_QAPI);
-
-    init_dfl_pathnames();
-    pid_filepath = dfl_pathnames.pidfile;
-    state_dir = dfl_pathnames.state_dir;
 
     while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
         switch (ch) {
         case 'm':
-            method = optarg;
+            g_free(config->method);
+            config->method = g_strdup(optarg);
             break;
         case 'p':
-            path = optarg;
+            g_free(config->channel_path);
+            config->channel_path = g_strdup(optarg);
             break;
         case 'l':
-            log_filepath = optarg;
+            g_free(config->log_filepath);
+            config->log_filepath = g_strdup(optarg);
             break;
         case 'f':
-            pid_filepath = optarg;
+            g_free(config->pid_filepath);
+            config->pid_filepath = g_strdup(optarg);
             break;
 #ifdef CONFIG_FSFREEZE
         case 'F':
-            fsfreeze_hook = optarg ? optarg : QGA_FSFREEZE_HOOK_DEFAULT;
+            g_free(config->fsfreeze_hook);
+            config->fsfreeze_hook = g_strdup(optarg ?: QGA_FSFREEZE_HOOK_DEFAULT);
             break;
 #endif
         case 't':
-             state_dir = optarg;
-             break;
+            g_free(config->state_dir);
+            config->state_dir = g_strdup(optarg);
+            break;
         case 'v':
             /* enable all log levels */
-            log_level = G_LOG_LEVEL_MASK;
+            config->log_level = G_LOG_LEVEL_MASK;
             break;
         case 'V':
             printf("QEMU Guest Agent %s\n", QEMU_VERSION);
-            return 0;
+            exit(EXIT_SUCCESS);
         case 'd':
-            daemonize = 1;
+            config->daemonize = 1;
+            break;
+        case 'D':
+            config->dumpconf = 1;
             break;
         case 'b': {
             if (is_help_option(optarg)) {
                 qmp_for_each_command(ga_print_cmd, NULL);
-                return 0;
-            }
-            for (j = 0, i = 0, len = strlen(optarg); i < len; i++) {
-                if (optarg[i] == ',') {
-                    optarg[i] = 0;
-                    blacklist = g_list_append(blacklist, &optarg[j]);
-                    j = i + 1;
-                }
-            }
-            if (j < i) {
-                blacklist = g_list_append(blacklist, &optarg[j]);
+                exit(EXIT_SUCCESS);
             }
+            config->blacklist = g_list_concat(config->blacklist,
+                                             split_list(optarg, ","));
             break;
         }
 #ifdef _WIN32
         case 's':
-            service = optarg;
-            if (strcmp(service, "install") == 0) {
-                const char *fixed_state_dir;
-
-                /* If the user passed the "-t" option, we save that state dir
-                 * in the service. Otherwise we let the service fetch the state
-                 * dir from the environment when it starts.
-                 */
-                fixed_state_dir = (state_dir == dfl_pathnames.state_dir) ?
-                                  NULL :
-                                  state_dir;
+            config->service = optarg;
+            if (strcmp(config->service, "install") == 0) {
                 if (ga_install_vss_provider()) {
-                    return EXIT_FAILURE;
+                    exit(EXIT_FAILURE);
                 }
-                if (ga_install_service(path, log_filepath, fixed_state_dir)) {
-                    return EXIT_FAILURE;
+                if (ga_install_service(config->channel_path,
+                                       config->log_filepath, config->state_dir)) {
+                    exit(EXIT_FAILURE);
                 }
-                return 0;
-            } else if (strcmp(service, "uninstall") == 0) {
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "uninstall") == 0) {
                 ga_uninstall_vss_provider();
-                return ga_uninstall_service();
-            } else if (strcmp(service, "vss-install") == 0) {
+                exit(ga_uninstall_service());
+            } else if (strcmp(config->service, "vss-install") == 0) {
                 if (ga_install_vss_provider()) {
-                    return EXIT_FAILURE;
+                    exit(EXIT_FAILURE);
                 }
-                return EXIT_SUCCESS;
-            } else if (strcmp(service, "vss-uninstall") == 0) {
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "vss-uninstall") == 0) {
                 ga_uninstall_vss_provider();
-                return EXIT_SUCCESS;
+                exit(EXIT_SUCCESS);
             } else {
                 printf("Unknown service command.\n");
-                return EXIT_FAILURE;
+                exit(EXIT_FAILURE);
             }
             break;
 #endif
         case 'h':
             usage(argv[0]);
-            return 0;
+            exit(EXIT_SUCCESS);
         case '?':
             g_print("Unknown option, try '%s --help' for more information.\n",
                     argv[0]);
-            return EXIT_FAILURE;
+            exit(EXIT_FAILURE);
         }
     }
+}
 
-#ifdef _WIN32
-    /* On win32 the state directory is application specific (be it the default
-     * or a user override). We got past the command line parsing; let's create
-     * the directory (with any intermediate directories). If we run into an
-     * error later on, we won't try to clean up the directory, it is considered
-     * persistent.
-     */
-    if (g_mkdir_with_parents(state_dir, S_IRWXU) == -1) {
-        g_critical("unable to create (an ancestor of) the state directory"
-                   " '%s': %s", state_dir, strerror(errno));
-        return EXIT_FAILURE;
-    }
-#endif
-
-    s = g_malloc0(sizeof(GAState));
-    s->log_level = log_level;
-    s->log_file = stderr;
+static void config_free(GAConfig *config)
+{
+    g_free(config->method);
+    g_free(config->log_filepath);
+    g_free(config->pid_filepath);
+    g_free(config->state_dir);
+    g_free(config->channel_path);
+    g_free(config->bliststr);
 #ifdef CONFIG_FSFREEZE
-    s->fsfreeze_hook = fsfreeze_hook;
+    g_free(config->fsfreeze_hook);
 #endif
-    g_log_set_default_handler(ga_log, s);
-    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
-    ga_enable_logging(s);
-    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
-                                                 state_dir);
-    s->pstate_filepath = g_strdup_printf("%s/qga.state", state_dir);
-    s->frozen = false;
+    g_free(config);
+}
 
+static bool check_is_frozen(GAState *s)
+{
 #ifndef _WIN32
     /* check if a previous instance of qemu-ga exited with filesystems' state
      * marked as frozen. this could be a stale value (a non-qemu-ga process
@@ -1113,32 +1206,56 @@ int main(int argc, char **argv)
                   " guest-fsfreeze-thaw is issued, or filesystems are"
                   " manually unfrozen and the file %s is removed",
                   s->state_filepath_isfrozen);
-        s->frozen = true;
+        return true;
+    }
+#endif
+    return false;
+}
+
+static int run_agent(GAState *s, GAConfig *config)
+{
+    ga_state = s;
+
+    g_log_set_default_handler(ga_log, s);
+    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
+    ga_enable_logging(s);
+
+#ifdef _WIN32
+    /* On win32 the state directory is application specific (be it the default
+     * or a user override). We got past the command line parsing; let's create
+     * the directory (with any intermediate directories). If we run into an
+     * error later on, we won't try to clean up the directory, it is considered
+     * persistent.
+     */
+    if (g_mkdir_with_parents(config->state_dir, S_IRWXU) == -1) {
+        g_critical("unable to create (an ancestor of) the state directory"
+                   " '%s': %s", config->state_dir, strerror(errno));
+        return EXIT_FAILURE;
     }
 #endif
 
     if (ga_is_frozen(s)) {
-        if (daemonize) {
+        if (config->daemonize) {
             /* delay opening/locking of pidfile till filesystems are unfrozen */
-            s->deferred_options.pid_filepath = pid_filepath;
+            s->deferred_options.pid_filepath = config->pid_filepath;
             become_daemon(NULL);
         }
-        if (log_filepath) {
+        if (config->log_filepath) {
             /* delay opening the log file till filesystems are unfrozen */
-            s->deferred_options.log_filepath = log_filepath;
+            s->deferred_options.log_filepath = config->log_filepath;
         }
         ga_disable_logging(s);
         qmp_for_each_command(ga_disable_non_whitelisted, NULL);
     } else {
-        if (daemonize) {
-            become_daemon(pid_filepath);
+        if (config->daemonize) {
+            become_daemon(config->pid_filepath);
         }
-        if (log_filepath) {
-            FILE *log_file = ga_open_logfile(log_filepath);
+        if (config->log_filepath) {
+            FILE *log_file = ga_open_logfile(config->log_filepath);
             if (!log_file) {
                 g_critical("unable to open specified log file: %s",
                            strerror(errno));
-                goto out_bad;
+                return EXIT_FAILURE;
             }
             s->log_file = log_file;
         }
@@ -1149,17 +1266,18 @@ int main(int argc, char **argv)
                                s->pstate_filepath,
                                ga_is_frozen(s))) {
         g_critical("failed to load persistent state");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 
-    blacklist = ga_command_blacklist_init(blacklist);
-    if (blacklist) {
-        s->blacklist = blacklist;
+    config->blacklist = ga_command_blacklist_init(config->blacklist);
+    if (config->blacklist) {
+        GList *l = config->blacklist;
+        s->blacklist = config->blacklist;
         do {
-            g_debug("disabling command: %s", (char *)blacklist->data);
-            qmp_disable_command(blacklist->data);
-            blacklist = g_list_next(blacklist);
-        } while (blacklist);
+            g_debug("disabling command: %s", (char *)l->data);
+            qmp_disable_command(l->data);
+            l = g_list_next(l);
+        } while (l);
     }
     s->command_state = ga_command_state_new();
     ga_command_state_init(s, s->command_state);
@@ -1169,19 +1287,19 @@ int main(int argc, char **argv)
 #ifndef _WIN32
     if (!register_signal_handlers()) {
         g_critical("failed to register signal handlers");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #endif
 
     s->main_loop = g_main_loop_new(NULL, false);
-    if (!channel_init(ga_state, method, path)) {
+    if (!channel_init(ga_state, config->method, config->channel_path)) {
         g_critical("failed to initialize guest agent channel");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #ifndef _WIN32
     g_main_loop_run(ga_state->main_loop);
 #else
-    if (daemonize) {
+    if (config->daemonize) {
         SERVICE_TABLE_ENTRY service_table[] = {
             { (char *)QGA_SERVICE_NAME, service_main }, { NULL, NULL } };
         StartServiceCtrlDispatcher(service_table);
@@ -1190,17 +1308,87 @@ int main(int argc, char **argv)
     }
 #endif
 
-    ga_command_state_cleanup_all(ga_state->command_state);
-    ga_channel_free(ga_state->channel);
+    return EXIT_SUCCESS;
+}
+
+static void free_blacklist_entry(gpointer entry, gpointer unused)
+{
+    g_free(entry);
+}
+
+int main(int argc, char **argv)
+{
+    int ret = EXIT_SUCCESS;
+    GAState *s = g_new0(GAState, 1);
+    GAConfig *config = g_new0(GAConfig, 1);
+
+    config->log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
 
-    if (daemonize) {
-        unlink(pid_filepath);
+    module_call_init(MODULE_INIT_QAPI);
+
+    init_dfl_pathnames();
+    config_load(config);
+    config_parse(config, argc, argv);
+
+    if (config->pid_filepath == NULL) {
+        config->pid_filepath = g_strdup(dfl_pathnames.pidfile);
     }
-    return 0;
 
-out_bad:
-    if (daemonize) {
-        unlink(pid_filepath);
+    if (config->state_dir == NULL) {
+        config->state_dir = g_strdup(dfl_pathnames.state_dir);
+    }
+
+    if (config->method == NULL) {
+        config->method = g_strdup("virtio-serial");
+    }
+
+    if (config->channel_path == NULL) {
+        if (strcmp(config->method, "virtio-serial") == 0) {
+            /* try the default path for the virtio-serial port */
+            config->channel_path = g_strdup(QGA_VIRTIO_PATH_DEFAULT);
+        } else if (strcmp(config->method, "isa-serial") == 0) {
+            /* try the default path for the serial port - COM1 */
+            config->channel_path = g_strdup(QGA_SERIAL_PATH_DEFAULT);
+        } else {
+            g_critical("must specify a path for this channel");
+            ret = EXIT_FAILURE;
+            goto end;
+        }
+    }
+
+    s->log_level = config->log_level;
+    s->log_file = stderr;
+#ifdef CONFIG_FSFREEZE
+    s->fsfreeze_hook = config->fsfreeze_hook;
+#endif
+    s->pstate_filepath = g_strdup_printf("%s/qga.state", config->state_dir);
+    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
+                                                 config->state_dir);
+    s->frozen = check_is_frozen(s);
+
+    if (config->dumpconf) {
+        config_dump(config);
+        goto end;
+    }
+
+    ret = run_agent(s, config);
+
+end:
+    if (s->command_state) {
+        ga_command_state_cleanup_all(s->command_state);
+    }
+    if (s->channel) {
+        ga_channel_free(s->channel);
+    }
+    g_list_foreach(config->blacklist, free_blacklist_entry, NULL);
+    g_free(s->pstate_filepath);
+    g_free(s->state_filepath_isfrozen);
+
+    if (config->daemonize) {
+        unlink(config->pid_filepath);
     }
-    return EXIT_FAILURE;
+
+    config_free(config);
+
+    return ret;
 }
diff --git a/qemu/qga/qapi-schema.json b/qemu/qga/qapi-schema.json
index 18e3cc37d..c21f3084d 100644
--- a/qemu/qga/qapi-schema.json
+++ b/qemu/qga/qapi-schema.json
@@ -12,6 +12,7 @@
 ##
 
 ##
+# @guest-sync-delimited:
 #
 # Echo back a unique integer value, and prepend to response a
 # leading sentinel byte (0xFF) the client can check scan for.
@@ -41,7 +42,7 @@
 # Returns: The unique integer id passed in by the client
 #
 # Since: 1.1
-# ##
+##
 { 'command': 'guest-sync-delimited',
   'data':    { 'id': 'int' },
   'returns': 'int' }
@@ -313,24 +314,53 @@
   'data': { 'position': 'int', 'eof': 'bool' } }
 
 ##
+# @QGASeek:
+#
+# Symbolic names for use in @guest-file-seek
+#
+# @set: Set to the specified offset (same effect as 'whence':0)
+# @cur: Add offset to the current location (same effect as 'whence':1)
+# @end: Add offset to the end of the file (same effect as 'whence':2)
+#
+# Since: 2.6
+##
+{ 'enum': 'QGASeek', 'data': [ 'set', 'cur', 'end' ] }
+
+##
+# @GuestFileWhence:
+#
+# Controls the meaning of offset to @guest-file-seek.
+#
+# @value: Integral value (0 for set, 1 for cur, 2 for end), available
+#         for historical reasons, and might differ from the host's or
+#         guest's SEEK_* values (since: 0.15)
+# @name: Symbolic name, and preferred interface
+#
+# Since: 2.6
+##
+{ 'alternate': 'GuestFileWhence',
+  'data': { 'value': 'int', 'name': 'QGASeek' } }
+
+##
 # @guest-file-seek:
 #
 # Seek to a position in the file, as with fseek(), and return the
 # current file position afterward. Also encapsulates ftell()'s
-# functionality, just Set offset=0, whence=SEEK_CUR.
+# functionality, with offset=0 and whence=1.
 #
 # @handle: filehandle returned by guest-file-open
 #
 # @offset: bytes to skip over in the file stream
 #
-# @whence: SEEK_SET, SEEK_CUR, or SEEK_END, as with fseek()
+# @whence: Symbolic or numeric code for interpreting offset
 #
 # Returns: @GuestFileSeek on success.
 #
 # Since: 0.15.0
 ##
 { 'command': 'guest-file-seek',
-  'data':    { 'handle': 'int', 'offset': 'int', 'whence': 'int' },
+  'data':    { 'handle': 'int', 'offset': 'int',
+               'whence': 'GuestFileWhence' },
   'returns': 'GuestFileSeek' }
 
 ##
@@ -793,7 +823,7 @@
 # scheme. Refer to the documentation of the guest operating system
 # in question to determine what is supported.
 #
-# Note all guest operating systems will support use of the
+# Not all guest operating systems will support use of the
 # @crypted flag, as they may require the clear-text password
 #
 # The @password parameter must always be base64 encoded before
@@ -929,3 +959,70 @@
 ##
 { 'command': 'guest-get-memory-block-info',
   'returns': 'GuestMemoryBlockInfo' }
+
+# @GuestExecStatus:
+#
+# @exited: true if process has already terminated.
+# @exitcode: #optional process exit code if it was normally terminated.
+# @signal: #optional signal number (linux) or unhandled exception code
+#       (windows) if the process was abnormally terminated.
+# @out-data: #optional base64-encoded stdout of the process
+# @err-data: #optional base64-encoded stderr of the process
+#       Note: @out-data and @err-data are present only
+#       if 'capture-output' was specified for 'guest-exec'
+# @out-truncated: #optional true if stdout was not fully captured
+#       due to size limitation.
+# @err-truncated: #optional true if stderr was not fully captured
+#       due to size limitation.
+#
+# Since: 2.5
+##
+{ 'struct': 'GuestExecStatus',
+  'data': { 'exited': 'bool', '*exitcode': 'int', '*signal': 'int',
+            '*out-data': 'str', '*err-data': 'str',
+            '*out-truncated': 'bool', '*err-truncated': 'bool' }}
+##
+# @guest-exec-status
+#
+# Check status of process associated with PID retrieved via guest-exec.
+# Reap the process and associated metadata if it has exited.
+#
+# @pid: pid returned from guest-exec
+#
+# Returns: GuestExecStatus on success.
+#
+# Since 2.5
+##
+{ 'command': 'guest-exec-status',
+  'data':    { 'pid': 'int' },
+  'returns': 'GuestExecStatus' }
+
+##
+# @GuestExec:
+# @pid: pid of child process in guest OS
+#
+#Since: 2.5
+##
+{ 'struct': 'GuestExec',
+  'data': { 'pid': 'int'} }
+
+##
+# @guest-exec:
+#
+# Execute a command in the guest
+#
+# @path: path or executable name to execute
+# @arg: #optional argument list to pass to executable
+# @env: #optional environment variables to pass to executable
+# @input-data: #optional data to be passed to process stdin (base64 encoded)
+# @capture-output: #optional bool flag to enable capture of
+#                  stdout/stderr of running process. defaults to false.
+#
+# Returns: PID on success.
+#
+# Since: 2.5
+##
+{ 'command': 'guest-exec',
+  'data':    { 'path': 'str', '*arg': ['str'], '*env': ['str'],
+               '*input-data': 'str', '*capture-output': 'bool' },
+  'returns': 'GuestExec' }
diff --git a/qemu/qga/service-win32.c b/qemu/qga/service-win32.c
index aef41f04f..72437587b 100644
--- a/qemu/qga/service-win32.c
+++ b/qemu/qga/service-win32.c
@@ -10,8 +10,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <windows.h>
 #include "qga/service-win32.h"
diff --git a/qemu/qga/vss-win32.c b/qemu/qga/vss-win32.c
index 0e4095736..9a0e46356 100644
--- a/qemu/qga/vss-win32.c
+++ b/qemu/qga/vss-win32.c
@@ -10,7 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <windows.h>
 #include "qga/guest-agent-core.h"
 #include "qga/vss-win32.h"
@@ -150,11 +150,11 @@ void qga_vss_fsfreeze(int *nr_volume, Error **errp, bool freeze)
     const char *func_name = freeze ? "requester_freeze" : "requester_thaw";
     QGAVSSRequesterFunc func;
     ErrorSet errset = {
-        .error_set = (ErrorSetFunc)error_set_win32,
-        .errp = (void **)errp,
-        .err_class = ERROR_CLASS_GENERIC_ERROR
+        .error_setg_win32_wrapper = error_setg_win32_internal,
+        .errp = errp,
     };
 
+    g_assert(errp);             /* requester.cpp requires it */
     func = (QGAVSSRequesterFunc)GetProcAddress(provider_lib, func_name);
     if (!func) {
         error_setg_win32(errp, GetLastError(), "failed to load %s from %s",
diff --git a/qemu/qga/vss-win32.h b/qemu/qga/vss-win32.h
index 298927dfa..4d1d15081 100644
--- a/qemu/qga/vss-win32.h
+++ b/qemu/qga/vss-win32.h
@@ -13,7 +13,6 @@
 #ifndef VSS_WIN32_H
 #define VSS_WIN32_H
 
-#include "qapi/error.h"
 
 bool vss_init(bool init_requester);
 void vss_deinit(bool deinit_requester);
diff --git a/qemu/qga/vss-win32/install.cpp b/qemu/qga/vss-win32/install.cpp
index b0e4426c7..cd9cdb4a2 100644
--- a/qemu/qga/vss-win32/install.cpp
+++ b/qemu/qga/vss-win32/install.cpp
@@ -10,8 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "vss-common.h"
 #include "inc/win2003/vscoordint.h"
diff --git a/qemu/qga/vss-win32/provider.cpp b/qemu/qga/vss-win32/provider.cpp
index d5129f8f6..d977393e3 100644
--- a/qemu/qga/vss-win32/provider.cpp
+++ b/qemu/qga/vss-win32/provider.cpp
@@ -10,7 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "vss-common.h"
 #include "inc/win2003/vscoordint.h"
 #include "inc/win2003/vsprov.h"
diff --git a/qemu/qga/vss-win32/requester.cpp b/qemu/qga/vss-win32/requester.cpp
index 922e74ddf..889052ded 100644
--- a/qemu/qga/vss-win32/requester.cpp
+++ b/qemu/qga/vss-win32/requester.cpp
@@ -10,10 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "vss-common.h"
 #include "requester.h"
-#include "assert.h"
 #include "inc/win2003/vswriter.h"
 #include "inc/win2003/vsbackup.h"
 
@@ -23,10 +22,12 @@
 /* Call QueryStatus every 10 ms while waiting for frozen event */
 #define VSS_TIMEOUT_EVENT_MSEC 10
 
-#define err_set(e, err, fmt, ...) \
-    ((e)->error_set((e)->errp, err, (e)->err_class, fmt, ## __VA_ARGS__))
+#define err_set(e, err, fmt, ...)                                           \
+    ((e)->error_setg_win32_wrapper((e)->errp, __FILE__, __LINE__, __func__, \
+                                   err, fmt, ## __VA_ARGS__))
+/* Bad idea, works only when (e)->errp != NULL: */
 #define err_is_set(e) ((e)->errp && *(e)->errp)
-
+/* To lift this restriction, error_propagate(), like we do in QEMU code */
 
 /* Handle to VSSAPI.DLL */
 static HMODULE hLib;
diff --git a/qemu/qga/vss-win32/requester.h b/qemu/qga/vss-win32/requester.h
index 374f9b8d1..2a39d734a 100644
--- a/qemu/qga/vss-win32/requester.h
+++ b/qemu/qga/vss-win32/requester.h
@@ -14,19 +14,21 @@
 #define VSS_WIN32_REQUESTER_H
 
 #include <basetyps.h>           /* STDAPI */
-#include "qemu/compiler.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct Error;
+
 /* Callback to set Error; used to avoid linking glib to the DLL */
-typedef void (*ErrorSetFunc)(void **errp, int win32_err, int err_class,
-                             const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+typedef void (*ErrorSetFunc)(struct Error **errp,
+                             const char *src, int line, const char *func,
+                             int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 typedef struct ErrorSet {
-    ErrorSetFunc error_set;
-    void **errp;
-    int err_class;
+    ErrorSetFunc error_setg_win32_wrapper;
+    struct Error **errp;        /* restriction: must not be null */
 } ErrorSet;
 
 STDAPI requester_init(void);
diff --git a/qemu/qga/vss-win32/vss-common.h b/qemu/qga/vss-win32/vss-common.h
index ce14e1429..91dae0c38 100644
--- a/qemu/qga/vss-win32/vss-common.h
+++ b/qemu/qga/vss-win32/vss-common.h
@@ -14,7 +14,6 @@
 #define VSS_WIN32_H
 
 #define __MIDL_user_allocate_free_DEFINED__
-#include "config-host.h"
 #include <windows.h>
 #include <shlwapi.h>
 
diff --git a/qemu/qjson.c b/qemu/qjson.c
index e478802a4..b65ca6ee5 100644
--- a/qemu/qjson.c
+++ b/qemu/qjson.c
@@ -11,8 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <qapi/qmp/qstring.h>
-#include <stdbool.h>
 #include <glib.h>
 #include <qjson.h>
 #include <qemu/module.h>
diff --git a/qemu/qmp-commands.hx b/qemu/qmp-commands.hx
index ba630b1e7..de896a5a3 100644
--- a/qemu/qmp-commands.hx
+++ b/qemu/qmp-commands.hx
@@ -63,7 +63,7 @@ EQMP
     {
         .name       = "quit",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_quit,
+        .mhandler.cmd_new = qmp_marshal_quit,
     },
 
 SQMP
@@ -84,7 +84,7 @@ EQMP
     {
         .name       = "eject",
         .args_type  = "force:-f,device:B",
-        .mhandler.cmd_new = qmp_marshal_input_eject,
+        .mhandler.cmd_new = qmp_marshal_eject,
     },
 
 SQMP
@@ -110,7 +110,7 @@ EQMP
     {
         .name       = "change",
         .args_type  = "device:B,target:F,arg:s?",
-        .mhandler.cmd_new = qmp_marshal_input_change,
+        .mhandler.cmd_new = qmp_marshal_change,
     },
 
 SQMP
@@ -146,7 +146,7 @@ EQMP
     {
         .name       = "screendump",
         .args_type  = "filename:F",
-        .mhandler.cmd_new = qmp_marshal_input_screendump,
+        .mhandler.cmd_new = qmp_marshal_screendump,
     },
 
 SQMP
@@ -169,7 +169,7 @@ EQMP
     {
         .name       = "stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_stop,
+        .mhandler.cmd_new = qmp_marshal_stop,
     },
 
 SQMP
@@ -190,7 +190,7 @@ EQMP
     {
         .name       = "cont",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_cont,
+        .mhandler.cmd_new = qmp_marshal_cont,
     },
 
 SQMP
@@ -211,7 +211,7 @@ EQMP
     {
         .name       = "system_wakeup",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_wakeup,
+        .mhandler.cmd_new = qmp_marshal_system_wakeup,
     },
 
 SQMP
@@ -232,7 +232,7 @@ EQMP
     {
         .name       = "system_reset",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_reset,
+        .mhandler.cmd_new = qmp_marshal_system_reset,
     },
 
 SQMP
@@ -253,7 +253,7 @@ EQMP
     {
         .name       = "system_powerdown",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_powerdown,
+        .mhandler.cmd_new = qmp_marshal_system_powerdown,
     },
 
 SQMP
@@ -310,7 +310,7 @@ EQMP
     {
         .name       = "device_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_del,
+        .mhandler.cmd_new = qmp_marshal_device_del,
     },
 
 SQMP
@@ -321,19 +321,24 @@ Remove a device.
 
 Arguments:
 
-- "id": the device's ID (json-string)
+- "id": the device's ID or QOM path (json-string)
 
 Example:
 
 -> { "execute": "device_del", "arguments": { "id": "net1" } }
 <- { "return": {} }
 
+Example:
+
+-> { "execute": "device_del", "arguments": { "id": "/machine/peripheral-anon/device[0]" } }
+<- { "return": {} }
+
 EQMP
 
     {
         .name       = "send-key",
         .args_type  = "keys:q,hold-time:i?",
-        .mhandler.cmd_new = qmp_marshal_input_send_key,
+        .mhandler.cmd_new = qmp_marshal_send_key,
     },
 
 SQMP
@@ -364,7 +369,7 @@ EQMP
     {
         .name       = "cpu",
         .args_type  = "index:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu,
+        .mhandler.cmd_new = qmp_marshal_cpu,
     },
 
 SQMP
@@ -389,7 +394,7 @@ EQMP
     {
         .name       = "cpu-add",
         .args_type  = "id:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu_add,
+        .mhandler.cmd_new = qmp_marshal_cpu_add,
     },
 
 SQMP
@@ -412,7 +417,7 @@ EQMP
     {
         .name       = "memsave",
         .args_type  = "val:l,size:i,filename:s,cpu:i?",
-        .mhandler.cmd_new = qmp_marshal_input_memsave,
+        .mhandler.cmd_new = qmp_marshal_memsave,
     },
 
 SQMP
@@ -441,7 +446,7 @@ EQMP
     {
         .name       = "pmemsave",
         .args_type  = "val:l,size:i,filename:s",
-        .mhandler.cmd_new = qmp_marshal_input_pmemsave,
+        .mhandler.cmd_new = qmp_marshal_pmemsave,
     },
 
 SQMP
@@ -469,7 +474,7 @@ EQMP
     {
         .name       = "inject-nmi",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_inject_nmi,
+        .mhandler.cmd_new = qmp_marshal_inject_nmi,
     },
 
 SQMP
@@ -492,7 +497,7 @@ EQMP
     {
         .name       = "ringbuf-write",
         .args_type  = "device:s,data:s,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_write,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_write,
     },
 
 SQMP
@@ -507,8 +512,6 @@ Arguments:
 - "data": data to write (json-string)
 - "format": data format (json-string, optional)
           - Possible values: "utf8" (default), "base64"
-            Bug: invalid base64 is currently not rejected.
-            Whitespace *is* invalid.
 
 Example:
 
@@ -523,7 +526,7 @@ EQMP
     {
         .name       = "ringbuf-read",
         .args_type  = "device:s,size:i,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_read,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_read,
     },
 
 SQMP
@@ -559,7 +562,7 @@ EQMP
     {
         .name       = "xen-save-devices-state",
         .args_type  = "filename:F",
-    .mhandler.cmd_new = qmp_marshal_input_xen_save_devices_state,
+    .mhandler.cmd_new = qmp_marshal_xen_save_devices_state,
     },
 
 SQMP
@@ -586,7 +589,7 @@ EQMP
     {
         .name       = "xen-set-global-dirty-log",
         .args_type  = "enable:b",
-        .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log,
+        .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
     },
 
 SQMP
@@ -610,7 +613,7 @@ EQMP
     {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
-        .mhandler.cmd_new = qmp_marshal_input_migrate,
+        .mhandler.cmd_new = qmp_marshal_migrate,
     },
 
 SQMP
@@ -643,7 +646,7 @@ EQMP
     {
         .name       = "migrate_cancel",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_cancel,
+        .mhandler.cmd_new = qmp_marshal_migrate_cancel,
     },
 
 SQMP
@@ -664,7 +667,7 @@ EQMP
     {
         .name       = "migrate-incoming",
         .args_type  = "uri:s",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_incoming,
+        .mhandler.cmd_new = qmp_marshal_migrate_incoming,
     },
 
 SQMP
@@ -692,7 +695,7 @@ EQMP
     {
         .name       = "migrate-set-cache-size",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_cache_size,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_cache_size,
     },
 
 SQMP
@@ -713,9 +716,28 @@ Example:
 
 EQMP
     {
+        .name       = "migrate-start-postcopy",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_migrate_start_postcopy,
+    },
+
+SQMP
+migrate-start-postcopy
+----------------------
+
+Switch an in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+
+Example:
+-> { "execute": "migrate-start-postcopy" }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-migrate-cache-size",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_cache_size,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_cache_size,
     },
 
 SQMP
@@ -737,7 +759,7 @@ EQMP
     {
         .name       = "migrate_set_speed",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_speed,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_speed,
     },
 
 SQMP
@@ -760,7 +782,7 @@ EQMP
     {
         .name       = "migrate_set_downtime",
         .args_type  = "value:T",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_downtime,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_downtime,
     },
 
 SQMP
@@ -785,7 +807,7 @@ EQMP
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
         .help       = "set migration information for remote display",
-        .mhandler.cmd_new = qmp_marshal_input_client_migrate_info,
+        .mhandler.cmd_new = qmp_marshal_client_migrate_info,
     },
 
 SQMP
@@ -816,10 +838,10 @@ EQMP
 
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:b,protocol:s,begin:i?,end:i?,format:s?",
-        .params     = "-p protocol [begin] [length] [format]",
+        .args_type  = "paging:b,protocol:s,detach:b?,begin:i?,end:i?,format:s?",
+        .params     = "-p protocol [-d] [begin] [length] [format]",
         .help       = "dump guest memory to file",
-        .mhandler.cmd_new = qmp_marshal_input_dump_guest_memory,
+        .mhandler.cmd_new = qmp_marshal_dump_guest_memory,
     },
 
 SQMP
@@ -833,6 +855,9 @@ Arguments:
 - "paging": do paging to get guest's memory mapping (json-bool)
 - "protocol": destination file(started with "file:") or destination file
               descriptor (started with "fd:") (json-string)
+- "detach": if specified, command will return immediately, without waiting
+            for the dump to finish. The user can track progress using
+            "query-dump". (json-bool)
 - "begin": the starting physical address. It's optional, and should be specified
            with length together (json-int)
 - "length": the memory size, in bytes. It's optional, and should be specified
@@ -855,7 +880,7 @@ EQMP
     {
         .name       = "query-dump-guest-memory-capability",
         .args_type  = "",
-    .mhandler.cmd_new = qmp_marshal_input_query_dump_guest_memory_capability,
+    .mhandler.cmd_new = qmp_marshal_query_dump_guest_memory_capability,
     },
 
 SQMP
@@ -873,6 +898,55 @@ Example:
 EQMP
 
     {
+        .name       = "query-dump",
+        .args_type  = "",
+        .params     = "",
+        .help       = "query background dump status",
+        .mhandler.cmd_new = qmp_marshal_query_dump,
+    },
+
+SQMP
+query-dump
+----------
+
+Query background dump status.
+
+Arguments: None.
+
+Example:
+
+-> { "execute": "query-dump" }
+<- { "return": { "status": "active", "completed": 1024000,
+                 "total": 2048000 } }
+
+EQMP
+
+#if defined TARGET_S390X
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .mhandler.cmd_new = qmp_marshal_dump_skeys,
+    },
+#endif
+
+SQMP
+dump-skeys
+----------
+
+Save guest storage keys to file.
+
+Arguments:
+
+- "filename": file path (json-string)
+
+Example:
+
+-> { "execute": "dump-skeys", "arguments": { "filename": "/tmp/skeys" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "netdev_add",
         .args_type  = "netdev:O",
         .mhandler.cmd_new = qmp_netdev_add,
@@ -892,7 +966,9 @@ Arguments:
 
 Example:
 
--> { "execute": "netdev_add", "arguments": { "type": "user", "id": "netdev1" } }
+-> { "execute": "netdev_add",
+     "arguments": { "type": "user", "id": "netdev1",
+                    "dnssearch": "example.org" } }
 <- { "return": {} }
 
 Note: The supported device options are the same ones supported by the '-netdev'
@@ -904,7 +980,7 @@ EQMP
     {
         .name       = "netdev_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_netdev_del,
+        .mhandler.cmd_new = qmp_marshal_netdev_del,
     },
 
 SQMP
@@ -928,7 +1004,7 @@ EQMP
     {
         .name       = "object-add",
         .args_type  = "qom-type:s,id:s,props:q?",
-        .mhandler.cmd_new = qmp_object_add,
+        .mhandler.cmd_new = qmp_marshal_object_add,
     },
 
 SQMP
@@ -954,7 +1030,7 @@ EQMP
     {
         .name       = "object-del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_object_del,
+        .mhandler.cmd_new = qmp_marshal_object_del,
     },
 
 SQMP
@@ -979,7 +1055,7 @@ EQMP
     {
         .name       = "block_resize",
         .args_type  = "device:s?,node-name:s?,size:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_resize,
+        .mhandler.cmd_new = qmp_marshal_block_resize,
     },
 
 SQMP
@@ -1004,7 +1080,7 @@ EQMP
     {
         .name       = "block-stream",
         .args_type  = "device:B,base:s?,speed:o?,backing-file:s?,on-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_block_stream,
+        .mhandler.cmd_new = qmp_marshal_block_stream,
     },
 
 SQMP
@@ -1047,7 +1123,7 @@ EQMP
     {
         .name       = "block-commit",
         .args_type  = "device:B,base:s?,top:s?,backing-file:s?,speed:o?",
-        .mhandler.cmd_new = qmp_marshal_input_block_commit,
+        .mhandler.cmd_new = qmp_marshal_block_commit,
     },
 
 SQMP
@@ -1111,7 +1187,7 @@ EQMP
         .name       = "drive-backup",
         .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
                       "bitmap:s?,on-source-error:s?,on-target-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_backup,
+        .mhandler.cmd_new = qmp_marshal_drive_backup,
     },
 
 SQMP
@@ -1165,7 +1241,7 @@ EQMP
         .name       = "blockdev-backup",
         .args_type  = "sync:s,device:B,target:B,speed:i?,"
                       "on-source-error:s?,on-target-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_backup,
+        .mhandler.cmd_new = qmp_marshal_blockdev_backup,
     },
 
 SQMP
@@ -1205,44 +1281,55 @@ EQMP
     {
         .name       = "block-job-set-speed",
         .args_type  = "device:B,speed:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_set_speed,
+        .mhandler.cmd_new = qmp_marshal_block_job_set_speed,
     },
 
     {
         .name       = "block-job-cancel",
         .args_type  = "device:B,force:b?",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_cancel,
+        .mhandler.cmd_new = qmp_marshal_block_job_cancel,
     },
     {
         .name       = "block-job-pause",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_pause,
+        .mhandler.cmd_new = qmp_marshal_block_job_pause,
     },
     {
         .name       = "block-job-resume",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_resume,
+        .mhandler.cmd_new = qmp_marshal_block_job_resume,
     },
     {
         .name       = "block-job-complete",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_complete,
+        .mhandler.cmd_new = qmp_marshal_block_job_complete,
     },
     {
         .name       = "transaction",
-        .args_type  = "actions:q",
-        .mhandler.cmd_new = qmp_marshal_input_transaction,
+        .args_type  = "actions:q,properties:q?",
+        .mhandler.cmd_new = qmp_marshal_transaction,
     },
 
 SQMP
 transaction
 -----------
 
-Atomically operate on one or more block devices.  The only supported operations
-for now are drive-backup, internal and external snapshotting.  A list of
-dictionaries is accepted, that contains the actions to be performed.
-If there is any failure performing any of the operations, all operations
-for the group are abandoned.
+Atomically operate on one or more block devices.  Operations that are
+currently supported:
+
+    - drive-backup
+    - blockdev-backup
+    - blockdev-snapshot-sync
+    - blockdev-snapshot-internal-sync
+    - abort
+    - block-dirty-bitmap-add
+    - block-dirty-bitmap-clear
+
+Refer to the qemu/qapi-schema.json file for minimum required QEMU
+versions for these operations.  A list of dictionaries is accepted,
+that contains the actions to be performed.  If there is any failure
+performing any of the operations, all operations for the group are
+abandoned.
 
 For external snapshots, the dictionary contains the device, the file to use for
 the new snapshot, and the format.  The default format, if not specified, is
@@ -1269,8 +1356,12 @@ it later with qemu-img or other command.
 Arguments:
 
 actions array:
-    - "type": the operation to perform.  The only supported
-      value is "blockdev-snapshot-sync". (json-string)
+    - "type": the operation to perform (json-string).  Possible
+              values: "drive-backup", "blockdev-backup",
+                      "blockdev-snapshot-sync",
+                      "blockdev-snapshot-internal-sync",
+                      "abort", "block-dirty-bitmap-add",
+                      "block-dirty-bitmap-clear"
     - "data": a dictionary.  The contents depend on the value
       of "type".  When "type" is "blockdev-snapshot-sync":
       - "device": device name to snapshot (json-string)
@@ -1310,7 +1401,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-add",
         .args_type  = "node:B,name:s,granularity:i?",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_add,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_add,
     },
 
 SQMP
@@ -1338,7 +1429,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-remove",
         .args_type  = "node:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_remove,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_remove,
     },
 
 SQMP
@@ -1366,7 +1457,7 @@ EQMP
     {
         .name       = "block-dirty-bitmap-clear",
         .args_type  = "node:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_dirty_bitmap_clear,
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_clear,
     },
 
 SQMP
@@ -1395,7 +1486,7 @@ EQMP
     {
         .name       = "blockdev-snapshot-sync",
         .args_type  = "device:s?,node-name:s?,snapshot-file:s,snapshot-node-name:s?,format:s?,mode:s?",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_sync,
     },
 
 SQMP
@@ -1429,9 +1520,47 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-snapshot",
+        .args_type  = "node:s,overlay:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot,
+    },
+
+SQMP
+blockdev-snapshot
+-----------------
+Since 2.5
+
+Create a snapshot, by installing 'node' as the backing image of
+'overlay'. Additionally, if 'node' is associated with a block
+device, the block device changes to using 'overlay' as its new active
+image.
+
+Arguments:
+
+- "node": device that will have a snapshot created (json-string)
+- "overlay": device that will have 'node' as its backing image (json-string)
+
+Example:
+
+-> { "execute": "blockdev-add",
+                "arguments": { "options": { "driver": "qcow2",
+                                            "node-name": "node1534",
+                                            "file": { "driver": "file",
+                                                      "filename": "hd1.qcow2" },
+                                            "backing": "" } } }
+
+<- { "return": {} }
+
+-> { "execute": "blockdev-snapshot", "arguments": { "node": "ide-hd0",
+                                                    "overlay": "node1534" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "blockdev-snapshot-internal-sync",
         .args_type  = "device:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_internal_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_internal_sync,
     },
 
 SQMP
@@ -1461,7 +1590,7 @@ EQMP
         .name       = "blockdev-snapshot-delete-internal-sync",
         .args_type  = "device:B,id:s?,name:s?",
         .mhandler.cmd_new =
-                      qmp_marshal_input_blockdev_snapshot_delete_internal_sync,
+                      qmp_marshal_blockdev_snapshot_delete_internal_sync,
     },
 
 SQMP
@@ -1505,7 +1634,7 @@ EQMP
                       "on-source-error:s?,on-target-error:s?,"
                       "unmap:b?,"
                       "granularity:i?,buf-size:i?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
+        .mhandler.cmd_new = qmp_marshal_drive_mirror,
     },
 
 SQMP
@@ -1533,7 +1662,7 @@ Arguments:
 - "speed": maximum speed of the streaming job, in bytes per second
   (json-int)
 - "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
-- "buf_size": maximum amount of data in flight from source to target, in bytes
+- "buf-size": maximum amount of data in flight from source to target, in bytes
   (json-int, default 10M)
 - "sync": what parts of the disk image should be copied to the destination;
   possibilities include "full" for all the disk, "top" for only the sectors
@@ -1563,9 +1692,57 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-mirror",
+        .args_type  = "sync:s,device:B,target:B,replaces:s?,speed:i?,"
+                      "on-source-error:s?,on-target-error:s?,"
+                      "granularity:i?,buf-size:i?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_mirror,
+    },
+
+SQMP
+blockdev-mirror
+------------
+
+Start mirroring a block device's writes to another block device. target
+specifies the target of mirror operation.
+
+Arguments:
+
+- "device": device name to operate on (json-string)
+- "target": device name to mirror to (json-string)
+- "replaces": the block driver node name to replace when finished
+              (json-string, optional)
+- "speed": maximum speed of the streaming job, in bytes per second
+  (json-int)
+- "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
+- "buf_size": maximum amount of data in flight from source to target, in bytes
+  (json-int, default 10M)
+- "sync": what parts of the disk image should be copied to the destination;
+  possibilities include "full" for all the disk, "top" for only the sectors
+  allocated in the topmost image, or "none" to only replicate new I/O
+  (MirrorSyncMode).
+- "on-source-error": the action to take on an error on the source
+  (BlockdevOnError, default 'report')
+- "on-target-error": the action to take on an error on the target
+  (BlockdevOnError, default 'report')
+
+The default value of the granularity is the image cluster size clamped
+between 4096 and 65536, if the image format defines one.  If the format
+does not define a cluster size, the default value of the granularity
+is 65536.
+
+Example:
+
+-> { "execute": "blockdev-mirror", "arguments": { "device": "ide-hd0",
+                                                  "target": "target0",
+                                                  "sync": "full" } }
+<- { "return": {} }
+
+EQMP
+    {
         .name       = "change-backing-file",
         .args_type  = "device:s,image-node-name:s,backing-file:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_backing_file,
+        .mhandler.cmd_new = qmp_marshal_change_backing_file,
     },
 
 SQMP
@@ -1604,7 +1781,7 @@ EQMP
     {
         .name       = "balloon",
         .args_type  = "value:M",
-        .mhandler.cmd_new = qmp_marshal_input_balloon,
+        .mhandler.cmd_new = qmp_marshal_balloon,
     },
 
 SQMP
@@ -1627,7 +1804,7 @@ EQMP
     {
         .name       = "set_link",
         .args_type  = "name:s,up:b",
-        .mhandler.cmd_new = qmp_marshal_input_set_link,
+        .mhandler.cmd_new = qmp_marshal_set_link,
     },
 
 SQMP
@@ -1653,7 +1830,7 @@ EQMP
         .args_type  = "fdname:s",
         .params     = "getfd name",
         .help       = "receive a file descriptor via SCM rights and assign it a name",
-        .mhandler.cmd_new = qmp_marshal_input_getfd,
+        .mhandler.cmd_new = qmp_marshal_getfd,
     },
 
 SQMP
@@ -1686,7 +1863,7 @@ EQMP
         .args_type  = "fdname:s",
         .params     = "closefd name",
         .help       = "close a file descriptor previously passed via SCM rights",
-        .mhandler.cmd_new = qmp_marshal_input_closefd,
+        .mhandler.cmd_new = qmp_marshal_closefd,
     },
 
 SQMP
@@ -1711,7 +1888,7 @@ EQMP
         .args_type  = "fdset-id:i?,opaque:s?",
         .params     = "add-fd fdset-id opaque",
         .help       = "Add a file descriptor, that was passed via SCM rights, to an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_add_fd,
+        .mhandler.cmd_new = qmp_marshal_add_fd,
     },
 
 SQMP
@@ -1750,7 +1927,7 @@ EQMP
         .args_type  = "fdset-id:i,fd:i?",
         .params     = "remove-fd fdset-id fd",
         .help       = "Remove a file descriptor from an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_remove_fd,
+        .mhandler.cmd_new = qmp_marshal_remove_fd,
     },
 
 SQMP
@@ -1782,7 +1959,7 @@ EQMP
         .name       = "query-fdsets",
         .args_type  = "",
         .help       = "Return information describing all fd sets",
-        .mhandler.cmd_new = qmp_marshal_input_query_fdsets,
+        .mhandler.cmd_new = qmp_marshal_query_fdsets,
     },
 
 SQMP
@@ -1831,7 +2008,7 @@ EQMP
     {
         .name       = "block_passwd",
         .args_type  = "device:s?,node-name:s?,password:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_passwd,
+        .mhandler.cmd_new = qmp_marshal_block_passwd,
     },
 
 SQMP
@@ -1856,8 +2033,8 @@ EQMP
 
     {
         .name       = "block_set_io_throttle",
-        .args_type  = "device:B,bps:l,bps_rd:l,bps_wr:l,iops:l,iops_rd:l,iops_wr:l,bps_max:l?,bps_rd_max:l?,bps_wr_max:l?,iops_max:l?,iops_rd_max:l?,iops_wr_max:l?,iops_size:l?,group:s?",
-        .mhandler.cmd_new = qmp_marshal_input_block_set_io_throttle,
+        .args_type  = "device:B,bps:l,bps_rd:l,bps_wr:l,iops:l,iops_rd:l,iops_wr:l,bps_max:l?,bps_rd_max:l?,bps_wr_max:l?,iops_max:l?,iops_rd_max:l?,iops_wr_max:l?,bps_max_length:l?,bps_rd_max_length:l?,bps_wr_max_length:l?,iops_max_length:l?,iops_rd_max_length:l?,iops_wr_max_length:l?,iops_size:l?,group:s?",
+        .mhandler.cmd_new = qmp_marshal_block_set_io_throttle,
     },
 
 SQMP
@@ -1875,14 +2052,20 @@ Arguments:
 - "iops": total I/O operations per second (json-int)
 - "iops_rd": read I/O operations per second (json-int)
 - "iops_wr": write I/O operations per second (json-int)
-- "bps_max":  total max in bytes (json-int)
-- "bps_rd_max":  read max in bytes (json-int)
-- "bps_wr_max":  write max in bytes (json-int)
-- "iops_max":  total I/O operations max (json-int)
-- "iops_rd_max":  read I/O operations max (json-int)
-- "iops_wr_max":  write I/O operations max (json-int)
-- "iops_size":  I/O size in bytes when limiting (json-int)
-- "group": throttle group name (json-string)
+- "bps_max": total throughput limit during bursts, in bytes (json-int, optional)
+- "bps_rd_max": read throughput limit during bursts, in bytes (json-int, optional)
+- "bps_wr_max": write throughput limit during bursts, in bytes (json-int, optional)
+- "iops_max": total I/O operations per second during bursts (json-int, optional)
+- "iops_rd_max": read I/O operations per second during bursts (json-int, optional)
+- "iops_wr_max": write I/O operations per second during bursts (json-int, optional)
+- "bps_max_length": maximum length of the @bps_max burst period, in seconds (json-int, optional)
+- "bps_rd_max_length": maximum length of the @bps_rd_max burst period, in seconds (json-int, optional)
+- "bps_wr_max_length": maximum length of the @bps_wr_max burst period, in seconds (json-int, optional)
+- "iops_max_length": maximum length of the @iops_max burst period, in seconds (json-int, optional)
+- "iops_rd_max_length": maximum length of the @iops_rd_max burst period, in seconds (json-int, optional)
+- "iops_wr_max_length": maximum length of the @iops_wr_max burst period, in seconds (json-int, optional)
+- "iops_size":  I/O size in bytes when limiting (json-int, optional)
+- "group": throttle group name (json-string, optional)
 
 Example:
 
@@ -1899,6 +2082,7 @@ Example:
                                                "iops_max": 0,
                                                "iops_rd_max": 0,
                                                "iops_wr_max": 0,
+                                               "bps_max_length": 60,
                                                "iops_size": 0 } }
 <- { "return": {} }
 
@@ -1907,7 +2091,7 @@ EQMP
     {
         .name       = "set_password",
         .args_type  = "protocol:s,password:s,connected:s?",
-        .mhandler.cmd_new = qmp_marshal_input_set_password,
+        .mhandler.cmd_new = qmp_marshal_set_password,
     },
 
 SQMP
@@ -1933,7 +2117,7 @@ EQMP
     {
         .name       = "expire_password",
         .args_type  = "protocol:s,time:s",
-        .mhandler.cmd_new = qmp_marshal_input_expire_password,
+        .mhandler.cmd_new = qmp_marshal_expire_password,
     },
 
 SQMP
@@ -1958,7 +2142,7 @@ EQMP
     {
         .name       = "add_client",
         .args_type  = "protocol:s,fdname:s,skipauth:b?,tls:b?",
-        .mhandler.cmd_new = qmp_marshal_input_add_client,
+        .mhandler.cmd_new = qmp_marshal_add_client,
     },
 
 SQMP
@@ -2009,7 +2193,7 @@ EQMP
     {
         .name       = "human-monitor-command",
         .args_type  = "command-line:s,cpu-index:i?",
-        .mhandler.cmd_new = qmp_marshal_input_human_monitor_command,
+        .mhandler.cmd_new = qmp_marshal_human_monitor_command,
     },
 
 SQMP
@@ -2088,7 +2272,7 @@ EQMP
     {
         .name       = "query-version",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_version,
+        .mhandler.cmd_new = qmp_marshal_query_version,
     },
 
 SQMP
@@ -2125,7 +2309,7 @@ EQMP
     {
         .name       = "query-commands",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_commands,
+        .mhandler.cmd_new = qmp_marshal_query_commands,
     },
 
 SQMP
@@ -2162,7 +2346,24 @@ EQMP
     {
         .name       = "query-events",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_events,
+        .mhandler.cmd_new = qmp_marshal_query_events,
+    },
+
+SQMP
+query-qmp-schema
+----------------
+
+Return the QMP wire schema.  The returned value is a json-array of
+named schema entities.  Entities are commands, events and various
+types.  See docs/qapi-code-gen.txt for information on their structure
+and intended use.
+
+EQMP
+
+    {
+        .name       = "query-qmp-schema",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_query_qmp_schema,
     },
 
 SQMP
@@ -2207,7 +2408,7 @@ EQMP
     {
         .name       = "query-chardev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev,
+        .mhandler.cmd_new = qmp_marshal_query_chardev,
     },
 
 SQMP
@@ -2248,7 +2449,7 @@ EQMP
     {
         .name       = "query-chardev-backends",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev_backends,
+        .mhandler.cmd_new = qmp_marshal_query_chardev_backends,
     },
 
 SQMP
@@ -2432,7 +2633,7 @@ EQMP
     {
         .name       = "query-block",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block,
+        .mhandler.cmd_new = qmp_marshal_query_block,
     },
 
 SQMP
@@ -2456,12 +2657,70 @@ Each json-object contain the following:
     - "wr_total_time_ns": total time spend on writes in nano-seconds (json-int)
     - "rd_total_time_ns": total time spend on reads in nano-seconds (json-int)
     - "flush_total_time_ns": total time spend on cache flushes in nano-seconds (json-int)
-    - "wr_highest_offset": Highest offset of a sector written since the
-                           BlockDriverState has been opened (json-int)
+    - "wr_highest_offset": The offset after the greatest byte written to the
+                           BlockDriverState since it has been opened (json-int)
     - "rd_merged": number of read requests that have been merged into
                    another request (json-int)
     - "wr_merged": number of write requests that have been merged into
                    another request (json-int)
+    - "idle_time_ns": time since the last I/O operation, in
+                      nanoseconds. If the field is absent it means
+                      that there haven't been any operations yet
+                      (json-int, optional)
+    - "failed_rd_operations": number of failed read operations
+                              (json-int)
+    - "failed_wr_operations": number of failed write operations
+                              (json-int)
+    - "failed_flush_operations": number of failed flush operations
+                               (json-int)
+    - "invalid_rd_operations": number of invalid read operations
+                               (json-int)
+    - "invalid_wr_operations": number of invalid write operations
+                               (json-int)
+    - "invalid_flush_operations": number of invalid flush operations
+                                  (json-int)
+    - "account_invalid": whether invalid operations are included in
+                         the last access statistics (json-bool)
+    - "account_failed": whether failed operations are included in the
+                         latency and last access statistics
+                         (json-bool)
+    - "timed_stats": A json-array containing statistics collected in
+                     specific intervals, with the following members:
+        - "interval_length": interval used for calculating the
+                             statistics, in seconds (json-int)
+        - "min_rd_latency_ns": minimum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_wr_latency_ns": minimum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_flush_latency_ns": minimum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "max_rd_latency_ns": maximum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_wr_latency_ns": maximum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_flush_latency_ns": maximum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_latency_ns": average latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_wr_latency_ns": average latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_flush_latency_ns": average latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_queue_depth": average number of pending read
+                                operations in the defined interval
+                                (json-number)
+        - "avg_wr_queue_depth": average number of pending write
+                                operations in the defined interval
+                                (json-number).
 - "parent": Contains recursively the statistics of the underlying
             protocol (e.g. the host file for a qcow2 image). If there is
             no underlying protocol, this field is omitted
@@ -2486,7 +2745,10 @@ Example:
                   "flush_total_times_ns":49653
                   "flush_operations":61,
                   "rd_merged":0,
-                  "wr_merged":0
+                  "wr_merged":0,
+                  "idle_time_ns":2953431879,
+                  "account_invalid":true,
+                  "account_failed":false
                }
             },
             "stats":{
@@ -2500,7 +2762,10 @@ Example:
                "rd_total_times_ns":3465673657
                "flush_total_times_ns":49653,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "idle_time_ns":2953431879,
+               "account_invalid":true,
+               "account_failed":false
             }
          },
          {
@@ -2516,7 +2781,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2532,7 +2799,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2548,7 +2817,9 @@ Example:
                "rd_total_times_ns":0
                "flush_total_times_ns":0,
                "rd_merged":0,
-               "wr_merged":0
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          }
       ]
@@ -2559,7 +2830,7 @@ EQMP
     {
         .name       = "query-blockstats",
         .args_type  = "query-nodes:b?",
-        .mhandler.cmd_new = qmp_marshal_input_query_blockstats,
+        .mhandler.cmd_new = qmp_marshal_query_blockstats,
     },
 
 SQMP
@@ -2574,6 +2845,8 @@ Return a json-array. Each CPU is represented by a json-object, which contains:
 - "current": true if this is the current CPU, false otherwise (json-bool)
 - "halted": true if the cpu is halted, false otherwise (json-bool)
 - "qom_path": path to the CPU object in the QOM tree (json-str)
+- "arch": architecture of the cpu, which determines what additional
+          keys will be present (json-str)
 - Current program counter. The key's name depends on the architecture:
      "pc": i386/x86_64 (json-int)
      "nip": PPC (json-int)
@@ -2591,6 +2864,7 @@ Example:
             "current":true,
             "halted":false,
             "qom_path":"/machine/unattached/device[0]",
+            "arch":"x86",
             "pc":3227107138,
             "thread_id":3134
          },
@@ -2599,6 +2873,7 @@ Example:
             "current":false,
             "halted":true,
             "qom_path":"/machine/unattached/device[2]",
+            "arch":"x86",
             "pc":7108165,
             "thread_id":3135
          }
@@ -2610,7 +2885,7 @@ EQMP
     {
         .name       = "query-cpus",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpus,
+        .mhandler.cmd_new = qmp_marshal_query_cpus,
     },
 
 SQMP
@@ -2649,7 +2924,7 @@ EQMP
     {
         .name       = "query-iothreads",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_iothreads,
+        .mhandler.cmd_new = qmp_marshal_query_iothreads,
     },
 
 SQMP
@@ -2866,7 +3141,7 @@ EQMP
     {
         .name       = "query-pci",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_pci,
+        .mhandler.cmd_new = qmp_marshal_query_pci,
     },
 
 SQMP
@@ -2890,7 +3165,7 @@ EQMP
     {
         .name       = "query-kvm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_kvm,
+        .mhandler.cmd_new = qmp_marshal_query_kvm,
     },
 
 SQMP
@@ -2930,7 +3205,7 @@ EQMP
     {
         .name       = "query-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_status,
+        .mhandler.cmd_new = qmp_marshal_query_status,
     },
 
 SQMP
@@ -2974,7 +3249,7 @@ EQMP
     {
         .name       = "query-mice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_mice,
+        .mhandler.cmd_new = qmp_marshal_query_mice,
     },
 
 SQMP
@@ -3037,12 +3312,12 @@ EQMP
     {
         .name       = "query-vnc",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_vnc,
+        .mhandler.cmd_new = qmp_marshal_query_vnc,
     },
     {
         .name       = "query-vnc-servers",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_vnc_servers,
+        .mhandler.cmd_new = qmp_marshal_query_vnc_servers,
     },
 
 SQMP
@@ -3119,7 +3394,7 @@ EQMP
     {
         .name       = "query-spice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_spice,
+        .mhandler.cmd_new = qmp_marshal_query_spice,
     },
 #endif
 
@@ -3143,7 +3418,7 @@ EQMP
     {
         .name       = "query-name",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_name,
+        .mhandler.cmd_new = qmp_marshal_query_name,
     },
 
 SQMP
@@ -3166,7 +3441,7 @@ EQMP
     {
         .name       = "query-uuid",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_uuid,
+        .mhandler.cmd_new = qmp_marshal_query_uuid,
     },
 
 SQMP
@@ -3215,7 +3490,7 @@ EQMP
     {
         .name       = "query-command-line-options",
         .args_type  = "option:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_command_line_options,
+        .mhandler.cmd_new = qmp_marshal_query_command_line_options,
     },
 
 SQMP
@@ -3393,7 +3668,7 @@ EQMP
     {
         .name       = "query-migrate",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate,
+        .mhandler.cmd_new = qmp_marshal_query_migrate,
     },
 
 SQMP
@@ -3406,7 +3681,9 @@ Enable/Disable migration capabilities
 - "rdma-pin-all": pin all pages when using RDMA during migration
 - "auto-converge": throttle down guest to help convergence of migration
 - "zero-blocks": compress zero blocks during block migration
+- "compress": use multiple compression threads to accelerate live migration
 - "events": generate events for each migration state change
+- "postcopy-ram": postcopy mode for live migration
 
 Arguments:
 
@@ -3421,7 +3698,7 @@ EQMP
         .name       = "migrate-set-capabilities",
         .args_type  = "capabilities:q",
         .params     = "capability:s,state:b",
-	.mhandler.cmd_new = qmp_marshal_input_migrate_set_capabilities,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_capabilities,
     },
 SQMP
 query-migrate-capabilities
@@ -3434,20 +3711,31 @@ Query current migration capabilities
          - "rdma-pin-all" : RDMA Pin Page state (json-bool)
          - "auto-converge" : Auto Converge state (json-bool)
          - "zero-blocks" : Zero Blocks state (json-bool)
+         - "compress": Multiple compression threads state (json-bool)
+         - "events": Migration state change event state (json-bool)
+         - "postcopy-ram": postcopy ram state (json-bool)
 
 Arguments:
 
 Example:
 
 -> { "execute": "query-migrate-capabilities" }
-<- { "return": [ { "state": false, "capability": "xbzrle" } ] }
+<- {"return": [
+     {"state": false, "capability": "xbzrle"},
+     {"state": false, "capability": "rdma-pin-all"},
+     {"state": false, "capability": "auto-converge"},
+     {"state": false, "capability": "zero-blocks"},
+     {"state": false, "capability": "compress"},
+     {"state": true, "capability": "events"},
+     {"state": false, "capability": "postcopy-ram"}
+   ]}
 
 EQMP
 
     {
         .name       = "query-migrate-capabilities",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_capabilities,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_capabilities,
     },
 
 SQMP
@@ -3459,6 +3747,10 @@ Set migration parameters
 - "compress-level": set compression level during migration (json-int)
 - "compress-threads": set compression thread count for migration (json-int)
 - "decompress-threads": set decompression thread count for migration (json-int)
+- "x-cpu-throttle-initial": set initial percentage of time guest cpus are
+                           throttled for auto-converge (json-int)
+- "x-cpu-throttle-increment": set throttle increasing percentage for
+                             auto-converge (json-int)
 
 Arguments:
 
@@ -3472,8 +3764,8 @@ EQMP
     {
         .name       = "migrate-set-parameters",
         .args_type  =
-            "compress-level:i?,compress-threads:i?,decompress-threads:i?",
-	.mhandler.cmd_new = qmp_marshal_input_migrate_set_parameters,
+            "compress-level:i?,compress-threads:i?,decompress-threads:i?,x-cpu-throttle-initial:i?,x-cpu-throttle-increment:i?",
+        .mhandler.cmd_new = qmp_marshal_migrate_set_parameters,
     },
 SQMP
 query-migrate-parameters
@@ -3485,6 +3777,10 @@ Query current migration parameters
          - "compress-level" : compression level value (json-int)
          - "compress-threads" : compression thread count value (json-int)
          - "decompress-threads" : decompression thread count value (json-int)
+         - "x-cpu-throttle-initial" : initial percentage of time guest cpus are
+                                      throttled (json-int)
+         - "x-cpu-throttle-increment" : throttle increasing percentage for
+                                        auto-converge (json-int)
 
 Arguments:
 
@@ -3493,9 +3789,11 @@ Example:
 -> { "execute": "query-migrate-parameters" }
 <- {
       "return": {
-         "decompress-threads", 2,
-         "compress-threads", 8,
-         "compress-level", 1
+         "decompress-threads": 2,
+         "x-cpu-throttle-increment": 10,
+         "compress-threads": 8,
+         "compress-level": 1,
+         "x-cpu-throttle-initial": 20
       }
    }
 
@@ -3504,7 +3802,7 @@ EQMP
     {
         .name       = "query-migrate-parameters",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_parameters,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_parameters,
     },
 
 SQMP
@@ -3532,88 +3830,88 @@ EQMP
     {
         .name       = "query-balloon",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_balloon,
+        .mhandler.cmd_new = qmp_marshal_query_balloon,
     },
 
     {
         .name       = "query-block-jobs",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block_jobs,
+        .mhandler.cmd_new = qmp_marshal_query_block_jobs,
     },
 
     {
         .name       = "qom-list",
         .args_type  = "path:s",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list,
+        .mhandler.cmd_new = qmp_marshal_qom_list,
     },
 
     {
         .name       = "qom-set",
 	.args_type  = "path:s,property:s,value:q",
-	.mhandler.cmd_new = qmp_qom_set,
+        .mhandler.cmd_new = qmp_marshal_qom_set,
     },
 
     {
         .name       = "qom-get",
 	.args_type  = "path:s,property:s",
-	.mhandler.cmd_new = qmp_qom_get,
+        .mhandler.cmd_new = qmp_marshal_qom_get,
     },
 
     {
         .name       = "nbd-server-start",
-        .args_type  = "addr:q",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_start,
+        .args_type  = "addr:q,tls-creds:s?",
+        .mhandler.cmd_new = qmp_marshal_nbd_server_start,
     },
     {
         .name       = "nbd-server-add",
         .args_type  = "device:B,writable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_add,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_add,
     },
     {
         .name       = "nbd-server-stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_stop,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_stop,
     },
 
     {
         .name       = "change-vnc-password",
         .args_type  = "password:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_vnc_password,
+        .mhandler.cmd_new = qmp_marshal_change_vnc_password,
     },
     {
         .name       = "qom-list-types",
         .args_type  = "implements:s?,abstract:b?",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list_types,
+        .mhandler.cmd_new = qmp_marshal_qom_list_types,
     },
 
     {
         .name       = "device-list-properties",
         .args_type  = "typename:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_list_properties,
+        .mhandler.cmd_new = qmp_marshal_device_list_properties,
     },
 
     {
         .name       = "query-machines",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_machines,
+        .mhandler.cmd_new = qmp_marshal_query_machines,
     },
 
     {
         .name       = "query-cpu-definitions",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpu_definitions,
+        .mhandler.cmd_new = qmp_marshal_query_cpu_definitions,
     },
 
     {
         .name       = "query-target",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_target,
+        .mhandler.cmd_new = qmp_marshal_query_target,
     },
 
     {
         .name       = "query-tpm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm,
+        .mhandler.cmd_new = qmp_marshal_query_tpm,
     },
 
 SQMP
@@ -3647,7 +3945,7 @@ EQMP
     {
         .name       = "query-tpm-models",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_models,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_models,
     },
 
 SQMP
@@ -3668,7 +3966,7 @@ EQMP
     {
         .name       = "query-tpm-types",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_types,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_types,
     },
 
 SQMP
@@ -3689,7 +3987,7 @@ EQMP
     {
         .name       = "chardev-add",
         .args_type  = "id:s,backend:q",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_add,
+        .mhandler.cmd_new = qmp_marshal_chardev_add,
     },
 
 SQMP
@@ -3726,7 +4024,7 @@ EQMP
     {
         .name       = "chardev-remove",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_remove,
+        .mhandler.cmd_new = qmp_marshal_chardev_remove,
     },
 
 
@@ -3749,7 +4047,7 @@ EQMP
     {
         .name       = "query-rx-filter",
         .args_type  = "name:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rx_filter,
+        .mhandler.cmd_new = qmp_marshal_query_rx_filter,
     },
 
 SQMP
@@ -3815,7 +4113,7 @@ EQMP
     {
         .name       = "blockdev-add",
         .args_type  = "options:q",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_add,
+        .mhandler.cmd_new = qmp_marshal_blockdev_add,
     },
 
 SQMP
@@ -3825,8 +4123,8 @@ blockdev-add
 Add a block device.
 
 This command is still a work in progress.  It doesn't support all
-block drivers, it lacks a matching blockdev-del, and more.  Stay away
-from it unless you want to help with its development.
+block drivers among other things.  Stay away from it unless you want
+to help with its development.
 
 Arguments:
 
@@ -3872,9 +4170,237 @@ Example (2):
 EQMP
 
     {
+        .name       = "x-blockdev-del",
+        .args_type  = "id:s?,node-name:s?",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_del,
+    },
+
+SQMP
+x-blockdev-del
+------------
+Since 2.5
+
+Deletes a block device thas has been added using blockdev-add.
+The selected device can be either a block backend or a graph node.
+
+In the former case the backend will be destroyed, along with its
+inserted medium if there's any. The command will fail if the backend
+or its medium are in use.
+
+In the latter case the node will be destroyed. The command will fail
+if the node is attached to a block backend or is otherwise being
+used.
+
+One of "id" or "node-name" must be specified, but not both.
+
+This command is still a work in progress and is considered
+experimental. Stay away from it unless you want to help with its
+development.
+
+Arguments:
+
+- "id": Name of the block backend device to delete (json-string, optional)
+- "node-name": Name of the graph node to delete (json-string, optional)
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": {
+         "options": {
+             "driver": "qcow2",
+             "id": "drive0",
+             "file": {
+                 "driver": "file",
+                 "filename": "test.qcow2"
+             }
+         }
+     }
+   }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-del",
+     "arguments": { "id": "drive0" }
+   }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-open-tray",
+        .args_type  = "device:s,force:b?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_open_tray,
+    },
+
+SQMP
+blockdev-open-tray
+------------------
+
+Opens a block device's tray. If there is a block driver state tree inserted as a
+medium, it will become inaccessible to the guest (but it will remain associated
+to the block device, so closing the tray will make it accessible again).
+
+If the tray was already open before, this will be a no-op.
+
+Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+which no such event will be generated, these include:
+- if the guest has locked the tray, @force is false and the guest does not
+  respond to the eject request
+- if the BlockBackend denoted by @device does not have a guest device attached
+  to it
+- if the guest device does not have an actual tray and is empty, for instance
+  for floppy disk drives
+
+Arguments:
+
+- "device": block device name (json-string)
+- "force": if false (the default), an eject request will be sent to the guest if
+           it has locked the tray (and the tray will not be opened immediately);
+           if true, the tray will be opened regardless of whether it is locked
+           (json-bool, optional)
+
+Example:
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751016,
+                    "microseconds": 716996 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-close-tray",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_close_tray,
+    },
+
+SQMP
+blockdev-close-tray
+-------------------
+
+Closes a block device's tray. If there is a block driver state tree associated
+with the block device (which is currently ejected), that tree will be loaded as
+the medium.
+
+If the tray was already closed before, this will be a no-op.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "blockdev-close-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751345,
+                    "microseconds": 272147 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": false } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-remove-medium",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_remove_medium,
+    },
+
+SQMP
+x-blockdev-remove-medium
+------------------------
+
+Removes a medium (a block driver state tree) from a block device. That block
+device's tray must currently be open (unless there is no attached guest device).
+
+If the tray is open and there is no medium inserted, this will be a no-op.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "error": { "class": "GenericError",
+                "desc": "Tray of device 'ide1-cd0' is not open" } }
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751627,
+                    "microseconds": 549958 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-insert-medium",
+        .args_type  = "device:s,node-name:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_insert_medium,
+    },
+
+SQMP
+x-blockdev-insert-medium
+------------------------
+
+Inserts a medium (a block driver state tree) into a block device. That block
+device's tray must currently be open (unless there is no attached guest device)
+and there must be no medium inserted already.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+- "node-name": root node of the BDS tree to insert into the block device
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": { "options": { "node-name": "node0",
+                                 "driver": "raw",
+                                 "file": { "driver": "file",
+                                           "filename": "fedora.iso" } } } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-insert-medium",
+     "arguments": { "device": "ide1-cd0",
+                    "node-name": "node0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-named-block-nodes",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_named_block_nodes,
+        .mhandler.cmd_new = qmp_marshal_query_named_block_nodes,
     },
 
 SQMP
@@ -3934,9 +4460,62 @@ Example:
 EQMP
 
     {
+        .name       = "blockdev-change-medium",
+        .args_type  = "device:B,filename:F,format:s?,read-only-mode:s?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_change_medium,
+    },
+
+SQMP
+blockdev-change-medium
+----------------------
+
+Changes the medium inserted into a block device by ejecting the current medium
+and loading a new image file which is inserted as the new medium.
+
+Arguments:
+
+- "device": device name (json-string)
+- "filename": filename of the new image (json-string)
+- "format": format of the new image (json-string, optional)
+- "read-only-mode": new read-only mode (json-string, optional)
+          - Possible values: "retain" (default), "read-only", "read-write"
+
+Examples:
+
+1. Change a removable medium
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "ide1-cd0",
+                            "filename": "/srv/images/Fedora-12-x86_64-DVD.iso",
+                            "format": "raw" } }
+<- { "return": {} }
+
+2. Load a read-only medium into a writable drive
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "retain" } }
+
+<- { "error":
+     { "class": "GenericError",
+       "desc": "Could not open '/srv/images/ro.img': Permission denied" } }
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "read-only" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-memdev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memdev,
+        .mhandler.cmd_new = qmp_marshal_query_memdev,
     },
 
 SQMP
@@ -3974,7 +4553,7 @@ EQMP
     {
         .name       = "query-memory-devices",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memory_devices,
+        .mhandler.cmd_new = qmp_marshal_query_memory_devices,
     },
 
 SQMP
@@ -4001,7 +4580,7 @@ EQMP
     {
         .name       = "query-acpi-ospm-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_acpi_ospm_status,
+        .mhandler.cmd_new = qmp_marshal_query_acpi_ospm_status,
     },
 
 SQMP
@@ -4024,7 +4603,7 @@ EQMP
     {
         .name       = "rtc-reset-reinjection",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_rtc_reset_reinjection,
+        .mhandler.cmd_new = qmp_marshal_rtc_reset_reinjection,
     },
 #endif
 
@@ -4045,7 +4624,7 @@ EQMP
     {
         .name       = "trace-event-get-state",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_get_state,
+        .mhandler.cmd_new = qmp_marshal_trace_event_get_state,
     },
 
 SQMP
@@ -4063,7 +4642,7 @@ EQMP
     {
         .name       = "trace-event-set-state",
         .args_type  = "name:s,enable:b,ignore-unavailable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_set_state,
+        .mhandler.cmd_new = qmp_marshal_trace_event_set_state,
     },
 
 SQMP
@@ -4079,21 +4658,22 @@ Example:
 EQMP
 
     {
-        .name       = "x-input-send-event",
+        .name       = "input-send-event",
         .args_type  = "console:i?,events:q",
-        .mhandler.cmd_new = qmp_marshal_input_x_input_send_event,
+        .mhandler.cmd_new = qmp_marshal_input_send_event,
     },
 
 SQMP
-@x-input-send-event
+@input-send-event
 -----------------
 
 Send input event to guest.
 
 Arguments:
 
-- "console": console index. (json-int, optional)
-- "events": list of input events.
+- "device": display device (json-string, optional)
+- "head": display head (json-int, optional)
+- "events": list of input events
 
 The consoles are visible in the qom tree, under
 /backend/console[$index]. They have a device link and head property, so
@@ -4105,24 +4685,24 @@ Example (1):
 
 Press left mouse button.
 
--> { "execute": "x-input-send-event",
-    "arguments": { "console": 0,
+-> { "execute": "input-send-event",
+    "arguments": { "device": "video0",
                    "events": [ { "type": "btn",
-                    "data" : { "down": true, "button": "Left" } } ] } }
+                   "data" : { "down": true, "button": "left" } } ] } }
 <- { "return": {} }
 
--> { "execute": "x-input-send-event",
-    "arguments": { "console": 0,
+-> { "execute": "input-send-event",
+    "arguments": { "device": "video0",
                    "events": [ { "type": "btn",
-                    "data" : { "down": false, "button": "Left" } } ] } }
+                   "data" : { "down": false, "button": "left" } } ] } }
 <- { "return": {} }
 
 Example (2):
 
 Press ctrl-alt-del.
 
--> { "execute": "x-input-send-event",
-     "arguments": { "console": 0, "events": [
+-> { "execute": "input-send-event",
+     "arguments": { "events": [
         { "type": "key", "data" : { "down": true,
           "key": {"type": "qcode", "data": "ctrl" } } },
         { "type": "key", "data" : { "down": true,
@@ -4135,10 +4715,10 @@ Example (3):
 
 Move mouse pointer to absolute coordinates (20000, 400).
 
--> { "execute": "x-input-send-event" ,
-  "arguments": { "console": 0, "events": [
-               { "type": "abs", "data" : { "axis": "X", "value" : 20000 } },
-               { "type": "abs", "data" : { "axis": "Y", "value" : 400 } } ] } }
+-> { "execute": "input-send-event" ,
+  "arguments": { "events": [
+               { "type": "abs", "data" : { "axis": "x", "value" : 20000 } },
+               { "type": "abs", "data" : { "axis": "y", "value" : 400 } } ] } }
 <- { "return": {} }
 
 EQMP
@@ -4146,7 +4726,7 @@ EQMP
     {
         .name       = "block-set-write-threshold",
         .args_type  = "node-name:s,write-threshold:l",
-        .mhandler.cmd_new = qmp_marshal_input_block_set_write_threshold,
+        .mhandler.cmd_new = qmp_marshal_block_set_write_threshold,
     },
 
 SQMP
@@ -4174,7 +4754,7 @@ EQMP
     {
         .name       = "query-rocker",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker,
+        .mhandler.cmd_new = qmp_marshal_query_rocker,
     },
 
 SQMP
@@ -4195,7 +4775,7 @@ EQMP
     {
         .name       = "query-rocker-ports",
         .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_ports,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_ports,
     },
 
 SQMP
@@ -4220,7 +4800,7 @@ EQMP
     {
         .name       = "query-rocker-of-dpa-flows",
         .args_type  = "name:s,tbl-id:i?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_of_dpa_flows,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_flows,
     },
 
 SQMP
@@ -4249,7 +4829,7 @@ EQMP
     {
         .name       = "query-rocker-of-dpa-groups",
         .args_type  = "name:s,type:i?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rocker_of_dpa_groups,
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_groups,
     },
 
 SQMP
@@ -4273,3 +4853,30 @@ Example:
                  {"type": 0, "out-pport": 0, "pport": 0, "vlan-id": 3840,
                   "pop-vlan": 1, "id": 251658240}
    ]}
+
+EQMP
+
+#if defined TARGET_ARM
+    {
+        .name       = "query-gic-capabilities",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_query_gic_capabilities,
+    },
+#endif
+
+SQMP
+query-gic-capabilities
+---------------
+
+Return a list of GICCapability objects, describing supported GIC
+(Generic Interrupt Controller) versions.
+
+Arguments: None
+
+Example:
+
+-> { "execute": "query-gic-capabilities" }
+<- { "return": [{ "version": 2, "emulated": true, "kernel": false },
+                { "version": 3, "emulated": false, "kernel": true } ] }
+
+EQMP
diff --git a/qemu/qmp.c b/qemu/qmp.c
index 403805a55..9d0953bc2 100644
--- a/qemu/qmp.c
+++ b/qemu/qmp.c
@@ -13,7 +13,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
@@ -24,6 +25,7 @@
 #include "sysemu/arch_init.h"
 #include "hw/qdev.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qom/qom-qobject.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qobject.h"
@@ -49,14 +51,20 @@ VersionInfo *qmp_query_version(Error **errp)
 {
     VersionInfo *info = g_new0(VersionInfo, 1);
     const char *version = QEMU_VERSION;
-    char *tmp;
+    const char *tmp;
+    int err;
 
     info->qemu = g_new0(VersionTriple, 1);
-    info->qemu->major = strtol(version, &tmp, 10);
+    err = qemu_strtoll(version, &tmp, 10, &info->qemu->major);
+    assert(err == 0);
     tmp++;
-    info->qemu->minor = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->minor);
+    assert(err == 0);
     tmp++;
-    info->qemu->micro = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->micro);
+    assert(err == 0);
     info->package = g_strdup(QEMU_PKGVERSION);
 
     return info;
@@ -95,6 +103,13 @@ void qmp_quit(Error **errp)
 
 void qmp_stop(Error **errp)
 {
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
+
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         autostart = 0;
     } else {
@@ -151,9 +166,9 @@ VncInfo2List *qmp_query_vnc_servers(Error **errp)
  * #ifdef CONFIG_SPICE.  Necessary for an accurate query-commands
  * result.  However, the QAPI schema is blissfully unaware of that,
  * and the QAPI code generator happily generates a dead
- * qmp_marshal_input_query_spice() that calls qmp_query_spice().
- * Provide it one, or else linking fails.
- * FIXME Educate the QAPI schema on CONFIG_SPICE.
+ * qmp_marshal_query_spice() that calls qmp_query_spice().  Provide it
+ * one, or else linking fails.  FIXME Educate the QAPI schema on
+ * CONFIG_SPICE.
  */
 SpiceInfo *qmp_query_spice(Error **errp)
 {
@@ -164,8 +179,16 @@ SpiceInfo *qmp_query_spice(Error **errp)
 void qmp_cont(Error **errp)
 {
     Error *local_err = NULL;
+    BlockBackend *blk;
     BlockDriverState *bs;
 
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
+
     if (runstate_needs_reset()) {
         error_setg(errp, "Resetting the Virtual Machine is required");
         return;
@@ -173,8 +196,8 @@ void qmp_cont(Error **errp)
         return;
     }
 
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        bdrv_iostatus_reset(bs);
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        blk_iostatus_reset(blk);
     }
     for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
         bdrv_add_key(bs, NULL, &local_err);
@@ -184,6 +207,18 @@ void qmp_cont(Error **errp)
         }
     }
 
+    /* Continuing after completed migration. Images have been inactivated to
+     * allow the destination to take control. Need to get control back now. */
+    if (runstate_check(RUN_STATE_FINISH_MIGRATE) ||
+        runstate_check(RUN_STATE_POSTMIGRATE))
+    {
+        bdrv_invalidate_cache_all(&local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         autostart = 1;
     } else {
@@ -202,6 +237,7 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
     bool ambiguous = false;
     ObjectPropertyInfoList *props = NULL;
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
 
     obj = object_resolve_path(path, &ambiguous);
     if (obj == NULL) {
@@ -214,7 +250,8 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
         return NULL;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    object_property_iter_init(&iter, obj);
+    while ((prop = object_property_iter_next(&iter))) {
         ObjectPropertyInfoList *entry = g_malloc0(sizeof(*entry));
 
         entry->value = g_malloc0(sizeof(ObjectPropertyInfo));
@@ -228,12 +265,9 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp)
     return props;
 }
 
-/* FIXME: teach qapi about how to pass through Visitors */
-void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp)
+void qmp_qom_set(const char *path, const char *property, QObject *value,
+                 Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
-    QObject *value = qdict_get(qdict, "value");
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
@@ -246,20 +280,18 @@ void qmp_qom_set(QDict *qdict, QObject **ret, Error **errp)
     object_property_set_qobject(obj, value, property, errp);
 }
 
-void qmp_qom_get(QDict *qdict, QObject **ret, Error **errp)
+QObject *qmp_qom_get(const char *path, const char *property, Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
     if (!obj) {
         error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
                   "Device '%s' not found", path);
-        return;
+        return NULL;
     }
 
-    *ret = object_property_get_qobject(obj, property, errp);
+    return object_property_get_qobject(obj, property, errp);
 }
 
 void qmp_set_password(const char *protocol, const char *password,
@@ -411,7 +443,8 @@ void qmp_change(const char *device, const char *target,
     if (strcmp(device, "vnc") == 0) {
         qmp_change_vnc(target, has_arg, arg, errp);
     } else {
-        qmp_change_blockdev(device, target, arg, errp);
+        qmp_blockdev_change_medium(device, target, has_arg, arg, false, 0,
+                                   errp);
     }
 }
 
@@ -499,6 +532,7 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
     ObjectClass *klass;
     Object *obj;
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
     DevicePropertyInfoList *prop_list = NULL;
 
     klass = object_class_by_name(typename);
@@ -514,9 +548,21 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
         return NULL;
     }
 
+    if (object_class_is_abstract(klass)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "name",
+                   "non-abstract device type");
+        return NULL;
+    }
+
+    if (DEVICE_CLASS(klass)->cannot_destroy_with_object_finalize_yet) {
+        error_setg(errp, "Can't list properties of device '%s'", typename);
+        return NULL;
+    }
+
     obj = object_new(typename);
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    object_property_iter_init(&iter, obj);
+    while ((prop = object_property_iter_next(&iter))) {
         DevicePropertyInfo *info;
         DevicePropertyInfoList *entry;
 
@@ -601,67 +647,13 @@ void qmp_add_client(const char *protocol, const char *fdname,
     close(fd);
 }
 
-void object_add(const char *type, const char *id, const QDict *qdict,
-                Visitor *v, Error **errp)
-{
-    Object *obj;
-    ObjectClass *klass;
-    const QDictEntry *e;
-    Error *local_err = NULL;
-
-    klass = object_class_by_name(type);
-    if (!klass) {
-        error_setg(errp, "invalid object type: %s", type);
-        return;
-    }
-
-    if (!object_class_dynamic_cast(klass, TYPE_USER_CREATABLE)) {
-        error_setg(errp, "object type '%s' isn't supported by object-add",
-                   type);
-        return;
-    }
-
-    if (object_class_is_abstract(klass)) {
-        error_setg(errp, "object type '%s' is abstract", type);
-        return;
-    }
-
-    obj = object_new(type);
-    if (qdict) {
-        for (e = qdict_first(qdict); e; e = qdict_next(qdict, e)) {
-            object_property_set(obj, v, e->key, &local_err);
-            if (local_err) {
-                goto out;
-            }
-        }
-    }
-
-    object_property_add_child(object_get_objects_root(),
-                              id, obj, &local_err);
-    if (local_err) {
-        goto out;
-    }
-
-    user_creatable_complete(obj, &local_err);
-    if (local_err) {
-        object_property_del(object_get_objects_root(),
-                            id, &error_abort);
-        goto out;
-    }
-out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    object_unref(obj);
-}
 
-void qmp_object_add(QDict *qdict, QObject **ret, Error **errp)
+void qmp_object_add(const char *type, const char *id,
+                    bool has_props, QObject *props, Error **errp)
 {
-    const char *type = qdict_get_str(qdict, "qom-type");
-    const char *id = qdict_get_str(qdict, "id");
-    QObject *props = qdict_get(qdict, "props");
     const QDict *pdict = NULL;
     QmpInputVisitor *qiv;
+    Object *obj;
 
     if (props) {
         pdict = qobject_to_qdict(props);
@@ -672,27 +664,17 @@ void qmp_object_add(QDict *qdict, QObject **ret, Error **errp)
     }
 
     qiv = qmp_input_visitor_new(props);
-    object_add(type, id, pdict, qmp_input_get_visitor(qiv), errp);
+    obj = user_creatable_add_type(type, id, pdict,
+                                  qmp_input_get_visitor(qiv), errp);
     qmp_input_visitor_cleanup(qiv);
+    if (obj) {
+        object_unref(obj);
+    }
 }
 
 void qmp_object_del(const char *id, Error **errp)
 {
-    Object *container;
-    Object *obj;
-
-    container = object_get_objects_root();
-    obj = object_resolve_path_component(container, id);
-    if (!obj) {
-        error_setg(errp, "object id not found");
-        return;
-    }
-
-    if (!user_creatable_can_be_deleted(USER_CREATABLE(obj), errp)) {
-        error_setg(errp, "%s is in use, can not be deleted", id);
-        return;
-    }
-    object_unparent(obj);
+    user_creatable_del(id, errp);
 }
 
 MemoryDeviceInfoList *qmp_query_memory_devices(Error **errp)
diff --git a/qemu/qobject/Makefile.objs b/qemu/qobject/Makefile.objs
index 0031e8b69..bed55084b 100644
--- a/qemu/qobject/Makefile.objs
+++ b/qemu/qobject/Makefile.objs
@@ -1,2 +1,2 @@
 util-obj-y = qnull.o qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
-util-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
+util-obj-y += qjson.o qobject.o json-lexer.o json-streamer.o json-parser.o
diff --git a/qemu/qobject/json-lexer.c b/qemu/qobject/json-lexer.c
index b19623e22..496374d9a 100644
--- a/qemu/qobject/json-lexer.c
+++ b/qemu/qobject/json-lexer.c
@@ -11,10 +11,7 @@
  *
  */
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/qmp/qint.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
 
@@ -30,7 +27,7 @@
  */
 
 enum json_lexer_state {
-    IN_ERROR = 0,
+    IN_ERROR = 0,               /* must really be 0, see json_lexer[] */
     IN_DQ_UCODE3,
     IN_DQ_UCODE2,
     IN_DQ_UCODE1,
@@ -62,6 +59,8 @@ enum json_lexer_state {
     IN_START,
 };
 
+QEMU_BUILD_BUG_ON((int)JSON_MIN <= (int)IN_START);
+
 #define TERMINAL(state) [0 ... 0x7F] = (state)
 
 /* Return whether TERMINAL is a terminal state and the transition to it
@@ -71,6 +70,8 @@ enum json_lexer_state {
             (json_lexer[(old_state)][0] == (terminal))
 
 static const uint8_t json_lexer[][256] =  {
+    /* Relies on default initialization to IN_ERROR! */
+
     /* double quote string */
     [IN_DQ_UCODE3] = {
         ['0' ... '9'] = IN_DQ_STRING,
@@ -253,12 +254,12 @@ static const uint8_t json_lexer[][256] =  {
         ['0'] = IN_ZERO,
         ['1' ... '9'] = IN_NONZERO_NUMBER,
         ['-'] = IN_NEG_NONZERO_NUMBER,
-        ['{'] = JSON_OPERATOR,
-        ['}'] = JSON_OPERATOR,
-        ['['] = JSON_OPERATOR,
-        [']'] = JSON_OPERATOR,
-        [','] = JSON_OPERATOR,
-        [':'] = JSON_OPERATOR,
+        ['{'] = JSON_LCURLY,
+        ['}'] = JSON_RCURLY,
+        ['['] = JSON_LSQUARE,
+        [']'] = JSON_RSQUARE,
+        [','] = JSON_COMMA,
+        [':'] = JSON_COLON,
         ['a' ... 'z'] = IN_KEYWORD,
         ['%'] = IN_ESCAPE,
         [' '] = IN_WHITESPACE,
@@ -272,7 +273,7 @@ void json_lexer_init(JSONLexer *lexer, JSONLexerEmitter func)
 {
     lexer->emit = func;
     lexer->state = IN_START;
-    lexer->token = qstring_new();
+    lexer->token = g_string_sized_new(3);
     lexer->x = lexer->y = 0;
 }
 
@@ -287,14 +288,20 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
     }
 
     do {
+        assert(lexer->state <= ARRAY_SIZE(json_lexer));
         new_state = json_lexer[lexer->state][(uint8_t)ch];
         char_consumed = !TERMINAL_NEEDED_LOOKAHEAD(lexer->state, new_state);
         if (char_consumed) {
-            qstring_append_chr(lexer->token, ch);
+            g_string_append_c(lexer->token, ch);
         }
 
         switch (new_state) {
-        case JSON_OPERATOR:
+        case JSON_LCURLY:
+        case JSON_RCURLY:
+        case JSON_LSQUARE:
+        case JSON_RSQUARE:
+        case JSON_COLON:
+        case JSON_COMMA:
         case JSON_ESCAPE:
         case JSON_INTEGER:
         case JSON_FLOAT:
@@ -303,8 +310,7 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
             lexer->emit(lexer, lexer->token, new_state, lexer->x, lexer->y);
             /* fall through */
         case JSON_SKIP:
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             break;
         case IN_ERROR:
@@ -322,8 +328,7 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
              * induce an error/flush state.
              */
             lexer->emit(lexer, lexer->token, JSON_ERROR, lexer->x, lexer->y);
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             lexer->state = new_state;
             return 0;
@@ -336,10 +341,9 @@ static int json_lexer_feed_char(JSONLexer *lexer, char ch, bool flush)
     /* Do not let a single token grow to an arbitrarily large size,
      * this is a security consideration.
      */
-    if (lexer->token->length > MAX_TOKEN_SIZE) {
+    if (lexer->token->len > MAX_TOKEN_SIZE) {
         lexer->emit(lexer, lexer->token, lexer->state, lexer->x, lexer->y);
-        QDECREF(lexer->token);
-        lexer->token = qstring_new();
+        g_string_truncate(lexer->token, 0);
         lexer->state = IN_START;
     }
 
@@ -369,5 +373,5 @@ int json_lexer_flush(JSONLexer *lexer)
 
 void json_lexer_destroy(JSONLexer *lexer)
 {
-    QDECREF(lexer->token);
+    g_string_free(lexer->token, true);
 }
diff --git a/qemu/qobject/json-parser.c b/qemu/qobject/json-parser.c
index ac991ba3c..67ed72731 100644
--- a/qemu/qobject/json-parser.c
+++ b/qemu/qobject/json-parser.c
@@ -1,5 +1,5 @@
 /*
- * JSON Parser 
+ * JSON Parser
  *
  * Copyright IBM, Corp. 2009
  *
@@ -11,8 +11,8 @@
  *
  */
 
-#include <stdarg.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qint.h"
@@ -22,15 +22,13 @@
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-lexer.h"
+#include "qapi/qmp/json-streamer.h"
 
 typedef struct JSONParserContext
 {
     Error *err;
-    struct {
-        QObject **buf;
-        size_t pos;
-        size_t count;
-    } tokens;
+    JSONToken *current;
+    GQueue *buf;
 } JSONParserContext;
 
 #define BUG_ON(cond) assert(!(cond))
@@ -47,58 +45,10 @@ typedef struct JSONParserContext
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
 
 /**
- * Token manipulators
- *
- * tokens are dictionaries that contain a type, a string value, and geometry information
- * about a token identified by the lexer.  These are routines that make working with
- * these objects a bit easier.
- */
-static const char *token_get_value(QObject *obj)
-{
-    return qdict_get_str(qobject_to_qdict(obj), "token");
-}
-
-static JSONTokenType token_get_type(QObject *obj)
-{
-    return qdict_get_int(qobject_to_qdict(obj), "type");
-}
-
-static int token_is_operator(QObject *obj, char op)
-{
-    const char *val;
-
-    if (token_get_type(obj) != JSON_OPERATOR) {
-        return 0;
-    }
-
-    val = token_get_value(obj);
-
-    return (val[0] == op) && (val[1] == 0);
-}
-
-static int token_is_keyword(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_KEYWORD) {
-        return 0;
-    }
-
-    return strcmp(token_get_value(obj), value) == 0;
-}
-
-static int token_is_escape(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_ESCAPE) {
-        return 0;
-    }
-
-    return (strcmp(token_get_value(obj), value) == 0);
-}
-
-/**
  * Error handler
  */
 static void GCC_FMT_ATTR(3, 4) parse_error(JSONParserContext *ctxt,
-                                           QObject *token, const char *msg, ...)
+                                           JSONToken *token, const char *msg, ...)
 {
     va_list ap;
     char message[1024];
@@ -176,9 +126,10 @@ static int hex2decimal(char ch)
  *      \t
  *      \u four-hex-digits 
  */
-static QString *qstring_from_escaped_str(JSONParserContext *ctxt, QObject *token)
+static QString *qstring_from_escaped_str(JSONParserContext *ctxt,
+                                         JSONToken *token)
 {
-    const char *ptr = token_get_value(token);
+    const char *ptr = token->str;
     QString *str;
     int double_quote = 1;
 
@@ -274,73 +225,34 @@ out:
     return NULL;
 }
 
-static QObject *parser_context_pop_token(JSONParserContext *ctxt)
-{
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    ctxt->tokens.pos++;
-    return token;
-}
-
-/* Note: parser_context_{peek|pop}_token do not increment the
- * token object's refcount. In both cases the references will continue
- * to be tracked and cleaned up in parser_context_free(), so do not
- * attempt to free the token object.
+/* Note: the token object returned by parser_context_peek_token or
+ * parser_context_pop_token is deleted as soon as parser_context_pop_token
+ * is called again.
  */
-static QObject *parser_context_peek_token(JSONParserContext *ctxt)
-{
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    return token;
-}
-
-static JSONParserContext parser_context_save(JSONParserContext *ctxt)
-{
-    JSONParserContext saved_ctxt = {0};
-    saved_ctxt.tokens.pos = ctxt->tokens.pos;
-    saved_ctxt.tokens.count = ctxt->tokens.count;
-    saved_ctxt.tokens.buf = ctxt->tokens.buf;
-    return saved_ctxt;
-}
-
-static void parser_context_restore(JSONParserContext *ctxt,
-                                   JSONParserContext saved_ctxt)
+static JSONToken *parser_context_pop_token(JSONParserContext *ctxt)
 {
-    ctxt->tokens.pos = saved_ctxt.tokens.pos;
-    ctxt->tokens.count = saved_ctxt.tokens.count;
-    ctxt->tokens.buf = saved_ctxt.tokens.buf;
+    g_free(ctxt->current);
+    assert(!g_queue_is_empty(ctxt->buf));
+    ctxt->current = g_queue_pop_head(ctxt->buf);
+    return ctxt->current;
 }
 
-static void tokens_append_from_iter(QObject *obj, void *opaque)
+static JSONToken *parser_context_peek_token(JSONParserContext *ctxt)
 {
-    JSONParserContext *ctxt = opaque;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
-    qobject_incref(obj);
+    assert(!g_queue_is_empty(ctxt->buf));
+    return g_queue_peek_head(ctxt->buf);
 }
 
-static JSONParserContext *parser_context_new(QList *tokens)
+static JSONParserContext *parser_context_new(GQueue *tokens)
 {
     JSONParserContext *ctxt;
-    size_t count;
 
     if (!tokens) {
         return NULL;
     }
 
-    count = qlist_size(tokens);
-    if (count == 0) {
-        return NULL;
-    }
-
     ctxt = g_malloc0(sizeof(JSONParserContext));
-    ctxt->tokens.pos = 0;
-    ctxt->tokens.count = count;
-    ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
-    qlist_iter(tokens, tokens_append_from_iter, ctxt);
-    ctxt->tokens.pos = 0;
+    ctxt->buf = tokens;
 
     return ctxt;
 }
@@ -348,12 +260,12 @@ static JSONParserContext *parser_context_new(QList *tokens)
 /* to support error propagation, ctxt->err must be freed separately */
 static void parser_context_free(JSONParserContext *ctxt)
 {
-    int i;
     if (ctxt) {
-        for (i = 0; i < ctxt->tokens.count; i++) {
-            qobject_decref(ctxt->tokens.buf[i]);
+        while (!g_queue_is_empty(ctxt->buf)) {
+            parser_context_pop_token(ctxt);
         }
-        g_free(ctxt->tokens.buf);
+        g_free(ctxt->current);
+        g_queue_free(ctxt->buf);
         g_free(ctxt);
     }
 }
@@ -363,8 +275,8 @@ static void parser_context_free(JSONParserContext *ctxt)
  */
 static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
 {
-    QObject *key = NULL, *token = NULL, *value, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    QObject *key = NULL, *value;
+    JSONToken *peek, *token;
 
     peek = parser_context_peek_token(ctxt);
     if (peek == NULL) {
@@ -384,7 +296,7 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(token, ':')) {
+    if (token->type != JSON_COLON) {
         parse_error(ctxt, token, "missing : in object pair");
         goto out;
     }
@@ -402,7 +314,6 @@ static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
     return 0;
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     qobject_decref(key);
 
     return -1;
@@ -411,17 +322,10 @@ out:
 static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
 {
     QDict *dict = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '{')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LCURLY);
 
     dict = qdict_new();
 
@@ -431,7 +335,7 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(peek, '}')) {
+    if (peek->type != JSON_RCURLY) {
         if (parse_pair(ctxt, dict, ap) == -1) {
             goto out;
         }
@@ -442,8 +346,8 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
             goto out;
         }
 
-        while (!token_is_operator(token, '}')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RCURLY) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in dict");
                 goto out;
             }
@@ -465,7 +369,6 @@ static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
     return QOBJECT(dict);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(dict);
     return NULL;
 }
@@ -473,17 +376,10 @@ out:
 static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
 {
     QList *list = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '[')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LSQUARE);
 
     list = qlist_new();
 
@@ -493,7 +389,7 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
         goto out;
     }
 
-    if (!token_is_operator(peek, ']')) {
+    if (peek->type != JSON_RSQUARE) {
         QObject *obj;
 
         obj = parse_value(ctxt, ap);
@@ -510,8 +406,8 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
             goto out;
         }
 
-        while (!token_is_operator(token, ']')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RSQUARE) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in list");
                 goto out;
             }
@@ -537,99 +433,68 @@ static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
     return QOBJECT(list);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(list);
     return NULL;
 }
 
 static QObject *parse_keyword(JSONParserContext *ctxt)
 {
-    QObject *token, *ret;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (token_get_type(token) != JSON_KEYWORD) {
-        goto out;
-    }
+    assert(token && token->type == JSON_KEYWORD);
 
-    if (token_is_keyword(token, "true")) {
-        ret = QOBJECT(qbool_from_bool(true));
-    } else if (token_is_keyword(token, "false")) {
-        ret = QOBJECT(qbool_from_bool(false));
-    } else if (token_is_keyword(token, "null")) {
-        ret = qnull();
-    } else {
-        parse_error(ctxt, token, "invalid keyword `%s'", token_get_value(token));
-        goto out;
+    if (!strcmp(token->str, "true")) {
+        return QOBJECT(qbool_from_bool(true));
+    } else if (!strcmp(token->str, "false")) {
+        return QOBJECT(qbool_from_bool(false));
+    } else if (!strcmp(token->str, "null")) {
+        return qnull();
     }
-
-    return ret;
-
-out: 
-    parser_context_restore(ctxt, saved_ctxt);
-
+    parse_error(ctxt, token, "invalid keyword '%s'", token->str);
     return NULL;
 }
 
 static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *token = NULL, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     if (ap == NULL) {
-        goto out;
+        return NULL;
     }
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (token_is_escape(token, "%p")) {
-        obj = va_arg(*ap, QObject *);
-    } else if (token_is_escape(token, "%i")) {
-        obj = QOBJECT(qbool_from_bool(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%ld")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long)));
-    } else if (token_is_escape(token, "%lld") ||
-               token_is_escape(token, "%I64d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long long)));
-    } else if (token_is_escape(token, "%s")) {
-        obj = QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
-    } else if (token_is_escape(token, "%f")) {
-        obj = QOBJECT(qfloat_from_double(va_arg(*ap, double)));
-    } else {
-        goto out;
+    assert(token && token->type == JSON_ESCAPE);
+
+    if (!strcmp(token->str, "%p")) {
+        return va_arg(*ap, QObject *);
+    } else if (!strcmp(token->str, "%i")) {
+        return QOBJECT(qbool_from_bool(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%ld")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long)));
+    } else if (!strcmp(token->str, "%lld") ||
+               !strcmp(token->str, "%I64d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long long)));
+    } else if (!strcmp(token->str, "%s")) {
+        return QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
+    } else if (!strcmp(token->str, "%f")) {
+        return QOBJECT(qfloat_from_double(va_arg(*ap, double)));
     }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
     return NULL;
 }
 
 static QObject *parse_literal(JSONParserContext *ctxt)
 {
-    QObject *token, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
+    assert(token);
 
-    switch (token_get_type(token)) {
+    switch (token->type) {
     case JSON_STRING:
-        obj = QOBJECT(qstring_from_escaped_str(ctxt, token));
-        break;
+        return QOBJECT(qstring_from_escaped_str(ctxt, token));
     case JSON_INTEGER: {
         /* A possibility exists that this is a whole-valued float where the
          * fractional part was left out due to being 0 (.0). It's not a big
@@ -646,56 +511,57 @@ static QObject *parse_literal(JSONParserContext *ctxt)
         int64_t value;
 
         errno = 0; /* strtoll doesn't set errno on success */
-        value = strtoll(token_get_value(token), NULL, 10);
+        value = strtoll(token->str, NULL, 10);
         if (errno != ERANGE) {
-            obj = QOBJECT(qint_from_int(value));
-            break;
+            return QOBJECT(qint_from_int(value));
         }
         /* fall through to JSON_FLOAT */
     }
     case JSON_FLOAT:
-        /* FIXME dependent on locale */
-        obj = QOBJECT(qfloat_from_double(strtod(token_get_value(token), NULL)));
-        break;
+        /* FIXME dependent on locale; a pervasive issue in QEMU */
+        /* FIXME our lexer matches RFC 7159 in forbidding Inf or NaN,
+         * but those might be useful extensions beyond JSON */
+        return QOBJECT(qfloat_from_double(strtod(token->str, NULL)));
     default:
-        goto out;
+        abort();
     }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
-    return NULL;
 }
 
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *obj;
+    JSONToken *token;
 
-    obj = parse_object(ctxt, ap);
-    if (obj == NULL) {
-        obj = parse_array(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_escape(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_keyword(ctxt);
-    } 
-    if (obj == NULL) {
-        obj = parse_literal(ctxt);
+    token = parser_context_peek_token(ctxt);
+    if (token == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        return NULL;
     }
 
-    return obj;
+    switch (token->type) {
+    case JSON_LCURLY:
+        return parse_object(ctxt, ap);
+    case JSON_LSQUARE:
+        return parse_array(ctxt, ap);
+    case JSON_ESCAPE:
+        return parse_escape(ctxt, ap);
+    case JSON_INTEGER:
+    case JSON_FLOAT:
+    case JSON_STRING:
+        return parse_literal(ctxt);
+    case JSON_KEYWORD:
+        return parse_keyword(ctxt);
+    default:
+        parse_error(ctxt, token, "expecting value");
+        return NULL;
+    }
 }
 
-QObject *json_parser_parse(QList *tokens, va_list *ap)
+QObject *json_parser_parse(GQueue *tokens, va_list *ap)
 {
     return json_parser_parse_err(tokens, ap, NULL);
 }
 
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp)
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp)
 {
     JSONParserContext *ctxt = parser_context_new(tokens);
     QObject *result;
diff --git a/qemu/qobject/json-streamer.c b/qemu/qobject/json-streamer.c
index 1b2f9b1d1..02516853a 100644
--- a/qemu/qobject/json-streamer.c
+++ b/qemu/qobject/json-streamer.c
@@ -11,50 +11,56 @@
  *
  */
 
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qdict.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
+#define MAX_TOKEN_COUNT (2ULL << 20)
 #define MAX_NESTING (1ULL << 10)
 
-static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTokenType type, int x, int y)
+static void json_message_free_tokens(JSONMessageParser *parser)
+{
+    if (parser->tokens) {
+        g_queue_free(parser->tokens);
+        parser->tokens = NULL;
+    }
+}
+
+static void json_message_process_token(JSONLexer *lexer, GString *input,
+                                       JSONTokenType type, int x, int y)
 {
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
-    QDict *dict;
-
-    if (type == JSON_OPERATOR) {
-        switch (qstring_get_str(token)[0]) {
-        case '{':
-            parser->brace_count++;
-            break;
-        case '}':
-            parser->brace_count--;
-            break;
-        case '[':
-            parser->bracket_count++;
-            break;
-        case ']':
-            parser->bracket_count--;
-            break;
-        default:
-            break;
-        }
+    JSONToken *token;
+
+    switch (type) {
+    case JSON_LCURLY:
+        parser->brace_count++;
+        break;
+    case JSON_RCURLY:
+        parser->brace_count--;
+        break;
+    case JSON_LSQUARE:
+        parser->bracket_count++;
+        break;
+    case JSON_RSQUARE:
+        parser->bracket_count--;
+        break;
+    default:
+        break;
     }
 
-    dict = qdict_new();
-    qdict_put(dict, "type", qint_from_int(type));
-    QINCREF(token);
-    qdict_put(dict, "token", token);
-    qdict_put(dict, "x", qint_from_int(x));
-    qdict_put(dict, "y", qint_from_int(y));
+    token = g_malloc(sizeof(JSONToken) + input->len + 1);
+    token->type = type;
+    memcpy(token->str, input->str, input->len);
+    token->str[input->len] = 0;
+    token->x = x;
+    token->y = y;
 
-    parser->token_size += token->length;
+    parser->token_size += input->len;
 
-    qlist_append(parser->tokens, dict);
+    g_queue_push_tail(parser->tokens, token);
 
     if (type == JSON_ERROR) {
         goto out_emit_bad;
@@ -64,41 +70,39 @@ static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTok
          parser->bracket_count == 0)) {
         goto out_emit;
     } else if (parser->token_size > MAX_TOKEN_SIZE ||
-               parser->bracket_count > MAX_NESTING ||
-               parser->brace_count > MAX_NESTING) {
+               g_queue_get_length(parser->tokens) > MAX_TOKEN_COUNT ||
+               parser->bracket_count + parser->brace_count > MAX_NESTING) {
         /* Security consideration, we limit total memory allocated per object
          * and the maximum recursion depth that a message can force.
          */
-        goto out_emit;
+        goto out_emit_bad;
     }
 
     return;
 
 out_emit_bad:
-    /* clear out token list and tell the parser to emit and error
+    /*
+     * Clear out token list and tell the parser to emit an error
      * indication by passing it a NULL list
      */
-    QDECREF(parser->tokens);
-    parser->tokens = NULL;
+    json_message_free_tokens(parser);
 out_emit:
     /* send current list of tokens to parser and reset tokenizer */
     parser->brace_count = 0;
     parser->bracket_count = 0;
+    /* parser->emit takes ownership of parser->tokens.  */
     parser->emit(parser, parser->tokens);
-    if (parser->tokens) {
-        QDECREF(parser->tokens);
-    }
-    parser->tokens = qlist_new();
+    parser->tokens = g_queue_new();
     parser->token_size = 0;
 }
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *))
+                              void (*func)(JSONMessageParser *, GQueue *))
 {
     parser->emit = func;
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    parser->tokens = qlist_new();
+    parser->tokens = g_queue_new();
     parser->token_size = 0;
 
     json_lexer_init(&parser->lexer, json_message_process_token);
@@ -118,5 +122,5 @@ int json_message_parser_flush(JSONMessageParser *parser)
 void json_message_parser_destroy(JSONMessageParser *parser)
 {
     json_lexer_destroy(&parser->lexer);
-    QDECREF(parser->tokens);
+    json_message_free_tokens(parser);
 }
diff --git a/qemu/qobject/qbool.c b/qemu/qobject/qbool.c
index 5ff69f0b2..0606bbd2a 100644
--- a/qemu/qobject/qbool.c
+++ b/qemu/qobject/qbool.c
@@ -11,17 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qbool_destroy_obj(QObject *obj);
-
-static const QType qbool_type = {
-    .code = QTYPE_QBOOL,
-    .destroy = qbool_destroy_obj,
-};
-
 /**
  * qbool_from_bool(): Create a new QBool from a bool
  *
@@ -32,8 +26,8 @@ QBool *qbool_from_bool(bool value)
     QBool *qb;
 
     qb = g_malloc(sizeof(*qb));
+    qobject_init(QOBJECT(qb), QTYPE_QBOOL);
     qb->value = value;
-    QOBJECT_INIT(qb, &qbool_type);
 
     return qb;
 }
@@ -51,9 +45,9 @@ bool qbool_get_bool(const QBool *qb)
  */
 QBool *qobject_to_qbool(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QBOOL)
+    if (!obj || qobject_type(obj) != QTYPE_QBOOL) {
         return NULL;
-
+    }
     return container_of(obj, QBool, base);
 }
 
@@ -61,7 +55,7 @@ QBool *qobject_to_qbool(const QObject *obj)
  * qbool_destroy_obj(): Free all memory allocated by a
  * QBool object
  */
-static void qbool_destroy_obj(QObject *obj)
+void qbool_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qbool(obj));
diff --git a/qemu/qobject/qdict.c b/qemu/qobject/qdict.c
index 67b1a58ab..a1285361c 100644
--- a/qemu/qobject/qdict.c
+++ b/qemu/qobject/qdict.c
@@ -10,6 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qfloat.h"
 #include "qapi/qmp/qdict.h"
@@ -18,13 +19,7 @@
 #include "qapi/qmp/qobject.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
-
-static void qdict_destroy_obj(QObject *obj);
-
-static const QType qdict_type = {
-    .code = QTYPE_QDICT,
-    .destroy = qdict_destroy_obj,
-};
+#include "qemu/cutils.h"
 
 /**
  * qdict_new(): Create a new QDict
@@ -36,7 +31,7 @@ QDict *qdict_new(void)
     QDict *qdict;
 
     qdict = g_malloc0(sizeof(*qdict));
-    QOBJECT_INIT(qdict, &qdict_type);
+    qobject_init(QOBJECT(qdict), QTYPE_QDICT);
 
     return qdict;
 }
@@ -46,9 +41,9 @@ QDict *qdict_new(void)
  */
 QDict *qobject_to_qdict(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QDICT)
+    if (!obj || qobject_type(obj) != QTYPE_QDICT) {
         return NULL;
-
+    }
     return container_of(obj, QDict, base);
 }
 
@@ -184,8 +179,7 @@ size_t qdict_size(const QDict *qdict)
 /**
  * qdict_get_obj(): Get a QObject of a specific type
  */
-static QObject *qdict_get_obj(const QDict *qdict, const char *key,
-                              qtype_code type)
+static QObject *qdict_get_obj(const QDict *qdict, const char *key, QType type)
 {
     QObject *obj;
 
@@ -229,8 +223,7 @@ double qdict_get_double(const QDict *qdict, const char *key)
  */
 int64_t qdict_get_int(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QINT);
-    return qint_get_int(qobject_to_qint(obj));
+    return qint_get_int(qobject_to_qint(qdict_get(qdict, key)));
 }
 
 /**
@@ -243,8 +236,7 @@ int64_t qdict_get_int(const QDict *qdict, const char *key)
  */
 bool qdict_get_bool(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QBOOL);
-    return qbool_get_bool(qobject_to_qbool(obj));
+    return qbool_get_bool(qobject_to_qbool(qdict_get(qdict, key)));
 }
 
 /**
@@ -270,7 +262,7 @@ QList *qdict_get_qlist(const QDict *qdict, const char *key)
  */
 QDict *qdict_get_qdict(const QDict *qdict, const char *key)
 {
-    return qobject_to_qdict(qdict_get_obj(qdict, key, QTYPE_QDICT));
+    return qobject_to_qdict(qdict_get(qdict, key));
 }
 
 /**
@@ -284,8 +276,7 @@ QDict *qdict_get_qdict(const QDict *qdict, const char *key)
  */
 const char *qdict_get_str(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QSTRING);
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstring_get_str(qobject_to_qstring(qdict_get(qdict, key)));
 }
 
 /**
@@ -298,13 +289,9 @@ const char *qdict_get_str(const QDict *qdict, const char *key)
 int64_t qdict_get_try_int(const QDict *qdict, const char *key,
                           int64_t def_value)
 {
-    QObject *obj;
-
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QINT)
-        return def_value;
+    QInt *qint = qobject_to_qint(qdict_get(qdict, key));
 
-    return qint_get_int(qobject_to_qint(obj));
+    return qint ? qint_get_int(qint) : def_value;
 }
 
 /**
@@ -316,13 +303,9 @@ int64_t qdict_get_try_int(const QDict *qdict, const char *key,
  */
 bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value)
 {
-    QObject *obj;
-
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QBOOL)
-        return def_value;
+    QBool *qbool = qobject_to_qbool(qdict_get(qdict, key));
 
-    return qbool_get_bool(qobject_to_qbool(obj));
+    return qbool ? qbool_get_bool(qbool) : def_value;
 }
 
 /**
@@ -335,13 +318,9 @@ bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value)
  */
 const char *qdict_get_try_str(const QDict *qdict, const char *key)
 {
-    QObject *obj;
-
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QSTRING)
-        return NULL;
+    QString *qstr = qobject_to_qstring(qdict_get(qdict, key));
 
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstr ? qstring_get_str(qstr) : NULL;
 }
 
 /**
@@ -456,7 +435,7 @@ void qdict_del(QDict *qdict, const char *key)
 /**
  * qdict_destroy_obj(): Free all the memory allocated by a QDict
  */
-static void qdict_destroy_obj(QObject *obj)
+void qdict_destroy_obj(QObject *obj)
 {
     int i;
     QDict *qdict;
diff --git a/qemu/qobject/qfloat.c b/qemu/qobject/qfloat.c
index 7de0992db..d5da84770 100644
--- a/qemu/qobject/qfloat.c
+++ b/qemu/qobject/qfloat.c
@@ -11,17 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qfloat.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qfloat_destroy_obj(QObject *obj);
-
-static const QType qfloat_type = {
-    .code = QTYPE_QFLOAT,
-    .destroy = qfloat_destroy_obj,
-};
-
 /**
  * qfloat_from_int(): Create a new QFloat from a float
  *
@@ -32,8 +26,8 @@ QFloat *qfloat_from_double(double value)
     QFloat *qf;
 
     qf = g_malloc(sizeof(*qf));
+    qobject_init(QOBJECT(qf), QTYPE_QFLOAT);
     qf->value = value;
-    QOBJECT_INIT(qf, &qfloat_type);
 
     return qf;
 }
@@ -51,9 +45,9 @@ double qfloat_get_double(const QFloat *qf)
  */
 QFloat *qobject_to_qfloat(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QFLOAT)
+    if (!obj || qobject_type(obj) != QTYPE_QFLOAT) {
         return NULL;
-
+    }
     return container_of(obj, QFloat, base);
 }
 
@@ -61,7 +55,7 @@ QFloat *qobject_to_qfloat(const QObject *obj)
  * qfloat_destroy_obj(): Free all memory allocated by a
  * QFloat object
  */
-static void qfloat_destroy_obj(QObject *obj)
+void qfloat_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qfloat(obj));
diff --git a/qemu/qobject/qint.c b/qemu/qobject/qint.c
index 86b9b04f0..d7d1b3021 100644
--- a/qemu/qobject/qint.c
+++ b/qemu/qobject/qint.c
@@ -10,17 +10,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qint_destroy_obj(QObject *obj);
-
-static const QType qint_type = {
-    .code = QTYPE_QINT,
-    .destroy = qint_destroy_obj,
-};
-
 /**
  * qint_from_int(): Create a new QInt from an int64_t
  *
@@ -31,8 +25,8 @@ QInt *qint_from_int(int64_t value)
     QInt *qi;
 
     qi = g_malloc(sizeof(*qi));
+    qobject_init(QOBJECT(qi), QTYPE_QINT);
     qi->value = value;
-    QOBJECT_INIT(qi, &qint_type);
 
     return qi;
 }
@@ -50,9 +44,9 @@ int64_t qint_get_int(const QInt *qi)
  */
 QInt *qobject_to_qint(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QINT)
+    if (!obj || qobject_type(obj) != QTYPE_QINT) {
         return NULL;
-
+    }
     return container_of(obj, QInt, base);
 }
 
@@ -60,7 +54,7 @@ QInt *qobject_to_qint(const QObject *obj)
  * qint_destroy_obj(): Free all memory allocated by a
  * QInt object
  */
-static void qint_destroy_obj(QObject *obj)
+void qint_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qint(obj));
diff --git a/qemu/qobject/qjson.c b/qemu/qobject/qjson.c
index 33f8ef530..ef160d211 100644
--- a/qemu/qobject/qjson.c
+++ b/qemu/qobject/qjson.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
@@ -20,6 +21,7 @@
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qfloat.h"
 #include "qapi/qmp/qdict.h"
+#include "qemu/unicode.h"
 
 typedef struct JSONParsingState
 {
@@ -28,7 +30,7 @@ typedef struct JSONParsingState
     QObject *result;
 } JSONParsingState;
 
-static void parse_json(JSONMessageParser *parser, QList *tokens)
+static void parse_json(JSONMessageParser *parser, GQueue *tokens)
 {
     JSONParsingState *s = container_of(parser, JSONParsingState, parser);
     s->result = json_parser_parse(tokens, s->ap);
@@ -237,6 +239,15 @@ static void to_json(const QObject *obj, QString *str, int pretty, int indent)
         char buffer[1024];
         int len;
 
+        /* FIXME: snprintf() is locale dependent; but JSON requires
+         * numbers to be formatted as if in the C locale. Dependence
+         * on C locale is a pervasive issue in QEMU. */
+        /* FIXME: This risks printing Inf or NaN, which are not valid
+         * JSON values. */
+        /* FIXME: the default precision of 6 for %f often causes
+         * rounding errors; we should be using DBL_DECIMAL_DIG (17),
+         * and only rounding to a shorter number if the result would
+         * still produce the same floating point value.  */
         len = snprintf(buffer, sizeof(buffer), "%f", qfloat_get_double(val));
         while (len > 0 && buffer[len - 1] == '0') {
             len--;
@@ -247,7 +258,7 @@ static void to_json(const QObject *obj, QString *str, int pretty, int indent)
         } else {
             buffer[len] = 0;
         }
-        
+
         qstring_append(str, buffer);
         break;
     }
diff --git a/qemu/qobject/qlist.c b/qemu/qobject/qlist.c
index 1ced0de58..1ec74de2b 100644
--- a/qemu/qobject/qlist.c
+++ b/qemu/qobject/qlist.c
@@ -10,18 +10,12 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qlist.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
 
-static void qlist_destroy_obj(QObject *obj);
-
-static const QType qlist_type = {
-    .code = QTYPE_QLIST,
-    .destroy = qlist_destroy_obj,
-};
- 
 /**
  * qlist_new(): Create a new QList
  *
@@ -32,8 +26,8 @@ QList *qlist_new(void)
     QList *qlist;
 
     qlist = g_malloc(sizeof(*qlist));
+    qobject_init(QOBJECT(qlist), QTYPE_QLIST);
     QTAILQ_INIT(&qlist->head);
-    QOBJECT_INIT(qlist, &qlist_type);
 
     return qlist;
 }
@@ -142,17 +136,16 @@ size_t qlist_size(const QList *qlist)
  */
 QList *qobject_to_qlist(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QLIST) {
+    if (!obj || qobject_type(obj) != QTYPE_QLIST) {
         return NULL;
     }
-
     return container_of(obj, QList, base);
 }
 
 /**
  * qlist_destroy_obj(): Free all the memory allocated by a QList
  */
-static void qlist_destroy_obj(QObject *obj)
+void qlist_destroy_obj(QObject *obj)
 {
     QList *qlist;
     QListEntry *entry, *next_entry;
diff --git a/qemu/qobject/qnull.c b/qemu/qobject/qnull.c
index 9873e266e..c124d0585 100644
--- a/qemu/qobject/qnull.c
+++ b/qemu/qobject/qnull.c
@@ -10,20 +10,11 @@
  * or later.  See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/qobject.h"
 
-static void qnull_destroy_obj(QObject *obj)
-{
-    assert(0);
-}
-
-static const QType qnull_type = {
-    .code = QTYPE_QNULL,
-    .destroy = qnull_destroy_obj,
-};
-
 QObject qnull_ = {
-    .type = &qnull_type,
+    .type = QTYPE_QNULL,
     .refcnt = 1,
 };
diff --git a/qemu/qobject/qobject.c b/qemu/qobject/qobject.c
new file mode 100644
index 000000000..cd41fb940
--- /dev/null
+++ b/qemu/qobject/qobject.c
@@ -0,0 +1,35 @@
+/*
+ * QObject
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qfloat.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qlist.h"
+#include "qapi/qmp/qstring.h"
+
+static void (*qdestroy[QTYPE__MAX])(QObject *) = {
+    [QTYPE_NONE] = NULL,               /* No such object exists */
+    [QTYPE_QNULL] = NULL,              /* qnull_ is indestructible */
+    [QTYPE_QINT] = qint_destroy_obj,
+    [QTYPE_QSTRING] = qstring_destroy_obj,
+    [QTYPE_QDICT] = qdict_destroy_obj,
+    [QTYPE_QLIST] = qlist_destroy_obj,
+    [QTYPE_QFLOAT] = qfloat_destroy_obj,
+    [QTYPE_QBOOL] = qbool_destroy_obj,
+};
+
+void qobject_destroy(QObject *obj)
+{
+    assert(!obj->refcnt);
+    assert(QTYPE_QNULL < obj->type && obj->type < QTYPE__MAX);
+    qdestroy[obj->type](obj);
+}
diff --git a/qemu/qobject/qstring.c b/qemu/qobject/qstring.c
index 607b7a142..5da7b5f37 100644
--- a/qemu/qobject/qstring.c
+++ b/qemu/qobject/qstring.c
@@ -10,17 +10,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
-static void qstring_destroy_obj(QObject *obj);
-
-static const QType qstring_type = {
-    .code = QTYPE_QSTRING,
-    .destroy = qstring_destroy_obj,
-};
-
 /**
  * qstring_new(): Create a new empty QString
  *
@@ -49,6 +43,7 @@ QString *qstring_from_substr(const char *str, int start, int end)
     QString *qstring;
 
     qstring = g_malloc(sizeof(*qstring));
+    qobject_init(QOBJECT(qstring), QTYPE_QSTRING);
 
     qstring->length = end - start + 1;
     qstring->capacity = qstring->length;
@@ -57,7 +52,6 @@ QString *qstring_from_substr(const char *str, int start, int end)
     memcpy(qstring->string, str + start, qstring->length);
     qstring->string[qstring->length] = 0;
 
-    QOBJECT_INIT(qstring, &qstring_type);
 
     return qstring;
 }
@@ -117,9 +111,9 @@ void qstring_append_chr(QString *qstring, int c)
  */
 QString *qobject_to_qstring(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QSTRING)
+    if (!obj || qobject_type(obj) != QTYPE_QSTRING) {
         return NULL;
-
+    }
     return container_of(obj, QString, base);
 }
 
@@ -138,7 +132,7 @@ const char *qstring_get_str(const QString *qstring)
  * qstring_destroy_obj(): Free all memory allocated by a QString
  * object
  */
-static void qstring_destroy_obj(QObject *obj)
+void qstring_destroy_obj(QObject *obj)
 {
     QString *qs;
 
diff --git a/qemu/qom/Makefile.objs b/qemu/qom/Makefile.objs
index 985003bd0..516349eec 100644
--- a/qemu/qom/Makefile.objs
+++ b/qemu/qom/Makefile.objs
@@ -1,3 +1,4 @@
-common-obj-y = object.o container.o qom-qobject.o
-common-obj-y += cpu.o
-common-obj-y += object_interfaces.o
+qom-obj-y = object.o container.o qom-qobject.o
+qom-obj-y += object_interfaces.o
+
+common-obj-y = cpu.o
diff --git a/qemu/qom/container.c b/qemu/qom/container.c
index 62b1648ad..c9eb49b01 100644
--- a/qemu/qom/container.c
+++ b/qemu/qom/container.c
@@ -10,9 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "qemu/module.h"
-#include <assert.h>
 
 static const TypeInfo container_info = {
     .name          = "container",
diff --git a/qemu/qom/cpu.c b/qemu/qom/cpu.c
index eb9cfeca1..c9007d3d0 100644
--- a/qemu/qom/cpu.c
+++ b/qemu/qom/cpu.c
@@ -18,11 +18,14 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qom/cpu.h"
 #include "sysemu/kvm.h"
 #include "qemu/notify.h"
 #include "qemu/log.h"
+#include "exec/log.h"
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 
@@ -114,6 +117,8 @@ void cpu_reset_interrupt(CPUState *cpu, int mask)
 void cpu_exit(CPUState *cpu)
 {
     cpu->exit_request = 1;
+    /* Ensure cpu_exec will see the exit request after TCG has exited.  */
+    smp_wmb();
     cpu->tcg_exit_req = 1;
 }
 
@@ -128,7 +133,7 @@ int cpu_write_elf32_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
 static int cpu_common_write_elf32_qemunote(WriteCoreDumpFunction f,
                                            CPUState *cpu, void *opaque)
 {
-    return -1;
+    return 0;
 }
 
 int cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cpu,
@@ -157,7 +162,7 @@ int cpu_write_elf64_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
 static int cpu_common_write_elf64_qemunote(WriteCoreDumpFunction f,
                                            CPUState *cpu, void *opaque)
 {
-    return -1;
+    return 0;
 }
 
 int cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cpu,
@@ -186,6 +191,14 @@ static int cpu_common_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg)
     return 0;
 }
 
+static bool cpu_common_debug_check_watchpoint(CPUState *cpu, CPUWatchpoint *wp)
+{
+    /* If no extra check is required, QEMU watchpoint match can be considered
+     * as an architectural match.
+     */
+    return true;
+}
+
 bool target_words_bigendian(void);
 static bool cpu_common_virtio_is_big_endian(CPUState *cpu)
 {
@@ -247,8 +260,9 @@ static void cpu_common_reset(CPUState *cpu)
     cpu->mem_io_vaddr = 0;
     cpu->icount_extra = 0;
     cpu->icount_decr.u32 = 0;
-    cpu->can_do_io = 0;
+    cpu->can_do_io = 1;
     cpu->exception_index = -1;
+    cpu->crash_occurred = false;
     memset(cpu->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
 }
 
@@ -314,6 +328,7 @@ static void cpu_common_initfn(Object *obj)
 
     cpu->cpu_index = -1;
     cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+    qemu_mutex_init(&cpu->work_mutex);
     QTAILQ_INIT(&cpu->breakpoints);
     QTAILQ_INIT(&cpu->watchpoints);
 }
@@ -348,6 +363,7 @@ static void cpu_class_init(ObjectClass *klass, void *data)
     k->gdb_write_register = cpu_common_gdb_write_register;
     k->virtio_is_big_endian = cpu_common_virtio_is_big_endian;
     k->debug_excp_handler = cpu_common_noop;
+    k->debug_check_watchpoint = cpu_common_debug_check_watchpoint;
     k->cpu_exec_enter = cpu_common_noop;
     k->cpu_exec_exit = cpu_common_noop;
     k->cpu_exec_interrupt = cpu_common_exec_interrupt;
diff --git a/qemu/qom/object.c b/qemu/qom/object.c
index eea8edf3d..8e6e68dff 100644
--- a/qemu/qom/object.c
+++ b/qemu/qom/object.c
@@ -10,9 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qom/object.h"
 #include "qom/object_interfaces.h"
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qapi/visitor.h"
 #include "qapi-visit.h"
 #include "qapi/string-input-visitor.h"
@@ -204,7 +206,7 @@ static bool type_is_ancestor(TypeImpl *type, TypeImpl *target_type)
 {
     assert(target_type);
 
-    /* Check if typename is a direct ancestor of type */
+    /* Check if target_type is a direct ancestor of type */
     while (type) {
         if (type == target_type) {
             return true;
@@ -242,6 +244,16 @@ static void type_initialize_interface(TypeImpl *ti, TypeImpl *interface_type,
                                            iface_impl->class);
 }
 
+static void object_property_free(gpointer data)
+{
+    ObjectProperty *prop = data;
+
+    g_free(prop->name);
+    g_free(prop->type);
+    g_free(prop->description);
+    g_free(prop);
+}
+
 static void type_initialize(TypeImpl *ti)
 {
     TypeImpl *parent;
@@ -261,9 +273,11 @@ static void type_initialize(TypeImpl *ti)
         GSList *e;
         int i;
 
-        g_assert(parent->class_size <= ti->class_size);
+        g_assert_cmpint(parent->class_size, <=, ti->class_size);
         memcpy(ti->class, parent->class, parent->class_size);
         ti->class->interfaces = NULL;
+        ti->class->properties = g_hash_table_new_full(
+            g_str_hash, g_str_equal, g_free, object_property_free);
 
         for (e = parent->class->interfaces; e; e = e->next) {
             InterfaceClass *iface = e->data;
@@ -288,6 +302,9 @@ static void type_initialize(TypeImpl *ti)
 
             type_initialize_interface(ti, t, t);
         }
+    } else {
+        ti->class->properties = g_hash_table_new_full(
+            g_str_hash, g_str_equal, g_free, object_property_free);
     }
 
     ti->class->type = ti;
@@ -333,14 +350,15 @@ void object_initialize_with_type(void *data, size_t size, TypeImpl *type)
     g_assert(type != NULL);
     type_initialize(type);
 
-    g_assert(type->instance_size >= sizeof(Object));
+    g_assert_cmpint(type->instance_size, >=, sizeof(Object));
     g_assert(type->abstract == false);
-    g_assert(size >= type->instance_size);
+    g_assert_cmpint(size, >=, type->instance_size);
 
     memset(obj, 0, type->instance_size);
     obj->class = type->class;
     object_ref(obj);
-    QTAILQ_INIT(&obj->properties);
+    obj->properties = g_hash_table_new_full(g_str_hash, g_str_equal,
+                                            NULL, object_property_free);
     object_init_with_type(obj, type);
     object_post_init_with_type(obj, type);
 }
@@ -359,29 +377,51 @@ static inline bool object_property_is_child(ObjectProperty *prop)
 
 static void object_property_del_all(Object *obj)
 {
-    while (!QTAILQ_EMPTY(&obj->properties)) {
-        ObjectProperty *prop = QTAILQ_FIRST(&obj->properties);
-
-        QTAILQ_REMOVE(&obj->properties, prop, node);
-
-        if (prop->release) {
-            prop->release(obj, prop->name, prop->opaque);
+    ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
+    bool released;
+
+    do {
+        released = false;
+        g_hash_table_iter_init(&iter, obj->properties);
+        while (g_hash_table_iter_next(&iter, &key, &value)) {
+            prop = value;
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+                released = true;
+                break;
+            }
+            g_hash_table_iter_remove(&iter);
         }
+    } while (released);
 
-        g_free(prop->name);
-        g_free(prop->type);
-        g_free(prop->description);
-        g_free(prop);
-    }
+    g_hash_table_unref(obj->properties);
 }
 
 static void object_property_del_child(Object *obj, Object *child, Error **errp)
 {
     ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
+        if (object_property_is_child(prop) && prop->opaque == child) {
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+            }
+            break;
+        }
+    }
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
         if (object_property_is_child(prop) && prop->opaque == child) {
-            object_property_del(obj, prop->name, errp);
+            g_hash_table_iter_remove(&iter);
             break;
         }
     }
@@ -413,7 +453,7 @@ static void object_finalize(void *data)
     object_property_del_all(obj);
     object_deinit(obj, ti);
 
-    g_assert(obj->ref == 0);
+    g_assert_cmpint(obj->ref, ==, 0);
     if (obj->free) {
         obj->free(obj);
     }
@@ -775,23 +815,44 @@ void object_class_foreach(void (*fn)(ObjectClass *klass, void *opaque),
     enumerating_types = false;
 }
 
-int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
-                         void *opaque)
+static int do_object_child_foreach(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque, bool recurse)
 {
-    ObjectProperty *prop, *next;
+    GHashTableIter iter;
+    ObjectProperty *prop;
     int ret = 0;
 
-    QTAILQ_FOREACH_SAFE(prop, &obj->properties, node, next) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (object_property_is_child(prop)) {
-            ret = fn(prop->opaque, opaque);
+            Object *child = prop->opaque;
+
+            ret = fn(child, opaque);
             if (ret != 0) {
                 break;
             }
+            if (recurse) {
+                do_object_child_foreach(child, fn, opaque, true);
+            }
         }
     }
     return ret;
 }
 
+int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
+                         void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, false);
+}
+
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, true);
+}
+
 static void object_class_get_list_tramp(ObjectClass *klass, void *opaque)
 {
     GSList **list = opaque;
@@ -814,7 +875,7 @@ void object_ref(Object *obj)
     if (!obj) {
         return;
     }
-     atomic_inc(&obj->ref);
+    atomic_inc(&obj->ref);
 }
 
 void object_unref(Object *obj)
@@ -822,7 +883,7 @@ void object_unref(Object *obj)
     if (!obj) {
         return;
     }
-    g_assert(obj->ref > 0);
+    g_assert_cmpint(obj->ref, >, 0);
 
     /* parent always holds a reference to its children */
     if (atomic_fetch_dec(&obj->ref) == 1) {
@@ -860,13 +921,11 @@ object_property_add(Object *obj, const char *name, const char *type,
         return ret;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            error_setg(errp, "attempt to add duplicate property '%s'"
-                       " to object (type '%s')", name,
-                       object_get_typename(obj));
-            return NULL;
-        }
+    if (object_property_find(obj, name, NULL) != NULL) {
+        error_setg(errp, "attempt to add duplicate property '%s'"
+                   " to object (type '%s')", name,
+                   object_get_typename(obj));
+        return NULL;
     }
 
     prop = g_malloc0(sizeof(*prop));
@@ -879,7 +938,41 @@ object_property_add(Object *obj, const char *name, const char *type,
     prop->release = release;
     prop->opaque = opaque;
 
-    QTAILQ_INSERT_TAIL(&obj->properties, prop, node);
+    g_hash_table_insert(obj->properties, prop->name, prop);
+    return prop;
+}
+
+ObjectProperty *
+object_class_property_add(ObjectClass *klass,
+                          const char *name,
+                          const char *type,
+                          ObjectPropertyAccessor *get,
+                          ObjectPropertyAccessor *set,
+                          ObjectPropertyRelease *release,
+                          void *opaque,
+                          Error **errp)
+{
+    ObjectProperty *prop;
+
+    if (object_class_property_find(klass, name, NULL) != NULL) {
+        error_setg(errp, "attempt to add duplicate property '%s'"
+                   " to object (type '%s')", name,
+                   object_class_get_name(klass));
+        return NULL;
+    }
+
+    prop = g_malloc0(sizeof(*prop));
+
+    prop->name = g_strdup(name);
+    prop->type = g_strdup(type);
+
+    prop->get = get;
+    prop->set = set;
+    prop->release = release;
+    prop->opaque = opaque;
+
+    g_hash_table_insert(klass->properties, g_strdup(name), prop);
+
     return prop;
 }
 
@@ -887,34 +980,76 @@ ObjectProperty *object_property_find(Object *obj, const char *name,
                                      Error **errp)
 {
     ObjectProperty *prop;
+    ObjectClass *klass = object_get_class(obj);
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            return prop;
-        }
+    prop = object_class_property_find(klass, name, NULL);
+    if (prop) {
+        return prop;
+    }
+
+    prop = g_hash_table_lookup(obj->properties, name);
+    if (prop) {
+        return prop;
     }
 
     error_setg(errp, "Property '.%s' not found", name);
     return NULL;
 }
 
+void object_property_iter_init(ObjectPropertyIterator *iter,
+                               Object *obj)
+{
+    g_hash_table_iter_init(&iter->iter, obj->properties);
+    iter->nextclass = object_get_class(obj);
+}
+
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter)
+{
+    gpointer key, val;
+    while (!g_hash_table_iter_next(&iter->iter, &key, &val)) {
+        if (!iter->nextclass) {
+            return NULL;
+        }
+        g_hash_table_iter_init(&iter->iter, iter->nextclass->properties);
+        iter->nextclass = object_class_get_parent(iter->nextclass);
+    }
+    return val;
+}
+
+ObjectProperty *object_class_property_find(ObjectClass *klass, const char *name,
+                                           Error **errp)
+{
+    ObjectProperty *prop;
+    ObjectClass *parent_klass;
+
+    parent_klass = object_class_get_parent(klass);
+    if (parent_klass) {
+        prop = object_class_property_find(parent_klass, name, NULL);
+        if (prop) {
+            return prop;
+        }
+    }
+
+    prop = g_hash_table_lookup(klass->properties, name);
+    if (!prop) {
+        error_setg(errp, "Property '.%s' not found", name);
+    }
+    return prop;
+}
+
 void object_property_del(Object *obj, const char *name, Error **errp)
 {
-    ObjectProperty *prop = object_property_find(obj, name, errp);
-    if (prop == NULL) {
+    ObjectProperty *prop = g_hash_table_lookup(obj->properties, name);
+
+    if (!prop) {
+        error_setg(errp, "Property '.%s' not found", name);
         return;
     }
 
     if (prop->release) {
         prop->release(obj, name, prop->opaque);
     }
-
-    QTAILQ_REMOVE(&obj->properties, prop, node);
-
-    g_free(prop->name);
-    g_free(prop->type);
-    g_free(prop->description);
-    g_free(prop);
+    g_hash_table_remove(obj->properties, name);
 }
 
 void object_property_get(Object *obj, Visitor *v, const char *name,
@@ -928,7 +1063,7 @@ void object_property_get(Object *obj, Visitor *v, const char *name,
     if (!prop->get) {
         error_setg(errp, QERR_PERMISSION_DENIED);
     } else {
-        prop->get(obj, v, prop->opaque, name, errp);
+        prop->get(obj, v, name, prop->opaque, errp);
     }
 }
 
@@ -943,7 +1078,7 @@ void object_property_set(Object *obj, Visitor *v, const char *name,
     if (!prop->set) {
         error_setg(errp, QERR_PERMISSION_DENIED);
     } else {
-        prop->set(obj, v, prop->opaque, name, errp);
+        prop->set(obj, v, name, prop->opaque, errp);
     }
 }
 
@@ -1079,6 +1214,7 @@ typedef struct EnumProperty {
 int object_property_get_enum(Object *obj, const char *name,
                              const char *typename, Error **errp)
 {
+    Error *err = NULL;
     StringOutputVisitor *sov;
     StringInputVisitor *siv;
     char *str;
@@ -1100,12 +1236,17 @@ int object_property_get_enum(Object *obj, const char *name,
     enumprop = prop->opaque;
 
     sov = string_output_visitor_new(false);
-    object_property_get(obj, string_output_get_visitor(sov), name, errp);
+    object_property_get(obj, string_output_get_visitor(sov), name, &err);
+    if (err) {
+        error_propagate(errp, err);
+        string_output_visitor_cleanup(sov);
+        return 0;
+    }
     str = string_output_get_string(sov);
     siv = string_input_visitor_new(str);
     string_output_visitor_cleanup(sov);
-    visit_type_enum(string_input_get_visitor(siv),
-                    &ret, enumprop->strings, NULL, name, errp);
+    visit_type_enum(string_input_get_visitor(siv), name, &ret,
+                    enumprop->strings, errp);
 
     g_free(str);
     string_input_visitor_cleanup(siv);
@@ -1116,51 +1257,56 @@ int object_property_get_enum(Object *obj, const char *name,
 void object_property_get_uint16List(Object *obj, const char *name,
                                     uint16List **list, Error **errp)
 {
+    Error *err = NULL;
     StringOutputVisitor *ov;
     StringInputVisitor *iv;
     char *str;
 
     ov = string_output_visitor_new(false);
     object_property_get(obj, string_output_get_visitor(ov),
-                        name, errp);
+                        name, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto out;
+    }
     str = string_output_get_string(ov);
     iv = string_input_visitor_new(str);
-    visit_type_uint16List(string_input_get_visitor(iv),
-                          list, NULL, errp);
+    visit_type_uint16List(string_input_get_visitor(iv), NULL, list, errp);
 
     g_free(str);
-    string_output_visitor_cleanup(ov);
     string_input_visitor_cleanup(iv);
+out:
+    string_output_visitor_cleanup(ov);
 }
 
 void object_property_parse(Object *obj, const char *string,
                            const char *name, Error **errp)
 {
-    StringInputVisitor *mi;
-    mi = string_input_visitor_new(string);
-    object_property_set(obj, string_input_get_visitor(mi), name, errp);
+    StringInputVisitor *siv;
+    siv = string_input_visitor_new(string);
+    object_property_set(obj, string_input_get_visitor(siv), name, errp);
 
-    string_input_visitor_cleanup(mi);
+    string_input_visitor_cleanup(siv);
 }
 
 char *object_property_print(Object *obj, const char *name, bool human,
                             Error **errp)
 {
-    StringOutputVisitor *mo;
+    StringOutputVisitor *sov;
     char *string = NULL;
     Error *local_err = NULL;
 
-    mo = string_output_visitor_new(human);
-    object_property_get(obj, string_output_get_visitor(mo), name, &local_err);
+    sov = string_output_visitor_new(human);
+    object_property_get(obj, string_output_get_visitor(sov), name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
     }
 
-    string = string_output_get_string(mo);
+    string = string_output_get_string(sov);
 
 out:
-    string_output_visitor_cleanup(mo);
+    string_output_visitor_cleanup(sov);
     return string;
 }
 
@@ -1190,14 +1336,15 @@ Object *object_get_objects_root(void)
     return container_get(object_get_root(), "/objects");
 }
 
-static void object_get_child_property(Object *obj, Visitor *v, void *opaque,
-                                      const char *name, Error **errp)
+static void object_get_child_property(Object *obj, Visitor *v,
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     Object *child = opaque;
     gchar *path;
 
     path = object_get_canonical_path(child);
-    visit_type_str(v, &path, name, errp);
+    visit_type_str(v, name, &path, errp);
     g_free(path);
 }
 
@@ -1259,8 +1406,9 @@ typedef struct {
     ObjectPropertyLinkFlags flags;
 } LinkProperty;
 
-static void object_get_link_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void object_get_link_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     LinkProperty *lprop = opaque;
     Object **child = lprop->child;
@@ -1268,11 +1416,11 @@ static void object_get_link_property(Object *obj, Visitor *v, void *opaque,
 
     if (*child) {
         path = object_get_canonical_path(*child);
-        visit_type_str(v, &path, name, errp);
+        visit_type_str(v, name, &path, errp);
         g_free(path);
     } else {
         path = (gchar *)"";
-        visit_type_str(v, &path, name, errp);
+        visit_type_str(v, name, &path, errp);
     }
 }
 
@@ -1299,8 +1447,8 @@ static Object *object_resolve_link(Object *obj, const char *name,
     target = object_resolve_path_type(path, target_type, &ambiguous);
 
     if (ambiguous) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "Path '%s' does not uniquely identify an object", path);
+        error_setg(errp, "Path '%s' does not uniquely identify an object",
+                   path);
     } else if (!target) {
         target = object_resolve_path(path, &ambiguous);
         if (target || ambiguous) {
@@ -1316,8 +1464,9 @@ static Object *object_resolve_link(Object *obj, const char *name,
     return target;
 }
 
-static void object_set_link_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void object_set_link_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     Error *local_err = NULL;
     LinkProperty *prop = opaque;
@@ -1326,7 +1475,7 @@ static void object_set_link_property(Object *obj, Visitor *v, void *opaque,
     Object *new_target = NULL;
     char *path = NULL;
 
-    visit_type_str(v, &path, name, &local_err);
+    visit_type_str(v, name, &path, &local_err);
 
     if (!local_err && strcmp(path, "") != 0) {
         new_target = object_resolve_link(obj, name, path, &local_err);
@@ -1422,11 +1571,13 @@ void object_property_add_const_link(Object *obj, const char *name,
 gchar *object_get_canonical_path_component(Object *obj)
 {
     ObjectProperty *prop = NULL;
+    GHashTableIter iter;
 
     g_assert(obj);
     g_assert(obj->parent != NULL);
 
-    QTAILQ_FOREACH(prop, &obj->parent->properties, node) {
+    g_hash_table_iter_init(&iter, obj->parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (!object_property_is_child(prop)) {
             continue;
         }
@@ -1510,11 +1661,13 @@ static Object *object_resolve_partial_path(Object *parent,
                                               bool *ambiguous)
 {
     Object *obj;
+    GHashTableIter iter;
     ObjectProperty *prop;
 
     obj = object_resolve_abs_path(parent, parts, typename, 0);
 
-    QTAILQ_FOREACH(prop, &parent->properties, node) {
+    g_hash_table_iter_init(&iter, parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         Object *found;
 
         if (!object_property_is_child(prop)) {
@@ -1576,27 +1729,31 @@ typedef struct StringProperty
     void (*set)(Object *, const char *, Error **);
 } StringProperty;
 
-static void property_get_str(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+static void property_get_str(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     StringProperty *prop = opaque;
     char *value;
+    Error *err = NULL;
 
-    value = prop->get(obj, errp);
-    if (value) {
-        visit_type_str(v, &value, name, errp);
-        g_free(value);
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
+
+    visit_type_str(v, name, &value, errp);
+    g_free(value);
 }
 
-static void property_set_str(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+static void property_set_str(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     StringProperty *prop = opaque;
     char *value;
     Error *local_err = NULL;
 
-    visit_type_str(v, &value, name, &local_err);
+    visit_type_str(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1635,30 +1792,59 @@ void object_property_add_str(Object *obj, const char *name,
     }
 }
 
+void object_class_property_add_str(ObjectClass *klass, const char *name,
+                                   char *(*get)(Object *, Error **),
+                                   void (*set)(Object *, const char *,
+                                               Error **),
+                                   Error **errp)
+{
+    Error *local_err = NULL;
+    StringProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, "string",
+                              get ? property_get_str : NULL,
+                              set ? property_set_str : NULL,
+                              property_release_str,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
 typedef struct BoolProperty
 {
     bool (*get)(Object *, Error **);
     void (*set)(Object *, bool, Error **);
 } BoolProperty;
 
-static void property_get_bool(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void property_get_bool(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     BoolProperty *prop = opaque;
     bool value;
+    Error *err = NULL;
+
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
 
-    value = prop->get(obj, errp);
-    visit_type_bool(v, &value, name, errp);
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void property_set_bool(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void property_set_bool(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     BoolProperty *prop = opaque;
     bool value;
     Error *local_err = NULL;
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1696,23 +1882,56 @@ void object_property_add_bool(Object *obj, const char *name,
     }
 }
 
-static void property_get_enum(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+void object_class_property_add_bool(ObjectClass *klass, const char *name,
+                                    bool (*get)(Object *, Error **),
+                                    void (*set)(Object *, bool, Error **),
+                                    Error **errp)
+{
+    Error *local_err = NULL;
+    BoolProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, "bool",
+                              get ? property_get_bool : NULL,
+                              set ? property_set_bool : NULL,
+                              property_release_bool,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
+static void property_get_enum(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     EnumProperty *prop = opaque;
     int value;
+    Error *err = NULL;
 
-    value = prop->get(obj, errp);
-    visit_type_enum(v, &value, prop->strings, NULL, name, errp);
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    visit_type_enum(v, name, &value, prop->strings, errp);
 }
 
-static void property_set_enum(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void property_set_enum(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     EnumProperty *prop = opaque;
     int value;
+    Error *err = NULL;
 
-    visit_type_enum(v, &value, prop->strings, NULL, name, errp);
+    visit_type_enum(v, name, &value, prop->strings, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
     prop->set(obj, value, errp);
 }
 
@@ -1748,12 +1967,37 @@ void object_property_add_enum(Object *obj, const char *name,
     }
 }
 
+void object_class_property_add_enum(ObjectClass *klass, const char *name,
+                                    const char *typename,
+                                    const char * const *strings,
+                                    int (*get)(Object *, Error **),
+                                    void (*set)(Object *, int, Error **),
+                                    Error **errp)
+{
+    Error *local_err = NULL;
+    EnumProperty *prop = g_malloc(sizeof(*prop));
+
+    prop->strings = strings;
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, typename,
+                              get ? property_get_enum : NULL,
+                              set ? property_set_enum : NULL,
+                              property_release_enum,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
 typedef struct TMProperty {
     void (*get)(Object *, struct tm *, Error **);
 } TMProperty;
 
-static void property_get_tm(Object *obj, Visitor *v, void *opaque,
-                            const char *name, Error **errp)
+static void property_get_tm(Object *obj, Visitor *v, const char *name,
+                            void *opaque, Error **errp)
 {
     TMProperty *prop = opaque;
     Error *err = NULL;
@@ -1764,31 +2008,31 @@ static void property_get_tm(Object *obj, Visitor *v, void *opaque,
         goto out;
     }
 
-    visit_start_struct(v, NULL, "struct tm", name, 0, &err);
+    visit_start_struct(v, name, NULL, 0, &err);
     if (err) {
         goto out;
     }
-    visit_type_int32(v, &value.tm_year, "tm_year", &err);
+    visit_type_int32(v, "tm_year", &value.tm_year, &err);
     if (err) {
         goto out_end;
     }
-    visit_type_int32(v, &value.tm_mon, "tm_mon", &err);
+    visit_type_int32(v, "tm_mon", &value.tm_mon, &err);
     if (err) {
         goto out_end;
     }
-    visit_type_int32(v, &value.tm_mday, "tm_mday", &err);
+    visit_type_int32(v, "tm_mday", &value.tm_mday, &err);
     if (err) {
         goto out_end;
     }
-    visit_type_int32(v, &value.tm_hour, "tm_hour", &err);
+    visit_type_int32(v, "tm_hour", &value.tm_hour, &err);
     if (err) {
         goto out_end;
     }
-    visit_type_int32(v, &value.tm_min, "tm_min", &err);
+    visit_type_int32(v, "tm_min", &value.tm_min, &err);
     if (err) {
         goto out_end;
     }
-    visit_type_int32(v, &value.tm_sec, "tm_sec", &err);
+    visit_type_int32(v, "tm_sec", &value.tm_sec, &err);
     if (err) {
         goto out_end;
     }
@@ -1827,41 +2071,56 @@ void object_property_add_tm(Object *obj, const char *name,
     }
 }
 
+void object_class_property_add_tm(ObjectClass *klass, const char *name,
+                                  void (*get)(Object *, struct tm *, Error **),
+                                  Error **errp)
+{
+    Error *local_err = NULL;
+    TMProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+
+    object_class_property_add(klass, name, "struct tm",
+                              get ? property_get_tm : NULL, NULL,
+                              property_release_tm,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
 static char *qdev_get_type(Object *obj, Error **errp)
 {
     return g_strdup(object_get_typename(obj));
 }
 
-static void property_get_uint8_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint8_ptr(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     uint8_t value = *(uint8_t *)opaque;
-    visit_type_uint8(v, &value, name, errp);
+    visit_type_uint8(v, name, &value, errp);
 }
 
-static void property_get_uint16_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint16_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint16_t value = *(uint16_t *)opaque;
-    visit_type_uint16(v, &value, name, errp);
+    visit_type_uint16(v, name, &value, errp);
 }
 
-static void property_get_uint32_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint32_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint32_t value = *(uint32_t *)opaque;
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
-static void property_get_uint64_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint64_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint64_t value = *(uint64_t *)opaque;
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 void object_property_add_uint8_ptr(Object *obj, const char *name,
@@ -1871,6 +2130,13 @@ void object_property_add_uint8_ptr(Object *obj, const char *name,
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint8_ptr(ObjectClass *klass, const char *name,
+                                         const uint8_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint8", property_get_uint8_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint16_ptr(Object *obj, const char *name,
                                     const uint16_t *v, Error **errp)
 {
@@ -1878,6 +2144,13 @@ void object_property_add_uint16_ptr(Object *obj, const char *name,
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint16_ptr(ObjectClass *klass, const char *name,
+                                          const uint16_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint16", property_get_uint16_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint32_ptr(Object *obj, const char *name,
                                     const uint32_t *v, Error **errp)
 {
@@ -1885,6 +2158,13 @@ void object_property_add_uint32_ptr(Object *obj, const char *name,
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint32_ptr(ObjectClass *klass, const char *name,
+                                          const uint32_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint32", property_get_uint32_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint64_ptr(Object *obj, const char *name,
                                     const uint64_t *v, Error **errp)
 {
@@ -1892,21 +2172,28 @@ void object_property_add_uint64_ptr(Object *obj, const char *name,
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint64_ptr(ObjectClass *klass, const char *name,
+                                          const uint64_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint64", property_get_uint64_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 typedef struct {
     Object *target_obj;
     char *target_name;
 } AliasProperty;
 
-static void property_get_alias(Object *obj, struct Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+static void property_get_alias(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     AliasProperty *prop = opaque;
 
     object_property_get(prop->target_obj, v, prop->target_name, errp);
 }
 
-static void property_set_alias(Object *obj, struct Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+static void property_set_alias(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     AliasProperty *prop = opaque;
 
@@ -1989,6 +2276,23 @@ void object_property_set_description(Object *obj, const char *name,
     op->description = g_strdup(description);
 }
 
+void object_class_property_set_description(ObjectClass *klass,
+                                           const char *name,
+                                           const char *description,
+                                           Error **errp)
+{
+    ObjectProperty *op;
+
+    op = g_hash_table_lookup(klass->properties, name);
+    if (!op) {
+        error_setg(errp, "Property '.%s' not found", name);
+        return;
+    }
+
+    g_free(op->description);
+    op->description = g_strdup(description);
+}
+
 static void object_instance_init(Object *obj)
 {
     object_property_add_str(obj, "type", qdev_get_type, NULL, NULL);
diff --git a/qemu/qom/object_interfaces.c b/qemu/qom/object_interfaces.c
index a66cd6026..393189024 100644
--- a/qemu/qom/object_interfaces.c
+++ b/qemu/qom/object_interfaces.c
@@ -1,5 +1,10 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qom/object_interfaces.h"
 #include "qemu/module.h"
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/opts-visitor.h"
 
 void user_creatable_complete(Object *obj, Error **errp)
 {
@@ -30,6 +35,179 @@ bool user_creatable_can_be_deleted(UserCreatable *uc, Error **errp)
     }
 }
 
+
+Object *user_creatable_add(const QDict *qdict,
+                           Visitor *v, Error **errp)
+{
+    char *type = NULL;
+    char *id = NULL;
+    Object *obj = NULL;
+    Error *local_err = NULL, *end_err = NULL;
+    QDict *pdict;
+
+    pdict = qdict_clone_shallow(qdict);
+
+    visit_start_struct(v, NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    qdict_del(pdict, "qom-type");
+    visit_type_str(v, "qom-type", &type, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+
+    qdict_del(pdict, "id");
+    visit_type_str(v, "id", &id, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+
+    obj = user_creatable_add_type(type, id, pdict, v, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+
+ out_visit:
+    visit_end_struct(v, &end_err);
+    if (end_err) {
+        error_propagate(&local_err, end_err);
+        if (obj) {
+            user_creatable_del(id, NULL);
+        }
+        goto out;
+    }
+
+out:
+    QDECREF(pdict);
+    g_free(id);
+    g_free(type);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        object_unref(obj);
+        return NULL;
+    }
+    return obj;
+}
+
+
+Object *user_creatable_add_type(const char *type, const char *id,
+                                const QDict *qdict,
+                                Visitor *v, Error **errp)
+{
+    Object *obj;
+    ObjectClass *klass;
+    const QDictEntry *e;
+    Error *local_err = NULL;
+
+    klass = object_class_by_name(type);
+    if (!klass) {
+        error_setg(errp, "invalid object type: %s", type);
+        return NULL;
+    }
+
+    if (!object_class_dynamic_cast(klass, TYPE_USER_CREATABLE)) {
+        error_setg(errp, "object type '%s' isn't supported by object-add",
+                   type);
+        return NULL;
+    }
+
+    if (object_class_is_abstract(klass)) {
+        error_setg(errp, "object type '%s' is abstract", type);
+        return NULL;
+    }
+
+    obj = object_new(type);
+    if (qdict) {
+        for (e = qdict_first(qdict); e; e = qdict_next(qdict, e)) {
+            object_property_set(obj, v, e->key, &local_err);
+            if (local_err) {
+                goto out;
+            }
+        }
+    }
+
+    object_property_add_child(object_get_objects_root(),
+                              id, obj, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    user_creatable_complete(obj, &local_err);
+    if (local_err) {
+        object_property_del(object_get_objects_root(),
+                            id, &error_abort);
+        goto out;
+    }
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        object_unref(obj);
+        return NULL;
+    }
+    return obj;
+}
+
+
+Object *user_creatable_add_opts(QemuOpts *opts, Error **errp)
+{
+    OptsVisitor *ov;
+    QDict *pdict;
+    Object *obj = NULL;
+
+    ov = opts_visitor_new(opts);
+    pdict = qemu_opts_to_qdict(opts, NULL);
+
+    obj = user_creatable_add(pdict, opts_get_visitor(ov), errp);
+    opts_visitor_cleanup(ov);
+    QDECREF(pdict);
+    return obj;
+}
+
+
+int user_creatable_add_opts_foreach(void *opaque, QemuOpts *opts, Error **errp)
+{
+    bool (*type_predicate)(const char *) = opaque;
+    Object *obj = NULL;
+    Error *err = NULL;
+    const char *type;
+
+    type = qemu_opt_get(opts, "qom-type");
+    if (type && type_predicate &&
+        !type_predicate(type)) {
+        return 0;
+    }
+
+    obj = user_creatable_add_opts(opts, &err);
+    if (!obj) {
+        error_report_err(err);
+        return -1;
+    }
+    object_unref(obj);
+    return 0;
+}
+
+
+void user_creatable_del(const char *id, Error **errp)
+{
+    Object *container;
+    Object *obj;
+
+    container = object_get_objects_root();
+    obj = object_resolve_path_component(container, id);
+    if (!obj) {
+        error_setg(errp, "object '%s' not found", id);
+        return;
+    }
+
+    if (!user_creatable_can_be_deleted(USER_CREATABLE(obj), errp)) {
+        error_setg(errp, "object '%s' is in use, can not be deleted", id);
+        return;
+    }
+    object_unparent(obj);
+}
+
 static void register_types(void)
 {
     static const TypeInfo uc_interface_info = {
diff --git a/qemu/qom/qom-qobject.c b/qemu/qom/qom-qobject.c
index 6384b8e98..e6b17c1f1 100644
--- a/qemu/qom/qom-qobject.c
+++ b/qemu/qom/qom-qobject.c
@@ -9,6 +9,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qom/object.h"
 #include "qom/qom-qobject.h"
@@ -19,11 +21,11 @@
 void object_property_set_qobject(Object *obj, QObject *value,
                                  const char *name, Error **errp)
 {
-    QmpInputVisitor *mi;
-    mi = qmp_input_visitor_new(value);
-    object_property_set(obj, qmp_input_get_visitor(mi), name, errp);
+    QmpInputVisitor *qiv;
+    qiv = qmp_input_visitor_new(value);
+    object_property_set(obj, qmp_input_get_visitor(qiv), name, errp);
 
-    qmp_input_visitor_cleanup(mi);
+    qmp_input_visitor_cleanup(qiv);
 }
 
 QObject *object_property_get_qobject(Object *obj, const char *name,
@@ -31,14 +33,14 @@ QObject *object_property_get_qobject(Object *obj, const char *name,
 {
     QObject *ret = NULL;
     Error *local_err = NULL;
-    QmpOutputVisitor *mo;
+    QmpOutputVisitor *qov;
 
-    mo = qmp_output_visitor_new();
-    object_property_get(obj, qmp_output_get_visitor(mo), name, &local_err);
+    qov = qmp_output_visitor_new();
+    object_property_get(obj, qmp_output_get_visitor(qov), name, &local_err);
     if (!local_err) {
-        ret = qmp_output_get_qobject(mo);
+        ret = qmp_output_get_qobject(qov);
     }
     error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
+    qmp_output_visitor_cleanup(qov);
     return ret;
 }
diff --git a/qemu/qtest.c b/qemu/qtest.c
index 05cefd280..87575bc0b 100644
--- a/qemu/qtest.c
+++ b/qemu/qtest.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/qtest.h"
 #include "hw/qdev.h"
 #include "sysemu/char.h"
diff --git a/qemu/replay/Makefile.objs b/qemu/replay/Makefile.objs
new file mode 100644
index 000000000..fcb3f74d6
--- /dev/null
+++ b/qemu/replay/Makefile.objs
@@ -0,0 +1,6 @@
+common-obj-y += replay.o
+common-obj-y += replay-internal.o
+common-obj-y += replay-events.o
+common-obj-y += replay-time.o
+common-obj-y += replay-input.o
+common-obj-y += replay-char.o
diff --git a/qemu/replay/replay-char.c b/qemu/replay/replay-char.c
new file mode 100755
index 000000000..23b692297
--- /dev/null
+++ b/qemu/replay/replay-char.c
@@ -0,0 +1,168 @@
+/*
+ * replay-char.c
+ *
+ * Copyright (c) 2010-2016 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/char.h"
+
+/* Char drivers that generate qemu_chr_be_write events
+   that should be saved into the log. */
+static CharDriverState **char_drivers;
+static int drivers_count;
+
+/* Char event attributes. */
+typedef struct CharEvent {
+    int id;
+    uint8_t *buf;
+    size_t len;
+} CharEvent;
+
+static int find_char_driver(CharDriverState *chr)
+{
+    int i = 0;
+    for ( ; i < drivers_count ; ++i) {
+        if (char_drivers[i] == chr) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void replay_register_char_driver(CharDriverState *chr)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+    char_drivers = g_realloc(char_drivers,
+                             sizeof(*char_drivers) * (drivers_count + 1));
+    char_drivers[drivers_count++] = chr;
+}
+
+void replay_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    CharEvent *event = g_malloc0(sizeof(CharEvent));
+
+    event->id = find_char_driver(s);
+    if (event->id < 0) {
+        fprintf(stderr, "Replay: cannot find char driver\n");
+        exit(1);
+    }
+    event->buf = g_malloc(len);
+    memcpy(event->buf, buf, len);
+    event->len = len;
+
+    replay_add_event(REPLAY_ASYNC_EVENT_CHAR_READ, event, NULL, 0);
+}
+
+void replay_event_char_read_run(void *opaque)
+{
+    CharEvent *event = (CharEvent *)opaque;
+
+    qemu_chr_be_write_impl(char_drivers[event->id], event->buf,
+                           (int)event->len);
+
+    g_free(event->buf);
+    g_free(event);
+}
+
+void replay_event_char_read_save(void *opaque)
+{
+    CharEvent *event = (CharEvent *)opaque;
+
+    replay_put_byte(event->id);
+    replay_put_array(event->buf, event->len);
+}
+
+void *replay_event_char_read_load(void)
+{
+    CharEvent *event = g_malloc0(sizeof(CharEvent));
+
+    event->id = replay_get_byte();
+    replay_get_array_alloc(&event->buf, &event->len);
+
+    return event;
+}
+
+void replay_char_write_event_save(int res, int offset)
+{
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_WRITE);
+    replay_put_dword(res);
+    replay_put_dword(offset);
+    replay_mutex_unlock();
+}
+
+void replay_char_write_event_load(int *res, int *offset)
+{
+    replay_account_executed_instructions();
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_CHAR_WRITE)) {
+        *res = replay_get_dword();
+        *offset = replay_get_dword();
+        replay_finish_event();
+        replay_mutex_unlock();
+    } else {
+        replay_mutex_unlock();
+        error_report("Missing character write event in the replay log");
+        exit(1);
+    }
+}
+
+int replay_char_read_all_load(uint8_t *buf)
+{
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_CHAR_READ_ALL)) {
+        size_t size;
+        int res;
+        replay_get_array(buf, &size);
+        replay_finish_event();
+        replay_mutex_unlock();
+        res = (int)size;
+        assert(res >= 0);
+        return res;
+    } else if (replay_next_event_is(EVENT_CHAR_READ_ALL_ERROR)) {
+        int res = replay_get_dword();
+        replay_finish_event();
+        replay_mutex_unlock();
+        return res;
+    } else {
+        replay_mutex_unlock();
+        error_report("Missing character read all event in the replay log");
+        exit(1);
+    }
+}
+
+void replay_char_read_all_save_error(int res)
+{
+    assert(res < 0);
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_READ_ALL_ERROR);
+    replay_put_dword(res);
+    replay_mutex_unlock();
+}
+
+void replay_char_read_all_save_buf(uint8_t *buf, int offset)
+{
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_READ_ALL);
+    replay_put_array(buf, offset);
+    replay_mutex_unlock();
+}
diff --git a/qemu/replay/replay-events.c b/qemu/replay/replay-events.c
new file mode 100644
index 000000000..3807245ae
--- /dev/null
+++ b/qemu/replay/replay-events.c
@@ -0,0 +1,311 @@
+/*
+ * replay-events.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "block/aio.h"
+#include "ui/input.h"
+
+typedef struct Event {
+    ReplayAsyncEventKind event_kind;
+    void *opaque;
+    void *opaque2;
+    uint64_t id;
+
+    QTAILQ_ENTRY(Event) events;
+} Event;
+
+static QTAILQ_HEAD(, Event) events_list = QTAILQ_HEAD_INITIALIZER(events_list);
+static unsigned int read_event_kind = -1;
+static uint64_t read_id = -1;
+static int read_checkpoint = -1;
+
+static bool events_enabled;
+
+/* Functions */
+
+static void replay_run_event(Event *event)
+{
+    switch (event->event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        aio_bh_call(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        qemu_input_event_send_impl(NULL, (InputEvent *)event->opaque);
+        qapi_free_InputEvent((InputEvent *)event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        qemu_input_event_sync_impl();
+        break;
+    case REPLAY_ASYNC_EVENT_CHAR_READ:
+        replay_event_char_read_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK:
+        aio_bh_call(event->opaque);
+        break;
+    default:
+        error_report("Replay: invalid async event ID (%d) in the queue",
+                    event->event_kind);
+        exit(1);
+        break;
+    }
+}
+
+void replay_enable_events(void)
+{
+    events_enabled = true;
+}
+
+bool replay_has_events(void)
+{
+    return !QTAILQ_EMPTY(&events_list);
+}
+
+void replay_flush_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+void replay_disable_events(void)
+{
+    if (replay_mode != REPLAY_MODE_NONE) {
+        events_enabled = false;
+        /* Flush events queue before waiting of completion */
+        replay_flush_events();
+    }
+}
+
+void replay_clear_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        QTAILQ_REMOVE(&events_list, event, events);
+
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+/*! Adds specified async event to the queue */
+void replay_add_event(ReplayAsyncEventKind event_kind,
+                      void *opaque,
+                      void *opaque2, uint64_t id)
+{
+    assert(event_kind < REPLAY_ASYNC_COUNT);
+
+    if (!replay_file || replay_mode == REPLAY_MODE_NONE
+        || !events_enabled) {
+        Event e;
+        e.event_kind = event_kind;
+        e.opaque = opaque;
+        e.opaque2 = opaque2;
+        e.id = id;
+        replay_run_event(&e);
+        return;
+    }
+
+    Event *event = g_malloc0(sizeof(Event));
+    event->event_kind = event_kind;
+    event->opaque = opaque;
+    event->opaque2 = opaque2;
+    event->id = id;
+
+    replay_mutex_lock();
+    QTAILQ_INSERT_TAIL(&events_list, event, events);
+    replay_mutex_unlock();
+}
+
+void replay_bh_schedule_event(QEMUBH *bh)
+{
+    if (replay_mode != REPLAY_MODE_NONE && events_enabled) {
+        uint64_t id = replay_get_current_step();
+        replay_add_event(REPLAY_ASYNC_EVENT_BH, bh, NULL, id);
+    } else {
+        qemu_bh_schedule(bh);
+    }
+}
+
+void replay_add_input_event(struct InputEvent *event)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT, event, NULL, 0);
+}
+
+void replay_add_input_sync_event(void)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT_SYNC, NULL, NULL, 0);
+}
+
+void replay_block_event(QEMUBH *bh, uint64_t id)
+{
+    if (replay_mode != REPLAY_MODE_NONE && events_enabled) {
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK, bh, NULL, id);
+    } else {
+        qemu_bh_schedule(bh);
+    }
+}
+
+static void replay_save_event(Event *event, int checkpoint)
+{
+    if (replay_mode != REPLAY_MODE_PLAY) {
+        /* put the event into the file */
+        replay_put_event(EVENT_ASYNC);
+        replay_put_byte(checkpoint);
+        replay_put_byte(event->event_kind);
+
+        /* save event-specific data */
+        switch (event->event_kind) {
+        case REPLAY_ASYNC_EVENT_BH:
+            replay_put_qword(event->id);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT:
+            replay_save_input_event(event->opaque);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+            break;
+        case REPLAY_ASYNC_EVENT_CHAR_READ:
+            replay_event_char_read_save(event->opaque);
+            break;
+        case REPLAY_ASYNC_EVENT_BLOCK:
+            replay_put_qword(event->id);
+            break;
+        default:
+            error_report("Unknown ID %" PRId64 " of replay event", event->id);
+            exit(1);
+        }
+    }
+}
+
+/* Called with replay mutex locked */
+void replay_save_events(int checkpoint)
+{
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_save_event(event, checkpoint);
+
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+}
+
+static Event *replay_read_event(int checkpoint)
+{
+    Event *event;
+    if (read_event_kind == -1) {
+        read_checkpoint = replay_get_byte();
+        read_event_kind = replay_get_byte();
+        read_id = -1;
+        replay_check_error();
+    }
+
+    if (checkpoint != read_checkpoint) {
+        return NULL;
+    }
+
+    /* Events that has not to be in the queue */
+    switch (read_event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        if (read_id == -1) {
+            read_id = replay_get_qword();
+        }
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = replay_read_input_event();
+        return event;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = 0;
+        return event;
+    case REPLAY_ASYNC_EVENT_CHAR_READ:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = replay_event_char_read_load();
+        return event;
+    case REPLAY_ASYNC_EVENT_BLOCK:
+        if (read_id == -1) {
+            read_id = replay_get_qword();
+        }
+        break;
+    default:
+        error_report("Unknown ID %d of replay event", read_event_kind);
+        exit(1);
+        break;
+    }
+
+    QTAILQ_FOREACH(event, &events_list, events) {
+        if (event->event_kind == read_event_kind
+            && (read_id == -1 || read_id == event->id)) {
+            break;
+        }
+    }
+
+    if (event) {
+        QTAILQ_REMOVE(&events_list, event, events);
+    } else {
+        return NULL;
+    }
+
+    /* Read event-specific data */
+
+    return event;
+}
+
+/* Called with replay mutex locked */
+void replay_read_events(int checkpoint)
+{
+    while (replay_data_kind == EVENT_ASYNC) {
+        Event *event = replay_read_event(checkpoint);
+        if (!event) {
+            break;
+        }
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+
+        g_free(event);
+        replay_finish_event();
+        read_event_kind = -1;
+    }
+}
+
+void replay_init_events(void)
+{
+    read_event_kind = -1;
+}
+
+void replay_finish_events(void)
+{
+    events_enabled = false;
+    replay_clear_events();
+}
+
+bool replay_events_enabled(void)
+{
+    return events_enabled;
+}
diff --git a/qemu/replay/replay-input.c b/qemu/replay/replay-input.c
new file mode 100644
index 000000000..06babe0ec
--- /dev/null
+++ b/qemu/replay/replay-input.c
@@ -0,0 +1,169 @@
+/*
+ * replay-input.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/notify.h"
+#include "ui/input.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi-visit.h"
+
+static InputEvent *qapi_clone_InputEvent(InputEvent *src)
+{
+    QmpOutputVisitor *qov;
+    QmpInputVisitor *qiv;
+    Visitor *ov, *iv;
+    QObject *obj;
+    InputEvent *dst = NULL;
+
+    qov = qmp_output_visitor_new();
+    ov = qmp_output_get_visitor(qov);
+    visit_type_InputEvent(ov, NULL, &src, &error_abort);
+    obj = qmp_output_get_qobject(qov);
+    qmp_output_visitor_cleanup(qov);
+    if (!obj) {
+        return NULL;
+    }
+
+    qiv = qmp_input_visitor_new(obj);
+    iv = qmp_input_get_visitor(qiv);
+    visit_type_InputEvent(iv, NULL, &dst, &error_abort);
+    qmp_input_visitor_cleanup(qiv);
+    qobject_decref(obj);
+
+    return dst;
+}
+
+void replay_save_input_event(InputEvent *evt)
+{
+    InputKeyEvent *key;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
+    replay_put_dword(evt->type);
+
+    switch (evt->type) {
+    case INPUT_EVENT_KIND_KEY:
+        key = evt->u.key.data;
+        replay_put_dword(key->key->type);
+
+        switch (key->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            replay_put_qword(key->key->u.number.data);
+            replay_put_byte(key->down);
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            replay_put_dword(key->key->u.qcode.data);
+            replay_put_byte(key->down);
+            break;
+        case KEY_VALUE_KIND__MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        btn = evt->u.btn.data;
+        replay_put_dword(btn->button);
+        replay_put_byte(btn->down);
+        break;
+    case INPUT_EVENT_KIND_REL:
+        move = evt->u.rel.data;
+        replay_put_dword(move->axis);
+        replay_put_qword(move->value);
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        move = evt->u.abs.data;
+        replay_put_dword(move->axis);
+        replay_put_qword(move->value);
+        break;
+    case INPUT_EVENT_KIND__MAX:
+        /* keep gcc happy */
+        break;
+    }
+}
+
+InputEvent *replay_read_input_event(void)
+{
+    InputEvent evt;
+    KeyValue keyValue;
+    InputKeyEvent key;
+    key.key = &keyValue;
+    InputBtnEvent btn;
+    InputMoveEvent rel;
+    InputMoveEvent abs;
+
+    evt.type = replay_get_dword();
+    switch (evt.type) {
+    case INPUT_EVENT_KIND_KEY:
+        evt.u.key.data = &key;
+        evt.u.key.data->key->type = replay_get_dword();
+
+        switch (evt.u.key.data->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            evt.u.key.data->key->u.number.data = replay_get_qword();
+            evt.u.key.data->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            evt.u.key.data->key->u.qcode.data = (QKeyCode)replay_get_dword();
+            evt.u.key.data->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND__MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        evt.u.btn.data = &btn;
+        evt.u.btn.data->button = (InputButton)replay_get_dword();
+        evt.u.btn.data->down = replay_get_byte();
+        break;
+    case INPUT_EVENT_KIND_REL:
+        evt.u.rel.data = &rel;
+        evt.u.rel.data->axis = (InputAxis)replay_get_dword();
+        evt.u.rel.data->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        evt.u.abs.data = &abs;
+        evt.u.abs.data->axis = (InputAxis)replay_get_dword();
+        evt.u.abs.data->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND__MAX:
+        /* keep gcc happy */
+        break;
+    }
+
+    return qapi_clone_InputEvent(&evt);
+}
+
+void replay_input_event(QemuConsole *src, InputEvent *evt)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_event(qapi_clone_InputEvent(evt));
+    } else {
+        qemu_input_event_send_impl(src, evt);
+    }
+}
+
+void replay_input_sync_event(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_sync_event();
+    } else {
+        qemu_input_event_sync_impl();
+    }
+}
diff --git a/qemu/replay/replay-internal.c b/qemu/replay/replay-internal.c
new file mode 100644
index 000000000..5835e8def
--- /dev/null
+++ b/qemu/replay/replay-internal.c
@@ -0,0 +1,207 @@
+/*
+ * replay-internal.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+
+unsigned int replay_data_kind = -1;
+static unsigned int replay_has_unread_data;
+
+/* Mutex to protect reading and writing events to the log.
+   replay_data_kind and replay_has_unread_data are also protected
+   by this mutex.
+   It also protects replay events queue which stores events to be
+   written or read to the log. */
+static QemuMutex lock;
+
+/* File for replay writing */
+FILE *replay_file;
+
+void replay_put_byte(uint8_t byte)
+{
+    if (replay_file) {
+        putc(byte, replay_file);
+    }
+}
+
+void replay_put_event(uint8_t event)
+{
+    assert(event < EVENT_COUNT);
+    replay_put_byte(event);
+}
+
+
+void replay_put_word(uint16_t word)
+{
+    replay_put_byte(word >> 8);
+    replay_put_byte(word);
+}
+
+void replay_put_dword(uint32_t dword)
+{
+    replay_put_word(dword >> 16);
+    replay_put_word(dword);
+}
+
+void replay_put_qword(int64_t qword)
+{
+    replay_put_dword(qword >> 32);
+    replay_put_dword(qword);
+}
+
+void replay_put_array(const uint8_t *buf, size_t size)
+{
+    if (replay_file) {
+        replay_put_dword(size);
+        fwrite(buf, 1, size, replay_file);
+    }
+}
+
+uint8_t replay_get_byte(void)
+{
+    uint8_t byte = 0;
+    if (replay_file) {
+        byte = getc(replay_file);
+    }
+    return byte;
+}
+
+uint16_t replay_get_word(void)
+{
+    uint16_t word = 0;
+    if (replay_file) {
+        word = replay_get_byte();
+        word = (word << 8) + replay_get_byte();
+    }
+
+    return word;
+}
+
+uint32_t replay_get_dword(void)
+{
+    uint32_t dword = 0;
+    if (replay_file) {
+        dword = replay_get_word();
+        dword = (dword << 16) + replay_get_word();
+    }
+
+    return dword;
+}
+
+int64_t replay_get_qword(void)
+{
+    int64_t qword = 0;
+    if (replay_file) {
+        qword = replay_get_dword();
+        qword = (qword << 32) + replay_get_dword();
+    }
+
+    return qword;
+}
+
+void replay_get_array(uint8_t *buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        if (fread(buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_get_array_alloc(uint8_t **buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        *buf = g_malloc(*size);
+        if (fread(*buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_check_error(void)
+{
+    if (replay_file) {
+        if (feof(replay_file)) {
+            error_report("replay file is over");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_PAUSED);
+        } else if (ferror(replay_file)) {
+            error_report("replay file is over or something goes wrong");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_INTERNAL_ERROR);
+        }
+    }
+}
+
+void replay_fetch_data_kind(void)
+{
+    if (replay_file) {
+        if (!replay_has_unread_data) {
+            replay_data_kind = replay_get_byte();
+            if (replay_data_kind == EVENT_INSTRUCTION) {
+                replay_state.instructions_count = replay_get_dword();
+            }
+            replay_check_error();
+            replay_has_unread_data = 1;
+            if (replay_data_kind >= EVENT_COUNT) {
+                error_report("Replay: unknown event kind %d", replay_data_kind);
+                exit(1);
+            }
+        }
+    }
+}
+
+void replay_finish_event(void)
+{
+    replay_has_unread_data = 0;
+    replay_fetch_data_kind();
+}
+
+void replay_mutex_init(void)
+{
+    qemu_mutex_init(&lock);
+}
+
+void replay_mutex_destroy(void)
+{
+    qemu_mutex_destroy(&lock);
+}
+
+void replay_mutex_lock(void)
+{
+    qemu_mutex_lock(&lock);
+}
+
+void replay_mutex_unlock(void)
+{
+    qemu_mutex_unlock(&lock);
+}
+
+/*! Saves cached instructions. */
+void replay_save_instructions(void)
+{
+    if (replay_file && replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        int diff = (int)(replay_get_current_step() - replay_state.current_step);
+        if (diff > 0) {
+            replay_put_event(EVENT_INSTRUCTION);
+            replay_put_dword(diff);
+            replay_state.current_step += diff;
+        }
+        replay_mutex_unlock();
+    }
+}
diff --git a/qemu/replay/replay-internal.h b/qemu/replay/replay-internal.h
new file mode 100644
index 000000000..efbf14c8a
--- /dev/null
+++ b/qemu/replay/replay-internal.h
@@ -0,0 +1,158 @@
+#ifndef REPLAY_INTERNAL_H
+#define REPLAY_INTERNAL_H
+
+/*
+ * replay-internal.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+
+enum ReplayEvents {
+    /* for instruction event */
+    EVENT_INSTRUCTION,
+    /* for software interrupt */
+    EVENT_INTERRUPT,
+    /* for emulated exceptions */
+    EVENT_EXCEPTION,
+    /* for async events */
+    EVENT_ASYNC,
+    /* for shutdown request */
+    EVENT_SHUTDOWN,
+    /* for character device write event */
+    EVENT_CHAR_WRITE,
+    /* for character device read all event */
+    EVENT_CHAR_READ_ALL,
+    EVENT_CHAR_READ_ALL_ERROR,
+    /* for clock read/writes */
+    /* some of greater codes are reserved for clocks */
+    EVENT_CLOCK,
+    EVENT_CLOCK_LAST = EVENT_CLOCK + REPLAY_CLOCK_COUNT - 1,
+    /* for checkpoint event */
+    /* some of greater codes are reserved for checkpoints */
+    EVENT_CHECKPOINT,
+    EVENT_CHECKPOINT_LAST = EVENT_CHECKPOINT + CHECKPOINT_COUNT - 1,
+    /* end of log event */
+    EVENT_END,
+    EVENT_COUNT
+};
+
+/* Asynchronous events IDs */
+
+enum ReplayAsyncEventKind {
+    REPLAY_ASYNC_EVENT_BH,
+    REPLAY_ASYNC_EVENT_INPUT,
+    REPLAY_ASYNC_EVENT_INPUT_SYNC,
+    REPLAY_ASYNC_EVENT_CHAR_READ,
+    REPLAY_ASYNC_EVENT_BLOCK,
+    REPLAY_ASYNC_COUNT
+};
+
+typedef enum ReplayAsyncEventKind ReplayAsyncEventKind;
+
+typedef struct ReplayState {
+    /*! Cached clock values. */
+    int64_t cached_clock[REPLAY_CLOCK_COUNT];
+    /*! Current step - number of processed instructions and timer events. */
+    uint64_t current_step;
+    /*! Number of instructions to be executed before other events happen. */
+    int instructions_count;
+} ReplayState;
+extern ReplayState replay_state;
+
+extern unsigned int replay_data_kind;
+
+/* File for replay writing */
+extern FILE *replay_file;
+
+void replay_put_byte(uint8_t byte);
+void replay_put_event(uint8_t event);
+void replay_put_word(uint16_t word);
+void replay_put_dword(uint32_t dword);
+void replay_put_qword(int64_t qword);
+void replay_put_array(const uint8_t *buf, size_t size);
+
+uint8_t replay_get_byte(void);
+uint16_t replay_get_word(void);
+uint32_t replay_get_dword(void);
+int64_t replay_get_qword(void);
+void replay_get_array(uint8_t *buf, size_t *size);
+void replay_get_array_alloc(uint8_t **buf, size_t *size);
+
+/* Mutex functions for protecting replay log file */
+
+void replay_mutex_init(void);
+void replay_mutex_destroy(void);
+void replay_mutex_lock(void);
+void replay_mutex_unlock(void);
+
+/*! Checks error status of the file. */
+void replay_check_error(void);
+
+/*! Finishes processing of the replayed event and fetches
+    the next event from the log. */
+void replay_finish_event(void);
+/*! Reads data type from the file and stores it in the
+    replay_data_kind variable. */
+void replay_fetch_data_kind(void);
+
+/*! Saves queued events (like instructions and sound). */
+void replay_save_instructions(void);
+
+/*! Skips async events until some sync event will be found.
+    \return true, if event was found */
+bool replay_next_event_is(int event);
+
+/*! Reads next clock value from the file.
+    If clock kind read from the file is different from the parameter,
+    the value is not used. */
+void replay_read_next_clock(unsigned int kind);
+
+/* Asynchronous events queue */
+
+/*! Initializes events' processing internals */
+void replay_init_events(void);
+/*! Clears internal data structures for events handling */
+void replay_finish_events(void);
+/*! Enables storing events in the queue */
+void replay_enable_events(void);
+/*! Flushes events queue */
+void replay_flush_events(void);
+/*! Clears events list before loading new VM state */
+void replay_clear_events(void);
+/*! Returns true if there are any unsaved events in the queue */
+bool replay_has_events(void);
+/*! Saves events from queue into the file */
+void replay_save_events(int checkpoint);
+/*! Read events from the file into the input queue */
+void replay_read_events(int checkpoint);
+/*! Adds specified async event to the queue */
+void replay_add_event(ReplayAsyncEventKind event_kind, void *opaque,
+                      void *opaque2, uint64_t id);
+
+/* Input events */
+
+/*! Saves input event to the log */
+void replay_save_input_event(InputEvent *evt);
+/*! Reads input event from the log */
+InputEvent *replay_read_input_event(void);
+/*! Adds input event to the queue */
+void replay_add_input_event(struct InputEvent *event);
+/*! Adds input sync event to the queue */
+void replay_add_input_sync_event(void);
+
+/* Character devices */
+
+/*! Called to run char device read event. */
+void replay_event_char_read_run(void *opaque);
+/*! Writes char read event to the file. */
+void replay_event_char_read_save(void *opaque);
+/*! Reads char event read from the file. */
+void *replay_event_char_read_load(void);
+
+#endif
diff --git a/qemu/replay/replay-time.c b/qemu/replay/replay-time.c
new file mode 100644
index 000000000..fffe072c5
--- /dev/null
+++ b/qemu/replay/replay-time.c
@@ -0,0 +1,65 @@
+/*
+ * replay-time.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock)
+{
+    replay_save_instructions();
+
+    if (replay_file) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_CLOCK + kind);
+        replay_put_qword(clock);
+        replay_mutex_unlock();
+    }
+
+    return clock;
+}
+
+void replay_read_next_clock(ReplayClockKind kind)
+{
+    unsigned int read_kind = replay_data_kind - EVENT_CLOCK;
+
+    assert(read_kind == kind);
+
+    int64_t clock = replay_get_qword();
+
+    replay_check_error();
+    replay_finish_event();
+
+    replay_state.cached_clock[read_kind] = clock;
+}
+
+/*! Reads next clock event from the input. */
+int64_t replay_read_clock(ReplayClockKind kind)
+{
+    replay_account_executed_instructions();
+
+    if (replay_file) {
+        int64_t ret;
+        replay_mutex_lock();
+        if (replay_next_event_is(EVENT_CLOCK + kind)) {
+            replay_read_next_clock(kind);
+        }
+        ret = replay_state.cached_clock[kind];
+        replay_mutex_unlock();
+
+        return ret;
+    }
+
+    error_report("REPLAY INTERNAL ERROR %d", __LINE__);
+    exit(1);
+}
diff --git a/qemu/replay/replay.c b/qemu/replay/replay.c
new file mode 100644
index 000000000..167fd2942
--- /dev/null
+++ b/qemu/replay/replay.c
@@ -0,0 +1,354 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/timer.h"
+#include "qemu/main-loop.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
+
+/* Current version of the replay mechanism.
+   Increase it when file format changes. */
+#define REPLAY_VERSION              0xe02004
+/* Size of replay log header */
+#define HEADER_SIZE                 (sizeof(uint32_t) + sizeof(uint64_t))
+
+ReplayMode replay_mode = REPLAY_MODE_NONE;
+
+/* Name of replay file  */
+static char *replay_filename;
+ReplayState replay_state;
+static GSList *replay_blockers;
+
+bool replay_next_event_is(int event)
+{
+    bool res = false;
+
+    /* nothing to skip - not all instructions used */
+    if (replay_state.instructions_count != 0) {
+        assert(replay_data_kind == EVENT_INSTRUCTION);
+        return event == EVENT_INSTRUCTION;
+    }
+
+    while (true) {
+        if (event == replay_data_kind) {
+            res = true;
+        }
+        switch (replay_data_kind) {
+        case EVENT_SHUTDOWN:
+            replay_finish_event();
+            qemu_system_shutdown_request();
+            break;
+        default:
+            /* clock, time_t, checkpoint and other events */
+            return res;
+        }
+    }
+    return res;
+}
+
+uint64_t replay_get_current_step(void)
+{
+    return cpu_get_icount_raw();
+}
+
+int replay_get_instructions(void)
+{
+    int res = 0;
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_INSTRUCTION)) {
+        res = replay_state.instructions_count;
+    }
+    replay_mutex_unlock();
+    return res;
+}
+
+void replay_account_executed_instructions(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_mutex_lock();
+        if (replay_state.instructions_count > 0) {
+            int count = (int)(replay_get_current_step()
+                              - replay_state.current_step);
+            replay_state.instructions_count -= count;
+            replay_state.current_step += count;
+            if (replay_state.instructions_count == 0) {
+                assert(replay_data_kind == EVENT_INSTRUCTION);
+                replay_finish_event();
+                /* Wake up iothread. This is required because
+                   timers will not expire until clock counters
+                   will be read from the log. */
+                qemu_notify_event();
+            }
+        }
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_exception(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_exception();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+    }
+
+    return res;
+}
+
+bool replay_interrupt(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_interrupt();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+    }
+    return res;
+}
+
+void replay_shutdown_request(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_SHUTDOWN);
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    bool res = false;
+    assert(EVENT_CHECKPOINT + checkpoint <= EVENT_CHECKPOINT_LAST);
+    replay_save_instructions();
+
+    if (!replay_file) {
+        return true;
+    }
+
+    replay_mutex_lock();
+
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        if (replay_next_event_is(EVENT_CHECKPOINT + checkpoint)) {
+            replay_finish_event();
+        } else if (replay_data_kind != EVENT_ASYNC) {
+            res = false;
+            goto out;
+        }
+        replay_read_events(checkpoint);
+        /* replay_read_events may leave some unread events.
+           Return false if not all of the events associated with
+           checkpoint were processed */
+        res = replay_data_kind != EVENT_ASYNC;
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_put_event(EVENT_CHECKPOINT + checkpoint);
+        replay_save_events(checkpoint);
+        res = true;
+    }
+out:
+    replay_mutex_unlock();
+    return res;
+}
+
+static void replay_enable(const char *fname, int mode)
+{
+    const char *fmode = NULL;
+    assert(!replay_file);
+
+    switch (mode) {
+    case REPLAY_MODE_RECORD:
+        fmode = "wb";
+        break;
+    case REPLAY_MODE_PLAY:
+        fmode = "rb";
+        break;
+    default:
+        fprintf(stderr, "Replay: internal error: invalid replay mode\n");
+        exit(1);
+    }
+
+    atexit(replay_finish);
+
+    replay_mutex_init();
+
+    replay_file = fopen(fname, fmode);
+    if (replay_file == NULL) {
+        fprintf(stderr, "Replay: open %s: %s\n", fname, strerror(errno));
+        exit(1);
+    }
+
+    replay_filename = g_strdup(fname);
+
+    replay_mode = mode;
+    replay_data_kind = -1;
+    replay_state.instructions_count = 0;
+    replay_state.current_step = 0;
+
+    /* skip file header for RECORD and check it for PLAY */
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        unsigned int version = replay_get_dword();
+        if (version != REPLAY_VERSION) {
+            fprintf(stderr, "Replay: invalid input log file version\n");
+            exit(1);
+        }
+        /* go to the beginning */
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+        replay_fetch_data_kind();
+    }
+
+    replay_init_events();
+}
+
+void replay_configure(QemuOpts *opts)
+{
+    const char *fname;
+    const char *rr;
+    ReplayMode mode = REPLAY_MODE_NONE;
+    Location loc;
+
+    if (!opts) {
+        return;
+    }
+
+    loc_push_none(&loc);
+    qemu_opts_loc_restore(opts);
+
+    rr = qemu_opt_get(opts, "rr");
+    if (!rr) {
+        /* Just enabling icount */
+        goto out;
+    } else if (!strcmp(rr, "record")) {
+        mode = REPLAY_MODE_RECORD;
+    } else if (!strcmp(rr, "replay")) {
+        mode = REPLAY_MODE_PLAY;
+    } else {
+        error_report("Invalid icount rr option: %s", rr);
+        exit(1);
+    }
+
+    fname = qemu_opt_get(opts, "rrfile");
+    if (!fname) {
+        error_report("File name not specified for replay");
+        exit(1);
+    }
+
+    replay_enable(fname, mode);
+
+out:
+    loc_pop(&loc);
+}
+
+void replay_start(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    if (replay_blockers) {
+        error_reportf_err(replay_blockers->data, "Record/replay: ");
+        exit(1);
+    }
+    if (!use_icount) {
+        error_report("Please enable icount to use record/replay");
+        exit(1);
+    }
+
+    /* Timer for snapshotting will be set up here. */
+
+    replay_enable_events();
+}
+
+void replay_finish(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    replay_save_instructions();
+
+    /* finalize the file */
+    if (replay_file) {
+        if (replay_mode == REPLAY_MODE_RECORD) {
+            /* write end event */
+            replay_put_event(EVENT_END);
+
+            /* write header */
+            fseek(replay_file, 0, SEEK_SET);
+            replay_put_dword(REPLAY_VERSION);
+        }
+
+        fclose(replay_file);
+        replay_file = NULL;
+    }
+    if (replay_filename) {
+        g_free(replay_filename);
+        replay_filename = NULL;
+    }
+
+    replay_finish_events();
+    replay_mutex_destroy();
+}
+
+void replay_add_blocker(Error *reason)
+{
+    replay_blockers = g_slist_prepend(replay_blockers, reason);
+}
diff --git a/qemu/roms/Makefile b/qemu/roms/Makefile
index 7b3f15632..7bd125273 100644
--- a/qemu/roms/Makefile
+++ b/qemu/roms/Makefile
@@ -35,7 +35,7 @@ powerpc_cross_prefix := $(call find-cross-prefix,powerpc)
 x86_64_cross_prefix := $(call find-cross-prefix,x86_64)
 
 # tag our seabios builds
-SEABIOS_VERSION="$(shell cd seabios; git describe --tags --long) by qemu-project.org"
+SEABIOS_EXTRAVERSION="-prebuilt.qemu-project.org"
 
 #
 # EfiRom utility is shipped with edk2 / tianocore, in BaseTools/
@@ -64,7 +64,6 @@ default:
 bios: build-seabios-config-seabios-128k build-seabios-config-seabios-256k
 	cp seabios/builds/seabios-128k/bios.bin ../pc-bios/bios.bin
 	cp seabios/builds/seabios-256k/bios.bin ../pc-bios/bios-256k.bin
-	cp seabios/builds/seabios-256k/src/fw/*dsdt.aml ../pc-bios/
 
 seavgabios: $(patsubst %,seavgabios-%,$(vgabios_variants))
 
@@ -78,12 +77,12 @@ build-seabios-config-%: config.%
 	mkdir -p seabios/builds/$*
 	cp $< seabios/builds/$*/.config
 	$(MAKE) -C seabios \
-		VERSION=$(SEABIOS_VERSION) \
+		EXTRAVERSION=$(SEABIOS_EXTRAVERSION) \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
 		OUT=$(CURDIR)/seabios/builds/$*/ oldnoconfig
 	$(MAKE) -C seabios \
-		VERSION=$(SEABIOS_VERSION) \
+		EXTRAVERSION=$(SEABIOS_EXTRAVERSION) \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
 		OUT=$(CURDIR)/seabios/builds/$*/ all
@@ -120,20 +119,17 @@ efi-rom-%: build-pxe-roms build-efi-roms
 		-ec ipxe/src/bin-x86_64-efi/$(VID)$(DID).efidrv \
 		-o ../pc-bios/efi-$*.rom
 
-build-pxe-roms: ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-pxe-roms:
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin/%.rom,$(pxerom_targets))
 
-build-efi-roms: build-pxe-roms ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-efi-roms: build-pxe-roms
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin-i386-efi/%.efidrv,$(pxerom_targets)) \
 		$(patsubst %,bin-x86_64-efi/%.efidrv,$(pxerom_targets))
 
-ipxe/src/config/local/%: config.ipxe.%
-	cp $< $@
-
 
 slof:
 	$(MAKE) -C SLOF CROSS=$(powerpc64_cross_prefix) qemu
diff --git a/qemu/roms/SLOF/README b/qemu/roms/SLOF/README
index 58e929427..294458880 100644
--- a/qemu/roms/SLOF/README
+++ b/qemu/roms/SLOF/README
@@ -11,6 +11,8 @@ Index
 2.2 Overview of the source code
 2.4 Extending the Forth engine
 3.0 Limitations
+4.0 Submitting patches
+5.0 Coding style
 
 
 1.0 Introduction to Slimline Open Firmware
@@ -236,6 +238,34 @@ To add primitives:
  On a JS21 all memory configurations should work.
 
 
+4.0 Submitting patches
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Patches for SLOF should be made against https://github.com/aik/SLOF,
+the master branch and posted to slof@lists.ozlabs.org.
+The patches must be signed using "Signed-off-by" tag with a real name to
+confirm that you certify the Developer Certificate of Origin  Version 1.1,
+see [3] for details.
+
+
+5.0 Coding style
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+New C code submitted to SLOF should follow the coding style guidelines
+for the Linux kernel [4] with the following exceptions:
+
+- in the event that you require a specific width, use a standard type
+  like int32_t, uint32_t, uint64_t, etc. Don't use Linux kernel internal
+  types like u32, __u32 or __le32.
+
+New Forth code should use 4 space indentations and no tabs. Patches for
+the old code should keep the existing style which usually is
+3 space indentation.
+
+New assembly code submitted to SLOF should follow the coding style
+guidelines for the Linux kernel [4], i.e. indent with tabs, not with spaces.
+
+
 Documentation
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -244,3 +274,9 @@ Documentation
 
 [2] PAPR Standard, Power.org(TM) Standard for Power Architecture(R) Platform
     Requirements (Workstation, Server), Version 2.4, December 7, 2009
+
+[3] Developer Certificate of Origin Version 1.1
+    http://developercertificate.org/
+
+[4] Linux kernel coding style
+    https://github.com/torvalds/linux/blob/master/Documentation/CodingStyle
diff --git a/qemu/roms/SLOF/VERSION b/qemu/roms/SLOF/VERSION
index 20bdb2eb7..aded5537b 100644
--- a/qemu/roms/SLOF/VERSION
+++ b/qemu/roms/SLOF/VERSION
@@ -1 +1 @@
-20150429
+20160223
diff --git a/qemu/roms/SLOF/board-js2x/llfw/stage2.lds b/qemu/roms/SLOF/board-js2x/llfw/stage2.lds
index f91f0658a..e6315c3c8 100644
--- a/qemu/roms/SLOF/board-js2x/llfw/stage2.lds
+++ b/qemu/roms/SLOF/board-js2x/llfw/stage2.lds
@@ -45,7 +45,8 @@ SECTIONS {
 	__bss_end = .;
 	__bss_size = (__bss_end - __bss_start);
 
-	__toc_start = .;
+	. = ALIGN(256);
+	__toc_start = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	.got       :
 	{
 		 *(.toc .got)
diff --git a/qemu/roms/SLOF/board-js2x/llfw/stage2_head.S b/qemu/roms/SLOF/board-js2x/llfw/stage2_head.S
index 5460bfebb..f3f5e0c8c 100644
--- a/qemu/roms/SLOF/board-js2x/llfw/stage2_head.S
+++ b/qemu/roms/SLOF/board-js2x/llfw/stage2_head.S
@@ -79,8 +79,6 @@ bsscdone:
 	/* ------------------------------------ */
 ASM_ENTRY(toc_init)	
 	LOAD64(r2, __toc_start)
-	addi	r2,r2,0x4000
-	addi	r2,r2,0x4000
 	blr
 
 	/* ------------------------------------ */
diff --git a/qemu/roms/SLOF/board-js2x/slof/Makefile b/qemu/roms/SLOF/board-js2x/slof/Makefile
index ab3e683a4..4cdd5fa36 100644
--- a/qemu/roms/SLOF/board-js2x/slof/Makefile
+++ b/qemu/roms/SLOF/board-js2x/slof/Makefile
@@ -57,6 +57,7 @@ OF_FFS_FILES = \
 	$(SLOFBRDDIR)/attu.fs \
 	$(SLOFBRDDIR)/cpu.fs \
 	$(SLOFBRDDIR)/ioapic.fs \
+	$(SLOFBRDDIR)/dma-function.fs \
 	$(SLOFBRDDIR)/pci-bridge_1022_7460.fs \
 	$(SLOFBRDDIR)/pci-device_1014_028c.fs \
 	$(SLOFBRDDIR)/pci-device_1014_02bd.fs \
diff --git a/qemu/roms/SLOF/board-js2x/slof/dma-function.fs b/qemu/roms/SLOF/board-js2x/slof/dma-function.fs
new file mode 100644
index 000000000..2e314cdaa
--- /dev/null
+++ b/qemu/roms/SLOF/board-js2x/slof/dma-function.fs
@@ -0,0 +1,31 @@
+\ *****************************************************************************
+\ * Copyright (c) 2004, 2011 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+: dma-alloc ( ... size -- virt )
+   \ ." dma-alloc called: " .s cr
+   alloc-mem
+;
+
+: dma-free ( virt size -- )
+   \ ." dma-free called: " .s cr
+   free-mem
+;
+
+: dma-map-in ( ... virt size cacheable? -- devaddr )
+   \ ." dma-map-in called: " .s cr
+   2drop
+;
+
+: dma-map-out ( virt devaddr size -- )
+   \ ." dma-map-out called: " .s cr
+   2drop drop
+;
diff --git a/qemu/roms/SLOF/board-js2x/slof/helper.fs b/qemu/roms/SLOF/board-js2x/slof/helper.fs
index 34d60da1f..1e2b03063 100644
--- a/qemu/roms/SLOF/board-js2x/slof/helper.fs
+++ b/qemu/roms/SLOF/board-js2x/slof/helper.fs
@@ -26,3 +26,14 @@
    s" , " $cat
    bdate2human $cat encode-string THEN
 ;
+
+: invert-region ( addr len -- )
+   2dup or 7 and CASE
+      0 OF 3 rshift 0 ?DO dup dup rx@ -1 xor swap rx! xa1+ LOOP ENDOF
+      4 OF 2 rshift 0 ?DO dup dup rl@ -1 xor swap rl! la1+ LOOP ENDOF
+      3 and
+      2 OF 1 rshift 0 ?DO dup dup rw@ -1 xor swap rw! wa1+ LOOP ENDOF
+      dup OF 0 ?DO dup dup rb@ -1 xor swap rb! 1+ LOOP ENDOF
+   ENDCASE
+   drop
+;
diff --git a/qemu/roms/SLOF/board-qemu/llfw/stage2.lds b/qemu/roms/SLOF/board-qemu/llfw/stage2.lds
index e060dd189..28c9dca93 100644
--- a/qemu/roms/SLOF/board-qemu/llfw/stage2.lds
+++ b/qemu/roms/SLOF/board-qemu/llfw/stage2.lds
@@ -49,7 +49,8 @@ SECTIONS {
 	__bss_end = .;
 	__bss_size = (__bss_end - __bss_start);
 
-	__toc_start = .;
+	. = ALIGN(256);
+	__toc_start = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	.got :
 	{
 		 *(.toc .got)
diff --git a/qemu/roms/SLOF/board-qemu/llfw/stage2_head.S b/qemu/roms/SLOF/board-qemu/llfw/stage2_head.S
index c56b117ce..adf75547b 100644
--- a/qemu/roms/SLOF/board-qemu/llfw/stage2_head.S
+++ b/qemu/roms/SLOF/board-qemu/llfw/stage2_head.S
@@ -77,8 +77,6 @@ bsscdone:
 	/* ------------------------------------ */
 ASM_ENTRY(toc_init)
 	LOAD64(r2, __toc_start)
-	addi	r2,r2,0x4000
-	addi	r2,r2,0x4000
 	blr
 
 	/* ------------------------------------ */
diff --git a/qemu/roms/SLOF/board-qemu/slof/Makefile b/qemu/roms/SLOF/board-qemu/slof/Makefile
index 283f77d32..a000a2519 100644
--- a/qemu/roms/SLOF/board-qemu/slof/Makefile
+++ b/qemu/roms/SLOF/board-qemu/slof/Makefile
@@ -66,9 +66,13 @@ USB_FFS_FILES = \
 
 VIO_FFS_FILES = \
 	$(SLOFBRDDIR)/pci-device_1af4_1000.fs \
+	$(SLOFBRDDIR)/pci-device_1af4_1041.fs \
 	$(SLOFBRDDIR)/pci-device_1af4_1001.fs \
+	$(SLOFBRDDIR)/pci-device_1af4_1042.fs \
 	$(SLOFBRDDIR)/pci-device_1af4_1004.fs \
+	$(SLOFBRDDIR)/pci-device_1af4_1048.fs \
 	$(SLOFBRDDIR)/pci-device_1af4_1009.fs \
+	$(SLOFBRDDIR)/pci-device_1af4_1050.fs \
 	$(SLOFBRDDIR)/vio-hvterm.fs \
 	$(SLOFBRDDIR)/vio-vscsi.fs \
 	$(SLOFBRDDIR)/vio-veth.fs \
@@ -103,6 +107,7 @@ OF_FFS_FILES = \
 	$(SLOFBRDDIR)/pci-device_1013_00b8.fs \
 	$(SLOFBRDDIR)/pci-device_8086_100e.fs \
 	$(SLOFBRDDIR)/e1k.fs \
+	$(SLOFBRDDIR)/qemu-vga.fs \
 	$(FCODE_FFS_FILES)
 
 # Uncomment the following line to enable the USB code:
diff --git a/qemu/roms/SLOF/board-qemu/slof/OF.fs b/qemu/roms/SLOF/board-qemu/slof/OF.fs
index 561d89225..69ee5c122 100644
--- a/qemu/roms/SLOF/board-qemu/slof/OF.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/OF.fs
@@ -134,10 +134,6 @@ check-boot-menu
 \ Grab rtas from qemu
 #include "rtas.fs"
 
-390 cp
-
-#include "virtio.fs"
-
 3f0 cp
 
 #include "tree.fs"
diff --git a/qemu/roms/SLOF/slof/fs/archsupport.fs b/qemu/roms/SLOF/board-qemu/slof/archsupport.fs
index cc4668769..a8ace3cc5 100644
--- a/qemu/roms/SLOF/slof/fs/archsupport.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/archsupport.fs
@@ -10,15 +10,16 @@
 \ *     IBM Corporation - initial implementation
 \ ****************************************************************************/
 
-\ Qemu supports max 256cpus, 32K will be able to accomodate the fdt changes if
-\ needed.
-8000 VALUE size
+\ 2 MiB FDT buffer size is enough to accommodate 255 CPU cores
+\ and 16 TiB of maxmem specification.
+200000 CONSTANT cas-buffer-size
 : ibm,client-architecture-support         ( vec -- err? )
     \ Store require parameters in nvram
     \ to come back to right boot device
     \ Allocate memory for H_CALL
-    size alloc-mem                        ( vec memaddr )
-    swap over size                        ( memaddr vec memaddr size )
+    cas-buffer-size alloc-mem             ( vec memaddr )
+    dup 0= IF ." out of memory during ibm,client-architecture-support" cr THEN
+    swap over cas-buffer-size             ( memaddr vec memaddr size )
     \ make h_call to hypervisor
     hv-cas 0= IF                          ( memaddr )
 	dup l@ 1 >= IF                    \ Version number >= 1
@@ -34,5 +35,5 @@
     ELSE
 	TRUE
     THEN
-    >r size free-mem r>
+    >r cas-buffer-size free-mem r>
 ;
diff --git a/qemu/roms/SLOF/board-qemu/slof/helper.fs b/qemu/roms/SLOF/board-qemu/slof/helper.fs
index 96da49894..40d4abc3a 100644
--- a/qemu/roms/SLOF/board-qemu/slof/helper.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/helper.fs
@@ -33,3 +33,16 @@
   swap -
 ;
 
+: invert-region-cs ( addr len cellsize -- )
+   >r over swap r@ rshift r> swap 1 hv-logical-memop drop
+;
+
+: invert-region ( addr len -- )
+   2dup or 7 and CASE
+      0 OF 3 invert-region-cs ENDOF
+      4 OF 2 invert-region-cs ENDOF
+      3 and
+      2 OF 1 invert-region-cs ENDOF
+      dup OF 0 invert-region-cs ENDOF
+   ENDCASE
+;
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
index a5c3584f9..22ea45d5c 100644
--- a/qemu/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1234_1111.fs
@@ -10,233 +10,6 @@
 \ *     IBM Corporation - initial implementation
 \ ****************************************************************************/
 
-my-space pci-device-generic-setup
-
-\ Defaults, overriden from qemu
-d# 800 VALUE disp-width
-d# 600 VALUE disp-height
-d#   8 VALUE disp-depth
-
-\ Determine base address
-10 config-l@ translate-my-address f not AND VALUE fb-base
-
-\ Fixed up later
--1 VALUE io-base
-
-\ We support only one instance
-false VALUE is-installed?
-
-: vga-io-xlate ( port -- addr )
-  io-base -1 = IF
-    dup translate-my-address fff not and to io-base
-  THEN
-  io-base +
-;
-
-: vga-w! ( value port -- )
-  vga-io-xlate rw!-le
-;
-
-: vga-w@ ( port -- value )
-  vga-io-xlate rw@-le
-;
-
-: vga-b! ( value port -- )
-  vga-io-xlate rb!
-;
-
-: vga-b@ ( port -- value )
-  vga-io-xlate rb@
-;
-
-: vbe!	( value index -- )
-  1ce vga-w! 1d0 vga-w!
-;
-
-: vbe@	( index -- value )
-  1ce vga-w! 1d0 vga-w@
-;
-
-: color! ( r g b number -- ) 
-   3c8 vga-b!
-   rot 3c9 vga-b!
-   swap 3c9 vga-b!
-   3c9 vga-b!
-;
-
-: color@ ( number -- r g b ) 
-   3c8 vga-b!
-   3c9 vga-b@
-   3c9 vga-b@
-   3c9 vga-b@
-;
-
-: set-colors ( adr number #numbers -- )
-   over 3c8 vga-b!
-   swap DO
-     rb@ 3c9 vga-b!
-     rb@ 3c9 vga-b!
-     rb@ 3c9 vga-b!
-   LOOP
-   3drop
-;
-
-: get-colors ( adr number #numbers -- )
-   3drop
-;
-
-include graphics.fs
-
-\ qemu fake VBE IO registers
-0 CONSTANT VBE_DISPI_INDEX_ID
-1 CONSTANT VBE_DISPI_INDEX_XRES
-2 CONSTANT VBE_DISPI_INDEX_YRES
-3 CONSTANT VBE_DISPI_INDEX_BPP
-4 CONSTANT VBE_DISPI_INDEX_ENABLE
-5 CONSTANT VBE_DISPI_INDEX_BANK
-6 CONSTANT VBE_DISPI_INDEX_VIRT_WIDTH
-7 CONSTANT VBE_DISPI_INDEX_VIRT_HEIGHT
-8 CONSTANT VBE_DISPI_INDEX_X_OFFSET
-9 CONSTANT VBE_DISPI_INDEX_Y_OFFSET
-a CONSTANT VBE_DISPI_INDEX_NB
-
-\ ENABLE register
-00 CONSTANT VBE_DISPI_DISABLED
-01 CONSTANT VBE_DISPI_ENABLED
-02 CONSTANT VBE_DISPI_GETCAPS
-20 CONSTANT VBE_DISPI_8BIT_DAC
-40 CONSTANT VBE_DISPI_LFB_ENABLED
-80 CONSTANT VBE_DISPI_NOCLEARMEM
-
-: init-mode
-  0 3c0 vga-b!
-  VBE_DISPI_DISABLED VBE_DISPI_INDEX_ENABLE vbe!
-  0 VBE_DISPI_INDEX_X_OFFSET vbe!
-  0 VBE_DISPI_INDEX_Y_OFFSET vbe!
-  disp-width VBE_DISPI_INDEX_XRES vbe!
-  disp-height VBE_DISPI_INDEX_YRES vbe!
-  disp-depth VBE_DISPI_INDEX_BPP vbe!
-  VBE_DISPI_ENABLED VBE_DISPI_8BIT_DAC or VBE_DISPI_INDEX_ENABLE vbe!
-  0 3c0 vga-b!
-  20 3c0 vga-b!
-;
-
-: clear-screen
-  fb-base disp-width disp-height disp-depth 7 + 8 / * * 0 rfill
-;
-
-: read-settings
-  s" qemu,graphic-width" get-chosen IF
-     decode-int to disp-width 2drop
-  THEN
-  s" qemu,graphic-height" get-chosen IF
-     decode-int to disp-height 2drop
-  THEN
-  s" qemu,graphic-depth" get-chosen IF
-     decode-int nip nip     
-       dup 8 =
-       over f = or
-       over 10 = or
-       over 20 = or IF
-         to disp-depth
-       ELSE
-         ." Unsupported bit depth, using 8bpp " drop cr
-       THEN
-  THEN
-;
-
-: add-legacy-reg
-  \ add legacy I/O Ports / Memory regions to assigned-addresses
-  \ see PCI Bus Binding Revision 2.1 Section 7.
-  s" reg" get-node get-property IF
-    \ "reg" does not exist, create new
-    encode-start
-  ELSE
-    \ "reg" does exist, copy it 
-    encode-bytes
-  THEN
-  \ I/O Range 0x1ce-0x1d2
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  1ce encode-64+ 4 encode-64+ \ addr size
-  \ I/O Range 0x3B0-0x3BB
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  3b0 encode-64+ c encode-64+ \ addr size
-  \ I/O Range 0x3C0-0x3DF
-  my-space a1000000 or encode-int+ \ non-relocatable, aliased I/O space
-  3c0 encode-64+ 20 encode-64+ \ addr size
-  \ Memory Range 0xA0000-0xBFFFF
-  my-space a2000000 or encode-int+ \ non-relocatable, <1MB Memory space
-  a0000 encode-64+ 20000 encode-64+ \ addr size
-  s" reg" property \ store "reg" property
-;
-
-: setup-properties
-   \ Shouldn't this be done from open ?
-   disp-width encode-int s" width" property
-   disp-height encode-int s" height" property
-   disp-width disp-depth 7 + 8 / * encode-int s" linebytes" property
-   disp-depth encode-int s" depth" property
-   s" ISO8859-1" encode-string s" character-set" property \ i hope this is ok...
-   \ add "device_type" property
-   s" display" device-type
-   s" qemu,std-vga" encode-string s" compatible" property
-   \ XXX We don't create an "address" property because Linux doesn't know what
-   \ to do with it for >32-bit
-;
-
-\ words for installation/removal, needed by is-install/is-remove, see display.fs
-: display-remove ( -- ) 
-;
-
-: hcall-invert-screen ( -- )
-    frame-buffer-adr frame-buffer-adr 3
-    screen-height screen-width * screen-depth * /x /
-    1 hv-logical-memop
-    drop
-;
-
-: hcall-blink-screen ( -- )
-    \ 32 msec delay for visually noticing the blink
-    hcall-invert-screen 20 ms hcall-invert-screen
-;
-
-: display-install ( -- )
-    is-installed? NOT IF
-        ." Installing QEMU fb" cr
-        fb-base to frame-buffer-adr
-        clear-screen
-        default-font 
-        set-font
-        disp-width disp-height
-        disp-width char-width / disp-height char-height /
-        disp-depth 7 + 8 /                      ( width height #lines #cols depth )
-        fb-install
-	['] hcall-invert-screen to invert-screen
-	['] hcall-blink-screen to blink-screen
-         true to is-installed?
-    THEN
-;
-
-: set-alias
-    s" screen" find-alias 0= IF
-      \ no previous screen alias defined, define it...
-      s" screen" get-node node>path set-alias
-    ELSE
-       drop
-    THEN 
-;
-
-
 ." qemu vga" cr
 
-pci-master-enable
-pci-mem-enable
-pci-io-enable
-add-legacy-reg
-read-settings
-init-mode
-init-default-palette
-setup-properties
-' display-install is-install
-' display-remove is-remove
-set-alias
+s" qemu-vga.fs" included
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1001.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1001.fs
index fb2463467..db0bb3fb8 100644
--- a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1001.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1001.fs
@@ -22,13 +22,4 @@ pci-io-enable
 
 s" virtio-block.fs" included
 
-\ Allocate memory for virtio queue:
-virtiodev 0 virtio-get-qsize virtio-vring-size
-1000 CLAIM VALUE queue-addr
-
-\ Write queue address into device:
-queue-addr c rshift
-virtiodev vd>base @ 8 +
-rl!-le
-
 pci-device-disable
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1009.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1009.fs
index 03964a6db..9c8be249d 100644
--- a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1009.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1009.fs
@@ -22,13 +22,4 @@ pci-io-enable
 
 s" virtio-fs.fs" included
 
-\ Allocate memory for virtio queue:
-virtiodev 0 virtio-get-qsize virtio-vring-size
-1000 CLAIM VALUE queue-addr
-
-\ Write queue address into device:
-queue-addr c rshift
-virtiodev vd>base @ 8 +
-rl!-le
-
 pci-device-disable
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1041.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1041.fs
new file mode 100644
index 000000000..552b0ef8a
--- /dev/null
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1041.fs
@@ -0,0 +1,15 @@
+\ *****************************************************************************
+\ * Copyright (c) 2016 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+\ Device ID 1041 is for virtio-net non-transitional device.
+\ Include the driver for virtio-net
+s" pci-device_1af4_1000.fs" included
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1042.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1042.fs
new file mode 100644
index 000000000..2b0a848d4
--- /dev/null
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1042.fs
@@ -0,0 +1,15 @@
+\ *****************************************************************************
+\ * Copyright (c) 2016 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+\ Device ID 1042 is for virtio-blk non-transitional device.
+\ Include the driver for virtio-blk
+s" pci-device_1af4_1001.fs" included
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1048.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1048.fs
new file mode 100644
index 000000000..055ad8960
--- /dev/null
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1048.fs
@@ -0,0 +1,15 @@
+\ *****************************************************************************
+\ * Copyright (c) 2016 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+\ Device ID 1048 is for virtio-scsi non-transitional device.
+\ Include the driver for virtio-scsi
+s" pci-device_1af4_1004.fs" included
diff --git a/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs
new file mode 100644
index 000000000..516056aad
--- /dev/null
+++ b/qemu/roms/SLOF/board-qemu/slof/pci-device_1af4_1050.fs
@@ -0,0 +1,15 @@
+\ *****************************************************************************
+\ * Copyright (c) 2015 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+s" virtio [ vga ]" type cr
+
+s" qemu-vga.fs" included
diff --git a/qemu/roms/SLOF/board-qemu/slof/qemu-vga.fs b/qemu/roms/SLOF/board-qemu/slof/qemu-vga.fs
new file mode 100644
index 000000000..3f4c237fc
--- /dev/null
+++ b/qemu/roms/SLOF/board-qemu/slof/qemu-vga.fs
@@ -0,0 +1,198 @@
+\ *****************************************************************************
+\ * Copyright (c) 2015 IBM Corporation
+\ * All rights reserved.
+\ * This program and the accompanying materials
+\ * are made available under the terms of the BSD License
+\ * which accompanies this distribution, and is available at
+\ * http://www.opensource.org/licenses/bsd-license.php
+\ *
+\ * Contributors:
+\ *     IBM Corporation - initial implementation
+\ ****************************************************************************/
+
+my-space pci-device-generic-setup
+
+\ Defaults, overriden from qemu
+d# 800 VALUE disp-width
+d# 600 VALUE disp-height
+d#   8 VALUE disp-depth
+
+: map-in  " map-in" my-phandle parent $call-static ;
+: map-out  " map-out" my-phandle parent $call-static ;
+
+\ Determine base address
+0 0  my-space h# 02000010 + 1  map-in VALUE fb-base
+0 0  my-space h# 02000018 + 1 map-in VALUE reg-base
+
+\ We support only one instance
+false VALUE is-installed?
+
+: vga-w! ( value port -- )
+  3c0 - reg-base 400 + + rw!-le
+;
+
+: vga-w@ ( port -- value )
+  3c0 - reg-base 400 + + rw@-le
+;
+
+: vga-b! ( value port -- )
+  3c0 - reg-base 400 + + rb!
+;
+
+: vga-b@ ( port -- value )
+  3c0 - reg-base 400 + + rb@
+;
+
+: vbe!	( value index -- )
+  1 << reg-base 500 + + rw!-le
+;
+
+: vbe@	( index -- value )
+  1 << reg-base 500 + + rw@-le
+;
+
+: color! ( r g b number -- )
+   3c8 vga-b!
+   rot 3c9 vga-b!
+   swap 3c9 vga-b!
+   3c9 vga-b!
+;
+
+: color@ ( number -- r g b )
+   3c8 vga-b!
+   3c9 vga-b@
+   3c9 vga-b@
+   3c9 vga-b@
+;
+
+: set-colors ( adr number #numbers -- )
+   over 3c8 vga-b!
+   swap DO
+     rb@ 3c9 vga-b!
+     rb@ 3c9 vga-b!
+     rb@ 3c9 vga-b!
+   LOOP
+   3drop
+;
+
+: get-colors ( adr number #numbers -- )
+   3drop
+;
+
+include graphics.fs
+
+\ qemu fake VBE IO registers
+0 CONSTANT VBE_DISPI_INDEX_ID
+1 CONSTANT VBE_DISPI_INDEX_XRES
+2 CONSTANT VBE_DISPI_INDEX_YRES
+3 CONSTANT VBE_DISPI_INDEX_BPP
+4 CONSTANT VBE_DISPI_INDEX_ENABLE
+5 CONSTANT VBE_DISPI_INDEX_BANK
+6 CONSTANT VBE_DISPI_INDEX_VIRT_WIDTH
+7 CONSTANT VBE_DISPI_INDEX_VIRT_HEIGHT
+8 CONSTANT VBE_DISPI_INDEX_X_OFFSET
+9 CONSTANT VBE_DISPI_INDEX_Y_OFFSET
+a CONSTANT VBE_DISPI_INDEX_NB
+
+\ ENABLE register
+00 CONSTANT VBE_DISPI_DISABLED
+01 CONSTANT VBE_DISPI_ENABLED
+02 CONSTANT VBE_DISPI_GETCAPS
+20 CONSTANT VBE_DISPI_8BIT_DAC
+40 CONSTANT VBE_DISPI_LFB_ENABLED
+80 CONSTANT VBE_DISPI_NOCLEARMEM
+
+: init-mode
+  0 3c0 vga-b!
+  VBE_DISPI_DISABLED VBE_DISPI_INDEX_ENABLE vbe!
+  0 VBE_DISPI_INDEX_X_OFFSET vbe!
+  0 VBE_DISPI_INDEX_Y_OFFSET vbe!
+  disp-width VBE_DISPI_INDEX_XRES vbe!
+  disp-height VBE_DISPI_INDEX_YRES vbe!
+  disp-depth VBE_DISPI_INDEX_BPP vbe!
+  VBE_DISPI_ENABLED VBE_DISPI_8BIT_DAC or VBE_DISPI_INDEX_ENABLE vbe!
+  0 3c0 vga-b!
+  20 3c0 vga-b!
+;
+
+: clear-screen
+  fb-base disp-width disp-height disp-depth 7 + 8 / * * 0 rfill
+;
+
+: read-settings
+  s" qemu,graphic-width" get-chosen IF
+     decode-int to disp-width 2drop
+  THEN
+  s" qemu,graphic-height" get-chosen IF
+     decode-int to disp-height 2drop
+  THEN
+  s" qemu,graphic-depth" get-chosen IF
+     decode-int nip nip
+       dup 8 =
+       over f = or
+       over 10 = or
+       over 20 = or IF
+         to disp-depth
+       ELSE
+         ." Unsupported bit depth, using 8bpp " drop cr
+       THEN
+  THEN
+;
+
+: setup-properties
+   \ Shouldn't this be done from open ?
+   disp-width encode-int s" width" property
+   disp-height encode-int s" height" property
+   disp-width disp-depth 7 + 8 / * encode-int s" linebytes" property
+   disp-depth encode-int s" depth" property
+   s" ISO8859-1" encode-string s" character-set" property \ i hope this is ok...
+   \ add "device_type" property
+   s" display" device-type
+   s" qemu,std-vga" encode-string s" compatible" property
+   \ XXX We don't create an "address" property because Linux doesn't know what
+   \ to do with it for >32-bit
+;
+
+\ words for installation/removal, needed by is-install/is-remove, see display.fs
+: display-remove ( -- )
+;
+
+: slow-blink-screen ( -- )
+    \ 32 msec delay for visually noticing the blink
+    invert-screen 20 ms invert-screen
+;
+
+: display-install ( -- )
+    is-installed? NOT IF
+        ." Installing QEMU fb" cr
+        fb-base to frame-buffer-adr
+        clear-screen
+        default-font
+        set-font
+        disp-width disp-height
+        disp-width char-width / disp-height char-height /
+        disp-depth 7 + 8 /                      ( width height #lines #cols depth )
+        fb-install
+	['] slow-blink-screen to blink-screen
+	true to is-installed?
+    THEN
+;
+
+: set-alias
+    s" screen" find-alias 0= IF
+      \ no previous screen alias defined, define it...
+      s" screen" get-node node>path set-alias
+    ELSE
+       drop
+    THEN
+;
+
+pci-master-enable
+pci-mem-enable
+read-settings
+init-mode
+init-default-palette
+setup-properties
+' display-install is-install
+' display-remove is-remove
+set-alias
diff --git a/qemu/roms/SLOF/board-qemu/slof/tree.fs b/qemu/roms/SLOF/board-qemu/slof/tree.fs
index 4aba4c53f..78dafab71 100644
--- a/qemu/roms/SLOF/board-qemu/slof/tree.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/tree.fs
@@ -26,6 +26,8 @@
 \ 2 encode-int s" #size-cells" property
 \ s" chrp" device-type
 
+#include "archsupport.fs"
+
 480 cp
 
 \ See 3.6.5, and the PowerPC OF binding document.
diff --git a/qemu/roms/SLOF/board-qemu/slof/virtio-block.fs b/qemu/roms/SLOF/board-qemu/slof/virtio-block.fs
index ea388fb00..bc9013eea 100644
--- a/qemu/roms/SLOF/board-qemu/slof/virtio-block.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/virtio-block.fs
@@ -23,8 +23,7 @@ FALSE VALUE initialized?
 
 INSTANCE VARIABLE deblocker
 
-/vd-len BUFFER: virtiodev
-virtiodev virtio-setup-vd
+virtio-setup-vd VALUE virtiodev
 
 \ Quiesce the virtqueue of this device so that no more background
 \ transactions can be pending.
diff --git a/qemu/roms/SLOF/board-qemu/slof/virtio-fs.fs b/qemu/roms/SLOF/board-qemu/slof/virtio-fs.fs
index 8632b465f..3898d0b7d 100644
--- a/qemu/roms/SLOF/board-qemu/slof/virtio-fs.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/virtio-fs.fs
@@ -20,8 +20,7 @@ FALSE VALUE initialized?
 
 2000 CONSTANT VIRTFS-BUF-SIZE \ 8k
 
-/vd-len BUFFER: virtiodev
-virtiodev virtio-setup-vd
+virtio-setup-vd VALUE virtiodev
 
 \
 \ Support methods.
diff --git a/qemu/roms/SLOF/board-qemu/slof/virtio-net.fs b/qemu/roms/SLOF/board-qemu/slof/virtio-net.fs
index 412b34fa6..b16fffe39 100644
--- a/qemu/roms/SLOF/board-qemu/slof/virtio-net.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/virtio-net.fs
@@ -16,20 +16,28 @@ s" network" device-type
 
 INSTANCE VARIABLE obp-tftp-package
 
-/vd-len BUFFER: virtiodev
-virtiodev virtio-setup-vd
+virtio-setup-vd VALUE virtiodev
 0 VALUE virtio-net-priv
 0 VALUE open-count
 
+\ Set up MAC address from config virtqueue
+6 BUFFER: local-mac
+: setup-mac  ( -- )
+   s" local-mac-address" get-node get-property not IF 2drop EXIT THEN
+   6 0 DO
+      virtiodev i 1 virtio-get-config
+      local-mac i + c!
+   LOOP
+   local-mac 6 encode-bytes  s" local-mac-address"  property
+;
+
 : open  ( -- okay? )
    open-count 0= IF
       open IF
          \ my-unit 1 rtas-set-tce-bypass
-         s" local-mac-address" get-node get-property not IF
-            virtiodev virtio-net-open dup not IF ." virtio-net-open failed" EXIT THEN
-            drop TO virtio-net-priv
-         THEN
-         true
+         virtiodev virtio-net-open not IF ." virtio-net-open failed" false EXIT THEN
+         TO virtio-net-priv
+         setup-mac true
       ELSE
          false
       THEN
@@ -77,17 +85,6 @@ virtiodev virtio-setup-vd
    s" ping" obp-tftp-package @ $call-method
 ;
 
-\ Set up MAC address from config virtqueue
-6 BUFFER: local-mac
-: setup-mac  ( -- )
-   6 0 DO
-      virtiodev i 1 virtio-get-config
-      local-mac i + c!
-   LOOP
-   local-mac 6 encode-bytes  s" local-mac-address"  property
-;
-setup-mac
-
 : setup-alias  ( -- )
    " net" get-next-alias ?dup IF
       get-node node>path set-alias
diff --git a/qemu/roms/SLOF/board-qemu/slof/virtio-scsi.fs b/qemu/roms/SLOF/board-qemu/slof/virtio-scsi.fs
index ca5fb13aa..4fedeeeb1 100644
--- a/qemu/roms/SLOF/board-qemu/slof/virtio-scsi.fs
+++ b/qemu/roms/SLOF/board-qemu/slof/virtio-scsi.fs
@@ -22,8 +22,7 @@ FALSE CONSTANT virtio-scsi-debug
 
 FALSE VALUE initialized?
 
-/vd-len BUFFER: virtiodev
-virtiodev virtio-setup-vd
+virtio-setup-vd VALUE virtiodev
 
 STRUCT \ virtio-scsi-config
     /l FIELD vs-cfg>num-queues
diff --git a/qemu/roms/SLOF/board-qemu/slof/virtio.fs b/qemu/roms/SLOF/board-qemu/slof/virtio.fs
deleted file mode 100644
index 818c1320e..000000000
--- a/qemu/roms/SLOF/board-qemu/slof/virtio.fs
+++ /dev/null
@@ -1,35 +0,0 @@
-\ *****************************************************************************
-\ * Copyright (c) 2011 IBM Corporation
-\ * All rights reserved.
-\ * This program and the accompanying materials
-\ * are made available under the terms of the BSD License
-\ * which accompanies this distribution, and is available at
-\ * http://www.opensource.org/licenses/bsd-license.php
-\ *
-\ * Contributors:
-\ *     IBM Corporation - initial implementation
-\ ****************************************************************************/
-
-\ This struct must match "struct virtio_device" in virtio.h!
-STRUCT
-   /n FIELD vd>base
-   /l FIELD vd>type
-CONSTANT /vd-len
-
-
-\ Initialize virtiodev structure for the current node
-: virtio-setup-vd  ( vdstruct -- )
-   >r
-   \ Does it have a "class-code" property? If yes, assume we're a PCI device
-   s" class-code" get-node get-property 0= IF
-      \ Set up for PCI device interface
-      2drop
-      s" 10 config-l@ translate-my-address 3 not AND" evaluate
-      ( io-base ) r@ vd>base !
-      0 r@ vd>type l!
-   ELSE
-      ." unsupported virtio interface!" cr
-      1 r@ vd>type l!
-   THEN
-   r> drop
-;
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netapps/netboot.c b/qemu/roms/SLOF/clients/net-snk/app/netapps/netboot.c
index cf20b5915..bb1db03e3 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netapps/netboot.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netapps/netboot.c
@@ -332,7 +332,13 @@ int dhcp(char *ret_buffer, filename_ip_t * fn_ip, unsigned int retries, int flag
 	int i = (int) retries+1;
 	int rc = -1;
 
-	printf("    ");
+	printf("  Requesting information via DHCP%s:     ",
+	       flags == F_IPV4 ? "v4" : flags == F_IPV6 ? "v6" : "");
+
+	if (flags != F_IPV6)
+		dhcpv4_generate_transaction_id();
+	if (flags != F_IPV4)
+		dhcpv6_generate_transaction_id();
 
 	do {
 		printf("\b\b\b%03d", i-1);
@@ -353,7 +359,6 @@ int dhcp(char *ret_buffer, filename_ip_t * fn_ip, unsigned int retries, int flag
 			set_ipv6_address(fn_ip->fd, 0);
 			rc = dhcpv6(ret_buffer, fn_ip);
 			if (rc == 0) {
-				printf("\n");
 				memcpy(&fn_ip->own_ip6, get_ipv6_address(), 16);
 				break;
 			}
@@ -362,11 +367,23 @@ int dhcp(char *ret_buffer, filename_ip_t * fn_ip, unsigned int retries, int flag
 		if (rc != -1) /* either success or non-dhcp failure */
 			break;
 	} while (1);
-	printf("\b\b\b\b");
+	printf("\b\b\b\bdone\n");
 
 	return rc;
 }
 
+/**
+ * Seed the random number generator with our mac and current timestamp
+ */
+static void seed_rng(uint8_t mac[])
+{
+	unsigned int seed;
+
+	asm volatile("mftbl %0" : "=r"(seed));
+	seed ^= (mac[2] << 24) | (mac[3] << 16) | (mac[4] << 8) | mac[5];
+	srand(seed);
+}
+
 int
 netboot(int argc, char *argv[])
 {
@@ -388,8 +405,7 @@ netboot(int argc, char *argv[])
 	int32_t block_size = strtol(argv[5], 0, 10);
 	uint8_t own_mac[6];
 
-	printf("\n");
-	printf(" Bootloader 1.6 \n");
+	puts("\n Initializing NIC");
 	memset(&fn_ip, 0, sizeof(filename_ip_t));
 
 	/***********************************************************
@@ -438,6 +454,8 @@ netboot(int argc, char *argv[])
 	// init ethernet layer
 	set_mac_address(own_mac);
 
+	seed_rng(own_mac);
+
 	if (argc > 6) {
 		parse_args(argv[6], &obp_tftp_args);
 		if(obp_tftp_args.bootp_retries - rc < DEFAULT_BOOT_RETRIES)
@@ -468,10 +486,8 @@ netboot(int argc, char *argv[])
 		}
 	}
 	else if (ip_version == 6) {
-		if (memcmp(&obp_tftp_args.ci6addr, null_ip6, 16) != 0
-		    && memcmp(&obp_tftp_args.si6addr, null_ip6, 16) != 0
+		if (memcmp(&obp_tftp_args.si6addr, null_ip6, 16) != 0
 		    && obp_tftp_args.filename[0] != 0) {
-
 			memcpy(&fn_ip.server_ip6.addr[0],
 			       &obp_tftp_args.si6addr.addr, 16);
 			obp_tftp_args.ip_init = IP_INIT_IPV6_MANUAL;
@@ -484,7 +500,6 @@ netboot(int argc, char *argv[])
 	// construction of fn_ip from parameter
 	switch(obp_tftp_args.ip_init) {
 	case IP_INIT_BOOTP:
-		printf("  Requesting IP address via BOOTP: ");
 		// if giaddr in not specified, then we have to identify
 		// the BOOTP server via broadcasts
 		if(memcmp(obp_tftp_args.giaddr, null_ip, 4) == 0) {
@@ -499,19 +514,25 @@ netboot(int argc, char *argv[])
 		rc = bootp(ret_buffer, &fn_ip, obp_tftp_args.bootp_retries);
 		break;
 	case IP_INIT_DHCP:
-		printf("  Requesting IP address via DHCPv4: ");
 		rc = dhcp(ret_buffer, &fn_ip, obp_tftp_args.bootp_retries, F_IPV4);
 		break;
 	case IP_INIT_DHCPV6_STATELESS:
-		printf("  Requesting information via DHCPv6: ");
 		rc = dhcp(ret_buffer, &fn_ip,
 			  obp_tftp_args.bootp_retries, F_IPV6);
 		break;
 	case IP_INIT_IPV6_MANUAL:
-		set_ipv6_address(fn_ip.fd, &obp_tftp_args.ci6addr);
+		if (memcmp(&obp_tftp_args.ci6addr, null_ip6, 16)) {
+			set_ipv6_address(fn_ip.fd, &obp_tftp_args.ci6addr);
+		} else {
+			/*
+			 * If no client address has been specified, then
+			 * use a link-local or stateless autoconfig address
+			 */
+			set_ipv6_address(fn_ip.fd, NULL);
+			memcpy(&fn_ip.own_ip6, get_ipv6_address(), 16);
+		}
 		break;
 	case IP_INIT_DEFAULT:
-		printf("  Requesting IP address via DHCP: ");
 		rc = dhcp(ret_buffer, &fn_ip, obp_tftp_args.bootp_retries, 0);
 		break;
 	case IP_INIT_NONE:
@@ -548,10 +569,15 @@ netboot(int argc, char *argv[])
 		return -101;
 	}
 
-	if(ip_version == 4)
-		printf("%d.%d.%d.%d\n",
+	if (ip_version == 4) {
+		printf("  Using IPv4 address: %d.%d.%d.%d\n",
 			((fn_ip.own_ip >> 24) & 0xFF), ((fn_ip.own_ip >> 16) & 0xFF),
 			((fn_ip.own_ip >>  8) & 0xFF), ( fn_ip.own_ip        & 0xFF));
+	} else if (ip_version == 6) {
+		char ip6_str[40];
+		ipv6_to_str(fn_ip.own_ip6.addr, ip6_str);
+		printf("  Using IPv6 address: %s\n", ip6_str);
+	}
 
 	if (rc == -2) {
 		sprintf(buf,
@@ -818,7 +844,7 @@ int parse_tftp_args(char buffer[], char *server_ip, char filename[], int fd,
 		tmp = raw + 7;
 		tmp[j] = '\0';
 		strcpy(domainname, tmp);
-		if (dns_get_ip(fd, (int8_t *)domainname, server_ip6, 6) == 0) {
+		if (dns_get_ip(fd, domainname, server_ip6, 6) == 0) {
 			printf("\n DNS failed for IPV6\n");
                         return -1;
                 }
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/bootp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/bootp.c
index 1bc6efe5b..6d58cef7d 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/bootp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/bootp.c
@@ -232,7 +232,7 @@ bootp(char *ret_buffer, filename_ip_t * fn_ip, unsigned int retries)
 	int i = (int) retries+1;
 	fn_ip->own_ip = 0;
 
-	printf("   ");
+	printf("  Requesting IP address via BOOTP:    ");
 
 	response_buffer = ret_buffer;
 
@@ -249,6 +249,7 @@ bootp(char *ret_buffer, filename_ip_t * fn_ip, unsigned int retries)
 		 * in case the previous one was lost. And because we don't
 		 * trust the network cable we keep on doing this 30 times */
 	} while (receive_bootp(fn_ip) != 0);
-	printf("\b\b\b");
+
+	printf("\b\b\bdone\n");
 	return 0;
 }
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.c
index 5f26f3afb..7e2e88ccf 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.c
@@ -11,7 +11,7 @@
  *****************************************************************************/
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> ALGORITHMS <<<<<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/******************************* ALGORITHMS ******************************/
 
 /** \file dhcp.c <pre>
  * **************** State-transition diagram for DHCP client  *************
@@ -41,13 +41,14 @@
  * </pre> */
 
 
-/*>>>>>>>>>>>>>>>>>>>>> DEFINITIONS & DECLARATIONS <<<<<<<<<<<<<<<<<<<<<<*/
+/********************** DEFINITIONS & DECLARATIONS ***********************/
 
 #include <dhcp.h>
 #include <ethernet.h>
 #include <ipv4.h>
 #include <udp.h>
 #include <dns.h>
+#include <netapps/args.h>
 
 #include <stdio.h>
 #include <string.h>
@@ -110,11 +111,11 @@ static uint8_t dhcp_magic[] = {0x63, 0x82, 0x53, 0x63};
  *  If flag[i] == TRUE then field for i-th option retains valid value and
  *  information from this field may retrived (in case of receiving) or will
  *  be transmitted (in case of transmitting).
- *  
+ *
  */
 typedef struct {
 	uint8_t    flag[256];         /**< Show if corresponding opt. is valid */
-	uint8_t    request_list[256]; /**< o.55 If i-th member is TRUE, then i-th  
+	uint8_t    request_list[256]; /**< o.55 If i-th member is TRUE, then i-th
 	                                  option will be requested from server */
 	uint32_t   server_ID;         /**< o.54 Identifies DHCP-server         */
 	uint32_t   requested_IP;      /**< o.50 Must be filled in DHCP-Request */
@@ -132,65 +133,57 @@ typedef struct {
 static uint8_t dhcp_state;
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>> PROTOTYPES <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<*/
-
-static int32_t
-dhcp_attempt(int fd);
+/***************************** PROTOTYPES ********************************/
 
-static int32_t
-dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct);
+static int32_t dhcp_attempt(int fd);
 
-static int32_t
-dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
-                    dhcp_options_t * opt_struct);
+static int32_t dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct);
 
-static int8_t
-dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
-                   uint8_t src_options[], uint32_t src_len);
+static int32_t dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
+				   dhcp_options_t * opt_struct);
 
-static int8_t
-dhcp_find_option(uint8_t options[], uint32_t len,
-                 uint8_t op_code, uint32_t * op_offset);
-
-static void
-dhcp_append_option(uint8_t dst_options[], uint32_t * dst_len,
-                   uint8_t * new_option);
+static int8_t dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
+				 uint8_t src_options[], uint32_t src_len);
 
-static void
-dhcp_combine_option(uint8_t dst_options[], uint32_t * dst_len,
-                    uint32_t dst_offset, uint8_t * new_option);
+static int8_t dhcp_find_option(uint8_t options[], uint32_t len,
+			       uint8_t op_code, uint32_t * op_offset);
 
-static void
-dhcp_send_discover(int fd);
+static void dhcp_append_option(uint8_t dst_options[], uint32_t * dst_len,
+			       uint8_t * new_option);
 
-static void
-dhcp_send_request(int fd);
+static void dhcp_combine_option(uint8_t dst_options[], uint32_t * dst_len,
+				uint32_t dst_offset, uint8_t * new_option);
 
-static uint8_t
-strtoip(int8_t * str, uint32_t * ip);
+static void dhcp_send_discover(int fd);
 
+static void dhcp_send_request(int fd);
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>> LOCAL VARIABLES <<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/***************************** LOCAL VARIABLES ***************************/
 
 static uint8_t  ether_packet[ETH_MTU_SIZE];
 static uint32_t dhcp_own_ip        = 0;
 static uint32_t dhcp_server_ip     = 0;
 static uint32_t dhcp_siaddr_ip     = 0;
-static int8_t   dhcp_filename[256];
-static int8_t   dhcp_tftp_name[256];
+static char   dhcp_filename[256];
+static char   dhcp_tftp_name[256];
+static uint32_t dhcp_xid;
 
 static char   * response_buffer;
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>> IMPLEMENTATION <<<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/***************************** IMPLEMENTATION ****************************/
 
-int32_t
-dhcpv4(char *ret_buffer, filename_ip_t * fn_ip) {
+void dhcpv4_generate_transaction_id(void)
+{
+	dhcp_xid = (rand() << 16) ^ rand();
+}
 
+int32_t dhcpv4(char *ret_buffer, filename_ip_t *fn_ip)
+{
 	uint32_t dhcp_tftp_ip     = 0;
 	int fd = fn_ip->fd;
 
-	strcpy((char *) dhcp_filename, "");
-	strcpy((char *) dhcp_tftp_name, "");
+	strcpy(dhcp_filename, "");
+	strcpy(dhcp_tftp_name, "");
 
 	response_buffer = ret_buffer;
 
@@ -204,11 +197,11 @@ dhcpv4(char *ret_buffer, filename_ip_t * fn_ip) {
 		dhcp_siaddr_ip = fn_ip->server_ip;
 	}
 	if(fn_ip->filename[0] != 0) {
-		strcpy((char *) dhcp_filename, (char *) fn_ip->filename);
+		strcpy(dhcp_filename, (char *) fn_ip->filename);
 	}
 
 	// TFTP SERVER
-	if (!strlen((char *) dhcp_tftp_name)) {
+	if (!strlen(dhcp_tftp_name)) {
 		if (!dhcp_siaddr_ip) {
 			// ERROR: TFTP name is not presented
 			return -3;
@@ -219,9 +212,9 @@ dhcpv4(char *ret_buffer, filename_ip_t * fn_ip) {
 	}
 	else {
 		// TFTP server defined by its name
-		if (!strtoip(dhcp_tftp_name, &(dhcp_tftp_ip))) {
-			if (!dns_get_ip(fd, dhcp_tftp_name, (uint8_t *)&(dhcp_tftp_ip), 4)) {
-				// DNS error - can't obtain TFTP-server name  
+		if (!strtoip(dhcp_tftp_name, (char *)&dhcp_tftp_ip)) {
+			if (!dns_get_ip(fd, dhcp_tftp_name, (uint8_t *)&dhcp_tftp_ip, 4)) {
+				// DNS error - can't obtain TFTP-server name
 				// Use TFTP-ip from siaddr field, if presented
 				if (dhcp_siaddr_ip) {
 					dhcp_tftp_ip = dhcp_siaddr_ip;
@@ -237,7 +230,7 @@ dhcpv4(char *ret_buffer, filename_ip_t * fn_ip) {
 	// Store configuration info into filename_ip strucutre
 	fn_ip -> own_ip = dhcp_own_ip;
 	fn_ip -> server_ip = dhcp_tftp_ip;
-	strcpy((char *) fn_ip -> filename, (char *) dhcp_filename);
+	strcpy((char *) fn_ip -> filename, dhcp_filename);
 
 	return 0;
 }
@@ -245,8 +238,8 @@ dhcpv4(char *ret_buffer, filename_ip_t * fn_ip) {
 /**
  * DHCP: Tries o obtain DHCP parameters, refer to state-transition diagram
  */
-static int32_t
-dhcp_attempt(int fd) {
+static int32_t dhcp_attempt(int fd)
+{
 	int sec;
 
 	// Send DISCOVER message and switch DHCP-client to SELECT state
@@ -270,7 +263,7 @@ dhcp_attempt(int fd) {
 		} while (get_timer() > 0);
 	}
 
-	// timeout 
+	// timeout
 	return 0;
 }
 
@@ -278,7 +271,7 @@ dhcp_attempt(int fd) {
  * DHCP: Supplements DHCP-message with options stored in structure.
  *       For more information about option coding see dhcp_options_t.
  *
- * @param  opt_field     Points to the "vend" field of DHCP-message  
+ * @param  opt_field     Points to the "vend" field of DHCP-message
  *                       (destination)
  * @param  opt_struct    this structure stores info about the options which
  *                       will be added to DHCP-message (source)
@@ -286,8 +279,8 @@ dhcp_attempt(int fd) {
  *                       FALSE - error condition occurs.
  * @see                  dhcp_options_t
  */
-static int32_t
-dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct) {
+static int32_t dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct)
+{
 	uint8_t * options = opt_field;
 	uint16_t i, sum; // used to define is any options set
 
@@ -380,7 +373,7 @@ dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct) {
  * DHCP: Extracts encoded options from DHCP-message into the structure.
  *       For more information about option coding see dhcp_options_t.
  *
- * @param  opt_field     Points to the "options" field of DHCP-message  
+ * @param  opt_field     Points to the "options" field of DHCP-message
  *                       (source).
  * @param  opt_len       Length of "options" field.
  * @param  opt_struct    this structure stores info about the options which
@@ -389,10 +382,10 @@ dhcp_encode_options(uint8_t * opt_field, dhcp_options_t * opt_struct) {
  *                       FALSE - error condition occurs.
  * @see                  dhcp_options_t
  */
-static int32_t
-dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
-                    dhcp_options_t * opt_struct) {
-	int32_t offset = 0;
+static int32_t dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
+				   dhcp_options_t * opt_struct)
+{
+	uint32_t offset = 0;
 
 	memset(opt_struct, 0, sizeof(dhcp_options_t));
 
@@ -407,30 +400,30 @@ dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
 		switch(opt_field[offset]) {
 		case DHCP_OVERLOAD :
 			opt_struct -> overload = opt_field[offset + 2];
-			offset += 2 + opt_field[offset + 1]; 
+			offset += 2 + opt_field[offset + 1];
 			break;
 
 		case DHCP_REQUESTED_IP :
 			opt_struct -> requested_IP = htonl(* (uint32_t *) (opt_field + offset + 2));
-			offset += 2 + opt_field[offset + 1]; 
+			offset += 2 + opt_field[offset + 1];
 			break;
 
 		case DHCP_MASK :
 			opt_struct -> flag[DHCP_MASK] = 1;
 			opt_struct -> subnet_mask = htonl(* (uint32_t *) (opt_field + offset + 2));
-			offset += 2 + opt_field[offset + 1]; 
+			offset += 2 + opt_field[offset + 1];
 			break;
 
 		case DHCP_DNS :
 			opt_struct -> flag[DHCP_DNS] = 1;
 			opt_struct -> dns_IP = htonl(* (uint32_t *) (opt_field + offset + 2));
-			offset += 2 + opt_field[offset + 1]; 
+			offset += 2 + opt_field[offset + 1];
 			break;
 
 		case DHCP_ROUTER :
 			opt_struct -> flag[DHCP_ROUTER] = 1;
 			opt_struct -> router_IP = htonl(* (uint32_t *) (opt_field + offset + 2));
-			offset += 2 + opt_field[offset + 1]; 
+			offset += 2 + opt_field[offset + 1];
 			break;
 
 		case DHCP_MSG_TYPE :
@@ -492,11 +485,12 @@ dhcp_decode_options(uint8_t opt_field[], uint32_t opt_len,
  *                       FALSE - error condition occurs.
  */
 static int8_t dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
-                                 uint8_t src_options[], uint32_t src_len) {
-	int32_t dst_offset, src_offset = 0;
+				 uint8_t src_options[], uint32_t src_len)
+{
+	uint32_t dst_offset, src_offset = 0;
 
 	// remove ENDOPT if presented
-	if (dhcp_find_option(dst_options, * dst_len, DHCP_ENDOPT, (uint32_t *) &dst_offset))
+	if (dhcp_find_option(dst_options, * dst_len, DHCP_ENDOPT, &dst_offset))
 		* dst_len = dst_offset;
 
 	while (src_offset < src_len) {
@@ -509,7 +503,7 @@ static int8_t dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
 		default:
 			if (dhcp_find_option(dst_options, * dst_len,
 			                     src_options[src_offset],
-			                     (uint32_t *) &dst_offset)) {
+			                     &dst_offset)) {
 				dhcp_combine_option(dst_options, dst_len,
 				                    dst_offset,
 				                    (uint8_t *) src_options +
@@ -522,7 +516,7 @@ static int8_t dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
 		}
 	}
 
-	if (src_offset == src_len) 
+	if (src_offset == src_len)
 		return 1;
 	return 0;
 }
@@ -540,7 +534,8 @@ static int8_t dhcp_merge_options(uint8_t dst_options[], uint32_t * dst_len,
  *                       FALSE - option wasn't find.
  */
 static int8_t dhcp_find_option(uint8_t options[], uint32_t len,
-                               uint8_t op_code, uint32_t * op_offset) {
+			       uint8_t op_code, uint32_t * op_offset)
+{
 	uint32_t srch_offset = 0;
 	* op_offset = 0;
 
@@ -568,9 +563,9 @@ static int8_t dhcp_find_option(uint8_t options[], uint32_t len,
  * @param  dst_len       length of the "options" field (modified)
  * @param  new_option    points to an option in another list (src)
  */
-static void
-dhcp_append_option(uint8_t dst_options[], uint32_t * dst_len,
-                   uint8_t * new_option) {
+static void dhcp_append_option(uint8_t dst_options[], uint32_t * dst_len,
+			       uint8_t * new_option)
+{
 	memcpy(dst_options + ( * dst_len), new_option, 2 + (* (new_option + 1)));
 	* dst_len += 2 + *(new_option + 1);
 }
@@ -586,10 +581,9 @@ dhcp_append_option(uint8_t dst_options[], uint32_t * dst_len,
  * @param  dst_offset    offset of the option from beginning of the list
  * @param  new_option    points to an option in another list (src)
  */
-static void
-dhcp_combine_option(uint8_t dst_options[], uint32_t * dst_len,
-                    uint32_t dst_offset, uint8_t * new_option) {
-
+static void dhcp_combine_option(uint8_t dst_options[], uint32_t * dst_len,
+				uint32_t dst_offset, uint8_t * new_option)
+{
 	uint8_t tmp_buffer[1024]; // use to provide safe memcpy
 	uint32_t tail_len;
 
@@ -612,8 +606,8 @@ dhcp_combine_option(uint8_t dst_options[], uint32_t * dst_len,
 /**
  * DHCP: Sends DHCP-Discover message. Looks for DHCP servers.
  */
-static void
-dhcp_send_discover(int fd) {
+static void dhcp_send_discover(int fd)
+{
 	uint32_t packetsize = sizeof(struct iphdr) +
 	                      sizeof(struct udphdr) + sizeof(struct btphdr);
 	struct btphdr *btph;
@@ -627,6 +621,7 @@ dhcp_send_discover(int fd) {
 	btph -> op = 1;
 	btph -> htype = 1;
 	btph -> hlen = 6;
+	btph -> xid = dhcp_xid;
 	memcpy(btph -> chaddr, get_mac_address(), 6);
 
 	memset(&opt, 0, sizeof(dhcp_options_t));
@@ -655,8 +650,8 @@ dhcp_send_discover(int fd) {
 /**
  * DHCP: Sends DHCP-Request message. Asks for acknowledgment to occupy IP.
  */
-static void
-dhcp_send_request(int fd) {
+static void dhcp_send_request(int fd)
+{
 	uint32_t packetsize = sizeof(struct iphdr) +
 	                      sizeof(struct udphdr) + sizeof(struct btphdr);
 	struct btphdr *btph;
@@ -670,6 +665,7 @@ dhcp_send_request(int fd) {
 	btph -> op = 1;
 	btph -> htype = 1;
 	btph -> hlen = 6;
+	btph -> xid = dhcp_xid;
 	memcpy(btph -> chaddr, get_mac_address(), 6);
 
 	memset(&opt, 0, sizeof(dhcp_options_t));
@@ -704,7 +700,8 @@ dhcp_send_request(int fd) {
 /**
  * DHCP: Sends DHCP-Release message. Releases occupied IP.
  */
-void dhcp_send_release(int fd) {
+void dhcp_send_release(int fd)
+{
 	uint32_t packetsize = sizeof(struct iphdr) +
 	                      sizeof(struct udphdr) + sizeof(struct btphdr);
 	struct btphdr *btph;
@@ -718,6 +715,7 @@ void dhcp_send_release(int fd) {
 	btph -> op = 1;
 	btph -> htype = 1;
 	btph -> hlen = 6;
+	btph -> xid = dhcp_xid;
 	strcpy((char *) btph -> file, "");
 	memcpy(btph -> chaddr, get_mac_address(), 6);
 	btph -> ciaddr = htonl(dhcp_own_ip);
@@ -730,7 +728,7 @@ void dhcp_send_release(int fd) {
 
 	dhcp_encode_options(btph -> vend, &opt);
 
-	fill_udphdr(&ether_packet[sizeof(struct iphdr)], 
+	fill_udphdr(&ether_packet[sizeof(struct iphdr)],
 	            sizeof(struct btphdr) + sizeof(struct udphdr),
 	            UDPPORT_BOOTPC, UDPPORT_BOOTPS);
 	fill_iphdr(ether_packet, sizeof(struct btphdr) +
@@ -753,18 +751,21 @@ void dhcp_send_release(int fd) {
  * @see               btphdr
  */
 
-int8_t
-handle_dhcp(int fd, uint8_t * packet, int32_t packetsize) {
+int8_t handle_dhcp(int fd, uint8_t * packet, int32_t packetsize)
+{
 	struct btphdr * btph;
 	struct iphdr * iph;
 	dhcp_options_t opt;
 
-	memset(&opt, 0, sizeof(dhcp_options_t));  
+	memset(&opt, 0, sizeof(dhcp_options_t));
 	btph = (struct btphdr *) packet;
 	iph = (struct iphdr *) packet - sizeof(struct udphdr) -
 	      sizeof(struct iphdr);
-	if (btph -> op != 2)
-		return -1; // it is not Boot Reply
+
+	if (btph->op != 2)
+		return -1;		/* It is not a Bootp/DHCP reply */
+	if (btph->xid != dhcp_xid)
+		return -1;		/* The transaction ID does not match */
 
 	if (memcmp(btph -> vend, dhcp_magic, 4)) {
 		// It is BootP - RFC 951
@@ -788,7 +789,7 @@ handle_dhcp(int fd, uint8_t * packet, int32_t packetsize) {
 	}
 
 
-	// decode options  
+	// decode options
 	if (!dhcp_decode_options(btph -> vend, packetsize -
 	                         sizeof(struct btphdr) + sizeof(btph -> vend),
 	                         &opt)) {
@@ -902,7 +903,7 @@ handle_dhcp(int fd, uint8_t * packet, int32_t packetsize) {
 				else {
 					strcpy((char *) dhcp_filename, "");
 					if (opt.overload != DHCP_OVERLOAD_FILE &&
-						opt.overload != DHCP_OVERLOAD_BOTH && 
+						opt.overload != DHCP_OVERLOAD_BOTH &&
 						strlen((char *) btph -> file)) {
 						strncpy((char *) dhcp_filename,
 						        (char *) btph->file,
@@ -952,47 +953,3 @@ handle_dhcp(int fd, uint8_t * packet, int32_t packetsize) {
 
 	return 0;
 }
-
-/**
- * DHCP: Converts "255.255.255.255" -> 32-bit long IP
- *
- * @param  str        string to be converted
- * @param  ip         in case of SUCCESS - 32-bit long IP
-                      in case of FAULT - zero
- * @return            TRUE - IP converted successfully;
- *                    FALSE - error condition occurs (e.g. bad format)
- */
-static uint8_t
-strtoip(int8_t * str, uint32_t * ip) {
-	int8_t ** ptr = &str;
-	int16_t i = 0, res, len;
-	char octet[256];
-
-	* ip = 0;
-
-	while (**ptr != 0) {
-		if (i > 3 || !isdigit(**ptr))
-			return 0;
-		if (strstr((char *) * ptr, ".") != NULL) {
-			len = (int16_t) ((int8_t *) strstr((char *) * ptr, ".") - 
-			      (int8_t *) (* ptr));
-			strncpy(octet, (char *) * ptr, len); octet[len] = 0;
-			* ptr += len;
-		}
-		else {
-			strcpy(octet, (char *) * ptr);
-			* ptr += strlen(octet);
-		}
-		res = strtol(octet, NULL, 10);
-		if ((res > 255) || (res < 0))
-			return 0;
-		* ip = ((* ip) << 8) + res;
-		i++;
-		if (** ptr == '.')
-			(*ptr)++;
-	}
-
-	if (i != 4)
-		return 0;
-	return 1;
-}
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.h
index 69dd49d4a..54fb1eed3 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcp.h
@@ -43,6 +43,7 @@ struct btphdr {
 	uint8_t vend[64];    /**< Optional parameters field (DHCP-options)     */
 };
 
+void dhcpv4_generate_transaction_id(void);
 int bootp(char *ret_buffer, filename_ip_t *, unsigned int);
 int dhcpv4(char *ret_buffer, filename_ip_t *);
 void dhcp_send_release(int fd);
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.c
index 4deef30f2..d0a22d555 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.c
@@ -27,13 +27,15 @@ static uint8_t tid[3];
 static uint32_t dhcpv6_state = -1;
 static filename_ip_t *my_fn_ip;
 
-static void
-generate_transaction_id(void)
+static struct ip6addr_list_entry all_dhcpv6_ll; /* All DHCPv6 servers address */
+
+void
+dhcpv6_generate_transaction_id(void)
 {
-	/* TODO: as per RFC 3315 transaction IDs should be generated randomly */
-	tid[0] = 1;
-	tid[1] = 2;
-	tid[2] = 4;
+	/* As per RFC 3315 transaction IDs should be generated randomly */
+	tid[0] = rand();
+	tid[1] = rand();
+	tid[2] = rand();
 }
 
 static void
@@ -45,8 +47,6 @@ send_info_request(int fd)
 
 	memset(ether_packet, 0, ETH_MTU_SIZE);
 
-	generate_transaction_id();
-
 	/* Get an IPv6 packet */
 	payload_length = sizeof(struct udphdr) + sizeof(struct dhcp_message_header);
 	fill_ip6hdr (ether_packet + sizeof(struct ethhdr),
@@ -72,16 +72,14 @@ send_info_request(int fd)
 	dhcph->option.el_time.length = 2;
 	dhcph->option.el_time.time = 0x190; /* 4000 ms */
 	dhcph->option.option_request_option.code = DHCPV6_OPTION_ORO;
-	dhcph->option.option_request_option.length= 6;
+	dhcph->option.option_request_option.length = DHCPV6_OPTREQUEST_NUMOPTS * 2;
 	dhcph->option.option_request_option.option_code[0] = DHCPV6_OPTION_DNS_SERVERS;
 	dhcph->option.option_request_option.option_code[1] = DHCPV6_OPTION_DOMAIN_LIST;
 	dhcph->option.option_request_option.option_code[2] = DHCPV6_OPTION_BOOT_URL;
 
-
 	send_ipv6(fd, ether_packet + sizeof(struct ethhdr),
-	         sizeof(struct ethhdr)+ sizeof(struct ip6hdr)
-		 + sizeof(struct udphdr)
-	         + sizeof( struct dhcp_message_header) );
+		  sizeof(struct ip6hdr) + sizeof(struct udphdr)
+		  + sizeof(struct dhcp_message_header));
 }
 
 static int32_t
@@ -119,6 +117,9 @@ dhcpv6 ( char *ret_buffer, void *fn_ip)
 {
 	int fd;
 
+	all_dhcpv6_ll.addr.part.prefix = 0xff02000000000000ULL;
+	all_dhcpv6_ll.addr.part.interface_id = 0x10002ULL;
+
 	my_fn_ip = (filename_ip_t *) fn_ip;
 	fd = my_fn_ip->fd;
 
@@ -129,8 +130,7 @@ dhcpv6 ( char *ret_buffer, void *fn_ip)
 	return 0;
 }
 
-static struct dhcp6_received_options *
-dhcp6_process_options (uint8_t *option, int32_t option_length)
+static void dhcp6_process_options (uint8_t *option, int32_t option_length)
 {
 	struct dhcp_boot_url *option_boot_url;
 	struct client_identifier *option_clientid;
@@ -138,24 +138,19 @@ dhcp6_process_options (uint8_t *option, int32_t option_length)
 	struct dhcp_dns *option_dns;
 	struct dhcp_dns_list *option_dns_list;
 	struct dhcp6_gen_option *option_gen;
-	struct dhcp6_received_options *received_options;
 	char buffer[256];
 
-
-	received_options = malloc (sizeof(struct dhcp6_received_options));
 	while (option_length > 0) {
 		switch ((uint16_t) *(option+1)) {
 		case DHCPV6_OPTION_CLIENTID:
 			option_clientid = (struct client_identifier *) option;
 			option = option +  option_clientid->length + 4;
 			option_length = option_length - option_clientid->length - 4;
-			received_options->client_id = 1;
 			break;
 		case DHCPV6_OPTION_SERVERID:
 			option_serverid = (struct server_identifier *) option;
 			option = option +  option_serverid->length + 4;
 			option_length = option_length - option_serverid->length - 4;
-			received_options->server_id = 1;
 			break;
 		case DHCPV6_OPTION_DNS_SERVERS:
 			option_dns = (struct dhcp_dns *) option;
@@ -184,7 +179,7 @@ dhcp6_process_options (uint8_t *option, int32_t option_length)
 					    (char *)my_fn_ip->filename,
 					    (int)my_fn_ip->fd,
 					    option_boot_url->length) == -1)
-				return NULL;
+				return;
 			break;
 		default:
 			option_gen = (struct dhcp6_gen_option *) option;
@@ -192,8 +187,6 @@ dhcp6_process_options (uint8_t *option, int32_t option_length)
 			option_length = option_length - option_gen->length - 4;
 		}
 	}
-
-	return received_options;
 }
 
 uint32_t
@@ -205,6 +198,9 @@ handle_dhcpv6(uint8_t * packet, int32_t packetsize)
 	struct dhcp_message_reply *reply;
 	reply = (struct dhcp_message_reply *) packet;
 
+	if (memcmp(reply->transaction_id, tid, 3))
+		return -1;			/* Wrong transaction ID */
+
 	if (reply->type == 7)
 		dhcpv6_state = DHCP_STATUSCODE_SUCCESS;
 
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.h
index 078a9f11f..fb77da648 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dhcpv6.h
@@ -77,6 +77,7 @@
 #define DUID_LL		3 /* DUID based on Link-layer Address */
 
 /* Prototypes */
+void dhcpv6_generate_transaction_id(void);
 int32_t dhcpv6 ( char *ret_buffer, void *fn_ip);
 uint32_t handle_dhcpv6(uint8_t * , int32_t);
 
@@ -102,6 +103,8 @@ struct server_identifier {
 	uint8_t mac[6];
 };
 
+#define DHCPV6_OPTREQUEST_NUMOPTS 3
+
 struct dhcp_info_request {
 	struct client_identifier client_id;
 	struct elapsed_time {
@@ -112,7 +115,7 @@ struct dhcp_info_request {
 	struct option_request {
 		uint16_t code;
 		uint16_t length;
-		uint16_t option_code[5];
+		uint16_t option_code[DHCPV6_OPTREQUEST_NUMOPTS];
 	} option_request_option;
 };
 
@@ -141,12 +144,6 @@ struct dhcp_boot_url {
 	uint8_t url[256];
 };
 
-struct dhcp6_received_options {
-	uint8_t filename;
-	uint8_t ip;
-	uint8_t client_id;
-	uint8_t server_id;
-};
 struct dhcp_message_reply {
 	uint8_t type;			    /* Message type   */
 	uint8_t transaction_id[3];          /* Transaction id */
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.c
index 0ab1346c9..a5a36a18e 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.c
@@ -133,7 +133,7 @@ dns_init(uint32_t _dns_server_ip, uint8_t _dns_server_ipv6[16], uint8_t ip_versi
  *                   FALSE - error condition occurs.
  */
 int8_t
-dns_get_ip(int fd, int8_t * url, uint8_t * domain_ip, uint8_t ip_version)
+dns_get_ip(int fd, char* url, uint8_t * domain_ip, uint8_t ip_version)
 {
 	/* this counter is used so that we abort after 30 DNS request */
 	int32_t i;
@@ -143,7 +143,7 @@ dns_get_ip(int fd, int8_t * url, uint8_t * domain_ip, uint8_t ip_version)
 	(* domain_ip) = 0;
 
 	// Retrieve host name from URL
-	if (!urltohost((char *) url, (char *) host_name)) {
+	if (!urltohost(url, (char *) host_name)) {
 		printf("\nERROR:\t\t\tBad URL!\n");
 		return 0;
 	}
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.h
index 82eea4e4d..b8756afca 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/dns.h
@@ -20,7 +20,7 @@
 extern int8_t dns_init(uint32_t _dns_server_ip, uint8_t _dns_server_ipv6[16], uint8_t ip_version);
 
 /* For given URL retrieves IPv4 from DNS-server. */
-extern int8_t dns_get_ip(int fd, int8_t * url, uint8_t * domain_ip, uint8_t ip_version);
+extern int8_t dns_get_ip(int fd, char * url, uint8_t * domain_ip, uint8_t ip_version);
 
 /* Handles DNS-packets, which are detected by receive_ether. */
 extern int32_t handle_dns(uint8_t * packet, int32_t packetsize);
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ethernet.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/ethernet.c
index bbfd6d1c3..1e03a0bf3 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ethernet.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ethernet.c
@@ -11,7 +11,7 @@
  *****************************************************************************/
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> ALGORITHMS <<<<<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/******************************* ALGORITHMS ******************************/
 
 /** \file netbase.c <pre>
  * *********************** Receive-handle diagram *************************
@@ -36,12 +36,12 @@
  *  | APPLICATION        +----------------+-----------+
  *  V                    |                            |
  * upper               DNS (handle_dns)      BootP / DHCP (handle_bootp_client)
- * 
+ *
  * ************************************************************************
  * </pre> */
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>> DEFINITIONS & DECLARATIONS <<<<<<<<<<<<<<<<<<<<*/
+/************************ DEFINITIONS & DECLARATIONS *********************/
 
 #include <ethernet.h>
 #include <string.h>
@@ -50,22 +50,22 @@
 #include <ipv6.h>
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> LOCAL VARIABLES <<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** LOCAL VARIABLES **************************/
 
 static uint8_t ether_packet[ETH_MTU_SIZE];
 static uint8_t own_mac[6] = {0, 0, 0, 0, 0, 0};
 static uint8_t multicast_mac[] = {0x01, 0x00, 0x5E};
 static const uint8_t broadcast_mac[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> IMPLEMENTATION <<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** IMPLEMENTATION ***************************/
 
 /**
  * Ethernet: Set the own MAC address to initializes ethernet layer.
  *
  * @param  own_mac  own hardware-address (MAC)
  */
-void
-set_mac_address(const uint8_t * _own_mac) {
+void set_mac_address(const uint8_t * _own_mac)
+{
 	if (_own_mac)
 		memcpy(own_mac, _own_mac, 6);
 	else
@@ -77,19 +77,19 @@ set_mac_address(const uint8_t * _own_mac) {
  *
  * @return  own hardware-address (MAC)
  */
-const uint8_t *
-get_mac_address(void) {
+const uint8_t *get_mac_address(void)
+{
 	return own_mac;
 }
 
 /**
  * Ethernet: Check if given multicast address is a multicast MAC address
- *           starting with 0x3333 
+ *           starting with 0x3333
  *
- * @return  true or false 
+ * @return  true or false
  */
-static uint8_t
-is_multicast_mac(uint8_t * mac) {
+static uint8_t is_multicast_mac(uint8_t * mac)
+{
 
     	uint16_t mc = 0x3333;
     	if (memcmp(mac, &mc, 2) == 0)
@@ -98,7 +98,6 @@ is_multicast_mac(uint8_t * mac) {
 	return 0;
 }
 
-
 /**
  * Ethernet: Receives an ethernet-packet and handles it according to
  *      Receive-handle diagram.
@@ -107,8 +106,8 @@ is_multicast_mac(uint8_t * mac) {
  * @return  ZERO - packet was handled or no packets received;
  *          NON ZERO - error condition occurs.
  */
-int32_t
-receive_ether(int fd) {
+int32_t receive_ether(int fd)
+{
 	int32_t bytes_received;
 	struct ethhdr * ethh;
 
@@ -118,7 +117,10 @@ receive_ether(int fd) {
 	if (!bytes_received) // No messages
 		return 0;
 
-	if (bytes_received < sizeof(struct ethhdr))
+	if (bytes_received < 0)
+		return -1; /* recv() failed */
+
+	if ((size_t) bytes_received < sizeof(struct ethhdr))
 		return -1; // packet is too small
 
 	ethh = (struct ethhdr *) ether_packet;
@@ -176,9 +178,9 @@ send_ether(int fd, void* buffer, int len)
  * @see                fill_dnshdr
  * @see                fill_btphdr
  */
-void
-fill_ethhdr(uint8_t * packet, uint16_t eth_type,
-            const uint8_t * src_mac, const uint8_t * dest_mac) {
+void fill_ethhdr(uint8_t * packet, uint16_t eth_type,
+		 const uint8_t * src_mac, const uint8_t * dest_mac)
+{
 	struct ethhdr * ethh = (struct ethhdr *) packet;
 
 	ethh -> type = htons(eth_type);
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/icmpv6.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/icmpv6.c
index be6cc110f..c104f7015 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/icmpv6.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/icmpv6.c
@@ -40,9 +40,8 @@ send_router_solicitation (int fd)
 			  sizeof(struct ip6hdr));
 
 	/* Destination is "All routers multicast address" (link-local) */
-	dest_addr.part.prefix       = all_routers_ll.addr.part.prefix;
-	dest_addr.part.interface_id = all_routers_ll.addr.part.interface_id;
-
+	dest_addr.part.prefix       = 0xff02000000000000ULL;
+	dest_addr.part.interface_id = 2;
 
 	/* Fill IPv6 header */
 	fill_ip6hdr (ether_packet + sizeof(struct ethhdr),
@@ -78,8 +77,8 @@ handle_prefixoption (uint8_t *option)
 	prefix_option = (struct option_prefix *) option;
 	memcpy( &(prefix.addr), &(prefix_option->prefix.addr), IPV6_ADDR_LENGTH);
 
-	/* Link-local adresses in RAs are nonsense                  */
-	if ( (IPV6_LL_PREFIX & (prefix_option->prefix.part.prefix)) == IPV6_LL_PREFIX )
+	/* Link-local adresses in RAs are nonsense */
+	if (ip6_is_linklocal(&prefix))
 		return;
 
 	if (prefix_option->preferred_lifetime > prefix_option->valid_lifetime)
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.c
index 8185de5e1..2b92c77c4 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.c
@@ -11,7 +11,7 @@
  *****************************************************************************/
 
 
-/*>>>>>>>>>>>>>>>>>>>>> DEFINITIONS & DECLARATIONS <<<<<<<<<<<<<<<<<<<<<<*/
+/********************** DEFINITIONS & DECLARATIONS ***********************/
 
 #include <ipv4.h>
 #include <udp.h>
@@ -81,32 +81,26 @@ struct icmphdr {
 	} payload;
 };
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> PROTOTYPES <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** PROTOTYPES *******************************/
 
-static unsigned short
-checksum(unsigned short *packet, int words);
+static unsigned short checksum(unsigned short *packet, int words);
 
-static void
-arp_send_request(int fd, uint32_t dest_ip);
+static void arp_send_request(int fd, uint32_t dest_ip);
 
-static void
-arp_send_reply(int fd, uint32_t src_ip, uint8_t * src_mac);
+static void arp_send_reply(int fd, uint32_t src_ip, uint8_t * src_mac);
 
-static void
-fill_arphdr(uint8_t * packet, uint8_t opcode,
-            const uint8_t * src_mac, uint32_t src_ip,
-            const uint8_t * dest_mac, uint32_t dest_ip);
+static void fill_arphdr(uint8_t * packet, uint8_t opcode,
+			const uint8_t * src_mac, uint32_t src_ip,
+			const uint8_t * dest_mac, uint32_t dest_ip);
 
-static arp_entry_t*
-lookup_mac_addr(uint32_t ipv4_addr);
+static arp_entry_t *lookup_mac_addr(uint32_t ipv4_addr);
 
-static void
-fill_udp_checksum(struct iphdr *ipv4_hdr);
+static void fill_udp_checksum(struct iphdr *ipv4_hdr);
 
-static int8_t
-handle_icmp(int fd, struct iphdr * iph, uint8_t * packet, int32_t packetsize);
+static int8_t handle_icmp(int fd, struct iphdr * iph, uint8_t * packet,
+			  int32_t packetsize);
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> LOCAL VARIABLES <<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** LOCAL VARIABLES **************************/
 
 /* Routing parameters */
 static uint32_t own_ip       = 0;
@@ -126,18 +120,19 @@ static       uint8_t multicast_mac[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
 static unsigned int arp_consumer = 0;
 static unsigned int arp_producer = 0;
 static arp_entry_t  arp_table[ARP_ENTRIES];
-static arp_entry_t  pending_pkt;
+
+static uint8_t pending_pkt_frame[ETH_MTU_SIZE];
+static int pending_pkt_len;
 
 /* Function pointer send_ip. Points either to send_ipv4() or send_ipv6() */
 int   (*send_ip) (int fd, void *, int);
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>> IMPLEMENTATION <<<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/***************************** IMPLEMENTATION ****************************/
 
 /**
  * IPv4: Initialize the environment for the IPv4 layer.
  */
-static void
-ipv4_init(void)
+static void ipv4_init(void)
 {
 	int i;
 
@@ -153,7 +148,7 @@ ipv4_init(void)
 		arp_table[i].pkt_pending = 0;
 	}
 
-	/* Set IP send function to send_ipv4() */ 
+	/* Set IP send function to send_ipv4() */
 	send_ip = &send_ipv4;
 }
 
@@ -162,8 +157,7 @@ ipv4_init(void)
  *
  * @param  _own_ip  client IPv4 address (e.g. 127.0.0.1)
  */
-void
-set_ipv4_address(uint32_t _own_ip)
+void set_ipv4_address(uint32_t _own_ip)
 {
 	own_ip = _own_ip;
 	ipv4_init();
@@ -174,8 +168,7 @@ set_ipv4_address(uint32_t _own_ip)
  *
  * @return client IPv4 address (e.g. 127.0.0.1)
  */
-uint32_t
-get_ipv4_address(void)
+uint32_t get_ipv4_address(void)
 {
 	return own_ip;
 }
@@ -185,8 +178,7 @@ get_ipv4_address(void)
  *
  * @param  _own_ip  multicast IPv4 address (224.0.0.0 - 239.255.255.255)
  */
-void
-set_ipv4_multicast(uint32_t _multicast_ip)
+void set_ipv4_multicast(uint32_t _multicast_ip)
 {
 	// is this IP Multicast out of range (224.0.0.0 - 239.255.255.255)
 	if((htonl(_multicast_ip) < 0xE0000000)
@@ -210,8 +202,7 @@ set_ipv4_multicast(uint32_t _multicast_ip)
  *
  * @return multicast IPv4 address (224.0.0.0 - 239.255.255.255 or 0 if not set)
  */
-uint32_t
-get_ipv4_multicast(void)
+uint32_t get_ipv4_multicast(void)
 {
 	return multicast_ip;
 }
@@ -221,8 +212,7 @@ get_ipv4_multicast(void)
  *
  * @param  _router_ip   router IPv4 address
  */
-void
-set_ipv4_router(uint32_t _router_ip)
+void set_ipv4_router(uint32_t _router_ip)
 {
 	router_ip = _router_ip;
 	ipv4_init();
@@ -233,8 +223,7 @@ set_ipv4_router(uint32_t _router_ip)
  *
  * @return router IPv4 address
  */
-uint32_t
-get_ipv4_router(void)
+uint32_t get_ipv4_router(void)
 {
 	return router_ip;
 }
@@ -244,8 +233,7 @@ get_ipv4_router(void)
  *
  * @param  _subnet_mask   netmask of the own IPv4 address
  */
-void
-set_ipv4_netmask(uint32_t _subnet_mask)
+void set_ipv4_netmask(uint32_t _subnet_mask)
 {
 	subnet_mask = _subnet_mask;
 	ipv4_init();
@@ -256,8 +244,7 @@ set_ipv4_netmask(uint32_t _subnet_mask)
  *
  * @return netmask of the own IPv4 address
  */
-uint32_t
-get_ipv4_netmask(void)
+uint32_t get_ipv4_netmask(void)
 {
 	return subnet_mask;
 }
@@ -280,9 +267,9 @@ get_ipv4_netmask(void)
  * @see                fill_dnshdr
  * @see                fill_btphdr
  */
-void
-fill_iphdr(uint8_t * packet, uint16_t packetsize,
-           uint8_t ip_proto, uint32_t ip_src, uint32_t ip_dst) {
+void fill_iphdr(uint8_t * packet, uint16_t packetsize,
+           uint8_t ip_proto, uint32_t ip_src, uint32_t ip_dst)
+{
 	struct iphdr * iph = (struct iphdr *) packet;
 
 	iph -> ip_hlv = 0x45;
@@ -308,8 +295,7 @@ fill_iphdr(uint8_t * packet, uint16_t packetsize,
  * @see               receive_ether
  * @see               iphdr
  */
-int8_t
-handle_ipv4(int fd, uint8_t * ip_packet, int32_t packetsize)
+int8_t handle_ipv4(int fd, uint8_t * ip_packet, uint32_t packetsize)
 {
 	struct iphdr * iph;
 	int32_t old_sum;
@@ -422,8 +408,7 @@ handle_ipv4(int fd, uint8_t * ip_packet, int32_t packetsize)
  * @see               receive_ether
  * @see               iphdr
  */
-int
-send_ipv4(int fd, void* buffer, int len)
+int send_ipv4(int fd, void* buffer, int len)
 {
 	arp_entry_t *arp_entry = 0;
 	struct iphdr *ip;
@@ -506,13 +491,11 @@ send_ipv4(int fd, void* buffer, int len)
 		arp_entry->pkt_pending = 1;
 		arp_entry->ipv4_addr = ip_dst;
 		memset(arp_entry->mac_addr, 0, 6);
-		pending_pkt.ipv4_addr = ip_dst;
-		memset(pending_pkt.mac_addr, 0, 6);
-		fill_ethhdr (pending_pkt.eth_frame, htons(ETHERTYPE_IP),
+		fill_ethhdr (pending_pkt_frame, htons(ETHERTYPE_IP),
 		             get_mac_address(), null_mac_addr);
-		memcpy(&pending_pkt.eth_frame[sizeof(struct ethhdr)],
+		memcpy(&pending_pkt_frame[sizeof(struct ethhdr)],
 		       buffer, len);
-		pending_pkt.eth_len = len + sizeof(struct ethhdr);
+		pending_pkt_len = len + sizeof(struct ethhdr);
 
 		set_timer(TICKS_SEC);
 		do {
@@ -538,11 +521,9 @@ send_ipv4(int fd, void* buffer, int len)
  *
  * @param  ipv4_hdr    Points to the place where IPv4-header starts.
  */
-
-static void
-fill_udp_checksum(struct iphdr *ipv4_hdr)
+static void fill_udp_checksum(struct iphdr *ipv4_hdr)
 {
-	int i;
+	unsigned i;
 	unsigned long checksum = 0;
 	struct iphdr ip_hdr;
 	char *ptr;
@@ -585,8 +566,7 @@ fill_udp_checksum(struct iphdr *ipv4_hdr)
  * @return            Checksum
  * @see               iphdr
  */
-static unsigned short
-checksum(unsigned short * packet, int words)
+static unsigned short checksum(unsigned short * packet, int words)
 {
 	unsigned long checksum;
 
@@ -598,8 +578,7 @@ checksum(unsigned short * packet, int words)
 	return ~checksum;
 }
 
-static arp_entry_t*
-lookup_mac_addr(uint32_t ipv4_addr)
+static arp_entry_t* lookup_mac_addr(uint32_t ipv4_addr)
 {
 	unsigned int i;
 
@@ -618,8 +597,7 @@ lookup_mac_addr(uint32_t ipv4_addr)
  * @param  fd        socket fd
  * @param  dest_ip   IP of the host which MAC should be obtained
  */
-static void
-arp_send_request(int fd, uint32_t dest_ip)
+static void arp_send_request(int fd, uint32_t dest_ip)
 {
 	arp_entry_t *arp_entry = &arp_table[arp_producer];
 
@@ -642,8 +620,7 @@ arp_send_request(int fd, uint32_t dest_ip)
  * @param  src_ip    requester IP address (foreign IP)
  * @param  src_mac   requester MAC address (foreign MAC)
  */
-static void
-arp_send_reply(int fd, uint32_t src_ip, uint8_t * src_mac)
+static void arp_send_reply(int fd, uint32_t src_ip, uint8_t * src_mac)
 {
 	arp_entry_t *arp_entry = &arp_table[arp_producer];
 
@@ -674,10 +651,9 @@ arp_send_reply(int fd, uint32_t src_ip, uint8_t * src_mac)
  * @see                arphdr
  * @see                fill_ethhdr
  */
-static void
-fill_arphdr(uint8_t * packet, uint8_t opcode,
-	    const uint8_t * src_mac, uint32_t src_ip,
-	    const uint8_t * dest_mac, uint32_t dest_ip)
+static void fill_arphdr(uint8_t * packet, uint8_t opcode,
+                        const uint8_t * src_mac, uint32_t src_ip,
+                        const uint8_t * dest_mac, uint32_t dest_ip)
 {
 	struct arphdr * arph = (struct arphdr *) packet;
 
@@ -706,8 +682,7 @@ fill_arphdr(uint8_t * packet, uint8_t opcode,
  * @see               receive_ether
  * @see               arphdr
  */
-int8_t
-handle_arp(int fd, uint8_t * packet, int32_t packetsize)
+int8_t handle_arp(int fd, uint8_t * packet, uint32_t packetsize)
 {
 	struct arphdr * arph = (struct arphdr *) packet;
 
@@ -754,11 +729,11 @@ handle_arp(int fd, uint8_t * packet, int32_t packetsize)
 
 		// do we have something to send
 		if (arp_table[i].pkt_pending) {
-			struct ethhdr * ethh = (struct ethhdr *) pending_pkt.eth_frame;
+			struct ethhdr * ethh = (struct ethhdr *) pending_pkt_frame;
 			memcpy(ethh -> dest_mac, arp_table[i].mac_addr, 6);
 
-			send_ether(fd, pending_pkt.eth_frame, pending_pkt.eth_len);
-			pending_pkt.pkt_pending = 0;
+			send_ether(fd, pending_pkt_frame, pending_pkt_len);
+			arp_table[i].pkt_pending = 0;
 			arp_table[i].eth_len = 0;
 		}
 		return 0; // no error
@@ -780,8 +755,7 @@ handle_arp(int fd, uint8_t * packet, int32_t packetsize)
  * @param  fd            socket descriptor
  * @param  _ping_dst_ip  destination IPv4 address
  */
-void
-ping_ipv4(int fd, uint32_t _ping_dst_ip)
+void ping_ipv4(int fd, uint32_t _ping_dst_ip)
 {
 	unsigned char packet[sizeof(struct iphdr) + sizeof(struct icmphdr)];
 	struct icmphdr *icmp;
@@ -814,8 +788,7 @@ ping_ipv4(int fd, uint32_t _ping_dst_ip)
  *
  * @return  ping_dst_ip  host IPv4 address
  */
-uint32_t
-pong_ipv4(void)
+uint32_t pong_ipv4(void)
 {
 	return ping_dst_ip;
 }
@@ -830,8 +803,8 @@ pong_ipv4(void)
  *                      NON ZERO - packet was not handled (e.g. bad format)
  * @see                 handle_ipv4
  */
-static int8_t
-handle_icmp(int fd, struct iphdr * iph, uint8_t * packet, int32_t packetsize)
+static int8_t handle_icmp(int fd, struct iphdr * iph, uint8_t * packet,
+			  int32_t packetsize)
 {
 	struct icmphdr *icmp = (struct icmphdr *) packet;
 
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.h
index eb719f8b2..18821ea74 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv4.h
@@ -60,7 +60,7 @@ struct arphdr {
 	uint32_t dest_ip;    /**< Proto address of target of this packet       */
 } __attribute((packed));
 
-/*>>>>>>>>>>>>> Initialization of the IPv4 network layer. <<<<<<<<<<<<<*/
+/************** Initialization of the IPv4 network layer. **************/
 extern void     set_ipv4_address(uint32_t own_ip);
 extern uint32_t get_ipv4_address(void);
 extern void     set_ipv4_multicast(uint32_t multicast_ip);
@@ -88,9 +88,9 @@ extern void ping_ipv4(int fd, uint32_t _ping_dst_ip);
 extern uint32_t pong_ipv4(void);
 
 /* Handles IPv4-packets that are detected by receive_ether. */
-extern int8_t handle_ipv4(int fd, uint8_t * packet, int32_t packetsize);
+extern int8_t handle_ipv4(int fd, uint8_t * packet, uint32_t packetsize);
 
 /* Handles ARP-packets that are detected by receive_ether. */
-extern int8_t handle_arp(int fd, uint8_t * packet, int32_t packetsize);
+extern int8_t handle_arp(int fd, uint8_t * packet, uint32_t packetsize);
 
 #endif
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.c
index 0cb0a2e7b..62d29ea86 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.c
@@ -37,15 +37,23 @@ static int ip6_is_multicast (ip6_addr_t * ip);
 
 /****************************** LOCAL VARIABLES **************************/
 
+/* List of Ipv6 Addresses */
+static struct ip6addr_list_entry *first_ip6;
+static struct ip6addr_list_entry *last_ip6;
+
 /* Own IPv6 address */
 static struct ip6addr_list_entry *own_ip6;
 
+/* All nodes link-local address */
+struct ip6addr_list_entry all_nodes_ll;
+
 /* Null IPv6 address */
 static ip6_addr_t null_ip6;
 
 /* helper variables */
 static uint8_t null_mac[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
 
+struct ip6_config ip6_state;
 
 /****************************** IMPLEMENTATION ***************************/
 
@@ -55,9 +63,10 @@ static uint8_t null_mac[] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
  * @param  fd            Socket descriptor
  * @param  _own_ip       client IPv6 address (e.g. ::1)
  */
-void
-set_ipv6_address (int fd, ip6_addr_t *_own_ip6)
+void set_ipv6_address(int fd, ip6_addr_t *_own_ip6)
 {
+	struct ip6addr_list_entry *ile;
+
 	own_ip6 = malloc (sizeof(struct ip6addr_list_entry));
 
 	/* If no address was passed as a parameter generate a link-local
@@ -73,6 +82,20 @@ set_ipv6_address (int fd, ip6_addr_t *_own_ip6)
 	ip6addr_add (own_ip6);
 
 	ipv6_init(fd);
+
+	/*
+	 * Check whether we've got a non-link-local address during
+	 * ipv6_init() and use that as preferred address if possible
+	 */
+	if (_own_ip6 == NULL) {
+		for (ile = first_ip6; ile != NULL ; ile = ile->next) {
+			if (!ip6_is_multicast(&ile->addr) &&
+			    !ip6_is_linklocal(&ile->addr)) {
+				own_ip6 = ile;
+				break;
+			}
+		}
+	}
 }
 
 /**
@@ -80,8 +103,7 @@ set_ipv6_address (int fd, ip6_addr_t *_own_ip6)
  *
  * @return pointer to client IPv6 address (e.g. ::1)
  */
-ip6_addr_t *
-get_ipv6_address (void)
+ip6_addr_t *get_ipv6_address(void)
 {
 	return (ip6_addr_t *) &(own_ip6->addr);
 }
@@ -92,8 +114,7 @@ get_ipv6_address (void)
  * @return 0 - IPv6 address is not in list
  *         1 - IPv6 address is in list
  */
-static int8_t
-find_ip6addr (ip6_addr_t *ip)
+static int8_t find_ip6addr(ip6_addr_t *ip)
 {
 	struct ip6addr_list_entry *n = NULL;
 
@@ -119,8 +140,7 @@ find_ip6addr (ip6_addr_t *ip)
  * @see handle_udp
  * @see ip6hdr
  */
-int8_t
-handle_ipv6 (int fd, uint8_t * ip6_packet, int32_t packetsize)
+int8_t handle_ipv6(int fd, uint8_t * ip6_packet, uint32_t packetsize)
 {
 
 	struct ip6hdr *ip6 = NULL;
@@ -164,11 +184,9 @@ handle_ipv6 (int fd, uint8_t * ip6_packet, int32_t packetsize)
  * @see                fill_dnshdr
  * @see                fill_btphdr
  */
-void
-fill_ip6hdr (uint8_t * packet, uint16_t packetsize,
-             uint8_t ip_proto, ip6_addr_t *ip6_src, ip6_addr_t *ip6_dst)
+void fill_ip6hdr(uint8_t * packet, uint16_t packetsize,
+		 uint8_t ip_proto, ip6_addr_t *ip6_src, ip6_addr_t *ip6_dst)
 {
-
 	struct ip6hdr * ip6h = (struct ip6hdr *) packet;
 
 	ip6h->ver_tc_fl = 6 << 28;	// set version to 6
@@ -184,8 +202,7 @@ fill_ip6hdr (uint8_t * packet, uint16_t packetsize,
  *      See RFC 4291 "IP Version 6 Addressing Architecture"
  *
  */
-uint64_t
-mac2eui64 (const uint8_t *mac)
+uint64_t mac2eui64(const uint8_t *mac)
 {
 	uint8_t eui64id[8];
 	uint64_t retid;
@@ -205,8 +222,7 @@ mac2eui64 (const uint8_t *mac)
  * @param  own_mac    MAC of NIC
  * @return ll_addr    pointer to newly created link-local address
  */
-ip6_addr_t *
-ip6_create_ll_address (const uint8_t *own_mac)
+ip6_addr_t *ip6_create_ll_address(const uint8_t *own_mac)
 {
 	ip6_addr_t *ll_addr;
 
@@ -223,8 +239,7 @@ ip6_create_ll_address (const uint8_t *own_mac)
  * @param  struct ip6_addr_list_entry *ip6
  * @return true or false
  */
-int8_t
-unknown_prefix (ip6_addr_t *ip)
+int8_t unknown_prefix(ip6_addr_t *ip)
 {
 	struct ip6addr_list_entry *node;
 
@@ -240,8 +255,7 @@ unknown_prefix (ip6_addr_t *ip)
  * @return NULL - malloc failed
  *	   ! NULL - pointer to new prefix_info
  */
-struct prefix_info *
-ip6_create_prefix_info ()
+struct prefix_info *ip6_create_prefix_info()
 {
 	struct prefix_info *prfx_info;
 
@@ -259,8 +273,7 @@ ip6_create_prefix_info ()
  * @param  ip6_addr prefix (as received in RA)
  * @return NULL - pointer to new ip6addr_list entry
  */
-void *
-ip6_prefix2addr (ip6_addr_t prefix)
+void *ip6_prefix2addr(ip6_addr_t prefix)
 {
 	struct ip6addr_list_entry *new_address;
 	uint64_t interface_id;
@@ -287,8 +300,7 @@ ip6_prefix2addr (ip6_addr_t prefix)
  * @return  0 - passed pointer = NULL;
  *	    1 - ok
  */
-int8_t
-ip6addr_add (struct ip6addr_list_entry *new_address)
+int8_t ip6addr_add(struct ip6addr_list_entry *new_address)
 {
 	struct ip6addr_list_entry *solicited_node;
 
@@ -332,8 +344,7 @@ ip6addr_add (struct ip6addr_list_entry *new_address)
  *
  * @param  fd            socket fd
  */
-static void
-ipv6_init (int fd)
+static void ipv6_init(int fd)
 {
 	int i = 0;
 
@@ -349,21 +360,9 @@ ipv6_init (int fd)
 	/* Multicast addresses */
 	all_nodes_ll.addr.part.prefix         = 0xff02000000000000;
 	all_nodes_ll.addr.part.interface_id   = 1;
-	all_dhcpv6_ll.addr.part.prefix        = 0xff02000000000000ULL;
-	all_dhcpv6_ll.addr.part.interface_id  = 0x10002ULL;
-	all_routers_ll.addr.part.prefix       = 0xff02000000000000;
-	all_routers_ll.addr.part.interface_id      = 2;
-
 	ip6addr_add(&all_nodes_ll);
-	/* ... */
 
-	/* Router list */
-	first_router = NULL;
-	last_router = first_router;
-
-	/* Init Neighbour cache */
-	first_neighbor = NULL;
-	last_neighbor  = first_neighbor;
+	ndp_init();
 
 	send_router_solicitation (fd);
 	for(i=0; i < 4 && !is_ra_received(); i++) {
@@ -382,8 +381,7 @@ ipv6_init (int fd)
  * @param  ip6_addr ip_1
  * @param  ip6_addr ip_2
  */
-int8_t
-ip6_cmp (ip6_addr_t *ip_1, ip6_addr_t *ip_2)
+int8_t ip6_cmp(ip6_addr_t *ip_1, ip6_addr_t *ip_2)
 {
 	return ((int8_t) !memcmp( &(ip_1->addr[0]), &(ip_2->addr[0]),
 		IPV6_ADDR_LENGTH ));
@@ -396,11 +394,9 @@ ip6_cmp (ip6_addr_t *ip_1, ip6_addr_t *ip_2)
  * @param  *ip    - pointer to IPv6 address
  * @return true or false
  */
-int
-ip6_is_multicast (ip6_addr_t * ip)
+int ip6_is_multicast(ip6_addr_t * ip)
 {
-	uint8_t mc = 0xFF;
-	return ! memcmp(&ip->addr[0], &mc, 1);
+	return ip->addr[0] == 0xFF;
 }
 
 /**
@@ -408,17 +404,11 @@ ip6_is_multicast (ip6_addr_t * ip)
  *      (e.g. UDP or ICMPv6)
  *
  * @param  *ip    - pointer to IPv6 address
+ * @param  *mc_mac  pointer to an array with 6 bytes (for the MAC address)
  * @return pointer to Multicast MAC address
  */
-static uint8_t *
-ip6_to_multicast_mac (ip6_addr_t * ip)
+static uint8_t *ip6_to_multicast_mac(ip6_addr_t * ip, uint8_t *mc_mac)
 {
-	uint8_t *mc_mac;
-
-	mc_mac = malloc(ETH_ALEN);
-	if (!mc_mac)
-		return NULL;
-
 	mc_mac[0] = 0x33;
 	mc_mac[1] = 0x33;
 	memcpy (mc_mac+2, (uint8_t *) &(ip->addr)+12, 4);
@@ -437,8 +427,8 @@ ip6_to_multicast_mac (ip6_addr_t * ip)
  *				    starting from *packet
  * @return checksum
  */
-static unsigned short
-ip6_checksum (struct ip6hdr *ip6h, unsigned short *packet, int words)
+static unsigned short ip6_checksum(struct ip6hdr *ip6h, unsigned short *packet,
+				   int words)
 {
 	int i=0;
 	unsigned long checksum;
@@ -478,8 +468,7 @@ ip6_checksum (struct ip6hdr *ip6h, unsigned short *packet, int words)
  * @see receive_ether
  * @see ip6hdr
  */
-int
-send_ipv6 (int fd, void* buffer, int len)
+int send_ipv6(int fd, void* buffer, int len)
 {
 	struct neighbor *n;
 	struct ip6hdr *ip6h;
@@ -519,17 +508,9 @@ send_ipv6 (int fd, void* buffer, int len)
 
 	n = find_neighbor (&ip_dst);
 
-	// If packet is a neighbor solicitation
-	if (icmp6h->type == ICMPV6_NEIGHBOUR_SOLICITATION) {
-		mac_addr = ip6_to_multicast_mac (&ip_dst);
-		fill_ethhdr( buffer-sizeof(struct ethhdr), htons(ETHERTYPE_IPv6),
-			     get_mac_address(),
-			     mac_addr);
-	}
-
 	// If address is a multicast address, create a proper mac address
-	else if (ip6_is_multicast (&ip_dst)) {
-		mac_addr = ip6_to_multicast_mac (&ip_dst);
+	if (ip6_is_multicast (&ip_dst)) {
+		mac_addr = ip6_to_multicast_mac (&ip_dst, mac);
 	}
 	else {
 		// Check if the MAC address is already cached
@@ -572,8 +553,7 @@ send_ipv6 (int fd, void* buffer, int len)
 	return send_ether (fd, n->eth_frame, len + sizeof(struct ethhdr));
 }
 
-static int
-check_colons(const char *str)
+static int check_colons(const char *str)
 {
 	char *pch, *prv;
 	int col = 0;
@@ -595,7 +575,7 @@ check_colons(const char *str)
 	dprintf("The number of  col : %d \n",col);
 	dprintf("The number of dcol : %d \n",dcol);
 
-	if((dcol > 1) ||                      /* Cannot have 2 "::" */ 
+	if((dcol > 1) ||                      /* Cannot have 2 "::" */
 	   ((dcol == 1) && (col > 5)) ||      /* Too many ':'s */
 	   ((dcol == 0) && (col != 7)) ) {    /* Too few ':'s */
 		dprintf(" exiting for check_colons \n");
@@ -605,8 +585,7 @@ check_colons(const char *str)
 	return (col+dcol);
 }
 
-static int
-ipv6str_to_bytes(const char *str, char *ip)
+static int ipv6str_to_bytes(const char *str, char *ip)
 {
 	char block[5];
 	int res;
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.h
index b496364f3..72c6ee22f 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ipv6.h
@@ -26,6 +26,7 @@
 
 #define IPV6_ADDR_LENGTH	 16 /* Size of IPv6 adress in bytes */
 #define IPV6_LL_PREFIX		 0xFE80000000000000ULL
+#define IPV6_LL_PREFIX_MASK	 0xFFC0000000000000ULL
 #define IPV6_SOLIC_NODE_PREFIX   0xFF02000000000000ULL
 #define IPV6_SOLIC_NODE_IFACE_ID 0x00000001FF000000ULL
 
@@ -126,32 +127,17 @@ struct ip6_config {
 	uint8_t managed_mode:1,
 		other_config:1,
 		reserved:6;
-} ip6_state;
+};
 
 /******************** VARIABLES **********************************************/
 /* Function pointer send_ip. Points either to send_ipv4() or send_ipv6() */
 extern int   (*send_ip) (int fd, void *, int);
 
-/* IPv6 link-local multicast addresses */
-struct ip6addr_list_entry all_routers_ll; // Routers
-struct ip6addr_list_entry all_dhcpv6_ll;  // DHCPv6 servers
-struct ip6addr_list_entry all_nodes_ll;   // All IPv6 nodes
-
-/* List of Ipv6 Addresses */
-struct ip6addr_list_entry *first_ip6;
-struct ip6addr_list_entry *last_ip6;
-
-/* Neighbor cache */
-struct neighbor *first_neighbor;
-struct neighbor *last_neighbor;
-
-/* Router list */
-struct router *first_router;
-struct router *last_router;
+extern struct ip6_config ip6_state;
 
 /******************** FUNCTIONS *********************************************/
 /* Handles IPv6-packets that are detected by receive_ether. */
-int8_t handle_ipv6(int fd, uint8_t * ip6_packet, int32_t packetsize);
+int8_t handle_ipv6(int fd, uint8_t * ip6_packet, uint32_t packetsize);
 
 /* Fill IPv6 header */
 void fill_ip6hdr(uint8_t * packet, uint16_t packetsize,
@@ -179,6 +165,12 @@ void * ip6_prefix2addr (ip6_addr_t prefix);
 /* Compare IPv6 adresses */
 int8_t ip6_cmp( ip6_addr_t *ip_1, ip6_addr_t *ip_2 );
 
+/* Check if it is a link-local address */
+static inline int ip6_is_linklocal(ip6_addr_t *ip)
+{
+	return (ip->part.prefix & IPV6_LL_PREFIX_MASK) == IPV6_LL_PREFIX;
+}
+
 /* Check if prefix is already in our list */
 int8_t unknown_prefix (ip6_addr_t *ip);
 
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.c
index ed9d61f4a..96faa8716 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.c
@@ -17,6 +17,14 @@
 #include <netlib/icmpv6.h>
 #include <netlib/ndp.h>
 
+/* Neighbor cache */
+static struct neighbor *first_neighbor;
+static struct neighbor *last_neighbor;
+
+/* Router list */
+static struct router *first_router;
+static struct router *last_router;
+
 /*
  * NET: add new router to list
  * @param  struct router nghb  - new router
@@ -145,3 +153,14 @@ find_neighbor (ip6_addr_t *ip)
 
 	return NULL; /* neighbor is unknown */
 }
+
+void ndp_init(void)
+{
+	/* Router list */
+	first_router = NULL;
+	last_router = first_router;
+
+	/* Init Neighbour cache */
+	first_neighbor = NULL;
+	last_neighbor  = first_neighbor;
+}
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.h
index ee5235fe1..c785c4897 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/ndp.h
@@ -59,6 +59,7 @@ struct neighbor {
 };
 
 /******************** FUNCTIONS *********************************************/
+void ndp_init(void);
 int8_t neighbor_add (struct neighbor *);
 void * neighbor_create (uint8_t *packet, struct packeth *headers);
 struct neighbor * find_neighbor (ip6_addr_t *);
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/tcp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/tcp.c
index 5511aa00a..faa0b83ac 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/tcp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/tcp.c
@@ -10,16 +10,14 @@
  *     IBM Corporation - initial implementation
  *****************************************************************************/
 
-/*>>>>>>>>>>>>>>>>>>>>>>> DEFINITIONS & DECLARATIONS <<<<<<<<<<<<<<<<<<<<*/
+/************************ DEFINITIONS & DECLARATIONS *********************/
 
 #include <tcp.h>
 #include <sys/socket.h>
 
+/****************************** LOCAL VARIABLES **************************/
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> LOCAL VARIABLES <<<<<<<<<<<<<<<<<<<<<<<<<*/
-
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> IMPLEMENTATION <<<<<<<<<<<<<<<<<<<<<<<<<<*/
-
+/****************************** IMPLEMENTATION ***************************/
 
 /**
  * TCP: Handles TCP-packets according to Receive-handle diagram.
@@ -29,13 +27,11 @@
  * @return            ZERO - packet handled successfully;
  *                    NON ZERO - packet was not handled (e.g. bad format)
  */
-int8_t
-handle_tcp(uint8_t * tcp_packet, int32_t packetsize)
+int8_t handle_tcp(uint8_t * tcp_packet, int32_t packetsize)
 {
 	return -1;
 }
 
-
 /**
  * NET: This function handles situation when "Destination unreachable"
  *      ICMP-error occurs during sending TCP-packet.
@@ -45,6 +41,6 @@ handle_tcp(uint8_t * tcp_packet, int32_t packetsize)
  * @param  packetsize length of the packet
  * @see               handle_icmp
  */
-void
-handle_tcp_dun(uint8_t * tcp_packet, uint32_t packetsize, uint8_t err_code) {
+void handle_tcp_dun(uint8_t * tcp_packet, uint32_t packetsize, uint8_t err_code)
+{
 }
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/tftp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/tftp.c
index 0a7c0ec63..c1197cf17 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/tftp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/tftp.c
@@ -50,13 +50,13 @@ static unsigned short block = 0;
 static unsigned short blocksize;
 static char blocksize_str[6];    /* Blocksize string for read request */
 static int received_len = 0;
-static int retries = 0;
+static unsigned int retries = 0;
 static int huge_load;
 static int len;
 static int tftp_finished = 0;
 static int lost_packets = 0;
-static int tftp_errno = 0; 
-static int ip_version = 0; 
+static int tftp_errno = 0;
+static int ip_version = 0;
 static short port_number = -1;
 static tftp_err_t *tftp_err;
 static filename_ip_t  *fn_ip;
@@ -69,8 +69,7 @@ static filename_ip_t  *fn_ip;
  */
 #ifdef __DEBUG__
 
-static void
-dump_package(unsigned char *buffer, unsigned int len)
+static void dump_package(unsigned char *buffer, unsigned int len)
 {
 	int i;
 
@@ -89,8 +88,7 @@ dump_package(unsigned char *buffer, unsigned int len)
  *
  * @fd:          Socket Descriptor
  */
-static void
-send_rrq(int fd)
+static void send_rrq(int fd)
 {
 	int ip_len = 0;
 	int ip6_payload_len    = 0;
@@ -121,7 +119,7 @@ send_rrq(int fd)
 			+ strlen("blksize") + strlen(blocksize_str) + 2;
 		ip_len = sizeof(struct ip6hdr) + ip6_payload_len;
 		fill_ip6hdr ((uint8_t *) ip6, ip6_payload_len, IPTYPE_UDP, get_ipv6_address(),
-			     &(fn_ip->server_ip6)); 
+			     &(fn_ip->server_ip6));
 
 	}
 	udp_len = htons(sizeof(struct udphdr)
@@ -158,8 +156,7 @@ send_rrq(int fd)
  * @blckno: block number
  * @dport:  UDP destination port
  */
-static void
-send_ack(int fd, int blckno, unsigned short dport)
+static void send_ack(int fd, int blckno, unsigned short dport)
 {
 	int ip_len 	       = 0;
 	int ip6_payload_len    = 0;
@@ -182,8 +179,7 @@ send_ack(int fd, int blckno, unsigned short dport)
 		ip6 = (struct ip6hdr *) packet;
 		udph = (struct udphdr *) (ip6 + 1);
 		ip6_payload_len = sizeof(struct udphdr) + 4;
-		ip_len = sizeof(struct ethhdr) + sizeof(struct ip6hdr) +
-		    	 ip6_payload_len;
+		ip_len = sizeof(struct ip6hdr) + ip6_payload_len;
 		fill_ip6hdr ((uint8_t *) ip6, ip6_payload_len, IPTYPE_UDP, get_ipv6_address(),
 			     &(fn_ip->server_ip6));
 	}
@@ -210,8 +206,7 @@ send_ack(int fd, int blckno, unsigned short dport)
  * @error_code:  Used sub code for error packet
  * @dport:       UDP destination port
  */
-static void
-send_error(int fd, int error_code, unsigned short dport)
+static void send_error(int fd, int error_code, unsigned short dport)
 {
 	int ip_len 	       = 0;
 	int ip6_payload_len    = 0;
@@ -234,8 +229,7 @@ send_error(int fd, int error_code, unsigned short dport)
 		ip6 = (struct ip6hdr *) packet;
 		udph = (struct udphdr *) (ip6 + 1);
 		ip6_payload_len = sizeof(struct udphdr) + 5;
-		ip_len = sizeof(struct ethhdr) + sizeof(struct ip6hdr) +
-		         ip6_payload_len; 
+		ip_len = sizeof(struct ip6hdr) + ip6_payload_len;
 		fill_ip6hdr ((uint8_t *) ip6, ip6_payload_len, IPTYPE_UDP, get_ipv6_address(),
 			    &(fn_ip->server_ip6));
 	}
@@ -256,8 +250,7 @@ send_error(int fd, int error_code, unsigned short dport)
 	return;
 }
 
-static void
-print_progress(int urgent, int received_bytes)
+static void print_progress(int urgent, int received_bytes)
 {
 	static unsigned int i = 1;
 	static int first = -1;
@@ -265,7 +258,7 @@ print_progress(int urgent, int received_bytes)
 	char buffer[100];
 	char *ptr;
 
-	// 1MB steps or 0x400 times or urgent 
+	// 1MB steps or 0x400 times or urgent
 	if(((received_bytes - last_bytes) >> 20) > 0
 	|| (i & 0x3FF) == 0 || urgent) {
 		if(!first) {
@@ -295,8 +288,7 @@ print_progress(int urgent, int received_bytes)
  * @param len  the length of the network packet
  * @return  the blocksize the server supports or 0 for error
  */
-static int
-get_blksize(unsigned char *buffer, unsigned int len)
+static int get_blksize(unsigned char *buffer, unsigned int len)
 {
 	unsigned char *orig = buffer;
 	/* skip all headers until tftp has been reached */
@@ -325,7 +317,7 @@ get_blksize(unsigned char *buffer, unsigned int len)
 }
 
 /**
- * Handle incoming tftp packets after read request was sent 
+ * Handle incoming tftp packets after read request was sent
  *
  * this function also prints out some status characters
  * \|-/ for each packet received
@@ -334,13 +326,12 @@ get_blksize(unsigned char *buffer, unsigned int len)
  * #+* for different unexpected TFTP packets (not very good)
  *
  * @param fd     socket descriptor
- * @param packet points to the UDP header of the packet 
+ * @param packet points to the UDP header of the packet
  * @param len    the length of the network packet
  * @return       ZERO if packet was handled successfully
- *               ERRORCODE if error occurred 
+ *               ERRORCODE if error occurred
  */
-int32_t
-handle_tftp(int fd, uint8_t *pkt, int32_t packetsize) 
+int32_t handle_tftp(int fd, uint8_t *pkt, int32_t packetsize)
 {
 	struct udphdr *udph;
 	struct tftphdr *tftp;
@@ -397,7 +388,7 @@ handle_tftp(int fd, uint8_t *pkt, int32_t packetsize)
 		case ENOUSER:
 			tftp_errno = -7;	// ERROR: no such user
 			break;
-		default:	
+		default:
 			tftp_errno = -1;	// ERROR: unknown error
 		}
 		goto error;
@@ -489,8 +480,7 @@ error:
  *
  * @param  err_code   Error Code (e.g. "Host unreachable")
  */
-void
-handle_tftp_dun(uint8_t err_code)
+void handle_tftp_dun(uint8_t err_code)
 {
 	tftp_errno = - err_code - 10;
 	tftp_finished = 1;
@@ -510,10 +500,9 @@ handle_tftp_dun(uint8_t err_code)
  * @return               ZERO - error condition occurs
  *                       NON ZERO - size of received file
  */
-int
-tftp(filename_ip_t * _fn_ip, unsigned char *_buffer, int _len,
-     unsigned int _retries, tftp_err_t * _tftp_err,
-     int32_t _mode, int32_t _blocksize, int _ip_version)
+int tftp(filename_ip_t * _fn_ip, unsigned char *_buffer, int _len,
+	 unsigned int _retries, tftp_err_t * _tftp_err,
+	 int32_t _mode, int32_t _blocksize, int _ip_version)
 {
 	retries     = _retries;
 	fn_ip       = _fn_ip;
@@ -592,6 +581,6 @@ tftp(filename_ip_t * _fn_ip, unsigned char *_buffer, int _len,
 	printf("\n");
 	if (lost_packets)
 		printf("Lost ACK packets: %d\n", lost_packets);
-		
+
 	return received_len;
 }
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.c b/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.c
index db29bc90f..5d16e52c6 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.c
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.c
@@ -10,7 +10,7 @@
  *     IBM Corporation - initial implementation
  *****************************************************************************/
 
-/*>>>>>>>>>>>>>>>>>>>>>>> DEFINITIONS & DECLARATIONS <<<<<<<<<<<<<<<<<<<<*/
+/************************ DEFINITIONS & DECLARATIONS *********************/
 
 #include <udp.h>
 #include <sys/socket.h>
@@ -25,7 +25,7 @@
 
 
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> LOCAL VARIABLES <<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** LOCAL VARIABLES **************************/
 
 
 #ifdef USE_MTFTP
@@ -33,17 +33,19 @@
 uint16_t net_tftp_uport;
 uint16_t net_mtftp_uport;
 
-void net_set_tftp_port(uint16_t tftp_port) {
+void net_set_tftp_port(uint16_t tftp_port)
+{
 	net_tftp_uport = tftp_port;
 }
 
-void net_set_mtftp_port(uint16_t tftp_port) {
+void net_set_mtftp_port(uint16_t tftp_port)
+{
 	net_mtftp_uport = tftp_port;
 }
 
 #endif
 
-/*>>>>>>>>>>>>>>>>>>>>>>>>>>>>> IMPLEMENTATION <<<<<<<<<<<<<<<<<<<<<<<<<<*/
+/****************************** IMPLEMENTATION ***************************/
 
 
 /**
@@ -56,8 +58,8 @@ void net_set_mtftp_port(uint16_t tftp_port) {
  * @see               receive_ether
  * @see               udphdr
  */
-int8_t
-handle_udp(int fd, uint8_t * udp_packet, int32_t packetsize) {
+int8_t handle_udp(int fd, uint8_t * udp_packet, uint32_t packetsize)
+{
 	struct udphdr * udph = (struct udphdr *) udp_packet;
 
 	if (packetsize < sizeof(struct udphdr))
@@ -108,8 +110,8 @@ handle_udp(int fd, uint8_t * udp_packet, int32_t packetsize) {
  * @param  packetsize length of the packet
  * @see               handle_icmp
  */
-void
-handle_udp_dun(uint8_t * udp_packet, uint32_t packetsize, uint8_t err_code) {
+void handle_udp_dun(uint8_t * udp_packet, uint32_t packetsize, uint8_t err_code)
+{
 	struct udphdr * udph = (struct udphdr *) udp_packet;
 
 	if (packetsize < sizeof(struct udphdr))
@@ -139,9 +141,9 @@ handle_udp_dun(uint8_t * udp_packet, uint32_t packetsize, uint8_t err_code) {
  * @see                fill_dnshdr
  * @see                fill_btphdr
  */
-void
-fill_udphdr(uint8_t * packet, uint16_t packetsize,
-            uint16_t src_port, uint16_t dest_port) {
+void fill_udphdr(uint8_t * packet, uint16_t packetsize,
+		 uint16_t src_port, uint16_t dest_port)
+{
 	struct udphdr * udph = (struct udphdr *) packet;
 
 	udph -> uh_sport = htons(src_port);
diff --git a/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.h b/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.h
index 1ba9332ce..f154542dd 100644
--- a/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.h
+++ b/qemu/roms/SLOF/clients/net-snk/app/netlib/udp.h
@@ -40,7 +40,7 @@ typedef int32_t *(*handle_upper_udp_t)(uint8_t *, int32_t);
 typedef void    *(*handle_upper_udp_dun_t)(uint8_t);
 
 /* Handles UDP-packets that are detected by any network layer. */
-extern int8_t handle_udp(int fd, uint8_t * udp_packet, int32_t packetsize);
+extern int8_t handle_udp(int fd, uint8_t * udp_packet, uint32_t packetsize);
 
 /* Handles UDP related ICMP-Dest.Unreachable packets that are detected by
  * the network layers. */
diff --git a/qemu/roms/SLOF/clients/net-snk/client.lds b/qemu/roms/SLOF/clients/net-snk/client.lds
index 39d04594e..c2086445b 100644
--- a/qemu/roms/SLOF/clients/net-snk/client.lds
+++ b/qemu/roms/SLOF/clients/net-snk/client.lds
@@ -44,10 +44,10 @@ SECTIONS {
 	  *(.opd)
 	}
 
-	. = ALIGN(0x10);
+	. = ALIGN(256);
 	.got :
 	{
-	  _got = .;
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
           *(.got)
 	  *(.toc)
 	  _got_end = .;
diff --git a/qemu/roms/SLOF/clients/net-snk/kernel/entry.S b/qemu/roms/SLOF/clients/net-snk/kernel/entry.S
index 8849fb9d1..bf10542bd 100644
--- a/qemu/roms/SLOF/clients/net-snk/kernel/entry.S
+++ b/qemu/roms/SLOF/clients/net-snk/kernel/entry.S
@@ -44,7 +44,7 @@ C_ENTRY(_entry)
 	bcl	20,31,over	# branch after pointer table
 base:	
 	.align  3
-.LCgot:		.quad   _got-base+0x8000
+.LCgot:		.quad   _got-base
 .LCstack:	.quad   _stack+STACKSIZE-0x80-base
 over:	
 	mflr	r8		# gpr 8 is the base
diff --git a/qemu/roms/SLOF/clients/takeover/client.lds b/qemu/roms/SLOF/clients/takeover/client.lds
index 2701d8e1e..0ab428a01 100644
--- a/qemu/roms/SLOF/clients/takeover/client.lds
+++ b/qemu/roms/SLOF/clients/takeover/client.lds
@@ -43,8 +43,8 @@ SECTIONS {
 
 	.got :
 	{
-	  . = ALIGN(8);
-	  _got = .;
+	  . = ALIGN(256);
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
           *(.got .toc)
 	  _got_end = .;
 	}  	
diff --git a/qemu/roms/SLOF/clients/takeover/entry.S b/qemu/roms/SLOF/clients/takeover/entry.S
index a1030eb40..ff482732d 100644
--- a/qemu/roms/SLOF/clients/takeover/entry.S
+++ b/qemu/roms/SLOF/clients/takeover/entry.S
@@ -21,7 +21,7 @@ _wrapclient:
 	bcl	20,31,over	# branch after pointer table
 base:
 	.align  3
-.LCgot:		.quad   _got-base+0x8000
+.LCgot:		.quad   _got-base
 over:
 	mflr	r8		# gpr 8 is the base
 	ld	r2, .LCgot-base(r8)	# load got pointer
diff --git a/qemu/roms/SLOF/clients/takeover/main.c b/qemu/roms/SLOF/clients/takeover/main.c
index 360d8eaed..1e1b02614 100644
--- a/qemu/roms/SLOF/clients/takeover/main.c
+++ b/qemu/roms/SLOF/clients/takeover/main.c
@@ -16,7 +16,7 @@
 #include <of.h>
 #include <pci.h>
 #include <cpu.h>
-#include <ioctl.h>
+#include <unistd.h>
 #include <takeover.h>
 
 extern void call_client_interface(of_arg_t *);
diff --git a/qemu/roms/SLOF/include/byteorder.h b/qemu/roms/SLOF/include/byteorder.h
index 60ca67267..d4a2c8ca7 100644
--- a/qemu/roms/SLOF/include/byteorder.h
+++ b/qemu/roms/SLOF/include/byteorder.h
@@ -19,38 +19,36 @@
 
 #include <stdint.h>
 
-static inline uint16_t
-bswap_16 (uint16_t x)
+typedef uint16_t le16;
+typedef uint32_t le32;
+typedef uint64_t le64;
+
+static inline uint16_t bswap_16 (uint16_t x)
 {
 	return __builtin_bswap16(x);
 }
 
-static inline uint32_t
-bswap_32 (uint32_t x)
+static inline uint32_t bswap_32 (uint32_t x)
 {
 	return __builtin_bswap32(x);
 }
 
-static inline uint64_t
-bswap_64 (uint64_t x)
+static inline uint64_t bswap_64 (uint64_t x)
 {
 	return __builtin_bswap64(x);
 }
 
-static inline void
-bswap_16p (uint16_t *x)
+static inline void bswap_16p (uint16_t *x)
 {
 	*x = __builtin_bswap16(*x);
 }
 
-static inline void
-bswap_32p (uint32_t *x)
+static inline void bswap_32p (uint32_t *x)
 {
 	*x = __builtin_bswap32(*x);
 }
 
-static inline void
-bswap_64p (uint64_t *x)
+static inline void bswap_64p (uint64_t *x)
 {
 	*x = __builtin_bswap64(*x);
 }
diff --git a/qemu/roms/SLOF/include/helpers.h b/qemu/roms/SLOF/include/helpers.h
index fb105345e..5b3d711ac 100644
--- a/qemu/roms/SLOF/include/helpers.h
+++ b/qemu/roms/SLOF/include/helpers.h
@@ -30,8 +30,10 @@ extern long SLOF_dma_map_in(void *virt, long size, int cacheable);
 extern void SLOF_dma_map_out(long phys, void *virt, long size);
 extern long SLOF_pci_config_read32(long offset);
 extern long SLOF_pci_config_read16(long offset);
+extern long SLOF_pci_config_read8(long offset);
 extern void SLOF_pci_config_write32(long offset, long value);
 extern void SLOF_pci_config_write16(long offset, long value);
+extern void SLOF_pci_config_write8(long offset, long value);
 extern void *SLOF_translate_my_address(void *addr);
 
 #define offset_of(type, member) ((long) &((type *)0)->member)
diff --git a/qemu/roms/SLOF/include/ppc970/cache.h b/qemu/roms/SLOF/include/ppc970/cache.h
index b74868986..500182ea6 100644
--- a/qemu/roms/SLOF/include/ppc970/cache.h
+++ b/qemu/roms/SLOF/include/ppc970/cache.h
@@ -55,8 +55,8 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 		case 0:			_MOVE(s, d, size, type_u); break; \
-		case sizeof(type_l):	_MOVE(s, d, size, type_l); break; \
-		case sizeof(type_w):	_MOVE(s, d, size, type_w); break; \
+		case 4:			_MOVE(s, d, size, type_l); break; \
+		case 2: case 6:		_MOVE(s, d, size, type_w); break; \
 		default:		_MOVE(s, d, size, type_c); break; \
 	}
 
@@ -78,9 +78,51 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTRMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 		case 0:			_RMOVE(s, d, size, type_u); break; \
-		case sizeof(type_l):	_RMOVE(s, d, size, type_l); break; \
-		case sizeof(type_w):	_RMOVE(s, d, size, type_w); break; \
+		case 4:			_RMOVE(s, d, size, type_l); break; \
+		case 2: case 6:		_RMOVE(s, d, size, type_w); break; \
 		default:		_RMOVE(s, d, size, type_c); break; \
 	}
 
+/* main RAM to IO memory move */
+#define FAST_MRMOVE_TYPED(s, d, size, t)	\
+{ \
+	t *s1 = (s), *d1 = (d); \
+	register t tmp; \
+	while (size > 0) { \
+		tmp = *s1++; SET_CI; *d1++ = tmp; CLR_CI; size -= sizeof(t); \
+	} \
+}
+
+#define FAST_MRMOVE(s, d, size) \
+	switch (((type_u)(s) | (type_u)(d) | (size)) & (sizeof(type_u)-1)) { \
+	case 0:		FAST_MRMOVE_TYPED(s, d, size, type_u); break; \
+	case 4:		FAST_MRMOVE_TYPED(s, d, size, type_l); break; \
+	case 2: case 6:	FAST_MRMOVE_TYPED(s, d, size, type_w); break; \
+	default:	FAST_MRMOVE_TYPED(s, d, size, type_c); break; \
+	}
+
+/* fill IO memory with pattern */
+#define FAST_RFILL_TYPED(dst, size, pat, t) \
+{ \
+	t *d1 = (dst); \
+	register t tmp = 0; \
+	int i = sizeof(t); \
+	while (i-- > 0) { \
+		tmp <<= 8; tmp |= pat & 0xff; \
+	} \
+	SET_CI; \
+	while (size > 0) { \
+		*d1++ = tmp; size -= sizeof(t); \
+	} \
+	CLR_CI; \
+}
+
+#define FAST_RFILL(dst, size, pat) \
+	switch (((type_u)dst | size) & (sizeof(type_u)-1)) { \
+	case 0:		FAST_RFILL_TYPED(dst, size, pat, type_u); break; \
+	case 4:		FAST_RFILL_TYPED(dst, size, pat, type_l); break; \
+	case 2: case 6:	FAST_RFILL_TYPED(dst, size, pat, type_w); break; \
+	default:	FAST_RFILL_TYPED(dst, size, pat, type_c); break; \
+	}
+
 #endif
diff --git a/qemu/roms/SLOF/include/ppcp7/cache.h b/qemu/roms/SLOF/include/ppcp7/cache.h
index dc6837196..3c02bb10d 100644
--- a/qemu/roms/SLOF/include/ppcp7/cache.h
+++ b/qemu/roms/SLOF/include/ppcp7/cache.h
@@ -81,8 +81,8 @@ cache_inhibited_access(uint64_t, 64)
 #define _FASTMOVE(s, d, size) \
 	switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) { \
 	case 0:			_MOVE(s, d, size, type_u); break;	\
-	case sizeof(type_l):	_MOVE(s, d, size, type_l); break;	\
-	case sizeof(type_w):	_MOVE(s, d, size, type_w); break;	\
+	case 4:			_MOVE(s, d, size, type_l); break;	\
+	case 2: case 6:		_MOVE(s, d, size, type_w); break;	\
 	default:		_MOVE(s, d, size, type_c); break;	\
 	}
 
@@ -116,12 +116,17 @@ static inline void ci_rmove(void *dst, void *src, unsigned long esize,
 #define _FASTRMOVE(s, d, size) do {					      \
 		switch (((type_u)s | (type_u)d | size) & (sizeof(type_u)-1)) {\
 		case 0:			ci_rmove(d,s,3,size>>3); break;	      \
-		case sizeof(type_l):	ci_rmove(d,s,2,size>>2); break;	      \
-		case sizeof(type_w):	ci_rmove(d,s,1,size>>1); break;	      \
+		case 4:			ci_rmove(d,s,2,size>>2); break;	      \
+		case 2: case 6:		ci_rmove(d,s,1,size>>1); break;	      \
 		default:		ci_rmove(d,s,0,size); break;	      \
 		}							      \
 	} while(0)
 
+#define FAST_MRMOVE(s, d, size) _FASTRMOVE(s, d, size)
+
+extern void fast_rfill(char *dst, long size, char pat);
+#define FAST_RFILL(dst, size, pat) fast_rfill(dst, size, pat)
+
 static inline uint16_t bswap16_load(uint64_t addr)
 {
 	unsigned int val;
diff --git a/qemu/roms/SLOF/lib/libc/include/stdlib.h b/qemu/roms/SLOF/lib/libc/include/stdlib.h
index dff57f577..5e0eda9ff 100644
--- a/qemu/roms/SLOF/lib/libc/include/stdlib.h
+++ b/qemu/roms/SLOF/lib/libc/include/stdlib.h
@@ -29,5 +29,6 @@ unsigned long int strtoul(const char *nptr, char **endptr, int base);
 long int strtol(const char *nptr, char **endptr, int base);
 
 int rand(void);
+void srand(unsigned int seed);
 
 #endif
diff --git a/qemu/roms/SLOF/lib/libc/stdio/vsnprintf.c b/qemu/roms/SLOF/lib/libc/stdio/vsnprintf.c
index e78fb3d8e..21dd04dfe 100644
--- a/qemu/roms/SLOF/lib/libc/stdio/vsnprintf.c
+++ b/qemu/roms/SLOF/lib/libc/stdio/vsnprintf.c
@@ -10,72 +10,110 @@
  *     IBM Corporation - initial implementation
  *****************************************************************************/
 
+#include <stdbool.h>
 #include "stdio.h"
 #include "stdlib.h"
 #include "string.h"
+#include "ctype.h"
 
-const static unsigned long long convert[] = {
+static const unsigned long long convert[] = {
 	0x0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF,
 	0xFFFFFFFFFFULL, 0xFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL
 };
 
-
-
 static int
-print_itoa(char **buffer,unsigned long value, unsigned short int base)
+print_str_fill(char **buffer, size_t bufsize, char *sizec,
+					const char *str, char c)
 {
-	const char zeichen[] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
-	static char sign = 0;
-
-	if(base <= 2 || base > 16)
-		return 0;
-
-	if(value < 0) {
-		sign = 1;
-		value *= -1;
-	}
+	int i, sizei, len;
+	char *bstart = *buffer;
 
-	if(value < base) {
-		if(sign) {
-			**buffer = '-';
+	sizei = strtoul(sizec, NULL, 10);
+	len = strlen(str);
+	if (sizei > len) {
+		for (i = 0;
+			(i < (sizei - len)) && ((*buffer - bstart) < bufsize);
+									i++) {
+			**buffer = c;
 			*buffer += 1;
-			sign = 0;
 		}
-		**buffer = zeichen[value];
-		*buffer += 1;
-	} else {
-		print_itoa(buffer, value / base, base);
-		**buffer = zeichen[(value % base)];
-		*buffer += 1;
 	}
-
 	return 1;
 }
 
+static int
+print_str(char **buffer, size_t bufsize, const char *str)
+{
+	char *bstart = *buffer;
+	size_t i;
+
+	for (i = 0; (i < strlen(str)) && ((*buffer - bstart) < bufsize); i++) {
+		**buffer = str[i];
+		*buffer += 1;
+	}
+	return 1;
+}
 
 static unsigned int
 print_intlen(unsigned long value, unsigned short int base)
 {
 	int i = 0;
 
-	while(value > 0) {
+	while (value > 0) {
 		value /= base;
 		i++;
 	}
-	if(i == 0) i = 1;
+	if (i == 0)
+		i = 1;
 	return i;
 }
 
+static int
+print_itoa(char **buffer, size_t bufsize, unsigned long value,
+					unsigned short base, bool upper)
+{
+	const char zeichen[] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
+	char c;
+	size_t i, len;
+
+	if(base <= 2 || base > 16)
+		return 0;
+
+	len = i = print_intlen(value, base);
+
+	/* Don't print to buffer if bufsize is not enough. */
+	if (len > bufsize)
+		return 0;
+
+	do {
+		c = zeichen[value % base];
+		if (upper)
+			c = toupper(c);
+
+		(*buffer)[--i] = c;
+		value /= base;
+	} while(value);
+
+	*buffer += len;
+
+	return 1;
+}
+
+
 
 static int
-print_fill(char **buffer, char *sizec, unsigned long size, unsigned short int base, char c, int optlen)
+print_fill(char **buffer, size_t bufsize, char *sizec, unsigned long size,
+				unsigned short int base, char c, int optlen)
 {
 	int i, sizei, len;
+	char *bstart = *buffer;
 
 	sizei = strtoul(sizec, NULL, 10);
  	len = print_intlen(size, base) + optlen;
-	if(sizei > len) {
-		for(i = 0; i < (sizei - len); i++) {
+	if (sizei > len) {
+		for (i = 0;
+			(i < (sizei - len)) && ((*buffer - bstart) < bufsize);
+									i++) {
 			**buffer = c;
 			*buffer += 1;
 		}
@@ -86,17 +124,18 @@ print_fill(char **buffer, char *sizec, unsigned long size, unsigned short int ba
 
 
 static int
-print_format(char **buffer, const char *format, void *var)
+print_format(char **buffer, size_t bufsize, const char *format, void *var)
 {
-	unsigned long start;
-	unsigned int i = 0, sizei = 0, len = 0, length_mod = sizeof(int);
+	char *start;
+	unsigned int i = 0, length_mod = sizeof(int);
 	unsigned long value = 0;
 	unsigned long signBit;
 	char *form, sizec[32];
 	char sign = ' ';
+	bool upper = false;
 
 	form  = (char *) format;
-	start = (unsigned long) *buffer;
+	start = *buffer;
 
 	form++;
 	if(*form == '0' || *form == '.') {
@@ -104,7 +143,7 @@ print_format(char **buffer, const char *format, void *var)
 		form++;
 	}
 
-	while(*form != '\0') {
+	while ((*form != '\0') && ((*buffer - start) < bufsize)) {
 		switch(*form) {
 			case 'u':
 			case 'd':
@@ -112,57 +151,59 @@ print_format(char **buffer, const char *format, void *var)
 				sizec[i] = '\0';
 				value = (unsigned long) var;
 				signBit = 0x1ULL << (length_mod * 8 - 1);
-				if (signBit & value) {
+				if ((*form != 'u') && (signBit & value)) {
 					**buffer = '-';
 					*buffer += 1;
 					value = (-(unsigned long)value) & convert[length_mod];
 				}
-				print_fill(buffer, sizec, value, 10, sign, 0);
-				print_itoa(buffer, value, 10);
+				print_fill(buffer, bufsize - (*buffer - start),
+						sizec, value, 10, sign, 0);
+				print_itoa(buffer, bufsize - (*buffer - start),
+							value, 10, upper);
 				break;
 			case 'X':
+				upper = true;
 			case 'x':
 				sizec[i] = '\0';
 				value = (unsigned long) var & convert[length_mod];
-				print_fill(buffer, sizec, value, 16, sign, 0);
-				print_itoa(buffer, value, 16);
+				print_fill(buffer, bufsize - (*buffer - start),
+						sizec, value, 16, sign, 0);
+				print_itoa(buffer, bufsize - (*buffer - start),
+							value, 16, upper);
 				break;
 			case 'O':
 			case 'o':
 				sizec[i] = '\0';
 				value = (long int) var & convert[length_mod];
-				print_fill(buffer, sizec, value, 8, sign, 0);
-				print_itoa(buffer, value, 8);
+				print_fill(buffer, bufsize - (*buffer - start),
+						sizec, value, 8, sign, 0);
+				print_itoa(buffer, bufsize - (*buffer - start),
+							value, 8, upper);
 				break;
 			case 'p':
 				sizec[i] = '\0';
-				print_fill(buffer, sizec, (unsigned long) var, 16, ' ', 2);
-				**buffer = '0';
-				*buffer += 1;	
-				**buffer = 'x';
-				*buffer += 1;
-				print_itoa(buffer,(unsigned long) var, 16);
+				print_fill(buffer, bufsize - (*buffer - start),
+					sizec, (unsigned long) var, 16, ' ', 2);
+				print_str(buffer, bufsize - (*buffer - start),
+									"0x");
+				print_itoa(buffer, bufsize - (*buffer - start),
+						(unsigned long) var, 16, upper);
 				break;
 			case 'c':
 				sizec[i] = '\0';
-				print_fill(buffer, sizec, 1, 10, ' ', 0);
+				print_fill(buffer, bufsize - (*buffer - start),
+							sizec, 1, 10, ' ', 0);
 				**buffer = (unsigned long) var;
 				*buffer += 1;
 				break;
 			case 's':
 				sizec[i] = '\0';
-				sizei = strtoul(sizec, NULL, 10);
-				len = strlen((char *) var);
-				if(sizei > len) {
-					for(i = 0; i < (sizei - len); i++) {
-						**buffer = ' ';
-						*buffer += 1;
-					}
-				}
-				for(i = 0; i < strlen((char *) var); i++) {
-					**buffer = ((char *) var)[i];
-					*buffer += 1;
-				}
+				print_str_fill(buffer,
+					bufsize - (*buffer - start), sizec,
+							(char *) var, ' ');
+
+				print_str(buffer, bufsize - (*buffer - start),
+								(char *) var);
 				break;
 			case 'l':
 				form++;
@@ -182,6 +223,9 @@ print_format(char **buffer, const char *format, void *var)
 					length_mod = sizeof(short int);
 				}
 				break;
+			case 'z':
+				length_mod = sizeof(size_t);
+				break;
 			default:
 				if(*form >= '0' && *form <= '9')
 					sizec[i++] = *form;
@@ -206,6 +250,16 @@ vsnprintf(char *buffer, size_t bufsize, const char *format, va_list arg)
 	bstart = buffer;
 	ptr = (char *) format;
 
+	/*
+	 * Return from here if size passed is zero, otherwise we would
+	 * overrun buffer while setting NULL character at the end.
+	 */
+	if (!buffer || !bufsize)
+		return 0;
+
+	/* Leave one space for NULL character */
+	bufsize--;
+
 	while(*ptr != '\0' && (buffer - bstart) < bufsize)
 	{
 		if(*ptr == '%') {
@@ -224,7 +278,9 @@ vsnprintf(char *buffer, size_t bufsize, const char *format, va_list arg)
 			if(*ptr == '%') {
 				*buffer++ = '%';
 			} else {
-				print_format(&buffer, formstr, va_arg(arg, void *));
+				print_format(&buffer,
+					bufsize - (buffer - bstart),
+					formstr, va_arg(arg, void *));
 			}
 			ptr++;
 		} else {
diff --git a/qemu/roms/SLOF/lib/libc/stdlib/rand.c b/qemu/roms/SLOF/lib/libc/stdlib/rand.c
index 87e3efd29..39f5a9a2c 100644
--- a/qemu/roms/SLOF/lib/libc/stdlib/rand.c
+++ b/qemu/roms/SLOF/lib/libc/stdlib/rand.c
@@ -18,7 +18,12 @@ static unsigned long _rand = 1;
 int
 rand(void)
 {
-	_rand = _rand * 25364735 + 34563;
+	_rand = _rand * 1237732973 + 34563;
 
-	return ((unsigned int) (_rand << 16) & RAND_MAX);
+	return ((unsigned int) (_rand >> 16) & RAND_MAX);
+}
+
+void srand(unsigned int seed)
+{
+	_rand = seed;
 }
diff --git a/qemu/roms/SLOF/lib/libhvcall/Makefile b/qemu/roms/SLOF/lib/libhvcall/Makefile
index 2a9b2d7d1..def532509 100644
--- a/qemu/roms/SLOF/lib/libhvcall/Makefile
+++ b/qemu/roms/SLOF/lib/libhvcall/Makefile
@@ -24,7 +24,7 @@ TARGET = ../libhvcall.a
 
 all: $(TARGET)
 
-SRCS = brokensc1.c
+SRCS = brokensc1.c rfill.c
 SRCSS = hvcall.S
 
 
diff --git a/qemu/roms/SLOF/lib/libhvcall/rfill.c b/qemu/roms/SLOF/lib/libhvcall/rfill.c
new file mode 100644
index 000000000..5407cd2a6
--- /dev/null
+++ b/qemu/roms/SLOF/lib/libhvcall/rfill.c
@@ -0,0 +1,38 @@
+/*****************************************************************************
+ * Fast function for filling cache-inhibited memory regions via h-call.
+ *
+ * Copyright 2015 Red Hat, Inc.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the BSD License
+ * which accompanies this distribution, and is available at
+ * http://www.opensource.org/licenses/bsd-license.php
+ *
+ * Contributors:
+ *     Thomas Huth, Red Hat Inc. - initial implementation
+ *****************************************************************************/
+
+#include <cache.h>
+#include <string.h>
+
+typedef unsigned long type_u;
+
+/**
+ * fast_rfill is the implementation of the FAST_RFILL macro with h-calls.
+ * This is defined here instead of cache.h since we need a temporary
+ * local buffer - and that caused stack size problems in engine() when
+ * we used it directly in the FAST_RFILL macro.
+ */
+void fast_rfill(char *dst, long size, char pat)
+{
+	type_u buf[64];
+
+	memset(buf, pat, size < sizeof(buf) ? size : sizeof(buf));
+
+	while (size > sizeof(buf)) {
+		FAST_MRMOVE(buf, dst, sizeof(buf));
+		dst += sizeof(buf);
+		size -= sizeof(buf);
+	}
+	FAST_MRMOVE(buf, dst, size);
+}
diff --git a/qemu/roms/SLOF/lib/libnvram/envvar.c b/qemu/roms/SLOF/lib/libnvram/envvar.c
index 87aaf27a0..ee943fce5 100644
--- a/qemu/roms/SLOF/lib/libnvram/envvar.c
+++ b/qemu/roms/SLOF/lib/libnvram/envvar.c
@@ -17,7 +17,7 @@
 #include "nvram.h"
 
 /* returns the offset of the first byte after the searched envvar */
-static int get_past_env_pos(partition_t part, char *envvar)
+static int get_past_env_pos(partition_t part, char *envvar, int evlen)
 {
 	int offset, len;
 	static char temp[256];
@@ -32,7 +32,7 @@ static int get_past_env_pos(partition_t part, char *envvar)
 		while((data=nvram_read_byte(offset++)) && len < 256) {
 			temp[len++]=data;
 		}
-		if (!strncmp(envvar, temp, strlen(envvar))) {
+		if (!strncmp(envvar, temp, evlen)) {
 			return offset;
 		}
 	} while (len);
@@ -43,16 +43,16 @@ static int get_past_env_pos(partition_t part, char *envvar)
 /**
  * @param partition name of the envvar partition
  * @param envvar name of the environment variable
+ * @param evlen string length of the envvar parameter
  * @return pointer to temporary string containing the value of envvar
  */
-
-char *get_env(partition_t part, char *envvar)
+char *nvram_get_env(partition_t part, char *envvar, int evlen)
 {
 	static char temp[256+1];
 	int len, offset;
 	uint8_t data;
 
-	DEBUG("get_env %s... ", envvar);
+	DEBUG("nvram_get_env %p... ", envvar);
 	if(!part.addr) {
 		/* ERROR: No environment variable partition */
 		DEBUG("invalid partition.\n");
@@ -68,7 +68,7 @@ char *get_env(partition_t part, char *envvar)
 		}
 		temp[len]=0;
 
-		if (!strncmp(envvar, temp, strlen(envvar))) {
+		if (!strncmp(envvar, temp, evlen)) {
 			int pos=0;
 			while (temp[pos]!='=' && pos < len) pos++;
 			// DEBUG("value='%s'\n", temp+pos+1); 
@@ -100,7 +100,7 @@ static int find_last_envvar(partition_t part)
 	return -1;
 }
 
-int add_env(partition_t part, char *envvar, char *value)
+int nvram_add_env(partition_t part, char *envvar, int evlen, char *value, int vallen)
 {
 	int freespace, last, len, offset;
 	unsigned int i;
@@ -112,7 +112,7 @@ int add_env(partition_t part, char *envvar, char *value)
 	freespace = part.addr+part.len-last;
 
 	/* how long is the entry we want to write? */
-	len = strlen(envvar) + strlen(value) + 2;
+	len = evlen + vallen + 2;
 
 	if(freespace<len) {
 		// TODO try to increase partition size
@@ -121,18 +121,18 @@ int add_env(partition_t part, char *envvar, char *value)
 
 	offset=last;
 
-	for(i=0; i<strlen(envvar); i++)
+	for (i = 0; i < evlen; i++)
 		nvram_write_byte(offset++, envvar[i]);
 
 	nvram_write_byte(offset++, '=');
 
-	for(i=0; i<strlen(value); i++)
+	for (i = 0; i < vallen; i++)
 		nvram_write_byte(offset++, value[i]);
 
 	return 0;
 }
 
-int del_env(partition_t part, char *envvar)
+int nvram_del_env(partition_t part, char *envvar, int evlen)
 {
 	int last, current, pos, i;
 	char *buffer;
@@ -141,7 +141,7 @@ int del_env(partition_t part, char *envvar)
 		return -1;
 
 	last=find_last_envvar(part);
-	current = pos = get_past_env_pos(part, envvar);
+	current = pos = get_past_env_pos(part, envvar, evlen);
 	
 	// TODO is this really required?
 	/* go back to non-0 value */
@@ -168,25 +168,25 @@ int del_env(partition_t part, char *envvar)
 	return 0;
 }
 
-int set_env(partition_t part, char *envvar, char *value)
+int nvram_set_env(partition_t part, char *envvar, int evlen, char *value, int vallen)
 {
 	char *oldvalue, *buffer;
 	int last, current, buffersize, i;
 
-	DEBUG("set_env %lx[%lx]: %s=%s\n", part.addr, part.len, envvar, value);
+	DEBUG("nvram_set_env %lx[%lx]: %p=>%p\n", part.addr, part.len, envvar, value);
 
 	if(!part.addr)
 		return -1;
 
 	/* Check whether the environment variable exists already */
-	oldvalue = get_env(part, envvar);
+	oldvalue = nvram_get_env(part, envvar, evlen);
 
-	if(oldvalue==NULL)
-		return add_env(part, envvar, value);
+	if (oldvalue == NULL)
+		return nvram_add_env(part, envvar, evlen, value, vallen);
 
 
 	/* The value did not change. So we succeeded! */
-	if(!strncmp(oldvalue, value, strlen(value)+1))
+	if (strlen(oldvalue) == vallen && !strncmp(oldvalue, value, vallen))
 		return 0;
 
 	/* we need to overwrite environment variables, back them up first */
@@ -195,7 +195,7 @@ int set_env(partition_t part, char *envvar, char *value)
 
 	/* allocate a buffer */
 	last=find_last_envvar(part);
-	current=get_past_env_pos(part, envvar);
+	current = get_past_env_pos(part, envvar, evlen);
 	buffersize = last - current;
 	buffer=get_nvram_buffer(buffersize);
 	if(!buffer)
@@ -214,7 +214,7 @@ int set_env(partition_t part, char *envvar, char *value)
 	current++;
 
 	/* Write the new value */
-	for(i=0; i<(int)strlen(value); i++) {
+	for(i = 0; i < vallen; i++) {
 		nvram_write_byte(current++, value[i]);
 	}
 	
diff --git a/qemu/roms/SLOF/lib/libnvram/libnvram.code b/qemu/roms/SLOF/lib/libnvram/libnvram.code
index 723941d3e..8481f57f5 100644
--- a/qemu/roms/SLOF/lib/libnvram/libnvram.code
+++ b/qemu/roms/SLOF/lib/libnvram/libnvram.code
@@ -11,17 +11,6 @@
  *****************************************************************************/
 #include <nvram.h>
 
-#define STRING_INIT(str)	\
-	char str[255];		\
-	char * str##_address;	\
-	int  str##_length;
-
-#define STRING_FROM_STACK(str)				\
-	str##_length = TOS.u; POP;				\
-	str##_address = TOS.a; POP;			\
-	memcpy(str, str##_address, str##_length);	\
-	memset(str + str##_length, 0, 255 - str##_length);
-
 PRIM(nvram_X2d_c_X40)
 	unsigned int offset = TOS.u;
 	TOS.u=nvram_read_byte(offset);
@@ -80,21 +69,18 @@ MIRP
 
 /* get-named-nvram-partition ( name.addr name.len -- addr len FAILED? ) */
 PRIM(get_X2d_named_X2d_nvram_X2d_partition)
-	STRING_INIT(name)
 	partition_t partition;
+	int namelen = TOS.n; POP;
 
-	STRING_FROM_STACK(name)
-	partition = get_partition(-1, name);
+	partition = get_partition_fs(TOS.a, namelen);
 
 	if(partition.len && partition.len != -1) {
-		PUSH;
 		TOS.u = partition.addr;
 		PUSH;
 		TOS.u = partition.len;
 		PUSH;
 		TOS.u = 0; // FALSE
 	} else {
-		PUSH;
 		TOS.u = -1; // TRUE
 	}
 MIRP
@@ -103,23 +89,16 @@ MIRP
 
 /* new-nvram-partition ( type name.addr name.len len -- part.offs part.len FALSE | TRUE) */
 PRIM(new_X2d_nvram_X2d_partition)
-	int type, len, i, slen;
-	char name[12], *addr;
+	int type, len, namelen;
 	partition_t partition;
+	char *name;
 
 	len = TOS.u; POP;
-	slen = TOS.u; POP;
-	addr = (char *)TOS.u; POP;
+	namelen = TOS.u; POP;
+	name = (char *)TOS.u; POP;
 	type = TOS.u; POP;
 
-	for (i=0; i<12; i++) {
-		if(slen>i)
-			name[i]=addr[i];
-		else
-			name[i]=0;
-	}
-
-	partition=new_nvram_partition(type, name, len);
+	partition = new_nvram_partition_fs(type, name, namelen, len);
 
 	if(!partition.len) {
 		PUSH; TOS.u = -1; // TRUE
@@ -165,15 +144,17 @@ MIRP
 
 // ( part.start part.len name.addr name.len -- var.addr var.len TRUE | false )
 PRIM(internal_X2d_get_X2d_env)
-	STRING_INIT(name)
+	char *name;
+	int namelen;
 	partition_t part;
 	char *val;
 
-	STRING_FROM_STACK(name)
+	namelen = TOS.u; POP;
+	name = TOS.a; POP;
 	part.len = TOS.u; POP;
 	part.addr = TOS.u; POP;
 
-	val=get_env(part, name);
+	val = nvram_get_env(part, name, namelen);
 	if(val) {
 		PUSH; TOS.a = val;
 		PUSH; TOS.u = strlen(val);
@@ -185,17 +166,19 @@ MIRP
 
 // ( part.start part.len name.addr name.len val.addr val.len -- FALSE|TRUE)
 PRIM(internal_X2d_add_X2d_env)
-	STRING_INIT(name)
-	STRING_INIT(value)
+	char *name, *val;
+	int namelen, vallen;
 	partition_t part;
 	int ret;
 
-	STRING_FROM_STACK(value)
-	STRING_FROM_STACK(name)
+	vallen = TOS.u; POP;
+	val = TOS.a; POP;
+	namelen = TOS.u; POP;
+	name = TOS.a; POP;
 	part.len = TOS.u; POP;
 	part.addr = TOS.u; POP;
 
-	ret=add_env(part, name, value);
+	ret = nvram_add_env(part, name, namelen, val, vallen);
 	if(ret) {
 		PUSH; TOS.u = -1; // TRUE
 	} else {
@@ -205,15 +188,17 @@ MIRP
 
 // ( part.addr part.len name.addr name.len -- FALSE|TRUE)
 PRIM(internal_X2d_del_X2d_env)
-	STRING_INIT(name)
+	char *name;
+	int namelen;
 	partition_t part;
 	int ret;
 
-	STRING_FROM_STACK(name);
+	namelen = TOS.u; POP;
+	name = TOS.a; POP;
 	part.len = TOS.u; POP;
 	part.addr = TOS.u; POP;
 
-	ret=del_env(part, name);
+	ret = nvram_del_env(part, name, namelen);
 	if(ret) {
 		PUSH; TOS.u = -1; // TRUE
 	} else {
@@ -224,17 +209,19 @@ MIRP
 
 // internal-set-env ( part.addr part.len name.addr name.len val.addr val.len -- FALSE|TRUE)
 PRIM(internal_X2d_set_X2d_env)
-	STRING_INIT(name)
-	STRING_INIT(value)
+	char *name, *value;
+	int namelen, valuelen;
 	partition_t part;
 	int ret;
 
-	STRING_FROM_STACK(value)
-	STRING_FROM_STACK(name)
+	valuelen = TOS.u; POP;
+	value = TOS.a; POP;
+	namelen = TOS.u; POP;
+	name = TOS.a; POP;
 	part.len = TOS.u; POP;
 	part.addr = TOS.u; POP;
 
-	ret=set_env(part, name, value);
+	ret = nvram_set_env(part, name, namelen, value, valuelen);
 	if(ret) {
 		PUSH; TOS.u = -1; // TRUE
 	} else {
diff --git a/qemu/roms/SLOF/lib/libnvram/nvram.c b/qemu/roms/SLOF/lib/libnvram/nvram.c
index 5c1137669..473814e5c 100644
--- a/qemu/roms/SLOF/lib/libnvram/nvram.c
+++ b/qemu/roms/SLOF/lib/libnvram/nvram.c
@@ -358,6 +358,17 @@ partition_t get_partition(unsigned int type, char *name)
 	return ret;
 }
 
+/* Get partition specified by a Forth string */
+partition_t get_partition_fs(char *name, int namelen)
+{
+	char buf[namelen + 1];
+
+	memcpy(buf, name, namelen);
+	buf[namelen] = 0;
+
+	return get_partition(-1, buf);
+}
+
 void erase_nvram(int offset, int len)
 {
 	int i;
@@ -466,6 +477,22 @@ partition_t new_nvram_partition(int type, char *name, int len)
 	return new_part;
 }
 
+partition_t new_nvram_partition_fs(int type, char *name, int namelen, int len)
+{
+	char buf[13];
+	int i;
+
+	for (i = 0; i < 12; i++) {
+		if (i < namelen)
+			buf[i] = name[i];
+		else
+			buf[i] = 0;
+	}
+	buf[12] = 0;
+
+	return new_nvram_partition(type, buf, len);
+}
+
 /**
  * @param partition   partition structure pointing to the partition to wipe.
  */
diff --git a/qemu/roms/SLOF/lib/libnvram/nvram.h b/qemu/roms/SLOF/lib/libnvram/nvram.h
index fa6bdd425..73fe44424 100644
--- a/qemu/roms/SLOF/lib/libnvram/nvram.h
+++ b/qemu/roms/SLOF/lib/libnvram/nvram.h
@@ -51,9 +51,11 @@ char *get_nvram_buffer(int len);
 void free_nvram_buffer(char *buffer);
 int nvramlog_printf(const char* fmt, ...);
 partition_t get_partition(unsigned int type, char *name);
+partition_t get_partition_fs(char *name, int namelen);
 void erase_nvram(int offset, int len);
 int wipe_partition(partition_t partition, int header_only);
 partition_t new_nvram_partition(int type, char *name, int len);
+partition_t new_nvram_partition_fs(int type, char *name, int namelen, int len);
 int increase_nvram_partition_size(partition_t partition, int newsize);
 int clear_nvram_partition(partition_t part);
 int delete_nvram_partition(partition_t part);
@@ -65,9 +67,9 @@ void nvram_init(uint32_t store_token, uint32_t fetch_token,
 unsigned int get_nvram_size(void);
 
 /* envvar.c */
-char *get_env(partition_t part, char *envvar);
-int add_env(partition_t part, char *envvar, char *value);
-int del_env(partition_t part, char *envvar);
-int set_env(partition_t part, char *envvar, char *value);
+char *nvram_get_env(partition_t part, char *envvar, int evlen);
+int nvram_add_env(partition_t part, char *envvar, int evlen, char *value, int vallen);
+int nvram_del_env(partition_t part, char *envvar, int evlen);
+int nvram_set_env(partition_t part, char *envvar, int evlen, char *val, int vlen);
 
 #endif
diff --git a/qemu/roms/SLOF/lib/libusb/usb-core.c b/qemu/roms/SLOF/lib/libusb/usb-core.c
index 6719c5726..4c720ce2f 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-core.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-core.c
@@ -383,8 +383,6 @@ int usb_hid_exit(void *vdev)
 	return true;
 }
 
-#define usb_get_intf_class(x) ((x & 0x00FF0000) >> 16)
-
 int usb_msc_init(void *vdev)
 {
 	struct usb_dev *dev;
@@ -420,7 +418,7 @@ int usb_msc_exit(void *vdev)
 	return true;
 }
 
-static int usb_msc_reset(struct usb_dev *dev)
+int usb_msc_reset(struct usb_dev *dev)
 {
 	struct usb_dev_req req;
 
@@ -477,7 +475,7 @@ static int usb_handle_device(struct usb_dev *dev, struct usb_dev_config_descr *c
 		case DESCR_TYPE_HUB:
 			break;
 		default:
-			printf("ptr %p desc_type %d\n", ptr, desc_type);
+			dprintf("ptr %p desc_type %d\n", ptr, desc_type);
 		}
 		ptr += desc_len;
 		len -= desc_len;
@@ -485,7 +483,7 @@ static int usb_handle_device(struct usb_dev *dev, struct usb_dev_config_descr *c
 	return true;
 }
 
-int setup_new_device(struct usb_dev *dev, unsigned int port)
+int usb_setup_new_device(struct usb_dev *dev, unsigned int port)
 {
 	struct usb_dev_descr descr;
 	struct usb_dev_config_descr cfg;
@@ -552,35 +550,6 @@ int setup_new_device(struct usb_dev *dev, unsigned int port)
 	if (!usb_handle_device(dev, &cfg, data, len))
 		goto fail_mem_free;
 
-	switch (usb_get_intf_class(dev->class)) {
-	case 3:
-		dprintf("HID found %06X\n", dev->class);
-		slof_usb_handle(dev);
-		break;
-	case 8:
-		dprintf("MASS STORAGE found %d %06X\n", dev->intf_num,
-			dev->class);
-		if ((dev->class & 0x50) != 0x50) { /* Bulk-only supported */
-			printf("Device not supported %06X\n", dev->class);
-			goto fail_mem_free;
-		}
-
-		if (!usb_msc_reset(dev)) {
-			printf("%s: bulk reset failed\n", __func__);
-			goto fail_mem_free;
-		}
-		SLOF_msleep(100);
-		slof_usb_handle(dev);
-		break;
-	case 9:
-		dprintf("HUB found\n");
-		slof_usb_handle(dev);
-		break;
-	default:
-		printf("USB Interface class -%x- Not supported\n", dev->class);
-		break;
-	}
-
 	SLOF_dma_free(data, len);
 	return true;
 fail_mem_free:
diff --git a/qemu/roms/SLOF/lib/libusb/usb-core.h b/qemu/roms/SLOF/lib/libusb/usb-core.h
index 7441979e9..a35df3485 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-core.h
+++ b/qemu/roms/SLOF/lib/libusb/usb-core.h
@@ -261,6 +261,8 @@ struct usb_hcd_ops {
 	unsigned int usb_type;
 };
 
+#define usb_get_intf_class(x) ((x & 0x00FF0000) >> 16)
+
 extern void usb_hcd_register(struct usb_hcd_ops *ops);
 extern struct usb_pipe *usb_get_pipe(struct usb_dev *dev, struct usb_ep_descr *ep,
 				char *buf, size_t len);
@@ -269,11 +271,12 @@ extern int usb_poll_intr(struct usb_pipe *pipe, uint8_t *buf);
 extern int usb_send_ctrl(struct usb_pipe *pipe, struct usb_dev_req *req, void *data);
 extern struct usb_dev *usb_devpool_get(void);
 extern void usb_devpool_put(struct usb_dev *);
-extern int setup_new_device(struct usb_dev *dev, unsigned int port);
-extern int slof_usb_handle(struct usb_dev *dev);
+extern int usb_setup_new_device(struct usb_dev *dev, unsigned int port);
+extern void usb_slof_populate_new_device(struct usb_dev *dev);
 extern int usb_dev_populate_pipe(struct usb_dev *dev, struct usb_ep_descr *ep,
 				void *buf, size_t len);
 extern int usb_hid_kbd_init(struct usb_dev *dev);
 extern int usb_hid_kbd_exit(struct usb_dev *dev);
+extern int usb_msc_reset(struct usb_dev *dev);
 extern void usb_msc_resetrecovery(struct usb_dev *dev);
 #endif
diff --git a/qemu/roms/SLOF/lib/libusb/usb-ehci.c b/qemu/roms/SLOF/lib/libusb/usb-ehci.c
index 4cca0da15..60af9e101 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-ehci.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-ehci.c
@@ -79,7 +79,9 @@ static int ehci_hub_check_ports(struct ehci_hcd *ehcd)
 			dprintf("usb-ehci: allocated device %p\n", dev);
 			dev->hcidev = ehcd->hcidev;
 			dev->speed = USB_HIGH_SPEED; /* TODO: Check for Low/Full speed device */
-			if (!setup_new_device(dev, i))
+			if (usb_setup_new_device(dev, i))
+				usb_slof_populate_new_device(dev);
+			else
 				printf("usb-ehci: unable to setup device on port %d\n", i);
 		}
 	}
diff --git a/qemu/roms/SLOF/lib/libusb/usb-hid.c b/qemu/roms/SLOF/lib/libusb/usb-hid.c
index f0cab8a69..ac6616aba 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-hid.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-hid.c
@@ -28,6 +28,10 @@
 #define HID_REQ_SET_IDLE                0x0A
 #define HID_REQ_SET_PROTOCOL            0x0B
 
+//key position for latin letters
+#define KEYP_LATIN_A 4
+#define KEYP_LATIN_Z 29
+
 //#define KEY_DEBUG
 
 /* HID SPEC - 7.2.6 Set_Protocol Request */
@@ -83,6 +87,8 @@ uint8_t set_leds;
 const uint8_t *key_std       = NULL;
 const uint8_t *key_std_shift = NULL;
 
+uint8_t ctrl; /* modifiers */
+
 /**
  * read character from Keyboard-Buffer
  *
@@ -111,6 +117,16 @@ static void write_key(uint8_t key)
 }
 
 /**
+ * Checks if keypos is a latin key
+ * @param  keypos
+ * @return -
+ */
+static bool is_latin(uint8_t keypos)
+{
+	return keypos >= KEYP_LATIN_A && keypos <= KEYP_LATIN_Z;
+}
+
+/**
  * Convert keyboard usage-ID to ANSI-Code
  *
  * @param   Ctrl=Modifier Byte
@@ -120,22 +136,24 @@ static void write_key(uint8_t key)
 static void get_char(uint8_t ctrl, uint8_t keypos)
 {
 	uint8_t ch;
+	bool caps = false;
 
 #ifdef KEY_DEBUG
 	printf("pos %02X\n", keypos);
 #endif
 
 	if (set_leds & LED_CAPS_LOCK)	                /* is CAPS Lock set ? */
-		ctrl |= MODIFIER_SHIFT;	                    /* simulate shift */
+		caps = true;
 
-	if (ctrl == 0) {
+	/* caps is a shift only for latin chars */
+	if ((!caps && ctrl == 0) || (caps && !is_latin(keypos))) {
 		ch = key_std[keypos];
 		if (ch != 0)
 			write_key(ch);
 		return;
 	}
 
-	if (ctrl & MODIFIER_SHIFT) {
+	if ((ctrl & MODIFIER_SHIFT) || caps) {
 		ch = key_std_shift[keypos];
 		if (ch != 0)
 			write_key(ch);
@@ -187,36 +205,38 @@ static void check_key_code(uint8_t *buf)
 					set_leds ^= LED_CAPS_LOCK;
 					break;
 
+				case 0x36:		                /*Shift pressed*/
+					ctrl |= MODIFIER_SHIFT;
+					break;
+				case 0xb6:		                /*Shift unpressed*/
+					ctrl &= ~MODIFIER_SHIFT;
+					break;
 				case 0x3a:	                        /* F1 */
 					write_key(0x1b);
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x31);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x50);
 					break;
 
 				case 0x3b:		                /* F2 */
 					write_key(0x1b);
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x32);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x51);
 					break;
 
 				case 0x3c:
 					write_key(0x1b);               /* F3 */
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x33);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x52);
 					break;
 
 				case 0x3d:
 					write_key(0x1b);		/* F4 */
 					write_key(0x5b);
-					write_key(0x31);
-					write_key(0x34);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x53);
 					break;
 
 				case 0x3e:
@@ -254,7 +274,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x42:
 					write_key(0x1b);		/* F9 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x30);
 					write_key(0x7e);
 					break;
@@ -262,7 +282,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x43:
 					write_key(0x1b);	       /* F10 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x31);
 					write_key(0x7e);
 					break;
@@ -270,7 +290,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x44:
 					write_key(0x1b);	       /* F11 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x33);
 					write_key(0x7e);
 					break;
@@ -278,7 +298,7 @@ static void check_key_code(uint8_t *buf)
 				case 0x45:
 					write_key(0x1b);	       /* F12 */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x34);
 					write_key(0x7e);
 					break;
@@ -290,36 +310,34 @@ static void check_key_code(uint8_t *buf)
 				case 0x49:
 					write_key(0x1b);	       /* INS */
 					write_key(0x5b);
-					write_key(0x31);
+					write_key(0x32);
 					write_key(0x7e);
 					break;
 
 				case 0x4a:
 					write_key(0x1b);	      /* HOME */
-					write_key(0x5b);
-					write_key(0x32);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x48);
 					break;
 
 				case 0x4b:
 					write_key(0x1b);	      /* PgUp */
 					write_key(0x5b);
-					write_key(0x33);
+					write_key(0x35);
 					write_key(0x7e);
 					break;
 
 				case 0x4c:
 					write_key(0x1b);	       /* DEL */
 					write_key(0x5b);
-					write_key(0x34);
+					write_key(0x33);
 					write_key(0x7e);
 					break;
 
 				case 0x4d:
 					write_key(0x1b);	       /* END */
-					write_key(0x5b);
-					write_key(0x35);
-					write_key(0x7e);
+					write_key(0x4f);
+					write_key(0x46);
 					break;
 
 				case 0x4e:
@@ -443,11 +461,8 @@ unsigned char usb_key_available(void *dev)
 
 unsigned char usb_read_keyb(void *vdev)
 {
-	if (!vdev)
-		return false;
-
-	while (usb_poll_key(vdev)) {
-		/* loop for all pending keys */
-	}
-	return read_key();
+	if (usb_key_available(vdev))
+		return read_key();
+	else
+		return 0;
 }
diff --git a/qemu/roms/SLOF/lib/libusb/usb-hub.c b/qemu/roms/SLOF/lib/libusb/usb-hub.c
index 7059cd019..bb8a30915 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-hub.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-hub.c
@@ -175,7 +175,9 @@ unsigned int usb_hub_init(void *hubdev)
 			newdev = usb_devpool_get();
 			dprintf("usb-hub: allocated device %p\n", newdev);
 			newdev->hcidev = dev->hcidev;
-			if (!setup_new_device(newdev, i))
+			if (usb_setup_new_device(newdev, i))
+				usb_slof_populate_new_device(newdev);
+			else
 				printf("usb-hub: unable to setup device on port %d\n", i);
 		}
 	}
diff --git a/qemu/roms/SLOF/lib/libusb/usb-ohci.c b/qemu/roms/SLOF/lib/libusb/usb-ohci.c
index 0e8400481..d06c754d1 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-ohci.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-ohci.c
@@ -192,7 +192,9 @@ static void ohci_hub_check_ports(struct ohci_hcd *ohcd)
 			dev = usb_devpool_get();
 			dprintf("usb-ohci: Device reset, setting up %p\n", dev);
 			dev->hcidev = ohcd->hcidev;
-			if (!setup_new_device(dev, i))
+			if (usb_setup_new_device(dev, i))
+				usb_slof_populate_new_device(dev);
+			else
 				printf("usb-ohci: unable to setup device on port %d\n", i);
 		}
 		if (port_status & RH_PS_PESC) {
@@ -252,7 +254,7 @@ static int ohci_alloc_pipe_pool(struct ohci_hcd *ohcd)
 		return false;
 
 	ohcd->pool_phys = opipe_phys = SLOF_dma_map_in(opipe, OHCI_PIPE_POOL_SIZE, true);
-	dprintf("usb-ohci: %s opipe %x, opipe_phys %x size %d count %d\n",
+	dprintf("usb-ohci: %s opipe %p, opipe_phys %lx size %ld count %d\n",
 		__func__, opipe, opipe_phys, sizeof(*opipe), count);
 	/* Although an array, link them*/
 	for (i = 0, curr = opipe, prev = NULL; i < count; i++, curr++) {
@@ -446,7 +448,7 @@ again:
 	/* Interrupt is there, read from done_head pointer */
 	td_phys = (struct ohci_td *)(uint64_t) le32_to_cpu(hcca->done_head);
 	if (!td_phys) {
-		dprintf("Again td_phys null %ld\n");
+		dprintf("Again td_phys null\n");
 		goto again;
 	}
 	hcca->done_head = 0;
@@ -553,7 +555,7 @@ static int ohci_send_ctrl(struct usb_pipe *pipe, struct usb_dev_req *req, void *
 	attr = EDA_FADDR(pipe->dev->addr) | EDA_MPS(pipe->mps) | EDA_SKIP;
 	ohci_fill_ed(ed, PTR_U32(td_phys), td_next, attr, 0);
 	ed->tailp = 0; /* HACK */
-	dprintf("usb-ohci: %s - td_start %x td_end %x req %x\n", __func__,
+	dprintf("usb-ohci: %s - td_start %p td_end %lx req %lx\n", __func__,
 		td_phys, td_next, req_phys);
 	mb();
 	ed->attr &= cpu_to_le32(~EDA_SKIP);
@@ -642,7 +644,7 @@ static int ohci_transfer_bulk(struct usb_pipe *pipe, void *td_ptr,
 
 	td = tds = (struct ohci_td *) td_ptr;
 	td_phys = (long)td_phys_ptr;
-	dprintf("usb-ohci: %s pipe %p data_phys %p len %d DIR_IN %d td %p td_phys %p\n",
+	dprintf("usb-ohci: %s pipe %p data_phys %p len %d DIR_IN %d td %p td_phys %lx\n",
 		__func__, pipe, data_phys, datalen, dir, td, td_phys);
 
 	if (!tds) {
@@ -672,7 +674,7 @@ static int ohci_transfer_bulk(struct usb_pipe *pipe, void *td_ptr,
 		| EDA_SKIP | pipe->dev->speed | EDA_EP(pipe->epno);
 	td_next = ohci_get_td_phys(td, tds, td_phys);
 	ohci_fill_ed(ed, td_phys, td_next, attr, 0);
-	dprintf("usb-ohci: %s - tds %p td %p\n", __func__, td_phys, td_next);
+	dprintf("usb-ohci: %s - tds %lx td %lx\n", __func__, td_phys, td_next);
 	mb();
 	ed->attr &= cpu_to_le32(~EDA_SKIP);
 
@@ -778,7 +780,7 @@ static int ohci_get_pipe_intr(struct usb_pipe *pipe, struct ohci_hcd *ohcd,
 		td->attr = cpu_to_le32(TDA_DP_IN | TDA_ROUNDING | TDA_CC);
 		td->next_td = cpu_to_le32(td_next);
 		td->be = cpu_to_le32(PTR_U32(ptr) + mps - 1);
-		dprintf("td %x td++ %x ptr %x be %x\n",
+		dprintf("td %p td++ %x ptr %p be %x\n",
 			td, le32_to_cpu(td->next_td),
 			ptr, (PTR_U32(ptr) + mps - 1));
 	}
diff --git a/qemu/roms/SLOF/lib/libusb/usb-slof.c b/qemu/roms/SLOF/lib/libusb/usb-slof.c
index de841f0fb..ff070559a 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-slof.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-slof.c
@@ -26,7 +26,7 @@
 #define dprintf(_x ...)
 #endif
 
-int slof_usb_handle(struct usb_dev *dev)
+static int slof_usb_handle(struct usb_dev *dev)
 {
 	struct slof_usb_dev sdev;
 	sdev.port = dev->port;
@@ -59,3 +59,35 @@ int slof_usb_handle(struct usb_dev *dev)
 	}
 	return true;
 }
+
+void usb_slof_populate_new_device(struct usb_dev *dev)
+{
+	switch (usb_get_intf_class(dev->class)) {
+	case 3:
+		dprintf("HID found %06X\n", dev->class);
+		slof_usb_handle(dev);
+		break;
+	case 8:
+		dprintf("MASS STORAGE found %d %06X\n", dev->intf_num,
+			dev->class);
+		if ((dev->class & 0x50) != 0x50) { /* Bulk-only supported */
+			printf("Device not supported %06X\n", dev->class);
+			break;
+		}
+
+		if (!usb_msc_reset(dev)) {
+			printf("%s: bulk reset failed\n", __func__);
+			break;
+		}
+		SLOF_msleep(100);
+		slof_usb_handle(dev);
+		break;
+	case 9:
+		dprintf("HUB found\n");
+		slof_usb_handle(dev);
+		break;
+	default:
+		printf("USB Interface class -%x- Not supported\n", dev->class);
+		break;
+	}
+}
diff --git a/qemu/roms/SLOF/lib/libusb/usb-xhci.c b/qemu/roms/SLOF/lib/libusb/usb-xhci.c
index 0c3d6e47f..858cd12f9 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-xhci.c
+++ b/qemu/roms/SLOF/lib/libusb/usb-xhci.c
@@ -225,11 +225,11 @@ static void xhci_handle_cmd_completion(struct xhci_hcd *xhcd,
 		xhcd->slot_id = 0;
 }
 
-static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
-					uint32_t event_type)
+static uint64_t xhci_poll_event(struct xhci_hcd *xhcd,
+				uint32_t event_type)
 {
 	struct xhci_event_trb *event;
-	uint64_t val;
+	uint64_t val, retval = 0;
 	uint32_t flags, time;
 	int index;
 
@@ -244,7 +244,7 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 		mb();
 		flags = le32_to_cpu(event->flags);
 		if (time < SLOF_GetTimer())
-			return NULL;
+			return 0;
 	}
 
 	mb();
@@ -273,6 +273,7 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 		break;
 	}
 	xhcd->ering.deq = (uint64_t) (event + 1);
+	retval = le64_to_cpu(event->addr);
 
 	event->addr = 0;
 	event->status = 0;
@@ -289,7 +290,11 @@ static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 	dprintf("Update start %x deq %x index %d\n",
 		xhcd->ering.trbs_dma, val, index/sizeof(*event));
 	write_reg64(&xhcd->run_regs->irs[0].erdp, val);
-	return event;
+
+	if (retval == 0)
+		return (uint64_t)event;
+	else
+		return retval;
 }
 
 static void xhci_send_cmd(struct xhci_hcd *xhcd, uint32_t field1,
@@ -388,10 +393,12 @@ static void xhci_init_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 	seg->deq = (uint64_t)seg->trbs;
 	memset((void *)seg->trbs, 0, size);
 
-	link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
-	link->addr = cpu_to_le64(seg->trbs_dma);
-	link->field2 = 0;
-	link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
+	if (type != TYPE_EVENT) {
+		link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
+		link->addr = cpu_to_le64(seg->trbs_dma);
+		link->field2 = 0;
+		link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
+	}
 	return;
 }
 
@@ -601,8 +608,10 @@ static bool xhci_alloc_dev(struct xhci_hcd *xhcd, uint32_t slot_id, uint32_t por
 	dev->port = newport;
 	dev->priv = xdev;
 	xdev->dev = dev;
-	if (setup_new_device(dev, newport))
+	if (usb_setup_new_device(dev, newport)) {
+		usb_slof_populate_new_device(dev);
 		return true;
+	}
 
 	xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
 fail_control_seg:
@@ -616,6 +625,7 @@ static void xhci_free_dev(struct xhci_dev *xdev)
 {
 	xhci_free_seg(&xdev->bulk_in, XHCI_DATA_TRBS_SIZE);
 	xhci_free_seg(&xdev->bulk_out, XHCI_DATA_TRBS_SIZE);
+	xhci_free_seg(&xdev->intr, XHCI_INTR_TRBS_SIZE);
 	xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 	xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 	xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
@@ -637,7 +647,25 @@ static bool usb3_dev_init(struct xhci_hcd *xhcd, uint32_t port)
 	return true;
 }
 
-static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
+static int xhci_device_present(uint32_t portsc, uint32_t usb_ver)
+{
+	if (usb_ver == USB_XHCI) {
+		/* Device present and enabled state */
+		if ((portsc & PORTSC_CCS) &&
+			(portsc & PORTSC_PP) &&
+			(portsc & PORTSC_PED)) {
+			return true;
+		}
+	} else if (usb_ver == USB_EHCI) {
+		/* Device present and in disabled state */
+		if ((portsc & PORTSC_CCS) && (portsc & PORTSC_CSC))
+			return true;
+	}
+	return false;
+}
+
+static int xhci_port_scan(struct xhci_hcd *xhcd,
+			uint32_t usb_ver)
 {
 	uint32_t num_ports, portsc, i;
 	struct xhci_op_regs *op;
@@ -645,7 +673,7 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	struct xhci_cap_regs *cap;
 	uint32_t xecp_off;
 	uint32_t *xecp_addr, *base;
-	uint32_t port_off = 1, port_cnt;
+	uint32_t port_off = 0, port_cnt;
 
 	dprintf("enter\n");
 
@@ -658,14 +686,14 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	base = (uint32_t *)cap;
 	while (xecp_off > 0) {
 		xecp_addr = base + xecp_off;
-		dprintf(stderr, "xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
+		dprintf("xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
 
 		if (XHCI_XECP_CAP_ID(read_reg32(xecp_addr)) == XHCI_XECP_CAP_SP &&
-		    XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == 3 &&
+		    XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == usb_ver &&
 		    XHCI_XECP_CAP_SP_MN(read_reg32(xecp_addr)) == 0) {
 			port_cnt = XHCI_XECP_CAP_SP_PC(read_reg32(xecp_addr + 2));
 			port_off = XHCI_XECP_CAP_SP_PO(read_reg32(xecp_addr + 2));
-			dprintf(stderr, "PortCount %d Portoffset %d\n", port_cnt, port_off);
+			dprintf("PortCount %d Portoffset %d\n", port_cnt, port_off);
 		}
 		base = xecp_addr;
 		xecp_off = XHCI_XECP_NEXT_PTR(read_reg32(xecp_addr));
@@ -675,10 +703,8 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	for (i = (port_off - 1); i < (port_off + port_cnt - 1); i++) {
 		prs = &op->prs[i];
 		portsc = read_reg32(&prs->portsc);
-		if ((portsc & PORTSC_CCS) &&
-			(portsc & PORTSC_PP) &&
-			(portsc & PORTSC_PED)) {
-			/* Device present and enabled */
+		if (xhci_device_present(portsc, usb_ver)) {
+			/* Device present */
 			dprintf("Device present on port %d\n", i);
 			/* Reset the port */
 			portsc = read_reg32(&prs->portsc);
@@ -701,6 +727,11 @@ static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 	return true;
 }
 
+static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
+{
+	return xhci_port_scan(xhcd, USB_XHCI) | xhci_port_scan(xhcd, USB_EHCI);
+}
+
 static bool xhci_hcd_init(struct xhci_hcd *xhcd)
 {
 	struct xhci_op_regs *op;
@@ -868,6 +899,18 @@ static bool xhci_hcd_exit(struct xhci_hcd *xhcd)
 		SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 		SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 	}
+
+	/*
+	 * QEMU implementation of XHCI doesn't implement halt
+	 * properly. It basically says that it's halted immediately
+	 * but doesn't actually terminate ongoing activities and
+	 * DMAs. This needs to be fixed in QEMU.
+	 *
+	 * For now, wait for 50ms grace time till qemu stops using
+	 * this device.
+	 */
+	SLOF_msleep(50);
+
 	return true;
 }
 
@@ -1079,18 +1122,17 @@ static inline struct xhci_seg *xhci_pipe_get_seg(struct usb_pipe *pipe)
 static inline void *xhci_get_trb(struct xhci_seg *seg)
 {
 	uint64_t val, enq;
-	uint32_t size;
+	int index;
 	struct xhci_link_trb *link;
 
 	enq = val = seg->enq;
 	val = val + XHCI_TRB_SIZE;
-	size = seg->size * XHCI_TRB_SIZE;
-        /* TRBs being a cyclic buffer, here we cycle back to beginning. */
-	if ((val % size) == 0) {
+	index = (enq - (uint64_t)seg->trbs) / XHCI_TRB_SIZE + 1;
+	dprintf("%s: enq %llx, val %llx %x\n", __func__, enq, val, index);
+	/* TRBs being a cyclic buffer, here we cycle back to beginning. */
+	if (index == (seg->size - 1)) {
+		dprintf("%s: rounding \n", __func__);
 		seg->enq = (uint64_t)seg->trbs;
-		enq = seg->enq;
-		seg->enq = seg->enq + XHCI_TRB_SIZE;
-		val = 0;
 		seg->cycle_state ^= seg->cycle_state;
 		link = (struct xhci_link_trb *) (seg->trbs + seg->size - 1);
 		link->addr = cpu_to_le64(seg->trbs_dma);
@@ -1105,6 +1147,12 @@ static inline void *xhci_get_trb(struct xhci_seg *seg)
 	return (void *)enq;
 }
 
+static uint64_t xhci_get_trb_phys(struct xhci_seg *seg, uint64_t trb)
+{
+	return seg->trbs_dma + (trb - (uint64_t)seg->trbs);
+}
+
+static int usb_kb = false;
 static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 			void *data, int datalen)
 {
@@ -1114,7 +1162,8 @@ static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 	struct xhci_transfer_trb *trb;
 	struct xhci_db_regs *dbr;
 	int ret = true;
-	uint32_t slot_id, epno;
+	uint32_t slot_id, epno, time;
+	uint64_t trb_phys, event_phys;
 
 	if (!pipe->dev || !pipe->dev->hcidev) {
 		dprintf(" NULL pointer\n");
@@ -1139,13 +1188,26 @@ static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
 	}
 
 	trb = xhci_get_trb(seg);
+	trb_phys = xhci_get_trb_phys(seg, (uint64_t)trb);
 	fill_normal_trb(trb, (void *)data, datalen);
 
 	epno = xhci_get_epno(pipe);
 	write_reg32(&dbr->db[slot_id], epno);
-	if (!xhci_poll_event(xhcd, 0)) {
-		dprintf("Bulk failed\n");
-		ret = false;
+
+	time = SLOF_GetTimer() + USB_TIMEOUT;
+	while (1) {
+		event_phys = xhci_poll_event(xhcd, 0);
+		if (event_phys == trb_phys) {
+			break;
+		} else if (event_phys == 0) { /* polling timed out */
+			ret = false;
+			break;
+		} else
+			usb_kb = true;
+
+		/* transfer timed out */
+		if (time < SLOF_GetTimer())
+			return false;
 	}
 	trb->addr = 0;
 	trb->len = 0;
@@ -1214,7 +1276,8 @@ static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
 
 	if (!seg->trbs) {
 		if (!xhci_alloc_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK)) {
-			dprintf("Failed allocating seg\n");
+			printf("usb-xhci: allocation failed for bulk endpoint\n");
+			return;
 		}
 	} else {
 		xhci_init_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK);
@@ -1235,6 +1298,61 @@ static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
 	xpipe->seg = seg;
 }
 
+static int xhci_get_pipe_intr(struct usb_pipe *pipe,
+			struct xhci_hcd *xhcd,
+			char *buf, size_t len)
+{
+	struct xhci_dev *xdev;
+	struct xhci_seg *seg;
+	struct xhci_pipe *xpipe;
+	struct xhci_control_ctx *ctrl;
+	struct xhci_ep_ctx *ep;
+	uint32_t x_epno, val, type;
+	struct usb_dev *dev;
+	struct xhci_transfer_trb *trb;
+
+	dev = pipe->dev;
+	if (dev->class != DEV_HID_KEYB)
+		return false;
+
+	xdev = dev->priv;
+	pipe->mps = 8;
+	seg = xhci_pipe_get_seg(pipe);
+	xpipe = xhci_pipe_get_xpipe(pipe);
+	type = EP_INT_IN;
+	seg = &xdev->intr;
+
+	if (!seg->trbs) {
+		if (!xhci_alloc_seg(seg, XHCI_INTR_TRBS_SIZE, TYPE_BULK)) {
+			printf("usb-xhci: allocation failed for interrupt endpoint\n");
+			return false;
+		}
+	} else {
+		xhci_init_seg(seg, XHCI_EVENT_TRBS_SIZE, TYPE_BULK);
+	}
+
+	xpipe->buf = buf;
+	xpipe->buf_phys = SLOF_dma_map_in(buf, len, false);
+	xpipe->buflen = len;
+
+	ctrl = xhci_get_control_ctx(&xdev->in_ctx);
+	x_epno = xhci_get_epno(pipe);
+	ep = xhci_get_ep_ctx(&xdev->in_ctx, xdev->ctx_size, x_epno);
+	val = EP_TYPE(type) | MAX_BURST(0) | ERROR_COUNT(3) |
+		MAX_PACKET_SIZE(pipe->mps);
+	ep->field2 = cpu_to_le32(val);
+	ep->deq_addr = cpu_to_le64(seg->trbs_dma | seg->cycle_state);
+	ep->field4 = cpu_to_le32(8);
+	ctrl->a_flags = cpu_to_le32(BIT(x_epno) | 0x1);
+	ctrl->d_flags = 0;
+	xhci_configure_ep(xhcd, xdev->slot_id, xdev->in_ctx.dma_addr);
+	xpipe->seg = seg;
+
+	trb = xhci_get_trb(seg);
+	fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
+	return true;
+}
+
 static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *ep, char *buf, size_t len)
 {
 	struct xhci_hcd *xhcd;
@@ -1264,6 +1382,12 @@ static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *
 	new->dir = (ep->bEndpointAddress & 0x80) >> 7;
 	new->epno = ep->bEndpointAddress & 0x0f;
 
+	if (new->type == USB_EP_TYPE_INTR) {
+		if (!xhci_get_pipe_intr(new, xhcd, buf, len)) {
+			printf("usb-xhci: %s alloc_intr failed  %p\n",
+				__func__, new);
+		}
+	}
 	if (new->type == USB_EP_TYPE_BULK)
 		xhci_init_bulk_ep(dev, new);
 
@@ -1284,6 +1408,10 @@ static void xhci_put_pipe(struct usb_pipe *pipe)
 	if (pipe->type == USB_EP_TYPE_BULK) {
 		xpipe = xhci_pipe_get_xpipe(pipe);
 		xpipe->seg = NULL;
+	} else if (pipe->type == USB_EP_TYPE_INTR) {
+		xpipe = xhci_pipe_get_xpipe(pipe);
+		SLOF_dma_map_out(xpipe->buf_phys, xpipe->buf, xpipe->buflen);
+		xpipe->seg = NULL;
 	}
 	if (xhcd->end)
 		xhcd->end->next = pipe;
@@ -1298,6 +1426,51 @@ static void xhci_put_pipe(struct usb_pipe *pipe)
 	dprintf("usb-xhci: %s exit\n", __func__);
 }
 
+static int xhci_poll_intr(struct usb_pipe *pipe, uint8_t *data)
+{
+	struct xhci_transfer_trb *trb;
+	struct xhci_seg *seg;
+	struct xhci_pipe *xpipe;
+	struct xhci_dev *xdev;
+	struct xhci_hcd *xhcd;
+	struct xhci_db_regs *dbr;
+	uint32_t x_epno;
+	uint8_t *buf, ret = 1;
+
+	if (!pipe || !pipe->dev || !pipe->dev->hcidev)
+		return 0;
+	xdev = pipe->dev->priv;
+	xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
+	x_epno = xhci_get_epno(pipe);
+	seg = xhci_pipe_get_seg(pipe);
+	xpipe = xhci_pipe_get_xpipe(pipe);
+
+	if (usb_kb == true) {
+		/* This event was consumed by bulk transfer */
+		usb_kb = false;
+		goto skip_poll;
+	}
+	buf = xpipe->buf;
+	memset(buf, 0, 8);
+
+	mb();
+	/* Ring the doorbell - x_epno */
+	dbr = xhcd->db_regs;
+	write_reg32(&dbr->db[xdev->slot_id], x_epno);
+	if (!xhci_poll_event(xhcd, 0)) {
+		printf("poll intr failed\n");
+		return 0;
+	}
+	mb();
+	memcpy(data, buf, 8);
+
+skip_poll:
+	trb = xhci_get_trb(seg);
+	fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
+	mb();
+	return ret;
+}
+
 struct usb_hcd_ops xhci_ops = {
 	.name          = "xhci-hcd",
 	.init          = xhci_init,
@@ -1305,6 +1478,7 @@ struct usb_hcd_ops xhci_ops = {
 	.usb_type      = USB_XHCI,
 	.get_pipe      = xhci_get_pipe,
 	.put_pipe      = xhci_put_pipe,
+	.poll_intr     = xhci_poll_intr,
 	.send_ctrl     = xhci_send_ctrl,
 	.transfer_bulk = xhci_transfer_bulk,
 	.next          = NULL,
diff --git a/qemu/roms/SLOF/lib/libusb/usb-xhci.h b/qemu/roms/SLOF/lib/libusb/usb-xhci.h
index faeb07ead..3fc7e7889 100644
--- a/qemu/roms/SLOF/lib/libusb/usb-xhci.h
+++ b/qemu/roms/SLOF/lib/libusb/usb-xhci.h
@@ -266,6 +266,7 @@ struct xhci_seg {
 #define XHCI_EVENT_TRBS_SIZE   4096
 #define XHCI_CONTROL_TRBS_SIZE 4096
 #define XHCI_DATA_TRBS_SIZE    4096
+#define XHCI_INTR_TRBS_SIZE    4096
 #define XHCI_ERST_NUM_SEGS     1
 
 #define XHCI_MAX_BULK_SIZE    0xF000
@@ -349,6 +350,7 @@ struct xhci_dev {
 	struct xhci_ctx in_ctx;
 	struct xhci_ctx out_ctx;
 	struct xhci_seg control;
+	struct xhci_seg intr;
 	struct xhci_seg bulk_in;
 	struct xhci_seg bulk_out;
 	uint32_t ctx_size;
@@ -381,6 +383,9 @@ struct xhci_hcd {
 struct xhci_pipe {
 	struct usb_pipe pipe;
 	struct xhci_seg *seg;
+	void *buf;
+	long buf_phys;
+	uint32_t buflen;
 };
 
 #endif	/* USB_XHCI_H */
diff --git a/qemu/roms/SLOF/lib/libvirtio/p9.c b/qemu/roms/SLOF/lib/libvirtio/p9.c
index a55662994..0e5953031 100644
--- a/qemu/roms/SLOF/lib/libvirtio/p9.c
+++ b/qemu/roms/SLOF/lib/libvirtio/p9.c
@@ -143,7 +143,7 @@ int p9_transaction(p9_connection_t *connection)
 {
 	int rc;
 	int tx_size = GET_SIZE;
-	int rx_size = connection->message_size;
+	uint32_t rx_size = connection->message_size;
 
 	if (transact == NULL) {
 		return P9_NO_TRANSPORT;
diff --git a/qemu/roms/SLOF/lib/libvirtio/p9.h b/qemu/roms/SLOF/lib/libvirtio/p9.h
index 7df9ef441..3a35e80ed 100644
--- a/qemu/roms/SLOF/lib/libvirtio/p9.h
+++ b/qemu/roms/SLOF/lib/libvirtio/p9.h
@@ -33,7 +33,7 @@
 #define P9_PARTIAL_WALK			1
 
 typedef int (*p9_transact_t)(void *opaque, uint8_t *tx, int tx_size,
-		uint8_t *rx, int *rx_size);
+		uint8_t *rx, uint32_t *rx_size);
 
 typedef struct {
 	uint32_t message_size;
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-9p.c b/qemu/roms/SLOF/lib/libvirtio/virtio-9p.c
index 5a5fd01da..fc5db9154 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-9p.c
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-9p.c
@@ -19,6 +19,7 @@
 #include "virtio-9p.h"
 #include "p9.h"
 
+static struct vqs vq;
 
 /**
  * Notes for 9P Server config:
@@ -86,7 +87,7 @@ static void dprint_buffer(const char *name, uint8_t *buffer, int length)
  * @return	0 = success, -ve = error.
  */
 static int virtio_9p_transact(void *opaque, uint8_t *tx, int tx_size, uint8_t *rx,
-			      int *rx_size)
+			      uint32_t *rx_size)
 {
 	struct virtio_device *dev = opaque;
 	struct vring_desc *desc;
@@ -165,6 +166,7 @@ int virtio_9p_init(struct virtio_device *dev, void *tx_buf, void *rx_buf,
 		   int buf_size)
 {
 	struct vring_avail *vq_avail;
+	int status = VIRTIO_STAT_ACKNOWLEDGE;
 
 	/* Check for double open */
 	if (__buf_size)
@@ -174,28 +176,31 @@ int virtio_9p_init(struct virtio_device *dev, void *tx_buf, void *rx_buf,
         dprintf("%s : device at %p\n", __func__, dev->base);
         dprintf("%s : type is %04x\n", __func__, dev->type);
 
-	/* Reset device */
-	// XXX That will clear the virtq base. We need to move
-	//     initializing it to here anyway
-	//
-	//	 virtio_reset_device(dev);
+	/* Keep it disabled until the driver is 1.0 capable */
+	dev->is_modern = false;
+
+	virtio_reset_device(dev);
 
 	/* Acknowledge device. */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE);
+	virtio_set_status(dev, status);
 
 	/* Tell HV that we know how to drive the device. */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE | VIRTIO_STAT_DRIVER);
+	status |= VIRTIO_STAT_DRIVER;
+	virtio_set_status(dev, status);
 
 	/* Device specific setup - we do not support special features */
 	virtio_set_guest_features(dev,  0);
 
+	if (virtio_queue_init_vq(dev, &vq, 0))
+		goto dev_error;
+
 	vq_avail = virtio_get_vring_avail(dev, 0);
 	vq_avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
 	vq_avail->idx = 0;
 
 	/* Tell HV that setup succeeded */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE | VIRTIO_STAT_DRIVER
-			  |VIRTIO_STAT_DRIVER_OK);
+	status |= VIRTIO_STAT_DRIVER_OK;
+	virtio_set_status(dev, status);
 
 	/* Setup 9P library. */
 	p9_reg_transport(virtio_9p_transact, dev,(uint8_t *)tx_buf,
@@ -203,6 +208,12 @@ int virtio_9p_init(struct virtio_device *dev, void *tx_buf, void *rx_buf,
 
 	dprintf("%s : complete\n", __func__);
 	return 0;
+
+dev_error:
+	printf("%s: failed\n", __func__);
+	status |= VIRTIO_STAT_FAILED;
+	virtio_set_status(dev, status);
+	return -1;
 }
 
 /**
@@ -228,7 +239,7 @@ void virtio_9p_shutdown(struct virtio_device *dev)
  * @param buffer[out]	Where to read the file to.
  * @return	+ve = amount of data read, -ve = error.
  */
-int virtio_9p_load(struct virtio_device *dev, const char *file_name, uint8_t *buffer)
+long virtio_9p_load(struct virtio_device *dev, const char *file_name, uint8_t *buffer)
 {
 	int rc;
 	uint16_t tag_len;
@@ -332,5 +343,5 @@ cleanup_connection:
 
 
 	dprintf("%s : complete, read %llu bytes\n", __func__, offset);
-	return rc == 0 ? offset : rc;
+	return rc == 0 ? (long)offset : rc;
 }
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-9p.h b/qemu/roms/SLOF/lib/libvirtio/virtio-9p.h
index 4bf47d078..db2cf6f11 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-9p.h
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-9p.h
@@ -26,7 +26,7 @@ typedef struct {
 int virtio_9p_init(struct virtio_device *dev, void *tx_buf, void *rx_buf,
 		   int buf_size);
 void virtio_9p_shutdown(struct virtio_device *dev);
-int virtio_9p_load(struct virtio_device *dev, const char *file_name, uint8_t *buffer);
+long virtio_9p_load(struct virtio_device *dev, const char *file_name, uint8_t *buffer);
 
 
 #endif /* VIRTIO_9P_H_ */
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-blk.c b/qemu/roms/SLOF/lib/libvirtio/virtio-blk.c
index 826f2ea0e..07ec1048f 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-blk.c
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-blk.c
@@ -13,10 +13,15 @@
 #include <stdio.h>
 #include <cpu.h>
 #include <helpers.h>
+#include <byteorder.h>
 #include "virtio.h"
 #include "virtio-blk.h"
+#include "virtio-internal.h"
 
 #define DEFAULT_SECTOR_SIZE 512
+#define DRIVER_FEATURE_SUPPORT  (VIRTIO_BLK_F_BLK_SIZE | VIRTIO_F_VERSION_1)
+
+static struct vqs vq;
 
 /**
  * Initialize virtio-block device.
@@ -27,39 +32,54 @@ virtioblk_init(struct virtio_device *dev)
 {
 	struct vring_avail *vq_avail;
 	int blk_size = DEFAULT_SECTOR_SIZE;
-	int features;
+	uint64_t features;
+	int status = VIRTIO_STAT_ACKNOWLEDGE;
 
 	/* Reset device */
-	// XXX That will clear the virtq base. We need to move
-	//     initializing it to here anyway
-	//
-	//	 virtio_reset_device(dev);
+	virtio_reset_device(dev);
 
 	/* Acknowledge device. */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE);
+	virtio_set_status(dev, status);
 
 	/* Tell HV that we know how to drive the device. */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER);
+	status |= VIRTIO_STAT_DRIVER;
+	virtio_set_status(dev, status);
+
+	if (dev->is_modern) {
+		/* Negotiate features and sets FEATURES_OK if successful */
+		if (virtio_negotiate_guest_features(dev, DRIVER_FEATURE_SUPPORT))
+			goto dev_error;
+
+		virtio_get_status(dev, &status);
+	} else {
+		/* Device specific setup - we support F_BLK_SIZE */
+		virtio_set_guest_features(dev,  VIRTIO_BLK_F_BLK_SIZE);
+	}
 
-	/* Device specific setup - we support F_BLK_SIZE */
-	virtio_set_guest_features(dev,  VIRTIO_BLK_F_BLK_SIZE);
+	if (virtio_queue_init_vq(dev, &vq, 0))
+		goto dev_error;
 
 	vq_avail = virtio_get_vring_avail(dev, 0);
-	vq_avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
+	vq_avail->flags = virtio_cpu_to_modern16(dev, VRING_AVAIL_F_NO_INTERRUPT);
 	vq_avail->idx = 0;
 
 	/* Tell HV that setup succeeded */
-	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER
-				|VIRTIO_STAT_DRIVER_OK);
+	status |= VIRTIO_STAT_DRIVER_OK;
+	virtio_set_status(dev, status);
 
-	virtio_get_host_features(dev, &features);
+	features = virtio_get_host_features(dev);
 	if (features & VIRTIO_BLK_F_BLK_SIZE) {
 		blk_size = virtio_get_config(dev,
-				offset_of(struct virtio_blk_cfg, blk_size),
-				sizeof(blk_size));
+					     offset_of(struct virtio_blk_cfg, blk_size),
+					     sizeof(blk_size));
 	}
 
 	return blk_size;
+dev_error:
+	printf("%s: failed\n", __func__);
+	status |= VIRTIO_STAT_FAILED;
+	virtio_set_status(dev, status);
+	return 0;
 }
 
 
@@ -77,6 +97,19 @@ virtioblk_shutdown(struct virtio_device *dev)
 	virtio_reset_device(dev);
 }
 
+static void fill_blk_hdr(struct virtio_blk_req *blkhdr, bool is_modern,
+                         uint32_t type, uint32_t ioprio, uint32_t sector)
+{
+	if (is_modern) {
+		blkhdr->type = cpu_to_le32(type);
+		blkhdr->ioprio = cpu_to_le32(ioprio);
+		blkhdr->sector = cpu_to_le64(sector);
+	} else {
+		blkhdr->type = type;
+		blkhdr->ioprio = ioprio;
+		blkhdr->sector = sector;
+	}
+}
 
 /**
  * Read blocks
@@ -87,7 +120,7 @@ virtioblk_shutdown(struct virtio_device *dev)
  * @return number of blocks that have been read successfully
  */
 int
-virtioblk_read(struct virtio_device *dev, char *buf, long blocknum, long cnt)
+virtioblk_read(struct virtio_device *dev, char *buf, uint64_t blocknum, long cnt)
 {
 	struct vring_desc *desc;
 	int id;
@@ -100,7 +133,7 @@ virtioblk_read(struct virtio_device *dev, char *buf, long blocknum, long cnt)
 	struct vring_used *vq_used;		/* "Used" vring */
 	volatile uint8_t status = -1;
 	volatile uint16_t *current_used_idx;
-	uint16_t last_used_idx;
+	uint16_t last_used_idx, avail_idx;
 	int blk_size = DEFAULT_SECTOR_SIZE;
 
 	//printf("virtioblk_read: dev=%p buf=%p blocknum=%li count=%li\n",
@@ -128,41 +161,38 @@ virtioblk_read(struct virtio_device *dev, char *buf, long blocknum, long cnt)
 	vq_avail = virtio_get_vring_avail(dev, 0);
 	vq_used = virtio_get_vring_used(dev, 0);
 
+	avail_idx = virtio_modern16_to_cpu(dev, vq_avail->idx);
+
 	last_used_idx = vq_used->idx;
 	current_used_idx = &vq_used->idx;
 
 	/* Set up header */
-	blkhdr.type = VIRTIO_BLK_T_IN | VIRTIO_BLK_T_BARRIER;
-	blkhdr.ioprio = 1;
-	blkhdr.sector = blocknum * blk_size / DEFAULT_SECTOR_SIZE;
+	fill_blk_hdr(&blkhdr, dev->is_modern, VIRTIO_BLK_T_IN | VIRTIO_BLK_T_BARRIER,
+		     1, blocknum * blk_size / DEFAULT_SECTOR_SIZE);
 
 	/* Determine descriptor index */
-	id = (vq_avail->idx * 3) % vq_size;
+	id = (avail_idx * 3) % vq_size;
 
 	/* Set up virtqueue descriptor for header */
 	desc = &vq_desc[id];
-	desc->addr = (uint64_t)&blkhdr;
-	desc->len = sizeof(struct virtio_blk_req);
-	desc->flags = VRING_DESC_F_NEXT;
-	desc->next = (id + 1) % vq_size;
+	virtio_fill_desc(desc, dev->is_modern, (uint64_t)&blkhdr,
+			 sizeof(struct virtio_blk_req),
+			 VRING_DESC_F_NEXT, (id + 1) % vq_size);
 
 	/* Set up virtqueue descriptor for data */
 	desc = &vq_desc[(id + 1) % vq_size];
-	desc->addr = (uint64_t)buf;
-	desc->len = cnt * blk_size;
-	desc->flags = VRING_DESC_F_NEXT | VRING_DESC_F_WRITE;
-	desc->next = (id + 2) % vq_size;
+	virtio_fill_desc(desc, dev->is_modern, (uint64_t)buf, cnt * blk_size,
+			 VRING_DESC_F_NEXT | VRING_DESC_F_WRITE,
+			 (id + 2) % vq_size);
 
 	/* Set up virtqueue descriptor for status */
 	desc = &vq_desc[(id + 2) % vq_size];
-	desc->addr = (uint64_t)&status;
-	desc->len = 1;
-	desc->flags = VRING_DESC_F_WRITE;
-	desc->next = 0;
+	virtio_fill_desc(desc, dev->is_modern, (uint64_t)&status, 1,
+			 VRING_DESC_F_WRITE, 0);
 
-	vq_avail->ring[vq_avail->idx % vq_size] = id;
+	vq_avail->ring[avail_idx % vq_size] = virtio_cpu_to_modern16 (dev, id);
 	mb();
-	vq_avail->idx += 1;
+	vq_avail->idx = virtio_cpu_to_modern16(dev, avail_idx + 1);
 
 	/* Tell HV that the queue is ready */
 	virtio_queue_notify(dev, 0);
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-blk.h b/qemu/roms/SLOF/lib/libvirtio/virtio-blk.h
index ac8bf2896..2e7b5926b 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-blk.h
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-blk.h
@@ -55,6 +55,6 @@ struct virtio_blk_req {
 
 extern int virtioblk_init(struct virtio_device *dev);
 extern void virtioblk_shutdown(struct virtio_device *dev);
-extern int virtioblk_read(struct virtio_device *dev, char *buf, long blocknum, long cnt);
+extern int virtioblk_read(struct virtio_device *dev, char *buf, uint64_t blocknum, long cnt);
 
 #endif  /* _VIRTIO_BLK_H */
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-internal.h b/qemu/roms/SLOF/lib/libvirtio/virtio-internal.h
new file mode 100644
index 000000000..08662eab7
--- /dev/null
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-internal.h
@@ -0,0 +1,48 @@
+/******************************************************************************
+ * Copyright (c) 2016 IBM Corporation
+ * All rights reserved.
+ * This program and the accompanying materials
+ * are made available under the terms of the BSD License
+ * which accompanies this distribution, and is available at
+ * http://www.opensource.org/licenses/bsd-license.php
+ *
+ * Contributors:
+ *     IBM Corporation - initial implementation
+ *****************************************************************************/
+
+#ifndef _LIBVIRTIO_INTERNAL_H
+#define _LIBVIRTIO_INTERNAL_H
+
+#include <byteorder.h>
+
+static inline uint16_t virtio_cpu_to_modern16(struct virtio_device *dev, uint16_t val)
+{
+	return dev->is_modern ? cpu_to_le16(val) : val;
+}
+
+static inline uint32_t virtio_cpu_to_modern32(struct virtio_device *dev, uint32_t val)
+{
+	return dev->is_modern ? cpu_to_le32(val) : val;
+}
+
+static inline uint64_t virtio_cpu_to_modern64(struct virtio_device *dev, uint64_t val)
+{
+	return dev->is_modern ? cpu_to_le64(val) : val;
+}
+
+static inline uint16_t virtio_modern16_to_cpu(struct virtio_device *dev, uint16_t val)
+{
+	return dev->is_modern ? le16_to_cpu(val) : val;
+}
+
+static inline uint32_t virtio_modern32_to_cpu(struct virtio_device *dev, uint32_t val)
+{
+	return dev->is_modern ? le32_to_cpu(val) : val;
+}
+
+static inline uint64_t virtio_modern64_to_cpu(struct virtio_device *dev, uint64_t val)
+{
+	return dev->is_modern ? le64_to_cpu(val) : val;
+}
+
+#endif /* _LIBVIRTIO_INTERNAL_H */
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-net.c b/qemu/roms/SLOF/lib/libvirtio/virtio-net.c
index 99c19d952..fc620a201 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-net.c
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-net.c
@@ -26,6 +26,7 @@
 #include <byteorder.h>
 #include "virtio.h"
 #include "virtio-net.h"
+#include "virtio-internal.h"
 
 #undef DEBUG
 //#define DEBUG
@@ -37,22 +38,13 @@
 
 #define sync()  asm volatile (" sync \n" ::: "memory")
 
-/* PCI virtio header offsets */
-#define VIRTIOHDR_DEVICE_FEATURES       0
-#define VIRTIOHDR_GUEST_FEATURES        4
-#define VIRTIOHDR_QUEUE_ADDRESS         8
-#define VIRTIOHDR_QUEUE_SIZE            12
-#define VIRTIOHDR_QUEUE_SELECT          14
-#define VIRTIOHDR_QUEUE_NOTIFY          16
-#define VIRTIOHDR_DEVICE_STATUS         18
-#define VIRTIOHDR_ISR_STATUS            19
-#define VIRTIOHDR_DEVICE_CONFIG         20
-#define VIRTIOHDR_MAC_ADDRESS           20
+#define DRIVER_FEATURE_SUPPORT  (VIRTIO_NET_F_MAC | VIRTIO_F_VERSION_1)
 
 struct virtio_device virtiodev;
-struct vqs vq[2];     /* Information about virtqueues */
+static struct vqs vq_rx;     /* Information about receive virtqueues */
+static struct vqs vq_tx;     /* Information about transmit virtqueues */
 
-/* See Virtio Spec, appendix C, "Device Operation" */ 
+/* See Virtio Spec, appendix C, "Device Operation" */
 struct virtio_net_hdr {
 	uint8_t  flags;
 	uint8_t  gso_type;
@@ -63,6 +55,18 @@ struct virtio_net_hdr {
 	// uint16_t  num_buffers;	/* Only if VIRTIO_NET_F_MRG_RXBUF */
 };
 
+static unsigned int net_hdr_size;
+
+struct virtio_net_hdr_v1 {
+	uint8_t  flags;
+	uint8_t  gso_type;
+	le16  hdr_len;
+	le16  gso_size;
+	le16  csum_start;
+	le16  csum_offset;
+	le16  num_buffers;
+};
+
 static uint16_t last_rx_idx;	/* Last index in RX "used" ring */
 
 /**
@@ -72,15 +76,13 @@ static uint16_t last_rx_idx;	/* Last index in RX "used" ring */
  */
 static int virtionet_init_pci(struct virtio_device *dev)
 {
-	int i;
-
 	dprintf("virtionet: doing virtionet_init_pci!\n");
 
 	if (!dev)
 		return -1;
 
-	virtiodev.base = dev->base;
-	virtiodev.type = dev->type;
+	/* make a copy of the device structure */
+	memcpy(&virtiodev, dev, sizeof(struct virtio_device));
 
 	/* Reset device */
 	virtio_reset_device(&virtiodev);
@@ -90,29 +92,11 @@ static int virtionet_init_pci(struct virtio_device *dev)
 	 * second the transmit queue, and the forth is the control queue for
 	 * networking options.
 	 * We are only interested in the receive and transmit queue here. */
-
-	for (i=VQ_RX; i<=VQ_TX; i++) {
-		/* Select ring (0=RX, 1=TX): */
-		vq[i].id = i-VQ_RX;
-		ci_write_16(virtiodev.base+VIRTIOHDR_QUEUE_SELECT,
-			    cpu_to_le16(vq[i].id));
-
-		vq[i].size = le16_to_cpu(ci_read_16(virtiodev.base+VIRTIOHDR_QUEUE_SIZE));
-		vq[i].desc = SLOF_alloc_mem_aligned(virtio_vring_size(vq[i].size), 4096);
-		if (!vq[i].desc) {
-			printf("memory allocation failed!\n");
-			return -1;
-		}
-		memset(vq[i].desc, 0, virtio_vring_size(vq[i].size));
-		ci_write_32(virtiodev.base+VIRTIOHDR_QUEUE_ADDRESS,
-			    cpu_to_le32((long)vq[i].desc / 4096));
-		vq[i].avail = (void*)vq[i].desc
-				    + vq[i].size * sizeof(struct vring_desc);
-		vq[i].used = (void*)VQ_ALIGN((long)vq[i].avail
-				    + vq[i].size * sizeof(struct vring_avail));
-
-		dprintf("%i: vq.id = %llx\nvq.size =%x\n vq.avail =%p\nvq.used=%p\n",
-			i, vq[i].id, vq[i].size, vq[i].avail, vq[i].used);
+	if (virtio_queue_init_vq(dev, &vq_rx, VQ_RX) ||
+	    virtio_queue_init_vq(dev, &vq_tx, VQ_TX)) {
+		virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER
+				  |VIRTIO_STAT_FAILED);
+		return -1;
 	}
 
 	/* Acknowledge device. */
@@ -129,6 +113,7 @@ static int virtionet_init_pci(struct virtio_device *dev)
 static int virtionet_init(net_driver_t *driver)
 {
 	int i;
+	int status = VIRTIO_STAT_ACKNOWLEDGE | VIRTIO_STAT_DRIVER;
 
 	dprintf("virtionet_init(%02x:%02x:%02x:%02x:%02x:%02x)\n",
 		driver->mac_addr[0], driver->mac_addr[1],
@@ -139,60 +124,69 @@ static int virtionet_init(net_driver_t *driver)
 		return 0;
 
 	/* Tell HV that we know how to drive the device. */
-	virtio_set_status(&virtiodev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER);
-
-	/* Device specific setup - we do not support special features right now */
-	virtio_set_guest_features(&virtiodev,  0);
+	virtio_set_status(&virtiodev, status);
+
+	/* Device specific setup */
+	if (virtiodev.is_modern) {
+		if (virtio_negotiate_guest_features(&virtiodev, DRIVER_FEATURE_SUPPORT))
+			goto dev_error;
+		net_hdr_size = sizeof(struct virtio_net_hdr_v1);
+		virtio_get_status(&virtiodev, &status);
+	} else {
+		net_hdr_size = sizeof(struct virtio_net_hdr);
+		virtio_set_guest_features(&virtiodev,  0);
+	}
 
 	/* Allocate memory for one transmit an multiple receive buffers */
-	vq[VQ_RX].buf_mem = SLOF_alloc_mem((BUFFER_ENTRY_SIZE+sizeof(struct virtio_net_hdr))
+	vq_rx.buf_mem = SLOF_alloc_mem((BUFFER_ENTRY_SIZE+net_hdr_size)
 				   * RX_QUEUE_SIZE);
-	if (!vq[VQ_RX].buf_mem) {
+	if (!vq_rx.buf_mem) {
 		printf("virtionet: Failed to allocate buffers!\n");
-		virtio_set_status(&virtiodev, VIRTIO_STAT_FAILED);
-		return -1;
+		goto dev_error;
 	}
 
 	/* Prepare receive buffer queue */
 	for (i = 0; i < RX_QUEUE_SIZE; i++) {
-		struct vring_desc *desc;
+		uint64_t addr = (uint64_t)vq_rx.buf_mem
+			+ i * (BUFFER_ENTRY_SIZE+net_hdr_size);
+		uint32_t id = i*2;
 		/* Descriptor for net_hdr: */
-		desc = &vq[VQ_RX].desc[i*2];
-		desc->addr = (uint64_t)vq[VQ_RX].buf_mem
-			     + i * (BUFFER_ENTRY_SIZE+sizeof(struct virtio_net_hdr));
-		desc->len = sizeof(struct virtio_net_hdr);
-		desc->flags = VRING_DESC_F_NEXT | VRING_DESC_F_WRITE;
-		desc->next = i*2+1;
+		virtio_fill_desc(&vq_rx.desc[id], virtiodev.is_modern, addr, net_hdr_size,
+				 VRING_DESC_F_NEXT | VRING_DESC_F_WRITE, id + 1);
 
 		/* Descriptor for data: */
-		desc = &vq[VQ_RX].desc[i*2+1];
-		desc->addr = vq[VQ_RX].desc[i*2].addr + sizeof(struct virtio_net_hdr);
-		desc->len = BUFFER_ENTRY_SIZE;
-		desc->flags = VRING_DESC_F_WRITE;
-		desc->next = 0;
+		virtio_fill_desc(&vq_rx.desc[id+1], virtiodev.is_modern, addr + net_hdr_size,
+				 BUFFER_ENTRY_SIZE, VRING_DESC_F_WRITE, 0);
 
-		vq[VQ_RX].avail->ring[i] = i*2;
+		vq_rx.avail->ring[i] = virtio_cpu_to_modern16(&virtiodev, id);
 	}
 	sync();
-	vq[VQ_RX].avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
-	vq[VQ_RX].avail->idx = RX_QUEUE_SIZE;
 
-	last_rx_idx = vq[VQ_RX].used->idx;
+	vq_rx.avail->flags = virtio_cpu_to_modern16(&virtiodev, VRING_AVAIL_F_NO_INTERRUPT);
+	vq_rx.avail->idx = virtio_cpu_to_modern16(&virtiodev, RX_QUEUE_SIZE);
+
+	last_rx_idx = virtio_modern16_to_cpu(&virtiodev, vq_rx.used->idx);
 
-	vq[VQ_TX].avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
-	vq[VQ_TX].avail->idx = 0;
+	vq_tx.avail->flags = virtio_cpu_to_modern16(&virtiodev, VRING_AVAIL_F_NO_INTERRUPT);
+	vq_tx.avail->idx = 0;
 
 	/* Tell HV that setup succeeded */
-	virtio_set_status(&virtiodev, VIRTIO_STAT_ACKNOWLEDGE
-				      |VIRTIO_STAT_DRIVER
-				      |VIRTIO_STAT_DRIVER_OK);
+	status |= VIRTIO_STAT_DRIVER_OK;
+	virtio_set_status(&virtiodev, status);
 
 	/* Tell HV that RX queues are ready */
 	virtio_queue_notify(&virtiodev, VQ_RX);
 
 	driver->running = 1;
-
+	for(i = 0; i < (int)sizeof(driver->mac_addr); i++) {
+		driver->mac_addr[i] = virtio_get_config(&virtiodev, i, 1);
+	}
 	return 0;
+
+dev_error:
+	status |= VIRTIO_STAT_FAILED;
+	virtio_set_status(&virtiodev, status);
+	return -1;
 }
 
 
@@ -225,9 +219,10 @@ static int virtionet_term(net_driver_t *driver)
  */
 static int virtionet_xmit(char *buf, int len)
 {
-	struct vring_desc *desc;
-	int id;
-	static struct virtio_net_hdr nethdr;
+	int id, idx;
+	static struct virtio_net_hdr_v1 nethdr_v1;
+	static struct virtio_net_hdr nethdr_legacy;
+	void *nethdr = &nethdr_legacy;
 
 	if (len > BUFFER_ENTRY_SIZE) {
 		printf("virtionet: Packet too big!\n");
@@ -236,28 +231,25 @@ static int virtionet_xmit(char *buf, int len)
 
 	dprintf("\nvirtionet_xmit(packet at %p, %d bytes)\n", buf, len);
 
-	memset(&nethdr, 0, sizeof(nethdr));
+	if (virtiodev.is_modern)
+		nethdr = &nethdr_v1;
+
+	memset(nethdr, 0, net_hdr_size);
 
 	/* Determine descriptor index */
-	id = (vq[VQ_TX].avail->idx * 2) % vq[VQ_TX].size;
+	idx = virtio_modern16_to_cpu(&virtiodev, vq_tx.avail->idx);
+	id = (idx * 2) % vq_tx.size;
 
 	/* Set up virtqueue descriptor for header */
-	desc = &vq[VQ_TX].desc[id];
-	desc->addr = (uint64_t)&nethdr;
-	desc->len = sizeof(struct virtio_net_hdr);
-	desc->flags = VRING_DESC_F_NEXT;
-	desc->next = id + 1;
+	virtio_fill_desc(&vq_tx.desc[id], virtiodev.is_modern, (uint64_t)nethdr,
+			 net_hdr_size, VRING_DESC_F_NEXT, id + 1);
 
 	/* Set up virtqueue descriptor for data */
-	desc = &vq[VQ_TX].desc[id+1];
-	desc->addr = (uint64_t)buf;
-	desc->len = len;
-	desc->flags = 0;
-	desc->next = 0;
+	virtio_fill_desc(&vq_tx.desc[id+1], virtiodev.is_modern, (uint64_t)buf, len, 0, 0);
 
-	vq[VQ_TX].avail->ring[vq[VQ_TX].avail->idx % vq[VQ_TX].size] = id;
+	vq_tx.avail->ring[idx % vq_tx.size] = virtio_cpu_to_modern16(&virtiodev, id);
 	sync();
-	vq[VQ_TX].avail->idx += 1;
+	vq_tx.avail->idx = virtio_cpu_to_modern16(&virtiodev, idx + 1);
 	sync();
 
 	/* Tell HV that TX queue is ready */
@@ -272,23 +264,24 @@ static int virtionet_xmit(char *buf, int len)
  */
 static int virtionet_receive(char *buf, int maxlen)
 {
-	int len = 0;
-	int id;
+	uint32_t len = 0;
+	uint32_t id, idx;
 
-	if (last_rx_idx == vq[VQ_RX].used->idx) {
+	idx = virtio_modern16_to_cpu(&virtiodev, vq_rx.used->idx);
+
+	if (last_rx_idx == idx) {
 		/* Nothing received yet */
 		return 0;
 	}
 
-	id = (vq[VQ_RX].used->ring[last_rx_idx % vq[VQ_RX].size].id + 1)
-	     % vq[VQ_RX].size;
-	len = vq[VQ_RX].used->ring[last_rx_idx % vq[VQ_RX].size].len
-	      - sizeof(struct virtio_net_hdr);
-
-	dprintf("virtionet_receive() last_rx_idx=%i, vq[VQ_RX].used->idx=%i,"
-		" id=%i len=%i\n", last_rx_idx, vq[VQ_RX].used->idx, id, len);
+	id = (virtio_modern32_to_cpu(&virtiodev, vq_rx.used->ring[last_rx_idx % vq_rx.size].id) + 1)
+		% vq_rx.size;
+	len = virtio_modern32_to_cpu(&virtiodev, vq_rx.used->ring[last_rx_idx % vq_rx.size].len)
+		- net_hdr_size;
+	dprintf("virtionet_receive() last_rx_idx=%i, vq_rx.used->idx=%i,"
+		" id=%i len=%i\n", last_rx_idx, vq_rx.used->idx, id, len);
 
-	if (len > maxlen) {
+	if (len > (uint32_t)maxlen) {
 		printf("virtio-net: Receive buffer not big enough!\n");
 		len = maxlen;
 	}
@@ -298,7 +291,7 @@ static int virtionet_receive(char *buf, int maxlen)
 	printf("\n");
 	int i;
 	for (i=0; i<64; i++) {
-		printf(" %02x", *(uint8_t*)(vq[VQ_RX].desc[id].addr+i));
+		printf(" %02x", *(uint8_t*)(vq_rx.desc[id].addr+i));
 		if ((i%16)==15)
 			printf("\n");
 	}
@@ -306,14 +299,14 @@ static int virtionet_receive(char *buf, int maxlen)
 #endif
 
 	/* Copy data to destination buffer */
-	memcpy(buf, (void*)vq[VQ_RX].desc[id].addr, len);
+	memcpy(buf, (void *)virtio_modern64_to_cpu(&virtiodev, vq_rx.desc[id].addr), len);
 
 	/* Move indices to next entries */
 	last_rx_idx = last_rx_idx + 1;
 
-	vq[VQ_RX].avail->ring[vq[VQ_RX].avail->idx % vq[VQ_RX].size] = id - 1;
+	vq_rx.avail->ring[idx % vq_rx.size] = virtio_cpu_to_modern16(&virtiodev, id - 1);
 	sync();
-	vq[VQ_RX].avail->idx += 1;
+	vq_rx.avail->idx = virtio_cpu_to_modern16(&virtiodev, idx + 1);
 
 	/* Tell HV that RX queue entry is ready */
 	virtio_queue_notify(&virtiodev, VQ_RX);
@@ -321,7 +314,7 @@ static int virtionet_receive(char *buf, int maxlen)
 	return len;
 }
 
-net_driver_t *virtionet_open(char *mac_addr, int len, struct virtio_device *dev)
+net_driver_t *virtionet_open(struct virtio_device *dev)
 {
 	net_driver_t *driver;
 
@@ -331,7 +324,6 @@ net_driver_t *virtionet_open(char *mac_addr, int len, struct virtio_device *dev)
 		return NULL;
 	}
 
-	memcpy(driver->mac_addr, mac_addr, 6);
 	driver->running = 0;
 
 	if (virtionet_init_pci(dev))
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-net.h b/qemu/roms/SLOF/lib/libvirtio/virtio-net.h
index bc7a189f7..c2d8ee336 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-net.h
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-net.h
@@ -23,19 +23,10 @@ enum {
 	VQ_TX = 1,	/* Transmit Queue */
 };
 
-struct vqs {
-	uint64_t id;	/* Queue ID */
-	uint32_t size;
-	void *buf_mem;
-	struct vring_desc *desc;
-	struct vring_avail *avail;
-	struct vring_used *used;
-};
-
-/* Device is identified by RX queue ID: */
-#define DEVICE_ID  vq[0].id
+/* VIRTIO_NET Feature bits */
+#define VIRTIO_NET_F_MAC       (1 << 5)
 
-extern net_driver_t *virtionet_open(char *mac_addr, int len, struct virtio_device *dev);
+extern net_driver_t *virtionet_open(struct virtio_device *dev);
 extern void virtionet_close(net_driver_t *driver);
 extern int virtionet_read(char *buf, int len);
 extern int virtionet_write(char *buf, int len);
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio-scsi.c b/qemu/roms/SLOF/lib/libvirtio/virtio-scsi.c
index 48289289a..04181b06c 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio-scsi.c
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio-scsi.c
@@ -15,6 +15,7 @@
 #include <cpu.h>
 #include <helpers.h>
 #include "virtio.h"
+#include "virtio-internal.h"
 #include "virtio-scsi.h"
 
 int virtioscsi_send(struct virtio_device *dev,
@@ -22,58 +23,54 @@ int virtioscsi_send(struct virtio_device *dev,
 		    struct virtio_scsi_resp_cmd *resp,
 		    int is_read, void *buf, uint64_t buf_len)
 {
-        struct vring_desc *desc;
-        struct vring_desc *vq_desc;		/* Descriptor vring */
-        struct vring_avail *vq_avail;		/* "Available" vring */
-        struct vring_used *vq_used;		/* "Used" vring */
-
-        volatile uint16_t *current_used_idx;
-        uint16_t last_used_idx;
-        int id;
-        uint32_t vq_size, time;
-
-        int vq = VIRTIO_SCSI_REQUEST_VQ;
-
-        vq_size = virtio_get_qsize(dev, vq);
-        vq_desc = virtio_get_vring_desc(dev, vq);
-        vq_avail = virtio_get_vring_avail(dev, vq);
-        vq_used = virtio_get_vring_used(dev, vq);
-
-        last_used_idx = vq_used->idx;
-        current_used_idx = &vq_used->idx;
-
-        /* Determine descriptor index */
-        id = (vq_avail->idx * 3) % vq_size;
-
-        desc = &vq_desc[id];
-        desc->addr = (uint64_t)req;
-        desc->len = sizeof(*req);
-        desc->flags = VRING_DESC_F_NEXT;
-        desc->next = (id + 1) % vq_size;
-
-        /* Set up virtqueue descriptor for data */
-        desc = &vq_desc[(id + 1) % vq_size];
-        desc->addr = (uint64_t)resp;
-        desc->len = sizeof(*resp);
-        desc->flags = VRING_DESC_F_NEXT | VRING_DESC_F_WRITE;
-        desc->next = (id + 2) % vq_size;
-
-        if (buf && buf_len) {
-                /* Set up virtqueue descriptor for status */
-                desc = &vq_desc[(id + 2) % vq_size];
-                desc->addr = (uint64_t)buf;
-                desc->len = buf_len;
-                desc->flags = is_read ? VRING_DESC_F_WRITE : 0;
-                desc->next = 0;
-        } else
-                desc->flags &= ~VRING_DESC_F_NEXT;
-
-        vq_avail->ring[vq_avail->idx % vq_size] = id;
-        mb();
-        vq_avail->idx += 1;
-
-        /* Tell HV that the vq is ready */
-        virtio_queue_notify(dev, vq);
+	struct vring_desc *vq_desc;		/* Descriptor vring */
+	struct vring_avail *vq_avail;		/* "Available" vring */
+	struct vring_used *vq_used;		/* "Used" vring */
+
+	volatile uint16_t *current_used_idx;
+	uint16_t last_used_idx, avail_idx;
+	int id;
+	uint32_t vq_size, time;
+
+	int vq = VIRTIO_SCSI_REQUEST_VQ;
+
+	vq_size = virtio_get_qsize(dev, vq);
+	vq_desc = virtio_get_vring_desc(dev, vq);
+	vq_avail = virtio_get_vring_avail(dev, vq);
+	vq_used = virtio_get_vring_used(dev, vq);
+
+	avail_idx = virtio_modern16_to_cpu(dev, vq_avail->idx);
+
+	last_used_idx = vq_used->idx;
+	current_used_idx = &vq_used->idx;
+
+	/* Determine descriptor index */
+	id = (avail_idx * 3) % vq_size;
+	virtio_fill_desc(&vq_desc[id], dev->is_modern, (uint64_t)req, sizeof(*req), VRING_DESC_F_NEXT,
+			 (id + 1) % vq_size);
+
+	/* Set up virtqueue descriptor for data */
+	if (buf && buf_len) {
+		virtio_fill_desc(&vq_desc[(id + 1) % vq_size], dev->is_modern,
+				 (uint64_t)resp, sizeof(*resp),
+				 VRING_DESC_F_NEXT | VRING_DESC_F_WRITE,
+				 (id + 2) % vq_size);
+		/* Set up virtqueue descriptor for status */
+		virtio_fill_desc(&vq_desc[(id + 2) % vq_size], dev->is_modern,
+				 (uint64_t)buf, buf_len,
+				 (is_read ? VRING_DESC_F_WRITE : 0), 0);
+	} else {
+		virtio_fill_desc(&vq_desc[(id + 1) % vq_size], dev->is_modern,
+				 (uint64_t)resp, sizeof(*resp),
+				 VRING_DESC_F_WRITE, 0);
+	}
+
+	vq_avail->ring[avail_idx % vq_size] = virtio_cpu_to_modern16(dev, id);
+	mb();
+	vq_avail->idx = virtio_cpu_to_modern16(dev, avail_idx + 1);
+
+	/* Tell HV that the vq is ready */
+	virtio_queue_notify(dev, vq);
 
 	/* Wait for host to consume the descriptor */
 	time = SLOF_GetTimer() + VIRTIO_TIMEOUT;
@@ -84,7 +81,7 @@ int virtioscsi_send(struct virtio_device *dev,
 			break;
 	}
 
-        return 0;
+	return 0;
 }
 
 /**
@@ -93,42 +90,55 @@ int virtioscsi_send(struct virtio_device *dev,
  */
 int virtioscsi_init(struct virtio_device *dev)
 {
-        struct vring_avail *vq_avail;
-        unsigned int idx = 0;
-        int qsize = 0;
-
-        /* Reset device */
-        // XXX That will clear the virtq base. We need to move
-        //     initializing it to here anyway
-        //
-        //     virtio_reset_device(dev);
-
-        /* Acknowledge device. */
-        virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE);
+	struct vring_avail *vq_avail;
+	unsigned int idx = 0;
+	int qsize = 0;
+	int status = VIRTIO_STAT_ACKNOWLEDGE;
 
-        /* Tell HV that we know how to drive the device. */
-        virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER);
-
-        /* Device specific setup - we do not support special features right now */
-        virtio_set_guest_features(dev,  0);
+	/* Reset device */
+	// XXX That will clear the virtq base. We need to move
+	//     initializing it to here anyway
+	//
+	//     virtio_reset_device(dev);
+
+	/* Acknowledge device. */
+	virtio_set_status(dev, status);
+
+	/* Tell HV that we know how to drive the device. */
+	status |= VIRTIO_STAT_DRIVER;
+	virtio_set_status(dev, status);
+
+	/* Device specific setup - we do not support special features right now */
+	if (dev->is_modern) {
+		if (virtio_negotiate_guest_features(dev, VIRTIO_F_VERSION_1))
+			goto dev_error;
+		virtio_get_status(dev, &status);
+	} else {
+		virtio_set_guest_features(dev, 0);
+	}
 
-        while(1) {
-                qsize = virtio_get_qsize(dev, idx);
-                if (!qsize)
-                        break;
-                virtio_vring_size(qsize);
+	while(1) {
+		qsize = virtio_get_qsize(dev, idx);
+		if (!qsize)
+			break;
+		virtio_vring_size(qsize);
 
-                vq_avail = virtio_get_vring_avail(dev, 0);
-                vq_avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
-                vq_avail->idx = 0;
-                idx++;
-        }
+		vq_avail = virtio_get_vring_avail(dev, idx);
+		vq_avail->flags = virtio_cpu_to_modern16(dev, VRING_AVAIL_F_NO_INTERRUPT);
+		vq_avail->idx = 0;
+		idx++;
+	}
 
 	/* Tell HV that setup succeeded */
- 	virtio_set_status(dev, VIRTIO_STAT_ACKNOWLEDGE|VIRTIO_STAT_DRIVER
-                          |VIRTIO_STAT_DRIVER_OK);
+	status |= VIRTIO_STAT_DRIVER_OK;
+	virtio_set_status(dev, status);
 
 	return 0;
+dev_error:
+	printf("%s: failed\n", __func__);
+	status |= VIRTIO_STAT_FAILED;
+	virtio_set_status(dev, status);
+	return -1;
 }
 
 /**
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio.c b/qemu/roms/SLOF/lib/libvirtio/virtio.c
index f9c00a67a..f189941c7 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio.c
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio.c
@@ -10,10 +10,16 @@
  *     IBM Corporation - initial implementation
  *****************************************************************************/
 
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stddef.h>
 #include <cpu.h>
 #include <cache.h>
 #include <byteorder.h>
 #include "virtio.h"
+#include "helpers.h"
 
 /* PCI virtio header offsets */
 #define VIRTIOHDR_DEVICE_FEATURES	0
@@ -26,6 +32,166 @@
 #define VIRTIOHDR_ISR_STATUS		19
 #define VIRTIOHDR_DEVICE_CONFIG 	20
 
+/* PCI defines */
+#define PCI_BASE_ADDR_SPACE_IO	0x01
+#define PCI_BASE_ADDR_SPACE_64BIT 0x04
+#define PCI_BASE_ADDR_MEM_MASK	(~0x0fUL)
+#define PCI_BASE_ADDR_IO_MASK	(~0x03UL)
+
+#define PCI_BASE_ADDR_REG_0	0x10
+#define PCI_CONFIG_CAP_REG	0x34
+
+#define PCI_CAP_ID_VNDR		0x9
+
+/* Common configuration */
+#define VIRTIO_PCI_CAP_COMMON_CFG       1
+/* Notifications */
+#define VIRTIO_PCI_CAP_NOTIFY_CFG       2
+/* ISR access */
+#define VIRTIO_PCI_CAP_ISR_CFG          3
+/* Device specific configuration */
+#define VIRTIO_PCI_CAP_DEVICE_CFG       4
+/* PCI configuration access */
+#define VIRTIO_PCI_CAP_PCI_CFG          5
+
+#define VIRTIO_PCI_CAP_VNDR     0	  /* Generic PCI field: PCI_CAP_ID_VNDR */
+#define VIRTIO_PCI_CAP_NEXT     1	  /* Generic PCI field: next ptr. */
+#define VIRTIO_PCI_CAP_LEN      2	  /* Generic PCI field: capability length */
+#define VIRTIO_PCI_CAP_CFG_TYPE 3	  /* Identifies the structure. */
+#define VIRTIO_PCI_CAP_BAR      4	  /* Where to find it. */
+#define VIRTIO_PCI_CAP_OFFSET   8	  /* Offset within bar. */
+#define VIRTIO_PCI_CAP_LENGTH  12	  /* Length of the structure, in bytes. */
+
+struct virtio_dev_common {
+	le32 dev_features_sel;
+	le32 dev_features;
+	le32 drv_features_sel;
+	le32 drv_features;
+	le16 msix_config;
+	le16 num_queues;
+	uint8_t dev_status;
+	uint8_t cfg_generation;
+
+	le16 q_select;
+	le16 q_size;
+	le16 q_msix_vec;
+	le16 q_enable;
+	le16 q_notify_off;
+	le64 q_desc;
+	le64 q_avail;
+	le64 q_used;
+} __attribute__ ((packed));
+
+/* virtio 1.0 Spec: 4.1.3 PCI Device Layout
+ *
+ * Fields of different sizes are present in the device configuration regions.
+ * All 64-bit, 32-bit and 16-bit fields are little-endian. 64-bit fields are to
+ * be treated as two 32-bit fields, with low 32 bit part followed by the high 32
+ * bit part.
+ */
+static void virtio_pci_write64(void *addr, uint64_t val)
+{
+	uint32_t hi = (val >> 32) & 0xFFFFFFFF;
+	uint32_t lo = val & 0xFFFFFFFF;
+
+	ci_write_32(addr, cpu_to_le32(lo));
+	ci_write_32(addr + 4, cpu_to_le32(hi));
+}
+
+static uint64_t virtio_pci_read64(void *addr)
+{
+	uint64_t hi, lo;
+
+	lo = le32_to_cpu(ci_read_32(addr));
+	hi = le32_to_cpu(ci_read_32(addr + 4));
+	return (hi << 32) | lo;
+}
+
+static void virtio_cap_set_base_addr(struct virtio_cap *cap, uint32_t offset)
+{
+	uint64_t addr;
+
+	addr = SLOF_pci_config_read32(PCI_BASE_ADDR_REG_0 + 4 * cap->bar);
+	if (addr & PCI_BASE_ADDR_SPACE_IO) {
+		addr = addr & PCI_BASE_ADDR_IO_MASK;
+		cap->is_io = 1;
+	} else {
+		if (addr & PCI_BASE_ADDR_SPACE_64BIT)
+			addr |= SLOF_pci_config_read32(PCI_BASE_ADDR_REG_0 + 4 * (cap->bar + 1)) << 32;
+		addr = addr & PCI_BASE_ADDR_MEM_MASK;
+		cap->is_io = 0;
+	}
+	addr = (uint64_t)SLOF_translate_my_address((void *)addr);
+	cap->addr = (void *)addr + offset;
+}
+
+static void virtio_process_cap(struct virtio_device *dev, uint8_t cap_ptr)
+{
+	struct virtio_cap *cap;
+	uint8_t cfg_type, bar;
+	uint32_t offset;
+
+	cfg_type = SLOF_pci_config_read8(cap_ptr + VIRTIO_PCI_CAP_CFG_TYPE);
+	bar = SLOF_pci_config_read8(cap_ptr + VIRTIO_PCI_CAP_BAR);
+	offset = SLOF_pci_config_read32(cap_ptr + VIRTIO_PCI_CAP_OFFSET);
+
+	switch(cfg_type) {
+	case VIRTIO_PCI_CAP_COMMON_CFG:
+		cap = &dev->common;
+		break;
+	case VIRTIO_PCI_CAP_NOTIFY_CFG:
+		cap = &dev->notify;
+		dev->notify_off_mul = SLOF_pci_config_read32(cap_ptr + sizeof(struct virtio_cap));
+		break;
+	case VIRTIO_PCI_CAP_ISR_CFG:
+		cap = &dev->isr;
+		break;
+	case VIRTIO_PCI_CAP_DEVICE_CFG:
+		cap = &dev->device;
+		break;
+	default:
+		return;
+	}
+
+	cap->bar = bar;
+	virtio_cap_set_base_addr(cap, offset);
+	cap->cap_id = cfg_type;
+}
+
+/**
+ * Reads the virtio device capabilities, gets called from SLOF routines The
+ * function determines legacy or modern device and sets up driver registers
+ */
+struct virtio_device *virtio_setup_vd(void)
+{
+	uint8_t cap_ptr, cap_vndr;
+	struct virtio_device *dev;
+
+	dev = SLOF_alloc_mem(sizeof(struct virtio_device));
+	if (!dev) {
+		printf("Failed to allocate memory");
+		return NULL;
+	}
+
+	cap_ptr = SLOF_pci_config_read8(PCI_CONFIG_CAP_REG);
+	while (cap_ptr != 0) {
+		cap_vndr = SLOF_pci_config_read8(cap_ptr + VIRTIO_PCI_CAP_VNDR);
+		if (cap_vndr == PCI_CAP_ID_VNDR)
+			virtio_process_cap(dev, cap_ptr);
+		cap_ptr = SLOF_pci_config_read8(cap_ptr+VIRTIO_PCI_CAP_NEXT);
+	}
+
+	if (dev->common.cap_id && dev->notify.cap_id &&
+	    dev->isr.cap_id && dev->device.cap_id) {
+		dev->is_modern = 1;
+	} else {
+		dev->is_modern = 0;
+		dev->legacy.cap_id = 0;
+		dev->legacy.bar = 0;
+		virtio_cap_set_base_addr(&dev->legacy, 0);
+	}
+	return dev;
+}
 
 /**
  * Calculate ring size according to queue size number
@@ -33,9 +199,9 @@
 unsigned long virtio_vring_size(unsigned int qsize)
 {
 	return VQ_ALIGN(sizeof(struct vring_desc) * qsize +
-                        sizeof(struct vring_avail) + sizeof(uint16_t) * qsize) +
-               VQ_ALIGN(sizeof(struct vring_used) +
-                        sizeof(struct vring_used_elem) * qsize);
+			sizeof(struct vring_avail) + sizeof(uint16_t) * qsize) +
+		VQ_ALIGN(sizeof(struct vring_used) +
+			 sizeof(struct vring_used_elem) * qsize);
 }
 
 
@@ -45,15 +211,22 @@ unsigned long virtio_vring_size(unsigned int qsize)
  * @param   queue virtio queue number
  * @return  number of elements
  */
-int virtio_get_qsize(struct virtio_device *dev, int queue)
+unsigned int virtio_get_qsize(struct virtio_device *dev, int queue)
 {
-	int size = 0;
+	unsigned int size = 0;
 
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_16(dev->base+VIRTIOHDR_QUEUE_SELECT,
+	if (dev->is_modern) {
+		void *addr = dev->common.addr + offset_of(struct virtio_dev_common, q_select);
+		ci_write_16(addr, cpu_to_le16(queue));
+		eieio();
+		addr = dev->common.addr + offset_of(struct virtio_dev_common, q_size);
+		size = le16_to_cpu(ci_read_16(addr));
+	}
+	else {
+		ci_write_16(dev->legacy.addr+VIRTIOHDR_QUEUE_SELECT,
 			    cpu_to_le16(queue));
 		eieio();
-		size = le16_to_cpu(ci_read_16(dev->base+VIRTIOHDR_QUEUE_SIZE));
+		size = le16_to_cpu(ci_read_16(dev->legacy.addr+VIRTIOHDR_QUEUE_SIZE));
 	}
 
 	return size;
@@ -70,12 +243,19 @@ struct vring_desc *virtio_get_vring_desc(struct virtio_device *dev, int queue)
 {
 	struct vring_desc *desc = 0;
 
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_16(dev->base+VIRTIOHDR_QUEUE_SELECT,
+	if (dev->is_modern) {
+		void *q_sel = dev->common.addr + offset_of(struct virtio_dev_common, q_select);
+		void *q_desc = dev->common.addr + offset_of(struct virtio_dev_common, q_desc);
+
+		ci_write_16(q_sel, cpu_to_le16(queue));
+		eieio();
+		desc = (void *)(virtio_pci_read64(q_desc));
+	} else {
+		ci_write_16(dev->legacy.addr+VIRTIOHDR_QUEUE_SELECT,
 			    cpu_to_le16(queue));
 		eieio();
 		desc = (void*)(4096L *
-		   le32_to_cpu(ci_read_32(dev->base+VIRTIOHDR_QUEUE_ADDRESS)));
+			       le32_to_cpu(ci_read_32(dev->legacy.addr+VIRTIOHDR_QUEUE_ADDRESS)));
 	}
 
 	return desc;
@@ -90,8 +270,18 @@ struct vring_desc *virtio_get_vring_desc(struct virtio_device *dev, int queue)
  */
 struct vring_avail *virtio_get_vring_avail(struct virtio_device *dev, int queue)
 {
-	return (void*)((uint64_t)virtio_get_vring_desc(dev, queue)
-			+ virtio_get_qsize(dev, queue) * sizeof(struct vring_desc));
+	if (dev->is_modern) {
+		void *q_sel = dev->common.addr + offset_of(struct virtio_dev_common, q_select);
+		void *q_avail = dev->common.addr + offset_of(struct virtio_dev_common, q_avail);
+
+		ci_write_16(q_sel, cpu_to_le16(queue));
+		eieio();
+		return (void *)(virtio_pci_read64(q_avail));
+	}
+	else {
+		return (void*)((uint64_t)virtio_get_vring_desc(dev, queue) +
+			       virtio_get_qsize(dev, queue) * sizeof(struct vring_desc));
+	}
 }
 
 
@@ -103,20 +293,46 @@ struct vring_avail *virtio_get_vring_avail(struct virtio_device *dev, int queue)
  */
 struct vring_used *virtio_get_vring_used(struct virtio_device *dev, int queue)
 {
-	return (void*)VQ_ALIGN((uint64_t)virtio_get_vring_avail(dev, queue)
-				  + virtio_get_qsize(dev, queue)
-				    * sizeof(struct vring_avail));
+	if (dev->is_modern) {
+		void *q_sel = dev->common.addr + offset_of(struct virtio_dev_common, q_select);
+		void *q_used = dev->common.addr + offset_of(struct virtio_dev_common, q_used);
+
+		ci_write_16(q_sel, cpu_to_le16(queue));
+		eieio();
+		return (void *)(virtio_pci_read64(q_used));
+	} else {
+		return (void*)VQ_ALIGN((uint64_t)virtio_get_vring_avail(dev, queue)
+				       + virtio_get_qsize(dev, queue)
+				       * sizeof(struct vring_avail));
+	}
 }
 
+/**
+ * Fill the virtio ring descriptor depending on the legacy mode or virtio 1.0
+ */
+void virtio_fill_desc(struct vring_desc *desc, bool is_modern,
+                      uint64_t addr, uint32_t len,
+                      uint16_t flags, uint16_t next)
+{
+	if (is_modern) {
+		desc->addr = cpu_to_le64(addr);
+		desc->len = cpu_to_le32(len);
+		desc->flags = cpu_to_le16(flags);
+		desc->next = cpu_to_le16(next);
+	} else {
+		desc->addr = addr;
+		desc->len = len;
+		desc->flags = flags;
+		desc->next = next;
+	}
+}
 
 /**
  * Reset virtio device
  */
 void virtio_reset_device(struct virtio_device *dev)
 {
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_8(dev->base+VIRTIOHDR_DEVICE_STATUS, 0);
-	}
+	virtio_set_status(dev, 0);
 }
 
 
@@ -125,25 +341,64 @@ void virtio_reset_device(struct virtio_device *dev)
  */
 void virtio_queue_notify(struct virtio_device *dev, int queue)
 {
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_16(dev->base+VIRTIOHDR_QUEUE_NOTIFY, cpu_to_le16(queue));
+	if (dev->is_modern) {
+		void *q_sel = dev->common.addr + offset_of(struct virtio_dev_common, q_select);
+		void *q_ntfy = dev->common.addr + offset_of(struct virtio_dev_common, q_notify_off);
+		void *addr;
+		uint16_t q_notify_off;
+
+		ci_write_16(q_sel, cpu_to_le16(queue));
+		eieio();
+		q_notify_off = le16_to_cpu(ci_read_16(q_ntfy));
+		addr = dev->notify.addr + q_notify_off * dev->notify_off_mul;
+		ci_write_16(addr, cpu_to_le16(queue));
+	} else {
+		ci_write_16(dev->legacy.addr+VIRTIOHDR_QUEUE_NOTIFY, cpu_to_le16(queue));
 	}
 }
 
 /**
  * Set queue address
  */
-void virtio_set_qaddr(struct virtio_device *dev, int queue, unsigned int qaddr)
+void virtio_set_qaddr(struct virtio_device *dev, int queue, unsigned long qaddr)
+{
+	if (dev->is_modern) {
+		uint64_t q_desc = qaddr;
+		uint64_t q_avail;
+		uint64_t q_used;
+		uint32_t q_size = virtio_get_qsize(dev, queue);
+
+		virtio_pci_write64(dev->common.addr + offset_of(struct virtio_dev_common, q_desc), q_desc);
+		q_avail = q_desc + q_size * sizeof(struct vring_desc);
+		virtio_pci_write64(dev->common.addr + offset_of(struct virtio_dev_common, q_avail), q_avail);
+		q_used = VQ_ALIGN(q_avail + sizeof(struct vring_avail) + sizeof(uint16_t) * q_size);
+		virtio_pci_write64(dev->common.addr + offset_of(struct virtio_dev_common, q_used), q_used);
+		ci_write_16(dev->common.addr + offset_of(struct virtio_dev_common, q_enable), cpu_to_le16(1));
+	} else {
+		uint32_t val = qaddr;
+		val = val >> 12;
+		ci_write_16(dev->legacy.addr+VIRTIOHDR_QUEUE_SELECT,
+			    cpu_to_le16(queue));
+		eieio();
+		ci_write_32(dev->legacy.addr+VIRTIOHDR_QUEUE_ADDRESS,
+			    cpu_to_le32(val));
+	}
+}
+
+int virtio_queue_init_vq(struct virtio_device *dev, struct vqs *vq, unsigned int id)
 {
-        if (dev->type == VIRTIO_TYPE_PCI) {
-                uint32_t val = qaddr;
-                val = val >> 12;
-                ci_write_16(dev->base+VIRTIOHDR_QUEUE_SELECT,
-                            cpu_to_le16(queue));
-                eieio();
-                ci_write_32(dev->base+VIRTIOHDR_QUEUE_ADDRESS,
-                            cpu_to_le32(val));
-        }
+	vq->size = virtio_get_qsize(dev, id);
+	vq->desc = SLOF_alloc_mem_aligned(virtio_vring_size(vq->size), 4096);
+	if (!vq->desc) {
+		printf("memory allocation failed!\n");
+		return -1;
+	}
+	memset(vq->desc, 0, virtio_vring_size(vq->size));
+	virtio_set_qaddr(dev, id, (unsigned long)vq->desc);
+	vq->avail = virtio_get_vring_avail(dev, id);
+	vq->used = virtio_get_vring_used(dev, id);
+	vq->id = id;
+	return 0;
 }
 
 /**
@@ -151,34 +406,109 @@ void virtio_set_qaddr(struct virtio_device *dev, int queue, unsigned int qaddr)
  */
 void virtio_set_status(struct virtio_device *dev, int status)
 {
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_8(dev->base+VIRTIOHDR_DEVICE_STATUS, status);
+	if (dev->is_modern) {
+		ci_write_8(dev->common.addr +
+			   offset_of(struct virtio_dev_common, dev_status), status);
+	} else {
+		ci_write_8(dev->legacy.addr+VIRTIOHDR_DEVICE_STATUS, status);
 	}
 }
 
+/**
+ * Get device status bits
+ */
+void virtio_get_status(struct virtio_device *dev, int *status)
+{
+	if (dev->is_modern) {
+		*status = ci_read_8(dev->common.addr +
+				    offset_of(struct virtio_dev_common, dev_status));
+	} else {
+		*status = ci_read_8(dev->legacy.addr+VIRTIOHDR_DEVICE_STATUS);
+	}
+}
 
 /**
  * Set guest feature bits
  */
-void virtio_set_guest_features(struct virtio_device *dev, int features)
+void virtio_set_guest_features(struct virtio_device *dev, uint64_t features)
 
 {
-	if (dev->type == VIRTIO_TYPE_PCI) {
-		ci_write_32(dev->base+VIRTIOHDR_GUEST_FEATURES, bswap_32(features));
+	if (dev->is_modern) {
+		uint32_t f1 = (features >> 32) & 0xFFFFFFFF;
+		uint32_t f0 = features & 0xFFFFFFFF;
+		void *addr = dev->common.addr;
+
+		ci_write_32(addr + offset_of(struct virtio_dev_common, drv_features_sel),
+			    cpu_to_le32(1));
+		ci_write_32(addr + offset_of(struct virtio_dev_common, drv_features),
+			    cpu_to_le32(f1));
+
+		ci_write_32(addr + offset_of(struct virtio_dev_common, drv_features_sel),
+			    cpu_to_le32(0));
+		ci_write_32(addr + offset_of(struct virtio_dev_common, drv_features),
+			    cpu_to_le32(f0));
+	} else {
+		ci_write_32(dev->legacy.addr+VIRTIOHDR_GUEST_FEATURES, cpu_to_le32(features));
 	}
 }
 
 /**
  * Get host feature bits
  */
-void virtio_get_host_features(struct virtio_device *dev, int *features)
+uint64_t virtio_get_host_features(struct virtio_device *dev)
 
 {
-	if (dev->type == VIRTIO_TYPE_PCI && features) {
-		*features = bswap_32(ci_read_32(dev->base+VIRTIOHDR_DEVICE_FEATURES));
+	uint64_t features = 0;
+	if (dev->is_modern) {
+		uint32_t f0 = 0, f1 = 0;
+		void *addr = dev->common.addr;
+
+		ci_write_32(addr + offset_of(struct virtio_dev_common, dev_features_sel),
+			    cpu_to_le32(1));
+		f1 = ci_read_32(addr +
+				offset_of(struct virtio_dev_common, dev_features));
+		ci_write_32(addr + offset_of(struct virtio_dev_common, dev_features_sel),
+			    cpu_to_le32(0));
+		f0 = ci_read_32(addr +
+				offset_of(struct virtio_dev_common, dev_features));
+
+		features = ((uint64_t)le32_to_cpu(f1) << 32) | le32_to_cpu(f0);
+	} else {
+		features = le32_to_cpu(ci_read_32(dev->legacy.addr+VIRTIOHDR_DEVICE_FEATURES));
 	}
+	return features;
 }
 
+int virtio_negotiate_guest_features(struct virtio_device *dev, uint64_t features)
+{
+	uint64_t host_features = 0;
+	int status;
+
+	/* Negotiate features */
+	host_features = virtio_get_host_features(dev);
+	if (!(host_features & VIRTIO_F_VERSION_1)) {
+		fprintf(stderr, "Device does not support virtio 1.0 %llx\n", host_features);
+		return -1;
+	}
+
+	virtio_set_guest_features(dev,  features);
+	host_features = virtio_get_host_features(dev);
+	if ((host_features & features) != features) {
+		fprintf(stderr, "Features error %llx\n", features);
+		return -1;
+	}
+
+	virtio_get_status(dev, &status);
+	status |= VIRTIO_STAT_FEATURES_OK;
+	virtio_set_status(dev, status);
+
+	/* Read back to verify the FEATURES_OK bit */
+	virtio_get_status(dev, &status);
+	if ((status & VIRTIO_STAT_FEATURES_OK) != VIRTIO_STAT_FEATURES_OK)
+		return -1;
+
+	return 0;
+}
 
 /**
  * Get additional config values
@@ -186,32 +516,38 @@ void virtio_get_host_features(struct virtio_device *dev, int *features)
 uint64_t virtio_get_config(struct virtio_device *dev, int offset, int size)
 {
 	uint64_t val = ~0ULL;
+	uint32_t hi, lo;
 	void *confbase;
 
-	switch (dev->type) {
-	 case VIRTIO_TYPE_PCI:
-		confbase = dev->base+VIRTIOHDR_DEVICE_CONFIG;
-		break;
-	 default:
-		return ~0ULL;
-	}
+	if (dev->is_modern)
+		confbase = dev->device.addr;
+	else
+		confbase = dev->legacy.addr+VIRTIOHDR_DEVICE_CONFIG;
+
 	switch (size) {
-	 case 1:
+	case 1:
 		val = ci_read_8(confbase+offset);
 		break;
-	 case 2:
+	case 2:
 		val = ci_read_16(confbase+offset);
+		if (dev->is_modern)
+			val = le16_to_cpu(val);
 		break;
-	 case 4:
+	case 4:
 		val = ci_read_32(confbase+offset);
+		if (dev->is_modern)
+			val = le32_to_cpu(val);
 		break;
-	 case 8:
+	case 8:
 		/* We don't support 8 bytes PIO accesses
 		 * in qemu and this is all PIO
 		 */
-		val = ci_read_32(confbase+offset);
-		val <<= 32;
-		val |= ci_read_32(confbase+offset+4);
+		lo = ci_read_32(confbase+offset);
+		hi = ci_read_32(confbase+offset+4);
+		if (dev->is_modern)
+			val = (uint64_t)le32_to_cpu(hi) << 32 | le32_to_cpu(lo);
+		else
+			val = (uint64_t)hi << 32 | lo;
 		break;
 	}
 
@@ -222,20 +558,19 @@ uint64_t virtio_get_config(struct virtio_device *dev, int offset, int size)
  * Get config blob
  */
 int __virtio_read_config(struct virtio_device *dev, void *dst,
-			  int offset, int len)
+			 int offset, int len)
 {
 	void *confbase;
 	unsigned char *buf = dst;
 	int i;
 
-	switch (dev->type) {
-	 case VIRTIO_TYPE_PCI:
-		confbase = dev->base+VIRTIOHDR_DEVICE_CONFIG;
-		break;
-	 default:
-		return 0;
-	}
+	if (dev->is_modern)
+		confbase = dev->device.addr;
+	else
+		confbase = dev->legacy.addr+VIRTIOHDR_DEVICE_CONFIG;
+
 	for (i = 0; i < len; i++)
 		buf[i] = ci_read_8(confbase + offset + i);
+
 	return len;
 }
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio.code b/qemu/roms/SLOF/lib/libvirtio/virtio.code
index 258b9bbda..8eec8f055 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio.code
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio.code
@@ -18,6 +18,11 @@
 
 /******** core virtio ********/
 
+// : virtio-setup-vd  ( -- dev )
+PRIM(virtio_X2d_setup_X2d_vd)
+	PUSH; TOS.a = virtio_setup_vd();
+MIRP
+
 // : virtio-vring-size  ( queuesize -- ringsize )
 PRIM(virtio_X2d_vring_X2d_size)
 	TOS.u = virtio_vring_size(TOS.u);
@@ -122,20 +127,18 @@ MIRP
 
 /******** virtio-net ********/
 
-// : virtio-net-open ( mac-addr-str len dev -- false | [ driver true ] )
+// : virtio-net-open ( dev -- false | [ driver true ] )
 PRIM(virtio_X2d_net_X2d_open)
 {
-	void *dev = TOS.a; POP;
-	int len = TOS.u; POP;
-        char *mac_addr = TOS.a;
+	void *dev = TOS.a;
 
-        net_driver_t *net_driver = virtionet_open(mac_addr, len, dev);
+	net_driver_t *net_driver = virtionet_open(dev);
 
-        if (net_driver) {
-                TOS.u = (unsigned long)net_driver; PUSH;
-                TOS.n = -1;
-        } else
-                TOS.n = 0;
+	if (net_driver) {
+		TOS.u = (unsigned long)net_driver; PUSH;
+		TOS.n = -1;
+	} else
+		TOS.n = 0;
 }
 MIRP
 
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio.h b/qemu/roms/SLOF/lib/libvirtio/virtio.h
index d5759b45a..0fee4baec 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio.h
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio.h
@@ -14,13 +14,23 @@
 #define _LIBVIRTIO_H
 
 #include <stdint.h>
+#include <stdbool.h>
 
 /* Device status bits */
 #define VIRTIO_STAT_ACKNOWLEDGE		1
 #define VIRTIO_STAT_DRIVER		2
 #define VIRTIO_STAT_DRIVER_OK		4
+#define VIRTIO_STAT_FEATURES_OK		8
+#define VIRTIO_STAT_NEEDS_RESET		64
 #define VIRTIO_STAT_FAILED		128
 
+#define BIT(x) (1UL << (x))
+
+/* VIRTIO 1.0 Device independent feature bits */
+#define VIRTIO_F_RING_INDIRECT_DESC	BIT(28)
+#define VIRTIO_F_RING_EVENT_IDX		BIT(29)
+#define VIRTIO_F_VERSION_1		BIT(32)
+
 #define VIRTIO_TIMEOUT		        5000 /* 5 sec timeout */
 
 /* Definitions for vring_desc.flags */
@@ -34,7 +44,7 @@ struct vring_desc {
 	uint32_t len;		/* Length */
 	uint16_t flags;		/* The flags as indicated above */
 	uint16_t next;		/* Next field if flags & NEXT */
-}; 
+};
 
 /* Definitions for vring_avail.flags */
 #define VRING_AVAIL_F_NO_INTERRUPT	1
@@ -44,8 +54,7 @@ struct vring_avail {
 	uint16_t flags;
 	uint16_t idx;
 	uint16_t ring[];
-}; 
-
+};
 
 /* Definitions for vring_used.flags */
 #define VRING_USED_F_NO_NOTIFY		1
@@ -61,27 +70,56 @@ struct vring_used {
 	struct vring_used_elem ring[];
 };
 
-#define VIRTIO_TYPE_PCI 0	/* For virtio-pci interface */
+/* Structure shared with SLOF and is 16bytes */
+struct virtio_cap {
+	void *addr;
+	uint8_t bar;
+	uint8_t is_io;
+	uint8_t cap_id;
+};
+
 struct virtio_device {
-	void *base;		/* base address */
-	int type;		/* VIRTIO_TYPE_PCI or VIRTIO_TYPE_VIO */
+	uint32_t is_modern;     /* Indicates whether to use virtio 1.0 */
+	struct virtio_cap legacy;
+	struct virtio_cap common;
+	struct virtio_cap notify;
+	struct virtio_cap isr;
+	struct virtio_cap device;
+	struct virtio_cap pci;
+	uint32_t notify_off_mul;
+};
+
+struct vqs {
+	uint64_t id;	/* Queue ID */
+	uint32_t size;
+	void *buf_mem;
+	struct vring_desc *desc;
+	struct vring_avail *avail;
+	struct vring_used *used;
 };
 
 /* Parts of the virtqueue are aligned on a 4096 byte page boundary */
 #define VQ_ALIGN(addr)	(((addr) + 0xfff) & ~0xfff)
 
 extern unsigned long virtio_vring_size(unsigned int qsize);
-extern int virtio_get_qsize(struct virtio_device *dev, int queue);
+extern unsigned int virtio_get_qsize(struct virtio_device *dev, int queue);
 extern struct vring_desc *virtio_get_vring_desc(struct virtio_device *dev, int queue);
 extern struct vring_avail *virtio_get_vring_avail(struct virtio_device *dev, int queue);
 extern struct vring_used *virtio_get_vring_used(struct virtio_device *dev, int queue);
+extern void virtio_fill_desc(struct vring_desc *desc, bool is_modern,
+                             uint64_t addr, uint32_t len,
+                             uint16_t flags, uint16_t next);
+extern int virtio_queue_init_vq(struct virtio_device *dev, struct vqs *vq, unsigned int id);
 
+extern struct virtio_device *virtio_setup_vd(void);
 extern void virtio_reset_device(struct virtio_device *dev);
 extern void virtio_queue_notify(struct virtio_device *dev, int queue);
 extern void virtio_set_status(struct virtio_device *dev, int status);
-extern void virtio_set_qaddr(struct virtio_device *dev, int queue, unsigned int qaddr);
-extern void virtio_set_guest_features(struct virtio_device *dev, int features);
-extern void virtio_get_host_features(struct virtio_device *dev, int *features);
+extern void virtio_get_status(struct virtio_device *dev, int *status);
+extern void virtio_set_qaddr(struct virtio_device *dev, int queue, unsigned long qaddr);
+extern void virtio_set_guest_features(struct virtio_device *dev, uint64_t features);
+extern uint64_t virtio_get_host_features(struct virtio_device *dev);
+extern int virtio_negotiate_guest_features(struct virtio_device *dev, uint64_t features);
 extern uint64_t virtio_get_config(struct virtio_device *dev, int offset, int size);
 extern int __virtio_read_config(struct virtio_device *dev, void *dst,
 				int offset, int len);
diff --git a/qemu/roms/SLOF/lib/libvirtio/virtio.in b/qemu/roms/SLOF/lib/libvirtio/virtio.in
index c36d127c7..195840e0f 100644
--- a/qemu/roms/SLOF/lib/libvirtio/virtio.in
+++ b/qemu/roms/SLOF/lib/libvirtio/virtio.in
@@ -10,6 +10,8 @@
  *     IBM Corporation - initial implementation
  *****************************************************************************/
 
+cod(virtio-setup-vd)
+
 cod(virtio-vring-size)
 cod(virtio-get-qsize)
 cod(virtio-get-config)
diff --git a/qemu/roms/SLOF/make.rules b/qemu/roms/SLOF/make.rules
index aebc4e360..cbc63530a 100644
--- a/qemu/roms/SLOF/make.rules
+++ b/qemu/roms/SLOF/make.rules
@@ -19,8 +19,12 @@
 ARCH := $(shell uname -p)
 
 # Auto-detect ppc64
-ifeq ($(ARCH), ppc64)
-CROSS          = ""
+ifeq (ppc64,$(findstring ppc64,$(ARCH)))
+  ifeq ($(ARCH), ppc64le)
+    EXTRA_CC	= -mbig -mabi=elfv1
+    EXTRA_LD	= -mbig
+  endif
+CROSS		?=
 else
 CROSS		?= powerpc64-linux-
 endif
@@ -31,8 +35,8 @@ HOSTCC		?= gcc
 HOSTCFLAGS	= -g -Wall -W -O2 -I. -I../include
 DD		= dd
 
-ONLY_CC 	= $(CROSS)gcc -m$(CELLSIZE)
-ONLY_AS 	= $(CROSS)as -m$(CELLSIZE)
+ONLY_CC 	= $(CROSS)gcc -m$(CELLSIZE) $(EXTRA_CC)
+ONLY_AS 	= $(CROSS)as -m$(CELLSIZE) $(EXTRA_LD)
 ONLY_LD 	= $(CROSS)ld -melf$(CELLSIZE)ppc
 
 # Verbose level:
diff --git a/qemu/roms/SLOF/rtas/reloc.S b/qemu/roms/SLOF/rtas/reloc.S
index e24d293d4..1b5b59a68 100644
--- a/qemu/roms/SLOF/rtas/reloc.S
+++ b/qemu/roms/SLOF/rtas/reloc.S
@@ -61,7 +61,7 @@ _rtas_start:
 ._rtas_entry_offset:	.quad   rtas_entry-_rtas_start
 ._rtas_config_offset:	.quad   rtas_config-_rtas_start
 ._rtas_stack:	        .quad   .stack-_rtas_start+RTAS_STACKSIZE-0x60
-._rtas_toc:	        .quad   _got-_rtas_start+0x8000
+._rtas_toc:	        .quad   _got-_rtas_start
 
 .over:	
 	mflr r8			# gpr 8 is the base
diff --git a/qemu/roms/SLOF/rtas/rtas.lds b/qemu/roms/SLOF/rtas/rtas.lds
index a5ba1daaf..30b18dd26 100644
--- a/qemu/roms/SLOF/rtas/rtas.lds
+++ b/qemu/roms/SLOF/rtas/rtas.lds
@@ -28,7 +28,8 @@ SECTIONS {
 	}
 	.got :
 	{
-	  _got = .;
+	  . = ALIGN(256);
+	  _got = DEFINED (.TOC.) ? .TOC. : ADDR (.got) + 0x8000;
 	  *(.got .toc)
 	}  	
 	.reloc :
diff --git a/qemu/roms/SLOF/rtas/rtas_entry.S b/qemu/roms/SLOF/rtas/rtas_entry.S
index 74693aa48..424137bf5 100644
--- a/qemu/roms/SLOF/rtas/rtas_entry.S
+++ b/qemu/roms/SLOF/rtas/rtas_entry.S
@@ -39,7 +39,7 @@ rtas_entry:
 	bcl 20,31,.over		# branch to over 
 .base:	
 	.align  3
-..got:			.quad   _got-.base+0x8000
+..got:			.quad   _got-.base
 ..stack:		.quad   .stack+RTAS_STACKSIZE-0x60-.base
 .over:	
 	mflr r8			# gpr 8 is the base
diff --git a/qemu/roms/SLOF/slof/entry.S b/qemu/roms/SLOF/slof/entry.S
index dcff57ba0..5372de357 100644
--- a/qemu/roms/SLOF/slof/entry.S
+++ b/qemu/roms/SLOF/slof/entry.S
@@ -207,4 +207,13 @@ call_client:
 	li 3, -1 # client app return
 	blr
 
+
+	# Call another function via pointer in r6
+	# (arguments can be provided in r3 to r5)
+	# Destination function should jump back to lr
+C_ENTRY(call_c)
+	mtctr	r6
+	bctr
+
+.global the_system_stack
 	.lcomm	the_system_stack, STACKSIZE, 16
diff --git a/qemu/roms/SLOF/slof/fs/base.fs b/qemu/roms/SLOF/slof/fs/base.fs
index e71e087eb..03e77e54f 100644
--- a/qemu/roms/SLOF/slof/fs/base.fs
+++ b/qemu/roms/SLOF/slof/fs/base.fs
@@ -579,8 +579,6 @@ defer cursor-off ( -- )
 #include "debug.fs"
 \ provide 7.5.3.1 Dictionary search
 #include "dictionary.fs"
-\ block data access for IO devices - ought to be implemented in engine
-#include "rmove.fs"
 \ provide a simple run time preprocessor
 #include <preprocessor.fs>
 
diff --git a/qemu/roms/SLOF/slof/fs/boot.fs b/qemu/roms/SLOF/slof/fs/boot.fs
index 9a0ded0c2..e0b628140 100644
--- a/qemu/roms/SLOF/slof/fs/boot.fs
+++ b/qemu/roms/SLOF/slof/fs/boot.fs
@@ -181,17 +181,13 @@ defer go ( -- )
       \ with watchdog timeout.
       4ec set-watchdog
    THEN
+   2dup " HALT" str= IF 2drop 0 EXIT THEN
    my-self >r current-node @ >r         \ Save my-self
    ." Trying to load: " $bootargs type ."  from: " 2dup type ."  ... "
    2dup open-dev dup IF
       dup to my-self
       dup ihandle>phandle set-node
       -rot                              ( ihandle devstr len )
-      my-args nip 0= IF
-	 2dup 1- + c@ [char] : <> IF    \ Add : to device path if missing
-	    1+ strdup 2dup 1- + [char] : swap c!
-	 THEN
-      THEN
       encode-string s" bootpath" set-chosen
       $bootargs encode-string s" bootargs" set-chosen
       get-load-base s" load" 3 pick ['] $call-method CATCH IF
@@ -211,7 +207,7 @@ defer go ( -- )
 
 : parse-load ( "{devlist}" -- success )	\ Parse-execute boot-device list
    cr BEGIN parse-word dup WHILE
-	 ( de-alias ) do-load dup 0< IF drop 0 THEN IF
+	 de-alias do-load dup 0< IF drop 0 THEN IF
 	    state-valid @ IF ."   Successfully loaded" cr THEN
 	    true 0d parse strdup load-list 2! EXIT
 	 THEN
diff --git a/qemu/roms/SLOF/slof/fs/client.fs b/qemu/roms/SLOF/slof/fs/client.fs
index 1b2bb0326..7d537a668 100644
--- a/qemu/roms/SLOF/slof/fs/client.fs
+++ b/qemu/roms/SLOF/slof/fs/client.fs
@@ -282,6 +282,18 @@ ALSO client-voc DEFINITIONS
 ;
 
 \
+\ Standard for Boot, defined in 6.3.2.5:
+\
+: boot  ( zstr -- )
+   zcount
+   debug-client-interface? IF
+      ." ci: boot " 2dup type cr
+   THEN
+   " boot " 2swap $cat " boot-command" $setenv (nvupdate)
+   reset-all
+;
+
+\
 \ User Interface, defined in 6.3.2.6
 \
 : interpret ( ... zstr -- result ... )
diff --git a/qemu/roms/SLOF/slof/fs/fbuffer.fs b/qemu/roms/SLOF/slof/fs/fbuffer.fs
index 756f05a95..47046087d 100644
--- a/qemu/roms/SLOF/slof/fs/fbuffer.fs
+++ b/qemu/roms/SLOF/slof/fs/fbuffer.fs
@@ -19,6 +19,7 @@
 0 VALUE screen-height
 0 VALUE screen-width
 0 VALUE screen-depth
+0 VALUE screen-line-bytes
 0 VALUE window-top
 0 VALUE window-left
 
@@ -54,10 +55,10 @@
 : fb8-background inverse? ;
 : fb8-foreground inverse? invert ;
 
-: fb8-lines2bytes ( #lines -- #bytes ) char-height * screen-width * screen-depth * ;
+: fb8-lines2bytes ( #lines -- #bytes ) char-height * screen-line-bytes * ;
 : fb8-columns2bytes ( #columns -- #bytes ) char-width * screen-depth * ;
 : fb8-line2addr ( line# -- addr )
-	char-height * window-top + screen-width * screen-depth *
+	char-height * window-top + screen-line-bytes *
 	frame-buffer-adr + window-left screen-depth * +
 ;
 
@@ -98,9 +99,10 @@ CREATE bitmap-buffer 400 4 * allot
 
 : fb8-toggle-cursor ( -- )
 	line# fb8-line2addr column# fb8-columns2bytes +
-	char-height 0 ?DO
-		char-width screen-depth * 0 ?DO dup dup rb@ -1 xor swap rb! 1+ LOOP
-		screen-width screen-depth * + char-width screen-depth * -
+	char-height 2 - screen-line-bytes * +
+	2 0 ?DO
+		dup char-width screen-depth * invert-region
+		screen-line-bytes +
 	LOOP drop
 ;
 
@@ -110,7 +112,7 @@ CREATE bitmap-buffer 400 4 * allot
 	line# fb8-line2addr column# fb8-columns2bytes + ( bitmap-buf fb-addr )
 	char-height 0 ?DO
 		2dup char-width screen-depth * mrmove
-		screen-width screen-depth * + >r char-width screen-depth * + r>
+		screen-line-bytes + >r char-width screen-depth * + r>
 	LOOP 2drop
     ELSE 2drop r> 3drop THEN
 ;
@@ -135,12 +137,12 @@ CREATE bitmap-buffer 400 4 * allot
 		fb8-columns2bytes swap fb8-columns2bytes tuck -
 		over r@ tuck + rot char-height 0 ?DO
 			3dup rmove
-			-rot screen-width screen-depth * tuck + -rot + swap rot
+			-rot screen-line-bytes tuck + -rot + swap rot
 		LOOP
 		3drop r>
 	THEN
 	char-height 0 ?DO
-		dup 2 pick fb8-erase-block screen-width screen-depth * +
+		dup 2 pick fb8-erase-block screen-line-bytes +
 	LOOP
 	2drop
 ;
@@ -153,12 +155,12 @@ CREATE bitmap-buffer 400 4 * allot
 		fb8-columns2bytes swap fb8-columns2bytes tuck -
 		over r@ + 2dup + r> swap >r rot char-height 0 ?DO
 			3dup rmove
-			-rot screen-width screen-depth * tuck + -rot + swap rot
+			-rot screen-line-bytes tuck + -rot + swap rot
 		LOOP
 		3drop r> over -
 	THEN
 	char-height 0 ?DO
-		dup 2 pick fb8-erase-block screen-width screen-depth * +
+		dup 2 pick fb8-erase-block screen-line-bytes +
 	LOOP
 	2drop
 ;
@@ -166,13 +168,11 @@ CREATE bitmap-buffer 400 4 * allot
 : fb8-reset-screen ( -- ) ( Left as no-op by design ) ;
 
 : fb8-erase-screen ( -- )
-	frame-buffer-adr screen-height screen-width * screen-depth * fb8-erase-block
+	frame-buffer-adr screen-height screen-line-bytes * fb8-erase-block
 ;
 
 : fb8-invert-screen ( -- )
-	frame-buffer-adr screen-height screen-width * screen-depth * 2dup /x / 0 ?DO
-		dup rx@ -1 xor over rx! xa1+
-	LOOP 3drop
+	frame-buffer-adr screen-height screen-line-bytes * invert-region
 ;
 
 : fb8-blink-screen ( -- ) fb8-invert-screen fb8-invert-screen ;
@@ -180,6 +180,7 @@ CREATE bitmap-buffer 400 4 * allot
 : fb8-install ( width height #columns #lines -- )
 	1 to screen-depth
 	2swap  to screen-height  to screen-width
+	screen-width to screen-line-bytes
 	screen-#rows min to #lines
 	screen-#columns min to #columns
 	screen-height char-height #lines * - 2/ to window-top
@@ -201,6 +202,7 @@ CREATE bitmap-buffer 400 4 * allot
 	>r
 	fb8-install
 	r> to screen-depth
+	screen-width screen-depth * to screen-line-bytes
 ;
 
 
diff --git a/qemu/roms/SLOF/slof/fs/little-endian.fs b/qemu/roms/SLOF/slof/fs/little-endian.fs
index f2e4e8d42..6b4779ee0 100644
--- a/qemu/roms/SLOF/slof/fs/little-endian.fs
+++ b/qemu/roms/SLOF/slof/fs/little-endian.fs
@@ -17,6 +17,9 @@ here c@ ef = CONSTANT ?littleendian
 
 ?bigendian [IF]
 
+: x!-le >r xbflip r> x! ;
+: x@-le x@ xbflip ;
+
 : l!-le  >r lbflip r> l! ;
 : l@-le  l@ lbflip ;
 
@@ -47,6 +50,9 @@ here c@ ef = CONSTANT ?littleendian
 
 [ELSE]
 
+: x!-le x! ;
+: x@-le x@ ;
+
 : l!-le  l! ;
 : l@-le  l@ ;
 
diff --git a/qemu/roms/SLOF/slof/fs/packages/disk-label.fs b/qemu/roms/SLOF/slof/fs/packages/disk-label.fs
index fe1c25e7a..e034d6408 100644
--- a/qemu/roms/SLOF/slof/fs/packages/disk-label.fs
+++ b/qemu/roms/SLOF/slof/fs/packages/disk-label.fs
@@ -20,6 +20,7 @@ false VALUE debug-disk-label?
 \ If we ever want to put a large kernel with initramfs from a PREP partition
 \ we might need to increase this value. The default value is 65536 blocks (32MB)
 d# 65536 value max-prep-partition-blocks
+d# 4096 CONSTANT block-array-size
 
 s" disk-label" device-name
 
@@ -152,8 +153,8 @@ CONSTANT /gpt-part-entry
 : init-block ( -- )
    s" block-size" ['] $call-parent CATCH IF ABORT" parent has no block-size." THEN
    to block-size
-   d# 4096 alloc-mem
-   dup d# 4096 erase
+   block-array-size alloc-mem
+   dup block-array-size erase
    to block
    debug-disk-label? IF
       ." init-block: block-size=" block-size .d ." block=0x" block u. cr
@@ -178,7 +179,8 @@ CONSTANT /gpt-part-entry
 \ This word returns true if the currently loaded block has _NO_ GPT partition id
 : no-gpt? ( -- true|false )
    0 read-sector
-   1 partition>part-entry part-entry>id c@ ee <>
+   1 partition>part-entry part-entry>id c@ ee <> IF true EXIT THEN
+   block mbr>magic w@-le aa55 <>
 ;
 
 : pc-extended-partition? ( part-entry-addr -- true|false )
@@ -266,7 +268,10 @@ CONSTANT /gpt-part-entry
 
 : try-dos-partition ( -- okay? )
    \ Read partition table and check magic.
-   no-mbr? IF cr ." No DOS disk-label found." cr false EXIT THEN
+   no-mbr? IF
+       debug-disk-label? IF cr ." No DOS disk-label found." cr THEN
+       false EXIT
+   THEN
 
    count-dos-logical-partitions TO dos-logical-partitions
 
@@ -320,6 +325,14 @@ CONSTANT /gpt-part-entry
 
 \ Load from first active DOS boot partition.
 
+: fat-bootblock? ( addr -- flag )
+   \ byte 0-2 of the bootblock is a jump instruction in
+   \ all FAT filesystems.
+   \ e9 and eb are jump instructions in x86 assembler.
+   dup c@ e9 = IF drop true EXIT THEN
+   dup c@ eb = swap 2+ c@ 90 = and
+;
+
 \ NOTE: block-size is always 512 bytes for DOS partition tables.
 
 : load-from-dos-boot-partition ( addr -- size )
@@ -352,60 +365,103 @@ CONSTANT /gpt-part-entry
    drop 0
 ;
 
-\ Check for GPT PReP partition GUID
-9E1A2D38     CONSTANT GPT-PREP-PARTITION-1
-C612         CONSTANT GPT-PREP-PARTITION-2
-4316         CONSTANT GPT-PREP-PARTITION-3
-AA26         CONSTANT GPT-PREP-PARTITION-4
-8B49521E5A8B CONSTANT GPT-PREP-PARTITION-5
+\ Check for GPT PReP partition GUID. Only first 3 blocks are
+\ byte-swapped treating last two blocks as contigous for simplifying
+\ comparison
+9E1A2D38            CONSTANT GPT-PREP-PARTITION-1
+C612                CONSTANT GPT-PREP-PARTITION-2
+4316                CONSTANT GPT-PREP-PARTITION-3
+AA268B49521E5A8B    CONSTANT GPT-PREP-PARTITION-4
 
 : gpt-prep-partition? ( -- true|false )
-   block gpt-part-entry>part-type-guid l@-le GPT-PREP-PARTITION-1 = IF
-      block gpt-part-entry>part-type-guid 4 + w@-le
-      GPT-PREP-PARTITION-2 = IF
-         block gpt-part-entry>part-type-guid 6 + w@-le
-         GPT-PREP-PARTITION-3 = IF
-            block gpt-part-entry>part-type-guid 8 + w@
-            GPT-PREP-PARTITION-4 = IF
-               block gpt-part-entry>part-type-guid a + w@
-               block gpt-part-entry>part-type-guid c + l@ swap lxjoin
-               GPT-PREP-PARTITION-5 = IF
-                   TRUE EXIT
-               THEN
-            THEN
-         THEN
-      THEN
+   block gpt-part-entry>part-type-guid
+   dup l@-le     GPT-PREP-PARTITION-1 <> IF drop false EXIT THEN
+   dup 4 + w@-le GPT-PREP-PARTITION-2 <> IF drop false EXIT THEN
+   dup 6 + w@-le GPT-PREP-PARTITION-3 <> IF drop false EXIT THEN
+       8 + x@    GPT-PREP-PARTITION-4 =
+;
+
+\ Check for GPT MSFT BASIC DATA GUID - fat based
+EBD0A0A2            CONSTANT GPT-BASIC-DATA-PARTITION-1
+B9E5                CONSTANT GPT-BASIC-DATA-PARTITION-2
+4433                CONSTANT GPT-BASIC-DATA-PARTITION-3
+87C068B6B72699C7    CONSTANT GPT-BASIC-DATA-PARTITION-4
+
+: gpt-basic-data-partition? ( -- true|false )
+   block gpt-part-entry>part-type-guid
+   dup l@-le     GPT-BASIC-DATA-PARTITION-1 <> IF drop false EXIT THEN
+   dup 4 + w@-le GPT-BASIC-DATA-PARTITION-2 <> IF drop false EXIT THEN
+   dup 6 + w@-le GPT-BASIC-DATA-PARTITION-3 <> IF drop false EXIT THEN
+       8 + x@    GPT-BASIC-DATA-PARTITION-4 =
+;
+
+\
+\ GPT Signature
+\ ("EFI PART", 45h 46h 49h 20h 50h 41h 52h 54h)
+\
+4546492050415254 CONSTANT GPT-SIGNATURE
+
+\ The routine checks whether the protective MBR has GPT ID and then
+\ reads the gpt data from the sector. Also set the seek position and
+\ the partition size used in caller routines.
+
+: get-gpt-partition ( -- true|false )
+   no-gpt? IF false EXIT THEN
+   debug-disk-label? IF cr ." GPT partition found " cr  THEN
+   1 read-sector
+   block gpt>part-entry-lba x@-le
+   block-size * to seek-pos
+   block gpt>part-entry-size l@-le to gpt-part-size
+   gpt-part-size block-array-size > IF
+      cr ." GPT part size exceeds buffer allocated " cr
+      false exit
    THEN
-   FALSE
+   block gpt>signature x@ GPT-SIGNATURE =
 ;
 
 : load-from-gpt-prep-partition ( addr -- size )
-   no-gpt? IF drop FALSE EXIT THEN
-   debug-disk-label? IF
-      cr ." GPT partition found " cr
-   THEN
-   1 read-sector block gpt>part-entry-lba l@-le
-   block-size * to seek-pos
-   block gpt>part-entry-size l@-le to gpt-part-size
-   block gpt>num-part-entry l@-le dup 0= IF FALSE EXIT THEN
+   get-gpt-partition 0= IF false EXIT THEN
+   block gpt>num-part-entry l@-le dup 0= IF false exit THEN
    1+ 1 ?DO
       seek-pos 0 seek drop
       block gpt-part-size read drop gpt-prep-partition? IF
-         debug-disk-label? IF
-            ." GPT PReP partition found " cr
-         THEN
-         block gpt-part-entry>first-lba x@ xbflip
-         block gpt-part-entry>last-lba x@ xbflip
-         over - 1+                 ( addr offset len )
-         swap                      ( addr len offset )
-         block-size * to part-offset
-         0 0 seek drop             ( addr len )
-         block-size * read         ( size )
+         debug-disk-label? IF  ." GPT PReP partition found " cr THEN
+         block gpt-part-entry>first-lba x@-le ( addr first-lba )
+         block gpt-part-entry>last-lba x@-le  ( addr first-lba last-lba)
+         over - 1+                            ( addr first-lba blocks )
+         swap                                 ( addr blocks first-lba )
+         block-size * to part-offset          ( addr blocks )
+         0 0 seek drop                        ( addr blocks )
+         block-size * read                    ( size )
+         UNLOOP EXIT
+     THEN
+     seek-pos gpt-part-size + to seek-pos
+   LOOP
+   false
+;
+
+: try-gpt-dos-partition ( -- true|false )
+   get-gpt-partition 0= IF false EXIT THEN
+   block gpt>num-part-entry l@-le dup 0= IF false EXIT THEN
+   1+ 1 ?DO
+      seek-pos 0 seek drop
+      block gpt-part-size read drop
+      gpt-basic-data-partition? IF
+         debug-disk-label? IF ." GPT BASIC DATA partition found " cr THEN
+         block gpt-part-entry>first-lba x@-le       ( first-lba )
+         dup to part-start                          ( first-lba )
+         block gpt-part-entry>last-lba x@-le        ( first-lba last-lba )
+         over - 1+                                  ( first-lba s1 )
+         block-size * to part-size                  ( first-lba )
+         block-size * to part-offset                ( )
+         0 0 seek drop
+         block block-size read drop
+         block fat-bootblock?                    ( true|false )
          UNLOOP EXIT
       THEN
-      seek-pos gpt-part-size i * + to seek-pos
+      seek-pos gpt-part-size + to seek-pos
    LOOP
-   FALSE
+   false
 ;
 
 \ Extract the boot loader path from a bootinfo.txt file
@@ -493,7 +549,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
    debug-disk-label? IF ." Trying CHRP boot " .s cr THEN
    1 disk-chrp-boot !
-   dup load-chrp-boot-file ?dup 0 <> IF .s cr nip EXIT THEN
+   dup load-chrp-boot-file ?dup 0 <> IF nip EXIT THEN
    0 disk-chrp-boot !
 
    debug-disk-label? IF ." Trying GPT boot " .s cr THEN
@@ -558,14 +614,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 : try-dos-files ( -- found? )
    no-mbr? IF false EXIT THEN
 
-   \ block 0 byte 0-2 is a jump instruction in all FAT
-   \ filesystems.
-   \ e9 and eb are jump instructions in x86 assembler.
-   block c@ e9 <> IF
-      block c@ eb <>
-      block 2+ c@ 90 <> or
-      IF false EXIT THEN
-   THEN
+   block fat-bootblock? 0= IF false EXIT THEN
    s" fat-files" (interpose-filesystem)
    true
 ;
@@ -600,6 +649,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
 : try-partitions ( -- found? )
    try-dos-partition IF try-files EXIT THEN
+   try-gpt-dos-partition IF try-files EXIT THEN
    \ try-iso9660-partition IF try-files EXIT THEN
    \ ... more partition types here...
    false
@@ -610,7 +660,7 @@ AA26         CONSTANT GPT-PREP-PARTITION-4
 
 : close ( -- )
    debug-disk-label? IF ." Closing disk-label: block=0x" block u. ." block-size=" block-size .d cr THEN
-   block d# 4096 free-mem
+   block block-array-size free-mem
 ;
 
 
diff --git a/qemu/roms/SLOF/slof/fs/packages/fat-files.fs b/qemu/roms/SLOF/slof/fs/packages/fat-files.fs
index 0cec3664e..d9194527e 100644
--- a/qemu/roms/SLOF/slof/fs/packages/fat-files.fs
+++ b/qemu/roms/SLOF/slof/fs/packages/fat-files.fs
@@ -18,6 +18,7 @@ INSTANCE VARIABLE sectors/cluster
 INSTANCE VARIABLE #reserved-sectors
 INSTANCE VARIABLE #fats
 INSTANCE VARIABLE #root-entries
+INSTANCE VARIABLE fat32-root-cluster
 INSTANCE VARIABLE total-#sectors
 INSTANCE VARIABLE media-descriptor
 INSTANCE VARIABLE sectors/fat
@@ -59,9 +60,18 @@ INSTANCE VARIABLE next-cluster
 : read-cluster ( cluster# -- )
   dup bytes/cluster @ * cluster-offset @ + bytes/cluster @ read-data
   read-fat dup #clusters @ >= IF drop 0 THEN next-cluster ! ;
+
 : read-dir ( cluster# -- )
-  ?dup 0= IF root-offset @ #root-entries @ 20 * read-data 0 next-cluster !
-  ELSE read-cluster THEN ;
+    ?dup 0= IF
+        #root-entries @ 0= IF
+            fat32-root-cluster @ read-cluster
+        ELSE
+            root-offset @ #root-entries @ 20 * read-data 0 next-cluster !
+        THEN
+    ELSE
+        read-cluster
+    THEN
+;
 
 : .time ( x -- )
   base @ >r decimal
@@ -137,6 +147,7 @@ CREATE dos-name b allot
 
   \ For FAT32:
   sectors/fat @ 0= IF data @ 24 + 4c@ bljoin sectors/fat ! THEN
+  #root-entries @ 0= IF data @ 2c + 4c@ bljoin ELSE 0 THEN fat32-root-cluster !
 
   \ XXX add other FAT32 stuff (offsets 28, 2c, 30)
 
diff --git a/qemu/roms/SLOF/slof/fs/pci-config-bridge.fs b/qemu/roms/SLOF/slof/fs/pci-config-bridge.fs
index 689325318..1efbcd804 100644
--- a/qemu/roms/SLOF/slof/fs/pci-config-bridge.fs
+++ b/qemu/roms/SLOF/slof/fs/pci-config-bridge.fs
@@ -81,26 +81,6 @@
    2drop 
 ;
 
-: dma-alloc ( ... size -- virt )
-   \ ." dma-alloc called: " .s cr
-   alloc-mem
-;
-
-: dma-free ( virt size -- )
-   \ ." dma-free called: " .s cr
-   free-mem
-;
-
-: dma-map-in ( ... virt size cacheable? -- devaddr )
-   \ ." dma-map-in called: " .s cr
-   2drop
-;
-
-: dma-map-out ( virt devaddr size -- )
-   \ ." dma-map-out called: " .s cr
-   2drop drop
-;
-
 : dma-sync ( virt devaddr size -- )
    \ XXX should we add at least a memory barrier here?
    \ ." dma-sync called: " .s cr
diff --git a/qemu/roms/SLOF/slof/fs/pci-scan.fs b/qemu/roms/SLOF/slof/fs/pci-scan.fs
index b8b9fe61f..2fdf0e8f5 100644
--- a/qemu/roms/SLOF/slof/fs/pci-scan.fs
+++ b/qemu/roms/SLOF/slof/fs/pci-scan.fs
@@ -110,10 +110,13 @@ here 100 allot CONSTANT pci-device-vec
         dup 100000 + pci-next-mem !             \ and write back with 1MB for bridge
         over 24 + rtas-config-w@                \ check if 64bit support
         1 and IF                                \ IF 64 bit support
-                2dup 20 rshift                  \ | keep upper 32 bits
-                swap 28 + rtas-config-l!        \ | and write it into the Base-Upper32-bits
-                pci-max-mem @ 20 rshift         \ | fetch max Limit address and keep upper 32 bits
-                2 pick 2C + rtas-config-l!      \ | and set the Limit
+                pci-next-mem64 @ 100000000 #aligned \ | read the current Value of 64-bit and align to 4GB boundary
+                dup 100000000 + pci-next-mem64 x!   \ | and write back with 1GB for bridge
+                2 pick swap                         \ |
+                20 rshift                           \ | keep upper 32 bits
+                swap 28 + rtas-config-l!            \ | and write it into the Base-Upper32-bits
+                pci-max-mem64 @ 20 rshift           \ | fetch max Limit address and keep upper 32 bits
+                2 pick 2C + rtas-config-l!          \ | and set the Limit
         THEN                                    \ FI
         10 rshift                               \ keep upper 16 bits
         pci-max-mem @ 1- FFFF0000 and or        \ and Insert mmem Limit (set it to max)
@@ -129,8 +132,12 @@ here 100 allot CONSTANT pci-device-vec
         1-                                      \ make limit one less than boundary
         over 24 + rtas-config-w@                \ check if 64bit support
         1 and IF                                \ IF 64 bit support
-                2dup 20 rshift                  \ | keep upper 32 bits
-                swap 2C + rtas-config-l!        \ | and write it into the Limit-Upper32-bits
+                pci-next-mem64 @ 100000000 #aligned \ | Reat current value of 64-bar and align at 4GB
+                dup pci-next-mem64 x!               \ | and write it back
+                1-                                  \ | make limite one less than boundary
+                2 pick swap                         \ |
+                20 rshift                           \ | keep upper 32 bits
+                swap 2C + rtas-config-l!            \ | and write it into the Limit-Upper32-bits
         THEN                                    \ FI
         FFFF0000 and                            \ keep upper 16 bits
         over 24 + rtas-config-l@ 0000FFFF and   \ fetch original Value
diff --git a/qemu/roms/SLOF/slof/fs/rmove.fs b/qemu/roms/SLOF/slof/fs/rmove.fs
deleted file mode 100644
index c28dba9c4..000000000
--- a/qemu/roms/SLOF/slof/fs/rmove.fs
+++ /dev/null
@@ -1,53 +0,0 @@
-\ *****************************************************************************
-\ * Copyright (c) 2004, 2008 IBM Corporation
-\ * All rights reserved.
-\ * This program and the accompanying materials
-\ * are made available under the terms of the BSD License
-\ * which accompanies this distribution, and is available at
-\ * http://www.opensource.org/licenses/bsd-license.php
-\ *
-\ * Contributors:
-\ *     IBM Corporation - initial implementation
-\ ****************************************************************************/
-
-defer '(r@)
-defer '(r!)
-1 VALUE /(r)
-
-
-\ The rest of the code already implemented in prim.in
-\ In the end all of this should be moved over there and this file terminated
-
-: (rfill) ( addr size pattern 'r! /r -- )
-	to /(r) to '(r!) ff and
-	dup 8 lshift or dup 10 lshift or dup 20 lshift or
-	-rot bounds ?do dup i '(r!) /(r) +loop drop
-;
-
-: (fwrmove) ( src dest size -- )
-	>r 0 -rot r> bounds ?do + dup '(r@) i '(r!) /(r) dup +loop 2drop
-;
-
-\ Move from main to device memory
-: mrmove ( src dest size -- )
-	3dup or or 7 AND CASE
-		0 OF ['] x@ ['] rx! /x ENDOF
-		4 OF ['] l@ ['] rl! /l ENDOF
-		2 OF ['] w@ ['] rw! /w ENDOF
-		dup OF ['] c@ ['] rb! /c ENDOF
-	ENDCASE
-	( We already know that source and destination do not overlap )
-	to /(r) to '(r!) to '(r@) (fwrmove)
-;
-
-: rfill ( addr size pattern -- )
-	3dup drop or 7 AND CASE
-		0 OF ['] rx! /x ENDOF
-		4 OF ['] rl! /l ENDOF
-		2 OF ['] rw! /w ENDOF
-		dup OF ['] rb! /c ENDOF
-	ENDCASE (rfill)
-;
-
-
-
diff --git a/qemu/roms/SLOF/slof/fs/root.fs b/qemu/roms/SLOF/slof/fs/root.fs
index 21c710951..952b00e75 100644
--- a/qemu/roms/SLOF/slof/fs/root.fs
+++ b/qemu/roms/SLOF/slof/fs/root.fs
@@ -77,7 +77,6 @@ finish-device
 
 : open true ;
 : close ;
-#include <archsupport.fs>
 
 \ Finish root
 finish-device
diff --git a/qemu/roms/SLOF/slof/fs/terminal.fs b/qemu/roms/SLOF/slof/fs/terminal.fs
index 582bedeb3..dc82e7bf4 100644
--- a/qemu/roms/SLOF/slof/fs/terminal.fs
+++ b/qemu/roms/SLOF/slof/fs/terminal.fs
@@ -167,6 +167,7 @@ false VALUE stopcsi
 CREATE twtracebuf 4000 allot twtracebuf 4000 erase
 twtracebuf VALUE twbp
 0 VALUE twbc
+0 VALUE twtrace-enabled?
 
 : twtrace
 	twbc 4000 = IF 0 to twbc twtracebuf to twbp THEN
@@ -176,7 +177,7 @@ twtracebuf VALUE twbp
 : terminal-write ( addr len -- actual-len )
  	cursor-off
 	tuck bounds ?DO i c@
-		twtrace
+		twtrace-enabled? IF twtrace THEN
 		esc-on IF esc-process
 		ELSE CASE
 			1B OF true to esc-on ENDOF
diff --git a/qemu/roms/SLOF/slof/helpers.c b/qemu/roms/SLOF/slof/helpers.c
index d7c1888b4..48c34a65c 100644
--- a/qemu/roms/SLOF/slof/helpers.c
+++ b/qemu/roms/SLOF/slof/helpers.c
@@ -114,6 +114,13 @@ long SLOF_pci_config_read16(long offset)
 	return forth_pop();
 }
 
+long SLOF_pci_config_read8(long offset)
+{
+	forth_push(offset);
+	forth_eval("config-b@");
+	return forth_pop();
+}
+
 void SLOF_pci_config_write32(long offset, long value)
 {
 	forth_push(value);
@@ -128,6 +135,13 @@ void SLOF_pci_config_write16(long offset, long value)
 	forth_eval("config-w!");
 }
 
+void SLOF_pci_config_write8(long offset, long value)
+{
+	forth_push(value);
+	forth_push(offset);
+	forth_eval("config-b!");
+}
+
 void *SLOF_translate_my_address(void *addr)
 {
 	forth_push((long)addr);
diff --git a/qemu/roms/SLOF/slof/paflof.c b/qemu/roms/SLOF/slof/paflof.c
index 624955fba..2fc25c81e 100644
--- a/qemu/roms/SLOF/slof/paflof.c
+++ b/qemu/roms/SLOF/slof/paflof.c
@@ -19,6 +19,7 @@
 #undef unix
 
 #include "paflof.h"
+#include <stdio.h>
 #include <string.h>
 #include <stdint.h>
 #include <ctype.h>
@@ -43,6 +44,9 @@ unsigned char hash_table[HASHSIZE*CELLSIZE];
 
 #include ISTR(TARG,c)
 
+static int did_stackwarning;
+extern char the_system_stack[];
+
 // the actual engine
 long engine(int mode, long param_1, long param_2)
 {
@@ -84,6 +88,11 @@ long engine(int mode, long param_1, long param_2)
 		c_return[1].a = &dummy;
 	}
 
+	if ((char *)&ip < the_system_stack && !did_stackwarning) {
+		puts("ERROR: stack overflow in engine()!");
+		did_stackwarning = 1;
+	}
+
 	if (mode & ENGINE_MODE_PARAM_2) {
 		(++dp)->n = param_2;
 	}
diff --git a/qemu/roms/SLOF/slof/ppc64.c b/qemu/roms/SLOF/slof/ppc64.c
index 20d927069..619d95ec7 100644
--- a/qemu/roms/SLOF/slof/ppc64.c
+++ b/qemu/roms/SLOF/slof/ppc64.c
@@ -42,24 +42,7 @@ cell *the_heap_start = &the_heap[0];
 cell *the_heap_end = &the_heap[HEAP_SIZE / CELLSIZE];
 
 extern void io_putchar(unsigned char);
-
-
-static unsigned long __attribute__((noinline))
-call_c(cell arg0, cell arg1, cell arg2, cell entry)
-{
-	register unsigned long r3 asm("r3") = arg0.u;
-	register unsigned long r4 asm("r4") = arg1.u;
-	register unsigned long r5 asm("r5") = arg2.u;
-	register unsigned long r6 = entry.u         ;
-
-	asm volatile("mflr 31 ; mtctr %4 ; bctrl ; mtlr 31"
-		     : "=r" (r3)
-		     : "r" (r3), "r" (r4), "r" (r5), "r" (r6)
-		     : "ctr", "r6", "r7", "r8", "r9", "r10", "r11",
-		       "r12", "r13", "r31", "lr", "cc");
-
-	return r3;
-}
+extern unsigned long call_c(cell arg0, cell arg1, cell arg2, cell entry);
 
 
 long
diff --git a/qemu/roms/SLOF/slof/prim.code b/qemu/roms/SLOF/slof/prim.code
index 9fbed7168..bb9e036a9 100644
--- a/qemu/roms/SLOF/slof/prim.code
+++ b/qemu/roms/SLOF/slof/prim.code
@@ -520,6 +520,19 @@ PRIM(RMOVE)
 
 	MIRP
 
+PRIM(MRMOVE)
+	type_u size = TOS.u; POP;
+	void *d = TOS.a; POP;
+	void *s = TOS.a; POP;
+	FAST_MRMOVE(s, d, size);
+	MIRP
+
+PRIM(RFILL)
+	type_u pat = TOS.u; POP;
+	type_u size = TOS.u; POP;
+	void *dst = TOS.a; POP;
+	FAST_RFILL(dst, size, pat);
+	MIRP
 
 // String compare, case insensitive:
 // : string=ci  ( str1 len1 str2 len2 -- equal? )
diff --git a/qemu/roms/SLOF/slof/prim.in b/qemu/roms/SLOF/slof/prim.in
index 7a0d6a2ed..855f59262 100644
--- a/qemu/roms/SLOF/slof/prim.in
+++ b/qemu/roms/SLOF/slof/prim.in
@@ -104,8 +104,9 @@ cod(SEMICOLON)
 cod(EXECUTE)
 
 cod(MOVE)
-// cod(RMOVE64)
 cod(RMOVE)
+cod(MRMOVE)
+cod(RFILL)
 cod(ZCOUNT)
 con(HASH-SIZE HASHSIZE)
 cod(HASH)
diff --git a/qemu/roms/config.ipxe.general.h b/qemu/roms/config.ipxe.general.h
deleted file mode 100644
index 619ee4c15..000000000
--- a/qemu/roms/config.ipxe.general.h
+++ /dev/null
@@ -1,4 +0,0 @@
-#undef BANNER_TIMEOUT
-#define BANNER_TIMEOUT 30
-#undef ROM_BANNER_TIMEOUT
-#define ROM_BANNER_TIMEOUT 0
diff --git a/qemu/roms/config.seabios-128k b/qemu/roms/config.seabios-128k
index c719ba689..0a9da77a5 100644
--- a/qemu/roms/config.seabios-128k
+++ b/qemu/roms/config.seabios-128k
@@ -3,6 +3,8 @@
 CONFIG_QEMU=y
 CONFIG_ROM_SIZE=128
 CONFIG_XEN=n
+CONFIG_USB_OHCI=n
 CONFIG_USB_XHCI=n
 CONFIG_USB_UAS=n
 CONFIG_SDCARD=n
+CONFIG_TCGBIOS=n
diff --git a/qemu/roms/ipxe/COPYING b/qemu/roms/ipxe/COPYING
index a43ea2126..342330bb9 100644
--- a/qemu/roms/ipxe/COPYING
+++ b/qemu/roms/ipxe/COPYING
@@ -1,339 +1,12 @@
-		    GNU GENERAL PUBLIC LICENSE
-		       Version 2, June 1991
+In general iPXE files are licensed under the GPL.  For historical
+reasons, individual files may contain their own licence declarations.
+Most builds of iPXE do not contain all iPXE code (in particular, most
+builds will include only one driver), and so the overall licence can
+vary depending on what target you are building.
 
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.
-                          675 Mass Ave, Cambridge, MA 02139, USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
+The resultant applicable licence(s) for any particular build can be
+determined by using "make bin/xxxxxxx.yyy.licence"; for example:
 
-			    Preamble
+  make bin/rtl8139.rom.licence
 
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Library General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-		    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-			    NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-	Appendix: How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) 19yy  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program; if not, write to the Free Software
-    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) 19yy name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Library General
-Public License instead of this License.
+to determine the resultant licence(s) for the build bin/rtl8139.rom
diff --git a/qemu/roms/ipxe/COPYING.GPLv2 b/qemu/roms/ipxe/COPYING.GPLv2
new file mode 100644
index 000000000..d159169d1
--- /dev/null
+++ b/qemu/roms/ipxe/COPYING.GPLv2
@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/qemu/roms/ipxe/COPYING.UBDL b/qemu/roms/ipxe/COPYING.UBDL
new file mode 100644
index 000000000..780ddcd77
--- /dev/null
+++ b/qemu/roms/ipxe/COPYING.UBDL
@@ -0,0 +1,59 @@
+UNMODIFIED BINARY DISTRIBUTION LICENCE
+
+
+PREAMBLE
+
+The GNU General Public License provides a legal guarantee that
+software covered by it remains free (in the sense of freedom, not
+price).  It achieves this guarantee by imposing obligations on anyone
+who chooses to distribute the software.
+
+Some of these obligations may be seen as unnecessarily burdensome.  In
+particular, when the source code for the software is already publicly
+and freely available, there is minimal value in imposing upon each
+distributor the obligation to provide the complete source code (or an
+equivalent written offer to provide the complete source code).
+
+This Licence allows for the distribution of unmodified binaries built
+from publicly available source code, without imposing the obligations
+of the GNU General Public License upon anyone who chooses to
+distribute only the unmodified binaries built from that source code.
+
+The extra permissions granted by this Licence apply only to unmodified
+binaries built from source code which has already been made available
+to the public in accordance with the terms of the GNU General Public
+Licence.  Nothing in this Licence allows for the creation of
+closed-source modified versions of the Program.  Any modified versions
+of the Program are subject to the usual terms and conditions of the
+GNU General Public License.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+This Licence applies to any Program or other work which contains a
+notice placed by the copyright holder saying it may be distributed
+under the terms of this Unmodified Binary Distribution Licence.  All
+terms used in the text of this Licence are to be interpreted as they
+are used in version 2 of the GNU General Public License as published
+by the Free Software Foundation.
+
+If you have made this Program available to the public in both source
+code and executable form in accordance with the terms of the GNU
+General Public License as published by the Free Software Foundation;
+either version 2 of the License, or (at your option) any later
+version, then you are hereby granted an additional permission to use,
+copy, and distribute the unmodified executable form of this Program
+(the "Unmodified Binary") without restriction, including the right to
+permit persons to whom the Unmodified Binary is furnished to do
+likewise, subject to the following conditions:
+
+- when started running, the Program must display an announcement which
+  includes the details of your existing publication of the Program
+  made in accordance with the terms of the GNU General Public License.
+  For example, the Program could display the URL of the publicly
+  available source code from which the Unmodified Binary was built.
+
+- when exercising your right to grant permissions under this Licence,
+  you do not need to refer directly to the text of this Licence, but
+  you may not grant permissions beyond those granted to you by this
+  Licence.
diff --git a/qemu/roms/ipxe/COPYRIGHTS b/qemu/roms/ipxe/COPYRIGHTS
deleted file mode 100644
index 342330bb9..000000000
--- a/qemu/roms/ipxe/COPYRIGHTS
+++ /dev/null
@@ -1,12 +0,0 @@
-In general iPXE files are licensed under the GPL.  For historical
-reasons, individual files may contain their own licence declarations.
-Most builds of iPXE do not contain all iPXE code (in particular, most
-builds will include only one driver), and so the overall licence can
-vary depending on what target you are building.
-
-The resultant applicable licence(s) for any particular build can be
-determined by using "make bin/xxxxxxx.yyy.licence"; for example:
-
-  make bin/rtl8139.rom.licence
-
-to determine the resultant licence(s) for the build bin/rtl8139.rom
diff --git a/qemu/roms/ipxe/src/Makefile b/qemu/roms/ipxe/src/Makefile
index b742d1283..2a9cc9e8f 100644
--- a/qemu/roms/ipxe/src/Makefile
+++ b/qemu/roms/ipxe/src/Makefile
@@ -83,11 +83,13 @@ SRCDIRS		+= drivers/block
 SRCDIRS		+= drivers/nvs
 SRCDIRS		+= drivers/bitbash
 SRCDIRS		+= drivers/infiniband
+SRCDIRS		+= drivers/usb
 SRCDIRS		+= interface/pxe interface/efi interface/smbios
 SRCDIRS		+= interface/bofm
 SRCDIRS		+= interface/xen
+SRCDIRS		+= interface/hyperv
 SRCDIRS		+= tests
-SRCDIRS		+= crypto crypto/axtls crypto/matrixssl
+SRCDIRS		+= crypto crypto/mishmash
 SRCDIRS		+= hci hci/commands hci/tui
 SRCDIRS		+= hci/mucurses hci/mucurses/widgets
 SRCDIRS		+= hci/keymap
diff --git a/qemu/roms/ipxe/src/Makefile.housekeeping b/qemu/roms/ipxe/src/Makefile.housekeeping
index 1a75d3939..03800c8ef 100644
--- a/qemu/roms/ipxe/src/Makefile.housekeeping
+++ b/qemu/roms/ipxe/src/Makefile.housekeeping
@@ -157,17 +157,6 @@ SP_FLAGS := $(shell $(SP_TEST) && $(ECHO) '-fno-stack-protector')
 WORKAROUND_CFLAGS += $(SP_FLAGS)
 endif
 
-# Some widespread patched versions of gcc include -fPIE -Wl,-pie by
-# default.  Note that gcc will exit *successfully* if it fails to
-# recognise an option that starts with "no", so we have to test for
-# output on stderr instead of checking the exit status.
-#
-ifeq ($(CCTYPE),gcc)
-PIE_TEST = [ -z "`$(CC) -fno-PIE -nopie -x c -c /dev/null -o /dev/null 2>&1`" ]
-PIE_FLAGS := $(shell $(PIE_TEST) && $(ECHO) '-fno-PIE -nopie')
-WORKAROUND_CFLAGS += $(PIE_FLAGS)
-endif
-
 # gcc 4.4 generates .eh_frame sections by default, which distort the
 # output of "size".  Inhibit this.
 #
@@ -533,6 +522,7 @@ endif
 #
 COMPILE_c	= $(CC) $(CFLAGS) $(CFLAGS_c) $(OBJ_CFLAGS)
 RULE_c		= $(Q)$(COMPILE_c) -c $< -o $@ $(POST_O)
+RULE_c_to_ids.o = $(Q)$(ECHO_E) '$(OBJ_IDS_ASM_NL)' | $(ASSEMBLE_S) -o $@
 RULE_c_to_dbg%.o = $(Q)$(COMPILE_c) -DDBGLVL_MAX=$* -c $< -o $@ $(POST_O)
 RULE_c_to_c	= $(Q)$(COMPILE_c) -E -c $< > $@
 RULE_c_to_s	= $(Q)$(COMPILE_c) -S -g0 -c $< -o $@
@@ -543,7 +533,7 @@ RULE_S		= $(Q)$(PREPROCESS_S) $< | $(ASSEMBLE_S) -o $@
 RULE_S_to_dbg%.o = $(Q)$(PREPROCESS_S) -DDBGLVL_MAX=$* $< | $(ASSEMBLE_S) -o $@
 RULE_S_to_s	= $(Q)$(PREPROCESS_S) $< > $@
 
-DEBUG_TARGETS	+= dbg%.o c s
+GENERIC_TARGETS	+= ids.o dbg%.o c s
 
 # List of embedded images included in the last build of embedded.o.
 # This is needed in order to correctly rebuild embedded.o whenever the
@@ -775,8 +765,6 @@ define deps_template_parts
 	$(Q)$(CPP) $(CFLAGS) $(CFLAGS_$(2)) $(CFLAGS_$(3)) -DOBJECT=$(3) \
 		-Wno-error -M $(1) -MG -MP | \
 		sed 's/\.o\s*:/_DEPS +=/' > $(BIN)/deps/$(1).d
-	$(Q)$(if $(findstring drivers/,$(1)),\
-	   $(PERL) $(PARSEROM) $(1) >> $(BIN)/deps/$(1).d)
 endef
 
 # rules_template : generate rules for a given source file
@@ -796,7 +784,7 @@ $$(BIN)/$(3).o : $(1) $$(MAKEDEPS) $$(POST_O_DEPS) $$($(3)_DEPS)
 	$$(QM)$(ECHO) "  [BUILD] $$@"
 	$$(RULE_$(2))
 BOBJS += $$(BIN)/$(3).o
-$(foreach TGT,$(DEBUG_TARGETS),$(if $(RULE_$(2)_to_$(TGT)),$(NEWLINE)$(call rules_template_target,$(1),$(2),$(3),$(TGT))))
+$(foreach TGT,$(GENERIC_TARGETS),$(if $(RULE_$(2)_to_$(TGT)),$(NEWLINE)$(call rules_template_target,$(1),$(2),$(3),$(TGT))))
 $$(BIN)/deps/$(1).d : $$($(3)_DEPS)
 TAGS : $$($(3)_DEPS)
 endef
@@ -824,7 +812,7 @@ endef
 
 # Generate the dependency files
 #
-$(BIN)/deps/%.d : % $(MAKEDEPS) $(PARSEROM)
+$(BIN)/deps/%.d : % $(MAKEDEPS)
 	$(call deps_template_file,$<)
 
 # Calculate list of dependency files
@@ -866,10 +854,69 @@ endif
 endif
 endif
 
-# The following variables are created by the rules files
+# Files to be parsed using parserom.pl
+#
+ROM_SRCS	= $(foreach SRC,$(AUTO_SRCS),\
+		    $(if $(findstring drivers/,$(SRC)),$(SRC)))
+romsrcs :
+	@$(ECHO) $(ROM_SRCS)
+
+# List of files to be parsed using parserom.pl
+#
+ROM_SRCS_LIST	:= $(BIN)/.rom.list
+ifeq ($(wildcard $(ROM_SRCS_LIST)),)
+ROM_SRCS_OLD := <invalid>
+else
+ROM_SRCS_OLD := $(shell cat $(ROM_SRCS_LIST))
+endif
+ifneq ($(ROM_SRCS_OLD),$(ROM_SRCS))
+$(shell $(ECHO) "$(ROM_SRCS)" > $(ROM_SRCS_LIST))
+endif
+
+$(ROM_SRCS_LIST) : $(MAKEDEPS)
+
+VERYCLEANUP	+= $(ROM_SRCS_LIST)
+
+# ROM definition file
+#
+ROMDEFS		= $(BIN)/.rom.defs
+$(ROMDEFS) : $(ROM_SRCS) $(ROM_SRCS_LIST) $(PARSEROM) $(MAKEDEPS)
+	$(QM)$(ECHO) "  [PARSEROM]"
+	$(Q)$(PERL) $(PARSEROM) $(ROM_SRCS) > $@
+
+VERYCLEANUP	+= $(ROMDEFS)
+
+# Evaluate ROM definition file
+ifdef NEED_DEPS
+ifneq ($(ROM_SRCS),)
+-include $(ROMDEFS)
+endif
+endif
+
+# Device ID tables (using IDs from ROM definition file)
+#
+define obj_pci_id_asm
+	.section ".pci_devlist.$(1)", "a", @progbits
+	.globl pci_devlist_$(1)
+pci_devlist_$(1):
+	.short ( 0x$(1) & 0xffff )
+
+endef
+define obj_isa_id_asm
+endef
+OBJ_IDS_ASM	= $(foreach ROM,$(ROMS_$(OBJECT)),$(call obj_$(ROM_TYPE_$(ROM))_id_asm,$(ROM)))
+OBJ_IDS_ASM_NL	= $(subst $(NEWLINE),\n,$(OBJ_IDS_ASM))
+$(BIN)/%.ids :
+	@$(ECHO_E) '$(OBJ_IDS_ASM_NL)'
+
+BOBJS	+= $(patsubst %,$(BIN)/%.ids.o,$(DRIVERS))
+
+# The following variables are created by the autogenerated rules
 #
 bobjs :
 	@$(ECHO) $(BOBJS)
+drivers_% :
+	@$(ECHO) $(DRIVERS_$*)
 drivers :
 	@$(ECHO) $(DRIVERS)
 .PHONY : drivers
@@ -900,6 +947,11 @@ $(BIN)/NIC : $(AUTO_DEPS)
 	@perl -ne 'chomp; print "$$1\n" if /\# NIC\t(.*)$$/' $^ >> $@
 CLEANUP		+= $(BIN)/NIC	# Doesn't match the $(BIN)/*.* pattern
 
+# Select drivers to be included in the all-drivers build
+#
+DRIVERS_ipxe	= $(DRIVERS_net) $(DRIVERS_infiniband) \
+		  $(DRIVERS_xen) $(DRIVERS_hyperv)
+
 # Analyse a target name (e.g. "bin/dfe538--prism2_pci.rom.tmp") and
 # derive the variables:
 # 
@@ -908,7 +960,6 @@ CLEANUP		+= $(BIN)/NIC	# Doesn't match the $(BIN)/*.* pattern
 # TGT_DRIVERS  : the driver for each element (e.g. "rtl8139 prism2_pci")
 # TGT_ROM_NAME : the ROM name (e.g. "dfe538")
 #
-DRIVERS_ipxe	= $(DRIVERS)
 CARD_DRIVER	= $(firstword $(DRIVER_$(1)) $(1))
 TGT_ELEMENTS	= $(subst --, ,$(firstword $(subst ., ,$(notdir $@))))
 TGT_ROM_NAME	= $(firstword $(TGT_ELEMENTS))
@@ -941,6 +992,8 @@ TGT_PCI_DEVICE	= $(PCI_DEVICE_$(TGT_ROM_NAME))
 TGT_LD_DRIVERS	= $(subst -,_,$(patsubst %,obj_%,$(TGT_DRIVERS)))
 TGT_LD_IDS	= pci_vendor_id=$(firstword $(TGT_PCI_VENDOR) 0) \
 		  pci_device_id=$(firstword $(TGT_PCI_DEVICE) 0)
+TGT_LD_DEVLIST	= $(foreach ELEM,$(TGT_ELEMENTS),$(if $(PCI_VENDOR_$(ELEM)),\
+		    pci_devlist_$(patsubst 0x%,%,$(PCI_VENDOR_$(ELEM)))$(patsubst 0x%,%,$(PCI_DEVICE_$(ELEM)))))
 TGT_LD_ENTRY	= _$(TGT_PREFIX)_start
 
 # Calculate linker flags based on link-time options for the current
@@ -951,7 +1004,8 @@ TGT_LD_ENTRY	= _$(TGT_PREFIX)_start
 #		 "-u obj_zpciprefix -u obj_rtl8139 -u obj_prism2_pci
 #		  --defsym pci_vendor=0x1186 --defsym pci_device=0x1300")
 #
-TGT_LD_FLAGS	= $(foreach SYM,$(TGT_LD_ENTRY) $(TGT_LD_DRIVERS) obj_config,\
+TGT_LD_FLAGS	= $(foreach SYM,$(TGT_LD_ENTRY) $(TGT_LD_DRIVERS) \
+		    $(TGT_LD_DEVLIST) obj_config,\
 		    -u $(SYM) --defsym check_$(SYM)=$(SYM) ) \
 		  $(patsubst %,--defsym %,$(TGT_LD_IDS)) \
 		  -e $(TGT_LD_ENTRY)
@@ -981,6 +1035,7 @@ $(BIN)/%.info :
 	@$(ECHO)
 	@$(ECHO) 'LD driver symbols    : $(TGT_LD_DRIVERS)'
 	@$(ECHO) 'LD ID symbols        : $(TGT_LD_IDS)'
+	@$(ECHO) 'LD devlist symbols   : $(TGT_LD_DEVLIST)'
 	@$(ECHO) 'LD entry point       : $(TGT_LD_ENTRY)'
 	@$(ECHO)
 	@$(ECHO) 'LD target flags      : $(TGT_LD_FLAGS)'
@@ -1012,7 +1067,7 @@ BLIB		= $(BIN)/blib.a
 $(BLIB) : $(BLIB_OBJS) $(BLIB_LIST) $(MAKEDEPS)
 	$(Q)$(RM) $(BLIB)
 	$(QM)$(ECHO) "  [AR] $@"
-	$(Q)$(AR) r $@ $(BLIB_OBJS)
+	$(Q)$(AR) r $@ $(sort $(BLIB_OBJS))
 	$(Q)$(RANLIB) $@
 blib : $(BLIB)
 
@@ -1231,15 +1286,12 @@ endif # defined(BIN)
 #
 # The compression utilities
 #
-$(NRV2B) : util/nrv2b.c $(MAKEDEPS)
-	$(QM)$(ECHO) "  [HOSTCC] $@"
-	$(Q)$(HOST_CC) $(HOST_CFLAGS) -DENCODE -DDECODE -DMAIN -DVERBOSE \
-		       -DNDEBUG -DBITSIZE=32 -DENDIAN=0 -o $@ $<
-CLEANUP	+= $(NRV2B)
 
-$(ZBIN) : util/zbin.c util/nrv2b.c $(MAKEDEPS)
+ZBIN_LDFLAGS := -llzma
+
+$(ZBIN) : util/zbin.c $(MAKEDEPS)
 	$(QM)$(ECHO) "  [HOSTCC] $@"
-	$(Q)$(HOST_CC) $(HOST_CFLAGS) -o $@ $<
+	$(Q)$(HOST_CC) $(HOST_CFLAGS) $< $(ZBIN_LDFLAGS) -o $@
 CLEANUP += $(ZBIN)
 
 ###############################################################################
@@ -1319,31 +1371,6 @@ endif
 
 ###############################################################################
 #
-# Auto-incrementing build serial number.  Append "bs" to your list of
-# build targets to get a serial number printed at the end of the
-# build.  Enable -DBUILD_SERIAL in order to see it when the code runs.
-#
-BUILDSERIAL_H		= config/.buildserial.h
-BUILDSERIAL_NOW		= config/.buildserial.now
-BUILDSERIAL_NEXT	= config/.buildserial.next
-
-$(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT) :
-	$(ECHO) 1 > $@
-
-$(BUILDSERIAL_H) : $(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT)
-	$(ECHO) '#define BUILD_SERIAL_NUM $(shell cat $<)' > $@
-
-ifeq ($(filter bs,$(MAKECMDGOALS)),bs)
-$(shell diff -q $(BUILDSERIAL_NOW) $(BUILDSERIAL_NEXT) > /dev/null || \
-	cp -f $(BUILDSERIAL_NEXT) $(BUILDSERIAL_NOW))
-endif
-
-bs : $(BUILDSERIAL_NOW)
-	@$(ECHO) $$(( $(shell cat $<) + 1 )) > $(BUILDSERIAL_NEXT)
-	@$(ECHO) "Build serial number is $(shell cat $<)"
-
-###############################################################################
-#
 # Build the TAGS file(s) for emacs
 #
 TAGS :
diff --git a/qemu/roms/ipxe/src/arch/i386/Makefile b/qemu/roms/ipxe/src/arch/i386/Makefile
index 4925cc4e6..99f875314 100644
--- a/qemu/roms/ipxe/src/arch/i386/Makefile
+++ b/qemu/roms/ipxe/src/arch/i386/Makefile
@@ -69,6 +69,17 @@ CFLAGS		+= -fshort-wchar
 #
 CFLAGS			+= -Ui386
 
+# Some widespread patched versions of gcc include -fPIE -Wl,-pie by
+# default.  Note that gcc will exit *successfully* if it fails to
+# recognise an option that starts with "no", so we have to test for
+# output on stderr instead of checking the exit status.
+#
+ifeq ($(CCTYPE),gcc)
+PIE_TEST = [ -z "`$(CC) -fno-PIE -nopie -x c -c /dev/null -o /dev/null 2>&1`" ]
+PIE_FLAGS := $(shell $(PIE_TEST) && $(ECHO) '-fno-PIE -nopie')
+WORKAROUND_CFLAGS += $(PIE_FLAGS)
+endif
+
 # Define version string for lkrnprefix.S
 #
 CFLAGS_lkrnprefix	+= -DVERSION="\"$(VERSION)\""
diff --git a/qemu/roms/ipxe/src/arch/i386/core/basemem_packet.c b/qemu/roms/ipxe/src/arch/i386/core/basemem_packet.c
index 06ffa3bbd..9f5fbf330 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/basemem_packet.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/basemem_packet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/core/cachedhcp.c b/qemu/roms/ipxe/src/arch/i386/core/cachedhcp.c
index 3cac28e7d..a5c624035 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/cachedhcp.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/cachedhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/core/gdbmach.c b/qemu/roms/ipxe/src/arch/i386/core/gdbmach.c
index 4d6897f7d..d92a4ac08 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/gdbmach.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/gdbmach.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/core/patch_cf.S b/qemu/roms/ipxe/src/arch/i386/core/patch_cf.S
index 97a62f494..4365563fe 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/patch_cf.S
+++ b/qemu/roms/ipxe/src/arch/i386/core/patch_cf.S
@@ -14,9 +14,13 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/qemu/roms/ipxe/src/arch/i386/core/pci_autoboot.c b/qemu/roms/ipxe/src/arch/i386/core/pci_autoboot.c
index a3eb1f97d..337598091 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/pci_autoboot.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/pci_autoboot.c
@@ -16,9 +16,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/device.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/core/rdtsc_timer.c b/qemu/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
index 2f31afc66..e720a239c 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/rdtsc_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -27,7 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <assert.h>
 #include <ipxe/timer.h>
-#include <ipxe/timer2.h>
+#include <ipxe/pit8254.h>
 
 /**
  * Number of TSC ticks per microsecond
@@ -56,10 +60,10 @@ static void rdtsc_udelay ( unsigned long usecs ) {
 			elapsed = ( currticks() - start );
 		} while ( elapsed < ( usecs * rdtsc_ticks_per_usec ) );
 	} else {
-		/* Not yet calibrated; use timer2 and calibrate
+		/* Not yet calibrated; use 8254 PIT and calibrate
 		 * based on result.
 		 */
-		timer2_udelay ( usecs );
+		pit8254_udelay ( usecs );
 		elapsed = ( currticks() - start );
 		rdtsc_ticks_per_usec = ( elapsed / usecs );
 		DBG ( "RDTSC timer calibrated: %ld ticks in %ld usecs "
diff --git a/qemu/roms/ipxe/src/arch/i386/core/relocate.c b/qemu/roms/ipxe/src/arch/i386/core/relocate.c
index 5fbf2d2c2..54ad387e4 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/relocate.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/relocate.c
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * The linker passes in the symbol _max_align, which is the alignment
diff --git a/qemu/roms/ipxe/src/arch/i386/core/runtime.c b/qemu/roms/ipxe/src/arch/i386/core/runtime.c
index 18ca7936e..d160fee04 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/runtime.c
+++ b/qemu/roms/ipxe/src/arch/i386/core/runtime.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/core/setjmp.S b/qemu/roms/ipxe/src/arch/i386/core/setjmp.S
index 03727148c..81d3b4911 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/setjmp.S
+++ b/qemu/roms/ipxe/src/arch/i386/core/setjmp.S
@@ -1,42 +1,64 @@
-/* setjmp and longjmp. Use of these functions is deprecated. */
-
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
 	.code32
-	
-/**************************************************************************
-SETJMP - Save stack context for non-local goto
-**************************************************************************/
+
+	/* Must match jmp_buf structure layout */
+	.struct	0
+env_retaddr:	.long	0
+env_stack:	.long	0
+env_ebx:	.long	0
+env_esi:	.long	0
+env_edi:	.long	0
+env_ebp:	.long	0
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
 	.globl	setjmp
 setjmp:
-	movl	4(%esp),%ecx		/* jmpbuf */
-	movl	0(%esp),%edx		/* return address */
-	movl	%edx,0(%ecx)
-	movl	%ebx,4(%ecx)
-	movl	%esp,8(%ecx)
-	movl	%ebp,12(%ecx)
-	movl	%esi,16(%ecx)
-	movl	%edi,20(%ecx)
-	movl	$0,%eax
+	/* Get jmp_buf pointer in %edx */
+	movl	4(%esp),%edx
+	/* Save return address */
+	movl	0(%esp),%eax
+	movl	%eax, env_retaddr(%edx)
+	/* Save stack pointer */
+	movl	%esp, env_stack(%edx)
+	/* Save other registers */
+	movl	%ebx, env_ebx(%edx)
+	movl	%esi, env_esi(%edx)
+	movl	%edi, env_edi(%edx)
+	movl	%ebp, env_ebp(%edx)
+	/* Return 0 when returning as setjmp() */
+	xorl	%eax, %eax
 	ret
+	.size	setjmp, . - setjmp
 
-/**************************************************************************
-LONGJMP - Non-local jump to a saved stack context
-**************************************************************************/
+/*
+ * Non-local jump to a saved stack context
+ */
 	.globl	longjmp
 longjmp:
-	movl	4(%esp),%edx		/* jumpbuf */
-	movl	8(%esp),%eax		/* result */
-	movl	0(%edx),%ecx
-	movl	4(%edx),%ebx
-	movl	8(%edx),%esp
-	movl	12(%edx),%ebp
-	movl	16(%edx),%esi
-	movl	20(%edx),%edi
-	cmpl	$0,%eax
-	jne	1f
-	movl	$1,%eax
-1:	movl	%ecx,0(%esp)
+	/* Get jmp_buf pointer in %edx */
+	movl	4(%esp),%edx
+	/* Get result in %eax */
+	movl	8(%esp),%eax
+	/* Force result to non-zero */
+	testl	%eax, %eax
+	jnz	1f
+	incl	%eax
+1:	/* Restore stack pointer */
+	movl	env_stack(%edx), %esp
+	/* Restore other registers */
+	movl	env_ebx(%edx), %ebx
+	movl	env_esi(%edx), %esi
+	movl	env_edi(%edx), %edi
+	movl	env_ebp(%edx), %ebp
+	/* Replace return address on the new stack */
+	popl	%ecx	/* discard */
+	pushl	env_retaddr(%edx)
+	/* Return to setjmp() caller */
 	ret
+	.size	longjmp, . - longjmp
diff --git a/qemu/roms/ipxe/src/arch/i386/core/stack.S b/qemu/roms/ipxe/src/arch/i386/core/stack.S
index 737ec0eed..98f1cd9b9 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/stack.S
+++ b/qemu/roms/ipxe/src/arch/i386/core/stack.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/qemu/roms/ipxe/src/arch/i386/core/stack16.S b/qemu/roms/ipxe/src/arch/i386/core/stack16.S
index 523f0288b..4bc6f081a 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/stack16.S
+++ b/qemu/roms/ipxe/src/arch/i386/core/stack16.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/qemu/roms/ipxe/src/arch/i386/core/timer2.c b/qemu/roms/ipxe/src/arch/i386/core/timer2.c
deleted file mode 100644
index 077866562..000000000
--- a/qemu/roms/ipxe/src/arch/i386/core/timer2.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * arch/i386/core/i386_timer.c
- *
- * Use the "System Timer 2" to implement the udelay callback in
- * the BIOS timer driver. Also used to calibrate the clock rate
- * in the RTDSC timer driver.
- * 
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stddef.h>
-#include <ipxe/timer2.h>
-#include <ipxe/io.h>
-
-/* Timers tick over at this rate */
-#define TIMER2_TICKS_PER_SEC	1193180U
-
-/* Parallel Peripheral Controller Port B */
-#define	PPC_PORTB	0x61
-
-/* Meaning of the port bits */
-#define	PPCB_T2OUT	0x20	/* Bit 5 */
-#define	PPCB_SPKR	0x02	/* Bit 1 */
-#define	PPCB_T2GATE	0x01	/* Bit 0 */
-
-/* Ports for the 8254 timer chip */
-#define	TIMER2_PORT	0x42
-#define	TIMER_MODE_PORT	0x43
-
-/* Meaning of the mode bits */
-#define	TIMER0_SEL	0x00
-#define	TIMER1_SEL	0x40
-#define	TIMER2_SEL	0x80
-#define	READBACK_SEL	0xC0
-
-#define	LATCH_COUNT	0x00
-#define	LOBYTE_ACCESS	0x10
-#define	HIBYTE_ACCESS	0x20
-#define	WORD_ACCESS	0x30
-
-#define	MODE0		0x00
-#define	MODE1		0x02
-#define	MODE2		0x04
-#define	MODE3		0x06
-#define	MODE4		0x08
-#define	MODE5		0x0A
-
-#define	BINARY_COUNT	0x00
-#define	BCD_COUNT	0x01
-
-static void load_timer2 ( unsigned int ticks ) {
-	/*
-	 * Now let's take care of PPC channel 2
-	 *
-	 * Set the Gate high, program PPC channel 2 for mode 0,
-	 * (interrupt on terminal count mode), binary count,
-	 * load 5 * LATCH count, (LSB and MSB) to begin countdown.
-	 *
-	 * Note some implementations have a bug where the high bits byte
-	 * of channel 2 is ignored.
-	 */
-	/* Set up the timer gate, turn off the speaker */
-	/* Set the Gate high, disable speaker */
-	outb((inb(PPC_PORTB) & ~PPCB_SPKR) | PPCB_T2GATE, PPC_PORTB);
-	/* binary, mode 0, LSB/MSB, Ch 2 */
-	outb(TIMER2_SEL|WORD_ACCESS|MODE0|BINARY_COUNT, TIMER_MODE_PORT);
-	/* LSB of ticks */
-	outb(ticks & 0xFF, TIMER2_PORT);
-	/* MSB of ticks */
-	outb(ticks >> 8, TIMER2_PORT);
-}
-
-static int timer2_running ( void ) {
-	return ((inb(PPC_PORTB) & PPCB_T2OUT) == 0);
-}
-
-void timer2_udelay ( unsigned long usecs ) {
-	load_timer2 ( ( usecs * TIMER2_TICKS_PER_SEC ) / ( 1000 * 1000 ) );
-	while (timer2_running()) {
-		/* Do nothing */
-	}
-}
diff --git a/qemu/roms/ipxe/src/arch/i386/core/virtaddr.S b/qemu/roms/ipxe/src/arch/i386/core/virtaddr.S
index 5e5d77352..425591570 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/virtaddr.S
+++ b/qemu/roms/ipxe/src/arch/i386/core/virtaddr.S
@@ -4,7 +4,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #include "librm.h"
 		
diff --git a/qemu/roms/ipxe/src/arch/i386/drivers/net/undi.c b/qemu/roms/ipxe/src/arch/i386/drivers/net/undi.c
index 2bc54824c..9820cf629 100644
--- a/qemu/roms/ipxe/src/arch/i386/drivers/net/undi.c
+++ b/qemu/roms/ipxe/src/arch/i386/drivers/net/undi.c
@@ -68,10 +68,6 @@ static int undipci_probe ( struct pci_device *pci ) {
 	struct undi_rom *undirom;
 	int rc;
 
-	/* Ignore non-network devices */
-	if ( PCI_BASE_CLASS ( pci->class ) != PCI_BASE_CLASS_NETWORK )
-		return -ENOTTY;
-
 	/* Allocate UNDI device structure */
 	undi = zalloc ( sizeof ( *undi ) );
 	if ( ! undi )
@@ -138,12 +134,13 @@ static void undipci_remove ( struct pci_device *pci ) {
 }
 
 static struct pci_device_id undipci_nics[] = {
-PCI_ROM ( 0xffff, 0xffff, "undipci", "UNDI (PCI)", 0 ),
+	PCI_ROM ( 0xffff, 0xffff, "undipci", "UNDI (PCI)", 0 ),
 };
 
 struct pci_driver undipci_driver __pci_driver_fallback = {
 	.ids = undipci_nics,
 	.id_count = ( sizeof ( undipci_nics ) / sizeof ( undipci_nics[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_NETWORK, PCI_ANY_ID, PCI_ANY_ID ),
 	.probe = undipci_probe,
 	.remove = undipci_remove,
 };
diff --git a/qemu/roms/ipxe/src/arch/i386/drivers/net/undiload.c b/qemu/roms/ipxe/src/arch/i386/drivers/net/undiload.c
index 77134dcb8..7160ee384 100644
--- a/qemu/roms/ipxe/src/arch/i386/drivers/net/undiload.c
+++ b/qemu/roms/ipxe/src/arch/i386/drivers/net/undiload.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/drivers/net/undionly.c b/qemu/roms/ipxe/src/arch/i386/drivers/net/undionly.c
index 028fac5d9..70dbe4bfd 100644
--- a/qemu/roms/ipxe/src/arch/i386/drivers/net/undionly.c
+++ b/qemu/roms/ipxe/src/arch/i386/drivers/net/undionly.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/drivers/net/undipreload.c b/qemu/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
index 81d7a80eb..fca771843 100644
--- a/qemu/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
+++ b/qemu/roms/ipxe/src/arch/i386/drivers/net/undipreload.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <undipreload.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
index b23f2c356..6a46081aa 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/basemem.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
index bd73838b5..63413cdc1 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/bios_console.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <realmode.h>
@@ -39,6 +43,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ATTR_FCOL_YELLOW	0x06
 #define ATTR_FCOL_WHITE		0x07
 
+#define ATTR_BLINK		0x80
+
 #define ATTR_BCOL_MASK		0x70
 #define ATTR_BCOL_BLACK		0x00
 #define ATTR_BCOL_BLUE		0x10
@@ -137,8 +143,12 @@ static void bios_handle_sgr ( struct ansiesc_context *ctx __unused,
 			bios_attr = ATTR_DEFAULT;
 		} else if ( aspect == 1 ) {
 			bios_attr |= ATTR_BOLD;
+		} else if ( aspect == 5 ) {
+			bios_attr |= ATTR_BLINK;
 		} else if ( aspect == 22 ) {
 			bios_attr &= ~ATTR_BOLD;
+		} else if ( aspect == 25 ) {
+			bios_attr &= ~ATTR_BLINK;
 		} else if ( ( aspect >= 30 ) && ( aspect <= 39 ) ) {
 			bios_attr &= ~ATTR_FCOL_MASK;
 			bios_attr |= bios_attr_fcols[ aspect - 30 ];
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
index cea17ef8e..d5d97b482 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/e820mangler.S
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
index e5f713728..15f4d772f 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/fakee820.c
@@ -14,9 +14,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <biosint.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
index 8f3069e18..253c601ff 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/hidemem.c
@@ -14,9 +14,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
index 0937a7ce2..bcacecd6a 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/memmap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
index 5c74b0431..20ec35d75 100644
--- a/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
+++ b/qemu/roms/ipxe/src/arch/i386/firmware/pcbios/pnpbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c b/qemu/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
index 523724ab0..473b97f97 100644
--- a/qemu/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
+++ b/qemu/roms/ipxe/src/arch/i386/hci/commands/pxe_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <ipxe/netdevice.h>
@@ -23,7 +27,7 @@
 #include <hci/ifmgmt_cmd.h>
 #include <pxe_call.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/image/bootsector.c b/qemu/roms/ipxe/src/arch/i386/image/bootsector.c
index 9a089e6bb..dba87613c 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/bootsector.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/bootsector.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/image/bzimage.c b/qemu/roms/ipxe/src/arch/i386/image/bzimage.c
index 4865c394c..a64206cd3 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/bzimage.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/bzimage.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/image/elfboot.c b/qemu/roms/ipxe/src/arch/i386/image/elfboot.c
index 0f6957f02..dc3568929 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/elfboot.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/elfboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <elf.h>
@@ -75,6 +79,27 @@ static int elfboot_exec ( struct image *image ) {
 }
 
 /**
+ * Check that ELF segment uses flat physical addressing
+ *
+ * @v image		ELF file
+ * @v phdr		ELF program header
+ * @v dest		Destination address
+ * @ret rc		Return status code
+ */
+static int elfboot_check_segment ( struct image *image, Elf_Phdr *phdr,
+				   physaddr_t dest ) {
+
+	/* Check that ELF segment uses flat physical addressing */
+	if ( phdr->p_vaddr != dest ) {
+		DBGC ( image, "ELF %p uses virtual addressing (phys %x, "
+		       "virt %x)\n", image, phdr->p_paddr, phdr->p_vaddr );
+		return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+/**
  * Probe ELF image
  *
  * @v image		ELF file
@@ -91,14 +116,24 @@ static int elfboot_probe ( struct image *image ) {
 		[EI_DATA]	= ELFDATA2LSB,
 		[EI_VERSION]	= EV_CURRENT,
 	};
+	physaddr_t entry;
+	physaddr_t max;
+	int rc;
 
 	/* Read ELF header */
 	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
 	if ( memcmp ( ehdr.e_ident, e_ident, sizeof ( e_ident ) ) != 0 ) {
-		DBG ( "Invalid ELF identifier\n" );
+		DBGC ( image, "Invalid ELF identifier\n" );
 		return -ENOEXEC;
 	}
 
+	/* Check that this image uses flat physical addressing */
+	if ( ( rc = elf_segments ( image, &ehdr, elfboot_check_segment,
+				   &entry, &max ) ) != 0 ) {
+		DBGC ( image, "Unloadable ELF image\n" );
+		return rc;
+	}
+
 	return 0;
 }
 
diff --git a/qemu/roms/ipxe/src/arch/i386/image/initrd.c b/qemu/roms/ipxe/src/arch/i386/image/initrd.c
index eaba3a645..80c197417 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/initrd.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/initrd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <initrd.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/image/multiboot.c b/qemu/roms/ipxe/src/arch/i386/image/multiboot.c
index 86b0bc12d..0c85df708 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/multiboot.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/multiboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/image/pxe_image.c b/qemu/roms/ipxe/src/arch/i386/image/pxe_image.c
index dc28f6082..5b0f6eb89 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/pxe_image.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/pxe_image.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -34,6 +38,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/netdevice.h>
 #include <ipxe/features.h>
 #include <ipxe/console.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/IndustryStandard/PeImage.h>
 
 FEATURE ( FEATURE_IMAGE, "PXE", DHCP_EB_FEATURE_PXE, 1 );
 
@@ -121,9 +127,45 @@ int pxe_probe ( struct image *image ) {
 	return 0;
 }
 
+/**
+ * Probe PXE image (with rejection of potential EFI images)
+ *
+ * @v image		PXE file
+ * @ret rc		Return status code
+ */
+int pxe_probe_no_mz ( struct image *image ) {
+	uint16_t magic;
+	int rc;
+
+	/* Probe PXE image */
+	if ( ( rc = pxe_probe ( image ) ) != 0 )
+		return rc;
+
+	/* Reject image with an "MZ" signature which may indicate an
+	 * EFI image incorrectly handed out to a BIOS system.
+	 */
+	if ( image->len >= sizeof ( magic ) ) {
+		copy_from_user ( &magic, image->data, 0, sizeof ( magic ) );
+		if ( magic == cpu_to_le16 ( EFI_IMAGE_DOS_SIGNATURE ) ) {
+			DBGC ( image, "IMAGE %p may be an EFI image\n",
+			       image );
+			return -ENOTTY;
+		}
+	}
+
+	return 0;
+}
+
 /** PXE image type */
-struct image_type pxe_image_type __image_type ( PROBE_PXE ) = {
-	.name = "PXE",
-	.probe = pxe_probe,
-	.exec = pxe_exec,
+struct image_type pxe_image_type[] __image_type ( PROBE_PXE ) = {
+	{
+		.name = "PXE-NBP",
+		.probe = pxe_probe_no_mz,
+		.exec = pxe_exec,
+	},
+	{
+		.name = "PXE-NBP (may be EFI?)",
+		.probe = pxe_probe,
+		.exec = pxe_exec,
+	},
 };
diff --git a/qemu/roms/ipxe/src/arch/i386/image/sdi.c b/qemu/roms/ipxe/src/arch/i386/image/sdi.c
index df1c3a868..fa2d0b73f 100644
--- a/qemu/roms/ipxe/src/arch/i386/image/sdi.c
+++ b/qemu/roms/ipxe/src/arch/i386/image/sdi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/basemem.h b/qemu/roms/ipxe/src/arch/i386/include/basemem.h
index c477c7fe2..01c2ea917 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/basemem.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/basemem.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/basemem_packet.h b/qemu/roms/ipxe/src/arch/i386/include/basemem_packet.h
index 3cb477671..def6dee31 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/basemem_packet.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/basemem_packet.h
@@ -1,7 +1,7 @@
 #ifndef BASEMEM_PACKET_H
 #define BASEMEM_PACKET_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bios.h b/qemu/roms/ipxe/src/arch/i386/include/bios.h
index 0754b1168..988bbc62b 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bios.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bios.h
@@ -1,7 +1,7 @@
 #ifndef BIOS_H
 #define BIOS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define BDA_SEG 0x0040
 #define BDA_EQUIPMENT_WORD 0x0010
diff --git a/qemu/roms/ipxe/src/arch/i386/include/biosint.h b/qemu/roms/ipxe/src/arch/i386/include/biosint.h
index ab466af3c..67d6a3811 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/biosint.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/biosint.h
@@ -6,7 +6,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/byteswap.h b/qemu/roms/ipxe/src/arch/i386/include/bits/byteswap.h
index 0d9cb967c..53b6a454d 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/byteswap.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/byteswap.h
@@ -9,7 +9,7 @@
 
 #include <stdint.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static inline __attribute__ (( always_inline, const )) uint16_t
 __bswap_variable_16 ( uint16_t x ) {
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/compiler.h b/qemu/roms/ipxe/src/arch/i386/include/bits/compiler.h
index d48b4b385..87201135f 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/compiler.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/compiler.h
@@ -1,7 +1,10 @@
 #ifndef _BITS_COMPILER_H
 #define _BITS_COMPILER_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_386_NONE
 
 #ifndef ASSEMBLY
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/endian.h b/qemu/roms/ipxe/src/arch/i386/include/bits/endian.h
deleted file mode 100644
index 841885424..000000000
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/endian.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef ETHERBOOT_BITS_ENDIAN_H
-#define ETHERBOOT_BITS_ENDIAN_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define __BYTE_ORDER __LITTLE_ENDIAN
-
-#endif /* ETHERBOOT_BITS_ENDIAN_H */
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/entropy.h b/qemu/roms/ipxe/src/arch/i386/include/bits/entropy.h
index 6dcceec6d..bfeb5e3b5 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/entropy.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/rtc_entropy.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/hyperv.h b/qemu/roms/ipxe/src/arch/i386/include/bits/hyperv.h
new file mode 100644
index 000000000..3565c8a83
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/hyperv.h
@@ -0,0 +1,72 @@
+#ifndef _BITS_HYPERV_H
+#define _BITS_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Issue hypercall
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v code		Call code
+ * @v in		Input parameters
+ * @v out		Output parameters
+ * @ret status		Status code
+ */
+static inline __attribute__ (( always_inline )) int
+hv_call ( struct hv_hypervisor *hv, unsigned int code, const void *in,
+	  void *out ) {
+	void *hypercall = hv->hypercall;
+	uint32_t in_phys;
+	uint32_t out_phys;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint16_t result;
+
+	in_phys = ( ( __builtin_constant_p ( in ) && ( in == NULL ) )
+		       ? 0 : virt_to_phys ( in ) );
+	out_phys = ( ( __builtin_constant_p ( out ) && ( out == NULL ) )
+		       ? 0 : virt_to_phys ( out ) );
+	__asm__ __volatile__ ( "call *%9"
+			       : "=a" ( result ), "=c" ( discard_ecx ),
+				 "=d" ( discard_edx )
+			       : "d" ( 0 ), "a" ( code ),
+				 "b" ( 0 ), "c" ( in_phys ),
+				 "D" ( 0 ), "S" ( out_phys ),
+				 "m" ( hypercall ) );
+	return result;
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bits		Bit field
+ * @v bit		Bit to set
+ */
+static inline __attribute__ (( always_inline )) void
+hv_set_bit ( void *bits, unsigned int bit ) {
+	struct {
+		uint32_t dword[ ( bit / 32 ) + 1 ];
+	} *dwords = bits;
+
+	/* Set bit using "lock bts".  Inform compiler that any memory
+	 * from the start of the bit field up to and including the
+	 * dword containing this bit may be modified.  (This is
+	 * overkill but shouldn't matter in practice since we're
+	 * unlikely to subsequently read other bits from the same bit
+	 * field.)
+	 */
+	__asm__ __volatile__ ( "lock bts %1, %0"
+			       : "+m" ( *dwords ) : "Ir" ( bit ) );
+}
+
+#endif /* _BITS_HYPERV_H */
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/nap.h b/qemu/roms/ipxe/src/arch/i386/include/bits/nap.h
index 64066e6ab..e8bcfd13b 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/nap.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_nap.h>
 #include <ipxe/efi/efix86_nap.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/profile.h b/qemu/roms/ipxe/src/arch/i386/include/bits/profile.h
index f3ee54ae9..e184d7b51 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/profile.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/reboot.h b/qemu/roms/ipxe/src/arch/i386/include/bits/reboot.h
index 5b09e95f7..803dacfe4 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/reboot.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_reboot.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/sanboot.h b/qemu/roms/ipxe/src/arch/i386/include/bits/sanboot.h
index 9c77a4d42..f02d2e649 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/sanboot.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_sanboot.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/smbios.h b/qemu/roms/ipxe/src/arch/i386/include/bits/smbios.h
index cc79eec51..2ab31e74b 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/smbios.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_smbios.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/stdint.h b/qemu/roms/ipxe/src/arch/i386/include/bits/stdint.h
index 8edf13192..fe1f9946a 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/stdint.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/stdint.h
@@ -1,7 +1,7 @@
 #ifndef _BITS_STDINT_H
 #define _BITS_STDINT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 typedef __SIZE_TYPE__		size_t;
 typedef signed long		ssize_t;
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/strings.h b/qemu/roms/ipxe/src/arch/i386/include/bits/strings.h
index 092bcb593..453545f00 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/strings.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/strings.h
@@ -1,7 +1,51 @@
 #ifndef _BITS_STRINGS_H
 #define _BITS_STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+	long lsb_minus_one;
+
+	/* If the input value is zero, the BSF instruction returns
+	 * ZF=0 and leaves an undefined value in the output register.
+	 * Perform this check in C rather than asm so that it can be
+	 * omitted in cases where the compiler is able to prove that
+	 * the input is non-zero.
+	 */
+	if ( value ) {
+		__asm__ ( "bsfl %1, %0"
+			  : "=r" ( lsb_minus_one )
+			  : "rm" ( value ) );
+		return ( lsb_minus_one + 1 );
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	unsigned long high = ( value >> 32 );
+	unsigned long low = ( value >> 0 );
+
+	if ( low ) {
+		return ( __ffsl ( low ) );
+	} else if ( high ) {
+		return ( 32 + __ffsl ( high ) );
+	} else {
+		return 0;
+	}
+}
 
 /**
  * Find last (i.e. most significant) set bit
@@ -13,7 +57,7 @@ static inline __attribute__ (( always_inline )) int __flsl ( long value ) {
 	long msb_minus_one;
 
 	/* If the input value is zero, the BSR instruction returns
-	 * ZF=1 and leaves an undefined value in the output register.
+	 * ZF=0 and leaves an undefined value in the output register.
 	 * Perform this check in C rather than asm so that it can be
 	 * omitted in cases where the compiler is able to prove that
 	 * the input is non-zero.
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/time.h b/qemu/roms/ipxe/src/arch/i386/include/bits/time.h
index 24dd020e9..6a5d63d32 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/time.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/rtc_time.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/timer.h b/qemu/roms/ipxe/src/arch/i386/include/bits/timer.h
index 50b676b77..f7d86d78c 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/timer.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bios_timer.h>
 #include <ipxe/rdtsc_timer.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/uaccess.h b/qemu/roms/ipxe/src/arch/i386/include/bits/uaccess.h
index 2bb52e021..aac09ba95 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/uaccess.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/uaccess.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <librm.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bits/umalloc.h b/qemu/roms/ipxe/src/arch/i386/include/bits/umalloc.h
index 54fb006f0..113f16fd1 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bits/umalloc.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bits/umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/memtop_umalloc.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bootsector.h b/qemu/roms/ipxe/src/arch/i386/include/bootsector.h
index 8730fbfcc..c5d35aae3 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bootsector.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bootsector.h
@@ -6,7 +6,7 @@
  * x86 bootsector image format
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int call_bootsector ( unsigned int segment, unsigned int offset,
 			     unsigned int drive );
diff --git a/qemu/roms/ipxe/src/arch/i386/include/bzimage.h b/qemu/roms/ipxe/src/arch/i386/include/bzimage.h
index 7e42e3188..4933ce5b1 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/bzimage.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/bzimage.h
@@ -1,7 +1,7 @@
 #ifndef _BZIMAGE_H
 #define _BZIMAGE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h b/qemu/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
index 184177219..c17c1ea5e 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/efi/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/fakee820.h b/qemu/roms/ipxe/src/arch/i386/include/fakee820.h
index 9d00fb670..552b1e48d 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/fakee820.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/fakee820.h
@@ -1,7 +1,7 @@
 #ifndef _FAKEE820_H
 #define _FAKEE820_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void fake_e820 ( void );
 extern void unfake_e820 ( void );
diff --git a/qemu/roms/ipxe/src/arch/i386/include/initrd.h b/qemu/roms/ipxe/src/arch/i386/include/initrd.h
index a5659f43c..ddb3e5a45 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/initrd.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/initrd.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/int13.h b/qemu/roms/ipxe/src/arch/i386/include/int13.h
index e337ca1d1..f82a583c6 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/int13.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/int13.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
index 5b684c041..c9b82c1e5 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_PCBIOS
 #define NAP_PREFIX_pcbios
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
index a0845328d..3f6df9073 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_PCBIOS
 #define REBOOT_PREFIX_pcbios
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
index 689227b70..1a86b7d57 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SANBOOT_PCBIOS
 #define SANBOOT_PREFIX_pcbios
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
index d8c7f648a..9f7f9c8ff 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_PCBIOS
 #define SMBIOS_PREFIX_pcbios
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
index f9fc80412..6b88a623c 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/bios_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_PCBIOS
 #define TIMER_PREFIX_pcbios
@@ -15,7 +15,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define TIMER_PREFIX_pcbios __pcbios_
 #endif
 
-#include <ipxe/timer2.h>
+#include <ipxe/pit8254.h>
 
 /**
  * Delay for a fixed number of microseconds
@@ -25,9 +25,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 static inline __always_inline void
 TIMER_INLINE ( pcbios, udelay ) ( unsigned long usecs ) {
 	/* BIOS timer is not high-resolution enough for udelay(), so
-	 * we use timer2
+	 * we use the 8254 Programmable Interval Timer.
 	 */
-	timer2_udelay ( usecs );
+	pit8254_udelay ( usecs );
 }
 
 /**
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
index 3a9eb2495..6312adaa4 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/errno/pcbios.h
@@ -10,7 +10,7 @@
  * for the PC-BIOS platform.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_error.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
index 72a0f714f..bc3d85506 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/guestrpc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/vmware.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
index 001648fe5..dee055d16 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/memtop_umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UMALLOC_MEMTOP
 #define UMALLOC_PREFIX_memtop
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/msr.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/msr.h
index c88e26a39..5705318fd 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/msr.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/msr.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Read model-specific register
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
index 472e14007..598f4bb08 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rdtsc_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_RDTSC
 #define TIMER_PREFIX_rdtsc
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
index 6c3cf2104..e214745d0 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
index c0dfe3f88..cb8c7f49e 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/rtc_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_RTC
 #define TIME_PREFIX_rtc
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/timer2.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/timer2.h
deleted file mode 100644
index 322a3ed59..000000000
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/timer2.h
+++ /dev/null
@@ -1,14 +0,0 @@
-#ifndef _IPXE_TIMER2_H
-#define _IPXE_TIMER2_H
-
-/** @file
- *
- * Timer chip control
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-extern void timer2_udelay ( unsigned long usecs );
-
-#endif /* _IPXE_TIMER2_H */
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
index 48cd6a7b7..efc8f2cb8 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/vesafb.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/ipxe/vmware.h b/qemu/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
index 2ac65f436..24f60a03a 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/ipxe/vmware.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/librm.h b/qemu/roms/ipxe/src/arch/i386/include/librm.h
index c8ba72b53..a8a578a39 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/librm.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/librm.h
@@ -1,7 +1,7 @@
 #ifndef LIBRM_H
 #define LIBRM_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Segment selectors as used in our protected-mode GDTs.
  *
@@ -170,18 +170,6 @@ extern uint16_t __text16 ( rm_cs );
 extern uint16_t __text16 ( rm_ds );
 #define rm_ds __use_text16 ( rm_ds )
 
-/**
- * Convert segment:offset address to user buffer
- *
- * @v segment		Real-mode segment
- * @v offset		Real-mode offset
- * @ret buffer		User buffer
- */
-static inline __always_inline userptr_t
-real_to_user ( unsigned int segment, unsigned int offset ) {
-	return ( phys_to_user ( ( segment << 4 ) + offset ) );
-}
-
 extern uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size );
 extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/limits.h b/qemu/roms/ipxe/src/arch/i386/include/limits.h
index 031b6c57a..bb48b75ab 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/limits.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/limits.h
@@ -1,7 +1,7 @@
 #ifndef LIMITS_H
 #define LIMITS_H	1
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Number of bits in a `char' */
 #define CHAR_BIT	8
diff --git a/qemu/roms/ipxe/src/arch/i386/include/memsizes.h b/qemu/roms/ipxe/src/arch/i386/include/memsizes.h
index 7b217494a..f115f7574 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/memsizes.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/memsizes.h
@@ -1,7 +1,7 @@
 #ifndef _MEMSIZES_H
 #define _MEMSIZES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <basemem.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/multiboot.h b/qemu/roms/ipxe/src/arch/i386/include/multiboot.h
index 44614c73a..ae09df6c7 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/multiboot.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/multiboot.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h b/qemu/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
index a36d9cfa1..e07e4c192 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pcbios/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pnpbios.h b/qemu/roms/ipxe/src/arch/i386/include/pnpbios.h
index 4c20e73ed..d14873700 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pnpbios.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pnpbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* BIOS segment address */
 #define BIOS_SEG 0xf000
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pxe.h b/qemu/roms/ipxe/src/arch/i386/include/pxe.h
index b95b0cce5..66d752683 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pxe.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pxe.h
@@ -1,7 +1,7 @@
 #ifndef PXE_H
 #define PXE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include "pxe_types.h"
 #include "pxe_error.h"
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pxe_api.h b/qemu/roms/ipxe/src/arch/i386/include/pxe_api.h
index e4396efb2..3110d26da 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pxe_api.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pxe_api.h
@@ -17,6 +17,10 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  * As an alternative, at your option, you may use this file under the
  * following terms, known as the "MIT license":
  *
@@ -49,7 +53,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include "pxe_types.h"
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pxe_call.h b/qemu/roms/ipxe/src/arch/i386/include/pxe_call.h
index 45af46549..cbd548318 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pxe_call.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pxe_call.h
@@ -6,7 +6,7 @@
  * PXE API entry point
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_api.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pxe_error.h b/qemu/roms/ipxe/src/arch/i386/include/pxe_error.h
index a1398cbd4..51298e665 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pxe_error.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pxe_error.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @defgroup pxeerrors PXE error codes
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pxe_types.h b/qemu/roms/ipxe/src/arch/i386/include/pxe_types.h
index db8214591..483666e33 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pxe_types.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/pxe_types.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h> /* PXE status codes */
diff --git a/qemu/roms/ipxe/src/arch/i386/include/realmode.h b/qemu/roms/ipxe/src/arch/i386/include/realmode.h
index dafc5a32a..4defd3b97 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/realmode.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/realmode.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Declaration of variables in .data16
@@ -65,6 +65,18 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /**
+ * Convert segment:offset address to user buffer
+ *
+ * @v segment		Real-mode segment
+ * @v offset		Real-mode offset
+ * @ret buffer		User buffer
+ */
+static inline __always_inline userptr_t
+real_to_user ( unsigned int segment, unsigned int offset ) {
+	return ( phys_to_user ( ( segment << 4 ) + offset ) );
+}
+
+/**
  * Copy data to base memory
  *
  * @v dest_seg		Destination segment
diff --git a/qemu/roms/ipxe/src/arch/i386/include/registers.h b/qemu/roms/ipxe/src/arch/i386/include/registers.h
index 06d236524..d9aa3c376 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/registers.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/registers.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/rtc.h b/qemu/roms/ipxe/src/arch/i386/include/rtc.h
index 2a6abbae5..6294b63e3 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/rtc.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/rtc.h
@@ -13,7 +13,7 @@
  *    http://wiki.osdev.org/CMOS
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pic8259.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/sdi.h b/qemu/roms/ipxe/src/arch/i386/include/sdi.h
index fc486402d..806c3f194 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/sdi.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/sdi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** SDI image header */
 struct sdi_header {
diff --git a/qemu/roms/ipxe/src/arch/i386/include/setjmp.h b/qemu/roms/ipxe/src/arch/i386/include/setjmp.h
index 5d3c11b69..fe1a9ef4d 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/setjmp.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/setjmp.h
@@ -1,40 +1,50 @@
-#ifndef ETHERBOOT_SETJMP_H
-#define ETHERBOOT_SETJMP_H
+#ifndef _SETJMP_H
+#define _SETJMP_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
 
 /** A jump buffer */
 typedef struct {
+	/** Saved return address */
 	uint32_t retaddr;
+	/** Saved stack pointer */
+	uint32_t stack;
+	/** Saved %ebx */
 	uint32_t ebx;
-	uint32_t esp;
-	uint32_t ebp;
+	/** Saved %esi */
 	uint32_t esi;
+	/** Saved %edi */
 	uint32_t edi;
+	/** Saved %ebp */
+	uint32_t ebp;
 } jmp_buf[1];
 
 /** A real-mode-extended jump buffer */
 typedef struct {
+	/** Jump buffer */
 	jmp_buf env;
-	uint16_t rm_ss;
-	uint16_t rm_sp;
+	/** Real-mode stack pointer */
+	segoff_t rm_stack;
 } rmjmp_buf[1];
 
-extern int __asmcall setjmp ( jmp_buf env );
-extern void __asmcall longjmp ( jmp_buf env, int val );
+extern int __asmcall __attribute__ (( returns_twice ))
+setjmp ( jmp_buf env );
+
+extern void __asmcall __attribute__ (( noreturn ))
+longjmp ( jmp_buf env, int val );
 
-#define rmsetjmp( _env ) ( {			\
-	(_env)->rm_ss = rm_ss;			\
-	(_env)->rm_sp = rm_sp;			\
-	setjmp ( (_env)->env ); } )		\
+#define rmsetjmp( _env ) ( {					\
+	(_env)->rm_stack.segment = rm_ss;			\
+	(_env)->rm_stack.offset = rm_sp;			\
+	setjmp ( (_env)->env ); } )				\
 
-#define rmlongjmp( _env, _val ) do {		\
-	rm_ss = (_env)->rm_ss;			\
-	rm_sp = (_env)->rm_sp;			\
-	longjmp ( (_env)->env, (_val) );	\
+#define rmlongjmp( _env, _val ) do {				\
+	rm_ss = (_env)->rm_stack.segment;			\
+	rm_sp = (_env)->rm_stack.offset;			\
+	longjmp ( (_env)->env, (_val) );			\
 	} while ( 0 )
 
-#endif /* ETHERBOOT_SETJMP_H */
+#endif /* _SETJMP_H */
diff --git a/qemu/roms/ipxe/src/arch/i386/include/undi.h b/qemu/roms/ipxe/src/arch/i386/include/undi.h
index 325fcbbf9..7a5624f93 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/undi.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/undi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef ASSEMBLY
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/undiload.h b/qemu/roms/ipxe/src/arch/i386/include/undiload.h
index 426830e8d..235e7a79e 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/undiload.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/undiload.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct undi_device;
 struct undi_rom;
diff --git a/qemu/roms/ipxe/src/arch/i386/include/undinet.h b/qemu/roms/ipxe/src/arch/i386/include/undinet.h
index c3c17c11a..2798c4466 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/undinet.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/undinet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct undi_device;
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/undipreload.h b/qemu/roms/ipxe/src/arch/i386/include/undipreload.h
index de9b8fb52..57f493cec 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/undipreload.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/undipreload.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <realmode.h>
 #include <undi.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/include/undirom.h b/qemu/roms/ipxe/src/arch/i386/include/undirom.h
index 86d7077b5..1c530118d 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/undirom.h
+++ b/qemu/roms/ipxe/src/arch/i386/include/undirom.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <pxe_types.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/apm.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
index 3b13e1cd0..50b19cb81 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/apm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
index 1e7de756b..f1ba8297b 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_nap.c
@@ -1,7 +1,7 @@
 #include <ipxe/nap.h>
 #include <realmode.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Save power by halting the CPU until the next interrupt
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
index 68546b2e5..10a1ecb89 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
index dd7897e29..a8c0fc325 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_smbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
index 65bbf9e01..3299c9aae 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/bios_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
index a193defa3..3b8e80438 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/biosint.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Hook INT vector
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
index 1c7a8128f..f0450da90 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c
new file mode 100644
index 000000000..2414c6909
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/int13con.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <ipxe/console.h>
+#include <ipxe/init.h>
+#include <realmode.h>
+#include <int13.h>
+#include <config/console.h>
+
+/** @file
+ *
+ * INT13 disk log console
+ *
+ */
+
+/* Set default console usage if applicable */
+#if ! ( defined ( CONSOLE_INT13 ) && CONSOLE_EXPLICIT ( CONSOLE_INT13 ) )
+#undef CONSOLE_INT13
+#define CONSOLE_INT13 ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
+#endif
+
+/** Disk drive number */
+#define INT13CON_DRIVE 0x80
+
+/** Log partition type */
+#define INT13CON_PARTITION_TYPE 0xe0
+
+/** Maximum number of outstanding unwritten characters */
+#define INT13CON_MAX_UNWRITTEN 64
+
+/** Log partition header */
+struct int13con_header {
+	/** Magic signature */
+	char magic[10];
+} __attribute__ (( packed ));
+
+/** Log partition magic signature */
+#define INT13CON_MAGIC "iPXE LOG\n\n"
+
+/** Sector buffer */
+static uint8_t __bss16_array ( int13con_buffer, [INT13_BLKSIZE] );
+#define int13con_buffer __use_data16 ( int13con_buffer )
+
+/** Disk address packet */
+static struct int13_disk_address __bss16 ( int13con_address );
+#define int13con_address __use_data16 ( int13con_address )
+
+/** Current LBA */
+static uint64_t int13con_lba;
+
+/** Maximum LBA */
+static uint64_t int13con_max_lba;
+
+/** Current offset within sector */
+static size_t int13con_offset;
+
+/** Number of unwritten characters */
+static size_t int13con_unwritten;
+
+struct console_driver int13con __console_driver;
+
+/**
+ * Read/write disk sector
+ *
+ * @v op		Operation
+ * @v lba		Logical block address
+ * @ret rc		Return status code
+ */
+static int int13con_rw ( unsigned int op, uint64_t lba ) {
+	uint8_t error;
+
+	/* Construct disk address packet */
+	int13con_address.bufsize = sizeof ( int13con_address );
+	int13con_address.count = 1;
+	int13con_address.buffer.segment = rm_ds;
+	int13con_address.buffer.offset = __from_data16 ( int13con_buffer );
+	int13con_address.lba = lba;
+
+	/* Issue INT13 */
+	__asm__ ( REAL_CODE ( "int $0x13\n\t" )
+		  : "=a" ( error )
+		  : "0" ( op << 8 ), "d" ( INT13CON_DRIVE ),
+		    "S" ( __from_data16 ( &int13con_address ) ) );
+	if ( error ) {
+		DBG ( "INT13CON operation %04x failed: %02x\n",
+		      op, error );
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * Write character to console
+ *
+ * @v character		Character
+ */
+static void int13con_putchar ( int character ) {
+	static int busy;
+	int rc;
+
+	/* Ignore if we are already mid-logging */
+	if ( busy )
+		return;
+	busy = 1;
+
+	/* Write character to buffer */
+	int13con_buffer[int13con_offset++] = character;
+	int13con_unwritten++;
+
+	/* Write sector to disk, if applicable */
+	if ( ( int13con_offset == INT13_BLKSIZE ) ||
+	     ( int13con_unwritten == INT13CON_MAX_UNWRITTEN ) ||
+	     ( character == '\n' ) ) {
+
+		/* Write sector to disk */
+		if ( ( rc = int13con_rw ( INT13_EXTENDED_WRITE,
+					  int13con_lba ) ) != 0 ) {
+			DBG ( "INT13CON could not write log\n" );
+			/* Ignore and continue; there's nothing we can do */
+		}
+
+		/* Reset count of unwritten characters */
+		int13con_unwritten = 0;
+	}
+
+	/* Move to next sector, if applicable */
+	if ( int13con_offset == INT13_BLKSIZE ) {
+
+		/* Disable console if we have run out of space */
+		if ( int13con_lba >= int13con_max_lba )
+			int13con.disabled = 1;
+
+		/* Clear log buffer */
+		memset ( int13con_buffer, 0, sizeof ( int13con_buffer ) );
+		int13con_offset = 0;
+
+		/* Move to next sector */
+		int13con_lba++;
+	}
+
+	/* Clear busy flag */
+	busy = 0;
+}
+
+/**
+ * Find log partition
+ *
+ * @ret rc		Return status code
+ */
+static int int13con_find ( void ) {
+	struct master_boot_record *mbr =
+		( ( struct master_boot_record * ) int13con_buffer );
+	struct int13con_header *hdr =
+		( ( struct int13con_header * ) int13con_buffer );
+	struct partition_table_entry part[4];
+	unsigned int i;
+	int rc;
+
+	/* Read MBR */
+	if ( ( rc = int13con_rw ( INT13_EXTENDED_READ, 0 ) ) != 0 ) {
+		DBG ( "INT13CON could not read MBR: %s\n", strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check MBR magic */
+	if ( mbr->magic != INT13_MBR_MAGIC ) {
+		DBG ( "INT13CON incorrect MBR magic\n" );
+		DBG2_HDA ( 0, mbr, sizeof ( *mbr ) );
+		return -EINVAL;
+	}
+
+	/* Look for magic partition */
+	memcpy ( part, mbr->partitions, sizeof ( part ) );
+	for ( i = 0 ; i < ( sizeof ( part ) / sizeof ( part[0] ) ) ; i++ ) {
+
+		/* Skip partitions of the wrong type */
+		if ( part[i].type != INT13CON_PARTITION_TYPE )
+			continue;
+
+		/* Read partition header */
+		if ( ( rc = int13con_rw ( INT13_EXTENDED_READ,
+					  part[i].start ) ) != 0 ) {
+			DBG ( "INT13CON partition %d could not read header: "
+			      "%s\n", ( i + 1 ), strerror ( rc ) );
+			continue;
+		}
+
+		/* Check partition header */
+		if ( memcmp ( hdr->magic, INT13CON_MAGIC,
+			      sizeof ( hdr->magic ) ) != 0 ) {
+			DBG ( "INT13CON partition %d bad magic\n", ( i + 1 ) );
+			DBG2_HDA ( 0, hdr, sizeof ( *hdr ) );
+			continue;
+		}
+
+		/* Found log partition */
+		DBG ( "INT13CON partition %d at [%08x,%08x)\n", ( i + 1 ),
+		      part[i].start, ( part[i].start + part[i].length ) );
+		int13con_lba = part[i].start;
+		int13con_max_lba = ( part[i].start + part[i].length - 1 );
+
+		/* Initialise log buffer */
+		memset ( &int13con_buffer[ sizeof ( *hdr ) ], 0,
+			 ( sizeof ( int13con_buffer ) - sizeof ( *hdr ) ) );
+		int13con_offset = sizeof ( hdr->magic );
+
+		return 0;
+	}
+
+	DBG ( "INT13CON found no log partition\n" );
+	return -ENOENT;
+}
+
+/**
+ * Initialise INT13 console
+ *
+ */
+static void int13con_init ( void ) {
+	uint8_t error;
+	uint16_t check;
+	unsigned int discard_c;
+	unsigned int discard_d;
+	int rc;
+
+	/* Check for INT13 extensions */
+	__asm__ __volatile__ ( REAL_CODE ( "int $0x13\n\t"
+					   "setc %%al\n\t" )
+			       : "=a" ( error ), "=b" ( check ),
+				 "=c" ( discard_c ), "=d" ( discard_d )
+			       : "0" ( INT13_EXTENSION_CHECK << 8 ),
+				 "1" ( 0x55aa ), "3" ( INT13CON_DRIVE ) );
+	if ( error || ( check != 0xaa55 ) ) {
+		DBG ( "INT13CON missing extensions (%02x,%04x)\n",
+		      error, check );
+		return;
+	}
+
+	/* Locate log partition */
+	if ( ( rc = int13con_find() ) != 0)
+		return;
+
+	/* Enable console */
+	int13con.disabled = 0;
+}
+
+/**
+ * INT13 console initialisation function
+ */
+struct init_fn int13con_init_fn __init_fn ( INIT_CONSOLE ) = {
+	.initialise = int13con_init,
+};
+
+/** INT13 console driver */
+struct console_driver int13con __console_driver = {
+	.putchar = int13con_putchar,
+	.disabled = CONSOLE_DISABLED,
+	.usage = CONSOLE_INT13,
+};
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
index c382e3c36..957f8e324 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/memtop_umalloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
index 61873039f..34efa0b39 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/pcibios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
@@ -38,7 +42,11 @@ static int pcibios_num_bus ( void ) {
 	int discard_a, discard_D;
 	uint8_t max_bus;
 
-	__asm__ __volatile__ ( REAL_CODE ( "stc\n\t"
+	/* We issue this call using flat real mode, to work around a
+	 * bug in some HP BIOSes.
+	 */
+	__asm__ __volatile__ ( REAL_CODE ( "call flatten_real_mode\n\t"
+					   "stc\n\t"
 					   "int $0x1a\n\t"
 					   "jnc 1f\n\t"
 					   "xorw %%cx, %%cx\n\t"
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
index fad421c2a..9aab03c03 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
index 67041d4ca..cdbeac8d5 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/rtc_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
index 2adc7b040..9cf2bf29e 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pcbios/vesafb.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
index 657d47b6c..104313666 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_call.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 #include <ipxe/init.h>
@@ -342,6 +346,7 @@ int pxe_start_nbp ( void ) {
 	return 0;
 }
 
+REQUIRING_SYMBOL ( pxe_api_call );
 REQUIRE_OBJECT ( pxe_preboot );
 REQUIRE_OBJECT ( pxe_undi );
 REQUIRE_OBJECT ( pxe_udp );
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
index 6274264ff..07852cd50 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_entry.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
index 9d1896507..f92dae0d1 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_exit_hook.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <realmode.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
index 6e9610294..456ffb5fd 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_file.c
@@ -31,9 +31,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 FEATURE ( FEATURE_MISC, "PXEXT", DHCP_EB_FEATURE_PXE_EXT, 2 );
 
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
index 695af3b93..e6a2e072a 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_loader.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/init.h>
 #include "pxe.h"
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
index 534352b2b..6e09080bc 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_preboot.c
@@ -22,9 +22,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -174,18 +178,16 @@ pxenv_get_cached_info ( struct s_PXENV_GET_CACHED_INFO *get_cached_info ) {
 	}
 	info = &cached_info[idx];
 
-	/* Construct cached version of packet, if not already constructed. */
-	if ( ! info->dhcphdr.op ) {
-		/* Construct DHCP packet */
-		creator = &pxe_dhcp_packet_creators[idx];
-		if ( ( rc = creator->create ( pxe_netdev, info,
-					      sizeof ( *info ) ) ) != 0 ) {
-			DBGC ( &pxe_netdev, " failed to build packet: %s\n",
-			       strerror ( rc ) );
-			goto err;
-		}
+	/* Construct DHCP packet */
+	creator = &pxe_dhcp_packet_creators[idx];
+	if ( ( rc = creator->create ( pxe_netdev, info,
+				      sizeof ( *info ) ) ) != 0 ) {
+		DBGC ( &pxe_netdev, " failed to build packet: %s\n",
+		       strerror ( rc ) );
+		goto err;
 	}
 
+	/* Copy packet (if applicable) */
 	len = get_cached_info->BufferSize;
 	if ( len == 0 ) {
 		/* Point client at our cached buffer.
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
index f4801bad0..068d8a7b2 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_tftp.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -36,6 +40,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/xfer.h>
 #include <ipxe/open.h>
 #include <ipxe/process.h>
+#include <ipxe/uri.h>
+#include <realmode.h>
 #include <pxe.h>
 
 /** A PXE TFTP connection */
@@ -170,11 +176,10 @@ static struct pxe_tftp_connection pxe_tftp = {
  * @v blksize		Requested block size
  * @ret rc		Return status code
  */
-static int pxe_tftp_open ( uint32_t ipaddress, unsigned int port,
-			   const unsigned char *filename, size_t blksize,
-			   int sizeonly ) {
-	char uri_string[PXE_TFTP_URI_LEN];
+static int pxe_tftp_open ( IP4_t ipaddress, UDP_PORT_t port,
+			   UINT8_t *filename, UINT16_t blksize ) {
 	struct in_addr address;
+	struct uri *uri;
 	int rc;
 
 	/* Reset PXE TFTP connection structure */
@@ -185,19 +190,20 @@ static int pxe_tftp_open ( uint32_t ipaddress, unsigned int port,
 	pxe_tftp.blksize = blksize;
 	pxe_tftp.rc = -EINPROGRESS;
 
-	/* Construct URI string */
+	/* Construct URI */
 	address.s_addr = ipaddress;
-	if ( ! port )
-		port = htons ( TFTP_PORT );
-	snprintf ( uri_string, sizeof ( uri_string ), "tftp%s://%s:%d%s%s",
-		   sizeonly ? "size" : "", inet_ntoa ( address ),
-		   ntohs ( port ), ( ( filename[0] == '/' ) ? "" : "/" ),
-		   filename );
-	DBG ( " %s", uri_string );
+	DBG ( " %s", inet_ntoa ( address ) );
+	if ( port )
+		DBG ( ":%d", ntohs ( port ) );
+	DBG ( ":%s", filename );
+	uri = tftp_uri ( address, ntohs ( port ), ( ( char * ) filename ) );
+	if ( ! uri ) {
+		DBG ( " could not create URI\n" );
+		return -ENOMEM;
+	}
 
 	/* Open PXE TFTP connection */
-	if ( ( rc = xfer_open_uri_string ( &pxe_tftp.xfer,
-					   uri_string ) ) != 0 ) {
+	if ( ( rc = xfer_open_uri ( &pxe_tftp.xfer, uri ) ) != 0 ) {
 		DBG ( " could not open (%s)\n", strerror ( rc ) );
 		return rc;
 	}
@@ -259,8 +265,7 @@ static PXENV_EXIT_t pxenv_tftp_open ( struct s_PXENV_TFTP_OPEN *tftp_open ) {
 	if ( ( rc = pxe_tftp_open ( tftp_open->ServerIPAddress,
 				    tftp_open->TFTPPort,
 				    tftp_open->FileName,
-				    tftp_open->PacketSize,
-				    0) ) != 0 ) {
+				    tftp_open->PacketSize ) ) != 0 ) {
 		tftp_open->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
@@ -483,7 +488,7 @@ PXENV_EXIT_t pxenv_tftp_read_file ( struct s_PXENV_TFTP_READ_FILE
 
 	/* Open TFTP file */
 	if ( ( rc = pxe_tftp_open ( tftp_read_file->ServerIPAddress, 0,
-				    tftp_read_file->FileName, 0, 0 ) ) != 0 ) {
+				    tftp_read_file->FileName, 0 ) ) != 0 ) {
 		tftp_read_file->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
@@ -553,7 +558,7 @@ static PXENV_EXIT_t pxenv_tftp_get_fsize ( struct s_PXENV_TFTP_GET_FSIZE
 
 	/* Open TFTP file */
 	if ( ( rc = pxe_tftp_open ( tftp_get_fsize->ServerIPAddress, 0,
-				    tftp_get_fsize->FileName, 0, 1 ) ) != 0 ) {
+				    tftp_get_fsize->FileName, 0 ) ) != 0 ) {
 		tftp_get_fsize->Status = PXENV_STATUS ( rc );
 		return PXENV_EXIT_FAILURE;
 	}
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
index 32bc39c8e..071cb59db 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_udp.c
@@ -11,6 +11,7 @@
 #include <ipxe/udp.h>
 #include <ipxe/uaccess.h>
 #include <ipxe/process.h>
+#include <realmode.h>
 #include <pxe.h>
 
 /*
@@ -30,9 +31,25 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A PXE UDP pseudo-header */
+struct pxe_udp_pseudo_header {
+	/** Source IP address */
+	IP4_t src_ip;
+	/** Source port */
+	UDP_PORT_t s_port;
+	/** Destination IP address */
+	IP4_t dest_ip;
+	/** Destination port */
+	UDP_PORT_t d_port;
+} __attribute__ (( packed ));
 
 /** A PXE UDP connection */
 struct pxe_udp_connection {
@@ -40,8 +57,8 @@ struct pxe_udp_connection {
 	struct interface xfer;
 	/** Local address */
 	struct sockaddr_in local;
-	/** Current PXENV_UDP_READ parameter block */
-	struct s_PXENV_UDP_READ *pxenv_udp_read;
+	/** List of received packets */
+	struct list_head list;
 };
 
 /**
@@ -58,45 +75,38 @@ struct pxe_udp_connection {
 static int pxe_udp_deliver ( struct pxe_udp_connection *pxe_udp,
 			     struct io_buffer *iobuf,
 			     struct xfer_metadata *meta ) {
-	struct s_PXENV_UDP_READ *pxenv_udp_read = pxe_udp->pxenv_udp_read;
+	struct pxe_udp_pseudo_header *pshdr;
 	struct sockaddr_in *sin_src;
 	struct sockaddr_in *sin_dest;
-	userptr_t buffer;
-	size_t len;
-	int rc = 0;
-
-	if ( ! pxenv_udp_read ) {
-		DBG ( "PXE discarded UDP packet\n" );
-		rc = -ENOBUFS;
-		goto done;
-	}
-
-	/* Copy packet to buffer and record length */
-	buffer = real_to_user ( pxenv_udp_read->buffer.segment,
-				pxenv_udp_read->buffer.offset );
-	len = iob_len ( iobuf );
-	if ( len > pxenv_udp_read->buffer_size )
-		len = pxenv_udp_read->buffer_size;
-	copy_to_user ( buffer, 0, iobuf->data, len );
-	pxenv_udp_read->buffer_size = len;
+	int rc;
 
-	/* Fill in source/dest information */
+	/* Extract metadata */
 	assert ( meta );
 	sin_src = ( struct sockaddr_in * ) meta->src;
 	assert ( sin_src );
 	assert ( sin_src->sin_family == AF_INET );
-	pxenv_udp_read->src_ip = sin_src->sin_addr.s_addr;
-	pxenv_udp_read->s_port = sin_src->sin_port;
 	sin_dest = ( struct sockaddr_in * ) meta->dest;
 	assert ( sin_dest );
 	assert ( sin_dest->sin_family == AF_INET );
-	pxenv_udp_read->dest_ip = sin_dest->sin_addr.s_addr;
-	pxenv_udp_read->d_port = sin_dest->sin_port;
 
-	/* Mark as received */
-	pxe_udp->pxenv_udp_read = NULL;
+	/* Construct pseudo-header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *pshdr ) ) ) != 0 ) {
+		DBG ( "PXE could not prepend pseudo-header\n" );
+		rc = -ENOMEM;
+		goto drop;
+	}
+	pshdr = iob_push ( iobuf, sizeof ( *pshdr ) );
+	pshdr->src_ip = sin_src->sin_addr.s_addr;
+	pshdr->s_port = sin_src->sin_port;
+	pshdr->dest_ip = sin_dest->sin_addr.s_addr;
+	pshdr->d_port = sin_dest->sin_port;
 
- done:
+	/* Add to queue */
+	list_add_tail ( &iobuf->list, &pxe_udp->list );
+
+	return 0;
+
+ drop:
 	free_iob ( iobuf );
 	return rc;
 }
@@ -116,6 +126,7 @@ static struct pxe_udp_connection pxe_udp = {
 	.local = {
 		.sin_family = AF_INET,
 	},
+	.list = LIST_HEAD_INIT ( pxe_udp.list ),
 };
 
 /**
@@ -205,11 +216,20 @@ static PXENV_EXIT_t pxenv_udp_open ( struct s_PXENV_UDP_OPEN *pxenv_udp_open ) {
  */
 static PXENV_EXIT_t
 pxenv_udp_close ( struct s_PXENV_UDP_CLOSE *pxenv_udp_close ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
 	DBG ( "PXENV_UDP_CLOSE\n" );
 
 	/* Close UDP connection */
 	intf_restart ( &pxe_udp.xfer, 0 );
 
+	/* Discard any received packets */
+	list_for_each_entry_safe ( iobuf, tmp, &pxe_udp.list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+
 	pxenv_udp_close->Status = PXENV_STATUS_SUCCESS;
 	return PXENV_EXIT_SUCCESS;
 }
@@ -365,20 +385,32 @@ pxenv_udp_write ( struct s_PXENV_UDP_WRITE *pxenv_udp_write ) {
 static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	struct in_addr dest_ip_wanted = { .s_addr = pxenv_udp_read->dest_ip };
 	struct in_addr dest_ip;
+	struct io_buffer *iobuf;
+	struct pxe_udp_pseudo_header *pshdr;
 	uint16_t d_port_wanted = pxenv_udp_read->d_port;
 	uint16_t d_port;
+	userptr_t buffer;
+	size_t len;
+
+	/* Try receiving a packet, if the queue is empty */
+	if ( list_empty ( &pxe_udp.list ) )
+		step();
 
-	/* Try receiving a packet */
-	pxe_udp.pxenv_udp_read = pxenv_udp_read;
-	step();
-	if ( pxe_udp.pxenv_udp_read ) {
+	/* Remove first packet from the queue */
+	iobuf = list_first_entry ( &pxe_udp.list, struct io_buffer, list );
+	if ( ! iobuf ) {
 		/* No packet received */
 		DBG2 ( "PXENV_UDP_READ\n" );
-		pxe_udp.pxenv_udp_read = NULL;
 		goto no_packet;
 	}
-	dest_ip.s_addr = pxenv_udp_read->dest_ip;
-	d_port = pxenv_udp_read->d_port;
+	list_del ( &iobuf->list );
+
+	/* Strip pseudo-header */
+	assert ( iob_len ( iobuf ) >= sizeof ( *pshdr ) );
+	pshdr = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *pshdr ) );
+	dest_ip.s_addr = pshdr->dest_ip;
+	d_port = pshdr->d_port;
 	DBG ( "PXENV_UDP_READ" );
 
 	/* Filter on destination address and/or port */
@@ -386,14 +418,29 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	     ( dest_ip_wanted.s_addr != dest_ip.s_addr ) ) {
 		DBG ( " wrong IP %s", inet_ntoa ( dest_ip ) );
 		DBG ( " (wanted %s)\n", inet_ntoa ( dest_ip_wanted ) );
-		goto no_packet;
+		goto drop;
 	}
 	if ( d_port_wanted && ( d_port_wanted != d_port ) ) {
 		DBG ( " wrong port %d", htons ( d_port ) );
 		DBG ( " (wanted %d)\n", htons ( d_port_wanted ) );
-		goto no_packet;
+		goto drop;
 	}
 
+	/* Copy packet to buffer and record length */
+	buffer = real_to_user ( pxenv_udp_read->buffer.segment,
+				pxenv_udp_read->buffer.offset );
+	len = iob_len ( iobuf );
+	if ( len > pxenv_udp_read->buffer_size )
+		len = pxenv_udp_read->buffer_size;
+	copy_to_user ( buffer, 0, iobuf->data, len );
+	pxenv_udp_read->buffer_size = len;
+
+	/* Fill in source/dest information */
+	pxenv_udp_read->src_ip = pshdr->src_ip;
+	pxenv_udp_read->s_port = pshdr->s_port;
+	pxenv_udp_read->dest_ip = pshdr->dest_ip;
+	pxenv_udp_read->d_port = pshdr->d_port;
+
 	DBG ( " %04x:%04x+%x %s:", pxenv_udp_read->buffer.segment,
 	      pxenv_udp_read->buffer.offset, pxenv_udp_read->buffer_size,
 	      inet_ntoa ( *( ( struct in_addr * ) &pxenv_udp_read->src_ip ) ));
@@ -401,9 +448,14 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) {
 	      inet_ntoa ( *( ( struct in_addr * ) &pxenv_udp_read->dest_ip ) ),
 	      ntohs ( pxenv_udp_read->d_port ) );
 
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
 	pxenv_udp_read->Status = PXENV_STATUS_SUCCESS;
 	return PXENV_EXIT_SUCCESS;
 
+ drop:
+	free_iob ( iobuf );
  no_packet:
 	pxenv_udp_read->Status = PXENV_STATUS_FAILURE;
 	return PXENV_EXIT_FAILURE;
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
index 29e586ed2..2eb68178a 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/pxe/pxe_undi.c
@@ -21,9 +21,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c b/qemu/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
index 1854501de..69d94c407 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/syslinux/comboot_call.c
@@ -41,8 +41,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/image.h>
 #include <ipxe/version.h>
 #include <usr/imgmgmt.h>
-#include "config/console.h"
-#include "config/serial.h"
 
 /** The "SYSLINUX" version string */
 static char __bss16_array ( syslinux_version, [32] );
@@ -86,7 +84,6 @@ rmjmp_buf comboot_return;
 /* Mode flags set by INT 22h AX=0017h */
 static uint16_t comboot_graphics_mode = 0;
 
-
 /**
  * Print a string with a particular terminator
  */
@@ -261,8 +258,10 @@ static __asmcall void int21 ( struct i386_all_regs *ix86 ) {
 		break;
 
 	case 0x04: /* Write Character to Serial Port */
-		serial_putc ( ix86->regs.dl );
-		ix86->flags &= ~CF;
+		if ( serial_console.base ) {
+			uart_transmit ( &serial_console, ix86->regs.dl );
+			ix86->flags &= ~CF;
+		}
 		break;
 
 	case 0x09: /* Write DOS String to Console */
@@ -455,15 +454,16 @@ static __asmcall void int22 ( struct i386_all_regs *ix86 ) {
 		break;
 
 	case 0x000B: /* Get Serial Console Configuration */
-#if defined(CONSOLE_SERIAL) && !defined(COMPRESERVE)
-		ix86->regs.dx = COMCONSOLE;
-		ix86->regs.cx = 115200 / COMSPEED;
-		ix86->regs.bx = 0;
-#else
-		ix86->regs.dx = 0;
-#endif
+		if ( serial_console.base ) {
+			ix86->regs.dx = ( ( intptr_t ) serial_console.base );
+			ix86->regs.cx = serial_console.divisor;
+			ix86->regs.bx = 0;
+			ix86->flags &= ~CF;
+		}
+		break;
 
-		ix86->flags &= ~CF;
+	case 0x000C: /* Perform final cleanup */
+		shutdown_boot();
 		break;
 
 	case 0x000E: /* Get configuration file name */
@@ -712,3 +712,6 @@ void unhook_comboot_interrupts ( ) {
 	unhook_bios_interrupt ( 0x22, ( unsigned int ) int22_wrapper,
 				&int22_vector );
 }
+
+/* Avoid dragging in serial console support unconditionally */
+struct uart serial_console __attribute__ (( weak ));
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c b/qemu/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
index 390fc5545..ef7ee8151 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/vmware/guestrpc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c b/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
index c6b9fff12..f7df4f75b 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmconsole.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmware.c b/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
index 8074e6118..a415465fb 100644
--- a/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
+++ b/qemu/roms/ipxe/src/arch/i386/interface/vmware/vmware.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/bootpart.S b/qemu/roms/ipxe/src/arch/i386/prefix/bootpart.S
index 968da1a38..6d0c6034a 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/bootpart.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/bootpart.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BOOT_SEG	0x07c0
 #define EXEC_SEG	0x0100
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/exeprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/exeprefix.S
index cb61287d3..5c648d51d 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/exeprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/exeprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Initial temporary stack size */
 #define EXE_STACK_SIZE 0x400
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/hdprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/hdprefix.S
index 876bfe1be..1d012d80b 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/hdprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/hdprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.arch i386
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/isaromprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
index e28208089..fb49819ee 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/isaromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BUSTYPE "ISAR"
 #define _rom_start _isarom_start
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
index 27ed231e7..6e43cd26a 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/kkkpxeprefix.S
@@ -5,12 +5,10 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/* Since we have the whole stack, we can use cached DHCP information */
-REQUIRE_OBJECT ( pxeparent_dhcp )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Provide the PXENV_FILE_EXIT_HOOK API call */
+REQUIRING_SYMBOL ( _kkkpxe_start )
 REQUIRE_OBJECT ( pxe_exit_hook )
 
 #define PXELOADER_KEEP_UNDI
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
index d177d7d62..3c17dbdb1 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/kkpxeprefix.S
@@ -3,10 +3,7 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/* Since we have the whole stack, we can use cached DHCP information */
-REQUEST_OBJECT ( pxeparent_dhcp )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXELOADER_KEEP_UNDI
 #define PXELOADER_KEEP_PXE
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
index c75608172..200006d83 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/kpxeprefix.S
@@ -3,7 +3,7 @@
  *****************************************************************************
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXELOADER_KEEP_UNDI
 #define _pxe_start _kpxe_start
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/libprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/libprefix.S
index 7c1ece791..7d5c1ed53 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/libprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/libprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
@@ -296,11 +300,9 @@ copy_bytes:
  * Zero bytes
  *
  * Parameters:
- *   %ds:esi : source address
  *   %es:edi : destination address
  *   %ecx : length
  * Returns:
- *   %ds:esi : next source address
  *   %es:edi : next destination address
  * Corrupts:
  *   None
@@ -396,8 +398,10 @@ process_bytes:
 	movw	%ax, %fs
 	movw	%ax, %gs
 
+#ifdef NDEBUG
 	/* Call memcpy()-like function */
 	call	*%bx
+#endif
 
 	/* Return to (flat) real mode */
 	movl	%cr0, %eax
@@ -411,6 +415,20 @@ process_bytes:
 	popw	%fs
 	popw	%gs
 
+#ifndef NDEBUG
+	/* Call memcpy()-like function in flat real mode (to allow for
+	 * debug output via INT 10).
+	 */
+	pushw	%ds
+	pushw	%es
+	xorw	%ax, %ax
+	movw	%ax, %ds
+	movw	%ax, %es
+	call	*%bx
+	popw	%es
+	popw	%ds
+#endif
+
 	/* Restore GDT */
 	data32 lgdt -8(%bp)
 	addw	$( 8 /* saved GDT */ + ( PM_DS + 8 ) /* GDT on stack */ ), %sp
@@ -442,11 +460,11 @@ process_bytes:
 
 	/* Convert %ds:esi and %es:edi back to physical addresses */
 	xorl	%eax, %eax
-	movw	%ds, %cx
+	movw    %ds, %ax
 	shll	$4, %eax
 	addl	%eax, %esi
 	xorl	%eax, %eax
-	movw	%es, %cx
+	movw    %es, %ax
 	shll	$4, %eax
 	addl	%eax, %edi
 
@@ -678,12 +696,21 @@ install:
 	.globl install_prealloc
 install_prealloc:
 	progress "install_prealloc:\n"
-	/* Save registers */
+	/* Save registers on external stack */
 	pushal
 	pushw	%ds
 	pushw	%es
 	cld			/* Sanity: clear the direction flag asap */
 
+	/* Switch to temporary stack in .bss16 */
+	pushw	%ss
+	popw	%ds
+	movl	%esp, %ecx
+	movw	%bx, %ss
+	movl	$_data16_memsz, %esp
+	pushw	%ds
+	pushl	%ecx
+
 	/* Set up %ds for (read-only) access to .prefix */
 	pushw	%cs
 	popw	%ds
@@ -710,6 +737,7 @@ install_prealloc:
 	popl	%esi
 
 #ifndef KEEP_IT_REAL
+
 	/* Access high memory by enabling the A20 gate.  (We will
 	 * already have 4GB segment limits as a result of calling
 	 * install_block.)
@@ -778,7 +806,7 @@ payload_death_message:
 	movzwl	%bx, %edi
 	shll	$4, %edi
 	movl	$_data16_filesz, %ecx
-	movl	$_data16_memsz, %edx
+	movl	$_data16_filesz, %edx	/* do not zero our temporary stack */
 	call	install_block		/* .data16 */
 
 	/* Set up %ds for access to .data16 */
@@ -787,11 +815,8 @@ payload_death_message:
 	/* Restore decompression temporary area physical address */
 	popl	%edi
 
-#ifdef KEEP_IT_REAL
-	/* Initialise libkir */
-	movw	%ax, (init_libkir_vector+2)
-	lcall	*init_libkir_vector
-#else
+#ifndef KEEP_IT_REAL
+
 	/* Find a suitable decompression temporary area, if none specified */
 	pushl	%eax
 	testl	%edi, %edi
@@ -823,6 +848,22 @@ payload_death_message:
 	call	install_block
 	popl	%edi
 
+#endif /* KEEP_IT_REAL */
+
+	/* Switch back to original stack and zero .bss16 */
+	addr32 lss %ss:(%esp), %esp
+	pushl	%edi
+	pushw	%es
+	movw	%bx, %es
+	movl	$_data16_filesz, %edi
+	movl	$_data16_memsz, %ecx
+	subl	%edi, %ecx
+	call	zero_bytes
+	popw	%es
+	popl	%edi
+
+#ifndef KEEP_IT_REAL
+
 	/* Initialise librm at current location */
 	progress "  init_librm\n"
 	movw	%ax, (init_librm_vector+2)
@@ -834,7 +875,6 @@ payload_death_message:
 	incb	memmap_post
 	decl	%ebp
 1:
-
 	/* Call relocate() to determine target address for relocation.
 	 * relocate() will return with %esi, %edi and %ecx set up
 	 * ready for the copy to the new location.
@@ -857,7 +897,14 @@ payload_death_message:
 	/* Initialise librm at new location */
 	progress "  init_librm\n"
 	lcall	*init_librm_vector
-#endif
+
+#else /* KEEP_IT_REAL */
+
+	/* Initialise libkir */
+	movw	%ax, (init_libkir_vector+2)
+	lcall	*init_libkir_vector
+
+#endif /* KEEP_IT_REAL */
 
 	/* Close access to payload */
 	progress "  close_payload\n"
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
index 259bc6ba5..64135e14b 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/lkrnprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL_ANY )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BZI_LOAD_HIGH_ADDR 0x100000
 
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/mbr.S b/qemu/roms/ipxe/src/arch/i386/prefix/mbr.S
index adfe20410..a1e237de8 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/mbr.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/mbr.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/mromprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/mromprefix.S
index 4c94457c2..b636b92af 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/mromprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/mromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PCIBIOS_READ_CONFIG_WORD	0xb109
 #define PCIBIOS_READ_CONFIG_DWORD	0xb10a
@@ -463,6 +467,7 @@ pci_set_mem_access:
 	.org	0x00
 mromheader:
 	.word	0xaa55			/* BIOS extension signature */
+	.byte	0x01			/* Dummy size (BIOS bug workaround) */
 	.org	0x18
 	.word	mpciheader
 	.org	0x1a
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/nbiprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
index 06e7df5b7..16c79566c 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/nbiprefix.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.code16
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/nullprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/nullprefix.S
index 032d41e0f..bd0ff339e 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/nullprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/nullprefix.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.org	0
 	.text
 	.arch i386
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/pciromprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
index 45ba31f50..5a5a49647 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/pciromprefix.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define BUSTYPE "PCIR"
 #define _rom_start _pcirom_start
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/pxeprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
index 6e29c7949..465ce4345 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/pxeprefix.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define PXENV_UNDI_SHUTDOWN		0x0005
 #define	PXENV_UNDI_GET_NIC_TYPE		0x0012
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/romprefix.S b/qemu/roms/ipxe/src/arch/i386/prefix/romprefix.S
index 7bc4fe8cd..18dda2b37 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/romprefix.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/romprefix.S
@@ -6,9 +6,10 @@
  * table so using a noticeable amount of stack space is a no-no.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #include <config/general.h>
+#include <config/branding.h>
 
 #define PNP_SIGNATURE ( '$' + ( 'P' << 8 ) + ( 'n' << 16 ) + ( 'P' << 24 ) )
 #define PMM_SIGNATURE ( '$' + ( 'P' << 8 ) + ( 'M' << 16 ) + ( 'M' << 24 ) )
@@ -90,7 +91,7 @@ pciheader:
 	.ascii	"PCIR"			/* Signature */
 	.word	pci_vendor_id		/* Vendor identification */ 
 	.word	pci_device_id		/* Device identification */
-	.word	0x0000			/* Device list pointer */
+	.word	( pci_devlist - pciheader ) /* Device list pointer */
 	.word	pciheader_len		/* PCI data structure length */
 	.byte	0x03			/* PCI data structure revision */
 	.byte	0x02, 0x00, 0x00	/* Class code */
@@ -106,6 +107,17 @@ pciheader_runtime_length:
 	.equ pciheader_len, . - pciheader
 	.size pciheader, . - pciheader
 
+	/* PCI additional device list (filled in by linker) */
+	.section ".pci_devlist.00000000", "a", @progbits
+pci_devlist:
+	.previous
+	.section ".pci_devlist.ffffffff", "a", @progbits
+pci_devlist_end:
+	.short	0x0000 /* List terminator */
+	.previous
+	/* Ensure that terminator is always present */
+	.reloc pciheader, RELOC_TYPE_NONE, pci_devlist_end
+
 	.section ".zinfo.fixup", "a", @progbits	/* Compressor fixups */
 	.ascii	ZINFO_TYPE_ADxW
 	.long	pciheader_image_length
@@ -573,7 +585,7 @@ get_pmm_decompress_to:
  * Note to hardware vendors:
  *
  * If you wish to brand this boot ROM, please do so by defining the
- * strings PRODUCT_NAME and PRODUCT_SHORT_NAME in config/general.h.
+ * strings PRODUCT_NAME and PRODUCT_SHORT_NAME in config/branding.h.
  *
  * While nothing in the GPL prevents you from removing all references
  * to iPXE or http://ipxe.org, we prefer you not to do so.
@@ -589,7 +601,10 @@ init_message:
 	.ascii	"\n"
 	.ascii	PRODUCT_NAME
 	.ascii	"\n"
-	.asciz	"iPXE (http://ipxe.org)"
+	.ascii	PRODUCT_SHORT_NAME
+	.ascii	" ("
+	.ascii	PRODUCT_URI
+	.asciz	")"
 	.size	init_message, . - init_message
 .ifeqs	BUSTYPE, "PCIR"
 init_message_pci:
@@ -771,7 +786,9 @@ exec:	/* Set %ds = %cs */
 
 	/* Store PCI bus:dev.fn, if applicable */
 .ifeqs	BUSTYPE, "PCIR"
+#ifdef AUTOBOOT_ROM_FILTER
 	movw	%ax, autoboot_busdevfn
+#endif /* AUTOBOOT_ROM_FILTER */
 .endif
 
 	/* Call main() */
@@ -870,3 +887,9 @@ wait_for_tick:
 	popl	%eax
 	ret
 	.size wait_for_tick, . - wait_for_tick
+
+/* Drag in objects via _rom_start */
+REQUIRING_SYMBOL ( _rom_start )
+
+/* Drag in ROM configuration */
+REQUIRE_OBJECT ( config_romprefix )
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/undiloader.S b/qemu/roms/ipxe/src/arch/i386/prefix/undiloader.S
index 74bb59041..5cace44b7 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/undiloader.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/undiloader.S
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.text
 	.code16
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/unlzma.S b/qemu/roms/ipxe/src/arch/i386/prefix/unlzma.S
new file mode 100644
index 000000000..8d4b3c1a8
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/unlzma.S
@@ -0,0 +1,942 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/****************************************************************************
+ *
+ * This file provides the decompress() and decompress16() functions
+ * which can be called in order to decompress an LZMA-compressed
+ * image.  The code is modelled on the public-domain "XZ Embedded"
+ * implementation as used by the Linux kernel.  Symbol names are
+ * chosen to match the XZ Embedded implementation where possible, for
+ * ease of reference.
+ *
+ * This code is optimised for size rather than speed, since the amount
+ * of data to be decompressed is trivially small by modern standards.
+ *
+ * The same basic assembly code is used to compile both decompress()
+ * and decompress16().
+ *
+ * Note that these functions require large amounts of stack space.
+ *
+ ****************************************************************************
+ */
+
+	.text
+	.arch i586
+	.section ".prefix.lib", "ax", @progbits
+
+#ifdef CODE16
+#define ADDR16
+#define ADDR32 addr32
+#define decompress decompress16
+	.code16
+#else /* CODE16 */
+#define ADDR16 addr16
+#define ADDR32
+	.code32
+#endif /* CODE16 */
+
+/****************************************************************************
+ * Debugging
+ ****************************************************************************
+ *
+ * This code will usually run in 16-bit protected mode, in which case
+ * only the 0xe9 debug port (present on some virtual machines) can be
+ * used.
+ *
+ * To debug on real hardware, build with DEBUG=libprefix.  This will
+ * cause this code to be called in flat real mode, and so DEBUG_INT10
+ * may be used.
+ */
+
+/* Enable debugging via 0xe9 debug port */
+#define DEBUG_E9 0
+
+/* Enable debugging via BIOS INT 10 (works only when in flat real mode) */
+#define DEBUG_INT10 0
+
+#if ( DEBUG_E9 || DEBUG_INT10 )
+	.macro	print_character, reg
+	pushfl
+	pushw	%ax
+	pushw	%bx
+	pushw	%bp
+	movb 	\reg, %al
+	movw	$0x0007, %bx
+	movb	$0x0e, %ah
+#if DEBUG_E9
+	outb	%al, $0xe9
+#endif
+#if DEBUG_INT10
+	cmpb	$('\n'), %al
+	jne	L\@
+	int	$0x10
+	movb	$('\r'), %al
+L\@:	int	$0x10
+#endif
+	popw	%bp
+	popw	%bx
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_nibble
+	pushfl
+	pushw	%ax
+	cmpb	$10, %al
+	sbb	$0x69, %al
+	das
+	print_character %al
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_byte, reg
+	pushfl
+	pushw	%ax
+	movb	\reg, %al
+	pushw	%ax
+	shrb	$4, %al
+	print_hex_nibble
+	popw	%ax
+	andb	$0x0f, %al
+	print_hex_nibble
+	popw	%ax
+	popfl
+	.endm
+
+	.macro	print_hex_word, reg
+	pushw	%ax
+	movw	\reg, %ax
+	print_hex_byte %ah
+	print_hex_byte %al
+	popw	%ax
+	.endm
+
+	.macro	print_hex_dword, reg
+	pushl	%eax
+	movl	\reg, %eax
+	rorl	$16, %eax
+	print_hex_word %ax
+	rorl	$16, %eax
+	print_hex_word %ax
+	popl	%eax
+	.endm
+#else
+	.macro	print_character, char
+	.endm
+	.macro	print_hex_byte, reg
+	.endm
+	.macro	print_hex_word, reg
+	.endm
+	.macro	print_hex_dword, reg
+	.endm
+#endif
+
+/****************************************************************************
+ * LZMA parameters and data structures
+ ****************************************************************************
+ */
+
+/* LZMA decompressor states (as used in XZ Embedded) */
+#define STATE_LIT_LIT 0x00
+#define STATE_MATCH_LIT_LIT 0x01
+#define STATE_REP_LIT_LIT 0x02
+#define STATE_SHORTREP_LIT_LIT 0x03
+#define STATE_MATCH_LIT 0x04
+#define STATE_REP_LIT 0x05
+#define STATE_SHORTREP_LIT 0x06
+#define STATE_LIT_MATCH 0x07
+#define STATE_LIT_LONGREP 0x08
+#define STATE_LIT_SHORTREP 0x09
+#define STATE_NONLIT_MATCH 0x0a
+#define STATE_NONLIT_REP 0x0b
+
+/* LZMA maximum decompressor state in which most recent symbol was a literal */
+#define STATE_LIT_MAX 0x06
+
+/* LZMA number of literal context bits ("lc=" parameter) */
+#define LZMA_LC 2
+
+	.struct	0
+lzma_len_dec:
+choice:		.word	0
+choice2:	.word	0
+low:		.rept	( 1 << 3 )
+		.word	0
+		.endr
+mid:		.rept	( 1 << 3 )
+		.word	0
+		.endr
+high:		.rept	( 1 << 8 )
+		.word	0
+		.endr
+	.equ	sizeof__lzma_len_dec, . - lzma_len_dec
+	.previous
+
+	.struct	0
+lzma_dec:
+out_start:	.long	0
+rc_code:	.long	0
+rc_range:	.long	0
+len:		.word	0
+reps:
+rep0:		.long	0
+rep1:		.long	0
+rep2:		.long	0
+rep3:		.long	0
+probs:
+is_match:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep:		.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep0:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep1:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep2:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+is_rep0_long:	.word	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+dist_slot:	.rept	( 4 * ( 1 << 6 ) )
+		.word	0
+		.endr
+dist_special:	.rept	( ( 1 << ( 14 / 2 ) ) - 14 )
+		.word	0
+		.endr
+dist_align:	.rept	( 1 << 4 )
+		.word	0
+		.endr
+match_len_dec:	.space	sizeof__lzma_len_dec
+rep_len_dec:	.space	sizeof__lzma_len_dec
+literal:	.rept	( ( 1 << LZMA_LC ) * 0x300 )
+		.word	0
+		.endr
+	.align	4
+	.equ	sizeof__lzma_dec, . - lzma_dec
+	.previous
+
+	/* Some binutils versions seem not to handle .struct/.previous */
+	.section ".prefix.lib", "ax", @progbits
+
+/*****************************************************************************
+ * Normalise range encoder
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : current range
+ *****************************************************************************
+ */
+rc_normalise:
+	/* Check if rc_range is less than 1<<24 */
+	testb	$0xff, (rc_range+3)(%ebp)
+	jnz	1f
+	/* If it is, shift in a new byte from the compressed input data */
+	shll	$8, rc_range(%ebp)
+	shll	$8, rc_code(%ebp)
+	ADDR32 lodsb
+	movb	%al, (rc_code+0)(%ebp)
+1:	/* Return current range */
+	movl	rc_range(%ebp), %eax
+	ret
+	.size	rc_normalise, . - rc_normalise
+
+/*****************************************************************************
+ * Decode single range-encoded bit using a probability estimate
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate pointer (offset from %ebp)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   CF : decoded bit
+ *   ZF : inverse of decoded bit
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bit:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%edx
+	/* Perform normalisation */
+	call	rc_normalise
+	/* Calculate bound in %eax and probability estimate in %dx */
+	shrl	$11, %eax
+	movzwl	(%ebp,%ebx), %edx
+	mul	%edx /* will zero %edx */
+	movw	(%ebp,%ebx), %dx
+	/* Compare code against bound */
+	cmpl	%eax, rc_code(%ebp)
+	jae	2f
+1:	/* Code is less than bound */
+	movl	%eax, rc_range(%ebp)
+	negw	%dx
+	addw	$(1<<11), %dx
+	shrw	$5, %dx
+	addw	%dx, (%ebp,%ebx)
+	xorw	%ax, %ax	/* Clear CF, set ZF */
+	jmp	99f
+2:	/* Code is greater than or equal to bound */
+	subl	%eax, rc_range(%ebp)
+	subl	%eax, rc_code(%ebp)
+	shrw	$5, %dx
+	subw	%dx, (%ebp,%ebx)
+	incw	%dx		/* Clear ZF (%dx is 11-bit; can never wrap) */
+	stc			/* Set CF */
+99:	/* Restore registers and return */
+	popl	%edx
+	popl	%eax
+	ret
+	.size	rc_bit, . - rc_bit
+
+/*****************************************************************************
+ * Decode MSB-first bittree
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree:
+	/* Preserve registers */
+	pushl	%edi
+	pushw	%cx
+	movl	%ebx, %edi
+	/* Initialise registers */
+	movl	$1, %eax
+1:	/* Decode bit */
+	leaw	(%edi,%eax,2), %bx	/* high word always zero anyway */
+	call	rc_bit
+	rclw	%ax
+	ADDR16 loop 1b
+	/* Restore registers, clear unwanted high bit of result, and return */
+	movl	%edi, %ebx
+	popw	%cx
+	popl	%edi
+	btrw	%cx, %ax
+	ret
+	.size	rc_bittree, . - rc_bittree
+
+/*****************************************************************************
+ * Decode LSB-first bittree
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree_reverse:
+	/* Preserve registers */
+	pushw	%cx
+	/* Decode bittree */
+	call	rc_bittree
+1:	/* Reverse result */
+	rcrb	%al
+	rclb	%ah
+	ADDR16 loop 1b
+	shrw	$8, %ax
+	/* Restore registers and return */
+	popw	%cx
+	ret
+	.size	rc_bittree_reverse, . - rc_bittree_reverse
+
+/*****************************************************************************
+ * Decode MSB-first bittree with optional match byte
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : probability estimate set pointer (offset from %ebp)
+ *   %cl : match byte
+ *   %ch : 1 to use match byte, 0 to ignore match byte
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bittree
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_bittree_match:
+	/* Preserve registers */
+	pushl	%edi
+	pushw	%cx
+	pushw	%dx
+	movl	%ebx, %edi
+	/* Initialise registers */
+	movl	$1, %eax
+1:	/* Decode bit */
+	rolb	$1, %cl
+	movw	%cx, %dx
+	andb	%dh, %dl		/* match_bit in %dl */
+	movw	%dx, %bx
+	addb	%bl, %bh
+	xorb	%bl, %bl
+	addw	%ax, %bx		/* offset + match_bit + symbol */
+	leaw	(%edi,%ebx,2), %bx	/* high word always zero anyway */
+	call	rc_bit
+	rclw	%ax
+	movb	%al, %dh
+	notb	%dh
+	xorb	%dh, %dl
+	andb	%dl, %ch		/* offset &= ( match_bit ^ bit ) */
+	testb	%ah, %ah
+	jz	1b
+	/* Restore registers, clear unwanted high bit of result, and return */
+	movl	%edi, %ebx
+	popw	%dx
+	popw	%cx
+	popl	%edi
+	xorb	%ah, %ah
+	ret
+	.size	rc_bittree_match, . - rc_bittree_match
+
+/*****************************************************************************
+ * Decode direct bits (no probability estimates)
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %cx : number of bits to decode
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %eax : decoded bits
+ * Corrupts:
+ *   none
+ *****************************************************************************
+ */
+rc_direct:
+	/* Preserve registers */
+	pushl	%ebx
+	pushw	%cx
+	pushl	%edx
+	/* Initialise registers */
+	xorl	%edx, %edx
+1:	/* Perform normalisation */
+	call	rc_normalise
+	/* Decode bit */
+	shrl	$1, %eax
+	movl	%eax, rc_range(%ebp)
+	movl	rc_code(%ebp), %ebx
+	subl	%eax, %ebx
+	js	2f
+	movl	%ebx, rc_code(%ebp)
+2:	rcll	%ebx
+	rcll	%edx
+	xorb	$1, %dl
+	ADDR16 loop 1b
+	/* Restore registers and return */
+	movl	%edx, %eax
+	popl	%edx
+	popw	%cx
+	popl	%ebx
+	ret
+	.size	rc_direct, . - rc_direct
+
+/*****************************************************************************
+ * Decode an LZMA literal
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer (updated)
+ *   %edx : LZMA state
+ *   CF : end of payload marker found (always zero)
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Literals are coded as an eight-bit tree, using a match byte if the
+ * previous symbol was not a literal.
+ *
+ */
+lzma_literal:
+	/* Get most recent output byte, if available */
+	xorl	%ebx, %ebx
+	cmpl	%edi, out_start(%ebp)
+	je	1f
+	movb	%es:-1(%edi), %bh
+1:	/* Locate probability estimate set */
+	shrb	$( 8 - LZMA_LC ), %bh
+	shlb	$1, %bh
+	leaw	literal(%ebx,%ebx,2), %bx
+	/* Get match byte, if applicable */
+	xorw	%cx, %cx
+	cmpb	$STATE_LIT_MAX, %dl
+	jbe	1f
+	movl	rep0(%ebp), %eax
+	notl	%eax
+	movb	%es:(%edi,%eax), %cl
+	movb	$1, %ch
+1:	/* Decode bittree */
+	call	rc_bittree_match
+	/* Store output byte */
+	ADDR32 stosb
+	print_hex_byte %al
+	print_character $(' ')
+	/* Update LZMA state */
+	subb	$3, %dl
+	jns	1f
+	xorb	%dl, %dl
+1:	cmpb	$7, %dl
+	jb	1f
+	subb	$3, %dl
+1:	/* Clear CF and return */
+	clc
+	ret
+	.size	lzma_literal, . - lzma_literal
+
+/*****************************************************************************
+ * Decode an LZMA length
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %ebx : length parameter pointer (offset from %ebp)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ * Corrupts:
+ *   %ebx
+ *****************************************************************************
+ *
+ * Lengths are encoded as:
+ *
+ *   "0" + 3 bits    : lengths 2-9 ("low")
+ *   "10" + 3 bits   : lengths 10-17 ("mid")
+ *   "11" + 8 bits   : lengths 18-273 ("high")
+ */
+lzma_len:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%ecx
+	pushl	%edi
+	movl	%ebx, %edi
+	/* Start by assuming three bits and a base length of 2 */
+	movw	$3, %cx
+	movw	$2, len(%ebp)
+	/* Check low-length choice bit */
+	leal	choice(%edi), %ebx
+	call	rc_bit
+	leal	low(%edi), %ebx
+	jz	1f
+	/* Check high-length choice bit */
+	leal	choice2(%edi), %ebx
+	call	rc_bit
+	leal	mid(%edi), %ebx
+	movb	$10, len(%ebp)
+	jz	1f
+	leal	high(%edi), %ebx
+	movb	$8, %cl
+	movb	$18, len(%ebp)
+1:	/* Get encoded length */
+	call	rc_bittree
+	addw	%ax, len(%ebp)
+	/* Restore registers and return */
+	movl	%edi, %ebx
+	popl	%edi
+	popl	%ecx
+	popl	%eax
+	ret
+	.size	lzma_len, . - lzma_len
+
+/*****************************************************************************
+ * Copy (possibly repeated) matched data
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %cl : repeated match distance index (for repeated matches)
+ *   %eax : match distance (for non-repeated matches)
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   CF : match distance is out of range
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ */
+match:	/* Update repeated match list */
+	print_character $('[')
+	movl	$3, %ecx
+	jmp	1f
+match_rep:
+	print_character $('[')
+	print_character $('R')
+	print_hex_byte %cl
+	print_character $('=')
+	movzbl	%cl, %ecx
+	movl	reps(%ebp,%ecx,4), %eax
+	jcxz	2f
+1:	movl	(reps-4)(%ebp,%ecx,4), %ebx
+	movl	%ebx, reps(%ebp,%ecx,4)
+	loop	1b
+	movl	%eax, rep0(%ebp)
+2:	/* Preserve registers */
+	pushl	%esi
+	/* Get stored match length */
+	movzwl	len(%ebp), %ecx
+	print_hex_dword	%eax
+	print_character $('+')
+	print_hex_word %cx
+	print_character $(']')
+	print_character $(' ')
+	/* Abort with CF set if match distance is out of range */
+	movl	out_start(%ebp), %esi
+	negl	%esi
+	leal	-1(%edi,%esi), %esi
+	cmpl	%eax, %esi
+	jc	99f
+	/* Perform copy */
+	notl	%eax
+	leal	(%edi,%eax), %esi
+	ADDR32 es rep movsb
+99:	/* Restore registers and return */
+	popl	%esi
+	ret
+	.size	match, . - match
+
+/*****************************************************************************
+ * Decode an LZMA match
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Matches are encoded as an LZMA length followed by a 6-bit "distance
+ * slot" code, 0-26 fixed-probability bits, and 0-5 context encoded
+ * bits.
+ */
+lzma_match:
+	/* Preserve registers */
+	pushl	%edi
+	/* Update LZMA state */
+	cmpb	$STATE_LIT_MAX, %dl
+	movb	$STATE_LIT_MATCH, %dl
+	jbe	1f
+	movb	$STATE_NONLIT_MATCH, %dl
+1:	/* Decode length */
+	movl	$match_len_dec, %ebx
+	call	lzma_len
+	/* Decode distance slot */
+	movw	len(%ebp), %bx
+	subw	$2, %bx
+	cmpw	$4, %bx
+	jb	1f
+	movw	$3, %bx
+1:	shlw	$7, %bx
+	addw	$dist_slot, %bx
+	movw	$6, %cx
+	call	rc_bittree
+	/* Distance slots 0-3 are literal distances */
+	cmpb	$4, %al
+	jb	99f
+	/* Determine initial bits: 10/11 for even/odd distance codes */
+	movl	%eax, %edi
+	andw	$1, %di
+	orw	$2, %di
+	/* Determine number of context-encoded bits */
+	movw	%ax, %cx
+	shrb	$1, %cl
+	decb	%cl
+	/* Select context to be used in absence of fixed-probability bits */
+	movl	%edi, %ebx
+	shlw	%cl, %bx
+	subw	%ax, %bx
+	leaw	(dist_special-2)(%ebx,%ebx), %bx
+	/* Decode fixed-probability bits, if any */
+	cmpb	$6, %cl
+	jb	1f
+	subb	$4, %cl
+	shll	%cl, %edi
+	call	rc_direct
+	orl	%eax, %edi
+	/* Select context to be used in presence of fixed-probability bits */
+	movb	$4, %cl
+	movl	$dist_align, %ebx
+1:	/* Decode context-encoded bits */
+	shll	%cl, %edi
+	call	rc_bittree_reverse
+	orl	%edi, %eax
+99:	/* Restore registers and tail-call */
+	popl	%edi
+	jmp	match
+	.size	lzma_match, . - lzma_match
+
+/*****************************************************************************
+ * Decode an LZMA repeated match
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ *
+ * Repeated matches are encoded as:
+ *
+ *   "00"	 : shortrep0 (implicit length 1)
+ *   "01" + len  : longrep0
+ *   "10" + len  : longrep1
+ *   "110" + len : longrep2
+ *   "111" + len : longrep3
+ */
+lzma_rep_match:
+	/* Initially assume longrep0 */
+	movw	$(STATE_LIT_LONGREP << 8), %cx
+	/* Get is_rep0 bit */
+	leal	is_rep0(,%edx,2), %ebx
+	call	rc_bit
+	jnz	1f
+	/* Get is_rep0_long bit */
+	leal	is_rep0_long(,%edx,2), %ebx
+	call	rc_bit
+	jnz	98f
+	movw	$1, len(%ebp)
+	movb	$STATE_LIT_SHORTREP, %ch
+	jmp	99f
+1:	/* Get is_rep1 bit */
+	incb	%cl
+	leal	is_rep1(,%edx,2), %ebx
+	call	rc_bit
+	jz	98f
+	/* Get is_rep2 bit */
+	incb	%cl
+	leal	is_rep2(,%edx,2), %ebx
+	call	rc_bit
+	adcb	$0, %cl
+98:	/* Decode length */
+	movl	$rep_len_dec, %ebx
+	call	lzma_len
+99:	/* Update LZMA state */
+	cmpb	$STATE_LIT_MAX, %dl
+	movb	%ch, %dl
+	jbe	1f
+	movb	$STATE_NONLIT_REP, %dl
+1:	/* Tail call */
+	jmp	match_rep
+	.size	lzma_match, . - lzma_match
+
+/*****************************************************************************
+ * Decode one LZMA symbol
+ *
+ * Parameters:
+ *   %ss:%ebp : LZMA parameter block
+ *   %ds:%esi : compressed input data pointer
+ *   %es:%edi : uncompressed output data pointer
+ *   %edx : LZMA state
+ * Returns:
+ *   %ds:%esi : compressed input data pointer (possibly updated)
+ *   %es:%edi : uncompressed output data pointer (updated)
+ *   %edx : LZMA state
+ *   CF : end of payload marker found
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *****************************************************************************
+ */
+lzma_decode:
+	/* Get is_match bit */
+	leal	is_match(,%edx,2), %ebx
+	call	rc_bit
+	jz	lzma_literal
+	/* Get is_rep bit */
+	leal	is_rep(,%edx,2), %ebx
+	call	rc_bit
+	jz	lzma_match
+	jmp	lzma_rep_match
+	.size	lzma_decode, . - lzma_decode
+
+/****************************************************************************
+ * Undo effect of branch-call-jump (BCJ) filter
+ *
+ * Parameters:
+ *   %es:%esi : start of uncompressed output data (note %es)
+ *   %es:%edi : end of uncompressed output data
+ * Returns:
+ * Corrupts:
+ *   %eax
+ *   %ebx
+ *   %ecx
+ *   %edx
+ *   %esi
+ *****************************************************************************
+ */
+bcj_filter:
+	/* Store (negative) start of data in %edx */
+	movl	%esi, %edx
+	negl	%edx
+	/* Calculate limit in %ecx */
+	leal	-5(%edi,%edx), %ecx
+1:	/* Calculate offset in %ebx */
+	leal	(%esi,%edx), %ebx
+	/* Check for end of data */
+	cmpl	%ecx, %ebx
+	ja	99f
+	/* Check for an opcode which would be followed by a rel32 address */
+	ADDR32 es lodsb
+	andb	$0xfe, %al
+	cmpb	$0xe8, %al
+	jne	1b
+	/* Get current jump target value in %eax */
+	ADDR32 es lodsl
+	/* Convert absolute addresses in the range [0,limit) back to
+	 * relative addresses in the range [-offset,limit-offset).
+	 */
+	cmpl	%ecx, %eax
+	jae	2f
+	subl	%ebx,%es:-4(%esi)
+2:	/* Convert negative numbers in the range [-offset,0) back to
+	 * positive numbers in the range [limit-offset,limit).
+	 */
+	notl	%eax	/* Range is now [0,offset) */
+	cmpl	%ebx, %eax
+	jae	1b
+	addl	%ecx,%es:-4(%esi)
+	jmp	1b
+99:	/* Return */
+	ret
+	.size	bcj_filter, . - bcj_filter
+
+/****************************************************************************
+ * decompress (real-mode or 16/32-bit protected-mode near call)
+ *
+ * Decompress data
+ *
+ * Parameters (passed via registers):
+ *   %ds:%esi : Start of compressed input data
+ *   %es:%edi : Start of output buffer
+ * Returns:
+ *   %ds:%esi - End of compressed input data
+ *   %es:%edi - End of decompressed output data
+ *   All other registers are preserved
+ *
+ * NOTE: It would be possible to build a smaller version of the
+ * decompression code for -DKEEP_IT_REAL by using 16-bit registers
+ * where possible.
+ ****************************************************************************
+ */
+	.globl	decompress
+decompress:
+	/* Preserve registers */
+	pushl	%eax
+	pushl	%ebx
+	pushl	%ecx
+	pushl	%edx
+	pushl	%ebp
+	/* Allocate parameter block */
+	subl	$sizeof__lzma_dec, %esp
+	movl	%esp, %ebp
+	/* Zero parameter block and set all probabilities to 0.5 */
+	pushl	%edi
+	pushw	%es
+	pushw	%ss
+	popw	%es
+	movl	%ebp, %edi
+	xorl	%eax, %eax
+	movl	$( sizeof__lzma_dec / 4 ), %ecx
+	ADDR32 rep stosl
+	leal	probs(%ebp), %edi
+	movw	$( ( 1 << 11 ) / 2 ), %ax
+	movl	$( ( sizeof__lzma_dec - probs ) / 2 ), %ecx
+	ADDR32 rep stosw
+	popw	%es
+	popl	%edi
+	/* Initialise remaining parameters */
+	movl	%edi, out_start(%ebp)
+	print_character $('\n')
+	ADDR32 lodsb	/* discard initial byte */
+	print_hex_byte %al
+	ADDR32 lodsl
+	bswapl	%eax
+	print_hex_dword %eax
+	print_character $('\n')
+	movl	%eax, rc_code(%ebp)
+	decl	rc_range(%ebp)
+	movl	$STATE_LIT_LIT, %edx
+1:	/* Decompress until we reach end of buffer */
+	call	lzma_decode
+	jnc	1b
+	call	rc_normalise
+	print_character $('\n')
+	/* Undo BCJ filter */
+	pushl	%esi
+	movl	out_start(%ebp), %esi
+	call	bcj_filter
+	popl	%esi
+	/* Restore registers and return */
+	addl	$sizeof__lzma_dec, %esp
+	popl	%ebp
+	popl	%edx
+	popl	%ecx
+	popl	%ebx
+	popl	%eax
+	ret
+
+	/* Specify minimum amount of stack space required */
+	.globl	_min_decompress_stack
+	.equ	_min_decompress_stack, ( sizeof__lzma_dec + 512 /* margin */ )
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/unnrv2b16.S b/qemu/roms/ipxe/src/arch/i386/prefix/unlzma16.S
index b24c2846f..32b43f0dc 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/unnrv2b16.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/unlzma16.S
@@ -3,7 +3,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 #define CODE16
-#include "unnrv2b.S"
+#include "unlzma.S"
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/unnrv2b.S b/qemu/roms/ipxe/src/arch/i386/prefix/unnrv2b.S
deleted file mode 100644
index f5724c134..000000000
--- a/qemu/roms/ipxe/src/arch/i386/prefix/unnrv2b.S
+++ /dev/null
@@ -1,184 +0,0 @@
-/* 
- * Copyright (C) 1996-2002 Markus Franz Xaver Johannes Oberhumer
- *
- * This file is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * Originally this code was part of ucl the data compression library
- * for upx the ``Ultimate Packer of eXecutables''.
- *
- * - Converted to gas assembly, and refitted to work with etherboot.
- *   Eric Biederman 20 Aug 2002
- *
- * - Structure modified to be a subroutine call rather than an
- *   executable prefix.
- *   Michael Brown 30 Mar 2004
- *
- * - Modified to be compilable as either 16-bit or 32-bit code.
- *   Michael Brown 9 Mar 2005
- */
-
-FILE_LICENCE ( GPL2_OR_LATER )
-
-/****************************************************************************
- * This file provides the decompress() and decompress16() functions
- * which can be called in order to decompress an image compressed with
- * the nrv2b utility in src/util.
- *
- * These functions are designed to be called by the prefix.  They are
- * position-independent code.
- *
- * The same basic assembly code is used to compile both
- * decompress() and decompress16().
- ****************************************************************************
- */
-
-	.text
-	.arch i386
-	.section ".prefix.lib", "ax", @progbits
-
-#ifdef CODE16
-/****************************************************************************
- * decompress16 (real-mode near call, position independent)
- *
- * Decompress data in 16-bit mode
- *
- * Parameters (passed via registers):
- *   %ds:%esi - Start of compressed input data
- *   %es:%edi - Start of output buffer
- * Returns:
- *   %ds:%esi - End of compressed input data
- *   %es:%edi - End of decompressed output data
- *   All other registers are preserved
- *
- * NOTE: It would be possible to build a smaller version of the
- * decompression code for -DKEEP_IT_REAL by using
- *    #define REG(x) x
- * to use 16-bit registers where possible.  This would impose limits
- * that the compressed data size must be in the range [1,65533-%si]
- * and the uncompressed data size must be in the range [1,65536-%di]
- * (where %si and %di are the input values for those registers).  Note
- * particularly that the lower limit is 1, not 0, and that the upper
- * limit on the input (compressed) data really is 65533, since the
- * algorithm may read up to three bytes beyond the end of the input
- * data, since it reads dwords.
- ****************************************************************************
- */
-
-#define REG(x) e ## x
-#define ADDR32 addr32
-
-	.code16
-	.globl	decompress16
-decompress16:
-	
-#else /* CODE16 */
-
-/****************************************************************************
- * decompress (32-bit protected-mode near call, position independent)
- *
- * Parameters (passed via registers):
- *   %ds:%esi - Start of compressed input data
- *   %es:%edi - Start of output buffer
- * Returns:
- *   %ds:%esi - End of compressed input data
- *   %es:%edi - End of decompressed output data
- *   All other registers are preserved
- ****************************************************************************
- */
-
-#define REG(x) e ## x
-#define ADDR32
-	
-	.code32
-	.globl	decompress
-decompress:
-
-#endif /* CODE16 */
-
-#define xAX	REG(ax)
-#define xCX	REG(cx)
-#define xBP	REG(bp)
-#define xSI	REG(si)
-#define xDI	REG(di)
-
-	/* Save registers */
-	push	%xAX
-	pushl	%ebx
-	push	%xCX
-	push	%xBP
-	/* Do the decompression */
-	cld
-	xor	%xBP, %xBP
-	dec	%xBP		/* last_m_off = -1 */
-	jmp	dcl1_n2b
-	
-decompr_literals_n2b:
-	ADDR32 movsb
-decompr_loop_n2b:
-	addl	%ebx, %ebx
-	jnz	dcl2_n2b
-dcl1_n2b:
-	call	getbit32
-dcl2_n2b:
-	jc	decompr_literals_n2b
-	xor	%xAX, %xAX
-	inc	%xAX		/* m_off = 1 */
-loop1_n2b:
-	call	getbit1
-	adc	%xAX, %xAX	/* m_off = m_off*2 + getbit() */
-	call	getbit1
-	jnc	loop1_n2b	/* while(!getbit()) */
-	sub	$3, %xAX
-	jb	decompr_ebpeax_n2b	/* if (m_off == 2) goto decompr_ebpeax_n2b ? */
-	shl	$8, %xAX	
-	ADDR32 movb (%xSI), %al	/* m_off = (m_off - 3)*256 + src[ilen++] */
-	inc	%xSI
-	xor	$-1, %xAX
-	jz	decompr_end_n2b	/* if (m_off == 0xffffffff) goto decomp_end_n2b */
-	mov	%xAX, %xBP	/* last_m_off = m_off ?*/
-decompr_ebpeax_n2b:
-	xor	%xCX, %xCX
-	call	getbit1
-	adc	%xCX, %xCX	/* m_len = getbit() */
-	call	getbit1
-	adc	%xCX, %xCX	/* m_len = m_len*2 + getbit()) */
-	jnz	decompr_got_mlen_n2b	/* if (m_len == 0) goto decompr_got_mlen_n2b */
-	inc	%xCX		/* m_len++ */
-loop2_n2b:
-	call	getbit1	
-	adc	%xCX, %xCX	/* m_len = m_len*2 + getbit() */
-	call	getbit1
-	jnc	loop2_n2b	/* while(!getbit()) */
-	inc	%xCX
-	inc	%xCX		/* m_len += 2 */
-decompr_got_mlen_n2b:
-	cmp	$-0xd00, %xBP
-	adc	$1, %xCX	/* m_len = m_len + 1 + (last_m_off > 0xd00) */
-	push	%xSI
-	ADDR32 lea (%xBP,%xDI), %xSI	/* m_pos = dst + olen + -m_off  */
-	rep
-	es ADDR32 movsb		/* dst[olen++] = *m_pos++ while(m_len > 0) */
-	pop	%xSI
-	jmp	decompr_loop_n2b
-
-
-getbit1:
-	addl	%ebx, %ebx
-	jnz	1f
-getbit32:
-	ADDR32 movl (%xSI), %ebx
-	sub	$-4, %xSI	/* sets carry flag */
-	adcl	%ebx, %ebx
-1:
-	ret
-
-decompr_end_n2b:
-	/* Restore registers and return */
-	pop	%xBP
-	pop	%xCX
-	popl	%ebx
-	pop	%xAX
-	ret
diff --git a/qemu/roms/ipxe/src/arch/i386/prefix/usbdisk.S b/qemu/roms/ipxe/src/arch/i386/prefix/usbdisk.S
index fa7d1956e..9676406e2 100644
--- a/qemu/roms/ipxe/src/arch/i386/prefix/usbdisk.S
+++ b/qemu/roms/ipxe/src/arch/i386/prefix/usbdisk.S
@@ -1,3 +1,5 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
 	.text
 	.arch i386
 	.section ".prefix", "awx", @progbits
@@ -6,18 +8,27 @@
 
 #include "mbr.S"
 
-/* Partition table: ZIP-compatible partition 4, 64 heads, 32 sectors/track */
+/* Partition table: 64 heads, 32 sectors/track (ZIP-drive compatible) */
 	.org 446
 	.space 16
 	.space 16
-	.space 16
-	.byte 0x80, 0x01, 0x01, 0x00
-	.byte 0xeb, 0x3f, 0x20, 0x01
+	/* Partition 3: log partition (for CONSOLE_INT13) */
+	.byte 0x00, 0x01, 0x01, 0x00
+	.byte 0xe0, 0x3f, 0x20, 0x00
 	.long 0x00000020
-	.long 0x00000fe0
+	.long 0x000007e0
+	/* Partition 4: boot partition */
+	.byte 0x80, 0x00, 0x01, 0x01
+	.byte 0xeb, 0x3f, 0x20, 0x02
+	.long 0x00000800
+	.long 0x00001000
 
 	.org 510
 	.byte 0x55, 0xaa
 
-/* Skip to start of partition */
+/* Skip to start of log partition */
 	.org 32 * 512
+	.ascii "iPXE LOG\n\n"
+
+/* Skip to start of boot partition */
+	.org 2048 * 512
diff --git a/qemu/roms/ipxe/src/arch/i386/scripts/i386.lds b/qemu/roms/ipxe/src/arch/i386/scripts/i386.lds
index 98f95cb23..38c89e14b 100644
--- a/qemu/roms/ipxe/src/arch/i386/scripts/i386.lds
+++ b/qemu/roms/ipxe/src/arch/i386/scripts/i386.lds
@@ -27,6 +27,13 @@ SECTIONS {
     PROVIDE ( _max_align = 16 );
 
     /*
+     * Allow decompressor to require a minimum amount of temporary stack
+     * space.
+     *
+     */
+    PROVIDE ( _min_decompress_stack = 0 );
+
+    /*
      * The prefix
      *
      */
@@ -34,6 +41,7 @@ SECTIONS {
     .prefix 0x0 : AT ( _prefix_lma ) {
 	_prefix = .;
 	*(.prefix)
+	*(SORT(.pci_devlist.*))
 	*(.prefix.*)
 	_mprefix = .;
     } .bss.prefix (NOLOAD) : AT ( _end_lma ) {
@@ -87,6 +95,7 @@ SECTIONS {
 	*(.bss16.*)
 	*(.stack16)
 	*(.stack16.*)
+	. = MAX ( ., _mdata16 + _min_decompress_stack );
 	_edata16 = .;
     }
     _data16_filesz	= ABSOLUTE ( _mdata16 ) - ABSOLUTE ( _data16 );
diff --git a/qemu/roms/ipxe/src/arch/i386/transitions/liba20.S b/qemu/roms/ipxe/src/arch/i386/transitions/liba20.S
index 684697525..6c1e1f62f 100644
--- a/qemu/roms/ipxe/src/arch/i386/transitions/liba20.S
+++ b/qemu/roms/ipxe/src/arch/i386/transitions/liba20.S
@@ -16,9 +16,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
  *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 	.arch i386
 
diff --git a/qemu/roms/ipxe/src/arch/i386/transitions/libkir.S b/qemu/roms/ipxe/src/arch/i386/transitions/libkir.S
index 1176fcced..fa9459d52 100644
--- a/qemu/roms/ipxe/src/arch/i386/transitions/libkir.S
+++ b/qemu/roms/ipxe/src/arch/i386/transitions/libkir.S
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /****************************************************************************
  * This file defines libkir: an interface between external and
diff --git a/qemu/roms/ipxe/src/arch/i386/transitions/librm.S b/qemu/roms/ipxe/src/arch/i386/transitions/librm.S
index 2e447b030..863e22415 100644
--- a/qemu/roms/ipxe/src/arch/i386/transitions/librm.S
+++ b/qemu/roms/ipxe/src/arch/i386/transitions/librm.S
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER )
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
 
 /* Drag in local definitions */
 #include "librm.h"
diff --git a/qemu/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c b/qemu/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
index cc4765de2..becb02677 100644
--- a/qemu/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
+++ b/qemu/roms/ipxe/src/arch/i386/transitions/librm_mgmt.c
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/profile.h>
@@ -103,7 +103,7 @@ void init_idt ( void ) {
 				( uint32_t ) vec->next );
 		set_interrupt_vector ( intr, vec );
 	}
-	DBGC ( &intr_vec[0], "INTn vector at %p+%xn (phys %#lx+%xn)\n",
+	DBGC ( &intr_vec[0], "INTn vector at %p+%zxn (phys %#lx+%zxn)\n",
 	       intr_vec, sizeof ( intr_vec[0] ),
 	       virt_to_phys ( intr_vec ), sizeof ( intr_vec[0] ) );
 
diff --git a/qemu/roms/ipxe/src/arch/i386/transitions/librm_test.c b/qemu/roms/ipxe/src/arch/i386/transitions/librm_test.c
index e07cfccdd..f1a517eda 100644
--- a/qemu/roms/ipxe/src/arch/i386/transitions/librm_test.c
+++ b/qemu/roms/ipxe/src/arch/i386/transitions/librm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -114,4 +118,5 @@ struct self_test librm_test __self_test = {
 	.exec = librm_test_exec,
 };
 
+REQUIRING_SYMBOL ( librm_test );
 REQUIRE_OBJECT ( test );
diff --git a/qemu/roms/ipxe/src/arch/x86/Makefile b/qemu/roms/ipxe/src/arch/x86/Makefile
index e555587df..98c49b98d 100644
--- a/qemu/roms/ipxe/src/arch/x86/Makefile
+++ b/qemu/roms/ipxe/src/arch/x86/Makefile
@@ -9,9 +9,14 @@ SRCDIRS		+= arch/x86/interface/efi
 SRCDIRS		+= arch/x86/prefix
 SRCDIRS		+= arch/x86/hci/commands
 SRCDIRS		+= arch/x86/drivers/xen
+SRCDIRS		+= arch/x86/drivers/hyperv
 
 # breaks building some of the linux-related objects
 CFLAGS		+= -Ulinux
 
 # disable valgrind
 CFLAGS		+= -DNVALGRIND
+
+# Include Hyper-V driver in the all-drivers build
+#
+DRIVERS_hyperv += hyperv
diff --git a/qemu/roms/ipxe/src/arch/x86/Makefile.efi b/qemu/roms/ipxe/src/arch/x86/Makefile.efi
index 13a69d9f7..f73bc7d5d 100644
--- a/qemu/roms/ipxe/src/arch/x86/Makefile.efi
+++ b/qemu/roms/ipxe/src/arch/x86/Makefile.efi
@@ -17,7 +17,7 @@ NON_AUTO_MEDIA	+= efirom
 
 # Include SNP driver in the all-drivers build
 #
-DRIVERS += snp
+DRIVERS_net += snp
 
 # Rules for building EFI files
 #
diff --git a/qemu/roms/ipxe/src/arch/x86/core/cpuid.c b/qemu/roms/ipxe/src/arch/x86/core/cpuid.c
index 5908f4419..bc5a6c68c 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/cpuid.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/cpuid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/cpuid.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/core/cpuid_settings.c b/qemu/roms/ipxe/src/arch/x86/core/cpuid_settings.c
index 42dea9336..08bd3918a 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/cpuid_settings.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/cpuid_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/core/debugcon.c b/qemu/roms/ipxe/src/arch/x86/core/debugcon.c
index 263cb4af1..60de61f55 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/debugcon.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/debugcon.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/x86/core/pcidirect.c b/qemu/roms/ipxe/src/arch/x86/core/pcidirect.c
index dbc8317b8..9b8e6b1d9 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/pcidirect.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/pcidirect.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/io.h>
 #include <ipxe/pci.h>
diff --git a/qemu/roms/ipxe/src/arch/i386/core/pic8259.c b/qemu/roms/ipxe/src/arch/x86/core/pic8259.c
index 0a9ea2e03..0a9ea2e03 100644
--- a/qemu/roms/ipxe/src/arch/i386/core/pic8259.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/pic8259.c
diff --git a/qemu/roms/ipxe/src/arch/x86/core/pit8254.c b/qemu/roms/ipxe/src/arch/x86/core/pit8254.c
new file mode 100644
index 000000000..da2099263
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/core/pit8254.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/io.h>
+#include <ipxe/pit8254.h>
+
+/** @file
+ *
+ * 8254 Programmable Interval Timer
+ *
+ */
+
+/**
+ * Delay for a fixed number of timer ticks using the speaker channel
+ *
+ * @v ticks		Number of timer ticks for which to delay
+ */
+void pit8254_speaker_delay ( unsigned int ticks ) {
+	uint8_t spkr;
+	uint8_t cmd;
+	uint8_t low;
+	uint8_t high;
+
+	/* Sanity check */
+	assert ( ticks <= 0xffff );
+
+	/* Disable speaker, set speaker channel gate input high */
+	spkr = inb ( PIT8254_SPKR );
+	spkr &= ~PIT8254_SPKR_ENABLE;
+	spkr |= PIT8254_SPKR_GATE;
+	outb ( spkr, PIT8254_SPKR );
+
+	/* Program speaker channel to "interrupt" on terminal count */
+	cmd = ( PIT8254_CMD_CHANNEL ( PIT8254_CH_SPKR ) |
+		PIT8254_CMD_ACCESS_LOHI | PIT8254_CMD_OP_TERMINAL |
+		PIT8254_CMD_BINARY );
+	low = ( ( ticks >> 0 ) & 0xff );
+	high = ( ( ticks >> 8 ) & 0xff );
+	outb ( cmd, PIT8254_CMD );
+	outb ( low, PIT8254_DATA ( PIT8254_CH_SPKR ) );
+	outb ( high, PIT8254_DATA ( PIT8254_CH_SPKR ) );
+
+	/* Wait for channel to "interrupt" */
+	do {
+		spkr = inb ( PIT8254_SPKR );
+	} while ( ! ( spkr & PIT8254_SPKR_OUT ) );
+}
diff --git a/qemu/roms/ipxe/src/arch/x86/core/vram_settings.c b/qemu/roms/ipxe/src/arch/x86/core/vram_settings.c
new file mode 100644
index 000000000..9c169b40c
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/core/vram_settings.c
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/uaccess.h>
+#include <ipxe/settings.h>
+
+/** @file
+ *
+ * Video RAM dump
+ *
+ */
+
+/** Video RAM base address */
+#define VRAM_BASE 0xb8000
+
+/** Video RAM length */
+#define VRAM_LEN \
+	( 80 /* columns */ * 25 /* rows */ * 2 /* bytes per character */ )
+
+/**
+ * Fetch video RAM setting
+ *
+ * @v data		Buffer to fill with setting data
+ * @v len		Length of buffer
+ * @ret len		Length of setting data, or negative error
+ */
+static int vram_fetch ( void *data, size_t len ) {
+	userptr_t vram = phys_to_user ( VRAM_BASE );
+
+	/* Copy video RAM */
+	if ( len > VRAM_LEN )
+		len = VRAM_LEN;
+	copy_from_user ( data, vram, 0, len );
+
+	return VRAM_LEN;
+}
+
+/** Video RAM setting */
+const struct setting vram_setting __setting ( SETTING_MISC, vram ) = {
+	.name = "vram",
+	.description = "Video RAM",
+	.type = &setting_type_base64,
+	.scope = &builtin_scope,
+};
+
+/** Video RAM built-in setting */
+struct builtin_setting vram_builtin_setting __builtin_setting = {
+	.setting = &vram_setting,
+	.fetch = vram_fetch,
+};
diff --git a/qemu/roms/ipxe/src/arch/x86/core/x86_bigint.c b/qemu/roms/ipxe/src/arch/x86/core/x86_bigint.c
index 418ac2309..6413b2fa8 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/x86_bigint.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/x86_bigint.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/core/x86_io.c b/qemu/roms/ipxe/src/arch/x86/core/x86_io.c
index 9b2d2d935..3081fa8b9 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/x86_io.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/x86_io.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/io.h>
 #include <ipxe/x86_io.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/core/x86_string.c b/qemu/roms/ipxe/src/arch/x86/core/x86_string.c
index d48347c96..7d5e4a5f1 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/x86_string.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/x86_string.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 /** @file
@@ -23,7 +27,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 
@@ -104,87 +108,3 @@ void * __memmove ( void *dest, const void *src, size_t len ) {
 		return __memcpy_reverse ( dest, src, len );
 	}
 }
-
-/**
- * Swap memory areas
- *
- * @v dest		Destination address
- * @v src		Source address
- * @v len		Length
- * @ret dest		Destination address
- */
-void * memswap ( void *dest, void *src, size_t len ) {
-	size_t discard_c;
-	int discard;
-
-	__asm__ __volatile__ ( "\n1:\n\t"
-			       "dec %2\n\t"
-			       "js 2f\n\t"
-			       "movb (%0,%2), %b3\n\t"
-			       "xchgb (%1,%2), %b3\n\t"
-			       "movb %b3, (%0,%2)\n\t"
-			       "jmp 1b\n\t"
-			       "2:\n\t"
-			       : "=r" ( src ), "=r" ( dest ),
-				 "=&c" ( discard_c ), "=&q" ( discard )
-			       : "0" ( src ), "1" ( dest ), "2" ( len )
-			       : "memory" );
-
-	return dest;
-}
-
-/**
- * Calculate length of string
- *
- * @v string		String
- * @ret len		Length (excluding NUL)
- */
-size_t strlen ( const char *string ) {
-	const char *discard_D;
-	size_t len_plus_one;
-
-	__asm__ __volatile__ ( "repne scasb\n\t"
-			       "not %1\n\t"
-			       : "=&D" ( discard_D ), "=&c" ( len_plus_one )
-			       : "0" ( string ), "1" ( -1UL ), "a" ( 0 ) );
-
-	return ( len_plus_one - 1 );
-}
-
-/**
- * Compare strings (up to a specified length)
- *
- * @v str1		First string
- * @v str2		Second string
- * @v len		Maximum length
- * @ret diff		Difference
- */
-int strncmp ( const char *str1, const char *str2, size_t len ) {
-	const void *discard_S;
-	const void *discard_D;
-	size_t discard_c;
-	int diff;
-
-	__asm__ __volatile__ ( "\n1:\n\t"
-			       "dec %2\n\t"
-			       "js 2f\n\t"
-			       "lodsb\n\t"
-			       "scasb\n\t"
-			       "jne 3f\n\t"
-			       "testb %b3, %b3\n\t"
-			       "jnz 1b\n\t"
-			       /* Equal */
-			       "\n2:\n\t"
-			       "xor %3, %3\n\t"
-			       "jmp 4f\n\t"
-			       /* Not equal; CF indicates difference */
-			       "\n3:\n\t"
-			       "sbb %3, %3\n\t"
-			       "orb $1, %b3\n\t"
-			       "\n4:\n\t"
-			       : "=&S" ( discard_S ), "=&D" ( discard_D ),
-				 "=&c" ( discard_c ), "=&a" ( diff )
-			       : "0" ( str1 ), "1" ( str2 ), "2" ( len ) );
-
-	return diff;
-}
diff --git a/qemu/roms/ipxe/src/arch/x86/core/x86_tcpip.c b/qemu/roms/ipxe/src/arch/x86/core/x86_tcpip.c
index 8a4ce5152..88042f5f7 100644
--- a/qemu/roms/ipxe/src/arch/x86/core/x86_tcpip.c
+++ b/qemu/roms/ipxe/src/arch/x86/core/x86_tcpip.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/arch/x86/core/x86_uart.c b/qemu/roms/ipxe/src/arch/x86/core/x86_uart.c
new file mode 100644
index 000000000..e455775bf
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/core/x86_uart.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+#include <errno.h>
+#include <ipxe/uart.h>
+
+/** UART port bases */
+static uint16_t uart_base[] = {
+	[COM1] = 0x3f8,
+	[COM2] = 0x2f8,
+	[COM3] = 0x3e8,
+	[COM4] = 0x2e8,
+};
+
+/**
+ * Select UART port
+ *
+ * @v uart		UART
+ * @v port		Port number, or 0 to disable
+ * @ret rc		Return status code
+ */
+int uart_select ( struct uart *uart, unsigned int port ) {
+	int rc;
+
+	/* Set new UART base */
+	if ( port >= ( sizeof ( uart_base ) / sizeof ( uart_base[0] ) ) ) {
+		rc = -ENODEV;
+		goto err;
+	}
+	uart->base = ( ( void * ) ( intptr_t ) uart_base[port] );
+
+	/* Check that UART exists */
+	if ( ( rc = uart_exists ( uart ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	uart->base = NULL;
+	return rc;
+}
diff --git a/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c b/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c
new file mode 100644
index 000000000..f73829bd5
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.c
@@ -0,0 +1,597 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V driver
+ *
+ */
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <pic8259.h>
+#include <ipxe/malloc.h>
+#include <ipxe/device.h>
+#include <ipxe/cpuid.h>
+#include <ipxe/msr.h>
+#include <ipxe/hyperv.h>
+#include <ipxe/vmbus.h>
+#include "hyperv.h"
+
+/** Maximum time to wait for a message response
+ *
+ * This is a policy decision.
+ */
+#define HV_MESSAGE_MAX_WAIT_MS 1000
+
+/**
+ * Convert a Hyper-V status code to an iPXE status code
+ *
+ * @v status		Hyper-V status code
+ * @ret rc		iPXE status code (before negation)
+ */
+#define EHV( status ) EPLATFORM ( EINFO_EPLATFORM, (status) )
+
+/**
+ * Allocate zeroed pages
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v ...		Page addresses to fill in, terminated by NULL
+ * @ret rc		Return status code
+ */
+__attribute__ (( sentinel )) int
+hv_alloc_pages ( struct hv_hypervisor *hv, ... ) {
+	va_list args;
+	void **page;
+	int i;
+
+	/* Allocate and zero pages */
+	va_start ( args, hv );
+	for ( i = 0 ; ( ( page = va_arg ( args, void ** ) ) != NULL ); i++ ) {
+		*page = malloc_dma ( PAGE_SIZE, PAGE_SIZE );
+		if ( ! *page )
+			goto err_alloc;
+		memset ( *page, 0, PAGE_SIZE );
+	}
+	va_end ( args );
+
+	return 0;
+
+ err_alloc:
+	va_end ( args );
+	va_start ( args, hv );
+	for ( ; i >= 0 ; i-- ) {
+		page = va_arg ( args, void ** );
+		free_dma ( *page, PAGE_SIZE );
+	}
+	va_end ( args );
+	return -ENOMEM;
+}
+
+/**
+ * Free pages
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v ...		Page addresses, terminated by NULL
+ */
+__attribute__ (( sentinel )) void
+hv_free_pages ( struct hv_hypervisor *hv, ... ) {
+	va_list args;
+	void *page;
+
+	va_start ( args, hv );
+	while ( ( page = va_arg ( args, void * ) ) != NULL )
+		free_dma ( page, PAGE_SIZE );
+	va_end ( args );
+}
+
+/**
+ * Allocate message buffer
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_alloc_message ( struct hv_hypervisor *hv ) {
+
+	/* Allocate buffer.  Must be aligned to at least 8 bytes and
+	 * must not cross a page boundary, so align on its own size.
+	 */
+	hv->message = malloc_dma ( sizeof ( *hv->message ),
+				   sizeof ( *hv->message ) );
+	if ( ! hv->message )
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * Free message buffer
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_free_message ( struct hv_hypervisor *hv ) {
+
+	/* Free buffer */
+	free_dma ( hv->message, sizeof ( *hv->message ) );
+}
+
+/**
+ * Check whether or not we are running in Hyper-V
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_check_hv ( struct hv_hypervisor *hv ) {
+	struct x86_features features;
+	uint32_t interface_id;
+	uint32_t discard_ebx;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint32_t available;
+	uint32_t permissions;
+
+	/* Check for presence of a hypervisor (not necessarily Hyper-V) */
+	x86_features ( &features );
+	if ( ! ( features.intel.ecx & CPUID_FEATURES_INTEL_ECX_HYPERVISOR ) ) {
+		DBGC ( hv, "HV %p not running in a hypervisor\n", hv );
+		return -ENODEV;
+	}
+
+	/* Check that hypervisor is Hyper-V */
+	cpuid ( HV_CPUID_INTERFACE_ID, &interface_id, &discard_ebx,
+		&discard_ecx, &discard_edx );
+	if ( interface_id != HV_INTERFACE_ID ) {
+		DBGC ( hv, "HV %p not running in Hyper-V (interface ID "
+		       "%#08x)\n", hv, interface_id );
+		return -ENODEV;
+	}
+
+	/* Check that required features and privileges are available */
+	cpuid ( HV_CPUID_FEATURES, &available, &permissions, &discard_ecx,
+		&discard_edx );
+	if ( ! ( available & HV_FEATURES_AVAIL_HYPERCALL_MSR ) ) {
+		DBGC ( hv, "HV %p has no hypercall MSRs (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -ENODEV;
+	}
+	if ( ! ( available & HV_FEATURES_AVAIL_SYNIC_MSR ) ) {
+		DBGC ( hv, "HV %p has no SynIC MSRs (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -ENODEV;
+	}
+	if ( ! ( permissions & HV_FEATURES_PERM_POST_MESSAGES ) ) {
+		DBGC ( hv, "HV %p cannot post messages (features %08x:%08x)\n",
+		       hv, available, permissions );
+		return -EACCES;
+	}
+	if ( ! ( permissions & HV_FEATURES_PERM_SIGNAL_EVENTS ) ) {
+		DBGC ( hv, "HV %p cannot signal events (features %08x:%08x)",
+		       hv, available, permissions );
+		return -EACCES;
+	}
+
+	return 0;
+}
+
+/**
+ * Map hypercall page
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_map_hypercall ( struct hv_hypervisor *hv ) {
+	union {
+		struct {
+			uint32_t ebx;
+			uint32_t ecx;
+			uint32_t edx;
+		} __attribute__ (( packed ));
+		char text[ 13 /* "bbbbccccdddd" + NUL */ ];
+	} vendor_id;
+	uint32_t build;
+	uint32_t version;
+	uint32_t discard_eax;
+	uint32_t discard_ecx;
+	uint32_t discard_edx;
+	uint64_t guest_os_id;
+	uint64_t hypercall;
+
+	/* Report guest OS identity */
+	guest_os_id = rdmsr ( HV_X64_MSR_GUEST_OS_ID );
+	if ( guest_os_id != 0 ) {
+		DBGC ( hv, "HV %p guest OS ID MSR already set to %#08llx\n",
+		       hv, guest_os_id );
+		return -EBUSY;
+	}
+	guest_os_id = HV_GUEST_OS_ID_IPXE;
+	DBGC2 ( hv, "HV %p guest OS ID MSR is %#08llx\n", hv, guest_os_id );
+	wrmsr ( HV_X64_MSR_GUEST_OS_ID, guest_os_id );
+
+	/* Get hypervisor system identity (for debugging) */
+	cpuid ( HV_CPUID_VENDOR_ID, &discard_eax, &vendor_id.ebx,
+		&vendor_id.ecx, &vendor_id.edx );
+	vendor_id.text[ sizeof ( vendor_id.text ) - 1 ] = '\0';
+	cpuid ( HV_CPUID_HYPERVISOR_ID, &build, &version, &discard_ecx,
+		&discard_edx );
+	DBGC ( hv, "HV %p detected \"%s\" version %d.%d build %d\n", hv,
+	       vendor_id.text, ( version >> 16 ), ( version & 0xffff ), build );
+
+	/* Map hypercall page */
+	hypercall = rdmsr ( HV_X64_MSR_HYPERCALL );
+	hypercall &= ( PAGE_SIZE - 1 );
+	hypercall |= ( virt_to_phys ( hv->hypercall ) | HV_HYPERCALL_ENABLE );
+	DBGC2 ( hv, "HV %p hypercall MSR is %#08llx\n", hv, hypercall );
+	wrmsr ( HV_X64_MSR_HYPERCALL, hypercall );
+
+	return 0;
+}
+
+/**
+ * Unmap hypercall page
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_unmap_hypercall ( struct hv_hypervisor *hv ) {
+	uint64_t hypercall;
+	uint64_t guest_os_id;
+
+	/* Unmap the hypercall page */
+	hypercall = rdmsr ( HV_X64_MSR_HYPERCALL );
+	hypercall &= ( ( PAGE_SIZE - 1 ) & ~HV_HYPERCALL_ENABLE );
+	DBGC2 ( hv, "HV %p hypercall MSR is %#08llx\n", hv, hypercall );
+	wrmsr ( HV_X64_MSR_HYPERCALL, hypercall );
+
+	/* Reset the guest OS identity */
+	guest_os_id = 0;
+	DBGC2 ( hv, "HV %p guest OS ID MSR is %#08llx\n", hv, guest_os_id );
+	wrmsr ( HV_X64_MSR_GUEST_OS_ID, guest_os_id );
+}
+
+/**
+ * Map synthetic interrupt controller
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int hv_map_synic ( struct hv_hypervisor *hv ) {
+	uint64_t simp;
+	uint64_t siefp;
+	uint64_t scontrol;
+
+	/* Map SynIC message page */
+	simp = rdmsr ( HV_X64_MSR_SIMP );
+	simp &= ( PAGE_SIZE - 1 );
+	simp |= ( virt_to_phys ( hv->synic.message ) | HV_SIMP_ENABLE );
+	DBGC2 ( hv, "HV %p SIMP MSR is %#08llx\n", hv, simp );
+	wrmsr ( HV_X64_MSR_SIMP, simp );
+
+	/* Map SynIC event page */
+	siefp = rdmsr ( HV_X64_MSR_SIEFP );
+	siefp &= ( PAGE_SIZE - 1 );
+	siefp |= ( virt_to_phys ( hv->synic.event ) | HV_SIEFP_ENABLE );
+	DBGC2 ( hv, "HV %p SIEFP MSR is %#08llx\n", hv, siefp );
+	wrmsr ( HV_X64_MSR_SIEFP, siefp );
+
+	/* Enable SynIC */
+	scontrol = rdmsr ( HV_X64_MSR_SCONTROL );
+	scontrol |= HV_SCONTROL_ENABLE;
+	DBGC2 ( hv, "HV %p SCONTROL MSR is %#08llx\n", hv, scontrol );
+	wrmsr ( HV_X64_MSR_SCONTROL, scontrol );
+
+	return 0;
+}
+
+/**
+ * Unmap synthetic interrupt controller
+ *
+ * @v hv		Hyper-V hypervisor
+ */
+static void hv_unmap_synic ( struct hv_hypervisor *hv ) {
+	uint64_t scontrol;
+	uint64_t siefp;
+	uint64_t simp;
+
+	/* Disable SynIC */
+	scontrol = rdmsr ( HV_X64_MSR_SCONTROL );
+	scontrol &= ~HV_SCONTROL_ENABLE;
+	DBGC2 ( hv, "HV %p SCONTROL MSR is %#08llx\n", hv, scontrol );
+	wrmsr ( HV_X64_MSR_SCONTROL, scontrol );
+
+	/* Unmap SynIC event page */
+	siefp = rdmsr ( HV_X64_MSR_SIEFP );
+	siefp &= ( ( PAGE_SIZE - 1 ) & ~HV_SIEFP_ENABLE );
+	DBGC2 ( hv, "HV %p SIEFP MSR is %#08llx\n", hv, siefp );
+	wrmsr ( HV_X64_MSR_SIEFP, siefp );
+
+	/* Unmap SynIC message page */
+	simp = rdmsr ( HV_X64_MSR_SIMP );
+	simp &= ( ( PAGE_SIZE - 1 ) & ~HV_SIMP_ENABLE );
+	DBGC2 ( hv, "HV %p SIMP MSR is %#08llx\n", hv, simp );
+	wrmsr ( HV_X64_MSR_SIMP, simp );
+}
+
+/**
+ * Enable synthetic interrupt
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ */
+void hv_enable_sint ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	unsigned long msr = HV_X64_MSR_SINT ( sintx );
+	uint64_t sint;
+
+	/* Enable synthetic interrupt
+	 *
+	 * We have to enable the interrupt, otherwise messages will
+	 * not be delivered (even though the documentation implies
+	 * that polling for messages is possible).  We enable AutoEOI
+	 * and hook the interrupt to the obsolete IRQ13 (FPU
+	 * exception) vector, which will be implemented as a no-op.
+	 */
+	sint = rdmsr ( msr );
+	sint &= ~( HV_SINT_MASKED | HV_SINT_VECTOR_MASK );
+	sint |= ( HV_SINT_AUTO_EOI |
+		  HV_SINT_VECTOR ( IRQ_INT ( 13 /* See comment above */ ) ) );
+	DBGC2 ( hv, "HV %p SINT%d MSR is %#08llx\n", hv, sintx, sint );
+	wrmsr ( msr, sint );
+}
+
+/**
+ * Disable synthetic interrupt
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ */
+void hv_disable_sint ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	unsigned long msr = HV_X64_MSR_SINT ( sintx );
+	uint64_t sint;
+
+	/* Disable synthetic interrupt */
+	sint = rdmsr ( msr );
+	sint &= ~HV_SINT_AUTO_EOI;
+	sint |= HV_SINT_MASKED;
+	DBGC2 ( hv, "HV %p SINT%d MSR is %#08llx\n", hv, sintx, sint );
+	wrmsr ( msr, sint );
+}
+
+/**
+ * Post message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v id		Connection ID
+ * @v type		Message type
+ * @v data		Message
+ * @v len		Length of message
+ * @ret rc		Return status code
+ */
+int hv_post_message ( struct hv_hypervisor *hv, unsigned int id,
+		      unsigned int type, const void *data, size_t len ) {
+	struct hv_post_message *msg = &hv->message->posted;
+	int status;
+	int rc;
+
+	/* Sanity check */
+	assert ( len <= sizeof ( msg->data ) );
+
+	/* Construct message */
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = cpu_to_le32 ( id );
+	msg->type = cpu_to_le32 ( type );
+	msg->len = cpu_to_le32 ( len );
+	memcpy ( msg->data, data, len );
+	DBGC2 ( hv, "HV %p connection %d posting message type %#08x:\n",
+		hv, id, type );
+	DBGC2_HDA ( hv, 0, msg->data, len );
+
+	/* Post message */
+	if ( ( status = hv_call ( hv, HV_POST_MESSAGE, msg, NULL ) ) != 0 ) {
+		rc = -EHV ( status );
+		DBGC ( hv, "HV %p could not post message to %#08x: %s\n",
+		       hv, id, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Wait for received message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v sintx		Synthetic interrupt number
+ * @ret rc		Return status code
+ */
+int hv_wait_for_message ( struct hv_hypervisor *hv, unsigned int sintx ) {
+	struct hv_message *msg = &hv->message->received;
+	struct hv_message *src = &hv->synic.message[sintx];
+	unsigned int retries;
+	size_t len;
+
+	/* Wait for message to arrive */
+	for ( retries = 0 ; retries < HV_MESSAGE_MAX_WAIT_MS ; retries++ ) {
+
+		/* Check for message */
+		if ( src->type ) {
+
+			/* Copy message */
+			memset ( msg, 0, sizeof ( *msg ) );
+			len = src->len;
+			assert ( len <= sizeof ( *msg ) );
+			memcpy ( msg, src,
+				 ( offsetof ( typeof ( *msg ), data ) + len ) );
+			DBGC2 ( hv, "HV %p SINT%d received message type "
+				"%#08x:\n", hv, sintx,
+				le32_to_cpu ( msg->type ) );
+			DBGC2_HDA ( hv, 0, msg->data, len );
+
+			/* Consume message */
+			src->type = 0;
+
+			return 0;
+		}
+
+		/* Trigger message delivery */
+		wrmsr ( HV_X64_MSR_EOM, 0 );
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( hv, "HV %p SINT%d timed out waiting for message\n",
+	       hv, sintx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Signal event
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v id		Connection ID
+ * @v flag		Flag number
+ * @ret rc		Return status code
+ */
+int hv_signal_event ( struct hv_hypervisor *hv, unsigned int id,
+		      unsigned int flag ) {
+	struct hv_signal_event *event = &hv->message->signalled;
+	int status;
+	int rc;
+
+	/* Construct event */
+	memset ( event, 0, sizeof ( *event ) );
+	event->id = cpu_to_le32 ( id );
+	event->flag = cpu_to_le16 ( flag );
+
+	/* Signal event */
+	if ( ( status = hv_call ( hv, HV_SIGNAL_EVENT, event, NULL ) ) != 0 ) {
+		rc = -EHV ( status );
+		DBGC ( hv, "HV %p could not signal event to %#08x: %s\n",
+		       hv, id, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Probe root device
+ *
+ * @v rootdev		Root device
+ * @ret rc		Return status code
+ */
+static int hv_probe ( struct root_device *rootdev ) {
+	struct hv_hypervisor *hv;
+	int rc;
+
+	/* Allocate and initialise structure */
+	hv = zalloc ( sizeof ( *hv ) );
+	if ( ! hv ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Check we are running in Hyper-V */
+	if ( ( rc = hv_check_hv ( hv ) ) != 0 )
+		goto err_check_hv;
+
+	/* Allocate pages */
+	if ( ( rc = hv_alloc_pages ( hv, &hv->hypercall, &hv->synic.message,
+				     &hv->synic.event, NULL ) ) != 0 )
+		goto err_alloc_pages;
+
+	/* Allocate message buffer */
+	if ( ( rc = hv_alloc_message ( hv ) ) != 0 )
+		goto err_alloc_message;
+
+	/* Map hypercall page */
+	if ( ( rc = hv_map_hypercall ( hv ) ) != 0 )
+		goto err_map_hypercall;
+
+	/* Map synthetic interrupt controller */
+	if ( ( rc = hv_map_synic ( hv ) ) != 0 )
+		goto err_map_synic;
+
+	/* Probe Hyper-V devices */
+	if ( ( rc = vmbus_probe ( hv, &rootdev->dev ) ) != 0 )
+		goto err_vmbus_probe;
+
+	rootdev_set_drvdata ( rootdev, hv );
+	return 0;
+
+	vmbus_remove ( hv, &rootdev->dev );
+ err_vmbus_probe:
+	hv_unmap_synic ( hv );
+ err_map_synic:
+	hv_unmap_hypercall ( hv );
+ err_map_hypercall:
+	hv_free_message ( hv );
+ err_alloc_message:
+	hv_free_pages ( hv, hv->hypercall, hv->synic.message, hv->synic.event,
+			NULL );
+ err_alloc_pages:
+ err_check_hv:
+	free ( hv );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove root device
+ *
+ * @v rootdev		Root device
+ */
+static void hv_remove ( struct root_device *rootdev ) {
+	struct hv_hypervisor *hv = rootdev_get_drvdata ( rootdev );
+
+	vmbus_remove ( hv, &rootdev->dev );
+	hv_unmap_synic ( hv );
+	hv_unmap_hypercall ( hv );
+	hv_free_message ( hv );
+	hv_free_pages ( hv, hv->hypercall, hv->synic.message, hv->synic.event,
+			NULL );
+	free ( hv );
+}
+
+/** Hyper-V root device driver */
+static struct root_driver hv_root_driver = {
+	.probe = hv_probe,
+	.remove = hv_remove,
+};
+
+/** Hyper-V root device */
+struct root_device hv_root_device __root_device = {
+	.dev = { .name = "Hyper-V" },
+	.driver = &hv_root_driver,
+};
+
+/* Drag in objects via hv_root_device */
+REQUIRING_SYMBOL ( hv_root_device );
+
+/* Drag in netvsc driver */
+REQUIRE_OBJECT ( netvsc );
diff --git a/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h b/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h
new file mode 100644
index 000000000..0d09beb44
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/drivers/hyperv/hyperv.h
@@ -0,0 +1,57 @@
+#ifndef _HYPERV_H
+#define _HYPERV_H
+
+/** @file
+ *
+ * Hyper-V driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Get vendor identification */
+#define HV_CPUID_VENDOR_ID 0x40000000UL
+
+/** Get interface identification */
+#define HV_CPUID_INTERFACE_ID 0x40000001UL
+
+/** Get hypervisor identification */
+#define HV_CPUID_HYPERVISOR_ID 0x40000002UL
+
+/** Get hypervisor features */
+#define HV_CPUID_FEATURES 0x40000003UL
+
+/** SynIC MSRs are available */
+#define HV_FEATURES_AVAIL_SYNIC_MSR 0x00000004UL
+
+/** Hypercall MSRs are available */
+#define HV_FEATURES_AVAIL_HYPERCALL_MSR 0x00000020UL
+
+/** Guest may post messages */
+#define HV_FEATURES_PERM_POST_MESSAGES 0x00000010UL
+
+/** Guest may signal events */
+#define HV_FEATURES_PERM_SIGNAL_EVENTS 0x00000020UL
+
+/** Guest OS identity MSR */
+#define HV_X64_MSR_GUEST_OS_ID 0x40000000UL
+
+/** Hypercall page MSR */
+#define HV_X64_MSR_HYPERCALL 0x40000001UL
+
+/** SynIC control MSR */
+#define HV_X64_MSR_SCONTROL 0x40000080UL
+
+/** SynIC event flags page MSR */
+#define HV_X64_MSR_SIEFP 0x40000082UL
+
+/** SynIC message page MSR */
+#define HV_X64_MSR_SIMP 0x40000083UL
+
+/** SynIC end of message MSR */
+#define HV_X64_MSR_EOM 0x40000084UL
+
+/** SynIC interrupt source MSRs */
+#define HV_X64_MSR_SINT(x) ( 0x40000090UL + (x) )
+
+#endif /* _HYPERV_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.c b/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
index 7406ca68d..7ac32d54c 100644
--- a/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
+++ b/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
@@ -492,5 +496,8 @@ struct pci_driver hvm_driver __pci_driver = {
 	.remove = hvm_remove,
 };
 
+/* Drag in objects via hvm_driver */
+REQUIRING_SYMBOL ( hvm_driver );
+
 /* Drag in netfront driver */
 REQUIRE_OBJECT ( netfront );
diff --git a/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.h b/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
index 325d20d66..72ed94f6d 100644
--- a/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
+++ b/qemu/roms/ipxe/src/arch/x86/drivers/xen/hvm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/xen.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c b/qemu/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
index c4e35d179..d73ce2a3e 100644
--- a/qemu/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
+++ b/qemu/roms/ipxe/src/arch/x86/hci/commands/cpuid_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/bigint.h b/qemu/roms/ipxe/src/arch/x86/include/bits/bigint.h
index d3449af5a..c9bb6ea45 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/bigint.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/bigint.h
@@ -6,7 +6,7 @@
  * Big integer support
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/endian.h b/qemu/roms/ipxe/src/arch/x86/include/bits/endian.h
new file mode 100644
index 000000000..85718cfdd
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/endian.h
@@ -0,0 +1,8 @@
+#ifndef _BITS_ENDIAN_H
+#define _BITS_ENDIAN_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#define __BYTE_ORDER __LITTLE_ENDIAN
+
+#endif /* _BITS_ENDIAN_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/errfile.h b/qemu/roms/ipxe/src/arch/x86/include/bits/errfile.h
index 624575621..0d1617d20 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/errfile.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/errfile.h
@@ -1,7 +1,7 @@
 #ifndef _BITS_ERRFILE_H
 #define _BITS_ERRFILE_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @addtogroup errfile Error file identifiers
@@ -21,6 +21,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_guestinfo	( ERRFILE_ARCH | ERRFILE_CORE | 0x000a0000 )
 #define ERRFILE_apm		( ERRFILE_ARCH | ERRFILE_CORE | 0x000b0000 )
 #define ERRFILE_vesafb		( ERRFILE_ARCH | ERRFILE_CORE | 0x000c0000 )
+#define ERRFILE_int13con	( ERRFILE_ARCH | ERRFILE_CORE | 0x000d0000 )
 
 #define ERRFILE_bootsector     ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_bzimage	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00010000 )
@@ -46,9 +47,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_timer_rdtsc   ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_timer_bios    ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00010000 )
 #define ERRFILE_hvm	      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00020000 )
+#define ERRFILE_hyperv	      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00030000 )
+#define ERRFILE_x86_uart      ( ERRFILE_ARCH | ERRFILE_DRIVER | 0x00040000 )
 
 #define ERRFILE_cpuid_cmd      ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00000000 )
 #define ERRFILE_cpuid_settings ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00010000 )
+#define ERRFILE_efi_entropy    ( ERRFILE_ARCH | ERRFILE_OTHER | 0x00020000 )
 
 /** @} */
 
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/io.h b/qemu/roms/ipxe/src/arch/x86/include/bits/io.h
index cb1b67a6f..60c2e3edf 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/io.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/x86_io.h>
 
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/pci_io.h b/qemu/roms/ipxe/src/arch/x86/include/bits/pci_io.h
index 01b12326e..b41e562ee 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/pci_io.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/pci_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/pcibios.h>
 #include <ipxe/pcidirect.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/string.h b/qemu/roms/ipxe/src/arch/x86/include/bits/string.h
index dce994983..c26fe30d5 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/string.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/string.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -28,8 +32,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#define __HAVE_ARCH_MEMCPY
-
 extern void * __memcpy ( void *dest, const void *src, size_t len );
 extern void * __memcpy_reverse ( void *dest, const void *src, size_t len );
 
@@ -169,8 +171,6 @@ memcpy ( void *dest, const void *src, size_t len ) {
 	}
 }
 
-#define __HAVE_ARCH_MEMMOVE
-
 extern void * __memmove ( void *dest, const void *src, size_t len );
 
 /**
@@ -196,8 +196,6 @@ memmove ( void *dest, const void *src, size_t len ) {
 	}
 }
 
-#define __HAVE_ARCH_MEMSET
-
 /**
  * Fill memory region
  *
@@ -206,7 +204,8 @@ memmove ( void *dest, const void *src, size_t len ) {
  * @v len		Length
  * @ret dest		Destination address
  */
-static inline void * memset ( void *dest, int fill, size_t len ) {
+static inline __attribute__ (( always_inline )) void *
+__memset ( void *dest, int fill, size_t len ) {
 	void *discard_D;
 	size_t discard_c;
 
@@ -217,16 +216,129 @@ static inline void * memset ( void *dest, int fill, size_t len ) {
 	return dest;
 }
 
-#define __HAVE_ARCH_MEMSWAP
+/**
+ * Fill memory region with zero (where length is a compile-time constant)
+ *
+ * @v dest		Destination address
+ * @v len		Length
+ * @ret dest		Destination address
+ */
+static inline __attribute__ (( always_inline )) void *
+__constant_memset_zero ( void *dest, size_t len ) {
+	union {
+		uint32_t u32[2];
+		uint16_t u16[4];
+		uint8_t  u8[8];
+	} __attribute__ (( __may_alias__ )) *dest_u = dest;
+	void *edi;
+	uint32_t eax;
+
+	switch ( len ) {
+	case 0 : /* 0 bytes */
+		return dest;
+
+	/* Single-register moves.  Almost certainly better than a
+	 * string operation.  We can avoid clobbering any registers,
+	 * we can reuse a zero that happens to already be in a
+	 * register, and we can optimise away the code entirely if the
+	 * memset() is used to clear a region which then gets
+	 * immediately overwritten.
+	 */
+	case 1 : /* 3 bytes */
+		dest_u->u8[0] = 0;
+		return dest;
+	case 2: /* 5 bytes */
+		dest_u->u16[0] = 0;
+		return dest;
+	case 4: /* 6 bytes */
+		dest_u->u32[0] = 0;
+		return dest;
+
+	/* Double-register moves.  Very probably better than a string
+	 * operation.
+	 */
+	case 3 : /* 9 bytes */
+		dest_u->u16[0] = 0;
+		dest_u->u8[2]  = 0;
+		return dest;
+	case 5 : /* 10 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u8[4]  = 0;
+		return dest;
+	case 6 : /* 12 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u16[2] = 0;
+		return dest;
+	case 8 : /* 13 bytes */
+		dest_u->u32[0] = 0;
+		dest_u->u32[1] = 0;
+		return dest;
+	}
+
+	/* As with memcpy(), we can potentially save space by using
+	 * multiple single-byte "stos" instructions instead of loading
+	 * up ecx and using "rep stosb".
+	 *
+	 * "load ecx, rep movsb" is 7 bytes, plus an average of 1 byte
+	 * to allow for saving/restoring ecx 50% of the time.
+	 *
+	 * "stosl" and "stosb" are 1 byte each, "stosw" is two bytes.
+	 *
+	 * The calculations are therefore the same as for memcpy(),
+	 * giving a cutoff point of around 26 bytes.
+	 */
 
-extern void * memswap ( void *dest, void *src, size_t len );
+	edi = dest;
+	eax = 0;
+
+	if ( len >= 26 )
+		return __memset ( dest, 0, len );
 
-#define __HAVE_ARCH_STRNCMP
+	if ( len >= 6*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 5*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 4*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 3*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 2*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( len >= 1*4 )
+		__asm__ __volatile__ ( "stosl" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( ( len % 4 ) >= 2 )
+		__asm__ __volatile__ ( "stosw" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
+	if ( ( len % 2 ) >= 1 )
+		__asm__ __volatile__ ( "stosb" : "=&D" ( edi ), "=&a" ( eax )
+				       : "0" ( edi ), "1" ( eax ) : "memory" );
 
-extern int strncmp ( const char *str1, const char *str2, size_t len );
+	return dest;
+}
 
-#define __HAVE_ARCH_STRLEN
+/**
+ * Fill memory region
+ *
+ * @v dest		Destination address
+ * @v fill		Fill pattern
+ * @v len		Length
+ * @ret dest		Destination address
+ */
+static inline __attribute__ (( always_inline )) void *
+memset ( void *dest, int fill, size_t len ) {
 
-extern size_t strlen ( const char *string );
+	if ( __builtin_constant_p ( fill ) && ( fill == 0 ) &&
+	     __builtin_constant_p ( len ) ) {
+		return __constant_memset_zero ( dest, len );
+	} else {
+		return __memset ( dest, fill, len );
+	}
+}
 
 #endif /* X86_BITS_STRING_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/tcpip.h b/qemu/roms/ipxe/src/arch/x86/include/bits/tcpip.h
index a4b335eb1..5c2baffcf 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/tcpip.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/tcpip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern uint16_t x86_tcpip_continue_chksum ( uint16_t partial,
 					    const void *data, size_t len );
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/uart.h b/qemu/roms/ipxe/src/arch/x86/include/bits/uart.h
new file mode 100644
index 000000000..e09cd3f4c
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/uart.h
@@ -0,0 +1,41 @@
+#ifndef _BITS_UART_H
+#define _BITS_UART_H
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Write to UART register
+ *
+ * @v uart		UART
+ * @v addr		Register address
+ * @v data		Data
+ */
+static inline __attribute__ (( always_inline )) void
+uart_write ( struct uart *uart, unsigned int addr, uint8_t data ) {
+	outb ( data, ( uart->base + addr ) );
+}
+
+/**
+ * Read from UART register
+ *
+ * @v uart		UART
+ * @v addr		Register address
+ * @ret data		Data
+ */
+static inline __attribute__ (( always_inline )) uint8_t
+uart_read ( struct uart *uart, unsigned int addr ) {
+	return inb ( uart->base + addr );
+}
+
+extern int uart_select ( struct uart *uart, unsigned int port );
+
+#endif /* _BITS_UART_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/include/bits/xen.h b/qemu/roms/ipxe/src/arch/x86/include/bits/xen.h
index dbccf1b77..fc065ea38 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/bits/xen.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/bits/xen.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Hypercall registers */
 #ifdef __x86_64__
@@ -161,4 +161,23 @@ xen_hypercall_5 ( struct xen_hypervisor *xen, unsigned int hypercall,
 	return retval;
 }
 
+/**
+ * Test and clear pending event
+ *
+ * @v xen		Xen hypervisor
+ * @v port		Event channel port
+ * @ret pending		Event was pending
+ */
+static inline __attribute__ (( always_inline )) uint8_t
+xenevent_pending ( struct xen_hypervisor *xen, evtchn_port_t port ) {
+	uint8_t pending;
+
+	__asm__ __volatile__ ( "lock btr %2, %0\n\t"
+			       "setc %1\n\t"
+			       : "+m" ( xen->shared->evtchn_pending ),
+				 "=a"  ( pending )
+			       : "Ir" ( port ) );
+	return pending;
+}
+
 #endif /* _BITS_XEN_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
index 2f78dfca1..da85d0b88 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/cpuid.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
@@ -39,6 +39,9 @@ struct x86_features {
 /** Get standard features */
 #define CPUID_FEATURES 0x00000001UL
 
+/** Hypervisor is present */
+#define CPUID_FEATURES_INTEL_ECX_HYPERVISOR 0x80000000UL
+
 /** Get largest extended function */
 #define CPUID_AMD_MAX_FN 0x80000000UL
 
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
index e85a272b3..1a391c9b6 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/efi/efix86_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_EFIX86
 #define NAP_PREFIX_efix86
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
index 36af7fcde..7e1bcd814 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcibios.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_PCBIOS
 #define PCIAPI_PREFIX_pcbios
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
index 7fa7c4fa7..d924f2f20 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pcidirect.h
@@ -1,7 +1,7 @@
 #ifndef _PCIDIRECT_H
 #define _PCIDIRECT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/io.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h
new file mode 100644
index 000000000..00b0ab164
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/pit8254.h
@@ -0,0 +1,81 @@
+#ifndef _IPXE_PIT8254_H
+#define _IPXE_PIT8254_H
+
+/** @file
+ *
+ * 8254 Programmable Interval Timer
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** IRQ0 channel */
+#define PIT8254_CH_IRQ0 0
+
+/** PC speaker channel */
+#define PIT8254_CH_SPKR 2
+
+/** Timer frequency (1.193182MHz) */
+#define PIT8254_HZ 1193182UL
+
+/** Data port */
+#define PIT8254_DATA(channel) ( 0x40 + (channel) )
+
+/** Mode/command register */
+#define PIT8254_CMD 0x43
+
+/** Select channel */
+#define PIT8254_CMD_CHANNEL(channel) ( (channel) << 6 )
+
+/** Access modes */
+#define PIT8254_CMD_ACCESS_LATCH 0x00	/**< Latch count value command */
+#define PIT8254_CMD_ACCESS_LO	0x10	/**< Low byte only */
+#define PIT8254_CMD_ACCESS_HI	0x20	/**< High byte only */
+#define PIT8254_CMD_ACCESS_LOHI	0x30	/**< Low-byte, high-byte pair */
+
+/* Operating modes */
+#define PIT8254_CMD_OP_TERMINAL	0x00	/**< Interrupt on terminal count */
+#define PIT8254_CMD_OP_ONESHOT	0x02	/**< Hardware re-triggerable one-shot */
+#define PIT8254_CMD_OP_RATE	0x04	/**< Rate generator */
+#define PIT8254_CMD_OP_SQUARE	0x06	/**< Square wave generator */
+#define PIT8254_CMD_OP_SWSTROBE	0x08	/**< Software triggered strobe */
+#define PIT8254_CMD_OP_HWSTROBE	0x0a	/**< Hardware triggered strobe */
+#define PIT8254_CMD_OP_RATE2	0x0c	/**< Rate generator (duplicate) */
+#define PIT8254_CMD_OP_SQUARE2	0x0e	/**< Square wave generator (duplicate)*/
+
+/** Binary mode */
+#define PIT8254_CMD_BINARY 0x00
+
+/** BCD mode */
+#define PIT8254_CMD_BCD 0x01
+
+/** PC speaker control register */
+#define PIT8254_SPKR 0x61
+
+/** PC speaker channel gate */
+#define PIT8254_SPKR_GATE 0x01
+
+/** PC speaker enabled */
+#define PIT8254_SPKR_ENABLE 0x02
+
+/** PC speaker channel output */
+#define PIT8254_SPKR_OUT 0x20
+
+extern void pit8254_speaker_delay ( unsigned int ticks );
+
+/**
+ * Delay for a fixed number of microseconds
+ *
+ * @v usecs		Number of microseconds for which to delay
+ */
+static inline __attribute__ (( always_inline )) void
+pit8254_udelay ( unsigned long usecs ) {
+
+	/* Delays are invariably compile-time constants; force the
+	 * multiplication and division to take place at compilation
+	 * time rather than runtime.
+	 */
+	pit8254_speaker_delay ( ( usecs * PIT8254_HZ ) / 1000000 );
+}
+
+#endif /* _IPXE_PIT8254_H */
diff --git a/qemu/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h b/qemu/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
index 9e68f4e78..5214e9fbb 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/ipxe/x86_io.h
@@ -15,7 +15,7 @@
  * physically fit into a machine with such an old CPU anyway.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef IOAPI_X86
 #define IOAPI_PREFIX_x86
diff --git a/qemu/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h b/qemu/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
index e83fd9d87..d60905f22 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/linux/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _LINUX_DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE(GPL2_OR_LATER_OR_UBDL);
 
 #include <ipxe/dhcp.h>
 
diff --git a/qemu/roms/ipxe/src/arch/i386/include/pic8259.h b/qemu/roms/ipxe/src/arch/x86/include/pic8259.h
index a07e97d30..f02e62909 100644
--- a/qemu/roms/ipxe/src/arch/i386/include/pic8259.h
+++ b/qemu/roms/ipxe/src/arch/x86/include/pic8259.h
@@ -4,16 +4,13 @@
  * Initially written by Michael Brown (mcb30).
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef PIC8259_H
 #define PIC8259_H
 
 #include <ipxe/io.h>
 
-/* For segoff_t */
-#include "realmode.h"
-
 #define IRQ_PIC_CUTOFF 8
 
 /* 8259 register locations */
diff --git a/qemu/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c b/qemu/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c
new file mode 100644
index 000000000..a54bd12e6
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86/interface/efi/efi_entropy.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <ipxe/entropy.h>
+#include <ipxe/crc32.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/Protocol/Rng.h>
+
+/** @file
+ *
+ * EFI entropy source
+ *
+ */
+
+/** Random number generator protocol */
+static EFI_RNG_PROTOCOL *efirng;
+EFI_REQUEST_PROTOCOL ( EFI_RNG_PROTOCOL, &efirng );
+
+/** Minimum number of bytes to request from RNG
+ *
+ * The UEFI spec states (for no apparently good reason) that "When a
+ * Deterministic Random Bit Generator (DRBG) is used on the output of
+ * a (raw) entropy source, its security level must be at least 256
+ * bits."  The EDK2 codebase (mis)interprets this to mean that the
+ * call to GetRNG() should fail if given a buffer less than 32 bytes.
+ *
+ * Incidentally, nothing in the EFI RNG protocol provides any way to
+ * report the actual amount of entropy returned by GetRNG().
+ */
+#define EFI_ENTROPY_RNG_LEN 32
+
+/** Time (in 100ns units) to delay waiting for timer tick
+ *
+ * In theory, UEFI allows us to specify a trigger time of zero to
+ * simply wait for the next timer tick.  In practice, specifying zero
+ * seems to often return immediately, which produces almost no
+ * entropy.  Specify a delay of 1000ns to try to force an existent
+ * delay.
+ */
+#define EFI_ENTROPY_TRIGGER_TIME 10
+
+/** Event used to wait for timer tick */
+static EFI_EVENT tick;
+
+/**
+ * Enable entropy gathering
+ *
+ * @ret rc		Return status code
+ */
+static int efi_entropy_enable ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	EFI_STATUS efirc;
+	int rc;
+
+	DBGC ( &tick, "ENTROPY %s RNG protocol\n",
+	       ( efirng ? "has" : "has no" ) );
+
+	/* Create timer tick event */
+	if ( ( efirc = bs->CreateEvent ( EVT_TIMER, TPL_NOTIFY, NULL, NULL,
+					 &tick ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not create event: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Disable entropy gathering
+ *
+ */
+static void efi_entropy_disable ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+
+	/* Close timer tick event */
+	bs->CloseEvent ( tick );
+}
+
+/**
+ * Wait for a timer tick
+ *
+ * @ret low		TSC low-order bits, or negative error
+ */
+static int efi_entropy_tick ( void ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	UINTN index;
+	uint16_t low;
+	uint32_t discard_d;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Wait for next timer tick */
+	if ( ( efirc = bs->SetTimer ( tick, TimerRelative,
+				      EFI_ENTROPY_TRIGGER_TIME ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not set timer: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+	if ( ( efirc = bs->WaitForEvent ( 1, &tick, &index ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not wait for timer tick: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Get current TSC low-order bits */
+	__asm__ __volatile__ ( "rdtsc" : "=a" ( low ), "=d" ( discard_d ) );
+
+	return low;
+}
+
+/**
+ * Get noise sample from timer ticks
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise_ticks ( noise_sample_t *noise ) {
+	int before;
+	int after;
+	int rc;
+
+	/* Wait for a timer tick */
+	before = efi_entropy_tick();
+	if ( before < 0 ) {
+		rc = before;
+		return rc;
+	}
+
+	/* Wait for another timer tick */
+	after = efi_entropy_tick();
+	if ( after < 0 ) {
+		rc = after;
+		return rc;
+	}
+
+	/* Use TSC delta as noise sample */
+	*noise = ( after - before );
+
+	return 0;
+}
+
+/**
+ * Get noise sample from RNG protocol
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise_rng ( noise_sample_t *noise ) {
+	uint8_t buf[EFI_ENTROPY_RNG_LEN];
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Fail if we have no EFI RNG protocol */
+	if ( ! efirng )
+		return -ENOTSUP;
+
+	/* Get the minimum allowed number of random bytes */
+	if ( ( efirc = efirng->GetRNG ( efirng, NULL, EFI_ENTROPY_RNG_LEN,
+					buf ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( &tick, "ENTROPY could not read from RNG: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Reduce random bytes to a single noise sample.  This seems
+	 * like overkill, but we have no way of knowing how much
+	 * entropy is actually present in the bytes returned by the
+	 * RNG protocol.
+	 */
+	*noise = crc32_le ( 0, buf, sizeof ( buf ) );
+
+	return 0;
+}
+
+/**
+ * Get noise sample
+ *
+ * @ret noise		Noise sample
+ * @ret rc		Return status code
+ */
+static int efi_get_noise ( noise_sample_t *noise ) {
+	int rc;
+
+	/* Try RNG first, falling back to timer ticks */
+	if ( ( ( rc = efi_get_noise_rng ( noise ) ) != 0 ) &&
+	     ( ( rc = efi_get_noise_ticks ( noise ) ) != 0 ) )
+		return rc;
+
+	return 0;
+}
+
+PROVIDE_ENTROPY_INLINE ( efi, min_entropy_per_sample );
+PROVIDE_ENTROPY ( efi, entropy_enable, efi_entropy_enable );
+PROVIDE_ENTROPY ( efi, entropy_disable, efi_entropy_disable );
+PROVIDE_ENTROPY ( efi, get_noise, efi_get_noise );
diff --git a/qemu/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c b/qemu/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
index b05421fab..3ebf0bd68 100644
--- a/qemu/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
+++ b/qemu/roms/ipxe/src/arch/x86/interface/efi/efix86_nap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nap.h>
 #include <ipxe/efi/efi.h>
diff --git a/qemu/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c b/qemu/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
index 3daefd00a..4fbb19ff7 100644
--- a/qemu/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
+++ b/qemu/roms/ipxe/src/arch/x86/prefix/efidrvprefix.c
@@ -21,7 +21,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdlib.h>
 #include <ipxe/init.h>
+#include <ipxe/device.h>
 #include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_driver.h>
 
 /**
  * EFI entry point
@@ -44,3 +46,36 @@ EFI_STATUS EFIAPI _efidrv_start ( EFI_HANDLE image_handle,
 
 	return 0;
 }
+
+/**
+ * Probe EFI root bus
+ *
+ * @v rootdev		EFI root device
+ */
+static int efi_probe ( struct root_device *rootdev __unused ) {
+
+	/* Do nothing */
+	return 0;
+}
+
+/**
+ * Remove EFI root bus
+ *
+ * @v rootdev		EFI root device
+ */
+static void efi_remove ( struct root_device *rootdev __unused ) {
+
+	efi_driver_disconnect_all();
+}
+
+/** EFI root device driver */
+static struct root_driver efi_root_driver = {
+	.probe = efi_probe,
+	.remove = efi_remove,
+};
+
+/** EFI root device */
+struct root_device efi_root_device __root_device = {
+	.dev = { .name = "EFI" },
+	.driver = &efi_root_driver,
+};
diff --git a/qemu/roms/ipxe/src/arch/x86/prefix/efiprefix.c b/qemu/roms/ipxe/src/arch/x86/prefix/efiprefix.c
index b0bf99c65..18b931e68 100644
--- a/qemu/roms/ipxe/src/arch/x86/prefix/efiprefix.c
+++ b/qemu/roms/ipxe/src/arch/x86/prefix/efiprefix.c
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_snp.h>
 #include <ipxe/efi/efi_autoboot.h>
+#include <ipxe/efi/efi_watchdog.h>
 
 /**
  * EFI entry point
@@ -49,6 +50,9 @@ EFI_STATUS EFIAPI _efi_start ( EFI_HANDLE image_handle,
 	/* Claim SNP devices for use by iPXE */
 	efi_snp_claim();
 
+	/* Start watchdog holdoff timer */
+	efi_watchdog_start();
+
 	/* Call to main() */
 	if ( ( rc = main() ) != 0 ) {
 		efirc = EFIRC ( rc );
@@ -56,6 +60,7 @@ EFI_STATUS EFIAPI _efi_start ( EFI_HANDLE image_handle,
 	}
 
  err_main:
+	efi_watchdog_stop();
 	efi_snp_release();
 	efi_loaded_image->Unload ( image_handle );
 	efi_driver_reconnect_all();
diff --git a/qemu/roms/ipxe/src/arch/x86_64/Makefile b/qemu/roms/ipxe/src/arch/x86_64/Makefile
index b687f3407..48c0aa1af 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/Makefile
+++ b/qemu/roms/ipxe/src/arch/x86_64/Makefile
@@ -40,6 +40,7 @@ endif
 
 # x86_64-specific directories containing source files
 #
+SRCDIRS		+= arch/x86_64/core
 SRCDIRS		+= arch/x86_64/prefix
 
 # Include common x86 Makefile
diff --git a/qemu/roms/ipxe/src/arch/x86_64/core/setjmp.S b/qemu/roms/ipxe/src/arch/x86_64/core/setjmp.S
new file mode 100644
index 000000000..e43200d7b
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86_64/core/setjmp.S
@@ -0,0 +1,65 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+	.text
+	.code64
+
+	/* Must match jmp_buf structure layout */
+	.struct	0
+env_retaddr:	.quad	0
+env_stack:	.quad	0
+env_rbx:	.quad	0
+env_rbp:	.quad	0
+env_r12:	.quad	0
+env_r13:	.quad	0
+env_r14:	.quad	0
+env_r15:	.quad	0
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
+	.globl	setjmp
+setjmp:
+	/* Save return address */
+	movq	0(%rsp), %rax
+	movq	%rax, env_retaddr(%rdi)
+	/* Save stack pointer */
+	movq	%rsp, env_stack(%rdi)
+	/* Save other registers */
+	movq	%rbx, env_rbx(%rdi)
+	movq	%rbp, env_rbp(%rdi)
+	movq	%r12, env_r12(%rdi)
+	movq	%r13, env_r13(%rdi)
+	movq	%r14, env_r14(%rdi)
+	movq	%r15, env_r15(%rdi)
+	/* Return 0 when returning as setjmp() */
+	xorq	%rax, %rax
+	ret
+	.size	setjmp, . - setjmp
+
+/*
+ * Non-local jump to a saved stack context
+ */
+	.globl	longjmp
+longjmp:
+	/* Get result in %rax */
+	movq	%rsi, %rax
+	/* Force result to non-zero */
+	testq	%rax, %rax
+	jnz	1f
+	incq	%rax
+1:	/* Restore stack pointer */
+	movq	env_stack(%rdi), %rsp
+	/* Restore other registers */
+	movq	env_rbx(%rdi), %rbx
+	movq	env_rbp(%rdi), %rbp
+	movq	env_r12(%rdi), %r12
+	movq	env_r13(%rdi), %r13
+	movq	env_r14(%rdi), %r14
+	movq	env_r15(%rdi), %r15
+	/* Replace return address on the new stack */
+	popq	%rcx	/* discard */
+	pushq	env_retaddr(%rdi)
+	/* Return to setjmp() caller */
+	ret
+	.size	longjmp, . - longjmp
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
index 2e472d98a..d8c5098ef 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/byteswap.h
@@ -9,7 +9,7 @@
 
 #include <stdint.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static inline __attribute__ (( always_inline, const )) uint16_t
 __bswap_variable_16 ( uint16_t x ) {
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/compiler.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
index 51a7eaae2..f70b2e517 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/compiler.h
@@ -1,6 +1,9 @@
 #ifndef _BITS_COMPILER_H
 #define _BITS_COMPILER_H
 
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_X86_64_NONE
+
 #ifndef ASSEMBLY
 
 /** Declare a function with standard calling conventions */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/endian.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/endian.h
deleted file mode 100644
index 413e702db..000000000
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/endian.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef ETHERBOOT_BITS_ENDIAN_H
-#define ETHERBOOT_BITS_ENDIAN_H
-
-#define __BYTE_ORDER __LITTLE_ENDIAN
-
-#endif /* ETHERBOOT_BITS_ENDIAN_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/entropy.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
index 9c64c833b..a9b3bc10e 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/entropy.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_ENTROPY_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h
new file mode 100644
index 000000000..845c182f7
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/hyperv.h
@@ -0,0 +1,75 @@
+#ifndef _BITS_HYPERV_H
+#define _BITS_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/**
+ * Issue hypercall
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v code		Call code
+ * @v in		Input parameters
+ * @v out		Output parameters
+ * @ret status		Status code
+ */
+static inline __attribute__ (( always_inline )) int
+hv_call ( struct hv_hypervisor *hv, unsigned int code, const void *in,
+	  void *out ) {
+	void *hypercall = hv->hypercall;
+	register uint64_t rcx asm ( "rcx" );
+	register uint64_t rdx asm ( "rdx" );
+	register uint64_t r8 asm ( "r8" );
+	uint64_t in_phys;
+	uint64_t out_phys;
+	uint16_t result;
+
+	in_phys = ( ( __builtin_constant_p ( in ) && ( in == NULL ) )
+		       ? 0 : virt_to_phys ( in ) );
+	out_phys = ( ( __builtin_constant_p ( out ) && ( out == NULL ) )
+		       ? 0 : virt_to_phys ( out ) );
+	rcx = code;
+	rdx = in_phys;
+	r8 = out_phys;
+	__asm__ __volatile__ ( "call *%4"
+			       : "=a" ( result ), "+r" ( rcx ), "+r" ( rdx ),
+				 "+r" ( r8 )
+			       : "m" ( hypercall )
+			       : "r9", "r10", "r11", "xmm0", "xmm1", "xmm2",
+				 "xmm3", "xmm4", "xmm5" );
+	return result;
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bits		Bit field
+ * @v bit		Bit to set
+ */
+static inline __attribute__ (( always_inline )) void
+hv_set_bit ( void *bits, unsigned int bit ) {
+	struct {
+		uint64_t qword[ ( bit / 64 ) + 1 ];
+	} *qwords = bits;
+
+	/* Set bit using "lock bts".  Inform compiler that any memory
+	 * from the start of the bit field up to and including the
+	 * qword containing this bit may be modified.  (This is
+	 * overkill but shouldn't matter in practice since we're
+	 * unlikely to subsequently read other bits from the same bit
+	 * field.)
+	 */
+	__asm__ __volatile__ ( "lock bts %1, %0"
+			       : "+m" ( *qwords ) : "Ir" ( bit ) );
+}
+
+#endif /* _BITS_HYPERV_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/profile.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/profile.h
index 6fc16d84b..b7c74fbe7 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/profile.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/reboot.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
index f1bce0540..f9bcd6a7b 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/reboot.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_REBOOT_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
index d33d03cbe..dcab830f6 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/sanboot.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_SANBOOT_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/strings.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/strings.h
index 6ee99a500..3b7911f3b 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/strings.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/strings.h
@@ -1,7 +1,43 @@
 #ifndef _BITS_STRINGS_H
 #define _BITS_STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	long long lsb_minus_one;
+
+	/* If the input value is zero, the BSF instruction returns
+	 * ZF=0 and leaves an undefined value in the output register.
+	 * Perform this check in C rather than asm so that it can be
+	 * omitted in cases where the compiler is able to prove that
+	 * the input is non-zero.
+	 */
+	if ( value ) {
+		__asm__ ( "bsfq %1, %0"
+			  : "=r" ( lsb_minus_one )
+			  : "rm" ( value ) );
+		return ( lsb_minus_one + 1 );
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+
+	return __ffsll ( value );
+}
 
 /**
  * Find last (i.e. most significant) set bit
@@ -13,7 +49,7 @@ static inline __attribute__ (( always_inline )) int __flsll ( long long value ){
 	long long msb_minus_one;
 
 	/* If the input value is zero, the BSR instruction returns
-	 * ZF=1 and leaves an undefined value in the output register.
+	 * ZF=0 and leaves an undefined value in the output register.
 	 * Perform this check in C rather than asm so that it can be
 	 * omitted in cases where the compiler is able to prove that
 	 * the input is non-zero.
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/bits/time.h b/qemu/roms/ipxe/src/arch/x86_64/include/bits/time.h
index 59b355359..aa74fac8c 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/bits/time.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/bits/time.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _BITS_TIME_H */
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h b/qemu/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
index 9a4790fdc..6511c1ad3 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/efi/ipxe/dhcp_arch.h
@@ -4,7 +4,7 @@
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
+ * License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -13,7 +13,12 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _DHCP_ARCH_H
@@ -24,7 +29,7 @@
  * Architecture-specific DHCP options
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h b/qemu/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
index a5816ac35..316243b69 100644
--- a/qemu/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/ipxe/msr.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Read model-specific register
diff --git a/qemu/roms/ipxe/src/arch/x86_64/include/setjmp.h b/qemu/roms/ipxe/src/arch/x86_64/include/setjmp.h
new file mode 100644
index 000000000..69835d9fa
--- /dev/null
+++ b/qemu/roms/ipxe/src/arch/x86_64/include/setjmp.h
@@ -0,0 +1,34 @@
+#ifndef _SETJMP_H
+#define _SETJMP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** A jump buffer */
+typedef struct {
+	/** Saved return address */
+	uint64_t retaddr;
+	/** Saved stack pointer */
+	uint64_t stack;
+	/** Saved %rbx */
+	uint64_t rbx;
+	/** Saved %rbp */
+	uint64_t rbp;
+	/** Saved %r12 */
+	uint64_t r12;
+	/** Saved %r13 */
+	uint64_t r13;
+	/** Saved %r14 */
+	uint64_t r14;
+	/** Saved %r15 */
+	uint64_t r15;
+} jmp_buf[1];
+
+extern int __asmcall __attribute__ (( returns_twice ))
+setjmp ( jmp_buf env );
+
+extern void __asmcall __attribute__ (( noreturn ))
+longjmp ( jmp_buf env, int val );
+
+#endif /* _SETJMP_H */
diff --git a/qemu/roms/ipxe/src/config/.gitignore b/qemu/roms/ipxe/src/config/.gitignore
deleted file mode 100644
index 8e94f32fe..000000000
--- a/qemu/roms/ipxe/src/config/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.buildserial.*
diff --git a/qemu/roms/ipxe/src/config/branding.h b/qemu/roms/ipxe/src/config/branding.h
new file mode 100644
index 000000000..73f00af95
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/branding.h
@@ -0,0 +1,174 @@
+#ifndef CONFIG_BRANDING_H
+#define CONFIG_BRANDING_H
+
+/** @file
+ *
+ * Branding configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * Branding
+ *
+ * Vendors may use these strings to add their own branding to iPXE.
+ * PRODUCT_NAME is displayed prior to any iPXE branding in startup
+ * messages, and PRODUCT_SHORT_NAME is used where a brief product
+ * label is required (e.g. in BIOS boot selection menus).
+ *
+ * To minimise end-user confusion, it's probably a good idea to either
+ * make PRODUCT_SHORT_NAME a substring of PRODUCT_NAME or leave it as
+ * "iPXE".
+ *
+ */
+#define PRODUCT_NAME ""
+#define PRODUCT_SHORT_NAME "iPXE"
+#define PRODUCT_URI "http://ipxe.org"
+
+/*
+ * Tag line
+ *
+ * If your PRODUCT_SHORT_NAME is longer than the four characters used
+ * by "iPXE", then the standard tag line "Open Source Network Boot
+ * Firmware" is unlikely to fit neatly onto the screen.
+ */
+#define PRODUCT_TAG_LINE "Open Source Network Boot Firmware"
+
+/*
+ * Error messages
+ *
+ * iPXE error messages comprise a summary error message
+ * (e.g. "Permission denied") and a 32-bit error number.  This number
+ * is incorporated into an error URI such as
+ *
+ *   "No such file or directory (http://ipxe.org/2d0c613b)"
+ *
+ * or
+ *
+ *   "Operation not supported (http://ipxe.org/3c092003)"
+ *
+ * Users may browse to the URI within the error message, which is
+ * provided by a database running on the iPXE web site
+ * (http://ipxe.org).  This database provides details for all possible
+ * errors generated by iPXE, including:
+ *
+ * - the detailed error message (e.g. "Not an OCSP signing
+ *   certificate") to complement the summary message (e.g. "Permission
+ *   denied") which is compiled into the iPXE binary.
+ *
+ * - an instruction to the user to upgrade, if the error cannot be
+ *   generated by the latest version of iPXE.
+ *
+ * - hints on how to fix the error (e.g. "This error indicates that
+ *   the file was not found on the TFTP server.  Check that you can
+ *   retrieve the file using an alternative TFTP client, such as
+ *   tftp-hpa on Linux.")
+ *
+ * - details of which source file within the iPXE codebase generated
+ *   the error.
+ *
+ * - a direct link to the line(s) of code which generated the error.
+ *
+ * If you have a customer support team and would like your customers
+ * to contact your support team for all problems, instead of using the
+ * existing support infrastructure provided by http://ipxe.org, then
+ * you may define a custom URI to be included within error messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the 32-bit error number.
+ */
+#define PRODUCT_ERROR_URI "http://ipxe.org/%08x"
+
+/*
+ * Command help messages
+ *
+ * iPXE command help messages include a URI constructed from the
+ * command name, such as
+ *
+ *   "See http://ipxe.org/cmd/vcreate for further information"
+ *
+ * The iPXE web site includes documentation for the commands provided
+ * by the iPXE shell, including:
+ *
+ * - details of the command syntax (e.g. "vcreate --tag <tag>
+ *   [--priority <priority>] <trunk interface>").
+ *
+ * - example usages of the command (e.g. "vcreate --tag 123 net0")
+ *
+ * - a formal description of the command (e.g. "Create a VLAN network
+ *   interface on an existing trunk network interface. The new network
+ *   interface will be named by appending a hyphen and the VLAN tag
+ *   value to the trunk network interface name.")
+ *
+ * - details of the possible exit statuses from the command.
+ *
+ * - links to documentation for related commands (e.g. "vdestroy")
+ *
+ * - links to documentation for relevant build options (e.g. "VLAN_CMD").
+ *
+ * - general hints and tips on using the command.
+ *
+ * If you want to provide your own documentation for all of the
+ * commands provided by the iPXE shell, rather than using the existing
+ * support infrastructure provided by http://ipxe.org, then you may
+ * define a custom URI to be included within command help messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the command name.
+ *
+ * [ Please also note that the existing documentation is licensed
+ *   under Creative Commons terms which require attribution to the
+ *   iPXE project and prohibit the alteration or removal of any
+ *   references to "iPXE". ]
+ */
+#define PRODUCT_COMMAND_URI "http://ipxe.org/cmd/%s"
+
+/*
+ * Setting help messages
+ *
+ * iPXE setting help messages include a URI constructed from the
+ * setting name, such as
+ *
+ *   "http://ipxe.org/cfg/initiator-iqn"
+ *
+ * The iPXE web site includes documentation for the settings used by
+ * iPXE, including:
+ *
+ * - details of the corresponding DHCP option number.
+ *
+ * - details of the corresponding ISC dhcpd option name.
+ *
+ * - examples of using the setting from the iPXE command line, or in
+ *   iPXE scripts.
+ *
+ * - examples of configuring the setting via a DHCP server.
+ *
+ * - a formal description of the setting.
+ *
+ * - links to documentation for related settings.
+ *
+ * - links to documentation for relevant build options.
+ *
+ * - general notes about the setting.
+ *
+ * If you want to provide your own documentation for all of the
+ * settings used by iPXE, rather than using the existing support
+ * infrastructure provided by http://ipxe.org, then you may define a
+ * custom URI to be included within setting help messages.
+ *
+ * Note that the custom URI is a printf() format string which must
+ * include a format specifier for the setting name.
+ *
+ * [ Please also note that the existing documentation is licensed
+ *   under Creative Commons terms which require attribution to the
+ *   iPXE project and prohibit the alteration or removal of any
+ *   references to "iPXE". ]
+ */
+#define PRODUCT_SETTING_URI "http://ipxe.org/cfg/%s"
+
+#include <config/local/branding.h>
+
+#endif /* CONFIG_BRANDING_H */
diff --git a/qemu/roms/ipxe/src/config/colour.h b/qemu/roms/ipxe/src/config/colour.h
index 57d20c1db..98198f12f 100644
--- a/qemu/roms/ipxe/src/config/colour.h
+++ b/qemu/roms/ipxe/src/config/colour.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define COLOR_NORMAL_FG		COLOR_WHITE
 #define COLOR_NORMAL_BG		COLOR_BLUE
diff --git a/qemu/roms/ipxe/src/config/config.c b/qemu/roms/ipxe/src/config/config.c
index 6c8b9551a..1dd912c1d 100644
--- a/qemu/roms/ipxe/src/config/config.c
+++ b/qemu/roms/ipxe/src/config/config.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 #include <config/console.h>
@@ -30,33 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * in the final iPXE executable built.
  */
 
-/*
- * Build ID string calculations
- *
- */
-#undef XSTR
-#undef STR
-#define XSTR(s) STR(s)
-#define STR(s) #s
-
-#ifdef BUILD_SERIAL
-#include "config/.buildserial.h"
-#define BUILD_SERIAL_STR " #" XSTR(BUILD_SERIAL_NUM)
-#else
-#define BUILD_SERIAL_STR ""
-#endif
-
-#ifdef BUILD_ID
-#define BUILD_ID_STR " " BUILD_ID
-#else
-#define BUILD_ID_STR ""
-#endif
-
-#if defined(BUILD_ID) || defined(BUILD_SERIAL)
-#define BUILD_STRING " [build" BUILD_ID_STR BUILD_SERIAL_STR "]"
-#else
-#define BUILD_STRING ""
-#endif
+PROVIDE_REQUIRING_SYMBOL();
 
 /*
  * Drag in all requested console types
@@ -67,7 +55,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 REQUIRE_OBJECT ( bios_console );
 #endif
 #ifdef CONSOLE_SERIAL
-REQUIRE_OBJECT ( serial_console );
+REQUIRE_OBJECT ( serial );
 #endif
 #ifdef CONSOLE_DIRECT_VGA
 REQUIRE_OBJECT ( video_subr );
@@ -96,6 +84,9 @@ REQUIRE_OBJECT ( debugcon );
 #ifdef CONSOLE_VESAFB
 REQUIRE_OBJECT ( vesafb );
 #endif
+#ifdef CONSOLE_INT13
+REQUIRE_OBJECT ( int13con );
+#endif
 
 /*
  * Drag in all requested network protocols
@@ -149,6 +140,9 @@ REQUIRE_OBJECT ( slam );
 #ifdef SANBOOT_PROTO_ISCSI
 REQUIRE_OBJECT ( iscsi );
 #endif
+#ifdef SANBOOT_PROTO_HTTP
+REQUIRE_OBJECT ( httpblock );
+#endif
 
 /*
  * Drag in all requested resolvers
@@ -349,6 +343,9 @@ REQUIRE_OBJECT ( cpuid_settings );
 #ifdef MEMMAP_SETTINGS
 REQUIRE_OBJECT ( memmap_settings );
 #endif
+#ifdef VRAM_SETTINGS
+REQUIRE_OBJECT ( vram_settings );
+#endif
 
 /*
  * Drag in selected keyboard map
diff --git a/qemu/roms/ipxe/src/config/config_crypto.c b/qemu/roms/ipxe/src/config/config_crypto.c
new file mode 100644
index 000000000..1e125d8ab
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/config_crypto.c
@@ -0,0 +1,76 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/crypto.h>
+
+/** @file
+ *
+ * Cryptographic configuration
+ *
+ * Cryptographic configuration is slightly messy since we need to drag
+ * in objects based on combinations of build options.
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/* RSA and MD5 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_MD5 )
+REQUIRE_OBJECT ( rsa_md5 );
+#endif
+
+/* RSA and SHA-1 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( rsa_sha1 );
+#endif
+
+/* RSA and SHA-224 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA224 )
+REQUIRE_OBJECT ( rsa_sha224 );
+#endif
+
+/* RSA and SHA-256 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( rsa_sha256 );
+#endif
+
+/* RSA and SHA-384 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA384 )
+REQUIRE_OBJECT ( rsa_sha384 );
+#endif
+
+/* RSA and SHA-512 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_DIGEST_SHA512 )
+REQUIRE_OBJECT ( rsa_sha512 );
+#endif
+
+/* RSA, AES-CBC, and SHA-1 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
+    defined ( CRYPTO_DIGEST_SHA1 )
+REQUIRE_OBJECT ( rsa_aes_cbc_sha1 );
+#endif
+
+/* RSA, AES-CBC, and SHA-256 */
+#if defined ( CRYPTO_PUBKEY_RSA ) && defined ( CRYPTO_CIPHER_AES_CBC ) && \
+    defined ( CRYPTO_DIGEST_SHA256 )
+REQUIRE_OBJECT ( rsa_aes_cbc_sha256 );
+#endif
diff --git a/qemu/roms/ipxe/src/config/config_ethernet.c b/qemu/roms/ipxe/src/config/config_ethernet.c
index d13bd6144..de7a07c57 100644
--- a/qemu/roms/ipxe/src/config/config_ethernet.c
+++ b/qemu/roms/ipxe/src/config/config_ethernet.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Ethernet-specific protocols
  */
@@ -24,3 +40,6 @@ REQUIRE_OBJECT ( aoe );
 #ifdef NET_PROTO_FCOE
 REQUIRE_OBJECT ( fcoe );
 #endif
+#ifdef NET_PROTO_STP
+REQUIRE_OBJECT ( stp );
+#endif
diff --git a/qemu/roms/ipxe/src/config/config_fc.c b/qemu/roms/ipxe/src/config/config_fc.c
index 414646994..33fc9462a 100644
--- a/qemu/roms/ipxe/src/config/config_fc.c
+++ b/qemu/roms/ipxe/src/config/config_fc.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Fibre Channel-specific commands
  *
diff --git a/qemu/roms/ipxe/src/config/config_http.c b/qemu/roms/ipxe/src/config/config_http.c
new file mode 100644
index 000000000..3f198d228
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/config_http.c
@@ -0,0 +1,45 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/general.h>
+
+/** @file
+ *
+ * HTTP extensions
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in HTTP extensions
+ */
+#ifdef HTTP_AUTH_BASIC
+REQUIRE_OBJECT ( httpbasic );
+#endif
+#ifdef HTTP_AUTH_DIGEST
+REQUIRE_OBJECT ( httpdigest );
+#endif
+#ifdef HTTP_ENC_PEERDIST
+REQUIRE_OBJECT ( peerdist );
+#endif
diff --git a/qemu/roms/ipxe/src/config/config_infiniband.c b/qemu/roms/ipxe/src/config/config_infiniband.c
index 432e621d0..a742e7559 100644
--- a/qemu/roms/ipxe/src/config/config_infiniband.c
+++ b/qemu/roms/ipxe/src/config/config_infiniband.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in Infiniband-specific protocols
  */
diff --git a/qemu/roms/ipxe/src/config/config_net80211.c b/qemu/roms/ipxe/src/config/config_net80211.c
index b33c363b1..343617548 100644
--- a/qemu/roms/ipxe/src/config/config_net80211.c
+++ b/qemu/roms/ipxe/src/config/config_net80211.c
@@ -1,8 +1,18 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -15,6 +25,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in 802.11-specific commands
  *
diff --git a/qemu/roms/ipxe/src/config/config_romprefix.c b/qemu/roms/ipxe/src/config/config_romprefix.c
index 85f1e78ab..21921b867 100644
--- a/qemu/roms/ipxe/src/config/config_romprefix.c
+++ b/qemu/roms/ipxe/src/config/config_romprefix.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Provide UNDI loader if PXE stack is requested
  *
diff --git a/qemu/roms/ipxe/src/config/config_route.c b/qemu/roms/ipxe/src/config/config_route.c
index 33e18cdd3..c0b4ee91d 100644
--- a/qemu/roms/ipxe/src/config/config_route.c
+++ b/qemu/roms/ipxe/src/config/config_route.c
@@ -1,11 +1,25 @@
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/general.h>
 
@@ -15,6 +29,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+PROVIDE_REQUIRING_SYMBOL();
+
 /*
  * Drag in routing management for relevant protocols
  *
diff --git a/qemu/roms/ipxe/src/config/config_usb.c b/qemu/roms/ipxe/src/config/config_usb.c
new file mode 100644
index 000000000..dc0e6e6af
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/config_usb.c
@@ -0,0 +1,52 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/usb.h>
+
+/** @file
+ *
+ * USB configuration options
+ *
+ */
+
+PROVIDE_REQUIRING_SYMBOL();
+
+/*
+ * Drag in USB controllers
+ */
+#ifdef USB_HCD_XHCI
+REQUIRE_OBJECT ( xhci );
+#endif
+#ifdef USB_HCD_EHCI
+REQUIRE_OBJECT ( ehci );
+#endif
+#ifdef USB_HCD_UHCI
+REQUIRE_OBJECT ( uhci );
+#endif
+
+/*
+ * Drag in USB peripherals
+ */
+#ifdef USB_KEYBOARD
+REQUIRE_OBJECT ( usbkbd );
+#endif
diff --git a/qemu/roms/ipxe/src/config/console.h b/qemu/roms/ipxe/src/config/console.h
index 908ec5a0b..ffa5cf50d 100644
--- a/qemu/roms/ipxe/src/config/console.h
+++ b/qemu/roms/ipxe/src/config/console.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
@@ -23,6 +23,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 //#define	CONSOLE_VMWARE		/* VMware logfile console */
 //#define	CONSOLE_DEBUGCON	/* Debug port console */
 //#define	CONSOLE_VESAFB		/* VESA framebuffer console */
+//#define	CONSOLE_INT13		/* INT13 disk log console */
 
 #define	KEYBOARD_MAP	us
 
diff --git a/qemu/roms/ipxe/src/config/crypto.h b/qemu/roms/ipxe/src/config/crypto.h
index 1e021b0fb..bccfc04b8 100644
--- a/qemu/roms/ipxe/src/config/crypto.h
+++ b/qemu/roms/ipxe/src/config/crypto.h
@@ -7,7 +7,40 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** RSA public-key algorithm */
+#define CRYPTO_PUBKEY_RSA
+
+/** AES-CBC block cipher */
+#define CRYPTO_CIPHER_AES_CBC
+
+/** MD5 digest algorithm
+ *
+ * Note that use of MD5 is implicit when using TLSv1.1 or earlier.
+ */
+#define CRYPTO_DIGEST_MD5
+
+/** SHA-1 digest algorithm
+ *
+ * Note that use of SHA-1 is implicit when using TLSv1.1 or earlier.
+ */
+#define CRYPTO_DIGEST_SHA1
+
+/** SHA-224 digest algorithm */
+#define CRYPTO_DIGEST_SHA224
+
+/** SHA-256 digest algorithm
+ *
+ * Note that use of SHA-256 is implicit when using TLSv1.2.
+ */
+#define CRYPTO_DIGEST_SHA256
+
+/** SHA-384 digest algorithm */
+#define CRYPTO_DIGEST_SHA384
+
+/** SHA-512 digest algorithm */
+#define CRYPTO_DIGEST_SHA512
 
 /** Margin of error (in seconds) allowed in signed timestamps
  *
diff --git a/qemu/roms/ipxe/src/config/defaults.h b/qemu/roms/ipxe/src/config/defaults.h
index 389c0b07b..32d6dbcce 100644
--- a/qemu/roms/ipxe/src/config/defaults.h
+++ b/qemu/roms/ipxe/src/config/defaults.h
@@ -1,7 +1,7 @@
 #ifndef CONFIG_DEFAULTS_H
 #define CONFIG_DEFAULTS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define CONFIG_DEFAULTS(_platform) <config/defaults/_platform.h>
 
diff --git a/qemu/roms/ipxe/src/config/defaults/efi.h b/qemu/roms/ipxe/src/config/defaults/efi.h
index 4276d9366..cdf41c54d 100644
--- a/qemu/roms/ipxe/src/config/defaults/efi.h
+++ b/qemu/roms/ipxe/src/config/defaults/efi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define UACCESS_EFI
 #define IOAPI_X86
@@ -19,8 +19,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define SMBIOS_EFI
 #define SANBOOT_NULL
 #define BOFM_EFI
-#define ENTROPY_NULL
-#define TIME_NULL
+#define ENTROPY_EFI
+#define TIME_EFI
 #define REBOOT_EFI
 
 #define	IMAGE_EFI		/* EFI image support */
diff --git a/qemu/roms/ipxe/src/config/defaults/pcbios.h b/qemu/roms/ipxe/src/config/defaults/pcbios.h
index 7debc8d2f..3ed8343ce 100644
--- a/qemu/roms/ipxe/src/config/defaults/pcbios.h
+++ b/qemu/roms/ipxe/src/config/defaults/pcbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define UACCESS_LIBRM
 #define IOAPI_X86
@@ -35,6 +35,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define	SANBOOT_PROTO_AOE	/* AoE protocol */
 #define	SANBOOT_PROTO_IB_SRP	/* Infiniband SCSI RDMA protocol */
 #define	SANBOOT_PROTO_FCP	/* Fibre Channel protocol */
+#define SANBOOT_PROTO_HTTP	/* HTTP SAN protocol */
+
+#define	USB_HCD_XHCI		/* xHCI USB host controller */
+#define	USB_HCD_EHCI		/* EHCI USB host controller */
+#define	USB_HCD_UHCI		/* UHCI USB host controller */
+#define	USB_KEYBOARD		/* USB keyboards */
 
 #define	REBOOT_CMD		/* Reboot command */
 #define	CPUID_CMD		/* x86 CPU feature detection command */
diff --git a/qemu/roms/ipxe/src/config/dhcp.h b/qemu/roms/ipxe/src/config/dhcp.h
new file mode 100644
index 000000000..49fe16b92
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/dhcp.h
@@ -0,0 +1,87 @@
+#ifndef CONFIG_DHCP_H
+#define CONFIG_DHCP_H
+
+/** @file
+ *
+ * DHCP configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * DHCP and PXE Boot Server timeout parameters
+ *
+ * Initial and final timeout for DHCP discovery
+ *
+ * The PXE spec indicates discover request are sent 4 times, with
+ * timeouts of 4, 8, 16, 32 seconds.  iPXE by default uses 1, 2, 4, 8.
+ */
+#define DHCP_DISC_START_TIMEOUT_SEC	1
+#define DHCP_DISC_END_TIMEOUT_SEC	10
+//#define DHCP_DISC_START_TIMEOUT_SEC	4	/* as per PXE spec */
+//#define DHCP_DISC_END_TIMEOUT_SEC	32	/* as per PXE spec */
+
+/*
+ * ProxyDHCP offers are given precedence by continue to wait for them
+ * after a valid DHCPOFFER is received.  We'll wait through this
+ * timeout for it.  The PXE spec indicates waiting through the 4 & 8
+ * second timeouts, iPXE by default stops after 2.
+ */
+#define DHCP_DISC_PROXY_TIMEOUT_SEC	2
+//#define DHCP_DISC_PROXY_TIMEOUT_SEC	11	/* as per PXE spec */
+
+/*
+ * Per the PXE spec, requests are also tried 4 times, but at timeout
+ * intervals of 1, 2, 3, 4 seconds.  To adapt this to an exponential
+ * backoff timer, we can either do 1, 2, 4, 8, ie. 4 retires with a
+ * longer interval or start at 0 (0.25s) for 0.25, 0.5, 1, 2, 4,
+ * ie. one extra try and shorter initial timeouts.  iPXE by default
+ * does a combination of both, starting at 0 and going through the 8
+ * second timeout.
+ */
+#define DHCP_REQ_START_TIMEOUT_SEC	0
+#define DHCP_REQ_END_TIMEOUT_SEC	10
+//#define DHCP_REQ_END_TIMEOUT_SEC	4	/* as per PXE spec */
+
+/*
+ * A ProxyDHCP offer without PXE options also goes through a request
+ * phase using these same parameters, but note the early break below.
+ */
+#define DHCP_PROXY_START_TIMEOUT_SEC	0
+#define DHCP_PROXY_END_TIMEOUT_SEC	10
+//#define DHCP_PROXY_END_TIMEOUT_SEC	8	/* as per PXE spec */
+
+/*
+ * A ProxyDHCP request timeout should not induce a failure condition,
+ * so we always want to break before the above set of timers expire.
+ * The iPXE default value of 2 breaks at the first timeout after 2
+ * seconds, which will be after the 2 second timeout.
+ */
+#define DHCP_REQ_PROXY_TIMEOUT_SEC	2
+//#define DHCP_REQ_PROXY_TIMEOUT_SEC	7	/* as per PXE spec */
+
+/*
+ * Per the PXE spec, a PXE boot server request is also be retried 4
+ * times at timeouts of 1, 2, 3, 4.  iPXE uses the same timeouts as
+ * discovery, 1, 2, 4, 8, but will move on to the next server if
+ * available after an elapsed time greater than 3 seconds, therefore
+ * effectively only sending 3 tries at timeouts of 1, 2, 4.
+ */
+#define PXEBS_START_TIMEOUT_SEC		1
+#define PXEBS_END_TIMEOUT_SEC		10
+//#define PXEBS_START_TIMEOUT_SEC	0	/* as per PXE spec */
+//#define PXEBS_END_TIMEOUT_SEC		8	/* as per PXE spec */
+
+/*
+ * Increment to the next PXE Boot server, if available, after this
+ * this much time has elapsed.
+ */
+#define PXEBS_MAX_TIMEOUT_SEC		3
+//#define PXEBS_MAX_TIMEOUT_SEC		7	/* as per PXE spec */
+
+#include <config/local/dhcp.h>
+
+#endif /* CONFIG_DHCP_H */
diff --git a/qemu/roms/ipxe/src/config/entropy.h b/qemu/roms/ipxe/src/config/entropy.h
index 7de2f6737..c79060fd5 100644
--- a/qemu/roms/ipxe/src/config/entropy.h
+++ b/qemu/roms/ipxe/src/config/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/fault.h b/qemu/roms/ipxe/src/config/fault.h
new file mode 100644
index 000000000..5024a8ff3
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/fault.h
@@ -0,0 +1,34 @@
+#ifndef CONFIG_FAULT_H
+#define CONFIG_FAULT_H
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/* Drop every N transmitted or received network packets */
+#define	NETDEV_DISCARD_RATE 0
+
+/* Drop every N transmitted or received PeerDist discovery packets */
+#define PEERDISC_DISCARD_RATE 0
+
+/* Annul every N PeerDist download attempts */
+#define PEERBLK_ANNUL_RATE 0
+
+/* Stall every N PeerDist download attempts */
+#define PEERBLK_STALL_RATE 0
+
+/* Abort every N PeerDist download attempts */
+#define PEERBLK_ABORT_RATE 0
+
+/* Corrupt every N received PeerDist packets */
+#define PEERBLK_CORRUPT_RATE 0
+
+#include <config/local/fault.h>
+
+#endif /* CONFIG_FAULT_H */
diff --git a/qemu/roms/ipxe/src/config/general.h b/qemu/roms/ipxe/src/config/general.h
index 539203457..ee15f6bf1 100644
--- a/qemu/roms/ipxe/src/config/general.h
+++ b/qemu/roms/ipxe/src/config/general.h
@@ -7,27 +7,11 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
 /*
- * Branding
- *
- * Vendors may use these strings to add their own branding to iPXE.
- * PRODUCT_NAME is displayed prior to any iPXE branding in startup
- * messages, and PRODUCT_SHORT_NAME is used where a brief product
- * label is required (e.g. in BIOS boot selection menus).
- *
- * To minimise end-user confusion, it's probably a good idea to either
- * make PRODUCT_SHORT_NAME a substring of PRODUCT_NAME or leave it as
- * "iPXE".
- *
- */
-#define PRODUCT_NAME ""
-#define PRODUCT_SHORT_NAME "iPXE"
-
-/*
  * Banner timeout configuration
  *
  * This controls the timeout for the "Press Ctrl-B for the iPXE
@@ -53,6 +37,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define	NET_PROTO_IPV4		/* IPv4 protocol */
 #undef	NET_PROTO_IPV6		/* IPv6 protocol */
 #undef	NET_PROTO_FCOE		/* Fibre Channel over Ethernet protocol */
+#define	NET_PROTO_STP		/* Spanning Tree protocol */
 
 /*
  * PXE support
@@ -82,6 +67,15 @@ FILE_LICENCE ( GPL2_OR_LATER );
 //#undef	SANBOOT_PROTO_AOE	/* AoE protocol */
 //#undef	SANBOOT_PROTO_IB_SRP	/* Infiniband SCSI RDMA protocol */
 //#undef	SANBOOT_PROTO_FCP	/* Fibre Channel protocol */
+//#undef	SANBOOT_PROTO_HTTP	/* HTTP SAN protocol */
+
+/*
+ * HTTP extensions
+ *
+ */
+#define HTTP_AUTH_BASIC		/* Basic authentication */
+#define HTTP_AUTH_DIGEST	/* Digest authentication */
+//#define HTTP_ENC_PEERDIST	/* PeerDist content encoding */
 
 /*
  * 802.11 cryptosystems and handshaking protocols
@@ -156,6 +150,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 #undef	NONPNP_HOOK_INT19	/* Hook INT19 on non-PnP BIOSes */
+#define	AUTOBOOT_ROM_FILTER	/* Autoboot only devices matching our ROM */
 
 /*
  * Error message tables to include
@@ -170,7 +165,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#define	NETDEV_DISCARD_RATE 0	/* Drop every N packets (0=>no drop) */
 #undef	BUILD_SERIAL		/* Include an automatic build serial
 				 * number.  Add "bs" to the list of
 				 * make targets.  For example:
@@ -181,6 +175,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #undef	GDBSERIAL		/* Remote GDB debugging over serial */
 #undef	GDBUDP			/* Remote GDB debugging over UDP
 				 * (both may be set) */
+//#define EFI_DOWNGRADE_UX	/* Downgrade UEFI user experience */
 
 #include <config/named.h>
 #include NAMED_CONFIG(general.h)
diff --git a/qemu/roms/ipxe/src/config/ioapi.h b/qemu/roms/ipxe/src/config/ioapi.h
index ce19c6d71..abe5a50ce 100644
--- a/qemu/roms/ipxe/src/config/ioapi.h
+++ b/qemu/roms/ipxe/src/config/ioapi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/named.h b/qemu/roms/ipxe/src/config/named.h
index 36efdabdd..ddde6f0a6 100644
--- a/qemu/roms/ipxe/src/config/named.h
+++ b/qemu/roms/ipxe/src/config/named.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* config/<name>/<header>.h */
 #ifdef CONFIG
diff --git a/qemu/roms/ipxe/src/config/nap.h b/qemu/roms/ipxe/src/config/nap.h
index 187af4289..e4fe97964 100644
--- a/qemu/roms/ipxe/src/config/nap.h
+++ b/qemu/roms/ipxe/src/config/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/tests/qapi-schema/alternate-good.err b/qemu/roms/ipxe/src/config/qemu/colour.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/alternate-good.err
+++ b/qemu/roms/ipxe/src/config/qemu/colour.h
diff --git a/qemu/tests/qapi-schema/data-array-empty.out b/qemu/roms/ipxe/src/config/qemu/console.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-array-empty.out
+++ b/qemu/roms/ipxe/src/config/qemu/console.h
diff --git a/qemu/tests/qapi-schema/data-array-unknown.out b/qemu/roms/ipxe/src/config/qemu/crypto.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-array-unknown.out
+++ b/qemu/roms/ipxe/src/config/qemu/crypto.h
diff --git a/qemu/roms/ipxe/src/config/qemu/general.h b/qemu/roms/ipxe/src/config/qemu/general.h
new file mode 100644
index 000000000..30f60d3f7
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/qemu/general.h
@@ -0,0 +1,10 @@
+/* Disable entry during POST */
+#undef ROM_BANNER_TIMEOUT
+#define ROM_BANNER_TIMEOUT 0
+
+/* Extend banner timeout */
+#undef BANNER_TIMEOUT
+#define BANNER_TIMEOUT 30
+
+/* Work around missing EFI_PXE_BASE_CODE_PROTOCOL */
+#define EFI_DOWNGRADE_UX
diff --git a/qemu/tests/qapi-schema/data-int.out b/qemu/roms/ipxe/src/config/qemu/serial.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-int.out
+++ b/qemu/roms/ipxe/src/config/qemu/serial.h
diff --git a/qemu/tests/qapi-schema/data-member-array-bad.out b/qemu/roms/ipxe/src/config/qemu/settings.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-member-array-bad.out
+++ b/qemu/roms/ipxe/src/config/qemu/settings.h
diff --git a/qemu/tests/qapi-schema/data-member-array.err b/qemu/roms/ipxe/src/config/qemu/sideband.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-member-array.err
+++ b/qemu/roms/ipxe/src/config/qemu/sideband.h
diff --git a/qemu/tests/qapi-schema/data-member-unknown.out b/qemu/roms/ipxe/src/config/qemu/usb.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-member-unknown.out
+++ b/qemu/roms/ipxe/src/config/qemu/usb.h
diff --git a/qemu/roms/ipxe/src/config/reboot.h b/qemu/roms/ipxe/src/config/reboot.h
index 240ef87be..2d1648e7b 100644
--- a/qemu/roms/ipxe/src/config/reboot.h
+++ b/qemu/roms/ipxe/src/config/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/sanboot.h b/qemu/roms/ipxe/src/config/sanboot.h
index 1d7f5f177..ccc4bda1f 100644
--- a/qemu/roms/ipxe/src/config/sanboot.h
+++ b/qemu/roms/ipxe/src/config/sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/serial.h b/qemu/roms/ipxe/src/config/serial.h
index 08368efdb..27040dc54 100644
--- a/qemu/roms/ipxe/src/config/serial.h
+++ b/qemu/roms/ipxe/src/config/serial.h
@@ -13,11 +13,6 @@
 
 FILE_LICENCE ( GPL2_OR_LATER );
 
-#define	COM1		0x3f8
-#define	COM2		0x2f8
-#define	COM3		0x3e8
-#define	COM4		0x2e8
-
 #define	COMCONSOLE	COM1		/* I/O port address */
 
 /* Keep settings from a previous user of the serial port (e.g. lilo or
diff --git a/qemu/roms/ipxe/src/config/settings.h b/qemu/roms/ipxe/src/config/settings.h
index 42fe9cc81..01feaaa87 100644
--- a/qemu/roms/ipxe/src/config/settings.h
+++ b/qemu/roms/ipxe/src/config/settings.h
@@ -7,12 +7,13 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define	PCI_SETTINGS	/* PCI device settings */
 //#define	CPUID_SETTINGS	/* CPUID settings */
 //#define	MEMMAP_SETTINGS	/* Memory map settings */
 //#define	VMWARE_SETTINGS	/* VMware GuestInfo settings */
+//#define	VRAM_SETTINGS	/* Video RAM dump settings */
 
 #include <config/named.h>
 #include NAMED_CONFIG(settings.h)
diff --git a/qemu/roms/ipxe/src/config/sideband.h b/qemu/roms/ipxe/src/config/sideband.h
index 039bb5d09..dd704f9bb 100644
--- a/qemu/roms/ipxe/src/config/sideband.h
+++ b/qemu/roms/ipxe/src/config/sideband.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 //#define	CONFIG_BOFM	/* IBM's BladeCenter Open Fabric Manager */
 
diff --git a/qemu/roms/ipxe/src/config/time.h b/qemu/roms/ipxe/src/config/time.h
index 0576211fd..678f6f864 100644
--- a/qemu/roms/ipxe/src/config/time.h
+++ b/qemu/roms/ipxe/src/config/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/timer.h b/qemu/roms/ipxe/src/config/timer.h
index abd669851..5a54d398c 100644
--- a/qemu/roms/ipxe/src/config/timer.h
+++ b/qemu/roms/ipxe/src/config/timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/umalloc.h b/qemu/roms/ipxe/src/config/umalloc.h
index 245c6b4aa..832dd21d1 100644
--- a/qemu/roms/ipxe/src/config/umalloc.h
+++ b/qemu/roms/ipxe/src/config/umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <config/defaults.h>
 
diff --git a/qemu/roms/ipxe/src/config/usb.h b/qemu/roms/ipxe/src/config/usb.h
new file mode 100644
index 000000000..52e82eaad
--- /dev/null
+++ b/qemu/roms/ipxe/src/config/usb.h
@@ -0,0 +1,33 @@
+#ifndef CONFIG_USB_H
+#define CONFIG_USB_H
+
+/** @file
+ *
+ * USB configuration
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <config/defaults.h>
+
+/*
+ * USB host controllers (all enabled by default)
+ *
+ */
+//#undef	USB_HCD_XHCI		/* xHCI USB host controller */
+//#undef	USB_HCD_EHCI		/* EHCI USB host controller */
+//#undef	USB_HCD_UHCI		/* UHCI USB host controller */
+
+/*
+ * USB peripherals
+ *
+ */
+//#undef	USB_KEYBOARD		/* USB keyboards */
+
+#include <config/named.h>
+#include NAMED_CONFIG(usb.h)
+#include <config/local/usb.h>
+#include LOCAL_NAMED_CONFIG(usb.h)
+
+#endif /* CONFIG_USB_H */
diff --git a/qemu/roms/ipxe/src/config/vbox/general.h b/qemu/roms/ipxe/src/config/vbox/general.h
index 27d15daf2..06b45f1a8 100644
--- a/qemu/roms/ipxe/src/config/vbox/general.h
+++ b/qemu/roms/ipxe/src/config/vbox/general.h
@@ -1,25 +1,17 @@
 /* Disabled from config/defaults/pcbios.h */
 
-#undef IMAGE_ELF
 #undef SANBOOT_PROTO_ISCSI
 #undef SANBOOT_PROTO_AOE
 #undef SANBOOT_PROTO_IB_SRP
 #undef SANBOOT_PROTO_FCP
-#undef REBOOT_CMD
-#undef CPUID_CMD
 
 /* Disabled from config/general.h */
 
-#undef DOWNLOAD_PROTO_HTTP
 #undef CRYPTO_80211_WEP
 #undef CRYPTO_80211_WPA
 #undef CRYPTO_80211_WPA2
 #undef IWMGMT_CMD
-#undef FCMGMT_CMD
-#undef SANBOOT_CMD
 #undef MENU_CMD
-#undef LOGIN_CMD
-#undef SYNC_CMD
 
 /* Ensure ROM banner is not displayed */
 
diff --git a/qemu/tests/qapi-schema/data-unknown.out b/qemu/roms/ipxe/src/config/vbox/usb.h
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/data-unknown.out
+++ b/qemu/roms/ipxe/src/config/vbox/usb.h
diff --git a/qemu/roms/ipxe/src/core/acpi.c b/qemu/roms/ipxe/src/core/acpi.c
index 330f50631..b0ccfa78d 100644
--- a/qemu/roms/ipxe/src/core/acpi.c
+++ b/qemu/roms/ipxe/src/core/acpi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/acpi.h>
diff --git a/qemu/roms/ipxe/src/core/ansicol.c b/qemu/roms/ipxe/src/core/ansicol.c
index 142a00f8d..ddf9ba77c 100644
--- a/qemu/roms/ipxe/src/core/ansicol.c
+++ b/qemu/roms/ipxe/src/core/ansicol.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/core/ansicoldef.c b/qemu/roms/ipxe/src/core/ansicoldef.c
index dd89f3b70..6d8598e11 100644
--- a/qemu/roms/ipxe/src/core/ansicoldef.c
+++ b/qemu/roms/ipxe/src/core/ansicoldef.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/core/ansiesc.c b/qemu/roms/ipxe/src/core/ansiesc.c
index ca9a73ce0..7f545db0e 100644
--- a/qemu/roms/ipxe/src/core/ansiesc.c
+++ b/qemu/roms/ipxe/src/core/ansiesc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/qemu/roms/ipxe/src/core/asprintf.c b/qemu/roms/ipxe/src/core/asprintf.c
index 03cf45cfc..00edf8e11 100644
--- a/qemu/roms/ipxe/src/core/asprintf.c
+++ b/qemu/roms/ipxe/src/core/asprintf.c
@@ -4,7 +4,7 @@
 #include <stdio.h>
 #include <errno.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Write a formatted string to newly allocated memory.
diff --git a/qemu/roms/ipxe/src/core/assert.c b/qemu/roms/ipxe/src/core/assert.c
index 0791ea7b9..294e766be 100644
--- a/qemu/roms/ipxe/src/core/assert.c
+++ b/qemu/roms/ipxe/src/core/assert.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/base16.c b/qemu/roms/ipxe/src/core/base16.c
index bf9cc21bb..f9e0f3364 100644
--- a/qemu/roms/ipxe/src/core/base16.c
+++ b/qemu/roms/ipxe/src/core/base16.c
@@ -15,14 +15,20 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
-#include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
+#include <ipxe/string.h>
+#include <ipxe/vsprintf.h>
 #include <ipxe/base16.h>
 
 /** @file
@@ -32,48 +38,42 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /**
- * Base16-encode data
+ * Encode hexadecimal string (with optional byte separator character)
  *
+ * @v separator		Byte separator character, or 0 for no separator
  * @v raw		Raw data
- * @v len		Length of raw data
- * @v encoded		Buffer for encoded string
- *
- * The buffer must be the correct length for the encoded string.  Use
- * something like
- *
- *     char buf[ base16_encoded_len ( len ) + 1 ];
- *
- * (the +1 is for the terminating NUL) to provide a buffer of the
- * correct size.
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
  */
-void base16_encode ( const uint8_t *raw, size_t len, char *encoded ) {
-	const uint8_t *raw_bytes = raw;
-	char *encoded_bytes = encoded;
-	size_t remaining = len;
-
-	/* Encode each byte */
-	for ( ; remaining-- ; encoded_bytes += 2 ) {
-		sprintf ( encoded_bytes, "%02x", *(raw_bytes++) );
+size_t hex_encode ( char separator, const void *raw, size_t raw_len,
+		    char *data, size_t len ) {
+	const uint8_t *bytes = raw;
+	const char delimiter[2] = { separator, '\0' };
+	size_t used = 0;
+	unsigned int i;
+
+	if ( len )
+		data[0] = 0; /* Ensure that a terminating NUL exists */
+	for ( i = 0 ; i < raw_len ; i++ ) {
+		used += ssnprintf ( ( data + used ), ( len - used ),
+				    "%s%02x", ( used ? delimiter : "" ),
+				    bytes[i] );
 	}
-
-	/* Ensure terminating NUL exists even if length was zero */
-	*encoded_bytes = '\0';
-
-	DBG ( "Base16-encoded to \"%s\":\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( strlen ( encoded ) == base16_encoded_len ( len ) );
+	return used;
 }
 
 /**
- * Decode hexadecimal string
+ * Decode hexadecimal string (with optional byte separator character)
  *
- * @v encoded		Encoded string
  * @v separator		Byte separator character, or 0 for no separator
+ * @v encoded		Encoded string
  * @v data		Buffer
  * @v len		Length of buffer
  * @ret len		Length of data, or negative error
  */
-int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
+int hex_decode ( char separator, const char *encoded, void *data, size_t len ) {
 	uint8_t *out = data;
 	unsigned int count = 0;
 	unsigned int sixteens;
@@ -87,13 +87,13 @@ int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
 
 		/* Extract digits.  Note that either digit may be NUL,
 		 * which would be interpreted as an invalid value by
-		 * strtoul_charval(); there is therefore no need for an
+		 * digit_value(); there is therefore no need for an
 		 * explicit end-of-string check.
 		 */
-		sixteens = strtoul_charval ( *(encoded++) );
+		sixteens = digit_value ( *(encoded++) );
 		if ( sixteens >= 16 )
 			return -EINVAL;
-		units = strtoul_charval ( *(encoded++) );
+		units = digit_value ( *(encoded++) );
 		if ( units >= 16 )
 			return -EINVAL;
 
@@ -105,31 +105,3 @@ int hex_decode ( const char *encoded, char separator, void *data, size_t len ) {
 	}
 	return count;
 }
-
-/**
- * Base16-decode data
- *
- * @v encoded		Encoded string
- * @v raw		Raw data
- * @ret len		Length of raw data, or negative error
- *
- * The buffer must be large enough to contain the decoded data.  Use
- * something like
- *
- *     char buf[ base16_decoded_max_len ( encoded ) ];
- *
- * to provide a buffer of the correct size.
- */
-int base16_decode ( const char *encoded, uint8_t *raw ) {
-	int len;
-
-	len = hex_decode ( encoded, 0, raw, -1UL );
-	if ( len < 0 )
-		return len;
-
-	DBG ( "Base16-decoded \"%s\" to:\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( len <= ( int ) base16_decoded_max_len ( encoded ) );
-
-	return len;
-}
diff --git a/qemu/roms/ipxe/src/core/base64.c b/qemu/roms/ipxe/src/core/base64.c
index bdaf70957..e452f7d41 100644
--- a/qemu/roms/ipxe/src/core/base64.c
+++ b/qemu/roms/ipxe/src/core/base64.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -39,80 +43,73 @@ static const char base64[64] =
  * Base64-encode data
  *
  * @v raw		Raw data
- * @v len		Length of raw data
- * @v encoded		Buffer for encoded string
- *
- * The buffer must be the correct length for the encoded string.  Use
- * something like
- *
- *     char buf[ base64_encoded_len ( len ) + 1 ];
- *
- * (the +1 is for the terminating NUL) to provide a buffer of the
- * correct size.
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
  */
-void base64_encode ( const uint8_t *raw, size_t len, char *encoded ) {
+size_t base64_encode ( const void *raw, size_t raw_len, char *data,
+		       size_t len ) {
 	const uint8_t *raw_bytes = ( ( const uint8_t * ) raw );
-	uint8_t *encoded_bytes = ( ( uint8_t * ) encoded );
-	size_t raw_bit_len = ( 8 * len );
+	size_t raw_bit_len = ( 8 * raw_len );
+	size_t used = 0;
 	unsigned int bit;
 	unsigned int byte;
 	unsigned int shift;
 	unsigned int tmp;
 
-	for ( bit = 0 ; bit < raw_bit_len ; bit += 6 ) {
+	for ( bit = 0 ; bit < raw_bit_len ; bit += 6, used++ ) {
 		byte = ( bit / 8 );
 		shift = ( bit % 8 );
 		tmp = ( raw_bytes[byte] << shift );
-		if ( ( byte + 1 ) < len )
+		if ( ( byte + 1 ) < raw_len )
 			tmp |= ( raw_bytes[ byte + 1 ] >> ( 8 - shift ) );
 		tmp = ( ( tmp >> 2 ) & 0x3f );
-		*(encoded_bytes++) = base64[tmp];
+		if ( used < len )
+			data[used] = base64[tmp];
+	}
+	for ( ; ( bit % 8 ) != 0 ; bit += 6, used++ ) {
+		if ( used < len )
+			data[used] = '=';
 	}
-	for ( ; ( bit % 8 ) != 0 ; bit += 6 )
-		*(encoded_bytes++) = '=';
-	*(encoded_bytes++) = '\0';
+	if ( used < len )
+		data[used] = '\0';
+	if ( len )
+		data[ len - 1 ] = '\0'; /* Ensure terminator exists */
 
-	DBG ( "Base64-encoded to \"%s\":\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( strlen ( encoded ) == base64_encoded_len ( len ) );
+	return used;
 }
 
 /**
  * Base64-decode string
  *
  * @v encoded		Encoded string
- * @v raw		Raw data
- * @ret len		Length of raw data, or negative error
- *
- * The buffer must be large enough to contain the decoded data.  Use
- * something like
- *
- *     char buf[ base64_decoded_max_len ( encoded ) ];
- *
- * to provide a buffer of the correct size.
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of data, or negative error
  */
-int base64_decode ( const char *encoded, uint8_t *raw ) {
-	const uint8_t *encoded_bytes = ( ( const uint8_t * ) encoded );
-	uint8_t *raw_bytes = ( ( uint8_t * ) raw );
-	uint8_t encoded_byte;
+int base64_decode ( const char *encoded, void *data, size_t len ) {
+	const char *in = encoded;
+	uint8_t *out = data;
+	uint8_t in_char;
 	char *match;
-	int decoded;
+	int in_bits;
 	unsigned int bit = 0;
 	unsigned int pad_count = 0;
-	size_t len;
+	size_t offset;
 
-	/* Zero the raw data */
-	memset ( raw, 0, base64_decoded_max_len ( encoded ) );
+	/* Zero the output buffer */
+	memset ( data, 0, len );
 
 	/* Decode string */
-	while ( ( encoded_byte = *(encoded_bytes++) ) ) {
+	while ( ( in_char = *(in++) ) ) {
 
 		/* Ignore whitespace characters */
-		if ( isspace ( encoded_byte ) )
+		if ( isspace ( in_char ) )
 			continue;
 
 		/* Process pad characters */
-		if ( encoded_byte == '=' ) {
+		if ( in_char == '=' ) {
 			if ( pad_count >= 2 ) {
 				DBG ( "Base64-encoded string \"%s\" has too "
 				      "many pad characters\n", encoded );
@@ -129,18 +126,22 @@ int base64_decode ( const char *encoded, uint8_t *raw ) {
 		}
 
 		/* Process normal characters */
-		match = strchr ( base64, encoded_byte );
+		match = strchr ( base64, in_char );
 		if ( ! match ) {
 			DBG ( "Base64-encoded string \"%s\" contains invalid "
-			      "character '%c'\n", encoded, encoded_byte );
+			      "character '%c'\n", encoded, in_char );
 			return -EINVAL;
 		}
-		decoded = ( match - base64 );
+		in_bits = ( match - base64 );
 
 		/* Add to raw data */
-		decoded <<= 2;
-		raw_bytes[ bit / 8 ] |= ( decoded >> ( bit % 8 ) );
-		raw_bytes[ bit / 8 + 1 ] |= ( decoded << ( 8 - ( bit % 8 ) ) );
+		in_bits <<= 2;
+		offset = ( bit / 8 );
+		if ( offset < len )
+			out[offset] |= ( in_bits >> ( bit % 8 ) );
+		offset++;
+		if ( offset < len )
+			out[offset] |= ( in_bits << ( 8 - ( bit % 8 ) ) );
 		bit += 6;
 	}
 
@@ -150,12 +151,7 @@ int base64_decode ( const char *encoded, uint8_t *raw ) {
 		      "%d\n", encoded, bit );
 		return -EINVAL;
 	}
-	len = ( bit / 8 );
-
-	DBG ( "Base64-decoded \"%s\" to:\n", encoded );
-	DBG_HDA ( 0, raw, len );
-	assert ( len <= base64_decoded_max_len ( encoded ) );
 
 	/* Return length in bytes */
-	return ( len );
+	return ( bit / 8 );
 }
diff --git a/qemu/roms/ipxe/src/core/basename.c b/qemu/roms/ipxe/src/core/basename.c
index b534a7886..f4f929517 100644
--- a/qemu/roms/ipxe/src/core/basename.c
+++ b/qemu/roms/ipxe/src/core/basename.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/core/bitmap.c b/qemu/roms/ipxe/src/core/bitmap.c
index 0d1152327..2aac33870 100644
--- a/qemu/roms/ipxe/src/core/bitmap.c
+++ b/qemu/roms/ipxe/src/core/bitmap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/bitmap.h>
diff --git a/qemu/roms/ipxe/src/core/blockdev.c b/qemu/roms/ipxe/src/core/blockdev.c
index 9d118cb2f..c219d9673 100644
--- a/qemu/roms/ipxe/src/core/blockdev.c
+++ b/qemu/roms/ipxe/src/core/blockdev.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/interface.h>
diff --git a/qemu/roms/ipxe/src/core/blocktrans.c b/qemu/roms/ipxe/src/core/blocktrans.c
new file mode 100644
index 000000000..3f32f9cf8
--- /dev/null
+++ b/qemu/roms/ipxe/src/core/blocktrans.c
@@ -0,0 +1,261 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Block device translator
+ *
+ */
+
+#include <stdlib.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/xfer.h>
+#include <ipxe/blockdev.h>
+#include <ipxe/blocktrans.h>
+
+/**
+ * Reallocate block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int blktrans_xferbuf_realloc ( struct xfer_buffer *xferbuf,
+				      size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Record length, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* We have a (non-reallocatable) data buffer */
+		return -ENOTSUP;
+
+	} else {
+
+		/* Record length (for block device capacity) */
+		xferbuf->len = len;
+		return 0;
+	}
+}
+
+/**
+ * Write data to block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void blktrans_xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+				     const void *data, size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Write data to buffer, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* Write data to buffer */
+		copy_to_user ( blktrans->buffer, offset, data, len );
+
+	} else {
+
+		/* Sanity check */
+		assert ( len == 0 );
+	}
+}
+
+/**
+ * Read data from block device translator data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void blktrans_xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+				    void *data, size_t len ) {
+	struct block_translator *blktrans =
+		container_of ( xferbuf, struct block_translator, xferbuf );
+
+	/* Read data from buffer, if applicable */
+	if ( blktrans->buffer ) {
+
+		/* Read data from buffer */
+		copy_from_user ( data, blktrans->buffer, offset, len );
+
+	} else {
+
+		/* Sanity check */
+		assert ( len == 0 );
+	}
+}
+
+/** Block device translator data transfer buffer operations */
+static struct xfer_buffer_operations blktrans_xferbuf_operations = {
+	.realloc = blktrans_xferbuf_realloc,
+	.write = blktrans_xferbuf_write,
+	.read = blktrans_xferbuf_read,
+};
+
+/**
+ * Close block device translator
+ *
+ * @v blktrans		Block device translator
+ * @v rc		Reason for close
+ */
+static void blktrans_close ( struct block_translator *blktrans, int rc ) {
+	struct block_device_capacity capacity;
+
+	/* Report block device capacity, if applicable */
+	if ( ( rc == 0 ) && ( blktrans->blksize ) ) {
+
+		/* Construct block device capacity */
+		capacity.blocks =
+			( blktrans->xferbuf.len / blktrans->blksize );
+		capacity.blksize = blktrans->blksize;
+		capacity.max_count = -1U;
+
+		/* Report block device capacity */
+		block_capacity ( &blktrans->block, &capacity );
+	}
+
+	/* Shut down interfaces */
+	intf_shutdown ( &blktrans->xfer, rc );
+	intf_shutdown ( &blktrans->block, rc );
+}
+
+/**
+ * Deliver data
+ *
+ * @v blktrans		Block device translator
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int blktrans_deliver ( struct block_translator *blktrans,
+			      struct io_buffer *iobuf,
+			      struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Deliver to buffer */
+	if ( ( rc = xferbuf_deliver ( &blktrans->xferbuf, iob_disown ( iobuf ),
+				      meta ) ) != 0 ) {
+		DBGC ( blktrans, "BLKTRANS %p could not deliver: %s\n",
+		       blktrans, strerror ( rc ) );
+		goto err;
+	}
+
+	return 0;
+
+ err:
+	blktrans_close ( blktrans, rc );
+	return rc;
+}
+
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v blktrans		Block device translator
+ * @ret xferbuf		Data transfer buffer
+ */
+static struct xfer_buffer *
+blktrans_buffer ( struct block_translator *blktrans ) {
+
+	return &blktrans->xferbuf;
+}
+
+/** Block device translator block device interface operations */
+static struct interface_operation blktrans_block_operations[] = {
+	INTF_OP ( intf_close, struct block_translator *, blktrans_close ),
+};
+
+/** Block device translator block device interface descriptor */
+static struct interface_descriptor blktrans_block_desc =
+	INTF_DESC_PASSTHRU ( struct block_translator, block,
+			     blktrans_block_operations, xfer );
+
+/** Block device translator data transfer interface operations */
+static struct interface_operation blktrans_xfer_operations[] = {
+	INTF_OP ( xfer_deliver, struct block_translator *, blktrans_deliver ),
+	INTF_OP ( xfer_buffer, struct block_translator *, blktrans_buffer ),
+	INTF_OP ( intf_close, struct block_translator *, blktrans_close ),
+};
+
+/** Block device translator data transfer interface descriptor */
+static struct interface_descriptor blktrans_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct block_translator, xfer,
+			     blktrans_xfer_operations, block );
+
+/**
+ * Insert block device translator
+ *
+ * @v block		Block device interface
+ * @v buffer		Data buffer (or UNULL)
+ * @v size		Length of data buffer, or block size
+ * @ret rc		Return status code
+ */
+int block_translate ( struct interface *block, userptr_t buffer, size_t size ) {
+	struct block_translator *blktrans;
+	int rc;
+
+	/* Allocate and initialise structure */
+	blktrans = zalloc ( sizeof ( *blktrans ) );
+	if ( ! blktrans ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ref_init ( &blktrans->refcnt, NULL );
+	intf_init ( &blktrans->block, &blktrans_block_desc, &blktrans->refcnt );
+	intf_init ( &blktrans->xfer, &blktrans_xfer_desc, &blktrans->refcnt );
+	blktrans->xferbuf.op = &blktrans_xferbuf_operations;
+	blktrans->buffer = buffer;
+	if ( buffer ) {
+		blktrans->xferbuf.len = size;
+	} else {
+		blktrans->blksize = size;
+	}
+
+	/* Attach to interfaces, mortalise self, and return */
+	assert ( block->dest != &null_intf );
+	intf_plug_plug ( &blktrans->xfer, block->dest );
+	intf_plug_plug ( &blktrans->block, block );
+	ref_put ( &blktrans->refcnt );
+
+	DBGC2 ( blktrans, "BLKTRANS %p created", blktrans );
+	if ( buffer ) {
+		DBGC2 ( blktrans, " for %#lx+%#zx",
+			user_to_phys ( buffer, 0 ), size );
+	}
+	DBGC2 ( blktrans, "\n" );
+	return 0;
+
+	ref_put ( &blktrans->refcnt );
+ err_alloc:
+	return rc;
+}
diff --git a/qemu/roms/ipxe/src/core/console.c b/qemu/roms/ipxe/src/core/console.c
index 141d8f0f0..7fd00036f 100644
--- a/qemu/roms/ipxe/src/core/console.c
+++ b/qemu/roms/ipxe/src/core/console.c
@@ -5,7 +5,7 @@
 
 /** @file */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Current console usage */
 int console_usage = CONSOLE_USAGE_STDOUT;
diff --git a/qemu/roms/ipxe/src/core/cpio.c b/qemu/roms/ipxe/src/core/cpio.c
index 3a5f4d2b6..080c72daf 100644
--- a/qemu/roms/ipxe/src/core/cpio.c
+++ b/qemu/roms/ipxe/src/core/cpio.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/ctype.c b/qemu/roms/ipxe/src/core/ctype.c
index c812346a0..891af71ea 100644
--- a/qemu/roms/ipxe/src/core/ctype.c
+++ b/qemu/roms/ipxe/src/core/ctype.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -31,11 +35,12 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /**
  * Check to see if character is a space
  *
- * @v c			Character
+ * @v character		Character
  * @ret isspace		Character is a space
  */
-int isspace ( int c ) {
-	switch ( c ) {
+int isspace ( int character ) {
+
+	switch ( character ) {
 	case ' ' :
 	case '\f' :
 	case '\n' :
diff --git a/qemu/roms/ipxe/src/core/cwuri.c b/qemu/roms/ipxe/src/core/cwuri.c
index 5865552a0..612f0b179 100644
--- a/qemu/roms/ipxe/src/core/cwuri.c
+++ b/qemu/roms/ipxe/src/core/cwuri.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/uri.h>
diff --git a/qemu/roms/ipxe/src/core/debug.c b/qemu/roms/ipxe/src/core/debug.c
index 7ded47089..def5d8b09 100644
--- a/qemu/roms/ipxe/src/core/debug.c
+++ b/qemu/roms/ipxe/src/core/debug.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/core/debug_md5.c b/qemu/roms/ipxe/src/core/debug_md5.c
index f049ac757..d0dbad9ed 100644
--- a/qemu/roms/ipxe/src/core/debug_md5.c
+++ b/qemu/roms/ipxe/src/core/debug_md5.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/core/device.c b/qemu/roms/ipxe/src/core/device.c
index 330f95c5a..77d7b719b 100644
--- a/qemu/roms/ipxe/src/core/device.c
+++ b/qemu/roms/ipxe/src/core/device.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/list.h>
diff --git a/qemu/roms/ipxe/src/core/downloader.c b/qemu/roms/ipxe/src/core/downloader.c
index ec69db6b1..d745f3617 100644
--- a/qemu/roms/ipxe/src/core/downloader.c
+++ b/qemu/roms/ipxe/src/core/downloader.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
@@ -29,7 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/uaccess.h>
 #include <ipxe/umalloc.h>
 #include <ipxe/image.h>
-#include <ipxe/profile.h>
+#include <ipxe/xferbuf.h>
 #include <ipxe/downloader.h>
 
 /** @file
@@ -38,14 +42,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-/** Receive profiler */
-static struct profiler downloader_rx_profiler __profiler =
-	{ .name = "downloader.rx" };
-
-/** Data copy profiler */
-static struct profiler downloader_copy_profiler __profiler =
-	{ .name = "downloader.copy" };
-
 /** A downloader */
 struct downloader {
 	/** Reference count for this object */
@@ -58,8 +54,8 @@ struct downloader {
 
 	/** Image to contain downloaded file */
 	struct image *image;
-	/** Current position within image buffer */
-	size_t pos;
+	/** Data transfer buffer */
+	struct xfer_buffer buffer;
 };
 
 /**
@@ -92,42 +88,14 @@ static void downloader_finished ( struct downloader *downloader, int rc ) {
 			 downloader->image->name, strerror ( rc ) );
 	}
 
+	/* Update image length */
+	downloader->image->len = downloader->buffer.len;
+
 	/* Shut down interfaces */
 	intf_shutdown ( &downloader->xfer, rc );
 	intf_shutdown ( &downloader->job, rc );
 }
 
-/**
- * Ensure that download buffer is large enough for the specified size
- *
- * @v downloader	Downloader
- * @v len		Required minimum size
- * @ret rc		Return status code
- */
-static int downloader_ensure_size ( struct downloader *downloader,
-				    size_t len ) {
-	userptr_t new_buffer;
-
-	/* If buffer is already large enough, do nothing */
-	if ( len <= downloader->image->len )
-		return 0;
-
-	DBGC ( downloader, "Downloader %p extending to %zd bytes\n",
-	       downloader, len );
-
-	/* Extend buffer */
-	new_buffer = urealloc ( downloader->image->data, len );
-	if ( ! new_buffer ) {
-		DBGC ( downloader, "Downloader %p could not extend buffer to "
-		       "%zd bytes\n", downloader, len );
-		return -ENOSPC;
-	}
-	downloader->image->data = new_buffer;
-	downloader->image->len = len;
-
-	return 0;
-}
-
 /****************************************************************************
  *
  * Job control interface
@@ -148,8 +116,8 @@ static int downloader_progress ( struct downloader *downloader,
 	 * arrive out of order (e.g. with multicast protocols), but
 	 * it's a reasonable first approximation.
 	 */
-	progress->completed = downloader->pos;
-	progress->total = downloader->image->len;
+	progress->completed = downloader->buffer.pos;
+	progress->total = downloader->buffer.len;
 
 	return 0;
 }
@@ -171,44 +139,37 @@ static int downloader_progress ( struct downloader *downloader,
 static int downloader_xfer_deliver ( struct downloader *downloader,
 				     struct io_buffer *iobuf,
 				     struct xfer_metadata *meta ) {
-	size_t len;
-	size_t max;
 	int rc;
 
-	/* Start profiling */
-	profile_start ( &downloader_rx_profiler );
-
-	/* Calculate new buffer position */
-	if ( meta->flags & XFER_FL_ABS_OFFSET )
-		downloader->pos = 0;
-	downloader->pos += meta->offset;
-
-	/* Ensure that we have enough buffer space for this data */
-	len = iob_len ( iobuf );
-	max = ( downloader->pos + len );
-	if ( ( rc = downloader_ensure_size ( downloader, max ) ) != 0 )
-		goto done;
-
-	/* Copy data to buffer */
-	profile_start ( &downloader_copy_profiler );
-	copy_to_user ( downloader->image->data, downloader->pos,
-		       iobuf->data, len );
-	profile_stop ( &downloader_copy_profiler );
-
-	/* Update current buffer position */
-	downloader->pos += len;
-
- done:
-	free_iob ( iobuf );
-	if ( rc != 0 )
-		downloader_finished ( downloader, rc );
-	profile_stop ( &downloader_rx_profiler );
+	/* Add data to buffer */
+	if ( ( rc = xferbuf_deliver ( &downloader->buffer, iob_disown ( iobuf ),
+				      meta ) ) != 0 )
+		goto err_deliver;
+
+	return 0;
+
+ err_deliver:
+	downloader_finished ( downloader, rc );
 	return rc;
 }
 
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v downloader	Downloader
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+downloader_xfer_buffer ( struct downloader *downloader ) {
+
+	/* Provide direct access to underlying data transfer buffer */
+	return &downloader->buffer;
+}
+
 /** Downloader data transfer interface operations */
 static struct interface_operation downloader_xfer_operations[] = {
 	INTF_OP ( xfer_deliver, struct downloader *, downloader_xfer_deliver ),
+	INTF_OP ( xfer_buffer, struct downloader *, downloader_xfer_buffer ),
 	INTF_OP ( intf_close, struct downloader *, downloader_finished ),
 };
 
@@ -262,6 +223,7 @@ int create_downloader ( struct interface *job, struct image *image ) {
 	intf_init ( &downloader->xfer, &downloader_xfer_desc,
 		    &downloader->refcnt );
 	downloader->image = image_get ( image );
+	xferbuf_umalloc_init ( &downloader->buffer, &image->data );
 
 	/* Instantiate child objects and attach to our interfaces */
 	if ( ( rc = xfer_open_uri ( &downloader->xfer, image->uri ) ) != 0 )
diff --git a/qemu/roms/ipxe/src/core/edd.c b/qemu/roms/ipxe/src/core/edd.c
index d574ea6c0..a50b74ab1 100644
--- a/qemu/roms/ipxe/src/core/edd.c
+++ b/qemu/roms/ipxe/src/core/edd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/interface.h>
diff --git a/qemu/roms/ipxe/src/core/errno.c b/qemu/roms/ipxe/src/core/errno.c
index 06905561f..5de15bb92 100644
--- a/qemu/roms/ipxe/src/core/errno.c
+++ b/qemu/roms/ipxe/src/core/errno.c
@@ -1,6 +1,6 @@
 #include <errno.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/exec.c b/qemu/roms/ipxe/src/core/exec.c
index 1c85705ae..2c2ade0a5 100644
--- a/qemu/roms/ipxe/src/core/exec.c
+++ b/qemu/roms/ipxe/src/core/exec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/fault.c b/qemu/roms/ipxe/src/core/fault.c
new file mode 100644
index 000000000..63d3ccacf
--- /dev/null
+++ b/qemu/roms/ipxe/src/core/fault.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <errno.h>
+#include <ipxe/fault.h>
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+/**
+ * Inject fault with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (must be non-zero)
+ * @ret rc		Return status code
+ */
+int inject_fault_nonzero ( unsigned int rate ) {
+
+	/* Do nothing unless we want to inject a fault now */
+	if ( ( random() % rate ) != 0 )
+		return 0;
+
+	/* Generate error number here so that faults can be injected
+	 * into files that don't themselves have error file
+	 * identifiers (via errfile.h).
+	 */
+	return -EFAULT;
+}
+
+/**
+ * Corrupt data with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (must be non-zero)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+void inject_corruption_nonzero ( unsigned int rate, const void *data,
+				 size_t len ) {
+	uint8_t *writable;
+	size_t offset;
+
+	/* Do nothing if we have no data to corrupt */
+	if ( ! len )
+		return;
+
+	/* Do nothing unless we want to inject a fault now */
+	if ( ! inject_fault_nonzero ( rate ) )
+		return;
+
+	/* Get a writable pointer to the nominally read-only data */
+	writable = ( ( uint8_t * ) data );
+
+	/* Pick a random victim byte and zap it */
+	offset = ( random() % len );
+	writable[offset] ^= random();
+}
diff --git a/qemu/roms/ipxe/src/core/fbcon.c b/qemu/roms/ipxe/src/core/fbcon.c
index 72d6a6789..6d8b0086d 100644
--- a/qemu/roms/ipxe/src/core/fbcon.c
+++ b/qemu/roms/ipxe/src/core/fbcon.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/fnrec.c b/qemu/roms/ipxe/src/core/fnrec.c
index 3453c8b6a..0430817f8 100644
--- a/qemu/roms/ipxe/src/core/fnrec.c
+++ b/qemu/roms/ipxe/src/core/fnrec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/core/gdbserial.c b/qemu/roms/ipxe/src/core/gdbserial.c
index 6f78c88bf..0983f2557 100644
--- a/qemu/roms/ipxe/src/core/gdbserial.c
+++ b/qemu/roms/ipxe/src/core/gdbserial.c
@@ -15,35 +15,105 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <assert.h>
-#include <ipxe/serial.h>
+#include <ipxe/uart.h>
 #include <ipxe/gdbstub.h>
 #include <ipxe/gdbserial.h>
+#include <config/serial.h>
+
+/* UART port number */
+#ifdef COMCONSOLE
+#define GDBSERIAL_PORT COMCONSOLE
+#else
+#define GDBSERIAL_PORT 0
+#endif
+
+/* UART baud rate */
+#ifdef COMPRESERVE
+#define GDBSERIAL_BAUD 0
+#else
+#define GDBSERIAL_BAUD COMSPEED
+#endif
+
+/* UART line control register value */
+#ifdef COMPRESERVE
+#define GDBSERIAL_LCR 0
+#else
+#define GDBSERIAL_LCR UART_LCR_WPS ( COMDATA, COMPARITY, COMSTOP )
+#endif
+
+/** GDB serial UART */
+static struct uart gdbserial_uart;
 
 struct gdb_transport serial_gdb_transport __gdb_transport;
 
 static size_t gdbserial_recv ( char *buf, size_t len ) {
+
 	assert ( len > 0 );
-	buf [ 0 ] = serial_getc();
+	while ( ! uart_data_ready ( &gdbserial_uart ) ) {}
+	buf[0] = uart_receive ( &gdbserial_uart );
 	return 1;
 }
 
 static void gdbserial_send ( const char *buf, size_t len ) {
+
 	while ( len-- > 0 ) {
-		serial_putc ( *buf++ );
+		uart_transmit ( &gdbserial_uart, *buf++ );
 	}
 }
 
+static int gdbserial_init ( int argc, char **argv ) {
+	unsigned int port;
+	char *endp;
+
+	if ( argc == 0 ) {
+		port = GDBSERIAL_PORT;
+	} else if ( argc == 1 ) {
+		port = strtoul ( argv[0], &endp, 10 );
+		if ( *endp ) {
+			printf ( "serial: invalid port\n" );
+			return 1;
+		}
+	} else {
+		printf ( "serial: syntax <port>\n" );
+		return 1;
+	}
+
+	if ( ! gdbserial_configure ( port, GDBSERIAL_BAUD, GDBSERIAL_LCR ) ) {
+		printf ( "serial: unable to configure\n" );
+		return 1;
+	}
+
+	return 0;
+}
+
 struct gdb_transport serial_gdb_transport __gdb_transport = {
 	.name = "serial",
+	.init = gdbserial_init,
 	.recv = gdbserial_recv,
 	.send = gdbserial_send,
 };
 
-struct gdb_transport *gdbserial_configure ( void ) {
+struct gdb_transport * gdbserial_configure ( unsigned int port,
+					     unsigned int baud, uint8_t lcr ) {
+	int rc;
+
+	if ( ( rc = uart_select ( &gdbserial_uart, port ) ) != 0 )
+		return NULL;
+
+	if ( ( rc = uart_init ( &gdbserial_uart, baud, lcr ) ) != 0 )
+		return NULL;
+
 	return &serial_gdb_transport;
 }
diff --git a/qemu/roms/ipxe/src/core/gdbstub.c b/qemu/roms/ipxe/src/core/gdbstub.c
index af06118b2..6ad52d1a6 100644
--- a/qemu/roms/ipxe/src/core/gdbstub.c
+++ b/qemu/roms/ipxe/src/core/gdbstub.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/core/gdbudp.c b/qemu/roms/ipxe/src/core/gdbudp.c
index 5977547c8..e4613d137 100644
--- a/qemu/roms/ipxe/src/core/gdbudp.c
+++ b/qemu/roms/ipxe/src/core/gdbudp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/getkey.c b/qemu/roms/ipxe/src/core/getkey.c
index d69cfb44b..0f0f8b7c3 100644
--- a/qemu/roms/ipxe/src/core/getkey.c
+++ b/qemu/roms/ipxe/src/core/getkey.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ctype.h>
 #include <ipxe/console.h>
diff --git a/qemu/roms/ipxe/src/core/getopt.c b/qemu/roms/ipxe/src/core/getopt.c
index abc1edd6c..e6c3948d1 100644
--- a/qemu/roms/ipxe/src/core/getopt.c
+++ b/qemu/roms/ipxe/src/core/getopt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/image.c b/qemu/roms/ipxe/src/core/image.c
index ec4480238..529e3d72c 100644
--- a/qemu/roms/ipxe/src/core/image.c
+++ b/qemu/roms/ipxe/src/core/image.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
@@ -154,6 +158,32 @@ int image_set_cmdline ( struct image *image, const char *cmdline ) {
 }
 
 /**
+ * Determine image type
+ *
+ * @v image		Executable image
+ * @ret rc		Return status code
+ */
+static int image_probe ( struct image *image ) {
+	struct image_type *type;
+	int rc;
+
+	/* Try each type in turn */
+	for_each_table_entry ( type, IMAGE_TYPES ) {
+		if ( ( rc = type->probe ( image ) ) == 0 ) {
+			image->type = type;
+			DBGC ( image, "IMAGE %s is %s\n",
+			       image->name, type->name );
+			break;
+		}
+		DBGC ( image, "IMAGE %s is not %s: %s\n", image->name,
+		       type->name, strerror ( rc ) );
+	}
+
+	DBGC ( image, "IMAGE %s format not recognised\n", image->name );
+	return -ENOTSUP;
+}
+
+/**
  * Register executable image
  *
  * @v image		Executable image
@@ -185,6 +215,14 @@ int register_image ( struct image *image ) {
 	       image->name, user_to_phys ( image->data, 0 ),
 	       user_to_phys ( image->data, image->len ) );
 
+	/* Try to detect image type, if applicable.  Ignore failures,
+	 * since we expect to handle some unrecognised images
+	 * (e.g. kernel initrds, multiboot modules, random files
+	 * provided via our EFI virtual filesystem, etc).
+	 */
+	if ( ! image->type )
+		image_probe ( image );
+
 	return 0;
 }
 
@@ -223,36 +261,6 @@ struct image * find_image ( const char *name ) {
 }
 
 /**
- * Determine image type
- *
- * @v image		Executable image
- * @ret rc		Return status code
- */
-int image_probe ( struct image *image ) {
-	struct image_type *type;
-	int rc;
-
-	/* Succeed if we already have a type */
-	if ( image->type )
-		return 0;
-
-	/* Try each type in turn */
-	for_each_table_entry ( type, IMAGE_TYPES ) {
-		if ( ( rc = type->probe ( image ) ) == 0 ) {
-			image->type = type;
-			DBGC ( image, "IMAGE %s is %s\n",
-			       image->name, type->name );
-			return 0;
-		}
-		DBGC ( image, "IMAGE %s is not %s: %s\n", image->name,
-		       type->name, strerror ( rc ) );
-	}
-
-	DBGC ( image, "IMAGE %s format not recognised\n", image->name );
-	return -ENOEXEC;
-}
-
-/**
  * Execute image
  *
  * @v image		Executable image
@@ -284,9 +292,11 @@ int image_exec ( struct image *image ) {
 	 */
 	current_image = image_get ( image );
 
-	/* Check that this image can be selected for execution */
-	if ( ( rc = image_select ( image ) ) != 0 )
+	/* Check that this image can be executed */
+	if ( ! ( image->type && image->type->exec ) ) {
+		rc = -ENOEXEC;
 		goto err;
+	}
 
 	/* Check that image is trusted (if applicable) */
 	if ( require_trusted_images && ! ( image->flags & IMAGE_TRUSTED ) ) {
@@ -378,8 +388,8 @@ int image_replace ( struct image *replacement ) {
 	}
 
 	/* Check that the replacement image can be executed */
-	if ( ( rc = image_probe ( replacement ) ) != 0 )
-		return rc;
+	if ( ! ( replacement->type && replacement->type->exec ) )
+		return -ENOEXEC;
 
 	/* Clear any existing replacement */
 	image_put ( image->replacement );
@@ -400,16 +410,13 @@ int image_replace ( struct image *replacement ) {
  */
 int image_select ( struct image *image ) {
 	struct image *tmp;
-	int rc;
 
 	/* Unselect all other images */
 	for_each_image ( tmp )
 		tmp->flags &= ~IMAGE_SELECTED;
 
 	/* Check that this image can be executed */
-	if ( ( rc = image_probe ( image ) ) != 0 )
-		return rc;
-	if ( ! image->type->exec )
+	if ( ! ( image->type && image->type->exec ) )
 		return -ENOEXEC;
 
 	/* Mark image as selected */
@@ -468,9 +475,7 @@ int image_pixbuf ( struct image *image, struct pixel_buffer **pixbuf ) {
 	int rc;
 
 	/* Check that this image can be used to create a pixel buffer */
-	if ( ( rc = image_probe ( image ) ) != 0 )
-		return rc;
-	if ( ! image->type->pixbuf )
+	if ( ! ( image->type && image->type->pixbuf ) )
 		return -ENOTSUP;
 
 	/* Try creating pixel buffer */
diff --git a/qemu/roms/ipxe/src/core/init.c b/qemu/roms/ipxe/src/core/init.c
index 7ea0730fa..d91e44669 100644
--- a/qemu/roms/ipxe/src/core/init.c
+++ b/qemu/roms/ipxe/src/core/init.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/console.h>
diff --git a/qemu/roms/ipxe/src/core/interface.c b/qemu/roms/ipxe/src/core/interface.c
index 62f4621db..ba148c13d 100644
--- a/qemu/roms/ipxe/src/core/interface.c
+++ b/qemu/roms/ipxe/src/core/interface.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/interface.h>
@@ -307,3 +311,28 @@ void intf_restart ( struct interface *intf, int rc ) {
 	 */
 	intf->desc = desc;
 }
+
+/**
+ * Poke an object interface
+ *
+ * @v intf		Object interface
+ * @v type		Operation type
+ *
+ * This is a helper function to implement methods which take no
+ * parameters and return nothing.
+ */
+void intf_poke ( struct interface *intf,
+		 void ( type ) ( struct interface *intf ) ) {
+	struct interface *dest;
+	intf_poke_TYPE ( void * ) *op =
+		intf_get_dest_op_untyped ( intf, type, &dest );
+	void *object = intf_object ( dest );
+
+	if ( op ) {
+		op ( object );
+	} else {
+		/* Default is to do nothing */
+	}
+
+	intf_put ( dest );
+}
diff --git a/qemu/roms/ipxe/src/core/iobuf.c b/qemu/roms/ipxe/src/core/iobuf.c
index afc91d150..3e52ada4f 100644
--- a/qemu/roms/ipxe/src/core/iobuf.c
+++ b/qemu/roms/ipxe/src/core/iobuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <strings.h>
@@ -200,3 +204,33 @@ struct io_buffer * iob_concatenate ( struct list_head *list ) {
 
 	return concatenated;
 }
+
+/**
+ * Split I/O buffer
+ *
+ * @v iobuf		I/O buffer
+ * @v len		Length to split into a new I/O buffer
+ * @ret split		New I/O buffer, or NULL on allocation failure
+ *
+ * Split the first @c len bytes of the existing I/O buffer into a
+ * separate I/O buffer.  The resulting buffers are likely to have no
+ * headroom or tailroom.
+ *
+ * If this call fails, then the original buffer will be unmodified.
+ */
+struct io_buffer * iob_split ( struct io_buffer *iobuf, size_t len ) {
+	struct io_buffer *split;
+
+	/* Sanity checks */
+	assert ( len <= iob_len ( iobuf ) );
+
+	/* Allocate new I/O buffer */
+	split = alloc_iob ( len );
+	if ( ! split )
+		return NULL;
+
+	/* Copy in data */
+	memcpy ( iob_put ( split, len ), iobuf->data, len );
+	iob_pull ( iobuf, len );
+	return split;
+}
diff --git a/qemu/roms/ipxe/src/core/isqrt.c b/qemu/roms/ipxe/src/core/isqrt.c
index 35c918d19..c4d0571e7 100644
--- a/qemu/roms/ipxe/src/core/isqrt.c
+++ b/qemu/roms/ipxe/src/core/isqrt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/job.c b/qemu/roms/ipxe/src/core/job.c
index 674bec8b5..65df80056 100644
--- a/qemu/roms/ipxe/src/core/job.c
+++ b/qemu/roms/ipxe/src/core/job.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/core/linebuf.c b/qemu/roms/ipxe/src/core/linebuf.c
index 8fb2f86a7..c197e383c 100644
--- a/qemu/roms/ipxe/src/core/linebuf.c
+++ b/qemu/roms/ipxe/src/core/linebuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -39,7 +43,18 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * @ret line		Buffered line, or NULL if no line ready to read
  */
 char * buffered_line ( struct line_buffer *linebuf ) {
-	return ( linebuf->ready ? linebuf->data : NULL );
+	char *line = &linebuf->data[ linebuf->len ];
+
+	/* Fail unless we have a newly completed line to retrieve */
+	if ( ( linebuf->len == 0 ) || ( linebuf->consumed == 0 ) ||
+	     ( *(--line) != '\0' ) )
+		return NULL;
+
+	/* Identify start of line */
+	while ( ( line > linebuf->data ) && ( line[-1] != '\0' ) )
+		line--;
+
+	return line;
 }
 
 /**
@@ -48,10 +63,11 @@ char * buffered_line ( struct line_buffer *linebuf ) {
  * @v linebuf		Line buffer
  */
 void empty_line_buffer ( struct line_buffer *linebuf ) {
+
 	free ( linebuf->data );
 	linebuf->data = NULL;
 	linebuf->len = 0;
-	linebuf->ready = 0;
+	linebuf->consumed = 0;
 }
 
 /**
@@ -72,16 +88,13 @@ void empty_line_buffer ( struct line_buffer *linebuf ) {
  * should call empty_line_buffer() before freeing a @c struct @c
  * line_buffer.
  */
-ssize_t line_buffer ( struct line_buffer *linebuf,
-		      const char *data, size_t len ) {
+int line_buffer ( struct line_buffer *linebuf, const char *data, size_t len ) {
 	const char *eol;
 	size_t consume;
 	size_t new_len;
 	char *new_data;
-
-	/* Free any completed line from previous iteration */
-	if ( linebuf->ready )
-		empty_line_buffer ( linebuf );
+	char *lf;
+	char *cr;
 
 	/* Search for line terminator */
 	if ( ( eol = memchr ( data, '\n', len ) ) ) {
@@ -90,6 +103,10 @@ ssize_t line_buffer ( struct line_buffer *linebuf,
 		consume = len;
 	}
 
+	/* Reject any embedded NULs within the data to be consumed */
+	if ( memchr ( data, '\0', consume ) )
+		return -EINVAL;
+
 	/* Reallocate data buffer and copy in new data */
 	new_len = ( linebuf->len + consume );
 	new_data = realloc ( linebuf->data, ( new_len + 1 ) );
@@ -100,13 +117,27 @@ ssize_t line_buffer ( struct line_buffer *linebuf,
 	linebuf->data = new_data;
 	linebuf->len = new_len;
 
-	/* If we have reached end of line, trim the line and mark as ready */
+	/* If we have reached end of line, terminate the line */
 	if ( eol ) {
-		linebuf->data[--linebuf->len] = '\0'; /* trim NL */
-		if ( linebuf->data[linebuf->len - 1] == '\r' )
-			linebuf->data[--linebuf->len] = '\0'; /* trim CR */
-		linebuf->ready = 1;
+
+		/* Overwrite trailing LF (which must exist at this point) */
+		assert ( linebuf->len > 0 );
+		lf = &linebuf->data[ linebuf->len - 1 ];
+		assert ( *lf == '\n' );
+		*lf = '\0';
+
+		/* Trim (and overwrite) trailing CR, if present */
+		if ( linebuf->len > 1 ) {
+			cr = ( lf - 1 );
+			if ( *cr == '\r' ) {
+				linebuf->len--;
+				*cr = '\0';
+			}
+		}
 	}
 
+	/* Record consumed length */
+	linebuf->consumed = consume;
+
 	return consume;
 }
diff --git a/qemu/roms/ipxe/src/core/lineconsole.c b/qemu/roms/ipxe/src/core/lineconsole.c
index 1b6791cf3..bb3bfafc9 100644
--- a/qemu/roms/ipxe/src/core/lineconsole.c
+++ b/qemu/roms/ipxe/src/core/lineconsole.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/list.c b/qemu/roms/ipxe/src/core/list.c
index 77579d69a..5175c84ec 100644
--- a/qemu/roms/ipxe/src/core/list.c
+++ b/qemu/roms/ipxe/src/core/list.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/log.c b/qemu/roms/ipxe/src/core/log.c
index f160b4fc8..c08e4bb9b 100644
--- a/qemu/roms/ipxe/src/core/log.c
+++ b/qemu/roms/ipxe/src/core/log.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/main.c b/qemu/roms/ipxe/src/core/main.c
index db09e4c39..638dea9cf 100644
--- a/qemu/roms/ipxe/src/core/main.c
+++ b/qemu/roms/ipxe/src/core/main.c
@@ -12,7 +12,7 @@ Literature dealing with the network protocols:
 
 **************************************************************************/
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdio.h>
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * @ret rc		Return status code
  */
 __asmcall int main ( void ) {
+	int rc;
 
 	/* Perform one-time-only initialisation (e.g. heap) */
 	initialise();
@@ -35,9 +36,11 @@ __asmcall int main ( void ) {
 	startup();
 	printf ( "ok\n" );
 
-	ipxe ( NULL );
+	/* Attempt to boot */
+	if ( ( rc = ipxe ( NULL ) ) != 0 )
+		goto err_ipxe;
 
+ err_ipxe:
 	shutdown_exit();
-
-	return 0;
+	return rc;
 }
diff --git a/qemu/roms/ipxe/src/core/malloc.c b/qemu/roms/ipxe/src/core/malloc.c
index d9c07495d..b120c0325 100644
--- a/qemu/roms/ipxe/src/core/malloc.c
+++ b/qemu/roms/ipxe/src/core/malloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
@@ -106,6 +110,7 @@ static char heap[HEAP_SIZE] __attribute__ (( aligned ( __alignof__(void *) )));
 static inline void valgrind_make_blocks_defined ( void ) {
 	struct memory_block *block;
 
+	/* Do nothing unless running under Valgrind */
 	if ( RUNNING_ON_VALGRIND <= 0 )
 		return;
 
@@ -147,6 +152,7 @@ static inline void valgrind_make_blocks_noaccess ( void ) {
 	struct memory_block *block;
 	struct memory_block *prev = NULL;
 
+	/* Do nothing unless running under Valgrind */
 	if ( RUNNING_ON_VALGRIND <= 0 )
 		return;
 
@@ -267,24 +273,25 @@ static void discard_all_cache ( void ) {
 void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 	struct memory_block *block;
 	size_t align_mask;
+	size_t actual_size;
 	size_t pre_size;
 	ssize_t post_size;
 	struct memory_block *pre;
 	struct memory_block *post;
-	struct memory_block *ptr;
+	void *ptr;
 
 	/* Sanity checks */
 	assert ( size != 0 );
 	assert ( ( align == 0 ) || ( ( align & ( align - 1 ) ) == 0 ) );
-
 	valgrind_make_blocks_defined();
 	check_blocks();
 
 	/* Round up size to multiple of MIN_MEMBLOCK_SIZE and
 	 * calculate alignment mask.
 	 */
-	size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
-	align_mask = ( align - 1 ) | ( MIN_MEMBLOCK_SIZE - 1 );
+	actual_size = ( ( size + MIN_MEMBLOCK_SIZE - 1 ) &
+			~( MIN_MEMBLOCK_SIZE - 1 ) );
+	align_mask = ( ( align - 1 ) | ( MIN_MEMBLOCK_SIZE - 1 ) );
 
 	DBGC2 ( &heap, "Allocating %#zx (aligned %#zx+%zx)\n",
 		size, align, offset );
@@ -293,7 +300,7 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 		list_for_each_entry ( block, &free_blocks, list ) {
 			pre_size = ( ( offset - virt_to_phys ( block ) )
 				     & align_mask );
-			post_size = ( block->size - pre_size - size );
+			post_size = ( block->size - pre_size - actual_size );
 			if ( post_size >= 0 ) {
 				/* Split block into pre-block, block, and
 				 * post-block.  After this split, the "pre"
@@ -302,7 +309,7 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 */
 				pre   = block;
 				block = ( ( ( void * ) pre   ) + pre_size );
-				post  = ( ( ( void * ) block ) + size     );
+				post  = ( ( ( void * ) block ) + actual_size );
 				DBGC2 ( &heap, "[%p,%p) -> [%p,%p) + [%p,%p)\n",
 					pre, ( ( ( void * ) pre ) + pre->size ),
 					pre, block, post,
@@ -313,8 +320,8 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 * the heap).
 				 */
 				if ( (size_t) post_size >= MIN_MEMBLOCK_SIZE ) {
-					VALGRIND_MAKE_MEM_DEFINED ( post,
-							     sizeof ( *post ) );
+					VALGRIND_MAKE_MEM_UNDEFINED
+						( post, sizeof ( *post ) );
 					post->size = post_size;
 					list_add ( &post->list, &pre->list );
 				}
@@ -328,14 +335,18 @@ void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
 				 * it is too small, which can happen only at
 				 * the very start of the heap.
 				 */
-				if ( pre_size < MIN_MEMBLOCK_SIZE )
+				if ( pre_size < MIN_MEMBLOCK_SIZE ) {
 					list_del ( &pre->list );
+					VALGRIND_MAKE_MEM_NOACCESS
+						( pre, sizeof ( *pre ) );
+				}
 				/* Update total free memory */
-				freemem -= size;
+				freemem -= actual_size;
 				/* Return allocated block */
 				DBGC2 ( &heap, "Allocated [%p,%p)\n", block,
 					( ( ( void * ) block ) + size ) );
 				ptr = block;
+				VALGRIND_MAKE_MEM_UNDEFINED ( ptr, size );
 				goto done;
 			}
 		}
@@ -368,13 +379,16 @@ void free_memblock ( void *ptr, size_t size ) {
 	struct memory_block *freeing;
 	struct memory_block *block;
 	struct memory_block *tmp;
+	size_t actual_size;
 	ssize_t gap_before;
 	ssize_t gap_after = -1;
 
 	/* Allow for ptr==NULL */
 	if ( ! ptr )
 		return;
+	VALGRIND_MAKE_MEM_NOACCESS ( ptr, size );
 
+	/* Sanity checks */
 	valgrind_make_blocks_defined();
 	check_blocks();
 
@@ -382,9 +396,10 @@ void free_memblock ( void *ptr, size_t size ) {
 	 * would have used.
 	 */
 	assert ( size != 0 );
-	size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
+	actual_size = ( ( size + MIN_MEMBLOCK_SIZE - 1 ) &
+			~( MIN_MEMBLOCK_SIZE - 1 ) );
 	freeing = ptr;
-	VALGRIND_MAKE_MEM_DEFINED ( freeing, sizeof ( *freeing ) );
+	VALGRIND_MAKE_MEM_UNDEFINED ( freeing, sizeof ( *freeing ) );
 	DBGC2 ( &heap, "Freeing [%p,%p)\n",
 		freeing, ( ( ( void * ) freeing ) + size ) );
 
@@ -392,7 +407,7 @@ void free_memblock ( void *ptr, size_t size ) {
 	if ( ASSERTING ) {
 		list_for_each_entry ( block, &free_blocks, list ) {
 			if ( ( ( ( void * ) block ) <
-			       ( ( void * ) freeing + size ) ) &&
+			       ( ( void * ) freeing + actual_size ) ) &&
 			     ( ( void * ) freeing <
 			       ( ( void * ) block + block->size ) ) ) {
 				assert ( 0 );
@@ -407,7 +422,7 @@ void free_memblock ( void *ptr, size_t size ) {
 	}
 
 	/* Insert/merge into free list */
-	freeing->size = size;
+	freeing->size = actual_size;
 	list_for_each_entry_safe ( block, tmp, &free_blocks, list ) {
 		/* Calculate gaps before and after the "freeing" block */
 		gap_before = ( ( ( void * ) freeing ) - 
@@ -421,8 +436,10 @@ void free_memblock ( void *ptr, size_t size ) {
 				( ( ( void * ) freeing ) + freeing->size ),
 				block,
 				( ( ( void * ) freeing ) + freeing->size ) );
-			block->size += size;
+			block->size += actual_size;
 			list_del ( &block->list );
+			VALGRIND_MAKE_MEM_NOACCESS ( freeing,
+						     sizeof ( *freeing ) );
 			freeing = block;
 		}
 		/* Stop processing as soon as we reach a following block */
@@ -444,10 +461,11 @@ void free_memblock ( void *ptr, size_t size ) {
 			( ( ( void * ) block ) + block->size ) );
 		freeing->size += block->size;
 		list_del ( &block->list );
+		VALGRIND_MAKE_MEM_NOACCESS ( block, sizeof ( *block ) );
 	}
 
 	/* Update free memory counter */
-	freemem += size;
+	freemem += actual_size;
 
 	check_blocks();
 	valgrind_make_blocks_noaccess();
@@ -490,9 +508,9 @@ void * realloc ( void *old_ptr, size_t new_size ) {
 		new_block = alloc_memblock ( new_total_size, 1, 0 );
 		if ( ! new_block )
 			return NULL;
-		VALGRIND_MAKE_MEM_UNDEFINED ( new_block, offsetof ( struct autosized_block, data ) );
 		new_block->size = new_total_size;
-		VALGRIND_MAKE_MEM_NOACCESS ( new_block, offsetof ( struct autosized_block, data ) );
+		VALGRIND_MAKE_MEM_NOACCESS ( &new_block->size,
+					     sizeof ( new_block->size ) );
 		new_ptr = &new_block->data;
 		VALGRIND_MALLOCLIKE_BLOCK ( new_ptr, new_size, 0, 0 );
 	}
@@ -505,16 +523,16 @@ void * realloc ( void *old_ptr, size_t new_size ) {
 	if ( old_ptr && ( old_ptr != NOWHERE ) ) {
 		old_block = container_of ( old_ptr, struct autosized_block,
 					   data );
-		VALGRIND_MAKE_MEM_DEFINED ( old_block, offsetof ( struct autosized_block, data ) );
+		VALGRIND_MAKE_MEM_DEFINED ( &old_block->size,
+					    sizeof ( old_block->size ) );
 		old_total_size = old_block->size;
 		assert ( old_total_size != 0 );
 		old_size = ( old_total_size -
 			     offsetof ( struct autosized_block, data ) );
 		memcpy ( new_ptr, old_ptr,
 			 ( ( old_size < new_size ) ? old_size : new_size ) );
-		free_memblock ( old_block, old_total_size );
-		VALGRIND_MAKE_MEM_NOACCESS ( old_block, offsetof ( struct autosized_block, data ) );
 		VALGRIND_FREELIKE_BLOCK ( old_ptr, 0 );
+		free_memblock ( old_block, old_total_size );
 	}
 
 	if ( ASSERTED ) {
@@ -611,6 +629,7 @@ void mpopulate ( void *start, size_t len ) {
  */
 static void init_heap ( void ) {
 	VALGRIND_MAKE_MEM_NOACCESS ( heap, sizeof ( heap ) );
+	VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks, sizeof ( free_blocks ) );
 	mpopulate ( heap, sizeof ( heap ) );
 }
 
diff --git a/qemu/roms/ipxe/src/core/memblock.c b/qemu/roms/ipxe/src/core/memblock.c
index 1fd89b871..aecddc22c 100644
--- a/qemu/roms/ipxe/src/core/memblock.c
+++ b/qemu/roms/ipxe/src/core/memblock.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/memmap_settings.c b/qemu/roms/ipxe/src/core/memmap_settings.c
index 0f6d0abf5..fab3e5f3a 100644
--- a/qemu/roms/ipxe/src/core/memmap_settings.c
+++ b/qemu/roms/ipxe/src/core/memmap_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/core/menu.c b/qemu/roms/ipxe/src/core/menu.c
index 8d42e1f83..ab5b0c7f5 100644
--- a/qemu/roms/ipxe/src/core/menu.c
+++ b/qemu/roms/ipxe/src/core/menu.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/misc.c b/qemu/roms/ipxe/src/core/misc.c
deleted file mode 100644
index eaceddfea..000000000
--- a/qemu/roms/ipxe/src/core/misc.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/**************************************************************************
-MISC Support Routines
-**************************************************************************/
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdlib.h>
-#include <ctype.h>
-#include <byteswap.h>
-#include <ipxe/in.h>
-#include <ipxe/timer.h>
-
-/**************************************************************************
-INET_ATON - Convert an ascii x.x.x.x to binary form
-**************************************************************************/
-int inet_aton ( const char *cp, struct in_addr *inp ) {
-	const char *p = cp;
-	const char *digits_start;
-	unsigned long ip = 0;
-	unsigned long val;
-	int j;
-	for(j = 0; j <= 3; j++) {
-		digits_start = p;
-		val = strtoul(p, ( char ** ) &p, 10);
-		if ((p == digits_start) || (val > 255)) return 0;
-		if ( ( j < 3 ) && ( *(p++) != '.' ) ) return 0;
-		ip = (ip << 8) | val;
-	}
-	if ( *p == '\0' ) {
-		inp->s_addr = htonl(ip);
-		return 1;
-	}
-	return 0;
-}
-
-unsigned int strtoul_charval ( unsigned int charval ) {
-
-	if ( charval >= 'a' ) {
-		charval = ( charval - 'a' + 10 );
-	} else if ( charval >= 'A' ) {
-		charval = ( charval - 'A' + 10 );
-	} else if ( charval <= '9' ) {
-		charval = ( charval - '0' );
-	}
-
-	return charval;
-}
-
-unsigned long strtoul ( const char *p, char **endp, int base ) {
-	unsigned long ret = 0;
-	int negative = 0;
-	unsigned int charval;
-
-	while ( isspace ( *p ) )
-		p++;
-
-	if ( *p == '-' ) {
-		negative = 1;
-		p++;
-	}
-
-	base = strtoul_base ( &p, base );
-
-	while ( 1 ) {
-		charval = strtoul_charval ( *p );
-		if ( charval >= ( unsigned int ) base )
-			break;
-		ret = ( ( ret * base ) + charval );
-		p++;
-	}
-
-	if ( negative )
-		ret = -ret;
-
-	if ( endp )
-		*endp = ( char * ) p;
-
-	return ( ret );
-}
-
-/*
- * Local variables:
- *  c-basic-offset: 8
- * End:
- */
diff --git a/qemu/roms/ipxe/src/core/monojob.c b/qemu/roms/ipxe/src/core/monojob.c
index 820fa31dc..817f21b2c 100644
--- a/qemu/roms/ipxe/src/core/monojob.c
+++ b/qemu/roms/ipxe/src/core/monojob.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/core/null_reboot.c b/qemu/roms/ipxe/src/core/null_reboot.c
index a3d5b2ef8..7be5612a3 100644
--- a/qemu/roms/ipxe/src/core/null_reboot.c
+++ b/qemu/roms/ipxe/src/core/null_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/core/null_sanboot.c b/qemu/roms/ipxe/src/core/null_sanboot.c
index 18c0dea84..2f7522c6c 100644
--- a/qemu/roms/ipxe/src/core/null_sanboot.c
+++ b/qemu/roms/ipxe/src/core/null_sanboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/sanboot.h>
diff --git a/qemu/roms/ipxe/src/core/null_time.c b/qemu/roms/ipxe/src/core/null_time.c
index 506c70b52..90041a456 100644
--- a/qemu/roms/ipxe/src/core/null_time.c
+++ b/qemu/roms/ipxe/src/core/null_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/nvo.c b/qemu/roms/ipxe/src/core/nvo.c
index e135d2b41..d2c9b5e73 100644
--- a/qemu/roms/ipxe/src/core/nvo.c
+++ b/qemu/roms/ipxe/src/core/nvo.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/core/open.c b/qemu/roms/ipxe/src/core/open.c
index b479c2975..9d665ffda 100644
--- a/qemu/roms/ipxe/src/core/open.c
+++ b/qemu/roms/ipxe/src/core/open.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/params.c b/qemu/roms/ipxe/src/core/params.c
index 93b834419..e1f66acca 100644
--- a/qemu/roms/ipxe/src/core/params.c
+++ b/qemu/roms/ipxe/src/core/params.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/parseopt.c b/qemu/roms/ipxe/src/core/parseopt.c
index d268c0594..66f60158c 100644
--- a/qemu/roms/ipxe/src/core/parseopt.c
+++ b/qemu/roms/ipxe/src/core/parseopt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
@@ -32,6 +36,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/params.h>
 #include <ipxe/timer.h>
 #include <ipxe/parseopt.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -343,7 +348,7 @@ void print_usage ( struct command_descriptor *cmd, char **argv ) {
 	}
 	if ( cmd->usage )
 		printf ( " %s", cmd->usage );
-	printf ( "\n\nSee http://ipxe.org/cmd/%s for further information\n",
+	printf ( "\n\nSee " PRODUCT_COMMAND_URI " for further information\n",
 		 argv[0] );
 }
 
diff --git a/qemu/roms/ipxe/src/core/pending.c b/qemu/roms/ipxe/src/core/pending.c
index 7bb0c2e00..96d0cf197 100644
--- a/qemu/roms/ipxe/src/core/pending.c
+++ b/qemu/roms/ipxe/src/core/pending.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/process.h>
diff --git a/qemu/roms/ipxe/src/core/pinger.c b/qemu/roms/ipxe/src/core/pinger.c
index 31ea2ce1c..0ff7bb9f2 100644
--- a/qemu/roms/ipxe/src/core/pinger.c
+++ b/qemu/roms/ipxe/src/core/pinger.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/pixbuf.c b/qemu/roms/ipxe/src/core/pixbuf.c
index 48f8e9f9a..41e18f8dc 100644
--- a/qemu/roms/ipxe/src/core/pixbuf.c
+++ b/qemu/roms/ipxe/src/core/pixbuf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/core/pool.c b/qemu/roms/ipxe/src/core/pool.c
new file mode 100644
index 000000000..0163405f7
--- /dev/null
+++ b/qemu/roms/ipxe/src/core/pool.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Pooled connections
+ *
+ */
+
+#include <assert.h>
+#include <ipxe/pool.h>
+
+/**
+ * Recycle this connection after closing
+ *
+ * @v intf		Data transfer interface
+ */
+void pool_recycle ( struct interface *intf ) {
+
+	intf_poke ( intf, pool_recycle );
+}
+
+/**
+ * Reopen a defunct connection
+ *
+ * @v intf		Data transfer interface
+ */
+void pool_reopen ( struct interface *intf ) {
+
+	intf_poke ( intf, pool_reopen );
+}
+
+/**
+ * Add connection to pool
+ *
+ * @v pool		Pooled connection
+ * @v list		List of pooled connections
+ * @v expiry		Expiry time
+ */
+void pool_add ( struct pooled_connection *pool, struct list_head *list,
+		unsigned long expiry ) {
+
+	/* Sanity check */
+	assert ( list_empty ( &pool->list ) );
+	assert ( ! timer_running ( &pool->timer ) );
+
+	/* Add to list of pooled connections */
+	list_add_tail ( &pool->list, list );
+
+	/* Start expiry timer */
+	start_timer_fixed ( &pool->timer, expiry );
+}
+
+/**
+ * Remove connection from pool
+ *
+ * @v pool		Pooled connection
+ */
+void pool_del ( struct pooled_connection *pool ) {
+
+	/* Remove from list of pooled connections */
+	list_del ( &pool->list );
+	INIT_LIST_HEAD ( &pool->list );
+
+	/* Stop expiry timer */
+	stop_timer ( &pool->timer );
+
+	/* Mark as a freshly recycled connection */
+	pool->flags = POOL_RECYCLED;
+}
+
+/**
+ * Close expired pooled connection
+ *
+ * @v timer		Expiry timer
+ * @v over		Failure indicator
+ */
+void pool_expired ( struct retry_timer *timer, int over __unused ) {
+	struct pooled_connection *pool =
+		container_of ( timer, struct pooled_connection, timer );
+
+	/* Sanity check */
+	assert ( ! list_empty ( &pool->list ) );
+
+	/* Remove from connection pool */
+	list_del ( &pool->list );
+	INIT_LIST_HEAD ( &pool->list );
+
+	/* Close expired connection */
+	pool->expired ( pool );
+}
diff --git a/qemu/roms/ipxe/src/core/posix_io.c b/qemu/roms/ipxe/src/core/posix_io.c
index 8460d0f51..35b52beeb 100644
--- a/qemu/roms/ipxe/src/core/posix_io.c
+++ b/qemu/roms/ipxe/src/core/posix_io.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/core/process.c b/qemu/roms/ipxe/src/core/process.c
index d341a2c37..69852c416 100644
--- a/qemu/roms/ipxe/src/core/process.c
+++ b/qemu/roms/ipxe/src/core/process.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/init.h>
diff --git a/qemu/roms/ipxe/src/core/profile.c b/qemu/roms/ipxe/src/core/profile.c
index 150e6b273..1075047b9 100644
--- a/qemu/roms/ipxe/src/core/profile.c
+++ b/qemu/roms/ipxe/src/core/profile.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/core/random.c b/qemu/roms/ipxe/src/core/random.c
index 8824dca3a..a74175a79 100644
--- a/qemu/roms/ipxe/src/core/random.c
+++ b/qemu/roms/ipxe/src/core/random.c
@@ -4,7 +4,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/timer.h>
diff --git a/qemu/roms/ipxe/src/core/refcnt.c b/qemu/roms/ipxe/src/core/refcnt.c
index 68a86120e..47c975a0b 100644
--- a/qemu/roms/ipxe/src/core/refcnt.c
+++ b/qemu/roms/ipxe/src/core/refcnt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/core/resolv.c b/qemu/roms/ipxe/src/core/resolv.c
index d59a8c0ad..1e3182b0b 100644
--- a/qemu/roms/ipxe/src/core/resolv.c
+++ b/qemu/roms/ipxe/src/core/resolv.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/core/serial.c b/qemu/roms/ipxe/src/core/serial.c
index 7e4460ab9..4ce025519 100644
--- a/qemu/roms/ipxe/src/core/serial.c
+++ b/qemu/roms/ipxe/src/core/serial.c
@@ -1,259 +1,184 @@
 /*
- * The serial port interface routines implement a simple polled i/o
- * interface to a standard serial port.  Due to the space restrictions
- * for the boot blocks, no BIOS support is used (since BIOS requires
- * expensive real/protected mode switches), instead the rudimentary
- * BIOS support is duplicated here.
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
  *
- * The base address and speed for the i/o port are passed from the
- * Makefile in the COMCONSOLE and CONSPEED preprocessor macros.  The
- * line control parameters are currently hard-coded to 8 bits, no
- * parity, 1 stop bit (8N1).  This can be changed in init_serial().
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Serial console
+ *
+ */
 
-#include "stddef.h"
+#include <stddef.h>
 #include <ipxe/init.h>
-#include <ipxe/io.h>
-#include <unistd.h>
+#include <ipxe/uart.h>
+#include <ipxe/console.h>
 #include <ipxe/serial.h>
-#include "config/serial.h"
-
-/* Set default values if none specified */
+#include <config/console.h>
+#include <config/serial.h>
 
-#ifndef COMCONSOLE
-#define COMCONSOLE	0x3f8
+/* Set default console usage if applicable */
+#if ! ( defined ( CONSOLE_SERIAL ) && CONSOLE_EXPLICIT ( CONSOLE_SERIAL ) )
+#undef CONSOLE_SERIAL
+#define CONSOLE_SERIAL ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
 #endif
 
-#ifndef COMSPEED
-#define COMSPEED	9600
-#endif
-
-#ifndef COMDATA
-#define COMDATA		8
+/* UART port number */
+#ifdef COMCONSOLE
+#define CONSOLE_PORT COMCONSOLE
+#else
+#define CONSOLE_PORT 0
 #endif
 
-#ifndef COMPARITY
-#define COMPARITY	0
+/* UART baud rate */
+#ifdef COMPRESERVE
+#define CONSOLE_BAUD 0
+#else
+#define CONSOLE_BAUD COMSPEED
 #endif
 
-#ifndef COMSTOP
-#define COMSTOP		1
+/* UART line control register value */
+#ifdef COMPRESERVE
+#define CONSOLE_LCR 0
+#else
+#define CONSOLE_LCR UART_LCR_WPS ( COMDATA, COMPARITY, COMSTOP )
 #endif
 
-#undef UART_BASE
-#define UART_BASE ( COMCONSOLE )
-
-#undef UART_BAUD
-#define UART_BAUD ( COMSPEED )
+/** Serial console UART */
+struct uart serial_console;
 
-#if ((115200%UART_BAUD) != 0)
-#error Bad ttys0 baud rate
-#endif
-
-#define COMBRD (115200/UART_BAUD)
+/**
+ * Print a character to serial console
+ *
+ * @v character		Character to be printed
+ */
+static void serial_putchar ( int character ) {
 
-/* Line Control Settings */
-#define UART_LCS ( ( ( (COMDATA) - 5 )	<< 0 ) | \
-		   ( ( (COMPARITY) )	<< 3 ) | \
-		   ( ( (COMSTOP) - 1 )	<< 2 ) )
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return;
 
-/* Data */
-#define UART_RBR 0x00
-#define UART_TBR 0x00
+	/* Transmit character */
+	uart_transmit ( &serial_console, character );
+}
 
-/* Control */
-#define UART_IER 0x01
-#define UART_IIR 0x02
-#define UART_FCR 0x02
-#define UART_LCR 0x03
-#define UART_MCR 0x04
-#define UART_DLL 0x00
-#define UART_DLM 0x01
+/**
+ * Get character from serial console
+ *
+ * @ret character	Character read from console
+ */
+static int serial_getchar ( void ) {
+	uint8_t data;
 
-/* Status */
-#define UART_LSR 0x05
-#define  UART_LSR_TEMPT 0x40	/* Transmitter empty */
-#define  UART_LSR_THRE  0x20	/* Transmit-hold-register empty */
-#define  UART_LSR_BI	0x10	/* Break interrupt indicator */
-#define  UART_LSR_FE	0x08	/* Frame error indicator */
-#define  UART_LSR_PE	0x04	/* Parity error indicator */
-#define  UART_LSR_OE	0x02	/* Overrun error indicator */
-#define  UART_LSR_DR	0x01	/* Receiver data ready */
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return 0;
 
-#define UART_MSR 0x06
-#define UART_SCR 0x07
+	/* Wait for data to be ready */
+	while ( ! uart_data_ready ( &serial_console ) ) {}
 
-#if defined(UART_MEM)
-#define uart_readb(addr) readb((addr))
-#define uart_writeb(val,addr) writeb((val),(addr))
-#else
-#define uart_readb(addr) inb((addr))
-#define uart_writeb(val,addr) outb((val),(addr))
-#endif
+	/* Receive data */
+	data = uart_receive ( &serial_console );
 
-/* Boolean for the state of serial driver initialization */
-int serial_initialized = 0;
+	/* Strip any high bit and convert DEL to backspace */
+	data &= 0x7f;
+	if ( data == 0x7f )
+		data = 0x08;
 
-/*
- * void serial_putc(int ch);
- *	Write character `ch' to port UART_BASE.
- */
-void serial_putc ( int ch ) {
-	int i;
-	int status;
-	i = 1000; /* timeout */
-	while(--i > 0) {
-		status = uart_readb(UART_BASE + UART_LSR);
-		if (status & UART_LSR_THRE) { 
-			/* TX buffer emtpy */
-			uart_writeb(ch, UART_BASE + UART_TBR);
-			break;
-		}
-		mdelay(2);
-	}
+	return data;
 }
 
-/*
- * int serial_getc(void);
- *	Read a character from port UART_BASE.
- */
-int serial_getc ( void ) {
-	int status;
-	int ch;
-	do {
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while((status & 1) == 0);
-	ch = uart_readb(UART_BASE + UART_RBR);	/* fetch (first) character */
-	ch &= 0x7f;				/* remove any parity bits we get */
-	if (ch == 0x7f) {			/* Make DEL... look like BS */
-		ch = 0x08;
-	}
-	return ch;
-}
-
-/*
- * int serial_ischar(void);
- *       If there is a character in the input buffer of port UART_BASE,
- *       return nonzero; otherwise return 0.
+/**
+ * Check for character ready to read from serial console
+ *
+ * @ret True		Character available to read
+ * @ret False		No character available to read
  */
-int serial_ischar ( void ) {
-	int status;
-	status = uart_readb(UART_BASE + UART_LSR);	/* line status reg; */
-	return status & 1;		/* rx char available */
-}
+static int serial_iskey ( void ) {
 
-/*
- * int serial_init(void);
- *	Initialize port UART_BASE to speed COMSPEED, line settings 8N1.
- */
-static void serial_init ( void ) {
-	int status;
-	int divisor, lcs;
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return 0;
 
-	DBG ( "Serial port %#x initialising\n", UART_BASE );
+	/* Check UART */
+	return uart_data_ready ( &serial_console );
+}
 
-	divisor = COMBRD;
-	lcs = UART_LCS;
+/** Serial console */
+struct console_driver serial_console_driver __console_driver = {
+	.putchar = serial_putchar,
+	.getchar = serial_getchar,
+	.iskey = serial_iskey,
+	.usage = CONSOLE_SERIAL,
+};
 
+/** Initialise serial console */
+static void serial_init ( void ) {
+	int rc;
 
-#ifdef COMPRESERVE
-	lcs = uart_readb(UART_BASE + UART_LCR) & 0x7f;
-	uart_writeb(0x80 | lcs, UART_BASE + UART_LCR);
-	divisor = (uart_readb(UART_BASE + UART_DLM) << 8) | uart_readb(UART_BASE + UART_DLL);
-	uart_writeb(lcs, UART_BASE + UART_LCR);
-#endif
+	/* Do nothing if we have no default port */
+	if ( ! CONSOLE_PORT )
+		return;
 
-	/* Set Baud Rate Divisor to COMSPEED, and test to see if the
-	 * serial port appears to be present.
-	 */
-	uart_writeb(0x80 | lcs, UART_BASE + UART_LCR);
-	uart_writeb(0xaa, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != 0xaa) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0x55, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != 0x55) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
+	/* Select UART */
+	if ( ( rc = uart_select ( &serial_console, CONSOLE_PORT ) ) != 0 ) {
+		DBG ( "Could not select UART %d: %s\n",
+		      CONSOLE_PORT, strerror ( rc ) );
+		return;
 	}
-	uart_writeb(divisor & 0xff, UART_BASE + UART_DLL);
-	if (uart_readb(UART_BASE + UART_DLL) != (divisor & 0xff)) {
-		DBG ( "Serial port %#x UART_DLL failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0xaa, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != 0xaa) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(0x55, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != 0x55) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb((divisor >> 8) & 0xff, UART_BASE + UART_DLM);
-	if (uart_readb(UART_BASE + UART_DLM) != ((divisor >> 8) & 0xff)) {
-		DBG ( "Serial port %#x UART_DLM failed\n", UART_BASE );
-		goto out;
-	}
-	uart_writeb(lcs, UART_BASE + UART_LCR);
-	
-	/* disable interrupts */
-	uart_writeb(0x0, UART_BASE + UART_IER);
 
-	/* enable fifos */
-	uart_writeb(0x01, UART_BASE + UART_FCR);
+	/* Initialise UART */
+	if ( ( rc = uart_init ( &serial_console, CONSOLE_BAUD,
+				CONSOLE_LCR ) ) != 0 ) {
+		DBG ( "Could not initialise UART %d baud %d LCR %#02x: %s\n",
+		      CONSOLE_PORT, CONSOLE_BAUD, CONSOLE_LCR, strerror ( rc ));
+		return;
+	}
+}
 
-	/* Set clear to send, so flow control works... */
-	uart_writeb((1<<1), UART_BASE + UART_MCR);
+/**
+ * Shut down serial console
+ *
+ * @v flags		Shutdown flags
+ */
+static void serial_shutdown ( int flags __unused ) {
 
-	/* Flush the input buffer. */
-	do {
-		/* rx buffer reg
-		 * throw away (unconditionally the first time)
-		 */
-	        (void) uart_readb(UART_BASE + UART_RBR);
-		/* line status reg */
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while(status & UART_LSR_DR);
+	/* Do nothing if we have no UART */
+	if ( ! serial_console.base )
+		return;
 
-	/* Note that serial support has been initialized */
-	serial_initialized = 1;
- out:
-	return;
-}
+	/* Flush any pending output */
+	uart_flush ( &serial_console );
 
-/*
- * void serial_fini(void);
- *	Cleanup our use of the serial port, in particular flush the
- *	output buffer so we don't accidentially lose characters.
- */
-static void serial_fini ( int flags __unused ) {
-	int i, status;
-	/* Flush the output buffer to avoid dropping characters,
-	 * if we are reinitializing the serial port.
-	 */
-	i = 10000; /* timeout */
-	do {
-		status = uart_readb(UART_BASE + UART_LSR);
-	} while((--i > 0) && !(status & UART_LSR_TEMPT));
-	/* Don't mark it as disabled; it's still usable */
+	/* Leave console enabled; it's still usable */
 }
 
-/**
- * Serial driver initialisation function
- *
- * Initialise serial port early on so that it is available to capture
- * early debug messages.
- */
-struct init_fn serial_init_fn __init_fn ( INIT_SERIAL ) = {
+/** Serial console initialisation function */
+struct init_fn serial_console_init_fn __init_fn ( INIT_CONSOLE ) = {
 	.initialise = serial_init,
 };
 
-/** Serial driver startup function */
+/** Serial console startup function */
 struct startup_fn serial_startup_fn __startup_fn ( STARTUP_EARLY ) = {
-	.shutdown = serial_fini,
+	.shutdown = serial_shutdown,
 };
diff --git a/qemu/roms/ipxe/src/core/serial_console.c b/qemu/roms/ipxe/src/core/serial_console.c
deleted file mode 100644
index de9b84ca7..000000000
--- a/qemu/roms/ipxe/src/core/serial_console.c
+++ /dev/null
@@ -1,42 +0,0 @@
-#include <ipxe/init.h>
-#include <ipxe/serial.h>
-#include <ipxe/console.h>
-#include <config/console.h>
-
-/** @file
- *
- * Serial console
- *
- */
-
-/* Set default console usage if applicable */
-#if ! ( defined ( CONSOLE_SERIAL ) && CONSOLE_EXPLICIT ( CONSOLE_SERIAL ) )
-#undef CONSOLE_SERIAL
-#define CONSOLE_SERIAL ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG )
-#endif
-
-struct console_driver serial_console __console_driver;
-
-static void serial_console_init ( void ) {
-	/*
-	 * Check if serial driver initialization is done.
-	 * If so, it's time to enable the serial console.
-	 */
-	if ( serial_initialized )
-		serial_console.disabled = 0;
-}
-
-struct console_driver serial_console __console_driver = {
-	.putchar = serial_putc,
-	.getchar = serial_getc,
-	.iskey = serial_ischar,
-	.disabled = CONSOLE_DISABLED,
-	.usage = CONSOLE_SERIAL,
-};
-
-/**
- * Serial console initialisation function
- */
-struct init_fn serial_console_init_fn __init_fn ( INIT_CONSOLE ) = {
-	.initialise = serial_console_init,
-};
diff --git a/qemu/roms/ipxe/src/core/settings.c b/qemu/roms/ipxe/src/core/settings.c
index 5e16b27d0..12e6c7d61 100644
--- a/qemu/roms/ipxe/src/core/settings.c
+++ b/qemu/roms/ipxe/src/core/settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -35,6 +39,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/uuid.h>
 #include <ipxe/uri.h>
 #include <ipxe/base16.h>
+#include <ipxe/base64.h>
 #include <ipxe/pci.h>
 #include <ipxe/init.h>
 #include <ipxe/version.h>
@@ -337,17 +342,20 @@ struct settings * autovivify_child_settings ( struct settings *parent,
  */
 const char * settings_name ( struct settings *settings ) {
 	static char buf[16];
-	char tmp[ sizeof ( buf ) ];
+	char tmp[ 1 /* '.' */ + sizeof ( buf ) ];
 
 	/* Find target settings block */
 	settings = settings_target ( settings );
 
 	/* Construct name */
-	for ( buf[2] = buf[0] = 0 ; settings ; settings = settings->parent ) {
-		memcpy ( tmp, buf, sizeof ( tmp ) );
-		snprintf ( buf, sizeof ( buf ), ".%s%s", settings->name, tmp );
+	buf[0] = '\0';
+	tmp[0] = '\0';
+	for ( ; settings->parent ; settings = settings->parent ) {
+		memcpy ( ( tmp + 1 ), buf, ( sizeof ( tmp ) - 1 ) );
+		snprintf ( buf, sizeof ( buf ), "%s%s", settings->name, tmp );
+		tmp[0] = '.';
 	}
-	return ( buf + 2 );
+	return buf;
 }
 
 /**
@@ -499,10 +507,10 @@ int register_settings ( struct settings *settings, struct settings *parent,
  */
 void unregister_settings ( struct settings *settings ) {
 	struct settings *child;
-	struct settings *tmp;
 
 	/* Unregister child settings */
-	list_for_each_entry_safe ( child, tmp, &settings->children, siblings ) {
+	while ( ( child = list_first_entry ( &settings->children,
+					     struct settings, siblings ) ) ) {
 		unregister_settings ( child );
 	}
 
@@ -1999,32 +2007,6 @@ const struct setting_type setting_type_uint32 __setting_type =
 	SETTING_TYPE_UINT ( SETTING_TYPE_INT32 );
 
 /**
- * Format hex string setting value
- *
- * @v delimiter		Byte delimiter
- * @v raw		Raw setting value
- * @v raw_len		Length of raw setting value
- * @v buf		Buffer to contain formatted value
- * @v len		Length of buffer
- * @ret len		Length of formatted value, or negative error
- */
-static int format_hex_setting ( const char *delimiter, const void *raw,
-				size_t raw_len, char *buf, size_t len ) {
-	const uint8_t *bytes = raw;
-	int used = 0;
-	unsigned int i;
-
-	if ( len )
-		buf[0] = 0; /* Ensure that a terminating NUL exists */
-	for ( i = 0 ; i < raw_len ; i++ ) {
-		used += ssnprintf ( ( buf + used ), ( len - used ),
-				    "%s%02x", ( used ? delimiter : "" ),
-				    bytes[i] );
-	}
-	return used;
-}
-
-/**
  * Parse hex string setting value (using colon delimiter)
  *
  * @v type		Setting type
@@ -2036,7 +2018,7 @@ static int format_hex_setting ( const char *delimiter, const void *raw,
  */
 static int parse_hex_setting ( const struct setting_type *type __unused,
 			       const char *value, void *buf, size_t len ) {
-	return hex_decode ( value, ':', buf, len );
+	return hex_decode ( ':', value, buf, len );
 }
 
 /**
@@ -2052,7 +2034,7 @@ static int parse_hex_setting ( const struct setting_type *type __unused,
 static int format_hex_colon_setting ( const struct setting_type *type __unused,
 				      const void *raw, size_t raw_len,
 				      char *buf, size_t len ) {
-	return format_hex_setting ( ":", raw, raw_len, buf, len );
+	return hex_encode ( ':', raw, raw_len, buf, len );
 }
 
 /**
@@ -2068,7 +2050,7 @@ static int format_hex_colon_setting ( const struct setting_type *type __unused,
 static int parse_hex_hyphen_setting ( const struct setting_type *type __unused,
 				      const char *value, void *buf,
 				      size_t len ) {
-	return hex_decode ( value, '-', buf, len );
+	return hex_decode ( '-', value, buf, len );
 }
 
 /**
@@ -2084,7 +2066,7 @@ static int parse_hex_hyphen_setting ( const struct setting_type *type __unused,
 static int format_hex_hyphen_setting ( const struct setting_type *type __unused,
 				       const void *raw, size_t raw_len,
 				       char *buf, size_t len ) {
-	return format_hex_setting ( "-", raw, raw_len, buf, len );
+	return hex_encode ( '-', raw, raw_len, buf, len );
 }
 
 /**
@@ -2099,7 +2081,7 @@ static int format_hex_hyphen_setting ( const struct setting_type *type __unused,
  */
 static int parse_hex_raw_setting ( const struct setting_type *type __unused,
 				   const char *value, void *buf, size_t len ) {
-	return hex_decode ( value, 0, buf, len );
+	return hex_decode ( 0, value, buf, len );
 }
 
 /**
@@ -2115,7 +2097,7 @@ static int parse_hex_raw_setting ( const struct setting_type *type __unused,
 static int format_hex_raw_setting ( const struct setting_type *type __unused,
 				    const void *raw, size_t raw_len,
 				    char *buf, size_t len ) {
-	return format_hex_setting ( "", raw, raw_len, buf, len );
+	return hex_encode ( 0, raw, raw_len, buf, len );
 }
 
 /** A hex-string setting (colon-delimited) */
@@ -2140,6 +2122,46 @@ const struct setting_type setting_type_hexraw __setting_type = {
 };
 
 /**
+ * Parse Base64-encoded setting value
+ *
+ * @v type		Setting type
+ * @v value		Formatted setting value
+ * @v buf		Buffer to contain raw value
+ * @v len		Length of buffer
+ * @v size		Integer size, in bytes
+ * @ret len		Length of raw value, or negative error
+ */
+static int parse_base64_setting ( const struct setting_type *type __unused,
+				  const char *value, void *buf, size_t len ) {
+
+	return base64_decode ( value, buf, len );
+}
+
+/**
+ * Format Base64-encoded setting value
+ *
+ * @v type		Setting type
+ * @v raw		Raw setting value
+ * @v raw_len		Length of raw setting value
+ * @v buf		Buffer to contain formatted value
+ * @v len		Length of buffer
+ * @ret len		Length of formatted value, or negative error
+ */
+static int format_base64_setting ( const struct setting_type *type __unused,
+				   const void *raw, size_t raw_len,
+				   char *buf, size_t len ) {
+
+	return base64_encode ( raw, raw_len, buf, len );
+}
+
+/** A Base64-encoded setting */
+const struct setting_type setting_type_base64 __setting_type = {
+	.name = "base64",
+	.parse = parse_base64_setting,
+	.format = format_base64_setting,
+};
+
+/**
  * Format UUID setting value
  *
  * @v type		Setting type
diff --git a/qemu/roms/ipxe/src/core/string.c b/qemu/roms/ipxe/src/core/string.c
index e53c283c2..3e658e54e 100644
--- a/qemu/roms/ipxe/src/core/string.c
+++ b/qemu/roms/ipxe/src/core/string.c
@@ -1,353 +1,501 @@
 /*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 2004 Tobias Lorenz
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
  *
- *  string handling functions
- *  based on linux/lib/string.c
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-FILE_LICENCE ( GPL2_ONLY );
-
-/*
- * stupid library routines.. The optimized versions should generally be found
- * as inline code in <asm-xx/string.h>
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
  *
- * These are buggy as well..
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  *
- * * Fri Jun 25 1999, Ingo Oeser <ioe@informatik.tu-chemnitz.de>
- * -  Added strsep() which will replace strtok() soon (because strsep() is
- *    reentrant and should be faster). Use only strsep() in new code, please.
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
- 
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
 
-/* *** FROM string.c *** */
+/** @file
+ *
+ * String functions
+ *
+ */
 
-#ifndef __HAVE_ARCH_STRCPY
 /**
- * strcpy - Copy a %NUL terminated string
- * @dest: Where to copy the string to
- * @src: Where to copy the string from
+ * Fill memory region
+ *
+ * @v dest		Destination region
+ * @v character		Fill character
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strcpy(char * dest,const char *src)
-{
-	char *tmp = dest;
+void * generic_memset ( void *dest, int character, size_t len ) {
+	uint8_t *dest_bytes = dest;
 
-	while ((*dest++ = *src++) != '\0')
-		/* nothing */;
-	return tmp;
+	while ( len-- )
+		*(dest_bytes++) = character;
+	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNCPY
 /**
- * strncpy - Copy a length-limited, %NUL-terminated string
- * @dest: Where to copy the string to
- * @src: Where to copy the string from
- * @count: The maximum number of bytes to copy
+ * Copy memory region
  *
- * Note that unlike userspace strncpy, this does not %NUL-pad the buffer.
- * However, the result is not %NUL-terminated if the source exceeds
- * @count bytes.
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strncpy(char * dest,const char *src,size_t count)
-{
-	char *tmp = dest;
-
-	while (count-- && (*dest++ = *src++) != '\0')
-		/* nothing */;
+void * generic_memcpy ( void *dest, const void *src, size_t len ) {
+	const uint8_t *src_bytes = src;
+	uint8_t *dest_bytes = dest;
 
-	return tmp;
+	while ( len-- )
+		*(dest_bytes++) = *(src_bytes++);
+	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCAT
 /**
- * strcat - Append one %NUL-terminated string to another
- * @dest: The string to be appended to
- * @src: The string to append to it
+ * Copy (possibly overlapping) memory region
+ *
+ * @v dest		Destination region
+ * @v src		Source region
+ * @v len		Length
+ * @ret dest		Destination region
  */
-char * strcat(char * dest, const char * src)
-{
-	char *tmp = dest;
-
-	while (*dest)
-		dest++;
-	while ((*dest++ = *src++) != '\0')
-		;
+void * generic_memmove ( void *dest, const void *src, size_t len ) {
+	const uint8_t *src_bytes = ( src + len );
+	uint8_t *dest_bytes = ( dest + len );
+
+	if ( dest < src )
+		return memcpy ( dest, src, len );
+	while ( len-- )
+		*(--dest_bytes) = *(--src_bytes);
+	return dest;
+}
 
-	return tmp;
+/**
+ * Compare memory regions
+ *
+ * @v first		First region
+ * @v second		Second region
+ * @v len		Length
+ * @ret diff		Difference
+ */
+int memcmp ( const void *first, const void *second, size_t len ) {
+	const uint8_t *first_bytes = first;
+	const uint8_t *second_bytes = second;
+	int diff;
+
+	while ( len-- ) {
+		diff = ( *(second_bytes++) - *(first_bytes++) );
+		if ( diff )
+			return diff;
+	}
+	return 0;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCMP
 /**
- * strcmp - Compare two strings
- * @cs: One string
- * @ct: Another string
+ * Find character within a memory region
+ *
+ * @v src		Source region
+ * @v character		Character to find
+ * @v len		Length
+ * @ret found		Found character, or NULL if not found
  */
-int strcmp(const char * cs,const char * ct)
-{
-	register signed char __res;
+void * memchr ( const void *src, int character, size_t len ) {
+	const uint8_t *src_bytes = src;
 
-	while (1) {
-		if ((__res = *cs - *ct++) != 0 || !*cs++)
-			break;
+	for ( ; len-- ; src_bytes++ ) {
+		if ( *src_bytes == character )
+			return ( ( void * ) src_bytes );
 	}
-
-	return __res;
+	return NULL;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNCMP
 /**
- * strncmp - Compare two length-limited strings
- * @cs: One string
- * @ct: Another string
- * @count: The maximum number of bytes to compare
+ * Swap memory regions
+ *
+ * @v first		First region
+ * @v second		Second region
+ * @v len		Length
+ * @ret first		First region
  */
-int strncmp(const char * cs,const char * ct,size_t count)
-{
-	register signed char __res = 0;
-
-	while (count) {
-		if ((__res = *cs - *ct++) != 0 || !*cs++)
-			break;
-		count--;
+void * memswap ( void *first, void *second, size_t len ) {
+	uint8_t *first_bytes = first;
+	uint8_t *second_bytes = second;
+	uint8_t temp;
+
+	for ( ; len-- ; first_bytes++, second_bytes++ ) {
+		temp = *first_bytes;
+		*first_bytes = *second_bytes;
+		*second_bytes = temp;
 	}
+	return first;
+}
+
+/**
+ * Compare strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @ret diff		Difference
+ */
+int strcmp ( const char *first, const char *second ) {
 
-	return __res;
+	return strncmp ( first, second, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCASECMP
-int strcasecmp(const char *a, const char *b)
-{
-	while (*a && *b && (*a & ~0x20) == (*b & ~0x20)) {a++; b++; }
-	return((*a & ~0x20) - (*b & ~0x20));
+/**
+ * Compare strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @v max		Maximum length to compare
+ * @ret diff		Difference
+ */
+int strncmp ( const char *first, const char *second, size_t max ) {
+	const uint8_t *first_bytes = ( ( const uint8_t * ) first );
+	const uint8_t *second_bytes = ( ( const uint8_t * ) second );
+	int diff;
+
+	for ( ; max-- ; first_bytes++, second_bytes++ ) {
+		diff = ( *second_bytes - *first_bytes );
+		if ( diff )
+			return diff;
+		if ( ! *first_bytes )
+			return 0;
+	}
+	return 0;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRCHR
 /**
- * strchr - Find the first occurrence of a character in a string
- * @s: The string to be searched
- * @c: The character to search for
+ * Compare case-insensitive strings
+ *
+ * @v first		First string
+ * @v second		Second string
+ * @ret diff		Difference
  */
-char * strchr(const char * s, int c)
-{
-	for(; *s != (char) c; ++s)
-		if (*s == '\0')
-			return NULL;
-	return (char *) s;
+int strcasecmp ( const char *first, const char *second ) {
+	const uint8_t *first_bytes = ( ( const uint8_t * ) first );
+	const uint8_t *second_bytes = ( ( const uint8_t * ) second );
+	int diff;
+
+	for ( ; ; first_bytes++, second_bytes++ ) {
+		diff = ( toupper ( *second_bytes ) -
+			 toupper ( *first_bytes ) );
+		if ( diff )
+			return diff;
+		if ( ! *first_bytes )
+			return 0;
+	}
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRRCHR
 /**
- * strrchr - Find the last occurrence of a character in a string
- * @s: The string to be searched
- * @c: The character to search for
+ * Get length of string
+ *
+ * @v src		String
+ * @ret len		Length
  */
-char * strrchr(const char * s, int c)
-{
-       const char *p = s + strlen(s);
-       do {
-           if (*p == (char)c)
-               return (char *)p;
-       } while (--p >= s);
-       return NULL;
+size_t strlen ( const char *src ) {
+
+	return strnlen ( src, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRLEN
 /**
- * strlen - Find the length of a string
- * @s: The string to be sized
+ * Get length of string
+ *
+ * @v src		String
+ * @v max		Maximum length
+ * @ret len		Length
  */
-size_t strlen(const char * s)
-{
-	const char *sc;
+size_t strnlen ( const char *src, size_t max ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	size_t len = 0;
 
-	for (sc = s; *sc != '\0'; ++sc)
-		/* nothing */;
-	return sc - s;
+	while ( max-- && *(src_bytes++) )
+		len++;
+	return len;
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRNLEN
 /**
- * strnlen - Find the length of a length-limited string
- * @s: The string to be sized
- * @count: The maximum number of bytes to search
+ * Find character within a string
+ *
+ * @v src		String
+ * @v character		Character to find
+ * @ret found		Found character, or NULL if not found
  */
-size_t strnlen(const char * s, size_t count)
-{
-	const char *sc;
+char * strchr ( const char *src, int character ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
 
-	for (sc = s; count-- && *sc != '\0'; ++sc)
-		/* nothing */;
-	return sc - s;
+	for ( ; ; src_bytes++ ) {
+		if ( *src_bytes == character )
+			return ( ( char * ) src_bytes );
+		if ( ! *src_bytes )
+			return NULL;
+	}
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMSET
 /**
- * memset - Fill a region of memory with the given value
- * @s: Pointer to the start of the area.
- * @c: The byte to fill the area with
- * @count: The size of the area.
+ * Find rightmost character within a string
  *
- * Do not use memset() to access IO space, use memset_io() instead.
+ * @v src		String
+ * @v character		Character to find
+ * @ret found		Found character, or NULL if not found
  */
-void * memset(void * s,int c,size_t count)
-{
-	char *xs = (char *) s;
+char * strrchr ( const char *src, int character ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	const uint8_t *start = src_bytes;
+
+	while ( *src_bytes )
+		src_bytes++;
+	for ( src_bytes-- ; src_bytes >= start ; src_bytes-- ) {
+		if ( *src_bytes == character )
+			return ( ( char * ) src_bytes );
+	}
+	return NULL;
+}
 
-	while (count--)
-		*xs++ = c;
+/**
+ * Find substring
+ *
+ * @v haystack		String
+ * @v needle		Substring
+ * @ret found		Found substring, or NULL if not found
+ */
+char * strstr ( const char *haystack, const char *needle ) {
+	size_t len = strlen ( needle );
 
-	return s;
+	for ( ; *haystack ; haystack++ ) {
+		if ( memcmp ( haystack, needle, len ) == 0 )
+			return ( ( char * ) haystack );
+	}
+	return NULL;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCPY
 /**
- * memcpy - Copy one area of memory to another
- * @dest: Where to copy to
- * @src: Where to copy from
- * @count: The size of the area.
+ * Copy string
  *
- * You should not use this function to access IO space, use memcpy_toio()
- * or memcpy_fromio() instead.
+ * @v dest		Destination string
+ * @v src		Source string
+ * @ret dest		Destination string
  */
-void * memcpy(void * dest,const void *src,size_t count)
-{
-	char *tmp = (char *) dest, *s = (char *) src;
+char * strcpy ( char *dest, const char *src ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	uint8_t *dest_bytes = ( ( uint8_t * ) dest );
+
+	/* We cannot use strncpy(), since that would pad the destination */
+	for ( ; ; src_bytes++, dest_bytes++ ) {
+		*dest_bytes = *src_bytes;
+		if ( ! *dest_bytes )
+			break;
+	}
+	return dest;
+}
 
-	while (count--)
-		*tmp++ = *s++;
+/**
+ * Copy string
+ *
+ * @v dest		Destination string
+ * @v src		Source string
+ * @v max		Maximum length
+ * @ret dest		Destination string
+ */
+char * strncpy ( char *dest, const char *src, size_t max ) {
+	const uint8_t *src_bytes = ( ( const uint8_t * ) src );
+	uint8_t *dest_bytes = ( ( uint8_t * ) dest );
 
+	for ( ; max ; max--, src_bytes++, dest_bytes++ ) {
+		*dest_bytes = *src_bytes;
+		if ( ! *dest_bytes )
+			break;
+	}
+	while ( max-- )
+		*(dest_bytes++) = '\0';
 	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMMOVE
 /**
- * memmove - Copy one area of memory to another
- * @dest: Where to copy to
- * @src: Where to copy from
- * @count: The size of the area.
+ * Concatenate string
  *
- * Unlike memcpy(), memmove() copes with overlapping areas.
+ * @v dest		Destination string
+ * @v src		Source string
+ * @ret dest		Destination string
  */
-void * memmove(void * dest,const void *src,size_t count)
-{
-	char *tmp, *s;
-
-	if (dest <= src) {
-		tmp = (char *) dest;
-		s = (char *) src;
-		while (count--)
-			*tmp++ = *s++;
-		}
-	else {
-		tmp = (char *) dest + count;
-		s = (char *) src + count;
-		while (count--)
-			*--tmp = *--s;
-		}
+char * strcat ( char *dest, const char *src ) {
 
+	strcpy ( ( dest + strlen ( dest ) ), src );
 	return dest;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCMP
 /**
- * memcmp - Compare two areas of memory
- * @cs: One area of memory
- * @ct: Another area of memory
- * @count: The size of the area.
+ * Duplicate string
+ *
+ * @v src		Source string
+ * @ret dup		Duplicated string, or NULL if allocation failed
  */
-int memcmp(const void * cs,const void * ct,size_t count)
-{
-	const unsigned char *su1, *su2;
-	int res = 0;
+char * strdup ( const char *src ) {
 
-	for( su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--)
-		if ((res = *su1 - *su2) != 0)
-			break;
-	return res;
+	return strndup ( src, ~( ( size_t ) 0 ) );
 }
-#endif
 
-#ifndef __HAVE_ARCH_STRSTR
 /**
- * strstr - Find the first substring in a %NUL terminated string
- * @s1: The string to be searched
- * @s2: The string to search for
+ * Duplicate string
+ *
+ * @v src		Source string
+ * @v max		Maximum length
+ * @ret dup		Duplicated string, or NULL if allocation failed
  */
-char * strstr(const char * s1,const char * s2)
-{
-	int l1, l2;
-
-	l2 = strlen(s2);
-	if (!l2)
-		return (char *) s1;
-	l1 = strlen(s1);
-	while (l1 >= l2) {
-		l1--;
-		if (!memcmp(s1,s2,l2))
-			return (char *) s1;
-		s1++;
-	}
-	return NULL;
+char * strndup ( const char *src, size_t max ) {
+	size_t len = strnlen ( src, max );
+        char *dup;
+
+        dup = malloc ( len + 1 /* NUL */ );
+        if ( dup ) {
+		memcpy ( dup, src, len );
+		dup[len] = '\0';
+        }
+        return dup;
+}
+
+/**
+ * Calculate digit value
+ *
+ * @v character		Digit character
+ * @ret digit		Digit value
+ *
+ * Invalid digits will be returned as a value greater than or equal to
+ * the numeric base.
+ */
+unsigned int digit_value ( unsigned int character ) {
+
+	if ( character >= 'a' )
+		return ( character - ( 'a' - 10 ) );
+	if ( character >= 'A' )
+		return ( character - ( 'A' - 10 ) );
+	if ( character <= '9' )
+		return ( character - '0' );
+	return character;
 }
-#endif
 
-#ifndef __HAVE_ARCH_MEMCHR
 /**
- * memchr - Find a character in an area of memory.
- * @s: The memory area
- * @c: The byte to search for
- * @n: The size of the area.
+ * Preprocess string for strtoul() or strtoull()
  *
- * returns the address of the first occurrence of @c, or %NULL
- * if @c is not found
+ * @v string		String
+ * @v negate		Final value should be negated
+ * @v base		Numeric base
+ * @ret string		Remaining string
  */
-void * memchr(const void *s, int c, size_t n)
-{
-	const unsigned char *p = s;
-	while (n-- != 0) {
-        	if ((unsigned char)c == *p++) {
-			return (void *)(p-1);
+static const char * strtoul_pre ( const char *string, int *negate, int *base ) {
+
+	/* Skip any leading whitespace */
+	while ( isspace ( *string ) )
+		string++;
+
+	/* Process arithmetic sign, if present */
+	*negate = 0;
+	if ( *string == '-' ) {
+		string++;
+		*negate = 1;
+	} else if ( *string == '+' ) {
+		string++;
+	}
+
+	/* Process base, if present */
+	if ( *base == 0 ) {
+		*base = 10;
+		if ( *string == '0' ) {
+			string++;
+			*base = 8;
+			if ( ( *string & ~0x20 ) == 'X' ) {
+				string++;
+				*base = 16;
+			}
 		}
 	}
-	return NULL;
+
+	return string;
 }
 
-#endif
+/**
+ * Convert string to numeric value
+ *
+ * @v string		String
+ * @v endp		End pointer (or NULL)
+ * @v base		Numeric base (or zero to autodetect)
+ * @ret value		Numeric value
+ */
+unsigned long strtoul ( const char *string, char **endp, int base ) {
+	unsigned long value = 0;
+	unsigned int digit;
+	int negate;
+
+	/* Preprocess string */
+	string = strtoul_pre ( string, &negate, &base );
+
+	/* Process digits */
+	for ( ; ; string++ ) {
+		digit = digit_value ( *string );
+		if ( digit >= ( unsigned int ) base )
+			break;
+		value = ( ( value * base ) + digit );
+	}
+
+	/* Negate value if, applicable */
+	if ( negate )
+		value = -value;
 
-char * strndup(const char *s, size_t n)
-{
-        size_t len = strnlen(s,n);
-        char *new;
+	/* Fill in end pointer, if applicable */
+	if ( endp )
+		*endp = ( ( char * ) string );
 
-        new = malloc(len+1);
-        if (new) {
-                new[len] = '\0';
-                memcpy(new,s,len);
-        }
-        return new;
+	return value;
 }
 
-char * strdup(const char *s) {
-	return strndup(s, ~((size_t)0));
+/**
+ * Convert string to numeric value
+ *
+ * @v string		String
+ * @v endp		End pointer (or NULL)
+ * @v base		Numeric base (or zero to autodetect)
+ * @ret value		Numeric value
+ */
+unsigned long long strtoull ( const char *string, char **endp, int base ) {
+	unsigned long long value = 0;
+	unsigned int digit;
+	int negate;
+
+	/* Preprocess string */
+	string = strtoul_pre ( string, &negate, &base );
+
+	/* Process digits */
+	for ( ; ; string++ ) {
+		digit = digit_value ( *string );
+		if ( digit >= ( unsigned int ) base )
+			break;
+		value = ( ( value * base ) + digit );
+	}
+
+	/* Negate value if, applicable */
+	if ( negate )
+		value = -value;
+
+	/* Fill in end pointer, if applicable */
+	if ( endp )
+		*endp = ( ( char * ) string );
+
+	return value;
 }
diff --git a/qemu/roms/ipxe/src/core/stringextra.c b/qemu/roms/ipxe/src/core/stringextra.c
index 0a509852e..18ffc6301 100644
--- a/qemu/roms/ipxe/src/core/stringextra.c
+++ b/qemu/roms/ipxe/src/core/stringextra.c
@@ -38,122 +38,6 @@ FILE_LICENCE ( GPL2_ONLY );
 
 /* *** FROM string.c *** */
 
-#ifndef __HAVE_ARCH_STRNICMP
-/**
- * strnicmp - Case insensitive, length-limited string comparison
- * @s1: One string
- * @s2: The other string
- * @len: the maximum number of characters to compare
- */
-int strnicmp(const char *s1, const char *s2, size_t len)
-{
-	/* Yes, Virginia, it had better be unsigned */
-	unsigned char c1, c2;
-
-	c1 = 0;	c2 = 0;
-	if (len) {
-		do {
-			c1 = *s1; c2 = *s2;
-			s1++; s2++;
-			if (!c1)
-				break;
-			if (!c2)
-				break;
-			if (c1 == c2)
-				continue;
-			c1 = tolower(c1);
-			c2 = tolower(c2);
-			if (c1 != c2)
-				break;
-		} while (--len);
-	}
-	return (int)c1 - (int)c2;
-}
-#endif
-
-char * ___strtok;
-
-#ifndef __HAVE_ARCH_STRNCAT
-/**
- * strncat - Append a length-limited, %NUL-terminated string to another
- * @dest: The string to be appended to
- * @src: The string to append to it
- * @count: The maximum numbers of bytes to copy
- *
- * Note that in contrast to strncpy, strncat ensures the result is
- * terminated.
- */
-char * strncat(char *dest, const char *src, size_t count)
-{
-	char *tmp = dest;
-
-	if (count) {
-		while (*dest)
-			dest++;
-		while ((*dest++ = *src++)) {
-			if (--count == 0) {
-				*dest = '\0';
-				break;
-			}
-		}
-	}
-
-	return tmp;
-}
-#endif
-
-#ifndef __HAVE_ARCH_STRSPN
-/**
- * strspn - Calculate the length of the initial substring of @s which only
- * 	contain letters in @accept
- * @s: The string to be searched
- * @accept: The string to search for
- */
-size_t strspn(const char *s, const char *accept)
-{
-	const char *p;
-	const char *a;
-	size_t count = 0;
-
-	for (p = s; *p != '\0'; ++p) {
-		for (a = accept; *a != '\0'; ++a) {
-			if (*p == *a)
-				break;
-		}
-		if (*a == '\0')
-			return count;
-		++count;
-	}
-
-	return count;
-}
-#endif
-
-#ifndef __HAVE_ARCH_STRCSPN
-/**
- * strcspn - Calculate the length of the initial substring of @s which only
- * 	contain letters not in @reject
- * @s: The string to be searched
- * @accept: The string to search for
- */
-size_t strcspn(const char *s, const char *reject)
-{
-	const char *p;
-	const char *r;
-	size_t count = 0;
-
-	for (p = s; *p != '\0'; ++p) {
-		for (r = reject; *r != '\0'; ++r) {
-			if (*p == *r)
-				return count;
-		}
-		++count;
-	}
-
-	return count;
-}
-#endif
-
 #ifndef __HAVE_ARCH_STRPBRK
 /**
  * strpbrk - Find the first occurrence of a set of characters
@@ -174,35 +58,6 @@ char * strpbrk(const char * cs,const char * ct)
 }
 #endif
 
-#ifndef __HAVE_ARCH_STRTOK
-/**
- * strtok - Split a string into tokens
- * @s: The string to be searched
- * @ct: The characters to search for
- *
- * WARNING: strtok is deprecated, use strsep instead.
- */
-char * strtok(char * s,const char * ct)
-{
-	char *sbegin, *send;
-
-	sbegin  = s ? s : ___strtok;
-	if (!sbegin) {
-		return NULL;
-	}
-	sbegin += strspn(sbegin,ct);
-	if (*sbegin == '\0') {
-		___strtok = NULL;
-		return( NULL );
-	}
-	send = strpbrk( sbegin, ct);
-	if (send && *send != '\0')
-		*send++ = '\0';
-	___strtok = send;
-	return (sbegin);
-}
-#endif
-
 #ifndef __HAVE_ARCH_STRSEP
 /**
  * strsep - Split a string into tokens
@@ -230,46 +85,3 @@ char * strsep(char **s, const char *ct)
 	return sbegin;
 }
 #endif
-
-#ifndef __HAVE_ARCH_BCOPY
-/**
- * bcopy - Copy one area of memory to another
- * @src: Where to copy from
- * @dest: Where to copy to
- * @count: The size of the area.
- *
- * Note that this is the same as memcpy(), with the arguments reversed.
- * memcpy() is the standard, bcopy() is a legacy BSD function.
- *
- * You should not use this function to access IO space, use memcpy_toio()
- * or memcpy_fromio() instead.
- */
-char * bcopy(const char * src, char * dest, int count)
-{
-	return memmove(dest,src,count);
-}
-#endif
-
-#ifndef __HAVE_ARCH_MEMSCAN
-/**
- * memscan - Find a character in an area of memory.
- * @addr: The memory area
- * @c: The byte to search for
- * @size: The size of the area.
- *
- * returns the address of the first occurrence of @c, or 1 byte past
- * the area if @c is not found
- */
-void * memscan(const void * addr, int c, size_t size)
-{
-	unsigned char * p = (unsigned char *) addr;
-
-	while (size) {
-		if (*p == c)
-			return (void *) p;
-		p++;
-		size--;
-	}
-  	return (void *) p;
-}
-#endif
diff --git a/qemu/roms/ipxe/src/core/strtoull.c b/qemu/roms/ipxe/src/core/strtoull.c
deleted file mode 100644
index 00986eef0..000000000
--- a/qemu/roms/ipxe/src/core/strtoull.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (C) 2006 Michael Brown <mbrown@fensystems.co.uk>
- * Copyright (C) 2010 Piotr Jaroszyński <p.jaroszynski@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdlib.h>
-#include <ctype.h>
-
-/*
- * Despite being exactly the same as strtoul() except the long long instead of
- * long it ends up being much bigger so provide a separate implementation in a
- * separate object so that it won't be linked in if not used.
- */
-unsigned long long strtoull ( const char *p, char **endp, int base ) {
-	unsigned long long ret = 0;
-	int negative = 0;
-	unsigned int charval;
-
-	while ( isspace ( *p ) )
-		p++;
-
-	if ( *p == '-' ) {
-		negative = 1;
-		p++;
-	}
-
-	base = strtoul_base ( &p, base );
-
-	while ( 1 ) {
-		charval = strtoul_charval ( *p );
-		if ( charval >= ( unsigned int ) base )
-			break;
-		ret = ( ( ret * base ) + charval );
-		p++;
-	}
-
-	if ( negative )
-		ret = -ret;
-
-	if ( endp )
-		*endp = ( char * ) p;
-
-	return ( ret );
-}
diff --git a/qemu/roms/ipxe/src/core/time.c b/qemu/roms/ipxe/src/core/time.c
index f70e1981d..29a924ebe 100644
--- a/qemu/roms/ipxe/src/core/time.c
+++ b/qemu/roms/ipxe/src/core/time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <time.h>
 
diff --git a/qemu/roms/ipxe/src/core/timer.c b/qemu/roms/ipxe/src/core/timer.c
index 18c2b2849..dbd89f12b 100644
--- a/qemu/roms/ipxe/src/core/timer.c
+++ b/qemu/roms/ipxe/src/core/timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <unistd.h>
 
diff --git a/qemu/roms/ipxe/src/core/uart.c b/qemu/roms/ipxe/src/core/uart.c
new file mode 100644
index 000000000..b85fe0767
--- /dev/null
+++ b/qemu/roms/ipxe/src/core/uart.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/uart.h>
+
+/** Timeout for transmit holding register to become empty */
+#define UART_THRE_TIMEOUT_MS 100
+
+/** Timeout for transmitter to become empty */
+#define UART_TEMT_TIMEOUT_MS 1000
+
+/**
+ * Transmit data
+ *
+ * @v uart		UART
+ * @v data		Data
+ */
+void uart_transmit ( struct uart *uart, uint8_t data ) {
+	unsigned int i;
+	uint8_t lsr;
+
+	/* Wait for transmitter holding register to become empty */
+	for ( i = 0 ; i < UART_THRE_TIMEOUT_MS ; i++ ) {
+		lsr = uart_read ( uart, UART_LSR );
+		if ( lsr & UART_LSR_THRE )
+			break;
+		mdelay ( 1 );
+	}
+
+	/* Transmit data (even if we timed out) */
+	uart_write ( uart, UART_THR, data );
+}
+
+/**
+ * Flush data
+ *
+ * @v uart		UART
+ */
+void uart_flush ( struct uart *uart ) {
+	unsigned int i;
+	uint8_t lsr;
+
+	/* Wait for transmitter and receiver to become empty */
+	for ( i = 0 ; i < UART_TEMT_TIMEOUT_MS ; i++ ) {
+		uart_read ( uart, UART_RBR );
+		lsr = uart_read ( uart, UART_LSR );
+		if ( ( lsr & UART_LSR_TEMT ) && ! ( lsr & UART_LSR_DR ) )
+			break;
+	}
+}
+
+/**
+ * Check for existence of UART
+ *
+ * @v uart		UART
+ * @ret rc		Return status code
+ */
+int uart_exists ( struct uart *uart ) {
+
+	/* Fail if no UART port is defined */
+	if ( ! uart->base )
+		return -ENODEV;
+
+	/* Fail if UART scratch register seems not to be present */
+	uart_write ( uart, UART_SCR, 0x18 );
+	if ( uart_read ( uart, UART_SCR ) != 0x18 )
+		return -ENODEV;
+	uart_write ( uart, UART_SCR, 0xae );
+	if ( uart_read ( uart, UART_SCR ) != 0xae )
+		return -ENODEV;
+
+	return 0;
+}
+
+/**
+ * Initialise UART
+ *
+ * @v uart		UART
+ * @v baud		Baud rate, or zero to leave unchanged
+ * @v lcr		Line control register value, or zero to leave unchanged
+ * @ret rc		Return status code
+ */
+int uart_init ( struct uart *uart, unsigned int baud, uint8_t lcr ) {
+	uint8_t dlm;
+	uint8_t dll;
+	int rc;
+
+	/* Check for existence of UART */
+	if ( ( rc = uart_exists ( uart ) ) != 0 )
+		return rc;
+
+	/* Configure divisor and line control register, if applicable */
+	if ( ! lcr )
+		lcr = uart_read ( uart, UART_LCR );
+	uart->lcr = lcr;
+	uart_write ( uart, UART_LCR, ( lcr | UART_LCR_DLAB ) );
+	if ( baud ) {
+		uart->divisor = ( UART_MAX_BAUD / baud );
+		dlm = ( ( uart->divisor >> 8 ) & 0xff );
+		dll = ( ( uart->divisor >> 0 ) & 0xff );
+		uart_write ( uart, UART_DLM, dlm );
+		uart_write ( uart, UART_DLL, dll );
+	} else {
+		dlm = uart_read ( uart, UART_DLM );
+		dll = uart_read ( uart, UART_DLL );
+		uart->divisor = ( ( dlm << 8 ) | dll );
+	}
+	uart_write ( uart, UART_LCR, ( lcr & ~UART_LCR_DLAB ) );
+
+	/* Disable interrupts */
+	uart_write ( uart, UART_IER, 0 );
+
+	/* Enable FIFOs */
+	uart_write ( uart, UART_FCR, UART_FCR_FE );
+
+	/* Assert DTR and RTS */
+	uart_write ( uart, UART_MCR, ( UART_MCR_DTR | UART_MCR_RTS ) );
+
+	/* Flush any stale data */
+	uart_flush ( uart );
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/core/uri.c b/qemu/roms/ipxe/src/core/uri.c
index 9ec21cee4..3b5f270fe 100644
--- a/qemu/roms/ipxe/src/core/uri.c
+++ b/qemu/roms/ipxe/src/core/uri.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -661,6 +665,7 @@ struct uri * resolve_uri ( const struct uri *base_uri,
  * Construct TFTP URI from next-server and filename
  *
  * @v next_server	Next-server address
+ * @v port		Port number, or zero to use the default port
  * @v filename		Filename
  * @ret uri		URI, or NULL on failure
  *
@@ -669,12 +674,18 @@ struct uri * resolve_uri ( const struct uri *base_uri,
  * generic URI parser.  We provide a mechanism for directly
  * constructing a TFTP URI from the next-server and filename.
  */
-struct uri * tftp_uri ( struct in_addr next_server, const char *filename ) {
+struct uri * tftp_uri ( struct in_addr next_server, unsigned int port,
+			const char *filename ) {
+	char buf[ 6 /* "65535" + NUL */ ];
 	struct uri uri;
 
 	memset ( &uri, 0, sizeof ( uri ) );
 	uri.scheme = "tftp";
 	uri.host = inet_ntoa ( next_server );
+	if ( port ) {
+		snprintf ( buf, sizeof ( buf ), "%d", port );
+		uri.port = buf;
+	}
 	uri.path = filename;
 	return uri_dup ( &uri );
 }
diff --git a/qemu/roms/ipxe/src/core/uuid.c b/qemu/roms/ipxe/src/core/uuid.c
index 27a249da8..b8d21de17 100644
--- a/qemu/roms/ipxe/src/core/uuid.c
+++ b/qemu/roms/ipxe/src/core/uuid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/core/version.c b/qemu/roms/ipxe/src/core/version.c
index 1e1e9daca..c984335c2 100644
--- a/qemu/roms/ipxe/src/core/version.c
+++ b/qemu/roms/ipxe/src/core/version.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -29,6 +33,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/features.h>
 #include <ipxe/version.h>
 #include <config/general.h>
+#include <config/branding.h>
 
 /**
  * Create wide-character version of string
diff --git a/qemu/roms/ipxe/src/core/vsprintf.c b/qemu/roms/ipxe/src/core/vsprintf.c
index 54811b11b..cb3bec5dd 100644
--- a/qemu/roms/ipxe/src/core/vsprintf.c
+++ b/qemu/roms/ipxe/src/core/vsprintf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/qemu/roms/ipxe/src/core/wchar.c b/qemu/roms/ipxe/src/core/wchar.c
index 7fabca470..860322820 100644
--- a/qemu/roms/ipxe/src/core/wchar.c
+++ b/qemu/roms/ipxe/src/core/wchar.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/core/xfer.c b/qemu/roms/ipxe/src/core/xfer.c
index 8d4bc9f53..112fee1bf 100644
--- a/qemu/roms/ipxe/src/core/xfer.c
+++ b/qemu/roms/ipxe/src/core/xfer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
@@ -134,18 +138,8 @@ size_t xfer_window ( struct interface *intf ) {
  * generating an xfer_window_changed() message.
  */
 void xfer_window_changed ( struct interface *intf ) {
-	struct interface *dest;
-	xfer_window_changed_TYPE ( void * ) *op =
-		intf_get_dest_op ( intf, xfer_window_changed, &dest );
-	void *object = intf_object ( dest );
 
-	if ( op ) {
-		op ( object );
-	} else {
-		/* Default is to do nothing */
-	}
-
-	intf_put ( dest );
+	intf_poke ( intf, xfer_window_changed );
 }
 
 /**
@@ -365,3 +359,34 @@ int xfer_seek ( struct interface *intf, off_t offset ) {
 
 	return xfer_deliver ( intf, iobuf, &meta );
 }
+
+/**
+ * Check that data is delivered strictly in order
+ *
+ * @v meta		Data transfer metadata
+ * @v pos		Current position
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+int xfer_check_order ( struct xfer_metadata *meta, size_t *pos, size_t len ) {
+	size_t new_pos;
+
+	/* Allow out-of-order zero-length packets (as used by xfer_seek()) */
+	if ( len == 0 )
+		return 0;
+
+	/* Calculate position of this delivery */
+	new_pos = *pos;
+	if ( meta->flags & XFER_FL_ABS_OFFSET )
+		new_pos = 0;
+	new_pos += meta->offset;
+
+	/* Fail if delivery position is not equal to current position */
+	if ( new_pos != *pos )
+		return -EPROTO;
+
+	/* Update current position */
+	*pos += len;
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/core/xferbuf.c b/qemu/roms/ipxe/src/core/xferbuf.c
index a0457feee..240118557 100644
--- a/qemu/roms/ipxe/src/core/xferbuf.c
+++ b/qemu/roms/ipxe/src/core/xferbuf.c
@@ -15,15 +15,21 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
 #include <ipxe/xfer.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/profile.h>
 #include <ipxe/xferbuf.h>
 
 /** @file
@@ -32,14 +38,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+/** Data delivery profiler */
+static struct profiler xferbuf_deliver_profiler __profiler =
+	{ .name = "xferbuf.deliver" };
+
+/** Data write profiler */
+static struct profiler xferbuf_write_profiler __profiler =
+	{ .name = "xferbuf.write" };
+
+/** Data read profiler */
+static struct profiler xferbuf_read_profiler __profiler =
+	{ .name = "xferbuf.read" };
+
 /**
- * Finish using data transfer buffer
+ * Free data transfer buffer
  *
  * @v xferbuf		Data transfer buffer
  */
-void xferbuf_done ( struct xfer_buffer *xferbuf ) {
-	free ( xferbuf->data );
-	xferbuf->data = NULL;
+void xferbuf_free ( struct xfer_buffer *xferbuf ) {
+
+	xferbuf->op->realloc ( xferbuf, 0 );
 	xferbuf->len = 0;
 	xferbuf->pos = 0;
 }
@@ -52,26 +70,78 @@ void xferbuf_done ( struct xfer_buffer *xferbuf ) {
  * @ret rc		Return status code
  */
 static int xferbuf_ensure_size ( struct xfer_buffer *xferbuf, size_t len ) {
-	void *new_data;
+	int rc;
 
 	/* If buffer is already large enough, do nothing */
 	if ( len <= xferbuf->len )
 		return 0;
 
 	/* Extend buffer */
-	new_data = realloc ( xferbuf->data, len );
-	if ( ! new_data ) {
+	if ( ( rc = xferbuf->op->realloc ( xferbuf, len ) ) != 0 ) {
 		DBGC ( xferbuf, "XFERBUF %p could not extend buffer to "
-		       "%zd bytes\n", xferbuf, len );
-		return -ENOSPC;
+		       "%zd bytes: %s\n", xferbuf, len, strerror ( rc ) );
+		return rc;
 	}
-	xferbuf->data = new_data;
 	xferbuf->len = len;
 
 	return 0;
 }
 
 /**
+ * Write to data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to write
+ * @v len		Length of data
+ */
+int xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+		    const void *data, size_t len ) {
+	size_t max_len;
+	int rc;
+
+	/* Check for overflow */
+	max_len = ( offset + len );
+	if ( max_len < offset )
+		return -EOVERFLOW;
+
+	/* Ensure buffer is large enough to contain this write */
+	if ( ( rc = xferbuf_ensure_size ( xferbuf, max_len ) ) != 0 )
+		return rc;
+
+	/* Copy data to buffer */
+	profile_start ( &xferbuf_write_profiler );
+	xferbuf->op->write ( xferbuf, offset, data, len );
+	profile_stop ( &xferbuf_write_profiler );
+
+	return 0;
+}
+
+/**
+ * Read from data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to write
+ * @v len		Length of data
+ */
+int xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+		   void *data, size_t len ) {
+
+	/* Check that read is within buffer range */
+	if ( ( offset > xferbuf->len ) ||
+	     ( len > ( xferbuf->len - offset ) ) )
+		return -ENOENT;
+
+	/* Copy data from buffer */
+	profile_start ( &xferbuf_read_profiler );
+	xferbuf->op->read ( xferbuf, offset, data, len );
+	profile_stop ( &xferbuf_read_profiler );
+
+	return 0;
+}
+
+/**
  * Add received data to data transfer buffer
  *
  * @v xferbuf		Data transfer buffer
@@ -81,28 +151,174 @@ static int xferbuf_ensure_size ( struct xfer_buffer *xferbuf, size_t len ) {
  */
 int xferbuf_deliver ( struct xfer_buffer *xferbuf, struct io_buffer *iobuf,
 		      struct xfer_metadata *meta ) {
-	size_t len;
-	size_t max;
+	size_t len = iob_len ( iobuf );
+	size_t pos;
 	int rc;
 
+	/* Start profiling */
+	profile_start ( &xferbuf_deliver_profiler );
+
 	/* Calculate new buffer position */
+	pos = xferbuf->pos;
 	if ( meta->flags & XFER_FL_ABS_OFFSET )
-		xferbuf->pos = 0;
-	xferbuf->pos += meta->offset;
+		pos = 0;
+	pos += meta->offset;
 
-	/* Ensure that we have enough buffer space for this data */
-	len = iob_len ( iobuf );
-	max = ( xferbuf->pos + len );
-	if ( ( rc = xferbuf_ensure_size ( xferbuf, max ) ) != 0 )
+	/* Write data to buffer */
+	if ( ( rc = xferbuf_write ( xferbuf, pos, iobuf->data, len ) ) != 0 )
 		goto done;
 
-	/* Copy data to buffer */
-	memcpy ( ( xferbuf->data + xferbuf->pos ), iobuf->data, len );
-
 	/* Update current buffer position */
-	xferbuf->pos += len;
+	xferbuf->pos = ( pos + len );
 
  done:
 	free_iob ( iobuf );
+	profile_stop ( &xferbuf_deliver_profiler );
 	return rc;
 }
+
+/**
+ * Reallocate malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int xferbuf_malloc_realloc ( struct xfer_buffer *xferbuf, size_t len ) {
+	void *new_data;
+
+	new_data = realloc ( xferbuf->data, len );
+	if ( ! new_data )
+		return -ENOSPC;
+	xferbuf->data = new_data;
+	return 0;
+}
+
+/**
+ * Write data to malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void xferbuf_malloc_write ( struct xfer_buffer *xferbuf, size_t offset,
+				   const void *data, size_t len ) {
+
+	memcpy ( ( xferbuf->data + offset ), data, len );
+}
+
+/**
+ * Read data from malloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void xferbuf_malloc_read ( struct xfer_buffer *xferbuf, size_t offset,
+				  void *data, size_t len ) {
+
+	memcpy ( data, ( xferbuf->data + offset ), len );
+}
+
+/** malloc()-based data buffer operations */
+struct xfer_buffer_operations xferbuf_malloc_operations = {
+	.realloc = xferbuf_malloc_realloc,
+	.write = xferbuf_malloc_write,
+	.read = xferbuf_malloc_read,
+};
+
+/**
+ * Reallocate umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v len		New length (or zero to free buffer)
+ * @ret rc		Return status code
+ */
+static int xferbuf_umalloc_realloc ( struct xfer_buffer *xferbuf, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+	userptr_t new_udata;
+
+	new_udata = urealloc ( *udata, len );
+	if ( ! new_udata )
+		return -ENOSPC;
+	*udata = new_udata;
+	return 0;
+}
+
+/**
+ * Write data to umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to copy
+ * @v len		Length of data
+ */
+static void xferbuf_umalloc_write ( struct xfer_buffer *xferbuf, size_t offset,
+				    const void *data, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+
+	copy_to_user ( *udata, offset, data, len );
+}
+
+/**
+ * Read data from umalloc()-based data buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v offset		Starting offset
+ * @v data		Data to read
+ * @v len		Length of data
+ */
+static void xferbuf_umalloc_read ( struct xfer_buffer *xferbuf, size_t offset,
+				   void *data, size_t len ) {
+	userptr_t *udata = xferbuf->data;
+
+	copy_from_user ( data, *udata, offset, len );
+}
+
+/** umalloc()-based data buffer operations */
+struct xfer_buffer_operations xferbuf_umalloc_operations = {
+	.realloc = xferbuf_umalloc_realloc,
+	.write = xferbuf_umalloc_write,
+	.read = xferbuf_umalloc_read,
+};
+
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v interface		Data transfer interface
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ *
+ * This call will check that the xfer_buffer() handler belongs to the
+ * destination interface which also provides xfer_deliver() for this
+ * interface.
+ *
+ * This is done to prevent accidental accesses to a data transfer
+ * buffer which may be located behind a non-transparent datapath via a
+ * series of pass-through interfaces.
+ */
+struct xfer_buffer * xfer_buffer ( struct interface *intf ) {
+	struct interface *dest;
+	xfer_buffer_TYPE ( void * ) *op =
+		intf_get_dest_op ( intf, xfer_buffer, &dest );
+	void *object = intf_object ( dest );
+	struct interface *xfer_deliver_dest;
+	struct xfer_buffer *xferbuf;
+
+	/* Check that this operation is provided by the same interface
+	 * which handles xfer_deliver().
+	 */
+	( void ) intf_get_dest_op ( intf, xfer_deliver, &xfer_deliver_dest );
+
+	if ( op && ( dest == xfer_deliver_dest ) ) {
+		xferbuf = op ( object );
+	} else {
+		/* Default is to not have a data transfer buffer */
+		xferbuf = NULL;
+	}
+
+	intf_put ( xfer_deliver_dest );
+	intf_put ( dest );
+	return xferbuf;
+}
diff --git a/qemu/roms/ipxe/src/crypto/aes.c b/qemu/roms/ipxe/src/crypto/aes.c
new file mode 100644
index 000000000..b9e206bfb
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/aes.c
@@ -0,0 +1,808 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * AES algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+#include <ipxe/cbc.h>
+#include <ipxe/aes.h>
+
+/** AES strides
+ *
+ * These are the strides (modulo 16) used to walk through the AES
+ * input state bytes in order of byte position after [Inv]ShiftRows.
+ */
+enum aes_stride {
+	/** Input stride for ShiftRows
+	 *
+	 *    0 4 8 c
+	 *     \ \ \
+	 *    1 5 9 d
+	 *     \ \ \
+	 *    2 6 a e
+	 *     \ \ \
+	 *    3 7 b f
+	 */
+	AES_STRIDE_SHIFTROWS = +5,
+	/** Input stride for InvShiftRows
+	 *
+	 *    0 4 8 c
+	 *     / / /
+	 *    1 5 9 d
+	 *     / / /
+	 *    2 6 a e
+	 *     / / /
+	 *    3 7 b f
+	 */
+	AES_STRIDE_INVSHIFTROWS = -3,
+};
+
+/** A single AES lookup table entry
+ *
+ * This represents the product (in the Galois field GF(2^8)) of an
+ * eight-byte vector multiplier with a single scalar multiplicand.
+ *
+ * The vector multipliers used for AES will be {1,1,1,3,2,1,1,3} for
+ * MixColumns and {1,9,13,11,14,9,13,11} for InvMixColumns.  This
+ * allows for the result of multiplying any single column of the
+ * [Inv]MixColumns matrix by a scalar value to be obtained simply by
+ * extracting the relevant four-byte subset from the lookup table
+ * entry.
+ *
+ * For example, to find the result of multiplying the second column of
+ * the MixColumns matrix by the scalar value 0x80:
+ *
+ * MixColumns column[0]: {			      2,    1,    1,    3 }
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * MixColumns column[2]: {		  1,    3,    2,    1		  }
+ * MixColumns column[3]: {	    1,    1,    3,    2			  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Scalar multiplicand:	   0x80
+ * Lookup table entry:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ *
+ * The second column of the MixColumns matrix is {3,2,1,1}.  The
+ * product of this column with the scalar value 0x80 can be obtained
+ * by extracting the relevant four-byte subset of the lookup table
+ * entry:
+ *
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Lookup table entry:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ * Product:		 {		     0x9b, 0x1b, 0x80, 0x80	  }
+ *
+ * The column lookups require only seven bytes of the eight-byte
+ * entry: the remaining (first) byte is used to hold the scalar
+ * multiplicand itself (i.e. the first byte of the vector multiplier
+ * is always chosen to be 1).
+ */
+union aes_table_entry {
+	/** Viewed as an array of bytes */
+	uint8_t byte[8];
+} __attribute__ (( packed ));
+
+/** An AES lookup table
+ *
+ * This represents the products (in the Galois field GF(2^8)) of a
+ * constant eight-byte vector multiplier with all possible 256 scalar
+ * multiplicands.
+ *
+ * The entries are indexed by the AES [Inv]SubBytes S-box output
+ * values (denoted S(N)).  This allows for the result of multiplying
+ * any single column of the [Inv]MixColumns matrix by S(N) to be
+ * obtained simply by extracting the relevant four-byte subset from
+ * the Nth table entry.  For example:
+ *
+ * Input byte (N):	   0x3a
+ * SubBytes output S(N):   0x80
+ * MixColumns column[1]: {			3,    2,    1,    1	  }
+ * Vector multiplier:	 {    1,    1,    1,    3,    2,    1,    1,    3 }
+ * Table entry[0x3a]:	 { 0x80, 0x80, 0x80, 0x9b, 0x1b, 0x80, 0x80, 0x9b }
+ * Product:		 {		     0x9b, 0x1b, 0x80, 0x80	  }
+ *
+ * Since the first byte of the eight-byte vector multiplier is always
+ * chosen to be 1, the value of S(N) may be lookup up by extracting
+ * the first byte of the Nth table entry.
+ */
+struct aes_table {
+	/** Table entries, indexed by S(N) */
+	union aes_table_entry entry[256];
+} __attribute__ (( aligned ( 8 ) ));
+
+/** AES MixColumns lookup table */
+static struct aes_table aes_mixcolumns;
+
+/** AES InvMixColumns lookup table */
+static struct aes_table aes_invmixcolumns;
+
+/**
+ * Multiply [Inv]MixColumns matrix column by scalar multiplicand
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ * @v column		[Inv]MixColumns matrix column index
+ * @ret product		Product of matrix column with scalar multiplicand
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_entry_column ( const union aes_table_entry *entry, unsigned int column ) {
+	const union {
+		uint8_t byte;
+		uint32_t column;
+	} __attribute__ (( may_alias )) *product;
+
+	/* Locate relevant four-byte subset */
+	product = container_of ( &entry->byte[ 4 - column ],
+				 typeof ( *product ), byte );
+
+	/* Extract this four-byte subset */
+	return product->column;
+}
+
+/**
+ * Multiply [Inv]MixColumns matrix column by S-boxed input byte
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v offset		Output byte offset (after [Inv]ShiftRows)
+ * @ret product		Product of matrix column with S(input byte)
+ *
+ * Note that the specified offset is not the offset of the input byte;
+ * it is the offset of the output byte which corresponds to the input
+ * byte.  This output byte offset is used to calculate both the input
+ * byte offset and to select the appropriate matric column.
+ *
+ * With a compile-time constant offset, this function will optimise
+ * down to a single "movzbl" (to extract the input byte) and will
+ * generate a single x86 memory reference expression which can then be
+ * used directly within a single "xorl" instruction.
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_column ( const struct aes_table *table, size_t stride,
+	     const union aes_matrix *in, size_t offset ) {
+	const union aes_table_entry *entry;
+	unsigned int byte;
+
+	/* Extract input byte corresponding to this output byte offset
+	 * (i.e. perform [Inv]ShiftRows).
+	 */
+	byte = in->byte[ ( stride * offset ) & 0xf ];
+
+	/* Locate lookup table entry for this input byte (i.e. perform
+	 * [Inv]SubBytes).
+	 */
+	entry = &table->entry[byte];
+
+	/* Multiply appropriate matrix column by this input byte
+	 * (i.e. perform [Inv]MixColumns).
+	 */
+	return aes_entry_column ( entry, ( offset & 0x3 ) );
+}
+
+/**
+ * Calculate intermediate round output column
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v key		AES round key
+ * @v column		Column index
+ * @ret output		Output column value
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_output ( const struct aes_table *table, size_t stride,
+	     const union aes_matrix *in, const union aes_matrix *key,
+	     unsigned int column ) {
+	size_t offset = ( column * 4 );
+
+	/* Perform [Inv]ShiftRows, [Inv]SubBytes, [Inv]MixColumns, and
+	 * AddRoundKey for this column.  The loop is unrolled to allow
+	 * for the required compile-time constant optimisations.
+	 */
+	return ( aes_column ( table, stride, in, ( offset + 0 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 1 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 2 ) ) ^
+		 aes_column ( table, stride, in, ( offset + 3 ) ) ^
+		 key->column[column] );
+}
+
+/**
+ * Perform a single intermediate round
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		AES round key
+ */
+static inline __attribute__ (( always_inline )) void
+aes_round ( const struct aes_table *table, size_t stride,
+	    const union aes_matrix *in, union aes_matrix *out,
+	    const union aes_matrix *key ) {
+
+	/* Perform [Inv]ShiftRows, [Inv]SubBytes, [Inv]MixColumns, and
+	 * AddRoundKey for all columns.  The loop is unrolled to allow
+	 * for the required compile-time constant optimisations.
+	 */
+	out->column[0] = aes_output ( table, stride, in, key, 0 );
+	out->column[1] = aes_output ( table, stride, in, key, 1 );
+	out->column[2] = aes_output ( table, stride, in, key, 2 );
+	out->column[3] = aes_output ( table, stride, in, key, 3 );
+}
+
+/**
+ * Perform encryption intermediate rounds
+ *
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		Round keys
+ * @v rounds		Number of rounds (must be odd)
+ *
+ * This function is deliberately marked as non-inlinable to ensure
+ * maximal availability of registers for GCC's register allocator,
+ * which has a tendency to otherwise spill performance-critical
+ * registers to the stack.
+ */
+static __attribute__ (( noinline )) void
+aes_encrypt_rounds ( union aes_matrix *in, union aes_matrix *out,
+		     const union aes_matrix *key, unsigned int rounds ) {
+	union aes_matrix *tmp;
+
+	/* Perform intermediate rounds */
+	do {
+		/* Perform one intermediate round */
+		aes_round ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS,
+			    in, out, key++ );
+
+		/* Swap input and output states for next round */
+		tmp = in;
+		in = out;
+		out = tmp;
+
+	} while ( --rounds );
+}
+
+/**
+ * Perform decryption intermediate rounds
+ *
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		Round keys
+ * @v rounds		Number of rounds (must be odd)
+ *
+ * As with aes_encrypt_rounds(), this function is deliberately marked
+ * as non-inlinable.
+ *
+ * This function could potentially use the same binary code as is used
+ * for encryption.  To compensate for the difference between ShiftRows
+ * and InvShiftRows, half of the input byte offsets would have to be
+ * modifiable at runtime (half by an offset of +4/-4, half by an
+ * offset of -4/+4 for ShiftRows/InvShiftRows).  This can be
+ * accomplished in x86 assembly within the number of available
+ * registers, but GCC's register allocator struggles to do so,
+ * resulting in a significant performance decrease due to registers
+ * being spilled to the stack.  We therefore use two separate but very
+ * similar binary functions based on the same C source.
+ */
+static __attribute__ (( noinline )) void
+aes_decrypt_rounds ( union aes_matrix *in, union aes_matrix *out,
+		     const union aes_matrix *key, unsigned int rounds ) {
+	union aes_matrix *tmp;
+
+	/* Perform intermediate rounds */
+	do {
+		/* Perform one intermediate round */
+		aes_round ( &aes_invmixcolumns, AES_STRIDE_INVSHIFTROWS,
+			    in, out, key++ );
+
+		/* Swap input and output states for next round */
+		tmp = in;
+		in = out;
+		out = tmp;
+
+	} while ( --rounds );
+}
+
+/**
+ * Perform standalone AddRoundKey
+ *
+ * @v state		AES state
+ * @v key		AES round key
+ */
+static inline __attribute__ (( always_inline )) void
+aes_addroundkey ( union aes_matrix *state, const union aes_matrix *key ) {
+
+	state->column[0] ^= key->column[0];
+	state->column[1] ^= key->column[1];
+	state->column[2] ^= key->column[2];
+	state->column[3] ^= key->column[3];
+}
+
+/**
+ * Perform final round
+ *
+ * @v table		AES lookup table
+ * @v stride		AES row shift stride
+ * @v in		AES input state
+ * @v out		AES output state
+ * @v key		AES round key
+ */
+static void aes_final ( const struct aes_table *table, size_t stride,
+			const union aes_matrix *in, union aes_matrix *out,
+			const union aes_matrix *key ) {
+	const union aes_table_entry *entry;
+	unsigned int byte;
+	size_t out_offset;
+	size_t in_offset;
+
+	/* Perform [Inv]ShiftRows and [Inv]SubBytes */
+	for ( out_offset = 0, in_offset = 0 ; out_offset < 16 ;
+	      out_offset++, in_offset = ( ( in_offset + stride ) & 0xf ) ) {
+
+		/* Extract input byte (i.e. perform [Inv]ShiftRows) */
+		byte = in->byte[in_offset];
+
+		/* Locate lookup table entry for this input byte
+		 * (i.e. perform [Inv]SubBytes).
+		 */
+		entry = &table->entry[byte];
+
+		/* Store output byte */
+		out->byte[out_offset] = entry->byte[0];
+	}
+
+	/* Perform AddRoundKey */
+	aes_addroundkey ( out, key );
+}
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ */
+static void aes_encrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix buffer[2];
+	union aes_matrix *in = &buffer[0];
+	union aes_matrix *out = &buffer[1];
+	unsigned int rounds = aes->rounds;
+
+	/* Sanity check */
+	assert ( len == sizeof ( *in ) );
+
+	/* Initialise input state */
+	memcpy ( in, src, sizeof ( *in ) );
+
+	/* Perform initial round (AddRoundKey) */
+	aes_addroundkey ( in, &aes->encrypt.key[0] );
+
+	/* Perform intermediate rounds (ShiftRows, SubBytes,
+	 * MixColumns, AddRoundKey).
+	 */
+	aes_encrypt_rounds ( in, out, &aes->encrypt.key[1], ( rounds - 2 ) );
+	in = out;
+
+	/* Perform final round (ShiftRows, SubBytes, AddRoundKey) */
+	out = dst;
+	aes_final ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS, in, out,
+		    &aes->encrypt.key[ rounds - 1 ] );
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ */
+static void aes_decrypt ( void *ctx, const void *src, void *dst, size_t len ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix buffer[2];
+	union aes_matrix *in = &buffer[0];
+	union aes_matrix *out = &buffer[1];
+	unsigned int rounds = aes->rounds;
+
+	/* Sanity check */
+	assert ( len == sizeof ( *in ) );
+
+	/* Initialise input state */
+	memcpy ( in, src, sizeof ( *in ) );
+
+	/* Perform initial round (AddRoundKey) */
+	aes_addroundkey ( in, &aes->decrypt.key[0] );
+
+	/* Perform intermediate rounds (InvShiftRows, InvSubBytes,
+	 * InvMixColumns, AddRoundKey).
+	 */
+	aes_decrypt_rounds ( in, out, &aes->decrypt.key[1], ( rounds - 2 ) );
+	in = out;
+
+	/* Perform final round (InvShiftRows, InvSubBytes, AddRoundKey) */
+	out = dst;
+	aes_final ( &aes_invmixcolumns, AES_STRIDE_INVSHIFTROWS, in, out,
+		    &aes->decrypt.key[ rounds - 1 ] );
+}
+
+/**
+ * Multiply a polynomial by (x) modulo (x^8 + x^4 + x^3 + x^2 + 1) in GF(2^8)
+ *
+ * @v poly		Polynomial to be multiplied
+ * @ret result		Result
+ */
+static __attribute__ (( const )) unsigned int aes_double ( unsigned int poly ) {
+
+	/* Multiply polynomial by (x), placing the resulting x^8
+	 * coefficient in the LSB (i.e. rotate byte left by one).
+	 */
+	poly = rol8 ( poly, 1 );
+
+	/* If coefficient of x^8 (in LSB) is non-zero, then reduce by
+	 * subtracting (x^8 + x^4 + x^3 + x^2 + 1) in GF(2^8).
+	 */
+	if ( poly & 0x01 ) {
+		poly ^= 0x01; /* Subtract x^8 (currently in LSB) */
+		poly ^= 0x1b; /* Subtract (x^4 + x^3 + x^2 + 1) */
+	}
+
+	return poly;
+}
+
+/**
+ * Fill in MixColumns lookup table entry
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ *
+ * The MixColumns lookup table vector multiplier is {1,1,1,3,2,1,1,3}.
+ */
+static void aes_mixcolumns_entry ( union aes_table_entry *entry ) {
+	unsigned int scalar_x_1;
+	unsigned int scalar_x;
+	unsigned int scalar;
+
+	/* Retrieve scalar multiplicand */
+	scalar = entry->byte[0];
+	entry->byte[1] = scalar;
+	entry->byte[2] = scalar;
+	entry->byte[5] = scalar;
+	entry->byte[6] = scalar;
+
+	/* Calculate scalar multiplied by (x) */
+	scalar_x = aes_double ( scalar );
+	entry->byte[4] = scalar_x;
+
+	/* Calculate scalar multiplied by (x + 1) */
+	scalar_x_1 = ( scalar_x ^ scalar );
+	entry->byte[3] = scalar_x_1;
+	entry->byte[7] = scalar_x_1;
+}
+
+/**
+ * Fill in InvMixColumns lookup table entry
+ *
+ * @v entry		AES lookup table entry for scalar multiplicand
+ *
+ * The InvMixColumns lookup table vector multiplier is {1,9,13,11,14,9,13,11}.
+ */
+static void aes_invmixcolumns_entry ( union aes_table_entry *entry ) {
+	unsigned int scalar_x3_x2_x;
+	unsigned int scalar_x3_x2_1;
+	unsigned int scalar_x3_x2;
+	unsigned int scalar_x3_x_1;
+	unsigned int scalar_x3_1;
+	unsigned int scalar_x3;
+	unsigned int scalar_x2;
+	unsigned int scalar_x;
+	unsigned int scalar;
+
+	/* Retrieve scalar multiplicand */
+	scalar = entry->byte[0];
+
+	/* Calculate scalar multiplied by (x) */
+	scalar_x = aes_double ( scalar );
+
+	/* Calculate scalar multiplied by (x^2) */
+	scalar_x2 = aes_double ( scalar_x );
+
+	/* Calculate scalar multiplied by (x^3) */
+	scalar_x3 = aes_double ( scalar_x2 );
+
+	/* Calculate scalar multiplied by (x^3 + 1) */
+	scalar_x3_1 = ( scalar_x3 ^ scalar );
+	entry->byte[1] = scalar_x3_1;
+	entry->byte[5] = scalar_x3_1;
+
+	/* Calculate scalar multiplied by (x^3 + x + 1) */
+	scalar_x3_x_1 = ( scalar_x3_1 ^ scalar_x );
+	entry->byte[3] = scalar_x3_x_1;
+	entry->byte[7] = scalar_x3_x_1;
+
+	/* Calculate scalar multiplied by (x^3 + x^2) */
+	scalar_x3_x2 = ( scalar_x3 ^ scalar_x2 );
+
+	/* Calculate scalar multiplied by (x^3 + x^2 + 1) */
+	scalar_x3_x2_1 = ( scalar_x3_x2 ^ scalar );
+	entry->byte[2] = scalar_x3_x2_1;
+	entry->byte[6] = scalar_x3_x2_1;
+
+	/* Calculate scalar multiplied by (x^3 + x^2 + x) */
+	scalar_x3_x2_x = ( scalar_x3_x2 ^ scalar_x );
+	entry->byte[4] = scalar_x3_x2_x;
+}
+
+/**
+ * Generate AES lookup tables
+ *
+ */
+static void aes_generate ( void ) {
+	union aes_table_entry *entry;
+	union aes_table_entry *inventry;
+	unsigned int poly = 0x01;
+	unsigned int invpoly = 0x01;
+	unsigned int transformed;
+	unsigned int i;
+
+	/* Iterate over non-zero values of GF(2^8) using generator (x + 1) */
+	do {
+
+		/* Multiply polynomial by (x + 1) */
+		poly ^= aes_double ( poly );
+
+		/* Divide inverse polynomial by (x + 1).  This code
+		 * fragment is taken directly from the Wikipedia page
+		 * on the Rijndael S-box.  An explanation of why it
+		 * works would be greatly appreciated.
+		 */
+		invpoly ^= ( invpoly << 1 );
+		invpoly ^= ( invpoly << 2 );
+		invpoly ^= ( invpoly << 4 );
+		if ( invpoly & 0x80 )
+			invpoly ^= 0x09;
+		invpoly &= 0xff;
+
+		/* Apply affine transformation */
+		transformed = ( 0x63 ^ invpoly ^ rol8 ( invpoly, 1 ) ^
+				rol8 ( invpoly, 2 ) ^ rol8 ( invpoly, 3 ) ^
+				rol8 ( invpoly, 4 ) );
+
+		/* Populate S-box (within MixColumns lookup table) */
+		aes_mixcolumns.entry[poly].byte[0] = transformed;
+
+	} while ( poly != 0x01 );
+
+	/* Populate zeroth S-box entry (which has no inverse) */
+	aes_mixcolumns.entry[0].byte[0] = 0x63;
+
+	/* Fill in MixColumns and InvMixColumns lookup tables */
+	for ( i = 0 ; i < 256 ; i++ ) {
+
+		/* Fill in MixColumns lookup table entry */
+		entry = &aes_mixcolumns.entry[i];
+		aes_mixcolumns_entry ( entry );
+
+		/* Populate inverse S-box (within InvMixColumns lookup table) */
+		inventry = &aes_invmixcolumns.entry[ entry->byte[0] ];
+		inventry->byte[0] = i;
+
+		/* Fill in InvMixColumns lookup table entry */
+		aes_invmixcolumns_entry ( inventry );
+	}
+}
+
+/**
+ * Rotate key column
+ *
+ * @v column		Key column
+ * @ret column		Updated key column
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_key_rotate ( uint32_t column ) {
+
+	return ( ( __BYTE_ORDER == __LITTLE_ENDIAN ) ?
+		 ror32 ( column, 8 ) : rol32 ( column, 8 ) );
+}
+
+/**
+ * Apply S-box to key column
+ *
+ * @v column		Key column
+ * @ret column		Updated key column
+ */
+static uint32_t aes_key_sbox ( uint32_t column ) {
+	unsigned int i;
+	uint8_t byte;
+
+	for ( i = 0 ; i < 4 ; i++ ) {
+		byte = ( column & 0xff );
+		byte = aes_mixcolumns.entry[byte].byte[0];
+		column = ( ( column & ~0xff ) | byte );
+		column = rol32 ( column, 8 );
+	}
+	return column;
+}
+
+/**
+ * Apply schedule round constant to key column
+ *
+ * @v column		Key column
+ * @v rcon		Round constant
+ * @ret column		Updated key column
+ */
+static inline __attribute__ (( always_inline )) uint32_t
+aes_key_rcon ( uint32_t column, unsigned int rcon ) {
+
+	return ( ( __BYTE_ORDER == __LITTLE_ENDIAN ) ?
+		 ( column ^ rcon ) : ( column ^ ( rcon << 24 ) ) );
+}
+
+/**
+ * Set key
+ *
+ * @v ctx		Context
+ * @v key		Key
+ * @v keylen		Key length
+ * @ret rc		Return status code
+ */
+static int aes_setkey ( void *ctx, const void *key, size_t keylen ) {
+	struct aes_context *aes = ctx;
+	union aes_matrix *enc;
+	union aes_matrix *dec;
+	union aes_matrix temp;
+	union aes_matrix zero;
+	unsigned int rcon = 0x01;
+	unsigned int rounds;
+	size_t offset = 0;
+	uint32_t *prev;
+	uint32_t *next;
+	uint32_t *end;
+	uint32_t tmp;
+
+	/* Generate lookup tables, if not already done */
+	if ( ! aes_mixcolumns.entry[0].byte[0] )
+		aes_generate();
+
+	/* Validate key length and calculate number of intermediate rounds */
+	switch ( keylen ) {
+	case ( 128 / 8 ) :
+		rounds = 11;
+		break;
+	case ( 192 / 8 ) :
+		rounds = 13;
+		break;
+	case ( 256 / 8 ) :
+		rounds = 15;
+		break;
+	default:
+		DBGC ( aes, "AES %p unsupported key length (%zd bits)\n",
+		       aes, ( keylen * 8 ) );
+		return -EINVAL;
+	}
+	aes->rounds = rounds;
+	enc = aes->encrypt.key;
+	end = enc[rounds].column;
+
+	/* Copy raw key */
+	memcpy ( enc, key, keylen );
+	prev = enc->column;
+	next = ( ( ( void * ) prev ) + keylen );
+	tmp = next[-1];
+
+	/* Construct expanded key */
+	while ( next < end ) {
+
+		/* If this is the first column of an expanded key
+		 * block, or the middle column of an AES-256 key
+		 * block, then apply the S-box.
+		 */
+		if ( ( offset == 0 ) || ( ( offset | keylen ) == 48 ) )
+			tmp = aes_key_sbox ( tmp );
+
+		/* If this is the first column of an expanded key
+		 * block then rotate and apply the round constant.
+		 */
+		if ( offset == 0 ) {
+			tmp = aes_key_rotate ( tmp );
+			tmp = aes_key_rcon ( tmp, rcon );
+			rcon = aes_double ( rcon );
+		}
+
+		/* XOR with previous key column */
+		tmp ^= *prev;
+
+		/* Store column */
+		*next = tmp;
+
+		/* Move to next column */
+		offset += sizeof ( *next );
+		if ( offset == keylen )
+			offset = 0;
+		next++;
+		prev++;
+	}
+	DBGC2 ( aes, "AES %p expanded %zd-bit key:\n", aes, ( keylen * 8 ) );
+	DBGC2_HDA ( aes, 0, &aes->encrypt, ( rounds * sizeof ( *enc ) ) );
+
+	/* Convert to decryption key */
+	memset ( &zero, 0, sizeof ( zero ) );
+	dec = &aes->decrypt.key[ rounds - 1 ];
+	memcpy ( dec--, enc++, sizeof ( *dec ) );
+	while ( dec > aes->decrypt.key ) {
+		/* Perform InvMixColumns (by reusing the encryption
+		 * final-round code to perform ShiftRows+SubBytes and
+		 * reusing the decryption intermediate-round code to
+		 * perform InvShiftRows+InvSubBytes+InvMixColumns, all
+		 * with a zero encryption key).
+		 */
+		aes_final ( &aes_mixcolumns, AES_STRIDE_SHIFTROWS,
+			    enc++, &temp, &zero );
+		aes_decrypt_rounds ( &temp, dec--, &zero, 1 );
+	}
+	memcpy ( dec--, enc++, sizeof ( *dec ) );
+	DBGC2 ( aes, "AES %p inverted %zd-bit key:\n", aes, ( keylen * 8 ) );
+	DBGC2_HDA ( aes, 0, &aes->decrypt, ( rounds * sizeof ( *dec ) ) );
+
+	return 0;
+}
+
+/**
+ * Set initialisation vector
+ *
+ * @v ctx		Context
+ * @v iv		Initialisation vector
+ */
+static void aes_setiv ( void *ctx __unused, const void *iv __unused ) {
+	/* Nothing to do */
+}
+
+/** Basic AES algorithm */
+struct cipher_algorithm aes_algorithm = {
+	.name = "aes",
+	.ctxsize = sizeof ( struct aes_context ),
+	.blocksize = AES_BLOCKSIZE,
+	.setkey = aes_setkey,
+	.setiv = aes_setiv,
+	.encrypt = aes_encrypt,
+	.decrypt = aes_decrypt,
+};
+
+/* AES in Electronic Codebook mode */
+ECB_CIPHER ( aes_ecb, aes_ecb_algorithm,
+	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
+
+/* AES in Cipher Block Chaining mode */
+CBC_CIPHER ( aes_cbc, aes_cbc_algorithm,
+	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
diff --git a/qemu/roms/ipxe/src/crypto/asn1.c b/qemu/roms/ipxe/src/crypto/asn1.c
index 6d880704f..aca12bf30 100644
--- a/qemu/roms/ipxe/src/crypto/asn1.c
+++ b/qemu/roms/ipxe/src/crypto/asn1.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/crypto/axtls/aes.c b/qemu/roms/ipxe/src/crypto/axtls/aes.c
deleted file mode 100644
index bd99a7097..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/aes.c
+++ /dev/null
@@ -1,457 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * AES implementation - this is a small code version. There are much faster
- * versions around but they are much larger in size (i.e. they use large 
- * submix tables).
- */
-
-#include <string.h>
-#include "os_port.h"
-#include "crypto.h"
-
-/* all commented out in skeleton mode */
-#ifndef CONFIG_SSL_SKELETON_MODE
-
-#define rot1(x) (((x) << 24) | ((x) >> 8))
-#define rot2(x) (((x) << 16) | ((x) >> 16))
-#define rot3(x) (((x) <<  8) | ((x) >> 24))
-
-/* 
- * This cute trick does 4 'mul by two' at once.  Stolen from
- * Dr B. R. Gladman <brg@gladman.uk.net> but I'm sure the u-(u>>7) is
- * a standard graphics trick
- * The key to this is that we need to xor with 0x1b if the top bit is set.
- * a 1xxx xxxx   0xxx 0xxx First we mask the 7bit,
- * b 1000 0000   0000 0000 then we shift right by 7 putting the 7bit in 0bit,
- * c 0000 0001   0000 0000 we then subtract (c) from (b)
- * d 0111 1111   0000 0000 and now we and with our mask
- * e 0001 1011   0000 0000
- */
-#define mt  0x80808080
-#define ml  0x7f7f7f7f
-#define mh  0xfefefefe
-#define mm  0x1b1b1b1b
-#define mul2(x,t)	((t)=((x)&mt), \
-			((((x)+(x))&mh)^(((t)-((t)>>7))&mm)))
-
-#define inv_mix_col(x,f2,f4,f8,f9) (\
-			(f2)=mul2(x,f2), \
-			(f4)=mul2(f2,f4), \
-			(f8)=mul2(f4,f8), \
-			(f9)=(x)^(f8), \
-			(f8)=((f2)^(f4)^(f8)), \
-			(f2)^=(f9), \
-			(f4)^=(f9), \
-			(f8)^=rot3(f2), \
-			(f8)^=rot2(f4), \
-			(f8)^rot1(f9))
-
-/*
- * AES S-box
- */
-static const uint8_t aes_sbox[256] =
-{
-	0x63,0x7C,0x77,0x7B,0xF2,0x6B,0x6F,0xC5,
-	0x30,0x01,0x67,0x2B,0xFE,0xD7,0xAB,0x76,
-	0xCA,0x82,0xC9,0x7D,0xFA,0x59,0x47,0xF0,
-	0xAD,0xD4,0xA2,0xAF,0x9C,0xA4,0x72,0xC0,
-	0xB7,0xFD,0x93,0x26,0x36,0x3F,0xF7,0xCC,
-	0x34,0xA5,0xE5,0xF1,0x71,0xD8,0x31,0x15,
-	0x04,0xC7,0x23,0xC3,0x18,0x96,0x05,0x9A,
-	0x07,0x12,0x80,0xE2,0xEB,0x27,0xB2,0x75,
-	0x09,0x83,0x2C,0x1A,0x1B,0x6E,0x5A,0xA0,
-	0x52,0x3B,0xD6,0xB3,0x29,0xE3,0x2F,0x84,
-	0x53,0xD1,0x00,0xED,0x20,0xFC,0xB1,0x5B,
-	0x6A,0xCB,0xBE,0x39,0x4A,0x4C,0x58,0xCF,
-	0xD0,0xEF,0xAA,0xFB,0x43,0x4D,0x33,0x85,
-	0x45,0xF9,0x02,0x7F,0x50,0x3C,0x9F,0xA8,
-	0x51,0xA3,0x40,0x8F,0x92,0x9D,0x38,0xF5,
-	0xBC,0xB6,0xDA,0x21,0x10,0xFF,0xF3,0xD2,
-	0xCD,0x0C,0x13,0xEC,0x5F,0x97,0x44,0x17,
-	0xC4,0xA7,0x7E,0x3D,0x64,0x5D,0x19,0x73,
-	0x60,0x81,0x4F,0xDC,0x22,0x2A,0x90,0x88,
-	0x46,0xEE,0xB8,0x14,0xDE,0x5E,0x0B,0xDB,
-	0xE0,0x32,0x3A,0x0A,0x49,0x06,0x24,0x5C,
-	0xC2,0xD3,0xAC,0x62,0x91,0x95,0xE4,0x79,
-	0xE7,0xC8,0x37,0x6D,0x8D,0xD5,0x4E,0xA9,
-	0x6C,0x56,0xF4,0xEA,0x65,0x7A,0xAE,0x08,
-	0xBA,0x78,0x25,0x2E,0x1C,0xA6,0xB4,0xC6,
-	0xE8,0xDD,0x74,0x1F,0x4B,0xBD,0x8B,0x8A,
-	0x70,0x3E,0xB5,0x66,0x48,0x03,0xF6,0x0E,
-	0x61,0x35,0x57,0xB9,0x86,0xC1,0x1D,0x9E,
-	0xE1,0xF8,0x98,0x11,0x69,0xD9,0x8E,0x94,
-	0x9B,0x1E,0x87,0xE9,0xCE,0x55,0x28,0xDF,
-	0x8C,0xA1,0x89,0x0D,0xBF,0xE6,0x42,0x68,
-	0x41,0x99,0x2D,0x0F,0xB0,0x54,0xBB,0x16,
-};
-
-/*
- * AES is-box
- */
-static const uint8_t aes_isbox[256] = 
-{
-    0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38,
-    0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb,
-    0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87,
-    0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb,
-    0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d,
-    0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e,
-    0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2,
-    0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25,
-    0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16,
-    0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92,
-    0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda,
-    0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84,
-    0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a,
-    0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06,
-    0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02,
-    0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b,
-    0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea,
-    0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73,
-    0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85,
-    0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e,
-    0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89,
-    0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b,
-    0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20,
-    0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4,
-    0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31,
-    0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f,
-    0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d,
-    0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef,
-    0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0,
-    0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61,
-    0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26,
-    0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d
-};
-
-static const unsigned char Rcon[30]=
-{
-	0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80,
-	0x1b,0x36,0x6c,0xd8,0xab,0x4d,0x9a,0x2f,
-	0x5e,0xbc,0x63,0xc6,0x97,0x35,0x6a,0xd4,
-	0xb3,0x7d,0xfa,0xef,0xc5,0x91,
-};
-
-/* ----- static functions ----- */
-static void AES_encrypt(const AES_CTX *ctx, uint32_t *data);
-static void AES_decrypt(const AES_CTX *ctx, uint32_t *data);
-
-/* Perform doubling in Galois Field GF(2^8) using the irreducible polynomial
-   x^8+x^4+x^3+x+1 */
-static unsigned char AES_xtime(uint32_t x)
-{
-	return (x&0x80) ? (x<<1)^0x1b : x<<1;
-}
-
-/**
- * Set up AES with the key/iv and cipher size.
- */
-void AES_set_key(AES_CTX *ctx, const uint8_t *key, 
-        const uint8_t *iv, AES_MODE mode)
-{
-    int i, ii;
-    uint32_t *W, tmp, tmp2;
-    const unsigned char *ip;
-    int words;
-
-    switch (mode)
-    {
-        case AES_MODE_128:
-            i = 10;
-            words = 4;
-            break;
-
-        case AES_MODE_256:
-            i = 14;
-            words = 8;
-            break;
-
-        default:        /* fail silently */
-            return;
-    }
-
-    ctx->rounds = i;
-    ctx->key_size = words;
-    W = ctx->ks;
-    for (i = 0; i < words; i+=2)
-    {
-        W[i+0]=	((uint32_t)key[ 0]<<24)|
-            ((uint32_t)key[ 1]<<16)|
-            ((uint32_t)key[ 2]<< 8)|
-            ((uint32_t)key[ 3]    );
-        W[i+1]=	((uint32_t)key[ 4]<<24)|
-            ((uint32_t)key[ 5]<<16)|
-            ((uint32_t)key[ 6]<< 8)|
-            ((uint32_t)key[ 7]    );
-        key += 8;
-    }
-
-    ip = Rcon;
-    ii = 4 * (ctx->rounds+1);
-    for (i = words; i<ii; i++)
-    {
-        tmp = W[i-1];
-
-        if ((i % words) == 0)
-        {
-            tmp2 =(uint32_t)aes_sbox[(tmp    )&0xff]<< 8;
-            tmp2|=(uint32_t)aes_sbox[(tmp>> 8)&0xff]<<16;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>16)&0xff]<<24;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>24)     ];
-            tmp=tmp2^(((unsigned int)*ip)<<24);
-            ip++;
-        }
-
-        if ((words == 8) && ((i % words) == 4))
-        {
-            tmp2 =(uint32_t)aes_sbox[(tmp    )&0xff]    ;
-            tmp2|=(uint32_t)aes_sbox[(tmp>> 8)&0xff]<< 8;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>16)&0xff]<<16;
-            tmp2|=(uint32_t)aes_sbox[(tmp>>24)     ]<<24;
-            tmp=tmp2;
-        }
-
-        W[i]=W[i-words]^tmp;
-    }
-
-    /* copy the iv across */
-    memcpy(ctx->iv, iv, 16);
-}
-
-/**
- * Change a key for decryption.
- */
-void AES_convert_key(AES_CTX *ctx)
-{
-    int i;
-    uint32_t *k,w,t1,t2,t3,t4;
-
-    k = ctx->ks;
-    k += 4;
-
-    for (i= ctx->rounds*4; i > 4; i--)
-    {
-        w= *k;
-        w = inv_mix_col(w,t1,t2,t3,t4);
-        *k++ =w;
-    }
-}
-
-/**
- * Encrypt a byte sequence (with a block size 16) using the AES cipher.
- */
-void AES_cbc_encrypt(AES_CTX *ctx, const uint8_t *msg, uint8_t *out, int length)
-{
-    int i;
-    uint32_t tin[4], tout[4], iv[4];
-
-    memcpy(iv, ctx->iv, AES_IV_SIZE);
-    for (i = 0; i < 4; i++)
-        tout[i] = ntohl(iv[i]);
-
-    for (length -= AES_BLOCKSIZE; length >= 0; length -= AES_BLOCKSIZE)
-    {
-        uint32_t msg_32[4];
-        uint32_t out_32[4];
-        memcpy(msg_32, msg, AES_BLOCKSIZE);
-        msg += AES_BLOCKSIZE;
-
-        for (i = 0; i < 4; i++)
-            tin[i] = ntohl(msg_32[i])^tout[i];
-
-        AES_encrypt(ctx, tin);
-
-        for (i = 0; i < 4; i++)
-        {
-            tout[i] = tin[i];
-            out_32[i] = htonl(tout[i]);
-        }
-
-        memcpy(out, out_32, AES_BLOCKSIZE);
-        out += AES_BLOCKSIZE;
-    }
-
-    for (i = 0; i < 4; i++)
-        iv[i] = htonl(tout[i]);
-    memcpy(ctx->iv, iv, AES_IV_SIZE);
-}
-
-/**
- * Decrypt a byte sequence (with a block size 16) using the AES cipher.
- */
-void AES_cbc_decrypt(AES_CTX *ctx, const uint8_t *msg, uint8_t *out, int length)
-{
-    int i;
-    uint32_t tin[4], xor[4], tout[4], data[4], iv[4];
-
-    memcpy(iv, ctx->iv, AES_IV_SIZE);
-    for (i = 0; i < 4; i++)
-        xor[i] = ntohl(iv[i]);
-
-    for (length -= 16; length >= 0; length -= 16)
-    {
-        uint32_t msg_32[4];
-        uint32_t out_32[4];
-        memcpy(msg_32, msg, AES_BLOCKSIZE);
-        msg += AES_BLOCKSIZE;
-
-        for (i = 0; i < 4; i++)
-        {
-            tin[i] = ntohl(msg_32[i]);
-            data[i] = tin[i];
-        }
-
-        AES_decrypt(ctx, data);
-
-        for (i = 0; i < 4; i++)
-        {
-            tout[i] = data[i]^xor[i];
-            xor[i] = tin[i];
-            out_32[i] = htonl(tout[i]);
-        }
-
-        memcpy(out, out_32, AES_BLOCKSIZE);
-        out += AES_BLOCKSIZE;
-    }
-
-    for (i = 0; i < 4; i++)
-        iv[i] = htonl(xor[i]);
-    memcpy(ctx->iv, iv, AES_IV_SIZE);
-}
-
-/**
- * Encrypt a single block (16 bytes) of data
- */
-static void AES_encrypt(const AES_CTX *ctx, uint32_t *data)
-{
-    /* To make this code smaller, generate the sbox entries on the fly.
-     * This will have a really heavy effect upon performance.
-     */
-    uint32_t tmp[4];
-    uint32_t tmp1, old_a0, a0, a1, a2, a3, row;
-    int curr_rnd;
-    int rounds = ctx->rounds; 
-    const uint32_t *k = ctx->ks;
-
-    /* Pre-round key addition */
-    for (row = 0; row < 4; row++)
-        data[row] ^= *(k++);
-
-    /* Encrypt one block. */
-    for (curr_rnd = 0; curr_rnd < rounds; curr_rnd++)
-    {
-        /* Perform ByteSub and ShiftRow operations together */
-        for (row = 0; row < 4; row++)
-        {
-            a0 = (uint32_t)aes_sbox[(data[row%4]>>24)&0xFF];
-            a1 = (uint32_t)aes_sbox[(data[(row+1)%4]>>16)&0xFF];
-            a2 = (uint32_t)aes_sbox[(data[(row+2)%4]>>8)&0xFF]; 
-            a3 = (uint32_t)aes_sbox[(data[(row+3)%4])&0xFF];
-
-            /* Perform MixColumn iff not last round */
-            if (curr_rnd < (rounds - 1))
-            {
-                tmp1 = a0 ^ a1 ^ a2 ^ a3;
-                old_a0 = a0;
-                a0 ^= tmp1 ^ AES_xtime(a0 ^ a1);
-                a1 ^= tmp1 ^ AES_xtime(a1 ^ a2);
-                a2 ^= tmp1 ^ AES_xtime(a2 ^ a3);
-                a3 ^= tmp1 ^ AES_xtime(a3 ^ old_a0);
-            }
-
-            tmp[row] = ((a0 << 24) | (a1 << 16) | (a2 << 8) | a3);
-        }
-
-        /* KeyAddition - note that it is vital that this loop is separate from
-           the MixColumn operation, which must be atomic...*/ 
-        for (row = 0; row < 4; row++)
-            data[row] = tmp[row] ^ *(k++);
-    }
-}
-
-/**
- * Decrypt a single block (16 bytes) of data
- */
-static void AES_decrypt(const AES_CTX *ctx, uint32_t *data)
-{ 
-    uint32_t tmp[4];
-    uint32_t xt0,xt1,xt2,xt3,xt4,xt5,xt6;
-    uint32_t a0, a1, a2, a3, row;
-    int curr_rnd;
-    int rounds = ctx->rounds;
-    const uint32_t *k = ctx->ks + ((rounds+1)*4);
-
-    /* pre-round key addition */
-    for (row=4; row > 0;row--)
-        data[row-1] ^= *(--k);
-
-    /* Decrypt one block */
-    for (curr_rnd = 0; curr_rnd < rounds; curr_rnd++)
-    {
-        /* Perform ByteSub and ShiftRow operations together */
-        for (row = 4; row > 0; row--)
-        {
-            a0 = aes_isbox[(data[(row+3)%4]>>24)&0xFF];
-            a1 = aes_isbox[(data[(row+2)%4]>>16)&0xFF];
-            a2 = aes_isbox[(data[(row+1)%4]>>8)&0xFF];
-            a3 = aes_isbox[(data[row%4])&0xFF];
-
-            /* Perform MixColumn iff not last round */
-            if (curr_rnd<(rounds-1))
-            {
-                /* The MDS cofefficients (0x09, 0x0B, 0x0D, 0x0E)
-                   are quite large compared to encryption; this 
-                   operation slows decryption down noticeably. */
-                xt0 = AES_xtime(a0^a1);
-                xt1 = AES_xtime(a1^a2);
-                xt2 = AES_xtime(a2^a3);
-                xt3 = AES_xtime(a3^a0);
-                xt4 = AES_xtime(xt0^xt1);
-                xt5 = AES_xtime(xt1^xt2);
-                xt6 = AES_xtime(xt4^xt5);
-
-                xt0 ^= a1^a2^a3^xt4^xt6;
-                xt1 ^= a0^a2^a3^xt5^xt6;
-                xt2 ^= a0^a1^a3^xt4^xt6;
-                xt3 ^= a0^a1^a2^xt5^xt6;
-                tmp[row-1] = ((xt0<<24)|(xt1<<16)|(xt2<<8)|xt3);
-            }
-            else
-                tmp[row-1] = ((a0<<24)|(a1<<16)|(a2<<8)|a3);
-        }
-
-        for (row = 4; row > 0; row--)
-            data[row-1] = tmp[row-1] ^ *(--k);
-    }
-}
-
-#endif
diff --git a/qemu/roms/ipxe/src/crypto/axtls/bigint.h b/qemu/roms/ipxe/src/crypto/axtls/bigint.h
deleted file mode 100644
index 1f38c53d6..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/bigint.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef BIGINT_HEADER
-#define BIGINT_HEADER
-
-#include "crypto.h"
-
-BI_CTX *bi_initialize(void);
-void bi_terminate(BI_CTX *ctx);
-void bi_permanent(bigint *bi);
-void bi_depermanent(bigint *bi);
-void bi_clear_cache(BI_CTX *ctx);
-void bi_free(BI_CTX *ctx, bigint *bi);
-bigint *bi_copy(bigint *bi);
-bigint *bi_clone(BI_CTX *ctx, const bigint *bi);
-void bi_export(BI_CTX *ctx, bigint *bi, uint8_t *data, int size);
-bigint *bi_import(BI_CTX *ctx, const uint8_t *data, int len);
-bigint *int_to_bi(BI_CTX *ctx, comp i);
-
-/* the functions that actually do something interesting */
-bigint *bi_add(BI_CTX *ctx, bigint *bia, bigint *bib);
-bigint *bi_subtract(BI_CTX *ctx, bigint *bia, 
-        bigint *bib, int *is_negative);
-bigint *bi_divide(BI_CTX *ctx, bigint *bia, bigint *bim, int is_mod);
-bigint *bi_multiply(BI_CTX *ctx, bigint *bia, bigint *bib);
-bigint *bi_mod_power(BI_CTX *ctx, bigint *bi, bigint *biexp);
-bigint *bi_mod_power2(BI_CTX *ctx, bigint *bi, bigint *bim, bigint *biexp);
-int bi_compare(bigint *bia, bigint *bib);
-void bi_set_mod(BI_CTX *ctx, bigint *bim, int mod_offset);
-void bi_free_mod(BI_CTX *ctx, int mod_offset);
-
-#ifdef CONFIG_SSL_FULL_MODE
-void bi_print(const char *label, bigint *bi);
-bigint *bi_str_import(BI_CTX *ctx, const char *data);
-#endif
-
-/**
- * @def bi_mod
- * Find the residue of B. bi_set_mod() must be called before hand.
- */
-#define bi_mod(A, B)      bi_divide(A, B, ctx->bi_mod[ctx->mod_offset], 1)
-
-/**
- * bi_residue() is technically the same as bi_mod(), but it uses the
- * appropriate reduction technique (which is bi_mod() when doing classical
- * reduction).
- */
-#if defined(CONFIG_BIGINT_MONTGOMERY)
-#define bi_residue(A, B)         bi_mont(A, B)
-bigint *bi_mont(BI_CTX *ctx, bigint *bixy);
-#elif defined(CONFIG_BIGINT_BARRETT)
-#define bi_residue(A, B)         bi_barrett(A, B)
-bigint *bi_barrett(BI_CTX *ctx, bigint *bi);
-#else /* if defined(CONFIG_BIGINT_CLASSICAL) */
-#define bi_residue(A, B)         bi_mod(A, B)
-#endif
-
-#ifdef CONFIG_BIGINT_SQUARE
-bigint *bi_square(BI_CTX *ctx, bigint *bi);
-#else
-#define bi_square(A, B)     bi_multiply(A, bi_copy(B), B)
-#endif
-
-#ifdef CONFIG_BIGINT_CRT
-bigint *bi_crt(BI_CTX *ctx, bigint *bi,
-        bigint *dP, bigint *dQ,
-        bigint *p, bigint *q,
-        bigint *qInv);
-#endif
-
-#endif
diff --git a/qemu/roms/ipxe/src/crypto/axtls/bigint_impl.h b/qemu/roms/ipxe/src/crypto/axtls/bigint_impl.h
deleted file mode 100644
index 09d8550ea..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/bigint_impl.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef BIGINT_IMPL_HEADER
-#define BIGINT_IMPL_HEADER
-
-/* Maintain a number of precomputed variables when doing reduction */
-#define BIGINT_M_OFFSET     0    /**< Normal modulo offset. */
-#ifdef CONFIG_BIGINT_CRT
-#define BIGINT_P_OFFSET     1    /**< p modulo offset. */
-#define BIGINT_Q_OFFSET     2    /**< q module offset. */
-#define BIGINT_NUM_MODS     3    /**< The number of modulus constants used. */
-#else
-#define BIGINT_NUM_MODS     1    
-#endif
-
-/* Architecture specific functions for big ints */
-#if defined(CONFIG_INTEGER_8BIT)
-#define COMP_RADIX          256U       /**< Max component + 1 */
-#define COMP_MAX            0xFFFFU/**< (Max dbl comp -1) */
-#define COMP_BIT_SIZE       8   /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      1   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    2   /**< Used For diagnostics only. */
-typedef uint8_t comp;	        /**< A single precision component. */
-typedef uint16_t long_comp;     /**< A double precision component. */
-typedef int16_t slong_comp;     /**< A signed double precision component. */
-#elif defined(CONFIG_INTEGER_16BIT)
-#define COMP_RADIX          65536U       /**< Max component + 1 */
-#define COMP_MAX            0xFFFFFFFFU/**< (Max dbl comp -1) */
-#define COMP_BIT_SIZE       16  /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      2   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    4   /**< Used For diagnostics only. */
-typedef uint16_t comp;	        /**< A single precision component. */
-typedef uint32_t long_comp;     /**< A double precision component. */
-typedef int32_t slong_comp;     /**< A signed double precision component. */
-#else /* regular 32 bit */
-#ifdef WIN32
-#define COMP_RADIX          4294967296i64         
-#define COMP_MAX            0xFFFFFFFFFFFFFFFFui64
-#else
-#define COMP_RADIX          4294967296ULL         /**< Max component + 1 */
-#define COMP_MAX            0xFFFFFFFFFFFFFFFFULL/**< (Max dbl comp -1) */
-#endif
-#define COMP_BIT_SIZE       32  /**< Number of bits in a component. */
-#define COMP_BYTE_SIZE      4   /**< Number of bytes in a component. */
-#define COMP_NUM_NIBBLES    8   /**< Used For diagnostics only. */
-typedef uint32_t comp;	        /**< A single precision component. */
-typedef uint64_t long_comp;     /**< A double precision component. */
-typedef int64_t slong_comp;     /**< A signed double precision component. */
-#endif
-
-/**
- * @struct  _bigint
- * @brief A big integer basic object
- */
-struct _bigint
-{
-    struct _bigint* next;       /**< The next bigint in the cache. */
-    short size;                 /**< The number of components in this bigint. */
-    short max_comps;            /**< The heapsize allocated for this bigint */
-    int refs;                   /**< An internal reference count. */
-    comp* comps;                /**< A ptr to the actual component data */
-};
-
-typedef struct _bigint bigint;  /**< An alias for _bigint */
-
-/**
- * Maintains the state of the cache, and a number of variables used in 
- * reduction.
- */
-typedef struct /**< A big integer "session" context. */
-{
-    bigint *active_list;                    /**< Bigints currently used. */
-    bigint *free_list;                      /**< Bigints not used. */
-    bigint *bi_radix;                       /**< The radix used. */
-    bigint *bi_mod[BIGINT_NUM_MODS];        /**< modulus */
-
-#if defined(CONFIG_BIGINT_MONTGOMERY)
-    bigint *bi_RR_mod_m[BIGINT_NUM_MODS];   /**< R^2 mod m */
-    bigint *bi_R_mod_m[BIGINT_NUM_MODS];    /**< R mod m */
-    comp N0_dash[BIGINT_NUM_MODS];
-#elif defined(CONFIG_BIGINT_BARRETT)
-    bigint *bi_mu[BIGINT_NUM_MODS];         /**< Storage for mu */
-#endif
-    bigint *bi_normalised_mod[BIGINT_NUM_MODS]; /**< Normalised mod storage. */
-    bigint **g;                 /**< Used by sliding-window. */
-    int window;                 /**< The size of the sliding window */
-    int active_count;           /**< Number of active bigints. */
-    int free_count;             /**< Number of free bigints. */
-
-#ifdef CONFIG_BIGINT_MONTGOMERY
-    uint8_t use_classical;      /**< Use classical reduction. */
-#endif
-    uint8_t mod_offset;         /**< The mod offset we are using */
-} BI_CTX;
-
-#ifndef WIN32
-#define max(a,b) ((a)>(b)?(a):(b))  /**< Find the maximum of 2 numbers. */
-#define min(a,b) ((a)<(b)?(a):(b))  /**< Find the minimum of 2 numbers. */
-#endif
-
-#define PERMANENT           0x7FFF55AA  /**< A magic number for permanents. */
-
-#endif
diff --git a/qemu/roms/ipxe/src/crypto/axtls/config.h b/qemu/roms/ipxe/src/crypto/axtls/config.h
deleted file mode 100644
index 32fa3bf03..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/config.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef AXTLS_CONFIG_H
-#define AXTLS_CONFIG_H
-
-/**
- * @file config.h
- *
- * Trick the axtls code into building within our build environment.
- */
-
-#define CONFIG_SSL_ENABLE_CLIENT 1
-#define CONFIG_BIGINT_CLASSICAL 1
-
-#endif
diff --git a/qemu/roms/ipxe/src/crypto/axtls/crypto.h b/qemu/roms/ipxe/src/crypto/axtls/crypto.h
deleted file mode 100644
index 2c4cda4de..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/crypto.h
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Copyright (c) 2007, Cameron Rich
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice,
- *   this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- *   this list of conditions and the following disclaimer in the documentation
- *   and/or other materials provided with the distribution.
- * * Neither the name of the axTLS project nor the names of its contributors
- *   may be used to endorse or promote products derived from this software
- *   without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/**
- * @file crypto.h
- */
-
-#ifndef HEADER_CRYPTO_H
-#define HEADER_CRYPTO_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "config.h"
-#include "bigint_impl.h"
-#include "bigint.h"
-
-#ifndef STDCALL
-#define STDCALL
-#endif
-#ifndef EXP_FUNC
-#define EXP_FUNC
-#endif
-
-
-/* enable features based on a 'super-set' capbaility. */
-#if defined(CONFIG_SSL_FULL_MODE)
-#define CONFIG_SSL_ENABLE_CLIENT
-#define CONFIG_SSL_CERT_VERIFICATION
-#elif defined(CONFIG_SSL_ENABLE_CLIENT)
-#define CONFIG_SSL_CERT_VERIFICATION
-#endif
-
-/**************************************************************************
- * AES declarations 
- **************************************************************************/
-
-#define AES_MAXROUNDS			14
-#define AES_BLOCKSIZE           16
-#define AES_IV_SIZE             16
-
-typedef struct aes_key_st 
-{
-    uint16_t rounds;
-    uint16_t key_size;
-    uint32_t ks[(AES_MAXROUNDS+1)*8];
-    uint8_t iv[AES_IV_SIZE];
-} AES_CTX;
-
-typedef enum
-{
-    AES_MODE_128,
-    AES_MODE_256
-} AES_MODE;
-
-void AES_set_key(AES_CTX *ctx, const uint8_t *key, 
-        const uint8_t *iv, AES_MODE mode);
-void AES_cbc_encrypt(AES_CTX *ctx, const uint8_t *msg, 
-        uint8_t *out, int length);
-void AES_cbc_decrypt(AES_CTX *ks, const uint8_t *in, uint8_t *out, int length);
-void AES_convert_key(AES_CTX *ctx);
-
-/**************************************************************************
- * RC4 declarations 
- **************************************************************************/
-
-typedef struct 
-{
-    uint8_t x, y, m[256];
-} RC4_CTX;
-
-void RC4_setup(RC4_CTX *s, const uint8_t *key, int length);
-void RC4_crypt(RC4_CTX *s, const uint8_t *msg, uint8_t *data, int length);
-
-/**************************************************************************
- * SHA1 declarations 
- **************************************************************************/
-
-#define SHA1_SIZE   20
-
-/*
- *  This structure will hold context information for the SHA-1
- *  hashing operation
- */
-typedef struct 
-{
-    uint32_t Intermediate_Hash[SHA1_SIZE/4]; /* Message Digest */
-    uint32_t Length_Low;            /* Message length in bits */
-    uint32_t Length_High;           /* Message length in bits */
-    uint16_t Message_Block_Index;   /* Index into message block array   */
-    uint8_t Message_Block[64];      /* 512-bit message blocks */
-} SHA1_CTX;
-
-void SHA1_Init(SHA1_CTX *);
-void SHA1_Update(SHA1_CTX *, const uint8_t * msg, int len);
-void SHA1_Final(uint8_t *digest, SHA1_CTX *);
-
-/**************************************************************************
- * MD2 declarations
- **************************************************************************/
-
-#define MD2_SIZE 16
-
-typedef struct
-{
-    unsigned char cksum[16];    /* checksum of the data block */
-    unsigned char state[48];    /* intermediate digest state */
-    unsigned char buffer[16];   /* data block being processed */
-    int left;                   /* amount of data in buffer */
-} MD2_CTX;
-
-EXP_FUNC void STDCALL MD2_Init(MD2_CTX *ctx);
-EXP_FUNC void STDCALL MD2_Update(MD2_CTX *ctx, const uint8_t *input, int ilen);
-EXP_FUNC void STDCALL MD2_Final(uint8_t *digest, MD2_CTX *ctx);
-
-/**************************************************************************
- * MD5 declarations
- **************************************************************************/
-
-#define MD5_SIZE    16
-
-typedef struct 
-{
-  uint32_t state[4];        /* state (ABCD) */
-  uint32_t count[2];        /* number of bits, modulo 2^64 (lsb first) */
-  uint8_t buffer[64];       /* input buffer */
-} MD5_CTX;
-
-EXP_FUNC void STDCALL MD5_Init(MD5_CTX *);
-EXP_FUNC void STDCALL MD5_Update(MD5_CTX *, const uint8_t *msg, int len);
-EXP_FUNC void STDCALL MD5_Final(uint8_t *digest, MD5_CTX *);
-
-/**************************************************************************
- * HMAC declarations 
- **************************************************************************/
-void hmac_md5(const uint8_t *msg, int length, const uint8_t *key, 
-        int key_len, uint8_t *digest);
-void hmac_sha1(const uint8_t *msg, int length, const uint8_t *key, 
-        int key_len, uint8_t *digest);
-
-/**************************************************************************
- * RSA declarations 
- **************************************************************************/
-
-typedef struct 
-{
-    bigint *m;              /* modulus */
-    bigint *e;              /* public exponent */
-    bigint *d;              /* private exponent */
-#ifdef CONFIG_BIGINT_CRT
-    bigint *p;              /* p as in m = pq */
-    bigint *q;              /* q as in m = pq */
-    bigint *dP;             /* d mod (p-1) */
-    bigint *dQ;             /* d mod (q-1) */
-    bigint *qInv;           /* q^-1 mod p */
-#endif
-    int num_octets;
-    BI_CTX *bi_ctx;
-} RSA_CTX;
-
-void RSA_priv_key_new(RSA_CTX **rsa_ctx, 
-        const uint8_t *modulus, int mod_len,
-        const uint8_t *pub_exp, int pub_len,
-        const uint8_t *priv_exp, int priv_len
-#ifdef CONFIG_BIGINT_CRT
-      , const uint8_t *p, int p_len,
-        const uint8_t *q, int q_len,
-        const uint8_t *dP, int dP_len,
-        const uint8_t *dQ, int dQ_len,
-        const uint8_t *qInv, int qInv_len
-#endif
-        );
-void RSA_pub_key_new(RSA_CTX **rsa_ctx, 
-        const uint8_t *modulus, int mod_len,
-        const uint8_t *pub_exp, int pub_len);
-void RSA_free(RSA_CTX *ctx);
-int RSA_decrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint8_t *out_data,
-        int is_decryption);
-bigint *RSA_private(const RSA_CTX *c, bigint *bi_msg);
-#if defined(CONFIG_SSL_CERT_VERIFICATION) || defined(CONFIG_SSL_GENERATE_X509_CERT)
-bigint *RSA_sign_verify(BI_CTX *ctx, const uint8_t *sig, int sig_len,
-        bigint *modulus, bigint *pub_exp);
-bigint *RSA_public(const RSA_CTX * c, bigint *bi_msg);
-int RSA_encrypt(const RSA_CTX *ctx, const uint8_t *in_data, uint16_t in_len, 
-        uint8_t *out_data, int is_signing);
-void RSA_print(const RSA_CTX *ctx);
-#endif
-
-/**************************************************************************
- * RNG declarations
- **************************************************************************/
-EXP_FUNC void STDCALL RNG_initialize(const uint8_t *seed_buf, int size);
-EXP_FUNC void STDCALL RNG_terminate(void);
-EXP_FUNC void STDCALL get_random(int num_rand_bytes, uint8_t *rand_data);
-void get_random_NZ(int num_rand_bytes, uint8_t *rand_data);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif 
diff --git a/qemu/roms/ipxe/src/crypto/axtls/os_port.h b/qemu/roms/ipxe/src/crypto/axtls/os_port.h
deleted file mode 100644
index 76313e204..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls/os_port.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef AXTLS_OS_PORT_H
-#define AXTLS_OS_PORT_H
-
-/**
- * @file os_port.h
- *
- * Trick the axtls code into building within our build environment.
- */
-
-#include <stdint.h>
-#include <byteswap.h>
-
-/** All imported axTLS files are licensed using the three-clause BSD licence */
-FILE_LICENCE ( BSD3 );
-
-/** We can't actually abort, since we are effectively a kernel... */
-#define abort() assert ( 0 )
-
-/** rsa.c uses alloca() */
-#define alloca( size ) __builtin_alloca ( size )
-
-#include <ipxe/random_nz.h>
-static inline void get_random_NZ ( int num_rand_bytes, uint8_t *rand_data ) {
-	/* AXTLS does not check for failures when generating random
-	 * data.  Rely on the fact that get_random_nz() does not
-	 * request prediction resistance (and so cannot introduce new
-	 * failures) and therefore any potential failure must already
-	 * have been encountered by e.g. tls_generate_random(), which
-	 * does check for failures.
-	 */
-	get_random_nz ( rand_data, num_rand_bytes );
-}
-
-/* Expose AES_encrypt() and AES_decrypt() in aes.o */
-#define aes 1
-#if OBJECT
-
-struct aes_key_st;
-
-static void AES_encrypt ( const struct aes_key_st *ctx, uint32_t *data );
-static void AES_decrypt ( const struct aes_key_st *ctx, uint32_t *data );
-
-void axtls_aes_encrypt ( void *ctx, uint32_t *data ) {
-	AES_encrypt ( ctx, data );
-}
-
-void axtls_aes_decrypt ( void *ctx, uint32_t *data ) {
-	AES_decrypt ( ctx, data );
-}
-
-#endif
-#undef aes
-
-#endif 
diff --git a/qemu/roms/ipxe/src/crypto/axtls_aes.c b/qemu/roms/ipxe/src/crypto/axtls_aes.c
deleted file mode 100644
index 7f93c0ed7..000000000
--- a/qemu/roms/ipxe/src/crypto/axtls_aes.c
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <string.h>
-#include <errno.h>
-#include <assert.h>
-#include <byteswap.h>
-#include <ipxe/crypto.h>
-#include <ipxe/cbc.h>
-#include <ipxe/aes.h>
-#include "crypto/axtls/crypto.h"
-
-/** @file
- *
- * AES algorithm
- *
- */
-
-/**
- * Set key
- *
- * @v ctx		Context
- * @v key		Key
- * @v keylen		Key length
- * @ret rc		Return status code
- */
-static int aes_setkey ( void *ctx, const void *key, size_t keylen ) {
-	struct aes_context *aes_ctx = ctx;
-	AES_MODE mode;
-	void *iv;
-
-	switch ( keylen ) {
-	case ( 128 / 8 ):
-		mode = AES_MODE_128;
-		break;
-	case ( 256 / 8 ):
-		mode = AES_MODE_256;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	/* IV is not a relevant concept at this stage; use a dummy
-	 * value that will have no side-effects.
-	 */
-	iv = &aes_ctx->axtls_ctx.iv;
-
-	AES_set_key ( &aes_ctx->axtls_ctx, key, iv, mode );
-
-	aes_ctx->decrypting = 0;
-
-	return 0;
-}
-
-/**
- * Set initialisation vector
- *
- * @v ctx		Context
- * @v iv		Initialisation vector
- */
-static void aes_setiv ( void *ctx __unused, const void *iv __unused ) {
-	/* Nothing to do */
-}
-
-/**
- * Call AXTLS' AES_encrypt() or AES_decrypt() functions
- *
- * @v axtls_ctx		AXTLS AES context
- * @v src		Data to process
- * @v dst		Buffer for output
- * @v func		AXTLS AES function to call
- */
-static void aes_call_axtls ( AES_CTX *axtls_ctx, const void *src, void *dst,
-			     void ( * func ) ( const AES_CTX *axtls_ctx,
-					       uint32_t *data ) ){
-	const uint32_t *srcl = src;
-	uint32_t *dstl = dst;
-	unsigned int i;
-
-	/* AXTLS' AES_encrypt() and AES_decrypt() functions both
-	 * expect to deal with an array of four dwords in host-endian
-	 * order.
-	 */
-	for ( i = 0 ; i < 4 ; i++ )
-		dstl[i] = ntohl ( srcl[i] );
-	func ( axtls_ctx, dstl );
-	for ( i = 0 ; i < 4 ; i++ )
-		dstl[i] = htonl ( dstl[i] );
-}
-
-/**
- * Encrypt data
- *
- * @v ctx		Context
- * @v src		Data to encrypt
- * @v dst		Buffer for encrypted data
- * @v len		Length of data
- */
-static void aes_encrypt ( void *ctx, const void *src, void *dst,
-			  size_t len ) {
-	struct aes_context *aes_ctx = ctx;
-
-	assert ( len == AES_BLOCKSIZE );
-	if ( aes_ctx->decrypting )
-		assert ( 0 );
-	aes_call_axtls ( &aes_ctx->axtls_ctx, src, dst, axtls_aes_encrypt );
-}
-
-/**
- * Decrypt data
- *
- * @v ctx		Context
- * @v src		Data to decrypt
- * @v dst		Buffer for decrypted data
- * @v len		Length of data
- */
-static void aes_decrypt ( void *ctx, const void *src, void *dst,
-			  size_t len ) {
-	struct aes_context *aes_ctx = ctx;
-
-	assert ( len == AES_BLOCKSIZE );
-	if ( ! aes_ctx->decrypting ) {
-		AES_convert_key ( &aes_ctx->axtls_ctx );
-		aes_ctx->decrypting = 1;
-	}
-	aes_call_axtls ( &aes_ctx->axtls_ctx, src, dst, axtls_aes_decrypt );
-}
-
-/** Basic AES algorithm */
-struct cipher_algorithm aes_algorithm = {
-	.name = "aes",
-	.ctxsize = sizeof ( struct aes_context ),
-	.blocksize = AES_BLOCKSIZE,
-	.setkey = aes_setkey,
-	.setiv = aes_setiv,
-	.encrypt = aes_encrypt,
-	.decrypt = aes_decrypt,
-};
-
-/* AES with cipher-block chaining */
-CBC_CIPHER ( aes_cbc, aes_cbc_algorithm,
-	     aes_algorithm, struct aes_context, AES_BLOCKSIZE );
diff --git a/qemu/roms/ipxe/src/crypto/bigint.c b/qemu/roms/ipxe/src/crypto/bigint.c
index 340128e2f..50f320302 100644
--- a/qemu/roms/ipxe/src/crypto/bigint.c
+++ b/qemu/roms/ipxe/src/crypto/bigint.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/crypto/cbc.c b/qemu/roms/ipxe/src/crypto/cbc.c
index 9bf0e8b49..0ba17ee48 100644
--- a/qemu/roms/ipxe/src/crypto/cbc.c
+++ b/qemu/roms/ipxe/src/crypto/cbc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/qemu/roms/ipxe/src/crypto/certstore.c b/qemu/roms/ipxe/src/crypto/certstore.c
index 77cf6ebb6..503ce499e 100644
--- a/qemu/roms/ipxe/src/crypto/certstore.c
+++ b/qemu/roms/ipxe/src/crypto/certstore.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/crypto/chap.c b/qemu/roms/ipxe/src/crypto/chap.c
index db64371c7..c90c16def 100644
--- a/qemu/roms/ipxe/src/crypto/chap.c
+++ b/qemu/roms/ipxe/src/crypto/chap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/crypto/cms.c b/qemu/roms/ipxe/src/crypto/cms.c
index b4a41de6c..bc2148e8a 100644
--- a/qemu/roms/ipxe/src/crypto/cms.c
+++ b/qemu/roms/ipxe/src/crypto/cms.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/crypto_null.c b/qemu/roms/ipxe/src/crypto/crypto_null.c
index ba05f7269..15a1c538b 100644
--- a/qemu/roms/ipxe/src/crypto/crypto_null.c
+++ b/qemu/roms/ipxe/src/crypto/crypto_null.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/crypto/deflate.c b/qemu/roms/ipxe/src/crypto/deflate.c
index 91a489961..e1c87d5fe 100644
--- a/qemu/roms/ipxe/src/crypto/deflate.c
+++ b/qemu/roms/ipxe/src/crypto/deflate.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <strings.h>
diff --git a/qemu/roms/ipxe/src/crypto/drbg.c b/qemu/roms/ipxe/src/crypto/drbg.c
index 9e0175d25..5c8b5e612 100644
--- a/qemu/roms/ipxe/src/crypto/drbg.c
+++ b/qemu/roms/ipxe/src/crypto/drbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/ecb.c b/qemu/roms/ipxe/src/crypto/ecb.c
new file mode 100644
index 000000000..3c9cf340c
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/ecb.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/crypto.h>
+#include <ipxe/ecb.h>
+
+/** @file
+ *
+ * Electronic codebook (ECB)
+ *
+ */
+
+/**
+ * Encrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to encrypt
+ * @v dst		Buffer for encrypted data
+ * @v len		Length of data
+ * @v raw_cipher	Underlying cipher algorithm
+ */
+void ecb_encrypt ( void *ctx, const void *src, void *dst, size_t len,
+		   struct cipher_algorithm *raw_cipher ) {
+	size_t blocksize = raw_cipher->blocksize;
+
+	assert ( ( len % blocksize ) == 0 );
+
+	while ( len ) {
+		cipher_encrypt ( raw_cipher, ctx, src, dst, blocksize );
+		dst += blocksize;
+		src += blocksize;
+		len -= blocksize;
+	}
+}
+
+/**
+ * Decrypt data
+ *
+ * @v ctx		Context
+ * @v src		Data to decrypt
+ * @v dst		Buffer for decrypted data
+ * @v len		Length of data
+ * @v raw_cipher	Underlying cipher algorithm
+ */
+void ecb_decrypt ( void *ctx, const void *src, void *dst, size_t len,
+		   struct cipher_algorithm *raw_cipher ) {
+	size_t blocksize = raw_cipher->blocksize;
+
+	assert ( ( len % blocksize ) == 0 );
+
+	while ( len ) {
+		cipher_decrypt ( raw_cipher, ctx, src, dst, blocksize );
+		dst += blocksize;
+		src += blocksize;
+		len -= blocksize;
+	}
+}
diff --git a/qemu/roms/ipxe/src/crypto/entropy.c b/qemu/roms/ipxe/src/crypto/entropy.c
index c7045840e..5acbc0258 100644
--- a/qemu/roms/ipxe/src/crypto/entropy.c
+++ b/qemu/roms/ipxe/src/crypto/entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/hash_df.c b/qemu/roms/ipxe/src/crypto/hash_df.c
index adf1d87e4..c1417e683 100644
--- a/qemu/roms/ipxe/src/crypto/hash_df.c
+++ b/qemu/roms/ipxe/src/crypto/hash_df.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/hmac.c b/qemu/roms/ipxe/src/crypto/hmac.c
index e9459198c..95a46195c 100644
--- a/qemu/roms/ipxe/src/crypto/hmac.c
+++ b/qemu/roms/ipxe/src/crypto/hmac.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/crypto/hmac_drbg.c b/qemu/roms/ipxe/src/crypto/hmac_drbg.c
index 1e5f732e2..6c1d5deb2 100644
--- a/qemu/roms/ipxe/src/crypto/hmac_drbg.c
+++ b/qemu/roms/ipxe/src/crypto/hmac_drbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/md5.c b/qemu/roms/ipxe/src/crypto/md5.c
index 122c7d59e..f9738b0ac 100644
--- a/qemu/roms/ipxe/src/crypto/md5.c
+++ b/qemu/roms/ipxe/src/crypto/md5.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c
new file mode 100644
index 000000000..06722c0e1
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha1.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha1.h>
+#include <ipxe/tls.h>
+
+/** TLS_RSA_WITH_AES_128_CBC_SHA cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_128_cbc_sha __tls_cipher_suite (03) = {
+	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
+	.key_len = ( 128 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+};
+
+/** TLS_RSA_WITH_AES_256_CBC_SHA cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_256_cbc_sha __tls_cipher_suite (04) = {
+	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
+	.key_len = ( 256 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha1_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c
new file mode 100644
index 000000000..c609eacea
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_aes_cbc_sha256.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/rsa.h>
+#include <ipxe/aes.h>
+#include <ipxe/sha256.h>
+#include <ipxe/tls.h>
+
+/** TLS_RSA_WITH_AES_128_CBC_SHA256 cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_128_cbc_sha256 __tls_cipher_suite(01)={
+	.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
+	.key_len = ( 128 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+};
+
+/** TLS_RSA_WITH_AES_256_CBC_SHA256 cipher suite */
+struct tls_cipher_suite tls_rsa_with_aes_256_cbc_sha256 __tls_cipher_suite(02)={
+	.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
+	.key_len = ( 256 / 8 ),
+	.pubkey = &rsa_algorithm,
+	.cipher = &aes_cbc_algorithm,
+	.digest = &sha256_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_md5.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_md5.c
new file mode 100644
index 000000000..ac828ac11
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_md5.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/md5.h>
+#include <ipxe/asn1.h>
+
+/** "md5WithRSAEncryption" object identifier */
+static uint8_t oid_md5_with_rsa_encryption[] =
+	{ ASN1_OID_MD5WITHRSAENCRYPTION };
+
+/** "md5WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm md5_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "md5WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &md5_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_md5_with_rsa_encryption ),
+};
+
+/** MD5 digestInfo prefix */
+static const uint8_t rsa_md5_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( MD5_DIGEST_SIZE, ASN1_OID_MD5 ) };
+
+/** MD5 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_md5_prefix __rsa_digestinfo_prefix = {
+	.digest = &md5_algorithm,
+	.data = rsa_md5_prefix_data,
+	.len = sizeof ( rsa_md5_prefix_data ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha1.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha1.c
new file mode 100644
index 000000000..39424bf2d
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha1.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha1.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha1WithRSAEncryption" object identifier */
+static uint8_t oid_sha1_with_rsa_encryption[] =
+	{ ASN1_OID_SHA1WITHRSAENCRYPTION };
+
+/** "sha1WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha1_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha1WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha1_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha1_with_rsa_encryption ),
+};
+
+/** SHA-1 digestInfo prefix */
+static const uint8_t rsa_sha1_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA1_DIGEST_SIZE, ASN1_OID_SHA1 ) };
+
+/** SHA-1 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha1_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha1_algorithm,
+	.data = rsa_sha1_prefix_data,
+	.len = sizeof ( rsa_sha1_prefix_data ),
+};
+
+/** RSA with SHA-1 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha1 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA1_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha1_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha224.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha224.c
new file mode 100644
index 000000000..5e8755aab
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha224.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha256.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha224WithRSAEncryption" object identifier */
+static uint8_t oid_sha224_with_rsa_encryption[] =
+	{ ASN1_OID_SHA224WITHRSAENCRYPTION };
+
+/** "sha224WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha224_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha224WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha224_with_rsa_encryption ),
+};
+
+/** SHA-224 digestInfo prefix */
+static const uint8_t rsa_sha224_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA224_DIGEST_SIZE, ASN1_OID_SHA224 ) };
+
+/** SHA-224 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha224_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha224_algorithm,
+	.data = rsa_sha224_prefix_data,
+	.len = sizeof ( rsa_sha224_prefix_data ),
+};
+
+/** RSA with SHA-224 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha224 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA224_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha224_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha256.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha256.c
new file mode 100644
index 000000000..b44af5f19
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha256.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha256.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha256WithRSAEncryption" object identifier */
+static uint8_t oid_sha256_with_rsa_encryption[] =
+	{ ASN1_OID_SHA256WITHRSAENCRYPTION };
+
+/** "sha256WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha256_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha256WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha256_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha256_with_rsa_encryption ),
+};
+
+/** SHA-256 digestInfo prefix */
+static const uint8_t rsa_sha256_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA256_DIGEST_SIZE, ASN1_OID_SHA256 ) };
+
+/** SHA-256 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha256_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha256_algorithm,
+	.data = rsa_sha256_prefix_data,
+	.len = sizeof ( rsa_sha256_prefix_data ),
+};
+
+/** RSA with SHA-256 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha256 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA256_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha256_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha384.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha384.c
new file mode 100644
index 000000000..af22a2bf0
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha384.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha512.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha384WithRSAEncryption" object identifier */
+static uint8_t oid_sha384_with_rsa_encryption[] =
+	{ ASN1_OID_SHA384WITHRSAENCRYPTION };
+
+/** "sha384WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha384_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha384WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha384_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha384_with_rsa_encryption ),
+};
+
+/** SHA-384 digestInfo prefix */
+static const uint8_t rsa_sha384_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA384_DIGEST_SIZE, ASN1_OID_SHA384 ) };
+
+/** SHA-384 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha384_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha384_algorithm,
+	.data = rsa_sha384_prefix_data,
+	.len = sizeof ( rsa_sha384_prefix_data ),
+};
+
+/** RSA with SHA-384 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha384 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA384_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha384_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha512.c b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha512.c
new file mode 100644
index 000000000..29ee15493
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/mishmash/rsa_sha512.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/rsa.h>
+#include <ipxe/sha512.h>
+#include <ipxe/asn1.h>
+#include <ipxe/tls.h>
+
+/** "sha512WithRSAEncryption" object identifier */
+static uint8_t oid_sha512_with_rsa_encryption[] =
+	{ ASN1_OID_SHA512WITHRSAENCRYPTION };
+
+/** "sha512WithRSAEncryption" OID-identified algorithm */
+struct asn1_algorithm sha512_with_rsa_encryption_algorithm __asn1_algorithm = {
+	.name = "sha512WithRSAEncryption",
+	.pubkey = &rsa_algorithm,
+	.digest = &sha512_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_with_rsa_encryption ),
+};
+
+/** SHA-512 digestInfo prefix */
+static const uint8_t rsa_sha512_prefix_data[] =
+	{ RSA_DIGESTINFO_PREFIX ( SHA512_DIGEST_SIZE, ASN1_OID_SHA512 ) };
+
+/** SHA-512 digestInfo prefix */
+struct rsa_digestinfo_prefix rsa_sha512_prefix __rsa_digestinfo_prefix = {
+	.digest = &sha512_algorithm,
+	.data = rsa_sha512_prefix_data,
+	.len = sizeof ( rsa_sha512_prefix_data ),
+};
+
+/** RSA with SHA-512 signature hash algorithm */
+struct tls_signature_hash_algorithm tls_rsa_sha512 __tls_sig_hash_algorithm = {
+	.code = {
+		.signature = TLS_RSA_ALGORITHM,
+		.hash = TLS_SHA512_ALGORITHM,
+	},
+	.pubkey = &rsa_algorithm,
+	.digest = &sha512_algorithm,
+};
diff --git a/qemu/roms/ipxe/src/crypto/null_entropy.c b/qemu/roms/ipxe/src/crypto/null_entropy.c
index c56d5e76f..d1e1a6f73 100644
--- a/qemu/roms/ipxe/src/crypto/null_entropy.c
+++ b/qemu/roms/ipxe/src/crypto/null_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/ocsp.c b/qemu/roms/ipxe/src/crypto/ocsp.c
index 66e47c57e..5df55bc96 100644
--- a/qemu/roms/ipxe/src/crypto/ocsp.c
+++ b/qemu/roms/ipxe/src/crypto/ocsp.c
@@ -233,7 +233,7 @@ static int ocsp_uri_string ( struct ocsp_check *ocsp ) {
 		goto err_path_base64;
 	}
 	base64_encode ( ocsp->request.builder.data, ocsp->request.builder.len,
-			path_base64_string );
+			path_base64_string, path_len );
 
 	/* URI-encode the Base64-encoded request */
 	memset ( &path_uri, 0, sizeof ( path_uri ) );
diff --git a/qemu/roms/ipxe/src/crypto/privkey.c b/qemu/roms/ipxe/src/crypto/privkey.c
index e010649c0..a6043bd1e 100644
--- a/qemu/roms/ipxe/src/crypto/privkey.c
+++ b/qemu/roms/ipxe/src/crypto/privkey.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/crypto/random_nz.c b/qemu/roms/ipxe/src/crypto/random_nz.c
index f1d2e187d..5fe576e05 100644
--- a/qemu/roms/ipxe/src/crypto/random_nz.c
+++ b/qemu/roms/ipxe/src/crypto/random_nz.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/rbg.c b/qemu/roms/ipxe/src/crypto/rbg.c
index e2d06978c..943b288c3 100644
--- a/qemu/roms/ipxe/src/crypto/rbg.c
+++ b/qemu/roms/ipxe/src/crypto/rbg.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/rootcert.c b/qemu/roms/ipxe/src/crypto/rootcert.c
index ae28905ac..00ea1647e 100644
--- a/qemu/roms/ipxe/src/crypto/rootcert.c
+++ b/qemu/roms/ipxe/src/crypto/rootcert.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <ipxe/crypto.h>
diff --git a/qemu/roms/ipxe/src/crypto/rsa.c b/qemu/roms/ipxe/src/crypto/rsa.c
index 0ab7b2ad3..36109280d 100644
--- a/qemu/roms/ipxe/src/crypto/rsa.c
+++ b/qemu/roms/ipxe/src/crypto/rsa.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -28,9 +32,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/crypto.h>
 #include <ipxe/bigint.h>
 #include <ipxe/random_nz.h>
-#include <ipxe/md5.h>
-#include <ipxe/sha1.h>
-#include <ipxe/sha256.h>
 #include <ipxe/rsa.h>
 
 /** @file
@@ -49,18 +50,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** "rsaEncryption" object identifier */
 static uint8_t oid_rsa_encryption[] = { ASN1_OID_RSAENCRYPTION };
 
-/** "md5WithRSAEncryption" object identifier */
-static uint8_t oid_md5_with_rsa_encryption[] =
-	{ ASN1_OID_MD5WITHRSAENCRYPTION };
-
-/** "sha1WithRSAEncryption" object identifier */
-static uint8_t oid_sha1_with_rsa_encryption[] =
-	{ ASN1_OID_SHA1WITHRSAENCRYPTION };
-
-/** "sha256WithRSAEncryption" object identifier */
-static uint8_t oid_sha256_with_rsa_encryption[] =
-	{ ASN1_OID_SHA256WITHRSAENCRYPTION };
-
 /** "rsaEncryption" OID-identified algorithm */
 struct asn1_algorithm rsa_encryption_algorithm __asn1_algorithm = {
 	.name = "rsaEncryption",
@@ -69,63 +58,6 @@ struct asn1_algorithm rsa_encryption_algorithm __asn1_algorithm = {
 	.oid = ASN1_OID_CURSOR ( oid_rsa_encryption ),
 };
 
-/** "md5WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm md5_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "md5WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &md5_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_md5_with_rsa_encryption ),
-};
-
-/** "sha1WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm sha1_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "sha1WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &sha1_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_sha1_with_rsa_encryption ),
-};
-
-/** "sha256WithRSAEncryption" OID-identified algorithm */
-struct asn1_algorithm sha256_with_rsa_encryption_algorithm __asn1_algorithm = {
-	.name = "sha256WithRSAEncryption",
-	.pubkey = &rsa_algorithm,
-	.digest = &sha256_algorithm,
-	.oid = ASN1_OID_CURSOR ( oid_sha256_with_rsa_encryption ),
-};
-
-/** MD5 digestInfo prefix */
-static const uint8_t rsa_md5_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( MD5_DIGEST_SIZE, ASN1_OID_MD5 ) };
-
-/** SHA-1 digestInfo prefix */
-static const uint8_t rsa_sha1_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( SHA1_DIGEST_SIZE, ASN1_OID_SHA1 ) };
-
-/** SHA-256 digestInfo prefix */
-static const uint8_t rsa_sha256_prefix_data[] =
-	{ RSA_DIGESTINFO_PREFIX ( SHA256_DIGEST_SIZE, ASN1_OID_SHA256 ) };
-
-/** MD5 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_md5_prefix __rsa_digestinfo_prefix = {
-	.digest = &md5_algorithm,
-	.data = rsa_md5_prefix_data,
-	.len = sizeof ( rsa_md5_prefix_data ),
-};
-
-/** SHA-1 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_sha1_prefix __rsa_digestinfo_prefix = {
-	.digest = &sha1_algorithm,
-	.data = rsa_sha1_prefix_data,
-	.len = sizeof ( rsa_sha1_prefix_data ),
-};
-
-/** SHA-256 digestInfo prefix */
-struct rsa_digestinfo_prefix rsa_sha256_prefix __rsa_digestinfo_prefix = {
-	.digest = &sha256_algorithm,
-	.data = rsa_sha256_prefix_data,
-	.len = sizeof ( rsa_sha256_prefix_data ),
-};
-
 /**
  * Identify RSA prefix
  *
diff --git a/qemu/roms/ipxe/src/crypto/sha1.c b/qemu/roms/ipxe/src/crypto/sha1.c
index e1bef669e..51866f4b7 100644
--- a/qemu/roms/ipxe/src/crypto/sha1.c
+++ b/qemu/roms/ipxe/src/crypto/sha1.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/crypto/sha224.c b/qemu/roms/ipxe/src/crypto/sha224.c
new file mode 100644
index 000000000..be25f24e9
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/sha224.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-224 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha256.h>
+
+/** SHA-224 initial digest values */
+static const struct sha256_digest sha224_init_digest = {
+	.h = {
+		cpu_to_be32 ( 0xc1059ed8 ),
+		cpu_to_be32 ( 0x367cd507 ),
+		cpu_to_be32 ( 0x3070dd17 ),
+		cpu_to_be32 ( 0xf70e5939 ),
+		cpu_to_be32 ( 0xffc00b31 ),
+		cpu_to_be32 ( 0x68581511 ),
+		cpu_to_be32 ( 0x64f98fa7 ),
+		cpu_to_be32 ( 0xbefa4fa4 ),
+	},
+};
+
+/**
+ * Initialise SHA-224 algorithm
+ *
+ * @v ctx		SHA-224 context
+ */
+static void sha224_init ( void *ctx ) {
+	struct sha256_context *context = ctx;
+
+	sha256_family_init ( context, &sha224_init_digest, SHA224_DIGEST_SIZE );
+}
+
+/** SHA-224 algorithm */
+struct digest_algorithm sha224_algorithm = {
+	.name		= "sha224",
+	.ctxsize	= sizeof ( struct sha256_context ),
+	.blocksize	= sizeof ( union sha256_block ),
+	.digestsize	= SHA224_DIGEST_SIZE,
+	.init		= sha224_init,
+	.update		= sha256_update,
+	.final		= sha256_final,
+};
+
+/** "sha224" object identifier */
+static uint8_t oid_sha224[] = { ASN1_OID_SHA224 };
+
+/** "sha224" OID-identified algorithm */
+struct asn1_algorithm oid_sha224_algorithm __asn1_algorithm = {
+	.name = "sha224",
+	.digest = &sha224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha224 ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/sha256.c b/qemu/roms/ipxe/src/crypto/sha256.c
index 36e02b3c2..0360d8d16 100644
--- a/qemu/roms/ipxe/src/crypto/sha256.c
+++ b/qemu/roms/ipxe/src/crypto/sha256.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -47,11 +51,11 @@ struct sha256_variables {
 	uint32_t f;
 	uint32_t g;
 	uint32_t h;
-	uint32_t w[64];
+	uint32_t w[SHA256_ROUNDS];
 } __attribute__ (( packed ));
 
 /** SHA-256 constants */
-static const uint32_t k[64] = {
+static const uint32_t k[SHA256_ROUNDS] = {
 	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1,
 	0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
 	0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
@@ -65,6 +69,37 @@ static const uint32_t k[64] = {
 	0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
 };
 
+/** SHA-256 initial digest values */
+static const struct sha256_digest sha256_init_digest = {
+	.h = {
+		cpu_to_be32 ( 0x6a09e667 ),
+		cpu_to_be32 ( 0xbb67ae85 ),
+		cpu_to_be32 ( 0x3c6ef372 ),
+		cpu_to_be32 ( 0xa54ff53a ),
+		cpu_to_be32 ( 0x510e527f ),
+		cpu_to_be32 ( 0x9b05688c ),
+		cpu_to_be32 ( 0x1f83d9ab ),
+		cpu_to_be32 ( 0x5be0cd19 ),
+	},
+};
+
+/**
+ * Initialise SHA-256 family algorithm
+ *
+ * @v context		SHA-256 context
+ * @v init		Initial digest values
+ * @v digestsize	Digest size
+ */
+void sha256_family_init ( struct sha256_context *context,
+			  const struct sha256_digest *init,
+			  size_t digestsize ) {
+
+	context->len = 0;
+	context->digestsize = digestsize;
+	memcpy ( &context->ddd.dd.digest, init,
+		 sizeof ( context->ddd.dd.digest ) );
+}
+
 /**
  * Initialise SHA-256 algorithm
  *
@@ -73,15 +108,8 @@ static const uint32_t k[64] = {
 static void sha256_init ( void *ctx ) {
 	struct sha256_context *context = ctx;
 
-	context->ddd.dd.digest.h[0] = cpu_to_be32 ( 0x6a09e667 );
-	context->ddd.dd.digest.h[1] = cpu_to_be32 ( 0xbb67ae85 );
-	context->ddd.dd.digest.h[2] = cpu_to_be32 ( 0x3c6ef372 );
-	context->ddd.dd.digest.h[3] = cpu_to_be32 ( 0xa54ff53a );
-	context->ddd.dd.digest.h[4] = cpu_to_be32 ( 0x510e527f );
-	context->ddd.dd.digest.h[5] = cpu_to_be32 ( 0x9b05688c );
-	context->ddd.dd.digest.h[6] = cpu_to_be32 ( 0x1f83d9ab );
-	context->ddd.dd.digest.h[7] = cpu_to_be32 ( 0x5be0cd19 );
-	context->len = 0;
+	sha256_family_init ( context, &sha256_init_digest,
+			     sizeof ( struct sha256_digest ) );
 }
 
 /**
@@ -139,7 +167,7 @@ static void sha256_digest ( struct sha256_context *context ) {
 	}
 
 	/* Initialise w[16..63] */
-	for ( i = 16 ; i < 64 ; i++ ) {
+	for ( i = 16 ; i < SHA256_ROUNDS ; i++ ) {
 		s0 = ( ror32 ( w[i-15], 7 ) ^ ror32 ( w[i-15], 18 ) ^
 		       ( w[i-15] >> 3 ) );
 		s1 = ( ror32 ( w[i-2], 17 ) ^ ror32 ( w[i-2], 19 ) ^
@@ -148,7 +176,7 @@ static void sha256_digest ( struct sha256_context *context ) {
 	}
 
 	/* Main loop */
-	for ( i = 0 ; i < 64 ; i++ ) {
+	for ( i = 0 ; i < SHA256_ROUNDS ; i++ ) {
 		s0 = ( ror32 ( *a, 2 ) ^ ror32 ( *a, 13 ) ^ ror32 ( *a, 22 ) );
 		maj = ( ( *a & *b ) ^ ( *a & *c ) ^ ( *b & *c ) );
 		t2 = ( s0 + maj );
@@ -186,7 +214,7 @@ static void sha256_digest ( struct sha256_context *context ) {
  * @v data		Data
  * @v len		Length of data
  */
-static void sha256_update ( void *ctx, const void *data, size_t len ) {
+void sha256_update ( void *ctx, const void *data, size_t len ) {
 	struct sha256_context *context = ctx;
 	const uint8_t *byte = data;
 	size_t offset;
@@ -209,7 +237,7 @@ static void sha256_update ( void *ctx, const void *data, size_t len ) {
  * @v ctx		SHA-256 context
  * @v out		Output buffer
  */
-static void sha256_final ( void *ctx, void *out ) {
+void sha256_final ( void *ctx, void *out ) {
 	struct sha256_context *context = ctx;
 	uint64_t len_bits;
 	uint8_t pad;
@@ -230,8 +258,7 @@ static void sha256_final ( void *ctx, void *out ) {
 	assert ( ( context->len % sizeof ( context->ddd.dd.data ) ) == 0 );
 
 	/* Copy out final digest */
-	memcpy ( out, &context->ddd.dd.digest,
-		 sizeof ( context->ddd.dd.digest ) );
+	memcpy ( out, &context->ddd.dd.digest, context->digestsize );
 }
 
 /** SHA-256 algorithm */
diff --git a/qemu/roms/ipxe/src/crypto/sha384.c b/qemu/roms/ipxe/src/crypto/sha384.c
new file mode 100644
index 000000000..017751826
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/sha384.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-384 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-384 initial digest values */
+static const struct sha512_digest sha384_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0xcbbb9d5dc1059ed8ULL ),
+		cpu_to_be64 ( 0x629a292a367cd507ULL ),
+		cpu_to_be64 ( 0x9159015a3070dd17ULL ),
+		cpu_to_be64 ( 0x152fecd8f70e5939ULL ),
+		cpu_to_be64 ( 0x67332667ffc00b31ULL ),
+		cpu_to_be64 ( 0x8eb44a8768581511ULL ),
+		cpu_to_be64 ( 0xdb0c2e0d64f98fa7ULL ),
+		cpu_to_be64 ( 0x47b5481dbefa4fa4ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-384 algorithm
+ *
+ * @v ctx		SHA-384 context
+ */
+static void sha384_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha384_init_digest, SHA384_DIGEST_SIZE );
+}
+
+/** SHA-384 algorithm */
+struct digest_algorithm sha384_algorithm = {
+	.name		= "sha384",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA384_DIGEST_SIZE,
+	.init		= sha384_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha384" object identifier */
+static uint8_t oid_sha384[] = { ASN1_OID_SHA384 };
+
+/** "sha384" OID-identified algorithm */
+struct asn1_algorithm oid_sha384_algorithm __asn1_algorithm = {
+	.name = "sha384",
+	.digest = &sha384_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha384 ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/sha512.c b/qemu/roms/ipxe/src/crypto/sha512.c
new file mode 100644
index 000000000..814f44563
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/sha512.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <byteswap.h>
+#include <assert.h>
+#include <ipxe/rotate.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512 variables */
+struct sha512_variables {
+	/* This layout matches that of struct sha512_digest_data,
+	 * allowing for efficient endianness-conversion,
+	 */
+	uint64_t a;
+	uint64_t b;
+	uint64_t c;
+	uint64_t d;
+	uint64_t e;
+	uint64_t f;
+	uint64_t g;
+	uint64_t h;
+	uint64_t w[SHA512_ROUNDS];
+} __attribute__ (( packed ));
+
+/** SHA-512 constants */
+static const uint64_t k[SHA512_ROUNDS] = {
+	0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
+	0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
+	0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL,
+	0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
+	0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
+	0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
+	0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL,
+	0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
+	0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL,
+	0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
+	0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
+	0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
+	0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL,
+	0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
+	0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
+	0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
+	0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL,
+	0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
+	0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL,
+	0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
+	0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
+	0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
+	0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL,
+	0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
+	0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
+	0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
+	0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL
+};
+
+/** SHA-512 initial digest values */
+static const struct sha512_digest sha512_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x6a09e667f3bcc908ULL ),
+		cpu_to_be64 ( 0xbb67ae8584caa73bULL ),
+		cpu_to_be64 ( 0x3c6ef372fe94f82bULL ),
+		cpu_to_be64 ( 0xa54ff53a5f1d36f1ULL ),
+		cpu_to_be64 ( 0x510e527fade682d1ULL ),
+		cpu_to_be64 ( 0x9b05688c2b3e6c1fULL ),
+		cpu_to_be64 ( 0x1f83d9abfb41bd6bULL ),
+		cpu_to_be64 ( 0x5be0cd19137e2179ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512 family algorithm
+ *
+ * @v context		SHA-512 context
+ * @v init		Initial digest values
+ * @v digestsize	Digest size
+ */
+void sha512_family_init ( struct sha512_context *context,
+			  const struct sha512_digest *init,
+			  size_t digestsize ) {
+
+	context->len = 0;
+	context->digestsize = digestsize;
+	memcpy ( &context->ddq.dd.digest, init,
+		 sizeof ( context->ddq.dd.digest ) );
+}
+
+/**
+ * Initialise SHA-512 algorithm
+ *
+ * @v ctx		SHA-512 context
+ */
+static void sha512_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_init_digest,
+			     sizeof ( struct sha512_digest ) );
+}
+
+/**
+ * Calculate SHA-512 digest of accumulated data
+ *
+ * @v context		SHA-512 context
+ */
+static void sha512_digest ( struct sha512_context *context ) {
+        union {
+		union sha512_digest_data_qwords ddq;
+		struct sha512_variables v;
+	} u;
+	uint64_t *a = &u.v.a;
+	uint64_t *b = &u.v.b;
+	uint64_t *c = &u.v.c;
+	uint64_t *d = &u.v.d;
+	uint64_t *e = &u.v.e;
+	uint64_t *f = &u.v.f;
+	uint64_t *g = &u.v.g;
+	uint64_t *h = &u.v.h;
+	uint64_t *w = u.v.w;
+	uint64_t s0;
+	uint64_t s1;
+	uint64_t maj;
+	uint64_t t1;
+	uint64_t t2;
+	uint64_t ch;
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
+	linker_assert ( &u.ddq.dd.digest.h[0] == a, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[1] == b, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[2] == c, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[3] == d, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[4] == e, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[5] == f, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[6] == g, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.digest.h[7] == h, sha512_bad_layout );
+	linker_assert ( &u.ddq.dd.data.qword[0] == w, sha512_bad_layout );
+
+	DBGC ( context, "SHA512 digesting:\n" );
+	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
+		   sizeof ( context->ddq.dd.digest ) );
+	DBGC_HDA ( context, context->len, &context->ddq.dd.data,
+		   sizeof ( context->ddq.dd.data ) );
+
+	/* Convert h[0..7] to host-endian, and initialise a, b, c, d,
+	 * e, f, g, h, and w[0..15]
+	 */
+	for ( i = 0 ; i < ( sizeof ( u.ddq.qword ) /
+			    sizeof ( u.ddq.qword[0] ) ) ; i++ ) {
+		be64_to_cpus ( &context->ddq.qword[i] );
+		u.ddq.qword[i] = context->ddq.qword[i];
+	}
+
+	/* Initialise w[16..79] */
+	for ( i = 16 ; i < SHA512_ROUNDS ; i++ ) {
+		s0 = ( ror64 ( w[i-15], 1 ) ^ ror64 ( w[i-15], 8 ) ^
+		       ( w[i-15] >> 7 ) );
+		s1 = ( ror64 ( w[i-2], 19 ) ^ ror64 ( w[i-2], 61 ) ^
+		       ( w[i-2] >> 6 ) );
+		w[i] = ( w[i-16] + s0 + w[i-7] + s1 );
+	}
+
+	/* Main loop */
+	for ( i = 0 ; i < SHA512_ROUNDS ; i++ ) {
+		s0 = ( ror64 ( *a, 28 ) ^ ror64 ( *a, 34 ) ^ ror64 ( *a, 39 ) );
+		maj = ( ( *a & *b ) ^ ( *a & *c ) ^ ( *b & *c ) );
+		t2 = ( s0 + maj );
+		s1 = ( ror64 ( *e, 14 ) ^ ror64 ( *e, 18 ) ^ ror64 ( *e, 41 ) );
+		ch = ( ( *e & *f ) ^ ( (~*e) & *g ) );
+		t1 = ( *h + s1 + ch + k[i] + w[i] );
+		*h = *g;
+		*g = *f;
+		*f = *e;
+		*e = ( *d + t1 );
+		*d = *c;
+		*c = *b;
+		*b = *a;
+		*a = ( t1 + t2 );
+		DBGC2 ( context, "%2d : %016llx %016llx %016llx %016llx "
+			"%016llx %016llx %016llx %016llx\n",
+			i, *a, *b, *c, *d, *e, *f, *g, *h );
+	}
+
+	/* Add chunk to hash and convert back to big-endian */
+	for ( i = 0 ; i < 8 ; i++ ) {
+		context->ddq.dd.digest.h[i] =
+			cpu_to_be64 ( context->ddq.dd.digest.h[i] +
+				      u.ddq.dd.digest.h[i] );
+	}
+
+	DBGC ( context, "SHA512 digested:\n" );
+	DBGC_HDA ( context, 0, &context->ddq.dd.digest,
+		   sizeof ( context->ddq.dd.digest ) );
+}
+
+/**
+ * Accumulate data with SHA-512 algorithm
+ *
+ * @v ctx		SHA-512 context
+ * @v data		Data
+ * @v len		Length of data
+ */
+void sha512_update ( void *ctx, const void *data, size_t len ) {
+	struct sha512_context *context = ctx;
+	const uint8_t *byte = data;
+	size_t offset;
+
+	/* Accumulate data a byte at a time, performing the digest
+	 * whenever we fill the data buffer
+	 */
+	while ( len-- ) {
+		offset = ( context->len % sizeof ( context->ddq.dd.data ) );
+		context->ddq.dd.data.byte[offset] = *(byte++);
+		context->len++;
+		if ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 )
+			sha512_digest ( context );
+	}
+}
+
+/**
+ * Generate SHA-512 digest
+ *
+ * @v ctx		SHA-512 context
+ * @v out		Output buffer
+ */
+void sha512_final ( void *ctx, void *out ) {
+	struct sha512_context *context = ctx;
+	uint64_t len_bits_hi;
+	uint64_t len_bits_lo;
+	uint8_t pad;
+
+	/* Record length before pre-processing */
+	len_bits_hi = 0;
+	len_bits_lo = cpu_to_be64 ( ( ( uint64_t ) context->len ) * 8 );
+
+	/* Pad with a single "1" bit followed by as many "0" bits as required */
+	pad = 0x80;
+	do {
+		sha512_update ( ctx, &pad, sizeof ( pad ) );
+		pad = 0x00;
+	} while ( ( context->len % sizeof ( context->ddq.dd.data ) ) !=
+		  offsetof ( typeof ( context->ddq.dd.data ), final.len_hi ) );
+
+	/* Append length (in bits) */
+	sha512_update ( ctx, &len_bits_hi, sizeof ( len_bits_hi ) );
+	sha512_update ( ctx, &len_bits_lo, sizeof ( len_bits_lo ) );
+	assert ( ( context->len % sizeof ( context->ddq.dd.data ) ) == 0 );
+
+	/* Copy out final digest */
+	memcpy ( out, &context->ddq.dd.digest, context->digestsize );
+}
+
+/** SHA-512 algorithm */
+struct digest_algorithm sha512_algorithm = {
+	.name		= "sha512",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= sizeof ( struct sha512_digest ),
+	.init		= sha512_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512" object identifier */
+static uint8_t oid_sha512[] = { ASN1_OID_SHA512 };
+
+/** "sha512" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_algorithm __asn1_algorithm = {
+	.name = "sha512",
+	.digest = &sha512_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512 ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/sha512_224.c b/qemu/roms/ipxe/src/crypto/sha512_224.c
new file mode 100644
index 000000000..8c37b566b
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/sha512_224.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512/224 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512/224 initial digest values */
+static const struct sha512_digest sha512_224_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x8c3d37c819544da2ULL ),
+		cpu_to_be64 ( 0x73e1996689dcd4d6ULL ),
+		cpu_to_be64 ( 0x1dfab7ae32ff9c82ULL ),
+		cpu_to_be64 ( 0x679dd514582f9fcfULL ),
+		cpu_to_be64 ( 0x0f6d2b697bd44da8ULL ),
+		cpu_to_be64 ( 0x77e36f7304c48942ULL ),
+		cpu_to_be64 ( 0x3f9d85a86a1d36c8ULL ),
+		cpu_to_be64 ( 0x1112e6ad91d692a1ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512/224 algorithm
+ *
+ * @v ctx		SHA-512/224 context
+ */
+static void sha512_224_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_224_init_digest,
+			     SHA512_224_DIGEST_SIZE );
+}
+
+/** SHA-512/224 algorithm */
+struct digest_algorithm sha512_224_algorithm = {
+	.name		= "sha512/224",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA512_224_DIGEST_SIZE,
+	.init		= sha512_224_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512_224" object identifier */
+static uint8_t oid_sha512_224[] = { ASN1_OID_SHA512_224 };
+
+/** "sha512_224" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_224_algorithm __asn1_algorithm = {
+	.name = "sha512/224",
+	.digest = &sha512_224_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_224 ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/sha512_256.c b/qemu/roms/ipxe/src/crypto/sha512_256.c
new file mode 100644
index 000000000..f8afaf3e3
--- /dev/null
+++ b/qemu/roms/ipxe/src/crypto/sha512_256.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512/256 algorithm
+ *
+ */
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/crypto.h>
+#include <ipxe/asn1.h>
+#include <ipxe/sha512.h>
+
+/** SHA-512/256 initial digest values */
+static const struct sha512_digest sha512_256_init_digest = {
+	.h = {
+		cpu_to_be64 ( 0x22312194fc2bf72cULL ),
+		cpu_to_be64 ( 0x9f555fa3c84c64c2ULL ),
+		cpu_to_be64 ( 0x2393b86b6f53b151ULL ),
+		cpu_to_be64 ( 0x963877195940eabdULL ),
+		cpu_to_be64 ( 0x96283ee2a88effe3ULL ),
+		cpu_to_be64 ( 0xbe5e1e2553863992ULL ),
+		cpu_to_be64 ( 0x2b0199fc2c85b8aaULL ),
+		cpu_to_be64 ( 0x0eb72ddc81c52ca2ULL ),
+	},
+};
+
+/**
+ * Initialise SHA-512/256 algorithm
+ *
+ * @v ctx		SHA-512/256 context
+ */
+static void sha512_256_init ( void *ctx ) {
+	struct sha512_context *context = ctx;
+
+	sha512_family_init ( context, &sha512_256_init_digest,
+			     SHA512_256_DIGEST_SIZE );
+}
+
+/** SHA-512/256 algorithm */
+struct digest_algorithm sha512_256_algorithm = {
+	.name		= "sha512/256",
+	.ctxsize	= sizeof ( struct sha512_context ),
+	.blocksize	= sizeof ( union sha512_block ),
+	.digestsize	= SHA512_256_DIGEST_SIZE,
+	.init		= sha512_256_init,
+	.update		= sha512_update,
+	.final		= sha512_final,
+};
+
+/** "sha512_256" object identifier */
+static uint8_t oid_sha512_256[] = { ASN1_OID_SHA512_256 };
+
+/** "sha512_256" OID-identified algorithm */
+struct asn1_algorithm oid_sha512_256_algorithm __asn1_algorithm = {
+	.name = "sha512/256",
+	.digest = &sha512_256_algorithm,
+	.oid = ASN1_OID_CURSOR ( oid_sha512_256 ),
+};
diff --git a/qemu/roms/ipxe/src/crypto/x509.c b/qemu/roms/ipxe/src/crypto/x509.c
index 4a02dad14..43a4ca17a 100644
--- a/qemu/roms/ipxe/src/crypto/x509.c
+++ b/qemu/roms/ipxe/src/crypto/x509.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
@@ -139,7 +143,8 @@ const char * x509_name ( struct x509_certificate *cert ) {
 	} else {
 		/* Certificate has no commonName: use SHA-1 fingerprint */
 		x509_fingerprint ( cert, digest, fingerprint );
-		base16_encode ( fingerprint, sizeof ( fingerprint ), buf );
+		base16_encode ( fingerprint, sizeof ( fingerprint ),
+				buf, sizeof ( buf ) );
 	}
 	return buf;
 }
@@ -1761,5 +1766,11 @@ int x509_validate_chain ( struct x509_chain *chain, time_t time,
 	return -EACCES_USELESS;
 }
 
+/* Drag in objects via x509_validate() */
+REQUIRING_SYMBOL ( x509_validate );
+
 /* Drag in certificate store */
 REQUIRE_OBJECT ( certstore );
+
+/* Drag in crypto configuration */
+REQUIRE_OBJECT ( config_crypto );
diff --git a/qemu/roms/ipxe/src/drivers/bitbash/bitbash.c b/qemu/roms/ipxe/src/drivers/bitbash/bitbash.c
index 23ca30356..9b24f716c 100644
--- a/qemu/roms/ipxe/src/drivers/bitbash/bitbash.c
+++ b/qemu/roms/ipxe/src/drivers/bitbash/bitbash.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/bitbash.h>
 
diff --git a/qemu/roms/ipxe/src/drivers/bitbash/i2c_bit.c b/qemu/roms/ipxe/src/drivers/bitbash/i2c_bit.c
index decc8d80e..707d9447d 100644
--- a/qemu/roms/ipxe/src/drivers/bitbash/i2c_bit.c
+++ b/qemu/roms/ipxe/src/drivers/bitbash/i2c_bit.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/drivers/bitbash/spi_bit.c b/qemu/roms/ipxe/src/drivers/bitbash/spi_bit.c
index 1b39d72fa..04fddc20b 100644
--- a/qemu/roms/ipxe/src/drivers/bitbash/spi_bit.c
+++ b/qemu/roms/ipxe/src/drivers/bitbash/spi_bit.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/drivers/block/ata.c b/qemu/roms/ipxe/src/drivers/block/ata.c
index c9b87c20c..b1c6855a0 100644
--- a/qemu/roms/ipxe/src/drivers/block/ata.c
+++ b/qemu/roms/ipxe/src/drivers/block/ata.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/block/scsi.c b/qemu/roms/ipxe/src/drivers/block/scsi.c
index 64d692986..fd5f82b9f 100644
--- a/qemu/roms/ipxe/src/drivers/block/scsi.c
+++ b/qemu/roms/ipxe/src/drivers/block/scsi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/bus/cdc.c b/qemu/roms/ipxe/src/drivers/bus/cdc.c
new file mode 100644
index 000000000..373a03072
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/bus/cdc.c
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+
+/** @file
+ *
+ * USB Communications Device Class (CDC)
+ *
+ */
+
+/**
+ * Locate CDC union functional descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @ret desc		Union functional descriptor, or NULL if not found
+ */
+struct cdc_union_descriptor *
+cdc_union_descriptor ( struct usb_configuration_descriptor *config,
+		       struct usb_interface_descriptor *interface ) {
+	struct cdc_union_descriptor *desc;
+
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_CS_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->subtype == CDC_SUBTYPE_UNION ) )
+			return desc;
+	}
+	return NULL;
+}
diff --git a/qemu/roms/ipxe/src/drivers/bus/pci.c b/qemu/roms/ipxe/src/drivers/bus/pci.c
index 4a8d00b54..6fbedd940 100644
--- a/qemu/roms/ipxe/src/drivers/bus/pci.c
+++ b/qemu/roms/ipxe/src/drivers/bus/pci.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -58,8 +62,8 @@ static unsigned long pci_bar ( struct pci_device *pci, unsigned int reg ) {
 	uint32_t high;
 
 	pci_read_config_dword ( pci, reg, &low );
-	if ( ( low & (PCI_BASE_ADDRESS_SPACE|PCI_BASE_ADDRESS_MEM_TYPE_MASK) )
-	     == (PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64) ){
+	if ( ( low & (PCI_BASE_ADDRESS_SPACE_IO|PCI_BASE_ADDRESS_MEM_TYPE_MASK))
+	     == PCI_BASE_ADDRESS_MEM_TYPE_64 ) {
 		pci_read_config_dword ( pci, reg + 4, &high );
 		if ( high ) {
 			if ( sizeof ( unsigned long ) > sizeof ( uint32_t ) ) {
@@ -93,10 +97,10 @@ unsigned long pci_bar_start ( struct pci_device *pci, unsigned int reg ) {
 	unsigned long bar;
 
 	bar = pci_bar ( pci, reg );
-	if ( (bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_MEMORY ){
-		return ( bar & PCI_BASE_ADDRESS_MEM_MASK );
+	if ( bar & PCI_BASE_ADDRESS_SPACE_IO ) {
+		return ( bar & ~PCI_BASE_ADDRESS_IO_MASK );
 	} else {
-		return ( bar & PCI_BASE_ADDRESS_IO_MASK );
+		return ( bar & ~PCI_BASE_ADDRESS_MEM_MASK );
 	}
 }
 
@@ -122,11 +126,11 @@ static void pci_read_bases ( struct pci_device *pci ) {
 		if ( bar & PCI_BASE_ADDRESS_SPACE_IO ) {
 			if ( ! pci->ioaddr )
 				pci->ioaddr = 
-					( bar & PCI_BASE_ADDRESS_IO_MASK );
+					( bar & ~PCI_BASE_ADDRESS_IO_MASK );
 		} else {
 			if ( ! pci->membase )
 				pci->membase =
-					( bar & PCI_BASE_ADDRESS_MEM_MASK );
+					( bar & ~PCI_BASE_ADDRESS_MEM_MASK );
 			/* Skip next BAR if 64-bit */
 			if ( bar & PCI_BASE_ADDRESS_MEM_TYPE_64 )
 				reg += 4;
@@ -181,7 +185,7 @@ int pci_read_config ( struct pci_device *pci ) {
 		pci->busdevfn = PCI_FIRST_FUNC ( pci->busdevfn );
 		pci_read_config_byte ( pci, PCI_HEADER_TYPE, &hdrtype );
 		pci->busdevfn = busdevfn;
-		if ( ! ( hdrtype & 0x80 ) )
+		if ( ! ( hdrtype & PCI_HEADER_TYPE_MULTI ) )
 			return -ENODEV;
 	}
 
@@ -253,6 +257,8 @@ int pci_find_driver ( struct pci_device *pci ) {
 	unsigned int i;
 
 	for_each_table_entry ( driver, PCI_DRIVERS ) {
+		if ( ( driver->class.class ^ pci->class ) & driver->class.mask )
+			continue;
 		for ( i = 0 ; i < driver->id_count ; i++ ) {
 			id = &driver->ids[i];
 			if ( ( id->vendor != PCI_ANY_ID ) &&
@@ -334,14 +340,15 @@ static int pcibus_probe ( struct root_device *rootdev ) {
 
 		/* Look for a driver */
 		if ( ( rc = pci_find_driver ( pci ) ) != 0 ) {
-			DBGC ( pci, PCI_FMT " (%04x:%04x) has no driver\n",
-			       PCI_ARGS ( pci ), pci->vendor, pci->device );
+			DBGC ( pci, PCI_FMT " (%04x:%04x class %06x) has no "
+			       "driver\n", PCI_ARGS ( pci ), pci->vendor,
+			       pci->device, pci->class );
 			continue;
 		}
 
 		/* Add to device hierarchy */
 		pci->dev.parent = &rootdev->dev;
-		list_add ( &pci->dev.siblings, &rootdev->dev.children);
+		list_add ( &pci->dev.siblings, &rootdev->dev.children );
 
 		/* Look for a driver */
 		if ( ( rc = pci_probe ( pci ) ) == 0 ) {
diff --git a/qemu/roms/ipxe/src/drivers/bus/pci_settings.c b/qemu/roms/ipxe/src/drivers/bus/pci_settings.c
index db20452e0..1cb9fa5a3 100644
--- a/qemu/roms/ipxe/src/drivers/bus/pci_settings.c
+++ b/qemu/roms/ipxe/src/drivers/bus/pci_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/drivers/bus/pcibackup.c b/qemu/roms/ipxe/src/drivers/bus/pcibackup.c
index 6b592e893..fecad8192 100644
--- a/qemu/roms/ipxe/src/drivers/bus/pcibackup.c
+++ b/qemu/roms/ipxe/src/drivers/bus/pcibackup.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
diff --git a/qemu/roms/ipxe/src/drivers/bus/pciextra.c b/qemu/roms/ipxe/src/drivers/bus/pciextra.c
index c4417e0cb..82287fb86 100644
--- a/qemu/roms/ipxe/src/drivers/bus/pciextra.c
+++ b/qemu/roms/ipxe/src/drivers/bus/pciextra.c
@@ -1,4 +1,4 @@
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/pci.h>
@@ -26,7 +26,7 @@ int pci_find_capability ( struct pci_device *pci, int cap ) {
 		return 0;
 
 	pci_read_config_byte ( pci, PCI_HEADER_TYPE, &hdr_type );
-	switch ( hdr_type & 0x7F ) {
+	switch ( hdr_type & PCI_HEADER_TYPE_MASK ) {
 	case PCI_HEADER_TYPE_NORMAL:
 	case PCI_HEADER_TYPE_BRIDGE:
 	default:
@@ -38,13 +38,13 @@ int pci_find_capability ( struct pci_device *pci, int cap ) {
 	}
 	while ( ttl-- && pos >= 0x40 ) {
 		pos &= ~3;
-		pci_read_config_byte ( pci, pos + PCI_CAP_LIST_ID, &id );
+		pci_read_config_byte ( pci, pos + PCI_CAP_ID, &id );
 		DBG ( "PCI Capability: %d\n", id );
 		if ( id == 0xff )
 			break;
 		if ( id == cap )
 			return pos;
-		pci_read_config_byte ( pci, pos + PCI_CAP_LIST_NEXT, &pos );
+		pci_read_config_byte ( pci, pos + PCI_CAP_NEXT, &pos );
 	}
 	return 0;
 }
@@ -76,9 +76,9 @@ unsigned long pci_bar_size ( struct pci_device *pci, unsigned int reg ) {
 	/* Restore the original command register. This reenables decoding. */
 	pci_write_config_word ( pci, PCI_COMMAND, cmd );
 	if ( start & PCI_BASE_ADDRESS_SPACE_IO ) {
-		size &= PCI_BASE_ADDRESS_IO_MASK;
+		size &= ~PCI_BASE_ADDRESS_IO_MASK;
 	} else {
-		size &= PCI_BASE_ADDRESS_MEM_MASK;
+		size &= ~PCI_BASE_ADDRESS_MEM_MASK;
 	}
 	/* Find the lowest bit set */
 	size = size & ~( size - 1 );
diff --git a/qemu/roms/ipxe/src/drivers/bus/pcivpd.c b/qemu/roms/ipxe/src/drivers/bus/pcivpd.c
index 0b7a879fe..243b1f779 100644
--- a/qemu/roms/ipxe/src/drivers/bus/pcivpd.c
+++ b/qemu/roms/ipxe/src/drivers/bus/pcivpd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/bus/usb.c b/qemu/roms/ipxe/src/drivers/bus/usb.c
new file mode 100644
index 000000000..2019e3341
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/bus/usb.c
@@ -0,0 +1,2128 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+
+/** @file
+ *
+ * Universal Serial Bus (USB)
+ *
+ */
+
+/** List of USB buses */
+struct list_head usb_buses = LIST_HEAD_INIT ( usb_buses );
+
+/** List of changed ports */
+static struct list_head usb_changed = LIST_HEAD_INIT ( usb_changed );
+
+/** List of halted endpoints */
+static struct list_head usb_halted = LIST_HEAD_INIT ( usb_halted );
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB speed name (for debugging)
+ *
+ * @v speed		Speed
+ * @ret name		Speed name
+ */
+static inline const char * usb_speed_name ( unsigned int speed ) {
+	static const char *exponents[4] = { "", "k", "M", "G" };
+	static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
+	unsigned int mantissa;
+	unsigned int exponent;
+
+	/* Extract mantissa and exponent */
+	mantissa = USB_SPEED_MANTISSA ( speed );
+	exponent = USB_SPEED_EXPONENT ( speed );
+
+	/* Name speed */
+	switch ( speed ) {
+	case USB_SPEED_NONE:		return "DETACHED";
+	case USB_SPEED_LOW:		return "low";
+	case USB_SPEED_FULL:		return "full";
+	case USB_SPEED_HIGH:		return "high";
+	case USB_SPEED_SUPER:		return "super";
+	default:
+		snprintf ( buf, sizeof ( buf ), "%d%sbps",
+			   mantissa, exponents[exponent] );
+		return buf;
+	}
+}
+
+/**
+ * Transcribe USB BCD-coded value (for debugging)
+ *
+ * @v bcd		BCD-coded value
+ * @ret string		Transcribed value
+ */
+static inline const char * usb_bcd ( uint16_t bcd ) {
+	static char buf[ 6 /* "xx.xx" + NUL */ ];
+	uint8_t high = ( bcd >> 8 );
+	uint8_t low = ( bcd >> 0 );
+
+	snprintf ( buf, sizeof ( buf ), "%x.%02x", high, low );
+	return buf;
+}
+
+/******************************************************************************
+ *
+ * USB descriptors
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Locate USB interface association descriptor
+ *
+ * @v config		Configuraton descriptor
+ * @v first		First interface number
+ * @ret desc		Interface association descriptor, or NULL if not found
+ */
+static struct usb_interface_association_descriptor *
+usb_interface_association_descriptor ( struct usb_configuration_descriptor
+								       *config,
+				       unsigned int first ) {
+	struct usb_interface_association_descriptor *desc;
+
+	/* Find a matching interface association descriptor */
+	for_each_config_descriptor ( desc, config ) {
+		if ( ( desc->header.type ==
+		       USB_INTERFACE_ASSOCIATION_DESCRIPTOR ) &&
+		     ( desc->first == first ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB interface descriptor
+ *
+ * @v config		Configuraton descriptor
+ * @v interface		Interface number
+ * @v alternate		Alternate setting
+ * @ret desc		Interface descriptor, or NULL if not found
+ */
+struct usb_interface_descriptor *
+usb_interface_descriptor ( struct usb_configuration_descriptor *config,
+			   unsigned int interface, unsigned int alternate ) {
+	struct usb_interface_descriptor *desc;
+
+	/* Find a matching interface descriptor */
+	for_each_config_descriptor ( desc, config ) {
+		if ( ( desc->header.type == USB_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->interface == interface ) &&
+		     ( desc->alternate == alternate ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB endpoint descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @v type		Endpoint (internal) type
+ * @v index		Endpoint index
+ * @ret desc		Descriptor, or NULL if not found
+ */
+struct usb_endpoint_descriptor *
+usb_endpoint_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface,
+			  unsigned int type, unsigned int index ) {
+	struct usb_endpoint_descriptor *desc;
+	unsigned int attributes = ( type & USB_ENDPOINT_ATTR_TYPE_MASK );
+	unsigned int direction = ( type & USB_DIR_IN );
+
+	/* Find a matching endpoint descriptor */
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_ENDPOINT_DESCRIPTOR ) &&
+		     ( ( desc->attributes &
+			 USB_ENDPOINT_ATTR_TYPE_MASK ) == attributes ) &&
+		     ( ( desc->endpoint & USB_DIR_IN ) == direction ) &&
+		     ( index-- == 0 ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Locate USB endpoint companion descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v desc		Endpoint descriptor
+ * @ret descx		Companion descriptor, or NULL if not found
+ */
+struct usb_endpoint_companion_descriptor *
+usb_endpoint_companion_descriptor ( struct usb_configuration_descriptor *config,
+				    struct usb_endpoint_descriptor *desc ) {
+	struct usb_endpoint_companion_descriptor *descx;
+
+	/* Get companion descriptor, if present */
+	descx = container_of ( usb_next_descriptor ( &desc->header ),
+			       struct usb_endpoint_companion_descriptor,
+			       header );
+	return ( ( usb_is_within_config ( config, &descx->header ) &&
+		   descx->header.type == USB_ENDPOINT_COMPANION_DESCRIPTOR )
+		 ? descx : NULL );
+}
+
+/******************************************************************************
+ *
+ * USB endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB endpoint name (for debugging)
+ *
+ * @v ep		USB endpoint
+ * @ret name		Endpoint name
+ */
+const char * usb_endpoint_name ( struct usb_endpoint *ep ) {
+	static char buf[ 9 /* "EPxx OUT" + NUL */ ];
+	unsigned int address = ep->address;
+
+	snprintf ( buf, sizeof ( buf ), "EP%d%s",
+		   ( address & USB_ENDPOINT_MAX ),
+		   ( address ?
+		     ( ( address & USB_ENDPOINT_IN ) ? " IN" : " OUT" ) : "" ));
+	return buf;
+}
+
+/**
+ * Describe USB endpoint from device configuration
+ *
+ * @v ep		USB endpoint
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @v type		Endpoint (internal) type
+ * @v index		Endpoint index
+ * @ret rc		Return status code
+ */
+int usb_endpoint_described ( struct usb_endpoint *ep,
+			     struct usb_configuration_descriptor *config,
+			     struct usb_interface_descriptor *interface,
+			     unsigned int type, unsigned int index ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	struct usb_endpoint_descriptor *desc;
+	struct usb_endpoint_companion_descriptor *descx;
+	unsigned int sizes;
+	unsigned int burst;
+	unsigned int interval;
+	size_t mtu;
+
+	/* Locate endpoint descriptor */
+	desc = usb_endpoint_descriptor ( config, interface, type, index );
+	if ( ! desc )
+		return -ENOENT;
+
+	/* Locate companion descriptor, if any */
+	descx = usb_endpoint_companion_descriptor ( config, desc );
+
+	/* Calculate MTU and burst size */
+	sizes = le16_to_cpu ( desc->sizes );
+	mtu = USB_ENDPOINT_MTU ( sizes );
+	burst = ( descx ? descx->burst : USB_ENDPOINT_BURST ( sizes ) );
+
+	/* Calculate interval */
+	if ( ( type & USB_ENDPOINT_ATTR_TYPE_MASK ) ==
+	     USB_ENDPOINT_ATTR_INTERRUPT ) {
+		if ( port->speed >= USB_SPEED_HIGH ) {
+			/* 2^(desc->interval-1) is a microframe count */
+			interval = ( 1 << ( desc->interval - 1 ) );
+		} else {
+			/* desc->interval is a (whole) frame count */
+			interval = ( desc->interval << 3 );
+		}
+	} else {
+		/* desc->interval is a microframe count */
+		interval = desc->interval;
+	}
+
+	/* Describe endpoint */
+	usb_endpoint_describe ( ep, desc->endpoint, desc->attributes,
+				mtu, burst, interval );
+	return 0;
+}
+
+/**
+ * Open USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int idx = USB_ENDPOINT_IDX ( ep->address );
+	int rc;
+
+	/* Populate host controller operations */
+	ep->host = &usb->port->hub->bus->op->endpoint;
+
+	/* Add to endpoint list */
+	if ( usb->ep[idx] != NULL ) {
+		DBGC ( usb, "USB %s %s is already open\n",
+		       usb->name, usb_endpoint_name ( ep ) );
+		rc = -EALREADY;
+		goto err_already;
+	}
+	usb->ep[idx] = ep;
+	INIT_LIST_HEAD ( &ep->halted );
+
+	/* Open endpoint */
+	if ( ( rc = ep->host->open ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not open: %s\n", usb->name,
+		       usb_endpoint_name ( ep ), strerror ( rc ) );
+		goto err_open;
+	}
+	ep->open = 1;
+
+	DBGC2 ( usb, "USB %s %s opened with MTU %zd, burst %d, interval %d\n",
+		usb->name, usb_endpoint_name ( ep ), ep->mtu, ep->burst,
+		ep->interval );
+	return 0;
+
+	ep->open = 0;
+	ep->host->close ( ep );
+ err_open:
+	usb->ep[idx] = NULL;
+ err_already:
+	if ( ep->max )
+		usb_flush ( ep );
+	return rc;
+}
+
+/**
+ * Clear transaction translator (if applicable)
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_clear_tt ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *tt;
+	int rc;
+
+	/* Do nothing if this is a periodic endpoint */
+	if ( ep->attributes & USB_ENDPOINT_ATTR_PERIODIC )
+		return 0;
+
+	/* Do nothing if this endpoint is not behind a transaction translator */
+	tt = usb_transaction_translator ( usb );
+	if ( ! tt )
+		return 0;
+
+	/* Clear transaction translator buffer */
+	if ( ( rc = tt->hub->driver->clear_tt ( tt->hub, tt, ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not clear transaction translator: "
+		       "%s\n", usb->name, usb_endpoint_name ( ep ),
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Close USB endpoint
+ *
+ * @v ep		USB endpoint
+ */
+void usb_endpoint_close ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int idx = USB_ENDPOINT_IDX ( ep->address );
+
+	/* Sanity checks */
+	assert ( usb->ep[idx] == ep );
+
+	/* Close endpoint */
+	ep->open = 0;
+	ep->host->close ( ep );
+	assert ( ep->fill == 0 );
+
+	/* Remove from endpoint list */
+	usb->ep[idx] = NULL;
+	list_del ( &ep->halted );
+
+	/* Discard any recycled buffers, if applicable */
+	if ( ep->max )
+		usb_flush ( ep );
+
+	/* Clear transaction translator, if applicable */
+	usb_endpoint_clear_tt ( ep );
+}
+
+/**
+ * Reset USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int type;
+	int rc;
+
+	/* Sanity check */
+	assert ( ! list_empty ( &ep->halted ) );
+
+	/* Reset endpoint */
+	if ( ( rc = ep->host->reset ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not reset: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Clear transaction translator, if applicable */
+	if ( ( rc = usb_endpoint_clear_tt ( ep ) ) != 0 )
+		return rc;
+
+	/* Clear endpoint halt, if applicable */
+	type = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	if ( ( type != USB_ENDPOINT_ATTR_CONTROL ) &&
+	     ( ( rc = usb_clear_feature ( usb, USB_RECIP_ENDPOINT,
+					  USB_ENDPOINT_HALT,
+					  ep->address ) ) != 0 ) ) {
+		DBGC ( usb, "USB %s %s could not clear endpoint halt: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Remove from list of halted endpoints */
+	list_del ( &ep->halted );
+	INIT_LIST_HEAD ( &ep->halted );
+
+	DBGC ( usb, "USB %s %s reset\n",
+	       usb->name, usb_endpoint_name ( ep ) );
+	return 0;
+}
+
+/**
+ * Update endpoint MTU
+ *
+ * @v ep		USB endpoint
+ * @v mtu		New MTU
+ * @ret rc		Return status code
+ */
+static int usb_endpoint_mtu ( struct usb_endpoint *ep, size_t mtu ) {
+	struct usb_device *usb = ep->usb;
+	int rc;
+
+	/* Update MTU */
+	ep->mtu = mtu;
+	if ( ( rc = ep->host->mtu ( ep ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not update MTU: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Enqueue USB message transfer
+ *
+ * @v ep		USB endpoint
+ * @v request		Request
+ * @v value		Value parameter
+ * @v index		Index parameter
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * The I/O buffer must have sufficient headroom to contain a setup
+ * packet.
+ */
+int usb_message ( struct usb_endpoint *ep, unsigned int request,
+		  unsigned int value, unsigned int index,
+		  struct io_buffer *iobuf ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	struct usb_setup_packet *packet;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Sanity check */
+	assert ( iob_headroom ( iobuf ) >= sizeof ( *packet ) );
+
+	/* Fail immediately if device has been unplugged */
+	if ( port->speed == USB_SPEED_NONE )
+		return -ENODEV;
+
+	/* Reset endpoint if required */
+	if ( ( ! list_empty ( &ep->halted ) ) &&
+	     ( ( rc = usb_endpoint_reset ( ep ) ) != 0 ) )
+		return rc;
+
+	/* Zero input data buffer (if applicable) */
+	if ( request & USB_DIR_IN )
+		memset ( iobuf->data, 0, len );
+
+	/* Construct setup packet */
+	packet = iob_push ( iobuf, sizeof ( *packet ) );
+	packet->request = cpu_to_le16 ( request );
+	packet->value = cpu_to_le16 ( value );
+	packet->index = cpu_to_le16 ( index );
+	packet->len = cpu_to_le16 ( len );
+
+	/* Enqueue message transfer */
+	if ( ( rc = ep->host->message ( ep, iobuf ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not enqueue message transfer: "
+		       "%s\n", usb->name, usb_endpoint_name ( ep ),
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Increment fill level */
+	ep->fill++;
+
+	return 0;
+}
+
+/**
+ * Enqueue USB stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+int usb_stream ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+		 int terminate ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *port = usb->port;
+	int rc;
+
+	/* Fail immediately if device has been unplugged */
+	if ( port->speed == USB_SPEED_NONE )
+		return -ENODEV;
+
+	/* Reset endpoint if required */
+	if ( ( ! list_empty ( &ep->halted ) ) &&
+	     ( ( rc = usb_endpoint_reset ( ep ) ) != 0 ) )
+		return rc;
+
+	/* Enqueue stream transfer */
+	if ( ( rc = ep->host->stream ( ep, iobuf, terminate ) ) != 0 ) {
+		DBGC ( usb, "USB %s %s could not enqueue stream transfer: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return rc;
+	}
+
+	/* Increment fill level */
+	ep->fill++;
+
+	return 0;
+}
+
+/**
+ * Complete transfer (possibly with error)
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+void usb_complete_err ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			int rc ) {
+	struct usb_device *usb = ep->usb;
+
+	/* Decrement fill level */
+	assert ( ep->fill > 0 );
+	ep->fill--;
+
+	/* Schedule reset, if applicable */
+	if ( ( rc != 0 ) && ep->open ) {
+		DBGC ( usb, "USB %s %s completion failed: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		list_del ( &ep->halted );
+		list_add_tail ( &ep->halted, &usb_halted );
+	}
+
+	/* Report completion */
+	ep->driver->complete ( ep, iobuf, rc );
+}
+
+/******************************************************************************
+ *
+ * Endpoint refilling
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Prefill endpoint recycled buffer list
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_prefill ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	size_t len = ( ep->len ? ep->len : ep->mtu );
+	unsigned int fill;
+	int rc;
+
+	/* Sanity checks */
+	assert ( ep->fill == 0 );
+	assert ( ep->max > 0 );
+	assert ( list_empty ( &ep->recycled ) );
+
+	/* Fill recycled buffer list */
+	for ( fill = 0 ; fill < ep->max ; fill++ ) {
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_iob ( len );
+		if ( ! iobuf ) {
+			rc = -ENOMEM;
+			goto err_alloc;
+		}
+
+		/* Add to recycled buffer list */
+		list_add_tail ( &iobuf->list, &ep->recycled );
+	}
+
+	return 0;
+
+ err_alloc:
+	usb_flush ( ep );
+	return rc;
+}
+
+/**
+ * Refill endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+int usb_refill ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	size_t len = ( ep->len ? ep->len : ep->mtu );
+	int rc;
+
+	/* Sanity checks */
+	assert ( ep->open );
+	assert ( ep->max > 0 );
+
+	/* Refill endpoint */
+	while ( ep->fill < ep->max ) {
+
+		/* Get or allocate buffer */
+		if ( list_empty ( &ep->recycled ) ) {
+			/* Recycled buffer list is empty; allocate new buffer */
+			iobuf = alloc_iob ( len );
+			if ( ! iobuf )
+				return -ENOMEM;
+		} else {
+			/* Get buffer from recycled buffer list */
+			iobuf = list_first_entry ( &ep->recycled,
+						   struct io_buffer, list );
+			assert ( iobuf != NULL );
+			list_del ( &iobuf->list );
+		}
+
+		/* Reset buffer to maximum size */
+		assert ( iob_len ( iobuf ) <= len );
+		iob_put ( iobuf, ( len - iob_len ( iobuf ) ) );
+
+		/* Enqueue buffer */
+		if ( ( rc = usb_stream ( ep, iobuf, 0 ) ) != 0 ) {
+			list_add ( &iobuf->list, &ep->recycled );
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Discard endpoint recycled buffer list
+ *
+ * @v ep		USB endpoint
+ */
+void usb_flush ( struct usb_endpoint *ep ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Sanity checks */
+	assert ( ! ep->open );
+	assert ( ep->max > 0 );
+
+	/* Free all I/O buffers */
+	list_for_each_entry_safe ( iobuf, tmp, &ep->recycled, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Control endpoint
+ *
+ ******************************************************************************
+ */
+
+/** USB control transfer pseudo-header */
+struct usb_control_pseudo_header {
+	/** Completion status */
+	int rc;
+};
+
+/**
+ * Complete USB control transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usb_control_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_control_pseudo_header *pshdr;
+
+	/* Record completion status in buffer */
+	pshdr = iob_push ( iobuf, sizeof ( *pshdr ) );
+	pshdr->rc = rc;
+
+	/* Add to list of completed I/O buffers */
+	list_add_tail ( &iobuf->list, &usb->complete );
+}
+
+/** USB control endpoint driver operations */
+static struct usb_endpoint_driver_operations usb_control_operations = {
+	.complete = usb_control_complete,
+};
+
+/**
+ * Issue USB control transaction
+ *
+ * @v usb		USB device
+ * @v request		Request
+ * @v value		Value parameter
+ * @v index		Index parameter
+ * @v data		Data buffer (if any)
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+int usb_control ( struct usb_device *usb, unsigned int request,
+		  unsigned int value, unsigned int index, void *data,
+		  size_t len ) {
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_endpoint *ep = &usb->control;
+	struct io_buffer *iobuf;
+	struct io_buffer *cmplt;
+	union {
+		struct usb_setup_packet setup;
+		struct usb_control_pseudo_header pshdr;
+	} *headroom;
+	struct usb_control_pseudo_header *pshdr;
+	unsigned int i;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( sizeof ( *headroom ) + len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	iob_reserve ( iobuf, sizeof ( *headroom ) );
+	iob_put ( iobuf, len );
+	if ( request & USB_DIR_IN ) {
+		memset ( data, 0, len );
+	} else {
+		memcpy ( iobuf->data, data, len );
+	}
+
+	/* Enqueue message */
+	if ( ( rc = usb_message ( ep, request, value, index, iobuf ) ) != 0 )
+		goto err_message;
+
+	/* Wait for completion */
+	for ( i = 0 ; i < USB_CONTROL_MAX_WAIT_MS ; i++ ) {
+
+		/* Poll bus */
+		usb_poll ( bus );
+
+		/* Check for completion */
+		while ( ( cmplt = list_first_entry ( &usb->complete,
+						     struct io_buffer,
+						     list ) ) ) {
+
+			/* Remove from completion list */
+			list_del ( &cmplt->list );
+
+			/* Extract and strip completion status */
+			pshdr = cmplt->data;
+			iob_pull ( cmplt, sizeof ( *pshdr ) );
+			rc = pshdr->rc;
+
+			/* Discard stale completions */
+			if ( cmplt != iobuf ) {
+				DBGC ( usb, "USB %s stale control completion: "
+				       "%s\n", usb->name, strerror ( rc ) );
+				DBGC_HDA ( usb, 0, cmplt->data,
+					   iob_len ( cmplt ) );
+				free_iob ( cmplt );
+				continue;
+			}
+
+			/* Fail immediately if completion was in error */
+			if ( rc != 0 ) {
+				DBGC ( usb, "USB %s control %04x:%04x:%04x "
+				       "failed: %s\n", usb->name, request,
+				       value, index, strerror ( rc ) );
+				free_iob ( cmplt );
+				return rc;
+			}
+
+			/* Copy completion to data buffer, if applicable */
+			assert ( iob_len ( cmplt ) <= len );
+			if ( request & USB_DIR_IN )
+				memcpy ( data, cmplt->data, iob_len ( cmplt ) );
+			free_iob ( cmplt );
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( usb, "USB %s timed out waiting for control %04x:%04x:%04x\n",
+	       usb->name, request, value, index );
+	return -ETIMEDOUT;
+
+ err_message:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Get USB string descriptor
+ *
+ * @v usb		USB device
+ * @v index		String index
+ * @v language		Language ID
+ * @v buf		Data buffer
+ * @v len		Length of buffer
+ * @ret len		String length (excluding NUL), or negative error
+ */
+int usb_get_string_descriptor ( struct usb_device *usb, unsigned int index,
+				unsigned int language, char *buf, size_t len ) {
+	size_t max = ( len ? ( len - 1 /* NUL */ ) : 0 );
+	struct {
+		struct usb_descriptor_header header;
+		uint16_t character[max];
+	} __attribute__ (( packed )) *desc;
+	unsigned int actual;
+	unsigned int i;
+	int rc;
+
+	/* Allocate buffer for string */
+	desc = malloc ( sizeof ( *desc ) );
+	if ( ! desc ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Get descriptor */
+	if ( ( rc = usb_get_descriptor ( usb, 0, USB_STRING_DESCRIPTOR, index,
+					 language, &desc->header,
+					 sizeof ( *desc ) ) ) != 0 )
+		goto err_get_descriptor;
+
+	/* Copy to buffer */
+	actual = ( ( desc->header.len - sizeof ( desc->header ) ) /
+		   sizeof ( desc->character[0] ) );
+	for ( i = 0 ; ( ( i < actual ) && ( i < max ) ) ; i++ )
+		buf[i] = le16_to_cpu ( desc->character[i] );
+	if ( len )
+		buf[i] = '\0';
+
+	/* Free buffer */
+	free ( desc );
+
+	return actual;
+
+ err_get_descriptor:
+	free ( desc );
+ err_alloc:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB device driver
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Describe USB function
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @v first		First interface number
+ * @ret rc		Return status code
+ */
+static int usb_function ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config,
+			  unsigned int first ) {
+	struct usb_device *usb = func->usb;
+	struct usb_interface_association_descriptor *association;
+	struct usb_interface_descriptor *interface;
+	struct cdc_union_descriptor *cdc_union;
+	unsigned int i;
+
+	/* First, look for an interface association descriptor */
+	association = usb_interface_association_descriptor ( config, first );
+	if ( association ) {
+
+		/* Sanity check */
+		if ( association->count > config->interfaces ) {
+			DBGC ( usb, "USB %s has invalid association [%d-%d)\n",
+			       func->name, association->first,
+			       ( association->first + association->count ) );
+			return -ERANGE;
+		}
+
+		/* Describe function */
+		memcpy ( &func->class, &association->class,
+			 sizeof ( func->class ) );
+		func->count = association->count;
+		for ( i = 0 ; i < association->count ; i++ )
+			func->interface[i] = ( association->first + i );
+		return 0;
+	}
+
+	/* Next, look for an interface descriptor */
+	interface = usb_interface_descriptor ( config, first, 0 );
+	if ( ! interface ) {
+		DBGC ( usb, "USB %s has no interface descriptor\n",
+		       func->name );
+		return -ENOENT;
+	}
+
+	/* Describe function */
+	memcpy ( &func->class, &interface->class, sizeof ( func->class ) );
+	func->count = 1;
+	func->interface[0] = first;
+
+	/* Look for a CDC union descriptor, if applicable */
+	if ( ( func->class.class == USB_CLASS_CDC ) &&
+	     ( cdc_union = cdc_union_descriptor ( config, interface ) ) ) {
+
+		/* Determine interface count */
+		func->count = ( ( cdc_union->header.len -
+				  offsetof ( typeof ( *cdc_union ),
+					     interface[0] ) ) /
+				sizeof ( cdc_union->interface[0] ) );
+		if ( func->count > config->interfaces ) {
+			DBGC ( usb, "USB %s has invalid union functional "
+			       "descriptor with %d interfaces\n",
+			       func->name, func->count );
+			return -ERANGE;
+		}
+
+		/* Describe function */
+		for ( i = 0 ; i < func->count ; i++ )
+			func->interface[i] = cdc_union->interface[i];
+
+		return 0;
+	}
+
+	return 0;
+}
+
+/**
+ * Check for a USB device ID match
+ *
+ * @v func		USB function
+ * @v id		Device ID
+ * @ret matches		Device ID matches
+ */
+static int
+usb_device_id_matches ( struct usb_function *func, struct usb_device_id *id ) {
+
+	return ( ( ( id->vendor == func->dev.desc.vendor ) ||
+		   ( id->vendor == USB_ANY_ID ) ) &&
+		 ( ( id->product == func->dev.desc.device ) ||
+		   ( id->product == USB_ANY_ID ) ) &&
+		 ( id->class.class == func->class.class ) &&
+		 ( id->class.subclass == func->class.subclass ) &&
+		 ( id->class.protocol == func->class.protocol ) );
+}
+
+/**
+ * Probe USB device driver
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usb_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_driver *driver;
+	struct usb_device_id *id;
+	unsigned int i;
+	int rc;
+
+	/* Look for a matching driver */
+	for_each_table_entry ( driver, USB_DRIVERS ) {
+		for ( i = 0 ; i < driver->id_count ; i++ ) {
+
+			/* Check for a matching ID */
+			id = &driver->ids[i];
+			if ( ! usb_device_id_matches ( func, id ) )
+				continue;
+
+			/* Probe driver */
+			if ( ( rc = driver->probe ( func, config ) ) != 0 ) {
+				DBGC ( usb, "USB %s failed to probe driver %s: "
+				       "%s\n", func->name, id->name,
+				       strerror ( rc ) );
+				/* Continue trying other drivers */
+				continue;
+			}
+
+			/* Record driver */
+			func->driver = driver;
+			func->dev.driver_name = id->name;
+			return 0;
+		}
+	}
+
+	/* No driver found */
+	DBGC ( usb, "USB %s %04x:%04x class %d:%d:%d has no driver\n",
+	       func->name, func->dev.desc.vendor, func->dev.desc.device,
+	       func->class.class, func->class.subclass, func->class.protocol );
+	return -ENOENT;
+}
+
+/**
+ * Remove USB device driver
+ *
+ * @v func		USB function
+ */
+static void usb_remove ( struct usb_function *func ) {
+
+	/* Remove driver */
+	func->driver->remove ( func );
+}
+
+/**
+ * Probe all USB device drivers
+ *
+ * @v usb		USB device
+ * @v config		Configuration descriptor
+ */
+static void
+usb_probe_all ( struct usb_device *usb,
+		struct usb_configuration_descriptor *config ) {
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_function *func;
+	uint8_t used[config->interfaces];
+	unsigned int first;
+	unsigned int i;
+	int rc;
+
+	/* Identify each function in turn */
+	memset ( used, 0, sizeof ( used ) );
+	for ( first = 0 ; first < config->interfaces ; first++ ) {
+
+		/* Skip interfaces already used */
+		if ( used[first] )
+			continue;
+
+		/* Allocate and initialise structure */
+		func = zalloc ( sizeof ( *func ) +
+				( config->interfaces *
+				  sizeof ( func->interface[0] ) ) );
+		if ( ! func )
+			goto err_alloc;
+		func->name = func->dev.name;
+		func->usb = usb;
+		func->dev.desc.bus_type = BUS_TYPE_USB;
+		func->dev.desc.location = usb->address;
+		func->dev.desc.vendor = le16_to_cpu ( usb->device.vendor );
+		func->dev.desc.device = le16_to_cpu ( usb->device.product );
+		snprintf ( func->dev.name, sizeof ( func->dev.name ),
+			   "%s-%d.%d", usb->name, config->config, first );
+		INIT_LIST_HEAD ( &func->dev.children );
+		func->dev.parent = bus->dev;
+
+		/* Identify function */
+		if ( ( rc = usb_function ( func, config, first ) ) != 0 )
+			goto err_function;
+		assert ( func->count <= config->interfaces );
+
+		/* Mark interfaces as used */
+		for ( i = 0 ; i < func->count ; i++ ) {
+			if ( func->interface[i] >= config->interfaces ) {
+				DBGC ( usb, "USB %s has invalid interface %d\n",
+				       func->name, func->interface[i] );
+				goto err_interface;
+			}
+			used[ func->interface[i] ] = 1;
+		}
+
+		/* Probe device driver */
+		if ( ( rc = usb_probe ( func, config ) ) != 0 )
+			goto err_probe;
+		DBGC ( usb, "USB %s %04x:%04x class %d:%d:%d interfaces ",
+		       func->name, func->dev.desc.vendor, func->dev.desc.device,
+		       func->class.class, func->class.subclass,
+		       func->class.protocol );
+		for ( i = 0 ; i < func->count ; i++ )
+			DBGC ( usb, "%s%d", ( i ? "," : "" ),
+			       func->interface[i] );
+		DBGC ( usb, " using driver %s\n", func->dev.driver_name );
+
+		/* Add to list of functions */
+		list_add ( &func->list, &usb->functions );
+
+		/* Add to device hierarchy */
+		list_add_tail ( &func->dev.siblings, &bus->dev->children );
+
+		continue;
+
+		list_del ( &func->dev.siblings );
+		list_del ( &func->list );
+		usb_remove ( func );
+	err_probe:
+		free ( func );
+	err_alloc:
+	err_interface:
+	err_function:
+		/* Continue registering other functions */
+		continue;
+	}
+}
+
+/**
+ * Remove all device drivers
+ *
+ * @v usb		USB device
+ */
+static void usb_remove_all ( struct usb_device *usb ) {
+	struct usb_function *func;
+	struct usb_function *tmp;
+
+	/* Remove all functions */
+	list_for_each_entry_safe ( func, tmp, &usb->functions, list ) {
+
+		/* Remove device driver */
+		usb_remove ( func );
+
+		/* Remove from device hierarchy */
+		assert ( list_empty ( &func->dev.children ) );
+		list_del ( &func->dev.siblings );
+
+		/* Remove from list of functions */
+		list_del ( &func->list );
+
+		/* Free function */
+		free ( func );
+	}
+}
+
+/**
+ * Select USB device configuration
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @ret rc		Return status code
+ */
+static int usb_configure ( struct usb_device *usb, unsigned int index ) {
+	struct usb_configuration_descriptor partial;
+	struct usb_configuration_descriptor *config;
+	size_t len;
+	int rc;
+
+	/* Read first part of configuration descriptor to get size */
+	if ( ( rc = usb_get_config_descriptor ( usb, index, &partial,
+						sizeof ( partial ) ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get configuration descriptor %d: "
+		       "%s\n", usb->name, index, strerror ( rc ) );
+		goto err_get_partial;
+	}
+	len = le16_to_cpu ( partial.len );
+	if ( len < sizeof ( partial ) ) {
+		DBGC ( usb, "USB %s underlength configuraton descriptor %d\n",
+		       usb->name, index );
+		rc = -EINVAL;
+		goto err_partial_len;
+	}
+
+	/* Allocate buffer for whole configuration descriptor */
+	config = malloc ( len );
+	if ( ! config ) {
+		rc = -ENOMEM;
+		goto err_alloc_config;
+	}
+
+	/* Read whole configuration descriptor */
+	if ( ( rc = usb_get_config_descriptor ( usb, index, config,
+						len ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get configuration descriptor %d: "
+		       "%s\n", usb->name, index, strerror ( rc ) );
+		goto err_get_config_descriptor;
+	}
+	if ( config->len != partial.len ) {
+		DBGC ( usb, "USB %s bad configuration descriptor %d length\n",
+		       usb->name, index );
+		rc = -EINVAL;
+		goto err_config_len;
+	}
+
+	/* Set configuration */
+	if ( ( rc = usb_set_configuration ( usb, config->config ) ) != 0){
+		DBGC ( usb, "USB %s could not set configuration %d: %s\n",
+		       usb->name, config->config, strerror ( rc ) );
+		goto err_set_configuration;
+	}
+
+	/* Probe USB device drivers */
+	usb_probe_all ( usb, config );
+
+	/* Free configuration descriptor */
+	free ( config );
+
+	return 0;
+
+	usb_remove_all ( usb );
+	usb_set_configuration ( usb, 0 );
+ err_set_configuration:
+ err_config_len:
+ err_get_config_descriptor:
+	free ( config );
+ err_alloc_config:
+ err_partial_len:
+ err_get_partial:
+	return rc;
+}
+
+/**
+ * Clear USB device configuration
+ *
+ * @v usb		USB device
+ */
+static void usb_deconfigure ( struct usb_device *usb ) {
+	unsigned int i;
+
+	/* Remove device drivers */
+	usb_remove_all ( usb );
+
+	/* Sanity checks */
+	for ( i = 0 ; i < ( sizeof ( usb->ep ) / sizeof ( usb->ep[0] ) ) ; i++){
+		if ( i != USB_ENDPOINT_IDX ( USB_EP0_ADDRESS ) )
+			assert ( usb->ep[i] == NULL );
+	}
+
+	/* Clear device configuration */
+	usb_set_configuration ( usb, 0 );
+}
+
+/**
+ * Find and select a supported USB device configuration
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int usb_configure_any ( struct usb_device *usb ) {
+	unsigned int index;
+	int rc = -ENOENT;
+
+	/* Attempt all configuration indexes */
+	for ( index = 0 ; index < usb->device.configurations ; index++ ) {
+
+		/* Attempt this configuration index */
+		if ( ( rc = usb_configure ( usb, index ) ) != 0 )
+			continue;
+
+		/* If we have no drivers, then try the next configuration */
+		if ( list_empty ( &usb->functions ) ) {
+			rc = -ENOTSUP;
+			usb_deconfigure ( usb );
+			continue;
+		}
+
+		return 0;
+	}
+
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB device
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB device
+ *
+ * @v port		USB port
+ * @ret usb		USB device, or NULL on allocation failure
+ */
+static struct usb_device * alloc_usb ( struct usb_port *port ) {
+	struct usb_hub *hub = port->hub;
+	struct usb_bus *bus = hub->bus;
+	struct usb_device *usb;
+
+	/* Allocate and initialise structure */
+	usb = zalloc ( sizeof ( *usb ) );
+	if ( ! usb )
+		return NULL;
+	snprintf ( usb->name, sizeof ( usb->name ), "%s%c%d", hub->name,
+		   ( hub->usb ? '.' : '-' ), port->address );
+	usb->port = port;
+	INIT_LIST_HEAD ( &usb->functions );
+	usb->host = &bus->op->device;
+	usb_endpoint_init ( &usb->control, usb, &usb_control_operations );
+	INIT_LIST_HEAD ( &usb->complete );
+
+	return usb;
+}
+
+/**
+ * Register USB device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int register_usb ( struct usb_device *usb ) {
+	struct usb_port *port = usb->port;
+	struct usb_hub *hub = port->hub;
+	struct usb_bus *bus = hub->bus;
+	unsigned int protocol;
+	size_t mtu;
+	int rc;
+
+	/* Add to port */
+	if ( port->usb != NULL ) {
+		DBGC ( hub, "USB hub %s port %d is already registered to %s\n",
+		       hub->name, port->address, port->usb->name );
+		rc = -EALREADY;
+		goto err_already;
+	}
+	port->usb = usb;
+
+	/* Add to bus device list */
+	list_add_tail ( &usb->list, &bus->devices );
+
+	/* Enable device */
+	if ( ( rc = hub->driver->enable ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not enable: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_enable;
+	}
+
+	/* Allow recovery interval since port may have been reset */
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Get device speed */
+	if ( ( rc = hub->driver->speed ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not get speed: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_speed;
+	}
+	DBGC2 ( usb, "USB %s attached as %s-speed device\n",
+		usb->name, usb_speed_name ( port->speed ) );
+
+	/* Open device */
+	if ( ( rc = usb->host->open ( usb ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not open: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Describe control endpoint */
+	mtu = USB_EP0_DEFAULT_MTU ( port->speed );
+	usb_endpoint_describe ( &usb->control, USB_EP0_ADDRESS,
+				USB_EP0_ATTRIBUTES, mtu, USB_EP0_BURST,
+				USB_EP0_INTERVAL );
+
+	/* Open control endpoint */
+	if ( ( rc = usb_endpoint_open ( &usb->control ) ) != 0 )
+		goto err_open_control;
+	assert ( usb_endpoint ( usb, USB_EP0_ADDRESS ) == &usb->control );
+
+	/* Assign device address */
+	if ( ( rc = usb->host->address ( usb ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not set address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_address;
+	}
+	DBGC2 ( usb, "USB %s assigned address %d\n", usb->name, usb->address );
+
+	/* Allow recovery interval after Set Address command */
+	mdelay ( USB_SET_ADDRESS_RECOVER_DELAY_MS );
+
+	/* Read first part of device descriptor to get EP0 MTU */
+	if ( ( rc = usb_get_mtu ( usb, &usb->device ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get MTU: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_get_mtu;
+	}
+
+	/* Calculate EP0 MTU */
+	protocol = le16_to_cpu ( usb->device.protocol );
+	mtu = ( ( protocol < USB_PROTO_3_0 ) ?
+		usb->device.mtu : ( 1 << usb->device.mtu ) );
+	DBGC2 ( usb, "USB %s has control MTU %zd (guessed %zd)\n",
+		usb->name, mtu, usb->control.mtu );
+
+	/* Update MTU */
+	if ( ( rc = usb_endpoint_mtu ( &usb->control, mtu ) ) != 0 )
+		goto err_mtu;
+
+	/* Read whole device descriptor */
+	if ( ( rc = usb_get_device_descriptor ( usb, &usb->device ) ) != 0 ) {
+		DBGC ( usb, "USB %s could not get device descriptor: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_get_device_descriptor;
+	}
+	DBGC ( usb, "USB %s addr %d %04x:%04x class %d:%d:%d (v%s, %s-speed, "
+	       "MTU %zd)\n", usb->name, usb->address,
+	       le16_to_cpu ( usb->device.vendor ),
+	       le16_to_cpu ( usb->device.product ), usb->device.class.class,
+	       usb->device.class.subclass, usb->device.class.protocol,
+	       usb_bcd ( le16_to_cpu ( usb->device.protocol ) ),
+	       usb_speed_name ( port->speed ), usb->control.mtu );
+
+	/* Configure device */
+	if ( ( rc = usb_configure_any ( usb ) ) != 0 )
+		goto err_configure_any;
+
+	return 0;
+
+	usb_deconfigure ( usb );
+ err_configure_any:
+ err_get_device_descriptor:
+ err_mtu:
+ err_get_mtu:
+ err_address:
+	usb_endpoint_close ( &usb->control );
+ err_open_control:
+	usb->host->close ( usb );
+ err_open:
+ err_speed:
+	hub->driver->disable ( hub, port );
+ err_enable:
+	list_del ( &usb->list );
+	port->usb = NULL;
+ err_already:
+	return rc;
+}
+
+/**
+ * Unregister USB device
+ *
+ * @v usb		USB device
+ */
+static void unregister_usb ( struct usb_device *usb ) {
+	struct usb_port *port = usb->port;
+	struct usb_hub *hub = port->hub;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Sanity checks */
+	assert ( port->usb == usb );
+
+	/* Clear device configuration */
+	usb_deconfigure ( usb );
+
+	/* Close control endpoint */
+	usb_endpoint_close ( &usb->control );
+
+	/* Discard any stale control completions */
+	list_for_each_entry_safe ( iobuf, tmp, &usb->complete, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+
+	/* Close device */
+	usb->host->close ( usb );
+
+	/* Disable port */
+	hub->driver->disable ( hub, port );
+
+	/* Remove from bus device list */
+	list_del ( &usb->list );
+
+	/* Remove from port */
+	port->usb = NULL;
+}
+
+/**
+ * Free USB device
+ *
+ * @v usb		USB device
+ */
+static void free_usb ( struct usb_device *usb ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	for ( i = 0 ; i < ( sizeof ( usb->ep ) / sizeof ( usb->ep[0] ) ) ; i++ )
+		assert ( usb->ep[i] == NULL );
+	assert ( list_empty ( &usb->functions ) );
+	assert ( list_empty ( &usb->complete ) );
+
+	/* Free device */
+	free ( usb );
+}
+
+/******************************************************************************
+ *
+ * USB device hotplug event handling
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Handle newly attached USB device
+ *
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int usb_attached ( struct usb_port *port ) {
+	struct usb_device *usb;
+	int rc;
+
+	/* Mark port as attached */
+	port->attached = 1;
+
+	/* Sanity checks */
+	assert ( port->usb == NULL );
+
+	/* Allocate USB device */
+	usb = alloc_usb ( port );
+	if ( ! usb ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Register USB device */
+	if ( ( rc = register_usb ( usb ) ) != 0 )
+		goto err_register;
+
+	return 0;
+
+	unregister_usb ( usb );
+ err_register:
+	free_usb ( usb );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Handle newly detached USB device
+ *
+ * @v port		USB port
+ */
+static void usb_detached ( struct usb_port *port ) {
+	struct usb_device *usb = port->usb;
+
+	/* Mark port as detached */
+	port->attached = 0;
+
+	/* Do nothing if we have no USB device */
+	if ( ! usb )
+		return;
+
+	/* Unregister USB device */
+	unregister_usb ( usb );
+
+	/* Free USB device */
+	free_usb ( usb );
+}
+
+/**
+ * Handle newly attached or detached USB device
+ *
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int usb_hotplugged ( struct usb_port *port ) {
+	struct usb_hub *hub = port->hub;
+	int rc;
+
+	/* Get current port speed */
+	if ( ( rc = hub->driver->speed ( hub, port ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s port %d could not get speed: %s\n",
+		       hub->name, port->address, strerror ( rc ) );
+		goto err_speed;
+	}
+
+	/* Detach device, if applicable */
+	if ( port->attached && ( port->disconnected || ! port->speed ) )
+		usb_detached ( port );
+
+	/* Attach device, if applicable */
+	if ( port->speed && ( ! port->attached ) &&
+	     ( ( rc = usb_attached ( port ) ) != 0 ) )
+		goto err_attached;
+
+ err_attached:
+ err_speed:
+	/* Clear any recorded disconnections */
+	port->disconnected = 0;
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * USB process
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Report port status change
+ *
+ * @v port		USB port
+ */
+void usb_port_changed ( struct usb_port *port ) {
+
+	/* Record hub port status change */
+	list_del ( &port->changed );
+	list_add_tail ( &port->changed, &usb_changed );
+}
+
+/**
+ * Handle newly attached or detached USB device
+ *
+ */
+static void usb_hotplug ( void ) {
+	struct usb_port *port;
+
+	/* Handle any changed ports, allowing for the fact that the
+	 * port list may change as we perform hotplug actions.
+	 */
+	while ( ! list_empty ( &usb_changed ) ) {
+
+		/* Get first changed port */
+		port = list_first_entry ( &usb_changed, struct usb_port,
+					  changed );
+		assert ( port != NULL );
+
+		/* Remove from list of changed ports */
+		list_del ( &port->changed );
+		INIT_LIST_HEAD ( &port->changed );
+
+		/* Perform appropriate hotplug action */
+		usb_hotplugged ( port );
+	}
+}
+
+/**
+ * USB process
+ *
+ * @v process		USB process
+ */
+static void usb_step ( struct process *process __unused ) {
+	struct usb_bus *bus;
+	struct usb_endpoint *ep;
+
+	/* Poll all buses */
+	for_each_usb_bus ( bus )
+		usb_poll ( bus );
+
+	/* Attempt to reset first halted endpoint in list, if any.  We
+	 * do not attempt to process the complete list, since this
+	 * would require extra code to allow for the facts that the
+	 * halted endpoint list may change as we do so, and that
+	 * resetting an endpoint may fail.
+	 */
+	if ( ( ep = list_first_entry ( &usb_halted, struct usb_endpoint,
+				       halted ) ) != NULL )
+		usb_endpoint_reset ( ep );
+
+	/* Handle any changed ports */
+	usb_hotplug();
+}
+
+/** USB process */
+PERMANENT_PROCESS ( usb_process, usb_step );
+
+/******************************************************************************
+ *
+ * USB hub
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB hub
+ *
+ * @v bus		USB bus
+ * @v usb		Underlying USB device, if any
+ * @v ports		Number of ports
+ * @v driver		Hub driver operations
+ * @ret hub		USB hub, or NULL on allocation failure
+ */
+struct usb_hub * alloc_usb_hub ( struct usb_bus *bus, struct usb_device *usb,
+				 unsigned int ports,
+				 struct usb_hub_driver_operations *driver ) {
+	struct usb_hub *hub;
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Allocate and initialise structure */
+	hub = zalloc ( sizeof ( *hub ) + ( ports * sizeof ( hub->port[0] ) ) );
+	if ( ! hub )
+		return NULL;
+	hub->name = ( usb ? usb->name : bus->name );
+	hub->bus = bus;
+	hub->usb = usb;
+	if ( usb )
+		hub->protocol = usb->port->protocol;
+	hub->ports = ports;
+	hub->driver = driver;
+	hub->host = &bus->op->hub;
+
+	/* Initialise port list */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		port->hub = hub;
+		port->address = i;
+		if ( usb )
+			port->protocol = usb->port->protocol;
+		INIT_LIST_HEAD ( &port->changed );
+	}
+
+	return hub;
+}
+
+/**
+ * Register USB hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+int register_usb_hub ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct usb_port *port;
+	unsigned int i;
+	int rc;
+
+	/* Add to hub list */
+	list_add_tail ( &hub->list, &bus->hubs );
+
+	/* Open hub (host controller) */
+	if ( ( rc = hub->host->open ( hub ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s could not open: %s\n",
+		       hub->name, strerror ( rc ) );
+		goto err_host_open;
+	}
+
+	/* Open hub (driver) */
+	if ( ( rc = hub->driver->open ( hub ) ) != 0 ) {
+		DBGC ( hub, "USB hub %s could not open: %s\n",
+		       hub->name, strerror ( rc ) );
+		goto err_driver_open;
+	}
+
+	/* Delay to allow ports to stabilise */
+	mdelay ( USB_PORT_DELAY_MS );
+
+	/* Mark all ports as changed */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		usb_port_changed ( port );
+	}
+
+	/* Some hubs seem to defer reporting device connections until
+	 * their interrupt endpoint is polled for the first time.
+	 * Poll the bus once now in order to pick up any such
+	 * connections.
+	 */
+	usb_poll ( bus );
+
+	return 0;
+
+	hub->driver->close ( hub );
+ err_driver_open:
+	hub->host->close ( hub );
+ err_host_open:
+	list_del ( &hub->list );
+	return rc;
+}
+
+/**
+ * Unregister USB hub
+ *
+ * @v hub		USB hub
+ */
+void unregister_usb_hub ( struct usb_hub *hub ) {
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Detach all devices */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		if ( port->attached )
+			usb_detached ( port );
+	}
+
+	/* Close hub (driver) */
+	hub->driver->close ( hub );
+
+	/* Close hub (host controller) */
+	hub->host->close ( hub );
+
+	/* Cancel any pending port status changes */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		list_del ( &port->changed );
+		INIT_LIST_HEAD ( &port->changed );
+	}
+
+	/* Remove from hub list */
+	list_del ( &hub->list );
+}
+
+/**
+ * Free USB hub
+ *
+ * @v hub		USB hub
+ */
+void free_usb_hub ( struct usb_hub *hub ) {
+	struct usb_port *port;
+	unsigned int i;
+
+	/* Sanity checks */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		port = usb_port ( hub, i );
+		assert ( ! port->attached );
+		assert ( port->usb == NULL );
+		assert ( list_empty ( &port->changed ) );
+	}
+
+	/* Free hub */
+	free ( hub );
+}
+
+/******************************************************************************
+ *
+ * USB bus
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate USB bus
+ *
+ * @v dev		Underlying hardware device
+ * @v ports		Number of root hub ports
+ * @v mtu		Largest transfer allowed on the bus
+ * @v op		Host controller operations
+ * @ret bus		USB bus, or NULL on allocation failure
+ */
+struct usb_bus * alloc_usb_bus ( struct device *dev, unsigned int ports,
+				 size_t mtu, struct usb_host_operations *op ) {
+	struct usb_bus *bus;
+
+	/* Allocate and initialise structure */
+	bus = zalloc ( sizeof ( *bus ) );
+	if ( ! bus )
+		goto err_alloc_bus;
+	bus->name = dev->name;
+	bus->dev = dev;
+	bus->mtu = mtu;
+	bus->op = op;
+	INIT_LIST_HEAD ( &bus->devices );
+	INIT_LIST_HEAD ( &bus->hubs );
+	bus->host = &bus->op->bus;
+
+	/* Allocate root hub */
+	bus->hub = alloc_usb_hub ( bus, NULL, ports, &op->root );
+	if ( ! bus->hub )
+		goto err_alloc_hub;
+
+	return bus;
+
+	free_usb_hub ( bus->hub );
+ err_alloc_hub:
+	free ( bus );
+ err_alloc_bus:
+	return NULL;
+}
+
+/**
+ * Register USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+int register_usb_bus ( struct usb_bus *bus ) {
+	int rc;
+
+	/* Sanity checks */
+	assert ( bus->hub != NULL );
+
+	/* Open bus */
+	if ( ( rc = bus->host->open ( bus ) ) != 0 )
+		goto err_open;
+
+	/* Add to list of USB buses */
+	list_add_tail ( &bus->list, &usb_buses );
+
+	/* Register root hub */
+	if ( ( rc = register_usb_hub ( bus->hub ) ) != 0 )
+		goto err_register_hub;
+
+	/* Attach any devices already present */
+	usb_hotplug();
+
+	return 0;
+
+	unregister_usb_hub ( bus->hub );
+ err_register_hub:
+	list_del ( &bus->list );
+	bus->host->close ( bus );
+ err_open:
+	return rc;
+}
+
+/**
+ * Unregister USB bus
+ *
+ * @v bus		USB bus
+ */
+void unregister_usb_bus ( struct usb_bus *bus ) {
+
+	/* Sanity checks */
+	assert ( bus->hub != NULL );
+
+	/* Unregister root hub */
+	unregister_usb_hub ( bus->hub );
+
+	/* Remove from list of USB buses */
+	list_del ( &bus->list );
+
+	/* Close bus */
+	bus->host->close ( bus );
+
+	/* Sanity checks */
+	assert ( list_empty ( &bus->devices ) );
+	assert ( list_empty ( &bus->hubs ) );
+}
+
+/**
+ * Free USB bus
+ *
+ * @v bus		USB bus
+ */
+void free_usb_bus ( struct usb_bus *bus ) {
+	struct usb_endpoint *ep;
+	struct usb_port *port;
+
+	/* Sanity checks */
+	assert ( list_empty ( &bus->devices ) );
+	assert ( list_empty ( &bus->hubs ) );
+	list_for_each_entry ( ep, &usb_halted, halted )
+		assert ( ep->usb->port->hub->bus != bus );
+	list_for_each_entry ( port, &usb_changed, changed )
+		assert ( port->hub->bus != bus );
+
+	/* Free root hub */
+	free_usb_hub ( bus->hub );
+
+	/* Free bus */
+	free ( bus );
+}
+
+/**
+ * Find USB bus by device location
+ *
+ * @v bus_type		Bus type
+ * @v location		Bus location
+ * @ret bus		USB bus, or NULL
+ */
+struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
+					    unsigned int location ) {
+	struct usb_bus *bus;
+
+	for_each_usb_bus ( bus ) {
+		if ( ( bus->dev->desc.bus_type == bus_type ) &&
+		     ( bus->dev->desc.location == location ) )
+			return bus;
+	}
+
+	return NULL;
+}
+
+/******************************************************************************
+ *
+ * USB address assignment
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate device address
+ *
+ * @v bus		USB bus
+ * @ret address		Device address, or negative error
+ */
+int usb_alloc_address ( struct usb_bus *bus ) {
+	unsigned int address;
+
+	/* Find first free device address */
+	address = ffsll ( ~bus->addresses );
+	if ( ! address )
+		return -ENOENT;
+
+	/* Mark address as used */
+	bus->addresses |= ( 1ULL << ( address - 1 ) );
+
+	return address;
+}
+
+/**
+ * Free device address
+ *
+ * @v bus		USB bus
+ * @v address		Device address
+ */
+void usb_free_address ( struct usb_bus *bus, unsigned int address ) {
+
+	/* Sanity check */
+	assert ( address > 0 );
+	assert ( bus->addresses & ( 1ULL << ( address - 1 ) ) );
+
+	/* Mark address as free */
+	bus->addresses &= ~( 1ULL << ( address - 1 ) );
+}
+
+/******************************************************************************
+ *
+ * USB bus topology
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get USB route string
+ *
+ * @v usb		USB device
+ * @ret route		USB route string
+ */
+unsigned int usb_route_string ( struct usb_device *usb ) {
+	struct usb_device *parent;
+	unsigned int route;
+
+	/* Navigate up to root hub, constructing route string as we go */
+	for ( route = 0 ; ( parent = usb->port->hub->usb ) ; usb = parent ) {
+		route <<= 4;
+		route |= ( ( usb->port->address > 0xf ) ?
+			   0xf : usb->port->address );
+	}
+
+	return route;
+}
+
+/**
+ * Get USB depth
+ *
+ * @v usb		USB device
+ * @ret depth		Hub depth
+ */
+unsigned int usb_depth ( struct usb_device *usb ) {
+	struct usb_device *parent;
+	unsigned int depth;
+
+	/* Navigate up to root hub, constructing depth as we go */
+	for ( depth = 0 ; ( parent = usb->port->hub->usb ) ; usb = parent )
+		depth++;
+
+	return depth;
+}
+
+/**
+ * Get USB root hub port
+ *
+ * @v usb		USB device
+ * @ret port		Root hub port
+ */
+struct usb_port * usb_root_hub_port ( struct usb_device *usb ) {
+	struct usb_device *parent;
+
+	/* Navigate up to root hub */
+	while ( ( parent = usb->port->hub->usb ) )
+		usb = parent;
+
+	return usb->port;
+}
+
+/**
+ * Get USB transaction translator
+ *
+ * @v usb		USB device
+ * @ret port		Transaction translator port, or NULL
+ */
+struct usb_port * usb_transaction_translator ( struct usb_device *usb ) {
+	struct usb_device *parent;
+
+	/* Navigate up to root hub.  If we find a low-speed or
+	 * full-speed port with a higher-speed parent device, then
+	 * that port is the transaction translator.
+	 */
+	for ( ; ( parent = usb->port->hub->usb ) ; usb = parent ) {
+		if ( ( usb->port->speed <= USB_SPEED_FULL ) &&
+		     ( parent->port->speed > USB_SPEED_FULL ) )
+			return usb->port;
+	}
+
+	return NULL;
+}
+
+/* Drag in objects via register_usb_bus() */
+REQUIRING_SYMBOL ( register_usb_bus );
+
+/* Drag in USB configuration */
+REQUIRE_OBJECT ( config_usb );
+
+/* Drag in hub driver */
+REQUIRE_OBJECT ( usbhub );
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/arbel.c b/qemu/roms/ipxe/src/drivers/infiniband/arbel.c
index 1a56ff9af..2a6c32dec 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/arbel.c
+++ b/qemu/roms/ipxe/src/drivers/infiniband/arbel.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/arbel.h b/qemu/roms/ipxe/src/drivers/infiniband/arbel.h
index c0303f1bc..73394cd9a 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/arbel.h
+++ b/qemu/roms/ipxe/src/drivers/infiniband/arbel.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/linda.c b/qemu/roms/ipxe/src/drivers/infiniband/linda.c
index 4afda1208..a6ae9f529 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/linda.c
+++ b/qemu/roms/ipxe/src/drivers/infiniband/linda.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/linda.h b/qemu/roms/ipxe/src/drivers/infiniband/linda.h
index 72ce70868..46a920a17 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/linda.h
+++ b/qemu/roms/ipxe/src/drivers/infiniband/linda.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c
index 9979b346e..e22f2349a 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c
+++ b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/infiniband/qib7322.h b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.h
index 63abe221b..72797b240 100644
--- a/qemu/roms/ipxe/src/drivers/infiniband/qib7322.h
+++ b/qemu/roms/ipxe/src/drivers/infiniband/qib7322.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/net/amd8111e.h b/qemu/roms/ipxe/src/drivers/net/amd8111e.h
index 2000df158..8ecd159af 100644
--- a/qemu/roms/ipxe/src/drivers/net/amd8111e.h
+++ b/qemu/roms/ipxe/src/drivers/net/amd8111e.h
@@ -16,6 +16,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  * USA
 
 Module Name:
@@ -36,7 +40,7 @@ Revision History:
 	3.0.1
 */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef _AMD811E_H
 #define _AMD811E_H
diff --git a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ani.h b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
index dbd4d4d5b..ba87ba0fd 100644
--- a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
+++ b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ani.h
@@ -125,7 +125,7 @@ struct ar5416AniState {
 	u8 mrcCCKOff;
 	u8 spurImmunityLevel;
 	u8 firstepLevel;
-	u8 ofdmWeakSigDetectOff;
+	u8 ofdmWeakSigDetect;
 	u8 cckWeakSigThreshold;
 	u32 listenTime;
 	int32_t rssiThrLow;
diff --git a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
index ff7df497f..76ca79cba 100644
--- a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
+++ b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ani.c
@@ -177,7 +177,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 
 	rssi = BEACON_RSSI(ah);
 	if (rssi > aniState->rssiThrHigh) {
-		if (!aniState->ofdmWeakSigDetectOff) {
+		if (aniState->ofdmWeakSigDetect) {
 			if (ath9k_hw_ani_control(ah,
 					 ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 					 0)) {
@@ -192,7 +192,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 			return;
 		}
 	} else if (rssi > aniState->rssiThrLow) {
-		if (aniState->ofdmWeakSigDetectOff)
+		if (!aniState->ofdmWeakSigDetect)
 			ath9k_hw_ani_control(ah,
 				     ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				     1);
@@ -202,7 +202,7 @@ static void ath9k_hw_ani_ofdm_err_trigger_old(struct ath_hw *ah)
 		return;
 	} else {
 		if ((ah->dev->channels + ah->dev->channel)->band == NET80211_BAND_2GHZ) {
-			if (!aniState->ofdmWeakSigDetectOff)
+			if (aniState->ofdmWeakSigDetect)
 				ath9k_hw_ani_control(ah,
 				     ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				     0);
@@ -360,7 +360,7 @@ static void ath9k_hw_ani_lower_immunity_old(struct ath_hw *ah)
 	if (rssi > aniState->rssiThrHigh) {
 		/* XXX: Handle me */
 	} else if (rssi > aniState->rssiThrLow) {
-		if (aniState->ofdmWeakSigDetectOff) {
+		if (!aniState->ofdmWeakSigDetect) {
 			if (ath9k_hw_ani_control(ah,
 				 ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
 				 1) == 1)
@@ -436,9 +436,9 @@ static void ath9k_ani_reset_old(struct ath_hw *ah)
 	if (aniState->spurImmunityLevel != 0)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_SPUR_IMMUNITY_LEVEL,
 				     aniState->spurImmunityLevel);
-	if (aniState->ofdmWeakSigDetectOff)
+	if (!aniState->ofdmWeakSigDetect)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_OFDM_WEAK_SIGNAL_DETECTION,
-				     !aniState->ofdmWeakSigDetectOff);
+				     aniState->ofdmWeakSigDetect);
 	if (aniState->cckWeakSigThreshold)
 		ath9k_hw_ani_control(ah, ATH9K_ANI_CCK_WEAK_SIGNAL_THR,
 				     aniState->cckWeakSigThreshold);
@@ -709,8 +709,8 @@ void ath9k_hw_ani_init(struct ath_hw *ah)
 
 		ani->rssiThrHigh = ATH9K_ANI_RSSI_THR_HIGH;
 		ani->rssiThrLow = ATH9K_ANI_RSSI_THR_LOW;
-		ani->ofdmWeakSigDetectOff =
-			!ATH9K_ANI_USE_OFDM_WEAK_SIG;
+		ani->ofdmWeakSigDetect =
+			ATH9K_ANI_USE_OFDM_WEAK_SIG;
 		ani->cckNoiseImmunityLevel = ATH9K_ANI_CCK_DEF_LEVEL;
 	}
 
diff --git a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
index 60e87e9e2..2b6c133cb 100644
--- a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
+++ b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar5008_phy.c
@@ -1141,12 +1141,12 @@ static int ar5008_hw_ani_control_old(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1215,10 +1215,10 @@ static int ar5008_hw_ani_control_old(struct ath_hw *ah,
 
 	DBG2("ath9k: ANI parameters:\n");
 	DBG2(
-		"noiseImmunityLevel=%d, spurImmunityLevel=%d, ofdmWeakSigDetectOff=%d\n",
+		"noiseImmunityLevel=%d, spurImmunityLevel=%d, ofdmWeakSigDetect=%d\n",
 		aniState->noiseImmunityLevel,
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff);
+		aniState->ofdmWeakSigDetect);
 	DBG2(
 		"cckWeakSigThreshold=%d, firstepLevel=%d, listenTime=%d\n",
 		aniState->cckWeakSigThreshold,
@@ -1307,18 +1307,18 @@ static int ar5008_hw_ani_control_new(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			DBG2("ath9k: "
 				"** ch %d: ofdm weak signal: %s=>%s\n",
 				chan->channel,
-				!aniState->ofdmWeakSigDetectOff ?
+				aniState->ofdmWeakSigDetect ?
 				"on" : "off",
 				on ? "on" : "off");
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1467,7 +1467,7 @@ static int ar5008_hw_ani_control_new(struct ath_hw *ah,
 	DBG2("ath9k: "
 		"ANI parameters: SI=%d, ofdmWS=%s FS=%d MRCcck=%s listenTime=%d ofdmErrs=%d cckErrs=%d\n",
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff ? "on" : "off",
+		aniState->ofdmWeakSigDetect ? "on" : "off",
 		aniState->firstepLevel,
 		!aniState->mrcCCKOff ? "on" : "off",
 		aniState->listenTime,
@@ -1554,7 +1554,7 @@ static void ar5008_hw_ani_cache_ini_regs(struct ath_hw *ah)
 	/* these levels just got reset to defaults by the INI */
 	aniState->spurImmunityLevel = ATH9K_ANI_SPUR_IMMUNE_LVL_NEW;
 	aniState->firstepLevel = ATH9K_ANI_FIRSTEP_LVL_NEW;
-	aniState->ofdmWeakSigDetectOff = !ATH9K_ANI_USE_OFDM_WEAK_SIG;
+	aniState->ofdmWeakSigDetect = ATH9K_ANI_USE_OFDM_WEAK_SIG;
 	aniState->mrcCCKOff = 1; /* not available on pre AR9003 */
 }
 
diff --git a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
index 6103040ab..2244b775a 100644
--- a/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
+++ b/qemu/roms/ipxe/src/drivers/net/ath/ath9k/ath9k_ar9003_phy.c
@@ -859,18 +859,18 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 			REG_CLR_BIT(ah, AR_PHY_SFCORR_LOW,
 				    AR_PHY_SFCORR_LOW_USE_SELF_CORR_LOW);
 
-		if (!on != aniState->ofdmWeakSigDetectOff) {
+		if (on != aniState->ofdmWeakSigDetect) {
 			DBG2("ath9k: "
 				"** ch %d: ofdm weak signal: %s=>%s\n",
 				chan->channel,
-				!aniState->ofdmWeakSigDetectOff ?
+				aniState->ofdmWeakSigDetect ?
 				"on" : "off",
 				on ? "on" : "off");
 			if (on)
 				ah->stats.ast_ani_ofdmon++;
 			else
 				ah->stats.ast_ani_ofdmoff++;
-			aniState->ofdmWeakSigDetectOff = !on;
+			aniState->ofdmWeakSigDetect = on;
 		}
 		break;
 	}
@@ -1013,7 +1013,7 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 			      AR_PHY_MRC_CCK_ENABLE, is_on);
 		REG_RMW_FIELD(ah, AR_PHY_MRC_CCK_CTRL,
 			      AR_PHY_MRC_CCK_MUX_REG, is_on);
-		if (!is_on != aniState->mrcCCKOff) {
+		if (!(is_on != aniState->mrcCCKOff)) {
 			DBG2("ath9k: "
 				"** ch %d: MRC CCK: %s=>%s\n",
 				chan->channel,
@@ -1037,7 +1037,7 @@ static int ar9003_hw_ani_control(struct ath_hw *ah,
 	DBG2("ath9k: "
 		"ANI parameters: SI=%d, ofdmWS=%s FS=%d MRCcck=%s listenTime=%d ofdmErrs=%d cckErrs=%d\n",
 		aniState->spurImmunityLevel,
-		!aniState->ofdmWeakSigDetectOff ? "on" : "off",
+		aniState->ofdmWeakSigDetect ? "on" : "off",
 		aniState->firstepLevel,
 		!aniState->mrcCCKOff ? "on" : "off",
 		aniState->listenTime,
@@ -1137,7 +1137,7 @@ static void ar9003_hw_ani_cache_ini_regs(struct ath_hw *ah)
 	/* these levels just got reset to defaults by the INI */
 	aniState->spurImmunityLevel = ATH9K_ANI_SPUR_IMMUNE_LVL_NEW;
 	aniState->firstepLevel = ATH9K_ANI_FIRSTEP_LVL_NEW;
-	aniState->ofdmWeakSigDetectOff = !ATH9K_ANI_USE_OFDM_WEAK_SIG;
+	aniState->ofdmWeakSigDetect = ATH9K_ANI_USE_OFDM_WEAK_SIG;
 	aniState->mrcCCKOff = !ATH9K_ANI_ENABLE_MRC_CCK;
 }
 
diff --git a/qemu/roms/ipxe/src/drivers/net/atl1e.c b/qemu/roms/ipxe/src/drivers/net/atl1e.c
index 1ff0f0d10..d010d8c4a 100644
--- a/qemu/roms/ipxe/src/drivers/net/atl1e.c
+++ b/qemu/roms/ipxe/src/drivers/net/atl1e.c
@@ -224,7 +224,7 @@ static int atl1e_sw_init(struct atl1e_adapter *adapter)
 	adapter->link_duplex = FULL_DUPLEX;
 
 	/* PCI config space info */
-	pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id);
+	pci_read_config_byte(pdev, PCI_REVISION, &rev_id);
 
 	phy_status_data = AT_READ_REG(hw, REG_PHY_STATUS);
 	/* nic type */
diff --git a/qemu/roms/ipxe/src/drivers/net/davicom.c b/qemu/roms/ipxe/src/drivers/net/davicom.c
index a4870a729..9d3d8b915 100644
--- a/qemu/roms/ipxe/src/drivers/net/davicom.c
+++ b/qemu/roms/ipxe/src/drivers/net/davicom.c
@@ -340,6 +340,7 @@ static void davicom_media_chk(struct nic * nic __unused)
   csr6 = 0x00200000;	/* SF */
   outl(csr6, ioaddr + CSR6);
 
+#define PCI_VENDOR_ID_DAVICOM		0x1282
 #define	PCI_DEVICE_ID_DM9009		0x9009
   if (vendor == PCI_VENDOR_ID_DAVICOM && dev_id == PCI_DEVICE_ID_DM9009) {
     /* Set to 10BaseT mode for DM9009 */
diff --git a/qemu/roms/ipxe/src/drivers/net/dm96xx.c b/qemu/roms/ipxe/src/drivers/net/dm96xx.c
new file mode 100644
index 000000000..58d8dd964
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/dm96xx.c
@@ -0,0 +1,671 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include "dm96xx.h"
+
+/** @file
+ *
+ * Davicom DM96xx USB Ethernet driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Register operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset device
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_reset ( struct dm96xx_device *dm96xx ) {
+	int ncr;
+	int rc;
+
+	/* Reset device */
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_NCR,
+					    DM96XX_NCR_RST ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not reset: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Wait for reset to complete */
+	udelay ( DM96XX_RESET_DELAY_US );
+
+	/* Check that reset has completed */
+	ncr = dm96xx_read_register ( dm96xx, DM96XX_NCR );
+	if ( ncr < 0 ) {
+		rc = ncr;
+		DBGC ( dm96xx, "DM96XX %p failed to reset: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+	if ( ncr & DM96XX_NCR_RST ) {
+		DBGC ( dm96xx, "DM96XX %p failed to reset (NCR=%#02x)\n",
+		       dm96xx, ncr );
+		return -EIO;
+	}
+
+	return 0;
+}
+
+/**
+ * Read MAC address
+ *
+ * @v dm96xx		DM96xx device
+ * @v mac		MAC address to fill in
+ * @ret rc		Return status code
+ */
+static int dm96xx_read_mac ( struct dm96xx_device *dm96xx, uint8_t *mac ) {
+	int rc;
+
+	/* Read MAC address */
+	if ( ( rc = dm96xx_read_registers ( dm96xx, DM96XX_PAR, mac,
+					    ETH_ALEN ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not read MAC address: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write MAC address
+ *
+ * @v dm96xx		DM96xx device
+ * @v mac		MAC address
+ * @ret rc		Return status code
+ */
+static int dm96xx_write_mac ( struct dm96xx_device *dm96xx, uint8_t *mac ) {
+	int rc;
+
+	/* Write MAC address */
+	if ( ( rc = dm96xx_write_registers ( dm96xx, DM96XX_PAR, mac,
+					     ETH_ALEN ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write MAC address: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Update link status based on network status register
+ *
+ * @v dm96xx		DM96xx device
+ * @v nsr		Network status register
+ */
+static void dm96xx_link_nsr ( struct dm96xx_device *dm96xx, unsigned int nsr ) {
+	struct net_device *netdev = dm96xx->netdev;
+
+	if ( nsr & DM96XX_NSR_LINKST ) {
+		if ( ! netdev_link_ok ( netdev ) )
+			netdev_link_up ( netdev );
+	} else {
+		if ( netdev_link_ok ( netdev ) )
+			netdev_link_down ( netdev );
+	}
+}
+
+/**
+ * Get link status
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_check_link ( struct dm96xx_device *dm96xx ) {
+	int nsr;
+	int rc;
+
+	/* Read network status register */
+	nsr = dm96xx_read_register ( dm96xx, DM96XX_NSR );
+	if ( nsr < 0 ) {
+		rc = nsr;
+		DBGC ( dm96xx, "DM96XX %p could not read network status: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Update link status */
+	dm96xx_link_nsr ( dm96xx, nsr );
+
+	return 0;
+}
+
+/**
+ * Set DM9601-compatible RX header mode
+ *
+ * @v dm96xx		DM96xx device
+ * @ret rc		Return status code
+ */
+static int dm96xx_rx_mode ( struct dm96xx_device *dm96xx ) {
+	int chipr;
+	int mode_ctl;
+	int rc;
+
+	/* Get chip revision */
+	chipr = dm96xx_read_register ( dm96xx, DM96XX_CHIPR );
+	if ( chipr < 0 ) {
+		rc = chipr;
+		DBGC ( dm96xx, "DM96XX %p could not read chip revision: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Do nothing if device is a DM9601 anyway */
+	if ( chipr == DM96XX_CHIPR_9601 )
+		return 0;
+
+	/* Read current mode control */
+	mode_ctl = dm96xx_read_register ( dm96xx, DM96XX_MODE_CTL );
+	if ( mode_ctl < 0 ) {
+		rc = mode_ctl;
+		DBGC ( dm96xx, "DM96XX %p could not read mode control: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Write mode control */
+	mode_ctl &= ~DM96XX_MODE_CTL_MODE;
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_MODE_CTL,
+					    mode_ctl ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write mode control: %s\n",
+		       dm96xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_intr_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.intr );
+	struct net_device *netdev = dm96xx->netdev;
+	struct dm96xx_interrupt *intr;
+	size_t len = iob_len ( iobuf );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p interrupt failed: %s\n",
+		       dm96xx, strerror ( rc ) );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *intr ) ) {
+		DBGC ( dm96xx, "DM96XX %p underlength interrupt:\n", dm96xx );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, -EINVAL );
+		goto done;
+	}
+	intr = iobuf->data;
+
+	/* Update link status */
+	dm96xx_link_nsr ( dm96xx, intr->nsr );
+
+ done:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_intr_operations = {
+	.complete = dm96xx_intr_complete,
+};
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_in_complete ( struct usb_endpoint *ep,
+				 struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.in );
+	struct net_device *netdev = dm96xx->netdev;
+	struct dm96xx_rx_header *header;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open ) {
+		free_iob ( iobuf );
+		return;
+	}
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p bulk IN failed: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < ( sizeof ( *header ) + 4 /* CRC */ ) ) {
+		DBGC ( dm96xx, "DM96XX %p underlength bulk IN\n", dm96xx );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto err;
+	}
+
+	/* Strip header and CRC */
+	header = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *header ) );
+	iob_unput ( iobuf, 4 /* CRC */ );
+
+	/* Check status */
+	if ( header->rsr & ~DM96XX_RSR_MF ) {
+		DBGC ( dm96xx, "DM96XX %p receive error %02x:\n",
+		       dm96xx, header->rsr );
+		DBGC_HDA ( dm96xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+	return;
+
+ err:
+	/* Hand off to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_in_operations = {
+	.complete = dm96xx_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v dm96xx		DM96xx device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int dm96xx_out_transmit ( struct dm96xx_device *dm96xx,
+				 struct io_buffer *iobuf ) {
+	struct dm96xx_tx_header *header;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *header ) ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->len = cpu_to_le16 ( len );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &dm96xx->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void dm96xx_out_complete ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int rc ) {
+	struct dm96xx_device *dm96xx = container_of ( ep, struct dm96xx_device,
+						      usbnet.out );
+	struct net_device *netdev = dm96xx->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations dm96xx_out_operations = {
+	.complete = dm96xx_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int dm96xx_open ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	unsigned int rcr;
+	int rc;
+
+	/* Set DM9601-compatible RX header mode */
+	if ( ( rc = dm96xx_rx_mode ( dm96xx ) ) != 0 )
+		goto err_rx_mode;
+
+	/* Write MAC address */
+	if ( ( rc = dm96xx_write_mac ( dm96xx, netdev->ll_addr ) ) != 0 )
+		goto err_write_mac;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &dm96xx->usbnet ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not open: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Set receive filters */
+	rcr = ( DM96XX_RCR_ALL | DM96XX_RCR_RUNT | DM96XX_RCR_PRMSC |
+		DM96XX_RCR_RXEN );
+	if ( ( rc = dm96xx_write_register ( dm96xx, DM96XX_RCR, rcr ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not write receive filters: "
+		       "%s\n", dm96xx, strerror ( rc ) );
+		goto err_write_rcr;
+	}
+
+	/* Update link status */
+	if ( ( rc = dm96xx_check_link ( dm96xx ) ) != 0 )
+		goto err_check_link;
+
+	return 0;
+
+ err_check_link:
+ err_write_rcr:
+	usbnet_close ( &dm96xx->usbnet );
+ err_open:
+ err_write_mac:
+ err_rx_mode:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void dm96xx_close ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &dm96xx->usbnet );
+
+	/* Reset device */
+	dm96xx_reset ( dm96xx );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int dm96xx_transmit ( struct net_device *netdev,
+			     struct io_buffer *iobuf ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = dm96xx_out_transmit ( dm96xx, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void dm96xx_poll ( struct net_device *netdev ) {
+	struct dm96xx_device *dm96xx = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( dm96xx->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &dm96xx->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+}
+
+/** DM96xx network device operations */
+static struct net_device_operations dm96xx_operations = {
+	.open		= dm96xx_open,
+	.close		= dm96xx_close,
+	.transmit	= dm96xx_transmit,
+	.poll		= dm96xx_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int dm96xx_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct dm96xx_device *dm96xx;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *dm96xx ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &dm96xx_operations );
+	netdev->dev = &func->dev;
+	dm96xx = netdev->priv;
+	memset ( dm96xx, 0, sizeof ( *dm96xx ) );
+	dm96xx->usb = usb;
+	dm96xx->bus = usb->port->hub->bus;
+	dm96xx->netdev = netdev;
+	usbnet_init ( &dm96xx->usbnet, func, &dm96xx_intr_operations,
+		      &dm96xx_in_operations, &dm96xx_out_operations );
+	usb_refill_init ( &dm96xx->usbnet.intr, 0, DM96XX_INTR_MAX_FILL );
+	usb_refill_init ( &dm96xx->usbnet.in, DM96XX_IN_MTU,
+			  DM96XX_IN_MAX_FILL );
+	DBGC ( dm96xx, "DM96XX %p on %s\n", dm96xx, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &dm96xx->usbnet, config ) ) != 0 ) {
+		DBGC ( dm96xx, "DM96XX %p could not describe: %s\n",
+		       dm96xx, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Reset device */
+	if ( ( rc = dm96xx_reset ( dm96xx ) ) != 0 )
+		goto err_reset;
+
+	/* Read MAC address */
+	if ( ( rc = dm96xx_read_mac ( dm96xx, netdev->hw_addr ) ) != 0 )
+		goto err_read_mac;
+
+	/* Get initial link status */
+	if ( ( rc = dm96xx_check_link ( dm96xx ) ) != 0 )
+		goto err_check_link;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, netdev );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_check_link:
+ err_read_mac:
+ err_reset:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void dm96xx_remove ( struct usb_function *func ) {
+	struct net_device *netdev = usb_func_get_drvdata ( func );
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** DM96xx device IDs */
+static struct usb_device_id dm96xx_ids[] = {
+	{
+		.name = "dm9601-corega",
+		.vendor = 0x07aa,
+		.product = 0x9601,
+	},
+	{
+		.name = "dm9601",
+		.vendor = 0x0a46,
+		.product = 0x9601,
+	},
+	{
+		.name = "zt6688",
+		.vendor = 0x0a46,
+		.product = 0x6688,
+	},
+	{
+		.name = "st268",
+		.vendor = 0x0a46,
+		.product = 0x0268,
+	},
+	{
+		.name = "adm8515",
+		.vendor = 0x0a46,
+		.product = 0x8515,
+	},
+	{
+		.name = "dm9601-hirose",
+		.vendor = 0x0a47,
+		.product = 0x9601,
+	},
+	{
+		.name = "dm9601-8101",
+		.vendor = 0x0fe6,
+		.product = 0x8101,
+	},
+	{
+		.name = "dm9601-9700",
+		.vendor = 0x0fe6,
+		.product = 0x9700,
+	},
+	{
+		.name = "dm9000e",
+		.vendor = 0x0a46,
+		.product = 0x9000,
+	},
+	{
+		.name = "dm9620",
+		.vendor = 0x0a46,
+		.product = 0x9620,
+	},
+	{
+		.name = "dm9621A",
+		.vendor = 0x0a46,
+		.product = 0x9621,
+	},
+	{
+		.name = "dm9622",
+		.vendor = 0x0a46,
+		.product = 0x9622,
+	},
+	{
+		.name = "dm962Oa",
+		.vendor = 0x0a46,
+		.product = 0x0269,
+	},
+	{
+		.name = "dm9621a",
+		.vendor = 0x0a46,
+		.product = 0x1269,
+	},
+};
+
+/** Davicom DM96xx driver */
+struct usb_driver dm96xx_driver __usb_driver = {
+	.ids = dm96xx_ids,
+	.id_count = ( sizeof ( dm96xx_ids ) / sizeof ( dm96xx_ids[0] ) ),
+	.probe = dm96xx_probe,
+	.remove = dm96xx_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/dm96xx.h b/qemu/roms/ipxe/src/drivers/net/dm96xx.h
new file mode 100644
index 000000000..43a1a4e30
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/dm96xx.h
@@ -0,0 +1,194 @@
+#ifndef _DM96XX_H
+#define _DM96XX_H
+
+/** @file
+ *
+ * Davicom DM96xx USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/if_ether.h>
+
+/** Read register(s) */
+#define DM96XX_READ_REGISTER					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x00 ) )
+
+/** Write register(s) */
+#define DM96XX_WRITE_REGISTER					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x01 ) )
+
+/** Write single register */
+#define DM96XX_WRITE1_REGISTER					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0x03 ) )
+
+/** Network control register */
+#define DM96XX_NCR 0x00
+#define DM96XX_NCR_RST		0x01	/**< Software reset */
+
+/** Network status register */
+#define DM96XX_NSR 0x01
+#define DM96XX_NSR_LINKST	0x40	/**< Link status */
+
+/** Receive control register */
+#define DM96XX_RCR 0x05
+#define DM96XX_RCR_ALL		0x08	/**< Pass all multicast */
+#define DM96XX_RCR_RUNT		0x04	/**< Pass runt packet */
+#define DM96XX_RCR_PRMSC	0x02	/**< Promiscuous mode */
+#define DM96XX_RCR_RXEN		0x01	/**< RX enable */
+
+/** Receive status register */
+#define DM96XX_RSR 0x06
+#define DM96XX_RSR_MF		0x40	/**< Multicast frame */
+
+/** PHY address registers */
+#define DM96XX_PAR 0x10
+
+/** Chip revision register */
+#define DM96XX_CHIPR 0x2c
+#define DM96XX_CHIPR_9601	0x00	/**< DM9601 */
+#define DM96XX_CHIPR_9620	0x01	/**< DM9620 */
+
+/** RX header control/status register (DM9620+ only) */
+#define DM96XX_MODE_CTL 0x91
+#define DM96XX_MODE_CTL_MODE	0x80	/**< 4-byte header mode */
+
+/** DM96xx interrupt data */
+struct dm96xx_interrupt {
+	/** Network status register */
+	uint8_t nsr;
+	/** Transmit status registers */
+	uint8_t tsr[2];
+	/** Receive status register */
+	uint8_t rsr;
+	/** Receive overflow counter register */
+	uint8_t rocr;
+	/** Receive packet counter */
+	uint8_t rxc;
+	/** Transmit packet counter */
+	uint8_t txc;
+	/** General purpose register */
+	uint8_t gpr;
+} __attribute__ (( packed ));
+
+/** DM96xx receive header */
+struct dm96xx_rx_header {
+	/** Packet status */
+	uint8_t rsr;
+	/** Packet length (excluding this header, including CRC) */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** DM96xx transmit header */
+struct dm96xx_tx_header {
+	/** Packet length (excluding this header) */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** A DM96xx network device */
+struct dm96xx_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+};
+
+/**
+ * Read registers
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_read_registers ( struct dm96xx_device *dm96xx, unsigned int offset,
+			void *data, size_t len ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_READ_REGISTER, 0, offset,
+			     data, len );
+}
+
+/**
+ * Read register
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @ret value		Register value, or negative error
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_read_register ( struct dm96xx_device *dm96xx, unsigned int offset ) {
+	uint8_t value;
+	int rc;
+
+	if ( ( rc = dm96xx_read_registers ( dm96xx, offset, &value,
+					    sizeof ( value ) ) ) != 0 )
+		return rc;
+	return value;
+}
+
+/**
+ * Write registers
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_write_registers ( struct dm96xx_device *dm96xx, unsigned int offset,
+			 void *data, size_t len ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_WRITE_REGISTER, 0, offset,
+			     data, len );
+}
+
+/**
+ * Write register
+ *
+ * @v dm96xx		DM96xx device
+ * @v offset		Register offset
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+dm96xx_write_register ( struct dm96xx_device *dm96xx, unsigned int offset,
+			uint8_t value ) {
+
+	return usb_control ( dm96xx->usb, DM96XX_WRITE1_REGISTER, value,
+			     offset, NULL, 0 );
+}
+
+/** Reset delay (in microseconds) */
+#define DM96XX_RESET_DELAY_US 10
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define DM96XX_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define DM96XX_IN_MAX_FILL 8
+
+/** Bulk IN buffer size */
+#define DM96XX_IN_MTU					\
+	( 4 /* DM96xx header */ + ETH_FRAME_LEN +	\
+	  4 /* possible VLAN header */ + 4 /* CRC */ )
+
+#endif /* _DM96XX_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/dmfe.c b/qemu/roms/ipxe/src/drivers/net/dmfe.c
index aae40fce7..2ea0d2b2b 100644
--- a/qemu/roms/ipxe/src/drivers/net/dmfe.c
+++ b/qemu/roms/ipxe/src/drivers/net/dmfe.c
@@ -462,7 +462,7 @@ static int dmfe_probe ( struct nic *nic, struct pci_device *pci ) {
 	       pci->id->name, pci->vendor, pci->device);
 
 	/* Read Chip revision */
-	pci_read_config_dword(pci, PCI_REVISION_ID, &dev_rev);
+	pci_read_config_dword(pci, PCI_REVISION, &dev_rev);
 	dprintf(("Revision %lX\n", dev_rev));
 
 	/* point to private storage */
diff --git a/qemu/roms/ipxe/src/drivers/net/ecm.c b/qemu/roms/ipxe/src/drivers/net/ecm.c
new file mode 100644
index 000000000..8c84ea9e9
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/ecm.c
@@ -0,0 +1,520 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <errno.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/base16.h>
+#include <ipxe/profile.h>
+#include <ipxe/usb.h>
+#include "ecm.h"
+
+/** @file
+ *
+ * CDC-ECM USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler ecm_intr_profiler __profiler =
+	{ .name = "ecm.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler ecm_in_profiler __profiler =
+	{ .name = "ecm.in" };
+
+/** Bulk OUT profiler */
+static struct profiler ecm_out_profiler __profiler =
+	{ .name = "ecm.out" };
+
+/******************************************************************************
+ *
+ * Ethernet functional descriptor
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Locate Ethernet functional descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v interface		Interface descriptor
+ * @ret desc		Descriptor, or NULL if not found
+ */
+struct ecm_ethernet_descriptor *
+ecm_ethernet_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface ) {
+	struct ecm_ethernet_descriptor *desc;
+
+	for_each_interface_descriptor ( desc, config, interface ) {
+		if ( ( desc->header.type == USB_CS_INTERFACE_DESCRIPTOR ) &&
+		     ( desc->subtype == CDC_SUBTYPE_ETHERNET ) )
+			return desc;
+	}
+	return NULL;
+}
+
+/**
+ * Get hardware MAC address
+ *
+ * @v usb		USB device
+ * @v desc		Ethernet functional descriptor
+ * @v hw_addr		Hardware address to fill in
+ * @ret rc		Return status code
+ */
+int ecm_fetch_mac ( struct usb_device *usb,
+		    struct ecm_ethernet_descriptor *desc, uint8_t *hw_addr ) {
+	char buf[ base16_encoded_len ( ETH_ALEN ) + 1 /* NUL */ ];
+	int len;
+	int rc;
+
+	/* Fetch MAC address string */
+	len = usb_get_string_descriptor ( usb, desc->mac, 0, buf,
+					  sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	/* Sanity check */
+	if ( len != ( ( int ) ( sizeof ( buf ) - 1 /* NUL */ ) ) )
+		return -EINVAL;
+
+	/* Decode MAC address */
+	len = base16_decode ( buf, hw_addr, ETH_ALEN );
+	if ( len < 0 ) {
+		rc = len;
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * CDC-ECM communications interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_intr_complete ( struct usb_endpoint *ep,
+				struct io_buffer *iobuf, int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.intr );
+	struct net_device *netdev = ecm->netdev;
+	struct usb_setup_packet *message;
+	size_t len = iob_len ( iobuf );
+
+	/* Profile completions */
+	profile_start ( &ecm_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Drop packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( ecm, "ECM %p interrupt failed: %s\n",
+		       ecm, strerror ( rc ) );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *message ) ) {
+		DBGC ( ecm, "ECM %p underlength interrupt:\n", ecm );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto error;
+	}
+	message = iobuf->data;
+
+	/* Parse message header */
+	switch ( message->request ) {
+
+	case cpu_to_le16 ( CDC_NETWORK_CONNECTION ) :
+		if ( message->value && ! netdev_link_ok ( netdev ) ) {
+			DBGC ( ecm, "ECM %p link up\n", ecm );
+			netdev_link_up ( netdev );
+		} else if ( netdev_link_ok ( netdev ) && ! message->value ) {
+			DBGC ( ecm, "ECM %p link down\n", ecm );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	case cpu_to_le16 ( CDC_CONNECTION_SPEED_CHANGE ) :
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( ecm, "ECM %p unrecognised interrupt:\n", ecm );
+		DBGC_HDA ( ecm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -ENOTSUP;
+		goto error;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+	profile_stop ( &ecm_intr_profiler );
+
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+	return;
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations ecm_intr_operations = {
+	.complete = ecm_intr_complete,
+};
+
+/******************************************************************************
+ *
+ * CDC-ECM data interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_in_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			      int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.in );
+	struct net_device *netdev = ecm->netdev;
+
+	/* Profile receive completions */
+	profile_start ( &ecm_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( ecm, "ECM %p bulk IN failed: %s\n",
+		       ecm, strerror ( rc ) );
+		goto error;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	profile_stop ( &ecm_in_profiler );
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations ecm_in_operations = {
+	.complete = ecm_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v ecm		CDC-ECM device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ecm_out_transmit ( struct ecm_device *ecm,
+			      struct io_buffer *iobuf ) {
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &ecm_out_profiler );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &ecm->usbnet.out, iobuf, 1 ) ) != 0 )
+		return rc;
+
+	profile_stop ( &ecm_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ecm_out_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			       int rc ) {
+	struct ecm_device *ecm = container_of ( ep, struct ecm_device,
+						usbnet.out );
+	struct net_device *netdev = ecm->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations ecm_out_operations = {
+	.complete = ecm_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int ecm_open ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+	struct usb_device *usb = ecm->usb;
+	unsigned int filter;
+	int rc;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &ecm->usbnet ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not open: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Set packet filter */
+	filter = ( ECM_PACKET_TYPE_PROMISCUOUS |
+		   ECM_PACKET_TYPE_ALL_MULTICAST |
+		   ECM_PACKET_TYPE_DIRECTED |
+		   ECM_PACKET_TYPE_BROADCAST );
+	if ( ( rc = usb_control ( usb, ECM_SET_ETHERNET_PACKET_FILTER,
+				  filter, ecm->usbnet.comms, NULL, 0 ) ) != 0 ){
+		DBGC ( ecm, "ECM %p could not set packet filter: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_set_filter;
+	}
+
+	return 0;
+
+ err_set_filter:
+	usbnet_close ( &ecm->usbnet );
+ err_open:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void ecm_close ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &ecm->usbnet );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ecm_transmit ( struct net_device *netdev,
+			  struct io_buffer *iobuf ) {
+	struct ecm_device *ecm = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = ecm_out_transmit ( ecm, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void ecm_poll ( struct net_device *netdev ) {
+	struct ecm_device *ecm = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( ecm->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &ecm->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+}
+
+/** CDC-ECM network device operations */
+static struct net_device_operations ecm_operations = {
+	.open		= ecm_open,
+	.close		= ecm_close,
+	.transmit	= ecm_transmit,
+	.poll		= ecm_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int ecm_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct ecm_device *ecm;
+	struct usb_interface_descriptor *comms;
+	struct ecm_ethernet_descriptor *ethernet;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *ecm ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &ecm_operations );
+	netdev->dev = &func->dev;
+	ecm = netdev->priv;
+	memset ( ecm, 0, sizeof ( *ecm ) );
+	ecm->usb = usb;
+	ecm->bus = usb->port->hub->bus;
+	ecm->netdev = netdev;
+	usbnet_init ( &ecm->usbnet, func, &ecm_intr_operations,
+		      &ecm_in_operations, &ecm_out_operations );
+	usb_refill_init ( &ecm->usbnet.intr, 0, ECM_INTR_MAX_FILL );
+	usb_refill_init ( &ecm->usbnet.in, ECM_IN_MTU, ECM_IN_MAX_FILL );
+	DBGC ( ecm, "ECM %p on %s\n", ecm, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &ecm->usbnet, config ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not describe: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Locate Ethernet descriptor */
+	comms = usb_interface_descriptor ( config, ecm->usbnet.comms, 0 );
+	assert ( comms != NULL );
+	ethernet = ecm_ethernet_descriptor ( config, comms );
+	if ( ! ethernet ) {
+		DBGC ( ecm, "ECM %p has no Ethernet descriptor\n", ecm );
+		rc = -EINVAL;
+		goto err_ethernet;
+	}
+
+	/* Fetch MAC address */
+	if ( ( rc = ecm_fetch_mac ( usb, ethernet, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( ecm, "ECM %p could not fetch MAC address: %s\n",
+		       ecm, strerror ( rc ) );
+		goto err_fetch_mac;
+	}
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, ecm );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_fetch_mac:
+ err_ethernet:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void ecm_remove ( struct usb_function *func ) {
+	struct ecm_device *ecm = usb_func_get_drvdata ( func );
+	struct net_device *netdev = ecm->netdev;
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** CDC-ECM device IDs */
+static struct usb_device_id ecm_ids[] = {
+	{
+		.name = "cdc-ecm",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_CDC,
+			.subclass = USB_SUBCLASS_CDC_ECM,
+			.protocol = 0,
+		},
+	},
+};
+
+/** CDC-ECM driver */
+struct usb_driver ecm_driver __usb_driver = {
+	.ids = ecm_ids,
+	.id_count = ( sizeof ( ecm_ids ) / sizeof ( ecm_ids[0] ) ),
+	.probe = ecm_probe,
+	.remove = ecm_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/ecm.h b/qemu/roms/ipxe/src/drivers/net/ecm.h
new file mode 100644
index 000000000..83d324bdc
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/ecm.h
@@ -0,0 +1,93 @@
+#ifndef _ECM_H
+#define _ECM_H
+
+/** @file
+ *
+ * CDC-ECM USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/cdc.h>
+
+/** CDC-ECM subclass */
+#define USB_SUBCLASS_CDC_ECM 0x06
+
+/** Set Ethernet packet filter */
+#define ECM_SET_ETHERNET_PACKET_FILTER					\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x43 ) )
+
+/** Ethernet packet types */
+enum ecm_ethernet_packet_filter {
+	/** Promiscuous mode */
+	ECM_PACKET_TYPE_PROMISCUOUS = 0x0001,
+	/** All multicast packets */
+	ECM_PACKET_TYPE_ALL_MULTICAST = 0x0002,
+	/** Unicast packets */
+	ECM_PACKET_TYPE_DIRECTED = 0x0004,
+	/** Broadcast packets */
+	ECM_PACKET_TYPE_BROADCAST = 0x0008,
+	/** Specified multicast packets */
+	ECM_PACKET_TYPE_MULTICAST = 0x0010,
+};
+
+/** An Ethernet Functional Descriptor */
+struct ecm_ethernet_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Descriptor subtype */
+	uint8_t subtype;
+	/** MAC address string */
+	uint8_t mac;
+	/** Ethernet statistics bitmap */
+	uint32_t statistics;
+	/** Maximum segment size */
+	uint16_t mtu;
+	/** Multicast filter configuration */
+	uint16_t mcast;
+	/** Number of wake-on-LAN filters */
+	uint8_t wol;
+} __attribute__ (( packed ));
+
+/** A CDC-ECM network device */
+struct ecm_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+};
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define ECM_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define ECM_IN_MAX_FILL 8
+
+/** Bulk IN buffer size
+ *
+ * This is a policy decision.
+ */
+#define ECM_IN_MTU ( ETH_FRAME_LEN + 4 /* possible VLAN header */ )
+
+extern struct ecm_ethernet_descriptor *
+ecm_ethernet_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface );
+extern int ecm_fetch_mac ( struct usb_device *usb,
+			   struct ecm_ethernet_descriptor *desc,
+			   uint8_t *hw_addr );
+
+#endif /* _ECM_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/eepro.c b/qemu/roms/ipxe/src/drivers/net/eepro.c
index 909482bcc..97b4c4061 100644
--- a/qemu/roms/ipxe/src/drivers/net/eepro.c
+++ b/qemu/roms/ipxe/src/drivers/net/eepro.c
@@ -27,8 +27,18 @@ has 34 pins, the top row of 2 are not used.
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -591,9 +601,9 @@ static int eepro_probe ( struct nic *nic, struct isa_device *isa ) {
 		l_eepro = 0;
 		name = "Intel 82595-based LAN card";
 	}
-	station_addr.saddr[0] = swap16(station_addr.saddr[0]);
-	station_addr.saddr[1] = swap16(station_addr.saddr[1]);
-	station_addr.saddr[2] = swap16(station_addr.saddr[2]);
+	station_addr.saddr[0] = bswap_16(station_addr.saddr[0]);
+	station_addr.saddr[1] = bswap_16(station_addr.saddr[1]);
+	station_addr.saddr[2] = bswap_16(station_addr.saddr[2]);
 	for (i = 0; i < ETH_ALEN; i++) {
 		nic->node_addr[i] = station_addr.caddr[i];
 	}
diff --git a/qemu/roms/ipxe/src/drivers/net/eepro100.c b/qemu/roms/ipxe/src/drivers/net/eepro100.c
index ede0a1a4b..1046cda39 100644
--- a/qemu/roms/ipxe/src/drivers/net/eepro100.c
+++ b/qemu/roms/ipxe/src/drivers/net/eepro100.c
@@ -1136,7 +1136,6 @@ PCI_ROM(0x8086, 0x2449, "82562em",       "Intel EtherExpressPro100 82562EM", 0),
 PCI_ROM(0x8086, 0x2459, "82562-1",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x245d, "82562-2",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x1050, "82562ez",       "Intel 82562EZ Network Connection", 0),
-PCI_ROM(0x8086, 0x1051, "eepro100-1051", "Intel 82801EB/ER (ICH5/ICH5R) Chipset Ethernet Controller", 0),
 PCI_ROM(0x8086, 0x1065, "82562-3",       "Intel 82562 based Fast Ethernet Connection", 0),
 PCI_ROM(0x8086, 0x5200, "eepro100-5200", "Intel EtherExpress PRO/100 Intelligent Server", 0),
 PCI_ROM(0x8086, 0x5201, "eepro100-5201", "Intel EtherExpress PRO/100 Intelligent Server", 0),
diff --git a/qemu/roms/ipxe/src/drivers/net/efi/nii.c b/qemu/roms/ipxe/src/drivers/net/efi/nii.c
index d0d7da95a..b91848f5c 100644
--- a/qemu/roms/ipxe/src/drivers/net/efi/nii.c
+++ b/qemu/roms/ipxe/src/drivers/net/efi/nii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <strings.h>
@@ -168,6 +172,9 @@ struct nii_nic {
 	/** Saved task priority level */
 	EFI_TPL saved_tpl;
 
+	/** Media status is supported */
+	int media;
+
 	/** Current transmit buffer */
 	struct io_buffer *txbuf;
 	/** Current receive buffer */
@@ -408,6 +415,13 @@ static int nii_issue_cpb_db ( struct nii_nic *nii, unsigned int op, void *cpb,
 	cdb.IFnum = nii->nii->IfNum;
 
 	/* Issue command */
+	DBGC2 ( nii, "NII %s issuing %02x:%04x ifnum %d%s%s\n",
+		nii->dev.name, cdb.OpCode, cdb.OpFlags, cdb.IFnum,
+		( cpb ? " cpb" : "" ), ( db ? " db" : "" ) );
+	if ( cpb )
+		DBGC2_HD ( nii, cpb, cpb_len );
+	if ( db )
+		DBGC2_HD ( nii, db, db_len );
 	nii->issue ( ( intptr_t ) &cdb );
 
 	/* Check completion status */
@@ -552,6 +566,7 @@ static int nii_get_init_info ( struct nii_nic *nii,
 	nii->buffer_len = db.MemoryRequired;
 	nii->mtu = ( db.FrameDataLen + db.MediaHeaderLen );
 	netdev->max_pkt_len = nii->mtu;
+	nii->media = ( stat & PXE_STATFLAGS_GET_STATUS_NO_MEDIA_SUPPORTED );
 
 	return 0;
 }
@@ -560,10 +575,12 @@ static int nii_get_init_info ( struct nii_nic *nii,
  * Initialise UNDI
  *
  * @v nii		NII NIC
+ * @v flags		Flags
  * @ret rc		Return status code
  */
-static int nii_initialise ( struct nii_nic *nii ) {
+static int nii_initialise_flags ( struct nii_nic *nii, unsigned int flags ) {
 	PXE_CPB_INITIALIZE cpb;
+	PXE_DB_INITIALIZE db;
 	unsigned int op;
 	int stat;
 	int rc;
@@ -580,10 +597,13 @@ static int nii_initialise ( struct nii_nic *nii ) {
 	cpb.MemoryAddr = ( ( intptr_t ) nii->buffer );
 	cpb.MemoryLength = nii->buffer_len;
 
+	/* Construct data block */
+	memset ( &db, 0, sizeof ( db ) );
+
 	/* Issue command */
-	op = NII_OP ( PXE_OPCODE_INITIALIZE,
-		      PXE_OPFLAGS_INITIALIZE_DO_NOT_DETECT_CABLE );
-	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_INITIALIZE, flags );
+	if ( ( stat = nii_issue_cpb_db ( nii, op, &cpb, sizeof ( cpb ),
+					 &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not initialise: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -599,6 +619,36 @@ static int nii_initialise ( struct nii_nic *nii ) {
 }
 
 /**
+ * Initialise UNDI
+ *
+ * @v nii		NII NIC
+ * @ret rc		Return status code
+ */
+static int nii_initialise ( struct nii_nic *nii ) {
+	unsigned int flags;
+
+	/* Initialise UNDI */
+	flags = PXE_OPFLAGS_INITIALIZE_DO_NOT_DETECT_CABLE;
+	return nii_initialise_flags ( nii, flags );
+}
+
+/**
+ * Initialise UNDI and detect cable
+ *
+ * @v nii		NII NIC
+ * @ret rc		Return status code
+ */
+static int nii_initialise_and_detect ( struct nii_nic *nii ) {
+	unsigned int flags;
+
+	/* Initialise UNDI and detect cable.  This is required to work
+	 * around bugs in some Emulex NII drivers.
+	 */
+	flags = PXE_OPFLAGS_INITIALIZE_DETECT_CABLE;
+	return nii_initialise_flags ( nii, flags );
+}
+
+/**
  * Shut down UNDI
  *
  * @v nii		NII NIC
@@ -630,6 +680,7 @@ static void nii_shutdown ( struct nii_nic *nii ) {
 static int nii_get_station_address ( struct nii_nic *nii,
 				     struct net_device *netdev ) {
 	PXE_DB_STATION_ADDRESS db;
+	unsigned int op;
 	int stat;
 	int rc;
 
@@ -638,8 +689,9 @@ static int nii_get_station_address ( struct nii_nic *nii,
 		goto err_initialise;
 
 	/* Issue command */
-	if ( ( stat = nii_issue_db ( nii, PXE_OPCODE_STATION_ADDRESS, &db,
-				     sizeof ( db ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_STATION_ADDRESS,
+		      PXE_OPFLAGS_STATION_ADDRESS_READ );
+	if ( ( stat = nii_issue_db ( nii, op, &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not get station address: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -669,18 +721,25 @@ static int nii_get_station_address ( struct nii_nic *nii,
  */
 static int nii_set_station_address ( struct nii_nic *nii,
 				     struct net_device *netdev ) {
+	uint32_t implementation = nii->undi->Implementation;
 	PXE_CPB_STATION_ADDRESS cpb;
+	unsigned int op;
 	int stat;
 	int rc;
 
+	/* Fail if setting station address is unsupported */
+	if ( ! ( implementation & PXE_ROMID_IMP_STATION_ADDR_SETTABLE ) )
+		return -ENOTSUP;
+
 	/* Construct parameter block */
 	memset ( &cpb, 0, sizeof ( cpb ) );
 	memcpy ( cpb.StationAddr, netdev->ll_addr,
 		 netdev->ll_protocol->ll_addr_len );
 
 	/* Issue command */
-	if ( ( stat = nii_issue_cpb ( nii, PXE_OPCODE_STATION_ADDRESS,
-				      &cpb, sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_STATION_ADDRESS,
+	              PXE_OPFLAGS_STATION_ADDRESS_WRITE );
+	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not set station address: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -697,21 +756,28 @@ static int nii_set_station_address ( struct nii_nic *nii,
  * @ret rc		Return status code
  */
 static int nii_set_rx_filters ( struct nii_nic *nii ) {
+	uint32_t implementation = nii->undi->Implementation;
+	unsigned int flags;
 	unsigned int op;
 	int stat;
 	int rc;
 
+	/* Construct receive filter set */
+	flags = ( PXE_OPFLAGS_RECEIVE_FILTER_ENABLE |
+		  PXE_OPFLAGS_RECEIVE_FILTER_UNICAST );
+	if ( implementation & PXE_ROMID_IMP_BROADCAST_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_BROADCAST;
+	if ( implementation & PXE_ROMID_IMP_PROMISCUOUS_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_PROMISCUOUS;
+	if ( implementation & PXE_ROMID_IMP_PROMISCUOUS_MULTICAST_RX_SUPPORTED )
+		flags |= PXE_OPFLAGS_RECEIVE_FILTER_ALL_MULTICAST;
+
 	/* Issue command */
-	op = NII_OP ( PXE_OPCODE_RECEIVE_FILTERS,
-		      ( PXE_OPFLAGS_RECEIVE_FILTER_ENABLE |
-			PXE_OPFLAGS_RECEIVE_FILTER_UNICAST |
-			PXE_OPFLAGS_RECEIVE_FILTER_BROADCAST |
-			PXE_OPFLAGS_RECEIVE_FILTER_PROMISCUOUS |
-			PXE_OPFLAGS_RECEIVE_FILTER_ALL_MULTICAST ) );
+	op = NII_OP ( PXE_OPCODE_RECEIVE_FILTERS, flags );
 	if ( ( stat = nii_issue ( nii, op ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
-		DBGC ( nii, "NII %s could not set receive filters: %s\n",
-		       nii->dev.name, strerror ( rc ) );
+		DBGC ( nii, "NII %s could not set receive filters %#04x: %s\n",
+		       nii->dev.name, flags, strerror ( rc ) );
 		return rc;
 	}
 
@@ -729,6 +795,7 @@ static int nii_transmit ( struct net_device *netdev,
 			  struct io_buffer *iobuf ) {
 	struct nii_nic *nii = netdev->priv;
 	PXE_CPB_TRANSMIT cpb;
+	unsigned int op;
 	int stat;
 	int rc;
 
@@ -745,8 +812,10 @@ static int nii_transmit ( struct net_device *netdev,
 	cpb.MediaheaderLen = netdev->ll_protocol->ll_header_len;
 
 	/* Transmit packet */
-	if ( ( stat = nii_issue_cpb ( nii, PXE_OPCODE_TRANSMIT, &cpb,
-				      sizeof ( cpb ) ) ) < 0 ) {
+	op = NII_OP ( PXE_OPCODE_TRANSMIT,
+		      ( PXE_OPFLAGS_TRANSMIT_WHOLE |
+			PXE_OPFLAGS_TRANSMIT_DONT_BLOCK ) );
+	if ( ( stat = nii_issue_cpb ( nii, op, &cpb, sizeof ( cpb ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not transmit: %s\n",
 		       nii->dev.name, strerror ( rc ) );
@@ -772,12 +841,7 @@ static void nii_poll_tx ( struct net_device *netdev, unsigned int stat ) {
 		return;
 
 	/* Sanity check */
-	if ( ! nii->txbuf ) {
-		DBGC ( nii, "NII %s reported spurious TX completion\n",
-		       nii->dev.name );
-		netdev_tx_err ( netdev, NULL, -EPIPE );
-		return;
-	}
+	assert ( nii->txbuf != NULL );
 
 	/* Complete transmission */
 	iobuf = nii->txbuf;
@@ -869,11 +933,14 @@ static void nii_poll ( struct net_device *netdev ) {
 	int stat;
 	int rc;
 
+	/* Construct data block */
+	memset ( &db, 0, sizeof ( db ) );
+
 	/* Get status */
 	op = NII_OP ( PXE_OPCODE_GET_STATUS,
 		      ( PXE_OPFLAGS_GET_INTERRUPT_STATUS |
-			PXE_OPFLAGS_GET_TRANSMITTED_BUFFERS |
-			PXE_OPFLAGS_GET_MEDIA_STATUS ) );
+			( nii->txbuf ? PXE_OPFLAGS_GET_TRANSMITTED_BUFFERS : 0)|
+			( nii->media ? PXE_OPFLAGS_GET_MEDIA_STATUS : 0 ) ) );
 	if ( ( stat = nii_issue_db ( nii, op, &db, sizeof ( db ) ) ) < 0 ) {
 		rc = -EIO_STAT ( stat );
 		DBGC ( nii, "NII %s could not get status: %s\n",
@@ -882,13 +949,15 @@ static void nii_poll ( struct net_device *netdev ) {
 	}
 
 	/* Process any TX completions */
-	nii_poll_tx ( netdev, stat );
+	if ( nii->txbuf )
+		nii_poll_tx ( netdev, stat );
 
 	/* Process any RX completions */
 	nii_poll_rx ( netdev );
 
 	/* Check for link state changes */
-	nii_poll_link ( netdev, stat );
+	if ( nii->media )
+		nii_poll_link ( netdev, stat );
 }
 
 /**
@@ -901,8 +970,18 @@ static int nii_open ( struct net_device *netdev ) {
 	struct nii_nic *nii = netdev->priv;
 	int rc;
 
-	/* Initialise NIC */
-	if ( ( rc = nii_initialise ( nii ) ) != 0 )
+	/* Initialise NIC
+	 *
+	 * Some Emulex NII drivers have a bug which prevents packets
+	 * from being sent or received unless we specifically ask it
+	 * to detect cable presence during initialisation.  Work
+	 * around these buggy drivers by requesting cable detection at
+	 * this point, even though we don't care about link state here
+	 * (and would prefer to have the NIC initialise even if no
+	 * cable is present, to match the behaviour of all other iPXE
+	 * drivers).
+	 */
+	if ( ( rc = nii_initialise_and_detect ( nii ) ) != 0 )
 		goto err_initialise;
 
 	/* Attempt to set station address */
@@ -1023,8 +1102,9 @@ int nii_start ( struct efi_device *efidev ) {
 		nii->issue = ( ( ( void * ) nii->undi ) +
 			       nii->undi->EntryPoint );
 	}
-	DBGC ( nii, "NII %s using UNDI v%x.%x at %p entry %p\n", nii->dev.name,
-	       nii->nii->MajorVer, nii->nii->MinorVer, nii->undi, nii->issue );
+	DBGC ( nii, "NII %s using UNDI v%x.%x at %p entry %p impl %#08x\n",
+	       nii->dev.name, nii->nii->MajorVer, nii->nii->MinorVer,
+	       nii->undi, nii->issue, nii->undi->Implementation );
 
 	/* Open PCI I/O protocols and locate BARs */
 	if ( ( rc = nii_pci_open ( nii ) ) != 0 )
@@ -1048,6 +1128,10 @@ int nii_start ( struct efi_device *efidev ) {
 	DBGC ( nii, "NII %s registered as %s for %p %s\n", nii->dev.name,
 	       netdev->name, device, efi_handle_name ( device ) );
 
+	/* Set initial link state (if media detection is not supported) */
+	if ( ! nii->media )
+		netdev_link_up ( netdev );
+
 	return 0;
 
 	unregister_netdev ( netdev );
diff --git a/qemu/roms/ipxe/src/drivers/net/efi/nii.h b/qemu/roms/ipxe/src/drivers/net/efi/nii.h
index de0ac687b..c10be9db5 100644
--- a/qemu/roms/ipxe/src/drivers/net/efi/nii.h
+++ b/qemu/roms/ipxe/src/drivers/net/efi/nii.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct efi_device;
 
diff --git a/qemu/roms/ipxe/src/drivers/net/efi/snp.c b/qemu/roms/ipxe/src/drivers/net/efi/snp.c
index 2b5fc8618..acfcfba9f 100644
--- a/qemu/roms/ipxe/src/drivers/net/efi/snp.c
+++ b/qemu/roms/ipxe/src/drivers/net/efi/snp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/efi/efi.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/efi/snponly.c b/qemu/roms/ipxe/src/drivers/net/efi/snponly.c
index 99f264bca..73abfdbf4 100644
--- a/qemu/roms/ipxe/src/drivers/net/efi/snponly.c
+++ b/qemu/roms/ipxe/src/drivers/net/efi/snponly.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/etherfabric.c b/qemu/roms/ipxe/src/drivers/net/etherfabric.c
index 5e0efb1e1..29d117443 100644
--- a/qemu/roms/ipxe/src/drivers/net/etherfabric.c
+++ b/qemu/roms/ipxe/src/drivers/net/etherfabric.c
@@ -3176,7 +3176,7 @@ falcon_probe_nic_variant ( struct efab_nic *efab, struct pci_device *pci )
 	uint8_t revision;
 
 	/* PCI revision */
-	pci_read_config_byte ( pci, PCI_CLASS_REVISION, &revision );
+	pci_read_config_byte ( pci, PCI_REVISION, &revision );
 	efab->pci_revision = revision;
 
 	/* Asic vs FPGA */
diff --git a/qemu/roms/ipxe/src/drivers/net/forcedeth.c b/qemu/roms/ipxe/src/drivers/net/forcedeth.c
index d8ece9a7a..79938cbbb 100644
--- a/qemu/roms/ipxe/src/drivers/net/forcedeth.c
+++ b/qemu/roms/ipxe/src/drivers/net/forcedeth.c
@@ -1749,10 +1749,8 @@ forcedeth_map_regs ( struct forcedeth_private *priv )
 	for ( reg = PCI_BASE_ADDRESS_0; reg <= PCI_BASE_ADDRESS_5; reg += 4 ) {
 		pci_read_config_dword ( priv->pci_dev, reg, &bar );
 
-		if ( ( ( bar & PCI_BASE_ADDRESS_SPACE ) ==
-			 PCI_BASE_ADDRESS_SPACE_MEMORY ) &&
-		       ( pci_bar_size ( priv->pci_dev, reg ) >=
-			 register_size ) ) {
+		if ( ( ! ( bar & PCI_BASE_ADDRESS_SPACE_IO ) ) &&
+		     ( pci_bar_size ( priv->pci_dev, reg ) >= register_size ) ){
 			addr = pci_bar_start ( priv->pci_dev, reg );
 			break;
 		}
diff --git a/qemu/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c b/qemu/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
index aace5ad56..fc7021c38 100644
--- a/qemu/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
+++ b/qemu/roms/ipxe/src/drivers/net/igbvf/igbvf_main.c
@@ -461,7 +461,7 @@ static int __devinit igbvf_sw_init ( struct igbvf_adapter *adapter )
         hw->vendor_id = pdev->vendor;
         hw->device_id = pdev->device;
 
-        pci_read_config_byte ( pdev, PCI_REVISION_ID, &hw->revision_id );
+        pci_read_config_byte ( pdev, PCI_REVISION, &hw->revision_id );
 
         pci_read_config_word ( pdev, PCI_COMMAND, &hw->bus.pci_cmd_word );
 
diff --git a/qemu/roms/ipxe/src/drivers/net/intel.c b/qemu/roms/ipxe/src/drivers/net/intel.c
index a89f947b2..6309e9aa5 100644
--- a/qemu/roms/ipxe/src/drivers/net/intel.c
+++ b/qemu/roms/ipxe/src/drivers/net/intel.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -248,32 +252,6 @@ static int intel_fetch_mac ( struct intel_nic *intel, uint8_t *hw_addr ) {
 
 /******************************************************************************
  *
- * Diagnostics
- *
- ******************************************************************************
- */
-
-/**
- * Dump diagnostic information
- *
- * @v intel		Intel device
- */
-static void __attribute__ (( unused )) intel_diag ( struct intel_nic *intel ) {
-
-	DBGC ( intel, "INTEL %p TX %04x(%02x)/%04x(%02x) "
-	       "RX %04x(%02x)/%04x(%02x)\n", intel,
-	       ( intel->tx.cons & 0xffff ),
-	       readl ( intel->regs + intel->tx.reg + INTEL_xDH ),
-	       ( intel->tx.prod & 0xffff ),
-	       readl ( intel->regs + intel->tx.reg + INTEL_xDT ),
-	       ( intel->rx.cons & 0xffff ),
-	       readl ( intel->regs + intel->rx.reg + INTEL_xDH ),
-	       ( intel->rx.prod & 0xffff ),
-	       readl ( intel->regs + intel->rx.reg + INTEL_xDT ) );
-}
-
-/******************************************************************************
- *
  * Device reset
  *
  ******************************************************************************
@@ -371,6 +349,67 @@ static void intel_check_link ( struct net_device *netdev ) {
 
 /******************************************************************************
  *
+ * Descriptors
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Populate transmit descriptor
+ *
+ * @v tx		Transmit descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_tx ( struct intel_descriptor *tx, physaddr_t addr,
+			 size_t len ) {
+
+	/* Populate transmit descriptor */
+	tx->address = cpu_to_le64 ( addr );
+	tx->length = cpu_to_le16 ( len );
+	tx->flags = 0;
+	tx->command = ( INTEL_DESC_CMD_RS | INTEL_DESC_CMD_IFCS |
+			INTEL_DESC_CMD_EOP );
+	tx->status = 0;
+}
+
+/**
+ * Populate advanced transmit descriptor
+ *
+ * @v tx		Transmit descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_tx_adv ( struct intel_descriptor *tx, physaddr_t addr,
+			     size_t len ) {
+
+	/* Populate advanced transmit descriptor */
+	tx->address = cpu_to_le64 ( addr );
+	tx->length = cpu_to_le16 ( len );
+	tx->flags = INTEL_DESC_FL_DTYP_DATA;
+	tx->command = ( INTEL_DESC_CMD_DEXT | INTEL_DESC_CMD_RS |
+			INTEL_DESC_CMD_IFCS | INTEL_DESC_CMD_EOP );
+	tx->status = cpu_to_le32 ( INTEL_DESC_STATUS_PAYLEN ( len ) );
+}
+
+/**
+ * Populate receive descriptor
+ *
+ * @v rx		Receive descriptor
+ * @v addr		Data buffer address
+ * @v len		Length of data
+ */
+void intel_describe_rx ( struct intel_descriptor *rx, physaddr_t addr,
+			 size_t len __unused ) {
+
+	/* Populate transmit descriptor */
+	rx->address = cpu_to_le64 ( addr );
+	rx->length = 0;
+	rx->status = 0;
+}
+
+/******************************************************************************
+ *
  * Network device interface
  *
  ******************************************************************************
@@ -479,10 +518,7 @@ void intel_refill_rx ( struct intel_nic *intel ) {
 
 		/* Populate receive descriptor */
 		address = virt_to_bus ( iobuf->data );
-		rx->address = cpu_to_le64 ( address );
-		rx->length = 0;
-		rx->status = 0;
-		rx->errors = 0;
+		intel->rx.describe ( rx, address, 0 );
 
 		/* Record I/O buffer */
 		assert ( intel->rx_iobuf[rx_idx] == NULL );
@@ -568,6 +604,13 @@ static int intel_open ( struct net_device *netdev ) {
 	/* Update link state */
 	intel_check_link ( netdev );
 
+	/* Apply required errata */
+	if ( intel->flags & INTEL_VMWARE ) {
+		DBGC ( intel, "INTEL %p applying VMware errata workaround\n",
+		       intel );
+		intel->force_icr = INTEL_IRQ_RXT0;
+	}
+
 	return 0;
 
 	intel_destroy_ring ( intel, &intel->rx );
@@ -617,6 +660,7 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 	unsigned int tx_idx;
 	unsigned int tx_tail;
 	physaddr_t address;
+	size_t len;
 
 	/* Get next transmit descriptor */
 	if ( ( intel->tx.prod - intel->tx.cons ) >= INTEL_TX_FILL ) {
@@ -629,11 +673,8 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 
 	/* Populate transmit descriptor */
 	address = virt_to_bus ( iobuf->data );
-	tx->address = cpu_to_le64 ( address );
-	tx->length = cpu_to_le16 ( iob_len ( iobuf ) );
-	tx->command = ( INTEL_DESC_CMD_RS | INTEL_DESC_CMD_IFCS |
-			INTEL_DESC_CMD_EOP );
-	tx->status = 0;
+	len = iob_len ( iobuf );
+	intel->tx.describe ( tx, address, len );
 	wmb();
 
 	/* Notify card that there are packets ready to transmit */
@@ -644,7 +685,7 @@ int intel_transmit ( struct net_device *netdev, struct io_buffer *iobuf ) {
 
 	DBGC2 ( intel, "INTEL %p TX %d is [%llx,%llx)\n", intel, tx_idx,
 		( ( unsigned long long ) address ),
-		( ( unsigned long long ) address + iob_len ( iobuf ) ) );
+		( ( unsigned long long ) address + len ) );
 
 	return 0;
 }
@@ -667,7 +708,7 @@ void intel_poll_tx ( struct net_device *netdev ) {
 		tx = &intel->tx.desc[tx_idx];
 
 		/* Stop if descriptor is still in use */
-		if ( ! ( tx->status & INTEL_DESC_STATUS_DD ) )
+		if ( ! ( tx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
 			return;
 
 		DBGC2 ( intel, "INTEL %p TX %d complete\n", intel, tx_idx );
@@ -698,7 +739,7 @@ void intel_poll_rx ( struct net_device *netdev ) {
 		rx = &intel->rx.desc[rx_idx];
 
 		/* Stop if descriptor is still in use */
-		if ( ! ( rx->status & INTEL_DESC_STATUS_DD ) )
+		if ( ! ( rx->status & cpu_to_le32 ( INTEL_DESC_STATUS_DD ) ) )
 			return;
 
 		/* Populate I/O buffer */
@@ -708,10 +749,10 @@ void intel_poll_rx ( struct net_device *netdev ) {
 		iob_put ( iobuf, len );
 
 		/* Hand off to network stack */
-		if ( rx->errors ) {
+		if ( rx->status & cpu_to_le32 ( INTEL_DESC_STATUS_RXE ) ) {
 			DBGC ( intel, "INTEL %p RX %d error (length %zd, "
-			       "errors %02x)\n",
-			       intel, rx_idx, len, rx->errors );
+			       "status %08x)\n", intel, rx_idx, len,
+			       le32_to_cpu ( rx->status ) );
 			netdev_rx_err ( netdev, iobuf, -EIO );
 		} else {
 			DBGC2 ( intel, "INTEL %p RX %d complete (length %zd)\n",
@@ -736,6 +777,7 @@ static void intel_poll ( struct net_device *netdev ) {
 	icr = readl ( intel->regs + INTEL_ICR );
 	profile_stop ( &intel_vm_poll_profiler );
 	profile_exclude ( &intel_vm_poll_profiler );
+	icr |= intel->force_icr;
 	if ( ! icr )
 		return;
 
@@ -755,6 +797,14 @@ static void intel_poll ( struct net_device *netdev ) {
 	if ( icr & INTEL_IRQ_LSC )
 		intel_check_link ( netdev );
 
+	/* Check for unexpected interrupts */
+	if ( icr & ~( INTEL_IRQ_TXDW | INTEL_IRQ_TXQE | INTEL_IRQ_LSC |
+		      INTEL_IRQ_RXDMT0 | INTEL_IRQ_RXT0 | INTEL_IRQ_RXO ) ) {
+		DBGC ( intel, "INTEL %p unexpected ICR %08x\n", intel, icr );
+		/* Report as a TX error */
+		netdev_tx_err ( netdev, NULL, -ENOTSUP );
+	}
+
 	/* Refill RX ring */
 	intel_refill_rx ( intel );
 }
@@ -817,8 +867,10 @@ static int intel_probe ( struct pci_device *pci ) {
 	memset ( intel, 0, sizeof ( *intel ) );
 	intel->port = PCI_FUNC ( pci->busdevfn );
 	intel->flags = pci->id->driver_data;
-	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTEL_TD );
-	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTEL_RD );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTEL_TD,
+			  intel_describe_tx );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTEL_RD,
+			  intel_describe_rx );
 
 	/* Fix up PCI device */
 	adjust_pci_device ( pci );
@@ -895,7 +947,7 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x100c, "82544gc", "82544GC (Copper)", 0 ),
 	PCI_ROM ( 0x8086, 0x100d, "82544gc-l", "82544GC (LOM)", 0 ),
 	PCI_ROM ( 0x8086, 0x100e, "82540em", "82540EM", 0 ),
-	PCI_ROM ( 0x8086, 0x100f, "82545em", "82545EM (Copper)", 0 ),
+	PCI_ROM ( 0x8086, 0x100f, "82545em", "82545EM (Copper)", INTEL_VMWARE ),
 	PCI_ROM ( 0x8086, 0x1010, "82546eb", "82546EB (Copper)", 0 ),
 	PCI_ROM ( 0x8086, 0x1011, "82545em-f", "82545EM (Fiber)", 0 ),
 	PCI_ROM ( 0x8086, 0x1012, "82546eb-f", "82546EB (Fiber)", 0 ),
@@ -998,6 +1050,12 @@ static struct pci_device_id intel_nics[] = {
 	PCI_ROM ( 0x8086, 0x1533, "i210", "I210", 0 ),
 	PCI_ROM ( 0x8086, 0x153a, "i217lm", "I217-LM", 0 ),
 	PCI_ROM ( 0x8086, 0x153b, "i217v", "I217-V", 0 ),
+	PCI_ROM ( 0x8086, 0x1559, "i218v", "I218-V", 0),
+	PCI_ROM ( 0x8086, 0x155a, "i218lm", "I218-LM", 0),
+	PCI_ROM ( 0x8086, 0x15a0, "i218lm-2", "I218-LM", 0 ),
+	PCI_ROM ( 0x8086, 0x15a1, "i218v-2", "I218-V", 0 ),
+	PCI_ROM ( 0x8086, 0x15a2, "i218lm-3", "I218-LM", 0 ),
+	PCI_ROM ( 0x8086, 0x15a3, "i218v-3", "I218-V", 0 ),
 	PCI_ROM ( 0x8086, 0x294c, "82566dc-2", "82566DC-2", 0 ),
 	PCI_ROM ( 0x8086, 0x2e6e, "cemedia", "CE Media Processor", 0 ),
 };
diff --git a/qemu/roms/ipxe/src/drivers/net/intel.h b/qemu/roms/ipxe/src/drivers/net/intel.h
index 8c4479bb4..ce9e3f467 100644
--- a/qemu/roms/ipxe/src/drivers/net/intel.h
+++ b/qemu/roms/ipxe/src/drivers/net/intel.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
@@ -22,33 +22,38 @@ struct intel_descriptor {
 	uint64_t address;
 	/** Length */
 	uint16_t length;
-	/** Reserved */
-	uint8_t reserved_a;
+	/** Flags */
+	uint8_t flags;
 	/** Command */
 	uint8_t command;
 	/** Status */
-	uint8_t status;
-	/** Errors */
-	uint8_t errors;
-	/** Reserved */
-	uint16_t reserved_b;
+	uint32_t status;
 } __attribute__ (( packed ));
 
-/** Packet descriptor command bits */
-enum intel_descriptor_command {
-	/** Report status */
-	INTEL_DESC_CMD_RS = 0x08,
-	/** Insert frame checksum (CRC) */
-	INTEL_DESC_CMD_IFCS = 0x02,
-	/** End of packet */
-	INTEL_DESC_CMD_EOP = 0x01,
-};
+/** Descriptor type */
+#define INTEL_DESC_FL_DTYP( dtyp ) ( (dtyp) << 4 )
+#define INTEL_DESC_FL_DTYP_DATA INTEL_DESC_FL_DTYP ( 0x03 )
 
-/** Packet descriptor status bits */
-enum intel_descriptor_status {
-	/** Descriptor done */
-	INTEL_DESC_STATUS_DD = 0x01,
-};
+/** Descriptor extension */
+#define INTEL_DESC_CMD_DEXT 0x20
+
+/** Report status */
+#define INTEL_DESC_CMD_RS 0x08
+
+/** Insert frame checksum (CRC) */
+#define INTEL_DESC_CMD_IFCS 0x02
+
+/** End of packet */
+#define INTEL_DESC_CMD_EOP 0x01
+
+/** Descriptor done */
+#define INTEL_DESC_STATUS_DD 0x00000001UL
+
+/** Receive error */
+#define INTEL_DESC_STATUS_RXE 0x00000100UL
+
+/** Payload length */
+#define INTEL_DESC_STATUS_PAYLEN( len ) ( (len) << 14 )
 
 /** Device Control Register */
 #define INTEL_CTRL 0x00000UL
@@ -91,7 +96,9 @@ enum intel_descriptor_status {
 /** Interrupt Cause Read Register */
 #define INTEL_ICR 0x000c0UL
 #define INTEL_IRQ_TXDW		0x00000001UL	/**< Transmit descriptor done */
+#define INTEL_IRQ_TXQE		0x00000002UL	/**< Transmit queue empty */
 #define INTEL_IRQ_LSC		0x00000004UL	/**< Link status change */
+#define INTEL_IRQ_RXDMT0	0x00000010UL	/**< Receive queue low */
 #define INTEL_IRQ_RXT0		0x00000080UL	/**< Receive timer */
 #define INTEL_IRQ_RXO		0x00000400UL	/**< Receive overrun */
 
@@ -207,6 +214,15 @@ struct intel_ring {
 	unsigned int reg;
 	/** Length (in bytes) */
 	size_t len;
+
+	/** Populate descriptor
+	 *
+	 * @v desc		Descriptor
+	 * @v addr		Data buffer address
+	 * @v len		Length of data
+	 */
+	void ( * describe ) ( struct intel_descriptor *desc, physaddr_t addr,
+			      size_t len );
 };
 
 /**
@@ -215,12 +231,39 @@ struct intel_ring {
  * @v ring		Descriptor ring
  * @v count		Number of descriptors
  * @v reg		Descriptor register block
+ * @v describe		Method to populate descriptor
  */
 static inline __attribute__ (( always_inline)) void
-intel_init_ring ( struct intel_ring *ring, unsigned int count,
-		  unsigned int reg ) {
+intel_init_ring ( struct intel_ring *ring, unsigned int count, unsigned int reg,
+		  void ( * describe ) ( struct intel_descriptor *desc,
+					physaddr_t addr, size_t len ) ) {
+
 	ring->len = ( count * sizeof ( ring->desc[0] ) );
 	ring->reg = reg;
+	ring->describe = describe;
+}
+
+/** An Intel virtual function mailbox */
+struct intel_mailbox {
+	/** Mailbox control register */
+	unsigned int ctrl;
+	/** Mailbox memory base */
+	unsigned int mem;
+};
+
+/**
+ * Initialise mailbox
+ *
+ * @v mbox		Mailbox
+ * @v ctrl		Mailbox control register
+ * @v mem		Mailbox memory register base
+ */
+static inline __attribute__ (( always_inline )) void
+intel_init_mbox ( struct intel_mailbox *mbox, unsigned int ctrl,
+		  unsigned int mem ) {
+
+	mbox->ctrl = ctrl;
+	mbox->mem = mem;
 }
 
 /** An Intel network card */
@@ -231,6 +274,8 @@ struct intel_nic {
 	unsigned int port;
 	/** Flags */
 	unsigned int flags;
+	/** Forced interrupts */
+	unsigned int force_icr;
 
 	/** EEPROM */
 	struct nvs_device eeprom;
@@ -239,6 +284,9 @@ struct intel_nic {
 	/** EEPROM address shift */
 	unsigned int eerd_addr_shift;
 
+	/** Mailbox */
+	struct intel_mailbox mbox;
+
 	/** Transmit descriptor ring */
 	struct intel_ring tx;
 	/** Receive descriptor ring */
@@ -251,8 +299,35 @@ struct intel_nic {
 enum intel_flags {
 	/** PBS/PBA errata workaround required */
 	INTEL_PBS_ERRATA = 0x0001,
+	/** VMware missing interrupt workaround required */
+	INTEL_VMWARE = 0x0002,
 };
 
+/**
+ * Dump diagnostic information
+ *
+ * @v intel		Intel device
+ */
+static inline void intel_diag ( struct intel_nic *intel ) {
+
+	DBGC ( intel, "INTEL %p TX %04x(%02x)/%04x(%02x) "
+	       "RX %04x(%02x)/%04x(%02x)\n", intel,
+	       ( intel->tx.cons & 0xffff ),
+	       readl ( intel->regs + intel->tx.reg + INTEL_xDH ),
+	       ( intel->tx.prod & 0xffff ),
+	       readl ( intel->regs + intel->tx.reg + INTEL_xDT ),
+	       ( intel->rx.cons & 0xffff ),
+	       readl ( intel->regs + intel->rx.reg + INTEL_xDH ),
+	       ( intel->rx.prod & 0xffff ),
+	       readl ( intel->regs + intel->rx.reg + INTEL_xDT ) );
+}
+
+extern void intel_describe_tx ( struct intel_descriptor *tx,
+				physaddr_t addr, size_t len );
+extern void intel_describe_tx_adv ( struct intel_descriptor *tx,
+				    physaddr_t addr, size_t len );
+extern void intel_describe_rx ( struct intel_descriptor *rx,
+				physaddr_t addr, size_t len );
 extern int intel_create_ring ( struct intel_nic *intel,
 			       struct intel_ring *ring );
 extern void intel_destroy_ring ( struct intel_nic *intel,
diff --git a/qemu/roms/ipxe/src/drivers/net/intelvf.c b/qemu/roms/ipxe/src/drivers/net/intelvf.c
new file mode 100644
index 000000000..ac6fea745
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/intelvf.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/io.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include "intelvf.h"
+
+/** @file
+ *
+ * Intel 10/100/1000 virtual function network card driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Mailbox messages
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Write message to mailbox
+ *
+ * @v intel		Intel device
+ * @v msg		Message
+ */
+static void intelvf_mbox_write ( struct intel_nic *intel,
+				 const union intelvf_msg *msg ) {
+	unsigned int i;
+
+	/* Write message */
+	DBGC2 ( intel, "INTEL %p sending message", intel );
+	for ( i = 0 ; i < ( sizeof ( *msg ) / sizeof ( msg->dword[0] ) ) ; i++){
+		DBGC2 ( intel, "%c%08x", ( i ? ':' : ' ' ), msg->dword[i] );
+		writel ( msg->dword[i], ( intel->regs + intel->mbox.mem +
+					  ( i * sizeof ( msg->dword[0] ) ) ) );
+	}
+	DBGC2 ( intel, "\n" );
+}
+
+/**
+ * Read message from mailbox
+ *
+ * @v intel		Intel device
+ * @v msg		Message
+ */
+static void intelvf_mbox_read ( struct intel_nic *intel,
+				union intelvf_msg *msg ) {
+	unsigned int i;
+
+	/* Read message */
+	DBGC2 ( intel, "INTEL %p received message", intel );
+	for ( i = 0 ; i < ( sizeof ( *msg ) / sizeof ( msg->dword[0] ) ) ; i++){
+		msg->dword[i] = readl ( intel->regs + intel->mbox.mem +
+					( i * sizeof ( msg->dword[0] ) ) );
+		DBGC2 ( intel, "%c%08x", ( i ? ':' : ' ' ), msg->dword[i] );
+	}
+	DBGC2 ( intel, "\n" );
+}
+
+/**
+ * Poll mailbox
+ *
+ * @v intel		Intel device
+ * @ret rc		Return status code
+ *
+ * Note that polling the mailbox may fail if the underlying PF is
+ * reset.
+ */
+int intelvf_mbox_poll ( struct intel_nic *intel ) {
+	struct intel_mailbox *mbox = &intel->mbox;
+	union intelvf_msg msg;
+	uint32_t ctrl;
+
+	/* Get mailbox status */
+	ctrl = readl ( intel->regs + mbox->ctrl );
+
+	/* Fail if a reset is in progress */
+	if ( ctrl & INTELVF_MBCTRL_RSTI )
+		return -EPIPE;
+
+	/* Acknowledge (and ignore) any received messages */
+	if ( ctrl & INTELVF_MBCTRL_PFSTS ) {
+		intelvf_mbox_read ( intel, &msg );
+		writel ( INTELVF_MBCTRL_ACK, intel->regs + mbox->ctrl );
+	}
+
+	return 0;
+}
+
+/**
+ * Wait for PF reset to complete
+ *
+ * @v intel		Intel device
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_wait ( struct intel_nic *intel ) {
+	unsigned int i;
+	int rc;
+
+	/* Wait until a poll completes successfully */
+	for ( i = 0 ; i < INTELVF_MBOX_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for successful poll */
+		if ( ( rc = intelvf_mbox_poll ( intel ) ) == 0 )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( intel, "INTEL %p timed out waiting for reset\n", intel );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Send/receive mailbox message
+ *
+ * @v intel		Intel device
+ * @v msg		Message buffer
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_msg ( struct intel_nic *intel, union intelvf_msg *msg ) {
+	struct intel_mailbox *mbox = &intel->mbox;
+	uint32_t ctrl;
+	uint32_t seen = 0;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( ! ( msg->hdr & INTELVF_MSG_RESPONSE ) );
+
+	/* Handle mailbox */
+	for ( i = 0 ; i < INTELVF_MBOX_MAX_WAIT_MS ; i++ ) {
+
+		/* Attempt to claim mailbox, if we have not yet sent
+		 * our message.
+		 */
+		if ( ! ( seen & INTELVF_MBCTRL_VFU ) )
+			writel ( INTELVF_MBCTRL_VFU, intel->regs + mbox->ctrl );
+
+		/* Get mailbox status and record observed flags */
+		ctrl = readl ( intel->regs + mbox->ctrl );
+		seen |= ctrl;
+
+		/* If a reset is in progress, clear VFU and abort */
+		if ( ctrl & INTELVF_MBCTRL_RSTI ) {
+			writel ( 0, intel->regs + mbox->ctrl );
+			return -EPIPE;
+		}
+
+		/* Write message to mailbox, if applicable.  This
+		 * potentially overwrites a message sent by the PF (if
+		 * the PF has simultaneously released PFU (thus
+		 * allowing our VFU) and asserted PFSTS), but that
+		 * doesn't really matter since there are no
+		 * unsolicited PF->VF messages that require the actual
+		 * message content to be observed.
+		 */
+		if ( ctrl & INTELVF_MBCTRL_VFU )
+			intelvf_mbox_write ( intel, msg );
+
+		/* Read message from mailbox, if applicable. */
+		if ( ( seen & INTELVF_MBCTRL_VFU ) &&
+		     ( seen & INTELVF_MBCTRL_PFACK ) &&
+		     ( ctrl & INTELVF_MBCTRL_PFSTS ) )
+			intelvf_mbox_read ( intel, msg );
+
+		/* Acknowledge received message (if applicable),
+		 * release VFU lock, and send message (if applicable).
+		 */
+		ctrl = ( ( ( ctrl & INTELVF_MBCTRL_PFSTS ) ?
+			   INTELVF_MBCTRL_ACK : 0 ) |
+			 ( ( ctrl & INTELVF_MBCTRL_VFU ) ?
+			   INTELVF_MBCTRL_REQ : 0 ) );
+		writel ( ctrl, intel->regs + mbox->ctrl );
+
+		/* Exit successfully if we have received a response */
+		if ( msg->hdr & INTELVF_MSG_RESPONSE ) {
+
+			/* Sanity check */
+			assert ( seen & INTELVF_MBCTRL_VFU );
+			assert ( seen & INTELVF_MBCTRL_PFACK );
+			assert ( seen & INTELVF_MBCTRL_PFSTS );
+
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( intel, "INTEL %p timed out waiting for mailbox (seen %08x)\n",
+	       intel, seen );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Send reset message and get initial MAC address
+ *
+ * @v intel		Intel device
+ * @v hw_addr		Hardware address to fill in, or NULL
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_reset ( struct intel_nic *intel, uint8_t *hw_addr ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send reset message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_RESET;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p reset failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_RESET ) {
+		DBGC ( intel, "INTEL %p reset unexpected response:\n", intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Fill in MAC address, if applicable */
+	if ( hw_addr ) {
+		if ( msg.hdr & INTELVF_MSG_ACK ) {
+			memcpy ( hw_addr, msg.mac.mac, sizeof ( msg.mac.mac ) );
+			DBGC ( intel, "INTEL %p reset assigned MAC address "
+			       "%s\n", intel, eth_ntoa ( hw_addr ) );
+		} else {
+			eth_random_addr ( hw_addr );
+			DBGC ( intel, "INTEL %p reset generated MAC address "
+			       "%s\n", intel, eth_ntoa ( hw_addr ) );
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Send set MAC address message
+ *
+ * @v intel		Intel device
+ * @v ll_addr		Link-layer address
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_set_mac ( struct intel_nic *intel, const uint8_t *ll_addr ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MAC address message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_SET_MAC;
+	memcpy ( msg.mac.mac, ll_addr, sizeof ( msg.mac.mac ) );
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p set MAC address failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_SET_MAC ) {
+		DBGC ( intel, "INTEL %p set MAC address unexpected response:\n",
+		       intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that we were allowed to set the MAC address */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p set MAC address refused\n", intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/**
+ * Send set MTU message
+ *
+ * @v intel		Intel device
+ * @v mtu		Maximum packet size
+ * @ret rc		Return status code
+ */
+int intelvf_mbox_set_mtu ( struct intel_nic *intel, size_t mtu ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MTU message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELVF_MSG_TYPE_SET_MTU;
+	msg.mtu.mtu = mtu;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p set MTU failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELVF_MSG_TYPE_SET_MTU ) {
+		DBGC ( intel, "INTEL %p set MTU unexpected response:\n",
+		       intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that we were allowed to set the MTU */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p set MTU refused\n", intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/drivers/net/intelvf.h b/qemu/roms/ipxe/src/drivers/net/intelvf.h
new file mode 100644
index 000000000..d2f98d874
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/intelvf.h
@@ -0,0 +1,109 @@
+#ifndef _INTELVF_H
+#define _INTELVF_H
+
+/** @file
+ *
+ * Intel 10/100/1000 virtual function network card driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include "intel.h"
+
+/** Intel VF BAR size */
+#define INTELVF_BAR_SIZE ( 16 * 1024 )
+
+/** Mailbox Control Register */
+#define INTELVF_MBCTRL 0x0c40UL
+#define INTELVF_MBCTRL_REQ	0x00000001UL	/**< Request for PF ready */
+#define INTELVF_MBCTRL_ACK	0x00000002UL	/**< PF message received */
+#define INTELVF_MBCTRL_VFU	0x00000004UL	/**< Buffer taken by VF */
+#define INTELVF_MBCTRL_PFU	0x00000008UL	/**< Buffer taken to PF */
+#define INTELVF_MBCTRL_PFSTS	0x00000010UL	/**< PF wrote a message */
+#define INTELVF_MBCTRL_PFACK	0x00000020UL	/**< PF acknowledged message */
+#define INTELVF_MBCTRL_RSTI	0x00000040UL	/**< PF reset in progress */
+#define INTELVF_MBCTRL_RSTD	0x00000080UL	/**< PF reset complete */
+
+/** Mailbox Memory Register Base */
+#define INTELVF_MBMEM 0x0800UL
+
+/** Reset mailbox message */
+#define INTELVF_MSG_TYPE_RESET 0x00000001UL
+
+/** Set MAC address mailbox message */
+#define INTELVF_MSG_TYPE_SET_MAC 0x00000002UL
+
+/** Set MTU mailbox message */
+#define INTELVF_MSG_TYPE_SET_MTU 0x00000005UL
+
+/** Control ("ping") mailbox message */
+#define INTELVF_MSG_TYPE_CONTROL 0x00000100UL
+
+/** Message type mask */
+#define INTELVF_MSG_TYPE_MASK 0x0000ffffUL
+
+/** Message NACK flag */
+#define INTELVF_MSG_NACK 0x40000000UL
+
+/** Message ACK flag */
+#define INTELVF_MSG_ACK 0x80000000UL
+
+/** Message is a response */
+#define INTELVF_MSG_RESPONSE ( INTELVF_MSG_ACK | INTELVF_MSG_NACK )
+
+/** MAC address mailbox message */
+struct intelvf_msg_mac {
+	/** Message header */
+	uint32_t hdr;
+	/** MAC address */
+	uint8_t mac[ETH_ALEN];
+	/** Alignment padding */
+	uint8_t reserved[ (-ETH_ALEN) & 0x3 ];
+} __attribute__ (( packed ));
+
+/** Version number mailbox message */
+struct intelvf_msg_version {
+	/** Message header */
+	uint32_t hdr;
+	/** API version */
+	uint32_t version;
+} __attribute__ (( packed ));
+
+/** MTU mailbox message */
+struct intelvf_msg_mtu {
+	/** Message header */
+	uint32_t hdr;
+	/** Maximum packet size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Mailbox message */
+union intelvf_msg {
+	/** Message header */
+	uint32_t hdr;
+	/** MAC address message */
+	struct intelvf_msg_mac mac;
+	/** Version number message */
+	struct intelvf_msg_version version;
+	/** MTU message */
+	struct intelvf_msg_mtu mtu;
+	/** Raw dwords */
+	uint32_t dword[0];
+};
+
+/** Maximum time to wait for mailbox message
+ *
+ * This is a policy decision.
+ */
+#define INTELVF_MBOX_MAX_WAIT_MS 500
+
+extern int intelvf_mbox_msg ( struct intel_nic *intel, union intelvf_msg *msg );
+extern int intelvf_mbox_poll ( struct intel_nic *intel );
+extern int intelvf_mbox_wait ( struct intel_nic *intel );
+extern int intelvf_mbox_reset ( struct intel_nic *intel, uint8_t *hw_addr );
+extern int intelvf_mbox_set_mac ( struct intel_nic *intel,
+				  const uint8_t *ll_addr );
+extern int intelvf_mbox_set_mtu ( struct intel_nic *intel, size_t mtu );
+
+#endif /* _INTELVF_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/intelx.c b/qemu/roms/ipxe/src/drivers/net/intelx.c
index d69900e41..982b74f12 100644
--- a/qemu/roms/ipxe/src/drivers/net/intelx.c
+++ b/qemu/roms/ipxe/src/drivers/net/intelx.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -392,8 +396,10 @@ static int intelx_probe ( struct pci_device *pci ) {
 	netdev->dev = &pci->dev;
 	memset ( intel, 0, sizeof ( *intel ) );
 	intel->port = PCI_FUNC ( pci->busdevfn );
-	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELX_TD );
-	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELX_RD );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELX_TD,
+			  intel_describe_tx );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELX_RD,
+			  intel_describe_rx );
 
 	/* Fix up PCI device */
 	adjust_pci_device ( pci );
@@ -458,10 +464,15 @@ static void intelx_remove ( struct pci_device *pci ) {
 
 /** PCI device IDs */
 static struct pci_device_id intelx_nics[] = {
-	PCI_ROM ( 0x8086, 0x10fb, "82599", "82599", 0 ),
-	PCI_ROM ( 0x8086, 0x1528, "x540at2", "X540-AT2", 0 ),
-	PCI_ROM ( 0x8086, 0x154d, "x520", "X520", 0 ),
-	PCI_ROM ( 0x8086, 0x1557, "82599", "82599", 0 ),
+	PCI_ROM ( 0x8086, 0x10f7, "82599-kx4", "82599 (KX/KX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x10f8, "82599-combo-backplane", "82599 (combined backplane; KR/KX4/KX)", 0 ),
+	PCI_ROM ( 0x8086, 0x10f9, "82599-cx4", "82599 (CX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x10fb, "82599-sfp", "82599 (SFI/SFP+)", 0 ),
+	PCI_ROM ( 0x8086, 0x10fc, "82599-xaui", "82599 (XAUI/BX4)", 0 ),
+	PCI_ROM ( 0x8086, 0x1528, "x540t", "X540-AT2/X540-BT2", 0 ),
+	PCI_ROM ( 0x8086, 0x154d, "82599-sfp-sf2", "82599 (SFI/SFP+)", 0 ),
+	PCI_ROM ( 0x8086, 0x1557, "82599en-sfp", "82599 (Single Port SFI Only)", 0 ),
+	PCI_ROM ( 0x8086, 0x1560, "x540t1", "X540-AT2/X540-BT2 (with single port NVM)", 0 ),
 };
 
 /** PCI driver */
diff --git a/qemu/roms/ipxe/src/drivers/net/intelx.h b/qemu/roms/ipxe/src/drivers/net/intelx.h
index 60bb294d5..6383dfcad 100644
--- a/qemu/roms/ipxe/src/drivers/net/intelx.h
+++ b/qemu/roms/ipxe/src/drivers/net/intelx.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/intelxvf.c b/qemu/roms/ipxe/src/drivers/net/intelxvf.c
new file mode 100644
index 000000000..05e34c127
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/intelxvf.c
@@ -0,0 +1,466 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <ipxe/io.h>
+#include <ipxe/pci.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include "intelxvf.h"
+
+/** @file
+ *
+ * Intel 10 Gigabit Ethernet virtual function network card driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Diagnostics
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Dump statistics
+ *
+ * @v intel		Intel device
+ */
+static __attribute__ (( unused )) void
+intelxvf_stats ( struct intel_nic *intel ) {
+
+	DBGC ( intel, "INTEL %p TX %d (%#x%08x) RX %d (%#x%08x) multi %d\n",
+	       intel, readl ( intel->regs + INTELXVF_GPTC ),
+	       readl ( intel->regs + INTELXVF_GOTCH ),
+	       readl ( intel->regs + INTELXVF_GOTCL ),
+	       readl ( intel->regs + INTELXVF_GPRC ),
+	       readl ( intel->regs + INTELXVF_GORCH ),
+	       readl ( intel->regs + INTELXVF_GORCL ),
+	       readl ( intel->regs + INTELXVF_MPRC ) );
+}
+
+/******************************************************************************
+ *
+ * Device reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset hardware
+ *
+ * @v intel		Intel device
+ */
+static void intelxvf_reset ( struct intel_nic *intel ) {
+
+	/* Perform a function-level reset */
+	writel ( INTELXVF_CTRL_RST, intel->regs + INTELXVF_CTRL );
+}
+
+/******************************************************************************
+ *
+ * Link state
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check link state
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_check_link ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t links;
+
+	/* Read link status */
+	links = readl ( intel->regs + INTELXVF_LINKS );
+	DBGC ( intel, "INTEL %p link status is %08x\n", intel, links );
+
+	/* Update network device */
+	if ( links & INTELXVF_LINKS_UP ) {
+		netdev_link_up ( netdev );
+	} else {
+		netdev_link_down ( netdev );
+	}
+}
+
+/******************************************************************************
+ *
+ * Mailbox messages
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Send negotiate API version message
+ *
+ * @v intel		Intel device
+ * @v version		Requested version
+ * @ret rc		Return status code
+ */
+static int intelxvf_mbox_version ( struct intel_nic *intel,
+				   unsigned int version ) {
+	union intelvf_msg msg;
+	int rc;
+
+	/* Send set MTU message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.hdr = INTELXVF_MSG_TYPE_VERSION;
+	msg.version.version = version;
+	if ( ( rc = intelvf_mbox_msg ( intel, &msg ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p negotiate API version failed: %s\n",
+		       intel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check response */
+	if ( ( msg.hdr & INTELVF_MSG_TYPE_MASK ) != INTELXVF_MSG_TYPE_VERSION ){
+		DBGC ( intel, "INTEL %p negotiate API version unexpected "
+		       "response:\n", intel );
+		DBGC_HDA ( intel, 0, &msg, sizeof ( msg ) );
+		return -EPROTO;
+	}
+
+	/* Check that this version is supported */
+	if ( ! ( msg.hdr & INTELVF_MSG_ACK ) ) {
+		DBGC ( intel, "INTEL %p negotiate API version failed\n",
+		       intel );
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int intelxvf_open ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t srrctl;
+	uint32_t dca_rxctrl;
+	int rc;
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+
+	/* Notify PF that reset is complete */
+	if ( ( rc = intelvf_mbox_reset ( intel, NULL ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not reset: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_reset;
+	}
+
+	/* Negotiate API version 1.1.  If we do not negotiate at least
+	 * this version, then the RX datapath will remain disabled if
+	 * the PF has jumbo frames enabled.
+	 *
+	 * Ignore failures, since the host may not actually support
+	 * v1.1.
+	 */
+	intelxvf_mbox_version ( intel, INTELXVF_MSG_VERSION_1_1 );
+
+	/* Set MAC address */
+	if ( ( rc = intelvf_mbox_set_mac ( intel, netdev->ll_addr ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not set MAC address: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_set_mac;
+	}
+
+	/* Set MTU */
+	if ( ( rc = intelvf_mbox_set_mtu ( intel, netdev->max_pkt_len ) ) != 0){
+		DBGC ( intel, "INTEL %p could not set MTU %zd: %s\n",
+		       intel, netdev->max_pkt_len, strerror ( rc ) );
+		goto err_mbox_set_mtu;
+	}
+
+	/* Create transmit descriptor ring */
+	if ( ( rc = intel_create_ring ( intel, &intel->tx ) ) != 0 )
+		goto err_create_tx;
+
+	/* Create receive descriptor ring */
+	if ( ( rc = intel_create_ring ( intel, &intel->rx ) ) != 0 )
+		goto err_create_rx;
+
+	/* Allocate interrupt vectors */
+	writel ( ( INTELXVF_IVAR_RX0_DEFAULT | INTELXVF_IVAR_RX0_VALID |
+		   INTELXVF_IVAR_TX0_DEFAULT | INTELXVF_IVAR_TX0_VALID ),
+		 intel->regs + INTELXVF_IVAR );
+	writel ( ( INTELXVF_IVARM_MBOX_DEFAULT | INTELXVF_IVARM_MBOX_VALID ),
+		 intel->regs + INTELXVF_IVARM );
+
+	/* Configure receive buffer sizes and set receive descriptor type */
+	srrctl = readl ( intel->regs + INTELXVF_SRRCTL );
+	srrctl &= ~( INTELXVF_SRRCTL_BSIZE_MASK |
+		     INTELXVF_SRRCTL_DESCTYPE_MASK );
+	srrctl |= ( INTELXVF_SRRCTL_BSIZE_DEFAULT |
+		    INTELXVF_SRRCTL_DESCTYPE_DEFAULT );
+	writel ( srrctl, intel->regs + INTELXVF_SRRCTL );
+
+	/* Clear "must-be-zero" bit for direct cache access (DCA).  We
+	 * leave DCA disabled anyway, but if we do not clear this bit
+	 * then the received packets contain garbage data.
+	 */
+	dca_rxctrl = readl ( intel->regs + INTELXVF_DCA_RXCTRL );
+	dca_rxctrl &= ~INTELXVF_DCA_RXCTRL_MUST_BE_ZERO;
+	writel ( dca_rxctrl, intel->regs + INTELXVF_DCA_RXCTRL );
+
+	/* Fill receive ring */
+	intel_refill_rx ( intel );
+
+	/* Update link state */
+	intelxvf_check_link ( netdev );
+
+	return 0;
+
+	intel_destroy_ring ( intel, &intel->rx );
+ err_create_rx:
+	intel_destroy_ring ( intel, &intel->tx );
+ err_create_tx:
+ err_mbox_set_mtu:
+ err_mbox_set_mac:
+ err_mbox_reset:
+	intelxvf_reset ( intel );
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_close ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+
+	/* Destroy receive descriptor ring */
+	intel_destroy_ring ( intel, &intel->rx );
+
+	/* Discard any unused receive buffers */
+	intel_empty_rx ( intel );
+
+	/* Destroy transmit descriptor ring */
+	intel_destroy_ring ( intel, &intel->tx );
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void intelxvf_poll ( struct net_device *netdev ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t eicr;
+	int rc;
+
+	/* Check for and acknowledge interrupts */
+	eicr = readl ( intel->regs + INTELXVF_EICR );
+	if ( ! eicr )
+		return;
+
+	/* Poll for TX completions, if applicable */
+	if ( eicr & INTELXVF_EIRQ_TX0 )
+		intel_poll_tx ( netdev );
+
+	/* Poll for RX completions, if applicable */
+	if ( eicr & INTELXVF_EIRQ_RX0 )
+		intel_poll_rx ( netdev );
+
+	/* Poll for mailbox messages, if applicable */
+	if ( eicr & INTELXVF_EIRQ_MBOX ) {
+
+		/* Poll mailbox */
+		if ( ( rc = intelvf_mbox_poll ( intel ) ) != 0 ) {
+			DBGC ( intel, "INTEL %p mailbox poll failed!\n",
+			       intel );
+			netdev_rx_err ( netdev, NULL, rc );
+		}
+
+		/* Update link state */
+		intelxvf_check_link ( netdev );
+	}
+
+	/* Refill RX ring */
+	intel_refill_rx ( intel );
+}
+
+/**
+ * Enable or disable interrupts
+ *
+ * @v netdev		Network device
+ * @v enable		Interrupts should be enabled
+ */
+static void intelxvf_irq ( struct net_device *netdev, int enable ) {
+	struct intel_nic *intel = netdev->priv;
+	uint32_t mask;
+
+	mask = ( INTELXVF_EIRQ_MBOX | INTELXVF_EIRQ_TX0 | INTELXVF_EIRQ_RX0 );
+	if ( enable ) {
+		writel ( mask, intel->regs + INTELXVF_EIMS );
+	} else {
+		writel ( mask, intel->regs + INTELXVF_EIMC );
+	}
+}
+
+/** Network device operations */
+static struct net_device_operations intelxvf_operations = {
+	.open		= intelxvf_open,
+	.close		= intelxvf_close,
+	.transmit	= intel_transmit,
+	.poll		= intelxvf_poll,
+	.irq		= intelxvf_irq,
+};
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int intelxvf_probe ( struct pci_device *pci ) {
+	struct net_device *netdev;
+	struct intel_nic *intel;
+	int rc;
+
+	/* Allocate and initialise net device */
+	netdev = alloc_etherdev ( sizeof ( *intel ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &intelxvf_operations );
+	intel = netdev->priv;
+	pci_set_drvdata ( pci, netdev );
+	netdev->dev = &pci->dev;
+	memset ( intel, 0, sizeof ( *intel ) );
+	intel_init_mbox ( &intel->mbox, INTELXVF_MBCTRL, INTELXVF_MBMEM );
+	intel_init_ring ( &intel->tx, INTEL_NUM_TX_DESC, INTELXVF_TD,
+			  intel_describe_tx_adv );
+	intel_init_ring ( &intel->rx, INTEL_NUM_RX_DESC, INTELXVF_RD,
+			  intel_describe_rx );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	intel->regs = ioremap ( pci->membase, INTELVF_BAR_SIZE );
+	if ( ! intel->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Reset the function */
+	intelxvf_reset ( intel );
+
+	/* Send reset message and fetch MAC address */
+	if ( ( rc = intelvf_mbox_reset ( intel, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( intel, "INTEL %p could not reset and fetch MAC: %s\n",
+		       intel, strerror ( rc ) );
+		goto err_mbox_reset;
+	}
+
+	/* Reset the function (since we will not respond to Control
+	 * ("ping") mailbox messages until the network device is opened.
+	 */
+	intelxvf_reset ( intel );
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register_netdev;
+
+	/* Set initial link state */
+	intelxvf_check_link ( netdev );
+
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register_netdev:
+ err_mbox_reset:
+	intelxvf_reset ( intel );
+	iounmap ( intel->regs );
+ err_ioremap:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void intelxvf_remove ( struct pci_device *pci ) {
+	struct net_device *netdev = pci_get_drvdata ( pci );
+	struct intel_nic *intel = netdev->priv;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+
+	/* Reset the NIC */
+	intelxvf_reset ( intel );
+
+	/* Free network device */
+	iounmap ( intel->regs );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** PCI device IDs */
+static struct pci_device_id intelxvf_nics[] = {
+	PCI_ROM ( 0x8086, 0x10ed, "82599-vf", "82599 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x1515, "x540-vf", "X540 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x1565, "x550-vf", "X550 VF", 0 ),
+	PCI_ROM ( 0x8086, 0x15a8, "x552-vf", "X552 VF", 0 ),
+};
+
+/** PCI driver */
+struct pci_driver intelxvf_driver __pci_driver = {
+	.ids = intelxvf_nics,
+	.id_count = ( sizeof ( intelxvf_nics ) / sizeof ( intelxvf_nics[0] ) ),
+	.probe = intelxvf_probe,
+	.remove = intelxvf_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/intelxvf.h b/qemu/roms/ipxe/src/drivers/net/intelxvf.h
new file mode 100644
index 000000000..ad046a65c
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/intelxvf.h
@@ -0,0 +1,104 @@
+#ifndef _INTELXVF_H
+#define _INTELXVF_H
+
+/** @file
+ *
+ * Intel 10 Gigabit Ethernet virtual function network card driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include "intelvf.h"
+
+/** Control Register */
+#define INTELXVF_CTRL 0x0000UL
+#define INTELXVF_CTRL_RST	0x04000000UL	/**< Function-level reset */
+
+/** Link Status Register */
+#define INTELXVF_LINKS 0x0010UL
+#define INTELXVF_LINKS_UP	0x40000000UL	/**< Link up */
+
+/** Extended Interrupt Cause Read Register */
+#define INTELXVF_EICR 0x0100UL
+#define INTELXVF_EIRQ_RX0	0x00000001UL	/**< RX queue 0 (via IVAR) */
+#define INTELXVF_EIRQ_TX0	0x00000002UL	/**< TX queue 0 (via IVAR) */
+#define INTELXVF_EIRQ_MBOX	0x00000004UL	/**< Mailbox (via IVARM) */
+
+/** Extended Interrupt Mask Set/Read Register */
+#define INTELXVF_EIMS 0x0108UL
+
+/** Extended Interrupt Mask Clear Register */
+#define INTELXVF_EIMC 0x010cUL
+
+/** Interrupt Vector Allocation Register */
+#define INTELXVF_IVAR 0x0120UL
+#define INTELXVF_IVAR_RX0(bit)	( (bit) << 0 )	/**< RX queue 0 allocation */
+#define INTELXVF_IVAR_RX0_DEFAULT INTELXVF_IVAR_RX0 ( 0x00 )
+#define INTELXVF_IVAR_RX0_MASK	INTELXVF_IVAR_RX0 ( 0x01 )
+#define INTELXVF_IVAR_RX0_VALID	0x00000080UL	/**< RX queue 0 valid */
+#define INTELXVF_IVAR_TX0(bit)	( (bit) << 8 )	/**< TX queue 0 allocation */
+#define INTELXVF_IVAR_TX0_DEFAULT INTELXVF_IVAR_TX0 ( 0x01 )
+#define INTELXVF_IVAR_TX0_MASK	INTELXVF_IVAR_TX0 ( 0x01 )
+#define INTELXVF_IVAR_TX0_VALID	0x00008000UL	/**< TX queue 0 valid */
+
+/** Interrupt Vector Allocation Miscellaneous Register */
+#define INTELXVF_IVARM 0x0140UL
+#define INTELXVF_IVARM_MBOX(bit) ( (bit) << 0 )	/**< Mailbox allocation */
+#define INTELXVF_IVARM_MBOX_DEFAULT INTELXVF_IVARM_MBOX ( 0x02 )
+#define INTELXVF_IVARM_MBOX_MASK INTELXVF_IVARM_MBOX ( 0x03 )
+#define INTELXVF_IVARM_MBOX_VALID 0x00000080UL	/**< Mailbox valid */
+
+/** Mailbox Memory Register Base */
+#define INTELXVF_MBMEM 0x0200UL
+
+/** Mailbox Control Register */
+#define INTELXVF_MBCTRL 0x02fcUL
+
+/** Receive Descriptor register block */
+#define INTELXVF_RD 0x1000UL
+
+/** RX DCA Control Register */
+#define INTELXVF_DCA_RXCTRL 0x100cUL
+#define INTELXVF_DCA_RXCTRL_MUST_BE_ZERO 0x00001000UL /**< Must be zero */
+
+/** Split Receive Control Register */
+#define INTELXVF_SRRCTL 0x1014UL
+#define INTELXVF_SRRCTL_BSIZE(kb) ( (kb) << 0 )	/**< Receive buffer size */
+#define INTELXVF_SRRCTL_BSIZE_DEFAULT INTELXVF_SRRCTL_BSIZE ( 0x02 )
+#define INTELXVF_SRRCTL_BSIZE_MASK INTELXVF_SRRCTL_BSIZE ( 0x1f )
+#define INTELXVF_SRRCTL_DESCTYPE(typ) ( (typ) << 25 ) /**< Descriptor type */
+#define INTELXVF_SRRCTL_DESCTYPE_DEFAULT INTELXVF_SRRCTL_DESCTYPE ( 0x00 )
+#define INTELXVF_SRRCTL_DESCTYPE_MASK INTELXVF_SRRCTL_DESCTYPE ( 0x07 )
+
+/** Good Packets Received Count */
+#define INTELXVF_GPRC 0x101c
+
+/** Good Packets Received Count Low */
+#define INTELXVF_GORCL 0x1020
+
+/** Good Packets Received Count High */
+#define INTELXVF_GORCH 0x1024
+
+/* Multicast Packets Received Count */
+#define INTELXVF_MPRC 0x1034
+
+/** Transmit Descriptor register block */
+#define INTELXVF_TD 0x2000UL
+
+/** Good Packets Transmitted Count */
+#define INTELXVF_GPTC 0x201c
+
+/** Good Packets Transmitted Count Low */
+#define INTELXVF_GOTCL 0x2020
+
+/** Good Packets Transmitted Count High */
+#define INTELXVF_GOTCH 0x2024
+
+/** Negotiate API version mailbox message */
+#define INTELXVF_MSG_TYPE_VERSION 0x00000008UL
+
+/** API version 1.1 */
+#define INTELXVF_MSG_VERSION_1_1 0x00000002UL
+
+#endif /* _INTELXVF_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/ipoib.c b/qemu/roms/ipxe/src/drivers/net/ipoib.c
index 1b5391776..6552d764e 100644
--- a/qemu/roms/ipxe/src/drivers/net/ipoib.c
+++ b/qemu/roms/ipxe/src/drivers/net/ipoib.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -29,8 +33,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/errortab.h>
 #include <ipxe/malloc.h>
 #include <ipxe/if_arp.h>
+#include <ipxe/arp.h>
 #include <ipxe/if_ether.h>
 #include <ipxe/ethernet.h>
+#include <ipxe/ip.h>
 #include <ipxe/iobuf.h>
 #include <ipxe/netdevice.h>
 #include <ipxe/infiniband.h>
@@ -44,6 +50,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
  * IP over Infiniband
  */
 
+/* Disambiguate the various error causes */
+#define ENXIO_ARP_REPLY __einfo_error ( EINFO_ENXIO_ARP_REPLY )
+#define EINFO_ENXIO_ARP_REPLY						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x01,				\
+			  "Missing REMAC for ARP reply target address" )
+#define ENXIO_NON_IPV4 __einfo_error ( EINFO_ENXIO_NON_IPV4 )
+#define EINFO_ENXIO_NON_IPV4						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x02,				\
+			  "Missing REMAC for non-IPv4 packet" )
+#define ENXIO_ARP_SENT __einfo_error ( EINFO_ENXIO_ARP_SENT )
+#define EINFO_ENXIO_ARP_SENT						\
+	__einfo_uniqify ( EINFO_ENXIO, 0x03,				\
+			  "Missing REMAC for IPv4 packet (ARP sent)" )
+
 /** Number of IPoIB send work queue entries */
 #define IPOIB_NUM_SEND_WQES 2
 
@@ -96,6 +116,8 @@ struct errortab ipoib_errors[] __errortab = {
 	__einfo_errortab ( EINFO_EINPROGRESS_JOINING ),
 };
 
+static struct net_device_operations ipoib_operations;
+
 /****************************************************************************
  *
  * IPoIB REMAC cache
@@ -124,8 +146,10 @@ static struct ipoib_mac * ipoib_find_remac ( struct ipoib_device *ipoib,
 					     const struct ipoib_remac *remac ) {
 	struct ipoib_peer *peer;
 
-	/* Check for broadcast REMAC */
-	if ( is_broadcast_ether_addr ( remac ) )
+	/* Check for broadcast or multicast REMAC.  We transmit
+	 * multicasts as broadcasts for simplicity.
+	 */
+	if ( is_multicast_ether_addr ( remac ) )
 		return &ipoib->broadcast;
 
 	/* Try to find via REMAC cache */
@@ -202,14 +226,20 @@ static void ipoib_flush_remac ( struct ipoib_device *ipoib ) {
  * @ret discarded	Number of cached items discarded
  */
 static unsigned int ipoib_discard_remac ( void ) {
-	struct ib_device *ibdev;
+	struct net_device *netdev;
 	struct ipoib_device *ipoib;
 	struct ipoib_peer *peer;
 	unsigned int discarded = 0;
 
 	/* Try to discard one cache entry for each IPoIB device */
-	for_each_ibdev ( ibdev ) {
-		ipoib = ib_get_ownerdata ( ibdev );
+	for_each_netdev ( netdev ) {
+
+		/* Skip non-IPoIB devices */
+		if ( netdev->op != &ipoib_operations )
+			continue;
+		ipoib = netdev->priv;
+
+		/* Discard least recently used cache entry (if any) */
 		list_for_each_entry_reverse ( peer, &ipoib->peers, list ) {
 			list_del ( &peer->list );
 			free ( peer );
@@ -222,7 +252,7 @@ static unsigned int ipoib_discard_remac ( void ) {
 }
 
 /** IPoIB cache discarder */
-struct cache_discarder ipoib_discarder __cache_discarder ( CACHE_NORMAL ) = {
+struct cache_discarder ipoib_discarder __cache_discarder ( CACHE_EXPENSIVE ) = {
 	.discard = ipoib_discard_remac,
 };
 
@@ -324,8 +354,11 @@ static int ipoib_translate_tx_arp ( struct net_device *netdev,
 	/* Look up REMAC, if applicable */
 	if ( arphdr->ar_op == ARPOP_REPLY ) {
 		target_ha = ipoib_find_remac ( ipoib, arp_target_pa ( arphdr ));
-		if ( ! target_ha )
-			return -ENXIO;
+		if ( ! target_ha ) {
+			DBGC ( ipoib, "IPoIB %p no REMAC for %s ARP reply\n",
+			       ipoib, eth_ntoa ( arp_target_pa ( arphdr ) ) );
+			return -ENXIO_ARP_REPLY;
+		}
 	}
 
 	/* Construct new packet */
@@ -461,6 +494,7 @@ static int ipoib_transmit ( struct net_device *netdev,
 	struct ipoib_device *ipoib = netdev->priv;
 	struct ib_device *ibdev = ipoib->ibdev;
 	struct ethhdr *ethhdr;
+	struct iphdr *iphdr;
 	struct ipoib_hdr *ipoib_hdr;
 	struct ipoib_mac *mac;
 	struct ib_address_vector dest;
@@ -485,9 +519,34 @@ static int ipoib_transmit ( struct net_device *netdev,
 	iob_pull ( iobuf, sizeof ( *ethhdr ) );
 
 	/* Identify destination address */
-	mac = ipoib_find_remac ( ipoib, ( ( void *) ethhdr->h_dest ) );
-	if ( ! mac )
-		return -ENXIO;
+	mac = ipoib_find_remac ( ipoib, ( ( void * ) ethhdr->h_dest ) );
+	if ( ! mac ) {
+		/* Generate a new ARP request (if possible) to trigger
+		 * population of the REMAC cache entry.
+		 */
+		if ( ( net_proto != htons ( ETH_P_IP ) ) ||
+		     ( iob_len ( iobuf ) < sizeof ( *iphdr ) ) ) {
+			DBGC ( ipoib, "IPoIB %p no REMAC for %s non-IPv4 "
+			       "packet type %04x\n", ipoib,
+			       eth_ntoa ( ethhdr->h_dest ),
+			       ntohs ( net_proto ) );
+			return -ENXIO_NON_IPV4;
+		}
+		iphdr = iobuf->data;
+		if ( ( rc = arp_tx_request ( netdev, &ipv4_protocol,
+					     &iphdr->dest, &iphdr->src ) ) !=0){
+			DBGC ( ipoib, "IPoIB %p could not ARP for %s/%s/",
+			       ipoib, eth_ntoa ( ethhdr->h_dest ),
+			       inet_ntoa ( iphdr->dest ) );
+			DBGC ( ipoib, "%s: %s\n", inet_ntoa ( iphdr->src ),
+			       strerror ( rc ) );
+			return rc;
+		}
+		DBGC ( ipoib, "IPoIB %p no REMAC for %s/%s/", ipoib,
+		       eth_ntoa ( ethhdr->h_dest ), inet_ntoa ( iphdr->dest ) );
+		DBGC  ( ipoib, "%s\n", inet_ntoa ( iphdr->src ) );
+		return -ENXIO_ARP_SENT;
+	}
 
 	/* Translate packet if applicable */
 	if ( ( rc = ipoib_translate_tx ( netdev, iobuf, net_proto ) ) != 0 )
@@ -732,7 +791,8 @@ static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
 	int rc;
 
 	/* Leave existing broadcast group */
-	ipoib_leave_broadcast_group ( ipoib );
+	if ( ipoib->qp )
+		ipoib_leave_broadcast_group ( ipoib );
 
 	/* Update MAC address based on potentially-new GID prefix */
 	memcpy ( &ipoib->mac.gid.s.prefix, &ibdev->gid.s.prefix,
@@ -747,7 +807,7 @@ static void ipoib_link_state_changed ( struct ib_device *ibdev ) {
 	netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
 
 	/* Join new broadcast group */
-	if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) &&
+	if ( ib_is_open ( ibdev ) && ib_link_ok ( ibdev ) && ipoib->qp &&
 	     ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) ) {
 		DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
 		       "%s\n", ipoib, strerror ( rc ) );
@@ -835,7 +895,9 @@ static void ipoib_close ( struct net_device *netdev ) {
 
 	/* Tear down the queues */
 	ib_destroy_qp ( ibdev, ipoib->qp );
+	ipoib->qp = NULL;
 	ib_destroy_cq ( ibdev, ipoib->cq );
+	ipoib->cq = NULL;
 
 	/* Close IB device */
 	ib_close ( ibdev );
diff --git a/qemu/roms/ipxe/src/drivers/net/legacy.c b/qemu/roms/ipxe/src/drivers/net/legacy.c
index 4edbef162..73a80194f 100644
--- a/qemu/roms/ipxe/src/drivers/net/legacy.c
+++ b/qemu/roms/ipxe/src/drivers/net/legacy.c
@@ -17,7 +17,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct nic nic;
 
diff --git a/qemu/roms/ipxe/src/drivers/net/mii.c b/qemu/roms/ipxe/src/drivers/net/mii.c
index c4d32514d..9b297029a 100644
--- a/qemu/roms/ipxe/src/drivers/net/mii.c
+++ b/qemu/roms/ipxe/src/drivers/net/mii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <unistd.h>
@@ -111,3 +115,35 @@ int mii_reset ( struct mii_interface *mii ) {
 	DBGC ( mii, "MII %p timed out waiting for reset\n", mii );
 	return -ETIMEDOUT;
 }
+
+/**
+ * Update link status via MII
+ *
+ * @v mii		MII interface
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+int mii_check_link ( struct mii_interface *mii, struct net_device *netdev ) {
+	int bmsr;
+	int link;
+	int rc;
+
+	/* Read BMSR */
+	bmsr = mii_read ( mii, MII_BMSR );
+	if ( bmsr < 0 ) {
+		rc = bmsr;
+		return rc;
+	}
+
+	/* Report link status */
+	link = ( bmsr & BMSR_LSTATUS );
+	DBGC ( mii, "MII %p link %s (BMSR %#04x)\n",
+	       mii, ( link ? "up" : "down" ), bmsr );
+	if ( link ) {
+		netdev_link_up ( netdev );
+	} else {
+		netdev_link_down ( netdev );
+	}
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/drivers/net/myson.c b/qemu/roms/ipxe/src/drivers/net/myson.c
index 6abb55660..84a550596 100644
--- a/qemu/roms/ipxe/src/drivers/net/myson.c
+++ b/qemu/roms/ipxe/src/drivers/net/myson.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/myson.h b/qemu/roms/ipxe/src/drivers/net/myson.h
index 8d7cc5855..05a6b8a58 100644
--- a/qemu/roms/ipxe/src/drivers/net/myson.h
+++ b/qemu/roms/ipxe/src/drivers/net/myson.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/if_ether.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/ncm.c b/qemu/roms/ipxe/src/drivers/net/ncm.c
new file mode 100644
index 000000000..10728d2a1
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/ncm.c
@@ -0,0 +1,672 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/profile.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include "ecm.h"
+#include "ncm.h"
+
+/** @file
+ *
+ * CDC-NCM USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler ncm_intr_profiler __profiler =
+	{ .name = "ncm.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler ncm_in_profiler __profiler =
+	{ .name = "ncm.in" };
+
+/** Bulk IN per-datagram profiler */
+static struct profiler ncm_in_datagram_profiler __profiler =
+	{ .name = "ncm.in_dgram" };
+
+/** Bulk OUT profiler */
+static struct profiler ncm_out_profiler __profiler =
+	{ .name = "ncm.out" };
+
+/******************************************************************************
+ *
+ * CDC-NCM communications interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_intr_complete ( struct usb_endpoint *ep,
+				struct io_buffer *iobuf, int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.intr );
+	struct net_device *netdev = ncm->netdev;
+	struct usb_setup_packet *message;
+	size_t len = iob_len ( iobuf );
+
+	/* Profile completions */
+	profile_start ( &ncm_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( ncm, "NCM %p interrupt failed: %s\n",
+		       ncm, strerror ( rc ) );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Extract message header */
+	if ( len < sizeof ( *message ) ) {
+		DBGC ( ncm, "NCM %p underlength interrupt:\n", ncm );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto error;
+	}
+	message = iobuf->data;
+
+	/* Parse message header */
+	switch ( message->request ) {
+
+	case cpu_to_le16 ( CDC_NETWORK_CONNECTION ) :
+		if ( message->value ) {
+			DBGC ( ncm, "NCM %p link up\n", ncm );
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( ncm, "NCM %p link down\n", ncm );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	case cpu_to_le16 ( CDC_CONNECTION_SPEED_CHANGE ) :
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( ncm, "NCM %p unrecognised interrupt:\n", ncm );
+		DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+		goto error;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+	profile_stop ( &ncm_intr_profiler );
+
+	return;
+
+ error:
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+ ignore:
+	free_iob ( iobuf );
+	return;
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations ncm_intr_operations = {
+	.complete = ncm_intr_complete,
+};
+
+/******************************************************************************
+ *
+ * CDC-NCM data interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Prefill bulk IN endpoint
+ *
+ * @v ncm		CDC-NCM device
+ * @ret rc		Return status code
+ */
+static int ncm_in_prefill ( struct ncm_device *ncm ) {
+	struct usb_bus *bus = ncm->bus;
+	size_t mtu;
+	unsigned int count;
+	int rc;
+
+	/* Some devices have a very small number of internal buffers,
+	 * and rely on being able to pack multiple packets into each
+	 * buffer.  We therefore want to use large buffers if
+	 * possible.  However, large allocations have a reasonable
+	 * chance of failure, especially if this is not the first or
+	 * only device to be opened.
+	 *
+	 * We therefore attempt to find a usable buffer size, starting
+	 * large and working downwards until allocation succeeds.
+	 * Smaller buffers will still work, albeit with a higher
+	 * chance of packet loss and so lower overall throughput.
+	 */
+	for ( mtu = ncm->mtu ; mtu >= NCM_MIN_NTB_INPUT_SIZE ; mtu >>= 1 ) {
+
+		/* Attempt allocation at this MTU */
+		if ( mtu > NCM_MAX_NTB_INPUT_SIZE )
+			continue;
+		if ( mtu > bus->mtu )
+			continue;
+		count = ( NCM_IN_MIN_SIZE / mtu );
+		if ( count < NCM_IN_MIN_COUNT )
+			count = NCM_IN_MIN_COUNT;
+		if ( ( count * mtu ) > NCM_IN_MAX_SIZE )
+			continue;
+		usb_refill_init ( &ncm->usbnet.in, mtu, count );
+		if ( ( rc = usb_prefill ( &ncm->usbnet.in ) ) != 0 ) {
+			DBGC ( ncm, "NCM %p could not prefill %dx %zd-byte "
+			       "buffers for bulk IN\n", ncm, count, mtu );
+			continue;
+		}
+
+		DBGC ( ncm, "NCM %p using %dx %zd-byte buffers for bulk IN\n",
+		       ncm, count, mtu );
+		return 0;
+	}
+
+	DBGC ( ncm, "NCM %p could not prefill bulk IN endpoint\n", ncm );
+	return -ENOMEM;
+}
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_in_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			      int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.in );
+	struct net_device *netdev = ncm->netdev;
+	struct ncm_transfer_header *nth;
+	struct ncm_datagram_pointer *ndp;
+	struct ncm_datagram_descriptor *desc;
+	struct io_buffer *pkt;
+	unsigned int remaining;
+	size_t ndp_offset;
+	size_t ndp_len;
+	size_t pkt_offset;
+	size_t pkt_len;
+	size_t headroom;
+	size_t len;
+
+	/* Profile overall bulk IN completion */
+	profile_start ( &ncm_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto ignore;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( ncm, "NCM %p bulk IN failed: %s\n",
+		       ncm, strerror ( rc ) );
+		goto error;
+	}
+
+	/* Locate transfer header */
+	len = iob_len ( iobuf );
+	if ( sizeof ( *nth ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NTH:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	nth = iobuf->data;
+
+	/* Locate datagram pointer */
+	ndp_offset = le16_to_cpu ( nth->offset );
+	if ( ( ndp_offset + sizeof ( *ndp ) ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NDP:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	ndp = ( iobuf->data + ndp_offset );
+	ndp_len = le16_to_cpu ( ndp->header_len );
+	if ( ndp_len < offsetof ( typeof ( *ndp ), desc ) ) {
+		DBGC ( ncm, "NCM %p NDP header length too short:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+	if ( ( ndp_offset + ndp_len ) > len ) {
+		DBGC ( ncm, "NCM %p packet too short for NDP:\n", ncm );
+		rc = -EINVAL;
+		goto error;
+	}
+
+	/* Process datagrams */
+	remaining = ( ( ndp_len - offsetof ( typeof ( *ndp ), desc ) ) /
+		      sizeof ( ndp->desc[0] ) );
+	for ( desc = ndp->desc ; remaining && desc->offset ; remaining-- ) {
+
+		/* Profile individual datagrams */
+		profile_start ( &ncm_in_datagram_profiler );
+
+		/* Locate datagram */
+		pkt_offset = le16_to_cpu ( desc->offset );
+		pkt_len = le16_to_cpu ( desc->len );
+		if ( pkt_len < ETH_HLEN ) {
+			DBGC ( ncm, "NCM %p underlength datagram:\n", ncm );
+			rc = -EINVAL;
+			goto error;
+		}
+		if ( ( pkt_offset + pkt_len ) > len ) {
+			DBGC ( ncm, "NCM %p datagram exceeds packet:\n", ncm );
+			rc = -EINVAL;
+			goto error;
+		}
+
+		/* Move to next descriptor */
+		desc++;
+
+		/* Copy data to a new I/O buffer.  Our USB buffers may
+		 * be very large and so we choose to recycle the
+		 * buffers directly rather than attempt reallocation
+		 * while the device is running.  We therefore copy the
+		 * data to a new I/O buffer even if this is the only
+		 * (or last) packet within the buffer.
+		 *
+		 * We reserve enough space at the start of each buffer
+		 * to allow for our own transmission header, to
+		 * support protocols such as ARP which may modify the
+		 * received packet and reuse the same I/O buffer for
+		 * transmission.
+		 */
+		headroom = ( sizeof ( struct ncm_ntb_header ) + ncm->padding );
+		pkt = alloc_iob ( headroom + pkt_len );
+		if ( ! pkt ) {
+			/* Record error and continue */
+			netdev_rx_err ( netdev, NULL, -ENOMEM );
+			continue;
+		}
+		iob_reserve ( pkt, headroom );
+		memcpy ( iob_put ( pkt, pkt_len ),
+			 ( iobuf->data + pkt_offset ), pkt_len );
+
+		/* Strip CRC, if present */
+		if ( ndp->magic & cpu_to_le32 ( NCM_DATAGRAM_POINTER_MAGIC_CRC))
+			iob_unput ( pkt, 4 /* CRC32 */ );
+
+		/* Hand off to network stack */
+		netdev_rx ( netdev, pkt );
+		profile_stop ( &ncm_in_datagram_profiler );
+	}
+
+	/* Recycle I/O buffer */
+	usb_recycle ( &ncm->usbnet.in, iobuf );
+	profile_stop ( &ncm_in_profiler );
+
+	return;
+
+ error:
+	/* Record error against network device */
+	DBGC_HDA ( ncm, 0, iobuf->data, iob_len ( iobuf ) );
+	netdev_rx_err ( netdev, NULL, rc );
+ ignore:
+	usb_recycle ( &ncm->usbnet.in, iobuf );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations ncm_in_operations = {
+	.complete = ncm_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v ncm		CDC-NCM device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ncm_out_transmit ( struct ncm_device *ncm,
+			      struct io_buffer *iobuf ) {
+	struct ncm_ntb_header *header;
+	size_t len = iob_len ( iobuf );
+	size_t header_len = ( sizeof ( *header ) + ncm->padding );
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &ncm_out_profiler );
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, header_len ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, header_len );
+
+	/* Populate header */
+	header->nth.magic = cpu_to_le32 ( NCM_TRANSFER_HEADER_MAGIC );
+	header->nth.header_len = cpu_to_le16 ( sizeof ( header->nth ) );
+	header->nth.sequence = cpu_to_le16 ( ncm->sequence );
+	header->nth.len = cpu_to_le16 ( iob_len ( iobuf ) );
+	header->nth.offset =
+		cpu_to_le16 ( offsetof ( typeof ( *header ), ndp ) );
+	header->ndp.magic = cpu_to_le32 ( NCM_DATAGRAM_POINTER_MAGIC );
+	header->ndp.header_len = cpu_to_le16 ( sizeof ( header->ndp ) +
+					       sizeof ( header->desc ) );
+	header->ndp.offset = cpu_to_le16 ( 0 );
+	header->desc[0].offset = cpu_to_le16 ( header_len );
+	header->desc[0].len = cpu_to_le16 ( len );
+	memset ( &header->desc[1], 0, sizeof ( header->desc[1] ) );
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &ncm->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	/* Increment sequence number */
+	ncm->sequence++;
+
+	profile_stop ( &ncm_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void ncm_out_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			       int rc ) {
+	struct ncm_device *ncm = container_of ( ep, struct ncm_device,
+						usbnet.out );
+	struct net_device *netdev = ncm->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations ncm_out_operations = {
+	.complete = ncm_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int ncm_open ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+	struct usb_device *usb = ncm->usb;
+	struct ncm_set_ntb_input_size size;
+	int rc;
+
+	/* Reset sequence number */
+	ncm->sequence = 0;
+
+	/* Prefill I/O buffers */
+	if ( ( rc = ncm_in_prefill ( ncm ) ) != 0 )
+		goto err_prefill;
+
+	/* Set maximum input size */
+	memset ( &size, 0, sizeof ( size ) );
+	size.mtu = cpu_to_le32 ( ncm->usbnet.in.len );
+	if ( ( rc = usb_control ( usb, NCM_SET_NTB_INPUT_SIZE, 0,
+				  ncm->usbnet.comms, &size,
+				  sizeof ( size ) ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not set input size to %zd: %s\n",
+		       ncm, ncm->usbnet.in.len, strerror ( rc ) );
+		goto err_set_ntb_input_size;
+	}
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &ncm->usbnet ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not open: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_open;
+	}
+
+	return 0;
+
+	usbnet_close ( &ncm->usbnet );
+ err_open:
+ err_set_ntb_input_size:
+	usb_flush ( &ncm->usbnet.in );
+ err_prefill:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void ncm_close ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &ncm->usbnet );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ncm_transmit ( struct net_device *netdev,
+			  struct io_buffer *iobuf ) {
+	struct ncm_device *ncm = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = ncm_out_transmit ( ncm, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void ncm_poll ( struct net_device *netdev ) {
+	struct ncm_device *ncm = netdev->priv;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( ncm->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &ncm->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+
+}
+
+/** CDC-NCM network device operations */
+static struct net_device_operations ncm_operations = {
+	.open		= ncm_open,
+	.close		= ncm_close,
+	.transmit	= ncm_transmit,
+	.poll		= ncm_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int ncm_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct ncm_device *ncm;
+	struct usb_interface_descriptor *comms;
+	struct ecm_ethernet_descriptor *ethernet;
+	struct ncm_ntb_parameters params;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *ncm ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &ncm_operations );
+	netdev->dev = &func->dev;
+	ncm = netdev->priv;
+	memset ( ncm, 0, sizeof ( *ncm ) );
+	ncm->usb = usb;
+	ncm->bus = usb->port->hub->bus;
+	ncm->netdev = netdev;
+	usbnet_init ( &ncm->usbnet, func, &ncm_intr_operations,
+		      &ncm_in_operations, &ncm_out_operations );
+	usb_refill_init ( &ncm->usbnet.intr, 0, NCM_INTR_COUNT );
+	DBGC ( ncm, "NCM %p on %s\n", ncm, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &ncm->usbnet, config ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not describe: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Locate Ethernet descriptor */
+	comms = usb_interface_descriptor ( config, ncm->usbnet.comms, 0 );
+	assert ( comms != NULL );
+	ethernet = ecm_ethernet_descriptor ( config, comms );
+	if ( ! ethernet ) {
+		DBGC ( ncm, "NCM %p has no Ethernet descriptor\n", ncm );
+		rc = -EINVAL;
+		goto err_ethernet;
+	}
+
+	/* Fetch MAC address */
+	if ( ( rc = ecm_fetch_mac ( usb, ethernet, netdev->hw_addr ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not fetch MAC address: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_fetch_mac;
+	}
+
+	/* Get NTB parameters */
+	if ( ( rc = usb_control ( usb, NCM_GET_NTB_PARAMETERS, 0,
+				  ncm->usbnet.comms, &params,
+				  sizeof ( params ) ) ) != 0 ) {
+		DBGC ( ncm, "NCM %p could not get NTB parameters: %s\n",
+		       ncm, strerror ( rc ) );
+		goto err_ntb_parameters;
+	}
+
+	/* Get maximum supported input size */
+	ncm->mtu = le32_to_cpu ( params.in.mtu );
+	DBGC2 ( ncm, "NCM %p maximum IN size is %zd bytes\n", ncm, ncm->mtu );
+
+	/* Calculate transmit padding */
+	ncm->padding = ( ( le16_to_cpu ( params.out.remainder ) -
+			   sizeof ( struct ncm_ntb_header ) - ETH_HLEN ) &
+			 ( le16_to_cpu ( params.out.divisor ) - 1 ) );
+	DBGC2 ( ncm, "NCM %p using %zd-byte transmit padding\n",
+		ncm, ncm->padding );
+	assert ( ( ( sizeof ( struct ncm_ntb_header ) + ncm->padding +
+		     ETH_HLEN ) % le16_to_cpu ( params.out.divisor ) ) ==
+		 le16_to_cpu ( params.out.remainder ) );
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, ncm );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_ntb_parameters:
+ err_fetch_mac:
+ err_ethernet:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void ncm_remove ( struct usb_function *func ) {
+	struct ncm_device *ncm = usb_func_get_drvdata ( func );
+	struct net_device *netdev = ncm->netdev;
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** CDC-NCM device IDs */
+static struct usb_device_id ncm_ids[] = {
+	{
+		.name = "cdc-ncm",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_CDC,
+			.subclass = USB_SUBCLASS_CDC_NCM,
+			.protocol = 0,
+		},
+	},
+};
+
+/** CDC-NCM driver */
+struct usb_driver ncm_driver __usb_driver = {
+	.ids = ncm_ids,
+	.id_count = ( sizeof ( ncm_ids ) / sizeof ( ncm_ids[0] ) ),
+	.probe = ncm_probe,
+	.remove = ncm_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/ncm.h b/qemu/roms/ipxe/src/drivers/net/ncm.h
new file mode 100644
index 000000000..a9565a56b
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/ncm.h
@@ -0,0 +1,173 @@
+#ifndef _NCM_H
+#define _NCM_H
+
+/** @file
+ *
+ * CDC-NCM USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/usb.h>
+#include <ipxe/cdc.h>
+#include <byteswap.h>
+#include "ecm.h"
+
+/** CDC-NCM subclass */
+#define USB_SUBCLASS_CDC_NCM 0x0d
+
+/** Get NTB parameters */
+#define NCM_GET_NTB_PARAMETERS						\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x80 ) )
+
+/** NTB datagram parameters */
+struct ncm_ntb_datagram_parameters {
+	/** Maximum size */
+	uint32_t mtu;
+	/** Alignment divisor */
+	uint16_t divisor;
+	/** Alignment remainder */
+	uint16_t remainder;
+	/** Alignment modulus */
+	uint16_t modulus;
+} __attribute__ (( packed ));
+
+/** NTB parameters */
+struct ncm_ntb_parameters {
+	/** Length */
+	uint16_t len;
+	/** Supported formats */
+	uint16_t formats;
+	/** IN datagram parameters */
+	struct ncm_ntb_datagram_parameters in;
+	/** Reserved */
+	uint16_t reserved;
+	/** OUT datagram parameters */
+	struct ncm_ntb_datagram_parameters out;
+	/** Maximum number of datagrams per OUT NTB */
+	uint16_t max;
+} __attribute__ (( packed ));
+
+/** Set NTB input size */
+#define NCM_SET_NTB_INPUT_SIZE						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x86 ) )
+
+/** Set NTB input size */
+struct ncm_set_ntb_input_size {
+	/** Maximum size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Minimum allowed NTB input size */
+#define NCM_MIN_NTB_INPUT_SIZE 2048
+
+/** Maximum allowed NTB input size (16-bit) */
+#define NCM_MAX_NTB_INPUT_SIZE 65536
+
+/** CDC-NCM transfer header (16-bit) */
+struct ncm_transfer_header {
+	/** Signature */
+	uint32_t magic;
+	/** Header length */
+	uint16_t header_len;
+	/** Sequence number */
+	uint16_t sequence;
+	/** Total length */
+	uint16_t len;
+	/** Offset of first datagram pointer */
+	uint16_t offset;
+} __attribute__ (( packed ));
+
+/** CDC-NCM transfer header magic */
+#define NCM_TRANSFER_HEADER_MAGIC 0x484d434eUL
+
+/** CDC-NCM datagram descriptor (16-bit) */
+struct ncm_datagram_descriptor {
+	/** Starting offset */
+	uint16_t offset;
+	/** Length */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** CDC-NCM datagram pointer (16-bit) */
+struct ncm_datagram_pointer {
+	/** Signature */
+	uint32_t magic;
+	/** Header length */
+	uint16_t header_len;
+	/** Offset of next datagram pointer */
+	uint16_t offset;
+	/** Datagram descriptors
+	 *
+	 * Must be terminated by an empty descriptor.
+	 */
+	struct ncm_datagram_descriptor desc[0];
+} __attribute__ (( packed ));
+
+/** CDC-NCM datagram pointer magic */
+#define NCM_DATAGRAM_POINTER_MAGIC 0x304d434eUL
+
+/** CDC-NCM datagram pointer CRC present flag */
+#define NCM_DATAGRAM_POINTER_MAGIC_CRC 0x01000000UL
+
+/** NTB constructed for transmitted packets (excluding padding)
+ *
+ * This is a policy decision.
+ */
+struct ncm_ntb_header {
+	/** Transfer header */
+	struct ncm_transfer_header nth;
+	/** Datagram pointer */
+	struct ncm_datagram_pointer ndp;
+	/** Datagram descriptors */
+	struct ncm_datagram_descriptor desc[2];
+} __attribute__ (( packed ));
+
+/** A CDC-NCM network device */
+struct ncm_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+
+	/** Maximum supported NTB input size */
+	size_t mtu;
+	/** Transmitted packet sequence number */
+	uint16_t sequence;
+	/** Alignment padding required on transmitted packets */
+	size_t padding;
+};
+
+/** Bulk IN ring minimum buffer count
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MIN_COUNT 3
+
+/** Bulk IN ring minimum total buffer size
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MIN_SIZE 16384
+
+/** Bulk IN ring maximum total buffer size
+ *
+ * This is a policy decision.
+ */
+#define NCM_IN_MAX_SIZE 131072
+
+/** Interrupt ring buffer count
+ *
+ * This is a policy decision.
+ */
+#define NCM_INTR_COUNT 2
+
+#endif /* _NCM_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/netfront.c b/qemu/roms/ipxe/src/drivers/net/netfront.c
index 4b816329e..2f4bbf2a0 100644
--- a/qemu/roms/ipxe/src/drivers/net/netfront.c
+++ b/qemu/roms/ipxe/src/drivers/net/netfront.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -135,7 +139,7 @@ static int netfront_read_mac ( struct netfront_nic *netfront, void *hw_addr ) {
 		xendev->key, mac );
 
 	/* Decode MAC address */
-	len = hex_decode ( mac, ':', hw_addr, ETH_ALEN );
+	len = hex_decode ( ':', mac, hw_addr, ETH_ALEN );
 	if ( len < 0 ) {
 		rc = len;
 		DBGC ( netfront, "NETFRONT %s could not decode MAC address "
@@ -593,6 +597,11 @@ static int netfront_open ( struct net_device *netdev ) {
 					  "feature-no-csum-offload" ) ) != 0 )
 		goto err_feature_no_csum_offload;
 
+	/* Inform backend that we will send notifications for RX requests */
+	if ( ( rc = netfront_write_flag ( netfront,
+					  "feature-rx-notify" ) ) != 0 )
+		goto err_feature_rx_notify;
+
 	/* Set state to Connected */
 	if ( ( rc = xenbus_set_state ( xendev, XenbusStateConnected ) ) != 0 ) {
 		DBGC ( netfront, "NETFRONT %s could not set state=\"%d\": %s\n",
@@ -618,6 +627,8 @@ static int netfront_open ( struct net_device *netdev ) {
  err_backend_wait:
 	netfront_reset ( netfront );
  err_set_state:
+	netfront_rm ( netfront, "feature-rx-notify" );
+ err_feature_rx_notify:
 	netfront_rm ( netfront, "feature-no-csum-offload" );
  err_feature_no_csum_offload:
 	netfront_rm ( netfront, "request-rx-copy" );
@@ -661,6 +672,7 @@ static void netfront_close ( struct net_device *netdev ) {
 	}
 
 	/* Delete flags */
+	netfront_rm ( netfront, "feature-rx-notify" );
 	netfront_rm ( netfront, "feature-no-csum-offload" );
 	netfront_rm ( netfront, "request-rx-copy" );
 
diff --git a/qemu/roms/ipxe/src/drivers/net/netfront.h b/qemu/roms/ipxe/src/drivers/net/netfront.h
index b3f899f3c..38fd0a77e 100644
--- a/qemu/roms/ipxe/src/drivers/net/netfront.h
+++ b/qemu/roms/ipxe/src/drivers/net/netfront.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/io/netif.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/netvsc.c b/qemu/roms/ipxe/src/drivers/net/netvsc.c
new file mode 100644
index 000000000..d269cd63e
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/netvsc.c
@@ -0,0 +1,848 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V network virtual service client
+ *
+ * The network virtual service client (NetVSC) connects to the network
+ * virtual service provider (NetVSP) via the Hyper-V virtual machine
+ * bus (VMBus).  It provides a transport layer for RNDIS packets.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/rndis.h>
+#include <ipxe/vmbus.h>
+#include "netvsc.h"
+
+/**
+ * Send control message and wait for completion
+ *
+ * @v netvsc		NetVSC device
+ * @v xrid		Relative transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_control ( struct netvsc_device *netvsc, unsigned int xrid,
+			    const void *data, size_t len ) {
+	uint64_t xid = ( NETVSC_BASE_XID + xrid );
+	unsigned int i;
+	int rc;
+
+	/* Send control message */
+	if ( ( rc = vmbus_send_control ( netvsc->vmdev, xid, data, len ) ) !=0){
+		DBGC ( netvsc, "NETVSC %s could not send control message: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Record transaction ID */
+	netvsc->wait_xrid = xrid;
+
+	/* Wait for operation to complete */
+	for ( i = 0 ; i < NETVSC_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for completion */
+		if ( ! netvsc->wait_xrid )
+			return netvsc->wait_rc;
+
+		/* Poll VMBus device */
+		vmbus_poll ( netvsc->vmdev );
+
+		/* Delay for 1ms */
+		mdelay ( 1 );
+	}
+
+	DBGC ( netvsc, "NETVSC %s timed out waiting for XRID %d\n",
+	       netvsc->name, xrid );
+	vmbus_dump_channel ( netvsc->vmdev );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Handle generic completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_completed ( struct netvsc_device *netvsc __unused,
+			      const void *data __unused, size_t len __unused ) {
+	return 0;
+}
+
+/**
+ * Initialise communication
+ *
+ * @v netvsc		NetVSC device
+ * @ret rc		Return status code
+ */
+static int netvsc_initialise ( struct netvsc_device *netvsc ) {
+	struct netvsc_init_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_INIT_MSG );
+	msg.min = cpu_to_le32 ( NETVSC_VERSION_1 );
+	msg.max = cpu_to_le32 ( NETVSC_VERSION_1 );
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, NETVSC_INIT_XRID, &msg,
+				     sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not initialise: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle initialisation completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int
+netvsc_initialised ( struct netvsc_device *netvsc, const void *data,
+		     size_t len ) {
+	const struct netvsc_init_completion *cmplt = data;
+
+	/* Check completion */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( netvsc, "NETVSC %s underlength initialisation "
+		       "completion (%zd bytes)\n", netvsc->name, len );
+		return -EINVAL;
+	}
+	if ( cmplt->header.type != cpu_to_le32 ( NETVSC_INIT_CMPLT ) ) {
+		DBGC ( netvsc, "NETVSC %s unexpected initialisation completion "
+		       "type %d\n", netvsc->name,
+		       le32_to_cpu ( cmplt->header.type ) );
+		return -EPROTO;
+	}
+	if ( cmplt->status != cpu_to_le32 ( NETVSC_OK ) ) {
+		DBGC ( netvsc, "NETVSC %s initialisation failure status %d\n",
+		       netvsc->name, le32_to_cpu ( cmplt->status ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Set NDIS version
+ *
+ * @v netvsc		NetVSC device
+ * @ret rc		Return status code
+ */
+static int netvsc_ndis_version ( struct netvsc_device *netvsc ) {
+	struct netvsc_ndis_version_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_NDIS_VERSION_MSG );
+	msg.major = cpu_to_le32 ( NETVSC_NDIS_MAJOR );
+	msg.minor = cpu_to_le32 ( NETVSC_NDIS_MINOR );
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, NETVSC_NDIS_VERSION_XRID,
+				     &msg, sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not set NDIS version: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Establish data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_establish_buffer ( struct netvsc_device *netvsc,
+				     struct netvsc_buffer *buffer ) {
+	struct netvsc_establish_buffer_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( buffer->establish_type );
+	msg.gpadl = cpu_to_le32 ( buffer->gpadl );
+	msg.pageset = buffer->pages.pageset; /* Already protocol-endian */
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, buffer->establish_xrid, &msg,
+				     sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not establish buffer: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle establish receive data buffer completion
+ *
+ * @v netvsc		NetVSC device
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_rx_established_buffer ( struct netvsc_device *netvsc,
+					  const void *data, size_t len ) {
+	const struct netvsc_rx_establish_buffer_completion *cmplt = data;
+
+	/* Check completion */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( netvsc, "NETVSC %s underlength buffer completion (%zd "
+		       "bytes)\n", netvsc->name, len );
+		return -EINVAL;
+	}
+	if ( cmplt->header.type != cpu_to_le32 ( NETVSC_RX_ESTABLISH_CMPLT ) ) {
+		DBGC ( netvsc, "NETVSC %s unexpected buffer completion type "
+		       "%d\n", netvsc->name, le32_to_cpu ( cmplt->header.type));
+		return -EPROTO;
+	}
+	if ( cmplt->status != cpu_to_le32 ( NETVSC_OK ) ) {
+		DBGC ( netvsc, "NETVSC %s buffer failure status %d\n",
+		       netvsc->name, le32_to_cpu ( cmplt->status ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Revoke data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_revoke_buffer ( struct netvsc_device *netvsc,
+				  struct netvsc_buffer *buffer ) {
+	struct netvsc_revoke_buffer_message msg;
+	int rc;
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( buffer->revoke_type );
+	msg.pageset = buffer->pages.pageset; /* Already protocol-endian */
+
+	/* Send message and wait for completion */
+	if ( ( rc = netvsc_control ( netvsc, buffer->revoke_xrid,
+				     &msg, sizeof ( msg ) ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not revoke buffer: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle received control packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_control ( struct vmbus_device *vmdev, uint64_t xid,
+				 const void *data, size_t len ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+
+	DBGC ( netvsc, "NETVSC %s received unsupported control packet "
+	       "(%08llx):\n", netvsc->name, xid );
+	DBGC_HDA ( netvsc, 0, data, len );
+	return -ENOTSUP;
+}
+
+/**
+ * Handle received data packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @v list		List of I/O buffers
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_data ( struct vmbus_device *vmdev, uint64_t xid,
+			      const void *data, size_t len,
+			      struct list_head *list ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+	const struct netvsc_rndis_message *msg = data;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( netvsc, "NETVSC %s received underlength RNDIS packet "
+		       "(%zd bytes)\n", netvsc->name, len );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+	if ( msg->header.type != cpu_to_le32 ( NETVSC_RNDIS_MSG ) ) {
+		DBGC ( netvsc, "NETVSC %s received unexpected RNDIS packet "
+		       "type %d\n", netvsc->name,
+		       le32_to_cpu ( msg->header.type ) );
+		rc = -EINVAL;
+		goto err_sanity;
+	}
+
+	/* Send completion back to host */
+	if ( ( rc = vmbus_send_completion ( vmdev, xid, NULL, 0 ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not send completion: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_completion;
+	}
+
+	/* Hand off to RNDIS */
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		rndis_rx ( rndis, iob_disown ( iobuf ) );
+	}
+
+	return 0;
+
+ err_completion:
+ err_sanity:
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+	return rc;
+}
+
+/**
+ * Handle received completion packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_completion ( struct vmbus_device *vmdev, uint64_t xid,
+				    const void *data, size_t len ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+	struct io_buffer *iobuf;
+	int ( * completion ) ( struct netvsc_device *netvsc,
+			       const void *data, size_t len );
+	unsigned int xrid = ( xid - NETVSC_BASE_XID );
+	unsigned int tx_id;
+	int rc;
+
+	/* Handle transmit completion, if applicable */
+	tx_id = ( xrid - NETVSC_TX_BASE_XRID );
+	if ( ( tx_id < NETVSC_TX_NUM_DESC ) &&
+	     ( ( iobuf = netvsc->tx.iobufs[tx_id] ) != NULL ) ) {
+
+		/* Free buffer ID */
+		netvsc->tx.iobufs[tx_id] = NULL;
+		netvsc->tx.ids[ ( netvsc->tx.id_cons++ ) &
+				( netvsc->tx.count - 1 ) ] = tx_id;
+
+		/* Hand back to RNDIS */
+		rndis_tx_complete ( rndis, iobuf );
+		return 0;
+	}
+
+	/* Otherwise determine completion handler */
+	if ( xrid == NETVSC_INIT_XRID ) {
+		completion = netvsc_initialised;
+	} else if ( xrid == NETVSC_RX_ESTABLISH_XRID ) {
+		completion = netvsc_rx_established_buffer;
+	} else if ( ( netvsc->wait_xrid != 0 ) &&
+		    ( xrid == netvsc->wait_xrid ) ) {
+		completion = netvsc_completed;
+	} else {
+		DBGC ( netvsc, "NETVSC %s received unexpected completion "
+		       "(%08llx)\n", netvsc->name, xid );
+		return -EPIPE;
+	}
+
+	/* Hand off to completion handler */
+	rc = completion ( netvsc, data, len );
+
+	/* Record completion handler result if applicable */
+	if ( xrid == netvsc->wait_xrid ) {
+		netvsc->wait_xrid = 0;
+		netvsc->wait_rc = rc;
+	}
+
+	return rc;
+}
+
+/**
+ * Handle received cancellation packet
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @ret rc		Return status code
+ */
+static int netvsc_recv_cancellation ( struct vmbus_device *vmdev,
+				      uint64_t xid ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+	struct netvsc_device *netvsc = rndis->priv;
+
+	DBGC ( netvsc, "NETVSC %s received unsupported cancellation packet "
+	       "(%08llx):\n", netvsc->name, xid );
+	return -ENOTSUP;
+}
+
+/** VMBus channel operations */
+static struct vmbus_channel_operations netvsc_channel_operations = {
+	.recv_control = netvsc_recv_control,
+	.recv_data = netvsc_recv_data,
+	.recv_completion = netvsc_recv_completion,
+	.recv_cancellation = netvsc_recv_cancellation,
+};
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v rndis		RNDIS device
+ */
+static void netvsc_poll ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	struct vmbus_device *vmdev = netvsc->vmdev;
+
+	/* Poll VMBus device */
+	while ( vmbus_has_data ( vmdev ) )
+		vmbus_poll ( vmdev );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * If this method returns success then the RNDIS device must
+ * eventually report completion via rndis_tx_complete().
+ */
+static int netvsc_transmit ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	struct rndis_header *header = iobuf->data;
+	struct netvsc_rndis_message msg;
+	unsigned int tx_id;
+	unsigned int xrid;
+	uint64_t xid;
+	int rc;
+
+	/* Sanity check */
+	assert ( iob_len ( iobuf ) >= sizeof ( *header ) );
+	assert ( iob_len ( iobuf ) == le32_to_cpu ( header->len ) );
+
+	/* Check that we have space in the transmit ring */
+	if ( netvsc_ring_is_full ( &netvsc->tx ) )
+		return rndis_tx_defer ( rndis, iobuf );
+
+	/* Allocate buffer ID and calculate transaction ID */
+	tx_id = netvsc->tx.ids[ netvsc->tx.id_prod & ( netvsc->tx.count - 1 ) ];
+	assert ( netvsc->tx.iobufs[tx_id] == NULL );
+	xrid = ( NETVSC_TX_BASE_XRID + tx_id );
+	xid = ( NETVSC_BASE_XID + xrid );
+
+	/* Construct message */
+	memset ( &msg, 0, sizeof ( msg ) );
+	msg.header.type = cpu_to_le32 ( NETVSC_RNDIS_MSG );
+	msg.channel = ( ( header->type == cpu_to_le32 ( RNDIS_PACKET_MSG ) ) ?
+			NETVSC_RNDIS_DATA : NETVSC_RNDIS_CONTROL );
+	msg.buffer = cpu_to_le32 ( NETVSC_RNDIS_NO_BUFFER );
+
+	/* Send message */
+	if ( ( rc = vmbus_send_data ( netvsc->vmdev, xid, &msg, sizeof ( msg ),
+				      iobuf ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not send RNDIS message: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Store I/O buffer and consume buffer ID */
+	netvsc->tx.iobufs[tx_id] = iobuf;
+	netvsc->tx.id_prod++;
+
+	return 0;
+}
+
+/**
+ * Cancel transmission
+ *
+ * @v netvsc		NetVSC device
+ * @v iobuf		I/O buffer
+ * @v tx_id		Transmission ID
+ */
+static void netvsc_cancel_transmit ( struct netvsc_device *netvsc,
+				     struct io_buffer *iobuf,
+				     unsigned int tx_id ) {
+	unsigned int xrid;
+	uint64_t xid;
+
+	/* Send cancellation */
+	xrid = ( NETVSC_TX_BASE_XRID + tx_id );
+	xid = ( NETVSC_BASE_XID + xrid );
+	DBGC ( netvsc, "NETVSC %s cancelling transmission %#x\n",
+	       netvsc->name, tx_id );
+	vmbus_send_cancellation ( netvsc->vmdev, xid );
+
+	/* Report back to RNDIS */
+	rndis_tx_complete_err ( netvsc->rndis, iobuf, -ECANCELED );
+}
+
+/**
+ * Create descriptor ring
+ *
+ * @v netvsc		NetVSC device
+ * @v ring		Descriptor ring
+ * @ret rc		Return status code
+ */
+static int netvsc_create_ring ( struct netvsc_device *netvsc __unused,
+				struct netvsc_ring *ring ) {
+	unsigned int i;
+
+	/* Initialise buffer ID ring */
+	for ( i = 0 ; i < ring->count ; i++ ) {
+		ring->ids[i] = i;
+		assert ( ring->iobufs[i] == NULL );
+	}
+	ring->id_prod = 0;
+	ring->id_cons = 0;
+
+	return 0;
+}
+
+/**
+ * Destroy descriptor ring
+ *
+ * @v netvsc		NetVSC device
+ * @v ring		Descriptor ring
+ * @v discard		Method used to discard outstanding buffer, or NULL
+ */
+static void netvsc_destroy_ring ( struct netvsc_device *netvsc,
+				  struct netvsc_ring *ring,
+				  void ( * discard ) ( struct netvsc_device *,
+						       struct io_buffer *,
+						       unsigned int ) ) {
+	struct io_buffer *iobuf;
+	unsigned int i;
+
+	/* Flush any outstanding buffers */
+	for ( i = 0 ; i < ring->count ; i++ ) {
+		iobuf = ring->iobufs[i];
+		if ( ! iobuf )
+			continue;
+		ring->iobufs[i] = NULL;
+		ring->ids[ ( ring->id_cons++ ) & ( ring->count - 1 ) ] = i;
+		if ( discard )
+			discard ( netvsc, iobuf, i );
+	}
+
+	/* Sanity check */
+	assert ( netvsc_ring_is_empty ( ring ) );
+}
+
+/**
+ * Copy data from data buffer
+ *
+ * @v pages		Transfer page set
+ * @v data		Data buffer
+ * @v offset		Offset within page set
+ * @v len		Length within page set
+ * @ret rc		Return status code
+ */
+static int netvsc_buffer_copy ( struct vmbus_xfer_pages *pages, void *data,
+				size_t offset, size_t len ) {
+	struct netvsc_buffer *buffer =
+		container_of ( pages, struct netvsc_buffer, pages );
+
+	/* Sanity check */
+	if ( ( offset > buffer->len ) || ( len > ( buffer->len - offset ) ) )
+		return -ERANGE;
+
+	/* Copy data from buffer */
+	copy_from_user ( data, buffer->data, offset, len );
+
+	return 0;
+}
+
+/** Transfer page set operations */
+static struct vmbus_xfer_pages_operations netvsc_xfer_pages_operations = {
+	.copy = netvsc_buffer_copy,
+};
+
+/**
+ * Create data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ * @ret rc		Return status code
+ */
+static int netvsc_create_buffer ( struct netvsc_device *netvsc,
+				  struct netvsc_buffer *buffer ) {
+	struct vmbus_device *vmdev = netvsc->vmdev;
+	int gpadl;
+	int rc;
+
+	/* Allocate receive buffer */
+	buffer->data = umalloc ( buffer->len );
+	if ( ! buffer->data ) {
+		DBGC ( netvsc, "NETVSC %s could not allocate %zd-byte buffer\n",
+		       netvsc->name, buffer->len );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Establish GPA descriptor list */
+	gpadl = vmbus_establish_gpadl ( vmdev, buffer->data, buffer->len );
+	if ( gpadl < 0 ) {
+		rc = gpadl;
+		DBGC ( netvsc, "NETVSC %s could not establish GPADL: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_establish_gpadl;
+	}
+	buffer->gpadl = gpadl;
+
+	/* Register transfer page set */
+	if ( ( rc = vmbus_register_pages ( vmdev, &buffer->pages ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not register transfer pages: "
+		       "%s\n", netvsc->name, strerror ( rc ) );
+		goto err_register_pages;
+	}
+
+	return 0;
+
+	vmbus_unregister_pages ( vmdev, &buffer->pages );
+ err_register_pages:
+	vmbus_gpadl_teardown ( vmdev, gpadl );
+ err_establish_gpadl:
+	ufree ( buffer->data );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Destroy data buffer
+ *
+ * @v netvsc		NetVSC device
+ * @v buffer		Data buffer
+ */
+static void netvsc_destroy_buffer ( struct netvsc_device *netvsc,
+				    struct netvsc_buffer *buffer ) {
+	struct vmbus_device *vmdev = netvsc->vmdev;
+	int rc;
+
+	/* Unregister transfer pages */
+	vmbus_unregister_pages ( vmdev, &buffer->pages );
+
+	/* Tear down GPA descriptor list */
+	if ( ( rc = vmbus_gpadl_teardown ( vmdev, buffer->gpadl ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not tear down GPADL: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		/* Death is imminent.  The host may well continue to
+		 * write to the data buffer.  The best we can do is
+		 * leak memory for now and hope that the host doesn't
+		 * write to this region after we load an OS.
+		 */
+		return;
+	}
+
+	/* Free buffer */
+	ufree ( buffer->data );
+}
+
+/**
+ * Open device
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int netvsc_open ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+	int rc;
+
+	/* Initialise receive buffer */
+	if ( ( rc = netvsc_create_buffer ( netvsc, &netvsc->rx ) ) != 0 )
+		goto err_create_rx;
+
+	/* Open channel */
+	if ( ( rc = vmbus_open ( netvsc->vmdev, &netvsc_channel_operations,
+				 PAGE_SIZE, PAGE_SIZE, NETVSC_MTU ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not open VMBus: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_vmbus_open;
+	}
+
+	/* Initialise communication with NetVSP */
+	if ( ( rc = netvsc_initialise ( netvsc ) ) != 0 )
+		goto err_initialise;
+	if ( ( rc = netvsc_ndis_version ( netvsc ) ) != 0 )
+		goto err_ndis_version;
+
+	/* Initialise transmit ring */
+	if ( ( rc = netvsc_create_ring ( netvsc, &netvsc->tx ) ) != 0 )
+		goto err_create_tx;
+
+	/* Establish receive buffer */
+	if ( ( rc = netvsc_establish_buffer ( netvsc, &netvsc->rx ) ) != 0 )
+		goto err_establish_rx;
+
+	return 0;
+
+	netvsc_revoke_buffer ( netvsc, &netvsc->rx );
+ err_establish_rx:
+	netvsc_destroy_ring ( netvsc, &netvsc->tx, NULL );
+ err_create_tx:
+ err_ndis_version:
+ err_initialise:
+	vmbus_close ( netvsc->vmdev );
+ err_vmbus_open:
+	netvsc_destroy_buffer ( netvsc, &netvsc->rx );
+ err_create_rx:
+	return rc;
+}
+
+/**
+ * Close device
+ *
+ * @v rndis		RNDIS device
+ */
+static void netvsc_close ( struct rndis_device *rndis ) {
+	struct netvsc_device *netvsc = rndis->priv;
+
+	/* Revoke receive buffer */
+	netvsc_revoke_buffer ( netvsc, &netvsc->rx );
+
+	/* Destroy transmit ring */
+	netvsc_destroy_ring ( netvsc, &netvsc->tx, netvsc_cancel_transmit );
+
+	/* Close channel */
+	vmbus_close ( netvsc->vmdev );
+
+	/* Destroy receive buffer */
+	netvsc_destroy_buffer ( netvsc, &netvsc->rx );
+}
+
+/** RNDIS operations */
+static struct rndis_operations netvsc_operations = {
+	.open = netvsc_open,
+	.close = netvsc_close,
+	.transmit = netvsc_transmit,
+	.poll = netvsc_poll,
+};
+
+/**
+ * Probe device
+ *
+ * @v vmdev		VMBus device
+ * @ret rc		Return status code
+ */
+static int netvsc_probe ( struct vmbus_device *vmdev ) {
+	struct netvsc_device *netvsc;
+	struct rndis_device *rndis;
+	int rc;
+
+	/* Allocate and initialise structure */
+	rndis = alloc_rndis ( sizeof ( *netvsc ) );
+	if ( ! rndis ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	rndis_init ( rndis, &netvsc_operations );
+	rndis->netdev->dev = &vmdev->dev;
+	netvsc = rndis->priv;
+	netvsc->vmdev = vmdev;
+	netvsc->rndis = rndis;
+	netvsc->name = vmdev->dev.name;
+	netvsc_init_ring ( &netvsc->tx, NETVSC_TX_NUM_DESC,
+			   netvsc->tx_iobufs, netvsc->tx_ids );
+	netvsc_init_buffer ( &netvsc->rx, NETVSC_RX_BUF_PAGESET,
+			     &netvsc_xfer_pages_operations,
+			     NETVSC_RX_ESTABLISH_MSG, NETVSC_RX_ESTABLISH_XRID,
+			     NETVSC_RX_REVOKE_MSG, NETVSC_RX_REVOKE_XRID,
+			     NETVSC_RX_BUF_LEN );
+	vmbus_set_drvdata ( vmdev, rndis );
+
+	/* Register RNDIS device */
+	if ( ( rc = register_rndis ( rndis ) ) != 0 ) {
+		DBGC ( netvsc, "NETVSC %s could not register: %s\n",
+		       netvsc->name, strerror ( rc ) );
+		goto err_register;
+	}
+
+	return 0;
+
+	unregister_rndis ( rndis );
+ err_register:
+	free_rndis ( rndis );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v vmdev		VMBus device
+ */
+static void netvsc_remove ( struct vmbus_device *vmdev ) {
+	struct rndis_device *rndis = vmbus_get_drvdata ( vmdev );
+
+	/* Unregister RNDIS device */
+	unregister_rndis ( rndis );
+
+	/* Free RNDIS device */
+	free_rndis ( rndis );
+}
+
+/** NetVSC driver */
+struct vmbus_driver netvsc_driver __vmbus_driver = {
+	.name = "netvsc",
+	.type = VMBUS_TYPE ( 0xf8615163, 0xdf3e, 0x46c5, 0x913f,
+			     0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e ),
+	.probe = netvsc_probe,
+	.remove = netvsc_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/netvsc.h b/qemu/roms/ipxe/src/drivers/net/netvsc.h
new file mode 100644
index 000000000..39eeb891c
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/netvsc.h
@@ -0,0 +1,365 @@
+#ifndef _NETVSC_H
+#define _NETVSC_H
+
+/** @file
+ *
+ * Hyper-V network virtual service client
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Maximum supported NetVSC message length */
+#define NETVSC_MTU 512
+
+/** Maximum time to wait for a transaction to complete
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_MAX_WAIT_MS 1000
+
+/** Number of transmit ring entries
+ *
+ * Must be a power of two.  This is a policy decision.  This value
+ * must be sufficiently small to guarantee that we never run out of
+ * space in the VMBus outbound ring buffer.
+ */
+#define NETVSC_TX_NUM_DESC 32
+
+/** RX data buffer page set ID
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_RX_BUF_PAGESET 0xbead
+
+/** RX data buffer length
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_RX_BUF_LEN ( 16 * PAGE_SIZE )
+
+/** Base transaction ID
+ *
+ * This is a policy decision.
+ */
+#define NETVSC_BASE_XID 0x18ae0000UL
+
+/** Relative transaction IDs */
+enum netvsc_xrid {
+	/** Transmit descriptors (one per transmit buffer ID) */
+	NETVSC_TX_BASE_XRID = 0,
+	/** Initialisation */
+	NETVSC_INIT_XRID = ( NETVSC_TX_BASE_XRID + NETVSC_TX_NUM_DESC ),
+	/** NDIS version */
+	NETVSC_NDIS_VERSION_XRID,
+	/** Establish receive buffer */
+	NETVSC_RX_ESTABLISH_XRID,
+	/** Revoke receive buffer */
+	NETVSC_RX_REVOKE_XRID,
+};
+
+/** NetVSC status codes */
+enum netvsc_status {
+	NETVSC_NONE = 0,
+	NETVSC_OK = 1,
+	NETVSC_FAIL = 2,
+	NETVSC_TOO_NEW = 3,
+	NETVSC_TOO_OLD = 4,
+	NETVSC_BAD_PACKET = 5,
+	NETVSC_BUSY = 6,
+	NETVSC_UNSUPPORTED = 7,
+};
+
+/** NetVSC message header */
+struct netvsc_header {
+	/** Type */
+	uint32_t type;
+} __attribute__ (( packed ));
+
+/** NetVSC initialisation message */
+#define NETVSC_INIT_MSG 1
+
+/** NetVSC initialisation message */
+struct netvsc_init_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Minimum supported protocol version */
+	uint32_t min;
+	/** Maximum supported protocol version */
+	uint32_t max;
+	/** Reserved */
+	uint8_t reserved[20];
+} __attribute__ (( packed ));
+
+/** Oldest known NetVSC protocol version */
+#define NETVSC_VERSION_1 2 /* sic */
+
+/** NetVSC initialisation completion */
+#define NETVSC_INIT_CMPLT 2
+
+/** NetVSC initialisation completion */
+struct netvsc_init_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Protocol version */
+	uint32_t version;
+	/** Maximum memory descriptor list length */
+	uint32_t max_mdl_len;
+	/** Status */
+	uint32_t status;
+	/** Reserved */
+	uint8_t reserved[16];
+} __attribute__ (( packed ));
+
+/** NetVSC NDIS version message */
+#define NETVSC_NDIS_VERSION_MSG 100
+
+/** NetVSC NDIS version message */
+struct netvsc_ndis_version_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Reserved */
+	uint8_t reserved[20];
+} __attribute__ (( packed ));
+
+/** NetVSC NDIS major version */
+#define NETVSC_NDIS_MAJOR 6
+
+/** NetVSC NDIS minor version */
+#define NETVSC_NDIS_MINOR 1
+
+/** NetVSC establish receive data buffer message */
+#define NETVSC_RX_ESTABLISH_MSG 101
+
+/** NetVSC establish receive data buffer completion */
+#define NETVSC_RX_ESTABLISH_CMPLT 102
+
+/** NetVSC revoke receive data buffer message */
+#define NETVSC_RX_REVOKE_MSG 103
+
+/** NetVSC establish transmit data buffer message */
+#define NETVSC_TX_ESTABLISH_MSG 104
+
+/** NetVSC establish transmit data buffer completion */
+#define NETVSC_TX_ESTABLISH_CMPLT 105
+
+/** NetVSC revoke transmit data buffer message */
+#define NETVSC_TX_REVOKE_MSG 106
+
+/** NetVSC establish data buffer message */
+struct netvsc_establish_buffer_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Reserved */
+	uint8_t reserved[22];
+} __attribute__ (( packed ));
+
+/** NetVSC receive data buffer section */
+struct netvsc_rx_buffer_section {
+	/** Starting offset */
+	uint32_t start;
+	/** Subsection length */
+	uint32_t len;
+	/** Number of subsections */
+	uint32_t count;
+	/** Ending offset */
+	uint32_t end;
+} __attribute__ (( packed ));
+
+/** NetVSC establish receive data buffer completion */
+struct netvsc_rx_establish_buffer_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Status */
+	uint32_t status;
+	/** Number of sections (must be 1) */
+	uint32_t count;
+	/** Section descriptors */
+	struct netvsc_rx_buffer_section section[1];
+} __attribute__ (( packed ));
+
+/** NetVSC establish transmit data buffer completion */
+struct netvsc_tx_establish_buffer_completion {
+	/** Message header */
+	struct netvsc_header header;
+	/** Status */
+	uint32_t status;
+	/** Section length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** NetVSC revoke data buffer message */
+struct netvsc_revoke_buffer_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Reserved */
+	uint8_t reserved[26];
+} __attribute__ (( packed ));
+
+/** NetVSC RNDIS message */
+#define NETVSC_RNDIS_MSG 107
+
+/** NetVSC RNDIS message */
+struct netvsc_rndis_message {
+	/** Message header */
+	struct netvsc_header header;
+	/** RNDIS channel */
+	uint32_t channel;
+	/** Buffer index (or NETVSC_RNDIS_NO_BUFFER) */
+	uint32_t buffer;
+	/** Buffer length */
+	uint32_t len;
+	/** Reserved */
+	uint8_t reserved[16];
+} __attribute__ (( packed ));
+
+/** RNDIS data channel (for RNDIS_PACKET_MSG only) */
+#define NETVSC_RNDIS_DATA 0
+
+/** RNDIS control channel (for all other RNDIS messages) */
+#define NETVSC_RNDIS_CONTROL 1
+
+/** "No buffer used" index */
+#define NETVSC_RNDIS_NO_BUFFER 0xffffffffUL
+
+/** A NetVSC descriptor ring */
+struct netvsc_ring {
+	/** Number of descriptors */
+	unsigned int count;
+	/** I/O buffers, indexed by buffer ID */
+	struct io_buffer **iobufs;
+	/** Buffer ID ring */
+	uint8_t *ids;
+	/** Buffer ID producer counter */
+	unsigned int id_prod;
+	/** Buffer ID consumer counter */
+	unsigned int id_cons;
+};
+
+/**
+ * Initialise descriptor ring
+ *
+ * @v ring		Descriptor ring
+ * @v count		Maximum number of used descriptors
+ * @v iobufs		I/O buffers
+ * @v ids		Buffer IDs
+ */
+static inline __attribute__ (( always_inline )) void
+netvsc_init_ring ( struct netvsc_ring *ring, unsigned int count,
+		   struct io_buffer **iobufs, uint8_t *ids ) {
+
+	ring->count = count;
+	ring->iobufs = iobufs;
+	ring->ids = ids;
+}
+
+/**
+ * Check whether or not descriptor ring is full
+ *
+ * @v ring		Descriptor ring
+ * @v is_full		Ring is full
+ */
+static inline __attribute__ (( always_inline )) int
+netvsc_ring_is_full ( struct netvsc_ring *ring ) {
+	unsigned int fill_level;
+
+	fill_level = ( ring->id_prod - ring->id_cons );
+	assert ( fill_level <= ring->count );
+	return ( fill_level >= ring->count );
+}
+
+/**
+ * Check whether or not descriptor ring is empty
+ *
+ * @v ring		Descriptor ring
+ * @v is_empty		Ring is empty
+ */
+static inline __attribute__ (( always_inline )) int
+netvsc_ring_is_empty ( struct netvsc_ring *ring ) {
+
+	return ( ring->id_prod == ring->id_cons );
+}
+
+/** A NetVSC data buffer */
+struct netvsc_buffer {
+	/** Transfer page set */
+	struct vmbus_xfer_pages pages;
+	/** Establish data buffer message type */
+	uint8_t establish_type;
+	/** Establish data buffer relative transaction ID */
+	uint8_t establish_xrid;
+	/** Revoke data buffer message type */
+	uint8_t revoke_type;
+	/** Revoke data buffer relative transaction ID */
+	uint8_t revoke_xrid;
+	/** Buffer length */
+	size_t len;
+	/** Buffer */
+	userptr_t data;
+	/** GPADL ID */
+	unsigned int gpadl;
+};
+
+/**
+ * Initialise data buffer
+ *
+ * @v buffer		Data buffer
+ * @v pageset		Page set ID
+ * @v op		Page set operations
+ * @v establish_type	Establish data buffer message type
+ * @v establish_xrid	Establish data buffer relative transaction ID
+ * @v revoke_type	Revoke data buffer message type
+ * @v revoke_type	Revoke data buffer relative transaction ID
+ * @v len		Required length
+ */
+static inline __attribute__ (( always_inline )) void
+netvsc_init_buffer ( struct netvsc_buffer *buffer, uint16_t pageset,
+		     struct vmbus_xfer_pages_operations *op,
+		     uint8_t establish_type, uint8_t establish_xrid,
+		     uint8_t revoke_type, uint8_t revoke_xrid, size_t len ) {
+
+	buffer->pages.pageset = cpu_to_le16 ( pageset );
+	buffer->pages.op = op;
+	buffer->establish_type = establish_type;
+	buffer->establish_xrid = establish_xrid;
+	buffer->revoke_type = revoke_type;
+	buffer->revoke_xrid = revoke_xrid;
+	buffer->len = len;
+}
+
+/** A NetVSC device */
+struct netvsc_device {
+	/** VMBus device */
+	struct vmbus_device *vmdev;
+	/** RNDIS device */
+	struct rndis_device *rndis;
+	/** Name */
+	const char *name;
+
+	/** Transmit ring */
+	struct netvsc_ring tx;
+	/** Transmit buffer IDs */
+	uint8_t tx_ids[NETVSC_TX_NUM_DESC];
+	/** Transmit I/O buffers */
+	struct io_buffer *tx_iobufs[NETVSC_TX_NUM_DESC];
+
+	/** Receive buffer */
+	struct netvsc_buffer rx;
+
+	/** Relative transaction ID for current blocking transaction */
+	unsigned int wait_xrid;
+	/** Return status code for current blocking transaction */
+	int wait_rc;
+};
+
+#endif /* _NETVSC_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/phantom/nx_bitops.h b/qemu/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
index 15f3d3767..1687b6952 100644
--- a/qemu/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
+++ b/qemu/roms/ipxe/src/drivers/net/phantom/nx_bitops.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/net/phantom/phantom.c b/qemu/roms/ipxe/src/drivers/net/phantom/phantom.c
index e70ded08c..38b66743c 100644
--- a/qemu/roms/ipxe/src/drivers/net/phantom/phantom.c
+++ b/qemu/roms/ipxe/src/drivers/net/phantom/phantom.c
@@ -16,9 +16,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/phantom/phantom.h b/qemu/roms/ipxe/src/drivers/net/phantom/phantom.h
index 1647168ba..967603409 100644
--- a/qemu/roms/ipxe/src/drivers/net/phantom/phantom.h
+++ b/qemu/roms/ipxe/src/drivers/net/phantom/phantom.h
@@ -19,9 +19,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/net/phantom/phantom_hw.h b/qemu/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
index 7dfff52b2..016730de3 100644
--- a/qemu/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
+++ b/qemu/roms/ipxe/src/drivers/net/phantom/phantom_hw.h
@@ -19,9 +19,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/net/pnic.c b/qemu/roms/ipxe/src/drivers/net/pnic.c
index 4170cc640..ca64299ea 100644
--- a/qemu/roms/ipxe/src/drivers/net/pnic.c
+++ b/qemu/roms/ipxe/src/drivers/net/pnic.c
@@ -6,8 +6,18 @@ Bochs Pseudo NIC driver for Etherboot
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  *
  * See pnic_api.h for an explanation of the Bochs Pseudo NIC.
  */
diff --git a/qemu/roms/ipxe/src/drivers/net/prism2.c b/qemu/roms/ipxe/src/drivers/net/prism2.c
index ab974264c..4331f2cd0 100644
--- a/qemu/roms/ipxe/src/drivers/net/prism2.c
+++ b/qemu/roms/ipxe/src/drivers/net/prism2.c
@@ -9,8 +9,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
diff --git a/qemu/roms/ipxe/src/drivers/net/prism2_pci.c b/qemu/roms/ipxe/src/drivers/net/prism2_pci.c
index 72549babf..69ddf0fb0 100644
--- a/qemu/roms/ipxe/src/drivers/net/prism2_pci.c
+++ b/qemu/roms/ipxe/src/drivers/net/prism2_pci.c
@@ -10,8 +10,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -40,8 +50,6 @@ static void prism2_pci_disable ( struct nic *nic ) {
 
 static struct pci_device_id prism2_pci_nics[] = {
 PCI_ROM(0x1260, 0x3873, "prism2_pci",	"Harris Semiconductor Prism2.5 clone", 0),
-PCI_ROM(0x1260, 0x3873, "hwp01170",	"ActionTec HWP01170", 0),
-PCI_ROM(0x1260, 0x3873, "dwl520",	"DLink DWL-520", 0),
 };
 
 PCI_DRIVER ( prism2_pci_driver, prism2_pci_nics, PCI_NO_CLASS );
diff --git a/qemu/roms/ipxe/src/drivers/net/prism2_plx.c b/qemu/roms/ipxe/src/drivers/net/prism2_plx.c
index 2098f7f09..a73b0e087 100644
--- a/qemu/roms/ipxe/src/drivers/net/prism2_plx.c
+++ b/qemu/roms/ipxe/src/drivers/net/prism2_plx.c
@@ -10,8 +10,18 @@ $Id$
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -44,10 +54,10 @@ static int prism2_find_plx ( hfa384x_t *hw, struct pci_device *p )
   
   /* Obtain all memory and IO base addresses */
   pci_read_config_dword( p, PLX_LOCAL_CONFIG_REGISTER_BASE, &plx_lcr);
-  plx_lcr &= PCI_BASE_ADDRESS_IO_MASK;
+  plx_lcr &= ~PCI_BASE_ADDRESS_IO_MASK;
   pci_read_config_dword( p, PRISM2_PLX_ATTR_MEM_BASE, &attr_mem);
   pci_read_config_dword( p, PRISM2_PLX_IO_BASE, &iobase);
-  iobase &= PCI_BASE_ADDRESS_IO_MASK;
+  iobase &= ~PCI_BASE_ADDRESS_IO_MASK;
 
   /* Fill out hw structure */
   hw->iobase = iobase;
diff --git a/qemu/roms/ipxe/src/drivers/net/realtek.c b/qemu/roms/ipxe/src/drivers/net/realtek.c
index 0aca8c77f..022b59324 100644
--- a/qemu/roms/ipxe/src/drivers/net/realtek.c
+++ b/qemu/roms/ipxe/src/drivers/net/realtek.c
@@ -17,9 +17,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -194,7 +198,6 @@ static int realtek_init_eeprom ( struct net_device *netdev ) {
 		DBGC ( rtl, "REALTEK %p EEPROM is a 93C46\n", rtl );
 		init_at93c46 ( &rtl->eeprom, 16 );
 	}
-	rtl->eeprom.bus = &rtl->spibit.bus;
 
 	/* Check for EEPROM presence.  Some onboard NICs will have no
 	 * EEPROM connected, with the BIOS being responsible for
@@ -1085,6 +1088,7 @@ static void realtek_detect ( struct realtek_nic *rtl ) {
 			       rtl );
 			rtl->legacy = 1;
 		}
+		rtl->eeprom.bus = &rtl->spibit.bus;
 	}
 }
 
@@ -1132,7 +1136,8 @@ static int realtek_probe ( struct pci_device *pci ) {
 	realtek_detect ( rtl );
 
 	/* Initialise EEPROM */
-	if ( ( rc = realtek_init_eeprom ( netdev ) ) == 0 ) {
+	if ( rtl->eeprom.bus &&
+	     ( ( rc = realtek_init_eeprom ( netdev ) ) == 0 ) ) {
 
 		/* Read MAC address from EEPROM */
 		if ( ( rc = nvs_read ( &rtl->eeprom.nvs, RTL_EEPROM_MAC,
diff --git a/qemu/roms/ipxe/src/drivers/net/realtek.h b/qemu/roms/ipxe/src/drivers/net/realtek.h
index ac33405e8..b1ce7f98f 100644
--- a/qemu/roms/ipxe/src/drivers/net/realtek.h
+++ b/qemu/roms/ipxe/src/drivers/net/realtek.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/spi.h>
 #include <ipxe/spi_bit.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
index 8851d1bfb..5f97480fa 100644
--- a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
+++ b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8180.c
@@ -3,15 +3,23 @@
 FILE_LICENCE(GPL2_OR_LATER);
 
 #include <ipxe/pci.h>
+#include "rtl818x.h"
 
-REQUIRE_OBJECT(rtl818x);
-REQUIRE_OBJECT(rtl8180_grf5101);
-REQUIRE_OBJECT(rtl8180_max2820);
-REQUIRE_OBJECT(rtl8180_sa2400);
-
-static struct pci_device_id rtl8180_nics[] __unused = {
+static struct pci_device_id rtl8180_nics[] = {
 	PCI_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
 	PCI_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
 	PCI_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
 	PCI_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
 };
+
+struct pci_driver rtl8180_driver __pci_driver = {
+	.ids            = rtl8180_nics,
+	.id_count       = sizeof(rtl8180_nics) / sizeof(rtl8180_nics[0]),
+	.probe		= rtl818x_probe,
+	.remove		= rtl818x_remove,
+};
+
+REQUIRING_SYMBOL(rtl8180_driver);
+REQUIRE_OBJECT(rtl8180_grf5101);
+REQUIRE_OBJECT(rtl8180_max2820);
+REQUIRE_OBJECT(rtl8180_sa2400);
diff --git a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
index fd27e5c8c..234978cea 100644
--- a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
+++ b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl8185.c
@@ -3,12 +3,20 @@
 FILE_LICENCE(GPL2_OR_LATER);
 
 #include <ipxe/pci.h>
-
-REQUIRE_OBJECT(rtl818x);
-REQUIRE_OBJECT(rtl8185_rtl8225);
+#include "rtl818x.h"
 
 static struct pci_device_id rtl8185_nics[] __unused = {
 	PCI_ROM(0x10ec, 0x8185, "rtl8185", "Realtek 8185", 0),
 	PCI_ROM(0x1799, 0x700f, "f5d7000", "Belkin F5D7000", 0),
 	PCI_ROM(0x1799, 0x701f, "f5d7010", "Belkin F5D7010", 0),
 };
+
+struct pci_driver rtl8185_driver __pci_driver = {
+	.ids            = rtl8185_nics,
+	.id_count       = sizeof(rtl8185_nics) / sizeof(rtl8185_nics[0]),
+	.probe		= rtl818x_probe,
+	.remove		= rtl818x_remove,
+};
+
+REQUIRING_SYMBOL(rtl8185_driver);
+REQUIRE_OBJECT(rtl8185_rtl8225);
diff --git a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
index cf4c7556f..8b3c206d4 100644
--- a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
+++ b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.c
@@ -649,7 +649,7 @@ struct net80211_device_operations rtl818x_operations = {
 	.config = rtl818x_config,
 };
 
-static int rtl818x_probe(struct pci_device *pdev )
+int rtl818x_probe(struct pci_device *pdev )
 {
 	struct net80211_device *dev;
 	struct rtl818x_priv *priv;
@@ -820,7 +820,7 @@ static int rtl818x_probe(struct pci_device *pdev )
 	return err;
 }
 
-static void rtl818x_remove(struct pci_device *pdev)
+void rtl818x_remove(struct pci_device *pdev)
 {
 	struct net80211_device *dev = pci_get_drvdata(pdev);
 
@@ -830,25 +830,3 @@ static void rtl818x_remove(struct pci_device *pdev)
 	net80211_unregister(dev);
 	net80211_free(dev);
 }
-
-/* Hide PCI_ROM definitions in here from parserom.pl; the definitions
-   that should be used are in rtl8180.c and rtl8185.c. */
-#define RTL_ROM PCI_ROM
-
-static struct pci_device_id rtl818x_nics[] = {
-	RTL_ROM(0x10ec, 0x8185, "rtl8185", "Realtek 8185", 0),
-	RTL_ROM(0x1799, 0x700f, "f5d7000", "Belkin F5D7000", 0),
-	RTL_ROM(0x1799, 0x701f, "f5d7010", "Belkin F5D7010", 0),
-
-	RTL_ROM(0x10ec, 0x8180, "rtl8180", "Realtek 8180", 0),
-	RTL_ROM(0x1799, 0x6001, "f5d6001", "Belkin F5D6001", 0),
-	RTL_ROM(0x1799, 0x6020, "f5d6020", "Belkin F5D6020", 0),
-	RTL_ROM(0x1186, 0x3300, "dwl510",  "D-Link DWL-510", 0),
-};
-
-struct pci_driver rtl818x_driver __pci_driver = {
-	.ids            = rtl818x_nics,
-	.id_count       = sizeof(rtl818x_nics) / sizeof(rtl818x_nics[0]),
-	.probe		= rtl818x_probe,
-	.remove		= rtl818x_remove,
-};
diff --git a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
index 4e57d0bd3..ae4b8a96f 100644
--- a/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
+++ b/qemu/roms/ipxe/src/drivers/net/rtl818x/rtl818x.h
@@ -19,6 +19,7 @@
 
 #include <ipxe/spi_bit.h>
 #include <ipxe/tables.h>
+#include <ipxe/net80211.h>
 
 FILE_LICENCE(GPL2_ONLY);
 
@@ -356,4 +357,7 @@ struct rtl818x_rf_ops {
 	void (*conf_erp)(struct net80211_device *dev); /* set based on dev->erp_flags */
 };
 
+extern int rtl818x_probe(struct pci_device *pdev );
+extern void rtl818x_remove(struct pci_device *pdev);
+
 #endif /* RTL818X_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/skeleton.c b/qemu/roms/ipxe/src/drivers/net/skeleton.c
index 365111b8d..0435b9d0e 100644
--- a/qemu/roms/ipxe/src/drivers/net/skeleton.c
+++ b/qemu/roms/ipxe/src/drivers/net/skeleton.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/skeleton.h b/qemu/roms/ipxe/src/drivers/net/skeleton.h
index 3de2afa5b..2ab01bd56 100644
--- a/qemu/roms/ipxe/src/drivers/net/skeleton.h
+++ b/qemu/roms/ipxe/src/drivers/net/skeleton.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Skeleton BAR size */
 #define SKELETON_BAR_SIZE 256
diff --git a/qemu/roms/ipxe/src/drivers/net/smsc75xx.c b/qemu/roms/ipxe/src/drivers/net/smsc75xx.c
new file mode 100644
index 000000000..017e02a59
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/smsc75xx.c
@@ -0,0 +1,1057 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/profile.h>
+#include "smsc75xx.h"
+
+/** @file
+ *
+ * SMSC LAN75xx USB Ethernet driver
+ *
+ */
+
+/** Interrupt completion profiler */
+static struct profiler smsc75xx_intr_profiler __profiler =
+	{ .name = "smsc75xx.intr" };
+
+/** Bulk IN completion profiler */
+static struct profiler smsc75xx_in_profiler __profiler =
+	{ .name = "smsc75xx.in" };
+
+/** Bulk OUT profiler */
+static struct profiler smsc75xx_out_profiler __profiler =
+	{ .name = "smsc75xx.out" };
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Write register (without byte-swapping)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static int smsc75xx_raw_writel ( struct smsc75xx_device *smsc75xx,
+				 unsigned int address, uint32_t value ) {
+	int rc;
+
+	/* Write register */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_REGISTER_WRITE, 0,
+				  address, &value, sizeof ( value ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not write %03x: %s\n",
+		       smsc75xx, address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write register
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @v value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+smsc75xx_writel ( struct smsc75xx_device *smsc75xx, unsigned int address,
+		  uint32_t value ) {
+	int rc;
+
+	/* Write register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, address,
+					  cpu_to_le32 ( value ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Read register (without byte-swapping)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @ret value		Register value
+ * @ret rc		Return status code
+ */
+static int smsc75xx_raw_readl ( struct smsc75xx_device *smsc75xx,
+				unsigned int address, uint32_t *value ) {
+	int rc;
+
+	/* Read register */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_REGISTER_READ, 0,
+				  address, value, sizeof ( *value ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not read %03x: %s\n",
+		       smsc75xx, address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Read register
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		Register address
+ * @ret value		Register value
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+smsc75xx_readl ( struct smsc75xx_device *smsc75xx, unsigned int address,
+		 uint32_t *value ) {
+	int rc;
+
+	/* Read register */
+	if ( ( rc = smsc75xx_raw_readl ( smsc75xx, address, value ) ) != 0 )
+		return rc;
+	le32_to_cpus ( value );
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * EEPROM access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Wait for EEPROM to become idle
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_eeprom_wait ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t e2p_cmd;
+	unsigned int i;
+	int rc;
+
+	/* Wait for EPC_BSY to become clear */
+	for ( i = 0 ; i < SMSC75XX_EEPROM_MAX_WAIT_MS ; i++ ) {
+
+		/* Read E2P_CMD and check EPC_BSY */
+		if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_E2P_CMD,
+					     &e2p_cmd ) ) != 0 )
+			return rc;
+		if ( ! ( e2p_cmd & SMSC75XX_E2P_CMD_EPC_BSY ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p timed out waiting for EEPROM\n",
+	       smsc75xx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Read byte from EEPROM
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		EEPROM address
+ * @ret byte		Byte read, or negative error
+ */
+static int smsc75xx_eeprom_read_byte ( struct smsc75xx_device *smsc75xx,
+				       unsigned int address ) {
+	uint32_t e2p_cmd;
+	uint32_t e2p_data;
+	int rc;
+
+	/* Wait for EEPROM to become idle */
+	if ( ( rc = smsc75xx_eeprom_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Initiate read command */
+	e2p_cmd = ( SMSC75XX_E2P_CMD_EPC_BSY | SMSC75XX_E2P_CMD_EPC_CMD_READ |
+		    SMSC75XX_E2P_CMD_EPC_ADDR ( address ) );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_E2P_CMD,
+				      e2p_cmd ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_eeprom_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Read EEPROM data */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_E2P_DATA,
+				     &e2p_data ) ) != 0 )
+		return rc;
+
+	return SMSC75XX_E2P_DATA_GET ( e2p_data );
+}
+
+/**
+ * Read data from EEPROM
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v address		EEPROM address
+ * @v data		Data buffer
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static int smsc75xx_eeprom_read ( struct smsc75xx_device *smsc75xx,
+				  unsigned int address, void *data,
+				  size_t len ) {
+	uint8_t *bytes;
+	int byte;
+
+	/* Read bytes */
+	for ( bytes = data ; len-- ; address++, bytes++ ) {
+		byte = smsc75xx_eeprom_read_byte ( smsc75xx, address );
+		if ( byte < 0 )
+			return byte;
+		*bytes = byte;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * MII access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Wait for MII to become idle
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_mii_wait ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t mii_access;
+	unsigned int i;
+	int rc;
+
+	/* Wait for MIIBZY to become clear */
+	for ( i = 0 ; i < SMSC75XX_MII_MAX_WAIT_MS ; i++ ) {
+
+		/* Read MII_ACCESS and check MIIBZY */
+		if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_MII_ACCESS,
+					     &mii_access ) ) != 0 )
+			return rc;
+		if ( ! ( mii_access & SMSC75XX_MII_ACCESS_MIIBZY ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p timed out waiting for MII\n",
+	       smsc75xx );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Read from MII register
+ *
+ * @v mii		MII interface
+ * @v reg		Register address
+ * @ret value		Data read, or negative error
+ */
+static int smsc75xx_mii_read ( struct mii_interface *mii, unsigned int reg ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( mii, struct smsc75xx_device, mii );
+	uint32_t mii_access;
+	uint32_t mii_data;
+	int rc;
+
+	/* Wait for MII to become idle */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Initiate read command */
+	mii_access = ( SMSC75XX_MII_ACCESS_PHY_ADDRESS |
+		       SMSC75XX_MII_ACCESS_MIIRINDA ( reg ) |
+		       SMSC75XX_MII_ACCESS_MIIBZY );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_ACCESS,
+				      mii_access ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Read MII data */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_MII_DATA,
+				     &mii_data ) ) != 0 )
+		return rc;
+
+	return SMSC75XX_MII_DATA_GET ( mii_data );
+}
+
+/**
+ * Write to MII register
+ *
+ * @v mii		MII interface
+ * @v reg		Register address
+ * @v data		Data to write
+ * @ret rc		Return status code
+ */
+static int smsc75xx_mii_write ( struct mii_interface *mii, unsigned int reg,
+				unsigned int data ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( mii, struct smsc75xx_device, mii );
+	uint32_t mii_access;
+	uint32_t mii_data;
+	int rc;
+
+	/* Wait for MII to become idle */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	/* Write MII data */
+	mii_data = SMSC75XX_MII_DATA_SET ( data );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_DATA,
+				      mii_data ) ) != 0 )
+		return rc;
+
+	/* Initiate write command */
+	mii_access = ( SMSC75XX_MII_ACCESS_PHY_ADDRESS |
+		       SMSC75XX_MII_ACCESS_MIIRINDA ( reg ) |
+		       SMSC75XX_MII_ACCESS_MIIWNR |
+		       SMSC75XX_MII_ACCESS_MIIBZY );
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MII_ACCESS,
+				      mii_access ) ) != 0 )
+		return rc;
+
+	/* Wait for command to complete */
+	if ( ( rc = smsc75xx_mii_wait ( smsc75xx ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/** MII operations */
+static struct mii_operations smsc75xx_mii_operations = {
+	.read = smsc75xx_mii_read,
+	.write = smsc75xx_mii_write,
+};
+
+/**
+ * Check link status
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_check_link ( struct smsc75xx_device *smsc75xx ) {
+	struct net_device *netdev = smsc75xx->netdev;
+	int intr;
+	int rc;
+
+	/* Read PHY interrupt source */
+	intr = mii_read ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_SOURCE );
+	if ( intr < 0 ) {
+		rc = intr;
+		DBGC ( smsc75xx, "SMSC75XX %p could not get PHY interrupt "
+		       "source: %s\n", smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Acknowledge PHY interrupt */
+	if ( ( rc = mii_write ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_SOURCE,
+				intr ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not acknowledge PHY "
+		       "interrupt: %s\n", smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Check link status */
+	if ( ( rc = mii_check_link ( &smsc75xx->mii, netdev ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not check link: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	DBGC ( smsc75xx, "SMSC75XX %p link %s (intr %#04x)\n",
+	       smsc75xx, ( netdev_link_ok ( netdev ) ? "up" : "down" ), intr );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Statistics (for debugging)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get statistics
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v stats		Statistics to fill in
+ * @ret rc		Return status code
+ */
+static int smsc75xx_get_statistics ( struct smsc75xx_device *smsc75xx,
+				     struct smsc75xx_statistics *stats ) {
+	int rc;
+
+	/* Get statistics */
+	if ( ( rc = usb_control ( smsc75xx->usb, SMSC75XX_GET_STATISTICS, 0, 0,
+				  stats, sizeof ( *stats ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not get statistics: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Dump statistics (for debugging)
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_dump_statistics ( struct smsc75xx_device *smsc75xx ) {
+	struct smsc75xx_statistics stats;
+	int rc;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return 0;
+
+	/* Get statistics */
+	if ( ( rc = smsc75xx_get_statistics ( smsc75xx, &stats ) ) != 0 )
+		return rc;
+
+	/* Dump statistics */
+	DBGC ( smsc75xx, "SMSC75XX %p RXE fcs %d aln %d frg %d jab %d und %d "
+	       "ovr %d drp %d\n", smsc75xx, le32_to_cpu ( stats.rx.err.fcs ),
+	       le32_to_cpu ( stats.rx.err.alignment ),
+	       le32_to_cpu ( stats.rx.err.fragment ),
+	       le32_to_cpu ( stats.rx.err.jabber ),
+	       le32_to_cpu ( stats.rx.err.undersize ),
+	       le32_to_cpu ( stats.rx.err.oversize ),
+	       le32_to_cpu ( stats.rx.err.dropped ) );
+	DBGC ( smsc75xx, "SMSC75XX %p RXB ucast %d bcast %d mcast %d\n",
+	       smsc75xx, le32_to_cpu ( stats.rx.byte.unicast ),
+	       le32_to_cpu ( stats.rx.byte.broadcast ),
+	       le32_to_cpu ( stats.rx.byte.multicast ) );
+	DBGC ( smsc75xx, "SMSC75XX %p RXF ucast %d bcast %d mcast %d pause "
+	       "%d\n", smsc75xx, le32_to_cpu ( stats.rx.frame.unicast ),
+	       le32_to_cpu ( stats.rx.frame.broadcast ),
+	       le32_to_cpu ( stats.rx.frame.multicast ),
+	       le32_to_cpu ( stats.rx.frame.pause ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXE fcs %d def %d car %d cnt %d sgl %d "
+	       "mul %d exc %d lat %d\n", smsc75xx,
+	       le32_to_cpu ( stats.tx.err.fcs ),
+	       le32_to_cpu ( stats.tx.err.deferral ),
+	       le32_to_cpu ( stats.tx.err.carrier ),
+	       le32_to_cpu ( stats.tx.err.count ),
+	       le32_to_cpu ( stats.tx.err.single ),
+	       le32_to_cpu ( stats.tx.err.multiple ),
+	       le32_to_cpu ( stats.tx.err.excessive ),
+	       le32_to_cpu ( stats.tx.err.late ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXB ucast %d bcast %d mcast %d\n",
+	       smsc75xx, le32_to_cpu ( stats.tx.byte.unicast ),
+	       le32_to_cpu ( stats.tx.byte.broadcast ),
+	       le32_to_cpu ( stats.tx.byte.multicast ) );
+	DBGC ( smsc75xx, "SMSC75XX %p TXF ucast %d bcast %d mcast %d pause "
+	       "%d\n", smsc75xx, le32_to_cpu ( stats.tx.frame.unicast ),
+	       le32_to_cpu ( stats.tx.frame.broadcast ),
+	       le32_to_cpu ( stats.tx.frame.multicast ),
+	       le32_to_cpu ( stats.tx.frame.pause ) );
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Device reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Reset device
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_reset ( struct smsc75xx_device *smsc75xx ) {
+	uint32_t hw_cfg;
+	int rc;
+
+	/* Reset device */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_HW_CFG,
+				      SMSC75XX_HW_CFG_LRST ) ) != 0 )
+		return rc;
+
+	/* Wait for reset to complete */
+	udelay ( SMSC75XX_RESET_DELAY_US );
+
+	/* Check that reset has completed */
+	if ( ( rc = smsc75xx_readl ( smsc75xx, SMSC75XX_HW_CFG,
+				     &hw_cfg ) ) != 0 )
+		return rc;
+	if ( hw_cfg & SMSC75XX_HW_CFG_LRST ) {
+		DBGC ( smsc75xx, "SMSC75XX %p failed to reset\n", smsc75xx );
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_intr_complete ( struct usb_endpoint *ep,
+				     struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.intr );
+	struct net_device *netdev = smsc75xx->netdev;
+	struct smsc75xx_interrupt *intr;
+
+	/* Profile completions */
+	profile_start ( &smsc75xx_intr_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p interrupt failed: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+
+	/* Extract interrupt data */
+	if ( iob_len ( iobuf ) != sizeof ( *intr ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p malformed interrupt\n",
+		       smsc75xx );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		netdev_rx_err ( netdev, NULL, rc );
+		goto done;
+	}
+	intr = iobuf->data;
+
+	/* Record interrupt status */
+	smsc75xx->int_sts = le32_to_cpu ( intr->int_sts );
+	profile_stop ( &smsc75xx_intr_profiler );
+
+ done:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_intr_operations = {
+	.complete = smsc75xx_intr_complete,
+};
+
+/**
+ * Complete bulk IN transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_in_complete ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.in );
+	struct net_device *netdev = smsc75xx->netdev;
+	struct smsc75xx_rx_header *header;
+
+	/* Profile completions */
+	profile_start ( &smsc75xx_in_profiler );
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open ) {
+		free_iob ( iobuf );
+		return;
+	}
+
+	/* Record USB errors against the network device */
+	if ( rc != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p bulk IN failed: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < ( sizeof ( *header ) ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p underlength bulk IN\n",
+		       smsc75xx );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto err;
+	}
+
+	/* Strip header */
+	header = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *header ) );
+
+	/* Check for errors */
+	if ( header->command & cpu_to_le32 ( SMSC75XX_RX_RED ) ) {
+		DBGC ( smsc75xx, "SMSC75XX %p receive error (%08x):\n",
+		       smsc75xx, le32_to_cpu ( header->command ) );
+		DBGC_HDA ( smsc75xx, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	profile_stop ( &smsc75xx_in_profiler );
+	return;
+
+ err:
+	/* Hand off to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/** Bulk IN endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_in_operations = {
+	.complete = smsc75xx_in_complete,
+};
+
+/**
+ * Transmit packet
+ *
+ * @v smsc75xx		SMSC75xx device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int smsc75xx_out_transmit ( struct smsc75xx_device *smsc75xx,
+				   struct io_buffer *iobuf ) {
+	struct smsc75xx_tx_header *header;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Profile transmissions */
+	profile_start ( &smsc75xx_out_profiler );
+
+	/* Prepend header */
+	if ( ( rc = iob_ensure_headroom ( iobuf, sizeof ( *header ) ) ) != 0 )
+		return rc;
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->command = cpu_to_le32 ( SMSC75XX_TX_FCS | len );
+	header->tag = 0;
+	header->mss = 0;
+
+	/* Enqueue I/O buffer */
+	if ( ( rc = usb_stream ( &smsc75xx->usbnet.out, iobuf, 0 ) ) != 0 )
+		return rc;
+
+	profile_stop ( &smsc75xx_out_profiler );
+	return 0;
+}
+
+/**
+ * Complete bulk OUT transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void smsc75xx_out_complete ( struct usb_endpoint *ep,
+				    struct io_buffer *iobuf, int rc ) {
+	struct smsc75xx_device *smsc75xx =
+		container_of ( ep, struct smsc75xx_device, usbnet.out );
+	struct net_device *netdev = smsc75xx->netdev;
+
+	/* Report TX completion */
+	netdev_tx_complete_err ( netdev, iobuf, rc );
+}
+
+/** Bulk OUT endpoint operations */
+static struct usb_endpoint_driver_operations smsc75xx_out_operations = {
+	.complete = smsc75xx_out_complete,
+};
+
+/******************************************************************************
+ *
+ * Network device interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int smsc75xx_open ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	union smsc75xx_mac mac;
+	int rc;
+
+	/* Clear stored interrupt status */
+	smsc75xx->int_sts = 0;
+
+	/* Copy MAC address */
+	memset ( &mac, 0, sizeof ( mac ) );
+	memcpy ( mac.raw, netdev->ll_addr, ETH_ALEN );
+
+	/* Configure bulk IN empty response */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_HW_CFG,
+				      SMSC75XX_HW_CFG_BIR ) ) != 0 )
+		goto err_hw_cfg;
+
+	/* Open USB network device */
+	if ( ( rc = usbnet_open ( &smsc75xx->usbnet ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not open: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Configure interrupt endpoint */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_INT_EP_CTL,
+				      ( SMSC75XX_INT_EP_CTL_RDFO_EN |
+					SMSC75XX_INT_EP_CTL_PHY_EN ) ) ) != 0 )
+		goto err_int_ep_ctl;
+
+	/* Configure bulk IN delay */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_BULK_IN_DLY,
+				      SMSC75XX_BULK_IN_DLY_SET ( 0 ) ) ) != 0 )
+		goto err_bulk_in_dly;
+
+	/* Configure receive filters */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_RFE_CTL,
+				      ( SMSC75XX_RFE_CTL_AB |
+					SMSC75XX_RFE_CTL_AM |
+					SMSC75XX_RFE_CTL_AU ) ) ) != 0 )
+		goto err_rfe_ctl;
+
+	/* Configure receive FIFO */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_FCT_RX_CTL,
+				      ( SMSC75XX_FCT_RX_CTL_EN |
+					SMSC75XX_FCT_RX_CTL_BAD ) ) ) != 0 )
+		goto err_fct_rx_ctl;
+
+	/* Configure transmit FIFO */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_FCT_TX_CTL,
+				      SMSC75XX_FCT_TX_CTL_EN ) ) != 0 )
+		goto err_fct_tx_ctl;
+
+	/* Configure receive datapath */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MAC_RX,
+				      ( SMSC75XX_MAC_RX_MAX_SIZE_DEFAULT |
+					SMSC75XX_MAC_RX_FCS |
+					SMSC75XX_MAC_RX_EN ) ) ) != 0 )
+		goto err_mac_rx;
+
+	/* Configure transmit datapath */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_MAC_TX,
+				      SMSC75XX_MAC_TX_EN ) ) != 0 )
+		goto err_mac_tx;
+
+	/* Write MAC address high register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_RX_ADDRH,
+					  mac.addr.h ) ) != 0 )
+		goto err_rx_addrh;
+
+	/* Write MAC address low register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_RX_ADDRL,
+					  mac.addr.l ) ) != 0 )
+		goto err_rx_addrl;
+
+	/* Write MAC address perfect filter high register */
+	mac.addr.h |= cpu_to_le32 ( SMSC75XX_ADDR_FILTH_VALID );
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_ADDR_FILTH ( 0 ),
+					  mac.addr.h ) ) != 0 )
+		goto err_addr_filth;
+
+	/* Write MAC address perfect filter low register */
+	if ( ( rc = smsc75xx_raw_writel ( smsc75xx, SMSC75XX_ADDR_FILTL ( 0 ),
+					  mac.addr.l ) ) != 0 )
+		goto err_addr_filtl;
+
+	/* Enable PHY interrupts */
+	if ( ( rc = mii_write ( &smsc75xx->mii, SMSC75XX_MII_PHY_INTR_MASK,
+				( SMSC75XX_PHY_INTR_ANEG_DONE |
+				  SMSC75XX_PHY_INTR_LINK_DOWN ) ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not set PHY interrupt "
+		       "mask: %s\n", smsc75xx, strerror ( rc ) );
+		goto err_phy_intr_mask;
+	}
+
+	/* Update link status */
+	smsc75xx_check_link ( smsc75xx );
+
+	return 0;
+
+ err_phy_intr_mask:
+ err_addr_filtl:
+ err_addr_filth:
+ err_rx_addrl:
+ err_rx_addrh:
+ err_mac_tx:
+ err_mac_rx:
+ err_fct_tx_ctl:
+ err_fct_rx_ctl:
+ err_rfe_ctl:
+ err_bulk_in_dly:
+ err_int_ep_ctl:
+	usbnet_close ( &smsc75xx->usbnet );
+ err_open:
+ err_hw_cfg:
+	smsc75xx_reset ( smsc75xx );
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void smsc75xx_close ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+
+	/* Close USB network device */
+	usbnet_close ( &smsc75xx->usbnet );
+
+	/* Dump statistics (for debugging) */
+	smsc75xx_dump_statistics ( smsc75xx );
+
+	/* Reset device */
+	smsc75xx_reset ( smsc75xx );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int smsc75xx_transmit ( struct net_device *netdev,
+			       struct io_buffer *iobuf ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	int rc;
+
+	/* Transmit packet */
+	if ( ( rc = smsc75xx_out_transmit ( smsc75xx, iobuf ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void smsc75xx_poll ( struct net_device *netdev ) {
+	struct smsc75xx_device *smsc75xx = netdev->priv;
+	uint32_t int_sts;
+	int rc;
+
+	/* Poll USB bus */
+	usb_poll ( smsc75xx->bus );
+
+	/* Refill endpoints */
+	if ( ( rc = usbnet_refill ( &smsc75xx->usbnet ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+
+	/* Do nothing more unless there are interrupts to handle */
+	int_sts = smsc75xx->int_sts;
+	if ( ! int_sts )
+		return;
+
+	/* Check link status if applicable */
+	if ( int_sts & SMSC75XX_INT_STS_PHY_INT ) {
+		smsc75xx_check_link ( smsc75xx );
+		int_sts &= ~SMSC75XX_INT_STS_PHY_INT;
+	}
+
+	/* Record RX FIFO overflow if applicable */
+	if ( int_sts & SMSC75XX_INT_STS_RDFO_INT ) {
+		DBGC2 ( smsc75xx, "SMSC75XX %p RX FIFO overflowed\n",
+			smsc75xx );
+		netdev_rx_err ( netdev, NULL, -ENOBUFS );
+		int_sts &= ~SMSC75XX_INT_STS_RDFO_INT;
+	}
+
+	/* Check for unexpected interrupts */
+	if ( int_sts ) {
+		DBGC ( smsc75xx, "SMSC75XX %p unexpected interrupt %#08x\n",
+		       smsc75xx, int_sts );
+		netdev_rx_err ( netdev, NULL, -ENOTTY );
+	}
+
+	/* Clear interrupts */
+	if ( ( rc = smsc75xx_writel ( smsc75xx, SMSC75XX_INT_STS,
+				      smsc75xx->int_sts ) ) != 0 )
+		netdev_rx_err ( netdev, NULL, rc );
+	smsc75xx->int_sts = 0;
+}
+
+/** SMSC75xx network device operations */
+static struct net_device_operations smsc75xx_operations = {
+	.open		= smsc75xx_open,
+	.close		= smsc75xx_close,
+	.transmit	= smsc75xx_transmit,
+	.poll		= smsc75xx_poll,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int smsc75xx_probe ( struct usb_function *func,
+			    struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct net_device *netdev;
+	struct smsc75xx_device *smsc75xx;
+	int rc;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *smsc75xx ) );
+	if ( ! netdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	netdev_init ( netdev, &smsc75xx_operations );
+	netdev->dev = &func->dev;
+	smsc75xx = netdev->priv;
+	memset ( smsc75xx, 0, sizeof ( *smsc75xx ) );
+	smsc75xx->usb = usb;
+	smsc75xx->bus = usb->port->hub->bus;
+	smsc75xx->netdev = netdev;
+	usbnet_init ( &smsc75xx->usbnet, func, &smsc75xx_intr_operations,
+		      &smsc75xx_in_operations, &smsc75xx_out_operations );
+	usb_refill_init ( &smsc75xx->usbnet.intr, 0, SMSC75XX_INTR_MAX_FILL );
+	usb_refill_init ( &smsc75xx->usbnet.in, SMSC75XX_IN_MTU,
+			  SMSC75XX_IN_MAX_FILL );
+	mii_init ( &smsc75xx->mii, &smsc75xx_mii_operations );
+	DBGC ( smsc75xx, "SMSC75XX %p on %s\n", smsc75xx, func->name );
+
+	/* Describe USB network device */
+	if ( ( rc = usbnet_describe ( &smsc75xx->usbnet, config ) ) != 0 ) {
+		DBGC ( smsc75xx, "SMSC75XX %p could not describe: %s\n",
+		       smsc75xx, strerror ( rc ) );
+		goto err_describe;
+	}
+
+	/* Reset device */
+	if ( ( rc = smsc75xx_reset ( smsc75xx ) ) != 0 )
+		goto err_reset;
+
+	/* Read MAC address */
+	if ( ( rc = smsc75xx_eeprom_read ( smsc75xx, SMSC75XX_EEPROM_MAC,
+					   netdev->hw_addr, ETH_ALEN ) ) != 0 )
+		goto err_eeprom_read;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 )
+		goto err_register;
+
+	usb_func_set_drvdata ( func, netdev );
+	return 0;
+
+	unregister_netdev ( netdev );
+ err_register:
+ err_eeprom_read:
+ err_reset:
+ err_describe:
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void smsc75xx_remove ( struct usb_function *func ) {
+	struct net_device *netdev = usb_func_get_drvdata ( func );
+
+	unregister_netdev ( netdev );
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
+
+/** SMSC75xx device IDs */
+static struct usb_device_id smsc75xx_ids[] = {
+	{
+		.name = "smsc7500",
+		.vendor = 0x0424,
+		.product = 0x7500,
+		.class = { 0xff, 0x00, 0xff },
+	},
+	{
+		.name = "smsc7505",
+		.vendor = 0x0424,
+		.product = 0x7505,
+		.class = { 0xff, 0x00, 0xff },
+	},
+};
+
+/** SMSC LAN75xx driver */
+struct usb_driver smsc75xx_driver __usb_driver = {
+	.ids = smsc75xx_ids,
+	.id_count = ( sizeof ( smsc75xx_ids ) / sizeof ( smsc75xx_ids[0] ) ),
+	.probe = smsc75xx_probe,
+	.remove = smsc75xx_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/net/smsc75xx.h b/qemu/roms/ipxe/src/drivers/net/smsc75xx.h
new file mode 100644
index 000000000..2463b72a1
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/net/smsc75xx.h
@@ -0,0 +1,309 @@
+#ifndef _SMSC75XX_H
+#define _SMSC75XX_H
+
+/** @file
+ *
+ * SMSC LAN75xx USB Ethernet driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+#include <ipxe/if_ether.h>
+#include <ipxe/mii.h>
+
+/** Register write command */
+#define SMSC75XX_REGISTER_WRITE					\
+	( USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa0 ) )
+
+/** Register read command */
+#define SMSC75XX_REGISTER_READ					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa1 ) )
+
+/** Get statistics command */
+#define SMSC75XX_GET_STATISTICS					\
+	( USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE |	\
+	  USB_REQUEST_TYPE ( 0xa2 ) )
+
+/** Interrupt status register */
+#define SMSC75XX_INT_STS 0x00c
+#define SMSC75XX_INT_STS_RDFO_INT	0x00400000UL	/**< RX FIFO overflow */
+#define SMSC75XX_INT_STS_PHY_INT	0x00020000UL	/**< PHY interrupt */
+
+/** Hardware configuration register */
+#define SMSC75XX_HW_CFG 0x010
+#define SMSC75XX_HW_CFG_BIR		0x00000080UL	/**< Bulk IN use NAK */
+#define SMSC75XX_HW_CFG_LRST		0x00000002UL	/**< Soft lite reset */
+
+/** Interrupt endpoint control register */
+#define SMSC75XX_INT_EP_CTL 0x038
+#define SMSC75XX_INT_EP_CTL_RDFO_EN	0x00400000UL	/**< RX FIFO overflow */
+#define SMSC75XX_INT_EP_CTL_PHY_EN	0x00020000UL	/**< PHY interrupt */
+
+/** Bulk IN delay register */
+#define SMSC75XX_BULK_IN_DLY 0x03c
+#define SMSC75XX_BULK_IN_DLY_SET(ticks)	( (ticks) << 0 ) /**< Delay / 16.7ns */
+
+/** EEPROM command register */
+#define SMSC75XX_E2P_CMD 0x040
+#define SMSC75XX_E2P_CMD_EPC_BSY	0x80000000UL	/**< EPC busy */
+#define SMSC75XX_E2P_CMD_EPC_CMD_READ	0x00000000UL	/**< READ command */
+#define SMSC75XX_E2P_CMD_EPC_ADDR(addr) ( (addr) << 0 )	/**< EPC address */
+
+/** EEPROM data register */
+#define SMSC75XX_E2P_DATA 0x044
+#define SMSC75XX_E2P_DATA_GET(e2p_data) \
+	( ( (e2p_data) >> 0 ) & 0xff )			/**< EEPROM data */
+
+/** MAC address EEPROM address */
+#define SMSC75XX_EEPROM_MAC 0x01
+
+/** Receive filtering engine control register */
+#define SMSC75XX_RFE_CTL 0x060
+#define SMSC75XX_RFE_CTL_AB		0x00000400UL	/**< Accept broadcast */
+#define SMSC75XX_RFE_CTL_AM		0x00000200UL	/**< Accept multicast */
+#define SMSC75XX_RFE_CTL_AU		0x00000100UL	/**< Accept unicast */
+
+/** FIFO controller RX FIFO control register */
+#define SMSC75XX_FCT_RX_CTL 0x090
+#define SMSC75XX_FCT_RX_CTL_EN		0x80000000UL	/**< FCT RX enable */
+#define SMSC75XX_FCT_RX_CTL_BAD		0x02000000UL	/**< Store bad frames */
+
+/** FIFO controller TX FIFO control register */
+#define SMSC75XX_FCT_TX_CTL 0x094
+#define SMSC75XX_FCT_TX_CTL_EN		0x80000000UL	/**< FCT TX enable */
+
+/** MAC receive register */
+#define SMSC75XX_MAC_RX 0x104
+#define SMSC75XX_MAC_RX_MAX_SIZE(mtu)	( (mtu) << 16 )	/**< Max frame size */
+#define SMSC75XX_MAC_RX_MAX_SIZE_DEFAULT \
+	SMSC75XX_MAC_RX_MAX_SIZE ( ETH_FRAME_LEN + 4 /* VLAN */ + 4 /* CRC */ )
+#define SMSC75XX_MAC_RX_FCS		0x00000010UL	/**< FCS stripping */
+#define SMSC75XX_MAC_RX_EN		0x00000001UL	/**< RX enable */
+
+/** MAC transmit register */
+#define SMSC75XX_MAC_TX 0x108
+#define SMSC75XX_MAC_TX_EN		0x00000001UL	/**< TX enable */
+
+/** MAC receive address high register */
+#define SMSC75XX_RX_ADDRH 0x118
+
+/** MAC receive address low register */
+#define SMSC75XX_RX_ADDRL 0x11c
+
+/** MII access register */
+#define SMSC75XX_MII_ACCESS 0x120
+#define SMSC75XX_MII_ACCESS_PHY_ADDRESS	0x00000800UL	/**< PHY address */
+#define SMSC75XX_MII_ACCESS_MIIRINDA(addr) ( (addr) << 6 ) /**< MII register */
+#define SMSC75XX_MII_ACCESS_MIIWNR	0x00000002UL	/**< MII write */
+#define SMSC75XX_MII_ACCESS_MIIBZY	0x00000001UL	/**< MII busy */
+
+/** MII data register */
+#define SMSC75XX_MII_DATA 0x124
+#define SMSC75XX_MII_DATA_SET(data)	( (data) << 0 )	/**< Set data */
+#define SMSC75XX_MII_DATA_GET(mii_data) \
+	( ( (mii_data) >> 0 ) & 0xffff )		/**< Get data */
+
+/** PHY interrupt source MII register */
+#define SMSC75XX_MII_PHY_INTR_SOURCE 29
+
+/** PHY interrupt mask MII register */
+#define SMSC75XX_MII_PHY_INTR_MASK 30
+
+/** PHY interrupt: auto-negotiation complete */
+#define SMSC75XX_PHY_INTR_ANEG_DONE	0x0040
+
+/** PHY interrupt: link down */
+#define SMSC75XX_PHY_INTR_LINK_DOWN	0x0010
+
+/** MAC address perfect filter N high register */
+#define SMSC75XX_ADDR_FILTH(n) ( 0x300 + ( 8 * (n) ) )
+#define SMSC75XX_ADDR_FILTH_VALID	0x80000000UL	/**< Address valid */
+
+/** MAC address perfect filter N low register */
+#define SMSC75XX_ADDR_FILTL(n) ( 0x304 + ( 8 * (n) ) )
+
+/** MAC address */
+union smsc75xx_mac {
+	/** MAC receive address registers */
+	struct {
+		/** MAC receive address low register */
+		uint32_t l;
+		/** MAC receive address high register */
+		uint32_t h;
+	} __attribute__ (( packed )) addr;
+	/** Raw MAC address */
+	uint8_t raw[ETH_ALEN];
+};
+
+/** Receive packet header */
+struct smsc75xx_rx_header {
+	/** RX command word */
+	uint32_t command;
+	/** VLAN tag */
+	uint16_t vtag;
+	/** Checksum */
+	uint16_t csum;
+	/** Two-byte padding used to align Ethernet payload */
+	uint16_t pad;
+} __attribute__ (( packed ));
+
+/** Receive error detected */
+#define SMSC75XX_RX_RED 0x00400000UL
+
+/** Transmit packet header */
+struct smsc75xx_tx_header {
+	/** TX command word */
+	uint32_t command;
+	/** VLAN tag */
+	uint16_t tag;
+	/** Maximum segment size */
+	uint16_t mss;
+} __attribute__ (( packed ));
+
+/** Insert frame checksum and pad */
+#define SMSC75XX_TX_FCS 0x00400000UL
+
+/** Interrupt packet format */
+struct smsc75xx_interrupt {
+	/** Current value of INT_STS register */
+	uint32_t int_sts;
+} __attribute__ (( packed ));
+
+/** Byte count statistics */
+struct smsc75xx_byte_statistics {
+	/** Unicast byte count */
+	uint32_t unicast;
+	/** Broadcast byte count */
+	uint32_t broadcast;
+	/** Multicast byte count */
+	uint32_t multicast;
+} __attribute__ (( packed ));
+
+/** Frame count statistics */
+struct smsc75xx_frame_statistics {
+	/** Unicast frames */
+	uint32_t unicast;
+	/** Broadcast frames */
+	uint32_t broadcast;
+	/** Multicast frames */
+	uint32_t multicast;
+	/** Pause frames */
+	uint32_t pause;
+	/** Frames by length category */
+	uint32_t len[7];
+} __attribute__ (( packed ));
+
+/** Receive error statistics */
+struct smsc75xx_rx_error_statistics {
+	/** FCS errors */
+	uint32_t fcs;
+	/** Alignment errors */
+	uint32_t alignment;
+	/** Fragment errors */
+	uint32_t fragment;
+	/** Jabber errors */
+	uint32_t jabber;
+	/** Undersize frame errors */
+	uint32_t undersize;
+	/** Oversize frame errors */
+	uint32_t oversize;
+	/** Dropped frame errors */
+	uint32_t dropped;
+} __attribute__ (( packed ));
+
+/** Receive statistics */
+struct smsc75xx_rx_statistics {
+	/** Error statistics */
+	struct smsc75xx_rx_error_statistics err;
+	/** Byte count statistics */
+	struct smsc75xx_byte_statistics byte;
+	/** Frame count statistics */
+	struct smsc75xx_frame_statistics frame;
+} __attribute__ (( packed ));
+
+/** Transmit error statistics */
+struct smsc75xx_tx_error_statistics {
+	/** FCS errors */
+	uint32_t fcs;
+	/** Excess deferral errors */
+	uint32_t deferral;
+	/** Carrier errors */
+	uint32_t carrier;
+	/** Bad byte count */
+	uint32_t count;
+	/** Single collisions */
+	uint32_t single;
+	/** Multiple collisions */
+	uint32_t multiple;
+	/** Excession collisions */
+	uint32_t excessive;
+	/** Late collisions */
+	uint32_t late;
+} __attribute__ (( packed ));
+
+/** Transmit statistics */
+struct smsc75xx_tx_statistics {
+	/** Error statistics */
+	struct smsc75xx_tx_error_statistics err;
+	/** Byte count statistics */
+	struct smsc75xx_byte_statistics byte;
+	/** Frame count statistics */
+	struct smsc75xx_frame_statistics frame;
+} __attribute__ (( packed ));
+
+/** Statistics */
+struct smsc75xx_statistics {
+	/** Receive statistics */
+	struct smsc75xx_rx_statistics rx;
+	/** Transmit statistics */
+	struct smsc75xx_tx_statistics tx;
+} __attribute__ (( packed ));
+
+/** A SMSC75xx network device */
+struct smsc75xx_device {
+	/** USB device */
+	struct usb_device *usb;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Network device */
+	struct net_device *netdev;
+	/** USB network device */
+	struct usbnet_device usbnet;
+	/** MII interface */
+	struct mii_interface mii;
+	/** Interrupt status */
+	uint32_t int_sts;
+};
+
+/** Reset delay (in microseconds) */
+#define SMSC75XX_RESET_DELAY_US 2
+
+/** Maximum time to wait for EEPROM (in milliseconds) */
+#define SMSC75XX_EEPROM_MAX_WAIT_MS 100
+
+/** Maximum time to wait for MII (in milliseconds) */
+#define SMSC75XX_MII_MAX_WAIT_MS 100
+
+/** Interrupt maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define SMSC75XX_INTR_MAX_FILL 2
+
+/** Bulk IN maximum fill level
+ *
+ * This is a policy decision.
+ */
+#define SMSC75XX_IN_MAX_FILL 8
+
+/** Bulk IN buffer size */
+#define SMSC75XX_IN_MTU						\
+	( sizeof ( struct smsc75xx_rx_header ) +		\
+	  ETH_FRAME_LEN + 4 /* possible VLAN header */ )
+
+#endif /* _SMSC75XX_H */
diff --git a/qemu/roms/ipxe/src/drivers/net/sundance.c b/qemu/roms/ipxe/src/drivers/net/sundance.c
index eef7c9c7c..9127fa2cd 100644
--- a/qemu/roms/ipxe/src/drivers/net/sundance.c
+++ b/qemu/roms/ipxe/src/drivers/net/sundance.c
@@ -601,7 +601,7 @@ static int sundance_probe ( struct nic *nic, struct pci_device *pci ) {
 	sdc->nic_name = pci->id->name;
 	sdc->mtu = mtu;
 
-	pci_read_config_byte(pci, PCI_REVISION_ID, &sdc->pci_rev_id);
+	pci_read_config_byte(pci, PCI_REVISION, &sdc->pci_rev_id);
 
 	DBG ( "Device revision id: %hx\n", sdc->pci_rev_id );
 
diff --git a/qemu/roms/ipxe/src/drivers/net/tg3/tg3.c b/qemu/roms/ipxe/src/drivers/net/tg3/tg3.c
index 32ca1609c..42bfa2d99 100644
--- a/qemu/roms/ipxe/src/drivers/net/tg3/tg3.c
+++ b/qemu/roms/ipxe/src/drivers/net/tg3/tg3.c
@@ -928,6 +928,7 @@ static struct pci_device_id tg3_nics[] = {
 	PCI_ROM(0x14e4, 0x16b6, "14e4-16b6", "14e4-16b6", 0),
 	PCI_ROM(0x14e4, 0x1657, "14e4-1657", "14e4-1657", 0),
 	PCI_ROM(0x14e4, 0x165f, "14e4-165f", "14e4-165f", 0),
+	PCI_ROM(0x14e4, 0x1686, "14e4-1686", "14e4-1686", 0),
 	PCI_ROM(0x1148, 0x4400, "1148-4400", "1148-4400", 0),
 	PCI_ROM(0x1148, 0x4500, "1148-4500", "1148-4500", 0),
 	PCI_ROM(0x173b, 0x03e8, "173b-03e8", "173b-03e8", 0),
diff --git a/qemu/roms/ipxe/src/drivers/net/tg3/tg3.h b/qemu/roms/ipxe/src/drivers/net/tg3/tg3.h
index 660368394..2b85b065b 100644
--- a/qemu/roms/ipxe/src/drivers/net/tg3/tg3.h
+++ b/qemu/roms/ipxe/src/drivers/net/tg3/tg3.h
@@ -131,6 +131,10 @@
 #define PCI_DEVICE_ID_TIGON3_5901_2	0x170e
 #define PCI_DEVICE_ID_TIGON3_5906	0x1712
 #define PCI_DEVICE_ID_TIGON3_5906M	0x1713
+#define PCI_VENDOR_ID_COMPAQ		0x0e11
+#define PCI_VENDOR_ID_IBM		0x1014
+#define PCI_VENDOR_ID_DELL		0x1028
+#define PCI_VENDOR_ID_3COM		0x10b7
 /* </pci_ids.h> */
 
 #define SPEED_10			10
@@ -185,6 +189,7 @@
 #define  TG3PCI_DEVICE_TIGON3_57761	 0x16b0
 #define  TG3PCI_DEVICE_TIGON3_57762	 0x1682
 #define  TG3PCI_DEVICE_TIGON3_57765	 0x16b4
+#define  TG3PCI_DEVICE_TIGON3_57766	 0x1686
 #define  TG3PCI_DEVICE_TIGON3_57791	 0x16b2
 #define  TG3PCI_DEVICE_TIGON3_57795	 0x16b6
 #define  TG3PCI_DEVICE_TIGON3_5719	 0x1657
diff --git a/qemu/roms/ipxe/src/drivers/net/tg3/tg3_hw.c b/qemu/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
index 3a481aba3..50353cf36 100644
--- a/qemu/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
+++ b/qemu/roms/ipxe/src/drivers/net/tg3/tg3_hw.c
@@ -436,6 +436,7 @@ int tg3_get_invariants(struct tg3 *tp)
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57761 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57762 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57765 ||
+			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57766 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57791 ||
 			 tp->pdev->device == TG3PCI_DEVICE_TIGON3_57795)
 			pci_read_config_dword(tp->pdev,
diff --git a/qemu/roms/ipxe/src/drivers/net/virtio-net.c b/qemu/roms/ipxe/src/drivers/net/virtio-net.c
index d5fd81979..533ccb0c6 100644
--- a/qemu/roms/ipxe/src/drivers/net/virtio-net.c
+++ b/qemu/roms/ipxe/src/drivers/net/virtio-net.c
@@ -20,7 +20,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <stdlib.h>
@@ -131,8 +131,8 @@ static void virtnet_enqueue_iob ( struct net_device *netdev,
 		},
 	};
 
-	DBGC ( virtnet, "VIRTIO-NET %p enqueuing iobuf %p on vq %d\n",
-	       virtnet, iobuf, vq_idx );
+	DBGC2 ( virtnet, "VIRTIO-NET %p enqueuing iobuf %p on vq %d\n",
+		virtnet, iobuf, vq_idx );
 
 	vring_add_buf ( vq, list, out, in, iobuf, 0 );
 	vring_kick ( virtnet->ioaddr, vq, 1 );
@@ -256,8 +256,8 @@ static void virtnet_process_tx_packets ( struct net_device *netdev ) {
 	while ( vring_more_used ( tx_vq ) ) {
 		struct io_buffer *iobuf = vring_get_buf ( tx_vq, NULL );
 
-		DBGC ( virtnet, "VIRTIO-NET %p tx complete iobuf %p\n",
-		       virtnet, iobuf );
+		DBGC2 ( virtnet, "VIRTIO-NET %p tx complete iobuf %p\n",
+			virtnet, iobuf );
 
 		netdev_tx_complete ( netdev, iobuf );
 	}
@@ -283,8 +283,8 @@ static void virtnet_process_rx_packets ( struct net_device *netdev ) {
 		iob_unput ( iobuf, RX_BUF_SIZE );
 		iob_put ( iobuf, len - sizeof ( struct virtio_net_hdr ) );
 
-		DBGC ( virtnet, "VIRTIO-NET %p rx complete iobuf %p len %zd\n",
-		       virtnet, iobuf, iob_len ( iobuf ) );
+		DBGC2 ( virtnet, "VIRTIO-NET %p rx complete iobuf %p len %zd\n",
+			virtnet, iobuf, iob_len ( iobuf ) );
 
 		/* Pass completed packet to the network stack */
 		netdev_rx ( netdev, iobuf );
diff --git a/qemu/roms/ipxe/src/drivers/net/vmxnet3.c b/qemu/roms/ipxe/src/drivers/net/vmxnet3.c
index 31082bf6f..8d4f4b843 100644
--- a/qemu/roms/ipxe/src/drivers/net/vmxnet3.c
+++ b/qemu/roms/ipxe/src/drivers/net/vmxnet3.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/drivers/net/vmxnet3.h b/qemu/roms/ipxe/src/drivers/net/vmxnet3.h
index db313d4b8..a1671d9dd 100644
--- a/qemu/roms/ipxe/src/drivers/net/vmxnet3.h
+++ b/qemu/roms/ipxe/src/drivers/net/vmxnet3.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/drivers/net/vxge/vxge.c b/qemu/roms/ipxe/src/drivers/net/vxge/vxge.c
index bf20ec43c..d50ac05b5 100644
--- a/qemu/roms/ipxe/src/drivers/net/vxge/vxge.c
+++ b/qemu/roms/ipxe/src/drivers/net/vxge/vxge.c
@@ -5,10 +5,11 @@
  * as "vxge" even though the code is in vxge_* named files.
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE(GPL2_OR_LATER_OR_UBDL);
 
 #include <ipxe/pci.h>
 
+PROVIDE_REQUIRING_SYMBOL();
 REQUIRE_OBJECT(vxge_main);
 
 /** vxge PCI IDs for util/parserom.pl which are put into bin/NIC */
diff --git a/qemu/roms/ipxe/src/drivers/net/vxge/vxge_main.c b/qemu/roms/ipxe/src/drivers/net/vxge/vxge_main.c
index 130eab617..8b099c0e2 100644
--- a/qemu/roms/ipxe/src/drivers/net/vxge/vxge_main.c
+++ b/qemu/roms/ipxe/src/drivers/net/vxge/vxge_main.c
@@ -509,7 +509,7 @@ vxge_probe(struct pci_device *pdev)
 	vxge_debug(VXGE_INFO, "vxge_probe for device " PCI_FMT "\n",
 			PCI_ARGS(pdev));
 
-	pci_read_config_byte(pdev, PCI_REVISION_ID, &revision);
+	pci_read_config_byte(pdev, PCI_REVISION, &revision);
 	titan1 = is_titan1(pdev->device, revision);
 
 	mmio_start = pci_bar_start(pdev, PCI_BASE_ADDRESS_0);
diff --git a/qemu/roms/ipxe/src/drivers/net/w89c840.c b/qemu/roms/ipxe/src/drivers/net/w89c840.c
index ce638ab99..d8144a8ce 100644
--- a/qemu/roms/ipxe/src/drivers/net/w89c840.c
+++ b/qemu/roms/ipxe/src/drivers/net/w89c840.c
@@ -641,7 +641,9 @@ static int w89c840_probe ( struct nic *nic, struct pci_device *p ) {
 
     ioaddr = ioaddr & ~3; /* Mask the bit that says "this is an io addr" */
 
+#define PCI_VENDOR_ID_WINBOND2		0x1050
 #define PCI_DEVICE_ID_WINBOND2_89C840   0x0840
+#define PCI_VENDOR_ID_COMPEX		0x11f6
 #define PCI_DEVICE_ID_COMPEX_RL100ATX   0x2011
 
     /* From Matt Hortman <mbhortman@acpthinclient.com> */
diff --git a/qemu/roms/ipxe/src/drivers/nvs/nvs.c b/qemu/roms/ipxe/src/drivers/nvs/nvs.c
index ccb2145bd..af7c466c4 100644
--- a/qemu/roms/ipxe/src/drivers/nvs/nvs.c
+++ b/qemu/roms/ipxe/src/drivers/nvs/nvs.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/drivers/nvs/nvsvpd.c b/qemu/roms/ipxe/src/drivers/nvs/nvsvpd.c
index 33148d5b9..3e88531c7 100644
--- a/qemu/roms/ipxe/src/drivers/nvs/nvsvpd.c
+++ b/qemu/roms/ipxe/src/drivers/nvs/nvsvpd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/drivers/nvs/spi.c b/qemu/roms/ipxe/src/drivers/nvs/spi.c
index 84613b9dd..dcfe1af91 100644
--- a/qemu/roms/ipxe/src/drivers/nvs/spi.c
+++ b/qemu/roms/ipxe/src/drivers/nvs/spi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/drivers/nvs/threewire.c b/qemu/roms/ipxe/src/drivers/nvs/threewire.c
index 53f1ad8de..547f35382 100644
--- a/qemu/roms/ipxe/src/drivers/nvs/threewire.c
+++ b/qemu/roms/ipxe/src/drivers/nvs/threewire.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/drivers/usb/ehci.c b/qemu/roms/ipxe/src/drivers/usb/ehci.c
new file mode 100644
index 000000000..4124692a6
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/ehci.c
@@ -0,0 +1,1994 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include <ipxe/init.h>
+#include "ehci.h"
+
+/** @file
+ *
+ * USB Enhanced Host Controller Interface (EHCI) driver
+ *
+ */
+
+/**
+ * Construct error code from transfer descriptor status
+ *
+ * @v status		Transfer descriptor status
+ * @ret rc		Error code
+ *
+ * Bits 2-5 of the status code provide some indication as to the root
+ * cause of the error.  We incorporate these into the error code as
+ * reported to usb_complete_err().
+ */
+#define EIO_STATUS( status ) EUNIQ ( EINFO_EIO, ( ( (status) >> 2 ) & 0xf ) )
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise device
+ *
+ * @v ehci		EHCI device
+ * @v regs		MMIO registers
+ */
+static void ehci_init ( struct ehci_device *ehci, void *regs ) {
+	uint32_t hcsparams;
+	uint32_t hccparams;
+	size_t caplength;
+
+	/* Locate capability and operational registers */
+	ehci->cap = regs;
+	caplength = readb ( ehci->cap + EHCI_CAP_CAPLENGTH );
+	ehci->op = ( ehci->cap + caplength );
+	DBGC2 ( ehci, "EHCI %s cap %08lx op %08lx\n", ehci->name,
+		virt_to_phys ( ehci->cap ), virt_to_phys ( ehci->op ) );
+
+	/* Read structural parameters */
+	hcsparams = readl ( ehci->cap + EHCI_CAP_HCSPARAMS );
+	ehci->ports = EHCI_HCSPARAMS_PORTS ( hcsparams );
+	DBGC ( ehci, "EHCI %s has %d ports\n", ehci->name, ehci->ports );
+
+	/* Read capability parameters 1 */
+	hccparams = readl ( ehci->cap + EHCI_CAP_HCCPARAMS );
+	ehci->addr64 = EHCI_HCCPARAMS_ADDR64 ( hccparams );
+	ehci->flsize = ( EHCI_HCCPARAMS_FLSIZE ( hccparams ) ?
+			 EHCI_FLSIZE_SMALL : EHCI_FLSIZE_DEFAULT );
+	ehci->eecp = EHCI_HCCPARAMS_EECP ( hccparams );
+	DBGC2 ( ehci, "EHCI %s %d-bit flsize %d\n", ehci->name,
+		( ehci->addr64 ? 64 : 32 ), ehci->flsize );
+}
+
+/**
+ * Find extended capability
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ * @v id		Capability ID
+ * @v offset		Offset to previous extended capability instance, or zero
+ * @ret offset		Offset to extended capability, or zero if not found
+ */
+static unsigned int ehci_extended_capability ( struct ehci_device *ehci,
+					       struct pci_device *pci,
+					       unsigned int id,
+					       unsigned int offset ) {
+	uint32_t eecp;
+
+	/* Locate the extended capability */
+	while ( 1 ) {
+
+		/* Locate first or next capability as applicable */
+		if ( offset ) {
+			pci_read_config_dword ( pci, offset, &eecp );
+			offset = EHCI_EECP_NEXT ( eecp );
+		} else {
+			offset = ehci->eecp;
+		}
+		if ( ! offset )
+			return 0;
+
+		/* Check if this is the requested capability */
+		pci_read_config_dword ( pci, offset, &eecp );
+		if ( EHCI_EECP_ID ( eecp ) == id )
+			return offset;
+	}
+}
+
+/**
+ * Calculate buffer alignment
+ *
+ * @v len		Length
+ * @ret align		Buffer alignment
+ *
+ * Determine alignment required for a buffer which must be aligned to
+ * at least EHCI_MIN_ALIGN and which must not cross a page boundary.
+ */
+static inline size_t ehci_align ( size_t len ) {
+	size_t align;
+
+	/* Align to own length (rounded up to a power of two) */
+	align = ( 1 << fls ( len - 1 ) );
+
+	/* Round up to EHCI_MIN_ALIGN if needed */
+	if ( align < EHCI_MIN_ALIGN )
+		align = EHCI_MIN_ALIGN;
+
+	return align;
+}
+
+/**
+ * Check control data structure reachability
+ *
+ * @v ehci		EHCI device
+ * @v ptr		Data structure pointer
+ * @ret rc		Return status code
+ */
+static int ehci_ctrl_reachable ( struct ehci_device *ehci, void *ptr ) {
+	physaddr_t phys = virt_to_phys ( ptr );
+	uint32_t segment;
+
+	/* Always reachable in a 32-bit build */
+	if ( sizeof ( physaddr_t ) <= sizeof ( uint32_t ) )
+		return 0;
+
+	/* Reachable only if control segment matches in a 64-bit build */
+	segment = ( ( ( uint64_t ) phys ) >> 32 );
+	if ( segment == ehci->ctrldssegment )
+		return 0;
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * USB legacy support
+ *
+ ******************************************************************************
+ */
+
+/** Prevent the release of ownership back to BIOS */
+static int ehci_legacy_prevent_release;
+
+/**
+ * Initialise USB legacy support
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_init ( struct ehci_device *ehci,
+			       struct pci_device *pci ) {
+	unsigned int legacy;
+	uint8_t bios;
+
+	/* Locate USB legacy support capability (if present) */
+	legacy = ehci_extended_capability ( ehci, pci, EHCI_EECP_ID_LEGACY, 0 );
+	if ( ! legacy ) {
+		/* Not an error; capability may not be present */
+		DBGC ( ehci, "EHCI %s has no USB legacy support capability\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Check if legacy USB support is enabled */
+	pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ), &bios );
+	if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
+		/* Not an error; already owned by OS */
+		DBGC ( ehci, "EHCI %s USB legacy support already disabled\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Record presence of USB legacy support capability */
+	ehci->legacy = legacy;
+}
+
+/**
+ * Claim ownership from BIOS
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_claim ( struct ehci_device *ehci,
+				struct pci_device *pci ) {
+	unsigned int legacy = ehci->legacy;
+	uint32_t ctlsts;
+	uint8_t bios;
+	unsigned int i;
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! legacy )
+		return;
+
+	/* Claim ownership */
+	pci_write_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_OS ),
+				EHCI_USBLEGSUP_OS_OWNED );
+
+	/* Wait for BIOS to release ownership */
+	for ( i = 0 ; i < EHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if BIOS has released ownership */
+		pci_read_config_byte ( pci, ( legacy + EHCI_USBLEGSUP_BIOS ),
+				       &bios );
+		if ( ! ( bios & EHCI_USBLEGSUP_BIOS_OWNED ) ) {
+			DBGC ( ehci, "EHCI %s claimed ownership from BIOS\n",
+			       ehci->name );
+			pci_read_config_dword ( pci, ( legacy +
+						       EHCI_USBLEGSUP_CTLSTS ),
+						&ctlsts );
+			if ( ctlsts ) {
+				DBGC ( ehci, "EHCI %s warning: BIOS retained "
+				       "SMIs: %08x\n", ehci->name, ctlsts );
+			}
+			return;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* BIOS did not release ownership.  Claim it forcibly by
+	 * disabling all SMIs.
+	 */
+	DBGC ( ehci, "EHCI %s could not claim ownership from BIOS: forcibly "
+	       "disabling SMIs\n", ehci->name );
+	pci_write_config_dword ( pci, ( legacy + EHCI_USBLEGSUP_CTLSTS ), 0 );
+}
+
+/**
+ * Release ownership back to BIOS
+ *
+ * @v ehci		EHCI device
+ * @v pci		PCI device
+ */
+static void ehci_legacy_release ( struct ehci_device *ehci,
+				  struct pci_device *pci ) {
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! ehci->legacy )
+		return;
+
+	/* Do nothing if releasing ownership is prevented */
+	if ( ehci_legacy_prevent_release ) {
+		DBGC ( ehci, "EHCI %s not releasing ownership to BIOS\n",
+		       ehci->name );
+		return;
+	}
+
+	/* Release ownership */
+	pci_write_config_byte ( pci, ( ehci->legacy + EHCI_USBLEGSUP_OS ), 0 );
+	DBGC ( ehci, "EHCI %s released ownership to BIOS\n", ehci->name );
+}
+
+/******************************************************************************
+ *
+ * Companion controllers
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Poll child companion controllers
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_poll_companions ( struct ehci_device *ehci ) {
+	struct usb_bus *bus;
+	struct device_description *desc;
+
+	/* Poll any USB buses belonging to child companion controllers */
+	for_each_usb_bus ( bus ) {
+
+		/* Get underlying devices description */
+		desc = &bus->dev->desc;
+
+		/* Skip buses that are not PCI devices */
+		if ( desc->bus_type != BUS_TYPE_PCI )
+			continue;
+
+		/* Skip buses that are not part of the same PCI device */
+		if ( PCI_FIRST_FUNC ( desc->location ) !=
+		     PCI_FIRST_FUNC ( ehci->bus->dev->desc.location ) )
+			continue;
+
+		/* Skip buses that are not UHCI or OHCI PCI devices */
+		if ( ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
+						  PCI_CLASS_SERIAL_USB,
+						  PCI_CLASS_SERIAL_USB_UHCI ))&&
+		     ( desc->class != PCI_CLASS ( PCI_CLASS_SERIAL,
+						  PCI_CLASS_SERIAL_USB,
+						  PCI_CLASS_SERIAL_USB_OHCI ) ))
+			continue;
+
+		/* Poll child companion controller bus */
+		DBGC2 ( ehci, "EHCI %s polling companion %s\n",
+			ehci->name, bus->name );
+		usb_poll ( bus );
+	}
+}
+
+/**
+ * Locate EHCI companion controller
+ *
+ * @v pci		PCI device
+ * @ret busdevfn	EHCI companion controller bus:dev.fn (if any)
+ */
+unsigned int ehci_companion ( struct pci_device *pci ) {
+	struct pci_device tmp;
+	unsigned int busdevfn;
+	int rc;
+
+	/* Look for an EHCI function on the same PCI device */
+	busdevfn = pci->busdevfn;
+	while ( ++busdevfn <= PCI_LAST_FUNC ( pci->busdevfn ) ) {
+		pci_init ( &tmp, busdevfn );
+		if ( ( rc = pci_read_config ( &tmp ) ) != 0 )
+			continue;
+		if ( tmp.class == PCI_CLASS ( PCI_CLASS_SERIAL,
+					      PCI_CLASS_SERIAL_USB,
+					      PCI_CLASS_SERIAL_USB_EHCI ) )
+			return busdevfn;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start EHCI device
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_run ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+
+	/* Set run/stop bit */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd &= ~EHCI_USBCMD_FLSIZE_MASK;
+	usbcmd |= ( EHCI_USBCMD_RUN | EHCI_USBCMD_FLSIZE ( ehci->flsize ) |
+		    EHCI_USBCMD_PERIODIC | EHCI_USBCMD_ASYNC );
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+}
+
+/**
+ * Stop EHCI device
+ *
+ * @v ehci		EHCI device
+ * @ret rc		Return status code
+ */
+static int ehci_stop ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd &= ~( EHCI_USBCMD_RUN | EHCI_USBCMD_PERIODIC |
+		     EHCI_USBCMD_ASYNC );
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < EHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+		if ( usbsts & EHCI_USBSTS_HCH )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s timed out waiting for stop\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset EHCI device
+ *
+ * @v ehci		EHCI device
+ * @ret rc		Return status code
+ */
+static int ehci_reset ( struct ehci_device *ehci ) {
+	uint32_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The EHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = ehci_stop ( ehci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	writel ( EHCI_USBCMD_HCRST, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < EHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+		if ( ! ( usbcmd & EHCI_USBCMD_HCRST ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s timed out waiting for reset\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer descriptor rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer descriptor ring
+ *
+ * @v ehci		EHCI device
+ * @v ring		Transfer descriptor ring
+ * @ret rc		Return status code
+ */
+static int ehci_ring_alloc ( struct ehci_device *ehci,
+			     struct ehci_ring *ring ) {
+	struct ehci_transfer_descriptor *desc;
+	struct ehci_transfer_descriptor *next;
+	unsigned int i;
+	size_t len;
+	uint32_t link;
+	int rc;
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+
+	/* Allocate I/O buffers */
+	ring->iobuf = zalloc ( EHCI_RING_COUNT * sizeof ( ring->iobuf[0] ) );
+	if ( ! ring->iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc_iobuf;
+	}
+
+	/* Allocate queue head */
+	ring->head = malloc_dma ( sizeof ( *ring->head ),
+				  ehci_align ( sizeof ( *ring->head ) ) );
+	if ( ! ring->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_queue;
+	}
+	if ( ( rc = ehci_ctrl_reachable ( ehci, ring->head ) ) != 0 ) {
+		DBGC ( ehci, "EHCI %s queue head unreachable\n", ehci->name );
+		goto err_unreachable_queue;
+	}
+	memset ( ring->head, 0, sizeof ( *ring->head ) );
+
+	/* Allocate transfer descriptors */
+	len = ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) );
+	ring->desc = malloc_dma ( len, sizeof ( ring->desc[0] ) );
+	if ( ! ring->desc ) {
+		rc = -ENOMEM;
+		goto err_alloc_desc;
+	}
+	memset ( ring->desc, 0, len );
+
+	/* Initialise transfer descriptors */
+	for ( i = 0 ; i < EHCI_RING_COUNT ; i++ ) {
+		desc = &ring->desc[i];
+		if ( ( rc = ehci_ctrl_reachable ( ehci, desc ) ) != 0 ) {
+			DBGC ( ehci, "EHCI %s descriptor unreachable\n",
+			       ehci->name );
+			goto err_unreachable_desc;
+		}
+		next = &ring->desc[ ( i + 1 ) % EHCI_RING_COUNT ];
+		link = virt_to_phys ( next );
+		desc->next = cpu_to_le32 ( link );
+		desc->alt = cpu_to_le32 ( link );
+	}
+
+	/* Initialise queue head */
+	link = virt_to_phys ( &ring->desc[0] );
+	ring->head->cache.next = cpu_to_le32 ( link );
+
+	return 0;
+
+ err_unreachable_desc:
+	free_dma ( ring->desc, len );
+ err_alloc_desc:
+ err_unreachable_queue:
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+ err_alloc_queue:
+	free ( ring->iobuf );
+ err_alloc_iobuf:
+	return rc;
+}
+
+/**
+ * Free transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ */
+static void ehci_ring_free ( struct ehci_ring *ring ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ehci_ring_fill ( ring ) == 0 );
+	for ( i = 0 ; i < EHCI_RING_COUNT ; i++ )
+		assert ( ring->iobuf[i] == NULL );
+
+	/* Free transfer descriptors */
+	free_dma ( ring->desc, ( EHCI_RING_COUNT * sizeof ( ring->desc[0] ) ) );
+
+	/* Free queue head */
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+
+	/* Free I/O buffers */
+	free ( ring->iobuf );
+}
+
+/**
+ * Enqueue transfer descriptors
+ *
+ * @v ehci		EHCI device
+ * @v ring		Transfer descriptor ring
+ * @v iobuf		I/O buffer
+ * @v xfers		Transfers
+ * @v count		Number of transfers
+ * @ret rc		Return status code
+ */
+static int ehci_enqueue ( struct ehci_device *ehci, struct ehci_ring *ring,
+			  struct io_buffer *iobuf,
+			  const struct ehci_transfer *xfer,
+			  unsigned int count ) {
+	struct ehci_transfer_descriptor *desc;
+	physaddr_t phys;
+	void *data;
+	size_t len;
+	size_t offset;
+	size_t frag_len;
+	unsigned int toggle;
+	unsigned int index;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( iobuf != NULL );
+	assert ( count > 0 );
+
+	/* Fail if ring does not have sufficient space */
+	if ( ehci_ring_remaining ( ring ) < count )
+		return -ENOBUFS;
+
+	/* Fail if any portion is unreachable */
+	for ( i = 0 ; i < count ; i++ ) {
+		phys = ( virt_to_phys ( xfer[i].data ) + xfer[i].len - 1 );
+		if ( ( phys > 0xffffffffUL ) && ( ! ehci->addr64 ) )
+			return -ENOTSUP;
+	}
+
+	/* Enqueue each transfer, recording the I/O buffer with the last */
+	for ( ; count ; ring->prod++, xfer++ ) {
+
+		/* Populate descriptor header */
+		index = ( ring->prod % EHCI_RING_COUNT );
+		desc = &ring->desc[index];
+		toggle = ( xfer->flags & EHCI_FL_TOGGLE );
+		assert ( xfer->len <= EHCI_LEN_MASK );
+		assert ( EHCI_FL_TOGGLE == EHCI_LEN_TOGGLE );
+		desc->len = cpu_to_le16 ( xfer->len | toggle );
+		desc->flags = ( xfer->flags | EHCI_FL_CERR_MAX );
+
+		/* Populate buffer pointers */
+		data = xfer->data;
+		len = xfer->len;
+		for ( i = 0 ; len ; i++ ) {
+
+			/* Calculate length of this fragment */
+			phys = virt_to_phys ( data );
+			offset = ( phys & ( EHCI_PAGE_ALIGN - 1 ) );
+			frag_len = ( EHCI_PAGE_ALIGN - offset );
+			if ( frag_len > len )
+				frag_len = len;
+
+			/* Sanity checks */
+			assert ( ( i == 0 ) || ( offset == 0 ) );
+			assert ( i < ( sizeof ( desc->low ) /
+				       sizeof ( desc->low[0] ) ) );
+
+			/* Populate buffer pointer */
+			desc->low[i] = cpu_to_le32 ( phys );
+			if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) {
+				desc->high[i] =
+					cpu_to_le32 ( ((uint64_t) phys) >> 32 );
+			}
+
+			/* Move to next fragment */
+			data += frag_len;
+			len -= frag_len;
+		}
+
+		/* Ensure everything is valid before activating descriptor */
+		wmb();
+		desc->status = EHCI_STATUS_ACTIVE;
+
+		/* Record I/O buffer against last ring index */
+		if ( --count == 0 )
+			ring->iobuf[index] = iobuf;
+	}
+
+	return 0;
+}
+
+/**
+ * Dequeue a transfer descriptor
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret iobuf		I/O buffer (or NULL)
+ */
+static struct io_buffer * ehci_dequeue ( struct ehci_ring *ring ) {
+	struct ehci_transfer_descriptor *desc;
+	struct io_buffer *iobuf;
+	unsigned int index = ( ring->cons % EHCI_RING_COUNT );
+
+	/* Sanity check */
+	assert ( ehci_ring_fill ( ring ) > 0 );
+
+	/* Mark descriptor as inactive (and not halted) */
+	desc = &ring->desc[index];
+	desc->status = 0;
+
+	/* Retrieve I/O buffer */
+	iobuf = ring->iobuf[index];
+	ring->iobuf[index] = NULL;
+
+	/* Update consumer counter */
+	ring->cons++;
+
+	return iobuf;
+}
+
+/******************************************************************************
+ *
+ * Schedule management
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get link value for a queue head
+ *
+ * @v queue		Queue head
+ * @ret link		Link value
+ */
+static inline uint32_t ehci_link_qh ( struct ehci_queue_head *queue ) {
+
+	return ( virt_to_phys ( queue ) | EHCI_LINK_TYPE_QH );
+}
+
+/**
+ * (Re)build asynchronous schedule
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_async_schedule ( struct ehci_device *ehci ) {
+	struct ehci_endpoint *endpoint;
+	struct ehci_queue_head *queue;
+	uint32_t link;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	link = ehci_link_qh ( ehci->head );
+	list_for_each_entry_reverse ( endpoint, &ehci->async, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		link = ehci_link_qh ( queue );
+	}
+	ehci->head->link = cpu_to_le32 ( link );
+	wmb();
+}
+
+/**
+ * Add endpoint to asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_async_add ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+
+	/* Add to end of schedule */
+	list_add_tail ( &endpoint->schedule, &ehci->async );
+
+	/* Rebuild schedule */
+	ehci_async_schedule ( ehci );
+}
+
+/**
+ * Remove endpoint from asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_async_del ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &ehci->async, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	ehci_async_schedule ( ehci );
+
+	/* Request notification when asynchronous schedule advances */
+	usbcmd = readl ( ehci->op + EHCI_OP_USBCMD );
+	usbcmd |= EHCI_USBCMD_ASYNC_ADVANCE;
+	writel ( usbcmd, ehci->op + EHCI_OP_USBCMD );
+
+	/* Wait for asynchronous schedule to advance */
+	for ( i = 0 ; i < EHCI_ASYNC_ADVANCE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for asynchronous schedule advancing */
+		usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+		if ( usbsts & EHCI_USBSTS_ASYNC_ADVANCE ) {
+			usbsts &= ~EHCI_USBSTS_CHANGE;
+			usbsts |= EHCI_USBSTS_ASYNC_ADVANCE;
+			writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* Bad things will probably happen now */
+	DBGC ( ehci, "EHCI %s timed out waiting for asynchronous schedule "
+	       "to advance\n", ehci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * (Re)build periodic schedule
+ *
+ * @v ehci		EHCI device
+ */
+static void ehci_periodic_schedule ( struct ehci_device *ehci ) {
+	struct ehci_endpoint *endpoint;
+	struct ehci_queue_head *queue;
+	uint32_t link;
+	unsigned int frames;
+	unsigned int max_interval;
+	unsigned int i;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	DBGCP ( ehci, "EHCI %s periodic schedule: ", ehci->name );
+	link = EHCI_LINK_TERMINATE;
+	list_for_each_entry_reverse ( endpoint, &ehci->periodic, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		DBGCP ( ehci, "%s%d",
+			( ( link == EHCI_LINK_TERMINATE ) ? "" : "<-" ),
+			endpoint->ep->interval );
+		link = ehci_link_qh ( queue );
+	}
+	DBGCP ( ehci, "\n" );
+
+	/* Populate periodic frame list */
+	DBGCP ( ehci, "EHCI %s periodic frame list:", ehci->name );
+	frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+	for ( i = 0 ; i < frames ; i++ ) {
+
+		/* Calculate maximum interval (in microframes) which
+		 * may appear as part of this frame list.
+		 */
+		if ( i == 0 ) {
+			/* Start of list: include all endpoints */
+			max_interval = -1U;
+		} else {
+			/* Calculate highest power-of-two frame interval */
+			max_interval = ( 1 << ( ffs ( i ) - 1 ) );
+			/* Convert to microframes */
+			max_interval <<= 3;
+			/* Round up to nearest 2^n-1 */
+			max_interval = ( ( max_interval << 1 ) - 1 );
+		}
+
+		/* Find first endpoint in schedule satisfying this
+		 * maximum interval constraint.
+		 */
+		link = EHCI_LINK_TERMINATE;
+		list_for_each_entry ( endpoint, &ehci->periodic, schedule ) {
+			if ( endpoint->ep->interval <= max_interval ) {
+				queue = endpoint->ring.head;
+				link = ehci_link_qh ( queue );
+				DBGCP ( ehci, " %d:%d",
+					i, endpoint->ep->interval );
+				break;
+			}
+		}
+		ehci->frame[i].link = cpu_to_le32 ( link );
+	}
+	wmb();
+	DBGCP ( ehci, "\n" );
+}
+
+/**
+ * Add endpoint to periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_periodic_add ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	struct ehci_endpoint *before;
+	unsigned int interval = endpoint->ep->interval;
+
+	/* Find first endpoint with a smaller interval */
+	list_for_each_entry ( before, &ehci->periodic, schedule ) {
+		if ( before->ep->interval < interval )
+			break;
+	}
+	list_add_tail ( &endpoint->schedule, &before->schedule );
+
+	/* Rebuild schedule */
+	ehci_periodic_schedule ( ehci );
+}
+
+/**
+ * Remove endpoint from periodic schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_periodic_del ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &ehci->periodic, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	ehci_periodic_schedule ( ehci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+
+	return 0;
+}
+
+/**
+ * Add endpoint to appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_schedule_add ( struct ehci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		ehci_periodic_add ( endpoint );
+	} else {
+		ehci_async_add ( endpoint );
+	}
+}
+
+/**
+ * Remove endpoint from appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_schedule_del ( struct ehci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		return ehci_periodic_del ( endpoint );
+	} else {
+		return ehci_async_del ( endpoint );
+	}
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Determine endpoint characteristics
+ *
+ * @v ep		USB endpoint
+ * @ret chr		Endpoint characteristics
+ */
+static uint32_t ehci_endpoint_characteristics ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	uint32_t chr;
+
+	/* Determine basic characteristics */
+	chr = ( EHCI_CHR_ADDRESS ( usb->address ) |
+		EHCI_CHR_ENDPOINT ( ep->address ) |
+		EHCI_CHR_MAX_LEN ( ep->mtu ) );
+
+	/* Control endpoints require manual control of the data toggle */
+	if ( attr == USB_ENDPOINT_ATTR_CONTROL )
+		chr |= EHCI_CHR_TOGGLE;
+
+	/* Determine endpoint speed */
+	if ( usb->port->speed == USB_SPEED_HIGH ) {
+		chr |= EHCI_CHR_EPS_HIGH;
+	} else {
+		if ( usb->port->speed == USB_SPEED_FULL ) {
+			chr |= EHCI_CHR_EPS_FULL;
+		} else {
+			chr |= EHCI_CHR_EPS_LOW;
+		}
+		if ( attr == USB_ENDPOINT_ATTR_CONTROL )
+			chr |= EHCI_CHR_CONTROL;
+	}
+
+	return chr;
+}
+
+/**
+ * Determine endpoint capabilities
+ *
+ * @v ep		USB endpoint
+ * @ret cap		Endpoint capabilities
+ */
+static uint32_t ehci_endpoint_capabilities ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct usb_port *tt = usb_transaction_translator ( usb );
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	uint32_t cap;
+	unsigned int i;
+
+	/* Determine basic capabilities */
+	cap = EHCI_CAP_MULT ( ep->burst + 1 );
+
+	/* Determine interrupt schedule mask, if applicable */
+	if ( ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) &&
+	     ( ( ep->interval != 0 ) /* avoid infinite loop */ ) ) {
+		for ( i = 0 ; i < 8 /* microframes per frame */ ;
+		      i += ep->interval ) {
+			cap |= EHCI_CAP_INTR_SCHED ( i );
+		}
+	}
+
+	/* Set transaction translator hub address and port, if applicable */
+	if ( tt ) {
+		assert ( tt->hub->usb );
+		cap |= ( EHCI_CAP_TT_HUB ( tt->hub->usb->address ) |
+			 EHCI_CAP_TT_PORT ( tt->address ) );
+		if ( attr == USB_ENDPOINT_ATTR_INTERRUPT )
+			cap |= EHCI_CAP_SPLIT_SCHED_DEFAULT;
+	}
+
+	return cap;
+}
+
+/**
+ * Update endpoint characteristics and capabilities
+ *
+ * @v ep		USB endpoint
+ */
+static void ehci_endpoint_update ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_queue_head *head;
+
+	/* Update queue characteristics and capabilities */
+	head = endpoint->ring.head;
+	head->chr = cpu_to_le32 ( ehci_endpoint_characteristics ( ep ) );
+	head->cap = cpu_to_le32 ( ehci_endpoint_capabilities ( ep ) );
+}
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct ehci_endpoint *endpoint;
+	int rc;
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	endpoint->ehci = ehci;
+	endpoint->ep = ep;
+	usb_endpoint_set_hostdata ( ep, endpoint );
+
+	/* Initialise descriptor ring */
+	if ( ( rc = ehci_ring_alloc ( ehci, &endpoint->ring ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Update queue characteristics and capabilities */
+	ehci_endpoint_update ( ep );
+
+	/* Add to list of endpoints */
+	list_add_tail ( &endpoint->list, &ehci->endpoints );
+
+	/* Add to schedule */
+	ehci_schedule_add ( endpoint );
+
+	return 0;
+
+	ehci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void ehci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	struct usb_device *usb = ep->usb;
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Remove from schedule */
+	if ( ( rc = ehci_schedule_del ( endpoint ) ) != 0 ) {
+		/* No way to prevent hardware from continuing to
+		 * access the memory, so leak it.
+		 */
+		DBGC ( ehci, "EHCI %s %s could not unschedule: %s\n",
+		       usb->name, usb_endpoint_name ( ep ), strerror ( rc ) );
+		return;
+	}
+
+	/* Cancel any incomplete transfers */
+	while ( ehci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = ehci_dequeue ( &endpoint->ring );
+		if ( iobuf )
+			usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Remove from list of endpoints */
+	list_del ( &endpoint->list );
+
+	/* Free descriptor ring */
+	ehci_ring_free ( &endpoint->ring );
+
+	/* Free endpoint */
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_ring *ring = &endpoint->ring;
+	struct ehci_transfer_descriptor *cache = &ring->head->cache;
+	uint32_t link;
+
+	/* Sanity checks */
+	assert ( ! ( cache->status & EHCI_STATUS_ACTIVE ) );
+	assert ( cache->status & EHCI_STATUS_HALTED );
+
+	/* Reset residual count */
+	ring->residual = 0;
+
+	/* Reset data toggle */
+	cache->len = 0;
+
+	/* Prepare to restart at next unconsumed descriptor */
+	link = virt_to_phys ( &ring->desc[ ring->cons % EHCI_RING_COUNT ] );
+	cache->next = cpu_to_le32 ( link );
+
+	/* Restart ring */
+	wmb();
+	cache->status = 0;
+
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_mtu ( struct usb_endpoint *ep ) {
+
+	/* Update endpoint characteristics and capabilities */
+	ehci_endpoint_update ( ep );
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	struct usb_setup_packet *packet;
+	unsigned int input;
+	struct ehci_transfer xfers[3];
+	struct ehci_transfer *xfer = xfers;
+	size_t len;
+	int rc;
+
+	/* Construct setup stage */
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	packet = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *packet ) );
+	xfer->data = packet;
+	xfer->len = sizeof ( *packet );
+	xfer->flags = EHCI_FL_PID_SETUP;
+	xfer++;
+
+	/* Construct data stage, if applicable */
+	len = iob_len ( iobuf );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len ) {
+		xfer->data = iobuf->data;
+		xfer->len = len;
+		xfer->flags = ( EHCI_FL_TOGGLE |
+				( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT ) );
+		xfer++;
+	}
+
+	/* Construct status stage */
+	xfer->data = NULL;
+	xfer->len = 0;
+	xfer->flags = ( EHCI_FL_TOGGLE | EHCI_FL_IOC |
+			( ( len && input ) ? EHCI_FL_PID_OUT : EHCI_FL_PID_IN));
+	xfer++;
+
+	/* Enqueue transfer */
+	if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
+				   ( xfer - xfers ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int ehci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct ehci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct ehci_device *ehci = endpoint->ehci;
+	unsigned int input = ( ep->address & USB_DIR_IN );
+	struct ehci_transfer xfers[2];
+	struct ehci_transfer *xfer = xfers;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Create transfer */
+	xfer->data = iobuf->data;
+	xfer->len = len;
+	xfer->flags = ( EHCI_FL_IOC |
+			( input ? EHCI_FL_PID_IN : EHCI_FL_PID_OUT ) );
+	xfer++;
+	if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
+		xfer->data = NULL;
+		xfer->len = 0;
+		assert ( ! input );
+		xfer->flags = ( EHCI_FL_IOC | EHCI_FL_PID_OUT );
+		xfer++;
+	}
+
+	/* Enqueue transfer */
+	if ( ( rc = ehci_enqueue ( ehci, &endpoint->ring, iobuf, xfers,
+				   ( xfer - xfers ) ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Poll for completions
+ *
+ * @v endpoint		Endpoint
+ */
+static void ehci_endpoint_poll ( struct ehci_endpoint *endpoint ) {
+	struct ehci_device *ehci = endpoint->ehci;
+	struct ehci_ring *ring = &endpoint->ring;
+	struct ehci_transfer_descriptor *desc;
+	struct usb_endpoint *ep = endpoint->ep;
+	struct usb_device *usb = ep->usb;
+	struct io_buffer *iobuf;
+	unsigned int index;
+	unsigned int status;
+	int rc;
+
+	/* Consume all completed descriptors */
+	while ( ehci_ring_fill ( &endpoint->ring ) ) {
+
+		/* Stop if we reach an uncompleted descriptor */
+		rmb();
+		index = ( ring->cons % EHCI_RING_COUNT );
+		desc = &ring->desc[index];
+		status = desc->status;
+		if ( status & EHCI_STATUS_ACTIVE )
+			break;
+
+		/* Consume this descriptor */
+		iobuf = ehci_dequeue ( ring );
+
+		/* If we have encountered an error, then consume all
+		 * remaining descriptors in this transaction, report
+		 * the error to the USB core, and stop further
+		 * processing.
+		 */
+		if ( status & EHCI_STATUS_HALTED ) {
+			rc = -EIO_STATUS ( status );
+			DBGC ( ehci, "EHCI %s %s completion %d failed (status "
+			       "%02x): %s\n", usb->name,
+			       usb_endpoint_name ( ep ), index, status,
+			       strerror ( rc ) );
+			while ( ! iobuf )
+				iobuf = ehci_dequeue ( ring );
+			usb_complete_err ( endpoint->ep, iobuf, rc );
+			return;
+		}
+
+		/* Accumulate residual data count */
+		ring->residual += ( le16_to_cpu ( desc->len ) & EHCI_LEN_MASK );
+
+		/* If this is not the end of a transaction (i.e. has
+		 * no I/O buffer), then continue to next descriptor.
+		 */
+		if ( ! iobuf )
+			continue;
+
+		/* Update I/O buffer length */
+		iob_unput ( iobuf, ring->residual );
+		ring->residual = 0;
+
+		/* Report completion to USB core */
+		usb_complete ( endpoint->ep, iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int ehci_device_open ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( usb->port->hub->bus );
+
+	usb_set_hostdata ( usb, ehci );
+	return 0;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void ehci_device_close ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = ehci->bus;
+
+	/* Free device address, if assigned */
+	if ( usb->address )
+		usb_free_address ( bus, usb->address );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int ehci_device_address ( struct usb_device *usb ) {
+	struct ehci_device *ehci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = ehci->bus;
+	struct usb_endpoint *ep0 = usb_endpoint ( usb, USB_EP0_ADDRESS );
+	int address;
+	int rc;
+
+	/* Sanity checks */
+	assert ( usb->address == 0 );
+	assert ( ep0 != NULL );
+
+	/* Allocate device address */
+	address = usb_alloc_address ( bus );
+	if ( address < 0 ) {
+		rc = address;
+		DBGC ( ehci, "EHCI %s could not allocate address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_alloc_address;
+	}
+
+	/* Set address */
+	if ( ( rc = usb_set_address ( usb, address ) ) != 0 )
+		goto err_set_address;
+
+	/* Update device address */
+	usb->address = address;
+
+	/* Update control endpoint characteristics and capabilities */
+	ehci_endpoint_update ( ep0 );
+
+	return 0;
+
+ err_set_address:
+	usb_free_address ( bus, address );
+ err_alloc_address:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int ehci_hub_open ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void ehci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int ehci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Route all ports to EHCI controller */
+	writel ( EHCI_CONFIGFLAG_CF, ehci->op + EHCI_OP_CONFIGFLAG );
+
+	/* Enable power to all ports */
+	for ( i = 1 ; i <= ehci->ports ; i++ ) {
+		portsc = readl ( ehci->op + EHCI_OP_PORTSC ( i ) );
+		portsc &= ~EHCI_PORTSC_CHANGE;
+		portsc |= EHCI_PORTSC_PP;
+		writel ( portsc, ehci->op + EHCI_OP_PORTSC ( i ) );
+	}
+
+	/* Wait 20ms after potentially enabling power to a port */
+	mdelay ( EHCI_PORT_POWER_DELAY_MS );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, ehci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void ehci_root_close ( struct usb_hub *hub ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+
+	/* Route all ports back to companion controllers */
+	writel ( 0, ehci->op + EHCI_OP_CONFIGFLAG );
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int line;
+	unsigned int i;
+
+	/* Check for a low-speed device */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	line = EHCI_PORTSC_LINE_STATUS ( portsc );
+	if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
+		DBGC ( ehci, "EHCI %s-%d detected low-speed device: "
+		       "disowning\n", ehci->name, port->address );
+		goto disown;
+	}
+
+	/* Reset port */
+	portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
+	portsc |= EHCI_PORTSC_PR;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_DELAY_MS );
+	portsc &= ~EHCI_PORTSC_PR;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < EHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+		if ( ! ( portsc & EHCI_PORTSC_PR ) ) {
+			if ( portsc & EHCI_PORTSC_PED )
+				return 0;
+			DBGC ( ehci, "EHCI %s-%d not enabled after reset: "
+			       "disowning\n", ehci->name, port->address );
+			goto disown;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( ehci, "EHCI %s-%d timed out waiting for port to reset\n",
+	       ehci->name, port->address );
+	return -ETIMEDOUT;
+
+ disown:
+	/* Disown port */
+	portsc &= ~EHCI_PORTSC_CHANGE;
+	portsc |= EHCI_PORTSC_OWNER;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Delay to allow child companion controllers to settle */
+	mdelay ( EHCI_DISOWN_DELAY_MS );
+
+	/* Poll child companion controllers */
+	ehci_poll_companions ( ehci );
+
+	return -ENODEV;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+
+	/* Disable port */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	portsc &= ~( EHCI_PORTSC_PED | EHCI_PORTSC_CHANGE );
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int ehci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int speed;
+	unsigned int line;
+	int ccs;
+	int csc;
+	int ped;
+
+	/* Read port status */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	DBGC2 ( ehci, "EHCI %s-%d status is %08x\n",
+		ehci->name, port->address, portsc );
+	ccs = ( portsc & EHCI_PORTSC_CCS );
+	csc = ( portsc & EHCI_PORTSC_CSC );
+	ped = ( portsc & EHCI_PORTSC_PED );
+	line = EHCI_PORTSC_LINE_STATUS ( portsc );
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= csc;
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Determine port speed */
+	if ( ! ccs ) {
+		/* Port not connected */
+		speed = USB_SPEED_NONE;
+	} else if ( line == EHCI_PORTSC_LINE_STATUS_LOW ) {
+		/* Detected as low-speed */
+		speed = USB_SPEED_LOW;
+	} else if ( ped ) {
+		/* Port already enabled: must be high-speed */
+		speed = USB_SPEED_HIGH;
+	} else {
+		/* Not low-speed and not yet enabled.  Could be either
+		 * full-speed or high-speed; we can't yet tell.
+		 */
+		speed = USB_SPEED_FULL;
+	}
+	port->speed = speed;
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int ehci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( ehci, "EHCI %s-%d nonsensical CLEAR_TT for %s %s\n", ehci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/**
+ * Poll for port status changes
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ */
+static void ehci_root_poll ( struct usb_hub *hub, struct usb_port *port ) {
+	struct ehci_device *ehci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	uint32_t change;
+
+	/* Do nothing unless something has changed */
+	portsc = readl ( ehci->op + EHCI_OP_PORTSC ( port->address ) );
+	change = ( portsc & EHCI_PORTSC_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & EHCI_PORTSC_CSC );
+	writel ( portsc, ehci->op + EHCI_OP_PORTSC ( port->address ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int ehci_bus_open ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	unsigned int frames;
+	size_t len;
+	int rc;
+
+	/* Sanity checks */
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+
+	/* Allocate and initialise asynchronous queue head */
+	ehci->head = malloc_dma ( sizeof ( *ehci->head ),
+				  ehci_align ( sizeof ( *ehci->head ) ) );
+	if ( ! ehci->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_head;
+	}
+	memset ( ehci->head, 0, sizeof ( *ehci->head ) );
+	ehci->head->chr = cpu_to_le32 ( EHCI_CHR_HEAD );
+	ehci->head->cache.next = cpu_to_le32 ( EHCI_LINK_TERMINATE );
+	ehci->head->cache.status = EHCI_STATUS_HALTED;
+	ehci_async_schedule ( ehci );
+	writel ( virt_to_phys ( ehci->head ),
+		 ehci->op + EHCI_OP_ASYNCLISTADDR );
+
+	/* Use async queue head to determine control data structure segment */
+	ehci->ctrldssegment =
+		( ( ( uint64_t ) virt_to_phys ( ehci->head ) ) >> 32 );
+	if ( ehci->addr64 ) {
+		writel ( ehci->ctrldssegment, ehci->op + EHCI_OP_CTRLDSSEGMENT);
+	} else if ( ehci->ctrldssegment ) {
+		DBGC ( ehci, "EHCI %s CTRLDSSEGMENT not supported\n",
+		       ehci->name );
+		rc = -ENOTSUP;
+		goto err_ctrldssegment;
+	}
+
+	/* Allocate periodic frame list */
+	frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+	len = ( frames * sizeof ( ehci->frame[0] ) );
+	ehci->frame = malloc_dma ( len, EHCI_PAGE_ALIGN );
+	if ( ! ehci->frame ) {
+		rc = -ENOMEM;
+		goto err_alloc_frame;
+	}
+	if ( ( rc = ehci_ctrl_reachable ( ehci, ehci->frame ) ) != 0 ) {
+		DBGC ( ehci, "EHCI %s frame list unreachable\n", ehci->name );
+		goto err_unreachable_frame;
+	}
+	ehci_periodic_schedule ( ehci );
+	writel ( virt_to_phys ( ehci->frame ),
+		 ehci->op + EHCI_OP_PERIODICLISTBASE );
+
+	/* Start controller */
+	ehci_run ( ehci );
+
+	return 0;
+
+	ehci_stop ( ehci );
+ err_unreachable_frame:
+	free_dma ( ehci->frame, len );
+ err_alloc_frame:
+ err_ctrldssegment:
+	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+ err_alloc_head:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void ehci_bus_close ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	unsigned int frames = EHCI_PERIODIC_FRAMES ( ehci->flsize );
+
+	/* Sanity checks */
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+
+	/* Stop controller */
+	ehci_stop ( ehci );
+
+	/* Free periodic frame list */
+	free_dma ( ehci->frame, ( frames * sizeof ( ehci->frame[0] ) ) );
+
+	/* Free asynchronous schedule */
+	free_dma ( ehci->head, sizeof ( *ehci->head ) );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void ehci_bus_poll ( struct usb_bus *bus ) {
+	struct ehci_device *ehci = usb_bus_get_hostdata ( bus );
+	struct usb_hub *hub = bus->hub;
+	struct ehci_endpoint *endpoint;
+	unsigned int i;
+	uint32_t usbsts;
+	uint32_t change;
+
+	/* Do nothing unless something has changed */
+	usbsts = readl ( ehci->op + EHCI_OP_USBSTS );
+	assert ( usbsts & EHCI_USBSTS_ASYNC );
+	assert ( usbsts & EHCI_USBSTS_PERIODIC );
+	assert ( ! ( usbsts & EHCI_USBSTS_HCH ) );
+	change = ( usbsts & EHCI_USBSTS_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Acknowledge changes */
+	writel ( usbsts, ehci->op + EHCI_OP_USBSTS );
+
+	/* Process completions, if applicable */
+	if ( change & ( EHCI_USBSTS_USBINT | EHCI_USBSTS_USBERRINT ) ) {
+
+		/* Iterate over all endpoints looking for completed
+		 * descriptors.  We trust that completion handlers are
+		 * minimal and will not do anything that could
+		 * plausibly affect the endpoint list itself.
+		 */
+		list_for_each_entry ( endpoint, &ehci->endpoints, list )
+			ehci_endpoint_poll ( endpoint );
+	}
+
+	/* Process port status changes, if applicable */
+	if ( change & EHCI_USBSTS_PORT ) {
+
+		/* Iterate over all ports looking for status changes */
+		for ( i = 1 ; i <= ehci->ports ; i++ )
+			ehci_root_poll ( hub, usb_port ( hub, i ) );
+	}
+
+	/* Report fatal errors */
+	if ( change & EHCI_USBSTS_SYSERR )
+		DBGC ( ehci, "EHCI %s host system error\n", ehci->name );
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations ehci_operations = {
+	.endpoint = {
+		.open = ehci_endpoint_open,
+		.close = ehci_endpoint_close,
+		.reset = ehci_endpoint_reset,
+		.mtu = ehci_endpoint_mtu,
+		.message = ehci_endpoint_message,
+		.stream = ehci_endpoint_stream,
+	},
+	.device = {
+		.open = ehci_device_open,
+		.close = ehci_device_close,
+		.address = ehci_device_address,
+	},
+	.bus = {
+		.open = ehci_bus_open,
+		.close = ehci_bus_close,
+		.poll = ehci_bus_poll,
+	},
+	.hub = {
+		.open = ehci_hub_open,
+		.close = ehci_hub_close,
+	},
+	.root = {
+		.open = ehci_root_open,
+		.close = ehci_root_close,
+		.enable = ehci_root_enable,
+		.disable = ehci_root_disable,
+		.speed = ehci_root_speed,
+		.clear_tt = ehci_root_clear_tt,
+	},
+};
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int ehci_probe ( struct pci_device *pci ) {
+	struct ehci_device *ehci;
+	struct usb_port *port;
+	unsigned long bar_start;
+	size_t bar_size;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	ehci = zalloc ( sizeof ( *ehci ) );
+	if ( ! ehci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ehci->name = pci->dev.name;
+	INIT_LIST_HEAD ( &ehci->endpoints );
+	INIT_LIST_HEAD ( &ehci->async );
+	INIT_LIST_HEAD ( &ehci->periodic );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	bar_start = pci_bar_start ( pci, EHCI_BAR );
+	bar_size = pci_bar_size ( pci, EHCI_BAR );
+	ehci->regs = ioremap ( bar_start, bar_size );
+	if ( ! ehci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Initialise EHCI device */
+	ehci_init ( ehci, ehci->regs );
+
+	/* Initialise USB legacy support and claim ownership */
+	ehci_legacy_init ( ehci, pci );
+	ehci_legacy_claim ( ehci, pci );
+
+	/* Reset device */
+	if ( ( rc = ehci_reset ( ehci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	ehci->bus = alloc_usb_bus ( &pci->dev, ehci->ports, EHCI_MTU,
+				    &ehci_operations );
+	if ( ! ehci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( ehci->bus, ehci );
+	usb_hub_set_drvdata ( ehci->bus->hub, ehci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= ehci->ports ; i++ ) {
+		port = usb_port ( ehci->bus->hub, i );
+		port->protocol = USB_PROTO_2_0;
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( ehci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, ehci );
+	return 0;
+
+	unregister_usb_bus ( ehci->bus );
+ err_register:
+	free_usb_bus ( ehci->bus );
+ err_alloc_bus:
+	ehci_reset ( ehci );
+ err_reset:
+	ehci_legacy_release ( ehci, pci );
+	iounmap ( ehci->regs );
+ err_ioremap:
+	free ( ehci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void ehci_remove ( struct pci_device *pci ) {
+	struct ehci_device *ehci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = ehci->bus;
+
+	unregister_usb_bus ( bus );
+	assert ( list_empty ( &ehci->async ) );
+	assert ( list_empty ( &ehci->periodic ) );
+	free_usb_bus ( bus );
+	ehci_reset ( ehci );
+	ehci_legacy_release ( ehci, pci );
+	iounmap ( ehci->regs );
+	free ( ehci );
+}
+
+/** EHCI PCI device IDs */
+static struct pci_device_id ehci_ids[] = {
+	PCI_ROM ( 0xffff, 0xffff, "ehci", "EHCI", 0 ),
+};
+
+/** EHCI PCI driver */
+struct pci_driver ehci_driver __pci_driver = {
+	.ids = ehci_ids,
+	.id_count = ( sizeof ( ehci_ids ) / sizeof ( ehci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_EHCI ),
+	.probe = ehci_probe,
+	.remove = ehci_remove,
+};
+
+/**
+ * Prepare for exit
+ *
+ * @v booting		System is shutting down for OS boot
+ */
+static void ehci_shutdown ( int booting ) {
+	/* If we are shutting down to boot an OS, then prevent the
+	 * release of ownership back to BIOS.
+	 */
+	ehci_legacy_prevent_release = booting;
+}
+
+/** Startup/shutdown function */
+struct startup_fn ehci_startup __startup_fn ( STARTUP_LATE ) = {
+	.shutdown = ehci_shutdown,
+};
diff --git a/qemu/roms/ipxe/src/drivers/usb/ehci.h b/qemu/roms/ipxe/src/drivers/usb/ehci.h
new file mode 100644
index 000000000..42e282e92
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/ehci.h
@@ -0,0 +1,544 @@
+#ifndef _IPXE_EHCI_H
+#define _IPXE_EHCI_H
+
+/** @file
+ *
+ * USB Enhanced Host Controller Interface (EHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the periodic frame list (which is
+ * page-aligned), data structures used by EHCI generally require
+ * 32-byte alignment and must not cross a 4kB page boundary.  We
+ * simplify this requirement by aligning each structure on its own
+ * size, with a minimum of a 32 byte alignment.
+ */
+#define EHCI_MIN_ALIGN 32
+
+/** Maximum transfer size
+ *
+ * EHCI allows for transfers of up to 20kB with page-alignment, or
+ * 16kB with arbitrary alignment.
+ */
+#define EHCI_MTU 16384
+
+/** Page-alignment required for some data structures */
+#define EHCI_PAGE_ALIGN 4096
+
+/** EHCI PCI BAR */
+#define EHCI_BAR PCI_BASE_ADDRESS_0
+
+/** Capability register length */
+#define EHCI_CAP_CAPLENGTH 0x00
+
+/** Host controller interface version number */
+#define EHCI_CAP_HCIVERSION 0x02
+
+/** Structural parameters */
+#define EHCI_CAP_HCSPARAMS 0x04
+
+/** Number of ports */
+#define EHCI_HCSPARAMS_PORTS(params) ( ( (params) >> 0 ) & 0x0f )
+
+/** Capability parameters */
+#define EHCI_CAP_HCCPARAMS 0x08
+
+/** 64-bit addressing capability */
+#define EHCI_HCCPARAMS_ADDR64(params) ( ( (params) >> 0 ) & 0x1 )
+
+/** Programmable frame list flag */
+#define EHCI_HCCPARAMS_FLSIZE(params) ( ( (params) >> 1 ) & 0x1 )
+
+/** EHCI extended capabilities pointer */
+#define EHCI_HCCPARAMS_EECP(params) ( ( ( (params) >> 8 ) & 0xff ) )
+
+/** EHCI extended capability ID */
+#define EHCI_EECP_ID(eecp) ( ( (eecp) >> 0 ) & 0xff )
+
+/** Next EHCI extended capability pointer */
+#define EHCI_EECP_NEXT(eecp) ( ( ( (eecp) >> 8 ) & 0xff ) )
+
+/** USB legacy support extended capability */
+#define EHCI_EECP_ID_LEGACY 1
+
+/** USB legacy support BIOS owned semaphore */
+#define EHCI_USBLEGSUP_BIOS 0x02
+
+/** USB legacy support BIOS ownership flag */
+#define EHCI_USBLEGSUP_BIOS_OWNED 0x01
+
+/** USB legacy support OS owned semaphore */
+#define EHCI_USBLEGSUP_OS 0x03
+
+/** USB legacy support OS ownership flag */
+#define EHCI_USBLEGSUP_OS_OWNED 0x01
+
+/** USB legacy support control/status */
+#define EHCI_USBLEGSUP_CTLSTS 0x04
+
+/** USB command register */
+#define EHCI_OP_USBCMD 0x00
+
+/** Run/stop */
+#define EHCI_USBCMD_RUN 0x00000001UL
+
+/** Host controller reset */
+#define EHCI_USBCMD_HCRST 0x00000002UL
+
+/** Frame list size */
+#define EHCI_USBCMD_FLSIZE(flsize) ( (flsize) << 2 )
+
+/** Frame list size mask */
+#define EHCI_USBCMD_FLSIZE_MASK EHCI_USBCMD_FLSIZE ( 3 )
+
+/** Default frame list size */
+#define EHCI_FLSIZE_DEFAULT 0
+
+/** Smallest allowed frame list size */
+#define EHCI_FLSIZE_SMALL 2
+
+/** Number of elements in frame list */
+#define EHCI_PERIODIC_FRAMES(flsize) ( 1024 >> (flsize) )
+
+/** Periodic schedule enable */
+#define EHCI_USBCMD_PERIODIC 0x00000010UL
+
+/** Asynchronous schedule enable */
+#define EHCI_USBCMD_ASYNC 0x00000020UL
+
+/** Asyncchronous schedule advance doorbell */
+#define EHCI_USBCMD_ASYNC_ADVANCE 0x000040UL
+
+/** USB status register */
+#define EHCI_OP_USBSTS 0x04
+
+/** USB interrupt */
+#define EHCI_USBSTS_USBINT 0x00000001UL
+
+/** USB error interrupt */
+#define EHCI_USBSTS_USBERRINT 0x00000002UL
+
+/** Port change detect */
+#define EHCI_USBSTS_PORT 0x00000004UL
+
+/** Frame list rollover */
+#define EHCI_USBSTS_ROLLOVER 0x00000008UL
+
+/** Host system error */
+#define EHCI_USBSTS_SYSERR 0x00000010UL
+
+/** Asynchronous schedule advanced */
+#define EHCI_USBSTS_ASYNC_ADVANCE 0x00000020UL
+
+/** Periodic schedule enabled */
+#define EHCI_USBSTS_PERIODIC 0x00004000UL
+
+/** Asynchronous schedule enabled */
+#define EHCI_USBSTS_ASYNC 0x00008000UL
+
+/** Host controller halted */
+#define EHCI_USBSTS_HCH 0x00001000UL
+
+/** USB status change mask */
+#define EHCI_USBSTS_CHANGE						\
+	( EHCI_USBSTS_USBINT | EHCI_USBSTS_USBERRINT |			\
+	  EHCI_USBSTS_PORT | EHCI_USBSTS_ROLLOVER |			\
+	  EHCI_USBSTS_SYSERR | EHCI_USBSTS_ASYNC_ADVANCE )
+
+/** USB interrupt enable register */
+#define EHCI_OP_USBINTR 0x08
+
+/** Frame index register */
+#define EHCI_OP_FRINDEX 0x0c
+
+/** Control data structure segment register */
+#define EHCI_OP_CTRLDSSEGMENT 0x10
+
+/** Periodic frame list base address register */
+#define EHCI_OP_PERIODICLISTBASE 0x14
+
+/** Current asynchronous list address register */
+#define EHCI_OP_ASYNCLISTADDR 0x18
+
+/** Configure flag register */
+#define EHCI_OP_CONFIGFLAG 0x40
+
+/** Configure flag */
+#define EHCI_CONFIGFLAG_CF 0x00000001UL
+
+/** Port status and control register */
+#define EHCI_OP_PORTSC(port) ( 0x40 + ( (port) << 2 ) )
+
+/** Current connect status */
+#define EHCI_PORTSC_CCS 0x00000001UL
+
+/** Connect status change */
+#define EHCI_PORTSC_CSC 0x00000002UL
+
+/** Port enabled */
+#define EHCI_PORTSC_PED 0x00000004UL
+
+/** Port enabled/disabled change */
+#define EHCI_PORTSC_PEC 0x00000008UL
+
+/** Over-current change */
+#define EHCI_PORTSC_OCC 0x00000020UL
+
+/** Port reset */
+#define EHCI_PORTSC_PR 0x00000100UL
+
+/** Line status */
+#define EHCI_PORTSC_LINE_STATUS(portsc) ( ( (portsc) >> 10 ) & 0x3 )
+
+/** Line status: low-speed device */
+#define EHCI_PORTSC_LINE_STATUS_LOW 0x1
+
+/** Port power */
+#define EHCI_PORTSC_PP 0x00001000UL
+
+/** Port owner */
+#define EHCI_PORTSC_OWNER 0x00002000UL
+
+/** Port status change mask */
+#define EHCI_PORTSC_CHANGE \
+	( EHCI_PORTSC_CSC | EHCI_PORTSC_PEC | EHCI_PORTSC_OCC )
+
+/** List terminator */
+#define EHCI_LINK_TERMINATE 0x00000001UL
+
+/** Frame list type */
+#define EHCI_LINK_TYPE(type) ( (type) << 1 )
+
+/** Queue head type */
+#define EHCI_LINK_TYPE_QH EHCI_LINK_TYPE ( 1 )
+
+/** A periodic frame list entry */
+struct ehci_periodic_frame {
+	/** First queue head */
+	uint32_t link;
+} __attribute__ (( packed ));
+
+/** A transfer descriptor */
+struct ehci_transfer_descriptor {
+	/** Next transfer descriptor */
+	uint32_t next;
+	/** Alternate next transfer descriptor */
+	uint32_t alt;
+	/** Status */
+	uint8_t status;
+	/** Flags */
+	uint8_t flags;
+	/** Transfer length */
+	uint16_t len;
+	/** Buffer pointers (low 32 bits) */
+	uint32_t low[5];
+	/** Extended buffer pointers (high 32 bits) */
+	uint32_t high[5];
+	/** Reserved */
+	uint8_t reserved[12];
+} __attribute__ (( packed ));
+
+/** Transaction error */
+#define EHCI_STATUS_XACT_ERR 0x08
+
+/** Babble detected */
+#define EHCI_STATUS_BABBLE 0x10
+
+/** Data buffer error */
+#define EHCI_STATUS_BUFFER 0x20
+
+/** Halted */
+#define EHCI_STATUS_HALTED 0x40
+
+/** Active */
+#define EHCI_STATUS_ACTIVE 0x80
+
+/** PID code */
+#define EHCI_FL_PID(code) ( (code) << 0 )
+
+/** OUT token */
+#define EHCI_FL_PID_OUT EHCI_FL_PID ( 0 )
+
+/** IN token */
+#define EHCI_FL_PID_IN EHCI_FL_PID ( 1 )
+
+/** SETUP token */
+#define EHCI_FL_PID_SETUP EHCI_FL_PID ( 2 )
+
+/** Error counter */
+#define EHCI_FL_CERR( count ) ( (count) << 2 )
+
+/** Error counter maximum value */
+#define EHCI_FL_CERR_MAX EHCI_FL_CERR ( 3 )
+
+/** Interrupt on completion */
+#define EHCI_FL_IOC 0x80
+
+/** Length mask */
+#define EHCI_LEN_MASK 0x7fff
+
+/** Data toggle */
+#define EHCI_LEN_TOGGLE 0x8000
+
+/** A queue head */
+struct ehci_queue_head {
+	/** Horizontal link pointer */
+	uint32_t link;
+	/** Endpoint characteristics */
+	uint32_t chr;
+	/** Endpoint capabilities */
+	uint32_t cap;
+	/** Current transfer descriptor */
+	uint32_t current;
+	/** Transfer descriptor cache */
+	struct ehci_transfer_descriptor cache;
+} __attribute__ (( packed ));
+
+/** Device address */
+#define EHCI_CHR_ADDRESS( address ) ( (address) << 0 )
+
+/** Endpoint number */
+#define EHCI_CHR_ENDPOINT( address ) ( ( (address) & 0xf ) << 8 )
+
+/** Endpoint speed */
+#define EHCI_CHR_EPS( eps ) ( (eps) << 12 )
+
+/** Full-speed endpoint */
+#define EHCI_CHR_EPS_FULL EHCI_CHR_EPS ( 0 )
+
+/** Low-speed endpoint */
+#define EHCI_CHR_EPS_LOW EHCI_CHR_EPS ( 1 )
+
+/** High-speed endpoint */
+#define EHCI_CHR_EPS_HIGH EHCI_CHR_EPS ( 2 )
+
+/** Explicit data toggles */
+#define EHCI_CHR_TOGGLE 0x00004000UL
+
+/** Head of reclamation list flag */
+#define EHCI_CHR_HEAD 0x00008000UL
+
+/** Maximum packet length */
+#define EHCI_CHR_MAX_LEN( len ) ( (len) << 16 )
+
+/** Control endpoint flag */
+#define EHCI_CHR_CONTROL 0x08000000UL
+
+/** Interrupt schedule mask */
+#define EHCI_CAP_INTR_SCHED( uframe ) ( 1 << ( (uframe) + 0 ) )
+
+/** Split completion schedule mask */
+#define EHCI_CAP_SPLIT_SCHED( uframe ) ( 1 << ( (uframe) + 8 ) )
+
+/** Default split completion schedule mask
+ *
+ * We schedule all split starts in microframe 0, on the assumption
+ * that we will never have to deal with more than sixteen actively
+ * interrupting devices via the same transaction translator.  We
+ * schedule split completions for all remaining microframes after
+ * microframe 1 (in which the low-speed or full-speed transaction is
+ * assumed to execute).  This is a very crude approximation designed
+ * to avoid the need for calculating exactly when low-speed and
+ * full-speed transactions will execute.  Since we only ever deal with
+ * interrupt endpoints (rather than isochronous endpoints), the volume
+ * of periodic traffic is extremely low, and this approximation should
+ * remain valid.
+ */
+#define EHCI_CAP_SPLIT_SCHED_DEFAULT					\
+	( EHCI_CAP_SPLIT_SCHED ( 2 ) | EHCI_CAP_SPLIT_SCHED ( 3 ) |	\
+	  EHCI_CAP_SPLIT_SCHED ( 4 ) | EHCI_CAP_SPLIT_SCHED ( 5 ) |	\
+	  EHCI_CAP_SPLIT_SCHED ( 6 ) | EHCI_CAP_SPLIT_SCHED ( 7 ) )
+
+/** Transaction translator hub address */
+#define EHCI_CAP_TT_HUB( address ) ( (address) << 16 )
+
+/** Transaction translator port number */
+#define EHCI_CAP_TT_PORT( port ) ( (port) << 23 )
+
+/** High-bandwidth pipe multiplier */
+#define EHCI_CAP_MULT( mult ) ( (mult) << 30 )
+
+/** A transfer descriptor ring */
+struct ehci_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+
+	/** Residual untransferred data */
+	size_t residual;
+
+	/** I/O buffers */
+	struct io_buffer **iobuf;
+
+	/** Queue head */
+	struct ehci_queue_head *head;
+	/** Transfer descriptors */
+	struct ehci_transfer_descriptor *desc;
+};
+
+/** Number of transfer descriptors in a ring
+ *
+ * This is a policy decision.
+ */
+#define EHCI_RING_COUNT 64
+
+/**
+ * Calculate space used in transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+ehci_ring_fill ( struct ehci_ring *ring ) {
+	unsigned int fill;
+
+	fill = ( ring->prod - ring->cons );
+	assert ( fill <= EHCI_RING_COUNT );
+	return fill;
+}
+
+/**
+ * Calculate space remaining in transfer descriptor ring
+ *
+ * @v ring		Transfer descriptor ring
+ * @ret remaining	Number of entries remaining
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+ehci_ring_remaining ( struct ehci_ring *ring ) {
+	unsigned int fill = ehci_ring_fill ( ring );
+
+	return ( EHCI_RING_COUNT - fill );
+}
+
+/** Time to delay after enabling power to a port
+ *
+ * This is not mandated by EHCI; we use the value given for xHCI.
+ */
+#define EHCI_PORT_POWER_DELAY_MS 20
+
+/** Time to delay after releasing ownership of a port
+ *
+ * This is a policy decision.
+ */
+#define EHCI_DISOWN_DELAY_MS 100
+
+/** Maximum time to wait for BIOS to release ownership
+ *
+ * This is a policy decision.
+ */
+#define EHCI_USBLEGSUP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for asynchronous schedule to advance
+ *
+ * This is a policy decision.
+ */
+#define EHCI_ASYNC_ADVANCE_MAX_WAIT_MS 100
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define EHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define EHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a port reset to complete
+ *
+ * This is a policy decision.
+ */
+#define EHCI_PORT_RESET_MAX_WAIT_MS 500
+
+/** An EHCI transfer */
+struct ehci_transfer {
+	/** Data buffer */
+	void *data;
+	/** Length */
+	size_t len;
+	/** Flags
+	 *
+	 * This is the bitwise OR of zero or more EHCI_FL_XXX values.
+	 * The low 8 bits are copied to the flags byte within the
+	 * transfer descriptor; the remaining bits hold flags
+	 * meaningful only to our driver code.
+	 */
+	unsigned int flags;
+};
+
+/** Set initial data toggle */
+#define EHCI_FL_TOGGLE 0x8000
+
+/** An EHCI device */
+struct ehci_device {
+	/** Registers */
+	void *regs;
+	/** Name */
+	const char *name;
+
+	/** Capability registers */
+	void *cap;
+	/** Operational registers */
+	void *op;
+
+	/** Number of ports */
+	unsigned int ports;
+	/** 64-bit addressing capability */
+	int addr64;
+	/** Frame list size */
+	unsigned int flsize;
+	/** EHCI extended capabilities offset */
+	unsigned int eecp;
+
+	/** USB legacy support capability (if present and enabled) */
+	unsigned int legacy;
+
+	/** Control data structure segment */
+	uint32_t ctrldssegment;
+	/** Asynchronous queue head */
+	struct ehci_queue_head *head;
+	/** Periodic frame list */
+	struct ehci_periodic_frame *frame;
+
+	/** List of all endpoints */
+	struct list_head endpoints;
+	/** Asynchronous schedule */
+	struct list_head async;
+	/** Periodic schedule
+	 *
+	 * Listed in decreasing order of endpoint interval.
+	 */
+	struct list_head periodic;
+
+	/** USB bus */
+	struct usb_bus *bus;
+};
+
+/** An EHCI endpoint */
+struct ehci_endpoint {
+	/** EHCI device */
+	struct ehci_device *ehci;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** List of all endpoints */
+	struct list_head list;
+	/** Endpoint schedule */
+	struct list_head schedule;
+
+	/** Transfer descriptor ring */
+	struct ehci_ring ring;
+};
+
+extern unsigned int ehci_companion ( struct pci_device *pci );
+
+#endif /* _IPXE_EHCI_H */
diff --git a/qemu/roms/ipxe/src/drivers/usb/uhci.c b/qemu/roms/ipxe/src/drivers/usb/uhci.c
new file mode 100644
index 000000000..b6bb92560
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/uhci.c
@@ -0,0 +1,1577 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <strings.h>
+#include <unistd.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include "ehci.h"
+#include "uhci.h"
+
+/** @file
+ *
+ * USB Universal Host Controller Interface (UHCI) driver
+ *
+ */
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check that address is reachable
+ *
+ * @v addr		Address
+ * @v len		Length
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline)) int
+uhci_reachable ( void *addr, size_t len ) {
+	physaddr_t phys = virt_to_phys ( addr );
+
+	/* Always reachable in a 32-bit build */
+	if ( sizeof ( physaddr_t ) <= sizeof ( uint32_t ) )
+		return 0;
+
+	/* Reachable if below 4GB */
+	if ( ( ( phys + len - 1 ) & ~0xffffffffULL ) == 0 )
+		return 0;
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start UHCI device
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_run ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+
+	/* Set run/stop bit */
+	usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+	usbcmd |= ( UHCI_USBCMD_RUN | UHCI_USBCMD_MAX64 );
+	outw ( usbcmd, uhci->regs + UHCI_USBCMD );
+}
+
+/**
+ * Stop UHCI device
+ *
+ * @v uhci		UHCI device
+ * @ret rc		Return status code
+ */
+static int uhci_stop ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+	uint16_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+	usbcmd &= ~UHCI_USBCMD_RUN;
+	outw ( usbcmd, uhci->regs + UHCI_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < UHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = inw ( uhci->regs + UHCI_USBSTS );
+		if ( usbsts & UHCI_USBSTS_HCHALTED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s timed out waiting for stop\n", uhci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset UHCI device
+ *
+ * @v uhci		UHCI device
+ * @ret rc		Return status code
+ */
+static int uhci_reset ( struct uhci_device *uhci ) {
+	uint16_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The UHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = uhci_stop ( uhci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	outw ( UHCI_USBCMD_HCRESET, uhci->regs + UHCI_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < UHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = inw ( uhci->regs + UHCI_USBCMD );
+		if ( ! ( usbcmd & UHCI_USBCMD_HCRESET ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s timed out waiting for reset\n", uhci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer descriptor rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret rc		Return status code
+ */
+static int uhci_ring_alloc ( struct uhci_ring *ring ) {
+	int rc;
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+
+	/* Allocate queue head */
+	ring->head = malloc_dma ( sizeof ( *ring->head ), UHCI_ALIGN );
+	if ( ! ring->head ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	if ( ( rc = uhci_reachable ( ring->head,
+				     sizeof ( *ring->head ) ) ) != 0 )
+		goto err_unreachable;
+
+	/* Initialise queue head */
+	ring->head->current = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+
+	return 0;
+
+ err_unreachable:
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free transfer ring
+ *
+ * @v ring		Transfer ring
+ */
+static void uhci_ring_free ( struct uhci_ring *ring ) {
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( uhci_ring_fill ( ring ) == 0 );
+	for ( i = 0 ; i < UHCI_RING_COUNT ; i++ )
+		assert ( ring->xfer[i] == NULL );
+
+	/* Free queue head */
+	free_dma ( ring->head, sizeof ( *ring->head ) );
+}
+
+/**
+ * Enqueue new transfer
+ *
+ * @v ring		Transfer ring
+ * @v iobuf		I/O buffer
+ * @v count		Number of descriptors
+ * @ret rc		Return status code
+ */
+static int uhci_enqueue ( struct uhci_ring *ring, struct io_buffer *iobuf,
+			  unsigned int count ) {
+	struct uhci_transfer *xfer;
+	struct uhci_transfer *end;
+	struct uhci_transfer_descriptor *desc;
+	unsigned int index = ( ring->prod % UHCI_RING_COUNT );
+	uint32_t link;
+	size_t len;
+	int rc;
+
+	/* Sanity check */
+	assert ( count > 0 );
+	assert ( iobuf != NULL );
+
+	/* Check for space in ring */
+	if ( ! uhci_ring_remaining ( ring ) ) {
+		rc = -ENOBUFS;
+		goto err_ring_full;
+	}
+
+	/* Check for reachability of I/O buffer */
+	if ( ( rc = uhci_reachable ( iobuf->data, iob_len ( iobuf ) ) ) != 0 )
+		goto err_unreachable_iobuf;
+
+	/* Allocate transfer */
+	xfer = malloc ( sizeof ( *xfer ) );
+	if ( ! xfer ) {
+		rc = -ENOMEM;
+		goto err_alloc_xfer;
+	}
+
+	/* Initialise transfer */
+	xfer->prod = 0;
+	xfer->cons = 0;
+	xfer->len = 0;
+	xfer->iobuf = iobuf;
+
+	/* Allocate transfer descriptors */
+	len = ( count * sizeof ( xfer->desc[0] ) );
+	xfer->desc = malloc_dma ( len, UHCI_ALIGN );
+	if ( ! xfer->desc ) {
+		rc = -ENOMEM;
+		goto err_alloc_desc;
+	}
+	if ( ( rc = uhci_reachable ( xfer->desc, len ) ) != 0 )
+		goto err_unreachable_desc;
+
+	/* Initialise transfer descriptors */
+	memset ( xfer->desc, 0, len );
+	desc = xfer->desc;
+	for ( ; --count ; desc++ ) {
+		link = ( virt_to_phys ( desc + 1 ) | UHCI_LINK_DEPTH_FIRST );
+		desc->link = cpu_to_le32 ( link );
+		desc->flags = ring->flags;
+	}
+	desc->link = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+	desc->flags = ( ring->flags | UHCI_FL_IOC );
+
+	/* Add to ring */
+	wmb();
+	link = virt_to_phys ( xfer->desc );
+	if ( uhci_ring_fill ( ring ) > 0 ) {
+		end = ring->end;
+		end->desc[ end->prod - 1 ].link = cpu_to_le32 ( link );
+	} else {
+		ring->head->current = cpu_to_le32 ( link );
+	}
+	assert ( ring->xfer[index] == NULL );
+	ring->xfer[index] = xfer;
+	ring->end = xfer;
+	ring->prod++;
+
+	return 0;
+
+ err_unreachable_desc:
+	free_dma ( xfer->desc, len );
+ err_alloc_desc:
+	free ( xfer );
+ err_alloc_xfer:
+ err_unreachable_iobuf:
+ err_ring_full:
+	return rc;
+}
+
+/**
+ * Describe transfer
+ *
+ * @v ring		Transfer ring
+ * @v data		Data
+ * @v len		Length of data
+ * @v pid		Packet ID
+ */
+static void uhci_describe ( struct uhci_ring *ring, void *data,
+			    size_t len, uint8_t pid ) {
+	struct uhci_transfer *xfer = ring->end;
+	struct uhci_transfer_descriptor *desc;
+	size_t frag_len;
+	uint32_t control;
+
+	do {
+		/* Calculate fragment length */
+		frag_len = len;
+		if ( frag_len > ring->mtu )
+			frag_len = ring->mtu;
+
+		/* Populate descriptor */
+		desc = &xfer->desc[xfer->prod++];
+		if ( pid == USB_PID_IN )
+			desc->flags |= UHCI_FL_SPD;
+		control = ( ring->control | UHCI_CONTROL_PID ( pid ) |
+			    UHCI_CONTROL_LEN ( frag_len ) );
+		desc->control = cpu_to_le32 ( control );
+		if ( data )
+			desc->data = virt_to_phys ( data );
+		wmb();
+		desc->status = UHCI_STATUS_ACTIVE;
+
+		/* Update data toggle */
+		ring->control ^= UHCI_CONTROL_TOGGLE;
+
+		/* Move to next descriptor */
+		data += frag_len;
+		len -= frag_len;
+
+	} while ( len );
+}
+
+/**
+ * Dequeue transfer
+ *
+ * @v ring		Transfer ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * uhci_dequeue ( struct uhci_ring *ring ) {
+	unsigned int index = ( ring->cons % UHCI_RING_COUNT );
+	struct io_buffer *iobuf;
+	struct uhci_transfer *xfer;
+	size_t len;
+
+	/* Sanity checks */
+	assert ( uhci_ring_fill ( ring ) > 0 );
+
+	/* Consume transfer */
+	xfer = ring->xfer[index];
+	assert ( xfer != NULL );
+	assert ( xfer->desc != NULL );
+	iobuf = xfer->iobuf;
+	assert ( iobuf != NULL );
+	ring->xfer[index] = NULL;
+	ring->cons++;
+
+	/* Free transfer descriptors */
+	len = ( xfer->prod * sizeof ( xfer->desc[0] ) );
+	free_dma ( xfer->desc, len );
+
+	/* Free transfer */
+	free ( xfer );
+
+	return iobuf;
+}
+
+/**
+ * Restart ring
+ *
+ * @v ring		Transfer ring
+ * @v toggle		Expected data toggle for next descriptor
+ */
+static void uhci_restart ( struct uhci_ring *ring, uint32_t toggle ) {
+	struct uhci_transfer *xfer;
+	struct uhci_transfer_descriptor *desc;
+	struct uhci_transfer_descriptor *first;
+	uint32_t link;
+	unsigned int i;
+	unsigned int j;
+
+	/* Sanity check */
+	assert ( ring->head->current == cpu_to_le32 ( UHCI_LINK_TERMINATE ) );
+
+	/* If ring is empty, then just update the data toggle for the
+	 * next descriptor.
+	 */
+	if ( uhci_ring_fill ( ring ) == 0 ) {
+		ring->control &= ~UHCI_CONTROL_TOGGLE;
+		ring->control |= toggle;
+		return;
+	}
+
+	/* If expected toggle does not match the toggle in the first
+	 * unconsumed descriptor, then invert all toggles.
+	 */
+	xfer = ring->xfer[ ring->cons % UHCI_RING_COUNT ];
+	assert ( xfer != NULL );
+	assert ( xfer->cons == 0 );
+	first = &xfer->desc[0];
+	if ( ( le32_to_cpu ( first->control ) ^ toggle ) & UHCI_CONTROL_TOGGLE){
+
+		/* Invert toggle on all unconsumed transfer descriptors */
+		for ( i = ring->cons ; i != ring->prod ; i++ ) {
+			xfer = ring->xfer[ i % UHCI_RING_COUNT ];
+			assert ( xfer != NULL );
+			assert ( xfer->cons == 0 );
+			for ( j = 0 ; j < xfer->prod ; j++ ) {
+				desc = &xfer->desc[j];
+				desc->control ^=
+					cpu_to_le32 ( UHCI_CONTROL_TOGGLE );
+			}
+		}
+
+		/* Invert toggle for next descriptor to be enqueued */
+		ring->control ^= UHCI_CONTROL_TOGGLE;
+	}
+
+	/* Restart ring at first unconsumed transfer */
+	link = virt_to_phys ( first );
+	wmb();
+	ring->head->current = cpu_to_le32 ( link );
+}
+
+/******************************************************************************
+ *
+ * Schedule management
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get link value for a queue head
+ *
+ * @v queue		Queue head
+ * @ret link		Link value
+ */
+static inline uint32_t uhci_link_qh ( struct uhci_queue_head *queue ) {
+
+	return ( virt_to_phys ( queue ) | UHCI_LINK_TYPE_QH );
+}
+
+/**
+ * (Re)build asynchronous schedule
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_async_schedule ( struct uhci_device *uhci ) {
+	struct uhci_endpoint *endpoint;
+	struct uhci_queue_head *queue;
+	uint32_t end;
+	uint32_t link;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	link = end = uhci_link_qh ( uhci->head );
+	list_for_each_entry_reverse ( endpoint, &uhci->async, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		link = uhci_link_qh ( queue );
+	}
+	if ( link == end )
+		link = UHCI_LINK_TERMINATE;
+	uhci->head->link = cpu_to_le32 ( link );
+	wmb();
+}
+
+/**
+ * Add endpoint to asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_async_add ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Add to end of schedule */
+	list_add_tail ( &endpoint->schedule, &uhci->async );
+
+	/* Rebuild schedule */
+	uhci_async_schedule ( uhci );
+}
+
+/**
+ * Remove endpoint from asynchronous schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_async_del ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &uhci->async, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	uhci_async_schedule ( uhci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+}
+
+/**
+ * (Re)build periodic schedule
+ *
+ * @v uhci		UHCI device
+ */
+static void uhci_periodic_schedule ( struct uhci_device *uhci ) {
+	struct uhci_endpoint *endpoint;
+	struct uhci_queue_head *queue;
+	uint32_t link;
+	uint32_t end;
+	unsigned int max_interval;
+	unsigned int i;
+
+	/* Build schedule in reverse order of execution.  Provided
+	 * that we only ever add or remove single endpoints, this can
+	 * safely run concurrently with hardware execution of the
+	 * schedule.
+	 */
+	DBGCP ( uhci, "UHCI %s periodic schedule: ", uhci->name );
+	link = end = uhci_link_qh ( uhci->head );
+	list_for_each_entry_reverse ( endpoint, &uhci->periodic, schedule ) {
+		queue = endpoint->ring.head;
+		queue->link = cpu_to_le32 ( link );
+		wmb();
+		DBGCP ( uhci, "%s%d", ( ( link == end ) ? "" : "<-" ),
+			endpoint->ep->interval );
+		link = uhci_link_qh ( queue );
+	}
+	DBGCP ( uhci, "\n" );
+
+	/* Populate periodic frame list */
+	DBGCP ( uhci, "UHCI %s periodic frame list:", uhci->name );
+	for ( i = 0 ; i < UHCI_FRAMES ; i++ ) {
+
+		/* Calculate maximum interval (in microframes) which
+		 * may appear as part of this frame list.
+		 */
+		if ( i == 0 ) {
+			/* Start of list: include all endpoints */
+			max_interval = -1U;
+		} else {
+			/* Calculate highest power-of-two frame interval */
+			max_interval = ( 1 << ( ffs ( i ) - 1 ) );
+			/* Convert to microframes */
+			max_interval <<= 3;
+			/* Round up to nearest 2^n-1 */
+			max_interval = ( ( max_interval << 1 ) - 1 );
+		}
+
+		/* Find first endpoint in schedule satisfying this
+		 * maximum interval constraint.
+		 */
+		link = uhci_link_qh ( uhci->head );
+		list_for_each_entry ( endpoint, &uhci->periodic, schedule ) {
+			if ( endpoint->ep->interval <= max_interval ) {
+				queue = endpoint->ring.head;
+				link = uhci_link_qh ( queue );
+				DBGCP ( uhci, " %d:%d",
+					i, endpoint->ep->interval );
+				break;
+			}
+		}
+		uhci->frame->link[i] = cpu_to_le32 ( link );
+	}
+	wmb();
+	DBGCP ( uhci, "\n" );
+}
+
+/**
+ * Add endpoint to periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_periodic_add ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+	struct uhci_endpoint *before;
+	unsigned int interval = endpoint->ep->interval;
+
+	/* Find first endpoint with a smaller interval */
+	list_for_each_entry ( before, &uhci->periodic, schedule ) {
+		if ( before->ep->interval < interval )
+			break;
+	}
+	list_add_tail ( &endpoint->schedule, &before->schedule );
+
+	/* Rebuild schedule */
+	uhci_periodic_schedule ( uhci );
+}
+
+/**
+ * Remove endpoint from periodic schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_periodic_del ( struct uhci_endpoint *endpoint ) {
+	struct uhci_device *uhci = endpoint->uhci;
+
+	/* Remove from schedule */
+	list_check_contains_entry ( endpoint, &uhci->periodic, schedule );
+	list_del ( &endpoint->schedule );
+
+	/* Rebuild schedule */
+	uhci_periodic_schedule ( uhci );
+
+	/* Delay for a whole USB frame (with a 100% safety margin) */
+	mdelay ( 2 );
+}
+
+/**
+ * Add endpoint to appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_schedule_add ( struct uhci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		uhci_periodic_add ( endpoint );
+	} else {
+		uhci_async_add ( endpoint );
+	}
+}
+
+/**
+ * Remove endpoint from appropriate schedule
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_schedule_del ( struct uhci_endpoint *endpoint ) {
+	struct usb_endpoint *ep = endpoint->ep;
+	unsigned int attr = ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+
+	if ( attr == USB_ENDPOINT_ATTR_INTERRUPT ) {
+		uhci_periodic_del ( endpoint );
+	} else {
+		uhci_async_del ( endpoint );
+	}
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct uhci_endpoint *endpoint;
+	int rc;
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	endpoint->uhci = uhci;
+	endpoint->ep = ep;
+	usb_endpoint_set_hostdata ( ep, endpoint );
+
+	/* Initialise descriptor ring */
+	if ( ( rc = uhci_ring_alloc ( &endpoint->ring ) ) != 0 )
+		goto err_ring_alloc;
+	endpoint->ring.mtu = ep->mtu;
+	endpoint->ring.flags = UHCI_FL_CERR_MAX;
+	if ( usb->port->speed < USB_SPEED_FULL )
+		endpoint->ring.flags |= UHCI_FL_LS;
+	endpoint->ring.control = ( UHCI_CONTROL_DEVICE ( usb->address ) |
+				   UHCI_CONTROL_ENDPOINT ( ep->address ) );
+
+	/* Add to list of endpoints */
+	list_add_tail ( &endpoint->list, &uhci->endpoints );
+
+	/* Add to schedule */
+	uhci_schedule_add ( endpoint );
+
+	return 0;
+
+	uhci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void uhci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct io_buffer *iobuf;
+
+	/* Remove from schedule */
+	uhci_schedule_del ( endpoint );
+
+	/* Cancel any incomplete transfers */
+	while ( uhci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = uhci_dequeue ( &endpoint->ring );
+		if ( iobuf )
+			usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Remove from list of endpoints */
+	list_del ( &endpoint->list );
+
+	/* Free descriptor ring */
+	uhci_ring_free ( &endpoint->ring );
+
+	/* Free endpoint */
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+
+	/* Restart ring */
+	uhci_restart ( ring, 0 );
+
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_mtu ( struct usb_endpoint *ep ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+
+	/* Update endpoint MTU */
+	endpoint->ring.mtu = ep->mtu;
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+	struct usb_setup_packet *packet;
+	unsigned int count;
+	size_t len;
+	int input;
+	int rc;
+
+	/* Calculate number of descriptors */
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	len = ( iob_len ( iobuf ) - sizeof ( *packet ) );
+	count = ( 1 /* setup stage */ +
+		  ( ( len + ring->mtu - 1 ) / ring->mtu ) /* data stage */ +
+		  1 /* status stage */ );
+
+	/* Enqueue transfer */
+	if ( ( rc = uhci_enqueue ( ring, iobuf, count ) ) != 0 )
+		return rc;
+
+	/* Describe setup stage */
+	packet = iobuf->data;
+	ring->control &= ~UHCI_CONTROL_TOGGLE;
+	uhci_describe ( ring, packet, sizeof ( *packet ), USB_PID_SETUP );
+	iob_pull ( iobuf, sizeof ( *packet ) );
+
+	/* Describe data stage, if applicable */
+	assert ( ring->control & UHCI_CONTROL_TOGGLE );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len ) {
+		uhci_describe ( ring, iobuf->data, len,
+				( input ? USB_PID_IN : USB_PID_OUT ) );
+	}
+
+	/* Describe status stage */
+	ring->control |= UHCI_CONTROL_TOGGLE;
+	uhci_describe ( ring, NULL, 0,
+			( ( len && input ) ? USB_PID_OUT : USB_PID_IN ) );
+
+	/* Sanity check */
+	assert ( ring->end->prod == count );
+
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int uhci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct uhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct uhci_ring *ring = &endpoint->ring;
+	unsigned int count;
+	size_t len;
+	int input;
+	int zlp;
+	int rc;
+
+	/* Calculate number of descriptors */
+	len = iob_len ( iobuf );
+	zlp = ( terminate && ( ( len & ( ring->mtu - 1 ) ) == 0 ) );
+	count = ( ( ( len + ring->mtu - 1 ) / ring->mtu ) + ( zlp ? 1 : 0 ) );
+
+	/* Enqueue transfer */
+	if ( ( rc = uhci_enqueue ( ring, iobuf, count ) ) != 0 )
+		return rc;
+
+	/* Describe data packet */
+	input = ( ep->address & USB_DIR_IN );
+	uhci_describe ( ring, iobuf->data, len,
+			( input ? USB_PID_IN : USB_PID_OUT ) );
+
+	/* Describe zero-length packet, if applicable */
+	if ( zlp )
+		uhci_describe ( ring, NULL, 0, USB_PID_OUT );
+
+	/* Sanity check */
+	assert ( ring->end->prod == count );
+
+	return 0;
+}
+
+/**
+ * Check if transfer is a message transfer
+ *
+ * @v xfer		UHCI transfer
+ * @ret is_message	Transfer is a message transfer
+ */
+static inline int uhci_is_message ( struct uhci_transfer *xfer ) {
+	struct uhci_transfer_descriptor *desc = &xfer->desc[0];
+
+	return ( ( desc->control & cpu_to_le32 ( UHCI_CONTROL_PID_MASK ) ) ==
+		 cpu_to_le32 ( UHCI_CONTROL_PID ( USB_PID_SETUP ) ) );
+}
+
+/**
+ * Poll for completions
+ *
+ * @v endpoint		Endpoint
+ */
+static void uhci_endpoint_poll ( struct uhci_endpoint *endpoint ) {
+	struct uhci_ring *ring = &endpoint->ring;
+	struct uhci_device *uhci = endpoint->uhci;
+	struct usb_endpoint *ep = endpoint->ep;
+	struct usb_device *usb = ep->usb;
+	struct uhci_transfer *xfer;
+	struct uhci_transfer_descriptor *desc;
+	struct io_buffer *iobuf;
+	unsigned int index;
+	uint32_t link;
+	uint32_t toggle;
+	uint32_t control;
+	uint16_t actual;
+	size_t len;
+
+	/* Consume all completed descriptors */
+	while ( uhci_ring_fill ( ring ) ) {
+
+		/* Stop if we reach an uncompleted descriptor */
+		index = ( ring->cons % UHCI_RING_COUNT );
+		xfer = ring->xfer[index];
+		assert ( xfer != NULL );
+		assert ( xfer->cons < xfer->prod );
+		desc = &xfer->desc[xfer->cons];
+		rmb();
+		if ( desc->status & UHCI_STATUS_ACTIVE )
+			break;
+		control = le32_to_cpu ( desc->control );
+		actual = le16_to_cpu ( desc->actual );
+
+		/* Update data length, if applicable */
+		if ( UHCI_DATA_PACKET ( control ) )
+			xfer->len += UHCI_ACTUAL_LEN ( actual );
+
+		/* If we have encountered an error, then deactivate
+		 * the queue head (to prevent further hardware
+		 * accesses to this transfer), consume the transfer,
+		 * and report the error to the USB core.
+		 */
+		if ( desc->status & UHCI_STATUS_STALLED ) {
+			DBGC ( uhci, "UHCI %s %s completion %d.%d failed "
+			       "(status %02x)\n", usb->name,
+			       usb_endpoint_name ( ep ), index,
+			       xfer->cons, desc->status );
+			link = UHCI_LINK_TERMINATE;
+			ring->head->current = cpu_to_le32 ( link );
+			wmb();
+			iobuf = uhci_dequeue ( ring );
+			usb_complete_err ( ep, iobuf, -EIO );
+			break;
+		}
+
+		/* Consume this descriptor */
+		xfer->cons++;
+
+		/* Check for short packets */
+		if ( UHCI_SHORT_PACKET ( control, actual ) ) {
+
+			/* Sanity checks */
+			assert ( desc->flags & UHCI_FL_SPD );
+			link = virt_to_phys ( desc );
+			assert ( ( le32_to_cpu ( ring->head->current ) &
+				   ~( UHCI_ALIGN - 1 ) ) == link );
+
+			/* If this is a message transfer, then restart
+			 * at the status stage.
+			 */
+			if ( uhci_is_message ( xfer ) ) {
+				xfer->cons = ( xfer->prod - 1 );
+				link = virt_to_phys ( &xfer->desc[xfer->cons] );
+				ring->head->current = cpu_to_le32 ( link );
+				break;
+			}
+
+			/* Otherwise, this is a stream transfer.
+			 * First, prevent further hardware access to
+			 * this transfer.
+			 */
+			link = UHCI_LINK_TERMINATE;
+			ring->head->current = cpu_to_le32 ( link );
+			wmb();
+
+			/* Determine expected data toggle for next descriptor */
+			toggle = ( ( control ^ UHCI_CONTROL_TOGGLE ) &
+				   UHCI_CONTROL_TOGGLE );
+
+			/* Consume this transfer */
+			len = xfer->len;
+			iobuf = uhci_dequeue ( ring );
+
+			/* Update packet length */
+			assert ( len <= iob_len ( iobuf ) );
+			iob_unput ( iobuf, ( iob_len ( iobuf ) - len ) );
+
+			/* Restart ring */
+			uhci_restart ( ring, toggle );
+
+		} else if ( xfer->cons == xfer->prod ) {
+
+			/* Completed a transfer: consume it */
+			len = xfer->len;
+			iobuf = uhci_dequeue ( ring );
+			assert ( len == iob_len ( iobuf ) );
+
+		} else {
+
+			/* Not a short packet and not yet complete:
+			 * continue processing.
+			 */
+			continue;
+		}
+
+		/* Report completion to USB core */
+		usb_complete ( ep, iobuf );
+	}
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int uhci_device_open ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( usb->port->hub->bus );
+
+	usb_set_hostdata ( usb, uhci );
+	return 0;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void uhci_device_close ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = uhci->bus;
+
+	/* Free device address, if assigned */
+	if ( usb->address )
+		usb_free_address ( bus, usb->address );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int uhci_device_address ( struct usb_device *usb ) {
+	struct uhci_device *uhci = usb_get_hostdata ( usb );
+	struct usb_bus *bus = uhci->bus;
+	struct usb_endpoint *ep0 = usb_endpoint ( usb, USB_EP0_ADDRESS );
+	struct uhci_endpoint *endpoint0 = usb_endpoint_get_hostdata ( ep0 );
+	int address;
+	int rc;
+
+	/* Sanity checks */
+	assert ( usb->address == 0 );
+	assert ( ep0 != NULL );
+
+	/* Allocate device address */
+	address = usb_alloc_address ( bus );
+	if ( address < 0 ) {
+		rc = address;
+		DBGC ( uhci, "UHCI %s could not allocate address: %s\n",
+		       usb->name, strerror ( rc ) );
+		goto err_alloc_address;
+	}
+
+	/* Set address */
+	if ( ( rc = usb_set_address ( usb, address ) ) != 0 )
+		goto err_set_address;
+
+	/* Update device address */
+	usb->address = address;
+	endpoint0->ring.control |= UHCI_CONTROL_DEVICE ( address );
+
+	return 0;
+
+ err_set_address:
+	usb_free_address ( bus, address );
+ err_alloc_address:
+	return rc;
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int uhci_hub_open ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void uhci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int uhci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, uhci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void uhci_root_close ( struct usb_hub *hub ) {
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+	unsigned int i;
+
+	/* Reset port */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	portsc |= UHCI_PORTSC_PR;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_DELAY_MS );
+	portsc &= ~UHCI_PORTSC_PR;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Enable port */
+	portsc |= UHCI_PORTSC_PED;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+	mdelay ( USB_RESET_RECOVER_DELAY_MS );
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < UHCI_PORT_ENABLE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+		if ( portsc & UHCI_PORTSC_PED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( uhci, "UHCI %s-%d timed out waiting for port to enable "
+	       "(status %04x)\n",  uhci->name, port->address, portsc );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+
+	/* Disable port */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	portsc &= ~UHCI_PORTSC_PED;
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int uhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	struct pci_device pci;
+	uint16_t portsc;
+	unsigned int speed;
+
+	/* Read port status */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	if ( ! ( portsc & UHCI_PORTSC_CCS ) ) {
+		/* Port not connected */
+		speed = USB_SPEED_NONE;
+	} else if ( uhci->companion &&
+		    ! find_usb_bus_by_location ( BUS_TYPE_PCI,
+						 uhci->companion ) ) {
+		/* Defer connection detection until companion
+		 * controller has been enumerated.
+		 */
+		pci_init ( &pci, uhci->companion );
+		DBGC ( uhci, "UHCI %s-%d deferring for companion " PCI_FMT "\n",
+		       uhci->name, port->address, PCI_ARGS ( &pci ) );
+		speed = USB_SPEED_NONE;
+	} else if ( portsc & UHCI_PORTSC_LS ) {
+		/* Low-speed device */
+		speed = USB_SPEED_LOW;
+	} else {
+		/* Full-speed device */
+		speed = USB_SPEED_FULL;
+	}
+	port->speed = speed;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & UHCI_PORTSC_CSC );
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int uhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( uhci, "UHCI %s-%d nonsensical CLEAR_TT for %s %s\n", uhci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/**
+ * Poll for port status changes
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ */
+static void uhci_root_poll ( struct usb_hub *hub, struct usb_port *port ) {
+	struct uhci_device *uhci = usb_hub_get_drvdata ( hub );
+	uint16_t portsc;
+	uint16_t change;
+
+	/* Do nothing unless something has changed */
+	portsc = inw ( uhci->regs + UHCI_PORTSC ( port->address ) );
+	change = ( portsc & UHCI_PORTSC_CHANGE );
+	if ( ! change )
+		return;
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= ( portsc & UHCI_PORTSC_CSC );
+	outw ( portsc, uhci->regs + UHCI_PORTSC ( port->address ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int uhci_bus_open ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+	int rc;
+
+	/* Sanity checks */
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+
+	/* Allocate and initialise asynchronous queue head */
+	uhci->head = malloc_dma ( sizeof ( *uhci->head ), UHCI_ALIGN );
+	if ( ! uhci->head ) {
+		rc = -ENOMEM;
+		goto err_alloc_head;
+	}
+	if ( ( rc = uhci_reachable ( uhci->head, sizeof ( *uhci->head ) ) ) !=0)
+		goto err_unreachable_head;
+	memset ( uhci->head, 0, sizeof ( *uhci->head ) );
+	uhci->head->current = cpu_to_le32 ( UHCI_LINK_TERMINATE );
+	uhci_async_schedule ( uhci );
+
+	/* Allocate periodic frame list */
+	uhci->frame = malloc_dma ( sizeof ( *uhci->frame ),
+				   sizeof ( *uhci->frame ) );
+	if ( ! uhci->frame ) {
+		rc = -ENOMEM;
+		goto err_alloc_frame;
+	}
+	if ( ( rc = uhci_reachable ( uhci->frame,
+				     sizeof ( *uhci->frame ) ) ) != 0 )
+		goto err_unreachable_frame;
+	uhci_periodic_schedule ( uhci );
+	outl ( virt_to_phys ( uhci->frame ), uhci->regs + UHCI_FLBASEADD );
+
+	/* Start controller */
+	uhci_run ( uhci );
+
+	return 0;
+
+	uhci_stop ( uhci );
+ err_unreachable_frame:
+	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+ err_alloc_frame:
+ err_unreachable_head:
+	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+ err_alloc_head:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void uhci_bus_close ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+
+	/* Sanity checks */
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+
+	/* Stop controller */
+	uhci_stop ( uhci );
+
+	/* Free periodic frame list */
+	free_dma ( uhci->frame, sizeof ( *uhci->frame ) );
+
+	/* Free asynchronous schedule */
+	free_dma ( uhci->head, sizeof ( *uhci->head ) );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void uhci_bus_poll ( struct usb_bus *bus ) {
+	struct uhci_device *uhci = usb_bus_get_hostdata ( bus );
+	struct usb_hub *hub = bus->hub;
+	struct uhci_endpoint *endpoint;
+	unsigned int i;
+
+	/* UHCI defers interrupts (including short packet detection)
+	 * until the end of the frame.  This can result in bulk IN
+	 * endpoints remaining halted for much of the time, waiting
+	 * for software action to reset the data toggles.  We
+	 * therefore ignore USBSTS and unconditionally poll all
+	 * endpoints for completed transfer descriptors.
+	 *
+	 * As with EHCI, we trust that completion handlers are minimal
+	 * and will not do anything that could plausibly affect the
+	 * endpoint list itself.
+	 */
+	list_for_each_entry ( endpoint, &uhci->endpoints, list )
+		uhci_endpoint_poll ( endpoint );
+
+	/* UHCI provides no single bit to indicate that a port status
+	 * change has occurred.  We therefore unconditionally iterate
+	 * over all ports looking for status changes.
+	 */
+	for ( i = 1 ; i <= UHCI_PORTS ; i++ )
+		uhci_root_poll ( hub, usb_port ( hub, i ) );
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations uhci_operations = {
+	.endpoint = {
+		.open = uhci_endpoint_open,
+		.close = uhci_endpoint_close,
+		.reset = uhci_endpoint_reset,
+		.mtu = uhci_endpoint_mtu,
+		.message = uhci_endpoint_message,
+		.stream = uhci_endpoint_stream,
+	},
+	.device = {
+		.open = uhci_device_open,
+		.close = uhci_device_close,
+		.address = uhci_device_address,
+	},
+	.bus = {
+		.open = uhci_bus_open,
+		.close = uhci_bus_close,
+		.poll = uhci_bus_poll,
+	},
+	.hub = {
+		.open = uhci_hub_open,
+		.close = uhci_hub_close,
+	},
+	.root = {
+		.open = uhci_root_open,
+		.close = uhci_root_close,
+		.enable = uhci_root_enable,
+		.disable = uhci_root_disable,
+		.speed = uhci_root_speed,
+		.clear_tt = uhci_root_clear_tt,
+	},
+};
+
+/**
+ * Locate EHCI companion controller (when no EHCI support is present)
+ *
+ * @v pci		PCI device
+ * @ret busdevfn	EHCI companion controller bus:dev.fn (if any)
+ */
+__weak unsigned int ehci_companion ( struct pci_device *pci __unused ) {
+	return 0;
+}
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int uhci_probe ( struct pci_device *pci ) {
+	struct uhci_device *uhci;
+	struct usb_port *port;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	uhci = zalloc ( sizeof ( *uhci ) );
+	if ( ! uhci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	uhci->name = pci->dev.name;
+	INIT_LIST_HEAD ( &uhci->endpoints );
+	INIT_LIST_HEAD ( &uhci->async );
+	INIT_LIST_HEAD ( &uhci->periodic );
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Identify EHCI companion controller, if any */
+	uhci->companion = ehci_companion ( pci );
+
+	/* Claim ownership from BIOS.  (There is no release mechanism
+	 * for UHCI.)
+	 */
+	pci_write_config_word ( pci, UHCI_USBLEGSUP, UHCI_USBLEGSUP_DEFAULT );
+
+	/* Map registers */
+	uhci->regs = pci->ioaddr;
+	if ( ! uhci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Reset device */
+	if ( ( rc = uhci_reset ( uhci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	uhci->bus = alloc_usb_bus ( &pci->dev, UHCI_PORTS, UHCI_MTU,
+				    &uhci_operations );
+	if ( ! uhci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( uhci->bus, uhci );
+	usb_hub_set_drvdata ( uhci->bus->hub, uhci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= UHCI_PORTS ; i++ ) {
+		port = usb_port ( uhci->bus->hub, i );
+		port->protocol = USB_PROTO_2_0;
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( uhci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, uhci );
+	return 0;
+
+	unregister_usb_bus ( uhci->bus );
+ err_register:
+	free_usb_bus ( uhci->bus );
+ err_alloc_bus:
+	uhci_reset ( uhci );
+ err_reset:
+ err_ioremap:
+	free ( uhci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void uhci_remove ( struct pci_device *pci ) {
+	struct uhci_device *uhci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = uhci->bus;
+
+	unregister_usb_bus ( bus );
+	assert ( list_empty ( &uhci->async ) );
+	assert ( list_empty ( &uhci->periodic ) );
+	free_usb_bus ( bus );
+	uhci_reset ( uhci );
+	free ( uhci );
+}
+
+/** UHCI PCI device IDs */
+static struct pci_device_id uhci_ids[] = {
+	PCI_ROM ( 0xffff, 0xffff, "uhci", "UHCI", 0 ),
+};
+
+/** UHCI PCI driver */
+struct pci_driver uhci_driver __pci_driver = {
+	.ids = uhci_ids,
+	.id_count = ( sizeof ( uhci_ids ) / sizeof ( uhci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_UHCI ),
+	.probe = uhci_probe,
+	.remove = uhci_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/usb/uhci.h b/qemu/roms/ipxe/src/drivers/usb/uhci.h
new file mode 100644
index 000000000..ba4c28f7e
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/uhci.h
@@ -0,0 +1,350 @@
+#ifndef _IPXE_UHCI_H
+#define _IPXE_UHCI_H
+
+/** @file
+ *
+ * USB Universal Host Controller Interface (UHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the frame list (which is page-aligned), data
+ * structures used by UHCI generally require 16-byte alignment.
+ */
+#define UHCI_ALIGN 16
+
+/** Number of ports */
+#define UHCI_PORTS 2
+
+/** Maximum transfer size */
+#define UHCI_MTU 1280
+
+/** I/O BAR size */
+#define UHCI_BAR_SIZE 0x14
+
+/** USB command register */
+#define UHCI_USBCMD 0x00
+
+/** Max packet is 64 bytes */
+#define UHCI_USBCMD_MAX64 0x0080
+
+/** Host controller reset */
+#define UHCI_USBCMD_HCRESET 0x0002
+
+/** Run/stop */
+#define UHCI_USBCMD_RUN 0x0001
+
+/** USB status register */
+#define UHCI_USBSTS 0x02
+
+/** Host controller halted */
+#define UHCI_USBSTS_HCHALTED 0x0020
+
+/** USB interrupt */
+#define UHCI_USBSTS_USBINT 0x0001
+
+/** Frame list base address register */
+#define UHCI_FLBASEADD 0x08
+
+/** Port status and control register */
+#define UHCI_PORTSC(port) ( 0x0e + ( (port) << 1 ) )
+
+/** Port reset */
+#define UHCI_PORTSC_PR 0x0200
+
+/** Low-speed device attached */
+#define UHCI_PORTSC_LS 0x0100
+
+/** Port enabled/disabled change */
+#define UHCI_PORTSC_PEC 0x0008
+
+/** Port enabled */
+#define UHCI_PORTSC_PED 0x0004
+
+/** Connect status change */
+#define UHCI_PORTSC_CSC 0x0002
+
+/** Current connect status */
+#define UHCI_PORTSC_CCS 0x0001
+
+/** Port status change mask */
+#define UHCI_PORTSC_CHANGE ( UHCI_PORTSC_CSC | UHCI_PORTSC_PEC )
+
+/** Depth-first processing */
+#define UHCI_LINK_DEPTH_FIRST 0x00000004UL
+
+/** Queue head type */
+#define UHCI_LINK_TYPE_QH 0x00000002UL
+
+/** List terminator */
+#define UHCI_LINK_TERMINATE 0x00000001UL
+
+/** Number of frames in frame list */
+#define UHCI_FRAMES 1024
+
+/** A frame list */
+struct uhci_frame_list {
+	/** Link pointer */
+	uint32_t link[UHCI_FRAMES];
+} __attribute__ (( packed ));
+
+/** A transfer descriptor */
+struct uhci_transfer_descriptor {
+	/** Link pointer */
+	uint32_t link;
+	/** Actual length */
+	uint16_t actual;
+	/** Status */
+	uint8_t status;
+	/** Flags */
+	uint8_t flags;
+	/** Control */
+	uint32_t control;
+	/** Buffer pointer */
+	uint32_t data;
+} __attribute__ (( packed ));
+
+/** Length mask */
+#define UHCI_LEN_MASK 0x7ff
+
+/** Actual length */
+#define UHCI_ACTUAL_LEN( actual ) ( ( (actual) + 1 ) & UHCI_LEN_MASK )
+
+/** Active */
+#define UHCI_STATUS_ACTIVE 0x80
+
+/** Stalled */
+#define UHCI_STATUS_STALLED 0x40
+
+/** Data buffer error */
+#define UHCI_STATUS_BUFFER 0x20
+
+/** Babble detected */
+#define UHCI_STATUS_BABBLE 0x10
+
+/** NAK received */
+#define UHCI_STATUS_NAK 0x08
+
+/** CRC/timeout error */
+#define UHCI_STATUS_CRC_TIMEOUT 0x04
+
+/** Bitstuff error */
+#define UHCI_STATUS_BITSTUFF 0x02
+
+/** Short packet detect */
+#define UHCI_FL_SPD 0x20
+
+/** Error counter */
+#define UHCI_FL_CERR( count ) ( (count) << 3 )
+
+/** Error counter maximum value */
+#define UHCI_FL_CERR_MAX UHCI_FL_CERR ( 3 )
+
+/** Low speed device */
+#define UHCI_FL_LS 0x04
+
+/** Interrupt on completion */
+#define UHCI_FL_IOC 0x01
+
+/** Packet ID */
+#define UHCI_CONTROL_PID( pid ) ( (pid) << 0 )
+
+/** Packet ID mask */
+#define UHCI_CONTROL_PID_MASK UHCI_CONTROL_PID ( 0xff )
+
+/** Device address */
+#define UHCI_CONTROL_DEVICE( address ) ( (address) << 8 )
+
+/** Endpoint address */
+#define UHCI_CONTROL_ENDPOINT( address ) ( (address) << 15 )
+
+/** Data toggle */
+#define UHCI_CONTROL_TOGGLE ( 1 << 19 )
+
+/** Data length */
+#define UHCI_CONTROL_LEN( len ) ( ( ( (len) - 1 ) & UHCI_LEN_MASK ) << 21 )
+
+/** Check for data packet
+ *
+ * This check is based on the fact that only USB_PID_SETUP has bit 2
+ * set.
+ */
+#define UHCI_DATA_PACKET( control ) ( ! ( control & 0x04 ) )
+
+/** Check for short packet */
+#define UHCI_SHORT_PACKET( control, actual ) \
+	( ( ( (control) >> 21 ) ^ (actual) ) & UHCI_LEN_MASK )
+
+/** USB legacy support register (in PCI configuration space) */
+#define UHCI_USBLEGSUP 0xc0
+
+/** USB legacy support default value */
+#define UHCI_USBLEGSUP_DEFAULT 0x2000
+
+/** A queue head */
+struct uhci_queue_head {
+	/** Horizontal link pointer */
+	uint32_t link;
+	/** Current transfer descriptor */
+	uint32_t current;
+} __attribute__ (( packed ));
+
+/** A single UHCI transfer
+ *
+ * UHCI hardware is extremely simple, and requires software to build
+ * the entire packet schedule (including manually handling all of the
+ * data toggles).  The hardware requires at least 16 bytes of transfer
+ * descriptors per 64 bytes of transmitted/received data.  We allocate
+ * the transfer descriptors at the time that the transfer is enqueued,
+ * to avoid the need to allocate unreasonably large blocks when the
+ * endpoint is opened.
+ */
+struct uhci_transfer {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+	/** Completed data length */
+	size_t len;
+
+	/** Transfer descriptors */
+	struct uhci_transfer_descriptor *desc;
+
+	/** I/O buffer */
+	struct io_buffer *iobuf;
+};
+
+/** Number of transfer descriptors in a ring
+ *
+ * This is a policy decision.
+ */
+#define UHCI_RING_COUNT 16
+
+/** A transfer ring */
+struct uhci_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+
+	/** Maximum packet length */
+	size_t mtu;
+	/** Base flags
+	 *
+	 * This incorporates the CERR and LS bits
+	 */
+	uint8_t flags;
+	/** Base control word
+	 *
+	 * This incorporates the device address, the endpoint address,
+	 * and the data toggle for the next descriptor to be enqueued.
+	 */
+	uint32_t control;
+
+	/** Transfers */
+	struct uhci_transfer *xfer[UHCI_RING_COUNT];
+	/** End of transfer ring (if non-empty) */
+	struct uhci_transfer *end;
+
+	/** Queue head */
+	struct uhci_queue_head *head;
+};
+
+/**
+ * Calculate space used in transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+uhci_ring_fill ( struct uhci_ring *ring ) {
+	unsigned int fill;
+
+	fill = ( ring->prod - ring->cons );
+	assert ( fill <= UHCI_RING_COUNT );
+	return fill;
+}
+
+/**
+ * Calculate space remaining in transfer ring
+ *
+ * @v ring		Transfer ring
+ * @ret remaining	Number of entries remaining
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+uhci_ring_remaining ( struct uhci_ring *ring ) {
+	unsigned int fill = uhci_ring_fill ( ring );
+
+	return ( UHCI_RING_COUNT - fill );
+}
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define UHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define UHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a port to be enabled
+ *
+ * This is a policy decision.
+ */
+#define UHCI_PORT_ENABLE_MAX_WAIT_MS 500
+
+/** A UHCI device */
+struct uhci_device {
+	/** Registers */
+	unsigned long regs;
+	/** Name */
+	const char *name;
+
+	/** EHCI companion controller bus:dev.fn address (if any) */
+	unsigned int companion;
+
+	/** Asynchronous queue head */
+	struct uhci_queue_head *head;
+	/** Frame list */
+	struct uhci_frame_list *frame;
+
+	/** List of all endpoints */
+	struct list_head endpoints;
+	/** Asynchronous schedule */
+	struct list_head async;
+	/** Periodic schedule
+	 *
+	 * Listed in decreasing order of endpoint interval.
+	 */
+	struct list_head periodic;
+
+	/** USB bus */
+	struct usb_bus *bus;
+};
+
+/** A UHCI endpoint */
+struct uhci_endpoint {
+	/** UHCI device */
+	struct uhci_device *uhci;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** List of all endpoints */
+	struct list_head list;
+	/** Endpoint schedule */
+	struct list_head schedule;
+
+	/** Transfer ring */
+	struct uhci_ring ring;
+};
+
+#endif /* _IPXE_UHCI_H */
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbhid.c b/qemu/roms/ipxe/src/drivers/usb/usbhid.c
new file mode 100644
index 000000000..c74535a05
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbhid.c
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbhid.h>
+
+/** @file
+ *
+ * USB human interface devices (HID)
+ *
+ */
+
+/**
+ * Open USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @ret rc		Return status code
+ */
+int usbhid_open ( struct usb_hid *hid ) {
+	int rc;
+
+	/* Open interrupt IN endpoint */
+	if ( ( rc = usb_endpoint_open ( &hid->in ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not open interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_open_in;
+	}
+
+	/* Refill interrupt IN endpoint */
+	if ( ( rc = usb_refill ( &hid->in ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not refill interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_refill_in;
+	}
+
+	/* Open interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb &&
+	     ( ( rc = usb_endpoint_open ( &hid->out ) ) != 0 ) ) {
+		DBGC ( hid, "HID %s could not open interrupt OUT: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		goto err_open_out;
+	}
+
+	return 0;
+
+	usb_endpoint_close ( &hid->out );
+ err_open_out:
+ err_refill_in:
+	usb_endpoint_close ( &hid->in );
+ err_open_in:
+	return rc;
+}
+
+/**
+ * Close USB human interface device
+ *
+ * @v hid		USB human interface device
+ */
+void usbhid_close ( struct usb_hid *hid ) {
+
+	/* Close interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb )
+		usb_endpoint_close ( &hid->out );
+
+	/* Close interrupt IN endpoint */
+	usb_endpoint_close ( &hid->in );
+}
+
+/**
+ * Refill USB human interface device endpoints
+ *
+ * @v hid		USB human interface device
+ * @ret rc		Return status code
+ */
+int usbhid_refill ( struct usb_hid *hid ) {
+	int rc;
+
+	/* Refill interrupt IN endpoint */
+	if ( ( rc = usb_refill ( &hid->in ) ) != 0 )
+		return rc;
+
+	/* Refill interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb && ( ( rc = usb_refill ( &hid->out ) ) != 0 ) )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Describe USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+int usbhid_describe ( struct usb_hid *hid,
+		      struct usb_configuration_descriptor *config ) {
+	struct usb_interface_descriptor *desc;
+	int rc;
+
+	/* Locate interface descriptor */
+	desc = usb_interface_descriptor ( config, hid->func->interface[0], 0 );
+	if ( ! desc ) {
+		DBGC ( hid, "HID %s has no interface descriptor\n",
+		       hid->func->name );
+		return -EINVAL;
+	}
+
+	/* Describe interrupt IN endpoint */
+	if ( ( rc = usb_endpoint_described ( &hid->in, config, desc,
+					     USB_INTERRUPT_IN, 0 ) ) != 0 ) {
+		DBGC ( hid, "HID %s could not describe interrupt IN: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Describe interrupt OUT endpoint, if applicable */
+	if ( hid->out.usb &&
+	     ( ( rc = usb_endpoint_described ( &hid->out, config, desc,
+					       USB_INTERRUPT_OUT, 0 ) ) != 0 )){
+		DBGC ( hid, "HID %s could not describe interrupt OUT: %s\n",
+		       hid->func->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbhub.c b/qemu/roms/ipxe/src/drivers/usb/usbhub.c
new file mode 100644
index 000000000..bf2a20005
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbhub.c
@@ -0,0 +1,547 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/usb.h>
+#include "usbhub.h"
+
+/** @file
+ *
+ * USB hub driver
+ *
+ */
+
+/**
+ * Refill interrupt ring
+ *
+ * @v hubdev		Hub device
+ */
+static void hub_refill ( struct usb_hub_device *hubdev ) {
+	int rc;
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &hubdev->intr ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not refill interrupt: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		/* Continue attempting to refill */
+		return;
+	}
+
+	/* Stop refill process */
+	process_del ( &hubdev->refill );
+}
+
+/** Refill process descriptor */
+static struct process_descriptor hub_refill_desc =
+	PROC_DESC ( struct usb_hub_device, refill, hub_refill );
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void hub_complete ( struct usb_endpoint *ep,
+			   struct io_buffer *iobuf, int rc ) {
+	struct usb_hub_device *hubdev =
+		container_of ( ep, struct usb_hub_device, intr );
+	struct usb_hub *hub = hubdev->hub;
+	uint8_t *data = iobuf->data;
+	unsigned int bits = ( 8 * iob_len ( iobuf ) );
+	unsigned int i;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto done;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( hubdev, "HUB %s interrupt failed: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		DBGC_HDA ( hubdev, 0, iobuf->data, iob_len ( iobuf ) );
+		goto done;
+	}
+
+	/* Report any port status changes */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+
+		/* Sanity check */
+		if ( i > bits ) {
+			DBGC ( hubdev, "HUB %s underlength interrupt:\n",
+			       hubdev->name );
+			DBGC_HDA ( hubdev, 0, iobuf->data, iob_len ( iobuf ) );
+			goto done;
+		}
+
+		/* Report port status change if applicable */
+		if ( data[ i / 8 ] & ( 1 << ( i % 8 ) ) ) {
+			DBGC2 ( hubdev, "HUB %s port %d status changed\n",
+				hubdev->name, i );
+			usb_port_changed ( usb_port ( hub, i ) );
+		}
+	}
+
+ done:
+	/* Start refill process */
+	process_add ( &hubdev->refill );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations usb_hub_intr_operations = {
+	.complete = hub_complete,
+};
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int hub_open ( struct usb_hub *hub ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	unsigned int i;
+	int rc;
+
+	/* Ensure ports are powered */
+	for ( i = 1 ; i <= hub->ports ; i++ ) {
+		if ( ( rc = usb_hub_set_port_feature ( usb, i,
+						       USB_HUB_PORT_POWER,
+						       0 ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not apply power: "
+			       "%s\n", hubdev->name, i, strerror ( rc ) );
+			goto err_power;
+		}
+	}
+
+	/* Open interrupt endpoint */
+	if ( ( rc = usb_endpoint_open ( &hubdev->intr ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not register interrupt: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Start refill process */
+	process_add ( &hubdev->refill );
+
+	/* Refill interrupt ring */
+	hub_refill ( hubdev );
+
+	return 0;
+
+	usb_endpoint_close ( &hubdev->intr );
+ err_open:
+ err_power:
+	return rc;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void hub_close ( struct usb_hub *hub ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+
+	/* Close interrupt endpoint */
+	usb_endpoint_close ( &hubdev->intr );
+
+	/* Stop refill process */
+	process_del ( &hubdev->refill );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	struct usb_hub_port_status status;
+	unsigned int current;
+	unsigned int i;
+	int rc;
+
+	/* Initiate reset if applicable */
+	if ( ( hub->protocol < USB_PROTO_3_0 ) &&
+	     ( ( rc = usb_hub_set_port_feature ( usb, port->address,
+						 USB_HUB_PORT_RESET, 0 ) )!=0)){
+		DBGC ( hubdev, "HUB %s port %d could not initiate reset: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < USB_HUB_ENABLE_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for port being enabled */
+		if ( ( rc = usb_hub_get_port_status ( usb, port->address,
+						      &status ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not get status: "
+			       "%s\n", hubdev->name, port->address,
+			       strerror ( rc ) );
+			return rc;
+		}
+		current = le16_to_cpu ( status.current );
+		if ( current & ( 1 << USB_HUB_PORT_ENABLE ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( hubdev, "HUB %s port %d timed out waiting for enable\n",
+	       hubdev->name, port->address );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	int rc;
+
+	/* Disable port */
+	if ( ( rc = usb_hub_clear_port_feature ( usb, port->address,
+						 USB_HUB_PORT_ENABLE, 0 ) )!=0){
+		DBGC ( hubdev, "HUB %s port %d could not disable: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Clear port status change bits
+ *
+ * @v hubdev		USB hub device
+ * @v port		Port number
+ * @v changed		Port status change bits
+ * @ret rc		Return status code
+ */
+static int hub_clear_changes ( struct usb_hub_device *hubdev,
+			       unsigned int port, uint16_t changed ) {
+	struct usb_device *usb = hubdev->usb;
+	unsigned int bit;
+	unsigned int feature;
+	int rc;
+
+	/* Clear each set bit */
+	for ( bit = 0 ; bit < 16 ; bit++ ) {
+
+		/* Skip unset bits */
+		if ( ! ( changed & ( 1 << bit ) ) )
+			continue;
+
+		/* Skip unused features */
+		feature = USB_HUB_C_FEATURE ( bit );
+		if ( ! ( hubdev->features & ( 1 << feature ) ) )
+			continue;
+
+		/* Clear bit */
+		if ( ( rc = usb_hub_clear_port_feature ( usb, port,
+							 feature, 0 ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s port %d could not clear feature "
+			       "%d: %s\n", hubdev->name, port, feature,
+			       strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Update port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int hub_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	struct usb_hub_port_status status;
+	unsigned int current;
+	unsigned int changed;
+	int rc;
+
+	/* Get port status */
+	if ( ( rc = usb_hub_get_port_status ( usb, port->address,
+					      &status ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s port %d could not get status: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+	current = le16_to_cpu ( status.current );
+	changed = le16_to_cpu ( status.changed );
+	DBGC2 ( hubdev, "HUB %s port %d status is %04x:%04x\n",
+		hubdev->name, port->address, changed, current );
+
+	/* Update port speed */
+	if ( current & ( 1 << USB_HUB_PORT_CONNECTION ) ) {
+		if ( hub->protocol >= USB_PROTO_3_0 ) {
+			port->speed = USB_SPEED_SUPER;
+		} else if ( current & ( 1 << USB_HUB_PORT_LOW_SPEED ) ) {
+			port->speed = USB_SPEED_LOW;
+		} else if ( current & ( 1 << USB_HUB_PORT_HIGH_SPEED ) ) {
+			port->speed = USB_SPEED_HIGH;
+		} else {
+			port->speed = USB_SPEED_FULL;
+		}
+	} else {
+		port->speed = USB_SPEED_NONE;
+	}
+
+	/* Record disconnections */
+	port->disconnected |= ( changed & ( 1 << USB_HUB_PORT_CONNECTION ) );
+
+	/* Clear port status change bits */
+	if ( ( rc = hub_clear_changes ( hubdev, port->address, changed ) ) != 0)
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int hub_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+			  struct usb_endpoint *ep ) {
+	struct usb_hub_device *hubdev = usb_hub_get_drvdata ( hub );
+	struct usb_device *usb = hubdev->usb;
+	int rc;
+
+	/* Clear transaction translator buffer.  All hubs must support
+	 * single-TT operation; we simplify our code by supporting
+	 * only this configuration.
+	 */
+	if ( ( rc = usb_hub_clear_tt_buffer ( usb, ep->usb->address,
+					      ep->address, ep->attributes,
+					      USB_HUB_TT_SINGLE ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s port %d could not clear TT buffer: %s\n",
+		       hubdev->name, port->address, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/** USB hub operations */
+static struct usb_hub_driver_operations hub_operations = {
+	.open = hub_open,
+	.close = hub_close,
+	.enable = hub_enable,
+	.disable = hub_disable,
+	.speed = hub_speed,
+	.clear_tt = hub_clear_tt,
+};
+
+/**
+ * Probe USB hub
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int hub_probe ( struct usb_function *func,
+		       struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_bus *bus = usb->port->hub->bus;
+	struct usb_hub_device *hubdev;
+	struct usb_interface_descriptor *interface;
+	union usb_hub_descriptor desc;
+	unsigned int depth;
+	unsigned int ports;
+	int enhanced;
+	int rc;
+
+	/* Allocate and initialise structure */
+	hubdev = zalloc ( sizeof ( *hubdev ) );
+	if ( ! hubdev ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	enhanced = ( usb->port->protocol >= USB_PROTO_3_0 );
+	hubdev->name = func->name;
+	hubdev->usb = usb;
+	hubdev->features =
+		( enhanced ? USB_HUB_FEATURES_ENHANCED : USB_HUB_FEATURES );
+	usb_endpoint_init ( &hubdev->intr, usb, &usb_hub_intr_operations );
+	usb_refill_init ( &hubdev->intr, 0, USB_HUB_INTR_FILL );
+	process_init_stopped ( &hubdev->refill, &hub_refill_desc, NULL );
+
+	/* Locate hub interface descriptor */
+	interface = usb_interface_descriptor ( config, func->interface[0], 0 );
+	if ( ! interface ) {
+		DBGC ( hubdev, "HUB %s has no interface descriptor\n",
+		       hubdev->name );
+		rc = -EINVAL;
+		goto err_interface;
+	}
+
+	/* Locate interrupt endpoint descriptor */
+	if ( ( rc = usb_endpoint_described ( &hubdev->intr, config, interface,
+					     USB_INTERRUPT_IN, 0 ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not describe interrupt endpoint: "
+		       "%s\n", hubdev->name, strerror ( rc ) );
+		goto err_endpoint;
+	}
+
+	/* Set hub depth */
+	depth = usb_depth ( usb );
+	if ( enhanced ) {
+		if ( ( rc = usb_hub_set_hub_depth ( usb, depth ) ) != 0 ) {
+			DBGC ( hubdev, "HUB %s could not set hub depth to %d: "
+			       "%s\n", hubdev->name, depth, strerror ( rc ) );
+			goto err_set_hub_depth;
+		}
+	}
+
+	/* Get hub descriptor */
+	if ( ( rc = usb_hub_get_descriptor ( usb, enhanced, &desc ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not get hub descriptor: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_hub_descriptor;
+	}
+	ports = desc.basic.ports;
+	DBGC ( hubdev, "HUB %s has %d ports at depth %d%s\n", hubdev->name,
+	       ports, depth, ( enhanced ? " (enhanced)" : "" ) );
+
+	/* Allocate hub */
+	hubdev->hub = alloc_usb_hub ( bus, usb, ports, &hub_operations );
+	if ( ! hubdev->hub ) {
+		rc = -ENOMEM;
+		goto err_alloc_hub;
+	}
+	usb_hub_set_drvdata ( hubdev->hub, hubdev );
+
+	/* Register hub */
+	if ( ( rc = register_usb_hub ( hubdev->hub ) ) != 0 ) {
+		DBGC ( hubdev, "HUB %s could not register: %s\n",
+		       hubdev->name, strerror ( rc ) );
+		goto err_register_hub;
+	}
+
+	usb_func_set_drvdata ( func, hubdev );
+	return 0;
+
+	unregister_usb_hub ( hubdev->hub );
+ err_register_hub:
+	free_usb_hub ( hubdev->hub );
+ err_alloc_hub:
+ err_hub_descriptor:
+ err_set_hub_depth:
+ err_endpoint:
+ err_interface:
+	free ( hubdev );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove USB hub
+ *
+ * @v func		USB function
+ * @ret rc		Return status code
+ */
+static void hub_remove ( struct usb_function *func ) {
+	struct usb_hub_device *hubdev = usb_func_get_drvdata ( func );
+	struct usb_hub *hub = hubdev->hub;
+	struct usb_device *usb = hubdev->usb;
+	struct usb_port *port;
+	unsigned int i;
+
+	/* If hub has been unplugged, mark all ports as unplugged */
+	if ( usb->port->speed == USB_SPEED_NONE ) {
+		for ( i = 1 ; i <= hub->ports ; i++ ) {
+			port = usb_port ( hub, i );
+			port->speed = USB_SPEED_NONE;
+		}
+	}
+
+	/* Unregister hub */
+	unregister_usb_hub ( hubdev->hub );
+	assert ( ! process_running ( &hubdev->refill ) );
+
+	/* Free hub */
+	free_usb_hub ( hubdev->hub );
+
+	/* Free hub device */
+	free ( hubdev );
+}
+
+/** USB hub device IDs */
+static struct usb_device_id hub_ids[] = {
+	{
+		.name = "hub-1",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HUB,
+			.subclass = 0,
+			.protocol = 0,
+		},
+	},
+	{
+		.name = "hub-2",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HUB,
+			.subclass = 0,
+			.protocol = 1,
+		},
+	},
+};
+
+/** USB hub driver */
+struct usb_driver usb_hub_driver __usb_driver = {
+	.ids = hub_ids,
+	.id_count = ( sizeof ( hub_ids ) / sizeof ( hub_ids[0] ) ),
+	.probe = hub_probe,
+	.remove = hub_remove,
+};
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbhub.h b/qemu/roms/ipxe/src/drivers/usb/usbhub.h
new file mode 100644
index 000000000..d7d8f9610
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbhub.h
@@ -0,0 +1,279 @@
+#ifndef _USBHUB_H
+#define _USBHUB_H
+
+/** @file
+ *
+ * USB hubs
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+#include <ipxe/list.h>
+#include <ipxe/process.h>
+
+/** Request recipient is a port */
+#define USB_HUB_RECIP_PORT ( 3 << 0 )
+
+/** A basic USB hub descriptor */
+struct usb_hub_descriptor_basic {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Number of ports */
+	uint8_t ports;
+	/** Characteristics */
+	uint16_t characteristics;
+	/** Power-on delay (in 2ms intervals */
+	uint8_t delay;
+	/** Controller current (in mA) */
+	uint8_t current;
+} __attribute__ (( packed ));
+
+/** A basic USB hub descriptor */
+#define USB_HUB_DESCRIPTOR 41
+
+/** An enhanced USB hub descriptor */
+struct usb_hub_descriptor_enhanced {
+	/** Basic USB hub descriptor */
+	struct usb_hub_descriptor_basic basic;
+	/** Header decode latency */
+	uint8_t latency;
+	/** Maximum delay */
+	uint16_t delay;
+	/** Removable device bitmask */
+	uint16_t removable;
+} __attribute__ (( packed ));
+
+/** An enhanced USB hub descriptor */
+#define USB_HUB_DESCRIPTOR_ENHANCED 42
+
+/** A USB hub descriptor */
+union usb_hub_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Basic hub descriptor */
+	struct usb_hub_descriptor_basic basic;
+	/** Enhanced hub descriptor */
+	struct usb_hub_descriptor_enhanced enhanced;
+} __attribute__ (( packed ));
+
+/** Port status */
+struct usb_hub_port_status {
+	/** Current status */
+	uint16_t current;
+	/** Changed status */
+	uint16_t changed;
+} __attribute__ (( packed ));
+
+/** Current connect status feature */
+#define USB_HUB_PORT_CONNECTION 0
+
+/** Port enabled/disabled feature */
+#define USB_HUB_PORT_ENABLE 1
+
+/** Port reset feature */
+#define USB_HUB_PORT_RESET 4
+
+/** Port power feature */
+#define USB_HUB_PORT_POWER 8
+
+/** Low-speed device attached */
+#define USB_HUB_PORT_LOW_SPEED 9
+
+/** High-speed device attached */
+#define USB_HUB_PORT_HIGH_SPEED 10
+
+/** Connect status changed */
+#define USB_HUB_C_PORT_CONNECTION 16
+
+/** Port enable/disable changed */
+#define	USB_HUB_C_PORT_ENABLE 17
+
+/** Suspend changed */
+#define USB_HUB_C_PORT_SUSPEND 18
+
+/** Over-current indicator changed */
+#define USB_HUB_C_PORT_OVER_CURRENT 19
+
+/** Reset changed */
+#define USB_HUB_C_PORT_RESET 20
+
+/** Link state changed */
+#define USB_HUB_C_PORT_LINK_STATE 25
+
+/** Configuration error */
+#define USB_HUB_C_PORT_CONFIG_ERROR 26
+
+/** Calculate feature from change bit number */
+#define USB_HUB_C_FEATURE( bit ) ( 16 + (bit) )
+
+/** USB features */
+#define USB_HUB_FEATURES						\
+	( ( 1 << USB_HUB_C_PORT_CONNECTION ) |				\
+	  ( 1 << USB_HUB_C_PORT_ENABLE ) |				\
+	  ( 1 << USB_HUB_C_PORT_SUSPEND ) |				\
+	  ( 1 << USB_HUB_C_PORT_OVER_CURRENT ) |			\
+	  ( 1 << USB_HUB_C_PORT_RESET ) )
+
+/** USB features for enhanced hubs */
+#define USB_HUB_FEATURES_ENHANCED					\
+	( ( 1 << USB_HUB_C_PORT_CONNECTION ) |				\
+	  ( 1 << USB_HUB_C_PORT_OVER_CURRENT ) |			\
+	  ( 1 << USB_HUB_C_PORT_RESET ) |				\
+	  ( 1 << USB_HUB_C_PORT_LINK_STATE ) |				\
+	  ( 1 << USB_HUB_C_PORT_CONFIG_ERROR ) )
+
+/** Set hub depth */
+#define USB_HUB_SET_HUB_DEPTH						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_DEVICE |		\
+	  USB_REQUEST_TYPE ( 12 ) )
+
+/** Clear transaction translator buffer */
+#define USB_HUB_CLEAR_TT_BUFFER						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_HUB_RECIP_PORT |		\
+	  USB_REQUEST_TYPE ( 8 ) )
+
+/**
+ * Get hub descriptor
+ *
+ * @v usb		USB device
+ * @v enhanced		Hub is an enhanced hub
+ * @v data		Hub descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_get_descriptor ( struct usb_device *usb, int enhanced,
+			 union usb_hub_descriptor *data ) {
+	unsigned int desc;
+	size_t len;
+
+	/* Determine descriptor type and length */
+	desc = ( enhanced ? USB_HUB_DESCRIPTOR_ENHANCED : USB_HUB_DESCRIPTOR );
+	len = ( enhanced ? sizeof ( data->enhanced ) : sizeof ( data->basic ) );
+
+	return usb_get_descriptor ( usb, USB_TYPE_CLASS, desc, 0, 0,
+				    &data->header, len );
+}
+
+/**
+ * Get port status
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v status		Port status descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_get_port_status ( struct usb_device *usb, unsigned int port,
+			  struct usb_hub_port_status *status ) {
+
+	return usb_get_status ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				port, status, sizeof ( *status ) );
+}
+
+/**
+ * Clear port feature
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v feature		Feature to clear
+ * @v index		Index (when clearing a port indicator)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_clear_port_feature ( struct usb_device *usb, unsigned int port,
+			     unsigned int feature, unsigned int index ) {
+
+	return usb_clear_feature ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				   feature, ( ( index << 8 ) | port ) );
+}
+
+/**
+ * Set port feature
+ *
+ * @v usb		USB device
+ * @v port		Port address
+ * @v feature		Feature to clear
+ * @v index		Index (when clearing a port indicator)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_set_port_feature ( struct usb_device *usb, unsigned int port,
+			   unsigned int feature, unsigned int index ) {
+
+	return usb_set_feature ( usb, ( USB_TYPE_CLASS | USB_HUB_RECIP_PORT ),
+				 feature, ( ( index << 8 ) | port ) );
+}
+
+/**
+ * Set hub depth
+ *
+ * @v usb		USB device
+ * @v depth		Hub depth
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_set_hub_depth ( struct usb_device *usb, unsigned int depth ) {
+
+	return usb_control ( usb, USB_HUB_SET_HUB_DEPTH, depth, 0, NULL, 0 );
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v usb		USB device
+ * @v device		Device address
+ * @v endpoint		Endpoint address
+ * @v attributes	Endpoint attributes
+ * @v tt_port		Transaction translator port (or 1 for single-TT hubs)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_hub_clear_tt_buffer ( struct usb_device *usb, unsigned int device,
+			  unsigned int endpoint, unsigned int attributes,
+			  unsigned int tt_port ) {
+	unsigned int value;
+
+	/* Calculate value */
+	value = ( ( ( endpoint & USB_ENDPOINT_MAX ) << 0 ) | ( device << 4 ) |
+		  ( ( attributes & USB_ENDPOINT_ATTR_TYPE_MASK ) << 11 ) |
+		  ( ( endpoint & USB_ENDPOINT_IN ) << 8 ) );
+
+	return usb_control ( usb, USB_HUB_CLEAR_TT_BUFFER, value,
+			     tt_port, NULL, 0 );
+}
+
+/** Transaction translator port value for single-TT hubs */
+#define USB_HUB_TT_SINGLE 1
+
+/** A USB hub device */
+struct usb_hub_device {
+	/** Name */
+	const char *name;
+	/** USB device */
+	struct usb_device *usb;
+	/** USB hub */
+	struct usb_hub *hub;
+	/** Features */
+	unsigned int features;
+
+	/** Interrupt endpoint */
+	struct usb_endpoint intr;
+	/** Interrupt endpoint refill process */
+	struct process refill;
+};
+
+/** Interrupt ring fill level
+ *
+ * This is a policy decision.
+ */
+#define USB_HUB_INTR_FILL 4
+
+/** Maximum time to wait for port to become enabled
+ *
+ * This is a policy decision.
+ */
+#define USB_HUB_ENABLE_MAX_WAIT_MS 100
+
+#endif /* _USBHUB_H */
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbkbd.c b/qemu/roms/ipxe/src/drivers/usb/usbkbd.c
new file mode 100644
index 000000000..ea94f2e63
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbkbd.c
@@ -0,0 +1,509 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/console.h>
+#include <ipxe/keys.h>
+#include <ipxe/usb.h>
+#include "usbkbd.h"
+
+/** @file
+ *
+ * USB keyboard driver
+ *
+ */
+
+/** List of USB keyboards */
+static LIST_HEAD ( usb_keyboards );
+
+/******************************************************************************
+ *
+ * Keyboard map
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Map USB keycode to iPXE key
+ *
+ * @v keycode		Keycode
+ * @v modifiers		Modifiers
+ * @ret key		iPXE key
+ *
+ * Key codes are defined in the USB HID Usage Tables Keyboard/Keypad
+ * page.
+ */
+static unsigned int usbkbd_map ( unsigned int keycode,
+				 unsigned int modifiers ) {
+	unsigned int key;
+
+	if ( keycode < USBKBD_KEY_A ) {
+		/* Not keys */
+		key = 0;
+	} else if ( keycode <= USBKBD_KEY_Z ) {
+		/* Alphabetic keys */
+		key = ( keycode - USBKBD_KEY_A + 'a' );
+		if ( modifiers & USBKBD_CTRL ) {
+			key -= ( 'a' - CTRL_A );
+		} else if ( modifiers & USBKBD_SHIFT ) {
+			key -= ( 'a' - 'A' );
+		}
+	} else if ( keycode <= USBKBD_KEY_0 ) {
+		/* Numeric key row */
+		if ( modifiers & USBKBD_SHIFT ) {
+			key = "!@#$%^&*()" [ keycode - USBKBD_KEY_1 ];
+		} else {
+			key = ( ( ( keycode - USBKBD_KEY_1 + 1 ) % 10 ) + '0' );
+		}
+	} else if ( keycode <= USBKBD_KEY_SPACE ) {
+		/* Unmodifiable keys */
+		static const uint8_t unmodifable[] =
+			{ LF, ESC, BACKSPACE, TAB, ' ' };
+		key = unmodifable[ keycode - USBKBD_KEY_ENTER ];
+	} else if ( keycode <= USBKBD_KEY_SLASH ) {
+		/* Punctuation keys */
+		if ( modifiers & USBKBD_SHIFT ) {
+			key = "_+{}|~:\"~<>?" [ keycode - USBKBD_KEY_MINUS ];
+		} else {
+			key = "-=[]\\#;'`,./" [ keycode - USBKBD_KEY_MINUS ];
+		}
+	} else if ( keycode <= USBKBD_KEY_UP ) {
+		/* Special keys */
+		static const uint16_t special[] = {
+			0, 0, 0, 0, 0, KEY_F5, KEY_F6, KEY_F7, KEY_F8, KEY_F9,
+			KEY_F10, KEY_F11, KEY_F12, 0, 0, 0, KEY_IC, KEY_HOME,
+			KEY_PPAGE, KEY_DC, KEY_END, KEY_NPAGE, KEY_RIGHT,
+			KEY_LEFT, KEY_DOWN, KEY_UP
+		};
+		key = special[ keycode - USBKBD_KEY_CAPSLOCK ];
+	} else {
+		key = 0;
+	}
+
+	return key;
+}
+
+/******************************************************************************
+ *
+ * Keyboard buffer
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Insert keypress into keyboard buffer
+ *
+ * @v kbd		USB keyboard
+ * @v keycode		Keycode
+ * @v modifiers		Modifiers
+ */
+static void usbkbd_produce ( struct usb_keyboard *kbd, unsigned int keycode,
+			     unsigned int modifiers ) {
+	unsigned int key;
+
+	/* Map to iPXE key */
+	key = usbkbd_map ( keycode, modifiers );
+
+	/* Do nothing if this keycode has no corresponding iPXE key */
+	if ( ! key ) {
+		DBGC ( kbd, "KBD %s has no key for keycode %#02x:%#02x\n",
+		       kbd->name, modifiers, keycode );
+		return;
+	}
+
+	/* Check for buffer overrun */
+	if ( usbkbd_fill ( kbd ) >= USBKBD_BUFSIZE ) {
+		DBGC ( kbd, "KBD %s buffer overrun (key %#02x)\n",
+		       kbd->name, key );
+		return;
+	}
+
+	/* Insert into buffer */
+	kbd->key[ ( kbd->prod++ ) % USBKBD_BUFSIZE ] = key;
+	DBGC2 ( kbd, "KBD %s key %#02x produced\n", kbd->name, key );
+}
+
+/**
+ * Consume character from keyboard buffer
+ *
+ * @v kbd		USB keyboard
+ * @ret character	Character
+ */
+static unsigned int usbkbd_consume ( struct usb_keyboard *kbd ) {
+	static char buf[] = "\x1b[xx~";
+	char *tmp = &buf[2];
+	unsigned int key;
+	unsigned int character;
+	unsigned int ansi_n;
+	unsigned int len;
+
+	/* Sanity check */
+	assert ( usbkbd_fill ( kbd ) > 0 );
+
+	/* Get current keypress */
+	key = kbd->key[ kbd->cons % USBKBD_BUFSIZE ];
+
+	/* If this is a straightforward key, just consume and return it */
+	if ( key < KEY_MIN ) {
+		kbd->cons++;
+		DBGC2 ( kbd, "KBD %s key %#02x consumed\n", kbd->name, key );
+		return key;
+	}
+
+	/* Construct ANSI sequence */
+	ansi_n = KEY_ANSI_N ( key );
+	if ( ansi_n )
+		tmp += sprintf ( tmp, "%d", ansi_n );
+	*(tmp++) = KEY_ANSI_TERMINATOR ( key );
+	*tmp = '\0';
+	len = ( tmp - buf );
+	assert ( len < sizeof ( buf ) );
+	if ( kbd->subcons == 0 ) {
+		DBGC2 ( kbd, "KBD %s key %#02x consumed as ^[%s\n",
+			kbd->name, key, &buf[1] );
+	}
+
+	/* Extract character from ANSI sequence */
+	assert ( kbd->subcons < len );
+	character = buf[ kbd->subcons++ ];
+
+	/* Consume key if applicable */
+	if ( kbd->subcons == len ) {
+		kbd->cons++;
+		kbd->subcons = 0;
+	}
+
+	return character;
+}
+
+/******************************************************************************
+ *
+ * Keyboard report
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Check for presence of keycode in report
+ *
+ * @v report		Keyboard report
+ * @v keycode		Keycode (must be non-zero)
+ * @ret has_keycode	Keycode is present in report
+ */
+static int usbkbd_has_keycode ( struct usb_keyboard_report *report,
+				unsigned int keycode ) {
+	unsigned int i;
+
+	/* Check for keycode */
+	for ( i = 0 ; i < ( sizeof ( report->keycode ) /
+			    sizeof ( report->keycode[0] ) ) ; i++ ) {
+		if ( report->keycode[i] == keycode )
+			return keycode;
+	}
+
+	return 0;
+}
+
+/**
+ * Handle keyboard report
+ *
+ * @v kbd		USB keyboard
+ * @v new		New keyboard report
+ */
+static void usbkbd_report ( struct usb_keyboard *kbd,
+			    struct usb_keyboard_report *new ) {
+	struct usb_keyboard_report *old = &kbd->report;
+	unsigned int keycode;
+	unsigned int i;
+
+	/* Check if current key has been released */
+	if ( kbd->keycode && ! usbkbd_has_keycode ( new, kbd->keycode ) ) {
+		DBGC2 ( kbd, "KBD %s keycode %#02x released\n",
+			kbd->name, kbd->keycode );
+		kbd->keycode = 0;
+	}
+
+	/* Decrement auto-repeat hold-off timer, if applicable */
+	if ( kbd->holdoff )
+		kbd->holdoff--;
+
+	/* Check if a new key has been pressed */
+	for ( i = 0 ; i < ( sizeof ( new->keycode ) /
+			    sizeof ( new->keycode[0] ) ) ; i++ ) {
+
+		/* Ignore keys present in the previous report */
+		keycode = new->keycode[i];
+		if ( ( keycode == 0 ) || usbkbd_has_keycode ( old, keycode ) )
+			continue;
+		DBGC2 ( kbd, "KBD %s keycode %#02x pressed\n",
+			kbd->name, keycode );
+
+		/* Insert keypress into keyboard buffer */
+		usbkbd_produce ( kbd, keycode, new->modifiers );
+
+		/* Record as most recent keycode */
+		kbd->keycode = keycode;
+
+		/* Start auto-repeat hold-off timer */
+		kbd->holdoff = USBKBD_HOLDOFF;
+	}
+
+	/* Insert auto-repeated keypress into keyboard buffer, if applicable */
+	if ( kbd->keycode && ! kbd->holdoff )
+		usbkbd_produce ( kbd, kbd->keycode, new->modifiers );
+
+	/* Record report */
+	memcpy ( old, new, sizeof ( *old ) );
+}
+
+/******************************************************************************
+ *
+ * Interrupt endpoint
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Complete interrupt transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v rc		Completion status code
+ */
+static void usbkbd_complete ( struct usb_endpoint *ep,
+			      struct io_buffer *iobuf, int rc ) {
+	struct usb_keyboard *kbd = container_of ( ep, struct usb_keyboard,
+						  hid.in );
+	struct usb_keyboard_report *report;
+
+	/* Ignore packets cancelled when the endpoint closes */
+	if ( ! ep->open )
+		goto drop;
+
+	/* Ignore packets with errors */
+	if ( rc != 0 ) {
+		DBGC ( kbd, "KBD %s interrupt IN failed: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto drop;
+	}
+
+	/* Ignore underlength packets */
+	if ( iob_len ( iobuf ) < sizeof ( *report ) ) {
+		DBGC ( kbd, "KBD %s underlength report:\n", kbd->name );
+		DBGC_HDA ( kbd, 0, iobuf->data, iob_len ( iobuf ) );
+		goto drop;
+	}
+	report = iobuf->data;
+
+	/* Handle keyboard report */
+	usbkbd_report ( kbd, report );
+
+ drop:
+	/* Recycle I/O buffer */
+	usb_recycle ( &kbd->hid.in, iobuf );
+}
+
+/** Interrupt endpoint operations */
+static struct usb_endpoint_driver_operations usbkbd_operations = {
+	.complete = usbkbd_complete,
+};
+
+/******************************************************************************
+ *
+ * USB interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Probe device
+ *
+ * @v func		USB function
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbkbd_probe ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config ) {
+	struct usb_device *usb = func->usb;
+	struct usb_keyboard *kbd;
+	int rc;
+
+	/* Allocate and initialise structure */
+	kbd = zalloc ( sizeof ( *kbd ) );
+	if ( ! kbd ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	kbd->name = func->name;
+	kbd->bus = usb->port->hub->bus;
+	usbhid_init ( &kbd->hid, func, &usbkbd_operations, NULL );
+	usb_refill_init ( &kbd->hid.in, sizeof ( kbd->report ),
+			  USBKBD_INTR_MAX_FILL );
+
+	/* Describe USB human interface device */
+	if ( ( rc = usbhid_describe ( &kbd->hid, config ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not describe: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_describe;
+	}
+	DBGC ( kbd, "KBD %s using %s (len %zd)\n",
+	       kbd->name, usb_endpoint_name ( &kbd->hid.in ), kbd->hid.in.mtu );
+
+	/* Set boot protocol */
+	if ( ( rc = usbhid_set_protocol ( usb, func->interface[0],
+					  USBHID_PROTOCOL_BOOT ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not set boot protocol: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_set_protocol;
+	}
+
+	/* Set idle time */
+	if ( ( rc = usbhid_set_idle ( usb, func->interface[0], 0,
+				      USBKBD_IDLE_DURATION ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not set idle time: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_set_idle;
+	}
+
+	/* Open USB human interface device */
+	if ( ( rc = usbhid_open ( &kbd->hid ) ) != 0 ) {
+		DBGC ( kbd, "KBD %s could not open: %s\n",
+		       kbd->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Add to list of USB keyboards */
+	list_add_tail ( &kbd->list, &usb_keyboards );
+
+	usb_func_set_drvdata ( func, kbd );
+	return 0;
+
+	usbhid_close ( &kbd->hid );
+ err_open:
+ err_set_idle:
+ err_set_protocol:
+ err_describe:
+	free ( kbd );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove device
+ *
+ * @v func		USB function
+ */
+static void usbkbd_remove ( struct usb_function *func ) {
+	struct usb_keyboard *kbd = usb_func_get_drvdata ( func );
+
+	/* Remove from list of USB keyboards */
+	list_del ( &kbd->list );
+
+	/* Close USB human interface device */
+	usbhid_close ( &kbd->hid );
+
+	/* Free device */
+	free ( kbd );
+}
+
+/** USB keyboard device IDs */
+static struct usb_device_id usbkbd_ids[] = {
+	{
+		.name = "kbd",
+		.vendor = USB_ANY_ID,
+		.product = USB_ANY_ID,
+		.class = {
+			.class = USB_CLASS_HID,
+			.subclass = USB_SUBCLASS_HID_BOOT,
+			.protocol = USBKBD_PROTOCOL,
+		},
+	},
+};
+
+/** USB keyboard driver */
+struct usb_driver usbkbd_driver __usb_driver = {
+	.ids = usbkbd_ids,
+	.id_count = ( sizeof ( usbkbd_ids ) / sizeof ( usbkbd_ids[0] ) ),
+	.probe = usbkbd_probe,
+	.remove = usbkbd_remove,
+};
+
+/******************************************************************************
+ *
+ * Console interface
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Read a character from the console
+ *
+ * @ret character	Character read
+ */
+static int usbkbd_getchar ( void ) {
+	struct usb_keyboard *kbd;
+
+	/* Consume first available key */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		if ( usbkbd_fill ( kbd ) )
+			return usbkbd_consume ( kbd );
+	}
+
+	return 0;
+}
+
+/**
+ * Check for available input
+ *
+ * @ret is_available	Input is available
+ */
+static int usbkbd_iskey ( void ) {
+	struct usb_keyboard *kbd;
+	unsigned int fill;
+
+	/* Poll all USB keyboards and refill endpoints */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		usb_poll ( kbd->bus );
+		usb_refill ( &kbd->hid.in );
+	}
+
+	/* Check for a non-empty keyboard buffer */
+	list_for_each_entry ( kbd, &usb_keyboards, list ) {
+		fill = usbkbd_fill ( kbd );
+		if ( fill )
+			return fill;
+	}
+
+	return 0;
+}
+
+/** USB keyboard console */
+struct console_driver usbkbd_console __console_driver = {
+	.getchar = usbkbd_getchar,
+	.iskey = usbkbd_iskey,
+};
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbkbd.h b/qemu/roms/ipxe/src/drivers/usb/usbkbd.h
new file mode 100644
index 000000000..7eab24e46
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbkbd.h
@@ -0,0 +1,154 @@
+#ifndef _USBKBD_H
+#define _USBKBD_H
+
+/** @file
+ *
+ * USB keyboard driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbhid.h>
+
+/** Keyboard protocol */
+#define USBKBD_PROTOCOL 1
+
+/** A USB keyboard report */
+struct usb_keyboard_report {
+	/** Modifier keys */
+	uint8_t modifiers;
+	/** Reserved */
+	uint8_t reserved;
+	/** Keycodes */
+	uint8_t keycode[6];
+} __attribute__ (( packed ));
+
+/** USB modifier keys */
+enum usb_keyboard_modifier {
+	/** Left Ctrl key */
+	USBKBD_CTRL_LEFT = 0x01,
+	/** Left Shift key */
+	USBKBD_SHIFT_LEFT = 0x02,
+	/** Left Alt key */
+	USBKBD_ALT_LEFT = 0x04,
+	/** Left GUI key */
+	USBKBD_GUI_LEFT = 0x08,
+	/** Right Ctrl key */
+	USBKBD_CTRL_RIGHT = 0x10,
+	/** Right Shift key */
+	USBKBD_SHIFT_RIGHT = 0x20,
+	/** Right Alt key */
+	USBKBD_ALT_RIGHT = 0x40,
+	/** Right GUI key */
+	USBKBD_GUI_RIGHT = 0x80,
+};
+
+/** Either Ctrl key */
+#define USBKBD_CTRL ( USBKBD_CTRL_LEFT | USBKBD_CTRL_RIGHT )
+
+/** Either Shift key */
+#define USBKBD_SHIFT ( USBKBD_SHIFT_LEFT | USBKBD_SHIFT_RIGHT )
+
+/** Either Alt key */
+#define USBKBD_ALT ( USBKBD_ALT_LEFT | USBKBD_ALT_RIGHT )
+
+/** Either GUI key */
+#define USBKBD_GUI ( USBKBD_GUI_LEFT | USBKBD_GUI_RIGHT )
+
+/** USB keycodes */
+enum usb_keycode {
+	USBKBD_KEY_A = 0x04,
+	USBKBD_KEY_Z = 0x1d,
+	USBKBD_KEY_1 = 0x1e,
+	USBKBD_KEY_0 = 0x27,
+	USBKBD_KEY_ENTER = 0x28,
+	USBKBD_KEY_SPACE = 0x2c,
+	USBKBD_KEY_MINUS = 0x2d,
+	USBKBD_KEY_SLASH = 0x38,
+	USBKBD_KEY_CAPSLOCK = 0x39,
+	USBKBD_KEY_UP = 0x52,
+};
+
+/** Keyboard idle duration (in 4ms units)
+ *
+ * This is a policy decision.  We choose to use an autorepeat rate of
+ * approximately 40ms.
+ */
+#define USBKBD_IDLE_DURATION 10 /* 10 x 4ms = 40ms */
+
+/** Keyboard auto-repeat hold-off (in units of USBKBD_IDLE_DURATION)
+ *
+ * This is a policy decision.  We choose to use an autorepeat delay of
+ * approximately 500ms.
+ */
+#define USBKBD_HOLDOFF 12 /* 12 x 40ms = 480ms */
+
+/** Interrupt endpoint maximum fill level
+ *
+ * When idling, we are likely to poll the USB endpoint at only the
+ * 18.2Hz system timer tick rate.  With a typical observed bInterval
+ * of 10ms (which will be rounded down to 8ms by the HCI drivers),
+ * this gives approximately 7 completions per poll.
+ */
+#define USBKBD_INTR_MAX_FILL 8
+
+/** Keyboard buffer size
+ *
+ * Must be a power of two.
+ */
+#define USBKBD_BUFSIZE 8
+
+/** A USB keyboard device */
+struct usb_keyboard {
+	/** Name */
+	const char *name;
+	/** List of all USB keyboards */
+	struct list_head list;
+
+	/** USB bus */
+	struct usb_bus *bus;
+	/** USB human interface device */
+	struct usb_hid hid;
+
+	/** Most recent keyboard report */
+	struct usb_keyboard_report report;
+	/** Most recently pressed non-modifier key (if any) */
+	unsigned int keycode;
+	/** Autorepeat hold-off time (in number of completions reported) */
+	unsigned int holdoff;
+
+	/** Keyboard buffer
+	 *
+	 * This stores iPXE key values.
+	 */
+	unsigned int key[USBKBD_BUFSIZE];
+	/** Keyboard buffer producer counter */
+	unsigned int prod;
+	/** Keyboard buffer consumer counter */
+	unsigned int cons;
+	/** Keyboard buffer sub-consumer counter
+	 *
+	 * This represents the index within the ANSI escape sequence
+	 * corresponding to an iPXE key value.
+	 */
+	unsigned int subcons;
+};
+
+/**
+ * Calculate keyboard buffer fill level
+ *
+ * @v kbd		USB keyboard
+ * @ret fill		Keyboard buffer fill level
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+usbkbd_fill ( struct usb_keyboard *kbd ) {
+	unsigned int fill = ( kbd->prod - kbd->cons );
+
+	assert ( fill <= USBKBD_BUFSIZE );
+	return fill;
+}
+
+#endif /* _USBKBD_H */
diff --git a/qemu/roms/ipxe/src/drivers/usb/usbnet.c b/qemu/roms/ipxe/src/drivers/usb/usbnet.c
new file mode 100644
index 000000000..b92336d05
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/usbnet.c
@@ -0,0 +1,284 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <ipxe/usb.h>
+#include <ipxe/usbnet.h>
+
+/** @file
+ *
+ * USB network devices
+ *
+ * USB network devices use a variety of packet formats and interface
+ * descriptors, but tend to have several features in common:
+ *
+ *  - a single interrupt endpoint using the generic refill mechanism
+ *
+ *  - a single bulk IN endpoint using the generic refill mechanism
+ *
+ *  - a single bulk OUT endpoint
+ *
+ *  - optional use of an alternate setting to enable the data interface
+ *
+ */
+
+/**
+ * Open USB network device
+ *
+ * @v usbnet		USB network device
+ * @ret rc		Return status code
+ */
+int usbnet_open ( struct usbnet_device *usbnet ) {
+	struct usb_device *usb = usbnet->func->usb;
+	int rc;
+
+	/* Open interrupt endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->intr ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open interrupt: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_intr;
+	}
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &usbnet->intr ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not refill interrupt: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_refill_intr;
+	}
+
+	/* Select alternate setting for data interface, if applicable */
+	if ( usbnet->alternate &&
+	     ( ( rc = usb_set_interface ( usb, usbnet->data,
+					  usbnet->alternate ) ) != 0 ) ) {
+		DBGC ( usbnet, "USBNET %s could not set alternate interface "
+		       "%d: %s\n", usbnet->func->name, usbnet->alternate,
+		       strerror ( rc ) );
+		goto err_set_interface;
+	}
+
+	/* Open bulk IN endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->in ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open bulk IN: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_in;
+	}
+
+	/* Open bulk OUT endpoint */
+	if ( ( rc = usb_endpoint_open ( &usbnet->out ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not open bulk OUT: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_open_out;
+	}
+
+	/* Refill bulk IN endpoint */
+	if ( ( rc = usb_refill ( &usbnet->in ) ) != 0 ) {
+		DBGC ( usbnet, "USBNET %s could not refill bulk IN: %s\n",
+		       usbnet->func->name, strerror ( rc ) );
+		goto err_refill_in;
+	}
+
+	return 0;
+
+ err_refill_in:
+	usb_endpoint_close ( &usbnet->out );
+ err_open_out:
+	usb_endpoint_close ( &usbnet->in );
+ err_open_in:
+	if ( usbnet->alternate )
+		usb_set_interface ( usb, usbnet->data, 0 );
+ err_set_interface:
+ err_refill_intr:
+	usb_endpoint_close ( &usbnet->intr );
+ err_open_intr:
+	return rc;
+}
+
+/**
+ * Close USB network device
+ *
+ * @v usbnet		USB network device
+ */
+void usbnet_close ( struct usbnet_device *usbnet ) {
+	struct usb_device *usb = usbnet->func->usb;
+
+	/* Close bulk OUT endpoint */
+	usb_endpoint_close ( &usbnet->out );
+
+	/* Close bulk IN endpoint */
+	usb_endpoint_close ( &usbnet->in );
+
+	/* Reset alternate setting for data interface, if applicable */
+	if ( usbnet->alternate )
+		usb_set_interface ( usb, usbnet->data, 0 );
+
+	/* Close interrupt endpoint */
+	usb_endpoint_close ( &usbnet->intr );
+}
+
+/**
+ * Refill USB network device bulk IN and interrupt endpoints
+ *
+ * @v usbnet		USB network device
+ * @ret rc		Return status code
+ */
+int usbnet_refill ( struct usbnet_device *usbnet ) {
+	int rc;
+
+	/* Refill bulk IN endpoint */
+	if ( ( rc = usb_refill ( &usbnet->in ) ) != 0 )
+		return rc;
+
+	/* Refill interrupt endpoint */
+	if ( ( rc = usb_refill ( &usbnet->intr ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Describe communications interface and interrupt endpoint
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbnet_comms_describe ( struct usbnet_device *usbnet,
+				   struct usb_configuration_descriptor *config){
+	struct usb_interface_descriptor *desc;
+	unsigned int comms;
+	unsigned int i;
+	int rc;
+
+	/* Iterate over all available interfaces */
+	for ( i = 0 ; i < usbnet->func->count ; i++ ) {
+
+		/* Get interface number */
+		comms = usbnet->func->interface[i];
+
+		/* Locate interface descriptor */
+		desc = usb_interface_descriptor ( config, comms, 0 );
+		if ( ! desc )
+			continue;
+
+		/* Describe interrupt endpoint */
+		if ( ( rc = usb_endpoint_described ( &usbnet->intr, config,
+						     desc, USB_INTERRUPT_IN,
+						     0 ) ) != 0 )
+			continue;
+
+		/* Record communications interface */
+		usbnet->comms = comms;
+		DBGC ( usbnet, "USBNET %s found communications interface %d\n",
+		       usbnet->func->name, comms );
+		return 0;
+	}
+
+	DBGC ( usbnet, "USBNET %s found no communications interface\n",
+	       usbnet->func->name );
+	return -ENOENT;
+}
+
+/**
+ * Describe data interface and bulk endpoints
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+static int usbnet_data_describe ( struct usbnet_device *usbnet,
+				  struct usb_configuration_descriptor *config ){
+	struct usb_interface_descriptor *desc;
+	unsigned int data;
+	unsigned int alt;
+	unsigned int i;
+	int rc;
+
+	/* Iterate over all available interfaces */
+	for ( i = 0 ; i < usbnet->func->count ; i++ ) {
+
+		/* Get interface number */
+		data = usbnet->func->interface[i];
+
+		/* Iterate over all existent alternate settings */
+		for ( alt = 0 ; ; alt++ ) {
+
+			/* Locate interface descriptor */
+			desc = usb_interface_descriptor ( config, data, alt );
+			if ( ! desc )
+				break;
+
+			/* Describe bulk IN endpoint */
+			if ( ( rc = usb_endpoint_described ( &usbnet->in,
+							     config, desc,
+							     USB_BULK_IN,
+							     0 ) ) != 0 )
+				continue;
+
+			/* Describe bulk OUT endpoint */
+			if ( ( rc = usb_endpoint_described ( &usbnet->out,
+							     config, desc,
+							     USB_BULK_OUT,
+							     0 ) ) != 0 )
+				continue;
+
+			/* Record data interface and alternate setting */
+			usbnet->data = data;
+			usbnet->alternate = alt;
+			DBGC ( usbnet, "USBNET %s found data interface %d",
+			       usbnet->func->name, data );
+			if ( alt )
+				DBGC ( usbnet, " using alternate %d", alt );
+			DBGC ( usbnet, "\n" );
+			return 0;
+		}
+	}
+
+	DBGC ( usbnet, "USBNET %s found no data interface\n",
+	       usbnet->func->name );
+	return -ENOENT;
+}
+
+/**
+ * Describe USB network device interfaces
+ *
+ * @v usbnet		USB network device
+ * @v config		Configuration descriptor
+ * @ret rc		Return status code
+ */
+int usbnet_describe ( struct usbnet_device *usbnet,
+		      struct usb_configuration_descriptor *config ) {
+	int rc;
+
+	/* Describe communications interface */
+	if ( ( rc = usbnet_comms_describe ( usbnet, config ) ) != 0 )
+		return rc;
+
+	/* Describe data interface */
+	if ( ( rc = usbnet_data_describe ( usbnet, config ) ) != 0 )
+		return rc;
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/drivers/usb/xhci.c b/qemu/roms/ipxe/src/drivers/usb/xhci.c
new file mode 100644
index 000000000..49e67316b
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/xhci.c
@@ -0,0 +1,3321 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <strings.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/umalloc.h>
+#include <ipxe/pci.h>
+#include <ipxe/usb.h>
+#include <ipxe/init.h>
+#include <ipxe/profile.h>
+#include "xhci.h"
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+/** Message transfer profiler */
+static struct profiler xhci_message_profiler __profiler =
+	{ .name = "xhci.message" };
+
+/** Stream transfer profiler */
+static struct profiler xhci_stream_profiler __profiler =
+	{ .name = "xhci.stream" };
+
+/** Event ring profiler */
+static struct profiler xhci_event_profiler __profiler =
+	{ .name = "xhci.event" };
+
+/** Transfer event profiler */
+static struct profiler xhci_transfer_profiler __profiler =
+	{ .name = "xhci.transfer" };
+
+/* Disambiguate the various error causes */
+#define EIO_DATA							\
+	__einfo_error ( EINFO_EIO_DATA )
+#define EINFO_EIO_DATA							\
+	__einfo_uniqify ( EINFO_EIO, ( 2 - 0 ),				\
+			  "Data buffer error" )
+#define EIO_BABBLE							\
+	__einfo_error ( EINFO_EIO_BABBLE )
+#define EINFO_EIO_BABBLE						\
+	__einfo_uniqify ( EINFO_EIO, ( 3 - 0 ),				\
+			  "Babble detected" )
+#define EIO_USB								\
+	__einfo_error ( EINFO_EIO_USB )
+#define EINFO_EIO_USB							\
+	__einfo_uniqify ( EINFO_EIO, ( 4 - 0 ),				\
+			  "USB transaction error" )
+#define EIO_TRB								\
+	__einfo_error ( EINFO_EIO_TRB )
+#define EINFO_EIO_TRB							\
+	__einfo_uniqify ( EINFO_EIO, ( 5 - 0 ),				\
+			  "TRB error" )
+#define EIO_STALL							\
+	__einfo_error ( EINFO_EIO_STALL )
+#define EINFO_EIO_STALL							\
+	__einfo_uniqify ( EINFO_EIO, ( 6 - 0 ),				\
+			  "Stall error" )
+#define EIO_RESOURCE							\
+	__einfo_error ( EINFO_EIO_RESOURCE )
+#define EINFO_EIO_RESOURCE						\
+	__einfo_uniqify ( EINFO_EIO, ( 7 - 0 ),				\
+			  "Resource error" )
+#define EIO_BANDWIDTH							\
+	__einfo_error ( EINFO_EIO_BANDWIDTH )
+#define EINFO_EIO_BANDWIDTH						\
+	__einfo_uniqify ( EINFO_EIO, ( 8 - 0 ),				\
+			  "Bandwidth error" )
+#define EIO_NO_SLOTS							\
+	__einfo_error ( EINFO_EIO_NO_SLOTS )
+#define EINFO_EIO_NO_SLOTS						\
+	__einfo_uniqify ( EINFO_EIO, ( 9 - 0 ),				\
+			  "No slots available" )
+#define EIO_STREAM_TYPE							\
+	__einfo_error ( EINFO_EIO_STREAM_TYPE )
+#define EINFO_EIO_STREAM_TYPE						\
+	__einfo_uniqify ( EINFO_EIO, ( 10 - 0 ),			\
+			  "Invalid stream type" )
+#define EIO_SLOT							\
+	__einfo_error ( EINFO_EIO_SLOT )
+#define EINFO_EIO_SLOT							\
+	__einfo_uniqify ( EINFO_EIO, ( 11 - 0 ),			\
+			  "Slot not enabled" )
+#define EIO_ENDPOINT							\
+	__einfo_error ( EINFO_EIO_ENDPOINT )
+#define EINFO_EIO_ENDPOINT						\
+	__einfo_uniqify ( EINFO_EIO, ( 12 - 0 ),			\
+			  "Endpoint not enabled" )
+#define EIO_SHORT							\
+	__einfo_error ( EINFO_EIO_SHORT )
+#define EINFO_EIO_SHORT							\
+	__einfo_uniqify ( EINFO_EIO, ( 13 - 0 ),			\
+			  "Short packet" )
+#define EIO_UNDERRUN							\
+	__einfo_error ( EINFO_EIO_UNDERRUN )
+#define EINFO_EIO_UNDERRUN						\
+	__einfo_uniqify ( EINFO_EIO, ( 14 - 0 ),			\
+			  "Ring underrun" )
+#define EIO_OVERRUN							\
+	__einfo_error ( EINFO_EIO_OVERRUN )
+#define EINFO_EIO_OVERRUN						\
+	__einfo_uniqify ( EINFO_EIO, ( 15 - 0 ),			\
+			  "Ring overrun" )
+#define EIO_VF_RING_FULL						\
+	__einfo_error ( EINFO_EIO_VF_RING_FULL )
+#define EINFO_EIO_VF_RING_FULL						\
+	__einfo_uniqify ( EINFO_EIO, ( 16 - 0 ),			\
+			  "Virtual function event ring full" )
+#define EIO_PARAMETER							\
+	__einfo_error ( EINFO_EIO_PARAMETER )
+#define EINFO_EIO_PARAMETER						\
+	__einfo_uniqify ( EINFO_EIO, ( 17 - 0 ),			\
+			  "Parameter error" )
+#define EIO_BANDWIDTH_OVERRUN						\
+	__einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
+#define EINFO_EIO_BANDWIDTH_OVERRUN					\
+	__einfo_uniqify ( EINFO_EIO, ( 18 - 0 ),			\
+			  "Bandwidth overrun" )
+#define EIO_CONTEXT							\
+	__einfo_error ( EINFO_EIO_CONTEXT )
+#define EINFO_EIO_CONTEXT						\
+	__einfo_uniqify ( EINFO_EIO, ( 19 - 0 ),			\
+			  "Context state error" )
+#define EIO_NO_PING							\
+	__einfo_error ( EINFO_EIO_NO_PING )
+#define EINFO_EIO_NO_PING						\
+	__einfo_uniqify ( EINFO_EIO, ( 20 - 0 ),			\
+			  "No ping response" )
+#define EIO_RING_FULL							\
+	__einfo_error ( EINFO_EIO_RING_FULL )
+#define EINFO_EIO_RING_FULL						\
+	__einfo_uniqify ( EINFO_EIO, ( 21 - 0 ),			\
+			  "Event ring full" )
+#define EIO_INCOMPATIBLE						\
+	__einfo_error ( EINFO_EIO_INCOMPATIBLE )
+#define EINFO_EIO_INCOMPATIBLE						\
+	__einfo_uniqify ( EINFO_EIO, ( 22 - 0 ),			\
+			  "Incompatible device" )
+#define EIO_MISSED							\
+	__einfo_error ( EINFO_EIO_MISSED )
+#define EINFO_EIO_MISSED						\
+	__einfo_uniqify ( EINFO_EIO, ( 23 - 0 ),			\
+			  "Missed service error" )
+#define EIO_CMD_STOPPED							\
+	__einfo_error ( EINFO_EIO_CMD_STOPPED )
+#define EINFO_EIO_CMD_STOPPED						\
+	__einfo_uniqify ( EINFO_EIO, ( 24 - 0 ),			\
+			  "Command ring stopped" )
+#define EIO_CMD_ABORTED							\
+	__einfo_error ( EINFO_EIO_CMD_ABORTED )
+#define EINFO_EIO_CMD_ABORTED						\
+	__einfo_uniqify ( EINFO_EIO, ( 25 - 0 ),			\
+			  "Command aborted" )
+#define EIO_STOP							\
+	__einfo_error ( EINFO_EIO_STOP )
+#define EINFO_EIO_STOP							\
+	__einfo_uniqify ( EINFO_EIO, ( 26 - 0 ),			\
+			  "Stopped" )
+#define EIO_STOP_LEN							\
+	__einfo_error ( EINFO_EIO_STOP_LEN )
+#define EINFO_EIO_STOP_LEN						\
+	__einfo_uniqify ( EINFO_EIO, ( 27 - 0 ),			\
+			  "Stopped - length invalid" )
+#define EIO_STOP_SHORT							\
+	__einfo_error ( EINFO_EIO_STOP_SHORT )
+#define EINFO_EIO_STOP_SHORT						\
+	__einfo_uniqify ( EINFO_EIO, ( 28 - 0 ),			\
+			  "Stopped - short packet" )
+#define EIO_LATENCY							\
+	__einfo_error ( EINFO_EIO_LATENCY )
+#define EINFO_EIO_LATENCY						\
+	__einfo_uniqify ( EINFO_EIO, ( 29 - 0 ),			\
+			  "Maximum exit latency too large" )
+#define EIO_ISOCH							\
+	__einfo_error ( EINFO_EIO_ISOCH )
+#define EINFO_EIO_ISOCH							\
+	__einfo_uniqify ( EINFO_EIO, ( 31 - 0 ),			\
+			  "Isochronous buffer overrun" )
+#define EPROTO_LOST							\
+	__einfo_error ( EINFO_EPROTO_LOST )
+#define EINFO_EPROTO_LOST						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ),			\
+			  "Event lost" )
+#define EPROTO_UNDEFINED						\
+	__einfo_error ( EINFO_EPROTO_UNDEFINED )
+#define EINFO_EPROTO_UNDEFINED						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ),			\
+			  "Undefined error" )
+#define EPROTO_STREAM_ID						\
+	__einfo_error ( EINFO_EPROTO_STREAM_ID )
+#define EINFO_EPROTO_STREAM_ID						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ),			\
+			  "Invalid stream ID" )
+#define EPROTO_SECONDARY						\
+	__einfo_error ( EINFO_EPROTO_SECONDARY )
+#define EINFO_EPROTO_SECONDARY						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ),			\
+			  "Secondary bandwidth error" )
+#define EPROTO_SPLIT							\
+	__einfo_error ( EINFO_EPROTO_SPLIT )
+#define EINFO_EPROTO_SPLIT						\
+	__einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ),			\
+			  "Split transaction error" )
+#define ECODE(code)							\
+	( ( (code) < 32 ) ?						\
+	  EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE,	\
+		  EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE,		\
+		  EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE,		\
+		  EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN,	\
+		  EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER,		\
+		  EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING,	\
+		  EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED,		\
+		  EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP,		\
+		  EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY,		\
+		  EIO_ISOCH ) :						\
+	  ( (code) < 64 ) ?						\
+	  EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST,		\
+		  EPROTO_UNDEFINED, EPROTO_STREAM_ID,			\
+		  EPROTO_SECONDARY, EPROTO_SPLIT ) :			\
+	  EFAULT )
+
+/******************************************************************************
+ *
+ * Register access
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Initialise device
+ *
+ * @v xhci		xHCI device
+ * @v regs		MMIO registers
+ */
+static void xhci_init ( struct xhci_device *xhci, void *regs ) {
+	uint32_t hcsparams1;
+	uint32_t hcsparams2;
+	uint32_t hccparams1;
+	uint32_t pagesize;
+	size_t caplength;
+	size_t rtsoff;
+	size_t dboff;
+
+	/* Locate capability, operational, runtime, and doorbell registers */
+	xhci->cap = regs;
+	caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
+	rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
+	dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
+	xhci->op = ( xhci->cap + caplength );
+	xhci->run = ( xhci->cap + rtsoff );
+	xhci->db = ( xhci->cap + dboff );
+	DBGC2 ( xhci, "XHCI %s cap %08lx op %08lx run %08lx db %08lx\n",
+		xhci->name, virt_to_phys ( xhci->cap ),
+		virt_to_phys ( xhci->op ), virt_to_phys ( xhci->run ),
+		virt_to_phys ( xhci->db ) );
+
+	/* Read structural parameters 1 */
+	hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
+	xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
+	xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
+	xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
+	DBGC ( xhci, "XHCI %s has %d slots %d intrs %d ports\n",
+	       xhci->name, xhci->slots, xhci->intrs, xhci->ports );
+
+	/* Read structural parameters 2 */
+	hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
+	xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
+	DBGC2 ( xhci, "XHCI %s needs %d scratchpads\n",
+		xhci->name, xhci->scratchpads );
+
+	/* Read capability parameters 1 */
+	hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
+	xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
+	xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
+	xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
+
+	/* Read page size */
+	pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+	xhci->pagesize = XHCI_PAGESIZE ( pagesize );
+	assert ( xhci->pagesize != 0 );
+	assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
+	DBGC2 ( xhci, "XHCI %s page size %zd bytes\n",
+		xhci->name, xhci->pagesize );
+}
+
+/**
+ * Find extended capability
+ *
+ * @v xhci		xHCI device
+ * @v id		Capability ID
+ * @v offset		Offset to previous extended capability instance, or zero
+ * @ret offset		Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
+					       unsigned int id,
+					       unsigned int offset ) {
+	uint32_t xecp;
+	unsigned int next;
+
+	/* Locate the extended capability */
+	while ( 1 ) {
+
+		/* Locate first or next capability as applicable */
+		if ( offset ) {
+			xecp = readl ( xhci->cap + offset );
+			next = XHCI_XECP_NEXT ( xecp );
+		} else {
+			next = xhci->xecp;
+		}
+		if ( ! next )
+			return 0;
+		offset += next;
+
+		/* Check if this is the requested capability */
+		xecp = readl ( xhci->cap + offset );
+		if ( XHCI_XECP_ID ( xecp ) == id )
+			return offset;
+	}
+}
+
+/**
+ * Write potentially 64-bit register
+ *
+ * @v xhci		xHCI device
+ * @v value		Value
+ * @v reg		Register address
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
+
+	/* If this is a 32-bit build, then this can never fail
+	 * (allowing the compiler to optimise out the error path).
+	 */
+	if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
+		writel ( value, reg );
+		writel ( 0, ( reg + sizeof ( uint32_t ) ) );
+		return 0;
+	}
+
+	/* If the device does not support 64-bit addresses and this
+	 * address is outside the 32-bit address space, then fail.
+	 */
+	if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
+		DBGC ( xhci, "XHCI %s cannot access address %lx\n",
+		       xhci->name, value );
+		return -ENOTSUP;
+	}
+
+	/* If this is a 64-bit build, then writeq() is available */
+	writeq ( value, reg );
+	return 0;
+}
+
+/**
+ * Calculate buffer alignment
+ *
+ * @v len		Length
+ * @ret align		Buffer alignment
+ *
+ * Determine alignment required for a buffer which must be aligned to
+ * at least XHCI_MIN_ALIGN and which must not cross a page boundary.
+ */
+static inline size_t xhci_align ( size_t len ) {
+	size_t align;
+
+	/* Align to own length (rounded up to a power of two) */
+	align = ( 1 << fls ( len - 1 ) );
+
+	/* Round up to XHCI_MIN_ALIGN if needed */
+	if ( align < XHCI_MIN_ALIGN )
+		align = XHCI_MIN_ALIGN;
+
+	return align;
+}
+
+/**
+ * Calculate device context offset
+ *
+ * @v xhci		xHCI device
+ * @v ctx		Context index
+ */
+static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
+						  unsigned int ctx ) {
+
+	return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
+}
+
+/**
+ * Calculate input context offset
+ *
+ * @v xhci		xHCI device
+ * @v ctx		Context index
+ */
+static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
+						 unsigned int ctx ) {
+
+	return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
+}
+
+/******************************************************************************
+ *
+ * Diagnostics
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Dump host controller registers
+ *
+ * @v xhci		xHCI device
+ */
+static inline void xhci_dump ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	uint32_t pagesize;
+	uint32_t dnctrl;
+	uint32_t config;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return;
+
+	/* Dump USBCMD */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	DBGC ( xhci, "XHCI %s USBCMD %08x%s%s\n", xhci->name, usbcmd,
+	       ( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
+	       ( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
+
+	/* Dump USBSTS */
+	usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+	DBGC ( xhci, "XHCI %s USBSTS %08x%s\n", xhci->name, usbsts,
+	       ( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
+
+	/* Dump PAGESIZE */
+	pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
+	DBGC ( xhci, "XHCI %s PAGESIZE %08x\n", xhci->name, pagesize );
+
+	/* Dump DNCTRL */
+	dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
+	DBGC ( xhci, "XHCI %s DNCTRL %08x\n", xhci->name, dnctrl );
+
+	/* Dump CONFIG */
+	config = readl ( xhci->op + XHCI_OP_CONFIG );
+	DBGC ( xhci, "XHCI %s CONFIG %08x\n", xhci->name, config );
+}
+
+/**
+ * Dump port registers
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ */
+static inline void xhci_dump_port ( struct xhci_device *xhci,
+				    unsigned int port ) {
+	uint32_t portsc;
+	uint32_t portpmsc;
+	uint32_t portli;
+	uint32_t porthlpmc;
+
+	/* Do nothing unless debugging is enabled */
+	if ( ! DBG_LOG )
+		return;
+
+	/* Dump PORTSC */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTSC %08x%s%s%s%s psiv=%d\n",
+	       xhci->name, port, portsc,
+	       ( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
+	       ( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
+	       XHCI_PORTSC_PSIV ( portsc ) );
+
+	/* Dump PORTPMSC */
+	portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTPMSC %08x\n", xhci->name, port, portpmsc );
+
+	/* Dump PORTLI */
+	portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTLI %08x\n", xhci->name, port, portli );
+
+	/* Dump PORTHLPMC */
+	porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
+	DBGC ( xhci, "XHCI %s-%d PORTHLPMC %08x\n",
+	       xhci->name, port, porthlpmc );
+}
+
+/******************************************************************************
+ *
+ * USB legacy support
+ *
+ ******************************************************************************
+ */
+
+/** Prevent the release of ownership back to BIOS */
+static int xhci_legacy_prevent_release;
+
+/**
+ * Initialise USB legacy support
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_init ( struct xhci_device *xhci ) {
+	unsigned int legacy;
+	uint8_t bios;
+
+	/* Locate USB legacy support capability (if present) */
+	legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
+	if ( ! legacy ) {
+		/* Not an error; capability may not be present */
+		DBGC ( xhci, "XHCI %s has no USB legacy support capability\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Check if legacy USB support is enabled */
+	bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
+	if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+		/* Not an error; already owned by OS */
+		DBGC ( xhci, "XHCI %s USB legacy support already disabled\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Record presence of USB legacy support capability */
+	xhci->legacy = legacy;
+}
+
+/**
+ * Claim ownership from BIOS
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_claim ( struct xhci_device *xhci ) {
+	uint32_t ctlsts;
+	uint8_t bios;
+	unsigned int i;
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! xhci->legacy )
+		return;
+
+	/* Claim ownership */
+	writeb ( XHCI_USBLEGSUP_OS_OWNED,
+		 xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+
+	/* Wait for BIOS to release ownership */
+	for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if BIOS has released ownership */
+		bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
+		if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
+			DBGC ( xhci, "XHCI %s claimed ownership from BIOS\n",
+			       xhci->name );
+			ctlsts = readl ( xhci->cap + xhci->legacy +
+					 XHCI_USBLEGSUP_CTLSTS );
+			if ( ctlsts ) {
+				DBGC ( xhci, "XHCI %s warning: BIOS retained "
+				       "SMIs: %08x\n", xhci->name, ctlsts );
+			}
+			return;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* BIOS did not release ownership.  Claim it forcibly by
+	 * disabling all SMIs.
+	 */
+	DBGC ( xhci, "XHCI %s could not claim ownership from BIOS: forcibly "
+	       "disabling SMIs\n", xhci->name );
+	writel ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_CTLSTS );
+}
+
+/**
+ * Release ownership back to BIOS
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_legacy_release ( struct xhci_device *xhci ) {
+
+	/* Do nothing unless legacy support capability is present */
+	if ( ! xhci->legacy )
+		return;
+
+	/* Do nothing if releasing ownership is prevented */
+	if ( xhci_legacy_prevent_release ) {
+		DBGC ( xhci, "XHCI %s not releasing ownership to BIOS\n",
+		       xhci->name );
+		return;
+	}
+
+	/* Release ownership */
+	writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
+	DBGC ( xhci, "XHCI %s released ownership to BIOS\n", xhci->name );
+}
+
+/******************************************************************************
+ *
+ * Supported protocols
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Transcribe port speed (for debugging)
+ *
+ * @v psi		Protocol speed ID
+ * @ret speed		Transcribed speed
+ */
+static inline const char * xhci_speed_name ( uint32_t psi ) {
+	static const char *exponents[4] = { "", "k", "M", "G" };
+	static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
+	unsigned int mantissa;
+	unsigned int exponent;
+
+	/* Extract mantissa and exponent */
+	mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+	exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+
+	/* Transcribe speed */
+	snprintf ( buf, sizeof ( buf ), "%d%sbps",
+		   mantissa, exponents[exponent] );
+	return buf;
+}
+
+/**
+ * Find supported protocol extended capability for a port
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret supported	Offset to extended capability, or zero if not found
+ */
+static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
+					      unsigned int port ) {
+	unsigned int supported = 0;
+	unsigned int offset;
+	unsigned int count;
+	uint32_t ports;
+
+	/* Iterate over all supported protocol structures */
+	while ( ( supported = xhci_extended_capability ( xhci,
+							 XHCI_XECP_ID_SUPPORTED,
+							 supported ) ) ) {
+
+		/* Determine port range */
+		ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+		offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
+		count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
+
+		/* Check if port lies within this range */
+		if ( ( port - offset ) < count )
+			return supported;
+	}
+
+	DBGC ( xhci, "XHCI %s-%d has no supported protocol\n",
+	       xhci->name, port );
+	return 0;
+}
+
+/**
+ * Find port protocol
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret protocol	USB protocol, or zero if not found
+ */
+static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
+					 unsigned int port ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	union {
+		uint32_t raw;
+		char text[5];
+	} name;
+	unsigned int protocol;
+	unsigned int type;
+	unsigned int psic;
+	unsigned int psiv;
+	unsigned int i;
+	uint32_t revision;
+	uint32_t ports;
+	uint32_t slot;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return 0;
+
+	/* Determine protocol version */
+	revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
+	protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
+
+	/* Describe port protocol */
+	if ( DBG_EXTRA ) {
+		name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
+						 XHCI_SUPPORTED_NAME ) );
+		name.text[4] = '\0';
+		slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+		type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+		DBGC2 ( xhci, "XHCI %s-%d %sv%04x type %d",
+			xhci->name, port, name.text, protocol, type );
+		ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+		psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+		if ( psic ) {
+			DBGC2 ( xhci, " speeds" );
+			for ( i = 0 ; i < psic ; i++ ) {
+				psi = readl ( xhci->cap + supported +
+					      XHCI_SUPPORTED_PSI ( i ) );
+				psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+				DBGC2 ( xhci, " %d:%s", psiv,
+					xhci_speed_name ( psi ) );
+			}
+		}
+		if ( xhci->quirks & XHCI_BAD_PSIV )
+			DBGC2 ( xhci, " (ignored)" );
+		DBGC2 ( xhci, "\n" );
+	}
+
+	return protocol;
+}
+
+/**
+ * Find port slot type
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @ret type		Slot type, or negative error
+ */
+static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int type;
+	uint32_t slot;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get slot type */
+	slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
+	type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
+
+	return type;
+}
+
+/**
+ * Find port speed
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @v psiv		Protocol speed ID value
+ * @ret speed		Port speed, or negative error
+ */
+static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
+			     unsigned int psiv ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int psic;
+	unsigned int mantissa;
+	unsigned int exponent;
+	unsigned int speed;
+	unsigned int i;
+	uint32_t ports;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get protocol speed ID count */
+	ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+	psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+	/* Use the default mappings if applicable */
+	if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
+		switch ( psiv ) {
+		case XHCI_SPEED_LOW :	return USB_SPEED_LOW;
+		case XHCI_SPEED_FULL :	return USB_SPEED_FULL;
+		case XHCI_SPEED_HIGH :	return USB_SPEED_HIGH;
+		case XHCI_SPEED_SUPER :	return USB_SPEED_SUPER;
+		default:
+			DBGC ( xhci, "XHCI %s-%d non-standard PSI value %d\n",
+			       xhci->name, port, psiv );
+			return -ENOTSUP;
+		}
+	}
+
+	/* Iterate over PSI dwords looking for a match */
+	for ( i = 0 ; i < psic ; i++ ) {
+		psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+		if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
+			mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+			exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+			speed = USB_SPEED ( mantissa, exponent );
+			return speed;
+		}
+	}
+
+	DBGC ( xhci, "XHCI %s-%d spurious PSI value %d\n",
+	       xhci->name, port, psiv );
+	return -ENOENT;
+}
+
+/**
+ * Find protocol speed ID value
+ *
+ * @v xhci		xHCI device
+ * @v port		Port number
+ * @v speed		USB speed
+ * @ret psiv		Protocol speed ID value, or negative error
+ */
+static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
+			    unsigned int speed ) {
+	unsigned int supported = xhci_supported_protocol ( xhci, port );
+	unsigned int psic;
+	unsigned int mantissa;
+	unsigned int exponent;
+	unsigned int psiv;
+	unsigned int i;
+	uint32_t ports;
+	uint32_t psi;
+
+	/* Fail if there is no supported protocol */
+	if ( ! supported )
+		return -ENOTSUP;
+
+	/* Get protocol speed ID count */
+	ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
+	psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
+
+	/* Use the default mappings if applicable */
+	if ( ( psic == 0 ) || ( xhci->quirks & XHCI_BAD_PSIV ) ) {
+		switch ( speed ) {
+		case USB_SPEED_LOW :	return XHCI_SPEED_LOW;
+		case USB_SPEED_FULL :	return XHCI_SPEED_FULL;
+		case USB_SPEED_HIGH :	return XHCI_SPEED_HIGH;
+		case USB_SPEED_SUPER :	return XHCI_SPEED_SUPER;
+		default:
+			DBGC ( xhci, "XHCI %s-%d non-standard speed %d\n",
+			       xhci->name, port, speed );
+			return -ENOTSUP;
+		}
+	}
+
+	/* Iterate over PSI dwords looking for a match */
+	for ( i = 0 ; i < psic ; i++ ) {
+		psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
+		mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
+		exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
+		if ( speed == USB_SPEED ( mantissa, exponent ) ) {
+			psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
+			return psiv;
+		}
+	}
+
+	DBGC ( xhci, "XHCI %s-%d unrepresentable speed %#x\n",
+	       xhci->name, port, speed );
+	return -ENOENT;
+}
+
+/******************************************************************************
+ *
+ * Device context base address array
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate device context base address array
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
+	size_t len;
+	physaddr_t dcbaap;
+	int rc;
+
+	/* Allocate and initialise structure.  Must be at least
+	 * 64-byte aligned and must not cross a page boundary, so
+	 * align on its own size (rounded up to a power of two and
+	 * with a minimum of 64 bytes).
+	 */
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+	xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! xhci->dcbaa ) {
+		DBGC ( xhci, "XHCI %s could not allocate DCBAA\n", xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset ( xhci->dcbaa, 0, len );
+
+	/* Program DCBAA pointer */
+	dcbaap = virt_to_phys ( xhci->dcbaa );
+	if ( ( rc = xhci_writeq ( xhci, dcbaap,
+				  xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
+		goto err_writeq;
+
+	DBGC2 ( xhci, "XHCI %s DCBAA at [%08lx,%08lx)\n",
+		xhci->name, dcbaap, ( dcbaap + len ) );
+	return 0;
+
+ err_writeq:
+	free_dma ( xhci->dcbaa, len );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free device context base address array
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
+	size_t len;
+	unsigned int i;
+
+	/* Sanity check */
+	for ( i = 0 ; i <= xhci->slots ; i++ )
+		assert ( xhci->dcbaa[i] == 0 );
+
+	/* Clear DCBAA pointer */
+	xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
+
+	/* Free DCBAA */
+	len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
+	free_dma ( xhci->dcbaa, len );
+}
+
+/******************************************************************************
+ *
+ * Scratchpad buffers
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate scratchpad buffers
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
+	size_t array_len;
+	size_t len;
+	physaddr_t phys;
+	unsigned int i;
+	int rc;
+
+	/* Do nothing if no scratchpad buffers are used */
+	if ( ! xhci->scratchpads )
+		return 0;
+
+	/* Allocate scratchpads */
+	len = ( xhci->scratchpads * xhci->pagesize );
+	xhci->scratchpad = umalloc ( len );
+	if ( ! xhci->scratchpad ) {
+		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffers\n",
+		       xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset_user ( xhci->scratchpad, 0, 0, len );
+
+	/* Allocate scratchpad array */
+	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+	xhci->scratchpad_array =
+		malloc_dma ( array_len, xhci_align ( array_len ) );
+	if ( ! xhci->scratchpad_array ) {
+		DBGC ( xhci, "XHCI %s could not allocate scratchpad buffer "
+		       "array\n", xhci->name );
+		rc = -ENOMEM;
+		goto err_alloc_array;
+	}
+
+	/* Populate scratchpad array */
+	for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
+		phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
+		xhci->scratchpad_array[i] = phys;
+	}
+
+	/* Set scratchpad array pointer */
+	assert ( xhci->dcbaa != NULL );
+	xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
+
+	DBGC2 ( xhci, "XHCI %s scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
+		xhci->name, user_to_phys ( xhci->scratchpad, 0 ),
+		user_to_phys ( xhci->scratchpad, len ),
+		virt_to_phys ( xhci->scratchpad_array ),
+		( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
+	return 0;
+
+	free_dma ( xhci->scratchpad_array, array_len );
+ err_alloc_array:
+	ufree ( xhci->scratchpad );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Free scratchpad buffers
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
+	size_t array_len;
+
+	/* Do nothing if no scratchpad buffers are used */
+	if ( ! xhci->scratchpads )
+		return;
+
+	/* Clear scratchpad array pointer */
+	assert ( xhci->dcbaa != NULL );
+	xhci->dcbaa[0] = 0;
+
+	/* Free scratchpad array */
+	array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
+	free_dma ( xhci->scratchpad_array, array_len );
+
+	/* Free scratchpads */
+	ufree ( xhci->scratchpad );
+}
+
+/******************************************************************************
+ *
+ * Run / stop / reset
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Start xHCI device
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_run ( struct xhci_device *xhci ) {
+	uint32_t config;
+	uint32_t usbcmd;
+
+	/* Configure number of device slots */
+	config = readl ( xhci->op + XHCI_OP_CONFIG );
+	config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
+	config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
+	writel ( config, xhci->op + XHCI_OP_CONFIG );
+
+	/* Set run/stop bit */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	usbcmd |= XHCI_USBCMD_RUN;
+	writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+}
+
+/**
+ * Stop xHCI device
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_stop ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	uint32_t usbsts;
+	unsigned int i;
+
+	/* Clear run/stop bit */
+	usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+	usbcmd &= ~XHCI_USBCMD_RUN;
+	writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
+
+	/* Wait for device to stop */
+	for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if device is stopped */
+		usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
+		if ( usbsts & XHCI_USBSTS_HCH )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s timed out waiting for stop\n", xhci->name );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Reset xHCI device
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_reset ( struct xhci_device *xhci ) {
+	uint32_t usbcmd;
+	unsigned int i;
+	int rc;
+
+	/* The xHCI specification states that resetting a running
+	 * device may result in undefined behaviour, so try stopping
+	 * it first.
+	 */
+	if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
+		/* Ignore errors and attempt to reset the device anyway */
+	}
+
+	/* Reset device */
+	writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
+
+	/* Wait for reset to complete */
+	for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check if reset is complete */
+		usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
+		if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s timed out waiting for reset\n", xhci->name );
+	return -ETIMEDOUT;
+}
+
+/******************************************************************************
+ *
+ * Transfer request blocks
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate transfer request block ring
+ *
+ * @v xhci		xHCI device
+ * @v ring		TRB ring
+ * @v shift		Ring size (log2)
+ * @v slot		Device slot
+ * @v target		Doorbell target
+ * @v stream		Doorbell stream ID
+ * @ret rc		Return status code
+ */
+static int xhci_ring_alloc ( struct xhci_device *xhci,
+			     struct xhci_trb_ring *ring,
+			     unsigned int shift, unsigned int slot,
+			     unsigned int target, unsigned int stream ) {
+	struct xhci_trb_link *link;
+	unsigned int count;
+	int rc;
+
+	/* Sanity check */
+	assert ( shift > 0 );
+
+	/* Initialise structure */
+	memset ( ring, 0, sizeof ( *ring ) );
+	ring->shift = shift;
+	count = ( 1U << shift );
+	ring->mask = ( count - 1 );
+	ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
+	ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
+	ring->dbval = XHCI_DBVAL ( target, stream );
+
+	/* Allocate I/O buffers */
+	ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
+	if ( ! ring->iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc_iobuf;
+	}
+
+	/* Allocate TRBs */
+	ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
+	if ( ! ring->trb ) {
+		rc = -ENOMEM;
+		goto err_alloc_trb;
+	}
+	memset ( ring->trb, 0, ring->len );
+
+	/* Initialise Link TRB */
+	link = &ring->trb[count].link;
+	link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
+	link->flags = XHCI_TRB_TC;
+	link->type = XHCI_TRB_LINK;
+	ring->link = link;
+
+	return 0;
+
+	free_dma ( ring->trb, ring->len );
+ err_alloc_trb:
+	free ( ring->iobuf );
+ err_alloc_iobuf:
+	return rc;
+}
+
+/**
+ * Reset transfer request block ring
+ *
+ * @v ring		TRB ring
+ */
+static void xhci_ring_reset ( struct xhci_trb_ring *ring ) {
+	unsigned int count = ( 1U << ring->shift );
+
+	/* Reset producer and consumer counters */
+	ring->prod = 0;
+	ring->cons = 0;
+
+	/* Reset TRBs (except Link TRB) */
+	memset ( ring->trb, 0, ( count * sizeof ( ring->trb[0] ) ) );
+}
+
+/**
+ * Free transfer request block ring
+ *
+ * @v ring		TRB ring
+ */
+static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
+	unsigned int count = ( 1U << ring->shift );
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( ring->cons == ring->prod );
+	for ( i = 0 ; i < count ; i++ )
+		assert ( ring->iobuf[i] == NULL );
+
+	/* Free TRBs */
+	free_dma ( ring->trb, ring->len );
+
+	/* Free I/O buffers */
+	free ( ring->iobuf );
+}
+
+/**
+ * Enqueue a transfer request block
+ *
+ * @v ring		TRB ring
+ * @v iobuf		I/O buffer (if any)
+ * @v trb		Transfer request block (with empty Cycle flag)
+ * @ret rc		Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
+			  const union xhci_trb *trb ) {
+	union xhci_trb *dest;
+	unsigned int prod;
+	unsigned int mask;
+	unsigned int index;
+	unsigned int cycle;
+
+	/* Sanity check */
+	assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
+
+	/* Fail if ring is full */
+	if ( ! xhci_ring_remaining ( ring ) )
+		return -ENOBUFS;
+
+	/* Update producer counter (and link TRB, if applicable) */
+	prod = ring->prod++;
+	mask = ring->mask;
+	cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
+	index = ( prod & mask );
+	if ( index == 0 )
+		ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
+
+	/* Record I/O buffer */
+	ring->iobuf[index] = iobuf;
+
+	/* Enqueue TRB */
+	dest = &ring->trb[index];
+	dest->template.parameter = trb->template.parameter;
+	dest->template.status = trb->template.status;
+	wmb();
+	dest->template.control = ( trb->template.control |
+				   cpu_to_le32 ( cycle ) );
+
+	return 0;
+}
+
+/**
+ * Dequeue a transfer request block
+ *
+ * @v ring		TRB ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
+	struct io_buffer *iobuf;
+	unsigned int cons;
+	unsigned int mask;
+	unsigned int index;
+
+	/* Sanity check */
+	assert ( xhci_ring_fill ( ring ) != 0 );
+
+	/* Update consumer counter */
+	cons = ring->cons++;
+	mask = ring->mask;
+	index = ( cons & mask );
+
+	/* Retrieve I/O buffer */
+	iobuf = ring->iobuf[index];
+	ring->iobuf[index] = NULL;
+
+	return iobuf;
+}
+
+/**
+ * Enqueue multiple transfer request blocks
+ *
+ * @v ring		TRB ring
+ * @v iobuf		I/O buffer
+ * @v trbs		Transfer request blocks (with empty Cycle flag)
+ * @v count		Number of transfer request blocks
+ * @ret rc		Return status code
+ *
+ * This operation does not implicitly ring the doorbell register.
+ */
+static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
+				struct io_buffer *iobuf,
+				const union xhci_trb *trbs,
+				unsigned int count ) {
+	const union xhci_trb *trb = trbs;
+	int rc;
+
+	/* Sanity check */
+	assert ( iobuf != NULL );
+
+	/* Fail if ring does not have sufficient space */
+	if ( xhci_ring_remaining ( ring ) < count )
+		return -ENOBUFS;
+
+	/* Enqueue each TRB, recording the I/O buffer with the final TRB */
+	while ( count-- ) {
+		rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
+		assert ( rc == 0 ); /* Should never be able to fail */
+	}
+
+	return 0;
+}
+
+/**
+ * Dequeue multiple transfer request blocks
+ *
+ * @v ring		TRB ring
+ * @ret iobuf		I/O buffer
+ */
+static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
+	struct io_buffer *iobuf;
+
+	/* Dequeue TRBs until we reach the final TRB for an I/O buffer */
+	do {
+		iobuf = xhci_dequeue ( ring );
+	} while ( iobuf == NULL );
+
+	return iobuf;
+}
+
+/**
+ * Ring doorbell register
+ *
+ * @v ring		TRB ring
+ */
+static inline __attribute__ (( always_inline )) void
+xhci_doorbell ( struct xhci_trb_ring *ring ) {
+
+	wmb();
+	writel ( ring->dbval, ring->db );
+}
+
+/******************************************************************************
+ *
+ * Command and event rings
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Allocate command ring
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_command_alloc ( struct xhci_device *xhci ) {
+	physaddr_t crp;
+	int rc;
+
+	/* Allocate TRB ring */
+	if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
+				      0, 0, 0 ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Program command ring control register */
+	crp = virt_to_phys ( xhci->command.trb );
+	if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
+				  xhci->op + XHCI_OP_CRCR ) ) != 0 )
+		goto err_writeq;
+
+	DBGC2 ( xhci, "XHCI %s CRCR at [%08lx,%08lx)\n",
+		xhci->name, crp, ( crp + xhci->command.len ) );
+	return 0;
+
+ err_writeq:
+	xhci_ring_free ( &xhci->command );
+ err_ring_alloc:
+	return rc;
+}
+
+/**
+ * Free command ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_command_free ( struct xhci_device *xhci ) {
+
+	/* Sanity check */
+	assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+
+	/* Clear command ring control register */
+	xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
+
+	/* Free TRB ring */
+	xhci_ring_free ( &xhci->command );
+}
+
+/**
+ * Allocate event ring
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static int xhci_event_alloc ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	unsigned int count;
+	size_t len;
+	int rc;
+
+	/* Allocate event ring */
+	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+	len = ( count * sizeof ( event->trb[0] ) );
+	event->trb = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! event->trb ) {
+		rc = -ENOMEM;
+		goto err_alloc_trb;
+	}
+	memset ( event->trb, 0, len );
+
+	/* Allocate event ring segment table */
+	event->segment = malloc_dma ( sizeof ( event->segment[0] ),
+				      xhci_align ( sizeof (event->segment[0])));
+	if ( ! event->segment ) {
+		rc = -ENOMEM;
+		goto err_alloc_segment;
+	}
+	memset ( event->segment, 0, sizeof ( event->segment[0] ) );
+	event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
+	event->segment[0].count = cpu_to_le32 ( count );
+
+	/* Program event ring registers */
+	writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
+				  xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
+		goto err_writeq_erdp;
+	if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
+				  xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
+		goto err_writeq_erstba;
+
+	DBGC2 ( xhci, "XHCI %s event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
+		xhci->name, virt_to_phys ( event->trb ),
+		( virt_to_phys ( event->trb ) + len ),
+		virt_to_phys ( event->segment ),
+		( virt_to_phys ( event->segment ) +
+		  sizeof (event->segment[0] ) ) );
+	return 0;
+
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+ err_writeq_erstba:
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+ err_writeq_erdp:
+	free_dma ( event->trb, len );
+ err_alloc_segment:
+	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+ err_alloc_trb:
+	return rc;
+}
+
+/**
+ * Free event ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_event_free ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	unsigned int count;
+	size_t len;
+
+	/* Clear event ring registers */
+	writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
+	xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
+
+	/* Free event ring segment table */
+	free_dma ( event->segment, sizeof ( event->segment[0] ) );
+
+	/* Free event ring */
+	count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
+	len = ( count * sizeof ( event->trb[0] ) );
+	free_dma ( event->trb, len );
+}
+
+/**
+ * Handle transfer event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Transfer event TRB
+ */
+static void xhci_transfer ( struct xhci_device *xhci,
+			    struct xhci_trb_transfer *trb ) {
+	struct xhci_slot *slot;
+	struct xhci_endpoint *endpoint;
+	struct io_buffer *iobuf;
+	int rc;
+
+	/* Profile transfer events */
+	profile_start ( &xhci_transfer_profiler );
+
+	/* Identify slot */
+	if ( ( trb->slot > xhci->slots ) ||
+	     ( ( slot = xhci->slot[trb->slot] ) == NULL ) ) {
+		DBGC ( xhci, "XHCI %s transfer event invalid slot %d:\n",
+		       xhci->name, trb->slot );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Identify endpoint */
+	if ( ( trb->endpoint > XHCI_CTX_END ) ||
+	     ( ( endpoint = slot->endpoint[trb->endpoint] ) == NULL ) ) {
+		DBGC ( xhci, "XHCI %s slot %d transfer event invalid epid "
+		       "%d:\n", xhci->name, slot->id, trb->endpoint );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Dequeue TRB(s) */
+	iobuf = xhci_dequeue_multi ( &endpoint->ring );
+	assert ( iobuf != NULL );
+
+	/* Check for errors */
+	if ( ! ( ( trb->code == XHCI_CMPLT_SUCCESS ) ||
+		 ( trb->code == XHCI_CMPLT_SHORT ) ) ) {
+
+		/* Construct error */
+		rc = -ECODE ( trb->code );
+		DBGC ( xhci, "XHCI %s slot %d ctx %d failed (code %d): %s\n",
+		       xhci->name, slot->id, endpoint->ctx, trb->code,
+		       strerror ( rc ) );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+
+		/* Sanity check */
+		assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
+			 != XHCI_ENDPOINT_RUNNING );
+
+		/* Report failure to USB core */
+		usb_complete_err ( endpoint->ep, iobuf, rc );
+		return;
+	}
+
+	/* Record actual transfer size */
+	iob_unput ( iobuf, le16_to_cpu ( trb->residual ) );
+
+	/* Sanity check (for successful completions only) */
+	assert ( xhci_ring_consumed ( &endpoint->ring ) ==
+		 le64_to_cpu ( trb->transfer ) );
+
+	/* Report completion to USB core */
+	usb_complete ( endpoint->ep, iobuf );
+	profile_stop ( &xhci_transfer_profiler );
+}
+
+/**
+ * Handle command completion event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Command completion event
+ */
+static void xhci_complete ( struct xhci_device *xhci,
+			    struct xhci_trb_complete *trb ) {
+	int rc;
+
+	/* Ignore "command ring stopped" notifications */
+	if ( trb->code == XHCI_CMPLT_CMD_STOPPED ) {
+		DBGC2 ( xhci, "XHCI %s command ring stopped\n", xhci->name );
+		return;
+	}
+
+	/* Ignore unexpected completions */
+	if ( ! xhci->pending ) {
+		rc = -ECODE ( trb->code );
+		DBGC ( xhci, "XHCI %s unexpected completion (code %d): %s\n",
+		       xhci->name, trb->code, strerror ( rc ) );
+		DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+		return;
+	}
+
+	/* Dequeue command TRB */
+	xhci_dequeue ( &xhci->command );
+
+	/* Sanity check */
+	assert ( xhci_ring_consumed ( &xhci->command ) ==
+		 le64_to_cpu ( trb->command ) );
+
+	/* Record completion */
+	memcpy ( xhci->pending, trb, sizeof ( *xhci->pending ) );
+	xhci->pending = NULL;
+}
+
+/**
+ * Handle port status event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Port status event
+ */
+static void xhci_port_status ( struct xhci_device *xhci,
+			       struct xhci_trb_port_status *trb ) {
+	struct usb_port *port = usb_port ( xhci->bus->hub, trb->port );
+	uint32_t portsc;
+
+	/* Sanity check */
+	assert ( ( trb->port > 0 ) && ( trb->port <= xhci->ports ) );
+
+	/* Record disconnections and clear changes */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( trb->port ) );
+	port->disconnected |= ( portsc & XHCI_PORTSC_CSC );
+	portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( trb->port ) );
+
+	/* Report port status change */
+	usb_port_changed ( port );
+}
+
+/**
+ * Handle host controller event
+ *
+ * @v xhci		xHCI device
+ * @v trb		Host controller event
+ */
+static void xhci_host_controller ( struct xhci_device *xhci,
+				   struct xhci_trb_host_controller *trb ) {
+	int rc;
+
+	/* Construct error */
+	rc = -ECODE ( trb->code );
+	DBGC ( xhci, "XHCI %s host controller event (code %d): %s\n",
+	       xhci->name, trb->code, strerror ( rc ) );
+}
+
+/**
+ * Poll event ring
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_event_poll ( struct xhci_device *xhci ) {
+	struct xhci_event_ring *event = &xhci->event;
+	union xhci_trb *trb;
+	unsigned int shift = XHCI_EVENT_TRBS_LOG2;
+	unsigned int count = ( 1 << shift );
+	unsigned int mask = ( count - 1 );
+	unsigned int consumed;
+	unsigned int type;
+
+	/* Poll for events */
+	profile_start ( &xhci_event_profiler );
+	for ( consumed = 0 ; ; consumed++ ) {
+
+		/* Stop if we reach an empty TRB */
+		rmb();
+		trb = &event->trb[ event->cons & mask ];
+		if ( ! ( ( trb->common.flags ^
+			   ( event->cons >> shift ) ) & XHCI_TRB_C ) )
+			break;
+
+		/* Handle TRB */
+		type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
+		switch ( type ) {
+
+		case XHCI_TRB_TRANSFER :
+			xhci_transfer ( xhci, &trb->transfer );
+			break;
+
+		case XHCI_TRB_COMPLETE :
+			xhci_complete ( xhci, &trb->complete );
+			break;
+
+		case XHCI_TRB_PORT_STATUS:
+			xhci_port_status ( xhci, &trb->port );
+			break;
+
+		case XHCI_TRB_HOST_CONTROLLER:
+			xhci_host_controller ( xhci, &trb->host );
+			break;
+
+		default:
+			DBGC ( xhci, "XHCI %s unrecognised event %#x\n:",
+			       xhci->name, event->cons );
+			DBGC_HDA ( xhci, virt_to_phys ( trb ),
+				   trb, sizeof ( *trb ) );
+			break;
+		}
+
+		/* Consume this TRB */
+		event->cons++;
+	}
+
+	/* Update dequeue pointer if applicable */
+	if ( consumed ) {
+		xhci_writeq ( xhci, virt_to_phys ( trb ),
+			      xhci->run + XHCI_RUN_ERDP ( 0 ) );
+		profile_stop ( &xhci_event_profiler );
+	}
+}
+
+/**
+ * Abort command
+ *
+ * @v xhci		xHCI device
+ */
+static void xhci_abort ( struct xhci_device *xhci ) {
+	physaddr_t crp;
+
+	/* Abort the command */
+	DBGC2 ( xhci, "XHCI %s aborting command\n", xhci->name );
+	xhci_writeq ( xhci, XHCI_CRCR_CA, xhci->op + XHCI_OP_CRCR );
+
+	/* Allow time for command to abort */
+	mdelay ( XHCI_COMMAND_ABORT_DELAY_MS );
+
+	/* Sanity check */
+	assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
+
+	/* Consume (and ignore) any final command status */
+	xhci_event_poll ( xhci );
+
+	/* Reset the command ring control register */
+	xhci_ring_reset ( &xhci->command );
+	crp = virt_to_phys ( xhci->command.trb );
+	xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ), xhci->op + XHCI_OP_CRCR );
+}
+
+/**
+ * Issue command and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @v trb		Transfer request block (with empty Cycle flag)
+ * @ret rc		Return status code
+ *
+ * On a successful completion, the TRB will be overwritten with the
+ * completion.
+ */
+static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
+	struct xhci_trb_complete *complete = &trb->complete;
+	unsigned int i;
+	int rc;
+
+	/* Record the pending command */
+	xhci->pending = trb;
+
+	/* Enqueue the command */
+	if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
+		goto err_enqueue;
+
+	/* Ring the command doorbell */
+	xhci_doorbell ( &xhci->command );
+
+	/* Wait for the command to complete */
+	for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
+
+		/* Poll event ring */
+		xhci_event_poll ( xhci );
+
+		/* Check for completion */
+		if ( ! xhci->pending ) {
+			if ( complete->code != XHCI_CMPLT_SUCCESS ) {
+				rc = -ECODE ( complete->code );
+				DBGC ( xhci, "XHCI %s command failed (code "
+				       "%d): %s\n", xhci->name, complete->code,
+				       strerror ( rc ) );
+				DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
+				return rc;
+			}
+			return 0;
+		}
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	/* Timeout */
+	DBGC ( xhci, "XHCI %s timed out waiting for completion\n", xhci->name );
+	rc = -ETIMEDOUT;
+
+	/* Abort command */
+	xhci_abort ( xhci );
+
+ err_enqueue:
+	xhci->pending = NULL;
+	return rc;
+}
+
+/**
+ * Issue NOP and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @ret rc		Return status code
+ */
+static inline int xhci_nop ( struct xhci_device *xhci ) {
+	union xhci_trb trb;
+	struct xhci_trb_common *nop = &trb.common;
+	int rc;
+
+	/* Construct command */
+	memset ( nop, 0, sizeof ( *nop ) );
+	nop->flags = XHCI_TRB_IOC;
+	nop->type = XHCI_TRB_NOP_CMD;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enable slot
+ *
+ * @v xhci		xHCI device
+ * @v type		Slot type
+ * @ret slot		Device slot ID, or negative error
+ */
+static inline int xhci_enable_slot ( struct xhci_device *xhci,
+				     unsigned int type ) {
+	union xhci_trb trb;
+	struct xhci_trb_enable_slot *enable = &trb.enable;
+	struct xhci_trb_complete *enabled = &trb.complete;
+	unsigned int slot;
+	int rc;
+
+	/* Construct command */
+	memset ( enable, 0, sizeof ( *enable ) );
+	enable->slot = type;
+	enable->type = XHCI_TRB_ENABLE_SLOT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s could not enable new slot: %s\n",
+		       xhci->name, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Extract slot number */
+	slot = enabled->slot;
+
+	DBGC2 ( xhci, "XHCI %s slot %d enabled\n", xhci->name, slot );
+	return slot;
+}
+
+/**
+ * Disable slot
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @ret rc		Return status code
+ */
+static inline int xhci_disable_slot ( struct xhci_device *xhci,
+				      unsigned int slot ) {
+	union xhci_trb trb;
+	struct xhci_trb_disable_slot *disable = &trb.disable;
+	int rc;
+
+	/* Construct command */
+	memset ( disable, 0, sizeof ( *disable ) );
+	disable->type = XHCI_TRB_DISABLE_SLOT;
+	disable->slot = slot;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s could not disable slot %d: %s\n",
+		       xhci->name, slot, strerror ( rc ) );
+		return rc;
+	}
+
+	DBGC2 ( xhci, "XHCI %s slot %d disabled\n", xhci->name, slot );
+	return 0;
+}
+
+/**
+ * Issue context-based command and wait for completion
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v type		TRB type
+ * @v populate		Input context populater
+ * @ret rc		Return status code
+ */
+static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
+			  struct xhci_endpoint *endpoint, unsigned int type,
+			  void ( * populate ) ( struct xhci_device *xhci,
+						struct xhci_slot *slot,
+						struct xhci_endpoint *endpoint,
+						void *input ) ) {
+	union xhci_trb trb;
+	struct xhci_trb_context *context = &trb.context;
+	size_t len;
+	void *input;
+	int rc;
+
+	/* Allocate an input context */
+	len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
+	input = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! input ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	memset ( input, 0, len );
+
+	/* Populate input context */
+	populate ( xhci, slot, endpoint, input );
+
+	/* Construct command */
+	memset ( context, 0, sizeof ( *context ) );
+	context->type = type;
+	context->input = cpu_to_le64 ( virt_to_phys ( input ) );
+	context->slot = slot->id;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
+		goto err_command;
+
+ err_command:
+	free_dma ( input, len );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Populate address device input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_address_device_input ( struct xhci_device *xhci,
+					struct xhci_slot *slot,
+					struct xhci_endpoint *endpoint,
+					void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Sanity checks */
+	assert ( endpoint->ctx == XHCI_CTX_EP0 );
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << XHCI_CTX_EP0 ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
+							slot->route ) );
+	slot_ctx->port = slot->port;
+	slot_ctx->tt_id = slot->tt_id;
+	slot_ctx->tt_port = slot->tt_port;
+
+	/* Populate control endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
+	ep_ctx->type = XHCI_EP_TYPE_CONTROL;
+	ep_ctx->burst = endpoint->ep->burst;
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+					XHCI_EP_DCS );
+	ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
+}
+
+/**
+ * Address device
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @ret rc		Return status code
+ */
+static inline int xhci_address_device ( struct xhci_device *xhci,
+					struct xhci_slot *slot ) {
+	struct usb_device *usb = slot->usb;
+	struct xhci_slot_context *slot_ctx;
+	int rc;
+
+	/* Assign device address */
+	if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
+				   XHCI_TRB_ADDRESS_DEVICE,
+				   xhci_address_device_input ) ) != 0 )
+		return rc;
+
+	/* Get assigned address */
+	slot_ctx = ( slot->context +
+		     xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
+	usb->address = slot_ctx->address;
+	DBGC2 ( xhci, "XHCI %s assigned address %d to %s\n",
+		xhci->name, usb->address, usb->name );
+
+	return 0;
+}
+
+/**
+ * Populate configure endpoint input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
+					    struct xhci_slot *slot,
+					    struct xhci_endpoint *endpoint,
+					    void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << endpoint->ctx ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							( slot->ports ? 1 : 0 ),
+							slot->psiv, 0 ) );
+	slot_ctx->ports = slot->ports;
+
+	/* Populate endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+	ep_ctx->interval = endpoint->interval;
+	ep_ctx->type = endpoint->type;
+	ep_ctx->burst = endpoint->ep->burst;
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+	ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
+					XHCI_EP_DCS );
+	ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
+}
+
+/**
+ * Configure endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
+					    struct xhci_slot *slot,
+					    struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Configure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_CONFIGURE_ENDPOINT,
+				   xhci_configure_endpoint_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d configured\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Populate deconfigure endpoint input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void
+xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
+				  struct xhci_slot *slot __unused,
+				  struct xhci_endpoint *endpoint,
+				  void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
+	control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							0, 0, 0 ) );
+}
+
+/**
+ * Deconfigure endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
+					      struct xhci_slot *slot,
+					      struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Deconfigure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_CONFIGURE_ENDPOINT,
+				   xhci_deconfigure_endpoint_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d deconfigured\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Populate evaluate context input context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @v input		Input context
+ */
+static void xhci_evaluate_context_input ( struct xhci_device *xhci,
+					  struct xhci_slot *slot __unused,
+					  struct xhci_endpoint *endpoint,
+					  void *input ) {
+	struct xhci_control_context *control_ctx;
+	struct xhci_slot_context *slot_ctx;
+	struct xhci_endpoint_context *ep_ctx;
+
+	/* Populate control context */
+	control_ctx = input;
+	control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
+					 ( 1 << endpoint->ctx ) );
+
+	/* Populate slot context */
+	slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
+	slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
+							0, 0, 0 ) );
+
+	/* Populate endpoint context */
+	ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
+	ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
+}
+
+/**
+ * Evaluate context
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_evaluate_context ( struct xhci_device *xhci,
+					  struct xhci_slot *slot,
+					  struct xhci_endpoint *endpoint ) {
+	int rc;
+
+	/* Configure endpoint */
+	if ( ( rc = xhci_context ( xhci, slot, endpoint,
+				   XHCI_TRB_EVALUATE_CONTEXT,
+				   xhci_evaluate_context_input ) ) != 0 )
+		return rc;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d (re-)evaluated\n",
+		xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
+					struct xhci_slot *slot,
+					struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_reset_endpoint *reset = &trb.reset;
+	int rc;
+
+	/* Construct command */
+	memset ( reset, 0, sizeof ( *reset ) );
+	reset->slot = slot->id;
+	reset->endpoint = endpoint->ctx;
+	reset->type = XHCI_TRB_RESET_ENDPOINT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not reset endpoint "
+		       "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
+		       endpoint->context->state, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Stop endpoint
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
+				       struct xhci_slot *slot,
+				       struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_stop_endpoint *stop = &trb.stop;
+	int rc;
+
+	/* Construct command */
+	memset ( stop, 0, sizeof ( *stop ) );
+	stop->slot = slot->id;
+	stop->endpoint = endpoint->ctx;
+	stop->type = XHCI_TRB_STOP_ENDPOINT;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not stop endpoint "
+		       "in state %d: %s\n", xhci->name, slot->id, endpoint->ctx,
+		       endpoint->context->state, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Set transfer ring dequeue pointer
+ *
+ * @v xhci		xHCI device
+ * @v slot		Device slot
+ * @v endpoint		Endpoint
+ * @ret rc		Return status code
+ */
+static inline int
+xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
+			      struct xhci_slot *slot,
+			      struct xhci_endpoint *endpoint ) {
+	union xhci_trb trb;
+	struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
+	struct xhci_trb_ring *ring = &endpoint->ring;
+	unsigned int cons;
+	unsigned int mask;
+	unsigned int index;
+	unsigned int dcs;
+	int rc;
+
+	/* Construct command */
+	memset ( dequeue, 0, sizeof ( *dequeue ) );
+	cons = ring->cons;
+	mask = ring->mask;
+	dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
+	index = ( cons & mask );
+	dequeue->dequeue =
+		cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
+	dequeue->slot = slot->id;
+	dequeue->endpoint = endpoint->ctx;
+	dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
+
+	/* Issue command and wait for completion */
+	if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
+		DBGC ( xhci, "XHCI %s slot %d ctx %d could not set TR dequeue "
+		       "pointer in state %d: %s\n", xhci->name, slot->id,
+		       endpoint->ctx, endpoint->context->state, strerror ( rc));
+		return rc;
+	}
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Endpoint operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
+	struct usb_device *usb = ep->usb;
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	struct xhci_endpoint *endpoint;
+	unsigned int ctx;
+	unsigned int type;
+	unsigned int interval;
+	int rc;
+
+	/* Calculate context index */
+	ctx = XHCI_CTX ( ep->address );
+	assert ( slot->endpoint[ctx] == NULL );
+
+	/* Calculate endpoint type */
+	type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
+	if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
+		type = XHCI_EP_TYPE_CONTROL;
+	if ( ep->address & USB_DIR_IN )
+		type |= XHCI_EP_TYPE_IN;
+
+	/* Calculate interval */
+	if ( type & XHCI_EP_TYPE_PERIODIC ) {
+		interval = ( fls ( ep->interval ) - 1 );
+	} else {
+		interval = ep->interval;
+	}
+
+	/* Allocate and initialise structure */
+	endpoint = zalloc ( sizeof ( *endpoint ) );
+	if ( ! endpoint ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	usb_endpoint_set_hostdata ( ep, endpoint );
+	slot->endpoint[ctx] = endpoint;
+	endpoint->xhci = xhci;
+	endpoint->slot = slot;
+	endpoint->ep = ep;
+	endpoint->ctx = ctx;
+	endpoint->type = type;
+	endpoint->interval = interval;
+	endpoint->context = ( ( ( void * ) slot->context ) +
+			      xhci_device_context_offset ( xhci, ctx ) );
+
+	/* Allocate transfer ring */
+	if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
+				      XHCI_TRANSFER_TRBS_LOG2,
+				      slot->id, ctx, 0 ) ) != 0 )
+		goto err_ring_alloc;
+
+	/* Configure endpoint, if applicable */
+	if ( ( ctx != XHCI_CTX_EP0 ) &&
+	     ( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
+		goto err_configure_endpoint;
+
+	DBGC2 ( xhci, "XHCI %s slot %d ctx %d ring [%08lx,%08lx)\n",
+		xhci->name, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
+		( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
+	return 0;
+
+	xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+ err_configure_endpoint:
+	xhci_ring_free ( &endpoint->ring );
+ err_ring_alloc:
+	slot->endpoint[ctx] = NULL;
+	free ( endpoint );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Close endpoint
+ *
+ * @v ep		USB endpoint
+ */
+static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	struct io_buffer *iobuf;
+	unsigned int ctx = endpoint->ctx;
+
+	/* Deconfigure endpoint, if applicable */
+	if ( ctx != XHCI_CTX_EP0 )
+		xhci_deconfigure_endpoint ( xhci, slot, endpoint );
+
+	/* Cancel any incomplete transfers */
+	while ( xhci_ring_fill ( &endpoint->ring ) ) {
+		iobuf = xhci_dequeue_multi ( &endpoint->ring );
+		usb_complete_err ( ep, iobuf, -ECANCELED );
+	}
+
+	/* Free endpoint */
+	xhci_ring_free ( &endpoint->ring );
+	slot->endpoint[ctx] = NULL;
+	free ( endpoint );
+}
+
+/**
+ * Reset endpoint
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	int rc;
+
+	/* Reset endpoint context */
+	if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
+		return rc;
+
+	/* Set transfer ring dequeue pointer */
+	if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
+		return rc;
+
+	/* Ring doorbell to resume processing */
+	xhci_doorbell ( &endpoint->ring );
+
+	DBGC ( xhci, "XHCI %s slot %d ctx %d reset\n",
+	       xhci->name, slot->id, endpoint->ctx );
+	return 0;
+}
+
+/**
+ * Update MTU
+ *
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct xhci_slot *slot = endpoint->slot;
+	struct xhci_device *xhci = slot->xhci;
+	int rc;
+
+	/* Evalulate context */
+	if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Enqueue message transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_message ( struct usb_endpoint *ep,
+				   struct io_buffer *iobuf ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	struct usb_setup_packet *packet;
+	unsigned int input;
+	size_t len;
+	union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
+			     1 /* status */ ];
+	union xhci_trb *trb = trbs;
+	struct xhci_trb_setup *setup;
+	struct xhci_trb_data *data;
+	struct xhci_trb_status *status;
+	int rc;
+
+	/* Profile message transfers */
+	profile_start ( &xhci_message_profiler );
+
+	/* Construct setup stage TRB */
+	memset ( trbs, 0, sizeof ( trbs ) );
+	assert ( iob_len ( iobuf ) >= sizeof ( *packet ) );
+	packet = iobuf->data;
+	iob_pull ( iobuf, sizeof ( *packet ) );
+	setup = &(trb++)->setup;
+	memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
+	setup->len = cpu_to_le32 ( sizeof ( *packet ) );
+	setup->flags = XHCI_TRB_IDT;
+	setup->type = XHCI_TRB_SETUP;
+	len = iob_len ( iobuf );
+	input = ( packet->request & cpu_to_le16 ( USB_DIR_IN ) );
+	if ( len )
+		setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
+
+	/* Construct data stage TRB, if applicable */
+	if ( len ) {
+		data = &(trb++)->data;
+		data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+		data->len = cpu_to_le32 ( len );
+		data->type = XHCI_TRB_DATA;
+		data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
+	}
+
+	/* Construct status stage TRB */
+	status = &(trb++)->status;
+	status->flags = XHCI_TRB_IOC;
+	status->type = XHCI_TRB_STATUS;
+	status->direction =
+		( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
+
+	/* Enqueue TRBs */
+	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
+					 ( trb - trbs ) ) ) != 0 )
+		return rc;
+
+	/* Ring the doorbell */
+	xhci_doorbell ( &endpoint->ring );
+
+	profile_stop ( &xhci_message_profiler );
+	return 0;
+}
+
+/**
+ * Enqueue stream transfer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ * @v terminate		Terminate using a short packet
+ * @ret rc		Return status code
+ */
+static int xhci_endpoint_stream ( struct usb_endpoint *ep,
+				  struct io_buffer *iobuf, int terminate ) {
+	struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
+	union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ];
+	union xhci_trb *trb = trbs;
+	struct xhci_trb_normal *normal;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Profile stream transfers */
+	profile_start ( &xhci_stream_profiler );
+
+	/* Construct normal TRBs */
+	memset ( &trbs, 0, sizeof ( trbs ) );
+	normal = &(trb++)->normal;
+	normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
+	normal->len = cpu_to_le32 ( len );
+	normal->type = XHCI_TRB_NORMAL;
+	if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
+		normal->flags = XHCI_TRB_CH;
+		normal = &(trb++)->normal;
+		normal->type = XHCI_TRB_NORMAL;
+	}
+	normal->flags = XHCI_TRB_IOC;
+
+	/* Enqueue TRBs */
+	if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
+					 ( trb - trbs ) ) ) != 0 )
+		return rc;
+
+	/* Ring the doorbell */
+	xhci_doorbell ( &endpoint->ring );
+
+	profile_stop ( &xhci_stream_profiler );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Device operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open device
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int xhci_device_open ( struct usb_device *usb ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
+	struct usb_port *tt = usb_transaction_translator ( usb );
+	struct xhci_slot *slot;
+	struct xhci_slot *tt_slot;
+	size_t len;
+	int type;
+	int id;
+	int rc;
+
+	/* Determine applicable slot type */
+	type = xhci_port_slot_type ( xhci, usb->port->address );
+	if ( type < 0 ) {
+		rc = type;
+		DBGC ( xhci, "XHCI %s-%d has no slot type\n",
+		       xhci->name, usb->port->address );
+		goto err_type;
+	}
+
+	/* Allocate a device slot number */
+	id = xhci_enable_slot ( xhci, type );
+	if ( id < 0 ) {
+		rc = id;
+		goto err_enable_slot;
+	}
+	assert ( ( id > 0 ) && ( ( unsigned int ) id <= xhci->slots ) );
+	assert ( xhci->slot[id] == NULL );
+
+	/* Allocate and initialise structure */
+	slot = zalloc ( sizeof ( *slot ) );
+	if ( ! slot ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	usb_set_hostdata ( usb, slot );
+	xhci->slot[id] = slot;
+	slot->xhci = xhci;
+	slot->usb = usb;
+	slot->id = id;
+	if ( tt ) {
+		tt_slot = usb_get_hostdata ( tt->hub->usb );
+		slot->tt_id = tt_slot->id;
+		slot->tt_port = tt->address;
+	}
+
+	/* Allocate a device context */
+	len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+	slot->context = malloc_dma ( len, xhci_align ( len ) );
+	if ( ! slot->context ) {
+		rc = -ENOMEM;
+		goto err_alloc_context;
+	}
+	memset ( slot->context, 0, len );
+
+	/* Set device context base address */
+	assert ( xhci->dcbaa[id] == 0 );
+	xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
+
+	DBGC2 ( xhci, "XHCI %s slot %d device context [%08lx,%08lx) for %s\n",
+		xhci->name, slot->id, virt_to_phys ( slot->context ),
+		( virt_to_phys ( slot->context ) + len ), usb->name );
+	return 0;
+
+	xhci->dcbaa[id] = 0;
+	free_dma ( slot->context, len );
+ err_alloc_context:
+	xhci->slot[id] = NULL;
+	free ( slot );
+ err_alloc:
+	xhci_disable_slot ( xhci, id );
+ err_enable_slot:
+ err_type:
+	return rc;
+}
+
+/**
+ * Close device
+ *
+ * @v usb		USB device
+ */
+static void xhci_device_close ( struct usb_device *usb ) {
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
+	unsigned int id = slot->id;
+	int rc;
+
+	/* Disable slot */
+	if ( ( rc = xhci_disable_slot ( xhci, id ) ) != 0 ) {
+		/* Slot is still enabled.  Leak the slot context,
+		 * since the controller may still write to this
+		 * memory, and leave the DCBAA entry intact.
+		 *
+		 * If the controller later reports that this same slot
+		 * has been re-enabled, then some assertions will be
+		 * triggered.
+		 */
+		DBGC ( xhci, "XHCI %s slot %d leaking context memory\n",
+		       xhci->name, slot->id );
+		slot->context = NULL;
+	}
+
+	/* Free slot */
+	if ( slot->context ) {
+		free_dma ( slot->context, len );
+		xhci->dcbaa[id] = 0;
+	}
+	xhci->slot[id] = NULL;
+	free ( slot );
+}
+
+/**
+ * Assign device address
+ *
+ * @v usb		USB device
+ * @ret rc		Return status code
+ */
+static int xhci_device_address ( struct usb_device *usb ) {
+	struct xhci_slot *slot = usb_get_hostdata ( usb );
+	struct xhci_device *xhci = slot->xhci;
+	struct usb_port *port = usb->port;
+	struct usb_port *root_port;
+	int psiv;
+	int rc;
+
+	/* Calculate route string */
+	slot->route = usb_route_string ( usb );
+
+	/* Calculate root hub port number */
+	root_port = usb_root_hub_port ( usb );
+	slot->port = root_port->address;
+
+	/* Calculate protocol speed ID */
+	psiv = xhci_port_psiv ( xhci, slot->port, port->speed );
+	if ( psiv < 0 ) {
+		rc = psiv;
+		return rc;
+	}
+	slot->psiv = psiv;
+
+	/* Address device */
+	if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Bus operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open USB bus
+ *
+ * @v bus		USB bus
+ * @ret rc		Return status code
+ */
+static int xhci_bus_open ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	int rc;
+
+	/* Allocate device slot array */
+	xhci->slot = zalloc ( ( xhci->slots + 1 ) * sizeof ( xhci->slot[0] ) );
+	if ( ! xhci->slot ) {
+		rc = -ENOMEM;
+		goto err_slot_alloc;
+	}
+
+	/* Allocate device context base address array */
+	if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
+		goto err_dcbaa_alloc;
+
+	/* Allocate scratchpad buffers */
+	if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
+		goto err_scratchpad_alloc;
+
+	/* Allocate command ring */
+	if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
+		goto err_command_alloc;
+
+	/* Allocate event ring */
+	if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
+		goto err_event_alloc;
+
+	/* Start controller */
+	xhci_run ( xhci );
+
+	return 0;
+
+	xhci_stop ( xhci );
+	xhci_event_free ( xhci );
+ err_event_alloc:
+	xhci_command_free ( xhci );
+ err_command_alloc:
+	xhci_scratchpad_free ( xhci );
+ err_scratchpad_alloc:
+	xhci_dcbaa_free ( xhci );
+ err_dcbaa_alloc:
+	free ( xhci->slot );
+ err_slot_alloc:
+	return rc;
+}
+
+/**
+ * Close USB bus
+ *
+ * @v bus		USB bus
+ */
+static void xhci_bus_close ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	unsigned int i;
+
+	/* Sanity checks */
+	assert ( xhci->slot != NULL );
+	for ( i = 0 ; i <= xhci->slots ; i++ )
+		assert ( xhci->slot[i] == NULL );
+
+	xhci_stop ( xhci );
+	xhci_event_free ( xhci );
+	xhci_command_free ( xhci );
+	xhci_scratchpad_free ( xhci );
+	xhci_dcbaa_free ( xhci );
+	free ( xhci->slot );
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static void xhci_bus_poll ( struct usb_bus *bus ) {
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+
+	/* Poll event ring */
+	xhci_event_poll ( xhci );
+}
+
+/******************************************************************************
+ *
+ * Hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int xhci_hub_open ( struct usb_hub *hub ) {
+	struct xhci_slot *slot;
+
+	/* Do nothing if this is the root hub */
+	if ( ! hub->usb )
+		return 0;
+
+	/* Get device slot */
+	slot = usb_get_hostdata ( hub->usb );
+
+	/* Update device slot hub parameters.  We don't inform the
+	 * hardware of this information until the hub's interrupt
+	 * endpoint is opened, since the only mechanism for so doing
+	 * provided by the xHCI specification is a Configure Endpoint
+	 * command, and we can't issue that command until we have a
+	 * non-EP0 endpoint to configure.
+	 */
+	slot->ports = hub->ports;
+
+	return 0;
+}
+
+/**
+ * Close hub
+ *
+ * @v hub		USB hub
+ */
+static void xhci_hub_close ( struct usb_hub *hub __unused ) {
+
+	/* Nothing to do */
+}
+
+/******************************************************************************
+ *
+ * Root hub operations
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open root hub
+ *
+ * @v hub		USB hub
+ * @ret rc		Return status code
+ */
+static int xhci_root_open ( struct usb_hub *hub ) {
+	struct usb_bus *bus = hub->bus;
+	struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
+	struct usb_port *port;
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Enable power to all ports */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+		portsc &= XHCI_PORTSC_PRESERVE;
+		portsc |= XHCI_PORTSC_PP;
+		writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+	}
+
+	/* xHCI spec requires us to potentially wait 20ms after
+	 * enabling power to a port.
+	 */
+	mdelay ( XHCI_PORT_POWER_DELAY_MS );
+
+	/* USB3 ports may power up as Disabled */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
+		port = usb_port ( hub, i );
+		if ( ( port->protocol >= USB_PROTO_3_0 ) &&
+		     ( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
+		       XHCI_PORTSC_PLS_DISABLED ) ) {
+			/* Force link state to RxDetect */
+			portsc &= XHCI_PORTSC_PRESERVE;
+			portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
+			writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
+		}
+	}
+
+	/* Some xHCI cards seem to require an additional delay after
+	 * setting the link state to RxDetect.
+	 */
+	mdelay ( XHCI_LINK_STATE_DELAY_MS );
+
+	/* Record hub driver private data */
+	usb_hub_set_drvdata ( hub, xhci );
+
+	return 0;
+}
+
+/**
+ * Close root hub
+ *
+ * @v hub		USB hub
+ */
+static void xhci_root_close ( struct usb_hub *hub ) {
+
+	/* Clear hub driver private data */
+	usb_hub_set_drvdata ( hub, NULL );
+}
+
+/**
+ * Enable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_enable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int i;
+
+	/* Reset port */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	portsc &= XHCI_PORTSC_PRESERVE;
+	portsc |= XHCI_PORTSC_PR;
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	/* Wait for port to become enabled */
+	for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
+
+		/* Check port status */
+		portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+		if ( portsc & XHCI_PORTSC_PED )
+			return 0;
+
+		/* Delay */
+		mdelay ( 1 );
+	}
+
+	DBGC ( xhci, "XHCI %s-%d timed out waiting for port to enable\n",
+	       xhci->name, port->address );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Disable port
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_disable ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+
+	/* Disable port */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	portsc &= XHCI_PORTSC_PRESERVE;
+	portsc |= XHCI_PORTSC_PED;
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	return 0;
+}
+
+/**
+ * Update root hub port speed
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @ret rc		Return status code
+ */
+static int xhci_root_speed ( struct usb_hub *hub, struct usb_port *port ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+	uint32_t portsc;
+	unsigned int psiv;
+	int ccs;
+	int ped;
+	int csc;
+	int speed;
+	int rc;
+
+	/* Read port status */
+	portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
+	DBGC2 ( xhci, "XHCI %s-%d status is %08x\n",
+		xhci->name, port->address, portsc );
+	ccs = ( portsc & XHCI_PORTSC_CCS );
+	ped = ( portsc & XHCI_PORTSC_PED );
+	csc = ( portsc & XHCI_PORTSC_CSC );
+	psiv = XHCI_PORTSC_PSIV ( portsc );
+
+	/* Record disconnections and clear changes */
+	port->disconnected |= csc;
+	portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
+	writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
+
+	/* Port speed is not valid unless port is connected */
+	if ( ! ccs ) {
+		port->speed = USB_SPEED_NONE;
+		return 0;
+	}
+
+	/* For USB2 ports, the PSIV field is not valid until the port
+	 * completes reset and becomes enabled.
+	 */
+	if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
+		port->speed = USB_SPEED_FULL;
+		return 0;
+	}
+
+	/* Get port speed and map to generic USB speed */
+	speed = xhci_port_speed ( xhci, port->address, psiv );
+	if ( speed < 0 ) {
+		rc = speed;
+		return rc;
+	}
+
+	port->speed = speed;
+	return 0;
+}
+
+/**
+ * Clear transaction translator buffer
+ *
+ * @v hub		USB hub
+ * @v port		USB port
+ * @v ep		USB endpoint
+ * @ret rc		Return status code
+ */
+static int xhci_root_clear_tt ( struct usb_hub *hub, struct usb_port *port,
+				struct usb_endpoint *ep ) {
+	struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
+
+	/* Should never be called; this is a root hub */
+	DBGC ( xhci, "XHCI %s-%d nonsensical CLEAR_TT for %s %s\n", xhci->name,
+	       port->address, ep->usb->name, usb_endpoint_name ( ep ) );
+
+	return -ENOTSUP;
+}
+
+/******************************************************************************
+ *
+ * PCI interface
+ *
+ ******************************************************************************
+ */
+
+/** USB host controller operations */
+static struct usb_host_operations xhci_operations = {
+	.endpoint = {
+		.open = xhci_endpoint_open,
+		.close = xhci_endpoint_close,
+		.reset = xhci_endpoint_reset,
+		.mtu = xhci_endpoint_mtu,
+		.message = xhci_endpoint_message,
+		.stream = xhci_endpoint_stream,
+	},
+	.device = {
+		.open = xhci_device_open,
+		.close = xhci_device_close,
+		.address = xhci_device_address,
+	},
+	.bus = {
+		.open = xhci_bus_open,
+		.close = xhci_bus_close,
+		.poll = xhci_bus_poll,
+	},
+	.hub = {
+		.open = xhci_hub_open,
+		.close = xhci_hub_close,
+	},
+	.root = {
+		.open = xhci_root_open,
+		.close = xhci_root_close,
+		.enable = xhci_root_enable,
+		.disable = xhci_root_disable,
+		.speed = xhci_root_speed,
+		.clear_tt = xhci_root_clear_tt,
+	},
+};
+
+/**
+ * Fix Intel PCH-specific quirks
+ *
+ * @v xhci		xHCI device
+ * @v pci		PCI device
+ */
+static void xhci_pch_fix ( struct xhci_device *xhci, struct pci_device *pci ) {
+	struct xhci_pch *pch = &xhci->pch;
+	uint32_t xusb2pr;
+	uint32_t xusb2prm;
+	uint32_t usb3pssen;
+	uint32_t usb3prm;
+
+	/* Enable SuperSpeed capability.  Do this before rerouting
+	 * USB2 ports, so that USB3 devices connect at SuperSpeed.
+	 */
+	pci_read_config_dword ( pci, XHCI_PCH_USB3PSSEN, &usb3pssen );
+	pci_read_config_dword ( pci, XHCI_PCH_USB3PRM, &usb3prm );
+	if ( usb3prm & ~usb3pssen ) {
+		DBGC ( xhci, "XHCI %s enabling SuperSpeed on ports %08x\n",
+		       xhci->name, ( usb3prm & ~usb3pssen ) );
+	}
+	pch->usb3pssen = usb3pssen;
+	usb3pssen |= usb3prm;
+	pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, usb3pssen );
+
+	/* Route USB2 ports from EHCI to xHCI */
+	pci_read_config_dword ( pci, XHCI_PCH_XUSB2PR, &xusb2pr );
+	pci_read_config_dword ( pci, XHCI_PCH_XUSB2PRM, &xusb2prm );
+	if ( xusb2prm & ~xusb2pr ) {
+		DBGC ( xhci, "XHCI %s routing ports %08x from EHCI to xHCI\n",
+		       xhci->name, ( xusb2prm & ~xusb2pr ) );
+	}
+	pch->xusb2pr = xusb2pr;
+	xusb2pr |= xusb2prm;
+	pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, xusb2pr );
+}
+
+/**
+ * Undo Intel PCH-specific quirk fixes
+ *
+ * @v xhci		xHCI device
+ * @v pci		PCI device
+ */
+static void xhci_pch_undo ( struct xhci_device *xhci, struct pci_device *pci ) {
+	struct xhci_pch *pch = &xhci->pch;
+
+	/* Restore USB2 port routing to original state */
+	pci_write_config_dword ( pci, XHCI_PCH_XUSB2PR, pch->xusb2pr );
+
+	/* Restore SuperSpeed capability to original state */
+	pci_write_config_dword ( pci, XHCI_PCH_USB3PSSEN, pch->usb3pssen );
+}
+
+/**
+ * Probe PCI device
+ *
+ * @v pci		PCI device
+ * @ret rc		Return status code
+ */
+static int xhci_probe ( struct pci_device *pci ) {
+	struct xhci_device *xhci;
+	struct usb_port *port;
+	unsigned long bar_start;
+	size_t bar_size;
+	unsigned int i;
+	int rc;
+
+	/* Allocate and initialise structure */
+	xhci = zalloc ( sizeof ( *xhci ) );
+	if ( ! xhci ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	xhci->name = pci->dev.name;
+	xhci->quirks = pci->id->driver_data;
+
+	/* Fix up PCI device */
+	adjust_pci_device ( pci );
+
+	/* Map registers */
+	bar_start = pci_bar_start ( pci, XHCI_BAR );
+	bar_size = pci_bar_size ( pci, XHCI_BAR );
+	xhci->regs = ioremap ( bar_start, bar_size );
+	if ( ! xhci->regs ) {
+		rc = -ENODEV;
+		goto err_ioremap;
+	}
+
+	/* Initialise xHCI device */
+	xhci_init ( xhci, xhci->regs );
+
+	/* Initialise USB legacy support and claim ownership */
+	xhci_legacy_init ( xhci );
+	xhci_legacy_claim ( xhci );
+
+	/* Fix Intel PCH-specific quirks, if applicable */
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_fix ( xhci, pci );
+
+	/* Reset device */
+	if ( ( rc = xhci_reset ( xhci ) ) != 0 )
+		goto err_reset;
+
+	/* Allocate USB bus */
+	xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports, XHCI_MTU,
+				    &xhci_operations );
+	if ( ! xhci->bus ) {
+		rc = -ENOMEM;
+		goto err_alloc_bus;
+	}
+	usb_bus_set_hostdata ( xhci->bus, xhci );
+	usb_hub_set_drvdata ( xhci->bus->hub, xhci );
+
+	/* Set port protocols */
+	for ( i = 1 ; i <= xhci->ports ; i++ ) {
+		port = usb_port ( xhci->bus->hub, i );
+		port->protocol = xhci_port_protocol ( xhci, i );
+	}
+
+	/* Register USB bus */
+	if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
+		goto err_register;
+
+	pci_set_drvdata ( pci, xhci );
+	return 0;
+
+	unregister_usb_bus ( xhci->bus );
+ err_register:
+	free_usb_bus ( xhci->bus );
+ err_alloc_bus:
+	xhci_reset ( xhci );
+ err_reset:
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_undo ( xhci, pci );
+	xhci_legacy_release ( xhci );
+	iounmap ( xhci->regs );
+ err_ioremap:
+	free ( xhci );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove PCI device
+ *
+ * @v pci		PCI device
+ */
+static void xhci_remove ( struct pci_device *pci ) {
+	struct xhci_device *xhci = pci_get_drvdata ( pci );
+	struct usb_bus *bus = xhci->bus;
+
+	unregister_usb_bus ( bus );
+	free_usb_bus ( bus );
+	xhci_reset ( xhci );
+	if ( xhci->quirks & XHCI_PCH )
+		xhci_pch_undo ( xhci, pci );
+	xhci_legacy_release ( xhci );
+	iounmap ( xhci->regs );
+	free ( xhci );
+}
+
+/** XHCI PCI device IDs */
+static struct pci_device_id xhci_ids[] = {
+	PCI_ROM ( 0x8086, 0x9d2f, "xhci-skylake", "xHCI (Skylake)", ( XHCI_PCH | XHCI_BAD_PSIV ) ),
+	PCI_ROM ( 0x8086, 0xffff, "xhci-pch", "xHCI (Intel PCH)", XHCI_PCH ),
+	PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
+};
+
+/** XHCI PCI driver */
+struct pci_driver xhci_driver __pci_driver = {
+	.ids = xhci_ids,
+	.id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
+	.class = PCI_CLASS_ID ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
+				PCI_CLASS_SERIAL_USB_XHCI ),
+	.probe = xhci_probe,
+	.remove = xhci_remove,
+};
+
+/**
+ * Prepare for exit
+ *
+ * @v booting		System is shutting down for OS boot
+ */
+static void xhci_shutdown ( int booting ) {
+	/* If we are shutting down to boot an OS, then prevent the
+	 * release of ownership back to BIOS.
+	 */
+	xhci_legacy_prevent_release = booting;
+}
+
+/** Startup/shutdown function */
+struct startup_fn xhci_startup __startup_fn ( STARTUP_LATE ) = {
+	.shutdown = xhci_shutdown,
+};
diff --git a/qemu/roms/ipxe/src/drivers/usb/xhci.h b/qemu/roms/ipxe/src/drivers/usb/xhci.h
new file mode 100644
index 000000000..83bf71e7e
--- /dev/null
+++ b/qemu/roms/ipxe/src/drivers/usb/xhci.h
@@ -0,0 +1,1150 @@
+#ifndef _IPXE_XHCI_H
+#define _IPXE_XHCI_H
+
+/** @file
+ *
+ * USB eXtensible Host Controller Interface (xHCI) driver
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <assert.h>
+#include <ipxe/pci.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/usb.h>
+
+/** Minimum alignment required for data structures
+ *
+ * With the exception of the scratchpad buffer pages (which are
+ * page-aligned), data structures used by xHCI generally require from
+ * 16 to 64 byte alignment and must not cross an (xHCI) page boundary.
+ * We simplify this requirement by aligning each structure on its own
+ * size, with a minimum of a 64 byte alignment.
+ */
+#define XHCI_MIN_ALIGN 64
+
+/** Maximum transfer size */
+#define XHCI_MTU 65536
+
+/** xHCI PCI BAR */
+#define XHCI_BAR PCI_BASE_ADDRESS_0
+
+/** Capability register length */
+#define XHCI_CAP_CAPLENGTH 0x00
+
+/** Host controller interface version number */
+#define XHCI_CAP_HCIVERSION 0x02
+
+/** Structural parameters 1 */
+#define XHCI_CAP_HCSPARAMS1 0x04
+
+/** Number of device slots */
+#define XHCI_HCSPARAMS1_SLOTS(params) ( ( (params) >> 0 ) & 0xff )
+
+/** Number of interrupters */
+#define XHCI_HCSPARAMS1_INTRS(params) ( ( (params) >> 8 ) & 0x3ff )
+
+/** Number of ports */
+#define XHCI_HCSPARAMS1_PORTS(params) ( ( (params) >> 24 ) & 0xff )
+
+/** Structural parameters 2 */
+#define XHCI_CAP_HCSPARAMS2 0x08
+
+/** Number of page-sized scratchpad buffers */
+#define XHCI_HCSPARAMS2_SCRATCHPADS(params) \
+	( ( ( (params) >> 16 ) & 0x3e0 ) | ( ( (params) >> 27 ) & 0x1f ) )
+
+/** Capability parameters */
+#define XHCI_CAP_HCCPARAMS1 0x10
+
+/** 64-bit addressing capability */
+#define XHCI_HCCPARAMS1_ADDR64(params) ( ( (params) >> 0 ) & 0x1 )
+
+/** Context size shift */
+#define XHCI_HCCPARAMS1_CSZ_SHIFT(params) ( 5 + ( ( (params) >> 2 ) & 0x1 ) )
+
+/** xHCI extended capabilities pointer */
+#define XHCI_HCCPARAMS1_XECP(params) ( ( ( (params) >> 16 ) & 0xffff ) << 2 )
+
+/** Doorbell offset */
+#define XHCI_CAP_DBOFF 0x14
+
+/** Runtime register space offset */
+#define XHCI_CAP_RTSOFF 0x18
+
+/** xHCI extended capability ID */
+#define XHCI_XECP_ID(xecp) ( ( (xecp) >> 0 ) & 0xff )
+
+/** Next xHCI extended capability pointer */
+#define XHCI_XECP_NEXT(xecp) ( ( ( (xecp) >> 8 ) & 0xff ) << 2 )
+
+/** USB legacy support extended capability */
+#define XHCI_XECP_ID_LEGACY 1
+
+/** USB legacy support BIOS owned semaphore */
+#define XHCI_USBLEGSUP_BIOS 0x02
+
+/** USB legacy support BIOS ownership flag */
+#define XHCI_USBLEGSUP_BIOS_OWNED 0x01
+
+/** USB legacy support OS owned semaphore */
+#define XHCI_USBLEGSUP_OS 0x03
+
+/** USB legacy support OS ownership flag */
+#define XHCI_USBLEGSUP_OS_OWNED 0x01
+
+/** USB legacy support control/status */
+#define XHCI_USBLEGSUP_CTLSTS 0x04
+
+/** Supported protocol extended capability */
+#define XHCI_XECP_ID_SUPPORTED 2
+
+/** Supported protocol revision */
+#define XHCI_SUPPORTED_REVISION 0x00
+
+/** Supported protocol minor revision */
+#define XHCI_SUPPORTED_REVISION_VER(revision) ( ( (revision) >> 16 ) & 0xffff )
+
+/** Supported protocol name */
+#define XHCI_SUPPORTED_NAME 0x04
+
+/** Supported protocol ports */
+#define XHCI_SUPPORTED_PORTS 0x08
+
+/** Supported protocol port offset */
+#define XHCI_SUPPORTED_PORTS_OFFSET(ports) ( ( (ports) >> 0 ) & 0xff )
+
+/** Supported protocol port count */
+#define XHCI_SUPPORTED_PORTS_COUNT(ports) ( ( (ports) >> 8 ) & 0xff )
+
+/** Supported protocol PSI count */
+#define XHCI_SUPPORTED_PORTS_PSIC(ports) ( ( (ports) >> 28 ) & 0x0f )
+
+/** Supported protocol slot */
+#define XHCI_SUPPORTED_SLOT 0x0c
+
+/** Supported protocol slot type */
+#define XHCI_SUPPORTED_SLOT_TYPE(slot) ( ( (slot) >> 0 ) & 0x1f )
+
+/** Supported protocol PSI */
+#define XHCI_SUPPORTED_PSI(index) ( 0x10 + ( (index) * 4 ) )
+
+/** Supported protocol PSI value */
+#define XHCI_SUPPORTED_PSI_VALUE(psi) ( ( (psi) >> 0 ) & 0x0f )
+
+/** Supported protocol PSI mantissa */
+#define XHCI_SUPPORTED_PSI_MANTISSA(psi) ( ( (psi) >> 16 ) & 0xffff )
+
+/** Supported protocol PSI exponent */
+#define XHCI_SUPPORTED_PSI_EXPONENT(psi) ( ( (psi) >> 4 ) & 0x03 )
+
+/** Default PSI values */
+enum xhci_default_psi_value {
+	/** Full speed (12Mbps) */
+	XHCI_SPEED_FULL = 1,
+	/** Low speed (1.5Mbps) */
+	XHCI_SPEED_LOW = 2,
+	/** High speed (480Mbps) */
+	XHCI_SPEED_HIGH = 3,
+	/** Super speed */
+	XHCI_SPEED_SUPER = 4,
+};
+
+/** USB command register */
+#define XHCI_OP_USBCMD 0x00
+
+/** Run/stop */
+#define XHCI_USBCMD_RUN 0x00000001UL
+
+/** Host controller reset */
+#define XHCI_USBCMD_HCRST 0x00000002UL
+
+/** USB status register */
+#define XHCI_OP_USBSTS 0x04
+
+/** Host controller halted */
+#define XHCI_USBSTS_HCH 0x00000001UL
+
+/** Page size register */
+#define XHCI_OP_PAGESIZE 0x08
+
+/** Page size */
+#define XHCI_PAGESIZE(pagesize) ( (pagesize) << 12 )
+
+/** Device notifcation control register */
+#define XHCI_OP_DNCTRL 0x14
+
+/** Command ring control register */
+#define XHCI_OP_CRCR 0x18
+
+/** Command ring cycle state */
+#define XHCI_CRCR_RCS 0x00000001UL
+
+/** Command abort */
+#define XHCI_CRCR_CA 0x00000004UL
+
+/** Command ring running */
+#define XHCI_CRCR_CRR 0x00000008UL
+
+/** Device context base address array pointer */
+#define XHCI_OP_DCBAAP 0x30
+
+/** Configure register */
+#define XHCI_OP_CONFIG 0x38
+
+/** Maximum device slots enabled */
+#define XHCI_CONFIG_MAX_SLOTS_EN(slots) ( (slots) << 0 )
+
+/** Maximum device slots enabled mask */
+#define XHCI_CONFIG_MAX_SLOTS_EN_MASK \
+	XHCI_CONFIG_MAX_SLOTS_EN ( 0xff )
+
+/** Port status and control register */
+#define XHCI_OP_PORTSC(port) ( 0x400 - 0x10 + ( (port) << 4 ) )
+
+/** Current connect status */
+#define XHCI_PORTSC_CCS 0x00000001UL
+
+/** Port enabled */
+#define XHCI_PORTSC_PED 0x00000002UL
+
+/** Port reset */
+#define XHCI_PORTSC_PR 0x00000010UL
+
+/** Port link state */
+#define XHCI_PORTSC_PLS(pls) ( (pls) << 5 )
+
+/** Disabled port link state */
+#define XHCI_PORTSC_PLS_DISABLED XHCI_PORTSC_PLS ( 4 )
+
+/** RxDetect port link state */
+#define XHCI_PORTSC_PLS_RXDETECT XHCI_PORTSC_PLS ( 5 )
+
+/** Port link state mask */
+#define XHCI_PORTSC_PLS_MASK XHCI_PORTSC_PLS ( 0xf )
+
+/** Port power */
+#define XHCI_PORTSC_PP 0x00000200UL
+
+/** Time to delay after enabling power to a port */
+#define XHCI_PORT_POWER_DELAY_MS 20
+
+/** Port speed ID value */
+#define XHCI_PORTSC_PSIV(portsc) ( ( (portsc) >> 10 ) & 0xf )
+
+/** Port indicator control */
+#define XHCI_PORTSC_PIC(indicators) ( (indicators) << 14 )
+
+/** Port indicator control mask */
+#define XHCI_PORTSC_PIC_MASK XHCI_PORTSC_PIC ( 3 )
+
+/** Port link state write strobe */
+#define XHCI_PORTSC_LWS 0x00010000UL
+
+/** Time to delay after writing the port link state */
+#define XHCI_LINK_STATE_DELAY_MS 20
+
+/** Connect status change */
+#define XHCI_PORTSC_CSC 0x00020000UL
+
+/** Port enabled/disabled change */
+#define XHCI_PORTSC_PEC 0x00040000UL
+
+/** Warm port reset change */
+#define XHCI_PORTSC_WRC 0x00080000UL
+
+/** Over-current change */
+#define XHCI_PORTSC_OCC 0x00100000UL
+
+/** Port reset change */
+#define XHCI_PORTSC_PRC 0x00200000UL
+
+/** Port link state change */
+#define XHCI_PORTSC_PLC 0x00400000UL
+
+/** Port config error change */
+#define XHCI_PORTSC_CEC 0x00800000UL
+
+/** Port status change mask */
+#define XHCI_PORTSC_CHANGE					\
+	( XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | XHCI_PORTSC_WRC |	\
+	  XHCI_PORTSC_OCC | XHCI_PORTSC_PRC | XHCI_PORTSC_PLC |	\
+	  XHCI_PORTSC_CEC )
+
+/** Port status and control bits which should be preserved
+ *
+ * The port status and control register is a horrendous mix of
+ * differing semantics.  Some bits are written to only when a separate
+ * write strobe bit is set.  Some bits should be preserved when
+ * modifying other bits.  Some bits will be cleared if written back as
+ * a one.  Most excitingly, the "port enabled" bit has the semantics
+ * that 1=enabled, 0=disabled, yet writing a 1 will disable the port.
+ */
+#define XHCI_PORTSC_PRESERVE ( XHCI_PORTSC_PP | XHCI_PORTSC_PIC_MASK )
+
+/** Port power management status and control register */
+#define XHCI_OP_PORTPMSC(port) ( 0x404 - 0x10 + ( (port) << 4 ) )
+
+/** Port link info register */
+#define XHCI_OP_PORTLI(port) ( 0x408 - 0x10 + ( (port) << 4 ) )
+
+/** Port hardware link power management control register */
+#define XHCI_OP_PORTHLPMC(port) ( 0x40c - 0x10 + ( (port) << 4 ) )
+
+/** Event ring segment table size register */
+#define XHCI_RUN_ERSTSZ(intr) ( 0x28 + ( (intr) << 5 ) )
+
+/** Event ring segment table base address register */
+#define XHCI_RUN_ERSTBA(intr) ( 0x30 + ( (intr) << 5 ) )
+
+/** Event ring dequeue pointer register */
+#define XHCI_RUN_ERDP(intr) ( 0x38 + ( (intr) << 5 ) )
+
+/** A transfer request block template */
+struct xhci_trb_template {
+	/** Parameter */
+	uint64_t parameter;
+	/** Status */
+	uint32_t status;
+	/** Control */
+	uint32_t control;
+};
+
+/** A transfer request block */
+struct xhci_trb_common {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** Transfer request block cycle bit flag */
+#define XHCI_TRB_C 0x01
+
+/** Transfer request block toggle cycle bit flag */
+#define XHCI_TRB_TC 0x02
+
+/** Transfer request block chain flag */
+#define XHCI_TRB_CH 0x10
+
+/** Transfer request block interrupt on completion flag */
+#define XHCI_TRB_IOC 0x20
+
+/** Transfer request block immediate data flag */
+#define XHCI_TRB_IDT 0x40
+
+/** Transfer request block type */
+#define XHCI_TRB_TYPE(type) ( (type) << 2 )
+
+/** Transfer request block type mask */
+#define XHCI_TRB_TYPE_MASK XHCI_TRB_TYPE ( 0x3f )
+
+/** A normal transfer request block */
+struct xhci_trb_normal {
+	/** Data buffer */
+	uint64_t data;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A normal transfer request block */
+#define XHCI_TRB_NORMAL XHCI_TRB_TYPE ( 1 )
+
+/** Construct TD size field */
+#define XHCI_TD_SIZE(remaining) \
+	( ( ( (remaining) <= 0xf ) ? remaining : 0xf ) << 17 )
+
+/** A setup stage transfer request block */
+struct xhci_trb_setup {
+	/** Setup packet */
+	struct usb_setup_packet packet;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Transfer direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A setup stage transfer request block */
+#define XHCI_TRB_SETUP XHCI_TRB_TYPE ( 2 )
+
+/** Setup stage input data direction */
+#define XHCI_SETUP_IN 3
+
+/** Setup stage output data direction */
+#define XHCI_SETUP_OUT 2
+
+/** A data stage transfer request block */
+struct xhci_trb_data {
+	/** Data buffer */
+	uint64_t data;
+	/** Length */
+	uint32_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Transfer direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved;
+} __attribute__ (( packed ));
+
+/** A data stage transfer request block */
+#define XHCI_TRB_DATA XHCI_TRB_TYPE ( 3 )
+
+/** Input data direction */
+#define XHCI_DATA_IN 0x01
+
+/** Output data direction */
+#define XHCI_DATA_OUT 0x00
+
+/** A status stage transfer request block */
+struct xhci_trb_status {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Direction */
+	uint8_t direction;
+	/** Reserved */
+	uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A status stage transfer request block */
+#define XHCI_TRB_STATUS XHCI_TRB_TYPE ( 4 )
+
+/** Input status direction */
+#define XHCI_STATUS_IN 0x01
+
+/** Output status direction */
+#define XHCI_STATUS_OUT 0x00
+
+/** A link transfer request block */
+struct xhci_trb_link {
+	/** Next ring segment */
+	uint64_t next;
+	/** Reserved */
+	uint32_t reserved_a;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A link transfer request block */
+#define XHCI_TRB_LINK XHCI_TRB_TYPE ( 6 )
+
+/** A no-op transfer request block */
+#define XHCI_TRB_NOP XHCI_TRB_TYPE ( 8 )
+
+/** An enable slot transfer request block */
+struct xhci_trb_enable_slot {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Slot type */
+	uint8_t slot;
+	/** Reserved */
+	uint8_t reserved_c;
+} __attribute__ (( packed ));
+
+/** An enable slot transfer request block */
+#define XHCI_TRB_ENABLE_SLOT XHCI_TRB_TYPE ( 9 )
+
+/** A disable slot transfer request block */
+struct xhci_trb_disable_slot {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint8_t reserved_c;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A disable slot transfer request block */
+#define XHCI_TRB_DISABLE_SLOT XHCI_TRB_TYPE ( 10 )
+
+/** A context transfer request block */
+struct xhci_trb_context {
+	/** Input context */
+	uint64_t input;
+	/** Reserved */
+	uint32_t reserved_a;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint8_t reserved_b;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** An address device transfer request block */
+#define XHCI_TRB_ADDRESS_DEVICE XHCI_TRB_TYPE ( 11 )
+
+/** A configure endpoint transfer request block */
+#define XHCI_TRB_CONFIGURE_ENDPOINT XHCI_TRB_TYPE ( 12 )
+
+/** An evaluate context transfer request block */
+#define XHCI_TRB_EVALUATE_CONTEXT XHCI_TRB_TYPE ( 13 )
+
+/** A reset endpoint transfer request block */
+struct xhci_trb_reset_endpoint {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A reset endpoint transfer request block */
+#define XHCI_TRB_RESET_ENDPOINT XHCI_TRB_TYPE ( 14 )
+
+/** A stop endpoint transfer request block */
+struct xhci_trb_stop_endpoint {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint32_t reserved_b;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A stop endpoint transfer request block */
+#define XHCI_TRB_STOP_ENDPOINT XHCI_TRB_TYPE ( 15 )
+
+/** A set transfer ring dequeue pointer transfer request block */
+struct xhci_trb_set_tr_dequeue_pointer {
+	/** Dequeue pointer */
+	uint64_t dequeue;
+	/** Reserved */
+	uint32_t reserved;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A set transfer ring dequeue pointer transfer request block */
+#define XHCI_TRB_SET_TR_DEQUEUE_POINTER XHCI_TRB_TYPE ( 16 )
+
+/** A no-op command transfer request block */
+#define XHCI_TRB_NOP_CMD XHCI_TRB_TYPE ( 23 )
+
+/** A transfer event transfer request block */
+struct xhci_trb_transfer {
+	/** Transfer TRB pointer */
+	uint64_t transfer;
+	/** Residual transfer length */
+	uint16_t residual;
+	/** Reserved */
+	uint8_t reserved;
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Endpoint ID */
+	uint8_t endpoint;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A transfer event transfer request block */
+#define XHCI_TRB_TRANSFER XHCI_TRB_TYPE ( 32 )
+
+/** A command completion event transfer request block */
+struct xhci_trb_complete {
+	/** Command TRB pointer */
+	uint64_t command;
+	/** Parameter */
+	uint8_t parameter[3];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Virtual function ID */
+	uint8_t vf;
+	/** Slot ID */
+	uint8_t slot;
+} __attribute__ (( packed ));
+
+/** A command completion event transfer request block */
+#define XHCI_TRB_COMPLETE XHCI_TRB_TYPE ( 33 )
+
+/** xHCI completion codes */
+enum xhci_completion_code {
+	/** Success */
+	XHCI_CMPLT_SUCCESS = 1,
+	/** Short packet */
+	XHCI_CMPLT_SHORT = 13,
+	/** Command ring stopped */
+	XHCI_CMPLT_CMD_STOPPED = 24,
+};
+
+/** A port status change transfer request block */
+struct xhci_trb_port_status {
+	/** Reserved */
+	uint8_t reserved_a[3];
+	/** Port ID */
+	uint8_t port;
+	/** Reserved */
+	uint8_t reserved_b[7];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A port status change transfer request block */
+#define XHCI_TRB_PORT_STATUS XHCI_TRB_TYPE ( 34 )
+
+/** A port status change transfer request block */
+struct xhci_trb_host_controller {
+	/** Reserved */
+	uint64_t reserved_a;
+	/** Reserved */
+	uint8_t reserved_b[3];
+	/** Completion code */
+	uint8_t code;
+	/** Flags */
+	uint8_t flags;
+	/** Type */
+	uint8_t type;
+	/** Reserved */
+	uint16_t reserved_c;
+} __attribute__ (( packed ));
+
+/** A port status change transfer request block */
+#define XHCI_TRB_HOST_CONTROLLER XHCI_TRB_TYPE ( 37 )
+
+/** A transfer request block */
+union xhci_trb {
+	/** Template */
+	struct xhci_trb_template template;
+	/** Common fields */
+	struct xhci_trb_common common;
+	/** Normal TRB */
+	struct xhci_trb_normal normal;
+	/** Setup stage TRB */
+	struct xhci_trb_setup setup;
+	/** Data stage TRB */
+	struct xhci_trb_data data;
+	/** Status stage TRB */
+	struct xhci_trb_status status;
+	/** Link TRB */
+	struct xhci_trb_link link;
+	/** Enable slot TRB */
+	struct xhci_trb_enable_slot enable;
+	/** Disable slot TRB */
+	struct xhci_trb_disable_slot disable;
+	/** Input context TRB */
+	struct xhci_trb_context context;
+	/** Reset endpoint TRB */
+	struct xhci_trb_reset_endpoint reset;
+	/** Stop endpoint TRB */
+	struct xhci_trb_stop_endpoint stop;
+	/** Set transfer ring dequeue pointer TRB */
+	struct xhci_trb_set_tr_dequeue_pointer dequeue;
+	/** Transfer event */
+	struct xhci_trb_transfer transfer;
+	/** Command completion event */
+	struct xhci_trb_complete complete;
+	/** Port status changed event */
+	struct xhci_trb_port_status port;
+	/** Host controller event */
+	struct xhci_trb_host_controller host;
+} __attribute__ (( packed ));
+
+/** An input control context */
+struct xhci_control_context {
+	/** Drop context flags */
+	uint32_t drop;
+	/** Add context flags */
+	uint32_t add;
+	/** Reserved */
+	uint32_t reserved_a[5];
+	/** Configuration value */
+	uint8_t config;
+	/** Interface number */
+	uint8_t intf;
+	/** Alternate setting */
+	uint8_t alt;
+	/** Reserved */
+	uint8_t reserved_b;
+} __attribute__ (( packed ));
+
+/** A slot context */
+struct xhci_slot_context {
+	/** Device info */
+	uint32_t info;
+	/** Maximum exit latency */
+	uint16_t latency;
+	/** Root hub port number */
+	uint8_t port;
+	/** Number of downstream ports */
+	uint8_t ports;
+	/** TT hub slot ID */
+	uint8_t tt_id;
+	/** TT port number */
+	uint8_t tt_port;
+	/** Interrupter target */
+	uint16_t intr;
+	/** USB address */
+	uint8_t address;
+	/** Reserved */
+	uint16_t reserved_a;
+	/** Slot state */
+	uint8_t state;
+	/** Reserved */
+	uint32_t reserved_b[4];
+} __attribute__ (( packed ));
+
+/** Construct slot context device info */
+#define XHCI_SLOT_INFO( entries, hub, speed, route ) \
+	( ( (entries) << 27 ) | ( (hub) << 26 ) | ( (speed) << 20 ) | (route) )
+
+/** An endpoint context */
+struct xhci_endpoint_context {
+	/** Endpoint state */
+	uint8_t state;
+	/** Stream configuration */
+	uint8_t stream;
+	/** Polling interval */
+	uint8_t interval;
+	/** Max ESIT payload high */
+	uint8_t esit_high;
+	/** Endpoint type */
+	uint8_t type;
+	/** Maximum burst size */
+	uint8_t burst;
+	/** Maximum packet size */
+	uint16_t mtu;
+	/** Transfer ring dequeue pointer */
+	uint64_t dequeue;
+	/** Average TRB length */
+	uint16_t trb_len;
+	/** Max ESIT payload low */
+	uint16_t esit_low;
+	/** Reserved */
+	uint32_t reserved[3];
+} __attribute__ (( packed ));
+
+/** Endpoint states */
+enum xhci_endpoint_state {
+	/** Endpoint is disabled */
+	XHCI_ENDPOINT_DISABLED = 0,
+	/** Endpoint is running */
+	XHCI_ENDPOINT_RUNNING = 1,
+	/** Endpoint is halted due to a USB Halt condition */
+	XHCI_ENDPOINT_HALTED = 2,
+	/** Endpoint is stopped */
+	XHCI_ENDPOINT_STOPPED = 3,
+	/** Endpoint is halted due to a TRB error */
+	XHCI_ENDPOINT_ERROR = 4,
+};
+
+/** Endpoint state mask */
+#define XHCI_ENDPOINT_STATE_MASK 0x07
+
+/** Endpoint type */
+#define XHCI_EP_TYPE(type) ( (type) << 3 )
+
+/** Control endpoint type */
+#define XHCI_EP_TYPE_CONTROL XHCI_EP_TYPE ( 4 )
+
+/** Input endpoint type */
+#define XHCI_EP_TYPE_IN XHCI_EP_TYPE ( 4 )
+
+/** Periodic endpoint type */
+#define XHCI_EP_TYPE_PERIODIC XHCI_EP_TYPE ( 1 )
+
+/** Endpoint dequeue cycle state */
+#define XHCI_EP_DCS 0x00000001UL
+
+/** Control endpoint average TRB length */
+#define XHCI_EP0_TRB_LEN 8
+
+/** An event ring segment */
+struct xhci_event_ring_segment {
+	/** Base address */
+	uint64_t base;
+	/** Number of TRBs */
+	uint32_t count;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** A transfer request block command/transfer ring */
+struct xhci_trb_ring {
+	/** Producer counter */
+	unsigned int prod;
+	/** Consumer counter */
+	unsigned int cons;
+	/** Ring size (log2) */
+	unsigned int shift;
+	/** Ring counter mask */
+	unsigned int mask;
+
+	/** I/O buffers */
+	struct io_buffer **iobuf;
+
+	/** Transfer request blocks */
+	union xhci_trb *trb;
+	/** Length of transfer request blocks */
+	size_t len;
+	/** Link TRB (if applicable) */
+	struct xhci_trb_link *link;
+
+	/** Doorbell register */
+	void *db;
+	/** Doorbell register value */
+	uint32_t dbval;
+};
+
+/** An event ring */
+struct xhci_event_ring {
+	/** Consumer counter */
+	unsigned int cons;
+	/** Event ring segment table */
+	struct xhci_event_ring_segment *segment;
+	/** Transfer request blocks */
+	union xhci_trb *trb;
+};
+
+/**
+ * Calculate doorbell register value
+ *
+ * @v target		Doorbell target
+ * @v stream		Doorbell stream ID
+ * @ret dbval		Doorbell register value
+ */
+#define XHCI_DBVAL( target, stream ) ( (target) | ( (stream) << 16 ) )
+
+/**
+ * Calculate space used in TRB ring
+ *
+ * @v ring		TRB ring
+ * @ret fill		Number of entries used
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_fill ( struct xhci_trb_ring *ring ) {
+
+	return ( ring->prod - ring->cons );
+}
+
+/**
+ * Calculate space remaining in TRB ring
+ *
+ * @v ring		TRB ring
+ * @ret remaining	Number of entries remaining
+ *
+ * xHCI does not allow us to completely fill a ring; there must be at
+ * least one free entry (excluding the Link TRB).
+ */
+static inline __attribute__ (( always_inline )) unsigned int
+xhci_ring_remaining ( struct xhci_trb_ring *ring ) {
+	unsigned int fill = xhci_ring_fill ( ring );
+
+	/* We choose to utilise rings with ( 2^n + 1 ) entries, with
+	 * the final entry being a Link TRB.  The maximum fill level
+	 * is therefore
+	 *
+	 *   ( ( 2^n + 1 ) - 1 (Link TRB) - 1 (one slot always empty)
+	 *       == ( 2^n - 1 )
+	 *
+	 * which is therefore equal to the ring mask.
+	 */
+	assert ( fill <= ring->mask );
+	return ( ring->mask - fill );
+}
+
+/**
+ * Calculate physical address of most recently consumed TRB
+ *
+ * @v ring		TRB ring
+ * @ret trb		TRB physical address
+ */
+static inline __attribute__ (( always_inline )) physaddr_t
+xhci_ring_consumed ( struct xhci_trb_ring *ring ) {
+	unsigned int index = ( ( ring->cons - 1 ) & ring->mask );
+
+	return virt_to_phys ( &ring->trb[index] );
+}
+
+/** Slot context index */
+#define XHCI_CTX_SLOT 0
+
+/** Calculate context index from USB endpoint address */
+#define XHCI_CTX(address)						\
+	( (address) ? ( ( ( (address) & 0x0f ) << 1 ) |			\
+			( ( (address) & 0x80 ) >> 7 ) ) : 1 )
+
+/** Endpoint zero context index */
+#define XHCI_CTX_EP0 XHCI_CTX ( 0x00 )
+
+/** End of contexts */
+#define XHCI_CTX_END 32
+
+/** Device context index */
+#define XHCI_DCI(ctx) ( (ctx) + 0 )
+
+/** Input context index */
+#define XHCI_ICI(ctx) ( (ctx) + 1 )
+
+/** Number of TRBs (excluding Link TRB) in the command ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_CMD_TRBS_LOG2 2
+
+/** Number of TRBs in the event ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_EVENT_TRBS_LOG2 6
+
+/** Number of TRBs in a transfer ring
+ *
+ * This is a policy decision.
+ */
+#define XHCI_TRANSFER_TRBS_LOG2 6
+
+/** Maximum time to wait for BIOS to release ownership
+ *
+ * This is a policy decision.
+ */
+#define XHCI_USBLEGSUP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for host controller to stop
+ *
+ * This is a policy decision.
+ */
+#define XHCI_STOP_MAX_WAIT_MS 100
+
+/** Maximum time to wait for reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_RESET_MAX_WAIT_MS 500
+
+/** Maximum time to wait for a command to complete
+ *
+ * The "address device" command involves waiting for a response to a
+ * USB control transaction, and so we must wait for up to the 5000ms
+ * that USB allows for devices to respond to control transactions.
+ */
+#define XHCI_COMMAND_MAX_WAIT_MS USB_CONTROL_MAX_WAIT_MS
+
+/** Time to delay after aborting a command
+ *
+ * This is a policy decision
+ */
+#define XHCI_COMMAND_ABORT_DELAY_MS 500
+
+/** Maximum time to wait for a port reset to complete
+ *
+ * This is a policy decision.
+ */
+#define XHCI_PORT_RESET_MAX_WAIT_MS 500
+
+/** Intel PCH quirk */
+struct xhci_pch {
+	/** USB2 port routing register original value */
+	uint32_t xusb2pr;
+	/** USB3 port SuperSpeed enable register original value */
+	uint32_t usb3pssen;
+};
+
+/** Intel PCH quirk flag */
+#define XHCI_PCH 0x0001
+
+/** Intel PCH USB2 port routing register */
+#define XHCI_PCH_XUSB2PR 0xd0
+
+/** Intel PCH USB2 port routing mask register */
+#define XHCI_PCH_XUSB2PRM 0xd4
+
+/** Intel PCH SuperSpeed enable register */
+#define XHCI_PCH_USB3PSSEN 0xd8
+
+/** Intel PCH USB3 port routing mask register */
+#define XHCI_PCH_USB3PRM 0xdc
+
+/** Invalid protocol speed ID values quirk */
+#define XHCI_BAD_PSIV 0x0002
+
+/** An xHCI device */
+struct xhci_device {
+	/** Registers */
+	void *regs;
+	/** Name */
+	const char *name;
+	/** Quirks */
+	unsigned int quirks;
+
+	/** Capability registers */
+	void *cap;
+	/** Operational registers */
+	void *op;
+	/** Runtime registers */
+	void *run;
+	/** Doorbell registers */
+	void *db;
+
+	/** Number of device slots */
+	unsigned int slots;
+	/** Number of interrupters */
+	unsigned int intrs;
+	/** Number of ports */
+	unsigned int ports;
+
+	/** Number of page-sized scratchpad buffers */
+	unsigned int scratchpads;
+
+	/** 64-bit addressing capability */
+	int addr64;
+	/** Context size shift */
+	unsigned int csz_shift;
+	/** xHCI extended capabilities offset */
+	unsigned int xecp;
+
+	/** Page size */
+	size_t pagesize;
+
+	/** USB legacy support capability (if present and enabled) */
+	unsigned int legacy;
+
+	/** Device context base address array */
+	uint64_t *dcbaa;
+
+	/** Scratchpad buffer area */
+	userptr_t scratchpad;
+	/** Scratchpad buffer array */
+	uint64_t *scratchpad_array;
+
+	/** Command ring */
+	struct xhci_trb_ring command;
+	/** Event ring */
+	struct xhci_event_ring event;
+	/** Current command (if any) */
+	union xhci_trb *pending;
+
+	/** Device slots, indexed by slot ID */
+	struct xhci_slot **slot;
+
+	/** USB bus */
+	struct usb_bus *bus;
+
+	/** Intel PCH quirk */
+	struct xhci_pch pch;
+};
+
+/** An xHCI device slot */
+struct xhci_slot {
+	/** xHCI device */
+	struct xhci_device *xhci;
+	/** USB device */
+	struct usb_device *usb;
+	/** Slot ID */
+	unsigned int id;
+	/** Slot context */
+	struct xhci_slot_context *context;
+	/** Route string */
+	unsigned int route;
+	/** Root hub port number */
+	unsigned int port;
+	/** Protocol speed ID */
+	unsigned int psiv;
+	/** Number of ports (if this device is a hub) */
+	unsigned int ports;
+	/** Transaction translator slot ID */
+	unsigned int tt_id;
+	/** Transaction translator port */
+	unsigned int tt_port;
+	/** Endpoints, indexed by context ID */
+	struct xhci_endpoint *endpoint[XHCI_CTX_END];
+};
+
+/** An xHCI endpoint */
+struct xhci_endpoint {
+	/** xHCI device */
+	struct xhci_device *xhci;
+	/** xHCI slot */
+	struct xhci_slot *slot;
+	/** USB endpoint */
+	struct usb_endpoint *ep;
+	/** Context index */
+	unsigned int ctx;
+	/** Endpoint type */
+	unsigned int type;
+	/** Endpoint interval */
+	unsigned int interval;
+	/** Endpoint context */
+	struct xhci_endpoint_context *context;
+	/** Transfer ring */
+	struct xhci_trb_ring ring;
+};
+
+#endif /* _IPXE_XHCI_H */
diff --git a/qemu/roms/ipxe/src/hci/commands/autoboot_cmd.c b/qemu/roms/ipxe/src/hci/commands/autoboot_cmd.c
index 62235a278..56f39a1ce 100644
--- a/qemu/roms/ipxe/src/hci/commands/autoboot_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/autoboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -25,7 +29,7 @@
 #include <hci/ifmgmt_cmd.h>
 #include <usr/autoboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/config_cmd.c b/qemu/roms/ipxe/src/hci/commands/config_cmd.c
index b81c866ff..ad415e045 100644
--- a/qemu/roms/ipxe/src/hci/commands/config_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/config_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <string.h>
@@ -26,7 +30,7 @@
 #include <ipxe/settings.h>
 #include <ipxe/settings_ui.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/console_cmd.c b/qemu/roms/ipxe/src/hci/commands/console_cmd.c
index d2eae59f0..ba472b9f6 100644
--- a/qemu/roms/ipxe/src/hci/commands/console_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/console_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/dhcp_cmd.c b/qemu/roms/ipxe/src/hci/commands/dhcp_cmd.c
index feeb55ee5..45a922b51 100644
--- a/qemu/roms/ipxe/src/hci/commands/dhcp_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/dhcp_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/fcmgmt_cmd.c b/qemu/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
index 1c199b5dc..97f10f4dd 100644
--- a/qemu/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/fcmgmt_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/gdbstub_cmd.c b/qemu/roms/ipxe/src/hci/commands/gdbstub_cmd.c
index 33890aebc..c4a831e7a 100644
--- a/qemu/roms/ipxe/src/hci/commands/gdbstub_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/gdbstub_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/ifmgmt_cmd.c b/qemu/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
index 5307c9423..c89af2e81 100644
--- a/qemu/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/ifmgmt_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/image_cmd.c b/qemu/roms/ipxe/src/hci/commands/image_cmd.c
index a9e831bf5..4a7c500a4 100644
--- a/qemu/roms/ipxe/src/hci/commands/image_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/image_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/image_trust_cmd.c b/qemu/roms/ipxe/src/hci/commands/image_trust_cmd.c
index ca59a858a..f9d6b5b3e 100644
--- a/qemu/roms/ipxe/src/hci/commands/image_trust_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/image_trust_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
@@ -169,6 +173,9 @@ struct command image_trust_commands[] __command = {
 	},
 };
 
+/* Drag in objects via command list */
+REQUIRING_SYMBOL ( image_trust_commands );
+
 /* Drag in objects typically required for signature verification */
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( md5 );
diff --git a/qemu/roms/ipxe/src/hci/commands/ipstat_cmd.c b/qemu/roms/ipxe/src/hci/commands/ipstat_cmd.c
index d565dc0ae..763e4dfd6 100644
--- a/qemu/roms/ipxe/src/hci/commands/ipstat_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/ipstat_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/login_cmd.c b/qemu/roms/ipxe/src/hci/commands/login_cmd.c
index f5db427d5..c9e196437 100644
--- a/qemu/roms/ipxe/src/hci/commands/login_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/login_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <string.h>
@@ -23,7 +27,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/login_ui.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/lotest_cmd.c b/qemu/roms/ipxe/src/hci/commands/lotest_cmd.c
index 0fa031bcb..a989932d4 100644
--- a/qemu/roms/ipxe/src/hci/commands/lotest_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/lotest_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/menu_cmd.c b/qemu/roms/ipxe/src/hci/commands/menu_cmd.c
index 66a6262e6..76bce8695 100644
--- a/qemu/roms/ipxe/src/hci/commands/menu_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/menu_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/neighbour_cmd.c b/qemu/roms/ipxe/src/hci/commands/neighbour_cmd.c
index a1e052439..816e87357 100644
--- a/qemu/roms/ipxe/src/hci/commands/neighbour_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/neighbour_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/nvo_cmd.c b/qemu/roms/ipxe/src/hci/commands/nvo_cmd.c
index e63dab08e..ac0d60651 100644
--- a/qemu/roms/ipxe/src/hci/commands/nvo_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/nvo_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdint.h>
@@ -29,7 +33,7 @@
 #include <ipxe/parseopt.h>
 #include <readline/readline.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/param_cmd.c b/qemu/roms/ipxe/src/hci/commands/param_cmd.c
index 6cf096d00..bff04f2ff 100644
--- a/qemu/roms/ipxe/src/hci/commands/param_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/param_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/pci_cmd.c b/qemu/roms/ipxe/src/hci/commands/pci_cmd.c
index f5145fb35..a2a811aa0 100644
--- a/qemu/roms/ipxe/src/hci/commands/pci_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/pci_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -23,7 +27,7 @@
 #include <ipxe/command.h>
 #include <ipxe/parseopt.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/ping_cmd.c b/qemu/roms/ipxe/src/hci/commands/ping_cmd.c
index 34807696f..ab271e75a 100644
--- a/qemu/roms/ipxe/src/hci/commands/ping_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/ping_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/poweroff_cmd.c b/qemu/roms/ipxe/src/hci/commands/poweroff_cmd.c
index 9d487d330..afdf12dde 100644
--- a/qemu/roms/ipxe/src/hci/commands/poweroff_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/poweroff_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -24,7 +28,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/reboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/profstat_cmd.c b/qemu/roms/ipxe/src/hci/commands/profstat_cmd.c
index e4c9e5a24..dc6f649e3 100644
--- a/qemu/roms/ipxe/src/hci/commands/profstat_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/profstat_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/reboot_cmd.c b/qemu/roms/ipxe/src/hci/commands/reboot_cmd.c
index 485939e42..45d54cc2c 100644
--- a/qemu/roms/ipxe/src/hci/commands/reboot_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/reboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <getopt.h>
@@ -22,7 +26,7 @@
 #include <ipxe/parseopt.h>
 #include <ipxe/reboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/route_cmd.c b/qemu/roms/ipxe/src/hci/commands/route_cmd.c
index cc5ffc2f2..8aa535363 100644
--- a/qemu/roms/ipxe/src/hci/commands/route_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/route_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <getopt.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/sanboot_cmd.c b/qemu/roms/ipxe/src/hci/commands/sanboot_cmd.c
index 5954b6326..24ec8bc4e 100644
--- a/qemu/roms/ipxe/src/hci/commands/sanboot_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/sanboot_cmd.c
@@ -15,6 +15,10 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #include <stdio.h>
@@ -27,7 +31,7 @@
 #include <ipxe/sanboot.h>
 #include <usr/autoboot.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/commands/sync_cmd.c b/qemu/roms/ipxe/src/hci/commands/sync_cmd.c
index adf7e3cc6..54799d422 100644
--- a/qemu/roms/ipxe/src/hci/commands/sync_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/sync_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/hci/commands/vlan_cmd.c b/qemu/roms/ipxe/src/hci/commands/vlan_cmd.c
index 5d7298220..8a2f0c749 100644
--- a/qemu/roms/ipxe/src/hci/commands/vlan_cmd.c
+++ b/qemu/roms/ipxe/src/hci/commands/vlan_cmd.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/hci/editstring.c b/qemu/roms/ipxe/src/hci/editstring.c
index 5f6f04d51..8cbce0767 100644
--- a/qemu/roms/ipxe/src/hci/editstring.c
+++ b/qemu/roms/ipxe/src/hci/editstring.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <assert.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/hci/jumpscroll.c b/qemu/roms/ipxe/src/hci/jumpscroll.c
new file mode 100644
index 000000000..dd6bcac2b
--- /dev/null
+++ b/qemu/roms/ipxe/src/hci/jumpscroll.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Jump scrolling
+ *
+ */
+
+#include <assert.h>
+#include <ipxe/keys.h>
+#include <ipxe/jumpscroll.h>
+
+/**
+ * Handle keypress
+ *
+ * @v scroll		Jump scroller
+ * @v key		Key pressed by user
+ * @ret move		Scroller movement, or zero
+ */
+int jump_scroll_key ( struct jump_scroller *scroll, int key ) {
+
+	/* Sanity checks */
+	assert ( scroll->rows != 0 );
+	assert ( scroll->count != 0 );
+	assert ( scroll->current < scroll->count );
+	assert ( scroll->first < scroll->count );
+	assert ( scroll->first <= scroll->current );
+	assert ( scroll->current < ( scroll->first + scroll->rows ) );
+
+	/* Handle key, if applicable */
+	switch ( key ) {
+	case KEY_UP:
+		return -1;
+	case KEY_DOWN:
+		return +1;
+	case KEY_PPAGE:
+		return ( scroll->first - scroll->current - 1 );
+	case KEY_NPAGE:
+		return ( scroll->first - scroll->current + scroll->rows );
+	case KEY_HOME:
+		return -( scroll->count );
+	case KEY_END:
+		return +( scroll->count );
+	default:
+		return 0;
+	}
+}
+
+/**
+ * Move scroller
+ *
+ * @v scroll		Jump scroller
+ * @v move		Scroller movement
+ * @ret move		Continuing scroller movement (if applicable)
+ */
+int jump_scroll_move ( struct jump_scroller *scroll, int move ) {
+	int current = scroll->current;
+	int last = ( scroll->count - 1 );
+
+	/* Sanity checks */
+	assert ( move != 0 );
+	assert ( scroll->count != 0 );
+
+	/* Move to the new current item */
+	current += move;
+
+	/* Check for start/end of list */
+	if ( current < 0 ) {
+		/* We have attempted to move before the start of the
+		 * list.  Move to the start of the list and continue
+		 * moving forwards (if applicable).
+		 */
+		scroll->current = 0;
+		return +1;
+	} else if ( current > last ) {
+		/* We have attempted to move after the end of the
+		 * list.  Move to the end of the list and continue
+		 * moving backwards (if applicable).
+		 */
+		scroll->current = last;
+		return -1;
+	} else {
+		/* Update the current item and continue moving in the
+		 * same direction (if applicable).
+		 */
+		scroll->current = current;
+		return ( ( move > 0 ) ? +1 : -1 );
+	}
+}
+
+/**
+ * Jump scroll to new page (if applicable)
+ *
+ * @v scroll		Jump scroller
+ * @ret jumped		Jumped to a new page
+ */
+int jump_scroll ( struct jump_scroller *scroll ) {
+	unsigned int index;
+
+	/* Sanity checks */
+	assert ( scroll->rows != 0 );
+	assert ( scroll->count != 0 );
+	assert ( scroll->current < scroll->count );
+	assert ( scroll->first < scroll->count );
+
+	/* Do nothing if we are already on the correct page */
+	index = ( scroll->current - scroll->first );
+	if ( index < scroll->rows )
+		return 0;
+
+	/* Move to required page */
+	while ( scroll->first < scroll->current )
+		scroll->first += scroll->rows;
+	while ( scroll->first > scroll->current )
+		scroll->first -= scroll->rows;
+
+	return 1;
+}
diff --git a/qemu/roms/ipxe/src/hci/mucurses/alert.c b/qemu/roms/ipxe/src/hci/mucurses/alert.c
index 00e959a89..7dc61c222 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/alert.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/alert.c
@@ -7,6 +7,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Audible signal
  *
diff --git a/qemu/roms/ipxe/src/hci/mucurses/ansi_screen.c b/qemu/roms/ipxe/src/hci/mucurses/ansi_screen.c
index 1d3143f89..1cf3309dd 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/ansi_screen.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/ansi_screen.c
@@ -3,7 +3,7 @@
 #include <ipxe/ansicol.h>
 #include <ipxe/console.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static void ansiscr_reset(struct _curses_screen *scr) __nonnull;
 static void ansiscr_movetoyx(struct _curses_screen *scr,
diff --git a/qemu/roms/ipxe/src/hci/mucurses/clear.c b/qemu/roms/ipxe/src/hci/mucurses/clear.c
index f5e52ca20..2054f72cc 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/clear.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/clear.c
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Clear a window to the bottom from current cursor position
diff --git a/qemu/roms/ipxe/src/hci/mucurses/colour.c b/qemu/roms/ipxe/src/hci/mucurses/colour.c
index c1359c868..b0c480b1f 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/colour.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/colour.c
@@ -1,6 +1,6 @@
 #include <curses.h>
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct colour_pair {
 	short fcol;
diff --git a/qemu/roms/ipxe/src/hci/mucurses/cursor.h b/qemu/roms/ipxe/src/hci/mucurses/cursor.h
index 16b7d27c2..2e0c896a6 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/cursor.h
+++ b/qemu/roms/ipxe/src/hci/mucurses/cursor.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct cursor_pos {
 	unsigned int y, x;
diff --git a/qemu/roms/ipxe/src/hci/mucurses/edging.c b/qemu/roms/ipxe/src/hci/mucurses/edging.c
index eccd32422..e938d338b 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/edging.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/edging.c
@@ -8,6 +8,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Draw borders from single-byte characters and renditions around a
  * window
diff --git a/qemu/roms/ipxe/src/hci/mucurses/kb.c b/qemu/roms/ipxe/src/hci/mucurses/kb.c
index b38c8c146..8face14d8 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/kb.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/kb.c
@@ -8,6 +8,8 @@
  * MuCurses keyboard input handling functions
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #define INPUT_DELAY 		200 // half-blocking delay timer resolution (ms)
 #define INPUT_DELAY_TIMEOUT 	1000 // half-blocking delay timeout
 
diff --git a/qemu/roms/ipxe/src/hci/mucurses/mucurses.c b/qemu/roms/ipxe/src/hci/mucurses/mucurses.c
index b67445baf..98a8a2c59 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/mucurses.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/mucurses.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 static void _wupdcurs ( WINDOW *win ) __nonnull;
 void _wputch ( WINDOW *win, chtype ch, int wrap ) __nonnull;
diff --git a/qemu/roms/ipxe/src/hci/mucurses/mucurses.h b/qemu/roms/ipxe/src/hci/mucurses/mucurses.h
index 7ac1086ac..270394787 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/mucurses.h
+++ b/qemu/roms/ipxe/src/hci/mucurses/mucurses.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #define WRAP 0
 #define NOWRAP 1
diff --git a/qemu/roms/ipxe/src/hci/mucurses/print.c b/qemu/roms/ipxe/src/hci/mucurses/print.c
index 9c682588b..e8831c58f 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/print.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/print.c
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Add a single-byte character and rendition to a window and advance
diff --git a/qemu/roms/ipxe/src/hci/mucurses/print_nadv.c b/qemu/roms/ipxe/src/hci/mucurses/print_nadv.c
index ee472e685..3a44e5bd2 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/print_nadv.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/print_nadv.c
@@ -8,6 +8,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Add string of single-byte characters and renditions to a window
  *
diff --git a/qemu/roms/ipxe/src/hci/mucurses/slk.c b/qemu/roms/ipxe/src/hci/mucurses/slk.c
index 600658e75..660eb65c0 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/slk.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/slk.c
@@ -11,6 +11,8 @@
  * Soft label key functions
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #define MIN_SPACE_SIZE 2
 
 #define SLK_MAX_LABEL_LEN 8
diff --git a/qemu/roms/ipxe/src/hci/mucurses/widgets/editbox.c b/qemu/roms/ipxe/src/hci/mucurses/widgets/editbox.c
index 630a66e0b..210de4481 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/widgets/editbox.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/widgets/editbox.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <assert.h>
diff --git a/qemu/roms/ipxe/src/hci/mucurses/winattrs.c b/qemu/roms/ipxe/src/hci/mucurses/winattrs.c
index f549d7519..97a5a18b3 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/winattrs.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/winattrs.c
@@ -6,7 +6,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Get the background rendition attributes for a window
diff --git a/qemu/roms/ipxe/src/hci/mucurses/windows.c b/qemu/roms/ipxe/src/hci/mucurses/windows.c
index 63d0af08c..7f39bdea2 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/windows.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/windows.c
@@ -9,6 +9,8 @@
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 /**
  * Delete a window
  *
diff --git a/qemu/roms/ipxe/src/hci/mucurses/wininit.c b/qemu/roms/ipxe/src/hci/mucurses/wininit.c
index b75abba44..dd84d2f1d 100644
--- a/qemu/roms/ipxe/src/hci/mucurses/wininit.c
+++ b/qemu/roms/ipxe/src/hci/mucurses/wininit.c
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Initialise console environment
diff --git a/qemu/roms/ipxe/src/hci/readline.c b/qemu/roms/ipxe/src/hci/readline.c
index 40aa59787..83a2e0b90 100644
--- a/qemu/roms/ipxe/src/hci/readline.c
+++ b/qemu/roms/ipxe/src/hci/readline.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/hci/shell.c b/qemu/roms/ipxe/src/hci/shell.c
index c1a543849..276eb3527 100644
--- a/qemu/roms/ipxe/src/hci/shell.c
+++ b/qemu/roms/ipxe/src/hci/shell.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -28,6 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/command.h>
 #include <ipxe/parseopt.h>
 #include <ipxe/shell.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -36,7 +41,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /** The shell prompt string */
-static const char shell_prompt[] = "iPXE> ";
+static const char shell_prompt[] = PRODUCT_SHORT_NAME "> ";
 
 /**
  * "help" command
diff --git a/qemu/roms/ipxe/src/hci/strerror.c b/qemu/roms/ipxe/src/hci/strerror.c
index 9356e9e0a..1bba8c620 100644
--- a/qemu/roms/ipxe/src/hci/strerror.c
+++ b/qemu/roms/ipxe/src/hci/strerror.c
@@ -2,6 +2,7 @@
 #include <string.h>
 #include <stdio.h>
 #include <ipxe/errortab.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -18,7 +19,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Find error description
@@ -74,7 +75,7 @@ static struct errortab * find_closest_error ( int errno ) {
  * call to strerror().
  *
  */
-const char * strerror ( int errno ) {
+char * strerror ( int errno ) {
 	static char errbuf[64];
 	struct errortab *errortab;
 
@@ -88,11 +89,11 @@ const char * strerror ( int errno ) {
 	/* Construct the error message */
 	if ( errortab ) {
 		snprintf ( errbuf, sizeof ( errbuf ),
-			   "%s (http://ipxe.org/%08x)",
+			   "%s (" PRODUCT_ERROR_URI ")",
 			   errortab->text, errno );
 	} else {
 		snprintf ( errbuf, sizeof ( errbuf ),
-			   "Error %#08x (http://ipxe.org/%08x)",
+			   "Error %#08x (" PRODUCT_ERROR_URI ")",
 			   errno, errno );
 	}
 
diff --git a/qemu/roms/ipxe/src/hci/tui/login_ui.c b/qemu/roms/ipxe/src/hci/tui/login_ui.c
index 996b68a0a..3c55325d5 100644
--- a/qemu/roms/ipxe/src/hci/tui/login_ui.c
+++ b/qemu/roms/ipxe/src/hci/tui/login_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/hci/tui/menu_ui.c b/qemu/roms/ipxe/src/hci/tui/menu_ui.c
index 0a9566def..f9dd9d100 100644
--- a/qemu/roms/ipxe/src/hci/tui/menu_ui.c
+++ b/qemu/roms/ipxe/src/hci/tui/menu_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -32,6 +36,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/timer.h>
 #include <ipxe/console.h>
 #include <ipxe/ansicol.h>
+#include <ipxe/jumpscroll.h>
 #include <ipxe/menu.h>
 
 /* Screen layout */
@@ -46,12 +51,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 struct menu_ui {
 	/** Menu */
 	struct menu *menu;
-	/** Number of menu items */
-	unsigned int count;
-	/** Currently selected item */
-	int selected;
-	/** First visible item */
-	int first_visible;
+	/** Jump scroller */
+	struct jump_scroller scroll;
 	/** Timeout (0=indefinite) */
 	unsigned long timeout;
 };
@@ -80,7 +81,7 @@ static struct menu_item * menu_item ( struct menu *menu, unsigned int index ) {
  * @v ui		Menu user interface
  * @v index		Index
  */
-static void draw_menu_item ( struct menu_ui *ui, int index ) {
+static void draw_menu_item ( struct menu_ui *ui, unsigned int index ) {
 	struct menu_item *item;
 	unsigned int row_offset;
 	char buf[ MENU_COLS + 1 /* NUL */ ];
@@ -90,7 +91,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 	size_t len;
 
 	/* Move to start of row */
-	row_offset = ( index - ui->first_visible );
+	row_offset = ( index - ui->scroll.first );
 	move ( ( MENU_ROW + row_offset ), MENU_COL );
 
 	/* Get menu item */
@@ -102,7 +103,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 			color_set ( CPAIR_SEPARATOR, NULL );
 
 		/* Highlight if this is the selected item */
-		if ( index == ui->selected ) {
+		if ( index == ui->scroll.current ) {
 			color_set ( CPAIR_SELECT, NULL );
 			attron ( A_BOLD );
 		}
@@ -121,7 +122,7 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 			snprintf ( timeout_buf, sizeof ( timeout_buf ), "(%ld)",
 				   ( ( ui->timeout + TICKS_PER_SEC - 1 ) /
 				     TICKS_PER_SEC ) );
-		if ( ( index == ui->selected ) && ( ui->timeout != 0 ) ) {
+		if ( ( index == ui->scroll.current ) && ( ui->timeout != 0 ) ) {
 			memcpy ( ( buf + MENU_COLS - MENU_PAD - timeout_len ),
 				 timeout_buf, timeout_len );
 		}
@@ -150,24 +151,17 @@ static void draw_menu_item ( struct menu_ui *ui, int index ) {
 static void draw_menu_items ( struct menu_ui *ui ) {
 	unsigned int i;
 
-	/* Jump scroll to correct point in list */
-	while ( ui->first_visible < ui->selected )
-		ui->first_visible += MENU_ROWS;
-	while ( ui->first_visible > ui->selected )
-		ui->first_visible -= MENU_ROWS;
-
 	/* Draw ellipses before and/or after the list as necessary */
 	color_set ( CPAIR_SEPARATOR, NULL );
 	mvaddstr ( ( MENU_ROW - 1 ), ( MENU_COL + MENU_PAD ),
-		   ( ( ui->first_visible > 0 ) ? "..." : "   " ) );
+		   ( jump_scroll_is_first ( &ui->scroll ) ? "   " : "..." ) );
 	mvaddstr ( ( MENU_ROW + MENU_ROWS ), ( MENU_COL + MENU_PAD ),
-		   ( ( ( ui->first_visible + MENU_ROWS ) < ui->count ) ?
-		     "..." : "   " ) );
+		   ( jump_scroll_is_last ( &ui->scroll ) ? "   " : "..." ) );
 	color_set ( CPAIR_NORMAL, NULL );
 
 	/* Draw visible items */
 	for ( i = 0 ; i < MENU_ROWS ; i++ )
-		draw_menu_item ( ui, ( ui->first_visible + i ) );
+		draw_menu_item ( ui, ( ui->scroll.first + i ) );
 }
 
 /**
@@ -180,8 +174,7 @@ static void draw_menu_items ( struct menu_ui *ui ) {
 static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 	struct menu_item *item;
 	unsigned long timeout;
-	unsigned int delta;
-	int current;
+	unsigned int previous;
 	int key;
 	int i;
 	int move;
@@ -190,7 +183,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 
 	do {
 		/* Record current selection */
-		current = ui->selected;
+		previous = ui->scroll.current;
 
 		/* Calculate timeout as remainder of current second */
 		timeout = ( ui->timeout % TICKS_PER_SEC );
@@ -209,27 +202,11 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 			/* Cancel any timeout */
 			ui->timeout = 0;
 
-			/* Handle key */
+			/* Handle scroll keys */
+			move = jump_scroll_key ( &ui->scroll, key );
+
+			/* Handle other keys */
 			switch ( key ) {
-			case KEY_UP:
-				move = -1;
-				break;
-			case KEY_DOWN:
-				move = +1;
-				break;
-			case KEY_PPAGE:
-				move = ( ui->first_visible - ui->selected - 1 );
-				break;
-			case KEY_NPAGE:
-				move = ( ui->first_visible - ui->selected
-					 + MENU_ROWS );
-				break;
-			case KEY_HOME:
-				move = -ui->count;
-				break;
-			case KEY_END:
-				move = +ui->count;
-				break;
 			case ESC:
 			case CTRL_C:
 				rc = -ECANCELED;
@@ -247,7 +224,7 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 						i++;
 						continue;
 					}
-					ui->selected = i;
+					ui->scroll.current = i;
 					if ( item->label ) {
 						chosen = 1;
 					} else {
@@ -260,31 +237,22 @@ static int menu_loop ( struct menu_ui *ui, struct menu_item **selected ) {
 
 		/* Move selection, if applicable */
 		while ( move ) {
-			ui->selected += move;
-			if ( ui->selected < 0 ) {
-				ui->selected = 0;
-				move = +1;
-			} else if ( ui->selected >= ( int ) ui->count ) {
-				ui->selected = ( ui->count - 1 );
-				move = -1;
-			}
-			item = menu_item ( ui->menu, ui->selected );
+			move = jump_scroll_move ( &ui->scroll, move );
+			item = menu_item ( ui->menu, ui->scroll.current );
 			if ( item->label )
 				break;
-			move = ( ( move > 0 ) ? +1 : -1 );
 		}
 
 		/* Redraw selection if necessary */
-		if ( ( ui->selected != current ) || ( timeout != 0 ) ) {
-			draw_menu_item ( ui, current );
-			delta = ( ui->selected - ui->first_visible );
-			if ( delta >= MENU_ROWS )
+		if ( ( ui->scroll.current != previous ) || ( timeout != 0 ) ) {
+			draw_menu_item ( ui, previous );
+			if ( jump_scroll ( &ui->scroll ) )
 				draw_menu_items ( ui );
-			draw_menu_item ( ui, ui->selected );
+			draw_menu_item ( ui, ui->scroll.current );
 		}
 
 		/* Record selection */
-		item = menu_item ( ui->menu, ui->selected );
+		item = menu_item ( ui->menu, ui->scroll.current );
 		assert ( item != NULL );
 		assert ( item->label != NULL );
 		*selected = item;
@@ -313,21 +281,22 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	/* Initialise UI */
 	memset ( &ui, 0, sizeof ( ui ) );
 	ui.menu = menu;
+	ui.scroll.rows = MENU_ROWS;
 	ui.timeout = timeout;
 	list_for_each_entry ( item, &menu->items, list ) {
 		if ( item->label ) {
 			if ( ! labelled_count )
-				ui.selected = ui.count;
+				ui.scroll.current = ui.scroll.count;
 			labelled_count++;
 			if ( select ) {
 				if ( strcmp ( select, item->label ) == 0 )
-					ui.selected = ui.count;
+					ui.scroll.current = ui.scroll.count;
 			} else {
 				if ( item->is_default )
-					ui.selected = ui.count;
+					ui.scroll.current = ui.scroll.count;
 			}
 		}
-		ui.count++;
+		ui.scroll.count++;
 	}
 	if ( ! labelled_count ) {
 		/* Menus with no labelled items cannot be selected
@@ -349,8 +318,9 @@ int show_menu ( struct menu *menu, unsigned long timeout,
 	snprintf ( buf, sizeof ( buf ), "%s", ui.menu->title );
 	mvprintw ( TITLE_ROW, ( ( COLS - strlen ( buf ) ) / 2 ), "%s", buf );
 	attroff ( A_BOLD );
+	jump_scroll ( &ui.scroll );
 	draw_menu_items ( &ui );
-	draw_menu_item ( &ui, ui.selected );
+	draw_menu_item ( &ui, ui.scroll.current );
 
 	/* Enter main loop */
 	rc = menu_loop ( &ui, selected );
diff --git a/qemu/roms/ipxe/src/hci/tui/settings_ui.c b/qemu/roms/ipxe/src/hci/tui/settings_ui.c
index 221839730..be421cc0a 100644
--- a/qemu/roms/ipxe/src/hci/tui/settings_ui.c
+++ b/qemu/roms/ipxe/src/hci/tui/settings_ui.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <stdarg.h>
@@ -29,7 +33,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/editbox.h>
 #include <ipxe/keys.h>
 #include <ipxe/ansicol.h>
+#include <ipxe/jumpscroll.h>
 #include <ipxe/settings_ui.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -47,7 +53,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define INSTRUCTION_ROW		( LINES - 2U )
 #define INSTRUCTION_PAD "     "
 
-/** Layout of text within a setting widget */
+/** Layout of text within a setting row */
 #define SETTING_ROW_TEXT( cols ) struct {				\
 	char start[0];							\
 	char pad1[1];							\
@@ -63,8 +69,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 	char nul;							\
 } __attribute__ (( packed ))
 
-/** A setting row widget */
-struct setting_row_widget {
+/** A settings user interface row */
+struct settings_ui_row {
 	/** Target configuration settings block
 	 *
 	 * Valid only for rows that lead to new settings blocks.
@@ -82,8 +88,6 @@ struct setting_row_widget {
 	struct setting setting;
 	/** Screen row */
 	unsigned int row;
-	/** Screen column */
-	unsigned int col;
 	/** Edit box widget used for editing setting */
 	struct edit_box editbox;
 	/** Editing in progress flag */
@@ -92,28 +96,24 @@ struct setting_row_widget {
 	char value[256]; /* enough size for a DHCP string */
 };
 
-/** A settings widget */
-struct setting_widget {
+/** A settings user interface */
+struct settings_ui {
 	/** Settings block */
 	struct settings *settings;
-	/** Number of rows */
-	unsigned int num_rows;
-	/** Current row index */
-	unsigned int current;
-        /** Index of the first visible row, for scrolling. */
-	unsigned int first_visible;
-	/** Active row */
-	struct setting_row_widget row;
+	/** Jump scroller */
+	struct jump_scroller scroll;
+	/** Current row */
+	struct settings_ui_row row;
 };
 
 /**
- * Select a setting row
+ * Select a setting
  *
- * @v widget		Setting widget
+ * @v ui		Settings user interface
  * @v index		Index of setting row
- * @ret count		Number of settings rows
+ * @ret count		Number of setting rows
  */
-static unsigned int select_setting_row ( struct setting_widget *widget,
+static unsigned int select_setting_row ( struct settings_ui *ui,
 					 unsigned int index ) {
 	SETTING_ROW_TEXT ( COLS ) *text;
 	struct settings *settings;
@@ -122,25 +122,22 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 	unsigned int count = 0;
 
 	/* Initialise structure */
-	memset ( &widget->row, 0, sizeof ( widget->row ) );
-	widget->current = index;
-	widget->row.row = ( SETTINGS_LIST_ROW + index - widget->first_visible );
-	widget->row.col = SETTINGS_LIST_COL;
+	memset ( &ui->row, 0, sizeof ( ui->row ) );
+	ui->row.row = ( SETTINGS_LIST_ROW + index - ui->scroll.first );
 
 	/* Include parent settings block, if applicable */
-	if ( widget->settings->parent && ( count++ == index ) ) {
-		widget->row.settings = widget->settings->parent;
-		snprintf ( widget->row.value, sizeof ( widget->row.value ),
+	if ( ui->settings->parent && ( count++ == index ) ) {
+		ui->row.settings = ui->settings->parent;
+		snprintf ( ui->row.value, sizeof ( ui->row.value ),
 			   "../" );
 	}
 
 	/* Include any child settings blocks, if applicable */
-	list_for_each_entry ( settings, &widget->settings->children, siblings ){
+	list_for_each_entry ( settings, &ui->settings->children, siblings ) {
 		if ( count++ == index ) {
-			widget->row.settings = settings;
-			snprintf ( widget->row.value,
-				   sizeof ( widget->row.value ), "%s/",
-				   settings->name );
+			ui->row.settings = settings;
+			snprintf ( ui->row.value, sizeof ( ui->row.value ),
+				   "%s/", settings->name );
 		}
 	}
 
@@ -148,7 +145,7 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 	for_each_table_entry ( setting, SETTINGS ) {
 
 		/* Skip inapplicable settings */
-		if ( ! setting_applies ( widget->settings, setting ) )
+		if ( ! setting_applies ( ui->settings, setting ) )
 			continue;
 
 		/* Skip duplicate settings */
@@ -158,18 +155,16 @@ static unsigned int select_setting_row ( struct setting_widget *widget,
 
 		/* Read current setting value and origin */
 		if ( count++ == index ) {
-			fetchf_setting ( widget->settings, setting,
-					 &widget->row.origin,
-					 &widget->row.setting,
-					 widget->row.value,
-					 sizeof ( widget->row.value ) );
+			fetchf_setting ( ui->settings, setting, &ui->row.origin,
+					 &ui->row.setting, ui->row.value,
+					 sizeof ( ui->row.value ) );
 		}
 	}
 
 	/* Initialise edit box */
-	init_editbox ( &widget->row.editbox, widget->row.value,
-		       sizeof ( widget->row.value ), NULL, widget->row.row,
-		       ( widget->row.col +
+	init_editbox ( &ui->row.editbox, ui->row.value,
+		       sizeof ( ui->row.value ), NULL, ui->row.row,
+		       ( SETTINGS_LIST_COL +
 			 offsetof ( typeof ( *text ), u.setting.value ) ),
 		       sizeof ( text->u.setting.value ), 0 );
 
@@ -197,9 +192,9 @@ static size_t string_copy ( char *dest, const char *src, size_t len ) {
 /**
  * Draw setting row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_setting_row ( struct setting_widget *widget ) {
+static void draw_setting_row ( struct settings_ui *ui ) {
 	SETTING_ROW_TEXT ( COLS ) text;
 	unsigned int curs_offset;
 	char *value;
@@ -209,12 +204,12 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 	text.nul = '\0';
 
 	/* Construct row content */
-	if ( widget->row.settings ) {
+	if ( ui->row.settings ) {
 
 		/* Construct space-padded name */
 		curs_offset = ( offsetof ( typeof ( text ), u.settings ) +
 				string_copy ( text.u.settings,
-					      widget->row.value,
+					      ui->row.value,
 					      sizeof ( text.u.settings ) ) );
 
 	} else {
@@ -222,11 +217,11 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 		/* Construct dot-padded name */
 		memset ( text.u.setting.name, '.',
 			 sizeof ( text.u.setting.name ) );
-		string_copy ( text.u.setting.name, widget->row.setting.name,
+		string_copy ( text.u.setting.name, ui->row.setting.name,
 			      sizeof ( text.u.setting.name ) );
 
 		/* Construct space-padded value */
-		value = widget->row.value;
+		value = ui->row.value;
 		if ( ! *value )
 			value = "<not specified>";
 		curs_offset = ( offsetof ( typeof ( text ), u.setting.value ) +
@@ -235,37 +230,34 @@ static void draw_setting_row ( struct setting_widget *widget ) {
 	}
 
 	/* Print row */
-	if ( ( widget->row.origin == widget->settings ) ||
-	     ( widget->row.settings != NULL ) ) {
+	if ( ( ui->row.origin == ui->settings ) || ( ui->row.settings != NULL ))
 		attron ( A_BOLD );
-	}
-	mvprintw ( widget->row.row, widget->row.col, "%s", text.start );
+	mvprintw ( ui->row.row, SETTINGS_LIST_COL, "%s", text.start );
 	attroff ( A_BOLD );
-	move ( widget->row.row, widget->row.col + curs_offset );
+	move ( ui->row.row, ( SETTINGS_LIST_COL + curs_offset ) );
 }
 
 /**
- * Edit setting widget
+ * Edit setting ui
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  * @v key		Key pressed by user
  * @ret key		Key returned to application, or zero
  */
-static int edit_setting ( struct setting_widget *widget, int key ) {
-	assert ( widget->row.setting.name != NULL );
-	widget->row.editing = 1;
-	return edit_editbox ( &widget->row.editbox, key );
+static int edit_setting ( struct settings_ui *ui, int key ) {
+	assert ( ui->row.setting.name != NULL );
+	ui->row.editing = 1;
+	return edit_editbox ( &ui->row.editbox, key );
 }
 
 /**
- * Save setting widget value back to configuration settings
+ * Save setting ui value back to configuration settings
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static int save_setting ( struct setting_widget *widget ) {
-	assert ( widget->row.setting.name != NULL );
-	return storef_setting ( widget->settings, &widget->row.setting,
-				widget->row.value );
+static int save_setting ( struct settings_ui *ui ) {
+	assert ( ui->row.setting.name != NULL );
+	return storef_setting ( ui->settings, &ui->row.setting, ui->row.value );
 }
 
 /**
@@ -340,15 +332,15 @@ static void alert ( const char *fmt, ... ) {
 /**
  * Draw title row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_title_row ( struct setting_widget *widget ) {
+static void draw_title_row ( struct settings_ui *ui ) {
 	const char *name;
 
 	clearmsg ( TITLE_ROW );
-	name = settings_name ( widget->settings );
+	name = settings_name ( ui->settings );
 	attron ( A_BOLD );
-	msg ( TITLE_ROW, "iPXE configuration settings%s%s",
+	msg ( TITLE_ROW, PRODUCT_SHORT_NAME " configuration settings%s%s",
 	      ( name[0] ? " - " : "" ), name );
 	attroff ( A_BOLD );
 }
@@ -356,89 +348,73 @@ static void draw_title_row ( struct setting_widget *widget ) {
 /**
  * Draw information row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_info_row ( struct setting_widget *widget ) {
+static void draw_info_row ( struct settings_ui *ui ) {
 	char buf[32];
 
 	/* Draw nothing unless this row represents a setting */
 	clearmsg ( INFO_ROW );
 	clearmsg ( INFO_ROW + 1 );
-	if ( ! widget->row.setting.name )
+	if ( ! ui->row.setting.name )
 		return;
 
 	/* Determine a suitable setting name */
-	setting_name ( ( widget->row.origin ?
-			 widget->row.origin : widget->settings ),
-		       &widget->row.setting, buf, sizeof ( buf ) );
+	setting_name ( ( ui->row.origin ?
+			 ui->row.origin : ui->settings ),
+		       &ui->row.setting, buf, sizeof ( buf ) );
 
 	/* Draw row */
 	attron ( A_BOLD );
-	msg ( INFO_ROW, "%s - %s", buf, widget->row.setting.description );
+	msg ( INFO_ROW, "%s - %s", buf, ui->row.setting.description );
 	attroff ( A_BOLD );
 	color_set ( CPAIR_URL, NULL );
-	msg ( ( INFO_ROW + 1 ), "http://ipxe.org/cfg/%s",
-	      widget->row.setting.name );
+	msg ( ( INFO_ROW + 1 ), PRODUCT_SETTING_URI, ui->row.setting.name );
 	color_set ( CPAIR_NORMAL, NULL );
 }
 
 /**
  * Draw instruction row
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  */
-static void draw_instruction_row ( struct setting_widget *widget ) {
+static void draw_instruction_row ( struct settings_ui *ui ) {
 
 	clearmsg ( INSTRUCTION_ROW );
-	if ( widget->row.editing ) {
+	if ( ui->row.editing ) {
 		msg ( INSTRUCTION_ROW,
 		      "Enter - accept changes" INSTRUCTION_PAD
 		      "Ctrl-C - discard changes" );
 	} else {
 		msg ( INSTRUCTION_ROW,
 		      "%sCtrl-X - exit configuration utility",
-		      ( ( widget->row.origin == widget->settings ) ?
+		      ( ( ui->row.origin == ui->settings ) ?
 			"Ctrl-D - delete setting" INSTRUCTION_PAD : "" ) );
 	}
 }
 
 /**
- * Reveal setting row
+ * Draw the current block of setting rows
  *
- * @v widget		Setting widget
- * @v index		Index of setting row
+ * @v ui		Settings UI
  */
-static void reveal_setting_row ( struct setting_widget *widget,
-				 unsigned int index ) {
+static void draw_setting_rows ( struct settings_ui *ui ) {
 	unsigned int i;
 
-	/* Simply return if setting N is already on-screen. */
-	if ( index - widget->first_visible < SETTINGS_LIST_ROWS )
-		return;
-
-	/* Jump scroll to make the specified setting row visible. */
-	while ( widget->first_visible < index )
-		widget->first_visible += SETTINGS_LIST_ROWS;
-	while ( widget->first_visible > index )
-		widget->first_visible -= SETTINGS_LIST_ROWS;
-
-	/* Draw ellipses before and/or after the settings list to
-	 * represent any invisible settings.
-	 */
-	mvaddstr ( SETTINGS_LIST_ROW - 1,
-		   SETTINGS_LIST_COL + 1,
-		   widget->first_visible > 0 ? "..." : "   " );
-	mvaddstr ( SETTINGS_LIST_ROW + SETTINGS_LIST_ROWS,
-		   SETTINGS_LIST_COL + 1,
-		   ( ( widget->first_visible + SETTINGS_LIST_ROWS )
-		     < widget->num_rows ? "..." : "   " ) );
+	/* Draw ellipses before and/or after the list as necessary */
+	color_set ( CPAIR_SEPARATOR, NULL );
+	mvaddstr ( ( SETTINGS_LIST_ROW - 1 ), ( SETTINGS_LIST_COL + 1 ),
+		   jump_scroll_is_first ( &ui->scroll ) ? "   " : "..." );
+	mvaddstr ( ( SETTINGS_LIST_ROW + SETTINGS_LIST_ROWS ),
+		   ( SETTINGS_LIST_COL + 1 ),
+		   jump_scroll_is_last ( &ui->scroll ) ? "   " : "..." );
+	color_set ( CPAIR_NORMAL, NULL );
 
 	/* Draw visible settings. */
-	for ( i = 0; i < SETTINGS_LIST_ROWS; i++ ) {
-		if ( ( widget->first_visible + i ) < widget->num_rows ) {
-			select_setting_row ( widget,
-					     widget->first_visible + i );
-			draw_setting_row ( widget );
+	for ( i = 0 ; i < SETTINGS_LIST_ROWS ; i++ ) {
+		if ( ( ui->scroll.first + i ) < ui->scroll.count ) {
+			select_setting_row ( ui, ( ui->scroll.first + i ) );
+			draw_setting_row ( ui );
 		} else {
 			clearmsg ( SETTINGS_LIST_ROW + i );
 		}
@@ -446,69 +422,72 @@ static void reveal_setting_row ( struct setting_widget *widget,
 }
 
 /**
- * Reveal setting row
+ * Select settings block
  *
- * @v widget		Setting widget
+ * @v ui		Settings UI
  * @v settings		Settings block
  */
-static void init_widget ( struct setting_widget *widget,
-			  struct settings *settings ) {
-
-	widget->settings = settings_target ( settings );
-	widget->num_rows = select_setting_row ( widget, 0 );
-	widget->first_visible = SETTINGS_LIST_ROWS;
-	draw_title_row ( widget );
-	reveal_setting_row ( widget, 0 );
-	select_setting_row ( widget, 0 );
+static void select_settings ( struct settings_ui *ui,
+			      struct settings *settings ) {
+
+	ui->settings = settings_target ( settings );
+	ui->scroll.count = select_setting_row ( ui, 0 );
+	ui->scroll.rows = SETTINGS_LIST_ROWS;
+	ui->scroll.current = 0;
+	ui->scroll.first = 0;
+	draw_title_row ( ui );
+	draw_setting_rows ( ui );
+	select_setting_row ( ui, 0 );
 }
 
 static int main_loop ( struct settings *settings ) {
-	struct setting_widget widget;
+	struct settings_ui ui;
+	unsigned int previous;
 	int redraw = 1;
 	int move;
-	unsigned int next;
 	int key;
 	int rc;
 
 	/* Print initial screen content */
 	color_set ( CPAIR_NORMAL, NULL );
-	memset ( &widget, 0, sizeof ( widget ) );
-	init_widget ( &widget, settings );
+	memset ( &ui, 0, sizeof ( ui ) );
+	select_settings ( &ui, settings );
 
 	while ( 1 ) {
 
 		/* Redraw rows if necessary */
 		if ( redraw ) {
-			draw_info_row ( &widget );
-			draw_instruction_row ( &widget );
-			color_set ( ( widget.row.editing ?
+			draw_info_row ( &ui );
+			draw_instruction_row ( &ui );
+			color_set ( ( ui.row.editing ?
 				      CPAIR_EDIT : CPAIR_SELECT ), NULL );
-			draw_setting_row ( &widget );
+			draw_setting_row ( &ui );
 			color_set ( CPAIR_NORMAL, NULL );
-			curs_set ( widget.row.editing );
+			curs_set ( ui.row.editing );
 			redraw = 0;
 		}
 
-		if ( widget.row.editing ) {
+		/* Edit setting, if we are currently editing */
+		if ( ui.row.editing ) {
 
 			/* Sanity check */
-			assert ( widget.row.setting.name != NULL );
+			assert ( ui.row.setting.name != NULL );
 
 			/* Redraw edit box */
 			color_set ( CPAIR_EDIT, NULL );
-			draw_editbox ( &widget.row.editbox );
+			draw_editbox ( &ui.row.editbox );
 			color_set ( CPAIR_NORMAL, NULL );
 
 			/* Process keypress */
-			key = edit_setting ( &widget, getkey ( 0 ) );
+			key = edit_setting ( &ui, getkey ( 0 ) );
 			switch ( key ) {
 			case CR:
 			case LF:
-				if ( ( rc = save_setting ( &widget ) ) != 0 )
+				if ( ( rc = save_setting ( &ui ) ) != 0 )
 					alert ( " %s ", strerror ( rc ) );
 				/* Fall through */
 			case CTRL_C:
-				select_setting_row ( &widget, widget.current );
+				select_setting_row ( &ui, ui.scroll.current );
 				redraw = 1;
 				break;
 			default:
@@ -516,72 +495,52 @@ static int main_loop ( struct settings *settings ) {
 				break;
 			}
 
-		} else {
+			continue;
+		}
 
-			/* Process keypress */
-			key = getkey ( 0 );
-			move = 0;
-			switch ( key ) {
-			case KEY_UP:
-				move = -1;
-				break;
-			case KEY_DOWN:
-				move = +1;
-				break;
-			case KEY_PPAGE:
-				move = ( widget.first_visible -
-					 widget.current - 1 );
-				break;
-			case KEY_NPAGE:
-				move = ( widget.first_visible - widget.current
-					 + SETTINGS_LIST_ROWS );
-				break;
-			case KEY_HOME:
-				move = -widget.num_rows;
-				break;
-			case KEY_END:
-				move = +widget.num_rows;
-				break;
-			case CTRL_D:
-				if ( ! widget.row.setting.name )
-					break;
-				if ( ( rc = delete_setting ( widget.settings,
-						&widget.row.setting ) ) != 0 ) {
-					alert ( " %s ", strerror ( rc ) );
-				}
-				select_setting_row ( &widget, widget.current );
+		/* Otherwise, navigate through settings */
+		key = getkey ( 0 );
+		move = jump_scroll_key ( &ui.scroll, key );
+		if ( move ) {
+			previous = ui.scroll.current;
+			jump_scroll_move ( &ui.scroll, move );
+			if ( ui.scroll.current != previous ) {
+				draw_setting_row ( &ui );
 				redraw = 1;
+				if ( jump_scroll ( &ui.scroll ) )
+					draw_setting_rows ( &ui );
+				select_setting_row ( &ui, ui.scroll.current );
+			}
+			continue;
+		}
+
+		/* Handle non-navigation keys */
+		switch ( key ) {
+		case CTRL_D:
+			if ( ! ui.row.setting.name )
 				break;
-			case CTRL_X:
-				return 0;
-			case CR:
-			case LF:
-				if ( widget.row.settings ) {
-					init_widget ( &widget,
-						      widget.row.settings );
-					redraw = 1;
-				}
-				/* Fall through */
-			default:
-				if ( widget.row.setting.name ) {
-					edit_setting ( &widget, key );
-					redraw = 1;
-				}
-				break;
+			if ( ( rc = delete_setting ( ui.settings,
+						     &ui.row.setting ) ) != 0 ){
+				alert ( " %s ", strerror ( rc ) );
+			}
+			select_setting_row ( &ui, ui.scroll.current );
+			redraw = 1;
+			break;
+		case CTRL_X:
+			return 0;
+		case CR:
+		case LF:
+			if ( ui.row.settings ) {
+				select_settings ( &ui, ui.row.settings );
+				redraw = 1;
 			}
-			if ( move ) {
-				next = ( widget.current + move );
-				if ( ( int ) next < 0 )
-					next = 0;
-				if ( next >= widget.num_rows )
-					next = ( widget.num_rows - 1 );
-				if ( next != widget.current ) {
-					draw_setting_row ( &widget );
-					redraw = 1;
-					reveal_setting_row ( &widget, next );
-					select_setting_row ( &widget, next );
-				}
+			/* Fall through */
+		default:
+			if ( ui.row.setting.name ) {
+				edit_setting ( &ui, key );
+				redraw = 1;
 			}
+			break;
 		}
 	}
 }
diff --git a/qemu/roms/ipxe/src/image/elf.c b/qemu/roms/ipxe/src/image/elf.c
index 51636a8e9..5c2f9db25 100644
--- a/qemu/roms/ipxe/src/image/elf.c
+++ b/qemu/roms/ipxe/src/image/elf.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -36,27 +40,54 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/image.h>
 #include <ipxe/elf.h>
 
-typedef Elf32_Ehdr	Elf_Ehdr;
-typedef Elf32_Phdr	Elf_Phdr;
-typedef Elf32_Off	Elf_Off;
-#define ELFCLASS	ELFCLASS32
-
 /**
  * Load ELF segment into memory
  *
  * @v image		ELF file
  * @v phdr		ELF program header
+ * @v dest		Destination address
+ * @ret rc		Return status code
+ */
+static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
+			      physaddr_t dest ) {
+	userptr_t buffer = phys_to_user ( dest );
+	int rc;
+
+	DBGC ( image, "ELF %p loading segment [%x,%x) to [%lx,%lx,%lx)\n",
+	       image, phdr->p_offset, ( phdr->p_offset + phdr->p_filesz ),
+	       dest, ( dest + phdr->p_filesz ), ( dest + phdr->p_memsz ) );
+
+	/* Verify and prepare segment */
+	if ( ( rc = prep_segment ( buffer, phdr->p_filesz,
+				   phdr->p_memsz ) ) != 0 ) {
+		DBGC ( image, "ELF %p could not prepare segment: %s\n",
+		       image, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Copy image to segment */
+	memcpy_user ( buffer, 0, image->data, phdr->p_offset, phdr->p_filesz );
+
+	return 0;
+}
+
+/**
+ * Process ELF segment
+ *
+ * @v image		ELF file
  * @v ehdr		ELF executable header
+ * @v phdr		ELF program header
+ * @v process		Segment processor
  * @ret entry		Entry point, if found
  * @ret max		Maximum used address
  * @ret rc		Return status code
  */
-static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
-			      Elf_Ehdr *ehdr, physaddr_t *entry,
-			      physaddr_t *max ) {
+static int elf_segment ( struct image *image, Elf_Ehdr *ehdr, Elf_Phdr *phdr,
+			 int ( * process ) ( struct image *image,
+					     Elf_Phdr *phdr, physaddr_t dest ),
+			 physaddr_t *entry, physaddr_t *max ) {
 	physaddr_t dest;
 	physaddr_t end;
-	userptr_t buffer;
 	unsigned long e_offset;
 	int rc;
 
@@ -82,28 +113,15 @@ static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
 		       image );
 		return -ENOEXEC;
 	}
-	buffer = phys_to_user ( dest );
 	end = ( dest + phdr->p_memsz );
 
-	DBGC ( image, "ELF %p loading segment [%x,%x) to [%x,%x,%x)\n", image,
-	       phdr->p_offset, ( phdr->p_offset + phdr->p_filesz ),
-	       phdr->p_paddr, ( phdr->p_paddr + phdr->p_filesz ),
-	       ( phdr->p_paddr + phdr->p_memsz ) );
-
-	/* Verify and prepare segment */
-	if ( ( rc = prep_segment ( buffer, phdr->p_filesz,
-				   phdr->p_memsz ) ) != 0 ) {
-		DBGC ( image, "ELF %p could not prepare segment: %s\n",
-		       image, strerror ( rc ) );
-		return rc;
-	}
-
 	/* Update maximum used address, if applicable */
 	if ( end > *max )
 		*max = end;
 
-	/* Copy image to segment */
-	memcpy_user ( buffer, 0, image->data, phdr->p_offset, phdr->p_filesz );
+	/* Process segment */
+	if ( ( rc = process ( image, phdr, dest ) ) != 0 )
+		return rc;
 
 	/* Set execution address, if it lies within this segment */
 	if ( ( e_offset = ( ehdr->e_entry - dest ) ) < phdr->p_filesz ) {
@@ -124,62 +142,85 @@ static int elf_load_segment ( struct image *image, Elf_Phdr *phdr,
 }
 
 /**
- * Load ELF image into memory
+ * Process ELF segments
  *
  * @v image		ELF file
- * @ret entry		Entry point
+ * @v ehdr		ELF executable header
+ * @v process		Segment processor
+ * @ret entry		Entry point, if found
  * @ret max		Maximum used address
  * @ret rc		Return status code
  */
-int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max ) {
-	static const uint8_t e_ident[] = {
-		[EI_MAG0]	= ELFMAG0,
-		[EI_MAG1]	= ELFMAG1,
-		[EI_MAG2]	= ELFMAG2,
-		[EI_MAG3]	= ELFMAG3,
-		[EI_CLASS]	= ELFCLASS,
-	};
-	Elf_Ehdr ehdr;
+int elf_segments ( struct image *image, Elf_Ehdr *ehdr,
+		   int ( * process ) ( struct image *image, Elf_Phdr *phdr,
+				       physaddr_t dest ),
+		   physaddr_t *entry, physaddr_t *max ) {
 	Elf_Phdr phdr;
 	Elf_Off phoff;
 	unsigned int phnum;
 	int rc;
 
-	/* Read ELF header */
-	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
-	if ( memcmp ( &ehdr.e_ident[EI_MAG0], e_ident,
-		      sizeof ( e_ident ) ) != 0 ) {
-		DBGC ( image, "ELF %p has invalid signature\n", image );
-		return -ENOEXEC;
-	}
-
 	/* Initialise maximum used address */
 	*max = 0;
 
 	/* Invalidate entry point */
 	*entry = 0;
 
-	/* Read ELF program headers */
-	for ( phoff = ehdr.e_phoff , phnum = ehdr.e_phnum ; phnum ;
-	      phoff += ehdr.e_phentsize, phnum-- ) {
+	/* Read and process ELF program headers */
+	for ( phoff = ehdr->e_phoff , phnum = ehdr->e_phnum ; phnum ;
+	      phoff += ehdr->e_phentsize, phnum-- ) {
 		if ( phoff > image->len ) {
 			DBGC ( image, "ELF %p program header %d outside "
 			       "image\n", image, phnum );
 			return -ENOEXEC;
 		}
 		copy_from_user ( &phdr, image->data, phoff, sizeof ( phdr ) );
-		if ( ( rc = elf_load_segment ( image, &phdr, &ehdr,
-					       entry, max ) ) != 0 ) {
+		if ( ( rc = elf_segment ( image, ehdr, &phdr, process,
+					  entry, max ) ) != 0 )
 			return rc;
-		}
 	}
 
 	/* Check for a valid execution address */
 	if ( ! *entry ) {
 		DBGC ( image, "ELF %p entry point %lx outside image\n",
-		       image, ( ( unsigned long ) ehdr.e_entry ) );
+		       image, ( ( unsigned long ) ehdr->e_entry ) );
+		return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+/**
+ * Load ELF image into memory
+ *
+ * @v image		ELF file
+ * @ret entry		Entry point
+ * @ret max		Maximum used address
+ * @ret rc		Return status code
+ */
+int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max ) {
+	static const uint8_t e_ident[] = {
+		[EI_MAG0]	= ELFMAG0,
+		[EI_MAG1]	= ELFMAG1,
+		[EI_MAG2]	= ELFMAG2,
+		[EI_MAG3]	= ELFMAG3,
+		[EI_CLASS]	= ELFCLASS,
+	};
+	Elf_Ehdr ehdr;
+	int rc;
+
+	/* Read ELF header */
+	copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) );
+	if ( memcmp ( &ehdr.e_ident[EI_MAG0], e_ident,
+		      sizeof ( e_ident ) ) != 0 ) {
+		DBGC ( image, "ELF %p has invalid signature\n", image );
 		return -ENOEXEC;
 	}
 
+	/* Load ELF segments into memory */
+	if ( ( rc = elf_segments ( image, &ehdr, elf_load_segment,
+				   entry, max ) ) != 0 )
+		return rc;
+
 	return 0;
 }
diff --git a/qemu/roms/ipxe/src/image/embedded.c b/qemu/roms/ipxe/src/image/embedded.c
index 6358378fb..48dd86851 100644
--- a/qemu/roms/ipxe/src/image/embedded.c
+++ b/qemu/roms/ipxe/src/image/embedded.c
@@ -6,7 +6,7 @@
  * fetching over the network.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/image.h>
diff --git a/qemu/roms/ipxe/src/image/png.c b/qemu/roms/ipxe/src/image/png.c
index c14608553..5c4bcb3a0 100644
--- a/qemu/roms/ipxe/src/image/png.c
+++ b/qemu/roms/ipxe/src/image/png.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/image/pnm.c b/qemu/roms/ipxe/src/image/pnm.c
index af9e571a2..f24b28841 100644
--- a/qemu/roms/ipxe/src/image/pnm.c
+++ b/qemu/roms/ipxe/src/image/pnm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/image/script.c b/qemu/roms/ipxe/src/image/script.c
index 5328da8b4..28050868a 100644
--- a/qemu/roms/ipxe/src/image/script.c
+++ b/qemu/roms/ipxe/src/image/script.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/image/segment.c b/qemu/roms/ipxe/src/image/segment.c
index 86fe42662..2d0f2f0fc 100644
--- a/qemu/roms/ipxe/src/image/segment.c
+++ b/qemu/roms/ipxe/src/image/segment.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/include/.gitignore b/qemu/roms/ipxe/src/include/.gitignore
deleted file mode 100644
index de1598ef3..000000000
--- a/qemu/roms/ipxe/src/include/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.buildserial.h
diff --git a/qemu/roms/ipxe/src/include/assert.h b/qemu/roms/ipxe/src/include/assert.h
index a33f6017c..07f3ecb8c 100644
--- a/qemu/roms/ipxe/src/include/assert.h
+++ b/qemu/roms/ipxe/src/include/assert.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NDEBUG
 #define ASSERTING 0
diff --git a/qemu/roms/ipxe/src/include/big_bswap.h b/qemu/roms/ipxe/src/include/big_bswap.h
deleted file mode 100644
index 6c375a573..000000000
--- a/qemu/roms/ipxe/src/include/big_bswap.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef ETHERBOOT_BIG_BSWAP_H
-#define ETHERBOOT_BIG_BSWAP_H
-
-#define htonll(x)	(x)
-#define ntohll(x)	(x)
-#define ntohl(x) 	(x)
-#define htonl(x) 	(x)
-#define ntohs(x) 	(x)
-#define htons(x) 	(x)
-#define cpu_to_le64(x)	__bswap_64(x)
-#define cpu_to_le32(x)	__bswap_32(x)
-#define cpu_to_le16(x)	__bswap_16(x)
-#define cpu_to_be64(x)	(x)
-#define cpu_to_be32(x)	(x)
-#define cpu_to_be16(x)	(x)
-#define le64_to_cpu(x)	__bswap_64(x)
-#define le32_to_cpu(x)	__bswap_32(x)
-#define le16_to_cpu(x)	__bswap_16(x)
-#define be64_to_cpu(x)	(x)
-#define be32_to_cpu(x)	(x)
-#define be16_to_cpu(x)	(x)
-#define cpu_to_le64s(x) __bswap_64s(x)
-#define cpu_to_le32s(x) __bswap_32s(x)
-#define cpu_to_le16s(x) __bswap_16s(x)
-#define cpu_to_be64s(x) do {} while (0)
-#define cpu_to_be32s(x) do {} while (0)
-#define cpu_to_be16s(x) do {} while (0)
-#define le64_to_cpus(x) __bswap_64s(x)
-#define le32_to_cpus(x) __bswap_32s(x)
-#define le16_to_cpus(x) __bswap_16s(x)
-#define be64_to_cpus(x) do {} while (0)
-#define be32_to_cpus(x) do {} while (0)
-#define be16_to_cpus(x) do {} while (0)
-
-#endif /* ETHERBOOT_BIG_BSWAP_H */
diff --git a/qemu/roms/ipxe/src/include/byteswap.h b/qemu/roms/ipxe/src/include/byteswap.h
index 466759cf8..d1028c579 100644
--- a/qemu/roms/ipxe/src/include/byteswap.h
+++ b/qemu/roms/ipxe/src/include/byteswap.h
@@ -1,59 +1,138 @@
-#ifndef ETHERBOOT_BYTESWAP_H
-#define ETHERBOOT_BYTESWAP_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include "endian.h"
-#include "bits/byteswap.h"
-
-#define __bswap_constant_16(x) \
-	((uint16_t)((((uint16_t)(x) & 0x00ff) << 8) | \
-		    (((uint16_t)(x) & 0xff00) >> 8)))
-
-#define __bswap_constant_32(x) \
-	((uint32_t)((((uint32_t)(x) & 0x000000ffU) << 24) | \
-		    (((uint32_t)(x) & 0x0000ff00U) <<  8) | \
-		    (((uint32_t)(x) & 0x00ff0000U) >>  8) | \
-		    (((uint32_t)(x) & 0xff000000U) >> 24)))
-
-#define __bswap_constant_64(x) \
-	((uint64_t)((((uint64_t)(x) & 0x00000000000000ffULL) << 56) | \
-		    (((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \
-		    (((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \
-		    (((uint64_t)(x) & 0x00000000ff000000ULL) <<  8) | \
-		    (((uint64_t)(x) & 0x000000ff00000000ULL) >>  8) | \
-		    (((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \
-		    (((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \
-		    (((uint64_t)(x) & 0xff00000000000000ULL) >> 56)))
-
-#define __bswap_16(x) \
-	((uint16_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_16(x) : \
-	__bswap_variable_16(x)))
-
-#define __bswap_32(x) \
-	((uint32_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_32(x) : \
-	__bswap_variable_32(x)))
-
-#define __bswap_64(x) \
-	((uint64_t)(__builtin_constant_p(x) ? \
-	__bswap_constant_64(x) : \
-	__bswap_variable_64(x)))
+#ifndef BYTESWAP_H
+#define BYTESWAP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <endian.h>
+#include <bits/byteswap.h>
+
+/**
+ * Byte-swap a 16-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_16( value )					\
+	( ( ( (value) & 0x00ff ) << 8 ) |				\
+	  ( ( (value) & 0xff00 ) >> 8 ) )
+
+/**
+ * Byte-swap a 32-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_32( value ) \
+	( ( ( (value) & 0x000000ffUL ) << 24 ) |			\
+	  ( ( (value) & 0x0000ff00UL ) <<  8 ) |			\
+	  ( ( (value) & 0x00ff0000UL ) >>  8 ) |			\
+	  ( ( (value) & 0xff000000UL ) >> 24 ) )
+
+/**
+ * Byte-swap a 64-bit constant
+ *
+ * @v value		Constant value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_constant_64( value )					\
+	( ( ( (value) & 0x00000000000000ffULL ) << 56 ) |		\
+	  ( ( (value) & 0x000000000000ff00ULL ) << 40 ) |		\
+	  ( ( (value) & 0x0000000000ff0000ULL ) << 24 ) |		\
+	  ( ( (value) & 0x00000000ff000000ULL ) <<  8 ) |		\
+	  ( ( (value) & 0x000000ff00000000ULL ) >>  8 ) |		\
+	  ( ( (value) & 0x0000ff0000000000ULL ) >> 24 ) |		\
+	  ( ( (value) & 0x00ff000000000000ULL ) >> 40 ) |		\
+	  ( ( (value) & 0xff00000000000000ULL ) >> 56 ) )
+
+/**
+ * Byte-swap a 16-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_16( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint16_t ) __bswap_constant_16 ( ( uint16_t ) (value) ) ) \
+	  : __bswap_variable_16 (value) )
+#define bswap_16( value ) __bswap_16 (value)
+
+/**
+ * Byte-swap a 32-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_32( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint32_t ) __bswap_constant_32 ( ( uint32_t ) (value) ) ) \
+	  : __bswap_variable_32 (value) )
+#define bswap_32( value ) __bswap_32 (value)
+
+/**
+ * Byte-swap a 64-bit value
+ *
+ * @v value		Value
+ * @ret swapped		Byte-swapped value
+ */
+#define __bswap_64( value )						\
+	( __builtin_constant_p (value) ?				\
+	  ( ( uint64_t ) __bswap_constant_64 ( ( uint64_t ) (value) ) ) \
+          : __bswap_variable_64 (value) )
+#define bswap_64( value ) __bswap_64 (value)
 
 #if __BYTE_ORDER == __LITTLE_ENDIAN
-#include "little_bswap.h"
+#define __cpu_to_leNN( bits, value ) (value)
+#define __cpu_to_beNN( bits, value ) __bswap_ ## bits (value)
+#define __leNN_to_cpu( bits, value ) (value)
+#define __beNN_to_cpu( bits, value ) __bswap_ ## bits (value)
+#define __cpu_to_leNNs( bits, ptr ) do { } while ( 0 )
+#define __cpu_to_beNNs( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __leNN_to_cpus( bits, ptr ) do { } while ( 0 )
+#define __beNN_to_cpus( bits, ptr ) __bswap_ ## bits ## s (ptr)
 #endif
+
 #if __BYTE_ORDER == __BIG_ENDIAN
-#include "big_bswap.h"
+#define __cpu_to_leNN( bits, value ) __bswap_ ## bits (value)
+#define __cpu_to_beNN( bits, value ) (value)
+#define __leNN_to_cpu( bits, value ) __bswap_ ## bits (value)
+#define __beNN_to_cpu( bits, value ) (value)
+#define __cpu_to_leNNs( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __cpu_to_beNNs( bits, ptr ) do { } while ( 0 )
+#define __leNN_to_cpus( bits, ptr ) __bswap_ ## bits ## s (ptr)
+#define __beNN_to_cpus( bits, ptr ) do { } while ( 0 )
 #endif
 
-/* Make routines available to all */
-#define swap64(x)	__bswap_64(x)
-#define swap32(x)	__bswap_32(x)
-#define swap16(x)	__bswap_16(x)
-#define bswap_64(x)	__bswap_64(x)
-#define bswap_32(x)	__bswap_32(x)
-#define bswap_16(x)	__bswap_16(x)
-	
-#endif /* ETHERBOOT_BYTESWAP_H */
+#define cpu_to_le16( value ) __cpu_to_leNN ( 16, value )
+#define cpu_to_le32( value ) __cpu_to_leNN ( 32, value )
+#define cpu_to_le64( value ) __cpu_to_leNN ( 64, value )
+#define cpu_to_be16( value ) __cpu_to_beNN ( 16, value )
+#define cpu_to_be32( value ) __cpu_to_beNN ( 32, value )
+#define cpu_to_be64( value ) __cpu_to_beNN ( 64, value )
+#define le16_to_cpu( value ) __leNN_to_cpu ( 16, value )
+#define le32_to_cpu( value ) __leNN_to_cpu ( 32, value )
+#define le64_to_cpu( value ) __leNN_to_cpu ( 64, value )
+#define be16_to_cpu( value ) __beNN_to_cpu ( 16, value )
+#define be32_to_cpu( value ) __beNN_to_cpu ( 32, value )
+#define be64_to_cpu( value ) __beNN_to_cpu ( 64, value )
+#define cpu_to_le16s( ptr ) __cpu_to_leNNs ( 16, ptr )
+#define cpu_to_le32s( ptr ) __cpu_to_leNNs ( 32, ptr )
+#define cpu_to_le64s( ptr ) __cpu_to_leNNs ( 64, ptr )
+#define cpu_to_be16s( ptr ) __cpu_to_beNNs ( 16, ptr )
+#define cpu_to_be32s( ptr ) __cpu_to_beNNs ( 32, ptr )
+#define cpu_to_be64s( ptr ) __cpu_to_beNNs ( 64, ptr )
+#define le16_to_cpus( ptr ) __leNN_to_cpus ( 16, ptr )
+#define le32_to_cpus( ptr ) __leNN_to_cpus ( 32, ptr )
+#define le64_to_cpus( ptr ) __leNN_to_cpus ( 64, ptr )
+#define be16_to_cpus( ptr ) __beNN_to_cpus ( 16, ptr )
+#define be32_to_cpus( ptr ) __beNN_to_cpus ( 32, ptr )
+#define be64_to_cpus( ptr ) __beNN_to_cpus ( 64, ptr )
+
+#define htonll( value ) cpu_to_be64 (value)
+#define ntohll( value ) be64_to_cpu (value)
+#define htonl( value ) cpu_to_be32 (value)
+#define ntohl( value ) be32_to_cpu (value)
+#define htons( value ) cpu_to_be16 (value)
+#define ntohs( value ) be16_to_cpu (value)
+
+#endif /* BYTESWAP_H */
diff --git a/qemu/roms/ipxe/src/include/compiler.h b/qemu/roms/ipxe/src/include/compiler.h
index 3f5c913a0..ca82f9523 100644
--- a/qemu/roms/ipxe/src/include/compiler.h
+++ b/qemu/roms/ipxe/src/include/compiler.h
@@ -57,101 +57,100 @@
  * @{
  */
 
-/** Provide a symbol within this object file */
+/**
+ * Provide a symbol within this object file
+ *
+ * @v symbol		Symbol name
+ */
 #ifdef ASSEMBLY
-#define PROVIDE_SYMBOL( _sym )				\
-	.section ".provided", "a", @nobits ;		\
-	.hidden _sym ;					\
-	.globl	_sym ;					\
-	_sym: ;						\
+#define PROVIDE_SYMBOL( symbol )				\
+	.section ".provided", "a", @nobits ;			\
+	.hidden symbol ;					\
+	.globl	symbol ;					\
+	symbol: ;						\
 	.previous
-#else /* ASSEMBLY */
-#define PROVIDE_SYMBOL( _sym )				\
-	char _sym[0]					\
+#else
+#define PROVIDE_SYMBOL( symbol )				\
+	char symbol[0]						\
 	  __attribute__ (( section ( ".provided" ) ))
-#endif /* ASSEMBLY */
+#endif
 
-/** Require a symbol within this object file
+/**
+ * Request a symbol
+ *
+ * @v symbol		Symbol name
  *
- * The symbol is referenced by a relocation in a discarded section, so
- * if it is not available at link time the link will fail.
+ * Request a symbol to be included within the link.  If the symbol
+ * cannot be found, the link will succeed anyway.
  */
 #ifdef ASSEMBLY
-#define REQUIRE_SYMBOL( _sym )				\
-	.section ".discard", "a", @progbits ;		\
-	.extern	_sym ;					\
-	.long	_sym ;					\
-	.previous
-#else /* ASSEMBLY */
-#define REQUIRE_SYMBOL( _sym )				\
-	extern char _sym;				\
-	static char * _C2 ( _C2 ( __require_, _sym ), _C2 ( _, __LINE__ ) ) \
-		__attribute__ (( section ( ".discard" ), used )) \
-		= &_sym
+#define REQUEST_SYMBOL( symbol )				\
+	.equ __request_ ## symbol, symbol
+#else
+#define REQUEST_SYMBOL( symbol )				\
+	__asm__ ( ".equ __request_" #symbol ", " #symbol )
 #endif
 
-/** Request that a symbol be available at runtime
+/**
+ * Require a symbol
+ *
+ * @v symbol		Symbol name
  *
- * The requested symbol is entered as undefined into the symbol table
- * for this object, so the linker will pull in other object files as
- * necessary to satisfy the reference. However, the undefined symbol
- * is not referenced in any relocations, so the link can still succeed
- * if no file contains it.
+ * Require a symbol to be included within the link.  If the symbol
+ * cannot be found, the link will fail.
  *
- * A symbol passed to this macro may not be referenced anywhere
- * else in the file. If you want to do that, see IMPORT_SYMBOL().
+ * To use this macro within a file, you must also specify the file's
+ * "requiring symbol" using the REQUIRING_SYMBOL() or
+ * PROVIDE_REQUIRING_SYMBOL() macros.
  */
 #ifdef ASSEMBLY
-#define REQUEST_SYMBOL( _sym )				\
-	.equ	__need_ ## _sym, _sym
-#else /* ASSEMBLY */
-#define REQUEST_SYMBOL( _sym )				\
-	__asm__ ( ".equ\t__need_" #_sym ", " #_sym )
-#endif /* ASSEMBLY */
+#define REQUIRE_SYMBOL( symbol )				\
+	.reloc __requiring_symbol__, RELOC_TYPE_NONE, symbol
+#else
+#define REQUIRE_SYMBOL( symbol )				\
+	__asm__ ( ".reloc __requiring_symbol__, "		\
+		  _S2 ( RELOC_TYPE_NONE ) ", " #symbol )
+#endif
 
-/** Set up a symbol to be usable in another file by IMPORT_SYMBOL()
+/**
+ * Specify the file's requiring symbol
+ *
+ * @v symbol		Symbol name
  *
- * The symbol must already be marked as global.
+ * REQUIRE_SYMBOL() works by defining a dummy relocation record
+ * against a nominated "requiring symbol".  The presence of the
+ * nominated requiring symbol will drag in all of the symbols
+ * specified using REQUIRE_SYMBOL().
  */
-#define EXPORT_SYMBOL( _sym )	PROVIDE_SYMBOL ( __export_ ## _sym )
+#ifdef ASSEMBLY
+#define REQUIRING_SYMBOL( symbol )				\
+	.equ __requiring_symbol__, symbol
+#else
+#define REQUIRING_SYMBOL( symbol )				\
+	__asm__ ( ".equ __requiring_symbol__, " #symbol )
+#endif
 
-/** Make a symbol usable to this file if available at link time
- *
- * If no file passed to the linker contains the symbol, it will have
- * @c NULL value to future uses. Keep in mind that the symbol value is
- * really the @e address of a variable or function; see the code
- * snippet below.
- *
- * In C using IMPORT_SYMBOL, you must specify the declaration as the
- * second argument, for instance
- *
- * @code
- *   IMPORT_SYMBOL ( my_func, int my_func ( int arg ) );
- *   IMPORT_SYMBOL ( my_var, int my_var );
- *
- *   void use_imports ( void ) {
- * 	if ( my_func && &my_var )
- * 	   my_var = my_func ( my_var );
- *   }
- * @endcode
- *
- * GCC considers a weak declaration to override a strong one no matter
- * which comes first, so it is safe to include a header file declaring
- * the imported symbol normally, but providing the declaration to
- * IMPORT_SYMBOL is still required.
+/**
+ * Provide a file's requiring symbol
  *
- * If no EXPORT_SYMBOL declaration exists for the imported symbol in
- * another file, the behavior will be most likely be identical to that
- * for an unavailable symbol.
+ * If the file contains no symbols that can be used as the requiring
+ * symbol, you can provide a dummy one-byte-long symbol using
+ * PROVIDE_REQUIRING_SYMBOL().
  */
 #ifdef ASSEMBLY
-#define IMPORT_SYMBOL( _sym )				\
-	REQUEST_SYMBOL ( __export_ ## _sym ) ;		\
-	.weak	_sym
-#else /* ASSEMBLY */
-#define IMPORT_SYMBOL( _sym, _decl )			\
-	REQUEST_SYMBOL ( __export_ ## _sym ) ;		\
-	extern _decl __attribute__ (( weak ))
+#define PROVIDE_REQUIRING_SYMBOL()				\
+	.section ".tbl.requiring_symbols", "a", @progbits ;	\
+	__requiring_symbol__:	.byte 0 ;			\
+	.size __requiring_symbol__, . - __requiring_symbol__ ;	\
+	.previous
+#else
+#define PROVIDE_REQUIRING_SYMBOL()				\
+	__asm__ ( ".section \".tbl.requiring_symbols\", "	\
+		  "         \"a\", @progbits\n"			\
+		  "__requiring_symbol__:\t.byte 0\n"		\
+		  ".size __requiring_symbol__, "		\
+		  "      . - __requiring_symbol__\n"		\
+		  ".previous" )
 #endif
 
 /** @} */
@@ -163,20 +162,33 @@
 
 #define PREFIX_OBJECT( _prefix ) _C2 ( _prefix, OBJECT )
 #define OBJECT_SYMBOL PREFIX_OBJECT ( obj_ )
-#define REQUEST_EXPANDED( _sym ) REQUEST_SYMBOL ( _sym )
-#define CONFIG_SYMBOL PREFIX_OBJECT ( obj_config_ )
 
 /** Always provide the symbol for the current object (defined by -DOBJECT) */
 PROVIDE_SYMBOL ( OBJECT_SYMBOL );
 
-/** Pull in an object-specific configuration file if available */
-REQUEST_EXPANDED ( CONFIG_SYMBOL );
-
-/** Explicitly require another object */
-#define REQUIRE_OBJECT( _obj ) REQUIRE_SYMBOL ( obj_ ## _obj )
+/**
+ * Request an object
+ *
+ * @v object		Object name
+ *
+ * Request an object to be included within the link.  If the object
+ * cannot be found, the link will succeed anyway.
+ */
+#define REQUEST_OBJECT( object ) REQUEST_SYMBOL ( obj_ ## object )
 
-/** Pull in another object if it exists */
-#define REQUEST_OBJECT( _obj ) REQUEST_SYMBOL ( obj_ ## _obj )
+/**
+ * Require an object
+ *
+ * @v object		Object name
+ *
+ * Require an object to be included within the link.  If the object
+ * cannot be found, the link will fail.
+ *
+ * To use this macro within a file, you must also specify the file's
+ * "requiring symbol" using the REQUIRING_SYMBOL() or
+ * PROVIDE_REQUIRING_SYMBOL() macros.
+ */
+#define REQUIRE_OBJECT( object ) REQUIRE_SYMBOL ( obj_ ## object )
 
 /** @} */
 
@@ -195,14 +207,6 @@ REQUEST_EXPANDED ( CONFIG_SYMBOL );
  */
 #define __weak		__attribute__ (( weak, noinline ))
 
-/** Prevent a function from being optimized away without inlining
- *
- * Calls to functions with void return type that contain no code in their body
- * may be removed by gcc's optimizer even when inlining is inhibited. Placing
- * this macro in the body of the function prevents that from occurring.
- */
-#define __keepme	asm("");
-
 #endif
 
 /** @defgroup dbg Debugging infrastructure
@@ -730,13 +734,24 @@ int __debug_disable;
 #define FILE_LICENCE_MIT \
 	PROVIDE_SYMBOL ( PREFIX_OBJECT ( __licence__mit__ ) )
 
+/** Declare a file as being under GPLv2+ or UBDL
+ *
+ * This licence declaration is applicable when a file states itself to
+ * be licensed under the GNU GPL; "either version 2 of the License, or
+ * (at your option) any later version" and also states that it may be
+ * distributed under the terms of the Unmodified Binary Distribution
+ * Licence (as given in the file COPYING.UBDL).
+ */
+#define FILE_LICENCE_GPL2_OR_LATER_OR_UBDL \
+	PROVIDE_SYMBOL ( PREFIX_OBJECT ( __licence__gpl2_or_later_or_ubdl__ ) )
+
 /** Declare a particular licence as applying to a file */
 #define FILE_LICENCE( _licence ) FILE_LICENCE_ ## _licence
 
 /** @} */
 
-/* This file itself is under GPLv2-or-later */
-FILE_LICENCE ( GPL2_OR_LATER );
+/* This file itself is under GPLv2+/UBDL */
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/compiler.h>
 
diff --git a/qemu/roms/ipxe/src/include/ctype.h b/qemu/roms/ipxe/src/include/ctype.h
index e92ecb1c0..0d79ecd19 100644
--- a/qemu/roms/ipxe/src/include/ctype.h
+++ b/qemu/roms/ipxe/src/include/ctype.h
@@ -4,30 +4,114 @@
 /** @file
  *
  * Character types
+ *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#define isdigit(c)	((c) >= '0' && (c) <= '9')
-#define islower(c)	((c) >= 'a' && (c) <= 'z')
-#define isupper(c)	((c) >= 'A' && (c) <= 'Z')
-#define isxdigit(c)	(isdigit(c) || ((c) >= 'A' && (c) <= 'F') || ((c) >= 'a' && (c) <= 'f'))
-#define isprint(c)	((c) >= ' ' && (c) <= '~' )
+/**
+ * Check if character is a decimal digit
+ *
+ * @v character		ASCII character
+ * @ret is_digit	Character is a decimal digit
+ */
+static inline int isdigit ( int character ) {
 
-static inline unsigned char tolower(unsigned char c)
-{
-	if (isupper(c))
-		c -= 'A'-'a';
-	return c;
+	return ( ( character >= '0' ) && ( character <= '9' ) );
 }
 
-static inline unsigned char toupper(unsigned char c)
-{
-	if (islower(c))
-		c -= 'a'-'A';
-	return c;
+/**
+ * Check if character is a hexadecimal digit
+ *
+ * @v character		ASCII character
+ * @ret is_xdigit	Character is a hexadecimal digit
+ */
+static inline int isxdigit ( int character ) {
+
+	return ( ( ( character >= '0' ) && ( character <= '9' ) ) ||
+		 ( ( character >= 'A' ) && ( character <= 'F' ) ) ||
+		 ( ( character >= 'a' ) && ( character <= 'f' ) ) );
+}
+
+/**
+ * Check if character is an upper-case letter
+ *
+ * @v character		ASCII character
+ * @ret is_upper	Character is an upper-case letter
+ */
+static inline int isupper ( int character ) {
+
+	return ( ( character >= 'A' ) && ( character <= 'Z' ) );
+}
+
+/**
+ * Check if character is a lower-case letter
+ *
+ * @v character		ASCII character
+ * @ret is_lower	Character is a lower-case letter
+ */
+static inline int islower ( int character ) {
+
+	return ( ( character >= 'a' ) && ( character <= 'z' ) );
+}
+
+/**
+ * Check if character is alphabetic
+ *
+ * @v character		ASCII character
+ * @ret is_alpha	Character is alphabetic
+ */
+static inline int isalpha ( int character ) {
+
+	return ( isupper ( character ) || islower ( character ) );
+}
+
+/**
+ * Check if character is alphanumeric
+ *
+ * @v character		ASCII character
+ * @ret is_alnum	Character is alphanumeric
+ */
+static inline int isalnum ( int character ) {
+
+	return ( isalpha ( character ) || isdigit ( character ) );
+}
+
+/**
+ * Check if character is printable
+ *
+ * @v character		ASCII character
+ * @ret is_print	Character is printable
+ */
+static inline int isprint ( int character ) {
+
+	return ( ( character >= ' ' ) && ( character <= '~' ) );
+}
+
+/**
+ * Convert character to lower case
+ *
+ * @v character		Character
+ * @v character		Lower-case character
+ */
+static inline int tolower ( int character ) {
+
+	return ( isupper ( character ) ?
+		 ( character - 'A' + 'a' ) : character );
+}
+
+/**
+ * Convert character to upper case
+ *
+ * @v character		Character
+ * @v character		Upper-case character
+ */
+static inline int toupper ( int character ) {
+
+	return ( islower ( character ) ?
+		 ( character - 'a' + 'A' ) : character );
 }
 
-extern int isspace ( int c );
+extern int isspace ( int character );
 
 #endif /* _CTYPE_H */
diff --git a/qemu/roms/ipxe/src/include/curses.h b/qemu/roms/ipxe/src/include/curses.h
index f16f9d7d0..04060fe27 100644
--- a/qemu/roms/ipxe/src/include/curses.h
+++ b/qemu/roms/ipxe/src/include/curses.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #undef  ERR
 #define ERR	(-1)
diff --git a/qemu/roms/ipxe/src/include/elf.h b/qemu/roms/ipxe/src/include/elf.h
index 04022b687..18f755a21 100644
--- a/qemu/roms/ipxe/src/include/elf.h
+++ b/qemu/roms/ipxe/src/include/elf.h
@@ -1,234 +1,81 @@
 #ifndef ELF_H
 #define ELF_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define EI_NIDENT	16	/* Size of e_ident array. */
-
-/* Values for e_type. */
-#define ET_NONE		0	/* No file type */
-#define ET_REL		1	/* Relocatable file */
-#define ET_EXEC		2	/* Executable file */
-#define ET_DYN		3	/* Shared object file */
-#define ET_CORE		4	/* Core file */
-
-/* Values for e_machine (architecute). */
-#define EM_NONE		 0		/* No machine */
-#define EM_M32		 1		/* AT&T WE 32100 */
-#define EM_SPARC	 2		/* SUN SPARC */
-#define EM_386		 3		/* Intel 80386+ */
-#define EM_68K		 4		/* Motorola m68k family */
-#define EM_88K		 5		/* Motorola m88k family */
-#define EM_486		 6		/* Perhaps disused */
-#define EM_860		 7		/* Intel 80860 */
-#define EM_MIPS		 8		/* MIPS R3000 big-endian */
-#define EM_S370		 9		/* IBM System/370 */
-#define EM_MIPS_RS3_LE	10		/* MIPS R3000 little-endian */
-
-#define EM_PARISC	15		/* HPPA */
-#define EM_VPP500	17		/* Fujitsu VPP500 */
-#define EM_SPARC32PLUS	18		/* Sun's "v8plus" */
-#define EM_960		19		/* Intel 80960 */
-#define EM_PPC		20		/* PowerPC */
-#define EM_PPC64	21		/* PowerPC 64-bit */
-#define EM_S390		22		/* IBM S390 */
-
-#define EM_V800		36		/* NEC V800 series */
-#define EM_FR20		37		/* Fujitsu FR20 */
-#define EM_RH32		38		/* TRW RH-32 */
-#define EM_RCE		39		/* Motorola RCE */
-#define EM_ARM		40		/* ARM */
-#define EM_FAKE_ALPHA	41		/* Digital Alpha */
-#define EM_SH		42		/* Hitachi SH */
-#define EM_SPARCV9	43		/* SPARC v9 64-bit */
-#define EM_TRICORE	44		/* Siemens Tricore */
-#define EM_ARC		45		/* Argonaut RISC Core */
-#define EM_H8_300	46		/* Hitachi H8/300 */
-#define EM_H8_300H	47		/* Hitachi H8/300H */
-#define EM_H8S		48		/* Hitachi H8S */
-#define EM_H8_500	49		/* Hitachi H8/500 */
-#define EM_IA_64	50		/* Intel Merced */
-#define EM_MIPS_X	51		/* Stanford MIPS-X */
-#define EM_COLDFIRE	52		/* Motorola Coldfire */
-#define EM_68HC12	53		/* Motorola M68HC12 */
-#define EM_MMA		54		/* Fujitsu MMA Multimedia Accelerator*/
-#define EM_PCP		55		/* Siemens PCP */
-#define EM_NCPU		56		/* Sony nCPU embeeded RISC */
-#define EM_NDR1		57		/* Denso NDR1 microprocessor */
-#define EM_STARCORE	58		/* Motorola Start*Core processor */
-#define EM_ME16		59		/* Toyota ME16 processor */
-#define EM_ST100	60		/* STMicroelectronic ST100 processor */
-#define EM_TINYJ	61		/* Advanced Logic Corp. Tinyj emb.fam*/
-#define EM_X86_64	62		/* AMD x86-64 architecture */
-#define EM_PDSP		63		/* Sony DSP Processor */
-
-#define EM_FX66		66		/* Siemens FX66 microcontroller */
-#define EM_ST9PLUS	67		/* STMicroelectronics ST9+ 8/16 mc */
-#define EM_ST7		68		/* STmicroelectronics ST7 8 bit mc */
-#define EM_68HC16	69		/* Motorola MC68HC16 microcontroller */
-#define EM_68HC11	70		/* Motorola MC68HC11 microcontroller */
-#define EM_68HC08	71		/* Motorola MC68HC08 microcontroller */
-#define EM_68HC05	72		/* Motorola MC68HC05 microcontroller */
-#define EM_SVX		73		/* Silicon Graphics SVx */
-#define EM_AT19		74		/* STMicroelectronics ST19 8 bit mc */
-#define EM_VAX		75		/* Digital VAX */
-#define EM_CRIS		76		/* Axis Communications 32-bit embedded processor */
-#define EM_JAVELIN	77		/* Infineon Technologies 32-bit embedded processor */
-#define EM_FIREPATH	78		/* Element 14 64-bit DSP Processor */
-#define EM_ZSP		79		/* LSI Logic 16-bit DSP Processor */
-#define EM_MMIX		80		/* Donald Knuth's educational 64-bit processor */
-#define EM_HUANY	81		/* Harvard University machine-independent object files */
-#define EM_PRISM	82		/* SiTera Prism */
-#define EM_AVR		83		/* Atmel AVR 8-bit microcontroller */
-#define EM_FR30		84		/* Fujitsu FR30 */
-#define EM_D10V		85		/* Mitsubishi D10V */
-#define EM_D30V		86		/* Mitsubishi D30V */
-#define EM_V850		87		/* NEC v850 */
-#define EM_M32R		88		/* Mitsubishi M32R */
-#define EM_MN10300	89		/* Matsushita MN10300 */
-#define EM_MN10200	90		/* Matsushita MN10200 */
-#define EM_PJ		91		/* picoJava */
-#define EM_OPENRISC	92		/* OpenRISC 32-bit embedded processor */
-#define EM_ARC_A5	93		/* ARC Cores Tangent-A5 */
-#define EM_XTENSA	94		/* Tensilica Xtensa Architecture */
-#define EM_NUM		95
-
-/* Values for p_type. */
-#define PT_NULL		0	/* Unused entry. */
-#define PT_LOAD		1	/* Loadable segment. */
-#define PT_DYNAMIC	2	/* Dynamic linking information segment. */
-#define PT_INTERP	3	/* Pathname of interpreter. */
-#define PT_NOTE		4	/* Auxiliary information. */
-#define PT_SHLIB	5	/* Reserved (not used). */
-#define PT_PHDR		6	/* Location of program header itself. */
-
-/* Values for p_flags. */
-#define PF_X		0x1	/* Executable. */
-#define PF_W		0x2	/* Writable. */
-#define PF_R		0x4	/* Readable. */
-
-
-#define	ELF_PROGRAM_RETURNS_BIT	0x8000000	/* e_flags bit 31 */
-
-#define EI_MAG0		0
-#define ELFMAG0		0x7f
-
-#define EI_MAG1		1
-#define ELFMAG1		'E'
-
-#define EI_MAG2		2
-#define ELFMAG2		'L'
-
-#define EI_MAG3		3
-#define ELFMAG3		'F'
-
-#define ELFMAG		"\177ELF"
-#define	SELFMAG		4
-
-#define EI_CLASS	4	/* File class byte index */
-#define ELFCLASSNONE	0	/* Invalid class */
-#define ELFCLASS32	1	/* 32-bit objects */
-#define ELFCLASS64	2	/* 64-bit objects */
+/**
+ * @file
+ *
+ * ELF headers
+ *
+ */
 
-#define EI_DATA		5	/* Data encodeing byte index */
-#define ELFDATANONE	0	/* Invalid data encoding */
-#define ELFDATA2LSB	1	/* 2's complement little endian */
-#define ELFDATA2MSB	2	/* 2's complement big endian */
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#define EI_VERSION	6	/* File version byte index */
-				/* Value must be EV_CURRENT */
+#include <stdint.h>
 
-#define EV_NONE		0	/* Invalid ELF Version */
-#define EV_CURRENT	1	/* Current version */
+typedef uint32_t Elf32_Addr;
+typedef uint16_t Elf32_Half;
+typedef uint32_t Elf32_Off;
+typedef int32_t Elf32_Sword;
+typedef uint32_t Elf32_Word;
 
-#define ELF32_PHDR_SIZE (8*4)	/* Size of an elf program header */
+/** Length of ELF identifier */
+#define EI_NIDENT 16
 
-#ifndef ASSEMBLY
+/** ELF header */
+typedef struct {
+	unsigned char e_ident[EI_NIDENT];
+	Elf32_Half e_type;
+	Elf32_Half e_machine;
+	Elf32_Word e_version;
+	Elf32_Addr e_entry;
+	Elf32_Off e_phoff;
+	Elf32_Off e_shoff;
+	Elf32_Word e_flags;
+	Elf32_Half e_ehsize;
+	Elf32_Half e_phentsize;
+	Elf32_Half e_phnum;
+	Elf32_Half e_shentsize;
+	Elf32_Half e_shnum;
+	Elf32_Half e_shstrndx;
+} Elf32_Ehdr;
 
-#include <stdint.h>
+/* ELF identifier indexes */
+#define EI_MAG0 0
+#define EI_MAG1 1
+#define EI_MAG2 2
+#define EI_MAG3 3
+#define EI_CLASS 4
+#define EI_DATA 5
+#define EI_VERSION 6
 
-/*
- * ELF definitions common to all 32-bit architectures.
- */
+/* ELF magic signature bytes */
+#define ELFMAG0 0x7f
+#define ELFMAG1 'E'
+#define ELFMAG2 'L'
+#define ELFMAG3 'F'
 
-typedef uint32_t	Elf32_Addr;
-typedef uint16_t	Elf32_Half;
-typedef uint32_t	Elf32_Off;
-typedef int32_t		Elf32_Sword;
-typedef uint32_t	Elf32_Word;
-typedef uint32_t	Elf32_Size;
+/* ELF classes */
+#define ELFCLASS32 1
 
-typedef uint64_t	Elf64_Addr;
-typedef uint16_t	Elf64_Half;
-typedef uint64_t	Elf64_Off;
-typedef int32_t		Elf64_Sword;
-typedef uint32_t	Elf64_Word;
-typedef uint64_t	Elf64_Size;
+/* ELF data encodings */
+#define ELFDATA2LSB 1
 
-/*
- * ELF header.
- */
-typedef struct {
-	unsigned char	e_ident[EI_NIDENT];	/* File identification. */
-	Elf32_Half	e_type;		/* File type. */
-	Elf32_Half	e_machine;	/* Machine architecture. */
-	Elf32_Word	e_version;	/* ELF format version. */
-	Elf32_Addr	e_entry;	/* Entry point. */
-	Elf32_Off	e_phoff;	/* Program header file offset. */
-	Elf32_Off	e_shoff;	/* Section header file offset. */
-	Elf32_Word	e_flags;	/* Architecture-specific flags. */
-	Elf32_Half	e_ehsize;	/* Size of ELF header in bytes. */
-	Elf32_Half	e_phentsize;	/* Size of program header entry. */
-	Elf32_Half	e_phnum;	/* Number of program header entries. */
-	Elf32_Half	e_shentsize;	/* Size of section header entry. */
-	Elf32_Half	e_shnum;	/* Number of section header entries. */
-	Elf32_Half	e_shstrndx;	/* Section name strings section. */
-} Elf32_Ehdr;
-
-typedef struct {
-	unsigned char	e_ident[EI_NIDENT];	/* File identification. */
-	Elf64_Half	e_type;		/* File type. */
-	Elf64_Half	e_machine;	/* Machine architecture. */
-	Elf64_Word	e_version;	/* ELF format version. */
-	Elf64_Addr	e_entry;	/* Entry point. */
-	Elf64_Off	e_phoff;	/* Program header file offset. */
-	Elf64_Off	e_shoff;	/* Section header file offset. */
-	Elf64_Word	e_flags;	/* Architecture-specific flags. */
-	Elf64_Half	e_ehsize;	/* Size of ELF header in bytes. */
-	Elf64_Half	e_phentsize;	/* Size of program header entry. */
-	Elf64_Half	e_phnum;	/* Number of program header entries. */
-	Elf64_Half	e_shentsize;	/* Size of section header entry. */
-	Elf64_Half	e_shnum;	/* Number of section header entries. */
-	Elf64_Half	e_shstrndx;	/* Section name strings section. */
-} Elf64_Ehdr;
+/* ELF versions */
+#define EV_CURRENT 1
 
-/*
- * Program header.
- */
+/** ELF program header */
 typedef struct {
-	Elf32_Word	p_type;		/* Entry type. */
-	Elf32_Off	p_offset;	/* File offset of contents. */
-	Elf32_Addr	p_vaddr;	/* Virtual address (not used). */
-	Elf32_Addr	p_paddr;	/* Physical address. */
-	Elf32_Size	p_filesz;	/* Size of contents in file. */
-	Elf32_Size	p_memsz;	/* Size of contents in memory. */
-	Elf32_Word	p_flags;	/* Access permission flags. */
-	Elf32_Size	p_align;	/* Alignment in memory and file. */
+	Elf32_Word p_type;
+	Elf32_Off p_offset;
+	Elf32_Addr p_vaddr;
+	Elf32_Addr p_paddr;
+	Elf32_Word p_filesz;
+	Elf32_Word p_memsz;
+	Elf32_Word p_flags;
+	Elf32_Word p_align;
 } Elf32_Phdr;
 
-typedef struct {
-	Elf64_Word	p_type;		/* Entry type. */
-	Elf64_Word	p_flags;	/* Access permission flags. */
-	Elf64_Off	p_offset;	/* File offset of contents. */
-	Elf64_Addr	p_vaddr;	/* Virtual address (not used). */
-	Elf64_Addr	p_paddr;	/* Physical address. */
-	Elf64_Size	p_filesz;	/* Size of contents in file. */
-	Elf64_Size	p_memsz;	/* Size of contents in memory. */
-	Elf64_Size	p_align;	/* Alignment in memory and file. */
-} Elf64_Phdr;
-
-/* Standardized Elf image notes for booting... The name for all of these is ELFBoot */
-
-#endif /* ASSEMBLY */
+/* ELF segment types */
+#define PT_LOAD 1
 
 #endif /* ELF_H */
diff --git a/qemu/roms/ipxe/src/include/endian.h b/qemu/roms/ipxe/src/include/endian.h
index 9682cf9b4..79c3163ee 100644
--- a/qemu/roms/ipxe/src/include/endian.h
+++ b/qemu/roms/ipxe/src/include/endian.h
@@ -1,21 +1,22 @@
-#ifndef ETHERBOOT_ENDIAN_H
-#define ETHERBOOT_ENDIAN_H
+#ifndef _ENDIAN_H
+#define _ENDIAN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/* Definitions for byte order, according to significance of bytes,
-   from low addresses to high addresses.  The value is what you get by
-   putting '4' in the most significant byte, '3' in the second most
-   significant byte, '2' in the second least significant byte, and '1'
-   in the least significant byte, and then writing down one digit for
-   each byte, starting with the byte at the lowest address at the left,
-   and proceeding to the byte with the highest address at the right.  */
+/** Constant representing little-endian byte order
+ *
+ * Little-endian systems should define BYTE_ORDER as LITTLE_ENDIAN.
+ * This constant is intended to be used only at compile time.
+ */
+#define __LITTLE_ENDIAN 0x44332211UL
 
-#define __LITTLE_ENDIAN 1234
-#define __BIG_ENDIAN    4321
-#define __PDP_ENDIAN    3412
+/** Constant representing big-endian byte order
+ *
+ * Big-endian systems should define BYTE_ORDER as BIG_ENDIAN.
+ * This constant is intended to be used only at compile time.
+ */
+#define __BIG_ENDIAN 0x11223344UL
 
 #include "bits/endian.h"
 
-
-#endif /* ETHERBOOT_ENDIAN_H */
+#endif /* _ENDIAN_H */
diff --git a/qemu/roms/ipxe/src/include/errno.h b/qemu/roms/ipxe/src/include/errno.h
index bcc4a8816..036479aff 100644
--- a/qemu/roms/ipxe/src/include/errno.h
+++ b/qemu/roms/ipxe/src/include/errno.h
@@ -15,12 +15,16 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef ERRNO_H
 #define ERRNO_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/include/getopt.h b/qemu/roms/ipxe/src/include/getopt.h
index 0fe43567e..db3de1786 100644
--- a/qemu/roms/ipxe/src/include/getopt.h
+++ b/qemu/roms/ipxe/src/include/getopt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 
diff --git a/qemu/roms/ipxe/src/include/hci/ifmgmt_cmd.h b/qemu/roms/ipxe/src/include/hci/ifmgmt_cmd.h
index 913b911d8..5debf85c2 100644
--- a/qemu/roms/ipxe/src/include/hci/ifmgmt_cmd.h
+++ b/qemu/roms/ipxe/src/include/hci/ifmgmt_cmd.h
@@ -15,12 +15,16 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
 #ifndef _IFMGMT_CMD_H
 #define _IFMGMT_CMD_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/parseopt.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/acpi.h b/qemu/roms/ipxe/src/include/ipxe/acpi.h
index 282b6d92d..2ccd691ed 100644
--- a/qemu/roms/ipxe/src/include/ipxe/acpi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/acpi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/aes.h b/qemu/roms/ipxe/src/include/ipxe/aes.h
index 4e44f9853..0432e43ee 100644
--- a/qemu/roms/ipxe/src/include/ipxe/aes.h
+++ b/qemu/roms/ipxe/src/include/ipxe/aes.h
@@ -1,31 +1,51 @@
 #ifndef _IPXE_AES_H
 #define _IPXE_AES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+/** @file
+ *
+ * AES algorithm
+ *
+ */
 
-struct cipher_algorithm;
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/** Basic AES blocksize */
+#include <ipxe/crypto.h>
+
+/** AES blocksize */
 #define AES_BLOCKSIZE 16
 
-#include "crypto/axtls/crypto.h"
+/** Maximum number of AES rounds */
+#define AES_MAX_ROUNDS 15
+
+/** AES matrix */
+union aes_matrix {
+	/** Viewed as an array of bytes */
+	uint8_t byte[16];
+	/** Viewed as an array of four-byte columns */
+	uint32_t column[4];
+} __attribute__ (( packed ));
+
+/** AES round keys */
+struct aes_round_keys {
+	/** Round keys */
+	union aes_matrix key[AES_MAX_ROUNDS];
+};
 
 /** AES context */
 struct aes_context {
-	/** AES context for AXTLS */
-	AES_CTX axtls_ctx;
-	/** Cipher is being used for decrypting */
-	int decrypting;
+	/** Encryption keys */
+	struct aes_round_keys encrypt;
+	/** Decryption keys */
+	struct aes_round_keys decrypt;
+	/** Number of rounds */
+	unsigned int rounds;
 };
 
 /** AES context size */
 #define AES_CTX_SIZE sizeof ( struct aes_context )
 
-/* AXTLS functions */
-extern void axtls_aes_encrypt ( const AES_CTX *ctx, uint32_t *data );
-extern void axtls_aes_decrypt ( const AES_CTX *ctx, uint32_t *data );
-
 extern struct cipher_algorithm aes_algorithm;
+extern struct cipher_algorithm aes_ecb_algorithm;
 extern struct cipher_algorithm aes_cbc_algorithm;
 
 int aes_wrap ( const void *kek, const void *src, void *dest, int nblk );
diff --git a/qemu/roms/ipxe/src/include/ipxe/ansicol.h b/qemu/roms/ipxe/src/include/ipxe/ansicol.h
index 707d1599d..2b54ecaca 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ansicol.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ansicol.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <curses.h> /* For COLOR_RED etc. */
diff --git a/qemu/roms/ipxe/src/include/ipxe/ansiesc.h b/qemu/roms/ipxe/src/include/ipxe/ansiesc.h
index c1c74481d..80bc83308 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ansiesc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ansiesc.h
@@ -26,7 +26,7 @@
  * 
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ansiesc_context;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/aoe.h b/qemu/roms/ipxe/src/include/ipxe/aoe.h
index 60f3bd959..0c656e7c2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/aoe.h
+++ b/qemu/roms/ipxe/src/include/ipxe/aoe.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/api.h b/qemu/roms/ipxe/src/include/ipxe/api.h
index 838b8936e..d05d3b07a 100644
--- a/qemu/roms/ipxe/src/include/ipxe/api.h
+++ b/qemu/roms/ipxe/src/include/ipxe/api.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @defgroup Single-implementation APIs
  *
diff --git a/qemu/roms/ipxe/src/include/ipxe/arp.h b/qemu/roms/ipxe/src/include/ipxe/arp.h
index e30ae6b76..5822fa095 100644
--- a/qemu/roms/ipxe/src/include/ipxe/arp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/arp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 #include <ipxe/netdevice.h>
@@ -57,4 +57,8 @@ static inline int arp_tx ( struct io_buffer *iobuf, struct net_device *netdev,
 			      &arp_discovery, net_source, ll_source );
 }
 
+extern int arp_tx_request ( struct net_device *netdev,
+			    struct net_protocol *net_protocol,
+			    const void *net_dest, const void *net_source );
+
 #endif /* _IPXE_ARP_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/asn1.h b/qemu/roms/ipxe/src/include/ipxe/asn1.h
index d12524ddb..5fbd58281 100644
--- a/qemu/roms/ipxe/src/include/ipxe/asn1.h
+++ b/qemu/roms/ipxe/src/include/ipxe/asn1.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <time.h>
@@ -141,6 +141,24 @@ struct asn1_builder_header {
 	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
 	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 11 )
 
+/** ASN.1 OID for sha384WithRSAEncryption (1.2.840.113549.1.1.12) */
+#define ASN1_OID_SHA384WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 12 )
+
+/** ASN.1 OID for sha512WithRSAEncryption (1.2.840.113549.1.1.13) */
+#define ASN1_OID_SHA512WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 13 )
+
+/** ASN.1 OID for sha224WithRSAEncryption (1.2.840.113549.1.1.14) */
+#define ASN1_OID_SHA224WITHRSAENCRYPTION			\
+	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_TRIPLE ( 113549 ), ASN1_OID_SINGLE ( 1 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 14 )
+
 /** ASN.1 OID for id-md5 (1.2.840.113549.2.5) */
 #define ASN1_OID_MD5						\
 	ASN1_OID_INITIAL ( 1, 2 ), ASN1_OID_DOUBLE ( 840 ),	\
@@ -160,6 +178,41 @@ struct asn1_builder_header {
 	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
 	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 1 )
 
+/** ASN.1 OID for id-sha384 (2.16.840.1.101.3.4.2.2) */
+#define ASN1_OID_SHA384						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 2 )
+
+/** ASN.1 OID for id-sha512 (2.16.840.1.101.3.4.2.3) */
+#define ASN1_OID_SHA512						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 3 )
+
+/** ASN.1 OID for id-sha224 (2.16.840.1.101.3.4.2.4) */
+#define ASN1_OID_SHA224						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 4 )
+
+/** ASN.1 OID for id-sha512-224 (2.16.840.1.101.3.4.2.5) */
+#define ASN1_OID_SHA512_224						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 5 )
+
+/** ASN.1 OID for id-sha512-256 (2.16.840.1.101.3.4.2.6) */
+#define ASN1_OID_SHA512_256						\
+	ASN1_OID_INITIAL ( 2, 16 ), ASN1_OID_DOUBLE ( 840 ),	\
+	ASN1_OID_SINGLE ( 1 ), ASN1_OID_SINGLE ( 101 ),		\
+	ASN1_OID_SINGLE ( 3 ), ASN1_OID_SINGLE ( 4 ),		\
+	ASN1_OID_SINGLE ( 2 ), ASN1_OID_SINGLE ( 6 )
+
 /** ASN.1 OID for commonName (2.5.4.3) */
 #define ASN1_OID_COMMON_NAME					\
 	ASN1_OID_INITIAL ( 2, 5 ), ASN1_OID_SINGLE ( 4 ),	\
diff --git a/qemu/roms/ipxe/src/include/ipxe/ata.h b/qemu/roms/ipxe/src/include/ipxe/ata.h
index b7f02d655..a10cfafcc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ata.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ata.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * An ATA Logical Block Address
diff --git a/qemu/roms/ipxe/src/include/ipxe/base16.h b/qemu/roms/ipxe/src/include/ipxe/base16.h
index 60e3f2315..8c44da17e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/base16.h
+++ b/qemu/roms/ipxe/src/include/ipxe/base16.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -32,9 +32,36 @@ static inline size_t base16_decoded_max_len ( const char *encoded ) {
 	return ( ( strlen ( encoded ) + 1 ) / 2 );
 }
 
-extern void base16_encode ( const uint8_t *raw, size_t len, char *encoded );
-extern int hex_decode ( const char *string, char separator, void *data,
+extern size_t hex_encode ( char separator, const void *raw, size_t raw_len,
+			   char *data, size_t len );
+extern int hex_decode ( char separator, const char *encoded, void *data,
 			size_t len );
-extern int base16_decode ( const char *encoded, uint8_t *raw );
+
+/**
+ * Base16-encode data
+ *
+ * @v raw		Raw data
+ * @v raw_len		Length of raw data
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Encoded length
+ */
+static inline __attribute__ (( always_inline )) size_t
+base16_encode ( const void *raw, size_t raw_len, char *data, size_t len ) {
+	return hex_encode ( 0, raw, raw_len, data, len );
+}
+
+/**
+ * Base16-decode data
+ *
+ * @v encoded		Encoded string
+ * @v data		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of data, or negative error
+ */
+static inline __attribute__ (( always_inline )) int
+base16_decode ( const char *encoded, void *data, size_t len ) {
+	return hex_decode ( 0, encoded, data, len );
+}
 
 #endif /* _IPXE_BASE16_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/base64.h b/qemu/roms/ipxe/src/include/ipxe/base64.h
index 5fe134dc8..0c70d8382 100644
--- a/qemu/roms/ipxe/src/include/ipxe/base64.h
+++ b/qemu/roms/ipxe/src/include/ipxe/base64.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -35,7 +35,8 @@ static inline size_t base64_decoded_max_len ( const char *encoded ) {
 	return ( ( ( strlen ( encoded ) + 4 - 1 ) / 4 ) * 3 );
 }
 
-extern void base64_encode ( const uint8_t *raw, size_t len, char *encoded );
-extern int base64_decode ( const char *encoded, uint8_t *raw );
+extern size_t base64_encode ( const void *raw, size_t raw_len, char *data,
+			      size_t len );
+extern int base64_decode ( const char *encoded, void *data, size_t len );
 
 #endif /* _IPXE_BASE64_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/bigint.h b/qemu/roms/ipxe/src/include/ipxe/bigint.h
index 97fbce245..2f99f8445 100644
--- a/qemu/roms/ipxe/src/include/ipxe/bigint.h
+++ b/qemu/roms/ipxe/src/include/ipxe/bigint.h
@@ -6,7 +6,7 @@
  * Big integer support
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Define a big-integer type
diff --git a/qemu/roms/ipxe/src/include/ipxe/bitbash.h b/qemu/roms/ipxe/src/include/ipxe/bitbash.h
index 69d5d9e3e..2a2e475d0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/bitbash.h
+++ b/qemu/roms/ipxe/src/include/ipxe/bitbash.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct bit_basher;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/bitmap.h b/qemu/roms/ipxe/src/include/ipxe/bitmap.h
index b18584c1f..38aca694b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/bitmap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/bitmap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/bitops.h b/qemu/roms/ipxe/src/include/ipxe/bitops.h
index 73e859f41..220ab0fe7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/bitops.h
+++ b/qemu/roms/ipxe/src/include/ipxe/bitops.h
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/include/ipxe/blockdev.h b/qemu/roms/ipxe/src/include/ipxe/blockdev.h
index 9f0a9f787..418c43004 100644
--- a/qemu/roms/ipxe/src/include/ipxe/blockdev.h
+++ b/qemu/roms/ipxe/src/include/ipxe/blockdev.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/blocktrans.h b/qemu/roms/ipxe/src/include/ipxe/blocktrans.h
new file mode 100644
index 000000000..fee71b96c
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/blocktrans.h
@@ -0,0 +1,38 @@
+#ifndef _IPXE_BLOCKTRANS_H
+#define _IPXE_BLOCKTRANS_H
+
+/** @file
+ *
+ * Block device translator
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/uaccess.h>
+
+/** A block device translator */
+struct block_translator {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Block device interface */
+	struct interface block;
+	/** Data transfer interface */
+	struct interface xfer;
+
+	/** Data transfer buffer */
+	struct xfer_buffer xferbuf;
+	/** Data buffer */
+	userptr_t buffer;
+	/** Block size */
+	size_t blksize;
+};
+
+extern int block_translate ( struct interface *block,
+			     userptr_t buffer, size_t size );
+
+#endif /* _IPXE_BLOCKTRANS_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/bofm.h b/qemu/roms/ipxe/src/include/ipxe/bofm.h
index 1da47f651..bc994ea8b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/bofm.h
+++ b/qemu/roms/ipxe/src/include/ipxe/bofm.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/cbc.h b/qemu/roms/ipxe/src/include/ipxe/cbc.h
index fae376577..18a94e144 100644
--- a/qemu/roms/ipxe/src/include/ipxe/cbc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/cbc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/crypto.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/cdc.h b/qemu/roms/ipxe/src/include/ipxe/cdc.h
new file mode 100644
index 000000000..f1799cd9a
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/cdc.h
@@ -0,0 +1,55 @@
+#ifndef _IPXE_CDC_H
+#define _IPXE_CDC_H
+
+/** @file
+ *
+ * USB Communications Device Class (CDC)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** Class code for communications devices */
+#define USB_CLASS_CDC 2
+
+/** Union functional descriptor */
+struct cdc_union_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Descriptor subtype */
+	uint8_t subtype;
+	/** Interfaces (variable-length) */
+	uint8_t interface[1];
+} __attribute__ (( packed ));
+
+/** Union functional descriptor subtype */
+#define CDC_SUBTYPE_UNION 6
+
+/** Ethernet descriptor subtype */
+#define CDC_SUBTYPE_ETHERNET 15
+
+/** Network connection notification */
+#define CDC_NETWORK_CONNECTION						\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x00 ) )
+
+/** Connection speed change notification */
+#define CDC_CONNECTION_SPEED_CHANGE					\
+	( USB_DIR_IN | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x2a ) )
+
+/** Connection speed change notification */
+struct cdc_connection_speed_change {
+	/** Downlink bit rate, in bits per second */
+	uint32_t down;
+	/** Uplink bit rate, in bits per second */
+	uint32_t up;
+} __attribute__ (( packed ));
+
+extern struct cdc_union_descriptor *
+cdc_union_descriptor ( struct usb_configuration_descriptor *config,
+		       struct usb_interface_descriptor *interface );
+
+#endif /* _IPXE_CDC_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/certstore.h b/qemu/roms/ipxe/src/include/ipxe/certstore.h
index 7456db621..49b3b512c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/certstore.h
+++ b/qemu/roms/ipxe/src/include/ipxe/certstore.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/asn1.h>
 #include <ipxe/x509.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/chap.h b/qemu/roms/ipxe/src/include/ipxe/chap.h
index fce48f3ea..7c693e29d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/chap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/chap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/md5.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/cms.h b/qemu/roms/ipxe/src/include/ipxe/cms.h
index e026ebd2f..7adf724b2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/cms.h
+++ b/qemu/roms/ipxe/src/include/ipxe/cms.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <time.h>
 #include <ipxe/asn1.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/command.h b/qemu/roms/ipxe/src/include/ipxe/command.h
index 432da1abb..a208e7d8f 100644
--- a/qemu/roms/ipxe/src/include/ipxe/command.h
+++ b/qemu/roms/ipxe/src/include/ipxe/command.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_COMMAND_H
 #define _IPXE_COMMAND_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/console.h b/qemu/roms/ipxe/src/include/ipxe/console.h
index 4b90c9cec..1b764aaca 100644
--- a/qemu/roms/ipxe/src/include/ipxe/console.h
+++ b/qemu/roms/ipxe/src/include/ipxe/console.h
@@ -16,7 +16,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct pixel_buffer;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/cpio.h b/qemu/roms/ipxe/src/include/ipxe/cpio.h
index 277232808..0637c531d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/cpio.h
+++ b/qemu/roms/ipxe/src/include/ipxe/cpio.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A CPIO archive header
  *
diff --git a/qemu/roms/ipxe/src/include/ipxe/crc32.h b/qemu/roms/ipxe/src/include/ipxe/crc32.h
index 38ac1b31f..30d2fe66c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/crc32.h
+++ b/qemu/roms/ipxe/src/include/ipxe/crc32.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_CRC32_H
 #define _IPXE_CRC32_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/crypto.h b/qemu/roms/ipxe/src/include/ipxe/crypto.h
index 3eda5ec6e..fc0d8b22b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/crypto.h
+++ b/qemu/roms/ipxe/src/include/ipxe/crypto.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/deflate.h b/qemu/roms/ipxe/src/include/ipxe/deflate.h
index 19c5125eb..b751aa9a3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/deflate.h
+++ b/qemu/roms/ipxe/src/include/ipxe/deflate.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/device.h b/qemu/roms/ipxe/src/include/ipxe/device.h
index 7202a6966..d81417e8e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/device.h
+++ b/qemu/roms/ipxe/src/include/ipxe/device.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/tables.h>
@@ -63,10 +63,16 @@ struct device_description {
 /** Xen bus type */
 #define BUS_TYPE_XEN 8
 
+/** Hyper-V bus type */
+#define BUS_TYPE_HV 9
+
+/** USB bus type */
+#define BUS_TYPE_USB 10
+
 /** A hardware device */
 struct device {
 	/** Name */
-	char name[16];
+	char name[32];
 	/** Driver name */
 	const char *driver_name;
 	/** Device description */
@@ -93,6 +99,8 @@ struct root_device {
 	struct device dev;
 	/** Root device driver */
 	struct root_driver *driver;
+	/** Driver-private data */
+	void *priv;
 };
 
 /** A root device driver */
@@ -123,6 +131,27 @@ struct root_driver {
 /** Declare a root device */
 #define __root_device __table_entry ( ROOT_DEVICES, 01 )
 
+/**
+ * Set root device driver-private data
+ *
+ * @v rootdev		Root device
+ * @v priv		Private data
+ */
+static inline void rootdev_set_drvdata ( struct root_device *rootdev,
+					 void *priv ){
+	rootdev->priv = priv;
+}
+
+/**
+ * Get root device driver-private data
+ *
+ * @v rootdev		Root device
+ * @ret priv		Private data
+ */
+static inline void * rootdev_get_drvdata ( struct root_device *rootdev ) {
+	return rootdev->priv;
+}
+
 extern int device_keep_count;
 
 /**
diff --git a/qemu/roms/ipxe/src/include/ipxe/dhcp.h b/qemu/roms/ipxe/src/include/ipxe/dhcp.h
index bcfb85cc1..a11db3497 100644
--- a/qemu/roms/ipxe/src/include/ipxe/dhcp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/dhcp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
@@ -639,16 +639,6 @@ struct dhcphdr {
  */
 #define DHCP_MIN_LEN 552
 
-/** Timeouts for sending DHCP packets */
-#define DHCP_MIN_TIMEOUT ( 1 * TICKS_PER_SEC )
-#define DHCP_MAX_TIMEOUT ( 10 * TICKS_PER_SEC )
-
-/** Maximum time that we will wait for ProxyDHCP responses */
-#define PROXYDHCP_MAX_TIMEOUT ( 2 * TICKS_PER_SEC )
-
-/** Maximum time that we will wait for Boot Server responses */
-#define PXEBS_MAX_TIMEOUT ( 3 * TICKS_PER_SEC )
-
 /** Settings block name used for DHCP responses */
 #define DHCP_SETTINGS_NAME "dhcp"
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/dhcpopts.h b/qemu/roms/ipxe/src/include/ipxe/dhcpopts.h
index c5af5d749..707fda4a8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/dhcpopts.h
+++ b/qemu/roms/ipxe/src/include/ipxe/dhcpopts.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/dhcppkt.h b/qemu/roms/ipxe/src/include/ipxe/dhcppkt.h
index 3179a6bb0..f13dfc93d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/dhcppkt.h
+++ b/qemu/roms/ipxe/src/include/ipxe/dhcppkt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/dhcp.h>
 #include <ipxe/dhcpopts.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/dhcpv6.h b/qemu/roms/ipxe/src/include/ipxe/dhcpv6.h
index 2636b8ab2..9307b6cae 100644
--- a/qemu/roms/ipxe/src/include/ipxe/dhcpv6.h
+++ b/qemu/roms/ipxe/src/include/ipxe/dhcpv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/dns.h b/qemu/roms/ipxe/src/include/ipxe/dns.h
index 4f6cab3a4..738dea6e4 100644
--- a/qemu/roms/ipxe/src/include/ipxe/dns.h
+++ b/qemu/roms/ipxe/src/include/ipxe/dns.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/downloader.h b/qemu/roms/ipxe/src/include/ipxe/downloader.h
index de1a2e75e..ccb1abfef 100644
--- a/qemu/roms/ipxe/src/include/ipxe/downloader.h
+++ b/qemu/roms/ipxe/src/include/ipxe/downloader.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct interface;
 struct image;
diff --git a/qemu/roms/ipxe/src/include/ipxe/drbg.h b/qemu/roms/ipxe/src/include/ipxe/drbg.h
index 6374e7787..ed2b3757a 100644
--- a/qemu/roms/ipxe/src/include/ipxe/drbg.h
+++ b/qemu/roms/ipxe/src/include/ipxe/drbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/sha256.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ecb.h b/qemu/roms/ipxe/src/include/ipxe/ecb.h
new file mode 100644
index 000000000..4e6aa3c81
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/ecb.h
@@ -0,0 +1,55 @@
+#ifndef _IPXE_ECB_H
+#define _IPXE_ECB_H
+
+/** @file
+ *
+ * Electronic codebook (ECB)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/crypto.h>
+
+extern void ecb_encrypt ( void *ctx, const void *src, void *dst,
+			  size_t len, struct cipher_algorithm *raw_cipher );
+extern void ecb_decrypt ( void *ctx, const void *src, void *dst,
+			  size_t len, struct cipher_algorithm *raw_cipher );
+
+/**
+ * Create a cipher-block chaining mode of behaviour of an existing cipher
+ *
+ * @v _ecb_name		Name for the new ECB cipher
+ * @v _ecb_cipher	New cipher algorithm
+ * @v _raw_cipher	Underlying cipher algorithm
+ * @v _raw_context	Context structure for the underlying cipher
+ * @v _blocksize	Cipher block size
+ */
+#define ECB_CIPHER( _ecb_name, _ecb_cipher, _raw_cipher, _raw_context,	\
+		    _blocksize )					\
+static int _ecb_name ## _setkey ( void *ctx, const void *key,		\
+				  size_t keylen ) {			\
+	return cipher_setkey ( &_raw_cipher, ctx, key, keylen );	\
+}									\
+static void _ecb_name ## _setiv ( void *ctx, const void *iv ) {		\
+	cipher_setiv ( &_raw_cipher, ctx, iv );				\
+}									\
+static void _ecb_name ## _encrypt ( void *ctx, const void *src,		\
+				    void *dst, size_t len ) {		\
+	ecb_encrypt ( ctx, src, dst, len, &_raw_cipher );		\
+}									\
+static void _ecb_name ## _decrypt ( void *ctx, const void *src,		\
+				    void *dst, size_t len ) {		\
+	ecb_decrypt ( ctx, src, dst, len, &_raw_cipher );		\
+}									\
+struct cipher_algorithm _ecb_cipher = {					\
+	.name		= #_ecb_name,					\
+	.ctxsize	= sizeof ( _raw_context ),			\
+	.blocksize	= _blocksize,					\
+	.setkey		= _ecb_name ## _setkey,				\
+	.setiv		= _ecb_name ## _setiv,				\
+	.encrypt	= _ecb_name ## _encrypt,			\
+	.decrypt	= _ecb_name ## _decrypt,			\
+};
+
+#endif /* _IPXE_ECB_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/edd.h b/qemu/roms/ipxe/src/include/ipxe/edd.h
index 0c25593d5..1914fd0b0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/edd.h
+++ b/qemu/roms/ipxe/src/include/ipxe/edd.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/editbox.h b/qemu/roms/ipxe/src/include/ipxe/editbox.h
index 9122dbbf3..2c70e0b6b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/editbox.h
+++ b/qemu/roms/ipxe/src/include/ipxe/editbox.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <curses.h>
 #include <ipxe/editstring.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/editstring.h b/qemu/roms/ipxe/src/include/ipxe/editstring.h
index 2ef546a63..a00a8adaa 100644
--- a/qemu/roms/ipxe/src/include/ipxe/editstring.h
+++ b/qemu/roms/ipxe/src/include/ipxe/editstring.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** An editable string */
 struct edit_string {
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h b/qemu/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
index 1294459f9..7466814fa 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/ProcessorBind.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_EFI_PROCESSOR_BIND_H
 #define _IPXE_EFI_PROCESSOR_BIND_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * EFI header files rely on having the CPU architecture directory
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h b/qemu/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h
new file mode 100644
index 000000000..f04efbb03
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/Protocol/Rng.h
@@ -0,0 +1,158 @@
+/** @file
+  EFI_RNG_PROTOCOL as defined in UEFI 2.4.
+  The UEFI Random Number Generator Protocol is used to provide random bits for use
+  in applications, or entropy for seeding other random number generators.
+
+Copyright (c) 2013, Intel Corporation. All rights reserved.<BR>
+This program and the accompanying materials are licensed and made available under
+the terms and conditions of the BSD License that accompanies this distribution.
+The full text of the license may be found at
+http://opensource.org/licenses/bsd-license.php.
+
+THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
+WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
+
+**/
+
+#ifndef __EFI_RNG_PROTOCOL_H__
+#define __EFI_RNG_PROTOCOL_H__
+
+FILE_LICENCE ( BSD3 );
+
+///
+/// Global ID for the Random Number Generator Protocol
+///
+#define EFI_RNG_PROTOCOL_GUID \
+  { \
+    0x3152bca5, 0xeade, 0x433d, {0x86, 0x2e, 0xc0, 0x1c, 0xdc, 0x29, 0x1f, 0x44 } \
+  }
+
+typedef struct _EFI_RNG_PROTOCOL EFI_RNG_PROTOCOL;
+
+///
+/// A selection of EFI_RNG_PROTOCOL algorithms.
+/// The algorithms listed are optional, not meant to be exhaustive and be argmented by
+/// vendors or other industry standards.
+///
+
+typedef EFI_GUID EFI_RNG_ALGORITHM;
+
+///
+/// The algorithms corresponds to SP800-90 as defined in
+/// NIST SP 800-90, "Recommendation for Random Number Generation Using Deterministic Random
+/// Bit Generators", March 2007.
+///
+#define EFI_RNG_ALGORITHM_SP800_90_HASH_256_GUID \
+  { \
+    0xa7af67cb, 0x603b, 0x4d42, {0xba, 0x21, 0x70, 0xbf, 0xb6, 0x29, 0x3f, 0x96 } \
+  }
+#define EFI_RNG_ALGORITHM_SP800_90_HMAC_256_GUID \
+  { \
+    0xc5149b43, 0xae85, 0x4f53, {0x99, 0x82, 0xb9, 0x43, 0x35, 0xd3, 0xa9, 0xe7 } \
+  }
+#define EFI_RNG_ALGORITHM_SP800_90_CTR_256_GUID \
+  { \
+    0x44f0de6e, 0x4d8c, 0x4045, {0xa8, 0xc7, 0x4d, 0xd1, 0x68, 0x85, 0x6b, 0x9e } \
+  }
+///
+/// The algorithms correspond to X9.31 as defined in
+/// NIST, "Recommended Random Number Generator Based on ANSI X9.31 Appendix A.2.4 Using
+/// the 3-Key Triple DES and AES Algorithm", January 2005.
+///
+#define EFI_RNG_ALGORITHM_X9_31_3DES_GUID \
+  { \
+    0x63c4785a, 0xca34, 0x4012, {0xa3, 0xc8, 0x0b, 0x6a, 0x32, 0x4f, 0x55, 0x46 } \
+  }
+#define EFI_RNG_ALGORITHM_X9_31_AES_GUID \
+  { \
+    0xacd03321, 0x777e, 0x4d3d, {0xb1, 0xc8, 0x20, 0xcf, 0xd8, 0x88, 0x20, 0xc9 } \
+  }
+///
+/// The "raw" algorithm, when supported, is intended to provide entropy directly from
+/// the source, without it going through some deterministic random bit generator.
+///
+#define EFI_RNG_ALGORITHM_RAW \
+  { \
+    0xe43176d7, 0xb6e8, 0x4827, {0xb7, 0x84, 0x7f, 0xfd, 0xc4, 0xb6, 0x85, 0x61 } \
+  }
+
+/**
+  Returns information about the random number generation implementation.
+
+  @param[in]     This                 A pointer to the EFI_RNG_PROTOCOL instance.
+  @param[in,out] RNGAlgorithmListSize On input, the size in bytes of RNGAlgorithmList.
+                                      On output with a return code of EFI_SUCCESS, the size
+                                      in bytes of the data returned in RNGAlgorithmList. On output
+                                      with a return code of EFI_BUFFER_TOO_SMALL,
+                                      the size of RNGAlgorithmList required to obtain the list.
+  @param[out] RNGAlgorithmList        A caller-allocated memory buffer filled by the driver
+                                      with one EFI_RNG_ALGORITHM element for each supported
+                                      RNG algorithm. The list must not change across multiple
+                                      calls to the same driver. The first algorithm in the list
+                                      is the default algorithm for the driver.
+
+  @retval EFI_SUCCESS                 The RNG algorithm list was returned successfully.
+  @retval EFI_UNSUPPORTED             The services is not supported by this driver.
+  @retval EFI_DEVICE_ERROR            The list of algorithms could not be retrieved due to a
+                                      hardware or firmware error.
+  @retval EFI_INVALID_PARAMETER       One or more of the parameters are incorrect.
+  @retval EFI_BUFFER_TOO_SMALL        The buffer RNGAlgorithmList is too small to hold the result.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_RNG_GET_INFO) (
+  IN EFI_RNG_PROTOCOL             *This,
+  IN OUT UINTN                    *RNGAlgorithmListSize,
+  OUT EFI_RNG_ALGORITHM           *RNGAlgorithmList
+  );
+
+/**
+  Produces and returns an RNG value using either the default or specified RNG algorithm.
+
+  @param[in]  This                    A pointer to the EFI_RNG_PROTOCOL instance.
+  @param[in]  RNGAlgorithm            A pointer to the EFI_RNG_ALGORITHM that identifies the RNG
+                                      algorithm to use. May be NULL in which case the function will
+                                      use its default RNG algorithm.
+  @param[in]  RNGValueLength          The length in bytes of the memory buffer pointed to by
+                                      RNGValue. The driver shall return exactly this numbers of bytes.
+  @param[out] RNGValue                A caller-allocated memory buffer filled by the driver with the
+                                      resulting RNG value.
+
+  @retval EFI_SUCCESS                 The RNG value was returned successfully.
+  @retval EFI_UNSUPPORTED             The algorithm specified by RNGAlgorithm is not supported by
+                                      this driver.
+  @retval EFI_DEVICE_ERROR            An RNG value could not be retrieved due to a hardware or
+                                      firmware error.
+  @retval EFI_NOT_READY               There is not enough random data available to satisfy the length
+                                      requested by RNGValueLength.
+  @retval EFI_INVALID_PARAMETER       RNGValue is NULL or RNGValueLength is zero.
+
+**/
+typedef
+EFI_STATUS
+(EFIAPI *EFI_RNG_GET_RNG) (
+  IN EFI_RNG_PROTOCOL            *This,
+  IN EFI_RNG_ALGORITHM           *RNGAlgorithm, OPTIONAL
+  IN UINTN                       RNGValueLength,
+  OUT UINT8                      *RNGValue
+  );
+
+///
+/// The Random Number Generator (RNG) protocol provides random bits for use in
+/// applications, or entropy for seeding other random number generators.
+///
+struct _EFI_RNG_PROTOCOL {
+  EFI_RNG_GET_INFO                GetInfo;
+  EFI_RNG_GET_RNG                 GetRNG;
+};
+
+extern EFI_GUID gEfiRngProtocolGuid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Hash256Guid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Hmac256Guid;
+extern EFI_GUID gEfiRngAlgorithmSp80090Ctr256Guid;
+extern EFI_GUID gEfiRngAlgorithmX9313DesGuid;
+extern EFI_GUID gEfiRngAlgorithmX931AesGuid;
+extern EFI_GUID gEfiRngAlgorithmRaw;
+
+#endif
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
index d4a26850c..1d5ddc8c3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_autoboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void efi_set_autoboot ( void );
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_driver.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_driver.h
index e16a24daa..f497df3e3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_driver.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_driver.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_entropy.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_entropy.h
new file mode 100644
index 000000000..39a667355
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_entropy.h
@@ -0,0 +1,35 @@
+#ifndef _IPXE_EFI_ENTROPY_H
+#define _IPXE_EFI_ENTROPY_H
+
+/** @file
+ *
+ * EFI entropy source
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+#ifdef ENTROPY_EFI
+#define ENTROPY_PREFIX_efi
+#else
+#define ENTROPY_PREFIX_efi __efi_
+#endif
+
+/**
+ * min-entropy per sample
+ *
+ * @ret min_entropy	min-entropy of each sample
+ */
+static inline __always_inline double
+ENTROPY_INLINE ( efi, min_entropy_per_sample ) ( void ) {
+
+	/* We use essentially the same mechanism as for the BIOS
+	 * RTC-based entropy source, and so assume the same
+	 * min-entropy per sample.
+	 */
+	return 1.3;
+}
+
+#endif /* _IPXE_EFI_ENTROPY_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_hii.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_hii.h
index 8e94bbe7e..bbec31194 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_hii.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_hii.h
@@ -6,7 +6,7 @@
  * EFI human interface infrastructure
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <ipxe/efi/Uefi/UefiInternalFormRepresentation.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci.h
index af36613d9..6dd945f05 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/pci.h>
 #include <ipxe/efi/efi.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
index 498a0388b..887d5ee14 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_pci_api.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_EFI
 #define PCIAPI_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_reboot.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
index 33921b913..249cae8c5 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_EFI
 #define REBOOT_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_smbios.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
index 7642e5bc5..d890d5460 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_EFI
 #define SMBIOS_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_snp.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_snp.h
index a18bced5f..1e5c66626 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_snp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_snp.h
@@ -18,6 +18,9 @@
 #include <ipxe/efi/Protocol/HiiDatabase.h>
 #include <ipxe/efi/Protocol/LoadFile.h>
 
+/** SNP transmit completion ring size */
+#define EFI_SNP_NUM_TX 32
+
 /** An SNP device */
 struct efi_snp_device {
 	/** List of SNP devices */
@@ -34,20 +37,16 @@ struct efi_snp_device {
 	EFI_SIMPLE_NETWORK_MODE mode;
 	/** Started flag */
 	int started;
-	/** Outstanding TX packet count (via "interrupt status")
-	 *
-	 * Used in order to generate TX completions.
-	 */
-	unsigned int tx_count_interrupts;
-	/** Outstanding TX packet count (via "recycled tx buffers")
-	 *
-	 * Used in order to generate TX completions.
-	 */
-	unsigned int tx_count_txbufs;
-	/** Outstanding RX packet count (via "interrupt status") */
-	unsigned int rx_count_interrupts;
-	/** Outstanding RX packet count (via WaitForPacket event) */
-	unsigned int rx_count_events;
+	/** Pending interrupt status */
+	unsigned int interrupts;
+	/** Transmit completion ring */
+	VOID *tx[EFI_SNP_NUM_TX];
+	/** Transmit completion ring producer counter */
+	unsigned int tx_prod;
+	/** Transmit completion ring consumer counter */
+	unsigned int tx_cons;
+	/** Receive queue */
+	struct list_head rx;
 	/** The network interface identifier */
 	EFI_NETWORK_INTERFACE_IDENTIFIER_PROTOCOL nii;
 	/** Component name protocol */
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_strings.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_strings.h
index 023ccda07..2f241537e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_strings.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_strings.h
@@ -6,7 +6,7 @@
  * EFI strings
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_time.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_time.h
new file mode 100644
index 000000000..099994b57
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_time.h
@@ -0,0 +1,20 @@
+#ifndef _IPXE_EFI_TIME_H
+#define _IPXE_EFI_TIME_H
+
+/** @file
+ *
+ * EFI time source
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+#ifdef TIME_EFI
+#define TIME_PREFIX_efi
+#else
+#define TIME_PREFIX_efi __efi_
+#endif
+
+#endif /* _IPXE_EFI_TIME_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_timer.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_timer.h
index b10543d6c..c03765393 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_timer.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_EFI
 #define TIMER_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
index 870a089b2..3cc750405 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_uaccess.h
@@ -10,7 +10,7 @@
  * no-ops.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UACCESS_EFI
 #define UACCESS_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
index 911e69a96..4eb2a5f9b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_umalloc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef UMALLOC_EFI
 #define UMALLOC_PREFIX_efi
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_utils.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_utils.h
index 9164be190..57268daf7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_utils.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_utils.h
@@ -6,7 +6,7 @@
  * EFI utilities
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Protocol/DevicePath.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h
new file mode 100644
index 000000000..4a56b9a29
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_watchdog.h
@@ -0,0 +1,31 @@
+#ifndef _IPXE_EFI_WATCHDOG_H
+#define _IPXE_EFI_WATCHDOG_H
+
+/** @file
+ *
+ * EFI watchdog holdoff timer
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern struct retry_timer efi_watchdog;
+
+/**
+ * Start EFI watchdog holdoff timer
+ *
+ */
+static inline void efi_watchdog_start ( void ) {
+
+	start_timer_nodelay ( &efi_watchdog );
+}
+
+/**
+ * Stop EFI watchdog holdoff timer
+ *
+ */
+static inline void efi_watchdog_stop ( void ) {
+
+	stop_timer ( &efi_watchdog );
+}
+
+#endif /* _IPXE_EFI_WATCHDOG_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/efi/efi_wrap.h b/qemu/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
index 7579e0fe9..d8ed1a5cc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/efi/efi_wrap.h
@@ -6,7 +6,7 @@
  * EFI driver interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/eisa.h b/qemu/roms/ipxe/src/include/ipxe/eisa.h
index 22a1ed94e..e7dac1f39 100644
--- a/qemu/roms/ipxe/src/include/ipxe/eisa.h
+++ b/qemu/roms/ipxe/src/include/ipxe/eisa.h
@@ -1,7 +1,7 @@
 #ifndef EISA_H
 #define EISA_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/isa_ids.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/elf.h b/qemu/roms/ipxe/src/include/ipxe/elf.h
index ec675c047..033c3f7a8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/elf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/elf.h
@@ -8,10 +8,21 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdint.h>
+#include <ipxe/image.h>
 #include <elf.h>
 
+typedef Elf32_Ehdr	Elf_Ehdr;
+typedef Elf32_Phdr	Elf_Phdr;
+typedef Elf32_Off	Elf_Off;
+#define ELFCLASS	ELFCLASS32
+
+extern int elf_segments ( struct image *image, Elf_Ehdr *ehdr,
+			  int ( * process ) ( struct image *image,
+					      Elf_Phdr *phdr, physaddr_t dest ),
+			  physaddr_t *entry, physaddr_t *max );
 extern int elf_load ( struct image *image, physaddr_t *entry, physaddr_t *max );
 
 #endif /* _IPXE_ELF_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/eltorito.h b/qemu/roms/ipxe/src/include/ipxe/eltorito.h
index 3302b38b6..27e361b16 100644
--- a/qemu/roms/ipxe/src/include/ipxe/eltorito.h
+++ b/qemu/roms/ipxe/src/include/ipxe/eltorito.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iso9660.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/entropy.h b/qemu/roms/ipxe/src/include/ipxe/entropy.h
index adf325e79..beeb3abfa 100644
--- a/qemu/roms/ipxe/src/include/ipxe/entropy.h
+++ b/qemu/roms/ipxe/src/include/ipxe/entropy.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -54,6 +54,7 @@ typedef uint8_t entropy_sample_t;
 
 /* Include all architecture-independent entropy API headers */
 #include <ipxe/null_entropy.h>
+#include <ipxe/efi/efi_entropy.h>
 #include <ipxe/linux/linux_entropy.h>
 
 /* Include all architecture-dependent entropy API headers */
diff --git a/qemu/roms/ipxe/src/include/ipxe/errfile.h b/qemu/roms/ipxe/src/include/ipxe/errfile.h
index f809337ff..e21c95938 100644
--- a/qemu/roms/ipxe/src/include/ipxe/errfile.h
+++ b/qemu/roms/ipxe/src/include/ipxe/errfile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/errfile.h>
 
@@ -68,6 +68,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_fbcon		       ( ERRFILE_CORE | 0x001c0000 )
 #define ERRFILE_ansicol		       ( ERRFILE_CORE | 0x001d0000 )
 #define ERRFILE_ansicoldef	       ( ERRFILE_CORE | 0x001e0000 )
+#define ERRFILE_fault		       ( ERRFILE_CORE | 0x001f0000 )
+#define ERRFILE_blocktrans	       ( ERRFILE_CORE | 0x00200000 )
 
 #define ERRFILE_eisa		     ( ERRFILE_DRIVER | 0x00000000 )
 #define ERRFILE_isa		     ( ERRFILE_DRIVER | 0x00010000 )
@@ -76,12 +78,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_pci		     ( ERRFILE_DRIVER | 0x00040000 )
 #define ERRFILE_linux		     ( ERRFILE_DRIVER | 0x00050000 )
 #define ERRFILE_pcivpd		     ( ERRFILE_DRIVER | 0x00060000 )
+#define ERRFILE_usb		     ( ERRFILE_DRIVER | 0x00070000 )
+#define ERRFILE_usbhub		     ( ERRFILE_DRIVER | 0x00080000 )
+#define ERRFILE_xhci		     ( ERRFILE_DRIVER | 0x00090000 )
+#define ERRFILE_ehci		     ( ERRFILE_DRIVER | 0x000a0000 )
+#define ERRFILE_uhci		     ( ERRFILE_DRIVER | 0x000b0000 )
+#define ERRFILE_usbhid		     ( ERRFILE_DRIVER | 0x000c0000 )
+#define ERRFILE_usbkbd		     ( ERRFILE_DRIVER | 0x000d0000 )
 
 #define ERRFILE_nvs		     ( ERRFILE_DRIVER | 0x00100000 )
 #define ERRFILE_spi		     ( ERRFILE_DRIVER | 0x00110000 )
 #define ERRFILE_i2c_bit		     ( ERRFILE_DRIVER | 0x00120000 )
 #define ERRFILE_spi_bit		     ( ERRFILE_DRIVER | 0x00130000 )
 #define ERRFILE_nvsvpd		     ( ERRFILE_DRIVER | 0x00140000 )
+#define ERRFILE_uart		     ( ERRFILE_DRIVER | 0x00150000 )
 
 #define ERRFILE_3c509		     ( ERRFILE_DRIVER | 0x00200000 )
 #define ERRFILE_bnx2		     ( ERRFILE_DRIVER | 0x00210000 )
@@ -157,7 +167,11 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_snp		     ( ERRFILE_DRIVER | 0x00680000 )
 #define ERRFILE_netfront	     ( ERRFILE_DRIVER | 0x00690000 )
 #define ERRFILE_nii		     ( ERRFILE_DRIVER | 0x006a0000 )
-
+#define ERRFILE_netvsc		     ( ERRFILE_DRIVER | 0x006b0000 )
+#define ERRFILE_ecm		     ( ERRFILE_DRIVER | 0x006c0000 )
+#define ERRFILE_ncm		     ( ERRFILE_DRIVER | 0x006d0000 )
+#define ERRFILE_usbnet		     ( ERRFILE_DRIVER | 0x006e0000 )
+#define ERRFILE_dm96xx		     ( ERRFILE_DRIVER | 0x006f0000 )
 #define ERRFILE_scsi		     ( ERRFILE_DRIVER | 0x00700000 )
 #define ERRFILE_arbel		     ( ERRFILE_DRIVER | 0x00710000 )
 #define ERRFILE_hermon		     ( ERRFILE_DRIVER | 0x00720000 )
@@ -165,6 +179,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_ata		     ( ERRFILE_DRIVER | 0x00740000 )
 #define ERRFILE_srp		     ( ERRFILE_DRIVER | 0x00750000 )
 #define ERRFILE_qib7322		     ( ERRFILE_DRIVER | 0x00760000 )
+#define ERRFILE_smsc75xx	     ( ERRFILE_DRIVER | 0x00770000 )
+#define ERRFILE_intelvf		     ( ERRFILE_DRIVER | 0x00780000 )
+#define ERRFILE_intelxvf	     ( ERRFILE_DRIVER | 0x00790000 )
 
 #define ERRFILE_aoe			( ERRFILE_NET | 0x00000000 )
 #define ERRFILE_arp			( ERRFILE_NET | 0x00010000 )
@@ -227,6 +244,17 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_ping			( ERRFILE_NET | 0x003a0000 )
 #define ERRFILE_dhcpv6			( ERRFILE_NET | 0x003b0000 )
 #define ERRFILE_nfs_uri			( ERRFILE_NET | 0x003c0000 )
+#define ERRFILE_rndis			( ERRFILE_NET | 0x003d0000 )
+#define ERRFILE_pccrc			( ERRFILE_NET | 0x003e0000 )
+#define ERRFILE_stp			( ERRFILE_NET | 0x003f0000 )
+#define ERRFILE_pccrd			( ERRFILE_NET | 0x00400000 )
+#define ERRFILE_httpconn		( ERRFILE_NET | 0x00410000 )
+#define ERRFILE_httpauth		( ERRFILE_NET | 0x00420000 )
+#define ERRFILE_httpbasic		( ERRFILE_NET | 0x00430000 )
+#define ERRFILE_httpdigest		( ERRFILE_NET | 0x00440000 )
+#define ERRFILE_peerdisc		( ERRFILE_NET | 0x00450000 )
+#define ERRFILE_peerblk			( ERRFILE_NET | 0x00460000 )
+#define ERRFILE_peermux			( ERRFILE_NET | 0x00470000 )
 
 #define ERRFILE_image		      ( ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_elf		      ( ERRFILE_IMAGE | 0x00010000 )
@@ -245,7 +273,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_imgmgmt		      ( ERRFILE_OTHER | 0x00050000 )
 #define ERRFILE_pxe_tftp	      ( ERRFILE_OTHER | 0x00060000 )
 #define ERRFILE_pxe_udp		      ( ERRFILE_OTHER | 0x00070000 )
-#define ERRFILE_axtls_aes	      ( ERRFILE_OTHER | 0x00080000 )
+#define ERRFILE_aes	 	      ( ERRFILE_OTHER | 0x00080000 )
 #define ERRFILE_cipher		      ( ERRFILE_OTHER | 0x00090000 )
 #define ERRFILE_image_cmd	      ( ERRFILE_OTHER | 0x000a0000 )
 #define ERRFILE_uri_test	      ( ERRFILE_OTHER | 0x000b0000 )
@@ -308,6 +336,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_xengrant	      ( ERRFILE_OTHER | 0x00440000 )
 #define ERRFILE_efi_utils	      ( ERRFILE_OTHER | 0x00450000 )
 #define ERRFILE_efi_wrap	      ( ERRFILE_OTHER | 0x00460000 )
+#define ERRFILE_vmbus		      ( ERRFILE_OTHER | 0x00470000 )
+#define ERRFILE_efi_time	      ( ERRFILE_OTHER | 0x00480000 )
+#define ERRFILE_efi_watchdog	      ( ERRFILE_OTHER | 0x00490000 )
 
 /** @} */
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/errno/efi.h b/qemu/roms/ipxe/src/include/ipxe/errno/efi.h
index 2d2c50176..9f010f5fb 100644
--- a/qemu/roms/ipxe/src/include/ipxe/errno/efi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/errno/efi.h
@@ -21,7 +21,7 @@
  * as-is.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Uefi/UefiBaseType.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/errno/linux.h b/qemu/roms/ipxe/src/include/ipxe/errno/linux.h
index 11309b4ad..99133c816 100644
--- a/qemu/roms/ipxe/src/include/ipxe/errno/linux.h
+++ b/qemu/roms/ipxe/src/include/ipxe/errno/linux.h
@@ -10,7 +10,7 @@
  * directly as our platform error codes.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Convert platform error code to platform component of iPXE error code
diff --git a/qemu/roms/ipxe/src/include/ipxe/errortab.h b/qemu/roms/ipxe/src/include/ipxe/errortab.h
index a2f6a70f5..4fe81a6be 100644
--- a/qemu/roms/ipxe/src/include/ipxe/errortab.h
+++ b/qemu/roms/ipxe/src/include/ipxe/errortab.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/eth_slow.h b/qemu/roms/ipxe/src/include/ipxe/eth_slow.h
index 00509197d..f6d731b3b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/eth_slow.h
+++ b/qemu/roms/ipxe/src/include/ipxe/eth_slow.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Slow protocols header */
 struct eth_slow_header {
diff --git a/qemu/roms/ipxe/src/include/ipxe/ethernet.h b/qemu/roms/ipxe/src/include/ipxe/ethernet.h
index d1263d7c3..dd04e00ce 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ethernet.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ethernet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/netdevice.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fakedhcp.h b/qemu/roms/ipxe/src/include/ipxe/fakedhcp.h
index ea06b06dc..d016b5237 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fakedhcp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fakedhcp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/fault.h b/qemu/roms/ipxe/src/include/ipxe/fault.h
new file mode 100644
index 000000000..356296c35
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/fault.h
@@ -0,0 +1,53 @@
+#ifndef _IPXE_FAULT_H
+#define _IPXE_FAULT_H
+
+/** @file
+ *
+ * Fault injection
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <config/fault.h>
+
+extern int inject_fault_nonzero ( unsigned int rate );
+extern void inject_corruption_nonzero ( unsigned int rate, const void *data,
+					size_t len );
+
+/**
+ * Inject fault with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (zero for no faults)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+inject_fault ( unsigned int rate ) {
+
+	/* Force dead code elimination in non-fault-injecting builds */
+	if ( rate == 0 )
+		return 0;
+
+	return inject_fault_nonzero ( rate );
+}
+
+/**
+ * Corrupt data with a specified probability
+ *
+ * @v rate		Reciprocal of fault probability (zero for no faults)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) void
+inject_corruption ( unsigned int rate, const void *data, size_t len ) {
+
+	/* Force dead code elimination in non-fault-injecting builds */
+	if ( rate == 0 )
+		return;
+
+	return inject_corruption_nonzero ( rate, data, len );
+}
+
+#endif /* _IPXE_FAULT_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/fbcon.h b/qemu/roms/ipxe/src/include/ipxe/fbcon.h
index 0538449ac..d442bb918 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fbcon.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fbcon.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ansiesc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fc.h b/qemu/roms/ipxe/src/include/ipxe/fc.h
index 6fdef092d..840d11f62 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fc.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fcels.h b/qemu/roms/ipxe/src/include/ipxe/fcels.h
index 45fa69a4a..02f755115 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fcels.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fcels.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fcns.h b/qemu/roms/ipxe/src/include/ipxe/fcns.h
index e25d9b9d5..9011a7be7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fcns.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fcns.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fcoe.h b/qemu/roms/ipxe/src/include/ipxe/fcoe.h
index 6ba5b406a..b61e82fea 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fcoe.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fcoe.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/fcp.h b/qemu/roms/ipxe/src/include/ipxe/fcp.h
index f6922bc7c..853ca13f6 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fcp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fcp.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/fc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/features.h b/qemu/roms/ipxe/src/include/ipxe/features.h
index d8b8b2184..e86a2d226 100644
--- a/qemu/roms/ipxe/src/include/ipxe/features.h
+++ b/qemu/roms/ipxe/src/include/ipxe/features.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @defgroup featurecat Feature categories
diff --git a/qemu/roms/ipxe/src/include/ipxe/fragment.h b/qemu/roms/ipxe/src/include/ipxe/fragment.h
index e311ad1e4..0069e5e08 100644
--- a/qemu/roms/ipxe/src/include/ipxe/fragment.h
+++ b/qemu/roms/ipxe/src/include/ipxe/fragment.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ftp.h b/qemu/roms/ipxe/src/include/ipxe/ftp.h
index cbab12d2c..3180f1631 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ftp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ftp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** FTP default port */
 #define FTP_PORT 21
diff --git a/qemu/roms/ipxe/src/include/ipxe/gdbserial.h b/qemu/roms/ipxe/src/include/ipxe/gdbserial.h
index a3b56173c..e1040c94e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/gdbserial.h
+++ b/qemu/roms/ipxe/src/include/ipxe/gdbserial.h
@@ -7,15 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
 
 struct gdb_transport;
 
-/**
- * Set up the serial transport
- *
- * @ret transport suitable for starting the GDB stub or NULL on error
- */
-struct gdb_transport *gdbserial_configure ( void );
+extern struct gdb_transport * gdbserial_configure ( unsigned int port,
+						    unsigned int baud,
+						    uint8_t lcr );
 
 #endif /* _IPXE_GDBSERIAL_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/gdbstub.h b/qemu/roms/ipxe/src/include/ipxe/gdbstub.h
index 319606747..13ca33ddb 100644
--- a/qemu/roms/ipxe/src/include/ipxe/gdbstub.h
+++ b/qemu/roms/ipxe/src/include/ipxe/gdbstub.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/gdbudp.h b/qemu/roms/ipxe/src/include/ipxe/gdbudp.h
index db7a451c9..a1c091522 100644
--- a/qemu/roms/ipxe/src/include/ipxe/gdbudp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/gdbudp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct sockaddr_in;
 struct gdb_transport;
diff --git a/qemu/roms/ipxe/src/include/ipxe/hash_df.h b/qemu/roms/ipxe/src/include/ipxe/hash_df.h
index 607a4a610..e57682446 100644
--- a/qemu/roms/ipxe/src/include/ipxe/hash_df.h
+++ b/qemu/roms/ipxe/src/include/ipxe/hash_df.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/hidemem.h b/qemu/roms/ipxe/src/include/ipxe/hidemem.h
index ddc9cd8b3..cc8d5ee37 100644
--- a/qemu/roms/ipxe/src/include/ipxe/hidemem.h
+++ b/qemu/roms/ipxe/src/include/ipxe/hidemem.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/hmac.h b/qemu/roms/ipxe/src/include/ipxe/hmac.h
index d5ec0868d..09d3e273d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/hmac.h
+++ b/qemu/roms/ipxe/src/include/ipxe/hmac.h
@@ -6,7 +6,7 @@
  * Keyed-Hashing for Message Authentication
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/crypto.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/hmac_drbg.h b/qemu/roms/ipxe/src/include/ipxe/hmac_drbg.h
index 8dfd2924f..a0f22da75 100644
--- a/qemu/roms/ipxe/src/include/ipxe/hmac_drbg.h
+++ b/qemu/roms/ipxe/src/include/ipxe/hmac_drbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/http.h b/qemu/roms/ipxe/src/include/ipxe/http.h
index cf8c0c7fa..a0dff7d00 100644
--- a/qemu/roms/ipxe/src/include/ipxe/http.h
+++ b/qemu/roms/ipxe/src/include/ipxe/http.h
@@ -7,7 +7,26 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/process.h>
+#include <ipxe/retry.h>
+#include <ipxe/linebuf.h>
+#include <ipxe/pool.h>
+#include <ipxe/tables.h>
+
+struct http_transaction;
+
+/******************************************************************************
+ *
+ * HTTP URI schemes
+ *
+ ******************************************************************************
+ */
 
 /** HTTP default port */
 #define HTTP_PORT 80
@@ -15,10 +34,469 @@ FILE_LICENCE ( GPL2_OR_LATER );
 /** HTTPS default port */
 #define HTTPS_PORT 443
 
-extern int http_open_filter ( struct interface *xfer, struct uri *uri,
-			      unsigned int default_port,
-			      int ( * filter ) ( struct interface *,
-						 const char *,
-						 struct interface ** ) );
+/** An HTTP URI scheme */
+struct http_scheme {
+	/** Scheme name (e.g. "http" or "https") */
+	const char *name;
+	/** Default port */
+	unsigned int port;
+	/** Transport-layer filter (if any)
+	 *
+	 * @v xfer		Data transfer interface
+	 * @v name		Host name
+	 * @v next		Next interface
+	 * @ret rc		Return status code
+	 */
+	int ( * filter ) ( struct interface *xfer, const char *name,
+			   struct interface **next );
+};
+
+/** HTTP scheme table */
+#define HTTP_SCHEMES __table ( struct http_scheme, "http_schemes" )
+
+/** Declare an HTTP scheme */
+#define __http_scheme __table_entry ( HTTP_SCHEMES, 01 )
+
+/******************************************************************************
+ *
+ * Connections
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP connection
+ *
+ * This represents a potentially reusable connection to an HTTP
+ * server.
+ */
+struct http_connection {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Connection URI
+	 *
+	 * This encapsulates the server (and protocol) used for the
+	 * connection.  This may be the origin server or a proxy
+	 * server.
+	 */
+	struct uri *uri;
+	/** HTTP scheme */
+	struct http_scheme *scheme;
+	/** Transport layer interface */
+	struct interface socket;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Pooled connection */
+	struct pooled_connection pool;
+};
+
+/******************************************************************************
+ *
+ * HTTP methods
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP method */
+struct http_method {
+	/** Method name (e.g. "GET" or "POST") */
+	const char *name;
+};
+
+extern struct http_method http_head;
+extern struct http_method http_get;
+extern struct http_method http_post;
+
+/******************************************************************************
+ *
+ * Requests
+ *
+ ******************************************************************************
+ */
+
+/** HTTP Digest authentication client nonce count
+ *
+ * We choose to generate a new client nonce each time.
+ */
+#define HTTP_DIGEST_NC "00000001"
+
+/** HTTP Digest authentication client nonce length
+ *
+ * We choose to use a 32-bit hex client nonce.
+ */
+#define HTTP_DIGEST_CNONCE_LEN 8
+
+/** HTTP Digest authentication response length
+ *
+ * The Digest authentication response is a Base16-encoded 16-byte MD5
+ * checksum.
+ */
+#define HTTP_DIGEST_RESPONSE_LEN 32
+
+/** HTTP request range descriptor */
+struct http_request_range {
+	/** Range start */
+	size_t start;
+	/** Range length, or zero for no range request */
+	size_t len;
+};
+
+/** HTTP request content descriptor */
+struct http_request_content {
+	/** Content type (if any) */
+	const char *type;
+	/** Content data (if any) */
+	const void *data;
+	/** Content length */
+	size_t len;
+};
+
+/** HTTP request authentication descriptor */
+struct http_request_auth {
+	/** Authentication scheme (if any) */
+	struct http_authentication *auth;
+	/** Username */
+	const char *username;
+	/** Password */
+	const char *password;
+	/** Quality of protection */
+	const char *qop;
+	/** Algorithm */
+	const char *algorithm;
+	/** Client nonce */
+	char cnonce[ HTTP_DIGEST_CNONCE_LEN + 1 /* NUL */ ];
+	/** Response */
+	char response[ HTTP_DIGEST_RESPONSE_LEN + 1 /* NUL */ ];
+};
+
+/** An HTTP request
+ *
+ * This represents a single request to be sent to a server, including
+ * the values required to construct all headers.
+ *
+ * Pointers within this structure must point to storage which is
+ * guaranteed to remain valid for the lifetime of the containing HTTP
+ * transaction.
+ */
+struct http_request {
+	/** Method */
+	struct http_method *method;
+	/** Request URI string */
+	const char *uri;
+	/** Server host name */
+	const char *host;
+	/** Range descriptor */
+	struct http_request_range range;
+	/** Content descriptor */
+	struct http_request_content content;
+	/** Authentication descriptor */
+	struct http_request_auth auth;
+};
+
+/** An HTTP request header */
+struct http_request_header {
+	/** Header name (e.g. "User-Agent") */
+	const char *name;
+	/** Construct remaining header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v buf		Buffer
+	 * @v len		Length of buffer
+	 * @ret len		Header length if present, or negative error
+	 */
+	int ( * format ) ( struct http_transaction *http, char *buf,
+			   size_t len );
+};
+
+/** HTTP request header table */
+#define HTTP_REQUEST_HEADERS \
+	__table ( struct http_request_header, "http_request_headers" )
+
+/** Declare an HTTP request header */
+#define __http_request_header __table_entry ( HTTP_REQUEST_HEADERS, 01 )
+
+/******************************************************************************
+ *
+ * Responses
+ *
+ ******************************************************************************
+ */
+
+/** HTTP response transfer descriptor */
+struct http_response_transfer {
+	/** Transfer encoding */
+	struct http_transfer_encoding *encoding;
+};
+
+/** HTTP response content descriptor */
+struct http_response_content {
+	/** Content length (may be zero) */
+	size_t len;
+	/** Content encoding */
+	struct http_content_encoding *encoding;
+};
+
+/** HTTP response authorization descriptor */
+struct http_response_auth {
+	/** Authentication scheme (if any) */
+	struct http_authentication *auth;
+	/** Realm */
+	const char *realm;
+	/** Quality of protection */
+	const char *qop;
+	/** Algorithm */
+	const char *algorithm;
+	/** Nonce */
+	const char *nonce;
+	/** Opaque */
+	const char *opaque;
+};
+
+/** An HTTP response
+ *
+ * This represents a single response received from the server,
+ * including all values parsed from headers.
+ *
+ * Pointers within this structure may point into the raw response
+ * buffer, and so should be invalidated when the response buffer is
+ * modified or discarded.
+ */
+struct http_response {
+	/** Raw response header lines
+	 *
+	 * This is the raw response data received from the server, up
+	 * to and including the terminating empty line.  String
+	 * pointers within the response may point into this data
+	 * buffer; NUL terminators will be added (overwriting the
+	 * original terminating characters) as needed.
+	 */
+	struct line_buffer headers;
+	/** Status code
+	 *
+	 * This is the raw HTTP numeric status code (e.g. 404).
+	 */
+	unsigned int status;
+	/** Return status code
+	 *
+	 * This is the iPXE return status code corresponding to the
+	 * HTTP status code (e.g. -ENOENT).
+	 */
+	int rc;
+	/** Redirection location */
+	const char *location;
+	/** Transfer descriptor */
+	struct http_response_transfer transfer;
+	/** Content descriptor */
+	struct http_response_content content;
+	/** Authorization descriptor */
+	struct http_response_auth auth;
+	/** Retry delay (in seconds) */
+	unsigned int retry_after;
+	/** Flags */
+	unsigned int flags;
+};
+
+/** HTTP response flags */
+enum http_response_flags {
+	/** Keep connection alive after close */
+	HTTP_RESPONSE_KEEPALIVE = 0x0001,
+	/** Content length specified */
+	HTTP_RESPONSE_CONTENT_LEN = 0x0002,
+	/** Transaction may be retried on failure */
+	HTTP_RESPONSE_RETRY = 0x0004,
+};
+
+/** An HTTP response header */
+struct http_response_header {
+	/** Header name (e.g. "Transfer-Encoding") */
+	const char *name;
+	/** Parse header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v line		Remaining header line
+	 * @ret rc		Return status code
+	 */
+	int ( * parse ) ( struct http_transaction *http, char *line );
+};
+
+/** HTTP response header table */
+#define HTTP_RESPONSE_HEADERS \
+	__table ( struct http_response_header, "http_response_headers" )
+
+/** Declare an HTTP response header */
+#define __http_response_header __table_entry ( HTTP_RESPONSE_HEADERS, 01 )
+
+/******************************************************************************
+ *
+ * Transactions
+ *
+ ******************************************************************************
+ */
+
+/** HTTP transaction state */
+struct http_state {
+	/** Transmit data
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * tx ) ( struct http_transaction *http );
+	/** Receive data
+	 *
+	 * @v http		HTTP transaction
+	 * @v iobuf		I/O buffer (may be claimed)
+	 * @ret rc		Return status code
+	 */
+	int ( * rx ) ( struct http_transaction *http,
+		       struct io_buffer **iobuf );
+	/** Server connection closed
+	 *
+	 * @v http		HTTP transaction
+	 * @v rc		Reason for close
+	 */
+	void ( * close ) ( struct http_transaction *http, int rc );
+};
+
+/** An HTTP transaction */
+struct http_transaction {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Content-decoded interface */
+	struct interface content;
+	/** Transfer-decoded interface */
+	struct interface transfer;
+	/** Server connection */
+	struct interface conn;
+	/** Transmit process */
+	struct process process;
+	/** Reconnection timer */
+	struct retry_timer timer;
+
+	/** Request URI */
+	struct uri *uri;
+	/** Request */
+	struct http_request request;
+	/** Response */
+	struct http_response response;
+	/** Temporary line buffer */
+	struct line_buffer linebuf;
+
+	/** Transaction state */
+	struct http_state *state;
+	/** Accumulated transfer-decoded length */
+	size_t len;
+	/** Chunk length remaining */
+	size_t remaining;
+};
+
+/******************************************************************************
+ *
+ * Transfer encoding
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP transfer encoding */
+struct http_transfer_encoding {
+	/** Name */
+	const char *name;
+	/** Initialise transfer encoding
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * init ) ( struct http_transaction *http );
+	/** Receive data state */
+	struct http_state state;
+};
+
+/** HTTP transfer encoding table */
+#define HTTP_TRANSFER_ENCODINGS \
+	__table ( struct http_transfer_encoding, "http_transfer_encodings" )
+
+/** Declare an HTTP transfer encoding */
+#define __http_transfer_encoding __table_entry ( HTTP_TRANSFER_ENCODINGS, 01 )
+
+/******************************************************************************
+ *
+ * Content encoding
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP content encoding */
+struct http_content_encoding {
+	/** Name */
+	const char *name;
+	/** Check if content encoding is supported for this request
+	 *
+	 * @v http		HTTP transaction
+	 * @ret supported	Content encoding is supported for this request
+	 */
+	int ( * supported ) ( struct http_transaction *http );
+	/** Initialise content encoding
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * init ) ( struct http_transaction *http );
+};
+
+/** HTTP content encoding table */
+#define HTTP_CONTENT_ENCODINGS \
+	__table ( struct http_content_encoding, "http_content_encodings" )
+
+/** Declare an HTTP content encoding */
+#define __http_content_encoding __table_entry ( HTTP_CONTENT_ENCODINGS, 01 )
+
+/******************************************************************************
+ *
+ * Authentication
+ *
+ ******************************************************************************
+ */
+
+/** An HTTP authentication scheme */
+struct http_authentication {
+	/** Name (e.g. "Digest") */
+	const char *name;
+	/** Perform authentication
+	 *
+	 * @v http		HTTP transaction
+	 * @ret rc		Return status code
+	 */
+	int ( * authenticate ) ( struct http_transaction *http );
+	/** Construct remaining "Authorization" header line
+	 *
+	 * @v http		HTTP transaction
+	 * @v buf		Buffer
+	 * @v len		Length of buffer
+	 * @ret len		Header length if present, or negative error
+	 */
+	int ( * format ) ( struct http_transaction *http, char *buf,
+			   size_t len );
+};
+
+/** HTTP authentication scheme table */
+#define HTTP_AUTHENTICATIONS \
+	__table ( struct http_authentication, "http_authentications" )
+
+/** Declare an HTTP authentication scheme */
+#define __http_authentication __table_entry ( HTTP_AUTHENTICATIONS, 01 )
+
+/******************************************************************************
+ *
+ * General
+ *
+ ******************************************************************************
+ */
+
+extern char * http_token ( char **line, char **value );
+extern int http_connect ( struct interface *xfer, struct uri *uri );
+extern int http_open ( struct interface *xfer, struct http_method *method,
+		       struct uri *uri, struct http_request_range *range,
+		       struct http_request_content *content );
+extern int http_open_uri ( struct interface *xfer, struct uri *uri );
 
 #endif /* _IPXE_HTTP_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/hyperv.h b/qemu/roms/ipxe/src/include/ipxe/hyperv.h
new file mode 100644
index 000000000..c61e2a083
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/hyperv.h
@@ -0,0 +1,232 @@
+#ifndef _IPXE_HYPERV_H
+#define _IPXE_HYPERV_H
+
+/** @file
+ *
+ * Hyper-V interface
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/io.h>
+
+/** Hyper-V interface identification */
+#define HV_INTERFACE_ID 0x31237648 /* "Hv#1" */
+
+/** Guest OS identity for iPXE
+ *
+ * This field comprises:
+ *
+ * Bit  63    : set to 1 to indicate an open source OS
+ * Bits 62:56 : OS Type
+ * Bits 55:48 : OS ID
+ * Bits 47:16 : Version
+ * Bits 15:0  : Build number
+ *
+ * There appears to be no central registry for the "OS Type".  The
+ * specification states that "Linux is 0x100", and the FreeBSD source
+ * states that "FreeBSD is 0x200".  Both of these statements are
+ * actually referring to the combined "OS Type" and "OS ID" field.
+ *
+ * We choose to use 0x98ae: this is generated by setting bit 63 (to
+ * indicate an open source OS) and setting the OS Type+ID equal to the
+ * PnP vendor ID used in romprefix.S.  No version information or build
+ * number is included.
+ */
+#define HV_GUEST_OS_ID_IPXE ( ( 1ULL << 63 ) | ( 0x18aeULL << 48 ) )
+
+/** Enable hypercall page */
+#define HV_HYPERCALL_ENABLE 0x00000001UL
+
+/** Enable SynIC */
+#define HV_SCONTROL_ENABLE 0x00000001UL
+
+/** Enable SynIC event flags */
+#define HV_SIEFP_ENABLE 0x00000001UL
+
+/** Enable SynIC messages */
+#define HV_SIMP_ENABLE 0x00000001UL
+
+/** Perform implicit EOI upon synthetic interrupt delivery */
+#define HV_SINT_AUTO_EOI 0x00020000UL
+
+/** Mask synthetic interrupt */
+#define HV_SINT_MASKED 0x00010000UL
+
+/** Synthetic interrupt vector */
+#define HV_SINT_VECTOR(x) ( (x) << 0 )
+
+/** Synthetic interrupt vector mask */
+#define HV_SINT_VECTOR_MASK HV_SINT_VECTOR ( 0xff )
+
+/** Post message */
+#define HV_POST_MESSAGE 0x005c
+
+/** A posted message
+ *
+ * This is the input parameter list for the HvPostMessage hypercall.
+ */
+struct hv_post_message {
+	/** Connection ID */
+	uint32_t id;
+	/** Padding */
+	uint32_t reserved;
+	/** Type */
+	uint32_t type;
+	/** Length of message */
+	uint32_t len;
+	/** Message */
+	uint8_t data[240];
+} __attribute__ (( packed ));
+
+/** A received message
+ *
+ * This is the HV_MESSAGE structure from the Hypervisor Top-Level
+ * Functional Specification.  The field order given in the
+ * documentation is incorrect.
+ */
+struct hv_message {
+	/** Type */
+	uint32_t type;
+	/** Length of message */
+	uint8_t len;
+	/** Flags */
+	uint8_t flags;
+	/** Padding */
+	uint16_t reserved;
+	/** Origin */
+	uint64_t origin;
+	/** Message */
+	uint8_t data[240];
+} __attribute__ (( packed ));
+
+/** Signal event */
+#define HV_SIGNAL_EVENT 0x005d
+
+/** A signalled event */
+struct hv_signal_event {
+	/** Connection ID */
+	uint32_t id;
+	/** Flag number */
+	uint16_t flag;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A received event */
+struct hv_event {
+	/** Event flags */
+	uint8_t flags[256];
+} __attribute__ (( packed ));
+
+/** A monitor trigger group
+ *
+ * This is the HV_MONITOR_TRIGGER_GROUP structure from the Hypervisor
+ * Top-Level Functional Specification.
+ */
+struct hv_monitor_trigger {
+	/** Pending events */
+	uint32_t pending;
+	/** Armed events */
+	uint32_t armed;
+} __attribute__ (( packed ));
+
+/** A monitor parameter set
+ *
+ * This is the HV_MONITOR_PARAMETER structure from the Hypervisor
+ * Top-Level Functional Specification.
+ */
+struct hv_monitor_parameter {
+	/** Connection ID */
+	uint32_t id;
+	/** Flag number */
+	uint16_t flag;
+	/** Reserved */
+	uint16_t reserved;
+} __attribute__ (( packed ));
+
+/** A monitor page
+ *
+ * This is the HV_MONITOR_PAGE structure from the Hypervisor Top-Level
+ * Functional Specification.
+ */
+struct hv_monitor {
+	/** Flags */
+	uint32_t flags;
+	/** Reserved */
+	uint8_t reserved_a[4];
+	/** Trigger groups */
+	struct hv_monitor_trigger trigger[4];
+	/** Reserved */
+	uint8_t reserved_b[536];
+	/** Latencies */
+	uint16 latency[4][32];
+	/** Reserved */
+	uint8_t reserved_c[256];
+	/** Parameters */
+	struct hv_monitor_parameter param[4][32];
+	/** Reserved */
+	uint8_t reserved_d[1984];
+} __attribute__ (( packed ));
+
+/** A synthetic interrupt controller */
+struct hv_synic {
+	/** Message page */
+	struct hv_message *message;
+	/** Event flag page */
+	struct hv_event *event;
+};
+
+/** A message buffer */
+union hv_message_buffer {
+	/** Posted message */
+	struct hv_post_message posted;
+	/** Received message */
+	struct hv_message received;
+	/** Signalled event */
+	struct hv_signal_event signalled;
+};
+
+/** A Hyper-V hypervisor */
+struct hv_hypervisor {
+	/** Hypercall page */
+	void *hypercall;
+	/** Synthetic interrupt controller (SynIC) */
+	struct hv_synic synic;
+	/** Message buffer */
+	union hv_message_buffer *message;
+	/** Virtual machine bus */
+	struct vmbus *vmbus;
+};
+
+#include <bits/hyperv.h>
+
+/**
+ * Calculate the number of pages covering an address range
+ *
+ * @v data		Start of data
+ * @v len		Length of data (must be non-zero)
+ * @ret pfn_count	Number of pages covered
+ */
+static inline unsigned int hv_pfn_count ( physaddr_t data, size_t len ) {
+	unsigned int first_pfn = ( data / PAGE_SIZE );
+	unsigned int last_pfn = ( ( data + len - 1 ) / PAGE_SIZE );
+
+	return ( last_pfn - first_pfn + 1 );
+}
+
+extern __attribute__ (( sentinel )) int
+hv_alloc_pages ( struct hv_hypervisor *hv, ... );
+extern __attribute__ (( sentinel )) void
+hv_free_pages ( struct hv_hypervisor *hv, ... );
+extern void hv_enable_sint ( struct hv_hypervisor *hv, unsigned int sintx );
+extern void hv_disable_sint ( struct hv_hypervisor *hv, unsigned int sintx );
+extern int hv_post_message ( struct hv_hypervisor *hv, unsigned int id,
+			     unsigned int type, const void *data, size_t len );
+extern int hv_wait_for_message ( struct hv_hypervisor *hv, unsigned int sintx );
+extern int hv_signal_event ( struct hv_hypervisor *hv, unsigned int id,
+			     unsigned int flag );
+
+#endif /* _IPXE_HYPERV_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/i2c.h b/qemu/roms/ipxe/src/include/ipxe/i2c.h
index c1f5a9bbd..46970515c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/i2c.h
+++ b/qemu/roms/ipxe/src/include/ipxe/i2c.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/bitbash.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_cm.h b/qemu/roms/ipxe/src/include/ipxe/ib_cm.h
index 7d08cd9b1..4913eebae 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_cm.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_cm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 #include <ipxe/retry.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_mad.h b/qemu/roms/ipxe/src/include/ipxe/ib_mad.h
index b8694833e..ae1eea7e4 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_mad.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_mad.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ib_packet.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_mcast.h b/qemu/roms/ipxe/src/include/ipxe/ib_mcast.h
index a5c22a03e..564066975 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_mcast.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_mcast.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_mi.h b/qemu/roms/ipxe/src/include/ipxe/ib_mi.h
index 5c5415b71..c7c8143ba 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_mi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_mi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/retry.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_packet.h b/qemu/roms/ipxe/src/include/ipxe/ib_packet.h
index a959967cb..f275fcb09 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_packet.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_packet.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ib_device;
 struct ib_queue_pair;
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_pathrec.h b/qemu/roms/ipxe/src/include/ipxe/ib_pathrec.h
index 1fe67f87d..a4e11ebe3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_pathrec.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_pathrec.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_sma.h b/qemu/roms/ipxe/src/include/ipxe/ib_sma.h
index fa355c652..74003d045 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_sma.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_sma.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct ib_device;
 struct ib_mad_interface;
diff --git a/qemu/roms/ipxe/src/include/ipxe/ib_smc.h b/qemu/roms/ipxe/src/include/ipxe/ib_smc.h
index 259d2cde1..f9b96b1bd 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ib_smc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ib_smc.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/infiniband.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/icmp.h b/qemu/roms/ipxe/src/include/ipxe/icmp.h
index 0480ddfaf..803f8e019 100644
--- a/qemu/roms/ipxe/src/include/ipxe/icmp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/icmp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iobuf.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/icmpv6.h b/qemu/roms/ipxe/src/include/ipxe/icmpv6.h
index b5ea54eab..0474ddca8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/icmpv6.h
+++ b/qemu/roms/ipxe/src/include/ipxe/icmpv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
@@ -40,6 +40,18 @@ struct icmpv6_handler {
 /** Declare an ICMPv6 handler */
 #define __icmpv6_handler __table_entry ( ICMPV6_HANDLERS, 01 )
 
+/** ICMPv6 destination unreachable */
+#define ICMPV6_DESTINATION_UNREACHABLE 1
+
+/** ICMPv6 packet too big */
+#define ICMPV6_PACKET_TOO_BIG 2
+
+/** ICMPv6 time exceeded */
+#define ICMPV6_TIME_EXCEEDED 3
+
+/** ICMPv6 parameter problem */
+#define ICMPV6_PARAMETER_PROBLEM 4
+
 /** ICMPv6 echo request */
 #define ICMPV6_ECHO_REQUEST 128
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/if_arp.h b/qemu/roms/ipxe/src/include/ipxe/if_arp.h
index fd36e9c67..4eb1f80b7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/if_arp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/if_arp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/if_ether.h b/qemu/roms/ipxe/src/include/ipxe/if_ether.h
index a7e237349..58d91b976 100644
--- a/qemu/roms/ipxe/src/include/ipxe/if_ether.h
+++ b/qemu/roms/ipxe/src/include/ipxe/if_ether.h
@@ -1,7 +1,7 @@
 #ifndef	_IPXE_IF_ETHER_H
 #define	_IPXE_IF_ETHER_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/image.h b/qemu/roms/ipxe/src/include/ipxe/image.h
index 5d7080a75..6abd7a2d2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/image.h
+++ b/qemu/roms/ipxe/src/include/ipxe/image.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 #include <ipxe/list.h>
@@ -163,7 +163,6 @@ extern int image_set_cmdline ( struct image *image, const char *cmdline );
 extern int register_image ( struct image *image );
 extern void unregister_image ( struct image *image );
 struct image * find_image ( const char *name );
-extern int image_probe ( struct image *image );
 extern int image_exec ( struct image *image );
 extern int image_replace ( struct image *replacement );
 extern int image_select ( struct image *image );
diff --git a/qemu/roms/ipxe/src/include/ipxe/in.h b/qemu/roms/ipxe/src/include/ipxe/in.h
index de96ca22a..0ebf441c2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/in.h
+++ b/qemu/roms/ipxe/src/include/ipxe/in.h
@@ -1,9 +1,10 @@
 #ifndef	_IPXE_IN_H
 #define	_IPXE_IN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
+#include <byteswap.h>
 #include <ipxe/socket.h>
 
 /* Protocol numbers */
@@ -15,17 +16,22 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 /* IP address constants */
 
-#define INADDR_NONE 0xffffffff
+#define INADDR_NONE		htonl ( 0xffffffff )
 
-#define INADDR_BROADCAST 0xffffffff
+#define INADDR_BROADCAST	htonl ( 0xffffffff )
 
-#define	IN_CLASSA(addr)		( ( (addr) & 0x80000000 ) == 0x00000000 )
-#define	IN_CLASSA_NET		0xff000000
-#define	IN_CLASSB(addr)		( ( (addr) & 0xc0000000 ) == 0x80000000 )
-#define	IN_CLASSB_NET		0xffff0000
-#define	IN_CLASSC(addr)		( ( (addr) & 0xe0000000 ) == 0xc0000000 )
-#define	IN_CLASSC_NET		0xffffff00
-#define IN_MULTICAST(addr)	( ( (addr) & 0xf0000000 ) == 0xe0000000 )
+#define	INADDR_NET_CLASSA	htonl ( 0xff000000 )
+#define	INADDR_NET_CLASSB	htonl ( 0xffff0000 )
+#define	INADDR_NET_CLASSC	htonl ( 0xffffff00 )
+
+#define	IN_IS_CLASSA( addr ) \
+	( ( (addr) & htonl ( 0x80000000 ) ) == htonl ( 0x00000000 ) )
+#define	IN_IS_CLASSB( addr ) \
+	( ( (addr) & htonl ( 0xc0000000 ) ) == htonl ( 0x80000000 ) )
+#define	IN_IS_CLASSC( addr ) \
+	( ( (addr) & htonl ( 0xe0000000 ) ) == htonl ( 0xc0000000 ) )
+#define IN_IS_MULTICAST( addr ) \
+	( ( (addr) & htonl ( 0xf0000000 ) ) == htonl ( 0xe0000000 ) )
 
 /**
  * IP address structure
@@ -63,6 +69,9 @@ struct in6_addr {
 	( ( *( ( const uint16_t * ) (addr) ) & htons ( 0xffc0 ) ) ==	\
 	  htons ( 0xfe80 ) )
 
+#define IN6_IS_ADDR_NONGLOBAL( addr )					\
+	( IN6_IS_ADDR_LINKLOCAL (addr) || IN6_IS_ADDR_MULTICAST (addr) )
+
 /**
  * IPv4 socket address
  */
@@ -76,6 +85,11 @@ struct sockaddr_in {
 	uint16_t sin_flags;
 	/** TCP/IP port (part of struct @c sockaddr_tcpip) */
 	uint16_t sin_port;
+	/** Scope ID (part of struct @c sockaddr_tcpip)
+	 *
+	 * For multicast addresses, this is the network device index.
+	 */
+        uint16_t sin_scope_id;
 	/** IPv4 address */
 	struct in_addr sin_addr;
 	/** Padding
@@ -87,6 +101,7 @@ struct sockaddr_in {
 		  ( sizeof ( sa_family_t ) /* sin_family */ +
 		    sizeof ( uint16_t ) /* sin_flags */ +
 		    sizeof ( uint16_t ) /* sin_port */ +
+		    sizeof ( uint16_t ) /* sin_scope_id */ +
 		    sizeof ( struct in_addr ) /* sin_addr */ ) ];
 } __attribute__ (( packed, may_alias ));
 
@@ -103,9 +118,10 @@ struct sockaddr_in6 {
 	uint16_t sin6_flags;
 	/** TCP/IP port (part of struct @c sockaddr_tcpip) */
 	uint16_t sin6_port;
-	/** Scope ID
+	/** Scope ID (part of struct @c sockaddr_tcpip)
 	 *
-	 * For link-local addresses, this is the network device index.
+	 * For link-local or multicast addresses, this is the network
+	 * device index.
 	 */
         uint16_t sin6_scope_id;
 	/** IPv6 address */
diff --git a/qemu/roms/ipxe/src/include/ipxe/infiniband.h b/qemu/roms/ipxe/src/include/ipxe/infiniband.h
index f546ea61b..87cfe5082 100644
--- a/qemu/roms/ipxe/src/include/ipxe/infiniband.h
+++ b/qemu/roms/ipxe/src/include/ipxe/infiniband.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/init.h b/qemu/roms/ipxe/src/include/ipxe/init.h
index 19c5925bf..025cfaf37 100644
--- a/qemu/roms/ipxe/src/include/ipxe/init.h
+++ b/qemu/roms/ipxe/src/include/ipxe/init.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_INIT_H
 #define _IPXE_INIT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
@@ -26,10 +26,9 @@ struct init_fn {
  */
 
 #define INIT_EARLY	01	/**< Early initialisation */
-#define INIT_SERIAL	02	/**< Serial driver initialisation */
-#define	INIT_CONSOLE	03	/**< Console initialisation */
-#define INIT_NORMAL	04	/**< Normal initialisation */
-#define INIT_LATE	05	/**< Late initialisation */
+#define	INIT_CONSOLE	02	/**< Console initialisation */
+#define INIT_NORMAL	03	/**< Normal initialisation */
+#define INIT_LATE	04	/**< Late initialisation */
 
 /** @} */
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/interface.h b/qemu/roms/ipxe/src/include/ipxe/interface.h
index a474aaad0..a8d823775 100644
--- a/qemu/roms/ipxe/src/include/ipxe/interface.h
+++ b/qemu/roms/ipxe/src/include/ipxe/interface.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/refcnt.h>
@@ -145,6 +145,11 @@ extern void intf_close ( struct interface *intf, int rc );
 extern void intf_shutdown ( struct interface *intf, int rc );
 extern void intf_restart ( struct interface *intf, int rc );
 
+extern void intf_poke ( struct interface *intf,
+			void ( type ) ( struct interface *intf ) );
+#define intf_poke_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
 extern struct interface_descriptor null_intf_desc;
 extern struct interface null_intf;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/io.h b/qemu/roms/ipxe/src/include/ipxe/io.h
index 29ccfd1fa..af767915d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/io.h
+++ b/qemu/roms/ipxe/src/include/ipxe/io.h
@@ -16,7 +16,7 @@
  * the address parameter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/iobuf.h b/qemu/roms/ipxe/src/include/ipxe/iobuf.h
index b2b0cb440..27d285d44 100644
--- a/qemu/roms/ipxe/src/include/ipxe/iobuf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/iobuf.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <assert.h>
@@ -217,5 +217,6 @@ extern void free_iob ( struct io_buffer *iobuf );
 extern void iob_pad ( struct io_buffer *iobuf, size_t min_len );
 extern int iob_ensure_headroom ( struct io_buffer *iobuf, size_t len );
 extern struct io_buffer * iob_concatenate ( struct list_head *list );
+extern struct io_buffer * iob_split ( struct io_buffer *iobuf, size_t len );
 
 #endif /* _IPXE_IOBUF_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/ip.h b/qemu/roms/ipxe/src/include/ipxe/ip.h
index 1a93a552e..285be6dcd 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ip.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ipoib.h b/qemu/roms/ipxe/src/include/ipxe/ipoib.h
index 68ff8df49..b34dd32d0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ipoib.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ipoib.h
@@ -6,7 +6,7 @@
  * IP over Infiniband
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/if_arp.h>
 #include <ipxe/infiniband.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ipstat.h b/qemu/roms/ipxe/src/include/ipxe/ipstat.h
index c554c1859..b34ed5fcf 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ipstat.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ipstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/ipv6.h b/qemu/roms/ipxe/src/include/ipxe/ipv6.h
index 48aaf677e..b500382c1 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ipv6.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ipv6.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/isa_ids.h b/qemu/roms/ipxe/src/include/ipxe/isa_ids.h
index 1faf1148d..d815bda34 100644
--- a/qemu/roms/ipxe/src/include/ipxe/isa_ids.h
+++ b/qemu/roms/ipxe/src/include/ipxe/isa_ids.h
@@ -19,7 +19,7 @@
  * the underlying "meaning" is big-endian.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <byteswap.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/isapnp.h b/qemu/roms/ipxe/src/include/ipxe/isapnp.h
index b0b0e98d6..59beac986 100644
--- a/qemu/roms/ipxe/src/include/ipxe/isapnp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/isapnp.h
@@ -17,6 +17,10 @@
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 *    02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
 *
 *    Portions of this code:
 *		Copyright (C) 2001  P.J.H.Fox (fox@roestock.demon.co.uk)
@@ -34,7 +38,7 @@
 *
 ***************************************************************************/
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef ISAPNP_H
 #define ISAPNP_H
diff --git a/qemu/roms/ipxe/src/include/ipxe/iscsi.h b/qemu/roms/ipxe/src/include/ipxe/iscsi.h
index be71360a0..c75ff4188 100644
--- a/qemu/roms/ipxe/src/include/ipxe/iscsi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/iscsi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/socket.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/iso9660.h b/qemu/roms/ipxe/src/include/ipxe/iso9660.h
index 02c2ae377..34cb8f0a1 100644
--- a/qemu/roms/ipxe/src/include/ipxe/iso9660.h
+++ b/qemu/roms/ipxe/src/include/ipxe/iso9660.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/isqrt.h b/qemu/roms/ipxe/src/include/ipxe/isqrt.h
index 58ed42f0c..68255d1bc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/isqrt.h
+++ b/qemu/roms/ipxe/src/include/ipxe/isqrt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern unsigned long isqrt ( unsigned long value );
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/job.h b/qemu/roms/ipxe/src/include/ipxe/job.h
index a2369f7c2..7e1bd8109 100644
--- a/qemu/roms/ipxe/src/include/ipxe/job.h
+++ b/qemu/roms/ipxe/src/include/ipxe/job.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/jumpscroll.h b/qemu/roms/ipxe/src/include/ipxe/jumpscroll.h
new file mode 100644
index 000000000..7a5b111c1
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/jumpscroll.h
@@ -0,0 +1,50 @@
+#ifndef _IPXE_JUMPSCROLL_H
+#define _IPXE_JUMPSCROLL_H
+
+/** @file
+ *
+ * Jump scrolling
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** A jump scroller */
+struct jump_scroller {
+	/** Maximum number of visible rows */
+	unsigned int rows;
+	/** Total number of items */
+	unsigned int count;
+	/** Currently selected item */
+	unsigned int current;
+	/** First visible item */
+	unsigned int first;
+};
+
+/**
+ * Check if jump scroller is currently on first page
+ *
+ * @v scroll		Jump scroller
+ * @ret is_first	Scroller is currently on first page
+ */
+static inline int jump_scroll_is_first ( struct jump_scroller *scroll ) {
+
+	return ( scroll->first == 0 );
+}
+
+/**
+ * Check if jump scroller is currently on last page
+ *
+ * @v scroll		Jump scroller
+ * @ret is_last		Scroller is currently on last page
+ */
+static inline int jump_scroll_is_last ( struct jump_scroller *scroll ) {
+
+	return ( ( scroll->first + scroll->rows ) >= scroll->count );
+}
+
+extern int jump_scroll_key ( struct jump_scroller *scroll, int key );
+extern int jump_scroll_move ( struct jump_scroller *scroll, int move );
+extern int jump_scroll ( struct jump_scroller *scroll );
+
+#endif /* _IPXE_JUMPSCROLL_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/keymap.h b/qemu/roms/ipxe/src/include/ipxe/keymap.h
index 9ac42a6b1..0f1b0c656 100644
--- a/qemu/roms/ipxe/src/include/ipxe/keymap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/keymap.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/keys.h b/qemu/roms/ipxe/src/include/ipxe/keys.h
index 8b13550b9..d15267a1f 100644
--- a/qemu/roms/ipxe/src/include/ipxe/keys.h
+++ b/qemu/roms/ipxe/src/include/ipxe/keys.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Symbolic names for some standard ASCII characters
@@ -58,6 +58,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 #define KEY_ANSI( n, terminator ) ( 0x100 * ( (n) + 1 ) + (terminator) )
+#define KEY_ANSI_N( key ) ( ( (key) / 0x100 ) - 1 )
+#define KEY_ANSI_TERMINATOR( key ) ( (key) & 0xff )
 
 #define KEY_MIN		0x101
 #define KEY_UP		KEY_ANSI ( 0, 'A' )	/**< Up arrow */
diff --git a/qemu/roms/ipxe/src/include/ipxe/linebuf.h b/qemu/roms/ipxe/src/include/ipxe/linebuf.h
index 706ef2554..630278a04 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linebuf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linebuf.h
@@ -7,24 +7,24 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
 
 /** A line buffer */
 struct line_buffer {
-	/** Current string in the buffer */
+	/** Data buffer */
 	char *data;
-	/** Length of current string, excluding the terminating NUL */
+	/** Length of buffered data */
 	size_t len;
-	/** String is ready to read */
-	int ready;
+	/** Most recently consumed length */
+	size_t consumed;
 };
 
 extern char * buffered_line ( struct line_buffer *linebuf );
-extern ssize_t line_buffer ( struct line_buffer *linebuf,
-			     const char *data, size_t len );
+extern int line_buffer ( struct line_buffer *linebuf,
+			 const char *data, size_t len );
 extern void empty_line_buffer ( struct line_buffer *linebuf );
 
 #endif /* _IPXE_LINEBUF_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/lineconsole.h b/qemu/roms/ipxe/src/include/ipxe/lineconsole.h
index 925c0accc..31117e73c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/lineconsole.h
+++ b/qemu/roms/ipxe/src/include/ipxe/lineconsole.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/ansiesc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_entropy.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
index bd89bd52f..afef6fe19 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_entropy.h
@@ -3,11 +3,11 @@
 
 /** @file
  *
- * iPXE entropy API for linux
+ * /dev/random-based entropy source
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef ENTROPY_LINUX
 #define ENTROPY_PREFIX_linux
@@ -23,10 +23,12 @@ FILE_LICENCE(GPL2_OR_LATER);
 static inline __always_inline double
 ENTROPY_INLINE ( linux, min_entropy_per_sample ) ( void ) {
 
-	/* We read single bytes from /dev/random and assume that each
-	 * contains full entropy.
+	/* linux_get_noise() reads a single byte from /dev/random,
+	 * which is supposed to block until a sufficient amount of
+	 * entropy is available.  We therefore assume that each sample
+	 * contains exactly 8 bits of entropy.
 	 */
-	return 8;
+	return 8.0;
 }
 
 #endif /* _IPXE_LINUX_ENTROPY_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_nap.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_nap.h
index 5bac7242f..d072886c7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_nap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_LINUX
 #define NAP_PREFIX_linux
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_pci.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_pci.h
index 439166733..22ae7f1bc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_pci.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_pci.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef PCIAPI_LINUX
 #define PCIAPI_PREFIX_linux
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_smbios.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
index 6d51e13ba..16c6d8acd 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_smbios.h
@@ -3,11 +3,11 @@
 
 /** @file
  *
- * iPXE SMBIOS API for linux
+ * iPXE SMBIOS API for Linux
  *
  */
 
-FILE_LICENCE(GPL2_OR_LATER);
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SMBIOS_LINUX
 #define SMBIOS_PREFIX_linux
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_time.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_time.h
index 93a257730..872ef5ade 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_time.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_LINUX
 #define TIME_PREFIX_linux
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_timer.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_timer.h
index 379507417..7f46e36b2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_timer.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_timer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIMER_LINUX
 #define TIMER_PREFIX_linux
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
index e4d16d9e0..acd919a85 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_uaccess.h
@@ -1,116 +1,108 @@
-/*
- * Copyright (C) 2010 Piotr Jaroszyński <p.jaroszynski@gmail.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
 #ifndef _IPXE_LINUX_UACCESS_H
 #define _IPXE_LINUX_UACCESS_H
 
-FILE_LICENCE(GPL2_OR_LATER);
-
 /** @file
  *
- * iPXE user access API for linux
+ * iPXE user access API for Linux
+ *
+ * We run with no distinction between internal and external addresses,
+ * so can use trivial_virt_to_user() et al.
  *
- * In linux userspace virtual == user == phys addresses.
- * Physical addresses also being the same is wrong, but there is no general way
- * of converting userspace addresses to physical as what appears to be
- * contiguous in userspace is physically fragmented.
- * Currently only the DMA memory is special-cased, but its conversion to bus
- * addresses is done in phys_to_bus.
- * This is known to break virtio as it is passing phys addresses to the virtual
- * device.
+ * We have no concept of the underlying physical addresses, since
+ * these are not exposed to userspace.  We provide a stub
+ * implementation of user_to_phys() since this is required by
+ * alloc_memblock().  We provide no implementation of phys_to_user();
+ * any code attempting to access physical addresses will therefore
+ * (correctly) fail to link.
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #ifdef UACCESS_LINUX
 #define UACCESS_PREFIX_linux
 #else
 #define UACCESS_PREFIX_linux __linux_
 #endif
 
-static inline __always_inline userptr_t
-UACCESS_INLINE(linux, phys_to_user)(unsigned long phys_addr)
-{
-	return phys_addr;
-}
-
+/**
+ * Convert user buffer to physical address
+ *
+ * @v userptr		User pointer
+ * @v offset		Offset from user pointer
+ * @ret phys_addr	Physical address
+ */
 static inline __always_inline unsigned long
-UACCESS_INLINE(linux, user_to_phys)(userptr_t userptr, off_t offset)
-{
-	return userptr + offset;
+UACCESS_INLINE ( linux, user_to_phys ) ( userptr_t userptr, off_t offset ) {
+
+	/* We do not know the real underlying physical address.  We
+	 * provide this stub implementation only because it is
+	 * required by alloc_memblock() (which allocates memory with
+	 * specified physical address alignment).  We assume that the
+	 * low-order bits of virtual addresses match the low-order
+	 * bits of physical addresses, and so simply returning the
+	 * virtual address will suffice for the purpose of determining
+	 * alignment.
+	 */
+	return ( userptr + offset );
 }
 
 static inline __always_inline userptr_t
-UACCESS_INLINE(linux, virt_to_user)(volatile const void *addr)
-{
-	return trivial_virt_to_user(addr);
+UACCESS_INLINE ( linux, virt_to_user ) ( volatile const void *addr ) {
+	return trivial_virt_to_user ( addr );
 }
 
 static inline __always_inline void *
-UACCESS_INLINE(linux, user_to_virt)(userptr_t userptr, off_t offset)
-{
-	return trivial_user_to_virt(userptr, offset);
+UACCESS_INLINE ( linux, user_to_virt ) ( userptr_t userptr, off_t offset ) {
+	return trivial_user_to_virt ( userptr, offset );
 }
 
 static inline __always_inline userptr_t
-UACCESS_INLINE(linux, userptr_add)(userptr_t userptr, off_t offset)
-{
-	return trivial_userptr_add(userptr, offset);
+UACCESS_INLINE ( linux, userptr_add ) ( userptr_t userptr, off_t offset ) {
+	return trivial_userptr_add ( userptr, offset );
 }
 
 static inline __always_inline off_t
-UACCESS_INLINE(linux, userptr_sub)(userptr_t userptr, userptr_t subtrahend)
-{
+UACCESS_INLINE ( linux, userptr_sub ) ( userptr_t userptr,
+					userptr_t subtrahend ) {
 	return trivial_userptr_sub ( userptr, subtrahend );
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memcpy_user)(userptr_t dest, off_t dest_off, userptr_t src, off_t src_off, size_t len)
-{
-	trivial_memcpy_user(dest, dest_off, src, src_off, len);
+UACCESS_INLINE ( linux, memcpy_user ) ( userptr_t dest, off_t dest_off,
+					userptr_t src, off_t src_off,
+					size_t len ) {
+	trivial_memcpy_user ( dest, dest_off, src, src_off, len );
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memmove_user)(userptr_t dest, off_t dest_off, userptr_t src, off_t src_off, size_t len)
-{
-	trivial_memmove_user(dest, dest_off, src, src_off, len);
+UACCESS_INLINE ( linux, memmove_user ) ( userptr_t dest, off_t dest_off,
+					 userptr_t src, off_t src_off,
+					 size_t len ) {
+	trivial_memmove_user ( dest, dest_off, src, src_off, len );
 }
 
 static inline __always_inline int
-UACCESS_INLINE(linux, memcmp_user)(userptr_t first, off_t first_off, userptr_t second, off_t second_off, size_t len)
-{
-	return trivial_memcmp_user(first, first_off, second, second_off, len);
+UACCESS_INLINE ( linux, memcmp_user ) ( userptr_t first, off_t first_off,
+					userptr_t second, off_t second_off,
+					size_t len ) {
+	return trivial_memcmp_user ( first, first_off, second, second_off, len);
 }
 
 static inline __always_inline void
-UACCESS_INLINE(linux, memset_user)(userptr_t buffer, off_t offset, int c, size_t len)
-{
-	trivial_memset_user(buffer, offset, c, len);
+UACCESS_INLINE ( linux, memset_user ) ( userptr_t buffer, off_t offset,
+					int c, size_t len ) {
+	trivial_memset_user ( buffer, offset, c, len );
 }
 
 static inline __always_inline size_t
-UACCESS_INLINE(linux, strlen_user)(userptr_t buffer, off_t offset)
-{
-	return trivial_strlen_user(buffer, offset);
+UACCESS_INLINE ( linux, strlen_user ) ( userptr_t buffer, off_t offset ) {
+	return trivial_strlen_user ( buffer, offset );
 }
 
 static inline __always_inline off_t
-UACCESS_INLINE(linux, memchr_user)(userptr_t buffer, off_t offset, int c, size_t len)
-{
-	return trivial_memchr_user(buffer, offset, c, len);
+UACCESS_INLINE ( linux, memchr_user ) ( userptr_t buffer, off_t offset,
+					int c, size_t len ) {
+	return trivial_memchr_user ( buffer, offset, c, len );
 }
 
 #endif /* _IPXE_LINUX_UACCESS_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h b/qemu/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
index 4de55ecf3..1811d0bc6 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux/linux_umalloc.h
@@ -1,14 +1,14 @@
 #ifndef _IPXE_LINUX_UMALLOC_H
 #define _IPXE_LINUX_UMALLOC_H
 
-FILE_LICENCE(GPL2_OR_LATER);
-
 /** @file
  *
- * iPXE user memory allocation API for linux
+ * iPXE user memory allocation API for Linux
  *
  */
 
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
 #ifdef UMALLOC_LINUX
 #define UMALLOC_PREFIX_linux
 #else
diff --git a/qemu/roms/ipxe/src/include/ipxe/linux_compat.h b/qemu/roms/ipxe/src/include/ipxe/linux_compat.h
index 6f6ed97d7..4704c4817 100644
--- a/qemu/roms/ipxe/src/include/ipxe/linux_compat.h
+++ b/qemu/roms/ipxe/src/include/ipxe/linux_compat.h
@@ -10,7 +10,7 @@
  * intended to be a substitute for proper porting.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/list.h b/qemu/roms/ipxe/src/include/ipxe/list.h
index 581ec9806..6a9b76f91 100644
--- a/qemu/roms/ipxe/src/include/ipxe/list.h
+++ b/qemu/roms/ipxe/src/include/ipxe/list.h
@@ -9,7 +9,7 @@
  * list.h.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <assert.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/login_ui.h b/qemu/roms/ipxe/src/include/ipxe/login_ui.h
index 01e5479f7..313e07349 100644
--- a/qemu/roms/ipxe/src/include/ipxe/login_ui.h
+++ b/qemu/roms/ipxe/src/include/ipxe/login_ui.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int login_ui ( void );
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/malloc.h b/qemu/roms/ipxe/src/include/ipxe/malloc.h
index bbd6cb898..dd158b8e6 100644
--- a/qemu/roms/ipxe/src/include/ipxe/malloc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/malloc.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * Prototypes for the standard functions (malloc() et al) are in
@@ -77,8 +77,8 @@ static inline void * __malloc malloc_dma ( size_t size, size_t phys_align ) {
  * If @c ptr is NULL, no action is taken.
  */
 static inline void free_dma ( void *ptr, size_t size ) {
-	free_memblock ( ptr, size );
 	VALGRIND_FREELIKE_BLOCK ( ptr, 0 );
+	free_memblock ( ptr, size );
 }
 
 /** A cache discarder */
diff --git a/qemu/roms/ipxe/src/include/ipxe/mca.h b/qemu/roms/ipxe/src/include/ipxe/mca.h
index d86dab195..11470ec93 100644
--- a/qemu/roms/ipxe/src/include/ipxe/mca.h
+++ b/qemu/roms/ipxe/src/include/ipxe/mca.h
@@ -5,7 +5,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifndef MCA_H
 #define MCA_H
diff --git a/qemu/roms/ipxe/src/include/ipxe/md5.h b/qemu/roms/ipxe/src/include/ipxe/md5.h
index 860bc4769..05c3974c8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/md5.h
+++ b/qemu/roms/ipxe/src/include/ipxe/md5.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/memblock.h b/qemu/roms/ipxe/src/include/ipxe/memblock.h
index 13af3e433..2bb38c460 100644
--- a/qemu/roms/ipxe/src/include/ipxe/memblock.h
+++ b/qemu/roms/ipxe/src/include/ipxe/memblock.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/menu.h b/qemu/roms/ipxe/src/include/ipxe/menu.h
index f2b3caccc..3cc99be48 100644
--- a/qemu/roms/ipxe/src/include/ipxe/menu.h
+++ b/qemu/roms/ipxe/src/include/ipxe/menu.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/mii.h b/qemu/roms/ipxe/src/include/ipxe/mii.h
index f53ad4a62..c2245b49e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/mii.h
+++ b/qemu/roms/ipxe/src/include/ipxe/mii.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <mii.h>
 #include <ipxe/netdevice.h>
@@ -114,5 +114,7 @@ mii_dump ( struct mii_interface *mii ) {
 
 extern int mii_restart ( struct mii_interface *mii );
 extern int mii_reset ( struct mii_interface *mii );
+extern int mii_check_link ( struct mii_interface *mii,
+			    struct net_device *netdev );
 
 #endif /* _IPXE_MII_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/monojob.h b/qemu/roms/ipxe/src/include/ipxe/monojob.h
index aedc37eca..1661d91c2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/monojob.h
+++ b/qemu/roms/ipxe/src/include/ipxe/monojob.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct interface;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/mount.h b/qemu/roms/ipxe/src/include/ipxe/mount.h
index ca958117a..2d42ba080 100644
--- a/qemu/roms/ipxe/src/include/ipxe/mount.h
+++ b/qemu/roms/ipxe/src/include/ipxe/mount.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** NFS MOUNT protocol number */
 #define ONCRPC_MOUNT 100005
diff --git a/qemu/roms/ipxe/src/include/ipxe/nap.h b/qemu/roms/ipxe/src/include/ipxe/nap.h
index afc887910..f4de778c4 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/nap.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ndp.h b/qemu/roms/ipxe/src/include/ipxe/ndp.h
index 7388f938e..1815236f5 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ndp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ndp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/in.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/neighbour.h b/qemu/roms/ipxe/src/include/ipxe/neighbour.h
index f2a3946f1..1c1d1b6ca 100644
--- a/qemu/roms/ipxe/src/include/ipxe/neighbour.h
+++ b/qemu/roms/ipxe/src/include/ipxe/neighbour.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/net80211_err.h b/qemu/roms/ipxe/src/include/ipxe/net80211_err.h
index 7df3d0d85..32ccc257f 100644
--- a/qemu/roms/ipxe/src/include/ipxe/net80211_err.h
+++ b/qemu/roms/ipxe/src/include/ipxe/net80211_err.h
@@ -10,7 +10,7 @@
  * Copyright (c) 2009 Joshua Oreman <oremanj@rwcr.net>.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/include/ipxe/netdevice.h b/qemu/roms/ipxe/src/include/ipxe/netdevice.h
index 95ad1cf1b..a1d207ffc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/netdevice.h
+++ b/qemu/roms/ipxe/src/include/ipxe/netdevice.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/list.h>
@@ -15,6 +15,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/refcnt.h>
 #include <ipxe/settings.h>
 #include <ipxe/interface.h>
+#include <ipxe/retry.h>
 
 struct io_buffer;
 struct net_device;
@@ -36,13 +37,12 @@ struct device;
 
 /** Maximum length of a link-layer header
  *
- * The longest currently-supported link-layer header is for 802.11: a
- * 24-byte frame header plus an 8-byte 802.3 LLC/SNAP header, plus a
- * possible 4-byte VLAN header.  (The IPoIB link-layer pseudo-header
- * doesn't actually include link-layer addresses; see ipoib.c for
- * details.)
+ * The longest currently-supported link-layer header is for RNDIS: an
+ * 8-byte RNDIS header, a 32-byte RNDIS packet message header, a
+ * 14-byte Ethernet header and a possible 4-byte VLAN header.  Round
+ * up to 64 bytes.
  */
-#define MAX_LL_HEADER_LEN 36
+#define MAX_LL_HEADER_LEN 64
 
 /** Maximum length of a network-layer address */
 #define MAX_NET_ADDR_LEN 16
@@ -393,6 +393,8 @@ struct net_device {
 	 * indicates the error preventing link-up.
 	 */
 	int link_rc;
+	/** Link block timer */
+	struct retry_timer link_block;
 	/** Maximum packet length
 	 *
 	 * This length includes any link-layer headers.
@@ -428,6 +430,14 @@ struct net_device {
 /** Network device receive queue processing is frozen */
 #define NETDEV_RX_FROZEN 0x0004
 
+/** Network device interrupts are unsupported
+ *
+ * This flag can be used by a network device to indicate that
+ * interrupts are not supported despite the presence of an irq()
+ * method.
+ */
+#define NETDEV_IRQ_UNSUPPORTED 0x0008
+
 /** Link-layer protocol table */
 #define LL_PROTOCOLS __table ( struct ll_protocol, "ll_protocols" )
 
@@ -615,6 +625,17 @@ netdev_link_ok ( struct net_device *netdev ) {
 }
 
 /**
+ * Check link block state of network device
+ *
+ * @v netdev		Network device
+ * @ret link_blocked	Link is blocked
+ */
+static inline __attribute__ (( always_inline )) int
+netdev_link_blocked ( struct net_device *netdev ) {
+	return ( timer_running ( &netdev->link_block ) );
+}
+
+/**
  * Check whether or not network device is open
  *
  * @v netdev		Network device
@@ -633,7 +654,8 @@ netdev_is_open ( struct net_device *netdev ) {
  */
 static inline __attribute__ (( always_inline )) int
 netdev_irq_supported ( struct net_device *netdev ) {
-	return ( netdev->op->irq != NULL );
+	return ( ( netdev->op->irq != NULL ) &&
+		 ! ( netdev->state & NETDEV_IRQ_UNSUPPORTED ) );
 }
 
 /**
@@ -662,6 +684,9 @@ extern void netdev_rx_freeze ( struct net_device *netdev );
 extern void netdev_rx_unfreeze ( struct net_device *netdev );
 extern void netdev_link_err ( struct net_device *netdev, int rc );
 extern void netdev_link_down ( struct net_device *netdev );
+extern void netdev_link_block ( struct net_device *netdev,
+				unsigned long timeout );
+extern void netdev_link_unblock ( struct net_device *netdev );
 extern int netdev_tx ( struct net_device *netdev, struct io_buffer *iobuf );
 extern void netdev_tx_defer ( struct net_device *netdev,
 			      struct io_buffer *iobuf );
diff --git a/qemu/roms/ipxe/src/include/ipxe/nfs.h b/qemu/roms/ipxe/src/include/ipxe/nfs.h
index 498ed5a27..69b8b5381 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nfs.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nfs.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** NFS protocol number */
 #define ONCRPC_NFS 100003
diff --git a/qemu/roms/ipxe/src/include/ipxe/nfs_open.h b/qemu/roms/ipxe/src/include/ipxe/nfs_open.h
index caba977f7..8572c41b3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nfs_open.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nfs_open.h
@@ -7,6 +7,6 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #endif /* _IPXE_NFS_OPEN_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/nfs_uri.h b/qemu/roms/ipxe/src/include/ipxe/nfs_uri.h
index d88bd6f65..aaa6d3749 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nfs_uri.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nfs_uri.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uri.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/null_entropy.h b/qemu/roms/ipxe/src/include/ipxe/null_entropy.h
index 646d1a17e..91adefa69 100644
--- a/qemu/roms/ipxe/src/include/ipxe/null_entropy.h
+++ b/qemu/roms/ipxe/src/include/ipxe/null_entropy.h
@@ -9,7 +9,7 @@
  * security-sensitive environment.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/null_nap.h b/qemu/roms/ipxe/src/include/ipxe/null_nap.h
index 0c0704bc7..17145b48b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/null_nap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/null_nap.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef NAP_NULL
 #define NAP_PREFIX_null
diff --git a/qemu/roms/ipxe/src/include/ipxe/null_reboot.h b/qemu/roms/ipxe/src/include/ipxe/null_reboot.h
index 3de36c5b3..5de38afc0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/null_reboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/null_reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef REBOOT_NULL
 #define REBOOT_PREFIX_null
diff --git a/qemu/roms/ipxe/src/include/ipxe/null_sanboot.h b/qemu/roms/ipxe/src/include/ipxe/null_sanboot.h
index 2b3a2c74d..58f03339f 100644
--- a/qemu/roms/ipxe/src/include/ipxe/null_sanboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/null_sanboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef SANBOOT_NULL
 #define SANBOOT_PREFIX_null
diff --git a/qemu/roms/ipxe/src/include/ipxe/null_time.h b/qemu/roms/ipxe/src/include/ipxe/null_time.h
index 2b72cdf50..d2b15194b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/null_time.h
+++ b/qemu/roms/ipxe/src/include/ipxe/null_time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #ifdef TIME_NULL
 #define TIME_PREFIX_null
diff --git a/qemu/roms/ipxe/src/include/ipxe/nvo.h b/qemu/roms/ipxe/src/include/ipxe/nvo.h
index 1a629da78..7a3c7a3db 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nvo.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nvo.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/dhcpopts.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/nvs.h b/qemu/roms/ipxe/src/include/ipxe/nvs.h
index 4733123cf..5789f4c0d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nvs.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nvs.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/nvsvpd.h b/qemu/roms/ipxe/src/include/ipxe/nvsvpd.h
index 3450e5c71..4c50daf85 100644
--- a/qemu/roms/ipxe/src/include/ipxe/nvsvpd.h
+++ b/qemu/roms/ipxe/src/include/ipxe/nvsvpd.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nvs.h>
 #include <ipxe/pcivpd.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/ocsp.h b/qemu/roms/ipxe/src/include/ipxe/ocsp.h
index 387e28f81..71fa41dc9 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ocsp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ocsp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <time.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/oncrpc.h b/qemu/roms/ipxe/src/include/ipxe/oncrpc.h
index 76c1260f2..071468711 100644
--- a/qemu/roms/ipxe/src/include/ipxe/oncrpc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/oncrpc.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** ONC RCP Version */
 #define ONCRPC_VERS 2
diff --git a/qemu/roms/ipxe/src/include/ipxe/oncrpc_iob.h b/qemu/roms/ipxe/src/include/ipxe/oncrpc_iob.h
index 4858d96b5..b55043770 100644
--- a/qemu/roms/ipxe/src/include/ipxe/oncrpc_iob.h
+++ b/qemu/roms/ipxe/src/include/ipxe/oncrpc_iob.h
@@ -13,7 +13,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Add a string to the end of an I/O buffer
diff --git a/qemu/roms/ipxe/src/include/ipxe/open.h b/qemu/roms/ipxe/src/include/ipxe/open.h
index a522f0cd1..43d4cdc66 100644
--- a/qemu/roms/ipxe/src/include/ipxe/open.h
+++ b/qemu/roms/ipxe/src/include/ipxe/open.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/params.h b/qemu/roms/ipxe/src/include/ipxe/params.h
index c2d82d9cf..dd3292efc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/params.h
+++ b/qemu/roms/ipxe/src/include/ipxe/params.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/parseopt.h b/qemu/roms/ipxe/src/include/ipxe/parseopt.h
index 840de7497..829b3431c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/parseopt.h
+++ b/qemu/roms/ipxe/src/include/ipxe/parseopt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pccrc.h b/qemu/roms/ipxe/src/include/ipxe/pccrc.h
new file mode 100644
index 000000000..7f0963428
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/pccrc.h
@@ -0,0 +1,447 @@
+#ifndef _IPXE_PCCRC_H
+#define _IPXE_PCCRC_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC]
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <byteswap.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/crypto.h>
+
+/******************************************************************************
+ *
+ * Content Information versioning
+ *
+ ******************************************************************************
+ *
+ * Note that version 1 data structures are little-endian, but version
+ * 2 data structures are big-endian.
+ */
+
+/** Content Information version number */
+union peerdist_info_version {
+	/** Raw version number
+	 *
+	 * Always little-endian, regardless of whether the
+	 * encompassing structure is version 1 (little-endian) or
+	 * version 2 (big-endian).
+	 */
+	uint16_t raw;
+	/** Major:minor version number */
+	struct {
+		/** Minor version number */
+		uint8_t minor;
+		/** Major version number */
+		uint8_t major;
+	} __attribute__ (( packed ));
+} __attribute__ (( packed ));
+
+/** Content Information version 1 */
+#define PEERDIST_INFO_V1 0x0100
+
+/** Content Information version 2 */
+#define PEERDIST_INFO_V2 0x0200
+
+/******************************************************************************
+ *
+ * Content Information version 1
+ *
+ ******************************************************************************
+ */
+
+/** Content Information version 1 data structure header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1 {
+	/** Version number */
+	union peerdist_info_version version;
+	/** Hash algorithm
+	 *
+	 * This is a @c PEERDIST_INFO_V1_HASH_XXX constant.
+	 */
+	uint32_t hash;
+	/** Length to skip in first segment
+	 *
+	 * Length at the start of the first segment which is not
+	 * included within the content range.
+	 */
+	uint32_t first;
+	/** Length to read in last segment, or zero
+	 *
+	 * Length within the last segment which is included within the
+	 * content range.  A zero value indicates that the whole of
+	 * the last segment is included within the content range.
+	 */
+	uint32_t last;
+	/** Number of segments within the content information */
+	uint32_t segments;
+	/* Followed by a variable-length array of segment descriptions
+	 * and a list of variable-length block descriptions:
+	 *
+	 * peerdist_info_v1_segment_t(digestsize) segment[segments];
+	 * peerdist_info_v1_block_t(digestsize, block0.blocks) block0;
+	 * peerdist_info_v1_block_t(digestsize, block1.blocks) block1;
+	 * ...
+	 * peerdist_info_v1_block_t(digestsize, blockN.blocks) blockN;
+	 */
+} __attribute__ (( packed ));
+
+/** SHA-256 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA256 0x0000800cUL
+
+/** SHA-384 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA384 0x0000800dUL
+
+/** SHA-512 hash algorithm */
+#define PEERDIST_INFO_V1_HASH_SHA512 0x0000800eUL
+
+/** Content Information version 1 segment description header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1_segment {
+	/** Offset of this segment within the content */
+	uint64_t offset;
+	/** Length of this segment
+	 *
+	 * Should always be 32MB, except for the last segment within
+	 * the content.
+	 */
+	uint32_t len;
+	/** Block size for this segment
+	 *
+	 * Should always be 64kB.  Note that the last block within the
+	 * last segment may actually be less than 64kB.
+	 */
+	uint32_t blksize;
+	/* Followed by two variable-length hashes:
+	 *
+	 * uint8_t hash[digestsize];
+	 * uint8_t secret[digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 *
+	 * Note that the hash is taken over (the hashes of all blocks
+	 * within) the entire segment, even if the blocks do not
+	 * intersect the content range (and so do not appear within
+	 * the block list).  It therefore functions only as a segment
+	 * identifier; it cannot be used to verify the content of the
+	 * segment (since we may not download all blocks within the
+	 * segment).
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 1 segment description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v1_segment_t( digestsize )			\
+	struct {							\
+		struct peerdist_info_v1_segment segment;		\
+		uint8_t hash[digestsize];				\
+		uint8_t secret[digestsize];				\
+	} __attribute__ (( packed ))
+
+/** Content Information version 1 block description header
+ *
+ * All fields are little-endian.
+ */
+struct peerdist_info_v1_block {
+	/** Number of blocks within the block description
+	 *
+	 * This is the number of blocks within the segment which
+	 * overlap the content range.  It may therefore be less than
+	 * the number of blocks within the segment.
+	 */
+	uint32_t blocks;
+	/* Followed by an array of variable-length hashes:
+	 *
+	 * uint8_t hash[blocks][digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 */
+ } __attribute__ (( packed ));
+
+/** Content Information version 1 block description
+ *
+ * @v digestsize	Digest size
+ * @v blocks		Number of blocks
+ */
+#define peerdist_info_v1_block_t( digestsize, blocks )			\
+	struct {							\
+		struct peerdist_info_v1_block block;			\
+		uint8_t hash[blocks][digestsize];			\
+	} __attribute__ (( packed ))
+
+/******************************************************************************
+ *
+ * Content Information version 2
+ *
+ ******************************************************************************
+ */
+
+/** Content Information version 2 data structure header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2 {
+	/** Version number */
+	union peerdist_info_version version;
+	/** Hash algorithm
+	 *
+	 * This is a @c PEERDIST_INFO_V2_HASH_XXX constant.
+	 */
+	uint8_t hash;
+	/** Offset of the first segment within the content */
+	uint64_t offset;
+	/** Index of the first segment within the content */
+	uint64_t index;
+	/** Length to skip in first segment
+	 *
+	 * Length at the start of the first segment which is not
+	 * included within the content range.
+	 */
+	uint32_t first;
+	/** Length of content range, or zero
+	 *
+	 * Length of the content range.  A zero indicates that
+	 * everything up to the end of the last segment is included in
+	 * the content range.
+	 */
+	uint64_t len;
+	/* Followed by a list of chunk descriptions */
+} __attribute__ (( packed ));
+
+/** SHA-512 hash algorithm with output truncated to first 256 bits */
+#define PEERDIST_INFO_V2_HASH_SHA512_TRUNC 0x04
+
+/** Content Information version 2 chunk description header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2_chunk {
+	/** Chunk type */
+	uint8_t type;
+	/** Chunk data length */
+	uint32_t len;
+	/* Followed by an array of segment descriptions:
+	 *
+	 * peerdist_info_v2_segment_t(digestsize) segment[segments]
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm, and segments is equal to @c len divided by the
+	 * size of each segment array entry.
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 2 chunk description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v2_chunk_t( digestsize )				\
+	struct {							\
+		struct peerdist_info_v2_chunk chunk;			\
+		peerdist_info_v2_segment_t ( digestsize ) segment[0];	\
+	} __attribute__ (( packed ))
+
+/** Chunk type */
+#define PEERDIST_INFO_V2_CHUNK_TYPE 0x00
+
+/** Content Information version 2 segment description header
+ *
+ * All fields are big-endian.
+ */
+struct peerdist_info_v2_segment {
+	/** Segment length */
+	uint32_t len;
+	/* Followed by two variable-length hashes:
+	 *
+	 * uint8_t hash[digestsize];
+	 * uint8_t secret[digestsize];
+	 *
+	 * where digestsize is the digest size for the selected hash
+	 * algorithm.
+	 */
+} __attribute__ (( packed ));
+
+/** Content Information version 2 segment description
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_info_v2_segment_t( digestsize )			\
+	struct {							\
+		struct peerdist_info_v2_segment segment;		\
+		uint8_t hash[digestsize];				\
+		uint8_t secret[digestsize];				\
+	} __attribute__ (( packed ))
+
+/******************************************************************************
+ *
+ * Content Information
+ *
+ ******************************************************************************
+ */
+
+/** Maximum digest size for any supported algorithm
+ *
+ * The largest digest size that we support is for SHA-512 at 64 bytes
+ */
+#define PEERDIST_DIGEST_MAX_SIZE 64
+
+/** Raw content information */
+struct peerdist_raw {
+	/** Data buffer */
+	userptr_t data;
+	/** Length of data buffer */
+	size_t len;
+};
+
+/** A content range */
+struct peerdist_range {
+	/** Start offset */
+	size_t start;
+	/** End offset */
+	size_t end;
+};
+
+/** Content information */
+struct peerdist_info {
+	/** Raw content information */
+	struct peerdist_raw raw;
+
+	/** Content information operations */
+	struct peerdist_info_operations *op;
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Digest size
+	 *
+	 * Note that this may be shorter than the digest size of the
+	 * digest algorithm.  The truncation does not always take
+	 * place as soon as a digest is calculated.  For example,
+	 * version 2 content information uses SHA-512 with a truncated
+	 * digest size of 32 (256 bits), but the segment identifier
+	 * ("HoHoDk") is calculated by using HMAC with the full
+	 * SHA-512 digest and then truncating the HMAC output, rather
+	 * than by simply using HMAC with the truncated SHA-512
+	 * digest.  This is, of course, totally undocumented.
+	 */
+	size_t digestsize;
+	/** Content range */
+	struct peerdist_range range;
+	/** Trimmed content range */
+	struct peerdist_range trim;
+	/** Number of segments within the content information */
+	unsigned int segments;
+};
+
+/** A content information segment */
+struct peerdist_info_segment {
+	/** Content information */
+	const struct peerdist_info *info;
+	/** Segment index */
+	unsigned int index;
+
+	/** Content range
+	 *
+	 * Note that this range may exceed the overall content range.
+	 */
+	struct peerdist_range range;
+	/** Number of blocks within this segment */
+	unsigned int blocks;
+	/** Block size */
+	size_t blksize;
+	/** Segment hash of data
+	 *
+	 * This is MS-PCCRC's "HoD".
+	 */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment secret
+	 *
+	 * This is MS-PCCRC's "Ke = Kp".
+	 */
+	uint8_t secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment identifier
+	 *
+	 * This is MS-PCCRC's "HoHoDk".
+	 */
+	uint8_t id[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/** Magic string constant used to calculate segment identifier
+ *
+ * Note that the MS-PCCRC specification states that this constant is
+ *
+ *   "the null-terminated ASCII string constant "MS_P2P_CACHING";
+ *    string literals are all ASCII strings with NULL terminators
+ *    unless otherwise noted."
+ *
+ * The specification lies.  This constant is a UTF-16LE string, not an
+ * ASCII string.  The terminating wNUL *is* included within the
+ * constant.
+ */
+#define PEERDIST_SEGMENT_ID_MAGIC L"MS_P2P_CACHING"
+
+/** A content information block */
+struct peerdist_info_block {
+	/** Content information segment */
+	const struct peerdist_info_segment *segment;
+	/** Block index */
+	unsigned int index;
+
+	/** Content range
+	 *
+	 * Note that this range may exceed the overall content range.
+	 */
+	struct peerdist_range range;
+	/** Trimmed content range */
+	struct peerdist_range trim;
+	/** Block hash */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/** Content information operations */
+struct peerdist_info_operations {
+	/**
+	 * Populate content information
+	 *
+	 * @v info		Content information to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * info ) ( struct peerdist_info *info );
+	/**
+	 * Populate content information segment
+	 *
+	 * @v segment		Content information segment to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * segment ) ( struct peerdist_info_segment *segment );
+	/**
+	 * Populate content information block
+	 *
+	 * @v block		Content information block to fill in
+	 * @ret rc		Return status code
+	 */
+	int ( * block ) ( struct peerdist_info_block *block );
+};
+
+extern struct digest_algorithm sha512_trunc_algorithm;
+
+extern int peerdist_info ( userptr_t data, size_t len,
+			   struct peerdist_info *info );
+extern int peerdist_info_segment ( const struct peerdist_info *info,
+				   struct peerdist_info_segment *segment,
+				   unsigned int index );
+extern int peerdist_info_block ( const struct peerdist_info_segment *segment,
+				 struct peerdist_info_block *block,
+				 unsigned int index );
+
+#endif /* _IPXE_PCCRC_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/pccrd.h b/qemu/roms/ipxe/src/include/ipxe/pccrd.h
new file mode 100644
index 000000000..3daa92f29
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/pccrd.h
@@ -0,0 +1,47 @@
+#ifndef _IPXE_PCCRD_H
+#define _IPXE_PCCRD_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Discovery Protocol [MS-PCCRD]
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** PeerDist discovery port */
+#define PEERDIST_DISCOVERY_PORT 3702
+
+/** PeerDist discovery IPv4 address (239.255.255.250) */
+#define PEERDIST_DISCOVERY_IPV4 \
+	( ( 239 << 24 ) | ( 255 << 16 ) | ( 255 << 8 ) | ( 250 << 0 ) )
+
+/** PeerDist discovery IPv6 address (ff02::c) */
+#define PEERDIST_DISCOVERY_IPV6 \
+	{ 0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xc }
+
+/** A PeerDist discovery reply block count */
+struct peerdist_discovery_block_count {
+	/** Count (as an eight-digit hex value) */
+	char hex[8];
+} __attribute__ (( packed ));
+
+/** A PeerDist discovery reply */
+struct peerdist_discovery_reply {
+	/** List of segment ID strings
+	 *
+	 * The list is terminated with a zero-length string.
+	 */
+	char *ids;
+	/** List of peer locations
+	 *
+	 * The list is terminated with a zero-length string.
+	 */
+	char *locations;
+};
+
+extern char * peerdist_discovery_request ( const char *uuid, const char *id );
+extern int peerdist_discovery_reply ( char *data, size_t len,
+				      struct peerdist_discovery_reply *reply );
+
+#endif /* _IPXE_PCCRD_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/pccrr.h b/qemu/roms/ipxe/src/include/ipxe/pccrr.h
new file mode 100644
index 000000000..1ea86c40d
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/pccrr.h
@@ -0,0 +1,376 @@
+#ifndef _IPXE_PCCRR_H
+#define _IPXE_PCCRR_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Retrieval Protocol [MS-PCCRR]
+ *
+ * All fields are in network byte order.
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/uaccess.h>
+
+/** Magic retrieval URI path */
+#define PEERDIST_MAGIC_PATH "/116B50EB-ECE2-41ac-8429-9F9E963361B7/"
+
+/** Retrieval protocol version */
+union peerdist_msg_version {
+	/** Raw version number */
+	uint32_t raw;
+	/** Major:minor version number */
+	struct {
+		/** Minor version number */
+		uint16_t minor;
+		/** Major version number */
+		uint16_t major;
+	} __attribute__ (( packed ));
+} __attribute__ (( packed ));
+
+/** Retrieval protocol version 1.0 */
+#define PEERDIST_MSG_VERSION_1_0 0x00000001UL
+
+/** Retrieval protocol version 2.0 */
+#define PEERDIST_MSG_VERSION_2_0 0x00000002UL
+
+/** Retrieval protocol supported versions */
+struct peerdist_msg_versions {
+	/** Minimum supported protocol version */
+	union peerdist_msg_version min;
+	/** Maximum supported protocol version */
+	union peerdist_msg_version max;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block range */
+struct peerdist_msg_range {
+	/** First block in range */
+	uint32_t first;
+	/** Number of blocks in range */
+	uint32_t count;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol segment ID header */
+struct peerdist_msg_segment {
+	/** Digest size (i.e. length of segment ID) */
+	uint32_t digestsize;
+	/* Followed by a single variable-length ID and padding:
+	 *
+	 * uint8_t id[digestsize];
+	 * uint8_t pad[ (-digestsize) & 0x3 ];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol segment ID
+ *
+ * @v digestsize	Digest size
+ */
+#define peerdist_msg_segment_t( digestsize )				\
+	struct {							\
+		struct peerdist_msg_segment segment;			\
+		uint8_t id[digestsize];					\
+		uint8_t pad[ ( -(digestsize) ) & 0x3 ];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block range list header */
+struct peerdist_msg_ranges {
+	/** Number of ranges */
+	uint32_t count;
+	/* Followed by an array of block ranges:
+	 *
+	 * struct peerdist_msg_range range[count];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block range list
+ *
+ * @v count		Number of ranges
+ */
+#define peerdist_msg_ranges_t( count )					\
+	struct {							\
+		struct peerdist_msg_ranges ranges;			\
+		struct peerdist_msg_range range[count];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol data block header */
+struct peerdist_msg_block {
+	/** Length of data block */
+	uint32_t len;
+	/* Followed by the (encrypted) data block:
+	 *
+	 * uint8_t data[len];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol data block */
+#define peerdist_msg_block_t( len )					\
+	struct {							\
+		struct peerdist_msg_block block;			\
+		uint8_t data[len];					\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol initialisation vector header */
+struct peerdist_msg_iv {
+	/** Cipher block size */
+	uint32_t blksize;
+	/* Followed by the initialisation vector:
+	 *
+	 * uint8_t data[blksize];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol initialisation vector */
+#define peerdist_msg_iv_t( blksize )					\
+	struct {							\
+		struct peerdist_msg_iv iv;				\
+		uint8_t data[blksize];					\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol useless VRF data header */
+struct peerdist_msg_useless_vrf {
+	/** Length of useless VRF data */
+	uint32_t len;
+	/* Followed by a variable-length useless VRF data block and
+	 * padding:
+	 *
+	 * uint8_t data[len];
+	 * uint8_t pad[ (-len) & 0x3 ];
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol useless VRF data */
+#define peerdist_msg_useless_vrf_t( vrf_len )				\
+	struct {							\
+		struct peerdist_msg_useless_vrf vrf;			\
+		uint8_t data[vrf_len];					\
+		uint8_t pad[ ( -(vrf_len) ) & 0x3 ];			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol message header */
+struct peerdist_msg_header {
+	/** Protocol version
+	 *
+	 * This is the protocol version in which the message type was
+	 * first defined.
+	 */
+	union peerdist_msg_version version;
+	/** Message type */
+	uint32_t type;
+	/** Message size (including this header) */
+	uint32_t len;
+	/** Cryptographic algorithm ID */
+	uint32_t algorithm;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol cryptographic algorithm IDs */
+enum peerdist_msg_algorithm {
+	/** No encryption */
+	PEERDIST_MSG_PLAINTEXT = 0x00000000UL,
+	/** AES-128 in CBC mode */
+	PEERDIST_MSG_AES_128_CBC = 0x00000001UL,
+	/** AES-192 in CBC mode */
+	PEERDIST_MSG_AES_192_CBC = 0x00000002UL,
+	/** AES-256 in CBC mode */
+	PEERDIST_MSG_AES_256_CBC = 0x00000003UL,
+};
+
+/** Retrieval protocol transport response header */
+struct peerdist_msg_transport_header {
+	/** Length (excluding this header)
+	 *
+	 * This seems to be identical in both purpose and value to the
+	 * length found within the message header, and therefore
+	 * serves no useful purpose.
+	 */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation request */
+struct peerdist_msg_nego_req {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/** Supported versions */
+	struct peerdist_msg_versions versions;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation request version */
+#define PEERDIST_MSG_NEGO_REQ_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol negotiation request type */
+#define PEERDIST_MSG_NEGO_REQ_TYPE 0x00000000UL
+
+/** Retrieval protocol negotiation response */
+struct peerdist_msg_nego_resp {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/** Supported versions */
+	struct peerdist_msg_versions versions;
+} __attribute__ (( packed ));
+
+/** Retrieval protocol negotiation response version */
+#define PEERDIST_MSG_NEGO_RESP_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol negotiation response type */
+#define PEERDIST_MSG_NEGO_RESP_TYPE 0x00000001UL
+
+/** Retrieval protocol block list request header */
+struct peerdist_msg_getblklist {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID and a block range list:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block list request
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ */
+#define peerdist_msg_getblklist_t( digestsize, count )			\
+	struct {							\
+		struct peerdist_msg_getblklist getblklist;		\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block list request version */
+#define PEERDIST_MSG_GETBLKLIST_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block list request type */
+#define PEERDIST_MSG_GETBLKLIST_TYPE 0x00000002UL
+
+/** Retrieval protocol block fetch request header */
+struct peerdist_msg_getblks {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block range list, and a useless
+	 * VRF block:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 * peerdist_msg_vrf_t(vrf_len) vrf;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block fetch request
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ * @v vrf_len		Length of uselessness
+ */
+#define peerdist_msg_getblks_t( digestsize, count, vrf_len )		\
+	struct {							\
+		struct peerdist_msg_getblks getblks;			\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+		peerdist_msg_useless_vrf_t ( vrf_len );			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block fetch request version */
+#define PEERDIST_MSG_GETBLKS_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block fetch request type */
+#define PEERDIST_MSG_GETBLKS_TYPE 0x00000003UL
+
+/** Retrieval protocol block list response header */
+struct peerdist_msg_blklist {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block range list, and a next
+	 * block index:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * peerdist_msg_ranges_t(count) ranges;
+	 * uint32_t next;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block list response
+ *
+ * @v digestsize	Digest size
+ * @v count		Block range count
+ */
+#define peerdist_msg_blklist_t( digestsize, count )			\
+	struct {							\
+		struct peerdist_msg_blklist blklist;			\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		peerdist_msg_ranges_t ( count ) ranges;			\
+		uint32_t next;						\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block list response version */
+#define PEERDIST_MSG_BLKLIST_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block list response type */
+#define PEERDIST_MSG_BLKLIST_TYPE 0x00000004UL
+
+/** Retrieval protocol block fetch response header */
+struct peerdist_msg_blk {
+	/** Message header */
+	struct peerdist_msg_header hdr;
+	/* Followed by a segment ID, a block index, a next block
+	 * index, a data block, a useless VRF block, and an
+	 * initialisation vector:
+	 *
+	 * peerdist_msg_segment_t(digestsize) segment;
+	 * uint32_t index;
+	 * uint32_t next;
+	 * peerdist_msg_block_t(len) data;
+	 * peerdist_msg_useless_vrf_t(vrf_len) vrf;
+	 * peerdist_msg_iv_t(blksize) iv;
+	 */
+} __attribute__ (( packed ));
+
+/** Retrieval protocol block fetch response
+ *
+ * @v digestsize	Digest size
+ * @v len		Data block length
+ * @v vrf_len		Length of uselessness
+ * @v blksize		Cipher block size
+ */
+#define peerdist_msg_blk_t( digestsize, len, vrf_len, blksize )		\
+	struct {							\
+		struct peerdist_msg_blk blk;				\
+		peerdist_msg_segment_t ( digestsize ) segment;		\
+		uint32_t index;						\
+		uint32_t next;						\
+		peerdist_msg_block_t ( len ) block;			\
+		peerdist_msg_useless_vrf_t ( vrf_len ) vrf;		\
+		peerdist_msg_iv_t ( blksize ) iv;			\
+	} __attribute__ (( packed ))
+
+/** Retrieval protocol block fetch response version */
+#define PEERDIST_MSG_BLK_VERSION PEERDIST_MSG_VERSION_1_0
+
+/** Retrieval protocol block fetch response type */
+#define PEERDIST_MSG_BLK_TYPE 0x00000005UL
+
+/**
+ * Parse retrieval protocol block fetch response
+ *
+ * @v raw		Raw data
+ * @v raw_len		Length of raw data
+ * @v digestsize	Digest size
+ * @v blksize		Cipher block size
+ * @v blk		Structure to fill in
+ * @ret rc		Return status code
+ */
+#define peerdist_msg_blk( raw, raw_len, digestsize, blksize, blk ) ( {	\
+	assert ( sizeof ( (blk)->segment.id ) == (digestsize) );	\
+	assert ( sizeof ( (blk)->block.data ) == 0 );			\
+	assert ( sizeof ( (blk)->vrf.data ) == 0 );			\
+	assert ( sizeof ( (blk)->iv.data ) == blksize );		\
+	peerdist_msg_blk_untyped ( (raw), (raw_len), (digestsize),	\
+				   (blksize), blk );			\
+	} )
+
+extern int peerdist_msg_blk_untyped ( userptr_t raw, size_t raw_len,
+				      size_t digestsize, size_t blksize,
+				      void *out );
+
+#endif /* _IPXE_PCCRR_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/pci.h b/qemu/roms/ipxe/src/include/ipxe/pci.h
index 692771ebe..a841e00ff 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pci.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pci.h
@@ -1,268 +1,132 @@
 #ifndef	_IPXE_PCI_H
 #define _IPXE_PCI_H
 
-/*
- * Support for NE2000 PCI clones added David Monro June 1997
- * Generalised for other PCI NICs by Ken Yap July 1997
- * PCI support rewritten by Michael Brown 2006
+/** @file
+ *
+ * PCI bus
  *
- * Most of this is taken from /usr/src/linux/include/linux/pci.h.
- */
-
-/*
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
 #include <ipxe/pci_io.h>
-#include "pci_ids.h"
 
-/*
- * PCI constants
- *
- */
-
-#define PCI_COMMAND_IO			0x1	/* Enable response in I/O space */
-#define PCI_COMMAND_MEM			0x2	/* Enable response in mem space */
-#define PCI_COMMAND_MASTER		0x4	/* Enable bus mastering */
-
-#define PCI_CACHE_LINE_SIZE		0x0c	/* 8 bits */
-#define PCI_LATENCY_TIMER		0x0d	/* 8 bits */
-
-#define PCI_COMMAND_SPECIAL		0x8	/* Enable response to special cycles */
-#define PCI_COMMAND_INVALIDATE		0x10	/* Use memory write and invalidate */
-#define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
-#define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
-#define  PCI_COMMAND_WAIT 	0x80	/* Enable address/data stepping */
-#define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
-#define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
-#define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
-
-
-#define PCI_VENDOR_ID           0x00    /* 16 bits */
-#define PCI_DEVICE_ID           0x02    /* 16 bits */
-#define PCI_COMMAND             0x04    /* 16 bits */
-
-#define PCI_STATUS		0x06	/* 16 bits */
-#define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
-#define  PCI_STATUS_66MHZ	0x20	/* Support 66 Mhz PCI 2.1 bus */
-#define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
-#define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
-#define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
-#define  PCI_STATUS_DEVSEL_MASK	0x600	/* DEVSEL timing */
-#define  PCI_STATUS_DEVSEL_FAST	0x000	
-#define  PCI_STATUS_DEVSEL_MEDIUM 0x200
-#define  PCI_STATUS_DEVSEL_SLOW 0x400
-#define  PCI_STATUS_SIG_TARGET_ABORT 0x800 /* Set on target abort */
-#define  PCI_STATUS_REC_TARGET_ABORT 0x1000 /* Master ack of " */
-#define  PCI_STATUS_REC_MASTER_ABORT 0x2000 /* Set on master abort */
-#define  PCI_STATUS_SIG_SYSTEM_ERROR 0x4000 /* Set when we drive SERR */
-#define  PCI_STATUS_DETECTED_PARITY 0x8000 /* Set on parity error */
-
-#define PCI_REVISION            0x08    /* 8 bits  */
-#define PCI_REVISION_ID         0x08    /* 8 bits  */
-#define PCI_CLASS_REVISION      0x08    /* 32 bits  */
-#define PCI_CLASS_CODE          0x0b    /* 8 bits */
-#define PCI_SUBCLASS_CODE       0x0a    /* 8 bits */
-#define PCI_HEADER_TYPE         0x0e    /* 8 bits */
-#define  PCI_HEADER_TYPE_NORMAL	0
-#define  PCI_HEADER_TYPE_BRIDGE 1
-#define  PCI_HEADER_TYPE_CARDBUS 2
-
-
-/* Header type 0 (normal devices) */
-#define PCI_CARDBUS_CIS		0x28
+/** PCI vendor ID */
+#define PCI_VENDOR_ID		0x00
+
+/** PCI device ID */
+#define PCI_DEVICE_ID		0x02
+
+/** PCI command */
+#define PCI_COMMAND		0x04
+#define PCI_COMMAND_IO			0x0001	/**< I/O space */
+#define PCI_COMMAND_MEM			0x0002	/**< Memory space */
+#define PCI_COMMAND_MASTER		0x0004	/**< Bus master */
+#define PCI_COMMAND_INVALIDATE		0x0010	/**< Mem. write & invalidate */
+#define PCI_COMMAND_PARITY		0x0040	/**< Parity error response */
+#define PCI_COMMAND_SERR		0x0100	/**< SERR# enable */
+#define PCI_COMMAND_INTX_DISABLE	0x0400	/**< Interrupt disable */
+
+/** PCI status */
+#define PCI_STATUS		0x06
+#define PCI_STATUS_CAP_LIST		0x0010	/**< Capabilities list */
+#define PCI_STATUS_PARITY		0x0100	/**< Master data parity error */
+#define PCI_STATUS_REC_TARGET_ABORT	0x1000	/**< Received target abort */
+#define PCI_STATUS_REC_MASTER_ABORT	0x2000	/**< Received master abort */
+#define PCI_STATUS_SIG_SYSTEM_ERROR	0x4000	/**< Signalled system error */
+#define PCI_STATUS_DETECTED_PARITY	0x8000	/**< Detected parity error */
+
+/** PCI revision */
+#define PCI_REVISION		0x08
+
+/** PCI cache line size */
+#define PCI_CACHE_LINE_SIZE	0x0c
+
+/** PCI latency timer */
+#define PCI_LATENCY_TIMER	0x0d
+
+/** PCI header type */
+#define PCI_HEADER_TYPE         0x0e
+#define PCI_HEADER_TYPE_NORMAL		0x00	/**< Normal header */
+#define PCI_HEADER_TYPE_BRIDGE		0x01	/**< PCI-to-PCI bridge header */
+#define PCI_HEADER_TYPE_CARDBUS		0x02	/**< CardBus header */
+#define PCI_HEADER_TYPE_MASK		0x7f	/**< Header type mask */
+#define PCI_HEADER_TYPE_MULTI		0x80	/**< Multi-function device */
+
+/** PCI base address registers */
+#define PCI_BASE_ADDRESS(n)	( 0x10 + ( 4 * (n) ) )
+#define PCI_BASE_ADDRESS_0	PCI_BASE_ADDRESS ( 0 )
+#define PCI_BASE_ADDRESS_1	PCI_BASE_ADDRESS ( 1 )
+#define PCI_BASE_ADDRESS_2	PCI_BASE_ADDRESS ( 2 )
+#define PCI_BASE_ADDRESS_3	PCI_BASE_ADDRESS ( 3 )
+#define PCI_BASE_ADDRESS_4	PCI_BASE_ADDRESS ( 4 )
+#define PCI_BASE_ADDRESS_5	PCI_BASE_ADDRESS ( 5 )
+#define PCI_BASE_ADDRESS_SPACE_IO	0x00000001UL	/**< I/O BAR */
+#define	PCI_BASE_ADDRESS_IO_MASK	0x00000003UL	/**< I/O BAR mask */
+#define PCI_BASE_ADDRESS_MEM_TYPE_64	0x00000004UL	/**< 64-bit memory */
+#define PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x00000006UL	/**< Memory type mask */
+#define	PCI_BASE_ADDRESS_MEM_MASK	0x0000000fUL	/**< Memory BAR mask */
+
+/** PCI subsystem vendor ID */
 #define PCI_SUBSYSTEM_VENDOR_ID	0x2c
+
+/** PCI subsystem ID */
 #define PCI_SUBSYSTEM_ID	0x2e  
 
-#define PCI_BASE_ADDRESS_0      0x10    /* 32 bits */
-#define PCI_BASE_ADDRESS_1      0x14    /* 32 bits */
-#define PCI_BASE_ADDRESS_2      0x18    /* 32 bits */
-#define PCI_BASE_ADDRESS_3      0x1c    /* 32 bits */
-#define PCI_BASE_ADDRESS_4      0x20    /* 32 bits */
-#define PCI_BASE_ADDRESS_5      0x24    /* 32 bits */
-
-#define PCI_BASE_ADDRESS_SPACE		0x01    /* 0 = memory, 1 = I/O */
-#define PCI_BASE_ADDRESS_SPACE_IO	0x01
-#define PCI_BASE_ADDRESS_SPACE_MEMORY	0x00
-
-#define PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x06
-#define PCI_BASE_ADDRESS_MEM_TYPE_32	0x00	/* 32 bit address */
-#define PCI_BASE_ADDRESS_MEM_TYPE_1M	0x02	/* Below 1M [obsolete] */
-#define PCI_BASE_ADDRESS_MEM_TYPE_64	0x04	/* 64 bit address */
-#define	PCI_BASE_ADDRESS_MEM_MASK	(~0x0f)
-#define	PCI_BASE_ADDRESS_IO_MASK	(~0x03)
-#define	PCI_ROM_ADDRESS		0x30	/* 32 bits */
-#define	PCI_ROM_ADDRESS_ENABLE	0x01	/* Write 1 to enable ROM,
-					   bits 31..11 are address,
-					   10..2 are reserved */
-
-#define PCI_CAPABILITY_LIST	0x34	/* Offset of first capability list entry */
-
-#define PCI_INTERRUPT_LINE	0x3c	/* IRQ number (0-15) */
-#define PCI_INTERRUPT_PIN	0x3d	/* IRQ pin on PCI bus (A-D) */
-
-/* Header type 1 (PCI-to-PCI bridges) */
-#define PCI_PRIMARY_BUS		0x18	/* Primary bus number */
-#define PCI_SECONDARY_BUS	0x19	/* Secondary bus number */
-#define PCI_SUBORDINATE_BUS	0x1a	/* Highest bus number behind the bridge */
-#define PCI_SEC_LATENCY_TIMER	0x1b	/* Latency timer for secondary interface */
-#define PCI_IO_BASE		0x1c	/* I/O range behind the bridge */
-#define PCI_IO_LIMIT		0x1d
-#define  PCI_IO_RANGE_TYPE_MASK	0x0f	/* I/O bridging type */
-#define  PCI_IO_RANGE_TYPE_16	0x00
-#define  PCI_IO_RANGE_TYPE_32	0x01
-#define  PCI_IO_RANGE_MASK	~0x0f
-#define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
-#define PCI_MEMORY_BASE		0x20	/* Memory range behind */
-#define PCI_MEMORY_LIMIT	0x22
-#define  PCI_MEMORY_RANGE_TYPE_MASK 0x0f
-#define  PCI_MEMORY_RANGE_MASK	~0x0f
-#define PCI_PREF_MEMORY_BASE	0x24	/* Prefetchable memory range behind */
-#define PCI_PREF_MEMORY_LIMIT	0x26
-#define  PCI_PREF_RANGE_TYPE_MASK 0x0f
-#define  PCI_PREF_RANGE_TYPE_32	0x00
-#define  PCI_PREF_RANGE_TYPE_64	0x01
-#define  PCI_PREF_RANGE_MASK	~0x0f
-#define PCI_PREF_BASE_UPPER32	0x28	/* Upper half of prefetchable memory range */
-#define PCI_PREF_LIMIT_UPPER32	0x2c
-#define PCI_IO_BASE_UPPER16	0x30	/* Upper half of I/O addresses */
-#define PCI_IO_LIMIT_UPPER16	0x32
-/* 0x34 same as for htype 0 */
-/* 0x35-0x3b is reserved */
-#define PCI_ROM_ADDRESS1	0x38	/* Same as PCI_ROM_ADDRESS, but for htype 1 */
-/* 0x3c-0x3d are same as for htype 0 */
-#define PCI_BRIDGE_CONTROL	0x3e
-#define  PCI_BRIDGE_CTL_PARITY	0x01	/* Enable parity detection on secondary interface */
-#define  PCI_BRIDGE_CTL_SERR	0x02	/* The same for SERR forwarding */
-#define  PCI_BRIDGE_CTL_NO_ISA	0x04	/* Disable bridging of ISA ports */
-#define  PCI_BRIDGE_CTL_VGA	0x08	/* Forward VGA addresses */
-#define  PCI_BRIDGE_CTL_MASTER_ABORT 0x20  /* Report master aborts */
-#define  PCI_BRIDGE_CTL_BUS_RESET 0x40	/* Secondary bus reset */
-#define  PCI_BRIDGE_CTL_FAST_BACK 0x80	/* Fast Back2Back enabled on secondary interface */
+/** PCI expansion ROM base address */
+#define	PCI_ROM_ADDRESS		0x30
 
+/** PCI capabilities pointer */
+#define PCI_CAPABILITY_LIST	0x34
+
+/** CardBus capabilities pointer */
 #define PCI_CB_CAPABILITY_LIST	0x14
 
-/* Capability lists */
-
-#define PCI_CAP_LIST_ID		0	/* Capability ID */
-#define  PCI_CAP_ID_PM		0x01	/* Power Management */
-#define  PCI_CAP_ID_AGP		0x02	/* Accelerated Graphics Port */
-#define  PCI_CAP_ID_VPD		0x03	/* Vital Product Data */
-#define  PCI_CAP_ID_SLOTID	0x04	/* Slot Identification */
-#define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
-#define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
-#define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific */
-#define  PCI_CAP_ID_EXP		0x10	/* PCI Express */
-#define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
-#define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
-#define PCI_CAP_SIZEOF		4
-
-/* Power Management Registers */
-
-#define PCI_PM_PMC              2       /* PM Capabilities Register */
-#define  PCI_PM_CAP_VER_MASK	0x0007	/* Version */
-#define  PCI_PM_CAP_PME_CLOCK	0x0008	/* PME clock required */
-#define  PCI_PM_CAP_RESERVED    0x0010  /* Reserved field */
-#define  PCI_PM_CAP_DSI		0x0020	/* Device specific initialization */
-#define  PCI_PM_CAP_AUX_POWER	0x01C0	/* Auxiliary power support mask */
-#define  PCI_PM_CAP_D1		0x0200	/* D1 power state support */
-#define  PCI_PM_CAP_D2		0x0400	/* D2 power state support */
-#define  PCI_PM_CAP_PME		0x0800	/* PME pin supported */
-#define  PCI_PM_CAP_PME_MASK    0xF800  /* PME Mask of all supported states */
-#define  PCI_PM_CAP_PME_D0      0x0800  /* PME# from D0 */
-#define  PCI_PM_CAP_PME_D1      0x1000  /* PME# from D1 */
-#define  PCI_PM_CAP_PME_D2      0x2000  /* PME# from D2 */
-#define  PCI_PM_CAP_PME_D3      0x4000  /* PME# from D3 (hot) */
-#define  PCI_PM_CAP_PME_D3cold  0x8000  /* PME# from D3 (cold) */
-#define PCI_PM_CTRL		4	/* PM control and status register */
-#define  PCI_PM_CTRL_STATE_MASK	0x0003	/* Current power state (D0 to D3) */
-#define  PCI_PM_CTRL_PME_ENABLE	0x0100	/* PME pin enable */
-#define  PCI_PM_CTRL_DATA_SEL_MASK	0x1e00	/* Data select (??) */
-#define  PCI_PM_CTRL_DATA_SCALE_MASK	0x6000	/* Data scale (??) */
-#define  PCI_PM_CTRL_PME_STATUS	0x8000	/* PME pin status */
-#define PCI_PM_PPB_EXTENSIONS	6	/* PPB support extensions (??) */
-#define  PCI_PM_PPB_B2_B3	0x40	/* Stop clock when in D3hot (??) */
-#define  PCI_PM_BPCC_ENABLE	0x80	/* Bus power/clock control enable (??) */
-#define PCI_PM_DATA_REGISTER	7	/* (??) */
-#define PCI_PM_SIZEOF		8
-
-/* AGP registers */
-
-#define PCI_AGP_VERSION		2	/* BCD version number */
-#define PCI_AGP_RFU		3	/* Rest of capability flags */
-#define PCI_AGP_STATUS		4	/* Status register */
-#define  PCI_AGP_STATUS_RQ_MASK	0xff000000	/* Maximum number of requests - 1 */
-#define  PCI_AGP_STATUS_SBA	0x0200	/* Sideband addressing supported */
-#define  PCI_AGP_STATUS_64BIT	0x0020	/* 64-bit addressing supported */
-#define  PCI_AGP_STATUS_FW	0x0010	/* FW transfers supported */
-#define  PCI_AGP_STATUS_RATE4	0x0004	/* 4x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE2	0x0002	/* 2x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE1	0x0001	/* 1x transfer rate supported */
-#define PCI_AGP_COMMAND		8	/* Control register */
-#define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
-#define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
-#define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
-#define  PCI_AGP_COMMAND_64BIT	0x0020 	/* Allow processing of 64-bit addresses */
-#define  PCI_AGP_COMMAND_FW	0x0010 	/* Force FW transfers */
-#define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
-#define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
-#define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
-#define PCI_AGP_SIZEOF		12
-
-/* Slot Identification */
-
-#define PCI_SID_ESR		2	/* Expansion Slot Register */
-#define  PCI_SID_ESR_NSLOTS	0x1f	/* Number of expansion slots available */
-#define  PCI_SID_ESR_FIC	0x20	/* First In Chassis Flag */
-#define PCI_SID_CHASSIS_NR	3	/* Chassis Number */
-
-/* Message Signalled Interrupts registers */
-
-#define PCI_MSI_FLAGS		2	/* Various flags */
-#define  PCI_MSI_FLAGS_64BIT	0x80	/* 64-bit addresses allowed */
-#define  PCI_MSI_FLAGS_QSIZE	0x70	/* Message queue size configured */
-#define  PCI_MSI_FLAGS_QMASK	0x0e	/* Maximum queue size available */
-#define  PCI_MSI_FLAGS_ENABLE	0x01	/* MSI feature enabled */
-#define PCI_MSI_RFU		3	/* Rest of capability flags */
-#define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
-#define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
-#define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
-#define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
-
-/* Advanced Error Reporting */
-
-#define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
-#define  PCI_ERR_UNC_TRAIN	0x00000001	/* Training */
-#define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
-#define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
-#define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
-#define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
-#define  PCI_ERR_UNC_COMP_ABORT	0x00008000	/* Completer Abort */
-#define  PCI_ERR_UNC_UNX_COMP	0x00010000	/* Unexpected Completion */
-#define  PCI_ERR_UNC_RX_OVER	0x00020000	/* Receiver Overflow */
-#define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
-#define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
-#define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
-#define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
-	/* Same bits as above */
-#define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
-	/* Same bits as above */
-#define PCI_ERR_COR_STATUS	16	/* Correctable Error Status */
-#define  PCI_ERR_COR_RCVR	0x00000001	/* Receiver Error Status */
-#define  PCI_ERR_COR_BAD_TLP	0x00000040	/* Bad TLP Status */
-#define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
-#define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
-#define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
-#define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
-	/* Same bits as above */
+/** PCI interrupt line */
+#define PCI_INTERRUPT_LINE	0x3c
+
+/** Capability ID */
+#define PCI_CAP_ID		0x00
+#define PCI_CAP_ID_PM			0x01	/**< Power management */
+#define PCI_CAP_ID_VPD			0x03	/**< Vital product data */
+#define PCI_CAP_ID_VNDR			0x09	/**< Vendor-specific */
+#define PCI_CAP_ID_EXP			0x10	/**< PCI Express */
+
+/** Next capability */
+#define PCI_CAP_NEXT		0x01
+
+/** Power management control and status */
+#define PCI_PM_CTRL		0x04
+#define PCI_PM_CTRL_STATE_MASK		0x0003	/**< Current power state */
+#define PCI_PM_CTRL_PME_ENABLE		0x0100	/**< PME pin enable */
+#define PCI_PM_CTRL_PME_STATUS		0x8000	/**< PME pin status */
+
+/** Uncorrectable error status */
+#define PCI_ERR_UNCOR_STATUS	0x04
+
+/** Network controller */
+#define PCI_CLASS_NETWORK	0x02
+
+/** Serial bus controller */
+#define PCI_CLASS_SERIAL	0x0c
+#define PCI_CLASS_SERIAL_USB		0x03	/**< USB controller */
+#define PCI_CLASS_SERIAL_USB_UHCI	 0x00	/**< UHCI USB controller */
+#define PCI_CLASS_SERIAL_USB_OHCI	 0x10	/**< OHCI USB controller */
+#define PCI_CLASS_SERIAL_USB_EHCI	 0x20	/**< ECHI USB controller */
+#define PCI_CLASS_SERIAL_USB_XHCI	 0x30	/**< xHCI USB controller */
+
+/** Construct PCI class
+ *
+ * @v base		Base class (or PCI_ANY_ID)
+ * @v sub		Subclass (or PCI_ANY_ID)
+ * @v progif		Programming interface (or PCI_ANY_ID)
+ */
+#define PCI_CLASS( base, sub, progif )					\
+	( ( ( (base) & 0xff ) << 16 ) |	( ( (sub) & 0xff ) << 8 ) |	\
+	  ( ( (progif) & 0xff) << 0 ) )
 
 /** A PCI device ID list entry */
 struct pci_device_id {
@@ -279,6 +143,27 @@ struct pci_device_id {
 /** Match-anything ID */
 #define PCI_ANY_ID 0xffff
 
+/** A PCI class ID */
+struct pci_class_id {
+	/** Class */
+	uint32_t class;
+	/** Class mask */
+	uint32_t mask;
+};
+
+/** Construct PCI class ID
+ *
+ * @v base		Base class (or PCI_ANY_ID)
+ * @v sub		Subclass (or PCI_ANY_ID)
+ * @v progif		Programming interface (or PCI_ANY_ID)
+ */
+#define PCI_CLASS_ID( base, sub, progif ) {				   \
+	.class = PCI_CLASS ( base, sub, progif ),			   \
+	.mask = ( ( ( ( (base) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 16 ) |   \
+		  ( ( ( (sub) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 8 ) |	   \
+		  ( ( ( (progif) == PCI_ANY_ID ) ? 0x00 : 0xff ) << 0 ) ), \
+	}
+
 /** A PCI device */
 struct pci_device {
 	/** Generic device */
@@ -322,6 +207,8 @@ struct pci_driver {
 	struct pci_device_id *ids;
 	/** Number of entries in PCI ID table */
 	unsigned int id_count;
+	/** PCI class ID */
+	struct pci_class_id class;
 	/**
 	 * Probe device
 	 *
@@ -352,6 +239,7 @@ struct pci_driver {
 #define PCI_BUSDEVFN( bus, slot, func )	\
 	( ( (bus) << 8 ) | ( (slot) << 3 ) | ( (func) << 0 ) )
 #define PCI_FIRST_FUNC( busdevfn )	( (busdevfn) & ~0x07 )
+#define PCI_LAST_FUNC( busdevfn )	( (busdevfn) | 0x07 )
 
 #define PCI_BASE_CLASS( class )		( (class) >> 16 )
 #define PCI_SUB_CLASS( class )		( ( (class) >> 8 ) & 0xff )
diff --git a/qemu/roms/ipxe/src/include/ipxe/pci_ids.h b/qemu/roms/ipxe/src/include/ipxe/pci_ids.h
deleted file mode 100644
index 25c7782bc..000000000
--- a/qemu/roms/ipxe/src/include/ipxe/pci_ids.h
+++ /dev/null
@@ -1,351 +0,0 @@
-#ifndef _IPXE_PCI_IDS_H
-#define _IPXE_PCI_IDS_H
-
-/*
- *	PCI Class, Vendor and Device IDs
- *
- *	Please keep sorted.
- */
-
-FILE_LICENCE ( GPL2_ONLY );
-
-/* Device classes and subclasses */
-
-#define PCI_CLASS_NOT_DEFINED		0x0000
-#define PCI_CLASS_NOT_DEFINED_VGA	0x0001
-
-#define PCI_BASE_CLASS_STORAGE		0x01
-#define PCI_CLASS_STORAGE_SCSI		0x0100
-#define PCI_CLASS_STORAGE_IDE		0x0101
-#define PCI_CLASS_STORAGE_FLOPPY	0x0102
-#define PCI_CLASS_STORAGE_IPI		0x0103
-#define PCI_CLASS_STORAGE_RAID		0x0104
-#define PCI_CLASS_STORAGE_OTHER		0x0180
-
-#define PCI_BASE_CLASS_NETWORK		0x02
-#define PCI_CLASS_NETWORK_ETHERNET	0x0200
-#define PCI_CLASS_NETWORK_TOKEN_RING	0x0201
-#define PCI_CLASS_NETWORK_FDDI		0x0202
-#define PCI_CLASS_NETWORK_ATM		0x0203
-#define PCI_CLASS_NETWORK_OTHER		0x0280
-
-#define PCI_BASE_CLASS_DISPLAY		0x03
-#define PCI_CLASS_DISPLAY_VGA		0x0300
-#define PCI_CLASS_DISPLAY_XGA		0x0301
-#define PCI_CLASS_DISPLAY_3D		0x0302
-#define PCI_CLASS_DISPLAY_OTHER		0x0380
-
-#define PCI_BASE_CLASS_MULTIMEDIA	0x04
-#define PCI_CLASS_MULTIMEDIA_VIDEO	0x0400
-#define PCI_CLASS_MULTIMEDIA_AUDIO	0x0401
-#define PCI_CLASS_MULTIMEDIA_PHONE	0x0402
-#define PCI_CLASS_MULTIMEDIA_OTHER	0x0480
-
-#define PCI_BASE_CLASS_MEMORY		0x05
-#define PCI_CLASS_MEMORY_RAM		0x0500
-#define PCI_CLASS_MEMORY_FLASH		0x0501
-#define PCI_CLASS_MEMORY_OTHER		0x0580
-
-#define PCI_BASE_CLASS_BRIDGE		0x06
-#define PCI_CLASS_BRIDGE_HOST		0x0600
-#define PCI_CLASS_BRIDGE_ISA		0x0601
-#define PCI_CLASS_BRIDGE_EISA		0x0602
-#define PCI_CLASS_BRIDGE_MC		0x0603
-#define PCI_CLASS_BRIDGE_PCI		0x0604
-#define PCI_CLASS_BRIDGE_PCMCIA		0x0605
-#define PCI_CLASS_BRIDGE_NUBUS		0x0606
-#define PCI_CLASS_BRIDGE_CARDBUS	0x0607
-#define PCI_CLASS_BRIDGE_RACEWAY	0x0608
-#define PCI_CLASS_BRIDGE_OTHER		0x0680
-
-#define PCI_BASE_CLASS_COMMUNICATION	0x07
-#define PCI_CLASS_COMMUNICATION_SERIAL	0x0700
-#define PCI_CLASS_COMMUNICATION_PARALLEL 0x0701
-#define PCI_CLASS_COMMUNICATION_MULTISERIAL 0x0702
-#define PCI_CLASS_COMMUNICATION_MODEM	0x0703
-#define PCI_CLASS_COMMUNICATION_OTHER	0x0780
-
-#define PCI_BASE_CLASS_SYSTEM		0x08
-#define PCI_CLASS_SYSTEM_PIC		0x0800
-#define PCI_CLASS_SYSTEM_DMA		0x0801
-#define PCI_CLASS_SYSTEM_TIMER		0x0802
-#define PCI_CLASS_SYSTEM_RTC		0x0803
-#define PCI_CLASS_SYSTEM_PCI_HOTPLUG	0x0804
-#define PCI_CLASS_SYSTEM_OTHER		0x0880
-
-#define PCI_BASE_CLASS_INPUT		0x09
-#define PCI_CLASS_INPUT_KEYBOARD	0x0900
-#define PCI_CLASS_INPUT_PEN		0x0901
-#define PCI_CLASS_INPUT_MOUSE		0x0902
-#define PCI_CLASS_INPUT_SCANNER		0x0903
-#define PCI_CLASS_INPUT_GAMEPORT	0x0904
-#define PCI_CLASS_INPUT_OTHER		0x0980
-
-#define PCI_BASE_CLASS_DOCKING		0x0a
-#define PCI_CLASS_DOCKING_GENERIC	0x0a00
-#define PCI_CLASS_DOCKING_OTHER		0x0a80
-
-#define PCI_BASE_CLASS_PROCESSOR	0x0b
-#define PCI_CLASS_PROCESSOR_386		0x0b00
-#define PCI_CLASS_PROCESSOR_486		0x0b01
-#define PCI_CLASS_PROCESSOR_PENTIUM	0x0b02
-#define PCI_CLASS_PROCESSOR_ALPHA	0x0b10
-#define PCI_CLASS_PROCESSOR_POWERPC	0x0b20
-#define PCI_CLASS_PROCESSOR_MIPS	0x0b30
-#define PCI_CLASS_PROCESSOR_CO		0x0b40
-
-#define PCI_BASE_CLASS_SERIAL		0x0c
-#define PCI_CLASS_SERIAL_FIREWIRE	0x0c00
-#define PCI_CLASS_SERIAL_ACCESS		0x0c01
-#define PCI_CLASS_SERIAL_SSA		0x0c02
-#define PCI_CLASS_SERIAL_USB		0x0c03
-#define PCI_CLASS_SERIAL_FIBER		0x0c04
-#define PCI_CLASS_SERIAL_SMBUS		0x0c05
-
-#define PCI_BASE_CLASS_INTELLIGENT	0x0e
-#define PCI_CLASS_INTELLIGENT_I2O	0x0e00
-
-#define PCI_BASE_CLASS_SATELLITE	0x0f
-#define PCI_CLASS_SATELLITE_TV		0x0f00
-#define PCI_CLASS_SATELLITE_AUDIO	0x0f01
-#define PCI_CLASS_SATELLITE_VOICE	0x0f03
-#define PCI_CLASS_SATELLITE_DATA	0x0f04
-
-#define PCI_BASE_CLASS_CRYPT		0x10
-#define PCI_CLASS_CRYPT_NETWORK		0x1000
-#define PCI_CLASS_CRYPT_ENTERTAINMENT	0x1001
-#define PCI_CLASS_CRYPT_OTHER		0x1080
-
-#define PCI_BASE_CLASS_SIGNAL_PROCESSING 0x11
-#define PCI_CLASS_SP_DPIO		0x1100
-#define PCI_CLASS_SP_OTHER		0x1180
-
-#define PCI_CLASS_OTHERS		0xff
-
-/* Vendors */
-
-#define PCI_VENDOR_ID_DYNALINK		0x0675
-#define PCI_VENDOR_ID_BERKOM			0x0871
-#define PCI_VENDOR_ID_COMPAQ		0x0e11
-#define PCI_VENDOR_ID_NCR		0x1000
-#define PCI_VENDOR_ID_LSI_LOGIC		0x1000
-#define PCI_VENDOR_ID_ATI		0x1002
-#define PCI_VENDOR_ID_VLSI		0x1004
-#define PCI_VENDOR_ID_ADL		0x1005
-#define PCI_VENDOR_ID_NS		0x100b
-#define PCI_VENDOR_ID_TSENG		0x100c
-#define PCI_VENDOR_ID_WEITEK		0x100e
-#define PCI_VENDOR_ID_DEC		0x1011
-#define PCI_VENDOR_ID_CIRRUS		0x1013
-#define PCI_VENDOR_ID_IBM		0x1014
-#define PCI_VENDOR_ID_COMPEX2		0x101a
-/* pci.ids says "AT&T GIS (NCR)" */
-#define PCI_VENDOR_ID_WD		0x101c
-#define PCI_VENDOR_ID_AMI		0x101e
-#define PCI_VENDOR_ID_AMD		0x1022
-#define PCI_VENDOR_ID_TRIDENT		0x1023
-#define PCI_VENDOR_ID_AI		0x1025
-#define PCI_VENDOR_ID_DELL              0x1028
-#define PCI_VENDOR_ID_MATROX		0x102B
-#define PCI_VENDOR_ID_CT		0x102c
-#define PCI_VENDOR_ID_MIRO		0x1031
-#define PCI_VENDOR_ID_NEC		0x1033
-#define PCI_VENDOR_ID_FD		0x1036
-#define PCI_VENDOR_ID_SIS         	0x1039
-#define PCI_VENDOR_ID_SI		0x1039
-#define PCI_VENDOR_ID_HP		0x103c
-#define PCI_VENDOR_ID_PCTECH		0x1042
-#define PCI_VENDOR_ID_ASUSTEK		0x1043
-#define PCI_VENDOR_ID_DPT		0x1044
-#define PCI_VENDOR_ID_OPTI		0x1045
-#define PCI_VENDOR_ID_ELSA		0x1048
-#define PCI_VENDOR_ID_ELSA		0x1048
-#define PCI_VENDOR_ID_SGS		0x104a
-#define PCI_VENDOR_ID_BUSLOGIC		      0x104B
-#define PCI_VENDOR_ID_TI		0x104c
-#define PCI_VENDOR_ID_SONY		0x104d
-#define PCI_VENDOR_ID_OAK		0x104e
-/* Winbond have two vendor IDs! See 0x10ad as well */
-#define PCI_VENDOR_ID_WINBOND2		0x1050
-#define PCI_VENDOR_ID_ANIGMA		0x1051
-#define PCI_VENDOR_ID_EFAR		0x1055
-#define PCI_VENDOR_ID_MOTOROLA		0x1057
-#define PCI_VENDOR_ID_MOTOROLA_OOPS	0x1507
-#define PCI_VENDOR_ID_PROMISE		0x105a
-#define PCI_VENDOR_ID_N9		0x105d
-#define PCI_VENDOR_ID_UMC		0x1060
-#define PCI_VENDOR_ID_X			0x1061
-#define PCI_VENDOR_ID_MYLEX		0x1069
-#define PCI_VENDOR_ID_PICOP		0x1066
-#define PCI_VENDOR_ID_APPLE		0x106b
-#define PCI_VENDOR_ID_YAMAHA		0x1073
-#define PCI_VENDOR_ID_NEXGEN		0x1074
-#define PCI_VENDOR_ID_QLOGIC		0x1077
-#define PCI_VENDOR_ID_CYRIX		0x1078
-#define PCI_VENDOR_ID_LEADTEK		0x107d
-#define PCI_VENDOR_ID_INTERPHASE	0x107e
-#define PCI_VENDOR_ID_CONTAQ		0x1080
-#define PCI_VENDOR_ID_FOREX		0x1083
-#define PCI_VENDOR_ID_OLICOM		0x108d
-#define PCI_VENDOR_ID_SUN		0x108e
-#define PCI_VENDOR_ID_CMD		0x1095
-#define PCI_VENDOR_ID_VISION		0x1098
-#define PCI_VENDOR_ID_BROOKTREE		0x109e
-#define PCI_VENDOR_ID_SIERRA		0x10a8
-#define PCI_VENDOR_ID_SGI		0x10a9
-#define PCI_VENDOR_ID_ACC		0x10aa
-#define PCI_VENDOR_ID_WINBOND		0x10ad
-#define PCI_VENDOR_ID_DATABOOK		0x10b3
-#define PCI_VENDOR_ID_PLX		0x10b5
-#define PCI_VENDOR_ID_MADGE		0x10b6
-#define PCI_VENDOR_ID_3COM		0x10b7
-#define PCI_VENDOR_ID_SMC		0x10b8
-#define PCI_VENDOR_ID_SUNDANCE		0x13F0
-#define PCI_VENDOR_ID_AL		0x10b9
-#define PCI_VENDOR_ID_MITSUBISHI	0x10ba
-#define PCI_VENDOR_ID_SURECOM		0x10bd
-#define PCI_VENDOR_ID_NEOMAGIC		0x10c8
-#define PCI_VENDOR_ID_ASP		0x10cd
-#define PCI_VENDOR_ID_MACRONIX		0x10d9
-#define PCI_VENDOR_ID_TCONRAD		0x10da
-#define PCI_VENDOR_ID_CERN		0x10dc
-#define PCI_VENDOR_ID_NVIDIA			0x10de
-#define PCI_VENDOR_ID_IMS		0x10e0
-#define PCI_VENDOR_ID_TEKRAM2		0x10e1
-#define PCI_VENDOR_ID_TUNDRA		0x10e3
-#define PCI_VENDOR_ID_AMCC		0x10e8
-#define PCI_VENDOR_ID_INTERG		0x10ea
-#define PCI_VENDOR_ID_REALTEK		0x10ec
-#define PCI_VENDOR_ID_XILINX		0x10ee
-#define PCI_VENDOR_ID_TRUEVISION	0x10fa
-#define PCI_VENDOR_ID_INIT		0x1101
-#define PCI_VENDOR_ID_CREATIVE		0x1102
-/* duplicate: ECTIVA */
-#define PCI_VENDOR_ID_ECTIVA		0x1102
-/* duplicate: CREATIVE */
-#define PCI_VENDOR_ID_TTI		0x1103
-#define PCI_VENDOR_ID_VIA		0x1106
-#define PCI_VENDOR_ID_VIATEC		0x1106
-#define PCI_VENDOR_ID_SIEMENS           0x110A
-#define PCI_VENDOR_ID_SMC2		0x1113
-#define PCI_VENDOR_ID_VORTEX		0x1119
-#define PCI_VENDOR_ID_EF		0x111a
-#define PCI_VENDOR_ID_IDT		0x111d
-#define PCI_VENDOR_ID_FORE		0x1127
-#define PCI_VENDOR_ID_IMAGINGTECH	0x112f
-#define PCI_VENDOR_ID_PHILIPS		0x1131
-#define PCI_VENDOR_ID_EICON		0x1133
-#define PCI_VENDOR_ID_CYCLONE		0x113c
-#define PCI_VENDOR_ID_ALLIANCE		0x1142
-#define PCI_VENDOR_ID_SYSKONNECT	0x1148
-#define PCI_VENDOR_ID_VMIC		0x114a
-#define PCI_VENDOR_ID_DIGI		0x114f
-#define PCI_VENDOR_ID_MUTECH		0x1159
-#define PCI_VENDOR_ID_XIRCOM		0x115d
-#define PCI_VENDOR_ID_RENDITION		0x1163
-#define PCI_VENDOR_ID_SERVERWORKS	  0x1166
-#define PCI_VENDOR_ID_SBE		0x1176
-#define PCI_VENDOR_ID_TOSHIBA		0x1179
-#define PCI_VENDOR_ID_RICOH		0x1180
-#define	PCI_VENDOR_ID_DLINK		0x1186
-#define PCI_VENDOR_ID_ARTOP		0x1191
-#define PCI_VENDOR_ID_ZEITNET		0x1193
-#define PCI_VENDOR_ID_OMEGA		0x119b
-#define PCI_VENDOR_ID_FUJITSU_ME	0x119e
-#define PCI_SUBVENDOR_ID_KEYSPAN	0x11a9
-#define PCI_VENDOR_ID_GALILEO		0x11ab
-#define PCI_VENDOR_ID_LINKSYS		0x11ad
-#define PCI_VENDOR_ID_LITEON		0x11ad
-#define PCI_VENDOR_ID_V3		0x11b0
-#define PCI_VENDOR_ID_NP		0x11bc
-#define PCI_VENDOR_ID_ATT		0x11c1
-#define PCI_VENDOR_ID_SPECIALIX		0x11cb
-#define PCI_VENDOR_ID_AURAVISION	0x11d1
-#define PCI_VENDOR_ID_ANALOG_DEVICES	0x11d4
-#define PCI_VENDOR_ID_IKON		0x11d5
-#define PCI_VENDOR_ID_ZORAN		0x11de
-#define PCI_VENDOR_ID_KINETIC		0x11f4
-#define PCI_VENDOR_ID_COMPEX		0x11f6
-#define PCI_VENDOR_ID_RP		0x11fe
-#define PCI_VENDOR_ID_CYCLADES		0x120e
-#define PCI_VENDOR_ID_ESSENTIAL		0x120f
-#define PCI_VENDOR_ID_O2		0x1217
-#define PCI_VENDOR_ID_3DFX		0x121a
-#define PCI_VENDOR_ID_SIGMADES		0x1236
-#define PCI_VENDOR_ID_CCUBE		0x123f
-#define PCI_VENDOR_ID_AVM		0x1244
-#define PCI_VENDOR_ID_DIPIX		0x1246
-#define PCI_VENDOR_ID_STALLION		0x124d
-#define PCI_VENDOR_ID_OPTIBASE		0x1255
-#define PCI_VENDOR_ID_ESS		0x125d
-#define PCI_VENDOR_ID_HARRIS        	0x1260
-#define PCI_VENDOR_ID_SATSAGEM		0x1267
-#define PCI_VENDOR_ID_HUGHES		0x1273
-#define PCI_VENDOR_ID_ENSONIQ		0x1274
-#define PCI_VENDOR_ID_ROCKWELL		0x127A
-#define PCI_VENDOR_ID_DAVICOM		0x1282
-#define PCI_VENDOR_ID_ITE		0x1283
-/* formerly Platform Tech */
-#define PCI_VENDOR_ID_ESS_OLD		0x1285
-#define PCI_VENDOR_ID_ALTEON		0x12ae
-#define PCI_VENDOR_ID_USR		0x12B9
-#define	PCI_VENDOR_ID_HOLTEK		0x12c3
-#define PCI_SUBVENDOR_ID_CONNECT_TECH			0x12c4
-#define PCI_VENDOR_ID_PICTUREL		0x12c5
-#define PCI_VENDOR_ID_NVIDIA_SGS	0x12d2
-#define PCI_SUBVENDOR_ID_CHASE_PCIFAST		0x12E0
-#define PCI_SUBVENDOR_ID_CHASE_PCIRAS		0x124D
-#define PCI_VENDOR_ID_AUREAL		0x12eb
-#define PCI_VENDOR_ID_CBOARDS		0x1307
-#define PCI_VENDOR_ID_SIIG		0x131f
-#define PCI_VENDOR_ID_ADMTEK            0x1317
-#define PCI_VENDOR_ID_DOMEX		0x134a
-#define PCI_VENDOR_ID_QUATECH		0x135C
-#define PCI_VENDOR_ID_SEALEVEL		0x135e
-#define PCI_VENDOR_ID_HYPERCOPE		0x1365
-#define PCI_VENDOR_ID_KAWASAKI		0x136b
-#define PCI_VENDOR_ID_LMC		0x1376
-#define PCI_VENDOR_ID_NETGEAR		0x1385
-#define PCI_VENDOR_ID_APPLICOM		0x1389
-#define PCI_VENDOR_ID_MOXA		0x1393
-#define PCI_VENDOR_ID_CCD		0x1397
-#define PCI_VENDOR_ID_MICROGATE		0x13c0
-#define PCI_VENDOR_ID_3WARE		0x13C1
-#define PCI_VENDOR_ID_ABOCOM		0x13D1
-#define PCI_VENDOR_ID_CMEDIA		0x13f6
-#define PCI_VENDOR_ID_LAVA		0x1407
-#define PCI_VENDOR_ID_TIMEDIA		0x1409
-#define PCI_VENDOR_ID_OXSEMI		0x1415
-#define PCI_VENDOR_ID_AIRONET		0x14b9
-#define PCI_VENDOR_ID_MYRICOM		0x14c1
-#define PCI_VENDOR_ID_TITAN		0x14D2
-#define PCI_VENDOR_ID_PANACOM		0x14d4
-#define PCI_VENDOR_ID_BROADCOM		0x14e4
-#define PCI_VENDOR_ID_SYBA		0x1592
-#define PCI_VENDOR_ID_MORETON		0x15aa
-#define PCI_VENDOR_ID_ZOLTRIX		0x15b0
-#define PCI_VENDOR_ID_PDC		0x15e9
-#define PCI_VENDOR_ID_FSC		0x1734
-#define PCI_VENDOR_ID_SYMPHONY		0x1c1c
-#define PCI_VENDOR_ID_TEKRAM		0x1de1
-#define PCI_VENDOR_ID_3DLABS		0x3d3d
-#define PCI_VENDOR_ID_AVANCE		0x4005
-#define PCI_VENDOR_ID_AKS		0x416c
-#define PCI_VENDOR_ID_NETVIN		0x4a14
-#define PCI_VENDOR_ID_S3		0x5333
-#define PCI_VENDOR_ID_DCI		0x6666
-#define PCI_VENDOR_ID_GENROCO		0x5555
-#define PCI_VENDOR_ID_INTEL		0x8086
-#define PCI_VENDOR_ID_COMPUTONE		0x8e0e
-#define PCI_SUBVENDOR_ID_COMPUTONE	0x8e0e
-#define PCI_VENDOR_ID_KTI		0x8e2e
-#define PCI_VENDOR_ID_ADAPTEC		0x9004
-#define PCI_VENDOR_ID_ADAPTEC2		0x9005
-#define PCI_VENDOR_ID_ATRONICS		0x907f
-#define PCI_VENDOR_ID_HOLTEK2		0x9412
-#define PCI_VENDOR_ID_NETMOS		0x9710
-#define PCI_SUBVENDOR_ID_EXSYS		0xd84d
-#define PCI_VENDOR_ID_TIGERJET		0xe159
-#define PCI_VENDOR_ID_ARK		0xedd8
-
-#endif /* _IPXE_PCI_IDS_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/pci_io.h b/qemu/roms/ipxe/src/include/ipxe/pci_io.h
index 781b77fe1..10e69763e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pci_io.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pci_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pcibackup.h b/qemu/roms/ipxe/src/include/ipxe/pcibackup.h
index b9f55cf71..159d25392 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pcibackup.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pcibackup.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/pcivpd.h b/qemu/roms/ipxe/src/include/ipxe/pcivpd.h
index 0abf8a956..fefb69740 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pcivpd.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pcivpd.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/peerblk.h b/qemu/roms/ipxe/src/include/ipxe/peerblk.h
new file mode 100644
index 000000000..6fc9172f6
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/peerblk.h
@@ -0,0 +1,144 @@
+#ifndef _IPXE_PEERBLK_H
+#define _IPXE_PEERBLK_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol block downloads
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/crypto.h>
+#include <ipxe/aes.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/retry.h>
+#include <ipxe/process.h>
+#include <ipxe/pccrc.h>
+#include <ipxe/peerdisc.h>
+
+/** A PeerDist retrieval protocol decryption buffer descriptor */
+struct peerdist_block_decrypt {
+	/** Data transfer buffer */
+	struct xfer_buffer *xferbuf;
+	/** Offset within data transfer buffer */
+	size_t offset;
+	/** Length to use from data transfer buffer */
+	size_t len;
+};
+
+/** PeerDist retrieval protocol decryption data transfer buffer indices */
+enum peerdist_block_decrypt_index {
+	/** Data before the trimmed content */
+	PEERBLK_BEFORE = 0,
+	/** Data within the trimmed content */
+	PEERBLK_DURING,
+	/** Data after the trimmed content */
+	PEERBLK_AFTER,
+	/** Number of decryption buffers */
+	PEERBLK_NUM_BUFFERS
+};
+
+/** A PeerDist block download */
+struct peerdist_block {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Raw data interface */
+	struct interface raw;
+	/** Retrieval protocol interface */
+	struct interface retrieval;
+
+	/** Original URI */
+	struct uri *uri;
+	/** Content range of this block */
+	struct peerdist_range range;
+	/** Trimmed range of this block */
+	struct peerdist_range trim;
+	/** Offset of first byte in trimmed range within overall download */
+	size_t offset;
+
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Digest size
+	 *
+	 * Note that this may be shorter than the digest size of the
+	 * digest algorithm.
+	 */
+	size_t digestsize;
+	/** Digest context (statically allocated at instantiation time) */
+	void *digestctx;
+
+	/** Cipher algorithm */
+	struct cipher_algorithm *cipher;
+	/** Cipher context (dynamically allocated as needed) */
+	void *cipherctx;
+
+	/** Segment index */
+	unsigned int segment;
+	/** Segment identifier */
+	uint8_t id[PEERDIST_DIGEST_MAX_SIZE];
+	/** Segment secret */
+	uint8_t secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Block index */
+	unsigned int block;
+	/** Block hash */
+	uint8_t hash[PEERDIST_DIGEST_MAX_SIZE];
+
+	/** Current position (relative to incoming data stream) */
+	size_t pos;
+	/** Start of trimmed content (relative to incoming data stream) */
+	size_t start;
+	/** End of trimmed content (relative to incoming data stream) */
+	size_t end;
+	/** Data buffer */
+	struct xfer_buffer buffer;
+
+	/** Decryption process */
+	struct process process;
+	/** Decryption data buffer descriptors */
+	struct peerdist_block_decrypt decrypt[PEERBLK_NUM_BUFFERS];
+	/** Remaining decryption length */
+	size_t cipher_remaining;
+	/** Remaining digest length (excluding AES padding bytes) */
+	size_t digest_remaining;
+
+	/** Discovery client */
+	struct peerdisc_client discovery;
+	/** Current position in discovered peer list */
+	struct peerdisc_peer *peer;
+	/** Retry timer */
+	struct retry_timer timer;
+	/** Number of full attempt cycles completed */
+	unsigned int cycles;
+	/** Most recent attempt failure */
+	int rc;
+
+	/** Time at which block download was started */
+	unsigned long started;
+	/** Time at which most recent attempt was started */
+	unsigned long attempted;
+};
+
+/** Retrieval protocol block fetch response (including transport header)
+ *
+ * @v digestsize	Digest size
+ * @v len		Data block length
+ * @v vrf_len		Length of uselessness
+ * @v blksize		Cipher block size
+ */
+#define peerblk_msg_blk_t( digestsize, len, vrf_len, blksize )		\
+	struct {							\
+		struct peerdist_msg_transport_header hdr;		\
+		peerdist_msg_blk_t ( digestsize, len, vrf_len,		\
+				     blksize ) msg;			\
+	} __attribute__ (( packed ))
+
+extern int peerblk_open ( struct interface *xfer, struct uri *uri,
+			  struct peerdist_info_block *block );
+
+#endif /* _IPXE_PEERBLK_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/peerdisc.h b/qemu/roms/ipxe/src/include/ipxe/peerdisc.h
new file mode 100644
index 000000000..f08ccaae2
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/peerdisc.h
@@ -0,0 +1,116 @@
+#ifndef _IPXE_PEERDISC_H
+#define _IPXE_PEERDISC_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol peer discovery
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/list.h>
+#include <ipxe/tables.h>
+#include <ipxe/retry.h>
+#include <ipxe/socket.h>
+#include <ipxe/interface.h>
+#include <ipxe/pccrc.h>
+
+/** A PeerDist discovery socket */
+struct peerdisc_socket {
+	/** Name */
+	const char *name;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Socket address */
+	union {
+		struct sockaddr sa;
+		struct sockaddr_in sin;
+		struct sockaddr_in6 sin6;
+	} address;
+};
+
+/** PeerDist discovery socket table */
+#define PEERDISC_SOCKETS __table ( struct peerdisc_socket, "peerdisc_sockets" )
+
+/** Declare a PeerDist discovery socket */
+#define __peerdisc_socket __table_entry ( PEERDISC_SOCKETS, 01 )
+
+/** A PeerDist discovery segment */
+struct peerdisc_segment {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** List of segments */
+	struct list_head list;
+	/** Segment identifier string
+	 *
+	 * This is MS-PCCRC's "HoHoDk", transcribed as an upper-case
+	 * Base16-encoded string.
+	 */
+	const char *id;
+	/** Message UUID string */
+	const char *uuid;
+	/** List of discovered peers
+	 *
+	 * The list of peers may be appended to during the lifetime of
+	 * the discovery segment.  Discovered peers will not be
+	 * removed from the list until the last discovery has been
+	 * closed; this allows users to safely maintain a pointer to a
+	 * current position within the list.
+	 */
+	struct list_head peers;
+	/** List of active clients */
+	struct list_head clients;
+	/** Transmission timer */
+	struct retry_timer timer;
+};
+
+/** A PeerDist discovery peer */
+struct peerdisc_peer {
+	/** List of peers */
+	struct list_head list;
+	/** Peer location */
+	char location[0];
+};
+
+/** A PeerDist discovery client */
+struct peerdisc_client {
+	/** Discovery segment */
+	struct peerdisc_segment *segment;
+	/** List of clients */
+	struct list_head list;
+	/** Operations */
+	struct peerdisc_client_operations *op;
+};
+
+/** PeerDist discovery client operations */
+struct peerdisc_client_operations {
+	/** New peers have been discovered
+	 *
+	 * @v peerdisc		PeerDist discovery client
+	 */
+	void ( * discovered ) ( struct peerdisc_client *peerdisc );
+};
+
+/**
+ * Initialise PeerDist discovery
+ *
+ * @v peerdisc		PeerDist discovery client
+ * @v op		Discovery operations
+ */
+static inline __attribute__ (( always_inline )) void
+peerdisc_init ( struct peerdisc_client *peerdisc,
+		struct peerdisc_client_operations *op ) {
+
+	peerdisc->op = op;
+}
+
+extern unsigned int peerdisc_timeout_secs;
+
+extern int peerdisc_open ( struct peerdisc_client *peerdisc, const void *id,
+			   size_t len );
+extern void peerdisc_close ( struct peerdisc_client *peerdisc );
+
+#endif /* _IPXE_PEERDISC_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/peermux.h b/qemu/roms/ipxe/src/include/ipxe/peermux.h
new file mode 100644
index 000000000..44cbdb9d6
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/peermux.h
@@ -0,0 +1,73 @@
+#ifndef _IPXE_PEERMUX_H
+#define _IPXE_PEERMUX_H
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol multiplexer
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/list.h>
+#include <ipxe/refcnt.h>
+#include <ipxe/interface.h>
+#include <ipxe/process.h>
+#include <ipxe/uri.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/pccrc.h>
+
+/** Maximum number of concurrent block downloads */
+#define PEERMUX_MAX_BLOCKS 32
+
+/** PeerDist download content information cache */
+struct peerdist_info_cache {
+	/** Content information */
+	struct peerdist_info info;
+	/** Content information segment */
+	struct peerdist_info_segment segment;
+	/** Content information block */
+	struct peerdist_info_block block;
+};
+
+/** A PeerDist multiplexed block download */
+struct peerdist_multiplexed_block {
+	/** PeerDist download multiplexer */
+	struct peerdist_multiplexer *peermux;
+	/** List of multiplexed blocks */
+	struct list_head list;
+	/** Data transfer interface */
+	struct interface xfer;
+};
+
+/** A PeerDist download multiplexer */
+struct peerdist_multiplexer {
+	/** Reference count */
+	struct refcnt refcnt;
+	/** Data transfer interface */
+	struct interface xfer;
+	/** Content information interface */
+	struct interface info;
+	/** Original URI */
+	struct uri *uri;
+
+	/** Content information data transfer buffer */
+	struct xfer_buffer buffer;
+	/** Content information cache */
+	struct peerdist_info_cache cache;
+
+	/** Block download initiation process */
+	struct process process;
+	/** List of busy block downloads */
+	struct list_head busy;
+	/** List of idle block downloads */
+	struct list_head idle;
+	/** Block downloads */
+	struct peerdist_multiplexed_block block[PEERMUX_MAX_BLOCKS];
+};
+
+extern int peermux_filter ( struct interface *xfer, struct interface *info,
+			    struct uri *uri );
+
+#endif /* _IPXE_PEERMUX_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/pending.h b/qemu/roms/ipxe/src/include/ipxe/pending.h
index e6a369813..be6ed05a1 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pending.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pending.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A pending operation */
 struct pending_operation {
diff --git a/qemu/roms/ipxe/src/include/ipxe/ping.h b/qemu/roms/ipxe/src/include/ipxe/ping.h
index 6cd376b6f..c55bd1ab2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/ping.h
+++ b/qemu/roms/ipxe/src/include/ipxe/ping.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/iobuf.h>
 #include <ipxe/tcpip.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pinger.h b/qemu/roms/ipxe/src/include/ipxe/pinger.h
index 9932df6b0..227f002dc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pinger.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pinger.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/interface.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pixbuf.h b/qemu/roms/ipxe/src/include/ipxe/pixbuf.h
index 106b666e6..615744812 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pixbuf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pixbuf.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/png.h b/qemu/roms/ipxe/src/include/ipxe/png.h
index f51d1e6fe..3505eefc8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/png.h
+++ b/qemu/roms/ipxe/src/include/ipxe/png.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pnm.h b/qemu/roms/ipxe/src/include/ipxe/pnm.h
index 536c14d5f..860968cbc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/pnm.h
+++ b/qemu/roms/ipxe/src/include/ipxe/pnm.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/pool.h b/qemu/roms/ipxe/src/include/ipxe/pool.h
new file mode 100644
index 000000000..27066e9b3
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/pool.h
@@ -0,0 +1,127 @@
+#ifndef _IPXE_POOL_H
+#define _IPXE_POOL_H
+
+/** @file
+ *
+ * Pooled connections
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/interface.h>
+#include <ipxe/list.h>
+#include <ipxe/retry.h>
+
+/** A pooled connection */
+struct pooled_connection {
+	/** List of pooled connections
+	 *
+	 * Note that each connecton in the pool has a running expiry
+	 * timer which holds a reference to the connection.  We
+	 * therefore do not require the connection pool list to hold a
+	 * reference for each pooled connection.
+	 */
+	struct list_head list;
+	/** Expiry timer */
+	struct retry_timer timer;
+	/** Close expired pooled connection
+	 *
+	 * @v pool		Pooled connection
+	 */
+	void ( * expired ) ( struct pooled_connection *pool );
+	/** Flags */
+	unsigned int flags;
+};
+
+/** Pooled connection flags */
+enum pooled_connection_flags {
+	/** Connection should be recycled after closing */
+	POOL_RECYCLABLE = 0x0001,
+	/** Connection has been recycled */
+	POOL_RECYCLED = 0x0002,
+	/** Connection is known to be alive */
+	POOL_ALIVE = 0x0004,
+};
+
+extern void pool_add ( struct pooled_connection *pool, struct list_head *list,
+		       unsigned long expiry );
+extern void pool_del ( struct pooled_connection *pool );
+extern void pool_expired ( struct retry_timer *timer, int over );
+
+/**
+ * Initialise a pooled connection
+ *
+ * @v pool		Pooled connection
+ * @v expired		Close expired pooled connection method
+ * @v refcnt		Containing object reference counter
+ */
+static inline __attribute__ (( always_inline )) void
+pool_init ( struct pooled_connection *pool,
+	    void ( * expired ) ( struct pooled_connection *pool ),
+	    struct refcnt *refcnt ) {
+
+	INIT_LIST_HEAD ( &pool->list );
+	timer_init ( &pool->timer, pool_expired, refcnt );
+	pool->expired = expired;
+}
+
+/**
+ * Mark pooled connection as recyclable
+ *
+ * @v pool		Pooled connection
+ */
+static inline __attribute__ (( always_inline )) void
+pool_recyclable ( struct pooled_connection *pool ) {
+
+	pool->flags |= POOL_RECYCLABLE;
+}
+
+/**
+ * Mark pooled connection as alive
+ *
+ * @v pool		Pooled connection
+ */
+static inline __attribute__ (( always_inline )) void
+pool_alive ( struct pooled_connection *pool ) {
+
+	pool->flags |= POOL_ALIVE;
+}
+
+/**
+ * Check if pooled connection is recyclable
+ *
+ * @v pool		Pooled connection
+ * @ret recyclable	Pooled connection is recyclable
+ */
+static inline __attribute__ (( always_inline )) int
+pool_is_recyclable ( struct pooled_connection *pool ) {
+
+	return ( pool->flags & POOL_RECYCLABLE );
+}
+
+/**
+ * Check if pooled connection is reopenable
+ *
+ * @v pool		Pooled connection
+ * @ret reopenable	Pooled connection is reopenable
+ */
+static inline __attribute__ (( always_inline )) int
+pool_is_reopenable ( struct pooled_connection *pool ) {
+
+	/* A connection is reopenable if it has been recycled but is
+	 * not yet known to be alive.
+	 */
+	return ( ( pool->flags & POOL_RECYCLED ) &
+		 ( ! ( pool->flags & POOL_ALIVE ) ) );
+}
+
+extern void pool_recycle ( struct interface *intf );
+#define pool_recycle_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
+extern void pool_reopen ( struct interface *intf );
+#define pool_reopen_TYPE( object_type ) \
+	typeof ( void ( object_type ) )
+
+#endif /* _IPXE_POOL_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/portmap.h b/qemu/roms/ipxe/src/include/ipxe/portmap.h
index 9b735bbca..681ca2ec2 100644
--- a/qemu/roms/ipxe/src/include/ipxe/portmap.h
+++ b/qemu/roms/ipxe/src/include/ipxe/portmap.h
@@ -10,7 +10,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** PORTMAP default port */
 #define PORTMAP_PORT   111
diff --git a/qemu/roms/ipxe/src/include/ipxe/posix_io.h b/qemu/roms/ipxe/src/include/ipxe/posix_io.h
index 11f3bb5c9..1a73b5e86 100644
--- a/qemu/roms/ipxe/src/include/ipxe/posix_io.h
+++ b/qemu/roms/ipxe/src/include/ipxe/posix_io.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/uaccess.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/privkey.h b/qemu/roms/ipxe/src/include/ipxe/privkey.h
index 39049ac9f..81108b6bf 100644
--- a/qemu/roms/ipxe/src/include/ipxe/privkey.h
+++ b/qemu/roms/ipxe/src/include/ipxe/privkey.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/asn1.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/process.h b/qemu/roms/ipxe/src/include/ipxe/process.h
index 2c76ff260..d600508e7 100644
--- a/qemu/roms/ipxe/src/include/ipxe/process.h
+++ b/qemu/roms/ipxe/src/include/ipxe/process.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/profile.h b/qemu/roms/ipxe/src/include/ipxe/profile.h
index 3a745fcfa..b6d2b19e0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/profile.h
+++ b/qemu/roms/ipxe/src/include/ipxe/profile.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <bits/profile.h>
 #include <ipxe/tables.h>
@@ -186,4 +186,18 @@ profile_exclude ( struct profiler *profiler ) {
 		profile_excluded += profile_elapsed ( profiler );
 }
 
+/**
+ * Record profiling sample in custom units
+ *
+ * @v profiler		Profiler
+ * @v sample		Profiling sample
+ */
+static inline __attribute__ (( always_inline )) void
+profile_custom ( struct profiler *profiler, unsigned long sample ) {
+
+	/* If profiling is active then update stats */
+	if ( PROFILING )
+		profile_update ( profiler, sample );
+}
+
 #endif /* _IPXE_PROFILE_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/random_nz.h b/qemu/roms/ipxe/src/include/ipxe/random_nz.h
index 6bb80d2ab..4c433fa38 100644
--- a/qemu/roms/ipxe/src/include/ipxe/random_nz.h
+++ b/qemu/roms/ipxe/src/include/ipxe/random_nz.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/rarp.h b/qemu/roms/ipxe/src/include/ipxe/rarp.h
index f84301a43..9054db21a 100644
--- a/qemu/roms/ipxe/src/include/ipxe/rarp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/rarp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/netdevice.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/rbg.h b/qemu/roms/ipxe/src/include/ipxe/rbg.h
index 9689142f8..758238a65 100644
--- a/qemu/roms/ipxe/src/include/ipxe/rbg.h
+++ b/qemu/roms/ipxe/src/include/ipxe/rbg.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/drbg.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/reboot.h b/qemu/roms/ipxe/src/include/ipxe/reboot.h
index 97e0d5fb6..33606d9d5 100644
--- a/qemu/roms/ipxe/src/include/ipxe/reboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/reboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/reboot.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/refcnt.h b/qemu/roms/ipxe/src/include/ipxe/refcnt.h
index 0e8b8658c..7f489abc9 100644
--- a/qemu/roms/ipxe/src/include/ipxe/refcnt.h
+++ b/qemu/roms/ipxe/src/include/ipxe/refcnt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <assert.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/resolv.h b/qemu/roms/ipxe/src/include/ipxe/resolv.h
index d9868a5d7..ff48d35ca 100644
--- a/qemu/roms/ipxe/src/include/ipxe/resolv.h
+++ b/qemu/roms/ipxe/src/include/ipxe/resolv.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/retry.h b/qemu/roms/ipxe/src/include/ipxe/retry.h
index c514822b2..76d45fbd0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/retry.h
+++ b/qemu/roms/ipxe/src/include/ipxe/retry.h
@@ -7,14 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/list.h>
 
-/** Default timeout value */
+/** Default minimum timeout value (in ticks) */
 #define DEFAULT_MIN_TIMEOUT ( TICKS_PER_SEC / 4 )
 
-/** Limit after which the timeout will be deemed permanent */
+/** Default maximum timeout value (in ticks) */
 #define DEFAULT_MAX_TIMEOUT ( 10 * TICKS_PER_SEC )
 
 /** A retry timer */
@@ -25,16 +25,18 @@ struct retry_timer {
 	unsigned int running;
 	/** Timeout value (in ticks) */
 	unsigned long timeout;
-	/** Minimum timeout value (in ticks)
+	/** Minimum timeout value (in ticks), or zero to use default
 	 *
-	 * A value of zero means "use default timeout."
+	 * The timeout will never be reduced below this value.
 	 */
-	unsigned long min_timeout;
-	/** Maximum timeout value before failure (in ticks)
+	unsigned long min;
+	/** Maximum timeout value (in ticks), or zero to use default
 	 *
-	 * A value of zero means "use default timeout."
+	 * The timeout will be deemed permanent (according to the
+	 * failure indicator passed to expired()) when it exceeds this
+	 * value.
 	 */
-	unsigned long max_timeout;
+	unsigned long max;
 	/** Start time (in ticks) */
 	unsigned long start;
 	/** Retry count */
@@ -46,7 +48,7 @@ struct retry_timer {
 	 *
 	 * The timer will already be stopped when this method is
 	 * called.  The failure indicator will be True if the retry
-	 * timeout has already exceeded @c MAX_TIMEOUT.
+	 * timeout has already exceeded @c max_timeout.
 	 */
 	void ( * expired ) ( struct retry_timer *timer, int over );
 	/** Reference counter
@@ -109,4 +111,18 @@ timer_running ( struct retry_timer *timer ) {
 	return ( timer->running );
 }
 
+/**
+ * Set minimum and maximum timeouts
+ *
+ * @v timer		Retry timer
+ * @v min		Minimum timeout (in ticks), or zero to use default
+ * @v max		Maximum timeout (in ticks), or zero to use default
+ */
+static inline __attribute__ (( always_inline )) void
+set_timer_limits ( struct retry_timer *timer, unsigned long min,
+		   unsigned long max ) {
+	timer->min = min;
+	timer->max = max;
+}
+
 #endif /* _IPXE_RETRY_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/rndis.h b/qemu/roms/ipxe/src/include/ipxe/rndis.h
new file mode 100644
index 000000000..bcb6d8e6a
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/rndis.h
@@ -0,0 +1,370 @@
+#ifndef _IPXE_RNDIS_H
+#define _IPXE_RNDIS_H
+
+/** @file
+ *
+ * Remote Network Driver Interface Specification
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/iobuf.h>
+
+/** Maximum time to wait for a transaction to complete
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_MAX_WAIT_MS 1000
+
+/** RNDIS message header */
+struct rndis_header {
+	/** Message type */
+	uint32_t type;
+	/** Message length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** RNDIS initialise message */
+#define RNDIS_INITIALISE_MSG 0x00000002UL
+
+/** RNDIS initialise message */
+struct rndis_initialise_message {
+	/** Request ID */
+	uint32_t id;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Maximum transfer size */
+	uint32_t mtu;
+} __attribute__ (( packed ));
+
+/** Request ID used for initialisation
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_INIT_ID 0xe110e110UL
+
+/** RNDIS major version */
+#define RNDIS_VERSION_MAJOR 1
+
+/** RNDIS minor version */
+#define RNDIS_VERSION_MINOR 0
+
+/** RNDIS maximum transfer size
+ *
+ * This is a policy decision.
+ */
+#define RNDIS_MTU 2048
+
+/** RNDIS initialise completion */
+#define RNDIS_INITIALISE_CMPLT 0x80000002UL
+
+/** RNDIS initialise completion */
+struct rndis_initialise_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+	/** Major version */
+	uint32_t major;
+	/** Minor version */
+	uint32_t minor;
+	/** Device flags */
+	uint32_t flags;
+	/** Medium */
+	uint32_t medium;
+	/** Maximum packets per transfer */
+	uint32_t max_pkts;
+	/** Maximum transfer size */
+	uint32_t mtu;
+	/** Packet alignment factor */
+	uint32_t align;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS halt message */
+#define RNDIS_HALT_MSG 0x00000003UL
+
+/** RNDIS halt message */
+struct rndis_halt_message {
+	/** Request ID */
+	uint32_t id;
+} __attribute__ (( packed ));
+
+/** RNDIS query OID message */
+#define RNDIS_QUERY_MSG 0x00000004UL
+
+/** RNDIS set OID message */
+#define RNDIS_SET_MSG 0x00000005UL
+
+/** RNDIS query or set OID message */
+struct rndis_oid_message {
+	/** Request ID */
+	uint32_t id;
+	/** Object ID */
+	uint32_t oid;
+	/** Information buffer length */
+	uint32_t len;
+	/** Information buffer offset */
+	uint32_t offset;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS query OID completion */
+#define RNDIS_QUERY_CMPLT 0x80000004UL
+
+/** RNDIS query OID completion */
+struct rndis_query_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+	/** Information buffer length */
+	uint32_t len;
+	/** Information buffer offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** RNDIS set OID completion */
+#define RNDIS_SET_CMPLT 0x80000005UL
+
+/** RNDIS set OID completion */
+struct rndis_set_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** RNDIS reset message */
+#define RNDIS_RESET_MSG 0x00000006UL
+
+/** RNDIS reset message */
+struct rndis_reset_message {
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS reset completion */
+#define RNDIS_RESET_CMPLT 0x80000006UL
+
+/** RNDIS reset completion */
+struct rndis_reset_completion {
+	/** Status */
+	uint32_t status;
+	/** Addressing reset */
+	uint32_t addr;
+} __attribute__ (( packed ));
+
+/** RNDIS indicate status message */
+#define RNDIS_INDICATE_STATUS_MSG 0x00000007UL
+
+/** RNDIS diagnostic information */
+struct rndis_diagnostic_info {
+	/** Status */
+	uint32_t status;
+	/** Error offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** RNDIS indicate status message */
+struct rndis_indicate_status_message {
+	/** Status */
+	uint32_t status;
+	/** Status buffer length */
+	uint32_t len;
+	/** Status buffer offset */
+	uint32_t offset;
+	/** Diagnostic information (optional) */
+	struct rndis_diagnostic_info diag[0];
+} __attribute__ (( packed ));
+
+/** RNDIS status codes */
+enum rndis_status {
+	/** Device is connected to a network medium */
+	RNDIS_STATUS_MEDIA_CONNECT = 0x4001000bUL,
+	/** Device is disconnected from the medium */
+	RNDIS_STATUS_MEDIA_DISCONNECT = 0x4001000cUL,
+	/** Unknown start-of-day status code */
+	RNDIS_STATUS_WTF_WORLD = 0x40020006UL,
+};
+
+/** RNDIS keepalive message */
+#define RNDIS_KEEPALIVE_MSG 0x00000008UL
+
+/** RNDIS keepalive message */
+struct rndis_keepalive_message {
+	/** Request ID */
+	uint32_t id;
+} __attribute__ (( packed ));
+
+/** RNDIS keepalive completion */
+#define RNDIS_KEEPALIVE_CMPLT 0x80000008UL
+
+/** RNDIS keepalive completion */
+struct rndis_keepalive_completion {
+	/** Request ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** RNDIS packet message */
+#define RNDIS_PACKET_MSG 0x00000001UL
+
+/** RNDIS packet field */
+struct rndis_packet_field {
+	/** Offset */
+	uint32_t offset;
+	/** Length */
+	uint32_t len;
+} __attribute__ (( packed ));
+
+/** RNDIS packet message */
+struct rndis_packet_message {
+	/** Data */
+	struct rndis_packet_field data;
+	/** Out-of-band data records */
+	struct rndis_packet_field oob;
+	/** Number of out-of-band data records */
+	uint32_t oob_count;
+	/** Per-packet information record */
+	struct rndis_packet_field ppi;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** RNDIS packet record */
+struct rndis_packet_record {
+	/** Length */
+	uint32_t len;
+	/** Type */
+	uint32_t type;
+	/** Offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** OID for packet filter */
+#define RNDIS_OID_GEN_CURRENT_PACKET_FILTER 0x0001010eUL
+
+/** Packet filter bits */
+enum rndis_packet_filter {
+	/** Unicast packets */
+	RNDIS_FILTER_UNICAST = 0x00000001UL,
+	/** Multicast packets */
+	RNDIS_FILTER_MULTICAST = 0x00000002UL,
+	/** All multicast packets */
+	RNDIS_FILTER_ALL_MULTICAST = 0x00000004UL,
+	/** Broadcast packets */
+	RNDIS_FILTER_BROADCAST = 0x00000008UL,
+	/** All packets */
+	RNDIS_FILTER_PROMISCUOUS = 0x00000020UL
+};
+
+/** OID for media status */
+#define RNDIS_OID_GEN_MEDIA_CONNECT_STATUS 0x00010114UL
+
+/** OID for permanent MAC address */
+#define RNDIS_OID_802_3_PERMANENT_ADDRESS 0x01010101UL
+
+/** OID for current MAC address */
+#define RNDIS_OID_802_3_CURRENT_ADDRESS	0x01010102UL
+
+struct rndis_device;
+
+/** RNDIS device operations */
+struct rndis_operations {
+	/**
+	 * Open RNDIS device
+	 *
+	 * @v rndis		RNDIS device
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct rndis_device *rndis );
+	/**
+	 * Close RNDIS device
+	 *
+	 * @v rndis		RNDIS device
+	 */
+	void ( * close ) ( struct rndis_device *rndis );
+	/**
+	 * Transmit packet
+	 *
+	 * @v rndis		RNDIS device
+	 * @v iobuf		I/O buffer
+	 * @ret rc		Return status code
+	 *
+	 * If this method returns success then the RNDIS device must
+	 * eventually report completion via rndis_tx_complete().
+	 */
+	int ( * transmit ) ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf );
+	/**
+	 * Poll for completed and received packets
+	 *
+	 * @v rndis		RNDIS device
+	 */
+	void ( * poll ) ( struct rndis_device *rndis );
+};
+
+/** An RNDIS device */
+struct rndis_device {
+	/** Network device */
+	struct net_device *netdev;
+	/** Device name */
+	const char *name;
+	/** RNDIS operations */
+	struct rndis_operations *op;
+	/** Driver private data */
+	void *priv;
+
+	/** Request ID for current blocking request */
+	unsigned int wait_id;
+	/** Return status code for current blocking request */
+	int wait_rc;
+};
+
+/**
+ * Initialise an RNDIS device
+ *
+ * @v rndis		RNDIS device
+ * @v op		RNDIS device operations
+ */
+static inline void rndis_init ( struct rndis_device *rndis,
+				struct rndis_operations *op ) {
+
+	rndis->op = op;
+}
+
+extern void rndis_tx_complete_err ( struct rndis_device *rndis,
+				    struct io_buffer *iobuf, int rc );
+extern int rndis_tx_defer ( struct rndis_device *rndis,
+			    struct io_buffer *iobuf );
+extern void rndis_rx ( struct rndis_device *rndis, struct io_buffer *iobuf );
+extern void rndis_rx_err ( struct rndis_device *rndis, struct io_buffer *iobuf,
+			   int rc );
+
+extern struct rndis_device * alloc_rndis ( size_t priv_len );
+extern int register_rndis ( struct rndis_device *rndis );
+extern void unregister_rndis ( struct rndis_device *rndis );
+extern void free_rndis ( struct rndis_device *rndis );
+
+/**
+ * Complete message transmission
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static inline void rndis_tx_complete ( struct rndis_device *rndis,
+				       struct io_buffer *iobuf ) {
+
+	rndis_tx_complete_err ( rndis, iobuf, 0 );
+}
+
+#endif /* _IPXE_RNDIS_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/rootcert.h b/qemu/roms/ipxe/src/include/ipxe/rootcert.h
index 6525df87a..d4be2e1bc 100644
--- a/qemu/roms/ipxe/src/include/ipxe/rootcert.h
+++ b/qemu/roms/ipxe/src/include/ipxe/rootcert.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/x509.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/rotate.h b/qemu/roms/ipxe/src/include/ipxe/rotate.h
index ba271ca74..b5693e3ca 100644
--- a/qemu/roms/ipxe/src/include/ipxe/rotate.h
+++ b/qemu/roms/ipxe/src/include/ipxe/rotate.h
@@ -6,10 +6,30 @@
  * Bit operations
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
+static inline __attribute__ (( always_inline )) uint8_t
+rol8 ( uint8_t data, unsigned int rotation ) {
+        return ( ( data << rotation ) | ( data >> ( 8 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint8_t
+ror8 ( uint8_t data, unsigned int rotation ) {
+        return ( ( data >> rotation ) | ( data << ( 8 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint16_t
+rol16 ( uint16_t data, unsigned int rotation ) {
+        return ( ( data << rotation ) | ( data >> ( 16 - rotation ) ) );
+}
+
+static inline __attribute__ (( always_inline )) uint16_t
+ror16 ( uint16_t data, unsigned int rotation ) {
+        return ( ( data >> rotation ) | ( data << ( 16 - rotation ) ) );
+}
+
 static inline __attribute__ (( always_inline )) uint32_t
 rol32 ( uint32_t data, unsigned int rotation ) {
         return ( ( data << rotation ) | ( data >> ( 32 - rotation ) ) );
diff --git a/qemu/roms/ipxe/src/include/ipxe/rsa.h b/qemu/roms/ipxe/src/include/ipxe/rsa.h
index 1a5ad8bab..d947eec73 100644
--- a/qemu/roms/ipxe/src/include/ipxe/rsa.h
+++ b/qemu/roms/ipxe/src/include/ipxe/rsa.h
@@ -6,8 +6,9 @@
  * RSA public-key cryptography
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
+#include <stdarg.h>
 #include <ipxe/crypto.h>
 #include <ipxe/bigint.h>
 #include <ipxe/asn1.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/sanboot.h b/qemu/roms/ipxe/src/include/ipxe/sanboot.h
index 14c8a5da4..57025f2c6 100644
--- a/qemu/roms/ipxe/src/include/ipxe/sanboot.h
+++ b/qemu/roms/ipxe/src/include/ipxe/sanboot.h
@@ -16,7 +16,7 @@
  * the address parameter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/sanboot.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/script.h b/qemu/roms/ipxe/src/include/ipxe/script.h
index 33420dae4..7e7a9a3a4 100644
--- a/qemu/roms/ipxe/src/include/ipxe/script.h
+++ b/qemu/roms/ipxe/src/include/ipxe/script.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/scsi.h b/qemu/roms/ipxe/src/include/ipxe/scsi.h
index 4428daac3..28b55b2d5 100644
--- a/qemu/roms/ipxe/src/include/ipxe/scsi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/scsi.h
@@ -11,7 +11,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Maximum block for READ/WRITE (10) commands */
 #define SCSI_MAX_BLOCK_10 0xffffffffULL
diff --git a/qemu/roms/ipxe/src/include/ipxe/segment.h b/qemu/roms/ipxe/src/include/ipxe/segment.h
index 37bed0e19..9d5ecbd9b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/segment.h
+++ b/qemu/roms/ipxe/src/include/ipxe/segment.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/serial.h b/qemu/roms/ipxe/src/include/ipxe/serial.h
index b47b1d125..83be59c31 100644
--- a/qemu/roms/ipxe/src/include/ipxe/serial.h
+++ b/qemu/roms/ipxe/src/include/ipxe/serial.h
@@ -3,15 +3,14 @@
 
 /** @file
  *
- * Serial driver functions
+ * Serial console
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-extern void serial_putc ( int ch );
-extern int serial_getc ( void );
-extern int serial_ischar ( void );
-extern int serial_initialized;
+#include <ipxe/uart.h>
+
+extern struct uart serial_console;
 
 #endif /* _IPXE_SERIAL_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/settings.h b/qemu/roms/ipxe/src/include/ipxe/settings.h
index d6929ecd0..95a553cc8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/settings.h
+++ b/qemu/roms/ipxe/src/include/ipxe/settings.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
@@ -415,6 +415,7 @@ extern const struct setting_type setting_type_uint32 __setting_type;
 extern const struct setting_type setting_type_hex __setting_type;
 extern const struct setting_type setting_type_hexhyp __setting_type;
 extern const struct setting_type setting_type_hexraw __setting_type;
+extern const struct setting_type setting_type_base64 __setting_type;
 extern const struct setting_type setting_type_uuid __setting_type;
 extern const struct setting_type setting_type_busdevfn __setting_type;
 extern const struct setting_type setting_type_dnssl __setting_type;
diff --git a/qemu/roms/ipxe/src/include/ipxe/settings_ui.h b/qemu/roms/ipxe/src/include/ipxe/settings_ui.h
index 5f7be30cc..0bf21935d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/settings_ui.h
+++ b/qemu/roms/ipxe/src/include/ipxe/settings_ui.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct settings;
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/sha256.h b/qemu/roms/ipxe/src/include/ipxe/sha256.h
index 9aa9f3e57..e234cce33 100644
--- a/qemu/roms/ipxe/src/include/ipxe/sha256.h
+++ b/qemu/roms/ipxe/src/include/ipxe/sha256.h
@@ -7,11 +7,14 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
 
+/** SHA-256 number of rounds */
+#define SHA256_ROUNDS 64
+
 /** An SHA-256 digest */
 struct sha256_digest {
 	/** Hash output */
@@ -58,6 +61,8 @@ union sha256_digest_data_dwords {
 struct sha256_context {
 	/** Amount of accumulated data */
 	size_t len;
+	/** Digest size */
+	size_t digestsize;
 	/** Digest and accumulated data */
 	union sha256_digest_data_dwords ddd;
 } __attribute__ (( packed ));
@@ -68,6 +73,16 @@ struct sha256_context {
 /** SHA-256 digest size */
 #define SHA256_DIGEST_SIZE sizeof ( struct sha256_digest )
 
+/** SHA-224 digest size */
+#define SHA224_DIGEST_SIZE ( SHA256_DIGEST_SIZE * 224 / 256 )
+
+extern void sha256_family_init ( struct sha256_context *context,
+				 const struct sha256_digest *init,
+				 size_t digestsize );
+extern void sha256_update ( void *ctx, const void *data, size_t len );
+extern void sha256_final ( void *ctx, void *out );
+
 extern struct digest_algorithm sha256_algorithm;
+extern struct digest_algorithm sha224_algorithm;
 
 #endif /* _IPXE_SHA256_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/sha512.h b/qemu/roms/ipxe/src/include/ipxe/sha512.h
new file mode 100644
index 000000000..8e22d8357
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/sha512.h
@@ -0,0 +1,98 @@
+#ifndef _IPXE_SHA512_H
+#define _IPXE_SHA512_H
+
+/** @file
+ *
+ * SHA-512 algorithm
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/crypto.h>
+
+/** SHA-512 number of rounds */
+#define SHA512_ROUNDS 80
+
+/** An SHA-512 digest */
+struct sha512_digest {
+	/** Hash output */
+	uint64_t h[8];
+};
+
+/** An SHA-512 data block */
+union sha512_block {
+	/** Raw bytes */
+	uint8_t byte[128];
+	/** Raw qwords */
+	uint64_t qword[16];
+	/** Final block structure */
+	struct {
+		/** Padding */
+		uint8_t pad[112];
+		/** High 64 bits of length in bits */
+		uint64_t len_hi;
+		/** Low 64 bits of length in bits */
+		uint64_t len_lo;
+	} final;
+};
+
+/** SHA-512 digest and data block
+ *
+ * The order of fields within this structure is designed to minimise
+ * code size.
+ */
+struct sha512_digest_data {
+	/** Digest of data already processed */
+	struct sha512_digest digest;
+	/** Accumulated data */
+	union sha512_block data;
+} __attribute__ (( packed ));
+
+/** SHA-512 digest and data block */
+union sha512_digest_data_qwords {
+	/** Digest and data block */
+	struct sha512_digest_data dd;
+	/** Raw qwords */
+	uint64_t qword[ sizeof ( struct sha512_digest_data ) /
+			sizeof ( uint64_t ) ];
+};
+
+/** An SHA-512 context */
+struct sha512_context {
+	/** Amount of accumulated data */
+	size_t len;
+	/** Digest size */
+	size_t digestsize;
+	/** Digest and accumulated data */
+	union sha512_digest_data_qwords ddq;
+} __attribute__ (( packed ));
+
+/** SHA-512 context size */
+#define SHA512_CTX_SIZE sizeof ( struct sha512_context )
+
+/** SHA-512 digest size */
+#define SHA512_DIGEST_SIZE sizeof ( struct sha512_digest )
+
+/** SHA-384 digest size */
+#define SHA384_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 384 / 512 )
+
+/** SHA-512/256 digest size */
+#define SHA512_256_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 256 / 512 )
+
+/** SHA-512/224 digest size */
+#define SHA512_224_DIGEST_SIZE ( SHA512_DIGEST_SIZE * 224 / 512 )
+
+extern void sha512_family_init ( struct sha512_context *context,
+				 const struct sha512_digest *init,
+				 size_t digestsize );
+extern void sha512_update ( void *ctx, const void *data, size_t len );
+extern void sha512_final ( void *ctx, void *out );
+
+extern struct digest_algorithm sha512_algorithm;
+extern struct digest_algorithm sha384_algorithm;
+extern struct digest_algorithm sha512_256_algorithm;
+extern struct digest_algorithm sha512_224_algorithm;
+
+#endif /* IPXE_SHA512_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/shell.h b/qemu/roms/ipxe/src/include/ipxe/shell.h
index faa32f422..0d574e028 100644
--- a/qemu/roms/ipxe/src/include/ipxe/shell.h
+++ b/qemu/roms/ipxe/src/include/ipxe/shell.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** Shell stop states */
 enum shell_stop_state {
diff --git a/qemu/roms/ipxe/src/include/ipxe/smbios.h b/qemu/roms/ipxe/src/include/ipxe/smbios.h
index ef5892a21..24b05ed62 100644
--- a/qemu/roms/ipxe/src/include/ipxe/smbios.h
+++ b/qemu/roms/ipxe/src/include/ipxe/smbios.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/api.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/socket.h b/qemu/roms/ipxe/src/include/ipxe/socket.h
index 7cb3912f4..8c70ea4c0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/socket.h
+++ b/qemu/roms/ipxe/src/include/ipxe/socket.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/spi.h b/qemu/roms/ipxe/src/include/ipxe/spi.h
index d92d1aec9..83b53bce3 100644
--- a/qemu/roms/ipxe/src/include/ipxe/spi.h
+++ b/qemu/roms/ipxe/src/include/ipxe/spi.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/nvs.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/spi_bit.h b/qemu/roms/ipxe/src/include/ipxe/spi_bit.h
index 9cfa7b825..049d30a22 100644
--- a/qemu/roms/ipxe/src/include/ipxe/spi_bit.h
+++ b/qemu/roms/ipxe/src/include/ipxe/spi_bit.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/spi.h>
 #include <ipxe/bitbash.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/stp.h b/qemu/roms/ipxe/src/include/ipxe/stp.h
new file mode 100644
index 000000000..3d85e5ba4
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/stp.h
@@ -0,0 +1,76 @@
+#ifndef _IPXE_STP_H
+#define _IPXE_STP_H
+
+/** @file
+ *
+ * Spanning Tree Protocol (STP)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/if_ether.h>
+
+/** "Protocol" value for STP
+ *
+ * This is the concatenated {DSAP,SSAP} value used internally by iPXE
+ * as the network-layer protocol for LLC frames.
+ */
+#define ETH_P_STP 0x4242
+
+/** A switch identifier */
+struct stp_switch {
+	/** Priotity */
+	uint16_t priority;
+	/** MAC address */
+	uint8_t mac[ETH_ALEN];
+} __attribute__ (( packed ));
+
+/** A Spanning Tree bridge protocol data unit */
+struct stp_bpdu {
+	/** LLC DSAP */
+	uint8_t dsap;
+	/** LLC SSAP */
+	uint8_t ssap;
+	/** LLC control field */
+	uint8_t control;
+	/** Protocol ID */
+	uint16_t protocol;
+	/** Protocol version */
+	uint8_t version;
+	/** Message type */
+	uint8_t type;
+	/** Flags */
+	uint8_t flags;
+	/** Root switch */
+	struct stp_switch root;
+	/** Root path cost */
+	uint32_t cost;
+	/** Sender switch */
+	struct stp_switch sender;
+	/** Port */
+	uint16_t port;
+	/** Message age */
+	uint16_t age;
+	/** Maximum age */
+	uint16_t max;
+	/** Hello time */
+	uint16_t hello;
+	/** Forward delay */
+	uint16_t delay;
+} __attribute__ (( packed ));
+
+/** Spanning Tree protocol ID */
+#define STP_PROTOCOL 0x0000
+
+/** Rapid Spanning Tree protocol version */
+#define STP_VERSION_RSTP 0x02
+
+/** Rapid Spanning Tree bridge PDU type */
+#define STP_TYPE_RSTP 0x02
+
+/** Port is forwarding */
+#define STP_FL_FORWARDING 0x20
+
+#endif /* _IPXE_STP_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/string.h b/qemu/roms/ipxe/src/include/ipxe/string.h
new file mode 100644
index 000000000..a8cbe8faa
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/string.h
@@ -0,0 +1,14 @@
+#ifndef _IPXE_STRING_H
+#define _IPXE_STRING_H
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+extern unsigned int digit_value ( unsigned int digit );
+
+#endif /* _IPXE_STRING_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/syslog.h b/qemu/roms/ipxe/src/include/ipxe/syslog.h
index 131692654..138440d66 100644
--- a/qemu/roms/ipxe/src/include/ipxe/syslog.h
+++ b/qemu/roms/ipxe/src/include/ipxe/syslog.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <syslog.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/tables.h b/qemu/roms/ipxe/src/include/ipxe/tables.h
index e35ce8220..60f8efdea 100644
--- a/qemu/roms/ipxe/src/include/ipxe/tables.h
+++ b/qemu/roms/ipxe/src/include/ipxe/tables.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_TABLES_H
 #define _IPXE_TABLES_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @page ifdef_harmful #ifdef considered harmful
  *
diff --git a/qemu/roms/ipxe/src/include/ipxe/tcp.h b/qemu/roms/ipxe/src/include/ipxe/tcp.h
index 9baa6391c..063ebaa4b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/tcp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/tcp.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tcpip.h>
 
@@ -79,6 +79,48 @@ struct tcp_window_scale_padded_option {
  */
 #define TCP_RX_WINDOW_SCALE 9
 
+/** TCP selective acknowledgement permitted option */
+struct tcp_sack_permitted_option {
+	uint8_t kind;
+	uint8_t length;
+} __attribute__ (( packed ));
+
+/** Padded TCP selective acknowledgement permitted option (used for sending) */
+struct tcp_sack_permitted_padded_option {
+	uint8_t nop[2];
+	struct tcp_sack_permitted_option spopt;
+} __attribute__ (( packed ));
+
+/** Code for the TCP selective acknowledgement permitted option */
+#define TCP_OPTION_SACK_PERMITTED 4
+
+/** TCP selective acknowledgement option */
+struct tcp_sack_option {
+	uint8_t kind;
+	uint8_t length;
+} __attribute__ (( packed ));
+
+/** TCP selective acknowledgement block */
+struct tcp_sack_block {
+	uint32_t left;
+	uint32_t right;
+} __attribute__ (( packed ));
+
+/** Maximum number of selective acknowledgement blocks
+ *
+ * This allows for the presence of the TCP timestamp option.
+ */
+#define TCP_SACK_MAX 3
+
+/** Padded TCP selective acknowledgement option (used for sending) */
+struct tcp_sack_padded_option {
+	uint8_t nop[2];
+	struct tcp_sack_option sackopt;
+} __attribute__ (( packed ));
+
+/** Code for the TCP selective acknowledgement option */
+#define TCP_OPTION_SACK 5
+
 /** TCP timestamp option */
 struct tcp_timestamp_option {
 	uint8_t kind;
@@ -102,6 +144,8 @@ struct tcp_options {
 	const struct tcp_mss_option *mssopt;
 	/** Window scale option, if present */
 	const struct tcp_window_scale_option *wsopt;
+	/** SACK permitted option, if present */
+	const struct tcp_sack_permitted_option *spopt;
 	/** Timestamp option, if present */
 	const struct tcp_timestamp_option *tsopt;
 };
@@ -376,6 +420,13 @@ static inline int tcp_in_window ( uint32_t seq, uint32_t start,
 	return ( ( seq - start ) < len );
 }
 
+/** TCP finish wait time
+ *
+ * Currently set to one second, since we should not allow a slowly
+ * responding server to substantially delay a call to shutdown().
+ */
+#define TCP_FINISH_TIMEOUT ( 1 * TICKS_PER_SEC )
+
 extern struct tcpip_protocol tcp_protocol __tcpip_protocol;
 
 #endif /* _IPXE_TCP_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/tcpip.h b/qemu/roms/ipxe/src/include/ipxe/tcpip.h
index 200630d6b..3cfc8e3ac 100644
--- a/qemu/roms/ipxe/src/include/ipxe/tcpip.h
+++ b/qemu/roms/ipxe/src/include/ipxe/tcpip.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/socket.h>
@@ -48,6 +48,12 @@ struct sockaddr_tcpip {
 	uint16_t st_flags;
 	/** TCP/IP port */
 	uint16_t st_port;
+	/** Scope ID
+	 *
+	 * For link-local or multicast addresses, this is the network
+	 * device index.
+	 */
+        uint16_t st_scope_id;
 	/** Padding
 	 *
 	 * This ensures that a struct @c sockaddr_tcpip is large
@@ -57,7 +63,8 @@ struct sockaddr_tcpip {
 	char pad[ sizeof ( struct sockaddr ) -
 		  ( sizeof ( sa_family_t ) /* st_family */ +
 		    sizeof ( uint16_t ) /* st_flags */ +
-		    sizeof ( uint16_t ) /* st_port */ ) ];
+		    sizeof ( uint16_t ) /* st_port */ +
+		    sizeof ( uint16_t ) /* st_scope_id */ ) ];
 } __attribute__ (( packed, may_alias ));
 
 /** 
diff --git a/qemu/roms/ipxe/src/include/ipxe/test.h b/qemu/roms/ipxe/src/include/ipxe/test.h
index 028ee29fb..0b65c299c 100644
--- a/qemu/roms/ipxe/src/include/ipxe/test.h
+++ b/qemu/roms/ipxe/src/include/ipxe/test.h
@@ -1,7 +1,7 @@
 #ifndef _IPXE_TEST_H
 #define _IPXE_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/include/ipxe/tftp.h b/qemu/roms/ipxe/src/include/ipxe/tftp.h
index aecafa2ae..e3661e1ac 100644
--- a/qemu/roms/ipxe/src/include/ipxe/tftp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/tftp.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/time.h b/qemu/roms/ipxe/src/include/ipxe/time.h
index 673fe098a..4c5bb2a00 100644
--- a/qemu/roms/ipxe/src/include/ipxe/time.h
+++ b/qemu/roms/ipxe/src/include/ipxe/time.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <sys/time.h>
 #include <ipxe/api.h>
@@ -44,6 +44,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 /* Include all architecture-independent time API headers */
 #include <ipxe/null_time.h>
+#include <ipxe/efi/efi_time.h>
 #include <ipxe/linux/linux_time.h>
 
 /* Include all architecture-dependent time API headers */
diff --git a/qemu/roms/ipxe/src/include/ipxe/timer.h b/qemu/roms/ipxe/src/include/ipxe/timer.h
index d0309655d..82fbb6764 100644
--- a/qemu/roms/ipxe/src/include/ipxe/timer.h
+++ b/qemu/roms/ipxe/src/include/ipxe/timer.h
@@ -9,7 +9,7 @@
  * for a monotonically increasing tick counter.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/timer.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/tls.h b/qemu/roms/ipxe/src/include/ipxe/tls.h
index 586da26ec..7d982c326 100644
--- a/qemu/roms/ipxe/src/include/ipxe/tls.h
+++ b/qemu/roms/ipxe/src/include/ipxe/tls.h
@@ -7,7 +7,7 @@
  * Transport Layer Security Protocol
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
@@ -20,6 +20,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/x509.h>
 #include <ipxe/pending.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/tables.h>
 
 /** A TLS header */
 struct tls_header {
@@ -85,7 +86,10 @@ struct tls_header {
 /* TLS hash algorithm identifiers */
 #define TLS_MD5_ALGORITHM 1
 #define TLS_SHA1_ALGORITHM 2
+#define TLS_SHA224_ALGORITHM 3
 #define TLS_SHA256_ALGORITHM 4
+#define TLS_SHA384_ALGORITHM 5
+#define TLS_SHA512_ALGORITHM 6
 
 /* TLS signature algorithm identifiers */
 #define TLS_RSA_ALGORITHM 1
@@ -101,6 +105,9 @@ struct tls_header {
 #define TLS_MAX_FRAGMENT_LENGTH_2048 3
 #define TLS_MAX_FRAGMENT_LENGTH_4096 4
 
+/* TLS signature algorithms extension */
+#define TLS_SIGNATURE_ALGORITHMS 13
+
 /** TLS RX state machine state */
 enum tls_rx_state {
 	TLS_RX_HEADER = 0,
@@ -131,6 +138,14 @@ struct tls_cipher_suite {
 	uint16_t code;
 };
 
+/** TLS cipher suite table */
+#define TLS_CIPHER_SUITES						\
+	__table ( struct tls_cipher_suite, "tls_cipher_suites" )
+
+/** Declare a TLS cipher suite */
+#define __tls_cipher_suite( pref )					\
+	__table_entry ( TLS_CIPHER_SUITES, pref )
+
 /** A TLS cipher specification */
 struct tls_cipherspec {
 	/** Cipher suite */
@@ -165,6 +180,19 @@ struct tls_signature_hash_algorithm {
 	struct tls_signature_hash_id code;
 };
 
+/** TLS signature hash algorithm table
+ *
+ * Note that the default (TLSv1.1 and earlier) algorithm using
+ * MD5+SHA1 is never explicitly specified.
+ */
+#define TLS_SIG_HASH_ALGORITHMS						\
+	__table ( struct tls_signature_hash_algorithm,			\
+		  "tls_sig_hash_algorithms" )
+
+/** Declare a TLS signature hash algorithm */
+#define __tls_sig_hash_algorithm					\
+	__table_entry ( TLS_SIG_HASH_ALGORITHMS, 01 )
+
 /** TLS pre-master secret */
 struct tls_pre_master_secret {
 	/** TLS version */
diff --git a/qemu/roms/ipxe/src/include/ipxe/uaccess.h b/qemu/roms/ipxe/src/include/ipxe/uaccess.h
index 055bb2ca7..a3f78566a 100644
--- a/qemu/roms/ipxe/src/include/ipxe/uaccess.h
+++ b/qemu/roms/ipxe/src/include/ipxe/uaccess.h
@@ -19,7 +19,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/uart.h b/qemu/roms/ipxe/src/include/ipxe/uart.h
new file mode 100644
index 000000000..c63eae615
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/uart.h
@@ -0,0 +1,132 @@
+#ifndef _IPXE_UART_H
+#define _IPXE_UART_H
+
+/** @file
+ *
+ * 16550-compatible UART
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** Transmitter holding register */
+#define UART_THR 0x00
+
+/** Receiver buffer register */
+#define UART_RBR 0x00
+
+/** Interrupt enable register */
+#define UART_IER 0x01
+
+/** FIFO control register */
+#define UART_FCR 0x02
+#define UART_FCR_FE	0x01	/**< FIFO enable */
+
+/** Line control register */
+#define UART_LCR 0x03
+#define UART_LCR_WLS0	0x01	/**< Word length select bit 0 */
+#define UART_LCR_WLS1	0x02	/**< Word length select bit 1 */
+#define UART_LCR_STB	0x04	/**< Number of stop bits */
+#define UART_LCR_PEN	0x08	/**< Parity enable */
+#define UART_LCR_EPS	0x10	/**< Even parity select */
+#define UART_LCR_DLAB	0x80	/**< Divisor latch access bit */
+
+#define UART_LCR_WORD_LEN(x)	( ( (x) - 5 ) << 0 )	/**< Word length */
+#define UART_LCR_STOP_BITS(x)	( ( (x) - 1 ) << 2 )	/**< Stop bits */
+#define UART_LCR_PARITY(x)	( ( (x) - 0 ) << 3 )	/**< Parity */
+
+/**
+ * Calculate line control register value
+ *
+ * @v word_len		Word length (5-8)
+ * @v parity		Parity (0=none, 1=odd, 3=even)
+ * @v stop_bits		Stop bits (1-2)
+ * @ret lcr		Line control register value
+ */
+#define UART_LCR_WPS( word_len, parity, stop_bits )	\
+	( UART_LCR_WORD_LEN ( (word_len) ) |		\
+	  UART_LCR_PARITY ( (parity) ) |		\
+	  UART_LCR_STOP_BITS ( (stop_bits) ) )
+
+/** Default LCR value: 8 data bits, no parity, one stop bit */
+#define UART_LCR_8N1 UART_LCR_WPS ( 8, 0, 1 )
+
+/** Modem control register */
+#define UART_MCR 0x04
+#define UART_MCR_DTR	0x01	/**< Data terminal ready */
+#define UART_MCR_RTS	0x02	/**< Request to send */
+
+/** Line status register */
+#define UART_LSR 0x05
+#define UART_LSR_DR	0x01	/**< Data ready */
+#define UART_LSR_THRE	0x20	/**< Transmitter holding register empty */
+#define UART_LSR_TEMT	0x40	/**< Transmitter empty */
+
+/** Scratch register */
+#define UART_SCR 0x07
+
+/** Divisor latch (least significant byte) */
+#define UART_DLL 0x00
+
+/** Divisor latch (most significant byte) */
+#define UART_DLM 0x01
+
+/** Maximum baud rate */
+#define UART_MAX_BAUD 115200
+
+/** A 16550-compatible UART */
+struct uart {
+	/** I/O port base address */
+	void *base;
+	/** Baud rate divisor */
+	uint16_t divisor;
+	/** Line control register */
+	uint8_t lcr;
+};
+
+/** Symbolic names for port indexes */
+enum uart_port {
+	COM1 = 1,
+	COM2 = 2,
+	COM3 = 3,
+	COM4 = 4,
+};
+
+#include <bits/uart.h>
+
+void uart_write ( struct uart *uart, unsigned int addr, uint8_t data );
+uint8_t uart_read ( struct uart *uart, unsigned int addr );
+int uart_select ( struct uart *uart, unsigned int port );
+
+/**
+ * Check if received data is ready
+ *
+ * @v uart		UART
+ * @ret ready		Data is ready
+ */
+static inline int uart_data_ready ( struct uart *uart ) {
+	uint8_t lsr;
+
+	lsr = uart_read ( uart, UART_LSR );
+	return ( lsr & UART_LSR_DR );
+}
+
+/**
+ * Receive data
+ *
+ * @v uart		UART
+ * @ret data		Data
+ */
+static inline uint8_t uart_receive ( struct uart *uart ) {
+
+	return uart_read ( uart, UART_RBR );
+}
+
+extern void uart_transmit ( struct uart *uart, uint8_t data );
+extern void uart_flush ( struct uart *uart );
+extern int uart_exists ( struct uart *uart );
+extern int uart_init ( struct uart *uart, unsigned int baud, uint8_t lcr );
+
+#endif /* _IPXE_UART_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/udp.h b/qemu/roms/ipxe/src/include/ipxe/udp.h
index 5717ef213..7b0de4dc0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/udp.h
+++ b/qemu/roms/ipxe/src/include/ipxe/udp.h
@@ -9,7 +9,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/iobuf.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/umalloc.h b/qemu/roms/ipxe/src/include/ipxe/umalloc.h
index 4b25e182a..3892ef53b 100644
--- a/qemu/roms/ipxe/src/include/ipxe/umalloc.h
+++ b/qemu/roms/ipxe/src/include/ipxe/umalloc.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/api.h>
 #include <config/umalloc.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/uri.h b/qemu/roms/ipxe/src/include/ipxe/uri.h
index 7613d578d..00e5a24c4 100644
--- a/qemu/roms/ipxe/src/include/ipxe/uri.h
+++ b/qemu/roms/ipxe/src/include/ipxe/uri.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
@@ -203,7 +203,7 @@ extern char * resolve_path ( const char *base_path,
 			     const char *relative_path );
 extern struct uri * resolve_uri ( const struct uri *base_uri,
 				  struct uri *relative_uri );
-extern struct uri * tftp_uri ( struct in_addr next_server,
+extern struct uri * tftp_uri ( struct in_addr next_server, unsigned int port,
 			       const char *filename );
 extern void churi ( struct uri *uri );
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/usb.h b/qemu/roms/ipxe/src/include/ipxe/usb.h
new file mode 100644
index 000000000..ab060b8f4
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/usb.h
@@ -0,0 +1,1319 @@
+#ifndef _IPXE_USB_H
+#define _IPXE_USB_H
+
+/** @file
+ *
+ * Universal Serial Bus (USB)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/list.h>
+#include <ipxe/device.h>
+#include <ipxe/process.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/tables.h>
+
+/** USB protocols */
+enum usb_protocol {
+	/** USB 2.0 */
+	USB_PROTO_2_0 = 0x0200,
+	/** USB 3.0 */
+	USB_PROTO_3_0 = 0x0300,
+	/** USB 3.1 */
+	USB_PROTO_3_1 = 0x0301,
+};
+
+/** Define a USB speed
+ *
+ * @v mantissa		Mantissa
+ * @v exponent		Exponent (in engineering terms: 1=k, 2=M, 3=G)
+ * @ret speed		USB speed
+ */
+#define USB_SPEED( mantissa, exponent ) ( (exponent << 16) | (mantissa) )
+
+/** Extract USB speed mantissa */
+#define USB_SPEED_MANTISSA(speed) ( (speed) & 0xffff )
+
+/** Extract USB speed exponent */
+#define USB_SPEED_EXPONENT(speed) ( ( (speed) >> 16 ) & 0x3 )
+
+/** USB device speeds */
+enum usb_speed {
+	/** Not connected */
+	USB_SPEED_NONE = 0,
+	/** Low speed (1.5Mbps) */
+	USB_SPEED_LOW = USB_SPEED ( 1500, 1 ),
+	/** Full speed (12Mbps) */
+	USB_SPEED_FULL = USB_SPEED ( 12, 2 ),
+	/** High speed (480Mbps) */
+	USB_SPEED_HIGH = USB_SPEED ( 480, 2 ),
+	/** Super speed (5Gbps) */
+	USB_SPEED_SUPER = USB_SPEED ( 5, 3 ),
+};
+
+/** USB packet IDs */
+enum usb_pid {
+	/** IN PID */
+	USB_PID_IN = 0x69,
+	/** OUT PID */
+	USB_PID_OUT = 0xe1,
+	/** SETUP PID */
+	USB_PID_SETUP = 0x2d,
+};
+
+/** A USB setup data packet */
+struct usb_setup_packet {
+	/** Request */
+	uint16_t request;
+	/** Value paramer */
+	uint16_t value;
+	/** Index parameter */
+	uint16_t index;
+	/** Length of data stage */
+	uint16_t len;
+} __attribute__ (( packed ));
+
+/** Data transfer is from host to device */
+#define USB_DIR_OUT ( 0 << 7 )
+
+/** Data transfer is from device to host */
+#define USB_DIR_IN ( 1 << 7 )
+
+/** Standard request type */
+#define USB_TYPE_STANDARD ( 0 << 5 )
+
+/** Class-specific request type */
+#define USB_TYPE_CLASS ( 1 << 5 )
+
+/** Vendor-specific request type */
+#define USB_TYPE_VENDOR ( 2 << 5 )
+
+/** Request recipient is the device */
+#define USB_RECIP_DEVICE ( 0 << 0 )
+
+/** Request recipient is an interface */
+#define USB_RECIP_INTERFACE ( 1 << 0 )
+
+/** Request recipient is an endpoint */
+#define USB_RECIP_ENDPOINT ( 2 << 0 )
+
+/** Construct USB request type */
+#define USB_REQUEST_TYPE(type) ( (type) << 8 )
+
+/** Get status */
+#define USB_GET_STATUS ( USB_DIR_IN | USB_REQUEST_TYPE ( 0 ) )
+
+/** Clear feature */
+#define USB_CLEAR_FEATURE ( USB_DIR_OUT | USB_REQUEST_TYPE ( 1 ) )
+
+/** Set feature */
+#define USB_SET_FEATURE ( USB_DIR_OUT | USB_REQUEST_TYPE ( 3 ) )
+
+/** Set address */
+#define USB_SET_ADDRESS ( USB_DIR_OUT | USB_REQUEST_TYPE ( 5 ) )
+
+/** Get descriptor */
+#define USB_GET_DESCRIPTOR ( USB_DIR_IN | USB_REQUEST_TYPE ( 6 ) )
+
+/** Set descriptor */
+#define USB_SET_DESCRIPTOR ( USB_DIR_OUT | USB_REQUEST_TYPE ( 7 ) )
+
+/** Get configuration */
+#define USB_GET_CONFIGURATION ( USB_DIR_IN | USB_REQUEST_TYPE ( 8 ) )
+
+/** Set configuration */
+#define USB_SET_CONFIGURATION ( USB_DIR_OUT | USB_REQUEST_TYPE ( 9 ) )
+
+/** Get interface */
+#define USB_GET_INTERFACE \
+	( USB_DIR_IN | USB_RECIP_INTERFACE | USB_REQUEST_TYPE ( 10 ) )
+
+/** Set interface */
+#define USB_SET_INTERFACE \
+	( USB_DIR_OUT | USB_RECIP_INTERFACE | USB_REQUEST_TYPE ( 11 ) )
+
+/** Endpoint halt feature */
+#define USB_ENDPOINT_HALT 0
+
+/** A USB class code tuple */
+struct usb_class {
+	/** Class code */
+	uint8_t class;
+	/** Subclass code */
+	uint8_t subclass;
+	/** Protocol code */
+	uint8_t protocol;
+} __attribute__ (( packed ));
+
+/** Class code for USB hubs */
+#define USB_CLASS_HUB 9
+
+/** A USB descriptor header */
+struct usb_descriptor_header {
+	/** Length of descriptor */
+	uint8_t len;
+	/** Descriptor type */
+	uint8_t type;
+} __attribute__ (( packed ));
+
+/** A USB device descriptor */
+struct usb_device_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** USB specification release number in BCD */
+	uint16_t protocol;
+	/** Device class */
+	struct usb_class class;
+	/** Maximum packet size for endpoint zero */
+	uint8_t mtu;
+	/** Vendor ID */
+	uint16_t vendor;
+	/** Product ID */
+	uint16_t product;
+	/** Device release number in BCD */
+	uint16_t release;
+	/** Manufacturer string */
+	uint8_t manufacturer;
+	/** Product string */
+	uint8_t name;
+	/** Serial number string */
+	uint8_t serial;
+	/** Number of possible configurations */
+	uint8_t configurations;
+} __attribute__ (( packed ));
+
+/** A USB device descriptor */
+#define USB_DEVICE_DESCRIPTOR 1
+
+/** A USB configuration descriptor */
+struct usb_configuration_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Total length */
+	uint16_t len;
+	/** Number of interfaces */
+	uint8_t interfaces;
+	/** Configuration value */
+	uint8_t config;
+	/** Configuration string */
+	uint8_t name;
+	/** Attributes */
+	uint8_t attributes;
+	/** Maximum power consumption */
+	uint8_t power;
+} __attribute__ (( packed ));
+
+/** A USB configuration descriptor */
+#define USB_CONFIGURATION_DESCRIPTOR 2
+
+/** A USB string descriptor */
+struct usb_string_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** String */
+	char string[0];
+} __attribute__ (( packed ));
+
+/** A USB string descriptor */
+#define USB_STRING_DESCRIPTOR 3
+
+/** A USB interface descriptor */
+struct usb_interface_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Interface number */
+	uint8_t interface;
+	/** Alternate setting */
+	uint8_t alternate;
+	/** Number of endpoints */
+	uint8_t endpoints;
+	/** Interface class */
+	struct usb_class class;
+	/** Interface name */
+	uint8_t name;
+} __attribute__ (( packed ));
+
+/** A USB interface descriptor */
+#define USB_INTERFACE_DESCRIPTOR 4
+
+/** A USB endpoint descriptor */
+struct usb_endpoint_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Endpoint address */
+	uint8_t endpoint;
+	/** Attributes */
+	uint8_t attributes;
+	/** Maximum packet size and burst size */
+	uint16_t sizes;
+	/** Polling interval */
+	uint8_t interval;
+} __attribute__ (( packed ));
+
+/** A USB endpoint descriptor */
+#define USB_ENDPOINT_DESCRIPTOR 5
+
+/** Endpoint attribute transfer type mask */
+#define USB_ENDPOINT_ATTR_TYPE_MASK 0x03
+
+/** Endpoint periodic type */
+#define USB_ENDPOINT_ATTR_PERIODIC 0x01
+
+/** Control endpoint transfer type */
+#define USB_ENDPOINT_ATTR_CONTROL 0x00
+
+/** Bulk endpoint transfer type */
+#define USB_ENDPOINT_ATTR_BULK 0x02
+
+/** Interrupt endpoint transfer type */
+#define USB_ENDPOINT_ATTR_INTERRUPT 0x03
+
+/** Bulk OUT endpoint (internal) type */
+#define USB_BULK_OUT ( USB_ENDPOINT_ATTR_BULK | USB_DIR_OUT )
+
+/** Bulk IN endpoint (internal) type */
+#define USB_BULK_IN ( USB_ENDPOINT_ATTR_BULK | USB_DIR_IN )
+
+/** Interrupt IN endpoint (internal) type */
+#define USB_INTERRUPT_IN ( USB_ENDPOINT_ATTR_INTERRUPT | USB_DIR_IN )
+
+/** Interrupt OUT endpoint (internal) type */
+#define USB_INTERRUPT_OUT ( USB_ENDPOINT_ATTR_INTERRUPT | USB_DIR_OUT )
+
+/** USB endpoint MTU */
+#define USB_ENDPOINT_MTU(sizes) ( ( (sizes) >> 0 ) & 0x07ff )
+
+/** USB endpoint maximum burst size */
+#define USB_ENDPOINT_BURST(sizes) ( ( (sizes) >> 11 ) & 0x0003 )
+
+/** A USB endpoint companion descriptor */
+struct usb_endpoint_companion_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** Maximum burst size */
+	uint8_t burst;
+	/** Extended attributes */
+	uint8_t extended;
+	/** Number of bytes per service interval */
+	uint16_t periodic;
+} __attribute__ (( packed ));
+
+/** A USB endpoint companion descriptor */
+#define USB_ENDPOINT_COMPANION_DESCRIPTOR 48
+
+/** A USB interface association descriptor */
+struct usb_interface_association_descriptor {
+	/** Descriptor header */
+	struct usb_descriptor_header header;
+	/** First interface number */
+	uint8_t first;
+	/** Interface count */
+	uint8_t count;
+	/** Association class */
+	struct usb_class class;
+	/** Association name */
+	uint8_t name;
+} __attribute__ (( packed ));
+
+/** A USB interface association descriptor */
+#define USB_INTERFACE_ASSOCIATION_DESCRIPTOR 11
+
+/** A class-specific interface descriptor */
+#define USB_CS_INTERFACE_DESCRIPTOR 36
+
+/** A class-specific endpoint descriptor */
+#define USB_CS_ENDPOINT_DESCRIPTOR 37
+
+/**
+ * Get next USB descriptor
+ *
+ * @v desc		USB descriptor header
+ * @ret next		Next USB descriptor header
+ */
+static inline __attribute__ (( always_inline )) struct usb_descriptor_header *
+usb_next_descriptor ( struct usb_descriptor_header *desc ) {
+
+	return ( ( ( void * ) desc ) + desc->len );
+}
+
+/**
+ * Check that descriptor lies within a configuration descriptor
+ *
+ * @v config		Configuration descriptor
+ * @v desc		Descriptor header
+ * @v is_within		Descriptor is within the configuration descriptor
+ */
+static inline __attribute__ (( always_inline )) int
+usb_is_within_config ( struct usb_configuration_descriptor *config,
+		       struct usb_descriptor_header *desc ) {
+	struct usb_descriptor_header *end =
+		( ( ( void * ) config ) + le16_to_cpu ( config->len ) );
+
+	/* Check that descriptor starts within the configuration
+	 * descriptor, and that the length does not exceed the
+	 * configuration descriptor.  This relies on the fact that
+	 * usb_next_descriptor() needs to access only the first byte
+	 * of the descriptor in order to determine the length.
+	 */
+	return ( ( desc < end ) && ( usb_next_descriptor ( desc ) <= end ) );
+}
+
+/** Iterate over all configuration descriptors */
+#define for_each_config_descriptor( desc, config )			   \
+	for ( desc = container_of ( &(config)->header,			   \
+				    typeof ( *desc ), header ) ;	   \
+	      usb_is_within_config ( (config), &desc->header ) ;	   \
+	      desc = container_of ( usb_next_descriptor ( &desc->header ), \
+				    typeof ( *desc ), header ) )
+
+/** Iterate over all configuration descriptors within an interface descriptor */
+#define for_each_interface_descriptor( desc, config, interface )	   \
+	for ( desc = container_of ( usb_next_descriptor ( &(interface)->   \
+							  header ),	   \
+				    typeof ( *desc ), header ) ;	   \
+	      ( usb_is_within_config ( (config), &desc->header ) &&	   \
+		( desc->header.type != USB_INTERFACE_DESCRIPTOR ) ) ;	   \
+	      desc = container_of ( usb_next_descriptor ( &desc->header ), \
+				    typeof ( *desc ), header ) )
+
+/** A USB endpoint */
+struct usb_endpoint {
+	/** USB device */
+	struct usb_device *usb;
+	/** Endpoint address */
+	unsigned int address;
+	/** Attributes */
+	unsigned int attributes;
+	/** Maximum transfer size */
+	size_t mtu;
+	/** Maximum burst size */
+	unsigned int burst;
+	/** Interval (in microframes) */
+	unsigned int interval;
+
+	/** Endpoint is open */
+	int open;
+	/** Buffer fill level */
+	unsigned int fill;
+
+	/** List of halted endpoints */
+	struct list_head halted;
+
+	/** Host controller operations */
+	struct usb_endpoint_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+	/** Driver operations */
+	struct usb_endpoint_driver_operations *driver;
+
+	/** Recycled I/O buffer list */
+	struct list_head recycled;
+	/** Refill buffer length */
+	size_t len;
+	/** Maximum fill level */
+	unsigned int max;
+};
+
+/** USB endpoint host controller operations */
+struct usb_endpoint_host_operations {
+	/** Open endpoint
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_endpoint *ep );
+	/** Close endpoint
+	 *
+	 * @v ep		USB endpoint
+	 */
+	void ( * close ) ( struct usb_endpoint *ep );
+	/**
+	 * Reset endpoint
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * reset ) ( struct usb_endpoint *ep );
+	/** Update MTU
+	 *
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * mtu ) ( struct usb_endpoint *ep );
+	/** Enqueue message transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @ret rc		Return status code
+	 */
+	int ( * message ) ( struct usb_endpoint *ep,
+			    struct io_buffer *iobuf );
+	/** Enqueue stream transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @v terminate		Terminate using a short packet
+	 * @ret rc		Return status code
+	 */
+	int ( * stream ) ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			   int terminate );
+};
+
+/** USB endpoint driver operations */
+struct usb_endpoint_driver_operations {
+	/** Complete transfer
+	 *
+	 * @v ep		USB endpoint
+	 * @v iobuf		I/O buffer
+	 * @v rc		Completion status code
+	 */
+	void ( * complete ) ( struct usb_endpoint *ep,
+			      struct io_buffer *iobuf, int rc );
+};
+
+/** Control endpoint address */
+#define USB_EP0_ADDRESS 0x00
+
+/** Control endpoint attributes */
+#define USB_EP0_ATTRIBUTES 0x00
+
+/** Calculate default MTU based on device speed
+ *
+ * @v speed		Device speed
+ * @ret mtu		Default MTU
+ */
+#define USB_EP0_DEFAULT_MTU(speed)			\
+	( ( (speed) >= USB_SPEED_SUPER ) ? 512 :	\
+	  ( ( (speed) >= USB_SPEED_FULL ) ? 64 : 8 ) )
+
+/** Control endpoint maximum burst size */
+#define USB_EP0_BURST 0
+
+/** Control endpoint interval */
+#define USB_EP0_INTERVAL 0
+
+/** Maximum endpoint number */
+#define USB_ENDPOINT_MAX 0x0f
+
+/** Endpoint direction is in */
+#define USB_ENDPOINT_IN 0x80
+
+/** Construct endpoint index from endpoint address */
+#define USB_ENDPOINT_IDX(address)					\
+	( ( (address) & USB_ENDPOINT_MAX ) |				\
+	  ( ( (address) & USB_ENDPOINT_IN ) >> 3 ) )
+
+/**
+ * Initialise USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @v usb		USB device
+ * @v driver		Driver operations
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_init ( struct usb_endpoint *ep, struct usb_device *usb,
+		    struct usb_endpoint_driver_operations *driver ) {
+
+	ep->usb = usb;
+	ep->driver = driver;
+}
+
+/**
+ * Describe USB endpoint
+ *
+ * @v ep		USB endpoint
+ * @v address		Endpoint address
+ * @v attributes	Attributes
+ * @v mtu		Maximum packet size
+ * @v burst		Maximum burst size
+ * @v interval		Interval (in microframes)
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_describe ( struct usb_endpoint *ep, unsigned int address,
+			unsigned int attributes, size_t mtu,
+			unsigned int burst, unsigned int interval ) {
+
+	ep->address = address;
+	ep->attributes = attributes;
+	ep->mtu = mtu;
+	ep->burst = burst;
+	ep->interval = interval;
+}
+
+/**
+ * Set USB endpoint host controller private data
+ *
+ * @v ep		USB endpoint
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_endpoint_set_hostdata ( struct usb_endpoint *ep, void *priv ) {
+	ep->priv = priv;
+}
+
+/**
+ * Get USB endpoint host controller private data
+ *
+ * @v ep		USB endpoint
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_endpoint_get_hostdata ( struct usb_endpoint *ep ) {
+	return ep->priv;
+}
+
+extern const char * usb_endpoint_name ( struct usb_endpoint *ep );
+extern int
+usb_endpoint_described ( struct usb_endpoint *ep,
+			 struct usb_configuration_descriptor *config,
+			 struct usb_interface_descriptor *interface,
+			 unsigned int type, unsigned int index );
+extern int usb_endpoint_open ( struct usb_endpoint *ep );
+extern void usb_endpoint_close ( struct usb_endpoint *ep );
+extern int usb_message ( struct usb_endpoint *ep, unsigned int request,
+			 unsigned int value, unsigned int index,
+			 struct io_buffer *iobuf );
+extern int usb_stream ( struct usb_endpoint *ep, struct io_buffer *iobuf,
+			int terminate );
+extern void usb_complete_err ( struct usb_endpoint *ep,
+			       struct io_buffer *iobuf, int rc );
+
+/**
+ * Initialise USB endpoint refill
+ *
+ * @v ep		USB endpoint
+ * @v len		Refill buffer length (or zero to use endpoint's MTU)
+ * @v max		Maximum fill level
+ */
+static inline __attribute__ (( always_inline )) void
+usb_refill_init ( struct usb_endpoint *ep, size_t len, unsigned int max ) {
+
+	INIT_LIST_HEAD ( &ep->recycled );
+	ep->len = len;
+	ep->max = max;
+}
+
+/**
+ * Recycle I/O buffer
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+usb_recycle ( struct usb_endpoint *ep, struct io_buffer *iobuf ) {
+
+	list_add_tail ( &iobuf->list, &ep->recycled );
+}
+
+extern int usb_prefill ( struct usb_endpoint *ep );
+extern int usb_refill ( struct usb_endpoint *ep );
+extern void usb_flush ( struct usb_endpoint *ep );
+
+/**
+ * A USB function
+ *
+ * A USB function represents an association of interfaces within a USB
+ * device.
+ */
+struct usb_function {
+	/** Name */
+	const char *name;
+	/** USB device */
+	struct usb_device *usb;
+	/** Class */
+	struct usb_class class;
+	/** Number of interfaces */
+	unsigned int count;
+	/** Generic device */
+	struct device dev;
+	/** List of functions within this USB device */
+	struct list_head list;
+
+	/** Driver */
+	struct usb_driver *driver;
+	/** Driver private data */
+	void *priv;
+
+	/** List of interface numbers
+	 *
+	 * This must be the last field within the structure.
+	 */
+	uint8_t interface[0];
+};
+
+/**
+ * Set USB function driver private data
+ *
+ * @v func		USB function
+ * @v priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_func_set_drvdata ( struct usb_function *func, void *priv ) {
+	func->priv = priv;
+}
+
+/**
+ * Get USB function driver private data
+ *
+ * @v function		USB function
+ * @ret priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_func_get_drvdata ( struct usb_function *func ) {
+	return func->priv;
+}
+
+/** A USB device */
+struct usb_device {
+	/** Name */
+	char name[32];
+	/** USB port */
+	struct usb_port *port;
+	/** List of devices on this bus */
+	struct list_head list;
+	/** Device address, if assigned */
+	unsigned int address;
+	/** Device descriptor */
+	struct usb_device_descriptor device;
+	/** List of functions */
+	struct list_head functions;
+
+	/** Host controller operations */
+	struct usb_device_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+
+	/** Endpoint list */
+	struct usb_endpoint *ep[32];
+
+	/** Control endpoint */
+	struct usb_endpoint control;
+	/** Completed control transfers */
+	struct list_head complete;
+};
+
+/** USB device host controller operations */
+struct usb_device_host_operations {
+	/** Open device
+	 *
+	 * @v usb		USB device
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_device *usb );
+	/** Close device
+	 *
+	 * @v usb		USB device
+	 */
+	void ( * close ) ( struct usb_device *usb );
+	/** Assign device address
+	 *
+	 * @v usb		USB device
+	 * @ret rc		Return status code
+	 */
+	int ( * address ) ( struct usb_device *usb );
+};
+
+/**
+ * Set USB device host controller private data
+ *
+ * @v usb		USB device
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_set_hostdata ( struct usb_device *usb, void *priv ) {
+	usb->priv = priv;
+}
+
+/**
+ * Get USB device host controller private data
+ *
+ * @v usb		USB device
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_get_hostdata ( struct usb_device *usb ) {
+	return usb->priv;
+}
+
+/**
+ * Get USB endpoint
+ *
+ * @v usb		USB device
+ * @v address		Endpoint address
+ * @ret ep		USB endpoint, or NULL if not opened
+ */
+static inline struct usb_endpoint * usb_endpoint ( struct usb_device *usb,
+						   unsigned int address ) {
+
+	return usb->ep[ USB_ENDPOINT_IDX ( address ) ];
+}
+
+/** A USB port */
+struct usb_port {
+	/** USB hub */
+	struct usb_hub *hub;
+	/** Port address */
+	unsigned int address;
+	/** Port protocol */
+	unsigned int protocol;
+	/** Port speed */
+	unsigned int speed;
+	/** Port disconnection has been detected
+	 *
+	 * This should be set whenever the underlying hardware reports
+	 * a connection status change.
+	 */
+	int disconnected;
+	/** Port has an attached device */
+	int attached;
+	/** Currently attached device (if in use)
+	 *
+	 * Note that this field will be NULL if the attached device
+	 * has been freed (e.g. because there were no drivers found).
+	 */
+	struct usb_device *usb;
+	/** List of changed ports */
+	struct list_head changed;
+};
+
+/** A USB hub */
+struct usb_hub {
+	/** Name */
+	const char *name;
+	/** USB bus */
+	struct usb_bus *bus;
+	/** Underlying USB device, if any */
+	struct usb_device *usb;
+	/** Hub protocol */
+	unsigned int protocol;
+	/** Number of ports */
+	unsigned int ports;
+
+	/** List of hubs */
+	struct list_head list;
+
+	/** Host controller operations */
+	struct usb_hub_host_operations *host;
+	/** Driver operations */
+	struct usb_hub_driver_operations *driver;
+	/** Driver private data */
+	void *priv;
+
+	/** Port list
+	 *
+	 * This must be the last field within the structure.
+	 */
+	struct usb_port port[0];
+};
+
+/** USB hub host controller operations */
+struct usb_hub_host_operations {
+	/** Open hub
+	 *
+	 * @v hub		USB hub
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_hub *hub );
+	/** Close hub
+	 *
+	 * @v hub		USB hub
+	 */
+	void ( * close ) ( struct usb_hub *hub );
+};
+
+/** USB hub driver operations */
+struct usb_hub_driver_operations {
+	/** Open hub
+	 *
+	 * @v hub		USB hub
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_hub *hub );
+	/** Close hub
+	 *
+	 * @v hub		USB hub
+	 */
+	void ( * close ) ( struct usb_hub *hub );
+	/** Enable port
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * enable ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Disable port
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * disable ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Update port speed
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @ret rc		Return status code
+	 */
+	int ( * speed ) ( struct usb_hub *hub, struct usb_port *port );
+	/** Clear transaction translator buffer
+	 *
+	 * @v hub		USB hub
+	 * @v port		USB port
+	 * @v ep		USB endpoint
+	 * @ret rc		Return status code
+	 */
+	int ( * clear_tt ) ( struct usb_hub *hub, struct usb_port *port,
+			     struct usb_endpoint *ep );
+};
+
+/**
+ * Set USB hub driver private data
+ *
+ * @v hub		USB hub
+ * @v priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_hub_set_drvdata ( struct usb_hub *hub, void *priv ) {
+	hub->priv = priv;
+}
+
+/**
+ * Get USB hub driver private data
+ *
+ * @v hub		USB hub
+ * @ret priv		Driver private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_hub_get_drvdata ( struct usb_hub *hub ) {
+	return hub->priv;
+}
+
+/**
+ * Get USB port
+ *
+ * @v hub		USB hub
+ * @v address		Port address
+ * @ret port		USB port
+ */
+static inline __attribute__ (( always_inline )) struct usb_port *
+usb_port ( struct usb_hub *hub, unsigned int address ) {
+
+	return &hub->port[ address - 1 ];
+}
+
+/** A USB bus */
+struct usb_bus {
+	/** Name */
+	const char *name;
+	/** Underlying hardware device */
+	struct device *dev;
+	/** Host controller operations set */
+	struct usb_host_operations *op;
+
+	/** Largest transfer allowed on the bus */
+	size_t mtu;
+	/** Address in-use mask
+	 *
+	 * This is used only by buses which perform manual address
+	 * assignment.  USB allows for addresses in the range [1,127].
+	 * We use a simple bitmask which restricts us to the range
+	 * [1,64]; this is unlikely to be a problem in practice.  For
+	 * comparison: controllers which perform autonomous address
+	 * assignment (such as xHCI) typically allow for only 32
+	 * devices per bus anyway.
+	 */
+	unsigned long long addresses;
+
+	/** Root hub */
+	struct usb_hub *hub;
+
+	/** List of USB buses */
+	struct list_head list;
+	/** List of devices */
+	struct list_head devices;
+	/** List of hubs */
+	struct list_head hubs;
+
+	/** Host controller operations */
+	struct usb_bus_host_operations *host;
+	/** Host controller private data */
+	void *priv;
+};
+
+/** USB bus host controller operations */
+struct usb_bus_host_operations {
+	/** Open bus
+	 *
+	 * @v bus		USB bus
+	 * @ret rc		Return status code
+	 */
+	int ( * open ) ( struct usb_bus *bus );
+	/** Close bus
+	 *
+	 * @v bus		USB bus
+	 */
+	void ( * close ) ( struct usb_bus *bus );
+	/** Poll bus
+	 *
+	 * @v bus		USB bus
+	 */
+	void ( * poll ) ( struct usb_bus *bus );
+};
+
+/** USB host controller operations */
+struct usb_host_operations {
+	/** Endpoint operations */
+	struct usb_endpoint_host_operations endpoint;
+	/** Device operations */
+	struct usb_device_host_operations device;
+	/** Bus operations */
+	struct usb_bus_host_operations bus;
+	/** Hub operations */
+	struct usb_hub_host_operations hub;
+	/** Root hub operations */
+	struct usb_hub_driver_operations root;
+};
+
+/**
+ * Set USB bus host controller private data
+ *
+ * @v bus		USB bus
+ * @v priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void
+usb_bus_set_hostdata ( struct usb_bus *bus, void *priv ) {
+	bus->priv = priv;
+}
+
+/**
+ * Get USB bus host controller private data
+ *
+ * @v bus		USB bus
+ * @ret priv		Host controller private data
+ */
+static inline __attribute__ (( always_inline )) void *
+usb_bus_get_hostdata ( struct usb_bus *bus ) {
+	return bus->priv;
+}
+
+/**
+ * Poll USB bus
+ *
+ * @v bus		USB bus
+ */
+static inline __attribute__ (( always_inline )) void
+usb_poll ( struct usb_bus *bus ) {
+	bus->host->poll ( bus );
+}
+
+/** Iterate over all USB buses */
+#define for_each_usb_bus( bus ) \
+	list_for_each_entry ( (bus), &usb_buses, list )
+
+/**
+ * Complete transfer (without error)
+ *
+ * @v ep		USB endpoint
+ * @v iobuf		I/O buffer
+ */
+static inline __attribute__ (( always_inline )) void
+usb_complete ( struct usb_endpoint *ep, struct io_buffer *iobuf ) {
+	usb_complete_err ( ep, iobuf, 0 );
+}
+
+extern int usb_control ( struct usb_device *usb, unsigned int request,
+			 unsigned int value, unsigned int index, void *data,
+			 size_t len );
+extern int usb_get_string_descriptor ( struct usb_device *usb,
+				       unsigned int index,
+				       unsigned int language,
+				       char *buf, size_t len );
+
+/**
+ * Get status
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v index		Target index
+ * @v data		Status to fill in
+ * @v len		Length of status descriptor
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_status ( struct usb_device *usb, unsigned int type, unsigned int index,
+		 void *data, size_t len ) {
+
+	return usb_control ( usb, ( USB_GET_STATUS | type ), 0, index,
+			     data, len );
+}
+
+/**
+ * Clear feature
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v feature		Feature selector
+ * @v index		Target index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_clear_feature ( struct usb_device *usb, unsigned int type,
+		    unsigned int feature, unsigned int index ) {
+
+	return usb_control ( usb, ( USB_CLEAR_FEATURE | type ),
+			     feature, index, NULL, 0 );
+}
+
+/**
+ * Set feature
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v feature		Feature selector
+ * @v index		Target index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_feature ( struct usb_device *usb, unsigned int type,
+		  unsigned int feature, unsigned int index ) {
+
+	return usb_control ( usb, ( USB_SET_FEATURE | type ),
+			     feature, index, NULL, 0 );
+}
+
+/**
+ * Set address
+ *
+ * @v usb		USB device
+ * @v address		Device address
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_address ( struct usb_device *usb, unsigned int address ) {
+
+	return usb_control ( usb, USB_SET_ADDRESS, address, 0, NULL, 0 );
+}
+
+/**
+ * Get USB descriptor
+ *
+ * @v usb		USB device
+ * @v type		Request type
+ * @v desc		Descriptor type
+ * @v index		Descriptor index
+ * @v language		Language ID (for string descriptors)
+ * @v data		Descriptor to fill in
+ * @v len		Maximum length of descriptor
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_descriptor ( struct usb_device *usb, unsigned int type,
+		     unsigned int desc, unsigned int index,
+		     unsigned int language, struct usb_descriptor_header *data,
+		     size_t len ) {
+
+	return usb_control ( usb, ( USB_GET_DESCRIPTOR | type ),
+			     ( ( desc << 8 ) | index ), language, data, len );
+}
+
+/**
+ * Get first part of USB device descriptor (up to and including MTU)
+ *
+ * @v usb		USB device
+ * @v data		Device descriptor to (partially) fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_mtu ( struct usb_device *usb, struct usb_device_descriptor *data ) {
+
+	return usb_get_descriptor ( usb, 0, USB_DEVICE_DESCRIPTOR, 0, 0,
+				    &data->header,
+				    ( offsetof ( typeof ( *data ), mtu ) +
+				      sizeof ( data->mtu ) ) );
+}
+
+/**
+ * Get USB device descriptor
+ *
+ * @v usb		USB device
+ * @v data		Device descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_get_device_descriptor ( struct usb_device *usb,
+			    struct usb_device_descriptor *data ) {
+
+	return usb_get_descriptor ( usb, 0, USB_DEVICE_DESCRIPTOR, 0, 0,
+				    &data->header, sizeof ( *data ) );
+}
+
+/**
+ * Get USB configuration descriptor
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @v data		Configuration descriptor to fill in
+ * @ret rc		Return status code
+ */
+static inline __attribute (( always_inline )) int
+usb_get_config_descriptor ( struct usb_device *usb, unsigned int index,
+			    struct usb_configuration_descriptor *data,
+			    size_t len ) {
+
+	return usb_get_descriptor ( usb, 0, USB_CONFIGURATION_DESCRIPTOR, index,
+				    0, &data->header, len );
+}
+
+/**
+ * Set USB configuration
+ *
+ * @v usb		USB device
+ * @v index		Configuration index
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_configuration ( struct usb_device *usb, unsigned int index ) {
+
+	return usb_control ( usb, USB_SET_CONFIGURATION, index, 0, NULL, 0 );
+}
+
+/**
+ * Set USB interface alternate setting
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v alternate		Alternate setting
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usb_set_interface ( struct usb_device *usb, unsigned int interface,
+		    unsigned int alternate ) {
+
+	return usb_control ( usb, USB_SET_INTERFACE, alternate, interface,
+			     NULL, 0 );
+}
+
+extern struct list_head usb_buses;
+
+extern struct usb_interface_descriptor *
+usb_interface_descriptor ( struct usb_configuration_descriptor *config,
+			   unsigned int interface, unsigned int alternate );
+extern struct usb_endpoint_descriptor *
+usb_endpoint_descriptor ( struct usb_configuration_descriptor *config,
+			  struct usb_interface_descriptor *interface,
+			  unsigned int type, unsigned int index );
+extern struct usb_endpoint_companion_descriptor *
+usb_endpoint_companion_descriptor ( struct usb_configuration_descriptor *config,
+				    struct usb_endpoint_descriptor *desc );
+
+extern struct usb_hub * alloc_usb_hub ( struct usb_bus *bus,
+					struct usb_device *usb,
+					unsigned int ports,
+					struct usb_hub_driver_operations *op );
+extern int register_usb_hub ( struct usb_hub *hub );
+extern void unregister_usb_hub ( struct usb_hub *hub );
+extern void free_usb_hub ( struct usb_hub *hub );
+
+extern void usb_port_changed ( struct usb_port *port );
+
+extern struct usb_bus * alloc_usb_bus ( struct device *dev,
+					unsigned int ports, size_t mtu,
+					struct usb_host_operations *op );
+extern int register_usb_bus ( struct usb_bus *bus );
+extern void unregister_usb_bus ( struct usb_bus *bus );
+extern void free_usb_bus ( struct usb_bus *bus );
+extern struct usb_bus * find_usb_bus_by_location ( unsigned int bus_type,
+						   unsigned int location );
+
+extern int usb_alloc_address ( struct usb_bus *bus );
+extern void usb_free_address ( struct usb_bus *bus, unsigned int address );
+extern unsigned int usb_route_string ( struct usb_device *usb );
+extern unsigned int usb_depth ( struct usb_device *usb );
+extern struct usb_port * usb_root_hub_port ( struct usb_device *usb );
+extern struct usb_port * usb_transaction_translator ( struct usb_device *usb );
+
+/** Minimum reset time
+ *
+ * Section 7.1.7.5 of the USB2 specification states that root hub
+ * ports should assert reset signalling for at least 50ms.
+ */
+#define USB_RESET_DELAY_MS 50
+
+/** Reset recovery time
+ *
+ * Section 9.2.6.2 of the USB2 specification states that the
+ * "recovery" interval after a port reset is 10ms.
+ */
+#define USB_RESET_RECOVER_DELAY_MS 10
+
+/** Maximum time to wait for a control transaction to complete
+ *
+ * Section 9.2.6.1 of the USB2 specification states that the upper
+ * limit for commands to be processed is 5 seconds.
+ */
+#define USB_CONTROL_MAX_WAIT_MS 5000
+
+/** Set address recovery time
+ *
+ * Section 9.2.6.3 of the USB2 specification states that devices are
+ * allowed a 2ms recovery interval after receiving a new address.
+ */
+#define USB_SET_ADDRESS_RECOVER_DELAY_MS 2
+
+/** Time to wait for ports to stabilise
+ *
+ * Section 7.1.7.3 of the USB specification states that we must allow
+ * 100ms for devices to signal attachment, and an additional 100ms for
+ * connection debouncing.  (This delay is parallelised across all
+ * ports on a hub; we do not delay separately for each port.)
+ */
+#define USB_PORT_DELAY_MS 200
+
+/** A USB device ID */
+struct usb_device_id {
+	/** Name */
+	const char *name;
+	/** Vendor ID */
+	uint16_t vendor;
+	/** Product ID */
+	uint16_t product;
+	/** Class */
+	struct usb_class class;
+};
+
+/** Match-anything ID */
+#define USB_ANY_ID 0xffff
+
+/** A USB driver */
+struct usb_driver {
+	/** USB ID table */
+	struct usb_device_id *ids;
+	/** Number of entries in ID table */
+	unsigned int id_count;
+	/**
+	 * Probe device
+	 *
+	 * @v func		USB function
+	 * @v config		Configuration descriptor
+	 * @ret rc		Return status code
+	 */
+	int ( * probe ) ( struct usb_function *func,
+			  struct usb_configuration_descriptor *config );
+	/**
+	 * Remove device
+	 *
+	 * @v func		USB function
+	 */
+	void ( * remove ) ( struct usb_function *func );
+};
+
+/** USB driver table */
+#define USB_DRIVERS __table ( struct usb_driver, "usb_drivers" )
+
+/** Declare a USB driver */
+#define __usb_driver __table_entry ( USB_DRIVERS, 01 )
+
+#endif /* _IPXE_USB_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/usbhid.h b/qemu/roms/ipxe/src/include/ipxe/usbhid.h
new file mode 100644
index 000000000..fe9d84455
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/usbhid.h
@@ -0,0 +1,106 @@
+#ifndef _IPXE_USBHID_H
+#define _IPXE_USBHID_H
+
+/** @file
+ *
+ * USB human interface devices (HID)
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** Class code for human interface devices */
+#define USB_CLASS_HID 3
+
+/** Subclass code for boot devices */
+#define USB_SUBCLASS_HID_BOOT 1
+
+/** Set protocol */
+#define USBHID_SET_PROTOCOL						\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x0b ) )
+
+/** Boot protocol */
+#define USBHID_PROTOCOL_BOOT 0
+
+/** Report protocol */
+#define USBHID_PROTOCOL_REPORT 1
+
+/** Set idle time */
+#define USBHID_SET_IDLE							\
+	( USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE |		\
+	  USB_REQUEST_TYPE ( 0x0a ) )
+
+/** A USB human interface device */
+struct usb_hid {
+	/** USB function */
+	struct usb_function *func;
+	/** Interrupt IN endpoint */
+	struct usb_endpoint in;
+	/** Interrupt OUT endpoint (optional) */
+	struct usb_endpoint out;
+};
+
+/**
+ * Initialise USB human interface device
+ *
+ * @v hid		USB human interface device
+ * @v func		USB function
+ * @v in		Interrupt IN endpoint operations
+ * @v out		Interrupt OUT endpoint operations (or NULL)
+ */
+static inline __attribute__ (( always_inline )) void
+usbhid_init ( struct usb_hid *hid, struct usb_function *func,
+	      struct usb_endpoint_driver_operations *in,
+	      struct usb_endpoint_driver_operations *out ) {
+	struct usb_device *usb = func->usb;
+
+	hid->func = func;
+	usb_endpoint_init ( &hid->in, usb, in );
+	if ( out )
+		usb_endpoint_init ( &hid->out, usb, out );
+}
+
+/**
+ * Set protocol
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v protocol		HID protocol
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usbhid_set_protocol ( struct usb_device *usb, unsigned int interface,
+		      unsigned int protocol ) {
+
+	return usb_control ( usb, USBHID_SET_PROTOCOL, protocol, interface,
+			     NULL, 0 );
+}
+
+/**
+ * Set idle time
+ *
+ * @v usb		USB device
+ * @v interface		Interface number
+ * @v report		Report ID
+ * @v duration		Duration (in 4ms units)
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+usbhid_set_idle ( struct usb_device *usb, unsigned int interface,
+		  unsigned int report, unsigned int duration ) {
+
+	return usb_control ( usb, USBHID_SET_IDLE,
+			     ( ( duration << 8 ) | report ),
+			     interface, NULL, 0 );
+}
+
+extern int usbhid_open ( struct usb_hid *hid );
+extern void usbhid_close ( struct usb_hid *hid );
+extern int usbhid_refill ( struct usb_hid *hid );
+extern int usbhid_describe ( struct usb_hid *hid,
+			     struct usb_configuration_descriptor *config );
+
+#endif /* _IPXE_USBHID_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/usbnet.h b/qemu/roms/ipxe/src/include/ipxe/usbnet.h
new file mode 100644
index 000000000..33a8f3f58
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/usbnet.h
@@ -0,0 +1,62 @@
+#ifndef _IPXE_USBNET_H
+#define _IPXE_USBNET_H
+
+/** @file
+ *
+ * USB network devices
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/usb.h>
+
+/** A USB network device */
+struct usbnet_device {
+	/** USB function */
+	struct usb_function *func;
+
+	/** Communications interface */
+	unsigned int comms;
+	/** Data interface */
+	unsigned int data;
+	/** Alternate setting for data interface */
+	unsigned int alternate;
+
+	/** Interrupt endpoint */
+	struct usb_endpoint intr;
+	/** Bulk IN endpoint */
+	struct usb_endpoint in;
+	/** Bulk OUT endpoint */
+	struct usb_endpoint out;
+};
+
+/**
+ * Initialise USB network device
+ *
+ * @v usbnet		USB network device
+ * @v func		USB function
+ * @v intr		Interrupt endpoint operations
+ * @v in		Bulk IN endpoint operations
+ * @v out		Bulk OUT endpoint operations
+ */
+static inline __attribute__ (( always_inline )) void
+usbnet_init ( struct usbnet_device *usbnet, struct usb_function *func,
+	      struct usb_endpoint_driver_operations *intr,
+	      struct usb_endpoint_driver_operations *in,
+	      struct usb_endpoint_driver_operations *out ) {
+	struct usb_device *usb = func->usb;
+
+	usbnet->func = func;
+	usb_endpoint_init ( &usbnet->intr, usb, intr );
+	usb_endpoint_init ( &usbnet->in, usb, in );
+	usb_endpoint_init ( &usbnet->out, usb, out );
+}
+
+extern int usbnet_open ( struct usbnet_device *usbnet );
+extern void usbnet_close ( struct usbnet_device *usbnet );
+extern int usbnet_refill ( struct usbnet_device *usbnet );
+extern int usbnet_describe ( struct usbnet_device *usbnet,
+			     struct usb_configuration_descriptor *config );
+
+#endif /* _IPXE_USBNET_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/uuid.h b/qemu/roms/ipxe/src/include/ipxe/uuid.h
index ad515d0cb..6c45eb9aa 100644
--- a/qemu/roms/ipxe/src/include/ipxe/uuid.h
+++ b/qemu/roms/ipxe/src/include/ipxe/uuid.h
@@ -6,7 +6,7 @@
  * Universally unique IDs
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/validator.h b/qemu/roms/ipxe/src/include/ipxe/validator.h
index 23bdab423..0aee56eb0 100644
--- a/qemu/roms/ipxe/src/include/ipxe/validator.h
+++ b/qemu/roms/ipxe/src/include/ipxe/validator.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/interface.h>
 #include <ipxe/x509.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/version.h b/qemu/roms/ipxe/src/include/ipxe/version.h
index ae4275db1..a43a33425 100644
--- a/qemu/roms/ipxe/src/include/ipxe/version.h
+++ b/qemu/roms/ipxe/src/include/ipxe/version.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <wchar.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/vlan.h b/qemu/roms/ipxe/src/include/ipxe/vlan.h
index 083c21916..439e0c16d 100644
--- a/qemu/roms/ipxe/src/include/ipxe/vlan.h
+++ b/qemu/roms/ipxe/src/include/ipxe/vlan.h
@@ -8,7 +8,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A VLAN header */
 struct vlan_header {
diff --git a/qemu/roms/ipxe/src/include/ipxe/vmbus.h b/qemu/roms/ipxe/src/include/ipxe/vmbus.h
new file mode 100644
index 000000000..26fc578c6
--- /dev/null
+++ b/qemu/roms/ipxe/src/include/ipxe/vmbus.h
@@ -0,0 +1,634 @@
+#ifndef _IPXE_VMBUS_H
+#define _IPXE_VMBUS_H
+
+/** @file
+ *
+ * Hyper-V virtual machine bus
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <byteswap.h>
+#include <ipxe/uuid.h>
+#include <ipxe/device.h>
+#include <ipxe/tables.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/hyperv.h>
+
+/** VMBus message connection ID */
+#define VMBUS_MESSAGE_ID 1
+
+/** VMBus event connection ID */
+#define VMBUS_EVENT_ID 2
+
+/** VMBus message type */
+#define VMBUS_MESSAGE_TYPE 1
+
+/** VMBus message synthetic interrupt */
+#define VMBUS_MESSAGE_SINT 2
+
+/** VMBus version number */
+union vmbus_version {
+	/** Raw version */
+	uint32_t raw;
+	/** Major/minor version */
+	struct {
+		/** Minor version */
+		uint16_t minor;
+		/** Major version */
+		uint16_t major;
+	};
+} __attribute__ (( packed ));
+
+/** Known VMBus protocol versions */
+enum vmbus_raw_version {
+	/** Windows Server 2008 */
+	VMBUS_VERSION_WS2008 = ( ( 0 << 16 ) | ( 13 << 0 ) ),
+	/** Windows 7 */
+	VMBUS_VERSION_WIN7 = ( ( 1 << 16 ) | ( 1 << 0 ) ),
+	/** Windows 8 */
+	VMBUS_VERSION_WIN8 = ( ( 2 << 16 ) | ( 4 << 0 ) ),
+	/** Windows 8.1 */
+	VMBUS_VERSION_WIN8_1 = ( ( 3 << 16 ) | ( 0 << 0 ) ),
+};
+
+/** Guest physical address range descriptor */
+struct vmbus_gpa_range {
+	/** Byte count */
+	uint32_t len;
+	/** Starting byte offset */
+	uint32_t offset;
+	/** Page frame numbers
+	 *
+	 * The length of this array is implied by the byte count and
+	 * starting offset.
+	 */
+	uint64_t pfn[0];
+} __attribute__ (( packed ));
+
+/** VMBus message header */
+struct vmbus_message_header {
+	/** Message type */
+	uint32_t type;
+	/** Reserved */
+	uint32_t reserved;
+} __attribute__ (( packed ));
+
+/** VMBus message types */
+enum vmbus_message_type {
+	VMBUS_OFFER_CHANNEL = 1,
+	VMBUS_REQUEST_OFFERS = 3,
+	VMBUS_ALL_OFFERS_DELIVERED = 4,
+	VMBUS_OPEN_CHANNEL = 5,
+	VMBUS_OPEN_CHANNEL_RESULT = 6,
+	VMBUS_CLOSE_CHANNEL = 7,
+	VMBUS_GPADL_HEADER = 8,
+	VMBUS_GPADL_CREATED = 10,
+	VMBUS_GPADL_TEARDOWN = 11,
+	VMBUS_GPADL_TORNDOWN = 12,
+	VMBUS_INITIATE_CONTACT = 14,
+	VMBUS_VERSION_RESPONSE = 15,
+	VMBUS_UNLOAD = 16,
+	VMBUS_UNLOAD_RESPONSE = 17,
+};
+
+/** VMBus "offer channel" message */
+struct vmbus_offer_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel type */
+	union uuid type;
+	/** Channel instance */
+	union uuid instance;
+	/** Reserved */
+	uint8_t reserved_a[16];
+	/** Flags */
+	uint16_t flags;
+	/** Reserved */
+	uint8_t reserved_b[2];
+	/** User data */
+	uint8_t data[120];
+	/** Reserved */
+	uint8_t reserved_c[4];
+	/** Channel ID */
+	uint32_t channel;
+	/** Monitor ID */
+	uint8_t monitor;
+	/** Monitor exists */
+	uint8_t monitored;
+	/** Reserved */
+	uint8_t reserved[2];
+	/** Connection ID */
+	uint32_t connection;
+} __attribute__ (( packed ));
+
+/** VMBus "open channel" message */
+struct vmbus_open_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** Open ID */
+	uint32_t id;
+	/** Ring buffer GPADL ID */
+	uint32_t gpadl;
+	/** Reserved */
+	uint32_t reserved;
+	/** Outbound ring buffer size (in pages) */
+	uint32_t out_pages;
+	/** User-specific data */
+	uint8_t data[120];
+} __attribute__ (( packed ));
+
+/** VMBus "open channel result" message */
+struct vmbus_open_channel_result {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** Open ID */
+	uint32_t id;
+	/** Status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** VMBus "close channel" message */
+struct vmbus_close_channel {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL header" message */
+struct vmbus_gpadl_header {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Length of range descriptors */
+	uint16_t range_len;
+	/** Number of range descriptors */
+	uint16_t range_count;
+	/** Range descriptors */
+	struct vmbus_gpa_range range[0];
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL created" message */
+struct vmbus_gpadl_created {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+	/** Creation status */
+	uint32_t status;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL teardown" message */
+struct vmbus_gpadl_teardown {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Channel ID */
+	uint32_t channel;
+	/** GPADL ID */
+	uint32_t gpadl;
+} __attribute__ (( packed ));
+
+/** VMBus "GPADL torndown" message */
+struct vmbus_gpadl_torndown {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** GPADL ID */
+	uint32_t gpadl;
+} __attribute__ (( packed ));
+
+/** VMBus "initiate contact" message */
+struct vmbus_initiate_contact {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Requested version */
+	union vmbus_version version;
+	/** Target virtual CPU */
+	uint32_t vcpu;
+	/** Interrupt page base address */
+	uint64_t intr;
+	/** Parent to child monitor page base address */
+	uint64_t monitor_in;
+	/** Child to parent monitor page base address */
+	uint64_t monitor_out;
+} __attribute__ (( packed ));
+
+/** VMBus "version response" message */
+struct vmbus_version_response {
+	/** Message header */
+	struct vmbus_message_header header;
+	/** Version is supported */
+	uint8_t supported;
+	/** Reserved */
+	uint8_t reserved[3];
+	/** Version */
+	union vmbus_version version;
+} __attribute__ (( packed ));
+
+/** VMBus message */
+union vmbus_message {
+	/** Common message header */
+	struct vmbus_message_header header;
+	/** "Offer channel" message */
+	struct vmbus_offer_channel offer;
+	/** "Open channel" message */
+	struct vmbus_open_channel open;
+	/** "Open channel result" message */
+	struct vmbus_open_channel_result opened;
+	/** "Close channel" message */
+	struct vmbus_close_channel close;
+	/** "GPADL header" message */
+	struct vmbus_gpadl_header gpadlhdr;
+	/** "GPADL created" message */
+	struct vmbus_gpadl_created created;
+	/** "GPADL teardown" message */
+	struct vmbus_gpadl_teardown teardown;
+	/** "GPADL torndown" message */
+	struct vmbus_gpadl_torndown torndown;
+	/** "Initiate contact" message */
+	struct vmbus_initiate_contact initiate;
+	/** "Version response" message */
+	struct vmbus_version_response version;
+};
+
+/** VMBus packet header */
+struct vmbus_packet_header {
+	/** Type */
+	uint16_t type;
+	/** Length of packet header (in quadwords) */
+	uint16_t hdr_qlen;
+	/** Length of packet (in quadwords) */
+	uint16_t qlen;
+	/** Flags */
+	uint16_t flags;
+	/** Transaction ID
+	 *
+	 * This is an opaque token: we therefore treat it as
+	 * native-endian and don't worry about byte-swapping.
+	 */
+	uint64_t xid;
+} __attribute__ (( packed ));
+
+/** VMBus packet types */
+enum vmbus_packet_type {
+	VMBUS_DATA_INBAND = 6,
+	VMBUS_DATA_XFER_PAGES = 7,
+	VMBUS_DATA_GPA_DIRECT = 9,
+	VMBUS_CANCELLATION = 10,
+	VMBUS_COMPLETION = 11,
+};
+
+/** VMBus packet flags */
+enum vmbus_packet_flags {
+	VMBUS_COMPLETION_REQUESTED = 0x0001,
+};
+
+/** VMBus GPA direct header */
+struct vmbus_gpa_direct_header {
+	/** Packet header */
+	struct vmbus_packet_header header;
+	/** Reserved */
+	uint32_t reserved;
+	/** Number of range descriptors */
+	uint32_t range_count;
+	/** Range descriptors */
+	struct vmbus_gpa_range range[0];
+} __attribute__ (( packed ));
+
+/** VMBus transfer page range */
+struct vmbus_xfer_page_range {
+	/** Length */
+	uint32_t len;
+	/** Offset */
+	uint32_t offset;
+} __attribute__ (( packed ));
+
+/** VMBus transfer page header */
+struct vmbus_xfer_page_header {
+	/** Packet header */
+	struct vmbus_packet_header header;
+	/** Page set ID */
+	uint16_t pageset;
+	/** Sender owns page set */
+	uint8_t owner;
+	/** Reserved */
+	uint8_t reserved;
+	/** Number of range descriptors */
+	uint32_t range_count;
+	/** Range descriptors */
+	struct vmbus_xfer_page_range range[0];
+} __attribute__ (( packed ));
+
+/** Maximum expected size of VMBus packet header */
+#define VMBUS_PACKET_MAX_HEADER_LEN 64
+
+/** VMBus maximum-sized packet header */
+union vmbus_packet_header_max {
+	/** Common header */
+	struct vmbus_packet_header header;
+	/** GPA direct header */
+	struct vmbus_gpa_direct_header gpa;
+	/** Transfer page header */
+	struct vmbus_xfer_page_header xfer;
+	/** Padding to maximum supported size */
+	uint8_t padding[VMBUS_PACKET_MAX_HEADER_LEN];
+} __attribute__ (( packed ));
+
+/** VMBus packet footer */
+struct vmbus_packet_footer {
+	/** Reserved */
+	uint32_t reserved;
+	/** Producer index of the first byte of the packet */
+	uint32_t prod;
+} __attribute__ (( packed ));
+
+/** VMBus ring buffer
+ *
+ * This is the structure of the each of the ring buffers created when
+ * a VMBus channel is opened.
+ */
+struct vmbus_ring {
+	/** Producer index (modulo ring length) */
+	uint32_t prod;
+	/** Consumer index (modulo ring length) */
+	uint32_t cons;
+	/** Interrupt mask */
+	uint32_t intr_mask;
+	/** Reserved */
+	uint8_t reserved[4084];
+	/** Ring buffer contents */
+	uint8_t data[0];
+} __attribute__ (( packed ));
+
+/** VMBus interrupt page */
+struct vmbus_interrupt {
+	/** Inbound interrupts */
+	uint8_t in[ PAGE_SIZE / 2 ];
+	/** Outbound interrupts */
+	uint8_t out[ PAGE_SIZE / 2 ];
+} __attribute__ (( packed ));
+
+/** A virtual machine bus */
+struct vmbus {
+	/** Interrupt page */
+	struct vmbus_interrupt *intr;
+	/** Inbound notifications */
+	struct hv_monitor *monitor_in;
+	/** Outbound notifications */
+	struct hv_monitor *monitor_out;
+	/** Received message buffer */
+	const union vmbus_message *message;
+};
+
+struct vmbus_device;
+
+/** VMBus channel operations */
+struct vmbus_channel_operations {
+	/**
+	 * Handle received control packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_control ) ( struct vmbus_device *vmdev, uint64_t xid,
+				 const void *data, size_t len );
+	/**
+	 * Handle received data packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @v list		List of I/O buffers
+	 * @ret rc		Return status code
+	 *
+	 * This function takes ownership of the I/O buffer.  It should
+	 * eventually call vmbus_send_completion() to indicate to the
+	 * host that the buffer can be reused.
+	 */
+	int ( * recv_data ) ( struct vmbus_device *vmdev, uint64_t xid,
+			      const void *data, size_t len,
+			      struct list_head *list );
+	/**
+	 * Handle received completion packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @v data		Data
+	 * @v len		Length of data
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_completion ) ( struct vmbus_device *vmdev, uint64_t xid,
+				    const void *data, size_t len );
+	/**
+	 * Handle received cancellation packet
+	 *
+	 * @v vmdev		VMBus device
+	 * @v xid		Transaction ID
+	 * @ret rc		Return status code
+	 */
+	int ( * recv_cancellation ) ( struct vmbus_device *vmdev,
+				      uint64_t xid );
+};
+
+struct vmbus_xfer_pages;
+
+/** VMBus transfer page set operations */
+struct vmbus_xfer_pages_operations {
+	/**
+	 * Copy data from transfer page
+	 *
+	 * @v pages		Transfer page set
+	 * @v data		Data buffer
+	 * @v offset		Offset within page set
+	 * @v len		Length within page set
+	 * @ret rc		Return status code
+	 */
+	int ( * copy ) ( struct vmbus_xfer_pages *pages, void *data,
+			 size_t offset, size_t len );
+};
+
+/** VMBus transfer page set */
+struct vmbus_xfer_pages {
+	/** List of all transfer page sets */
+	struct list_head list;
+	/** Page set ID (in protocol byte order) */
+	uint16_t pageset;
+	/** Page set operations */
+	struct vmbus_xfer_pages_operations *op;
+};
+
+/** A VMBus device */
+struct vmbus_device {
+	/** Generic iPXE device */
+	struct device dev;
+	/** Hyper-V hypervisor */
+	struct hv_hypervisor *hv;
+
+	/** Channel ID */
+	unsigned int channel;
+	/** Monitor ID */
+	unsigned int monitor;
+	/** Signal channel
+	 *
+	 * @v vmdev		VMBus device
+	 */
+	void ( * signal ) ( struct vmbus_device *vmdev );
+
+	/** Outbound ring buffer length */
+	uint32_t out_len;
+	/** Inbound ring buffer length */
+	uint32_t in_len;
+	/** Outbound ring buffer */
+	struct vmbus_ring *out;
+	/** Inbound ring buffer */
+	struct vmbus_ring *in;
+	/** Ring buffer GPADL ID */
+	unsigned int gpadl;
+
+	/** Channel operations */
+	struct vmbus_channel_operations *op;
+	/** Maximum expected data packet length */
+	size_t mtu;
+	/** Packet buffer */
+	void *packet;
+	/** List of transfer page sets */
+	struct list_head pages;
+
+	/** Driver */
+	struct vmbus_driver *driver;
+	/** Driver-private data */
+	void *priv;
+};
+
+/** A VMBus device driver */
+struct vmbus_driver {
+	/** Name */
+	const char *name;
+	/** Device type */
+	union uuid type;
+	/** Probe device
+	 *
+	 * @v vmdev		VMBus device
+	 * @ret rc		Return status code
+	 */
+	int ( * probe ) ( struct vmbus_device *vmdev );
+	/** Remove device
+	 *
+	 * @v vmdev		VMBus device
+	 */
+	void ( * remove ) ( struct vmbus_device *vmdev );
+};
+
+/** VMBus device driver table */
+#define VMBUS_DRIVERS __table ( struct vmbus_driver, "vmbus_drivers" )
+
+/** Declare a VMBus device driver */
+#define __vmbus_driver __table_entry ( VMBUS_DRIVERS, 01 )
+
+/**
+ * Set VMBus device driver-private data
+ *
+ * @v vmdev		VMBus device
+ * @v priv		Private data
+ */
+static inline void vmbus_set_drvdata ( struct vmbus_device *vmdev, void *priv ){
+	vmdev->priv = priv;
+}
+
+/**
+ * Get VMBus device driver-private data
+ *
+ * @v vmdev		VMBus device
+ * @ret priv		Private data
+ */
+static inline void * vmbus_get_drvdata ( struct vmbus_device *vmdev ) {
+	return vmdev->priv;
+}
+
+/** Construct VMBus type */
+#define VMBUS_TYPE( a, b, c, d, e0, e1, e2, e3, e4, e5 ) {		\
+	.canonical = {							\
+		cpu_to_le32 ( a ), cpu_to_le16 ( b ),			\
+		cpu_to_le16 ( c ), cpu_to_be16 ( d ),			\
+		{ e0, e1, e2, e3, e4, e5 }				\
+	 } }
+
+/**
+ * Check if data is present in ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v has_data		Data is present
+ */
+static inline __attribute__ (( always_inline )) int
+vmbus_has_data ( struct vmbus_device *vmdev ) {
+
+	return ( vmdev->in->prod != vmdev->in->cons );
+}
+
+/**
+ * Register transfer page set
+ *
+ * @v vmdev		VMBus device
+ * @v pages		Transfer page set
+ * @ret rc		Return status code
+ */
+static inline __attribute__ (( always_inline )) int
+vmbus_register_pages ( struct vmbus_device *vmdev,
+		       struct vmbus_xfer_pages *pages ) {
+
+	list_add ( &pages->list, &vmdev->pages );
+	return 0;
+}
+
+/**
+ * Unregister transfer page set
+ *
+ * @v vmdev		VMBus device
+ * @v pages		Transfer page set
+ */
+static inline __attribute__ (( always_inline )) void
+vmbus_unregister_pages ( struct vmbus_device *vmdev,
+			 struct vmbus_xfer_pages *pages ) {
+
+	list_check_contains_entry ( pages, &vmdev->pages, list );
+	list_del ( &pages->list );
+}
+
+extern int vmbus_establish_gpadl ( struct vmbus_device *vmdev, userptr_t data,
+				   size_t len );
+extern int vmbus_gpadl_teardown ( struct vmbus_device *vmdev,
+				  unsigned int gpadl );
+extern int vmbus_open ( struct vmbus_device *vmdev,
+			struct vmbus_channel_operations *op,
+			size_t out_len, size_t in_len, size_t mtu );
+extern void vmbus_close ( struct vmbus_device *vmdev );
+extern int vmbus_send_control ( struct vmbus_device *vmdev, uint64_t xid,
+				const void *data, size_t len );
+extern int vmbus_send_data ( struct vmbus_device *vmdev, uint64_t xid,
+			     const void *data, size_t len,
+			     struct io_buffer *iobuf );
+extern int vmbus_send_completion ( struct vmbus_device *vmdev, uint64_t xid,
+				   const void *data, size_t len );
+extern int vmbus_send_cancellation ( struct vmbus_device *vmdev, uint64_t xid );
+extern int vmbus_poll ( struct vmbus_device *vmdev );
+extern void vmbus_dump_channel ( struct vmbus_device *vmdev );
+
+extern int vmbus_probe ( struct hv_hypervisor *hv, struct device *parent );
+extern void vmbus_remove ( struct hv_hypervisor *hv, struct device *parent );
+
+#endif /* _IPXE_VMBUS_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/vsprintf.h b/qemu/roms/ipxe/src/include/ipxe/vsprintf.h
index c48c97a87..9e6297715 100644
--- a/qemu/roms/ipxe/src/include/ipxe/vsprintf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/vsprintf.h
@@ -31,7 +31,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/x509.h b/qemu/roms/ipxe/src/include/ipxe/x509.h
index 802480e54..0daaf5e59 100644
--- a/qemu/roms/ipxe/src/include/ipxe/x509.h
+++ b/qemu/roms/ipxe/src/include/ipxe/x509.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xen.h b/qemu/roms/ipxe/src/include/ipxe/xen.h
index 60aabe03e..eac1145ad 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xen.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xen.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Define Xen interface version before including any Xen header files */
 #define __XEN_INTERFACE_VERSION__ 0x00040400
diff --git a/qemu/roms/ipxe/src/include/ipxe/xenbus.h b/qemu/roms/ipxe/src/include/ipxe/xenbus.h
index ef2b5496f..ec5782eed 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xenbus.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xenbus.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 #include <ipxe/tables.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xenevent.h b/qemu/roms/ipxe/src/include/ipxe/xenevent.h
index 1dd6a0c0b..f0bd3465e 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xenevent.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xenevent.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/event_channel.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xengrant.h b/qemu/roms/ipxe/src/include/ipxe/xengrant.h
index f9b3beb21..451a3ceee 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xengrant.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xengrant.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xenmem.h b/qemu/roms/ipxe/src/include/ipxe/xenmem.h
index 9b9aeda9c..dcc38d460 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xenmem.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xenmem.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/memory.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xenstore.h b/qemu/roms/ipxe/src/include/ipxe/xenstore.h
index f25f15704..892640755 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xenstore.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xenstore.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 
diff --git a/qemu/roms/ipxe/src/include/ipxe/xenver.h b/qemu/roms/ipxe/src/include/ipxe/xenver.h
index 5d678c5a3..b29dfb321 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xenver.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xenver.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/xen.h>
 #include <xen/version.h>
diff --git a/qemu/roms/ipxe/src/include/ipxe/xfer.h b/qemu/roms/ipxe/src/include/ipxe/xfer.h
index 1167e5cba..3a35fa924 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xfer.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xfer.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
@@ -103,5 +103,7 @@ extern int xfer_vprintf ( struct interface *intf,
 extern int __attribute__ (( format ( printf, 2, 3 ) ))
 xfer_printf ( struct interface *intf, const char *format, ... );
 extern int xfer_seek ( struct interface *intf, off_t offset );
+extern int xfer_check_order ( struct xfer_metadata *meta, size_t *pos,
+			      size_t len );
 
 #endif /* _IPXE_XFER_H */
diff --git a/qemu/roms/ipxe/src/include/ipxe/xferbuf.h b/qemu/roms/ipxe/src/include/ipxe/xferbuf.h
index 2ca871e59..cb0b1a0e8 100644
--- a/qemu/roms/ipxe/src/include/ipxe/xferbuf.h
+++ b/qemu/roms/ipxe/src/include/ipxe/xferbuf.h
@@ -7,10 +7,12 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/iobuf.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/interface.h>
 #include <ipxe/xfer.h>
 
 /** A data transfer buffer */
@@ -21,11 +23,83 @@ struct xfer_buffer {
 	size_t len;
 	/** Current offset within data */
 	size_t pos;
+	/** Data transfer buffer operations */
+	struct xfer_buffer_operations *op;
 };
 
-extern void xferbuf_done ( struct xfer_buffer *xferbuf );
+/** Data transfer buffer operations */
+struct xfer_buffer_operations {
+	/** Reallocate data buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v len		New length (or zero to free buffer)
+	 * @ret rc		Return status code
+	 */
+	int ( * realloc ) ( struct xfer_buffer *xferbuf, size_t len );
+	/** Write data to buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v offset		Starting offset
+	 * @v data		Data to write
+	 * @v len		Length of data
+	 *
+	 * This call is simply a wrapper for the appropriate
+	 * memcpy()-like operation: the caller is responsible for
+	 * ensuring that the write does not exceed the buffer length.
+	 */
+	void ( * write ) ( struct xfer_buffer *xferbuf, size_t offset,
+			   const void *data, size_t len );
+	/** Read data from buffer
+	 *
+	 * @v xferbuf		Data transfer buffer
+	 * @v offset		Starting offset
+	 * @v data		Data to read
+	 * @v len		Length of data
+	 *
+	 * This call is simply a wrapper for the appropriate
+	 * memcpy()-like operation: the caller is responsible for
+	 * ensuring that the read does not exceed the buffer length.
+	 */
+	void ( * read ) ( struct xfer_buffer *xferbuf, size_t offset,
+			  void *data, size_t len );
+};
+
+extern struct xfer_buffer_operations xferbuf_malloc_operations;
+extern struct xfer_buffer_operations xferbuf_umalloc_operations;
+
+/**
+ * Initialise malloc()-based data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ */
+static inline __attribute__ (( always_inline )) void
+xferbuf_malloc_init ( struct xfer_buffer *xferbuf ) {
+	xferbuf->op = &xferbuf_malloc_operations;
+}
+
+/**
+ * Initialise umalloc()-based data transfer buffer
+ *
+ * @v xferbuf		Data transfer buffer
+ * @v data		User pointer
+ */
+static inline __attribute__ (( always_inline )) void
+xferbuf_umalloc_init ( struct xfer_buffer *xferbuf, userptr_t *data ) {
+	xferbuf->data = data;
+	xferbuf->op = &xferbuf_umalloc_operations;
+}
+
+extern void xferbuf_free ( struct xfer_buffer *xferbuf );
+extern int xferbuf_write ( struct xfer_buffer *xferbuf, size_t offset,
+			   const void *data, size_t len );
+extern int xferbuf_read ( struct xfer_buffer *xferbuf, size_t offset,
+			  void *data, size_t len );
 extern int xferbuf_deliver ( struct xfer_buffer *xferbuf,
 			     struct io_buffer *iobuf,
 			     struct xfer_metadata *meta );
 
+extern struct xfer_buffer * xfer_buffer ( struct interface *intf );
+#define xfer_buffer_TYPE( object_type ) \
+	typeof ( struct xfer_buffer * ( object_type ) )
+
 #endif /* _IPXE_XFERBUF_H */
diff --git a/qemu/roms/ipxe/src/include/libgen.h b/qemu/roms/ipxe/src/include/libgen.h
index 7e94881a9..ae0861270 100644
--- a/qemu/roms/ipxe/src/include/libgen.h
+++ b/qemu/roms/ipxe/src/include/libgen.h
@@ -1,7 +1,7 @@
 #ifndef _LIBGEN_H
 #define _LIBGEN_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern char * basename ( char *path );
 extern char * dirname ( char *path );
diff --git a/qemu/roms/ipxe/src/include/little_bswap.h b/qemu/roms/ipxe/src/include/little_bswap.h
deleted file mode 100644
index 92dd26ba1..000000000
--- a/qemu/roms/ipxe/src/include/little_bswap.h
+++ /dev/null
@@ -1,37 +0,0 @@
-#ifndef ETHERBOOT_LITTLE_BSWAP_H
-#define ETHERBOOT_LITTLE_BSWAP_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#define htonll(x)	__bswap_64(x)
-#define ntohll(x)	__bswap_64(x)
-#define ntohl(x)	__bswap_32(x)
-#define htonl(x) 	__bswap_32(x)
-#define ntohs(x) 	__bswap_16(x)
-#define htons(x) 	__bswap_16(x)
-#define cpu_to_le64(x)	(x)
-#define cpu_to_le32(x)	(x)
-#define cpu_to_le16(x)	(x)
-#define cpu_to_be64(x)	__bswap_64(x)
-#define cpu_to_be32(x)	__bswap_32(x)
-#define cpu_to_be16(x)	__bswap_16(x)
-#define le64_to_cpu(x)	(x)
-#define le32_to_cpu(x)	(x)
-#define le16_to_cpu(x)	(x)
-#define be64_to_cpu(x)	__bswap_64(x)
-#define be32_to_cpu(x)	__bswap_32(x)
-#define be16_to_cpu(x)	__bswap_16(x)
-#define cpu_to_le64s(x) do {} while (0)
-#define cpu_to_le32s(x) do {} while (0)
-#define cpu_to_le16s(x) do {} while (0)
-#define cpu_to_be64s(x) __bswap_64s(x)
-#define cpu_to_be32s(x) __bswap_32s(x)
-#define cpu_to_be16s(x) __bswap_16s(x)
-#define le64_to_cpus(x) do {} while (0)
-#define le32_to_cpus(x) do {} while (0)
-#define le16_to_cpus(x) do {} while (0)
-#define be64_to_cpus(x) __bswap_64s(x)
-#define be32_to_cpus(x) __bswap_32s(x)
-#define be16_to_cpus(x) __bswap_16s(x)
-
-#endif /* ETHERBOOT_LITTLE_BSWAP_H */
diff --git a/qemu/roms/ipxe/src/include/nic.h b/qemu/roms/ipxe/src/include/nic.h
index 9aaede8a7..4c91f57a6 100644
--- a/qemu/roms/ipxe/src/include/nic.h
+++ b/qemu/roms/ipxe/src/include/nic.h
@@ -1,8 +1,18 @@
- /*
+/*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2, or (at
- * your option) any later version.
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
  */
 
 FILE_LICENCE ( GPL2_OR_LATER );
@@ -266,6 +276,7 @@ static inline void * legacy_isa_get_drvdata ( void *hwdev ) {
 	_name ## _isa_legacy_remove ( struct isa_device *isa ) {	  \
 		return legacy_remove ( isa, legacy_isa_get_drvdata,	  \
 				       _name ## _disable );		  \
-	}
+	}								  \
+	PROVIDE_REQUIRING_SYMBOL()
 
 #endif	/* NIC_H */
diff --git a/qemu/roms/ipxe/src/include/readline/readline.h b/qemu/roms/ipxe/src/include/readline/readline.h
index 0449a3f98..afafbbdf5 100644
--- a/qemu/roms/ipxe/src/include/readline/readline.h
+++ b/qemu/roms/ipxe/src/include/readline/readline.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A readline history entry */
 struct readline_history_entry {
diff --git a/qemu/roms/ipxe/src/include/stdarg.h b/qemu/roms/ipxe/src/include/stdarg.h
index f317238a9..89e94ce22 100644
--- a/qemu/roms/ipxe/src/include/stdarg.h
+++ b/qemu/roms/ipxe/src/include/stdarg.h
@@ -1,7 +1,7 @@
 #ifndef _STDARG_H
 #define _STDARG_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 typedef __builtin_va_list va_list;
 #define va_start( ap, last ) __builtin_va_start ( ap, last )
diff --git a/qemu/roms/ipxe/src/include/stddef.h b/qemu/roms/ipxe/src/include/stddef.h
index bf792771f..3c056294f 100644
--- a/qemu/roms/ipxe/src/include/stddef.h
+++ b/qemu/roms/ipxe/src/include/stddef.h
@@ -1,25 +1,43 @@
 #ifndef STDDEF_H
 #define STDDEF_H
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-/* for size_t */
 #include <stdint.h>
 
+/** EFI headers also define NULL */
 #undef NULL
-#define NULL ((void *)0)
 
-#undef offsetof
-#if ( defined ( __GNUC__ ) && ( __GNUC__ > 3 ) )
-#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
+/** Null pointer */
+#define NULL ( ( void * ) 0 )
+
+/**
+ * Get offset of a field within a structure
+ *
+ * @v type		Structure type
+ * @v field		Field within structure
+ * @ret offset		Offset within structure
+ */
+#if defined ( __GNUC__ ) && ( __GNUC__ > 3 )
+#define offsetof( type, field ) __builtin_offsetof ( type, field )
 #else
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#define offsetof( type, field ) ( ( size_t ) &( ( ( type * ) NULL )->field ) )
 #endif
 
-#undef container_of
-#define container_of(ptr, type, member) ({                      \
-	const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-	(type *)( (char *)__mptr - offsetof(type,member) );})
+/**
+ * Get containing structure
+ *
+ * @v ptr		Pointer to contained field
+ * @v type		Containing structure type
+ * @v field		Field within containing structure
+ * @ret container	Pointer to containing structure
+ */
+#define container_of( ptr, type, field ) ( {				\
+	type *__container;						\
+	const typeof ( __container->field ) *__field = (ptr);		\
+	__container = ( ( ( void * ) __field ) -			\
+			offsetof ( type, field ) );			\
+	__container; } )
 
 /* __WCHAR_TYPE__ is defined by gcc and will change if -fshort-wchar is used */
 #ifndef __WCHAR_TYPE__
diff --git a/qemu/roms/ipxe/src/include/stdint.h b/qemu/roms/ipxe/src/include/stdint.h
index 8cc9b84a5..0a239a517 100644
--- a/qemu/roms/ipxe/src/include/stdint.h
+++ b/qemu/roms/ipxe/src/include/stdint.h
@@ -1,7 +1,7 @@
 #ifndef _STDINT_H
 #define _STDINT_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /*
  * This is a standard predefined macro on all gcc's I've seen. It's
diff --git a/qemu/roms/ipxe/src/include/stdio.h b/qemu/roms/ipxe/src/include/stdio.h
index 91840af5b..a618482ce 100644
--- a/qemu/roms/ipxe/src/include/stdio.h
+++ b/qemu/roms/ipxe/src/include/stdio.h
@@ -1,7 +1,7 @@
 #ifndef _STDIO_H
 #define _STDIO_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
diff --git a/qemu/roms/ipxe/src/include/stdlib.h b/qemu/roms/ipxe/src/include/stdlib.h
index 2951522b8..d7748a07e 100644
--- a/qemu/roms/ipxe/src/include/stdlib.h
+++ b/qemu/roms/ipxe/src/include/stdlib.h
@@ -1,7 +1,7 @@
 #ifndef STDLIB_H
 #define STDLIB_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <assert.h>
@@ -13,31 +13,9 @@ FILE_LICENCE ( GPL2_OR_LATER );
  ****************************************************************************
  */
 
-static inline int strtoul_base ( const char **pp, int base )
-{
-	const char *p = *pp;
-
-	if ( base == 0 ) {
-		base = 10;
-		if ( *p == '0' ) {
-			p++;
-			base = 8;
-			if ( ( *p | 0x20 ) == 'x' ) {
-				p++;
-				base = 16;
-			}
-		}
-	}
-
-	*pp = p;
-
-	return base;
-}
-
-extern unsigned int strtoul_charval ( unsigned int charval );
-extern unsigned long strtoul ( const char *p, char **endp, int base );
-extern unsigned long long strtoull ( const char *p, char **endp, int base );
-
+extern unsigned long strtoul ( const char *string, char **endp, int base );
+extern unsigned long long strtoull ( const char *string, char **endp,
+				     int base );
 
 /*****************************************************************************
  *
diff --git a/qemu/roms/ipxe/src/include/string.h b/qemu/roms/ipxe/src/include/string.h
index 3482e1b22..0fab6c74b 100644
--- a/qemu/roms/ipxe/src/include/string.h
+++ b/qemu/roms/ipxe/src/include/string.h
@@ -1,52 +1,53 @@
-/*
- *  Copyright (C) 1991, 1992  Linus Torvalds
- *  Copyright (C) 2004 Tobias Lorenz
+#ifndef _STRING_H
+#define _STRING_H
+
+/** @file
  *
- *  string handling functions
- *  based on linux/include/linux/ctype.h
- *       and linux/include/linux/string.h
+ * String functions
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
-FILE_LICENCE ( GPL2_ONLY );
-
-#ifndef ETHERBOOT_STRING_H
-#define ETHERBOOT_STRING_H
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <bits/string.h>
 
-int __pure strnicmp(const char *s1, const char *s2, size_t len) __nonnull;
-char * strcpy(char * dest,const char *src) __nonnull;
-char * strncpy(char * dest,const char *src,size_t count) __nonnull;
-char * strcat(char * dest, const char * src) __nonnull;
-char * strncat(char *dest, const char *src, size_t count) __nonnull;
-int __pure strcmp(const char * cs,const char * ct) __nonnull;
-int __pure strncmp(const char * cs,const char * ct,
-				     size_t count) __nonnull;
-char * __pure strchr(const char * s, int c) __nonnull;
-char * __pure strrchr(const char * s, int c) __nonnull;
-size_t __pure strlen(const char * s) __nonnull;
-size_t __pure strnlen(const char * s, size_t count) __nonnull;
-size_t __pure strspn(const char *s, const char *accept) __nonnull;
-size_t __pure strcspn(const char *s, const char *reject) __nonnull;
-char * __pure strpbrk(const char * cs,const char * ct) __nonnull;
-char * strtok(char * s,const char * ct) __nonnull;
-char * strsep(char **s, const char *ct) __nonnull;
-void * memset(void * s,int c,size_t count) __nonnull;
+/* Architecture-specific code is expected to provide these functions,
+ * but may instead explicitly choose to use the generic versions.
+ */
+void * memset ( void *dest, int character, size_t len ) __nonnull;
 void * memcpy ( void *dest, const void *src, size_t len ) __nonnull;
-void * memmove(void * dest,const void *src,size_t count) __nonnull;
-int __pure memcmp(const void * cs,const void * ct,
-				    size_t count) __nonnull;
-void * __pure memscan(const void * addr, int c, size_t size) __nonnull;
-char * __pure strstr(const char * s1,const char * s2) __nonnull;
-void * __pure memchr(const void *s, int c, size_t n) __nonnull;
-char * __malloc strdup(const char *s) __nonnull;
-char * __malloc strndup(const char *s, size_t n) __nonnull;
+void * memmove ( void *dest, const void *src, size_t len ) __nonnull;
+extern void * generic_memset ( void *dest, int character,
+			       size_t len ) __nonnull;
+extern void * generic_memcpy ( void *dest, const void *src,
+			       size_t len ) __nonnull;
+extern void * generic_memmove ( void *dest, const void *src,
+				size_t len ) __nonnull;
+
+extern int __pure memcmp ( const void *first, const void *second,
+			   size_t len ) __nonnull;
+extern void * __pure memchr ( const void *src, int character,
+			      size_t len ) __nonnull;
+extern void * memswap ( void *dest, void *src, size_t len ) __nonnull;
+extern int __pure strcmp ( const char *first, const char *second ) __nonnull;
+extern int __pure strncmp ( const char *first, const char *second,
+			    size_t max ) __nonnull;
+extern size_t __pure strlen ( const char *src ) __nonnull;
+extern size_t __pure strnlen ( const char *src, size_t max ) __nonnull;
+extern char * __pure strchr ( const char *src, int character ) __nonnull;
+extern char * __pure strrchr ( const char *src, int character ) __nonnull;
+extern char * __pure strstr ( const char *haystack,
+			      const char *needle ) __nonnull;
+extern char * strcpy ( char *dest, const char *src ) __nonnull;
+extern char * strncpy ( char *dest, const char *src, size_t max ) __nonnull;
+extern char * strcat ( char *dest, const char *src ) __nonnull;
+extern char * __malloc strdup ( const char *src ) __nonnull;
+extern char * __malloc strndup ( const char *src, size_t max ) __nonnull;
+extern char * __pure strpbrk ( const char *string,
+			       const char *delim ) __nonnull;
+extern char * strsep ( char **string, const char *delim ) __nonnull;
 
-extern const char * __pure strerror ( int errno );
+extern char * __pure strerror ( int errno );
 
-#endif /* ETHERBOOT_STRING */
+#endif /* _STRING_H */
diff --git a/qemu/roms/ipxe/src/include/strings.h b/qemu/roms/ipxe/src/include/strings.h
index 6912a1e45..fab26dc28 100644
--- a/qemu/roms/ipxe/src/include/strings.h
+++ b/qemu/roms/ipxe/src/include/strings.h
@@ -1,12 +1,71 @@
 #ifndef _STRINGS_H
 #define _STRINGS_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
-#include <limits.h>
 #include <string.h>
 #include <bits/strings.h>
 
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int
+__constant_ffsll ( unsigned long long x ) {
+	int r = 0;
+
+	if ( ! ( x & 0x00000000ffffffffULL ) ) {
+		x >>= 32;
+		r += 32;
+	}
+	if ( ! ( x & 0x0000ffffUL ) ) {
+		x >>= 16;
+		r += 16;
+	}
+	if ( ! ( x & 0x00ff ) ) {
+		x >>= 8;
+		r += 8;
+	}
+	if ( ! ( x & 0x0f ) ) {
+		x >>= 4;
+		r += 4;
+	}
+	if ( ! ( x & 0x3 ) ) {
+		x >>= 2;
+		r += 2;
+	}
+	if ( ! ( x & 0x1 ) ) {
+		x >>= 1;
+		r += 1;
+	}
+	return ( x ? ( r + 1 ) : 0 );
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int
+__constant_ffsl ( unsigned long x ) {
+	return __constant_ffsll ( x );
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 static inline __attribute__ (( always_inline )) int
 __constant_flsll ( unsigned long long x ) {
 	int r = 0;
@@ -35,38 +94,100 @@ __constant_flsll ( unsigned long long x ) {
 		x >>= 1;
 		r += 1;
 	}
-	if ( x & 0x1 ) {
-		r += 1;
-	}
-	return r;
+	return ( x ? ( r + 1 ) : 0 );
 }
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 static inline __attribute__ (( always_inline )) int
 __constant_flsl ( unsigned long x ) {
 	return __constant_flsll ( x );
 }
 
+int __ffsll ( long long x );
+int __ffsl ( long x );
 int __flsll ( long long x );
 int __flsl ( long x );
 
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffsll( x ) \
+	( __builtin_constant_p ( x ) ? __constant_ffsll ( x ) : __ffsll ( x ) )
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffsl( x ) \
+	( __builtin_constant_p ( x ) ? __constant_ffsl ( x ) : __ffsl ( x ) )
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v x			Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+#define ffs( x ) ffsl ( x )
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define flsll( x ) \
 	( __builtin_constant_p ( x ) ? __constant_flsll ( x ) : __flsll ( x ) )
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define flsl( x ) \
 	( __builtin_constant_p ( x ) ? __constant_flsl ( x ) : __flsl ( x ) )
 
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v x			Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
 #define fls( x ) flsl ( x )
 
-extern int strcasecmp ( const char *s1, const char *s2 );
-
+/**
+ * Copy memory
+ *
+ * @v src		Source
+ * @v dest		Destination
+ * @v len		Length
+ */
 static inline __attribute__ (( always_inline )) void
-bcopy ( const void *src, void *dest, size_t n ) {
-	memmove ( dest, src, n );
+bcopy ( const void *src, void *dest, size_t len ) {
+	memmove ( dest, src, len );
 }
 
+/**
+ * Zero memory
+ *
+ * @v dest		Destination
+ * @v len		Length
+ */
 static inline __attribute__ (( always_inline )) void
-bzero ( void *s, size_t n ) {
-	memset ( s, 0, n );
+bzero ( void *dest, size_t len ) {
+	memset ( dest, 0, len );
 }
 
+int __pure strcasecmp ( const char *first, const char *second ) __nonnull;
+
 #endif /* _STRINGS_H */
diff --git a/qemu/roms/ipxe/src/include/sys/time.h b/qemu/roms/ipxe/src/include/sys/time.h
index 2647d3588..6e2a24447 100644
--- a/qemu/roms/ipxe/src/include/sys/time.h
+++ b/qemu/roms/ipxe/src/include/sys/time.h
@@ -6,7 +6,7 @@
  * Date and time
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/syslog.h b/qemu/roms/ipxe/src/include/syslog.h
index 93f32f867..748a4faec 100644
--- a/qemu/roms/ipxe/src/include/syslog.h
+++ b/qemu/roms/ipxe/src/include/syslog.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdarg.h>
 #include <ipxe/ansiesc.h>
diff --git a/qemu/roms/ipxe/src/include/time.h b/qemu/roms/ipxe/src/include/time.h
index 452a544bb..462ac6999 100644
--- a/qemu/roms/ipxe/src/include/time.h
+++ b/qemu/roms/ipxe/src/include/time.h
@@ -6,7 +6,7 @@
  * Date and time
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <sys/time.h>
 #include <ipxe/time.h>
diff --git a/qemu/roms/ipxe/src/include/unistd.h b/qemu/roms/ipxe/src/include/unistd.h
index 3a50a2521..d09e1ae30 100644
--- a/qemu/roms/ipxe/src/include/unistd.h
+++ b/qemu/roms/ipxe/src/include/unistd.h
@@ -1,7 +1,7 @@
 #ifndef _UNISTD_H
 #define _UNISTD_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/qemu/roms/ipxe/src/include/usr/autoboot.h b/qemu/roms/ipxe/src/include/usr/autoboot.h
index bc51aae79..4db226b9c 100644
--- a/qemu/roms/ipxe/src/include/usr/autoboot.h
+++ b/qemu/roms/ipxe/src/include/usr/autoboot.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/device.h>
 
@@ -35,7 +35,7 @@ extern int uriboot ( struct uri *filename, struct uri *root_path, int drive,
 extern struct uri *
 fetch_next_server_and_filename ( struct settings *settings );
 extern int netboot ( struct net_device *netdev );
-extern void ipxe ( struct net_device *netdev );
+extern int ipxe ( struct net_device *netdev );
 
 extern int pxe_menu_boot ( struct net_device *netdev );
 
diff --git a/qemu/roms/ipxe/src/include/usr/dhcpmgmt.h b/qemu/roms/ipxe/src/include/usr/dhcpmgmt.h
index af1eceb17..ed669eb9d 100644
--- a/qemu/roms/ipxe/src/include/usr/dhcpmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/dhcpmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct net_device;
 
diff --git a/qemu/roms/ipxe/src/include/usr/fcmgmt.h b/qemu/roms/ipxe/src/include/usr/fcmgmt.h
index 9441cefb4..eb568fd20 100644
--- a/qemu/roms/ipxe/src/include/usr/fcmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/fcmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct fc_port;
 struct fc_peer;
diff --git a/qemu/roms/ipxe/src/include/usr/ifmgmt.h b/qemu/roms/ipxe/src/include/usr/ifmgmt.h
index db77f1f1b..5c386327b 100644
--- a/qemu/roms/ipxe/src/include/usr/ifmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/ifmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 struct net_device;
 struct net_device_configurator;
diff --git a/qemu/roms/ipxe/src/include/usr/imgmgmt.h b/qemu/roms/ipxe/src/include/usr/imgmgmt.h
index 5e25c562b..806df0bfb 100644
--- a/qemu/roms/ipxe/src/include/usr/imgmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/imgmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/qemu/roms/ipxe/src/include/usr/imgtrust.h b/qemu/roms/ipxe/src/include/usr/imgtrust.h
index f47105af0..414e07a80 100644
--- a/qemu/roms/ipxe/src/include/usr/imgtrust.h
+++ b/qemu/roms/ipxe/src/include/usr/imgtrust.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/image.h>
 
diff --git a/qemu/roms/ipxe/src/include/usr/ipstat.h b/qemu/roms/ipxe/src/include/usr/ipstat.h
index 5ff8b40c3..803254bcb 100644
--- a/qemu/roms/ipxe/src/include/usr/ipstat.h
+++ b/qemu/roms/ipxe/src/include/usr/ipstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void ipstat ( void );
 
diff --git a/qemu/roms/ipxe/src/include/usr/lotest.h b/qemu/roms/ipxe/src/include/usr/lotest.h
index aa4bbac4d..ce0fe5eda 100644
--- a/qemu/roms/ipxe/src/include/usr/lotest.h
+++ b/qemu/roms/ipxe/src/include/usr/lotest.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int loopback_test ( struct net_device *sender,
 			   struct net_device *receiver, size_t mtu );
diff --git a/qemu/roms/ipxe/src/include/usr/neighmgmt.h b/qemu/roms/ipxe/src/include/usr/neighmgmt.h
index 3c2b704af..06f03716e 100644
--- a/qemu/roms/ipxe/src/include/usr/neighmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/neighmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void nstat ( void );
 
diff --git a/qemu/roms/ipxe/src/include/usr/pingmgmt.h b/qemu/roms/ipxe/src/include/usr/pingmgmt.h
index d4c2d6cd5..c7a8434be 100644
--- a/qemu/roms/ipxe/src/include/usr/pingmgmt.h
+++ b/qemu/roms/ipxe/src/include/usr/pingmgmt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 
diff --git a/qemu/roms/ipxe/src/include/usr/profstat.h b/qemu/roms/ipxe/src/include/usr/profstat.h
index 06ea251a0..b7812ca7f 100644
--- a/qemu/roms/ipxe/src/include/usr/profstat.h
+++ b/qemu/roms/ipxe/src/include/usr/profstat.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern void profstat ( void );
 
diff --git a/qemu/roms/ipxe/src/include/usr/prompt.h b/qemu/roms/ipxe/src/include/usr/prompt.h
index 57e43d2dc..8d3eeee3c 100644
--- a/qemu/roms/ipxe/src/include/usr/prompt.h
+++ b/qemu/roms/ipxe/src/include/usr/prompt.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int prompt ( const char *text, unsigned long timeout, int key );
 
diff --git a/qemu/roms/ipxe/src/include/usr/route.h b/qemu/roms/ipxe/src/include/usr/route.h
index b914f4b84..7ec4a3509 100644
--- a/qemu/roms/ipxe/src/include/usr/route.h
+++ b/qemu/roms/ipxe/src/include/usr/route.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/tables.h>
 
diff --git a/qemu/roms/ipxe/src/include/usr/sync.h b/qemu/roms/ipxe/src/include/usr/sync.h
index 0047d4ed9..b6f12ad6e 100644
--- a/qemu/roms/ipxe/src/include/usr/sync.h
+++ b/qemu/roms/ipxe/src/include/usr/sync.h
@@ -7,7 +7,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 extern int sync ( unsigned long timeout );
 
diff --git a/qemu/roms/ipxe/src/arch/x86/include/valgrind/memcheck.h b/qemu/roms/ipxe/src/include/valgrind/memcheck.h
index 7d4b56d31..7d4b56d31 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/valgrind/memcheck.h
+++ b/qemu/roms/ipxe/src/include/valgrind/memcheck.h
diff --git a/qemu/roms/ipxe/src/arch/x86/include/valgrind/valgrind.h b/qemu/roms/ipxe/src/include/valgrind/valgrind.h
index d48bbccae..d48bbccae 100644
--- a/qemu/roms/ipxe/src/arch/x86/include/valgrind/valgrind.h
+++ b/qemu/roms/ipxe/src/include/valgrind/valgrind.h
diff --git a/qemu/roms/ipxe/src/include/wchar.h b/qemu/roms/ipxe/src/include/wchar.h
index ba349aae8..d054b8d5b 100644
--- a/qemu/roms/ipxe/src/include/wchar.h
+++ b/qemu/roms/ipxe/src/include/wchar.h
@@ -1,7 +1,7 @@
 #ifndef WCHAR_H
 #define WCHAR_H
 
-FILE_LICENCE ( GPL2_ONLY );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 
diff --git a/qemu/roms/ipxe/src/interface/bofm/bofm.c b/qemu/roms/ipxe/src/interface/bofm/bofm.c
index b0e92b27c..545088dc6 100644
--- a/qemu/roms/ipxe/src/interface/bofm/bofm.c
+++ b/qemu/roms/ipxe/src/interface/bofm/bofm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_autoboot.c b/qemu/roms/ipxe/src/interface/efi/efi_autoboot.c
index ab0f36541..a9e807e23 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_autoboot.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_autoboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/efi_autoboot.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_bofm.c b/qemu/roms/ipxe/src/interface/efi/efi_bofm.c
index bdb705196..ea0e15f7f 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_bofm.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_bofm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <errno.h>
 #include <ipxe/bofm.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_debug.c b/qemu/roms/ipxe/src/interface/efi/efi_debug.c
index d23960140..473803951 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_debug.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_debug.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -326,7 +330,7 @@ const char * efi_devpath_text ( EFI_DEVICE_PATH_PROTOCOL *path ) {
 		max_len = ( ( sizeof ( text ) - 1 /* NUL */ ) / 2 /* "xx" */ );
 		if ( len > max_len )
 			len = max_len;
-		base16_encode ( start, len, text );
+		base16_encode ( start, len, text, sizeof ( text ) );
 		return text;
 	}
 
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_file.c b/qemu/roms/ipxe/src/interface/efi/efi_file.c
index 2ef3c5734..3715b70bf 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_file.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_file.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_guid.c b/qemu/roms/ipxe/src/interface/efi/efi_guid.c
index 52ba58ae4..ab1c91e9f 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_guid.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_guid.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/efi/efi.h>
 #include <ipxe/efi/Protocol/Arp.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_hii.c b/qemu/roms/ipxe/src/interface/efi/efi_hii.c
index 834060b54..0ea970e67 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_hii.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_hii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stddef.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_pci.c b/qemu/roms/ipxe/src/interface/efi/efi_pci.c
index 86c781d82..97ea72bb9 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_pci.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_pci.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_reboot.c b/qemu/roms/ipxe/src/interface/efi/efi_reboot.c
index 96638c48e..35919221e 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_reboot.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_reboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_snp.c b/qemu/roms/ipxe/src/interface/efi/efi_snp.c
index 67fba342e..3dfcc5e16 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_snp.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_snp.c
@@ -32,8 +32,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_driver.h>
 #include <ipxe/efi/efi_strings.h>
 #include <ipxe/efi/efi_utils.h>
+#include <ipxe/efi/efi_watchdog.h>
 #include <ipxe/efi/efi_snp.h>
 #include <usr/autoboot.h>
+#include <config/general.h>
 
 /** List of SNP devices */
 static LIST_HEAD ( efi_snp_devices );
@@ -41,6 +43,39 @@ static LIST_HEAD ( efi_snp_devices );
 /** Network devices are currently claimed for use by iPXE */
 static int efi_snp_claimed;
 
+/* Downgrade user experience if configured to do so
+ *
+ * The default UEFI user experience for network boot is somewhat
+ * excremental: only TFTP is available as a download protocol, and if
+ * anything goes wrong the user will be shown just a dot on an
+ * otherwise blank screen.  (Some programmer was clearly determined to
+ * win a bet that they could outshine Apple at producing uninformative
+ * error messages.)
+ *
+ * For comparison, the default iPXE user experience provides the
+ * option to use protocols designed more recently than 1980 (such as
+ * HTTP and iSCSI), and if anything goes wrong the the user will be
+ * shown one of over 1200 different error messages, complete with a
+ * link to a wiki page describing that specific error.
+ *
+ * We default to upgrading the user experience to match that available
+ * in a "legacy" BIOS environment, by installing our own instance of
+ * EFI_LOAD_FILE_PROTOCOL.
+ *
+ * Note that unfortunately we can't sensibly provide the choice of
+ * both options to the user in the same build, because the UEFI boot
+ * menu ignores the multitude of ways in which a network device handle
+ * can be described and opaquely labels both menu entries as just "EFI
+ * Network".
+ */
+#ifdef EFI_DOWNGRADE_UX
+static EFI_GUID dummy_load_file_protocol_guid = {
+	0x6f6c7323, 0x2077, 0x7523,
+	{ 0x6e, 0x68, 0x65, 0x6c, 0x70, 0x66, 0x75, 0x6c }
+};
+#define efi_load_file_protocol_guid dummy_load_file_protocol_guid
+#endif
+
 /**
  * Set EFI SNP mode state
  *
@@ -98,28 +133,43 @@ static void efi_snp_set_mode ( struct efi_snp_device *snpdev ) {
 }
 
 /**
+ * Flush transmit ring and receive queue
+ *
+ * @v snpdev		SNP device
+ */
+static void efi_snp_flush ( struct efi_snp_device *snpdev ) {
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+
+	/* Reset transmit completion ring */
+	snpdev->tx_prod = 0;
+	snpdev->tx_cons = 0;
+
+	/* Discard any queued receive buffers */
+	list_for_each_entry_safe ( iobuf, tmp, &snpdev->rx, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+}
+
+/**
  * Poll net device and count received packets
  *
  * @v snpdev		SNP device
  */
 static void efi_snp_poll ( struct efi_snp_device *snpdev ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct io_buffer *iobuf;
-	unsigned int before = 0;
-	unsigned int after = 0;
-	unsigned int arrived;
 
-	/* We have to report packet arrivals, and this is the easiest
-	 * way to fake it.
-	 */
-	list_for_each_entry ( iobuf, &snpdev->netdev->rx_queue, list )
-		before++;
+	/* Poll network device */
 	netdev_poll ( snpdev->netdev );
-	list_for_each_entry ( iobuf, &snpdev->netdev->rx_queue, list )
-		after++;
-	arrived = ( after - before );
 
-	snpdev->rx_count_interrupts += arrived;
-	snpdev->rx_count_events += arrived;
+	/* Retrieve any received packets */
+	while ( ( iobuf = netdev_rx_dequeue ( snpdev->netdev ) ) ) {
+		list_add_tail ( &iobuf->list, &snpdev->rx );
+		snpdev->interrupts |= EFI_SIMPLE_NETWORK_RECEIVE_INTERRUPT;
+		bs->SignalEvent ( &snpdev->snp.WaitForPacket );
+	}
 }
 
 /**
@@ -221,6 +271,7 @@ efi_snp_reset ( EFI_SIMPLE_NETWORK_PROTOCOL *snp, BOOLEAN ext_verify ) {
 
 	netdev_close ( snpdev->netdev );
 	efi_snp_set_state ( snpdev );
+	efi_snp_flush ( snpdev );
 
 	if ( ( rc = netdev_open ( snpdev->netdev ) ) != 0 ) {
 		DBGC ( snpdev, "SNPDEV %p could not reopen %s: %s\n",
@@ -251,6 +302,7 @@ efi_snp_shutdown ( EFI_SIMPLE_NETWORK_PROTOCOL *snp ) {
 
 	netdev_close ( snpdev->netdev );
 	efi_snp_set_state ( snpdev );
+	efi_snp_flush ( snpdev );
 
 	return 0;
 }
@@ -446,20 +498,22 @@ efi_snp_nvdata ( EFI_SIMPLE_NETWORK_PROTOCOL *snp, BOOLEAN read,
  *
  * @v snp		SNP interface
  * @v interrupts	Interrupt status, or NULL
- * @v txbufs		Recycled transmit buffer address, or NULL
+ * @v txbuf		Recycled transmit buffer address, or NULL
  * @ret efirc		EFI status code
  */
 static EFI_STATUS EFIAPI
 efi_snp_get_status ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
-		     UINT32 *interrupts, VOID **txbufs ) {
+		     UINT32 *interrupts, VOID **txbuf ) {
 	struct efi_snp_device *snpdev =
 		container_of ( snp, struct efi_snp_device, snp );
 
 	DBGC2 ( snpdev, "SNPDEV %p GET_STATUS", snpdev );
 
 	/* Fail if net device is currently claimed for use by iPXE */
-	if ( efi_snp_claimed )
+	if ( efi_snp_claimed ) {
+		DBGC2 ( snpdev, "\n" );
 		return EFI_NOT_READY;
+	}
 
 	/* Poll the network device */
 	efi_snp_poll ( snpdev );
@@ -468,47 +522,19 @@ efi_snp_get_status ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	 * to detect TX completions.
 	 */
 	if ( interrupts ) {
-		*interrupts = 0;
-		/* Report TX completions once queue is empty; this
-		 * avoids having to add hooks in the net device layer.
-		 */
-		if ( snpdev->tx_count_interrupts &&
-		     list_empty ( &snpdev->netdev->tx_queue ) ) {
-			*interrupts |= EFI_SIMPLE_NETWORK_TRANSMIT_INTERRUPT;
-			snpdev->tx_count_interrupts--;
-		}
-		/* Report RX */
-		if ( snpdev->rx_count_interrupts ) {
-			*interrupts |= EFI_SIMPLE_NETWORK_RECEIVE_INTERRUPT;
-			snpdev->rx_count_interrupts--;
-		}
+		*interrupts = snpdev->interrupts;
 		DBGC2 ( snpdev, " INTS:%02x", *interrupts );
+		snpdev->interrupts = 0;
 	}
 
-	/* TX completions.  It would be possible to design a more
-	 * idiotic scheme for this, but it would be a challenge.
-	 * According to the UEFI header file, txbufs will be filled in
-	 * with a list of "recycled transmit buffers" (i.e. completed
-	 * TX buffers).  Observant readers may care to note that
-	 * *txbufs is a void pointer.  Precisely how a list of
-	 * completed transmit buffers is meant to be represented as an
-	 * array of voids is left as an exercise for the reader.
-	 *
-	 * The only users of this interface (MnpDxe/MnpIo.c and
-	 * PxeBcDxe/Bc.c within the EFI dev kit) both just poll until
-	 * seeing a non-NULL result return in txbufs.  This is valid
-	 * provided that they do not ever attempt to transmit more
-	 * than one packet concurrently (and that TX never times out).
-	 */
-	if ( txbufs ) {
-		if ( snpdev->tx_count_txbufs &&
-		     list_empty ( &snpdev->netdev->tx_queue ) ) {
-			*txbufs = "Which idiot designed this API?";
-			snpdev->tx_count_txbufs--;
+	/* TX completions */
+	if ( txbuf ) {
+		if ( snpdev->tx_prod != snpdev->tx_cons ) {
+			*txbuf = snpdev->tx[snpdev->tx_cons++ % EFI_SNP_NUM_TX];
 		} else {
-			*txbufs = NULL;
+			*txbuf = NULL;
 		}
-		DBGC2 ( snpdev, " TX:%s", ( *txbufs ? "some" : "none" ) );
+		DBGC2 ( snpdev, " TX:%p", *txbuf );
 	}
 
 	DBGC2 ( snpdev, "\n" );
@@ -537,6 +563,7 @@ efi_snp_transmit ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	struct ll_protocol *ll_protocol = snpdev->netdev->ll_protocol;
 	struct io_buffer *iobuf;
 	size_t payload_len;
+	unsigned int tx_fill;
 	int rc;
 
 	DBGC2 ( snpdev, "SNPDEV %p TRANSMIT %p+%lx", snpdev, data,
@@ -624,12 +651,27 @@ efi_snp_transmit ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 		goto err_tx;
 	}
 
-	/* Record transmission as outstanding */
-	snpdev->tx_count_interrupts++;
-	snpdev->tx_count_txbufs++;
+	/* Record in transmit completion ring.  If we run out of
+	 * space, report the failure even though we have already
+	 * transmitted the packet.
+	 *
+	 * This allows us to report completions only for packets for
+	 * which we had reported successfully initiating transmission,
+	 * while continuing to support clients that never poll for
+	 * transmit completions.
+	 */
+	tx_fill = ( snpdev->tx_prod - snpdev->tx_cons );
+	if ( tx_fill >= EFI_SNP_NUM_TX ) {
+		DBGC ( snpdev, "SNPDEV %p TX completion ring full\n", snpdev );
+		rc = -ENOBUFS;
+		goto err_ring_full;
+	}
+	snpdev->tx[ snpdev->tx_prod++ % EFI_SNP_NUM_TX ] = data;
+	snpdev->interrupts |= EFI_SIMPLE_NETWORK_TRANSMIT_INTERRUPT;
 
 	return 0;
 
+ err_ring_full:
  err_tx:
  err_ll_push:
 	free_iob ( iobuf );
@@ -676,12 +718,13 @@ efi_snp_receive ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
 	efi_snp_poll ( snpdev );
 
 	/* Dequeue a packet, if one is available */
-	iobuf = netdev_rx_dequeue ( snpdev->netdev );
+	iobuf = list_first_entry ( &snpdev->rx, struct io_buffer, list );
 	if ( ! iobuf ) {
 		DBGC2 ( snpdev, "\n" );
 		rc = -EAGAIN;
 		goto out_no_packet;
 	}
+	list_del ( &iobuf->list );
 	DBGC2 ( snpdev, "+%zx\n", iob_len ( iobuf ) );
 
 	/* Return packet to caller */
@@ -721,9 +764,8 @@ efi_snp_receive ( EFI_SIMPLE_NETWORK_PROTOCOL *snp,
  * @v event		Event
  * @v context		Event context
  */
-static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event,
+static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event __unused,
 					     VOID *context ) {
-	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
 	struct efi_snp_device *snpdev = context;
 
 	DBGCP ( snpdev, "SNPDEV %p WAIT_FOR_PACKET\n", snpdev );
@@ -738,14 +780,6 @@ static VOID EFIAPI efi_snp_wait_for_packet ( EFI_EVENT event,
 
 	/* Poll the network device */
 	efi_snp_poll ( snpdev );
-
-	/* Fire event if packets have been received */
-	if ( snpdev->rx_count_events != 0 ) {
-		DBGC2 ( snpdev, "SNPDEV %p firing WaitForPacket event\n",
-			snpdev );
-		bs->SignalEvent ( event );
-		snpdev->rx_count_events--;
-	}
 }
 
 /** SNP interface */
@@ -837,6 +871,7 @@ efi_snp_load_file ( EFI_LOAD_FILE_PROTOCOL *load_file,
 	struct efi_snp_device *snpdev =
 		container_of ( load_file, struct efi_snp_device, load_file );
 	struct net_device *netdev = snpdev->netdev;
+	int rc;
 
 	/* Fail unless this is a boot attempt */
 	if ( ! booting ) {
@@ -848,14 +883,17 @@ efi_snp_load_file ( EFI_LOAD_FILE_PROTOCOL *load_file,
 	/* Claim network devices for use by iPXE */
 	efi_snp_claim();
 
+	/* Start watchdog holdoff timer */
+	efi_watchdog_start();
+
 	/* Boot from network device */
-	ipxe ( netdev );
+	if ( ( rc = ipxe ( netdev ) ) != 0 )
+		goto err_ipxe;
 
-	/* Release network devices for use via SNP */
+ err_ipxe:
+	efi_watchdog_stop();
 	efi_snp_release();
-
-	/* Assume boot process was aborted */
-	return EFI_ABORTED;
+	return EFIRC ( rc );
 }
 
 /** Load file protocol */
@@ -922,6 +960,7 @@ static int efi_snp_probe ( struct net_device *netdev ) {
 	}
 	snpdev->netdev = netdev_get ( netdev );
 	snpdev->efidev = efidev;
+	INIT_LIST_HEAD ( &snpdev->rx );
 
 	/* Sanity check */
 	if ( netdev->ll_protocol->ll_addr_len > sizeof ( EFI_MAC_ADDRESS ) ) {
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_snp_hii.c b/qemu/roms/ipxe/src/interface/efi/efi_snp_hii.c
index c49c76a32..720402bdb 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_snp_hii.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_snp_hii.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -59,6 +63,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/efi/efi_hii.h>
 #include <ipxe/efi/efi_snp.h>
 #include <ipxe/efi/efi_strings.h>
+#include <config/branding.h>
 
 /** EFI platform setup formset GUID */
 static EFI_GUID efi_hii_platform_setup_formset_guid
@@ -136,7 +141,7 @@ static void efi_snp_hii_questions ( struct efi_snp_device *snpdev,
 		previous = setting;
 		name_id = efi_ifr_string ( ifr, "%s", setting->name );
 		prompt_id = efi_ifr_string ( ifr, "%s", setting->description );
-		help_id = efi_ifr_string ( ifr, "http://ipxe.org/cfg/%s",
+		help_id = efi_ifr_string ( ifr, PRODUCT_SETTING_URI,
 					   setting->name );
 		question_id = setting->tag;
 		efi_ifr_string_op ( ifr, prompt_id, help_id,
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_strings.c b/qemu/roms/ipxe/src/interface/efi/efi_strings.c
index 751589b46..aa3afc64f 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_strings.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_strings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdarg.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_time.c b/qemu/roms/ipxe/src/interface/efi/efi_time.c
new file mode 100644
index 000000000..983a0ef5c
--- /dev/null
+++ b/qemu/roms/ipxe/src/interface/efi/efi_time.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <string.h>
+#include <errno.h>
+#include <time.h>
+#include <ipxe/time.h>
+#include <ipxe/efi/efi.h>
+
+/** @file
+ *
+ * EFI time source
+ *
+ */
+
+/**
+ * Get current time in seconds
+ *
+ * @ret time		Time, in seconds
+ */
+static time_t efi_get_time ( void ) {
+	EFI_RUNTIME_SERVICES *rs = efi_systab->RuntimeServices;
+	EFI_TIME time;
+	struct tm tm;
+	EFI_STATUS efirc;
+	int rc;
+
+	/* Get current time and date */
+	if ( ( efirc = rs->GetTime ( &time, NULL ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( rs, "EFITIME could not get system time: %s\n",
+		       strerror ( rc ) );
+		/* Nothing meaningful we can return */
+		return 0;
+	}
+
+	/* Construct broken-down time */
+	memset ( &tm, 0, sizeof ( tm ) );
+	tm.tm_sec = time.Second;
+	tm.tm_min = time.Minute;
+	tm.tm_hour = time.Hour;
+	tm.tm_mday = time.Day;
+	tm.tm_mon = ( time.Month - 1 );
+	tm.tm_year = ( time.Year - 1900 );
+	DBGC ( rs, "EFITIME is %04d-%02d-%02d %02d:%02d:%02d\n",
+	       ( tm.tm_year + 1900 ), ( tm.tm_mon + 1 ),
+	       tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec );
+
+	/* Convert to seconds since the Epoch */
+	return mktime ( &tm );
+}
+
+PROVIDE_TIME ( efi, time_now, efi_get_time );
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_timer.c b/qemu/roms/ipxe/src/interface/efi/efi_timer.c
index 7a1ff7869..81620c92c 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_timer.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_timer.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_uaccess.c b/qemu/roms/ipxe/src/interface/efi/efi_uaccess.c
index 8b429b9ee..e058be66b 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_uaccess.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_uaccess.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/uaccess.h>
 #include <ipxe/efi/efi.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_umalloc.c b/qemu/roms/ipxe/src/interface/efi/efi_umalloc.c
index 356efaa6f..e3f1dacc2 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_umalloc.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_umalloc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_watchdog.c b/qemu/roms/ipxe/src/interface/efi/efi_watchdog.c
new file mode 100644
index 000000000..7061f81d8
--- /dev/null
+++ b/qemu/roms/ipxe/src/interface/efi/efi_watchdog.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * EFI watchdog holdoff timer
+ *
+ */
+
+#include <errno.h>
+#include <string.h>
+#include <ipxe/retry.h>
+#include <ipxe/timer.h>
+#include <ipxe/efi/efi.h>
+#include <ipxe/efi/efi_watchdog.h>
+
+/** Watchdog holdoff interval (in seconds) */
+#define WATCHDOG_HOLDOFF_SECS 10
+
+/** Watchdog timeout (in seconds) */
+#define WATCHDOG_TIMEOUT_SECS ( 5 * 60 )
+
+/** Watchdog code (to be logged on watchdog timeout) */
+#define WATCHDOG_CODE 0x6950584544454144ULL
+
+/** Watchdog data (to be logged on watchdog timeout) */
+#define WATCHDOG_DATA L"iPXE";
+
+/**
+ * Hold off watchdog timer
+ *
+ * @v retry		Retry timer
+ * @v over		Failure indicator
+ */
+static void efi_watchdog_expired ( struct retry_timer *timer,
+				   int over __unused ) {
+	EFI_BOOT_SERVICES *bs = efi_systab->BootServices;
+	static CHAR16 data[] = WATCHDOG_DATA;
+	EFI_STATUS efirc;
+	int rc;
+
+	DBGC2 ( timer, "EFI holding off watchdog timer\n" );
+
+	/* Restart this holdoff timer */
+	start_timer_fixed ( timer, ( WATCHDOG_HOLDOFF_SECS * TICKS_PER_SEC ) );
+
+	/* Reset watchdog timer */
+	if ( ( efirc = bs->SetWatchdogTimer ( WATCHDOG_TIMEOUT_SECS,
+					      WATCHDOG_CODE, sizeof ( data ),
+					      data ) ) != 0 ) {
+		rc = -EEFI ( efirc );
+		DBGC ( timer, "EFI could not set watchdog timer: %s\n",
+		       strerror ( rc ) );
+		return;
+	}
+}
+
+/** Watchdog holdoff timer */
+struct retry_timer efi_watchdog = TIMER_INIT ( efi_watchdog_expired );
diff --git a/qemu/roms/ipxe/src/interface/efi/efi_wrap.c b/qemu/roms/ipxe/src/interface/efi/efi_wrap.c
index ff46b76ed..2ea184e97 100644
--- a/qemu/roms/ipxe/src/interface/efi/efi_wrap.c
+++ b/qemu/roms/ipxe/src/interface/efi/efi_wrap.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/interface/hyperv/vmbus.c b/qemu/roms/ipxe/src/interface/hyperv/vmbus.c
new file mode 100644
index 000000000..795929eae
--- /dev/null
+++ b/qemu/roms/ipxe/src/interface/hyperv/vmbus.c
@@ -0,0 +1,1333 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Hyper-V virtual machine bus
+ *
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <ipxe/nap.h>
+#include <ipxe/malloc.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/hyperv.h>
+#include <ipxe/vmbus.h>
+
+/** VMBus initial GPADL ID
+ *
+ * This is an opaque value with no meaning.  The Linux kernel uses
+ * 0xe1e10.
+ */
+#define VMBUS_GPADL_MAGIC 0x18ae0000
+
+/**
+ * Post message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v header		Message header
+ * @v len		Length of message (including header)
+ * @ret rc		Return status code
+ */
+static int vmbus_post_message ( struct hv_hypervisor *hv,
+				const struct vmbus_message_header *header,
+				size_t len ) {
+	struct vmbus *vmbus = hv->vmbus;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = hv_post_message ( hv, VMBUS_MESSAGE_ID, VMBUS_MESSAGE_TYPE,
+				      header, len ) ) != 0 ) {
+		DBGC ( vmbus, "VMBUS %p could not post message: %s\n",
+		       vmbus, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Post empty message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v type		Message type
+ * @ret rc		Return status code
+ */
+static int vmbus_post_empty_message ( struct hv_hypervisor *hv,
+				      unsigned int type ) {
+	struct vmbus_message_header header = { .type = cpu_to_le32 ( type ) };
+
+	return vmbus_post_message ( hv, &header, sizeof ( header ) );
+}
+
+/**
+ * Wait for received message
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_wait_for_message ( struct hv_hypervisor *hv ) {
+	struct vmbus *vmbus = hv->vmbus;
+	int rc;
+
+	/* Wait for message */
+	if ( ( rc = hv_wait_for_message ( hv, VMBUS_MESSAGE_SINT ) ) != 0 ) {
+		DBGC ( vmbus, "VMBUS %p failed waiting for message: %s\n",
+		       vmbus, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Sanity check */
+	if ( hv->message->received.type != cpu_to_le32 ( VMBUS_MESSAGE_TYPE ) ){
+		DBGC ( vmbus, "VMBUS %p invalid message type %d\n",
+		       vmbus, le32_to_cpu ( hv->message->received.type ) );
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * Initiate contact
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v raw		VMBus protocol (raw) version
+ * @ret rc		Return status code
+ */
+static int vmbus_initiate_contact ( struct hv_hypervisor *hv,
+				    unsigned int raw ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_version_response *version = &vmbus->message->version;
+	struct vmbus_initiate_contact initiate;
+	int rc;
+
+	/* Construct message */
+	memset ( &initiate, 0, sizeof ( initiate ) );
+	initiate.header.type = cpu_to_le32 ( VMBUS_INITIATE_CONTACT );
+	initiate.version.raw = cpu_to_le32 ( raw );
+	initiate.intr = virt_to_phys ( vmbus->intr );
+	initiate.monitor_in = virt_to_phys ( vmbus->monitor_in );
+	initiate.monitor_out = virt_to_phys ( vmbus->monitor_out );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &initiate.header,
+					 sizeof ( initiate ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( version->header.type != cpu_to_le32 ( VMBUS_VERSION_RESPONSE ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected version response type %d\n",
+		       vmbus, le32_to_cpu ( version->header.type ) );
+		return -EPROTO;
+	}
+	if ( ! version->supported ) {
+		DBGC ( vmbus, "VMBUS %p requested version not supported\n",
+		       vmbus );
+		return -ENOTSUP;
+	}
+	if ( version->version.raw != cpu_to_le32 ( raw ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected version %d.%d\n",
+		       vmbus, le16_to_cpu ( version->version.major ),
+		       le16_to_cpu ( version->version.minor ) );
+		return -EPROTO;
+	}
+
+	DBGC ( vmbus, "VMBUS %p initiated contact using version %d.%d\n",
+	       vmbus, le16_to_cpu ( version->version.major ),
+	       le16_to_cpu ( version->version.minor ) );
+	return 0;
+}
+
+/**
+ * Terminate contact
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_unload ( struct hv_hypervisor *hv ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_message_header *header = &vmbus->message->header;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = vmbus_post_empty_message ( hv, VMBUS_UNLOAD ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( header->type != cpu_to_le32 ( VMBUS_UNLOAD_RESPONSE ) ) {
+		DBGC ( vmbus, "VMBUS %p unexpected unload response type %d\n",
+		       vmbus, le32_to_cpu ( header->type ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Negotiate protocol version
+ *
+ * @v hv		Hyper-V hypervisor
+ * @ret rc		Return status code
+ */
+static int vmbus_negotiate_version ( struct hv_hypervisor *hv ) {
+	int rc;
+
+	/* We require the ability to disconnect from and reconnect to
+	 * VMBus; if we don't have this then there is no (viable) way
+	 * for a loaded operating system to continue to use any VMBus
+	 * devices.  (There is also a small but non-zero risk that the
+	 * host will continue to write to our interrupt and monitor
+	 * pages, since the VMBUS_UNLOAD message in earlier versions
+	 * is essentially a no-op.)
+	 *
+	 * This requires us to ensure that the host supports protocol
+	 * version 3.0 (VMBUS_VERSION_WIN8_1).  However, we can't
+	 * actually _use_ protocol version 3.0, since doing so causes
+	 * an iSCSI-booted Windows Server 2012 R2 VM to crash due to a
+	 * NULL pointer dereference in vmbus.sys.
+	 *
+	 * To work around this problem, we first ensure that we can
+	 * connect using protocol v3.0, then disconnect and reconnect
+	 * using the oldest known protocol.
+	 */
+
+	/* Initiate contact to check for required protocol support */
+	if ( ( rc = vmbus_initiate_contact ( hv, VMBUS_VERSION_WIN8_1 ) ) != 0 )
+		return rc;
+
+	/* Terminate contact */
+	if ( ( rc = vmbus_unload ( hv ) ) != 0 )
+		return rc;
+
+	/* Reinitiate contact using the oldest known protocol version */
+	if ( ( rc = vmbus_initiate_contact ( hv, VMBUS_VERSION_WS2008 ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Establish GPA descriptor list
+ *
+ * @v vmdev		VMBus device
+ * @v data		Data buffer
+ * @v len		Length of data buffer
+ * @ret gpadl		GPADL ID, or negative error
+ */
+int vmbus_establish_gpadl ( struct vmbus_device *vmdev, userptr_t data,
+			    size_t len ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	physaddr_t addr = user_to_phys ( data, 0 );
+	unsigned int pfn_count = hv_pfn_count ( addr, len );
+	struct {
+		struct vmbus_gpadl_header gpadlhdr;
+		struct vmbus_gpa_range range;
+		uint64_t pfn[pfn_count];
+	} __attribute__ (( packed )) gpadlhdr;
+	const struct vmbus_gpadl_created *created = &vmbus->message->created;
+	static unsigned int gpadl = VMBUS_GPADL_MAGIC;
+	unsigned int i;
+	int rc;
+
+	/* Allocate GPADL ID */
+	gpadl++;
+
+	/* Construct message */
+	memset ( &gpadlhdr, 0, sizeof ( gpadlhdr ) );
+	gpadlhdr.gpadlhdr.header.type = cpu_to_le32 ( VMBUS_GPADL_HEADER );
+	gpadlhdr.gpadlhdr.channel = cpu_to_le32 ( vmdev->channel );
+	gpadlhdr.gpadlhdr.gpadl = cpu_to_le32 ( gpadl );
+	gpadlhdr.gpadlhdr.range_len =
+		cpu_to_le16 ( ( sizeof ( gpadlhdr.range ) +
+				sizeof ( gpadlhdr.pfn ) ) );
+	gpadlhdr.gpadlhdr.range_count = cpu_to_le16 ( 1 );
+	gpadlhdr.range.len = cpu_to_le32 ( len );
+	gpadlhdr.range.offset = cpu_to_le32 ( addr & ( PAGE_SIZE - 1 ) );
+	for ( i = 0 ; i < pfn_count ; i++ )
+		gpadlhdr.pfn[i] = ( ( addr / PAGE_SIZE ) + i );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &gpadlhdr.gpadlhdr.header,
+					 sizeof ( gpadlhdr ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( created->header.type != cpu_to_le32 ( VMBUS_GPADL_CREATED ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( created->header.type ) );
+		return -EPROTO;
+	}
+	if ( created->channel != cpu_to_le32 ( vmdev->channel ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL channel %d\n",
+		       vmdev->dev.name, le32_to_cpu ( created->channel ) );
+		return -EPROTO;
+	}
+	if ( created->gpadl != cpu_to_le32 ( gpadl ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( created->gpadl ) );
+		return -EPROTO;
+	}
+	if ( created->status != 0 ) {
+		DBGC ( vmdev, "VMBUS %s GPADL creation failed: %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( created->status ) );
+		return -EPROTO;
+	}
+
+	DBGC ( vmdev, "VMBUS %s GPADL %#08x is [%08lx,%08lx)\n",
+	       vmdev->dev.name, gpadl, addr, ( addr + len ) );
+	return gpadl;
+}
+
+/**
+ * Tear down GPA descriptor list
+ *
+ * @v vmdev		VMBus device
+ * @v gpadl		GPADL ID
+ * @ret rc		Return status code
+ */
+int vmbus_gpadl_teardown ( struct vmbus_device *vmdev, unsigned int gpadl ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct vmbus_gpadl_teardown teardown;
+	const struct vmbus_gpadl_torndown *torndown = &vmbus->message->torndown;
+	int rc;
+
+	/* Construct message */
+	memset ( &teardown, 0, sizeof ( teardown ) );
+	teardown.header.type = cpu_to_le32 ( VMBUS_GPADL_TEARDOWN );
+	teardown.channel = cpu_to_le32 ( vmdev->channel );
+	teardown.gpadl = cpu_to_le32 ( gpadl );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &teardown.header,
+					 sizeof ( teardown ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( torndown->header.type != cpu_to_le32 ( VMBUS_GPADL_TORNDOWN ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( torndown->header.type ) );
+		return -EPROTO;
+	}
+	if ( torndown->gpadl != cpu_to_le32 ( gpadl ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected GPADL ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( torndown->gpadl ) );
+		return -EPROTO;
+	}
+
+	return 0;
+}
+
+/**
+ * Open VMBus channel
+ *
+ * @v vmdev		VMBus device
+ * @v op		Channel operations
+ * @v out_len		Outbound ring buffer length
+ * @v in_len		Inbound ring buffer length
+ * @v mtu		Maximum expected data packet length (including headers)
+ * @ret rc		Return status code
+ *
+ * Both outbound and inbound ring buffer lengths must be a power of
+ * two and a multiple of PAGE_SIZE.  The requirement to be a power of
+ * two is a policy decision taken to simplify the ring buffer indexing
+ * logic.
+ */
+int vmbus_open ( struct vmbus_device *vmdev,
+		 struct vmbus_channel_operations *op,
+		 size_t out_len, size_t in_len, size_t mtu ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct vmbus_open_channel open;
+	const struct vmbus_open_channel_result *opened =
+		&vmbus->message->opened;
+	size_t len;
+	void *ring;
+	void *packet;
+	int gpadl;
+	uint32_t open_id;
+	int rc;
+
+	/* Sanity checks */
+	assert ( ( out_len % PAGE_SIZE ) == 0 );
+	assert ( ( out_len & ( out_len - 1 ) ) == 0 );
+	assert ( ( in_len % PAGE_SIZE ) == 0 );
+	assert ( ( in_len & ( in_len - 1 ) ) == 0 );
+	assert ( mtu >= ( sizeof ( struct vmbus_packet_header ) +
+			  sizeof ( struct vmbus_packet_footer ) ) );
+
+	/* Allocate packet buffer */
+	packet = malloc ( mtu );
+	if ( ! packet ) {
+		rc = -ENOMEM;
+		goto err_alloc_packet;
+	}
+
+	/* Allocate ring buffer */
+	len = ( sizeof ( *vmdev->out ) + out_len +
+		sizeof ( *vmdev->in ) + in_len );
+	assert ( ( len % PAGE_SIZE ) == 0 );
+	ring = malloc_dma ( len, PAGE_SIZE );
+	if ( ! ring ) {
+		rc = -ENOMEM;
+		goto err_alloc_ring;
+	}
+	memset ( ring, 0, len );
+
+	/* Establish GPADL for ring buffer */
+	gpadl = vmbus_establish_gpadl ( vmdev, virt_to_user ( ring ), len );
+	if ( gpadl < 0 ) {
+		rc = gpadl;
+		goto err_establish;
+	}
+
+	/* Construct message */
+	memset ( &open, 0, sizeof ( open ) );
+	open.header.type = cpu_to_le32 ( VMBUS_OPEN_CHANNEL );
+	open.channel = cpu_to_le32 ( vmdev->channel );
+	open_id = random();
+	open.id = open_id; /* Opaque random value: endianness irrelevant */
+	open.gpadl = cpu_to_le32 ( gpadl );
+	open.out_pages = ( ( sizeof ( *vmdev->out ) / PAGE_SIZE ) +
+			   ( out_len / PAGE_SIZE ) );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &open.header,
+					 sizeof ( open ) ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+		return rc;
+
+	/* Check response */
+	if ( opened->header.type != cpu_to_le32 ( VMBUS_OPEN_CHANNEL_RESULT ) ){
+		DBGC ( vmdev, "VMBUS %s unexpected open response type %d\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->header.type ) );
+		return -EPROTO;
+	}
+	if ( opened->channel != cpu_to_le32 ( vmdev->channel ) ) {
+		DBGC ( vmdev, "VMBUS %s unexpected opened channel %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->channel ) );
+		return -EPROTO;
+	}
+	if ( opened->id != open_id /* Non-endian */ ) {
+		DBGC ( vmdev, "VMBUS %s unexpected open ID %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->id ) );
+		return -EPROTO;
+	}
+	if ( opened->status != 0 ) {
+		DBGC ( vmdev, "VMBUS %s open failed: %#08x\n",
+		       vmdev->dev.name, le32_to_cpu ( opened->status ) );
+		return -EPROTO;
+	}
+
+	/* Store channel parameters */
+	vmdev->out_len = out_len;
+	vmdev->in_len = in_len;
+	vmdev->out = ring;
+	vmdev->in = ( ring + sizeof ( *vmdev->out ) + out_len );
+	vmdev->gpadl = gpadl;
+	vmdev->op = op;
+	vmdev->mtu = mtu;
+	vmdev->packet = packet;
+
+	DBGC ( vmdev, "VMBUS %s channel GPADL %#08x ring "
+		"[%#08lx,%#08lx,%#08lx)\n", vmdev->dev.name, vmdev->gpadl,
+		virt_to_phys ( vmdev->out ), virt_to_phys ( vmdev->in ),
+		( virt_to_phys ( vmdev->out ) + len ) );
+	return 0;
+
+	vmbus_gpadl_teardown ( vmdev, vmdev->gpadl );
+ err_establish:
+	free_dma ( ring, len );
+ err_alloc_ring:
+	free ( packet );
+ err_alloc_packet:
+	return rc;
+}
+
+/**
+ * Close VMBus channel
+ *
+ * @v vmdev		VMBus device
+ */
+void vmbus_close ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus_close_channel close;
+	size_t len;
+	int rc;
+
+	/* Construct message */
+	memset ( &close, 0, sizeof ( close ) );
+	close.header.type = cpu_to_le32 ( VMBUS_CLOSE_CHANNEL );
+	close.channel = cpu_to_le32 ( vmdev->channel );
+
+	/* Post message */
+	if ( ( rc = vmbus_post_message ( hv, &close.header,
+					 sizeof ( close ) ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s failed to close: %s\n",
+		       vmdev->dev.name, strerror ( rc ) );
+		/* Continue to attempt to tear down GPADL, so that our
+		 * memory is no longer accessible by the remote VM.
+		 */
+	}
+
+	/* Tear down GPADL */
+	if ( ( rc = vmbus_gpadl_teardown ( vmdev,
+					   vmdev->gpadl ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s failed to tear down channel GPADL: "
+		       "%s\n", vmdev->dev.name, strerror ( rc ) );
+		/* We can't prevent the remote VM from continuing to
+		 * access this memory, so leak it.
+		 */
+		return;
+	}
+
+	/* Free ring buffer */
+	len = ( sizeof ( *vmdev->out ) + vmdev->out_len +
+		sizeof ( *vmdev->in ) + vmdev->in_len );
+	free_dma ( vmdev->out, len );
+	vmdev->out = NULL;
+	vmdev->in = NULL;
+
+	/* Free packet buffer */
+	free ( vmdev->packet );
+	vmdev->packet = NULL;
+
+	DBGC ( vmdev, "VMBUS %s closed\n", vmdev->dev.name );
+}
+
+/**
+ * Signal channel via monitor page
+ *
+ * @v vmdev		VMBus device
+ */
+static void vmbus_signal_monitor ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	struct hv_monitor_trigger *trigger;
+	unsigned int group;
+	unsigned int bit;
+
+	/* Set bit in monitor trigger group */
+	group = ( vmdev->monitor / ( 8 * sizeof ( trigger->pending ) ));
+	bit = ( vmdev->monitor % ( 8 * sizeof ( trigger->pending ) ) );
+	trigger = &vmbus->monitor_out->trigger[group];
+	hv_set_bit ( trigger, bit );
+}
+
+/**
+ * Signal channel via hypervisor event
+ *
+ * @v vmdev		VMBus device
+ */
+static void vmbus_signal_event ( struct vmbus_device *vmdev ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	int rc;
+
+	/* Signal hypervisor event */
+	if ( ( rc = hv_signal_event ( hv, VMBUS_EVENT_ID, 0 ) ) != 0 ) {
+		DBGC ( vmdev, "VMBUS %s could not signal event: %s\n",
+		       vmdev->dev.name, strerror ( rc ) );
+		return;
+	}
+}
+
+/**
+ * Fill outbound ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v prod		Producer index
+ * @v data		Data
+ * @v len		Length
+ * @ret prod		New producer index
+ *
+ * The caller must ensure that there is sufficient space in the ring
+ * buffer.
+ */
+static size_t vmbus_produce ( struct vmbus_device *vmdev, size_t prod,
+			      const void *data, size_t len ) {
+	size_t first;
+	size_t second;
+
+	/* Determine fragment lengths */
+	first = ( vmdev->out_len - prod );
+	if ( first > len )
+		first = len;
+	second = ( len - first );
+
+	/* Copy fragment(s) */
+	memcpy ( &vmdev->out->data[prod], data, first );
+	if ( second )
+		memcpy ( &vmdev->out->data[0], ( data + first ), second );
+
+	return ( ( prod + len ) & ( vmdev->out_len - 1 ) );
+}
+
+/**
+ * Consume inbound ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v cons		Consumer index
+ * @v data		Data buffer, or NULL
+ * @v len		Length to consume
+ * @ret cons		New consumer index
+ */
+static size_t vmbus_consume ( struct vmbus_device *vmdev, size_t cons,
+			      void *data, size_t len ) {
+	size_t first;
+	size_t second;
+
+	/* Determine fragment lengths */
+	first = ( vmdev->in_len - cons );
+	if ( first > len )
+		first = len;
+	second = ( len - first );
+
+	/* Copy fragment(s) */
+	memcpy ( data, &vmdev->in->data[cons], first );
+	if ( second )
+		memcpy ( ( data + first ), &vmdev->in->data[0], second );
+
+	return ( ( cons + len ) & ( vmdev->in_len - 1 ) );
+}
+
+/**
+ * Send packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v header		Packet header
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send a packet via the outbound ring buffer.  All fields in the
+ * packet header must be filled in, with the exception of the total
+ * packet length.
+ */
+static int vmbus_send ( struct vmbus_device *vmdev,
+			struct vmbus_packet_header *header,
+			const void *data, size_t len ) {
+	struct hv_hypervisor *hv = vmdev->hv;
+	struct vmbus *vmbus = hv->vmbus;
+	static uint8_t padding[ 8 - 1 ];
+	struct vmbus_packet_footer footer;
+	size_t header_len;
+	size_t pad_len;
+	size_t footer_len;
+	size_t ring_len;
+	size_t cons;
+	size_t prod;
+	size_t old_prod;
+	size_t fill;
+
+	/* Sanity check */
+	assert ( vmdev->out != NULL );
+
+	/* Calculate lengths */
+	header_len = ( le16_to_cpu ( header->hdr_qlen ) * 8 );
+	pad_len = ( ( -len ) & ( 8 - 1 ) );
+	footer_len = sizeof ( footer );
+	ring_len = ( header_len + len + pad_len + footer_len );
+
+	/* Check that we have enough room in the outbound ring buffer */
+	cons = le32_to_cpu ( vmdev->out->cons );
+	prod = le32_to_cpu ( vmdev->out->prod );
+	old_prod = prod;
+	fill = ( ( prod - cons ) & ( vmdev->out_len - 1 ) );
+	if ( ( fill + ring_len ) >= vmdev->out_len ) {
+		DBGC ( vmdev, "VMBUS %s ring buffer full\n", vmdev->dev.name );
+		return -ENOBUFS;
+	}
+
+	/* Complete header */
+	header->qlen = cpu_to_le16 ( ( ring_len - footer_len ) / 8 );
+
+	/* Construct footer */
+	footer.reserved = 0;
+	footer.prod = vmdev->out->prod;
+
+	/* Copy packet to buffer */
+	DBGC2 ( vmdev, "VMBUS %s sending:\n", vmdev->dev.name );
+	DBGC2_HDA ( vmdev, prod, header, header_len );
+	prod = vmbus_produce ( vmdev, prod, header, header_len );
+	DBGC2_HDA ( vmdev, prod, data, len );
+	prod = vmbus_produce ( vmdev, prod, data, len );
+	prod = vmbus_produce ( vmdev, prod, padding, pad_len );
+	DBGC2_HDA ( vmdev, prod, &footer, sizeof ( footer ) );
+	prod = vmbus_produce ( vmdev, prod, &footer, sizeof ( footer ) );
+	assert ( ( ( prod - old_prod ) & ( vmdev->out_len - 1 ) ) == ring_len );
+
+	/* Update producer index */
+	wmb();
+	vmdev->out->prod = cpu_to_le32 ( prod );
+
+	/* Return if we do not need to signal the host.  This follows
+	 * the logic of hv_need_to_signal() in the Linux driver.
+	 */
+	mb();
+	if ( vmdev->out->intr_mask )
+		return 0;
+	rmb();
+	cons = le32_to_cpu ( vmdev->out->cons );
+	if ( cons != old_prod )
+		return 0;
+
+	/* Set channel bit in interrupt page */
+	hv_set_bit ( vmbus->intr->out, vmdev->channel );
+
+	/* Signal the host */
+	vmdev->signal ( vmdev );
+
+	return 0;
+}
+
+/**
+ * Send control packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID (or zero to not request completion)
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_DATA_INBAND packet.
+ */
+int vmbus_send_control ( struct vmbus_device *vmdev, uint64_t xid,
+			 const void *data, size_t len ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_DATA_INBAND );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = ( xid ?
+			  cpu_to_le16 ( VMBUS_COMPLETION_REQUESTED ) : 0 );
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, data, len );
+}
+
+/**
+ * Send data packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_DATA_GPA_DIRECT packet.  The caller is
+ * responsible for ensuring that the I/O buffer remains untouched
+ * until the corresponding completion has been received.
+ */
+int vmbus_send_data ( struct vmbus_device *vmdev, uint64_t xid,
+		      const void *data, size_t len, struct io_buffer *iobuf ) {
+	physaddr_t addr = virt_to_phys ( iobuf->data );
+	unsigned int pfn_count = hv_pfn_count ( addr, iob_len ( iobuf ) );
+	struct {
+		struct vmbus_gpa_direct_header gpa;
+		struct vmbus_gpa_range range;
+		uint64_t pfn[pfn_count];
+	} __attribute__ (( packed )) *header = vmdev->packet;
+	unsigned int i;
+
+	/* Sanity check */
+	assert ( header != NULL );
+	assert ( sizeof ( *header ) <= vmdev->mtu );
+
+	/* Construct header in packet buffer */
+	header->gpa.header.type = cpu_to_le16 ( VMBUS_DATA_GPA_DIRECT );
+	header->gpa.header.hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->gpa.header.flags = cpu_to_le16 ( VMBUS_COMPLETION_REQUESTED );
+	header->gpa.header.xid = xid; /* Non-endian */
+	header->gpa.range_count = 1;
+	header->range.len = cpu_to_le32 ( iob_len ( iobuf ) );
+	header->range.offset = cpu_to_le32 ( addr & ( PAGE_SIZE - 1 ) );
+	for ( i = 0 ; i < pfn_count ; i++ )
+		header->pfn[i] = ( ( addr / PAGE_SIZE ) + i );
+
+	return vmbus_send ( vmdev, &header->gpa.header, data, len );
+}
+
+/**
+ * Send completion packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @v data		Data
+ * @v len		Length of data
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_COMPLETION packet.
+ */
+int vmbus_send_completion ( struct vmbus_device *vmdev, uint64_t xid,
+			    const void *data, size_t len ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_COMPLETION );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = 0;
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, data, len );
+}
+
+/**
+ * Send cancellation packet via ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @v xid		Transaction ID
+ * @ret rc		Return status code
+ *
+ * Send data using a VMBUS_CANCELLATION packet.
+ */
+int vmbus_send_cancellation ( struct vmbus_device *vmdev, uint64_t xid ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+
+	/* Construct header in packet buffer */
+	assert ( header != NULL );
+	header->type = cpu_to_le16 ( VMBUS_CANCELLATION );
+	header->hdr_qlen = cpu_to_le16 ( sizeof ( *header ) / 8 );
+	header->flags = 0;
+	header->xid = xid; /* Non-endian */
+
+	return vmbus_send ( vmdev, header, NULL, 0 );
+}
+
+/**
+ * Get transfer page set from pageset ID
+ *
+ * @v vmdev		VMBus device
+ * @v pageset		Page set ID (in protocol byte order)
+ * @ret pages		Page set, or NULL if not found
+ */
+static struct vmbus_xfer_pages * vmbus_xfer_pages ( struct vmbus_device *vmdev,
+						    uint16_t pageset ) {
+	struct vmbus_xfer_pages *pages;
+
+	/* Locate page set */
+	list_for_each_entry ( pages, &vmdev->pages, list ) {
+		if ( pages->pageset == pageset )
+			return pages;
+	}
+
+	DBGC ( vmdev, "VMBUS %s unrecognised page set ID %#04x\n",
+	       vmdev->dev.name, le16_to_cpu ( pageset ) );
+	return NULL;
+}
+
+/**
+ * Construct I/O buffer list from transfer pages
+ *
+ * @v vmdev		VMBus device
+ * @v header		Transfer page header
+ * @v list		I/O buffer list to populate
+ * @ret rc		Return status code
+ */
+static int vmbus_xfer_page_iobufs ( struct vmbus_device *vmdev,
+				    struct vmbus_packet_header *header,
+				    struct list_head *list ) {
+	struct vmbus_xfer_page_header *page_header =
+		container_of ( header, struct vmbus_xfer_page_header, header );
+	struct vmbus_xfer_pages *pages;
+	struct io_buffer *iobuf;
+	struct io_buffer *tmp;
+	size_t len;
+	size_t offset;
+	unsigned int range_count;
+	unsigned int i;
+	int rc;
+
+	/* Sanity check */
+	assert ( header->type == cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) );
+
+	/* Locate page set */
+	pages = vmbus_xfer_pages ( vmdev, page_header->pageset );
+	if ( ! pages ) {
+		rc = -ENOENT;
+		goto err_pages;
+	}
+
+	/* Allocate and populate I/O buffers */
+	range_count = le32_to_cpu ( page_header->range_count );
+	for ( i = 0 ; i < range_count ; i++ ) {
+
+		/* Parse header */
+		len = le32_to_cpu ( page_header->range[i].len );
+		offset = le32_to_cpu ( page_header->range[i].offset );
+
+		/* Allocate I/O buffer */
+		iobuf = alloc_iob ( len );
+		if ( ! iobuf ) {
+			DBGC ( vmdev, "VMBUS %s could not allocate %zd-byte "
+			       "I/O buffer\n", vmdev->dev.name, len );
+			rc = -ENOMEM;
+			goto err_alloc;
+		}
+
+		/* Add I/O buffer to list */
+		list_add ( &iobuf->list, list );
+
+		/* Populate I/O buffer */
+		if ( ( rc = pages->op->copy ( pages, iob_put ( iobuf, len ),
+					      offset, len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not populate I/O buffer "
+			       "range [%zd,%zd): %s\n",
+			       vmdev->dev.name, offset, len, strerror ( rc ) );
+			goto err_copy;
+		}
+	}
+
+	return 0;
+
+ err_copy:
+ err_alloc:
+	list_for_each_entry_safe ( iobuf, tmp, list, list ) {
+		list_del ( &iobuf->list );
+		free_iob ( iobuf );
+	}
+ err_pages:
+	return rc;
+}
+
+/**
+ * Poll ring buffer
+ *
+ * @v vmdev		VMBus device
+ * @ret rc		Return status code
+ */
+int vmbus_poll ( struct vmbus_device *vmdev ) {
+	struct vmbus_packet_header *header = vmdev->packet;
+	struct list_head list;
+	void *data;
+	size_t header_len;
+	size_t len;
+	size_t footer_len;
+	size_t ring_len;
+	size_t cons;
+	size_t old_cons;
+	uint64_t xid;
+	int rc;
+
+	/* Sanity checks */
+	assert ( vmdev->packet != NULL );
+	assert ( vmdev->in != NULL );
+
+	/* Return immediately if buffer is empty */
+	if ( ! vmbus_has_data ( vmdev ) )
+		return 0;
+	cons = le32_to_cpu ( vmdev->in->cons );
+	old_cons = cons;
+
+	/* Consume (start of) header */
+	cons = vmbus_consume ( vmdev, cons, header, sizeof ( *header ) );
+
+	/* Parse and sanity check header */
+	header_len = ( le16_to_cpu ( header->hdr_qlen ) * 8 );
+	if ( header_len < sizeof ( *header ) ) {
+		DBGC ( vmdev, "VMBUS %s received underlength header (%zd "
+		       "bytes)\n", vmdev->dev.name, header_len );
+		return -EINVAL;
+	}
+	len = ( ( le16_to_cpu ( header->qlen ) * 8 ) - header_len );
+	footer_len = sizeof ( struct vmbus_packet_footer );
+	ring_len = ( header_len + len + footer_len );
+	if ( ring_len > vmdev->mtu ) {
+		DBGC ( vmdev, "VMBUS %s received overlength packet (%zd "
+		       "bytes)\n", vmdev->dev.name, ring_len );
+		return -ERANGE;
+	}
+	xid = le64_to_cpu ( header->xid );
+
+	/* Consume remainder of packet */
+	cons = vmbus_consume ( vmdev, cons,
+			       ( ( ( void * ) header ) + sizeof ( *header ) ),
+			       ( ring_len - sizeof ( *header ) ) );
+	DBGC2 ( vmdev, "VMBUS %s received:\n", vmdev->dev.name );
+	DBGC2_HDA ( vmdev, old_cons, header, ring_len );
+	assert ( ( ( cons - old_cons ) & ( vmdev->in_len - 1 ) ) == ring_len );
+
+	/* Allocate I/O buffers, if applicable */
+	INIT_LIST_HEAD ( &list );
+	if ( header->type == cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) ) {
+		if ( ( rc = vmbus_xfer_page_iobufs ( vmdev, header,
+						     &list ) ) != 0 )
+			return rc;
+	}
+
+	/* Update producer index */
+	rmb();
+	vmdev->in->cons = cpu_to_le32 ( cons );
+
+	/* Handle packet */
+	data = ( ( ( void * ) header ) + header_len );
+	switch ( header->type ) {
+
+	case cpu_to_le16 ( VMBUS_DATA_INBAND ) :
+		if ( ( rc = vmdev->op->recv_control ( vmdev, xid, data,
+						      len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle control "
+			       "packet: %s\n",
+			       vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_DATA_XFER_PAGES ) :
+		if ( ( rc = vmdev->op->recv_data ( vmdev, xid, data, len,
+						   &list ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle data packet: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_COMPLETION ) :
+		if ( ( rc = vmdev->op->recv_completion ( vmdev, xid, data,
+							 len ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not handle completion: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	case cpu_to_le16 ( VMBUS_CANCELLATION ) :
+		if ( ( rc = vmdev->op->recv_cancellation ( vmdev, xid ) ) != 0){
+			DBGC ( vmdev, "VMBUS %s could not handle cancellation: "
+			       "%s\n", vmdev->dev.name, strerror ( rc ) );
+			return rc;
+		}
+		break;
+
+	default:
+		DBGC ( vmdev, "VMBUS %s unknown packet type %d\n",
+		       vmdev->dev.name, le16_to_cpu ( header->type ) );
+		return -ENOTSUP;
+	}
+
+	return 0;
+}
+
+/**
+ * Dump channel status (for debugging)
+ *
+ * @v vmdev		VMBus device
+ */
+void vmbus_dump_channel ( struct vmbus_device *vmdev ) {
+	size_t out_prod = le32_to_cpu ( vmdev->out->prod );
+	size_t out_cons = le32_to_cpu ( vmdev->out->cons );
+	size_t in_prod = le32_to_cpu ( vmdev->in->prod );
+	size_t in_cons = le32_to_cpu ( vmdev->in->cons );
+	size_t in_len;
+	size_t first;
+	size_t second;
+
+	/* Dump ring status */
+	DBGC ( vmdev, "VMBUS %s out %03zx:%03zx%s in %03zx:%03zx%s\n",
+	       vmdev->dev.name, out_prod, out_cons,
+	       ( vmdev->out->intr_mask ? "(m)" : "" ), in_prod, in_cons,
+	       ( vmdev->in->intr_mask ? "(m)" : "" ) );
+
+	/* Dump inbound ring contents, if any */
+	if ( in_prod != in_cons ) {
+		in_len = ( ( in_prod - in_cons ) &
+			   ( vmdev->in_len - 1 ) );
+		first = ( vmdev->in_len - in_cons );
+		if ( first > in_len )
+			first = in_len;
+		second = ( in_len - first );
+		DBGC_HDA ( vmdev, in_cons, &vmdev->in->data[in_cons], first );
+		DBGC_HDA ( vmdev, 0, &vmdev->in->data[0], second );
+	}
+}
+
+/**
+ * Find driver for VMBus device
+ *
+ * @v vmdev		VMBus device
+ * @ret driver		Driver, or NULL
+ */
+static struct vmbus_driver * vmbus_find_driver ( const union uuid *type ) {
+	struct vmbus_driver *vmdrv;
+
+	for_each_table_entry ( vmdrv, VMBUS_DRIVERS ) {
+		if ( memcmp ( &vmdrv->type, type, sizeof ( *type ) ) == 0 )
+			return vmdrv;
+	}
+	return NULL;
+}
+
+/**
+ * Probe channels
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ * @ret rc		Return status code
+ */
+static int vmbus_probe_channels ( struct hv_hypervisor *hv,
+				  struct device *parent ) {
+	struct vmbus *vmbus = hv->vmbus;
+	const struct vmbus_message_header *header = &vmbus->message->header;
+	const struct vmbus_offer_channel *offer = &vmbus->message->offer;
+	const union uuid *type;
+	struct vmbus_driver *driver;
+	struct vmbus_device *vmdev;
+	struct vmbus_device *tmp;
+	unsigned int channel;
+	int rc;
+
+	/* Post message */
+	if ( ( rc = vmbus_post_empty_message ( hv, VMBUS_REQUEST_OFFERS ) ) !=0)
+		goto err_post_message;
+
+	/* Collect responses */
+	while ( 1 ) {
+
+		/* Wait for response */
+		if ( ( rc = vmbus_wait_for_message ( hv ) ) != 0 )
+			goto err_wait_for_message;
+
+		/* Handle response */
+		if ( header->type == cpu_to_le32 ( VMBUS_OFFER_CHANNEL ) ) {
+
+			/* Parse offer */
+			type = &offer->type;
+			channel = le32_to_cpu ( offer->channel );
+			DBGC2 ( vmbus, "VMBUS %p offer %d type %s",
+				vmbus, channel, uuid_ntoa ( type ) );
+			if ( offer->monitored )
+				DBGC2 ( vmbus, " monitor %d", offer->monitor );
+			DBGC2 ( vmbus, "\n" );
+
+			/* Look for a driver */
+			driver = vmbus_find_driver ( type );
+			if ( ! driver ) {
+				DBGC2 ( vmbus, "VMBUS %p has no driver for "
+					"type %s\n", vmbus, uuid_ntoa ( type ));
+				/* Not a fatal error */
+				continue;
+			}
+
+			/* Allocate and initialise device */
+			vmdev = zalloc ( sizeof ( *vmdev ) );
+			if ( ! vmdev ) {
+				rc = -ENOMEM;
+				goto err_alloc_vmdev;
+			}
+			snprintf ( vmdev->dev.name, sizeof ( vmdev->dev.name ),
+				   "vmbus:%02x", channel );
+			vmdev->dev.desc.bus_type = BUS_TYPE_HV;
+			INIT_LIST_HEAD ( &vmdev->dev.children );
+			list_add_tail ( &vmdev->dev.siblings,
+					&parent->children );
+			vmdev->dev.parent = parent;
+			vmdev->hv = hv;
+			vmdev->channel = channel;
+			vmdev->monitor = offer->monitor;
+			vmdev->signal = ( offer->monitored ?
+					  vmbus_signal_monitor :
+					  vmbus_signal_event );
+			INIT_LIST_HEAD ( &vmdev->pages );
+			vmdev->driver = driver;
+			vmdev->dev.driver_name = driver->name;
+			DBGC ( vmdev, "VMBUS %s has driver \"%s\"\n",
+			       vmdev->dev.name, vmdev->driver->name );
+
+		} else if ( header->type ==
+			    cpu_to_le32 ( VMBUS_ALL_OFFERS_DELIVERED ) ) {
+
+			break;
+
+		} else {
+			DBGC ( vmbus, "VMBUS %p unexpected offer response type "
+			       "%d\n", vmbus, le32_to_cpu ( header->type ) );
+			rc = -EPROTO;
+			goto err_unexpected_offer;
+		}
+	}
+
+	/* Probe all devices.  We do this only after completing
+	 * enumeration since devices will need to send and receive
+	 * VMBus messages.
+	 */
+	list_for_each_entry ( vmdev, &parent->children, dev.siblings ) {
+		if ( ( rc = vmdev->driver->probe ( vmdev ) ) != 0 ) {
+			DBGC ( vmdev, "VMBUS %s could not probe: %s\n",
+			       vmdev->dev.name, strerror ( rc ) );
+			goto err_probe;
+		}
+	}
+
+	return 0;
+
+ err_probe:
+	/* Remove driver from each device that was already probed */
+	list_for_each_entry_continue_reverse ( vmdev, &parent->children,
+					       dev.siblings ) {
+		vmdev->driver->remove ( vmdev );
+	}
+ err_unexpected_offer:
+ err_alloc_vmdev:
+ err_wait_for_message:
+	/* Free any devices allocated (but potentially not yet probed) */
+	list_for_each_entry_safe ( vmdev, tmp, &parent->children,
+				   dev.siblings ) {
+		list_del ( &vmdev->dev.siblings );
+		free ( vmdev );
+	}
+ err_post_message:
+	return rc;
+}
+
+/**
+ * Remove channels
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ */
+static void vmbus_remove_channels ( struct hv_hypervisor *hv __unused,
+				    struct device *parent ) {
+	struct vmbus_device *vmdev;
+	struct vmbus_device *tmp;
+
+	/* Remove devices */
+	list_for_each_entry_safe ( vmdev, tmp, &parent->children,
+				   dev.siblings ) {
+		vmdev->driver->remove ( vmdev );
+		assert ( list_empty ( &vmdev->dev.children ) );
+		assert ( vmdev->out == NULL );
+		assert ( vmdev->in == NULL );
+		assert ( vmdev->packet == NULL );
+		assert ( list_empty ( &vmdev->pages ) );
+		list_del ( &vmdev->dev.siblings );
+		free ( vmdev );
+	}
+}
+
+/**
+ * Probe Hyper-V virtual machine bus
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ * @ret rc		Return status code
+ */
+int vmbus_probe ( struct hv_hypervisor *hv, struct device *parent ) {
+	struct vmbus *vmbus;
+	int rc;
+
+	/* Allocate and initialise structure */
+	vmbus = zalloc ( sizeof ( *vmbus ) );
+	if ( ! vmbus ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	hv->vmbus = vmbus;
+
+	/* Initialise message buffer pointer
+	 *
+	 * We use a pointer to the fixed-size Hyper-V received message
+	 * buffer.  This allows us to access fields within received
+	 * messages without first checking the message size: any
+	 * fields beyond the end of the message will read as zero.
+	 */
+	vmbus->message = ( ( void * ) hv->message->received.data );
+	assert ( sizeof ( *vmbus->message ) <=
+		 sizeof ( hv->message->received.data ) );
+
+	/* Allocate interrupt and monitor pages */
+	if ( ( rc = hv_alloc_pages ( hv, &vmbus->intr, &vmbus->monitor_in,
+				     &vmbus->monitor_out, NULL ) ) != 0 )
+		goto err_alloc_pages;
+
+	/* Enable message interrupt */
+	hv_enable_sint ( hv, VMBUS_MESSAGE_SINT );
+
+	/* Negotiate protocol version */
+	if ( ( rc = vmbus_negotiate_version ( hv ) ) != 0 )
+		goto err_negotiate_version;
+
+	/* Enumerate channels */
+	if ( ( rc = vmbus_probe_channels ( hv, parent ) ) != 0 )
+		goto err_probe_channels;
+
+	return 0;
+
+	vmbus_remove_channels ( hv, parent );
+ err_probe_channels:
+	vmbus_unload ( hv );
+ err_negotiate_version:
+	hv_disable_sint ( hv, VMBUS_MESSAGE_SINT );
+	hv_free_pages ( hv, vmbus->intr, vmbus->monitor_in, vmbus->monitor_out,
+			NULL );
+ err_alloc_pages:
+	free ( vmbus );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Remove Hyper-V virtual machine bus
+ *
+ * @v hv		Hyper-V hypervisor
+ * @v parent		Parent device
+ */
+void vmbus_remove ( struct hv_hypervisor *hv, struct device *parent ) {
+	struct vmbus *vmbus = hv->vmbus;
+
+	vmbus_remove_channels ( hv, parent );
+	vmbus_unload ( hv );
+	hv_disable_sint ( hv, VMBUS_MESSAGE_SINT );
+	hv_free_pages ( hv, vmbus->intr, vmbus->monitor_in, vmbus->monitor_out,
+			NULL );
+	free ( vmbus );
+}
diff --git a/qemu/roms/ipxe/src/interface/linux/linux_entropy.c b/qemu/roms/ipxe/src/interface/linux/linux_entropy.c
index 4671a48da..0f8e45d36 100644
--- a/qemu/roms/ipxe/src/interface/linux/linux_entropy.c
+++ b/qemu/roms/ipxe/src/interface/linux/linux_entropy.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/interface/linux/linux_pci.c b/qemu/roms/ipxe/src/interface/linux/linux_pci.c
index cbd825c18..0c140cb89 100644
--- a/qemu/roms/ipxe/src/interface/linux/linux_pci.c
+++ b/qemu/roms/ipxe/src/interface/linux/linux_pci.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/interface/linux/linux_time.c b/qemu/roms/ipxe/src/interface/linux/linux_time.c
index e3cbafec6..9e99fe9cd 100644
--- a/qemu/roms/ipxe/src/interface/linux/linux_time.c
+++ b/qemu/roms/ipxe/src/interface/linux/linux_time.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/interface/linux/linux_uaccess.c b/qemu/roms/ipxe/src/interface/linux/linux_uaccess.c
index 5ab0b6b65..ea2d8057c 100644
--- a/qemu/roms/ipxe/src/interface/linux/linux_uaccess.c
+++ b/qemu/roms/ipxe/src/interface/linux/linux_uaccess.c
@@ -27,7 +27,6 @@ FILE_LICENCE(GPL2_OR_LATER);
  *
  */
 
-PROVIDE_UACCESS_INLINE(linux, phys_to_user);
 PROVIDE_UACCESS_INLINE(linux, user_to_phys);
 PROVIDE_UACCESS_INLINE(linux, virt_to_user);
 PROVIDE_UACCESS_INLINE(linux, user_to_virt);
diff --git a/qemu/roms/ipxe/src/interface/smbios/smbios.c b/qemu/roms/ipxe/src/interface/smbios/smbios.c
index 856943428..1dcf819c2 100644
--- a/qemu/roms/ipxe/src/interface/smbios/smbios.c
+++ b/qemu/roms/ipxe/src/interface/smbios/smbios.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/interface/smbios/smbios_settings.c b/qemu/roms/ipxe/src/interface/smbios/smbios_settings.c
index 83e4320e9..5eadfa081 100644
--- a/qemu/roms/ipxe/src/interface/smbios/smbios_settings.c
+++ b/qemu/roms/ipxe/src/interface/smbios/smbios_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/interface/xen/xenbus.c b/qemu/roms/ipxe/src/interface/xen/xenbus.c
index ffc8aba3e..c328af443 100644
--- a/qemu/roms/ipxe/src/interface/xen/xenbus.c
+++ b/qemu/roms/ipxe/src/interface/xen/xenbus.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/interface/xen/xengrant.c b/qemu/roms/ipxe/src/interface/xen/xengrant.c
index be12b23dc..269cd5836 100644
--- a/qemu/roms/ipxe/src/interface/xen/xengrant.c
+++ b/qemu/roms/ipxe/src/interface/xen/xengrant.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <strings.h>
diff --git a/qemu/roms/ipxe/src/interface/xen/xenstore.c b/qemu/roms/ipxe/src/interface/xen/xenstore.c
index b96982927..23424a926 100644
--- a/qemu/roms/ipxe/src/interface/xen/xenstore.c
+++ b/qemu/roms/ipxe/src/interface/xen/xenstore.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdarg.h>
@@ -238,6 +242,10 @@ static int xenstore_response ( struct xen_hypervisor *xen, uint32_t req_id,
 	char *string;
 	int rc;
 
+	/* Wait for response to become available */
+	while ( ! xenevent_pending ( xen, xen->store.port ) )
+		cpu_nap();
+
 	/* Receive message header */
 	xenstore_recv ( xen, &msg, sizeof ( msg ) );
 	*len = msg.len;
diff --git a/qemu/roms/ipxe/src/net/80211/net80211.c b/qemu/roms/ipxe/src/net/80211/net80211.c
index 434944523..d4970ad5c 100644
--- a/qemu/roms/ipxe/src/net/80211/net80211.c
+++ b/qemu/roms/ipxe/src/net/80211/net80211.c
@@ -805,6 +805,10 @@ int net80211_register ( struct net80211_device *dev,
 		 NET80211_MAX_CHANNELS * sizeof ( dev->channels[0] ) );
 	dev->channel = 0;
 
+	/* Mark device as not supporting interrupts, if applicable */
+	if ( ! ops->irq )
+		dev->netdev->state |= NETDEV_IRQ_UNSUPPORTED;
+
 	list_add_tail ( &dev->list, &net80211_devices );
 	return register_netdev ( dev->netdev );
 }
@@ -2826,3 +2830,9 @@ struct errortab common_wireless_errors[] __errortab = {
 	__einfo_errortab ( EINFO_ECONNREFUSED_ASSOC_DENIED ),
 	__einfo_errortab ( EINFO_ECONNREFUSED_AUTH_ALGO_UNSUPP ),
 };
+
+/* Drag in objects via net80211_ll_protocol */
+REQUIRING_SYMBOL ( net80211_ll_protocol );
+
+/* Drag in 802.11 configuration */
+REQUIRE_OBJECT ( config_net80211 );
diff --git a/qemu/roms/ipxe/src/net/80211/wpa.c b/qemu/roms/ipxe/src/net/80211/wpa.c
index e2c4945f9..77f66d825 100644
--- a/qemu/roms/ipxe/src/net/80211/wpa.c
+++ b/qemu/roms/ipxe/src/net/80211/wpa.c
@@ -912,4 +912,5 @@ struct eapol_handler eapol_key_handler __eapol_handler = {
 };
 
 /* WPA always needs EAPOL in order to be useful */
+REQUIRING_SYMBOL ( eapol_key_handler );
 REQUIRE_OBJECT ( eapol );
diff --git a/qemu/roms/ipxe/src/net/80211/wpa_ccmp.c b/qemu/roms/ipxe/src/net/80211/wpa_ccmp.c
index f98ebea26..a073c6a3c 100644
--- a/qemu/roms/ipxe/src/net/80211/wpa_ccmp.c
+++ b/qemu/roms/ipxe/src/net/80211/wpa_ccmp.c
@@ -480,7 +480,7 @@ static void ccmp_kie_mic ( const void *kck, const void *msg, size_t len,
 {
 	u8 sha1_ctx[SHA1_CTX_SIZE];
 	u8 kckb[16];
-	u8 hash[SHA1_SIZE];
+	u8 hash[SHA1_DIGEST_SIZE];
 	size_t kck_len = 16;
 
 	memcpy ( kckb, kck, kck_len );
diff --git a/qemu/roms/ipxe/src/net/80211/wpa_tkip.c b/qemu/roms/ipxe/src/net/80211/wpa_tkip.c
index fa3e0763b..3b1934b59 100644
--- a/qemu/roms/ipxe/src/net/80211/wpa_tkip.c
+++ b/qemu/roms/ipxe/src/net/80211/wpa_tkip.c
@@ -136,7 +136,7 @@ static const u16 Sbox[256] = {
  */
 static inline u16 S ( u16 v )
 {
-	return Sbox[v & 0xFF] ^ swap16 ( Sbox[v >> 8] );
+	return Sbox[v & 0xFF] ^ bswap_16 ( Sbox[v >> 8] );
 }
 
 /**
diff --git a/qemu/roms/ipxe/src/net/aoe.c b/qemu/roms/ipxe/src/net/aoe.c
index a6d7b3e7b..2da8655b4 100644
--- a/qemu/roms/ipxe/src/net/aoe.c
+++ b/qemu/roms/ipxe/src/net/aoe.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/net/arp.c b/qemu/roms/ipxe/src/net/arp.c
index 261e681e1..1e27c44e7 100644
--- a/qemu/roms/ipxe/src/net/arp.c
+++ b/qemu/roms/ipxe/src/net/arp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -52,9 +56,9 @@ struct net_protocol arp_protocol __net_protocol;
  * @v net_source	Source network-layer address
  * @ret rc		Return status code
  */
-static int arp_tx_request ( struct net_device *netdev,
-			    struct net_protocol *net_protocol,
-			    const void *net_dest, const void *net_source ) {
+int arp_tx_request ( struct net_device *netdev,
+		     struct net_protocol *net_protocol,
+		     const void *net_dest, const void *net_source ) {
 	struct ll_protocol *ll_protocol = netdev->ll_protocol;
 	struct io_buffer *iobuf;
 	struct arphdr *arphdr;
diff --git a/qemu/roms/ipxe/src/net/dhcpopts.c b/qemu/roms/ipxe/src/net/dhcpopts.c
index 8cd19cf80..cdb632b46 100644
--- a/qemu/roms/ipxe/src/net/dhcpopts.c
+++ b/qemu/roms/ipxe/src/net/dhcpopts.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/dhcppkt.c b/qemu/roms/ipxe/src/net/dhcppkt.c
index a9a6d3a94..4e64f85e4 100644
--- a/qemu/roms/ipxe/src/net/dhcppkt.c
+++ b/qemu/roms/ipxe/src/net/dhcppkt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/eth_slow.c b/qemu/roms/ipxe/src/net/eth_slow.c
index db54b55a4..049c26cb3 100644
--- a/qemu/roms/ipxe/src/net/eth_slow.c
+++ b/qemu/roms/ipxe/src/net/eth_slow.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/net/ethernet.c b/qemu/roms/ipxe/src/net/ethernet.c
index 03978c2a8..6ddf05344 100644
--- a/qemu/roms/ipxe/src/net/ethernet.c
+++ b/qemu/roms/ipxe/src/net/ethernet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -43,6 +47,24 @@ FILE_LICENCE ( GPL2_OR_LATER );
 uint8_t eth_broadcast[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 /**
+ * Check if Ethernet packet has an 802.3 LLC header
+ *
+ * @v ethhdr		Ethernet header
+ * @ret is_llc		Packet has 802.3 LLC header
+ */
+static inline int eth_is_llc_packet ( struct ethhdr *ethhdr ) {
+	uint8_t len_msb;
+
+	/* Check if the protocol field contains a value short enough
+	 * to be a frame length.  The slightly convoluted form of the
+	 * comparison is designed to reduce to a single x86
+	 * instruction.
+	 */
+	len_msb = *( ( uint8_t * ) &ethhdr->h_protocol );
+	return ( len_msb < 0x06 );
+}
+
+/**
  * Add Ethernet link-layer header
  *
  * @v netdev		Network device
@@ -80,9 +102,14 @@ int eth_pull ( struct net_device *netdev __unused, struct io_buffer *iobuf,
 	       const void **ll_dest, const void **ll_source,
 	       uint16_t *net_proto, unsigned int *flags ) {
 	struct ethhdr *ethhdr = iobuf->data;
+	uint16_t *llc_proto;
 
-	/* Sanity check */
-	if ( iob_len ( iobuf ) < sizeof ( *ethhdr ) ) {
+	/* Sanity check.  While in theory we could receive a one-byte
+	 * packet, this will never happen in practice and performing
+	 * the combined length check here avoids the need for an
+	 * additional comparison if we detect an LLC frame.
+	 */
+	if ( iob_len ( iobuf ) < ( sizeof ( *ethhdr ) + sizeof ( *llc_proto ))){
 		DBG ( "Ethernet packet too short (%zd bytes)\n",
 		      iob_len ( iobuf ) );
 		return -EINVAL;
@@ -100,6 +127,17 @@ int eth_pull ( struct net_device *netdev __unused, struct io_buffer *iobuf,
 		   ( is_broadcast_ether_addr ( ethhdr->h_dest ) ?
 		     LL_BROADCAST : 0 ) );
 
+	/* If this is an LLC frame (with a length in place of the
+	 * protocol field), then use the next two bytes (which happen
+	 * to be the LLC DSAP and SSAP) as the protocol.  This allows
+	 * for minimal-overhead support for receiving (rare) LLC
+	 * frames, without requiring a full LLC protocol layer.
+	 */
+	if ( eth_is_llc_packet ( ethhdr ) ) {
+		llc_proto = ( &ethhdr->h_protocol + 1 );
+		*net_proto = *llc_proto;
+	}
+
 	return 0;
 }
 
@@ -235,5 +273,11 @@ struct net_device * alloc_etherdev ( size_t priv_size ) {
 	return netdev;
 }
 
+/* Drag in objects via ethernet_protocol */
+REQUIRING_SYMBOL ( ethernet_protocol );
+
+/* Drag in Ethernet configuration */
+REQUIRE_OBJECT ( config_ethernet );
+
 /* Drag in Ethernet slow protocols */
 REQUIRE_OBJECT ( eth_slow );
diff --git a/qemu/roms/ipxe/src/net/fakedhcp.c b/qemu/roms/ipxe/src/net/fakedhcp.c
index 3dec88b11..b6c456a59 100644
--- a/qemu/roms/ipxe/src/net/fakedhcp.c
+++ b/qemu/roms/ipxe/src/net/fakedhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/fc.c b/qemu/roms/ipxe/src/net/fc.c
index 58008995c..2e8070272 100644
--- a/qemu/roms/ipxe/src/net/fc.c
+++ b/qemu/roms/ipxe/src/net/fc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
@@ -1935,3 +1939,9 @@ struct fc_ulp * fc_ulp_get_port_id_type ( struct fc_port *port,
  err_peer_get_wwn:
 	return NULL;
 }
+
+/* Drag in objects via fc_ports */
+REQUIRING_SYMBOL ( fc_ports );
+
+/* Drag in Fibre Channel configuration */
+REQUIRE_OBJECT ( config_fc );
diff --git a/qemu/roms/ipxe/src/net/fcels.c b/qemu/roms/ipxe/src/net/fcels.c
index 1cfe90727..5fc27cef4 100644
--- a/qemu/roms/ipxe/src/net/fcels.c
+++ b/qemu/roms/ipxe/src/net/fcels.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/fcns.c b/qemu/roms/ipxe/src/net/fcns.c
index 3ca4ad557..be4dfea24 100644
--- a/qemu/roms/ipxe/src/net/fcns.c
+++ b/qemu/roms/ipxe/src/net/fcns.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/fcoe.c b/qemu/roms/ipxe/src/net/fcoe.c
index e9e404ec3..c3258f15e 100644
--- a/qemu/roms/ipxe/src/net/fcoe.c
+++ b/qemu/roms/ipxe/src/net/fcoe.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/fcp.c b/qemu/roms/ipxe/src/net/fcp.c
index 9c36a4c72..930bf7dd4 100644
--- a/qemu/roms/ipxe/src/net/fcp.c
+++ b/qemu/roms/ipxe/src/net/fcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/qemu/roms/ipxe/src/net/fragment.c b/qemu/roms/ipxe/src/net/fragment.c
index 410915b3b..781b9bc60 100644
--- a/qemu/roms/ipxe/src/net/fragment.c
+++ b/qemu/roms/ipxe/src/net/fragment.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/icmp.c b/qemu/roms/ipxe/src/net/icmp.c
index 1bbf8bd30..5371277e4 100644
--- a/qemu/roms/ipxe/src/net/icmp.c
+++ b/qemu/roms/ipxe/src/net/icmp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <byteswap.h>
diff --git a/qemu/roms/ipxe/src/net/icmpv4.c b/qemu/roms/ipxe/src/net/icmpv4.c
index 996ba1490..0858ff37f 100644
--- a/qemu/roms/ipxe/src/net/icmpv4.c
+++ b/qemu/roms/ipxe/src/net/icmpv4.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/net/icmpv6.c b/qemu/roms/ipxe/src/net/icmpv6.c
index 479800e7d..8555aaf0b 100644
--- a/qemu/roms/ipxe/src/net/icmpv6.c
+++ b/qemu/roms/ipxe/src/net/icmpv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
@@ -34,6 +38,65 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
+/* Disambiguate the various error causes */
+#define EHOSTUNREACH_ROUTE						\
+	__einfo_error ( EINFO_EHOSTUNREACH_ROUTE )
+#define EINFO_EHOSTUNREACH_ROUTE					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 0,			\
+			  "No route to destination" )
+#define EHOSTUNREACH_PROHIBITED						\
+	__einfo_error ( EINFO_EHOSTUNREACH_PROHIBITED )
+#define EINFO_EHOSTUNREACH_PROHIBITED					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 1,			\
+			  "Communication administratively prohibited" )
+#define EHOSTUNREACH_ADDRESS						\
+	__einfo_error ( EINFO_EHOSTUNREACH_ADDRESS )
+#define EINFO_EHOSTUNREACH_ADDRESS					\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 3,			\
+			  "Address unreachable" )
+#define EHOSTUNREACH_PORT						\
+	__einfo_error ( EINFO_EHOSTUNREACH_PORT )
+#define EINFO_EHOSTUNREACH_PORT						\
+	__einfo_uniqify ( EINFO_EHOSTUNREACH, 4,			\
+			  "Port unreachable" )
+#define EHOSTUNREACH_CODE( code )					\
+	EUNIQ ( EINFO_EHOSTUNREACH, ( (code) & 0x1f ),			\
+		EHOSTUNREACH_ROUTE, EHOSTUNREACH_PROHIBITED,		\
+		EHOSTUNREACH_ADDRESS, EHOSTUNREACH_PORT )
+
+#define ETIMEDOUT_HOP							\
+	__einfo_error ( EINFO_ETIMEDOUT_HOP )
+#define EINFO_ETIMEDOUT_HOP						\
+	__einfo_uniqify ( EINFO_ETIMEDOUT, 0,				\
+			  "Hop limit exceeded in transit" )
+#define ETIMEDOUT_REASSEMBLY						\
+	__einfo_error ( EINFO_ETIMEDOUT_REASSEMBLY )
+#define EINFO_ETIMEDOUT_REASSEMBLY					\
+	__einfo_uniqify ( EINFO_ETIMEDOUT, 1,				\
+			  "Fragment reassembly time exceeded" )
+#define ETIMEDOUT_CODE( code )						\
+	EUNIQ ( EINFO_ETIMEDOUT, ( (code) & 0x1f ),			\
+		ETIMEDOUT_HOP, ETIMEDOUT_REASSEMBLY )
+
+#define EPROTO_BAD_HEADER						\
+	__einfo_error ( EINFO_EPROTO_BAD_HEADER )
+#define EINFO_EPROTO_BAD_HEADER						\
+	__einfo_uniqify ( EINFO_EPROTO, 0,				\
+			  "Erroneous header field" )
+#define EPROTO_NEXT_HEADER						\
+	__einfo_error ( EINFO_EPROTO_NEXT_HEADER )
+#define EINFO_EPROTO_NEXT_HEADER					\
+	__einfo_uniqify ( EINFO_EPROTO, 1,				\
+			  "Unrecognised next header type" )
+#define EPROTO_OPTION							\
+	__einfo_error ( EINFO_EPROTO_OPTION )
+#define EINFO_EPROTO_OPTION						\
+	__einfo_uniqify ( EINFO_EPROTO, 2,				\
+			  "Unrecognised IPv6 option" )
+#define EPROTO_CODE( code )						\
+	EUNIQ ( EINFO_EPROTO, ( (code) & 0x1f ),			\
+		EPROTO_BAD_HEADER, EPROTO_NEXT_HEADER, EPROTO_OPTION )
+
 struct icmp_echo_protocol icmpv6_echo_protocol __icmp_echo_protocol;
 
 /**
@@ -144,8 +207,25 @@ static int icmpv6_rx ( struct io_buffer *iobuf, struct net_device *netdev,
 	/* Identify handler */
 	handler = icmpv6_handler ( icmp->type );
 	if ( ! handler ) {
-		DBGC ( netdev, "ICMPv6 unrecognised type %d\n", icmp->type );
-		rc = -ENOTSUP;
+		switch ( icmp->type ) {
+		case ICMPV6_DESTINATION_UNREACHABLE:
+			rc = -EHOSTUNREACH_CODE ( icmp->code );
+			break;
+		case ICMPV6_PACKET_TOO_BIG:
+			rc = -ERANGE;
+			break;
+		case ICMPV6_TIME_EXCEEDED:
+			rc = -ETIMEDOUT_CODE ( icmp->code );
+			break;
+		case ICMPV6_PARAMETER_PROBLEM:
+			rc = -EPROTO_CODE ( icmp->code );
+			break;
+		default:
+			DBGC ( netdev, "ICMPv6 unrecognised type %d code %d\n",
+			       icmp->type, icmp->code );
+			rc = -ENOTSUP;
+			break;
+		};
 		goto done;
 	}
 
diff --git a/qemu/roms/ipxe/src/net/infiniband.c b/qemu/roms/ipxe/src/net/infiniband.c
index 12d1d83ce..2e3d76d54 100644
--- a/qemu/roms/ipxe/src/net/infiniband.c
+++ b/qemu/roms/ipxe/src/net/infiniband.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -714,6 +718,9 @@ int ib_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	struct ib_multicast_gid *mgid;
 	int rc;
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Add to software multicast GID list */
 	mgid = zalloc ( sizeof ( *mgid ) );
 	if ( ! mgid ) {
@@ -747,6 +754,9 @@ void ib_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		       union ib_gid *gid ) {
 	struct ib_multicast_gid *mgid;
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Remove from hardware multicast GID list */
 	ibdev->op->mcast_detach ( ibdev, qp, gid );
 
@@ -995,5 +1005,11 @@ struct ib_device * last_opened_ibdev ( void ) {
 	return ibdev;
 }
 
+/* Drag in objects via register_ibdev() */
+REQUIRING_SYMBOL ( register_ibdev );
+
+/* Drag in Infiniband configuration */
+REQUIRE_OBJECT ( config_infiniband );
+
 /* Drag in IPoIB */
 REQUIRE_OBJECT ( ipoib );
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_cm.c b/qemu/roms/ipxe/src/net/infiniband/ib_cm.c
index 797639bc8..85982f09d 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_cm.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_cm.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_mcast.c b/qemu/roms/ipxe/src/net/infiniband/ib_mcast.c
index 0a5e72a37..fc4ff7f0a 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_mcast.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_mcast.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -146,6 +150,9 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	DBGC ( ibdev, "IBDEV %p QPN %lx joining " IB_GID_FMT "\n",
 	       ibdev, qp->qpn, IB_GID_ARGS ( gid ) );
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Initialise structure */
 	membership->qp = qp;
 	memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
@@ -195,6 +202,9 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	DBGC ( ibdev, "IBDEV %p QPN %lx leaving " IB_GID_FMT "\n",
 	       ibdev, qp->qpn, IB_GID_ARGS ( gid ) );
 
+	/* Sanity check */
+	assert ( qp != NULL );
+
 	/* Detach from multicast GID */
 	ib_mcast_detach ( ibdev, qp, &membership->gid );
 
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_mi.c b/qemu/roms/ipxe/src/net/infiniband/ib_mi.c
index ef6d539f1..b43212974 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_mi.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_mi.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_packet.c b/qemu/roms/ipxe/src/net/infiniband/ib_packet.c
index 6c850e39b..d3a22d309 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_packet.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_packet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_pathrec.c b/qemu/roms/ipxe/src/net/infiniband/ib_pathrec.c
index 1b95cbfa8..f9cbab87f 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_pathrec.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_pathrec.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_sma.c b/qemu/roms/ipxe/src/net/infiniband/ib_sma.c
index 86553732a..a05d7c924 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_sma.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_sma.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_smc.c b/qemu/roms/ipxe/src/net/infiniband/ib_smc.c
index 4d947d568..c1741b26c 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_smc.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_smc.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/infiniband/ib_srp.c b/qemu/roms/ipxe/src/net/infiniband/ib_srp.c
index 7b2b2b4ea..3700184c0 100644
--- a/qemu/roms/ipxe/src/net/infiniband/ib_srp.c
+++ b/qemu/roms/ipxe/src/net/infiniband/ib_srp.c
@@ -291,7 +291,7 @@ static int ib_srp_parse_byte_string ( const char *rp_comp, uint8_t *bytes,
 		return -EINVAL_BYTE_STRING_LEN;
 
 	/* Parse byte string */
-	decoded_size = base16_decode ( rp_comp, bytes );
+	decoded_size = base16_decode ( rp_comp, bytes, size );
 	if ( decoded_size < 0 )
 		return decoded_size;
 
diff --git a/qemu/roms/ipxe/src/net/iobpad.c b/qemu/roms/ipxe/src/net/iobpad.c
index 9cc8328e9..936b4bde4 100644
--- a/qemu/roms/ipxe/src/net/iobpad.c
+++ b/qemu/roms/ipxe/src/net/iobpad.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
diff --git a/qemu/roms/ipxe/src/net/ipv4.c b/qemu/roms/ipxe/src/net/ipv4.c
index 9c5cf2eb4..a54784049 100644
--- a/qemu/roms/ipxe/src/net/ipv4.c
+++ b/qemu/roms/ipxe/src/net/ipv4.c
@@ -1,3 +1,27 @@
+/*
+ * Copyright (C) 2006 Michael Brown <mbrown@fensystems.co.uk>.
+ * Copyright (C) 2006 Nikhil Chandru Rao
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
 #include <string.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -24,7 +48,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Unique IP datagram identification number (high byte) */
 static uint8_t next_ident_high = 0;
@@ -115,6 +139,7 @@ static void del_ipv4_miniroute ( struct ipv4_miniroute *miniroute ) {
 /**
  * Perform IPv4 routing
  *
+ * @v scope_id		Destination address scope ID
  * @v dest		Final destination address
  * @ret dest		Next hop destination address
  * @ret miniroute	Routing table entry to use, or NULL if no route
@@ -122,22 +147,42 @@ static void del_ipv4_miniroute ( struct ipv4_miniroute *miniroute ) {
  * If the route requires use of a gateway, the next hop destination
  * address will be overwritten with the gateway address.
  */
-static struct ipv4_miniroute * ipv4_route ( struct in_addr *dest ) {
+static struct ipv4_miniroute * ipv4_route ( unsigned int scope_id,
+					    struct in_addr *dest ) {
 	struct ipv4_miniroute *miniroute;
-	int local;
-	int has_gw;
 
 	/* Find first usable route in routing table */
 	list_for_each_entry ( miniroute, &ipv4_miniroutes, list ) {
+
+		/* Skip closed network devices */
 		if ( ! netdev_is_open ( miniroute->netdev ) )
 			continue;
-		local = ( ( ( dest->s_addr ^ miniroute->address.s_addr )
-			    & miniroute->netmask.s_addr ) == 0 );
-		has_gw = ( miniroute->gateway.s_addr );
-		if ( local || has_gw ) {
-			if ( ! local )
+
+		if ( IN_IS_MULTICAST ( dest->s_addr ) ) {
+
+			/* If destination is non-global, and the scope ID
+			 * matches this network device, then use this route.
+			 */
+			if ( miniroute->netdev->index == scope_id )
+				return miniroute;
+
+		} else {
+
+			/* If destination is an on-link global
+			 * address, then use this route.
+			 */
+			if ( ( ( dest->s_addr ^ miniroute->address.s_addr )
+			       & miniroute->netmask.s_addr ) == 0 )
+				return miniroute;
+
+			/* If destination is an off-link global
+			 * address, and we have a default gateway,
+			 * then use this route.
+			 */
+			if ( miniroute->gateway.s_addr ) {
 				*dest = miniroute->gateway;
-			return miniroute;
+				return miniroute;
+			}
 		}
 	}
 
@@ -156,7 +201,7 @@ static struct net_device * ipv4_netdev ( struct sockaddr_tcpip *st_dest ) {
 	struct ipv4_miniroute *miniroute;
 
 	/* Find routing table entry */
-	miniroute = ipv4_route ( &dest );
+	miniroute = ipv4_route ( sin_dest->sin_scope_id, &dest );
 	if ( ! miniroute )
 		return NULL;
 
@@ -290,8 +335,8 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 	if ( sin_src )
 		iphdr->src = sin_src->sin_addr;
 	if ( ( next_hop.s_addr != INADDR_BROADCAST ) &&
-	     ( ! IN_MULTICAST ( ntohl ( next_hop.s_addr ) ) ) &&
-	     ( ( miniroute = ipv4_route ( &next_hop ) ) != NULL ) ) {
+	     ( ( miniroute = ipv4_route ( sin_dest->sin_scope_id,
+					  &next_hop ) ) != NULL ) ) {
 		iphdr->src = miniroute->address;
 		netmask = miniroute->netmask;
 		netdev = miniroute->netdev;
@@ -329,7 +374,7 @@ static int ipv4_tx ( struct io_buffer *iobuf,
 		/* Broadcast address */
 		ipv4_stats.out_bcast_pkts++;
 		ll_dest = netdev->ll_broadcast;
-	} else if ( IN_MULTICAST ( ntohl ( next_hop.s_addr ) ) ) {
+	} else if ( IN_IS_MULTICAST ( next_hop.s_addr ) ) {
 		/* Multicast address */
 		ipv4_stats.out_mcast_pkts++;
 		if ( ( rc = netdev->ll_protocol->mc_hash ( AF_INET, &next_hop,
@@ -569,10 +614,42 @@ static int ipv4_arp_check ( struct net_device *netdev, const void *net_addr ) {
 }
 
 /**
+ * Parse IPv4 address
+ *
+ * @v string		IPv4 address string
+ * @ret in		IPv4 address to fill in
+ * @ret ok		IPv4 address is valid
+ *
+ * Note that this function returns nonzero iff the address is valid,
+ * to match the standard BSD API function of the same name.  Unlike
+ * most other iPXE functions, a zero therefore indicates failure.
+ */
+int inet_aton ( const char *string, struct in_addr *in ) {
+	const char *separator = "...";
+	uint8_t *byte = ( ( uint8_t * ) in );
+	char *endp;
+	unsigned long value;
+
+	while ( 1 ) {
+		value = strtoul ( string, &endp, 0 );
+		if ( string == endp )
+			return 0;
+		if ( value > 0xff )
+			return 0;
+		*(byte++) = value;
+		if ( *endp != *separator )
+			return 0;
+		if ( ! *(separator++) )
+			return 1;
+		string = ( endp + 1 );
+	}
+}
+
+/**
  * Convert IPv4 address to dotted-quad notation
  *
- * @v in	IP address
- * @ret string	IP address in dotted-quad notation
+ * @v in		IPv4 address
+ * @ret string		IPv4 address in dotted-quad notation
  */
 char * inet_ntoa ( struct in_addr in ) {
 	static char buf[16]; /* "xxx.xxx.xxx.xxx" */
@@ -583,10 +660,10 @@ char * inet_ntoa ( struct in_addr in ) {
 }
 
 /**
- * Transcribe IP address
+ * Transcribe IPv4 address
  *
- * @v net_addr	IP address
- * @ret string	IP address in dotted-quad notation
+ * @v net_addr		IPv4 address
+ * @ret string		IPv4 address in dotted-quad notation
  *
  */
 static const char * ipv4_ntoa ( const void *net_addr ) {
@@ -760,12 +837,12 @@ static int ipv4_create_routes ( void ) {
 		fetch_ipv4_setting ( settings, &netmask_setting, &netmask );
 		/* Calculate default netmask, if necessary */
 		if ( ! netmask.s_addr ) {
-			if ( IN_CLASSA ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSA_NET );
-			} else if ( IN_CLASSB ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSB_NET );
-			} else if ( IN_CLASSC ( ntohl ( address.s_addr ) ) ) {
-				netmask.s_addr = htonl ( IN_CLASSC_NET );
+			if ( IN_IS_CLASSA ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSA;
+			} else if ( IN_IS_CLASSB ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSB;
+			} else if ( IN_IS_CLASSC ( address.s_addr ) ) {
+				netmask.s_addr = INADDR_NET_CLASSC;
 			}
 		}
 		/* Get default gateway, if present */
@@ -785,5 +862,8 @@ struct settings_applicator ipv4_settings_applicator __settings_applicator = {
 	.apply = ipv4_create_routes,
 };
 
+/* Drag in objects via ipv4_protocol */
+REQUIRING_SYMBOL ( ipv4_protocol );
+
 /* Drag in ICMPv4 */
 REQUIRE_OBJECT ( icmpv4 );
diff --git a/qemu/roms/ipxe/src/net/ipv6.c b/qemu/roms/ipxe/src/net/ipv6.c
index 3c374168c..a75e72ddb 100644
--- a/qemu/roms/ipxe/src/net/ipv6.c
+++ b/qemu/roms/ipxe/src/net/ipv6.c
@@ -290,8 +290,7 @@ static struct ipv6_miniroute * ipv6_route ( unsigned int scope_id,
 		if ( ! ( miniroute->flags & IPV6_HAS_ADDRESS ) )
 			continue;
 
-		if ( IN6_IS_ADDR_LINKLOCAL ( *dest ) ||
-		     IN6_IS_ADDR_MULTICAST ( *dest ) ) {
+		if ( IN6_IS_ADDR_NONGLOBAL ( *dest ) ) {
 
 			/* If destination is non-global, and the scope ID
 			 * matches this network device, then use this route.
@@ -901,7 +900,7 @@ static const char * ipv6_sock_ntoa ( struct sockaddr *sa ) {
 	const char *netdev_name;
 
 	/* Identify network device, if applicable */
-	if ( IN6_IS_ADDR_LINKLOCAL ( in ) || IN6_IS_ADDR_MULTICAST ( in ) ) {
+	if ( IN6_IS_ADDR_NONGLOBAL ( in ) ) {
 		netdev = find_netdev_by_index ( sin6->sin6_scope_id );
 		netdev_name = ( netdev ? netdev->name : "UNKNOWN" );
 	} else {
@@ -956,14 +955,26 @@ static int ipv6_sock_aton ( const char *string, struct sockaddr *sa ) {
 	if ( ( rc = inet6_aton ( in_string, &in ) ) != 0 )
 		goto err_inet6_aton;
 
-	/* Parse network device name, if present */
+	/* Parse scope ID, if applicable */
 	if ( netdev_string ) {
+
+		/* Parse explicit network device name, if present */
 		netdev = find_netdev ( netdev_string );
 		if ( ! netdev ) {
 			rc = -ENODEV;
 			goto err_find_netdev;
 		}
 		sin6->sin6_scope_id = netdev->index;
+
+	} else if ( IN6_IS_ADDR_NONGLOBAL ( &in ) ) {
+
+		/* If no network device is explicitly specified for a
+		 * link-local or multicast address, default to using
+		 * "netX" (if existent).
+		 */
+		netdev = last_opened_netdev();
+		if ( netdev )
+			sin6->sin6_scope_id = netdev->index;
 	}
 
 	/* Copy IPv6 address portion to socket address */
@@ -1104,6 +1115,9 @@ struct net_driver ipv6_driver __net_driver = {
 	.remove = ipv6_remove,
 };
 
+/* Drag in objects via ipv6_protocol */
+REQUIRING_SYMBOL ( ipv6_protocol );
+
 /* Drag in ICMPv6 */
 REQUIRE_OBJECT ( icmpv6 );
 
diff --git a/qemu/roms/ipxe/src/net/neighbour.c b/qemu/roms/ipxe/src/net/neighbour.c
index e3026ce46..7f66d9992 100644
--- a/qemu/roms/ipxe/src/net/neighbour.c
+++ b/qemu/roms/ipxe/src/net/neighbour.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -91,8 +95,8 @@ static struct neighbour * neighbour_create ( struct net_device *netdev,
 	memcpy ( neighbour->net_dest, net_dest,
 		 net_protocol->net_addr_len );
 	timer_init ( &neighbour->timer, neighbour_expired, &neighbour->refcnt );
-	neighbour->timer.min_timeout = NEIGHBOUR_MIN_TIMEOUT;
-	neighbour->timer.max_timeout = NEIGHBOUR_MAX_TIMEOUT;
+	set_timer_limits ( &neighbour->timer, NEIGHBOUR_MIN_TIMEOUT,
+			   NEIGHBOUR_MAX_TIMEOUT );
 	INIT_LIST_HEAD ( &neighbour->tx_queue );
 
 	/* Transfer ownership to cache */
@@ -318,7 +322,7 @@ int neighbour_tx ( struct io_buffer *iobuf, struct net_device *netdev,
 			netdev->name, net_protocol->name,
 			net_protocol->ntoa ( net_dest ) );
 		list_add_tail ( &iobuf->list, &neighbour->tx_queue );
-		return -EAGAIN;
+		return 0;
 	}
 }
 
diff --git a/qemu/roms/ipxe/src/net/netdev_settings.c b/qemu/roms/ipxe/src/net/netdev_settings.c
index b3b2e68d8..edd4c4b9f 100644
--- a/qemu/roms/ipxe/src/net/netdev_settings.c
+++ b/qemu/roms/ipxe/src/net/netdev_settings.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <errno.h>
@@ -121,6 +125,10 @@ static int netdev_fetch_bustype ( struct net_device *netdev, void *data,
 		[BUS_TYPE_MCA] = "MCA",
 		[BUS_TYPE_ISA] = "ISA",
 		[BUS_TYPE_TAP] = "TAP",
+		[BUS_TYPE_EFI] = "EFI",
+		[BUS_TYPE_XEN] = "XEN",
+		[BUS_TYPE_HV] = "HV",
+		[BUS_TYPE_USB] = "USB",
 	};
 	struct device_description *desc = &netdev->dev->desc;
 	const char *bustype;
diff --git a/qemu/roms/ipxe/src/net/netdevice.c b/qemu/roms/ipxe/src/net/netdevice.c
index a55e6b7d7..7c40a2ac8 100644
--- a/qemu/roms/ipxe/src/net/netdevice.c
+++ b/qemu/roms/ipxe/src/net/netdevice.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -35,6 +39,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/device.h>
 #include <ipxe/errortab.h>
 #include <ipxe/profile.h>
+#include <ipxe/fault.h>
 #include <ipxe/vlan.h>
 #include <ipxe/netdevice.h>
 
@@ -157,6 +162,9 @@ void netdev_rx_unfreeze ( struct net_device *netdev ) {
  */
 void netdev_link_err ( struct net_device *netdev, int rc ) {
 
+	/* Stop link block timer */
+	stop_timer ( &netdev->link_block );
+
 	/* Record link state */
 	netdev->link_rc = rc;
 	if ( netdev->link_rc == 0 ) {
@@ -187,6 +195,50 @@ void netdev_link_down ( struct net_device *netdev ) {
 }
 
 /**
+ * Mark network device link as being blocked
+ *
+ * @v netdev		Network device
+ * @v timeout		Timeout (in ticks)
+ */
+void netdev_link_block ( struct net_device *netdev, unsigned long timeout ) {
+
+	/* Start link block timer */
+	if ( ! netdev_link_blocked ( netdev ) ) {
+		DBGC ( netdev, "NETDEV %s link blocked for %ld ticks\n",
+		       netdev->name, timeout );
+	}
+	start_timer_fixed ( &netdev->link_block, timeout );
+}
+
+/**
+ * Mark network device link as being unblocked
+ *
+ * @v netdev		Network device
+ */
+void netdev_link_unblock ( struct net_device *netdev ) {
+
+	/* Stop link block timer */
+	if ( netdev_link_blocked ( netdev ) )
+		DBGC ( netdev, "NETDEV %s link unblocked\n", netdev->name );
+	stop_timer ( &netdev->link_block );
+}
+
+/**
+ * Handle network device link block timer expiry
+ *
+ * @v timer		Link block timer
+ * @v fail		Failure indicator
+ */
+static void netdev_link_block_expired ( struct retry_timer *timer,
+					int fail __unused ) {
+	struct net_device *netdev =
+		container_of ( timer, struct net_device, link_block );
+
+	/* Assume link is no longer blocked */
+	DBGC ( netdev, "NETDEV %s link block expired\n", netdev->name );
+}
+
+/**
  * Record network device statistic
  *
  * @v stats		Network device statistics
@@ -252,11 +304,8 @@ int netdev_tx ( struct net_device *netdev, struct io_buffer *iobuf ) {
 	}
 
 	/* Discard packet (for test purposes) if applicable */
-	if ( ( NETDEV_DISCARD_RATE > 0 ) &&
-	     ( ( random() % NETDEV_DISCARD_RATE ) == 0 ) ) {
-		rc = -EAGAIN;
+	if ( ( rc = inject_fault ( NETDEV_DISCARD_RATE ) ) != 0 )
 		goto err;
-	}
 
 	/* Transmit packet */
 	if ( ( rc = netdev->op->transmit ( netdev, iobuf ) ) != 0 )
@@ -406,14 +455,14 @@ static void netdev_tx_flush ( struct net_device *netdev ) {
  * function takes ownership of the I/O buffer.
  */
 void netdev_rx ( struct net_device *netdev, struct io_buffer *iobuf ) {
+	int rc;
 
 	DBGC2 ( netdev, "NETDEV %s received %p (%p+%zx)\n",
 		netdev->name, iobuf, iobuf->data, iob_len ( iobuf ) );
 
 	/* Discard packet (for test purposes) if applicable */
-	if ( ( NETDEV_DISCARD_RATE > 0 ) &&
-	     ( ( random() % NETDEV_DISCARD_RATE ) == 0 ) ) {
-		netdev_rx_err ( netdev, iobuf, -EAGAIN );
+	if ( ( rc = inject_fault ( NETDEV_DISCARD_RATE ) ) != 0 ) {
+		netdev_rx_err ( netdev, iobuf, rc );
 		return;
 	}
 
@@ -541,7 +590,8 @@ static struct interface_descriptor netdev_config_desc =
 static void free_netdev ( struct refcnt *refcnt ) {
 	struct net_device *netdev =
 		container_of ( refcnt, struct net_device, refcnt );
-	
+
+	stop_timer ( &netdev->link_block );
 	netdev_tx_flush ( netdev );
 	netdev_rx_flush ( netdev );
 	clear_settings ( netdev_settings ( netdev ) );
@@ -571,6 +621,8 @@ struct net_device * alloc_netdev ( size_t priv_len ) {
 	if ( netdev ) {
 		ref_init ( &netdev->refcnt, free_netdev );
 		netdev->link_rc = -EUNKNOWN_LINK_STATUS;
+		timer_init ( &netdev->link_block, netdev_link_block_expired,
+			     &netdev->refcnt );
 		INIT_LIST_HEAD ( &netdev->tx_queue );
 		INIT_LIST_HEAD ( &netdev->tx_deferred );
 		INIT_LIST_HEAD ( &netdev->rx_queue );
@@ -624,11 +676,11 @@ int register_netdev ( struct net_device *netdev ) {
 	}
 
 	/* Record device index and create device name */
-	netdev->index = netdev_index++;
 	if ( netdev->name[0] == '\0' ) {
 		snprintf ( netdev->name, sizeof ( netdev->name ), "net%d",
-			   netdev->index );
+			   netdev_index );
 	}
+	netdev->index = ++netdev_index;
 
 	/* Use least significant bits of the link-layer address to
 	 * improve the randomness of the (non-cryptographic) random
diff --git a/qemu/roms/ipxe/src/net/nullnet.c b/qemu/roms/ipxe/src/net/nullnet.c
index 4ac50f64b..2948b38c0 100644
--- a/qemu/roms/ipxe/src/net/nullnet.c
+++ b/qemu/roms/ipxe/src/net/nullnet.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/net/pccrc.c b/qemu/roms/ipxe/src/net/pccrc.c
new file mode 100644
index 000000000..4cd82cd1c
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/pccrc.c
@@ -0,0 +1,818 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/sha256.h>
+#include <ipxe/sha512.h>
+#include <ipxe/hmac.h>
+#include <ipxe/base16.h>
+#include <ipxe/pccrc.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC]
+ *
+ */
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Transcribe hash value (for debugging)
+ *
+ * @v info		Content information
+ * @v hash		Hash value
+ * @ret string		Hash value string
+ */
+static inline const char *
+peerdist_info_hash_ntoa ( const struct peerdist_info *info, const void *hash ) {
+	static char buf[ ( 2 * PEERDIST_DIGEST_MAX_SIZE ) + 1 /* NUL */ ];
+	size_t digestsize = info->digestsize;
+
+	/* Sanity check */
+	assert ( info != NULL );
+	assert ( digestsize != 0 );
+	assert ( base16_encoded_len ( digestsize ) < sizeof ( buf ) );
+
+	/* Transcribe hash value */
+	base16_encode ( hash, digestsize, buf, sizeof ( buf ) );
+	return buf;
+}
+
+/**
+ * Get raw data
+ *
+ * @v info		Content information
+ * @v data		Data buffer
+ * @v offset		Starting offset
+ * @v len		Length
+ * @ret rc		Return status code
+ */
+static int peerdist_info_get ( const struct peerdist_info *info, void *data,
+			       size_t offset, size_t len ) {
+
+	/* Sanity check */
+	if ( ( offset > info->raw.len ) ||
+	     ( len > ( info->raw.len - offset ) ) ) {
+		DBGC ( info, "PCCRC %p data underrun at [%zx,%zx) of %zx\n",
+		       info, offset, ( offset + len ), info->raw.len );
+		return -ERANGE;
+	}
+
+	/* Copy data */
+	copy_from_user ( data, info->raw.data, offset, len );
+
+	return 0;
+}
+
+/**
+ * Populate segment hashes
+ *
+ * @v segment		Content information segment to fill in
+ * @v hash		Segment hash of data
+ * @v secret		Segment secret
+ */
+static void peerdist_info_segment_hash ( struct peerdist_info_segment *segment,
+					 const void *hash, const void *secret ){
+	const struct peerdist_info *info = segment->info;
+	struct digest_algorithm *digest = info->digest;
+	uint8_t ctx[digest->ctxsize];
+	size_t digestsize = info->digestsize;
+	size_t secretsize = digestsize;
+	static const uint16_t magic[] = PEERDIST_SEGMENT_ID_MAGIC;
+
+	/* Sanity check */
+	assert ( digestsize <= sizeof ( segment->hash ) );
+	assert ( digestsize <= sizeof ( segment->secret ) );
+	assert ( digestsize <= sizeof ( segment->id ) );
+
+	/* Get segment hash of data */
+	memcpy ( segment->hash, hash, digestsize );
+
+	/* Get segment secret */
+	memcpy ( segment->secret, secret, digestsize );
+
+	/* Calculate segment identifier */
+	hmac_init ( digest, ctx, segment->secret, &secretsize );
+	assert ( secretsize == digestsize );
+	hmac_update ( digest, ctx, segment->hash, digestsize );
+	hmac_update ( digest, ctx, magic, sizeof ( magic ) );
+	hmac_final ( digest, ctx, segment->secret, &secretsize, segment->id );
+	assert ( secretsize == digestsize );
+}
+
+/******************************************************************************
+ *
+ * Content Information version 1
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Get number of blocks within a block description
+ *
+ * @v info		Content information
+ * @v offset		Block description offset
+ * @ret blocks		Number of blocks, or negative error
+ */
+static int peerdist_info_v1_blocks ( const struct peerdist_info *info,
+				     size_t offset ) {
+	struct peerdist_info_v1_block raw;
+	unsigned int blocks;
+	int rc;
+
+	/* Get block description header */
+	if ( ( rc = peerdist_info_get ( info, &raw, offset,
+					sizeof ( raw ) ) ) != 0 )
+		return rc;
+
+	/* Calculate number of blocks */
+	blocks = le32_to_cpu ( raw.blocks );
+
+	return blocks;
+}
+
+/**
+ * Locate block description
+ *
+ * @v info		Content information
+ * @v index		Segment index
+ * @ret offset		Block description offset, or negative error
+ */
+static ssize_t peerdist_info_v1_block_offset ( const struct peerdist_info *info,
+					       unsigned int index ) {
+	size_t digestsize = info->digestsize;
+	unsigned int i;
+	size_t offset;
+	int blocks;
+	int rc;
+
+	/* Sanity check */
+	assert ( index < info->segments );
+
+	/* Calculate offset of first block description */
+	offset = ( sizeof ( struct peerdist_info_v1 ) +
+		   ( info->segments *
+		     sizeof ( peerdist_info_v1_segment_t ( digestsize ) ) ) );
+
+	/* Iterate over block descriptions until we find this segment */
+	for ( i = 0 ; i < index ; i++ ) {
+
+		/* Get number of blocks */
+		blocks = peerdist_info_v1_blocks ( info, offset );
+		if ( blocks < 0 ) {
+			rc = blocks;
+			DBGC ( info, "PCCRC %p segment %d could not get number "
+			       "of blocks: %s\n", info, i, strerror ( rc ) );
+			return rc;
+		}
+
+		/* Move to next block description */
+		offset += sizeof ( peerdist_info_v1_block_t ( digestsize,
+							      blocks ) );
+	}
+
+	return offset;
+}
+
+/**
+ * Populate content information
+ *
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1 ( struct peerdist_info *info ) {
+	struct peerdist_info_v1 raw;
+	struct peerdist_info_segment first;
+	struct peerdist_info_segment last;
+	size_t first_skip;
+	size_t last_skip;
+	size_t last_read;
+	int rc;
+
+	/* Get raw header */
+	if ( ( rc = peerdist_info_get ( info, &raw, 0, sizeof ( raw ) ) ) != 0){
+		DBGC ( info, "PCCRC %p could not get V1 content information: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	assert ( raw.version.raw == cpu_to_le16 ( PEERDIST_INFO_V1 ) );
+
+	/* Determine hash algorithm */
+	switch ( raw.hash ) {
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA256 ) :
+		info->digest = &sha256_algorithm;
+		break;
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA384 ) :
+		info->digest = &sha384_algorithm;
+		break;
+	case cpu_to_le32 ( PEERDIST_INFO_V1_HASH_SHA512 ) :
+		info->digest = &sha512_algorithm;
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported hash algorithm %#08x\n",
+		       info, le32_to_cpu ( raw.hash ) );
+		return -ENOTSUP;
+	}
+	info->digestsize = info->digest->digestsize;
+	assert ( info->digest != NULL );
+	DBGC2 ( info, "PCCRC %p using %s[%zd]\n",
+		info, info->digest->name, ( info->digestsize * 8 ) );
+
+	/* Calculate number of segments */
+	info->segments = le32_to_cpu ( raw.segments );
+
+	/* Get first segment */
+	if ( ( rc = peerdist_info_segment ( info, &first, 0 ) ) != 0 )
+		return rc;
+
+	/* Calculate range start offset */
+	info->range.start = first.range.start;
+
+	/* Calculate trimmed range start offset */
+	first_skip = le32_to_cpu ( raw.first );
+	info->trim.start = ( first.range.start + first_skip );
+
+	/* Get last segment */
+	if ( ( rc = peerdist_info_segment ( info, &last,
+					    ( info->segments - 1 ) ) ) != 0 )
+		return rc;
+
+	/* Calculate range end offset */
+	info->range.end = last.range.end;
+
+	/* Calculate trimmed range end offset */
+	if ( raw.last ) {
+		/* Explicit length to include from last segment is given */
+		last_read = le32_to_cpu ( raw.last );
+		last_skip = ( last.index ? 0 : first_skip );
+		info->trim.end = ( last.range.start + last_skip + last_read );
+	} else {
+		/* No explicit length given: range extends to end of segment */
+		info->trim.end = last.range.end;
+	}
+
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v segment		Content information segment to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1_segment ( struct peerdist_info_segment *segment ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v1_segment_t ( digestsize ) raw;
+	ssize_t raw_offset;
+	int blocks;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment->index < info->segments );
+
+	/* Get raw description */
+	raw_offset = ( sizeof ( struct peerdist_info_v1 ) +
+		       ( segment->index * sizeof ( raw ) ) );
+	if ( ( rc = peerdist_info_get ( info, &raw, raw_offset,
+					sizeof ( raw ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d could not get segment "
+		       "description: %s\n", info, segment->index,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	/* Calculate start offset of this segment */
+	segment->range.start = le64_to_cpu ( raw.segment.offset );
+
+	/* Calculate end offset of this segment */
+	segment->range.end = ( segment->range.start +
+			       le32_to_cpu ( raw.segment.len ) );
+
+	/* Calculate block size of this segment */
+	segment->blksize = le32_to_cpu ( raw.segment.blksize );
+
+	/* Locate block description for this segment */
+	raw_offset = peerdist_info_v1_block_offset ( info, segment->index );
+	if ( raw_offset < 0 ) {
+		rc = raw_offset;
+		return rc;
+	}
+
+	/* Get number of blocks */
+	blocks = peerdist_info_v1_blocks ( info, raw_offset );
+	if ( blocks < 0 ) {
+		rc = blocks;
+		DBGC ( info, "PCCRC %p segment %d could not get number of "
+		       "blocks: %s\n", info, segment->index, strerror ( rc ) );
+		return rc;
+	}
+	segment->blocks = blocks;
+
+	/* Calculate segment hashes */
+	peerdist_info_segment_hash ( segment, raw.hash, raw.secret );
+
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v block		Content information block to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v1_block ( struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v1_block_t ( digestsize, segment->blocks ) raw;
+	ssize_t raw_offset;
+	int rc;
+
+	/* Sanity checks */
+	assert ( block->index < segment->blocks );
+
+	/* Calculate start offset of this block */
+	block->range.start = ( segment->range.start +
+			       ( block->index * segment->blksize ) );
+
+	/* Calculate end offset of this block */
+	block->range.end = ( block->range.start + segment->blksize );
+	if ( block->range.end > segment->range.end )
+		block->range.end = segment->range.end;
+
+	/* Locate block description */
+	raw_offset = peerdist_info_v1_block_offset ( info, segment->index );
+	if ( raw_offset < 0 ) {
+		rc = raw_offset;
+		return rc;
+	}
+
+	/* Get block hash */
+	raw_offset += offsetof ( typeof ( raw ), hash[block->index] );
+	if ( ( rc = peerdist_info_get ( info, block->hash, raw_offset,
+					digestsize ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d block %d could not get "
+		       "hash: %s\n", info, segment->index, block->index,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/** Content information version 1 operations */
+static struct peerdist_info_operations peerdist_info_v1_operations = {
+	.info = peerdist_info_v1,
+	.segment = peerdist_info_v1_segment,
+	.block = peerdist_info_v1_block,
+};
+
+/******************************************************************************
+ *
+ * Content Information version 2
+ *
+ ******************************************************************************
+ */
+
+/** A segment cursor */
+struct peerdist_info_v2_cursor {
+	/** Raw data offset */
+	size_t offset;
+	/** Number of segments remaining within this chunk */
+	unsigned int remaining;
+	/** Accumulated segment length */
+	size_t len;
+};
+
+/**
+ * Initialise segment cursor
+ *
+ * @v cursor		Segment cursor
+ */
+static inline void
+peerdist_info_v2_cursor_init ( struct peerdist_info_v2_cursor *cursor ) {
+
+	/* Initialise cursor */
+	cursor->offset = ( sizeof ( struct peerdist_info_v2 ) +
+			   sizeof ( struct peerdist_info_v2_chunk ) );
+	cursor->remaining = 0;
+	cursor->len = 0;
+}
+
+/**
+ * Update segment cursor to next segment description
+ *
+ * @v info		Content information
+ * @v offset		Current offset
+ * @v remaining		Number of segments remaining within this chunk
+ * @ret rc		Return status code
+ */
+static int
+peerdist_info_v2_cursor_next ( const struct peerdist_info *info,
+			       struct peerdist_info_v2_cursor *cursor ) {
+	size_t digestsize = info->digestsize;
+	peerdist_info_v2_segment_t ( digestsize ) raw;
+	struct peerdist_info_v2_chunk chunk;
+	int rc;
+
+	/* Get chunk description if applicable */
+	if ( ! cursor->remaining ) {
+
+		/* Get chunk description */
+		if ( ( rc = peerdist_info_get ( info, &chunk,
+						( cursor->offset -
+						  sizeof ( chunk ) ),
+						sizeof ( chunk ) ) ) != 0 )
+			return rc;
+
+		/* Update number of segments remaining */
+		cursor->remaining = ( be32_to_cpu ( chunk.len ) /
+				      sizeof ( raw ) );
+	}
+
+	/* Get segment description header */
+	if ( ( rc = peerdist_info_get ( info, &raw.segment, cursor->offset,
+					sizeof ( raw.segment ) ) ) != 0 )
+		return rc;
+
+	/* Update cursor */
+	cursor->offset += sizeof ( raw );
+	cursor->remaining--;
+	if ( ! cursor->remaining )
+		cursor->offset += sizeof ( chunk );
+	cursor->len += be32_to_cpu ( raw.segment.len );
+
+	return 0;
+}
+
+/**
+ * Get number of segments and total length
+ *
+ * @v info		Content information
+ * @v len		Length to fill in
+ * @ret rc		Number of segments, or negative error
+ */
+static int peerdist_info_v2_segments ( const struct peerdist_info *info,
+				       size_t *len ) {
+	struct peerdist_info_v2_cursor cursor;
+	unsigned int segments;
+	int rc;
+
+	/* Iterate over all segments */
+	for ( peerdist_info_v2_cursor_init ( &cursor ), segments = 0 ;
+	      cursor.offset < info->raw.len ; segments++ ) {
+
+		/* Update segment cursor */
+		if ( ( rc = peerdist_info_v2_cursor_next ( info,
+							   &cursor ) ) != 0 ) {
+			DBGC ( info, "PCCRC %p segment %d could not update "
+			       "segment cursor: %s\n",
+			       info, segments, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Record accumulated length */
+	*len = cursor.len;
+
+	return segments;
+}
+
+/**
+ * Populate content information
+ *
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2 ( struct peerdist_info *info ) {
+	struct peerdist_info_v2 raw;
+	size_t len = 0;
+	int segments;
+	int rc;
+
+	/* Get raw header */
+	if ( ( rc = peerdist_info_get ( info, &raw, 0, sizeof ( raw ) ) ) != 0){
+		DBGC ( info, "PCCRC %p could not get V2 content information: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	assert ( raw.version.raw == cpu_to_le16 ( PEERDIST_INFO_V2 ) );
+
+	/* Determine hash algorithm */
+	switch ( raw.hash ) {
+	case PEERDIST_INFO_V2_HASH_SHA512_TRUNC :
+		info->digest = &sha512_algorithm;
+		info->digestsize = ( 256 / 8 );
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported hash algorithm %#02x\n",
+		       info, raw.hash );
+		return -ENOTSUP;
+	}
+	assert ( info->digest != NULL );
+	DBGC2 ( info, "PCCRC %p using %s[%zd]\n",
+		info, info->digest->name, ( info->digestsize * 8 ) );
+
+	/* Calculate number of segments and total length */
+	segments = peerdist_info_v2_segments ( info, &len );
+	if ( segments < 0 ) {
+		rc = segments;
+		DBGC ( info, "PCCRC %p could not get segment count and length: "
+		       "%s\n", info, strerror ( rc ) );
+		return rc;
+	}
+	info->segments = segments;
+
+	/* Calculate range start offset */
+	info->range.start = be64_to_cpu ( raw.offset );
+
+	/* Calculate trimmed range start offset */
+	info->trim.start = ( info->range.start + be32_to_cpu ( raw.first ) );
+
+	/* Calculate range end offset */
+	info->range.end = ( info->range.start + len );
+
+	/* Calculate trimmed range end offset */
+	info->trim.end = ( raw.len ? be64_to_cpu ( raw.len ) :
+			   info->range.end );
+
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v segment		Content information segment to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2_segment ( struct peerdist_info_segment *segment ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+	peerdist_info_v2_segment_t ( digestsize ) raw;
+	struct peerdist_info_v2_cursor cursor;
+	unsigned int index;
+	size_t len;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment->index < info->segments );
+
+	/* Iterate over all segments before the target segment */
+	for ( peerdist_info_v2_cursor_init ( &cursor ), index = 0 ;
+	      index < segment->index ; index++ ) {
+
+		/* Update segment cursor */
+		if ( ( rc = peerdist_info_v2_cursor_next ( info,
+							   &cursor ) ) != 0 ) {
+			DBGC ( info, "PCCRC %p segment %d could not update "
+			       "segment cursor: %s\n",
+			       info, index, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Get raw description */
+	if ( ( rc = peerdist_info_get ( info, &raw, cursor.offset,
+					sizeof ( raw ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p segment %d could not get segment "
+		       "description: %s\n",
+		       info, segment->index, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Calculate start offset of this segment */
+	segment->range.start = ( info->range.start + cursor.len );
+
+	/* Calculate end offset of this segment */
+	len = be32_to_cpu ( raw.segment.len );
+	segment->range.end = ( segment->range.start + len );
+
+	/* Model as a segment containing a single block */
+	segment->blocks = 1;
+	segment->blksize = len;
+
+	/* Calculate segment hashes */
+	peerdist_info_segment_hash ( segment, raw.hash, raw.secret );
+
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v block		Content information block to fill in
+ * @ret rc		Return status code
+ */
+static int peerdist_info_v2_block ( struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+
+	/* Sanity checks */
+	assert ( block->index < segment->blocks );
+
+	/* Model as a block covering the whole segment */
+	memcpy ( &block->range, &segment->range, sizeof ( block->range ) );
+	memcpy ( block->hash, segment->hash, digestsize );
+
+	return 0;
+}
+
+/** Content information version 2 operations */
+static struct peerdist_info_operations peerdist_info_v2_operations = {
+	.block = peerdist_info_v2_block,
+	.segment = peerdist_info_v2_segment,
+	.info = peerdist_info_v2,
+};
+
+/******************************************************************************
+ *
+ * Content Information
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Populate content information
+ *
+ * @v data		Raw data
+ * @v len		Length of raw data
+ * @v info		Content information to fill in
+ * @ret rc		Return status code
+ */
+int peerdist_info ( userptr_t data, size_t len, struct peerdist_info *info ) {
+	union peerdist_info_version version;
+	int rc;
+
+	/* Initialise structure */
+	memset ( info, 0, sizeof ( *info ) );
+	info->raw.data = data;
+	info->raw.len = len;
+
+	/* Get version */
+	if ( ( rc = peerdist_info_get ( info, &version, 0,
+					sizeof ( version ) ) ) != 0 ) {
+		DBGC ( info, "PCCRC %p could not get version: %s\n",
+		       info, strerror ( rc ) );
+		return rc;
+	}
+	DBGC2 ( info, "PCCRC %p version %d.%d\n",
+		info, version.major, version.minor );
+
+	/* Determine version */
+	switch ( version.raw ) {
+	case cpu_to_le16 ( PEERDIST_INFO_V1 ) :
+		info->op = &peerdist_info_v1_operations;
+		break;
+	case cpu_to_le16 ( PEERDIST_INFO_V2 ) :
+		info->op = &peerdist_info_v2_operations;
+		break;
+	default:
+		DBGC ( info, "PCCRC %p unsupported version %d.%d\n",
+		       info, version.major, version.minor );
+		return -ENOTSUP;
+	}
+	assert ( info->op != NULL );
+	assert ( info->op->info != NULL );
+
+	/* Populate content information */
+	if ( ( rc = info->op->info ( info ) ) != 0 )
+		return rc;
+
+	DBGC2 ( info, "PCCRC %p range [%08zx,%08zx) covers [%08zx,%08zx) with "
+		"%d segments\n", info, info->range.start, info->range.end,
+		info->trim.start, info->trim.end, info->segments );
+	return 0;
+}
+
+/**
+ * Populate content information segment
+ *
+ * @v info		Content information
+ * @v segment		Content information segment to fill in
+ * @v index		Segment index
+ * @ret rc		Return status code
+ */
+int peerdist_info_segment ( const struct peerdist_info *info,
+			    struct peerdist_info_segment *segment,
+			    unsigned int index ) {
+	int rc;
+
+	/* Sanity checks */
+	assert ( info != NULL );
+	assert ( info->op != NULL );
+	assert ( info->op->segment != NULL );
+	if ( index >= info->segments ) {
+		DBGC ( info, "PCCRC %p segment %d of [0,%d) out of range\n",
+		       info, index, info->segments );
+		return -ERANGE;
+	}
+
+	/* Initialise structure */
+	memset ( segment, 0, sizeof ( *segment ) );
+	segment->info = info;
+	segment->index = index;
+
+	/* Populate content information segment */
+	if ( ( rc = info->op->segment ( segment ) ) != 0 )
+		return rc;
+
+	DBGC2 ( info, "PCCRC %p segment %d range [%08zx,%08zx) with %d "
+		"blocks\n", info, segment->index, segment->range.start,
+		segment->range.end, segment->blocks );
+	DBGC2 ( info, "PCCRC %p segment %d digest %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->hash ) );
+	DBGC2 ( info, "PCCRC %p segment %d secret %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->secret ) );
+	DBGC2 ( info, "PCCRC %p segment %d identf %s\n", info, segment->index,
+		peerdist_info_hash_ntoa ( info, segment->id ) );
+	return 0;
+}
+
+/**
+ * Populate content information block
+ *
+ * @v segment		Content information segment
+ * @v block		Content information block to fill in
+ * @v index		Block index
+ * @ret rc		Return status code
+ */
+int peerdist_info_block ( const struct peerdist_info_segment *segment,
+			  struct peerdist_info_block *block,
+			  unsigned int index ) {
+	const struct peerdist_info *info = segment->info;
+	size_t start;
+	size_t end;
+	int rc;
+
+	/* Sanity checks */
+	assert ( segment != NULL );
+	assert ( info != NULL );
+	assert ( info->op != NULL );
+	assert ( info->op->block != NULL );
+	if ( index >= segment->blocks ) {
+		DBGC ( info, "PCCRC %p segment %d block %d of [0,%d) out of "
+		       "range\n", info, segment->index, index, segment->blocks);
+		return -ERANGE;
+	}
+
+	/* Initialise structure */
+	memset ( block, 0, sizeof ( *block ) );
+	block->segment = segment;
+	block->index = index;
+
+	/* Populate content information block */
+	if ( ( rc = info->op->block ( block ) ) != 0 )
+		return rc;
+
+	/* Calculate trimmed range */
+	start = block->range.start;
+	if ( start < info->trim.start )
+		start = info->trim.start;
+	end = block->range.end;
+	if ( end > info->trim.end )
+		end = info->trim.end;
+	if ( end < start )
+		end = start;
+	block->trim.start = start;
+	block->trim.end = end;
+
+	DBGC2 ( info, "PCCRC %p segment %d block %d hash %s\n",
+		info, segment->index, block->index,
+		peerdist_info_hash_ntoa ( info, block->hash ) );
+	DBGC2 ( info, "PCCRC %p segment %d block %d range [%08zx,%08zx) covers "
+		"[%08zx,%08zx)\n", info, segment->index, block->index,
+		block->range.start, block->range.end, block->trim.start,
+		block->trim.end );
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/net/pccrd.c b/qemu/roms/ipxe/src/net/pccrd.c
new file mode 100644
index 000000000..04b5dd86c
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/pccrd.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/pccrd.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Discovery Protocol [MS-PCCRD]
+ *
+ * This protocol manages to ingeniously combine the excessive
+ * verbosity of XML with a paucity of actual information.  For
+ * example: even in version 2.0 of the protocol it is still not
+ * possible to discover which peers hold a specific block within a
+ * given segment.
+ *
+ * For added bonus points, version 1.0 of the protocol is specified to
+ * use a case-sensitive string comparison (for SHA2 digest values) but
+ * nothing specifies whether the strings in question should be in
+ * upper or lower case.  There are example strings given in the
+ * specification, but the author skilfully manages to leave the issue
+ * unresolved by using the somewhat implausible digest value of
+ * "0200000000000000000000000000000000000000000000000000000000000000".
+ *
+ * Just in case you were thinking that the silver lining of the choice
+ * to use an XML-based protocol would be the ability to generate and
+ * process messages with standard tools, version 2.0 of the protocol
+ * places most of the critical information inside a Base64-encoded
+ * custom binary data structure.  Within an XML element, naturally.
+ *
+ * I hereby announce this specification to be the 2015 winner of the
+ * prestigious "UEFI HII API" award for incompetent design.
+ */
+
+/** Discovery request format */
+#define PEERDIST_DISCOVERY_REQUEST					      \
+	"<?xml version=\"1.0\" encoding=\"utf-8\"?>"			      \
+	"<soap:Envelope "						      \
+	    "xmlns:soap=\"http://www.w3.org/2003/05/soap-envelope\" "	      \
+	    "xmlns:wsa=\"http://schemas.xmlsoap.org/ws/2004/08/addressing\" " \
+	    "xmlns:wsd=\"http://schemas.xmlsoap.org/ws/2005/04/discovery\" "  \
+	    "xmlns:PeerDist=\"http://schemas.microsoft.com/p2p/"	      \
+			     "2007/09/PeerDistributionDiscovery\">"	      \
+	  "<soap:Header>"						      \
+	    "<wsa:To>"							      \
+	      "urn:schemas-xmlsoap-org:ws:2005:04:discovery"		      \
+	    "</wsa:To>"							      \
+	    "<wsa:Action>"						      \
+	      "http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe"	      \
+	    "</wsa:Action>"						      \
+	    "<wsa:MessageID>"						      \
+	      "urn:uuid:%s"						      \
+	    "</wsa:MessageID>"						      \
+	  "</soap:Header>"						      \
+	  "<soap:Body>"							      \
+	    "<wsd:Probe>"						      \
+	      "<wsd:Types>"						      \
+		"PeerDist:PeerDistData"					      \
+	      "</wsd:Types>"						      \
+	      "<wsd:Scopes MatchBy=\"http://schemas.xmlsoap.org/ws/"	      \
+				    "2005/04/discovery/strcmp0\">"	      \
+		"%s"							      \
+	      "</wsd:Scopes>"						      \
+	    "</wsd:Probe>"						      \
+	  "</soap:Body>"						      \
+	"</soap:Envelope>"
+
+/**
+ * Construct discovery request
+ *
+ * @v uuid		Message UUID string
+ * @v id		Segment identifier string
+ * @ret request		Discovery request, or NULL on failure
+ *
+ * The request is dynamically allocated; the caller must eventually
+ * free() the request.
+ */
+char * peerdist_discovery_request ( const char *uuid, const char *id ) {
+	char *request;
+	int len;
+
+	/* Construct request */
+	len = asprintf ( &request, PEERDIST_DISCOVERY_REQUEST, uuid, id );
+	if ( len < 0 )
+		return NULL;
+
+	return request;
+}
+
+/**
+ * Locate discovery reply tag
+ *
+ * @v data		Reply data (not NUL-terminated)
+ * @v len		Length of reply data
+ * @v tag		XML tag
+ * @ret found		Found tag (or NULL if not found)
+ */
+static char * peerdist_discovery_reply_tag ( char *data, size_t len,
+					     const char *tag ) {
+	size_t tag_len = strlen ( tag );
+
+	/* Search, allowing for the fact that the reply data is not
+	 * cleanly NUL-terminated and may contain embedded NULs due to
+	 * earlier parsing.
+	 */
+	for ( ; len >= tag_len ; data++, len-- ) {
+		if ( strncmp ( data, tag, tag_len ) == 0 )
+			return data;
+	}
+	return NULL;
+}
+
+/**
+ * Locate discovery reply values
+ *
+ * @v data		Reply data (not NUL-terminated, will be modified)
+ * @v len		Length of reply data
+ * @v name		XML tag name
+ * @ret values		Tag values (or NULL if not found)
+ *
+ * The reply data is modified by adding NULs and moving characters as
+ * needed to produce a NUL-separated list of values, terminated with a
+ * zero-length string.
+ *
+ * This is not supposed to be a full XML parser; it's supposed to
+ * include just enough functionality to allow PeerDist discovery to
+ * work with existing implementations.
+ */
+static char * peerdist_discovery_reply_values ( char *data, size_t len,
+						const char *name ) {
+	char buf[ 2 /* "</" */ + strlen ( name ) + 1 /* ">" */ + 1 /* NUL */ ];
+	char *open;
+	char *close;
+	char *start;
+	char *end;
+	char *in;
+	char *out;
+	char c;
+
+	/* Locate opening tag */
+	snprintf ( buf, sizeof ( buf ), "<%s>", name );
+	open = peerdist_discovery_reply_tag ( data, len, buf );
+	if ( ! open )
+		return NULL;
+	start = ( open + strlen ( buf ) );
+	len -= ( start - data );
+	data = start;
+
+	/* Locate closing tag */
+	snprintf ( buf, sizeof ( buf ), "</%s>", name );
+	close = peerdist_discovery_reply_tag ( data, len, buf );
+	if ( ! close )
+		return NULL;
+	assert ( close >= open );
+	end = close;
+
+	/* Strip initial whitespace, convert other whitespace
+	 * sequences to single NULs, add terminating pair of NULs.
+	 * This will probably overwrite part of the closing tag.
+	 */
+	for ( in = start, out = start ; in < end ; in++ ) {
+		c = *in;
+		if ( isspace ( c ) ) {
+			if ( ( out > start ) && ( out[-1] != '\0' ) )
+				*(out++) = '\0';
+		} else {
+			*(out++) = c;
+		}
+	}
+	*(out++) = '\0';
+	*(out++) = '\0';
+	assert ( out < ( close + strlen ( buf ) ) );
+
+	return start;
+}
+
+/**
+ * Parse discovery reply
+ *
+ * @v data		Reply data (not NUL-terminated, will be modified)
+ * @v len		Length of reply data
+ * @v reply		Discovery reply to fill in
+ * @ret rc		Return status code
+ *
+ * The discovery reply includes pointers to strings within the
+ * modified reply data.
+ */
+int peerdist_discovery_reply ( char *data, size_t len,
+			       struct peerdist_discovery_reply *reply ) {
+	static const struct peerdist_discovery_block_count zcount = {
+		.hex = "00000000",
+	};
+	struct peerdist_discovery_block_count *count;
+	unsigned int max;
+	unsigned int i;
+	char *scopes;
+	char *xaddrs;
+	char *blockcount;
+	char *in;
+	char *out;
+	size_t skip;
+
+	/* Find <wsd:Scopes> tag */
+	scopes = peerdist_discovery_reply_values ( data, len, "wsd:Scopes" );
+	if ( ! scopes ) {
+		DBGC ( reply, "PCCRD %p missing <wsd:Scopes> tag\n", reply );
+		return -ENOENT;
+	}
+
+	/* Find <wsd:XAddrs> tag */
+	xaddrs = peerdist_discovery_reply_values ( data, len, "wsd:XAddrs" );
+	if ( ! xaddrs ) {
+		DBGC ( reply, "PCCRD %p missing <wsd:XAddrs> tag\n", reply );
+		return -ENOENT;
+	}
+
+	/* Find <PeerDist:BlockCount> tag */
+	blockcount = peerdist_discovery_reply_values ( data, len,
+						       "PeerDist:BlockCount" );
+	if ( ! blockcount ) {
+		DBGC ( reply, "PCCRD %p missing <PeerDist:BlockCount> tag\n",
+		       reply );
+		return -ENOENT;
+	}
+
+	/* Determine maximum number of segments (according to number
+	 * of entries in the block count list).
+	 */
+	max = ( strlen ( blockcount ) / sizeof ( *count ) );
+	count = container_of ( blockcount,
+			       struct peerdist_discovery_block_count, hex[0] );
+
+	/* Eliminate any segments with a zero block count */
+	for ( i = 0, in = scopes, out = scopes ; *in ; i++, in += skip ) {
+
+		/* Fail if we have overrun the maximum number of segments */
+		if ( i >= max ) {
+			DBGC ( reply, "PCCRD %p too many segment IDs\n",
+			       reply );
+			return -EPROTO;
+		}
+
+		/* Delete segment if block count is zero */
+		skip = ( strlen ( in ) + 1 /* NUL */ );
+		if ( memcmp ( count[i].hex, zcount.hex,
+			      sizeof ( zcount.hex ) ) == 0 )
+			continue;
+		strcpy ( out, in );
+		out += skip;
+	}
+	out[0] = '\0'; /* Ensure list is terminated with a zero-length string */
+
+	/* Fill in discovery reply */
+	reply->ids = scopes;
+	reply->locations = xaddrs;
+
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/net/peerblk.c b/qemu/roms/ipxe/src/net/peerblk.c
new file mode 100644
index 000000000..fd7ea0893
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/peerblk.c
@@ -0,0 +1,1366 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/http.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/xfer.h>
+#include <ipxe/uri.h>
+#include <ipxe/timer.h>
+#include <ipxe/profile.h>
+#include <ipxe/fault.h>
+#include <ipxe/pccrr.h>
+#include <ipxe/peerblk.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol block downloads
+ *
+ */
+
+/** PeerDist decryption chunksize
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_DECRYPT_CHUNKSIZE 2048
+
+/** PeerDist raw block download attempt initial progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RAW_OPEN_TIMEOUT ( 10 * TICKS_PER_SEC )
+
+/** PeerDist raw block download attempt ongoing progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RAW_RX_TIMEOUT ( 15 * TICKS_PER_SEC )
+
+/** PeerDist retrieval protocol block download attempt initial progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RETRIEVAL_OPEN_TIMEOUT ( 3 * TICKS_PER_SEC )
+
+/** PeerDist retrieval protocol block download attempt ongoing progress timeout
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_RETRIEVAL_RX_TIMEOUT ( 5 * TICKS_PER_SEC )
+
+/** PeerDist maximum number of full download attempt cycles
+ *
+ * This is the maximum number of times that we will try a full cycle
+ * of download attempts (i.e. a retrieval protocol download attempt
+ * from each discovered peer plus a raw download attempt from the
+ * origin server).
+ *
+ * This is a policy decision.
+ */
+#define PEERBLK_MAX_ATTEMPT_CYCLES 4
+
+/** PeerDist block download profiler */
+static struct profiler peerblk_download_profiler __profiler =
+	{ .name = "peerblk.download" };
+
+/** PeerDist block download attempt success profiler */
+static struct profiler peerblk_attempt_success_profiler __profiler =
+	{ .name = "peerblk.attempt.success" };
+
+/** PeerDist block download attempt failure profiler */
+static struct profiler peerblk_attempt_failure_profiler __profiler =
+	{ .name = "peerblk.attempt.failure" };
+
+/** PeerDist block download attempt timeout profiler */
+static struct profiler peerblk_attempt_timeout_profiler __profiler =
+	{ .name = "peerblk.attempt.timeout" };
+
+/** PeerDist block download discovery success profiler */
+static struct profiler peerblk_discovery_success_profiler __profiler =
+	{ .name = "peerblk.discovery.success" };
+
+/** PeerDist block download discovery timeout profiler */
+static struct profiler peerblk_discovery_timeout_profiler __profiler =
+	{ .name = "peerblk.discovery.timeout" };
+
+/**
+ * Get profiling timestamp
+ *
+ * @ret timestamp	Timestamp
+ */
+static inline __attribute__ (( always_inline )) unsigned long
+peerblk_timestamp ( void ) {
+
+	if ( PROFILING ) {
+		return currticks();
+	} else {
+		return 0;
+	}
+}
+
+/**
+ * Free PeerDist block download
+ *
+ * @v refcnt		Reference count
+ */
+static void peerblk_free ( struct refcnt *refcnt ) {
+	struct peerdist_block *peerblk =
+		container_of ( refcnt, struct peerdist_block, refcnt );
+
+	uri_put ( peerblk->uri );
+	free ( peerblk->cipherctx );
+	free ( peerblk );
+}
+
+/**
+ * Reset PeerDist block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for reset
+ */
+static void peerblk_reset ( struct peerdist_block *peerblk, int rc ) {
+
+	/* Stop decryption process */
+	process_del ( &peerblk->process );
+
+	/* Stop timer */
+	stop_timer ( &peerblk->timer );
+
+	/* Abort any current download attempt */
+	intf_restart ( &peerblk->raw, rc );
+	intf_restart ( &peerblk->retrieval, rc );
+
+	/* Empty received data buffer */
+	xferbuf_free ( &peerblk->buffer );
+	peerblk->pos = 0;
+
+	/* Reset digest and free cipher context */
+	digest_init ( peerblk->digest, peerblk->digestctx );
+	free ( peerblk->cipherctx );
+	peerblk->cipherctx = NULL;
+	peerblk->cipher = NULL;
+
+	/* Reset trim thresholds */
+	peerblk->start = ( peerblk->trim.start - peerblk->range.start );
+	peerblk->end = ( peerblk->trim.end - peerblk->range.start );
+	assert ( peerblk->start <= peerblk->end );
+}
+
+/**
+ * Close PeerDist block download
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_close ( struct peerdist_block *peerblk, int rc ) {
+	unsigned long now = peerblk_timestamp();
+
+	/* Profile overall block download */
+	profile_custom ( &peerblk_download_profiler,
+			 ( now - peerblk->started ) );
+
+	/* Reset download attempt */
+	peerblk_reset ( peerblk, rc );
+
+	/* Close discovery */
+	peerdisc_close ( &peerblk->discovery );
+
+	/* Shut down all interfaces */
+	intf_shutdown ( &peerblk->retrieval, rc );
+	intf_shutdown ( &peerblk->raw, rc );
+	intf_shutdown ( &peerblk->xfer, rc );
+}
+
+/**
+ * Calculate offset within overall download
+ *
+ * @v peerblk		PeerDist block download
+ * @v pos		Position within incoming data stream
+ * @ret offset		Offset within overall download
+ */
+static inline __attribute__ (( always_inline )) size_t
+peerblk_offset ( struct peerdist_block *peerblk, size_t pos ) {
+
+	return ( ( pos - peerblk->start ) + peerblk->offset );
+}
+
+/**
+ * Deliver download attempt data block
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Original data transfer metadata
+ * @v pos		Position within incoming data stream
+ * @ret rc		Return status code
+ */
+static int peerblk_deliver ( struct peerdist_block *peerblk,
+			     struct io_buffer *iobuf,
+			     struct xfer_metadata *meta, size_t pos ) {
+	struct xfer_metadata xfer_meta;
+	size_t len = iob_len ( iobuf );
+	size_t start = pos;
+	size_t end = ( pos + len );
+	int rc;
+
+	/* Discard zero-length packets and packets which lie entirely
+	 * outside the trimmed range.
+	 */
+	if ( ( start >= peerblk->end ) || ( end <= peerblk->start ) ||
+	     ( len == 0 ) ) {
+		free_iob ( iobuf );
+		return 0;
+	}
+
+	/* Truncate data to within trimmed range */
+	if ( start < peerblk->start ) {
+		iob_pull ( iobuf, ( peerblk->start - start ) );
+		start = peerblk->start;
+	}
+	if ( end > peerblk->end ) {
+		iob_unput ( iobuf, ( end - peerblk->end ) );
+		end = peerblk->end;
+	}
+
+	/* Construct metadata */
+	memcpy ( &xfer_meta, meta, sizeof ( xfer_meta ) );
+	xfer_meta.flags |= XFER_FL_ABS_OFFSET;
+	xfer_meta.offset = peerblk_offset ( peerblk, start );
+
+	/* Deliver data */
+	if ( ( rc = xfer_deliver ( &peerblk->xfer, iob_disown ( iobuf ),
+				   &xfer_meta ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not deliver data: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Finish PeerDist block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_done ( struct peerdist_block *peerblk, int rc ) {
+	struct digest_algorithm *digest = peerblk->digest;
+	uint8_t hash[digest->digestsize];
+	unsigned long now = peerblk_timestamp();
+
+	/* Check for errors on completion */
+	if ( rc != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d attempt failed: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err;
+	}
+
+	/* Check digest */
+	digest_final ( digest, peerblk->digestctx, hash );
+	if ( memcmp ( hash, peerblk->hash, peerblk->digestsize ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d digest mismatch:\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		DBGC_HDA ( peerblk, 0, hash, peerblk->digestsize );
+		DBGC_HDA ( peerblk, 0, peerblk->hash, peerblk->digestsize );
+		rc = -EIO;
+		goto err;
+	}
+
+	/* Profile successful attempt */
+	profile_custom ( &peerblk_attempt_success_profiler,
+			 ( now - peerblk->attempted ) );
+
+	/* Close download */
+	peerblk_close ( peerblk, 0 );
+	return;
+
+ err:
+	/* Record failure reason and schedule a retry attempt */
+	profile_custom ( &peerblk_attempt_failure_profiler,
+			 ( now - peerblk->attempted ) );
+	peerblk_reset ( peerblk, rc );
+	peerblk->rc = rc;
+	start_timer_nodelay ( &peerblk->timer );
+}
+
+/******************************************************************************
+ *
+ * Raw block download attempts (using an HTTP range request)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open PeerDist raw block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @ret rc		Return status code
+ */
+static int peerblk_raw_open ( struct peerdist_block *peerblk ) {
+	struct http_request_range range;
+	int rc;
+
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d attempting raw range request\n",
+		peerblk, peerblk->segment, peerblk->block );
+
+	/* Construct HTTP range */
+	memset ( &range, 0, sizeof ( range ) );
+	range.start = peerblk->range.start;
+	range.len = ( peerblk->range.end - peerblk->range.start );
+
+	/* Initiate range request to retrieve block */
+	if ( ( rc = http_open ( &peerblk->raw, &http_get, peerblk->uri,
+				&range, NULL ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not create range "
+		       "request: %s\n", peerblk, peerblk->segment,
+		       peerblk->block, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Annul HTTP connection (for testing) if applicable.  Do not
+	 * report as an immediate error, in order to test our ability
+	 * to recover from a totally unresponsive HTTP server.
+	 */
+	if ( inject_fault ( PEERBLK_ANNUL_RATE ) )
+		intf_restart ( &peerblk->raw, 0 );
+
+	return 0;
+}
+
+/**
+ * Receive PeerDist raw data
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerblk_raw_rx ( struct peerdist_block *peerblk,
+			    struct io_buffer *iobuf,
+			    struct xfer_metadata *meta ) {
+	size_t len = iob_len ( iobuf );
+	size_t pos = peerblk->pos;
+	size_t mid = ( ( peerblk->range.end - peerblk->range.start ) / 2 );
+	int rc;
+
+	/* Corrupt received data (for testing) if applicable */
+	inject_corruption ( PEERBLK_CORRUPT_RATE, iobuf->data, len );
+
+	/* Fail if data is delivered out of order, since the streaming
+	 * digest requires strict ordering.
+	 */
+	if ( ( rc = xfer_check_order ( meta, &peerblk->pos, len ) ) != 0 )
+		goto err;
+
+	/* Add data to digest */
+	digest_update ( peerblk->digest, peerblk->digestctx, iobuf->data, len );
+
+	/* Deliver data */
+	if ( ( rc = peerblk_deliver ( peerblk, iob_disown ( iobuf ), meta,
+				      pos ) ) != 0 )
+		goto err;
+
+	/* Extend download attempt timer */
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RAW_RX_TIMEOUT );
+
+	/* Stall download attempt (for testing) if applicable */
+	if ( ( pos < mid ) && ( ( pos + len ) >= mid ) &&
+	     ( ( rc = inject_fault ( PEERBLK_STALL_RATE ) ) != 0 ) ) {
+		intf_restart ( &peerblk->raw, rc );
+	}
+
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	peerblk_done ( peerblk, rc );
+	return rc;
+}
+
+/**
+ * Close PeerDist raw block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_raw_close ( struct peerdist_block *peerblk, int rc ) {
+
+	/* Restart interface */
+	intf_restart ( &peerblk->raw, rc );
+
+	/* Fail immediately if we have an error */
+	if ( rc != 0 )
+		goto done;
+
+	/* Abort download attempt (for testing) if applicable */
+	if ( ( rc = inject_fault ( PEERBLK_ABORT_RATE ) ) != 0 )
+		goto done;
+
+ done:
+	/* Complete download attempt */
+	peerblk_done ( peerblk, rc );
+}
+
+/******************************************************************************
+ *
+ * Retrieval protocol block download attempts (using HTTP POST)
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Construct PeerDist retrieval protocol URI
+ *
+ * @v location		Peer location
+ * @ret uri		Retrieval URI, or NULL on error
+ */
+static struct uri * peerblk_retrieval_uri ( const char *location ) {
+	char uri_string[ 7 /* "http://" */ + strlen ( location ) +
+			 sizeof ( PEERDIST_MAGIC_PATH /* includes NUL */ ) ];
+
+	/* Construct URI string */
+	snprintf ( uri_string, sizeof ( uri_string ),
+		   ( "http://%s" PEERDIST_MAGIC_PATH ), location );
+
+	/* Parse URI string */
+	return parse_uri ( uri_string );
+}
+
+/**
+ * Open PeerDist retrieval protocol block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v location		Peer location
+ * @ret rc		Return status code
+ */
+static int peerblk_retrieval_open ( struct peerdist_block *peerblk,
+				    const char *location ) {
+	size_t digestsize = peerblk->digestsize;
+	peerdist_msg_getblks_t ( digestsize, 1, 0 ) req;
+	peerblk_msg_blk_t ( digestsize, 0, 0, 0 ) *rsp;
+	struct http_request_content content;
+	struct uri *uri;
+	int rc;
+
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d attempting retrieval from %s\n",
+		peerblk, peerblk->segment, peerblk->block, location );
+
+	/* Construct block fetch request */
+	memset ( &req, 0, sizeof ( req ) );
+	req.getblks.hdr.version.raw = htonl ( PEERDIST_MSG_GETBLKS_VERSION );
+	req.getblks.hdr.type = htonl ( PEERDIST_MSG_GETBLKS_TYPE );
+	req.getblks.hdr.len = htonl ( sizeof ( req ) );
+	req.getblks.hdr.algorithm = htonl ( PEERDIST_MSG_AES_128_CBC );
+	req.segment.segment.digestsize = htonl ( digestsize );
+	memcpy ( req.segment.id, peerblk->id, digestsize );
+	req.ranges.ranges.count = htonl ( 1 );
+	req.ranges.range[0].first = htonl ( peerblk->block );
+	req.ranges.range[0].count = htonl ( 1 );
+
+	/* Construct POST request content */
+	memset ( &content, 0, sizeof ( content ) );
+	content.data = &req;
+	content.len = sizeof ( req );
+
+	/* Construct URI */
+	if ( ( uri = peerblk_retrieval_uri ( location ) ) == NULL ) {
+		rc = -ENOMEM;
+		goto err_uri;
+	}
+
+	/* Update trim thresholds */
+	peerblk->start += offsetof ( typeof ( *rsp ), msg.vrf );
+	peerblk->end += offsetof ( typeof ( *rsp ), msg.vrf );
+
+	/* Initiate HTTP POST to retrieve block */
+	if ( ( rc = http_open ( &peerblk->retrieval, &http_post, uri,
+				NULL, &content ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not create retrieval "
+		       "request: %s\n", peerblk, peerblk->segment,
+		       peerblk->block, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Annul HTTP connection (for testing) if applicable.  Do not
+	 * report as an immediate error, in order to test our ability
+	 * to recover from a totally unresponsive HTTP server.
+	 */
+	if ( inject_fault ( PEERBLK_ANNUL_RATE ) )
+		intf_restart ( &peerblk->retrieval, 0 );
+
+ err_open:
+	uri_put ( uri );
+ err_uri:
+	return rc;
+}
+
+/**
+ * Receive PeerDist retrieval protocol data
+ *
+ * @v peerblk		PeerDist block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerblk_retrieval_rx ( struct peerdist_block *peerblk,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	size_t len = iob_len ( iobuf );
+	size_t start;
+	size_t end;
+	size_t before;
+	size_t after;
+	size_t cut;
+	int rc;
+
+	/* Some genius at Microsoft thought it would be a great idea
+	 * to place the AES-CBC initialisation vector *after* the
+	 * encrypted data, thereby making it logically impossible to
+	 * decrypt each packet as it arrives.
+	 *
+	 * To work around this mindless stupidity, we deliver the
+	 * ciphertext as-is and later use xfer_buffer() to obtain
+	 * access to the underlying data transfer buffer in order to
+	 * perform the decryption.
+	 *
+	 * There will be some data both before and after the bytes
+	 * corresponding to the trimmed plaintext: a MSG_BLK
+	 * header/footer, some block padding for the AES-CBC cipher,
+	 * and a possibly large quantity of unwanted ciphertext which
+	 * is excluded from the trimmed content range.  We store this
+	 * data in a local data transfer buffer.  If the amount of
+	 * data to be stored is too large, we will fail allocation and
+	 * so eventually fall back to using a range request (which
+	 * does not require this kind of temporary storage
+	 * allocation).
+	 */
+
+	/* Corrupt received data (for testing) if applicable */
+	inject_corruption ( PEERBLK_CORRUPT_RATE, iobuf->data, len );
+
+	/* Calculate start and end positions of this buffer */
+	start = peerblk->pos;
+	if ( meta->flags & XFER_FL_ABS_OFFSET )
+		start = 0;
+	start += meta->offset;
+	end = ( start + len );
+
+	/* Buffer any data before the trimmed content */
+	if ( ( start < peerblk->start ) && ( len > 0 ) ) {
+
+		/* Calculate length of data before the trimmed content */
+		before = ( peerblk->start - start );
+		if ( before > len )
+			before = len;
+
+		/* Buffer data before the trimmed content */
+		if ( ( rc = xferbuf_write ( &peerblk->buffer, start,
+					    iobuf->data, before ) ) != 0 ) {
+			DBGC ( peerblk, "PEERBLK %p %d.%d could not buffer "
+			       "data: %s\n", peerblk, peerblk->segment,
+			       peerblk->block, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Buffer any data after the trimmed content */
+	if ( ( end > peerblk->end ) && ( len > 0 ) ) {
+
+		/* Calculate length of data after the trimmed content */
+		after = ( end - peerblk->end );
+		if ( after > len )
+			after = len;
+
+		/* Buffer data after the trimmed content */
+		cut = ( peerblk->end - peerblk->start );
+		if ( ( rc = xferbuf_write ( &peerblk->buffer,
+					    ( end - after - cut ),
+					    ( iobuf->data + len - after ),
+					    after ) ) != 0 ) {
+			DBGC ( peerblk, "PEERBLK %p %d.%d could not buffer "
+			       "data: %s\n", peerblk, peerblk->segment,
+			       peerblk->block, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Deliver any remaining data */
+	if ( ( rc = peerblk_deliver ( peerblk, iob_disown ( iobuf ), meta,
+				      start ) ) != 0 )
+		goto err;
+
+	/* Update position */
+	peerblk->pos = end;
+
+	/* Extend download attempt timer */
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RETRIEVAL_RX_TIMEOUT );
+
+	/* Stall download attempt (for testing) if applicable */
+	if ( ( start < peerblk->end ) && ( end >= peerblk->end ) &&
+	     ( ( rc = inject_fault ( PEERBLK_STALL_RATE ) ) != 0 ) ) {
+		intf_restart ( &peerblk->retrieval, rc );
+	}
+
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	peerblk_done ( peerblk, rc );
+	return rc;
+}
+
+/**
+ * Parse retrieval protocol message header
+ *
+ * @v peerblk		PeerDist block download
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_header ( struct peerdist_block *peerblk ) {
+	struct {
+		struct peerdist_msg_transport_header hdr;
+		struct peerdist_msg_header msg;
+	} __attribute__ (( packed )) *msg = peerblk->buffer.data;
+	struct cipher_algorithm *cipher;
+	size_t len = peerblk->buffer.len;
+	size_t keylen = 0;
+	int rc;
+
+	/* Check message length */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for header "
+		       "(%zd bytes)\n", peerblk, peerblk->segment,
+		       peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check message type */
+	if ( msg->msg.type != htonl ( PEERDIST_MSG_BLK_TYPE ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d unexpected message type "
+		       "%#08x\n", peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.type ) );
+		return -EPROTO;
+	}
+
+	/* Determine cipher algorithm and key length */
+	cipher = &aes_cbc_algorithm;
+	switch ( msg->msg.algorithm ) {
+	case htonl ( PEERDIST_MSG_PLAINTEXT ) :
+		cipher = NULL;
+		break;
+	case htonl ( PEERDIST_MSG_AES_128_CBC ) :
+		keylen = ( 128 / 8 );
+		break;
+	case htonl ( PEERDIST_MSG_AES_192_CBC ) :
+		keylen = ( 192 / 8 );
+		break;
+	case htonl ( PEERDIST_MSG_AES_256_CBC ) :
+		keylen = ( 256 / 8 );
+		break;
+	default:
+		DBGC ( peerblk, "PEERBLK %p %d.%d unrecognised algorithm "
+		       "%#08x\n", peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.algorithm ) );
+		return -ENOTSUP;
+	}
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d using %s with %zd-bit key\n",
+		peerblk, peerblk->segment, peerblk->block,
+		( cipher ? cipher->name : "plaintext" ), ( 8 * keylen ) );
+
+	/* Sanity check key length against maximum secret length */
+	if ( keylen > peerblk->digestsize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d %zd-byte secret too short "
+		       "for %zd-bit key\n", peerblk, peerblk->segment,
+		       peerblk->block, peerblk->digestsize, ( 8 * keylen ) );
+		return -EPROTO;
+	}
+
+	/* Allocate cipher context.  Freeing the cipher context (on
+	 * error or otherwise) is handled by peerblk_reset().
+	 */
+	peerblk->cipher = cipher;
+	assert ( peerblk->cipherctx == NULL );
+	peerblk->cipherctx = malloc ( cipher->ctxsize );
+	if ( ! peerblk->cipherctx )
+		return -ENOMEM;
+
+	/* Initialise cipher */
+	if ( ( rc = cipher_setkey ( cipher, peerblk->cipherctx, peerblk->secret,
+				    keylen ) ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not set key: %s\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message segment and block details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data to fill in
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_block ( struct peerdist_block *peerblk,
+				 size_t *buf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	peerblk_msg_blk_t ( digestsize, 0, 0, 0 ) *msg = peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+	size_t data_len;
+	size_t total;
+
+	/* Check message length */
+	if ( len < offsetof ( typeof ( *msg ), msg.block.data ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "zero-length data (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check digest size */
+	if ( ntohl ( msg->msg.segment.segment.digestsize ) != digestsize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d incorrect digest size %d\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.segment.segment.digestsize ) );
+		return -EPROTO;
+	}
+
+	/* Check segment ID */
+	if ( memcmp ( msg->msg.segment.id, peerblk->id, digestsize ) != 0 ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d segment ID mismatch\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		return -EPROTO;
+	}
+
+	/* Check block ID */
+	if ( ntohl ( msg->msg.index ) != peerblk->block ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d block ID mismatch (got %d)\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.index ) );
+		return -EPROTO;
+	}
+
+	/* Check for missing blocks */
+	data_len = be32_to_cpu ( msg->msg.block.block.len );
+	if ( ! data_len ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d block not found\n",
+		       peerblk, peerblk->segment, peerblk->block );
+		return -ENOENT;
+	}
+
+	/* Check for underlength blocks */
+	if ( data_len < ( peerblk->range.end - peerblk->range.start ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d underlength block (%zd "
+		       "bytes)\n", peerblk, peerblk->segment, peerblk->block,
+		       data_len );
+		return -ERANGE;
+	}
+
+	/* Calculate buffered data length (i.e. excluding data which
+	 * was delivered to the final data transfer buffer).
+	 */
+	*buf_len = ( data_len - ( peerblk->end - peerblk->start ) );
+
+	/* Describe data before the trimmed content */
+	peerblk->decrypt[PEERBLK_BEFORE].xferbuf = &peerblk->buffer;
+	peerblk->decrypt[PEERBLK_BEFORE].offset =
+		offsetof ( typeof ( *msg ), msg.block.data );
+	peerblk->decrypt[PEERBLK_BEFORE].len =
+		( peerblk->start -
+		  offsetof ( typeof ( *msg ), msg.block.data ) );
+	total = peerblk->decrypt[PEERBLK_BEFORE].len;
+
+	/* Describe data within the trimmed content */
+	peerblk->decrypt[PEERBLK_DURING].offset =
+		peerblk_offset ( peerblk, peerblk->start );
+	peerblk->decrypt[PEERBLK_DURING].len =
+		( peerblk->end - peerblk->start );
+	total += peerblk->decrypt[PEERBLK_DURING].len;
+
+	/* Describe data after the trimmed content */
+	peerblk->decrypt[PEERBLK_AFTER].xferbuf = &peerblk->buffer;
+	peerblk->decrypt[PEERBLK_AFTER].offset = peerblk->start;
+	peerblk->decrypt[PEERBLK_AFTER].len =
+		( offsetof ( typeof ( *msg ), msg.block.data )
+		  + *buf_len - peerblk->start );
+	total += peerblk->decrypt[PEERBLK_AFTER].len;
+
+	/* Sanity check */
+	assert ( total == be32_to_cpu ( msg->msg.block.block.len ) );
+
+	/* Initialise cipher and digest lengths */
+	peerblk->cipher_remaining = total;
+	peerblk->digest_remaining =
+		( peerblk->range.end - peerblk->range.start );
+	assert ( peerblk->cipher_remaining >= peerblk->digest_remaining );
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message useless details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data
+ * @v vrf_len		Length of uselessness to fill in
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_useless ( struct peerdist_block *peerblk,
+				   size_t buf_len, size_t *vrf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	peerblk_msg_blk_t ( digestsize, buf_len, 0, 0 ) *msg =
+		peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+
+	/* Check message length */
+	if ( len < offsetof ( typeof ( *msg ), msg.vrf.data ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "zero-length uselessness (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Extract length of uselessness */
+	*vrf_len = be32_to_cpu ( msg->msg.vrf.vrf.len );
+
+	return 0;
+}
+
+/**
+ * Parse retrieval protocol message initialisation vector details
+ *
+ * @v peerblk		PeerDist block download
+ * @v buf_len		Length of buffered data
+ * @v vrf_len		Length of uselessness
+ * @ret rc		Return status code
+ */
+static int peerblk_parse_iv ( struct peerdist_block *peerblk, size_t buf_len,
+			      size_t vrf_len ) {
+	size_t digestsize = peerblk->digestsize;
+	size_t blksize = peerblk->cipher->blocksize;
+	peerblk_msg_blk_t ( digestsize, buf_len, vrf_len, blksize ) *msg =
+		peerblk->buffer.data;
+	size_t len = peerblk->buffer.len;
+
+	/* Check message length */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d message too short for "
+		       "initialisation vector (%zd bytes)\n", peerblk,
+		       peerblk->segment, peerblk->block, len );
+		return -ERANGE;
+	}
+
+	/* Check initialisation vector size */
+	if ( ntohl ( msg->msg.iv.iv.blksize ) != blksize ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d incorrect IV size %d\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       ntohl ( msg->msg.iv.iv.blksize ) );
+		return -EPROTO;
+	}
+
+	/* Set initialisation vector */
+	cipher_setiv ( peerblk->cipher, peerblk->cipherctx, msg->msg.iv.data );
+
+	return 0;
+}
+
+/**
+ * Read from decryption buffers
+ *
+ * @v peerblk		PeerDist block download
+ * @v data		Data buffer
+ * @v len		Length to read
+ * @ret rc		Return status code
+ */
+static int peerblk_decrypt_read ( struct peerdist_block *peerblk,
+				  void *data, size_t len ) {
+	struct peerdist_block_decrypt *decrypt = peerblk->decrypt;
+	size_t frag_len;
+	int rc;
+
+	/* Read from each decryption buffer in turn */
+	for ( ; len ; decrypt++, data += frag_len, len -= frag_len ) {
+
+		/* Calculate length to use from this buffer */
+		frag_len = decrypt->len;
+		if ( frag_len > len )
+			frag_len = len;
+		if ( ! frag_len )
+			continue;
+
+		/* Read from this buffer */
+		if ( ( rc = xferbuf_read ( decrypt->xferbuf, decrypt->offset,
+					   data, frag_len ) ) != 0 )
+			return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Write to decryption buffers and update offsets and lengths
+ *
+ * @v peerblk		PeerDist block download
+ * @v data		Data buffer
+ * @v len		Length to read
+ * @ret rc		Return status code
+ */
+static int peerblk_decrypt_write ( struct peerdist_block *peerblk,
+				   const void *data, size_t len ) {
+	struct peerdist_block_decrypt *decrypt = peerblk->decrypt;
+	size_t frag_len;
+	int rc;
+
+	/* Write to each decryption buffer in turn */
+	for ( ; len ; decrypt++, data += frag_len, len -= frag_len ) {
+
+		/* Calculate length to use from this buffer */
+		frag_len = decrypt->len;
+		if ( frag_len > len )
+			frag_len = len;
+		if ( ! frag_len )
+			continue;
+
+		/* Write to this buffer */
+		if ( ( rc = xferbuf_write ( decrypt->xferbuf, decrypt->offset,
+					    data, frag_len ) ) != 0 )
+			return rc;
+
+		/* Update offset and length */
+		decrypt->offset += frag_len;
+		decrypt->len -= frag_len;
+	}
+
+	return 0;
+}
+
+/**
+ * Decrypt one chunk of PeerDist retrieval protocol data
+ *
+ * @v peerblk		PeerDist block download
+ */
+static void peerblk_decrypt ( struct peerdist_block *peerblk ) {
+	struct cipher_algorithm *cipher = peerblk->cipher;
+	struct digest_algorithm *digest = peerblk->digest;
+	struct xfer_buffer *xferbuf;
+	size_t cipher_len;
+	size_t digest_len;
+	void *data;
+	int rc;
+
+	/* Sanity check */
+	assert ( ( PEERBLK_DECRYPT_CHUNKSIZE % cipher->blocksize ) == 0 );
+
+	/* Get the underlying data transfer buffer */
+	xferbuf = xfer_buffer ( &peerblk->xfer );
+	if ( ! xferbuf ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d has no underlying data "
+		       "transfer buffer\n", peerblk, peerblk->segment,
+		       peerblk->block );
+		rc = -ENOTSUP;
+		goto err_xfer_buffer;
+	}
+	peerblk->decrypt[PEERBLK_DURING].xferbuf = xferbuf;
+
+	/* Calculate cipher and digest lengths */
+	cipher_len = PEERBLK_DECRYPT_CHUNKSIZE;
+	if ( cipher_len > peerblk->cipher_remaining )
+		cipher_len = peerblk->cipher_remaining;
+	digest_len = cipher_len;
+	if ( digest_len > peerblk->digest_remaining )
+		digest_len = peerblk->digest_remaining;
+	assert ( ( cipher_len & ( cipher->blocksize - 1 ) ) == 0 );
+
+	/* Allocate temporary data buffer */
+	data = malloc ( cipher_len );
+	if ( ! data ) {
+		rc = -ENOMEM;
+		goto err_alloc_data;
+	}
+
+	/* Read ciphertext */
+	if ( ( rc = peerblk_decrypt_read ( peerblk, data, cipher_len ) ) != 0 ){
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not read ciphertext: "
+		       "%s\n", peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err_read;
+	}
+
+	/* Decrypt data */
+	cipher_decrypt ( cipher, peerblk->cipherctx, data, data, cipher_len );
+
+	/* Add data to digest */
+	digest_update ( digest, peerblk->digestctx, data, digest_len );
+
+	/* Write plaintext */
+	if ( ( rc = peerblk_decrypt_write ( peerblk, data, cipher_len ) ) != 0){
+		DBGC ( peerblk, "PEERBLK %p %d.%d could not write plaintext: "
+		       "%s\n", peerblk, peerblk->segment, peerblk->block,
+		       strerror ( rc ) );
+		goto err_write;
+	}
+
+	/* Consume input */
+	peerblk->cipher_remaining -= cipher_len;
+	peerblk->digest_remaining -= digest_len;
+
+	/* Free temporary data buffer */
+	free ( data );
+
+	/* Continue processing until all input is consumed */
+	if ( peerblk->cipher_remaining )
+		return;
+
+	/* Complete download attempt */
+	peerblk_done ( peerblk, 0 );
+	return;
+
+ err_write:
+ err_read:
+	free ( data );
+ err_alloc_data:
+ err_xfer_buffer:
+	peerblk_done ( peerblk, rc );
+}
+
+/**
+ * Close PeerDist retrieval protocol block download attempt
+ *
+ * @v peerblk		PeerDist block download
+ * @v rc		Reason for close
+ */
+static void peerblk_retrieval_close ( struct peerdist_block *peerblk, int rc ) {
+	size_t buf_len;
+	size_t vrf_len;
+
+	/* Restart interface */
+	intf_restart ( &peerblk->retrieval, rc );
+
+	/* Fail immediately if we have an error */
+	if ( rc != 0 )
+		goto done;
+
+	/* Abort download attempt (for testing) if applicable */
+	if ( ( rc = inject_fault ( PEERBLK_ABORT_RATE ) ) != 0 )
+		goto done;
+
+	/* Parse message header */
+	if ( ( rc = peerblk_parse_header ( peerblk ) ) != 0 )
+		goto done;
+
+	/* Parse message segment and block details */
+	if ( ( rc = peerblk_parse_block ( peerblk, &buf_len ) ) != 0 )
+		goto done;
+
+	/* If the block was plaintext, then there is nothing more to do */
+	if ( ! peerblk->cipher )
+		goto done;
+
+	/* Parse message useless details */
+	if ( ( rc = peerblk_parse_useless ( peerblk, buf_len, &vrf_len ) ) != 0)
+		goto done;
+
+	/* Parse message initialisation vector details */
+	if ( ( rc = peerblk_parse_iv ( peerblk, buf_len, vrf_len ) ) != 0 )
+		goto done;
+
+	/* Fail if decryption length is not aligned to the cipher block size */
+	if ( peerblk->cipher_remaining & ( peerblk->cipher->blocksize - 1 ) ) {
+		DBGC ( peerblk, "PEERBLK %p %d.%d unaligned data length %zd\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       peerblk->cipher_remaining );
+		rc = -EPROTO;
+		goto done;
+	}
+
+	/* Stop the download attempt timer: there is no point in
+	 * timing out while decrypting.
+	 */
+	stop_timer ( &peerblk->timer );
+
+	/* Start decryption process */
+	process_add ( &peerblk->process );
+	return;
+
+ done:
+	/* Complete download attempt */
+	peerblk_done ( peerblk, rc );
+}
+
+/******************************************************************************
+ *
+ * Retry policy
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Handle PeerDist retry timer expiry
+ *
+ * @v timer		Retry timer
+ * @v over		Failure indicator
+ */
+static void peerblk_expired ( struct retry_timer *timer, int over __unused ) {
+	struct peerdist_block *peerblk =
+		container_of ( timer, struct peerdist_block, timer );
+	struct peerdisc_segment *segment = peerblk->discovery.segment;
+	struct peerdisc_peer *head;
+	unsigned long now = peerblk_timestamp();
+	const char *location;
+	int rc;
+
+	/* Profile discovery timeout, if applicable */
+	if ( ( peerblk->peer == NULL ) && ( timer->timeout != 0 ) ) {
+		profile_custom ( &peerblk_discovery_timeout_profiler,
+				 ( now - peerblk->started ) );
+		DBGC ( peerblk, "PEERBLK %p %d.%d discovery timed out after "
+		       "%ld ticks\n", peerblk, peerblk->segment,
+		       peerblk->block, timer->timeout );
+	}
+
+	/* Profile download timeout, if applicable */
+	if ( ( peerblk->peer != NULL ) && ( timer->timeout != 0 ) ) {
+		profile_custom ( &peerblk_attempt_timeout_profiler,
+				 ( now - peerblk->attempted ) );
+		DBGC ( peerblk, "PEERBLK %p %d.%d timed out after %ld ticks\n",
+		       peerblk, peerblk->segment, peerblk->block,
+		       timer->timeout );
+	}
+
+	/* Abort any current download attempt */
+	peerblk_reset ( peerblk, -ETIMEDOUT );
+
+	/* Record attempt start time */
+	peerblk->attempted = now;
+
+	/* If we have exceeded our maximum number of attempt cycles
+	 * (each cycle comprising a retrieval protocol download from
+	 * each peer in the list followed by a raw download from the
+	 * origin server), then abort the overall download.
+	 */
+	head = list_entry ( &segment->peers, struct peerdisc_peer, list );
+	if ( ( peerblk->peer == head ) &&
+	     ( ++peerblk->cycles >= PEERBLK_MAX_ATTEMPT_CYCLES ) ) {
+		rc = peerblk->rc;
+		assert ( rc != 0 );
+		goto err;
+	}
+
+	/* If we have not yet made any download attempts, then move to
+	 * the start of the peer list.
+	 */
+	if ( peerblk->peer == NULL )
+		peerblk->peer = head;
+
+	/* Attempt retrieval protocol download from next usable peer */
+	list_for_each_entry_continue ( peerblk->peer, &segment->peers, list ) {
+
+		/* Attempt retrieval protocol download from this peer */
+		location = peerblk->peer->location;
+		if ( ( rc = peerblk_retrieval_open ( peerblk,
+						     location ) ) != 0 ) {
+			/* Non-fatal: continue to try next peer */
+			continue;
+		}
+
+		/* Start download attempt timer */
+		peerblk->rc = -ETIMEDOUT;
+		start_timer_fixed ( &peerblk->timer,
+				    PEERBLK_RETRIEVAL_OPEN_TIMEOUT );
+		return;
+	}
+
+	/* Attempt raw download */
+	if ( ( rc = peerblk_raw_open ( peerblk ) ) != 0 )
+		goto err;
+
+	/* Start download attempt timer */
+	peerblk->rc = -ETIMEDOUT;
+	start_timer_fixed ( &peerblk->timer, PEERBLK_RAW_OPEN_TIMEOUT );
+	return;
+
+ err:
+	peerblk_close ( peerblk, rc );
+}
+
+/**
+ * Handle PeerDist peer discovery
+ *
+ * @v discovery		PeerDist discovery client
+ */
+static void peerblk_discovered ( struct peerdisc_client *discovery ) {
+	struct peerdist_block *peerblk =
+		container_of ( discovery, struct peerdist_block, discovery );
+	unsigned long now = peerblk_timestamp();
+
+	/* Do nothing unless we are still waiting for the initial
+	 * discovery timeout.
+	 */
+	if ( ( peerblk->peer != NULL ) || ( peerblk->timer.timeout == 0 ) )
+		return;
+
+	/* Schedule an immediate retry */
+	start_timer_nodelay ( &peerblk->timer );
+
+	/* Profile discovery success */
+	profile_custom ( &peerblk_discovery_success_profiler,
+			 ( now - peerblk->started ) );
+}
+
+/******************************************************************************
+ *
+ * Opener
+ *
+ ******************************************************************************
+ */
+
+/** PeerDist block download data transfer interface operations */
+static struct interface_operation peerblk_xfer_operations[] = {
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_close ),
+};
+
+/** PeerDist block download data transfer interface descriptor */
+static struct interface_descriptor peerblk_xfer_desc =
+	INTF_DESC ( struct peerdist_block, xfer, peerblk_xfer_operations );
+
+/** PeerDist block download raw data interface operations */
+static struct interface_operation peerblk_raw_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_block *, peerblk_raw_rx ),
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_raw_close ),
+};
+
+/** PeerDist block download raw data interface descriptor */
+static struct interface_descriptor peerblk_raw_desc =
+	INTF_DESC ( struct peerdist_block, raw, peerblk_raw_operations );
+
+/** PeerDist block download retrieval protocol interface operations */
+static struct interface_operation peerblk_retrieval_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_block *, peerblk_retrieval_rx ),
+	INTF_OP ( intf_close, struct peerdist_block *, peerblk_retrieval_close),
+};
+
+/** PeerDist block download retrieval protocol interface descriptor */
+static struct interface_descriptor peerblk_retrieval_desc =
+	INTF_DESC ( struct peerdist_block, retrieval,
+		    peerblk_retrieval_operations );
+
+/** PeerDist block download decryption process descriptor */
+static struct process_descriptor peerblk_process_desc =
+	PROC_DESC ( struct peerdist_block, process, peerblk_decrypt );
+
+/** PeerDist block download discovery operations */
+static struct peerdisc_client_operations peerblk_discovery_operations = {
+	.discovered = peerblk_discovered,
+};
+
+/**
+ * Open PeerDist block download
+ *
+ * @v xfer		Data transfer interface
+ * @v uri		Original URI
+ * @v info		Content information block
+ * @ret rc		Return status code
+ */
+int peerblk_open ( struct interface *xfer, struct uri *uri,
+		   struct peerdist_info_block *block ) {
+	const struct peerdist_info_segment *segment = block->segment;
+	const struct peerdist_info *info = segment->info;
+	struct digest_algorithm *digest = info->digest;
+	struct peerdist_block *peerblk;
+	unsigned long timeout;
+	size_t digestsize;
+	int rc;
+
+	/* Allocate and initialise structure */
+	peerblk = zalloc ( sizeof ( *peerblk ) + digest->ctxsize );
+	if ( ! peerblk ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	ref_init ( &peerblk->refcnt, peerblk_free );
+	intf_init ( &peerblk->xfer, &peerblk_xfer_desc, &peerblk->refcnt );
+	intf_init ( &peerblk->raw, &peerblk_raw_desc, &peerblk->refcnt );
+	intf_init ( &peerblk->retrieval, &peerblk_retrieval_desc,
+		    &peerblk->refcnt );
+	peerblk->uri = uri_get ( uri );
+	memcpy ( &peerblk->range, &block->range, sizeof ( peerblk->range ) );
+	memcpy ( &peerblk->trim, &block->trim, sizeof ( peerblk->trim ) );
+	peerblk->offset = ( block->trim.start - info->trim.start );
+	peerblk->digest = info->digest;
+	peerblk->digestsize = digestsize = info->digestsize;
+	peerblk->digestctx = ( ( ( void * ) peerblk ) + sizeof ( *peerblk ) );
+	peerblk->segment = segment->index;
+	memcpy ( peerblk->id, segment->id, sizeof ( peerblk->id ) );
+	memcpy ( peerblk->secret, segment->secret, sizeof ( peerblk->secret ) );
+	peerblk->block = block->index;
+	memcpy ( peerblk->hash, block->hash, sizeof ( peerblk->hash ) );
+	xferbuf_malloc_init ( &peerblk->buffer );
+	process_init_stopped ( &peerblk->process, &peerblk_process_desc,
+			       &peerblk->refcnt );
+	peerdisc_init ( &peerblk->discovery, &peerblk_discovery_operations );
+	timer_init ( &peerblk->timer, peerblk_expired, &peerblk->refcnt );
+	DBGC2 ( peerblk, "PEERBLK %p %d.%d id %02x%02x%02x%02x%02x..."
+		"%02x%02x%02x [%08zx,%08zx)", peerblk, peerblk->segment,
+		peerblk->block, peerblk->id[0], peerblk->id[1], peerblk->id[2],
+		peerblk->id[3], peerblk->id[4], peerblk->id[ digestsize - 3 ],
+		peerblk->id[ digestsize - 2 ], peerblk->id[ digestsize - 1 ],
+		peerblk->range.start, peerblk->range.end );
+	if ( ( peerblk->trim.start != peerblk->range.start ) ||
+	     ( peerblk->trim.end != peerblk->range.end ) ) {
+		DBGC2 ( peerblk, " covers [%08zx,%08zx)",
+			peerblk->trim.start, peerblk->trim.end );
+	}
+	DBGC2 ( peerblk, "\n" );
+
+	/* Open discovery */
+	if ( ( rc = peerdisc_open ( &peerblk->discovery, peerblk->id,
+				    peerblk->digestsize ) ) != 0 )
+		goto err_open_discovery;
+
+	/* Schedule a retry attempt either immediately (if we already
+	 * have some peers) or after the discovery timeout.
+	 */
+	timeout = ( list_empty ( &peerblk->discovery.segment->peers ) ?
+		    ( peerdisc_timeout_secs * TICKS_PER_SEC ) : 0 );
+	start_timer_fixed ( &peerblk->timer, timeout );
+
+	/* Record start time */
+	peerblk->started = peerblk_timestamp();
+
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( xfer, &peerblk->xfer );
+	ref_put ( &peerblk->refcnt );
+	return 0;
+
+ err_open_discovery:
+	peerblk_close ( peerblk, rc );
+ err_alloc:
+	return rc;
+}
diff --git a/qemu/roms/ipxe/src/net/peerdisc.c b/qemu/roms/ipxe/src/net/peerdisc.c
new file mode 100644
index 000000000..5b0e98911
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/peerdisc.c
@@ -0,0 +1,551 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <errno.h>
+#include <assert.h>
+#include <ipxe/xfer.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/open.h>
+#include <ipxe/tcpip.h>
+#include <ipxe/uuid.h>
+#include <ipxe/base16.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/timer.h>
+#include <ipxe/fault.h>
+#include <ipxe/pccrd.h>
+#include <ipxe/peerdisc.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol peer discovery
+ *
+ */
+
+/** List of discovery segments */
+static LIST_HEAD ( peerdisc_segments );
+
+/** Number of repeated discovery attempts */
+#define PEERDISC_REPEAT_COUNT 2
+
+/** Time between repeated discovery attempts */
+#define PEERDISC_REPEAT_TIMEOUT ( 1 * TICKS_PER_SEC )
+
+/** Default discovery timeout (in seconds) */
+#define PEERDISC_DEFAULT_TIMEOUT_SECS 2
+
+/** Recommended discovery timeout (in seconds)
+ *
+ * We reduce the recommended discovery timeout whenever a segment
+ * fails to discover any peers, and restore the default value whenever
+ * a valid discovery reply is received.  We continue to send discovery
+ * requests even if the recommended timeout is reduced to zero.
+ *
+ * This strategy is intended to minimise discovery delays when no
+ * peers are available on the network, while allowing downloads to
+ * quickly switch back to using PeerDist acceleration if new peers
+ * become available.
+ */
+unsigned int peerdisc_timeout_secs = PEERDISC_DEFAULT_TIMEOUT_SECS;
+
+static struct peerdisc_segment * peerdisc_find ( const char *id );
+static int peerdisc_discovered ( struct peerdisc_segment *segment,
+				 const char *location );
+
+/******************************************************************************
+ *
+ * Discovery sockets
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open all PeerDist discovery sockets
+ *
+ * @ret rc		Return status code
+ */
+static int peerdisc_socket_open ( void ) {
+	struct peerdisc_socket *socket;
+	int rc;
+
+	/* Open each socket */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS ) {
+		if ( ( rc = xfer_open_socket ( &socket->xfer, SOCK_DGRAM,
+					       &socket->address.sa,
+					       NULL ) ) != 0 ) {
+			DBGC ( socket, "PEERDISC %s could not open socket: "
+			       "%s\n", socket->name, strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	return 0;
+
+ err:
+	for_each_table_entry_continue_reverse ( socket, PEERDISC_SOCKETS )
+		intf_restart ( &socket->xfer, rc );
+	return rc;
+}
+
+/**
+ * Attempt to transmit PeerDist discovery requests on all sockets
+ *
+ * @v uuid		Message UUID string
+ * @v id		Segment identifier string
+ */
+static void peerdisc_socket_tx ( const char *uuid, const char *id ) {
+	struct peerdisc_socket *socket;
+	struct net_device *netdev;
+	struct xfer_metadata meta;
+	union {
+		struct sockaddr sa;
+		struct sockaddr_tcpip st;
+	} address;
+	char *request;
+	size_t len;
+	int rc;
+
+	/* Construct discovery request */
+	request = peerdist_discovery_request ( uuid, id );
+	if ( ! request )
+		goto err_request;
+	len = strlen ( request );
+
+	/* Initialise data transfer metadata */
+	memset ( &meta, 0, sizeof ( meta ) );
+	meta.dest = &address.sa;
+
+	/* Send message on each socket */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS ) {
+
+		/* Initialise socket address */
+		memcpy ( &address.sa, &socket->address.sa,
+			 sizeof ( address.sa ) );
+
+		/* Send message on each open network device */
+		for_each_netdev ( netdev ) {
+
+			/* Skip unopened network devices */
+			if ( ! netdev_is_open ( netdev ) )
+				continue;
+			address.st.st_scope_id = netdev->index;
+
+			/* Discard request (for test purposes) if applicable */
+			if ( inject_fault ( PEERDISC_DISCARD_RATE ) )
+				continue;
+
+			/* Transmit request */
+			if ( ( rc = xfer_deliver_raw_meta ( &socket->xfer,
+							    request, len,
+							    &meta ) ) != 0 ) {
+				DBGC ( socket, "PEERDISC %s could not transmit "
+				       "via %s: %s\n", socket->name,
+				       netdev->name, strerror ( rc ) );
+				/* Contine to try other net devices/sockets */
+				continue;
+			}
+		}
+	}
+
+	free ( request );
+ err_request:
+	return;
+}
+
+/**
+ * Handle received PeerDist discovery reply
+ *
+ * @v socket		PeerDist discovery socket
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peerdisc_socket_rx ( struct peerdisc_socket *socket,
+				struct io_buffer *iobuf,
+				struct xfer_metadata *meta __unused ) {
+	struct peerdist_discovery_reply reply;
+	struct peerdisc_segment *segment;
+	char *id;
+	char *location;
+	int rc;
+
+	/* Discard reply (for test purposes) if applicable */
+	if ( ( rc = inject_fault ( PEERDISC_DISCARD_RATE ) ) != 0 )
+		goto err;
+
+	/* Parse reply */
+	if ( ( rc = peerdist_discovery_reply ( iobuf->data, iob_len ( iobuf ),
+					       &reply ) ) != 0 ) {
+		DBGC ( socket, "PEERDISC %s could not parse reply: %s\n",
+		       socket->name, strerror ( rc ) );
+		DBGC_HDA ( socket, 0, iobuf->data, iob_len ( iobuf ) );
+		goto err;
+	}
+
+	/* Any kind of discovery reply indicates that there are active
+	 * peers on a local network, so restore the recommended
+	 * discovery timeout to its default value for future requests.
+	 */
+	if ( peerdisc_timeout_secs != PEERDISC_DEFAULT_TIMEOUT_SECS ) {
+		DBGC ( socket, "PEERDISC %s restoring timeout to %d seconds\n",
+		       socket->name, PEERDISC_DEFAULT_TIMEOUT_SECS );
+	}
+	peerdisc_timeout_secs = PEERDISC_DEFAULT_TIMEOUT_SECS;
+
+	/* Iterate over segment IDs */
+	for ( id = reply.ids ; *id ; id += ( strlen ( id ) + 1 /* NUL */ ) ) {
+
+		/* Find corresponding segment */
+		segment = peerdisc_find ( id );
+		if ( ! segment ) {
+			DBGC ( socket, "PEERDISC %s ignoring reply for %s\n",
+			       socket->name, id );
+			continue;
+		}
+
+		/* Report all discovered peer locations */
+		for ( location = reply.locations ; *location ;
+		      location += ( strlen ( location ) + 1 /* NUL */ ) ) {
+
+			/* Report discovered peer location */
+			if ( ( rc = peerdisc_discovered ( segment,
+							  location ) ) != 0 )
+				goto err;
+		}
+	}
+
+ err:
+	free_iob ( iobuf );
+	return rc;
+}
+
+/**
+ * Close all PeerDist discovery sockets
+ *
+ * @v rc		Reason for close
+ */
+static void peerdisc_socket_close ( int rc ) {
+	struct peerdisc_socket *socket;
+
+	/* Close all sockets */
+	for_each_table_entry ( socket, PEERDISC_SOCKETS )
+		intf_restart ( &socket->xfer, rc );
+}
+
+/** PeerDist discovery socket interface operations */
+static struct interface_operation peerdisc_socket_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdisc_socket *, peerdisc_socket_rx ),
+};
+
+/** PeerDist discovery socket interface descriptor */
+static struct interface_descriptor peerdisc_socket_desc =
+	INTF_DESC ( struct peerdisc_socket, xfer, peerdisc_socket_operations );
+
+/** PeerDist discovery IPv4 socket */
+struct peerdisc_socket peerdisc_socket_ipv4 __peerdisc_socket = {
+	.name = "IPv4",
+	.address = {
+		.sin = {
+			.sin_family = AF_INET,
+			.sin_port = htons ( PEERDIST_DISCOVERY_PORT ),
+			.sin_addr.s_addr = htonl ( PEERDIST_DISCOVERY_IPV4 ),
+		},
+	},
+	.xfer = INTF_INIT ( peerdisc_socket_desc ),
+};
+
+/** PeerDist discovery IPv6 socket */
+struct peerdisc_socket peerdisc_socket_ipv6 __peerdisc_socket = {
+	.name = "IPv6",
+	.address = {
+		.sin6 = {
+			.sin6_family = AF_INET6,
+			.sin6_port = htons ( PEERDIST_DISCOVERY_PORT ),
+			.sin6_addr.s6_addr = PEERDIST_DISCOVERY_IPV6,
+		},
+	},
+	.xfer = INTF_INIT ( peerdisc_socket_desc ),
+};
+
+/******************************************************************************
+ *
+ * Discovery segments
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Free PeerDist discovery segment
+ *
+ * @v refcnt		Reference count
+ */
+static void peerdisc_free ( struct refcnt *refcnt ) {
+	struct peerdisc_segment *segment =
+		container_of ( refcnt, struct peerdisc_segment, refcnt );
+	struct peerdisc_peer *peer;
+	struct peerdisc_peer *tmp;
+
+	/* Free all discovered peers */
+	list_for_each_entry_safe ( peer, tmp, &segment->peers, list ) {
+		list_del ( &peer->list );
+		free ( peer );
+	}
+
+	/* Free segment */
+	free ( segment );
+}
+
+/**
+ * Find PeerDist discovery segment
+ *
+ * @v id		Segment ID
+ * @ret segment		PeerDist discovery segment, or NULL if not found
+ */
+static struct peerdisc_segment * peerdisc_find ( const char *id ) {
+	struct peerdisc_segment *segment;
+
+	/* Look for a matching segment */
+	list_for_each_entry ( segment, &peerdisc_segments, list ) {
+		if ( strcmp ( id, segment->id ) == 0 )
+			return segment;
+	}
+
+	return NULL;
+}
+
+/**
+ * Add discovered PeerDist peer
+ *
+ * @v segment		PeerDist discovery segment
+ * @v location		Peer location
+ * @ret rc		Return status code
+ */
+static int peerdisc_discovered ( struct peerdisc_segment *segment,
+				 const char *location ) {
+	struct peerdisc_peer *peer;
+	struct peerdisc_client *peerdisc;
+	struct peerdisc_client *tmp;
+
+	/* Ignore duplicate peers */
+	list_for_each_entry ( peer, &segment->peers, list ) {
+		if ( strcmp ( peer->location, location ) == 0 ) {
+			DBGC2 ( segment, "PEERDISC %p duplicate %s\n",
+				segment, location );
+			return 0;
+		}
+	}
+	DBGC2 ( segment, "PEERDISC %p discovered %s\n", segment, location );
+
+	/* Allocate and initialise structure */
+	peer = zalloc ( sizeof ( *peer ) + strlen ( location ) + 1 /* NUL */ );
+	if ( ! peer )
+		return -ENOMEM;
+	strcpy ( peer->location, location );
+
+	/* Add to end of list of peers */
+	list_add_tail ( &peer->list, &segment->peers );
+
+	/* Notify all clients */
+	list_for_each_entry_safe ( peerdisc, tmp, &segment->clients, list )
+		peerdisc->op->discovered ( peerdisc );
+
+	return 0;
+}
+
+/**
+ * Handle discovery timer expiry
+ *
+ * @v timer		Discovery timer
+ * @v over		Failure indicator
+ */
+static void peerdisc_expired ( struct retry_timer *timer, int over __unused ) {
+	struct peerdisc_segment *segment =
+		container_of ( timer, struct peerdisc_segment, timer );
+
+	/* Attempt to transmit discovery requests */
+	peerdisc_socket_tx ( segment->uuid, segment->id );
+
+	/* Schedule next transmission, if applicable */
+	if ( timer->count < PEERDISC_REPEAT_COUNT )
+		start_timer_fixed ( &segment->timer, PEERDISC_REPEAT_TIMEOUT );
+}
+
+/**
+ * Create PeerDist discovery segment
+ *
+ * @v id		Segment ID
+ * @ret segment		PeerDist discovery segment, or NULL on error
+ */
+static struct peerdisc_segment * peerdisc_create ( const char *id ) {
+	struct peerdisc_segment *segment;
+	union {
+		union uuid uuid;
+		uint32_t dword[ sizeof ( union uuid ) / sizeof ( uint32_t ) ];
+	} random_uuid;
+	size_t uuid_len;
+	size_t id_len;
+	char *uuid;
+	char *uuid_copy;
+	char *id_copy;
+	unsigned int i;
+
+	/* Generate a random message UUID.  This does not require high
+	 * quality randomness.
+	 */
+	for ( i = 0 ; i < ( sizeof ( random_uuid.dword ) /
+			    sizeof ( random_uuid.dword[0] ) ) ; i++ )
+		random_uuid.dword[i] = random();
+	uuid = uuid_ntoa ( &random_uuid.uuid );
+
+	/* Calculate string lengths */
+	id_len = ( strlen ( id ) + 1 /* NUL */ );
+	uuid_len = ( strlen ( uuid ) + 1 /* NUL */ );
+
+	/* Allocate and initialise structure */
+	segment = zalloc ( sizeof ( *segment ) + id_len + uuid_len );
+	if ( ! segment )
+		return NULL;
+	id_copy = ( ( ( void * ) segment ) + sizeof ( *segment ) );
+	memcpy ( id_copy, id, id_len );
+	uuid_copy = ( ( ( void * ) id_copy ) + id_len );
+	memcpy ( uuid_copy, uuid, uuid_len );
+	ref_init ( &segment->refcnt, peerdisc_free );
+	segment->id = id_copy;
+	segment->uuid = uuid_copy;
+	INIT_LIST_HEAD ( &segment->peers );
+	INIT_LIST_HEAD ( &segment->clients );
+	timer_init ( &segment->timer, peerdisc_expired, &segment->refcnt );
+	DBGC2 ( segment, "PEERDISC %p discovering %s\n", segment, segment->id );
+
+	/* Start discovery timer */
+	start_timer_nodelay ( &segment->timer );
+
+	/* Add to list of segments, transfer reference to list, and return */
+	list_add_tail ( &segment->list, &peerdisc_segments );
+	return segment;
+}
+
+/**
+ * Destroy PeerDist discovery segment
+ *
+ * @v segment		PeerDist discovery segment
+ */
+static void peerdisc_destroy ( struct peerdisc_segment *segment ) {
+
+	/* Sanity check */
+	assert ( list_empty ( &segment->clients ) );
+
+	/* Stop timer */
+	stop_timer ( &segment->timer );
+
+	/* Remove from list of segments and drop list's reference */
+	list_del ( &segment->list );
+	ref_put ( &segment->refcnt );
+}
+
+/******************************************************************************
+ *
+ * Discovery clients
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Open PeerDist discovery client
+ *
+ * @v peerdisc		PeerDist discovery client
+ * @v id		Segment ID
+ * @v len		Length of segment ID
+ * @ret rc		Return status code
+ */
+int peerdisc_open ( struct peerdisc_client *peerdisc, const void *id,
+		    size_t len ) {
+	struct peerdisc_segment *segment;
+	char id_string[ base16_encoded_len ( len ) + 1 /* NUL */ ];
+	char *id_chr;
+	int rc;
+
+	/* Construct ID string */
+	base16_encode ( id, len, id_string, sizeof ( id_string ) );
+	for ( id_chr = id_string ; *id_chr ; id_chr++ )
+		*id_chr = toupper ( *id_chr );
+
+	/* Sanity check */
+	assert ( peerdisc->segment == NULL );
+
+	/* Open socket if this is the first segment */
+	if ( list_empty ( &peerdisc_segments ) &&
+	     ( ( rc = peerdisc_socket_open() ) != 0 ) )
+		return rc;
+
+	/* Find or create segment */
+	if ( ! ( ( segment = peerdisc_find ( id_string ) ) ||
+		 ( segment = peerdisc_create ( id_string ) ) ) )
+		return -ENOMEM;
+
+	/* Add to list of clients */
+	ref_get ( &segment->refcnt );
+	peerdisc->segment = segment;
+	list_add_tail ( &peerdisc->list, &segment->clients );
+
+	return 0;
+}
+
+/**
+ * Close PeerDist discovery client
+ *
+ * @v peerdisc		PeerDist discovery client
+ */
+void peerdisc_close ( struct peerdisc_client *peerdisc ) {
+	struct peerdisc_segment *segment = peerdisc->segment;
+
+	/* Ignore if discovery is already closed */
+	if ( ! segment )
+		return;
+
+	/* If no peers were discovered, reduce the recommended
+	 * discovery timeout to minimise delays on future requests.
+	 */
+	if ( list_empty ( &segment->peers ) && peerdisc_timeout_secs ) {
+		peerdisc_timeout_secs--;
+		DBGC ( segment, "PEERDISC %p reducing timeout to %d "
+		       "seconds\n", peerdisc, peerdisc_timeout_secs );
+	}
+
+	/* Remove from list of clients */
+	peerdisc->segment = NULL;
+	list_del ( &peerdisc->list );
+	ref_put ( &segment->refcnt );
+
+	/* If this was the last clients, destroy the segment */
+	if ( list_empty ( &segment->clients ) )
+		peerdisc_destroy ( segment );
+
+	/* If there are no more segments, close the socket */
+	if ( list_empty ( &peerdisc_segments ) )
+		peerdisc_socket_close ( 0 );
+}
diff --git a/qemu/roms/ipxe/src/net/peerdist.c b/qemu/roms/ipxe/src/net/peerdist.c
new file mode 100644
index 000000000..48933f951
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/peerdist.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdio.h>
+#include <ipxe/http.h>
+#include <ipxe/peermux.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol
+ *
+ * This is quite possibly the ugliest protocol I have ever had the
+ * misfortune to encounter, and I've encountered multicast TFTP.
+ */
+
+/**
+ * Check whether or not to support PeerDist encoding for this request
+ *
+ * @v http		HTTP transaction
+ * @ret supported	PeerDist encoding is supported for this request
+ */
+static int http_peerdist_supported ( struct http_transaction *http ) {
+
+	/* Support PeerDist encoding only if we can directly access an
+	 * underlying data transfer buffer.  Direct access is required
+	 * in order to support decryption of data received via the
+	 * retrieval protocol (which provides the AES initialisation
+	 * vector only after all of the encrypted data has been
+	 * received).
+	 *
+	 * This test simultaneously ensures that we do not attempt to
+	 * use PeerDist encoding on a request which is itself a
+	 * PeerDist individual block download, since the individual
+	 * block downloads do not themselves provide direct access to
+	 * an underlying data transfer buffer.
+	 */
+	return ( xfer_buffer ( &http->xfer ) != NULL );
+}
+
+/**
+ * Format HTTP "X-P2P-PeerDist" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_p2p_peerdist ( struct http_transaction *http,
+				      char *buf, size_t len ) {
+	int supported = http_peerdist_supported ( http );
+	int missing;
+
+	/* PeerDist wants us to inform the server whenever we make a
+	 * request for data that was missing from local peers
+	 * (presumably for statistical purposes only).  We use the
+	 * heuristic of assuming that the combination of "this request
+	 * may not itself use PeerDist content encoding" and "this is
+	 * a range request" probably indicates that we are making a
+	 * PeerDist block raw range request for missing data.
+	 */
+	missing = ( http->request.range.len && ( ! supported ) );
+
+	/* Omit header if PeerDist encoding is not supported and we
+	 * are not reporting a missing data request.
+	 */
+	if ( ! ( supported || missing ) )
+		return 0;
+
+	/* Construct header */
+	return snprintf ( buf, len, "Version=1.1%s",
+			  ( missing ? ", MissingDataRequest=true" : "" ) );
+}
+
+/** HTTP "X-P2P-PeerDist" header */
+struct http_request_header http_request_p2p_peerdist __http_request_header = {
+	.name = "X-P2P-PeerDist",
+	.format = http_format_p2p_peerdist,
+};
+
+/**
+ * Format HTTP "X-P2P-PeerDistEx" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_p2p_peerdistex ( struct http_transaction *http,
+					char *buf, size_t len ) {
+	int supported = http_peerdist_supported ( http );
+
+	/* Omit header if PeerDist encoding is not supported */
+	if ( ! supported )
+		return 0;
+
+	/* Construct header */
+	return snprintf ( buf, len, ( "MinContentInformation=1.0, "
+				      "MaxContentInformation=2.0" ) );
+}
+
+/** HTTP "X-P2P-PeerDist" header */
+struct http_request_header http_request_p2p_peerdistex __http_request_header = {
+	.name = "X-P2P-PeerDistEx",
+	.format = http_format_p2p_peerdistex,
+};
+
+/**
+ * Initialise PeerDist content encoding
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_peerdist_init ( struct http_transaction *http ) {
+
+	return peermux_filter ( &http->content, &http->transfer, http->uri );
+}
+
+/** PeerDist HTTP content encoding */
+struct http_content_encoding peerdist_encoding __http_content_encoding = {
+	.name = "peerdist",
+	.supported = http_peerdist_supported,
+	.init = http_peerdist_init,
+};
diff --git a/qemu/roms/ipxe/src/net/peermux.c b/qemu/roms/ipxe/src/net/peermux.c
new file mode 100644
index 000000000..634c69992
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/peermux.c
@@ -0,0 +1,387 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdlib.h>
+#include <errno.h>
+#include <ipxe/uri.h>
+#include <ipxe/xferbuf.h>
+#include <ipxe/peerblk.h>
+#include <ipxe/peermux.h>
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval (PeerDist) protocol multiplexer
+ *
+ */
+
+/**
+ * Free PeerDist download multiplexer
+ *
+ * @v refcnt		Reference count
+ */
+static void peermux_free ( struct refcnt *refcnt ) {
+	struct peerdist_multiplexer *peermux =
+		container_of ( refcnt, struct peerdist_multiplexer, refcnt );
+
+	uri_put ( peermux->uri );
+	xferbuf_free ( &peermux->buffer );
+	free ( peermux );
+}
+
+/**
+ * Close PeerDist download multiplexer
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v rc		Reason for close
+ */
+static void peermux_close ( struct peerdist_multiplexer *peermux, int rc ) {
+	unsigned int i;
+
+	/* Stop block download initiation process */
+	process_del ( &peermux->process );
+
+	/* Shut down all block downloads */
+	for ( i = 0 ; i < PEERMUX_MAX_BLOCKS ; i++ )
+		intf_shutdown ( &peermux->block[i].xfer, rc );
+
+	/* Shut down all other interfaces (which may be connected to
+	 * the same object).
+	 */
+	intf_nullify ( &peermux->info ); /* avoid potential loops */
+	intf_shutdown ( &peermux->xfer, rc );
+	intf_shutdown ( &peermux->info, rc );
+}
+
+/**
+ * Receive content information
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peermux_info_deliver ( struct peerdist_multiplexer *peermux,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Add data to buffer */
+	if ( ( rc = xferbuf_deliver ( &peermux->buffer, iobuf, meta ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	peermux_close ( peermux, rc );
+	return rc;
+}
+
+/**
+ * Close content information interface
+ *
+ * @v peermux		PeerDist download multiplexer
+ * @v rc		Reason for close
+ */
+static void peermux_info_close ( struct peerdist_multiplexer *peermux, int rc ){
+	struct peerdist_info *info = &peermux->cache.info;
+	size_t len;
+
+	/* Terminate download on error */
+	if ( rc != 0 )
+		goto err;
+
+	/* Successfully closing the content information interface
+	 * indicates that the content information has been fully
+	 * received, and initiates the actual PeerDist download.
+	 */
+
+	/* Shut down content information interface */
+	intf_shutdown ( &peermux->info, rc );
+
+	/* Parse content information */
+	if ( ( rc = peerdist_info ( info->raw.data, peermux->buffer.len,
+				    info ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not parse content info: %s\n",
+		       peermux, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Notify recipient of total download size */
+	len = ( info->trim.end - info->trim.start );
+	if ( ( rc = xfer_seek ( &peermux->xfer, len ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not presize buffer: %s\n",
+		       peermux, strerror ( rc ) );
+		goto err;
+	}
+	xfer_seek ( &peermux->xfer, 0 );
+
+	/* Start block download process */
+	process_add ( &peermux->process );
+
+	return;
+
+ err:
+	peermux_close ( peermux, rc );
+}
+
+/**
+ * Initiate multiplexed block download
+ *
+ * @v peermux		PeerDist download multiplexer
+ */
+static void peermux_step ( struct peerdist_multiplexer *peermux ) {
+	struct peerdist_info *info = &peermux->cache.info;
+	struct peerdist_info_segment *segment = &peermux->cache.segment;
+	struct peerdist_info_block *block = &peermux->cache.block;
+	struct peerdist_multiplexed_block *peermblk;
+	unsigned int next_segment;
+	unsigned int next_block;
+	int rc;
+
+	/* Stop initiation process if all block downloads are busy */
+	peermblk = list_first_entry ( &peermux->idle,
+				      struct peerdist_multiplexed_block, list );
+	if ( ! peermblk ) {
+		process_del ( &peermux->process );
+		return;
+	}
+
+	/* Increment block index */
+	next_block = ( block->index + 1 );
+
+	/* Move to first/next segment, if applicable */
+	if ( next_block >= segment->blocks ) {
+
+		/* Reset block index */
+		next_block = 0;
+
+		/* Calculate segment index */
+		next_segment = ( segment->info ? ( segment->index + 1 ) : 0 );
+
+		/* If we have finished all segments and have no
+		 * remaining block downloads, then we are finished.
+		 */
+		if ( next_segment >= info->segments ) {
+			process_del ( &peermux->process );
+			if ( list_empty ( &peermux->busy ) )
+				peermux_close ( peermux, 0 );
+			return;
+		}
+
+		/* Get content information segment */
+		if ( ( rc = peerdist_info_segment ( info, segment,
+						    next_segment ) ) != 0 ) {
+			DBGC ( peermux, "PEERMUX %p could not get segment %d "
+			       "information: %s\n", peermux, next_segment,
+			       strerror ( rc ) );
+			goto err;
+		}
+	}
+
+	/* Get content information block */
+	if ( ( rc = peerdist_info_block ( segment, block, next_block ) ) != 0 ){
+		DBGC ( peermux, "PEERMUX %p could not get segment %d block "
+		       "%d information: %s\n", peermux, segment->index,
+		       next_block, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Ignore block if it lies entirely outside the trimmed range */
+	if ( block->trim.start == block->trim.end ) {
+		DBGC ( peermux, "PEERMUX %p skipping segment %d block %d\n",
+		       peermux, segment->index, block->index );
+		return;
+	}
+
+	/* Start downloading this block */
+	if ( ( rc = peerblk_open ( &peermblk->xfer, peermux->uri,
+				   block ) ) != 0 ) {
+		DBGC ( peermux, "PEERMUX %p could not start download for "
+		       "segment %d block %d: %s\n", peermux, segment->index,
+		       block->index, strerror ( rc ) );
+		goto err;
+	}
+
+	/* Move to list of busy block downloads */
+	list_del ( &peermblk->list );
+	list_add_tail ( &peermblk->list, &peermux->busy );
+
+	return;
+
+ err:
+	peermux_close ( peermux, rc );
+}
+
+/**
+ * Receive data from multiplexed block download
+ *
+ * @v peermblk		PeerDist multiplexed block download
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ * @ret rc		Return status code
+ */
+static int peermux_block_deliver ( struct peerdist_multiplexed_block *peermblk,
+				   struct io_buffer *iobuf,
+				   struct xfer_metadata *meta ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* Sanity check: all block downloads must use absolute
+	 * positions for all deliveries, since they run concurrently.
+	 */
+	assert ( meta->flags & XFER_FL_ABS_OFFSET );
+
+	/* We can't use a simple passthrough interface descriptor,
+	 * since there are multiple block download interfaces.
+	 */
+	return xfer_deliver ( &peermux->xfer, iob_disown ( iobuf ), meta );
+}
+
+/**
+ * Get multiplexed block download underlying data transfer buffer
+ *
+ * @v peermblk		PeerDist multiplexed download block
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+peermux_block_buffer ( struct peerdist_multiplexed_block *peermblk ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* We can't use a simple passthrough interface descriptor,
+	 * since there are multiple block download interfaces.
+	 */
+	return xfer_buffer ( &peermux->xfer );
+}
+
+/**
+ * Close multiplexed block download
+ *
+ * @v peermblk		PeerDist multiplexed block download
+ * @v rc		Reason for close
+ */
+static void peermux_block_close ( struct peerdist_multiplexed_block *peermblk,
+				  int rc ) {
+	struct peerdist_multiplexer *peermux = peermblk->peermux;
+
+	/* Move to list of idle downloads */
+	list_del ( &peermblk->list );
+	list_add_tail ( &peermblk->list, &peermux->idle );
+
+	/* If any error occurred, terminate the whole multiplexer */
+	if ( rc != 0 ) {
+		peermux_close ( peermux, rc );
+		return;
+	}
+
+	/* Restart data transfer interface */
+	intf_restart ( &peermblk->xfer, rc );
+
+	/* Restart block download initiation process */
+	process_add ( &peermux->process );
+}
+
+/** Data transfer interface operations */
+static struct interface_operation peermux_xfer_operations[] = {
+	INTF_OP ( intf_close, struct peerdist_multiplexer *, peermux_close ),
+};
+
+/** Data transfer interface descriptor */
+static struct interface_descriptor peermux_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct peerdist_multiplexer, xfer,
+			     peermux_xfer_operations, info );
+
+/** Content information interface operations */
+static struct interface_operation peermux_info_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_multiplexer *,
+		  peermux_info_deliver ),
+	INTF_OP ( intf_close, struct peerdist_multiplexer *,
+		  peermux_info_close ),
+};
+
+/** Content information interface descriptor */
+static struct interface_descriptor peermux_info_desc =
+	INTF_DESC_PASSTHRU ( struct peerdist_multiplexer, info,
+			     peermux_info_operations, xfer );
+
+/** Block download data transfer interface operations */
+static struct interface_operation peermux_block_operations[] = {
+	INTF_OP ( xfer_deliver, struct peerdist_multiplexed_block *,
+		  peermux_block_deliver ),
+	INTF_OP ( xfer_buffer, struct peerdist_multiplexed_block *,
+		  peermux_block_buffer ),
+	INTF_OP ( intf_close, struct peerdist_multiplexed_block *,
+		  peermux_block_close ),
+};
+
+/** Block download data transfer interface descriptor */
+static struct interface_descriptor peermux_block_desc =
+	INTF_DESC ( struct peerdist_multiplexed_block, xfer,
+		    peermux_block_operations );
+
+/** Block download initiation process descriptor */
+static struct process_descriptor peermux_process_desc =
+	PROC_DESC ( struct peerdist_multiplexer, process, peermux_step );
+
+/**
+ * Add PeerDist content-encoding filter
+ *
+ * @v xfer		Data transfer interface
+ * @v info		Content information interface
+ * @v uri		Original URI
+ * @ret rc		Return status code
+ */
+int peermux_filter ( struct interface *xfer, struct interface *info,
+		     struct uri *uri ) {
+	struct peerdist_multiplexer *peermux;
+	struct peerdist_multiplexed_block *peermblk;
+	unsigned int i;
+
+	/* Allocate and initialise structure */
+	peermux = zalloc ( sizeof ( *peermux ) );
+	if ( ! peermux )
+		return -ENOMEM;
+	ref_init ( &peermux->refcnt, peermux_free );
+	intf_init ( &peermux->xfer, &peermux_xfer_desc, &peermux->refcnt );
+	intf_init ( &peermux->info, &peermux_info_desc, &peermux->refcnt );
+	peermux->uri = uri_get ( uri );
+	xferbuf_umalloc_init ( &peermux->buffer,
+			       &peermux->cache.info.raw.data );
+	process_init_stopped ( &peermux->process, &peermux_process_desc,
+			       &peermux->refcnt );
+	INIT_LIST_HEAD ( &peermux->busy );
+	INIT_LIST_HEAD ( &peermux->idle );
+	for ( i = 0 ; i < PEERMUX_MAX_BLOCKS ; i++ ) {
+		peermblk = &peermux->block[i];
+		peermblk->peermux = peermux;
+		list_add_tail ( &peermblk->list, &peermux->idle );
+		intf_init ( &peermblk->xfer, &peermux_block_desc,
+			    &peermux->refcnt );
+	}
+
+	/* Attach to parent interfaces, mortalise self, and return */
+	intf_plug_plug ( &peermux->xfer, xfer );
+	intf_plug_plug ( &peermux->info, info );
+	ref_put ( &peermux->refcnt );
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/net/ping.c b/qemu/roms/ipxe/src/net/ping.c
index d9da87ade..3f4fa5c11 100644
--- a/qemu/roms/ipxe/src/net/ping.c
+++ b/qemu/roms/ipxe/src/net/ping.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <string.h>
diff --git a/qemu/roms/ipxe/src/net/rarp.c b/qemu/roms/ipxe/src/net/rarp.c
index 371145015..c194a404f 100644
--- a/qemu/roms/ipxe/src/net/rarp.c
+++ b/qemu/roms/ipxe/src/net/rarp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <byteswap.h>
diff --git a/qemu/roms/ipxe/src/net/retry.c b/qemu/roms/ipxe/src/net/retry.c
index 8f210bdcc..734567be5 100644
--- a/qemu/roms/ipxe/src/net/retry.c
+++ b/qemu/roms/ipxe/src/net/retry.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/timer.h>
@@ -35,7 +39,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  * This implementation of the timer is designed to satisfy RFC 2988
  * and therefore be usable as a TCP retransmission timer.
- *
  * 
  */
 
@@ -49,47 +52,59 @@ FILE_LICENCE ( GPL2_OR_LATER );
 static LIST_HEAD ( timers );
 
 /**
- * Start timer
+ * Start timer with a specified timeout
  *
  * @v timer		Retry timer
+ * @v timeout		Timeout, in ticks
  *
- * This starts the timer running with the current timeout value.  If
+ * This starts the timer running with the specified timeout value.  If
  * stop_timer() is not called before the timer expires, the timer will
  * be stopped and the timer's callback function will be called.
  */
-void start_timer ( struct retry_timer *timer ) {
+void start_timer_fixed ( struct retry_timer *timer, unsigned long timeout ) {
+
+	/* Add to list of running timers (if applicable) */
 	if ( ! timer->running ) {
 		list_add ( &timer->list, &timers );
 		ref_get ( timer->refcnt );
+		timer->running = 1;
 	}
+
+	/* Record start time */
 	timer->start = currticks();
-	timer->running = 1;
-
-	/* 0 means "use default timeout" */
-	if ( timer->min_timeout == 0 )
-		timer->min_timeout = DEFAULT_MIN_TIMEOUT;
-	/* We must never be less than MIN_TIMEOUT under any circumstances */
-	if ( timer->min_timeout < MIN_TIMEOUT )
-		timer->min_timeout = MIN_TIMEOUT;
-	/* Honor user-specified minimum timeout */
-	if ( timer->timeout < timer->min_timeout )
-		timer->timeout = timer->min_timeout;
-
-	DBG2 ( "Timer %p started at time %ld (expires at %ld)\n",
-	       timer, timer->start, ( timer->start + timer->timeout ) );
+
+	/* Record timeout */
+	timer->timeout = timeout;
+
+	DBGC2 ( timer, "Timer %p started at time %ld (expires at %ld)\n",
+		timer, timer->start, ( timer->start + timer->timeout ) );
 }
 
 /**
- * Start timer with a specified fixed timeout
+ * Start timer
  *
  * @v timer		Retry timer
- * @v timeout		Timeout, in ticks
+ *
+ * This starts the timer running with the current timeout value
+ * (rounded up to the minimum timeout value).  If stop_timer() is not
+ * called before the timer expires, the timer will be stopped and the
+ * timer's callback function will be called.
  */
-void start_timer_fixed ( struct retry_timer *timer, unsigned long timeout ) {
-	start_timer ( timer );
-	timer->timeout = timeout;
-	DBG2 ( "Timer %p expiry time changed to %ld\n",
-	       timer, ( timer->start + timer->timeout ) );
+void start_timer ( struct retry_timer *timer ) {
+	unsigned long timeout = timer->timeout;
+	unsigned long min;
+
+	/* Calculate minimum timeout */
+	min = ( timer->min ? timer->min : DEFAULT_MIN_TIMEOUT );
+	if ( min < MIN_TIMEOUT )
+		min = MIN_TIMEOUT;
+
+	/* Ensure timeout is at least the minimum */
+	if ( timeout < min )
+		timeout = min;
+
+	/* Start timer with this timeout */
+	start_timer_fixed ( timer, timeout );
 }
 
 /**
@@ -111,8 +126,8 @@ void stop_timer ( struct retry_timer *timer ) {
 	list_del ( &timer->list );
 	runtime = ( now - timer->start );
 	timer->running = 0;
-	DBG2 ( "Timer %p stopped at time %ld (ran for %ld)\n",
-	       timer, now, runtime );
+	DBGC2 ( timer, "Timer %p stopped at time %ld (ran for %ld)\n",
+		timer, now, runtime );
 
 	/* Update timer.  Variables are:
 	 *
@@ -135,8 +150,8 @@ void stop_timer ( struct retry_timer *timer ) {
 		timer->timeout -= ( timer->timeout >> 3 );
 		timer->timeout += ( runtime >> 1 );
 		if ( timer->timeout != old_timeout ) {
-			DBG ( "Timer %p timeout updated to %ld\n",
-			      timer, timer->timeout );
+			DBGC ( timer, "Timer %p timeout updated to %ld\n",
+			       timer, timer->timeout );
 		}
 	}
 
@@ -150,11 +165,12 @@ void stop_timer ( struct retry_timer *timer ) {
  */
 static void timer_expired ( struct retry_timer *timer ) {
 	struct refcnt *refcnt = timer->refcnt;
+	unsigned long max = ( timer->max ? timer->max : DEFAULT_MAX_TIMEOUT );
 	int fail;
 
 	/* Stop timer without performing RTT calculations */
-	DBG2 ( "Timer %p stopped at time %ld on expiry\n",
-	       timer, currticks() );
+	DBGC2 ( timer, "Timer %p stopped at time %ld on expiry\n",
+		timer, currticks() );
 	assert ( timer->running );
 	list_del ( &timer->list );
 	timer->running = 0;
@@ -162,12 +178,10 @@ static void timer_expired ( struct retry_timer *timer ) {
 
 	/* Back off the timeout value */
 	timer->timeout <<= 1;
-	if ( timer->max_timeout == 0 ) /* 0 means "use default timeout" */
-		timer->max_timeout = DEFAULT_MAX_TIMEOUT;
-	if ( ( fail = ( timer->timeout > timer->max_timeout ) ) )
-		timer->timeout = timer->max_timeout;
-	DBG ( "Timer %p timeout backed off to %ld\n",
-	      timer, timer->timeout );
+	if ( ( fail = ( timer->timeout > max ) ) )
+		timer->timeout = max;
+	DBGC ( timer, "Timer %p timeout backed off to %ld\n",
+	       timer, timer->timeout );
 
 	/* Call expiry callback */
 	timer->expired ( timer, fail );
diff --git a/qemu/roms/ipxe/src/net/rndis.c b/qemu/roms/ipxe/src/net/rndis.c
new file mode 100644
index 000000000..8c4fe8b30
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/rndis.c
@@ -0,0 +1,1052 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Remote Network Driver Interface Specification
+ *
+ */
+
+#include <unistd.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/device.h>
+#include <ipxe/rndis.h>
+
+/**
+ * Allocate I/O buffer
+ *
+ * @v len		Length
+ * @ret iobuf		I/O buffer, or NULL
+ */
+static struct io_buffer * rndis_alloc_iob ( size_t len ) {
+	struct rndis_header *header;
+	struct io_buffer *iobuf;
+
+	/* Allocate I/O buffer and reserve space */
+	iobuf = alloc_iob ( sizeof ( *header ) + len );
+	if ( iobuf )
+		iob_reserve ( iobuf, sizeof ( *header ) );
+
+	return iobuf;
+}
+
+/**
+ * Wait for completion
+ *
+ * @v rndis		RNDIS device
+ * @v wait_id		Request ID
+ * @ret rc		Return status code
+ */
+static int rndis_wait ( struct rndis_device *rndis, unsigned int wait_id ) {
+	unsigned int i;
+
+	/* Record query ID */
+	rndis->wait_id = wait_id;
+
+	/* Wait for operation to complete */
+	for ( i = 0 ; i < RNDIS_MAX_WAIT_MS ; i++ ) {
+
+		/* Check for completion */
+		if ( ! rndis->wait_id )
+			return rndis->wait_rc;
+
+		/* Poll RNDIS device */
+		rndis->op->poll ( rndis );
+
+		/* Delay for 1ms */
+		mdelay ( 1 );
+	}
+
+	DBGC ( rndis, "RNDIS %s timed out waiting for ID %#08x\n",
+	       rndis->name, wait_id );
+	return -ETIMEDOUT;
+}
+
+/**
+ * Transmit message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v type		Message type
+ * @ret rc		Return status code
+ */
+static int rndis_tx_message ( struct rndis_device *rndis,
+			      struct io_buffer *iobuf, unsigned int type ) {
+	struct rndis_header *header;
+	int rc;
+
+	/* Prepend RNDIS header */
+	header = iob_push ( iobuf, sizeof ( *header ) );
+	header->type = cpu_to_le32 ( type );
+	header->len = cpu_to_le32 ( iob_len ( iobuf ) );
+
+	/* Transmit message */
+	if ( ( rc = rndis->op->transmit ( rndis, iobuf ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not transmit: %s\n",
+		       rndis->name, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Complete message transmission
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v rc		Packet status code
+ */
+void rndis_tx_complete_err ( struct rndis_device *rndis,
+			     struct io_buffer *iobuf, int rc ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	size_t len = iob_len ( iobuf );
+
+	/* Sanity check */
+	if ( len < sizeof ( *header ) ) {
+		DBGC ( rndis, "RNDIS %s completed underlength transmission:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		netdev_tx_err ( netdev, NULL, -EINVAL );
+		return;
+	}
+	header = iobuf->data;
+
+	/* Complete buffer */
+	if ( header->type == cpu_to_le32 ( RNDIS_PACKET_MSG ) ) {
+		netdev_tx_complete_err ( netdev, iobuf, rc );
+	} else {
+		free_iob ( iobuf );
+	}
+}
+
+/**
+ * Transmit data packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int rndis_tx_data ( struct rndis_device *rndis,
+			   struct io_buffer *iobuf ) {
+	struct rndis_packet_message *msg;
+	size_t len = iob_len ( iobuf );
+	int rc;
+
+	/* Prepend packet message header */
+	msg = iob_push ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->data.offset = cpu_to_le32 ( sizeof ( *msg ) );
+	msg->data.len = cpu_to_le32 ( len );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, RNDIS_PACKET_MSG ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Defer transmitted packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ *
+ * As with netdev_tx_defer(), the caller must ensure that space in the
+ * transmit descriptor ring is freed up before calling
+ * rndis_tx_complete().
+ *
+ * Unlike netdev_tx_defer(), this call may fail.
+ */
+int rndis_tx_defer ( struct rndis_device *rndis, struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	struct rndis_packet_message *msg;
+
+	/* Fail unless this was a packet message.  Only packet
+	 * messages correspond to I/O buffers in the network device's
+	 * TX queue; other messages cannot be deferred in this way.
+	 */
+	assert ( iob_len ( iobuf ) >= sizeof ( *header ) );
+	header = iobuf->data;
+	if ( header->type != cpu_to_le32 ( RNDIS_PACKET_MSG ) )
+		return -ENOTSUP;
+
+	/* Strip RNDIS header and packet message header, to return
+	 * this packet to the state in which we received it.
+	 */
+	iob_pull ( iobuf, ( sizeof ( *header ) + sizeof ( *msg ) ) );
+
+	/* Defer packet */
+	netdev_tx_defer ( netdev, iobuf );
+
+	return 0;
+}
+
+/**
+ * Receive data packet
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_data ( struct rndis_device *rndis,
+			    struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_packet_message *msg;
+	size_t len = iob_len ( iobuf );
+	size_t data_offset;
+	size_t data_len;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength data packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	msg = iobuf->data;
+
+	/* Locate and sanity check data buffer */
+	data_offset = le32_to_cpu ( msg->data.offset );
+	data_len = le32_to_cpu ( msg->data.len );
+	if ( ( data_offset > len ) || ( data_len > ( len - data_offset ) ) ) {
+		DBGC ( rndis, "RNDIS %s data packet data exceeds packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_data;
+	}
+
+	/* Strip non-data portions */
+	iob_pull ( iobuf, data_offset );
+	iob_unput ( iobuf, ( iob_len ( iobuf ) - data_len ) );
+
+	/* Hand off to network stack */
+	netdev_rx ( netdev, iob_disown ( iobuf ) );
+
+	return;
+
+ err_data:
+ err_len:
+	/* Report error to network stack */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Transmit initialisation message
+ *
+ * @v rndis		RNDIS device
+ * @v id		Request ID
+ * @ret rc		Return status code
+ */
+static int rndis_tx_initialise ( struct rndis_device *rndis, unsigned int id ) {
+	struct io_buffer *iobuf;
+	struct rndis_initialise_message *msg;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = id; /* Non-endian */
+	msg->major = cpu_to_le32 ( RNDIS_VERSION_MAJOR );
+	msg->minor = cpu_to_le32 ( RNDIS_VERSION_MINOR );
+	msg->mtu = cpu_to_le32 ( RNDIS_MTU );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf,
+				       RNDIS_INITIALISE_MSG ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Receive initialisation completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_initialise ( struct rndis_device *rndis,
+				  struct io_buffer *iobuf ) {
+	struct rndis_initialise_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	unsigned int id;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength initialisation "
+		       "completion:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received initialisation completion "
+		       "failure %#08x\n", rndis->name,
+		       le32_to_cpu ( cmplt->status ) );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	free_iob ( iobuf );
+}
+
+/**
+ * Initialise RNDIS
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_initialise ( struct rndis_device *rndis ) {
+	int rc;
+
+	/* Transmit initialisation message */
+	if ( ( rc = rndis_tx_initialise ( rndis, RNDIS_INIT_ID ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = rndis_wait ( rndis, RNDIS_INIT_ID ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Transmit halt message
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_tx_halt ( struct rndis_device *rndis ) {
+	struct io_buffer *iobuf;
+	struct rndis_halt_message *msg;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+
+	/* Transmit message */
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, RNDIS_HALT_MSG ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Halt RNDIS
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ */
+static int rndis_halt ( struct rndis_device *rndis ) {
+	int rc;
+
+	/* Transmit halt message */
+	if ( ( rc = rndis_tx_halt ( rndis ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Transmit OID message
+ *
+ * @v rndis		RNDIS device
+ * @v oid		Object ID
+ * @v data		New OID value (or NULL to query current value)
+ * @v len		Length of new OID value
+ * @ret rc		Return status code
+ */
+static int rndis_tx_oid ( struct rndis_device *rndis, unsigned int oid,
+			  const void *data, size_t len ) {
+	struct io_buffer *iobuf;
+	struct rndis_oid_message *msg;
+	unsigned int type;
+	int rc;
+
+	/* Allocate I/O buffer */
+	iobuf = rndis_alloc_iob ( sizeof ( *msg ) + len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+
+	/* Construct message.  We use the OID as the request ID. */
+	msg = iob_put ( iobuf, sizeof ( *msg ) );
+	memset ( msg, 0, sizeof ( *msg ) );
+	msg->id = oid; /* Non-endian */
+	msg->oid = cpu_to_le32 ( oid );
+	msg->offset = cpu_to_le32 ( sizeof ( *msg ) );
+	msg->len = cpu_to_le32 ( len );
+	memcpy ( iob_put ( iobuf, len ), data, len );
+
+	/* Transmit message */
+	type = ( data ? RNDIS_SET_MSG : RNDIS_QUERY_MSG );
+	if ( ( rc = rndis_tx_message ( rndis, iobuf, type ) ) != 0 )
+		goto err_tx;
+
+	return 0;
+
+ err_tx:
+	free_iob ( iobuf );
+ err_alloc:
+	return rc;
+}
+
+/**
+ * Receive query OID completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_query_oid ( struct rndis_device *rndis,
+				 struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_query_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	size_t info_offset;
+	size_t info_len;
+	unsigned int id;
+	void *info;
+	uint32_t *link_status;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength query "
+		       "completion:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received query completion failure "
+		       "%#08x\n", rndis->name, le32_to_cpu ( cmplt->status ) );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Locate and sanity check information buffer */
+	info_offset = le32_to_cpu ( cmplt->offset );
+	info_len = le32_to_cpu ( cmplt->len );
+	if ( ( info_offset > len ) || ( info_len > ( len - info_offset ) ) ) {
+		DBGC ( rndis, "RNDIS %s query completion information exceeds "
+		       "packet:\n", rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_info;
+	}
+	info = ( ( ( void * ) cmplt ) + info_offset );
+
+	/* Handle OID */
+	switch ( id ) {
+
+	case RNDIS_OID_802_3_PERMANENT_ADDRESS:
+		if ( info_len > sizeof ( netdev->hw_addr ) )
+			info_len = sizeof ( netdev->hw_addr );
+		memcpy ( netdev->hw_addr, info, info_len );
+		break;
+
+	case RNDIS_OID_802_3_CURRENT_ADDRESS:
+		if ( info_len > sizeof ( netdev->ll_addr ) )
+			info_len = sizeof ( netdev->ll_addr );
+		memcpy ( netdev->ll_addr, info, info_len );
+		break;
+
+	case RNDIS_OID_GEN_MEDIA_CONNECT_STATUS:
+		if ( info_len != sizeof ( *link_status ) ) {
+			DBGC ( rndis, "RNDIS %s invalid link status:\n",
+			       rndis->name );
+			DBGC_HDA ( rndis, 0, iobuf->data, len );
+			rc = -EPROTO;
+			goto err_link_status;
+		}
+		link_status = info;
+		if ( *link_status == 0 ) {
+			DBGC ( rndis, "RNDIS %s link is up\n", rndis->name );
+			netdev_link_up ( netdev );
+		} else {
+			DBGC ( rndis, "RNDIS %s link is down: %#08x\n",
+			       rndis->name, le32_to_cpu ( *link_status ) );
+			netdev_link_down ( netdev );
+		}
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s unexpected query completion ID %#08x\n",
+		       rndis->name, id );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EPROTO;
+		goto err_id;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_id:
+ err_link_status:
+ err_info:
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/**
+ * Receive set OID completion
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_set_oid ( struct rndis_device *rndis,
+			       struct io_buffer *iobuf ) {
+	struct rndis_set_completion *cmplt;
+	size_t len = iob_len ( iobuf );
+	unsigned int id;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *cmplt ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength set completion:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	cmplt = iobuf->data;
+
+	/* Extract request ID */
+	id = cmplt->id; /* Non-endian */
+
+	/* Check status */
+	if ( cmplt->status ) {
+		DBGC ( rndis, "RNDIS %s received set completion failure "
+		       "%#08x\n", rndis->name, le32_to_cpu ( cmplt->status ) );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EIO;
+		goto err_status;
+	}
+
+	/* Success */
+	rc = 0;
+
+ err_status:
+	/* Record completion result if applicable */
+	if ( id == rndis->wait_id ) {
+		rndis->wait_id = 0;
+		rndis->wait_rc = rc;
+	}
+ err_len:
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+}
+
+/**
+ * Query or set OID
+ *
+ * @v rndis		RNDIS device
+ * @v oid		Object ID
+ * @v data		New OID value (or NULL to query current value)
+ * @v len		Length of new OID value
+ * @ret rc		Return status code
+ */
+static int rndis_oid ( struct rndis_device *rndis, unsigned int oid,
+		       const void *data, size_t len ) {
+	int rc;
+
+	/* Transmit query */
+	if ( ( rc = rndis_tx_oid ( rndis, oid, data, len ) ) != 0 )
+		return rc;
+
+	/* Wait for response */
+	if ( ( rc = rndis_wait ( rndis, oid ) ) != 0 )
+		return rc;
+
+	return 0;
+}
+
+/**
+ * Receive indicate status message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+static void rndis_rx_status ( struct rndis_device *rndis,
+			      struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_indicate_status_message *msg;
+	size_t len = iob_len ( iobuf );
+	unsigned int status;
+	int rc;
+
+	/* Sanity check */
+	if ( len < sizeof ( *msg ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength status message:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -EINVAL;
+		goto err_len;
+	}
+	msg = iobuf->data;
+
+	/* Extract status */
+	status = le32_to_cpu ( msg->status );
+
+	/* Handle status */
+	switch ( msg->status ) {
+
+	case RNDIS_STATUS_MEDIA_CONNECT:
+		DBGC ( rndis, "RNDIS %s link is up\n", rndis->name );
+		netdev_link_up ( netdev );
+		break;
+
+	case RNDIS_STATUS_MEDIA_DISCONNECT:
+		DBGC ( rndis, "RNDIS %s link is down\n", rndis->name );
+		netdev_link_down ( netdev );
+		break;
+
+	case RNDIS_STATUS_WTF_WORLD:
+		/* Ignore */
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s unexpected status %#08x:\n",
+		       rndis->name, status );
+		DBGC_HDA ( rndis, 0, iobuf->data, len );
+		rc = -ENOTSUP;
+		goto err_status;
+	}
+
+	/* Free I/O buffer */
+	free_iob ( iobuf );
+
+	return;
+
+ err_status:
+ err_len:
+	/* Report error via network device statistics */
+	netdev_rx_err ( netdev, iobuf, rc );
+}
+
+/**
+ * Receive RNDIS message
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v type		Message type
+ */
+static void rndis_rx_message ( struct rndis_device *rndis,
+			       struct io_buffer *iobuf, unsigned int type ) {
+	struct net_device *netdev = rndis->netdev;
+	int rc;
+
+	/* Handle packet */
+	switch ( type ) {
+
+	case RNDIS_PACKET_MSG:
+		rndis_rx_data ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_INITIALISE_CMPLT:
+		rndis_rx_initialise ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_QUERY_CMPLT:
+		rndis_rx_query_oid ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_SET_CMPLT:
+		rndis_rx_set_oid ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	case RNDIS_INDICATE_STATUS_MSG:
+		rndis_rx_status ( rndis, iob_disown ( iobuf ) );
+		break;
+
+	default:
+		DBGC ( rndis, "RNDIS %s received unexpected type %#08x\n",
+		       rndis->name, type );
+		DBGC_HDA ( rndis, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EPROTO;
+		goto err_type;
+	}
+
+	return;
+
+ err_type:
+	/* Report error via network device statistics */
+	netdev_rx_err ( netdev, iobuf, rc );
+}
+
+/**
+ * Receive packet from underlying transport layer
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ */
+void rndis_rx ( struct rndis_device *rndis, struct io_buffer *iobuf ) {
+	struct net_device *netdev = rndis->netdev;
+	struct rndis_header *header;
+	unsigned int type;
+	int rc;
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < sizeof ( *header ) ) {
+		DBGC ( rndis, "RNDIS %s received underlength packet:\n",
+		       rndis->name );
+		DBGC_HDA ( rndis, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto drop;
+	}
+	header = iobuf->data;
+
+	/* Parse and strip header */
+	type = le32_to_cpu ( header->type );
+	iob_pull ( iobuf, sizeof ( *header ) );
+
+	/* Handle message */
+	rndis_rx_message ( rndis, iob_disown ( iobuf ), type );
+
+	return;
+
+ drop:
+	/* Record error */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Discard packet from underlying transport layer
+ *
+ * @v rndis		RNDIS device
+ * @v iobuf		I/O buffer
+ * @v rc		Packet status code
+ */
+void rndis_rx_err ( struct rndis_device *rndis, struct io_buffer *iobuf,
+		    int rc ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Record error */
+	netdev_rx_err ( netdev, iob_disown ( iobuf ), rc );
+}
+
+/**
+ * Set receive filter
+ *
+ * @v rndis		RNDIS device
+ * @v filter		Receive filter
+ * @ret rc		Return status code
+ */
+static int rndis_filter ( struct rndis_device *rndis, unsigned int filter ) {
+	uint32_t value = cpu_to_le32 ( filter );
+	int rc;
+
+	/* Set receive filter */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_CURRENT_PACKET_FILTER,
+				&value, sizeof ( value ) ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not set receive filter to %#08x: "
+		       "%s\n", rndis->name, filter, strerror ( rc ) );
+		return rc;
+	}
+
+	return 0;
+}
+
+/**
+ * Open network device
+ *
+ * @v netdev		Network device
+ * @ret rc		Return status code
+ */
+static int rndis_open ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+	int rc;
+
+	/* Open RNDIS device */
+	if ( ( rc = rndis->op->open ( rndis ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not open: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Initialise RNDIS */
+	if ( ( rc = rndis_initialise ( rndis ) ) != 0 )
+		goto err_initialise;
+
+	/* Set receive filter */
+	if ( ( rc = rndis_filter ( rndis, ( RNDIS_FILTER_UNICAST |
+					    RNDIS_FILTER_MULTICAST |
+					    RNDIS_FILTER_ALL_MULTICAST |
+					    RNDIS_FILTER_BROADCAST |
+					    RNDIS_FILTER_PROMISCUOUS ) ) ) != 0)
+		goto err_set_filter;
+
+	/* Update link status */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_link;
+
+	return 0;
+
+ err_query_link:
+ err_set_filter:
+	rndis_halt ( rndis );
+ err_initialise:
+	rndis->op->close ( rndis );
+ err_open:
+	return rc;
+}
+
+/**
+ * Close network device
+ *
+ * @v netdev		Network device
+ */
+static void rndis_close ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Clear receive filter */
+	rndis_filter ( rndis, 0 );
+
+	/* Halt RNDIS device */
+	rndis_halt ( rndis );
+
+	/* Close RNDIS device */
+	rndis->op->close ( rndis );
+}
+
+/**
+ * Transmit packet
+ *
+ * @v netdev		Network device
+ * @v iobuf		I/O buffer
+ * @ret rc		Return status code
+ */
+static int rndis_transmit ( struct net_device *netdev,
+			    struct io_buffer *iobuf ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Transmit data packet */
+	return rndis_tx_data ( rndis, iobuf );
+}
+
+/**
+ * Poll for completed and received packets
+ *
+ * @v netdev		Network device
+ */
+static void rndis_poll ( struct net_device *netdev ) {
+	struct rndis_device *rndis = netdev->priv;
+
+	/* Poll RNDIS device */
+	rndis->op->poll ( rndis );
+}
+
+/** Network device operations */
+static struct net_device_operations rndis_operations = {
+	.open		= rndis_open,
+	.close		= rndis_close,
+	.transmit	= rndis_transmit,
+	.poll		= rndis_poll,
+};
+
+/**
+ * Allocate RNDIS device
+ *
+ * @v priv_len		Length of private data
+ * @ret rndis		RNDIS device, or NULL on allocation failure
+ */
+struct rndis_device * alloc_rndis ( size_t priv_len ) {
+	struct net_device *netdev;
+	struct rndis_device *rndis;
+
+	/* Allocate and initialise structure */
+	netdev = alloc_etherdev ( sizeof ( *rndis ) + priv_len );
+	if ( ! netdev )
+		return NULL;
+	netdev_init ( netdev, &rndis_operations );
+	rndis = netdev->priv;
+	rndis->netdev = netdev;
+	rndis->priv = ( ( ( void * ) rndis ) + sizeof ( *rndis ) );
+
+	return rndis;
+}
+
+/**
+ * Register RNDIS device
+ *
+ * @v rndis		RNDIS device
+ * @ret rc		Return status code
+ *
+ * Note that this routine will open and use the RNDIS device in order
+ * to query the MAC address.  The device must be immediately ready for
+ * use prior to registration.
+ */
+int register_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+	int rc;
+
+	/* Assign device name (for debugging) */
+	rndis->name = netdev->dev->name;
+
+	/* Register network device */
+	if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not register: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_register;
+	}
+
+	/* Open RNDIS device to read MAC addresses */
+	if ( ( rc = rndis->op->open ( rndis ) ) != 0 ) {
+		DBGC ( rndis, "RNDIS %s could not open: %s\n",
+		       rndis->name, strerror ( rc ) );
+		goto err_open;
+	}
+
+	/* Initialise RNDIS */
+	if ( ( rc = rndis_initialise ( rndis ) ) != 0 )
+		goto err_initialise;
+
+	/* Query permanent MAC address */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_802_3_PERMANENT_ADDRESS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_permanent;
+
+	/* Query current MAC address */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_802_3_CURRENT_ADDRESS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_current;
+
+	/* Get link status */
+	if ( ( rc = rndis_oid ( rndis, RNDIS_OID_GEN_MEDIA_CONNECT_STATUS,
+				NULL, 0 ) ) != 0 )
+		goto err_query_link;
+
+	/* Halt RNDIS device */
+	rndis_halt ( rndis );
+
+	/* Close RNDIS device */
+	rndis->op->close ( rndis );
+
+	return 0;
+
+ err_query_link:
+ err_query_current:
+ err_query_permanent:
+	rndis_halt ( rndis );
+ err_initialise:
+	rndis->op->close ( rndis );
+ err_open:
+	unregister_netdev ( netdev );
+ err_register:
+	return rc;
+}
+
+/**
+ * Unregister RNDIS device
+ *
+ * @v rndis		RNDIS device
+ */
+void unregister_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Unregister network device */
+	unregister_netdev ( netdev );
+}
+
+/**
+ * Free RNDIS device
+ *
+ * @v rndis		RNDIS device
+ */
+void free_rndis ( struct rndis_device *rndis ) {
+	struct net_device *netdev = rndis->netdev;
+
+	/* Free network device */
+	netdev_nullify ( netdev );
+	netdev_put ( netdev );
+}
diff --git a/qemu/roms/ipxe/src/net/socket.c b/qemu/roms/ipxe/src/net/socket.c
index 24f6a0892..2009ab237 100644
--- a/qemu/roms/ipxe/src/net/socket.c
+++ b/qemu/roms/ipxe/src/net/socket.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/net/stp.c b/qemu/roms/ipxe/src/net/stp.c
new file mode 100644
index 000000000..d4e65a1a2
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/stp.c
@@ -0,0 +1,152 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/netdevice.h>
+#include <ipxe/ethernet.h>
+#include <ipxe/iobuf.h>
+#include <ipxe/timer.h>
+#include <ipxe/stp.h>
+
+/** @file
+ *
+ * Spanning Tree Protocol (STP)
+ *
+ */
+
+/* Disambiguate the various error causes */
+#define ENOTSUP_PROTOCOL __einfo_error ( EINFO_ENOTSUP_PROTOCOL )
+#define EINFO_ENOTSUP_PROTOCOL					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Non-STP packet received" )
+#define ENOTSUP_VERSION __einfo_error ( EINFO_ENOTSUP_VERSION )
+#define EINFO_ENOTSUP_VERSION					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Legacy STP packet received" )
+#define ENOTSUP_TYPE __einfo_error ( EINFO_ENOTSUP_TYPE )
+#define EINFO_ENOTSUP_TYPE					\
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01,			\
+			  "Non-RSTP packet received" )
+
+/**
+ * Process incoming STP packets
+ *
+ * @v iobuf		I/O buffer
+ * @v netdev		Network device
+ * @v ll_source		Link-layer source address
+ * @v flags		Packet flags
+ * @ret rc		Return status code
+ */
+static int stp_rx ( struct io_buffer *iobuf, struct net_device *netdev,
+		    const void *ll_dest __unused,
+		    const void *ll_source __unused,
+		    unsigned int flags __unused ) {
+	struct stp_bpdu *stp;
+	unsigned int hello;
+	int rc;
+
+	/* Sanity check */
+	if ( iob_len ( iobuf ) < sizeof ( *stp ) ) {
+		DBGC ( netdev, "STP %s received underlength packet (%zd "
+		       "bytes):\n", netdev->name, iob_len ( iobuf ) );
+		DBGC_HDA ( netdev, 0, iobuf->data, iob_len ( iobuf ) );
+		rc = -EINVAL;
+		goto done;
+	}
+	stp = iobuf->data;
+
+	/* Ignore non-RSTP packets */
+	if ( stp->protocol != htons ( STP_PROTOCOL ) ) {
+		DBGC ( netdev, "STP %s ignoring non-STP packet (protocol "
+		       "%#04x)\n", netdev->name, ntohs ( stp->protocol ) );
+		rc = -ENOTSUP_PROTOCOL;
+		goto done;
+	}
+	if ( stp->version < STP_VERSION_RSTP ) {
+		DBGC ( netdev, "STP %s received legacy STP packet (version "
+		       "%#02x)\n", netdev->name, stp->version );
+		rc = -ENOTSUP_VERSION;
+		goto done;
+	}
+	if ( stp->type != STP_TYPE_RSTP ) {
+		DBGC ( netdev, "STP %s received non-RSTP packet (type %#02x)\n",
+		       netdev->name, stp->type );
+		rc = -ENOTSUP_TYPE;
+		goto done;
+	}
+
+	/* Dump information */
+	DBGC2 ( netdev, "STP %s %s port %#04x flags %#02x hello %d delay %d\n",
+		netdev->name, eth_ntoa ( stp->sender.mac ), ntohs ( stp->port ),
+		stp->flags, ntohs ( stp->hello ), ntohs ( stp->delay ) );
+
+	/* Check if port is forwarding */
+	if ( ! ( stp->flags & STP_FL_FORWARDING ) ) {
+		/* Port is not forwarding: block link for two hello times */
+		DBGC ( netdev, "STP %s %s port %#04x flags %#02x is not "
+		       "forwarding\n",
+		       netdev->name, eth_ntoa ( stp->sender.mac ),
+		       ntohs ( stp->port ), stp->flags );
+		hello = ( ( ntohs ( stp->hello ) * TICKS_PER_SEC ) / 256 );
+		netdev_link_block ( netdev, ( hello * 2 ) );
+		rc = -ENETUNREACH;
+		goto done;
+	}
+
+	/* Success */
+	if ( netdev_link_blocked ( netdev ) ) {
+		DBGC ( netdev, "STP %s %s port %#04x flags %#02x is "
+		       "forwarding\n",
+		       netdev->name, eth_ntoa ( stp->sender.mac ),
+		       ntohs ( stp->port ), stp->flags );
+	}
+	netdev_link_unblock ( netdev );
+	rc = 0;
+
+ done:
+	free_iob ( iobuf );
+	return rc;
+}
+
+/**
+ * Transcribe STP address
+ *
+ * @v net_addr		STP address
+ * @ret string		"<STP>"
+ *
+ * This operation is meaningless for the STP protocol.
+ */
+static const char * stp_ntoa ( const void *net_addr __unused ) {
+	return "<STP>";
+}
+
+/** STP network protocol */
+struct net_protocol stp_protocol __net_protocol = {
+	.name = "STP",
+	.net_proto = htons ( ETH_P_STP ),
+	.rx = stp_rx,
+	.ntoa = stp_ntoa,
+};
diff --git a/qemu/roms/ipxe/src/net/tcp.c b/qemu/roms/ipxe/src/net/tcp.c
index 987cb63e1..c69c83b85 100644
--- a/qemu/roms/ipxe/src/net/tcp.c
+++ b/qemu/roms/ipxe/src/net/tcp.c
@@ -26,7 +26,7 @@
  *
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** A TCP connection */
 struct tcp_connection {
@@ -101,8 +101,9 @@ struct tcp_connection {
 	 * Equivalent to Rcv.Wind.Scale in RFC 1323 terminology
 	 */
 	uint8_t rcv_win_scale;
-	/** Maximum receive window */
-	uint32_t max_rcv_win;
+
+	/** Selective acknowledgement list (in host-endian order) */
+	struct tcp_sack_block sack[TCP_SACK_MAX];
 
 	/** Transmit queue */
 	struct list_head tx_queue;
@@ -129,6 +130,8 @@ enum tcp_flags {
 	TCP_TS_ENABLED = 0x0002,
 	/** TCP acknowledgement is pending */
 	TCP_ACK_PENDING = 0x0004,
+	/** TCP selective acknowledgement is enabled */
+	TCP_SACK_ENABLED = 0x0008,
 };
 
 /** TCP internal header
@@ -143,6 +146,8 @@ struct tcp_rx_queued_header {
 	 * enqueued, and so excludes the SYN, if present.
 	 */
 	uint32_t seq;
+	/** Next SEQ value, in host-endian order */
+	uint32_t nxt;
 	/** Flags
 	 *
 	 * Only FIN is valid within this flags byte; all other flags
@@ -284,7 +289,6 @@ static int tcp_open ( struct interface *xfer, struct sockaddr *peer,
 	tcp->tcp_state = TCP_STATE_SENT ( TCP_SYN );
 	tcp_dump_state ( tcp );
 	tcp->snd_seq = random();
-	tcp->max_rcv_win = TCP_MAX_WINDOW_SIZE;
 	INIT_LIST_HEAD ( &tcp->tx_queue );
 	INIT_LIST_HEAD ( &tcp->rx_queue );
 	memcpy ( &tcp->peer, st_peer, sizeof ( tcp->peer ) );
@@ -396,6 +400,7 @@ static void tcp_close ( struct tcp_connection *tcp, int rc ) {
 
 		tcp->tcp_state |= TCP_STATE_SENT ( TCP_FIN );
 		tcp_dump_state ( tcp );
+		process_add ( &tcp->process );
 
 		/* Add a pending operation for the FIN */
 		pending_get ( &tcp->pending_flags );
@@ -450,6 +455,94 @@ static size_t tcp_xfer_window ( struct tcp_connection *tcp ) {
 }
 
 /**
+ * Find selective acknowledgement block
+ *
+ * @v tcp		TCP connection
+ * @v seq		SEQ value in SACK block (in host-endian order)
+ * @v sack		SACK block to fill in (in host-endian order)
+ * @ret len		Length of SACK block
+ */
+static uint32_t tcp_sack_block ( struct tcp_connection *tcp, uint32_t seq,
+				 struct tcp_sack_block *sack ) {
+	struct io_buffer *iobuf;
+	struct tcp_rx_queued_header *tcpqhdr;
+	uint32_t left = tcp->rcv_ack;
+	uint32_t right = left;
+
+	/* Find highest block which does not start after SEQ */
+	list_for_each_entry ( iobuf, &tcp->rx_queue, list ) {
+		tcpqhdr = iobuf->data;
+		if ( tcp_cmp ( tcpqhdr->seq, right ) > 0 ) {
+			if ( tcp_cmp ( tcpqhdr->seq, seq ) > 0 )
+				break;
+			left = tcpqhdr->seq;
+		}
+		if ( tcp_cmp ( tcpqhdr->nxt, right ) > 0 )
+			right = tcpqhdr->nxt;
+	}
+
+	/* Fail if this block does not contain SEQ */
+	if ( tcp_cmp ( right, seq ) < 0 )
+		return 0;
+
+	/* Populate SACK block */
+	sack->left = left;
+	sack->right = right;
+	return ( right - left );
+}
+
+/**
+ * Update TCP selective acknowledgement list
+ *
+ * @v tcp		TCP connection
+ * @v seq		SEQ value in first SACK block (in host-endian order)
+ * @ret count		Number of SACK blocks
+ */
+static unsigned int tcp_sack ( struct tcp_connection *tcp, uint32_t seq ) {
+	struct tcp_sack_block sack[TCP_SACK_MAX];
+	unsigned int old = 0;
+	unsigned int new = 0;
+	unsigned int i;
+	uint32_t len;
+
+	/* Populate first new SACK block */
+	len = tcp_sack_block ( tcp, seq, &sack[0] );
+	if ( len )
+		new++;
+
+	/* Populate remaining new SACK blocks based on old SACK blocks */
+	for ( old = 0 ; old < TCP_SACK_MAX ; old++ ) {
+
+		/* Stop if we run out of space in the new list */
+		if ( new == TCP_SACK_MAX )
+			break;
+
+		/* Skip empty old SACK blocks */
+		if ( tcp->sack[old].left == tcp->sack[old].right )
+			continue;
+
+		/* Populate new SACK block */
+		len = tcp_sack_block ( tcp, tcp->sack[old].left, &sack[new] );
+		if ( len == 0 )
+			continue;
+
+		/* Eliminate duplicates */
+		for ( i = 0 ; i < new ; i++ ) {
+			if ( sack[i].left == sack[new].left ) {
+				new--;
+				break;
+			}
+		}
+		new++;
+	}
+
+	/* Update SACK list */
+	memset ( tcp->sack, 0, sizeof ( tcp->sack ) );
+	memcpy ( tcp->sack, sack, ( new * sizeof ( tcp->sack[0] ) ) );
+	return new;
+}
+
+/**
  * Process TCP transmit queue
  *
  * @v tcp		TCP connection
@@ -493,9 +586,10 @@ static size_t tcp_process_tx_queue ( struct tcp_connection *tcp, size_t max_len,
 }
 
 /**
- * Transmit any outstanding data
+ * Transmit any outstanding data (with selective acknowledgement)
  *
  * @v tcp		TCP connection
+ * @v sack_seq		SEQ for first selective acknowledgement (if any)
  * 
  * Transmits any outstanding data on the connection.
  *
@@ -503,17 +597,22 @@ static size_t tcp_process_tx_queue ( struct tcp_connection *tcp, size_t max_len,
  * will have been started if necessary, and so the stack will
  * eventually attempt to retransmit the failed packet.
  */
-static void tcp_xmit ( struct tcp_connection *tcp ) {
+static void tcp_xmit_sack ( struct tcp_connection *tcp, uint32_t sack_seq ) {
 	struct io_buffer *iobuf;
 	struct tcp_header *tcphdr;
 	struct tcp_mss_option *mssopt;
 	struct tcp_window_scale_padded_option *wsopt;
 	struct tcp_timestamp_padded_option *tsopt;
+	struct tcp_sack_permitted_padded_option *spopt;
+	struct tcp_sack_padded_option *sackopt;
+	struct tcp_sack_block *sack;
 	void *payload;
 	unsigned int flags;
+	unsigned int sack_count;
+	unsigned int i;
 	size_t len = 0;
+	size_t sack_len;
 	uint32_t seq_len;
-	uint32_t app_win;
 	uint32_t max_rcv_win;
 	uint32_t max_representable_win;
 	int rc;
@@ -567,10 +666,9 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 	tcp_process_tx_queue ( tcp, len, iobuf, 0 );
 
 	/* Expand receive window if possible */
-	max_rcv_win = tcp->max_rcv_win;
-	app_win = xfer_window ( &tcp->xfer );
-	if ( max_rcv_win > app_win )
-		max_rcv_win = app_win;
+	max_rcv_win = xfer_window ( &tcp->xfer );
+	if ( max_rcv_win > TCP_MAX_WINDOW_SIZE )
+		max_rcv_win = TCP_MAX_WINDOW_SIZE;
 	max_representable_win = ( 0xffff << tcp->rcv_win_scale );
 	if ( max_rcv_win > max_representable_win )
 		max_rcv_win = max_representable_win;
@@ -590,6 +688,10 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 		wsopt->wsopt.kind = TCP_OPTION_WS;
 		wsopt->wsopt.length = sizeof ( wsopt->wsopt );
 		wsopt->wsopt.scale = TCP_RX_WINDOW_SCALE;
+		spopt = iob_push ( iobuf, sizeof ( *spopt ) );
+		memset ( spopt->nop, TCP_OPTION_NOP, sizeof ( spopt ) );
+		spopt->spopt.kind = TCP_OPTION_SACK_PERMITTED;
+		spopt->spopt.length = sizeof ( spopt->spopt );
 	}
 	if ( ( flags & TCP_SYN ) || ( tcp->flags & TCP_TS_ENABLED ) ) {
 		tsopt = iob_push ( iobuf, sizeof ( *tsopt ) );
@@ -599,6 +701,21 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 		tsopt->tsopt.tsval = htonl ( currticks() );
 		tsopt->tsopt.tsecr = htonl ( tcp->ts_recent );
 	}
+	if ( ( tcp->flags & TCP_SACK_ENABLED ) &&
+	     ( ! list_empty ( &tcp->rx_queue ) ) &&
+	     ( ( sack_count = tcp_sack ( tcp, sack_seq ) ) != 0 ) ) {
+		sack_len = ( sack_count * sizeof ( *sack ) );
+		sackopt = iob_push ( iobuf, ( sizeof ( *sackopt ) + sack_len ));
+		memset ( sackopt->nop, TCP_OPTION_NOP, sizeof ( sackopt->nop ));
+		sackopt->sackopt.kind = TCP_OPTION_SACK;
+		sackopt->sackopt.length =
+			( sizeof ( sackopt->sackopt ) + sack_len );
+		sack = ( ( ( void * ) sackopt ) + sizeof ( *sackopt ) );
+		for ( i = 0 ; i < sack_count ; i++, sack++ ) {
+			sack->left = htonl ( tcp->sack[i].left );
+			sack->right = htonl ( tcp->sack[i].right );
+		}
+	}
 	if ( len != 0 )
 		flags |= TCP_PSH;
 	tcphdr = iob_push ( iobuf, sizeof ( *tcphdr ) );
@@ -635,6 +752,17 @@ static void tcp_xmit ( struct tcp_connection *tcp ) {
 	profile_stop ( &tcp_tx_profiler );
 }
 
+/**
+ * Transmit any outstanding data
+ *
+ * @v tcp		TCP connection
+ */
+static void tcp_xmit ( struct tcp_connection *tcp ) {
+
+	/* Transmit without an explicit first SACK */
+	tcp_xmit_sack ( tcp, tcp->rcv_ack );
+}
+
 /** TCP process descriptor */
 static struct process_descriptor tcp_process_desc =
 	PROC_DESC_ONCE ( struct tcp_connection, process, tcp_xmit );
@@ -804,6 +932,12 @@ static void tcp_rx_opts ( struct tcp_connection *tcp, const void *data,
 		case TCP_OPTION_WS:
 			options->wsopt = data;
 			break;
+		case TCP_OPTION_SACK_PERMITTED:
+			options->spopt = data;
+			break;
+		case TCP_OPTION_SACK:
+			/* Ignore received SACKs */
+			break;
 		case TCP_OPTION_TS:
 			options->tsopt = data;
 			break;
@@ -823,6 +957,7 @@ static void tcp_rx_opts ( struct tcp_connection *tcp, const void *data,
  * @v seq_len		Sequence space length to consume
  */
 static void tcp_rx_seq ( struct tcp_connection *tcp, uint32_t seq_len ) {
+	unsigned int sack;
 
 	/* Sanity check */
 	assert ( seq_len > 0 );
@@ -840,6 +975,16 @@ static void tcp_rx_seq ( struct tcp_connection *tcp, uint32_t seq_len ) {
 	/* Update timestamp */
 	tcp->ts_recent = tcp->ts_val;
 
+	/* Update SACK list */
+	for ( sack = 0 ; sack < TCP_SACK_MAX ; sack++ ) {
+		if ( tcp->sack[sack].left == tcp->sack[sack].right )
+			continue;
+		if ( tcp_cmp ( tcp->sack[sack].left, tcp->rcv_ack ) < 0 )
+			tcp->sack[sack].left = tcp->rcv_ack;
+		if ( tcp_cmp ( tcp->sack[sack].right, tcp->rcv_ack ) < 0 )
+			tcp->sack[sack].right = tcp->rcv_ack;
+	}
+
 	/* Mark ACK as pending */
 	tcp->flags |= TCP_ACK_PENDING;
 }
@@ -860,6 +1005,8 @@ static int tcp_rx_syn ( struct tcp_connection *tcp, uint32_t seq,
 		tcp->rcv_ack = seq;
 		if ( options->tsopt )
 			tcp->flags |= TCP_TS_ENABLED;
+		if ( options->spopt )
+			tcp->flags |= TCP_SACK_ENABLED;
 		if ( options->wsopt ) {
 			tcp->snd_win_scale = options->wsopt->scale;
 			tcp->rcv_win_scale = TCP_RX_WINDOW_SCALE;
@@ -1070,6 +1217,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	struct io_buffer *queued;
 	size_t len;
 	uint32_t seq_len;
+	uint32_t nxt;
 
 	/* Calculate remaining flags and sequence length.  Note that
 	 * SYN, if present, has already been processed by this point.
@@ -1077,6 +1225,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	flags &= TCP_FIN;
 	len = iob_len ( iobuf );
 	seq_len = ( len + ( flags ? 1 : 0 ) );
+	nxt = ( seq + seq_len );
 
 	/* Discard immediately (to save memory) if:
 	 *
@@ -1087,7 +1236,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	 */
 	if ( ( ! ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) ) ||
 	     ( tcp_cmp ( seq, tcp->rcv_ack + tcp->rcv_win ) >= 0 ) ||
-	     ( tcp_cmp ( seq + seq_len, tcp->rcv_ack ) < 0 ) ||
+	     ( tcp_cmp ( nxt, tcp->rcv_ack ) < 0 ) ||
 	     ( seq_len == 0 ) ) {
 		free_iob ( iobuf );
 		return;
@@ -1096,6 +1245,7 @@ static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
 	/* Add internal header */
 	tcpqhdr = iob_push ( iobuf, sizeof ( *tcpqhdr ) );
 	tcpqhdr->seq = seq;
+	tcpqhdr->nxt = nxt;
 	tcpqhdr->flags = flags;
 
 	/* Add to RX queue */
@@ -1289,7 +1439,7 @@ static int tcp_rx ( struct io_buffer *iobuf,
 	if ( list_empty ( &tcp->rx_queue ) ) {
 		process_add ( &tcp->process );
 	} else {
-		tcp_xmit ( tcp );
+		tcp_xmit_sack ( tcp, seq );
 	}
 
 	/* If this packet was the last we expect to receive, set up
@@ -1328,24 +1478,12 @@ struct tcpip_protocol tcp_protocol __tcpip_protocol = {
 static unsigned int tcp_discard ( void ) {
 	struct tcp_connection *tcp;
 	struct io_buffer *iobuf;
-	struct tcp_rx_queued_header *tcpqhdr;
-	uint32_t max_win;
 	unsigned int discarded = 0;
 
 	/* Try to drop one queued RX packet from each connection */
 	list_for_each_entry ( tcp, &tcp_conns, list ) {
 		list_for_each_entry_reverse ( iobuf, &tcp->rx_queue, list ) {
 
-			/* Limit window to prevent future discards */
-			tcpqhdr = iobuf->data;
-			max_win = ( tcpqhdr->seq - tcp->rcv_ack );
-			if ( max_win < tcp->max_rcv_win ) {
-				DBGC ( tcp, "TCP %p reducing maximum window "
-				       "from %d to %d\n",
-				       tcp, tcp->max_rcv_win, max_win );
-				tcp->max_rcv_win = max_win;
-			}
-
 			/* Remove packet from queue */
 			list_del ( &iobuf->list );
 			free_iob ( iobuf );
@@ -1365,12 +1503,67 @@ struct cache_discarder tcp_discarder __cache_discarder ( CACHE_NORMAL ) = {
 };
 
 /**
+ * Find first TCP connection that has not yet been closed
+ *
+ * @ret tcp		First unclosed connection, or NULL
+ */
+static struct tcp_connection * tcp_first_unclosed ( void ) {
+	struct tcp_connection *tcp;
+
+	/* Find first connection which has not yet been closed */
+	list_for_each_entry ( tcp, &tcp_conns, list ) {
+		if ( ! ( tcp->flags & TCP_XFER_CLOSED ) )
+			return tcp;
+	}
+	return NULL;
+}
+
+/**
+ * Find first TCP connection that has not yet finished all operations
+ *
+ * @ret tcp		First unfinished connection, or NULL
+ */
+static struct tcp_connection * tcp_first_unfinished ( void ) {
+	struct tcp_connection *tcp;
+
+	/* Find first connection which has not yet closed gracefully,
+	 * or which still has a pending transmission (e.g. to ACK the
+	 * received FIN).
+	 */
+	list_for_each_entry ( tcp, &tcp_conns, list ) {
+		if ( ( ! TCP_CLOSED_GRACEFULLY ( tcp->tcp_state ) ) ||
+		     process_running ( &tcp->process ) ) {
+			return tcp;
+		}
+	}
+	return NULL;
+}
+
+/**
  * Shut down all TCP connections
  *
  */
 static void tcp_shutdown ( int booting __unused ) {
 	struct tcp_connection *tcp;
+	unsigned long start;
+	unsigned long elapsed;
+
+	/* Initiate a graceful close of all connections, allowing for
+	 * the fact that the connection list may change as we do so.
+	 */
+	while ( ( tcp = tcp_first_unclosed() ) ) {
+		DBGC ( tcp, "TCP %p closing for shutdown\n", tcp );
+		tcp_close ( tcp, -ECANCELED );
+	}
+
+	/* Wait for all connections to finish closing gracefully */
+	start = currticks();
+	while ( ( tcp = tcp_first_unfinished() ) &&
+		( ( elapsed = ( currticks() - start ) ) < TCP_FINISH_TIMEOUT )){
+		step();
+	}
 
+	/* Forcibly close any remaining connections */
 	while ( ( tcp = list_first_entry ( &tcp_conns, struct tcp_connection,
 					   list ) ) != NULL ) {
 		tcp->tcp_state = TCP_CLOSED;
@@ -1380,7 +1573,7 @@ static void tcp_shutdown ( int booting __unused ) {
 }
 
 /** TCP shutdown function */
-struct startup_fn tcp_startup_fn __startup_fn ( STARTUP_EARLY ) = {
+struct startup_fn tcp_startup_fn __startup_fn ( STARTUP_LATE ) = {
 	.shutdown = tcp_shutdown,
 };
 
diff --git a/qemu/roms/ipxe/src/net/tcp/http.c b/qemu/roms/ipxe/src/net/tcp/http.c
index 90bae9d7a..b000ed80f 100644
--- a/qemu/roms/ipxe/src/net/tcp/http.c
+++ b/qemu/roms/ipxe/src/net/tcp/http.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -26,26 +30,20 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#include <stddef.h>
 #include <ipxe/open.h>
 #include <ipxe/http.h>
 #include <ipxe/features.h>
 
 FEATURE ( FEATURE_PROTOCOL, "HTTP", DHCP_EB_FEATURE_HTTP, 1 );
 
-/**
- * Initiate an HTTP connection
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int http_open ( struct interface *xfer, struct uri *uri ) {
-	return http_open_filter ( xfer, uri, HTTP_PORT, NULL );
-}
-
 /** HTTP URI opener */
 struct uri_opener http_uri_opener __uri_opener = {
 	.scheme	= "http",
-	.open	= http_open,
+	.open	= http_open_uri,
+};
+
+/** HTTP URI scheme */
+struct http_scheme http_scheme __http_scheme = {
+	.name = "http",
+	.port = HTTP_PORT,
 };
diff --git a/qemu/roms/ipxe/src/net/tcp/httpauth.c b/qemu/roms/ipxe/src/net/tcp/httpauth.c
new file mode 100644
index 000000000..fb6dcd035
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/tcp/httpauth.c
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) authentication
+ *
+ */
+
+#include <stdio.h>
+#include <strings.h>
+#include <errno.h>
+#include <ipxe/http.h>
+
+/**
+ * Identify authentication scheme
+ *
+ * @v http		HTTP transaction
+ * @v name		Scheme name
+ * @ret auth		Authentication scheme, or NULL
+ */
+static struct http_authentication * http_authentication ( const char *name ) {
+	struct http_authentication *auth;
+
+	/* Identify authentication scheme */
+	for_each_table_entry ( auth, HTTP_AUTHENTICATIONS ) {
+		if ( strcasecmp ( name, auth->name ) == 0 )
+			return auth;
+	}
+
+	return NULL;
+}
+
+/** An HTTP "WWW-Authenticate" response field */
+struct http_www_authenticate_field {
+	/** Name */
+	const char *name;
+	/** Offset */
+	size_t offset;
+};
+
+/** Define an HTTP "WWW-Authenticate" response field */
+#define HTTP_WWW_AUTHENTICATE_FIELD( _name ) {				\
+		.name = #_name,						\
+		.offset = offsetof ( struct http_transaction,		\
+				     response.auth._name ),		\
+	}
+
+/**
+ * Set HTTP "WWW-Authenticate" response field value
+ *
+ * @v http		HTTP transaction
+ * @v field		Response field
+ * @v value		Field value
+ */
+static inline void
+http_www_auth_field ( struct http_transaction *http,
+		      struct http_www_authenticate_field *field, char *value ) {
+	char **ptr;
+
+	ptr = ( ( ( void * ) http ) + field->offset );
+	*ptr = value;
+}
+
+/** HTTP "WWW-Authenticate" fields */
+static struct http_www_authenticate_field http_www_auth_fields[] = {
+	HTTP_WWW_AUTHENTICATE_FIELD ( realm ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( qop ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( algorithm ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( nonce ),
+	HTTP_WWW_AUTHENTICATE_FIELD ( opaque ),
+};
+
+/**
+ * Parse HTTP "WWW-Authenticate" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_www_authenticate ( struct http_transaction *http,
+					 char *line ) {
+	struct http_www_authenticate_field *field;
+	char *name;
+	char *key;
+	char *value;
+	unsigned int i;
+
+	/* Get scheme name */
+	name = http_token ( &line, NULL );
+	if ( ! name ) {
+		DBGC ( http, "HTTP %p malformed WWW-Authenticate \"%s\"\n",
+		       http, value );
+		return -EPROTO;
+	}
+
+	/* Identify scheme */
+	http->response.auth.auth = http_authentication ( name );
+	if ( ! http->response.auth.auth ) {
+		DBGC ( http, "HTTP %p unrecognised authentication scheme "
+		       "\"%s\"\n", http, name );
+		return -ENOTSUP;
+	}
+
+	/* Process fields */
+	while ( ( key = http_token ( &line, &value ) ) ) {
+		for ( i = 0 ; i < ( sizeof ( http_www_auth_fields ) /
+				    sizeof ( http_www_auth_fields[0] ) ) ; i++){
+			field = &http_www_auth_fields[i];
+			if ( strcasecmp ( key, field->name ) == 0 )
+				http_www_auth_field ( http, field, value );
+		}
+	}
+
+	/* Allow HTTP request to be retried if the request had not
+	 * already tried authentication.
+	 */
+	if ( ! http->request.auth.auth )
+		http->response.flags |= HTTP_RESPONSE_RETRY;
+
+	return 0;
+}
+
+/** HTTP "WWW-Authenticate" header */
+struct http_response_header
+http_response_www_authenticate __http_response_header = {
+	.name = "WWW-Authenticate",
+	.parse = http_parse_www_authenticate,
+};
+
+/**
+ * Construct HTTP "Authorization" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_authorization ( struct http_transaction *http,
+				       char *buf, size_t len ) {
+	struct http_authentication *auth = http->request.auth.auth;
+	size_t used;
+	int auth_len;
+	int rc;
+
+	/* Do nothing unless we have an authentication scheme */
+	if ( ! auth )
+		return 0;
+
+	/* Construct header */
+	used = snprintf ( buf, len, "%s ", auth->name );
+	auth_len = auth->format ( http, ( buf + used ),
+				  ( ( used < len ) ? ( len - used ) : 0 ) );
+	if ( auth_len < 0 ) {
+		rc = auth_len;
+		return rc;
+	}
+	used += auth_len;
+
+	return used;
+}
+
+/** HTTP "Authorization" header */
+struct http_request_header http_request_authorization __http_request_header = {
+	.name = "Authorization",
+	.format = http_format_authorization,
+};
diff --git a/qemu/roms/ipxe/src/net/tcp/httpbasic.c b/qemu/roms/ipxe/src/net/tcp/httpbasic.c
new file mode 100644
index 000000000..7ed7de9e7
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/tcp/httpbasic.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) Basic authentication
+ *
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <ipxe/uri.h>
+#include <ipxe/base64.h>
+#include <ipxe/http.h>
+
+/* Disambiguate the various error causes */
+#define EACCES_USERNAME __einfo_error ( EINFO_EACCES_USERNAME )
+#define EINFO_EACCES_USERNAME						\
+	__einfo_uniqify ( EINFO_EACCES, 0x01,				\
+			  "No username available for Basic authentication" )
+
+/**
+ * Perform HTTP Basic authentication
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_basic_authenticate ( struct http_transaction *http ) {
+	struct http_request_auth *req = &http->request.auth;
+
+	/* Record username and password */
+	if ( ! http->uri->user ) {
+		DBGC ( http, "HTTP %p has no username for Basic "
+		       "authentication\n", http );
+		return -EACCES_USERNAME;
+	}
+	req->username = http->uri->user;
+	req->password = ( http->uri->password ? http->uri->password : "" );
+
+	return 0;
+}
+
+/**
+ * Construct HTTP "Authorization" header for Basic authentication
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_basic_auth ( struct http_transaction *http,
+				    char *buf, size_t len ) {
+	struct http_request_auth *req = &http->request.auth;
+	size_t user_pw_len = ( strlen ( req->username ) + 1 /* ":" */ +
+			       strlen ( req->password ) );
+	char user_pw[ user_pw_len + 1 /* NUL */ ];
+
+	/* Sanity checks */
+	assert ( req->username != NULL );
+	assert ( req->password != NULL );
+
+	/* Construct "user:password" string */
+	snprintf ( user_pw, sizeof ( user_pw ), "%s:%s",
+		   req->username, req->password );
+
+	/* Construct response */
+	return base64_encode ( user_pw, user_pw_len, buf, len );
+}
+
+/** HTTP Basic authentication scheme */
+struct http_authentication http_basic_auth __http_authentication = {
+	.name = "Basic",
+	.authenticate = http_basic_authenticate,
+	.format = http_format_basic_auth,
+};
+
+/* Drag in HTTP authentication support */
+REQUIRING_SYMBOL ( http_basic_auth );
+REQUIRE_OBJECT ( httpauth );
diff --git a/qemu/roms/ipxe/src/net/tcp/httpblock.c b/qemu/roms/ipxe/src/net/tcp/httpblock.c
new file mode 100644
index 000000000..e124ad2d6
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/tcp/httpblock.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) block device
+ *
+ */
+
+#include <stdint.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/blocktrans.h>
+#include <ipxe/blockdev.h>
+#include <ipxe/acpi.h>
+#include <ipxe/http.h>
+
+/** Block size used for HTTP block device requests */
+#define HTTP_BLKSIZE 512
+
+/**
+ * Read from block device
+ *
+ * @v http		HTTP transaction
+ * @v data		Data interface
+ * @v lba		Starting logical block address
+ * @v count		Number of logical blocks
+ * @v buffer		Data buffer
+ * @v len		Length of data buffer
+ * @ret rc		Return status code
+ */
+int http_block_read ( struct http_transaction *http, struct interface *data,
+		      uint64_t lba, unsigned int count, userptr_t buffer,
+		      size_t len ) {
+	struct http_request_range range;
+	int rc;
+
+	/* Sanity check */
+	assert ( len == ( count * HTTP_BLKSIZE ) );
+
+	/* Construct request range descriptor */
+	range.start = ( lba * HTTP_BLKSIZE );
+	range.len = len;
+
+	/* Start a range request to retrieve the block(s) */
+	if ( ( rc = http_open ( data, &http_get, http->uri, &range,
+				NULL ) ) != 0 )
+		goto err_open;
+
+	/* Insert block device translator */
+	if ( ( rc = block_translate ( data, buffer, len ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not insert block translator: %s\n",
+		       http, strerror ( rc ) );
+		goto err_translate;
+	}
+
+	return 0;
+
+ err_translate:
+	intf_restart ( data, rc );
+ err_open:
+	return rc;
+}
+
+/**
+ * Read block device capacity
+ *
+ * @v control		Control interface
+ * @v data		Data interface
+ * @ret rc		Return status code
+ */
+int http_block_read_capacity ( struct http_transaction *http,
+			       struct interface *data ) {
+	int rc;
+
+	/* Start a HEAD request to retrieve the capacity */
+	if ( ( rc = http_open ( data, &http_head, http->uri, NULL,
+				NULL ) ) != 0 )
+		goto err_open;
+
+	/* Insert block device translator */
+	if ( ( rc = block_translate ( data, UNULL, HTTP_BLKSIZE ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not insert block translator: %s\n",
+		       http, strerror ( rc ) );
+		goto err_translate;
+	}
+
+	return 0;
+
+ err_translate:
+	intf_restart ( data, rc );
+ err_open:
+	return rc;
+}
+
+/**
+ * Describe device in ACPI table
+ *
+ * @v http		HTTP transaction
+ * @v acpi		ACPI table
+ * @v len		Length of ACPI table
+ * @ret rc		Return status code
+ */
+int http_acpi_describe ( struct http_transaction *http,
+			 struct acpi_description_header *acpi, size_t len ) {
+
+	DBGC ( http, "HTTP %p cannot yet describe device in an ACPI table\n",
+	       http );
+	( void ) acpi;
+	( void ) len;
+	return 0;
+}
diff --git a/qemu/roms/ipxe/src/net/tcp/httpconn.c b/qemu/roms/ipxe/src/net/tcp/httpconn.c
new file mode 100644
index 000000000..7e4877b7b
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/tcp/httpconn.c
@@ -0,0 +1,309 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) connection management
+ *
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <byteswap.h>
+#include <ipxe/tcpip.h>
+#include <ipxe/uri.h>
+#include <ipxe/timer.h>
+#include <ipxe/xfer.h>
+#include <ipxe/open.h>
+#include <ipxe/pool.h>
+#include <ipxe/http.h>
+
+/** HTTP pooled connection expiry time */
+#define HTTP_CONN_EXPIRY ( 10 * TICKS_PER_SEC )
+
+/** HTTP connection pool */
+static LIST_HEAD ( http_connection_pool );
+
+/**
+ * Identify HTTP scheme
+ *
+ * @v uri		URI
+ * @ret scheme		HTTP scheme, or NULL
+ */
+static struct http_scheme * http_scheme ( struct uri *uri ) {
+	struct http_scheme *scheme;
+
+	/* Sanity check */
+	if ( ! uri->scheme )
+		return NULL;
+
+	/* Identify scheme */
+	for_each_table_entry ( scheme, HTTP_SCHEMES ) {
+		if ( strcmp ( uri->scheme, scheme->name ) == 0 )
+			return scheme;
+	}
+
+	return NULL;
+}
+
+/**
+ * Free HTTP connection
+ *
+ * @v refcnt		Reference count
+ */
+static void http_conn_free ( struct refcnt *refcnt ) {
+	struct http_connection *conn =
+		container_of ( refcnt, struct http_connection, refcnt );
+
+	/* Free connection */
+	uri_put ( conn->uri );
+	free ( conn );
+}
+
+/**
+ * Close HTTP connection
+ *
+ * @v conn		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_close ( struct http_connection *conn, int rc ) {
+
+	/* Remove from connection pool, if applicable */
+	pool_del ( &conn->pool );
+
+	/* Shut down interfaces */
+	intf_shutdown ( &conn->socket, rc );
+	intf_shutdown ( &conn->xfer, rc );
+	if ( rc == 0 ) {
+		DBGC2 ( conn, "HTTPCONN %p closed %s://%s\n",
+			conn, conn->scheme->name, conn->uri->host );
+	} else {
+		DBGC ( conn, "HTTPCONN %p closed %s://%s: %s\n",
+		       conn, conn->scheme->name, conn->uri->host,
+		       strerror ( rc ) );
+	}
+}
+
+/**
+ * Disconnect idle HTTP connection
+ *
+ * @v pool		Pooled connection
+ */
+static void http_conn_expired ( struct pooled_connection *pool ) {
+	struct http_connection *conn =
+		container_of ( pool, struct http_connection, pool );
+
+	/* Close connection */
+	http_conn_close ( conn, 0 /* Not an error to close idle connection */ );
+}
+
+/**
+ * Receive data from transport layer interface
+ *
+ * @v http		HTTP connection
+ * @v iobuf		I/O buffer
+ * @v meta		Transfer metadata
+ * @ret rc		Return status code
+ */
+static int http_conn_socket_deliver ( struct http_connection *conn,
+				      struct io_buffer *iobuf,
+				      struct xfer_metadata *meta ) {
+
+	/* Mark connection as alive */
+	pool_alive ( &conn->pool );
+
+	/* Pass on to data transfer interface */
+	return xfer_deliver ( &conn->xfer, iobuf, meta );
+}
+
+/**
+ * Close HTTP connection transport layer interface
+ *
+ * @v http		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_socket_close ( struct http_connection *conn, int rc ) {
+
+	/* If we are reopenable (i.e. we are a recycled connection
+	 * from the connection pool, and we have received no data from
+	 * the underlying socket since we were pooled), then suggest
+	 * that the client should reopen the connection.
+	 */
+	if ( pool_is_reopenable ( &conn->pool ) )
+		pool_reopen ( &conn->xfer );
+
+	/* Close the connection */
+	http_conn_close ( conn, rc );
+}
+
+/**
+ * Recycle this connection after closing
+ *
+ * @v http		HTTP connection
+ */
+static void http_conn_xfer_recycle ( struct http_connection *conn ) {
+
+	/* Mark connection as recyclable */
+	pool_recyclable ( &conn->pool );
+	DBGC2 ( conn, "HTTPCONN %p keepalive enabled\n", conn );
+}
+
+/**
+ * Close HTTP connection data transfer interface
+ *
+ * @v conn		HTTP connection
+ * @v rc		Reason for close
+ */
+static void http_conn_xfer_close ( struct http_connection *conn, int rc ) {
+
+	/* Add to the connection pool if keepalive is enabled and no
+	 * error occurred.
+	 */
+	if ( ( rc == 0 ) && pool_is_recyclable ( &conn->pool ) ) {
+		intf_restart ( &conn->xfer, rc );
+		pool_add ( &conn->pool, &http_connection_pool,
+			   HTTP_CONN_EXPIRY );
+		DBGC2 ( conn, "HTTPCONN %p pooled %s://%s\n",
+			conn, conn->scheme->name, conn->uri->host );
+		return;
+	}
+
+	/* Otherwise, close the connection */
+	http_conn_close ( conn, rc );
+}
+
+/** HTTP connection socket interface operations */
+static struct interface_operation http_conn_socket_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_connection *,
+		  http_conn_socket_deliver ),
+	INTF_OP ( intf_close, struct http_connection *,
+		  http_conn_socket_close ),
+};
+
+/** HTTP connection socket interface descriptor */
+static struct interface_descriptor http_conn_socket_desc =
+	INTF_DESC_PASSTHRU ( struct http_connection, socket,
+			     http_conn_socket_operations, xfer );
+
+/** HTTP connection data transfer interface operations */
+static struct interface_operation http_conn_xfer_operations[] = {
+	INTF_OP ( pool_recycle, struct http_connection *,
+		  http_conn_xfer_recycle ),
+	INTF_OP ( intf_close, struct http_connection *,
+		  http_conn_xfer_close ),
+};
+
+/** HTTP connection data transfer interface descriptor */
+static struct interface_descriptor http_conn_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_connection, xfer,
+			     http_conn_xfer_operations, socket );
+
+/**
+ * Connect to an HTTP server
+ *
+ * @v xfer		Data transfer interface
+ * @v uri		Connection URI
+ * @ret rc		Return status code
+ *
+ * HTTP connections are pooled.  The caller should be prepared to
+ * receive a pool_reopen() message.
+ */
+int http_connect ( struct interface *xfer, struct uri *uri ) {
+	struct http_connection *conn;
+	struct http_scheme *scheme;
+	struct sockaddr_tcpip server;
+	struct interface *socket;
+	int rc;
+
+	/* Identify scheme */
+	scheme = http_scheme ( uri );
+	if ( ! scheme )
+		return -ENOTSUP;
+
+	/* Sanity check */
+	if ( ! uri->host )
+		return -EINVAL;
+
+	/* Look for a reusable connection in the pool */
+	list_for_each_entry ( conn, &http_connection_pool, pool.list ) {
+
+		/* Sanity checks */
+		assert ( conn->uri != NULL );
+		assert ( conn->uri->host != NULL );
+
+		/* Reuse connection, if possible */
+		if ( ( scheme == conn->scheme ) &&
+		     ( strcmp ( uri->host, conn->uri->host ) == 0 ) ) {
+
+			/* Remove from connection pool, stop timer,
+			 * attach to parent interface, and return.
+			 */
+			pool_del ( &conn->pool );
+			intf_plug_plug ( &conn->xfer, xfer );
+			DBGC2 ( conn, "HTTPCONN %p reused %s://%s\n",
+				conn, conn->scheme->name, conn->uri->host );
+			return 0;
+		}
+	}
+
+	/* Allocate and initialise structure */
+	conn = zalloc ( sizeof ( *conn ) );
+	ref_init ( &conn->refcnt, http_conn_free );
+	conn->uri = uri_get ( uri );
+	conn->scheme = scheme;
+	intf_init ( &conn->socket, &http_conn_socket_desc, &conn->refcnt );
+	intf_init ( &conn->xfer, &http_conn_xfer_desc, &conn->refcnt );
+	pool_init ( &conn->pool, http_conn_expired, &conn->refcnt );
+
+	/* Open socket */
+	memset ( &server, 0, sizeof ( server ) );
+	server.st_port = htons ( uri_port ( uri, scheme->port ) );
+	socket = &conn->socket;
+	if ( scheme->filter &&
+	     ( ( rc = scheme->filter ( socket, uri->host, &socket ) ) != 0 ) )
+		goto err_filter;
+	if ( ( rc = xfer_open_named_socket ( socket, SOCK_STREAM,
+					     ( struct sockaddr * ) &server,
+					     uri->host, NULL ) ) != 0 )
+		goto err_open;
+
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( &conn->xfer, xfer );
+	ref_put ( &conn->refcnt );
+
+	DBGC2 ( conn, "HTTPCONN %p created %s://%s:%d\n", conn,
+		conn->scheme->name, conn->uri->host, ntohs ( server.st_port ) );
+	return 0;
+
+ err_open:
+ err_filter:
+	DBGC2 ( conn, "HTTPCONN %p could not create %s://%s: %s\n",
+		conn, conn->scheme->name, conn->uri->host, strerror ( rc ) );
+	http_conn_close ( conn, rc );
+	ref_put ( &conn->refcnt );
+	return rc;
+}
diff --git a/qemu/roms/ipxe/src/net/tcp/httpcore.c b/qemu/roms/ipxe/src/net/tcp/httpcore.c
index 1d1953e61..af3ca9780 100644
--- a/qemu/roms/ipxe/src/net/tcp/httpcore.c
+++ b/qemu/roms/ipxe/src/net/tcp/httpcore.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -40,35 +44,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/iobuf.h>
 #include <ipxe/xfer.h>
 #include <ipxe/open.h>
-#include <ipxe/socket.h>
-#include <ipxe/tcpip.h>
 #include <ipxe/process.h>
 #include <ipxe/retry.h>
 #include <ipxe/timer.h>
 #include <ipxe/linebuf.h>
-#include <ipxe/base64.h>
-#include <ipxe/base16.h>
-#include <ipxe/md5.h>
+#include <ipxe/xferbuf.h>
 #include <ipxe/blockdev.h>
 #include <ipxe/acpi.h>
 #include <ipxe/version.h>
 #include <ipxe/params.h>
 #include <ipxe/profile.h>
+#include <ipxe/vsprintf.h>
 #include <ipxe/http.h>
 
 /* Disambiguate the various error causes */
 #define EACCES_401 __einfo_error ( EINFO_EACCES_401 )
 #define EINFO_EACCES_401 \
 	__einfo_uniqify ( EINFO_EACCES, 0x01, "HTTP 401 Unauthorized" )
-#define EIO_OTHER __einfo_error ( EINFO_EIO_OTHER )
-#define EINFO_EIO_OTHER \
-	__einfo_uniqify ( EINFO_EIO, 0x01, "Unrecognised HTTP response code" )
-#define EIO_CONTENT_LENGTH __einfo_error ( EINFO_EIO_CONTENT_LENGTH )
-#define EINFO_EIO_CONTENT_LENGTH \
-	__einfo_uniqify ( EINFO_EIO, 0x02, "Content length mismatch" )
-#define EINVAL_RESPONSE __einfo_error ( EINFO_EINVAL_RESPONSE )
-#define EINFO_EINVAL_RESPONSE \
-	__einfo_uniqify ( EINFO_EINVAL, 0x01, "Invalid content length" )
+#define EINVAL_STATUS __einfo_error ( EINFO_EINVAL_STATUS )
+#define EINFO_EINVAL_STATUS \
+	__einfo_uniqify ( EINFO_EINVAL, 0x01, "Invalid status line" )
 #define EINVAL_HEADER __einfo_error ( EINFO_EINVAL_HEADER )
 #define EINFO_EINVAL_HEADER \
 	__einfo_uniqify ( EINFO_EINVAL, 0x02, "Invalid header" )
@@ -78,9 +73,27 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINVAL_CHUNK_LENGTH __einfo_error ( EINFO_EINVAL_CHUNK_LENGTH )
 #define EINFO_EINVAL_CHUNK_LENGTH \
 	__einfo_uniqify ( EINFO_EINVAL, 0x04, "Invalid chunk length" )
+#define EIO_OTHER __einfo_error ( EINFO_EIO_OTHER )
+#define EINFO_EIO_OTHER \
+	__einfo_uniqify ( EINFO_EIO, 0x01, "Unrecognised HTTP response code" )
+#define EIO_CONTENT_LENGTH __einfo_error ( EINFO_EIO_CONTENT_LENGTH )
+#define EINFO_EIO_CONTENT_LENGTH \
+	__einfo_uniqify ( EINFO_EIO, 0x02, "Content length mismatch" )
+#define EIO_4XX __einfo_error ( EINFO_EIO_4XX )
+#define EINFO_EIO_4XX \
+	__einfo_uniqify ( EINFO_EIO, 0x04, "HTTP 4xx Client Error" )
+#define EIO_5XX __einfo_error ( EINFO_EIO_5XX )
+#define EINFO_EIO_5XX \
+	__einfo_uniqify ( EINFO_EIO, 0x05, "HTTP 5xx Server Error" )
 #define ENOENT_404 __einfo_error ( EINFO_ENOENT_404 )
 #define EINFO_ENOENT_404 \
 	__einfo_uniqify ( EINFO_ENOENT, 0x01, "HTTP 404 Not Found" )
+#define ENOTSUP_CONNECTION __einfo_error ( EINFO_ENOTSUP_CONNECTION )
+#define EINFO_ENOTSUP_CONNECTION \
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x01, "Unsupported connection header" )
+#define ENOTSUP_TRANSFER __einfo_error ( EINFO_ENOTSUP_TRANSFER )
+#define EINFO_ENOTSUP_TRANSFER \
+	__einfo_uniqify ( EINFO_ENOTSUP, 0x02, "Unsupported transfer encoding" )
 #define EPERM_403 __einfo_error ( EINFO_EPERM_403 )
 #define EINFO_EPERM_403 \
 	__einfo_uniqify ( EINFO_EPERM, 0x01, "HTTP 403 Forbidden" )
@@ -88,9 +101,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define EINFO_EPROTO_UNSOLICITED \
 	__einfo_uniqify ( EINFO_EPROTO, 0x01, "Unsolicited data" )
 
-/** Block size used for HTTP block device request */
-#define HTTP_BLKSIZE 512
-
 /** Retry delay used when we cannot understand the Retry-After header */
 #define HTTP_RETRY_SECONDS 5
 
@@ -100,1069 +110,1711 @@ static struct profiler http_rx_profiler __profiler = { .name = "http.rx" };
 /** Data transfer profiler */
 static struct profiler http_xfer_profiler __profiler = { .name = "http.xfer" };
 
-/** HTTP flags */
-enum http_flags {
-	/** Request is waiting to be transmitted */
-	HTTP_TX_PENDING = 0x0001,
-	/** Fetch header only */
-	HTTP_HEAD_ONLY = 0x0002,
-	/** Client would like to keep connection alive */
-	HTTP_CLIENT_KEEPALIVE = 0x0004,
-	/** Server will keep connection alive */
-	HTTP_SERVER_KEEPALIVE = 0x0008,
-	/** Discard the current request and try again */
-	HTTP_TRY_AGAIN = 0x0010,
-	/** Provide Basic authentication details */
-	HTTP_BASIC_AUTH = 0x0020,
-	/** Provide Digest authentication details */
-	HTTP_DIGEST_AUTH = 0x0040,
-	/** Socket must be reopened */
-	HTTP_REOPEN_SOCKET = 0x0080,
+static struct http_state http_request;
+static struct http_state http_headers;
+static struct http_state http_trailers;
+static struct http_transfer_encoding http_transfer_identity;
+
+/******************************************************************************
+ *
+ * Methods
+ *
+ ******************************************************************************
+ */
+
+/** HTTP HEAD method */
+struct http_method http_head = {
+	.name = "HEAD",
 };
 
-/** HTTP receive state */
-enum http_rx_state {
-	HTTP_RX_RESPONSE = 0,
-	HTTP_RX_HEADER,
-	HTTP_RX_CHUNK_LEN,
-	/* In HTTP_RX_DATA, it is acceptable for the server to close
-	 * the connection (unless we are in the middle of a chunked
-	 * transfer).
-	 */
-	HTTP_RX_DATA,
-	/* In the following states, it is acceptable for the server to
-	 * close the connection.
-	 */
-	HTTP_RX_TRAILER,
-	HTTP_RX_IDLE,
-	HTTP_RX_DEAD,
+/** HTTP GET method */
+struct http_method http_get = {
+	.name = "GET",
 };
 
+/** HTTP POST method */
+struct http_method http_post = {
+	.name = "POST",
+};
+
+/******************************************************************************
+ *
+ * Utility functions
+ *
+ ******************************************************************************
+ */
+
 /**
- * An HTTP request
+ * Handle received HTTP line-buffered data
  *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v linebuf		Line buffer
+ * @ret rc		Return status code
  */
-struct http_request {
-	/** Reference count */
-	struct refcnt refcnt;
-	/** Data transfer interface */
-	struct interface xfer;
-	/** Partial transfer interface */
-	struct interface partial;
-
-	/** URI being fetched */
-	struct uri *uri;
-	/** Default port */
-	unsigned int default_port;
-	/** Filter (if any) */
-	int ( * filter ) ( struct interface *xfer,
-			   const char *name,
-			   struct interface **next );
-	/** Transport layer interface */
-	struct interface socket;
-
-	/** Flags */
-	unsigned int flags;
-	/** Starting offset of partial transfer (if applicable) */
-	size_t partial_start;
-	/** Length of partial transfer (if applicable) */
-	size_t partial_len;
-
-	/** TX process */
-	struct process process;
-
-	/** RX state */
-	enum http_rx_state rx_state;
-	/** Response code */
-	unsigned int code;
-	/** Received length */
-	size_t rx_len;
-	/** Length remaining (or 0 if unknown) */
-	size_t remaining;
-	/** HTTP is using Transfer-Encoding: chunked */
-	int chunked;
-	/** Current chunk length remaining (if applicable) */
-	size_t chunk_remaining;
-	/** Line buffer for received header lines */
-	struct line_buffer linebuf;
-	/** Receive data buffer (if applicable) */
-	userptr_t rx_buffer;
-
-	/** Authentication realm (if any) */
-	char *auth_realm;
-	/** Authentication nonce (if any) */
-	char *auth_nonce;
-	/** Authentication opaque string (if any) */
-	char *auth_opaque;
-
-	/** Request retry timer */
-	struct retry_timer timer;
-	/** Retry delay (in timer ticks) */
-	unsigned long retry_delay;
-};
+static int http_rx_linebuf ( struct http_transaction *http,
+			     struct io_buffer *iobuf,
+			     struct line_buffer *linebuf ) {
+	int consumed;
+	int rc;
+
+	/* Buffer received line */
+	consumed = line_buffer ( linebuf, iobuf->data, iob_len ( iobuf ) );
+	if ( consumed < 0 ) {
+		rc = consumed;
+		DBGC ( http, "HTTP %p could not buffer line: %s\n",
+		       http, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Consume line */
+	iob_pull ( iobuf, consumed );
+
+	return 0;
+}
 
 /**
- * Free HTTP request
+ * Get HTTP response token
  *
- * @v refcnt		Reference counter
+ * @v line		Line position
+ * @v value		Token value to fill in (if any)
+ * @ret token		Token, or NULL
+ */
+char * http_token ( char **line, char **value ) {
+	char *token;
+	char quote = '\0';
+	char c;
+
+	/* Avoid returning uninitialised data */
+	if ( value )
+		*value = NULL;
+
+	/* Skip any initial whitespace */
+	while ( isspace ( **line ) )
+		(*line)++;
+
+	/* Check for end of line and record token position */
+	if ( ! **line )
+		return NULL;
+	token = *line;
+
+	/* Scan for end of token */
+	while ( ( c = **line ) ) {
+
+		/* Terminate if we hit an unquoted whitespace */
+		if ( isspace ( c ) && ! quote )
+			break;
+
+		/* Terminate if we hit a closing quote */
+		if ( c == quote )
+			break;
+
+		/* Check for value separator */
+		if ( value && ( ! *value ) && ( c == '=' ) ) {
+
+			/* Terminate key portion of token */
+			*((*line)++) = '\0';
+
+			/* Check for quote character */
+			c = **line;
+			if ( ( c == '"' ) || ( c == '\'' ) ) {
+				quote = c;
+				(*line)++;
+			}
+
+			/* Record value portion of token */
+			*value = *line;
+
+		} else {
+
+			/* Move to next character */
+			(*line)++;
+		}
+	}
+
+	/* Terminate token, if applicable */
+	if ( c )
+		*((*line)++) = '\0';
+
+	return token;
+}
+
+/******************************************************************************
+ *
+ * Transactions
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Free HTTP transaction
+ *
+ * @v refcnt		Reference count
  */
 static void http_free ( struct refcnt *refcnt ) {
-	struct http_request *http =
-		container_of ( refcnt, struct http_request, refcnt );
+	struct http_transaction *http =
+		container_of ( refcnt, struct http_transaction, refcnt );
 
-	uri_put ( http->uri );
+	empty_line_buffer ( &http->response.headers );
 	empty_line_buffer ( &http->linebuf );
-	free ( http->auth_realm );
-	free ( http->auth_nonce );
-	free ( http->auth_opaque );
+	uri_put ( http->uri );
 	free ( http );
-};
+}
 
 /**
- * Close HTTP request
+ * Close HTTP transaction
  *
- * @v http		HTTP request
- * @v rc		Return status code
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static void http_close ( struct http_request *http, int rc ) {
-
-	/* Prevent further processing of any current packet */
-	http->rx_state = HTTP_RX_DEAD;
+static void http_close ( struct http_transaction *http, int rc ) {
 
-	/* Prevent reconnection */
-	http->flags &= ~HTTP_CLIENT_KEEPALIVE;
-
-	/* Remove process */
+	/* Stop process */
 	process_del ( &http->process );
 
-	/* Close all data transfer interfaces */
-	intf_shutdown ( &http->socket, rc );
-	intf_shutdown ( &http->partial, rc );
+	/* Stop timer */
+	stop_timer ( &http->timer );
+
+	/* Close all interfaces, allowing for the fact that the
+	 * content-decoded and transfer-decoded interfaces may be
+	 * connected to the same object.
+	 */
+	intf_shutdown ( &http->conn, rc );
+	intf_nullify ( &http->transfer );
+	intf_shutdown ( &http->content, rc );
+	intf_shutdown ( &http->transfer, rc );
 	intf_shutdown ( &http->xfer, rc );
 }
 
 /**
- * Open HTTP socket
+ * Close HTTP transaction with error (even if none specified)
  *
- * @v http		HTTP request
- * @ret rc		Return status code
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static int http_socket_open ( struct http_request *http ) {
-	struct uri *uri = http->uri;
-	struct sockaddr_tcpip server;
-	struct interface *socket;
+static void http_close_error ( struct http_transaction *http, int rc ) {
+
+	/* Treat any close as an error */
+	http_close ( http, ( rc ? rc : -EPIPE ) );
+}
+
+/**
+ * Reopen stale HTTP connection
+ *
+ * @v http		HTTP transaction
+ */
+static void http_reopen ( struct http_transaction *http ) {
 	int rc;
 
-	/* Open socket */
-	memset ( &server, 0, sizeof ( server ) );
-	server.st_port = htons ( uri_port ( uri, http->default_port ) );
-	socket = &http->socket;
-	if ( http->filter ) {
-		if ( ( rc = http->filter ( socket, uri->host, &socket ) ) != 0 )
-			return rc;
+	/* Close existing connection */
+	intf_restart ( &http->conn, -ECANCELED );
+
+	/* Reopen connection */
+	if ( ( rc = http_connect ( &http->conn, http->uri ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not reconnect: %s\n",
+		       http, strerror ( rc ) );
+		goto err_connect;
 	}
-	if ( ( rc = xfer_open_named_socket ( socket, SOCK_STREAM,
-					     ( struct sockaddr * ) &server,
-					     uri->host, NULL ) ) != 0 )
-		return rc;
 
-	return 0;
+	/* Reset state */
+	http->state = &http_request;
+
+	/* Reschedule transmission process */
+	process_add ( &http->process );
+
+	return;
+
+ err_connect:
+	http_close ( http, rc );
 }
 
 /**
- * Retry HTTP request
+ * Handle retry timer expiry
  *
  * @v timer		Retry timer
- * @v fail		Failure indicator
+ * @v over		Failure indicator
+ */
+static void http_expired ( struct retry_timer *timer, int over __unused ) {
+	struct http_transaction *http =
+		container_of ( timer, struct http_transaction, timer );
+
+	/* Reopen connection */
+	http_reopen ( http );
+}
+
+/**
+ * HTTP transmit process
+ *
+ * @v http		HTTP transaction
  */
-static void http_retry ( struct retry_timer *timer, int fail __unused ) {
-	struct http_request *http =
-		container_of ( timer, struct http_request, timer );
+static void http_step ( struct http_transaction *http ) {
 	int rc;
 
-	/* Reopen socket if required */
-	if ( http->flags & HTTP_REOPEN_SOCKET ) {
-		http->flags &= ~HTTP_REOPEN_SOCKET;
-		DBGC ( http, "HTTP %p reopening connection\n", http );
-		if ( ( rc = http_socket_open ( http ) ) != 0 ) {
-			http_close ( http, rc );
-			return;
+	/* Do nothing if we have nothing to transmit */
+	if ( ! http->state->tx )
+		return;
+
+	/* Do nothing until connection is ready */
+	if ( ! xfer_window ( &http->conn ) )
+		return;
+
+	/* Do nothing until data transfer interface is ready */
+	if ( ! xfer_window ( &http->xfer ) )
+		return;
+
+	/* Transmit data */
+	if ( ( rc = http->state->tx ( http ) ) != 0 )
+		goto err;
+
+	return;
+
+ err:
+	http_close ( http, rc );
+}
+
+/**
+ * Handle received HTTP data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v meta		Transfer metadata
+ * @ret rc		Return status code
+ *
+ * This function takes ownership of the I/O buffer.
+ */
+static int http_conn_deliver ( struct http_transaction *http,
+			       struct io_buffer *iobuf,
+			       struct xfer_metadata *meta __unused ) {
+	int rc;
+
+	/* Handle received data */
+	profile_start ( &http_rx_profiler );
+	while ( iobuf && iob_len ( iobuf ) ) {
+
+		/* Sanity check */
+		if ( ( ! http->state ) || ( ! http->state->rx ) ) {
+			DBGC ( http, "HTTP %p unexpected data\n", http );
+			rc = -EPROTO_UNSOLICITED;
+			goto err;
 		}
-	}
 
-	/* Retry the request if applicable */
-	if ( http->flags & HTTP_TRY_AGAIN ) {
-		http->flags &= ~HTTP_TRY_AGAIN;
-		DBGC ( http, "HTTP %p retrying request\n", http );
-		http->flags |= HTTP_TX_PENDING;
-		http->rx_state = HTTP_RX_RESPONSE;
-		process_add ( &http->process );
+		/* Receive (some) data */
+		if ( ( rc = http->state->rx ( http, &iobuf ) ) != 0 )
+			goto err;
 	}
+
+	/* Free I/O buffer, if applicable */
+	free_iob ( iobuf );
+
+	profile_stop ( &http_rx_profiler );
+	return 0;
+
+ err:
+	free_iob ( iobuf );
+	http_close ( http, rc );
+	return rc;
 }
 
 /**
- * Mark HTTP request as completed successfully
+ * Handle server connection close
  *
- * @v http		HTTP request
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
  */
-static void http_done ( struct http_request *http ) {
+static void http_conn_close ( struct http_transaction *http, int rc ) {
 
-	/* If we are not at an appropriate stage of the protocol
-	 * (including being in the middle of a chunked transfer),
-	 * force an error.
-	 */
-	if ( ( http->rx_state < HTTP_RX_DATA ) || ( http->chunked != 0 ) ) {
-		DBGC ( http, "HTTP %p connection closed unexpectedly in state "
-		       "%d\n", http, http->rx_state );
-		http_close ( http, -ECONNRESET );
-		return;
-	}
+	/* Sanity checks */
+	assert ( http->state != NULL );
+	assert ( http->state->close != NULL );
+
+	/* Restart server connection interface */
+	intf_restart ( &http->conn, rc );
+
+	/* Hand off to state-specific method */
+	http->state->close ( http, rc );
+}
 
-	/* If we had a Content-Length, and the received content length
-	 * isn't correct, force an error
+/**
+ * Handle received content-decoded data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer
+ * @v meta		Data transfer metadata
+ */
+static int http_content_deliver ( struct http_transaction *http,
+				  struct io_buffer *iobuf,
+				  struct xfer_metadata *meta ) {
+	int rc;
+
+	/* Ignore content if this is anything other than a successful
+	 * transfer.
 	 */
-	if ( http->remaining != 0 ) {
-		DBGC ( http, "HTTP %p incorrect length %zd, should be %zd\n",
-		       http, http->rx_len, ( http->rx_len + http->remaining ) );
-		http_close ( http, -EIO_CONTENT_LENGTH );
-		return;
+	if ( http->response.rc != 0 ) {
+		free_iob ( iobuf );
+		return 0;
 	}
 
-	/* Enter idle state */
-	http->rx_state = HTTP_RX_IDLE;
-	http->rx_len = 0;
-	assert ( http->remaining == 0 );
-	assert ( http->chunked == 0 );
-	assert ( http->chunk_remaining == 0 );
+	/* Deliver to data transfer interface */
+	profile_start ( &http_xfer_profiler );
+	if ( ( rc = xfer_deliver ( &http->xfer, iob_disown ( iobuf ),
+				   meta ) ) != 0 )
+		return rc;
+	profile_stop ( &http_xfer_profiler );
 
-	/* Close partial transfer interface */
-	if ( ! ( http->flags & HTTP_TRY_AGAIN ) )
-		intf_restart ( &http->partial, 0 );
+	return 0;
+}
 
-	/* Close everything unless we want to keep the connection alive */
-	if ( ! ( http->flags & ( HTTP_CLIENT_KEEPALIVE | HTTP_TRY_AGAIN ) ) ) {
-		http_close ( http, 0 );
-		return;
-	}
+/**
+ * Get underlying data transfer buffer
+ *
+ * @v http		HTTP transaction
+ * @ret xferbuf		Data transfer buffer, or NULL on error
+ */
+static struct xfer_buffer *
+http_content_buffer ( struct http_transaction *http ) {
 
-	/* If the server is not intending to keep the connection
-	 * alive, then close the socket and mark it as requiring
-	 * reopening.
+	/* Deny access to the data transfer buffer if this is anything
+	 * other than a successful transfer.
 	 */
-	if ( ! ( http->flags & HTTP_SERVER_KEEPALIVE ) ) {
-		intf_restart ( &http->socket, 0 );
-		http->flags &= ~HTTP_SERVER_KEEPALIVE;
-		http->flags |= HTTP_REOPEN_SOCKET;
-	}
+	if ( http->response.rc != 0 )
+		return NULL;
 
-	/* Start request retry timer */
-	start_timer_fixed ( &http->timer, http->retry_delay );
-	http->retry_delay = 0;
+	/* Hand off to data transfer interface */
+	return xfer_buffer ( &http->xfer );
 }
 
 /**
- * Convert HTTP response code to return status code
+ * Read from block device (when HTTP block device support is not present)
  *
- * @v response		HTTP response code
+ * @v http		HTTP transaction
+ * @v data		Data interface
+ * @v lba		Starting logical block address
+ * @v count		Number of logical blocks
+ * @v buffer		Data buffer
+ * @v len		Length of data buffer
  * @ret rc		Return status code
  */
-static int http_response_to_rc ( unsigned int response ) {
-	switch ( response ) {
-	case 200:
-	case 206:
-	case 301:
-	case 302:
-	case 303:
-		return 0;
-	case 404:
-		return -ENOENT_404;
-	case 403:
-		return -EPERM_403;
-	case 401:
-		return -EACCES_401;
-	default:
-		return -EIO_OTHER;
-	}
+__weak int http_block_read ( struct http_transaction *http __unused,
+			     struct interface *data __unused,
+			     uint64_t lba __unused, unsigned int count __unused,
+			     userptr_t buffer __unused, size_t len __unused ) {
+
+	return -ENOTSUP;
 }
 
 /**
- * Handle HTTP response
+ * Read block device capacity (when HTTP block device support is not present)
  *
- * @v http		HTTP request
- * @v response		HTTP response
+ * @v control		Control interface
+ * @v data		Data interface
  * @ret rc		Return status code
  */
-static int http_rx_response ( struct http_request *http, char *response ) {
-	char *spc;
-
-	DBGC ( http, "HTTP %p response \"%s\"\n", http, response );
+__weak int http_block_read_capacity ( struct http_transaction *http __unused,
+				      struct interface *data __unused ) {
 
-	/* Check response starts with "HTTP/" */
-	if ( strncmp ( response, "HTTP/", 5 ) != 0 )
-		return -EINVAL_RESPONSE;
+	return -ENOTSUP;
+}
 
-	/* Locate and store response code */
-	spc = strchr ( response, ' ' );
-	if ( ! spc )
-		return -EINVAL_RESPONSE;
-	http->code = strtoul ( spc, NULL, 10 );
+/**
+ * Describe device in ACPI table (when HTTP block device support is not present)
+ *
+ * @v http		HTTP transaction
+ * @v acpi		ACPI table
+ * @v len		Length of ACPI table
+ * @ret rc		Return status code
+ */
+__weak int http_acpi_describe ( struct http_transaction *http __unused,
+				struct acpi_description_header *acpi __unused,
+				size_t len __unused ) {
 
-	/* Move to receive headers */
-	http->rx_state = ( ( http->flags & HTTP_HEAD_ONLY ) ?
-			   HTTP_RX_TRAILER : HTTP_RX_HEADER );
-	return 0;
+	return -ENOTSUP;
 }
 
+/** HTTP data transfer interface operations */
+static struct interface_operation http_xfer_operations[] = {
+	INTF_OP ( block_read, struct http_transaction *, http_block_read ),
+	INTF_OP ( block_read_capacity, struct http_transaction *,
+		  http_block_read_capacity ),
+	INTF_OP ( acpi_describe, struct http_transaction *,
+		  http_acpi_describe ),
+	INTF_OP ( xfer_window_changed, struct http_transaction *, http_step ),
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP data transfer interface descriptor */
+static struct interface_descriptor http_xfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, xfer,
+			     http_xfer_operations, content );
+
+/** HTTP content-decoded interface operations */
+static struct interface_operation http_content_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_transaction *,
+		  http_content_deliver ),
+	INTF_OP ( xfer_buffer, struct http_transaction *, http_content_buffer ),
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP content-decoded interface descriptor */
+static struct interface_descriptor http_content_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, content,
+			     http_content_operations, xfer );
+
+/** HTTP transfer-decoded interface operations */
+static struct interface_operation http_transfer_operations[] = {
+	INTF_OP ( intf_close, struct http_transaction *, http_close ),
+};
+
+/** HTTP transfer-decoded interface descriptor */
+static struct interface_descriptor http_transfer_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, transfer,
+			     http_transfer_operations, conn );
+
+/** HTTP server connection interface operations */
+static struct interface_operation http_conn_operations[] = {
+	INTF_OP ( xfer_deliver, struct http_transaction *, http_conn_deliver ),
+	INTF_OP ( xfer_window_changed, struct http_transaction *, http_step ),
+	INTF_OP ( pool_reopen, struct http_transaction *, http_reopen ),
+	INTF_OP ( intf_close, struct http_transaction *, http_conn_close ),
+};
+
+/** HTTP server connection interface descriptor */
+static struct interface_descriptor http_conn_desc =
+	INTF_DESC_PASSTHRU ( struct http_transaction, conn,
+			     http_conn_operations, transfer );
+
+/** HTTP process descriptor */
+static struct process_descriptor http_process_desc =
+	PROC_DESC_ONCE ( struct http_transaction, process, http_step );
+
 /**
- * Handle HTTP Location header
+ * Open HTTP transaction
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v xfer		Data transfer interface
+ * @v method		Request method
+ * @v uri		Request URI
+ * @v range		Content range (if any)
+ * @v content		Request content (if any)
  * @ret rc		Return status code
  */
-static int http_rx_location ( struct http_request *http, char *value ) {
+int http_open ( struct interface *xfer, struct http_method *method,
+		struct uri *uri, struct http_request_range *range,
+		struct http_request_content *content ) {
+	struct http_transaction *http;
+	struct uri request_uri;
+	struct uri request_host;
+	size_t request_uri_len;
+	size_t request_host_len;
+	size_t content_len;
+	char *request_uri_string;
+	char *request_host_string;
+	void *content_data;
 	int rc;
 
-	/* Redirect to new location */
-	DBGC ( http, "HTTP %p redirecting to %s\n", http, value );
-	if ( ( rc = xfer_redirect ( &http->xfer, LOCATION_URI_STRING,
-				    value ) ) != 0 ) {
-		DBGC ( http, "HTTP %p could not redirect: %s\n",
+	/* Calculate request URI length */
+	memset ( &request_uri, 0, sizeof ( request_uri ) );
+	request_uri.path = ( uri->path ? uri->path : "/" );
+	request_uri.query = uri->query;
+	request_uri_len =
+		( format_uri ( &request_uri, NULL, 0 ) + 1 /* NUL */);
+
+	/* Calculate host name length */
+	memset ( &request_host, 0, sizeof ( request_host ) );
+	request_host.host = uri->host;
+	request_host.port = uri->port;
+	request_host_len =
+		( format_uri ( &request_host, NULL, 0 ) + 1 /* NUL */ );
+
+	/* Calculate request content length */
+	content_len = ( content ? content->len : 0 );
+
+	/* Allocate and initialise structure */
+	http = zalloc ( sizeof ( *http ) + request_uri_len + request_host_len +
+			content_len );
+	if ( ! http ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
+	request_uri_string = ( ( ( void * ) http ) + sizeof ( *http ) );
+	request_host_string = ( request_uri_string + request_uri_len );
+	content_data = ( request_host_string + request_host_len );
+	format_uri ( &request_uri, request_uri_string, request_uri_len );
+	format_uri ( &request_host, request_host_string, request_host_len );
+	ref_init ( &http->refcnt, http_free );
+	intf_init ( &http->xfer, &http_xfer_desc, &http->refcnt );
+	intf_init ( &http->content, &http_content_desc, &http->refcnt );
+	intf_init ( &http->transfer, &http_transfer_desc, &http->refcnt );
+	intf_init ( &http->conn, &http_conn_desc, &http->refcnt );
+	intf_plug_plug ( &http->transfer, &http->content );
+	process_init ( &http->process, &http_process_desc, &http->refcnt );
+	timer_init ( &http->timer, http_expired, &http->refcnt );
+	http->uri = uri_get ( uri );
+	http->request.method = method;
+	http->request.uri = request_uri_string;
+	http->request.host = request_host_string;
+	if ( range ) {
+		memcpy ( &http->request.range, range,
+			 sizeof ( http->request.range ) );
+	}
+	if ( content ) {
+		http->request.content.type = content->type;
+		http->request.content.data = content_data;
+		http->request.content.len = content_len;
+		memcpy ( content_data, content->data, content_len );
+	}
+	http->state = &http_request;
+	DBGC2 ( http, "HTTP %p %s://%s%s\n", http, http->uri->scheme,
+		http->request.host, http->request.uri );
+
+	/* Open connection */
+	if ( ( rc = http_connect ( &http->conn, uri ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not connect: %s\n",
 		       http, strerror ( rc ) );
-		return rc;
+		goto err_connect;
 	}
 
+	/* Attach to parent interface, mortalise self, and return */
+	intf_plug_plug ( &http->xfer, xfer );
+	ref_put ( &http->refcnt );
 	return 0;
+
+ err_connect:
+	http_close ( http, rc );
+	ref_put ( &http->refcnt );
+ err_alloc:
+	return rc;
 }
 
 /**
- * Handle HTTP Content-Length header
+ * Handle successful transfer completion
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_content_length ( struct http_request *http, char *value ) {
-	struct block_device_capacity capacity;
-	size_t content_len;
-	char *endp;
+static int http_transfer_complete ( struct http_transaction *http ) {
+	struct http_authentication *auth;
+	const char *location;
+	int rc;
 
-	/* Parse content length */
-	content_len = strtoul ( value, &endp, 10 );
-	if ( ! ( ( *endp == '\0' ) || isspace ( *endp ) ) ) {
-		DBGC ( http, "HTTP %p invalid Content-Length \"%s\"\n",
-		       http, value );
-		return -EINVAL_CONTENT_LENGTH;
+	/* Keep connection alive if applicable */
+	if ( http->response.flags & HTTP_RESPONSE_KEEPALIVE )
+		pool_recycle ( &http->conn );
+
+	/* Restart server connection interface */
+	intf_restart ( &http->conn, 0 );
+
+	/* No more data is expected */
+	http->state = NULL;
+
+	/* If transaction is successful, then close the
+	 * transfer-decoded interface.  The content encoding may
+	 * choose whether or not to immediately terminate the
+	 * transaction.
+	 */
+	if ( http->response.rc == 0 ) {
+		intf_shutdown ( &http->transfer, 0 );
+		return 0;
 	}
 
-	/* If we already have an expected content length, and this
-	 * isn't it, then complain
+	/* Perform redirection, if applicable */
+	if ( ( location = http->response.location ) ) {
+		DBGC2 ( http, "HTTP %p redirecting to \"%s\"\n",
+			http, location );
+		if ( ( rc = xfer_redirect ( &http->xfer, LOCATION_URI_STRING,
+					    location ) ) != 0 ) {
+			DBGC ( http, "HTTP %p could not redirect: %s\n",
+			       http, strerror ( rc ) );
+			return rc;
+		}
+		http_close ( http, 0 );
+		return 0;
+	}
+
+	/* Fail unless a retry is permitted */
+	if ( ! ( http->response.flags & HTTP_RESPONSE_RETRY ) )
+		return http->response.rc;
+
+	/* Perform authentication, if applicable */
+	if ( ( auth = http->response.auth.auth ) ) {
+		http->request.auth.auth = auth;
+		DBGC2 ( http, "HTTP %p performing %s authentication\n",
+			http, auth->name );
+		if ( ( rc = auth->authenticate ( http ) ) != 0 ) {
+			DBGC ( http, "HTTP %p could not authenticate: %s\n",
+			       http, strerror ( rc ) );
+			return rc;
+		}
+	}
+
+	/* Restart content decoding interfaces (which may be attached
+	 * to the same object).
 	 */
-	if ( http->remaining && ( http->remaining != content_len ) ) {
-		DBGC ( http, "HTTP %p incorrect Content-Length %zd (expected "
-		       "%zd)\n", http, content_len, http->remaining );
-		return -EIO_CONTENT_LENGTH;
+	intf_nullify ( &http->content );
+	intf_nullify ( &http->transfer );
+	intf_restart ( &http->content, http->response.rc );
+	intf_restart ( &http->transfer, http->response.rc );
+	http->content.desc = &http_content_desc;
+	http->transfer.desc = &http_transfer_desc;
+	intf_plug_plug ( &http->transfer, &http->content );
+	http->len = 0;
+	assert ( http->remaining == 0 );
+
+	/* Start timer to initiate retry */
+	DBGC2 ( http, "HTTP %p retrying after %d seconds\n",
+		http, http->response.retry_after );
+	start_timer_fixed ( &http->timer,
+			    ( http->response.retry_after * TICKS_PER_SEC ) );
+	return 0;
+}
+
+/******************************************************************************
+ *
+ * Requests
+ *
+ ******************************************************************************
+ */
+
+/**
+ * Construct HTTP request headers
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length, or negative error
+ */
+static int http_format_headers ( struct http_transaction *http, char *buf,
+				 size_t len ) {
+	struct http_request_header *header;
+	size_t used;
+	size_t remaining;
+	char *line;
+	int value_len;
+	int rc;
+
+	/* Construct request line */
+	used = ssnprintf ( buf, len, "%s %s HTTP/1.1",
+			   http->request.method->name, http->request.uri );
+	if ( used < len )
+		DBGC2 ( http, "HTTP %p TX %s\n", http, buf );
+	used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
+
+	/* Construct all headers */
+	for_each_table_entry ( header, HTTP_REQUEST_HEADERS ) {
+
+		/* Determine header value length */
+		value_len = header->format ( http, NULL, 0 );
+		if ( value_len < 0 ) {
+			rc = value_len;
+			return rc;
+		}
+
+		/* Skip zero-length headers */
+		if ( ! value_len )
+			continue;
+
+		/* Construct header */
+		line = ( buf + used );
+		used += ssnprintf ( ( buf + used ), ( len - used ), "%s: ",
+				    header->name );
+		remaining = ( ( used < len ) ? ( len - used ) : 0 );
+		used += header->format ( http, ( buf + used ), remaining );
+		if ( used < len )
+			DBGC2 ( http, "HTTP %p TX %s\n", http, line );
+		used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
 	}
-	if ( ! ( http->flags & HTTP_HEAD_ONLY ) )
-		http->remaining = content_len;
 
-	/* Do nothing more if we are retrying the request */
-	if ( http->flags & HTTP_TRY_AGAIN )
+	/* Construct terminating newline */
+	used += ssnprintf ( ( buf + used ), ( len - used ), "\r\n" );
+
+	return used;
+}
+
+/**
+ * Construct HTTP "Host" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_host ( struct http_transaction *http, char *buf,
+			      size_t len ) {
+
+	/* Construct host URI */
+	return snprintf ( buf, len, "%s", http->request.host );
+}
+
+/** HTTP "Host" header "*/
+struct http_request_header http_request_host __http_request_header = {
+	.name = "Host",
+	.format = http_format_host,
+};
+
+/**
+ * Construct HTTP "User-Agent" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_user_agent ( struct http_transaction *http __unused,
+				    char *buf, size_t len ) {
+
+	/* Construct user agent */
+	return snprintf ( buf, len, "iPXE/%s", product_version );
+}
+
+/** HTTP "User-Agent" header */
+struct http_request_header http_request_user_agent __http_request_header = {
+	.name = "User-Agent",
+	.format = http_format_user_agent,
+};
+
+/**
+ * Construct HTTP "Connection" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_connection ( struct http_transaction *http __unused,
+				    char *buf, size_t len ) {
+
+	/* Always request keep-alive */
+	return snprintf ( buf, len, "keep-alive" );
+}
+
+/** HTTP "Connection" header */
+struct http_request_header http_request_connection __http_request_header = {
+	.name = "Connection",
+	.format = http_format_connection,
+};
+
+/**
+ * Construct HTTP "Range" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_range ( struct http_transaction *http,
+			       char *buf, size_t len ) {
+
+	/* Construct range, if applicable */
+	if ( http->request.range.len ) {
+		return snprintf ( buf, len, "bytes=%zd-%zd",
+				  http->request.range.start,
+				  ( http->request.range.start +
+				    http->request.range.len - 1 ) );
+	} else {
 		return 0;
+	}
+}
+
+/** HTTP "Range" header */
+struct http_request_header http_request_range __http_request_header = {
+	.name = "Range",
+	.format = http_format_range,
+};
 
-	/* Use seek() to notify recipient of filesize */
-	xfer_seek ( &http->xfer, http->remaining );
-	xfer_seek ( &http->xfer, 0 );
+/**
+ * Construct HTTP "Content-Type" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_content_type ( struct http_transaction *http,
+				      char *buf, size_t len ) {
 
-	/* Report block device capacity if applicable */
-	if ( http->flags & HTTP_HEAD_ONLY ) {
-		capacity.blocks = ( content_len / HTTP_BLKSIZE );
-		capacity.blksize = HTTP_BLKSIZE;
-		capacity.max_count = -1U;
-		block_capacity ( &http->partial, &capacity );
+	/* Construct content type, if applicable */
+	if ( http->request.content.type ) {
+		return snprintf ( buf, len, "%s", http->request.content.type );
+	} else {
+		return 0;
 	}
-	return 0;
 }
 
+/** HTTP "Content-Type" header */
+struct http_request_header http_request_content_type __http_request_header = {
+	.name = "Content-Type",
+	.format = http_format_content_type,
+};
+
 /**
- * Handle HTTP Transfer-Encoding header
+ * Construct HTTP "Content-Length" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
- * @ret rc		Return status code
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
  */
-static int http_rx_transfer_encoding ( struct http_request *http, char *value ){
+static int http_format_content_length ( struct http_transaction *http,
+					char *buf, size_t len ) {
 
-	if ( strcasecmp ( value, "chunked" ) == 0 ) {
-		/* Mark connection as using chunked transfer encoding */
-		http->chunked = 1;
+	/* Construct content length, if applicable */
+	if ( http->request.content.len ) {
+		return snprintf ( buf, len, "%zd", http->request.content.len );
+	} else {
+		return 0;
 	}
+}
 
-	return 0;
+/** HTTP "Content-Length" header */
+struct http_request_header http_request_content_length __http_request_header = {
+	.name = "Content-Length",
+	.format = http_format_content_length,
+};
+
+/**
+ * Construct HTTP "Accept-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_accept_encoding ( struct http_transaction *http,
+					 char *buf, size_t len ) {
+	struct http_content_encoding *encoding;
+	const char *sep = "";
+	size_t used = 0;
+
+	/* Construct list of content encodings */
+	for_each_table_entry ( encoding, HTTP_CONTENT_ENCODINGS ) {
+		if ( encoding->supported && ( ! encoding->supported ( http ) ) )
+			continue;
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    "%s%s", sep, encoding->name );
+		sep = ", ";
+	}
+
+	return used;
 }
 
+/** HTTP "Accept-Encoding" header */
+struct http_request_header http_request_accept_encoding __http_request_header ={
+	.name = "Accept-Encoding",
+	.format = http_format_accept_encoding,
+};
+
 /**
- * Handle HTTP Connection header
+ * Transmit request
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_connection ( struct http_request *http, char *value ) {
+static int http_tx_request ( struct http_transaction *http ) {
+	struct io_buffer *iobuf;
+	int len;
+	int check_len;
+	int rc;
+
+	/* Calculate request length */
+	len = http_format_headers ( http, NULL, 0 );
+	if ( len < 0 ) {
+		rc = len;
+		DBGC ( http, "HTTP %p could not construct request: %s\n",
+		       http, strerror ( rc ) );
+		goto err_len;
+	}
+
+	/* Allocate I/O buffer */
+	iobuf = alloc_iob ( len + 1 /* NUL */ + http->request.content.len );
+	if ( ! iobuf ) {
+		rc = -ENOMEM;
+		goto err_alloc;
+	}
 
-	if ( strcasecmp ( value, "keep-alive" ) == 0 ) {
-		/* Mark connection as being kept alive by the server */
-		http->flags |= HTTP_SERVER_KEEPALIVE;
+	/* Construct request */
+	check_len = http_format_headers ( http, iob_put ( iobuf, len ),
+					  ( len + 1 /* NUL */ ) );
+	assert ( check_len == len );
+	memcpy ( iob_put ( iobuf, http->request.content.len ),
+		 http->request.content.data, http->request.content.len );
+
+	/* Deliver request */
+	if ( ( rc = xfer_deliver_iob ( &http->conn,
+				       iob_disown ( iobuf ) ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not deliver request: %s\n",
+		       http, strerror ( rc ) );
+		goto err_deliver;
 	}
 
+	/* Clear any previous response */
+	empty_line_buffer ( &http->response.headers );
+	memset ( &http->response, 0, sizeof ( http->response ) );
+
+	/* Move to response headers state */
+	http->state = &http_headers;
+
 	return 0;
+
+ err_deliver:
+	free_iob ( iobuf );
+ err_alloc:
+ err_len:
+	return rc;
 }
 
+/** HTTP request state */
+static struct http_state http_request = {
+	.tx = http_tx_request,
+	.close = http_close_error,
+};
+
+/******************************************************************************
+ *
+ * Response headers
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle WWW-Authenticate Basic header
+ * Parse HTTP status line
  *
- * @v http		HTTP request
- * @v params		Parameters
+ * @v http		HTTP transaction
+ * @v line		Status line
  * @ret rc		Return status code
  */
-static int http_rx_basic_auth ( struct http_request *http, char *params ) {
+static int http_parse_status ( struct http_transaction *http, char *line ) {
+	char *endp;
+	char *version;
+	char *vernum;
+	char *status;
+	int response_rc;
+
+	DBGC2 ( http, "HTTP %p RX %s\n", http, line );
+
+	/* Parse HTTP version */
+	version = http_token ( &line, NULL );
+	if ( ( ! version ) || ( strncmp ( version, "HTTP/", 5 ) != 0 ) ) {
+		DBGC ( http, "HTTP %p malformed version \"%s\"\n", http, line );
+		return -EINVAL_STATUS;
+	}
 
-	DBGC ( http, "HTTP %p Basic authentication required (%s)\n",
-	       http, params );
+	/* Keepalive is enabled by default for anything newer than HTTP/1.0 */
+	vernum = ( version + 5 /* "HTTP/" (presence already checked) */ );
+	if ( vernum[0] == '0' ) {
+		/* HTTP/0.x : keepalive not enabled by default */
+	} else if ( strncmp ( vernum, "1.0", 3 ) == 0 ) {
+		/* HTTP/1.0 : keepalive not enabled by default */
+	} else {
+		/* HTTP/1.1 or newer: keepalive enabled by default */
+		http->response.flags |= HTTP_RESPONSE_KEEPALIVE;
+	}
 
-	/* If we received a 401 Unauthorized response, then retry
-	 * using Basic authentication
-	 */
-	if ( ( http->code == 401 ) &&
-	     ( ! ( http->flags & HTTP_BASIC_AUTH ) ) &&
-	     ( http->uri->user != NULL ) ) {
-		http->flags |= ( HTTP_TRY_AGAIN | HTTP_BASIC_AUTH );
+	/* Parse status code */
+	status = line;
+	http->response.status = strtoul ( status, &endp, 10 );
+	if ( *endp != ' ' ) {
+		DBGC ( http, "HTTP %p malformed status code \"%s\"\n",
+		       http, status );
+		return -EINVAL_STATUS;
+	}
+
+	/* Convert HTTP status code to iPXE return status code */
+	if ( status[0] == '2' ) {
+		/* 2xx Success */
+		response_rc = 0;
+	} else if ( status[0] == '3' ) {
+		/* 3xx Redirection */
+		response_rc = -EXDEV;
+	} else if ( http->response.status == 401 ) {
+		/* 401 Unauthorized */
+		response_rc = -EACCES_401;
+	} else if ( http->response.status == 403 ) {
+		/* 403 Forbidden */
+		response_rc = -EPERM_403;
+	} else if ( http->response.status == 404 ) {
+		/* 404 Not Found */
+		response_rc = -ENOENT_404;
+	} else if ( status[0] == '4' ) {
+		/* 4xx Client Error (not already specified) */
+		response_rc = -EIO_4XX;
+	} else if ( status[0] == '5' ) {
+		/* 5xx Server Error */
+		response_rc = -EIO_5XX;
+	} else {
+		/* Unrecognised */
+		response_rc = -EIO_OTHER;
 	}
+	http->response.rc = response_rc;
 
 	return 0;
 }
 
 /**
- * Parse Digest authentication parameter
+ * Parse HTTP header
  *
- * @v params		Parameters
- * @v name		Parameter name (including trailing "=\"")
- * @ret value		Parameter value, or NULL
+ * @v http		HTTP transaction
+ * @v line		Header line
+ * @ret rc		Return status code
  */
-static char * http_digest_param ( char *params, const char *name ) {
-	char *key;
-	char *value;
-	char *terminator;
-
-	/* Locate parameter */
-	key = strstr ( params, name );
-	if ( ! key )
-		return NULL;
+static int http_parse_header ( struct http_transaction *http, char *line ) {
+	struct http_response_header *header;
+	char *name = line;
+	char *sep;
 
-	/* Extract value */
-	value = ( key + strlen ( name ) );
-	terminator = strchr ( value, '"' );
-	if ( ! terminator )
-		return NULL;
-	return strndup ( value, ( terminator - value ) );
+	DBGC2 ( http, "HTTP %p RX %s\n", http, line );
+
+	/* Extract header name */
+	sep = strstr ( line, ": " );
+	if ( ! sep ) {
+		DBGC ( http, "HTTP %p malformed header \"%s\"\n", http, line );
+		return -EINVAL_HEADER;
+	}
+	*sep = '\0';
+	line = ( sep + 2 /* ": " */ );
+
+	/* Process header, if recognised */
+	for_each_table_entry ( header, HTTP_RESPONSE_HEADERS ) {
+		if ( strcasecmp ( name, header->name ) == 0 )
+			return header->parse ( http, line );
+	}
+
+	/* Unrecognised headers should be ignored */
+	return 0;
 }
 
 /**
- * Handle WWW-Authenticate Digest header
+ * Parse HTTP response headers
  *
- * @v http		HTTP request
- * @v params		Parameters
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_digest_auth ( struct http_request *http, char *params ) {
+static int http_parse_headers ( struct http_transaction *http ) {
+	char *line;
+	char *next;
+	int rc;
 
-	DBGC ( http, "HTTP %p Digest authentication required (%s)\n",
-	       http, params );
+	/* Get status line */
+	line = http->response.headers.data;
+	assert ( line != NULL );
+	next = ( line + strlen ( line ) + 1 /* NUL */ );
 
-	/* If we received a 401 Unauthorized response, then retry
-	 * using Digest authentication
-	 */
-	if ( ( http->code == 401 ) &&
-	     ( ! ( http->flags & HTTP_DIGEST_AUTH ) ) &&
-	     ( http->uri->user != NULL ) ) {
-
-		/* Extract realm */
-		free ( http->auth_realm );
-		http->auth_realm = http_digest_param ( params, "realm=\"" );
-		if ( ! http->auth_realm ) {
-			DBGC ( http, "HTTP %p Digest prompt missing realm\n",
-			       http );
-			return -EINVAL_HEADER;
-		}
+	/* Parse status line */
+	if ( ( rc = http_parse_status ( http, line ) ) != 0 )
+		return rc;
 
-		/* Extract nonce */
-		free ( http->auth_nonce );
-		http->auth_nonce = http_digest_param ( params, "nonce=\"" );
-		if ( ! http->auth_nonce ) {
-			DBGC ( http, "HTTP %p Digest prompt missing nonce\n",
-			       http );
-			return -EINVAL_HEADER;
-		}
+	/* Process header lines */
+	while ( 1 ) {
 
-		/* Extract opaque */
-		free ( http->auth_opaque );
-		http->auth_opaque = http_digest_param ( params, "opaque=\"" );
-		if ( ! http->auth_opaque ) {
-			/* Not an error; "opaque" is optional */
-		}
+		/* Move to next line */
+		line = next;
+		next = ( line + strlen ( line ) + 1 /* NUL */ );
+
+		/* Stop on terminating blank line */
+		if ( ! line[0] )
+			return 0;
 
-		http->flags |= ( HTTP_TRY_AGAIN | HTTP_DIGEST_AUTH );
+		/* Process header line */
+		if ( ( rc = http_parse_header ( http, line ) ) != 0 )
+			return rc;
 	}
+}
 
+/**
+ * Parse HTTP "Location" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_location ( struct http_transaction *http, char *line ) {
+
+	/* Store location */
+	http->response.location = line;
 	return 0;
 }
 
-/** An HTTP WWW-Authenticate header handler */
-struct http_auth_header_handler {
-	/** Scheme (e.g. "Basic") */
-	const char *scheme;
-	/** Handle received parameters
-	 *
-	 * @v http	HTTP request
-	 * @v params	Parameters
-	 * @ret rc	Return status code
-	 */
-	int ( * rx ) ( struct http_request *http, char *params );
+/** HTTP "Location" header */
+struct http_response_header http_response_location __http_response_header = {
+	.name = "Location",
+	.parse = http_parse_location,
 };
 
-/** List of HTTP WWW-Authenticate header handlers */
-static struct http_auth_header_handler http_auth_header_handlers[] = {
-	{
-		.scheme = "Basic",
-		.rx = http_rx_basic_auth,
-	},
-	{
-		.scheme = "Digest",
-		.rx = http_rx_digest_auth,
-	},
-	{ NULL, NULL },
+/**
+ * Parse HTTP "Transfer-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_transfer_encoding ( struct http_transaction *http,
+					  char *line ) {
+	struct http_transfer_encoding *encoding;
+
+	/* Check for known transfer encodings */
+	for_each_table_entry ( encoding, HTTP_TRANSFER_ENCODINGS ) {
+		if ( strcasecmp ( line, encoding->name ) == 0 ) {
+			http->response.transfer.encoding = encoding;
+			return 0;
+		}
+	}
+
+	DBGC ( http, "HTTP %p unrecognised Transfer-Encoding \"%s\"\n",
+	       http, line );
+	return -ENOTSUP_TRANSFER;
+}
+
+/** HTTP "Transfer-Encoding" header */
+struct http_response_header
+http_response_transfer_encoding __http_response_header = {
+	.name = "Transfer-Encoding",
+	.parse = http_parse_transfer_encoding,
 };
 
 /**
- * Handle HTTP WWW-Authenticate header
+ * Parse HTTP "Connection" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
  * @ret rc		Return status code
  */
-static int http_rx_www_authenticate ( struct http_request *http, char *value ) {
-	struct http_auth_header_handler *handler;
-	char *separator;
-	char *scheme;
-	char *params;
-	int rc;
+static int http_parse_connection ( struct http_transaction *http, char *line ) {
 
-	/* Extract scheme */
-	separator = strchr ( value, ' ' );
-	if ( ! separator ) {
-		DBGC ( http, "HTTP %p malformed WWW-Authenticate header\n",
-		       http );
-		return -EINVAL_HEADER;
+	/* Check for known connection intentions */
+	if ( strcasecmp ( line, "keep-alive" ) == 0 ) {
+		http->response.flags |= HTTP_RESPONSE_KEEPALIVE;
+		return 0;
 	}
-	*separator = '\0';
-	scheme = value;
-	params = ( separator + 1 );
-
-	/* Hand off to header handler, if one exists */
-	for ( handler = http_auth_header_handlers; handler->scheme; handler++ ){
-		if ( strcasecmp ( scheme, handler->scheme ) == 0 ) {
-			if ( ( rc = handler->rx ( http, params ) ) != 0 )
-				return rc;
-			break;
-		}
+	if ( strcasecmp ( line, "close" ) == 0 ) {
+		http->response.flags &= ~HTTP_RESPONSE_KEEPALIVE;
+		return 0;
 	}
-	return 0;
+
+	DBGC ( http, "HTTP %p unrecognised Connection \"%s\"\n", http, line );
+	return -ENOTSUP_CONNECTION;
 }
 
+/** HTTP "Connection" header */
+struct http_response_header http_response_connection __http_response_header = {
+	.name = "Connection",
+	.parse = http_parse_connection,
+};
+
 /**
- * Handle HTTP Retry-After header
+ * Parse HTTP "Content-Length" header
  *
- * @v http		HTTP request
- * @v value		HTTP header value
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
  * @ret rc		Return status code
  */
-static int http_rx_retry_after ( struct http_request *http, char *value ) {
-	unsigned long seconds;
+static int http_parse_content_length ( struct http_transaction *http,
+				       char *line ) {
 	char *endp;
 
-	DBGC ( http, "HTTP %p retry requested (%s)\n", http, value );
+	/* Parse length */
+	http->response.content.len = strtoul ( line, &endp, 10 );
+	if ( *endp != '\0' ) {
+		DBGC ( http, "HTTP %p invalid Content-Length \"%s\"\n",
+		       http, line );
+		return -EINVAL_CONTENT_LENGTH;
+	}
 
-	/* If we received a 503 Service Unavailable response, then
-	 * retry after the specified number of seconds.  If the value
-	 * is not a simple number of seconds (e.g. a full HTTP date),
-	 * then retry after a fixed delay, since we don't have code
-	 * able to parse full HTTP dates.
-	 */
-	if ( http->code == 503 ) {
-		seconds = strtoul ( value, &endp, 10 );
-		if ( *endp != '\0' ) {
-			seconds = HTTP_RETRY_SECONDS;
-			DBGC ( http, "HTTP %p cannot understand \"%s\"; "
-			       "using %ld seconds\n", http, value, seconds );
+	/* Record that we have a content length (since it may be zero) */
+	http->response.flags |= HTTP_RESPONSE_CONTENT_LEN;
+
+	return 0;
+}
+
+/** HTTP "Content-Length" header */
+struct http_response_header
+http_response_content_length __http_response_header = {
+	.name = "Content-Length",
+	.parse = http_parse_content_length,
+};
+
+/**
+ * Parse HTTP "Content-Encoding" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_content_encoding ( struct http_transaction *http,
+					 char *line ) {
+	struct http_content_encoding *encoding;
+
+	/* Check for known content encodings */
+	for_each_table_entry ( encoding, HTTP_CONTENT_ENCODINGS ) {
+		if ( encoding->supported && ( ! encoding->supported ( http ) ) )
+			continue;
+		if ( strcasecmp ( line, encoding->name ) == 0 ) {
+			http->response.content.encoding = encoding;
+			return 0;
 		}
-		http->flags |= HTTP_TRY_AGAIN;
-		http->retry_delay = ( seconds * TICKS_PER_SEC );
 	}
 
+	/* Some servers (e.g. Apache) have a habit of specifying
+	 * unwarranted content encodings.  For example, if Apache
+	 * detects (via /etc/httpd/conf/magic) that a file's contents
+	 * are gzip-compressed, it will set "Content-Encoding: x-gzip"
+	 * regardless of the client's Accept-Encoding header.  The
+	 * only viable way to handle such servers is to treat unknown
+	 * content encodings as equivalent to "identity".
+	 */
+	DBGC ( http, "HTTP %p unrecognised Content-Encoding \"%s\"\n",
+	       http, line );
 	return 0;
 }
 
-/** An HTTP header handler */
-struct http_header_handler {
-	/** Name (e.g. "Content-Length") */
-	const char *header;
-	/** Handle received header
-	 *
-	 * @v http	HTTP request
-	 * @v value	HTTP header value
-	 * @ret rc	Return status code
-	 *
-	 * If an error is returned, the download will be aborted.
-	 */
-	int ( * rx ) ( struct http_request *http, char *value );
+/** HTTP "Content-Encoding" header */
+struct http_response_header
+http_response_content_encoding __http_response_header = {
+	.name = "Content-Encoding",
+	.parse = http_parse_content_encoding,
 };
 
-/** List of HTTP header handlers */
-static struct http_header_handler http_header_handlers[] = {
-	{
-		.header = "Location",
-		.rx = http_rx_location,
-	},
-	{
-		.header = "Content-Length",
-		.rx = http_rx_content_length,
-	},
-	{
-		.header = "Transfer-Encoding",
-		.rx = http_rx_transfer_encoding,
-	},
-	{
-		.header = "Connection",
-		.rx = http_rx_connection,
-	},
-	{
-		.header = "WWW-Authenticate",
-		.rx = http_rx_www_authenticate,
-	},
-	{
-		.header = "Retry-After",
-		.rx = http_rx_retry_after,
-	},
-	{ NULL, NULL }
+/**
+ * Parse HTTP "Retry-After" header
+ *
+ * @v http		HTTP transaction
+ * @v line		Remaining header line
+ * @ret rc		Return status code
+ */
+static int http_parse_retry_after ( struct http_transaction *http,
+				    char *line ) {
+	char *endp;
+
+	/* Try to parse value as a simple number of seconds */
+	http->response.retry_after = strtoul ( line, &endp, 10 );
+	if ( *endp != '\0' ) {
+		/* For any value which is not a simple number of
+		 * seconds (e.g. a full HTTP date), just retry after a
+		 * fixed delay, since we don't have code able to parse
+		 * full HTTP dates.
+		 */
+		http->response.retry_after = HTTP_RETRY_SECONDS;
+		DBGC ( http, "HTTP %p cannot understand Retry-After \"%s\"; "
+		       "using %d seconds\n", http, line, HTTP_RETRY_SECONDS );
+	}
+
+	/* Allow HTTP request to be retried after specified delay */
+	http->response.flags |= HTTP_RESPONSE_RETRY;
+
+	return 0;
+}
+
+/** HTTP "Retry-After" header */
+struct http_response_header http_response_retry_after __http_response_header = {
+	.name = "Retry-After",
+	.parse = http_parse_retry_after,
 };
 
 /**
- * Handle HTTP header
+ * Handle received HTTP headers
  *
- * @v http		HTTP request
- * @v header		HTTP header
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
  * @ret rc		Return status code
  */
-static int http_rx_header ( struct http_request *http, char *header ) {
-	struct http_header_handler *handler;
-	char *separator;
-	char *value;
+static int http_rx_headers ( struct http_transaction *http,
+			     struct io_buffer **iobuf ) {
+	struct http_transfer_encoding *transfer;
+	struct http_content_encoding *content;
+	char *line;
 	int rc;
 
-	/* An empty header line marks the end of this phase */
-	if ( ! header[0] ) {
-		empty_line_buffer ( &http->linebuf );
+	/* Buffer header line */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf,
+				      &http->response.headers ) ) != 0 )
+		return rc;
 
-		/* Handle response code */
-		if ( ! ( http->flags & HTTP_TRY_AGAIN ) ) {
-			if ( ( rc = http_response_to_rc ( http->code ) ) != 0 )
-				return rc;
-		}
+	/* Wait until we see the empty line marking end of headers */
+	line = buffered_line ( &http->response.headers );
+	if ( ( line == NULL ) || ( line[0] != '\0' ) )
+		return 0;
 
-		/* Move to next state */
-		if ( http->rx_state == HTTP_RX_HEADER ) {
-			DBGC ( http, "HTTP %p start of data\n", http );
-			http->rx_state = ( http->chunked ?
-					   HTTP_RX_CHUNK_LEN : HTTP_RX_DATA );
-			if ( ( http->partial_len != 0 ) &&
-			     ( ! ( http->flags & HTTP_TRY_AGAIN ) ) ) {
-				http->remaining = http->partial_len;
-			}
-			return 0;
-		} else {
-			DBGC ( http, "HTTP %p end of trailer\n", http );
-			http_done ( http );
-			return 0;
-		}
+	/* Process headers */
+	if ( ( rc = http_parse_headers ( http ) ) != 0 )
+		return rc;
+
+	/* Initialise content encoding, if applicable */
+	if ( ( content = http->response.content.encoding ) &&
+	     ( ( rc = content->init ( http ) ) != 0 ) ) {
+		DBGC ( http, "HTTP %p could not initialise %s content "
+		       "encoding: %s\n", http, content->name, strerror ( rc ) );
+		return rc;
 	}
 
-	DBGC ( http, "HTTP %p header \"%s\"\n", http, header );
+	/* Presize receive buffer, if we have a content length */
+	if ( http->response.content.len ) {
+		xfer_seek ( &http->transfer, http->response.content.len );
+		xfer_seek ( &http->transfer, 0 );
+	}
 
-	/* Split header at the ": " */
-	separator = strstr ( header, ": " );
-	if ( ! separator ) {
-		DBGC ( http, "HTTP %p malformed header\n", http );
-		return -EINVAL_HEADER;
+	/* Complete transfer if this is a HEAD request */
+	if ( http->request.method == &http_head ) {
+		if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+			return rc;
+		return 0;
 	}
-	*separator = '\0';
-	value = ( separator + 2 );
-
-	/* Hand off to header handler, if one exists */
-	for ( handler = http_header_handlers ; handler->header ; handler++ ) {
-		if ( strcasecmp ( header, handler->header ) == 0 ) {
-			if ( ( rc = handler->rx ( http, value ) ) != 0 )
-				return rc;
-			break;
-		}
+
+	/* Default to identity transfer encoding, if none specified */
+	if ( ! http->response.transfer.encoding )
+		http->response.transfer.encoding = &http_transfer_identity;
+
+	/* Move to transfer encoding-specific data state */
+	transfer = http->response.transfer.encoding;
+	http->state = &transfer->state;
+
+	/* Initialise transfer encoding */
+	if ( ( rc = transfer->init ( http ) ) != 0 ) {
+		DBGC ( http, "HTTP %p could not initialise %s transfer "
+		       "encoding: %s\n", http, transfer->name, strerror ( rc ));
+		return rc;
 	}
+
 	return 0;
 }
 
+/** HTTP response headers state */
+static struct http_state http_headers = {
+	.rx = http_rx_headers,
+	.close = http_close_error,
+};
+
+/******************************************************************************
+ *
+ * Identity transfer encoding
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle HTTP chunk length
+ * Initialise transfer encoding
  *
- * @v http		HTTP request
- * @v length		HTTP chunk length
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_rx_chunk_len ( struct http_request *http, char *length ) {
-	char *endp;
+static int http_init_transfer_identity ( struct http_transaction *http ) {
+	int rc;
 
-	/* Skip blank lines between chunks */
-	if ( length[0] == '\0' )
-		return 0;
+	/* Complete transfer immediately if we have a zero content length */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->response.content.len == 0 ) &&
+	     ( ( rc = http_transfer_complete ( http ) ) != 0 ) )
+		return rc;
 
-	/* Parse chunk length */
-	http->chunk_remaining = strtoul ( length, &endp, 16 );
-	if ( *endp != '\0' ) {
-		DBGC ( http, "HTTP %p invalid chunk length \"%s\"\n",
-		       http, length );
-		return -EINVAL_CHUNK_LENGTH;
-	}
+	return 0;
+}
 
-	/* Terminate chunked encoding if applicable */
-	if ( http->chunk_remaining == 0 ) {
-		DBGC ( http, "HTTP %p end of chunks\n", http );
-		http->chunked = 0;
-		http->rx_state = HTTP_RX_TRAILER;
-		return 0;
-	}
+/**
+ * Handle received data
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_transfer_identity ( struct http_transaction *http,
+				       struct io_buffer **iobuf ) {
+	size_t len = iob_len ( *iobuf );
+	int rc;
+
+	/* Update lengths */
+	http->len += len;
 
-	/* Use seek() to notify recipient of new filesize */
-	DBGC ( http, "HTTP %p start of chunk of length %zd\n",
-	       http, http->chunk_remaining );
-	if ( ! ( http->flags & HTTP_TRY_AGAIN ) ) {
-		xfer_seek ( &http->xfer,
-			    ( http->rx_len + http->chunk_remaining ) );
-		xfer_seek ( &http->xfer, http->rx_len );
+	/* Fail if this transfer would overrun the expected content
+	 * length (if any).
+	 */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->len > http->response.content.len ) ) {
+		DBGC ( http, "HTTP %p content length overrun\n", http );
+		return -EIO_CONTENT_LENGTH;
 	}
 
-	/* Start receiving data */
-	http->rx_state = HTTP_RX_DATA;
+	/* Hand off to content encoding */
+	if ( ( rc = xfer_deliver_iob ( &http->transfer,
+				       iob_disown ( *iobuf ) ) ) != 0 )
+		return rc;
+
+	/* Complete transfer if we have received the expected content
+	 * length (if any).
+	 */
+	if ( ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) &&
+	     ( http->len == http->response.content.len ) &&
+	     ( ( rc = http_transfer_complete ( http ) ) != 0 ) )
+		return rc;
 
 	return 0;
 }
 
-/** An HTTP line-based data handler */
-struct http_line_handler {
-	/** Handle line
-	 *
-	 * @v http	HTTP request
-	 * @v line	Line to handle
-	 * @ret rc	Return status code
+/**
+ * Handle server connection close
+ *
+ * @v http		HTTP transaction
+ * @v rc		Reason for close
+ */
+static void http_close_transfer_identity ( struct http_transaction *http,
+					   int rc ) {
+
+	/* Fail if any error occurred */
+	if ( rc != 0 )
+		goto err;
+
+	/* Fail if we have a content length (since we would have
+	 * already closed the connection if we had received the
+	 * correct content length).
 	 */
-	int ( * rx ) ( struct http_request *http, char *line );
-};
+	if ( http->response.flags & HTTP_RESPONSE_CONTENT_LEN ) {
+		DBGC ( http, "HTTP %p content length underrun\n", http );
+		rc = EIO_CONTENT_LENGTH;
+		goto err;
+	}
+
+	/* Indicate that transfer is complete */
+	if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+		goto err;
 
-/** List of HTTP line-based data handlers */
-static struct http_line_handler http_line_handlers[] = {
-	[HTTP_RX_RESPONSE]	= { .rx = http_rx_response },
-	[HTTP_RX_HEADER]	= { .rx = http_rx_header },
-	[HTTP_RX_CHUNK_LEN]	= { .rx = http_rx_chunk_len },
-	[HTTP_RX_TRAILER]	= { .rx = http_rx_header },
+	return;
+
+ err:
+	http_close ( http, rc );
+}
+
+/** Identity transfer encoding */
+static struct http_transfer_encoding http_transfer_identity = {
+	.name = "identity",
+	.init = http_init_transfer_identity,
+	.state = {
+		.rx = http_rx_transfer_identity,
+		.close = http_close_transfer_identity,
+	},
 };
 
+/******************************************************************************
+ *
+ * Chunked transfer encoding
+ *
+ ******************************************************************************
+ */
+
 /**
- * Handle new data arriving via HTTP connection
+ * Initialise transfer encoding
  *
- * @v http		HTTP request
- * @v iobuf		I/O buffer
- * @v meta		Data transfer metadata
+ * @v http		HTTP transaction
  * @ret rc		Return status code
  */
-static int http_socket_deliver ( struct http_request *http,
-				 struct io_buffer *iobuf,
-				 struct xfer_metadata *meta __unused ) {
-	struct http_line_handler *lh;
+static int http_init_transfer_chunked ( struct http_transaction *http ) {
+
+	/* Sanity checks */
+	assert ( http->remaining == 0 );
+	assert ( http->linebuf.len == 0 );
+
+	return 0;
+}
+
+/**
+ * Handle received chunk length
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_chunk_len ( struct http_transaction *http,
+			       struct io_buffer **iobuf ) {
 	char *line;
-	size_t data_len;
-	ssize_t line_len;
-	int rc = 0;
+	char *endp;
+	size_t len;
+	int rc;
 
-	profile_start ( &http_rx_profiler );
-	while ( iobuf && iob_len ( iobuf ) ) {
+	/* Receive into temporary line buffer */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf, &http->linebuf ) ) != 0 )
+		return rc;
 
-		switch ( http->rx_state ) {
-		case HTTP_RX_IDLE:
-			/* Receiving any data in this state is an error */
-			DBGC ( http, "HTTP %p received %zd bytes while %s\n",
-			       http, iob_len ( iobuf ),
-			       ( ( http->rx_state == HTTP_RX_IDLE ) ?
-				 "idle" : "dead" ) );
-			rc = -EPROTO_UNSOLICITED;
-			goto done;
-		case HTTP_RX_DEAD:
-			/* Do no further processing */
-			goto done;
-		case HTTP_RX_DATA:
-			/* Pass received data to caller */
-			data_len = iob_len ( iobuf );
-			if ( http->chunk_remaining &&
-			     ( http->chunk_remaining < data_len ) ) {
-				data_len = http->chunk_remaining;
-			}
-			if ( http->remaining &&
-			     ( http->remaining < data_len ) ) {
-				data_len = http->remaining;
-			}
-			if ( http->flags & HTTP_TRY_AGAIN ) {
-				/* Discard all received data */
-				iob_pull ( iobuf, data_len );
-			} else if ( http->rx_buffer != UNULL ) {
-				/* Copy to partial transfer buffer */
-				copy_to_user ( http->rx_buffer, http->rx_len,
-					       iobuf->data, data_len );
-				iob_pull ( iobuf, data_len );
-			} else if ( data_len < iob_len ( iobuf ) ) {
-				/* Deliver partial buffer as raw data */
-				profile_start ( &http_xfer_profiler );
-				rc = xfer_deliver_raw ( &http->xfer,
-							iobuf->data, data_len );
-				iob_pull ( iobuf, data_len );
-				if ( rc != 0 )
-					goto done;
-				profile_stop ( &http_xfer_profiler );
-			} else {
-				/* Deliver whole I/O buffer */
-				profile_start ( &http_xfer_profiler );
-				if ( ( rc = xfer_deliver_iob ( &http->xfer,
-						 iob_disown ( iobuf ) ) ) != 0 )
-					goto done;
-				profile_stop ( &http_xfer_profiler );
-			}
-			http->rx_len += data_len;
-			if ( http->chunk_remaining ) {
-				http->chunk_remaining -= data_len;
-				if ( http->chunk_remaining == 0 )
-					http->rx_state = HTTP_RX_CHUNK_LEN;
-			}
-			if ( http->remaining ) {
-				http->remaining -= data_len;
-				if ( ( http->remaining == 0 ) &&
-				     ( http->rx_state == HTTP_RX_DATA ) ) {
-					http_done ( http );
-				}
-			}
-			break;
-		case HTTP_RX_RESPONSE:
-		case HTTP_RX_HEADER:
-		case HTTP_RX_CHUNK_LEN:
-		case HTTP_RX_TRAILER:
-			/* In the other phases, buffer and process a
-			 * line at a time
-			 */
-			line_len = line_buffer ( &http->linebuf, iobuf->data,
-						 iob_len ( iobuf ) );
-			if ( line_len < 0 ) {
-				rc = line_len;
-				DBGC ( http, "HTTP %p could not buffer line: "
-				       "%s\n", http, strerror ( rc ) );
-				goto done;
-			}
-			iob_pull ( iobuf, line_len );
-			line = buffered_line ( &http->linebuf );
-			if ( line ) {
-				lh = &http_line_handlers[http->rx_state];
-				if ( ( rc = lh->rx ( http, line ) ) != 0 )
-					goto done;
-			}
-			break;
-		default:
-			assert ( 0 );
-			break;
-		}
+	/* Wait until we receive a non-empty line */
+	line = buffered_line ( &http->linebuf );
+	if ( ( line == NULL ) || ( line[0] == '\0' ) )
+		return 0;
+
+	/* Parse chunk length */
+	http->remaining = strtoul ( line, &endp, 16 );
+	if ( *endp != '\0' ) {
+		DBGC ( http, "HTTP %p invalid chunk length \"%s\"\n",
+		       http, line );
+		return -EINVAL_CHUNK_LENGTH;
 	}
 
- done:
-	if ( rc )
-		http_close ( http, rc );
-	free_iob ( iobuf );
-	profile_stop ( &http_rx_profiler );
-	return rc;
+	/* Empty line buffer */
+	empty_line_buffer ( &http->linebuf );
+
+	/* Update expected length */
+	len = ( http->len + http->remaining );
+	xfer_seek ( &http->transfer, len );
+	xfer_seek ( &http->transfer, http->len );
+
+	/* If chunk length is zero, then move to response trailers state */
+	if ( ! http->remaining )
+		http->state = &http_trailers;
+
+	return 0;
 }
 
 /**
- * Check HTTP socket flow control window
+ * Handle received chunk data
  *
- * @v http		HTTP request
- * @ret len		Length of window
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
  */
-static size_t http_socket_window ( struct http_request *http __unused ) {
+static int http_rx_chunk_data ( struct http_transaction *http,
+				struct io_buffer **iobuf ) {
+	struct io_buffer *payload;
+	uint8_t *crlf;
+	size_t len;
+	int rc;
 
-	/* Window is always open.  This is to prevent TCP from
-	 * stalling if our parent window is not currently open.
+	/* In the common case of a final chunk in a packet which also
+	 * includes the terminating CRLF, strip the terminating CRLF
+	 * (which we would ignore anyway) and hence avoid
+	 * unnecessarily copying the data.
 	 */
-	return ( ~( ( size_t ) 0 ) );
+	if ( iob_len ( *iobuf ) == ( http->remaining + 2 /* CRLF */ ) ) {
+		crlf = ( (*iobuf)->data + http->remaining );
+		if ( ( crlf[0] == '\r' ) && ( crlf[1] == '\n' ) )
+			iob_unput ( (*iobuf), 2 /* CRLF */ );
+	}
+	len = iob_len ( *iobuf );
+
+	/* Use whole/partial buffer as applicable */
+	if ( len <= http->remaining ) {
+
+		/* Whole buffer is to be consumed: decrease remaining
+		 * length and use original I/O buffer as payload.
+		 */
+		payload = iob_disown ( *iobuf );
+		http->len += len;
+		http->remaining -= len;
+
+	} else {
+
+		/* Partial buffer is to be consumed: copy data to a
+		 * temporary I/O buffer.
+		 */
+		payload = alloc_iob ( http->remaining );
+		if ( ! payload ) {
+			rc = -ENOMEM;
+			goto err;
+		}
+		memcpy ( iob_put ( payload, http->remaining ), (*iobuf)->data,
+			 http->remaining );
+		iob_pull ( *iobuf, http->remaining );
+		http->len += http->remaining;
+		http->remaining = 0;
+	}
+
+	/* Hand off to content encoding */
+	if ( ( rc = xfer_deliver_iob ( &http->transfer,
+				       iob_disown ( payload ) ) ) != 0 )
+		goto err;
+
+	return 0;
+
+ err:
+	assert ( payload == NULL );
+	return rc;
 }
 
 /**
- * Close HTTP socket
+ * Handle received chunked data
  *
- * @v http		HTTP request
- * @v rc		Reason for close
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
  */
-static void http_socket_close ( struct http_request *http, int rc ) {
+static int http_rx_transfer_chunked ( struct http_transaction *http,
+				      struct io_buffer **iobuf ) {
 
-	/* If we have an error, terminate */
-	if ( rc != 0 ) {
-		http_close ( http, rc );
-		return;
+	/* Handle as chunk length or chunk data as appropriate */
+	if ( http->remaining ) {
+		return http_rx_chunk_data ( http, iobuf );
+	} else {
+		return http_rx_chunk_len ( http, iobuf );
 	}
-
-	/* Mark HTTP request as complete */
-	http_done ( http );
 }
 
-/**
- * Generate HTTP Basic authorisation string
+/** Chunked transfer encoding */
+struct http_transfer_encoding http_transfer_chunked __http_transfer_encoding = {
+	.name = "chunked",
+	.init = http_init_transfer_chunked,
+	.state = {
+		.rx = http_rx_transfer_chunked,
+		.close = http_close_error,
+	},
+};
+
+/******************************************************************************
  *
- * @v http		HTTP request
- * @ret auth		Authorisation string, or NULL on error
+ * Response trailers
  *
- * The authorisation string is dynamically allocated, and must be
- * freed by the caller.
+ ******************************************************************************
  */
-static char * http_basic_auth ( struct http_request *http ) {
-	const char *user = http->uri->user;
-	const char *password = ( http->uri->password ?
-				 http->uri->password : "" );
-	size_t user_pw_len =
-		( strlen ( user ) + 1 /* ":" */ + strlen ( password ) );
-	char user_pw[ user_pw_len + 1 /* NUL */ ];
-	size_t user_pw_base64_len = base64_encoded_len ( user_pw_len );
-	char user_pw_base64[ user_pw_base64_len + 1 /* NUL */ ];
-	char *auth;
-	int len;
 
-	/* Sanity check */
-	assert ( user != NULL );
+/**
+ * Handle received HTTP trailer
+ *
+ * @v http		HTTP transaction
+ * @v iobuf		I/O buffer (may be claimed)
+ * @ret rc		Return status code
+ */
+static int http_rx_trailers ( struct http_transaction *http,
+			      struct io_buffer **iobuf ) {
+	char *line;
+	int rc;
 
-	/* Make "user:password" string from decoded fields */
-	snprintf ( user_pw, sizeof ( user_pw ), "%s:%s", user, password );
+	/* Buffer trailer line */
+	if ( ( rc = http_rx_linebuf ( http, *iobuf, &http->linebuf ) ) != 0 )
+		return rc;
 
-	/* Base64-encode the "user:password" string */
-	base64_encode ( ( void * ) user_pw, user_pw_len, user_pw_base64 );
+	/* Wait until we see the empty line marking end of trailers */
+	line = buffered_line ( &http->linebuf );
+	if ( ( line == NULL ) || ( line[0] != '\0' ) )
+		return 0;
 
-	/* Generate the authorisation string */
-	len = asprintf ( &auth, "Authorization: Basic %s\r\n",
-			 user_pw_base64 );
-	if ( len < 0 )
-		return NULL;
+	/* Empty line buffer */
+	empty_line_buffer ( &http->linebuf );
 
-	return auth;
+	/* Transfer is complete */
+	if ( ( rc = http_transfer_complete ( http ) ) != 0 )
+		return rc;
+
+	return 0;
 }
 
-/**
- * Generate HTTP Digest authorisation string
+/** HTTP response trailers state */
+static struct http_state http_trailers = {
+	.rx = http_rx_trailers,
+	.close = http_close_error,
+};
+
+/******************************************************************************
  *
- * @v http		HTTP request
- * @v method		HTTP method (e.g. "GET")
- * @v uri		HTTP request URI (e.g. "/index.html")
- * @ret auth		Authorisation string, or NULL on error
+ * Simple URI openers
  *
- * The authorisation string is dynamically allocated, and must be
- * freed by the caller.
+ ******************************************************************************
  */
-static char * http_digest_auth ( struct http_request *http,
-				 const char *method, const char *uri ) {
-	const char *user = http->uri->user;
-	const char *password = ( http->uri->password ?
-				 http->uri->password : "" );
-	const char *realm = http->auth_realm;
-	const char *nonce = http->auth_nonce;
-	const char *opaque = http->auth_opaque;
-	static const char colon = ':';
-	uint8_t ctx[MD5_CTX_SIZE];
-	uint8_t digest[MD5_DIGEST_SIZE];
-	char ha1[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char ha2[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char response[ base16_encoded_len ( sizeof ( digest ) ) + 1 /* NUL */ ];
-	char *auth;
-	int len;
-
-	/* Sanity checks */
-	assert ( user != NULL );
-	assert ( realm != NULL );
-	assert ( nonce != NULL );
-
-	/* Generate HA1 */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, user, strlen ( user ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, realm, strlen ( realm ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, password, strlen ( password ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), ha1 );
-
-	/* Generate HA2 */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, method, strlen ( method ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, uri, strlen ( uri ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), ha2 );
-
-	/* Generate response */
-	digest_init ( &md5_algorithm, ctx );
-	digest_update ( &md5_algorithm, ctx, ha1, strlen ( ha1 ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, nonce, strlen ( nonce ) );
-	digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
-	digest_update ( &md5_algorithm, ctx, ha2, strlen ( ha2 ) );
-	digest_final ( &md5_algorithm, ctx, digest );
-	base16_encode ( digest, sizeof ( digest ), response );
-
-	/* Generate the authorisation string */
-	len = asprintf ( &auth, "Authorization: Digest username=\"%s\", "
-			 "realm=\"%s\", nonce=\"%s\", uri=\"%s\", "
-			 "%s%s%sresponse=\"%s\"\r\n", user, realm, nonce, uri,
-			 ( opaque ? "opaque=\"" : "" ),
-			 ( opaque ? opaque : "" ),
-			 ( opaque ? "\", " : "" ), response );
-	if ( len < 0 )
-		return NULL;
-
-	return auth;
-}
 
 /**
- * Generate HTTP POST parameter list
+ * Construct HTTP parameter list
  *
- * @v http		HTTP request
+ * @v params		Parameter list
  * @v buf		Buffer to contain HTTP POST parameters
  * @v len		Length of buffer
  * @ret len		Length of parameter list (excluding terminating NUL)
  */
-static size_t http_post_params ( struct http_request *http,
-				 char *buf, size_t len ) {
+static size_t http_params ( struct parameters *params, char *buf, size_t len ) {
 	struct parameter *param;
 	ssize_t remaining = len;
 	size_t frag_len;
 
 	/* Add each parameter in the form "key=value", joined with "&" */
 	len = 0;
-	for_each_param ( param, http->uri->params ) {
+	for_each_param ( param, params ) {
 
 		/* Add the "&", if applicable */
 		if ( len ) {
@@ -1201,374 +1853,78 @@ static size_t http_post_params ( struct http_request *http,
 }
 
 /**
- * Generate HTTP POST body
+ * Open HTTP transaction for simple GET URI
  *
- * @v http		HTTP request
- * @ret post		I/O buffer containing POST body, or NULL on error
+ * @v xfer		Data transfer interface
+ * @v uri		Request URI
+ * @ret rc		Return status code
  */
-static struct io_buffer * http_post ( struct http_request *http ) {
-	struct io_buffer *post;
-	size_t len;
-	size_t check_len;
+static int http_open_get_uri ( struct interface *xfer, struct uri *uri ) {
 
-	/* Calculate length of parameter list */
-	len = http_post_params ( http, NULL, 0 );
-
-	/* Allocate parameter list */
-	post = alloc_iob ( len + 1 /* NUL */ );
-	if ( ! post )
-		return NULL;
-
-	/* Fill parameter list */
-	check_len = http_post_params ( http, iob_put ( post, len ),
-				       ( len + 1 /* NUL */ ) );
-	assert ( len == check_len );
-	DBGC ( http, "HTTP %p POST %s\n", http, ( ( char * ) post->data ) );
-
-	return post;
+	return http_open ( xfer, &http_get, uri, NULL, NULL );
 }
 
 /**
- * HTTP process
+ * Open HTTP transaction for simple POST URI
  *
- * @v http		HTTP request
+ * @v xfer		Data transfer interface
+ * @v uri		Request URI
+ * @ret rc		Return status code
  */
-static void http_step ( struct http_request *http ) {
-	struct io_buffer *post;
-	struct uri host_uri;
-	struct uri path_uri;
-	char *host_uri_string;
-	char *path_uri_string;
-	char *method;
-	char *range;
-	char *auth;
-	char *content;
-	int len;
+static int http_open_post_uri ( struct interface *xfer, struct uri *uri ) {
+	struct parameters *params = uri->params;
+	struct http_request_content content;
+	void *data;
+	size_t len;
+	size_t check_len;
 	int rc;
 
-	/* Do nothing if we have already transmitted the request */
-	if ( ! ( http->flags & HTTP_TX_PENDING ) )
-		return;
-
-	/* Do nothing until socket is ready */
-	if ( ! xfer_window ( &http->socket ) )
-		return;
-
-	/* Force a HEAD request if we have nowhere to send any received data */
-	if ( ( xfer_window ( &http->xfer ) == 0 ) &&
-	     ( http->rx_buffer == UNULL ) ) {
-		http->flags |= ( HTTP_HEAD_ONLY | HTTP_CLIENT_KEEPALIVE );
-	}
-
-	/* Determine method */
-	method = ( ( http->flags & HTTP_HEAD_ONLY ) ? "HEAD" :
-		   ( http->uri->params ? "POST" : "GET" ) );
+	/* Calculate length of parameter list */
+	len = http_params ( params, NULL, 0 );
 
-	/* Construct host URI */
-	memset ( &host_uri, 0, sizeof ( host_uri ) );
-	host_uri.host = http->uri->host;
-	host_uri.port = http->uri->port;
-	host_uri_string = format_uri_alloc ( &host_uri );
-	if ( ! host_uri_string ) {
+	/* Allocate temporary parameter list */
+	data = zalloc ( len + 1 /* NUL */ );
+	if ( ! data ) {
 		rc = -ENOMEM;
-		goto err_host_uri;
+		goto err_alloc;
 	}
 
-	/* Construct path URI */
-	memset ( &path_uri, 0, sizeof ( path_uri ) );
-	path_uri.path = ( http->uri->path ? http->uri->path : "/" );
-	path_uri.query = http->uri->query;
-	path_uri_string = format_uri_alloc ( &path_uri );
-	if ( ! path_uri_string ) {
-		rc = -ENOMEM;
-		goto err_path_uri;
-	}
+	/* Construct temporary parameter list */
+	check_len = http_params ( params, data, ( len + 1 /* NUL */ ) );
+	assert ( check_len == len );
 
-	/* Calculate range request parameters if applicable */
-	if ( http->partial_len ) {
-		len = asprintf ( &range, "Range: bytes=%zd-%zd\r\n",
-				 http->partial_start,
-				 ( http->partial_start + http->partial_len
-				   - 1 ) );
-		if ( len < 0 ) {
-			rc = len;
-			goto err_range;
-		}
-	} else {
-		range = NULL;
-	}
+	/* Construct request content */
+	content.type = "application/x-www-form-urlencoded";
+	content.data = data;
+	content.len = len;
 
-	/* Construct authorisation, if applicable */
-	if ( http->flags & HTTP_BASIC_AUTH ) {
-		auth = http_basic_auth ( http );
-		if ( ! auth ) {
-			rc = -ENOMEM;
-			goto err_auth;
-		}
-	} else if ( http->flags & HTTP_DIGEST_AUTH ) {
-		auth = http_digest_auth ( http, method, path_uri_string );
-		if ( ! auth ) {
-			rc = -ENOMEM;
-			goto err_auth;
-		}
-	} else {
-		auth = NULL;
-	}
+	/* Open HTTP transaction */
+	if ( ( rc = http_open ( xfer, &http_post, uri, NULL, &content ) ) != 0 )
+		goto err_open;
 
-	/* Construct POST content, if applicable */
-	if ( http->uri->params ) {
-		post = http_post ( http );
-		if ( ! post ) {
-			rc = -ENOMEM;
-			goto err_post;
-		}
-		len = asprintf ( &content, "Content-Type: "
-				 "application/x-www-form-urlencoded\r\n"
-				 "Content-Length: %zd\r\n", iob_len ( post ) );
-		if ( len < 0 ) {
-			rc = len;
-			goto err_content;
-		}
-	} else {
-		post = NULL;
-		content = NULL;
-	}
-
-	/* Mark request as transmitted */
-	http->flags &= ~HTTP_TX_PENDING;
-
-	/* Send request */
-	if ( ( rc = xfer_printf ( &http->socket,
-				  "%s %s HTTP/1.1\r\n"
-				  "User-Agent: iPXE/%s\r\n"
-				  "Host: %s\r\n"
-				  "%s%s%s%s"
-				  "\r\n",
-				  method, path_uri_string, product_version,
-				  host_uri_string,
-				  ( ( http->flags & HTTP_CLIENT_KEEPALIVE ) ?
-				    "Connection: keep-alive\r\n" : "" ),
-				  ( range ? range : "" ),
-				  ( auth ? auth : "" ),
-				  ( content ? content : "" ) ) ) != 0 ) {
-		goto err_xfer;
-	}
-
-	/* Send POST content, if applicable */
-	if ( post ) {
-		if ( ( rc = xfer_deliver_iob ( &http->socket,
-					       iob_disown ( post ) ) ) != 0 )
-			goto err_xfer_post;
-	}
-
- err_xfer_post:
- err_xfer:
-	free ( content );
- err_content:
-	free ( post );
- err_post:
-	free ( auth );
- err_auth:
-	free ( range );
- err_range:
-	free ( path_uri_string );
- err_path_uri:
-	free ( host_uri_string );
- err_host_uri:
-	if ( rc != 0 )
-		http_close ( http, rc );
-}
-
-/**
- * Check HTTP data transfer flow control window
- *
- * @v http		HTTP request
- * @ret len		Length of window
- */
-static size_t http_xfer_window ( struct http_request *http ) {
-
-	/* New block commands may be issued only when we are idle */
-	return ( ( http->rx_state == HTTP_RX_IDLE ) ? 1 : 0 );
-}
-
-/**
- * Initiate HTTP partial read
- *
- * @v http		HTTP request
- * @v partial		Partial transfer interface
- * @v offset		Starting offset
- * @v buffer		Data buffer
- * @v len		Length
- * @ret rc		Return status code
- */
-static int http_partial_read ( struct http_request *http,
-			       struct interface *partial,
-			       size_t offset, userptr_t buffer, size_t len ) {
-
-	/* Sanity check */
-	if ( http_xfer_window ( http ) == 0 )
-		return -EBUSY;
-
-	/* Initialise partial transfer parameters */
-	http->rx_buffer = buffer;
-	http->partial_start = offset;
-	http->partial_len = len;
-
-	/* Schedule request */
-	http->rx_state = HTTP_RX_RESPONSE;
-	http->flags = ( HTTP_TX_PENDING | HTTP_CLIENT_KEEPALIVE );
-	if ( ! len )
-		http->flags |= HTTP_HEAD_ONLY;
-	process_add ( &http->process );
-
-	/* Attach to parent interface and return */
-	intf_plug_plug ( &http->partial, partial );
-
-	return 0;
-}
-
-/**
- * Issue HTTP block device read
- *
- * @v http		HTTP request
- * @v block		Block data interface
- * @v lba		Starting logical block address
- * @v count		Number of blocks to transfer
- * @v buffer		Data buffer
- * @v len		Length of data buffer
- * @ret rc		Return status code
- */
-static int http_block_read ( struct http_request *http,
-			     struct interface *block,
-			     uint64_t lba, unsigned int count,
-			     userptr_t buffer, size_t len __unused ) {
-
-	return http_partial_read ( http, block, ( lba * HTTP_BLKSIZE ),
-				   buffer, ( count * HTTP_BLKSIZE ) );
-}
-
-/**
- * Read HTTP block device capacity
- *
- * @v http		HTTP request
- * @v block		Block data interface
- * @ret rc		Return status code
- */
-static int http_block_read_capacity ( struct http_request *http,
-				      struct interface *block ) {
-
-	return http_partial_read ( http, block, 0, 0, 0 );
-}
-
-/**
- * Describe HTTP device in an ACPI table
- *
- * @v http		HTTP request
- * @v acpi		ACPI table
- * @v len		Length of ACPI table
- * @ret rc		Return status code
- */
-static int http_acpi_describe ( struct http_request *http,
-				struct acpi_description_header *acpi,
-				size_t len ) {
-
-	DBGC ( http, "HTTP %p cannot yet describe device in an ACPI table\n",
-	       http );
-	( void ) acpi;
-	( void ) len;
-	return 0;
+ err_open:
+	free ( data );
+ err_alloc:
+	return rc;
 }
 
-/** HTTP socket interface operations */
-static struct interface_operation http_socket_operations[] = {
-	INTF_OP ( xfer_window, struct http_request *, http_socket_window ),
-	INTF_OP ( xfer_deliver, struct http_request *, http_socket_deliver ),
-	INTF_OP ( xfer_window_changed, struct http_request *, http_step ),
-	INTF_OP ( intf_close, struct http_request *, http_socket_close ),
-};
-
-/** HTTP socket interface descriptor */
-static struct interface_descriptor http_socket_desc =
-	INTF_DESC_PASSTHRU ( struct http_request, socket,
-			     http_socket_operations, xfer );
-
-/** HTTP partial transfer interface operations */
-static struct interface_operation http_partial_operations[] = {
-	INTF_OP ( intf_close, struct http_request *, http_close ),
-};
-
-/** HTTP partial transfer interface descriptor */
-static struct interface_descriptor http_partial_desc =
-	INTF_DESC ( struct http_request, partial, http_partial_operations );
-
-/** HTTP data transfer interface operations */
-static struct interface_operation http_xfer_operations[] = {
-	INTF_OP ( xfer_window, struct http_request *, http_xfer_window ),
-	INTF_OP ( block_read, struct http_request *, http_block_read ),
-	INTF_OP ( block_read_capacity, struct http_request *,
-		  http_block_read_capacity ),
-	INTF_OP ( intf_close, struct http_request *, http_close ),
-	INTF_OP ( acpi_describe, struct http_request *, http_acpi_describe ),
-};
-
-/** HTTP data transfer interface descriptor */
-static struct interface_descriptor http_xfer_desc =
-	INTF_DESC_PASSTHRU ( struct http_request, xfer,
-			     http_xfer_operations, socket );
-
-/** HTTP process descriptor */
-static struct process_descriptor http_process_desc =
-	PROC_DESC_ONCE ( struct http_request, process, http_step );
-
 /**
- * Initiate an HTTP connection, with optional filter
+ * Open HTTP transaction for simple URI
  *
  * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @v default_port	Default port number
- * @v filter		Filter to apply to socket, or NULL
+ * @v uri		Request URI
  * @ret rc		Return status code
  */
-int http_open_filter ( struct interface *xfer, struct uri *uri,
-		       unsigned int default_port,
-		       int ( * filter ) ( struct interface *xfer,
-					  const char *name,
-					  struct interface **next ) ) {
-	struct http_request *http;
-	int rc;
-
-	/* Sanity checks */
-	if ( ! uri->host )
-		return -EINVAL;
-
-	/* Allocate and populate HTTP structure */
-	http = zalloc ( sizeof ( *http ) );
-	if ( ! http )
-		return -ENOMEM;
-	ref_init ( &http->refcnt, http_free );
-	intf_init ( &http->xfer, &http_xfer_desc, &http->refcnt );
-	intf_init ( &http->partial, &http_partial_desc, &http->refcnt );
-	http->uri = uri_get ( uri );
-	http->default_port = default_port;
-	http->filter = filter;
-	intf_init ( &http->socket, &http_socket_desc, &http->refcnt );
-	process_init ( &http->process, &http_process_desc, &http->refcnt );
-	timer_init ( &http->timer, http_retry, &http->refcnt );
-	http->flags = HTTP_TX_PENDING;
+int http_open_uri ( struct interface *xfer, struct uri *uri ) {
 
-	/* Open socket */
-	if ( ( rc = http_socket_open ( http ) ) != 0 )
-		goto err;
-
-	/* Attach to parent interface, mortalise self, and return */
-	intf_plug_plug ( &http->xfer, xfer );
-	ref_put ( &http->refcnt );
-	return 0;
-
- err:
-	DBGC ( http, "HTTP %p could not create request: %s\n",
-	       http, strerror ( rc ) );
-	http_close ( http, rc );
-	ref_put ( &http->refcnt );
-	return rc;
+	/* Open GET/POST URI as applicable */
+	if ( uri->params ) {
+		return http_open_post_uri ( xfer, uri );
+	} else {
+		return http_open_get_uri ( xfer, uri );
+	}
 }
+
+/* Drag in HTTP extensions */
+REQUIRING_SYMBOL ( http_open );
+REQUIRE_OBJECT ( config_http );
diff --git a/qemu/roms/ipxe/src/net/tcp/httpdigest.c b/qemu/roms/ipxe/src/net/tcp/httpdigest.c
new file mode 100644
index 000000000..626dd7e9d
--- /dev/null
+++ b/qemu/roms/ipxe/src/net/tcp/httpdigest.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * @file
+ *
+ * Hyper Text Transfer Protocol (HTTP) Digest authentication
+ *
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <strings.h>
+#include <ipxe/uri.h>
+#include <ipxe/md5.h>
+#include <ipxe/base16.h>
+#include <ipxe/vsprintf.h>
+#include <ipxe/http.h>
+
+/* Disambiguate the various error causes */
+#define EACCES_USERNAME __einfo_error ( EINFO_EACCES_USERNAME )
+#define EINFO_EACCES_USERNAME						\
+	__einfo_uniqify ( EINFO_EACCES, 0x01,				\
+			  "No username available for Digest authentication" )
+
+/**
+ * Initialise HTTP Digest
+ *
+ * @v ctx		Digest context
+ * @v string		Initial string
+ */
+static void http_digest_init ( struct md5_context *ctx ) {
+
+	/* Initialise MD5 digest */
+	digest_init ( &md5_algorithm, ctx );
+}
+
+/**
+ * Update HTTP Digest with new data
+ *
+ * @v ctx		Digest context
+ * @v string		String to append
+ */
+static void http_digest_update ( struct md5_context *ctx, const char *string ) {
+	static const char colon = ':';
+
+	/* Add (possibly colon-separated) field to MD5 digest */
+	if ( ctx->len )
+		digest_update ( &md5_algorithm, ctx, &colon, sizeof ( colon ) );
+	digest_update ( &md5_algorithm, ctx, string, strlen ( string ) );
+}
+
+/**
+ * Finalise HTTP Digest
+ *
+ * @v ctx		Digest context
+ * @v out		Buffer for digest output
+ * @v len		Buffer length
+ */
+static void http_digest_final ( struct md5_context *ctx, char *out,
+				size_t len ) {
+	uint8_t digest[MD5_DIGEST_SIZE];
+
+	/* Finalise and base16-encode MD5 digest */
+	digest_final ( &md5_algorithm, ctx, digest );
+	base16_encode ( digest, sizeof ( digest ), out, len );
+}
+
+/**
+ * Perform HTTP Digest authentication
+ *
+ * @v http		HTTP transaction
+ * @ret rc		Return status code
+ */
+static int http_digest_authenticate ( struct http_transaction *http ) {
+	struct http_request_auth *req = &http->request.auth;
+	struct http_response_auth *rsp = &http->response.auth;
+	char ha1[ base16_encoded_len ( MD5_DIGEST_SIZE ) + 1 /* NUL */ ];
+	char ha2[ base16_encoded_len ( MD5_DIGEST_SIZE ) + 1 /* NUL */ ];
+	static const char md5sess[] = "MD5-sess";
+	static const char md5[] = "MD5";
+	struct md5_context ctx;
+
+	/* Check for required response parameters */
+	if ( ! rsp->realm ) {
+		DBGC ( http, "HTTP %p has no realm for Digest authentication\n",
+		       http );
+		return -EINVAL;
+	}
+	if ( ! rsp->nonce ) {
+		DBGC ( http, "HTTP %p has no nonce for Digest authentication\n",
+		       http );
+		return -EINVAL;
+	}
+
+	/* Record username and password */
+	if ( ! http->uri->user ) {
+		DBGC ( http, "HTTP %p has no username for Digest "
+		       "authentication\n", http );
+		return -EACCES_USERNAME;
+	}
+	req->username = http->uri->user;
+	req->password = ( http->uri->password ? http->uri->password : "" );
+
+	/* Handle quality of protection */
+	if ( rsp->qop ) {
+
+		/* Use "auth" in subsequent request */
+		req->qop = "auth";
+
+		/* Generate a client nonce */
+		snprintf ( req->cnonce, sizeof ( req->cnonce ),
+			   "%08lx", random() );
+
+		/* Determine algorithm */
+		req->algorithm = md5;
+		if ( rsp->algorithm &&
+		     ( strcasecmp ( rsp->algorithm, md5sess ) == 0 ) ) {
+			req->algorithm = md5sess;
+		}
+	}
+
+	/* Generate HA1 */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, req->username );
+	http_digest_update ( &ctx, rsp->realm );
+	http_digest_update ( &ctx, req->password );
+	http_digest_final ( &ctx, ha1, sizeof ( ha1 ) );
+	if ( req->algorithm == md5sess ) {
+		http_digest_init ( &ctx );
+		http_digest_update ( &ctx, ha1 );
+		http_digest_update ( &ctx, rsp->nonce );
+		http_digest_update ( &ctx, req->cnonce );
+		http_digest_final ( &ctx, ha1, sizeof ( ha1 ) );
+	}
+
+	/* Generate HA2 */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, http->request.method->name );
+	http_digest_update ( &ctx, http->request.uri );
+	http_digest_final ( &ctx, ha2, sizeof ( ha2 ) );
+
+	/* Generate response */
+	http_digest_init ( &ctx );
+	http_digest_update ( &ctx, ha1 );
+	http_digest_update ( &ctx, rsp->nonce );
+	if ( req->qop ) {
+		http_digest_update ( &ctx, HTTP_DIGEST_NC );
+		http_digest_update ( &ctx, req->cnonce );
+		http_digest_update ( &ctx, req->qop );
+	}
+	http_digest_update ( &ctx, ha2 );
+	http_digest_final ( &ctx, req->response, sizeof ( req->response ) );
+
+	return 0;
+}
+
+/**
+ * Construct HTTP "Authorization" header for Digest authentication
+ *
+ * @v http		HTTP transaction
+ * @v buf		Buffer
+ * @v len		Length of buffer
+ * @ret len		Length of header value, or negative error
+ */
+static int http_format_digest_auth ( struct http_transaction *http,
+				     char *buf, size_t len ) {
+	struct http_request_auth *req = &http->request.auth;
+	struct http_response_auth *rsp = &http->response.auth;
+	size_t used = 0;
+
+	/* Sanity checks */
+	assert ( rsp->realm != NULL );
+	assert ( rsp->nonce != NULL );
+	assert ( req->username != NULL );
+	if ( req->qop ) {
+		assert ( req->algorithm != NULL );
+		assert ( req->cnonce[0] != '\0' );
+	}
+	assert ( req->response[0] != '\0' );
+
+	/* Construct response */
+	used += ssnprintf ( ( buf + used ), ( len - used ),
+			    "realm=\"%s\", nonce=\"%s\", uri=\"%s\", "
+			    "username=\"%s\"", rsp->realm, rsp->nonce,
+			    http->request.uri, req->username );
+	if ( rsp->opaque ) {
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    ", opaque=\"%s\"", rsp->opaque );
+	}
+	if ( req->qop ) {
+		used += ssnprintf ( ( buf + used ), ( len - used ),
+				    ", qop=%s, algorithm=%s, cnonce=\"%s\", "
+				    "nc=" HTTP_DIGEST_NC, req->qop,
+				    req->algorithm, req->cnonce );
+	}
+	used += ssnprintf ( ( buf + used ), ( len - used ),
+			    ", response=\"%s\"", req->response );
+
+	return used;
+}
+
+/** HTTP Digest authentication scheme */
+struct http_authentication http_digest_auth __http_authentication = {
+	.name = "Digest",
+	.authenticate = http_digest_authenticate,
+	.format = http_format_digest_auth,
+};
+
+/* Drag in HTTP authentication support */
+REQUIRING_SYMBOL ( http_digest_auth );
+REQUIRE_OBJECT ( httpauth );
diff --git a/qemu/roms/ipxe/src/net/tcp/https.c b/qemu/roms/ipxe/src/net/tcp/https.c
index 6112acdae..e91000322 100644
--- a/qemu/roms/ipxe/src/net/tcp/https.c
+++ b/qemu/roms/ipxe/src/net/tcp/https.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * @file
@@ -26,7 +30,6 @@ FILE_LICENCE ( GPL2_OR_LATER );
  *
  */
 
-#include <stddef.h>
 #include <ipxe/open.h>
 #include <ipxe/tls.h>
 #include <ipxe/http.h>
@@ -34,19 +37,15 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 FEATURE ( FEATURE_PROTOCOL, "HTTPS", DHCP_EB_FEATURE_HTTPS, 1 );
 
-/**
- * Initiate an HTTPS connection
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int https_open ( struct interface *xfer, struct uri *uri ) {
-	return http_open_filter ( xfer, uri, HTTPS_PORT, add_tls );
-}
-
 /** HTTPS URI opener */
 struct uri_opener https_uri_opener __uri_opener = {
 	.scheme	= "https",
-	.open	= https_open,
+	.open	= http_open_uri,
+};
+
+/** HTTP URI scheme */
+struct http_scheme https_scheme __http_scheme = {
+	.name = "https",
+	.port = HTTPS_PORT,
+	.filter = add_tls,
 };
diff --git a/qemu/roms/ipxe/src/net/tcp/iscsi.c b/qemu/roms/ipxe/src/net/tcp/iscsi.c
index 03c6d0f23..019a4c14e 100644
--- a/qemu/roms/ipxe/src/net/tcp/iscsi.c
+++ b/qemu/roms/ipxe/src/net/tcp/iscsi.c
@@ -15,14 +15,19 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <ctype.h>
 #include <errno.h>
 #include <assert.h>
 #include <byteswap.h>
@@ -127,7 +132,7 @@ FEATURE ( FEATURE_PROTOCOL, "iSCSI", DHCP_EB_FEATURE_ISCSI, 1 );
 #define EPROTO_INVALID_LARGE_BINARY \
 	__einfo_error ( EINFO_EPROTO_INVALID_LARGE_BINARY )
 #define EINFO_EPROTO_INVALID_LARGE_BINARY \
-	__einfo_uniqify ( EINFO_EPROTO, 0x03, "Invalid large binary" )
+	__einfo_uniqify ( EINFO_EPROTO, 0x03, "Invalid large binary value" )
 #define EPROTO_INVALID_CHAP_RESPONSE \
 	__einfo_error ( EINFO_EPROTO_INVALID_CHAP_RESPONSE )
 #define EINFO_EPROTO_INVALID_CHAP_RESPONSE \
@@ -704,7 +709,7 @@ static int iscsi_build_login_request_strings ( struct iscsi_session *iscsi,
 		char buf[ base16_encoded_len ( iscsi->chap.response_len ) + 1 ];
 		assert ( iscsi->initiator_username != NULL );
 		base16_encode ( iscsi->chap.response, iscsi->chap.response_len,
-				buf );
+				buf, sizeof ( buf ) );
 		used += ssnprintf ( data + used, len - used,
 				    "CHAP_N=%s%cCHAP_R=0x%s%c",
 				    iscsi->initiator_username, 0, buf, 0 );
@@ -714,7 +719,7 @@ static int iscsi_build_login_request_strings ( struct iscsi_session *iscsi,
 		size_t challenge_len = ( sizeof ( iscsi->chap_challenge ) - 1 );
 		char buf[ base16_encoded_len ( challenge_len ) + 1 ];
 		base16_encode ( ( iscsi->chap_challenge + 1 ), challenge_len,
-				buf );
+				buf, sizeof ( buf ) );
 		used += ssnprintf ( data + used, len - used,
 				    "CHAP_I=%d%cCHAP_C=0x%s%c",
 				    iscsi->chap_challenge[0], 0, buf, 0 );
@@ -824,38 +829,27 @@ static int iscsi_tx_login_request ( struct iscsi_session *iscsi ) {
 }
 
 /**
- * Calculate maximum length of decoded large binary value
- *
- * @v encoded		Encoded large binary value
- * @v max_raw_len	Maximum length of raw data
- */
-static inline size_t
-iscsi_large_binary_decoded_max_len ( const char *encoded ) {
-	return ( strlen ( encoded ) ); /* Decoding never expands data */
-}
-
-/**
  * Decode large binary value
  *
  * @v encoded		Encoded large binary value
  * @v raw		Raw data
+ * @v len		Length of data buffer
  * @ret len		Length of raw data, or negative error
  */
-static int iscsi_large_binary_decode ( const char *encoded, uint8_t *raw ) {
-
-	if ( encoded[0] != '0' )
-		return -EPROTO_INVALID_LARGE_BINARY;
-
-	switch ( encoded[1] ) {
-	case 'x' :
-	case 'X' :
-		return base16_decode ( ( encoded + 2 ), raw );
-	case 'b' :
-	case 'B' :
-		return base64_decode ( ( encoded + 2 ), raw );
-	default:
-		return -EPROTO_INVALID_LARGE_BINARY;
+static int iscsi_large_binary_decode ( const char *encoded, uint8_t *raw,
+				       size_t len ) {
+
+	/* Check for initial '0x' or '0b' and decode as appropriate */
+	if ( *(encoded++) == '0' ) {
+		switch ( tolower ( *(encoded++) ) ) {
+		case 'x' :
+			return base16_decode ( encoded, raw, len );
+		case 'b' :
+			return base64_decode ( encoded, raw, len );
+		}
 	}
+
+	return -EPROTO_INVALID_LARGE_BINARY;
 }
 
 /**
@@ -982,19 +976,19 @@ static int iscsi_handle_chap_i_value ( struct iscsi_session *iscsi,
  */
 static int iscsi_handle_chap_c_value ( struct iscsi_session *iscsi,
 				       const char *value ) {
-	uint8_t buf[ iscsi_large_binary_decoded_max_len ( value ) ];
+	uint8_t buf[ strlen ( value ) ]; /* Decoding never expands data */
 	unsigned int i;
-	size_t len;
+	int len;
 	int rc;
 
 	/* Process challenge */
-	rc = iscsi_large_binary_decode ( value, buf );
-	if ( rc < 0 ) {
+	len = iscsi_large_binary_decode ( value, buf, sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
 		DBGC ( iscsi, "iSCSI %p invalid CHAP challenge \"%s\": %s\n",
 		       iscsi, value, strerror ( rc ) );
 		return rc;
 	}
-	len = rc;
 	chap_update ( &iscsi->chap, buf, len );
 
 	/* Build CHAP response */
@@ -1052,8 +1046,8 @@ static int iscsi_handle_chap_n_value ( struct iscsi_session *iscsi,
  */
 static int iscsi_handle_chap_r_value ( struct iscsi_session *iscsi,
 				       const char *value ) {
-	uint8_t buf[ iscsi_large_binary_decoded_max_len ( value ) ];
-	size_t len;
+	uint8_t buf[ strlen ( value ) ]; /* Decoding never expands data */
+	int len;
 	int rc;
 
 	/* Generate CHAP response for verification */
@@ -1073,16 +1067,16 @@ static int iscsi_handle_chap_r_value ( struct iscsi_session *iscsi,
 	chap_respond ( &iscsi->chap );
 
 	/* Process response */
-	rc = iscsi_large_binary_decode ( value, buf );
-	if ( rc < 0 ) {
+	len = iscsi_large_binary_decode ( value, buf, sizeof ( buf ) );
+	if ( len < 0 ) {
+		rc = len;
 		DBGC ( iscsi, "iSCSI %p invalid CHAP response \"%s\": %s\n",
 		       iscsi, value, strerror ( rc ) );
 		return rc;
 	}
-	len = rc;
 
 	/* Check CHAP response */
-	if ( len != iscsi->chap.response_len ) {
+	if ( len != ( int ) iscsi->chap.response_len ) {
 		DBGC ( iscsi, "iSCSI %p invalid CHAP response length\n",
 		       iscsi );
 		return -EPROTO_INVALID_CHAP_RESPONSE;
@@ -1445,8 +1439,10 @@ static void iscsi_tx_done ( struct iscsi_session *iscsi ) {
 	switch ( common->opcode & ISCSI_OPCODE_MASK ) {
 	case ISCSI_OPCODE_DATA_OUT:
 		iscsi_data_out_done ( iscsi );
+		break;
 	case ISCSI_OPCODE_LOGIN_REQUEST:
 		iscsi_login_request_done ( iscsi );
+		break;
 	default:
 		/* No action */
 		break;
diff --git a/qemu/roms/ipxe/src/net/tcp/syslogs.c b/qemu/roms/ipxe/src/net/tcp/syslogs.c
index 095afc543..0c07f86d5 100644
--- a/qemu/roms/ipxe/src/net/tcp/syslogs.c
+++ b/qemu/roms/ipxe/src/net/tcp/syslogs.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/net/tcpip.c b/qemu/roms/ipxe/src/net/tcpip.c
index 4bcbe64bb..5ad982fd1 100644
--- a/qemu/roms/ipxe/src/net/tcpip.c
+++ b/qemu/roms/ipxe/src/net/tcpip.c
@@ -17,7 +17,7 @@
  * TCP/IP transport-network layer interface
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * Process a received TCP/IP packet
@@ -235,7 +235,7 @@ int tcpip_bind ( struct sockaddr_tcpip *st_local,
 	/* Otherwise, find an available port in the range [1,1023] or
 	 * [1025,65535] as appropriate.
 	 */
-	min_port = ( ( ( ! flags ) & TCPIP_BIND_PRIVILEGED ) + 1 );
+	min_port = ( ( ( ~flags ) & TCPIP_BIND_PRIVILEGED ) + 1 );
 	max_port = ( ( flags & TCPIP_BIND_PRIVILEGED ) - 1 );
 	offset = random();
 	for ( i = 0 ; i <= max_port ; i++ ) {
diff --git a/qemu/roms/ipxe/src/net/tls.c b/qemu/roms/ipxe/src/net/tls.c
index 30ccc932e..db01fb291 100644
--- a/qemu/roms/ipxe/src/net/tls.c
+++ b/qemu/roms/ipxe/src/net/tls.c
@@ -179,20 +179,29 @@ static void tls_clear_cipher ( struct tls_session *tls,
  ******************************************************************************
  */
 
+/** A TLS 24-bit integer
+ *
+ * TLS uses 24-bit integers in several places, which are awkward to
+ * parse in C.
+ */
+typedef struct {
+	/** High byte */
+	uint8_t high;
+	/** Low word */
+	uint16_t low;
+} __attribute__ (( packed )) tls24_t;
+
 /**
  * Extract 24-bit field value
  *
  * @v field24		24-bit field
  * @ret value		Field value
  *
- * TLS uses 24-bit integers in several places, which are awkward to
- * parse in C.
  */
 static inline __attribute__ (( always_inline )) unsigned long
-tls_uint24 ( const uint8_t field24[3] ) {
-	const uint32_t *field32 __attribute__ (( may_alias )) =
-		( ( const void * ) field24 );
-	return ( be32_to_cpu ( *field32 ) >> 8 );
+tls_uint24 ( const tls24_t *field24 ) {
+
+	return ( ( field24->high << 16 ) | be16_to_cpu ( field24->low ) );
 }
 
 /**
@@ -200,13 +209,11 @@ tls_uint24 ( const uint8_t field24[3] ) {
  *
  * @v field24		24-bit field
  * @v value		Field value
- *
- * The field must be pre-zeroed.
  */
-static void tls_set_uint24 ( uint8_t field24[3], unsigned long value ) {
-	uint32_t *field32 __attribute__ (( may_alias )) =
-		( ( void * ) field24 );
-	*field32 |= cpu_to_be32 ( value << 8 );
+static void tls_set_uint24 ( tls24_t *field24, unsigned long value ) {
+
+	field24->high = ( value >> 16 );
+	field24->low = cpu_to_be16 ( value );
 }
 
 /**
@@ -659,41 +666,8 @@ struct tls_cipher_suite tls_cipher_suite_null = {
 	.digest = &digest_null,
 };
 
-/** Supported cipher suites, in order of preference */
-struct tls_cipher_suite tls_cipher_suites[] = {
-	{
-		.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA256 ),
-		.key_len = ( 256 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha256_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA256 ),
-		.key_len = ( 128 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha256_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_256_CBC_SHA ),
-		.key_len = ( 256 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha1_algorithm,
-	},
-	{
-		.code = htons ( TLS_RSA_WITH_AES_128_CBC_SHA ),
-		.key_len = ( 128 / 8 ),
-		.pubkey = &rsa_algorithm,
-		.cipher = &aes_cbc_algorithm,
-		.digest = &sha1_algorithm,
-	},
-};
-
 /** Number of supported cipher suites */
-#define TLS_NUM_CIPHER_SUITES \
-	( sizeof ( tls_cipher_suites ) / sizeof ( tls_cipher_suites[0] ) )
+#define TLS_NUM_CIPHER_SUITES table_num_entries ( TLS_CIPHER_SUITES )
 
 /**
  * Identify cipher suite
@@ -704,11 +678,9 @@ struct tls_cipher_suite tls_cipher_suites[] = {
 static struct tls_cipher_suite *
 tls_find_cipher_suite ( unsigned int cipher_suite ) {
 	struct tls_cipher_suite *suite;
-	unsigned int i;
 
 	/* Identify cipher suite */
-	for ( i = 0 ; i < TLS_NUM_CIPHER_SUITES ; i++ ) {
-		suite = &tls_cipher_suites[i];
+	for_each_table_entry ( suite, TLS_CIPHER_SUITES ) {
 		if ( suite->code == cipher_suite )
 			return suite;
 	}
@@ -841,26 +813,9 @@ static int tls_change_cipher ( struct tls_session *tls,
  ******************************************************************************
  */
 
-/** Supported signature and hash algorithms
- *
- * Note that the default (TLSv1.1 and earlier) algorithm using
- * MD5+SHA1 is never explicitly specified.
- */
-struct tls_signature_hash_algorithm tls_signature_hash_algorithms[] = {
-	{
-		.code = {
-			.signature = TLS_RSA_ALGORITHM,
-			.hash = TLS_SHA256_ALGORITHM,
-		},
-		.pubkey = &rsa_algorithm,
-		.digest = &sha256_algorithm,
-	},
-};
-
 /** Number of supported signature and hash algorithms */
-#define TLS_NUM_SIG_HASH_ALGORITHMS			\
-	( sizeof ( tls_signature_hash_algorithms ) /	\
-	  sizeof ( tls_signature_hash_algorithms[0] ) )
+#define TLS_NUM_SIG_HASH_ALGORITHMS \
+	table_num_entries ( TLS_SIG_HASH_ALGORITHMS )
 
 /**
  * Find TLS signature and hash algorithm
@@ -873,11 +828,9 @@ static struct tls_signature_hash_algorithm *
 tls_signature_hash_algorithm ( struct pubkey_algorithm *pubkey,
 			       struct digest_algorithm *digest ) {
 	struct tls_signature_hash_algorithm *sig_hash;
-	unsigned int i;
 
 	/* Identify signature and hash algorithm */
-	for ( i = 0 ; i < TLS_NUM_SIG_HASH_ALGORITHMS ; i++ ) {
-		sig_hash = &tls_signature_hash_algorithms[i];
+	for_each_table_entry ( sig_hash, TLS_SIG_HASH_ALGORITHMS ) {
 		if ( ( sig_hash->pubkey == pubkey ) &&
 		     ( sig_hash->digest == digest ) ) {
 			return sig_hash;
@@ -994,8 +947,17 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 			struct {
 				uint8_t max;
 			} __attribute__ (( packed )) max_fragment_length;
+			uint16_t signature_algorithms_type;
+			uint16_t signature_algorithms_len;
+			struct {
+				uint16_t len;
+				struct tls_signature_hash_id
+					code[TLS_NUM_SIG_HASH_ALGORITHMS];
+			} __attribute__ (( packed )) signature_algorithms;
 		} __attribute__ (( packed )) extensions;
 	} __attribute__ (( packed )) hello;
+	struct tls_cipher_suite *suite;
+	struct tls_signature_hash_algorithm *sighash;
 	unsigned int i;
 
 	memset ( &hello, 0, sizeof ( hello ) );
@@ -1005,8 +967,8 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 	hello.version = htons ( tls->version );
 	memcpy ( &hello.random, &tls->client_random, sizeof ( hello.random ) );
 	hello.cipher_suite_len = htons ( sizeof ( hello.cipher_suites ) );
-	for ( i = 0 ; i < TLS_NUM_CIPHER_SUITES ; i++ )
-		hello.cipher_suites[i] = tls_cipher_suites[i].code;
+	i = 0 ; for_each_table_entry ( suite, TLS_CIPHER_SUITES )
+		hello.cipher_suites[i++] = suite->code;
 	hello.compression_methods_len = sizeof ( hello.compression_methods );
 	hello.extensions_len = htons ( sizeof ( hello.extensions ) );
 	hello.extensions.server_name_type = htons ( TLS_SERVER_NAME );
@@ -1025,6 +987,14 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 		= htons ( sizeof ( hello.extensions.max_fragment_length ) );
 	hello.extensions.max_fragment_length.max
 		= TLS_MAX_FRAGMENT_LENGTH_4096;
+	hello.extensions.signature_algorithms_type
+		= htons ( TLS_SIGNATURE_ALGORITHMS );
+	hello.extensions.signature_algorithms_len
+		= htons ( sizeof ( hello.extensions.signature_algorithms ) );
+	hello.extensions.signature_algorithms.len
+		= htons ( sizeof ( hello.extensions.signature_algorithms.code));
+	i = 0 ; for_each_table_entry ( sighash, TLS_SIG_HASH_ALGORITHMS )
+		hello.extensions.signature_algorithms.code[i++] = sighash->code;
 
 	return tls_send_handshake ( tls, &hello, sizeof ( hello ) );
 }
@@ -1038,9 +1008,9 @@ static int tls_send_client_hello ( struct tls_session *tls ) {
 static int tls_send_certificate ( struct tls_session *tls ) {
 	struct {
 		uint32_t type_length;
-		uint8_t length[3];
+		tls24_t length;
 		struct {
-			uint8_t length[3];
+			tls24_t length;
 			uint8_t data[ tls->cert->raw.len ];
 		} __attribute__ (( packed )) certificates[1];
 	} __attribute__ (( packed )) *certificate;
@@ -1058,9 +1028,9 @@ static int tls_send_certificate ( struct tls_session *tls ) {
 		( cpu_to_le32 ( TLS_CERTIFICATE ) |
 		  htonl ( sizeof ( *certificate ) -
 			  sizeof ( certificate->type_length ) ) );
-	tls_set_uint24 ( certificate->length,
+	tls_set_uint24 ( &certificate->length,
 			 sizeof ( certificate->certificates ) );
-	tls_set_uint24 ( certificate->certificates[0].length,
+	tls_set_uint24 ( &certificate->certificates[0].length,
 			 sizeof ( certificate->certificates[0].data ) );
 	memcpy ( certificate->certificates[0].data,
 		 tls->cert->raw.data,
@@ -1412,7 +1382,7 @@ static int tls_parse_chain ( struct tls_session *tls,
 			     const void *data, size_t len ) {
 	const void *end = ( data + len );
 	const struct {
-		uint8_t length[3];
+		tls24_t length;
 		uint8_t data[0];
 	} __attribute__ (( packed )) *certificate;
 	size_t certificate_len;
@@ -1436,7 +1406,7 @@ static int tls_parse_chain ( struct tls_session *tls,
 
 		/* Extract raw certificate data */
 		certificate = data;
-		certificate_len = tls_uint24 ( certificate->length );
+		certificate_len = tls_uint24 ( &certificate->length );
 		next = ( certificate->data + certificate_len );
 		if ( next > end ) {
 			DBGC ( tls, "TLS %p overlength certificate:\n", tls );
@@ -1482,10 +1452,10 @@ static int tls_parse_chain ( struct tls_session *tls,
 static int tls_new_certificate ( struct tls_session *tls,
 				 const void *data, size_t len ) {
 	const struct {
-		uint8_t length[3];
+		tls24_t length;
 		uint8_t certificates[0];
 	} __attribute__ (( packed )) *certificate = data;
-	size_t certificates_len = tls_uint24 ( certificate->length );
+	size_t certificates_len = tls_uint24 ( &certificate->length );
 	const void *end = ( certificate->certificates + certificates_len );
 	int rc;
 
@@ -1634,12 +1604,12 @@ static int tls_new_handshake ( struct tls_session *tls,
 	while ( data != end ) {
 		const struct {
 			uint8_t type;
-			uint8_t length[3];
+			tls24_t length;
 			uint8_t payload[0];
 		} __attribute__ (( packed )) *handshake = data;
-		void *payload = &handshake->payload;
-		size_t payload_len = tls_uint24 ( handshake->length );
-		void *next = ( payload + payload_len );
+		const void *payload = &handshake->payload;
+		size_t payload_len = tls_uint24 ( &handshake->length );
+		const void *next = ( payload + payload_len );
 
 		/* Sanity check */
 		if ( next > end ) {
@@ -2637,3 +2607,9 @@ int add_tls ( struct interface *xfer, const char *name,
  err_alloc:
 	return rc;
 }
+
+/* Drag in objects via add_tls() */
+REQUIRING_SYMBOL ( add_tls );
+
+/* Drag in crypto configuration */
+REQUIRE_OBJECT ( config_crypto );
diff --git a/qemu/roms/ipxe/src/net/udp.c b/qemu/roms/ipxe/src/net/udp.c
index 76da67ecf..0f7dfb24a 100644
--- a/qemu/roms/ipxe/src/net/udp.c
+++ b/qemu/roms/ipxe/src/net/udp.c
@@ -17,7 +17,7 @@
  * UDP protocol
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /**
  * A UDP connection
diff --git a/qemu/roms/ipxe/src/net/udp/dhcp.c b/qemu/roms/ipxe/src/net/udp/dhcp.c
index 04fad04c2..aed5ee360 100644
--- a/qemu/roms/ipxe/src/net/udp/dhcp.c
+++ b/qemu/roms/ipxe/src/net/udp/dhcp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdlib.h>
@@ -44,6 +48,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/dhcppkt.h>
 #include <ipxe/dhcp_arch.h>
 #include <ipxe/features.h>
+#include <config/dhcp.h>
 
 /** @file
  *
@@ -149,30 +154,32 @@ struct dhcp_session_state {
 	 * @v dhcppkt		DHCP packet
 	 * @v peer		Destination address
 	 */
-	int ( * tx ) ( struct dhcp_session *dhcp,
-		       struct dhcp_packet *dhcppkt,
+	int ( * tx ) ( struct dhcp_session *dhcp, struct dhcp_packet *dhcppkt,
 		       struct sockaddr_in *peer );
-	/** Handle received packet
+	/**
+	 * Handle received packet
 	 *
 	 * @v dhcp		DHCP session
 	 * @v dhcppkt		DHCP packet
 	 * @v peer		DHCP server address
 	 * @v msgtype		DHCP message type
 	 * @v server_id		DHCP server ID
+	 * @v pseudo_id		DHCP server pseudo-ID
 	 */
-	void ( * rx ) ( struct dhcp_session *dhcp,
-			struct dhcp_packet *dhcppkt,
-			struct sockaddr_in *peer,
-			uint8_t msgtype, struct in_addr server_id );
-	/** Handle timer expiry
+	void ( * rx ) ( struct dhcp_session *dhcp, struct dhcp_packet *dhcppkt,
+			struct sockaddr_in *peer, uint8_t msgtype,
+			struct in_addr server_id, struct in_addr pseudo_id );
+	/**
+	 * Handle timer expiry
 	 *
 	 * @v dhcp		DHCP session
 	 */
 	void ( * expired ) ( struct dhcp_session *dhcp );
 	/** Transmitted message type */
 	uint8_t tx_msgtype;
-	/** Apply minimum timeout */
-	uint8_t apply_min_timeout;
+	/** Timeout parameters */
+	uint8_t min_timeout_sec;
+	uint8_t max_timeout_sec;
 };
 
 static struct dhcp_session_state dhcp_state_discover;
@@ -272,9 +279,9 @@ static void dhcp_set_state ( struct dhcp_session *dhcp,
 	dhcp->state = state;
 	dhcp->start = currticks();
 	stop_timer ( &dhcp->timer );
-	dhcp->timer.min_timeout =
-		( state->apply_min_timeout ? DHCP_MIN_TIMEOUT : 0 );
-	dhcp->timer.max_timeout = DHCP_MAX_TIMEOUT;
+	set_timer_limits ( &dhcp->timer,
+			   ( state->min_timeout_sec * TICKS_PER_SEC ),
+			   ( state->max_timeout_sec * TICKS_PER_SEC ) );
 	start_timer_nodelay ( &dhcp->timer );
 }
 
@@ -334,11 +341,13 @@ static int dhcp_discovery_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 				struct dhcp_packet *dhcppkt,
 				struct sockaddr_in *peer, uint8_t msgtype,
-				struct in_addr server_id ) {
+				struct in_addr server_id,
+				struct in_addr pseudo_id ) {
 	struct in_addr ip;
 	char vci[9]; /* "PXEClient" */
 	int vci_len;
@@ -350,8 +359,11 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify offered IP address */
 	ip = dhcppkt->dhcphdr->yiaddr;
@@ -392,10 +404,10 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	}
 
 	/* Select as ProxyDHCP offer, if applicable */
-	if ( server_id.s_addr && has_pxeclient &&
+	if ( pseudo_id.s_addr && has_pxeclient &&
 	     ( priority >= dhcp->proxy_priority ) ) {
 		dhcppkt_put ( dhcp->proxy_offer );
-		dhcp->proxy_server = server_id;
+		dhcp->proxy_server = pseudo_id;
 		dhcp->proxy_offer = dhcppkt_get ( dhcppkt );
 		dhcp->proxy_priority = priority;
 	}
@@ -415,7 +427,7 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 	/* If we can't yet transition to DHCPREQUEST, do nothing */
 	elapsed = ( currticks() - dhcp->start );
 	if ( ! ( dhcp->no_pxedhcp || dhcp->proxy_offer ||
-		 ( elapsed > PROXYDHCP_MAX_TIMEOUT ) ) )
+		 ( elapsed > DHCP_DISC_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) ) )
 		return;
 
 	/* Transition to DHCPREQUEST */
@@ -430,8 +442,18 @@ static void dhcp_discovery_rx ( struct dhcp_session *dhcp,
 static void dhcp_discovery_expired ( struct dhcp_session *dhcp ) {
 	unsigned long elapsed = ( currticks() - dhcp->start );
 
+	/* If link is blocked, defer DHCP discovery (and reset timeout) */
+	if ( netdev_link_blocked ( dhcp->netdev ) ) {
+		DBGC ( dhcp, "DHCP %p deferring discovery\n", dhcp );
+		start_timer_fixed ( &dhcp->timer,
+				    ( DHCP_DISC_START_TIMEOUT_SEC *
+				      TICKS_PER_SEC ) );
+		return;
+	}
+
 	/* Give up waiting for ProxyDHCP before we reach the failure point */
-	if ( dhcp->offer.s_addr && ( elapsed > PROXYDHCP_MAX_TIMEOUT ) ) {
+	if ( dhcp->offer.s_addr &&
+	     ( elapsed > DHCP_DISC_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) ) {
 		dhcp_set_state ( dhcp, &dhcp_state_request );
 		return;
 	}
@@ -447,7 +469,8 @@ static struct dhcp_session_state dhcp_state_discover = {
 	.rx			= dhcp_discovery_rx,
 	.expired		= dhcp_discovery_expired,
 	.tx_msgtype		= DHCPDISCOVER,
-	.apply_min_timeout	= 1,
+	.min_timeout_sec	= DHCP_DISC_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_DISC_END_TIMEOUT_SEC,
 };
 
 /**
@@ -493,11 +516,13 @@ static int dhcp_request_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_request_rx ( struct dhcp_session *dhcp,
 			      struct dhcp_packet *dhcppkt,
 			      struct sockaddr_in *peer, uint8_t msgtype,
-			      struct in_addr server_id ) {
+			      struct in_addr server_id,
+			      struct in_addr pseudo_id ) {
 	struct in_addr ip;
 	struct settings *parent;
 	struct settings *settings;
@@ -506,8 +531,11 @@ static void dhcp_request_rx ( struct dhcp_session *dhcp,
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify leased IP address */
 	ip = dhcppkt->dhcphdr->yiaddr;
@@ -584,7 +612,8 @@ static struct dhcp_session_state dhcp_state_request = {
 	.rx			= dhcp_request_rx,
 	.expired		= dhcp_request_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 0,
+	.min_timeout_sec	= DHCP_REQ_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_REQ_END_TIMEOUT_SEC,
 };
 
 /**
@@ -623,19 +652,26 @@ static int dhcp_proxy_tx ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_proxy_rx ( struct dhcp_session *dhcp,
 			    struct dhcp_packet *dhcppkt,
 			    struct sockaddr_in *peer, uint8_t msgtype,
-			    struct in_addr server_id ) {
+			    struct in_addr server_id,
+			    struct in_addr pseudo_id ) {
 	struct settings *settings = &dhcppkt->settings;
 	int rc;
 
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
+	if ( dhcp_has_pxeopts ( dhcppkt ) )
+		DBGC ( dhcp, " pxe" );
 	DBGC ( dhcp, "\n" );
 
 	/* Filter out unacceptable responses */
@@ -643,8 +679,9 @@ static void dhcp_proxy_rx ( struct dhcp_session *dhcp,
 		return;
 	if ( ( msgtype != DHCPOFFER ) && ( msgtype != DHCPACK ) )
 		return;
-	if ( server_id.s_addr /* Linux PXE server omits server ID */ &&
-	     ( server_id.s_addr != dhcp->proxy_server.s_addr ) )
+	if ( ( pseudo_id.s_addr != dhcp->proxy_server.s_addr ) )
+		return;
+	if ( ! dhcp_has_pxeopts ( dhcppkt ) )
 		return;
 
 	/* Register settings */
@@ -669,7 +706,7 @@ static void dhcp_proxy_expired ( struct dhcp_session *dhcp ) {
 	unsigned long elapsed = ( currticks() - dhcp->start );
 
 	/* Give up waiting for ProxyDHCP before we reach the failure point */
-	if ( elapsed > PROXYDHCP_MAX_TIMEOUT ) {
+	if ( elapsed > DHCP_REQ_PROXY_TIMEOUT_SEC * TICKS_PER_SEC ) {
 		dhcp_finished ( dhcp, 0 );
 		return;
 	}
@@ -685,7 +722,8 @@ static struct dhcp_session_state dhcp_state_proxy = {
 	.rx			= dhcp_proxy_rx,
 	.expired		= dhcp_proxy_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 0,
+	.min_timeout_sec	= DHCP_PROXY_START_TIMEOUT_SEC,
+	.max_timeout_sec	= DHCP_PROXY_END_TIMEOUT_SEC,
 };
 
 /**
@@ -753,19 +791,24 @@ static int dhcp_pxebs_accept ( struct dhcp_session *dhcp,
  * @v peer		DHCP server address
  * @v msgtype		DHCP message type
  * @v server_id		DHCP server ID
+ * @v pseudo_id		DHCP server pseudo-ID
  */
 static void dhcp_pxebs_rx ( struct dhcp_session *dhcp,
 			    struct dhcp_packet *dhcppkt,
 			    struct sockaddr_in *peer, uint8_t msgtype,
-			    struct in_addr server_id ) {
+			    struct in_addr server_id,
+			    struct in_addr pseudo_id ) {
 	struct dhcp_pxe_boot_menu_item menu_item = { 0, 0 };
 	int rc;
 
 	DBGC ( dhcp, "DHCP %p %s from %s:%d", dhcp,
 	       dhcp_msgtype_name ( msgtype ), inet_ntoa ( peer->sin_addr ),
 	       ntohs ( peer->sin_port ) );
-	if ( server_id.s_addr != peer->sin_addr.s_addr )
-		DBGC ( dhcp, " (%s)", inet_ntoa ( server_id ) );
+	if ( ( server_id.s_addr != peer->sin_addr.s_addr ) ||
+	     ( pseudo_id.s_addr != peer->sin_addr.s_addr ) ) {
+		DBGC ( dhcp, " (%s/", inet_ntoa ( server_id ) );
+		DBGC ( dhcp, "%s)", inet_ntoa ( pseudo_id ) );
+	}
 
 	/* Identify boot menu item */
 	dhcppkt_fetch ( dhcppkt, DHCP_PXE_BOOT_MENU_ITEM,
@@ -782,8 +825,7 @@ static void dhcp_pxebs_rx ( struct dhcp_session *dhcp,
 		return;
 	if ( menu_item.type != dhcp->pxe_type )
 		return;
-	if ( ! dhcp_pxebs_accept ( dhcp, ( server_id.s_addr ?
-					   server_id : peer->sin_addr ) ) )
+	if ( ! dhcp_pxebs_accept ( dhcp, pseudo_id ) )
 		return;
 
 	/* Register settings */
@@ -810,7 +852,7 @@ static void dhcp_pxebs_expired ( struct dhcp_session *dhcp ) {
 	/* Give up waiting before we reach the failure point, and fail
 	 * over to the next server in the attempt list
 	 */
-	if ( elapsed > PXEBS_MAX_TIMEOUT ) {
+	if ( elapsed > PXEBS_MAX_TIMEOUT_SEC * TICKS_PER_SEC ) {
 		dhcp->pxe_attempt++;
 		if ( dhcp->pxe_attempt->s_addr ) {
 			dhcp_set_state ( dhcp, &dhcp_state_pxebs );
@@ -832,7 +874,8 @@ static struct dhcp_session_state dhcp_state_pxebs = {
 	.rx			= dhcp_pxebs_rx,
 	.expired		= dhcp_pxebs_expired,
 	.tx_msgtype		= DHCPREQUEST,
-	.apply_min_timeout	= 1,
+	.min_timeout_sec	= PXEBS_START_TIMEOUT_SEC,
+	.max_timeout_sec	= PXEBS_END_TIMEOUT_SEC,
 };
 
 /****************************************************************************
@@ -1114,6 +1157,7 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	struct dhcphdr *dhcphdr;
 	uint8_t msgtype = 0;
 	struct in_addr server_id = { 0 };
+	struct in_addr pseudo_id;
 	int rc = 0;
 
 	/* Sanity checks */
@@ -1148,6 +1192,13 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	dhcppkt_fetch ( dhcppkt, DHCP_SERVER_IDENTIFIER,
 			&server_id, sizeof ( server_id ) );
 
+	/* Identify server pseudo-ID */
+	pseudo_id = server_id;
+	if ( ! pseudo_id.s_addr )
+		pseudo_id = dhcppkt->dhcphdr->siaddr;
+	if ( ! pseudo_id.s_addr )
+		pseudo_id = peer->sin_addr;
+
 	/* Check for matching transaction ID */
 	if ( dhcphdr->xid != dhcp->xid ) {
 		DBGC ( dhcp, "DHCP %p %s from %s:%d has bad transaction "
@@ -1170,7 +1221,7 @@ static int dhcp_deliver ( struct dhcp_session *dhcp,
 	}
 
 	/* Handle packet based on current state */
-	dhcp->state->rx ( dhcp, dhcppkt, peer, msgtype, server_id );
+	dhcp->state->rx ( dhcp, dhcppkt, peer, msgtype, server_id, pseudo_id );
 
  err_chaddr:
  err_xid:
diff --git a/qemu/roms/ipxe/src/net/udp/dhcpv6.c b/qemu/roms/ipxe/src/net/udp/dhcpv6.c
index f7736d08e..a63543775 100644
--- a/qemu/roms/ipxe/src/net/udp/dhcpv6.c
+++ b/qemu/roms/ipxe/src/net/udp/dhcpv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/net/udp/dns.c b/qemu/roms/ipxe/src/net/udp/dns.c
index fffe6e697..2d77477f6 100644
--- a/qemu/roms/ipxe/src/net/udp/dns.c
+++ b/qemu/roms/ipxe/src/net/udp/dns.c
@@ -18,9 +18,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/udp/slam.c b/qemu/roms/ipxe/src/net/udp/slam.c
index 3cb492d73..8b26bfb3c 100644
--- a/qemu/roms/ipxe/src/net/udp/slam.c
+++ b/qemu/roms/ipxe/src/net/udp/slam.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/net/udp/syslog.c b/qemu/roms/ipxe/src/net/udp/syslog.c
index d65d19ab8..b6eee6036 100644
--- a/qemu/roms/ipxe/src/net/udp/syslog.c
+++ b/qemu/roms/ipxe/src/net/udp/syslog.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/net/udp/tftp.c b/qemu/roms/ipxe/src/net/udp/tftp.c
index ee827ae3d..953bcb46a 100644
--- a/qemu/roms/ipxe/src/net/udp/tftp.c
+++ b/qemu/roms/ipxe/src/net/udp/tftp.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
@@ -149,8 +153,6 @@ enum {
 	TFTP_FL_RRQ_MULTICAST = 0x0004,
 	/** Perform MTFTP recovery on timeout */
 	TFTP_FL_MTFTP_RECOVERY = 0x0008,
-	/** Only get filesize and then abort the transfer */
-	TFTP_FL_SIZEONLY = 0x0010,
 };
 
 /** Maximum number of MTFTP open requests before falling back to TFTP */
@@ -759,14 +761,6 @@ static int tftp_rx_oack ( struct tftp_request *tftp, void *buf, size_t len ) {
 			goto done;
 	}
 
-	/* Abort request if only trying to determine file size */
-	if ( tftp->flags & TFTP_FL_SIZEONLY ) {
-		rc = 0;
-		tftp_send_error ( tftp, 0, "TFTP Aborted" );
-		tftp_done ( tftp, rc );
-		return rc;
-	}
-
 	/* Request next data block */
 	tftp_send_packet ( tftp );
 
@@ -794,13 +788,6 @@ static int tftp_rx_data ( struct tftp_request *tftp,
 	size_t data_len;
 	int rc;
 
-	if ( tftp->flags & TFTP_FL_SIZEONLY ) {
-		/* If we get here then server doesn't support SIZE option */
-		rc = -ENOTSUP;
-		tftp_send_error ( tftp, 0, "TFTP Aborted" );
-		goto done;
-	}
-
 	/* Sanity check */
 	if ( iob_len ( iobuf ) < sizeof ( *data ) ) {
 		DBGC ( tftp, "TFTP %p received underlength DATA packet "
@@ -1036,10 +1023,25 @@ static size_t tftp_xfer_window ( struct tftp_request *tftp ) {
 	return tftp->blksize;
 }
 
+/**
+ * Terminate download
+ *
+ * @v tftp		TFTP connection
+ * @v rc		Reason for close
+ */
+static void tftp_close ( struct tftp_request *tftp, int rc ) {
+
+	/* Abort download */
+	tftp_send_error ( tftp, 0, "TFTP Aborted" );
+
+	/* Close TFTP request */
+	tftp_done ( tftp, rc );
+}
+
 /** TFTP data transfer interface operations */
 static struct interface_operation tftp_xfer_operations[] = {
 	INTF_OP ( xfer_window, struct tftp_request *, tftp_xfer_window ),
-	INTF_OP ( intf_close, struct tftp_request *, tftp_done ),
+	INTF_OP ( intf_close, struct tftp_request *, tftp_close ),
 };
 
 /** TFTP data transfer interface descriptor */
@@ -1126,26 +1128,6 @@ struct uri_opener tftp_uri_opener __uri_opener = {
 };
 
 /**
- * Initiate TFTP-size request
- *
- * @v xfer		Data transfer interface
- * @v uri		Uniform Resource Identifier
- * @ret rc		Return status code
- */
-static int tftpsize_open ( struct interface *xfer, struct uri *uri ) {
-	return tftp_core_open ( xfer, uri, TFTP_PORT, NULL,
-				( TFTP_FL_RRQ_SIZES |
-				  TFTP_FL_SIZEONLY ) );
-
-}
-
-/** TFTP URI opener */
-struct uri_opener tftpsize_uri_opener __uri_opener = {
-	.scheme	= "tftpsize",
-	.open	= tftpsize_open,
-};
-
-/**
  * Initiate TFTM download
  *
  * @v xfer		Data transfer interface
diff --git a/qemu/roms/ipxe/src/net/validator.c b/qemu/roms/ipxe/src/net/validator.c
index 74d70e312..db968398a 100644
--- a/qemu/roms/ipxe/src/net/validator.c
+++ b/qemu/roms/ipxe/src/net/validator.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -79,7 +83,7 @@ static void validator_free ( struct refcnt *refcnt ) {
 	DBGC2 ( validator, "VALIDATOR %p freed\n", validator );
 	x509_chain_put ( validator->chain );
 	ocsp_put ( validator->ocsp );
-	xferbuf_done ( &validator->buffer );
+	xferbuf_free ( &validator->buffer );
 	free ( validator );
 }
 
@@ -250,7 +254,8 @@ static int validator_start_download ( struct validator *validator,
 	/* Generate URI string */
 	len = snprintf ( uri_string, uri_string_len, "%s/%08x.der?subject=",
 			 crosscert, crc );
-	base64_encode ( issuer->data, issuer->len, ( uri_string + len ) );
+	base64_encode ( issuer->data, issuer->len, ( uri_string + len ),
+			( uri_string_len - len ) );
 	DBGC ( validator, "VALIDATOR %p downloading cross-signed certificate "
 	       "from %s\n", validator, uri_string );
 
@@ -387,7 +392,7 @@ static void validator_xfer_close ( struct validator *validator, int rc ) {
 		goto err_append;
 
 	/* Free downloaded data */
-	xferbuf_done ( &validator->buffer );
+	xferbuf_free ( &validator->buffer );
 
 	/* Resume validation process */
 	process_add ( &validator->process );
@@ -552,6 +557,7 @@ int create_validator ( struct interface *job, struct x509_chain *chain ) {
 	process_init ( &validator->process, &validator_process_desc,
 		       &validator->refcnt );
 	validator->chain = x509_chain_get ( chain );
+	xferbuf_malloc_init ( &validator->buffer );
 
 	/* Attach parent interface, mortalise self, and return */
 	intf_plug_plug ( &validator->job, job );
diff --git a/qemu/roms/ipxe/src/net/vlan.c b/qemu/roms/ipxe/src/net/vlan.c
index b4ddde42d..f515c2dc9 100644
--- a/qemu/roms/ipxe/src/net/vlan.c
+++ b/qemu/roms/ipxe/src/net/vlan.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <string.h>
@@ -385,6 +389,10 @@ int vlan_create ( struct net_device *trunk, unsigned int tag,
 	snprintf ( netdev->name, sizeof ( netdev->name ), "%s-%d",
 		   trunk->name, vlan->tag );
 
+	/* Mark device as not supporting interrupts, if applicable */
+	if ( ! netdev_irq_supported ( trunk ) )
+		netdev->state |= NETDEV_IRQ_UNSUPPORTED;
+
 	/* Register VLAN device */
 	if ( ( rc = register_netdev ( netdev ) ) != 0 ) {
 		DBGC ( netdev, "VLAN %s could not register: %s\n",
diff --git a/qemu/roms/ipxe/src/tests/aes_cbc_test.c b/qemu/roms/ipxe/src/tests/aes_cbc_test.c
deleted file mode 100644
index 4ae3a92e5..000000000
--- a/qemu/roms/ipxe/src/tests/aes_cbc_test.c
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301, USA.
- */
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-/** @file
- *
- * AES-in-CBC-mode tests
- *
- * These test vectors are provided by NIST as part of the
- * Cryptographic Toolkit Examples, downloadable from:
- *
- *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_CBC.pdf
- *
- */
-
-/* Forcibly enable assertions */
-#undef NDEBUG
-
-#include <assert.h>
-#include <string.h>
-#include <ipxe/aes.h>
-#include <ipxe/test.h>
-#include "cbc_test.h"
-
-/** Define inline key */
-#define KEY(...) { __VA_ARGS__ }
-
-/** Define inline initialisation vector */
-#define IV(...) { __VA_ARGS__ }
-
-/** Define inline plaintext data */
-#define PLAINTEXT(...) { __VA_ARGS__ }
-
-/** Define inline ciphertext data */
-#define CIPHERTEXT(...) { __VA_ARGS__ }
-
-/** An AES-in-CBC-mode test */
-struct aes_cbc_test {
-	/** Key */
-	const void *key;
-	/** Length of key */
-	size_t key_len;
-	/** Initialisation vector */
-	const void *iv;
-	/** Length of initialisation vector */
-	size_t iv_len;
-	/** Plaintext */
-	const void *plaintext;
-	/** Length of plaintext */
-	size_t plaintext_len;
-	/** Ciphertext */
-	const void *ciphertext;
-	/** Length of ciphertext */
-	size_t ciphertext_len;
-};
-
-/**
- * Define an AES-in-CBC-mode test
- *
- * @v name		Test name
- * @v key_array		Key
- * @v iv_array		Initialisation vector
- * @v plaintext_array	Plaintext
- * @v ciphertext_array	Ciphertext
- * @ret test		AES-in-CBC-mode test
- */
-#define AES_CBC_TEST( name, key_array, iv_array, plaintext_array,	\
-		      ciphertext_array )				\
-	static const uint8_t name ## _key [] = key_array;		\
-	static const uint8_t name ## _iv [] = iv_array;			\
-	static const uint8_t name ## _plaintext [] = plaintext_array;	\
-	static const uint8_t name ## _ciphertext [] = ciphertext_array;	\
-	static struct aes_cbc_test name = {				\
-		.key = name ## _key,					\
-		.key_len = sizeof ( name ## _key ),			\
-		.iv = name ## _iv,					\
-		.iv_len = sizeof ( name ## _iv ),			\
-		.plaintext = name ## _plaintext,			\
-		.plaintext_len = sizeof ( name ## _plaintext ),		\
-		.ciphertext = name ## _ciphertext,			\
-		.ciphertext_len = sizeof ( name ## _ciphertext ),	\
-	}
-
-/**
- * Report AES-in-CBC-mode
- *
- * @v state		HMAC_DRBG internal state
- * @v test		Instantiation test
- */
-#define aes_cbc_ok( test ) do {						\
-	struct cipher_algorithm *cipher = &aes_cbc_algorithm;		\
-									\
-	assert ( (test)->iv_len == cipher->blocksize );			\
-	assert ( (test)->plaintext_len == (test)->ciphertext_len );	\
-	cbc_encrypt_ok ( cipher, (test)->key, (test)->key_len,		\
-			 (test)->iv, (test)->plaintext,			\
-			 (test)->ciphertext, (test)->plaintext_len );	\
-	cbc_decrypt_ok ( cipher, (test)->key, (test)->key_len,		\
-			 (test)->iv, (test)->ciphertext,		\
-			 (test)->plaintext, (test)->ciphertext_len );	\
-	} while ( 0 )
-
-/** CBC_AES128 */
-AES_CBC_TEST ( test_128,
-	KEY ( 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15,
-	      0x88, 0x09, 0xcf, 0x4f, 0x3c ),
-	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
-	     0x0b, 0x0c, 0x0d, 0x0e, 0x0f ),
-	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
-		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
-		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
-		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
-		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
-		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
-		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,
-		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 ),
-	CIPHERTEXT ( 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46,
-		     0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d,
-		     0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee,
-		     0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2,
-		     0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b,
-		     0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16,
-		     0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09,
-		     0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 ) );
-
-/** CBC_AES256 */
-AES_CBC_TEST ( test_256,
-	KEY ( 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae,
-	      0xf0, 0x85, 0x7d, 0x77, 0x81, 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61,
-	      0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 ),
-	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a,
-	     0x0b, 0x0c, 0x0d, 0x0e, 0x0f ),
-	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,
-		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,
-		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,
-		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,
-		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,
-		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
-		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,
-		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 ),
-	CIPHERTEXT ( 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba,
-		     0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6,
-		     0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d,
-		     0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d,
-		     0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf,
-		     0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61,
-		     0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc,
-		     0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b ) );
-
-/**
- * Perform AES-in-CBC-mode self-test
- *
- */
-static void aes_cbc_test_exec ( void ) {
-	struct cipher_algorithm *cipher = &aes_cbc_algorithm;
-
-	/* Correctness tests */
-	aes_cbc_ok ( &test_128 );
-	aes_cbc_ok ( &test_256 );
-
-	/* Speed tests */
-	DBG ( "AES128 encryption required %ld cycles per byte\n",
-	      cbc_cost_encrypt ( cipher, test_128.key_len ) );
-	DBG ( "AES128 decryption required %ld cycles per byte\n",
-	      cbc_cost_decrypt ( cipher, test_128.key_len ) );
-	DBG ( "AES256 encryption required %ld cycles per byte\n",
-	      cbc_cost_encrypt ( cipher, test_256.key_len ) );
-	DBG ( "AES256 decryption required %ld cycles per byte\n",
-	      cbc_cost_decrypt ( cipher, test_256.key_len ) );
-}
-
-/** AES-in-CBC-mode self-test */
-struct self_test aes_cbc_test __self_test = {
-	.name = "aes_cbc",
-	.exec = aes_cbc_test_exec,
-};
diff --git a/qemu/roms/ipxe/src/tests/aes_test.c b/qemu/roms/ipxe/src/tests/aes_test.c
new file mode 100644
index 000000000..ad66c734c
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/aes_test.c
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * AES tests
+ *
+ * These test vectors are provided by NIST as part of the
+ * Cryptographic Toolkit Examples, downloadable from:
+ *
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_Core_All.pdf
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_ECB.pdf
+ *    http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/AES_CBC.pdf
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <assert.h>
+#include <string.h>
+#include <ipxe/aes.h>
+#include <ipxe/test.h>
+#include "cipher_test.h"
+
+/** Key used for NIST 128-bit test vectors */
+#define AES_KEY_NIST_128						\
+	KEY ( 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab,	\
+	      0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c )
+
+/** Key used for NIST 192-bit test vectors */
+#define AES_KEY_NIST_192						\
+	KEY ( 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8,	\
+	      0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, 0x62, 0xf8,	\
+	      0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b )
+
+/** Key used for NIST 256-bit test vectors */
+#define AES_KEY_NIST_256						\
+	KEY ( 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b,	\
+	      0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, 0x1f, 0x35,	\
+	      0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10,	\
+	      0xa3, 0x09, 0x14, 0xdf, 0xf4 )
+
+/** Dummy initialisation vector used for NIST ECB-mode test vectors */
+#define AES_IV_NIST_DUMMY						\
+	IV ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	\
+	     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 )
+
+/** Initialisation vector used for NIST CBC-mode test vectors */
+#define AES_IV_NIST_CBC							\
+	IV ( 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,	\
+	     0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f )
+
+/** Plaintext used for NIST test vectors */
+#define AES_PLAINTEXT_NIST						\
+	PLAINTEXT ( 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96,	\
+		    0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a,	\
+		    0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c,	\
+		    0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51,	\
+		    0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11,	\
+		    0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,	\
+		    0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17,	\
+		    0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 )
+
+/** AES-128-ECB (same test as AES-128-Core) */
+CIPHER_TEST ( aes_128_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_128, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60,
+		     0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97,
+		     0xf5, 0xd3, 0xd5, 0x85, 0x03, 0xb9, 0x69, 0x9d,
+		     0xe7, 0x85, 0x89, 0x5a, 0x96, 0xfd, 0xba, 0xaf,
+		     0x43, 0xb1, 0xcd, 0x7f, 0x59, 0x8e, 0xce, 0x23,
+		     0x88, 0x1b, 0x00, 0xe3, 0xed, 0x03, 0x06, 0x88,
+		     0x7b, 0x0c, 0x78, 0x5e, 0x27, 0xe8, 0xad, 0x3f,
+		     0x82, 0x23, 0x20, 0x71, 0x04, 0x72, 0x5d, 0xd4 ) );
+
+/** AES-128-CBC */
+CIPHER_TEST ( aes_128_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_128, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46,
+		     0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d,
+		     0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee,
+		     0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2,
+		     0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b,
+		     0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16,
+		     0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09,
+		     0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 ) );
+
+/** AES-192-ECB (same test as AES-192-Core) */
+CIPHER_TEST ( aes_192_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_192, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f,
+		     0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc,
+		     0x97, 0x41, 0x04, 0x84, 0x6d, 0x0a, 0xd3, 0xad,
+		     0x77, 0x34, 0xec, 0xb3, 0xec, 0xee, 0x4e, 0xef,
+		     0xef, 0x7a, 0xfd, 0x22, 0x70, 0xe2, 0xe6, 0x0a,
+		     0xdc, 0xe0, 0xba, 0x2f, 0xac, 0xe6, 0x44, 0x4e,
+		     0x9a, 0x4b, 0x41, 0xba, 0x73, 0x8d, 0x6c, 0x72,
+		     0xfb, 0x16, 0x69, 0x16, 0x03, 0xc1, 0x8e, 0x0e ) );
+
+/** AES-192-CBC */
+CIPHER_TEST ( aes_192_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_192, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d,
+		     0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8,
+		     0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4,
+		     0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a,
+		     0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0,
+		     0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0,
+		     0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81,
+		     0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd ) );
+
+/** AES-256-ECB (same test as AES-256-Core) */
+CIPHER_TEST ( aes_256_ecb, &aes_ecb_algorithm,
+	AES_KEY_NIST_256, AES_IV_NIST_DUMMY, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c,
+		     0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8,
+		     0x59, 0x1c, 0xcb, 0x10, 0xd4, 0x10, 0xed, 0x26,
+		     0xdc, 0x5b, 0xa7, 0x4a, 0x31, 0x36, 0x28, 0x70,
+		     0xb6, 0xed, 0x21, 0xb9, 0x9c, 0xa6, 0xf4, 0xf9,
+		     0xf1, 0x53, 0xe7, 0xb1, 0xbe, 0xaf, 0xed, 0x1d,
+		     0x23, 0x30, 0x4b, 0x7a, 0x39, 0xf9, 0xf3, 0xff,
+		     0x06, 0x7d, 0x8d, 0x8f, 0x9e, 0x24, 0xec, 0xc7 ) );
+
+/** AES-256-CBC */
+CIPHER_TEST ( aes_256_cbc, &aes_cbc_algorithm,
+	AES_KEY_NIST_256, AES_IV_NIST_CBC, AES_PLAINTEXT_NIST,
+	CIPHERTEXT ( 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba,
+		     0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6,
+		     0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d,
+		     0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d,
+		     0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf,
+		     0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61,
+		     0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc,
+		     0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b ) );
+
+/**
+ * Perform AES self-test
+ *
+ */
+static void aes_test_exec ( void ) {
+	struct cipher_algorithm *ecb = &aes_ecb_algorithm;
+	struct cipher_algorithm *cbc = &aes_cbc_algorithm;
+	unsigned int keylen;
+
+	/* Correctness tests */
+	cipher_ok ( &aes_128_ecb );
+	cipher_ok ( &aes_128_cbc );
+	cipher_ok ( &aes_192_ecb );
+	cipher_ok ( &aes_192_cbc );
+	cipher_ok ( &aes_256_ecb );
+	cipher_ok ( &aes_256_cbc );
+
+	/* Speed tests */
+	for ( keylen = 128 ; keylen <= 256 ; keylen += 64 ) {
+		DBG ( "AES-%d-ECB encryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_encrypt ( ecb, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-ECB decryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_decrypt ( ecb, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-CBC encryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_encrypt ( cbc, ( keylen / 8 ) ) );
+		DBG ( "AES-%d-CBC decryption required %ld cycles per byte\n",
+		      keylen, cipher_cost_decrypt ( cbc, ( keylen / 8 ) ) );
+	}
+}
+
+/** AES self-test */
+struct self_test aes_test __self_test = {
+	.name = "aes",
+	.exec = aes_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/base16_test.c b/qemu/roms/ipxe/src/tests/base16_test.c
index 9b047b74c..46884aef7 100644
--- a/qemu/roms/ipxe/src/tests/base16_test.c
+++ b/qemu/roms/ipxe/src/tests/base16_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -73,30 +77,42 @@ BASE16 ( random_test,
  * Report a base16 encoding test result
  *
  * @v test		Base16 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base16_encode_ok( test ) do {					\
-	size_t len = base16_encoded_len ( (test)->len );		\
-	char buf[ len + 1 /* NUL */ ];					\
-	ok ( len == strlen ( (test)->encoded ) );			\
-	base16_encode ( (test)->data, (test)->len, buf );		\
-	ok ( strcmp ( (test)->encoded, buf ) == 0 );			\
-	} while ( 0 )
+static void base16_encode_okx ( struct base16_test *test, const char *file,
+				unsigned int line ) {
+	size_t len = base16_encoded_len ( test->len );
+	char buf[ len + 1 /* NUL */ ];
+	size_t check_len;
+
+	okx ( len == strlen ( test->encoded ), file, line );
+	check_len = base16_encode ( test->data, test->len, buf, sizeof ( buf ));
+	okx ( check_len == len, file, line );
+	okx ( strcmp ( test->encoded, buf ) == 0, file, line );
+}
+#define base16_encode_ok( test ) base16_encode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Report a base16 decoding test result
  *
  * @v test		Base16 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base16_decode_ok( test ) do {					\
-	size_t max_len = base16_decoded_max_len ( (test)->encoded );	\
-	uint8_t buf[max_len];						\
-	int len;							\
-	len = base16_decode ( (test)->encoded, buf );			\
-	ok ( len >= 0 );						\
-	ok ( ( size_t ) len <= max_len );				\
-	ok ( ( size_t ) len == (test)->len );				\
-	ok ( memcmp ( (test)->data, buf, len ) == 0 );			\
-	} while ( 0 )
+static void base16_decode_okx ( struct base16_test *test, const char *file,
+				unsigned int line ) {
+	size_t max_len = base16_decoded_max_len ( test->encoded );
+	uint8_t buf[max_len];
+	int len;
+
+	len = base16_decode ( test->encoded, buf, sizeof ( buf ) );
+	okx ( len >= 0, file, line );
+	okx ( ( size_t ) len <= max_len, file, line );
+	okx ( ( size_t ) len == test->len, file, line );
+	okx ( memcmp ( test->data, buf, len ) == 0, file, line );
+}
+#define base16_decode_ok( test ) base16_decode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Perform Base16 self-tests
diff --git a/qemu/roms/ipxe/src/tests/base64_test.c b/qemu/roms/ipxe/src/tests/base64_test.c
index c088298ca..0fc595d90 100644
--- a/qemu/roms/ipxe/src/tests/base64_test.c
+++ b/qemu/roms/ipxe/src/tests/base64_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -76,30 +80,42 @@ BASE64 ( random_test,
  * Report a base64 encoding test result
  *
  * @v test		Base64 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base64_encode_ok( test ) do {					\
-	size_t len = base64_encoded_len ( (test)->len );		\
-	char buf[ len + 1 /* NUL */ ];					\
-	ok ( len == strlen ( (test)->encoded ) );			\
-	base64_encode ( (test)->data, (test)->len, buf );		\
-	ok ( strcmp ( (test)->encoded, buf ) == 0 );			\
-	} while ( 0 )
+static void base64_encode_okx ( struct base64_test *test, const char *file,
+				unsigned int line ) {
+	size_t len = base64_encoded_len ( test->len );
+	char buf[ len + 1 /* NUL */ ];
+	size_t check_len;
+
+	okx ( len == strlen ( test->encoded ), file, line );
+	check_len = base64_encode ( test->data, test->len, buf, sizeof ( buf ));
+	okx ( check_len == len, file, line );
+	okx ( strcmp ( test->encoded, buf ) == 0, file, line );
+}
+#define base64_encode_ok( test ) base64_encode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Report a base64 decoding test result
  *
  * @v test		Base64 test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-#define base64_decode_ok( test ) do {					\
-	size_t max_len = base64_decoded_max_len ( (test)->encoded );	\
-	uint8_t buf[max_len];						\
-	int len;							\
-	len = base64_decode ( (test)->encoded, buf );			\
-	ok ( len >= 0 );						\
-	ok ( ( size_t ) len <= max_len );				\
-	ok ( ( size_t ) len == (test)->len );				\
-	ok ( memcmp ( (test)->data, buf, len ) == 0 );			\
-	} while ( 0 )
+static void base64_decode_okx ( struct base64_test *test, const char *file,
+				unsigned int line ) {
+	size_t max_len = base64_decoded_max_len ( test->encoded );
+	uint8_t buf[max_len];
+	int len;
+
+	len = base64_decode ( test->encoded, buf, sizeof ( buf ) );
+	okx ( len >= 0, file, line );
+	okx ( ( size_t ) len <= max_len, file, line );
+	okx ( ( size_t ) len == test->len, file, line );
+	okx ( memcmp ( test->data, buf, len ) == 0, file, line );
+}
+#define base64_decode_ok( test ) base64_decode_okx ( test, __FILE__, __LINE__ )
 
 /**
  * Perform Base64 self-tests
diff --git a/qemu/roms/ipxe/src/tests/bigint_test.c b/qemu/roms/ipxe/src/tests/bigint_test.c
index 75a80622f..8d40c3188 100644
--- a/qemu/roms/ipxe/src/tests/bigint_test.c
+++ b/qemu/roms/ipxe/src/tests/bigint_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/bofm_test.c b/qemu/roms/ipxe/src/tests/bofm_test.c
index e430d12d4..829924887 100644
--- a/qemu/roms/ipxe/src/tests/bofm_test.c
+++ b/qemu/roms/ipxe/src/tests/bofm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/tests/byteswap_test.c b/qemu/roms/ipxe/src/tests/byteswap_test.c
index a500218be..92bdb1d59 100644
--- a/qemu/roms/ipxe/src/tests/byteswap_test.c
+++ b/qemu/roms/ipxe/src/tests/byteswap_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/cbc_test.h b/qemu/roms/ipxe/src/tests/cbc_test.h
deleted file mode 100644
index ad9e6f341..000000000
--- a/qemu/roms/ipxe/src/tests/cbc_test.h
+++ /dev/null
@@ -1,57 +0,0 @@
-#ifndef _CBC_TEST_H
-#define _CBC_TEST_H
-
-FILE_LICENCE ( GPL2_OR_LATER );
-
-#include <stdint.h>
-#include <ipxe/crypto.h>
-#include <ipxe/test.h>
-
-extern int cbc_test_encrypt ( struct cipher_algorithm *cipher, const void *key,
-			      size_t key_len, const void *iv,
-			      const void *plaintext,
-			      const void *expected_ciphertext, size_t len );
-extern int cbc_test_decrypt ( struct cipher_algorithm *cipher, const void *key,
-			      size_t key_len, const void *iv,
-			      const void *ciphertext,
-			      const void *expected_plaintext, size_t len );
-extern unsigned long cbc_cost_encrypt ( struct cipher_algorithm *cipher,
-					size_t key_len );
-extern unsigned long cbc_cost_decrypt ( struct cipher_algorithm *cipher,
-					size_t key_len );
-
-/**
- * Report CBC encryption test result
- *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v plaintext			Plaintext data
- * @v expected_ciphertext	Expected ciphertext data
- * @v len			Length of data
- */
-#define cbc_encrypt_ok( cipher, key, key_len, iv, plaintext,		\
-			expected_ciphertext, len ) do {			\
-	ok ( cbc_test_encrypt ( cipher, key, key_len, iv, plaintext,	\
-				expected_ciphertext, len ) );		\
-	} while ( 0 )
-
-/**
- * Report CBC decryption test result
- *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v ciphertext		Ciphertext data
- * @v expected_plaintext	Expected plaintext data
- * @v len			Length of data
- */
-#define cbc_decrypt_ok( cipher, key, key_len, iv, ciphertext,		\
-			expected_plaintext, len ) do {			\
-	ok ( cbc_test_decrypt ( cipher, key, key_len, iv, ciphertext,	\
-				expected_plaintext, len ) );		\
-	} while ( 0 )
-
-#endif /* _CBC_TEST_H */
diff --git a/qemu/roms/ipxe/src/tests/cbc_test.c b/qemu/roms/ipxe/src/tests/cipher_test.c
index cb0f7bdea..800d6c138 100644
--- a/qemu/roms/ipxe/src/tests/cbc_test.c
+++ b/qemu/roms/ipxe/src/tests/cipher_test.c
@@ -15,13 +15,17 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * CBC self-tests
+ * Cipher self-tests
  *
  */
 
@@ -34,86 +38,90 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <assert.h>
 #include <ipxe/crypto.h>
 #include <ipxe/profile.h>
-#include "cbc_test.h"
+#include <ipxe/test.h>
+#include "cipher_test.h"
 
 /** Number of sample iterations for profiling */
 #define PROFILE_COUNT 16
 
 /**
- * Test CBC encryption
+ * Report a cipher encryption test result
  *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v plaintext			Plaintext data
- * @v expected_ciphertext	Expected ciphertext data
- * @v len			Length of data
- * @ret ok			Ciphertext is as expected
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int cbc_test_encrypt ( struct cipher_algorithm *cipher, const void *key,
-		       size_t key_len, const void *iv, const void *plaintext,
-		       const void *expected_ciphertext, size_t len ) {
+void cipher_encrypt_okx ( struct cipher_test *test, const char *file,
+			  unsigned int line ) {
+	struct cipher_algorithm *cipher = test->cipher;
+	size_t len = test->len;
 	uint8_t ctx[cipher->ctxsize];
 	uint8_t ciphertext[len];
-	int rc;
 
 	/* Initialise cipher */
-	rc = cipher_setkey ( cipher, ctx, key, key_len );
-	assert ( rc == 0 );
-	cipher_setiv ( cipher, ctx, iv );
+	okx ( cipher_setkey ( cipher, ctx, test->key, test->key_len ) == 0,
+	      file, line );
+	cipher_setiv ( cipher, ctx, test->iv );
 
 	/* Perform encryption */
-	cipher_encrypt ( cipher, ctx, plaintext, ciphertext, len );
+	cipher_encrypt ( cipher, ctx, test->plaintext, ciphertext, len );
 
-	/* Verify result */
-	return ( memcmp ( ciphertext, expected_ciphertext, len ) == 0 );
+	/* Compare against expected ciphertext */
+	okx ( memcmp ( ciphertext, test->ciphertext, len ) == 0, file, line );
 }
 
 /**
- * Test CBC decryption
+ * Report a cipher decryption test result
  *
- * @v cipher			Cipher algorithm
- * @v key			Key
- * @v key_len			Length of key
- * @v iv			Initialisation vector
- * @v ciphertext		Ciphertext data
- * @v expected_plaintext	Expected plaintext data
- * @v len			Length of data
- * @ret ok			Plaintext is as expected
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int cbc_test_decrypt ( struct cipher_algorithm *cipher, const void *key,
-		       size_t key_len, const void *iv, const void *ciphertext,
-		       const void *expected_plaintext, size_t len ) {
+void cipher_decrypt_okx ( struct cipher_test *test, const char *file,
+			  unsigned int line ) {
+	struct cipher_algorithm *cipher = test->cipher;
+	size_t len = test->len;
 	uint8_t ctx[cipher->ctxsize];
 	uint8_t plaintext[len];
-	int rc;
 
 	/* Initialise cipher */
-	rc = cipher_setkey ( cipher, ctx, key, key_len );
-	assert ( rc == 0 );
-	cipher_setiv ( cipher, ctx, iv );
+	okx ( cipher_setkey ( cipher, ctx, test->key, test->key_len ) == 0,
+	      file, line );
+	cipher_setiv ( cipher, ctx, test->iv );
 
 	/* Perform encryption */
-	cipher_decrypt ( cipher, ctx, ciphertext, plaintext, len );
+	cipher_decrypt ( cipher, ctx, test->ciphertext, plaintext, len );
+
+	/* Compare against expected plaintext */
+	okx ( memcmp ( plaintext, test->plaintext, len ) == 0, file, line );
+}
+
+/**
+ * Report a cipher encryption and decryption test result
+ *
+ * @v test		Cipher test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void cipher_okx ( struct cipher_test *test, const char *file,
+		  unsigned int line ) {
 
-	/* Verify result */
-	return ( memcmp ( plaintext, expected_plaintext, len ) == 0 );
+	cipher_encrypt_okx ( test, file, line );
+	cipher_decrypt_okx ( test, file, line );
 }
 
 /**
- * Calculate CBC encryption or decryption cost
+ * Calculate cipher encryption or decryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @v op			Encryption or decryption operation
  * @ret cost			Cost (in cycles per byte)
  */
-static unsigned long cbc_cost ( struct cipher_algorithm *cipher,
-				size_t key_len,
-				void ( * op ) ( struct cipher_algorithm *cipher,
-						void *ctx, const void *src,
-						void *dst, size_t len ) ) {
+static unsigned long
+cipher_cost ( struct cipher_algorithm *cipher, size_t key_len,
+	      void ( * op ) ( struct cipher_algorithm *cipher, void *ctx,
+			      const void *src, void *dst, size_t len ) ) {
 	static uint8_t random[8192]; /* Too large for stack */
 	uint8_t key[key_len];
 	uint8_t iv[cipher->blocksize];
@@ -153,25 +161,25 @@ static unsigned long cbc_cost ( struct cipher_algorithm *cipher,
 }
 
 /**
- * Calculate CBC encryption cost
+ * Calculate cipher encryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @ret cost			Cost (in cycles per byte)
  */
-unsigned long cbc_cost_encrypt ( struct cipher_algorithm *cipher,
-				 size_t key_len ) {
-	return cbc_cost ( cipher, key_len, cipher_encrypt );
+unsigned long cipher_cost_encrypt ( struct cipher_algorithm *cipher,
+				    size_t key_len ) {
+	return cipher_cost ( cipher, key_len, cipher_encrypt );
 }
 
 /**
- * Calculate CBC decryption cost
+ * Calculate cipher decryption cost
  *
  * @v cipher			Cipher algorithm
  * @v key_len			Length of key
  * @ret cost			Cost (in cycles per byte)
  */
-unsigned long cbc_cost_decrypt ( struct cipher_algorithm *cipher,
-				 size_t key_len ) {
-	return cbc_cost ( cipher, key_len, cipher_decrypt );
+unsigned long cipher_cost_decrypt ( struct cipher_algorithm *cipher,
+				    size_t key_len ) {
+	return cipher_cost ( cipher, key_len, cipher_decrypt );
 }
diff --git a/qemu/roms/ipxe/src/tests/cipher_test.h b/qemu/roms/ipxe/src/tests/cipher_test.h
new file mode 100644
index 000000000..d7c5aef8f
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/cipher_test.h
@@ -0,0 +1,111 @@
+#ifndef _CIPHER_TEST_H
+#define _CIPHER_TEST_H
+
+/** @file
+ *
+ * Cipher self-tests
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <ipxe/crypto.h>
+#include <ipxe/test.h>
+
+/** A cipher test */
+struct cipher_test {
+	/** Cipher algorithm */
+	struct cipher_algorithm *cipher;
+	/** Key */
+	const void *key;
+	/** Length of key */
+	size_t key_len;
+	/** Initialisation vector */
+	const void *iv;
+	/** Length of initialisation vector */
+	size_t iv_len;
+	/** Plaintext */
+	const void *plaintext;
+	/** Ciphertext */
+	const void *ciphertext;
+	/** Length of text */
+	size_t len;
+};
+
+/** Define inline key */
+#define KEY(...) { __VA_ARGS__ }
+
+/** Define inline initialisation vector */
+#define IV(...) { __VA_ARGS__ }
+
+/** Define inline plaintext data */
+#define PLAINTEXT(...) { __VA_ARGS__ }
+
+/** Define inline ciphertext data */
+#define CIPHERTEXT(...) { __VA_ARGS__ }
+
+/**
+ * Define a cipher test
+ *
+ * @v name		Test name
+ * @v CIPHER		Cipher algorithm
+ * @v KEY		Key
+ * @v IV		Initialisation vector
+ * @v PLAINTEXT		Plaintext
+ * @v CIPHERTEXT	Ciphertext
+ * @ret test		Cipher test
+ */
+#define CIPHER_TEST( name, CIPHER, KEY, IV, PLAINTEXT, CIPHERTEXT )	\
+	static const uint8_t name ## _key [] = KEY;			\
+	static const uint8_t name ## _iv [] = IV;			\
+	static const uint8_t name ## _plaintext [] = PLAINTEXT;		\
+	static const uint8_t name ## _ciphertext			\
+		[ sizeof ( name ## _plaintext ) ] = CIPHERTEXT;		\
+	static struct cipher_test name = {				\
+		.cipher = CIPHER,					\
+		.key = name ## _key,					\
+		.key_len = sizeof ( name ## _key ),			\
+		.iv = name ## _iv,					\
+		.iv_len = sizeof ( name ## _iv ),			\
+		.plaintext = name ## _plaintext,			\
+		.ciphertext = name ## _ciphertext,			\
+		.len = sizeof ( name ## _plaintext ),			\
+	}
+
+extern void cipher_encrypt_okx ( struct cipher_test *test, const char *file,
+				 unsigned int line );
+extern void cipher_decrypt_okx ( struct cipher_test *test, const char *file,
+				 unsigned int line );
+extern void cipher_okx ( struct cipher_test *test, const char *file,
+			 unsigned int line );
+extern unsigned long cipher_cost_encrypt ( struct cipher_algorithm *cipher,
+					   size_t key_len );
+extern unsigned long cipher_cost_decrypt ( struct cipher_algorithm *cipher,
+					   size_t key_len );
+
+/**
+ * Report a cipher encryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_encrypt_ok( test ) \
+	cipher_encrypt_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report a cipher decryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_decrypt_ok( test ) \
+	cipher_decrypt_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Report a cipher encryption and decryption test result
+ *
+ * @v test		Cipher test
+ */
+#define cipher_ok( test ) \
+	cipher_okx ( test, __FILE__, __LINE__ )
+
+#endif /* _CIPHER_TEST_H */
diff --git a/qemu/roms/ipxe/src/tests/cms_test.c b/qemu/roms/ipxe/src/tests/cms_test.c
index 8767504c0..b805a9974 100644
--- a/qemu/roms/ipxe/src/tests/cms_test.c
+++ b/qemu/roms/ipxe/src/tests/cms_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1470,6 +1474,7 @@ struct self_test cms_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( cms_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( md5 );
 REQUIRE_OBJECT ( sha1 );
diff --git a/qemu/roms/ipxe/src/tests/crc32_test.c b/qemu/roms/ipxe/src/tests/crc32_test.c
index 873f633a5..46cde0f7b 100644
--- a/qemu/roms/ipxe/src/tests/crc32_test.c
+++ b/qemu/roms/ipxe/src/tests/crc32_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/deflate_test.c b/qemu/roms/ipxe/src/tests/deflate_test.c
index 68c1aad96..20ff5b9a2 100644
--- a/qemu/roms/ipxe/src/tests/deflate_test.c
+++ b/qemu/roms/ipxe/src/tests/deflate_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/digest_test.c b/qemu/roms/ipxe/src/tests/digest_test.c
index 4df26c099..c3a128853 100644
--- a/qemu/roms/ipxe/src/tests/digest_test.c
+++ b/qemu/roms/ipxe/src/tests/digest_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -34,27 +38,47 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/profile.h>
 #include "digest_test.h"
 
+/** Maximum number of digest test fragments */
+#define NUM_DIGEST_TEST_FRAG 8
+
+/** A digest test fragment list */
+struct digest_test_fragments {
+	/** Fragment lengths */
+	size_t len[NUM_DIGEST_TEST_FRAG];
+};
+
+/** Digest test fragment lists */
+static struct digest_test_fragments digest_test_fragments[] = {
+	{ { 0, -1UL, } },
+	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
+	{ { 2, 0, 23, 4, 6, 1, 0 } },
+};
+
 /** Number of sample iterations for profiling */
 #define PROFILE_COUNT 16
 
 /**
- * Test digest algorithm
+ * Report a digest fragmented test result
  *
- * @v digest		Digest algorithm
- * @v fragments		Digest test fragment list, or NULL
- * @v data		Test data
- * @v len		Length of test data
- * @v expected		Expected digest value
- * @ret ok		Digest value is as expected
+ * @v test		Digest test
+ * @v fragments		Fragment list
+ * @v file		Test code file
+ * @v line		Test code line
  */
-int digest_test ( struct digest_algorithm *digest,
-		  struct digest_test_fragments *fragments,
-		  void *data, size_t len, void *expected ) {
+void digest_frag_okx ( struct digest_test *test,
+		       struct digest_test_fragments *fragments,
+		       const char *file, unsigned int line ) {
+	struct digest_algorithm *digest = test->digest;
 	uint8_t ctx[digest->ctxsize];
 	uint8_t out[digest->digestsize];
+	const void *data = test->data;
+	size_t len = test->len;
 	size_t frag_len = 0;
 	unsigned int i;
 
+	/* Sanity check */
+	okx ( test->expected_len == sizeof ( out ), file, line );
+
 	/* Initialise digest */
 	digest_init ( digest, ctx );
 
@@ -74,7 +98,28 @@ int digest_test ( struct digest_algorithm *digest,
 	digest_final ( digest, ctx, out );
 
 	/* Compare against expected output */
-	return ( memcmp ( expected, out, sizeof ( out ) ) == 0 );
+	okx ( memcmp ( test->expected, out, sizeof ( out ) ) == 0, file, line );
+}
+
+/**
+ * Report a digest test result
+ *
+ * @v test		Digest test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+void digest_okx ( struct digest_test *test, const char *file,
+		  unsigned int line ) {
+	unsigned int i;
+
+	/* Test with a single pass */
+	digest_frag_okx ( test, NULL, file, line );
+
+	/* Test with fragment lists */
+	for ( i = 0 ; i < ( sizeof ( digest_test_fragments ) /
+			    sizeof ( digest_test_fragments[0] ) ) ; i++ ) {
+		digest_frag_okx ( test, &digest_test_fragments[i], file, line );
+	}
 }
 
 /**
diff --git a/qemu/roms/ipxe/src/tests/digest_test.h b/qemu/roms/ipxe/src/tests/digest_test.h
index 49e06d1cb..abf1b834f 100644
--- a/qemu/roms/ipxe/src/tests/digest_test.h
+++ b/qemu/roms/ipxe/src/tests/digest_test.h
@@ -1,37 +1,115 @@
 #ifndef _DIGEST_TEST_H
 #define _DIGEST_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
 #include <ipxe/test.h>
 
-/** Maximum number of digest test fragments */
-#define NUM_DIGEST_TEST_FRAG 8
-
-/** A digest test fragment list */
-struct digest_test_fragments {
-	/** Fragment lengths */
-	size_t len[NUM_DIGEST_TEST_FRAG];
+/** A digest test */
+struct digest_test {
+	/** Digest algorithm */
+	struct digest_algorithm *digest;
+	/** Test data */
+	const void *data;
+	/** Length of test data */
+	size_t len;
+	/** Expected digest value */
+	const void *expected;
+	/** Expected digest length */
+	size_t expected_len;
 };
 
-extern int digest_test ( struct digest_algorithm *digest,
-			 struct digest_test_fragments *fragments,
-			 void *data, size_t len, void *expected );
-extern unsigned long digest_cost ( struct digest_algorithm *digest );
+/** Define inline test data */
+#define DATA(...) { __VA_ARGS__ }
+
+/** Define inline expected digest value */
+#define DIGEST(...) { __VA_ARGS__ }
+
+/**
+ * Define a digest test
+ *
+ * @v name		Test name
+ * @v DIGEST		Digest algorithm
+ * @v DATA		Test data
+ * @v EXPECTED		Expected digest value
+ * @ret test		Digest test
+ */
+#define DIGEST_TEST( name, DIGEST, DATA, EXPECTED )			\
+	static const uint8_t name ## _data[] = DATA;			\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct digest_test name = {				\
+		.digest = DIGEST,					\
+		.data = name ## _data,					\
+		.len = sizeof ( name ## _data ),			\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	};
+
+/** Standard test vector: empty data */
+#define DIGEST_EMPTY DATA()
+
+/** Standard test vector: NIST string "abc"
+ *
+ * The NIST Cryptographic Toolkit examples for all digest algorithms
+ * include a test vector which is the unterminated string
+ *
+ *   "abc"
+ */
+#define DIGEST_NIST_ABC							\
+	DATA ( 0x61, 0x62, 0x63 )
+
+/** Standard test vector: NIST string "abc...opq"
+ *
+ * The NIST Cryptographic Toolkit examples for all 32-bit digest
+ * algorithms (SHA-1 and the SHA-256 family) include a test vector
+ * which is the unterminated string
+ *
+ *   "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"
+ */
+#define DIGEST_NIST_ABC_OPQ						\
+	DATA ( 0x61, 0x62, 0x63, 0x64, 0x62, 0x63, 0x64, 0x65, 0x63,	\
+	       0x64, 0x65, 0x66, 0x64, 0x65, 0x66, 0x67, 0x65, 0x66,	\
+	       0x67, 0x68, 0x66, 0x67, 0x68, 0x69, 0x67, 0x68, 0x69,	\
+	       0x6a, 0x68, 0x69, 0x6a, 0x6b, 0x69, 0x6a, 0x6b, 0x6c,	\
+	       0x6a, 0x6b, 0x6c, 0x6d, 0x6b, 0x6c, 0x6d, 0x6e, 0x6c,	\
+	       0x6d, 0x6e, 0x6f, 0x6d, 0x6e, 0x6f, 0x70, 0x6e, 0x6f,	\
+	       0x70, 0x71 )
+
+/** Standard test vector: NIST string "abc...stu"
+ *
+ * The NIST Cryptographic Toolkit examples for all 64-bit digest
+ * algorithms (SHA-512 family) include a test vector which is the
+ * unterminated string
+ *
+ *   "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmn"
+ *   "hijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"
+ */
+#define DIGEST_NIST_ABC_STU						\
+	DATA ( 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x62,	\
+	       0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x63, 0x64,	\
+	       0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x64, 0x65, 0x66,	\
+	       0x67, 0x68, 0x69, 0x6a, 0x6b, 0x65, 0x66, 0x67, 0x68,	\
+	       0x69, 0x6a, 0x6b, 0x6c, 0x66, 0x67, 0x68, 0x69, 0x6a,	\
+	       0x6b, 0x6c, 0x6d, 0x67, 0x68, 0x69, 0x6a, 0x6b, 0x6c,	\
+	       0x6d, 0x6e, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e,	\
+	       0x6f, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70,	\
+	       0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x6b,	\
+	       0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x6c, 0x6d,	\
+	       0x6e, 0x6f, 0x70, 0x71, 0x72, 0x73, 0x6d, 0x6e, 0x6f,	\
+	       0x70, 0x71, 0x72, 0x73, 0x74, 0x6e, 0x6f, 0x70, 0x71,	\
+	       0x72, 0x73, 0x74, 0x75 )
 
 /**
- * Report digest test result
+ * Report a digest test result
  *
- * @v digest		Digest algorithm
- * @v fragments		Digest test fragment list, or NULL
- * @v data		Test data
- * @v len		Length of test data
- * @v expected		Expected digest value
+ * @v test		Digest test
  */
-#define digest_ok( digest, fragments, data, len, expected ) do {	\
-	ok ( digest_test ( digest, fragments, data, len, expected ) );	\
-	} while ( 0 )
+#define digest_ok(test) digest_okx ( test, __FILE__, __LINE__ )
+
+extern void digest_okx ( struct digest_test *test, const char *file,
+			 unsigned int line );
+extern unsigned long digest_cost ( struct digest_algorithm *digest );
 
 #endif /* _DIGEST_TEST_H */
diff --git a/qemu/roms/ipxe/src/tests/dns_test.c b/qemu/roms/ipxe/src/tests/dns_test.c
index 52f5f19f2..f08e7810f 100644
--- a/qemu/roms/ipxe/src/tests/dns_test.c
+++ b/qemu/roms/ipxe/src/tests/dns_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/entropy_sample.c b/qemu/roms/ipxe/src/tests/entropy_sample.c
index 95a662e3e..b45648c11 100644
--- a/qemu/roms/ipxe/src/tests/entropy_sample.c
+++ b/qemu/roms/ipxe/src/tests/entropy_sample.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/hash_df_test.c b/qemu/roms/ipxe/src/tests/hash_df_test.c
index 74c8d0f4d..0b7d56ad7 100644
--- a/qemu/roms/ipxe/src/tests/hash_df_test.c
+++ b/qemu/roms/ipxe/src/tests/hash_df_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/hmac_drbg_test.c b/qemu/roms/ipxe/src/tests/hmac_drbg_test.c
index 8cbf1cc8b..ddf9db2c5 100644
--- a/qemu/roms/ipxe/src/tests/hmac_drbg_test.c
+++ b/qemu/roms/ipxe/src/tests/hmac_drbg_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/ipv4_test.c b/qemu/roms/ipxe/src/tests/ipv4_test.c
new file mode 100644
index 000000000..f84a8b81f
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/ipv4_test.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * IPv4 tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <byteswap.h>
+#include <ipxe/in.h>
+#include <ipxe/test.h>
+
+/** Define inline IPv4 address */
+#define IPV4(a,b,c,d) \
+	htonl ( ( (a) << 24 ) | ( (b) << 16 ) | ( (c) << 8 ) | (d) )
+
+/**
+ * Report an inet_ntoa() test result
+ *
+ * @v addr		IPv4 address
+ * @v text		Expected textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_ntoa_okx ( uint32_t addr, const char *text, const char *file,
+			    unsigned int line ) {
+	struct in_addr in = { .s_addr = addr };
+	char *actual;
+
+	/* Format address */
+	actual = inet_ntoa ( in );
+	DBG ( "inet_ntoa ( %d.%d.%d.%d ) = %s\n",
+	      ( ( ntohl ( addr ) >> 24 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 16 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 8 ) & 0xff ),
+	      ( ( ntohl ( addr ) >> 0 ) & 0xff ), actual );
+	okx ( strcmp ( actual, text ) == 0, file, line );
+}
+#define inet_ntoa_ok( addr, text ) \
+	inet_ntoa_okx ( addr, text, __FILE__, __LINE__ )
+
+/**
+ * Report an inet_aton() test result
+ *
+ * @v text		Textual representation
+ * @v addr		Expected IPv4 address
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_aton_okx ( const char *text, uint32_t addr, const char *file,
+			    unsigned int line ) {
+	struct in_addr actual;
+
+	/* Parse address */
+	okx ( inet_aton ( text, &actual ) != 0, file, line );
+	DBG ( "inet_aton ( \"%s\" ) = %s\n", text, inet_ntoa ( actual ) );
+	okx ( actual.s_addr == addr, file, line );
+};
+#define inet_aton_ok( text, addr ) \
+	inet_aton_okx ( text, addr, __FILE__, __LINE__ )
+
+/**
+ * Report an inet_aton() failure test result
+ *
+ * @v text		Textual representation
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void inet_aton_fail_okx ( const char *text, const char *file,
+				 unsigned int line ) {
+	struct in_addr actual;
+
+	/* Attempt to parse address */
+	okx ( inet_aton ( text, &actual ) == 0, file, line );
+}
+#define inet_aton_fail_ok( text ) \
+	inet_aton_fail_okx ( text, __FILE__, __LINE__ )
+
+/**
+ * Perform IPv4 self-tests
+ *
+ */
+static void ipv4_test_exec ( void ) {
+
+	/* Address testing macros */
+	ok (   IN_IS_CLASSA ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSB ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSC ( IPV4 ( 10, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSA ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok (   IN_IS_CLASSB ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSC ( IPV4 ( 172, 16, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSA ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok ( ! IN_IS_CLASSB ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok (   IN_IS_CLASSC ( IPV4 ( 192, 168, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 127, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 8, 8, 8, 8 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 0, 0, 0, 0 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 223, 0, 0, 1 ) ) );
+	ok ( ! IN_IS_MULTICAST ( IPV4 ( 240, 0, 0, 1 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 224, 0, 0, 1 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 231, 89, 0, 2 ) ) );
+	ok (   IN_IS_MULTICAST ( IPV4 ( 239, 6, 1, 17 ) ) );
+
+	/* inet_ntoa() tests */
+	inet_ntoa_ok ( IPV4 ( 127, 0, 0, 1 ), "127.0.0.1" );
+	inet_ntoa_ok ( IPV4 ( 0, 0, 0, 0 ), "0.0.0.0" );
+	inet_ntoa_ok ( IPV4 ( 255, 255, 255, 255 ), "255.255.255.255" );
+	inet_ntoa_ok ( IPV4 ( 212, 13, 204, 60 ), "212.13.204.60" );
+
+	/* inet_aton() tests */
+	inet_aton_ok ( "212.13.204.60", IPV4 ( 212, 13, 204, 60 ) );
+	inet_aton_ok ( "127.0.0.1", IPV4 ( 127, 0, 0, 1 ) );
+
+	/* inet_aton() failure tests */
+	inet_aton_fail_ok ( "256.0.0.1" ); /* Byte out of range */
+	inet_aton_fail_ok ( "212.13.204.60.1" ); /* Too long */
+	inet_aton_fail_ok ( "127.0.0" ); /* Too short */
+	inet_aton_fail_ok ( "1.2.3.a" ); /* Invalid characters */
+	inet_aton_fail_ok ( "127.0..1" ); /* Missing bytes */
+}
+
+/** IPv4 self-test */
+struct self_test ipv4_test __self_test = {
+	.name = "ipv4",
+	.exec = ipv4_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/ipv6_test.c b/qemu/roms/ipxe/src/tests/ipv6_test.c
index e16fc7c3d..772eb1b82 100644
--- a/qemu/roms/ipxe/src/tests/ipv6_test.c
+++ b/qemu/roms/ipxe/src/tests/ipv6_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/linebuf_test.c b/qemu/roms/ipxe/src/tests/linebuf_test.c
index e06ac7d86..0dd486e9d 100644
--- a/qemu/roms/ipxe/src/tests/linebuf_test.c
+++ b/qemu/roms/ipxe/src/tests/linebuf_test.c
@@ -1,35 +1,320 @@
-#include <stdint.h>
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Line buffer self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <string.h>
-#include <stdio.h>
+#include <assert.h>
 #include <ipxe/linebuf.h>
+#include <ipxe/test.h>
 
-static const char data1[] = 
-"Hello world\r\n"
-"This is a reasonably nice set of lines\n"
-"with not many different terminators\r\n\r\n"
-"There should be exactly one blank line above\n"
-"and this line should never appear at all since it has no terminator";
+/** Define inline raw data */
+#define DATA(...) { __VA_ARGS__ }
 
-void linebuf_test ( void ) {
-	struct line_buffer linebuf;
-	const char *data = data1;
-	size_t len = ( sizeof ( data1 ) - 1 /* be mean; strip the NUL */ );
-	ssize_t frag_len;
-	char *line;
-
-	memset ( &linebuf, 0, sizeof ( linebuf ) );
-	while ( len ) {
-		frag_len = line_buffer ( &linebuf, data, len );
-		if ( frag_len < 0 ) {
-			printf ( "line_buffer() failed: %s\n",
-				 strerror ( frag_len ) );
+/** Define inline lines */
+#define LINES(...) { __VA_ARGS__ }
+
+/** A line buffer test */
+struct linebuf_test {
+	/** Raw data */
+	const void *data;
+	/** Length of raw data */
+	size_t len;
+	/** Expected sequence of lines */
+	const char **lines;
+	/** Number of expected lines */
+	unsigned int count;
+};
+
+/** Line buffer test expected failure indicator */
+static const char linebuf_failure[1];
+
+/**
+ * Define a line buffer test
+ *
+ * @v name		Test name
+ * @v DATA		Raw data
+ * @v LINES		Expected sequence of lines
+ * @ret test		Line buffer test
+ */
+#define LINEBUF_TEST( name, DATA, LINES )				\
+	static const char name ## _data[] = DATA;			\
+	static const char * name ## _lines[] = LINES;			\
+	static struct linebuf_test name = {				\
+		.data = name ## _data,					\
+		.len = ( sizeof ( name ## _data ) - 1 /* NUL */ ),	\
+		.lines = name ## _lines,				\
+		.count = ( sizeof ( name ## _lines ) /			\
+			   sizeof ( name ## _lines[0] ) ),		\
+	}
+
+/** Simple line buffer test */
+LINEBUF_TEST ( simple,
+	       ( "HTTP/1.1 200 OK\r\n"
+		 "Content-Length: 123\r\n"
+		 "Content-Type: text/plain\r\n"
+		 "\r\n" ),
+	       LINES ( "HTTP/1.1 200 OK",
+		       "Content-Length: 123",
+		       "Content-Type: text/plain",
+		       "" ) );
+
+/** Mixed line terminators */
+LINEBUF_TEST ( mixed,
+	       ( "LF only\n" "CRLF\r\n" "\n" "\n" "\r\n" "\r\n" "CR only\r" ),
+	       LINES ( "LF only", "CRLF", "", "", "", "",
+		       NULL /* \r should not be treated as a terminator */ ) );
+
+/** Split consumption: part 1 */
+LINEBUF_TEST ( split_1,
+	       ( "This line was" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 2 */
+LINEBUF_TEST ( split_2,
+	       ( " split across" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 3 */
+LINEBUF_TEST ( split_3,
+	       ( " multiple calls\r\nand so was this one\r" ),
+	       LINES ( "This line was split across multiple calls", NULL ) );
+
+/** Split consumption: part 4 */
+LINEBUF_TEST ( split_4,
+	       ( "\nbut not this one\r\n" ),
+	       LINES ( "and so was this one", "but not this one" ) );
+
+/** Split consumption: part 5 */
+LINEBUF_TEST ( split_5,
+	       ( "" ),
+	       LINES ( NULL ) );
+
+/** Split consumption: part 6 */
+LINEBUF_TEST ( split_6,
+	       ( "This line came after a zero-length call\r\n" ),
+	       LINES ( "This line came after a zero-length call" ) );
+
+/** Embedded NULs */
+LINEBUF_TEST ( embedded_nuls,
+	       ( "This\r\ntest\r\nincludes\r\n\r\nsome\0binary\0data\r\n" ),
+	       LINES ( "This", "test", "includes", "", linebuf_failure ) );
+
+/**
+ * Report line buffer initialisation test result
+ *
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_init_okx ( struct line_buffer *linebuf,
+				const char *file, unsigned int line ) {
+
+	/* Initialise line buffer */
+	memset ( linebuf, 0, sizeof ( *linebuf ) );
+	okx ( buffered_line ( linebuf ) == NULL, file, line );
+}
+#define linebuf_init_ok( linebuf ) \
+	linebuf_init_okx ( linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer consumption test result
+ *
+ * @v test		Line buffer test
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_consume_okx ( struct linebuf_test *test,
+				  struct line_buffer *linebuf,
+				  const char *file, unsigned int line ) {
+	const char *data = test->data;
+	size_t remaining = test->len;
+	int len;
+	unsigned int i;
+	const char *expected;
+	char *actual;
+	int rc;
+
+	DBGC ( test, "LINEBUF %p:\n", test );
+	DBGC_HDA ( test, 0, data, remaining );
+
+	/* Consume data one line at a time */
+	for ( i = 0 ; i < test->count ; i++ ) {
+
+		/* Add data to line buffer */
+		len = line_buffer ( linebuf, data, remaining );
+
+		/* Get buffered line, if any */
+		actual = buffered_line ( linebuf );
+		if ( len < 0 ) {
+			rc = len;
+			DBGC ( test, "LINEBUF %p %s\n", test, strerror ( rc ) );
+		} else if ( actual != NULL ) {
+			DBGC ( test, "LINEBUF %p \"%s\" (consumed %d)\n",
+			       test, actual, len );
+		} else {
+			DBGC ( test, "LINEBUF %p unterminated (consumed %d)\n",
+			       test, len );
+		}
+
+		/* Check for success/failure */
+		expected = test->lines[i];
+		if ( expected == linebuf_failure ) {
+			rc = len;
+			okx ( rc < 0, file, line );
+			okx ( remaining > 0, file, line );
 			return;
 		}
-		data += frag_len;
-		len -= frag_len;
-		if ( ( line = buffered_line ( &linebuf ) ) )
-			printf ( "\"%s\"\n", line );
+		okx ( len >= 0, file, line );
+		okx ( ( ( size_t ) len ) <= remaining, file, line );
+
+		/* Check expected result */
+		if ( expected == NULL ) {
+			okx ( actual == NULL, file, line );
+		} else {
+			okx ( actual != NULL, file, line );
+			okx ( strcmp ( actual, expected ) == 0, file, line );
+		}
+
+		/* Consume data */
+		data += len;
+		remaining -= len;
+	}
+
+	/* Check that all data was consumed */
+	okx ( remaining == 0, file, line );
+}
+#define linebuf_consume_ok( test, linebuf ) \
+	linebuf_consume_okx ( test, linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer accumulation test result
+ *
+ * @v test		Line buffer test
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_accumulated_okx ( struct linebuf_test *test,
+				      struct line_buffer *linebuf,
+				      const char *file, unsigned int line ) {
+	const char *actual;
+	const char *expected;
+	unsigned int i;
+
+	/* Check each accumulated line */
+	actual = linebuf->data;
+	for ( i = 0 ; i < test->count ; i++ ) {
+
+		/* Check accumulated line */
+		okx ( actual != NULL, file, line );
+		okx ( actual >= linebuf->data, file, line );
+		expected = test->lines[i];
+		if ( ( expected == NULL ) || ( expected == linebuf_failure ) )
+			return;
+		okx ( strcmp ( actual, expected ) == 0, file, line );
+
+		/* Move to next line */
+		actual += ( strlen ( actual ) + 1 /* NUL */ );
+		okx ( actual <= ( linebuf->data + linebuf->len ), file, line );
 	}
+}
+#define linebuf_accumulated_ok( test, linebuf ) \
+	linebuf_accumulated_okx ( test, linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer emptying test result
+ *
+ * @v linebuf		Line buffer
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_empty_okx ( struct line_buffer *linebuf,
+				const char *file, unsigned int line ) {
 
-	empty_line_buffer ( &linebuf );
+	/* Empty line buffer */
+	empty_line_buffer ( linebuf );
+	okx ( buffered_line ( linebuf ) == NULL, file, line );
 }
+#define linebuf_empty_ok( linebuf ) \
+	linebuf_empty_okx ( linebuf, __FILE__, __LINE__ )
+
+/**
+ * Report line buffer combined test result
+ *
+ * @v test		Line buffer test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void linebuf_okx ( struct linebuf_test *test, const char *file,
+			  unsigned int line ) {
+	struct line_buffer linebuf;
+
+	linebuf_init_okx ( &linebuf, file, line );
+	linebuf_consume_okx ( test, &linebuf, file, line );
+	linebuf_accumulated_okx ( test, &linebuf, file, line );
+	linebuf_empty_okx ( &linebuf, file, line );
+}
+#define linebuf_ok( test ) \
+	linebuf_okx ( test, __FILE__, __LINE__ )
+
+/**
+ * Perform line buffer self-tests
+ *
+ */
+static void linebuf_test_exec ( void ) {
+	struct line_buffer linebuf;
+
+	/* Basic tests */
+	linebuf_ok ( &simple );
+	linebuf_ok ( &mixed );
+
+	/* Split consumption test */
+	linebuf_init_ok ( &linebuf );
+	linebuf_consume_ok ( &split_1, &linebuf );
+	linebuf_consume_ok ( &split_2, &linebuf );
+	linebuf_consume_ok ( &split_3, &linebuf );
+	linebuf_consume_ok ( &split_4, &linebuf );
+	linebuf_consume_ok ( &split_5, &linebuf );
+	linebuf_consume_ok ( &split_6, &linebuf );
+	linebuf_empty_ok ( &linebuf );
+
+	/* Embedded NULs */
+	linebuf_ok ( &embedded_nuls );
+}
+
+/** Line buffer self-test */
+struct self_test linebuf_test __self_test = {
+	.name = "linebuf",
+	.exec = linebuf_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/list_test.c b/qemu/roms/ipxe/src/tests/list_test.c
index 35cbd5e5f..352c87da0 100644
--- a/qemu/roms/ipxe/src/tests/list_test.c
+++ b/qemu/roms/ipxe/src/tests/list_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/math_test.c b/qemu/roms/ipxe/src/tests/math_test.c
index e12b7939d..1a244f1eb 100644
--- a/qemu/roms/ipxe/src/tests/math_test.c
+++ b/qemu/roms/ipxe/src/tests/math_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -35,6 +39,26 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <ipxe/isqrt.h>
 
 /**
+ * Force a call to the non-constant implementation of ffsl()
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+__attribute__ (( noinline )) int ffsl_var ( long value ) {
+	return ffsl ( value );
+}
+
+/**
+ * Force a call to the non-constant implementation of ffsll()
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+__attribute__ (( noinline )) int ffsll_var ( long long value ) {
+	return ffsll ( value );
+}
+
+/**
  * Force a call to the non-constant implementation of flsl()
  *
  * @v value		Value
@@ -173,6 +197,44 @@ __attribute__ (( noinline )) int64_t s64mod_var ( int64_t dividend,
 }
 
 /**
+ * Report a ffsl() test result
+ *
+ * @v value		Value
+ * @v lsb		Expected LSB
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+ffsl_okx ( long value, int lsb, const char *file, unsigned int line ) {
+
+	/* Verify as a constant (requires to be inlined) */
+	okx ( ffsl ( value ) == lsb, file, line );
+
+	/* Verify as a non-constant */
+	okx ( ffsl_var ( value ) == lsb, file, line );
+}
+#define ffsl_ok( value, lsb ) ffsl_okx ( value, lsb, __FILE__, __LINE__ )
+
+/**
+ * Report a ffsll() test result
+ *
+ * @v value		Value
+ * @v lsb		Expected LSB
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static inline __attribute__ (( always_inline )) void
+ffsll_okx ( long long value, int lsb, const char *file, unsigned int line ) {
+
+	/* Verify as a constant (requires to be inlined) */
+	okx ( ffsll ( value ) == lsb, file, line );
+
+	/* Verify as a non-constant */
+	okx ( ffsll_var ( value ) == lsb, file, line );
+}
+#define ffsll_ok( value, lsb ) ffsll_okx ( value, lsb, __FILE__, __LINE__ )
+
+/**
  * Report a flsl() test result
  *
  * @v value		Value
@@ -270,6 +332,22 @@ static void s64divmod_okx ( int64_t dividend, int64_t divisor,
  */
 static void math_test_exec ( void ) {
 
+	/* Test ffsl() */
+	ffsl_ok ( 0, 0 );
+	ffsl_ok ( 1, 1 );
+	ffsl_ok ( 255, 1 );
+	ffsl_ok ( 256, 9 );
+	ffsl_ok ( 257, 1 );
+	ffsl_ok ( 0x54850596, 2 );
+	ffsl_ok ( 0x80000000, 32 );
+
+	/* Test ffsll() */
+	ffsll_ok ( 0, 0 );
+	ffsll_ok ( 1, 1 );
+	ffsll_ok ( 0x6d63623330ULL, 5 );
+	ffsll_ok ( 0x80000000UL, 32 );
+	ffsll_ok ( 0x8000000000000000ULL, 64 );
+
 	/* Test flsl() */
 	flsl_ok ( 0, 0 );
 	flsl_ok ( 1, 1 );
diff --git a/qemu/roms/ipxe/src/tests/md5_test.c b/qemu/roms/ipxe/src/tests/md5_test.c
index ba5f24c3e..e9ed2716a 100644
--- a/qemu/roms/ipxe/src/tests/md5_test.c
+++ b/qemu/roms/ipxe/src/tests/md5_test.c
@@ -15,82 +15,58 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
  * MD5 tests
  *
+ * Test inputs borrowed from NIST SHA-1 tests, with results calculated
+ * using md5sum.
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/md5.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An MD5 test vector */
-struct md5_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[MD5_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "md5sum /dev/null") */
+DIGEST_TEST ( md5_empty, &md5_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9,
+		       0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e ) );
 
-/** MD5 test vectors */
-static struct md5_test_vector md5_test_vectors[] = {
-	/* Test inputs borrowed from SHA-1 tests, with results
-	 * calculated using md5sum.
-	 */
-	{ NULL, 0,
-	  { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
-	    0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } },
-	{ "abc", 3,
-	  { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0,
-	    0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x82, 0x15, 0xef, 0x07, 0x96, 0xa2, 0x0b, 0xca,
-	    0xaa, 0xe1, 0x16, 0xd3, 0x87, 0x6c, 0x66, 0x4a } },
-};
+/* NIST test vector "abc" (digest obtained from "md5sum <data>") */
+DIGEST_TEST ( md5_nist_abc, &md5_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6,
+		       0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 ) );
 
-/** MD5 test fragment lists */
-static struct digest_test_fragments md5_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" (digest obtained from "md5sum <data>") */
+DIGEST_TEST ( md5_nist_abc_opq, &md5_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x82, 0x15, 0xef, 0x07, 0x96, 0xa2, 0x0b, 0xca, 0xaa,
+		       0xe1, 0x16, 0xd3, 0x87, 0x6c, 0x66, 0x4a ) );
 
 /**
  * Perform MD5 self-test
  *
  */
 static void md5_test_exec ( void ) {
-	struct digest_algorithm *digest = &md5_algorithm;
-	struct md5_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( md5_test_vectors ) /
-			    sizeof ( md5_test_vectors[0] ) ) ; i++ ) {
-		test = &md5_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( md5_test_fragments ) /
-				    sizeof ( md5_test_fragments[0] ) ) ; j++ ){
-			digest_ok ( digest, &md5_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &md5_empty );
+	digest_ok ( &md5_nist_abc );
+	digest_ok ( &md5_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "MD5 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "MD5 required %ld cycles per byte\n",
+	      digest_cost ( &md5_algorithm ) );
 }
 
 /** MD5 self-test */
diff --git a/qemu/roms/ipxe/src/tests/memcpy_test.c b/qemu/roms/ipxe/src/tests/memcpy_test.c
index f1e5503a6..0247c71d4 100644
--- a/qemu/roms/ipxe/src/tests/memcpy_test.c
+++ b/qemu/roms/ipxe/src/tests/memcpy_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/memset_test.c b/qemu/roms/ipxe/src/tests/memset_test.c
new file mode 100644
index 000000000..d96f83fa6
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/memset_test.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * memset() self-tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <string.h>
+#include <ipxe/test.h>
+
+/* Provide global functions to allow inspection of generated code */
+
+void memset_zero_0 ( void *dest ) { memset ( dest, 0, 0 ); }
+void memset_zero_1 ( void *dest ) { memset ( dest, 0, 1 ); }
+void memset_zero_2 ( void *dest ) { memset ( dest, 0, 2 ); }
+void memset_zero_3 ( void *dest ) { memset ( dest, 0, 3 ); }
+void memset_zero_4 ( void *dest ) { memset ( dest, 0, 4 ); }
+void memset_zero_5 ( void *dest ) { memset ( dest, 0, 5 ); }
+void memset_zero_6 ( void *dest ) { memset ( dest, 0, 6 ); }
+void memset_zero_7 ( void *dest ) { memset ( dest, 0, 7 ); }
+void memset_zero_8 ( void *dest ) { memset ( dest, 0, 8 ); }
+void memset_zero_9 ( void *dest ) { memset ( dest, 0, 9 ); }
+void memset_zero_10 ( void *dest ) { memset ( dest, 0, 10 ); }
+void memset_zero_11 ( void *dest ) { memset ( dest, 0, 11 ); }
+void memset_zero_12 ( void *dest ) { memset ( dest, 0, 12 ); }
+void memset_zero_13 ( void *dest ) { memset ( dest, 0, 13 ); }
+void memset_zero_14 ( void *dest ) { memset ( dest, 0, 14 ); }
+void memset_zero_15 ( void *dest ) { memset ( dest, 0, 15 ); }
+void memset_zero_16 ( void *dest ) { memset ( dest, 0, 16 ); }
+void memset_zero_17 ( void *dest ) { memset ( dest, 0, 17 ); }
+void memset_zero_18 ( void *dest ) { memset ( dest, 0, 18 ); }
+void memset_zero_19 ( void *dest ) { memset ( dest, 0, 19 ); }
+void memset_zero_20 ( void *dest ) { memset ( dest, 0, 20 ); }
+void memset_zero_21 ( void *dest ) { memset ( dest, 0, 21 ); }
+void memset_zero_22 ( void *dest ) { memset ( dest, 0, 22 ); }
+void memset_zero_23 ( void *dest ) { memset ( dest, 0, 23 ); }
+void memset_zero_24 ( void *dest ) { memset ( dest, 0, 24 ); }
+void memset_zero_25 ( void *dest ) { memset ( dest, 0, 25 ); }
+void memset_zero_26 ( void *dest ) { memset ( dest, 0, 26 ); }
+void memset_zero_27 ( void *dest ) { memset ( dest, 0, 27 ); }
+void memset_zero_28 ( void *dest ) { memset ( dest, 0, 28 ); }
+void memset_zero_29 ( void *dest ) { memset ( dest, 0, 29 ); }
+void memset_zero_30 ( void *dest ) { memset ( dest, 0, 30 ); }
+void memset_zero_31 ( void *dest ) { memset ( dest, 0, 31 ); }
+
+/**
+ * Force a call to the variable-length implementation of memset()
+ *
+ * @v dest		Destination address
+ * @v fill		Fill pattern
+ * @v len		Length of data
+ * @ret dest		Destination address
+ */
+__attribute__ (( noinline )) void * memset_var ( void *dest, unsigned int fill,
+						 size_t len ) {
+	return memset ( dest, fill, len );
+}
+
+/**
+ * Perform a constant-length memset() test
+ *
+ * @v len		Length of data
+ */
+#define MEMSET_TEST_CONSTANT( len ) do {				\
+		uint8_t dest_const[ 1 + len + 1 ];			\
+		uint8_t dest_var[ 1 + len + 1 ];			\
+		static uint8_t zero[len];				\
+		unsigned int i;						\
+									\
+		for ( i = 0 ; i < sizeof ( dest_const ) ; i++ )		\
+			dest_const[i] = 0xaa;				\
+		memset ( ( dest_const + 1 ), 0, len );			\
+		ok ( dest_const[0] == 0xaa );				\
+		ok ( dest_const[ sizeof ( dest_const ) - 1 ] == 0xaa );	\
+		ok ( memcmp ( ( dest_const + 1 ), zero, len ) == 0 );	\
+									\
+		for ( i = 0 ; i < sizeof ( dest_var ) ; i++ )		\
+			dest_var[i] = 0xbb;				\
+		memset_var ( ( dest_var + 1 ), 0, len );		\
+		ok ( dest_var[0] == 0xbb );				\
+		ok ( dest_var[ sizeof ( dest_var ) - 1 ] == 0xbb );	\
+		ok ( memcmp ( ( dest_var + 1 ), zero, len ) == 0 );	\
+	} while ( 0 )
+
+/**
+ * Perform memset() self-tests
+ *
+ */
+static void memset_test_exec ( void ) {
+
+	/* Constant-length tests */
+	MEMSET_TEST_CONSTANT ( 0 );
+	MEMSET_TEST_CONSTANT ( 1 );
+	MEMSET_TEST_CONSTANT ( 2 );
+	MEMSET_TEST_CONSTANT ( 3 );
+	MEMSET_TEST_CONSTANT ( 4 );
+	MEMSET_TEST_CONSTANT ( 5 );
+	MEMSET_TEST_CONSTANT ( 6 );
+	MEMSET_TEST_CONSTANT ( 7 );
+	MEMSET_TEST_CONSTANT ( 8 );
+	MEMSET_TEST_CONSTANT ( 9 );
+	MEMSET_TEST_CONSTANT ( 10 );
+	MEMSET_TEST_CONSTANT ( 11 );
+	MEMSET_TEST_CONSTANT ( 12 );
+	MEMSET_TEST_CONSTANT ( 13 );
+	MEMSET_TEST_CONSTANT ( 14 );
+	MEMSET_TEST_CONSTANT ( 15 );
+	MEMSET_TEST_CONSTANT ( 16 );
+	MEMSET_TEST_CONSTANT ( 17 );
+	MEMSET_TEST_CONSTANT ( 18 );
+	MEMSET_TEST_CONSTANT ( 19 );
+	MEMSET_TEST_CONSTANT ( 20 );
+	MEMSET_TEST_CONSTANT ( 21 );
+	MEMSET_TEST_CONSTANT ( 22 );
+	MEMSET_TEST_CONSTANT ( 23 );
+	MEMSET_TEST_CONSTANT ( 24 );
+	MEMSET_TEST_CONSTANT ( 25 );
+	MEMSET_TEST_CONSTANT ( 26 );
+	MEMSET_TEST_CONSTANT ( 27 );
+	MEMSET_TEST_CONSTANT ( 28 );
+	MEMSET_TEST_CONSTANT ( 29 );
+	MEMSET_TEST_CONSTANT ( 30 );
+	MEMSET_TEST_CONSTANT ( 31 );
+}
+
+/** memset() self-test */
+struct self_test memset_test __self_test = {
+	.name = "memset",
+	.exec = memset_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/ocsp_test.c b/qemu/roms/ipxe/src/tests/ocsp_test.c
index a318c185a..c6d458596 100644
--- a/qemu/roms/ipxe/src/tests/ocsp_test.c
+++ b/qemu/roms/ipxe/src/tests/ocsp_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1857,5 +1861,6 @@ struct self_test ocsp_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( ocsp_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( sha1 );
diff --git a/qemu/roms/ipxe/src/tests/pccrc_test.c b/qemu/roms/ipxe/src/tests/pccrc_test.c
new file mode 100644
index 000000000..f4ab573ac
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/pccrc_test.c
@@ -0,0 +1,529 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * Peer Content Caching and Retrieval: Content Identification [MS-PCCRC] tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <string.h>
+#include <assert.h>
+#include <ipxe/uaccess.h>
+#include <ipxe/pccrc.h>
+#include <ipxe/sha256.h>
+#include <ipxe/sha512.h>
+#include <ipxe/hmac.h>
+#include <ipxe/test.h>
+
+/** Define inline raw data */
+#define DATA(...) { __VA_ARGS__ }
+
+/**
+ * Define an inline content range
+ *
+ * @v START		Start offset
+ * @v END		End offset
+ * @ret range		Content range
+ */
+#define RANGE( START, END ) { .start = START, .end = END }
+
+/**
+ * Define an inline trimmed content range
+ *
+ * @v START		Start offset
+ * @v END		End offset
+ * @ret trim		Trimmed content range
+ */
+#define TRIM( START, END ) { .start = START, .end = END }
+
+/** A content information test */
+struct peerdist_info_test {
+	/** Raw content information */
+	const void *data;
+	/** Length of raw content information */
+	size_t len;
+	/** Expected digest algorithm */
+	struct digest_algorithm *expected_digest;
+	/** Expected digest size */
+	size_t expected_digestsize;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected trimmed content range */
+	struct peerdist_range expected_trim;
+	/** Expected number of segments */
+	unsigned int expected_segments;
+};
+
+/**
+ * Define a content information test
+ *
+ * @v name		Test name
+ * @v DATA		Raw content information
+ * @v DIGEST		Expected digest algorithm
+ * @v DIGESTSIZE	Expected digest size
+ * @v RANGE		Expected content range
+ * @v TRIM		Expected trimmer content range
+ * @v SEGMENTS		Expected number of segments
+ * @ret test		Content information test
+ *
+ * Raw content information can be obtained from PeerDist-capable web
+ * servers using wget's "--header" option to inject the relevant
+ * PeerDist headers.  For example:
+ *
+ *   wget --header "Accept-Encoding: peerdist" \
+ *        --header "X-P2P-PeerDist: Version=1.0" \
+ *	  http://peerdist.server.address/test.url -O - | xxd -i -c 11
+ *
+ * Version 1 content information can be retrieved using the headers:
+ *
+ *   Accept-Encoding: peerdist
+ *   X-P2P-PeerDist: Version=1.0
+ *
+ * Version 2 content information can be retrieved (from compatible
+ * servers) using the headers:
+ *
+ *   Accept-Encoding: peerdist
+ *   X-P2P-PeerDist: Version=1.1
+ *   X-P2P-PeerDistEx: MinContentInformation=2.0, MaxContentInformation=2.0
+ */
+#define PEERDIST_INFO_TEST( name, DATA, DIGEST, DIGESTSIZE, RANGE,	\
+			    TRIM, SEGMENTS )				\
+	static const uint8_t name ## _data[] = DATA;			\
+	static struct peerdist_info_test name = {			\
+		.data = name ## _data,					\
+		.len = sizeof ( name ## _data ),			\
+		.expected_digest = DIGEST,				\
+		.expected_digestsize = DIGESTSIZE,			\
+		.expected_range = RANGE,				\
+		.expected_trim = TRIM,					\
+		.expected_segments = SEGMENTS,				\
+	}
+
+/** A content information segment test */
+struct peerdist_info_segment_test {
+	/** Segment index */
+	unsigned int index;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected number of blocks */
+	unsigned int expected_blocks;
+	/** Expected block size */
+	size_t expected_blksize;
+	/** Expected segment hash of data */
+	uint8_t expected_hash[PEERDIST_DIGEST_MAX_SIZE];
+	/** Expected segment secret */
+	uint8_t expected_secret[PEERDIST_DIGEST_MAX_SIZE];
+	/** Expected segment identifier */
+	uint8_t expected_id[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/**
+ * Define a content information segment test
+ *
+ * @v name		Test name
+ * @v INDEX		Segment index
+ * @v RANGE		Expected content range
+ * @v BLOCKS		Expected number of blocks
+ * @v BLKSIZE		Expected block size
+ * @v HASH		Expected segment hash of data
+ * @v SECRET		Expected segment secret
+ * @v ID		Expected segment identifier
+ * @ret test		Content information segment test
+ */
+#define PEERDIST_INFO_SEGMENT_TEST( name, INDEX, RANGE, BLOCKS,		\
+				    BLKSIZE, HASH, SECRET, ID )		\
+	static struct peerdist_info_segment_test name = {		\
+		.index = INDEX,						\
+		.expected_range = RANGE,				\
+		.expected_blocks = BLOCKS,				\
+		.expected_blksize = BLKSIZE,				\
+		.expected_hash = HASH,					\
+		.expected_secret = SECRET,				\
+		.expected_id = ID,					\
+	}
+
+/** A content information block test */
+struct peerdist_info_block_test {
+	/** Block index */
+	unsigned int index;
+	/** Expected content range */
+	struct peerdist_range expected_range;
+	/** Expected trimmed content range */
+	struct peerdist_range expected_trim;
+	/** Expected hash of data */
+	uint8_t expected_hash[PEERDIST_DIGEST_MAX_SIZE];
+};
+
+/**
+ * Define a content information block test
+ *
+ * @v name		Test name
+ * @v INDEX		Block index
+ * @v RANGE		Expected content range
+ * @v TRIM		Expected trimmed content range
+ * @v HASH		Expected hash of data
+ * @ret test		Content information block test
+ */
+#define PEERDIST_INFO_BLOCK_TEST( name, INDEX, RANGE, TRIM, HASH )	\
+	static struct peerdist_info_block_test name = {			\
+		.index = INDEX,						\
+		.expected_range = RANGE,				\
+		.expected_trim = TRIM,					\
+		.expected_hash = HASH,					\
+	}
+
+/**
+ * Define a server passphrase
+ *
+ * @v name		Server passphrase name
+ * @v DATA		Raw server passphrase
+ *
+ * The server passphrase can be exported from a Windows BranchCache
+ * server using the command:
+ *
+ *   netsh branchcache exportkey exported.key somepassword
+ *
+ * and this encrypted exported key can be decrypted using the
+ * oSSL_key_dx or mcrypt_key_dx utilities found in the (prototype)
+ * Prequel project at https://fedorahosted.org/prequel/ :
+ *
+ *   oSSL_key_dx exported.key somepassword
+ *     or
+ *   mcrypt_key_dx exported.key somepassword
+ *
+ * Either command will display both the server passphrase and the
+ * "Server Secret".  Note that this latter is the version 1 server
+ * secret (i.e. the SHA-256 of the server passphrase); the
+ * corresponding version 2 server secret can be obtained by
+ * calculating the truncated SHA-512 of the server passphrase.
+ *
+ * We do not know the server passphrase during normal operation.  We
+ * use it in the self-tests only to check for typos and other errors
+ * in the test vectors, by checking that the segment secret defined in
+ * a content information segment test is as expected.
+ */
+#define SERVER_PASSPHRASE( name, DATA )					\
+	static uint8_t name[] = DATA
+
+/** Server passphrase used for these test vectors */
+SERVER_PASSPHRASE ( passphrase,
+      DATA ( 0x2a, 0x3d, 0x73, 0xeb, 0x43, 0x5e, 0x9f, 0x2b, 0x8a, 0x34, 0x42,
+	     0x67, 0xe7, 0x46, 0x7a, 0x3c, 0x73, 0x85, 0xc6, 0xe0, 0x55, 0xe2,
+	     0xb4, 0xd3, 0x0d, 0xfe, 0xc7, 0xc3, 0x8b, 0x0e, 0xd7, 0x2c ) );
+
+/** IIS logo (iis-85.png) content information version 1 */
+PEERDIST_INFO_TEST ( iis_85_png_v1,
+	DATA ( 0x00, 0x01, 0x0c, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x7e, 0x85, 0x01, 0x00, 0x00, 0x00, 0x01,
+	       0x00, 0xd8, 0xd9, 0x76, 0x35, 0x4a, 0x48, 0x72, 0xe9, 0x25, 0x76,
+	       0x18, 0x03, 0xf4, 0x58, 0xd9, 0xda, 0xaa, 0x67, 0xf8, 0xe3, 0x1c,
+	       0x63, 0x0f, 0xb7, 0x4e, 0x6a, 0x31, 0x2e, 0xf8, 0xa2, 0x5a, 0xba,
+	       0x11, 0xaf, 0xc0, 0xd7, 0x94, 0x92, 0x43, 0xf9, 0x4f, 0x9c, 0x1f,
+	       0xab, 0x35, 0xd9, 0xfd, 0x1e, 0x33, 0x1f, 0xcf, 0x78, 0x11, 0xa2,
+	       0xe0, 0x1d, 0x35, 0x87, 0xb3, 0x8d, 0x77, 0x0a, 0x29, 0xe2, 0x02,
+	       0x00, 0x00, 0x00, 0x73, 0xc1, 0x8a, 0xb8, 0x54, 0x91, 0x10, 0xf8,
+	       0xe9, 0x0e, 0x71, 0xbb, 0xc3, 0xab, 0x2a, 0xa8, 0xc4, 0x4d, 0x13,
+	       0xf4, 0x92, 0x94, 0x99, 0x25, 0x5b, 0x66, 0x0f, 0x24, 0xec, 0x77,
+	       0x80, 0x0b, 0x97, 0x4b, 0xdd, 0x65, 0x56, 0x7f, 0xde, 0xec, 0xcd,
+	       0xaf, 0xe4, 0x57, 0xa9, 0x50, 0x3b, 0x45, 0x48, 0xf6, 0x6e, 0xd3,
+	       0xb1, 0x88, 0xdc, 0xfd, 0xa0, 0xac, 0x38, 0x2b, 0x09, 0x71, 0x1a,
+	       0xcc ),
+	&sha256_algorithm, 32, RANGE ( 0, 99710 ), TRIM ( 0, 99710 ), 1 );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v1_s0, 0,
+	RANGE ( 0, 99710 ), 2, 65536,
+	DATA ( 0xd8, 0xd9, 0x76, 0x35, 0x4a, 0x48, 0x72, 0xe9, 0x25, 0x76, 0x18,
+	       0x03, 0xf4, 0x58, 0xd9, 0xda, 0xaa, 0x67, 0xf8, 0xe3, 0x1c, 0x63,
+	       0x0f, 0xb7, 0x4e, 0x6a, 0x31, 0x2e, 0xf8, 0xa2, 0x5a, 0xba ),
+	DATA ( 0x11, 0xaf, 0xc0, 0xd7, 0x94, 0x92, 0x43, 0xf9, 0x4f, 0x9c, 0x1f,
+	       0xab, 0x35, 0xd9, 0xfd, 0x1e, 0x33, 0x1f, 0xcf, 0x78, 0x11, 0xa2,
+	       0xe0, 0x1d, 0x35, 0x87, 0xb3, 0x8d, 0x77, 0x0a, 0x29, 0xe2 ),
+	DATA ( 0x49, 0x1b, 0x21, 0x7d, 0xbe, 0xe2, 0xb5, 0xf1, 0x2c, 0xa7, 0x9b,
+	       0x01, 0x5e, 0x06, 0xf4, 0xbb, 0xe6, 0x4f, 0x97, 0x45, 0xba, 0xd7,
+	       0x86, 0x7a, 0xef, 0x17, 0xde, 0x59, 0x92, 0x7e, 0xdc, 0xe9 ) );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v1_s0_b0, 0,
+	RANGE ( 0, 65536 ),
+	TRIM ( 0, 65536 ),
+	DATA ( 0x73, 0xc1, 0x8a, 0xb8, 0x54, 0x91, 0x10, 0xf8, 0xe9, 0x0e, 0x71,
+	       0xbb, 0xc3, 0xab, 0x2a, 0xa8, 0xc4, 0x4d, 0x13, 0xf4, 0x92, 0x94,
+	       0x99, 0x25, 0x5b, 0x66, 0x0f, 0x24, 0xec, 0x77, 0x80, 0x0b ) );
+
+/** IIS logo (iis-85.png) content information version 1 segment 0 block 1 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v1_s0_b1, 1,
+	RANGE ( 65536, 99710 ),
+	TRIM ( 65536, 99710 ),
+	DATA ( 0x97, 0x4b, 0xdd, 0x65, 0x56, 0x7f, 0xde, 0xec, 0xcd, 0xaf, 0xe4,
+	       0x57, 0xa9, 0x50, 0x3b, 0x45, 0x48, 0xf6, 0x6e, 0xd3, 0xb1, 0x88,
+	       0xdc, 0xfd, 0xa0, 0xac, 0x38, 0x2b, 0x09, 0x71, 0x1a, 0xcc ) );
+
+/** IIS logo (iis-85.png) content information version 2 */
+PEERDIST_INFO_TEST ( iis_85_png_v2,
+	DATA ( 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	       0x00, 0x00, 0x88, 0x00, 0x00, 0x99, 0xde, 0xe0, 0xd0, 0xc3, 0x58,
+	       0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32, 0xb5, 0xf1, 0x97, 0x87,
+	       0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e, 0x78, 0x1f, 0xae, 0x71,
+	       0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4, 0x58, 0x03, 0x7e, 0xd4, 0x04,
+	       0x11, 0x6b, 0xb6, 0x16, 0xd9, 0xb1, 0x41, 0x16, 0x08, 0x85, 0x20,
+	       0xc4, 0x7c, 0xdc, 0x50, 0xab, 0xce, 0xa3, 0xfa, 0xe1, 0x88, 0xa9,
+	       0x8e, 0xa2, 0x2d, 0xf3, 0xc0, 0x00, 0x00, 0xeb, 0xa0, 0x33, 0x81,
+	       0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21, 0x0f, 0x37,
+	       0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96, 0xa1, 0x30,
+	       0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc, 0xb8, 0xb6, 0xeb,
+	       0x77, 0x83, 0xe4, 0xf8, 0x07, 0x64, 0x7b, 0x63, 0xf1, 0x46, 0xb5,
+	       0x2f, 0x4a, 0xc8, 0x9c, 0xcc, 0x7a, 0xbf, 0x5f, 0xa1, 0x1a, 0xca,
+	       0xfc, 0x2a, 0xcf, 0x50, 0x28, 0x58, 0x6c ),
+	&sha512_algorithm, 32, RANGE ( 0, 99710 ), TRIM ( 0, 99710 ), 2 );
+
+/** IIS logo (iis-85.png) content information version 2 segment 0 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v2_s0, 0,
+	RANGE ( 0, 39390 ), 1, 39390,
+	DATA ( 0xe0, 0xd0, 0xc3, 0x58, 0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32,
+	       0xb5, 0xf1, 0x97, 0x87, 0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e,
+	       0x78, 0x1f, 0xae, 0x71, 0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4 ),
+	DATA ( 0x58, 0x03, 0x7e, 0xd4, 0x04, 0x11, 0x6b, 0xb6, 0x16, 0xd9, 0xb1,
+	       0x41, 0x16, 0x08, 0x85, 0x20, 0xc4, 0x7c, 0xdc, 0x50, 0xab, 0xce,
+	       0xa3, 0xfa, 0xe1, 0x88, 0xa9, 0x8e, 0xa2, 0x2d, 0xf3, 0xc0 ),
+	DATA ( 0x33, 0x71, 0xbb, 0xea, 0xdd, 0xb6, 0x23, 0x53, 0xad, 0xce, 0xf9,
+	       0x70, 0xa0, 0x6f, 0xdf, 0x65, 0x00, 0x1e, 0x04, 0x21, 0xf4, 0xc7,
+	       0x10, 0x82, 0x76, 0xb0, 0xc3, 0x7a, 0x9f, 0x9e, 0xc1, 0x0f ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 0 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v2_s0_b0, 0,
+	RANGE ( 0, 39390 ),
+	TRIM ( 0, 39390 ),
+	DATA ( 0xe0, 0xd0, 0xc3, 0x58, 0xe2, 0x68, 0x4b, 0x62, 0x33, 0x0d, 0x32,
+	       0xb5, 0xf1, 0x97, 0x87, 0x24, 0xa0, 0xd0, 0xa5, 0x2b, 0xdc, 0x5e,
+	       0x78, 0x1f, 0xae, 0x71, 0xff, 0x57, 0xa8, 0xbe, 0x3d, 0xd4 ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 1 */
+PEERDIST_INFO_SEGMENT_TEST ( iis_85_png_v2_s1, 1,
+	RANGE ( 39390, 99710 ), 1, 60320,
+	DATA ( 0x33, 0x81, 0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21,
+	       0x0f, 0x37, 0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96,
+	       0xa1, 0x30, 0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc ),
+	DATA ( 0xb8, 0xb6, 0xeb, 0x77, 0x83, 0xe4, 0xf8, 0x07, 0x64, 0x7b, 0x63,
+	       0xf1, 0x46, 0xb5, 0x2f, 0x4a, 0xc8, 0x9c, 0xcc, 0x7a, 0xbf, 0x5f,
+	       0xa1, 0x1a, 0xca, 0xfc, 0x2a, 0xcf, 0x50, 0x28, 0x58, 0x6c ),
+	DATA ( 0xd7, 0xe9, 0x24, 0x42, 0x5e, 0x8f, 0x4f, 0x88, 0xf0, 0x1d, 0xc6,
+	       0xa9, 0xbb, 0x1b, 0xc3, 0x7b, 0xe1, 0x13, 0xec, 0x79, 0x17, 0xc7,
+	       0x45, 0xd4, 0x96, 0x5c, 0x2b, 0x55, 0xfa, 0x16, 0x3a, 0x6e ) );
+
+/** IIS logo (iis-85.png) content information version 2 segment 1 block 0 */
+PEERDIST_INFO_BLOCK_TEST ( iis_85_png_v2_s1_b0, 0,
+	RANGE ( 39390, 99710 ),
+	TRIM ( 39390, 99710 ),
+	DATA ( 0x33, 0x81, 0xd0, 0xd0, 0xcb, 0x74, 0xf4, 0xb6, 0x13, 0xd8, 0x21,
+	       0x0f, 0x37, 0xf0, 0x02, 0xa0, 0x6f, 0x39, 0x10, 0x58, 0x60, 0x96,
+	       0xa1, 0x30, 0xd3, 0x43, 0x98, 0xc0, 0x8e, 0x66, 0xd7, 0xbc ) );
+
+/**
+ * Report content information test result
+ *
+ * @v test		Content information test
+ * @v info		Content information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void peerdist_info_okx ( struct peerdist_info_test *test,
+				struct peerdist_info *info,
+				const char *file, unsigned int line ) {
+
+	/* Parse content information */
+	okx ( peerdist_info ( virt_to_user ( test->data ), test->len,
+			      info ) == 0, file, line );
+
+	/* Verify content information */
+	okx ( info->raw.data == virt_to_user ( test->data ), file, line );
+	okx ( info->raw.len == test->len, file, line );
+	okx ( info->digest == test->expected_digest, file, line );
+	okx ( info->digestsize == test->expected_digestsize, file, line );
+	okx ( info->range.start == test->expected_range.start, file, line );
+	okx ( info->range.end == test->expected_range.end, file, line );
+	okx ( info->trim.start == test->expected_trim.start, file, line );
+	okx ( info->trim.end == test->expected_trim.end, file, line );
+	okx ( info->trim.start >= info->range.start, file, line );
+	okx ( info->trim.end <= info->range.end, file, line );
+	okx ( info->segments == test->expected_segments, file, line );
+}
+#define peerdist_info_ok( test, info ) \
+	peerdist_info_okx ( test, info, __FILE__, __LINE__ )
+
+/**
+ * Report content information segment test result
+ *
+ * @v test		Content information segment test
+ * @v info		Content information
+ * @v segment		Segment information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void peerdist_info_segment_okx ( struct peerdist_info_segment_test *test,
+					const struct peerdist_info *info,
+					struct peerdist_info_segment *segment,
+					const char *file, unsigned int line ) {
+	size_t digestsize = info->digestsize;
+
+	/* Parse content information segment */
+	okx ( peerdist_info_segment ( info, segment, test->index ) == 0,
+	      file, line );
+
+	/* Verify content information segment */
+	okx ( segment->info == info, file, line );
+	okx ( segment->index == test->index, file, line );
+	okx ( segment->range.start == test->expected_range.start, file, line );
+	okx ( segment->range.end == test->expected_range.end, file, line );
+	okx ( segment->blocks == test->expected_blocks, file, line );
+	okx ( segment->blksize == test->expected_blksize, file, line );
+	okx ( memcmp ( segment->hash, test->expected_hash,
+		       digestsize ) == 0, file, line );
+	okx ( memcmp ( segment->secret, test->expected_secret,
+		       digestsize ) == 0, file, line );
+	okx ( memcmp ( segment->id, test->expected_id,
+		       digestsize ) == 0, file, line );
+}
+#define peerdist_info_segment_ok( test, info, segment ) \
+	peerdist_info_segment_okx ( test, info, segment, __FILE__, __LINE__ )
+
+/**
+ * Report content information block test result
+ *
+ * @v test		Content information block test
+ * @v segment		Segment information
+ * @v block		Block information to fill in
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void
+peerdist_info_block_okx ( struct peerdist_info_block_test *test,
+			  const struct peerdist_info_segment *segment,
+			  struct peerdist_info_block *block,
+			  const char *file, unsigned int line ) {
+	const struct peerdist_info *info = segment->info;
+	size_t digestsize = info->digestsize;
+
+	/* Parse content information block */
+	okx ( peerdist_info_block ( segment, block, test->index ) == 0,
+	      file, line );
+
+	/* Verify content information block */
+	okx ( block->segment == segment, file, line );
+	okx ( block->index == test->index, file, line );
+	okx ( block->range.start == test->expected_range.start, file, line );
+	okx ( block->range.end == test->expected_range.end, file, line );
+	okx ( block->trim.start == test->expected_trim.start, file, line );
+	okx ( block->trim.end == test->expected_trim.end, file, line );
+	okx ( memcmp ( block->hash, test->expected_hash,
+		       digestsize ) == 0, file, line );
+}
+#define peerdist_info_block_ok( test, segment, block ) \
+	peerdist_info_block_okx ( test, segment, block, __FILE__, __LINE__ )
+
+/**
+ * Report server passphrase test result
+ *
+ * @v test		Content information segment test
+ * @v info		Content information
+ * @v pass		Server passphrase
+ * @v pass_len		Length of server passphrase
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void
+peerdist_info_passphrase_okx ( struct peerdist_info_segment_test *test,
+			       const struct peerdist_info *info,
+			       uint8_t *pass, size_t pass_len,
+			       const char *file, unsigned int line ) {
+	struct digest_algorithm *digest = info->digest;
+	uint8_t ctx[digest->ctxsize];
+	uint8_t secret[digest->digestsize];
+	uint8_t expected[digest->digestsize];
+	size_t digestsize = info->digestsize;
+	size_t secretsize = digestsize;
+
+	/* Calculate server secret */
+	digest_init ( digest, ctx );
+	digest_update ( digest, ctx, pass, pass_len );
+	digest_final ( digest, ctx, secret );
+
+	/* Calculate expected segment secret */
+	hmac_init ( digest, ctx, secret, &secretsize );
+	assert ( secretsize == digestsize );
+	hmac_update ( digest, ctx, test->expected_hash, digestsize );
+	hmac_final ( digest, ctx, secret, &secretsize, expected );
+	assert ( secretsize == digestsize );
+
+	/* Verify segment secret */
+	okx ( memcmp ( test->expected_secret, expected, digestsize ) == 0,
+	      file, line );
+}
+#define peerdist_info_passphrase_ok( test, info, pass, pass_len )	\
+	peerdist_info_passphrase_okx ( test, info, pass, pass_len,	\
+				       __FILE__, __LINE__ )
+
+/**
+ * Perform content information self-tests
+ *
+ */
+static void peerdist_info_test_exec ( void ) {
+	struct peerdist_info info;
+	struct peerdist_info_segment segment;
+	struct peerdist_info_block block;
+
+	/* IIS logo (iis-85.png) content information version 1 */
+	peerdist_info_ok ( &iis_85_png_v1, &info );
+	peerdist_info_passphrase_ok ( &iis_85_png_v1_s0, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v1_s0, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v1_s0_b0, &segment, &block );
+	peerdist_info_block_ok ( &iis_85_png_v1_s0_b1, &segment, &block );
+
+	/* IIS logo (iis-85.png) content information version 2 */
+	peerdist_info_ok ( &iis_85_png_v2, &info );
+	peerdist_info_passphrase_ok ( &iis_85_png_v2_s0, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v2_s0, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v2_s0_b0, &segment, &block );
+	peerdist_info_passphrase_ok ( &iis_85_png_v2_s1, &info,
+				      passphrase, sizeof ( passphrase ) );
+	peerdist_info_segment_ok ( &iis_85_png_v2_s1, &info, &segment );
+	peerdist_info_block_ok ( &iis_85_png_v2_s1_b0, &segment, &block );
+}
+
+/** Content information self-test */
+struct self_test peerdist_info_test __self_test = {
+	.name = "pccrc",
+	.exec = peerdist_info_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/pixbuf_test.c b/qemu/roms/ipxe/src/tests/pixbuf_test.c
index 15cd33dfd..aaa516bb2 100644
--- a/qemu/roms/ipxe/src/tests/pixbuf_test.c
+++ b/qemu/roms/ipxe/src/tests/pixbuf_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -53,8 +57,8 @@ void pixbuf_okx ( struct pixel_buffer_test *test, const char *file,
 	/* Correct image data pointer */
 	test->image->data = virt_to_user ( ( void * ) test->image->data );
 
-	/* Check that image is detected as PNM */
-	okx ( image_probe ( test->image ) == 0, file, line );
+	/* Check that image is detected as correct type */
+	okx ( register_image ( test->image ) == 0, file, line );
 	okx ( test->image->type == test->type, file, line );
 
 	/* Check that a pixel buffer can be created from the image */
@@ -73,4 +77,7 @@ void pixbuf_okx ( struct pixel_buffer_test *test, const char *file,
 
 		pixbuf_put ( pixbuf );
 	}
+
+	/* Unregister image */
+	unregister_image ( test->image );
 }
diff --git a/qemu/roms/ipxe/src/tests/pixbuf_test.h b/qemu/roms/ipxe/src/tests/pixbuf_test.h
index 394f7f5fa..d12829d89 100644
--- a/qemu/roms/ipxe/src/tests/pixbuf_test.h
+++ b/qemu/roms/ipxe/src/tests/pixbuf_test.h
@@ -1,7 +1,7 @@
 #ifndef _PIXBUF_TEST_H
 #define _PIXBUF_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/refcnt.h>
diff --git a/qemu/roms/ipxe/src/tests/png_test.c b/qemu/roms/ipxe/src/tests/png_test.c
index cf32f2034..e921aa2a6 100644
--- a/qemu/roms/ipxe/src/tests/png_test.c
+++ b/qemu/roms/ipxe/src/tests/png_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/pnm_test.c b/qemu/roms/ipxe/src/tests/pnm_test.c
index 26b0c0726..d57fdaaef 100644
--- a/qemu/roms/ipxe/src/tests/pnm_test.c
+++ b/qemu/roms/ipxe/src/tests/pnm_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/profile_test.c b/qemu/roms/ipxe/src/tests/profile_test.c
index 9d682bf2b..d2f8df211 100644
--- a/qemu/roms/ipxe/src/tests/profile_test.c
+++ b/qemu/roms/ipxe/src/tests/profile_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/pubkey_test.h b/qemu/roms/ipxe/src/tests/pubkey_test.h
index 7678453a9..cd65b8703 100644
--- a/qemu/roms/ipxe/src/tests/pubkey_test.h
+++ b/qemu/roms/ipxe/src/tests/pubkey_test.h
@@ -1,7 +1,7 @@
 #ifndef _PUBKEY_TEST_H
 #define _PUBKEY_TEST_H
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <ipxe/crypto.h>
diff --git a/qemu/roms/ipxe/src/tests/rsa_test.c b/qemu/roms/ipxe/src/tests/rsa_test.c
index 3b32c74bc..c0d05d263 100644
--- a/qemu/roms/ipxe/src/tests/rsa_test.c
+++ b/qemu/roms/ipxe/src/tests/rsa_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/setjmp_test.c b/qemu/roms/ipxe/src/tests/setjmp_test.c
new file mode 100644
index 000000000..50ad13f3c
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/setjmp_test.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * setjmp()/longjmp() tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stddef.h>
+#include <assert.h>
+#include <setjmp.h>
+#include <ipxe/test.h>
+
+/** A setjmp()/longjmp() test */
+struct setjmp_test {
+	/** Jump buffer */
+	jmp_buf env;
+	/** Expected value */
+	int expected;
+	/** Test code file */
+	const char *file;
+	/** Test code line */
+	unsigned int line;
+};
+
+/** Expected jump */
+static struct setjmp_test *jumped;
+
+/**
+ * Report a setjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ *
+ * This has to be implemented as a macro since if it were a function
+ * then the context saved by setjmp() would be invalidated when the
+ * function returned.
+ */
+#define setjmp_ok( test ) do {						\
+	int value;							\
+	/* Sanity check */						\
+	assert ( jumped == NULL );					\
+	/* Initialise test */						\
+	(test)->expected = 0;						\
+	(test)->file = __FILE__;					\
+	(test)->line = __LINE__;					\
+	/* Perform setjmp() */						\
+	value = setjmp ( (test)->env );					\
+	/* Report setjmp()/longjmp() result */				\
+	setjmp_return_ok ( (test), value );				\
+	} while ( 0 )
+
+/**
+ * Report a setjmp()/longjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ * @v value		Value returned from setjmp()
+ *
+ * This function ends up reporting results from either setjmp() or
+ * longjmp() tests (since calls to longjmp() will return via the
+ * corresponding setjmp()).  It therefore uses the test code file and
+ * line stored in the test structure, which will represent the line
+ * from which either setjmp() or longjmp() was called.
+ */
+static void setjmp_return_ok ( struct setjmp_test *test, int value ) {
+
+	/* Determine whether this was reached via setjmp() or longjmp() */
+	if ( value == 0 ) {
+		/* This is the initial call to setjmp() */
+		okx ( test->expected == 0, test->file, test->line );
+		okx ( jumped == NULL, test->file, test->line );
+	} else {
+		/* This is reached via a call to longjmp() */
+		okx ( value == test->expected, test->file, test->line );
+		okx ( jumped == test, test->file, test->line );
+	}
+
+	/* Clear expected jump */
+	jumped = NULL;
+}
+
+/**
+ * Report a longjmp() test result
+ *
+ * @v test		setjmp()/longjmp() test
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void longjmp_okx ( struct setjmp_test *test, int value,
+			  const char *file, unsigned int line ) {
+
+	/* Record expected value.  A zero passed to longjmp() should
+	 * result in setjmp() returning a value of one.
+	 */
+	test->expected = ( value ? value : 1 );
+
+	/* Record test code file and line */
+	test->file = file;
+	test->line = line;
+
+	/* Record expected jump */
+	jumped = test;
+
+	/* Perform longjmp().  Should return via setjmp_okx() */
+	longjmp ( test->env, value );
+
+	/* longjmp() should never return */
+	assert ( 0 );
+}
+#define longjmp_ok( test, value ) \
+	longjmp_okx ( test, value, __FILE__, __LINE__ )
+
+/**
+ * Perform setjmp()/longjmp() self-tests
+ *
+ */
+static void setjmp_test_exec ( void ) {
+	static struct setjmp_test alpha;
+	static struct setjmp_test beta;
+	static int iteration;
+
+	/* This is one of the very few situations in which the
+	 * "for-case" pattern is justified.
+	 */
+	for ( iteration = 0 ; iteration < 10 ; iteration++ ) {
+		DBGC ( jumped, "SETJMP test iteration %d\n", iteration );
+		switch ( iteration ) {
+		case 0: setjmp_ok ( &alpha ); break;
+		case 1: setjmp_ok ( &beta ); break;
+		case 2:	longjmp_ok ( &alpha, 0 );
+		case 3: longjmp_ok ( &alpha, 1 );
+		case 4: longjmp_ok ( &alpha, 2 );
+		case 5: longjmp_ok ( &beta, 17 );
+		case 6: longjmp_ok ( &beta, 29 );
+		case 7: longjmp_ok ( &alpha, -1 );
+		case 8: longjmp_ok ( &beta, 0 );
+		case 9: longjmp_ok ( &beta, 42 );
+		}
+	}
+}
+
+/** setjmp()/longjmp() self-test */
+struct self_test setjmp_test __self_test = {
+	.name = "setjmp",
+	.exec = setjmp_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/settings_test.c b/qemu/roms/ipxe/src/tests/settings_test.c
index 4ee6a10fa..f7fb35d0d 100644
--- a/qemu/roms/ipxe/src/tests/settings_test.c
+++ b/qemu/roms/ipxe/src/tests/settings_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -228,6 +232,12 @@ static struct setting test_hexraw_setting = {
 	.type = &setting_type_hexraw,
 };
 
+/** Test Base64 setting type */
+static struct setting test_base64_setting = {
+	.name = "test_base64",
+	.type = &setting_type_base64,
+};
+
 /** Test UUID setting type */
 static struct setting test_uuid_setting = {
 	.name = "test_uuid",
@@ -379,6 +389,15 @@ static void settings_test_exec ( void ) {
 			  0x17, 0x06, 0x39, 0x6b, 0xf4, 0x48, 0x4e ),
 		    "9e4b6eef36b646fe8f1706396bf4484e" );
 
+	/* "base64" setting type */
+	storef_ok ( &test_settings, &test_base64_setting,
+		    "cGFzc6\nNwaHJhc2U= ",
+		    RAW ( 0x70, 0x61, 0x73, 0x73, 0xa3, 0x70, 0x68, 0x72, 0x61,
+			  0x73, 0x65 ) );
+	fetchf_ok ( &test_settings, &test_base64_setting,
+		    RAW ( 0x80, 0x81, 0x82, 0x83, 0x84, 0x00, 0xff ),
+		    "gIGCg4QA/w==" );
+
 	/* "uuid" setting type (no store capability) */
 	fetchf_ok ( &test_settings, &test_uuid_setting,
 		    RAW ( 0x1a, 0x6a, 0x74, 0x9d, 0x0e, 0xda, 0x46, 0x1a,0xa8,
@@ -399,3 +418,7 @@ struct self_test settings_test __self_test = {
 	.name = "settings",
 	.exec = settings_test_exec,
 };
+
+/* Include real IPv6 setting type */
+REQUIRING_SYMBOL ( settings_test );
+REQUIRE_OBJECT ( ipv6 );
diff --git a/qemu/roms/ipxe/src/tests/sha1_test.c b/qemu/roms/ipxe/src/tests/sha1_test.c
index bcf761bdd..9f1d75686 100644
--- a/qemu/roms/ipxe/src/tests/sha1_test.c
+++ b/qemu/roms/ipxe/src/tests/sha1_test.c
@@ -15,87 +15,63 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
  * SHA-1 tests
  *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA1.pdf
+ *
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/sha1.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An SHA-1 test vector */
-struct sha1_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[SHA1_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "sha1sum /dev/null") */
+DIGEST_TEST ( sha1_empty, &sha1_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
+		       0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
+		       0x07, 0x09 ) );
 
-/** SHA-1 test vectors */
-static struct sha1_test_vector sha1_test_vectors[] = {
-	/* Empty test data
-	 *
-	 * Expected digest value obtained from "sha1sum /dev/null"
-	 */
-	{ NULL, 0,
-	  { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55,
-	    0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09 } },
-	/* Test data and expected digests taken from the NIST
-	 * Cryptographic Toolkit Algorithm Examples at
-	 * http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA1.pdf
-	 */
-	{ "abc", 3,
-	  { 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e,
-	    0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x84, 0x98, 0x3e, 0x44, 0x1c, 0x3b, 0xd2, 0x6e, 0xba, 0xae,
-	    0x4a, 0xa1, 0xf9, 0x51, 0x29, 0xe5, 0xe5, 0x46, 0x70, 0xf1 } },
-};
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha1_nist_abc, &sha1_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba,
+		       0x3e, 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0,
+		       0xd8, 0x9d ) );
 
-/** SHA-1 test fragment lists */
-static struct digest_test_fragments sha1_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha1_nist_abc_opq, &sha1_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x84, 0x98, 0x3e, 0x44, 0x1c, 0x3b, 0xd2, 0x6e, 0xba,
+		       0xae, 0x4a, 0xa1, 0xf9, 0x51, 0x29, 0xe5, 0xe5, 0x46,
+		       0x70, 0xf1 ) );
 
 /**
  * Perform SHA-1 self-test
  *
  */
 static void sha1_test_exec ( void ) {
-	struct digest_algorithm *digest = &sha1_algorithm;
-	struct sha1_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( sha1_test_vectors ) /
-			    sizeof ( sha1_test_vectors[0] ) ) ; i++ ) {
-		test = &sha1_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( sha1_test_fragments ) /
-				    sizeof ( sha1_test_fragments[0] ) ) ; j++ ){
-			digest_ok ( digest, &sha1_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &sha1_empty );
+	digest_ok ( &sha1_nist_abc );
+	digest_ok ( &sha1_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "SHA1 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "SHA1 required %ld cycles per byte\n",
+	      digest_cost ( &sha1_algorithm ) );
 }
 
 /** SHA-1 self-test */
diff --git a/qemu/roms/ipxe/src/tests/sha256_test.c b/qemu/roms/ipxe/src/tests/sha256_test.c
index 06a8cae25..3b4c423fd 100644
--- a/qemu/roms/ipxe/src/tests/sha256_test.c
+++ b/qemu/roms/ipxe/src/tests/sha256_test.c
@@ -15,93 +15,96 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
- * SHA-256 tests
+ * SHA-256 family tests
+ *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA256.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA224.pdf
  *
  */
 
-#include <stdint.h>
+/* Forcibly enable assertions */
+#undef NDEBUG
+
 #include <ipxe/sha256.h>
 #include <ipxe/test.h>
 #include "digest_test.h"
 
-/** An SHA-256 test vector */
-struct sha256_test_vector {
-	/** Test data */
-	void *data;
-	/** Test data length */
-	size_t len;
-	/** Expected digest */
-	uint8_t digest[SHA256_DIGEST_SIZE];
-};
+/* Empty test vector (digest obtained from "sha256sum /dev/null") */
+DIGEST_TEST ( sha256_empty, &sha256_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
+		       0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
+		       0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
+		       0x1b, 0x78, 0x52, 0xb8, 0x55 ) );
 
-/** SHA-256 test vectors */
-static struct sha256_test_vector sha256_test_vectors[] = {
-	/* Empty test data
-	 *
-	 * Expected digest value obtained from "sha256sum /dev/null"
-	 */
-	{ NULL, 0,
-	  { 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4,
-	    0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b,
-	    0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 } },
-	/* Test data and expected digests taken from the NIST
-	 * Cryptographic Toolkit Algorithm Examples at
-	 * http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA256.pdf
-	 */
-	{ "abc", 3,
-	  { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40,
-	    0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17,
-	    0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad } },
-	{ "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 56,
-	  { 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5, 0xc0, 0x26,
-	    0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff,
-	    0x21, 0x67, 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1 } },
-};
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha256_nist_abc, &sha256_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41,
+		       0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03,
+		       0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff,
+		       0x61, 0xf2, 0x00, 0x15, 0xad ) );
 
-/** SHA-256 test fragment lists */
-static struct digest_test_fragments sha256_test_fragments[] = {
-	{ { 0, -1UL, } },
-	{ { 1, 1, 1, 1, 1, 1, 1, 1 } },
-	{ { 2, 0, 23, 4, 6, 1, 0 } },
-};
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha256_nist_abc_opq, &sha256_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8, 0xe5,
+		       0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39, 0xa3, 0x3c,
+		       0xe4, 0x59, 0x64, 0xff, 0x21, 0x67, 0xf6, 0xec, 0xed,
+		       0xd4, 0x19, 0xdb, 0x06, 0xc1 ) );
+
+/* Empty test vector (digest obtained from "sha224sum /dev/null") */
+DIGEST_TEST ( sha224_empty, &sha224_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
+		       0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
+		       0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
+		       0x2f ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha224_nist_abc, &sha224_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x23, 0x09, 0x7d, 0x22, 0x34, 0x05, 0xd8, 0x22, 0x86,
+		       0x42, 0xa4, 0x77, 0xbd, 0xa2, 0x55, 0xb3, 0x2a, 0xad,
+		       0xbc, 0xe4, 0xbd, 0xa0, 0xb3, 0xf7, 0xe3, 0x6c, 0x9d,
+		       0xa7 ) );
+
+/* NIST test vector "abc...opq" */
+DIGEST_TEST ( sha224_nist_abc_opq, &sha224_algorithm, DIGEST_NIST_ABC_OPQ,
+	      DIGEST ( 0x75, 0x38, 0x8b, 0x16, 0x51, 0x27, 0x76, 0xcc, 0x5d,
+		       0xba, 0x5d, 0xa1, 0xfd, 0x89, 0x01, 0x50, 0xb0, 0xc6,
+		       0x45, 0x5c, 0xb4, 0xf5, 0x8b, 0x19, 0x52, 0x52, 0x25,
+		       0x25 ) );
 
 /**
- * Perform SHA-256 self-test
+ * Perform SHA-256 family self-test
  *
  */
 static void sha256_test_exec ( void ) {
-	struct digest_algorithm *digest = &sha256_algorithm;
-	struct sha256_test_vector *test;
-	unsigned long cost;
-	unsigned int i;
-	unsigned int j;
 
-	/* Correctness test */
-	for ( i = 0 ; i < ( sizeof ( sha256_test_vectors ) /
-			    sizeof ( sha256_test_vectors[0] ) ) ; i++ ) {
-		test = &sha256_test_vectors[i];
-		/* Test with a single pass */
-		digest_ok ( digest, NULL, test->data, test->len, test->digest );
-		/* Test with fragment lists */
-		for ( j = 0 ; j < ( sizeof ( sha256_test_fragments ) /
-				    sizeof ( sha256_test_fragments[0] )); j++ ){
-			digest_ok ( digest, &sha256_test_fragments[j],
-				    test->data, test->len, test->digest );
-		}
-	}
+	/* Correctness tests */
+	digest_ok ( &sha256_empty );
+	digest_ok ( &sha256_nist_abc );
+	digest_ok ( &sha256_nist_abc_opq );
+	digest_ok ( &sha224_empty );
+	digest_ok ( &sha224_nist_abc );
+	digest_ok ( &sha224_nist_abc_opq );
 
-	/* Speed test */
-	cost = digest_cost ( digest );
-	DBG ( "SHA256 required %ld cycles per byte\n", cost );
+	/* Speed tests */
+	DBG ( "SHA256 required %ld cycles per byte\n",
+	      digest_cost ( &sha256_algorithm ) );
+	DBG ( "SHA224 required %ld cycles per byte\n",
+	      digest_cost ( &sha224_algorithm ) );
 }
 
-/** SHA-256 self-test */
+/** SHA-256 family self-test */
 struct self_test sha256_test __self_test = {
 	.name = "sha256",
 	.exec = sha256_test_exec,
diff --git a/qemu/roms/ipxe/src/tests/sha512_test.c b/qemu/roms/ipxe/src/tests/sha512_test.c
new file mode 100644
index 000000000..be530ebad
--- /dev/null
+++ b/qemu/roms/ipxe/src/tests/sha512_test.c
@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** @file
+ *
+ * SHA-512 family tests
+ *
+ * NIST test vectors are taken from
+ *
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA384.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512_256.pdf
+ *  http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA512_224.pdf
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <ipxe/sha512.h>
+#include <ipxe/test.h>
+#include "digest_test.h"
+
+/* Empty test vector (digest obtained from "sha512sum /dev/null") */
+DIGEST_TEST ( sha512_empty, &sha512_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, 0xf1,
+		       0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, 0xd6, 0x20,
+		       0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, 0x83, 0xf4, 0xa9,
+		       0x21, 0xd3, 0x6c, 0xe9, 0xce, 0x47, 0xd0, 0xd1, 0x3c,
+		       0x5d, 0x85, 0xf2, 0xb0, 0xff, 0x83, 0x18, 0xd2, 0x87,
+		       0x7e, 0xec, 0x2f, 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41,
+		       0x7a, 0x81, 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda,
+		       0x3e ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_nist_abc, &sha512_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xdd, 0xaf, 0x35, 0xa1, 0x93, 0x61, 0x7a, 0xba, 0xcc,
+		       0x41, 0x73, 0x49, 0xae, 0x20, 0x41, 0x31, 0x12, 0xe6,
+		       0xfa, 0x4e, 0x89, 0xa9, 0x7e, 0xa2, 0x0a, 0x9e, 0xee,
+		       0xe6, 0x4b, 0x55, 0xd3, 0x9a, 0x21, 0x92, 0x99, 0x2a,
+		       0x27, 0x4f, 0xc1, 0xa8, 0x36, 0xba, 0x3c, 0x23, 0xa3,
+		       0xfe, 0xeb, 0xbd, 0x45, 0x4d, 0x44, 0x23, 0x64, 0x3c,
+		       0xe8, 0x0e, 0x2a, 0x9a, 0xc9, 0x4f, 0xa5, 0x4c, 0xa4,
+		       0x9f ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_nist_abc_stu, &sha512_algorithm, DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x8e, 0x95, 0x9b, 0x75, 0xda, 0xe3, 0x13, 0xda, 0x8c,
+		       0xf4, 0xf7, 0x28, 0x14, 0xfc, 0x14, 0x3f, 0x8f, 0x77,
+		       0x79, 0xc6, 0xeb, 0x9f, 0x7f, 0xa1, 0x72, 0x99, 0xae,
+		       0xad, 0xb6, 0x88, 0x90, 0x18, 0x50, 0x1d, 0x28, 0x9e,
+		       0x49, 0x00, 0xf7, 0xe4, 0x33, 0x1b, 0x99, 0xde, 0xc4,
+		       0xb5, 0x43, 0x3a, 0xc7, 0xd3, 0x29, 0xee, 0xb6, 0xdd,
+		       0x26, 0x54, 0x5e, 0x96, 0xe5, 0x5b, 0x87, 0x4b, 0xe9,
+		       0x09 ) );
+
+/* Empty test vector (digest obtained from "sha384sum /dev/null") */
+DIGEST_TEST ( sha384_empty, &sha384_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0x38, 0xb0, 0x60, 0xa7, 0x51, 0xac, 0x96, 0x38, 0x4c,
+		       0xd9, 0x32, 0x7e, 0xb1, 0xb1, 0xe3, 0x6a, 0x21, 0xfd,
+		       0xb7, 0x11, 0x14, 0xbe, 0x07, 0x43, 0x4c, 0x0c, 0xc7,
+		       0xbf, 0x63, 0xf6, 0xe1, 0xda, 0x27, 0x4e, 0xde, 0xbf,
+		       0xe7, 0x6f, 0x65, 0xfb, 0xd5, 0x1a, 0xd2, 0xf1, 0x48,
+		       0x98, 0xb9, 0x5b ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha384_nist_abc, &sha384_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0xcb, 0x00, 0x75, 0x3f, 0x45, 0xa3, 0x5e, 0x8b, 0xb5,
+		       0xa0, 0x3d, 0x69, 0x9a, 0xc6, 0x50, 0x07, 0x27, 0x2c,
+		       0x32, 0xab, 0x0e, 0xde, 0xd1, 0x63, 0x1a, 0x8b, 0x60,
+		       0x5a, 0x43, 0xff, 0x5b, 0xed, 0x80, 0x86, 0x07, 0x2b,
+		       0xa1, 0xe7, 0xcc, 0x23, 0x58, 0xba, 0xec, 0xa1, 0x34,
+		       0xc8, 0x25, 0xa7 ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha384_nist_abc_stu, &sha384_algorithm, DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x09, 0x33, 0x0c, 0x33, 0xf7, 0x11, 0x47, 0xe8, 0x3d,
+		       0x19, 0x2f, 0xc7, 0x82, 0xcd, 0x1b, 0x47, 0x53, 0x11,
+		       0x1b, 0x17, 0x3b, 0x3b, 0x05, 0xd2, 0x2f, 0xa0, 0x80,
+		       0x86, 0xe3, 0xb0, 0xf7, 0x12, 0xfc, 0xc7, 0xc7, 0x1a,
+		       0x55, 0x7e, 0x2d, 0xb9, 0x66, 0xc3, 0xe9, 0xfa, 0x91,
+		       0x74, 0x60, 0x39 ) );
+
+/* Empty test vector (digest obtained from "shasum -a 512256 /dev/null") */
+DIGEST_TEST ( sha512_256_empty, &sha512_256_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0xc6, 0x72, 0xb8, 0xd1, 0xef, 0x56, 0xed, 0x28, 0xab,
+		       0x87, 0xc3, 0x62, 0x2c, 0x51, 0x14, 0x06, 0x9b, 0xdd,
+		       0x3a, 0xd7, 0xb8, 0xf9, 0x73, 0x74, 0x98, 0xd0, 0xc0,
+		       0x1e, 0xce, 0xf0, 0x96, 0x7a ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_256_nist_abc, &sha512_256_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x53, 0x04, 0x8e, 0x26, 0x81, 0x94, 0x1e, 0xf9, 0x9b,
+		       0x2e, 0x29, 0xb7, 0x6b, 0x4c, 0x7d, 0xab, 0xe4, 0xc2,
+		       0xd0, 0xc6, 0x34, 0xfc, 0x6d, 0x46, 0xe0, 0xe2, 0xf1,
+		       0x31, 0x07, 0xe7, 0xaf, 0x23 ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_256_nist_abc_stu, &sha512_256_algorithm,
+	      DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x39, 0x28, 0xe1, 0x84, 0xfb, 0x86, 0x90, 0xf8, 0x40,
+		       0xda, 0x39, 0x88, 0x12, 0x1d, 0x31, 0xbe, 0x65, 0xcb,
+		       0x9d, 0x3e, 0xf8, 0x3e, 0xe6, 0x14, 0x6f, 0xea, 0xc8,
+		       0x61, 0xe1, 0x9b, 0x56, 0x3a ) );
+
+/* Empty test vector (digest obtained from "shasum -a 512224 /dev/null") */
+DIGEST_TEST ( sha512_224_empty, &sha512_224_algorithm, DIGEST_EMPTY,
+	      DIGEST ( 0x6e, 0xd0, 0xdd, 0x02, 0x80, 0x6f, 0xa8, 0x9e, 0x25,
+		       0xde, 0x06, 0x0c, 0x19, 0xd3, 0xac, 0x86, 0xca, 0xbb,
+		       0x87, 0xd6, 0xa0, 0xdd, 0xd0, 0x5c, 0x33, 0x3b, 0x84,
+		       0xf4 ) );
+
+/* NIST test vector "abc" */
+DIGEST_TEST ( sha512_224_nist_abc, &sha512_224_algorithm, DIGEST_NIST_ABC,
+	      DIGEST ( 0x46, 0x34, 0x27, 0x0f, 0x70, 0x7b, 0x6a, 0x54, 0xda,
+		       0xae, 0x75, 0x30, 0x46, 0x08, 0x42, 0xe2, 0x0e, 0x37,
+		       0xed, 0x26, 0x5c, 0xee, 0xe9, 0xa4, 0x3e, 0x89, 0x24,
+		       0xaa ) );
+
+/* NIST test vector "abc...stu" */
+DIGEST_TEST ( sha512_224_nist_abc_stu, &sha512_224_algorithm,
+	      DIGEST_NIST_ABC_STU,
+	      DIGEST ( 0x23, 0xfe, 0xc5, 0xbb, 0x94, 0xd6, 0x0b, 0x23, 0x30,
+		       0x81, 0x92, 0x64, 0x0b, 0x0c, 0x45, 0x33, 0x35, 0xd6,
+		       0x64, 0x73, 0x4f, 0xe4, 0x0e, 0x72, 0x68, 0x67, 0x4a,
+		       0xf9 ) );
+
+/**
+ * Perform SHA-512 family self-test
+ *
+ */
+static void sha512_test_exec ( void ) {
+
+	/* Correctness tests */
+	digest_ok ( &sha512_empty );
+	digest_ok ( &sha512_nist_abc );
+	digest_ok ( &sha512_nist_abc_stu );
+	digest_ok ( &sha384_empty );
+	digest_ok ( &sha384_nist_abc );
+	digest_ok ( &sha384_nist_abc_stu );
+	digest_ok ( &sha512_256_empty );
+	digest_ok ( &sha512_256_nist_abc );
+	digest_ok ( &sha512_256_nist_abc_stu );
+	digest_ok ( &sha512_224_empty );
+	digest_ok ( &sha512_224_nist_abc );
+	digest_ok ( &sha512_224_nist_abc_stu );
+
+	/* Speed tests */
+	DBG ( "SHA512 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_algorithm ) );
+	DBG ( "SHA384 required %ld cycles per byte\n",
+	      digest_cost ( &sha384_algorithm ) );
+	DBG ( "SHA512/256 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_256_algorithm ) );
+	DBG ( "SHA512/224 required %ld cycles per byte\n",
+	      digest_cost ( &sha512_224_algorithm ) );
+}
+
+/** SHA-512 family self-test */
+struct self_test sha512_test __self_test = {
+	.name = "sha512",
+	.exec = sha512_test_exec,
+};
diff --git a/qemu/roms/ipxe/src/tests/string_test.c b/qemu/roms/ipxe/src/tests/string_test.c
index 3b48d9f3d..4693b5f65 100644
--- a/qemu/roms/ipxe/src/tests/string_test.c
+++ b/qemu/roms/ipxe/src/tests/string_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -31,7 +35,10 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdint.h>
 #include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
+#include <strings.h>
+#include <ipxe/string.h>
 #include <ipxe/test.h>
 
 /**
@@ -63,6 +70,18 @@ static void string_test_exec ( void ) {
 	ok ( *(strchr ( "Testing", 'g' )) == 'g' );
 	ok ( strchr ( "Testing", 'x' ) == NULL );
 
+	/* Test strrchr() */
+	ok ( strrchr ( "", 'a' ) == NULL );
+	ok ( *(strrchr ( "Haystack", 'a' )) == 'a' );
+	ok ( *(strrchr ( "Haystack", 'k' )) == 'k' );
+	ok ( strrchr ( "Haystack", 'x' ) == NULL );
+
+	/* Test memchr() */
+	ok ( memchr ( "", '\0', 0 ) == NULL );
+	ok ( *((uint8_t *)memchr ( "post\0null", 'l', 9 )) == 'l' );
+	ok ( *((uint8_t *)memchr ( "post\0null", '\0', 9 )) == '\0' );
+	ok ( memchr ( "thingy", 'z', 6 ) == NULL );
+
 	/* Test strcmp() */
 	ok ( strcmp ( "", "" ) == 0 );
 	ok ( strcmp ( "Hello", "Hello" ) == 0 );
@@ -78,11 +97,31 @@ static void string_test_exec ( void ) {
 	ok ( strncmp ( "Goodbye", "Goodbye world", 32 ) != 0 );
 	ok ( strncmp ( "Goodbye", "Goodbye world", 7 ) == 0 );
 
+	/* Test strcasecmp() */
+	ok ( strcasecmp ( "", "" ) == 0 );
+	ok ( strcasecmp ( "Uncle Jack", "Uncle jack" ) == 0 );
+	ok ( strcasecmp ( "Uncle Jack", "Uncle" ) != 0 );
+	ok ( strcasecmp ( "Uncle", "Uncle Jack" ) != 0 );
+	ok ( strcasecmp ( "not", "equal" ) != 0 );
+
 	/* Test memcmp() */
 	ok ( memcmp ( "", "", 0 ) == 0 );
 	ok ( memcmp ( "Foo", "Foo", 3 ) == 0 );
 	ok ( memcmp ( "Foo", "Bar", 3 ) != 0 );
 
+	/* Test strstr() */
+	{
+		const char haystack[] = "find me!";
+		char *found;
+
+		found = strstr ( haystack, "find" );
+		ok ( found == &haystack[0] );
+		found = strstr ( haystack, "me" );
+		ok ( found == &haystack[5] );
+		found = strstr ( haystack, "me." );
+		ok ( found == NULL );
+	}
+
 	/* Test memset() */
 	{
 		static uint8_t test[7] = { '>', 1, 1, 1, 1, 1, '<' };
@@ -154,6 +193,107 @@ static void string_test_exec ( void ) {
 		ok ( strcmp ( dup, "hello" ) == 0 );
 		free ( dup );
 	}
+
+	/* Test strcpy() */
+	{
+		const char longer[7] = "copyme";
+		const char shorter[3] = "hi";
+		char dest[7];
+		char *copy;
+
+		copy = strcpy ( dest, longer );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, longer, 7 ) == 0 );
+		copy = strcpy ( dest, shorter );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, shorter, 3 ) == 0 );
+		ok ( memcmp ( ( dest + 3 ), ( longer + 3 ), 4 ) == 0 );
+	}
+
+	/* Test strncpy() */
+	{
+		const char src[5] = "copy";
+		const char orig[8] = { 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x' };
+		const char zero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+		char dest[8];
+		char *copy;
+
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 5 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 5 ) == 0 );
+		ok ( memcmp ( dest + 5, orig + 5, 3 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 4 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 4 ) == 0 );
+		ok ( memcmp ( dest + 4, orig + 4, 4 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, src, 8 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, src, 5 ) == 0 );
+		ok ( memcmp ( dest + 5, zero + 5, 3 ) == 0 );
+		memcpy ( dest, orig, sizeof ( dest ) );
+		copy = strncpy ( dest, "", 8 );
+		ok ( copy == dest );
+		ok ( memcmp ( dest, zero, 8 ) == 0 );
+	}
+
+	/* Test strcat() */
+	{
+		char buf[16] = "append";
+		char *dest;
+
+		dest = strcat ( buf, " this" );
+		ok ( dest == buf );
+		ok ( strcmp ( buf, "append this" ) == 0 );
+	}
+
+	/* Test digit_value() */
+	{
+		unsigned int i;
+		char buf[2];
+		for ( i = 0 ; i < 16 ; i++ ) {
+			snprintf ( buf, sizeof ( buf ), "%x", i );
+			ok ( digit_value ( buf[0] ) == i );
+			snprintf ( buf, sizeof ( buf ), "%X", i );
+			ok ( digit_value ( buf[0] ) == i );
+		}
+		ok ( digit_value ( 0 ) >= 16 );
+		ok ( digit_value ( 9 ) >= 16 );
+		ok ( digit_value ( '0' - 1 ) >= 16 );
+		ok ( digit_value ( '9' + 1 ) >= 16 );
+		ok ( digit_value ( 'A' - 1 ) >= 16 );
+		ok ( digit_value ( 'F' + 1 ) >= 16 );
+		ok ( digit_value ( 'a' - 1 ) >= 16 );
+		ok ( digit_value ( 'f' + 1 ) >= 16 );
+	}
+
+	/* Test strtoul() */
+	ok ( strtoul ( "12345", NULL, 0 ) == 12345UL );
+	ok ( strtoul ( "  741", NULL, 10 ) == 741UL );
+	ok ( strtoul ( " 555a", NULL, 0 ) == 555UL );
+	ok ( strtoul ( " 555a", NULL, 16 ) == 0x555aUL );
+	ok ( strtoul ( "-12", NULL, 0 ) == -12UL );
+	ok ( strtoul ( "+3", NULL, 0 ) == 3UL );
+	ok ( strtoul ( "721", NULL, 0 ) == 721UL );
+	ok ( strtoul ( "721", NULL, 8 ) == 0721UL );
+	ok ( strtoul ( "0721", NULL, 0 ) == 0721UL );
+	ok ( strtoul ( "", NULL, 0 ) == 0UL );
+	ok ( strtoul ( "\t0xcAfe", NULL, 0 ) == 0xcafeUL );
+	ok ( strtoul ( "0xffffffff", NULL, 0 ) == 0xffffffffUL );
+	{
+		static const char string[] = "123aHa.world";
+		char *endp;
+		ok ( strtoul ( string, &endp, 0 ) == 123UL );
+		ok ( endp == &string[3] );
+		ok ( strtoul ( string, &endp, 16 ) == 0x123aUL );
+		ok ( endp == &string[4] );
+		ok ( strtoul ( string, &endp, 26 ) ==
+		     ( ( ( ( ( 1 * 26 + 2 ) * 26 + 3 ) * 26 + 10 ) * 26
+			 + 17 ) * 26 + 10 ) );
+		ok ( endp == &string[6] );
+	}
 }
 
 /** String self-test */
diff --git a/qemu/roms/ipxe/src/tests/tcpip_test.c b/qemu/roms/ipxe/src/tests/tcpip_test.c
index 00c88ae32..759f886bc 100644
--- a/qemu/roms/ipxe/src/tests/tcpip_test.c
+++ b/qemu/roms/ipxe/src/tests/tcpip_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/test.c b/qemu/roms/ipxe/src/tests/test.c
index c05e72a76..67bd4cf89 100644
--- a/qemu/roms/ipxe/src/tests/test.c
+++ b/qemu/roms/ipxe/src/tests/test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/tests.c b/qemu/roms/ipxe/src/tests/tests.c
index 2b4b78c7c..54ce86677 100644
--- a/qemu/roms/ipxe/src/tests/tests.c
+++ b/qemu/roms/ipxe/src/tests/tests.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -26,6 +30,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
  */
 
 /* Drag in all applicable self-tests */
+PROVIDE_REQUIRING_SYMBOL();
+REQUIRE_OBJECT ( memset_test );
 REQUIRE_OBJECT ( memcpy_test );
 REQUIRE_OBJECT ( string_test );
 REQUIRE_OBJECT ( math_test );
@@ -37,12 +43,14 @@ REQUIRE_OBJECT ( base16_test );
 REQUIRE_OBJECT ( settings_test );
 REQUIRE_OBJECT ( time_test );
 REQUIRE_OBJECT ( tcpip_test );
+REQUIRE_OBJECT ( ipv4_test );
 REQUIRE_OBJECT ( ipv6_test );
 REQUIRE_OBJECT ( crc32_test );
 REQUIRE_OBJECT ( md5_test );
 REQUIRE_OBJECT ( sha1_test );
 REQUIRE_OBJECT ( sha256_test );
-REQUIRE_OBJECT ( aes_cbc_test );
+REQUIRE_OBJECT ( sha512_test );
+REQUIRE_OBJECT ( aes_test );
 REQUIRE_OBJECT ( hmac_drbg_test );
 REQUIRE_OBJECT ( hash_df_test );
 REQUIRE_OBJECT ( bigint_test );
@@ -56,3 +64,6 @@ REQUIRE_OBJECT ( png_test );
 REQUIRE_OBJECT ( dns_test );
 REQUIRE_OBJECT ( uri_test );
 REQUIRE_OBJECT ( profile_test );
+REQUIRE_OBJECT ( setjmp_test );
+REQUIRE_OBJECT ( pccrc_test );
+REQUIRE_OBJECT ( linebuf_test );
diff --git a/qemu/roms/ipxe/src/tests/time_test.c b/qemu/roms/ipxe/src/tests/time_test.c
index 28acebee6..3bf01dd1d 100644
--- a/qemu/roms/ipxe/src/tests/time_test.c
+++ b/qemu/roms/ipxe/src/tests/time_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/uri_test.c b/qemu/roms/ipxe/src/tests/uri_test.c
index 14f1b4ad0..da7fb8abe 100644
--- a/qemu/roms/ipxe/src/tests/uri_test.c
+++ b/qemu/roms/ipxe/src/tests/uri_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -66,6 +70,8 @@ struct uri_resolve_test {
 struct uri_tftp_test {
 	/** Next-server address */
 	struct in_addr next_server;
+	/** Port number */
+	unsigned int port;
 	/** Filename */
 	const char *filename;
 	/** URI */
@@ -330,7 +336,7 @@ static void uri_tftp_okx ( struct uri_tftp_test *test, const char *file,
 	size_t len;
 
 	/* Construct URI */
-	uri = tftp_uri ( test->next_server, test->filename );
+	uri = tftp_uri ( test->next_server, test->port, test->filename );
 	okx ( uri != NULL, file, line );
 	if ( uri ) {
 		uri_okx ( uri, &test->uri, file, line );
@@ -674,7 +680,7 @@ static struct uri_resolve_test uri_fragment = {
 
 /** TFTP URI with absolute path */
 static struct uri_tftp_test uri_tftp_absolute = {
-	{ .s_addr = htonl ( 0xc0a80002 ) /* 192.168.0.2 */ },
+	{ .s_addr = htonl ( 0xc0a80002 ) /* 192.168.0.2 */ }, 0,
 	"/absolute/path",
 	{
 		.scheme = "tftp",
@@ -686,7 +692,7 @@ static struct uri_tftp_test uri_tftp_absolute = {
 
 /** TFTP URI with relative path */
 static struct uri_tftp_test uri_tftp_relative = {
-	{ .s_addr = htonl ( 0xc0a80003 ) /* 192.168.0.3 */ },
+	{ .s_addr = htonl ( 0xc0a80003 ) /* 192.168.0.3 */ }, 0,
 	"relative/path",
 	{
 		.scheme = "tftp",
@@ -698,7 +704,7 @@ static struct uri_tftp_test uri_tftp_relative = {
 
 /** TFTP URI with path containing special characters */
 static struct uri_tftp_test uri_tftp_icky = {
-	{ .s_addr = htonl ( 0x0a000006 ) /* 10.0.0.6 */ },
+	{ .s_addr = htonl ( 0x0a000006 ) /* 10.0.0.6 */ }, 0,
 	"C:\\tftpboot\\icky#path",
 	{
 		.scheme = "tftp",
@@ -708,6 +714,19 @@ static struct uri_tftp_test uri_tftp_icky = {
 	"tftp://10.0.0.6/C%3A\\tftpboot\\icky%23path",
 };
 
+/** TFTP URI with custom port */
+static struct uri_tftp_test uri_tftp_port = {
+	{ .s_addr = htonl ( 0xc0a80001 ) /* 192.168.0.1 */ }, 4069,
+	"/another/path",
+	{
+		.scheme = "tftp",
+		.host = "192.168.0.1",
+		.port = "4069",
+		.path = "/another/path",
+	},
+	"tftp://192.168.0.1:4069/another/path",
+};
+
 /** Current working URI test */
 static struct uri_churi_test uri_churi[] = {
 	{
@@ -842,6 +861,7 @@ static void uri_test_exec ( void ) {
 	uri_tftp_ok ( &uri_tftp_absolute );
 	uri_tftp_ok ( &uri_tftp_relative );
 	uri_tftp_ok ( &uri_tftp_icky );
+	uri_tftp_ok ( &uri_tftp_port );
 
 	/* Current working URI tests */
 	uri_churi_ok ( uri_churi );
diff --git a/qemu/roms/ipxe/src/tests/vsprintf_test.c b/qemu/roms/ipxe/src/tests/vsprintf_test.c
index 11512ec8e..0ad4f1c56 100644
--- a/qemu/roms/ipxe/src/tests/vsprintf_test.c
+++ b/qemu/roms/ipxe/src/tests/vsprintf_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/tests/x509_test.c b/qemu/roms/ipxe/src/tests/x509_test.c
index fd39e12d2..658d5247c 100644
--- a/qemu/roms/ipxe/src/tests/x509_test.c
+++ b/qemu/roms/ipxe/src/tests/x509_test.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
@@ -1105,6 +1109,7 @@ struct self_test x509_test __self_test = {
 };
 
 /* Drag in algorithms required for tests */
+REQUIRING_SYMBOL ( x509_test );
 REQUIRE_OBJECT ( rsa );
 REQUIRE_OBJECT ( sha1 );
 REQUIRE_OBJECT ( sha256 );
diff --git a/qemu/roms/ipxe/src/usr/autoboot.c b/qemu/roms/ipxe/src/usr/autoboot.c
index 47476ae40..912543828 100644
--- a/qemu/roms/ipxe/src/usr/autoboot.c
+++ b/qemu/roms/ipxe/src/usr/autoboot.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -42,6 +46,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <usr/prompt.h>
 #include <usr/autoboot.h>
 #include <config/general.h>
+#include <config/branding.h>
 
 /** @file
  *
@@ -101,7 +106,7 @@ static struct uri * parse_next_server_and_filename ( struct in_addr next_server,
 	/* Construct a TFTP URI for the filename, if applicable */
 	if ( next_server.s_addr && filename[0] && ! uri_is_absolute ( uri ) ) {
 		uri_put ( uri );
-		uri = tftp_uri ( next_server, filename );
+		uri = tftp_uri ( next_server, 0, filename );
 		if ( ! uri )
 			return NULL;
 	}
@@ -173,6 +178,7 @@ int uriboot ( struct uri *filename, struct uri *root_path, int drive,
 	if ( filename ) {
 		if ( ( rc = imgdownload ( filename, 0, &image ) ) != 0 )
 			goto err_download;
+		imgstat ( image );
 		image->flags |= IMAGE_AUTO_UNREGISTER;
 		if ( ( rc = image_exec ( image ) ) != 0 ) {
 			printf ( "Could not boot image: %s\n",
@@ -434,9 +440,14 @@ int netboot ( struct net_device *netdev ) {
  * @ret is_autoboot	Network device matches the autoboot device
  */
 static int is_autoboot_busloc ( struct net_device *netdev ) {
+	struct device *dev;
 
-	return ( ( netdev->dev->desc.bus_type == autoboot_desc.bus_type ) &&
-		 ( netdev->dev->desc.location == autoboot_desc.location ) );
+	for ( dev = netdev->dev ; dev ; dev = dev->parent ) {
+		if ( ( dev->desc.bus_type == autoboot_desc.bus_type ) &&
+		     ( dev->desc.location == autoboot_desc.location ) )
+			return 1;
+	}
+	return 0;
 }
 
 /**
@@ -522,7 +533,8 @@ static int shell_banner ( void ) {
 
 	/* Prompt user */
 	printf ( "\n" );
-	return ( prompt ( "Press Ctrl-B for the iPXE command line...",
+	return ( prompt ( "Press Ctrl-B for the " PRODUCT_SHORT_NAME
+			  " command line...",
 			  ( ( BANNER_TIMEOUT * TICKS_PER_SEC ) / 10 ),
 			  CTRL_B ) == 0 );
 }
@@ -531,28 +543,29 @@ static int shell_banner ( void ) {
  * Main iPXE flow of execution
  *
  * @v netdev		Network device, or NULL
+ * @ret rc		Return status code
  */
-void ipxe ( struct net_device *netdev ) {
+int ipxe ( struct net_device *netdev ) {
 	struct feature *feature;
 	struct image *image;
 	char *scriptlet;
+	int rc;
 
 	/*
 	 * Print welcome banner
 	 *
 	 *
 	 * If you wish to brand this build of iPXE, please do so by
-	 * defining the string PRODUCT_NAME in config/general.h.
+	 * defining the string PRODUCT_NAME in config/branding.h.
 	 *
 	 * While nothing in the GPL prevents you from removing all
 	 * references to iPXE or http://ipxe.org, we prefer you not to
 	 * do so.
 	 *
 	 */
-	printf ( NORMAL "\n\n%s\n" BOLD "iPXE %s"
-		 NORMAL " -- Open Source Network Boot Firmware -- "
-		 CYAN "http://ipxe.org" NORMAL "\n"
-		 "Features:", product_name, product_version );
+	printf ( NORMAL "\n\n" PRODUCT_NAME "\n" BOLD PRODUCT_SHORT_NAME " %s"
+		 NORMAL " -- " PRODUCT_TAG_LINE " -- "
+		 CYAN PRODUCT_URI NORMAL "\nFeatures:", product_version );
 	for_each_table_entry ( feature, FEATURES )
 		printf ( " %s", feature->name );
 	printf ( "\n" );
@@ -560,28 +573,30 @@ void ipxe ( struct net_device *netdev ) {
 	/* Boot system */
 	if ( ( image = first_image() ) != NULL ) {
 		/* We have an embedded image; execute it */
-		image_exec ( image );
+		return image_exec ( image );
 	} else if ( shell_banner() ) {
 		/* User wants shell; just give them a shell */
-		shell();
+		return shell();
 	} else {
 		fetch_string_setting_copy ( NULL, &scriptlet_setting,
 					    &scriptlet );
 		if ( scriptlet ) {
 			/* User has defined a scriptlet; execute it */
-			system ( scriptlet );
+			rc = system ( scriptlet );
 			free ( scriptlet );
+			return rc;
 		} else {
 			/* Try booting.  If booting fails, offer the
 			 * user another chance to enter the shell.
 			 */
 			if ( netdev ) {
-				netboot ( netdev );
+				rc = netboot ( netdev );
 			} else {
-				autoboot();
+				rc = autoboot();
 			}
 			if ( shell_banner() )
-				shell();
+				rc = shell();
+			return rc;
 		}
 	}
 }
diff --git a/qemu/roms/ipxe/src/usr/dhcpmgmt.c b/qemu/roms/ipxe/src/usr/dhcpmgmt.c
index 23982b19c..dcb360b23 100644
--- a/qemu/roms/ipxe/src/usr/dhcpmgmt.c
+++ b/qemu/roms/ipxe/src/usr/dhcpmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/usr/fcmgmt.c b/qemu/roms/ipxe/src/usr/fcmgmt.c
index a30f37a71..6f626143f 100644
--- a/qemu/roms/ipxe/src/usr/fcmgmt.c
+++ b/qemu/roms/ipxe/src/usr/fcmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/usr/ifmgmt.c b/qemu/roms/ipxe/src/usr/ifmgmt.c
index 3d05895c2..aefdaa45d 100644
--- a/qemu/roms/ipxe/src/usr/ifmgmt.c
+++ b/qemu/roms/ipxe/src/usr/ifmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <string.h>
 #include <stdio.h>
@@ -99,11 +103,12 @@ static void ifstat_errors ( struct net_device_stats *stats,
  */
 void ifstat ( struct net_device *netdev ) {
 	printf ( "%s: %s using %s on %s (%s)\n"
-		 "  [Link:%s, TX:%d TXE:%d RX:%d RXE:%d]\n",
+		 "  [Link:%s%s, TX:%d TXE:%d RX:%d RXE:%d]\n",
 		 netdev->name, netdev_addr ( netdev ),
 		 netdev->dev->driver_name, netdev->dev->name,
 		 ( netdev_is_open ( netdev ) ? "open" : "closed" ),
 		 ( netdev_link_ok ( netdev ) ? "up" : "down" ),
+		 ( netdev_link_blocked ( netdev ) ? " (blocked)" : "" ),
 		 netdev->tx_stats.good, netdev->tx_stats.bad,
 		 netdev->rx_stats.good, netdev->rx_stats.bad );
 	if ( ! netdev_link_ok ( netdev ) ) {
diff --git a/qemu/roms/ipxe/src/usr/imgmgmt.c b/qemu/roms/ipxe/src/usr/imgmgmt.c
index c9c571640..352dd0242 100644
--- a/qemu/roms/ipxe/src/usr/imgmgmt.c
+++ b/qemu/roms/ipxe/src/usr/imgmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/usr/imgtrust.c b/qemu/roms/ipxe/src/usr/imgtrust.c
index da7ff2ef0..a269833a6 100644
--- a/qemu/roms/ipxe/src/usr/imgtrust.c
+++ b/qemu/roms/ipxe/src/usr/imgtrust.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdlib.h>
 #include <errno.h>
diff --git a/qemu/roms/ipxe/src/usr/ipstat.c b/qemu/roms/ipxe/src/usr/ipstat.c
index 95ad799dc..0f09cc2ff 100644
--- a/qemu/roms/ipxe/src/usr/ipstat.c
+++ b/qemu/roms/ipxe/src/usr/ipstat.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/ipstat.h>
diff --git a/qemu/roms/ipxe/src/usr/lotest.c b/qemu/roms/ipxe/src/usr/lotest.c
index ad7a2fad7..6b328713c 100644
--- a/qemu/roms/ipxe/src/usr/lotest.c
+++ b/qemu/roms/ipxe/src/usr/lotest.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/usr/neighmgmt.c b/qemu/roms/ipxe/src/usr/neighmgmt.c
index e4d21a208..9fd88f82b 100644
--- a/qemu/roms/ipxe/src/usr/neighmgmt.c
+++ b/qemu/roms/ipxe/src/usr/neighmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/neighbour.h>
diff --git a/qemu/roms/ipxe/src/usr/pingmgmt.c b/qemu/roms/ipxe/src/usr/pingmgmt.c
index 16b3ec994..bb33c5d47 100644
--- a/qemu/roms/ipxe/src/usr/pingmgmt.c
+++ b/qemu/roms/ipxe/src/usr/pingmgmt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdio.h>
diff --git a/qemu/roms/ipxe/src/usr/profstat.c b/qemu/roms/ipxe/src/usr/profstat.c
index 991427473..d80fa26b2 100644
--- a/qemu/roms/ipxe/src/usr/profstat.c
+++ b/qemu/roms/ipxe/src/usr/profstat.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/profile.h>
diff --git a/qemu/roms/ipxe/src/usr/prompt.c b/qemu/roms/ipxe/src/usr/prompt.c
index 957b4ab3d..fca0a157c 100644
--- a/qemu/roms/ipxe/src/usr/prompt.c
+++ b/qemu/roms/ipxe/src/usr/prompt.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /** @file
  *
diff --git a/qemu/roms/ipxe/src/usr/pxemenu.c b/qemu/roms/ipxe/src/usr/pxemenu.c
index b69905df1..2d05d3f51 100644
--- a/qemu/roms/ipxe/src/usr/pxemenu.c
+++ b/qemu/roms/ipxe/src/usr/pxemenu.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/qemu/roms/ipxe/src/usr/route.c b/qemu/roms/ipxe/src/usr/route.c
index ba4cc3221..690ba3b6b 100644
--- a/qemu/roms/ipxe/src/usr/route.c
+++ b/qemu/roms/ipxe/src/usr/route.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <ipxe/netdevice.h>
 #include <usr/route.h>
@@ -42,3 +46,7 @@ void route ( void ) {
 		}
 	}
 }
+
+/* Drag in routing management configuration */
+REQUIRING_SYMBOL ( route );
+REQUIRE_OBJECT ( config_route );
diff --git a/qemu/roms/ipxe/src/usr/route_ipv4.c b/qemu/roms/ipxe/src/usr/route_ipv4.c
index b4d1b7bf3..6260335ac 100644
--- a/qemu/roms/ipxe/src/usr/route_ipv4.c
+++ b/qemu/roms/ipxe/src/usr/route_ipv4.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/netdevice.h>
diff --git a/qemu/roms/ipxe/src/usr/route_ipv6.c b/qemu/roms/ipxe/src/usr/route_ipv6.c
index 6045f85bb..9e94b4a15 100644
--- a/qemu/roms/ipxe/src/usr/route_ipv6.c
+++ b/qemu/roms/ipxe/src/usr/route_ipv6.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdio.h>
 #include <ipxe/netdevice.h>
diff --git a/qemu/roms/ipxe/src/usr/sync.c b/qemu/roms/ipxe/src/usr/sync.c
index f7a04c44c..f599588ae 100644
--- a/qemu/roms/ipxe/src/usr/sync.c
+++ b/qemu/roms/ipxe/src/usr/sync.c
@@ -15,9 +15,13 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
  */
 
-FILE_LICENCE ( GPL2_OR_LATER );
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stddef.h>
 #include <ipxe/job.h>
diff --git a/qemu/roms/ipxe/src/util/Option/ROM.pm b/qemu/roms/ipxe/src/util/Option/ROM.pm
index 6c396730e..232cf16b8 100644
--- a/qemu/roms/ipxe/src/util/Option/ROM.pm
+++ b/qemu/roms/ipxe/src/util/Option/ROM.pm
@@ -529,6 +529,26 @@ sub new {
   return $hash;  
 }
 
+sub device_list {
+  my $hash = shift;
+  my $self = tied(%$hash);
+
+  my $device_list = $hash->{device_list};
+  return undef unless $device_list;
+
+  my @ids;
+  my $offset = ( $self->{offset} + $device_list );
+  while ( 1 ) {
+    my $raw = substr ( ${$self->{data}}, $offset, 2 );
+    my $id = unpack ( "S", $raw );
+    last unless $id;
+    push @ids, $id;
+    $offset += 2;
+  }
+
+  return @ids;
+}
+
 ##############################################################################
 #
 # Option::ROM::PnP
diff --git a/qemu/roms/ipxe/src/util/disrom.pl b/qemu/roms/ipxe/src/util/disrom.pl
index 574957acd..920a86b24 100755
--- a/qemu/roms/ipxe/src/util/disrom.pl
+++ b/qemu/roms/ipxe/src/util/disrom.pl
@@ -55,6 +55,10 @@ do {
     printf "  %-16s %s\n", "Signature:", $pci->{signature};
     printf "  %-16s 0x%04x\n", "Vendor ID:", $pci->{vendor_id};
     printf "  %-16s 0x%04x\n", "Device ID:", $pci->{device_id};
+    if ( $pci->{device_list} ) {
+      printf "  %-16s %s\n", "Device list:",
+	     ( join ( ", ", map { sprintf "0x%04x", $_ } $pci->device_list ) );
+    }
     printf "  %-16s 0x%02x%02x%02x\n", "Device class:",
 	   $pci->{base_class}, $pci->{sub_class}, $pci->{prog_intf};
     printf "  %-16s 0x%04x (%d)\n", "Image length:",
diff --git a/qemu/roms/ipxe/src/util/elf2efi.c b/qemu/roms/ipxe/src/util/elf2efi.c
index 45d539574..e68fa5d14 100644
--- a/qemu/roms/ipxe/src/util/elf2efi.c
+++ b/qemu/roms/ipxe/src/util/elf2efi.c
@@ -478,11 +478,13 @@ static void process_reloc ( bfd *bfd __attribute__ (( unused )),
 		/* Skip absolute symbols; the symbol value won't
 		 * change when the object is loaded.
 		 */
+	} else if ( ( strcmp ( howto->name, "R_386_NONE" ) == 0 ) ||
+		    ( strcmp ( howto->name, "R_X86_64_NONE" ) == 0 ) ) {
+		/* Ignore dummy relocations used by REQUIRE_SYMBOL() */
 	} else if ( strcmp ( howto->name, "R_X86_64_64" ) == 0 ) {
 		/* Generate an 8-byte PE relocation */
 		generate_pe_reloc ( pe_reltab, offset, 8 );
-	} else if ( ( strcmp ( howto->name, "R_386_32" ) == 0 ) ||
-		    ( strcmp ( howto->name, "R_X86_64_32" ) == 0 ) ) {
+	} else if ( strcmp ( howto->name, "R_386_32" ) == 0 ) {
 		/* Generate a 4-byte PE relocation */
 		generate_pe_reloc ( pe_reltab, offset, 4 );
 	} else if ( strcmp ( howto->name, "R_386_16" ) == 0 ) {
diff --git a/qemu/roms/ipxe/src/util/licence.pl b/qemu/roms/ipxe/src/util/licence.pl
index 0e43c7b4c..79e70fd65 100755
--- a/qemu/roms/ipxe/src/util/licence.pl
+++ b/qemu/roms/ipxe/src/util/licence.pl
@@ -37,6 +37,7 @@ my $known_licences = {
     desc => "GPL version 2 (or, at your option, any later version)",
     can_subsume => {
       gpl_any => 1,
+      gpl2_or_later_or_ubdl => 1,
       public_domain => 1,
       bsd3 => 1,
       bsd2 => 1,
@@ -49,6 +50,7 @@ my $known_licences = {
     can_subsume => {
       gpl_any => 1,
       gpl2_or_later => 1,
+      gpl2_or_later_or_ubdl => 1,
       public_domain => 1,
       bsd3 => 1,
       bsd2 => 1,
@@ -56,6 +58,17 @@ my $known_licences = {
       isc  => 1,
     },
   },
+  gpl2_or_later_or_ubdl => {
+    desc => ( "GPL version 2 (or, at your option, any later version) or ".
+	      "Unmodified Binary Distribution Licence" ),
+    can_subsume => {
+      public_domain => 1,
+      bsd3 => 1,
+      bsd2 => 1,
+      mit => 1,
+      isc => 1,
+    },
+  },
   public_domain => {
     desc => "Public Domain",
     can_subsume => {},
diff --git a/qemu/roms/ipxe/src/util/parserom.pl b/qemu/roms/ipxe/src/util/parserom.pl
index e278e6336..28df60652 100755
--- a/qemu/roms/ipxe/src/util/parserom.pl
+++ b/qemu/roms/ipxe/src/util/parserom.pl
@@ -1,66 +1,260 @@
 #!/usr/bin/env perl
 #
-# Parse PCI_ROM and ISA_ROM entries from a source file on stdin and
-# output the relevant Makefile variable definitions to stdout
+# Parse PCI_ROM and ISA_ROM entries from source file(s) specified as
+# arguments and output the relevant Makefile rules to STDOUT.
 #
-# Based upon portions of Ken Yap's genrules.pl
+# Originally based on portions of Ken Yap's genrules.pl. Completely
+# rewritten by Robin Smidsrød to be more maintainable.
 
 use strict;
 use warnings;
+use Getopt::Long;
 
-die "Syntax: $0 driver_source.c" unless @ARGV == 1;
-my $source = shift;
-open DRV, "<$source" or die "Could not open $source: $!\n";
+# Parse command-line options
+my @exclude_driver_classes = ();
+my @exclude_drivers = ();
+my $debug = 0;
+my $help = 0;
+GetOptions(
+    "exclude-driver-class=s" => \@exclude_driver_classes,
+    "exclude-driver=s"       => \@exclude_drivers,
+    "debug"                  => \$debug,
+    "help"                   => \$help,
+);
 
-( my $family, my $driver_name ) = ( $source =~ /^(.*?([^\/]+))\..$/ )
-    or die "Could not parse source file name \"$source\"\n";
+# Convert exclution arrays to lookup tables
+my $exclude_driver_class_map = { map { $_ => 1 } @exclude_driver_classes };
+my $exclude_driver_map       = { map { $_ => 1 } @exclude_drivers        };
 
-my $printed_family;
+# Ensure STDOUT and STDERR are synchronized if debugging
+if ( $debug ) {
+    STDOUT->autoflush(1);
+    STDERR->autoflush(1);
+}
+
+# Compile regular expressions here for slight performance boost
+my %RE = (
+    'parse_driver_class'    => qr{ drivers/ (\w+?) / }x,
+    'parse_family'          => qr{^ (?:\./)? (.*) \..+? $}x,
+    'find_rom_line'         => qr/^ \s* ( (PCI|ISA)_ROM \s* \( \s* (.*?) ) $/x,
+    'extract_pci_id'        => qr/^ \s* 0x([0-9A-Fa-f]{4}) \s* ,? \s* (.*) $/x,
+    'extract_quoted_string' => qr/^ \s* \" ([^\"]*?) \" \s* ,? \s* (.*) $/x,
+);
+
+# Show help if required arguments are missing or help was requested
+show_usage_and_exit() if $help or @ARGV < 1;
+
+# Process each source file specified
+process_source_file($_) for @ARGV;
+
+exit;
+
+sub show_usage_and_exit {
+    print STDERR <<"EOM";
+Syntax: $0 [<options>] <source-file> [<source-file>]
+Options:
+    --exclude-driver-class Exclude specified driver classes
+    --exclude-driver       Exclude specified drivers
+    --debug                Output debug information on STDERR
+    --help                 This help information
+EOM
+    exit 1;
+}
+
+# Figure out if source file is a driver and look for ROM declarations
+sub process_source_file {
+    my ($source_file) = @_;
+    return unless defined $source_file;
+    return unless length $source_file;
+    my $state = { 'source_file' => $source_file };
+    log_debug("SOURCE_FILE", $state->{source_file});
+    # Skip source files that aren't drivers
+    parse_driver_class( $state );
+    unless ( $state->{'driver_class'} ) {
+        log_debug("SKIP_NOT_DRIVER", $state->{source_file} );
+        return;
+    }
+    # Skip source files with driver classes that are explicitly excluded
+    if ( $exclude_driver_class_map->{ $state->{'driver_class'} } ) {
+        log_debug("SKIP_EXCL_CLASS", $state->{'driver_class'} );
+        return;
+    }
+    # Skip source files without driver information
+    parse_family( $state );
+    parse_driver_name( $state );
+    unless ( $state->{'family'} and $state->{'driver_name'} ) {
+        log_debug("SKIP_NO_DRV_INFO", $state->{source_file} );
+        return;
+    }
+    # Skip source files with drivers that are explicitly excluded
+    if ( $exclude_driver_map->{ $state->{'driver_name'} } ) {
+        log_debug("SKIP_EXCL_DRV", $state->{'driver_name'} );
+        return;
+    }
+    # Iterate through lines in source files looking for ROM declarations
+    # and # output Makefile rules
+    open( my $fh, "<", $state->{'source_file'} )
+        or die "Couldn't open $state->{source_file}: $!\n";
+    while (<$fh>) {
+        process_rom_decl($state, $1, $2, $3) if m/$RE{find_rom_line}/;
+    }
+    close($fh) or die "Couldn't close $source_file: $!\n";
+    return 1;
+}
+
+# Verify that the found ROM declaration is sane and dispatch to the right
+# handler depending on type
+sub process_rom_decl {
+    my ($state, $rom_line, $rom_type, $rom_decl) = @_;
+    return unless defined $rom_line;
+    return unless length $rom_line;
+    log_debug("ROM_LINE", $rom_line);
+    return unless defined $rom_type;
+    return unless length $rom_type;
+    log_debug("ROM_TYPE", $rom_type);
+    $state->{'type'} = lc $rom_type;
+    return process_pci_rom($state, $rom_decl) if $rom_type eq "PCI";
+    return process_isa_rom($state, $rom_decl) if $rom_type eq "ISA";
+    return;
+}
+
+# Extract values from PCI_ROM declaration lines and dispatch to
+# Makefile rule generator
+sub process_pci_rom {
+    my ($state, $decl) = @_;
+    return unless defined $decl;
+    return unless length $decl;
+    (my $vendor, $decl) = extract_pci_id($decl,        'PCI_VENDOR');
+    (my $device, $decl) = extract_pci_id($decl,        'PCI_DEVICE');
+    (my $image,  $decl) = extract_quoted_string($decl, 'IMAGE');
+    (my $desc,   $decl) = extract_quoted_string($decl, 'DESCRIPTION');
+    if ( $vendor and $device and $image and $desc ) {
+        print_make_rules( $state, "${vendor}${device}", $desc, $vendor, $device );
+        print_make_rules( $state, $image, $desc, $vendor, $device, 1 );
+    }
+    else {
+        log_debug("WARNING", "Malformed PCI_ROM macro on line $. of $state->{source_file}");
+    }
+    return 1;
+}
+
+# Extract values from ISA_ROM declaration lines and dispatch to
+# Makefile rule generator
+sub process_isa_rom {
+    my ($state, $decl) = @_;
+    return unless defined $decl;
+    return unless length $decl;
+    (my $image, $decl) = extract_quoted_string($decl, 'IMAGE');
+    (my $desc,  $decl) = extract_quoted_string($decl, 'DESCRIPTION');
+    if ( $image and $desc ) {
+        print_make_rules( $state, $image, $desc );
+    }
+    else {
+        log_debug("WARNING", "Malformed ISA_ROM macro on line $. of $state->{source_file}");
+    }
+    return 1;
+}
 
-sub rom {
-  ( my $type, my $image, my $desc, my $vendor, my $device, my $dup ) = @_;
-  my $ids = $vendor ? "$vendor,$device" : "-";
-  unless ( $printed_family ) {
+# Output Makefile rules for the specified ROM declarations
+sub print_make_rules {
+    my ( $state, my $image, my $desc, my $vendor, my $device, my $dup ) = @_;
+    unless ( $state->{'is_header_printed'} ) {
+        print "# NIC\t\n";
+        print "# NIC\tfamily\t$state->{family}\n";
+        print "DRIVERS_$state->{driver_class} += $state->{driver_name}\n";
+        print "DRIVERS += $state->{driver_name}\n";
+        print "\n";
+        $state->{'is_header_printed'} = 1;
+    }
+    return if $vendor and ( $vendor eq "ffff" or $device eq "ffff" );
+    my $ids = $vendor ? "$vendor,$device" : "-";
+    print "# NIC\t$image\t$ids\t$desc\n";
+    print "DRIVER_$image = $state->{driver_name}\n";
+    print "ROM_TYPE_$image = $state->{type}\n";
+    print "ROM_DESCRIPTION_$image = \"$desc\"\n";
+    print "PCI_VENDOR_$image = 0x$vendor\n" if $vendor;
+    print "PCI_DEVICE_$image = 0x$device\n" if $device;
+    print "ROMS += $image\n" unless $dup;
+    print "ROMS_$state->{driver_name} += $image\n" unless $dup;
     print "\n";
-    print "# NIC\t\n";
-    print "# NIC\tfamily\t$family\n";
-    print "DRIVERS += $driver_name\n";
-    $printed_family = 1;
-  }
-  print "\n";
-  return if ( $vendor && ( ( $vendor eq "ffff" ) || ( $device eq "ffff" ) ) );
-  print "# NIC\t$image\t$ids\t$desc\n";
-  print "DRIVER_$image = $driver_name\n";
-  print "ROM_TYPE_$image = $type\n";
-  print "ROM_DESCRIPTION_$image = \"$desc\"\n";
-  print "PCI_VENDOR_$image = 0x$vendor\n" if $vendor;
-  print "PCI_DEVICE_$image = 0x$device\n" if $device;
-  print "ROMS += $image\n" unless $dup;
-  print "ROMS_$driver_name += $image\n" unless $dup;
+    return 1;
+}
+
+# Driver class is whatever comes after the "drivers" part of the filename (relative path)
+sub parse_driver_class {
+    my ($state) = @_;
+    my $filename = $state->{'source_file'};
+    return unless defined $filename;
+    return unless length $filename;
+    if ( $filename =~ m/$RE{parse_driver_class}/ ) {
+        log_debug("DRIVER_CLASS", $1);
+        $state->{'driver_class'} = $1;
+    }
+    return;
+}
+
+# Family name is filename (relative path) without extension
+sub parse_family {
+    my ($state) = @_;
+    my $filename = $state->{'source_file'};
+    return unless defined $filename;
+    return unless length $filename;
+    if ( $filename =~ m/$RE{parse_family}/ ) {
+        log_debug("FAMILY", $1);
+        $state->{'family'} = $1;
+    }
+    return;
+}
+
+# Driver name is last part of family name
+sub parse_driver_name {
+    my ($state) = @_;
+    my $family = $state->{'family'};
+    return unless defined $family;
+    return unless length $family;
+    my @parts = split "/", $family;
+    $state->{'driver_name'} = $parts[-1];
+    log_debug("DRIVER", $state->{'driver_name'});
+    return;
 }
 
-while ( <DRV> ) {
-  next unless /(PCI|ISA)_ROM\s*\(/;
-
-  if ( /^\s*PCI_ROM\s*\(
-         \s*0x([0-9A-Fa-f]{4})\s*, # PCI vendor
-         \s*0x([0-9A-Fa-f]{4})\s*, # PCI device
-         \s*\"([^\"]*)\"\s*,	   # Image
-         \s*\"([^\"]*)\"\s*,	   # Description
-         \s*.*\s*		   # Driver data
-       \)/x ) {
-    ( my $vendor, my $device, my $image, my $desc ) = ( lc $1, lc $2, $3, $4 );
-    rom ( "pci", lc "${vendor}${device}", $desc, $vendor, $device );
-    rom ( "pci", $image, $desc, $vendor, $device, 1 );
-  } elsif ( /^\s*ISA_ROM\s*\(
-	      \s*\"([^\"]*)\"\s*,  # Image
-	      \s*\"([^\"]*)\"\s*   # Description
-	    \)/x ) {
-    ( my $image, my $desc ) = ( $1, $2 );
-    rom ( "isa", $image, $desc );
-  } else {
-    warn "Malformed PCI_ROM or ISA_ROM macro on line $. of $source\n";
-  }
+# Extract a PCI vendor/device ID e.g. 0x8086, possibly followed by a comma
+# Should always be 4-digit lower-case hex number
+sub extract_pci_id {
+    my ($str, $label) = @_;
+    return "", $str unless defined $str;
+    return "", $str unless length $str;
+    if ( $str =~ m/$RE{extract_pci_id}/ ) {
+        my $id = lc $1;
+        log_debug($label, $id);
+        return $id, $2;
+    }
+    return "", $str;
 }
 
-close DRV;
+# Extract a double-quoted string, possibly followed by a comma
+sub extract_quoted_string {
+    my ($str, $label) = @_;
+    return "", $str unless defined $str;
+    return "", $str unless length $str;
+    if ( $str =~ m/$RE{extract_quoted_string}/ ) {
+        log_debug($label, $1);
+        return $1, $2;
+    }
+    return "", $str;
+}
+
+# Output debug info to STDERR (off by default)
+sub log_debug {
+    my ($label, $str) = @_;
+    return unless $debug;
+    return unless defined $str;
+    print STDERR "\n" if $label eq 'SOURCE_FILE';
+    print STDERR "=";
+    if ( defined $label ) {
+        my $pad_count = 16 - length $label;
+        print STDERR $label . ":" . ( " " x $pad_count );
+    }
+    print STDERR $str . "\n";
+    return;
+}
diff --git a/qemu/roms/ipxe/src/util/relicense.pl b/qemu/roms/ipxe/src/util/relicense.pl
new file mode 100755
index 000000000..41954c1b3
--- /dev/null
+++ b/qemu/roms/ipxe/src/util/relicense.pl
@@ -0,0 +1,169 @@
+#!/usr/bin/perl -w
+
+=head1 NAME
+
+relicense.pl
+
+=head1 SYNOPSIS
+
+relicense.pl [options] -p <permissions file> <file> [<file>...]
+
+Option:
+
+    -p,--permitted=FILE	Specify file of emails with relicensing permission
+    -f,--force		Manually force relicensing
+    -h,--help		Display brief help message
+    -v,--verbose	Increase verbosity
+    -q,--quiet		Decrease verbosity
+
+=cut
+
+use File::Slurp;
+use IPC::Run qw ( run );
+use Getopt::Long;
+use Pod::Usage;
+use strict;
+use warnings;
+
+# Parse command-line options
+my $verbosity = 0;
+my $permfile;
+my $force;
+Getopt::Long::Configure ( "bundling", "auto_abbrev" );
+GetOptions (
+  'permitted|p=s' => \$permfile,
+  'force|f' => \$force,
+  'verbose|v+' => sub { $verbosity++; },
+  'quiet|q+' => sub { $verbosity--; },
+  'help|h' => sub { pod2usage ( 1 ); },
+) or die "Could not parse command-line options";
+pod2usage ( 1 ) unless @ARGV;
+
+# Read permitted emails file
+my @emails = ( $permfile ? read_file ( $permfile ) : () );
+chomp @emails;
+my $permitted = { map { /^.*<(\S+)>$/; ( $1 || $_ ) => 1 } @emails };
+
+# Define list of relicensable licences
+my $relicensable = {
+  GPL2_OR_LATER => 1,
+};
+
+# Define blurb to be added to copyright notice
+my $blurb = '
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.';
+
+# Process files
+my @succeeded;
+my @failed;
+while ( my $filename = shift @ARGV ) {
+
+  # Read file to determine existing licence
+  my $file = read_file ( $filename );
+  my @licences = ( $file =~ /^\s*FILE_LICENCE\s*\(\s*(\S+)\s*\)\s*;?$/mg );
+  die "No licence declaration in $filename\n" unless @licences;
+  die "Multiple licence declarations in $filename\n" if @licences > 1;
+  my $licence = $licences[0];
+
+  # Skip if file is already UBDL-licensed
+  next if $licence =~ /_OR_UBDL$/;
+
+  # Fail immediately if file is not a candidate for relicensing
+  if ( ! exists $relicensable->{$licence} ) {
+    print "Non-relicensable licence $licence in $filename\n";
+    push @failed, $filename;
+    next;
+  }
+
+  # Run git-blame
+  my $stdout;
+  my $stderr;
+  run [ "git", "blame", "-M", "-C", "-p", "-w", $filename ],
+      \undef, \$stdout, \$stderr
+      or die "git-blame $filename: $?";
+  die $stderr if $stderr;
+
+  # Process output
+  my @stdout = split ( /\n/, $stdout );
+  chomp @stdout;
+  my $details = {};
+  my $failures = 0;
+  while ( @stdout ) {
+
+    # Parse output
+    my $commit_line = shift @stdout;
+    ( my $commit, undef, my $lineno, undef, my $count ) =
+	( $commit_line =~
+	  /^([0-9a-f]{40})\s+([0-9]+)\s+([0-9]+)(\s+([0-9]+))?$/ )
+	or die "Malformed commit line \"$commit_line\"\n";
+    if ( $count ) {
+      $details->{$commit} ||= {};
+      while ( ! ( $stdout[0] =~ /^\t/ ) ) {
+	my $detail_line = shift @stdout;
+	( my $key, undef, my $value ) =
+	    ( $detail_line =~ /^([a-z-]+)(\s+(.+))?$/ )
+	    or die "Malformed detail line \"$detail_line\" for $commit_line\n";
+	$details->{$commit}->{$key} = $value;
+      }
+    }
+    die "Missing commit details for $commit_line\n"
+	unless %{$details->{$commit}};
+    my $code_line = shift @stdout;
+    ( my $line ) = ( $code_line =~ /^\t(.*)$/ )
+	or die "Malformed code line \"$code_line\" for $commit_line\n";
+
+    # Skip trivial lines and lines so common that they are likely to
+    # be misattributed by git-blame
+    next if $line =~ /^\s*$/;		# Empty lines
+    next if $line =~ /^\s*\/\*/;	# Start of comments
+    next if $line =~ /^\s*\*/;		# Middle (or end) of comments
+    next if $line =~ /^\s*\{\s*$/;	# Standalone opening braces
+    next if $line =~ /^\s*\};?\s*$/;	# Standalone closing braces
+    next if $line =~ /^\#include/;	# Header inclusions
+    next if $line =~ /^\s*return\s+0;/;	# return 0;
+    next if $line =~ /^\s*return\s+rc;/; # return rc;
+    next if $line =~ /^\s*PCI_ROM\s*\(.*\)\s*,\s*$/;	# PCI IDs
+    next if $line =~ /^\s*FILE_LICENCE\s*\(.*\)\s*;$/; # Licence declarations
+
+    # Identify author
+    my $author_mail = $details->{$commit}->{"author-mail"}
+    or die "Missing author email for $commit_line\n";
+    ( my $email ) = ( $author_mail =~ /^<(\S+)>$/ )
+	or die "Malformed author email \"$author_mail\" for $commit_line\n";
+    undef $email if exists $details->{$commit}->{boundary};
+
+    # Check for relicensing permission
+    next if defined $email && exists $permitted->{$email};
+
+    # Print out lines lacking permission
+    printf $filename."\n" unless $failures;
+    printf "%4d %-30s %s\n", $lineno, ( $email || "<root>" ), $line;
+    $failures++;
+  }
+
+  # Fail if there are any non-trivial lines lacking relicensing permission
+  if ( $failures && ! $force ) {
+    push @failed, $filename;
+    next;
+  }
+
+  # Modify FILE_LICENCE() line
+  $file =~ s/(^\s*FILE_LICENCE\s*\(\s*${licence})(\s*\)\s*;?$)/$1_OR_UBDL$2/m
+      or die "Could not modify FILE_LICENCE() in $filename\n";
+
+  # Modify copyright notice, if present
+  if ( $file =~ /GNU General Public License/i ) {
+    $file =~ s/(02110-1301, USA.$)/$1${blurb}/m
+	or die "Could not modify copyright notice in $filename\n";
+  }
+
+  # Write out modified file
+  write_file ( $filename, { atomic => 1 }, $file );
+  push @succeeded, $filename;
+}
+
+print "Relicensed: ".join ( " ", @succeeded )."\n" if @succeeded;
+die "Cannot relicense: ".join ( " ", @failed )."\n" if @failed;
diff --git a/qemu/roms/ipxe/src/util/zbin.c b/qemu/roms/ipxe/src/util/zbin.c
index 3b7cf95b3..1862a3827 100644
--- a/qemu/roms/ipxe/src/util/zbin.c
+++ b/qemu/roms/ipxe/src/util/zbin.c
@@ -1,13 +1,21 @@
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
 #include <sys/stat.h>
-
-#define ENCODE
-#define VERBOSE
-#include "nrv2b.c"
-FILE *infile, *outfile;
+#include <lzma.h>
 
 #define DEBUG 0
 
+/* LZMA filter choices.  Must match those used by unlzma.S */
+#define LZMA_LC 2
+#define LZMA_LP 0
+#define LZMA_PB 0
+
+/* LZMA preset choice.  This is a policy decision */
+#define LZMA_PRESET ( LZMA_PRESET_DEFAULT | LZMA_PRESET_EXTREME )
+
 struct input_file {
 	void *buf;
 	size_t len;
@@ -177,13 +185,75 @@ static int process_zinfo_copy ( struct input_file *input,
 	return 0;
 }
 
+#define OPCODE_CALL 0xe8
+#define OPCODE_JMP 0xe9
+
+static void bcj_filter ( void *data, size_t len ) {
+	struct {
+		uint8_t opcode;
+		int32_t target;
+	} __attribute__ (( packed )) *jump;
+	ssize_t limit = ( len - sizeof ( *jump ) );
+	ssize_t offset;
+
+	/* liblzma does include an x86 BCJ filter, but it's hideously
+	 * convoluted and undocumented.  This BCJ filter is
+	 * substantially simpler and achieves the same compression (at
+	 * the cost of requiring the decompressor to know the size of
+	 * the decompressed data, which we already have in iPXE).
+	 */
+	for ( offset = 0 ; offset <= limit ; offset++ ) {
+		jump = ( data + offset );
+
+		/* Skip instructions that are not followed by a rel32 address */
+		if ( ( jump->opcode != OPCODE_CALL ) &&
+		     ( jump->opcode != OPCODE_JMP ) )
+			continue;
+
+		/* Convert rel32 address to an absolute address.  To
+		 * avoid false positives (which damage the compression
+		 * ratio), we should check that the jump target is
+		 * within the range [0,limit).
+		 *
+		 * Some output values would then end up being mapped
+		 * from two distinct input values, making the
+		 * transformation irreversible.  To solve this, we
+		 * transform such values back into the part of the
+		 * range which would otherwise correspond to no input
+		 * values.
+		 */
+		if ( ( jump->target >= -offset ) &&
+		     ( jump->target < ( limit - offset ) ) ) {
+			/* Convert relative addresses in the range
+			 * [-offset,limit-offset) to absolute
+			 * addresses in the range [0,limit).
+			 */
+			jump->target += offset;
+		} else if ( ( jump->target >= ( limit - offset ) ) &&
+			    ( jump->target < limit ) ) {
+			/* Convert positive numbers in the range
+			 * [limit-offset,limit) to negative numbers in
+			 * the range [-offset,0).
+			 */
+			jump->target -= limit;
+		}
+		offset += sizeof ( jump->target );
+	};
+}
+
 static int process_zinfo_pack ( struct input_file *input,
 				struct output_file *output,
 				union zinfo_record *zinfo ) {
 	struct zinfo_pack *pack = &zinfo->pack;
 	size_t offset = pack->offset;
 	size_t len = pack->len;
-	unsigned long packed_len;
+	size_t packed_len = 0;
+	size_t remaining = ( output->max_len - output->len );
+	lzma_options_lzma options;
+	const lzma_filter filters[] = {
+		{ .id = LZMA_FILTER_LZMA1, .options = &options },
+		{ .id = LZMA_VLI_UNKNOWN }
+	};
 
 	if ( ( offset + len ) > input->len ) {
 		fprintf ( stderr, "Input buffer overrun on pack\n" );
@@ -196,9 +266,15 @@ static int process_zinfo_pack ( struct input_file *input,
 		return -1;
 	}
 
-	if ( ucl_nrv2b_99_compress ( ( input->buf + offset ), len,
-				     ( output->buf + output->len ),
-				     &packed_len, 0 ) != UCL_E_OK ) {
+	bcj_filter ( ( input->buf + offset ), len );
+
+	lzma_lzma_preset ( &options, LZMA_PRESET );
+	options.lc = LZMA_LC;
+	options.lp = LZMA_LP;
+	options.pb = LZMA_PB;
+	if ( lzma_raw_buffer_encode ( filters, NULL, ( input->buf + offset ),
+				      len, ( output->buf + output->len ),
+				      &packed_len, remaining ) != LZMA_OK ) {
 		fprintf ( stderr, "Compression failure\n" );
 		return -1;
 	}
@@ -206,7 +282,7 @@ static int process_zinfo_pack ( struct input_file *input,
 	if ( DEBUG ) {
 		fprintf ( stderr, "PACK [%#zx,%#zx) to [%#zx,%#zx)\n",
 			  offset, ( offset + len ), output->len,
-			  ( size_t )( output->len + packed_len ) );
+			  ( output->len + packed_len ) );
 	}
 
 	output->len += packed_len;
diff --git a/qemu/roms/openbios/Makefile.target b/qemu/roms/openbios/Makefile.target
index a7363e667..4c54105a7 100644
--- a/qemu/roms/openbios/Makefile.target
+++ b/qemu/roms/openbios/Makefile.target
@@ -15,7 +15,7 @@ HOSTCFLAGS+= -Wstrict-aliasing -Wwrite-strings -Wmissing-prototypes -Wnested-ext
 HOSTCFLAGS+= -W
 # Flags for dependency generation
 HOSTCFLAGS+= -MMD -MP -MT $@ -MF '$(*D)/$(*F).d'
-HOSTINCLUDES := -I$(SRCDIR)/include -I$(SRCDIR)/kernel -I$(SRCDIR)/kernel/include -I$(ODIR)/target/include
+HOSTINCLUDES := -I$(SRCDIR)/include -I$(SRCDIR)/kernel -I$(SRCDIR)/kernel/include -iquote $(ODIR)/target/include
 
 CC     := $(TARGET)gcc
 AS     := $(TARGET)as
diff --git a/qemu/roms/openbios/arch/ppc/qemu/init.c b/qemu/roms/openbios/arch/ppc/qemu/init.c
index 4fe8b7220..b76c5706f 100644
--- a/qemu/roms/openbios/arch/ppc/qemu/init.c
+++ b/qemu/roms/openbios/arch/ppc/qemu/init.c
@@ -302,6 +302,11 @@ cpu_generic_init(const struct cpudef *cpu)
     fword("encode-string");
     push_str("state");
     fword("property");
+
+    PUSH(0x20);
+    fword("encode-int");
+    push_str("reservation-granule-size");
+    fword("property");
 }
 
 static void
@@ -596,6 +601,11 @@ go(void)
 {
     ucell addr;
 
+    /* Insert copyright property for MacOS 9 and below */
+    if (find_dev("/rom/macos")) {
+        fword("insert-copyright-property");
+    }
+    
     feval("saved-program-state >sps.entry @");
     addr = POP();
 
@@ -680,6 +690,60 @@ static void ffilll(void)
     }   
 }
 
+/*
+ * adler32        ( adler buf len -- checksum )
+ *
+ * Adapted from Mark Adler's original implementation (zlib license)
+ *
+ * Both OS 9 and BootX require this word for payload validation.
+ */
+
+#define DO1(buf,i)  {s1 += buf[i]; s2 += s1;}
+#define DO2(buf,i)  DO1(buf,i); DO1(buf,i+1);
+#define DO4(buf,i)  DO2(buf,i); DO2(buf,i+2);
+#define DO8(buf,i)  DO4(buf,i); DO4(buf,i+4);
+#define DO16(buf)   DO8(buf,0); DO8(buf,8);
+
+static void adler32(void)
+{
+    uint32_t len = (uint32_t)POP();
+    char *buf = (char *)POP();
+    uint32_t adler = (uint32_t)POP();
+
+    if (buf == NULL) {
+        RET(-1);
+    }
+
+    uint32_t base = 65521;
+    uint32_t nmax = 5552;
+
+    uint32_t s1 = adler & 0xffff;
+    uint32_t s2 = (adler >> 16) & 0xffff;
+
+    uint32_t k;
+    while (len > 0) {
+        k = (len < nmax ? len : nmax);
+        len -= k;
+
+        while (k >= 16) {
+            DO16(buf);
+            buf += 16;
+            k -= 16;
+        }
+        if (k != 0) {
+            do {
+                s1 += *buf++;
+                s2 += s1;
+            } while (--k);
+        }
+
+        s1 %= base;
+        s2 %= base;
+    }
+
+    RET(s2 << 16 | s1);
+}
+
 void
 arch_of_init(void)
 {
@@ -945,6 +1009,9 @@ arch_of_init(void)
 
     /* Implementation of filll word (required by BootX) */
     bind_func("filll", ffilll);
+
+    /* Implementation of adler32 word (required by OS 9, BootX) */
+    bind_func("(adler32)", adler32);
     
     bind_func("platform-boot", boot);
     bind_func("(go)", go);
diff --git a/qemu/roms/openbios/arch/ppc/qemu/methods.c b/qemu/roms/openbios/arch/ppc/qemu/methods.c
index fd993daa9..930b47c4e 100644
--- a/qemu/roms/openbios/arch/ppc/qemu/methods.c
+++ b/qemu/roms/openbios/arch/ppc/qemu/methods.c
@@ -114,6 +114,8 @@ static void
 ciface_quiesce( unsigned long args[], unsigned long ret[] )
 {
 	usb_exit();
+
+	ob_ide_quiesce();
 #if 0
 	unsigned long msr;
 	/* This seems to be the correct thing to do - but I'm not sure */
@@ -164,21 +166,21 @@ DECLARE_UNNAMED_NODE( mmu, INSTALL_OPEN, 0 );
 DECLARE_NODE( mmu_ciface, 0, 0, "+/openprom/client-services" );
 
 
-/* ( phys size align --- base ) */
+/* ( [phys] size align --- base ) */
 static void
 mem_claim( void )
 {
 	ucell align = POP();
 	ucell size = POP();
-	ucell phys = POP();
-	ucell ret = ofmem_claim_phys( phys, size, align );
+	phys_addr_t phys = -1;
 
-	if( ret == -1 ) {
-		printk("MEM: claim failure\n");
-		throw( -13 );
-		return;
+	if (!align) {
+		phys = POP();
 	}
-	PUSH( ret );
+
+	phys = ofmem_claim_phys(phys, size, align);
+
+	PUSH(phys);
 }
 
 /* ( phys size --- ) */
@@ -188,24 +190,24 @@ mem_release( void )
 	POP(); POP();
 }
 
-/* ( phys size align --- base ) */
+/* ( [virt] size align --- base ) */
 static void
 mmu_claim( void )
 {
 	ucell align = POP();
 	ucell size = POP();
-	ucell phys = POP();
-	ucell ret = ofmem_claim_virt( phys, size, align );
+	ucell virt = -1;
 
-	if( ret == -1 ) {
-		printk("MMU: CLAIM failure\n");
-		throw( -13 );
-		return;
+	if (!align) {
+		virt = POP();
 	}
-	PUSH( ret );
+
+	virt = ofmem_claim_virt(virt, size, align);
+
+	PUSH(virt);
 }
 
-/* ( phys size --- ) */
+/* ( virt size --- ) */
 static void
 mmu_release( void )
 {
diff --git a/qemu/roms/openbios/arch/ppc/qemu/qemu.fs b/qemu/roms/openbios/arch/ppc/qemu/qemu.fs
index 458af1bc7..3d99a34a1 100644
--- a/qemu/roms/openbios/arch/ppc/qemu/qemu.fs
+++ b/qemu/roms/openbios/arch/ppc/qemu/qemu.fs
@@ -93,3 +93,48 @@ variable keyboard-phandle 0 keyboard-phandle !
 :noname
   set-defaults
 ; PREPOST-initializer
+
+\ -------------------------------------------------------------------------
+\ copyright property handling
+\ -------------------------------------------------------------------------
+
+: insert-copyright-property
+  \ As required for MacOS 9 and below
+  " Pbclevtug 1983-2001 Nccyr Pbzchgre, Vap. GUVF ZRFFNTR SBE PBZCNGVOVYVGL BAYL"
+  rot13-str encode-string " copyright"
+  " /" find-package if
+    " set-property" $find if
+      execute
+    else
+      3drop drop
+    then
+  then
+;
+
+: delete-copyright-property
+  \ Remove copyright property created above
+  active-package
+  " /" find-package if
+      active-package!
+      " copyright" delete-property
+  then
+  active-package!
+;
+
+: (exit)
+  \ Clean up before returning to the interpreter
+  delete-copyright-property
+;
+
+\ -------------------------------------------------------------------------
+\ Adler-32 wrapper
+\ -------------------------------------------------------------------------
+
+: adler32 ( adler buf len -- checksum )
+  " (adler32)" $find if
+    execute
+  else
+    ." Can't find " ( adler32-name ) type cr
+    3drop 0
+  then
+;
diff --git a/qemu/roms/openbios/arch/ppc/qemu/tree.fs b/qemu/roms/openbios/arch/ppc/qemu/tree.fs
index 1ed838397..5b6bbc6f7 100644
--- a/qemu/roms/openbios/arch/ppc/qemu/tree.fs
+++ b/qemu/roms/openbios/arch/ppc/qemu/tree.fs
@@ -42,6 +42,14 @@ new-device
 	: close ;
 finish-device
 
+new-device
+	" rom" device-name
+	h# ff800000 encode-int 0 encode-int encode+ " reg" property
+	1 encode-int " #address-cells" property
+	h# ff800000 encode-int h# 800000 encode-int encode+
+	h# ff800000 encode-int encode+ " ranges" property
+finish-device
+
 \ -------------------------------------------------------------
 \ /packages
 \ -------------------------------------------------------------
diff --git a/qemu/roms/openbios/arch/sparc64/call-client.S b/qemu/roms/openbios/arch/sparc64/call-client.S
index f365e3cb1..a8c0348e4 100644
--- a/qemu/roms/openbios/arch/sparc64/call-client.S
+++ b/qemu/roms/openbios/arch/sparc64/call-client.S
@@ -1,3 +1,5 @@
+#include "cpustate.h"
+
 	.globl	sparc64_of_client_interface, client_tba
 
 
@@ -9,151 +11,9 @@
  * behaviour of OBP.
  */
 
-#define SAVE_WINDOW_STATE(type) \
-	setx	client_window, %g6, %g1; \
-	rdpr	%cwp, %g7; \
-	stx	%g7, [%g1]; \
-	rdpr	%cansave, %g7; \
-	stx	%g7, [%g1 + 0x8]; \
-	rdpr	%canrestore, %g7; \
-	stx	%g7, [%g1 + 0x10]; \
-	rdpr	%otherwin, %g7; \
-	stx	%g7, [%g1 + 0x18]; \
-	rdpr	%wstate, %g7; \
-	stx	%g7, [%g1 + 0x20]; \
-	rdpr	%cleanwin, %g7; \
-	stx	%g7, [%g1 + 0x28]; \
-	\
-	stx	%o0, [%g1 + 0x30]; \
-	stx	%o1, [%g1 + 0x38]; \
-	stx	%o2, [%g1 + 0x40]; \
-	stx	%o3, [%g1 + 0x48]; \
-	stx	%o4, [%g1 + 0x50]; \
-	stx	%o5, [%g1 + 0x58]; \
-	stx	%o6, [%g1 + 0x60]; \
-	stx	%o7, [%g1 + 0x68]; \
-	\
-	rdpr	%pstate, %g7; \
-	stx	%g7, [%g1 + 0x70]; \
-	rd	%y, %g7; \
-	stx	%g7, [%g1 + 0x78]; \
-	rd	%fprs, %g7; \
-	stx	%g7, [%g1 + 0x80]; \
-	\
-	/* Now iterate through all of the windows saving all l and i registers */ \
-	add	%g1, 0x90, %g5; \
-	\
-	/* Get the number of windows in %g6 */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	inc	%g6; \
-	\
-save_cpu_window_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %cwp; \
-	stx	%l0, [%g5]; \
-	stx	%l1, [%g5 + 0x8]; \
-	stx	%l2, [%g5 + 0x10]; \
-	stx	%l3, [%g5 + 0x18]; \
-	stx	%l4, [%g5 + 0x20]; \
-	stx	%l5, [%g5 + 0x28]; \
-	stx	%l6, [%g5 + 0x30]; \
-	stx	%l7, [%g5 + 0x38]; \
-	stx	%i0, [%g5 + 0x40]; \
-	stx	%i1, [%g5 + 0x48]; \
-	stx	%i2, [%g5 + 0x50]; \
-	stx	%i3, [%g5 + 0x58]; \
-	stx	%i4, [%g5 + 0x60]; \
-	stx	%i5, [%g5 + 0x68]; \
-	stx	%i6, [%g5 + 0x70]; \
-	stx	%i7, [%g5 + 0x78]; \
-	bne	save_cpu_window_##type; \
-	 add	%g5, 0x80, %g5; \
-	\
-	/* For 8 windows with 16 registers to save in the window, memory required \
-	is 16*8*8 = 0x400 bytes */ \
-	\
-	/* Now we should be in window 0 so update the other window registers */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	dec	%g6; \
-	wrpr	%g6, %cansave; \
-	\
-	wrpr	%g0, %cleanwin; \
-	wrpr	%g0, %canrestore; \
-	wrpr	%g0, %otherwin;
-
-
-#define RESTORE_WINDOW_STATE(type) \
-	setx	client_window, %g6, %g1; \
-	\
-	/* Get the number of windows in %g6 */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	inc	%g6; \
-	\
-	/* Now iterate through all of the windows restoring all l and i registers */ \
-	add	%g1, 0x90, %g5; \
-	\
-restore_cpu_window_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %cwp; \
-	ldx	[%g5], %l0; \
-	ldx	[%g5 + 0x8], %l1; \
-	ldx	[%g5 + 0x10], %l2; \
-	ldx	[%g5 + 0x18], %l3; \
-	ldx	[%g5 + 0x20], %l4; \
-	ldx	[%g5 + 0x28], %l5; \
-	ldx	[%g5 + 0x30], %l6; \
-	ldx	[%g5 + 0x38], %l7; \
-	ldx	[%g5 + 0x40], %i0; \
-	ldx	[%g5 + 0x48], %i1; \
-	ldx	[%g5 + 0x50], %i2; \
-	ldx	[%g5 + 0x58], %i3; \
-	ldx	[%g5 + 0x60], %i4; \
-	ldx	[%g5 + 0x68], %i5; \
-	ldx	[%g5 + 0x70], %i6; \
-	ldx	[%g5 + 0x78], %i7; \
-	bne	restore_cpu_window_##type; \
-	 add	%g5, 0x80, %g5; \
-	\
-	/* Restore the window registers to their original value */ \
-	ldx	[%g1], %g7; \
-	wrpr	%g7, %cwp; \
-	ldx	[%g1 + 0x8], %g7; \
-	wrpr	%g7, %cansave; \
-	ldx	[%g1 + 0x10], %g7; \
-	wrpr	%g7, %canrestore; \
-	ldx	[%g1 + 0x18], %g7; \
-	wrpr	%g7, %otherwin; \
-	ldx	[%g1 + 0x20], %g7; \
-	wrpr	%g7, %wstate; \
-	ldx	[%g1 + 0x28], %g7; \
-	wrpr	%g7, %cleanwin; \
-	\
-	ldx	[%g1 + 0x30], %o0; \
-	ldx	[%g1 + 0x38], %o1; \
-	ldx	[%g1 + 0x40], %o2; \
-	ldx	[%g1 + 0x48], %o3; \
-	ldx	[%g1 + 0x50], %o4; \
-	ldx	[%g1 + 0x58], %o5; \
-	ldx	[%g1 + 0x60], %o6; \
-	ldx	[%g1 + 0x68], %o7; \
-	\
-	ldx	[%g1 + 0x70], %g7; \
-	wrpr	%g7, %pstate; \
-	ldx	[%g1 + 0x78], %g7; \
-	wr	%g7, 0, %y; \
-	ldx	[%g1 + 0x80], %g7; \
-	wr	%g7, 0, %fprs
-
-
 	.data
 	.align	8
 
-	.skip	16384
-openbios_stack:
-
 client_stack:
 	.xword	0
 client_tba:
@@ -176,15 +36,15 @@ client_window:
 sparc64_of_client_interface:
 
 	/* Save globals on callers stack */
-	add	%sp, -56, %sp
+	add	%sp, -248, %sp
 
-	stx	%g1, [%sp + 2047 + 0]
-	stx	%g2, [%sp + 2047 + 8]
-	stx	%g3, [%sp + 2047 + 16]
-	stx	%g4, [%sp + 2047 + 24]
-	stx	%g5, [%sp + 2047 + 32]
-	stx	%g6, [%sp + 2047 + 40]
-	stx	%g7, [%sp + 2047 + 48]
+	stx	%g1, [%sp + 2047 + 192]
+	stx	%g2, [%sp + 2047 + 200]
+	stx	%g3, [%sp + 2047 + 208]
+	stx	%g4, [%sp + 2047 + 216]
+	stx	%g5, [%sp + 2047 + 224]
+	stx	%g6, [%sp + 2047 + 232]
+	stx	%g7, [%sp + 2047 + 240]
 
 	/* Save client trap table */
 	setx	client_tba, %g6, %g7
@@ -196,22 +56,44 @@ sparc64_of_client_interface:
 	stx	%sp, [%g7]
 
 	/* Save windows */
-	SAVE_WINDOW_STATE(cif)
-
-	/* Move to OpenBIOS stack */
-	setx	openbios_stack - 2047 - 192, %g6, %g7
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	add	%g1, -CONTEXT_STATE_SIZE, %g1
+	stx	%g1, [%g7]
+	
+	SAVE_CPU_WINDOW_STATE(cif)
+
+	/* Move to OpenBIOS context stack */
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g6
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	sub	%g6, %g5, %g6
+	stx	%g6, [%g7]
+	
+	setx	- 2047 - 192, %g6, %g7
+	add	%g1, %g7, %g7
 	mov	%g7, %sp
 
 	/* Call client inteface */
 	call of_client_interface
 	 ldx	[%g1 + 0x30], %o0
 
-	setx	client_window, %g6, %g1
-	stx	%o0, [%g1 + 0x30]
-
 	/* Restore windows */
-	RESTORE_WINDOW_STATE(cif)
-
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	add	%g1, %g5, %g1
+	stx	%g1, [%g7]
+	
+	/* Return value */
+	stx	%o0, [%g1 + 0x30]
+	
+	RESTORE_CPU_WINDOW_STATE(cif)
+	
+	add	%g1, CONTEXT_STATE_SIZE, %g1
+	setx	_fcstack_ptr, %g6, %g7
+	stx	%g1, [%g7]
+	
 	/* Restore stack */
 	setx	client_stack, %g6, %g7
 	ldx	[%g7], %sp
@@ -222,15 +104,15 @@ sparc64_of_client_interface:
 	wrpr	%g6, %tba
 
 	/* Restore globals */
-	ldx	[%sp + 2047 + 0], %g1
-	ldx	[%sp + 2047 + 8], %g2
-	ldx	[%sp + 2047 + 16], %g3
-	ldx	[%sp + 2047 + 24], %g4
-	ldx	[%sp + 2047 + 32], %g5
-	ldx	[%sp + 2047 + 40], %g6
-	ldx	[%sp + 2047 + 48], %g7
-
-	add	%sp, 56, %sp
+	ldx	[%sp + 2047 + 192], %g1
+	ldx	[%sp + 2047 + 200], %g2
+	ldx	[%sp + 2047 + 208], %g3
+	ldx	[%sp + 2047 + 216], %g4
+	ldx	[%sp + 2047 + 224], %g5
+	ldx	[%sp + 2047 + 232], %g6
+	ldx	[%sp + 2047 + 240], %g7
+
+	add	%sp, 248, %sp
 
 	jmp	%o7+8
 	 nop
diff --git a/qemu/roms/openbios/arch/sparc64/context.c b/qemu/roms/openbios/arch/sparc64/context.c
index 2e7668958..98932ee9c 100644
--- a/qemu/roms/openbios/arch/sparc64/context.c
+++ b/qemu/roms/openbios/arch/sparc64/context.c
@@ -40,6 +40,10 @@ static uint8_t image_stack[IMAGE_STACK_SIZE];
 /* Pointer to startup context (physical address) */
 unsigned long __boot_ctx;
 
+/* Pointer to Forth context stack */
+void *_fcstack_ptr = &_efcstack;
+
+
 /*
  * Main starter
  * This is the C function that runs first.
diff --git a/qemu/roms/openbios/arch/sparc64/cpustate.h b/qemu/roms/openbios/arch/sparc64/cpustate.h
new file mode 100644
index 000000000..0c276bfd1
--- /dev/null
+++ b/qemu/roms/openbios/arch/sparc64/cpustate.h
@@ -0,0 +1,244 @@
+/*
+ *   Save/restore CPU state macros
+ *
+ *   Copyright (C) 2015 Mark Cave-Ayland (mark.cave-ayland@ilande.co.uk>)
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   version 2
+ *
+ */
+
+/* State size for context (see below) */
+#define CONTEXT_STATE_SIZE 0x510
+
+/* Stack size for context (allocated inline of the context stack) */
+#define CONTEXT_STACK_SIZE 0x2000
+
+/*
+ * SAVE_CPU_STATE and RESTORE_CPU_STATE are macros used to enable a context switch
+ * to C to occur within the MMU I/D TLB miss handlers.
+ *
+ * Because these handlers are called on a TLB miss, we cannot use flushw to store
+ * processor window state on the stack, as the memory areas used by each window's
+ * stack pointer may not be in the TLB, causing recursive TLB miss traps.
+ *
+ * For this reason, we save window state by manually rotating the window registers
+ * and saving their contents (along with other vital registers) into a special
+ * tlb_handler_stack defined above which is guaranteed to be locked in the TLB, and
+ * so won't cause issues with trap recursion.
+ *
+ * Once this process is complete, we remain in a TL=0, CWP=0 state (with IE=1 to allow
+ * window fill/spill traps if required), switch to our safe tlb_handler_stack and 
+ * invoke the miss handler.
+ */
+
+#define SAVE_CPU_WINDOW_STATE(type) \
+	/* Save window state into context at %g1 */ \
+	rdpr	%cwp, %g7; \
+	stx	%g7, [%g1]; \
+	rdpr	%cansave, %g7; \
+	stx	%g7, [%g1 + 0x8]; \
+	rdpr	%canrestore, %g7; \
+	stx	%g7, [%g1 + 0x10]; \
+	rdpr	%otherwin, %g7; \
+	stx	%g7, [%g1 + 0x18]; \
+	rdpr	%wstate, %g7; \
+	stx	%g7, [%g1 + 0x20]; \
+	rdpr	%cleanwin, %g7; \
+	stx	%g7, [%g1 + 0x28]; \
+	\
+	stx	%o0, [%g1 + 0x30]; \
+	stx	%o1, [%g1 + 0x38]; \
+	stx	%o2, [%g1 + 0x40]; \
+	stx	%o3, [%g1 + 0x48]; \
+	stx	%o4, [%g1 + 0x50]; \
+	stx	%o5, [%g1 + 0x58]; \
+	stx	%o6, [%g1 + 0x60]; \
+	stx	%o7, [%g1 + 0x68]; \
+	\
+	rdpr	%pstate, %g7; \
+	stx	%g7, [%g1 + 0x70]; \
+	rd	%y, %g7; \
+	stx	%g7, [%g1 + 0x78]; \
+	rd	%fprs, %g7; \
+	stx	%g7, [%g1 + 0x80]; \
+	rdpr    %tl, %g7; \
+	stx     %g7, [%g1 + 0x88]; \
+	\
+	/* Now iterate through all of the windows saving all l and i registers */ \
+	add	%g1, 0x90, %g5; \
+	\
+	/* Get the number of windows in %g6 */ \
+	rdpr	%ver, %g6; \
+	and	%g6, 0xf, %g6; \
+	\
+	mov     %g6, %g4; \
+	inc     %g4; \
+	\
+	/* Starting cwp in g7 */ \
+	rdpr    %cwp, %g7; \
+	\
+save_cpu_window_##type: \
+	wrpr    %g7, %cwp; \
+	stx	%l0, [%g5]; \
+	stx	%l1, [%g5 + 0x8]; \
+	stx	%l2, [%g5 + 0x10]; \
+	stx	%l3, [%g5 + 0x18]; \
+	stx	%l4, [%g5 + 0x20]; \
+	stx	%l5, [%g5 + 0x28]; \
+	stx	%l6, [%g5 + 0x30]; \
+	stx	%l7, [%g5 + 0x38]; \
+	stx	%i0, [%g5 + 0x40]; \
+	stx	%i1, [%g5 + 0x48]; \
+	stx	%i2, [%g5 + 0x50]; \
+	stx	%i3, [%g5 + 0x58]; \
+	stx	%i4, [%g5 + 0x60]; \
+	stx	%i5, [%g5 + 0x68]; \
+	stx	%i6, [%g5 + 0x70]; \
+	stx	%i7, [%g5 + 0x78]; \
+	dec	%g7; \
+	and	%g7, %g6, %g7; \
+	subcc	%g4, 1, %g4; \
+	bne	save_cpu_window_##type; \
+	 add	%g5, 0x80, %g5; \
+	\
+	/* For 8 windows with 16 registers to save in the window, memory required \
+	is 16*8*8 = 0x400 bytes */ \
+	\
+	/* Now we should be in window 0 so update the other window registers */ \
+	rdpr	%ver, %g6; \
+	and	%g6, 0xf, %g6; \
+	dec	%g6; \
+	wrpr	%g6, %cansave; \
+	\
+	wrpr	%g0, %cleanwin; \
+	wrpr	%g0, %canrestore; \
+	wrpr	%g0, %otherwin; \
+
+	
+#define SAVE_CPU_TRAP_STATE(type) \
+	/* Save trap state into context at %g1 */ \
+	add	%g1, 0x490, %g5; \
+	mov	4, %g6; \
+	\
+save_trap_state_##type: \
+	deccc	%g6; \
+	wrpr	%g6, %tl; \
+	rdpr	%tpc, %g7; \
+	stx	%g7, [%g5]; \
+	rdpr	%tnpc, %g7; \
+	stx	%g7, [%g5 + 0x8]; \
+	rdpr	%tstate, %g7; \
+	stx	%g7, [%g5 + 0x10]; \
+	rdpr	%tt, %g7; \
+	stx	%g7, [%g5 + 0x18]; \
+	bne	save_trap_state_##type; \
+	 add	%g5, 0x20, %g5; \
+	\
+	/* For 4 trap levels with 4 registers, memory required is \
+	4*8*4 = 0x80 bytes */
+
+/* Save all state into context at %g1 */
+#define SAVE_CPU_STATE(type) \
+	SAVE_CPU_WINDOW_STATE(type); \
+	SAVE_CPU_TRAP_STATE(type);
+
+
+#define RESTORE_CPU_WINDOW_STATE(type) \
+	/* Restore window state from context at %g1 */ \
+	\
+	/* Get the number of windows in %g6 */ \
+	rdpr	%ver, %g6; \
+	and	%g6, 0xf, %g6; \
+	\
+	mov	%g6, %g4; \
+	inc	%g4; \
+	\
+	/* Set starting window */ \
+	ldx	[%g1], %g7; \
+	\
+	/* Now iterate through all of the windows restoring all l and i registers */ \
+	add	%g1, 0x90, %g5; \
+	\
+restore_cpu_window_##type: \
+	wrpr	%g7, %cwp; \
+	ldx	[%g5], %l0; \
+	ldx	[%g5 + 0x8], %l1; \
+	ldx	[%g5 + 0x10], %l2; \
+	ldx	[%g5 + 0x18], %l3; \
+	ldx	[%g5 + 0x20], %l4; \
+	ldx	[%g5 + 0x28], %l5; \
+	ldx	[%g5 + 0x30], %l6; \
+	ldx	[%g5 + 0x38], %l7; \
+	ldx	[%g5 + 0x40], %i0; \
+	ldx	[%g5 + 0x48], %i1; \
+	ldx	[%g5 + 0x50], %i2; \
+	ldx	[%g5 + 0x58], %i3; \
+	ldx	[%g5 + 0x60], %i4; \
+	ldx	[%g5 + 0x68], %i5; \
+	ldx	[%g5 + 0x70], %i6; \
+	ldx	[%g5 + 0x78], %i7; \
+	dec	%g7; \
+	and	%g7, %g6, %g7; \
+	subcc	%g4, 1, %g4; \
+	bne	restore_cpu_window_##type; \
+	 add	%g5, 0x80, %g5; \
+	\
+	/* Restore the window registers to their original value */ \
+	ldx	[%g1], %g7; \
+	wrpr	%g7, %cwp; \
+	ldx	[%g1 + 0x8], %g7; \
+	wrpr	%g7, %cansave; \
+	ldx	[%g1 + 0x10], %g7; \
+	wrpr	%g7, %canrestore; \
+	ldx	[%g1 + 0x18], %g7; \
+	wrpr	%g7, %otherwin; \
+	ldx	[%g1 + 0x20], %g7; \
+	wrpr	%g7, %wstate; \
+	ldx	[%g1 + 0x28], %g7; \
+	wrpr	%g7, %cleanwin; \
+	\
+	ldx	[%g1 + 0x30], %o0; \
+	ldx	[%g1 + 0x38], %o1; \
+	ldx	[%g1 + 0x40], %o2; \
+	ldx	[%g1 + 0x48], %o3; \
+	ldx	[%g1 + 0x50], %o4; \
+	ldx	[%g1 + 0x58], %o5; \
+	ldx	[%g1 + 0x60], %o6; \
+	ldx	[%g1 + 0x68], %o7; \
+	\
+	ldx	[%g1 + 0x70], %g7; \
+	wrpr	%g7, %pstate; \
+	ldx	[%g1 + 0x78], %g7; \
+	wr	%g7, 0, %y; \
+	ldx	[%g1 + 0x80], %g7; \
+	wr	%g7, 0, %fprs; \
+
+
+#define RESTORE_CPU_TRAP_STATE(type) \
+	/* Restore trap state from context at %g1 */ \
+	add	%g1, 0x490, %g5; \
+	mov	4, %g6; \
+	\
+restore_trap_state_##type: \
+	deccc	%g6; \
+	wrpr	%g6, %tl; \
+	ldx	[%g5], %g7; \
+	wrpr	%g7, %tpc; \
+	ldx	[%g5 + 0x8], %g7; \
+	wrpr	%g7, %tnpc; \
+	ldx	[%g5 + 0x10], %g7; \
+	wrpr	%g7, %tstate; \
+	ldx	[%g5 + 0x18], %g7; \
+	wrpr	%g7, %tt; \
+	bne	restore_trap_state_##type; \
+	 add	%g5, 0x20, %g5; \
+	\
+	ldx	[%g1 + 0x88], %g7; \
+	wrpr	%g7, %tl
+
+/* Restore all state from context at %g1 */
+#define RESTORE_CPU_STATE(type) \
+	RESTORE_CPU_WINDOW_STATE(type); \
+	RESTORE_CPU_TRAP_STATE(type);
diff --git a/qemu/roms/openbios/arch/sparc64/ldscript b/qemu/roms/openbios/arch/sparc64/ldscript
index 54288e825..c5cc6a5ce 100644
--- a/qemu/roms/openbios/arch/sparc64/ldscript
+++ b/qemu/roms/openbios/arch/sparc64/ldscript
@@ -50,6 +50,11 @@ SECTIONS
 	*(.bss.*)
 	*(COMMON)
 
+	_fcstack = .;
+	. += 32768;
+	. = ALIGN(16);
+	_efcstack = .;
+	
 	_stack = .;
 	. += STACK_SIZE;
 	. = ALIGN(16);
diff --git a/qemu/roms/openbios/arch/sparc64/lib.c b/qemu/roms/openbios/arch/sparc64/lib.c
index e9101af52..4709ca8fe 100644
--- a/qemu/roms/openbios/arch/sparc64/lib.c
+++ b/qemu/roms/openbios/arch/sparc64/lib.c
@@ -458,10 +458,10 @@ NODE_METHODS(mmu) = {
 void ob_mmu_init(const char *cpuname, uint64_t ram_size)
 {
     /* memory node */
-    REGISTER_NODE_METHODS(memory, "/memory");
+    REGISTER_NODE(memory);
 
     /* MMU node */
-    REGISTER_NODE_METHODS(mmu, "/virtual-memory");
+    REGISTER_NODE(mmu);
 
     ofmem_register(find_dev("/memory"), find_dev("/virtual-memory"));
 
diff --git a/qemu/roms/openbios/arch/sparc64/vectors.S b/qemu/roms/openbios/arch/sparc64/vectors.S
index 927c1cdc7..9d86b6bd8 100644
--- a/qemu/roms/openbios/arch/sparc64/vectors.S
+++ b/qemu/roms/openbios/arch/sparc64/vectors.S
@@ -24,6 +24,7 @@
  */
 
 #define __ASSEMBLY__
+#include "cpustate.h"
 #include "pstate.h"
 #include <asm/asi.h>
 #define ASI_BP ASI_PHYS_BYPASS_EC_E
@@ -276,17 +277,8 @@ tl1_resv1f0:	BTRAPS(0x1f0) BTRAPS(0x1f8)
 
 	.section ".data"
 	.align 8
-	.globl tlb_handler_stack_top, tlb_handler_stack_pointer, obp_ticks_pointer
+	.globl obp_ticks_pointer
 
-	! Stack for the tlb MMU trap handlers
-tlb_handler_stack_bottom:
-	.skip 8192
-tlb_handler_stack_top:
-	.skip 8
-
-	! MMU trap handler stack pointer
-tlb_handler_stack_pointer:
-	.xword tlb_handler_stack_top
 
 	! Pointer to current tick value
 obp_ticks_pointer:
@@ -336,234 +328,30 @@ fill_32bit:
         restored
         retry
 
-/*
- * SAVE_CPU_STATE and RESTORE_CPU_STATE are macros used to enable a context switch
- * to C to occur within the MMU I/D TLB miss handlers.
- *
- * Because these handlers are called on a TLB miss, we cannot use flushw to store
- * processor window state on the stack, as the memory areas used by each window's
- * stack pointer may not be in the TLB, causing recursive TLB miss traps.
- *
- * For this reason, we save window state by manually rotating the window registers
- * and saving their contents (along with other vital registers) into a special
- * tlb_handler_stack defined above which is guaranteed to be locked in the TLB, and
- * so won't cause issues with trap recursion.
- *
- * Once this process is complete, we remain in a TL=0, CWP=0 state (with IE=1 to allow
- * window fill/spill traps if required), switch to our safe tlb_handler_stack and 
- * invoke the miss handler.
- */
-
-#define SAVE_CPU_STATE(type) \
-	/* Set up our exception stack pointer in %g1 */ \
-	setx	tlb_handler_stack_pointer, %g7, %g6; \
-	ldx	[%g6], %g1; \
-	add	%g1, -0x510, %g1; \
-	\
-	/* First save the various state registers */ \
-	rdpr	%cwp, %g7; \
-	stx	%g7, [%g1]; \
-	rdpr	%cansave, %g7; \
-	stx	%g7, [%g1 + 0x8]; \
-	rdpr	%canrestore, %g7; \
-	stx	%g7, [%g1 + 0x10]; \
-	rdpr	%otherwin, %g7; \
-	stx	%g7, [%g1 + 0x18]; \
-	rdpr	%wstate, %g7; \
-	stx	%g7, [%g1 + 0x20]; \
-	rdpr	%cleanwin, %g7; \
-	stx	%g7, [%g1 + 0x28]; \
-	rdpr	%pstate, %g7; \
-	stx	%g7, [%g1 + 0x30]; \
-	\
-	rd	%y, %g7; \
-	stx	%g7, [%g1 + 0x38]; \
-	rd	%fprs, %g7; \
-	stx	%g7, [%g1 + 0x40]; \
-	\
-	rdpr	%tl, %g7; \
-	stx	%g7, [%g1 + 0x48]; \
-	\
-	/* Trap state */ \
-	add	%g1, 0x50, %g5; \
-	mov	4, %g6; \
-	\
-save_trap_state_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %tl; \
-	rdpr	%tpc, %g7; \
-	stx	%g7, [%g5]; \
-	rdpr	%tnpc, %g7; \
-	stx	%g7, [%g5 + 0x8]; \
-	rdpr	%tstate, %g7; \
-	stx	%g7, [%g5 + 0x10]; \
-	rdpr	%tt, %g7; \
-	stx	%g7, [%g5 + 0x18]; \
-	bne	save_trap_state_##type; \
-	 add	%g5, 0x20, %g5; \
-	\
-	/* For 4 trap levels with 4 registers, memory required is 
-	4*8*4 = 0x80 bytes */ \
-	\
-	/* Save the o registers */ \
-	stx	%o0, [%g1 + 0xd0]; \
-	stx	%o1, [%g1 + 0xd8]; \
-	stx	%o2, [%g1 + 0xe0]; \
-	stx	%o3, [%g1 + 0xe8]; \
-	stx	%o4, [%g1 + 0xf0]; \
-	stx	%o5, [%g1 + 0xf8]; \
-	stx	%o6, [%g1 + 0x100]; \
-	stx	%o7, [%g1 + 0x108]; \
-	\
-	/* Now iterate through all of the windows saving all l and i registers */ \
-	add	%g1, 0x110, %g5; \
-	\
-	/* Get the number of windows in %g6 */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	inc	%g6; \
-	\
-save_cpu_window_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %cwp; \
-	stx	%l0, [%g5]; \
-	stx	%l1, [%g5 + 0x8]; \
-	stx	%l2, [%g5 + 0x10]; \
-	stx	%l3, [%g5 + 0x18]; \
-	stx	%l4, [%g5 + 0x20]; \
-	stx	%l5, [%g5 + 0x28]; \
-	stx	%l6, [%g5 + 0x30]; \
-	stx	%l7, [%g5 + 0x38]; \
-	stx	%i0, [%g5 + 0x40]; \
-	stx	%i1, [%g5 + 0x48]; \
-	stx	%i2, [%g5 + 0x50]; \
-	stx	%i3, [%g5 + 0x58]; \
-	stx	%i4, [%g5 + 0x60]; \
-	stx	%i5, [%g5 + 0x68]; \
-	stx	%i6, [%g5 + 0x70]; \
-	stx	%i7, [%g5 + 0x78]; \
-	bne	save_cpu_window_##type; \
-	 add	%g5, 0x80, %g5; \
-	\
-	/* For 8 windows with 16 registers to save in the window, memory required
-	is 16*8*8 = 0x400 bytes */ \
-	\
-	/* Now we should be in window 0 so update the other window registers */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	dec	%g6; \
-	wrpr	%g6, %cansave; \
-	\
-	wrpr	%g0, %cleanwin; \
-	wrpr	%g0, %canrestore; \
-	wrpr	%g0, %otherwin; \
-	\
-	/* Update our exception stack pointer */ \
-	setx	tlb_handler_stack_pointer, %g7, %g6; \
-	stx	%g1, [%g6];
-
-
-#define RESTORE_CPU_STATE(type) \
-	/* Set up our exception stack pointer in %g1 */ \
-	setx	tlb_handler_stack_pointer, %g7, %g6; \
-	ldx	[%g6], %g1; \
-	\
-	/* Get the number of windows in %g6 */ \
-	rdpr	%ver, %g6; \
-	and	%g6, 0xf, %g6; \
-	inc	%g6; \
-	\
-	/* Now iterate through all of the windows restoring all l and i registers */ \
-	add	%g1, 0x110, %g5; \
-	\
-restore_cpu_window_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %cwp; \
-	ldx	[%g5], %l0; \
-	ldx	[%g5 + 0x8], %l1; \
-	ldx	[%g5 + 0x10], %l2; \
-	ldx	[%g5 + 0x18], %l3; \
-	ldx	[%g5 + 0x20], %l4; \
-	ldx	[%g5 + 0x28], %l5; \
-	ldx	[%g5 + 0x30], %l6; \
-	ldx	[%g5 + 0x38], %l7; \
-	ldx	[%g5 + 0x40], %i0; \
-	ldx	[%g5 + 0x48], %i1; \
-	ldx	[%g5 + 0x50], %i2; \
-	ldx	[%g5 + 0x58], %i3; \
-	ldx	[%g5 + 0x60], %i4; \
-	ldx	[%g5 + 0x68], %i5; \
-	ldx	[%g5 + 0x70], %i6; \
-	ldx	[%g5 + 0x78], %i7; \
-	bne	restore_cpu_window_##type; \
-	 add	%g5, 0x80, %g5; \
-	\
-	/* Restore the window registers to their original value */ \
-	ldx	[%g1], %g7; \
-	wrpr	%g7, %cwp; \
-	ldx	[%g1 + 0x8], %g7; \
-	wrpr	%g7, %cansave; \
-	ldx	[%g1 + 0x10], %g7; \
-	wrpr	%g7, %canrestore; \
-	ldx	[%g1 + 0x18], %g7; \
-	wrpr	%g7, %otherwin; \
-	ldx	[%g1 + 0x20], %g7; \
-	wrpr	%g7, %wstate; \
-	ldx	[%g1 + 0x28], %g7; \
-	wrpr	%g7, %cleanwin; \
-	ldx	[%g1 + 0x30], %g7; \
-	wrpr	%g7, %pstate; \
-	\
-	/* Restore the o registers */ \
-	ldx	[%g1 + 0xd0], %o0; \
-	ldx	[%g1 + 0xd8], %o1; \
-	ldx	[%g1 + 0xe0], %o2; \
-	ldx	[%g1 + 0xe8], %o3; \
-	ldx	[%g1 + 0xf0], %o4; \
-	ldx	[%g1 + 0xf8], %o5; \
-	ldx	[%g1 + 0x100], %o6; \
-	ldx	[%g1 + 0x108], %o7; \
-	\
-	/* Restore the trap state */ \
-	add	%g1, 0x50, %g5; \
-	mov	4, %g6; \
-	\
-restore_trap_state_##type: \
-	deccc	%g6; \
-	wrpr	%g6, %tl; \
-	ldx	[%g5], %g7; \
-	wrpr	%g7, %tpc; \
-	ldx	[%g5 + 0x8], %g7; \
-	wrpr	%g7, %tnpc; \
-	ldx	[%g5 + 0x10], %g7; \
-	wrpr	%g7, %tstate; \
-	ldx	[%g5 + 0x18], %g7; \
-	wrpr	%g7, %tt; \
-	bne	restore_trap_state_##type; \
-	 add	%g5, 0x20, %g5; \
-	\
-	ldx	[%g1 + 0x38], %g7; \
-	wr	%g7, 0, %y; \
-	ldx	[%g1 + 0x40], %g7; \
-	wr	%g7, 0, %fprs; \
-	ldx	[%g1 + 0x48], %g7; \
-	wrpr	%g7, %tl; \
-	\
-	/* Restore exception stack pointer to previous value */ \
-	setx	tlb_handler_stack_pointer, %g7, %g6; \
-	add	%g1, 0x510, %g1; \
-	stx	%g1, [%g6];
-
 
         .globl reload_DMMU_tlb, reload_IMMU_tlb, bug
 
 reload_DMMU_tlb:
-
+	
+	/* Save CPU state to stack */
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	add	%g1, -CONTEXT_STATE_SIZE, %g1
+	stx	%g1, [%g7]
+	
 	SAVE_CPU_STATE(dtlb)
 
-	/* Switch to TLB locked stack space (note we add an additional 192 bytes required for
+	/* Switch to 8K TLB locked OpenBIOS stack (note we add an additional 192 bytes required for
 	   gcc to save its arguments when building with -O0) */
-	add 	%g1, -STACK_BIAS - 192, %sp
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g6
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	sub	%g6, %g5, %g6
+	stx	%g6, [%g7]
+	
+	setx	- 2047 - 192, %g6, %g7
+	add	%g1, %g7, %g7
+	mov	%g7, %sp
 
 	/* Enable interrupts for window spill/fill traps */
 	rdpr	%pstate, %g7
@@ -577,18 +365,44 @@ reload_DMMU_tlb:
 	rdpr	%pstate, %g7
 	andn	%g7, PSTATE_IE, %g7
 	wrpr	%g7, %pstate
-
+	
+	/* Restore CPU state from stack */
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	add	%g1, %g5, %g1
+	stx	%g1, [%g7]
+	
 	RESTORE_CPU_STATE(dtlb)
-
+	
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	add	%g1, CONTEXT_STATE_SIZE, %g1
+	stx	%g1, [%g7]
+	
         retry
 
 reload_IMMU_tlb:
-
+	
+	/* Save CPU state to stack */
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	add	%g1, -CONTEXT_STATE_SIZE, %g1
+	stx	%g1, [%g7]
+	
 	SAVE_CPU_STATE(itlb)
 
-	/* Switch to TLB locked stack space (note we add an additional 192 bytes required for
+	/* Switch to 8K TLB locked OpenBIOS stack (note we add an additional 192 bytes required for
 	   gcc to save its arguments when building with -O0) */
-	add 	%g1, -STACK_BIAS - 192, %sp
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g6
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	sub	%g6, %g5, %g6
+	stx	%g6, [%g7]
+	
+	setx	- 2047 - 192, %g6, %g7
+	add	%g1, %g7, %g7
+	mov	%g7, %sp
 
 	/* Enable interrupts for window spill/fill traps */
 	rdpr	%pstate, %g7
@@ -602,8 +416,20 @@ reload_IMMU_tlb:
 	rdpr	%pstate, %g7
 	andn	%g7, PSTATE_IE, %g7
 	wrpr	%g7, %pstate
-
+	
+	/* Restore CPU state from stack */
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	setx	CONTEXT_STACK_SIZE, %g4, %g5
+	add	%g1, %g5, %g1
+	stx	%g1, [%g7]
+	
 	RESTORE_CPU_STATE(itlb)
+	
+	setx	_fcstack_ptr, %g6, %g7
+	ldx	[%g7], %g1
+	add	%g1, CONTEXT_STATE_SIZE, %g1
+	stx	%g1, [%g7]
 
         retry
 
diff --git a/qemu/roms/openbios/config/scripts/switch-arch b/qemu/roms/openbios/config/scripts/switch-arch
index d5e2f7710..ab3b4ce69 100755
--- a/qemu/roms/openbios/config/scripts/switch-arch
+++ b/qemu/roms/openbios/config/scripts/switch-arch
@@ -17,48 +17,57 @@ if [ x"$1" = x -o "$1" = "-help" ]; then
   exit 0
 fi
 
-crosscflags()
+is_bigendian()
 {
-    host=$1
-    target=$2
-
-    if test "$host" = "powerpc" -o "$host" = "ppc" \
-        -o "$host" = "mips" -o "$host" = "s390" \
-        -o "$host" = "sparc32" -o "$host" = "sparc64" \
-        -o "$host" = "m68k" -o "$host" = "armv4b"; then
-        hostbigendian="yes"
+    cpu=$1
+
+    if test "$cpu" = "powerpc" -o "$cpu" = "ppc" \
+        -o "$cpu" = "powerpc64" -o "$cpu" = "ppc64" \
+        -o "$cpu" = "mips" -o "$cpu" = "s390" \
+        -o "$cpu" = "sparc32" -o "$cpu" = "sparc64" \
+        -o "$cpu" = "m68k" -o "$cpu" = "armv4b"; then
+        echo yes
     else
-        hostbigendian="no"
+        echo no
     fi
+}
 
-# host long bits test
-    if test "$host" = "sparc64" -o "$host" = "ia64" \
-        -o "$host" = "amd64" -o "$host" = "x86_64" \
-        -o "$host" = "alpha"; then
-        hostlongbits="64"
+longbits()
+{
+    cpu=$1
+    if test "$cpu" = "sparc64" -o "$cpu" = "ia64" \
+        -o "$cpu" = "amd64" -o "$cpu" = "x86_64" \
+        -o "$cpu" = "powerpc64" -o "$cpu" = "ppc64" \
+        -o "$cpu" = "alpha"; then
+        echo 64
     else
-        hostlongbits="32"
+        echo 32
     fi
+}
 
-    if test "$target" = "powerpc" -o "$target" = "ppc" \
-        -o "$target" = "powerpc64" -o "$target" = "ppc64" \
-        -o "$target" = "mips" -o "$target" = "s390" \
-        -o "$target" = "sparc32" -o "$target" = "sparc64" \
-        -o "$target" = "m68k" -o "$target" = "armv4b"; then
-        targetbigendian="yes"
-    else
-        targetbigendian="no"
-    fi
+basearch()
+{
+    arch=$1
+    case $arch in
+    powerpc|ppc64|powerpc64)
+        echo ppc
+        ;;
+    *)
+        echo $arch
+        ;;
+    esac
+}
 
-# target long bits test
-    if test "$target" = "sparc64" -o "$target" = "ia64" \
-        -o "$target" = "amd64"  -o "$target" = "x86_64" \
-        -o "$target" = "powerpc64" -o "$target" = "ppc64" \
-        -o "$target" = "alpha"; then
-        targetlongbits="64"
-    else
-        targetlongbits="32"
-    fi
+crosscflags()
+{
+    host=$1
+    target=$2
+
+    hostbigendian=$(is_bigendian $host)
+    hostlongbits=$(longbits $host)
+
+    targetbigendian=$(is_bigendian $target)
+    targetlongbits=$(longbits $target)
 
     if test "$targetbigendian" = "$hostbigendian"; then
         cflags="-USWAP_ENDIANNESS"
@@ -99,23 +108,27 @@ archname()
 
 select_prefix()
 {
-    TARGETS="${1}-unknown-linux-gnu- ${1}-linux-gnu- ${1}-linux- ${1}-elf- ${1}-eabi-"
+    BASEARCH=$(basearch $ARCH)
+    for target_arch ; do
+        TARGETS="${target_arch}-unknown-linux-gnu- ${target_arch}-linux-gnu- ${target_arch}-linux- ${target_arch}-elf- ${target_arch}-eabi-"
 
-    if [ x"$CROSS_COMPILE" != "x" ]; then
-        TARGETS=$CROSS_COMPILE
-    fi
+        if [ x"$CROSS_COMPILE" != "x" ]; then
+            TARGETS=$CROSS_COMPILE
+        fi
 
-    for TARGET in $TARGETS
-    do
-        if type ${TARGET}gcc > /dev/null 2>&1
-        then
+        for TARGET in $TARGETS
+        do
+            if type ${TARGET}gcc > /dev/null 2>&1
+            then
+                return
+            fi
+        done
+        if [ "$BASEARCH" = "$(basearch $HOSTARCH)" ]; then
+            TARGET=""
             return
         fi
     done
-    if [ "$ARCH" = "$HOSTARCH" ]; then
-        return
-    fi
-    echo "ERROR: no ${1} cross-compiler found !" 1>&2
+    echo "ERROR: no $* cross-compiler found !" 1>&2
     exit 1
 }
 
@@ -242,7 +255,6 @@ for ARCH in $arch_list; do
         esac
     done
 
-    BASEARCH=$ARCH
     case $ARCH in
         amd64)
         select_prefix x86_64
@@ -251,9 +263,10 @@ for ARCH in $arch_list; do
         ;;
 
         ppc)
-        select_prefix powerpc
+        select_prefix powerpc powerpc64
         if [ "$unix" = "no" ]; then
-            CFLAGS="-m32 -msoft-float -fno-builtin-bcopy -fno-builtin-log2"
+            # 604 cpu includes support for PReP as well as Mac
+            CFLAGS="-m32 -mcpu=604 -msoft-float -fno-builtin-bcopy -fno-builtin-log2"
             AS_FLAGS="-m32"
         else
             CFLAGS="-fno-builtin"
@@ -263,13 +276,14 @@ for ARCH in $arch_list; do
 
         ppc64)
         select_prefix powerpc64
-        CFLAGS="-Wa,-a64 -m64 -msoft-float -fno-builtin"
+
+        # 970 cpu is used in all 64-bit Macs but disable altivec
+        CFLAGS="-mcpu=970 -mno-altivec -Wa,-a64 -m64 -msoft-float -fno-builtin"
         AS_FLAGS="-Wa,-a64"
-        BASEARCH=ppc
         ;;
 
         sparc32)
-        select_prefix sparc
+        select_prefix sparc sparc64
         CFLAGS="-Wa,-xarch=v8 -Wa,-32 -m32 -mcpu=supersparc -fno-builtin"
         AS_FLAGS="-Wa,-xarch=v8 -Wa,-32"
         ;;
diff --git a/qemu/roms/openbios/drivers/cuda.c b/qemu/roms/openbios/drivers/cuda.c
index 9555dea49..ff5d22de2 100644
--- a/qemu/roms/openbios/drivers/cuda.c
+++ b/qemu/roms/openbios/drivers/cuda.c
@@ -144,8 +144,22 @@ static int cuda_adb_req (void *host, const uint8_t *snd_buf, int len,
  //   CUDA_DPRINTF("len: %d %02x\n", len, snd_buf[0]);
     len = cuda_request(host, ADB_PACKET, snd_buf, len, buffer);
     if (len > 1 && buffer[0] == ADB_PACKET) {
-        pos = buffer + 2;
-        len -= 2;
+        /* We handle 2 types of ADB packet here:
+               Normal: <type> <status> <data> ...
+               Error : <type> <status> <cmd> (<data> ...)
+           Ideally we should use buffer[1] (status) to determine whether this
+           is a normal or error packet but this requires a corresponding fix
+           in QEMU <= 2.4. Hence we temporarily handle it this way to ease
+           the transition. */
+        if (len > 2 && buffer[2] == snd_buf[0]) {
+            /* Error */
+            pos = buffer + 3;
+            len -= 3;
+        } else {
+            /* Normal */
+            pos = buffer + 2;
+            len -= 2;
+        }
     } else {
         pos = buffer + 1;
         len = -1;
@@ -380,7 +394,8 @@ powermgt_init(char *path)
 
 	ph = find_dev(buf);
 	set_property(ph, "device_type", "power-mgt", 10);
-	set_property(ph, "compatible", "power-mgt", 10);
+	set_property(ph, "mgt-kind", "min-consumption-pwm-led", strlen("min-consumption-pwm-led") + 1);
+	set_property(ph, "compatible", "cuda", strlen("cuda") + 1);
 }
 
 cuda_t *cuda_init (const char *path, phys_addr_t base)
diff --git a/qemu/roms/openbios/drivers/escc.c b/qemu/roms/openbios/drivers/escc.c
index 240043be3..1990e798d 100644
--- a/qemu/roms/openbios/drivers/escc.c
+++ b/qemu/roms/openbios/drivers/escc.c
@@ -380,12 +380,44 @@ ob_zs_init(phys_addr_t base, uint64_t offset, int intr, int slave, int keyboard)
 
 static void
 escc_add_channel(const char *path, const char *node, phys_addr_t addr,
-                 uint32_t offset)
+                 int esnum)
 {
     char buf[64], tty[32];
     phandle_t dnode, aliases;
-    int len;
-    cell props[2];
+
+    cell props[10];
+    ucell offset;
+    int index;
+    int legacy;
+    
+    int dbdma_offsets[2][2] = {
+        /* ch-b */
+        { 0x6, 0x7 },
+        /* ch-a */
+        { 0x4, 0x5 }
+    };
+    
+    int reg_offsets[2][2][3] = {
+        {
+            /* ch-b */
+            { 0x00, 0x10, 0x40 },
+            /* ch-a */
+            { 0x20, 0x30, 0x50 }
+        },{
+            /* legacy ch-b */
+            { 0x0, 0x2, 0x8 },
+            /* legacy ch-a */
+            { 0x4, 0x6, 0xa }
+        }
+    };
+    
+    switch (esnum) {
+        case 2: index = 1; legacy = 0; break;
+        case 3: index = 0; legacy = 0; break;
+        case 4: index = 1; legacy = 1; break;
+        case 5: index = 0; legacy = 1; break;
+        default: return;
+    }
 
     /* add device */
 
@@ -411,31 +443,49 @@ escc_add_channel(const char *path, const char *node, phys_addr_t addr,
     set_property(dnode, "device_type", "serial",
                  strlen("serial") + 1);
 
-    snprintf(buf, sizeof(buf), "ch-%s", node);
-    len = strlen(buf) + 1;
-    snprintf(buf + len, sizeof(buf) - len, "CHRP,es2");
-    set_property(dnode, "compatible", buf, len + 9);
+    snprintf(buf, sizeof(buf), "chrp,es%d", esnum);
+    set_property(dnode, "compatible", buf, 9);
 
-    props[0] = IO_ESCC_OFFSET + offset * 0x20;
-    props[1] = 0x00000020;
-    set_property(dnode, "reg", (char *)&props, 2 * sizeof(cell));
+    if (legacy) {
+        offset = IO_ESCC_LEGACY_OFFSET;
+    } else {
+        offset = IO_ESCC_OFFSET;
+    }
 
-    props[0] = addr + IO_ESCC_OFFSET + offset * 0x20;
+    props[0] = offset + reg_offsets[legacy][index][0];
+    props[1] = 0x1;
+    props[2] = offset + reg_offsets[legacy][index][1];
+    props[3] = 0x1;
+    props[4] = offset + reg_offsets[legacy][index][2];
+    props[5] = 0x1;
+    props[6] = 0x8000 + dbdma_offsets[index][0] * 0x100;
+    props[7] = 0x100;
+    props[8] = 0x8000 + dbdma_offsets[index][1] * 0x100;
+    props[9] = 0x100;
+    set_property(dnode, "reg", (char *)&props, 10 * sizeof(cell));
+
+    props[0] = addr + offset + reg_offsets[legacy][index][0];
     OLDWORLD(set_property(dnode, "AAPL,address",
             (char *)&props, 1 * sizeof(cell)));
 
-    props[0] = 0x00000010 - offset;
+    props[0] = 0x10 - index;
     OLDWORLD(set_property(dnode, "AAPL,interrupts",
             (char *)&props, 1 * sizeof(cell)));
 
-    props[0] = (0x24) + offset;
-    props[1] = 0;
+    props[0] = (0x24) + index;
+    props[1] = 0x1;
+    props[2] = dbdma_offsets[index][0];
+    props[3] = 0x0;
+    props[4] = dbdma_offsets[index][1];
+    props[5] = 0x0;
     NEWWORLD(set_property(dnode, "interrupts",
-             (char *)&props, 2 * sizeof(cell)));
+             (char *)&props, 6 * sizeof(cell)));
+
+    set_int_property(dnode, "slot-names", 0);
 
     device_end();
 
-    uart_init_line((unsigned char*)addr + IO_ESCC_OFFSET + offset * 0x20,
+    uart_init_line((unsigned char*)addr + offset + reg_offsets[legacy][index][0],
                    CONFIG_SERIAL_SPEED);
 }
 
@@ -464,13 +514,39 @@ escc_init(const char *path, phys_addr_t addr)
     set_property(dnode, "device_type", "escc",
                  strlen("escc") + 1);
     set_property(dnode, "compatible", "escc\0CHRP,es0", 14);
+    set_property(dnode, "ranges", "", 0);
 
     fword("finish-device");
 
-    escc_add_channel(buf, "a", addr, 1);
-    escc_add_channel(buf, "b", addr, 0);
+    escc_add_channel(buf, "a", addr, 2);
+    escc_add_channel(buf, "b", addr, 3);
 
     escc_serial_dev = (unsigned char *)addr + IO_ESCC_OFFSET +
                  (CONFIG_SERIAL_PORT ? 0 : 0x20);
+
+    push_str(path);
+    fword("find-device");
+    fword("new-device");
+
+    push_str("escc-legacy");
+    fword("device-name");
+
+    snprintf(buf, sizeof(buf), "%s/escc-legacy", path);
+
+    dnode = find_dev(buf);
+
+    set_int_property(dnode, "#address-cells", 1);
+    props[0] = __cpu_to_be32(IO_ESCC_LEGACY_OFFSET);
+    props[1] = __cpu_to_be32(IO_ESCC_LEGACY_SIZE);
+    set_property(dnode, "reg", (char *)&props, sizeof(props));
+    set_property(dnode, "device_type", "escc-legacy",
+                 strlen("escc-legacy") + 1);
+    set_property(dnode, "compatible", "chrp,es1", 9);
+    set_property(dnode, "ranges", "", 0);
+
+    fword("finish-device");
+
+    escc_add_channel(buf, "a", addr, 4);
+    escc_add_channel(buf, "b", addr, 5);
 }
 #endif
diff --git a/qemu/roms/openbios/drivers/escc.h b/qemu/roms/openbios/drivers/escc.h
index caaf00d40..e73f267b2 100644
--- a/qemu/roms/openbios/drivers/escc.h
+++ b/qemu/roms/openbios/drivers/escc.h
@@ -1,6 +1,8 @@
 
 #define IO_ESCC_SIZE    0x00001000
 #define IO_ESCC_OFFSET  0x00013000
+#define IO_ESCC_LEGACY_SIZE    0x00001000
+#define IO_ESCC_LEGACY_OFFSET  0x00012000
 
 #define ZS_REGS         8
 
diff --git a/qemu/roms/openbios/drivers/ide.c b/qemu/roms/openbios/drivers/ide.c
index 327c64a40..1da60c895 100644
--- a/qemu/roms/openbios/drivers/ide.c
+++ b/qemu/roms/openbios/drivers/ide.c
@@ -73,13 +73,13 @@ static inline void ide_add_channel(struct ide_channel *chan)
 	channels = chan;
 }
 
-static struct ide_channel *ide_seek_channel(const char *name)
+static struct ide_channel *ide_seek_channel(phandle_t ph)
 {
 	struct ide_channel *current;
 
 	current = channels;
 	while (current) {
-		if (!strcmp(current->name, name))
+		if (current->ph == ph)
 			return current;
 		current = current->next;
 	}
@@ -1247,11 +1247,10 @@ ob_ide_initialize(int *idx)
 static void
 ob_ide_open(int *idx)
 {
-	int ret=1, len;
+	int ret=1;
 	phandle_t ph;
 	struct ide_drive *drive;
 	struct ide_channel *chan;
-	char *idename;
 	int unit;
 
 	fword("my-unit");
@@ -1260,9 +1259,8 @@ ob_ide_open(int *idx)
 	fword("my-parent");
 	fword("ihandle>phandle");
 	ph=(phandle_t)POP();
-	idename=get_property(ph, "name", &len);
 
-	chan = ide_seek_channel(idename);
+	chan = ide_seek_channel(ph);
 	drive = &chan->drives[unit];
 	*(struct ide_drive **)idx = drive;
 
@@ -1380,9 +1378,6 @@ int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
 
 		chan = malloc(sizeof(struct ide_channel));
 
-		snprintf(chan->name, sizeof(chan->name),
-			 DEV_NAME, current_channel);
-
 		chan->mmio = 0;
 
 		for (j = 0; j < 8; j++)
@@ -1424,9 +1419,9 @@ int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
 
                 snprintf(nodebuff, sizeof(nodebuff), "%s/" DEV_NAME, path,
                          current_channel);
-		REGISTER_NAMED_NODE(ob_ide_ctrl, nodebuff);
+		REGISTER_NAMED_NODE_PHANDLE(ob_ide_ctrl, nodebuff, dnode);
 
-		dnode = find_dev(nodebuff);
+		chan->ph = dnode;
 
 #if !defined(CONFIG_PPC) && !defined(CONFIG_SPARC64)
 		props[0]=14; props[1]=0;
@@ -1468,11 +1463,9 @@ int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
 					break;
 			}
 			IDE_DPRINTF("%s]: %s\n", media, drive->model);
-                        snprintf(nodebuff, sizeof(nodebuff),
-                                 "%s/" DEV_NAME "/%s", path, current_channel,
-                                 media);
-			REGISTER_NAMED_NODE(ob_ide, nodebuff);
-			dnode=find_dev(nodebuff);
+			snprintf(nodebuff, sizeof(nodebuff), "%s/%s",
+				 get_path_from_ph(dnode), media);
+			REGISTER_NAMED_NODE_PHANDLE(ob_ide, nodebuff, dnode);
 			set_int_property(dnode, "reg", j);
 
 			/* create aliases */
@@ -1488,6 +1481,28 @@ int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
 	return 0;
 }
 
+void ob_ide_quiesce(void)
+{
+	struct ide_channel *channel;
+	int i;
+
+	channel = channels;
+	while (channel) {
+		for (i = 0; i < 2; i++) {
+			struct ide_drive *drive = &channel->drives[i];
+
+			if (!drive->present)
+				continue;
+
+			ob_ide_select_drive(drive);
+			ob_ide_software_reset(drive);
+			ob_ide_device_type_check(drive);
+		}
+
+		channel = channel->next;
+	}
+}
+
 #if defined(CONFIG_DRIVER_MACIO)
 static unsigned char
 macio_ide_inb(struct ide_channel *chan, unsigned int port)
@@ -1527,16 +1542,13 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 	struct ide_channel *chan;
 
 	/* IDE ports on Macs are numbered from 3.
-	 * Also see comments in macio.c:openpic_init() */
+	 * Also see comments in pci.c:ob_pci_host_set_interrupt_map() */
 	current_channel = 3;
 
-	for (i = 0; i < nb_channels; i++, current_channel++) {
+	for (i = 0; i < nb_channels; i++) {
 
 		chan = malloc(sizeof(struct ide_channel));
 
-		snprintf(chan->name, sizeof(chan->name),
-			 DEV_NAME, current_channel);
-
 		chan->mmio = addr + MACIO_IDE_OFFSET + i * MACIO_IDE_SIZE;
 
 		chan->obide_inb = macio_ide_inb;
@@ -1574,13 +1586,19 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 
                 snprintf(nodebuff, sizeof(nodebuff), "%s/" DEV_NAME, path,
                          current_channel);
-		REGISTER_NAMED_NODE(ob_ide_ctrl, nodebuff);
+		REGISTER_NAMED_NODE_PHANDLE(ob_ide_ctrl, nodebuff, dnode);
 
-		dnode = find_dev(nodebuff);
+		chan->ph = dnode;
 
 		set_property(dnode, "compatible", (is_oldworld() ?
 			     "heathrow-ata" : "keylargo-ata"), 13);
 
+		set_property(dnode, "model", ((current_channel == 3) ?
+			     "ata-3" : "ata-4"), strlen("ata-*") + 1);
+
+		set_property(dnode, "AAPL,connector", "ata",
+                             strlen("ata") + 1);
+
 		props[0] = 0x00000526;
 		props[1] = 0x00000085;
 		props[2] = 0x00000025;
@@ -1589,8 +1607,8 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 		props[5] = 0x00000000;
 		props[6] = 0x00000000;
 		props[7] = 0x00000000;
-		OLDWORLD(set_property(dnode, "AAPL,pio-timing",
-				      (char *)&props, 8*sizeof(props[0])));
+		set_property(dnode, "AAPL,pio-timing",
+				      (char *)&props, 8*sizeof(props[0]));
 
 		/* The first interrupt entry is the ide interrupt, the second
 		   the dbdma interrupt */
@@ -1633,9 +1651,9 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 		OLDWORLD(set_property(dnode, "AAPL,address",
 				      (char *)&props, 2*sizeof(props[0])));
 
-		props[0] = 0;
-		OLDWORLD(set_property(dnode, "AAPL,bus-id", (char*)props,
-			 1 * sizeof(props[0])));
+		props[0] = i;
+		set_property(dnode, "AAPL,bus-id", (char*)props,
+			 1 * sizeof(props[0]));
 		IDE_DPRINTF(DEV_NAME": [io ports 0x%lx]\n",
 		            current_channel, chan->mmio);
 
@@ -1663,11 +1681,9 @@ int macio_ide_init(const char *path, uint32_t addr, int nb_channels)
 					break;
 			}
 			IDE_DPRINTF("%s]: %s\n", media, drive->model);
-                        snprintf(nodebuff, sizeof(nodebuff),
-                                 "%s/" DEV_NAME "/%s", path, current_channel,
-                                 media);
-			REGISTER_NAMED_NODE(ob_ide, nodebuff);
-			dnode = find_dev(nodebuff);
+			snprintf(nodebuff, sizeof(nodebuff), "%s/%s",
+				 get_path_from_ph(dnode), media);
+			REGISTER_NAMED_NODE_PHANDLE(ob_ide, nodebuff, dnode);
 			set_int_property(dnode, "reg", j);
 
 			/* create aliases */
diff --git a/qemu/roms/openbios/drivers/ide.h b/qemu/roms/openbios/drivers/ide.h
index d6c4b9f5d..8983c8ecf 100644
--- a/qemu/roms/openbios/drivers/ide.h
+++ b/qemu/roms/openbios/drivers/ide.h
@@ -167,7 +167,7 @@ struct ide_drive {
 
 struct ide_channel {
 
-	char name[32];
+	phandle_t ph;
 	struct ide_channel *next;
 
 	/*
diff --git a/qemu/roms/openbios/drivers/obio.c b/qemu/roms/openbios/drivers/obio.c
index 7c135a362..4ac063188 100644
--- a/qemu/roms/openbios/drivers/obio.c
+++ b/qemu/roms/openbios/drivers/obio.c
@@ -26,8 +26,6 @@
 #define	PROMDEV_SCREEN	0		/* output to screen */
 #define	PROMDEV_TTYA	1		/* in/out to ttya */
 
-/* DECLARE data structures for the nodes.  */
-DECLARE_UNNAMED_NODE( ob_obio, INSTALL_OPEN, sizeof(int) );
 
 void
 ob_new_obio_device(const char *name, const char *type)
@@ -397,45 +395,6 @@ ob_smp_init(unsigned long mem_size)
 }
 
 static void
-ob_obio_open(__attribute__((unused))int *idx)
-{
-	int ret=1;
-	RET ( -ret );
-}
-
-static void
-ob_obio_close(__attribute__((unused))int *idx)
-{
-	selfword("close-deblocker");
-}
-
-static void
-ob_obio_initialize(__attribute__((unused))int *idx)
-{
-    push_str("/");
-    fword("find-device");
-    fword("new-device");
-
-    push_str("obio");
-    fword("device-name");
-
-    push_str("hierarchical");
-    fword("device-type");
-
-    PUSH(2);
-    fword("encode-int");
-    push_str("#address-cells");
-    fword("property");
-
-    PUSH(1);
-    fword("encode-int");
-    push_str("#size-cells");
-    fword("property");
-
-    fword("finish-device");
-}
-
-static void
 ob_set_obio_ranges(uint64_t base)
 {
     push_str("/obio");
@@ -458,27 +417,6 @@ ob_set_obio_ranges(uint64_t base)
     fword("property");
 }
 
-static void
-ob_obio_decodeunit(__attribute__((unused)) int *idx)
-{
-    fword("decode-unit-sbus");
-}
-
-
-static void
-ob_obio_encodeunit(__attribute__((unused)) int *idx)
-{
-    fword("encode-unit-sbus");
-}
-
-NODE_METHODS(ob_obio) = {
-	{ NULL,			ob_obio_initialize	},
-	{ "open",		ob_obio_open		},
-	{ "close",		ob_obio_close		},
-	{ "encode-unit",	ob_obio_encodeunit	},
-	{ "decode-unit",	ob_obio_decodeunit	},
-};
-
 
 int
 ob_obio_init(uint64_t slavio_base, unsigned long fd_offset,
@@ -491,10 +429,6 @@ ob_obio_init(uint64_t slavio_base, unsigned long fd_offset,
     // http://www.ibiblio.org/pub/historic-linux/early-ports/Sparc/NCR/NCR89C105.txt
 
     //printk("Initializing OBIO devices...\n");
-#if 0 // XXX
-    REGISTER_NAMED_NODE(ob_obio, "/obio");
-    device_end();
-#endif
     ob_set_obio_ranges(slavio_base);
 
     // Zilog Z8530 serial ports, see http://www.zilog.com
diff --git a/qemu/roms/openbios/drivers/pci.c b/qemu/roms/openbios/drivers/pci.c
index 366f4a17f..5062f302f 100644
--- a/qemu/roms/openbios/drivers/pci.c
+++ b/qemu/roms/openbios/drivers/pci.c
@@ -144,9 +144,16 @@ static void dump_reg_property(const char* description, int nreg, u32 *reg)
 }
 #endif
 
-static unsigned long pci_bus_addr_to_host_addr(uint32_t ba)
+static unsigned long pci_bus_addr_to_host_addr(int space, uint32_t ba)
 {
-    return arch->host_pci_base + (unsigned long)ba;
+    if (space == IO_SPACE) {
+        return arch->io_base + (unsigned long)ba;
+    } else if (space == MEMORY_SPACE_32) {
+        return arch->host_pci_base + (unsigned long)ba;
+    } else {
+        /* Return unaltered to aid debugging property values */
+        return (unsigned long)ba;
+    }
 }
 
 static void
@@ -340,22 +347,27 @@ ob_pci_encode_unit(int *idx)
 	        ss, dev, fn, buf);
 }
 
-/* ( pci-addr.lo pci-addr.hi size -- virt ) */
+/* ( pci-addr.lo pci-addr.mid pci-addr.hi size -- virt ) */
 
 static void
 ob_pci_map_in(int *idx)
 {
 	phys_addr_t phys;
 	uint32_t ba;
-	ucell size, virt;
+	ucell size, virt, tmp;
+	int space;
 
 	PCI_DPRINTF("ob_pci_bar_map_in idx=%p\n", idx);
 
 	size = POP();
+	tmp = POP();
 	POP();
 	ba = POP();
 
-	phys = pci_bus_addr_to_host_addr(ba);
+	/* Get the space from the pci-addr.hi */
+	space = ((tmp & PCI_RANGE_TYPE_MASK) >> 24);
+
+	phys = pci_bus_addr_to_host_addr(space, ba);
 
 #if defined(CONFIG_OFMEM)
 	ofmem_claim_phys(phys, size, 0);
@@ -448,13 +460,18 @@ static void pci_host_set_ranges(const pci_config_t *config)
 	int ncells;
 
 	ncells = 0;
-	/* first encode PCI configuration space */
-	{
-	    ncells += pci_encode_phys_addr(props + ncells, 0, CONFIGURATION_SPACE,
+	
+#ifdef CONFIG_SPARC64
+        /* While configuration space isn't mentioned in the IEEE-1275 PCI
+           bindings, it appears in the PCI host bridge ranges property in
+           real device trees. Hence we disable this range for all host
+           bridges except for SPARC, particularly as it causes Darwin/OS X
+           to incorrectly calculated PCI memory space ranges on PPC. */
+	ncells += pci_encode_phys_addr(props + ncells, 0, CONFIGURATION_SPACE,
                      config->dev, 0, 0);
         ncells += host_encode_phys_addr(props + ncells, arch->cfg_addr);
         ncells += pci_encode_size(props + ncells, arch->cfg_len);
-	}
+#endif
 
 	if (arch->io_base) {
 	    ncells += pci_encode_phys_addr(props + ncells, 0, IO_SPACE,
@@ -585,13 +602,18 @@ static void pci_set_AAPL_address(const pci_config_t *config)
 {
 	phandle_t dev = get_cur_dev();
 	cell props[7];
-	int ncells, i;
+	uint32_t mask;
+	int ncells, i, flags, space_code;
 
 	ncells = 0;
 	for (i = 0; i < 6; i++) {
 		if (!config->assigned[i] || !config->sizes[i])
 			continue;
-		props[ncells++] = config->assigned[i] & ~0x0000000F;
+		pci_decode_pci_addr(config->assigned[i],
+				    &flags, &space_code, &mask);
+
+		props[ncells++] = pci_bus_addr_to_host_addr(space_code,
+					config->assigned[i] & ~mask);
 	}
 	if (ncells)
 		set_property(dev, "AAPL,address", (char *)props,
@@ -752,13 +774,19 @@ int macio_keylargo_config_cb (const pci_config_t *config)
 int vga_config_cb (const pci_config_t *config)
 {
         unsigned long rom;
-        uint32_t rom_size, size;
+        uint32_t rom_size, size, mask;
+        int flags, space_code;
         phandle_t ph;
 
         if (config->assigned[0] != 0x00000000) {
             setup_video();
 
-            rom = pci_bus_addr_to_host_addr(config->assigned[1] & ~0x0000000F);
+            pci_decode_pci_addr(config->assigned[1],
+                &flags, &space_code, &mask);
+
+            rom = pci_bus_addr_to_host_addr(space_code,
+                                            config->assigned[1] & ~0x0000000F);
+
             rom_size = config->sizes[1];
 
             ph = get_cur_dev();
@@ -824,7 +852,7 @@ int ebus_config_cb(const pci_config_t *config)
         ncells += pci_encode_phys_addr(props + ncells,
                                        flags, space_code, config->dev,
                                        PCI_BASE_ADDR_0 + (i * sizeof(uint32_t)),
-                                       0);
+                                       config->assigned[i] & ~mask);
 
         props[ncells++] = config->sizes[i];
     }
@@ -997,7 +1025,10 @@ static void ob_pci_add_properties(phandle_t phandle,
 	}
 
 	pci_set_assigned_addresses(phandle, config, num_bars);
-	OLDWORLD(pci_set_AAPL_address(config));
+	
+	if (is_apple()) {
+		pci_set_AAPL_address(config);
+	}
 
 	PCI_DPRINTF("\n");
 }
@@ -1397,9 +1428,11 @@ static void ob_pci_set_available(phandle_t host, unsigned long mem_base, unsigne
 
 static void ob_pci_host_set_interrupt_map(phandle_t host)
 {
-    phandle_t dnode = 0;
-    u32 props[128];
-    int i;
+    phandle_t dnode = 0, pci_childnode = 0;
+    u32 props[128], intno;
+    int i, ncells, len;
+    u32 *val, addr;
+    char *reg;
 
 #if defined(CONFIG_PPC)
     phandle_t target_node;
@@ -1420,16 +1453,22 @@ static void ob_pci_host_set_interrupt_map(phandle_t host)
         target_node = find_dev("/pci/mac-io/escc/ch-b");
         set_int_property(target_node, "interrupt-parent", dnode);
 
+        target_node = find_dev("/pci/mac-io/escc-legacy/ch-a");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
+        target_node = find_dev("/pci/mac-io/escc-legacy/ch-b");
+        set_int_property(target_node, "interrupt-parent", dnode);
+
         /* QEMU only emulates 2 of the 3 ata buses currently */
         /* On a new world Mac these are not numbered but named by the
          * ATA version they support. Thus we have: ata-3, ata-3, ata-4
          * On g3beige they all called just ide.
-         * We take ata-3 and ata-4 which seems to work for both
-         * at least for clients we care about */
-        target_node = find_dev("/pci/mac-io/ata-3");
+         * We take 2 x ata-3 buses which seems to work for
+         * at least the clients we care about */
+        target_node = find_dev("/pci/mac-io/ata-3@20000");
         set_int_property(target_node, "interrupt-parent", dnode);
 
-        target_node = find_dev("/pci/mac-io/ata-4");
+        target_node = find_dev("/pci/mac-io/ata-3@21000");
         set_int_property(target_node, "interrupt-parent", dnode);
 
         target_node = find_dev("/pci/mac-io/via-cuda");
@@ -1437,69 +1476,61 @@ static void ob_pci_host_set_interrupt_map(phandle_t host)
 
         target_node = find_dev("/pci");
         set_int_property(target_node, "interrupt-parent", dnode);
-
-        /* openpic interrupt mapping */
-        for (i = 0; i < (7*8); i += 7) {
-            props[i + PCI_INT_MAP_PCI0] = 0;
-            props[i + PCI_INT_MAP_PCI1] = 0;
-            props[i + PCI_INT_MAP_PCI2] = 0;
-            props[i + PCI_INT_MAP_PCI_INT] = (i / 7) + 1; // starts at PINA=1
-            props[i + PCI_INT_MAP_PIC_HANDLE] = dnode;
-            props[i + PCI_INT_MAP_PIC_INT] = arch->irqs[i / 7];
-            props[i + PCI_INT_MAP_PIC_POL] = 3;
-        }
-        set_property(host, "interrupt-map", (char *)props, 7 * 8 * sizeof(props[0]));
-
-        props[PCI_INT_MAP_PCI0] = 0;
-        props[PCI_INT_MAP_PCI1] = 0;
-        props[PCI_INT_MAP_PCI2] = 0;
-        props[PCI_INT_MAP_PCI_INT] = 0x7;
-
-        set_property(host, "interrupt-map-mask", (char *)props, 4 * sizeof(props[0]));
     }
-#elif defined(CONFIG_SPARC64)
-    int ncells, len;
-    u32 *val, addr;
-    char *reg;
+#else
+    /* PCI host bridge is the default interrupt controller */
+    dnode = host;
+#endif
 
     /* Set interrupt-map for PCI devices with an interrupt pin present */
     ncells = 0;
 
     PUSH(host);
     fword("child");
-    dnode = POP();
-    while (dnode) {
-        if (get_int_property(dnode, "interrupts", &len)) {
-            reg = get_property(dnode, "reg", &len);
-            if (reg) {
+    pci_childnode = POP();
+    while (pci_childnode) {
+        intno = get_int_property(pci_childnode, "interrupts", &len);
+        if (len && intno) {
+            reg = get_property(pci_childnode, "reg", &len);
+            if (len && reg) {
                 val = (u32 *)reg;
 
                 for (i = 0; i < (len / sizeof(u32)); i += 5) {
                     addr = val[i];
 
                     /* Device address is in 1st 32-bit word of encoded PCI address for config space */
-                    if (!(addr & 0x03000000)) {
+                    if ((addr & PCI_RANGE_TYPE_MASK) == PCI_RANGE_CONFIG) {
+#if defined(CONFIG_SPARC64)
                         ncells += pci_encode_phys_addr(props + ncells, 0, 0, addr, 0, 0);
-                        props[ncells++] = 1;    /* always interrupt pin 1 for QEMU */
-                        props[ncells++] = host;
-                        props[ncells++] = SUN4U_INTERRUPT(addr, 1);
+                        props[ncells++] = intno;
+                        props[ncells++] = dnode;
+                        props[ncells++] = SUN4U_INTERRUPT(addr, intno);
+#elif defined(CONFIG_PPC)
+                        ncells += pci_encode_phys_addr(props + ncells, 0, 0, addr, 0, 0);
+                        props[ncells++] = intno;
+                        props[ncells++] = dnode;
+                        props[ncells++] = arch->irqs[intno - 1];
+                        props[ncells++] = 3;
+#else
+                        /* Keep compiler quiet */
+                        dnode = dnode;
+#endif
                     }
                 }
             }
         }
 
-        PUSH(dnode);
+        PUSH(pci_childnode);
         fword("peer");
-        dnode = POP();
+        pci_childnode = POP();
     }
     set_property(host, "interrupt-map", (char *)props, ncells * sizeof(props[0]));
 
     props[0] = 0x0000f800;
     props[1] = 0x0;
     props[2] = 0x0;
-    props[3] = 7;
+    props[3] = 0x7;
     set_property(host, "interrupt-map-mask", (char *)props, 4 * sizeof(props[0]));
-#endif
 }
 
 int ob_pci_init(void)
diff --git a/qemu/roms/openbios/drivers/pci.fs b/qemu/roms/openbios/drivers/pci.fs
index 563b652a4..a7b56e1f8 100644
--- a/qemu/roms/openbios/drivers/pci.fs
+++ b/qemu/roms/openbios/drivers/pci.fs
@@ -12,59 +12,19 @@
   rot encode-int encode+ 
   ;
 
-\ Get region offset for BAR reg
-: pci-bar-offset@ ( bar-reg -- off.lo off.hi -1 | 0 )
-  " reg" active-package get-package-property 0= if
-    begin
-      decode-phys    \ ( reg prop prop-len phys.lo phys.mid phys.hi )
-      ff and 5 pick = if
-        >r >r 3drop r> r>
-        -1 exit
-      else
-        2drop
-      then
-      \ Drop the size as we don't need it
-      decode-int drop decode-int drop
-      dup 0=
-    until
-    3drop
-    0 exit
-  else
-    0
-  then
-  ;
-
-\ Get region size for BAR reg
-: pci-bar-size@ ( bar-reg -- size )
-  " reg" active-package get-package-property 0= if
-    begin
-      decode-phys    \ ( reg prop prop-len phys.lo phys.mid phys.hi )
-      ff and 5 pick = if
-        2drop decode-int drop
-        decode-int
-        >r 3drop r>
-        exit
-      else
-        2drop decode-int drop
-        decode-int drop
-      then
-      dup 0=
-    until
-    3drop
-    0    \ default size of 0 if BAR not found
-  then
-  ;
-
-\ Get base address for configured BAR reg
-: pci-bar-base@ ( bar-reg -- addr.lo addr.hi -1 | 0 )
+\ Get PCI physical address and size for configured BAR reg
+: pci-bar>pci-addr ( bar-reg -- addr.lo addr.mid addr.hi size -1 | 0 )
   " assigned-addresses" active-package get-package-property 0= if
     begin
       decode-phys    \ ( reg prop prop-len phys.lo phys.mid phys.hi )
-      ff and 5 pick = if
-        >r >r 3drop r> r>
+      dup ff and 6 pick = if
+        >r >r >r rot drop
+        decode-int drop decode-int
+        -rot 2drop
+        r> swap r> r> rot
         -1 exit
       else
-        2drop
+        3drop
       then
       \ Drop the size as we don't need it
       decode-int drop decode-int drop
@@ -77,16 +37,4 @@
   then
   ;
 
-\ Get PCI bus address and size for configured BAR reg
-: pci-bar>pci-region  ( bar-reg -- addr.lo addr.hi size )
-  dup
-  >r pci-bar-offset@ if
-    swap r@ pci-bar-base@ if
-      swap d+
-    then
-    swap r@ pci-bar-size@
-  then
-  r> drop
-  ;
-
 [THEN]
diff --git a/qemu/roms/openbios/drivers/pci.h b/qemu/roms/openbios/drivers/pci.h
index 84a2b2cf6..d5aa5f84a 100644
--- a/qemu/roms/openbios/drivers/pci.h
+++ b/qemu/roms/openbios/drivers/pci.h
@@ -59,6 +59,15 @@
 #define PCI_MIN_GNT             0x3e    /* 8 bits */
 #define PCI_MAX_LAT             0x3f    /* 8 bits */
 
+#define PCI_RANGE_RELOCATABLE          0x80000000
+#define PCI_RANGE_PREFETCHABLE         0x40000000
+#define PCI_RANGE_ALIASED              0x20000000
+#define PCI_RANGE_TYPE_MASK            0x03000000
+#define PCI_RANGE_MMIO_64BIT           0x03000000
+#define PCI_RANGE_MMIO                 0x02000000
+#define PCI_RANGE_IOPORT               0x01000000
+#define PCI_RANGE_CONFIG               0x00000000
+
 typedef struct {
         u16     signature;
         u8      reserved[0x16];
diff --git a/qemu/roms/openbios/drivers/sbus.c b/qemu/roms/openbios/drivers/sbus.c
index a9b26c0a0..4caa59aaf 100644
--- a/qemu/roms/openbios/drivers/sbus.c
+++ b/qemu/roms/openbios/drivers/sbus.c
@@ -369,31 +369,6 @@ sbus_probe_slot_ss600mp(unsigned int slot, uint64_t base)
     }
 }
 
-static void
-ob_sbus_open(void)
-{
-	int ret=1;
-	RET ( -ret );
-}
-
-static void
-ob_sbus_close(void)
-{
-	selfword("close-deblocker");
-}
-
-static void
-ob_sbus_initialize(void)
-{
-}
-
-
-NODE_METHODS(ob_sbus_node) = {
-	{ NULL,			ob_sbus_initialize	},
-	{ "open",		ob_sbus_open		},
-	{ "close",		ob_sbus_close		},
-};
-
 struct sbus_offset {
     int slot, type;
     uint64_t base;
diff --git a/qemu/roms/openbios/drivers/usbohci_private.h b/qemu/roms/openbios/drivers/usbohci_private.h
index b3a723e21..99c964100 100644
--- a/qemu/roms/openbios/drivers/usbohci_private.h
+++ b/qemu/roms/openbios/drivers/usbohci_private.h
@@ -43,7 +43,7 @@
 	// FIXME: fake
 	typedef enum { CMD} reg;
 
-	enum {
+	extern enum {
 		NumberDownstreamPorts = 1<<0,
 		PowerSwitchingMode = 1<<8,
 		NoPowerSwitching = 1<<9,
@@ -53,17 +53,17 @@
 		PowerOnToPowerGoodTime = 1<<24
 	} HcRhDescriptorAReg;
 
-	enum {
+	extern enum {
 		NumberDownstreamPortsMask = MASK(0, 8),
 		PowerOnToPowerGoodTimeMask = MASK(24, 8)
 	} HcRhDescriptorAMask;
 
-	enum {
+	extern enum {
 		DeviceRemovable = 1<<0,
 		PortPowerControlMask = 1<<16
 	} HcRhDescriptorBReg;
 
-	enum {
+	extern enum {
 		CurrentConnectStatus		= 1<<0,
 		PortEnableStatus		= 1<<1,
 		PortSuspendStatus		= 1<<2,
@@ -77,7 +77,7 @@
 		PortOverCurrentIndicatorChange	= 1<<19,
 		PortResetStatusChange		= 1<<20
 	} HcRhPortStatusRead;
-	enum {
+	extern enum {
 		ClearPortEnable			= 1<<0,
 		SetPortEnable			= 1<<1,
 		SetPortSuspend			= 1<<2,
@@ -87,7 +87,7 @@
 		ClearPortPower			= 1<<9,
 	} HcRhPortStatusSet;
 
-	enum {
+	extern enum {
 		LocalPowerStatus = 1<<0,
 		OverCurrentIndicator = 1<<1,
 		DeviceRemoteWakeupEnable = 1<<15,
@@ -96,18 +96,18 @@
 		ClearRemoteWakeupEnable = 1<<31
 	} HcRhStatusReg;
 
-	enum {
+	extern enum {
 		FrameInterval = 1<<0,
 		FSLargestDataPacket = 1<<16,
 		FrameIntervalToggle = 1<<31
 	} HcFmIntervalOffset;
-	enum {
+	extern enum {
 		FrameIntervalMask = MASK(0, 14),
 		FSLargestDataPacketMask = MASK(16, 15),
 		FrameIntervalToggleMask = MASK(31, 1)
 	} HcFmIntervalMask;
 
-	enum {
+	extern enum {
 		ControlBulkServiceRatio = 1<<0,
 		PeriodicListEnable = 1<<2,
 		IsochronousEnable = 1<<3,
@@ -119,7 +119,7 @@
 		RemoteWakeupEnable = 1<<10
 	} HcControlReg;
 
-	enum {
+	extern enum {
 		ControlBulkServiceRatioMask = MASK(0, 2),
 		HostControllerFunctionalStateMask = MASK(6, 2)
 	} HcControlMask;
@@ -131,7 +131,7 @@
 		USBSuspend = 3*HostControllerFunctionalState
 	};
 
-	enum {
+	extern enum {
 		HostControllerReset = 1<<0,
 		ControlListFilled = 1<<1,
 		BulkListFilled = 1<<2,
@@ -139,16 +139,16 @@
 		SchedulingOverrunCount = 1<<16
 	} HcCommandStatusReg;
 
-	enum {
+	extern enum {
 		SchedulingOverrunCountMask = MASK(16, 2)
 	} HcCommandStatusMask;
 
-	enum {
+	extern enum {
 		FrameRemaining = 1<<0,
 		FrameRemainingToggle = 1<<31
 	} HcFmRemainingReg;
 
-	enum {
+	extern enum {
 		SchedulingOverrung = 1<<0,
 		WritebackDoneHead = 1<<1,
 		StartofFrame = 1<<2,
diff --git a/qemu/roms/openbios/drivers/vga.fs b/qemu/roms/openbios/drivers/vga.fs
index ec4c6c5f1..29a043a7a 100644
--- a/qemu/roms/openbios/drivers/vga.fs
+++ b/qemu/roms/openbios/drivers/vga.fs
@@ -109,16 +109,17 @@ h# 1 constant VBE_DISPI_ENABLED
 \ PCI
 \
 
-" pci-bar>pci-region" (find-xt) value pci-bar>pci-region-xt
-: pci-bar>pci-region pci-bar>pci-region-xt execute ;
+" pci-bar>pci-addr" (find-xt) value pci-bar>pci-addr-xt
+: pci-bar>pci-addr pci-bar>pci-addr-xt execute ;
 
 h# 10 constant cfg-bar0    \ Framebuffer BAR
 -1 value fb-addr
 
 : map-fb ( -- )
-  cfg-bar0 pci-bar>pci-region   \ ( pci-addr.lo pci-addr.hi size )
-  " pci-map-in" $call-parent
-  to fb-addr
+  cfg-bar0 pci-bar>pci-addr if   \ ( pci-addr.lo pci-addr.mid pci-addr.hi size )
+    " pci-map-in" $call-parent
+    to fb-addr
+  then
 ;
 
 \
diff --git a/qemu/roms/openbios/forth/lib/string.fs b/qemu/roms/openbios/forth/lib/string.fs
index eb6474917..f97db232f 100644
--- a/qemu/roms/openbios/forth/lib/string.fs
+++ b/qemu/roms/openbios/forth/lib/string.fs
@@ -125,3 +125,17 @@
 : parse-hex ( str len -- value )
   base @ hex -rot $number if 0 then swap base !
 ;
+
+
+\ -----------------------------------------------------
+\ miscellaneous functions
+\ -----------------------------------------------------
+
+: rot13 ( c - c )
+  dup upc [char] A [char] M between if d# 13 + exit then
+  dup upc [char] N [char] Z between if d# 13 - then
+;
+
+: rot13-str ( str len -- newstr len )
+  strdup 2dup bounds ?do i c@ rot13 i c! loop
+;
diff --git a/qemu/roms/openbios/forth/system/ciface.fs b/qemu/roms/openbios/forth/system/ciface.fs
index fd6c54efd..85a607627 100644
--- a/qemu/roms/openbios/forth/system/ciface.fs
+++ b/qemu/roms/openbios/forth/system/ciface.fs
@@ -326,6 +326,14 @@ external
 
 : exit ( -- )
   ." EXIT"
+  
+  \ Execute (exit) hook if one exists
+  s" (exit)" $find if
+    execute
+  else
+    2drop
+  then
+  
   outer-interpreter
 ;
 
diff --git a/qemu/roms/openbios/include/arch/ppc/types.h b/qemu/roms/openbios/include/arch/ppc/types.h
index 69b3db405..b2246d060 100644
--- a/qemu/roms/openbios/include/arch/ppc/types.h
+++ b/qemu/roms/openbios/include/arch/ppc/types.h
@@ -84,21 +84,13 @@ typedef uint32_t        prom_uarg_t;
 /* size named types */
 
 typedef unsigned char   u8;
-typedef unsigned char   __u8;
 typedef unsigned short u16;
-typedef unsigned short __u16;
 typedef unsigned int   u32;
-typedef unsigned int   __u32;
 typedef unsigned long long u64;
-typedef unsigned long long __u64;
 
 typedef signed char	s8;
-typedef signed char	__s8;
 typedef short		s16;
-typedef short		__s16;
 typedef int		s32;
-typedef int		__s32;
 typedef long long	s64;
-typedef long long	__s64;
 
 #endif
diff --git a/qemu/roms/openbios/include/arch/sparc64/io.h b/qemu/roms/openbios/include/arch/sparc64/io.h
index 2e4dfa37f..0f1a73284 100644
--- a/qemu/roms/openbios/include/arch/sparc64/io.h
+++ b/qemu/roms/openbios/include/arch/sparc64/io.h
@@ -9,7 +9,7 @@
 
 extern unsigned long va_shift; // Set in entry.S
 // Defined in ldscript
-extern char _start, _data, _stack, _estack, _end, _iomem;
+extern char _start, _data, _stack, _estack, _fcstack, _efcstack, _end, _iomem;
 
 // XXX check use and merge
 #define phys_to_virt(phys) ((void *) ((unsigned long) (phys)))
diff --git a/qemu/roms/openbios/include/drivers/drivers.h b/qemu/roms/openbios/include/drivers/drivers.h
index 3b83b12d1..48f81a870 100644
--- a/qemu/roms/openbios/include/drivers/drivers.h
+++ b/qemu/roms/openbios/include/drivers/drivers.h
@@ -52,6 +52,7 @@ void kbd_init(uint64_t base);
 /* drivers/ide.c */
 int ob_ide_init(const char *path, uint32_t io_port0, uint32_t ctl_port0,
                 uint32_t io_port1, uint32_t ctl_port1);
+void ob_ide_quiesce(void);
 int macio_ide_init(const char *path, uint32_t addr, int nb_channels);
 #endif
 #ifdef CONFIG_DRIVER_ESP
diff --git a/qemu/roms/openbios/include/libopenbios/bindings.h b/qemu/roms/openbios/include/libopenbios/bindings.h
index de9c77520..4ec978912 100644
--- a/qemu/roms/openbios/include/libopenbios/bindings.h
+++ b/qemu/roms/openbios/include/libopenbios/bindings.h
@@ -56,6 +56,7 @@ extern ihandle_t	open_package( const char *argstr, phandle_t ph );
 extern ihandle_t	open_dev( const char *spec );
 extern void		close_package( ihandle_t ih );
 extern void		close_dev( ihandle_t ih );
+extern char		*get_path_from_ph( phandle_t ph );
 
 /* property access */
 extern void		set_property( phandle_t ph, const char *name,
diff --git a/qemu/roms/openbios/libopenbios/bindings.c b/qemu/roms/openbios/libopenbios/bindings.c
index 5323421f5..4f7a99379 100644
--- a/qemu/roms/openbios/libopenbios/bindings.c
+++ b/qemu/roms/openbios/libopenbios/bindings.c
@@ -366,6 +366,14 @@ find_dev( const char *path )
 	return ret;
 }
 
+char *
+get_path_from_ph( phandle_t ph )
+{
+	PUSH(ph);
+	fword("get-package-path");
+	return pop_fstr_copy();
+}
+
 phandle_t
 dt_iter_begin( void )
 {
diff --git a/qemu/roms/openbios/libopenbios/bootinfo_load.c b/qemu/roms/openbios/libopenbios/bootinfo_load.c
index fa9e36bd4..f33678185 100644
--- a/qemu/roms/openbios/libopenbios/bootinfo_load.c
+++ b/qemu/roms/openbios/libopenbios/bootinfo_load.c
@@ -161,6 +161,12 @@ bootinfo_init_program(void)
 	feval("load-size");
 	size = POP();
 
+	/* Some bootinfo scripts contain a binary payload after the
+	   NULL-terminated Forth string such as OS 9. Restrict our
+	   size to just the Forth section, otherwise we end up trying
+	   to allocate memory for the entire binary which might fail. */
+	size = strnlen(base, size);
+
 	bootscript = malloc(size);
 	if (bootscript == NULL) {
 		DPRINTF("Can't malloc %d bytes\n", size);
diff --git a/qemu/roms/seabios/.version b/qemu/roms/seabios/.version
index a96337339..89c0c6acb 100644
--- a/qemu/roms/seabios/.version
+++ b/qemu/roms/seabios/.version
@@ -1 +1 @@
-rel-1.8.2-0-g33fbe13
+rel-1.9.1-0-gb3ef39f
diff --git a/qemu/roms/seabios/Makefile b/qemu/roms/seabios/Makefile
index 83cdff377..4e4092d07 100644
--- a/qemu/roms/seabios/Makefile
+++ b/qemu/roms/seabios/Makefile
@@ -34,15 +34,16 @@ SRCBOTH=misc.c stacks.c output.c string.c block.c cdrom.c disk.c mouse.c kbd.c \
     hw/usb.c hw/usb-uhci.c hw/usb-ohci.c hw/usb-ehci.c \
     hw/usb-hid.c hw/usb-msc.c hw/usb-uas.c \
     hw/blockcmd.c hw/floppy.c hw/ata.c hw/ramdisk.c \
-    hw/virtio-ring.c hw/virtio-pci.c hw/virtio-blk.c hw/virtio-scsi.c \
     hw/lsi-scsi.c hw/esp-scsi.c hw/megasas.c
 SRC16=$(SRCBOTH)
-SRC32FLAT=$(SRCBOTH) post.c memmap.c malloc.c romfile.c x86.c optionroms.c \
-    pmm.c font.c boot.c bootsplash.c jpeg.c bmp.c \
+SRC32FLAT=$(SRCBOTH) post.c e820map.c malloc.c romfile.c x86.c optionroms.c \
+    pmm.c font.c boot.c bootsplash.c jpeg.c bmp.c tcgbios.c sha1.c \
     hw/ahci.c hw/pvscsi.c hw/usb-xhci.c hw/usb-hub.c hw/sdcard.c \
-    fw/coreboot.c fw/lzmadecode.c fw/csm.c fw/biostables.c \
+    fw/coreboot.c fw/lzmadecode.c fw/multiboot.c fw/csm.c fw/biostables.c \
     fw/paravirt.c fw/shadow.c fw/pciinit.c fw/smm.c fw/smp.c fw/mtrr.c fw/xen.c \
-    fw/acpi.c fw/mptable.c fw/pirtable.c fw/smbios.c fw/romfile_loader.c
+    fw/acpi.c fw/mptable.c fw/pirtable.c fw/smbios.c fw/romfile_loader.c \
+    hw/virtio-ring.c hw/virtio-pci.c hw/virtio-blk.c hw/virtio-scsi.c \
+    hw/tpm_drivers.c
 SRC32SEG=string.c output.c pcibios.c apm.c stacks.c hw/pci.c hw/serialio.c
 DIRS=src src/hw src/fw vgasrc
 
@@ -50,6 +51,8 @@ DIRS=src src/hw src/fw vgasrc
 cc-option=$(shell if test -z "`$(1) $(2) -S -o /dev/null -xc /dev/null 2>&1`" \
     ; then echo "$(2)"; else echo "$(3)"; fi ;)
 
+EXTRAVERSION=
+
 CPPFLAGS = -P -MD -MT $@
 
 COMMONCFLAGS := -I$(OUT) -Isrc -Os -MD -g \
@@ -62,6 +65,7 @@ COMMONCFLAGS := -I$(OUT) -Isrc -Os -MD -g \
 COMMONCFLAGS += $(call cc-option,$(CC),-nopie,)
 COMMONCFLAGS += $(call cc-option,$(CC),-fno-stack-protector,)
 COMMONCFLAGS += $(call cc-option,$(CC),-fno-stack-protector-all,)
+COMMONCFLAGS += $(call cc-option,$(CC),-fstack-check=no,)
 COMMA := ,
 
 CFLAGS32FLAT := $(COMMONCFLAGS) -DMODE16=0 -DMODESEGMENT=0
@@ -152,10 +156,10 @@ $(OUT)romlayout.o: src/romlayout.S $(OUT)autoconf.h $(OUT)asm-offsets.h
 	@echo "  Compiling (16bit) $@"
 	$(Q)$(CC) $(CFLAGS16) -c -D__ASSEMBLY__ $< -o $@
 
-$(OUT)romlayout16.lds: $(OUT)ccode32flat.o $(OUT)code32seg.o $(OUT)ccode16.o $(OUT)romlayout.o scripts/layoutrom.py scripts/buildversion.sh
+$(OUT)romlayout16.lds: $(OUT)ccode32flat.o $(OUT)code32seg.o $(OUT)ccode16.o $(OUT)romlayout.o src/version.c scripts/layoutrom.py scripts/buildversion.py
 	@echo "  Building ld scripts"
-	$(Q)BUILD_VERSION="$(VERSION)" ./scripts/buildversion.sh $(OUT)version.c
-	$(Q)$(CC) $(CFLAGS32FLAT) -c $(OUT)version.c -o $(OUT)version.o
+	$(Q)$(PYTHON) ./scripts/buildversion.py -e "$(EXTRAVERSION)" -t "$(CC);$(AS);$(LD);$(OBJCOPY);$(OBJDUMP);$(STRIP)" $(OUT)autoversion.h
+	$(Q)$(CC) $(CFLAGS32FLAT) -c src/version.c -o $(OUT)version.o
 	$(Q)$(LD) $(LD32BIT_FLAG) -r $(OUT)ccode32flat.o $(OUT)version.o -o $(OUT)code32flat.o
 	$(Q)$(LD) $(LD32BIT_FLAG) -r $(OUT)ccode16.o $(OUT)romlayout.o -o $(OUT)code16.o
 	$(Q)$(OBJDUMP) -thr $(OUT)code32flat.o > $(OUT)code32flat.o.objdump
@@ -177,7 +181,7 @@ $(OUT)rom32seg.o: $(OUT)code32seg.o $(OUT)romlayout32seg.lds
 
 $(OUT)rom.o: $(OUT)rom16.strip.o $(OUT)rom32seg.strip.o $(OUT)code32flat.o $(OUT)romlayout32flat.lds
 	@echo "  Linking $@"
-	$(Q)$(LD) -T $(OUT)romlayout32flat.lds $(OUT)rom16.strip.o $(OUT)rom32seg.strip.o $(OUT)code32flat.o -o $@
+	$(Q)$(LD) -N -T $(OUT)romlayout32flat.lds $(OUT)rom16.strip.o $(OUT)rom32seg.strip.o $(OUT)code32flat.o -o $@
 
 $(OUT)bios.bin.prep: $(OUT)rom.o scripts/checkrom.py
 	@echo "  Prepping $@"
@@ -224,10 +228,10 @@ $(OUT)vgaentry.o: vgasrc/vgaentry.S $(OUT)autoconf.h $(OUT)asm-offsets.h
 	@echo "  Compiling (16bit) $@"
 	$(Q)$(CC) $(CFLAGS16) -c -D__ASSEMBLY__ $< -o $@
 
-$(OUT)vgarom.o: $(OUT)vgaccode16.o $(OUT)vgaentry.o $(OUT)vgasrc/vgalayout.lds scripts/buildversion.sh
+$(OUT)vgarom.o: $(OUT)vgaccode16.o $(OUT)vgaentry.o $(OUT)vgasrc/vgalayout.lds vgasrc/vgaversion.c scripts/buildversion.py
 	@echo "  Linking $@"
-	$(Q)BUILD_VERSION="$(VERSION)" ./scripts/buildversion.sh $(OUT)vgaversion.c VAR16
-	$(Q)$(CC) $(CFLAGS16) -c $(OUT)vgaversion.c -o $(OUT)vgaversion.o
+	$(Q)$(PYTHON) ./scripts/buildversion.py -e "$(EXTRAVERSION)" -t "$(CC);$(AS);$(LD);$(OBJCOPY);$(OBJDUMP);$(STRIP)" $(OUT)autovgaversion.h
+	$(Q)$(CC) $(CFLAGS16) -c vgasrc/vgaversion.c -o $(OUT)vgaversion.o
 	$(Q)$(LD) --gc-sections -T $(OUT)vgasrc/vgalayout.lds $(OUT)vgaccode16.o $(OUT)vgaentry.o $(OUT)vgaversion.o -o $@
 
 $(OUT)vgabios.bin.raw: $(OUT)vgarom.o
diff --git a/qemu/roms/seabios/docs/Build_overview.md b/qemu/roms/seabios/docs/Build_overview.md
index 26db22691..8c6b2f458 100644
--- a/qemu/roms/seabios/docs/Build_overview.md
+++ b/qemu/roms/seabios/docs/Build_overview.md
@@ -52,6 +52,34 @@ CSM_ENABLE'. The SeaBIOS binary will be included as a discrete file
 within the 'Flash Volume' which is created, and there are tools which
 will extract it and allow it to be replaced.
 
+Distribution builds
+===================
+
+If one is building a binary version of SeaBIOS as part of a package
+(such as an rpm) or for wide distribution, please provide the
+EXTRAVERSION field during the build. For example:
+
+`make EXTRAVERSION="-${RPM_PACKAGE_RELEASE}"`
+
+The EXTRAVERSION field should provide the package version (if
+applicable) and the name of the distribution (if that's not already
+obvious from the package version). This string will be appended to the
+main SeaBIOS version. The above information helps SeaBIOS developers
+correlate defect reports to the source code and build environment.
+
+If one is building a binary in a build environment that does not have
+access to the git tool or does not have the full SeaBIOS git repo
+available, then please use an official SeaBIOS release tar file as
+source. If building from a snapshot (where there is no official
+SeaBIOS tar) then one should generate a snapshot tar file on a machine
+that does support git using the scripts/tarball.sh tool. For example:
+
+`scripts/tarball.sh`
+
+The tarball.sh script encodes version information in the resulting tar
+file which the build can extract and include in the final binary. The
+above EXTRAVERSION field should still be set when building from a tar.
+
 Overview of files in the repository
 ===================================
 
@@ -61,11 +89,7 @@ drivers. The **src/fw/** directory contains source code for platform
 firmware initialization. The **src/std/** directory contains header
 files describing standard bios, firmware, and hardware interfaces.
 
-The **vgasrc/** directory contains code for VGA BIOS implementations.
-This code is separate from the main BIOS code in the src/ directory.
-When the build is configured to produce a VGA BIOS the resulting
-binary is found in out/vgabios.bin. The VGA BIOS code is always
-compiled in 16bit mode.
+The **vgasrc/** directory contains code for [SeaVGABIOS](SeaVGABIOS).
 
 The **scripts/** directory contains helper utilities for manipulating
 and building the final roms.
diff --git a/qemu/roms/seabios/docs/Contributing.md b/qemu/roms/seabios/docs/Contributing.md
new file mode 100644
index 000000000..d0f2b5b5e
--- /dev/null
+++ b/qemu/roms/seabios/docs/Contributing.md
@@ -0,0 +1,20 @@
+SeaBIOS welcomes contributions of code (either fixing bugs or adding
+new functionality). At a high level, the process to contribute a
+change is:
+
+1. [Obtain](Download) the current code and documentation
+2. Enhance and test the code locally
+3. Submit changes to the SeaBIOS [mailing list](Mailinglist) as a
+   patch
+4. Receive feedback, answer questions, and possibly provide updated
+   patches
+5. When accepted, a maintainer (Kevin O'Connor or Gerd Hoffman) will
+   commit the change to the master SeaBIOS repository
+
+The SeaBIOS patch submission process is similar to the
+[QEMU process](http://wiki.qemu.org/Contribute/SubmitAPatch). Please
+review the QEMU process for more details and tips on the best way to
+submit patches. The SeaBIOS C code does follow a slightly different
+coding style from QEMU (eg, mixed code and C99 style variable
+declarations are encouraged, braces are not required around single
+statement blocks), however patches in the QEMU style are acceptable.
diff --git a/qemu/roms/seabios/docs/Debugging.md b/qemu/roms/seabios/docs/Debugging.md
index 03567de4d..7ab5d02d8 100644
--- a/qemu/roms/seabios/docs/Debugging.md
+++ b/qemu/roms/seabios/docs/Debugging.md
@@ -55,8 +55,10 @@ rate.
 The tool can also timestamp the messages from the QEMU debug port. To
 use with QEMU run the following:
 
-`mkfifo qemudebugpipe`\
-`qemu -chardev pipe,path=qemudebugpipe,id=seabios -device isa-debugcon,iobase=0x402,chardev=seabios ...`
+```
+mkfifo qemudebugpipe
+qemu -chardev pipe,path=qemudebugpipe,id=seabios -device isa-debugcon,iobase=0x402,chardev=seabios ...
+```
 
 and then in another session:
 
@@ -84,20 +86,23 @@ bios 16bit code) or out/rom.o (to debug bios 32bit code). For example:
 
 `gdb out/rom16.o`
 
-Once in gdb, use the command "target remote localhost:1234" to have
+Once in gdb, use the command `target remote localhost:1234` to have
 gdb connect to QEMU. See the QEMU documentation for more information
 on using gdb and QEMU in this mode.
 
-When debugging 16bit code, also run the following commands in gdb:
+When debugging 16bit code it is necessary to load the 16bit symbols
+twice in order for gdb to properly handle break points.  To do this,
+run the following command `objcopy --adjust-vma 0xf0000 out/rom16.o
+rom16offset.o` and then run the following in gdb:
 
-`set architecture i8086`\
-`add-symbol-file out/rom16.o 0xf0000`
+```
+set architecture i8086
+add-symbol-file rom16offset.o 0
+```
 
-The second command loads the 16bit symbols a second time at an offset
-of 0xf0000, which helps gdb set and catch breakpoints correctly.
-
-To debug a VGA BIOS image, run "gdb out/vgarom.o" add use the gdb
-command "add-symbol-file out/vgarom.o 0xc0000" to load the 16bit VGA
+To debug a VGA BIOS image, run `gdb out/vgarom.o`, create a
+vgaromoffset.o file with offset 0xc0000, add use the gdb
+command `add-symbol-file out/vgaromoffset.o 0` to load the 16bit VGA
 BIOS symbols twice.
 
 If debugging the 32bit SeaBIOS initialization code with gdb, note that
diff --git a/qemu/roms/seabios/docs/Developer_Documentation.md b/qemu/roms/seabios/docs/Developer_Documentation.md
index d50455d36..24bf48a3e 100644
--- a/qemu/roms/seabios/docs/Developer_Documentation.md
+++ b/qemu/roms/seabios/docs/Developer_Documentation.md
@@ -10,15 +10,16 @@ page.
 See details on [building SeaBIOS](Build overview).
 
 There is also information on the SeaBIOS [Memory Model](Memory Model).
-
 Along with information on SeaBIOS [Execution and code flow](Execution
-and code flow).
+and code flow). A description of the process of linking the final
+SeaBIOS binary is available at [Linking overview](Linking overview).
 
-A description of the process of linking the final SeaBIOS binary is
-available at [Linking overview](Linking overview).
+The list of available runtime configuration items is at
+[runtime config](Runtime_config).
 
 To debug SeaBIOS and report problems see SeaBIOS
-[debugging](Debugging).
+[debugging](Debugging). To contribute changes to SeaBIOS see
+[contributing](Contributing).
 
 Useful links to specifications is available at [Developer
 links](Developer links).
diff --git a/qemu/roms/seabios/docs/Download.md b/qemu/roms/seabios/docs/Download.md
index a49c6fb74..9b1492ac5 100644
--- a/qemu/roms/seabios/docs/Download.md
+++ b/qemu/roms/seabios/docs/Download.md
@@ -9,8 +9,10 @@ The SeaBIOS project uses the [git](http://git-scm.com/) revision
 control system. To download the latest source from revision control,
 run:
 
-`$ git clone git://git.seabios.org/seabios.git seabios`\
-`$ cd seabios`
+```
+$ git clone git://git.seabios.org/seabios.git seabios
+$ cd seabios
+```
 
 There's also a [website](http://git.seabios.org/) to browse the latest
 source code online.
diff --git a/qemu/roms/seabios/docs/Execution_and_code_flow.md b/qemu/roms/seabios/docs/Execution_and_code_flow.md
index 9396ecaa4..a54776eef 100644
--- a/qemu/roms/seabios/docs/Execution_and_code_flow.md
+++ b/qemu/roms/seabios/docs/Execution_and_code_flow.md
@@ -36,7 +36,7 @@ process.
 
 The POST phase itself has several sub-phases.
 
-* The "preinit" sub-phase: code run prior to code relocation.
+* The "preinit" sub-phase: code run prior to [code relocation](Linking overview#Code relocation).
 * The "init" sub-phase: code to initialize internal variables and
   interfaces.
 * The "setup" sub-phase: code to setup hardware and drivers.
diff --git a/qemu/roms/seabios/docs/Linking_overview.md b/qemu/roms/seabios/docs/Linking_overview.md
index fb938b632..bcb8298c3 100644
--- a/qemu/roms/seabios/docs/Linking_overview.md
+++ b/qemu/roms/seabios/docs/Linking_overview.md
@@ -92,15 +92,9 @@ those situations where an address of a C function in another mode is
 required the build supports symbols with a special "\_cfuncX_"
 prefix. The layoutrom.py script detects these references and will emit
 a corresponding symbol definitions in the linker script that points to
-the C code of the specified mode. This is typically seen with code
-like:
-
-`extern void _cfunc32flat_process_op(void);`\
-`return call32(_cfunc32flat_process_op, 0, 0);`
-
-In the above example, when the build finds the symbol
-"\_cfunc32flat_process_op" it will emit that symbol with the physical
-address of the 32bit "flat" version of the process_op() C function.
+the C code of the specified mode. The call32() and stack_hop_back()
+macros automatically add the required prefix for C code, but the
+prefixes need to be explicitly added in assembler code.
 
 Build garbage collection
 ------------------------
diff --git a/qemu/roms/seabios/docs/Releases.md b/qemu/roms/seabios/docs/Releases.md
index 6a1ecd564..c24d3c0b7 100644
--- a/qemu/roms/seabios/docs/Releases.md
+++ b/qemu/roms/seabios/docs/Releases.md
@@ -1,6 +1,25 @@
 History of SeaBIOS releases. Please see [download](Download) for
 information on obtaining these releases.
 
+SeaBIOS 1.9.0
+=============
+
+Available on 20151117. Major changes in this release:
+
+* The default boot menu key is now the ESC key (instead of F12)
+* Initial support for Trusted Platform Module (TPM) hardware and BIOS calls
+* Initial support for chain loading SeaBIOS from Grub (via multiboot
+  support)
+* Initial support for booting from SD cards on real hardware
+* virtio 1.0 device support
+* The build will no longer include the build hostname or build time on
+  "clean" builds.  This makes the build binaries more "reproducible".
+* Basic support for running SeaBIOS on Baytrail Chromebooks
+* SeaVGABIOS improvements:
+    * Improved support for old versions of x86emu (the "leal"
+      instruction is now emulated)
+* Several bug fixes and code cleanups
+
 SeaBIOS 1.8.0
 =============
 
@@ -23,6 +42,16 @@ two release numbers (eg, 1.8) and stable releases will use three
 numbers (eg, 1.8.1). The prior behavior of using a forth number
 (eg, 1.7.5.1) for stable releases will no longer be used.
 
+SeaBIOS 1.8.1
+-------------
+
+Available on 20150316. Stable release containing only bug fixes.
+
+SeaBIOS 1.8.2
+-------------
+
+Available on 20150617. Stable release containing only bug fixes.
+
 SeaBIOS 1.7.5
 =============
 
diff --git a/qemu/roms/seabios/docs/Runtime_config.md b/qemu/roms/seabios/docs/Runtime_config.md
new file mode 100644
index 000000000..d6fea2827
--- /dev/null
+++ b/qemu/roms/seabios/docs/Runtime_config.md
@@ -0,0 +1,191 @@
+SeaBIOS can read several configuration items at runtime. On coreboot
+the configuration comes from files located in CBFS. When SeaBIOS runs
+natively on QEMU the files are passed from QEMU via the fw_cfg
+interface.
+
+This page documents the user visible configuration and control
+features that SeaBIOS supports.
+
+LZMA compression
+================
+
+On coreboot, when scanning files in CBFS, any filename that ends with
+a ".lzma" suffix will be treated as a raw file that is compressed with
+the lzma compression algorithm. This works for option ROMs,
+configuration files, floppy images, etc. . (This feature should not be
+used with embedded payloads - to compress payloads, use the standard
+section based compression algorithm that is built into the payload
+specification.)
+
+For example, the file **pci1106,3344.rom.lzma** would be treated the
+same as **pci1106,3344.rom**, but will be automatically uncompressed
+when accessed.
+
+A file is typically compressed with the lzma compression command line
+tool. For example:
+
+`lzma -zc /path/to/somefile.bin > somefile.bin.lzma`
+
+However, some recent versions of lzma no longer supply an uncompressed
+file size in the lzma header. (They instead populate the field with
+zero.) Unfortunately, SeaBIOS requires the uncompressed file size, so
+it may be necessary to use a different version of the lzma tool.
+
+File aliases
+============
+
+It is possible to create the equivalent of "symbolic links" so that
+one file's content appears under another name. To do this, create a
+**links** file with one line per link and each line having the format
+of "linkname" and "destname" separated by a space character. For
+example, the **links** file may look like:
+
+```
+pci1234,1000.rom somerom.rom
+pci1234,1001.rom somerom.rom
+pci1234,1002.rom somerom.rom
+```
+
+The above example would cause SeaBIOS to treat "pci1234,1000.rom" or
+"pci1234,1001.rom" as files with the same content as the file
+"somerom.rom".
+
+Option ROMs
+===========
+
+SeaBIOS will scan all of the PCI devices in the target machine for
+option ROMs on PCI devices. It recognizes option ROMs in files that
+have the form **pciVVVV,DDDD.rom**. The VVVV,DDDD should correspond to
+the PCI vendor and device id of a device in the machine. If a given
+file is found then SeaBIOS will deploy the file instead of attempting
+to extract an option ROM from the device. In addition to supplying
+option ROMs for on-board devices that do not store their own ROMs,
+this mechanism may be used to prevent a ROM on a specific device from
+running.
+
+SeaBIOS always deploys the VGA rom associated with the active VGA
+device before any other ROMs.
+
+In addition, SeaBIOS will also run any file in the directory
+**vgaroms/** as a VGA option ROM not specific to a device and files in
+**genroms/** as a generic option ROM not specific to a device. The
+ROMS in **vgaroms/** are run immediately after running the option ROM
+associated with the primary VGA device (if any were found), and the
+**genroms/** ROMs are run after all other PCI ROMs are run.
+
+Bootsplash images
+=================
+
+SeaBIOS can show a custom [JPEG](http://en.wikipedia.org/wiki/JPEG)
+image or [BMP](http://en.wikipedia.org/wiki/BMP_file_format) image
+during bootup. To enable this, add the JPEG file to flash with the
+name **bootsplash.jpg** or BMP file as **bootsplash.bmp**.
+
+The size of the image determines the video mode to use for showing the
+image. Make sure the dimensions of the image exactly correspond to an
+available video mode (eg, 640x480, or 1024x768), otherwise it will not
+be displayed.
+
+SeaBIOS will show the image during the wait for the boot menu (if the
+boot menu has been disabled, users will not see the image). The image
+should probably have "Press ESC for boot menu" embedded in it so users
+know they can enter the normal SeaBIOS boot menu. By default, the boot
+menu prompt (and thus graphical image) is shown for 2.5 seconds. This
+can be customized via a [configuration
+parameter](#Other_Configuration_items).
+
+The JPEG viewer in SeaBIOS uses a simplified decoding algorithm. It
+supports most common JPEGs, but does not support all possible formats.
+Please see the [trouble reporting section](Debugging) if a valid image
+isn't displayed properly.
+
+Payloads
+========
+
+On coreboot, SeaBIOS will treat all files found in the **img/**
+directory as a coreboot payload. Each payload file will be available
+for boot, and one can select from the available payloads in the
+bootmenu. SeaBIOS supports both uncompressed and lzma compressed
+payloads.
+
+Floppy images
+=============
+
+It is possible to embed an image of a floppy into a file. SeaBIOS can
+then boot from and redirect floppy BIOS calls to the image. This is
+mainly useful for legacy software (such as DOS utilities). To use this
+feature, place a floppy image into the directory **floppyimg/**.
+
+Using LZMA file compression with the [.lzma file
+suffix](#LZMA_compression) is a useful way to reduce the file
+size. Several floppy formats are available: 360K, 1.2MB, 720K, 1.44MB,
+2.88MB, 160K, 180K, 320K.
+
+The floppy image will appear as writable to the system, however all
+writes are discarded on reboot.
+
+When using this system, SeaBIOS reserves high-memory to store the
+floppy. The reserved memory is then no longer available for OS use, so
+this feature should only be used when needed.
+
+Configuring boot order
+======================
+
+The **bootorder** file may be used to configure the boot up order. The
+file should be ASCII text and contain one line per boot method. The
+description of each boot method follows an [Open
+Firmware](https://secure.wikimedia.org/wikipedia/en/wiki/Open_firmware)
+device path format. SeaBIOS will attempt to boot from each item in the
+file - first line of the file first.
+
+The easiest way to find the available boot methods is to look for
+"Searching bootorder for" in the SeaBIOS debug output. For example,
+one may see lines similar to:
+
+```
+Searching bootorder for: /pci@i0cf8/*@f/drive@1/disk@0
+Searching bootorder for: /pci@i0cf8/*@f,1/drive@2/disk@1
+Searching bootorder for: /pci@i0cf8/usb@10,4/*@2
+```
+
+The above represents the patterns SeaBIOS will search for in the
+bootorder file. However, it's safe to just copy and paste the pattern
+into bootorder. For example, the file:
+
+```
+/pci@i0cf8/usb@10,4/*@2
+/pci@i0cf8/*@f/drive@1/disk@0
+```
+
+will instruct SeaBIOS to attempt to boot from the given USB drive
+first and then attempt the given ATA harddrive second.
+
+SeaBIOS also supports a special "HALT" directive. If a line that
+contains "HALT" is found in the bootorder file then SeaBIOS will (by
+default) only attempt to boot from devices explicitly listed above
+HALT in the file.
+
+Other Configuration items
+=========================
+
+There are several additional configuration options available in the
+**etc/** directory.
+
+| Filename            | Description
+|---------------------|---------------------------------------------------
+| show-boot-menu      | Controls the display of the boot menu. Set to 0 to disable the boot menu.
+| boot-menu-message   | Customize the text boot menu message. Normally, when in text mode SeaBIOS will report the string "\\nPress ESC for boot menu.\\n\\n". This field allows the string to be changed. (This is a string field, and is added as a file containing the raw string.)
+| boot-menu-key       | Controls which key activates the boot menu. The value stored is the DOS scan code (eg, 0x86 for F12, 0x01 for Esc). If this field is set, be sure to also customize the **boot-menu-message** field above.
+| boot-menu-wait      | Amount of time (in milliseconds) to wait at the boot menu prompt before selecting the default boot.
+| boot-fail-wait      | If no boot devices are found SeaBIOS will reboot after 60 seconds. Set this to the amount of time (in milliseconds) to customize the reboot delay or set to -1 to disable rebooting when no boot devices are found
+| extra-pci-roots     | If the target machine has multiple independent root buses set this to a positive value. The SeaBIOS PCI probe will then search for the given number of extra root buses.
+| ps2-keyboard-spinup | Some laptops that emulate PS2 keyboards don't respond to keyboard commands immediately after powering on. One may specify the amount of time (in milliseconds) here to allow as additional time for the keyboard to become responsive. When this field is set, SeaBIOS will repeatedly attempt to detect the keyboard until the keyboard is found or the specified timeout is reached.
+| optionroms-checksum | Option ROMs are required to have correct checksums. However, some option ROMs in the wild don't correctly follow the specifications and have bad checksums. Set this to a zero value to allow SeaBIOS to execute them anyways.
+| pci-optionrom-exec  | Controls option ROM execution for roms found on PCI devices (as opposed to roms found in CBFS/fw_cfg).  Valid values are 0: Execute no ROMs, 1: Execute only VGA ROMs, 2: Execute all ROMs. The default is 2 (execute all ROMs).
+| s3-resume-vga-init  | Set this to a non-zero value to instruct SeaBIOS to run the vga rom on an S3 resume.
+| screen-and-debug    | Set this to a zero value to instruct SeaBIOS to not write characters it sends to the screen to the debug ports. This can be useful when using sgabios.
+| advertise-serial-debug-port | If using a serial debug port, one can set this file to a zero value to prevent SeaBIOS from listing that serial port as available for operating system use. This can be useful when running old DOS programs that are known to reset the baud rate of all advertised serial ports.
+| floppy0             | Set this to the type of the first floppy drive in the system (only type 4 for 3.5 inch drives is supported).
+| floppy1             | The type of the second floppy drive in the system. See the description of **floppy0** for more info.
+| threads             | By default, SeaBIOS will parallelize hardware initialization during bootup to reduce boot time. Multiple hardware devices can be initialized in parallel between vga initialization and option rom initialization. One can set this file to a value of zero to force hardware initialization to run serially. Alternatively, one can set this file to 2 to enable early hardware initialization that runs in parallel with vga, option rom initialization, and the boot menu.
+| sdcard*             | One may create one or more files with an "sdcard" prefix (eg, "etc/sdcard0") with the physical memory address of an SDHCI controller (one memory address per file).  This may be useful for SDHCI controllers that do not appear as PCI devices, but are mapped to a consistent memory address.
diff --git a/qemu/roms/seabios/docs/SeaBIOS.md b/qemu/roms/seabios/docs/SeaBIOS.md
index 831bfced9..e24913a64 100644
--- a/qemu/roms/seabios/docs/SeaBIOS.md
+++ b/qemu/roms/seabios/docs/SeaBIOS.md
@@ -10,6 +10,8 @@ information on using SeaBIOS in coreboot. Please see the
 [releases](Releases) page for information on recent releases. See the
 [download](Download) page to obtain SeaBIOS.
 
+[SeaVGABIOS](SeaVGABIOS) is a sub-project of SeaBIOS.
+
 Please join the [mailing list](Mailinglist) to contribute to
 SeaBIOS. Information on the internals of SeaBIOS is available on the
 [Developer Documentation](Developer Documentation) page.
diff --git a/qemu/roms/seabios/docs/SeaVGABIOS.md b/qemu/roms/seabios/docs/SeaVGABIOS.md
new file mode 100644
index 000000000..7ec27804d
--- /dev/null
+++ b/qemu/roms/seabios/docs/SeaVGABIOS.md
@@ -0,0 +1,39 @@
+SeaVGABIOS is a sub-project of the SeaBIOS project - it is an open
+source implementation of a 16bit X86
+[VGA BIOS](http://en.wikipedia.org/wiki/Video_BIOS). SeaVGABIOS is the
+default VGA BIOS on [QEMU](http://www.qemu.org/). SeaVGABIOS can also
+run natively on some X86 VGA hardware with
+[coreboot](http://www.coreboot.org/).
+
+Building SeaVGABIOS
+===================
+
+To build SeaVGABIOS, obtain the [code](Download), run `make
+menuconfig` and select the type of VGA BIOS to build in the "VGA ROM"
+menu. Once selected, run `make` and the final VGA BIOS binary will be
+located in "out/vgabios.bin".
+
+The choice of available VGA BIOSes within "make menuconfig" is
+dependent on whether CONFIG_QEMU, CONFIG_COREBOOT, or CONFIG_CSM is
+selected. Also, the debug options under the "Debugging" menu apply to
+SeaVGABIOS. All other options found in "make menuconfig" apply only to
+SeaBIOS and will not impact the SeaVGABIOS build.
+
+If SeaVGABIOS is needed for multiple different devices (eg, QEMU's
+cirrus emulation and QEMU's "dispi" emulation), then one must compile
+SeaVGABIOS multiple times with the appropriate config for each build.
+
+SeaVGABIOS code
+===============
+
+The source code for SeaVGABIOS is located in the SeaBIOS
+[git repository](Download). The main VGA BIOS code is located in the
+"vgasrc/" directory. The VGA BIOS code is always compiled in 16bit
+mode.
+
+The SeaVGABIOS builds to a separate binary from the main SeaBIOS
+binary, and much of the VGA BIOS code is separate from the main BIOS
+code. However, much of the SeaBIOS
+[developer documentation](Developer_Documentation) applies to
+SeaVGABIOS. To contribute, please join the
+[SeaBIOS mailing list](Mailinglist).
diff --git a/qemu/roms/seabios/scripts/buildversion.py b/qemu/roms/seabios/scripts/buildversion.py
new file mode 100755
index 000000000..46928984e
--- /dev/null
+++ b/qemu/roms/seabios/scripts/buildversion.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+# Generate version information for a program
+#
+# Copyright (C) 2015  Kevin O'Connor <kevin@koconnor.net>
+#
+# This file may be distributed under the terms of the GNU GPLv3 license.
+import sys, os, subprocess, shlex, time, socket, optparse, logging, traceback
+
+VERSION_FORMAT = """
+/* DO NOT EDIT!  This is an autogenerated file.  See scripts/buildversion.py. */
+#define BUILD_VERSION "%s"
+#define BUILD_TOOLS "%s"
+"""
+
+# Run program and return the specified output
+def check_output(prog):
+    logging.debug("Running %s" % (repr(prog),))
+    try:
+        process = subprocess.Popen(shlex.split(prog), stdout=subprocess.PIPE)
+        output = process.communicate()[0]
+        retcode = process.poll()
+    except OSError:
+        logging.debug("Exception on run: %s" % (traceback.format_exc(),))
+        return ""
+    logging.debug("Got (code=%s): %s" % (retcode, repr(output)))
+    if retcode:
+        return ""
+    try:
+        return output.decode()
+    except UnicodeError:
+        logging.debug("Exception on decode: %s" % (traceback.format_exc(),))
+        return ""
+
+# Obtain version info from "git" program
+def git_version():
+    if not os.path.exists('.git'):
+        logging.debug("No '.git' file/directory found")
+        return ""
+    ver = check_output("git describe --tags --long --dirty").strip()
+    logging.debug("Got git version: %s" % (repr(ver),))
+    return ver
+
+# Look for version in a ".version" file.  Official release tarballs
+# have this file (see scripts/tarball.sh).
+def file_version():
+    if not os.path.isfile('.version'):
+        logging.debug("No '.version' file found")
+        return ""
+    try:
+        f = open('.version', 'r')
+        ver = f.readline().strip()
+        f.close()
+    except OSError:
+        logging.debug("Exception on read: %s" % (traceback.format_exc(),))
+        return ""
+    logging.debug("Got .version: %s" % (repr(ver),))
+    return ver
+
+# Generate an output file with the version information
+def write_version(outfile, version, toolstr):
+    logging.debug("Write file %s and %s" % (repr(version), repr(toolstr)))
+    sys.stdout.write("Version: %s\n" % (version,))
+    f = open(outfile, 'w')
+    f.write(VERSION_FORMAT % (version, toolstr))
+    f.close()
+
+# Run "tool --version" for each specified tool and extract versions
+def tool_versions(tools):
+    tools = [t.strip() for t in tools.split(';')]
+    versions = ['', '']
+    success = 0
+    for tool in tools:
+        # Extract first line from "tool --version" output
+        verstr = check_output("%s --version" % (tool,)).split('\n')[0]
+        # Check if this tool looks like a binutils program
+        isbinutils = 0
+        if verstr.startswith('GNU '):
+            isbinutils = 1
+            verstr = verstr[4:]
+        # Extract version information and exclude program name
+        if ' ' not in verstr:
+            continue
+        prog, ver = verstr.split(' ', 1)
+        if not prog or not ver:
+            continue
+        # Check for any version conflicts
+        if versions[isbinutils] and versions[isbinutils] != ver:
+            logging.debug("Mixed version %s vs %s" % (
+                repr(versions[isbinutils]), repr(ver)))
+            versions[isbinutils] = "mixed"
+            continue
+        versions[isbinutils] = ver
+        success += 1
+    cleanbuild = versions[0] and versions[1] and success == len(tools)
+    return cleanbuild, "gcc: %s binutils: %s" % (versions[0], versions[1])
+
+def main():
+    usage = "%prog [options] <outputheader.h>"
+    opts = optparse.OptionParser(usage)
+    opts.add_option("-e", "--extra", dest="extra", default="",
+                    help="extra version string to append to version")
+    opts.add_option("-t", "--tools", dest="tools", default="",
+                    help="list of build programs to extract version from")
+    opts.add_option("-v", action="store_true", dest="verbose",
+                    help="enable debug messages")
+
+    options, args = opts.parse_args()
+    if len(args) != 1:
+        opts.error("Incorrect arguments")
+    outfile = args[0]
+    if options.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    cleanbuild, toolstr = tool_versions(options.tools)
+
+    ver = git_version()
+    cleanbuild = cleanbuild and 'dirty' not in ver
+    if not ver:
+        ver = file_version()
+        # We expect the "extra version" to contain information on the
+        # distributor and distribution package version (if
+        # applicable).  It is a "clean" build if this is a build from
+        # an official release tarball and the above info is present.
+        cleanbuild = cleanbuild and ver and options.extra != ""
+        if not ver:
+            ver = "?"
+    if not cleanbuild:
+        btime = time.strftime("%Y%m%d_%H%M%S")
+        hostname = socket.gethostname()
+        ver = "%s-%s-%s" % (ver, btime, hostname)
+    write_version(outfile, ver + options.extra, toolstr)
+
+if __name__ == '__main__':
+    main()
diff --git a/qemu/roms/seabios/scripts/buildversion.sh b/qemu/roms/seabios/scripts/buildversion.sh
deleted file mode 100755
index 516aff5b2..000000000
--- a/qemu/roms/seabios/scripts/buildversion.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/sh
-# Script to generate a C file with version information.
-OUTFILE="$1"
-VAR16MODE="$2"
-
-# Extract version info
-if [ -z "$BUILD_VERSION" ]; then
-    if [ -d .git -o -f .git ]; then
-        VERSION="`git describe --tags --long --dirty`"
-    elif [ -f .version ]; then
-        VERSION="`cat .version`"
-    else
-        VERSION="?"
-    fi
-    VERSION="${VERSION}-`date +"%Y%m%d_%H%M%S"`-`hostname`"
-else
-    VERSION="$BUILD_VERSION"
-fi
-echo "Version: ${VERSION}"
-
-# Build header file
-if [ "$VAR16MODE" = "VAR16" ]; then
-    cat > ${OUTFILE} <<EOF
-#include "types.h"
-char VERSION[] VAR16 = "${VERSION}";
-EOF
-else
-    cat > ${OUTFILE} <<EOF
-char VERSION[] = "${VERSION}";
-EOF
-fi
diff --git a/qemu/roms/seabios/scripts/checkrom.py b/qemu/roms/seabios/scripts/checkrom.py
index 377277db5..aced5e2cf 100755
--- a/qemu/roms/seabios/scripts/checkrom.py
+++ b/qemu/roms/seabios/scripts/checkrom.py
@@ -39,7 +39,7 @@ def main():
                 finalsize = 256*1024
     if datasize > finalsize:
         print("Error!  ROM doesn't fit (%d > %d)" % (datasize, finalsize))
-        print("   You have to either increate the size (CONFIG_ROM_SIZE)")
+        print("   You have to either increase the size (CONFIG_ROM_SIZE)")
         print("   or turn off some features (such as hardware support not")
         print("   needed) to make it fit.  Trying a more recent gcc version")
         print("   might work too.")
diff --git a/qemu/roms/seabios/scripts/checkstack.py b/qemu/roms/seabios/scripts/checkstack.py
index b49b6c8cc..5d9b0bfaf 100755
--- a/qemu/roms/seabios/scripts/checkstack.py
+++ b/qemu/roms/seabios/scripts/checkstack.py
@@ -2,7 +2,7 @@
 # Script that tries to find how much stack space each function in an
 # object is using.
 #
-# Copyright (C) 2008  Kevin O'Connor <kevin@koconnor.net>
+# Copyright (C) 2008-2015  Kevin O'Connor <kevin@koconnor.net>
 #
 # This file may be distributed under the terms of the GNU GPLv3 license.
 
@@ -26,85 +26,84 @@ OUTPUTDESC = """
 #    insn_addr:called_function [u+c,t,usage_to_yield_point]
 """
 
+class function:
+    def __init__(self, funcaddr, funcname):
+        self.funcaddr = funcaddr
+        self.funcname = funcname
+        self.basic_stack_usage = 0
+        self.max_stack_usage = None
+        self.yield_usage = -1
+        self.max_yield_usage = None
+        self.total_calls = 0
+        # called_funcs = [(insnaddr, calladdr, stackusage), ...]
+        self.called_funcs = []
+        self.subfuncs = {}
+    # Update function info with a found "yield" point.
+    def noteYield(self, stackusage):
+        if self.yield_usage < stackusage:
+            self.yield_usage = stackusage
+    # Update function info with a found "call" point.
+    def noteCall(self, insnaddr, calladdr, stackusage):
+        if (calladdr, stackusage) in self.subfuncs:
+            # Already noted a nearly identical call - ignore this one.
+            return
+        self.called_funcs.append((insnaddr, calladdr, stackusage))
+        self.subfuncs[(calladdr, stackusage)] = 1
+
 # Find out maximum stack usage for a function
-def calcmaxstack(funcs, funcaddr):
-    info = funcs[funcaddr]
-    # Find max of all nested calls.
-    maxusage = info[1]
-    maxyieldusage = doesyield = 0
-    if info[3] is not None:
-        maxyieldusage = info[3]
-        doesyield = 1
-    info[2] = maxusage
-    info[4] = info[3]
+def calcmaxstack(info, funcs):
+    if info.max_stack_usage is not None:
+        return
+    info.max_stack_usage = max_stack_usage = info.basic_stack_usage
+    info.max_yield_usage = max_yield_usage = info.yield_usage
+    total_calls = 0
     seenbefore = {}
-    totcalls = 0
-    for insnaddr, calladdr, usage in info[6]:
+    # Find max of all nested calls.
+    for insnaddr, calladdr, usage in info.called_funcs:
         callinfo = funcs.get(calladdr)
         if callinfo is None:
             continue
-        if callinfo[2] is None:
-            calcmaxstack(funcs, calladdr)
-        if callinfo[0] not in seenbefore:
-            seenbefore[callinfo[0]] = 1
-            totcalls += 1 + callinfo[5]
-        funcnameroot = callinfo[0].split('.')[0]
+        calcmaxstack(callinfo, funcs)
+        if callinfo.funcname not in seenbefore:
+            seenbefore[callinfo.funcname] = 1
+            total_calls += callinfo.total_calls + 1
+        funcnameroot = callinfo.funcname.split('.')[0]
         if funcnameroot in IGNORE:
             # This called function is ignored - don't contribute it to
             # the max stack.
             continue
+        totusage = usage + callinfo.max_stack_usage
+        totyieldusage = usage + callinfo.max_yield_usage
         if funcnameroot in STACKHOP:
-            if usage > maxusage:
-                maxusage = usage
-            if callinfo[4] is not None:
-                doesyield = 1
-                if usage > maxyieldusage:
-                    maxyieldusage = usage
-            continue
-        totusage = usage + callinfo[2]
-        if totusage > maxusage:
-            maxusage = totusage
-        if callinfo[4] is not None:
-            doesyield = 1
-            totyieldusage = usage + callinfo[4]
-            if totyieldusage > maxyieldusage:
-                maxyieldusage = totyieldusage
-    info[2] = maxusage
-    if doesyield:
-        info[4] = maxyieldusage
-    info[5] = totcalls
+            # Don't count children of this function
+            totusage = totyieldusage = usage
+        if totusage > max_stack_usage:
+            max_stack_usage = totusage
+        if callinfo.max_yield_usage >= 0 and totyieldusage > max_yield_usage:
+            max_yield_usage = totyieldusage
+    info.max_stack_usage = max_stack_usage
+    info.max_yield_usage = max_yield_usage
+    info.total_calls = total_calls
 
 # Try to arrange output so that functions that call each other are
 # near each other.
 def orderfuncs(funcaddrs, availfuncs):
-    l = [(availfuncs[funcaddr][5], availfuncs[funcaddr][0], funcaddr)
+    l = [(availfuncs[funcaddr].total_calls
+          , availfuncs[funcaddr].funcname, funcaddr)
          for funcaddr in funcaddrs if funcaddr in availfuncs]
     l.sort()
     l.reverse()
     out = []
     while l:
         count, name, funcaddr = l.pop(0)
-        if funcaddr not in availfuncs:
+        info = availfuncs.get(funcaddr)
+        if info is None:
             continue
-        calladdrs = [calls[1] for calls in availfuncs[funcaddr][6]]
+        calladdrs = [calls[1] for calls in info.called_funcs]
         del availfuncs[funcaddr]
-        out = out + orderfuncs(calladdrs, availfuncs) + [funcaddr]
+        out = out + orderfuncs(calladdrs, availfuncs) + [info]
     return out
 
-# Update function info with a found "yield" point.
-def noteYield(info, stackusage):
-    prevyield = info[3]
-    if prevyield is None or prevyield < stackusage:
-        info[3] = stackusage
-
-# Update function info with a found "call" point.
-def noteCall(info, subfuncs, insnaddr, calladdr, stackusage):
-    if (calladdr, stackusage) in subfuncs:
-        # Already noted a nearly identical call - ignore this one.
-        return
-    info[6].append((insnaddr, calladdr, stackusage))
-    subfuncs[(calladdr, stackusage)] = 1
-
 hex_s = r'[0-9a-f]+'
 re_func = re.compile(r'^(?P<funcaddr>' + hex_s + r') <(?P<func>.*)>:$')
 re_asm = re.compile(
@@ -114,11 +113,12 @@ re_asm = re.compile(
 re_usestack = re.compile(
     r'^(push[f]?[lw])|(sub.* [$](?P<num>0x' + hex_s + r'),%esp)$')
 
-def calc():
-    # funcs[funcaddr] = [funcname, basicstackusage, maxstackusage
-    #                    , yieldusage, maxyieldusage, totalcalls
-    #                    , [(insnaddr, calladdr, stackusage), ...]]
-    funcs = {-1: ['<indirect>', 0, 0, None, None, 0, []]}
+def main():
+    unknownfunc = function(None, "<unknown>")
+    indirectfunc = function(-1, '<indirect>')
+    unknownfunc.max_stack_usage = indirectfunc.max_stack_usage = 0
+    unknownfunc.max_yield_usage = indirectfunc.max_yield_usage = -1
+    funcs = {-1: indirectfunc}
     cur = None
     atstart = 0
     stackusage = 0
@@ -129,99 +129,93 @@ def calc():
         if m is not None:
             # Found function
             funcaddr = int(m.group('funcaddr'), 16)
-            funcs[funcaddr] = cur = [m.group('func'), 0, None, None, None, 0, []]
+            funcs[funcaddr] = cur = function(funcaddr, m.group('func'))
             stackusage = 0
             atstart = 1
-            subfuncs = {}
             continue
         m = re_asm.match(line)
-        if m is not None:
-            insn = m.group('insn')
-
-            im = re_usestack.match(insn)
-            if im is not None:
-                if insn.startswith('pushl') or insn.startswith('pushfl'):
-                    stackusage += 4
-                    continue
-                elif insn.startswith('pushw') or insn.startswith('pushfw'):
-                    stackusage += 2
-                    continue
-                stackusage += int(im.group('num'), 16)
-
-            if atstart:
-                if '%esp' in insn or insn.startswith('leal'):
-                    # Still part of initial header
-                    continue
-                cur[1] = stackusage
-                atstart = 0
-
-            insnaddr = m.group('insnaddr')
-            calladdr = m.group('calladdr')
-            if calladdr is None:
-                if insn.startswith('lcallw'):
-                    noteCall(cur, subfuncs, insnaddr, -1, stackusage + 4)
-                    noteYield(cur, stackusage + 4)
-                elif insn.startswith('int'):
-                    noteCall(cur, subfuncs, insnaddr, -1, stackusage + 6)
-                    noteYield(cur, stackusage + 6)
-                elif insn.startswith('sti'):
-                    noteYield(cur, stackusage)
-                else:
-                    # misc instruction
-                    continue
+        if m is None:
+            #print("other", repr(line))
+            continue
+        insn = m.group('insn')
+
+        im = re_usestack.match(insn)
+        if im is not None:
+            if insn.startswith('pushl') or insn.startswith('pushfl'):
+                stackusage += 4
+                continue
+            elif insn.startswith('pushw') or insn.startswith('pushfw'):
+                stackusage += 2
+                continue
+            stackusage += int(im.group('num'), 16)
+
+        if atstart:
+            if '%esp' in insn or insn.startswith('leal'):
+                # Still part of initial header
+                continue
+            cur.basic_stack_usage = stackusage
+            atstart = 0
+
+        insnaddr = m.group('insnaddr')
+        calladdr = m.group('calladdr')
+        if calladdr is None:
+            if insn.startswith('lcallw'):
+                cur.noteCall(insnaddr, -1, stackusage + 4)
+                cur.noteYield(stackusage + 4)
+            elif insn.startswith('int'):
+                cur.noteCall(insnaddr, -1, stackusage + 6)
+                cur.noteYield(stackusage + 6)
+            elif insn.startswith('sti'):
+                cur.noteYield(stackusage)
+            else:
+                # misc instruction
+                continue
+        else:
+            # Jump or call insn
+            calladdr = int(calladdr, 16)
+            ref = m.group('ref')
+            if '+' in ref:
+                # Inter-function jump.
+                pass
+            elif insn.startswith('j'):
+                # Tail call
+                cur.noteCall(insnaddr, calladdr, 0)
+            elif insn.startswith('calll'):
+                cur.noteCall(insnaddr, calladdr, stackusage + 4)
+            elif insn.startswith('callw'):
+                cur.noteCall(insnaddr, calladdr, stackusage + 2)
             else:
-                # Jump or call insn
-                calladdr = int(calladdr, 16)
-                ref = m.group('ref')
-                if '+' in ref:
-                    # Inter-function jump.
-                    pass
-                elif insn.startswith('j'):
-                    # Tail call
-                    noteCall(cur, subfuncs, insnaddr, calladdr, 0)
-                elif insn.startswith('calll'):
-                    noteCall(cur, subfuncs, insnaddr, calladdr, stackusage + 4)
-                elif insn.startswith('callw'):
-                    noteCall(cur, subfuncs, insnaddr, calladdr, stackusage + 2)
-                else:
-                    print("unknown call", ref)
-                    noteCall(cur, subfuncs, insnaddr, calladdr, stackusage)
-            # Reset stack usage to preamble usage
-            stackusage = cur[1]
-
-        #print("other", repr(line))
+                print("unknown call", ref)
+                cur.noteCall(insnaddr, calladdr, stackusage)
+        # Reset stack usage to preamble usage
+        stackusage = cur.basic_stack_usage
 
     # Calculate maxstackusage
-    for funcaddr, info in funcs.items():
-        if info[2] is not None:
-            continue
-        calcmaxstack(funcs, funcaddr)
+    for info in funcs.values():
+        calcmaxstack(info, funcs)
 
     # Sort functions for output
-    funcaddrs = orderfuncs(funcs.keys(), funcs.copy())
+    funcinfos = orderfuncs(funcs.keys(), funcs.copy())
 
     # Show all functions
     print(OUTPUTDESC)
-    for funcaddr in funcaddrs:
-        name, basicusage, maxusage, yieldusage, maxyieldusage, count, calls = \
-            funcs[funcaddr]
-        if maxusage == 0 and maxyieldusage is None:
+    for info in funcinfos:
+        if info.max_stack_usage == 0 and info.max_yield_usage < 0:
             continue
         yieldstr = ""
-        if maxyieldusage is not None:
-            yieldstr = ",%d" % maxyieldusage
-        print("\n%s[%d,%d%s]:" % (name, basicusage, maxusage, yieldstr))
-        for insnaddr, calladdr, stackusage in calls:
-            callinfo = funcs.get(calladdr, ("<unknown>", 0, 0, 0, None))
+        if info.max_yield_usage >= 0:
+            yieldstr = ",%d" % info.max_yield_usage
+        print("\n%s[%d,%d%s]:" % (info.funcname, info.basic_stack_usage
+                                  , info.max_stack_usage, yieldstr))
+        for insnaddr, calladdr, stackusage in info.called_funcs:
+            callinfo = funcs.get(calladdr, unknownfunc)
             yieldstr = ""
-            if callinfo[4] is not None:
-                yieldstr = ",%d" % (stackusage + callinfo[4])
+            if callinfo.max_yield_usage >= 0:
+                yieldstr = ",%d" % (stackusage + callinfo.max_yield_usage)
             print("    %04s:%-40s [%d+%d,%d%s]" % (
-                insnaddr, callinfo[0], stackusage, callinfo[1]
-                , stackusage+callinfo[2], yieldstr))
-
-def main():
-    calc()
+                insnaddr, callinfo.funcname, stackusage
+                , callinfo.basic_stack_usage
+                , stackusage+callinfo.max_stack_usage, yieldstr))
 
 if __name__ == '__main__':
     main()
diff --git a/qemu/roms/seabios/scripts/kconfig/lxdialog/util.c b/qemu/roms/seabios/scripts/kconfig/lxdialog/util.c
index f7abdeb92..2a0d182e8 100644
--- a/qemu/roms/seabios/scripts/kconfig/lxdialog/util.c
+++ b/qemu/roms/seabios/scripts/kconfig/lxdialog/util.c
@@ -376,7 +376,7 @@ void print_title(WINDOW *dialog, const char *title, int width)
 /*
  * Print a string of text in a window, automatically wrap around to the
  * next line if the string is too long to fit on one line. Newline
- * characters '\n' are propperly processed.  We start on a new line
+ * characters '\n' are properly processed.  We start on a new line
  * if there is no room for at least 4 nonblanks following a double-space.
  */
 void print_autowrap(WINDOW * win, const char *prompt, int width, int y, int x)
diff --git a/qemu/roms/seabios/scripts/layoutrom.py b/qemu/roms/seabios/scripts/layoutrom.py
index dd770fe49..b976fb056 100755
--- a/qemu/roms/seabios/scripts/layoutrom.py
+++ b/qemu/roms/seabios/scripts/layoutrom.py
@@ -161,6 +161,7 @@ def getSectionsPrefix(sections, prefix):
 # The sections (and associated information) to be placed in output rom
 class LayoutInfo:
     sections = None
+    config = None
     genreloc = None
     sec32init_start = sec32init_end = sec32init_align = None
     sec32low_start = sec32low_end = None
@@ -172,6 +173,7 @@ class LayoutInfo:
 # Determine final memory addresses for sections
 def doLayout(sections, config, genreloc):
     li = LayoutInfo()
+    li.config = config
     li.sections = sections
     li.genreloc = genreloc
     # Determine 16bit positions
@@ -399,6 +401,10 @@ def writeLinkerScripts(li, out16, out32seg, out32flat):
     filesections32flat = getSectionsFileid(li.sections, '32flat')
     out = outXRefs([], exportsyms=li.varlowsyms
                    , forcedelta=li.final_sec32low_start-li.sec32low_start)
+    multiboot_header = ""
+    if li.config.get('CONFIG_MULTIBOOT'):
+        multiboot_header = "LONG(0x1BADB002) LONG(0) LONG(-0x1BADB002)"
+        sec32all_start -= 3 * 4
     out += outXRefs(filesections32flat, exportsyms=[li.entrysym]) + """
     _reloc_min_align = 0x%x ;
     zonefseg_start = 0x%x ;
@@ -415,6 +421,7 @@ def writeLinkerScripts(li, out16, out32seg, out32flat):
     .text code32flat_start : {
 %s
 %s
+%s
         code32flat_end = ABSOLUTE(.) ;
     } :text
 """ % (li.sec32init_align,
@@ -428,6 +435,7 @@ def writeLinkerScripts(li, out16, out32seg, out32flat):
        li.sec32init_start,
        li.sec32init_end,
        sec32all_start,
+       multiboot_header,
        relocstr,
        outRelSections(li.sections, 'code32flat_start'))
     out = COMMONHEADER + out + COMMONTRAILER + """
diff --git a/qemu/roms/seabios/src/Kconfig b/qemu/roms/seabios/src/Kconfig
index 45ca59cf3..95bf087b8 100644
--- a/qemu/roms/seabios/src/Kconfig
+++ b/qemu/roms/seabios/src/Kconfig
@@ -20,7 +20,7 @@ choice
             Configure for an emulated machine (QEMU, Xen, KVM, or Bochs).
 
     config CSM
-       bool "Build as Compatibilty Support Module for EFI BIOS"
+       bool "Build as Compatibility Support Module for EFI BIOS"
        help
            Configure to be used by EFI firmware as Compatibility Support
            module (CSM) to provide legacy BIOS services.
@@ -96,12 +96,13 @@ endchoice
             the CBFS filesystem is at a non-standard location (eg,
             0xffe00000 if CBFS ends 2Meg below the end of flash).
 
-    config FLASH_FLOPPY
-        depends on COREBOOT_FLASH
-        bool "Floppy images in CBFS"
+    config MULTIBOOT
+        depends on COREBOOT
+        bool "multiboot support"
         default y
         help
-            Support floppy images in coreboot flash.
+            Add multiboot header in bios.bin.raw and accept files supplied
+            as multiboot modules.
     config ENTRY_EXTRASTACK
         bool "Use internal stack for 16bit interrupt entry points"
         default y
@@ -160,7 +161,7 @@ menu "Hardware support"
         help
             Support for AHCI disk code.
     config SDCARD
-        depends on DRIVES && QEMU_HARDWARE
+        depends on DRIVES
         bool "SD controllers"
         default y
         help
@@ -208,11 +209,18 @@ menu "Hardware support"
         help
             Support boot from LSI MegaRAID SAS scsi storage.
     config FLOPPY
-        depends on DRIVES
+        depends on DRIVES && HARDWARE_IRQ
         bool "Floppy controller"
         default y
         help
             Support floppy drive access.
+    config FLASH_FLOPPY
+        depends on DRIVES
+        bool "Floppy images from CBFS or fw_cfg"
+        default y
+        help
+            Support floppy images stored in coreboot flash or from
+            QEMU fw_cfg.
 
     config PS2PORT
         depends on KEYBOARD || MOUSE
@@ -291,6 +299,26 @@ menu "Hardware support"
         default y
         help
             Support parallel ports. This also enables int 17 parallel port calls.
+    config RTC_TIMER
+        bool "Real Time Clock (RTC) scheduling"
+        depends on HARDWARE_IRQ
+        default y
+        help
+            Support MC146818 Real Time Clock chip timer
+            interrupts. This also enables int 1583 and int 1586 calls.
+
+            Disabling this support does not disable access to the RTC
+            cmos registers.
+
+    config HARDWARE_IRQ
+        bool "Hardware interrupts"
+        default y
+        help
+            Program and support hardware interrupts using the i8259
+            programmable interrupt controller (PIC).  This option must
+            be enabled in order to support most boot loaders.  Only
+            disable this option if running on peculiar hardware known
+            not to support irq routing.
 
     config USE_SMM
         depends on QEMU
@@ -309,10 +337,16 @@ menu "Hardware support"
         help
             Initialize the Memory Type Range Registers (on emulators).
     config PMTIMER
-        bool "Use ACPI timer"
+        bool "Support ACPI timer"
         default y
         help
-            Use the ACPI timer instead of the TSC for timekeeping (on qemu).
+            Detect and use the ACPI timer for timekeeping.
+    config TSC_TIMER
+        bool "Support CPU timestamp counter as timer"
+        default y
+        help
+            Support for using the CPU timestamp counter as an internal
+            timing source.
 endmenu
 
 menu "BIOS interfaces"
@@ -421,6 +455,13 @@ menu "BIOS interfaces"
             modified by programs.  However, some old DOS high memory
             managers may require the UMB region to be read-only.
 
+    config TCGBIOS
+        depends on S3_RESUME
+        bool "TPM support and TCG BIOS extensions"
+        default y
+        help
+            Provide TPM support along with TCG BIOS extensions
+
 endmenu
 
 menu "BIOS Tables"
diff --git a/qemu/roms/seabios/src/biosvar.h b/qemu/roms/seabios/src/biosvar.h
index 58bcbcedb..f61fb6a50 100644
--- a/qemu/roms/seabios/src/biosvar.h
+++ b/qemu/roms/seabios/src/biosvar.h
@@ -8,11 +8,12 @@
 
 #include "config.h" // SEG_BDA
 #include "farptr.h" // GET_FARVAR
+#include "memmap.h" // SYMBOL
 #include "std/bda.h" // struct bios_data_area_s
 
 
 /****************************************************************
- * Interupt vector table
+ * Interrupt vector table
  ****************************************************************/
 
 #define GET_IVT(vector)                                         \
@@ -112,13 +113,12 @@ static inline u16 get_global_seg(void) {
  * "Low" memory variables
  ****************************************************************/
 
-extern u8 _zonelow_seg, zonelow_base[];
-#define SEG_LOW ((u32)&_zonelow_seg)
+#define SEG_LOW SYMBOL(_zonelow_seg)
 
 #if MODESEGMENT
 #define GET_LOW(var)            GET_FARVAR(SEG_LOW, (var))
 #define SET_LOW(var, val)       SET_FARVAR(SEG_LOW, (var), (val))
-#define LOWFLAT2LOW(var) ((typeof(var))((void*)(var) - (u32)zonelow_base))
+#define LOWFLAT2LOW(var) ((typeof(var))((void*)(var) - SYMBOL(zonelow_base)))
 #else
 #define GET_LOW(var)            (var)
 #define SET_LOW(var, val)       do { (var) = (val); } while (0)
diff --git a/qemu/roms/seabios/src/block.c b/qemu/roms/seabios/src/block.c
index 3f7ecb1d7..1762e2a33 100644
--- a/qemu/roms/seabios/src/block.c
+++ b/qemu/roms/seabios/src/block.c
@@ -10,9 +10,16 @@
 #include "hw/ata.h" // process_ata_op
 #include "hw/ahci.h" // process_ahci_op
 #include "hw/blockcmd.h" // cdb_*
+#include "hw/esp-scsi.h" // esp_scsi_process_op
+#include "hw/lsi-scsi.h" // lsi_scsi_process_op
+#include "hw/megasas.h" // megasas_process_op
 #include "hw/pci.h" // pci_bdf_to_bus
+#include "hw/pvscsi.h" // pvscsi_process_op
 #include "hw/rtc.h" // rtc_read
+#include "hw/usb-msc.h" // usb_process_op
+#include "hw/usb-uas.h" // uas_process_op
 #include "hw/virtio-blk.h" // process_virtio_blk_op
+#include "hw/virtio-scsi.h" // virtio_scsi_process_op
 #include "malloc.h" // malloc_low
 #include "output.h" // dprintf
 #include "stacks.h" // stack_hop
@@ -67,10 +74,8 @@ get_translation(struct drive_s *drive)
     u8 type = drive->type;
     if (CONFIG_QEMU && type == DTYPE_ATA) {
         // Emulators pass in the translation info via nvram.
-        u8 ataid = drive->cntl_id;
-        u8 channel = ataid / 2;
-        u8 translation = rtc_read(CMOS_BIOS_DISKTRANSFLAG + channel/2);
-        translation >>= 2 * (ataid % 4);
+        u8 translation = rtc_read(CMOS_BIOS_DISKTRANSFLAG + drive->cntl_id/4);
+        translation >>= 2 * (drive->cntl_id % 4);
         translation &= 0x03;
         return translation;
     }
@@ -282,11 +287,21 @@ map_floppy_drive(struct drive_s *drive)
  * Extended Disk Drive (EDD) get drive parameters
  ****************************************************************/
 
+// flags for bus_iface field in fill_generic_edd()
+#define EDD_ISA        0x01
+#define EDD_PCI        0x02
+#define EDD_BUS_MASK   0x0f
+#define EDD_ATA        0x10
+#define EDD_SCSI       0x20
+#define EDD_IFACE_MASK 0xf0
+
+// Fill in EDD info
 static int
-fill_generic_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf
-                 , u32 dpte_so, char *iface_type
-                 , int bdf, u8 channel, u16 iobase, u64 device_path)
+fill_generic_edd(struct segoff_s edd, struct drive_s *drive_gf
+                 , u32 dpte_so, u8 bus_iface, u32 iface_path, u32 device_path)
 {
+    u16 seg = edd.seg;
+    struct int13dpt_s *param_far = (void*)(edd.offset+0);
     u16 size = GET_FARVAR(seg, param_far->size);
     u16 t13 = size == 74;
 
@@ -335,7 +350,7 @@ fill_generic_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf
     SET_FARVAR(seg, param_far->size, 30);
     SET_FARVAR(seg, param_far->dpte.segoff, dpte_so);
 
-    if (size < 66 || !iface_type)
+    if (size < 66 || !bus_iface)
         return DISK_RET_SUCCESS;
 
     // EDD 3.x
@@ -344,32 +359,22 @@ fill_generic_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf
     SET_FARVAR(seg, param_far->reserved1, 0);
     SET_FARVAR(seg, param_far->reserved2, 0);
 
-    int i;
-    for (i=0; i<sizeof(param_far->iface_type); i++)
-        SET_FARVAR(seg, param_far->iface_type[i], GET_GLOBAL(iface_type[i]));
-
-    if (bdf != -1) {
-        SET_FARVAR(seg, param_far->host_bus[0], 'P');
-        SET_FARVAR(seg, param_far->host_bus[1], 'C');
-        SET_FARVAR(seg, param_far->host_bus[2], 'I');
-        SET_FARVAR(seg, param_far->host_bus[3], ' ');
-
-        u32 path = (pci_bdf_to_bus(bdf) | (pci_bdf_to_dev(bdf) << 8)
-                    | (pci_bdf_to_fn(bdf) << 16));
-        if (t13)
-            path |= channel << 24;
-
-        SET_FARVAR(seg, param_far->iface_path, path);
-    } else {
-        // ISA
-        SET_FARVAR(seg, param_far->host_bus[0], 'I');
-        SET_FARVAR(seg, param_far->host_bus[1], 'S');
-        SET_FARVAR(seg, param_far->host_bus[2], 'A');
-        SET_FARVAR(seg, param_far->host_bus[3], ' ');
-
-        SET_FARVAR(seg, param_far->iface_path, iobase);
+    const char *host_bus = "ISA ";
+    if ((bus_iface & EDD_BUS_MASK) == EDD_PCI) {
+        host_bus = "PCI ";
+        if (!t13)
+            // Phoenix v3 spec (pre t13) did not define the PCI channel field
+            iface_path &= 0x00ffffff;
     }
-
+    memcpy_far(seg, param_far->host_bus, SEG_BIOS, host_bus
+               , sizeof(param_far->host_bus));
+    SET_FARVAR(seg, param_far->iface_path, iface_path);
+
+    const char *iface_type = "ATA     ";
+    if ((bus_iface & EDD_IFACE_MASK) == EDD_SCSI)
+        iface_type = "SCSI    ";
+    memcpy_far(seg, param_far->iface_type, SEG_BIOS, iface_type
+               , sizeof(param_far->iface_type));
     if (t13) {
         SET_FARVAR(seg, param_far->t13.device_path[0], device_path);
         SET_FARVAR(seg, param_far->t13.device_path[1], 0);
@@ -386,10 +391,19 @@ fill_generic_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf
     return DISK_RET_SUCCESS;
 }
 
+// Build an EDD "iface_path" field for a PCI device
+static u32
+edd_pci_path(u16 bdf, u8 channel)
+{
+    return (pci_bdf_to_bus(bdf) | (pci_bdf_to_dev(bdf) << 8)
+            | (pci_bdf_to_fn(bdf) << 16) | ((u32)channel << 24));
+}
+
 struct dpte_s DefaultDPTE VARLOW;
 
+// EDD info for ATA and ATAPI drives
 static int
-fill_ata_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf)
+fill_ata_edd(struct segoff_s edd, struct drive_s *drive_gf)
 {
     if (!CONFIG_ATA)
         return DISK_RET_EPARAM;
@@ -440,109 +454,141 @@ fill_ata_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf)
     u8 sum = checksum_far(SEG_LOW, &DefaultDPTE, 15);
     SET_LOW(DefaultDPTE.checksum, -sum);
 
+    u32 bustype = EDD_ISA, ifpath = iobase1;
+    if (bdf >= 0) {
+        bustype = EDD_PCI;
+        ifpath = edd_pci_path(bdf, channel);
+    }
     return fill_generic_edd(
-        seg, param_far, drive_gf, SEGOFF(SEG_LOW, (u32)&DefaultDPTE).segoff
-        , "ATA     ", bdf, channel, iobase1, slave);
+        edd, drive_gf, SEGOFF(SEG_LOW, (u32)&DefaultDPTE).segoff
+        , bustype | EDD_ATA, ifpath, slave);
 }
 
+// Fill Extended Disk Drive (EDD) "Get drive parameters" info for a drive
 int noinline
-fill_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf)
+fill_edd(struct segoff_s edd, struct drive_s *drive_gf)
 {
     switch (GET_GLOBALFLAT(drive_gf->type)) {
     case DTYPE_ATA:
     case DTYPE_ATA_ATAPI:
-        return fill_ata_edd(seg, param_far, drive_gf);
+        return fill_ata_edd(edd, drive_gf);
     case DTYPE_VIRTIO_BLK:
     case DTYPE_VIRTIO_SCSI:
         return fill_generic_edd(
-            seg, param_far, drive_gf, 0xffffffff
-            , "SCSI    ", GET_GLOBALFLAT(drive_gf->cntl_id), 0, 0, 0);
+            edd, drive_gf, 0xffffffff, EDD_PCI | EDD_SCSI
+            , edd_pci_path(GET_GLOBALFLAT(drive_gf->cntl_id), 0), 0);
     default:
-        return fill_generic_edd(seg, param_far, drive_gf, 0, NULL, 0, 0, 0, 0);
+        return fill_generic_edd(edd, drive_gf, 0, 0, 0, 0);
     }
 }
 
 
 /****************************************************************
- * 16bit calling interface
+ * Disk driver dispatch
  ****************************************************************/
 
-int VISIBLE32FLAT
-process_atapi_op(struct disk_op_s *op)
+// Fallback handler for command requests not implemented by drivers
+int
+default_process_op(struct disk_op_s *op)
 {
     switch (op->command) {
-    case CMD_WRITE:
     case CMD_FORMAT:
-        return DISK_RET_EWRITEPROTECT;
+    case CMD_RESET:
+    case CMD_ISREADY:
+    case CMD_VERIFY:
+    case CMD_SEEK:
+        // Return success if the driver doesn't implement these commands
+        return DISK_RET_SUCCESS;
     default:
-        return scsi_process_op(op);
+        return DISK_RET_EPARAM;
     }
 }
 
-// Execute a disk_op request.
-int
-process_op(struct disk_op_s *op)
+// Command dispatch for disk drivers that run in both 16bit and 32bit mode
+static int
+process_op_both(struct disk_op_s *op)
 {
-    ASSERT16();
-    int ret, origcount = op->count;
-    if (origcount * GET_GLOBALFLAT(op->drive_gf->blksize) > 64*1024) {
-        op->count = 0;
-        return DISK_RET_EBOUNDARY;
-    }
-    u8 type = GET_GLOBALFLAT(op->drive_gf->type);
-    switch (type) {
-    case DTYPE_FLOPPY:
-        ret = process_floppy_op(op);
-        break;
-    case DTYPE_ATA:
-        ret = process_ata_op(op);
-        break;
-    case DTYPE_RAMDISK:
-        ret = process_ramdisk_op(op);
-        break;
-    case DTYPE_CDEMU:
-        ret = process_cdemu_op(op);
-        break;
-    case DTYPE_VIRTIO_BLK:
-        ret = process_virtio_blk_op(op);
-        break;
-    case DTYPE_AHCI: ;
-        extern void _cfunc32flat_process_ahci_op(void);
-        ret = call32(_cfunc32flat_process_ahci_op
-                     , (u32)MAKE_FLATPTR(GET_SEG(SS), op), DISK_RET_EPARAM);
-        break;
+    switch (GET_GLOBALFLAT(op->drive_gf->type)) {
     case DTYPE_ATA_ATAPI:
-        ret = process_atapi_op(op);
-        break;
-    case DTYPE_AHCI_ATAPI: ;
-        extern void _cfunc32flat_process_atapi_op(void);
-        ret = call32(_cfunc32flat_process_atapi_op
-                     , (u32)MAKE_FLATPTR(GET_SEG(SS), op), DISK_RET_EPARAM);
-        break;
-    case DTYPE_SDCARD: ;
-        extern void _cfunc32flat_process_sdcard_op(void);
-        ret = call32(_cfunc32flat_process_sdcard_op
-                     , (u32)MAKE_FLATPTR(GET_SEG(SS), op), DISK_RET_EPARAM);
-        break;
+        return ata_atapi_process_op(op);
     case DTYPE_USB:
+        return usb_process_op(op);
     case DTYPE_UAS:
-    case DTYPE_VIRTIO_SCSI:
+        return uas_process_op(op);
     case DTYPE_LSI_SCSI:
+        return lsi_scsi_process_op(op);
     case DTYPE_ESP_SCSI:
+        return esp_scsi_process_op(op);
     case DTYPE_MEGASAS:
-        ret = scsi_process_op(op);
-        break;
+        return megasas_process_op(op);
+    default:
+        if (!MODESEGMENT)
+            return DISK_RET_EPARAM;
+        // In 16bit mode and driver not found - try in 32bit mode
+        return call32(process_op_32, MAKE_FLATPTR(GET_SEG(SS), op)
+                      , DISK_RET_EPARAM);
+    }
+}
+
+// Command dispatch for disk drivers that only run in 32bit mode
+int VISIBLE32FLAT
+process_op_32(struct disk_op_s *op)
+{
+    ASSERT32FLAT();
+    switch (op->drive_gf->type) {
+    case DTYPE_VIRTIO_BLK:
+        return virtio_blk_process_op(op);
+    case DTYPE_AHCI:
+        return ahci_process_op(op);
+    case DTYPE_AHCI_ATAPI:
+        return ahci_atapi_process_op(op);
+    case DTYPE_SDCARD:
+        return sdcard_process_op(op);
     case DTYPE_USB_32:
+        return usb_process_op(op);
     case DTYPE_UAS_32:
-    case DTYPE_PVSCSI: ;
-        extern void _cfunc32flat_scsi_process_op(void);
-        ret = call32(_cfunc32flat_scsi_process_op
-                     , (u32)MAKE_FLATPTR(GET_SEG(SS), op), DISK_RET_EPARAM);
-        break;
+        return uas_process_op(op);
+    case DTYPE_VIRTIO_SCSI:
+        return virtio_scsi_process_op(op);
+    case DTYPE_PVSCSI:
+        return pvscsi_process_op(op);
     default:
-        ret = DISK_RET_EPARAM;
-        break;
+        return process_op_both(op);
     }
+}
+
+// Command dispatch for disk drivers that only run in 16bit mode
+static int
+process_op_16(struct disk_op_s *op)
+{
+    ASSERT16();
+    switch (GET_GLOBALFLAT(op->drive_gf->type)) {
+    case DTYPE_FLOPPY:
+        return floppy_process_op(op);
+    case DTYPE_ATA:
+        return ata_process_op(op);
+    case DTYPE_RAMDISK:
+        return ramdisk_process_op(op);
+    case DTYPE_CDEMU:
+        return cdemu_process_op(op);
+    default:
+        return process_op_both(op);
+    }
+}
+
+// Execute a disk_op_s request.
+int
+process_op(struct disk_op_s *op)
+{
+    int ret, origcount = op->count;
+    if (origcount * GET_GLOBALFLAT(op->drive_gf->blksize) > 64*1024) {
+        op->count = 0;
+        return DISK_RET_EBOUNDARY;
+    }
+    if (MODESEGMENT)
+        ret = process_op_16(op);
+    else
+        ret = process_op_32(op);
     if (ret && op->count == origcount)
         // If the count hasn't changed on error, assume no data transferred.
         op->count = 0;
@@ -578,5 +624,5 @@ send_disk_op(struct disk_op_s *op)
     if (! CONFIG_DRIVES)
         return -1;
 
-    return stack_hop((u32)op, GET_SEG(SS), __send_disk_op);
+    return stack_hop(__send_disk_op, op, GET_SEG(SS));
 }
diff --git a/qemu/roms/seabios/src/block.h b/qemu/roms/seabios/src/block.h
index 8182288d4..2ff359fb2 100644
--- a/qemu/roms/seabios/src/block.h
+++ b/qemu/roms/seabios/src/block.h
@@ -9,11 +9,19 @@
  ****************************************************************/
 
 struct disk_op_s {
-    u64 lba;
     void *buf_fl;
     struct drive_s *drive_gf;
-    u16 count;
     u8 command;
+    u16 count;
+    union {
+        // Commands: READ, WRITE, VERIFY, SEEK, FORMAT
+        u64 lba;
+        // Commands: SCSI
+        struct {
+            u16 blocksize;
+            void *cdbcmd;
+        };
+    };
 };
 
 #define CMD_RESET   0x00
@@ -23,6 +31,7 @@ struct disk_op_s {
 #define CMD_FORMAT  0x05
 #define CMD_SEEK    0x07
 #define CMD_ISREADY 0x10
+#define CMD_SCSI    0x20
 
 
 /****************************************************************
@@ -101,7 +110,8 @@ void map_floppy_drive(struct drive_s *drive);
 void map_hd_drive(struct drive_s *drive);
 void map_cd_drive(struct drive_s *drive);
 struct int13dpt_s;
-int fill_edd(u16 seg, struct int13dpt_s *param_far, struct drive_s *drive_gf);
+int fill_edd(struct segoff_s edd, struct drive_s *drive_gf);
+int default_process_op(struct disk_op_s *op);
 int process_op(struct disk_op_s *op);
 int send_disk_op(struct disk_op_s *op);
 int create_bounce_buf(void);
diff --git a/qemu/roms/seabios/src/bmp.c b/qemu/roms/seabios/src/bmp.c
index d8e76b789..96a2b3f22 100644
--- a/qemu/roms/seabios/src/bmp.c
+++ b/qemu/roms/seabios/src/bmp.c
@@ -1,6 +1,6 @@
 /*
 * Basic BMP data process and Raw picture data handle functions.
-* Could be used to adjust pixel data format, get infomation, etc.
+* Could be used to adjust pixel data format, get information, etc.
 *
 * Copyright (C) 2011 Wayne Xia <xiawenc@cn.ibm.com>
 *
diff --git a/qemu/roms/seabios/src/boot.c b/qemu/roms/seabios/src/boot.c
index f23e9e154..e0f73a385 100644
--- a/qemu/roms/seabios/src/boot.c
+++ b/qemu/roms/seabios/src/boot.c
@@ -19,6 +19,7 @@
 #include "std/disk.h" // struct mbr_s
 #include "string.h" // memset
 #include "util.h" // irqtimer_calc
+#include "tcgbios.h" // tpm_*
 
 
 /****************************************************************
@@ -111,9 +112,9 @@ build_pci_path(char *buf, int max, const char *devname, struct pci_device *pci)
     if (pci->parent) {
         p = build_pci_path(p, max, "pci-bridge", pci->parent);
     } else {
-        if (pci->rootbus)
-            p += snprintf(p, max, "/pci-root@%x", pci->rootbus);
         p += snprintf(p, buf+max-p, "%s", FW_PCI_DOMAIN);
+        if (pci->rootbus)
+            p += snprintf(p, buf+max-p, ",%x", pci->rootbus);
     }
 
     int dev = pci_bdf_to_dev(pci->bdf), fn = pci_bdf_to_fn(pci->bdf);
@@ -459,8 +460,8 @@ interactive_bootmenu(void)
         ;
 
     char *bootmsg = romfile_loadfile("etc/boot-menu-message", NULL);
-    int menukey = romfile_loadint("etc/boot-menu-key", 0x86);
-    printf("%s", bootmsg ?: "\nPress F12 for boot menu.\n\n");
+    int menukey = romfile_loadint("etc/boot-menu-key", 1);
+    printf("%s", bootmsg ?: "\nPress ESC for boot menu.\n\n");
     free(bootmsg);
 
     u32 menutime = romfile_loadint("etc/boot-menu-wait", DEFAULT_BOOTMENU_WAIT);
@@ -486,9 +487,15 @@ interactive_bootmenu(void)
                , strtcpy(desc, pos->description, ARRAY_SIZE(desc)));
     }
 
-    // Get key press
+    // Get key press.  If the menu key is ESC, do not restart boot unless
+    // 1.5 seconds have passed.  Otherwise users (trained by years of
+    // repeatedly hitting keys to enter the BIOS) will end up hitting ESC
+    // multiple times and immediately booting the primary boot device.
+    int esc_accepted_time = irqtimer_calc(menukey == 1 ? 1500 : 0);
     for (;;) {
         scan_code = get_keystroke(1000);
+        if (scan_code == 1 && !irqtimer_check(esc_accepted_time))
+            continue;
         if (scan_code >= 1 && scan_code <= maxmenu+1)
             break;
     }
@@ -622,6 +629,8 @@ boot_disk(u8 bootdrv, int checksig)
         }
     }
 
+    tpm_add_bcv(bootdrv, MAKE_FLATPTR(bootseg, 0), 512);
+
     /* Canonicalize bootseg:bootip */
     u16 bootip = (bootseg & 0x0fff) << 4;
     bootseg &= 0xf000;
@@ -645,6 +654,9 @@ boot_cdrom(struct drive_s *drive_g)
 
     u8 bootdrv = CDEmu.emulated_drive;
     u16 bootseg = CDEmu.load_segment;
+
+    tpm_add_cdrom(bootdrv, MAKE_FLATPTR(bootseg, 0), 512);
+
     /* Canonicalize bootseg:bootip */
     u16 bootip = (bootseg & 0x0fff) << 4;
     bootseg &= 0xf000;
diff --git a/qemu/roms/seabios/src/cdrom.c b/qemu/roms/seabios/src/cdrom.c
index 92f34f42b..a4f31adde 100644
--- a/qemu/roms/seabios/src/cdrom.c
+++ b/qemu/roms/seabios/src/cdrom.c
@@ -15,9 +15,7 @@
 #include "std/disk.h" // DISK_RET_SUCCESS
 #include "string.h" // memset
 #include "util.h" // cdrom_prepboot
-
-// Locks for removable devices
-u8 CDRom_locks[BUILD_MAX_EXTDRIVE] VARLOW;
+#include "tcgbios.h" // tpm_*
 
 
 /****************************************************************
@@ -88,7 +86,7 @@ cdemu_read(struct disk_op_s *op)
 }
 
 int
-process_cdemu_op(struct disk_op_s *op)
+cdemu_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_CDROM_EMU)
         return 0;
@@ -99,13 +97,8 @@ process_cdemu_op(struct disk_op_s *op)
     case CMD_WRITE:
     case CMD_FORMAT:
         return DISK_RET_EWRITEPROTECT;
-    case CMD_VERIFY:
-    case CMD_RESET:
-    case CMD_SEEK:
-    case CMD_ISREADY:
-        return DISK_RET_SUCCESS;
     default:
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
 
@@ -122,7 +115,6 @@ cdrom_prepboot(void)
     struct drive_s *drive = malloc_fseg(sizeof(*drive));
     if (!drive) {
         warn_noalloc();
-        free(drive);
         return;
     }
     cdemu_drive_gf = drive;
@@ -158,7 +150,7 @@ cdrom_boot(struct drive_s *drive)
     dop.lba = 0x11;
     dop.count = 1;
     dop.buf_fl = buffer;
-    ret = scsi_process_op(&dop);
+    ret = process_op(&dop);
     if (ret)
         return 3;
 
@@ -174,7 +166,7 @@ cdrom_boot(struct drive_s *drive)
     // And we read the Boot Catalog
     dop.lba = lba;
     dop.count = 1;
-    ret = scsi_process_op(&dop);
+    ret = process_op(&dop);
     if (ret)
         return 7;
 
@@ -192,6 +184,9 @@ cdrom_boot(struct drive_s *drive)
     if (buffer[0x20] != 0x88)
         return 11; // Bootable
 
+    /* measure 2048 bytes (one sector) */
+    tpm_add_cdrom_catalog(MAKE_FLATPTR(GET_SEG(SS), buffer), sizeof(buffer));
+
     // Fill in el-torito cdrom emulation fields.
     emulated_drive_gf = drive;
     u8 media = buffer[0x21];
@@ -220,7 +215,7 @@ cdrom_boot(struct drive_s *drive)
         if (count > 64*1024/CDROM_SECTOR_SIZE)
             count = 64*1024/CDROM_SECTOR_SIZE;
         dop.count = count;
-        ret = scsi_process_op(&dop);
+        ret = process_op(&dop);
         if (ret)
             return 12;
         nbsectors -= count;
diff --git a/qemu/roms/seabios/src/clock.c b/qemu/roms/seabios/src/clock.c
index 9ab0ac026..e83e0f338 100644
--- a/qemu/roms/seabios/src/clock.c
+++ b/qemu/roms/seabios/src/clock.c
@@ -8,6 +8,7 @@
 #include "biosvar.h" // SET_BDA
 #include "bregs.h" // struct bregs
 #include "hw/pic.h" // pic_eoi1
+#include "hw/ps2port.h" // ps2_check_event
 #include "hw/rtc.h" // rtc_read
 #include "hw/usb-hid.h" // usb_check_event
 #include "output.h" // debug_enter
@@ -55,7 +56,8 @@ clock_setup(void)
     }
 
     enable_hwirq(0, FUNC16(entry_08));
-    enable_hwirq(8, FUNC16(entry_70));
+    if (CONFIG_RTC_TIMER)
+        enable_hwirq(8, FUNC16(entry_70));
 }
 
 
@@ -239,6 +241,16 @@ handle_1a07(struct bregs *regs)
     set_success(regs);
 }
 
+static void
+handle_1abb(struct bregs *regs)
+{
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    dprintf(DEBUG_tcg, "16: Calling tpm_interrupt_handler\n");
+    call32(tpm_interrupt_handler32, MAKE_FLATPTR(GET_SEG(SS), regs), 0);
+}
+
 // Unsupported
 static void
 handle_1aXX(struct bregs *regs)
@@ -260,17 +272,15 @@ handle_1a(struct bregs *regs)
     case 0x05: handle_1a05(regs); break;
     case 0x06: handle_1a06(regs); break;
     case 0x07: handle_1a07(regs); break;
+    case 0xbb: handle_1abb(regs); break;
     default:   handle_1aXX(regs); break;
     }
 }
 
-// INT 08h System Timer ISR Entry Point
-void VISIBLE16
-handle_08(void)
+// Update main tick counter
+static void
+clock_update(void)
 {
-    debug_isr(DEBUG_ISR_08);
-
-    // Update counter
     u32 counter = GET_BDA(timer_counter);
     counter++;
     // compare to one days worth of timer ticks at 18.2 hz
@@ -284,6 +294,15 @@ handle_08(void)
     // Check for internal events.
     floppy_tick();
     usb_check_event();
+    ps2_check_event();
+}
+
+// INT 08h System Timer ISR Entry Point
+void VISIBLE16
+handle_08(void)
+{
+    debug_isr(DEBUG_ISR_08);
+    clock_update();
 
     // chain to user timer tick INT #0x1c
     struct bregs br;
@@ -294,6 +313,20 @@ handle_08(void)
     pic_eoi1();
 }
 
+u32 last_timer_check VARLOW;
+
+// Simulate timer irq on machines without hardware irqs
+void
+clock_poll_irq(void)
+{
+    if (CONFIG_HARDWARE_IRQ)
+        return;
+    if (!timer_check(GET_LOW(last_timer_check)))
+        return;
+    SET_LOW(last_timer_check, timer_calc(ticks_to_ms(1)));
+    clock_update();
+}
+
 
 /****************************************************************
  * IRQ based timer
@@ -359,6 +392,10 @@ clear_usertimer(void)
 void
 handle_1586(struct bregs *regs)
 {
+    if (!CONFIG_RTC_TIMER) {
+        set_code_unimplemented(regs, RET_EUNSUPPORTED);
+        return;
+    }
     // Use the rtc to wait for the specified time.
     u8 statusflag = 0;
     u32 count = (regs->cx << 16) | regs->dx;
@@ -402,6 +439,10 @@ handle_1583XX(struct bregs *regs)
 void
 handle_1583(struct bregs *regs)
 {
+    if (!CONFIG_RTC_TIMER) {
+        handle_1583XX(regs);
+        return;
+    }
     switch (regs->al) {
     case 0x00: handle_158300(regs); break;
     case 0x01: handle_158301(regs); break;
@@ -415,6 +456,8 @@ handle_1583(struct bregs *regs)
 void VISIBLE16
 handle_70(void)
 {
+    if (!CONFIG_RTC_TIMER)
+        return;
     debug_isr(DEBUG_ISR_70);
 
     // Check which modes are enabled and have occurred.
diff --git a/qemu/roms/seabios/src/config.h b/qemu/roms/seabios/src/config.h
index 6da067d0b..6c47f161c 100644
--- a/qemu/roms/seabios/src/config.h
+++ b/qemu/roms/seabios/src/config.h
@@ -22,6 +22,8 @@
 #define BUILD_MAX_EXTDRIVE 16
 // Number of bytes the smbios may be and still live in the f-segment
 #define BUILD_MAX_SMBIOS_FSEG     600
+// Maximum number of bytes the mptable may be and still be copied to f-segment
+#define BUILD_MAX_MPTABLE_FSEG    600
 
 #define BUILD_MODEL_ID      0xFC
 #define BUILD_SUBMODEL_ID   0x00
@@ -104,5 +106,6 @@
 #define DEBUG_unimplemented 2
 #define DEBUG_invalid 3
 #define DEBUG_thread 2
+#define DEBUG_tcg 20
 
 #endif // config.h
diff --git a/qemu/roms/seabios/src/disk.c b/qemu/roms/seabios/src/disk.c
index 0e0af24b3..3854d0024 100644
--- a/qemu/roms/seabios/src/disk.c
+++ b/qemu/roms/seabios/src/disk.c
@@ -407,6 +407,9 @@ disk_1344(struct bregs *regs, struct drive_s *drive_gf)
     extended_access(regs, drive_gf, CMD_VERIFY);
 }
 
+// Locks for removable devices
+u8 CDRom_locks[BUILD_MAX_EXTDRIVE] VARLOW;
+
 // lock
 static void
 disk_134500(struct bregs *regs, struct drive_s *drive_gf)
@@ -519,7 +522,7 @@ disk_1347(struct bregs *regs, struct drive_s *drive_gf)
 static void
 disk_1348(struct bregs *regs, struct drive_s *drive_gf)
 {
-    int ret = fill_edd(regs->ds, (void*)(regs->si+0), drive_gf);
+    int ret = fill_edd(SEGOFF(regs->ds, regs->si), drive_gf);
     disk_ret(regs, ret);
 }
 
diff --git a/qemu/roms/seabios/src/memmap.c b/qemu/roms/seabios/src/e820map.c
index e03f8d0bf..39445cf63 100644
--- a/qemu/roms/seabios/src/memmap.c
+++ b/qemu/roms/seabios/src/e820map.c
@@ -5,7 +5,7 @@
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
 #include "config.h" // BUILD_MAX_E820
-#include "memmap.h" // struct e820entry
+#include "e820map.h" // struct e820entry
 #include "output.h" // dprintf
 #include "string.h" // memmove
 
@@ -54,7 +54,6 @@ e820_type_name(u32 type)
     case E820_ACPI:     return "ACPI";
     case E820_NVS:      return "NVS";
     case E820_UNUSABLE: return "UNUSABLE";
-    case E820_HOLE:     return "HOLE";
     default:            return "UNKNOWN";
     }
 }
@@ -73,12 +72,14 @@ dump_map(void)
     }
 }
 
+#define E820_HOLE         ((u32)-1) // Used internally to remove entries
+
 // Add a new entry to the list.  This scans for overlaps and keeps the
 // list sorted.
 void
-add_e820(u64 start, u64 size, u32 type)
+e820_add(u64 start, u64 size, u32 type)
 {
-    dprintf(8, "Add to e820 map: %08x %08x %d\n", (u32)start, (u32)size, type);
+    dprintf(8, "Add to e820 map: %08llx %08llx %d\n", start, size, type);
 
     if (! size)
         // Huh?  Nothing to do.
@@ -136,9 +137,16 @@ add_e820(u64 start, u64 size, u32 type)
     //dump_map();
 }
 
+// Remove any definitions in a memory range (make a memory hole).
+void
+e820_remove(u64 start, u64 size)
+{
+    e820_add(start, size, E820_HOLE);
+}
+
 // Report on final memory locations.
 void
-memmap_prepboot(void)
+e820_prepboot(void)
 {
     dump_map();
 }
diff --git a/qemu/roms/seabios/src/e820map.h b/qemu/roms/seabios/src/e820map.h
new file mode 100644
index 000000000..de8b52300
--- /dev/null
+++ b/qemu/roms/seabios/src/e820map.h
@@ -0,0 +1,26 @@
+#ifndef __E820MAP_H
+#define __E820MAP_H
+
+#include "types.h" // u64
+
+#define E820_RAM          1
+#define E820_RESERVED     2
+#define E820_ACPI         3
+#define E820_NVS          4
+#define E820_UNUSABLE     5
+
+struct e820entry {
+    u64 start;
+    u64 size;
+    u32 type;
+};
+
+void e820_add(u64 start, u64 size, u32 type);
+void e820_remove(u64 start, u64 size);
+void e820_prepboot(void);
+
+// e820 map storage
+extern struct e820entry e820_list[];
+extern int e820_count;
+
+#endif // e820map.h
diff --git a/qemu/roms/seabios/src/fw/biostables.c b/qemu/roms/seabios/src/fw/biostables.c
index 50a891be8..9fb9ff9df 100644
--- a/qemu/roms/seabios/src/fw/biostables.c
+++ b/qemu/roms/seabios/src/fw/biostables.c
@@ -6,14 +6,15 @@
 
 #include "byteorder.h" // le32_to_cpu
 #include "config.h" // CONFIG_*
+#include "hw/pci.h" // pci_config_writeb
 #include "malloc.h" // malloc_fseg
+#include "memmap.h" // SYMBOL
 #include "output.h" // dprintf
-#include "hw/pci.h" // pci_config_writeb
+#include "romfile.h" // romfile_find
 #include "std/acpi.h" // struct rsdp_descriptor
 #include "std/mptable.h" // MPTABLE_SIGNATURE
 #include "std/pirtable.h" // struct pir_header
 #include "std/smbios.h" // struct smbios_entry_point
-#include "romfile.h"
 #include "string.h" // memcpy
 #include "util.h" // copy_table
 #include "x86.h" // outb
@@ -54,6 +55,11 @@ copy_mptable(void *pos)
         return;
     u32 length = p->length * 16;
     u16 mpclength = ((struct mptable_config_s *)p->physaddr)->length;
+    if (length + mpclength > BUILD_MAX_MPTABLE_FSEG) {
+        dprintf(1, "Skipping MPTABLE copy due to large size (%d bytes)\n"
+                , length + mpclength);
+        return;
+    }
     // Allocate final memory location.  (In theory the config
     // structure can go in high memory, but Linux kernels before
     // v2.6.30 crash with that.)
@@ -117,9 +123,8 @@ copy_acpi_rsdp(void *pos)
 
 void *find_acpi_rsdp(void)
 {
-    extern u8 zonefseg_start[], zonefseg_end[];
-    unsigned long start = (unsigned long)zonefseg_start;
-    unsigned long end = (unsigned long)zonefseg_end;
+    unsigned long start = SYMBOL(zonefseg_start);
+    unsigned long end = SYMBOL(zonefseg_end);
     unsigned long pos;
 
     for (pos = ALIGN(start, 0x10); pos <= ALIGN_DOWN(end, 0x10); pos += 0x10)
@@ -271,7 +276,7 @@ copy_smbios(void *pos)
     if (SMBiosAddr)
         return;
     struct smbios_entry_point *p = pos;
-    if (memcmp(p->anchor_string, "_SM_", 4))
+    if (p->signature != SMBIOS_SIGNATURE)
         return;
     if (checksum(pos, 0x10) != 0)
         return;
@@ -301,17 +306,42 @@ display_uuid(void)
             if (memcmp(uuid, empty_uuid, sizeof(empty_uuid)) == 0)
                 return;
 
-            printf("Machine UUID"
-                   " %02x%02x%02x%02x"
-                   "-%02x%02x"
-                   "-%02x%02x"
-                   "-%02x%02x"
-                   "-%02x%02x%02x%02x%02x%02x\n"
-                   , uuid[ 0], uuid[ 1], uuid[ 2], uuid[ 3]
-                   , uuid[ 4], uuid[ 5]
-                   , uuid[ 6], uuid[ 7]
-                   , uuid[ 8], uuid[ 9]
-                   , uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
+            /*
+             * According to SMBIOS v2.6 the first three fields are encoded in
+             * little-endian format.  Versions prior to v2.6 did not specify
+             * the encoding, but we follow dmidecode and assume big-endian
+             * encoding.
+             */
+            if (SMBiosAddr->smbios_major_version > 2 ||
+                (SMBiosAddr->smbios_major_version == 2 &&
+                 SMBiosAddr->smbios_minor_version >= 6)) {
+                printf("Machine UUID"
+                       " %02x%02x%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x%02x%02x%02x%02x\n"
+                       , uuid[ 3], uuid[ 2], uuid[ 1], uuid[ 0]
+                       , uuid[ 5], uuid[ 4]
+                       , uuid[ 7], uuid[ 6]
+                       , uuid[ 8], uuid[ 9]
+                       , uuid[10], uuid[11], uuid[12]
+                       , uuid[13], uuid[14], uuid[15]);
+            } else {
+                printf("Machine UUID"
+                       " %02x%02x%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x"
+                       "-%02x%02x%02x%02x%02x%02x\n"
+                       , uuid[ 0], uuid[ 1], uuid[ 2], uuid[ 3]
+                       , uuid[ 4], uuid[ 5]
+                       , uuid[ 6], uuid[ 7]
+                       , uuid[ 8], uuid[ 9]
+                       , uuid[10], uuid[11], uuid[12]
+                       , uuid[13], uuid[14], uuid[15]);
+            }
+
             return;
         }
 }
@@ -447,7 +477,7 @@ void
 smbios_setup(void)
 {
     if (smbios_romfile_setup())
-      return;
+        return;
     smbios_legacy_setup();
 }
 
diff --git a/qemu/roms/seabios/src/fw/coreboot.c b/qemu/roms/seabios/src/fw/coreboot.c
index 8fd84493b..4fe12928c 100644
--- a/qemu/roms/seabios/src/fw/coreboot.c
+++ b/qemu/roms/seabios/src/fw/coreboot.c
@@ -7,10 +7,10 @@
 #include "block.h" // MAXDESCSIZE
 #include "byteorder.h" // be32_to_cpu
 #include "config.h" // CONFIG_*
+#include "e820map.h" // e820_add
 #include "hw/pci.h" // pci_probe_devices
 #include "lzmadecode.h" // LzmaDecode
 #include "malloc.h" // free
-#include "memmap.h" // add_e820
 #include "output.h" // dprintf
 #include "paravirt.h" // PlatformRunningOn
 #include "romfile.h" // romfile_findprefix
@@ -184,12 +184,12 @@ coreboot_preinit(void)
         u32 type = m->type;
         if (type == CB_MEM_TABLE)
             type = E820_RESERVED;
-        add_e820(m->start, m->size, type);
+        e820_add(m->start, m->size, type);
     }
 
     // Ughh - coreboot likes to set a map at 0x0000-0x1000, but this
     // confuses grub.  So, override it.
-    add_e820(0, 16*1024, E820_RAM);
+    e820_add(0, 16*1024, E820_RAM);
 
     struct cb_cbmem_ref *cbref = find_cb_subtable(cbh, CB_TAG_CBMEM_CONSOLE);
     if (cbref) {
@@ -210,7 +210,7 @@ coreboot_preinit(void)
 fail:
     // No table found..  Use 16Megs as a dummy value.
     dprintf(1, "Unable to find coreboot table!\n");
-    add_e820(0, 16*1024*1024, E820_RAM);
+    e820_add(0, 16*1024*1024, E820_RAM);
     return;
 }
 
@@ -421,6 +421,13 @@ coreboot_cbfs_init(void)
         return;
 
     struct cbfs_header *hdr = *(void **)(CONFIG_CBFS_LOCATION - 4);
+    if ((u32)hdr & 0x03) {
+        dprintf(1, "Invalid CBFS pointer %p\n", hdr);
+        return;
+    }
+    if (CONFIG_CBFS_LOCATION && (u32)hdr > CONFIG_CBFS_LOCATION)
+        // Looks like the pointer is relative to CONFIG_CBFS_LOCATION
+        hdr = (void*)hdr + CONFIG_CBFS_LOCATION;
     if (hdr->magic != cpu_to_be32(CBFS_HEADER_MAGIC)) {
         dprintf(1, "Unable to find CBFS (ptr=%p; got %x not %x)\n"
                 , hdr, hdr->magic, cpu_to_be32(CBFS_HEADER_MAGIC));
@@ -503,7 +510,7 @@ cbfs_run_payload(struct cbfs_file *fhdr)
             break;
         case PAYLOAD_SEGMENT_ENTRY: {
             dprintf(1, "Calling addr %p\n", dest);
-            void (*func)() = dest;
+            void (*func)(void) = dest;
             func();
             return;
         }
diff --git a/qemu/roms/seabios/src/fw/csm.c b/qemu/roms/seabios/src/fw/csm.c
index 7cdb398f2..7cadd12e5 100644
--- a/qemu/roms/seabios/src/fw/csm.c
+++ b/qemu/roms/seabios/src/fw/csm.c
@@ -4,20 +4,21 @@
 //
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
-#include "bregs.h"
+#include "bregs.h" // struct bregs
 #include "config.h" // CONFIG_*
+#include "e820map.h" // e820_add
 #include "farptr.h" // MAKE_FLATPTR
-#include "hw/pci.h"
-#include "hw/pic.h"
-#include "malloc.h" // csm_malloc_preinit
-#include "memmap.h"
+#include "hw/pci.h" // pci_probe_devices
+#include "hw/pic.h" // pic_irqmask_read
+#include "malloc.h" // malloc_csm_preinit
+#include "memmap.h" // SYMBOL
 #include "output.h" // dprintf
+#include "paravirt.h" // qemu_preinit
 #include "stacks.h" // wait_threads
 #include "std/acpi.h" // RSDP_SIGNATURE
 #include "std/bda.h" // struct bios_data_area_s
 #include "std/optionrom.h" // struct rom_header
 #include "util.h" // copy_smbios
-#include "paravirt.h" // qemu_preinit
 
 #define UINT8 u8
 #define UINT16 u16
@@ -47,12 +48,11 @@ static void
 csm_return(struct bregs *regs)
 {
     u32 rommax = rom_get_max();
-    extern u8 final_readonly_start[];
 
     dprintf(3, "handle_csm returning AX=%04x\n", regs->ax);
 
     csm_compat_table.UmaAddress = rommax;
-    csm_compat_table.UmaSize = (u32)final_readonly_start - rommax;
+    csm_compat_table.UmaSize = SYMBOL(final_readonly_start) - rommax;
 
     PICMask = pic_irqmask_read();
     __csm_return(regs);
@@ -95,7 +95,7 @@ handle_csm_0000(struct bregs *regs)
     dprintf(3, "LoPmmMemory     %08x\n", csm_init_table->LowPmmMemory);
     dprintf(3, "LoPmmMemorySize %08x\n", csm_init_table->LowPmmMemorySizeInBytes);
 
-    csm_malloc_preinit(csm_init_table->LowPmmMemory,
+    malloc_csm_preinit(csm_init_table->LowPmmMemory,
                        csm_init_table->LowPmmMemorySizeInBytes,
                        csm_init_table->HiPmmMemory,
                        csm_init_table->HiPmmMemorySizeInBytes);
@@ -147,11 +147,11 @@ handle_csm_0002(struct bregs *regs)
     struct e820entry *p = (void *)csm_compat_table.E820Pointer;
     int i;
     for (i=0; i < csm_compat_table.E820Length / sizeof(struct e820entry); i++)
-        add_e820(p[i].start, p[i].size, p[i].type);
+        e820_add(p[i].start, p[i].size, p[i].type);
 
     if (csm_init_table->HiPmmMemorySizeInBytes > BUILD_MAX_HIGHTABLE) {
         u32 hi_pmm_end = csm_init_table->HiPmmMemory + csm_init_table->HiPmmMemorySizeInBytes;
-        add_e820(hi_pmm_end - BUILD_MAX_HIGHTABLE, BUILD_MAX_HIGHTABLE, E820_RESERVED);
+        e820_add(hi_pmm_end - BUILD_MAX_HIGHTABLE, BUILD_MAX_HIGHTABLE, E820_RESERVED);
     }
 
     // For PCIBIOS 1ab10e
@@ -183,6 +183,7 @@ handle_csm_0002(struct bregs *regs)
     struct bios_data_area_s *bda = MAKE_FLATPTR(SEG_BDA, 0);
     bda->hdcount = 0;
 
+    thread_setup();
     mathcp_setup();
     timer_setup();
     clock_setup();
diff --git a/qemu/roms/seabios/src/fw/dev-q35.h b/qemu/roms/seabios/src/fw/dev-q35.h
index c6f8bd9e7..201825deb 100644
--- a/qemu/roms/seabios/src/fw/dev-q35.h
+++ b/qemu/roms/seabios/src/fw/dev-q35.h
@@ -27,6 +27,9 @@
 #define ICH9_LPC_GEN_PMCON_1_SMI_LOCK  (1 << 4)
 #define ICH9_LPC_PORT_ELCR1            0x4d0
 #define ICH9_LPC_PORT_ELCR2            0x4d1
+#define ICH9_LPC_RCBA                  0xf0
+#define ICH9_LPC_RCBA_ADDR             0xfed1c000
+#define ICH9_LPC_RCBA_EN               0x1
 #define PCI_DEVICE_ID_INTEL_ICH9_SMBUS 0x2930
 #define ICH9_SMB_SMB_BASE              0x20
 #define ICH9_SMB_HOSTC                 0x40
diff --git a/qemu/roms/seabios/src/fw/multiboot.c b/qemu/roms/seabios/src/fw/multiboot.c
new file mode 100644
index 000000000..d9df06764
--- /dev/null
+++ b/qemu/roms/seabios/src/fw/multiboot.c
@@ -0,0 +1,111 @@
+// Multiboot interface support.
+//
+// Copyright (C) 2015  Vladimir Serbinenko <phcoder@gmail.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+#include "config.h" // CONFIG_*
+#include "malloc.h" // free
+#include "output.h" // dprintf
+#include "romfile.h" // romfile_add
+#include "std/multiboot.h" // MULTIBOOT_*
+#include "string.h" // memset
+#include "util.h" // multiboot_init
+
+struct mbfs_romfile_s {
+    struct romfile_s file;
+    void *data;
+};
+
+static int
+extract_filename(char *dest, char *src, size_t lim)
+{
+    char *ptr;
+    for (ptr = src; *ptr; ptr++) {
+        if (!(ptr == src || ptr[-1] == ' ' || ptr[-1] == '\t'))
+            continue;
+        /* memcmp stops early if it encounters \0 as it doesn't match name=.  */
+        if (memcmp(ptr, "name=", 5) == 0) {
+            int i;
+            char *optr = dest;
+            for (i = 0, ptr += 5; *ptr && *ptr != ' ' && i < lim; i++) {
+                *optr++ = *ptr++;
+            }
+            *optr++ = '\0';
+            return 1;
+        }
+    }
+    return 0;
+}
+
+// Copy a file to memory
+static int
+mbfs_copyfile(struct romfile_s *file, void *dst, u32 maxlen)
+{
+    struct mbfs_romfile_s *cfile;
+    cfile = container_of(file, struct mbfs_romfile_s, file);
+    u32 size = cfile->file.size;
+    void *src = cfile->data;
+
+    // Not compressed.
+    dprintf(3, "Copying data %d@%p to %d@%p\n", size, src, maxlen, dst);
+    if (size > maxlen) {
+        warn_noalloc();
+        return -1;
+    }
+    iomemcpy(dst, src, size);
+    return size;
+}
+
+u32 __VISIBLE entry_elf_eax, entry_elf_ebx;
+
+void
+multiboot_init(void)
+{
+    struct multiboot_info *mbi;
+    if (!CONFIG_MULTIBOOT)
+        return;
+    dprintf(1, "multiboot: eax=%x, ebx=%x\n", entry_elf_eax, entry_elf_ebx);
+    if (entry_elf_eax != MULTIBOOT_BOOTLOADER_MAGIC)
+        return;
+    mbi = (void *)entry_elf_ebx;
+    dprintf(1, "mbptr=%p\n", mbi);
+    dprintf(1, "flags=0x%x, mods=0x%x, mods_c=%d\n", mbi->flags, mbi->mods_addr,
+            mbi->mods_count);
+    if (!(mbi->flags & MULTIBOOT_INFO_MODS))
+        return;
+    int i;
+    struct multiboot_mod_list *mod = (void *)mbi->mods_addr;
+    for (i = 0; i < mbi->mods_count; i++) {
+        struct mbfs_romfile_s *cfile;
+        u8 *copy;
+        u32 len;
+        if (!mod[i].cmdline)
+            continue;
+        len = mod[i].mod_end - mod[i].mod_start;
+        cfile = malloc_tmp(sizeof(*cfile));
+        if (!cfile) {
+            warn_noalloc();
+            return;
+        }
+        memset(cfile, 0, sizeof(*cfile));
+        dprintf(1, "module %s, size 0x%x\n", (char *)mod[i].cmdline, len);
+        if (!extract_filename(cfile->file.name, (char *)mod[i].cmdline,
+                              sizeof(cfile->file.name))) {
+            free(cfile);
+            continue;
+        }
+        dprintf(1, "assigned file name <%s>\n", cfile->file.name);
+        cfile->file.size = len;
+        copy = malloc_tmp(len);
+        if (!copy) {
+            warn_noalloc();
+            free(cfile);
+            return;
+        }
+        memcpy(copy, (void *)mod[i].mod_start, len);
+        cfile->file.copy = mbfs_copyfile;
+        cfile->data = copy;
+        romfile_add(&cfile->file);
+    }
+}
diff --git a/qemu/roms/seabios/src/fw/paravirt.c b/qemu/roms/seabios/src/fw/paravirt.c
index db22ae8fc..3fae13a83 100644
--- a/qemu/roms/seabios/src/fw/paravirt.c
+++ b/qemu/roms/seabios/src/fw/paravirt.c
@@ -10,11 +10,11 @@
 
 #include "byteorder.h" // be32_to_cpu
 #include "config.h" // CONFIG_QEMU
+#include "e820map.h" // e820_add
 #include "hw/pci.h" // create_pirtable
 #include "hw/pci_regs.h" // PCI_DEVICE_ID
 #include "hw/rtc.h" // CMOS_*
 #include "malloc.h" // malloc_tmp
-#include "memmap.h" // add_e820
 #include "output.h" // dprintf
 #include "paravirt.h" // qemu_cfg_preinit
 #include "romfile.h" // romfile_loadint
@@ -23,6 +23,7 @@
 #include "util.h" // pci_setup
 #include "x86.h" // cpuid
 #include "xen.h" // xen_biostable_setup
+#include "stacks.h" // yield
 
 // Amount of continuous ram under 4Gig
 u32 RamSize;
@@ -30,6 +31,13 @@ u32 RamSize;
 u64 RamSizeOver4G;
 // Type of emulator platform.
 int PlatformRunningOn VARFSEG;
+// cfg_dma enabled
+int cfg_dma_enabled = 0;
+
+inline int qemu_cfg_dma_enabled(void)
+{
+    return cfg_dma_enabled;
+}
 
 /* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx.  It
  * should be used to determine that a VM is running under KVM.
@@ -114,10 +122,10 @@ qemu_preinit(void)
                | (rtc_read(CMOS_MEM_EXTMEM_HIGH) << 18))
               + 1 * 1024 * 1024);
     RamSize = rs;
-    add_e820(0, rs, E820_RAM);
+    e820_add(0, rs, E820_RAM);
 
     /* reserve 256KB BIOS area at the end of 4 GB */
-    add_e820(0xfffc0000, 256*1024, E820_RESERVED);
+    e820_add(0xfffc0000, 256*1024, E820_RESERVED);
 
     dprintf(1, "RamSize: 0x%08x [cmos]\n", RamSize);
 }
@@ -199,23 +207,63 @@ qemu_cfg_select(u16 f)
 }
 
 static void
+qemu_cfg_dma_transfer(void *address, u32 length, u32 control)
+{
+    QemuCfgDmaAccess access;
+
+    access.address = cpu_to_be64((u64)(u32)address);
+    access.length = cpu_to_be32(length);
+    access.control = cpu_to_be32(control);
+
+    barrier();
+
+    outl(cpu_to_be32((u32)&access), PORT_QEMU_CFG_DMA_ADDR_LOW);
+
+    while(be32_to_cpu(access.control) & ~QEMU_CFG_DMA_CTL_ERROR) {
+        yield();
+    }
+}
+
+static void
 qemu_cfg_read(void *buf, int len)
 {
-    insb(PORT_QEMU_CFG_DATA, buf, len);
+    if (len == 0) {
+        return;
+    }
+
+    if (qemu_cfg_dma_enabled()) {
+        qemu_cfg_dma_transfer(buf, len, QEMU_CFG_DMA_CTL_READ);
+    } else {
+        insb(PORT_QEMU_CFG_DATA, buf, len);
+    }
 }
 
 static void
 qemu_cfg_skip(int len)
 {
-    while (len--)
-        inb(PORT_QEMU_CFG_DATA);
+    if (len == 0) {
+        return;
+    }
+
+    if (qemu_cfg_dma_enabled()) {
+        qemu_cfg_dma_transfer(0, len, QEMU_CFG_DMA_CTL_SKIP);
+    } else {
+        while (len--)
+            inb(PORT_QEMU_CFG_DATA);
+    }
 }
 
 static void
 qemu_cfg_read_entry(void *buf, int e, int len)
 {
-    qemu_cfg_select(e);
-    qemu_cfg_read(buf, len);
+    if (qemu_cfg_dma_enabled()) {
+        u32 control = (e << 16) | QEMU_CFG_DMA_CTL_SELECT
+                        | QEMU_CFG_DMA_CTL_READ;
+        qemu_cfg_dma_transfer(buf, len, control);
+    } else {
+        qemu_cfg_select(e);
+        qemu_cfg_read(buf, len);
+    }
 }
 
 struct qemu_romfile_s {
@@ -230,9 +278,14 @@ qemu_cfg_read_file(struct romfile_s *file, void *dst, u32 maxlen)
         return -1;
     struct qemu_romfile_s *qfile;
     qfile = container_of(file, struct qemu_romfile_s, file);
-    qemu_cfg_select(qfile->select);
-    qemu_cfg_skip(qfile->skip);
-    qemu_cfg_read(dst, file->size);
+    if (qfile->skip == 0) {
+        /* Do it in one transfer */
+        qemu_cfg_read_entry(dst, qfile->select, file->size);
+    } else {
+        qemu_cfg_select(qfile->select);
+        qemu_cfg_skip(qfile->skip);
+        qemu_cfg_read(dst, file->size);
+    }
     return file->size;
 }
 
@@ -302,7 +355,7 @@ qemu_cfg_e820(void)
                 }
                 /* fall through */
             case E820_RESERVED:
-                add_e820(table[i].address, table[i].length, table[i].type);
+                e820_add(table[i].address, table[i].length, table[i].type);
                 break;
             default:
                 /*
@@ -324,13 +377,13 @@ qemu_cfg_e820(void)
         int i;
         for (i = 0; i < count32; i++) {
             qemu_cfg_read(&entry, sizeof(entry));
-            add_e820(entry.address, entry.length, entry.type);
+            e820_add(entry.address, entry.length, entry.type);
         }
     } else if (runningOnKVM()) {
         // Backwards compatibility - provide hard coded range.
         // 4 pages before the bios, 3 pages for vmx tss pages, the
         // other page for EPT real mode pagetable
-        add_e820(0xfffbc000, 4*4096, E820_RESERVED);
+        e820_add(0xfffbc000, 4*4096, E820_RESERVED);
     }
 
     // Check for memory over 4Gig in cmos
@@ -338,7 +391,7 @@ qemu_cfg_e820(void)
                 | ((u32)rtc_read(CMOS_MEM_HIGHMEM_MID) << 24)
                 | ((u64)rtc_read(CMOS_MEM_HIGHMEM_HIGH) << 32));
     RamSizeOver4G = high;
-    add_e820(0x100000000ull, high, E820_RAM);
+    e820_add(0x100000000ull, high, E820_RAM);
     dprintf(1, "RamSizeOver4G: 0x%016llx [cmos]\n", RamSizeOver4G);
 }
 
@@ -422,8 +475,18 @@ void qemu_cfg_init(void)
     for (i = 0; i < 4; i++)
         if (inb(PORT_QEMU_CFG_DATA) != sig[i])
             return;
+
     dprintf(1, "Found QEMU fw_cfg\n");
 
+    // Detect DMA interface.
+    u32 id;
+    qemu_cfg_read_entry(&id, QEMU_CFG_ID, sizeof(id));
+
+    if (id & QEMU_CFG_VERSION_DMA) {
+        dprintf(1, "QEMU fw_cfg DMA interface supported\n");
+        cfg_dma_enabled = 1;
+    }
+
     // Populate romfiles for legacy fw_cfg entries
     qemu_cfg_legacy();
 
diff --git a/qemu/roms/seabios/src/fw/paravirt.h b/qemu/roms/seabios/src/fw/paravirt.h
index 95ffb92ad..ed8e5f1f8 100644
--- a/qemu/roms/seabios/src/fw/paravirt.h
+++ b/qemu/roms/seabios/src/fw/paravirt.h
@@ -9,6 +9,12 @@
 #define PF_XEN      (1<<1)
 #define PF_KVM      (1<<2)
 
+typedef struct QemuCfgDmaAccess {
+    u32 control;
+    u32 length;
+    u64 address;
+} PACKED QemuCfgDmaAccess;
+
 extern u32 RamSize;
 extern u64 RamSizeOver4G;
 extern int PlatformRunningOn;
@@ -25,11 +31,23 @@ static inline int runningOnKVM(void) {
 }
 
 // Common paravirt ports.
-#define PORT_SMI_CMD           0x00b2
-#define PORT_SMI_STATUS        0x00b3
-#define PORT_QEMU_CFG_CTL      0x0510
-#define PORT_QEMU_CFG_DATA     0x0511
+#define PORT_SMI_CMD                0x00b2
+#define PORT_SMI_STATUS             0x00b3
+#define PORT_QEMU_CFG_CTL           0x0510
+#define PORT_QEMU_CFG_DATA          0x0511
+#define PORT_QEMU_CFG_DMA_ADDR_HIGH 0x0514
+#define PORT_QEMU_CFG_DMA_ADDR_LOW  0x0518
+
+// QEMU_CFG_DMA_CONTROL bits
+#define QEMU_CFG_DMA_CTL_ERROR   0x01
+#define QEMU_CFG_DMA_CTL_READ    0x02
+#define QEMU_CFG_DMA_CTL_SKIP    0x04
+#define QEMU_CFG_DMA_CTL_SELECT  0x08
+
+// QEMU_CFG_DMA ID bit
+#define QEMU_CFG_VERSION_DMA    2
 
+int qemu_cfg_dma_enabled(void);
 void qemu_preinit(void);
 void qemu_platform_setup(void);
 void qemu_cfg_init(void);
diff --git a/qemu/roms/seabios/src/fw/pciinit.c b/qemu/roms/seabios/src/fw/pciinit.c
index 46ae7090e..c31c2fa0c 100644
--- a/qemu/roms/seabios/src/fw/pciinit.c
+++ b/qemu/roms/seabios/src/fw/pciinit.c
@@ -9,13 +9,13 @@
 #include "config.h" // CONFIG_*
 #include "dev-q35.h" // Q35_HOST_BRIDGE_PCIEXBAR_ADDR
 #include "dev-piix.h" // PIIX_*
+#include "e820map.h" // e820_add
 #include "hw/ata.h" // PORT_ATA1_CMD_BASE
 #include "hw/pci.h" // pci_config_readl
 #include "hw/pci_ids.h" // PCI_VENDOR_ID_INTEL
 #include "hw/pci_regs.h" // PCI_COMMAND
 #include "list.h" // struct hlist_node
 #include "malloc.h" // free
-#include "memmap.h" // add_e820
 #include "output.h" // dprintf
 #include "paravirt.h" // RamSize
 #include "romfile.h" // romfile_loadint
@@ -183,6 +183,11 @@ static void mch_isa_bridge_setup(struct pci_device *dev, void *arg)
     /* acpi enable, SCI: IRQ9 000b = irq9*/
     pci_config_writeb(bdf, ICH9_LPC_ACPI_CTRL, ICH9_LPC_ACPI_CTRL_ACPI_EN);
 
+    /* set root complex register block BAR */
+    pci_config_writel(bdf, ICH9_LPC_RCBA,
+                      ICH9_LPC_RCBA_ADDR | ICH9_LPC_RCBA_EN);
+    e820_add(ICH9_LPC_RCBA_ADDR, 16*1024, E820_RESERVED);
+
     acpi_pm1a_cnt = acpi_pm_base + 0x04;
     pmtimer_setup(acpi_pm_base + 0x08);
 }
@@ -316,6 +321,10 @@ static void pci_bios_init_device(struct pci_device *pci)
     /* enable memory mappings */
     pci_config_maskw(bdf, PCI_COMMAND, 0,
                      PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_SERR);
+    /* enable SERR# for forwarding */
+    if (pci->header_type & PCI_HEADER_TYPE_BRIDGE)
+        pci_config_maskw(bdf, PCI_BRIDGE_CONTROL, 0,
+                         PCI_BRIDGE_CTL_SERR);
 }
 
 static void pci_bios_init_devices(void)
@@ -391,7 +400,7 @@ static void mch_mem_addr_setup(struct pci_device *dev, void *arg)
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR, 0);
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR + 4, upper);
     pci_config_writel(bdf, Q35_HOST_BRIDGE_PCIEXBAR, lower);
-    add_e820(addr, size, E820_RESERVED);
+    e820_add(addr, size, E820_RESERVED);
 
     /* setup pci i/o window (above mmconfig) */
     pcimem_start = addr + size;
@@ -636,9 +645,8 @@ pci_region_create_entry(struct pci_bus *bus, struct pci_device *dev,
     return entry;
 }
 
-static int pci_bus_hotplug_support(struct pci_bus *bus)
+static int pci_bus_hotplug_support(struct pci_bus *bus, u8 pcie_cap)
 {
-    u8 pcie_cap = pci_find_capability(bus->bus_dev, PCI_CAP_ID_EXP);
     u8 shpc_cap;
 
     if (pcie_cap) {
@@ -662,7 +670,7 @@ static int pci_bus_hotplug_support(struct pci_bus *bus)
         return downstream_port && slot_implemented;
     }
 
-    shpc_cap = pci_find_capability(bus->bus_dev, PCI_CAP_ID_SHPC);
+    shpc_cap = pci_find_capability(bus->bus_dev, PCI_CAP_ID_SHPC, 0);
     return !!shpc_cap;
 }
 
@@ -718,7 +726,8 @@ static int pci_bios_check_devices(struct pci_bus *busses)
              */
             parent = &busses[0];
         int type;
-        int hotplug_support = pci_bus_hotplug_support(s);
+        u8 pcie_cap = pci_find_capability(s->bus_dev, PCI_CAP_ID_EXP, 0);
+        int hotplug_support = pci_bus_hotplug_support(s, pcie_cap);
         for (type = 0; type < PCI_REGION_TYPE_COUNT; type++) {
             u64 align = (type == PCI_REGION_TYPE_IO) ?
                 PCI_BRIDGE_IO_MIN : PCI_BRIDGE_MEM_MIN;
@@ -727,7 +736,8 @@ static int pci_bios_check_devices(struct pci_bus *busses)
             if (pci_region_align(&s->r[type]) > align)
                  align = pci_region_align(&s->r[type]);
             u64 sum = pci_region_sum(&s->r[type]);
-            if (!sum && hotplug_support)
+            int resource_optional = pcie_cap && (type == PCI_REGION_TYPE_IO);
+            if (!sum && hotplug_support && !resource_optional)
                 sum = align; /* reserve min size for hot-plug */
             u64 size = ALIGN(sum, align);
             int is64 = pci_bios_bridge_region_is64(&s->r[type],
diff --git a/qemu/roms/seabios/src/fw/shadow.c b/qemu/roms/seabios/src/fw/shadow.c
index 4f00006bf..ee87d36e0 100644
--- a/qemu/roms/seabios/src/fw/shadow.c
+++ b/qemu/roms/seabios/src/fw/shadow.c
@@ -53,9 +53,8 @@ __make_bios_writable_intel(u16 bdf, u32 pam0)
         return;
 
     // Copy bios.
-    extern u8 code32flat_start[], code32flat_end[];
-    memcpy(code32flat_start, code32flat_start + BIOS_SRC_OFFSET
-           , code32flat_end - code32flat_start);
+    memcpy(VSYMBOL(code32flat_start), VSYMBOL(code32flat_start) + BIOS_SRC_OFFSET
+           , SYMBOL(code32flat_end) - SYMBOL(code32flat_start));
 }
 
 static void
@@ -65,7 +64,7 @@ make_bios_writable_intel(u16 bdf, u32 pam0)
     if (!(reg & 0x10)) {
         // QEMU doesn't fully implement the piix shadow capabilities -
         // if ram isn't backing the bios segment when shadowing is
-        // disabled, the code itself wont be in memory.  So, run the
+        // disabled, the code itself won't be in memory.  So, run the
         // code from the high-memory flash location.
         u32 pos = (u32)__make_bios_writable_intel + BIOS_SRC_OFFSET;
         void (*func)(u16 bdf, u32 pam0) = (void*)pos;
@@ -165,7 +164,6 @@ qemu_prep_reset(void)
     // QEMU doesn't map 0xc0000-0xfffff back to the original rom on a
     // reset, so do that manually before invoking a hard reset.
     make_bios_writable();
-    extern u8 code32flat_start[], code32flat_end[];
-    memcpy(code32flat_start, code32flat_start + BIOS_SRC_OFFSET
-           , code32flat_end - code32flat_start);
+    memcpy(VSYMBOL(code32flat_start), VSYMBOL(code32flat_start) + BIOS_SRC_OFFSET
+           , SYMBOL(code32flat_end) - SYMBOL(code32flat_start));
 }
diff --git a/qemu/roms/seabios/src/fw/smbios.c b/qemu/roms/seabios/src/fw/smbios.c
index dba054133..f3b5ad9dd 100644
--- a/qemu/roms/seabios/src/fw/smbios.c
+++ b/qemu/roms/seabios/src/fw/smbios.c
@@ -37,7 +37,7 @@ smbios_entry_point_setup(u16 max_structure_size,
 
     struct smbios_entry_point ep;
     memset(&ep, 0, sizeof(ep));
-    memcpy(ep.anchor_string, "_SM_", 4);
+    ep.signature = SMBIOS_SIGNATURE;
     ep.length = 0x1f;
     ep.smbios_major_version = 2;
     ep.smbios_minor_version = 4;
diff --git a/qemu/roms/seabios/src/fw/smm.c b/qemu/roms/seabios/src/fw/smm.c
index 6cb484e7e..8f042ee4d 100644
--- a/qemu/roms/seabios/src/fw/smm.c
+++ b/qemu/roms/seabios/src/fw/smm.c
@@ -64,11 +64,11 @@ handle_smi(u16 cs)
         return;
     u8 cmd = inb(PORT_SMI_CMD);
     struct smm_layout *smm = MAKE_FLATPTR(cs, 0);
+    u32 rev = smm->cpu.i32.smm_rev & SMM_REV_MASK;
     dprintf(DEBUG_HDL_smi, "handle_smi cmd=%x smbase=%p\n", cmd, smm);
 
     if (smm == (void*)BUILD_SMM_INIT_ADDR) {
         // relocate SMBASE to 0xa0000
-        u32 rev = smm->cpu.i32.smm_rev & SMM_REV_MASK;
         if (rev == SMM_REV_I32) {
             smm->cpu.i32.smm_base = BUILD_SMM_ADDR;
         } else if (rev == SMM_REV_I64) {
@@ -92,7 +92,7 @@ handle_smi(u16 cs)
     }
 
     if (CONFIG_CALL32_SMM && cmd == CALL32SMM_CMDID) {
-        if (smm->cpu.i32.smm_rev == SMM_REV_I32) {
+        if (rev == SMM_REV_I32) {
             u32 regs[8];
             memcpy(regs, &smm->cpu.i32.eax, sizeof(regs));
             if (smm->cpu.i32.ecx == CALL32SMM_ENTERID) {
@@ -107,7 +107,7 @@ handle_smi(u16 cs)
                 memcpy(&smm->cpu.i32.eax, regs, sizeof(regs));
                 smm->cpu.i32.eip = regs[3];
             }
-        } else if (smm->cpu.i64.smm_rev == SMM_REV_I64) {
+        } else if (rev == SMM_REV_I64) {
             u64 regs[8];
             memcpy(regs, &smm->cpu.i64.rdi, sizeof(regs));
             if ((u32)smm->cpu.i64.rcx == CALL32SMM_ENTERID) {
@@ -184,7 +184,7 @@ static void piix4_apmc_smm_setup(int isabdf, int i440_bdf)
 
     /* enable SMI generation */
     value = inl(acpi_pm_base + PIIX_PMIO_GLBCTL);
-    outl(acpi_pm_base + PIIX_PMIO_GLBCTL, value | PIIX_PMIO_GLBCTL_SMI_EN);
+    outl(value | PIIX_PMIO_GLBCTL_SMI_EN, acpi_pm_base + PIIX_PMIO_GLBCTL);
 
     smm_relocate_and_restore();
 
diff --git a/qemu/roms/seabios/src/fw/smp.c b/qemu/roms/seabios/src/fw/smp.c
index a466ea6e9..579acdbd0 100644
--- a/qemu/roms/seabios/src/fw/smp.c
+++ b/qemu/roms/seabios/src/fw/smp.c
@@ -52,9 +52,6 @@ handle_smp(void)
     if (!CONFIG_QEMU)
         return;
 
-    // Enable CPU caching
-    setcr0(getcr0() & ~(CR0_CD|CR0_NW));
-
     // Detect apic_id
     u32 eax, ebx, ecx, cpuid_features;
     cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
diff --git a/qemu/roms/seabios/src/fw/xen.c b/qemu/roms/seabios/src/fw/xen.c
index dd8e8afd4..3f19ef2dc 100644
--- a/qemu/roms/seabios/src/fw/xen.c
+++ b/qemu/roms/seabios/src/fw/xen.c
@@ -4,16 +4,17 @@
 //
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
-#include "config.h"
+#include "config.h" // CONFIG_XEN
+#include "e820map.h" // e820_add
 #include "hw/serialio.h" // DebugOutputPort
 #include "malloc.h" // memalign_high
-#include "memmap.h" // add_e820
+#include "memmap.h" // PAGE_SIZE
 #include "output.h" // dprintf
 #include "paravirt.h" // PlatformRunningOn
 #include "string.h" // memcpy
 #include "util.h" // copy_acpi_rsdp
 #include "x86.h" // cpuid
-#include "xen.h"
+#include "xen.h" // xen_extraversion_t
 
 #define INFO_PHYSICAL_ADDRESS 0x00001000
 
@@ -142,6 +143,6 @@ void xen_ramsize_preinit(void)
 
     for (i = 0; i < info->e820_nr; i++) {
         struct e820entry *e = &e820[i];
-        add_e820(e->start, e->size, e->type);
+        e820_add(e->start, e->size, e->type);
     }
 }
diff --git a/qemu/roms/seabios/src/hw/ahci.c b/qemu/roms/seabios/src/hw/ahci.c
index 3193d81a6..83b747cb2 100644
--- a/qemu/roms/seabios/src/hw/ahci.c
+++ b/qemu/roms/seabios/src/hw/ahci.c
@@ -213,7 +213,7 @@ static int ahci_command(struct ahci_port_s *port_gf, int iswrite, int isatapi,
 
 #define CDROM_CDB_SIZE 12
 
-int ahci_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+int ahci_atapi_process_op(struct disk_op_s *op)
 {
     if (! CONFIG_AHCI)
         return 0;
@@ -221,15 +221,14 @@ int ahci_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
     struct ahci_port_s *port_gf = container_of(
         op->drive_gf, struct ahci_port_s, drive);
     struct ahci_cmd_s *cmd = port_gf->cmd;
-    u8 *atapi = cdbcmd;
-    int i, rc;
 
+    if (op->command == CMD_WRITE || op->command == CMD_FORMAT)
+        return DISK_RET_EWRITEPROTECT;
+    int blocksize = scsi_fill_cmd(op, cmd->atapi, CDROM_CDB_SIZE);
+    if (blocksize < 0)
+        return default_process_op(op);
     sata_prep_atapi(&cmd->fis, blocksize);
-    for (i = 0; i < CDROM_CDB_SIZE; i++) {
-        cmd->atapi[i] = atapi[i];
-    }
-    rc = ahci_command(port_gf, 0, 1, op->buf_fl,
-                      op->count * blocksize);
+    int rc = ahci_command(port_gf, 0, 1, op->buf_fl, op->count * blocksize);
     if (rc < 0)
         return DISK_RET_EBADTRACK;
     return DISK_RET_SUCCESS;
@@ -296,8 +295,8 @@ ahci_disk_readwrite(struct disk_op_s *op, int iswrite)
 }
 
 // command demuxer
-int VISIBLE32FLAT
-process_ahci_op(struct disk_op_s *op)
+int
+ahci_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_AHCI)
         return 0;
@@ -306,15 +305,8 @@ process_ahci_op(struct disk_op_s *op)
         return ahci_disk_readwrite(op, 0);
     case CMD_WRITE:
         return ahci_disk_readwrite(op, 1);
-    case CMD_FORMAT:
-    case CMD_RESET:
-    case CMD_ISREADY:
-    case CMD_VERIFY:
-    case CMD_SEEK:
-        return DISK_RET_SUCCESS;
     default:
-        dprintf(1, "AHCI: unknown disk command %d\n", op->command);
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
 
@@ -405,6 +397,14 @@ static struct ahci_port_s* ahci_port_realloc(struct ahci_port_s *port)
     port->list = memalign_high(1024, 1024);
     port->fis = memalign_high(256, 256);
     port->cmd = memalign_high(256, 256);
+    if (!port->list || !port->fis || !port->cmd) {
+        warn_noalloc();
+        free(port->list);
+        free(port->fis);
+        free(port->cmd);
+        free(port);
+        return NULL;
+    }
 
     ahci_port_writel(port->ctrl, port->pnr, PORT_LST_ADDR, (u32)port->list);
     ahci_port_writel(port->ctrl, port->pnr, PORT_FIS_ADDR, (u32)port->fis);
diff --git a/qemu/roms/seabios/src/hw/ahci.h b/qemu/roms/seabios/src/hw/ahci.h
index c8c755a3f..fa11d6619 100644
--- a/qemu/roms/seabios/src/hw/ahci.h
+++ b/qemu/roms/seabios/src/hw/ahci.h
@@ -83,8 +83,8 @@ struct ahci_port_s {
 };
 
 void ahci_setup(void);
-int process_ahci_op(struct disk_op_s *op);
-int ahci_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int ahci_process_op(struct disk_op_s *op);
+int ahci_atapi_process_op(struct disk_op_s *op);
 
 #define AHCI_IRQ_ON_SG            (1 << 31)
 #define AHCI_CMD_ATAPI            (1 << 5)
diff --git a/qemu/roms/seabios/src/hw/ata.c b/qemu/roms/seabios/src/hw/ata.c
index d805706dd..fbbbbc1bb 100644
--- a/qemu/roms/seabios/src/hw/ata.c
+++ b/qemu/roms/seabios/src/hw/ata.c
@@ -552,7 +552,7 @@ ata_readwrite(struct disk_op_s *op, int iswrite)
 
 // 16bit command demuxer for ATA harddrives.
 int
-process_ata_op(struct disk_op_s *op)
+ata_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_ATA)
         return 0;
@@ -569,12 +569,8 @@ process_ata_op(struct disk_op_s *op)
         return DISK_RET_SUCCESS;
     case CMD_ISREADY:
         return isready(adrive_gf);
-    case CMD_FORMAT:
-    case CMD_VERIFY:
-    case CMD_SEEK:
-        return DISK_RET_SUCCESS;
     default:
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
 
@@ -587,11 +583,18 @@ process_ata_op(struct disk_op_s *op)
 
 // Low-level atapi command transmit function.
 int
-atapi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+ata_atapi_process_op(struct disk_op_s *op)
 {
     if (! CONFIG_ATA)
         return 0;
 
+    if (op->command == CMD_WRITE || op->command == CMD_FORMAT)
+        return DISK_RET_EWRITEPROTECT;
+    u8 cdbcmd[CDROM_CDB_SIZE];
+    int blocksize = scsi_fill_cmd(op, cdbcmd, sizeof(cdbcmd));
+    if (blocksize < 0)
+        return default_process_op(op);
+
     struct atadrive_s *adrive_gf = container_of(
         op->drive_gf, struct atadrive_s, drive);
     struct ata_channel_s *chan_gf = GET_GLOBALFLAT(adrive_gf->chan_gf);
@@ -715,7 +718,7 @@ init_atadrive(struct atadrive_s *dummy, u16 *buffer)
     memset(adrive, 0, sizeof(*adrive));
     adrive->chan_gf = dummy->chan_gf;
     adrive->slave = dummy->slave;
-    adrive->drive.cntl_id = adrive->chan_gf->chanid * 2 + dummy->slave;
+    adrive->drive.cntl_id = adrive->chan_gf->ataid * 2 + dummy->slave;
     adrive->drive.removable = (buffer[0] & 0x80) ? 1 : 0;
     return adrive;
 }
@@ -740,7 +743,7 @@ init_drive_atapi(struct atadrive_s *dummy, u16 *buffer)
     char model[MAXMODEL+1];
     char *desc = znprintf(MAXDESCSIZE
                           , "DVD/CD [ata%d-%d: %s ATAPI-%d %s]"
-                          , adrive->chan_gf->chanid, adrive->slave
+                          , adrive->chan_gf->ataid, adrive->slave
                           , ata_extract_model(model, MAXMODEL, buffer)
                           , ata_extract_version(buffer)
                           , (iscd ? "DVD/CD" : "Device"));
@@ -792,7 +795,7 @@ init_drive_ata(struct atadrive_s *dummy, u16 *buffer)
     char model[MAXMODEL+1];
     char *desc = znprintf(MAXDESCSIZE
                           , "ata%d-%d: %s ATA-%d Hard-Disk (%u %ciBytes)"
-                          , adrive->chan_gf->chanid, adrive->slave
+                          , adrive->chan_gf->ataid, adrive->slave
                           , ata_extract_model(model, MAXMODEL, buffer)
                           , ata_extract_version(buffer)
                           , (u32)adjsize, adjprefix);
@@ -866,7 +869,7 @@ ata_detect(void *data)
         u8 sc = inb(iobase1+ATA_CB_SC);
         u8 sn = inb(iobase1+ATA_CB_SN);
         dprintf(6, "ata_detect ata%d-%d: sc=%x sn=%x dh=%x\n"
-                , chan_gf->chanid, slave, sc, sn, dh);
+                , chan_gf->ataid, slave, sc, sn, dh);
         if (sc != 0x55 || sn != 0xaa || dh != newdh)
             continue;
 
@@ -913,16 +916,17 @@ ata_detect(void *data)
 
 // Initialize an ata controller and detect its drives.
 static void
-init_controller(struct pci_device *pci, int irq
+init_controller(struct pci_device *pci, int chanid, int irq
                 , u32 port1, u32 port2, u32 master)
 {
-    static int chanid = 0;
+    static int ataid = 0;
     struct ata_channel_s *chan_gf = malloc_fseg(sizeof(*chan_gf));
     if (!chan_gf) {
         warn_noalloc();
         return;
     }
-    chan_gf->chanid = chanid++;
+    chan_gf->ataid = ataid++;
+    chan_gf->chanid = chanid;
     chan_gf->irq = irq;
     chan_gf->pci_bdf = pci ? pci->bdf : -1;
     chan_gf->pci_tmp = pci;
@@ -930,7 +934,7 @@ init_controller(struct pci_device *pci, int irq
     chan_gf->iobase2 = port2;
     chan_gf->iomaster = master;
     dprintf(1, "ATA controller %d at %x/%x/%x (irq %d dev %x)\n"
-            , chanid, port1, port2, master, irq, chan_gf->pci_bdf);
+            , ataid, port1, port2, master, irq, chan_gf->pci_bdf);
     run_thread(ata_detect, chan_gf);
 }
 
@@ -966,7 +970,7 @@ init_pciata(struct pci_device *pci, u8 prog_if)
         port2 = PORT_ATA1_CTRL_BASE;
         irq = IRQ_ATA1;
     }
-    init_controller(pci, irq, port1, port2, master);
+    init_controller(pci, 0, irq, port1, port2, master);
 
     if (prog_if & 4) {
         port1 = (pci_config_readl(bdf, PCI_BASE_ADDRESS_2)
@@ -979,7 +983,7 @@ init_pciata(struct pci_device *pci, u8 prog_if)
         port2 = PORT_ATA2_CTRL_BASE;
         irq = IRQ_ATA2;
     }
-    init_controller(pci, irq, port1, port2, master ? master + 8 : 0);
+    init_controller(pci, 1, irq, port1, port2, master ? master + 8 : 0);
 }
 
 static void
@@ -1011,9 +1015,9 @@ ata_scan(void)
     if (CONFIG_QEMU && hlist_empty(&PCIDevices)) {
         // No PCI devices found - probably a QEMU "-M isapc" machine.
         // Try using ISA ports for ATA controllers.
-        init_controller(NULL, IRQ_ATA1
+        init_controller(NULL, 0, IRQ_ATA1
                         , PORT_ATA1_CMD_BASE, PORT_ATA1_CTRL_BASE, 0);
-        init_controller(NULL, IRQ_ATA2
+        init_controller(NULL, 1, IRQ_ATA2
                         , PORT_ATA2_CMD_BASE, PORT_ATA2_CTRL_BASE, 0);
         return;
     }
diff --git a/qemu/roms/seabios/src/hw/ata.h b/qemu/roms/seabios/src/hw/ata.h
index c73892bbe..cd14e59e9 100644
--- a/qemu/roms/seabios/src/hw/ata.h
+++ b/qemu/roms/seabios/src/hw/ata.h
@@ -11,6 +11,7 @@ struct ata_channel_s {
     u16 iomaster;
     u8  irq;
     u8  chanid;
+    u8  ataid;
     int pci_bdf;
     struct pci_device *pci_tmp;
 };
@@ -24,10 +25,9 @@ struct atadrive_s {
 // ata.c
 char *ata_extract_model(char *model, u32 size, u16 *buffer);
 int ata_extract_version(u16 *buffer);
-int cdrom_read(struct disk_op_s *op);
-int atapi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int ata_process_op(struct disk_op_s *op);
+int ata_atapi_process_op(struct disk_op_s *op);
 void ata_setup(void);
-int process_ata_op(struct disk_op_s *op);
 
 #define PORT_ATA2_CMD_BASE     0x0170
 #define PORT_ATA1_CMD_BASE     0x01f0
diff --git a/qemu/roms/seabios/src/hw/blockcmd.c b/qemu/roms/seabios/src/hw/blockcmd.c
index 78c0e65f4..0725b46db 100644
--- a/qemu/roms/seabios/src/hw/blockcmd.c
+++ b/qemu/roms/seabios/src/hw/blockcmd.c
@@ -5,67 +5,14 @@
 //
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
-#include "ahci.h" // atapi_cmd_data
-#include "ata.h" // atapi_cmd_data
 #include "biosvar.h" // GET_GLOBALFLAT
 #include "block.h" // struct disk_op_s
 #include "blockcmd.h" // struct cdb_request_sense
 #include "byteorder.h" // be32_to_cpu
-#include "esp-scsi.h" // esp_scsi_cmd_data
-#include "lsi-scsi.h" // lsi_scsi_cmd_data
-#include "megasas.h" // megasas_cmd_data
-#include "pvscsi.h" // pvscsi_cmd_data
 #include "output.h" // dprintf
 #include "std/disk.h" // DISK_RET_EPARAM
 #include "string.h" // memset
-#include "usb-msc.h" // usb_cmd_data
-#include "usb-uas.h" // usb_cmd_data
 #include "util.h" // timer_calc
-#include "virtio-scsi.h" // virtio_scsi_cmd_data
-
-// Route command to low-level handler.
-static int
-cdb_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
-{
-    u8 type = GET_GLOBALFLAT(op->drive_gf->type);
-    switch (type) {
-    case DTYPE_ATA_ATAPI:
-        return atapi_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_USB:
-        return usb_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_UAS:
-        return uas_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_VIRTIO_SCSI:
-        return virtio_scsi_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_LSI_SCSI:
-        return lsi_scsi_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_ESP_SCSI:
-        return esp_scsi_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_MEGASAS:
-        return megasas_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_USB_32:
-        if (!MODESEGMENT)
-            return usb_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_UAS_32:
-        if (!MODESEGMENT)
-            return uas_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_PVSCSI:
-        if (!MODESEGMENT)
-            return pvscsi_cmd_data(op, cdbcmd, blocksize);
-    case DTYPE_AHCI_ATAPI:
-        if (!MODESEGMENT)
-            return ahci_cmd_data(op, cdbcmd, blocksize);
-    default:
-        return DISK_RET_EPARAM;
-    }
-}
-
-// Determine if the command is a request to pull data from the device
-int
-cdb_is_read(u8 *cdbcmd, u16 blocksize)
-{
-    return blocksize && cdbcmd[0] != CDB_CMD_WRITE_10;
-}
 
 
 /****************************************************************
@@ -79,9 +26,12 @@ cdb_get_inquiry(struct disk_op_s *op, struct cdbres_inquiry *data)
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_INQUIRY;
     cmd.length = sizeof(*data);
+    op->command = CMD_SCSI;
     op->count = 1;
     op->buf_fl = data;
-    return cdb_cmd_data(op, &cmd, sizeof(*data));
+    op->cdbcmd = &cmd;
+    op->blocksize = sizeof(*data);
+    return process_op(op);
 }
 
 // Request SENSE
@@ -92,9 +42,12 @@ cdb_get_sense(struct disk_op_s *op, struct cdbres_request_sense *data)
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_REQUEST_SENSE;
     cmd.length = sizeof(*data);
+    op->command = CMD_SCSI;
     op->count = 1;
     op->buf_fl = data;
-    return cdb_cmd_data(op, &cmd, sizeof(*data));
+    op->cdbcmd = &cmd;
+    op->blocksize = sizeof(*data);
+    return process_op(op);
 }
 
 // Test unit ready
@@ -104,9 +57,12 @@ cdb_test_unit_ready(struct disk_op_s *op)
     struct cdb_request_sense cmd;
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_TEST_UNIT_READY;
+    op->command = CMD_SCSI;
     op->count = 0;
     op->buf_fl = NULL;
-    return cdb_cmd_data(op, &cmd, 0);
+    op->cdbcmd = &cmd;
+    op->blocksize = 0;
+    return process_op(op);
 }
 
 // Request capacity
@@ -116,9 +72,12 @@ cdb_read_capacity(struct disk_op_s *op, struct cdbres_read_capacity *data)
     struct cdb_read_capacity cmd;
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_READ_CAPACITY;
+    op->command = CMD_SCSI;
     op->count = 1;
     op->buf_fl = data;
-    return cdb_cmd_data(op, &cmd, sizeof(*data));
+    op->cdbcmd = &cmd;
+    op->blocksize = sizeof(*data);
+    return process_op(op);
 }
 
 // Mode sense, geometry page.
@@ -131,33 +90,12 @@ cdb_mode_sense_geom(struct disk_op_s *op, struct cdbres_mode_sense_geom *data)
     cmd.flags = 8; /* DBD */
     cmd.page = MODE_PAGE_HD_GEOMETRY;
     cmd.count = cpu_to_be16(sizeof(*data));
+    op->command = CMD_SCSI;
     op->count = 1;
     op->buf_fl = data;
-    return cdb_cmd_data(op, &cmd, sizeof(*data));
-}
-
-// Read sectors.
-static int
-cdb_read(struct disk_op_s *op)
-{
-    struct cdb_rwdata_10 cmd;
-    memset(&cmd, 0, sizeof(cmd));
-    cmd.command = CDB_CMD_READ_10;
-    cmd.lba = cpu_to_be32(op->lba);
-    cmd.count = cpu_to_be16(op->count);
-    return cdb_cmd_data(op, &cmd, GET_GLOBALFLAT(op->drive_gf->blksize));
-}
-
-// Write sectors.
-static int
-cdb_write(struct disk_op_s *op)
-{
-    struct cdb_rwdata_10 cmd;
-    memset(&cmd, 0, sizeof(cmd));
-    cmd.command = CDB_CMD_WRITE_10;
-    cmd.lba = cpu_to_be32(op->lba);
-    cmd.count = cpu_to_be16(op->count);
-    return cdb_cmd_data(op, &cmd, GET_GLOBALFLAT(op->drive_gf->blksize));
+    op->cdbcmd = &cmd;
+    op->blocksize = sizeof(*data);
+    return process_op(op);
 }
 
 
@@ -165,25 +103,36 @@ cdb_write(struct disk_op_s *op)
  * Main SCSI commands
  ****************************************************************/
 
-int VISIBLE32FLAT
-scsi_process_op(struct disk_op_s *op)
+// Create a scsi command request from a disk_op_s request
+int
+scsi_fill_cmd(struct disk_op_s *op, void *cdbcmd, int maxcdb)
 {
     switch (op->command) {
     case CMD_READ:
-        return cdb_read(op);
-    case CMD_WRITE:
-        return cdb_write(op);
-    case CMD_FORMAT:
-    case CMD_RESET:
-    case CMD_ISREADY:
-    case CMD_VERIFY:
-    case CMD_SEEK:
-        return DISK_RET_SUCCESS;
+    case CMD_WRITE: ;
+        struct cdb_rwdata_10 *cmd = cdbcmd;
+        memset(cmd, 0, maxcdb);
+        cmd->command = (op->command == CMD_READ ? CDB_CMD_READ_10
+                        : CDB_CMD_WRITE_10);
+        cmd->lba = cpu_to_be32(op->lba);
+        cmd->count = cpu_to_be16(op->count);
+        return GET_GLOBALFLAT(op->drive_gf->blksize);
+    case CMD_SCSI:
+        memcpy(cdbcmd, op->cdbcmd, maxcdb);
+        return op->blocksize;
     default:
-        return DISK_RET_EPARAM;
+        return -1;
     }
 }
 
+// Determine if the command is a request to pull data from the device
+int
+scsi_is_read(struct disk_op_s *op)
+{
+    return op->command == CMD_READ || (op->command == CMD_SCSI && op->blocksize);
+}
+
+// Check if a SCSI device is ready to receive commands
 int
 scsi_is_ready(struct disk_op_s *op)
 {
@@ -219,7 +168,7 @@ scsi_is_ready(struct disk_op_s *op)
 
         if (sense.asc == 0x04 && sense.ascq == 0x01 && !in_progress) {
             /* IN PROGRESS OF BECOMING READY */
-            printf("Waiting for device to detect medium... ");
+            dprintf(1, "Waiting for device to detect medium... ");
             /* Allow 30 seconds more */
             end = timer_calc(30000);
             in_progress = 1;
diff --git a/qemu/roms/seabios/src/hw/blockcmd.h b/qemu/roms/seabios/src/hw/blockcmd.h
index df12a6d42..b543f85eb 100644
--- a/qemu/roms/seabios/src/hw/blockcmd.h
+++ b/qemu/roms/seabios/src/hw/blockcmd.h
@@ -100,9 +100,9 @@ struct cdbres_mode_sense_geom {
 } PACKED;
 
 // blockcmd.c
-int cdb_is_read(u8 *cdbcmd, u16 blocksize);
 struct disk_op_s;
-int scsi_process_op(struct disk_op_s *op);
+int scsi_fill_cmd(struct disk_op_s *op, void *cdbcmd, int maxcdb);
+int scsi_is_read(struct disk_op_s *op);
 int scsi_is_ready(struct disk_op_s *op);
 struct drive_s;
 int scsi_drive_setup(struct drive_s *drive, const char *s, int prio);
diff --git a/qemu/roms/seabios/src/hw/esp-scsi.c b/qemu/roms/seabios/src/hw/esp-scsi.c
index 33cc44986..d4e47e3c5 100644
--- a/qemu/roms/seabios/src/hw/esp-scsi.c
+++ b/qemu/roms/seabios/src/hw/esp-scsi.c
@@ -76,10 +76,19 @@ esp_scsi_dma(u32 iobase, u32 buf, u32 len, int read)
     outb(read ? 0x83 : 0x03, iobase + ESP_DMA_CMD);
 }
 
-static int
-esp_scsi_cmd(struct esp_lun_s *llun_gf, struct disk_op_s *op,
-             u8 *cdbcmd, u16 target, u16 lun, u16 blocksize)
+int
+esp_scsi_process_op(struct disk_op_s *op)
 {
+    if (!CONFIG_ESP_SCSI)
+        return DISK_RET_EBADTRACK;
+    struct esp_lun_s *llun_gf =
+        container_of(op->drive_gf, struct esp_lun_s, drive);
+    u16 target = GET_GLOBALFLAT(llun_gf->target);
+    u16 lun = GET_GLOBALFLAT(llun_gf->lun);
+    u8 cdbcmd[16];
+    int blocksize = scsi_fill_cmd(op, cdbcmd, sizeof(cdbcmd));
+    if (blocksize < 0)
+        return default_process_op(op);
     u32 iobase = GET_GLOBALFLAT(llun_gf->iobase);
     int i, state;
     u8 status;
@@ -113,8 +122,7 @@ esp_scsi_cmd(struct esp_lun_s *llun_gf, struct disk_op_s *op,
             if (op->count && blocksize) {
                 /* Data phase.  */
                 u32 count = (u32)op->count * blocksize;
-                esp_scsi_dma(iobase, (u32)op->buf_fl, count,
-                             cdb_is_read(cdbcmd, blocksize));
+                esp_scsi_dma(iobase, (u32)op->buf_fl, count, scsi_is_read(op));
                 outb(ESP_CMD_TI | ESP_CMD_DMA, iobase + ESP_CMD);
                 continue;
             }
@@ -144,21 +152,6 @@ esp_scsi_cmd(struct esp_lun_s *llun_gf, struct disk_op_s *op,
     return DISK_RET_EBADTRACK;
 }
 
-int
-esp_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
-{
-    if (!CONFIG_ESP_SCSI)
-        return DISK_RET_EBADTRACK;
-
-    struct esp_lun_s *llun_gf =
-        container_of(op->drive_gf, struct esp_lun_s, drive);
-
-    return esp_scsi_cmd(llun_gf, op, cdbcmd,
-                        GET_GLOBALFLAT(llun_gf->target),
-                        GET_GLOBALFLAT(llun_gf->lun),
-                        blocksize);
-}
-
 static int
 esp_scsi_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
 {
diff --git a/qemu/roms/seabios/src/hw/esp-scsi.h b/qemu/roms/seabios/src/hw/esp-scsi.h
index dc555f395..0616d14b1 100644
--- a/qemu/roms/seabios/src/hw/esp-scsi.h
+++ b/qemu/roms/seabios/src/hw/esp-scsi.h
@@ -2,7 +2,7 @@
 #define __ESP_SCSI_H
 
 struct disk_op_s;
-int esp_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int esp_scsi_process_op(struct disk_op_s *op);
 void esp_scsi_setup(void);
 
 #endif /* __ESP_SCSI_H */
diff --git a/qemu/roms/seabios/src/hw/floppy.c b/qemu/roms/seabios/src/hw/floppy.c
index d60362a34..a14f7e093 100644
--- a/qemu/roms/seabios/src/hw/floppy.c
+++ b/qemu/roms/seabios/src/hw/floppy.c
@@ -613,7 +613,7 @@ floppy_format(struct disk_op_s *op)
 }
 
 int
-process_floppy_op(struct disk_op_s *op)
+floppy_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_FLOPPY)
         return 0;
diff --git a/qemu/roms/seabios/src/hw/lsi-scsi.c b/qemu/roms/seabios/src/hw/lsi-scsi.c
index b1d6bbf4b..ad3352886 100644
--- a/qemu/roms/seabios/src/hw/lsi-scsi.c
+++ b/qemu/roms/seabios/src/hw/lsi-scsi.c
@@ -50,12 +50,21 @@ struct lsi_lun_s {
     u8 lun;
 };
 
-static int
-lsi_scsi_cmd(struct lsi_lun_s *llun_gf, struct disk_op_s *op,
-             void *cdbcmd, u16 target, u16 lun, u16 blocksize)
+int
+lsi_scsi_process_op(struct disk_op_s *op)
 {
+    if (!CONFIG_LSI_SCSI)
+        return DISK_RET_EBADTRACK;
+    struct lsi_lun_s *llun_gf =
+        container_of(op->drive_gf, struct lsi_lun_s, drive);
+    u16 target = GET_GLOBALFLAT(llun_gf->target);
+    u16 lun = GET_GLOBALFLAT(llun_gf->lun);
+    u8 cdbcmd[16];
+    int blocksize = scsi_fill_cmd(op, cdbcmd, sizeof(cdbcmd));
+    if (blocksize < 0)
+        return default_process_op(op);
     u32 iobase = GET_GLOBALFLAT(llun_gf->iobase);
-    u32 dma = ((cdb_is_read(cdbcmd, blocksize) ? 0x01000000 : 0x00000000) |
+    u32 dma = ((scsi_is_read(op) ? 0x01000000 : 0x00000000) |
                (op->count * blocksize));
     u8 msgout[] = {
         0x80 | lun,                 // select lun
@@ -122,21 +131,6 @@ fail:
     return DISK_RET_EBADTRACK;
 }
 
-int
-lsi_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
-{
-    if (!CONFIG_LSI_SCSI)
-        return DISK_RET_EBADTRACK;
-
-    struct lsi_lun_s *llun_gf =
-        container_of(op->drive_gf, struct lsi_lun_s, drive);
-
-    return lsi_scsi_cmd(llun_gf, op, cdbcmd,
-                        GET_GLOBALFLAT(llun_gf->target),
-                        GET_GLOBALFLAT(llun_gf->lun),
-                        blocksize);
-}
-
 static int
 lsi_scsi_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
 {
diff --git a/qemu/roms/seabios/src/hw/lsi-scsi.h b/qemu/roms/seabios/src/hw/lsi-scsi.h
index 9c5a9b212..6baf4a162 100644
--- a/qemu/roms/seabios/src/hw/lsi-scsi.h
+++ b/qemu/roms/seabios/src/hw/lsi-scsi.h
@@ -2,7 +2,7 @@
 #define __LSI_SCSI_H
 
 struct disk_op_s;
-int lsi_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int lsi_scsi_process_op(struct disk_op_s *op);
 void lsi_scsi_setup(void);
 
 #endif /* __LSI_SCSI_H */
diff --git a/qemu/roms/seabios/src/hw/megasas.c b/qemu/roms/seabios/src/hw/megasas.c
index b2a65e48b..cb1a2a653 100644
--- a/qemu/roms/seabios/src/hw/megasas.c
+++ b/qemu/roms/seabios/src/hw/megasas.c
@@ -157,18 +157,20 @@ static int megasas_fire_cmd(u16 pci_id, u32 ioaddr,
 }
 
 int
-megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+megasas_process_op(struct disk_op_s *op)
 {
+    if (!CONFIG_MEGASAS)
+        return DISK_RET_EBADTRACK;
+    u8 cdb[16];
+    int blocksize = scsi_fill_cmd(op, cdb, sizeof(cdb));
+    if (blocksize < 0)
+        return default_process_op(op);
     struct megasas_lun_s *mlun_gf =
         container_of(op->drive_gf, struct megasas_lun_s, drive);
-    u8 *cdb = cdbcmd;
     struct megasas_cmd_frame *frame = GET_GLOBALFLAT(mlun_gf->frame);
     u16 pci_id = GET_GLOBALFLAT(mlun_gf->pci_id);
     int i;
 
-    if (!CONFIG_MEGASAS)
-        return DISK_RET_EBADTRACK;
-
     memset_fl(frame, 0, sizeof(*frame));
     SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);
     SET_LOWFLAT(frame->cmd_status, 0xFF);
@@ -241,7 +243,10 @@ static void megasas_scan_target(struct pci_device *pci, u32 iobase)
 {
     struct mfi_ld_list_s ld_list;
     struct megasas_cmd_frame *frame = memalign_tmp(256, sizeof(*frame));
-    int i;
+    if (!frame) {
+        warn_noalloc();
+        return;
+    }
 
     memset(&ld_list, 0, sizeof(ld_list));
     memset_fl(frame, 0, sizeof(*frame));
@@ -258,6 +263,7 @@ static void megasas_scan_target(struct pci_device *pci, u32 iobase)
 
     if (megasas_fire_cmd(pci->device, iobase, frame) == 0) {
         dprintf(2, "%d LD found\n", ld_list.count);
+        int i;
         for (i = 0; i < ld_list.count; i++) {
             dprintf(2, "LD %d:%d state 0x%x\n",
                     ld_list.lds[i].target, ld_list.lds[i].lun,
@@ -295,9 +301,9 @@ static int megasas_transition_to_ready(struct pci_device *pci, u32 ioaddr)
                 pci->device == PCI_DEVICE_ID_LSI_SAS2008 ||
                 pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
                 pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
-                outl(ioaddr + MFI_DB, mfi_flags);
+                outl(mfi_flags, ioaddr + MFI_DB);
             } else {
-                outl(ioaddr + MFI_IDB, mfi_flags);
+                outl(mfi_flags, ioaddr + MFI_IDB);
             }
             break;
         case MFI_STATE_OPERATIONAL:
@@ -306,7 +312,7 @@ static int megasas_transition_to_ready(struct pci_device *pci, u32 ioaddr)
                 pci->device == PCI_DEVICE_ID_LSI_SAS2008 ||
                 pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
                 pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
-                outl(ioaddr + MFI_DB, mfi_flags);
+                outl(mfi_flags, ioaddr + MFI_DB);
                 if (pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
                     pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
                     int j = 0;
@@ -321,7 +327,7 @@ static int megasas_transition_to_ready(struct pci_device *pci, u32 ioaddr)
                     }
                 }
             } else {
-                outw(ioaddr + MFI_IDB, mfi_flags);
+                outl(mfi_flags, ioaddr + MFI_IDB);
             }
             break;
         case MFI_STATE_READY:
diff --git a/qemu/roms/seabios/src/hw/megasas.h b/qemu/roms/seabios/src/hw/megasas.h
index 124042e1c..ed0e4f096 100644
--- a/qemu/roms/seabios/src/hw/megasas.h
+++ b/qemu/roms/seabios/src/hw/megasas.h
@@ -2,7 +2,7 @@
 #define __MEGASAS_H
 
 struct disk_op_s;
-int megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int megasas_process_op(struct disk_op_s *op);
 void megasas_setup(void);
 
 #endif /* __MEGASAS_H */
diff --git a/qemu/roms/seabios/src/hw/pci.c b/qemu/roms/seabios/src/hw/pci.c
index 0379b558e..a241d0675 100644
--- a/qemu/roms/seabios/src/hw/pci.c
+++ b/qemu/roms/seabios/src/hw/pci.c
@@ -221,16 +221,21 @@ pci_find_init_device(const struct pci_device_id *ids, void *arg)
     return NULL;
 }
 
-u8 pci_find_capability(struct pci_device *pci, u8 cap_id)
+u8 pci_find_capability(struct pci_device *pci, u8 cap_id, u8 cap)
 {
     int i;
-    u8 cap;
     u16 status = pci_config_readw(pci->bdf, PCI_STATUS);
 
     if (!(status & PCI_STATUS_CAP_LIST))
         return 0;
 
-    cap = pci_config_readb(pci->bdf, PCI_CAPABILITY_LIST);
+    if (cap == 0) {
+        /* find first */
+        cap = pci_config_readb(pci->bdf, PCI_CAPABILITY_LIST);
+    } else {
+        /* find next */
+        cap = pci_config_readb(pci->bdf, cap + PCI_CAP_LIST_NEXT);
+    }
     for (i = 0; cap && i <= 0xff; i++) {
         if (pci_config_readb(pci->bdf, cap + PCI_CAP_LIST_ID) == cap_id)
             return cap;
diff --git a/qemu/roms/seabios/src/hw/pci.h b/qemu/roms/seabios/src/hw/pci.h
index 0aaa84c1a..fc5e7b9bf 100644
--- a/qemu/roms/seabios/src/hw/pci.h
+++ b/qemu/roms/seabios/src/hw/pci.h
@@ -123,7 +123,7 @@ int pci_init_device(const struct pci_device_id *ids
                     , struct pci_device *pci, void *arg);
 struct pci_device *pci_find_init_device(const struct pci_device_id *ids
                                         , void *arg);
-u8 pci_find_capability(struct pci_device *pci, u8 cap_id);
+u8 pci_find_capability(struct pci_device *pci, u8 cap_id, u8 cap);
 int pci_bridge_has_region(struct pci_device *pci,
                           enum pci_region_type region_type);
 void pci_reboot(void);
diff --git a/qemu/roms/seabios/src/hw/pci_ids.h b/qemu/roms/seabios/src/hw/pci_ids.h
index 1cd4f7269..cdf9b3cbc 100644
--- a/qemu/roms/seabios/src/hw/pci_ids.h
+++ b/qemu/roms/seabios/src/hw/pci_ids.h
@@ -2616,8 +2616,12 @@
 #define PCI_DEVICE_ID_RME_DIGI32_8	0x9898
 
 #define PCI_VENDOR_ID_REDHAT_QUMRANET	0x1af4
-#define PCI_DEVICE_ID_VIRTIO_BLK	0x1001
-#define PCI_DEVICE_ID_VIRTIO_SCSI	0x1004
+/* virtio 0.9.5 ids (legacy/transitional devices) */
+#define PCI_DEVICE_ID_VIRTIO_BLK_09	0x1001
+#define PCI_DEVICE_ID_VIRTIO_SCSI_09	0x1004
+/* virtio 1.0 ids (modern devices) */
+#define PCI_DEVICE_ID_VIRTIO_BLK_10	0x1042
+#define PCI_DEVICE_ID_VIRTIO_SCSI_10	0x1048
 
 #define PCI_VENDOR_ID_VMWARE        0x15ad
 #define PCI_DEVICE_ID_VMWARE_PVSCSI 0x07C0
diff --git a/qemu/roms/seabios/src/hw/pic.c b/qemu/roms/seabios/src/hw/pic.c
index 6ff696765..d8b9764c7 100644
--- a/qemu/roms/seabios/src/hw/pic.c
+++ b/qemu/roms/seabios/src/hw/pic.c
@@ -13,12 +13,16 @@
 u16
 pic_irqmask_read(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return 0;
     return inb(PORT_PIC1_DATA) | (inb(PORT_PIC2_DATA) << 8);
 }
 
 void
 pic_irqmask_write(u16 mask)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     outb(mask, PORT_PIC1_DATA);
     outb(mask >> 8, PORT_PIC2_DATA);
 }
@@ -26,6 +30,8 @@ pic_irqmask_write(u16 mask)
 void
 pic_irqmask_mask(u16 off, u16 on)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     u8 pic1off = off, pic1on = on, pic2off = off>>8, pic2on = on>>8;
     outb((inb(PORT_PIC1_DATA) & ~pic1off) | pic1on, PORT_PIC1_DATA);
     outb((inb(PORT_PIC2_DATA) & ~pic2off) | pic2on, PORT_PIC2_DATA);
@@ -34,6 +40,8 @@ pic_irqmask_mask(u16 off, u16 on)
 void
 pic_reset(u8 irq0, u8 irq8)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     // Send ICW1 (select OCW1 + will send ICW4)
     outb(0x11, PORT_PIC1_CMD);
     outb(0x11, PORT_PIC2_CMD);
@@ -60,6 +68,8 @@ pic_setup(void)
 void
 enable_hwirq(int hwirq, struct segoff_s func)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     pic_irqmask_mask(1 << hwirq, 0);
     int vector;
     if (hwirq < 8)
@@ -72,6 +82,8 @@ enable_hwirq(int hwirq, struct segoff_s func)
 static u8
 pic_isr1_read(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return 0;
     // 0x0b == select OCW1 + read ISR
     outb(0x0b, PORT_PIC1_CMD);
     return inb(PORT_PIC1_CMD);
@@ -80,6 +92,8 @@ pic_isr1_read(void)
 static u8
 pic_isr2_read(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return 0;
     // 0x0b == select OCW1 + read ISR
     outb(0x0b, PORT_PIC2_CMD);
     return inb(PORT_PIC2_CMD);
diff --git a/qemu/roms/seabios/src/hw/pic.h b/qemu/roms/seabios/src/hw/pic.h
index 6947b6e81..f2d9f6130 100644
--- a/qemu/roms/seabios/src/hw/pic.h
+++ b/qemu/roms/seabios/src/hw/pic.h
@@ -34,6 +34,8 @@
 static inline void
 pic_eoi1(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     // Send eoi (select OCW2 + eoi)
     outb(0x20, PORT_PIC1_CMD);
 }
@@ -41,6 +43,8 @@ pic_eoi1(void)
 static inline void
 pic_eoi2(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     // Send eoi (select OCW2 + eoi)
     outb(0x20, PORT_PIC2_CMD);
     pic_eoi1();
diff --git a/qemu/roms/seabios/src/hw/ps2port.c b/qemu/roms/seabios/src/hw/ps2port.c
index 04995c881..d5504f71e 100644
--- a/qemu/roms/seabios/src/hw/ps2port.c
+++ b/qemu/roms/seabios/src/hw/ps2port.c
@@ -210,7 +210,7 @@ ps2_sendbyte(int aux, u8 command, int timeout)
     return 0;
 }
 
-u8 Ps2ctr VARLOW;
+u8 Ps2ctr VARLOW = I8042_CTR_KBDDIS | I8042_CTR_AUXDIS;
 
 static int
 __ps2_command(int aux, int command, u8 *param)
@@ -232,6 +232,7 @@ __ps2_command(int aux, int command, u8 *param)
     yield();
 
     // Enable port command is being sent to.
+    SET_LOW(Ps2ctr, newctr);
     if (aux)
         newctr &= ~I8042_CTR_AUXDIS;
     else
@@ -240,8 +241,8 @@ __ps2_command(int aux, int command, u8 *param)
     if (ret)
         goto fail;
 
-    if (command == ATKBD_CMD_RESET_BAT) {
-        // Reset is special wrt timeouts and bytes received.
+    if ((u8)command == (u8)ATKBD_CMD_RESET_BAT) {
+        // Reset is special wrt timeouts.
 
         // Send command.
         ret = ps2_sendbyte(aux, command, 1000);
@@ -253,11 +254,12 @@ __ps2_command(int aux, int command, u8 *param)
         if (ret < 0)
             goto fail;
         param[0] = ret;
-        ret = ps2_recvbyte(aux, 0, 100);
-        if (ret < 0)
-            // Some devices only respond with one byte on reset.
-            ret = 0;
-        param[1] = ret;
+        if (receive > 1) {
+            ret = ps2_recvbyte(aux, 0, 500);
+            if (ret < 0)
+                goto fail;
+            param[1] = ret;
+        }
     } else if (command == ATKBD_CMD_GETID) {
         // Getid is special wrt bytes received.
 
@@ -308,6 +310,7 @@ __ps2_command(int aux, int command, u8 *param)
 
 fail:
     // Restore interrupts and keyboard/mouse.
+    SET_LOW(Ps2ctr, ps2ctr);
     ret2 = i8042_command(I8042_CMD_CTL_WCTR, &ps2ctr);
     if (ret2)
         return ret2;
@@ -343,7 +346,8 @@ ps2_mouse_command(int command, u8 *param)
     if (command == PSMOUSE_CMD_ENABLE || command == PSMOUSE_CMD_DISABLE) {
         u8 ps2ctr = GET_LOW(Ps2ctr);
         if (command == PSMOUSE_CMD_ENABLE)
-            ps2ctr = (ps2ctr | I8042_CTR_AUXINT) & ~I8042_CTR_AUXDIS;
+            ps2ctr = ((ps2ctr | (CONFIG_HARDWARE_IRQ ? I8042_CTR_AUXINT : 0))
+                      & ~I8042_CTR_AUXDIS);
         else
             ps2ctr = (ps2ctr | I8042_CTR_AUXDIS) & ~I8042_CTR_AUXINT;
         SET_LOW(Ps2ctr, ps2ctr);
@@ -414,6 +418,31 @@ done:
     pic_eoi1();
 }
 
+// Check for ps2 activity on machines without hardware irqs
+void
+ps2_check_event(void)
+{
+    if (! CONFIG_PS2PORT || CONFIG_HARDWARE_IRQ)
+        return;
+    u8 ps2ctr = GET_LOW(Ps2ctr);
+    if ((ps2ctr & (I8042_CTR_KBDDIS|I8042_CTR_AUXDIS))
+        == (I8042_CTR_KBDDIS|I8042_CTR_AUXDIS))
+        return;
+    for (;;) {
+        u8 status = inb(PORT_PS2_STATUS);
+        if (!(status & I8042_STR_OBF))
+            break;
+        u8 data = inb(PORT_PS2_DATA);
+        if (status & I8042_STR_AUXDATA) {
+            if (!(ps2ctr & I8042_CTR_AUXDIS))
+                process_mouse(data);
+        } else {
+            if (!(ps2ctr & I8042_CTR_KBDDIS))
+                process_key(data);
+        }
+    }
+}
+
 
 /****************************************************************
  * Setup
@@ -446,9 +475,6 @@ ps2_keyboard_setup(void *data)
         return;
     }
 
-    // Disable keyboard and mouse events.
-    SET_LOW(Ps2ctr, I8042_CTR_KBDDIS | I8042_CTR_AUXDIS);
-
 
     /* ------------------- keyboard side ------------------------*/
     /* reset keyboard and self test  (keyboard side) */
@@ -482,7 +508,8 @@ ps2_keyboard_setup(void *data)
         return;
 
     // Keyboard Mode: disable mouse, scan code convert, enable kbd IRQ
-    SET_LOW(Ps2ctr, I8042_CTR_AUXDIS | I8042_CTR_XLATE | I8042_CTR_KBDINT);
+    Ps2ctr = (I8042_CTR_AUXDIS | I8042_CTR_XLATE
+              | (CONFIG_HARDWARE_IRQ ? I8042_CTR_KBDINT : 0));
 
     /* Enable keyboard */
     ret = ps2_kbd_command(ATKBD_CMD_ENABLE, NULL);
diff --git a/qemu/roms/seabios/src/hw/ps2port.h b/qemu/roms/seabios/src/hw/ps2port.h
index e5d9014b7..1338406ac 100644
--- a/qemu/roms/seabios/src/hw/ps2port.h
+++ b/qemu/roms/seabios/src/hw/ps2port.h
@@ -26,7 +26,7 @@
 #define ATKBD_CMD_GETID         0x02f2
 #define ATKBD_CMD_ENABLE        0x00f4
 #define ATKBD_CMD_RESET_DIS     0x00f5
-#define ATKBD_CMD_RESET_BAT     0x02ff
+#define ATKBD_CMD_RESET_BAT     0x01ff
 
 // Mouse commands
 #define PSMOUSE_CMD_SETSCALE11  0x00e6
@@ -61,6 +61,7 @@
 void i8042_reboot(void);
 int ps2_kbd_command(int command, u8 *param);
 int ps2_mouse_command(int command, u8 *param);
+void ps2_check_event(void);
 void ps2port_setup(void);
 
 #endif // ps2port.h
diff --git a/qemu/roms/seabios/src/hw/pvscsi.c b/qemu/roms/seabios/src/hw/pvscsi.c
index 601a551db..fa20efef7 100644
--- a/qemu/roms/seabios/src/hw/pvscsi.c
+++ b/qemu/roms/seabios/src/hw/pvscsi.c
@@ -11,6 +11,7 @@
 #include "blockcmd.h" // scsi_drive_setup
 #include "config.h" // CONFIG_*
 #include "malloc.h" // free
+#include "memmap.h" // PAGE_SHIFT, virt_to_phys
 #include "output.h" // dprintf
 #include "pci.h" // foreachpci
 #include "pci_ids.h" // PCI_DEVICE_ID_VMWARE_PVSCSI
@@ -19,7 +20,6 @@
 #include "std/disk.h" // DISK_RET_SUCCESS
 #include "string.h" // memset
 #include "util.h" // usleep
-#include "virtio-ring.h" // PAGE_SHIFT, virt_to_phys
 #include "x86.h" // writel
 
 #define MASK(n) ((1 << (n)) - 1)
@@ -197,29 +197,6 @@ pvscsi_init_rings(void *iobase, struct pvscsi_ring_dsc_s **ring_dsc)
     *ring_dsc = dsc;
 }
 
-static void pvscsi_fill_req(struct PVSCSIRingsState *s,
-                            struct PVSCSIRingReqDesc *req,
-                            u16 target, u16 lun, void *cdbcmd, u16 blocksize,
-                            struct disk_op_s *op)
-{
-    req->bus = 0;
-    req->target = target;
-    memset(req->lun, 0, sizeof(req->lun));
-    req->lun[1] = lun;
-    req->senseLen = 0;
-    req->senseAddr = 0;
-    req->cdbLen = 16;
-    req->vcpuHint = 0;
-    memcpy(req->cdb, cdbcmd, 16);
-    req->tag = SIMPLE_QUEUE_TAG;
-    req->flags = cdb_is_read(cdbcmd, blocksize) ?
-        PVSCSI_FLAG_CMD_DIR_TOHOST : PVSCSI_FLAG_CMD_DIR_TODEVICE;
-
-    req->dataLen = op->count * blocksize;
-    req->dataAddr = (u32)op->buf_fl;
-    s->reqProdIdx = s->reqProdIdx + 1;
-}
-
 static u32
 pvscsi_get_rsp(struct PVSCSIRingsState *s,
                struct PVSCSIRingCmpDesc *rsp)
@@ -229,10 +206,13 @@ pvscsi_get_rsp(struct PVSCSIRingsState *s,
     return status;
 }
 
-static int
-pvscsi_cmd(struct pvscsi_lun_s *plun, struct disk_op_s *op,
-           void *cdbcmd, u16 target, u16 lun, u16 blocksize)
+int
+pvscsi_process_op(struct disk_op_s *op)
 {
+    if (!CONFIG_PVSCSI)
+        return DISK_RET_EBADTRACK;
+    struct pvscsi_lun_s *plun =
+        container_of(op->drive_gf, struct pvscsi_lun_s, drive);
     struct pvscsi_ring_dsc_s *ring_dsc = plun->ring_dsc;
     struct PVSCSIRingsState *s = ring_dsc->ring_state;
     u32 req_entries = s->reqNumEntriesLog2;
@@ -248,7 +228,23 @@ pvscsi_cmd(struct pvscsi_lun_s *plun, struct disk_op_s *op,
     }
 
     req = ring_dsc->ring_reqs + (s->reqProdIdx & MASK(req_entries));
-    pvscsi_fill_req(s, req, target, lun, cdbcmd, blocksize, op);
+    int blocksize = scsi_fill_cmd(op, req->cdb, 16);
+    if (blocksize < 0)
+        return default_process_op(op);
+    req->bus = 0;
+    req->target = plun->target;
+    memset(req->lun, 0, sizeof(req->lun));
+    req->lun[1] = plun->lun;
+    req->senseLen = 0;
+    req->senseAddr = 0;
+    req->cdbLen = 16;
+    req->vcpuHint = 0;
+    req->tag = SIMPLE_QUEUE_TAG;
+    req->flags = scsi_is_read(op) ?
+        PVSCSI_FLAG_CMD_DIR_TOHOST : PVSCSI_FLAG_CMD_DIR_TODEVICE;
+    req->dataLen = op->count * blocksize;
+    req->dataAddr = (u32)op->buf_fl;
+    s->reqProdIdx = s->reqProdIdx + 1;
 
     pvscsi_kick_rw_io(plun->iobase);
     pvscsi_wait_intr_cmpl(plun->iobase);
@@ -259,18 +255,6 @@ pvscsi_cmd(struct pvscsi_lun_s *plun, struct disk_op_s *op,
     return status == 0 ? DISK_RET_SUCCESS : DISK_RET_EBADTRACK;
 }
 
-int
-pvscsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
-{
-    if (!CONFIG_PVSCSI)
-        return DISK_RET_EBADTRACK;
-
-    struct pvscsi_lun_s *plun =
-        container_of(op->drive_gf, struct pvscsi_lun_s, drive);
-
-    return pvscsi_cmd(plun, op, cdbcmd, plun->target, plun->lun, blocksize);
-}
-
 static int
 pvscsi_add_lun(struct pci_device *pci, void *iobase,
                struct pvscsi_ring_dsc_s *ring_dsc, u8 target, u8 lun)
diff --git a/qemu/roms/seabios/src/hw/pvscsi.h b/qemu/roms/seabios/src/hw/pvscsi.h
index fde9f0b98..5af7dcb0e 100644
--- a/qemu/roms/seabios/src/hw/pvscsi.h
+++ b/qemu/roms/seabios/src/hw/pvscsi.h
@@ -2,7 +2,7 @@
 #define _PVSCSI_H_
 
 struct disk_op_s;
-int pvscsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int pvscsi_process_op(struct disk_op_s *op);
 void pvscsi_setup(void);
 
 #endif /* _PVSCSI_H_ */
diff --git a/qemu/roms/seabios/src/hw/ramdisk.c b/qemu/roms/seabios/src/hw/ramdisk.c
index 1177bc00a..adec1d1b3 100644
--- a/qemu/roms/seabios/src/hw/ramdisk.c
+++ b/qemu/roms/seabios/src/hw/ramdisk.c
@@ -7,8 +7,9 @@
 #include "biosvar.h" // GET_GLOBALFLAT
 #include "block.h" // struct drive_s
 #include "bregs.h" // struct bregs
-#include "malloc.h" // malloc_fseg
-#include "memmap.h" // add_e820
+#include "e820map.h" // e820_add
+#include "malloc.h" // memalign_tmphigh
+#include "memmap.h" // PAGE_SIZE
 #include "output.h" // dprintf
 #include "romfile.h" // romfile_findprefix
 #include "stacks.h" // call16_int
@@ -41,7 +42,7 @@ ramdisk_setup(void)
         warn_noalloc();
         return;
     }
-    add_e820((u32)pos, size, E820_RESERVED);
+    e820_add((u32)pos, size, E820_RESERVED);
 
     // Copy image into ram.
     int ret = file->copy(file, pos, size);
@@ -53,7 +54,7 @@ ramdisk_setup(void)
     if (!drive)
         return;
     drive->type = DTYPE_RAMDISK;
-    dprintf(1, "Mapping CBFS floppy %s to addr %p\n", filename, pos);
+    dprintf(1, "Mapping floppy %s to addr %p\n", filename, pos);
     char *desc = znprintf(MAXDESCSIZE, "Ramdisk [%s]", &filename[10]);
     boot_add_floppy(drive, desc, bootprio_find_named_rom(filename, 0));
 }
@@ -91,7 +92,7 @@ ramdisk_copy(struct disk_op_s *op, int iswrite)
 }
 
 int
-process_ramdisk_op(struct disk_op_s *op)
+ramdisk_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_FLASH_FLOPPY)
         return 0;
@@ -101,11 +102,7 @@ process_ramdisk_op(struct disk_op_s *op)
         return ramdisk_copy(op, 0);
     case CMD_WRITE:
         return ramdisk_copy(op, 1);
-    case CMD_VERIFY:
-    case CMD_FORMAT:
-    case CMD_RESET:
-        return DISK_RET_SUCCESS;
     default:
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
diff --git a/qemu/roms/seabios/src/hw/rtc.c b/qemu/roms/seabios/src/hw/rtc.c
index 628d5429f..9649a5a79 100644
--- a/qemu/roms/seabios/src/hw/rtc.c
+++ b/qemu/roms/seabios/src/hw/rtc.c
@@ -30,6 +30,7 @@ rtc_write(u8 index, u8 val)
 void
 rtc_mask(u8 index, u8 off, u8 on)
 {
+    index |= NMI_DISABLE_BIT;
     outb(index, PORT_CMOS_INDEX);
     u8 val = inb(PORT_CMOS_DATA);
     outb((val & ~off) | on, PORT_CMOS_DATA);
@@ -62,6 +63,8 @@ rtc_updating(void)
 void
 rtc_setup(void)
 {
+    if (!CONFIG_RTC_TIMER)
+        return;
     rtc_write(CMOS_STATUS_A, 0x26);    // 32,768Khz src, 976.5625us updates
     rtc_mask(CMOS_STATUS_B, ~RTC_B_DSE, RTC_B_24HR);
     rtc_read(CMOS_STATUS_C);
@@ -73,6 +76,8 @@ int RTCusers VARLOW;
 void
 rtc_use(void)
 {
+    if (!CONFIG_RTC_TIMER)
+        return;
     int count = GET_LOW(RTCusers);
     SET_LOW(RTCusers, count+1);
     if (count)
@@ -84,6 +89,8 @@ rtc_use(void)
 void
 rtc_release(void)
 {
+    if (!CONFIG_RTC_TIMER)
+        return;
     int count = GET_LOW(RTCusers);
     SET_LOW(RTCusers, count-1);
     if (count != 1)
diff --git a/qemu/roms/seabios/src/hw/sdcard.c b/qemu/roms/seabios/src/hw/sdcard.c
index 6ff93c856..e01e1bb02 100644
--- a/qemu/roms/seabios/src/hw/sdcard.c
+++ b/qemu/roms/seabios/src/hw/sdcard.c
@@ -5,12 +5,12 @@
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
 #include "block.h" // struct drive_s
-#include "fw/paravirt.h" // runningOnQEMU
 #include "malloc.h" // malloc_fseg
 #include "output.h" // znprintf
 #include "pci.h" // pci_config_readl
 #include "pci_ids.h" // PCI_CLASS_SYSTEM_SDHCI
 #include "pci_regs.h" // PCI_BASE_ADDRESS_0
+#include "romfile.h" // romfile_findprefix
 #include "stacks.h" // wait_preempt
 #include "std/disk.h" // DISK_RET_SUCCESS
 #include "string.h" // memset
@@ -42,8 +42,8 @@ struct sdhci_s {
     u16 irq_signal;
     u16 error_signal;
     u16 auto_cmd12;
-    u8 pad_3E[2];
-    u64 cap;
+    u16 host_control2;
+    u32 cap_lo, cap_hi;
     u64 max_current;
     u16 force_auto_cmd12;
     u16 force_error;
@@ -56,25 +56,38 @@ struct sdhci_s {
 } PACKED;
 
 // SDHCI commands
-#define SC_ALL_SEND_CID         ((2<<8) | 0x21)
-#define SC_SEND_RELATIVE_ADDR   ((3<<8) | 0x22)
-#define SC_SELECT_DESELECT_CARD ((7<<8) | 0x23)
-#define SC_READ_SINGLE          ((17<<8) | 0x22)
-#define SC_READ_MULTIPLE        ((18<<8) | 0x22)
-#define SC_WRITE_SINGLE         ((24<<8) | 0x22)
-#define SC_WRITE_MULTIPLE       ((25<<8) | 0x22)
-#define SC_APP_CMD              ((55<<8) | 0x22)
-#define SC_APP_SEND_OP_COND ((41<<8) | 0x22)
+#define SCB_R0   0x00 // No response
+#define SCB_R48  0x1a // Response R1 (no data), R5, R6, R7
+#define SCB_R48d 0x3a // Response R1 (with data)
+#define SCB_R48b 0x1b // Response R1b, R5b
+#define SCB_R48o 0x02 // Response R3, R4
+#define SCB_R136 0x09 // Response R2
+#define SC_GO_IDLE_STATE        ((0<<8) | SCB_R0)
+#define SC_SEND_OP_COND         ((1<<8) | SCB_R48o)
+#define SC_ALL_SEND_CID         ((2<<8) | SCB_R136)
+#define SC_SEND_RELATIVE_ADDR   ((3<<8) | SCB_R48)
+#define SC_SELECT_DESELECT_CARD ((7<<8) | SCB_R48b)
+#define SC_SEND_IF_COND         ((8<<8) | SCB_R48)
+#define SC_SEND_EXT_CSD         ((8<<8) | SCB_R48d)
+#define SC_SEND_CSD             ((9<<8) | SCB_R136)
+#define SC_READ_SINGLE          ((17<<8) | SCB_R48d)
+#define SC_READ_MULTIPLE        ((18<<8) | SCB_R48d)
+#define SC_WRITE_SINGLE         ((24<<8) | SCB_R48d)
+#define SC_WRITE_MULTIPLE       ((25<<8) | SCB_R48d)
+#define SC_APP_CMD              ((55<<8) | SCB_R48)
+#define SC_APP_SEND_OP_COND ((41<<8) | SCB_R48o)
 
 // SDHCI irqs
 #define SI_CMD_COMPLETE (1<<0)
 #define SI_TRANS_DONE   (1<<1)
 #define SI_WRITE_READY  (1<<4)
 #define SI_READ_READY   (1<<5)
+#define SI_ERROR        (1<<15)
 
 // SDHCI present_state flags
-#define SP_CMD_INHIBIT (1<<0)
-#define SP_DAT_INHIBIT (1<<1)
+#define SP_CMD_INHIBIT   (1<<0)
+#define SP_DAT_INHIBIT   (1<<1)
+#define SP_CARD_INSERTED (1<<16)
 
 // SDHCI transfer_mode flags
 #define ST_BLOCKCOUNT (1<<1)
@@ -82,12 +95,43 @@ struct sdhci_s {
 #define ST_READ       (1<<4)
 #define ST_MULTIPLE   (1<<5)
 
+// SDHCI capabilities flags
+#define SD_CAPLO_V33             (1<<24)
+#define SD_CAPLO_V30             (1<<25)
+#define SD_CAPLO_V18             (1<<26)
+#define SD_CAPLO_BASECLOCK_SHIFT 8
+#define SD_CAPLO_BASECLOCK_MASK  0xff
+
+// SDHCI clock control flags
+#define SCC_INTERNAL_ENABLE (1<<0)
+#define SCC_STABLE          (1<<1)
+#define SCC_CLOCK_ENABLE    (1<<2)
+#define SCC_SDCLK_MASK      0xff
+#define SCC_SDCLK_SHIFT     8
+#define SCC_SDCLK_HI_MASK   0x300
+#define SCC_SDCLK_HI_RSHIFT 2
+
+// SDHCI power control flags
+#define SPC_POWER_ON (1<<0)
+#define SPC_V18      0x0a
+#define SPC_V30      0x0c
+#define SPC_V33      0x0e
+
+// SDHCI software reset flags
+#define SRF_ALL  0x01
+#define SRF_CMD  0x02
+#define SRF_DATA 0x04
+
 // SDHCI result flags
-#define SR_OCR_CCS (1<<30)
+#define SR_OCR_CCS     (1<<30)
+#define SR_OCR_NOTBUSY (1<<31)
 
 // SDHCI timeouts
-#define SDHCI_PIO_TIMEOUT      1000  // XXX - these are just made up
-#define SDHCI_TRANSFER_TIMEOUT 10000
+#define SDHCI_POWER_OFF_TIME   1
+#define SDHCI_POWER_ON_TIME    1
+#define SDHCI_CLOCK_ON_TIME    1 // 74 clock cycles
+#define SDHCI_POWERUP_TIMEOUT  1000
+#define SDHCI_PIO_TIMEOUT      1000  // XXX - this is just made up
 
 // Internal 'struct drive_s' storage for a detected card
 struct sddrive_s {
@@ -97,18 +141,18 @@ struct sddrive_s {
 };
 
 // SD card types
-#define SF_MMC  0
-#define SF_SDSC 1
-#define SF_SDHC 2
+#define SF_MMC          (1<<0)
+#define SF_HIGHCAPACITY (1<<1)
 
-// Repeatedly read a u16 register until the specific value is found
+// Repeatedly read a u16 register until any bit in a given mask is set
 static int
-waitw(u16 *reg, u16 mask, u16 value, u32 end)
+sdcard_waitw(u16 *reg, u16 mask)
 {
+    u32 end = timer_calc(SDHCI_PIO_TIMEOUT);
     for (;;) {
         u16 v = readw(reg);
-        if ((v & mask) == value)
-            return 0;
+        if (v & mask)
+            return v;
         if (timer_check(end)) {
             warn_timeout();
             return -1;
@@ -117,24 +161,49 @@ waitw(u16 *reg, u16 mask, u16 value, u32 end)
     }
 }
 
+// Send an sdhci reset
+static int
+sdcard_reset(struct sdhci_s *regs, int flags)
+{
+    writeb(&regs->software_reset, flags);
+    u32 end = timer_calc(SDHCI_PIO_TIMEOUT);
+    while (readb(&regs->software_reset))
+        if (timer_check(end)) {
+            warn_timeout();
+            return -1;
+        }
+    return 0;
+}
+
 // Send a command to the card.
 static int
 sdcard_pio(struct sdhci_s *regs, int cmd, u32 *param)
 {
-    u32 end = timer_calc(SDHCI_PIO_TIMEOUT);
-    u16 busyf = SP_CMD_INHIBIT | ((cmd & 0x03) == 0x03 ? SP_DAT_INHIBIT : 0);
-    int ret = waitw((u16*)&regs->present_state, busyf, 0, end);
-    if (ret)
-        return ret;
+    u32 state = readl(&regs->present_state);
+    dprintf(9, "sdcard_pio cmd %x %x %x\n", cmd, *param, state);
+    if ((state & SP_CMD_INHIBIT)
+        || ((cmd & 0x03) == 0x03 && state & SP_DAT_INHIBIT)) {
+        dprintf(1, "sdcard_pio not ready %x\n", state);
+        return -1;
+    }
     // Send command
     writel(&regs->arg, *param);
     writew(&regs->cmd, cmd);
-    ret = waitw(&regs->irq_status, SI_CMD_COMPLETE, SI_CMD_COMPLETE, end);
-    if (ret)
+    int ret = sdcard_waitw(&regs->irq_status, SI_ERROR|SI_CMD_COMPLETE);
+    if (ret < 0)
         return ret;
+    if (ret & SI_ERROR) {
+        u16 err = readw(&regs->error_irq_status);
+        dprintf(3, "sdcard_pio command stop (code=%x)\n", err);
+        sdcard_reset(regs, SRF_CMD|SRF_DATA);
+        writew(&regs->error_irq_status, err);
+        return -1;
+    }
     writew(&regs->irq_status, SI_CMD_COMPLETE);
     // Read response
     memcpy(param, regs->response, sizeof(regs->response));
+    dprintf(9, "sdcard cmd %x response %x %x %x %x\n"
+            , cmd, param[0], param[1], param[2], param[3]);
     return 0;
 }
 
@@ -155,24 +224,23 @@ sdcard_pio_transfer(struct sddrive_s *drive, int cmd, u32 addr
                     , void *data, int count)
 {
     // Send command
-    writel(&drive->regs->block_size, DISK_SECTOR_SIZE);
-    writew(&drive->regs->block_count, count); // XXX - SC_SET_BLOCK_COUNT?
+    writew(&drive->regs->block_size, DISK_SECTOR_SIZE);
+    writew(&drive->regs->block_count, count);
     int isread = cmd != SC_WRITE_SINGLE && cmd != SC_WRITE_MULTIPLE;
     u16 tmode = ((count > 1 ? ST_MULTIPLE|ST_AUTO_CMD12|ST_BLOCKCOUNT : 0)
                  | (isread ? ST_READ : 0));
     writew(&drive->regs->transfer_mode, tmode);
-    if (drive->card_type < SF_SDHC)
+    if (!(drive->card_type & SF_HIGHCAPACITY))
         addr *= DISK_SECTOR_SIZE;
     u32 param[4] = { addr };
     int ret = sdcard_pio(drive->regs, cmd, param);
     if (ret)
         return ret;
     // Read/write data
-    u32 end = timer_calc(SDHCI_TRANSFER_TIMEOUT);
     u16 cbit = isread ? SI_READ_READY : SI_WRITE_READY;
     while (count--) {
-        ret = waitw(&drive->regs->irq_status, cbit, cbit, end);
-        if (ret)
+        ret = sdcard_waitw(&drive->regs->irq_status, cbit);
+        if (ret < 0)
             return ret;
         writew(&drive->regs->irq_status, cbit);
         int i;
@@ -185,9 +253,8 @@ sdcard_pio_transfer(struct sddrive_s *drive, int cmd, u32 addr
         }
     }
     // Complete command
-    // XXX - SC_STOP_TRANSMISSION?
-    ret = waitw(&drive->regs->irq_status, SI_TRANS_DONE, SI_TRANS_DONE, end);
-    if (ret)
+    ret = sdcard_waitw(&drive->regs->irq_status, SI_TRANS_DONE);
+    if (ret < 0)
         return ret;
     writew(&drive->regs->irq_status, SI_TRANS_DONE);
     return 0;
@@ -208,8 +275,8 @@ sdcard_readwrite(struct disk_op_s *op, int iswrite)
     return DISK_RET_SUCCESS;
 }
 
-int VISIBLE32FLAT
-process_sdcard_op(struct disk_op_s *op)
+int
+sdcard_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_SDCARD)
         return 0;
@@ -218,14 +285,8 @@ process_sdcard_op(struct disk_op_s *op)
         return sdcard_readwrite(op, 0);
     case CMD_WRITE:
         return sdcard_readwrite(op, 1);
-    case CMD_FORMAT:
-    case CMD_RESET:
-    case CMD_ISREADY:
-    case CMD_VERIFY:
-    case CMD_SEEK:
-        return DISK_RET_SUCCESS;
     default:
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
 
@@ -234,75 +295,253 @@ process_sdcard_op(struct disk_op_s *op)
  * Setup
  ****************************************************************/
 
+static int
+sdcard_set_power(struct sdhci_s *regs)
+{
+    u32 cap = readl(&regs->cap_lo);
+    u32 volt, vbits;
+    if (cap & SD_CAPLO_V33) {
+        volt = 1<<20;
+        vbits = SPC_V33;
+    } else if (cap & SD_CAPLO_V30) {
+        volt = 1<<18;
+        vbits = SPC_V30;
+    } else if (cap & SD_CAPLO_V18) {
+        volt = 1<<7;
+        vbits = SPC_V18;
+    } else {
+        dprintf(1, "SD controller unsupported volt range (%x)\n", cap);
+        return -1;
+    }
+    writeb(&regs->power_control, 0);
+    msleep(SDHCI_POWER_OFF_TIME);
+    writeb(&regs->power_control, vbits | SPC_POWER_ON);
+    msleep(SDHCI_POWER_ON_TIME);
+    return volt;
+}
+
+static int
+sdcard_set_frequency(struct sdhci_s *regs, u32 khz)
+{
+    u16 ver = readw(&regs->controller_version);
+    u32 cap = readl(&regs->cap_lo);
+    u32 base_freq = (cap >> SD_CAPLO_BASECLOCK_SHIFT) & SD_CAPLO_BASECLOCK_MASK;
+    if (!base_freq) {
+        dprintf(1, "Unknown base frequency for SD controller\n");
+        return -1;
+    }
+    // Set new frequency
+    u32 divisor = DIV_ROUND_UP(base_freq * 1000, khz);
+    u16 creg;
+    if ((ver & 0xff) <= 0x01) {
+        divisor = divisor > 1 ? 1 << __fls(divisor-1) : 0;
+        creg = (divisor & SCC_SDCLK_MASK) << SCC_SDCLK_SHIFT;
+    } else {
+        divisor = DIV_ROUND_UP(divisor, 2);
+        creg = (divisor & SCC_SDCLK_MASK) << SCC_SDCLK_SHIFT;
+        creg |= (divisor & SCC_SDCLK_HI_MASK) >> SCC_SDCLK_HI_RSHIFT;
+    }
+    dprintf(3, "sdcard_set_frequency %d %d %x\n", base_freq, khz, creg);
+    writew(&regs->clock_control, 0);
+    writew(&regs->clock_control, creg | SCC_INTERNAL_ENABLE);
+    // Wait for frequency to become active
+    int ret = sdcard_waitw(&regs->clock_control, SCC_STABLE);
+    if (ret < 0)
+        return ret;
+    // Enable SD clock
+    writew(&regs->clock_control, creg | SCC_INTERNAL_ENABLE | SCC_CLOCK_ENABLE);
+    return 0;
+}
+
+// Obtain the disk size of an SD card
+static int
+sdcard_get_capacity(struct sddrive_s *drive, u8 *csd)
+{
+    // Original MMC/SD card capacity formula
+    u16 C_SIZE = (csd[6] >> 6) | (csd[7] << 2) | ((csd[8] & 0x03) << 10);
+    u8 C_SIZE_MULT = (csd[4] >> 7) | ((csd[5] & 0x03) << 1);
+    u8 READ_BL_LEN = csd[9] & 0x0f;
+    u32 count = (C_SIZE+1) << (C_SIZE_MULT + 2 + READ_BL_LEN - 9);
+    // Check for newer encoding formats.
+    u8 CSD_STRUCTURE = csd[14] >> 6;
+    if ((drive->card_type & SF_MMC) && CSD_STRUCTURE >= 2) {
+        // Get capacity from EXT_CSD register
+        u8 ext_csd[512];
+        int ret = sdcard_pio_transfer(drive, SC_SEND_EXT_CSD, 0, ext_csd, 1);
+        if (ret)
+            return ret;
+        count = *(u32*)&ext_csd[212];
+    } else if (!(drive->card_type & SF_MMC) && CSD_STRUCTURE >= 1) {
+        // High capacity SD card
+        u32 C_SIZE2 = csd[5] | (csd[6] << 8) | ((csd[7] & 0x3f) << 16);
+        count = (C_SIZE2+1) << (19-9);
+    }
+    // Fill drive struct and return
+    drive->drive.blksize = DISK_SECTOR_SIZE;
+    drive->drive.sectors = count;
+    return 0;
+}
+
 // Initialize an SD card
 static int
-sdcard_card_setup(struct sdhci_s *regs)
+sdcard_card_setup(struct sddrive_s *drive, int volt, int prio)
 {
-    // XXX - works on QEMU; probably wont on real hardware!
-    u32 param[4] = { 0x01 };
-    int ret = sdcard_pio_app(regs, SC_APP_SEND_OP_COND, param);
+    struct sdhci_s *regs = drive->regs;
+    // Set controller to initialization clock rate
+    int ret = sdcard_set_frequency(regs, 400);
+    if (ret)
+        return ret;
+    msleep(SDHCI_CLOCK_ON_TIME);
+    // Reset card
+    u32 param[4] = { };
+    ret = sdcard_pio(regs, SC_GO_IDLE_STATE, param);
     if (ret)
         return ret;
-    int card_type = (param[0] & SR_OCR_CCS) ? SF_SDHC : SF_SDSC;
+    // Let card know SDHC/SDXC is supported and confirm voltage
+    u32 hcs = 0, vrange = (volt >= (1<<15) ? 0x100 : 0x200) | 0xaa;
+    param[0] = vrange;
+    ret = sdcard_pio(regs, SC_SEND_IF_COND, param);
+    if (!ret && param[0] == vrange)
+        hcs = (1<<30);
+    // Verify SD card (instead of MMC or SDIO)
+    param[0] = 0x00;
+    ret = sdcard_pio_app(regs, SC_APP_SEND_OP_COND, param);
+    if (ret) {
+        // Check for MMC card
+        param[0] = 0x00;
+        ret = sdcard_pio(regs, SC_SEND_OP_COND, param);
+        if (ret)
+            return ret;
+        drive->card_type |= SF_MMC;
+        hcs = (1<<30);
+    }
+    // Init card
+    u32 end = timer_calc(SDHCI_POWERUP_TIMEOUT);
+    for (;;) {
+        param[0] = hcs | volt; // high-capacity support and voltage level
+        if (drive->card_type & SF_MMC)
+            ret = sdcard_pio(regs, SC_SEND_OP_COND, param);
+        else
+            ret = sdcard_pio_app(regs, SC_APP_SEND_OP_COND, param);
+        if (ret)
+            return ret;
+        if (param[0] & SR_OCR_NOTBUSY)
+            break;
+        if (timer_check(end)) {
+            warn_timeout();
+            return -1;
+        }
+        msleep(5); // Avoid flooding log when debugging
+    }
+    drive->card_type |= (param[0] & SR_OCR_CCS) ? SF_HIGHCAPACITY : 0;
+    // Select card (get cid, set rca, get csd, select card)
     param[0] = 0x00;
     ret = sdcard_pio(regs, SC_ALL_SEND_CID, param);
     if (ret)
         return ret;
-    param[0] = 0x01 << 16;
+    u8 cid[16];
+    memcpy(cid, param, sizeof(cid));
+    param[0] = drive->card_type & SF_MMC ? 0x0001 << 16 : 0x00;
     ret = sdcard_pio(regs, SC_SEND_RELATIVE_ADDR, param);
     if (ret)
         return ret;
-    u16 rca = param[0] >> 16;
+    u16 rca = drive->card_type & SF_MMC ? 0x0001 : param[0] >> 16;
+    param[0] = rca << 16;
+    ret = sdcard_pio(regs, SC_SEND_CSD, param);
+    if (ret)
+        return ret;
+    u8 csd[16];
+    memcpy(csd, param, sizeof(csd));
     param[0] = rca << 16;
     ret = sdcard_pio(regs, SC_SELECT_DESELECT_CARD, param);
     if (ret)
         return ret;
-    return card_type;
+    // Set controller to data transfer clock rate
+    ret = sdcard_set_frequency(regs, 25000);
+    if (ret)
+        return ret;
+    // Register drive
+    ret = sdcard_get_capacity(drive, csd);
+    if (ret)
+        return ret;
+    char pnm[7] = {};
+    int i;
+    for (i=0; i < (drive->card_type & SF_MMC ? 6 : 5); i++)
+        pnm[i] = cid[11-i];
+    char *desc = znprintf(MAXDESCSIZE, "%s %s %dMiB"
+                          , drive->card_type & SF_MMC ? "MMC drive" : "SD card"
+                          , pnm, (u32)(drive->drive.sectors >> 11));
+    dprintf(1, "Found sdcard at %p: %s\n", regs, desc);
+    boot_add_hd(&drive->drive, desc, prio);
+    return 0;
 }
 
 // Setup and configure an SD card controller
 static void
-sdcard_controller_setup(void *data)
+sdcard_controller_setup(struct sdhci_s *regs, int prio)
 {
-    struct pci_device *pci = data;
-    u16 bdf = pci->bdf;
-    wait_preempt();  // Avoid pci_config_readl when preempting
-    struct sdhci_s *regs = (void*)pci_config_readl(bdf, PCI_BASE_ADDRESS_0);
-    pci_config_maskw(bdf, PCI_COMMAND, 0,
-                     PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
-
     // Initialize controller
-    if (!runningOnQEMU())
-        // XXX - this init logic will probably only work on qemu!
+    u32 present_state = readl(&regs->present_state);
+    if (!(present_state & SP_CARD_INSERTED))
+        // No card present
         return;
+    dprintf(3, "sdhci@%p ver=%x cap=%x %x\n", regs
+            , readw(&regs->controller_version)
+            , readl(&regs->cap_lo), readl(&regs->cap_hi));
+    sdcard_reset(regs, SRF_ALL);
     writew(&regs->irq_signal, 0);
-    writew(&regs->irq_enable, 0xffff);
+    writew(&regs->irq_enable, 0x01ff);
+    writew(&regs->irq_status, readw(&regs->irq_status));
     writew(&regs->error_signal, 0);
-    writeb(&regs->power_control, 0x0f);
-    writew(&regs->clock_control, 0x0005);
-
-    // Initialize card
-    int card_type = sdcard_card_setup(regs);
-    if (card_type < 0)
+    writew(&regs->error_irq_enable, 0x01ff);
+    writew(&regs->error_irq_status, readw(&regs->error_irq_status));
+    writeb(&regs->timeout_control, 0x0e); // Set to max timeout
+    int volt = sdcard_set_power(regs);
+    if (volt < 0)
         return;
 
-    // Register drive
+    // Initialize card
     struct sddrive_s *drive = malloc_fseg(sizeof(*drive));
     if (!drive) {
         warn_noalloc();
-        return;
+        goto fail;
     }
     memset(drive, 0, sizeof(*drive));
     drive->drive.type = DTYPE_SDCARD;
-    drive->drive.blksize = DISK_SECTOR_SIZE;
-    drive->drive.sectors = (u64)-1; // XXX
     drive->regs = regs;
-    drive->card_type = card_type;
+    int ret = sdcard_card_setup(drive, volt, prio);
+    if (ret) {
+        free(drive);
+        goto fail;
+    }
+    return;
+fail:
+    writeb(&regs->power_control, 0);
+    writew(&regs->clock_control, 0);
+}
 
-    dprintf(1, "Found SD Card at %02x:%02x.%x\n"
-            , pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf), pci_bdf_to_fn(bdf));
-    char *desc = znprintf(MAXDESCSIZE, "SD Card"); // XXX
-    boot_add_hd(&drive->drive, desc, bootprio_find_pci_device(pci));
+static void
+sdcard_pci_setup(void *data)
+{
+    struct pci_device *pci = data;
+    wait_preempt();  // Avoid pci_config_readl when preempting
+    // XXX - bars dependent on slot index register in pci config space
+    u32 regs = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0);
+    regs &= PCI_BASE_ADDRESS_MEM_MASK;
+    pci_config_maskw(pci->bdf, PCI_COMMAND, 0,
+                     PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
+    int prio = bootprio_find_pci_device(pci);
+    sdcard_controller_setup((void*)regs, prio);
+}
+
+static void
+sdcard_romfile_setup(void *data)
+{
+    struct romfile_s *file = data;
+    int prio = bootprio_find_named_rom(file->name, 0);
+    u32 addr = romfile_loadint(file->name, 0);
+    dprintf(1, "Starting sdcard controller check at addr %x\n", addr);
+    sdcard_controller_setup((void*)addr, prio);
 }
 
 void
@@ -311,11 +550,19 @@ sdcard_setup(void)
     if (!CONFIG_SDCARD)
         return;
 
+    struct romfile_s *file = NULL;
+    for (;;) {
+        file = romfile_findprefix("etc/sdcard", file);
+        if (!file)
+            break;
+        run_thread(sdcard_romfile_setup, file);
+    }
+
     struct pci_device *pci;
     foreachpci(pci) {
         if (pci->class != PCI_CLASS_SYSTEM_SDHCI || pci->prog_if >= 2)
             // Not an SDHCI controller following SDHCI spec
             continue;
-        run_thread(sdcard_controller_setup, pci);
+        run_thread(sdcard_pci_setup, pci);
     }
 }
diff --git a/qemu/roms/seabios/src/hw/timer.c b/qemu/roms/seabios/src/hw/timer.c
index 5edc9fdbb..03d22b2f5 100644
--- a/qemu/roms/seabios/src/hw/timer.c
+++ b/qemu/roms/seabios/src/hw/timer.c
@@ -49,8 +49,8 @@
 #define PMTIMER_HZ 3579545      // Underlying Hz of the PM Timer
 #define PMTIMER_TO_PIT 3        // Ratio of pmtimer rate to pit rate
 
-u32 TimerKHz VARFSEG;
-u16 TimerPort VARFSEG;
+u32 TimerKHz VARFSEG = DIV_ROUND_UP(PMTIMER_HZ, 1000 * PMTIMER_TO_PIT);
+u16 TimerPort VARFSEG = PORT_PIT_COUNTER0;
 u8 ShiftTSC VARFSEG;
 
 
@@ -92,6 +92,7 @@ tsctimer_setup(void)
         t = (t + 1) >> 1;
     }
     TimerKHz = DIV_ROUND_UP((u32)t, 1000 * PMTIMER_TO_PIT);
+    TimerPort = 0;
 
     dprintf(1, "CPU Mhz=%u\n", (TimerKHz << ShiftTSC) / 1000);
 }
@@ -100,24 +101,16 @@ tsctimer_setup(void)
 void
 timer_setup(void)
 {
-    if (CONFIG_PMTIMER && TimerPort) {
-        dprintf(3, "pmtimer already configured; will not calibrate TSC\n");
+    if (!CONFIG_TSC_TIMER || (CONFIG_PMTIMER && TimerPort != PORT_PIT_COUNTER0))
         return;
-    }
 
+    // Check if CPU has a timestamp counter
     u32 eax, ebx, ecx, edx, cpuid_features = 0;
     cpuid(0, &eax, &ebx, &ecx, &edx);
     if (eax > 0)
         cpuid(1, &eax, &ebx, &ecx, &cpuid_features);
-
-    if (!(cpuid_features & CPUID_TSC)) {
-        TimerPort = PORT_PIT_COUNTER0;
-        TimerKHz = DIV_ROUND_UP(PMTIMER_HZ, 1000 * PMTIMER_TO_PIT);
-        dprintf(3, "386/486 class CPU. Using TSC emulation\n");
-        return;
-    }
-
-    tsctimer_setup();
+    if (cpuid_features & CPUID_TSC)
+        tsctimer_setup();
 }
 
 void
@@ -154,7 +147,7 @@ static u32
 timer_read(void)
 {
     u16 port = GET_GLOBAL(TimerPort);
-    if (!port)
+    if (CONFIG_TSC_TIMER && !port)
         // Read from CPU TSC
         return rdtscll() >> GET_GLOBAL(ShiftTSC);
     if (CONFIG_PMTIMER && port != PORT_PIT_COUNTER0)
@@ -249,6 +242,8 @@ ticks_from_ms(u32 ms)
 void
 pit_setup(void)
 {
+    if (!CONFIG_HARDWARE_IRQ)
+        return;
     // timer0: binary count, 16bit count, mode 2
     outb(PM_SEL_TIMER0|PM_ACCESS_WORD|PM_MODE2|PM_CNT_BINARY, PORT_PIT_MODE);
     // maximum count of 0000H = 18.2Hz
diff --git a/qemu/roms/seabios/src/hw/tpm_drivers.c b/qemu/roms/seabios/src/hw/tpm_drivers.c
new file mode 100644
index 000000000..444eac39b
--- /dev/null
+++ b/qemu/roms/seabios/src/hw/tpm_drivers.c
@@ -0,0 +1,291 @@
+// Implementation of a TPM driver for the TPM TIS interface
+//
+// Copyright (C) 2006-2011 IBM Corporation
+//
+// Authors:
+//     Stefan Berger <stefanb@linux.vnet.ibm.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+#include "config.h" // CONFIG_TPM_TIS_SHA1THRESHOLD
+#include "string.h" // memcpy
+#include "util.h" // msleep
+#include "x86.h" // readl
+#include "hw/tpm_drivers.h" // struct tpm_driver
+#include "tcgbios.h" // TCG_*
+
+static const u32 tis_default_timeouts[4] = {
+    TIS_DEFAULT_TIMEOUT_A,
+    TIS_DEFAULT_TIMEOUT_B,
+    TIS_DEFAULT_TIMEOUT_C,
+    TIS_DEFAULT_TIMEOUT_D,
+};
+
+static const u32 tpm_default_durations[3] = {
+    TPM_DEFAULT_DURATION_SHORT,
+    TPM_DEFAULT_DURATION_MEDIUM,
+    TPM_DEFAULT_DURATION_LONG,
+};
+
+/* determined values */
+static u32 tpm_default_dur[3];
+static u32 tpm_default_to[4];
+
+
+/* if device is not there, return '0', '1' otherwise */
+static u32 tis_probe(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u32 didvid = readl(TIS_REG(0, TIS_REG_DID_VID));
+
+    if ((didvid != 0) && (didvid != 0xffffffff))
+        rc = 1;
+
+    return rc;
+}
+
+static u32 tis_init(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 1;
+
+    writeb(TIS_REG(0, TIS_REG_INT_ENABLE), 0);
+
+    if (tpm_drivers[TIS_DRIVER_IDX].durations == NULL) {
+        u32 *durations = tpm_default_dur;
+        memcpy(durations, tpm_default_durations,
+               sizeof(tpm_default_durations));
+        tpm_drivers[TIS_DRIVER_IDX].durations = durations;
+    }
+
+    if (tpm_drivers[TIS_DRIVER_IDX].timeouts == NULL) {
+        u32 *timeouts = tpm_default_to;
+        memcpy(timeouts, tis_default_timeouts,
+               sizeof(tis_default_timeouts));
+        tpm_drivers[TIS_DRIVER_IDX].timeouts = timeouts;
+    }
+
+    return 1;
+}
+
+
+static void set_timeouts(u32 timeouts[4], u32 durations[3])
+{
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    u32 *tos = tpm_drivers[TIS_DRIVER_IDX].timeouts;
+    u32 *dus = tpm_drivers[TIS_DRIVER_IDX].durations;
+
+    if (tos && tos != tis_default_timeouts && timeouts)
+        memcpy(tos, timeouts, 4 * sizeof(u32));
+    if (dus && dus != tpm_default_durations && durations)
+        memcpy(dus, durations, 3 * sizeof(u32));
+}
+
+
+static u32 tis_wait_sts(u8 locty, u32 time, u8 mask, u8 expect)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 1;
+
+    while (time > 0) {
+        u8 sts = readb(TIS_REG(locty, TIS_REG_STS));
+        if ((sts & mask) == expect) {
+            rc = 0;
+            break;
+        }
+        msleep(1);
+        time--;
+    }
+    return rc;
+}
+
+static u32 tis_activate(u8 locty)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u8 acc;
+    int l;
+    u32 timeout_a = tpm_drivers[TIS_DRIVER_IDX].timeouts[TIS_TIMEOUT_TYPE_A];
+
+    if (!(readb(TIS_REG(locty, TIS_REG_ACCESS)) &
+          TIS_ACCESS_ACTIVE_LOCALITY)) {
+        /* release locality in use top-downwards */
+        for (l = 4; l >= 0; l--)
+            writeb(TIS_REG(l, TIS_REG_ACCESS),
+                   TIS_ACCESS_ACTIVE_LOCALITY);
+    }
+
+    /* request access to locality */
+    writeb(TIS_REG(locty, TIS_REG_ACCESS), TIS_ACCESS_REQUEST_USE);
+
+    acc = readb(TIS_REG(locty, TIS_REG_ACCESS));
+    if ((acc & TIS_ACCESS_ACTIVE_LOCALITY)) {
+        writeb(TIS_REG(locty, TIS_REG_STS), TIS_STS_COMMAND_READY);
+        rc = tis_wait_sts(locty, timeout_a,
+                          TIS_STS_COMMAND_READY, TIS_STS_COMMAND_READY);
+    }
+
+    return rc;
+}
+
+static u32 tis_find_active_locality(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u8 locty;
+
+    for (locty = 0; locty <= 4; locty++) {
+        if ((readb(TIS_REG(locty, TIS_REG_ACCESS)) &
+             TIS_ACCESS_ACTIVE_LOCALITY))
+            return locty;
+    }
+
+    tis_activate(0);
+
+    return 0;
+}
+
+static u32 tis_ready(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u8 locty = tis_find_active_locality();
+    u32 timeout_b = tpm_drivers[TIS_DRIVER_IDX].timeouts[TIS_TIMEOUT_TYPE_B];
+
+    writeb(TIS_REG(locty, TIS_REG_STS), TIS_STS_COMMAND_READY);
+    rc = tis_wait_sts(locty, timeout_b,
+                      TIS_STS_COMMAND_READY, TIS_STS_COMMAND_READY);
+
+    return rc;
+}
+
+static u32 tis_senddata(const u8 *const data, u32 len)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u32 offset = 0;
+    u32 end = 0;
+    u16 burst = 0;
+    u32 ctr = 0;
+    u8 locty = tis_find_active_locality();
+    u32 timeout_d = tpm_drivers[TIS_DRIVER_IDX].timeouts[TIS_TIMEOUT_TYPE_D];
+
+    do {
+        while (burst == 0 && ctr < timeout_d) {
+               burst = readl(TIS_REG(locty, TIS_REG_STS)) >> 8;
+            if (burst == 0) {
+                msleep(1);
+                ctr++;
+            }
+        }
+
+        if (burst == 0) {
+            rc = TCG_RESPONSE_TIMEOUT;
+            break;
+        }
+
+        while (1) {
+            writeb(TIS_REG(locty, TIS_REG_DATA_FIFO), data[offset++]);
+            burst--;
+
+            if (burst == 0 || offset == len)
+                break;
+        }
+
+        if (offset == len)
+            end = 1;
+    } while (end == 0);
+
+    return rc;
+}
+
+static u32 tis_readresp(u8 *buffer, u32 *len)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u32 offset = 0;
+    u32 sts;
+    u8 locty = tis_find_active_locality();
+
+    while (offset < *len) {
+        buffer[offset] = readb(TIS_REG(locty, TIS_REG_DATA_FIFO));
+        offset++;
+        sts = readb(TIS_REG(locty, TIS_REG_STS));
+        /* data left ? */
+        if ((sts & TIS_STS_DATA_AVAILABLE) == 0)
+            break;
+    }
+
+    *len = offset;
+
+    return rc;
+}
+
+
+static u32 tis_waitdatavalid(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u8 locty = tis_find_active_locality();
+    u32 timeout_c = tpm_drivers[TIS_DRIVER_IDX].timeouts[TIS_TIMEOUT_TYPE_C];
+
+    if (tis_wait_sts(locty, timeout_c, TIS_STS_VALID, TIS_STS_VALID) != 0)
+        rc = TCG_NO_RESPONSE;
+
+    return rc;
+}
+
+static u32 tis_waitrespready(enum tpmDurationType to_t)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    u32 rc = 0;
+    u8 locty = tis_find_active_locality();
+    u32 timeout = tpm_drivers[TIS_DRIVER_IDX].durations[to_t];
+
+    writeb(TIS_REG(locty ,TIS_REG_STS), TIS_STS_TPM_GO);
+
+    if (tis_wait_sts(locty, timeout,
+                     TIS_STS_DATA_AVAILABLE, TIS_STS_DATA_AVAILABLE) != 0)
+        rc = TCG_NO_RESPONSE;
+
+    return rc;
+}
+
+
+struct tpm_driver tpm_drivers[TPM_NUM_DRIVERS] = {
+    [TIS_DRIVER_IDX] =
+        {
+            .timeouts      = NULL,
+            .durations     = NULL,
+            .set_timeouts  = set_timeouts,
+            .probe         = tis_probe,
+            .init          = tis_init,
+            .activate      = tis_activate,
+            .ready         = tis_ready,
+            .senddata      = tis_senddata,
+            .readresp      = tis_readresp,
+            .waitdatavalid = tis_waitdatavalid,
+            .waitrespready = tis_waitrespready,
+            .sha1threshold = 100 * 1024,
+        },
+};
diff --git a/qemu/roms/seabios/src/hw/tpm_drivers.h b/qemu/roms/seabios/src/hw/tpm_drivers.h
new file mode 100644
index 000000000..34bb12d1c
--- /dev/null
+++ b/qemu/roms/seabios/src/hw/tpm_drivers.h
@@ -0,0 +1,90 @@
+#ifndef TPM_DRIVERS_H
+#define TPM_DRIVERS_H
+
+#include "types.h" // u32
+
+
+enum tpmDurationType {
+    TPM_DURATION_TYPE_SHORT = 0,
+    TPM_DURATION_TYPE_MEDIUM,
+    TPM_DURATION_TYPE_LONG,
+};
+
+/* low level driver implementation */
+struct tpm_driver {
+    u32 *timeouts;
+    u32 *durations;
+    void (*set_timeouts)(u32 timeouts[4], u32 durations[3]);
+    u32 (*probe)(void);
+    u32 (*init)(void);
+    u32 (*activate)(u8 locty);
+    u32 (*ready)(void);
+    u32 (*senddata)(const u8 *const data, u32 len);
+    u32 (*readresp)(u8 *buffer, u32 *len);
+    u32 (*waitdatavalid)(void);
+    u32 (*waitrespready)(enum tpmDurationType to_t);
+    /* the TPM will be used for buffers of sizes below the sha1threshold
+       for calculating the hash */
+    u32 sha1threshold;
+};
+
+extern struct tpm_driver tpm_drivers[];
+
+
+#define TIS_DRIVER_IDX       0
+#define TPM_NUM_DRIVERS      1
+
+#define TPM_INVALID_DRIVER  -1
+
+/* TIS driver */
+/* address of locality 0 (TIS) */
+#define TPM_TIS_BASE_ADDRESS        0xfed40000
+
+#define TIS_REG(LOCTY, REG) \
+    (void *)(TPM_TIS_BASE_ADDRESS + (LOCTY << 12) + REG)
+
+/* hardware registers */
+#define TIS_REG_ACCESS                 0x0
+#define TIS_REG_INT_ENABLE             0x8
+#define TIS_REG_INT_VECTOR             0xc
+#define TIS_REG_INT_STATUS             0x10
+#define TIS_REG_INTF_CAPABILITY        0x14
+#define TIS_REG_STS                    0x18
+#define TIS_REG_DATA_FIFO              0x24
+#define TIS_REG_DID_VID                0xf00
+#define TIS_REG_RID                    0xf04
+
+#define TIS_STS_VALID                  (1 << 7) /* 0x80 */
+#define TIS_STS_COMMAND_READY          (1 << 6) /* 0x40 */
+#define TIS_STS_TPM_GO                 (1 << 5) /* 0x20 */
+#define TIS_STS_DATA_AVAILABLE         (1 << 4) /* 0x10 */
+#define TIS_STS_EXPECT                 (1 << 3) /* 0x08 */
+#define TIS_STS_RESPONSE_RETRY         (1 << 1) /* 0x02 */
+
+#define TIS_ACCESS_TPM_REG_VALID_STS   (1 << 7) /* 0x80 */
+#define TIS_ACCESS_ACTIVE_LOCALITY     (1 << 5) /* 0x20 */
+#define TIS_ACCESS_BEEN_SEIZED         (1 << 4) /* 0x10 */
+#define TIS_ACCESS_SEIZE               (1 << 3) /* 0x08 */
+#define TIS_ACCESS_PENDING_REQUEST     (1 << 2) /* 0x04 */
+#define TIS_ACCESS_REQUEST_USE         (1 << 1) /* 0x02 */
+#define TIS_ACCESS_TPM_ESTABLISHMENT   (1 << 0) /* 0x01 */
+
+#define SCALER 10
+
+#define TIS_DEFAULT_TIMEOUT_A          (750  * SCALER)
+#define TIS_DEFAULT_TIMEOUT_B          (2000 * SCALER)
+#define TIS_DEFAULT_TIMEOUT_C          (750  * SCALER)
+#define TIS_DEFAULT_TIMEOUT_D          (750  * SCALER)
+
+enum tisTimeoutType {
+    TIS_TIMEOUT_TYPE_A = 0,
+    TIS_TIMEOUT_TYPE_B,
+    TIS_TIMEOUT_TYPE_C,
+    TIS_TIMEOUT_TYPE_D,
+};
+
+#define TPM_DEFAULT_DURATION_SHORT     (2000  * SCALER)
+#define TPM_DEFAULT_DURATION_MEDIUM    (20000 * SCALER)
+#define TPM_DEFAULT_DURATION_LONG      (60000 * SCALER)
+
+#endif /* TPM_DRIVERS_H */
diff --git a/qemu/roms/seabios/src/hw/usb-hid.h b/qemu/roms/seabios/src/hw/usb-hid.h
index ef34e7963..fd7b8f8be 100644
--- a/qemu/roms/seabios/src/hw/usb-hid.h
+++ b/qemu/roms/seabios/src/hw/usb-hid.h
@@ -4,10 +4,10 @@
 // usb-hid.c
 struct usbdevice_s;
 int usb_hid_setup(struct usbdevice_s *usbdev);
-inline int usb_kbd_active(void);
-inline int usb_kbd_command(int command, u8 *param);
-inline int usb_mouse_active(void);
-inline int usb_mouse_command(int command, u8 *param);
+int usb_kbd_active(void);
+int usb_kbd_command(int command, u8 *param);
+int usb_mouse_active(void);
+int usb_mouse_command(int command, u8 *param);
 void usb_check_event(void);
 
 
diff --git a/qemu/roms/seabios/src/hw/usb-msc.c b/qemu/roms/seabios/src/hw/usb-msc.c
index d90319f51..a234f13be 100644
--- a/qemu/roms/seabios/src/hw/usb-msc.c
+++ b/qemu/roms/seabios/src/hw/usb-msc.c
@@ -63,25 +63,27 @@ usb_msc_send(struct usbdrive_s *udrive_gf, int dir, void *buf, u32 bytes)
 
 // Low-level usb command transmit function.
 int
-usb_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+usb_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_USB_MSC)
         return 0;
 
-    dprintf(16, "usb_cmd_data id=%p write=%d count=%d bs=%d buf=%p\n"
-            , op->drive_gf, 0, op->count, blocksize, op->buf_fl);
+    dprintf(16, "usb_cmd_data id=%p write=%d count=%d buf=%p\n"
+            , op->drive_gf, 0, op->count, op->buf_fl);
     struct usbdrive_s *udrive_gf = container_of(
         op->drive_gf, struct usbdrive_s, drive);
 
     // Setup command block wrapper.
-    u32 bytes = blocksize * op->count;
     struct cbw_s cbw;
     memset(&cbw, 0, sizeof(cbw));
-    memcpy(cbw.CBWCB, cdbcmd, USB_CDB_SIZE);
+    int blocksize = scsi_fill_cmd(op, cbw.CBWCB, USB_CDB_SIZE);
+    if (blocksize < 0)
+        return default_process_op(op);
+    u32 bytes = blocksize * op->count;
     cbw.dCBWSignature = CBW_SIGNATURE;
     cbw.dCBWTag = 999; // XXX
     cbw.dCBWDataTransferLength = bytes;
-    cbw.bmCBWFlags = cdb_is_read(cdbcmd, blocksize) ? USB_DIR_IN : USB_DIR_OUT;
+    cbw.bmCBWFlags = scsi_is_read(op) ? USB_DIR_IN : USB_DIR_OUT;
     cbw.bCBWLUN = GET_GLOBALFLAT(udrive_gf->lun);
     cbw.bCBWCBLength = USB_CDB_SIZE;
 
diff --git a/qemu/roms/seabios/src/hw/usb-msc.h b/qemu/roms/seabios/src/hw/usb-msc.h
index c40d75556..ff3c38038 100644
--- a/qemu/roms/seabios/src/hw/usb-msc.h
+++ b/qemu/roms/seabios/src/hw/usb-msc.h
@@ -3,7 +3,7 @@
 
 // usb-msc.c
 struct disk_op_s;
-int usb_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int usb_process_op(struct disk_op_s *op);
 struct usbdevice_s;
 int usb_msc_setup(struct usbdevice_s *usbdev);
 
diff --git a/qemu/roms/seabios/src/hw/usb-uas.c b/qemu/roms/seabios/src/hw/usb-uas.c
index 6ef8d0912..10e38454a 100644
--- a/qemu/roms/seabios/src/hw/usb-uas.c
+++ b/qemu/roms/seabios/src/hw/usb-uas.c
@@ -91,7 +91,7 @@ struct uasdrive_s {
 };
 
 int
-uas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+uas_process_op(struct disk_op_s *op)
 {
     if (!CONFIG_USB_UAS)
         return DISK_RET_EBADTRACK;
@@ -104,7 +104,9 @@ uas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
     ui.hdr.id = UAS_UI_COMMAND;
     ui.hdr.tag = 0xdead;
     ui.command.lun[1] = GET_GLOBALFLAT(drive_gf->lun);
-    memcpy(ui.command.cdb, cdbcmd, sizeof(ui.command.cdb));
+    int blocksize = scsi_fill_cmd(op, ui.command.cdb, sizeof(ui.command.cdb));
+    if (blocksize < 0)
+        return default_process_op(op);
     int ret = usb_send_bulk(GET_GLOBALFLAT(drive_gf->command),
                             USB_DIR_OUT, MAKE_FLATPTR(GET_SEG(SS), &ui),
                             sizeof(ui.hdr) + sizeof(ui.command));
diff --git a/qemu/roms/seabios/src/hw/usb-uas.h b/qemu/roms/seabios/src/hw/usb-uas.h
index ad91c5f60..8b2f810e9 100644
--- a/qemu/roms/seabios/src/hw/usb-uas.h
+++ b/qemu/roms/seabios/src/hw/usb-uas.h
@@ -2,7 +2,7 @@
 #define __USB_UAS_H
 
 struct disk_op_s;
-int uas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int uas_process_op(struct disk_op_s *op);
 struct usbdevice_s;
 int usb_uas_setup(struct usbdevice_s *usbdev);
 
diff --git a/qemu/roms/seabios/src/hw/usb-xhci.c b/qemu/roms/seabios/src/hw/usb-xhci.c
index fd58334dc..654febaad 100644
--- a/qemu/roms/seabios/src/hw/usb-xhci.c
+++ b/qemu/roms/seabios/src/hw/usb-xhci.c
@@ -350,26 +350,41 @@ xhci_hub_reset(struct usbhub_s *hub, u32 port)
 {
     struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
     u32 portsc = readl(&xhci->pr[port].portsc);
-    int rc;
+    if (!(portsc & XHCI_PORTSC_CCS))
+        // Device no longer connected?!
+        return -1;
 
     switch (xhci_get_field(portsc, XHCI_PORTSC_PLS)) {
     case PLS_U0:
-        rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
+        // A USB3 port - controller automatically performs reset
         break;
     case PLS_POLLING:
+        // A USB2 port - perform device reset
         xhci_print_port_state(3, __func__, port, portsc);
-        portsc |= XHCI_PORTSC_PR;
-        writel(&xhci->pr[port].portsc, portsc);
-        if (wait_bit(&xhci->pr[port].portsc, XHCI_PORTSC_PED, XHCI_PORTSC_PED, 100) != 0)
-            return -1;
-        portsc = readl(&xhci->pr[port].portsc);
-        rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
+        writel(&xhci->pr[port].portsc, portsc | XHCI_PORTSC_PR);
         break;
     default:
-        rc = -1;
-        break;
+        return -1;
     }
 
+    // Wait for device to complete reset and be enabled
+    u32 end = timer_calc(100);
+    for (;;) {
+        portsc = readl(&xhci->pr[port].portsc);
+        if (!(portsc & XHCI_PORTSC_CCS))
+            // Device disconnected during reset
+            return -1;
+        if (portsc & XHCI_PORTSC_PED)
+            // Reset complete
+            break;
+        if (timer_check(end)) {
+            warn_timeout();
+            return -1;
+        }
+        yield();
+    }
+
+    int rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
     xhci_print_port_state(1, "XHCI", port, portsc);
     return rc;
 }
@@ -465,7 +480,7 @@ configure_xhci(void *data)
     xhci->evts->cs = 1;
 
     reg = readl(&xhci->caps->hcsparams2);
-    u32 spb = reg >> 27;
+    u32 spb = (reg >> 21 & 0x1f) << 5 | reg >> 27;
     if (spb) {
         dprintf(3, "%s: setup %d scratch pad buffers\n", __func__, spb);
         u64 *spba = memalign_high(64, sizeof(*spba) * spb);
@@ -921,8 +936,14 @@ xhci_alloc_pipe(struct usbdevice_s *usbdev
     usb_desc2pipe(&pipe->pipe, usbdev, epdesc);
     pipe->epid = epid;
     pipe->reqs.cs = 1;
-    if (eptype == USB_ENDPOINT_XFER_INT)
+    if (eptype == USB_ENDPOINT_XFER_INT) {
         pipe->buf = malloc_high(pipe->pipe.maxpacket);
+        if (!pipe->buf) {
+            warn_noalloc();
+            free(pipe);
+            return NULL;
+        }
+    }
 
     // Allocate input context and initialize endpoint info.
     struct xhci_inctx *in = xhci_alloc_inctx(usbdev, epid);
@@ -988,6 +1009,7 @@ xhci_alloc_pipe(struct usbdevice_s *usbdev
     return &pipe->pipe;
 
 fail:
+    free(pipe->buf);
     free(pipe);
     free(in);
     return NULL;
diff --git a/qemu/roms/seabios/src/hw/usb.c b/qemu/roms/seabios/src/hw/usb.c
index 1b4ea8bed..e46092c63 100644
--- a/qemu/roms/seabios/src/hw/usb.c
+++ b/qemu/roms/seabios/src/hw/usb.c
@@ -79,9 +79,8 @@ usb_poll_intr(struct usb_pipe *pipe_fl, void *data)
     case USB_TYPE_EHCI:
         return ehci_poll_intr(pipe_fl, data);
     case USB_TYPE_XHCI: ;
-        extern void _cfunc32flat_xhci_poll_intr(void);
-        return call32_params(_cfunc32flat_xhci_poll_intr, (u32)pipe_fl
-                             , (u32)MAKE_FLATPTR(GET_SEG(SS), (u32)data), 0, -1);
+        return call32_params(xhci_poll_intr, pipe_fl
+                             , MAKE_FLATPTR(GET_SEG(SS), data), 0, -1);
     }
 }
 
@@ -249,8 +248,10 @@ get_device_config(struct usb_pipe *pipe)
         return NULL;
 
     void *config = malloc_tmphigh(cfg.wTotalLength);
-    if (!config)
+    if (!config) {
+        warn_noalloc();
         return NULL;
+    }
     req.wLength = cfg.wTotalLength;
     ret = usb_send_default_control(pipe, &req, config);
     if (ret) {
diff --git a/qemu/roms/seabios/src/hw/virtio-blk.c b/qemu/roms/seabios/src/hw/virtio-blk.c
index e2dbd3c94..20a79ebba 100644
--- a/qemu/roms/seabios/src/hw/virtio-blk.c
+++ b/qemu/roms/seabios/src/hw/virtio-blk.c
@@ -25,7 +25,7 @@
 struct virtiodrive_s {
     struct drive_s drive;
     struct vring_virtqueue *vq;
-    u16 ioaddr;
+    struct vp_device vp;
 };
 
 static int
@@ -33,7 +33,7 @@ virtio_blk_op(struct disk_op_s *op, int write)
 {
     struct virtiodrive_s *vdrive_gf =
         container_of(op->drive_gf, struct virtiodrive_s, drive);
-    struct vring_virtqueue *vq = GET_GLOBALFLAT(vdrive_gf->vq);
+    struct vring_virtqueue *vq = vdrive_gf->vq;
     struct virtio_blk_outhdr hdr = {
         .type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN,
         .ioprio = 0,
@@ -42,15 +42,15 @@ virtio_blk_op(struct disk_op_s *op, int write)
     u8 status = VIRTIO_BLK_S_UNSUPP;
     struct vring_list sg[] = {
         {
-            .addr       = MAKE_FLATPTR(GET_SEG(SS), &hdr),
+            .addr       = (void*)(&hdr),
             .length     = sizeof(hdr),
         },
         {
             .addr       = op->buf_fl,
-            .length     = GET_GLOBALFLAT(vdrive_gf->drive.blksize) * op->count,
+            .length     = vdrive_gf->drive.blksize * op->count,
         },
         {
-            .addr       = MAKE_FLATPTR(GET_SEG(SS), &status),
+            .addr       = (void*)(&status),
             .length     = sizeof(status),
         },
     };
@@ -60,7 +60,7 @@ virtio_blk_op(struct disk_op_s *op, int write)
         vring_add_buf(vq, sg, 2, 1, 0, 0);
     else
         vring_add_buf(vq, sg, 1, 2, 0, 0);
-    vring_kick(GET_GLOBALFLAT(vdrive_gf->ioaddr), vq, 1);
+    vring_kick(&vdrive_gf->vp, vq, 1);
 
     /* Wait for reply */
     while (!vring_more_used(vq))
@@ -72,13 +72,13 @@ virtio_blk_op(struct disk_op_s *op, int write)
     /* Clear interrupt status register.  Avoid leaving interrupts stuck if
      * VRING_AVAIL_F_NO_INTERRUPT was ignored and interrupts were raised.
      */
-    vp_get_isr(GET_GLOBALFLAT(vdrive_gf->ioaddr));
+    vp_get_isr(&vdrive_gf->vp);
 
     return status == VIRTIO_BLK_S_OK ? DISK_RET_SUCCESS : DISK_RET_EBADTRACK;
 }
 
 int
-process_virtio_blk_op(struct disk_op_s *op)
+virtio_blk_process_op(struct disk_op_s *op)
 {
     if (! CONFIG_VIRTIO_BLK)
         return 0;
@@ -87,14 +87,8 @@ process_virtio_blk_op(struct disk_op_s *op)
         return virtio_blk_op(op, 0);
     case CMD_WRITE:
         return virtio_blk_op(op, 1);
-    case CMD_FORMAT:
-    case CMD_RESET:
-    case CMD_ISREADY:
-    case CMD_VERIFY:
-    case CMD_SEEK:
-        return DISK_RET_SUCCESS;
     default:
-        return DISK_RET_EPARAM;
+        return default_process_op(op);
     }
 }
 
@@ -102,6 +96,7 @@ static void
 init_virtio_blk(struct pci_device *pci)
 {
     u16 bdf = pci->bdf;
+    u8 status = VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER;
     dprintf(1, "found virtio-blk at %x:%x\n", pci_bdf_to_bus(bdf),
             pci_bdf_to_dev(bdf));
     struct virtiodrive_s *vdrive = malloc_fseg(sizeof(*vdrive));
@@ -113,47 +108,93 @@ init_virtio_blk(struct pci_device *pci)
     vdrive->drive.type = DTYPE_VIRTIO_BLK;
     vdrive->drive.cntl_id = bdf;
 
-    u16 ioaddr = vp_init_simple(bdf);
-    vdrive->ioaddr = ioaddr;
-    if (vp_find_vq(ioaddr, 0, &vdrive->vq) < 0 ) {
+    vp_init_simple(&vdrive->vp, pci);
+    if (vp_find_vq(&vdrive->vp, 0, &vdrive->vq) < 0 ) {
         dprintf(1, "fail to find vq for virtio-blk %x:%x\n",
                 pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
         goto fail;
     }
 
-    struct virtio_blk_config cfg;
-    vp_get(ioaddr, 0, &cfg, sizeof(cfg));
-
-    u32 f = vp_get_features(ioaddr);
-    vdrive->drive.blksize = (f & (1 << VIRTIO_BLK_F_BLK_SIZE)) ?
-        cfg.blk_size : DISK_SECTOR_SIZE;
-
-    vdrive->drive.sectors = cfg.capacity;
-    dprintf(3, "virtio-blk %x:%x blksize=%d sectors=%u\n",
-            pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
-            vdrive->drive.blksize, (u32)vdrive->drive.sectors);
-
-    if (vdrive->drive.blksize != DISK_SECTOR_SIZE) {
-        dprintf(1, "virtio-blk %x:%x block size %d is unsupported\n",
+    if (vdrive->vp.use_modern) {
+        struct vp_device *vp = &vdrive->vp;
+        u64 features = vp_get_features(vp);
+        u64 version1 = 1ull << VIRTIO_F_VERSION_1;
+        u64 blk_size = 1ull << VIRTIO_BLK_F_BLK_SIZE;
+        if (!(features & version1)) {
+            dprintf(1, "modern device without virtio_1 feature bit: %x:%x\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
+            goto fail;
+        }
+
+        features = features & (version1 | blk_size);
+        vp_set_features(vp, features);
+        status |= VIRTIO_CONFIG_S_FEATURES_OK;
+        vp_set_status(vp, status);
+        if (!(vp_get_status(vp) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+            dprintf(1, "device didn't accept features: %x:%x\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
+            goto fail;
+        }
+
+        vdrive->drive.sectors =
+            vp_read(&vp->device, struct virtio_blk_config, capacity);
+        if (features & blk_size) {
+            vdrive->drive.blksize =
+                vp_read(&vp->device, struct virtio_blk_config, blk_size);
+        } else {
+            vdrive->drive.blksize = DISK_SECTOR_SIZE;
+        }
+        if (vdrive->drive.blksize != DISK_SECTOR_SIZE) {
+            dprintf(1, "virtio-blk %x:%x block size %d is unsupported\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+                    vdrive->drive.blksize);
+            goto fail;
+        }
+        dprintf(3, "virtio-blk %x:%x blksize=%d sectors=%u\n",
                 pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
-                vdrive->drive.blksize);
-        goto fail;
+                vdrive->drive.blksize, (u32)vdrive->drive.sectors);
+
+        vdrive->drive.pchs.cylinder =
+            vp_read(&vp->device, struct virtio_blk_config, cylinders);
+        vdrive->drive.pchs.head =
+            vp_read(&vp->device, struct virtio_blk_config, heads);
+        vdrive->drive.pchs.sector =
+            vp_read(&vp->device, struct virtio_blk_config, sectors);
+    } else {
+        struct virtio_blk_config cfg;
+        vp_get_legacy(&vdrive->vp, 0, &cfg, sizeof(cfg));
+
+        u64 f = vp_get_features(&vdrive->vp);
+        vdrive->drive.blksize = (f & (1 << VIRTIO_BLK_F_BLK_SIZE)) ?
+            cfg.blk_size : DISK_SECTOR_SIZE;
+
+        vdrive->drive.sectors = cfg.capacity;
+        dprintf(3, "virtio-blk %x:%x blksize=%d sectors=%u\n",
+                pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+                vdrive->drive.blksize, (u32)vdrive->drive.sectors);
+
+        if (vdrive->drive.blksize != DISK_SECTOR_SIZE) {
+            dprintf(1, "virtio-blk %x:%x block size %d is unsupported\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+                    vdrive->drive.blksize);
+            goto fail;
+        }
+        vdrive->drive.pchs.cylinder = cfg.cylinders;
+        vdrive->drive.pchs.head = cfg.heads;
+        vdrive->drive.pchs.sector = cfg.sectors;
     }
 
-    vdrive->drive.pchs.cylinder = cfg.cylinders;
-    vdrive->drive.pchs.head = cfg.heads;
-    vdrive->drive.pchs.sector = cfg.sectors;
     char *desc = znprintf(MAXDESCSIZE, "Virtio disk PCI:%x:%x",
                           pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
 
     boot_add_hd(&vdrive->drive, desc, bootprio_find_pci_device(pci));
 
-    vp_set_status(ioaddr, VIRTIO_CONFIG_S_ACKNOWLEDGE |
-                  VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK);
+    status |= VIRTIO_CONFIG_S_DRIVER_OK;
+    vp_set_status(&vdrive->vp, status);
     return;
 
 fail:
-    vp_reset(ioaddr);
+    vp_reset(&vdrive->vp);
     free(vdrive->vq);
     free(vdrive);
 }
@@ -169,8 +210,9 @@ virtio_blk_setup(void)
 
     struct pci_device *pci;
     foreachpci(pci) {
-        if (pci->vendor != PCI_VENDOR_ID_REDHAT_QUMRANET
-            || pci->device != PCI_DEVICE_ID_VIRTIO_BLK)
+        if (pci->vendor != PCI_VENDOR_ID_REDHAT_QUMRANET ||
+            (pci->device != PCI_DEVICE_ID_VIRTIO_BLK_09 &&
+             pci->device != PCI_DEVICE_ID_VIRTIO_BLK_10))
             continue;
         init_virtio_blk(pci);
     }
diff --git a/qemu/roms/seabios/src/hw/virtio-blk.h b/qemu/roms/seabios/src/hw/virtio-blk.h
index b233c744b..157bed627 100644
--- a/qemu/roms/seabios/src/hw/virtio-blk.h
+++ b/qemu/roms/seabios/src/hw/virtio-blk.h
@@ -37,7 +37,7 @@ struct virtio_blk_outhdr {
 #define VIRTIO_BLK_S_UNSUPP     2
 
 struct disk_op_s;
-int process_virtio_blk_op(struct disk_op_s *op);
+int virtio_blk_process_op(struct disk_op_s *op);
 void virtio_blk_setup(void);
 
 #endif /* _VIRTIO_BLK_H */
diff --git a/qemu/roms/seabios/src/hw/virtio-pci.c b/qemu/roms/seabios/src/hw/virtio-pci.c
index b9b3ab1e3..6df519489 100644
--- a/qemu/roms/seabios/src/hw/virtio-pci.c
+++ b/qemu/roms/seabios/src/hw/virtio-pci.c
@@ -24,47 +24,153 @@
 #include "virtio-pci.h"
 #include "virtio-ring.h"
 
-int vp_find_vq(unsigned int ioaddr, int queue_index,
+u64 vp_get_features(struct vp_device *vp)
+{
+    u32 f0, f1;
+
+    if (vp->use_modern) {
+        vp_write(&vp->common, virtio_pci_common_cfg, device_feature_select, 0);
+        f0 = vp_read(&vp->common, virtio_pci_common_cfg, device_feature);
+        vp_write(&vp->common, virtio_pci_common_cfg, device_feature_select, 1);
+        f1 = vp_read(&vp->common, virtio_pci_common_cfg, device_feature);
+    } else {
+        f0 = vp_read(&vp->legacy, virtio_pci_legacy, host_features);
+        f1 = 0;
+    }
+    return ((u64)f1 << 32) | f0;
+}
+
+void vp_set_features(struct vp_device *vp, u64 features)
+{
+    u32 f0, f1;
+
+    f0 = features;
+    f1 = features >> 32;
+
+    if (vp->use_modern) {
+        vp_write(&vp->common, virtio_pci_common_cfg, guest_feature_select, 0);
+        vp_write(&vp->common, virtio_pci_common_cfg, guest_feature, f0);
+        vp_write(&vp->common, virtio_pci_common_cfg, guest_feature_select, 1);
+        vp_write(&vp->common, virtio_pci_common_cfg, guest_feature, f1);
+    } else {
+        vp_write(&vp->legacy, virtio_pci_legacy, guest_features, f0);
+    }
+}
+
+u8 vp_get_status(struct vp_device *vp)
+{
+    if (vp->use_modern) {
+        return vp_read(&vp->common, virtio_pci_common_cfg, device_status);
+    } else {
+        return vp_read(&vp->legacy, virtio_pci_legacy, status);
+    }
+}
+
+void vp_set_status(struct vp_device *vp, u8 status)
+{
+    if (status == 0)        /* reset */
+        return;
+    if (vp->use_modern) {
+        vp_write(&vp->common, virtio_pci_common_cfg, device_status, status);
+    } else {
+        vp_write(&vp->legacy, virtio_pci_legacy, status, status);
+    }
+}
+
+u8 vp_get_isr(struct vp_device *vp)
+{
+    if (vp->use_modern) {
+        return vp_read(&vp->isr, virtio_pci_isr, isr);
+    } else {
+        return vp_read(&vp->legacy, virtio_pci_legacy, isr);
+    }
+}
+
+void vp_reset(struct vp_device *vp)
+{
+    if (vp->use_modern) {
+        vp_write(&vp->common, virtio_pci_common_cfg, device_status, 0);
+        vp_read(&vp->isr, virtio_pci_isr, isr);
+    } else {
+        vp_write(&vp->legacy, virtio_pci_legacy, status, 0);
+        vp_read(&vp->legacy, virtio_pci_legacy, isr);
+    }
+}
+
+void vp_notify(struct vp_device *vp, struct vring_virtqueue *vq)
+{
+    if (vp->use_modern) {
+        u32 addr = vp->notify.addr +
+            vq->queue_notify_off *
+            vp->notify_off_multiplier;
+        if (vp->notify.is_io) {
+            outw(vq->queue_index, addr);
+        } else {
+            writew((void*)addr, vq->queue_index);
+        }
+        dprintf(9, "vp notify %x (%d) -- 0x%x\n",
+                addr, 2, vq->queue_index);
+    } else {
+        vp_write(&vp->legacy, virtio_pci_legacy, queue_notify, vq->queue_index);
+    }
+}
+
+int vp_find_vq(struct vp_device *vp, int queue_index,
                struct vring_virtqueue **p_vq)
 {
    u16 num;
 
    ASSERT32FLAT();
-   struct vring_virtqueue *vq = *p_vq = memalign_low(PAGE_SIZE, sizeof(*vq));
+   struct vring_virtqueue *vq = *p_vq = memalign_high(PAGE_SIZE, sizeof(*vq));
    if (!vq) {
        warn_noalloc();
        goto fail;
    }
    memset(vq, 0, sizeof(*vq));
 
-   /* select the queue */
 
-   outw(queue_index, ioaddr + VIRTIO_PCI_QUEUE_SEL);
+   /* select the queue */
+   if (vp->use_modern) {
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_select, queue_index);
+   } else {
+       vp_write(&vp->legacy, virtio_pci_legacy, queue_sel, queue_index);
+   }
 
    /* check if the queue is available */
-
-   num = inw(ioaddr + VIRTIO_PCI_QUEUE_NUM);
+   if (vp->use_modern) {
+       num = vp_read(&vp->common, virtio_pci_common_cfg, queue_size);
+       if (num > MAX_QUEUE_NUM) {
+           vp_write(&vp->common, virtio_pci_common_cfg, queue_size,
+                    MAX_QUEUE_NUM);
+           num = vp_read(&vp->common, virtio_pci_common_cfg, queue_size);
+       }
+   } else {
+       num = vp_read(&vp->legacy, virtio_pci_legacy, queue_num);
+   }
    if (!num) {
        dprintf(1, "ERROR: queue size is 0\n");
        goto fail;
    }
-
    if (num > MAX_QUEUE_NUM) {
        dprintf(1, "ERROR: queue size %d > %d\n", num, MAX_QUEUE_NUM);
        goto fail;
    }
 
    /* check if the queue is already active */
-
-   if (inl(ioaddr + VIRTIO_PCI_QUEUE_PFN)) {
-       dprintf(1, "ERROR: queue already active\n");
-       goto fail;
+   if (vp->use_modern) {
+       if (vp_read(&vp->common, virtio_pci_common_cfg, queue_enable)) {
+           dprintf(1, "ERROR: queue already active\n");
+           goto fail;
+       }
+   } else {
+       if (vp_read(&vp->legacy, virtio_pci_legacy, queue_pfn)) {
+           dprintf(1, "ERROR: queue already active\n");
+           goto fail;
+       }
    }
-
    vq->queue_index = queue_index;
 
    /* initialize the queue */
-
    struct vring * vr = &vq->vring;
    vring_init(vr, num, (unsigned char*)&vq->queue);
 
@@ -73,9 +179,23 @@ int vp_find_vq(unsigned int ioaddr, int queue_index,
     * NOTE: vr->desc is initialized by vring_init()
     */
 
-   outl((unsigned long)virt_to_phys(vr->desc) >> PAGE_SHIFT,
-        ioaddr + VIRTIO_PCI_QUEUE_PFN);
-
+   if (vp->use_modern) {
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_desc_lo,
+                (unsigned long)virt_to_phys(vr->desc));
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_desc_hi, 0);
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_avail_lo,
+                (unsigned long)virt_to_phys(vr->avail));
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_avail_hi, 0);
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_used_lo,
+                (unsigned long)virt_to_phys(vr->used));
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_used_hi, 0);
+       vp_write(&vp->common, virtio_pci_common_cfg, queue_enable, 1);
+       vq->queue_notify_off = vp_read(&vp->common, virtio_pci_common_cfg,
+                                      queue_notify_off);
+   } else {
+       vp_write(&vp->legacy, virtio_pci_legacy, queue_pfn,
+                (unsigned long)virt_to_phys(vr->desc) >> PAGE_SHIFT);
+   }
    return num;
 
 fail:
@@ -84,14 +204,76 @@ fail:
    return -1;
 }
 
-u16 vp_init_simple(u16 bdf)
+void vp_init_simple(struct vp_device *vp, struct pci_device *pci)
 {
-    u16 ioaddr = pci_config_readl(bdf, PCI_BASE_ADDRESS_0) &
-        PCI_BASE_ADDRESS_IO_MASK;
+    u8 cap = pci_find_capability(pci, PCI_CAP_ID_VNDR, 0);
+    struct vp_cap *vp_cap;
+    u32 addr, offset, mul;
+    u8 type;
+
+    memset(vp, 0, sizeof(*vp));
+    while (cap != 0) {
+        type = pci_config_readb(pci->bdf, cap +
+                                offsetof(struct virtio_pci_cap, cfg_type));
+        switch (type) {
+        case VIRTIO_PCI_CAP_COMMON_CFG:
+            vp_cap = &vp->common;
+            break;
+        case VIRTIO_PCI_CAP_NOTIFY_CFG:
+            vp_cap = &vp->notify;
+            mul = offsetof(struct virtio_pci_notify_cap, notify_off_multiplier);
+            vp->notify_off_multiplier = pci_config_readl(pci->bdf, cap + mul);
+            break;
+        case VIRTIO_PCI_CAP_ISR_CFG:
+            vp_cap = &vp->isr;
+            break;
+        case VIRTIO_PCI_CAP_DEVICE_CFG:
+            vp_cap = &vp->device;
+            break;
+        default:
+            vp_cap = NULL;
+            break;
+        }
+        if (vp_cap && !vp_cap->cap) {
+            vp_cap->cap = cap;
+            vp_cap->bar = pci_config_readb(pci->bdf, cap +
+                                           offsetof(struct virtio_pci_cap, bar));
+            offset = pci_config_readl(pci->bdf, cap +
+                                      offsetof(struct virtio_pci_cap, offset));
+            addr = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0 + 4 * vp_cap->bar);
+            if (addr & PCI_BASE_ADDRESS_SPACE_IO) {
+                vp_cap->is_io = 1;
+                addr &= PCI_BASE_ADDRESS_IO_MASK;
+            } else {
+                vp_cap->is_io = 0;
+                addr &= PCI_BASE_ADDRESS_MEM_MASK;
+            }
+            vp_cap->addr = addr + offset;
+            dprintf(3, "pci dev %x:%x virtio cap at 0x%x type %d "
+                    "bar %d at 0x%08x off +0x%04x [%s]\n",
+                    pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf),
+                    vp_cap->cap, type, vp_cap->bar, addr, offset,
+                    vp_cap->is_io ? "io" : "mmio");
+        }
+
+        cap = pci_find_capability(pci, PCI_CAP_ID_VNDR, cap);
+    }
+
+    if (vp->common.cap && vp->notify.cap && vp->isr.cap && vp->device.cap) {
+        dprintf(1, "pci dev %x:%x using modern (1.0) virtio mode\n",
+                pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf));
+        vp->use_modern = 1;
+    } else {
+        dprintf(1, "pci dev %x:%x using legacy (0.9.5) virtio mode\n",
+                pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf));
+        vp->legacy.bar = 0;
+        vp->legacy.addr = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0) &
+            PCI_BASE_ADDRESS_IO_MASK;
+        vp->legacy.is_io = 1;
+    }
 
-    vp_reset(ioaddr);
-    pci_config_maskw(bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
-    vp_set_status(ioaddr, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+    vp_reset(vp);
+    pci_config_maskw(pci->bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
+    vp_set_status(vp, VIRTIO_CONFIG_S_ACKNOWLEDGE |
                   VIRTIO_CONFIG_S_DRIVER );
-    return ioaddr;
 }
diff --git a/qemu/roms/seabios/src/hw/virtio-pci.h b/qemu/roms/seabios/src/hw/virtio-pci.h
index bc04b039e..b11c3555e 100644
--- a/qemu/roms/seabios/src/hw/virtio-pci.h
+++ b/qemu/roms/seabios/src/hw/virtio-pci.h
@@ -2,104 +2,210 @@
 #define _VIRTIO_PCI_H
 
 #include "x86.h" // inl
-
-/* A 32-bit r/o bitmask of the features supported by the host */
-#define VIRTIO_PCI_HOST_FEATURES        0
-
-/* A 32-bit r/w bitmask of features activated by the guest */
-#define VIRTIO_PCI_GUEST_FEATURES       4
-
-/* A 32-bit r/w PFN for the currently selected queue */
-#define VIRTIO_PCI_QUEUE_PFN            8
-
-/* A 16-bit r/o queue size for the currently selected queue */
-#define VIRTIO_PCI_QUEUE_NUM            12
-
-/* A 16-bit r/w queue selector */
-#define VIRTIO_PCI_QUEUE_SEL            14
-
-/* A 16-bit r/w queue notifier */
-#define VIRTIO_PCI_QUEUE_NOTIFY         16
-
-/* An 8-bit device status register.  */
-#define VIRTIO_PCI_STATUS               18
-
-/* An 8-bit r/o interrupt status register.  Reading the value will return the
- * current contents of the ISR and will also clear it.  This is effectively
- * a read-and-acknowledge. */
-#define VIRTIO_PCI_ISR                  19
+#include "biosvar.h" // GET_LOWFLAT
 
 /* The bit of the ISR which indicates a device configuration change. */
 #define VIRTIO_PCI_ISR_CONFIG           0x2
 
-/* The remaining space is defined by each driver as the per-driver
- * configuration space */
-#define VIRTIO_PCI_CONFIG               20
-
 /* Virtio ABI version, this must match exactly */
 #define VIRTIO_PCI_ABI_VERSION          0
 
-static inline u32 vp_get_features(unsigned int ioaddr)
-{
-   return inl(ioaddr + VIRTIO_PCI_HOST_FEATURES);
-}
-
-static inline void vp_set_features(unsigned int ioaddr, u32 features)
+/* --- virtio 0.9.5 (legacy) struct --------------------------------- */
+
+typedef struct virtio_pci_legacy {
+    u32 host_features;
+    u32 guest_features;
+    u32 queue_pfn;
+    u16 queue_num;
+    u16 queue_sel;
+    u16 queue_notify;
+    u8  status;
+    u8  isr;
+    u8  device[];
+} virtio_pci_legacy;
+
+/* --- virtio 1.0 (modern) structs ---------------------------------- */
+
+/* Common configuration */
+#define VIRTIO_PCI_CAP_COMMON_CFG       1
+/* Notifications */
+#define VIRTIO_PCI_CAP_NOTIFY_CFG       2
+/* ISR access */
+#define VIRTIO_PCI_CAP_ISR_CFG          3
+/* Device specific configuration */
+#define VIRTIO_PCI_CAP_DEVICE_CFG       4
+/* PCI configuration access */
+#define VIRTIO_PCI_CAP_PCI_CFG          5
+
+/* This is the PCI capability header: */
+struct virtio_pci_cap {
+    u8 cap_vndr;          /* Generic PCI field: PCI_CAP_ID_VNDR */
+    u8 cap_next;          /* Generic PCI field: next ptr. */
+    u8 cap_len;           /* Generic PCI field: capability length */
+    u8 cfg_type;          /* Identifies the structure. */
+    u8 bar;               /* Where to find it. */
+    u8 padding[3];        /* Pad to full dword. */
+    u32 offset;           /* Offset within bar. */
+    u32 length;           /* Length of the structure, in bytes. */
+};
+
+struct virtio_pci_notify_cap {
+    struct virtio_pci_cap cap;
+    u32 notify_off_multiplier;   /* Multiplier for queue_notify_off. */
+};
+
+typedef struct virtio_pci_common_cfg {
+    /* About the whole device. */
+    u32 device_feature_select;   /* read-write */
+    u32 device_feature;          /* read-only */
+    u32 guest_feature_select;    /* read-write */
+    u32 guest_feature;           /* read-write */
+    u16 msix_config;             /* read-write */
+    u16 num_queues;              /* read-only */
+    u8 device_status;            /* read-write */
+    u8 config_generation;        /* read-only */
+
+    /* About a specific virtqueue. */
+    u16 queue_select;            /* read-write */
+    u16 queue_size;              /* read-write, power of 2. */
+    u16 queue_msix_vector;       /* read-write */
+    u16 queue_enable;            /* read-write */
+    u16 queue_notify_off;        /* read-only */
+    u32 queue_desc_lo;           /* read-write */
+    u32 queue_desc_hi;           /* read-write */
+    u32 queue_avail_lo;          /* read-write */
+    u32 queue_avail_hi;          /* read-write */
+    u32 queue_used_lo;           /* read-write */
+    u32 queue_used_hi;           /* read-write */
+} virtio_pci_common_cfg;
+
+typedef struct virtio_pci_isr {
+    u8 isr;
+} virtio_pci_isr;
+
+/* --- driver structs ----------------------------------------------- */
+
+struct vp_cap {
+    u32 addr;
+    u8 cap;
+    u8 bar;
+    u8 is_io;
+};
+
+struct vp_device {
+    struct vp_cap common, notify, isr, device, legacy;
+    u32 notify_off_multiplier;
+    u8 use_modern;
+};
+
+static inline u64 _vp_read(struct vp_cap *cap, u32 offset, u8 size)
 {
-        outl(features, ioaddr + VIRTIO_PCI_GUEST_FEATURES);
+    u32 addr = cap->addr + offset;
+    u64 var;
+
+    if (cap->is_io) {
+        switch (size) {
+        case 8:
+            var = inl(addr);
+            var |= (u64)inl(addr+4) << 32;
+            break;
+        case 4:
+            var = inl(addr);
+            break;
+        case 2:
+            var = inw(addr);
+            break;
+        case 1:
+            var = inb(addr);
+            break;
+        default:
+            var = 0;
+        }
+    } else {
+        switch (size) {
+        case 8:
+            var = readl((void*)addr);
+            var |= (u64)readl((void*)(addr+4)) << 32;
+            break;
+        case 4:
+            var = readl((void*)addr);
+            break;
+        case 2:
+            var = readw((void*)addr);
+            break;
+        case 1:
+            var = readb((void*)addr);
+            break;
+        default:
+            var = 0;
+        }
+    }
+    dprintf(9, "vp read   %x (%d) -> 0x%llx\n", addr, size, var);
+    return var;
 }
 
-static inline void vp_get(unsigned int ioaddr, unsigned offset,
-                     void *buf, unsigned len)
+static inline void _vp_write(struct vp_cap *cap, u32 offset, u8 size, u64 var)
 {
-   u8 *ptr = buf;
-   unsigned i;
-
-   for (i = 0; i < len; i++)
-           ptr[i] = inb(ioaddr + VIRTIO_PCI_CONFIG + offset + i);
+    u32 addr = cap->addr + offset;
+
+    dprintf(9, "vp write  %x (%d) <- 0x%llx\n", addr, size, var);
+    if (cap->is_io) {
+        switch (size) {
+        case 4:
+            outl(var, addr);
+            break;
+        case 2:
+            outw(var, addr);
+            break;
+        case 1:
+            outb(var, addr);
+            break;
+        }
+    } else {
+        switch (size) {
+        case 4:
+            writel((void*)addr, var);
+            break;
+        case 2:
+            writew((void*)addr, var);
+            break;
+        case 1:
+            writeb((void*)addr, var);
+            break;
+        }
+    }
 }
 
-static inline u8 vp_get_status(unsigned int ioaddr)
-{
-   return inb(ioaddr + VIRTIO_PCI_STATUS);
-}
+#define vp_read(_cap, _struct, _field)        \
+    _vp_read(_cap, offsetof(_struct, _field), \
+             sizeof(((_struct *)0)->_field))
 
-static inline void vp_set_status(unsigned int ioaddr, u8 status)
-{
-   if (status == 0)        /* reset */
-           return;
-   outb(status, ioaddr + VIRTIO_PCI_STATUS);
-}
+#define vp_write(_cap, _struct, _field, _var)           \
+    _vp_write(_cap, offsetof(_struct, _field),          \
+             sizeof(((_struct *)0)->_field), _var)
 
-static inline u8 vp_get_isr(unsigned int ioaddr)
-{
-   return inb(ioaddr + VIRTIO_PCI_ISR);
-}
+u64 vp_get_features(struct vp_device *vp);
+void vp_set_features(struct vp_device *vp, u64 features);
 
-static inline void vp_reset(unsigned int ioaddr)
+static inline void vp_get_legacy(struct vp_device *vp, unsigned offset,
+                                 void *buf, unsigned len)
 {
-   outb(0, ioaddr + VIRTIO_PCI_STATUS);
-   (void)inb(ioaddr + VIRTIO_PCI_ISR);
-}
+    u8 *ptr = buf;
+    unsigned i;
 
-static inline void vp_notify(unsigned int ioaddr, int queue_index)
-{
-   outw(queue_index, ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
+    for (i = 0; i < len; i++)
+        ptr[i] = vp_read(&vp->legacy, virtio_pci_legacy, device[i]);
 }
 
-static inline void vp_del_vq(unsigned int ioaddr, int queue_index)
-{
-   /* select the queue */
-
-   outw(queue_index, ioaddr + VIRTIO_PCI_QUEUE_SEL);
-
-   /* deactivate the queue */
-
-   outl(0, ioaddr + VIRTIO_PCI_QUEUE_PFN);
-}
+u8 vp_get_status(struct vp_device *vp);
+void vp_set_status(struct vp_device *vp, u8 status);
+u8 vp_get_isr(struct vp_device *vp);
+void vp_reset(struct vp_device *vp);
 
+struct pci_device;
 struct vring_virtqueue;
-u16 vp_init_simple(u16 bdf);
-int vp_find_vq(unsigned int ioaddr, int queue_index,
+void vp_init_simple(struct vp_device *vp, struct pci_device *pci);
+void vp_notify(struct vp_device *vp, struct vring_virtqueue *vq);
+int vp_find_vq(struct vp_device *vp, int queue_index,
                struct vring_virtqueue **p_vq);
 #endif /* _VIRTIO_PCI_H_ */
diff --git a/qemu/roms/seabios/src/hw/virtio-ring.c b/qemu/roms/seabios/src/hw/virtio-ring.c
index 97e0b3487..7205a0acd 100644
--- a/qemu/roms/seabios/src/hw/virtio-ring.c
+++ b/qemu/roms/seabios/src/hw/virtio-ring.c
@@ -35,8 +35,8 @@
 
 int vring_more_used(struct vring_virtqueue *vq)
 {
-    struct vring_used *used = GET_LOWFLAT(vq->vring.used);
-    int more = GET_LOWFLAT(vq->last_used_idx) != GET_LOWFLAT(used->idx);
+    struct vring_used *used = vq->vring.used;
+    int more = vq->last_used_idx != used->idx;
     /* Make sure ring reads are done after idx read above. */
     smp_rmb();
     return more;
@@ -57,13 +57,13 @@ void vring_detach(struct vring_virtqueue *vq, unsigned int head)
     /* find end of given descriptor */
 
     i = head;
-    while (GET_LOWFLAT(desc[i].flags) & VRING_DESC_F_NEXT)
-        i = GET_LOWFLAT(desc[i].next);
+    while (desc[i].flags & VRING_DESC_F_NEXT)
+        i = desc[i].next;
 
     /* link it with free list and point to it */
 
-    SET_LOWFLAT(desc[i].next, GET_LOWFLAT(vq->free_head));
-    SET_LOWFLAT(vq->free_head, head);
+    desc[i].next = vq->free_head;
+    vq->free_head = head;
 }
 
 /*
@@ -77,22 +77,22 @@ int vring_get_buf(struct vring_virtqueue *vq, unsigned int *len)
 {
     struct vring *vr = &vq->vring;
     struct vring_used_elem *elem;
-    struct vring_used *used = GET_LOWFLAT(vq->vring.used);
+    struct vring_used *used = vq->vring.used;
     u32 id;
     int ret;
 
 //    BUG_ON(!vring_more_used(vq));
 
-    elem = &used->ring[GET_LOWFLAT(vq->last_used_idx) % GET_LOWFLAT(vr->num)];
-    id = GET_LOWFLAT(elem->id);
+    elem = &used->ring[vq->last_used_idx % vr->num];
+    id = elem->id;
     if (len != NULL)
-        *len = GET_LOWFLAT(elem->len);
+        *len = elem->len;
 
-    ret = GET_LOWFLAT(vq->vdata[id]);
+    ret = vq->vdata[id];
 
     vring_detach(vq, id);
 
-    SET_LOWFLAT(vq->last_used_idx, GET_LOWFLAT(vq->last_used_idx) + 1);
+    vq->last_used_idx = vq->last_used_idx + 1;
 
     return ret;
 }
@@ -104,46 +104,45 @@ void vring_add_buf(struct vring_virtqueue *vq,
 {
     struct vring *vr = &vq->vring;
     int i, av, head, prev;
-    struct vring_desc *desc = GET_LOWFLAT(vr->desc);
-    struct vring_avail *avail = GET_LOWFLAT(vr->avail);
+    struct vring_desc *desc = vr->desc;
+    struct vring_avail *avail = vr->avail;
 
     BUG_ON(out + in == 0);
 
     prev = 0;
-    head = GET_LOWFLAT(vq->free_head);
-    for (i = head; out; i = GET_LOWFLAT(desc[i].next), out--) {
-        SET_LOWFLAT(desc[i].flags, VRING_DESC_F_NEXT);
-        SET_LOWFLAT(desc[i].addr, (u64)virt_to_phys(list->addr));
-        SET_LOWFLAT(desc[i].len, list->length);
+    head = vq->free_head;
+    for (i = head; out; i = desc[i].next, out--) {
+        desc[i].flags = VRING_DESC_F_NEXT;
+        desc[i].addr = (u64)virt_to_phys(list->addr);
+        desc[i].len = list->length;
         prev = i;
         list++;
     }
-    for ( ; in; i = GET_LOWFLAT(desc[i].next), in--) {
-        SET_LOWFLAT(desc[i].flags, VRING_DESC_F_NEXT|VRING_DESC_F_WRITE);
-        SET_LOWFLAT(desc[i].addr, (u64)virt_to_phys(list->addr));
-        SET_LOWFLAT(desc[i].len, list->length);
+    for ( ; in; i = desc[i].next, in--) {
+        desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE;
+        desc[i].addr = (u64)virt_to_phys(list->addr);
+        desc[i].len = list->length;
         prev = i;
         list++;
     }
-    SET_LOWFLAT(desc[prev].flags,
-                GET_LOWFLAT(desc[prev].flags) & ~VRING_DESC_F_NEXT);
+    desc[prev].flags = desc[prev].flags & ~VRING_DESC_F_NEXT;
 
-    SET_LOWFLAT(vq->free_head, i);
+    vq->free_head = i;
 
-    SET_LOWFLAT(vq->vdata[head], index);
+    vq->vdata[head] = index;
 
-    av = (GET_LOWFLAT(avail->idx) + num_added) % GET_LOWFLAT(vr->num);
-    SET_LOWFLAT(avail->ring[av], head);
+    av = (avail->idx + num_added) % vr->num;
+    avail->ring[av] = head;
 }
 
-void vring_kick(unsigned int ioaddr, struct vring_virtqueue *vq, int num_added)
+void vring_kick(struct vp_device *vp, struct vring_virtqueue *vq, int num_added)
 {
     struct vring *vr = &vq->vring;
-    struct vring_avail *avail = GET_LOWFLAT(vr->avail);
+    struct vring_avail *avail = vr->avail;
 
     /* Make sure idx update is done after ring write. */
     smp_wmb();
-    SET_LOWFLAT(avail->idx, GET_LOWFLAT(avail->idx) + num_added);
+    avail->idx = avail->idx + num_added;
 
-    vp_notify(ioaddr, GET_LOWFLAT(vq->queue_index));
+    vp_notify(vp, vq);
 }
diff --git a/qemu/roms/seabios/src/hw/virtio-ring.h b/qemu/roms/seabios/src/hw/virtio-ring.h
index b7a7aafb2..7665fd54b 100644
--- a/qemu/roms/seabios/src/hw/virtio-ring.h
+++ b/qemu/roms/seabios/src/hw/virtio-ring.h
@@ -4,15 +4,6 @@
 #include "types.h" // u64
 #include "memmap.h" // PAGE_SIZE
 
-#define PAGE_SHIFT 12
-#define PAGE_MASK  (PAGE_SIZE-1)
-
-#define virt_to_phys(v) (unsigned long)(v)
-#define phys_to_virt(p) (void*)(p)
-/* Compiler barrier is enough as an x86 CPU does not reorder reads or writes */
-#define smp_rmb() barrier()
-#define smp_wmb() barrier()
-
 /* Status byte for guest to report progress, and synchronize features. */
 /* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
 #define VIRTIO_CONFIG_S_ACKNOWLEDGE     1
@@ -20,9 +11,14 @@
 #define VIRTIO_CONFIG_S_DRIVER          2
 /* Driver has used its parts of the config, and is happy */
 #define VIRTIO_CONFIG_S_DRIVER_OK       4
+/* Driver has finished configuring features */
+#define VIRTIO_CONFIG_S_FEATURES_OK     8
 /* We've given up on this device. */
 #define VIRTIO_CONFIG_S_FAILED          0x80
 
+/* v1.0 compliant. */
+#define VIRTIO_F_VERSION_1              32
+
 #define MAX_QUEUE_NUM      (128)
 
 #define VRING_DESC_F_NEXT  1
@@ -68,10 +64,9 @@ struct vring {
 };
 
 #define vring_size(num) \
-   (((((sizeof(struct vring_desc) * num) + \
-      (sizeof(struct vring_avail) + sizeof(u16) * num)) \
-         + PAGE_MASK) & ~PAGE_MASK) + \
-         (sizeof(struct vring_used) + sizeof(struct vring_used_elem) * num))
+    (ALIGN(sizeof(struct vring_desc) * num + sizeof(struct vring_avail) \
+           + sizeof(u16) * num, PAGE_SIZE)                              \
+     + sizeof(struct vring_used) + sizeof(struct vring_used_elem) * num)
 
 typedef unsigned char virtio_queue_t[vring_size(MAX_QUEUE_NUM)];
 
@@ -83,6 +78,7 @@ struct vring_virtqueue {
    u16 vdata[MAX_QUEUE_NUM];
    /* PCI */
    int queue_index;
+   int queue_notify_off;
 };
 
 struct vring_list {
@@ -90,42 +86,35 @@ struct vring_list {
   unsigned int length;
 };
 
-static inline void vring_init(struct vring *vr,
-                         unsigned int num, unsigned char *queue)
+static inline void
+vring_init(struct vring *vr, unsigned int num, unsigned char *queue)
 {
-   unsigned int i;
-   unsigned long pa;
-
    ASSERT32FLAT();
    vr->num = num;
 
    /* physical address of desc must be page aligned */
-
-   pa = virt_to_phys(queue);
-   pa = (pa + PAGE_MASK) & ~PAGE_MASK;
-   vr->desc = phys_to_virt(pa);
+   vr->desc = (void*)ALIGN((u32)queue, PAGE_SIZE);
 
    vr->avail = (struct vring_avail *)&vr->desc[num];
    /* disable interrupts */
    vr->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
 
    /* physical address of used must be page aligned */
+   vr->used = (void*)ALIGN((u32)&vr->avail->ring[num], PAGE_SIZE);
 
-   pa = virt_to_phys(&vr->avail->ring[num]);
-   pa = (pa + PAGE_MASK) & ~PAGE_MASK;
-   vr->used = phys_to_virt(pa);
-
+   int i;
    for (i = 0; i < num - 1; i++)
-           vr->desc[i].next = i + 1;
+       vr->desc[i].next = i + 1;
    vr->desc[i].next = 0;
 }
 
+struct vp_device;
 int vring_more_used(struct vring_virtqueue *vq);
 void vring_detach(struct vring_virtqueue *vq, unsigned int head);
 int vring_get_buf(struct vring_virtqueue *vq, unsigned int *len);
 void vring_add_buf(struct vring_virtqueue *vq, struct vring_list list[],
                    unsigned int out, unsigned int in,
                    int index, int num_added);
-void vring_kick(unsigned int ioaddr, struct vring_virtqueue *vq, int num_added);
+void vring_kick(struct vp_device *vp, struct vring_virtqueue *vq, int num_added);
 
 #endif /* _VIRTIO_RING_H_ */
diff --git a/qemu/roms/seabios/src/hw/virtio-scsi.c b/qemu/roms/seabios/src/hw/virtio-scsi.c
index 8f966875b..80afd04ca 100644
--- a/qemu/roms/seabios/src/hw/virtio-scsi.c
+++ b/qemu/roms/seabios/src/hw/virtio-scsi.c
@@ -27,35 +27,42 @@ struct virtio_lun_s {
     struct drive_s drive;
     struct pci_device *pci;
     struct vring_virtqueue *vq;
-    u16 ioaddr;
+    struct vp_device *vp;
     u16 target;
     u16 lun;
 };
 
-static int
-virtio_scsi_cmd(u16 ioaddr, struct vring_virtqueue *vq, struct disk_op_s *op,
-                void *cdbcmd, u16 target, u16 lun, u16 blocksize)
+int
+virtio_scsi_process_op(struct disk_op_s *op)
 {
+    if (! CONFIG_VIRTIO_SCSI)
+        return 0;
+    struct virtio_lun_s *vlun =
+        container_of(op->drive_gf, struct virtio_lun_s, drive);
+    struct vp_device *vp = vlun->vp;
+    struct vring_virtqueue *vq = vlun->vq;
     struct virtio_scsi_req_cmd req;
     struct virtio_scsi_resp_cmd resp;
     struct vring_list sg[3];
 
     memset(&req, 0, sizeof(req));
+    int blocksize = scsi_fill_cmd(op, req.cdb, 16);
+    if (blocksize < 0)
+        return default_process_op(op);
     req.lun[0] = 1;
-    req.lun[1] = target;
-    req.lun[2] = (lun >> 8) | 0x40;
-    req.lun[3] = (lun & 0xff);
-    memcpy(req.cdb, cdbcmd, 16);
+    req.lun[1] = vlun->target;
+    req.lun[2] = (vlun->lun >> 8) | 0x40;
+    req.lun[3] = (vlun->lun & 0xff);
 
     u32 len = op->count * blocksize;
-    int datain = cdb_is_read(cdbcmd, blocksize);
+    int datain = scsi_is_read(op);
     int in_num = (datain ? 2 : 1);
     int out_num = (len ? 3 : 2) - in_num;
 
-    sg[0].addr   = MAKE_FLATPTR(GET_SEG(SS), &req);
+    sg[0].addr   = (void*)(&req);
     sg[0].length = sizeof(req);
 
-    sg[out_num].addr   = MAKE_FLATPTR(GET_SEG(SS), &resp);
+    sg[out_num].addr   = (void*)(&resp);
     sg[out_num].length = sizeof(resp);
 
     if (len) {
@@ -66,7 +73,7 @@ virtio_scsi_cmd(u16 ioaddr, struct vring_virtqueue *vq, struct disk_op_s *op,
 
     /* Add to virtqueue and kick host */
     vring_add_buf(vq, sg, out_num, in_num, 0, 0);
-    vring_kick(ioaddr, vq, 1);
+    vring_kick(vp, vq, 1);
 
     /* Wait for reply */
     while (!vring_more_used(vq))
@@ -78,7 +85,7 @@ virtio_scsi_cmd(u16 ioaddr, struct vring_virtqueue *vq, struct disk_op_s *op,
     /* Clear interrupt status register.  Avoid leaving interrupts stuck if
      * VRING_AVAIL_F_NO_INTERRUPT was ignored and interrupts were raised.
      */
-    vp_get_isr(ioaddr);
+    vp_get_isr(vp);
 
     if (resp.response == VIRTIO_SCSI_S_OK && resp.status == 0) {
         return DISK_RET_SUCCESS;
@@ -86,21 +93,8 @@ virtio_scsi_cmd(u16 ioaddr, struct vring_virtqueue *vq, struct disk_op_s *op,
     return DISK_RET_EBADTRACK;
 }
 
-int
-virtio_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
-{
-    struct virtio_lun_s *vlun_gf =
-        container_of(op->drive_gf, struct virtio_lun_s, drive);
-
-    return virtio_scsi_cmd(GET_GLOBALFLAT(vlun_gf->ioaddr),
-                           GET_GLOBALFLAT(vlun_gf->vq), op, cdbcmd,
-                           GET_GLOBALFLAT(vlun_gf->target),
-                           GET_GLOBALFLAT(vlun_gf->lun),
-                           blocksize);
-}
-
 static int
-virtio_scsi_add_lun(struct pci_device *pci, u16 ioaddr,
+virtio_scsi_add_lun(struct pci_device *pci, struct vp_device *vp,
                     struct vring_virtqueue *vq, u16 target, u16 lun)
 {
     struct virtio_lun_s *vlun = malloc_fseg(sizeof(*vlun));
@@ -112,7 +106,7 @@ virtio_scsi_add_lun(struct pci_device *pci, u16 ioaddr,
     vlun->drive.type = DTYPE_VIRTIO_SCSI;
     vlun->drive.cntl_id = pci->bdf;
     vlun->pci = pci;
-    vlun->ioaddr = ioaddr;
+    vlun->vp = vp;
     vlun->vq = vq;
     vlun->target = target;
     vlun->lun = lun;
@@ -129,11 +123,11 @@ fail:
 }
 
 static int
-virtio_scsi_scan_target(struct pci_device *pci, u16 ioaddr,
+virtio_scsi_scan_target(struct pci_device *pci, struct vp_device *vp,
                         struct vring_virtqueue *vq, u16 target)
 {
     /* TODO: send REPORT LUNS.  For now, only LUN 0 is recognized.  */
-    int ret = virtio_scsi_add_lun(pci, ioaddr, vq, target, 0);
+    int ret = virtio_scsi_add_lun(pci, vp, vq, target, 0);
     return ret < 0 ? 0 : 1;
 }
 
@@ -144,19 +138,45 @@ init_virtio_scsi(struct pci_device *pci)
     dprintf(1, "found virtio-scsi at %x:%x\n", pci_bdf_to_bus(bdf),
             pci_bdf_to_dev(bdf));
     struct vring_virtqueue *vq = NULL;
-    u16 ioaddr = vp_init_simple(bdf);
-    if (vp_find_vq(ioaddr, 2, &vq) < 0 ) {
+    struct vp_device *vp = malloc_high(sizeof(*vp));
+    if (!vp) {
+        warn_noalloc();
+        return;
+    }
+    vp_init_simple(vp, pci);
+    u8 status = VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER;
+
+    if (vp->use_modern) {
+        u64 features = vp_get_features(vp);
+        u64 version1 = 1ull << VIRTIO_F_VERSION_1;
+        if (!(features & version1)) {
+            dprintf(1, "modern device without virtio_1 feature bit: %x:%x\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
+            goto fail;
+        }
+
+        vp_set_features(vp, version1);
+        status |= VIRTIO_CONFIG_S_FEATURES_OK;
+        vp_set_status(vp, status);
+        if (!(vp_get_status(vp) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+            dprintf(1, "device didn't accept features: %x:%x\n",
+                    pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
+            goto fail;
+        }
+    }
+
+    if (vp_find_vq(vp, 2, &vq) < 0 ) {
         dprintf(1, "fail to find vq for virtio-scsi %x:%x\n",
                 pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf));
         goto fail;
     }
 
-    vp_set_status(ioaddr, VIRTIO_CONFIG_S_ACKNOWLEDGE |
-                  VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_DRIVER_OK);
+    status |= VIRTIO_CONFIG_S_DRIVER_OK;
+    vp_set_status(vp, status);
 
     int i, tot;
     for (tot = 0, i = 0; i < 256; i++)
-        tot += virtio_scsi_scan_target(pci, ioaddr, vq, i);
+        tot += virtio_scsi_scan_target(pci, vp, vq, i);
 
     if (!tot)
         goto fail;
@@ -164,7 +184,8 @@ init_virtio_scsi(struct pci_device *pci)
     return;
 
 fail:
-    vp_reset(ioaddr);
+    vp_reset(vp);
+    free(vp);
     free(vq);
 }
 
@@ -179,8 +200,9 @@ virtio_scsi_setup(void)
 
     struct pci_device *pci;
     foreachpci(pci) {
-        if (pci->vendor != PCI_VENDOR_ID_REDHAT_QUMRANET
-            || pci->device != PCI_DEVICE_ID_VIRTIO_SCSI)
+        if (pci->vendor != PCI_VENDOR_ID_REDHAT_QUMRANET ||
+            (pci->device != PCI_DEVICE_ID_VIRTIO_SCSI_09 &&
+             pci->device != PCI_DEVICE_ID_VIRTIO_SCSI_10))
             continue;
         init_virtio_scsi(pci);
     }
diff --git a/qemu/roms/seabios/src/hw/virtio-scsi.h b/qemu/roms/seabios/src/hw/virtio-scsi.h
index 96c3701d2..7532cc98e 100644
--- a/qemu/roms/seabios/src/hw/virtio-scsi.h
+++ b/qemu/roms/seabios/src/hw/virtio-scsi.h
@@ -41,7 +41,7 @@ struct virtio_scsi_resp_cmd {
 #define VIRTIO_SCSI_S_OK            0
 
 struct disk_op_s;
-int virtio_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+int virtio_scsi_process_op(struct disk_op_s *op);
 void virtio_scsi_setup(void);
 
 #endif /* _VIRTIO_SCSI_H */
diff --git a/qemu/roms/seabios/src/list.h b/qemu/roms/seabios/src/list.h
index de656b9d6..94512e306 100644
--- a/qemu/roms/seabios/src/list.h
+++ b/qemu/roms/seabios/src/list.h
@@ -61,6 +61,16 @@ hlist_add_after(struct hlist_node *n, struct hlist_node *prev)
     hlist_add(n, &prev->next);
 }
 
+static inline void
+hlist_replace(struct hlist_node *old, struct hlist_node *new)
+{
+    new->next = old->next;
+    if (new->next)
+        new->next->pprev = &new->next;
+    new->pprev = old->pprev;
+    *new->pprev = new;
+}
+
 #define hlist_for_each_entry(pos, head, member)                         \
     for (pos = container_of((head)->first, typeof(*pos), member)        \
          ; pos != container_of(NULL, typeof(*pos), member)              \
diff --git a/qemu/roms/seabios/src/malloc.c b/qemu/roms/seabios/src/malloc.c
index c4cb17149..3733855ca 100644
--- a/qemu/roms/seabios/src/malloc.c
+++ b/qemu/roms/seabios/src/malloc.c
@@ -6,9 +6,10 @@
 
 #include "biosvar.h" // GET_BDA
 #include "config.h" // BUILD_BIOS_ADDR
+#include "e820map.h" // struct e820entry
 #include "list.h" // hlist_node
 #include "malloc.h" // _malloc
-#include "memmap.h" // struct e820entry
+#include "memmap.h" // PAGE_SIZE
 #include "output.h" // dprintf
 #include "stacks.h" // wait_preempt
 #include "std/optionrom.h" // OPTION_ROM_ALIGN
@@ -17,7 +18,7 @@
 // Information on a reserved area.
 struct allocinfo_s {
     struct hlist_node node;
-    void *data, *dataend, *allocend;
+    u32 range_start, range_end, alloc_size;
 };
 
 // Information on a tracked memory allocation.
@@ -46,98 +47,106 @@ static struct zone_s *Zones[] VARVERIFY32INIT = {
  ****************************************************************/
 
 // Find and reserve space from a given zone
-static void *
-allocSpace(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill)
+static u32
+alloc_new(struct zone_s *zone, u32 size, u32 align, struct allocinfo_s *fill)
 {
     struct allocinfo_s *info;
     hlist_for_each_entry(info, &zone->head, node) {
-        void *dataend = info->dataend;
-        void *allocend = info->allocend;
-        void *newallocend = (void*)ALIGN_DOWN((u32)allocend - size, align);
-        if (newallocend >= dataend && newallocend <= allocend) {
+        u32 alloc_end = info->range_start + info->alloc_size;
+        u32 range_end = info->range_end;
+        u32 new_range_end = ALIGN_DOWN(range_end - size, align);
+        if (new_range_end >= alloc_end && new_range_end <= range_end) {
             // Found space - now reserve it.
-            if (!fill)
-                fill = newallocend;
-            fill->data = newallocend;
-            fill->dataend = newallocend + size;
-            fill->allocend = allocend;
+            fill->range_start = new_range_end;
+            fill->range_end = range_end;
+            fill->alloc_size = size;
 
-            info->allocend = newallocend;
+            info->range_end = new_range_end;
             hlist_add_before(&fill->node, &info->node);
-            return newallocend;
+            return new_range_end;
         }
     }
-    return NULL;
+    return 0;
 }
 
-// Release space allocated with allocSpace()
-static void
-freeSpace(struct allocinfo_s *info)
+// Reserve space for a 'struct allocdetail_s' and fill
+static struct allocdetail_s *
+alloc_new_detail(struct allocdetail_s *temp)
 {
-    struct allocinfo_s *next = container_of_or_null(
-        info->node.next, struct allocinfo_s, node);
-    if (next && next->allocend == info->data)
-        next->allocend = info->allocend;
-    hlist_del(&info->node);
+    u32 detail_addr = alloc_new(&ZoneTmpHigh, sizeof(struct allocdetail_s)
+                                , MALLOC_MIN_ALIGN, &temp->detailinfo);
+    if (!detail_addr) {
+        detail_addr = alloc_new(&ZoneTmpLow, sizeof(struct allocdetail_s)
+                                , MALLOC_MIN_ALIGN, &temp->detailinfo);
+        if (!detail_addr) {
+            warn_noalloc();
+            return NULL;
+        }
+    }
+    struct allocdetail_s *detail = memremap(detail_addr, sizeof(*detail));
+
+    // Fill final 'detail' allocation from data in 'temp'
+    memcpy(detail, temp, sizeof(*detail));
+    hlist_replace(&temp->detailinfo.node, &detail->detailinfo.node);
+    hlist_replace(&temp->datainfo.node, &detail->datainfo.node);
+    return detail;
 }
 
 // Add new memory to a zone
 static void
-addSpace(struct zone_s *zone, void *start, void *end)
+alloc_add(struct zone_s *zone, u32 start, u32 end)
 {
     // Find position to add space
     struct allocinfo_s *info;
     struct hlist_node **pprev;
     hlist_for_each_entry_pprev(info, pprev, &zone->head, node) {
-        if (info->data < start)
+        if (info->range_start < start)
             break;
     }
 
     // Add space using temporary allocation info.
     struct allocdetail_s tempdetail;
-    tempdetail.datainfo.data = tempdetail.datainfo.dataend = start;
-    tempdetail.datainfo.allocend = end;
+    tempdetail.handle = MALLOC_DEFAULT_HANDLE;
+    tempdetail.datainfo.range_start = start;
+    tempdetail.datainfo.range_end = end;
+    tempdetail.datainfo.alloc_size = 0;
     hlist_add(&tempdetail.datainfo.node, pprev);
 
     // Allocate final allocation info.
-    struct allocdetail_s *detail = allocSpace(
-        &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, NULL);
-    if (!detail) {
-        detail = allocSpace(&ZoneTmpLow, sizeof(*detail)
-                            , MALLOC_MIN_ALIGN, NULL);
-        if (!detail) {
-            hlist_del(&tempdetail.datainfo.node);
-            warn_noalloc();
-            return;
-        }
-    }
+    struct allocdetail_s *detail = alloc_new_detail(&tempdetail);
+    if (!detail)
+        hlist_del(&tempdetail.datainfo.node);
+}
 
-    // Replace temp alloc space with final alloc space
-    pprev = tempdetail.datainfo.node.pprev;
-    hlist_del(&tempdetail.datainfo.node);
-    memcpy(&detail->datainfo, &tempdetail.datainfo, sizeof(detail->datainfo));
-    detail->handle = MALLOC_DEFAULT_HANDLE;
-    hlist_add(&detail->datainfo.node, pprev);
+// Release space allocated with alloc_new()
+static void
+alloc_free(struct allocinfo_s *info)
+{
+    struct allocinfo_s *next = container_of_or_null(
+        info->node.next, struct allocinfo_s, node);
+    if (next && next->range_end == info->range_start)
+        next->range_end = info->range_end;
+    hlist_del(&info->node);
 }
 
-// Search all zones for an allocation obtained from allocSpace()
+// Search all zones for an allocation obtained from alloc_new()
 static struct allocinfo_s *
-findAlloc(void *data)
+alloc_find(u32 data)
 {
     int i;
     for (i=0; i<ARRAY_SIZE(Zones); i++) {
         struct allocinfo_s *info;
         hlist_for_each_entry(info, &Zones[i]->head, node) {
-            if (info->data == data)
+            if (info->range_start == data)
                 return info;
         }
     }
     return NULL;
 }
 
-// Return the last sentinal node of a zone
+// Find the lowest memory range added by alloc_add()
 static struct allocinfo_s *
-findLast(struct zone_s *zone)
+alloc_find_lowest(struct zone_s *zone)
 {
     struct allocinfo_s *info, *last = NULL;
     hlist_for_each_entry(info, &zone->head, node) {
@@ -171,25 +180,25 @@ relocate_ebda(u32 newebda, u32 oldebda, u8 ebda_size)
 }
 
 // Support expanding the ZoneLow dynamically.
-static void *
+static u32
 zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill)
 {
     // Make sure to not move ebda while an optionrom is running.
     if (unlikely(wait_preempt())) {
-        void *data = allocSpace(&ZoneLow, size, align, fill);
+        u32 data = alloc_new(&ZoneLow, size, align, fill);
         if (data)
             return data;
     }
 
-    struct allocinfo_s *info = findLast(&ZoneLow);
+    struct allocinfo_s *info = alloc_find_lowest(&ZoneLow);
     if (!info)
-        return NULL;
-    u32 oldpos = (u32)info->allocend;
+        return 0;
+    u32 oldpos = info->range_end;
     u32 newpos = ALIGN_DOWN(oldpos - size, align);
-    u32 bottom = (u32)info->dataend;
+    u32 bottom = info->range_start + info->alloc_size;
     if (newpos >= bottom && newpos <= oldpos)
         // Space already present.
-        return allocSpace(&ZoneLow, size, align, fill);
+        return alloc_new(&ZoneLow, size, align, fill);
     u16 ebda_seg = get_ebda_seg();
     u32 ebda_pos = (u32)MAKE_FLATPTR(ebda_seg, 0);
     u8 ebda_size = GET_EBDA(ebda_seg, size);
@@ -201,21 +210,20 @@ zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill)
     u32 newebda = ALIGN_DOWN(newbottom - ebda_size * 1024, 1024);
     if (newebda < BUILD_EBDA_MINIMUM)
         // Not enough space.
-        return NULL;
+        return 0;
 
     // Move ebda
     int ret = relocate_ebda(newebda, ebda_pos, ebda_size);
     if (ret)
-        return NULL;
+        return 0;
 
     // Update zone
-    if (ebda_end == bottom) {
-        info->data = (void*)newbottom;
-        info->dataend = (void*)newbottom;
-    } else
-        addSpace(&ZoneLow, (void*)newbottom, (void*)ebda_end);
+    if (ebda_end == bottom)
+        info->range_start = newbottom;
+    else
+        alloc_add(&ZoneLow, newbottom, ebda_end);
 
-    return allocSpace(&ZoneLow, size, align, fill);
+    return alloc_new(&ZoneLow, size, align, fill);
 }
 
 
@@ -223,56 +231,69 @@ zonelow_expand(u32 size, u32 align, struct allocinfo_s *fill)
  * tracked memory allocations
  ****************************************************************/
 
-// Allocate memory from the given zone and track it as a PMM allocation
-void * __malloc
-_malloc(struct zone_s *zone, u32 size, u32 align)
+// Allocate physical memory from the given zone and track it as a PMM allocation
+u32
+malloc_palloc(struct zone_s *zone, u32 size, u32 align)
 {
     ASSERT32FLAT();
     if (!size)
-        return NULL;
-
-    // Find and reserve space for bookkeeping.
-    struct allocdetail_s *detail = allocSpace(
-        &ZoneTmpHigh, sizeof(*detail), MALLOC_MIN_ALIGN, NULL);
-    if (!detail) {
-        detail = allocSpace(&ZoneTmpLow, sizeof(*detail)
-                            , MALLOC_MIN_ALIGN, NULL);
-        if (!detail)
-            return NULL;
-    }
-    detail->handle = MALLOC_DEFAULT_HANDLE;
+        return 0;
 
     // Find and reserve space for main allocation
-    void *data = allocSpace(zone, size, align, &detail->datainfo);
+    struct allocdetail_s tempdetail;
+    tempdetail.handle = MALLOC_DEFAULT_HANDLE;
+    u32 data = alloc_new(zone, size, align, &tempdetail.datainfo);
     if (!CONFIG_MALLOC_UPPERMEMORY && !data && zone == &ZoneLow)
-        data = zonelow_expand(size, align, &detail->datainfo);
-    if (!data) {
-        freeSpace(&detail->detailinfo);
-        return NULL;
+        data = zonelow_expand(size, align, &tempdetail.datainfo);
+    if (!data)
+        return 0;
+
+    // Find and reserve space for bookkeeping.
+    struct allocdetail_s *detail = alloc_new_detail(&tempdetail);
+    if (!detail) {
+        alloc_free(&tempdetail.datainfo);
+        return 0;
     }
 
-    dprintf(8, "_malloc zone=%p size=%d align=%x ret=%p (detail=%p)\n"
+    dprintf(8, "phys_alloc zone=%p size=%d align=%x ret=%x (detail=%p)\n"
             , zone, size, align, data, detail);
 
     return data;
 }
 
-// Free a data block allocated with _malloc
+// Allocate virtual memory from the given zone
+void * __malloc
+_malloc(struct zone_s *zone, u32 size, u32 align)
+{
+    return memremap(malloc_palloc(zone, size, align), size);
+}
+
+// Free a data block allocated with phys_alloc
 int
-_free(void *data)
+malloc_pfree(u32 data)
 {
     ASSERT32FLAT();
-    struct allocinfo_s *info = findAlloc(data);
-    if (!info || data == (void*)info || data == info->dataend)
+    struct allocinfo_s *info = alloc_find(data);
+    if (!info || data == virt_to_phys(info) || !info->alloc_size)
         return -1;
     struct allocdetail_s *detail = container_of(
         info, struct allocdetail_s, datainfo);
-    dprintf(8, "_free %p (detail=%p)\n", data, detail);
-    freeSpace(info);
-    freeSpace(&detail->detailinfo);
+    dprintf(8, "phys_free %x (detail=%p)\n", data, detail);
+    alloc_free(info);
+    alloc_free(&detail->detailinfo);
     return 0;
 }
 
+void
+free(void *data)
+{
+    if (!data)
+        return;
+    int ret = malloc_pfree(virt_to_phys(data));
+    if (ret)
+        warn_internalerror();
+}
+
 // Find the amount of free space in a given zone.
 u32
 malloc_getspace(struct zone_s *zone)
@@ -282,7 +303,7 @@ malloc_getspace(struct zone_s *zone)
     u32 maxspace = 0;
     struct allocinfo_s *info;
     hlist_for_each_entry(info, &zone->head, node) {
-        u32 space = info->allocend - info->dataend;
+        u32 space = info->range_end - info->range_start - info->alloc_size;
         if (space > maxspace)
             maxspace = space;
     }
@@ -298,34 +319,34 @@ malloc_getspace(struct zone_s *zone)
 
 // Set a handle associated with an allocation.
 void
-malloc_sethandle(void *data, u32 handle)
+malloc_sethandle(u32 data, u32 handle)
 {
     ASSERT32FLAT();
-    struct allocinfo_s *info = findAlloc(data);
-    if (!info || data == (void*)info || data == info->dataend)
+    struct allocinfo_s *info = alloc_find(data);
+    if (!info || data == virt_to_phys(info) || !info->alloc_size)
         return;
     struct allocdetail_s *detail = container_of(
         info, struct allocdetail_s, datainfo);
     detail->handle = handle;
 }
 
-// Find the data block allocated with _malloc with a given handle.
-void *
+// Find the data block allocated with phys_alloc with a given handle.
+u32
 malloc_findhandle(u32 handle)
 {
     int i;
     for (i=0; i<ARRAY_SIZE(Zones); i++) {
         struct allocinfo_s *info;
         hlist_for_each_entry(info, &Zones[i]->head, node) {
-            if (info->data != (void*)info)
+            if (info->range_start != virt_to_phys(info))
                 continue;
             struct allocdetail_s *detail = container_of(
                 info, struct allocdetail_s, detailinfo);
             if (detail->handle == handle)
-                return detail->datainfo.data;
+                return detail->datainfo.range_start;
         }
     }
-    return NULL;
+    return 0;
 }
 
 
@@ -343,10 +364,9 @@ u32
 rom_get_max(void)
 {
     if (CONFIG_MALLOC_UPPERMEMORY)
-        return ALIGN_DOWN((u32)RomBase->allocend - OPROM_HEADER_RESERVE
+        return ALIGN_DOWN(RomBase->range_end - OPROM_HEADER_RESERVE
                           , OPTION_ROM_ALIGN);
-    extern u8 final_readonly_start[];
-    return (u32)final_readonly_start;
+    return SYMBOL(final_readonly_start);
 }
 
 // Return the end of the last deployed option rom.
@@ -364,9 +384,9 @@ rom_reserve(u32 size)
     if (newend > rom_get_max())
         return NULL;
     if (CONFIG_MALLOC_UPPERMEMORY) {
-        if (newend < (u32)zonelow_base)
-            newend = (u32)zonelow_base;
-        RomBase->data = RomBase->dataend = (void*)newend + OPROM_HEADER_RESERVE;
+        if (newend < SYMBOL(zonelow_base))
+            newend = SYMBOL(zonelow_base);
+        RomBase->range_start = newend + OPROM_HEADER_RESERVE;
     }
     return (void*)RomEnd;
 }
@@ -396,10 +416,10 @@ malloc_preinit(void)
     dprintf(3, "malloc preinit\n");
 
     // Don't declare any memory between 0xa0000 and 0x100000
-    add_e820(BUILD_LOWRAM_END, BUILD_BIOS_ADDR-BUILD_LOWRAM_END, E820_HOLE);
+    e820_remove(BUILD_LOWRAM_END, BUILD_BIOS_ADDR-BUILD_LOWRAM_END);
 
     // Mark known areas as reserved.
-    add_e820(BUILD_BIOS_ADDR, BUILD_BIOS_SIZE, E820_RESERVED);
+    e820_add(BUILD_BIOS_ADDR, BUILD_BIOS_SIZE, E820_RESERVED);
 
     // Populate temp high ram
     u32 highram = 0;
@@ -419,31 +439,30 @@ malloc_preinit(void)
                 e = newe;
             }
         }
-        addSpace(&ZoneTmpHigh, (void*)s, (void*)e);
+        alloc_add(&ZoneTmpHigh, s, e);
     }
 
     // Populate regions
-    addSpace(&ZoneTmpLow, (void*)BUILD_STACK_ADDR, (void*)BUILD_EBDA_MINIMUM);
+    alloc_add(&ZoneTmpLow, BUILD_STACK_ADDR, BUILD_EBDA_MINIMUM);
     if (highram) {
-        addSpace(&ZoneHigh, (void*)highram
-                 , (void*)highram + BUILD_MAX_HIGHTABLE);
-        add_e820(highram, BUILD_MAX_HIGHTABLE, E820_RESERVED);
+        alloc_add(&ZoneHigh, highram, highram + BUILD_MAX_HIGHTABLE);
+        e820_add(highram, BUILD_MAX_HIGHTABLE, E820_RESERVED);
     }
 }
 
 void
-csm_malloc_preinit(u32 low_pmm, u32 low_pmm_size, u32 hi_pmm, u32 hi_pmm_size)
+malloc_csm_preinit(u32 low_pmm, u32 low_pmm_size, u32 hi_pmm, u32 hi_pmm_size)
 {
     ASSERT32FLAT();
 
     if (hi_pmm_size > BUILD_MAX_HIGHTABLE) {
-        void *hi_pmm_end = (void *)hi_pmm + hi_pmm_size;
-        addSpace(&ZoneTmpHigh, (void *)hi_pmm, hi_pmm_end - BUILD_MAX_HIGHTABLE);
-        addSpace(&ZoneHigh, hi_pmm_end - BUILD_MAX_HIGHTABLE, hi_pmm_end);
+        u32 hi_pmm_end = hi_pmm + hi_pmm_size;
+        alloc_add(&ZoneTmpHigh, hi_pmm, hi_pmm_end - BUILD_MAX_HIGHTABLE);
+        alloc_add(&ZoneHigh, hi_pmm_end - BUILD_MAX_HIGHTABLE, hi_pmm_end);
     } else {
-        addSpace(&ZoneTmpHigh, (void *)hi_pmm, (void *)hi_pmm + hi_pmm_size);
+        alloc_add(&ZoneTmpHigh, hi_pmm, hi_pmm + hi_pmm_size);
     }
-    addSpace(&ZoneTmpLow, (void *)low_pmm, (void *)low_pmm + low_pmm_size);
+    alloc_add(&ZoneTmpLow, low_pmm, low_pmm + low_pmm_size);
 }
 
 u32 LegacyRamSize VARFSEG;
@@ -484,21 +503,21 @@ malloc_init(void)
     }
 
     // Initialize low-memory region
-    extern u8 varlow_start[], varlow_end[], final_varlow_start[];
-    memmove(final_varlow_start, varlow_start, varlow_end - varlow_start);
+    memmove(VSYMBOL(final_varlow_start), VSYMBOL(varlow_start)
+            , SYMBOL(varlow_end) - SYMBOL(varlow_start));
     if (CONFIG_MALLOC_UPPERMEMORY) {
-        addSpace(&ZoneLow, zonelow_base + OPROM_HEADER_RESERVE
-                 , final_varlow_start);
-        RomBase = findLast(&ZoneLow);
+        alloc_add(&ZoneLow, SYMBOL(zonelow_base) + OPROM_HEADER_RESERVE
+                  , SYMBOL(final_varlow_start));
+        RomBase = alloc_find_lowest(&ZoneLow);
     } else {
-        addSpace(&ZoneLow, (void*)ALIGN_DOWN((u32)final_varlow_start, 1024)
-                 , final_varlow_start);
+        alloc_add(&ZoneLow, ALIGN_DOWN(SYMBOL(final_varlow_start), 1024)
+                  , SYMBOL(final_varlow_start));
     }
 
     // Add space available in f-segment to ZoneFSeg
-    extern u8 zonefseg_start[], zonefseg_end[];
-    memset(zonefseg_start, 0, zonefseg_end - zonefseg_start);
-    addSpace(&ZoneFSeg, zonefseg_start, zonefseg_end);
+    memset(VSYMBOL(zonefseg_start), 0
+           , SYMBOL(zonefseg_end) - SYMBOL(zonefseg_start));
+    alloc_add(&ZoneFSeg, SYMBOL(zonefseg_start), SYMBOL(zonefseg_end));
 
     calcRamSize();
 }
@@ -521,19 +540,20 @@ malloc_prepboot(void)
 
     // Reserve more low-mem if needed.
     u32 endlow = GET_BDA(mem_size_kb)*1024;
-    add_e820(endlow, BUILD_LOWRAM_END-endlow, E820_RESERVED);
+    e820_add(endlow, BUILD_LOWRAM_END-endlow, E820_RESERVED);
 
     // Clear unused f-seg ram.
-    struct allocinfo_s *info = findLast(&ZoneFSeg);
-    memset(info->dataend, 0, info->allocend - info->dataend);
+    struct allocinfo_s *info = alloc_find_lowest(&ZoneFSeg);
+    u32 size = info->range_end - info->range_start;
+    memset(memremap(info->range_start, size), 0, size);
     dprintf(1, "Space available for UMB: %x-%x, %x-%x\n"
-            , RomEnd, base, (u32)info->dataend, (u32)info->allocend);
+            , RomEnd, base, info->range_start, info->range_end);
 
     // Give back unused high ram.
-    info = findLast(&ZoneHigh);
+    info = alloc_find_lowest(&ZoneHigh);
     if (info) {
-        u32 giveback = ALIGN_DOWN(info->allocend - info->dataend, PAGE_SIZE);
-        add_e820((u32)info->dataend, giveback, E820_RAM);
+        u32 giveback = ALIGN_DOWN(info->range_end-info->range_start, PAGE_SIZE);
+        e820_add(info->range_start, giveback, E820_RAM);
         dprintf(1, "Returned %d bytes of ZoneHigh\n", giveback);
     }
 
diff --git a/qemu/roms/seabios/src/malloc.h b/qemu/roms/seabios/src/malloc.h
index 2bcb5bf6d..960a7f800 100644
--- a/qemu/roms/seabios/src/malloc.h
+++ b/qemu/roms/seabios/src/malloc.h
@@ -9,17 +9,19 @@ u32 rom_get_max(void);
 u32 rom_get_last(void);
 struct rom_header *rom_reserve(u32 size);
 int rom_confirm(u32 size);
-void csm_malloc_preinit(u32 low_pmm, u32 low_pmm_size, u32 hi_pmm,
+void malloc_csm_preinit(u32 low_pmm, u32 low_pmm_size, u32 hi_pmm,
                         u32 hi_pmm_size);
 void malloc_preinit(void);
 extern u32 LegacyRamSize;
 void malloc_init(void);
 void malloc_prepboot(void);
+u32 malloc_palloc(struct zone_s *zone, u32 size, u32 align);
 void *_malloc(struct zone_s *zone, u32 size, u32 align);
-int _free(void *data);
+int malloc_pfree(u32 data);
+void free(void *data);
 u32 malloc_getspace(struct zone_s *zone);
-void malloc_sethandle(void *data, u32 handle);
-void *malloc_findhandle(u32 handle);
+void malloc_sethandle(u32 data, u32 handle);
+u32 malloc_findhandle(u32 handle);
 
 #define MALLOC_DEFAULT_HANDLE 0xFFFFFFFF
 // Minimum alignment of malloc'd memory
@@ -64,8 +66,5 @@ static inline void *memalign_tmp(u32 align, u32 size) {
         return ret;
     return memalign_tmplow(align, size);
 }
-static inline void free(void *data) {
-    _free(data);
-}
 
 #endif // malloc.h
diff --git a/qemu/roms/seabios/src/memmap.h b/qemu/roms/seabios/src/memmap.h
index 7bda56e2b..22bd4bcb8 100644
--- a/qemu/roms/seabios/src/memmap.h
+++ b/qemu/roms/seabios/src/memmap.h
@@ -1,29 +1,21 @@
-#ifndef __E820MAP_H
-#define __E820MAP_H
+#ifndef __MEMMAP_H
+#define __MEMMAP_H
 
-#include "types.h" // u64
-
-#define E820_RAM          1
-#define E820_RESERVED     2
-#define E820_ACPI         3
-#define E820_NVS          4
-#define E820_UNUSABLE     5
-#define E820_HOLE         ((u32)-1) // Useful for removing entries
-
-struct e820entry {
-    u64 start;
-    u64 size;
-    u32 type;
-};
-
-void add_e820(u64 start, u64 size, u32 type);
-void memmap_prepboot(void);
+#include "types.h" // u32
 
 // A typical OS page size
 #define PAGE_SIZE 4096
+#define PAGE_SHIFT 12
+
+static inline u32 virt_to_phys(void *v) {
+    return (u32)v;
+}
+static inline void *memremap(u32 addr, u32 len) {
+    return (void*)addr;
+}
 
-// e820 map storage
-extern struct e820entry e820_list[];
-extern int e820_count;
+// Return the value of a linker script symbol (see scripts/layoutrom.py)
+#define SYMBOL(SYM) ({ extern char SYM; (u32)&SYM; })
+#define VSYMBOL(SYM) ((void*)SYMBOL(SYM))
 
-#endif // e820map.h
+#endif // memmap.h
diff --git a/qemu/roms/seabios/src/misc.c b/qemu/roms/seabios/src/misc.c
index 8caaf31d8..f02237c36 100644
--- a/qemu/roms/seabios/src/misc.c
+++ b/qemu/roms/seabios/src/misc.c
@@ -56,7 +56,7 @@ void VISIBLE16
 handle_10(struct bregs *regs)
 {
     debug_enter(regs, DEBUG_HDL_10);
-    // dont do anything, since the VGA BIOS handles int10h requests
+    // don't do anything, since the VGA BIOS handles int10h requests
 }
 
 // NMI handler
diff --git a/qemu/roms/seabios/src/mouse.c b/qemu/roms/seabios/src/mouse.c
index 6d1f5b77e..b7ad7c62a 100644
--- a/qemu/roms/seabios/src/mouse.c
+++ b/qemu/roms/seabios/src/mouse.c
@@ -280,8 +280,7 @@ invoke_mouse_handler(void)
     if (!CONFIG_MOUSE)
         return;
     if (need_hop_back()) {
-        extern void _cfunc16_invoke_mouse_handler(void);
-        stack_hop_back(0, 0, _cfunc16_invoke_mouse_handler);
+        stack_hop_back(invoke_mouse_handler, 0, 0);
         return;
     }
     ASSERT16();
diff --git a/qemu/roms/seabios/src/optionroms.c b/qemu/roms/seabios/src/optionroms.c
index 93d9d2fe6..c81eff2d2 100644
--- a/qemu/roms/seabios/src/optionroms.c
+++ b/qemu/roms/seabios/src/optionroms.c
@@ -19,6 +19,9 @@
 #include "std/pnpbios.h" // PNP_SIGNATURE
 #include "string.h" // memset
 #include "util.h" // get_pnp_offset
+#include "tcgbios.h" // tpm_*
+
+static int EnforceChecksum, S3ResumeVga, RunPCIroms;
 
 
 /****************************************************************
@@ -60,8 +63,6 @@ call_bcv(u16 seg, u16 ip)
     __callrom(MAKE_FLATPTR(seg, 0), ip, 0);
 }
 
-static int EnforceChecksum;
-
 // Verify that an option rom looks valid
 static int
 is_valid_rom(struct rom_header *rom)
@@ -132,6 +133,8 @@ init_optionrom(struct rom_header *rom, u16 bdf, int isvga)
     if (newrom != rom)
         memmove(newrom, rom, rom->size * 512);
 
+    tpm_option_rom(newrom, rom->size * 512);
+
     if (isvga || get_pnp_rom(newrom))
         // Only init vga and PnP roms here.
         callrom(newrom, bdf);
@@ -180,19 +183,6 @@ deploy_romfile(struct romfile_s *file)
     return rom;
 }
 
-// Check if an option rom is at a hardcoded location or in CBFS.
-static struct rom_header *
-lookup_hardcode(struct pci_device *pci)
-{
-    char fname[17];
-    snprintf(fname, sizeof(fname), "pci%04x,%04x.rom"
-             , pci->vendor, pci->device);
-    struct romfile_s *file = romfile_find(fname);
-    if (file)
-        return deploy_romfile(file);
-    return NULL;
-}
-
 // Run all roms in a given CBFS directory.
 static void
 run_file_roms(const char *prefix, int isvga, u64 *sources)
@@ -321,21 +311,28 @@ fail:
 }
 
 // Attempt to map and initialize the option rom on a given PCI device.
-static int
+static void
 init_pcirom(struct pci_device *pci, int isvga, u64 *sources)
 {
     u16 bdf = pci->bdf;
     dprintf(4, "Attempting to init PCI bdf %02x:%02x.%x (vd %04x:%04x)\n"
             , pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf), pci_bdf_to_fn(bdf)
             , pci->vendor, pci->device);
-    struct rom_header *rom = lookup_hardcode(pci);
-    if (! rom)
+
+    char fname[17];
+    snprintf(fname, sizeof(fname), "pci%04x,%04x.rom"
+             , pci->vendor, pci->device);
+    struct romfile_s *file = romfile_find(fname);
+    struct rom_header *rom = NULL;
+    if (file)
+        rom = deploy_romfile(file);
+    else if (RunPCIroms > 1 || (RunPCIroms == 1 && isvga))
         rom = map_pcirom(pci);
     if (! rom)
         // No ROM present.
-        return -1;
+        return;
     setRomSource(sources, rom, RS_PCIROM | (u32)pci);
-    return init_optionrom(rom, bdf, isvga);
+    init_optionrom(rom, bdf, isvga);
 }
 
 
@@ -416,7 +413,6 @@ optionrom_setup(void)
  * VGA init
  ****************************************************************/
 
-static int S3ResumeVga;
 int ScreenAndDebug;
 struct rom_header *VgaROM;
 
@@ -432,6 +428,7 @@ vgarom_setup(void)
     // Load some config settings that impact VGA.
     EnforceChecksum = romfile_loadint("etc/optionroms-checksum", 1);
     S3ResumeVga = romfile_loadint("etc/s3-resume-vga-init", CONFIG_QEMU);
+    RunPCIroms = romfile_loadint("etc/pci-optionrom-exec", 2);
     ScreenAndDebug = romfile_loadint("etc/screen-and-debug", 1);
 
     if (CONFIG_OPTIONROMS_DEPLOYED) {
diff --git a/qemu/roms/seabios/src/output.c b/qemu/roms/seabios/src/output.c
index 45397b3f6..8a883889c 100644
--- a/qemu/roms/seabios/src/output.c
+++ b/qemu/roms/seabios/src/output.c
@@ -30,6 +30,7 @@ void
 debug_banner(void)
 {
     dprintf(1, "SeaBIOS (version %s)\n", VERSION);
+    dprintf(1, "BUILD: %s\n", BUILDINFO);
 }
 
 // Write a character to debug port(s).
diff --git a/qemu/roms/seabios/src/pmm.c b/qemu/roms/seabios/src/pmm.c
index 304faab2c..640341472 100644
--- a/qemu/roms/seabios/src/pmm.c
+++ b/qemu/roms/seabios/src/pmm.c
@@ -65,26 +65,26 @@ handle_pmm00(u16 *args)
         if (align < MALLOC_MIN_ALIGN)
             align = MALLOC_MIN_ALIGN;
     }
-    void *data;
+    u32 data;
     switch (flags & 3) {
     default:
     case 0:
         return 0;
     case 1:
-        data = _malloc(lowzone, size, align);
+        data = malloc_palloc(lowzone, size, align);
         break;
     case 2:
-        data = _malloc(highzone, size, align);
+        data = malloc_palloc(highzone, size, align);
         break;
     case 3: {
-        data = _malloc(lowzone, size, align);
+        data = malloc_palloc(lowzone, size, align);
         if (!data)
-            data = _malloc(highzone, size, align);
+            data = malloc_palloc(highzone, size, align);
     }
     }
     if (data && handle != MALLOC_DEFAULT_HANDLE)
         malloc_sethandle(data, handle);
-    return (u32)data;
+    return data;
 }
 
 // PMM - find
@@ -95,7 +95,7 @@ handle_pmm01(u16 *args)
     dprintf(3, "pmm01: handle=%x\n", handle);
     if (handle == MALLOC_DEFAULT_HANDLE)
         return 0;
-    return (u32)malloc_findhandle(handle);
+    return malloc_findhandle(handle);
 }
 
 // PMM - deallocate
@@ -104,7 +104,7 @@ handle_pmm02(u16 *args)
 {
     u32 buffer = *(u32*)&args[1];
     dprintf(3, "pmm02: buffer=%x\n", buffer);
-    int ret = _free((void*)buffer);
+    int ret = malloc_pfree(buffer);
     if (ret)
         // Error
         return 1;
diff --git a/qemu/roms/seabios/src/post.c b/qemu/roms/seabios/src/post.c
index 9ea5620c9..49c22b875 100644
--- a/qemu/roms/seabios/src/post.c
+++ b/qemu/roms/seabios/src/post.c
@@ -8,6 +8,7 @@
 #include "biosvar.h" // SET_BDA
 #include "bregs.h" // struct bregs
 #include "config.h" // CONFIG_*
+#include "e820map.h" // e820_add
 #include "fw/paravirt.h" // qemu_cfg_preinit
 #include "fw/xen.h" // xen_preinit
 #include "hw/ahci.h" // ahci_setup
@@ -24,10 +25,11 @@
 #include "hw/virtio-blk.h" // virtio_blk_setup
 #include "hw/virtio-scsi.h" // virtio_scsi_setup
 #include "malloc.h" // malloc_init
-#include "memmap.h" // add_e820
+#include "memmap.h" // SYMBOL
 #include "output.h" // dprintf
 #include "string.h" // memset
 #include "util.h" // kbd_init
+#include "tcgbios.h" // tpm_*
 
 
 /****************************************************************
@@ -88,9 +90,8 @@ bda_init(void)
 
     int esize = EBDA_SIZE_START;
     u16 ebda_seg = EBDA_SEGMENT_START;
-    extern u8 final_varlow_start[];
     if (!CONFIG_MALLOC_UPPERMEMORY)
-        ebda_seg = FLATPTR_TO_SEG(ALIGN_DOWN((u32)final_varlow_start, 1024)
+        ebda_seg = FLATPTR_TO_SEG(ALIGN_DOWN(SYMBOL(final_varlow_start), 1024)
                                   - EBDA_SIZE_START*1024);
     SET_BDA(ebda_seg, ebda_seg);
 
@@ -101,10 +102,10 @@ bda_init(void)
     memset(ebda, 0, sizeof(*ebda));
     ebda->size = esize;
 
-    add_e820((u32)ebda, BUILD_LOWRAM_END-(u32)ebda, E820_RESERVED);
+    e820_add((u32)ebda, BUILD_LOWRAM_END-(u32)ebda, E820_RESERVED);
 
     // Init extra stack
-    StackPos = (void*)(&ExtraStack[BUILD_EXTRA_STACK_SIZE] - zonelow_base);
+    StackPos = &ExtraStack[BUILD_EXTRA_STACK_SIZE] - SYMBOL(zonelow_base);
 }
 
 void
@@ -116,13 +117,13 @@ interface_init(void)
     // Setup romfile items.
     qemu_cfg_init();
     coreboot_cbfs_init();
+    multiboot_init();
 
     // Setup ivt/bda/ebda
     ivt_init();
     bda_init();
 
     // Other interfaces
-    thread_init();
     boot_init();
     bios32_init();
     pmm_init();
@@ -157,26 +158,32 @@ device_hardware_setup(void)
 static void
 platform_hardware_setup(void)
 {
-    // Enable CPU caching
-    setcr0(getcr0() & ~(CR0_CD|CR0_NW));
-
     // Make sure legacy DMA isn't running.
     dma_setup();
 
     // Init base pc hardware.
     pic_setup();
+    thread_setup();
     mathcp_setup();
-    timer_setup();
-    clock_setup();
 
     // Platform specific setup
     qemu_platform_setup();
     coreboot_platform_setup();
+
+    // Setup timers and periodic clock interrupt
+    timer_setup();
+    clock_setup();
+
+    // Initialize TPM
+    tpm_setup();
 }
 
 void
 prepareboot(void)
 {
+    // Change TPM phys. presence state befor leaving BIOS
+    tpm_prepboot();
+
     // Run BCVs
     bcv_prepboot();
 
@@ -184,7 +191,7 @@ prepareboot(void)
     cdrom_prepboot();
     pmm_prepboot();
     malloc_prepboot();
-    memmap_prepboot();
+    e820_prepboot();
 
     HaveRunPost = 2;
 
@@ -269,30 +276,27 @@ reloc_preinit(void *f, void *arg)
     void (*func)(void *) __noreturn = f;
     if (!CONFIG_RELOCATE_INIT)
         func(arg);
-    // Symbols populated by the build.
-    extern u8 code32flat_start[];
-    extern u8 _reloc_min_align;
-    extern u32 _reloc_abs_start[], _reloc_abs_end[];
-    extern u32 _reloc_rel_start[], _reloc_rel_end[];
-    extern u32 _reloc_init_start[], _reloc_init_end[];
-    extern u8 code32init_start[], code32init_end[];
 
     // Allocate space for init code.
-    u32 initsize = code32init_end - code32init_start;
-    u32 codealign = (u32)&_reloc_min_align;
+    u32 initsize = SYMBOL(code32init_end) - SYMBOL(code32init_start);
+    u32 codealign = SYMBOL(_reloc_min_align);
     void *codedest = memalign_tmp(codealign, initsize);
+    void *codesrc = VSYMBOL(code32init_start);
     if (!codedest)
         panic("No space for init relocation.\n");
 
     // Copy code and update relocs (init absolute, init relative, and runtime)
     dprintf(1, "Relocating init from %p to %p (size %d)\n"
-            , code32init_start, codedest, initsize);
-    s32 delta = codedest - (void*)code32init_start;
-    memcpy(codedest, code32init_start, initsize);
-    updateRelocs(codedest, _reloc_abs_start, _reloc_abs_end, delta);
-    updateRelocs(codedest, _reloc_rel_start, _reloc_rel_end, -delta);
-    updateRelocs(code32flat_start, _reloc_init_start, _reloc_init_end, delta);
-    if (f >= (void*)code32init_start && f < (void*)code32init_end)
+            , codesrc, codedest, initsize);
+    s32 delta = codedest - codesrc;
+    memcpy(codedest, codesrc, initsize);
+    updateRelocs(codedest, VSYMBOL(_reloc_abs_start), VSYMBOL(_reloc_abs_end)
+                 , delta);
+    updateRelocs(codedest, VSYMBOL(_reloc_rel_start), VSYMBOL(_reloc_rel_end)
+                 , -delta);
+    updateRelocs(VSYMBOL(code32flat_start), VSYMBOL(_reloc_init_start)
+                 , VSYMBOL(_reloc_init_end), delta);
+    if (f >= codesrc && f < VSYMBOL(code32init_end))
         func = f + delta;
 
     // Call function in relocated code.
diff --git a/qemu/roms/seabios/src/resume.c b/qemu/roms/seabios/src/resume.c
index 19031747c..a5465d877 100644
--- a/qemu/roms/seabios/src/resume.c
+++ b/qemu/roms/seabios/src/resume.c
@@ -16,6 +16,7 @@
 #include "std/bda.h" // struct bios_data_area_s
 #include "string.h" // memset
 #include "util.h" // dma_setup
+#include "tcgbios.h" // tpm_s3_resume
 
 // Handler for post calls that look like a resume.
 void VISIBLE16
@@ -99,6 +100,8 @@ s3_resume(void)
 
     pci_resume();
 
+    /* resume TPM before we may measure option roms */
+    tpm_s3_resume();
     s3_resume_vga();
 
     make_bios_readonly();
diff --git a/qemu/roms/seabios/src/romlayout.S b/qemu/roms/seabios/src/romlayout.S
index 93b6874e7..53cc0f5e3 100644
--- a/qemu/roms/seabios/src/romlayout.S
+++ b/qemu/roms/seabios/src/romlayout.S
@@ -22,18 +22,14 @@
 // %edx = return location (in 32bit mode)
 // Clobbers: ecx, flags, segment registers, cr0, idt/gdt
         DECLFUNC transition32
-transition32_nmi_off:
-        // transition32 when NMI and A20 are already initialized
-        movl %eax, %ecx
-        jmp 1f
+        .global transition32_nmi_off
 transition32:
-        movl %eax, %ecx
-
         // Disable irqs (and clear direction flag)
         cli
         cld
 
         // Disable nmi
+        movl %eax, %ecx
         movl $CMOS_RESET_CODE|NMI_DISABLE_BIT, %eax
         outb %al, $PORT_CMOS_INDEX
         inb $PORT_CMOS_DATA, %al
@@ -42,29 +38,31 @@ transition32:
         inb $PORT_A20, %al
         orb $A20_ENABLE_BIT, %al
         outb %al, $PORT_A20
+        movl %ecx, %eax
 
+transition32_nmi_off:
         // Set segment descriptors
-1:      lidtw %cs:pmode_IDT_info
+        lidtw %cs:pmode_IDT_info
         lgdtw %cs:rombios32_gdt_48
 
         // Enable protected mode
-        movl %cr0, %eax
-        orl $CR0_PE, %eax
-        movl %eax, %cr0
+        movl %cr0, %ecx
+        andl $~(CR0_PG|CR0_CD|CR0_NW), %ecx
+        orl $CR0_PE, %ecx
+        movl %ecx, %cr0
 
         // start 32bit protected mode code
-        ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 2f)
+        ljmpl $SEG32_MODE32_CS, $(BUILD_BIOS_ADDR + 1f)
 
         .code32
         // init data segments
-2:      movl $SEG32_MODE32_DS, %eax
-        movw %ax, %ds
-        movw %ax, %es
-        movw %ax, %ss
-        movw %ax, %fs
-        movw %ax, %gs
+1:      movl $SEG32_MODE32_DS, %ecx
+        movw %cx, %ds
+        movw %cx, %es
+        movw %cx, %ss
+        movw %cx, %fs
+        movw %cx, %gs
 
-        movl %ecx, %eax
         jmpl *%edx
         .code16
 
@@ -75,61 +73,47 @@ transition32:
         .global transition16big
         .code32
 transition16:
-        movl %eax, %ecx
-
-        // restore data segment limits to 0xffff
-        movl $SEG32_MODE16_DS, %eax
-        movw %ax, %ds
-        movw %ax, %es
-        movw %ax, %ss
-        movw %ax, %fs
-        movw %ax, %gs
-
-#if CONFIG_DISABLE_A20
-        // disable a20
-        inb $PORT_A20, %al
-        andb $~A20_ENABLE_BIT, %al
-        outb %al, $PORT_A20
-#endif
+        // Reset data segment limits
+        movl $SEG32_MODE16_DS, %ecx
+        movw %cx, %ds
+        movw %cx, %es
+        movw %cx, %ss
+        movw %cx, %fs
+        movw %cx, %gs
 
         // Jump to 16bit mode
         ljmpw $SEG32_MODE16_CS, $1f
 
 transition16big:
-        movl %eax, %ecx
-
-        movl $SEG32_MODE16BIG_DS, %eax
-        movw %ax, %ds
-        movw %ax, %es
-        movw %ax, %ss
-        movw %ax, %fs
-        movw %ax, %gs
+        movl $SEG32_MODE16BIG_DS, %ecx
+        movw %cx, %ds
+        movw %cx, %es
+        movw %cx, %ss
+        movw %cx, %fs
+        movw %cx, %gs
 
         ljmpw $SEG32_MODE16BIG_CS, $1f
 
         .code16
-1:
         // Disable protected mode
-        movl %cr0, %eax
-        andl $~CR0_PE, %eax
-        movl %eax, %cr0
+1:      movl %cr0, %ecx
+        andl $~CR0_PE, %ecx
+        movl %ecx, %cr0
 
         // far jump to flush CPU queue after transition to real mode
         ljmpw $SEG_BIOS, $2f
 
-2:
         // restore IDT to normal real-mode defaults
-        lidtw %cs:rmode_IDT_info
+2:      lidtw %cs:rmode_IDT_info
 
         // Clear segment registers
-        xorw %ax, %ax
-        movw %ax, %fs
-        movw %ax, %gs
-        movw %ax, %es
-        movw %ax, %ds
-        movw %ax, %ss  // Assume stack is in segment 0
+        xorw %cx, %cx
+        movw %cx, %fs
+        movw %cx, %gs
+        movw %cx, %es
+        movw %cx, %ds
+        movw %cx, %ss  // Assume stack is in segment 0
 
-        movl %ecx, %eax
         jmpl *%edx
 
 
@@ -264,7 +248,7 @@ entry_pmm:
         movl $_cfunc32flat_handle_pmm, %eax // Setup: call32(handle_pmm, args, -1)
         leal PUSHBREGS_size+12(%esp, %ecx), %edx // %edx points to start of args
         movl $-1, %ecx
-        calll call32
+        calll __call32
         movw %ax, BREGS_eax(%esp)       // Modify %ax:%dx to return %eax
         shrl $16, %eax
         movw %ax, BREGS_edx(%esp)
@@ -374,6 +358,8 @@ entry_bios32:
 entry_elf:
         cli
         cld
+        movl %eax, entry_elf_eax
+        movl %ebx, entry_elf_ebx
         lidtl (BUILD_BIOS_ADDR + pmode_IDT_info)
         lgdtl (BUILD_BIOS_ADDR + rombios32_gdt_48)
         movl $SEG32_MODE32_DS, %eax
@@ -562,7 +548,10 @@ entry_post:
         ENTRY_INTO32 _cfunc32flat_handle_post   // Normal entry point
 
         ORG 0xe2c3
-        IRQ_ENTRY 02
+        .global entry_02
+entry_02:
+        ENTRY handle_02  // NMI handler does not switch onto extra stack
+        iretw
 
         ORG 0xe3fe
         .global entry_13_official
diff --git a/qemu/roms/seabios/src/sha1.c b/qemu/roms/seabios/src/sha1.c
new file mode 100644
index 000000000..2ecb3cb89
--- /dev/null
+++ b/qemu/roms/seabios/src/sha1.c
@@ -0,0 +1,147 @@
+//  Support for Calculation of SHA1 in SW
+//
+//  Copyright (C) 2006-2011 IBM Corporation
+//
+//  Authors:
+//      Stefan Berger <stefanb@linux.vnet.ibm.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+//
+//  See: http://www.itl.nist.gov/fipspubs/fip180-1.htm
+//       RFC3174, Wikipedia's SHA1 alogrithm description
+//
+
+#include "config.h"
+#include "byteorder.h" // cpu_to_*, __swab64
+#include "sha1.h" // sha1
+#include "string.h" // memcpy
+#include "x86.h" // rol
+
+typedef struct _sha1_ctx {
+    u32 h[5];
+} sha1_ctx;
+
+
+static void
+sha1_block(u32 *w, sha1_ctx *ctx)
+{
+    u32 i;
+    u32 a,b,c,d,e,f;
+    u32 tmp;
+    u32 idx;
+
+    static const u32 sha_ko[4] = {
+        0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 };
+
+    /* change endianness of given data */
+    for (i = 0; i < 16; i++)
+        w[i] = be32_to_cpu(w[i]);
+
+    for (i = 16; i <= 79; i++) {
+        tmp = w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16];
+        w[i] = rol(tmp,1);
+    }
+
+    a = ctx->h[0];
+    b = ctx->h[1];
+    c = ctx->h[2];
+    d = ctx->h[3];
+    e = ctx->h[4];
+
+    for (i = 0; i <= 79; i++) {
+        if (i <= 19) {
+            f = (b & c) | ((b ^ 0xffffffff) & d);
+            idx = 0;
+        } else if (i <= 39) {
+            f = b ^ c ^ d;
+            idx = 1;
+        } else if (i <= 59) {
+            f = (b & c) | (b & d) | (c & d);
+            idx = 2;
+        } else {
+            f = b ^ c ^ d;
+            idx = 3;
+        }
+
+        tmp = rol(a, 5) +
+              f +
+              e +
+              sha_ko[idx] +
+              w[i];
+        e = d;
+        d = c;
+        c = rol(b, 30);
+        b = a;
+        a = tmp;
+    }
+
+    ctx->h[0] += a;
+    ctx->h[1] += b;
+    ctx->h[2] += c;
+    ctx->h[3] += d;
+    ctx->h[4] += e;
+}
+
+
+static void
+sha1_do(sha1_ctx *ctx, const u8 *data32, u32 length)
+{
+    u32 offset;
+    u16 num;
+    u32 bits = 0;
+    u32 w[80];
+    u64 tmp;
+
+    /* treat data in 64-byte chunks */
+    for (offset = 0; length - offset >= 64; offset += 64) {
+        memcpy(w, data32 + offset, 64);
+        sha1_block((u32 *)w, ctx);
+        bits += (64 * 8);
+    }
+
+    /* last block with less than 64 bytes */
+    num = length - offset;
+    bits += (num << 3);
+
+    memcpy(w, data32 + offset, num);
+    ((u8 *)w)[num] = 0x80;
+    if (64 - (num + 1) > 0)
+        memset( &((u8 *)w)[num + 1], 0x0, 64 - (num + 1));
+
+    if (num >= 56) {
+        /* cannot append number of bits here */
+        sha1_block((u32 *)w, ctx);
+        memset(w, 0x0, 60);
+    }
+
+    /* write number of bits to end of block */
+    tmp = __swab64(bits);
+    memcpy(&w[14], &tmp, 8);
+
+    sha1_block(w, ctx);
+
+    /* need to switch result's endianness */
+    for (num = 0; num < 5; num++)
+        ctx->h[num] = cpu_to_be32(ctx->h[num]);
+}
+
+
+u32
+sha1(const u8 *data, u32 length, u8 *hash)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    sha1_ctx ctx = {
+        .h[0] = 0x67452301,
+        .h[1] = 0xefcdab89,
+        .h[2] = 0x98badcfe,
+        .h[3] = 0x10325476,
+        .h[4] = 0xc3d2e1f0,
+    };
+
+    sha1_do(&ctx, data, length);
+    memcpy(hash, &ctx.h[0], 20);
+
+    return 0;
+}
diff --git a/qemu/roms/seabios/src/sha1.h b/qemu/roms/seabios/src/sha1.h
new file mode 100644
index 000000000..07aabf34f
--- /dev/null
+++ b/qemu/roms/seabios/src/sha1.h
@@ -0,0 +1,8 @@
+#ifndef __SHA1_H
+#define __SHA1_H
+
+#include "types.h" // u32
+
+u32 sha1(const u8 *data, u32 length, u8 *hash);
+
+#endif // sha1.h
diff --git a/qemu/roms/seabios/src/stacks.c b/qemu/roms/seabios/src/stacks.c
index 1dbdfe9bb..ef6a70775 100644
--- a/qemu/roms/seabios/src/stacks.c
+++ b/qemu/roms/seabios/src/stacks.c
@@ -1,6 +1,6 @@
 // Code for manipulating stack locations.
 //
-// Copyright (C) 2009-2014  Kevin O'Connor <kevin@koconnor.net>
+// Copyright (C) 2009-2015  Kevin O'Connor <kevin@koconnor.net>
 //
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
@@ -13,6 +13,7 @@
 #include "output.h" // dprintf
 #include "romfile.h" // romfile_loadint
 #include "stacks.h" // struct mutex_s
+#include "string.h" // memset
 #include "util.h" // useRTC
 
 #define MAIN_STACK_MAX (1024*1024)
@@ -27,40 +28,108 @@ struct {
     u8 cmosindex;
     u8 a20;
     u16 ss, fs, gs;
+    u32 cr0;
     struct descloc_s gdt;
-} Call32Data VARLOW;
+} Call16Data VARLOW;
 
-#define C32_SLOPPY 1
-#define C32_SMM    2
+#define C16_BIG 1
+#define C16_SMM 2
 
 int HaveSmmCall32 VARFSEG;
 
-// Backup state in preparation for call32_smm()
-static void
-call32_smm_prep(void)
-{
+// Backup state in preparation for call32
+static int
+call32_prep(u8 method)
+{
+    if (!CONFIG_CALL32_SMM || method != C16_SMM) {
+        // Backup cr0
+        u32 cr0 = cr0_read();
+        if (cr0 & CR0_PE)
+            // Called in 16bit protected mode?!
+            return -1;
+        SET_LOW(Call16Data.cr0, cr0);
+
+        // Backup fs/gs and gdt
+        SET_LOW(Call16Data.fs, GET_SEG(FS));
+        SET_LOW(Call16Data.gs, GET_SEG(GS));
+        struct descloc_s gdt;
+        sgdt(&gdt);
+        SET_LOW(Call16Data.gdt.length, gdt.length);
+        SET_LOW(Call16Data.gdt.addr, gdt.addr);
+
+        // Enable a20 and backup its previous state
+        SET_LOW(Call16Data.a20, set_a20(1));
+    }
+
+    // Backup ss
+    SET_LOW(Call16Data.ss, GET_SEG(SS));
+
     // Backup cmos index register and disable nmi
     u8 cmosindex = inb(PORT_CMOS_INDEX);
     outb(cmosindex | NMI_DISABLE_BIT, PORT_CMOS_INDEX);
     inb(PORT_CMOS_DATA);
-    SET_LOW(Call32Data.cmosindex, cmosindex);
-
-    // Backup ss
-    SET_LOW(Call32Data.ss, GET_SEG(SS));
+    SET_LOW(Call16Data.cmosindex, cmosindex);
 
-    SET_LOW(Call32Data.method, C32_SMM);
+    SET_LOW(Call16Data.method, method);
+    return 0;
 }
 
-// Restore state backed up during call32_smm()
-static void
-call32_smm_post(void)
+// Restore state backed up during call32
+static u8
+call32_post(void)
 {
-    SET_LOW(Call32Data.method, 0);
-    SET_LOW(Call32Data.ss, 0);
+    u8 method = GET_LOW(Call16Data.method);
+    SET_LOW(Call16Data.method, 0);
+    SET_LOW(Call16Data.ss, 0);
+
+    if (!CONFIG_CALL32_SMM || method != C16_SMM) {
+        // Restore a20
+        set_a20(GET_LOW(Call16Data.a20));
+
+        // Restore gdt and fs/gs
+        struct descloc_s gdt;
+        gdt.length = GET_LOW(Call16Data.gdt.length);
+        gdt.addr = GET_LOW(Call16Data.gdt.addr);
+        lgdt(&gdt);
+        SET_SEG(FS, GET_LOW(Call16Data.fs));
+        SET_SEG(GS, GET_LOW(Call16Data.gs));
+
+        // Restore cr0
+        u32 cr0_caching = GET_LOW(Call16Data.cr0) & (CR0_CD|CR0_NW);
+        if (cr0_caching)
+            cr0_mask(CR0_CD|CR0_NW, cr0_caching);
+    }
 
     // Restore cmos index register
-    outb(GET_LOW(Call32Data.cmosindex), PORT_CMOS_INDEX);
+    outb(GET_LOW(Call16Data.cmosindex), PORT_CMOS_INDEX);
     inb(PORT_CMOS_DATA);
+    return method;
+}
+
+// Force next call16() to restore to a pristine cpu environment state
+static void
+call16_override(int big)
+{
+    ASSERT32FLAT();
+    if (getesp() > BUILD_STACK_ADDR)
+        panic("call16_override with invalid stack\n");
+    memset(&Call16Data, 0, sizeof(Call16Data));
+    if (big) {
+        Call16Data.method = C16_BIG;
+        Call16Data.a20 = 1;
+    } else {
+        Call16Data.a20 = !CONFIG_DISABLE_A20;
+    }
+}
+
+// 16bit handler code called from call16() / call16_smm()
+u32 VISIBLE16
+call16_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
+{
+    u8 method = call32_post();
+    u32 ret = func(eax, edx);
+    call32_prep(method);
+    return ret;
 }
 
 #define ASM32_SWITCH16 "  .pushsection .text.32fseg." UNIQSEC "\n  .code16\n"
@@ -74,7 +143,7 @@ call32_smm(void *func, u32 eax)
 {
     ASSERT16();
     dprintf(9, "call32_smm %p %x\n", func, eax);
-    call32_smm_prep();
+    call32_prep(C16_SMM);
     u32 bkup_esp;
     asm volatile(
         // Backup esp / set esp to flat stack location
@@ -109,24 +178,12 @@ call32_smm(void *func, u32 eax)
         : "=&r" (bkup_esp), "+r" (eax)
         : "r" (func)
         : "eax", "ecx", "edx", "ebx", "cc", "memory");
-    call32_smm_post();
+    call32_post();
 
     dprintf(9, "call32_smm done %p %x\n", func, eax);
     return eax;
 }
 
-// 16bit handler code called from call16_smm()
-u32 VISIBLE16
-call16_smm_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
-{
-    if (!CONFIG_CALL32_SMM)
-        return eax;
-    call32_smm_post();
-    u32 ret = func(eax, edx);
-    call32_smm_prep();
-    return ret;
-}
-
 static u32
 call16_smm(u32 eax, u32 edx, void *func)
 {
@@ -135,7 +192,7 @@ call16_smm(u32 eax, u32 edx, void *func)
         return eax;
     func -= BUILD_BIOS_ADDR;
     dprintf(9, "call16_smm %p %x %x\n", func, eax, edx);
-    u32 stackoffset = Call32Data.ss << 4;
+    u32 stackoffset = Call16Data.ss << 4;
     asm volatile(
         // Restore esp
         "  subl %0, %%esp\n"
@@ -151,7 +208,7 @@ call16_smm(u32 eax, u32 edx, void *func)
         ASM32_SWITCH16
         "1:movl %1, %%eax\n"
         "  movl %3, %%ecx\n"
-        "  calll _cfunc16_call16_smm_helper\n"
+        "  calll _cfunc16_call16_helper\n"
         "  movl %%eax, %1\n"
 
         "  movl $" __stringify(CALL32SMM_CMDID) ", %%eax\n"
@@ -170,61 +227,18 @@ call16_smm(u32 eax, u32 edx, void *func)
     return eax;
 }
 
-// Backup state in preparation for call32_sloppy()
-static void
-call32_sloppy_prep(void)
-{
-    // Backup cmos index register and disable nmi
-    u8 cmosindex = inb(PORT_CMOS_INDEX);
-    outb(cmosindex | NMI_DISABLE_BIT, PORT_CMOS_INDEX);
-    inb(PORT_CMOS_DATA);
-    SET_LOW(Call32Data.cmosindex, cmosindex);
-
-    // Enable a20 and backup it's previous state
-    SET_LOW(Call32Data.a20, set_a20(1));
-
-    // Backup ss/fs/gs and gdt
-    SET_LOW(Call32Data.ss, GET_SEG(SS));
-    SET_LOW(Call32Data.fs, GET_SEG(FS));
-    SET_LOW(Call32Data.gs, GET_SEG(GS));
-    struct descloc_s gdt;
-    sgdt(&gdt);
-    SET_LOW(Call32Data.gdt.length, gdt.length);
-    SET_LOW(Call32Data.gdt.addr, gdt.addr);
-
-    SET_LOW(Call32Data.method, C32_SLOPPY);
-}
-
-// Restore state backed up during call32_sloppy()
-static void
-call32_sloppy_post(void)
-{
-    SET_LOW(Call32Data.method, 0);
-    SET_LOW(Call32Data.ss, 0);
-
-    // Restore gdt and fs/gs
-    struct descloc_s gdt;
-    gdt.length = GET_LOW(Call32Data.gdt.length);
-    gdt.addr = GET_LOW(Call32Data.gdt.addr);
-    lgdt(&gdt);
-    SET_SEG(FS, GET_LOW(Call32Data.fs));
-    SET_SEG(GS, GET_LOW(Call32Data.gs));
-
-    // Restore a20
-    set_a20(GET_LOW(Call32Data.a20));
-
-    // Restore cmos index register
-    outb(GET_LOW(Call32Data.cmosindex), PORT_CMOS_INDEX);
-    inb(PORT_CMOS_DATA);
-}
-
-// Call a C function in 32bit mode.  This clobbers the 16bit segment
-// selector registers.
-static u32
-call32_sloppy(void *func, u32 eax)
+// Call a 32bit SeaBIOS function from a 16bit SeaBIOS function.
+u32 VISIBLE16
+__call32(void *func, u32 eax, u32 errret)
 {
     ASSERT16();
-    call32_sloppy_prep();
+    if (CONFIG_CALL32_SMM && GET_GLOBAL(HaveSmmCall32))
+        return call32_smm(func, eax);
+    // Jump direclty to 32bit mode - this clobbers the 16bit segment
+    // selector registers.
+    int ret = call32_prep(C16_BIG);
+    if (ret)
+        return errret;
     u32 bkup_ss, bkup_esp;
     asm volatile(
         // Backup ss/esp / set esp to flat stack location
@@ -236,7 +250,7 @@ call32_sloppy(void *func, u32 eax)
 
         // Transition to 32bit mode, call func, return to 16bit
         "  movl $(" __stringify(BUILD_BIOS_ADDR) " + 1f), %%edx\n"
-        "  jmp transition32\n"
+        "  jmp transition32_nmi_off\n"
         ASM16_SWITCH32
         "1:calll *%3\n"
         "  movl $2f, %%edx\n"
@@ -250,136 +264,52 @@ call32_sloppy(void *func, u32 eax)
         : "=&r" (bkup_ss), "=&r" (bkup_esp), "+a" (eax)
         : "r" (func)
         : "ecx", "edx", "cc", "memory");
-    call32_sloppy_post();
+    call32_post();
     return eax;
 }
 
-// 16bit handler code called from call16_sloppy()
-u32 VISIBLE16
-call16_sloppy_helper(u32 eax, u32 edx, u32 (*func)(u32 eax, u32 edx))
-{
-    call32_sloppy_post();
-    u32 ret = func(eax, edx);
-    call32_sloppy_prep();
-    return ret;
-}
-
-// Jump back to 16bit mode while in 32bit mode from call32_sloppy()
+// Call a 16bit SeaBIOS function, restoring the mode from last call32().
 static u32
-call16_sloppy(u32 eax, u32 edx, void *func)
+call16(u32 eax, u32 edx, void *func)
 {
     ASSERT32FLAT();
     if (getesp() > MAIN_STACK_MAX)
-        panic("call16_sloppy with invalid stack\n");
+        panic("call16 with invalid stack\n");
+    if (CONFIG_CALL32_SMM && Call16Data.method == C16_SMM)
+        return call16_smm(eax, edx, func);
+
+    extern void transition16big(void);
+    extern void transition16(void);
+    void *thunk = transition16;
+    if (Call16Data.method == C16_BIG || in_post())
+        thunk = transition16big;
     func -= BUILD_BIOS_ADDR;
-    u32 stackseg = Call32Data.ss;
+    u32 stackseg = Call16Data.ss;
     asm volatile(
         // Transition to 16bit mode
         "  movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
-        "  jmp transition16big\n"
+        "  jmp *%%ecx\n"
         // Setup ss/esp and call func
         ASM32_SWITCH16
-        "1:movl %3, %%ecx\n"
-        "  shll $4, %3\n"
+        "1:movl %2, %%ecx\n"
+        "  shll $4, %2\n"
         "  movw %%cx, %%ss\n"
-        "  subl %3, %%esp\n"
+        "  subl %2, %%esp\n"
         "  movw %%cx, %%ds\n"
-        "  movl %2, %%edx\n"
-        "  movl %1, %%ecx\n"
-        "  calll _cfunc16_call16_sloppy_helper\n"
+        "  movl %4, %%edx\n"
+        "  movl %3, %%ecx\n"
+        "  calll _cfunc16_call16_helper\n"
         // Return to 32bit and restore esp
         "  movl $2f, %%edx\n"
-        "  jmp transition32\n"
-        ASM32_BACK32
-        "2:addl %3, %%esp\n"
-        : "+a" (eax)
-        : "r" (func), "r" (edx), "r" (stackseg)
-        : "edx", "ecx", "cc", "memory");
-    return eax;
-}
-
-// Call a 32bit SeaBIOS function from a 16bit SeaBIOS function.
-u32 VISIBLE16
-call32(void *func, u32 eax, u32 errret)
-{
-    ASSERT16();
-    if (CONFIG_CALL32_SMM && GET_GLOBAL(HaveSmmCall32))
-        return call32_smm(func, eax);
-    u32 cr0 = getcr0();
-    if (cr0 & CR0_PE)
-        // Called in 16bit protected mode?!
-        return errret;
-    return call32_sloppy(func, eax);
-}
-
-// Call a 16bit SeaBIOS function from a 32bit SeaBIOS function.
-static u32
-call16(u32 eax, u32 edx, void *func)
-{
-    ASSERT32FLAT();
-    if (getesp() > BUILD_STACK_ADDR)
-        panic("call16 with invalid stack\n");
-    func -= BUILD_BIOS_ADDR;
-    asm volatile(
-        // Transition to 16bit mode
-        "  movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
-        "  jmp transition16\n"
-        // Call func
-        ASM32_SWITCH16
-        "1:movl %2, %%edx\n"
-        "  calll *%1\n"
-        // Return to 32bit
-        "  movl $2f, %%edx\n"
-        "  jmp transition32\n"
-        ASM32_BACK32
-        "2:\n"
-        : "+a" (eax)
-        : "r" (func), "r" (edx)
-        : "edx", "ecx", "cc", "memory");
-    return eax;
-}
-
-// Call a 16bit SeaBIOS function in "big real" mode.
-static u32
-call16big(u32 eax, u32 edx, void *func)
-{
-    ASSERT32FLAT();
-    if (getesp() > BUILD_STACK_ADDR)
-        panic("call16big with invalid stack\n");
-    func -= BUILD_BIOS_ADDR;
-    asm volatile(
-        // Transition to 16bit mode
-        "  movl $(1f - " __stringify(BUILD_BIOS_ADDR) "), %%edx\n"
-        "  jmp transition16big\n"
-        // Call func
-        ASM32_SWITCH16
-        "1:movl %2, %%edx\n"
-        "  calll *%1\n"
-        // Return to 32bit
-        "  movl $2f, %%edx\n"
-        "  jmp transition32\n"
+        "  jmp transition32_nmi_off\n"
         ASM32_BACK32
-        "2:\n"
-        : "+a" (eax)
+        "2:addl %2, %%esp\n"
+        : "+a" (eax), "+c"(thunk), "+r"(stackseg)
         : "r" (func), "r" (edx)
-        : "edx", "ecx", "cc", "memory");
+        : "edx", "cc", "memory");
     return eax;
 }
 
-// Call a 16bit SeaBIOS function, restoring the mode from last call32().
-static u32
-call16_back(u32 eax, u32 edx, void *func)
-{
-    ASSERT32FLAT();
-    if (CONFIG_CALL32_SMM && Call32Data.method == C32_SMM)
-        return call16_smm(eax, edx, func);
-    if (Call32Data.method == C32_SLOPPY)
-        return call16_sloppy(eax, edx, func);
-    if (in_post())
-        return call16big(eax, edx, func);
-    return call16(eax, edx, func);
-}
-
 
 /****************************************************************
  * Extra 16bit stack
@@ -398,7 +328,7 @@ on_extra_stack(void)
 
 // Switch to the extra stack and call a function.
 u32
-stack_hop(u32 eax, u32 edx, void *func)
+__stack_hop(u32 eax, u32 edx, void *func)
 {
     if (on_extra_stack())
         return ((u32 (*)(u32, u32))func)(eax, edx);
@@ -431,10 +361,10 @@ stack_hop(u32 eax, u32 edx, void *func)
 
 // Switch back to original caller's stack and call a function.
 u32
-stack_hop_back(u32 eax, u32 edx, void *func)
+__stack_hop_back(u32 eax, u32 edx, void *func)
 {
     if (!MODESEGMENT)
-        return call16_back(eax, edx, func);
+        return call16(eax, edx, func);
     if (!MODE16 || !on_extra_stack())
         return ((u32 (*)(u32, u32))func)(eax, edx);
     ASSERT16();
@@ -474,8 +404,7 @@ void VISIBLE16
 _farcall16(struct bregs *callregs, u16 callregseg)
 {
     if (need_hop_back()) {
-        extern void _cfunc16__farcall16(void);
-        stack_hop_back((u32)callregs, callregseg, _cfunc16__farcall16);
+        stack_hop_back(_farcall16, callregs, callregseg);
         return;
     }
     ASSERT16();
@@ -486,18 +415,20 @@ _farcall16(struct bregs *callregs, u16 callregseg)
         : "ebx", "ecx", "esi", "edi", "cc", "memory");
 }
 
+// Invoke external 16bit code.
 void
 farcall16(struct bregs *callregs)
 {
-    extern void _cfunc16__farcall16(void);
-    call16((u32)callregs, 0, _cfunc16__farcall16);
+    call16_override(0);
+    _farcall16(callregs, 0);
 }
 
+// Invoke external 16bit code in "big real" mode.
 void
 farcall16big(struct bregs *callregs)
 {
-    extern void _cfunc16__farcall16(void);
-    call16big((u32)callregs, 0, _cfunc16__farcall16);
+    call16_override(1);
+    _farcall16(callregs, 0);
 }
 
 // Invoke a 16bit software interrupt.
@@ -507,7 +438,7 @@ __call16_int(struct bregs *callregs, u16 offset)
     callregs->code.offset = offset;
     if (!MODESEGMENT) {
         callregs->code.seg = SEG_BIOS;
-        _farcall16((void*)callregs - Call32Data.ss * 16, Call32Data.ss);
+        _farcall16((void*)callregs - Call16Data.ss * 16, Call16Data.ss);
         return;
     }
     callregs->code.seg = GET_SEG(CS);
@@ -520,7 +451,7 @@ reset(void)
 {
     extern void reset_vector(void) __noreturn;
     if (!MODE16)
-        call16_back(0, 0, reset_vector);
+        call16(0, 0, reset_vector);
     reset_vector();
 }
 
@@ -558,12 +489,13 @@ getCurThread(void)
     return (void*)ALIGN_DOWN(esp, THREADSTACKSIZE);
 }
 
-static int ThreadControl;
+static u8 CanInterrupt, ThreadControl;
 
 // Initialize the support for internal threads.
 void
-thread_init(void)
+thread_setup(void)
 {
+    CanInterrupt = 1;
     if (! CONFIG_THREADS)
         return;
     ThreadControl = romfile_loadint("etc/threads", 1);
@@ -573,7 +505,7 @@ thread_init(void)
 int
 threads_during_optionroms(void)
 {
-    return CONFIG_THREADS && ThreadControl == 2 && in_post();
+    return CONFIG_THREADS && CONFIG_RTC_TIMER && ThreadControl == 2 && in_post();
 }
 
 // Switch to next thread stack.
@@ -660,11 +592,17 @@ fail:
 void VISIBLE16
 check_irqs(void)
 {
+    if (!MODESEGMENT && !CanInterrupt) {
+        // Can't enable interrupts (PIC and/or IVT not yet setup)
+        cpu_relax();
+        return;
+    }
     if (need_hop_back()) {
-        extern void _cfunc16_check_irqs(void);
-        stack_hop_back(0, 0, _cfunc16_check_irqs);
+        stack_hop_back(check_irqs, 0, 0);
         return;
     }
+    if (MODE16)
+        clock_poll_irq();
     asm volatile("sti ; nop ; rep ; nop ; cli ; cld" : : :"memory");
 }
 
@@ -689,8 +627,7 @@ void VISIBLE16
 wait_irq(void)
 {
     if (need_hop_back()) {
-        extern void _cfunc16_wait_irq(void);
-        stack_hop_back(0, 0, _cfunc16_wait_irq);
+        stack_hop_back(wait_irq, 0, 0);
         return;
     }
     asm volatile("sti ; hlt ; cli ; cld": : :"memory");
@@ -700,8 +637,9 @@ wait_irq(void)
 void
 yield_toirq(void)
 {
-    if (!MODESEGMENT && have_threads()) {
-        // Threads still active - do a yield instead.
+    if (!CONFIG_HARDWARE_IRQ
+        || (!MODESEGMENT && (have_threads() || !CanInterrupt))) {
+        // Threads still active or irqs not available - do a yield instead.
         yield();
         return;
     }
@@ -794,9 +732,8 @@ yield_preempt(void)
 void
 check_preempt(void)
 {
-    extern void _cfunc32flat_yield_preempt(void);
     if (CONFIG_THREADS && GET_GLOBAL(CanPreempt) && have_threads())
-        call32(_cfunc32flat_yield_preempt, 0, 0);
+        call32(yield_preempt, 0, 0);
 }
 
 
@@ -817,11 +754,10 @@ call32_params_helper(struct call32_params_s *params)
 }
 
 u32
-call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret)
+__call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret)
 {
     ASSERT16();
     struct call32_params_s params = {func, eax, edx, ecx};
-    extern void _cfunc32flat_call32_params_helper(void);
-    return call32(_cfunc32flat_call32_params_helper
-                  , (u32)MAKE_FLATPTR(GET_SEG(SS), &params), errret);
+    return call32(call32_params_helper, MAKE_FLATPTR(GET_SEG(SS), &params)
+                  , errret);
 }
diff --git a/qemu/roms/seabios/src/stacks.h b/qemu/roms/seabios/src/stacks.h
index 82c4c3c85..c71bdc8e3 100644
--- a/qemu/roms/seabios/src/stacks.h
+++ b/qemu/roms/seabios/src/stacks.h
@@ -10,17 +10,27 @@
 
 // stacks.c
 extern int HaveSmmCall32;
-u32 call32(void *func, u32 eax, u32 errret);
+u32 __call32(void *func, u32 eax, u32 errret);
+#define call32(func, eax, errret) ({                            \
+        extern void _cfunc32flat_ ##func (void);                \
+        __call32( _cfunc32flat_ ##func , (u32)(eax), (errret)); \
+    })
 extern u8 ExtraStack[], *StackPos;
-u32 stack_hop(u32 eax, u32 edx, void *func);
-u32 stack_hop_back(u32 eax, u32 edx, void *func);
+u32 __stack_hop(u32 eax, u32 edx, void *func);
+#define stack_hop(func, eax, edx)               \
+    __stack_hop((u32)(eax), (u32)(edx), (func))
+u32 __stack_hop_back(u32 eax, u32 edx, void *func);
+#define stack_hop_back(func, eax, edx) ({                               \
+        extern void _cfunc16_ ##func (void);                            \
+        __stack_hop_back((u32)(eax), (u32)(edx), _cfunc16_ ##func );    \
+    })
 int on_extra_stack(void);
 struct bregs;
 void farcall16(struct bregs *callregs);
 void farcall16big(struct bregs *callregs);
 void __call16_int(struct bregs *callregs, u16 offset);
 #define call16_int(nr, callregs) do {                           \
-        extern void irq_trampoline_ ##nr ();                    \
+        extern void irq_trampoline_ ##nr (void);                \
         __call16_int((callregs), (u32)&irq_trampoline_ ##nr );  \
     } while (0)
 void reset(void);
@@ -28,7 +38,7 @@ extern struct thread_info MainThread;
 struct thread_info *getCurThread(void);
 void yield(void);
 void yield_toirq(void);
-void thread_init(void);
+void thread_setup(void);
 int threads_during_optionroms(void);
 void run_thread(void (*func)(void*), void *data);
 void wait_threads(void);
@@ -39,7 +49,12 @@ void start_preempt(void);
 void finish_preempt(void);
 int wait_preempt(void);
 void check_preempt(void);
-u32 call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret);
+u32 __call32_params(void *func, u32 eax, u32 edx, u32 ecx, u32 errret);
+#define call32_params(func, eax, edx, ecx, errret) ({                   \
+        extern void _cfunc32flat_ ##func (void);                        \
+        __call32_params( _cfunc32flat_ ##func , (u32)(eax), (u32)(edx)  \
+                        , (u32)(ecx), (errret));                        \
+    })
 
 // Inline functions
 
diff --git a/qemu/roms/seabios/src/std/acpi.h b/qemu/roms/seabios/src/std/acpi.h
index e0d9516ba..b672bbee4 100644
--- a/qemu/roms/seabios/src/std/acpi.h
+++ b/qemu/roms/seabios/src/std/acpi.h
@@ -294,4 +294,24 @@ struct acpi_table_mcfg {
     struct acpi_mcfg_allocation allocation[0];
 } PACKED;
 
+
+struct rsdt_descriptor {
+    ACPI_TABLE_HEADER_DEF
+    u32 entry[1];
+} PACKED;
+
+#define TCPA_SIGNATURE 0x41504354
+struct tcpa_descriptor_rev2
+{
+    ACPI_TABLE_HEADER_DEF
+    u16  platform_class;
+    u32  log_area_minimum_length;
+    u64  log_area_start_address;
+} PACKED;
+
+/* TCPA ACPI definitions */
+#define TCPA_ACPI_CLASS_CLIENT          0
+#define TCPA_ACPI_CLASS_SERVER          1
+
+
 #endif // acpi.h
diff --git a/qemu/roms/seabios/src/std/bda.h b/qemu/roms/seabios/src/std/bda.h
index c321266e2..4ad6605d4 100644
--- a/qemu/roms/seabios/src/std/bda.h
+++ b/qemu/roms/seabios/src/std/bda.h
@@ -7,7 +7,7 @@
 
 
 /****************************************************************
- * Interupt vector table
+ * Interrupt vector table
  ****************************************************************/
 
 struct rmode_IVT {
diff --git a/qemu/roms/seabios/src/std/multiboot.h b/qemu/roms/seabios/src/std/multiboot.h
new file mode 100644
index 000000000..6c9512703
--- /dev/null
+++ b/qemu/roms/seabios/src/std/multiboot.h
@@ -0,0 +1,260 @@
+/*  multiboot.h - Multiboot header file.  */
+/*  Copyright (C) 1999,2003,2007,2008,2009,2010  Free Software Foundation, Inc.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to
+ *  deal in the Software without restriction, including without limitation the
+ *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ *  sell copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL ANY
+ *  DEVELOPER OR DISTRIBUTOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *  WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+ *  IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef MULTIBOOT_HEADER
+#define MULTIBOOT_HEADER 1
+
+/* How many bytes from the start of the file we search for the header.  */
+#define MULTIBOOT_SEARCH			8192
+#define MULTIBOOT_HEADER_ALIGN			4
+
+/* The magic field should contain this.  */
+#define MULTIBOOT_HEADER_MAGIC			0x1BADB002
+
+/* This should be in %eax.  */
+#define MULTIBOOT_BOOTLOADER_MAGIC		0x2BADB002
+
+/* Alignment of multiboot modules.  */
+#define MULTIBOOT_MOD_ALIGN			0x00001000
+
+/* Alignment of the multiboot info structure.  */
+#define MULTIBOOT_INFO_ALIGN			0x00000004
+
+/* Flags set in the 'flags' member of the multiboot header.  */
+
+/* Align all boot modules on i386 page (4KB) boundaries.  */
+#define MULTIBOOT_PAGE_ALIGN			0x00000001
+
+/* Must pass memory information to OS.  */
+#define MULTIBOOT_MEMORY_INFO			0x00000002
+
+/* Must pass video information to OS.  */
+#define MULTIBOOT_VIDEO_MODE			0x00000004
+
+/* This flag indicates the use of the address fields in the header.  */
+#define MULTIBOOT_AOUT_KLUDGE			0x00010000
+
+/* Flags to be set in the 'flags' member of the multiboot info structure.  */
+
+/* is there basic lower/upper memory information? */
+#define MULTIBOOT_INFO_MEMORY			0x00000001
+/* is there a boot device set? */
+#define MULTIBOOT_INFO_BOOTDEV			0x00000002
+/* is the command-line defined? */
+#define MULTIBOOT_INFO_CMDLINE			0x00000004
+/* are there modules to do something with? */
+#define MULTIBOOT_INFO_MODS			0x00000008
+
+/* These next two are mutually exclusive */
+
+/* is there a symbol table loaded? */
+#define MULTIBOOT_INFO_AOUT_SYMS		0x00000010
+/* is there an ELF section header table? */
+#define MULTIBOOT_INFO_ELF_SHDR			0X00000020
+
+/* is there a full memory map? */
+#define MULTIBOOT_INFO_MEM_MAP			0x00000040
+
+/* Is there drive info?  */
+#define MULTIBOOT_INFO_DRIVE_INFO		0x00000080
+
+/* Is there a config table?  */
+#define MULTIBOOT_INFO_CONFIG_TABLE		0x00000100
+
+/* Is there a boot loader name?  */
+#define MULTIBOOT_INFO_BOOT_LOADER_NAME		0x00000200
+
+/* Is there a APM table?  */
+#define MULTIBOOT_INFO_APM_TABLE		0x00000400
+
+/* Is there video information?  */
+#define MULTIBOOT_INFO_VBE_INFO		        0x00000800
+#define MULTIBOOT_INFO_FRAMEBUFFER_INFO	        0x00001000
+
+#ifndef ASM_FILE
+
+typedef unsigned char		multiboot_uint8_t;
+typedef unsigned short		multiboot_uint16_t;
+typedef unsigned int		multiboot_uint32_t;
+typedef unsigned long long	multiboot_uint64_t;
+
+struct multiboot_header
+{
+  /* Must be MULTIBOOT_MAGIC - see above.  */
+  multiboot_uint32_t magic;
+
+  /* Feature flags.  */
+  multiboot_uint32_t flags;
+
+  /* The above fields plus this one must equal 0 mod 2^32. */
+  multiboot_uint32_t checksum;
+
+  /* These are only valid if MULTIBOOT_AOUT_KLUDGE is set.  */
+  multiboot_uint32_t header_addr;
+  multiboot_uint32_t load_addr;
+  multiboot_uint32_t load_end_addr;
+  multiboot_uint32_t bss_end_addr;
+  multiboot_uint32_t entry_addr;
+
+  /* These are only valid if MULTIBOOT_VIDEO_MODE is set.  */
+  multiboot_uint32_t mode_type;
+  multiboot_uint32_t width;
+  multiboot_uint32_t height;
+  multiboot_uint32_t depth;
+};
+
+/* The symbol table for a.out.  */
+struct multiboot_aout_symbol_table
+{
+  multiboot_uint32_t tabsize;
+  multiboot_uint32_t strsize;
+  multiboot_uint32_t addr;
+  multiboot_uint32_t reserved;
+};
+typedef struct multiboot_aout_symbol_table multiboot_aout_symbol_table_t;
+
+/* The section header table for ELF.  */
+struct multiboot_elf_section_header_table
+{
+  multiboot_uint32_t num;
+  multiboot_uint32_t size;
+  multiboot_uint32_t addr;
+  multiboot_uint32_t shndx;
+};
+typedef struct multiboot_elf_section_header_table multiboot_elf_section_header_table_t;
+
+struct multiboot_info
+{
+  /* Multiboot info version number */
+  multiboot_uint32_t flags;
+
+  /* Available memory from BIOS */
+  multiboot_uint32_t mem_lower;
+  multiboot_uint32_t mem_upper;
+
+  /* "root" partition */
+  multiboot_uint32_t boot_device;
+
+  /* Kernel command line */
+  multiboot_uint32_t cmdline;
+
+  /* Boot-Module list */
+  multiboot_uint32_t mods_count;
+  multiboot_uint32_t mods_addr;
+
+  union
+  {
+    multiboot_aout_symbol_table_t aout_sym;
+    multiboot_elf_section_header_table_t elf_sec;
+  } u;
+
+  /* Memory Mapping buffer */
+  multiboot_uint32_t mmap_length;
+  multiboot_uint32_t mmap_addr;
+
+  /* Drive Info buffer */
+  multiboot_uint32_t drives_length;
+  multiboot_uint32_t drives_addr;
+
+  /* ROM configuration table */
+  multiboot_uint32_t config_table;
+
+  /* Boot Loader Name */
+  multiboot_uint32_t boot_loader_name;
+
+  /* APM table */
+  multiboot_uint32_t apm_table;
+
+  /* Video */
+  multiboot_uint32_t vbe_control_info;
+  multiboot_uint32_t vbe_mode_info;
+  multiboot_uint16_t vbe_mode;
+  multiboot_uint16_t vbe_interface_seg;
+  multiboot_uint16_t vbe_interface_off;
+  multiboot_uint16_t vbe_interface_len;
+
+  multiboot_uint64_t framebuffer_addr;
+  multiboot_uint32_t framebuffer_pitch;
+  multiboot_uint32_t framebuffer_width;
+  multiboot_uint32_t framebuffer_height;
+  multiboot_uint8_t framebuffer_bpp;
+#define MULTIBOOT_FRAMEBUFFER_TYPE_INDEXED 0
+#define MULTIBOOT_FRAMEBUFFER_TYPE_RGB     1
+#define MULTIBOOT_FRAMEBUFFER_TYPE_EGA_TEXT	2
+  multiboot_uint8_t framebuffer_type;
+  union
+  {
+    struct
+    {
+      multiboot_uint32_t framebuffer_palette_addr;
+      multiboot_uint16_t framebuffer_palette_num_colors;
+    };
+    struct
+    {
+      multiboot_uint8_t framebuffer_red_field_position;
+      multiboot_uint8_t framebuffer_red_mask_size;
+      multiboot_uint8_t framebuffer_green_field_position;
+      multiboot_uint8_t framebuffer_green_mask_size;
+      multiboot_uint8_t framebuffer_blue_field_position;
+      multiboot_uint8_t framebuffer_blue_mask_size;
+    };
+  };
+};
+typedef struct multiboot_info multiboot_info_t;
+
+struct multiboot_color
+{
+  multiboot_uint8_t red;
+  multiboot_uint8_t green;
+  multiboot_uint8_t blue;
+};
+
+struct multiboot_mmap_entry
+{
+  multiboot_uint32_t size;
+  multiboot_uint64_t addr;
+  multiboot_uint64_t len;
+#define MULTIBOOT_MEMORY_AVAILABLE		1
+#define MULTIBOOT_MEMORY_RESERVED		2
+#define MULTIBOOT_MEMORY_ACPI_RECLAIMABLE       3
+#define MULTIBOOT_MEMORY_NVS                    4
+#define MULTIBOOT_MEMORY_BADRAM                 5
+  multiboot_uint32_t type;
+} __attribute__((packed));
+typedef struct multiboot_mmap_entry multiboot_memory_map_t;
+
+struct multiboot_mod_list
+{
+  /* the memory used goes from bytes 'mod_start' to 'mod_end-1' inclusive */
+  multiboot_uint32_t mod_start;
+  multiboot_uint32_t mod_end;
+
+  /* Module command line */
+  multiboot_uint32_t cmdline;
+
+  /* padding to take it to 16 bytes (must be zero) */
+  multiboot_uint32_t pad;
+};
+typedef struct multiboot_mod_list multiboot_module_t;
+
+#endif /* ! ASM_FILE */
+
+#endif /* ! MULTIBOOT_HEADER */
diff --git a/qemu/roms/seabios/src/std/smbios.h b/qemu/roms/seabios/src/std/smbios.h
index 05137167a..4ccf2ea34 100644
--- a/qemu/roms/seabios/src/std/smbios.h
+++ b/qemu/roms/seabios/src/std/smbios.h
@@ -3,11 +3,13 @@
 
 #include "types.h" // u32
 
+#define SMBIOS_SIGNATURE 0x5f4d535f // "_SM_"
+
 /* SMBIOS entry point -- must be written to a 16-bit aligned address
    between 0xf0000 and 0xfffff.
  */
 struct smbios_entry_point {
-    char anchor_string[4];
+    u32 signature;
     u8 checksum;
     u8 length;
     u8 smbios_major_version;
diff --git a/qemu/roms/seabios/src/string.c b/qemu/roms/seabios/src/string.c
index 2e4e43746..adb8198f8 100644
--- a/qemu/roms/seabios/src/string.c
+++ b/qemu/roms/seabios/src/string.c
@@ -227,7 +227,7 @@ strtcpy(char *dest, const char *src, size_t len)
     return dest;
 }
 
-// locate first occurance of character c in the string s
+// locate first occurrence of character c in the string s
 char *
 strchr(const char *s, int c)
 {
diff --git a/qemu/roms/seabios/src/string.h b/qemu/roms/seabios/src/string.h
index a557d6a44..d069989db 100644
--- a/qemu/roms/seabios/src/string.h
+++ b/qemu/roms/seabios/src/string.h
@@ -11,12 +11,12 @@ size_t strlen(const char *s);
 int memcmp_far(u16 s1seg, const void *s1, u16 s2seg, const void *s2, size_t n);
 int memcmp(const void *s1, const void *s2, size_t n);
 int strcmp(const char *s1, const char *s2);
-inline void memset_far(u16 d_seg, void *d_far, u8 c, size_t len);
-inline void memset16_far(u16 d_seg, void *d_far, u16 c, size_t len);
+void memset_far(u16 d_seg, void *d_far, u8 c, size_t len);
+void memset16_far(u16 d_seg, void *d_far, u16 c, size_t len);
 void *memset(void *s, int c, size_t n);
 void memset_fl(void *ptr, u8 val, size_t size);
-inline void memcpy_far(u16 d_seg, void *d_far
-                       , u16 s_seg, const void *s_far, size_t len);
+void memcpy_far(u16 d_seg, void *d_far
+                , u16 s_seg, const void *s_far, size_t len);
 void memcpy_fl(void *d_fl, const void *s_fl, size_t len);
 void *memcpy(void *d1, const void *s1, size_t len);
 #if MODESEGMENT == 0
diff --git a/qemu/roms/seabios/src/system.c b/qemu/roms/seabios/src/system.c
index 60a6fce58..438e60e2c 100644
--- a/qemu/roms/seabios/src/system.c
+++ b/qemu/roms/seabios/src/system.c
@@ -7,9 +7,9 @@
 
 #include "biosvar.h" // GET_GLOBAL
 #include "bregs.h" // struct bregs
+#include "e820map.h" // E820_RAM
 #include "hw/pic.h" // pic_reset
 #include "malloc.h" // LegacyRamSize
-#include "memmap.h" // E820_RAM
 #include "output.h" // debug_enter
 #include "string.h" // memcpy_far
 #include "util.h" // handle_1553
diff --git a/qemu/roms/seabios/src/tcgbios.c b/qemu/roms/seabios/src/tcgbios.c
new file mode 100644
index 000000000..09954825c
--- /dev/null
+++ b/qemu/roms/seabios/src/tcgbios.c
@@ -0,0 +1,1480 @@
+//  Implementation of the TCG BIOS extension according to the specification
+//  described in specs found at
+//  http://www.trustedcomputinggroup.org/resources/pc_client_work_group_specific_implementation_specification_for_conventional_bios
+//
+//  Copyright (C) 2006-2011, 2014, 2015 IBM Corporation
+//
+//  Authors:
+//      Stefan Berger <stefanb@linux.vnet.ibm.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+
+#include "config.h"
+
+#include "types.h"
+#include "byteorder.h" // cpu_to_*
+#include "hw/tpm_drivers.h" // tpm_drivers[]
+#include "farptr.h" // MAKE_FLATPTR
+#include "string.h" // checksum
+#include "tcgbios.h"// tpm_*, prototypes
+#include "util.h" // printf, get_keystroke
+#include "output.h" // dprintf
+#include "std/acpi.h"  // RSDP_SIGNATURE, rsdt_descriptor
+#include "bregs.h" // struct bregs
+#include "sha1.h" // sha1
+#include "fw/paravirt.h" // runningOnXen
+#include "std/smbios.h"
+
+static const u8 Startup_ST_CLEAR[] = { 0x00, TPM_ST_CLEAR };
+static const u8 Startup_ST_STATE[] = { 0x00, TPM_ST_STATE };
+
+static const u8 PhysicalPresence_CMD_ENABLE[]  = { 0x00, 0x20 };
+static const u8 PhysicalPresence_CMD_DISABLE[] = { 0x01, 0x00 };
+static const u8 PhysicalPresence_PRESENT[]     = { 0x00, 0x08 };
+static const u8 PhysicalPresence_NOT_PRESENT_LOCK[] = { 0x00, 0x14 };
+
+static const u8 CommandFlag_FALSE[1] = { 0x00 };
+static const u8 CommandFlag_TRUE[1]  = { 0x01 };
+
+static const u8 GetCapability_Permanent_Flags[] = {
+    0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04,
+    0x00, 0x00, 0x01, 0x08
+};
+
+static const u8 GetCapability_OwnerAuth[] = {
+    0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04,
+    0x00, 0x00, 0x01, 0x11
+};
+
+static const u8 GetCapability_Timeouts[] = {
+    0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04,
+    0x00, 0x00, 0x01, 0x15
+};
+
+static const u8 GetCapability_Durations[] = {
+    0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04,
+    0x00, 0x00, 0x01, 0x20
+};
+
+static u8 evt_separator[] = {0xff,0xff,0xff,0xff};
+
+
+#define RSDP_CAST(ptr)   ((struct rsdp_descriptor *)ptr)
+
+/* local function prototypes */
+
+static u32 tpm_calling_int19h(void);
+static u32 tpm_add_event_separators(void);
+static u32 tpm_start_option_rom_scan(void);
+static u32 tpm_smbios_measure(void);
+
+/* helper functions */
+
+static inline void *input_buf32(struct bregs *regs)
+{
+    return MAKE_FLATPTR(regs->es, regs->di);
+}
+
+static inline void *output_buf32(struct bregs *regs)
+{
+    return MAKE_FLATPTR(regs->ds, regs->si);
+}
+
+
+typedef struct {
+    u8            tpm_probed:1;
+    u8            tpm_found:1;
+    u8            tpm_working:1;
+    u8            if_shutdown:1;
+    u8            tpm_driver_to_use:4;
+} tpm_state_t;
+
+
+static tpm_state_t tpm_state = {
+    .tpm_driver_to_use = TPM_INVALID_DRIVER,
+};
+
+
+/********************************************************
+  Extensions for TCG-enabled BIOS
+ *******************************************************/
+
+
+static u32
+is_tpm_present(void)
+{
+    u32 rc = 0;
+    unsigned int i;
+
+    for (i = 0; i < TPM_NUM_DRIVERS; i++) {
+        struct tpm_driver *td = &tpm_drivers[i];
+        if (td->probe() != 0) {
+            td->init();
+            tpm_state.tpm_driver_to_use = i;
+            rc = 1;
+            break;
+        }
+    }
+
+    return rc;
+}
+
+static void
+probe_tpm(void)
+{
+    if (!tpm_state.tpm_probed) {
+        tpm_state.tpm_probed = 1;
+        tpm_state.tpm_found = (is_tpm_present() != 0);
+        tpm_state.tpm_working = tpm_state.tpm_found;
+    }
+}
+
+static int
+has_working_tpm(void)
+{
+    probe_tpm();
+
+    return tpm_state.tpm_working;
+}
+
+static struct tcpa_descriptor_rev2 *
+find_tcpa_by_rsdp(struct rsdp_descriptor *rsdp)
+{
+    u32 ctr = 0;
+    struct tcpa_descriptor_rev2 *tcpa = NULL;
+    struct rsdt_descriptor *rsdt;
+    u32 length;
+    u16 off;
+
+    rsdt   = (struct rsdt_descriptor *)rsdp->rsdt_physical_address;
+    if (!rsdt)
+        return NULL;
+
+    length = rsdt->length;
+    off = offsetof(struct rsdt_descriptor, entry);
+
+    while ((off + sizeof(rsdt->entry[0])) <= length) {
+        /* try all pointers to structures */
+        tcpa = (struct tcpa_descriptor_rev2 *)(int)rsdt->entry[ctr];
+
+        /* valid TCPA ACPI table ? */
+        if (tcpa->signature == TCPA_SIGNATURE &&
+            checksum((u8 *)tcpa, tcpa->length) == 0)
+            break;
+
+        tcpa = NULL;
+        off += sizeof(rsdt->entry[0]);
+        ctr++;
+    }
+
+    return tcpa;
+}
+
+
+static struct tcpa_descriptor_rev2 *
+find_tcpa_table(void)
+{
+    struct tcpa_descriptor_rev2 *tcpa = NULL;
+    struct rsdp_descriptor *rsdp = RsdpAddr;
+
+    if (rsdp)
+        tcpa = find_tcpa_by_rsdp(rsdp);
+    else
+        tpm_state.if_shutdown = 1;
+
+    if (!rsdp)
+        dprintf(DEBUG_tcg,
+                "TCGBIOS: RSDP was NOT found! -- Disabling interface.\n");
+    else if (!tcpa)
+        dprintf(DEBUG_tcg, "TCGBIOS: TCPA ACPI was NOT found!\n");
+
+    return tcpa;
+}
+
+
+static u8 *
+get_lasa_base_ptr(u32 *log_area_minimum_length)
+{
+    u8 *log_area_start_address = 0;
+    struct tcpa_descriptor_rev2 *tcpa = find_tcpa_table();
+
+    if (tcpa) {
+        log_area_start_address = (u8 *)(long)tcpa->log_area_start_address;
+        if (log_area_minimum_length)
+            *log_area_minimum_length = tcpa->log_area_minimum_length;
+    }
+
+    return log_area_start_address;
+}
+
+
+/* clear the ACPI log */
+static void
+reset_acpi_log(void)
+{
+    u32 log_area_minimum_length;
+    u8 *log_area_start_address = get_lasa_base_ptr(&log_area_minimum_length);
+
+    if (log_area_start_address)
+        memset(log_area_start_address, 0x0, log_area_minimum_length);
+}
+
+
+/*
+   initialize the TCPA ACPI subsystem; find the ACPI tables and determine
+   where the TCPA table is.
+ */
+static void
+tpm_acpi_init(void)
+{
+    tpm_state.if_shutdown = 0;
+    tpm_state.tpm_probed = 0;
+    tpm_state.tpm_found = 0;
+    tpm_state.tpm_working = 0;
+
+    if (!has_working_tpm()) {
+        tpm_state.if_shutdown = 1;
+        return;
+    }
+
+    reset_acpi_log();
+}
+
+
+static u32
+transmit(u8 locty, const struct iovec iovec[],
+         u8 *respbuffer, u32 *respbufferlen,
+         enum tpmDurationType to_t)
+{
+    u32 rc = 0;
+    u32 irc;
+    struct tpm_driver *td;
+    unsigned int i;
+
+    if (tpm_state.tpm_driver_to_use == TPM_INVALID_DRIVER)
+        return TCG_FATAL_COM_ERROR;
+
+    td = &tpm_drivers[tpm_state.tpm_driver_to_use];
+
+    irc = td->activate(locty);
+    if (irc != 0) {
+        /* tpm could not be activated */
+        return TCG_FATAL_COM_ERROR;
+    }
+
+    for (i = 0; iovec[i].length; i++) {
+        irc = td->senddata(iovec[i].data,
+                           iovec[i].length);
+        if (irc != 0)
+            return TCG_FATAL_COM_ERROR;
+    }
+
+    irc = td->waitdatavalid();
+    if (irc != 0)
+        return TCG_FATAL_COM_ERROR;
+
+    irc = td->waitrespready(to_t);
+    if (irc != 0)
+        return TCG_FATAL_COM_ERROR;
+
+    irc = td->readresp(respbuffer,
+                       respbufferlen);
+    if (irc != 0)
+        return TCG_FATAL_COM_ERROR;
+
+    td->ready();
+
+    return rc;
+}
+
+
+/*
+ * Send a TPM command with the given ordinal. Append the given buffer
+ * containing all data in network byte order to the command (this is
+ * the custom part per command) and expect a response of the given size.
+ * If a buffer is provided, the response will be copied into it.
+ */
+static u32
+build_and_send_cmd_od(u8 locty, u32 ordinal, const u8 *append, u32 append_size,
+                      u8 *resbuffer, u32 return_size, u32 *returnCode,
+                      const u8 *otherdata, u32 otherdata_size,
+                      enum tpmDurationType to_t)
+{
+#define MAX_APPEND_SIZE   sizeof(GetCapability_Timeouts)
+#define MAX_RESPONSE_SIZE sizeof(struct tpm_res_getcap_perm_flags)
+    u32 rc;
+    u8 ibuffer[TPM_REQ_HEADER_SIZE + MAX_APPEND_SIZE];
+    u8 obuffer[MAX_RESPONSE_SIZE];
+    struct tpm_req_header *trqh = (struct tpm_req_header *)ibuffer;
+    struct tpm_rsp_header *trsh = (struct tpm_rsp_header *)obuffer;
+    struct iovec iovec[3];
+    u32 obuffer_len = sizeof(obuffer);
+    u32 idx = 1;
+
+    if (append_size > MAX_APPEND_SIZE ||
+        return_size > MAX_RESPONSE_SIZE) {
+        dprintf(DEBUG_tcg, "TCGBIOS: size of requested buffers too big.");
+        return TCG_FIRMWARE_ERROR;
+    }
+
+    iovec[0].data   = trqh;
+    iovec[0].length = TPM_REQ_HEADER_SIZE + append_size;
+
+    if (otherdata) {
+        iovec[1].data   = (void *)otherdata;
+        iovec[1].length = otherdata_size;
+        idx = 2;
+    }
+
+    iovec[idx].data   = NULL;
+    iovec[idx].length = 0;
+
+    memset(ibuffer, 0x0, sizeof(ibuffer));
+    memset(obuffer, 0x0, sizeof(obuffer));
+
+    trqh->tag     = cpu_to_be16(TPM_TAG_RQU_CMD);
+    trqh->totlen  = cpu_to_be32(TPM_REQ_HEADER_SIZE + append_size +
+                                otherdata_size);
+    trqh->ordinal = cpu_to_be32(ordinal);
+
+    if (append_size)
+        memcpy((char *)trqh + sizeof(*trqh),
+               append, append_size);
+
+    rc = transmit(locty, iovec, obuffer, &obuffer_len, to_t);
+    if (rc)
+        return rc;
+
+    *returnCode = be32_to_cpu(trsh->errcode);
+
+    if (resbuffer)
+        memcpy(resbuffer, trsh, return_size);
+
+    return 0;
+}
+
+
+static u32
+build_and_send_cmd(u8 locty, u32 ordinal, const u8 *append, u32 append_size,
+                   u8 *resbuffer, u32 return_size, u32 *returnCode,
+                   enum tpmDurationType to_t)
+{
+    return build_and_send_cmd_od(locty, ordinal, append, append_size,
+                                 resbuffer, return_size, returnCode,
+                                 NULL, 0, to_t);
+}
+
+
+static u32
+determine_timeouts(void)
+{
+    u32 rc;
+    u32 returnCode;
+    struct tpm_res_getcap_timeouts timeouts;
+    struct tpm_res_getcap_durations durations;
+    struct tpm_driver *td = &tpm_drivers[tpm_state.tpm_driver_to_use];
+    u32 i;
+
+    rc = build_and_send_cmd(0, TPM_ORD_GetCapability,
+                            GetCapability_Timeouts,
+                            sizeof(GetCapability_Timeouts),
+                            (u8 *)&timeouts, sizeof(timeouts),
+                            &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: Return code from TPM_GetCapability(Timeouts)"
+            " = 0x%08x\n", returnCode);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    rc = build_and_send_cmd(0, TPM_ORD_GetCapability,
+                            GetCapability_Durations,
+                            sizeof(GetCapability_Durations),
+                            (u8 *)&durations, sizeof(durations),
+                            &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: Return code from TPM_GetCapability(Durations)"
+            " = 0x%08x\n", returnCode);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    for (i = 0; i < 3; i++)
+        durations.durations[i] = be32_to_cpu(durations.durations[i]);
+
+    for (i = 0; i < 4; i++)
+        timeouts.timeouts[i] = be32_to_cpu(timeouts.timeouts[i]);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: timeouts: %u %u %u %u\n",
+            timeouts.timeouts[0],
+            timeouts.timeouts[1],
+            timeouts.timeouts[2],
+            timeouts.timeouts[3]);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: durations: %u %u %u\n",
+            durations.durations[0],
+            durations.durations[1],
+            durations.durations[2]);
+
+
+    td->set_timeouts(timeouts.timeouts, durations.durations);
+
+    return 0;
+
+err_exit:
+    dprintf(DEBUG_tcg, "TCGBIOS: TPM malfunctioning (line %d).\n", __LINE__);
+
+    tpm_state.tpm_working = 0;
+    if (rc)
+        return rc;
+    return TCG_TCG_COMMAND_ERROR;
+}
+
+
+static u32
+tpm_startup(void)
+{
+    u32 rc;
+    u32 returnCode;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    dprintf(DEBUG_tcg, "TCGBIOS: Starting with TPM_Startup(ST_CLEAR)\n");
+    rc = build_and_send_cmd(0, TPM_ORD_Startup,
+                            Startup_ST_CLEAR, sizeof(Startup_ST_CLEAR),
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    dprintf(DEBUG_tcg, "Return code from TPM_Startup = 0x%08x\n",
+            returnCode);
+
+    if (CONFIG_COREBOOT) {
+        /* with other firmware on the system the TPM may already have been
+         * initialized
+         */
+        if (returnCode == TPM_INVALID_POSTINIT)
+            returnCode = 0;
+    }
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    rc = build_and_send_cmd(0, TPM_ORD_SelfTestFull, NULL, 0,
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_LONG);
+
+    dprintf(DEBUG_tcg, "Return code from TPM_SelfTestFull = 0x%08x\n",
+            returnCode);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    rc = build_and_send_cmd(3, TSC_ORD_ResetEstablishmentBit, NULL, 0,
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    dprintf(DEBUG_tcg, "Return code from TSC_ResetEstablishmentBit = 0x%08x\n",
+            returnCode);
+
+    if (rc || (returnCode != 0 && returnCode != TPM_BAD_LOCALITY))
+        goto err_exit;
+
+    rc = determine_timeouts();
+    if (rc)
+        goto err_exit;
+
+    rc = tpm_smbios_measure();
+    if (rc)
+        goto err_exit;
+
+    rc = tpm_start_option_rom_scan();
+    if (rc)
+        goto err_exit;
+
+    return 0;
+
+err_exit:
+    dprintf(DEBUG_tcg, "TCGBIOS: TPM malfunctioning (line %d).\n", __LINE__);
+
+    tpm_state.tpm_working = 0;
+    if (rc)
+        return rc;
+    return TCG_TCG_COMMAND_ERROR;
+}
+
+
+void
+tpm_setup(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    tpm_acpi_init();
+    if (runningOnXen())
+        return;
+
+    tpm_startup();
+}
+
+
+void
+tpm_prepboot(void)
+{
+    u32 rc;
+    u32 returnCode;
+
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    if (!has_working_tpm())
+        return;
+
+    rc = build_and_send_cmd(0, TPM_ORD_PhysicalPresence,
+                            PhysicalPresence_CMD_ENABLE,
+                            sizeof(PhysicalPresence_CMD_ENABLE),
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_SHORT);
+    if (rc || returnCode)
+        goto err_exit;
+
+    rc = build_and_send_cmd(0, TPM_ORD_PhysicalPresence,
+                            PhysicalPresence_NOT_PRESENT_LOCK,
+                            sizeof(PhysicalPresence_NOT_PRESENT_LOCK),
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_SHORT);
+    if (rc || returnCode)
+        goto err_exit;
+
+    rc = tpm_calling_int19h();
+    if (rc)
+        goto err_exit;
+
+    rc = tpm_add_event_separators();
+    if (rc)
+        goto err_exit;
+
+    return;
+
+err_exit:
+    dprintf(DEBUG_tcg, "TCGBIOS: TPM malfunctioning (line %d).\n", __LINE__);
+
+    tpm_state.tpm_working = 0;
+}
+
+static int
+is_valid_pcpes(struct pcpes *pcpes)
+{
+    return (pcpes->eventtype != 0);
+}
+
+
+static u8 *
+get_lasa_last_ptr(u16 *entry_count, u8 **log_area_start_address_next)
+{
+    struct pcpes *pcpes;
+    u32 log_area_minimum_length = 0;
+    u8 *log_area_start_address_base =
+        get_lasa_base_ptr(&log_area_minimum_length);
+    u8 *log_area_start_address_last = NULL;
+    u8 *end = log_area_start_address_base + log_area_minimum_length;
+    u32 size;
+
+    if (entry_count)
+        *entry_count = 0;
+
+    if (!log_area_start_address_base)
+        return NULL;
+
+    while (log_area_start_address_base < end) {
+        pcpes = (struct pcpes *)log_area_start_address_base;
+        if (!is_valid_pcpes(pcpes))
+            break;
+        if (entry_count)
+            (*entry_count)++;
+        size = pcpes->eventdatasize + offsetof(struct pcpes, event);
+        log_area_start_address_last = log_area_start_address_base;
+        log_area_start_address_base += size;
+    }
+
+    if (log_area_start_address_next)
+        *log_area_start_address_next = log_area_start_address_base;
+
+    return log_area_start_address_last;
+}
+
+
+static u32
+tpm_sha1_calc(const u8 *data, u32 length, u8 *hash)
+{
+    u32 rc;
+    u32 returnCode;
+    struct tpm_res_sha1start start;
+    struct tpm_res_sha1complete complete;
+    u32 blocks = length / 64;
+    u32 rest = length & 0x3f;
+    u32 numbytes, numbytes_no;
+    u32 offset = 0;
+
+    rc = build_and_send_cmd(0, TPM_ORD_SHA1Start,
+                            NULL, 0,
+                            (u8 *)&start, sizeof(start),
+                            &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    while (blocks > 0) {
+
+        numbytes = be32_to_cpu(start.max_num_bytes);
+        if (numbytes > blocks * 64)
+             numbytes = blocks * 64;
+
+        numbytes_no = cpu_to_be32(numbytes);
+
+        rc = build_and_send_cmd_od(0, TPM_ORD_SHA1Update,
+                                   (u8 *)&numbytes_no, sizeof(numbytes_no),
+                                   NULL, 0, &returnCode,
+                                   &data[offset], numbytes,
+                                   TPM_DURATION_TYPE_SHORT);
+
+        if (rc || returnCode)
+            goto err_exit;
+
+        offset += numbytes;
+        blocks -= (numbytes / 64);
+    }
+
+    numbytes_no = cpu_to_be32(rest);
+
+    rc = build_and_send_cmd_od(0, TPM_ORD_SHA1Complete,
+                              (u8 *)&numbytes_no, sizeof(numbytes_no),
+                              (u8 *)&complete, sizeof(complete),
+                              &returnCode,
+                              &data[offset], rest, TPM_DURATION_TYPE_SHORT);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    memcpy(hash, complete.hash, sizeof(complete.hash));
+
+    return 0;
+
+err_exit:
+    dprintf(DEBUG_tcg, "TCGBIOS: TPM SHA1 malfunctioning.\n");
+
+    tpm_state.tpm_working = 0;
+    if (rc)
+        return rc;
+    return TCG_TCG_COMMAND_ERROR;
+}
+
+
+static u32
+sha1_calc(const u8 *data, u32 length, u8 *hash)
+{
+    if (length < tpm_drivers[tpm_state.tpm_driver_to_use].sha1threshold)
+        return tpm_sha1_calc(data, length, hash);
+
+    return sha1(data, length, hash);
+}
+
+
+/*
+ * Extend the ACPI log with the given entry by copying the
+ * entry data into the log.
+ * Input
+ *  Pointer to the structure to be copied into the log
+ *
+ * Output:
+ *  lower 16 bits of return code contain entry number
+ *  if entry number is '0', then upper 16 bits contain error code.
+ */
+static u32
+tpm_extend_acpi_log(void *entry_ptr, u16 *entry_count)
+{
+    u32 log_area_minimum_length, size;
+    u8 *log_area_start_address_base =
+        get_lasa_base_ptr(&log_area_minimum_length);
+    u8 *log_area_start_address_next = NULL;
+    struct pcpes *pcpes = (struct pcpes *)entry_ptr;
+
+    get_lasa_last_ptr(entry_count, &log_area_start_address_next);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: LASA_BASE = %p, LASA_NEXT = %p\n",
+            log_area_start_address_base, log_area_start_address_next);
+
+    if (log_area_start_address_next == NULL || log_area_minimum_length == 0)
+        return TCG_PC_LOGOVERFLOW;
+
+    size = pcpes->eventdatasize + offsetof(struct pcpes, event);
+
+    if ((log_area_start_address_next + size - log_area_start_address_base) >
+        log_area_minimum_length) {
+        dprintf(DEBUG_tcg, "TCGBIOS: LOG OVERFLOW: size = %d\n", size);
+        return TCG_PC_LOGOVERFLOW;
+    }
+
+    memcpy(log_area_start_address_next, entry_ptr, size);
+
+    (*entry_count)++;
+
+    return 0;
+}
+
+
+static u32
+is_preboot_if_shutdown(void)
+{
+    return tpm_state.if_shutdown;
+}
+
+
+static u32
+shutdown_preboot_interface(void)
+{
+    u32 rc = 0;
+
+    if (!is_preboot_if_shutdown()) {
+        tpm_state.if_shutdown = 1;
+    } else {
+        rc = TCG_INTERFACE_SHUTDOWN;
+    }
+
+    return rc;
+}
+
+
+static void
+tpm_shutdown(void)
+{
+    reset_acpi_log();
+    shutdown_preboot_interface();
+}
+
+
+static u32
+pass_through_to_tpm(struct pttti *pttti, struct pttto *pttto)
+{
+    u32 rc = 0;
+    u32 resbuflen = 0;
+    struct tpm_req_header *trh;
+    u8 locty = 0;
+    struct iovec iovec[2];
+    const u32 *tmp;
+
+    if (is_preboot_if_shutdown()) {
+        rc = TCG_INTERFACE_SHUTDOWN;
+        goto err_exit;
+    }
+
+    trh = (struct tpm_req_header *)pttti->tpmopin;
+
+    if (pttti->ipblength < sizeof(struct pttti) + TPM_REQ_HEADER_SIZE ||
+        pttti->opblength < sizeof(struct pttto) ||
+        be32_to_cpu(trh->totlen)  + sizeof(struct pttti) > pttti->ipblength ) {
+        rc = TCG_INVALID_INPUT_PARA;
+        goto err_exit;
+    }
+
+    resbuflen = pttti->opblength - offsetof(struct pttto, tpmopout);
+
+    iovec[0].data   = pttti->tpmopin;
+    tmp = (const u32 *)&((u8 *)iovec[0].data)[2];
+    iovec[0].length = cpu_to_be32(*tmp);
+
+    iovec[1].data   = NULL;
+    iovec[1].length = 0;
+
+    rc = transmit(locty, iovec, pttto->tpmopout, &resbuflen,
+                  TPM_DURATION_TYPE_LONG /* worst case */);
+    if (rc)
+        goto err_exit;
+
+    pttto->opblength = offsetof(struct pttto, tpmopout) + resbuflen;
+    pttto->reserved  = 0;
+
+err_exit:
+    if (rc != 0) {
+        pttto->opblength = 4;
+        pttto->reserved = 0;
+    }
+
+    return rc;
+}
+
+
+static u32
+tpm_extend(u8 *hash, u32 pcrindex)
+{
+    u32 rc;
+    struct pttto_extend pttto;
+    struct pttti_extend pttti = {
+        .pttti = {
+            .ipblength = sizeof(struct pttti_extend),
+            .opblength = sizeof(struct pttto_extend),
+        },
+        .req = {
+            .tag      = cpu_to_be16(0xc1),
+            .totlen   = cpu_to_be32(sizeof(pttti.req)),
+            .ordinal  = cpu_to_be32(TPM_ORD_Extend),
+            .pcrindex = cpu_to_be32(pcrindex),
+        },
+    };
+
+    memcpy(pttti.req.digest, hash, sizeof(pttti.req.digest));
+
+    rc = pass_through_to_tpm(&pttti.pttti, &pttto.pttto);
+
+    if (rc == 0) {
+        if (pttto.pttto.opblength < TPM_RSP_HEADER_SIZE ||
+            pttto.pttto.opblength !=
+                sizeof(struct pttto) + be32_to_cpu(pttto.rsp.totlen) ||
+            be16_to_cpu(pttto.rsp.tag) != 0xc4) {
+            rc = TCG_FATAL_COM_ERROR;
+        }
+    }
+
+    if (rc)
+        tpm_shutdown();
+
+    return rc;
+}
+
+
+static u32
+hash_all(const struct hai *hai, u8 *hash)
+{
+    if (is_preboot_if_shutdown() != 0)
+        return TCG_INTERFACE_SHUTDOWN;
+
+    if (hai->ipblength != sizeof(struct hai) ||
+        hai->hashdataptr == 0 ||
+        hai->hashdatalen == 0 ||
+        hai->algorithmid != TPM_ALG_SHA)
+        return TCG_INVALID_INPUT_PARA;
+
+    return sha1_calc((const u8 *)hai->hashdataptr, hai->hashdatalen, hash);
+}
+
+
+static u32
+hash_log_event(const struct hlei *hlei, struct hleo *hleo)
+{
+    u32 rc = 0;
+    u16 size;
+    struct pcpes *pcpes;
+    u16 entry_count;
+
+    if (is_preboot_if_shutdown() != 0) {
+        rc = TCG_INTERFACE_SHUTDOWN;
+        goto err_exit;
+    }
+
+    size = hlei->ipblength;
+    if (size != sizeof(*hlei)) {
+        rc = TCG_INVALID_INPUT_PARA;
+        goto err_exit;
+    }
+
+    pcpes = (struct pcpes *)hlei->logdataptr;
+
+    if (pcpes->pcrindex >= 24 ||
+        pcpes->pcrindex  != hlei->pcrindex ||
+        pcpes->eventtype != hlei->logeventtype) {
+        rc = TCG_INVALID_INPUT_PARA;
+        goto err_exit;
+    }
+
+    if ((hlei->hashdataptr != 0) && (hlei->hashdatalen != 0)) {
+        rc = sha1_calc((const u8 *)hlei->hashdataptr,
+                       hlei->hashdatalen, pcpes->digest);
+        if (rc)
+            return rc;
+    }
+
+    rc = tpm_extend_acpi_log((void *)hlei->logdataptr, &entry_count);
+    if (rc)
+        goto err_exit;
+
+    /* updating the log was fine */
+    hleo->opblength = sizeof(struct hleo);
+    hleo->reserved  = 0;
+    hleo->eventnumber = entry_count;
+
+err_exit:
+    if (rc != 0) {
+        hleo->opblength = 2;
+        hleo->reserved = 0;
+    }
+
+    return rc;
+}
+
+
+static u32
+hash_log_extend_event(const struct hleei_short *hleei_s, struct hleeo *hleeo)
+{
+    u32 rc = 0;
+    struct hleo hleo;
+    struct hleei_long *hleei_l = (struct hleei_long *)hleei_s;
+    const void *logdataptr;
+    u32 logdatalen;
+    struct pcpes *pcpes;
+
+    /* short or long version? */
+    switch (hleei_s->ipblength) {
+    case sizeof(struct hleei_short):
+        /* short */
+        logdataptr = hleei_s->logdataptr;
+        logdatalen = hleei_s->logdatalen;
+    break;
+
+    case sizeof(struct hleei_long):
+        /* long */
+        logdataptr = hleei_l->logdataptr;
+        logdatalen = hleei_l->logdatalen;
+    break;
+
+    default:
+        /* bad input block */
+        rc = TCG_INVALID_INPUT_PARA;
+        goto err_exit;
+    }
+
+    pcpes = (struct pcpes *)logdataptr;
+
+    struct hlei hlei = {
+        .ipblength   = sizeof(hlei),
+        .hashdataptr = hleei_s->hashdataptr,
+        .hashdatalen = hleei_s->hashdatalen,
+        .pcrindex    = hleei_s->pcrindex,
+        .logeventtype= pcpes->eventtype,
+        .logdataptr  = logdataptr,
+        .logdatalen  = logdatalen,
+    };
+
+    rc = hash_log_event(&hlei, &hleo);
+    if (rc)
+        goto err_exit;
+
+    hleeo->opblength = sizeof(struct hleeo);
+    hleeo->reserved  = 0;
+    hleeo->eventnumber = hleo.eventnumber;
+
+    rc = tpm_extend(pcpes->digest, hleei_s->pcrindex);
+
+err_exit:
+    if (rc != 0) {
+        hleeo->opblength = 4;
+        hleeo->reserved  = 0;
+    }
+
+    return rc;
+
+}
+
+
+static u32
+tss(struct ti *ti, struct to *to)
+{
+    u32 rc = 0;
+
+    if (is_preboot_if_shutdown() == 0) {
+        rc = TCG_PC_UNSUPPORTED;
+    } else {
+        rc = TCG_INTERFACE_SHUTDOWN;
+    }
+
+    to->opblength = sizeof(struct to);
+    to->reserved  = 0;
+
+    return rc;
+}
+
+
+static u32
+compact_hash_log_extend_event(u8 *buffer,
+                              u32 info,
+                              u32 length,
+                              u32 pcrindex,
+                              u32 *edx_ptr)
+{
+    u32 rc = 0;
+    struct hleeo hleeo;
+    struct pcpes pcpes = {
+        .pcrindex      = pcrindex,
+        .eventtype     = EV_COMPACT_HASH,
+        .eventdatasize = sizeof(info),
+        .event         = info,
+    };
+    struct hleei_short hleei = {
+        .ipblength   = sizeof(hleei),
+        .hashdataptr = buffer,
+        .hashdatalen = length,
+        .pcrindex    = pcrindex,
+        .logdataptr  = &pcpes,
+        .logdatalen  = sizeof(pcpes),
+    };
+
+    rc = hash_log_extend_event(&hleei, &hleeo);
+    if (rc == 0)
+        *edx_ptr = hleeo.eventnumber;
+
+    return rc;
+}
+
+
+void VISIBLE32FLAT
+tpm_interrupt_handler32(struct bregs *regs)
+{
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    set_cf(regs, 0);
+
+    if (!has_working_tpm()) {
+        regs->eax = TCG_GENERAL_ERROR;
+        return;
+    }
+
+    switch ((enum irq_ids)regs->al) {
+    case TCG_StatusCheck:
+        if (is_tpm_present() == 0) {
+            /* no TPM available */
+            regs->eax = TCG_PC_TPM_NOT_PRESENT;
+        } else {
+            regs->eax = 0;
+            regs->ebx = TCG_MAGIC;
+            regs->ch = TCG_VERSION_MAJOR;
+            regs->cl = TCG_VERSION_MINOR;
+            regs->edx = 0x0;
+            regs->esi = (u32)get_lasa_base_ptr(NULL);
+            regs->edi =
+                  (u32)get_lasa_last_ptr(NULL, NULL);
+        }
+        break;
+
+    case TCG_HashLogExtendEvent:
+        regs->eax =
+            hash_log_extend_event(
+                  (struct hleei_short *)input_buf32(regs),
+                  (struct hleeo *)output_buf32(regs));
+        break;
+
+    case TCG_PassThroughToTPM:
+        regs->eax =
+            pass_through_to_tpm((struct pttti *)input_buf32(regs),
+                                (struct pttto *)output_buf32(regs));
+        break;
+
+    case TCG_ShutdownPreBootInterface:
+        regs->eax = shutdown_preboot_interface();
+        break;
+
+    case TCG_HashLogEvent:
+        regs->eax = hash_log_event((struct hlei*)input_buf32(regs),
+                                   (struct hleo*)output_buf32(regs));
+        break;
+
+    case TCG_HashAll:
+        regs->eax =
+            hash_all((struct hai*)input_buf32(regs),
+                     (u8 *)output_buf32(regs));
+        break;
+
+    case TCG_TSS:
+        regs->eax = tss((struct ti*)input_buf32(regs),
+                    (struct to*)output_buf32(regs));
+        break;
+
+    case TCG_CompactHashLogExtendEvent:
+        regs->eax =
+          compact_hash_log_extend_event((u8 *)input_buf32(regs),
+                                        regs->esi,
+                                        regs->ecx,
+                                        regs->edx,
+                                        &regs->edx);
+        break;
+
+    default:
+        set_cf(regs, 1);
+    }
+
+    return;
+}
+
+/*
+ * Add a measurement to the log; the data at data_seg:data/length are
+ * appended to the TCG_PCClientPCREventStruct
+ *
+ * Input parameters:
+ *  pcrIndex   : which PCR to extend
+ *  event_type : type of event; specs section on 'Event Types'
+ *  info       : pointer to info (e.g., string) to be added to log as-is
+ *  info_length: length of the info
+ *  data       : pointer to the data (i.e., string) to be added to the log
+ *  data_length: length of the data
+ */
+static u32
+tpm_add_measurement_to_log(u32 pcrIndex, u32 event_type,
+                           const char *info, u32 info_length,
+                           const u8 *data, u32 data_length)
+{
+    u32 rc = 0;
+    struct hleeo hleeo;
+    u8 _pcpes[offsetof(struct pcpes, event) + 400];
+    struct pcpes *pcpes = (struct pcpes *)_pcpes;
+
+    if (info_length < sizeof(_pcpes) - offsetof(struct pcpes, event)) {
+
+        pcpes->pcrindex      = pcrIndex;
+        pcpes->eventtype     = event_type;
+        memset(&pcpes->digest, 0x0, sizeof(pcpes->digest));
+        pcpes->eventdatasize = info_length;
+        memcpy(&pcpes->event, info, info_length);
+
+        struct hleei_short hleei = {
+            .ipblength   = sizeof(hleei),
+            .hashdataptr = data,
+            .hashdatalen = data_length,
+            .pcrindex    = pcrIndex,
+            .logdataptr  = _pcpes,
+            .logdatalen  = info_length + offsetof(struct pcpes, event),
+        };
+
+        rc = hash_log_extend_event(&hleei, &hleeo);
+    } else {
+        rc = TCG_GENERAL_ERROR;
+    }
+
+    return rc;
+}
+
+
+/*
+ * Add a measurement to the list of measurements
+ * pcrIndex   : PCR to be extended
+ * event_type : type of event; specs section on 'Event Types'
+ * data       : additional parameter; used as parameter for
+ *              'action index'
+ */
+static u32
+tpm_add_measurement(u32 pcrIndex,
+                    u16 event_type,
+                    const char *string)
+{
+    u32 rc;
+    u32 len;
+
+    switch (event_type) {
+    case EV_SEPARATOR:
+        len = sizeof(evt_separator);
+        rc = tpm_add_measurement_to_log(pcrIndex, event_type,
+                                        (char *)NULL, 0,
+                                        (u8 *)evt_separator, len);
+        break;
+
+    case EV_ACTION:
+        rc = tpm_add_measurement_to_log(pcrIndex, event_type,
+                                        string, strlen(string),
+                                        (u8 *)string, strlen(string));
+        break;
+
+    default:
+        rc = TCG_INVALID_INPUT_PARA;
+    }
+
+    return rc;
+}
+
+
+static u32
+tpm_calling_int19h(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    return tpm_add_measurement(4, EV_ACTION,
+                               "Calling INT 19h");
+}
+
+/*
+ * Add event separators for PCRs 0 to 7; specs on 'Measuring Boot Events'
+ */
+u32
+tpm_add_event_separators(void)
+{
+    u32 rc;
+    u32 pcrIndex = 0;
+
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    while (pcrIndex <= 7) {
+        rc = tpm_add_measurement(pcrIndex, EV_SEPARATOR, NULL);
+        if (rc)
+            break;
+        pcrIndex ++;
+    }
+
+    return rc;
+}
+
+
+/*
+ * Add a measurement regarding the boot device (CDRom, Floppy, HDD) to
+ * the list of measurements.
+ */
+static u32
+tpm_add_bootdevice(u32 bootcd, u32 bootdrv)
+{
+    const char *string;
+
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    switch (bootcd) {
+    case 0:
+        switch (bootdrv) {
+        case 0:
+            string = "Booting BCV device 00h (Floppy)";
+            break;
+
+        case 0x80:
+            string = "Booting BCV device 80h (HDD)";
+            break;
+
+        default:
+            string = "Booting unknown device";
+            break;
+        }
+
+        break;
+
+    default:
+        string = "Booting from CD ROM device";
+    }
+
+    return tpm_add_measurement_to_log(4, EV_ACTION,
+                                      string, strlen(string),
+                                      (u8 *)string, strlen(string));
+}
+
+
+/*
+ * Add measurement to the log about option rom scan
+ */
+u32
+tpm_start_option_rom_scan(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    return tpm_add_measurement(2, EV_ACTION,
+                               "Start Option ROM Scan");
+}
+
+
+/*
+ * Add measurement to the log about an option rom
+ */
+u32
+tpm_option_rom(const void *addr, u32 len)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    u32 rc;
+    struct pcctes_romex pcctes = {
+        .eventid = 7,
+        .eventdatasize = sizeof(u16) + sizeof(u16) + SHA1_BUFSIZE,
+    };
+
+    rc = sha1((const u8 *)addr, len, pcctes.digest);
+    if (rc)
+        return rc;
+
+    return tpm_add_measurement_to_log(2,
+                                      EV_EVENT_TAG,
+                                      (const char *)&pcctes, sizeof(pcctes),
+                                      (u8 *)&pcctes, sizeof(pcctes));
+}
+
+
+u32
+tpm_smbios_measure(void)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    u32 rc;
+    struct pcctes pcctes = {
+        .eventid = 1,
+        .eventdatasize = SHA1_BUFSIZE,
+    };
+    struct smbios_entry_point *sep = SMBiosAddr;
+
+    dprintf(DEBUG_tcg, "TCGBIOS: SMBIOS at %p\n", sep);
+
+    if (!sep)
+        return 0;
+
+    rc = sha1((const u8 *)sep->structure_table_address,
+              sep->structure_table_length, pcctes.digest);
+    if (rc)
+        return rc;
+
+    return tpm_add_measurement_to_log(1,
+                                      EV_EVENT_TAG,
+                                      (const char *)&pcctes, sizeof(pcctes),
+                                      (u8 *)&pcctes, sizeof(pcctes));
+}
+
+
+/*
+ * Add a measurement related to Initial Program Loader to the log.
+ * Creates two log entries.
+ *
+ * Input parameter:
+ *  bootcd : 0: MBR of hdd, 1: boot image, 2: boot catalog of El Torito
+ *  addr   : address where the IP data are located
+ *  length : IP data length in bytes
+ */
+static u32
+tpm_ipl(enum ipltype bootcd, const u8 *addr, u32 length)
+{
+    u32 rc;
+    const char *string;
+
+    switch (bootcd) {
+    case IPL_EL_TORITO_1:
+        /* specs: see section 'El Torito' */
+        string = "EL TORITO IPL";
+        rc = tpm_add_measurement_to_log(4, EV_IPL,
+                                        string, strlen(string),
+                                        addr, length);
+        break;
+
+    case IPL_EL_TORITO_2:
+        /* specs: see section 'El Torito' */
+        string = "BOOT CATALOG";
+        rc = tpm_add_measurement_to_log(5, EV_IPL_PARTITION_DATA,
+                                        string, strlen(string),
+                                        addr, length);
+        break;
+
+    default:
+        /* specs: see section 'Hard Disk Device or Hard Disk-Like Devices' */
+        /* equivalent to: dd if=/dev/hda ibs=1 count=440 | sha1sum */
+        string = "MBR";
+        rc = tpm_add_measurement_to_log(4, EV_IPL,
+                                        string, strlen(string),
+                                        addr, 0x1b8);
+
+        if (rc)
+            break;
+
+        /* equivalent to: dd if=/dev/hda ibs=1 count=72 skip=440 | sha1sum */
+        string = "MBR PARTITION_TABLE";
+        rc = tpm_add_measurement_to_log(5, EV_IPL_PARTITION_DATA,
+                                        string, strlen(string),
+                                        addr + 0x1b8, 0x48);
+    }
+
+    return rc;
+}
+
+u32
+tpm_add_bcv(u32 bootdrv, const u8 *addr, u32 length)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    u32 rc = tpm_add_bootdevice(0, bootdrv);
+    if (rc)
+        return rc;
+
+    return tpm_ipl(IPL_BCV, addr, length);
+}
+
+u32
+tpm_add_cdrom(u32 bootdrv, const u8 *addr, u32 length)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    u32 rc = tpm_add_bootdevice(1, bootdrv);
+    if (rc)
+        return rc;
+
+    return tpm_ipl(IPL_EL_TORITO_1, addr, length);
+}
+
+u32
+tpm_add_cdrom_catalog(const u8 *addr, u32 length)
+{
+    if (!CONFIG_TCGBIOS)
+        return 0;
+
+    if (!has_working_tpm())
+        return TCG_GENERAL_ERROR;
+
+    u32 rc = tpm_add_bootdevice(1, 0);
+    if (rc)
+        return rc;
+
+    return tpm_ipl(IPL_EL_TORITO_2, addr, length);
+}
+
+void
+tpm_s3_resume(void)
+{
+    u32 rc;
+    u32 returnCode;
+
+    if (!CONFIG_TCGBIOS)
+        return;
+
+    if (!has_working_tpm())
+        return;
+
+    dprintf(DEBUG_tcg, "TCGBIOS: Resuming with TPM_Startup(ST_STATE)\n");
+
+    rc = build_and_send_cmd(0, TPM_ORD_Startup,
+                            Startup_ST_STATE, sizeof(Startup_ST_STATE),
+                            NULL, 0, &returnCode, TPM_DURATION_TYPE_SHORT);
+
+    dprintf(DEBUG_tcg, "TCGBIOS: ReturnCode from TPM_Startup = 0x%08x\n",
+            returnCode);
+
+    if (rc || returnCode)
+        goto err_exit;
+
+    return;
+
+err_exit:
+    dprintf(DEBUG_tcg, "TCGBIOS: TPM malfunctioning (line %d).\n", __LINE__);
+
+    tpm_state.tpm_working = 0;
+}
diff --git a/qemu/roms/seabios/src/tcgbios.h b/qemu/roms/seabios/src/tcgbios.h
new file mode 100644
index 000000000..4b7eaabef
--- /dev/null
+++ b/qemu/roms/seabios/src/tcgbios.h
@@ -0,0 +1,375 @@
+#ifndef TCGBIOS_H
+#define TCGBIOS_H
+
+#include "types.h"
+
+/* Define for section 12.3 */
+#define TCG_PC_OK                       0x0
+#define TCG_PC_TPMERROR                 0x1
+#define TCG_PC_LOGOVERFLOW              0x2
+#define TCG_PC_UNSUPPORTED              0x3
+
+#define TPM_ALG_SHA                     0x4
+
+#define TCG_MAGIC                       0x41504354L
+#define TCG_VERSION_MAJOR               1
+#define TCG_VERSION_MINOR               2
+
+#define TPM_OK                          0x0
+#define TPM_RET_BASE                    0x1
+#define TCG_GENERAL_ERROR               (TPM_RET_BASE + 0x0)
+#define TCG_TPM_IS_LOCKED               (TPM_RET_BASE + 0x1)
+#define TCG_NO_RESPONSE                 (TPM_RET_BASE + 0x2)
+#define TCG_INVALID_RESPONSE            (TPM_RET_BASE + 0x3)
+#define TCG_INVALID_ACCESS_REQUEST      (TPM_RET_BASE + 0x4)
+#define TCG_FIRMWARE_ERROR              (TPM_RET_BASE + 0x5)
+#define TCG_INTEGRITY_CHECK_FAILED      (TPM_RET_BASE + 0x6)
+#define TCG_INVALID_DEVICE_ID           (TPM_RET_BASE + 0x7)
+#define TCG_INVALID_VENDOR_ID           (TPM_RET_BASE + 0x8)
+#define TCG_UNABLE_TO_OPEN              (TPM_RET_BASE + 0x9)
+#define TCG_UNABLE_TO_CLOSE             (TPM_RET_BASE + 0xa)
+#define TCG_RESPONSE_TIMEOUT            (TPM_RET_BASE + 0xb)
+#define TCG_INVALID_COM_REQUEST         (TPM_RET_BASE + 0xc)
+#define TCG_INVALID_ADR_REQUEST         (TPM_RET_BASE + 0xd)
+#define TCG_WRITE_BYTE_ERROR            (TPM_RET_BASE + 0xe)
+#define TCG_READ_BYTE_ERROR             (TPM_RET_BASE + 0xf)
+#define TCG_BLOCK_WRITE_TIMEOUT         (TPM_RET_BASE + 0x10)
+#define TCG_CHAR_WRITE_TIMEOUT          (TPM_RET_BASE + 0x11)
+#define TCG_CHAR_READ_TIMEOUT           (TPM_RET_BASE + 0x12)
+#define TCG_BLOCK_READ_TIMEOUT          (TPM_RET_BASE + 0x13)
+#define TCG_TRANSFER_ABORT              (TPM_RET_BASE + 0x14)
+#define TCG_INVALID_DRV_FUNCTION        (TPM_RET_BASE + 0x15)
+#define TCG_OUTPUT_BUFFER_TOO_SHORT     (TPM_RET_BASE + 0x16)
+#define TCG_FATAL_COM_ERROR             (TPM_RET_BASE + 0x17)
+#define TCG_INVALID_INPUT_PARA          (TPM_RET_BASE + 0x18)
+#define TCG_TCG_COMMAND_ERROR           (TPM_RET_BASE + 0x19)
+#define TCG_INTERFACE_SHUTDOWN          (TPM_RET_BASE + 0x20)
+//define TCG_PC_UNSUPPORTED             (TPM_RET_BASE + 0x21)
+#define TCG_PC_TPM_NOT_PRESENT          (TPM_RET_BASE + 0x22)
+#define TCG_PC_TPM_DEACTIVATED          (TPM_RET_BASE + 0x23)
+
+
+#define TPM_ORD_SelfTestFull             0x00000050
+#define TPM_ORD_ForceClear               0x0000005d
+#define TPM_ORD_GetCapability            0x00000065
+#define TPM_ORD_PhysicalEnable           0x0000006f
+#define TPM_ORD_PhysicalDisable          0x00000070
+#define TPM_ORD_SetOwnerInstall          0x00000071
+#define TPM_ORD_PhysicalSetDeactivated   0x00000072
+#define TPM_ORD_Startup                  0x00000099
+#define TPM_ORD_PhysicalPresence         0x4000000a
+#define TPM_ORD_Extend                   0x00000014
+#define TPM_ORD_SHA1Start                0x000000a0
+#define TPM_ORD_SHA1Update               0x000000a1
+#define TPM_ORD_SHA1Complete             0x000000a2
+#define TSC_ORD_ResetEstablishmentBit    0x4000000b
+
+
+#define TPM_ST_CLEAR                     0x1
+#define TPM_ST_STATE                     0x2
+#define TPM_ST_DEACTIVATED               0x3
+
+
+/* TPM command error codes */
+#define TPM_INVALID_POSTINIT             0x26
+#define TPM_BAD_LOCALITY                 0x3d
+
+/* TPM command tags */
+#define TPM_TAG_RQU_CMD                  0x00c1
+
+/* interrupt identifiers (al register) */
+enum irq_ids {
+    TCG_StatusCheck = 0,
+    TCG_HashLogExtendEvent = 1,
+    TCG_PassThroughToTPM = 2,
+    TCG_ShutdownPreBootInterface = 3,
+    TCG_HashLogEvent = 4,
+    TCG_HashAll = 5,
+    TCG_TSS = 6,
+    TCG_CompactHashLogExtendEvent = 7,
+};
+
+/* event types: 10.4.1 / table 11 */
+#define EV_POST_CODE             1
+#define EV_SEPARATOR             4
+#define EV_ACTION                5
+#define EV_EVENT_TAG             6
+#define EV_COMPACT_HASH         12
+#define EV_IPL                  13
+#define EV_IPL_PARTITION_DATA   14
+
+
+#define STATUS_FLAG_SHUTDOWN        (1 << 0)
+
+#define SHA1_BUFSIZE                20
+
+
+struct iovec
+{
+    size_t length;
+    void   *data;
+};
+
+
+/* Input and Output blocks for the TCG BIOS commands */
+
+struct hleei_short
+{
+    u16   ipblength;
+    u16   reserved;
+    const void *hashdataptr;
+    u32   hashdatalen;
+    u32   pcrindex;
+    const void *logdataptr;
+    u32   logdatalen;
+} PACKED;
+
+
+struct hleei_long
+{
+    u16   ipblength;
+    u16   reserved;
+    void *hashdataptr;
+    u32   hashdatalen;
+    u32   pcrindex;
+    u32   reserved2;
+    void *logdataptr;
+    u32   logdatalen;
+} PACKED;
+
+
+struct hleeo
+{
+    u16    opblength;
+    u16    reserved;
+    u32    eventnumber;
+    u8     digest[SHA1_BUFSIZE];
+} PACKED;
+
+
+struct pttti
+{
+    u16    ipblength;
+    u16    reserved;
+    u16    opblength;
+    u16    reserved2;
+    u8     tpmopin[0];
+} PACKED;
+
+
+struct pttto
+{
+    u16    opblength;
+    u16    reserved;
+    u8     tpmopout[0];
+};
+
+
+struct hlei
+{
+    u16    ipblength;
+    u16    reserved;
+    const void  *hashdataptr;
+    u32    hashdatalen;
+    u32    pcrindex;
+    u32    logeventtype;
+    const void  *logdataptr;
+    u32    logdatalen;
+} PACKED;
+
+
+struct hleo
+{
+    u16    opblength;
+    u16    reserved;
+    u32    eventnumber;
+} PACKED;
+
+
+struct hai
+{
+    u16    ipblength;
+    u16    reserved;
+    const void  *hashdataptr;
+    u32    hashdatalen;
+    u32    algorithmid;
+} PACKED;
+
+
+struct ti
+{
+    u16    ipblength;
+    u16    reserved;
+    u16    opblength;
+    u16    reserved2;
+    u8     tssoperandin[0];
+} PACKED;
+
+
+struct to
+{
+    u16    opblength;
+    u16    reserved;
+    u8     tssoperandout[0];
+} PACKED;
+
+
+struct pcpes
+{
+    u32    pcrindex;
+    u32    eventtype;
+    u8     digest[SHA1_BUFSIZE];
+    u32    eventdatasize;
+    u32    event;
+} PACKED;
+
+struct pcctes
+{
+    u32 eventid;
+    u32 eventdatasize;
+    u8  digest[SHA1_BUFSIZE];
+} PACKED;
+
+struct pcctes_romex
+{
+    u32 eventid;
+    u32 eventdatasize;
+    u16 reserved;
+    u16 pfa;
+    u8  digest[SHA1_BUFSIZE];
+} PACKED;
+
+
+#define TPM_REQ_HEADER \
+    u16    tag; \
+    u32    totlen; \
+    u32    ordinal;
+
+#define TPM_REQ_HEADER_SIZE  (sizeof(u16) + sizeof(u32) + sizeof(u32))
+
+#define TPM_RSP_HEADER \
+    u16    tag; \
+    u32    totlen; \
+    u32    errcode;
+
+#define TPM_RSP_HEADER_SIZE  (sizeof(u16) + sizeof(u32) + sizeof(u32))
+
+struct tpm_req_header {
+    TPM_REQ_HEADER;
+} PACKED;
+
+
+struct tpm_rsp_header {
+    TPM_RSP_HEADER;
+} PACKED;
+
+
+struct tpm_req_extend {
+    TPM_REQ_HEADER
+    u32    pcrindex;
+    u8     digest[SHA1_BUFSIZE];
+} PACKED;
+
+
+struct tpm_rsp_extend {
+    TPM_RSP_HEADER
+    u8     digest[SHA1_BUFSIZE];
+} PACKED;
+
+
+struct tpm_req_getcap_perm_flags {
+    TPM_REQ_HEADER
+    u32    capArea;
+    u32    subCapSize;
+    u32    subCap;
+} PACKED;
+
+
+struct tpm_permanent_flags {
+    u16    tag;
+    u8     flags[20];
+} PACKED;
+
+
+enum permFlagsIndex {
+    PERM_FLAG_IDX_DISABLE = 0,
+    PERM_FLAG_IDX_OWNERSHIP,
+    PERM_FLAG_IDX_DEACTIVATED,
+    PERM_FLAG_IDX_READPUBEK,
+    PERM_FLAG_IDX_DISABLEOWNERCLEAR,
+    PERM_FLAG_IDX_ALLOW_MAINTENANCE,
+    PERM_FLAG_IDX_PHYSICAL_PRESENCE_LIFETIME_LOCK,
+    PERM_FLAG_IDX_PHYSICAL_PRESENCE_HW_ENABLE,
+};
+
+
+struct tpm_res_getcap_perm_flags {
+    TPM_RSP_HEADER
+    u32    size;
+    struct tpm_permanent_flags perm_flags;
+} PACKED;
+
+
+struct tpm_res_getcap_ownerauth {
+    TPM_RSP_HEADER
+    u32    size;
+    u8     flag;
+} PACKED;
+
+
+struct tpm_res_getcap_timeouts {
+    TPM_RSP_HEADER
+    u32    size;
+    u32    timeouts[4];
+} PACKED;
+
+
+struct tpm_res_getcap_durations {
+    TPM_RSP_HEADER
+    u32    size;
+    u32    durations[3];
+} PACKED;
+
+
+struct tpm_res_sha1start {
+    TPM_RSP_HEADER
+    u32    max_num_bytes;
+} PACKED;
+
+
+struct tpm_res_sha1complete {
+    TPM_RSP_HEADER
+    u8     hash[20];
+} PACKED;
+
+struct pttti_extend {
+    struct pttti pttti;
+    struct tpm_req_extend req;
+} PACKED;
+
+
+struct pttto_extend {
+    struct pttto pttto;
+    struct tpm_rsp_extend rsp;
+} PACKED;
+
+
+enum ipltype {
+    IPL_BCV = 0,
+    IPL_EL_TORITO_1,
+    IPL_EL_TORITO_2
+};
+
+
+struct bregs;
+void tpm_interrupt_handler32(struct bregs *regs);
+
+void tpm_setup(void);
+void tpm_prepboot(void);
+void tpm_s3_resume(void);
+u32 tpm_add_bcv(u32 bootdrv, const u8 *addr, u32 length);
+u32 tpm_add_cdrom(u32 bootdrv, const u8 *addr, u32 length);
+u32 tpm_add_cdrom_catalog(const u8 *addr, u32 length);
+u32 tpm_option_rom(const void *addr, u32 len);
+
+#endif /* TCGBIOS_H */
diff --git a/qemu/roms/seabios/src/types.h b/qemu/roms/seabios/src/types.h
index 097372cdb..19d9f6c14 100644
--- a/qemu/roms/seabios/src/types.h
+++ b/qemu/roms/seabios/src/types.h
@@ -70,7 +70,7 @@ extern void __force_link_error__only_in_16bit(void) __noreturn;
 # define VARFSEG __section(".discard.varfseg." UNIQSEC) __VISIBLE __weak
 // Designate a variable at a specific address in the f-segment.
 # define VARFSEGFIXED(addr) __section(".discard.varfixed." UNIQSEC) __VISIBLE __weak
-// Verify a variable is only accessable via 32bit "init" functions
+// Verify a variable is only accessible via 32bit "init" functions
 # define VARVERIFY32INIT __section(".discard.varinit." UNIQSEC)
 // Designate top-level assembler as 16bit only.
 # define ASM16(code) __ASM(code)
diff --git a/qemu/roms/seabios/src/util.h b/qemu/roms/seabios/src/util.h
index 09bb8a9f3..cba3359d5 100644
--- a/qemu/roms/seabios/src/util.h
+++ b/qemu/roms/seabios/src/util.h
@@ -43,17 +43,17 @@ void enable_bootsplash(void);
 void disable_bootsplash(void);
 
 // cdrom.c
-extern u8 CDRom_locks[];
 extern struct eltorito_s CDEmu;
 extern struct drive_s *cdemu_drive_gf;
 struct disk_op_s;
-int process_cdemu_op(struct disk_op_s *op);
+int cdemu_process_op(struct disk_op_s *op);
 void cdrom_prepboot(void);
 int cdrom_boot(struct drive_s *drive_g);
 
 // clock.c
 void clock_setup(void);
 void handle_1583(struct bregs *regs);
+void clock_poll_irq(void);
 u32 irqtimer_calc_ticks(u32 count);
 u32 irqtimer_calc(u32 msecs);
 int irqtimer_check(u32 end);
@@ -75,6 +75,7 @@ u32 find_resume_vector(void);
 void acpi_reboot(void);
 void find_acpi_features(void);
 extern struct smbios_entry_point *SMBiosAddr;
+struct smbios_entry_point *get_smbios_entry_point();
 void copy_smbios(void *pos);
 void display_uuid(void);
 void copy_table(void *pos);
@@ -104,6 +105,9 @@ void mptable_setup(void);
 // fw/mtrr.c
 void mtrr_setup(void);
 
+// fw/multiboot.c
+void multiboot_init(void);
+
 // fw/pciinit.c
 extern const u8 pci_irqs[4];
 void pci_setup(void);
@@ -139,15 +143,15 @@ extern struct floppy_ext_dbt_s diskette_param_table2;
 void floppy_setup(void);
 struct drive_s *init_floppy(int floppyid, int ftype);
 int find_floppy_type(u32 size);
-int process_floppy_op(struct disk_op_s *op);
+int floppy_process_op(struct disk_op_s *op);
 void floppy_tick(void);
 
 // hw/ramdisk.c
 void ramdisk_setup(void);
-int process_ramdisk_op(struct disk_op_s *op);
+int ramdisk_process_op(struct disk_op_s *op);
 
 // hw/sdcard.c
-int process_sdcard_op(struct disk_op_s *op);
+int sdcard_process_op(struct disk_op_s *op);
 void sdcard_setup(void);
 
 // hw/timer.c
@@ -232,6 +236,6 @@ void vgahook_setup(struct pci_device *pci);
 
 
 // version (auto generated file out/version.c)
-extern const char VERSION[];
+extern const char VERSION[], BUILDINFO[];
 
 #endif // util.h
diff --git a/qemu/roms/seabios/src/version.c b/qemu/roms/seabios/src/version.c
new file mode 100644
index 000000000..a8a58cf09
--- /dev/null
+++ b/qemu/roms/seabios/src/version.c
@@ -0,0 +1,5 @@
+// Place build generated version into a C variable
+#include "autoversion.h"
+
+char VERSION[] = BUILD_VERSION;
+char BUILDINFO[] = BUILD_TOOLS;
diff --git a/qemu/roms/seabios/src/vgahooks.c b/qemu/roms/seabios/src/vgahooks.c
index 6a4acfeaf..48efb086c 100644
--- a/qemu/roms/seabios/src/vgahooks.c
+++ b/qemu/roms/seabios/src/vgahooks.c
@@ -124,7 +124,7 @@ getAMDRamSpeed(void)
 
 /* int 0x15 - 5f18
 
-   ECX = unknown/dont care
+   ECX = unknown/don't care
    EBX[3..0] Frame Buffer Size 2^N MiB
    EBX[7..4] Memory speed:
        0: SDR  66Mhz
diff --git a/qemu/roms/seabios/src/x86.h b/qemu/roms/seabios/src/x86.h
index 7798b1c17..53378e9ed 100644
--- a/qemu/roms/seabios/src/x86.h
+++ b/qemu/roms/seabios/src/x86.h
@@ -75,14 +75,22 @@ static inline void __cpuid(u32 index, u32 *eax, u32 *ebx, u32 *ecx, u32 *edx)
         : "0" (index));
 }
 
-static inline u32 getcr0(void) {
+static inline u32 cr0_read(void) {
     u32 cr0;
     asm("movl %%cr0, %0" : "=r"(cr0));
     return cr0;
 }
-static inline void setcr0(u32 cr0) {
+static inline void cr0_write(u32 cr0) {
     asm("movl %0, %%cr0" : : "r"(cr0));
 }
+static inline void cr0_mask(u32 off, u32 on) {
+    cr0_write((cr0_read() & ~off) | on);
+}
+static inline u16 cr0_vm86_read(void) {
+    u16 cr0;
+    asm("smsww %0" : "=r"(cr0));
+    return cr0;
+}
 
 static inline u64 rdmsr(u32 index)
 {
@@ -124,6 +132,13 @@ static inline u32 getesp(void) {
     return esp;
 }
 
+static inline u32 rol(u32 val, u16 rol) {
+    u32 res;
+    asm volatile("roll %%cl, %%eax"
+                 : "=a" (res) : "a" (val), "c" (rol));
+    return res;
+}
+
 static inline void outb(u8 value, u16 port) {
     __asm__ __volatile__("outb %b0, %w1" : : "a"(value), "Nd"(port));
 }
@@ -175,6 +190,14 @@ static inline void outsl(u16 port, u32 *data, u32 count) {
                  : "+c"(count), "+S"(data) : "d"(port) : "memory");
 }
 
+/* Compiler barrier is enough as an x86 CPU does not reorder reads or writes */
+static inline void smp_rmb(void) {
+    barrier();
+}
+static inline void smp_wmb(void) {
+    barrier();
+}
+
 static inline void writel(void *addr, u32 val) {
     barrier();
     *(volatile u32 *)addr = val;
diff --git a/qemu/roms/seabios/vgasrc/Kconfig b/qemu/roms/seabios/vgasrc/Kconfig
index 91d590ae2..f5098a4bd 100644
--- a/qemu/roms/seabios/vgasrc/Kconfig
+++ b/qemu/roms/seabios/vgasrc/Kconfig
@@ -58,6 +58,25 @@ menu "VGA ROM"
     endchoice
 
     choice
+        depends on VGA_BOCHS
+        prompt "bochs vga variant"
+        default VGA_BOCHS_STDVGA
+
+        config VGA_BOCHS_STDVGA
+            bool "qemu stdvga / bochs svga"
+
+        config VGA_BOCHS_VMWARE
+            bool "qemu vmware svga"
+
+        config VGA_BOCHS_QXL
+            bool "qemu qxl vga"
+
+        config VGA_BOCHS_VIRTIO
+            bool "qemu virtio vga"
+
+    endchoice
+
+    choice
         depends on VGA_GEODEGX2 || VGA_GEODELX
         prompt "Output Mode"
         default VGA_OUTPUT_CRT
@@ -141,7 +160,10 @@ menu "VGA ROM"
         hex
         prompt "PCI Vendor ID" if OVERRIDE_PCI_ID
         default 0x1013 if VGA_CIRRUS
-        default 0x1234 if VGA_BOCHS
+        default 0x1234 if VGA_BOCHS_STDVGA
+        default 0x15ad if VGA_BOCHS_VMWARE
+        default 0x1b36 if VGA_BOCHS_QXL
+        default 0x1af4 if VGA_BOCHS_VIRTIO
         default 0x100b if VGA_GEODEGX2
         default 0x1022 if VGA_GEODELX
         default 0x0000
@@ -153,7 +175,10 @@ menu "VGA ROM"
         hex
         prompt "PCI Vendor ID" if OVERRIDE_PCI_ID
         default 0x00b8 if VGA_CIRRUS
-        default 0x1111 if VGA_BOCHS
+        default 0x1111 if VGA_BOCHS_STDVGA
+        default 0x0405 if VGA_BOCHS_VMWARE
+        default 0x0100 if VGA_BOCHS_QXL
+        default 0x1050 if VGA_BOCHS_VIRTIO
         default 0x0030 if VGA_GEODEGX2
         default 0x2081 if VGA_GEODELX
         default 0x0000
diff --git a/qemu/roms/seabios/vgasrc/geodevga.h b/qemu/roms/seabios/vgasrc/geodevga.h
index 61d78084d..c99f54bb1 100644
--- a/qemu/roms/seabios/vgasrc/geodevga.h
+++ b/qemu/roms/seabios/vgasrc/geodevga.h
@@ -2,7 +2,7 @@
 //
 // Copyright (C) 2009 Chris Kindt
 //
-// Writen for Google Summer of Code 2009 for the coreboot project
+// Written for Google Summer of Code 2009 for the coreboot project
 //
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
diff --git a/qemu/roms/seabios/vgasrc/vgabios.c b/qemu/roms/seabios/vgasrc/vgabios.c
index 4aa50e1c1..f07e85bd3 100644
--- a/qemu/roms/seabios/vgasrc/vgabios.c
+++ b/qemu/roms/seabios/vgasrc/vgabios.c
@@ -304,6 +304,12 @@ vga_set_mode(int mode, int flags)
         SET_BDA(video_mode, 0xff);
     SET_BDA_EXT(vbe_mode, mode | (flags & MF_VBEFLAGS));
     SET_BDA_EXT(vgamode_offset, (u32)vmode_g);
+    if (CONFIG_VGA_ALLOCATE_EXTRA_STACK)
+        // Disable extra stack if it appears a modern OS is in use.
+        // This works around bugs in some versions of Windows (Vista
+        // and possibly later) when the stack is in the e-segment.
+        MASK_BDA_EXT(flags, BF_EXTRA_STACK
+                     , (flags & MF_LEGACY) ? BF_EXTRA_STACK : 0);
     if (memmodel == MM_TEXT) {
         SET_BDA(video_cols, width);
         SET_BDA(video_rows, height-1);
diff --git a/qemu/roms/seabios/vgasrc/vgabios.h b/qemu/roms/seabios/vgasrc/vgabios.h
index fd796f2e6..831f69407 100644
--- a/qemu/roms/seabios/vgasrc/vgabios.h
+++ b/qemu/roms/seabios/vgasrc/vgabios.h
@@ -62,7 +62,8 @@ struct gfx_op {
 #define GO_MEMSET  3
 #define GO_MEMMOVE 4
 
-// Custom internal storage in BDA
+// Custom internal storage in BDA (don't change here without also
+// updating vgaentry.S)
 #define VGA_CUSTOM_BDA 0xb9
 
 struct vga_bda_s {
@@ -74,6 +75,7 @@ struct vga_bda_s {
 #define BF_PM_MASK      0x0f
 #define BF_EMULATE_TEXT 0x10
 #define BF_SWCURSOR     0x20
+#define BF_EXTRA_STACK  0x40
 
 #define GET_BDA_EXT(var) \
     GET_FARVAR(SEG_BDA, ((struct vga_bda_s *)VGA_CUSTOM_BDA)->var)
diff --git a/qemu/roms/seabios/vgasrc/vgaentry.S b/qemu/roms/seabios/vgasrc/vgaentry.S
index d9ebdb93c..53be2b38c 100644
--- a/qemu/roms/seabios/vgasrc/vgaentry.S
+++ b/qemu/roms/seabios/vgasrc/vgaentry.S
@@ -86,14 +86,23 @@ entry_10:
         ENTRY_ARG_VGA handle_10
         iretw
 
+#define VGA_CUSTOM_BDA_FLAGS 0xb9
+#define BF_EXTRA_STACK 0x40
+
         // Entry point using extra stack
         DECLFUNC entry_10_extrastack
 entry_10_extrastack:
         cli
         cld
-        pushw %ds               // Set %ds:%eax to space on ExtraStack
+        pushw %ds
         pushl %eax
-        movw %cs:ExtraStackSeg, %ds
+
+        movw $SEG_BDA, %ax      // Check if extra stack is enabled
+        movw %ax, %ds
+        testb $BF_EXTRA_STACK, VGA_CUSTOM_BDA_FLAGS
+        jz 1f
+
+        movw %cs:ExtraStackSeg, %ds // Set %ds:%eax to space on ExtraStack
         movl $(CONFIG_VGA_EXTRA_STACK_SIZE-PUSHBREGS_size-16), %eax
         SAVEBREGS_POP_DSEAX     // Save registers on extra stack
         movl %esp, PUSHBREGS_size+8(%eax)
@@ -116,6 +125,11 @@ entry_10_extrastack:
         RESTOREBREGS_DSEAX
         iretw
 
+1:      // Use regular entry point if the extra stack is disabled
+        popl %eax
+        popw %ds
+        jmp entry_10
+
         // Timer irq handling
         DECLFUNC entry_timer_hook
 entry_timer_hook:
diff --git a/qemu/roms/seabios/vgasrc/vgafb.c b/qemu/roms/seabios/vgasrc/vgafb.c
index 1a94fcf70..5d1ecc936 100644
--- a/qemu/roms/seabios/vgasrc/vgafb.c
+++ b/qemu/roms/seabios/vgasrc/vgafb.c
@@ -273,28 +273,22 @@ gfx_direct(struct gfx_op *op)
     int bypp = DIV_ROUND_UP(depth, 8);
     void *dest_far = (fb + op->displaystart + op->y * op->linelength
                       + op->x * bypp);
+    u8 data[64];
+    int i;
     switch (op->op) {
     default:
-    case GO_READ8: {
-        u8 data[64];
+    case GO_READ8:
         memcpy_high(MAKE_FLATPTR(GET_SEG(SS), data), dest_far, bypp * 8);
-        int i;
         for (i=0; i<8; i++)
             op->pixels[i] = reverse_color(depth, *(u32*)&data[i*bypp]);
         break;
-    }
-    case GO_WRITE8: {
-        u8 data[64];
-        int i;
+    case GO_WRITE8:
         for (i=0; i<8; i++)
             *(u32*)&data[i*bypp] = get_color(depth, op->pixels[i]);
         memcpy_high(dest_far, MAKE_FLATPTR(GET_SEG(SS), data), bypp * 8);
         break;
-    }
-    case GO_MEMSET: {
+    case GO_MEMSET: ;
         u32 color = get_color(depth, op->pixels[0]);
-        u8 data[64];
-        int i;
         for (i=0; i<8; i++)
             *(u32*)&data[i*bypp] = color;
         memcpy_high(dest_far, MAKE_FLATPTR(GET_SEG(SS), data), bypp * 8);
@@ -303,7 +297,6 @@ gfx_direct(struct gfx_op *op)
             memcpy_high(dest_far + op->linelength * i
                         , dest_far, op->xlen * bypp);
         break;
-    }
     case GO_MEMMOVE: ;
         void *src_far = (fb + op->displaystart + op->srcy * op->linelength
                          + op->x * bypp);
diff --git a/qemu/roms/seabios/vgasrc/vgainit.c b/qemu/roms/seabios/vgasrc/vgainit.c
index 8d1226182..40997dbbd 100644
--- a/qemu/roms/seabios/vgasrc/vgainit.c
+++ b/qemu/roms/seabios/vgasrc/vgainit.c
@@ -150,6 +150,7 @@ vga_post(struct bregs *regs)
 {
     serial_debug_preinit();
     dprintf(1, "Start SeaVGABIOS (version %s)\n", VERSION);
+    dprintf(1, "VGABUILD: %s\n", BUILDINFO);
     debug_enter(regs, DEBUG_VGA_POST);
 
     if (CONFIG_VGA_PCI && !GET_GLOBAL(HaveRunInit)) {
diff --git a/qemu/roms/seabios/vgasrc/vgaversion.c b/qemu/roms/seabios/vgasrc/vgaversion.c
new file mode 100644
index 000000000..1ef5ddb79
--- /dev/null
+++ b/qemu/roms/seabios/vgasrc/vgaversion.c
@@ -0,0 +1,6 @@
+// Place build generated version into a C variable
+#include "autovgaversion.h"
+#include "types.h"
+
+char VERSION[] VAR16 = BUILD_VERSION;
+char BUILDINFO[] VAR16 = BUILD_TOOLS;
diff --git a/qemu/rules.mak b/qemu/rules.mak
index aec27f81d..d1ff31125 100644
--- a/qemu/rules.mak
+++ b/qemu/rules.mak
@@ -17,7 +17,7 @@ MAKEFLAGS += -rR
 QEMU_CXXFLAGS = -D__STDC_LIMIT_MACROS $(filter-out -Wstrict-prototypes -Wmissing-prototypes -Wnested-externs -Wold-style-declaration -Wold-style-definition -Wredundant-decls, $(QEMU_CFLAGS))
 
 # Flags for dependency generation
-QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
+QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(@D)/$(*F).d
 
 # Same as -I$(SRC_PATH) -I., but for the nested source/object directories
 QEMU_INCLUDES += -I$(<D) -I$(@D)
@@ -62,27 +62,9 @@ expand-objs = $(strip $(sort $(filter %.o,$1)) \
 # must link with the C++ compiler, not the plain C compiler.
 LINKPROG = $(or $(CXX),$(CC))
 
-ifeq ($(LIBTOOL),)
 LINK = $(call quiet-command, $(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
        $(call process-archive-undefs, $1) \
        $(version-obj-y) $(call extract-libs,$1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
-else
-LIBTOOL += $(if $(V),,--quiet)
-%.lo: %.c
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) $($*.o-cflags) -c -o $@ $<,"  lt CC $@")
-%.lo: %.rc
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=RC $(WINDRES) -I. -o $@ $<,"lt RC   $(TARGET_DIR)$@")
-%.lo: %.dtrace
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, " lt GEN $(TARGET_DIR)$@")
-
-LINK = $(call quiet-command,\
-       $(if $(filter %.lo %.la,$1),$(LIBTOOL) --mode=link --tag=CC \
-       )$(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
-       $(call process-archive-undefs, $1)\
-       $(if $(filter %.lo %.la,$1),$(version-lobj-y),$(version-obj-y)) \
-       $(if $(filter %.lo %.la,$1),$(LIBTOOLFLAGS)) \
-       $(call extract-libs,$(1:.lo=.o)) $(LIBS),$(if $(filter %.lo %.la,$1),"lt LINK ", "  LINK  ")"$(TARGET_DIR)$@")
-endif
 
 %.asm: %.S
 	$(call quiet-command,$(CPP) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -o $@ $<,"  CPP   $(TARGET_DIR)$@")
@@ -120,7 +102,7 @@ LD_REL := $(CC) -nostdlib -Wl,-r
 modules:
 
 %$(EXESUF): %.o
-	$(call LINK,$^)
+	$(call LINK,$(filter %.o %.a %.mo, $^))
 
 %.a:
 	$(call quiet-command,rm -f $@ && $(AR) rcs $@ $^,"  AR    $(TARGET_DIR)$@")
@@ -368,6 +350,6 @@ define unnest-vars
                 $(error $o added in $v but $o-objs is not set)))
         $(shell mkdir -p ./ $(sort $(dir $($v))))
         # Include all the .d files
-        $(eval -include $(addsuffix *.d, $(sort $(dir $($v)))))
+        $(eval -include $(patsubst %.o,%.d,$(patsubst %.mo,%.d,$($v))))
         $(eval $v := $(filter-out %/,$($v))))
 endef
diff --git a/qemu/scripts/acpi_extract.py b/qemu/scripts/acpi_extract.py
deleted file mode 100755
index 10c1ffb36..000000000
--- a/qemu/scripts/acpi_extract.py
+++ /dev/null
@@ -1,367 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, see <http://www.gnu.org/licenses/>.
-
-# Process mixed ASL/AML listing (.lst file) produced by iasl -l
-# Locate and execute ACPI_EXTRACT directives, output offset info
-#
-# Documentation of ACPI_EXTRACT_* directive tags:
-#
-# These directive tags output offset information from AML for BIOS runtime
-# table generation.
-# Each directive is of the form:
-# ACPI_EXTRACT_<TYPE> <array_name> <Operator> (...)
-# and causes the extractor to create an array
-# named <array_name> with offset, in the generated AML,
-# of an object of a given type in the following <Operator>.
-#
-# A directive must fit on a single code line.
-#
-# Object type in AML is verified, a mismatch causes a build failure.
-#
-# Directives and operators currently supported are:
-# ACPI_EXTRACT_NAME_DWORD_CONST - extract a Dword Const object from Name()
-# ACPI_EXTRACT_NAME_WORD_CONST - extract a Word Const object from Name()
-# ACPI_EXTRACT_NAME_BYTE_CONST - extract a Byte Const object from Name()
-# ACPI_EXTRACT_METHOD_STRING - extract a NameString from Method()
-# ACPI_EXTRACT_NAME_STRING - extract a NameString from Name()
-# ACPI_EXTRACT_PROCESSOR_START - start of Processor() block
-# ACPI_EXTRACT_PROCESSOR_STRING - extract a NameString from Processor()
-# ACPI_EXTRACT_PROCESSOR_END - offset at last byte of Processor() + 1
-# ACPI_EXTRACT_PKG_START - start of Package block
-#
-# ACPI_EXTRACT_ALL_CODE - create an array storing the generated AML bytecode
-#
-# ACPI_EXTRACT is not allowed anywhere else in code, except in comments.
-
-import re;
-import sys;
-import fileinput;
-
-aml = []
-asl = []
-output = {}
-debug = ""
-
-class asl_line:
-    line = None
-    lineno = None
-    aml_offset = None
-
-def die(diag):
-    sys.stderr.write("Error: %s; %s\n" % (diag, debug))
-    sys.exit(1)
-
-#Store an ASL command, matching AML offset, and input line (for debugging)
-def add_asl(lineno, line):
-    l = asl_line()
-    l.line = line
-    l.lineno = lineno
-    l.aml_offset = len(aml)
-    asl.append(l)
-
-#Store an AML byte sequence
-#Verify that offset output by iasl matches # of bytes so far
-def add_aml(offset, line):
-    o = int(offset, 16);
-    # Sanity check: offset must match size of code so far
-    if (o != len(aml)):
-        die("Offset 0x%x != 0x%x" % (o, len(aml)))
-    # Strip any trailing dots and ASCII dump after "
-    line = re.sub(r'\s*\.*\s*".*$',"", line)
-    # Strip traling whitespace
-    line = re.sub(r'\s+$',"", line)
-    # Strip leading whitespace
-    line = re.sub(r'^\s+',"", line)
-    # Split on whitespace
-    code = re.split(r'\s+', line)
-    for c in code:
-        # Require a legal hex number, two digits
-        if (not(re.search(r'^[0-9A-Fa-f][0-9A-Fa-f]$', c))):
-            die("Unexpected octet %s" % c);
-        aml.append(int(c, 16));
-
-# Process aml bytecode array, decoding AML
-def aml_pkglen_bytes(offset):
-    # PkgLength can be multibyte. Bits 8-7 give the # of extra bytes.
-    pkglenbytes = aml[offset] >> 6;
-    return pkglenbytes + 1
-
-def aml_pkglen(offset):
-    pkgstart = offset
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml[offset] & 0x3F
-    # If multibyte, first nibble only uses bits 0-3
-    if ((pkglenbytes > 1) and (pkglen & 0x30)):
-        die("PkgLen bytes 0x%x but first nibble 0x%x expected 0x0X" %
-            (pkglen, pkglen))
-    offset += 1
-    pkglenbytes -= 1
-    for i in range(pkglenbytes):
-        pkglen |= aml[offset + i] << (i * 8 + 4)
-    if (len(aml) < pkgstart + pkglen):
-        die("PckgLen 0x%x at offset 0x%x exceeds AML size 0x%x" %
-            (pkglen, offset, len(aml)))
-    return pkglen
-
-# Given method offset, find its NameString offset
-def aml_method_string(offset):
-    #0x14 MethodOp PkgLength NameString MethodFlags TermList
-    if (aml[offset] != 0x14):
-        die( "Method offset 0x%x: expected 0x14 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1;
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes;
-    return offset;
-
-# Given name offset, find its NameString offset
-def aml_name_string(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x08):
-        die( "Name offset 0x%x: expected 0x08 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1
-    # Block Name Modifier. Skip it.
-    if (aml[offset] == 0x5c or aml[offset] == 0x5e):
-        offset += 1
-    return offset;
-
-# Given data offset, find variable length byte buffer offset
-def aml_data_buffer(offset, length):
-    #0x11 PkgLength BufferSize ByteList
-    if (length > 63):
-        die( "Name offset 0x%x: expected a one byte PkgLength (length<=63)" %
-             (offset));
-    expect = [0x11, length+3, 0x0A, length]
-    if (aml[offset:offset+4] != expect):
-        die( "Name offset 0x%x: expected %s actual %s" %
-             (offset, expect, aml[offset:offset+4]))
-    return offset + len(expect)
-
-# Given data offset, find dword const offset
-def aml_data_dword_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0C):
-        die( "Name offset 0x%x: expected 0x0C actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Given data offset, find word const offset
-def aml_data_word_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0B):
-        die( "Name offset 0x%x: expected 0x0B actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Given data offset, find byte const offset
-def aml_data_byte_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0A):
-        die( "Name offset 0x%x: expected 0x0A actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Find name'd buffer
-def aml_name_buffer(offset, length):
-    return aml_data_buffer(aml_name_string(offset) + 4, length)
-
-# Given name offset, find dword const offset
-def aml_name_dword_const(offset):
-    return aml_data_dword_const(aml_name_string(offset) + 4)
-
-# Given name offset, find word const offset
-def aml_name_word_const(offset):
-    return aml_data_word_const(aml_name_string(offset) + 4)
-
-# Given name offset, find byte const offset
-def aml_name_byte_const(offset):
-    return aml_data_byte_const(aml_name_string(offset) + 4)
-
-def aml_device_start(offset):
-    #0x5B 0x82 DeviceOp PkgLength NameString
-    if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x82)):
-        die( "Name offset 0x%x: expected 0x5B 0x82 actual 0x%x 0x%x" %
-             (offset, aml[offset], aml[offset + 1]));
-    return offset
-
-def aml_device_string(offset):
-    #0x5B 0x82 DeviceOp PkgLength NameString
-    start = aml_device_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes
-    return offset
-
-def aml_device_end(offset):
-    start = aml_device_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml_pkglen(offset)
-    return offset + pkglen
-
-def aml_processor_start(offset):
-    #0x5B 0x83 ProcessorOp PkgLength NameString ProcID
-    if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x83)):
-        die( "Name offset 0x%x: expected 0x5B 0x83 actual 0x%x 0x%x" %
-             (offset, aml[offset], aml[offset + 1]));
-    return offset
-
-def aml_processor_string(offset):
-    #0x5B 0x83 ProcessorOp PkgLength NameString ProcID
-    start = aml_processor_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes
-    return offset
-
-def aml_processor_end(offset):
-    start = aml_processor_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml_pkglen(offset)
-    return offset + pkglen
-
-def aml_package_start(offset):
-    offset = aml_name_string(offset) + 4
-    # 0x12 PkgLength NumElements PackageElementList
-    if (aml[offset] != 0x12):
-        die( "Name offset 0x%x: expected 0x12 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1
-    return offset + aml_pkglen_bytes(offset) + 1
-
-lineno = 0
-for line in fileinput.input():
-    # Strip trailing newline
-    line = line.rstrip();
-    # line number and debug string to output in case of errors
-    lineno = lineno + 1
-    debug = "input line %d: %s" % (lineno, line)
-    #ASL listing: space, then line#, then ...., then code
-    pasl = re.compile('^\s+([0-9]+)(:\s\s|\.\.\.\.)\s*')
-    m = pasl.search(line)
-    if (m):
-        add_asl(lineno, pasl.sub("", line));
-    # AML listing: offset in hex, then ...., then code
-    paml = re.compile('^([0-9A-Fa-f]+)(:\s\s|\.\.\.\.)\s*')
-    m = paml.search(line)
-    if (m):
-        add_aml(m.group(1), paml.sub("", line))
-
-# Now go over code
-# Track AML offset of a previous non-empty ASL command
-prev_aml_offset = -1
-for i in range(len(asl)):
-    debug = "input line %d: %s" % (asl[i].lineno, asl[i].line)
-
-    l = asl[i].line
-
-    # skip if not an extract directive
-    a = len(re.findall(r'ACPI_EXTRACT', l))
-    if (not a):
-        # If not empty, store AML offset. Will be used for sanity checks
-        # IASL seems to put {}. at random places in the listing.
-        # Ignore any non-words for the purpose of this test.
-        m = re.search(r'\w+', l)
-        if (m):
-                prev_aml_offset = asl[i].aml_offset
-        continue
-
-    if (a > 1):
-        die("Expected at most one ACPI_EXTRACT per line, actual %d" % a)
-
-    mext = re.search(r'''
-                      ^\s* # leading whitespace
-                      /\*\s* # start C comment
-                      (ACPI_EXTRACT_\w+) # directive: group(1)
-                      \s+ # whitspace separates directive from array name
-                      (\w+) # array name: group(2)
-                      \s*\*/ # end of C comment
-                      \s*$ # trailing whitespace
-                      ''', l, re.VERBOSE)
-    if (not mext):
-        die("Stray ACPI_EXTRACT in input")
-
-    # previous command must have produced some AML,
-    # otherwise we are in a middle of a block
-    if (prev_aml_offset == asl[i].aml_offset):
-        die("ACPI_EXTRACT directive in the middle of a block")
-
-    directive = mext.group(1)
-    array = mext.group(2)
-    offset = asl[i].aml_offset
-
-    if (directive == "ACPI_EXTRACT_ALL_CODE"):
-        if array in output:
-            die("%s directive used more than once" % directive)
-        output[array] = aml
-        continue
-    if (directive == "ACPI_EXTRACT_NAME_BUFFER8"):
-        offset = aml_name_buffer(offset, 8)
-    elif (directive == "ACPI_EXTRACT_NAME_BUFFER16"):
-        offset = aml_name_buffer(offset, 16)
-    elif (directive == "ACPI_EXTRACT_NAME_DWORD_CONST"):
-        offset = aml_name_dword_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_WORD_CONST"):
-        offset = aml_name_word_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_BYTE_CONST"):
-        offset = aml_name_byte_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_STRING"):
-        offset = aml_name_string(offset)
-    elif (directive == "ACPI_EXTRACT_METHOD_STRING"):
-        offset = aml_method_string(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_START"):
-        offset = aml_device_start(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_STRING"):
-        offset = aml_device_string(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_END"):
-        offset = aml_device_end(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_START"):
-        offset = aml_processor_start(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_STRING"):
-        offset = aml_processor_string(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_END"):
-        offset = aml_processor_end(offset)
-    elif (directive == "ACPI_EXTRACT_PKG_START"):
-        offset = aml_package_start(offset)
-    else:
-        die("Unsupported directive %s" % directive)
-
-    if array not in output:
-        output[array] = []
-    output[array].append(offset)
-
-debug = "at end of file"
-
-def get_value_type(maxvalue):
-    #Use type large enough to fit the table
-    if (maxvalue >= 0x10000):
-            return "int"
-    elif (maxvalue >= 0x100):
-            return "short"
-    else:
-            return "char"
-
-# Pretty print output
-for array in output.keys():
-    otype = get_value_type(max(output[array]))
-    odata = []
-    for value in output[array]:
-        odata.append("0x%x" % value)
-    sys.stdout.write("static unsigned %s %s[] = {\n" % (otype, array))
-    sys.stdout.write(",\n".join(odata))
-    sys.stdout.write('\n};\n');
diff --git a/qemu/scripts/acpi_extract_preprocess.py b/qemu/scripts/acpi_extract_preprocess.py
deleted file mode 100755
index 69d10d621..000000000
--- a/qemu/scripts/acpi_extract_preprocess.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, see <http://www.gnu.org/licenses/>.
-
-# Read a preprocessed ASL listing and put each ACPI_EXTRACT
-# directive in a comment, to make iasl skip it.
-# We also put each directive on a new line, the machinery
-# in tools/acpi_extract.py requires this.
-
-import re;
-import sys;
-import fileinput;
-
-def die(diag):
-    sys.stderr.write("Error: %s\n" % (diag))
-    sys.exit(1)
-
-# Note: () around pattern make split return matched string as part of list
-psplit = re.compile(r''' (
-                          \b # At word boundary
-                          ACPI_EXTRACT_\w+ # directive
-                          \s+ # some whitespace
-                          \w+ # array name
-                         )''', re.VERBOSE);
-
-lineno = 0
-for line in fileinput.input():
-    # line number and debug string to output in case of errors
-    lineno = lineno + 1
-    debug = "input line %d: %s" % (lineno, line.rstrip())
-
-    s = psplit.split(line);
-    # The way split works, each odd item is the matching ACPI_EXTRACT directive.
-    # Put each in a comment, and on a line by itself.
-    for i in range(len(s)):
-        if (i % 2):
-            sys.stdout.write("\n/* %s */\n" % s[i])
-        else:
-            sys.stdout.write(s[i])
diff --git a/qemu/scripts/analyze-migration.py b/qemu/scripts/analyze-migration.py
index f6894bece..14553876a 100755
--- a/qemu/scripts/analyze-migration.py
+++ b/qemu/scripts/analyze-migration.py
@@ -252,6 +252,15 @@ class HTABSection(object):
     def getDict(self):
         return ""
 
+
+class ConfigurationSection(object):
+    def __init__(self, file):
+        self.file = file
+
+    def read(self):
+        name_len = self.file.read32()
+        name = self.file.readstr(len = name_len)
+
 class VMSDFieldGeneric(object):
     def __init__(self, desc, file):
         self.file = file
@@ -474,6 +483,7 @@ class MigrationDump(object):
     QEMU_VM_SECTION_FULL  = 0x04
     QEMU_VM_SUBSECTION    = 0x05
     QEMU_VM_VMDESCRIPTION = 0x06
+    QEMU_VM_CONFIGURATION = 0x07
     QEMU_VM_SECTION_FOOTER= 0x7e
 
     def __init__(self, filename):
@@ -514,6 +524,9 @@ class MigrationDump(object):
             section_type = file.read8()
             if section_type == self.QEMU_VM_EOF:
                 break
+            elif section_type == self.QEMU_VM_CONFIGURATION:
+                section = ConfigurationSection(file)
+                section.read()
             elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL:
                 section_id = file.read32()
                 name = file.readstr()
diff --git a/qemu/scripts/checkpatch.pl b/qemu/scripts/checkpatch.pl
index 7f0aae977..c9554ba64 100755
--- a/qemu/scripts/checkpatch.pl
+++ b/qemu/scripts/checkpatch.pl
@@ -141,44 +141,22 @@ our $Ident	= qr{
 		}x;
 our $Storage	= qr{extern|static|asmlinkage};
 our $Sparse	= qr{
-			__user|
-			__kernel|
-			__force|
-			__iomem|
-			__must_check|
-			__init_refok|
-			__kprobes|
-			__ref
+			__force
 		}x;
 
 # Notes to $Attribute:
-# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
 our $Attribute	= qr{
 			const|
-			__percpu|
-			__nocast|
-			__safe|
-			__bitwise__|
-			__packed__|
-			__packed2__|
-			__naked|
-			__maybe_unused|
-			__always_unused|
-			__noreturn|
-			__used|
-			__cold|
-			__noclone|
-			__deprecated|
-			__read_mostly|
-			__kprobes|
-			__(?:mem|cpu|dev|)(?:initdata|initconst|init\b)|
-			____cacheline_aligned|
-			____cacheline_aligned_in_smp|
-			____cacheline_internodealigned_in_smp|
-			__weak
+			volatile|
+			QEMU_NORETURN|
+			QEMU_WARN_UNUSED_RESULT|
+			QEMU_SENTINEL|
+			QEMU_ARTIFICIAL|
+			QEMU_PACKED|
+			GCC_FMT_ATTR
 		  }x;
 our $Modifier;
-our $Inline	= qr{inline|__always_inline|noinline};
+our $Inline	= qr{inline};
 our $Member	= qr{->$Ident|\.$Ident|\[[^]]*\]};
 our $Lval	= qr{$Ident(?:$Member)*};
 
@@ -215,14 +193,6 @@ our $typeTypedefs = qr{(?x:
         | QEMUBH                            # all uppercase
 )};
 
-our $logFunctions = qr{(?x:
-	printk|
-	pr_(debug|dbg|vdbg|devel|info|warning|err|notice|alert|crit|emerg|cont)|
-	(dev|netdev|netif)_(printk|dbg|vdbg|info|warn|err|notice|alert|crit|emerg|WARN)|
-	WARN|
-	panic
-)};
-
 our @typeList = (
 	qr{void},
 	qr{(?:unsigned\s+)?char},
@@ -242,21 +212,22 @@ our @typeList = (
 	qr{${Ident}_t},
 	qr{${Ident}_handler},
 	qr{${Ident}_handler_fn},
+	qr{target_(?:u)?long},
 );
+
+# This can be modified by sub possible.  Since it can be empty, be careful
+# about regexes that always match, because they can cause infinite loops.
 our @modifierList = (
-	qr{fastcall},
 );
 
-our $allowed_asm_includes = qr{(?x:
-	irq|
-	memory
-)};
-# memory.h: ARM has a custom one
-
 sub build_types {
-	my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
 	my $all = "(?x:  \n" . join("|\n  ", @typeList) . "\n)";
-	$Modifier	= qr{(?:$Attribute|$Sparse|$mods)};
+	if (@modifierList > 0) {
+		my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
+		$Modifier = qr{(?:$Attribute|$Sparse|$mods)};
+	} else {
+		$Modifier = qr{(?:$Attribute|$Sparse)};
+	}
 	$NonptrType	= qr{
 			(?:$Modifier\s+|const\s+)*
 			(?:
@@ -277,27 +248,6 @@ build_types();
 
 $chk_signoff = 0 if ($file);
 
-my @dep_includes = ();
-my @dep_functions = ();
-my $removal = "Documentation/feature-removal-schedule.txt";
-if ($tree && -f "$root/$removal") {
-	open(my $REMOVE, '<', "$root/$removal") ||
-				die "$P: $removal: open failed - $!\n";
-	while (<$REMOVE>) {
-		if (/^Check:\s+(.*\S)/) {
-			for my $entry (split(/[, ]+/, $1)) {
-				if ($entry =~ m@include/(.*)@) {
-					push(@dep_includes, $1);
-
-				} elsif ($entry !~ m@/@) {
-					push(@dep_functions, $entry);
-				}
-			}
-		}
-	}
-	close($REMOVE);
-}
-
 my @rawlines = ();
 my @lines = ();
 my $vname;
@@ -633,7 +583,7 @@ sub statement_block_size {
 	my ($stmt) = @_;
 
 	$stmt =~ s/(^|\n)./$1/g;
-	$stmt =~ s/^\s*{//;
+	$stmt =~ s/^\s*\{//;
 	$stmt =~ s/}\s*$//;
 	$stmt =~ s/^\s*//;
 	$stmt =~ s/\s*$//;
@@ -1061,7 +1011,9 @@ sub possible {
 			case|
 			else|
 			asm|__asm__|
-			do
+			do|
+			\#|
+			\#\#
 		)(?:\s|$)|
 		^(?:typedef|struct|enum)\b
 	    )}x;
@@ -1127,33 +1079,6 @@ sub CHK {
 	}
 }
 
-sub check_absolute_file {
-	my ($absolute, $herecurr) = @_;
-	my $file = $absolute;
-
-	##print "absolute<$absolute>\n";
-
-	# See if any suffix of this path is a path within the tree.
-	while ($file =~ s@^[^/]*/@@) {
-		if (-f "$root/$file") {
-			##print "file<$file>\n";
-			last;
-		}
-	}
-	if (! -f _)  {
-		return 0;
-	}
-
-	# It is, so see if the prefix is acceptable.
-	my $prefix = $absolute;
-	substr($prefix, -length($file)) = '';
-
-	##print "prefix<$prefix>\n";
-	if ($prefix ne ".../") {
-		WARN("use relative pathname instead of absolute in changelog text\n" . $herecurr);
-	}
-}
-
 sub process {
 	my $filename = shift;
 
@@ -1196,10 +1121,6 @@ sub process {
 	my %suppress_export;
 
 	# Pre-scan the patch sanitizing the lines.
-	# Pre-scan the patch looking for any __setup documentation.
-	#
-	my @setup_docs = ();
-	my $setup_docs = 0;
 
 	sanitise_line_reset();
 	my $line;
@@ -1207,13 +1128,6 @@ sub process {
 		$linenr++;
 		$line = $rawline;
 
-		if ($rawline=~/^\+\+\+\s+(\S+)/) {
-			$setup_docs = 0;
-			if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
-				$setup_docs = 1;
-			}
-			#next;
-		}
 		if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
 			$realline=$1-1;
 			if (defined $2) {
@@ -1272,10 +1186,6 @@ sub process {
 
 		#print "==>$rawline\n";
 		#print "-->$line\n";
-
-		if ($setup_docs && $line =~ /^\+/) {
-			push(@setup_docs, $line);
-		}
 	}
 
 	$prefix = '';
@@ -1350,9 +1260,6 @@ sub process {
 				WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
 			}
 
-			if ($realfile =~ m@^include/asm/@) {
-				ERROR("do not modify files in include/asm, change architecture specific files in include/asm-<architecture>\n" . "$here$rawline\n");
-			}
 			next;
 		}
 
@@ -1367,7 +1274,7 @@ sub process {
 # Check for incorrect file permissions
 		if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
 			my $permhere = $here . "FILE: $realfile\n";
-			if ($realfile =~ /(Makefile|Kconfig|\.c|\.cpp|\.h|\.S|\.tmpl)$/) {
+			if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) {
 				ERROR("do not set execute permissions for source files\n" . $permhere);
 			}
 		}
@@ -1392,20 +1299,6 @@ sub process {
 				$herecurr) if (!$emitted_corrupt++);
 		}
 
-# Check for absolute kernel paths.
-		if ($tree) {
-			while ($line =~ m{(?:^|\s)(/\S*)}g) {
-				my $file = $1;
-
-				if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
-				    check_absolute_file($1, $herecurr)) {
-					#
-				} else {
-					check_absolute_file($file, $herecurr);
-				}
-			}
-		}
-
 # UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
 		if (($realfile =~ /^$/ || $line =~ /^\+/) &&
 		    $rawline !~ m/^$UTF8*$/) {
@@ -1432,45 +1325,12 @@ sub process {
 			$rpt_cleaners = 1;
 		}
 
-# check for Kconfig help text having a real description
-# Only applies when adding the entry originally, after that we do not have
-# sufficient context to determine whether it is indeed long enough.
-		if ($realfile =~ /Kconfig/ &&
-		    $line =~ /\+\s*(?:---)?help(?:---)?$/) {
-			my $length = 0;
-			my $cnt = $realcnt;
-			my $ln = $linenr + 1;
-			my $f;
-			my $is_end = 0;
-			while ($cnt > 0 && defined $lines[$ln - 1]) {
-				$f = $lines[$ln - 1];
-				$cnt-- if ($lines[$ln - 1] !~ /^-/);
-				$is_end = $lines[$ln - 1] =~ /^\+/;
-				$ln++;
-
-				next if ($f =~ /^-/);
-				$f =~ s/^.//;
-				$f =~ s/#.*//;
-				$f =~ s/^\s+//;
-				next if ($f =~ /^$/);
-				if ($f =~ /^\s*config\s/) {
-					$is_end = 1;
-					last;
-				}
-				$length++;
-			}
-			WARN("please write a paragraph that describes the config symbol fully\n" . $herecurr) if ($is_end && $length < 4);
-			#print "is_end<$is_end> length<$length>\n";
-		}
-
 # check we are in a valid source file if not then ignore this hunk
 		next if ($realfile !~ /\.(h|c|cpp|s|S|pl|sh)$/);
 
 #80 column limit
-		if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
-		    $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
-		    !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:,|\)\s*;)\s*$/ ||
-		    $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+		if ($line =~ /^\+/ &&
+		    !($line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
 		    $length > 80)
 		{
 			WARN("line over 80 characters\n" . $herecurr);
@@ -1486,18 +1346,6 @@ sub process {
 			WARN("adding a line without newline at end of file\n" . $herecurr);
 		}
 
-# Blackfin: use hi/lo macros
-		if ($realfile =~ m@arch/blackfin/.*\.S$@) {
-			if ($line =~ /\.[lL][[:space:]]*=.*&[[:space:]]*0x[fF][fF][fF][fF]/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the LO() macro, not (... & 0xFFFF)\n" . $herevet);
-			}
-			if ($line =~ /\.[hH][[:space:]]*=.*>>[[:space:]]*16/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the HI() macro, not (... >> 16)\n" . $herevet);
-			}
-		}
-
 # check we are in a valid source file C or perl if not then ignore this hunk
 		next if ($realfile !~ /\.(h|c|cpp|pl)$/);
 
@@ -1516,16 +1364,6 @@ sub process {
 			WARN("CVS style keyword markers, these will _not_ be updated\n". $herecurr);
 		}
 
-# Blackfin: don't use __builtin_bfin_[cs]sync
-		if ($line =~ /__builtin_bfin_csync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the CSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-		if ($line =~ /__builtin_bfin_ssync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the SSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-
 # Check for potential 'bare' types
 		my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
 		    $realline_next);
@@ -1644,7 +1482,7 @@ sub process {
 			# 79 or 80 characters, it is no longer possible to add a space and an
 			# opening brace there)
 			if ($#ctx == 0 && $ctx !~ /{\s*/ &&
-			    defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/ &&
+			    defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*\{/ &&
 			    defined($lines[$ctx_ln - 2]) && length($lines[$ctx_ln - 2]) < 80) {
 				ERROR("that open brace { should be on the previous line\n" .
 					"$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
@@ -1684,7 +1522,7 @@ sub process {
 			my $continuation = 0;
 			my $check = 0;
 			$s =~ s/^.*\bdo\b//;
-			$s =~ s/^\s*{//;
+			$s =~ s/^\s*\{//;
 			if ($s =~ s/^\s*\\//) {
 				$continuation = 1;
 			}
@@ -1783,7 +1621,7 @@ sub process {
 		}
 
 # check for initialisation to aggregates open brace on the next line
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /(?:^|[^=])=\s*$/) {
 			ERROR("that open brace { should be on the previous line\n" . $hereprev);
 		}
@@ -1809,50 +1647,6 @@ sub process {
 		$line =~ s@//.*@@;
 		$opline =~ s@//.*@@;
 
-# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
-# the whole statement.
-#print "APW <$lines[$realline_next - 1]>\n";
-		if (defined $realline_next &&
-		    exists $lines[$realline_next - 1] &&
-		    !defined $suppress_export{$realline_next} &&
-		    ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-			# Handle definitions which produce identifiers with
-			# a prefix:
-			#   XXX(foo);
-			#   EXPORT_SYMBOL(something_foo);
-			my $name = $1;
-			if ($stat =~ /^.([A-Z_]+)\s*\(\s*($Ident)/ &&
-			    $name =~ /^${Ident}_$2/) {
-#print "FOO C name<$name>\n";
-				$suppress_export{$realline_next} = 1;
-
-			} elsif ($stat !~ /(?:
-				\n.}\s*$|
-				^.DEFINE_$Ident\(\Q$name\E\)|
-				^.DECLARE_$Ident\(\Q$name\E\)|
-				^.LIST_HEAD\(\Q$name\E\)|
-				^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
-				\b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
-			    )/x) {
-#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
-				$suppress_export{$realline_next} = 2;
-			} else {
-				$suppress_export{$realline_next} = 1;
-			}
-		}
-		if (!defined $suppress_export{$linenr} &&
-		    $prevline =~ /^.\s*$/ &&
-		    ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-#print "FOO B <$lines[$linenr - 1]>\n";
-			$suppress_export{$linenr} = 2;
-		}
-		if (defined $suppress_export{$linenr} &&
-		    $suppress_export{$linenr} == 2) {
-			WARN("EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
-		}
-
 # check for global initialisers.
 		if ($line =~ /^.$Type\s*$Ident\s*(?:\s+$Modifier)*\s*=\s*(0|NULL|false)\s*;/) {
 			ERROR("do not initialise globals to 0 or NULL\n" .
@@ -1900,67 +1694,37 @@ sub process {
 			}
 		}
 
-# # no BUG() or BUG_ON()
-# 		if ($line =~ /\b(BUG|BUG_ON)\b/) {
-# 			print "Try to use WARN_ON & Recovery code rather than BUG() or BUG_ON()\n";
-# 			print "$herecurr";
-# 			$clean = 0;
-# 		}
-
-		if ($line =~ /\bLINUX_VERSION_CODE\b/) {
-			WARN("LINUX_VERSION_CODE should be avoided, code should be for the version to which it is merged\n" . $herecurr);
-		}
-
-# printk should use KERN_* levels.  Note that follow on printk's on the
-# same line do not need a level, so we use the current block context
-# to try and find and validate the current printk.  In summary the current
-# printk includes all preceding printk's which have no newline on the end.
-# we assume the first bad printk is the one to report.
-		if ($line =~ /\bprintk\((?!KERN_)\s*"/) {
-			my $ok = 0;
-			for (my $ln = $linenr - 1; $ln >= $first_line; $ln--) {
-				#print "CHECK<$lines[$ln - 1]\n";
-				# we have a preceding printk if it ends
-				# with "\n" ignore it, else it is to blame
-				if ($lines[$ln - 1] =~ m{\bprintk\(}) {
-					if ($rawlines[$ln - 1] !~ m{\\n"}) {
-						$ok = 1;
-					}
-					last;
-				}
-			}
-			if ($ok == 0) {
-				WARN("printk() should include KERN_ facility level\n" . $herecurr);
-			}
-		}
-
 # function brace can't be on same line, except for #defines of do while,
 # or if closed on same line
-		if (($line=~/$Type\s*$Ident\(.*\).*\s{/) and
-		    !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) {
+		if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+		    !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
 			ERROR("open brace '{' following function declarations go on the next line\n" . $herecurr);
 		}
 
 # open braces for enum, union and struct go on the same line.
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
 			ERROR("open brace '{' following $1 go on the same line\n" . $hereprev);
 		}
 
 # missing space after union, struct or enum definition
 		if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) {
-		    WARN("missing space after $1 definition\n" . $herecurr);
+		    ERROR("missing space after $1 definition\n" . $herecurr);
 		}
 
 # check for spacing round square brackets; allowed:
 #  1. with a type on the left -- int [] a;
 #  2. at the beginning of a line for slice initialisers -- [0...10] = 5,
 #  3. inside a curly brace -- = { [0...10] = 5 }
+#  4. after a comma -- [1] = 5, [2] = 6
+#  5. in a macro definition -- #define abc(x) [x] = y
 		while ($line =~ /(.*?\s)\[/g) {
 			my ($where, $prefix) = ($-[1], $1);
 			if ($prefix !~ /$Type\s+$/ &&
 			    ($where != 0 || $prefix !~ /^.\s+$/) &&
-			    $prefix !~ /{\s+$/) {
+			    $prefix !~ /{\s+$/ &&
+			    $prefix !~ /\#\s*define[^(]*\([^)]*\)\s+$/ &&
+			    $prefix !~ /,\s+$/) {
 				ERROR("space prohibited before open square bracket '['\n" . $herecurr);
 			}
 		}
@@ -2091,7 +1855,7 @@ sub process {
                                 # not required when having a single },{ on one line
 				} elsif ($op eq ',') {
 					if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/ &&
-                                            ($elements[$n] . $elements[$n + 2]) !~ " *}{") {
+                                            ($elements[$n] . $elements[$n + 2]) !~ " *}\\{") {
 						ERROR("space required after that '$op' $at\n" . $hereptr);
 					}
 
@@ -2131,19 +1895,6 @@ sub process {
 						ERROR("space prohibited after that '$op' $at\n" . $hereptr);
 					}
 
-
-				# << and >> may either have or not have spaces both sides
-				} elsif ($op eq '<<' or $op eq '>>' or
-					 $op eq '&' or $op eq '^' or $op eq '|' or
-					 $op eq '+' or $op eq '-' or
-					 $op eq '*' or $op eq '/' or
-					 $op eq '%')
-				{
-					if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
-						ERROR("need consistent spacing around '$op' $at\n" .
-							$hereptr);
-					}
-
 				# A colon needs no spaces before when it is
 				# terminating a case value or a label.
 				} elsif ($opv eq ':C' || $opv eq ':L') {
@@ -2190,29 +1941,9 @@ sub process {
 			}
 		}
 
-# check for multiple assignments
-		if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
-			CHK("multiple assignments should be avoided\n" . $herecurr);
-		}
-
-## # check for multiple declarations, allowing for a function declaration
-## # continuation.
-## 		if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
-## 		    $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
-##
-## 			# Remove any bracketed sections to ensure we do not
-## 			# falsly report the parameters of functions.
-## 			my $ln = $line;
-## 			while ($ln =~ s/\([^\(\)]*\)//g) {
-## 			}
-## 			if ($ln =~ /,/) {
-## 				WARN("declaring multiple variables together should be avoided\n" . $herecurr);
-## 			}
-## 		}
-
 #need space before brace following if, while, etc
-		if (($line =~ /\(.*\){/ && $line !~ /\($Type\){/) ||
-		    $line =~ /do{/) {
+		if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+		    $line =~ /do\{/) {
 			ERROR("space required before the open brace '{'\n" . $herecurr);
 		}
 
@@ -2267,7 +1998,7 @@ sub process {
 		if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
 			my $name = $1;
 			if ($name ne 'EOF' && $name ne 'ERROR') {
-				CHK("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+				WARN("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
 			}
 		}
 
@@ -2398,22 +2129,6 @@ sub process {
 			WARN("Whitepspace after \\ makes next lines useless\n" . $herecurr);
 		}
 
-#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
-		if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
-			my $file = "$1.h";
-			my $checkfile = "include/linux/$file";
-			if (-f "$root/$checkfile" &&
-			    $realfile ne $checkfile &&
-			    $1 !~ /$allowed_asm_includes/)
-			{
-				if ($realfile =~ m{^arch/}) {
-					CHK("Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				} else {
-					WARN("Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				}
-			}
-		}
-
 # multi-statement macros should be enclosed in a do while loop, grab the
 # first statement and ensure its the whole macro if its not enclosed
 # in a known good container
@@ -2508,15 +2223,6 @@ sub process {
 			}
 		}
 
-# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
-# all assignments may have only one of the following with an assignment:
-#	.
-#	ALIGN(...)
-#	VMLINUX_SYMBOL(...)
-		if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
-			WARN("vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
-		}
-
 # check for missing bracing round if etc
 		if ($line =~ /(^.*)\bif\b/ && $line !~ /\#\s*if/) {
 			my ($level, $endln, @chunks) =
@@ -2551,7 +2257,7 @@ sub process {
 					my $spaced_block = $block;
 					$spaced_block =~ s/\n\+/ /g;
 
-					$seen++ if ($spaced_block =~ /^\s*{/);
+					$seen++ if ($spaced_block =~ /^\s*\{/);
 
                                         print "APW: cond<$cond> block<$block> allowed<$allowed>\n"
                                             if $dbg_adv_apw;
@@ -2644,64 +2350,23 @@ sub process {
 			}
 		}
 
-# don't include deprecated include files (uses RAW line)
-		for my $inc (@dep_includes) {
-			if ($rawline =~ m@^.\s*\#\s*include\s*\<$inc>@) {
-				ERROR("Don't use <$inc>: see Documentation/feature-removal-schedule.txt\n" . $herecurr);
-			}
-		}
-
-# don't use deprecated functions
-		for my $func (@dep_functions) {
-			if ($line =~ /\b$func\b/) {
-				ERROR("Don't use $func(): see Documentation/feature-removal-schedule.txt\n" . $herecurr);
-			}
-		}
-
 # no volatiles please
 		my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
 		if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
 			WARN("Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
 		}
 
-# SPIN_LOCK_UNLOCKED & RW_LOCK_UNLOCKED are deprecated
-		if ($line =~ /\b(SPIN_LOCK_UNLOCKED|RW_LOCK_UNLOCKED)/) {
-			ERROR("Use of $1 is deprecated: see Documentation/spinlocks.txt\n" . $herecurr);
-		}
-
 # warn about #if 0
 		if ($line =~ /^.\s*\#\s*if\s+0\b/) {
-			CHK("if this code is redundant consider removing it\n" .
+			WARN("if this code is redundant consider removing it\n" .
 				$herecurr);
 		}
 
-# check for needless kfree() checks
+# check for needless g_free() checks
 		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
 			my $expr = $1;
-			if ($line =~ /\bkfree\(\Q$expr\E\);/) {
-				WARN("kfree(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-# check for needless usb_free_urb() checks
-		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
-			my $expr = $1;
-			if ($line =~ /\busb_free_urb\(\Q$expr\E\);/) {
-				WARN("usb_free_urb(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-
-# prefer usleep_range over udelay
-		if ($line =~ /\budelay\s*\(\s*(\w+)\s*\)/) {
-			# ignore udelay's < 10, however
-			if (! (($1 =~ /(\d+)/) && ($1 < 10)) ) {
-				CHK("usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $line);
-			}
-		}
-
-# warn about unexpectedly long msleep's
-		if ($line =~ /\bmsleep\s*\((\d+)\);/) {
-			if ($1 < 20) {
-				WARN("msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $line);
+			if ($line =~ /\bg_free\(\Q$expr\E\);/) {
+				WARN("g_free(NULL) is safe this check is probably not required\n" . $hereprev);
 			}
 		}
 
@@ -2716,24 +2381,17 @@ sub process {
 		if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
 			ERROR("exactly one space required after that #$1\n" . $herecurr);
 		}
-
-# check for spinlock_t definitions without a comment.
-		if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
-		    $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
-			my $which = $1;
-			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("$1 definition without comment\n" . $herecurr);
-			}
-		}
 # check for memory barriers without a comment.
-		if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+		if ($line =~ /\b(smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
 			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("memory barrier without comment\n" . $herecurr);
+				WARN("memory barrier without comment\n" . $herecurr);
 			}
 		}
 # check of hardware specific defines
-		if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
-			CHK("architecture specific defines should be avoided\n" .  $herecurr);
+# we have e.g. CONFIG_LINUX and CONFIG_WIN32 for common cases
+# where they might be necessary.
+		if ($line =~ m@^.\s*\#\s*if.*\b__@) {
+			WARN("architecture specific defines should be avoided\n" .  $herecurr);
 		}
 
 # Check that the storage class is at the beginning of a declaration
@@ -2748,11 +2406,6 @@ sub process {
 			ERROR("inline keyword should sit between storage class and type\n" . $herecurr);
 		}
 
-# Check for __inline__ and __inline, prefer inline
-		if ($line =~ /\b(__inline__|__inline)\b/) {
-			WARN("plain inline is preferred over $1\n" . $herecurr);
-		}
-
 # check for sizeof(&)
 		if ($line =~ /\bsizeof\s*\(\s*\&/) {
 			WARN("sizeof(& should be avoided\n" . $herecurr);
@@ -2785,98 +2438,55 @@ sub process {
 			WARN("externs should be avoided in .c files\n" .  $herecurr);
 		}
 
-# checks for new __setup's
-		if ($rawline =~ /\b__setup\("([^"]*)"/) {
-			my $name = $1;
-
-			if (!grep(/$name/, @setup_docs)) {
-				CHK("__setup appears un-documented -- check Documentation/kernel-parameters.txt\n" . $herecurr);
+# check for pointless casting of g_malloc return
+		if ($line =~ /\*\s*\)\s*g_(try)?(m|re)alloc(0?)(_n)?\b/) {
+			if ($2 == 'm') {
+				WARN("unnecessary cast may hide bugs, use g_$1new$3 instead\n" . $herecurr);
+			} else {
+				WARN("unnecessary cast may hide bugs, use g_$1renew$3 instead\n" . $herecurr);
 			}
 		}
 
-# check for pointless casting of kmalloc return
-		if ($line =~ /\*\s*\)\s*k[czm]alloc\b/) {
-			WARN("unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
-		}
-
 # check for gcc specific __FUNCTION__
 		if ($line =~ /__FUNCTION__/) {
 			WARN("__func__ should be used instead of gcc specific __FUNCTION__\n"  . $herecurr);
 		}
 
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*(DECLARE_MUTEX|init_MUTEX)\s*\(/) {
-			WARN("mutexes are preferred for single holder semaphores\n" . $herecurr);
-		}
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*init_MUTEX_LOCKED\s*\(/) {
-			WARN("consider using a completion\n" . $herecurr);
-
-		}
-# recommend strict_strto* over simple_strto*
-		if ($line =~ /\bsimple_(strto.*?)\s*\(/) {
-			WARN("consider using strict_$1 in preference to simple_$1\n" . $herecurr);
+# recommend qemu_strto* over strto* for numeric conversions
+		if ($line =~ /\b(strto[^k].*?)\s*\(/) {
+			WARN("consider using qemu_$1 in preference to $1\n" . $herecurr);
 		}
-# check for __initcall(), use device_initcall() explicitly please
-		if ($line =~ /^.\s*__initcall\s*\(/) {
-			WARN("please use device_initcall() instead of __initcall()\n" . $herecurr);
+# check for module_init(), use category-specific init macros explicitly please
+		if ($line =~ /^module_init\s*\(/) {
+			WARN("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr);
 		}
 # check for various ops structs, ensure they are const.
-		my $struct_ops = qr{acpi_dock_ops|
-				address_space_operations|
-				backlight_ops|
-				block_device_operations|
-				dentry_operations|
-				dev_pm_ops|
-				dma_map_ops|
-				extent_io_ops|
-				file_lock_operations|
-				file_operations|
-				hv_ops|
-				ide_dma_ops|
-				intel_dvo_dev_ops|
-				item_operations|
-				iwl_ops|
-				kgdb_arch|
-				kgdb_io|
-				kset_uevent_ops|
-				lock_manager_operations|
-				microcode_ops|
-				mtrr_ops|
-				neigh_ops|
-				nlmsvc_binding|
-				pci_raw_ops|
-				pipe_buf_operations|
-				platform_hibernation_ops|
-				platform_suspend_ops|
-				proto_ops|
-				rpc_pipe_ops|
-				seq_operations|
-				snd_ac97_build_ops|
-				soc_pcmcia_socket_ops|
-				stacktrace_ops|
-				sysfs_ops|
-				tty_operations|
-				usb_mon_operations|
-				wd_ops}x;
+		my $struct_ops = qr{AIOCBInfo|
+				BdrvActionOps|
+				BlockDevOps|
+				BlockJobDriver|
+				DisplayChangeListenerOps|
+				GraphicHwOps|
+				IDEDMAOps|
+				KVMCapabilityInfo|
+				MemoryRegionIOMMUOps|
+				MemoryRegionOps|
+				MemoryRegionPortio|
+				QEMUFileOps|
+				SCSIBusInfo|
+				SCSIReqOps|
+				Spice[A-Z][a-zA-Z0-9]*Interface|
+				TPMDriverOps|
+				USBDesc[A-Z][a-zA-Z0-9]*|
+				VhostOps|
+				VMStateDescription|
+				VMStateInfo}x;
 		if ($line !~ /\bconst\b/ &&
-		    $line =~ /\bstruct\s+($struct_ops)\b/) {
+		    $line =~ /\b($struct_ops)\b/) {
 			WARN("struct $1 should normally be const\n" .
 				$herecurr);
 		}
 
-# use of NR_CPUS is usually wrong
-# ignore definitions of NR_CPUS and usage to define arrays as likely right
-		if ($line =~ /\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*if\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*define\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*$Declare\s.*\[[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*\.\.\.[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/)
-		{
-			WARN("usage of NR_CPUS is often wrong - consider using cpu_possible(), num_possible_cpus(), for_each_possible_cpu(), etc\n" . $herecurr);
-		}
-
 # check for %L{u,d,i} in strings
 		my $string;
 		while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
@@ -2888,29 +2498,46 @@ sub process {
 			}
 		}
 
-# whine mightly about in_atomic
-		if ($line =~ /\bin_atomic\s*\(/) {
-			if ($realfile =~ m@^drivers/@) {
-				ERROR("do not use in_atomic in drivers\n" . $herecurr);
-			} elsif ($realfile !~ m@^kernel/@) {
-				WARN("use of in_atomic() is incorrect outside core kernel code\n" . $herecurr);
-			}
+# QEMU specific tests
+		if ($rawline =~ /\b(?:Qemu|QEmu)\b/) {
+			WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr);
 		}
 
-# check for lockdep_set_novalidate_class
-		if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ ||
-		    $line =~ /__lockdep_no_validate__\s*\)/ ) {
-			if ($realfile !~ m@^kernel/lockdep@ &&
-			    $realfile !~ m@^include/linux/lockdep@ &&
-			    $realfile !~ m@^drivers/base/core@) {
-				ERROR("lockdep_no_validate class is reserved for device->mutex.\n" . $herecurr);
-			}
+# Qemu error function tests
+
+	# Find newlines in error messages
+	my $qemu_error_funcs = qr{error_setg|
+				error_setg_errno|
+				error_setg_win32|
+				error_set|
+				error_vreport|
+				error_report}x;
+
+	if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(\s*\".*\\n/) {
+		WARN("Error messages should not contain newlines\n" . $herecurr);
+	}
+
+	# Continue checking for error messages that contains newlines. This
+	# check handles cases where string literals are spread over multiple lines.
+	# Example:
+	# error_report("Error msg line #1"
+	#              "Error msg line #2\n");
+	my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"};
+	my $continued_str_literal = qr{\+\s*\".*\"};
+
+	if ($rawline =~ /$quoted_newline_regex/) {
+		# Backtrack to first line that does not contain only a quoted literal
+		# and assume that it is the start of the statement.
+		my $i = $linenr - 2;
+
+		while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) {
+			$i--;
 		}
 
-# QEMU specific tests
-		if ($rawline =~ /\b(?:Qemu|QEmu)\b/) {
-			WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr);
+		if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) {
+			WARN("Error messages should not contain newlines\n" . $herecurr);
 		}
+	}
 
 # check for non-portable ffs() calls that have portable alternatives in QEMU
 		if ($line =~ /\bffs\(/) {
diff --git a/qemu/scripts/clean-includes b/qemu/scripts/clean-includes
new file mode 100755
index 000000000..72b47f17f
--- /dev/null
+++ b/qemu/scripts/clean-includes
@@ -0,0 +1,165 @@
+#!/bin/sh -e
+#
+# Clean up QEMU #include lines by ensuring that qemu/osdep.h
+# is the first include listed in .c files, and no headers provided
+# by osdep.h itself are redundantly included in either .c or .h files.
+#
+# Copyright (c) 2015 Linaro Limited
+#
+# Authors:
+#  Peter Maydell <peter.maydell@linaro.org>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+# Usage:
+#   clean-includes [--git subjectprefix] file ...
+# or
+#   clean-includes [--git subjectprefix] --all
+#
+# If the --git subjectprefix option is given, then after making
+# the changes to the files this script will create a git commit
+# with the subject line "subjectprefix: Clean up includes"
+# and a boilerplate commit message.
+#
+# Using --all will cause clean-includes to run on the whole source
+# tree (excluding certain directories which are known not to need
+# handling).
+
+# This script requires Coccinelle to be installed.
+
+# .c files will have the osdep.h included added, and redundant
+# includes removed.
+# .h files will have redundant includes (including includes of osdep.h)
+# removed.
+# Other files (including C++ and ObjectiveC) can't be handled by this script.
+
+# The following one-liner may be handy for finding files to run this on.
+# However some caution is required regarding files that might be part
+# of the guest agent or standalone tests.
+
+# for i in `git ls-tree --name-only HEAD`  ; do test -f $i && \
+#   grep -E '^# *include' $i | head -1 | grep 'osdep.h' ; test $? != 0 && \
+#   echo $i ; done
+
+
+GIT=no
+
+# Extended regular expression defining files to ignore when using --all
+XDIRREGEX='^(tests/tcg|tests/multiboot|pc-bios|disas/libvixl)'
+
+if [ $# -ne 0 ] && [ "$1" = "--git" ]; then
+    if [ $# -eq 1 ]; then
+        echo "--git option requires an argument"
+        exit 1
+    fi
+    GITSUBJ="$2"
+    GIT=yes
+    shift
+    shift
+fi
+
+if [ $# -eq 0 ]; then
+    echo "Usage: clean-includes [--git subjectprefix] [--all | foo.c ...]"
+    echo "(modifies the files in place)"
+    exit 1
+fi
+
+if [ "$1" = "--all" ]; then
+    # We assume there are no files in the tree with spaces in their name
+    set -- $(git ls-files '*.[ch]' | grep -E -v "$XDIRREGEX")
+fi
+
+# Annoyingly coccinelle won't read a scriptfile unless its
+# name ends '.cocci', so write it out to a tempfile with the
+# right kind of name.
+COCCIFILE="$(mktemp --suffix=.cocci)"
+
+trap 'rm -f -- "$COCCIFILE"' INT TERM HUP EXIT
+
+cat >"$COCCIFILE" <<EOT
+@@
+@@
+
+(
++ #include "qemu/osdep.h"
+ #include "..."
+|
++ #include "qemu/osdep.h"
+ #include <...>
+)
+EOT
+
+
+for f in "$@"; do
+  case "$f" in
+    *.inc.c)
+      # These aren't standalone C source files
+      echo "SKIPPING $f (not a standalone source file)"
+      continue
+      ;;
+    *.c)
+      MODE=c
+      ;;
+    *include/qemu/osdep.h | \
+    *include/qemu/compiler.h | \
+    *include/standard-headers/ )
+      # Removing include lines from osdep.h itself would be counterproductive.
+      echo "SKIPPING $f (special case header)"
+      continue
+      ;;
+    *include/standard-headers/*)
+      echo "SKIPPING $f (autogenerated header)"
+      continue
+      ;;
+    *.h)
+      MODE=h
+      ;;
+    *)
+      echo "WARNING: ignoring $f (cannot handle non-C files)"
+      continue
+      ;;
+  esac
+
+  if [ "$MODE" = "c" ]; then
+    # First, use Coccinelle to add qemu/osdep.h before the first existing include
+    # (this will add two lines if the file uses both "..." and <...> #includes,
+    # but we will remove the extras in the next step)
+    spatch  --in-place --no-show-diff --cocci-file "$COCCIFILE" "$f"
+
+    # Now remove any duplicate osdep.h includes
+    perl -n -i -e 'print if !/#include "qemu\/osdep.h"/ || !$n++;' "$f"
+  else
+    # Remove includes of osdep.h itself
+    perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ ||
+                            ! (grep { $_ eq $1 } qw ("qemu/osdep.h"))' "$f"
+  fi
+
+  # Remove includes that osdep.h already provides
+  perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ ||
+                          ! (grep { $_ eq $1 } qw (
+           "config-host.h" "config-target.h" "qemu/compiler.h"
+           <setjmp.h> <stdarg.h> <stddef.h> <stdbool.h> <stdint.h> <sys/types.h>
+           <stdlib.h> <stdio.h> <string.h> <strings.h> <inttypes.h>
+           <limits.h> <unistd.h> <time.h> <ctype.h> <errno.h> <fcntl.h>
+           <sys/stat.h> <sys/time.h> <assert.h> <signal.h>
+           "sysemu/os-posix.h, sysemu/os-win32.h "glib-compat.h"
+           "qemu/typedefs.h"
+            ))' "$f"
+
+done
+
+if [ "$GIT" = "yes" ]; then
+    git add -- "$@"
+    git commit --signoff -F - <<EOF
+$GITSUBJ: Clean up includes
+
+Clean up includes so that osdep.h is included first and headers
+which it implies are not included manually.
+
+This commit was created with scripts/clean-includes.
+
+EOF
+
+fi
diff --git a/qemu/scripts/cocci-macro-file.h b/qemu/scripts/cocci-macro-file.h
new file mode 100644
index 000000000..eceb4be73
--- /dev/null
+++ b/qemu/scripts/cocci-macro-file.h
@@ -0,0 +1,119 @@
+/* Macro file for Coccinelle
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or, at your
+ * option, any later version.  See the COPYING file in the top-level directory.
+ */
+
+/* Coccinelle only does limited parsing of headers, and chokes on some idioms
+ * defined in compiler.h and queue.h.  Macros that Coccinelle must know about
+ * in order to parse .c files must be in a separate macro file---which is
+ * exactly what you're staring at now.
+ *
+ * To use this file, add the "--macro-file scripts/cocci-macro-file.h" to the
+ * Coccinelle command line.
+ */
+
+/* From qemu/compiler.h */
+#define QEMU_GNUC_PREREQ(maj, min) 1
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
+#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#define QEMU_SENTINEL __attribute__((sentinel))
+#define QEMU_ARTIFICIAL __attribute__((always_inline, artificial))
+#define QEMU_PACKED __attribute__((gcc_struct, packed))
+
+#define cat(x,y) x ## y
+#define cat2(x,y) cat(x,y)
+#define QEMU_BUILD_BUG_ON(x) \
+    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+
+#define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
+/* From qemu/queue.h */
+
+#define QLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *lh_first;  /* first element */                     \
+}
+
+#define QLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *le_next;   /* next element */                      \
+        struct type **le_prev;  /* address of previous next element */  \
+}
+
+/*
+ * Singly-linked List definitions.
+ */
+#define QSLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *slh_first; /* first element */                     \
+}
+
+#define QSLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QSLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *sle_next;  /* next element */                      \
+}
+
+/*
+ * Simple queue definitions.
+ */
+#define QSIMPLEQ_HEAD(name, type)                                       \
+struct name {                                                           \
+    struct type *sqh_first;    /* first element */                      \
+    struct type **sqh_last;    /* addr of last next element */          \
+}
+
+#define QSIMPLEQ_HEAD_INITIALIZER(head)                                 \
+    { NULL, &(head).sqh_first }
+
+#define QSIMPLEQ_ENTRY(type)                                            \
+struct {                                                                \
+    struct type *sqe_next;    /* next element */                        \
+}
+
+/*
+ * Tail queue definitions.
+ */
+#define Q_TAILQ_HEAD(name, type, qual)                                  \
+struct name {                                                           \
+        qual type *tqh_first;           /* first element */             \
+        qual type *qual *tqh_last;      /* addr of last next element */ \
+}
+#define QTAILQ_HEAD(name, type)                                         \
+struct name {                                                           \
+        type *tqh_first;      /* first element */                       \
+        type **tqh_last;      /* addr of last next element */           \
+}
+
+#define QTAILQ_HEAD_INITIALIZER(head)                                   \
+        { NULL, &(head).tqh_first }
+
+#define Q_TAILQ_ENTRY(type, qual)                                       \
+struct {                                                                \
+        qual type *tqe_next;            /* next element */              \
+        qual type *qual *tqe_prev;      /* address of previous next element */\
+}
+#define QTAILQ_ENTRY(type)                                              \
+struct {                                                                \
+        type *tqe_next;       /* next element */                        \
+        type **tqe_prev;      /* address of previous next element */    \
+}
diff --git a/qemu/scripts/coverity-model.c b/qemu/scripts/coverity-model.c
index 617f67d71..ee5bf9d07 100644
--- a/qemu/scripts/coverity-model.c
+++ b/qemu/scripts/coverity-model.c
@@ -236,6 +236,23 @@ void *g_try_realloc(void *ptr, size_t size)
     return g_try_realloc_n(ptr, 1, size);
 }
 
+/* Other memory allocation functions */
+
+void *g_memdup(const void *ptr, unsigned size)
+{
+    unsigned char *dup;
+    unsigned i;
+
+    if (!ptr) {
+        return NULL;
+    }
+
+    dup = g_malloc(size);
+    for (i = 0; i < size; i++)
+        dup[i] = ((unsigned char *)ptr)[i];
+    return dup;
+}
+
 /*
  * GLib string allocation functions
  */
@@ -325,6 +342,15 @@ char *g_strconcat(const char *s, ...)
 
 /* Other glib functions */
 
+typedef struct pollfd GPollFD;
+
+int poll();
+
+int g_poll (GPollFD *fds, unsigned nfds, int timeout)
+{
+    return poll(fds, nfds, timeout);
+}
+
 typedef struct _GIOChannel GIOChannel;
 GIOChannel *g_io_channel_unix_new(int fd)
 {
diff --git a/qemu/scripts/create_config b/qemu/scripts/create_config
index 546f88914..9cb176f1b 100755
--- a/qemu/scripts/create_config
+++ b/qemu/scripts/create_config
@@ -61,6 +61,15 @@ case $line in
     value=${line#*=}
     echo "#define $name $value"
     ;;
+ HAVE_*=y) # configuration
+    name=${line%=*}
+    echo "#define $name 1"
+    ;;
+ HAVE_*=*) # configuration
+    name=${line%=*}
+    value=${line#*=}
+    echo "#define $name $value"
+    ;;
  ARCH=*) # configuration
     arch=${line#*=}
     arch_name=`echo $arch | LC_ALL=C tr '[a-z]' '[A-Z]'`
diff --git a/qemu/scripts/dump-guest-memory.py b/qemu/scripts/dump-guest-memory.py
index dc8e44acf..c0a2e99f4 100644
--- a/qemu/scripts/dump-guest-memory.py
+++ b/qemu/scripts/dump-guest-memory.py
@@ -1,39 +1,456 @@
-# This python script adds a new gdb command, "dump-guest-memory". It
-# should be loaded with "source dump-guest-memory.py" at the (gdb)
-# prompt.
-#
-# Copyright (C) 2013, Red Hat, Inc.
-#
-# Authors:
-#   Laszlo Ersek <lersek@redhat.com>
-#
-# This work is licensed under the terms of the GNU GPL, version 2 or later. See
-# the COPYING file in the top-level directory.
-#
+"""
+This python script adds a new gdb command, "dump-guest-memory". It
+should be loaded with "source dump-guest-memory.py" at the (gdb)
+prompt.
+
+Copyright (C) 2013, Red Hat, Inc.
+
+Authors:
+   Laszlo Ersek <lersek@redhat.com>
+   Janosch Frank <frankja@linux.vnet.ibm.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or later. See
+the COPYING file in the top-level directory.
+"""
+
+import ctypes
+
+UINTPTR_T = gdb.lookup_type("uintptr_t")
+
+TARGET_PAGE_SIZE = 0x1000
+TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000
+
+# Special value for e_phnum. This indicates that the real number of
+# program headers is too large to fit into e_phnum. Instead the real
+# value is in the field sh_info of section 0.
+PN_XNUM = 0xFFFF
+
+EV_CURRENT = 1
+
+ELFCLASS32 = 1
+ELFCLASS64 = 2
+
+ELFDATA2LSB = 1
+ELFDATA2MSB = 2
+
+ET_CORE = 4
+
+PT_LOAD = 1
+PT_NOTE = 4
+
+EM_386 = 3
+EM_PPC = 20
+EM_PPC64 = 21
+EM_S390 = 22
+EM_AARCH = 183
+EM_X86_64 = 62
+
+class ELF(object):
+    """Representation of a ELF file."""
+
+    def __init__(self, arch):
+        self.ehdr = None
+        self.notes = []
+        self.segments = []
+        self.notes_size = 0
+        self.endianess = None
+        self.elfclass = ELFCLASS64
+
+        if arch == 'aarch64-le':
+            self.endianess = ELFDATA2LSB
+            self.elfclass = ELFCLASS64
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_AARCH
+
+        elif arch == 'aarch64-be':
+            self.endianess = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_AARCH
+
+        elif arch == 'X86_64':
+            self.endianess = ELFDATA2LSB
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_X86_64
+
+        elif arch == '386':
+            self.endianess = ELFDATA2LSB
+            self.elfclass = ELFCLASS32
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_386
+
+        elif arch == 's390':
+            self.endianess = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_S390
+
+        elif arch == 'ppc64-le':
+            self.endianess = ELFDATA2LSB
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_PPC64
+
+        elif arch == 'ppc64-be':
+            self.endianess = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianess, self.elfclass)
+            self.ehdr.e_machine = EM_PPC64
+
+        else:
+            raise gdb.GdbError("No valid arch type specified.\n"
+                               "Currently supported types:\n"
+                               "aarch64-be, aarch64-le, X86_64, 386, s390, "
+                               "ppc64-be, ppc64-le")
+
+        self.add_segment(PT_NOTE, 0, 0)
+
+    def add_note(self, n_name, n_desc, n_type):
+        """Adds a note to the ELF."""
+
+        note = get_arch_note(self.endianess, len(n_name), len(n_desc))
+        note.n_namesz = len(n_name) + 1
+        note.n_descsz = len(n_desc)
+        note.n_name = n_name.encode()
+        note.n_type = n_type
+
+        # Desc needs to be 4 byte aligned (although the 64bit spec
+        # specifies 8 byte). When defining n_desc as uint32 it will be
+        # automatically aligned but we need the memmove to copy the
+        # string into it.
+        ctypes.memmove(note.n_desc, n_desc.encode(), len(n_desc))
+
+        self.notes.append(note)
+        self.segments[0].p_filesz += ctypes.sizeof(note)
+        self.segments[0].p_memsz += ctypes.sizeof(note)
+
+    def add_segment(self, p_type, p_paddr, p_size):
+        """Adds a segment to the elf."""
+
+        phdr = get_arch_phdr(self.endianess, self.elfclass)
+        phdr.p_type = p_type
+        phdr.p_paddr = p_paddr
+        phdr.p_filesz = p_size
+        phdr.p_memsz = p_size
+        self.segments.append(phdr)
+        self.ehdr.e_phnum += 1
+
+    def to_file(self, elf_file):
+        """Writes all ELF structures to the the passed file.
+
+        Structure:
+        Ehdr
+        Segment 0:PT_NOTE
+        Segment 1:PT_LOAD
+        Segment N:PT_LOAD
+        Note    0..N
+        Dump contents
+        """
+        elf_file.write(self.ehdr)
+        off = ctypes.sizeof(self.ehdr) + \
+              len(self.segments) * ctypes.sizeof(self.segments[0])
+
+        for phdr in self.segments:
+            phdr.p_offset = off
+            elf_file.write(phdr)
+            off += phdr.p_filesz
+
+        for note in self.notes:
+            elf_file.write(note)
+
+
+def get_arch_note(endianess, len_name, len_desc):
+    """Returns a Note class with the specified endianess."""
+
+    if endianess == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    len_name = len_name + 1
+
+    class Note(superclass):
+        """Represents an ELF note, includes the content."""
+
+        _fields_ = [("n_namesz", ctypes.c_uint32),
+                    ("n_descsz", ctypes.c_uint32),
+                    ("n_type", ctypes.c_uint32),
+                    ("n_name", ctypes.c_char * len_name),
+                    ("n_desc", ctypes.c_uint32 * ((len_desc + 3) // 4))]
+    return Note()
+
+
+class Ident(ctypes.Structure):
+    """Represents the ELF ident array in the ehdr structure."""
+
+    _fields_ = [('ei_mag0', ctypes.c_ubyte),
+                ('ei_mag1', ctypes.c_ubyte),
+                ('ei_mag2', ctypes.c_ubyte),
+                ('ei_mag3', ctypes.c_ubyte),
+                ('ei_class', ctypes.c_ubyte),
+                ('ei_data', ctypes.c_ubyte),
+                ('ei_version', ctypes.c_ubyte),
+                ('ei_osabi', ctypes.c_ubyte),
+                ('ei_abiversion', ctypes.c_ubyte),
+                ('ei_pad', ctypes.c_ubyte * 7)]
+
+    def __init__(self, endianess, elfclass):
+        self.ei_mag0 = 0x7F
+        self.ei_mag1 = ord('E')
+        self.ei_mag2 = ord('L')
+        self.ei_mag3 = ord('F')
+        self.ei_class = elfclass
+        self.ei_data = endianess
+        self.ei_version = EV_CURRENT
+
+
+def get_arch_ehdr(endianess, elfclass):
+    """Returns a EHDR64 class with the specified endianess."""
+
+    if endianess == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    class EHDR64(superclass):
+        """Represents the 64 bit ELF header struct."""
+
+        _fields_ = [('e_ident', Ident),
+                    ('e_type', ctypes.c_uint16),
+                    ('e_machine', ctypes.c_uint16),
+                    ('e_version', ctypes.c_uint32),
+                    ('e_entry', ctypes.c_uint64),
+                    ('e_phoff', ctypes.c_uint64),
+                    ('e_shoff', ctypes.c_uint64),
+                    ('e_flags', ctypes.c_uint32),
+                    ('e_ehsize', ctypes.c_uint16),
+                    ('e_phentsize', ctypes.c_uint16),
+                    ('e_phnum', ctypes.c_uint16),
+                    ('e_shentsize', ctypes.c_uint16),
+                    ('e_shnum', ctypes.c_uint16),
+                    ('e_shstrndx', ctypes.c_uint16)]
+
+        def __init__(self):
+            super(superclass, self).__init__()
+            self.e_ident = Ident(endianess, elfclass)
+            self.e_type = ET_CORE
+            self.e_version = EV_CURRENT
+            self.e_ehsize = ctypes.sizeof(self)
+            self.e_phoff = ctypes.sizeof(self)
+            self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass))
+            self.e_phnum = 0
+
+
+    class EHDR32(superclass):
+        """Represents the 32 bit ELF header struct."""
+
+        _fields_ = [('e_ident', Ident),
+                    ('e_type', ctypes.c_uint16),
+                    ('e_machine', ctypes.c_uint16),
+                    ('e_version', ctypes.c_uint32),
+                    ('e_entry', ctypes.c_uint32),
+                    ('e_phoff', ctypes.c_uint32),
+                    ('e_shoff', ctypes.c_uint32),
+                    ('e_flags', ctypes.c_uint32),
+                    ('e_ehsize', ctypes.c_uint16),
+                    ('e_phentsize', ctypes.c_uint16),
+                    ('e_phnum', ctypes.c_uint16),
+                    ('e_shentsize', ctypes.c_uint16),
+                    ('e_shnum', ctypes.c_uint16),
+                    ('e_shstrndx', ctypes.c_uint16)]
+
+        def __init__(self):
+            super(superclass, self).__init__()
+            self.e_ident = Ident(endianess, elfclass)
+            self.e_type = ET_CORE
+            self.e_version = EV_CURRENT
+            self.e_ehsize = ctypes.sizeof(self)
+            self.e_phoff = ctypes.sizeof(self)
+            self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass))
+            self.e_phnum = 0
+
+    # End get_arch_ehdr
+    if elfclass == ELFCLASS64:
+        return EHDR64()
+    else:
+        return EHDR32()
+
+
+def get_arch_phdr(endianess, elfclass):
+    """Returns a 32 or 64 bit PHDR class with the specified endianess."""
+
+    if endianess == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    class PHDR64(superclass):
+        """Represents the 64 bit ELF program header struct."""
+
+        _fields_ = [('p_type', ctypes.c_uint32),
+                    ('p_flags', ctypes.c_uint32),
+                    ('p_offset', ctypes.c_uint64),
+                    ('p_vaddr', ctypes.c_uint64),
+                    ('p_paddr', ctypes.c_uint64),
+                    ('p_filesz', ctypes.c_uint64),
+                    ('p_memsz', ctypes.c_uint64),
+                    ('p_align', ctypes.c_uint64)]
+
+    class PHDR32(superclass):
+        """Represents the 32 bit ELF program header struct."""
+
+        _fields_ = [('p_type', ctypes.c_uint32),
+                    ('p_offset', ctypes.c_uint32),
+                    ('p_vaddr', ctypes.c_uint32),
+                    ('p_paddr', ctypes.c_uint32),
+                    ('p_filesz', ctypes.c_uint32),
+                    ('p_memsz', ctypes.c_uint32),
+                    ('p_flags', ctypes.c_uint32),
+                    ('p_align', ctypes.c_uint32)]
+
+    # End get_arch_phdr
+    if elfclass == ELFCLASS64:
+        return PHDR64()
+    else:
+        return PHDR32()
+
+
+def int128_get64(val):
+    """Returns low 64bit part of Int128 struct."""
+
+    assert val["hi"] == 0
+    return val["lo"]
+
+
+def qlist_foreach(head, field_str):
+    """Generator for qlists."""
+
+    var_p = head["lh_first"]
+    while var_p != 0:
+        var = var_p.dereference()
+        var_p = var[field_str]["le_next"]
+        yield var
+
+
+def qemu_get_ram_block(ram_addr):
+    """Returns the RAMBlock struct to which the given address belongs."""
+
+    ram_blocks = gdb.parse_and_eval("ram_list.blocks")
+
+    for block in qlist_foreach(ram_blocks, "next"):
+        if (ram_addr - block["offset"]) < block["used_length"]:
+            return block
+
+    raise gdb.GdbError("Bad ram offset %x" % ram_addr)
+
+
+def qemu_get_ram_ptr(ram_addr):
+    """Returns qemu vaddr for given guest physical address."""
+
+    block = qemu_get_ram_block(ram_addr)
+    return block["host"] + (ram_addr - block["offset"])
+
+
+def memory_region_get_ram_ptr(memory_region):
+    if memory_region["alias"] != 0:
+        return (memory_region_get_ram_ptr(memory_region["alias"].dereference())
+                + memory_region["alias_offset"])
+
+    return qemu_get_ram_ptr(memory_region["ram_block"]["offset"])
+
+
+def get_guest_phys_blocks():
+    """Returns a list of ram blocks.
+
+    Each block entry contains:
+    'target_start': guest block phys start address
+    'target_end':   guest block phys end address
+    'host_addr':    qemu vaddr of the block's start
+    """
+
+    guest_phys_blocks = []
+
+    print("guest RAM blocks:")
+    print("target_start     target_end       host_addr        message "
+          "count")
+    print("---------------- ---------------- ---------------- ------- "
+          "-----")
+
+    current_map_p = gdb.parse_and_eval("address_space_memory.current_map")
+    current_map = current_map_p.dereference()
+
+    # Conversion to int is needed for python 3
+    # compatibility. Otherwise range doesn't cast the value itself and
+    # breaks.
+    for cur in range(int(current_map["nr"])):
+        flat_range = (current_map["ranges"] + cur).dereference()
+        memory_region = flat_range["mr"].dereference()
+
+        # we only care about RAM
+        if not memory_region["ram"]:
+            continue
+
+        section_size = int128_get64(flat_range["addr"]["size"])
+        target_start = int128_get64(flat_range["addr"]["start"])
+        target_end = target_start + section_size
+        host_addr = (memory_region_get_ram_ptr(memory_region)
+                     + flat_range["offset_in_region"])
+        predecessor = None
+
+        # find continuity in guest physical address space
+        if len(guest_phys_blocks) > 0:
+            predecessor = guest_phys_blocks[-1]
+            predecessor_size = (predecessor["target_end"] -
+                                predecessor["target_start"])
+
+            # the memory API guarantees monotonically increasing
+            # traversal
+            assert predecessor["target_end"] <= target_start
+
+            # we want continuity in both guest-physical and
+            # host-virtual memory
+            if (predecessor["target_end"] < target_start or
+                predecessor["host_addr"] + predecessor_size != host_addr):
+                predecessor = None
+
+        if predecessor is None:
+            # isolated mapping, add it to the list
+            guest_phys_blocks.append({"target_start": target_start,
+                                      "target_end":   target_end,
+                                      "host_addr":    host_addr})
+            message = "added"
+        else:
+            # expand predecessor until @target_end; predecessor's
+            # start doesn't change
+            predecessor["target_end"] = target_end
+            message = "joined"
+
+        print("%016x %016x %016x %-7s %5u" %
+              (target_start, target_end, host_addr.cast(UINTPTR_T),
+               message, len(guest_phys_blocks)))
+
+    return guest_phys_blocks
+
+
 # The leading docstring doesn't have idiomatic Python formatting. It is
 # printed by gdb's "help" command (the first line is printed in the
 # "help data" summary), and it should match how other help texts look in
 # gdb.
-
-import struct
-
 class DumpGuestMemory(gdb.Command):
     """Extract guest vmcore from qemu process coredump.
 
-The sole argument is FILE, identifying the target file to write the
-guest vmcore to.
+The two required arguments are FILE and ARCH:
+FILE identifies the target file to write the guest vmcore to.
+ARCH specifies the architecture for which the core will be generated.
 
 This GDB command reimplements the dump-guest-memory QMP command in
 python, using the representation of guest memory as captured in the qemu
 coredump. The qemu process that has been dumped must have had the
-command line option "-machine dump-guest-core=on".
+command line option "-machine dump-guest-core=on" which is the default.
 
 For simplicity, the "paging", "begin" and "end" parameters of the QMP
 command are not supported -- no attempt is made to get the guest's
 internal paging structures (ie. paging=false is hard-wired), and guest
 memory is always fully dumped.
 
-Only x86_64 guests are supported.
+Currently aarch64-be, aarch64-le, X86_64, 386, s390, ppc64-be,
+ppc64-le guests are supported.
 
 The CORE/NT_PRSTATUS and QEMU notes (that is, the VCPUs' statuses) are
 not written to the vmcore. Preparing these would require context that is
@@ -47,293 +464,66 @@ deliberately called abort(), or it was dumped in response to a signal at
 a halfway fortunate point, then its coredump should be in reasonable
 shape and this command should mostly work."""
 
-    TARGET_PAGE_SIZE = 0x1000
-    TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000
-
-    # Various ELF constants
-    EM_X86_64   = 62        # AMD x86-64 target machine
-    ELFDATA2LSB = 1         # little endian
-    ELFCLASS64  = 2
-    ELFMAG      = "\x7FELF"
-    EV_CURRENT  = 1
-    ET_CORE     = 4
-    PT_LOAD     = 1
-    PT_NOTE     = 4
-
-    # Special value for e_phnum. This indicates that the real number of
-    # program headers is too large to fit into e_phnum. Instead the real
-    # value is in the field sh_info of section 0.
-    PN_XNUM = 0xFFFF
-
-    # Format strings for packing and header size calculation.
-    ELF64_EHDR = ("4s" # e_ident/magic
-                  "B"  # e_ident/class
-                  "B"  # e_ident/data
-                  "B"  # e_ident/version
-                  "B"  # e_ident/osabi
-                  "8s" # e_ident/pad
-                  "H"  # e_type
-                  "H"  # e_machine
-                  "I"  # e_version
-                  "Q"  # e_entry
-                  "Q"  # e_phoff
-                  "Q"  # e_shoff
-                  "I"  # e_flags
-                  "H"  # e_ehsize
-                  "H"  # e_phentsize
-                  "H"  # e_phnum
-                  "H"  # e_shentsize
-                  "H"  # e_shnum
-                  "H"  # e_shstrndx
-                 )
-    ELF64_PHDR = ("I"  # p_type
-                  "I"  # p_flags
-                  "Q"  # p_offset
-                  "Q"  # p_vaddr
-                  "Q"  # p_paddr
-                  "Q"  # p_filesz
-                  "Q"  # p_memsz
-                  "Q"  # p_align
-                 )
-
     def __init__(self):
         super(DumpGuestMemory, self).__init__("dump-guest-memory",
                                               gdb.COMMAND_DATA,
                                               gdb.COMPLETE_FILENAME)
-        self.uintptr_t     = gdb.lookup_type("uintptr_t")
-        self.elf64_ehdr_le = struct.Struct("<%s" % self.ELF64_EHDR)
-        self.elf64_phdr_le = struct.Struct("<%s" % self.ELF64_PHDR)
-
-    def int128_get64(self, val):
-        assert (val["hi"] == 0)
-        return val["lo"]
-
-    def qlist_foreach(self, head, field_str):
-        var_p = head["lh_first"]
-        while (var_p != 0):
-            var = var_p.dereference()
-            yield var
-            var_p = var[field_str]["le_next"]
-
-    def qemu_get_ram_block(self, ram_addr):
-        ram_blocks = gdb.parse_and_eval("ram_list.blocks")
-        for block in self.qlist_foreach(ram_blocks, "next"):
-            if (ram_addr - block["offset"] < block["length"]):
-                return block
-        raise gdb.GdbError("Bad ram offset %x" % ram_addr)
-
-    def qemu_get_ram_ptr(self, ram_addr):
-        block = self.qemu_get_ram_block(ram_addr)
-        return block["host"] + (ram_addr - block["offset"])
-
-    def memory_region_get_ram_ptr(self, mr):
-        if (mr["alias"] != 0):
-            return (self.memory_region_get_ram_ptr(mr["alias"].dereference()) +
-                    mr["alias_offset"])
-        return self.qemu_get_ram_ptr(mr["ram_addr"] & self.TARGET_PAGE_MASK)
-
-    def guest_phys_blocks_init(self):
-        self.guest_phys_blocks = []
-
-    def guest_phys_blocks_append(self):
-        print "guest RAM blocks:"
-        print ("target_start     target_end       host_addr        message "
-               "count")
-        print ("---------------- ---------------- ---------------- ------- "
-               "-----")
-
-        current_map_p = gdb.parse_and_eval("address_space_memory.current_map")
-        current_map = current_map_p.dereference()
-        for cur in range(current_map["nr"]):
-            flat_range   = (current_map["ranges"] + cur).dereference()
-            mr           = flat_range["mr"].dereference()
-
-            # we only care about RAM
-            if (not mr["ram"]):
-                continue
-
-            section_size = self.int128_get64(flat_range["addr"]["size"])
-            target_start = self.int128_get64(flat_range["addr"]["start"])
-            target_end   = target_start + section_size
-            host_addr    = (self.memory_region_get_ram_ptr(mr) +
-                            flat_range["offset_in_region"])
-            predecessor = None
-
-            # find continuity in guest physical address space
-            if (len(self.guest_phys_blocks) > 0):
-                predecessor = self.guest_phys_blocks[-1]
-                predecessor_size = (predecessor["target_end"] -
-                                    predecessor["target_start"])
-
-                # the memory API guarantees monotonically increasing
-                # traversal
-                assert (predecessor["target_end"] <= target_start)
-
-                # we want continuity in both guest-physical and
-                # host-virtual memory
-                if (predecessor["target_end"] < target_start or
-                    predecessor["host_addr"] + predecessor_size != host_addr):
-                    predecessor = None
-
-            if (predecessor is None):
-                # isolated mapping, add it to the list
-                self.guest_phys_blocks.append({"target_start": target_start,
-                                               "target_end"  : target_end,
-                                               "host_addr"   : host_addr})
-                message = "added"
-            else:
-                # expand predecessor until @target_end; predecessor's
-                # start doesn't change
-                predecessor["target_end"] = target_end
-                message = "joined"
-
-            print ("%016x %016x %016x %-7s %5u" %
-                   (target_start, target_end, host_addr.cast(self.uintptr_t),
-                    message, len(self.guest_phys_blocks)))
-
-    def cpu_get_dump_info(self):
-        # We can't synchronize the registers with KVM post-mortem, and
-        # the bits in (first_x86_cpu->env.hflags) seem to be stale; they
-        # may not reflect long mode for example. Hence just assume the
-        # most common values. This also means that instruction pointer
-        # etc. will be bogus in the dump, but at least the RAM contents
-        # should be valid.
-        self.dump_info = {"d_machine": self.EM_X86_64,
-                          "d_endian" : self.ELFDATA2LSB,
-                          "d_class"  : self.ELFCLASS64}
-
-    def encode_elf64_ehdr_le(self):
-        return self.elf64_ehdr_le.pack(
-                                 self.ELFMAG,                 # e_ident/magic
-                                 self.dump_info["d_class"],   # e_ident/class
-                                 self.dump_info["d_endian"],  # e_ident/data
-                                 self.EV_CURRENT,             # e_ident/version
-                                 0,                           # e_ident/osabi
-                                 "",                          # e_ident/pad
-                                 self.ET_CORE,                # e_type
-                                 self.dump_info["d_machine"], # e_machine
-                                 self.EV_CURRENT,             # e_version
-                                 0,                           # e_entry
-                                 self.elf64_ehdr_le.size,     # e_phoff
-                                 0,                           # e_shoff
-                                 0,                           # e_flags
-                                 self.elf64_ehdr_le.size,     # e_ehsize
-                                 self.elf64_phdr_le.size,     # e_phentsize
-                                 self.phdr_num,               # e_phnum
-                                 0,                           # e_shentsize
-                                 0,                           # e_shnum
-                                 0                            # e_shstrndx
-                                )
-
-    def encode_elf64_note_le(self):
-        return self.elf64_phdr_le.pack(self.PT_NOTE,         # p_type
-                                       0,                    # p_flags
-                                       (self.memory_offset -
-                                        len(self.note)),     # p_offset
-                                       0,                    # p_vaddr
-                                       0,                    # p_paddr
-                                       len(self.note),       # p_filesz
-                                       len(self.note),       # p_memsz
-                                       0                     # p_align
-                                      )
-
-    def encode_elf64_load_le(self, offset, start_hwaddr, range_size):
-        return self.elf64_phdr_le.pack(self.PT_LOAD, # p_type
-                                       0,            # p_flags
-                                       offset,       # p_offset
-                                       0,            # p_vaddr
-                                       start_hwaddr, # p_paddr
-                                       range_size,   # p_filesz
-                                       range_size,   # p_memsz
-                                       0             # p_align
-                                      )
-
-    def note_init(self, name, desc, type):
-        # name must include a trailing NUL
-        namesz = (len(name) + 1 + 3) / 4 * 4
-        descsz = (len(desc)     + 3) / 4 * 4
-        fmt = ("<"   # little endian
-               "I"   # n_namesz
-               "I"   # n_descsz
-               "I"   # n_type
-               "%us" # name
-               "%us" # desc
-               % (namesz, descsz))
-        self.note = struct.pack(fmt,
-                                len(name) + 1, len(desc), type, name, desc)
-
-    def dump_init(self):
-        self.guest_phys_blocks_init()
-        self.guest_phys_blocks_append()
-        self.cpu_get_dump_info()
-        # we have no way to retrieve the VCPU status from KVM
-        # post-mortem
-        self.note_init("NONE", "EMPTY", 0)
-
-        # Account for PT_NOTE.
-        self.phdr_num = 1
-
-        # We should never reach PN_XNUM for paging=false dumps: there's
-        # just a handful of discontiguous ranges after merging.
-        self.phdr_num += len(self.guest_phys_blocks)
-        assert (self.phdr_num < self.PN_XNUM)
-
-        # Calculate the ELF file offset where the memory dump commences:
-        #
-        #   ELF header
-        #   PT_NOTE
-        #   PT_LOAD: 1
-        #   PT_LOAD: 2
-        #   ...
-        #   PT_LOAD: len(self.guest_phys_blocks)
-        #   ELF note
-        #   memory dump
-        self.memory_offset = (self.elf64_ehdr_le.size +
-                              self.elf64_phdr_le.size * self.phdr_num +
-                              len(self.note))
-
-    def dump_begin(self, vmcore):
-        vmcore.write(self.encode_elf64_ehdr_le())
-        vmcore.write(self.encode_elf64_note_le())
-        running = self.memory_offset
+        self.elf = None
+        self.guest_phys_blocks = None
+
+    def dump_init(self, vmcore):
+        """Prepares and writes ELF structures to core file."""
+
+        # Needed to make crash happy, data for more useful notes is
+        # not available in a qemu core.
+        self.elf.add_note("NONE", "EMPTY", 0)
+
+        # We should never reach PN_XNUM for paging=false dumps,
+        # there's just a handful of discontiguous ranges after
+        # merging.
+        # The constant is needed to account for the PT_NOTE segment.
+        phdr_num = len(self.guest_phys_blocks) + 1
+        assert phdr_num < PN_XNUM
+
         for block in self.guest_phys_blocks:
-            range_size = block["target_end"] - block["target_start"]
-            vmcore.write(self.encode_elf64_load_le(running,
-                                                   block["target_start"],
-                                                   range_size))
-            running += range_size
-        vmcore.write(self.note)
+            block_size = block["target_end"] - block["target_start"]
+            self.elf.add_segment(PT_LOAD, block["target_start"], block_size)
+
+        self.elf.to_file(vmcore)
 
     def dump_iterate(self, vmcore):
+        """Writes guest core to file."""
+
         qemu_core = gdb.inferiors()[0]
         for block in self.guest_phys_blocks:
-            cur  = block["host_addr"]
+            cur = block["host_addr"]
             left = block["target_end"] - block["target_start"]
-            print ("dumping range at %016x for length %016x" %
-                   (cur.cast(self.uintptr_t), left))
-            while (left > 0):
-                chunk_size = min(self.TARGET_PAGE_SIZE, left)
+            print("dumping range at %016x for length %016x" %
+                  (cur.cast(UINTPTR_T), left))
+
+            while left > 0:
+                chunk_size = min(TARGET_PAGE_SIZE, left)
                 chunk = qemu_core.read_memory(cur, chunk_size)
                 vmcore.write(chunk)
-                cur  += chunk_size
+                cur += chunk_size
                 left -= chunk_size
 
-    def create_vmcore(self, filename):
-        vmcore = open(filename, "wb")
-        self.dump_begin(vmcore)
-        self.dump_iterate(vmcore)
-        vmcore.close()
-
     def invoke(self, args, from_tty):
+        """Handles command invocation from gdb."""
+
         # Unwittingly pressing the Enter key after the command should
         # not dump the same multi-gig coredump to the same file.
         self.dont_repeat()
 
         argv = gdb.string_to_argv(args)
-        if (len(argv) != 1):
-            raise gdb.GdbError("usage: dump-guest-memory FILE")
+        if len(argv) != 2:
+            raise gdb.GdbError("usage: dump-guest-memory FILE ARCH")
+
+        self.elf = ELF(argv[1])
+        self.guest_phys_blocks = get_guest_phys_blocks()
 
-        self.dump_init()
-        self.create_vmcore(argv[0])
+        with open(argv[0], "wb") as vmcore:
+            self.dump_init(vmcore)
+            self.dump_iterate(vmcore)
 
 DumpGuestMemory()
diff --git a/qemu/scripts/feature_to_c.sh b/qemu/scripts/feature_to_c.sh
index 888548e58..fb1f3363f 100644
--- a/qemu/scripts/feature_to_c.sh
+++ b/qemu/scripts/feature_to_c.sh
@@ -36,7 +36,7 @@ for input; do
   arrayname=xml_feature_`echo $input | sed 's,.*/,,; s/[-.]/_/g'`
 
   ${AWK:-awk} 'BEGIN { n = 0
-      printf "#include \"config.h\"\n"
+      printf "#include \"qemu/osdep.h\"\n"
       printf "#include \"qemu-common.h\"\n"
       printf "#include \"exec/gdbstub.h\"\n"
       print "static const char '$arrayname'[] = {"
diff --git a/qemu/scripts/get_maintainer.pl b/qemu/scripts/get_maintainer.pl
index f39630eb3..8261bcb1a 100755
--- a/qemu/scripts/get_maintainer.pl
+++ b/qemu/scripts/get_maintainer.pl
@@ -258,7 +258,7 @@ open (my $maint, '<', "${lk_path}MAINTAINERS")
 while (<$maint>) {
     my $line = $_;
 
-    if ($line =~ m/^(\C):\s*(.*)/) {
+    if ($line =~ m/^(.):\s*(.*)/) {
 	my $type = $1;
 	my $value = $2;
 
@@ -493,7 +493,7 @@ sub range_is_maintained {
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'S') {
@@ -511,7 +511,7 @@ sub range_has_maintainer {
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'M') {
@@ -560,7 +560,7 @@ sub get_maintainers {
 
 	    for ($i = $start; $i < $end; $i++) {
 		my $line = $typevalue[$i];
-		if ($line =~ m/^(\C):\s*(.*)/) {
+		if ($line =~ m/^(.):\s*(.*)/) {
 		    my $type = $1;
 		    my $value = $2;
 		    if ($type eq 'X') {
@@ -575,7 +575,7 @@ sub get_maintainers {
 	    if (!$exclude) {
 		for ($i = $start; $i < $end; $i++) {
 		    my $line = $typevalue[$i];
-		    if ($line =~ m/^(\C):\s*(.*)/) {
+		    if ($line =~ m/^(.):\s*(.*)/) {
 			my $type = $1;
 			my $value = $2;
 			if ($type eq 'F') {
@@ -636,7 +636,7 @@ sub get_maintainers {
 
     if ($email) {
 	if (! $interactive) {
-	    $email_git_fallback = 0 if @email_to > 0 || @list_to > 0 || $email_git || $email_git_blame;
+	    $email_git_fallback = 0 if @email_to > 0 || $email_git || $email_git_blame;
 	    if ($email_git_fallback) {
 	        print STDERR "get_maintainer.pl: No maintainers found, printing recent contributors.\n";
 	        print STDERR "get_maintainer.pl: Do not blindly cc: them on patches!  Use common sense.\n";
@@ -855,7 +855,7 @@ sub find_first_section {
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (($tv =~ m/^(\C):\s*(.*)/)) {
+	if (($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -869,7 +869,7 @@ sub find_starting_index {
 
     while ($index > 0) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index--;
@@ -883,7 +883,7 @@ sub find_ending_index {
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -909,7 +909,7 @@ sub get_maintainer_role {
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "S") {
@@ -968,7 +968,7 @@ sub add_categories {
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "L") {
@@ -1010,7 +1010,7 @@ sub add_categories {
 		if ($name eq "") {
 		    if ($i > 0) {
 			my $tv = $typevalue[$i - 1];
-			if ($tv =~ m/^(\C):\s*(.*)/) {
+			if ($tv =~ m/^(.):\s*(.*)/) {
 			    if ($1 eq "P") {
 				$name = $2;
 				$pvalue = format_email($name, $address, $email_usename);
diff --git a/qemu/scripts/kvm/kvm_stat b/qemu/scripts/kvm/kvm_stat
index 7e5d25612..769d884b6 100755
--- a/qemu/scripts/kvm/kvm_stat
+++ b/qemu/scripts/kvm/kvm_stat
@@ -12,285 +12,312 @@
 # the COPYING file in the top-level directory.
 
 import curses
-import sys, os, time, optparse, ctypes
-from ctypes import *
-
-class DebugfsProvider(object):
-    def __init__(self):
-        self.base = '/sys/kernel/debug/kvm'
-        self._fields = os.listdir(self.base)
-    def fields(self):
-        return self._fields
-    def select(self, fields):
-        self._fields = fields
-    def read(self):
-        def val(key):
-            return int(file(self.base + '/' + key).read())
-        return dict([(key, val(key)) for key in self._fields])
-
-vmx_exit_reasons = {
-    0: 'EXCEPTION_NMI',
-    1: 'EXTERNAL_INTERRUPT',
-    2: 'TRIPLE_FAULT',
-    7: 'PENDING_INTERRUPT',
-    8: 'NMI_WINDOW',
-    9: 'TASK_SWITCH',
-    10: 'CPUID',
-    12: 'HLT',
-    14: 'INVLPG',
-    15: 'RDPMC',
-    16: 'RDTSC',
-    18: 'VMCALL',
-    19: 'VMCLEAR',
-    20: 'VMLAUNCH',
-    21: 'VMPTRLD',
-    22: 'VMPTRST',
-    23: 'VMREAD',
-    24: 'VMRESUME',
-    25: 'VMWRITE',
-    26: 'VMOFF',
-    27: 'VMON',
-    28: 'CR_ACCESS',
-    29: 'DR_ACCESS',
-    30: 'IO_INSTRUCTION',
-    31: 'MSR_READ',
-    32: 'MSR_WRITE',
-    33: 'INVALID_STATE',
-    36: 'MWAIT_INSTRUCTION',
-    39: 'MONITOR_INSTRUCTION',
-    40: 'PAUSE_INSTRUCTION',
-    41: 'MCE_DURING_VMENTRY',
-    43: 'TPR_BELOW_THRESHOLD',
-    44: 'APIC_ACCESS',
-    48: 'EPT_VIOLATION',
-    49: 'EPT_MISCONFIG',
-    54: 'WBINVD',
-    55: 'XSETBV',
-    56: 'APIC_WRITE',
-    58: 'INVPCID',
+import sys
+import os
+import time
+import optparse
+import ctypes
+import fcntl
+import resource
+import struct
+import re
+from collections import defaultdict
+from time import sleep
+
+VMX_EXIT_REASONS = {
+    'EXCEPTION_NMI':        0,
+    'EXTERNAL_INTERRUPT':   1,
+    'TRIPLE_FAULT':         2,
+    'PENDING_INTERRUPT':    7,
+    'NMI_WINDOW':           8,
+    'TASK_SWITCH':          9,
+    'CPUID':                10,
+    'HLT':                  12,
+    'INVLPG':               14,
+    'RDPMC':                15,
+    'RDTSC':                16,
+    'VMCALL':               18,
+    'VMCLEAR':              19,
+    'VMLAUNCH':             20,
+    'VMPTRLD':              21,
+    'VMPTRST':              22,
+    'VMREAD':               23,
+    'VMRESUME':             24,
+    'VMWRITE':              25,
+    'VMOFF':                26,
+    'VMON':                 27,
+    'CR_ACCESS':            28,
+    'DR_ACCESS':            29,
+    'IO_INSTRUCTION':       30,
+    'MSR_READ':             31,
+    'MSR_WRITE':            32,
+    'INVALID_STATE':        33,
+    'MWAIT_INSTRUCTION':    36,
+    'MONITOR_INSTRUCTION':  39,
+    'PAUSE_INSTRUCTION':    40,
+    'MCE_DURING_VMENTRY':   41,
+    'TPR_BELOW_THRESHOLD':  43,
+    'APIC_ACCESS':          44,
+    'EPT_VIOLATION':        48,
+    'EPT_MISCONFIG':        49,
+    'WBINVD':               54,
+    'XSETBV':               55,
+    'APIC_WRITE':           56,
+    'INVPCID':              58,
 }
 
-svm_exit_reasons = {
-    0x000: 'READ_CR0',
-    0x003: 'READ_CR3',
-    0x004: 'READ_CR4',
-    0x008: 'READ_CR8',
-    0x010: 'WRITE_CR0',
-    0x013: 'WRITE_CR3',
-    0x014: 'WRITE_CR4',
-    0x018: 'WRITE_CR8',
-    0x020: 'READ_DR0',
-    0x021: 'READ_DR1',
-    0x022: 'READ_DR2',
-    0x023: 'READ_DR3',
-    0x024: 'READ_DR4',
-    0x025: 'READ_DR5',
-    0x026: 'READ_DR6',
-    0x027: 'READ_DR7',
-    0x030: 'WRITE_DR0',
-    0x031: 'WRITE_DR1',
-    0x032: 'WRITE_DR2',
-    0x033: 'WRITE_DR3',
-    0x034: 'WRITE_DR4',
-    0x035: 'WRITE_DR5',
-    0x036: 'WRITE_DR6',
-    0x037: 'WRITE_DR7',
-    0x040: 'EXCP_BASE',
-    0x060: 'INTR',
-    0x061: 'NMI',
-    0x062: 'SMI',
-    0x063: 'INIT',
-    0x064: 'VINTR',
-    0x065: 'CR0_SEL_WRITE',
-    0x066: 'IDTR_READ',
-    0x067: 'GDTR_READ',
-    0x068: 'LDTR_READ',
-    0x069: 'TR_READ',
-    0x06a: 'IDTR_WRITE',
-    0x06b: 'GDTR_WRITE',
-    0x06c: 'LDTR_WRITE',
-    0x06d: 'TR_WRITE',
-    0x06e: 'RDTSC',
-    0x06f: 'RDPMC',
-    0x070: 'PUSHF',
-    0x071: 'POPF',
-    0x072: 'CPUID',
-    0x073: 'RSM',
-    0x074: 'IRET',
-    0x075: 'SWINT',
-    0x076: 'INVD',
-    0x077: 'PAUSE',
-    0x078: 'HLT',
-    0x079: 'INVLPG',
-    0x07a: 'INVLPGA',
-    0x07b: 'IOIO',
-    0x07c: 'MSR',
-    0x07d: 'TASK_SWITCH',
-    0x07e: 'FERR_FREEZE',
-    0x07f: 'SHUTDOWN',
-    0x080: 'VMRUN',
-    0x081: 'VMMCALL',
-    0x082: 'VMLOAD',
-    0x083: 'VMSAVE',
-    0x084: 'STGI',
-    0x085: 'CLGI',
-    0x086: 'SKINIT',
-    0x087: 'RDTSCP',
-    0x088: 'ICEBP',
-    0x089: 'WBINVD',
-    0x08a: 'MONITOR',
-    0x08b: 'MWAIT',
-    0x08c: 'MWAIT_COND',
-    0x08d: 'XSETBV',
-    0x400: 'NPF',
+SVM_EXIT_REASONS = {
+    'READ_CR0':       0x000,
+    'READ_CR3':       0x003,
+    'READ_CR4':       0x004,
+    'READ_CR8':       0x008,
+    'WRITE_CR0':      0x010,
+    'WRITE_CR3':      0x013,
+    'WRITE_CR4':      0x014,
+    'WRITE_CR8':      0x018,
+    'READ_DR0':       0x020,
+    'READ_DR1':       0x021,
+    'READ_DR2':       0x022,
+    'READ_DR3':       0x023,
+    'READ_DR4':       0x024,
+    'READ_DR5':       0x025,
+    'READ_DR6':       0x026,
+    'READ_DR7':       0x027,
+    'WRITE_DR0':      0x030,
+    'WRITE_DR1':      0x031,
+    'WRITE_DR2':      0x032,
+    'WRITE_DR3':      0x033,
+    'WRITE_DR4':      0x034,
+    'WRITE_DR5':      0x035,
+    'WRITE_DR6':      0x036,
+    'WRITE_DR7':      0x037,
+    'EXCP_BASE':      0x040,
+    'INTR':           0x060,
+    'NMI':            0x061,
+    'SMI':            0x062,
+    'INIT':           0x063,
+    'VINTR':          0x064,
+    'CR0_SEL_WRITE':  0x065,
+    'IDTR_READ':      0x066,
+    'GDTR_READ':      0x067,
+    'LDTR_READ':      0x068,
+    'TR_READ':        0x069,
+    'IDTR_WRITE':     0x06a,
+    'GDTR_WRITE':     0x06b,
+    'LDTR_WRITE':     0x06c,
+    'TR_WRITE':       0x06d,
+    'RDTSC':          0x06e,
+    'RDPMC':          0x06f,
+    'PUSHF':          0x070,
+    'POPF':           0x071,
+    'CPUID':          0x072,
+    'RSM':            0x073,
+    'IRET':           0x074,
+    'SWINT':          0x075,
+    'INVD':           0x076,
+    'PAUSE':          0x077,
+    'HLT':            0x078,
+    'INVLPG':         0x079,
+    'INVLPGA':        0x07a,
+    'IOIO':           0x07b,
+    'MSR':            0x07c,
+    'TASK_SWITCH':    0x07d,
+    'FERR_FREEZE':    0x07e,
+    'SHUTDOWN':       0x07f,
+    'VMRUN':          0x080,
+    'VMMCALL':        0x081,
+    'VMLOAD':         0x082,
+    'VMSAVE':         0x083,
+    'STGI':           0x084,
+    'CLGI':           0x085,
+    'SKINIT':         0x086,
+    'RDTSCP':         0x087,
+    'ICEBP':          0x088,
+    'WBINVD':         0x089,
+    'MONITOR':        0x08a,
+    'MWAIT':          0x08b,
+    'MWAIT_COND':     0x08c,
+    'XSETBV':         0x08d,
+    'NPF':            0x400,
 }
 
 # EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h)
-aarch64_exit_reasons = {
-    0x00: 'UNKNOWN',
-    0x01: 'WFI',
-    0x03: 'CP15_32',
-    0x04: 'CP15_64',
-    0x05: 'CP14_MR',
-    0x06: 'CP14_LS',
-    0x07: 'FP_ASIMD',
-    0x08: 'CP10_ID',
-    0x0C: 'CP14_64',
-    0x0E: 'ILL_ISS',
-    0x11: 'SVC32',
-    0x12: 'HVC32',
-    0x13: 'SMC32',
-    0x15: 'SVC64',
-    0x16: 'HVC64',
-    0x17: 'SMC64',
-    0x18: 'SYS64',
-    0x20: 'IABT',
-    0x21: 'IABT_HYP',
-    0x22: 'PC_ALIGN',
-    0x24: 'DABT',
-    0x25: 'DABT_HYP',
-    0x26: 'SP_ALIGN',
-    0x28: 'FP_EXC32',
-    0x2C: 'FP_EXC64',
-    0x2F: 'SERROR',
-    0x30: 'BREAKPT',
-    0x31: 'BREAKPT_HYP',
-    0x32: 'SOFTSTP',
-    0x33: 'SOFTSTP_HYP',
-    0x34: 'WATCHPT',
-    0x35: 'WATCHPT_HYP',
-    0x38: 'BKPT32',
-    0x3A: 'VECTOR32',
-    0x3C: 'BRK64',
+AARCH64_EXIT_REASONS = {
+    'UNKNOWN':      0x00,
+    'WFI':          0x01,
+    'CP15_32':      0x03,
+    'CP15_64':      0x04,
+    'CP14_MR':      0x05,
+    'CP14_LS':      0x06,
+    'FP_ASIMD':     0x07,
+    'CP10_ID':      0x08,
+    'CP14_64':      0x0C,
+    'ILL_ISS':      0x0E,
+    'SVC32':        0x11,
+    'HVC32':        0x12,
+    'SMC32':        0x13,
+    'SVC64':        0x15,
+    'HVC64':        0x16,
+    'SMC64':        0x17,
+    'SYS64':        0x18,
+    'IABT':         0x20,
+    'IABT_HYP':     0x21,
+    'PC_ALIGN':     0x22,
+    'DABT':         0x24,
+    'DABT_HYP':     0x25,
+    'SP_ALIGN':     0x26,
+    'FP_EXC32':     0x28,
+    'FP_EXC64':     0x2C,
+    'SERROR':       0x2F,
+    'BREAKPT':      0x30,
+    'BREAKPT_HYP':  0x31,
+    'SOFTSTP':      0x32,
+    'SOFTSTP_HYP':  0x33,
+    'WATCHPT':      0x34,
+    'WATCHPT_HYP':  0x35,
+    'BKPT32':       0x38,
+    'VECTOR32':     0x3A,
+    'BRK64':        0x3C,
 }
 
 # From include/uapi/linux/kvm.h, KVM_EXIT_xxx
-userspace_exit_reasons = {
-     0: 'UNKNOWN',
-     1: 'EXCEPTION',
-     2: 'IO',
-     3: 'HYPERCALL',
-     4: 'DEBUG',
-     5: 'HLT',
-     6: 'MMIO',
-     7: 'IRQ_WINDOW_OPEN',
-     8: 'SHUTDOWN',
-     9: 'FAIL_ENTRY',
-    10: 'INTR',
-    11: 'SET_TPR',
-    12: 'TPR_ACCESS',
-    13: 'S390_SIEIC',
-    14: 'S390_RESET',
-    15: 'DCR',
-    16: 'NMI',
-    17: 'INTERNAL_ERROR',
-    18: 'OSI',
-    19: 'PAPR_HCALL',
-    20: 'S390_UCONTROL',
-    21: 'WATCHDOG',
-    22: 'S390_TSCH',
-    23: 'EPR',
-    24: 'SYSTEM_EVENT',
+USERSPACE_EXIT_REASONS = {
+    'UNKNOWN':          0,
+    'EXCEPTION':        1,
+    'IO':               2,
+    'HYPERCALL':        3,
+    'DEBUG':            4,
+    'HLT':              5,
+    'MMIO':             6,
+    'IRQ_WINDOW_OPEN':  7,
+    'SHUTDOWN':         8,
+    'FAIL_ENTRY':       9,
+    'INTR':             10,
+    'SET_TPR':          11,
+    'TPR_ACCESS':       12,
+    'S390_SIEIC':       13,
+    'S390_RESET':       14,
+    'DCR':              15,
+    'NMI':              16,
+    'INTERNAL_ERROR':   17,
+    'OSI':              18,
+    'PAPR_HCALL':       19,
+    'S390_UCONTROL':    20,
+    'WATCHDOG':         21,
+    'S390_TSCH':        22,
+    'EPR':              23,
+    'SYSTEM_EVENT':     24,
 }
 
-x86_exit_reasons = {
-    'vmx': vmx_exit_reasons,
-    'svm': svm_exit_reasons,
+IOCTL_NUMBERS = {
+    'SET_FILTER':  0x40082406,
+    'ENABLE':      0x00002400,
+    'DISABLE':     0x00002401,
+    'RESET':       0x00002403,
 }
 
-sc_perf_evt_open = None
-exit_reasons = None
+class Arch(object):
+    """Class that encapsulates global architecture specific data like
+    syscall and ioctl numbers.
+
+    """
+    @staticmethod
+    def get_arch():
+        machine = os.uname()[4]
+
+        if machine.startswith('ppc'):
+            return ArchPPC()
+        elif machine.startswith('aarch64'):
+            return ArchA64()
+        elif machine.startswith('s390'):
+            return ArchS390()
+        else:
+            # X86_64
+            for line in open('/proc/cpuinfo'):
+                if not line.startswith('flags'):
+                    continue
+
+                flags = line.split()
+                if 'vmx' in flags:
+                    return ArchX86(VMX_EXIT_REASONS)
+                if 'svm' in flags:
+                    return ArchX86(SVM_EXIT_REASONS)
+                return
+
+class ArchX86(Arch):
+    def __init__(self, exit_reasons):
+        self.sc_perf_evt_open = 298
+        self.ioctl_numbers = IOCTL_NUMBERS
+        self.exit_reasons = exit_reasons
+
+class ArchPPC(Arch):
+    def __init__(self):
+        self.sc_perf_evt_open = 319
+        self.ioctl_numbers = IOCTL_NUMBERS
+        self.ioctl_numbers['ENABLE'] = 0x20002400
+        self.ioctl_numbers['DISABLE'] = 0x20002401
 
-ioctl_numbers = {
-    'SET_FILTER' : 0x40082406,
-    'ENABLE'     : 0x00002400,
-    'DISABLE'    : 0x00002401,
-    'RESET'      : 0x00002403,
-}
+        # PPC comes in 32 and 64 bit and some generated ioctl
+        # numbers depend on the wordsize.
+        char_ptr_size = ctypes.sizeof(ctypes.c_char_p)
+        self.ioctl_numbers['SET_FILTER'] = 0x80002406 | char_ptr_size << 16
+
+class ArchA64(Arch):
+    def __init__(self):
+        self.sc_perf_evt_open = 241
+        self.ioctl_numbers = IOCTL_NUMBERS
+        self.exit_reasons = AARCH64_EXIT_REASONS
+
+class ArchS390(Arch):
+    def __init__(self):
+        self.sc_perf_evt_open = 331
+        self.ioctl_numbers = IOCTL_NUMBERS
+        self.exit_reasons = None
+
+ARCH = Arch.get_arch()
+
+
+def walkdir(path):
+    """Returns os.walk() data for specified directory.
+
+    As it is only a wrapper it returns the same 3-tuple of (dirpath,
+    dirnames, filenames).
+    """
+    return next(os.walk(path))
+
+
+def parse_int_list(list_string):
+    """Returns an int list from a string of comma separated integers and
+    integer ranges."""
+    integers = []
+    members = list_string.split(',')
 
-def x86_init(flag):
-    globals().update({
-        'sc_perf_evt_open' : 298,
-        'exit_reasons' : x86_exit_reasons[flag],
-    })
-
-def s390_init():
-    globals().update({
-        'sc_perf_evt_open' : 331
-    })
-
-def ppc_init():
-    globals().update({
-        'sc_perf_evt_open' : 319,
-        'ioctl_numbers' : {
-            'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16),
-            'ENABLE'     : 0x20002400,
-            'DISABLE'    : 0x20002401,
-        }
-    })
-
-def aarch64_init():
-    globals().update({
-        'sc_perf_evt_open' : 241,
-        'exit_reasons' : aarch64_exit_reasons,
-    })
-
-def detect_platform():
-    if os.uname()[4].startswith('ppc'):
-        ppc_init()
-        return
-    elif os.uname()[4].startswith('aarch64'):
-        aarch64_init()
-        return
-
-    for line in file('/proc/cpuinfo').readlines():
-        if line.startswith('flags'):
-            for flag in line.split():
-                if flag in x86_exit_reasons:
-                    x86_init(flag)
-                    return
-        elif line.startswith('vendor_id'):
-            for flag in line.split():
-                if flag == 'IBM/S390':
-                    s390_init()
-                    return
-
-detect_platform()
-
-def invert(d):
-    return dict((x[1], x[0]) for x in d.iteritems())
-
-filters = {}
-filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons))
-if exit_reasons:
-    filters['kvm_exit'] = ('exit_reason', invert(exit_reasons))
-
-import struct, array
-
-libc = ctypes.CDLL('libc.so.6')
+    for member in members:
+        if '-' not in member:
+            integers.append(int(member))
+        else:
+            int_range = member.split('-')
+            integers.extend(range(int(int_range[0]),
+                                  int(int_range[1]) + 1))
+
+    return integers
+
+
+def get_online_cpus():
+    with open('/sys/devices/system/cpu/online') as cpu_list:
+        cpu_string = cpu_list.readline()
+        return parse_int_list(cpu_string)
+
+
+def get_filters():
+    filters = {}
+    filters['kvm_userspace_exit'] = ('reason', USERSPACE_EXIT_REASONS)
+    if ARCH.exit_reasons:
+        filters['kvm_exit'] = ('exit_reason', ARCH.exit_reasons)
+    return filters
+
+libc = ctypes.CDLL('libc.so.6', use_errno=True)
 syscall = libc.syscall
-get_errno = libc.__errno_location
-get_errno.restype = POINTER(c_int)
 
 class perf_event_attr(ctypes.Structure):
     _fields_ = [('type', ctypes.c_uint32),
@@ -305,262 +332,350 @@ class perf_event_attr(ctypes.Structure):
                 ('bp_addr', ctypes.c_uint64),
                 ('bp_len', ctypes.c_uint64),
                 ]
-def _perf_event_open(attr, pid, cpu, group_fd, flags):
-    return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid),
-                   ctypes.c_int(cpu), ctypes.c_int(group_fd),
-                   ctypes.c_long(flags))
-
-PERF_TYPE_HARDWARE              = 0
-PERF_TYPE_SOFTWARE              = 1
-PERF_TYPE_TRACEPOINT            = 2
-PERF_TYPE_HW_CACHE              = 3
-PERF_TYPE_RAW                   = 4
-PERF_TYPE_BREAKPOINT            = 5
-
-PERF_SAMPLE_IP                  = 1 << 0
-PERF_SAMPLE_TID                 = 1 << 1
-PERF_SAMPLE_TIME                = 1 << 2
-PERF_SAMPLE_ADDR                = 1 << 3
-PERF_SAMPLE_READ                = 1 << 4
-PERF_SAMPLE_CALLCHAIN           = 1 << 5
-PERF_SAMPLE_ID                  = 1 << 6
-PERF_SAMPLE_CPU                 = 1 << 7
-PERF_SAMPLE_PERIOD              = 1 << 8
-PERF_SAMPLE_STREAM_ID           = 1 << 9
-PERF_SAMPLE_RAW                 = 1 << 10
-
-PERF_FORMAT_TOTAL_TIME_ENABLED  = 1 << 0
-PERF_FORMAT_TOTAL_TIME_RUNNING  = 1 << 1
-PERF_FORMAT_ID                  = 1 << 2
-PERF_FORMAT_GROUP               = 1 << 3
 
-import re
+    def __init__(self):
+        super(self.__class__, self).__init__()
+        self.type = PERF_TYPE_TRACEPOINT
+        self.size = ctypes.sizeof(self)
+        self.read_format = PERF_FORMAT_GROUP
+
+def perf_event_open(attr, pid, cpu, group_fd, flags):
+    return syscall(ARCH.sc_perf_evt_open, ctypes.pointer(attr),
+                   ctypes.c_int(pid), ctypes.c_int(cpu),
+                   ctypes.c_int(group_fd), ctypes.c_long(flags))
 
-sys_tracing = '/sys/kernel/debug/tracing'
+PERF_TYPE_TRACEPOINT = 2
+PERF_FORMAT_GROUP = 1 << 3
+
+PATH_DEBUGFS_TRACING = '/sys/kernel/debug/tracing'
+PATH_DEBUGFS_KVM = '/sys/kernel/debug/kvm'
 
 class Group(object):
-    def __init__(self, cpu):
+    def __init__(self):
         self.events = []
-        self.group_leader = None
-        self.cpu = cpu
-    def add_event(self, name, event_set, tracepoint, filter = None):
-        self.events.append(Event(group = self,
-                                 name = name, event_set = event_set,
-                                 tracepoint = tracepoint, filter = filter))
-        if len(self.events) == 1:
-            self.file = os.fdopen(self.events[0].fd)
+
+    def add_event(self, event):
+        self.events.append(event)
+
     def read(self):
-        bytes = 8 * (1 + len(self.events))
-        fmt = 'xxxxxxxx' + 'q' * len(self.events)
+        length = 8 * (1 + len(self.events))
+        read_format = 'xxxxxxxx' + 'Q' * len(self.events)
         return dict(zip([event.name for event in self.events],
-                        struct.unpack(fmt, self.file.read(bytes))))
+                        struct.unpack(read_format,
+                                      os.read(self.events[0].fd, length))))
 
 class Event(object):
-    def __init__(self, group, name, event_set, tracepoint, filter = None):
+    def __init__(self, name, group, trace_cpu, trace_point, trace_filter,
+                 trace_set='kvm'):
         self.name = name
-        attr = perf_event_attr()
-        attr.type = PERF_TYPE_TRACEPOINT
-        attr.size = ctypes.sizeof(attr)
-        id_path = os.path.join(sys_tracing, 'events', event_set,
-                               tracepoint, 'id')
-        id = int(file(id_path).read())
-        attr.config = id
-        attr.sample_type = (PERF_SAMPLE_RAW
-                            | PERF_SAMPLE_TIME
-                            | PERF_SAMPLE_CPU)
-        attr.sample_period = 1
-        attr.read_format = PERF_FORMAT_GROUP
+        self.fd = None
+        self.setup_event(group, trace_cpu, trace_point, trace_filter,
+                         trace_set)
+
+    def setup_event_attribute(self, trace_set, trace_point):
+        id_path = os.path.join(PATH_DEBUGFS_TRACING, 'events', trace_set,
+                               trace_point, 'id')
+
+        event_attr = perf_event_attr()
+        event_attr.config = int(open(id_path).read())
+        return event_attr
+
+    def setup_event(self, group, trace_cpu, trace_point, trace_filter,
+                    trace_set):
+        event_attr = self.setup_event_attribute(trace_set, trace_point)
+
         group_leader = -1
         if group.events:
             group_leader = group.events[0].fd
-        fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0)
+
+        fd = perf_event_open(event_attr, -1, trace_cpu,
+                             group_leader, 0)
         if fd == -1:
-            err = get_errno()[0]
-            raise Exception('perf_event_open failed, errno = ' + err.__str__())
-        if filter:
-            import fcntl
-            fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter)
+            err = ctypes.get_errno()
+            raise OSError(err, os.strerror(err),
+                          'while calling sys_perf_event_open().')
+
+        if trace_filter:
+            fcntl.ioctl(fd, ARCH.ioctl_numbers['SET_FILTER'],
+                        trace_filter)
+
         self.fd = fd
+
     def enable(self):
-        import fcntl
-        fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0)
+        fcntl.ioctl(self.fd, ARCH.ioctl_numbers['ENABLE'], 0)
+
     def disable(self):
-        import fcntl
-        fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0)
+        fcntl.ioctl(self.fd, ARCH.ioctl_numbers['DISABLE'], 0)
+
     def reset(self):
-        import fcntl
-        fcntl.ioctl(self.fd, ioctl_numbers['RESET'], 0)
+        fcntl.ioctl(self.fd, ARCH.ioctl_numbers['RESET'], 0)
 
 class TracepointProvider(object):
     def __init__(self):
-        path = os.path.join(sys_tracing, 'events', 'kvm')
-        fields = [f
-                  for f in os.listdir(path)
-                  if os.path.isdir(os.path.join(path, f))]
+        self.group_leaders = []
+        self.filters = get_filters()
+        self._fields = self.get_available_fields()
+        self.setup_traces()
+        self.fields = self._fields
+
+    def get_available_fields(self):
+        path = os.path.join(PATH_DEBUGFS_TRACING, 'events', 'kvm')
+        fields = walkdir(path)[1]
         extra = []
-        for f in fields:
-            if f in filters:
-                subfield, values = filters[f]
-                for name, number in values.iteritems():
-                    extra.append(f + '(' + name + ')')
+        for field in fields:
+            if field in self.filters:
+                filter_name_, filter_dicts = self.filters[field]
+                for name in filter_dicts:
+                    extra.append(field + '(' + name + ')')
         fields += extra
-        self._setup(fields)
-        self.select(fields)
-    def fields(self):
-        return self._fields
+        return fields
+
+    def setup_traces(self):
+        cpus = get_online_cpus()
+
+        # The constant is needed as a buffer for python libs, std
+        # streams and other files that the script opens.
+        newlim = len(cpus) * len(self._fields) + 50
+        try:
+            softlim_, hardlim = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+            if hardlim < newlim:
+                # Now we need CAP_SYS_RESOURCE, to increase the hard limit.
+                resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, newlim))
+            else:
+                # Raising the soft limit is sufficient.
+                resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, hardlim))
+
+        except ValueError:
+            sys.exit("NOFILE rlimit could not be raised to {0}".format(newlim))
 
-    def _online_cpus(self):
-        l = []
-        pattern = r'cpu([0-9]+)'
-        basedir = '/sys/devices/system/cpu'
-        for entry in os.listdir(basedir):
-            match = re.match(pattern, entry)
-            if not match:
-                continue
-            path = os.path.join(basedir, entry, 'online')
-            if os.path.exists(path) and open(path).read().strip() != '1':
-                continue
-            l.append(int(match.group(1)))
-        return l
-
-    def _setup(self, _fields):
-        self._fields = _fields
-        cpus = self._online_cpus()
-        import resource
-        nfiles = len(cpus) * 1000
-        resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles))
-        events = []
-        self.group_leaders = []
         for cpu in cpus:
-            group = Group(cpu)
-            for name in _fields:
+            group = Group()
+            for name in self._fields:
                 tracepoint = name
-                filter = None
-                m = re.match(r'(.*)\((.*)\)', name)
-                if m:
-                    tracepoint, sub = m.groups()
-                    filter = '%s==%d\0' % (filters[tracepoint][0],
-                                           filters[tracepoint][1][sub])
-                event = group.add_event(name, event_set = 'kvm',
-                                        tracepoint = tracepoint,
-                                        filter = filter)
+                tracefilter = None
+                match = re.match(r'(.*)\((.*)\)', name)
+                if match:
+                    tracepoint, sub = match.groups()
+                    tracefilter = ('%s==%d\0' %
+                                   (self.filters[tracepoint][0],
+                                    self.filters[tracepoint][1][sub]))
+
+                group.add_event(Event(name=name,
+                                      group=group,
+                                      trace_cpu=cpu,
+                                      trace_point=tracepoint,
+                                      trace_filter=tracefilter))
             self.group_leaders.append(group)
-    def select(self, fields):
+
+    def available_fields(self):
+        return self.get_available_fields()
+
+    @property
+    def fields(self):
+        return self._fields
+
+    @fields.setter
+    def fields(self, fields):
+        self._fields = fields
         for group in self.group_leaders:
-            for event in group.events:
+            for index, event in enumerate(group.events):
                 if event.name in fields:
                     event.reset()
                     event.enable()
                 else:
-                    event.disable()
+                    # Do not disable the group leader.
+                    # It would disable all of its events.
+                    if index != 0:
+                        event.disable()
+
     def read(self):
-        from collections import defaultdict
         ret = defaultdict(int)
         for group in self.group_leaders:
             for name, val in group.read().iteritems():
-                ret[name] += val
+                if name in self._fields:
+                    ret[name] += val
         return ret
 
-class Stats:
-    def __init__(self, providers, fields = None):
+class DebugfsProvider(object):
+    def __init__(self):
+        self._fields = self.get_available_fields()
+
+    def get_available_fields(self):
+        return walkdir(PATH_DEBUGFS_KVM)[2]
+
+    @property
+    def fields(self):
+        return self._fields
+
+    @fields.setter
+    def fields(self, fields):
+        self._fields = fields
+
+    def read(self):
+        def val(key):
+            return int(file(PATH_DEBUGFS_KVM + '/' + key).read())
+        return dict([(key, val(key)) for key in self._fields])
+
+class Stats(object):
+    def __init__(self, providers, fields=None):
         self.providers = providers
-        self.fields_filter = fields
-        self._update()
-    def _update(self):
+        self._fields_filter = fields
+        self.values = {}
+        self.update_provider_filters()
+
+    def update_provider_filters(self):
         def wanted(key):
-            import re
-            if not self.fields_filter:
+            if not self._fields_filter:
                 return True
-            return re.match(self.fields_filter, key) is not None
-        self.values = dict()
-        for d in providers:
-            provider_fields = [key for key in d.fields() if wanted(key)]
-            for key in provider_fields:
-                self.values[key] = None
-            d.select(provider_fields)
-    def set_fields_filter(self, fields_filter):
-        self.fields_filter = fields_filter
-        self._update()
+            return re.match(self._fields_filter, key) is not None
+
+        # As we reset the counters when updating the fields we can
+        # also clear the cache of old values.
+        self.values = {}
+        for provider in self.providers:
+            provider_fields = [key for key in provider.get_available_fields()
+                               if wanted(key)]
+            provider.fields = provider_fields
+
+    @property
+    def fields_filter(self):
+        return self._fields_filter
+
+    @fields_filter.setter
+    def fields_filter(self, fields_filter):
+        self._fields_filter = fields_filter
+        self.update_provider_filters()
+
     def get(self):
-        for d in providers:
-            new = d.read()
-            for key in d.fields():
+        for provider in self.providers:
+            new = provider.read()
+            for key in provider.fields:
                 oldval = self.values.get(key, (0, 0))
-                newval = new[key]
+                newval = new.get(key, 0)
                 newdelta = None
                 if oldval is not None:
                     newdelta = newval - oldval[0]
                 self.values[key] = (newval, newdelta)
         return self.values
 
-if not os.access('/sys/kernel/debug', os.F_OK):
-    print 'Please enable CONFIG_DEBUG_FS in your kernel'
-    sys.exit(1)
-if not os.access('/sys/kernel/debug/kvm', os.F_OK):
-    print "Please mount debugfs ('mount -t debugfs debugfs /sys/kernel/debug')"
-    print "and ensure the kvm modules are loaded"
-    sys.exit(1)
-
-label_width = 40
-number_width = 10
-
-def tui(screen, stats):
-    curses.use_default_colors()
-    curses.noecho()
-    drilldown = False
-    fields_filter = stats.fields_filter
-    def update_drilldown():
-        if not fields_filter:
-            if drilldown:
-                stats.set_fields_filter(None)
-            else:
-                stats.set_fields_filter(r'^[^\(]*$')
-    update_drilldown()
-    def refresh(sleeptime):
-        screen.erase()
-        screen.addstr(0, 0, 'kvm statistics')
-        screen.addstr(2, 1, 'Event')
-        screen.addstr(2, 1 + label_width + number_width - len('Total'), 'Total')
-        screen.addstr(2, 1 + label_width + number_width + 8 - len('Current'), 'Current')
+LABEL_WIDTH = 40
+NUMBER_WIDTH = 10
+
+class Tui(object):
+    def __init__(self, stats):
+        self.stats = stats
+        self.screen = None
+        self.drilldown = False
+        self.update_drilldown()
+
+    def __enter__(self):
+        """Initialises curses for later use.  Based on curses.wrapper
+           implementation from the Python standard library."""
+        self.screen = curses.initscr()
+        curses.noecho()
+        curses.cbreak()
+
+        # The try/catch works around a minor bit of
+        # over-conscientiousness in the curses module, the error
+        # return from C start_color() is ignorable.
+        try:
+            curses.start_color()
+        except:
+            pass
+
+        curses.use_default_colors()
+        return self
+
+    def __exit__(self, *exception):
+        """Resets the terminal to its normal state.  Based on curses.wrappre
+           implementation from the Python standard library."""
+        if self.screen:
+            self.screen.keypad(0)
+            curses.echo()
+            curses.nocbreak()
+            curses.endwin()
+
+    def update_drilldown(self):
+        if not self.stats.fields_filter:
+            self.stats.fields_filter = r'^[^\(]*$'
+
+        elif self.stats.fields_filter == r'^[^\(]*$':
+            self.stats.fields_filter = None
+
+    def refresh(self, sleeptime):
+        self.screen.erase()
+        self.screen.addstr(0, 0, 'kvm statistics - summary', curses.A_BOLD)
+        self.screen.addstr(2, 1, 'Event')
+        self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH -
+                           len('Total'), 'Total')
+        self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH + 8 -
+                           len('Current'), 'Current')
         row = 3
-        s = stats.get()
+        stats = self.stats.get()
         def sortkey(x):
-            if s[x][1]:
-                return (-s[x][1], -s[x][0])
+            if stats[x][1]:
+                return (-stats[x][1], -stats[x][0])
             else:
-                return (0, -s[x][0])
-        for key in sorted(s.keys(), key = sortkey):
-            if row >= screen.getmaxyx()[0]:
+                return (0, -stats[x][0])
+        for key in sorted(stats.keys(), key=sortkey):
+
+            if row >= self.screen.getmaxyx()[0]:
                 break
-            values = s[key]
+            values = stats[key]
             if not values[0] and not values[1]:
                 break
             col = 1
-            screen.addstr(row, col, key)
-            col += label_width
-            screen.addstr(row, col, '%10d' % (values[0],))
-            col += number_width
+            self.screen.addstr(row, col, key)
+            col += LABEL_WIDTH
+            self.screen.addstr(row, col, '%10d' % (values[0],))
+            col += NUMBER_WIDTH
             if values[1] is not None:
-                screen.addstr(row, col, '%8d' % (values[1] / sleeptime,))
+                self.screen.addstr(row, col, '%8d' % (values[1] / sleeptime,))
             row += 1
-        screen.refresh()
+        self.screen.refresh()
+
+    def show_filter_selection(self):
+        while True:
+            self.screen.erase()
+            self.screen.addstr(0, 0,
+                               "Show statistics for events matching a regex.",
+                               curses.A_BOLD)
+            self.screen.addstr(2, 0,
+                               "Current regex: {0}"
+                               .format(self.stats.fields_filter))
+            self.screen.addstr(3, 0, "New regex: ")
+            curses.echo()
+            regex = self.screen.getstr()
+            curses.noecho()
+            if len(regex) == 0:
+                return
+            try:
+                re.compile(regex)
+                self.stats.fields_filter = regex
+                return
+            except re.error:
+                continue
 
-    sleeptime = 0.25
-    while True:
-        refresh(sleeptime)
-        curses.halfdelay(int(sleeptime * 10))
-        sleeptime = 3
-        try:
-            c = screen.getkey()
-            if c == 'x':
-                drilldown = not drilldown
-                update_drilldown()
-            if c == 'q':
+    def show_stats(self):
+        sleeptime = 0.25
+        while True:
+            self.refresh(sleeptime)
+            curses.halfdelay(int(sleeptime * 10))
+            sleeptime = 3
+            try:
+                char = self.screen.getkey()
+                if char == 'x':
+                    self.drilldown = not self.drilldown
+                    self.update_drilldown()
+                if char == 'q':
+                    break
+                if char == 'f':
+                    self.show_filter_selection()
+            except KeyboardInterrupt:
                 break
-        except KeyboardInterrupt:
-            break
-        except curses.error:
-            continue
+            except curses.error:
+                continue
 
 def batch(stats):
     s = stats.get()
@@ -568,13 +683,13 @@ def batch(stats):
     s = stats.get()
     for key in sorted(s.keys()):
         values = s[key]
-        print '%-22s%10d%10d' % (key, values[0], values[1])
+        print '%-42s%10d%10d' % (key, values[0], values[1])
 
 def log(stats):
     keys = sorted(stats.get().iterkeys())
     def banner():
         for k in keys:
-            print '%10s' % k[0:9],
+            print '%s' % k,
         print
     def statline():
         s = stats.get()
@@ -590,57 +705,121 @@ def log(stats):
         statline()
         line += 1
 
-options = optparse.OptionParser()
-options.add_option('-1', '--once', '--batch',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'once',
-                   help = 'run in batch mode for one second',
-                   )
-options.add_option('-l', '--log',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'log',
-                   help = 'run in logging mode (like vmstat)',
-                   )
-options.add_option('-t', '--tracepoints',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'tracepoints',
-                   help = 'retrieve statistics from tracepoints',
-                   )
-options.add_option('-d', '--debugfs',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'debugfs',
-                   help = 'retrieve statistics from debugfs',
-                   )
-options.add_option('-f', '--fields',
-                   action = 'store',
-                   default = None,
-                   dest = 'fields',
-                   help = 'fields to display (regex)',
-                   )
-(options, args) = options.parse_args(sys.argv)
-
-providers = []
-if options.tracepoints:
-    providers.append(TracepointProvider())
-if options.debugfs:
-    providers.append(DebugfsProvider())
-
-if len(providers) == 0:
-    try:
-        providers = [TracepointProvider()]
-    except:
-        providers = [DebugfsProvider()]
-
-stats = Stats(providers, fields = options.fields)
-
-if options.log:
-    log(stats)
-elif not options.once:
-    import curses.wrapper
-    curses.wrapper(tui, stats)
-else:
-    batch(stats)
+def get_options():
+    description_text = """
+This script displays various statistics about VMs running under KVM.
+The statistics are gathered from the KVM debugfs entries and / or the
+currently available perf traces.
+
+The monitoring takes additional cpu cycles and might affect the VM's
+performance.
+
+Requirements:
+- Access to:
+    /sys/kernel/debug/kvm
+    /sys/kernel/debug/trace/events/*
+    /proc/pid/task
+- /proc/sys/kernel/perf_event_paranoid < 1 if user has no
+  CAP_SYS_ADMIN and perf events are used.
+- CAP_SYS_RESOURCE if the hard limit is not high enough to allow
+  the large number of files that are possibly opened.
+"""
+
+    class PlainHelpFormatter(optparse.IndentedHelpFormatter):
+        def format_description(self, description):
+            if description:
+                return description + "\n"
+            else:
+                return ""
+
+    optparser = optparse.OptionParser(description=description_text,
+                                      formatter=PlainHelpFormatter())
+    optparser.add_option('-1', '--once', '--batch',
+                         action='store_true',
+                         default=False,
+                         dest='once',
+                         help='run in batch mode for one second',
+                         )
+    optparser.add_option('-l', '--log',
+                         action='store_true',
+                         default=False,
+                         dest='log',
+                         help='run in logging mode (like vmstat)',
+                         )
+    optparser.add_option('-t', '--tracepoints',
+                         action='store_true',
+                         default=False,
+                         dest='tracepoints',
+                         help='retrieve statistics from tracepoints',
+                         )
+    optparser.add_option('-d', '--debugfs',
+                         action='store_true',
+                         default=False,
+                         dest='debugfs',
+                         help='retrieve statistics from debugfs',
+                         )
+    optparser.add_option('-f', '--fields',
+                         action='store',
+                         default=None,
+                         dest='fields',
+                         help='fields to display (regex)',
+                         )
+    (options, _) = optparser.parse_args(sys.argv)
+    return options
+
+def get_providers(options):
+    providers = []
+
+    if options.tracepoints:
+        providers.append(TracepointProvider())
+    if options.debugfs:
+        providers.append(DebugfsProvider())
+    if len(providers) == 0:
+        providers.append(TracepointProvider())
+
+    return providers
+
+def check_access(options):
+    if not os.path.exists('/sys/kernel/debug'):
+        sys.stderr.write('Please enable CONFIG_DEBUG_FS in your kernel.')
+        sys.exit(1)
+
+    if not os.path.exists(PATH_DEBUGFS_KVM):
+        sys.stderr.write("Please make sure, that debugfs is mounted and "
+                         "readable by the current user:\n"
+                         "('mount -t debugfs debugfs /sys/kernel/debug')\n"
+                         "Also ensure, that the kvm modules are loaded.\n")
+        sys.exit(1)
+
+    if not os.path.exists(PATH_DEBUGFS_TRACING) and (options.tracepoints
+                                                     or not options.debugfs):
+        sys.stderr.write("Please enable CONFIG_TRACING in your kernel "
+                         "when using the option -t (default).\n"
+                         "If it is enabled, make {0} readable by the "
+                         "current user.\n"
+                         .format(PATH_DEBUGFS_TRACING))
+        if options.tracepoints:
+            sys.exit(1)
+
+        sys.stderr.write("Falling back to debugfs statistics!\n")
+        options.debugfs = True
+        sleep(5)
+
+    return options
+
+def main():
+    options = get_options()
+    options = check_access(options)
+    providers = get_providers(options)
+    stats = Stats(providers, fields=options.fields)
+
+    if options.log:
+        log(stats)
+    elif not options.once:
+        with Tui(stats) as tui:
+            tui.show_stats()
+    else:
+        batch(stats)
+
+if __name__ == "__main__":
+    main()
diff --git a/qemu/scripts/ordereddict.py b/qemu/scripts/ordereddict.py
index 7242b5060..2d1d81370 100644
--- a/qemu/scripts/ordereddict.py
+++ b/qemu/scripts/ordereddict.py
@@ -22,6 +22,7 @@
 
 from UserDict import DictMixin
 
+
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
@@ -117,7 +118,7 @@ class OrderedDict(dict, DictMixin):
         if isinstance(other, OrderedDict):
             if len(self) != len(other):
                 return False
-            for p, q in  zip(self.items(), other.items()):
+            for p, q in zip(self.items(), other.items()):
                 if p != q:
                     return False
             return True
diff --git a/qemu/scripts/qapi-commands.py b/qemu/scripts/qapi-commands.py
index ca22acc1d..b570069fa 100644
--- a/qemu/scripts/qapi-commands.py
+++ b/qemu/scripts/qapi-commands.py
@@ -2,7 +2,7 @@
 # QAPI command marshaller generator
 #
 # Copyright IBM, Corp. 2011
-# Copyright (C) 2014-2015 Red Hat, Inc.
+# Copyright (C) 2014-2016 Red Hat, Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -12,274 +12,217 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
 
-def generate_command_decl(name, args, ret_type):
-    arglist=""
-    for argname, argtype, optional in parse_args(args):
-        argtype = c_type(argtype, is_param=True)
-        if optional:
-            arglist += "bool has_%s, " % c_name(argname)
-        arglist += "%s %s, " % (argtype, c_name(argname))
+
+def gen_command_decl(name, arg_type, ret_type):
     return mcgen('''
-%(ret_type)s qmp_%(name)s(%(args)sError **errp);
+%(c_type)s qmp_%(c_name)s(%(params)s);
 ''',
-                 ret_type=c_type(ret_type), name=c_name(name),
-                 args=arglist).strip()
-
-def gen_err_check(errvar):
-    if errvar:
-        return mcgen('''
-if (local_err) {
-    goto out;
-}
-''')
-    return ''
+                 c_type=(ret_type and ret_type.c_type()) or 'void',
+                 c_name=c_name(name),
+                 params=gen_params(arg_type, 'Error **errp'))
 
-def gen_sync_call(name, args, ret_type, indent=0):
-    ret = ""
-    arglist=""
-    retval=""
-    if ret_type:
-        retval = "retval = "
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            arglist += "has_%s, " % c_name(argname)
-        arglist += "%s, " % (c_name(argname))
-    push_indent(indent)
-    ret = mcgen('''
-%(retval)sqmp_%(name)s(%(args)s&local_err);
 
-''',
-                name=c_name(name), args=arglist, retval=retval).rstrip()
-    if ret_type:
-        ret += "\n" + gen_err_check('local_err')
-        ret += "\n" + mcgen(''''
-%(marshal_output_call)s
-''',
-                            marshal_output_call=gen_marshal_output_call(name, ret_type)).rstrip()
-    pop_indent(indent)
-    return ret.rstrip()
+def gen_call(name, arg_type, ret_type):
+    ret = ''
 
+    argstr = ''
+    if arg_type:
+        assert not arg_type.variants
+        for memb in arg_type.members:
+            if memb.optional:
+                argstr += 'arg.has_%s, ' % c_name(memb.name)
+            argstr += 'arg.%s, ' % c_name(memb.name)
 
-def gen_marshal_output_call(name, ret_type):
-    if not ret_type:
-        return ""
-    return "qmp_marshal_output_%s(retval, ret, &local_err);" % c_name(name)
+    lhs = ''
+    if ret_type:
+        lhs = 'retval = '
 
-def gen_visitor_input_containers_decl(args, obj):
-    ret = ""
+    ret = mcgen('''
 
-    push_indent()
-    if len(args) > 0:
-        ret += mcgen('''
-QmpInputVisitor *mi = qmp_input_visitor_new_strict(%(obj)s);
-QapiDeallocVisitor *md;
-Visitor *v;
-''',
-                     obj=obj)
-    pop_indent()
-
-    return ret.rstrip()
-
-def gen_visitor_input_vars_decl(args):
-    ret = ""
-    push_indent()
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            ret += mcgen('''
-bool has_%(argname)s = false;
-''',
-                         argname=c_name(argname))
-        if is_c_ptr(argtype):
-            ret += mcgen('''
-%(argtype)s %(argname)s = NULL;
+    %(lhs)sqmp_%(c_name)s(%(args)s&err);
 ''',
-                         argname=c_name(argname), argtype=c_type(argtype))
-        else:
-            ret += mcgen('''
-%(argtype)s %(argname)s = {0};
-''',
-                         argname=c_name(argname), argtype=c_type(argtype))
-
-    pop_indent()
-    return ret.rstrip()
-
-def gen_visitor_input_block(args, dealloc=False):
-    ret = ""
-    errparg = '&local_err'
-    errarg = 'local_err'
-
-    if len(args) == 0:
-        return ret
-
-    push_indent()
-
-    if dealloc:
-        errparg = 'NULL'
-        errarg = None;
-        ret += mcgen('''
-qmp_input_visitor_cleanup(mi);
-md = qapi_dealloc_visitor_new();
-v = qapi_dealloc_get_visitor(md);
-''')
-    else:
+                c_name=c_name(name), args=argstr, lhs=lhs)
+    if ret_type:
+        ret += gen_err_check()
         ret += mcgen('''
-v = qmp_input_get_visitor(mi);
-''')
 
-    for argname, argtype, optional in parse_args(args):
-        if optional:
-            ret += mcgen('''
-visit_optional(v, &has_%(c_name)s, "%(name)s", %(errp)s);
+    qmp_marshal_output_%(c_name)s(retval, ret, &err);
 ''',
-                         c_name=c_name(argname), name=argname, errp=errparg)
-            ret += gen_err_check(errarg)
-            ret += mcgen('''
-if (has_%(c_name)s) {
-''',
-                         c_name=c_name(argname))
-            push_indent()
-        ret += mcgen('''
-visit_type_%(visitor)s(v, &%(c_name)s, "%(name)s", %(errp)s);
-''',
-                     c_name=c_name(argname), name=argname, argtype=argtype,
-                     visitor=type_name(argtype), errp=errparg)
-        ret += gen_err_check(errarg)
-        if optional:
-            pop_indent()
-            ret += mcgen('''
-}
-''')
+                     c_name=ret_type.c_name())
+    return ret
 
-    if dealloc:
-        ret += mcgen('''
-qapi_dealloc_visitor_cleanup(md);
-''')
-    pop_indent()
-    return ret.rstrip()
 
-def gen_marshal_output(name, args, ret_type, middle_mode):
-    if not ret_type:
-        return ""
+def gen_marshal_output(ret_type):
+    return mcgen('''
 
-    ret = mcgen('''
-static void qmp_marshal_output_%(c_name)s(%(c_ret_type)s ret_in, QObject **ret_out, Error **errp)
+static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp)
 {
-    Error *local_err = NULL;
-    QmpOutputVisitor *mo = qmp_output_visitor_new();
-    QapiDeallocVisitor *md;
+    Error *err = NULL;
+    QmpOutputVisitor *qov = qmp_output_visitor_new();
+    QapiDeallocVisitor *qdv;
     Visitor *v;
 
-    v = qmp_output_get_visitor(mo);
-    visit_type_%(visitor)s(v, &ret_in, "unused", &local_err);
-    if (local_err) {
+    v = qmp_output_get_visitor(qov);
+    visit_type_%(c_name)s(v, "unused", &ret_in, &err);
+    if (err) {
         goto out;
     }
-    *ret_out = qmp_output_get_qobject(mo);
+    *ret_out = qmp_output_get_qobject(qov);
 
 out:
-    error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    visit_type_%(visitor)s(v, &ret_in, "unused", NULL);
-    qapi_dealloc_visitor_cleanup(md);
+    error_propagate(errp, err);
+    qmp_output_visitor_cleanup(qov);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
+    visit_type_%(c_name)s(v, "unused", &ret_in, NULL);
+    qapi_dealloc_visitor_cleanup(qdv);
 }
 ''',
-                c_ret_type=c_type(ret_type), c_name=c_name(name),
-                visitor=type_name(ret_type))
+                 c_type=ret_type.c_type(), c_name=ret_type.c_name())
 
-    return ret
 
-def gen_marshal_input_decl(name, args, ret_type, middle_mode):
-    ret = 'void qmp_marshal_input_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name)
+def gen_marshal_proto(name):
+    ret = 'void qmp_marshal_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name)
     if not middle_mode:
-        ret = "static " + ret
+        ret = 'static ' + ret
     return ret
 
-def gen_marshal_input(name, args, ret_type, middle_mode):
-    hdr = gen_marshal_input_decl(name, args, ret_type, middle_mode)
 
+def gen_marshal_decl(name):
+    return mcgen('''
+%(proto)s;
+''',
+                 proto=gen_marshal_proto(name))
+
+
+def gen_marshal(name, arg_type, ret_type):
     ret = mcgen('''
-%(header)s
+
+%(proto)s
 {
-    Error *local_err = NULL;
+    Error *err = NULL;
 ''',
-                header=hdr)
+                proto=gen_marshal_proto(name))
 
     if ret_type:
-        if is_c_ptr(ret_type):
-            retval = "    %s retval = NULL;" % c_type(ret_type)
-        else:
-            retval = "    %s retval;" % c_type(ret_type)
         ret += mcgen('''
-%(retval)s
+    %(c_type)s retval;
 ''',
-                     retval=retval)
+                     c_type=ret_type.c_type())
 
-    if len(args) > 0:
+    if arg_type and arg_type.members:
         ret += mcgen('''
-%(visitor_input_containers_decl)s
-%(visitor_input_vars_decl)s
-
-%(visitor_input_block)s
+    QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args));
+    QapiDeallocVisitor *qdv;
+    Visitor *v;
+    %(c_name)s arg = {0};
 
+    v = qmp_input_get_visitor(qiv);
+    visit_type_%(c_name)s_members(v, &arg, &err);
+    if (err) {
+        goto out;
+    }
 ''',
-                     visitor_input_containers_decl=gen_visitor_input_containers_decl(args, "QOBJECT(args)"),
-                     visitor_input_vars_decl=gen_visitor_input_vars_decl(args),
-                     visitor_input_block=gen_visitor_input_block(args))
+                     c_name=arg_type.c_name())
+
     else:
         ret += mcgen('''
 
     (void)args;
 ''')
 
-    ret += mcgen('''
-%(sync_call)s
-''',
-                 sync_call=gen_sync_call(name, args, ret_type, indent=4))
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
+    ret += gen_call(name, arg_type, ret_type)
+
+    # 'goto out' produced above for arg_type, and by gen_call() for ret_type
+    if (arg_type and arg_type.members) or ret_type:
         ret += mcgen('''
 
 out:
 ''')
     ret += mcgen('''
-    error_propagate(errp, local_err);
-%(visitor_input_block_cleanup)s
-}
+    error_propagate(errp, err);
+''')
+    if arg_type and arg_type.members:
+        ret += mcgen('''
+    qmp_input_visitor_cleanup(qiv);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
+    visit_type_%(c_name)s_members(v, &arg, NULL);
+    qapi_dealloc_visitor_cleanup(qdv);
 ''',
-                 visitor_input_block_cleanup=gen_visitor_input_block(args,
-                                                                     dealloc=True))
+                     c_name=arg_type.c_name())
+
+    ret += mcgen('''
+}
+''')
     return ret
 
-def gen_registry(commands):
-    registry=""
-    push_indent()
-    for cmd in commands:
-        options = 'QCO_NO_OPTIONS'
-        if not cmd.get('success-response', True):
-            options = 'QCO_NO_SUCCESS_RESP'
 
-        registry += mcgen('''
-qmp_register_command("%(name)s", qmp_marshal_input_%(c_name)s, %(opts)s);
+def gen_register_command(name, success_response):
+    options = 'QCO_NO_OPTIONS'
+    if not success_response:
+        options = 'QCO_NO_SUCCESS_RESP'
+
+    ret = mcgen('''
+    qmp_register_command("%(name)s", qmp_marshal_%(c_name)s, %(opts)s);
 ''',
-                     name=cmd['command'], c_name=c_name(cmd['command']),
-                     opts=options)
-    pop_indent()
+                name=name, c_name=c_name(name),
+                opts=options)
+    return ret
+
+
+def gen_registry(registry):
     ret = mcgen('''
+
 static void qmp_init_marshal(void)
 {
-%(registry)s
+''')
+    ret += registry
+    ret += mcgen('''
 }
 
 qapi_init(qmp_init_marshal);
-''',
-                registry=registry.rstrip())
+''')
     return ret
 
+
+class QAPISchemaGenCommandVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._regy = None
+        self._visited_ret_types = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._regy = ''
+        self._visited_ret_types = set()
+
+    def visit_end(self):
+        if not middle_mode:
+            self.defn += gen_registry(self._regy)
+        self._regy = None
+        self._visited_ret_types = None
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        if not gen:
+            return
+        self.decl += gen_command_decl(name, arg_type, ret_type)
+        if ret_type and ret_type not in self._visited_ret_types:
+            self._visited_ret_types.add(ret_type)
+            self.defn += gen_marshal_output(ret_type)
+        if middle_mode:
+            self.decl += gen_marshal_decl(name)
+        self.defn += gen_marshal(name, arg_type, ret_type)
+        if not middle_mode:
+            self._regy += gen_register_command(name, success_response)
+
+
 middle_mode = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -289,10 +232,6 @@ for o, a in opts:
     if o in ("-m", "--middle"):
         middle_mode = True
 
-exprs = parse_schema(input_file)
-commands = filter(lambda expr: expr.has_key('command'), exprs)
-commands = filter(lambda expr: not expr.has_key('gen'), commands)
-
 c_comment = '''
 /*
  * schema-defined QMP->QAPI command dispatch
@@ -327,6 +266,7 @@ h_comment = '''
                             c_comment, h_comment)
 
 fdef.write(mcgen('''
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/module.h"
 #include "qapi/qmp/types.h"
@@ -340,7 +280,7 @@ fdef.write(mcgen('''
 #include "%(prefix)sqmp-commands.h"
 
 ''',
-                prefix=prefix))
+                 prefix=prefix))
 
 fdecl.write(mcgen('''
 #include "%(prefix)sqapi-types.h"
@@ -348,29 +288,12 @@ fdecl.write(mcgen('''
 #include "qapi/error.h"
 
 ''',
-                 prefix=prefix))
-
-for cmd in commands:
-    arglist = []
-    ret_type = None
-    if cmd.has_key('data'):
-        arglist = cmd['data']
-    if cmd.has_key('returns'):
-        ret_type = cmd['returns']
-    ret = generate_command_decl(cmd['command'], arglist, ret_type) + "\n"
-    fdecl.write(ret)
-    if ret_type:
-        ret = gen_marshal_output(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-        fdef.write(ret)
-
-    if middle_mode:
-        fdecl.write('%s;\n' % gen_marshal_input_decl(cmd['command'], arglist, ret_type, middle_mode))
-
-    ret = gen_marshal_input(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-    fdef.write(ret)
+                  prefix=prefix))
 
-if not middle_mode:
-    ret = gen_registry(commands)
-    fdef.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenCommandVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/qemu/scripts/qapi-event.py b/qemu/scripts/qapi-event.py
index 56bc602a6..9b5c5b535 100644
--- a/qemu/scripts/qapi-event.py
+++ b/qemu/scripts/qapi-event.py
@@ -2,215 +2,151 @@
 # QAPI event generator
 #
 # Copyright (c) 2014 Wenchao Xia
+# Copyright (c) 2015-2016 Red Hat Inc.
 #
 # Authors:
 #  Wenchao Xia <wenchaoqemu@gmail.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 
-def _generate_event_api_name(event_name, params):
-    api_name = "void qapi_event_send_%s(" % c_name(event_name).lower();
-    l = len(api_name)
 
-    if params:
-        for argname, argentry, optional in parse_args(params):
-            if optional:
-                api_name += "bool has_%s,\n" % c_name(argname)
-                api_name += "".ljust(l)
+def gen_event_send_proto(name, arg_type):
+    return 'void qapi_event_send_%(c_name)s(%(param)s)' % {
+        'c_name': c_name(name.lower()),
+        'param': gen_params(arg_type, 'Error **errp')}
 
-            api_name += "%s %s,\n" % (c_type(argentry, is_param=True),
-                                      c_name(argname))
-            api_name += "".ljust(l)
-
-    api_name += "Error **errp)"
-    return api_name;
 
+def gen_event_send_decl(name, arg_type):
+    return mcgen('''
 
-# Following are the core functions that generate C APIs to emit event.
+%(proto)s;
+''',
+                 proto=gen_event_send_proto(name, arg_type))
 
-def generate_event_declaration(api_name):
-    return mcgen('''
 
-%(api_name)s;
+# Declare and initialize an object 'qapi' using parameters from gen_params()
+def gen_param_var(typ):
+    assert not typ.variants
+    ret = mcgen('''
+    %(c_name)s param = {
 ''',
-                 api_name = api_name)
+                c_name=typ.c_name())
+    sep = '        '
+    for memb in typ.members:
+        ret += sep
+        sep = ', '
+        if memb.optional:
+            ret += 'has_' + c_name(memb.name) + sep
+        if memb.type.name == 'str':
+            # Cast away const added in gen_params()
+            ret += '(char *)'
+        ret += c_name(memb.name)
+    ret += mcgen('''
+
+    };
+''')
+    return ret
 
-def generate_event_implement(api_name, event_name, params):
-    # step 1: declare any variables
-    ret = mcgen("""
 
-%(api_name)s
+def gen_event_send(name, arg_type):
+    # FIXME: Our declaration of local variables (and of 'errp' in the
+    # parameter list) can collide with exploded members of the event's
+    # data type passed in as parameters.  If this collision ever hits in
+    # practice, we can rename our local variables with a leading _ prefix,
+    # or split the code into a wrapper function that creates a boxed
+    # 'param' object then calls another to do the real work.
+    ret = mcgen('''
+
+%(proto)s
 {
     QDict *qmp;
-    Error *local_err = NULL;
+    Error *err = NULL;
     QMPEventFuncEmit emit;
-""",
-                api_name = api_name)
+''',
+                proto=gen_event_send_proto(name, arg_type))
 
-    if params:
-        ret += mcgen("""
+    if arg_type and arg_type.members:
+        ret += mcgen('''
     QmpOutputVisitor *qov;
     Visitor *v;
-    QObject *obj;
+''')
+        ret += gen_param_var(arg_type)
 
-""")
+    ret += mcgen('''
 
-    # step 2: check emit function, create a dict
-    ret += mcgen("""
     emit = qmp_event_get_func_emit();
     if (!emit) {
         return;
     }
 
-    qmp = qmp_event_build_dict("%(event_name)s");
+    qmp = qmp_event_build_dict("%(name)s");
 
-""",
-                 event_name = event_name)
+''',
+                 name=name)
 
-    # step 3: visit the params if params != None
-    if params:
-        ret += mcgen("""
+    if arg_type and arg_type.members:
+        ret += mcgen('''
     qov = qmp_output_visitor_new();
-    g_assert(qov);
-
     v = qmp_output_get_visitor(qov);
-    g_assert(v);
 
-    /* Fake visit, as if all members are under a structure */
-    visit_start_struct(v, NULL, "", "%(event_name)s", 0, &local_err);
-    if (local_err) {
-        goto clean;
-    }
-
-""",
-                event_name = event_name)
-
-        for argname, argentry, optional in parse_args(params):
-            if optional:
-                ret += mcgen("""
-    if (has_%(var)s) {
-""",
-                             var = c_name(argname))
-                push_indent()
-
-            if argentry == "str":
-                var_type = "(char **)"
-            else:
-                var_type = ""
-
-            ret += mcgen("""
-    visit_type_%(type)s(v, %(var_type)s&%(var)s, "%(name)s", &local_err);
-    if (local_err) {
-        goto clean;
-    }
-""",
-                         var_type = var_type,
-                         var = c_name(argname),
-                         type = type_name(argentry),
-                         name = argname)
-
-            if optional:
-                pop_indent()
-                ret += mcgen("""
+    visit_start_struct(v, "%(name)s", NULL, 0, &err);
+    if (err) {
+        goto out;
     }
-""")
-
-        ret += mcgen("""
-
-    visit_end_struct(v, &local_err);
-    if (local_err) {
-        goto clean;
+    visit_type_%(c_name)s_members(v, &param, &err);
+    visit_end_struct(v, err ? NULL : &err);
+    if (err) {
+        goto out;
     }
 
-    obj = qmp_output_get_qobject(qov);
-    g_assert(obj != NULL);
-
-    qdict_put_obj(qmp, "data", obj);
-""")
+    qdict_put_obj(qmp, "data", qmp_output_get_qobject(qov));
+''',
+                     name=name, c_name=arg_type.c_name())
 
-    # step 4: call qmp event api
-    ret += mcgen("""
-    emit(%(event_enum_value)s, qmp, &local_err);
+    ret += mcgen('''
+    emit(%(c_enum)s, qmp, &err);
 
-""",
-                 event_enum_value = event_enum_value)
+''',
+                 c_enum=c_enum_const(event_enum_name, name))
 
-    # step 5: clean up
-    if params:
-        ret += mcgen("""
- clean:
+    if arg_type and arg_type.members:
+        ret += mcgen('''
+out:
     qmp_output_visitor_cleanup(qov);
-""")
-    ret += mcgen("""
-    error_propagate(errp, local_err);
+''')
+    ret += mcgen('''
+    error_propagate(errp, err);
     QDECREF(qmp);
 }
-""")
-
+''')
     return ret
 
 
-# Following are the functions that generate an enum type for all defined
-# events, similar to qapi-types.py. Here we already have enum name and
-# values which were generated before and recorded in event_enum_*. It also
-# works around the issue that "import qapi-types" can't work.
-
-def generate_event_enum_decl(event_enum_name, event_enum_values):
-    lookup_decl = mcgen('''
-
-extern const char *%(event_enum_name)s_lookup[];
-''',
-                        event_enum_name = event_enum_name)
-
-    enum_decl = mcgen('''
-typedef enum %(event_enum_name)s
-{
-''',
-                      event_enum_name = event_enum_name)
-
-    # append automatically generated _MAX value
-    enum_max_value = c_enum_const(event_enum_name, "MAX")
-    enum_values = event_enum_values + [ enum_max_value ]
-
-    i = 0
-    for value in enum_values:
-        enum_decl += mcgen('''
-    %(value)s = %(i)d,
-''',
-                     value = value,
-                     i = i)
-        i += 1
-
-    enum_decl += mcgen('''
-} %(event_enum_name)s;
-''',
-                       event_enum_name = event_enum_name)
-
-    return lookup_decl + enum_decl
+class QAPISchemaGenEventVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._event_names = None
 
-def generate_event_enum_lookup(event_enum_name, event_enum_strings):
-    ret = mcgen('''
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._event_names = []
 
-const char *%(event_enum_name)s_lookup[] = {
-''',
-                event_enum_name = event_enum_name)
+    def visit_end(self):
+        self.decl += gen_enum(event_enum_name, self._event_names)
+        self.defn += gen_enum_lookup(event_enum_name, self._event_names)
+        self._event_names = None
 
-    i = 0
-    for string in event_enum_strings:
-        ret += mcgen('''
-    "%(string)s",
-''',
-                     string = string)
+    def visit_event(self, name, info, arg_type):
+        self.decl += gen_event_send_decl(name, arg_type)
+        self.defn += gen_event_send(name, arg_type)
+        self._event_names.append(name)
 
-    ret += mcgen('''
-    NULL,
-};
-''')
-    return ret
 
 (input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line()
 
@@ -248,6 +184,7 @@ h_comment = '''
                             c_comment, h_comment)
 
 fdef.write(mcgen('''
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "%(prefix)sqapi-event.h"
 #include "%(prefix)sqapi-visit.h"
@@ -265,35 +202,12 @@ fdecl.write(mcgen('''
 ''',
                   prefix=prefix))
 
-exprs = parse_schema(input_file)
-
-event_enum_name = prefix.upper().replace('-', '_') + "QAPIEvent"
-event_enum_values = []
-event_enum_strings = []
-
-for expr in exprs:
-    if expr.has_key('event'):
-        event_name = expr['event']
-        params = expr.get('data')
-        if params and len(params) == 0:
-            params = None
-
-        api_name = _generate_event_api_name(event_name, params)
-        ret = generate_event_declaration(api_name)
-        fdecl.write(ret)
-
-        # We need an enum value per event
-        event_enum_value = c_enum_const(event_enum_name, event_name)
-        ret = generate_event_implement(api_name, event_name, params)
-        fdef.write(ret)
-
-        # Record it, and generate enum later
-        event_enum_values.append(event_enum_value)
-        event_enum_strings.append(event_name)
+event_enum_name = c_name(prefix + "QAPIEvent", protect=False)
 
-ret = generate_event_enum_decl(event_enum_name, event_enum_values)
-fdecl.write(ret)
-ret = generate_event_enum_lookup(event_enum_name, event_enum_strings)
-fdef.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenEventVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/qemu/scripts/qapi-introspect.py b/qemu/scripts/qapi-introspect.py
new file mode 100644
index 000000000..e0f926be0
--- /dev/null
+++ b/qemu/scripts/qapi-introspect.py
@@ -0,0 +1,219 @@
+#
+# QAPI introspection generator
+#
+# Copyright (C) 2015-2016 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+from qapi import *
+
+
+# Caveman's json.dumps() replacement (we're stuck at Python 2.4)
+# TODO try to use json.dumps() once we get unstuck
+def to_json(obj, level=0):
+    if obj is None:
+        ret = 'null'
+    elif isinstance(obj, str):
+        ret = '"' + obj.replace('"', r'\"') + '"'
+    elif isinstance(obj, list):
+        elts = [to_json(elt, level + 1)
+                for elt in obj]
+        ret = '[' + ', '.join(elts) + ']'
+    elif isinstance(obj, dict):
+        elts = ['"%s": %s' % (key.replace('"', r'\"'),
+                              to_json(obj[key], level + 1))
+                for key in sorted(obj.keys())]
+        ret = '{' + ', '.join(elts) + '}'
+    else:
+        assert False                # not implemented
+    if level == 1:
+        ret = '\n' + ret
+    return ret
+
+
+def to_c_string(string):
+    return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
+
+
+class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor):
+    def __init__(self, unmask):
+        self._unmask = unmask
+        self.defn = None
+        self.decl = None
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_begin(self, schema):
+        self._schema = schema
+        self._jsons = []
+        self._used_types = []
+        self._name_map = {}
+
+    def visit_end(self):
+        # visit the types that are actually used
+        jsons = self._jsons
+        self._jsons = []
+        for typ in self._used_types:
+            typ.visit(self)
+        # generate C
+        # TODO can generate awfully long lines
+        jsons.extend(self._jsons)
+        name = prefix + 'qmp_schema_json'
+        self.decl = mcgen('''
+extern const char %(c_name)s[];
+''',
+                          c_name=c_name(name))
+        lines = to_json(jsons).split('\n')
+        c_string = '\n    '.join([to_c_string(line) for line in lines])
+        self.defn = mcgen('''
+const char %(c_name)s[] = %(c_string)s;
+''',
+                          c_name=c_name(name),
+                          c_string=c_string)
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_needed(self, entity):
+        # Ignore types on first pass; visit_end() will pick up used types
+        return not isinstance(entity, QAPISchemaType)
+
+    def _name(self, name):
+        if self._unmask:
+            return name
+        if name not in self._name_map:
+            self._name_map[name] = '%d' % len(self._name_map)
+        return self._name_map[name]
+
+    def _use_type(self, typ):
+        # Map the various integer types to plain int
+        if typ.json_type() == 'int':
+            typ = self._schema.lookup_type('int')
+        elif (isinstance(typ, QAPISchemaArrayType) and
+              typ.element_type.json_type() == 'int'):
+            typ = self._schema.lookup_type('intList')
+        # Add type to work queue if new
+        if typ not in self._used_types:
+            self._used_types.append(typ)
+        # Clients should examine commands and events, not types.  Hide
+        # type names to reduce the temptation.  Also saves a few
+        # characters.
+        if isinstance(typ, QAPISchemaBuiltinType):
+            return typ.name
+        if isinstance(typ, QAPISchemaArrayType):
+            return '[' + self._use_type(typ.element_type) + ']'
+        return self._name(typ.name)
+
+    def _gen_json(self, name, mtype, obj):
+        if mtype not in ('command', 'event', 'builtin', 'array'):
+            name = self._name(name)
+        obj['name'] = name
+        obj['meta-type'] = mtype
+        self._jsons.append(obj)
+
+    def _gen_member(self, member):
+        ret = {'name': member.name, 'type': self._use_type(member.type)}
+        if member.optional:
+            ret['default'] = None
+        return ret
+
+    def _gen_variants(self, tag_name, variants):
+        return {'tag': tag_name,
+                'variants': [self._gen_variant(v) for v in variants]}
+
+    def _gen_variant(self, variant):
+        return {'case': variant.name, 'type': self._use_type(variant.type)}
+
+    def visit_builtin_type(self, name, info, json_type):
+        self._gen_json(name, 'builtin', {'json-type': json_type})
+
+    def visit_enum_type(self, name, info, values, prefix):
+        self._gen_json(name, 'enum', {'values': values})
+
+    def visit_array_type(self, name, info, element_type):
+        element = self._use_type(element_type)
+        self._gen_json('[' + element + ']', 'array', {'element-type': element})
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        obj = {'members': [self._gen_member(m) for m in members]}
+        if variants:
+            obj.update(self._gen_variants(variants.tag_member.name,
+                                          variants.variants))
+        self._gen_json(name, 'object', obj)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._gen_json(name, 'alternate',
+                       {'members': [{'type': self._use_type(m.type)}
+                                    for m in variants.variants]})
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        ret_type = ret_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'command',
+                       {'arg-type': self._use_type(arg_type),
+                        'ret-type': self._use_type(ret_type)})
+
+    def visit_event(self, name, info, arg_type):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)})
+
+# Debugging aid: unmask QAPI schema's type names
+# We normally mask them, because they're not QMP wire ABI
+opt_unmask = False
+
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("u", ["unmask-non-abi-names"])
+
+for o, a in opts:
+    if o in ("-u", "--unmask-non-abi-names"):
+        opt_unmask = True
+
+c_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+h_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qmp-introspect.c', 'qmp-introspect.h',
+                            c_comment, h_comment)
+
+fdef.write(mcgen('''
+#include "qemu/osdep.h"
+#include "%(prefix)sqmp-introspect.h"
+
+''',
+                 prefix=prefix))
+
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenIntrospectVisitor(opt_unmask)
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
+
+close_output(fdef, fdecl)
diff --git a/qemu/scripts/qapi-types.py b/qemu/scripts/qapi-types.py
index e6eb4b613..437cf6c8e 100644
--- a/qemu/scripts/qapi-types.py
+++ b/qemu/scripts/qapi-types.py
@@ -2,271 +2,247 @@
 # QAPI types generator
 #
 # Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2016 Red Hat Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 
-def generate_fwd_builtin(name):
-    return mcgen('''
 
-typedef struct %(name)sList
-{
-    union {
-        %(type)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
-''',
-                 type=c_type(name),
-                 name=name)
+# variants must be emitted before their container; track what has already
+# been output
+objects_seen = set()
 
-def generate_fwd_struct(name):
-    return mcgen('''
 
-typedef struct %(name)s %(name)s;
+def gen_fwd_object_or_array(name):
+    return mcgen('''
 
-typedef struct %(name)sList
-{
-    union {
-        %(name)s *value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
+typedef struct %(c_name)s %(c_name)s;
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name))
+
 
-def generate_fwd_enum_struct(name):
+def gen_array(name, element_type):
     return mcgen('''
-typedef struct %(name)sList
-{
-    union {
-        %(name)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
+
+struct %(c_name)s {
+    %(c_name)s *next;
+    %(c_type)s value;
+};
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name), c_type=element_type.c_type())
 
-def generate_struct_fields(members):
-    ret = ''
 
-    for argname, argentry, optional in parse_args(members):
-        if optional:
+def gen_struct_members(members):
+    ret = ''
+    for memb in members:
+        if memb.optional:
             ret += mcgen('''
     bool has_%(c_name)s;
 ''',
-                         c_name=c_name(argname))
+                         c_name=c_name(memb.name))
         ret += mcgen('''
     %(c_type)s %(c_name)s;
 ''',
-                     c_type=c_type(argentry), c_name=c_name(argname))
-
+                     c_type=memb.type.c_type(), c_name=c_name(memb.name))
     return ret
 
-def generate_struct(expr):
 
-    structname = expr.get('struct', "")
-    members = expr['data']
-    base = expr.get('base')
+def gen_object(name, base, members, variants):
+    if name in objects_seen:
+        return ''
+    objects_seen.add(name)
 
-    ret = mcgen('''
-struct %(name)s
-{
-''',
-          name=c_name(structname))
+    ret = ''
+    if variants:
+        for v in variants.variants:
+            if isinstance(v.type, QAPISchemaObjectType):
+                ret += gen_object(v.type.name, v.type.base,
+                                  v.type.local_members, v.type.variants)
 
-    if base:
-        ret += generate_struct_fields({'base': base})
+    ret += mcgen('''
 
-    ret += generate_struct_fields(members)
+struct %(c_name)s {
+''',
+                 c_name=c_name(name))
 
-    # Make sure that all structs have at least one field; this avoids
-    # potential issues with attempting to malloc space for zero-length structs
-    # in C, and also incompatibility with C++ (where an empty struct is size 1).
-    if not base and not members:
+    if base:
+        if not base.is_implicit():
             ret += mcgen('''
-    char qapi_dummy_field_for_empty_struct;
-''')
-
-    ret += mcgen('''
-};
+    /* Members inherited from %(c_name)s: */
+''',
+                         c_name=base.c_name())
+        ret += gen_struct_members(base.members)
+        if not base.is_implicit():
+            ret += mcgen('''
+    /* Own members: */
 ''')
+    ret += gen_struct_members(members)
 
-    return ret
+    if variants:
+        ret += gen_variants(variants)
 
-def generate_enum_lookup(name, values):
-    ret = mcgen('''
-const char * const %(name)s_lookup[] = {
-''',
-                name=c_name(name))
-    i = 0
-    for value in values:
-        index = c_enum_const(name, value)
+    # Make sure that all structs have at least one member; this avoids
+    # potential issues with attempting to malloc space for zero-length
+    # structs in C, and also incompatibility with C++ (where an empty
+    # struct is size 1).
+    if not (base and base.members) and not members and not variants:
         ret += mcgen('''
-    [%(index)s] = "%(value)s",
-''',
-                     index = index, value = value)
+    char qapi_dummy_for_empty_struct;
+''')
 
-    max_index = c_enum_const(name, 'MAX')
     ret += mcgen('''
-    [%(max_index)s] = NULL,
 };
+''')
 
-''',
-        max_index=max_index)
     return ret
 
-def generate_enum(name, values):
-    name = c_name(name)
-    lookup_decl = mcgen('''
-extern const char * const %(name)s_lookup[];
-''',
-                name=name)
-
-    enum_decl = mcgen('''
-typedef enum %(name)s
-{
-''',
-                name=name)
 
-    # append automatically generated _MAX value
-    enum_values = values + [ 'MAX' ]
-
-    i = 0
-    for value in enum_values:
-        enum_full_value = c_enum_const(name, value)
-        enum_decl += mcgen('''
-    %(enum_full_value)s = %(i)d,
-''',
-                     enum_full_value = enum_full_value,
-                     i=i)
-        i += 1
+def gen_upcast(name, base):
+    # C makes const-correctness ugly.  We have to cast away const to let
+    # this function work for both const and non-const obj.
+    return mcgen('''
 
-    enum_decl += mcgen('''
-} %(name)s;
+static inline %(base)s *qapi_%(c_name)s_base(const %(c_name)s *obj)
+{
+    return (%(base)s *)obj;
+}
 ''',
-                 name=name)
+                 c_name=c_name(name), base=base.c_name())
 
-    return lookup_decl + enum_decl
-
-def generate_alternate_qtypes(expr):
-
-    name = expr['alternate']
-    members = expr['data']
 
+def gen_variants(variants):
     ret = mcgen('''
-const int %(name)s_qtypes[QTYPE_MAX] = {
+    union { /* union tag is @%(c_name)s */
 ''',
-                name=c_name(name))
-
-    for key in members:
-        qtype = find_alternate_member_qtype(members[key])
-        assert qtype, "Invalid alternate member"
+                c_name=c_name(variants.tag_member.name))
 
+    for var in variants.variants:
         ret += mcgen('''
-    [%(qtype)s] = %(enum_const)s,
+        %(c_type)s %(c_name)s;
 ''',
-                     qtype = qtype,
-                     enum_const = c_enum_const(name + 'Kind', key))
+                     c_type=var.type.c_unboxed_type(),
+                     c_name=c_name(var.name))
 
     ret += mcgen('''
-};
+    } u;
 ''')
-    return ret
-
-
-def generate_union(expr, meta):
 
-    name = c_name(expr[meta])
-    typeinfo = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
+    return ret
 
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        discriminator_type_name = enum_define['enum_name']
-    else:
-        discriminator_type_name = '%sKind' % (name)
 
+def gen_type_cleanup_decl(name):
     ret = mcgen('''
-struct %(name)s
-{
-    %(discriminator_type_name)s kind;
-    union {
-        void *data;
-''',
-                name=name,
-                discriminator_type_name=c_name(discriminator_type_name))
 
-    for key in typeinfo:
-        ret += mcgen('''
-        %(c_type)s %(c_name)s;
+void qapi_free_%(c_name)s(%(c_name)s *obj);
 ''',
-                     c_type=c_type(typeinfo[key]),
-                     c_name=c_name(key))
-
-    ret += mcgen('''
-    };
-''')
-
-    if base:
-        assert discriminator
-        base_fields = find_struct(base)['data'].copy()
-        del base_fields[discriminator]
-        ret += generate_struct_fields(base_fields)
-    else:
-        assert not discriminator
-
-    ret += mcgen('''
-};
-''')
-    if meta == 'alternate':
-        ret += mcgen('''
-extern const int %(name)s_qtypes[];
-''',
-            name=name)
-
-
+                c_name=c_name(name))
     return ret
 
-def generate_type_cleanup_decl(name):
-    ret = mcgen('''
-void qapi_free_%(name)s(%(c_type)s obj);
-''',
-                c_type=c_type(name), name=c_name(name))
-    return ret
 
-def generate_type_cleanup(name):
+def gen_type_cleanup(name):
     ret = mcgen('''
 
-void qapi_free_%(name)s(%(c_type)s obj)
+void qapi_free_%(c_name)s(%(c_name)s *obj)
 {
-    QapiDeallocVisitor *md;
+    QapiDeallocVisitor *qdv;
     Visitor *v;
 
     if (!obj) {
         return;
     }
 
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    visit_type_%(name)s(v, &obj, NULL, NULL);
-    qapi_dealloc_visitor_cleanup(md);
+    qdv = qapi_dealloc_visitor_new();
+    v = qapi_dealloc_get_visitor(qdv);
+    visit_type_%(c_name)s(v, NULL, &obj, NULL);
+    qapi_dealloc_visitor_cleanup(qdv);
 }
 ''',
-                c_type=c_type(name), name=c_name(name))
+                c_name=c_name(name))
     return ret
 
+
+class QAPISchemaGenTypeVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._fwdecl = None
+        self._btin = None
+
+    def visit_begin(self, schema):
+        # gen_object() is recursive, ensure it doesn't visit the empty type
+        objects_seen.add(schema.the_empty_object_type.name)
+        self.decl = ''
+        self.defn = ''
+        self._fwdecl = ''
+        self._btin = guardstart('QAPI_TYPES_BUILTIN')
+
+    def visit_end(self):
+        self.decl = self._fwdecl + self.decl
+        self._fwdecl = None
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_TYPES_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def _gen_type_cleanup(self, name):
+        self.decl += gen_type_cleanup_decl(name)
+        self.defn += gen_type_cleanup(name)
+
+    def visit_enum_type(self, name, info, values, prefix):
+        # Special case for our lone builtin enum type
+        # TODO use something cleaner than existence of info
+        if not info:
+            self._btin += gen_enum(name, values, prefix)
+            if do_builtins:
+                self.defn += gen_enum_lookup(name, values, prefix)
+        else:
+            self._fwdecl += gen_enum(name, values, prefix)
+            self.defn += gen_enum_lookup(name, values, prefix)
+
+    def visit_array_type(self, name, info, element_type):
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += gen_fwd_object_or_array(name)
+            self._btin += gen_array(name, element_type)
+            self._btin += gen_type_cleanup_decl(name)
+            if do_builtins:
+                self.defn += gen_type_cleanup(name)
+        else:
+            self._fwdecl += gen_fwd_object_or_array(name)
+            self.decl += gen_array(name, element_type)
+            self._gen_type_cleanup(name)
+
+    def visit_object_type(self, name, info, base, members, variants):
+        # Nothing to do for the special empty builtin
+        if name == 'q_empty':
+            return
+        self._fwdecl += gen_fwd_object_or_array(name)
+        self.decl += gen_object(name, base, members, variants)
+        if base and not base.is_implicit():
+            self.decl += gen_upcast(name, base)
+        # TODO Worth changing the visitor signature, so we could
+        # directly use rather than repeat type.is_implicit()?
+        if not name.startswith('q_'):
+            # implicit types won't be directly allocated/freed
+            self._gen_type_cleanup(name)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._fwdecl += gen_fwd_object_or_array(name)
+        self.decl += gen_object(name, None, [variants.tag_member], variants)
+        self._gen_type_cleanup(name)
+
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenTypeVisitor.visit_end().
 do_builtins = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -311,92 +287,17 @@ h_comment = '''
                             c_comment, h_comment)
 
 fdef.write(mcgen('''
+#include "qemu/osdep.h"
 #include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
-
 ''',
                  prefix=prefix))
 
-fdecl.write(mcgen('''
-#include <stdbool.h>
-#include <stdint.h>
-
-'''))
-
-exprs = parse_schema(input_file)
-
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_fwd_builtin(typename))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('struct'):
-        ret += generate_fwd_struct(expr['struct'])
-    elif expr.has_key('enum'):
-        ret += generate_enum(expr['enum'], expr['data']) + "\n"
-        ret += generate_fwd_enum_struct(expr['enum'])
-        fdef.write(generate_enum_lookup(expr['enum'], expr['data']))
-    elif expr.has_key('union'):
-        ret += generate_fwd_struct(expr['union']) + "\n"
-        enum_define = discriminator_find_enum_define(expr)
-        if not enum_define:
-            ret += generate_enum('%sKind' % expr['union'], expr['data'].keys())
-            fdef.write(generate_enum_lookup('%sKind' % expr['union'],
-                                            expr['data'].keys()))
-    elif expr.has_key('alternate'):
-        ret += generate_fwd_struct(expr['alternate']) + "\n"
-        ret += generate_enum('%sKind' % expr['alternate'], expr['data'].keys())
-        fdef.write(generate_enum_lookup('%sKind' % expr['alternate'],
-                                        expr['data'].keys()))
-        fdef.write(generate_alternate_qtypes(expr))
-    else:
-        continue
-    fdecl.write(ret)
-
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_type_cleanup_decl(typename + "List"))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    fdef.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-    for typename in builtin_types.keys():
-        fdef.write(generate_type_cleanup(typename + "List"))
-    fdef.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('struct'):
-        ret += generate_struct(expr) + "\n"
-        ret += generate_type_cleanup_decl(expr['struct'] + "List")
-        fdef.write(generate_type_cleanup(expr['struct'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['struct'])
-        fdef.write(generate_type_cleanup(expr['struct']) + "\n")
-    elif expr.has_key('union'):
-        ret += generate_union(expr, 'union')
-        ret += generate_type_cleanup_decl(expr['union'] + "List")
-        fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['union'])
-        fdef.write(generate_type_cleanup(expr['union']) + "\n")
-    elif expr.has_key('alternate'):
-        ret += generate_union(expr, 'alternate')
-        ret += generate_type_cleanup_decl(expr['alternate'] + "List")
-        fdef.write(generate_type_cleanup(expr['alternate'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['alternate'])
-        fdef.write(generate_type_cleanup(expr['alternate']) + "\n")
-    elif expr.has_key('enum'):
-        ret += generate_type_cleanup_decl(expr['enum'] + "List")
-        fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n")
-    else:
-        continue
-    fdecl.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenTypeVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/qemu/scripts/qapi-visit.py b/qemu/scripts/qapi-visit.py
index 5b9933648..31d233035 100644
--- a/qemu/scripts/qapi-visit.py
+++ b/qemu/scripts/qapi-visit.py
@@ -2,7 +2,7 @@
 # QAPI visitor generator
 #
 # Copyright IBM, Corp. 2011
-# Copyright (C) 2014-2015 Red Hat, Inc.
+# Copyright (C) 2014-2016 Red Hat, Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -12,86 +12,90 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
 
-implicit_structs = []
 
-def generate_visit_implicit_struct(type):
-    global implicit_structs
-    if type in implicit_structs:
-        return ''
-    implicit_structs.append(type)
+def gen_visit_decl(name, scalar=False):
+    c_type = c_name(name) + ' *'
+    if not scalar:
+        c_type += '*'
     return mcgen('''
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp);
+''',
+                 c_name=c_name(name), c_type=c_type)
 
-static void visit_type_implicit_%(c_type)s(Visitor *m, %(c_type)s **obj, Error **errp)
-{
-    Error *err = NULL;
 
-    visit_start_implicit_struct(m, (void **)obj, sizeof(%(c_type)s), &err);
-    if (!err) {
-        visit_type_%(c_type)s_fields(m, obj, errp);
-        visit_end_implicit_struct(m, &err);
-    }
-    error_propagate(errp, err);
-}
+def gen_visit_members_decl(name):
+    return mcgen('''
+
+void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp);
 ''',
-                 c_type=type_name(type))
+                 c_name=c_name(name))
 
-def generate_visit_struct_fields(name, members, base = None):
-    substructs = []
-    ret = ''
 
-    if base:
-        ret += generate_visit_implicit_struct(base)
-
-    ret += mcgen('''
+def gen_visit_object_members(name, base, members, variants):
+    ret = mcgen('''
 
-static void visit_type_%(name)s_fields(Visitor *m, %(name)s **obj, Error **errp)
+void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp)
 {
     Error *err = NULL;
+
 ''',
-                 name=c_name(name))
-    push_indent()
+                c_name=c_name(name))
 
     if base:
         ret += mcgen('''
-visit_type_implicit_%(type)s(m, &(*obj)->%(c_name)s, &err);
-if (err) {
-    goto out;
-}
+    visit_type_%(c_type)s_members(v, (%(c_type)s *)obj, &err);
 ''',
-                     type=type_name(base), c_name=c_name('base'))
+                     c_type=base.c_name())
+        ret += gen_err_check()
 
-    for argname, argentry, optional in parse_args(members):
-        if optional:
+    for memb in members:
+        if memb.optional:
             ret += mcgen('''
-visit_optional(m, &(*obj)->has_%(c_name)s, "%(name)s", &err);
-if (!err && (*obj)->has_%(c_name)s) {
+    if (visit_optional(v, "%(name)s", &obj->has_%(c_name)s)) {
 ''',
-                         c_name=c_name(argname), name=argname)
+                         name=memb.name, c_name=c_name(memb.name))
             push_indent()
-
         ret += mcgen('''
-visit_type_%(type)s(m, &(*obj)->%(c_name)s, "%(name)s", &err);
+    visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, &err);
 ''',
-                     type=type_name(argentry), c_name=c_name(argname),
-                     name=argname)
-
-        if optional:
+                     c_type=memb.type.c_name(), name=memb.name,
+                     c_name=c_name(memb.name))
+        ret += gen_err_check()
+        if memb.optional:
             pop_indent()
             ret += mcgen('''
-}
+    }
 ''')
+
+    if variants:
         ret += mcgen('''
-if (err) {
-    goto out;
-}
+    switch (obj->%(c_name)s) {
+''',
+                     c_name=c_name(variants.tag_member.name))
+
+        for var in variants.variants:
+            ret += mcgen('''
+    case %(case)s:
+        visit_type_%(c_type)s_members(v, &obj->u.%(c_name)s, &err);
+        break;
+''',
+                         case=c_enum_const(variants.tag_member.type.name,
+                                           var.name,
+                                           variants.tag_member.type.prefix),
+                         c_type=var.type.c_name(), c_name=c_name(var.name))
+
+        ret += mcgen('''
+    default:
+        abort();
+    }
 ''')
 
-    pop_indent()
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
+    # 'goto out' produced for base, for each member, and if variants were
+    # present
+    if base or members or variants:
         ret += mcgen('''
 
 out:
@@ -103,275 +107,209 @@ out:
     return ret
 
 
-def generate_visit_struct_body(name, members):
-    ret = mcgen('''
-    Error *err = NULL;
-
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err);
-    if (!err) {
-        if (*obj) {
-            visit_type_%(c_name)s_fields(m, obj, errp);
-        }
-        visit_end_struct(m, &err);
-    }
-    error_propagate(errp, err);
-''',
-                name=name, c_name=c_name(name))
-
-    return ret
-
-def generate_visit_struct(expr):
-
-    name = expr['struct']
-    members = expr['data']
-    base = expr.get('base')
-
-    ret = generate_visit_struct_fields(name, members, base)
-
-    ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
-{
-''',
-                 name=c_name(name))
-
-    ret += generate_visit_struct_body(name, members)
-
-    ret += mcgen('''
-}
-''')
-    return ret
-
-def generate_visit_list(name, members):
+def gen_visit_list(name, element_type):
+    # FIXME: if *obj is NULL on entry, and the first visit_next_list()
+    # assigns to *obj, while a later one fails, we should clean up *obj
+    # rather than leaving it non-NULL. As currently written, the caller must
+    # call qapi_free_FOOList() to avoid a memory leak of the partial FOOList.
     return mcgen('''
 
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
 {
     Error *err = NULL;
     GenericList *i, **prev;
 
-    visit_start_list(m, name, &err);
+    visit_start_list(v, name, &err);
     if (err) {
         goto out;
     }
 
     for (prev = (GenericList **)obj;
-         !err && (i = visit_next_list(m, prev, &err)) != NULL;
+         !err && (i = visit_next_list(v, prev, sizeof(**obj))) != NULL;
          prev = &i) {
-        %(name)sList *native_i = (%(name)sList *)i;
-        visit_type_%(name)s(m, &native_i->value, NULL, &err);
+        %(c_name)s *native_i = (%(c_name)s *)i;
+        visit_type_%(c_elt_type)s(v, NULL, &native_i->value, &err);
     }
 
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_list(m, &err);
+    visit_end_list(v);
 out:
     error_propagate(errp, err);
 }
 ''',
-                name=type_name(name))
+                 c_name=c_name(name), c_elt_type=element_type.c_name())
 
-def generate_visit_enum(name, members):
+
+def gen_visit_enum(name):
     return mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp)
 {
-    visit_type_enum(m, (int *)obj, %(name)s_lookup, "%(name)s", name, errp);
+    int value = *obj;
+    visit_type_enum(v, name, &value, %(c_name)s_lookup, errp);
+    *obj = value;
 }
 ''',
-                 name=c_name(name))
-
-def generate_visit_alternate(name, members):
-    ret = mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_implicit_struct(m, (void**) obj, sizeof(%(name)s), &err);
-    if (err) {
-        goto out;
-    }
-    visit_get_next_type(m, (int*) &(*obj)->kind, %(name)s_qtypes, name, &err);
-    if (err) {
-        goto out_end;
-    }
-    switch ((*obj)->kind) {
-''',
-                name=c_name(name))
-
-    # For alternate, always use the default enum type automatically generated
-    # as name + 'Kind'
-    disc_type = c_name(name) + 'Kind'
-
-    for key in members:
-        assert (members[key] in builtin_types.keys()
-            or find_struct(members[key])
-            or find_union(members[key])
-            or find_enum(members[key])), "Invalid alternate member"
-
-        enum_full_value = c_enum_const(disc_type, key)
-        ret += mcgen('''
-    case %(enum_full_value)s:
-        visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, name, &err);
-        break;
-''',
-                enum_full_value = enum_full_value,
-                c_type = type_name(members[key]),
-                c_name = c_name(key))
-
-    ret += mcgen('''
-    default:
-        abort();
-    }
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_implicit_struct(m, &err);
-out:
-    error_propagate(errp, err);
-}
-''')
-
-    return ret
-
-
-def generate_visit_union(expr):
-
-    name = expr['union']
-    members = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
-
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        # Use the enum type as discriminator
-        ret = ""
-        disc_type = c_name(enum_define['enum_name'])
-    else:
-        # There will always be a discriminator in the C switch code, by default
-        # it is an enum type generated silently
-        ret = generate_visit_enum(name + 'Kind', members.keys())
-        disc_type = c_name(name) + 'Kind'
+                 c_name=c_name(name))
 
-    if base:
-        assert discriminator
-        base_fields = find_struct(base)['data'].copy()
-        del base_fields[discriminator]
-        ret += generate_visit_struct_fields(name, base_fields)
 
-    if discriminator:
-        for key in members:
-            ret += generate_visit_implicit_struct(members[key])
+def gen_visit_alternate(name, variants):
+    promote_int = 'true'
+    ret = ''
+    for var in variants.variants:
+        if var.type.alternate_qtype() == 'QTYPE_QINT':
+            promote_int = 'false'
 
     ret += mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
 {
     Error *err = NULL;
 
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
+    visit_start_alternate(v, name, (GenericAlternate **)obj, sizeof(**obj),
+                          %(promote_int)s, &err);
     if (err) {
         goto out;
     }
-    if (*obj) {
+    switch ((*obj)->type) {
 ''',
-                 name=c_name(name))
+                 c_name=c_name(name), promote_int=promote_int)
 
-    if base:
+    for var in variants.variants:
         ret += mcgen('''
-        visit_type_%(name)s_fields(m, obj, &err);
-        if (err) {
-            goto out_obj;
-        }
+    case %(case)s:
 ''',
-                     name=c_name(name))
-
-    if not discriminator:
-        disc_key = "type"
-    else:
-        disc_key = discriminator
-    ret += mcgen('''
-        visit_type_%(disc_type)s(m, &(*obj)->kind, "%(disc_key)s", &err);
+                     case=var.type.alternate_qtype())
+        if isinstance(var.type, QAPISchemaObjectType):
+            ret += mcgen('''
+        visit_start_struct(v, name, NULL, 0, &err);
         if (err) {
-            goto out_obj;
-        }
-        if (!visit_start_union(m, !!(*obj)->data, &err) || err) {
-            goto out_obj;
+            break;
         }
-        switch ((*obj)->kind) {
+        visit_type_%(c_type)s_members(v, &(*obj)->u.%(c_name)s, &err);
+        error_propagate(errp, err);
+        err = NULL;
+        visit_end_struct(v, &err);
 ''',
-                 disc_type = disc_type,
-                 disc_key = disc_key)
-
-    for key in members:
-        if not discriminator:
-            fmt = 'visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, "data", &err);'
+                         c_type=var.type.c_name(),
+                         c_name=c_name(var.name))
         else:
-            fmt = 'visit_type_implicit_%(c_type)s(m, &(*obj)->%(c_name)s, &err);'
-
-        enum_full_value = c_enum_const(disc_type, key)
-        ret += mcgen('''
-        case %(enum_full_value)s:
-            ''' + fmt + '''
-            break;
+            ret += mcgen('''
+        visit_type_%(c_type)s(v, name, &(*obj)->u.%(c_name)s, &err);
 ''',
-                enum_full_value = enum_full_value,
-                c_type=type_name(members[key]),
-                c_name=c_name(key))
+                         c_type=var.type.c_name(),
+                         c_name=c_name(var.name))
+        ret += mcgen('''
+        break;
+''')
 
     ret += mcgen('''
-        default:
-            abort();
-        }
-out_obj:
-        error_propagate(errp, err);
-        err = NULL;
-        visit_end_union(m, !!(*obj)->data, &err);
-        error_propagate(errp, err);
-        err = NULL;
+    default:
+        error_setg(&err, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "%(name)s");
     }
-    visit_end_struct(m, &err);
+    visit_end_alternate(v);
 out:
     error_propagate(errp, err);
 }
-''')
-
-    return ret
-
-def generate_declaration(name, members, builtin_type=False):
-    ret = ""
-    if not builtin_type:
-        name = c_name(name)
-        ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp);
-''',
-                     name=name)
-
-    ret += mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
 ''',
                  name=name)
 
     return ret
 
-def generate_enum_declaration(name, members):
-    ret = mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
-''',
-                name=c_name(name))
 
-    return ret
-
-def generate_decl_enum(name, members):
+def gen_visit_object(name, base, members, variants):
+    # FIXME: if *obj is NULL on entry, and visit_start_struct() assigns to
+    # *obj, but then visit_type_FOO_members() fails, we should clean up *obj
+    # rather than leaving it non-NULL. As currently written, the caller must
+    # call qapi_free_FOO() to avoid a memory leak of the partial FOO.
     return mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp);
-''',
-                 name=c_name(name))
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+{
+    Error *err = NULL;
 
+    visit_start_struct(v, name, (void **)obj, sizeof(%(c_name)s), &err);
+    if (err) {
+        goto out;
+    }
+    if (!*obj) {
+        goto out_obj;
+    }
+    visit_type_%(c_name)s_members(v, *obj, &err);
+    error_propagate(errp, err);
+    err = NULL;
+out_obj:
+    visit_end_struct(v, &err);
+out:
+    error_propagate(errp, err);
+}
+''',
+                 c_name=c_name(name))
+
+
+class QAPISchemaGenVisitVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._btin = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._btin = guardstart('QAPI_VISIT_BUILTIN')
+
+    def visit_end(self):
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_VISIT_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def visit_enum_type(self, name, info, values, prefix):
+        # Special case for our lone builtin enum type
+        # TODO use something cleaner than existence of info
+        if not info:
+            self._btin += gen_visit_decl(name, scalar=True)
+            if do_builtins:
+                self.defn += gen_visit_enum(name)
+        else:
+            self.decl += gen_visit_decl(name, scalar=True)
+            self.defn += gen_visit_enum(name)
+
+    def visit_array_type(self, name, info, element_type):
+        decl = gen_visit_decl(name)
+        defn = gen_visit_list(name, element_type)
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += decl
+            if do_builtins:
+                self.defn += defn
+        else:
+            self.decl += decl
+            self.defn += defn
+
+    def visit_object_type(self, name, info, base, members, variants):
+        # Nothing to do for the special empty builtin
+        if name == 'q_empty':
+            return
+        self.decl += gen_visit_members_decl(name)
+        self.defn += gen_visit_object_members(name, base, members, variants)
+        # TODO Worth changing the visitor signature, so we could
+        # directly use rather than repeat type.is_implicit()?
+        if not name.startswith('q_'):
+            # only explicit types need an allocating visit
+            self.decl += gen_visit_decl(name)
+            self.defn += gen_visit_object(name, base, members, variants)
+
+    def visit_alternate_type(self, name, info, variants):
+        self.decl += gen_visit_decl(name)
+        self.defn += gen_visit_alternate(name, variants)
+
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenVisitVisitor.visit_end().
 do_builtins = False
 
 (input_file, output_dir, do_c, do_h, prefix, opts) = \
@@ -415,70 +353,25 @@ h_comment = '''
                             c_comment, h_comment)
 
 fdef.write(mcgen('''
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "%(prefix)sqapi-visit.h"
 ''',
-                 prefix = prefix))
+                 prefix=prefix))
 
 fdecl.write(mcgen('''
 #include "qapi/visitor.h"
+#include "qapi/qmp/qerror.h"
 #include "%(prefix)sqapi-types.h"
 
 ''',
                   prefix=prefix))
 
-exprs = parse_schema(input_file)
-
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-for typename in builtin_types.keys():
-    fdecl.write(generate_declaration(typename, None, builtin_type=True))
-fdecl.write(guardend("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    for typename in builtin_types.keys():
-        fdef.write(generate_visit_list(typename, None))
-
-for expr in exprs:
-    if expr.has_key('struct'):
-        ret = generate_visit_struct(expr)
-        ret += generate_visit_list(expr['struct'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_declaration(expr['struct'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('union'):
-        ret = generate_visit_union(expr)
-        ret += generate_visit_list(expr['union'], expr['data'])
-        fdef.write(ret)
-
-        enum_define = discriminator_find_enum_define(expr)
-        ret = ""
-        if not enum_define:
-            ret = generate_decl_enum('%sKind' % expr['union'],
-                                     expr['data'].keys())
-        ret += generate_declaration(expr['union'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('alternate'):
-        ret = generate_visit_alternate(expr['alternate'], expr['data'])
-        ret += generate_visit_list(expr['alternate'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_decl_enum('%sKind' % expr['alternate'],
-                                 expr['data'].keys())
-        ret += generate_declaration(expr['alternate'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('enum'):
-        ret = generate_visit_list(expr['enum'], expr['data'])
-        ret += generate_visit_enum(expr['enum'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_decl_enum(expr['enum'], expr['data'])
-        ret += generate_enum_declaration(expr['enum'], expr['data'])
-        fdecl.write(ret)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenVisitVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
 close_output(fdef, fdecl)
diff --git a/qemu/scripts/qapi.py b/qemu/scripts/qapi.py
index 06d7fc284..b13ae4789 100644
--- a/qemu/scripts/qapi.py
+++ b/qemu/scripts/qapi.py
@@ -2,7 +2,7 @@
 # QAPI helper library
 #
 # Copyright IBM, Corp. 2011
-# Copyright (c) 2013-2015 Red Hat Inc.
+# Copyright (c) 2013-2016 Red Hat Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -33,12 +33,15 @@ builtin_types = {
     'uint32':   'QTYPE_QINT',
     'uint64':   'QTYPE_QINT',
     'size':     'QTYPE_QINT',
+    'any':      None,           # any QType possible, actually
+    'QType':    'QTYPE_QSTRING',
 }
 
 # Whitelist of commands allowed to return a non-dictionary
 returns_whitelist = [
     # From QMP:
     'human-monitor-command',
+    'qom-get',
     'query-migrate-cache-size',
     'query-tpm-models',
     'query-tpm-types',
@@ -54,9 +57,18 @@ returns_whitelist = [
     'guest-set-vcpus',
     'guest-sync',
     'guest-sync-delimited',
+]
 
-    # From qapi-schema-test:
-    'user_def_cmd3',
+# Whitelist of entities allowed to violate case conventions
+case_whitelist = [
+    # From QMP:
+    'ACPISlotType',         # DIMM, visible through query-acpi-ospm-status
+    'CpuInfoMIPS',          # PC, visible through query-cpu
+    'CpuInfoTricore',       # PC, visible through query-cpu
+    'QapiErrorClass',       # all members, visible through errors
+    'UuidInfo',             # UUID, visible through query-uuid
+    'X86CPURegister32',     # all members, visible indirectly through qom-get
+    'q_obj_CpuInfo-base',   # CPU, visible through query-cpu
 ]
 
 enum_types = []
@@ -69,6 +81,7 @@ all_names = {}
 # Parsing the schema into expressions
 #
 
+
 def error_path(parent):
     res = ""
     while parent:
@@ -77,8 +90,10 @@ def error_path(parent):
         parent = parent['parent']
     return res
 
+
 class QAPISchemaError(Exception):
     def __init__(self, schema, msg):
+        Exception.__init__(self)
         self.fname = schema.fname
         self.msg = msg
         self.col = 1
@@ -94,8 +109,11 @@ class QAPISchemaError(Exception):
         return error_path(self.info) + \
             "%s:%d:%d: %s" % (self.fname, self.line, self.col, self.msg)
 
+
 class QAPIExprError(Exception):
     def __init__(self, expr_info, msg):
+        Exception.__init__(self)
+        assert expr_info
         self.info = expr_info
         self.msg = msg
 
@@ -103,9 +121,10 @@ class QAPIExprError(Exception):
         return error_path(self.info['parent']) + \
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
-class QAPISchema:
 
-    def __init__(self, fp, previously_included = [], incl_info = None):
+class QAPISchemaParser(object):
+
+    def __init__(self, fp, previously_included=[], incl_info=None):
         abs_fname = os.path.abspath(fp.name)
         fname = fp.name
         self.fname = fname
@@ -120,18 +139,18 @@ class QAPISchema:
         self.exprs = []
         self.accept()
 
-        while self.tok != None:
+        while self.tok is not None:
             expr_info = {'file': fname, 'line': self.line,
                          'parent': self.incl_info}
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
-                    raise QAPIExprError(expr_info, "Invalid 'include' directive")
+                    raise QAPIExprError(expr_info,
+                                        "Invalid 'include' directive")
                 include = expr["include"]
                 if not isinstance(include, str):
                     raise QAPIExprError(expr_info,
-                                        'Expected a file name (string), got: %s'
-                                        % include)
+                                        "Value of 'include' must be a string")
                 incl_abs_fname = os.path.join(os.path.dirname(abs_fname),
                                               include)
                 # catch inclusion cycle
@@ -146,11 +165,11 @@ class QAPISchema:
                     continue
                 try:
                     fobj = open(incl_abs_fname, 'r')
-                except IOError, e:
+                except IOError as e:
                     raise QAPIExprError(expr_info,
                                         '%s: %s' % (e.strerror, include))
-                exprs_include = QAPISchema(fobj, previously_included,
-                                           expr_info)
+                exprs_include = QAPISchemaParser(fobj, previously_included,
+                                                 expr_info)
                 self.exprs.extend(exprs_include.exprs)
             else:
                 expr_elem = {'expr': expr,
@@ -166,7 +185,7 @@ class QAPISchema:
 
             if self.tok == '#':
                 self.cursor = self.src.find('\n', self.cursor)
-            elif self.tok in ['{', '}', ':', ',', '[', ']']:
+            elif self.tok in "{}:,[]":
                 return
             elif self.tok == "'":
                 string = ''
@@ -190,7 +209,7 @@ class QAPISchema:
                             string += '\t'
                         elif ch == 'u':
                             value = 0
-                            for x in range(0, 4):
+                            for _ in range(0, 4):
                                 ch = self.src[self.cursor]
                                 self.cursor += 1
                                 if ch not in "0123456789abcdefABCDEF":
@@ -212,7 +231,7 @@ class QAPISchema:
                             string += ch
                         else:
                             raise QAPISchemaError(self,
-                                                  "Unknown escape \\%s" %ch)
+                                                  "Unknown escape \\%s" % ch)
                         esc = False
                     elif ch == "\\":
                         esc = True
@@ -272,7 +291,7 @@ class QAPISchema:
         if self.tok == ']':
             self.accept()
             return expr
-        if not self.tok in "{['tfn":
+        if self.tok not in "{['tfn":
             raise QAPISchemaError(self, 'Expected "{", "[", "]", string, '
                                   'boolean or "null"')
         while True:
@@ -302,17 +321,23 @@ class QAPISchema:
 
 #
 # Semantic analysis of schema expressions
+# TODO fold into QAPISchema
+# TODO catching name collisions in generated code would be nice
 #
 
-def find_base_fields(base):
+
+def find_base_members(base):
+    if isinstance(base, dict):
+        return base
     base_struct_define = find_struct(base)
     if not base_struct_define:
         return None
     return base_struct_define['data']
 
+
 # Return the qtype of an alternate branch, or None on error.
 def find_alternate_member_qtype(qapi_type):
-    if builtin_types.has_key(qapi_type):
+    if qapi_type in builtin_types:
         return builtin_types[qapi_type]
     elif find_struct(qapi_type):
         return "QTYPE_QDICT"
@@ -322,6 +347,7 @@ def find_alternate_member_qtype(qapi_type):
         return "QTYPE_QDICT"
     return None
 
+
 # Return the discriminator enum define if discriminator is specified as an
 # enum type, otherwise return None.
 def discriminator_find_enum_define(expr):
@@ -331,19 +357,26 @@ def discriminator_find_enum_define(expr):
     if not (discriminator and base):
         return None
 
-    base_fields = find_base_fields(base)
-    if not base_fields:
+    base_members = find_base_members(base)
+    if not base_members:
         return None
 
-    discriminator_type = base_fields.get(discriminator)
+    discriminator_type = base_members.get(discriminator)
     if not discriminator_type:
         return None
 
     return find_enum(discriminator_type)
 
-valid_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$')
-def check_name(expr_info, source, name, allow_optional = False,
-               enum_member = False):
+
+# Names must be letters, numbers, -, and _.  They must start with letter,
+# except for downstream extensions which must start with __RFQDN_.
+# Dots are only valid in the downstream extension prefix.
+valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
+                        '[a-zA-Z][a-zA-Z0-9_-]*$')
+
+
+def check_name(expr_info, source, name, allow_optional=False,
+               enum_member=False):
     global valid_name
     membername = name
 
@@ -358,31 +391,39 @@ def check_name(expr_info, source, name, allow_optional = False,
                                 % (source, name))
     # Enum members can start with a digit, because the generated C
     # code always prefixes it with the enum name
-    if enum_member:
-        membername = '_' + membername
-    if not valid_name.match(membername):
+    if enum_member and membername[0].isdigit():
+        membername = 'D' + membername
+    # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
+    # and 'q_obj_*' implicit type names.
+    if not valid_name.match(membername) or \
+       c_name(membername, False).startswith('q_'):
         raise QAPIExprError(expr_info,
                             "%s uses invalid name '%s'" % (source, name))
 
-def add_name(name, info, meta, implicit = False):
+
+def add_name(name, info, meta, implicit=False):
     global all_names
     check_name(info, "'%s'" % meta, name)
+    # FIXME should reject names that differ only in '_' vs. '.'
+    # vs. '-', because they're liable to clash in generated C.
     if name in all_names:
         raise QAPIExprError(info,
                             "%s '%s' is already defined"
                             % (all_names[name], name))
-    if not implicit and name[-4:] == 'Kind':
+    if not implicit and (name.endswith('Kind') or name.endswith('List')):
         raise QAPIExprError(info,
-                            "%s '%s' should not end in 'Kind'"
-                            % (meta, name))
+                            "%s '%s' should not end in '%s'"
+                            % (meta, name, name[-4:]))
     all_names[name] = meta
 
+
 def add_struct(definition, info):
     global struct_types
     name = definition['struct']
     add_name(name, info, 'struct')
     struct_types.append(definition)
 
+
 def find_struct(name):
     global struct_types
     for struct in struct_types:
@@ -390,12 +431,14 @@ def find_struct(name):
             return struct
     return None
 
+
 def add_union(definition, info):
     global union_types
     name = definition['union']
     add_name(name, info, 'union')
     union_types.append(definition)
 
+
 def find_union(name):
     global union_types
     for union in union_types:
@@ -403,11 +446,13 @@ def find_union(name):
             return union
     return None
 
-def add_enum(name, info, enum_values = None, implicit = False):
+
+def add_enum(name, info, enum_values=None, implicit=False):
     global enum_types
     add_name(name, info, 'enum', implicit)
     enum_types.append({"enum_name": name, "enum_values": enum_values})
 
+
 def find_enum(name):
     global enum_types
     for enum in enum_types:
@@ -415,21 +460,19 @@ def find_enum(name):
             return enum
     return None
 
+
 def is_enum(name):
-    return find_enum(name) != None
+    return find_enum(name) is not None
+
 
-def check_type(expr_info, source, value, allow_array = False,
-               allow_dict = False, allow_optional = False,
-               allow_star = False, allow_metas = []):
+def check_type(expr_info, source, value, allow_array=False,
+               allow_dict=False, allow_optional=False,
+               allow_metas=[]):
     global all_names
-    orig_value = value
 
     if value is None:
         return
 
-    if allow_star and value == '**':
-        return
-
     # Check if array type for value is okay
     if isinstance(value, list):
         if not allow_array:
@@ -440,103 +483,79 @@ def check_type(expr_info, source, value, allow_array = False,
                                 "%s: array type must contain single type name"
                                 % source)
         value = value[0]
-        orig_value = "array of %s" %value
 
     # Check if type name for value is okay
     if isinstance(value, str):
-        if value == '**':
-            raise QAPIExprError(expr_info,
-                                "%s uses '**' but did not request 'gen':false"
-                                % source)
-        if not value in all_names:
+        if value not in all_names:
             raise QAPIExprError(expr_info,
                                 "%s uses unknown type '%s'"
-                                % (source, orig_value))
+                                % (source, value))
         if not all_names[value] in allow_metas:
             raise QAPIExprError(expr_info,
                                 "%s cannot use %s type '%s'"
-                                % (source, all_names[value], orig_value))
+                                % (source, all_names[value], value))
         return
 
-    # value is a dictionary, check that each member is okay
-    if not isinstance(value, OrderedDict):
-        raise QAPIExprError(expr_info,
-                            "%s should be a dictionary" % source)
     if not allow_dict:
         raise QAPIExprError(expr_info,
                             "%s should be a type name" % source)
+
+    if not isinstance(value, OrderedDict):
+        raise QAPIExprError(expr_info,
+                            "%s should be a dictionary or type name" % source)
+
+    # value is a dictionary, check that each member is okay
     for (key, arg) in value.items():
         check_name(expr_info, "Member of %s" % source, key,
                    allow_optional=allow_optional)
+        if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
+            raise QAPIExprError(expr_info,
+                                "Member of %s uses reserved name '%s'"
+                                % (source, key))
         # Todo: allow dictionaries to represent default values of
         # an optional argument.
         check_type(expr_info, "Member '%s' of %s" % (key, source), arg,
-                   allow_array=True, allow_star=allow_star,
+                   allow_array=True,
                    allow_metas=['built-in', 'union', 'alternate', 'struct',
                                 'enum'])
 
-def check_member_clash(expr_info, base_name, data, source = ""):
-    base = find_struct(base_name)
-    assert base
-    base_members = base['data']
-    for key in data.keys():
-        if key.startswith('*'):
-            key = key[1:]
-        if key in base_members or "*" + key in base_members:
-            raise QAPIExprError(expr_info,
-                                "Member name '%s'%s clashes with base '%s'"
-                                % (key, source, base_name))
-    if base.get('base'):
-        check_member_clash(expr_info, base['base'], data, source)
 
 def check_command(expr, expr_info):
     name = expr['command']
-    allow_star = expr.has_key('gen')
 
     check_type(expr_info, "'data' for command '%s'" % name,
                expr.get('data'), allow_dict=True, allow_optional=True,
-               allow_metas=['union', 'struct'], allow_star=allow_star)
+               allow_metas=['struct'])
     returns_meta = ['union', 'struct']
     if name in returns_whitelist:
         returns_meta += ['built-in', 'alternate', 'enum']
     check_type(expr_info, "'returns' for command '%s'" % name,
-               expr.get('returns'), allow_array=True, allow_dict=True,
-               allow_optional=True, allow_metas=returns_meta,
-               allow_star=allow_star)
+               expr.get('returns'), allow_array=True,
+               allow_optional=True, allow_metas=returns_meta)
+
 
 def check_event(expr, expr_info):
     global events
     name = expr['event']
-    params = expr.get('data')
 
-    if name.upper() == 'MAX':
-        raise QAPIExprError(expr_info, "Event name 'MAX' cannot be created")
     events.append(name)
     check_type(expr_info, "'data' for event '%s'" % name,
                expr.get('data'), allow_dict=True, allow_optional=True,
-               allow_metas=['union', 'struct'])
+               allow_metas=['struct'])
+
 
 def check_union(expr, expr_info):
     name = expr['union']
     base = expr.get('base')
     discriminator = expr.get('discriminator')
     members = expr['data']
-    values = { 'MAX': '(automatic)' }
-
-    # If the object has a member 'base', its value must name a struct,
-    # and there must be a discriminator.
-    if base is not None:
-        if discriminator is None:
-            raise QAPIExprError(expr_info,
-                                "Union '%s' requires a discriminator to go "
-                                "along with base" %name)
 
     # Two types of unions, determined by discriminator.
 
     # With no discriminator it is a simple union.
     if discriminator is None:
         enum_define = None
-        allow_metas=['built-in', 'union', 'alternate', 'struct', 'enum']
+        allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
         if base is not None:
             raise QAPIExprError(expr_info,
                                 "Simple union '%s' must not have a base"
@@ -544,91 +563,78 @@ def check_union(expr, expr_info):
 
     # Else, it's a flat union.
     else:
-        # The object must have a string member 'base'.
-        if not isinstance(base, str):
+        # The object must have a string or dictionary 'base'.
+        check_type(expr_info, "'base' for union '%s'" % name,
+                   base, allow_dict=True, allow_optional=True,
+                   allow_metas=['struct'])
+        if not base:
             raise QAPIExprError(expr_info,
-                                "Flat union '%s' must have a string base field"
+                                "Flat union '%s' must have a base"
                                 % name)
-        base_fields = find_base_fields(base)
-        if not base_fields:
-            raise QAPIExprError(expr_info,
-                                "Base '%s' is not a valid struct"
-                                % base)
+        base_members = find_base_members(base)
+        assert base_members
 
         # The value of member 'discriminator' must name a non-optional
         # member of the base struct.
         check_name(expr_info, "Discriminator of flat union '%s'" % name,
                    discriminator)
-        discriminator_type = base_fields.get(discriminator)
+        discriminator_type = base_members.get(discriminator)
         if not discriminator_type:
             raise QAPIExprError(expr_info,
                                 "Discriminator '%s' is not a member of base "
                                 "struct '%s'"
                                 % (discriminator, base))
         enum_define = find_enum(discriminator_type)
-        allow_metas=['struct']
+        allow_metas = ['struct']
         # Do not allow string discriminator
         if not enum_define:
             raise QAPIExprError(expr_info,
                                 "Discriminator '%s' must be of enumeration "
                                 "type" % discriminator)
 
-    # Check every branch
+    # Check every branch; don't allow an empty union
+    if len(members) == 0:
+        raise QAPIExprError(expr_info,
+                            "Union '%s' cannot have empty 'data'" % name)
     for (key, value) in members.items():
         check_name(expr_info, "Member of union '%s'" % name, key)
 
-        # Each value must name a known type; furthermore, in flat unions,
-        # branches must be a struct with no overlapping member names
+        # Each value must name a known type
         check_type(expr_info, "Member '%s' of union '%s'" % (key, name),
                    value, allow_array=not base, allow_metas=allow_metas)
-        if base:
-            branch_struct = find_struct(value)
-            assert branch_struct
-            check_member_clash(expr_info, base, branch_struct['data'],
-                               " of branch '%s'" % key)
 
         # If the discriminator names an enum type, then all members
         # of 'data' must also be members of the enum type.
         if enum_define:
-            if not key in enum_define['enum_values']:
+            if key not in enum_define['enum_values']:
                 raise QAPIExprError(expr_info,
                                     "Discriminator value '%s' is not found in "
                                     "enum '%s'" %
                                     (key, enum_define["enum_name"]))
 
-        # Otherwise, check for conflicts in the generated enum
-        else:
-            c_key = camel_to_upper(key)
-            if c_key in values:
-                raise QAPIExprError(expr_info,
-                                    "Union '%s' member '%s' clashes with '%s'"
-                                    % (name, key, values[c_key]))
-            values[c_key] = key
 
 def check_alternate(expr, expr_info):
     name = expr['alternate']
     members = expr['data']
-    values = { 'MAX': '(automatic)' }
     types_seen = {}
 
-    # Check every branch
+    # Check every branch; require at least two branches
+    if len(members) < 2:
+        raise QAPIExprError(expr_info,
+                            "Alternate '%s' should have at least two branches "
+                            "in 'data'" % name)
     for (key, value) in members.items():
         check_name(expr_info, "Member of alternate '%s'" % name, key)
 
-        # Check for conflicts in the generated enum
-        c_key = camel_to_upper(key)
-        if c_key in values:
-            raise QAPIExprError(expr_info,
-                                "Alternate '%s' member '%s' clashes with '%s'"
-                                % (name, key, values[c_key]))
-        values[c_key] = key
-
         # Ensure alternates have no type conflicts.
         check_type(expr_info, "Member '%s' of alternate '%s'" % (key, name),
                    value,
                    allow_metas=['built-in', 'union', 'struct', 'enum'])
         qtype = find_alternate_member_qtype(value)
-        assert qtype
+        if not qtype:
+            raise QAPIExprError(expr_info,
+                                "Alternate '%s' member '%s' cannot use "
+                                "type '%s'" % (name, key, value))
         if qtype in types_seen:
             raise QAPIExprError(expr_info,
                                 "Alternate '%s' member '%s' can't "
@@ -636,23 +642,22 @@ def check_alternate(expr, expr_info):
                                 % (name, key, types_seen[qtype]))
         types_seen[qtype] = key
 
+
 def check_enum(expr, expr_info):
     name = expr['enum']
     members = expr.get('data')
-    values = { 'MAX': '(automatic)' }
+    prefix = expr.get('prefix')
 
     if not isinstance(members, list):
         raise QAPIExprError(expr_info,
                             "Enum '%s' requires an array for 'data'" % name)
+    if prefix is not None and not isinstance(prefix, str):
+        raise QAPIExprError(expr_info,
+                            "Enum '%s' requires a string for 'prefix'" % name)
     for member in members:
-        check_name(expr_info, "Member of enum '%s'" %name, member,
+        check_name(expr_info, "Member of enum '%s'" % name, member,
                    enum_member=True)
-        key = camel_to_upper(member)
-        if key in values:
-            raise QAPIExprError(expr_info,
-                                "Enum '%s' member '%s' clashes with '%s'"
-                                % (name, member, values[key]))
-        values[key] = member
+
 
 def check_struct(expr, expr_info):
     name = expr['struct']
@@ -662,8 +667,7 @@ def check_struct(expr, expr_info):
                allow_dict=True, allow_optional=True)
     check_type(expr_info, "'base' for struct '%s'" % name, expr.get('base'),
                allow_metas=['struct'])
-    if expr.get('base'):
-        check_member_clash(expr_info, expr['base'], expr['data'])
+
 
 def check_keys(expr_elem, meta, required, optional=[]):
     expr = expr_elem['expr']
@@ -672,22 +676,23 @@ def check_keys(expr_elem, meta, required, optional=[]):
     if not isinstance(name, str):
         raise QAPIExprError(info,
                             "'%s' key must have a string value" % meta)
-    required = required + [ meta ]
+    required = required + [meta]
     for (key, value) in expr.items():
-        if not key in required and not key in optional:
+        if key not in required and key not in optional:
             raise QAPIExprError(info,
                                 "Unknown key '%s' in %s '%s'"
                                 % (key, meta, name))
-        if (key == 'gen' or key == 'success-response') and value != False:
+        if (key == 'gen' or key == 'success-response') and value is not False:
             raise QAPIExprError(info,
                                 "'%s' of %s '%s' should only use false value"
                                 % (key, meta, name))
     for key in required:
-        if not expr.has_key(key):
+        if key not in expr:
             raise QAPIExprError(info,
                                 "Key '%s' is missing from %s '%s'"
                                 % (key, meta, name))
 
+
 def check_exprs(exprs):
     global all_names
 
@@ -697,24 +702,24 @@ def check_exprs(exprs):
     for expr_elem in exprs:
         expr = expr_elem['expr']
         info = expr_elem['info']
-        if expr.has_key('enum'):
-            check_keys(expr_elem, 'enum', ['data'])
+        if 'enum' in expr:
+            check_keys(expr_elem, 'enum', ['data'], ['prefix'])
             add_enum(expr['enum'], info, expr['data'])
-        elif expr.has_key('union'):
+        elif 'union' in expr:
             check_keys(expr_elem, 'union', ['data'],
                        ['base', 'discriminator'])
             add_union(expr, info)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             check_keys(expr_elem, 'alternate', ['data'])
             add_name(expr['alternate'], info, 'alternate')
-        elif expr.has_key('struct'):
+        elif 'struct' in expr:
             check_keys(expr_elem, 'struct', ['data'], ['base'])
             add_struct(expr, info)
-        elif expr.has_key('command'):
+        elif 'command' in expr:
             check_keys(expr_elem, 'command', [],
                        ['data', 'returns', 'gen', 'success-response'])
             add_name(expr['command'], info, 'command')
-        elif expr.has_key('event'):
+        elif 'event' in expr:
             check_keys(expr_elem, 'event', [], ['data'])
             add_name(expr['event'], info, 'event')
         else:
@@ -724,11 +729,11 @@ def check_exprs(exprs):
     # Try again for hidden UnionKind enum
     for expr_elem in exprs:
         expr = expr_elem['expr']
-        if expr.has_key('union'):
+        if 'union' in expr:
             if not discriminator_find_enum_define(expr):
                 add_enum('%sKind' % expr['union'], expr_elem['info'],
                          implicit=True)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
                      implicit=True)
 
@@ -737,51 +742,683 @@ def check_exprs(exprs):
         expr = expr_elem['expr']
         info = expr_elem['info']
 
-        if expr.has_key('enum'):
+        if 'enum' in expr:
             check_enum(expr, info)
-        elif expr.has_key('union'):
+        elif 'union' in expr:
             check_union(expr, info)
-        elif expr.has_key('alternate'):
+        elif 'alternate' in expr:
             check_alternate(expr, info)
-        elif expr.has_key('struct'):
+        elif 'struct' in expr:
             check_struct(expr, info)
-        elif expr.has_key('command'):
+        elif 'command' in expr:
             check_command(expr, info)
-        elif expr.has_key('event'):
+        elif 'event' in expr:
             check_event(expr, info)
         else:
             assert False, 'unexpected meta type'
 
-    return map(lambda expr_elem: expr_elem['expr'], exprs)
+    return exprs
 
-def parse_schema(fname):
-    try:
-        schema = QAPISchema(open(fname, "r"))
-        return check_exprs(schema.exprs)
-    except (QAPISchemaError, QAPIExprError), e:
-        print >>sys.stderr, e
-        exit(1)
 
 #
-# Code generation helpers
+# Schema compiler frontend
 #
 
-def parse_args(typeinfo):
-    if isinstance(typeinfo, str):
-        struct = find_struct(typeinfo)
-        assert struct != None
-        typeinfo = struct['data']
+class QAPISchemaEntity(object):
+    def __init__(self, name, info):
+        assert isinstance(name, str)
+        self.name = name
+        # For explicitly defined entities, info points to the (explicit)
+        # definition.  For builtins (and their arrays), info is None.
+        # For implicitly defined entities, info points to a place that
+        # triggered the implicit definition (there may be more than one
+        # such place).
+        self.info = info
+
+    def c_name(self):
+        return c_name(self.name)
+
+    def check(self, schema):
+        pass
+
+    def is_implicit(self):
+        return not self.info
+
+    def visit(self, visitor):
+        pass
+
+
+class QAPISchemaVisitor(object):
+    def visit_begin(self, schema):
+        pass
+
+    def visit_end(self):
+        pass
+
+    def visit_needed(self, entity):
+        # Default to visiting everything
+        return True
+
+    def visit_builtin_type(self, name, info, json_type):
+        pass
+
+    def visit_enum_type(self, name, info, values, prefix):
+        pass
+
+    def visit_array_type(self, name, info, element_type):
+        pass
+
+    def visit_object_type(self, name, info, base, members, variants):
+        pass
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        pass
+
+    def visit_alternate_type(self, name, info, variants):
+        pass
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        pass
 
-    for member in typeinfo:
-        argname = member
-        argentry = typeinfo[member]
+    def visit_event(self, name, info, arg_type):
+        pass
+
+
+class QAPISchemaType(QAPISchemaEntity):
+    # Return the C type for common use.
+    # For the types we commonly box, this is a pointer type.
+    def c_type(self):
+        pass
+
+    # Return the C type to be used in a parameter list.
+    def c_param_type(self):
+        return self.c_type()
+
+    # Return the C type to be used where we suppress boxing.
+    def c_unboxed_type(self):
+        return self.c_type()
+
+    def json_type(self):
+        pass
+
+    def alternate_qtype(self):
+        json2qtype = {
+            'string':  'QTYPE_QSTRING',
+            'number':  'QTYPE_QFLOAT',
+            'int':     'QTYPE_QINT',
+            'boolean': 'QTYPE_QBOOL',
+            'object':  'QTYPE_QDICT'
+        }
+        return json2qtype.get(self.json_type())
+
+
+class QAPISchemaBuiltinType(QAPISchemaType):
+    def __init__(self, name, json_type, c_type):
+        QAPISchemaType.__init__(self, name, None)
+        assert not c_type or isinstance(c_type, str)
+        assert json_type in ('string', 'number', 'int', 'boolean', 'null',
+                             'value')
+        self._json_type_name = json_type
+        self._c_type_name = c_type
+
+    def c_name(self):
+        return self.name
+
+    def c_type(self):
+        return self._c_type_name
+
+    def c_param_type(self):
+        if self.name == 'str':
+            return 'const ' + self._c_type_name
+        return self._c_type_name
+
+    def json_type(self):
+        return self._json_type_name
+
+    def visit(self, visitor):
+        visitor.visit_builtin_type(self.name, self.info, self.json_type())
+
+
+class QAPISchemaEnumType(QAPISchemaType):
+    def __init__(self, name, info, values, prefix):
+        QAPISchemaType.__init__(self, name, info)
+        for v in values:
+            assert isinstance(v, QAPISchemaMember)
+            v.set_owner(name)
+        assert prefix is None or isinstance(prefix, str)
+        self.values = values
+        self.prefix = prefix
+
+    def check(self, schema):
+        seen = {}
+        for v in self.values:
+            v.check_clash(self.info, seen)
+
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_enum_type()
+        return self.name.endswith('Kind')
+
+    def c_type(self):
+        return c_name(self.name)
+
+    def member_names(self):
+        return [v.name for v in self.values]
+
+    def json_type(self):
+        return 'string'
+
+    def visit(self, visitor):
+        visitor.visit_enum_type(self.name, self.info,
+                                self.member_names(), self.prefix)
+
+
+class QAPISchemaArrayType(QAPISchemaType):
+    def __init__(self, name, info, element_type):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(element_type, str)
+        self._element_type_name = element_type
+        self.element_type = None
+
+    def check(self, schema):
+        self.element_type = schema.lookup_type(self._element_type_name)
+        assert self.element_type
+
+    def is_implicit(self):
+        return True
+
+    def c_type(self):
+        return c_name(self.name) + pointer_suffix
+
+    def json_type(self):
+        return 'array'
+
+    def visit(self, visitor):
+        visitor.visit_array_type(self.name, self.info, self.element_type)
+
+
+class QAPISchemaObjectType(QAPISchemaType):
+    def __init__(self, name, info, base, local_members, variants):
+        # struct has local_members, optional base, and no variants
+        # flat union has base, variants, and no local_members
+        # simple union has local_members, variants, and no base
+        QAPISchemaType.__init__(self, name, info)
+        assert base is None or isinstance(base, str)
+        for m in local_members:
+            assert isinstance(m, QAPISchemaObjectTypeMember)
+            m.set_owner(name)
+        if variants is not None:
+            assert isinstance(variants, QAPISchemaObjectTypeVariants)
+            variants.set_owner(name)
+        self._base_name = base
+        self.base = None
+        self.local_members = local_members
+        self.variants = variants
+        self.members = None
+
+    def check(self, schema):
+        if self.members is False:               # check for cycles
+            raise QAPIExprError(self.info,
+                                "Object %s contains itself" % self.name)
+        if self.members:
+            return
+        self.members = False                    # mark as being checked
+        seen = OrderedDict()
+        if self._base_name:
+            self.base = schema.lookup_type(self._base_name)
+            assert isinstance(self.base, QAPISchemaObjectType)
+            self.base.check(schema)
+            self.base.check_clash(schema, self.info, seen)
+        for m in self.local_members:
+            m.check(schema)
+            m.check_clash(self.info, seen)
+        self.members = seen.values()
+        if self.variants:
+            self.variants.check(schema, seen)
+            assert self.variants.tag_member in self.members
+            self.variants.check_clash(schema, self.info, seen)
+
+    # Check that the members of this type do not cause duplicate JSON members,
+    # and update seen to track the members seen so far. Report any errors
+    # on behalf of info, which is not necessarily self.info
+    def check_clash(self, schema, info, seen):
+        assert not self.variants       # not implemented
+        for m in self.members:
+            m.check_clash(info, seen)
+
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_object_type(), as well as
+        # _def_predefineds()
+        return self.name.startswith('q_')
+
+    def c_name(self):
+        return QAPISchemaType.c_name(self)
+
+    def c_type(self):
+        assert not self.is_implicit()
+        return c_name(self.name) + pointer_suffix
+
+    def c_unboxed_type(self):
+        return c_name(self.name)
+
+    def json_type(self):
+        return 'object'
+
+    def visit(self, visitor):
+        visitor.visit_object_type(self.name, self.info,
+                                  self.base, self.local_members, self.variants)
+        visitor.visit_object_type_flat(self.name, self.info,
+                                       self.members, self.variants)
+
+
+class QAPISchemaMember(object):
+    role = 'member'
+
+    def __init__(self, name):
+        assert isinstance(name, str)
+        self.name = name
+        self.owner = None
+
+    def set_owner(self, name):
+        assert not self.owner
+        self.owner = name
+
+    def check_clash(self, info, seen):
+        cname = c_name(self.name)
+        if cname.lower() != cname and self.owner not in case_whitelist:
+            raise QAPIExprError(info,
+                                "%s should not use uppercase" % self.describe())
+        if cname in seen:
+            raise QAPIExprError(info,
+                                "%s collides with %s"
+                                % (self.describe(), seen[cname].describe()))
+        seen[cname] = self
+
+    def _pretty_owner(self):
+        owner = self.owner
+        if owner.startswith('q_obj_'):
+            # See QAPISchema._make_implicit_object_type() - reverse the
+            # mapping there to create a nice human-readable description
+            owner = owner[6:]
+            if owner.endswith('-arg'):
+                return '(parameter of %s)' % owner[:-4]
+            elif owner.endswith('-base'):
+                return '(base of %s)' % owner[:-5]
+            else:
+                assert owner.endswith('-wrapper')
+                # Unreachable and not implemented
+                assert False
+        if owner.endswith('Kind'):
+            # See QAPISchema._make_implicit_enum_type()
+            return '(branch of %s)' % owner[:-4]
+        return '(%s of %s)' % (self.role, owner)
+
+    def describe(self):
+        return "'%s' %s" % (self.name, self._pretty_owner())
+
+
+class QAPISchemaObjectTypeMember(QAPISchemaMember):
+    def __init__(self, name, typ, optional):
+        QAPISchemaMember.__init__(self, name)
+        assert isinstance(typ, str)
+        assert isinstance(optional, bool)
+        self._type_name = typ
+        self.type = None
+        self.optional = optional
+
+    def check(self, schema):
+        assert self.owner
+        self.type = schema.lookup_type(self._type_name)
+        assert self.type
+
+
+class QAPISchemaObjectTypeVariants(object):
+    def __init__(self, tag_name, tag_member, variants):
+        # Flat unions pass tag_name but not tag_member.
+        # Simple unions and alternates pass tag_member but not tag_name.
+        # After check(), tag_member is always set, and tag_name remains
+        # a reliable witness of being used by a flat union.
+        assert bool(tag_member) != bool(tag_name)
+        assert (isinstance(tag_name, str) or
+                isinstance(tag_member, QAPISchemaObjectTypeMember))
+        assert len(variants) > 0
+        for v in variants:
+            assert isinstance(v, QAPISchemaObjectTypeVariant)
+        self.tag_name = tag_name
+        self.tag_member = tag_member
+        self.variants = variants
+
+    def set_owner(self, name):
+        for v in self.variants:
+            v.set_owner(name)
+
+    def check(self, schema, seen):
+        if not self.tag_member:    # flat union
+            self.tag_member = seen[c_name(self.tag_name)]
+            assert self.tag_name == self.tag_member.name
+        assert isinstance(self.tag_member.type, QAPISchemaEnumType)
+        for v in self.variants:
+            v.check(schema)
+            # Union names must match enum values; alternate names are
+            # checked separately. Use 'seen' to tell the two apart.
+            if seen:
+                assert v.name in self.tag_member.type.member_names()
+                assert isinstance(v.type, QAPISchemaObjectType)
+                v.type.check(schema)
+
+    def check_clash(self, schema, info, seen):
+        for v in self.variants:
+            # Reset seen map for each variant, since qapi names from one
+            # branch do not affect another branch
+            assert isinstance(v.type, QAPISchemaObjectType)
+            v.type.check_clash(schema, info, dict(seen))
+
+
+class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
+    role = 'branch'
+
+    def __init__(self, name, typ):
+        QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
+
+
+class QAPISchemaAlternateType(QAPISchemaType):
+    def __init__(self, name, info, variants):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(variants, QAPISchemaObjectTypeVariants)
+        assert not variants.tag_name
+        variants.set_owner(name)
+        variants.tag_member.set_owner(self.name)
+        self.variants = variants
+
+    def check(self, schema):
+        self.variants.tag_member.check(schema)
+        # Not calling self.variants.check_clash(), because there's nothing
+        # to clash with
+        self.variants.check(schema, {})
+        # Alternate branch names have no relation to the tag enum values;
+        # so we have to check for potential name collisions ourselves.
+        seen = {}
+        for v in self.variants.variants:
+            v.check_clash(self.info, seen)
+
+    def c_type(self):
+        return c_name(self.name) + pointer_suffix
+
+    def json_type(self):
+        return 'value'
+
+    def visit(self, visitor):
+        visitor.visit_alternate_type(self.name, self.info, self.variants)
+
+
+class QAPISchemaCommand(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type, ret_type, gen, success_response):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        assert not ret_type or isinstance(ret_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+        self._ret_type_name = ret_type
+        self.ret_type = None
+        self.gen = gen
+        self.success_response = success_response
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert isinstance(self.arg_type, QAPISchemaObjectType)
+            assert not self.arg_type.variants   # not implemented
+        if self._ret_type_name:
+            self.ret_type = schema.lookup_type(self._ret_type_name)
+            assert isinstance(self.ret_type, QAPISchemaType)
+
+    def visit(self, visitor):
+        visitor.visit_command(self.name, self.info,
+                              self.arg_type, self.ret_type,
+                              self.gen, self.success_response)
+
+
+class QAPISchemaEvent(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert isinstance(self.arg_type, QAPISchemaObjectType)
+            assert not self.arg_type.variants   # not implemented
+
+    def visit(self, visitor):
+        visitor.visit_event(self.name, self.info, self.arg_type)
+
+
+class QAPISchema(object):
+    def __init__(self, fname):
+        try:
+            self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs)
+            self._entity_dict = {}
+            self._predefining = True
+            self._def_predefineds()
+            self._predefining = False
+            self._def_exprs()
+            self.check()
+        except (QAPISchemaError, QAPIExprError) as err:
+            print >>sys.stderr, err
+            exit(1)
+
+    def _def_entity(self, ent):
+        # Only the predefined types are allowed to not have info
+        assert ent.info or self._predefining
+        assert ent.name not in self._entity_dict
+        self._entity_dict[ent.name] = ent
+
+    def lookup_entity(self, name, typ=None):
+        ent = self._entity_dict.get(name)
+        if typ and not isinstance(ent, typ):
+            return None
+        return ent
+
+    def lookup_type(self, name):
+        return self.lookup_entity(name, QAPISchemaType)
+
+    def _def_builtin_type(self, name, json_type, c_type):
+        self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
+        # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
+        # qapi-types.h from a single .c, all arrays of builtins must be
+        # declared in the first file whether or not they are used.  Nicer
+        # would be to use lazy instantiation, while figuring out how to
+        # avoid compilation issues with multiple qapi-types.h.
+        self._make_array_type(name, None)
+
+    def _def_predefineds(self):
+        for t in [('str',    'string',  'char' + pointer_suffix),
+                  ('number', 'number',  'double'),
+                  ('int',    'int',     'int64_t'),
+                  ('int8',   'int',     'int8_t'),
+                  ('int16',  'int',     'int16_t'),
+                  ('int32',  'int',     'int32_t'),
+                  ('int64',  'int',     'int64_t'),
+                  ('uint8',  'int',     'uint8_t'),
+                  ('uint16', 'int',     'uint16_t'),
+                  ('uint32', 'int',     'uint32_t'),
+                  ('uint64', 'int',     'uint64_t'),
+                  ('size',   'int',     'uint64_t'),
+                  ('bool',   'boolean', 'bool'),
+                  ('any',    'value',   'QObject' + pointer_suffix)]:
+            self._def_builtin_type(*t)
+        self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
+                                                          None, [], None)
+        self._def_entity(self.the_empty_object_type)
+        qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
+                                                'qstring', 'qdict', 'qlist',
+                                                'qfloat', 'qbool'])
+        self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
+                                            'QTYPE'))
+
+    def _make_enum_members(self, values):
+        return [QAPISchemaMember(v) for v in values]
+
+    def _make_implicit_enum_type(self, name, info, values):
+        # See also QAPISchemaObjectTypeMember._pretty_owner()
+        name = name + 'Kind'   # Use namespace reserved by add_name()
+        self._def_entity(QAPISchemaEnumType(
+            name, info, self._make_enum_members(values), None))
+        return name
+
+    def _make_array_type(self, element_type, info):
+        name = element_type + 'List'   # Use namespace reserved by add_name()
+        if not self.lookup_type(name):
+            self._def_entity(QAPISchemaArrayType(name, info, element_type))
+        return name
+
+    def _make_implicit_object_type(self, name, info, role, members):
+        if not members:
+            return None
+        # See also QAPISchemaObjectTypeMember._pretty_owner()
+        name = 'q_obj_%s-%s' % (name, role)
+        if not self.lookup_entity(name, QAPISchemaObjectType):
+            self._def_entity(QAPISchemaObjectType(name, info, None,
+                                                  members, None))
+        return name
+
+    def _def_enum_type(self, expr, info):
+        name = expr['enum']
+        data = expr['data']
+        prefix = expr.get('prefix')
+        self._def_entity(QAPISchemaEnumType(
+            name, info, self._make_enum_members(data), prefix))
+
+    def _make_member(self, name, typ, info):
         optional = False
-        if member.startswith('*'):
-            argname = member[1:]
+        if name.startswith('*'):
+            name = name[1:]
             optional = True
-        # Todo: allow argentry to be OrderedDict, for providing the
-        # value of an optional argument.
-        yield (argname, argentry, optional)
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        return QAPISchemaObjectTypeMember(name, typ, optional)
+
+    def _make_members(self, data, info):
+        return [self._make_member(key, value, info)
+                for (key, value) in data.iteritems()]
+
+    def _def_struct_type(self, expr, info):
+        name = expr['struct']
+        base = expr.get('base')
+        data = expr['data']
+        self._def_entity(QAPISchemaObjectType(name, info, base,
+                                              self._make_members(data, info),
+                                              None))
+
+    def _make_variant(self, case, typ):
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _make_simple_variant(self, case, typ, info):
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        typ = self._make_implicit_object_type(
+            typ, info, 'wrapper', [self._make_member('data', typ, info)])
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _def_union_type(self, expr, info):
+        name = expr['union']
+        data = expr['data']
+        base = expr.get('base')
+        tag_name = expr.get('discriminator')
+        tag_member = None
+        if isinstance(base, dict):
+            base = (self._make_implicit_object_type(
+                    name, info, 'base', self._make_members(base, info)))
+        if tag_name:
+            variants = [self._make_variant(key, value)
+                        for (key, value) in data.iteritems()]
+            members = []
+        else:
+            variants = [self._make_simple_variant(key, value, info)
+                        for (key, value) in data.iteritems()]
+            typ = self._make_implicit_enum_type(name, info,
+                                                [v.name for v in variants])
+            tag_member = QAPISchemaObjectTypeMember('type', typ, False)
+            members = [tag_member]
+        self._def_entity(
+            QAPISchemaObjectType(name, info, base, members,
+                                 QAPISchemaObjectTypeVariants(tag_name,
+                                                              tag_member,
+                                                              variants)))
+
+    def _def_alternate_type(self, expr, info):
+        name = expr['alternate']
+        data = expr['data']
+        variants = [self._make_variant(key, value)
+                    for (key, value) in data.iteritems()]
+        tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
+        self._def_entity(
+            QAPISchemaAlternateType(name, info,
+                                    QAPISchemaObjectTypeVariants(None,
+                                                                 tag_member,
+                                                                 variants)))
+
+    def _def_command(self, expr, info):
+        name = expr['command']
+        data = expr.get('data')
+        rets = expr.get('returns')
+        gen = expr.get('gen', True)
+        success_response = expr.get('success-response', True)
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        if isinstance(rets, list):
+            assert len(rets) == 1
+            rets = self._make_array_type(rets[0], info)
+        self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
+                                           success_response))
+
+    def _def_event(self, expr, info):
+        name = expr['event']
+        data = expr.get('data')
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        self._def_entity(QAPISchemaEvent(name, info, data))
+
+    def _def_exprs(self):
+        for expr_elem in self.exprs:
+            expr = expr_elem['expr']
+            info = expr_elem['info']
+            if 'enum' in expr:
+                self._def_enum_type(expr, info)
+            elif 'struct' in expr:
+                self._def_struct_type(expr, info)
+            elif 'union' in expr:
+                self._def_union_type(expr, info)
+            elif 'alternate' in expr:
+                self._def_alternate_type(expr, info)
+            elif 'command' in expr:
+                self._def_command(expr, info)
+            elif 'event' in expr:
+                self._def_event(expr, info)
+            else:
+                assert False
+
+    def check(self):
+        for ent in self._entity_dict.values():
+            ent.check(self)
+
+    def visit(self, visitor):
+        visitor.visit_begin(self)
+        for (name, entity) in sorted(self._entity_dict.items()):
+            if visitor.visit_needed(entity):
+                entity.visit(visitor)
+        visitor.visit_end()
+
+
+#
+# Code generation helpers
+#
 
 def camel_case(name):
     new_name = ''
@@ -796,6 +1433,7 @@ def camel_case(name):
             new_name += ch.lower()
     return new_name
 
+
 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
 # ENUM24_Name -> ENUM24_NAME
@@ -810,19 +1448,22 @@ def camel_to_upper(value):
         c = c_fun_str[i]
         # When c is upper and no "_" appears before, do more checks
         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
-            # Case 1: next string is lower
-            # Case 2: previous string is digit
-            if (i < (l - 1) and c_fun_str[i + 1].islower()) or \
-            c_fun_str[i - 1].isdigit():
+            if i < l - 1 and c_fun_str[i + 1].islower():
+                new_name += '_'
+            elif c_fun_str[i - 1].isdigit():
                 new_name += '_'
         new_name += c
     return new_name.lstrip('_').upper()
 
-def c_enum_const(type_name, const_name):
-    return camel_to_upper(type_name + '_' + const_name)
+
+def c_enum_const(type_name, const_name, prefix=None):
+    if prefix is not None:
+        type_name = prefix
+    return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
 
 c_name_trans = string.maketrans('.-', '__')
 
+
 # Map @name to a valid C identifier.
 # If @protect, avoid returning certain ticklish identifiers (like
 # C keywords) by prepending "q_".
@@ -835,15 +1476,16 @@ c_name_trans = string.maketrans('.-', '__')
 def c_name(name, protect=True):
     # ANSI X3J11/88-090, 3.1.1
     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
-                     'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
-                     'for', 'goto', 'if', 'int', 'long', 'register', 'return',
-                     'short', 'signed', 'sizeof', 'static', 'struct', 'switch',
-                     'typedef', 'union', 'unsigned', 'void', 'volatile', 'while'])
+                     'default', 'do', 'double', 'else', 'enum', 'extern',
+                     'float', 'for', 'goto', 'if', 'int', 'long', 'register',
+                     'return', 'short', 'signed', 'sizeof', 'static',
+                     'struct', 'switch', 'typedef', 'union', 'unsigned',
+                     'void', 'volatile', 'while'])
     # ISO/IEC 9899:1999, 6.4.1
     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
     # ISO/IEC 9899:2011, 6.4.1
-    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', '_Noreturn',
-                     '_Static_assert', '_Thread_local'])
+    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
+                     '_Noreturn', '_Static_assert', '_Thread_local'])
     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
     # excluding _.*
     gcc_words = set(['asm', 'typeof'])
@@ -858,109 +1500,58 @@ def c_name(name, protect=True):
                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
     # namespace pollution:
-    polluted_words = set(['unix', 'errno'])
-    if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words):
+    polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
+    name = name.translate(c_name_trans)
+    if protect and (name in c89_words | c99_words | c11_words | gcc_words
+                    | cpp_words | polluted_words):
         return "q_" + name
-    return name.translate(c_name_trans)
-
-# Map type @name to the C typedef name for the list form.
-#
-# ['Name'] -> 'NameList', ['x-Foo'] -> 'x_FooList', ['int'] -> 'intList'
-def c_list_type(name):
-    return type_name(name) + 'List'
-
-# Map type @value to the C typedef form.
-#
-# Used for converting 'type' from a 'member':'type' qapi definition
-# into the alphanumeric portion of the type for a generated C parameter,
-# as well as generated C function names.  See c_type() for the rest of
-# the conversion such as adding '*' on pointer types.
-# 'int' -> 'int', '[x-Foo]' -> 'x_FooList', '__a.b_c' -> '__a_b_c'
-def type_name(value):
-    if type(value) == list:
-        return c_list_type(value[0])
-    if value in builtin_types.keys():
-        return value
-    return c_name(value)
+    return name
 
 eatspace = '\033EATSPACE.'
 pointer_suffix = ' *' + eatspace
 
-# Map type @name to its C type expression.
-# If @is_param, const-qualify the string type.
-#
-# This function is used for computing the full C type of 'member':'name'.
-# A special suffix is added in c_type() for pointer types, and it's
-# stripped in mcgen(). So please notice this when you check the return
-# value of c_type() outside mcgen().
-def c_type(value, is_param=False):
-    if value == 'str':
-        if is_param:
-            return 'const char' + pointer_suffix
-        return 'char' + pointer_suffix
-
-    elif value == 'int':
-        return 'int64_t'
-    elif (value == 'int8' or value == 'int16' or value == 'int32' or
-          value == 'int64' or value == 'uint8' or value == 'uint16' or
-          value == 'uint32' or value == 'uint64'):
-        return value + '_t'
-    elif value == 'size':
-        return 'uint64_t'
-    elif value == 'bool':
-        return 'bool'
-    elif value == 'number':
-        return 'double'
-    elif type(value) == list:
-        return c_list_type(value[0]) + pointer_suffix
-    elif is_enum(value):
-        return c_name(value)
-    elif value == None:
-        return 'void'
-    elif value in events:
-        return camel_case(value) + 'Event' + pointer_suffix
-    else:
-        # complex type name
-        assert isinstance(value, str) and value != ""
-        return c_name(value) + pointer_suffix
-
-def is_c_ptr(value):
-    return c_type(value).endswith(pointer_suffix)
 
 def genindent(count):
     ret = ""
-    for i in range(count):
+    for _ in range(count):
         ret += " "
     return ret
 
 indent_level = 0
 
+
 def push_indent(indent_amount=4):
     global indent_level
     indent_level += indent_amount
 
+
 def pop_indent(indent_amount=4):
     global indent_level
     indent_level -= indent_amount
 
+
+# Generate @code with @kwds interpolated.
+# Obey indent_level, and strip eatspace.
 def cgen(code, **kwds):
-    indent = genindent(indent_level)
-    lines = code.split('\n')
-    lines = map(lambda x: indent + x, lines)
-    return '\n'.join(lines) % kwds + '\n'
+    raw = code % kwds
+    if indent_level:
+        indent = genindent(indent_level)
+        # re.subn() lacks flags support before Python 2.7, use re.compile()
+        raw = re.subn(re.compile("^.", re.MULTILINE),
+                      indent + r'\g<0>', raw)
+        raw = raw[0]
+    return re.sub(re.escape(eatspace) + ' *', '', raw)
+
 
 def mcgen(code, **kwds):
-    raw = cgen('\n'.join(code.split('\n')[1:-1]), **kwds)
-    return re.sub(re.escape(eatspace) + ' *', '', raw)
+    if code[0] == '\n':
+        code = code[1:]
+    return cgen(code, **kwds)
 
-def basename(filename):
-    return filename.split("/")[-1]
 
 def guardname(filename):
-    guard = basename(filename).rsplit(".", 1)[0]
-    for substr in [".", " ", "-"]:
-        guard = guard.replace(substr, "_")
-    return guard.upper() + '_H'
+    return c_name(filename, protect=False).upper()
+
 
 def guardstart(name):
     return mcgen('''
@@ -971,6 +1562,7 @@ def guardstart(name):
 ''',
                  name=guardname(name))
 
+
 def guardend(name):
     return mcgen('''
 
@@ -979,18 +1571,99 @@ def guardend(name):
 ''',
                  name=guardname(name))
 
+
+def gen_enum_lookup(name, values, prefix=None):
+    ret = mcgen('''
+
+const char *const %(c_name)s_lookup[] = {
+''',
+                c_name=c_name(name))
+    for value in values:
+        index = c_enum_const(name, value, prefix)
+        ret += mcgen('''
+    [%(index)s] = "%(value)s",
+''',
+                     index=index, value=value)
+
+    max_index = c_enum_const(name, '_MAX', prefix)
+    ret += mcgen('''
+    [%(max_index)s] = NULL,
+};
+''',
+                 max_index=max_index)
+    return ret
+
+
+def gen_enum(name, values, prefix=None):
+    # append automatically generated _MAX value
+    enum_values = values + ['_MAX']
+
+    ret = mcgen('''
+
+typedef enum %(c_name)s {
+''',
+                c_name=c_name(name))
+
+    i = 0
+    for value in enum_values:
+        ret += mcgen('''
+    %(c_enum)s = %(i)d,
+''',
+                     c_enum=c_enum_const(name, value, prefix),
+                     i=i)
+        i += 1
+
+    ret += mcgen('''
+} %(c_name)s;
+''',
+                 c_name=c_name(name))
+
+    ret += mcgen('''
+
+extern const char *const %(c_name)s_lookup[];
+''',
+                 c_name=c_name(name))
+    return ret
+
+
+def gen_params(arg_type, extra):
+    if not arg_type:
+        return extra
+    assert not arg_type.variants
+    ret = ''
+    sep = ''
+    for memb in arg_type.members:
+        ret += sep
+        sep = ', '
+        if memb.optional:
+            ret += 'bool has_%s, ' % c_name(memb.name)
+        ret += '%s %s' % (memb.type.c_param_type(), c_name(memb.name))
+    if extra:
+        ret += sep + extra
+    return ret
+
+
+def gen_err_check():
+    return mcgen('''
+    if (err) {
+        goto out;
+    }
+''')
+
+
 #
 # Common command line parsing
 #
 
-def parse_command_line(extra_options = "", extra_long_options = []):
+
+def parse_command_line(extra_options="", extra_long_options=[]):
 
     try:
         opts, args = getopt.gnu_getopt(sys.argv[1:],
                                        "chp:o:" + extra_options,
                                        ["source", "header", "prefix=",
                                         "output-dir="] + extra_long_options)
-    except getopt.GetoptError, err:
+    except getopt.GetoptError as err:
         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
         sys.exit(1)
 
@@ -1003,6 +1676,12 @@ def parse_command_line(extra_options = "", extra_long_options = []):
     for oa in opts:
         o, a = oa
         if o in ("-p", "--prefix"):
+            match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
+            if match.end() != len(a):
+                print >>sys.stderr, \
+                    "%s: 'funny character '%s' in argument of --prefix" \
+                    % (sys.argv[0], a[match.end()])
+                sys.exit(1)
             prefix = a
         elif o in ("-o", "--output-dir"):
             output_dir = a + "/"
@@ -1028,16 +1707,19 @@ def parse_command_line(extra_options = "", extra_long_options = []):
 # Generate output files with boilerplate
 #
 
+
 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
                 c_comment, h_comment):
+    guard = guardname(prefix + h_file)
     c_file = output_dir + prefix + c_file
     h_file = output_dir + prefix + h_file
 
-    try:
-        os.makedirs(output_dir)
-    except os.error, e:
-        if e.errno != errno.EEXIST:
-            raise
+    if output_dir:
+        try:
+            os.makedirs(output_dir)
+        except os.error as e:
+            if e.errno != errno.EEXIST:
+                raise
 
     def maybe_open(really, name, opt):
         if really:
@@ -1053,7 +1735,7 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
 %(comment)s
 ''',
-                     comment = c_comment))
+                     comment=c_comment))
 
     fdecl.write(mcgen('''
 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
@@ -1062,10 +1744,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
 #define %(guard)s
 
 ''',
-                      comment = h_comment, guard = guardname(h_file)))
+                      comment=h_comment, guard=guard))
 
     return (fdef, fdecl)
 
+
 def close_output(fdef, fdecl):
     fdecl.write('''
 #endif
diff --git a/qemu/scripts/qemu-gdb.py b/qemu/scripts/qemu-gdb.py
index 6c7f4fbe5..b3f8e04f7 100644
--- a/qemu/scripts/qemu-gdb.py
+++ b/qemu/scripts/qemu-gdb.py
@@ -13,73 +13,20 @@
 # Contributions after 2012-01-13 are licensed under the terms of the
 # GNU GPL, version 2 or (at your option) any later version.
 
+# Usage:
+# At the (gdb) prompt, type "source scripts/qemu-gdb.py".
+# "help qemu" should then list the supported QEMU debug support commands.
 
 import gdb
 
-def isnull(ptr):
-    return ptr == gdb.Value(0).cast(ptr.type)
+import os, sys
 
-def int128(p):
-    return long(p['lo']) + (long(p['hi']) << 64)
+# Annoyingly, gdb doesn't put the directory of scripts onto the
+# module search path. Do it manually.
 
-def get_fs_base():
-    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS)'''
-    # %rsp - 120 is scratch space according to the SystemV ABI
-    old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
-    gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
-    fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
-    gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
-    return fs_base
+sys.path.append(os.path.dirname(__file__))
 
-def get_glibc_pointer_guard():
-    '''Fetch glibc pointer guard value'''
-    fs_base = get_fs_base()
-    return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
-
-def glibc_ptr_demangle(val, pointer_guard):
-    '''Undo effect of glibc's PTR_MANGLE()'''
-    return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
-
-def bt_jmpbuf(jmpbuf):
-    '''Backtrace a jmpbuf'''
-    JB_RBX  = 0
-    JB_RBP  = 1
-    JB_R12  = 2
-    JB_R13  = 3
-    JB_R14  = 4
-    JB_R15  = 5
-    JB_RSP  = 6
-    JB_PC   = 7
-
-    old_rbx = gdb.parse_and_eval('(uint64_t)$rbx')
-    old_rbp = gdb.parse_and_eval('(uint64_t)$rbp')
-    old_rsp = gdb.parse_and_eval('(uint64_t)$rsp')
-    old_r12 = gdb.parse_and_eval('(uint64_t)$r12')
-    old_r13 = gdb.parse_and_eval('(uint64_t)$r13')
-    old_r14 = gdb.parse_and_eval('(uint64_t)$r14')
-    old_r15 = gdb.parse_and_eval('(uint64_t)$r15')
-    old_rip = gdb.parse_and_eval('(uint64_t)$rip')
-
-    pointer_guard = get_glibc_pointer_guard()
-    gdb.execute('set $rbx = %s' % jmpbuf[JB_RBX])
-    gdb.execute('set $rbp = %s' % glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard))
-    gdb.execute('set $rsp = %s' % glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard))
-    gdb.execute('set $r12 = %s' % jmpbuf[JB_R12])
-    gdb.execute('set $r13 = %s' % jmpbuf[JB_R13])
-    gdb.execute('set $r14 = %s' % jmpbuf[JB_R14])
-    gdb.execute('set $r15 = %s' % jmpbuf[JB_R15])
-    gdb.execute('set $rip = %s' % glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard))
-
-    gdb.execute('bt')
-
-    gdb.execute('set $rbx = %s' % old_rbx)
-    gdb.execute('set $rbp = %s' % old_rbp)
-    gdb.execute('set $rsp = %s' % old_rsp)
-    gdb.execute('set $r12 = %s' % old_r12)
-    gdb.execute('set $r13 = %s' % old_r13)
-    gdb.execute('set $r14 = %s' % old_r14)
-    gdb.execute('set $r15 = %s' % old_r15)
-    gdb.execute('set $rip = %s' % old_rip)
+from qemugdb import aio, mtree, coroutine
 
 class QemuCommand(gdb.Command):
     '''Prefix for QEMU debug support commands'''
@@ -87,78 +34,14 @@ class QemuCommand(gdb.Command):
         gdb.Command.__init__(self, 'qemu', gdb.COMMAND_DATA,
                              gdb.COMPLETE_NONE, True)
 
-class CoroutineCommand(gdb.Command):
-    '''Display coroutine backtrace'''
-    def __init__(self):
-        gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA,
-                             gdb.COMPLETE_NONE)
-
-    def invoke(self, arg, from_tty):
-        argv = gdb.string_to_argv(arg)
-        if len(argv) != 1:
-            gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
-            return
-
-        coroutine_pointer = gdb.parse_and_eval(argv[0]).cast(gdb.lookup_type('CoroutineUContext').pointer())
-        bt_jmpbuf(coroutine_pointer['env']['__jmpbuf'])
+QemuCommand()
+coroutine.CoroutineCommand()
+mtree.MtreeCommand()
+aio.HandlersCommand()
 
-class MtreeCommand(gdb.Command):
-    '''Display the memory tree hierarchy'''
-    def __init__(self):
-        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
-                             gdb.COMPLETE_NONE)
-        self.queue = []
-    def invoke(self, arg, from_tty):
-        self.seen = set()
-        self.queue_root('address_space_memory')
-        self.queue_root('address_space_io')
-        self.process_queue()
-    def queue_root(self, varname):
-        ptr = gdb.parse_and_eval(varname)['root']
-        self.queue.append(ptr)
-    def process_queue(self):
-        while self.queue:
-            ptr = self.queue.pop(0)
-            if long(ptr) in self.seen:
-                continue
-            self.print_item(ptr)
-    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
-        self.seen.add(long(ptr))
-        addr = ptr['addr']
-        addr += offset
-        size = int128(ptr['size'])
-        alias = ptr['alias']
-        klass = ''
-        if not isnull(alias):
-            klass = ' (alias)'
-        elif not isnull(ptr['ops']):
-            klass = ' (I/O)'
-        elif bool(ptr['ram']):
-            klass = ' (RAM)'
-        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
-                  % ('  ' * level,
-                     long(addr),
-                     long(addr + (size - 1)),
-                     ptr['name'].string(),
-                     klass,
-                     ptr,
-                     ),
-                  gdb.STDOUT)
-        if not isnull(alias):
-            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
-                      ('  ' * level,
-                       alias['name'].string(),
-                       ptr['alias_offset'],
-                       alias,
-                       ),
-                      gdb.STDOUT)
-            self.queue.append(alias)
-        subregion = ptr['subregions']['tqh_first']
-        level += 1
-        while not isnull(subregion):
-            self.print_item(subregion, addr, level)
-            subregion = subregion['subregions_link']['tqe_next']
+coroutine.CoroutineSPFunction()
+coroutine.CoroutinePCFunction()
 
-QemuCommand()
-CoroutineCommand()
-MtreeCommand()
+# Default to silently passing through SIGUSR1, because QEMU sends it
+# to itself a lot.
+gdb.execute('handle SIGUSR1 pass noprint nostop')
diff --git a/qemu/scripts/qemugdb/__init__.py b/qemu/scripts/qemugdb/__init__.py
new file mode 100644
index 000000000..969f552b2
--- /dev/null
+++ b/qemu/scripts/qemugdb/__init__.py
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright (c) 2015 Linaro Ltd
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see
+# <http://www.gnu.org/licenses/gpl-2.0.html>
+#
+
+# We don't need to do anything in our init file currently.
+
+"""
+Support routines for debugging QEMU under GDB
+"""
+
+__license__    = "GPL version 2 or (at your option) any later version"
diff --git a/qemu/scripts/qemugdb/aio.py b/qemu/scripts/qemugdb/aio.py
new file mode 100644
index 000000000..2ba00c444
--- /dev/null
+++ b/qemu/scripts/qemugdb/aio.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+# GDB debugging support: aio/iohandler debug
+#
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Author: Dr. David Alan Gilbert <dgilbert@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+#
+
+import gdb
+from qemugdb import coroutine
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def dump_aiocontext(context, verbose):
+    '''Display a dump and backtrace for an aiocontext'''
+    cur = context['aio_handlers']['lh_first']
+    # Get pointers to functions we're going to process specially
+    sym_fd_coroutine_enter = gdb.parse_and_eval('fd_coroutine_enter')
+
+    while not isnull(cur):
+        entry = cur.dereference()
+        gdb.write('----\n%s\n' % entry)
+        if verbose and cur['io_read'] == sym_fd_coroutine_enter:
+            coptr = (cur['opaque'].cast(gdb.lookup_type('FDYieldUntilData').pointer()))['co']
+            coptr = coptr.cast(gdb.lookup_type('CoroutineUContext').pointer())
+            coroutine.bt_jmpbuf(coptr['env']['__jmpbuf'])
+        cur = cur['node']['le_next'];
+
+    gdb.write('----\n')
+
+class HandlersCommand(gdb.Command):
+    '''Display aio handlers'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu handlers', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        verbose = False
+        argv = gdb.string_to_argv(arg)
+
+        if len(argv) > 0 and argv[0] == '--verbose':
+            verbose = True
+            argv.pop(0)
+
+        if len(argv) > 1:
+            gdb.write('usage: qemu handlers [--verbose] [handler]\n')
+            return
+
+        if len(argv) == 1:
+            handlers_name = argv[0]
+        else:
+            handlers_name = 'qemu_aio_context'
+        dump_aiocontext(gdb.parse_and_eval(handlers_name), verbose)
diff --git a/qemu/scripts/qemugdb/coroutine.py b/qemu/scripts/qemugdb/coroutine.py
new file mode 100644
index 000000000..ab699794a
--- /dev/null
+++ b/qemu/scripts/qemugdb/coroutine.py
@@ -0,0 +1,119 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+import gdb
+
+VOID_PTR = gdb.lookup_type('void').pointer()
+
+def get_fs_base():
+    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS).  This is
+       pthread_self().'''
+    # %rsp - 120 is scratch space according to the SystemV ABI
+    old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
+    fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
+    return fs_base
+
+def pthread_self():
+    '''Fetch pthread_self() from the glibc start_thread function.'''
+    f = gdb.newest_frame()
+    while f.name() != 'start_thread':
+        f = f.older()
+        if f is None:
+            return get_fs_base()
+
+    try:
+        return f.read_var("arg")
+    except ValueError:
+        return get_fs_base()
+
+def get_glibc_pointer_guard():
+    '''Fetch glibc pointer guard value'''
+    fs_base = pthread_self()
+    return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
+
+def glibc_ptr_demangle(val, pointer_guard):
+    '''Undo effect of glibc's PTR_MANGLE()'''
+    return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
+
+def get_jmpbuf_regs(jmpbuf):
+    JB_RBX  = 0
+    JB_RBP  = 1
+    JB_R12  = 2
+    JB_R13  = 3
+    JB_R14  = 4
+    JB_R15  = 5
+    JB_RSP  = 6
+    JB_PC   = 7
+
+    pointer_guard = get_glibc_pointer_guard()
+    return {'rbx': jmpbuf[JB_RBX],
+        'rbp': glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard),
+        'rsp': glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard),
+        'r12': jmpbuf[JB_R12],
+        'r13': jmpbuf[JB_R13],
+        'r14': jmpbuf[JB_R14],
+        'r15': jmpbuf[JB_R15],
+        'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) }
+
+def bt_jmpbuf(jmpbuf):
+    '''Backtrace a jmpbuf'''
+    regs = get_jmpbuf_regs(jmpbuf)
+    old = dict()
+
+    for i in regs:
+        old[i] = gdb.parse_and_eval('(uint64_t)$%s' % i)
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, regs[i]))
+
+    gdb.execute('bt')
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, old[i]))
+
+def coroutine_to_jmpbuf(co):
+    coroutine_pointer = co.cast(gdb.lookup_type('CoroutineUContext').pointer())
+    return coroutine_pointer['env']['__jmpbuf']
+
+
+class CoroutineCommand(gdb.Command):
+    '''Display coroutine backtrace'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        argv = gdb.string_to_argv(arg)
+        if len(argv) != 1:
+            gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
+            return
+
+        bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0])))
+
+class CoroutineSPFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_sp')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rsp'].cast(VOID_PTR)
+
+class CoroutinePCFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_pc')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rip'].cast(VOID_PTR)
diff --git a/qemu/scripts/qemugdb/mtree.py b/qemu/scripts/qemugdb/mtree.py
new file mode 100644
index 000000000..cc8131c2e
--- /dev/null
+++ b/qemu/scripts/qemugdb/mtree.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+# 'qemu mtree' -- display the memory hierarchy
+
+import gdb
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def int128(p):
+    return int(p['lo']) + (int(p['hi']) << 64)
+
+class MtreeCommand(gdb.Command):
+    '''Display the memory tree hierarchy'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+        self.queue = []
+    def invoke(self, arg, from_tty):
+        self.seen = set()
+        self.queue_root('address_space_memory')
+        self.queue_root('address_space_io')
+        self.process_queue()
+    def queue_root(self, varname):
+        ptr = gdb.parse_and_eval(varname)['root']
+        self.queue.append(ptr)
+    def process_queue(self):
+        while self.queue:
+            ptr = self.queue.pop(0)
+            if int(ptr) in self.seen:
+                continue
+            self.print_item(ptr)
+    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
+        self.seen.add(int(ptr))
+        addr = ptr['addr']
+        addr += offset
+        size = int128(ptr['size'])
+        alias = ptr['alias']
+        klass = ''
+        if not isnull(alias):
+            klass = ' (alias)'
+        elif not isnull(ptr['ops']):
+            klass = ' (I/O)'
+        elif bool(ptr['ram']):
+            klass = ' (RAM)'
+        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
+                  % ('  ' * level,
+                     int(addr),
+                     int(addr + (size - 1)),
+                     ptr['name'].string(),
+                     klass,
+                     ptr,
+                     ),
+                  gdb.STDOUT)
+        if not isnull(alias):
+            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
+                      ('  ' * level,
+                       alias['name'].string(),
+                       ptr['alias_offset'],
+                       alias,
+                       ),
+                      gdb.STDOUT)
+            self.queue.append(alias)
+        subregion = ptr['subregions']['tqh_first']
+        level += 1
+        while not isnull(subregion):
+            self.print_item(subregion, addr, level)
+            subregion = subregion['subregions_link']['tqe_next']
+
diff --git a/qemu/scripts/qmp/qemu-ga-client b/qemu/scripts/qmp/qemu-ga-client
index 9908f2109..fd056056f 100755
--- a/qemu/scripts/qmp/qemu-ga-client
+++ b/qemu/scripts/qmp/qemu-ga-client
@@ -259,7 +259,7 @@ def main(address, cmd, args):
 
     try:
         client = QemuGuestAgentClient(address)
-    except QemuGuestAgent.error, e:
+    except QemuGuestAgent.error as e:
         import errno
 
         print(e)
diff --git a/qemu/scripts/qmp/qmp b/qemu/scripts/qmp/qmp
index 1db3c7ffe..514b539a6 100755
--- a/qemu/scripts/qmp/qmp
+++ b/qemu/scripts/qmp/qmp
@@ -91,8 +91,8 @@ def main(args):
         try:
             os.environ['QMP_PATH'] = path
             os.execvp(fullcmd, [fullcmd] + args)
-        except OSError, (errno, msg):
-            if errno == 2:
+        except OSError as exc:
+            if exc.errno == 2:
                 print 'Command "%s" not found.' % (fullcmd)
                 return 1
             raise
diff --git a/qemu/scripts/qmp/qmp-shell b/qemu/scripts/qmp/qmp-shell
index 65280d29d..0373b24b2 100755
--- a/qemu/scripts/qmp/qmp-shell
+++ b/qemu/scripts/qmp/qmp-shell
@@ -29,13 +29,47 @@
 # (QEMU) device_add driver=e1000 id=net1
 # {u'return': {}}
 # (QEMU)
+#
+# key=value pairs also support Python or JSON object literal subset notations,
+# without spaces. Dictionaries/objects {} are supported as are arrays [].
+#
+#    example-command arg-name1={'key':'value','obj'={'prop':"value"}}
+#
+# Both JSON and Python formatting should work, including both styles of
+# string literal quotes. Both paradigms of literal values should work,
+# including null/true/false for JSON and None/True/False for Python.
+#
+#
+# Transactions have the following multi-line format:
+#
+#    transaction(
+#    action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    ...
+#    action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    )
+#
+# One line transactions are also supported:
+#
+#    transaction( action-name1 ... )
+#
+# For example:
+#
+#     (QEMU) transaction(
+#     TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1
+#     TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0
+#     TRANS> )
+#     {"return": {}}
+#     (QEMU)
+#
+# Use the -v and -p options to activate the verbose and pretty-print options,
+# which will echo back the properly formatted JSON-compliant QMP that is being
+# sent to QEMU, which is useful for debugging and documentation generation.
 
 import qmp
 import json
 import ast
 import readline
 import sys
-import pprint
 
 class QMPCompleter(list):
     def complete(self, text, state):
@@ -68,11 +102,11 @@ class FuzzyJSON(ast.NodeTransformer):
 # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and
 #       _execute_cmd()). Let's design a better one.
 class QMPShell(qmp.QEMUMonitorProtocol):
-    def __init__(self, address, pp=None):
+    def __init__(self, address, pretty=False):
         qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address))
         self._greeting = None
         self._completer = None
-        self._pp = pp
+        self._pretty = pretty
         self._transmode = False
         self._actions = list()
 
@@ -196,16 +230,16 @@ class QMPShell(qmp.QEMUMonitorProtocol):
         return qmpcmd
 
     def _print(self, qmp):
-        jsobj = json.dumps(qmp)
-        if self._pp is not None:
-            self._pp.pprint(jsobj)
-        else:
-            print str(jsobj)
+        indent = None
+        if self._pretty:
+            indent = 4
+        jsobj = json.dumps(qmp, indent=indent)
+        print str(jsobj)
 
     def _execute_cmd(self, cmdline):
         try:
             qmpcmd = self.__build_cmd(cmdline)
-        except Exception, e:
+        except Exception as e:
             print 'Error while parsing command line: %s' % e
             print 'command format: <command-name> ',
             print '[arg-name1=arg1] ... [arg-nameN=argN]'
@@ -342,7 +376,7 @@ def main():
     addr = ''
     qemu = None
     hmp = False
-    pp = None
+    pretty = False
     verbose = False
 
     try:
@@ -352,9 +386,7 @@ def main():
                     fail_cmdline(arg)
                 hmp = True
             elif arg == "-p":
-                if pp is not None:
-                    fail_cmdline(arg)
-                pp = pprint.PrettyPrinter(indent=4)
+                pretty = True
             elif arg == "-v":
                 verbose = True
             else:
@@ -363,7 +395,7 @@ def main():
                 if hmp:
                     qemu = HMPShell(arg)
                 else:
-                    qemu = QMPShell(arg, pp)
+                    qemu = QMPShell(arg, pretty)
                 addr = arg
 
         if qemu is None:
diff --git a/qemu/scripts/qmp/qmp.py b/qemu/scripts/qmp/qmp.py
index 1d38e3e9e..779332f32 100644
--- a/qemu/scripts/qmp/qmp.py
+++ b/qemu/scripts/qmp/qmp.py
@@ -92,7 +92,7 @@ class QEMUMonitorProtocol:
         self.__sock.setblocking(0)
         try:
             self.__json_read()
-        except socket.error, err:
+        except socket.error as err:
             if err[0] == errno.EAGAIN:
                 # No data available
                 pass
@@ -150,7 +150,7 @@ class QEMUMonitorProtocol:
         """
         try:
             self.__sock.sendall(json.dumps(qmp_cmd))
-        except socket.error, err:
+        except socket.error as err:
             if err[0] == errno.EPIPE:
                 return
             raise socket.error(err)
diff --git a/qemu/scripts/texi2pod.pl b/qemu/scripts/texi2pod.pl
index 94097fb06..8767662d3 100755
--- a/qemu/scripts/texi2pod.pl
+++ b/qemu/scripts/texi2pod.pl
@@ -317,7 +317,7 @@ while(<$inf>) {
 	@columns = ();
 	for $column (split (/\s*\@tab\s*/, $1)) {
 	    # @strong{...} is used a @headitem work-alike
-	    $column =~ s/^\@strong{(.*)}$/$1/;
+	    $column =~ s/^\@strong\{(.*)\}$/$1/;
 	    push @columns, $column;
 	}
 	$_ = "\n=item ".join (" : ", @columns)."\n";
diff --git a/qemu/scripts/tracetool.py b/qemu/scripts/tracetool.py
index 83bde7bda..7b82959e8 100755
--- a/qemu/scripts/tracetool.py
+++ b/qemu/scripts/tracetool.py
@@ -71,7 +71,7 @@ def main(args):
 
     try:
         opts, args = getopt.getopt(args[1:], "", long_opts)
-    except getopt.GetoptError, err:
+    except getopt.GetoptError as err:
         error_opt(str(err))
 
     check_backends = False
@@ -132,7 +132,7 @@ def main(args):
     try:
         tracetool.generate(sys.stdin, arg_format, arg_backends,
                            binary=binary, probe_prefix=probe_prefix)
-    except tracetool.TracetoolError, e:
+    except tracetool.TracetoolError as e:
         error_opt(str(e))
 
 if __name__ == "__main__":
diff --git a/qemu/scripts/tracetool/__init__.py b/qemu/scripts/tracetool/__init__.py
index 181675f00..be24039c5 100644
--- a/qemu/scripts/tracetool/__init__.py
+++ b/qemu/scripts/tracetool/__init__.py
@@ -6,7 +6,7 @@ Machinery for generating tracing-related intermediate files.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -50,9 +50,14 @@ class Arguments:
         Parameters
         ----------
         args :
-            List of (type, name) tuples.
+            List of (type, name) tuples or Arguments objects.
         """
-        self._args = args
+        self._args = []
+        for arg in args:
+            if isinstance(arg, Arguments):
+                self._args.extend(arg._args)
+            else:
+                self._args.append(arg)
 
     def copy(self):
         """Create a new copy."""
@@ -83,6 +88,12 @@ class Arguments:
             res.append((arg_type, identifier))
         return Arguments(res)
 
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            return Arguments(self._args[index])
+        else:
+            return self._args[index]
+
     def __iter__(self):
         """Iterate over the (type, name) pairs."""
         return iter(self._args)
@@ -110,6 +121,10 @@ class Arguments:
         """List of argument types."""
         return [ type_ for type_, _ in self._args ]
 
+    def casted(self):
+        """List of argument names casted to their type."""
+        return ["(%s)%s" % (type_, name) for type_, name in self._args]
+
     def transform(self, *trans):
         """Return a new Arguments instance with transformed types.
 
@@ -146,9 +161,10 @@ class Event(object):
                       "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
                       "\s*")
 
-    _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec"])
+    _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"])
 
-    def __init__(self, name, props, fmt, args, orig=None):
+    def __init__(self, name, props, fmt, args, orig=None,
+                 event_trans=None, event_exec=None):
         """
         Parameters
         ----------
@@ -161,13 +177,19 @@ class Event(object):
         args : Arguments
             Event arguments.
         orig : Event or None
-            Original Event before transformation.
+            Original Event before transformation/generation.
+        event_trans : Event or None
+            Generated translation-time event ("tcg" property).
+        event_exec : Event or None
+            Generated execution-time event ("tcg" property).
 
         """
         self.name = name
         self.properties = props
         self.fmt = fmt
         self.args = args
+        self.event_trans = event_trans
+        self.event_exec = event_exec
 
         if orig is None:
             self.original = weakref.ref(self)
@@ -183,7 +205,7 @@ class Event(object):
     def copy(self):
         """Create a new copy."""
         return Event(self.name, list(self.properties), self.fmt,
-                     self.args.copy(), self)
+                     self.args.copy(), self, self.event_trans, self.event_exec)
 
     @staticmethod
     def build(line_str):
@@ -215,7 +237,13 @@ class Event(object):
         if "tcg" in props and isinstance(fmt, str):
             raise ValueError("Events with 'tcg' property must have two formats")
 
-        return Event(name, props, fmt, args)
+        event = Event(name, props, fmt, args)
+
+        # add implicit arguments when using the 'vcpu' property
+        import tracetool.vcpu
+        event = tracetool.vcpu.transform_event(event)
+
+        return event
 
     def __repr__(self):
         """Evaluable string representation for this object."""
@@ -270,6 +298,7 @@ def _read_events(fobj):
             event_trans.name += "_trans"
             event_trans.properties += ["tcg-trans"]
             event_trans.fmt = event.fmt[0]
+            # ignore TCG arguments
             args_trans = []
             for atrans, aorig in zip(
                     event_trans.transform(tracetool.transform.TCG_2_HOST).args,
@@ -277,13 +306,12 @@ def _read_events(fobj):
                 if atrans == aorig:
                     args_trans.append(atrans)
             event_trans.args = Arguments(args_trans)
-            event_trans = event_trans.copy()
 
             event_exec = event.copy()
             event_exec.name += "_exec"
             event_exec.properties += ["tcg-exec"]
             event_exec.fmt = event.fmt[1]
-            event_exec = event_exec.transform(tracetool.transform.TCG_2_HOST)
+            event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST)
 
             new_event = [event_trans, event_exec]
             event.event_trans, event.event_exec = new_event
diff --git a/qemu/scripts/tracetool/backend/stderr.py b/qemu/scripts/tracetool/backend/log.py
index ca5805462..e409b7326 100644
--- a/qemu/scripts/tracetool/backend/stderr.py
+++ b/qemu/scripts/tracetool/backend/log.py
@@ -20,11 +20,8 @@ PUBLIC = True
 
 
 def generate_h_begin(events):
-    out('#include <stdio.h>',
-        '#include <sys/time.h>',
-        '#include <sys/types.h>',
-        '#include <unistd.h>',
-        '#include "trace/control.h"',
+    out('#include "trace/control.h"',
+        '#include "qemu/log.h"',
         '')
 
 
@@ -36,10 +33,10 @@ def generate_h(event):
     out('    if (trace_event_get_state(%(event_id)s)) {',
         '        struct timeval _now;',
         '        gettimeofday(&_now, NULL);',
-        '        fprintf(stderr, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",',
-        '                        getpid(),',
-        '                        (size_t)_now.tv_sec, (size_t)_now.tv_usec',
-        '                        %(argnames)s);',
+        '        qemu_log_mask(LOG_TRACE, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",',
+        '                      getpid(),',
+        '                      (size_t)_now.tv_sec, (size_t)_now.tv_usec',
+        '                      %(argnames)s);',
         '    }',
         event_id="TRACE_" + event.name.upper(),
         name=event.name,
diff --git a/qemu/scripts/tracetool/backend/simple.py b/qemu/scripts/tracetool/backend/simple.py
index e8c2cd57e..3246c2001 100644
--- a/qemu/scripts/tracetool/backend/simple.py
+++ b/qemu/scripts/tracetool/backend/simple.py
@@ -42,7 +42,8 @@ def generate_h(event):
 
 
 def generate_c_begin(events):
-    out('#include "trace.h"',
+    out('#include "qemu/osdep.h"',
+        '#include "trace.h"',
         '#include "trace/control.h"',
         '#include "trace/simple.h"',
         '')
diff --git a/qemu/scripts/tracetool/format/events_c.py b/qemu/scripts/tracetool/format/events_c.py
index 2d97fa310..1cc6a49a7 100644
--- a/qemu/scripts/tracetool/format/events_c.py
+++ b/qemu/scripts/tracetool/format/events_c.py
@@ -19,6 +19,7 @@ from tracetool import out
 def generate(events, backend):
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
         '#include "trace.h"',
         '#include "trace/generated-events.h"',
         '#include "trace/control.h"',
@@ -27,7 +28,7 @@ def generate(events, backend):
     out('TraceEvent trace_events[TRACE_EVENT_COUNT] = {')
 
     for e in events:
-        out('    { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s, .dstate = 0 },',
+        out('    { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s },',
             id = "TRACE_" + e.name.upper(),
             name = e.name,
             sstate = "TRACE_%s_ENABLED" % e.name.upper())
diff --git a/qemu/scripts/tracetool/format/events_h.py b/qemu/scripts/tracetool/format/events_h.py
index 9f114a349..4529263e0 100644
--- a/qemu/scripts/tracetool/format/events_h.py
+++ b/qemu/scripts/tracetool/format/events_h.py
@@ -6,7 +6,7 @@ trace/generated-events.h
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -21,8 +21,6 @@ def generate(events, backend):
         '',
         '#ifndef TRACE__GENERATED_EVENTS_H',
         '#define TRACE__GENERATED_EVENTS_H',
-        '',
-        '#include <stdbool.h>',
         '')
 
     # event identifiers
@@ -43,7 +41,7 @@ def generate(events, backend):
         if "tcg-trans" in e.properties:
             # a single define for the two "sub-events"
             out('#define TRACE_%(name)s_ENABLED %(enabled)d',
-                name=e.original.original.name.upper(),
+                name=e.original.name.upper(),
                 enabled=enabled)
         out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled))
 
diff --git a/qemu/scripts/tracetool/format/h.py b/qemu/scripts/tracetool/format/h.py
index 9b3943002..083540621 100644
--- a/qemu/scripts/tracetool/format/h.py
+++ b/qemu/scripts/tracetool/format/h.py
@@ -6,7 +6,7 @@ trace/generated-tracers.h
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
diff --git a/qemu/scripts/tracetool/format/tcg_h.py b/qemu/scripts/tracetool/format/tcg_h.py
index f676b6662..e2331f251 100644
--- a/qemu/scripts/tracetool/format/tcg_h.py
+++ b/qemu/scripts/tracetool/format/tcg_h.py
@@ -6,14 +6,25 @@ Generate .h file for TCG code generation.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
 __email__      = "stefanha@linux.vnet.ibm.com"
 
 
-from tracetool import out
+from tracetool import out, Arguments
+import tracetool.vcpu
+
+
+def vcpu_transform_args(args):
+    assert len(args) == 1
+    return Arguments([
+        args,
+        # NOTE: this name must be kept in sync with the one in "tcg_h"
+        # NOTE: Current helper code uses TCGv_env (CPUArchState*)
+        ("TCGv_env", "__tcg_" + args.names()[0]),
+    ])
 
 
 def generate(events, backend):
@@ -23,8 +34,6 @@ def generate(events, backend):
         '#ifndef TRACE__GENERATED_TCG_TRACERS_H',
         '#define TRACE__GENERATED_TCG_TRACERS_H',
         '',
-        '#include <stdint.h>',
-        '',
         '#include "trace.h"',
         '#include "exec/helper-proto.h"',
         '',
@@ -35,21 +44,21 @@ def generate(events, backend):
         if "tcg-trans" not in e.properties:
             continue
 
-        # get the original event definition
-        e = e.original.original
-
         out('static inline void %(name_tcg)s(%(args)s)',
             '{',
-            name_tcg=e.api(e.QEMU_TRACE_TCG),
-            args=e.args)
+            name_tcg=e.original.api(e.QEMU_TRACE_TCG),
+            args=tracetool.vcpu.transform_args("tcg_h", e.original))
 
         if "disable" not in e.properties:
+            args_trans = e.original.event_trans.args
+            args_exec = tracetool.vcpu.transform_args(
+                "tcg_helper_c", e.original.event_exec, "wrapper")
             out('    %(name_trans)s(%(argnames_trans)s);',
                 '    gen_helper_%(name_exec)s(%(argnames_exec)s);',
-                name_trans=e.event_trans.api(e.QEMU_TRACE),
-                name_exec=e.event_exec.api(e.QEMU_TRACE),
-                argnames_trans=", ".join(e.event_trans.args.names()),
-                argnames_exec=", ".join(e.event_exec.args.names()))
+                name_trans=e.original.event_trans.api(e.QEMU_TRACE),
+                name_exec=e.original.event_exec.api(e.QEMU_TRACE),
+                argnames_trans=", ".join(args_trans.names()),
+                argnames_exec=", ".join(args_exec.names()))
 
         out('}')
 
diff --git a/qemu/scripts/tracetool/format/tcg_helper_c.py b/qemu/scripts/tracetool/format/tcg_helper_c.py
index 96655a059..a089b0bf0 100644
--- a/qemu/scripts/tracetool/format/tcg_helper_c.py
+++ b/qemu/scripts/tracetool/format/tcg_helper_c.py
@@ -6,15 +6,38 @@ Generate trace/generated-helpers.c.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
 __email__      = "stefanha@linux.vnet.ibm.com"
 
 
-from tracetool import out
+from tracetool import Arguments, out
 from tracetool.transform import *
+import tracetool.vcpu
+
+
+def vcpu_transform_args(args, mode):
+    assert len(args) == 1
+    # NOTE: this name must be kept in sync with the one in "tcg_h"
+    args = Arguments([(args.types()[0], "__tcg_" + args.names()[0])])
+    if mode == "code":
+        return Arguments([
+            # Does cast from helper requirements to tracing types
+            ("CPUState *", "ENV_GET_CPU(%s)" % args.names()[0]),
+        ])
+    else:
+        args = Arguments([
+            # NOTE: Current helper code uses TCGv_env (CPUArchState*)
+            ("CPUArchState *", args.names()[0]),
+        ])
+        if mode == "header":
+            return args
+        elif mode == "wrapper":
+            return args.transform(HOST_2_TCG)
+        else:
+            assert False
 
 
 def generate(events, backend):
@@ -23,6 +46,7 @@ def generate(events, backend):
 
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
         '#include "qemu-common.h"',
         '#include "trace.h"',
         '#include "exec/helper-proto.h"',
@@ -33,18 +57,18 @@ def generate(events, backend):
         if "tcg-exec" not in e.properties:
             continue
 
-        # tracetool.generate always transforms types to host
-        e_args = e.original.args
-
-        values = ["(%s)%s" % (t, n)
-                  for t, n in e.args.transform(TCG_2_TCG_HELPER_DEF)]
+        e_args_api = tracetool.vcpu.transform_args(
+            "tcg_helper_c", e.original, "header").transform(
+                HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF)
+        e_args_call = tracetool.vcpu.transform_args(
+            "tcg_helper_c", e, "code")
 
-        out('void %(name_tcg)s(%(args)s)',
+        out('void %(name_tcg)s(%(args_api)s)',
             '{',
-            '    %(name)s(%(values)s);',
+            '    %(name)s(%(args_call)s);',
             '}',
             name_tcg="helper_%s_proxy" % e.api(),
             name=e.api(),
-            args=e_args.transform(HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF),
-            values=", ".join(values),
+            args_api=e_args_api,
+            args_call=", ".join(e_args_call.casted()),
             )
diff --git a/qemu/scripts/tracetool/format/tcg_helper_h.py b/qemu/scripts/tracetool/format/tcg_helper_h.py
index a8ba7ba8e..dc76c15eb 100644
--- a/qemu/scripts/tracetool/format/tcg_helper_h.py
+++ b/qemu/scripts/tracetool/format/tcg_helper_h.py
@@ -6,7 +6,7 @@ Generate trace/generated-helpers.h.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -15,6 +15,7 @@ __email__      = "stefanha@linux.vnet.ibm.com"
 
 from tracetool import out
 from tracetool.transform import *
+import tracetool.vcpu
 
 
 def generate(events, backend):
@@ -29,11 +30,9 @@ def generate(events, backend):
         if "tcg-exec" not in e.properties:
             continue
 
-        # tracetool.generate always transforms types to host
-        e_args = e.original.args
-
         # TCG helper proxy declaration
         fmt = "DEF_HELPER_FLAGS_%(argc)d(%(name)s, %(flags)svoid%(types)s)"
+        e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "header")
         args = e_args.transform(HOST_2_TCG_COMPAT, HOST_2_TCG,
                                 TCG_2_TCG_HELPER_DECL)
         types = ", ".join(args.types())
diff --git a/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py b/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py
index cac5a878f..020f4422a 100644
--- a/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py
+++ b/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py
@@ -6,7 +6,7 @@ Generate trace/generated-helpers-wrappers.h.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -15,6 +15,7 @@ __email__      = "stefanha@linux.vnet.ibm.com"
 
 from tracetool import out
 from tracetool.transform import *
+import tracetool.vcpu
 
 
 def generate(events, backend):
@@ -33,7 +34,7 @@ def generate(events, backend):
             continue
 
         # tracetool.generate always transforms types to host
-        e_args = e.original.args
+        e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "wrapper")
 
         # mixed-type to TCG helper bridge
         args_tcg_compat = e_args.transform(HOST_2_TCG_COMPAT)
diff --git a/qemu/scripts/tracetool/format/ust_events_c.py b/qemu/scripts/tracetool/format/ust_events_c.py
index bc970936b..9967c7a82 100644
--- a/qemu/scripts/tracetool/format/ust_events_c.py
+++ b/qemu/scripts/tracetool/format/ust_events_c.py
@@ -22,6 +22,8 @@ def generate(events, backend):
 
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
+        '',
         '#define TRACEPOINT_DEFINE',
         '#define TRACEPOINT_CREATE_PROBES',
         '',
diff --git a/qemu/scripts/tracetool/transform.py b/qemu/scripts/tracetool/transform.py
index fc5e679ed..e18b05315 100644
--- a/qemu/scripts/tracetool/transform.py
+++ b/qemu/scripts/tracetool/transform.py
@@ -6,7 +6,7 @@ Type-transformation rules.
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -98,6 +98,7 @@ HOST_2_TCG = {
     "uint32_t": "TCGv_i32",
     "uint64_t": "TCGv_i64",
     "void *"  : "TCGv_ptr",
+    "CPUArchState *": "TCGv_env",
     None: _host_2_tcg,
     }
 
@@ -130,6 +131,7 @@ TCG_2_TCG_HELPER_DECL = {
     "TCGv_ptr": "ptr",
     "TCGv_i32": "i32",
     "TCGv_i64": "i64",
+    "TCGv_env": "env",
     None: _tcg_2_tcg_helper_decl_error,
     }
 
diff --git a/qemu/scripts/tracetool/vcpu.py b/qemu/scripts/tracetool/vcpu.py
new file mode 100644
index 000000000..452c7f589
--- /dev/null
+++ b/qemu/scripts/tracetool/vcpu.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generic management for the 'vcpu' property.
+
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2016, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import Arguments, try_import
+
+
+def transform_event(event):
+    """Transform event to comply with the 'vcpu' property (if present)."""
+    if "vcpu" in event.properties:
+        # events with 'tcg-trans' and 'tcg-exec' are auto-generated from
+        # already-patched events
+        assert "tcg-trans" not in event.properties
+        assert "tcg-exec" not in event.properties
+
+        event.args = Arguments([("CPUState *", "__cpu"), event.args])
+        if "tcg" in event.properties:
+            fmt = "\"cpu=%p \""
+            event.fmt = [fmt + event.fmt[0],
+                         fmt + event.fmt[1]]
+        else:
+            fmt = "\"cpu=%p \""
+            event.fmt = fmt + event.fmt
+    return event
+
+
+def transform_args(format, event, *args, **kwargs):
+    """Transforms the arguments to suit the specified format.
+
+    The format module must implement function 'vcpu_args', which receives the
+    implicit arguments added by the 'vcpu' property, and must return suitable
+    arguments for the given format.
+
+    The function is only called for events with the 'vcpu' property.
+
+    Parameters
+    ==========
+    format : str
+        Format module name.
+    event : Event
+    args, kwargs
+        Passed to 'vcpu_transform_args'.
+
+    Returns
+    =======
+    Arguments
+        The transformed arguments, including the non-implicit ones.
+
+    """
+    if "vcpu" in event.properties:
+        ok, func = try_import("tracetool.format." + format,
+                              "vcpu_transform_args")
+        assert ok
+        assert func
+        return Arguments([func(event.args[:1], *args, **kwargs),
+                          event.args[1:]])
+    else:
+        return event.args
diff --git a/qemu/scripts/update-acpi.sh b/qemu/scripts/update-acpi.sh
deleted file mode 100644
index b5f05ff3c..000000000
--- a/qemu/scripts/update-acpi.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-cd x86_64-softmmu
-for file in hw/i386/*.hex; do
-    cp -f $file ../$file.generated
-done
diff --git a/qemu/scripts/update-linux-headers.sh b/qemu/scripts/update-linux-headers.sh
index f0e830c2d..f7d62d974 100755
--- a/qemu/scripts/update-linux-headers.sh
+++ b/qemu/scripts/update-linux-headers.sh
@@ -28,38 +28,35 @@ if [ -z "$output" ]; then
     output="$PWD"
 fi
 
-cp_virtio() {
-    from=$1
+cp_portable() {
+    f=$1
     to=$2
-    virtio=$(find "$from" -name '*virtio*h' -o -name "input.h" -o -name "pci_regs.h")
-    if [ "$virtio" ]; then
-        rm -rf "$to"
-        mkdir -p "$to"
-        for f in $virtio; do
-            if
-                grep '#include' "$f" | grep -v -e 'linux/virtio' \
-                                             -e 'linux/types' \
-                                             -e 'linux/if_ether' \
-                                             -e 'sys/' \
-                                             > /dev/null
-            then
-                echo "Unexpected #include in input file $f".
-                exit 2
-            fi
-
-            header=$(basename "$f");
-            sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \
-                -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \
-                -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \
-                -e 's/__bitwise__//' \
-                -e 's/__attribute__((packed))/QEMU_PACKED/' \
-                -e 's/__inline__/inline/' \
-                -e '/sys\/ioctl.h/d' \
-                "$f" > "$to/$header";
-        done
+    if
+        grep '#include' "$f" | grep -v -e 'linux/virtio' \
+                                     -e 'linux/types' \
+                                     -e 'stdint' \
+                                     -e 'linux/if_ether' \
+                                     -e 'input-event-codes' \
+                                     -e 'sys/' \
+                                     > /dev/null
+    then
+        echo "Unexpected #include in input file $f".
+        exit 2
     fi
+
+    header=$(basename "$f");
+    sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \
+        -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/"\(input-event-codes\.h\)"/"standard-headers\/linux\/\1"/' \
+        -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \
+        -e 's/__bitwise__//' \
+        -e 's/__attribute__((packed))/QEMU_PACKED/' \
+        -e 's/__inline__/inline/' \
+        -e '/sys\/ioctl.h/d' \
+        -e 's/SW_MAX/SW_MAX_/' \
+        "$f" > "$to/$header";
 }
 
 # This will pick up non-directories too (eg "Kconfig") but we will
@@ -74,7 +71,7 @@ for arch in $ARCHLIST; do
     fi
 
     # Blacklist architectures which have KVM headers but are actually dead
-    if [ "$arch" = "ia64" ]; then
+    if [ "$arch" = "ia64" -o "$arch" = "mips" ]; then
         continue
     fi
 
@@ -82,23 +79,31 @@ for arch in $ARCHLIST; do
 
     rm -rf "$output/linux-headers/asm-$arch"
     mkdir -p "$output/linux-headers/asm-$arch"
-    for header in kvm.h kvm_para.h; do
+    for header in kvm.h kvm_para.h unistd.h; do
         cp "$tmpdir/include/asm/$header" "$output/linux-headers/asm-$arch"
     done
-    if [ $arch = x86 ]; then
-        cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86"
-    fi
     if [ $arch = powerpc ]; then
         cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/"
     fi
 
-    cp_virtio "$tmpdir/include/asm" "$output/include/standard-headers/asm-$arch"
+    rm -rf "$output/include/standard-headers/asm-$arch"
+    mkdir -p "$output/include/standard-headers/asm-$arch"
+    if [ $arch = s390 ]; then
+        cp_portable "$tmpdir/include/asm/kvm_virtio.h" "$output/include/standard-headers/asm-s390/"
+        cp_portable "$tmpdir/include/asm/virtio-ccw.h" "$output/include/standard-headers/asm-s390/"
+    fi
+    if [ $arch = x86 ]; then
+        cp_portable "$tmpdir/include/asm/hyperv.h" "$output/include/standard-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_x32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_64.h" "$output/linux-headers/asm-x86/"
+    fi
 done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
 for header in kvm.h kvm_para.h vfio.h vhost.h \
-              psci.h; do
+              psci.h userfaultfd.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done
 rm -rf "$output/linux-headers/asm-generic"
@@ -112,6 +117,9 @@ else
     cp "$linux/COPYING" "$output/linux-headers"
 fi
 
+cat <<EOF >$output/linux-headers/asm-x86/hyperv.h
+#include "standard-headers/asm-x86/hyperv.h"
+EOF
 cat <<EOF >$output/linux-headers/linux/virtio_config.h
 #include "standard-headers/linux/virtio_config.h"
 EOF
@@ -119,11 +127,18 @@ cat <<EOF >$output/linux-headers/linux/virtio_ring.h
 #include "standard-headers/linux/virtio_ring.h"
 EOF
 
-cp_virtio "$tmpdir/include/linux/" "$output/include/standard-headers/linux"
+rm -rf "$output/include/standard-headers/linux"
+mkdir -p "$output/include/standard-headers/linux"
+for i in "$tmpdir"/include/linux/*virtio*.h "$tmpdir/include/linux/input.h" \
+         "$tmpdir/include/linux/input-event-codes.h" \
+         "$tmpdir/include/linux/pci_regs.h"; do
+    cp_portable "$i" "$output/include/standard-headers/linux"
+done
 
 cat <<EOF >$output/include/standard-headers/linux/types.h
-#include <stdint.h>
-#include "qemu/compiler.h"
+/* For QEMU all types are already defined via osdep.h, so this
+ * header does not need to do anything.
+ */
 EOF
 cat <<EOF >$output/include/standard-headers/linux/if_ether.h
 #define ETH_ALEN    6
diff --git a/qemu/scripts/vmstate-static-checker.py b/qemu/scripts/vmstate-static-checker.py
index b6c0bbead..b5ecaf644 100755
--- a/qemu/scripts/vmstate-static-checker.py
+++ b/qemu/scripts/vmstate-static-checker.py
@@ -99,6 +99,7 @@ def get_changed_sec_name(sec):
     # Section names can change -- see commit 292b1634 for an example.
     changes = {
         "ICH9 LPC": "ICH9-LPC",
+        "e1000-82540em": "e1000",
     }
 
     for item in changes:
diff --git a/qemu/slirp/Makefile.objs b/qemu/slirp/Makefile.objs
index 2daa9dc58..6748e4f60 100644
--- a/qemu/slirp/Makefile.objs
+++ b/qemu/slirp/Makefile.objs
@@ -1,3 +1,5 @@
-common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o dnssearch.o
+common-obj-y = cksum.o if.o ip_icmp.o ip6_icmp.o ip6_input.o ip6_output.o \
+               ip_input.o ip_output.o dnssearch.o
 common-obj-y += slirp.o mbuf.o misc.o sbuf.o socket.o tcp_input.o tcp_output.o
-common-obj-y += tcp_subr.o tcp_timer.o udp.o bootp.o tftp.o arp_table.o
+common-obj-y += tcp_subr.o tcp_timer.o udp.o udp6.o bootp.o tftp.o arp_table.o \
+                ndp_table.o
diff --git a/qemu/slirp/arp_table.c b/qemu/slirp/arp_table.c
index bcaeb4486..354704355 100644
--- a/qemu/slirp/arp_table.c
+++ b/qemu/slirp/arp_table.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "slirp.h"
 
 void arp_table_add(Slirp *slirp, uint32_t ip_addr, uint8_t ethaddr[ETH_ALEN])
diff --git a/qemu/slirp/bootp.c b/qemu/slirp/bootp.c
index b7db9fa33..7b3232bdc 100644
--- a/qemu/slirp/bootp.c
+++ b/qemu/slirp/bootp.c
@@ -21,8 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <slirp.h>
 
+#if defined(_WIN32)
+/* Windows ntohl() returns an u_long value.
+ * Add a type cast to match the format strings. */
+# define ntohl(n) ((uint32_t)ntohl(n))
+#endif
+
 /* XXX: only DHCP is supported */
 
 #define LEASE_TIME (24 * 3600)
@@ -155,7 +162,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     dhcp_decode(bp, &dhcp_msg_type, &preq_addr);
     DPRINTF("bootp packet op=%d msgtype=%d", bp->bp_op, dhcp_msg_type);
     if (preq_addr.s_addr != htonl(0L))
-        DPRINTF(" req_addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF(" req_addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
     else
         DPRINTF("\n");
 
@@ -234,7 +241,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     q += 4;
 
     if (bc) {
-        DPRINTF("%s addr=%08x\n",
+        DPRINTF("%s addr=%08" PRIx32 "\n",
                 (dhcp_msg_type == DHCPDISCOVER) ? "offered" : "ack'ed",
                 ntohl(daddr.sin_addr.s_addr));
 
@@ -302,7 +309,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
     } else {
         static const char nak_msg[] = "requested address not available";
 
-        DPRINTF("nak'ed addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF("nak'ed addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
 
         *q++ = RFC2132_MSG_TYPE;
         *q++ = 1;
@@ -319,7 +326,7 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
 
     m->m_len = sizeof(struct bootp_t) -
         sizeof(struct ip) - sizeof(struct udphdr);
-    udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+    udp_output(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
 }
 
 void bootp_input(struct mbuf *m)
diff --git a/qemu/slirp/cksum.c b/qemu/slirp/cksum.c
index 63286600e..2ad0e6540 100644
--- a/qemu/slirp/cksum.c
+++ b/qemu/slirp/cksum.c
@@ -30,6 +30,7 @@
  * in_cksum.c,v 1.2 1994/08/02 07:48:16 davidg Exp
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 /*
@@ -137,3 +138,28 @@ cont:
 	REDUCE;
 	return (~sum & 0xffff);
 }
+
+int ip6_cksum(struct mbuf *m)
+{
+    /* TODO: Optimize this by being able to pass the ip6_pseudohdr to cksum
+     * separately from the mbuf */
+    struct ip6 save_ip, *ip = mtod(m, struct ip6 *);
+    struct ip6_pseudohdr *ih = mtod(m, struct ip6_pseudohdr *);
+    int sum;
+
+    save_ip = *ip;
+
+    ih->ih_src = save_ip.ip_src;
+    ih->ih_dst = save_ip.ip_dst;
+    ih->ih_pl = htonl((uint32_t)ntohs(save_ip.ip_pl));
+    ih->ih_zero_hi = 0;
+    ih->ih_zero_lo = 0;
+    ih->ih_nh = save_ip.ip_nh;
+
+    sum = cksum(m, ((int)sizeof(struct ip6_pseudohdr))
+                    + ntohl(ih->ih_pl));
+
+    *ip = save_ip;
+
+    return sum;
+}
diff --git a/qemu/slirp/dnssearch.c b/qemu/slirp/dnssearch.c
index 4c9064ecb..aed2f13af 100644
--- a/qemu/slirp/dnssearch.c
+++ b/qemu/slirp/dnssearch.c
@@ -22,9 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "slirp.h"
 
diff --git a/qemu/slirp/if.c b/qemu/slirp/if.c
index fb7acf87d..9b02180db 100644
--- a/qemu/slirp/if.c
+++ b/qemu/slirp/if.c
@@ -5,6 +5,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include "qemu/timer.h"
 
@@ -27,9 +28,9 @@ ifs_remque(struct mbuf *ifm)
 void
 if_init(Slirp *slirp)
 {
-    slirp->if_fastq.ifq_next = slirp->if_fastq.ifq_prev = &slirp->if_fastq;
-    slirp->if_batchq.ifq_next = slirp->if_batchq.ifq_prev = &slirp->if_batchq;
-    slirp->next_m = &slirp->if_batchq;
+    slirp->if_fastq.qh_link = slirp->if_fastq.qh_rlink = &slirp->if_fastq;
+    slirp->if_batchq.qh_link = slirp->if_batchq.qh_rlink = &slirp->if_batchq;
+    slirp->next_m = (struct mbuf *) &slirp->if_batchq;
 }
 
 /*
@@ -53,8 +54,8 @@ if_output(struct socket *so, struct mbuf *ifm)
 	int on_fastq = 1;
 
 	DEBUG_CALL("if_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ifm = %lx", (long)ifm);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ifm = %p", ifm);
 
 	/*
 	 * First remove the mbuf from m_usedlist,
@@ -73,7 +74,8 @@ if_output(struct socket *so, struct mbuf *ifm)
 	 * We mustn't put this packet back on the fastq (or we'll send it out of order)
 	 * XXX add cache here?
 	 */
-	for (ifq = slirp->if_batchq.ifq_prev; ifq != &slirp->if_batchq;
+	for (ifq = (struct mbuf *) slirp->if_batchq.qh_rlink;
+	     (struct quehead *) ifq != &slirp->if_batchq;
 	     ifq = ifq->ifq_prev) {
 		if (so == ifq->ifq_so) {
 			/* A match! */
@@ -85,7 +87,7 @@ if_output(struct socket *so, struct mbuf *ifm)
 
 	/* No match, check which queue to put it on */
 	if (so && (so->so_iptos & IPTOS_LOWDELAY)) {
-		ifq = slirp->if_fastq.ifq_prev;
+		ifq = (struct mbuf *) slirp->if_fastq.qh_rlink;
 		on_fastq = 1;
 		/*
 		 * Check if this packet is a part of the last
@@ -97,9 +99,9 @@ if_output(struct socket *so, struct mbuf *ifm)
 			goto diddit;
 		}
         } else {
-		ifq = slirp->if_batchq.ifq_prev;
+		ifq = (struct mbuf *) slirp->if_batchq.qh_rlink;
                 /* Set next_m if the queue was empty so far */
-                if (slirp->next_m == &slirp->if_batchq) {
+                if ((struct quehead *) slirp->next_m == &slirp->if_batchq) {
                     slirp->next_m = ifm;
                 }
         }
@@ -165,10 +167,10 @@ void if_start(Slirp *slirp)
     }
     slirp->if_start_busy = true;
 
-    if (slirp->if_fastq.ifq_next != &slirp->if_fastq) {
-        ifm_next = slirp->if_fastq.ifq_next;
+    if (slirp->if_fastq.qh_link != &slirp->if_fastq) {
+        ifm_next = (struct mbuf *) slirp->if_fastq.qh_link;
         next_from_batchq = false;
-    } else if (slirp->next_m != &slirp->if_batchq) {
+    } else if ((struct quehead *) slirp->next_m != &slirp->if_batchq) {
         /* Nothing on fastq, pick up from batchq via next_m */
         ifm_next = slirp->next_m;
         next_from_batchq = true;
@@ -181,19 +183,19 @@ void if_start(Slirp *slirp)
         from_batchq = next_from_batchq;
 
         ifm_next = ifm->ifq_next;
-        if (ifm_next == &slirp->if_fastq) {
+        if ((struct quehead *) ifm_next == &slirp->if_fastq) {
             /* No more packets in fastq, switch to batchq */
             ifm_next = slirp->next_m;
             next_from_batchq = true;
         }
-        if (ifm_next == &slirp->if_batchq) {
+        if ((struct quehead *) ifm_next == &slirp->if_batchq) {
             /* end of batchq */
             ifm_next = NULL;
         }
 
         /* Try to send packet unless it already expired */
         if (ifm->expiration_date >= now && !if_encap(slirp, ifm)) {
-            /* Packet is delayed due to pending ARP resolution */
+            /* Packet is delayed due to pending ARP or NDP resolution */
             continue;
         }
 
@@ -217,7 +219,7 @@ void if_start(Slirp *slirp)
                 /* Next packet in fastq is from the same session */
                 ifm_next = next;
                 next_from_batchq = false;
-            } else if (slirp->next_m == &slirp->if_batchq) {
+            } else if ((struct quehead *) slirp->next_m == &slirp->if_batchq) {
                 /* Set next_m and ifm_next if the session packet is now the
                  * only one on batchq */
                 slirp->next_m = ifm_next = next;
diff --git a/qemu/slirp/if.h b/qemu/slirp/if.h
index 33270239f..c7a5c5724 100644
--- a/qemu/slirp/if.h
+++ b/qemu/slirp/if.h
@@ -17,7 +17,7 @@
 #define IF_MRU 1500
 #define	IF_COMP IF_AUTOCOMP	/* Flags for compression */
 
-/* 2 for alignment, 14 for ethernet, 40 for TCP/IP */
-#define IF_MAXLINKHDR (2 + 14 + 40)
+/* 2 for alignment, 14 for ethernet */
+#define IF_MAXLINKHDR (2 + ETH_HLEN)
 
 #endif
diff --git a/qemu/slirp/ip6.h b/qemu/slirp/ip6.h
new file mode 100644
index 000000000..8ddfa242c
--- /dev/null
+++ b/qemu/slirp/ip6.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#ifndef SLIRP_IP6_H_
+#define SLIRP_IP6_H_
+
+#include "net/eth.h"
+
+#define ALLNODES_MULTICAST  { .s6_addr = \
+                            { 0xff, 0x02, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x01 } }
+
+#define SOLICITED_NODE_PREFIX { .s6_addr = \
+                            { 0xff, 0x02, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x01,\
+                            0xff, 0x00, 0x00, 0x00 } }
+
+#define LINKLOCAL_ADDR  { .s6_addr = \
+                        { 0xfe, 0x80, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x02 } }
+
+static inline bool in6_equal(const struct in6_addr *a, const struct in6_addr *b)
+{
+    return memcmp(a, b, sizeof(*a)) == 0;
+}
+
+static inline bool in6_equal_net(const struct in6_addr *a,
+                                 const struct in6_addr *b,
+                                 int prefix_len)
+{
+    if (memcmp(a, b, prefix_len / 8) != 0) {
+        return 0;
+    }
+
+    if (prefix_len % 8 == 0) {
+        return 1;
+    }
+
+    return a->s6_addr[prefix_len / 8] >> (8 - (prefix_len % 8))
+        == b->s6_addr[prefix_len / 8] >> (8 - (prefix_len % 8));
+}
+
+static inline bool in6_equal_mach(const struct in6_addr *a,
+                                  const struct in6_addr *b,
+                                  int prefix_len)
+{
+    if (memcmp(&(a->s6_addr[(prefix_len + 7) / 8]),
+               &(b->s6_addr[(prefix_len + 7) / 8]),
+               16 - (prefix_len + 7) / 8) != 0) {
+        return 0;
+    }
+
+    if (prefix_len % 8 == 0) {
+        return 1;
+    }
+
+    return (a->s6_addr[prefix_len / 8] & ((1U << (8 - (prefix_len % 8))) - 1))
+        == (b->s6_addr[prefix_len / 8] & ((1U << (8 - (prefix_len % 8))) - 1));
+}
+
+
+#define in6_equal_router(a)\
+    ((in6_equal_net(a, &slirp->vprefix_addr6, slirp->vprefix_len)\
+      && in6_equal_mach(a, &slirp->vhost_addr6, slirp->vprefix_len))\
+  || (in6_equal_net(a, &(struct in6_addr)LINKLOCAL_ADDR, 64)\
+      && in6_equal_mach(a, &slirp->vhost_addr6, 64)))
+
+#define in6_equal_dns(a)\
+    ((in6_equal_net(a, &slirp->vprefix_addr6, slirp->vprefix_len)\
+      && in6_equal_mach(a, &slirp->vnameserver_addr6, slirp->vprefix_len))\
+  || (in6_equal_net(a, &(struct in6_addr)LINKLOCAL_ADDR, 64)\
+      && in6_equal_mach(a, &slirp->vnameserver_addr6, 64)))
+
+#define in6_equal_host(a)\
+    (in6_equal_router(a) || in6_equal_dns(a))
+
+#define in6_solicitednode_multicast(a)\
+    (in6_equal_net(a, &(struct in6_addr)SOLICITED_NODE_PREFIX, 104))
+
+/* Compute emulated host MAC address from its ipv6 address */
+static inline void in6_compute_ethaddr(struct in6_addr ip,
+                                       uint8_t eth[ETH_ALEN])
+{
+    eth[0] = 0x52;
+    eth[1] = 0x56;
+    memcpy(&eth[2], &ip.s6_addr[16 - (ETH_ALEN - 2)], ETH_ALEN - 2);
+}
+
+/*
+ * Definitions for internet protocol version 6.
+ * Per RFC 2460, December 1998.
+ */
+#define IP6VERSION      6
+#define IP6_HOP_LIMIT 255
+
+/*
+ * Structure of an internet header, naked of options.
+ */
+struct ip6 {
+#ifdef HOST_WORDS_BIGENDIAN
+    uint32_t
+        ip_v:4,         /* version */
+        ip_tc_hi:4,     /* traffic class */
+        ip_tc_lo:4,
+        ip_fl_hi:4,     /* flow label */
+        ip_fl_lo:16;
+#else
+    uint32_t
+        ip_tc_hi:4,
+        ip_v:4,
+        ip_fl_hi:4,
+        ip_tc_lo:4,
+        ip_fl_lo:16;
+#endif
+    uint16_t    ip_pl;               /* payload length */
+    uint8_t     ip_nh;               /* next header */
+    uint8_t     ip_hl;               /* hop limit */
+    struct in6_addr ip_src, ip_dst;  /* source and dest address */
+} QEMU_PACKED;
+
+/*
+ * IPv6 pseudo-header used by upper-layer protocols
+ */
+struct ip6_pseudohdr {
+    struct      in6_addr ih_src;  /* source internet address */
+    struct      in6_addr ih_dst;  /* destination internet address */
+    uint32_t    ih_pl;            /* upper-layer packet length */
+    uint16_t    ih_zero_hi;       /* zero */
+    uint8_t     ih_zero_lo;       /* zero */
+    uint8_t     ih_nh;            /* next header */
+} QEMU_PACKED;
+
+
+#endif
diff --git a/qemu/slirp/ip6_icmp.c b/qemu/slirp/ip6_icmp.c
new file mode 100644
index 000000000..09571bcd6
--- /dev/null
+++ b/qemu/slirp/ip6_icmp.c
@@ -0,0 +1,424 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "ip6_icmp.h"
+#include "qemu/timer.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+#include <time.h>
+
+#define NDP_Interval g_rand_int_range(slirp->grand, \
+        NDP_MinRtrAdvInterval, NDP_MaxRtrAdvInterval)
+
+static void ra_timer_handler(void *opaque)
+{
+    Slirp *slirp = opaque;
+    timer_mod(slirp->ra_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + NDP_Interval);
+    ndp_send_ra(slirp);
+}
+
+void icmp6_init(Slirp *slirp)
+{
+    if (!slirp->in6_enabled) {
+        return;
+    }
+
+    slirp->ra_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, ra_timer_handler, slirp);
+    timer_mod(slirp->ra_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + NDP_Interval);
+}
+
+void icmp6_cleanup(Slirp *slirp)
+{
+    if (!slirp->in6_enabled) {
+        return;
+    }
+
+    timer_del(slirp->ra_timer);
+    timer_free(slirp->ra_timer);
+}
+
+static void icmp6_send_echoreply(struct mbuf *m, Slirp *slirp, struct ip6 *ip,
+        struct icmp6 *icmp)
+{
+    struct mbuf *t = m_get(slirp);
+    t->m_len = sizeof(struct ip6) + ntohs(ip->ip_pl);
+    memcpy(t->m_data, m->m_data, t->m_len);
+
+    /* IPv6 Packet */
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_dst = ip->ip_src;
+    rip->ip_src = ip->ip_dst;
+
+    /* ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_ECHO_REPLY;
+    ricmp->icmp6_cksum = 0;
+
+    /* Checksum */
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+void icmp6_send_error(struct mbuf *m, uint8_t type, uint8_t code)
+{
+    Slirp *slirp = m->slirp;
+    struct mbuf *t;
+    struct ip6 *ip = mtod(m, struct ip6 *);
+
+    DEBUG_CALL("icmp6_send_error");
+    DEBUG_ARGS((dfd, " type = %d, code = %d\n", type, code));
+
+    if (IN6_IS_ADDR_MULTICAST(&ip->ip_src) ||
+            IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)) {
+        /* TODO icmp error? */
+        return;
+    }
+
+    t = m_get(slirp);
+
+    /* IPv6 packet */
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = (struct in6_addr)LINKLOCAL_ADDR;
+    rip->ip_dst = ip->ip_src;
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &rip->ip_dst, addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("target = %s", addrstr);
+#endif
+
+    rip->ip_nh = IPPROTO_ICMPV6;
+    const int error_data_len = min(m->m_len,
+            IF_MTU - (sizeof(struct ip6) + ICMP6_ERROR_MINLEN));
+    rip->ip_pl = htons(ICMP6_ERROR_MINLEN + error_data_len);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = type;
+    ricmp->icmp6_code = code;
+    ricmp->icmp6_cksum = 0;
+
+    switch (type) {
+    case ICMP6_UNREACH:
+    case ICMP6_TIMXCEED:
+        ricmp->icmp6_err.unused = 0;
+        break;
+    case ICMP6_TOOBIG:
+        ricmp->icmp6_err.mtu = htonl(IF_MTU);
+        break;
+    case ICMP6_PARAMPROB:
+        /* TODO: Handle this case */
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+    t->m_data += ICMP6_ERROR_MINLEN;
+    memcpy(t->m_data, m->m_data, error_data_len);
+
+    /* Checksum */
+    t->m_data -= ICMP6_ERROR_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Send NDP Router Advertisement
+ */
+void ndp_send_ra(Slirp *slirp)
+{
+    DEBUG_CALL("ndp_send_ra");
+
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = (struct in6_addr)LINKLOCAL_ADDR;
+    rip->ip_dst = (struct in6_addr)ALLNODES_MULTICAST;
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_RA_MINLEN
+                        + NDPOPT_LINKLAYER_LEN
+                        + NDPOPT_PREFIXINFO_LEN);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_RA;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nra.chl = NDP_AdvCurHopLimit;
+    ricmp->icmp6_nra.M = NDP_AdvManagedFlag;
+    ricmp->icmp6_nra.O = NDP_AdvOtherConfigFlag;
+    ricmp->icmp6_nra.reserved = 0;
+    ricmp->icmp6_nra.lifetime = htons(NDP_AdvDefaultLifetime);
+    ricmp->icmp6_nra.reach_time = htonl(NDP_AdvReachableTime);
+    ricmp->icmp6_nra.retrans_time = htonl(NDP_AdvRetransTime);
+
+    /* Source link-layer address (NDP option) */
+    t->m_data += ICMP6_NDP_RA_MINLEN;
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_SOURCE;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(rip->ip_src, opt->ndpopt_linklayer);
+
+    /* Prefix information (NDP option) */
+    t->m_data += NDPOPT_LINKLAYER_LEN;
+    struct ndpopt *opt2 = mtod(t, struct ndpopt *);
+    opt2->ndpopt_type = NDPOPT_PREFIX_INFO;
+    opt2->ndpopt_len = NDPOPT_PREFIXINFO_LEN / 8;
+    opt2->ndpopt_prefixinfo.prefix_length = slirp->vprefix_len;
+    opt2->ndpopt_prefixinfo.L = 1;
+    opt2->ndpopt_prefixinfo.A = 1;
+    opt2->ndpopt_prefixinfo.reserved1 = 0;
+    opt2->ndpopt_prefixinfo.valid_lt = htonl(NDP_AdvValidLifetime);
+    opt2->ndpopt_prefixinfo.pref_lt = htonl(NDP_AdvPrefLifetime);
+    opt2->ndpopt_prefixinfo.reserved2 = 0;
+    opt2->ndpopt_prefixinfo.prefix = slirp->vprefix_addr6;
+
+    /* ICMPv6 Checksum */
+    t->m_data -= NDPOPT_LINKLAYER_LEN;
+    t->m_data -= ICMP6_NDP_RA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Send NDP Neighbor Solitication
+ */
+void ndp_send_ns(Slirp *slirp, struct in6_addr addr)
+{
+    DEBUG_CALL("ndp_send_ns");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &addr, addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("target = %s", addrstr);
+#endif
+
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = slirp->vhost_addr6;
+    rip->ip_dst = (struct in6_addr)SOLICITED_NODE_PREFIX;
+    memcpy(&rip->ip_dst.s6_addr[13], &addr.s6_addr[13], 3);
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_NS_MINLEN + NDPOPT_LINKLAYER_LEN);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_NS;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nns.reserved = 0;
+    ricmp->icmp6_nns.target = addr;
+
+    /* Build NDP option */
+    t->m_data += ICMP6_NDP_NS_MINLEN;
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_SOURCE;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(slirp->vhost_addr6, opt->ndpopt_linklayer);
+
+    /* ICMPv6 Checksum */
+    t->m_data -= ICMP6_NDP_NA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 1);
+}
+
+/*
+ * Send NDP Neighbor Advertisement
+ */
+static void ndp_send_na(Slirp *slirp, struct ip6 *ip, struct icmp6 *icmp)
+{
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = icmp->icmp6_nns.target;
+    if (IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)) {
+        rip->ip_dst = (struct in6_addr)ALLNODES_MULTICAST;
+    } else {
+        rip->ip_dst = ip->ip_src;
+    }
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_NA_MINLEN
+                        + NDPOPT_LINKLAYER_LEN);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_NA;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nna.R = NDP_IsRouter;
+    ricmp->icmp6_nna.S = !IN6_IS_ADDR_MULTICAST(&rip->ip_dst);
+    ricmp->icmp6_nna.O = 1;
+    ricmp->icmp6_nna.reserved_hi = 0;
+    ricmp->icmp6_nna.reserved_lo = 0;
+    ricmp->icmp6_nna.target = icmp->icmp6_nns.target;
+
+    /* Build NDP option */
+    t->m_data += ICMP6_NDP_NA_MINLEN;
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_TARGET;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(ricmp->icmp6_nna.target,
+                    opt->ndpopt_linklayer);
+
+    /* ICMPv6 Checksum */
+    t->m_data -= ICMP6_NDP_NA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Process a NDP message
+ */
+static void ndp_input(struct mbuf *m, Slirp *slirp, struct ip6 *ip,
+        struct icmp6 *icmp)
+{
+    m->m_len += ETH_HLEN;
+    m->m_data -= ETH_HLEN;
+    struct ethhdr *eth = mtod(m, struct ethhdr *);
+    m->m_len -= ETH_HLEN;
+    m->m_data += ETH_HLEN;
+
+    switch (icmp->icmp6_type) {
+    case ICMP6_NDP_RS:
+        DEBUG_CALL(" type = Router Solicitation");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_RS_MINLEN) {
+            /* Gratuitous NDP */
+            ndp_table_add(slirp, ip->ip_src, eth->h_source);
+
+            ndp_send_ra(slirp);
+        }
+        break;
+
+    case ICMP6_NDP_RA:
+        DEBUG_CALL(" type = Router Advertisement");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "Warning: guest sent NDP RA, but shouldn't");
+        break;
+
+    case ICMP6_NDP_NS:
+        DEBUG_CALL(" type = Neighbor Solicitation");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && !IN6_IS_ADDR_MULTICAST(&icmp->icmp6_nns.target)
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_NS_MINLEN
+                && (!IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)
+                    || in6_solicitednode_multicast(&ip->ip_dst))) {
+            if (in6_equal_host(&icmp->icmp6_nns.target)) {
+                /* Gratuitous NDP */
+                ndp_table_add(slirp, ip->ip_src, eth->h_source);
+                ndp_send_na(slirp, ip, icmp);
+            }
+        }
+        break;
+
+    case ICMP6_NDP_NA:
+        DEBUG_CALL(" type = Neighbor Advertisement");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_NA_MINLEN
+                && !IN6_IS_ADDR_MULTICAST(&icmp->icmp6_nna.target)
+                && (!IN6_IS_ADDR_MULTICAST(&ip->ip_dst)
+                    || icmp->icmp6_nna.S == 0)) {
+            ndp_table_add(slirp, ip->ip_src, eth->h_source);
+        }
+        break;
+
+    case ICMP6_NDP_REDIRECT:
+        DEBUG_CALL(" type = Redirect");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "Warning: guest sent NDP REDIRECT, but shouldn't");
+        break;
+    }
+}
+
+/*
+ * Process a received ICMPv6 message.
+ */
+void icmp6_input(struct mbuf *m)
+{
+    struct icmp6 *icmp;
+    struct ip6 *ip = mtod(m, struct ip6 *);
+    Slirp *slirp = m->slirp;
+    int hlen = sizeof(struct ip6);
+
+    DEBUG_CALL("icmp6_input");
+    DEBUG_ARG("m = %lx", (long) m);
+    DEBUG_ARG("m_len = %d", m->m_len);
+
+    if (ntohs(ip->ip_pl) < ICMP6_MINLEN) {
+        goto end;
+    }
+
+    if (ip6_cksum(m)) {
+        goto end;
+    }
+
+    m->m_len -= hlen;
+    m->m_data += hlen;
+    icmp = mtod(m, struct icmp6 *);
+    m->m_len += hlen;
+    m->m_data -= hlen;
+
+    DEBUG_ARG("icmp6_type = %d", icmp->icmp6_type);
+    switch (icmp->icmp6_type) {
+    case ICMP6_ECHO_REQUEST:
+        if (in6_equal_host(&ip->ip_dst)) {
+            icmp6_send_echoreply(m, slirp, ip, icmp);
+        } else {
+            /* TODO */
+            error_report("external icmpv6 not supported yet");
+        }
+        break;
+
+    case ICMP6_NDP_RS:
+    case ICMP6_NDP_RA:
+    case ICMP6_NDP_NS:
+    case ICMP6_NDP_NA:
+    case ICMP6_NDP_REDIRECT:
+        ndp_input(m, slirp, ip, icmp);
+        break;
+
+    case ICMP6_UNREACH:
+    case ICMP6_TOOBIG:
+    case ICMP6_TIMXCEED:
+    case ICMP6_PARAMPROB:
+        /* XXX? report error? close socket? */
+    default:
+        break;
+    }
+
+end:
+    m_free(m);
+}
diff --git a/qemu/slirp/ip6_icmp.h b/qemu/slirp/ip6_icmp.h
new file mode 100644
index 000000000..9460bf837
--- /dev/null
+++ b/qemu/slirp/ip6_icmp.h
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#ifndef SLIRP_NETINET_ICMP6_H_
+#define SLIRP_NETINET_ICMP6_H_
+
+/*
+ * Interface Control Message Protocol version 6 Definitions.
+ * Per RFC 4443, March 2006.
+ *
+ * Network Discover Protocol Definitions.
+ * Per RFC 4861, September 2007.
+ */
+
+struct icmp6_echo { /* Echo Messages */
+    uint16_t id;
+    uint16_t seq_num;
+};
+
+union icmp6_error_body {
+    uint32_t unused;
+    uint32_t pointer;
+    uint32_t mtu;
+};
+
+/*
+ * NDP Messages
+ */
+struct ndp_rs {     /* Router Solicitation Message */
+    uint32_t reserved;
+};
+
+struct ndp_ra {     /* Router Advertisement Message */
+    uint8_t chl;    /* Cur Hop Limit */
+#ifdef HOST_WORDS_BIGENDIAN
+    uint8_t
+        M:1,
+        O:1,
+        reserved:6;
+#else
+    uint8_t
+        reserved:6,
+        O:1,
+        M:1;
+#endif
+    uint16_t lifetime;      /* Router Lifetime */
+    uint32_t reach_time;    /* Reachable Time */
+    uint32_t retrans_time;  /* Retrans Timer */
+} QEMU_PACKED;
+
+struct ndp_ns {     /* Neighbor Solicitation Message */
+    uint32_t reserved;
+    struct in6_addr target; /* Target Address */
+} QEMU_PACKED;
+
+struct ndp_na {     /* Neighbor Advertisement Message */
+#ifdef HOST_WORDS_BIGENDIAN
+    uint32_t
+        R:1,                /* Router Flag */
+        S:1,                /* Solicited Flag */
+        O:1,                /* Override Flag */
+        reserved_hi:5,
+        reserved_lo:24;
+#else
+    uint32_t
+        reserved_hi:5,
+        O:1,
+        S:1,
+        R:1,
+        reserved_lo:24;
+#endif
+    struct in6_addr target; /* Target Address */
+} QEMU_PACKED;
+
+struct ndp_redirect {
+    uint32_t reserved;
+    struct in6_addr target; /* Target Address */
+    struct in6_addr dest;   /* Destination Address */
+} QEMU_PACKED;
+
+/*
+ * Structure of an icmpv6 header.
+ */
+struct icmp6 {
+    uint8_t     icmp6_type;         /* type of message, see below */
+    uint8_t     icmp6_code;         /* type sub code */
+    uint16_t    icmp6_cksum;        /* ones complement cksum of struct */
+    union {
+        union icmp6_error_body error_body;
+        struct icmp6_echo echo;
+        struct ndp_rs ndp_rs;
+        struct ndp_ra ndp_ra;
+        struct ndp_ns ndp_ns;
+        struct ndp_na ndp_na;
+        struct ndp_redirect ndp_redirect;
+    } icmp6_body;
+#define icmp6_err icmp6_body.error_body
+#define icmp6_echo icmp6_body.echo
+#define icmp6_nrs icmp6_body.ndp_rs
+#define icmp6_nra icmp6_body.ndp_ra
+#define icmp6_nns icmp6_body.ndp_ns
+#define icmp6_nna icmp6_body.ndp_na
+#define icmp6_redirect icmp6_body.ndp_redirect
+} QEMU_PACKED;
+
+#define ICMP6_MINLEN    4
+#define ICMP6_ERROR_MINLEN  8
+#define ICMP6_ECHO_MINLEN   8
+#define ICMP6_NDP_RS_MINLEN 8
+#define ICMP6_NDP_RA_MINLEN 16
+#define ICMP6_NDP_NS_MINLEN 24
+#define ICMP6_NDP_NA_MINLEN 24
+#define ICMP6_NDP_REDIRECT_MINLEN 40
+
+/*
+ * NDP Options
+ */
+struct ndpopt {
+    uint8_t     ndpopt_type;                    /* Option type */
+    uint8_t     ndpopt_len;                     /* /!\ In units of 8 octets */
+    union {
+        unsigned char   linklayer_addr[6];      /* Source/Target Link-layer */
+        struct prefixinfo {                     /* Prefix Information */
+            uint8_t     prefix_length;
+#ifdef HOST_WORDS_BIGENDIAN
+            uint8_t     L:1, A:1, reserved1:6;
+#else
+            uint8_t     reserved1:6, A:1, L:1;
+#endif
+            uint32_t    valid_lt;               /* Valid Lifetime */
+            uint32_t    pref_lt;                /* Preferred Lifetime */
+            uint32_t    reserved2;
+            struct in6_addr prefix;
+        } QEMU_PACKED prefixinfo;
+    } ndpopt_body;
+#define ndpopt_linklayer ndpopt_body.linklayer_addr
+#define ndpopt_prefixinfo ndpopt_body.prefixinfo
+} QEMU_PACKED;
+
+/* NDP options type */
+#define NDPOPT_LINKLAYER_SOURCE     1   /* Source Link-Layer Address */
+#define NDPOPT_LINKLAYER_TARGET     2   /* Target Link-Layer Address */
+#define NDPOPT_PREFIX_INFO          3   /* Prefix Information */
+
+/* NDP options size, in octets. */
+#define NDPOPT_LINKLAYER_LEN    8
+#define NDPOPT_PREFIXINFO_LEN   32
+
+/*
+ * Definition of type and code field values.
+ * Per https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xml
+ * Last Updated 2012-11-12
+ */
+
+/* Errors */
+#define ICMP6_UNREACH   1   /* Destination Unreachable */
+#define     ICMP6_UNREACH_NO_ROUTE      0   /* no route to dest */
+#define     ICMP6_UNREACH_DEST_PROHIB   1   /* com with dest prohibited */
+#define     ICMP6_UNREACH_SCOPE         2   /* beyond scope of src addr */
+#define     ICMP6_UNREACH_ADDRESS       3   /* address unreachable */
+#define     ICMP6_UNREACH_PORT          4   /* port unreachable */
+#define     ICMP6_UNREACH_SRC_FAIL      5   /* src addr failed */
+#define     ICMP6_UNREACH_REJECT_ROUTE  6   /* reject route to dest */
+#define     ICMP6_UNREACH_SRC_HDR_ERROR 7   /* error in src routing header */
+#define ICMP6_TOOBIG    2   /* Packet Too Big */
+#define ICMP6_TIMXCEED  3   /* Time Exceeded */
+#define     ICMP6_TIMXCEED_INTRANS      0   /* hop limit exceeded in transit */
+#define     ICMP6_TIMXCEED_REASS        1   /* ttl=0 in reass */
+#define ICMP6_PARAMPROB 4   /* Parameter Problem */
+#define     ICMP6_PARAMPROB_HDR_FIELD   0   /* err header field */
+#define     ICMP6_PARAMPROB_NXTHDR_TYPE 1   /* unrecognized Next Header type */
+#define     ICMP6_PARAMPROB_IPV6_OPT    2   /* unrecognized IPv6 option */
+
+/* Informational Messages */
+#define ICMP6_ECHO_REQUEST      128 /* Echo Request */
+#define ICMP6_ECHO_REPLY        129 /* Echo Reply */
+#define ICMP6_NDP_RS            133 /* Router Solicitation (NDP) */
+#define ICMP6_NDP_RA            134 /* Router Advertisement (NDP) */
+#define ICMP6_NDP_NS            135 /* Neighbor Solicitation (NDP) */
+#define ICMP6_NDP_NA            136 /* Neighbor Advertisement (NDP) */
+#define ICMP6_NDP_REDIRECT      137 /* Redirect Message (NDP) */
+
+/*
+ * Router Configuration Variables (rfc4861#section-6)
+ */
+#define NDP_IsRouter                1
+#define NDP_AdvSendAdvertisements   1
+#define NDP_MaxRtrAdvInterval       600000
+#define NDP_MinRtrAdvInterval       ((NDP_MaxRtrAdvInterval >= 9) ? \
+                                        NDP_MaxRtrAdvInterval / 3 : \
+                                        NDP_MaxRtrAdvInterval)
+#define NDP_AdvManagedFlag          0
+#define NDP_AdvOtherConfigFlag      0
+#define NDP_AdvLinkMTU              0
+#define NDP_AdvReachableTime        0
+#define NDP_AdvRetransTime          0
+#define NDP_AdvCurHopLimit          64
+#define NDP_AdvDefaultLifetime      ((3 * NDP_MaxRtrAdvInterval) / 1000)
+#define NDP_AdvValidLifetime        86400
+#define NDP_AdvOnLinkFlag           1
+#define NDP_AdvPrefLifetime         14400
+#define NDP_AdvAutonomousFlag       1
+
+void icmp6_init(Slirp *slirp);
+void icmp6_cleanup(Slirp *slirp);
+void icmp6_input(struct mbuf *);
+void icmp6_send_error(struct mbuf *m, uint8_t type, uint8_t code);
+void ndp_send_ra(Slirp *slirp);
+void ndp_send_ns(Slirp *slirp, struct in6_addr addr);
+
+#endif
diff --git a/qemu/slirp/ip6_input.c b/qemu/slirp/ip6_input.c
new file mode 100644
index 000000000..ac2e3ea88
--- /dev/null
+++ b/qemu/slirp/ip6_input.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "ip6_icmp.h"
+
+/*
+ * IP initialization: fill in IP protocol switch table.
+ * All protocols not implemented in kernel go to raw IP protocol handler.
+ */
+void ip6_init(Slirp *slirp)
+{
+    icmp6_init(slirp);
+}
+
+void ip6_cleanup(Slirp *slirp)
+{
+    icmp6_cleanup(slirp);
+}
+
+void ip6_input(struct mbuf *m)
+{
+    struct ip6 *ip6;
+    Slirp *slirp = m->slirp;
+
+    if (!slirp->in6_enabled) {
+        goto bad;
+    }
+
+    DEBUG_CALL("ip6_input");
+    DEBUG_ARG("m = %lx", (long)m);
+    DEBUG_ARG("m_len = %d", m->m_len);
+
+    if (m->m_len < sizeof(struct ip6)) {
+        goto bad;
+    }
+
+    ip6 = mtod(m, struct ip6 *);
+
+    if (ip6->ip_v != IP6VERSION) {
+        goto bad;
+    }
+
+    if (ntohs(ip6->ip_pl) > IF_MTU) {
+        icmp6_send_error(m, ICMP6_TOOBIG, 0);
+        goto bad;
+    }
+
+    /* check ip_ttl for a correct ICMP reply */
+    if (ip6->ip_hl == 0) {
+        icmp6_send_error(m, ICMP6_TIMXCEED, ICMP6_TIMXCEED_INTRANS);
+        goto bad;
+    }
+
+    /*
+     * Switch out to protocol's input routine.
+     */
+    switch (ip6->ip_nh) {
+    case IPPROTO_TCP:
+        NTOHS(ip6->ip_pl);
+        tcp_input(m, sizeof(struct ip6), (struct socket *)NULL, AF_INET6);
+        break;
+    case IPPROTO_UDP:
+        udp6_input(m);
+        break;
+    case IPPROTO_ICMPV6:
+        icmp6_input(m);
+        break;
+    default:
+        m_free(m);
+    }
+    return;
+bad:
+    m_free(m);
+}
diff --git a/qemu/slirp/ip6_output.c b/qemu/slirp/ip6_output.c
new file mode 100644
index 000000000..762cbfe89
--- /dev/null
+++ b/qemu/slirp/ip6_output.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+
+/* Number of packets queued before we start sending
+ * (to prevent allocing too many mbufs) */
+#define IF6_THRESH 10
+
+/*
+ * IPv6 output. The packet in mbuf chain m contains a IP header
+ */
+int ip6_output(struct socket *so, struct mbuf *m, int fast)
+{
+    struct ip6 *ip = mtod(m, struct ip6 *);
+
+    DEBUG_CALL("ip6_output");
+    DEBUG_ARG("so = %lx", (long)so);
+    DEBUG_ARG("m = %lx", (long)m);
+
+    /* Fill IPv6 header */
+    ip->ip_v = IP6VERSION;
+    ip->ip_hl = IP6_HOP_LIMIT;
+    ip->ip_tc_hi = 0;
+    ip->ip_tc_lo = 0;
+    ip->ip_fl_hi = 0;
+    ip->ip_fl_lo = 0;
+
+    if (fast) {
+        if_encap(m->slirp, m);
+    } else {
+        if_output(so, m);
+    }
+
+    return 0;
+}
diff --git a/qemu/slirp/ip_icmp.c b/qemu/slirp/ip_icmp.c
index 9f1cb08a1..5ffc7a683 100644
--- a/qemu/slirp/ip_icmp.c
+++ b/qemu/slirp/ip_icmp.c
@@ -30,6 +30,7 @@
  * ip_icmp.c,v 1.7 1995/05/30 08:09:42 rgrimes Exp
  */
 
+#include "qemu/osdep.h"
 #include "slirp.h"
 #include "ip_icmp.h"
 
@@ -37,7 +38,7 @@
 /* Be nice and tell them it's just a pseudo-ping packet */
 static const char icmp_ping_msg[] = "This is a pseudo-PING packet used by Slirp to emulate ICMP ECHO-REQUEST packets.\n";
 
-/* list of actions for icmp_error() on RX of an icmp message */
+/* list of actions for icmp_send_error() on RX of an icmp message */
 static const int icmp_flush[19] = {
 /*  ECHO REPLY (0)  */   0,
 		         1,
@@ -100,7 +101,7 @@ static int icmp_send(struct socket *so, struct mbuf *m, int hlen)
                (struct sockaddr *)&addr, sizeof(addr)) == -1) {
         DEBUG_MISC((dfd, "icmp_input icmp sendto tx errno = %d-%s\n",
                     errno, strerror(errno)));
-        icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
+        icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
         icmp_detach(so);
     }
 
@@ -125,7 +126,7 @@ icmp_input(struct mbuf *m, int hlen)
   Slirp *slirp = m->slirp;
 
   DEBUG_CALL("icmp_input");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
   DEBUG_ARG("m_len = %d", m->m_len);
 
   /*
@@ -157,12 +158,12 @@ icmp_input(struct mbuf *m, int hlen)
         goto freeit;
     } else {
       struct socket *so;
-      struct sockaddr_in addr;
+      struct sockaddr_storage addr;
       if ((so = socreate(slirp)) == NULL) goto freeit;
       if (icmp_send(so, m, hlen) == 0) {
         return;
       }
-      if(udp_attach(so) == -1) {
+      if (udp_attach(so, AF_INET) == -1) {
 	DEBUG_MISC((dfd,"icmp_input udp_attach errno = %d-%s\n",
 		    errno,strerror(errno)));
 	sofree(so);
@@ -170,8 +171,10 @@ icmp_input(struct mbuf *m, int hlen)
 	goto end_error;
       }
       so->so_m = m;
+      so->so_ffamily = AF_INET;
       so->so_faddr = ip->ip_dst;
       so->so_fport = htons(7);
+      so->so_lfamily = AF_INET;
       so->so_laddr = ip->ip_src;
       so->so_lport = htons(9);
       so->so_iptos = ip->ip_tos;
@@ -179,25 +182,14 @@ icmp_input(struct mbuf *m, int hlen)
       so->so_state = SS_ISFCONNECTED;
 
       /* Send the packet */
-      addr.sin_family = AF_INET;
-      if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-          slirp->vnetwork_addr.s_addr) {
-	/* It's an alias */
-	if (so->so_faddr.s_addr == slirp->vnameserver_addr.s_addr) {
-	  if (get_dns_addr(&addr.sin_addr) < 0)
-	    addr.sin_addr = loopback_addr;
-	} else {
-	  addr.sin_addr = loopback_addr;
-	}
-      } else {
-	addr.sin_addr = so->so_faddr;
-      }
-      addr.sin_port = so->so_fport;
+      addr = so->fhost.ss;
+      sotranslate_out(so, &addr);
+
       if(sendto(so->s, icmp_ping_msg, strlen(icmp_ping_msg), 0,
-		(struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		(struct sockaddr *)&addr, sockaddr_size(&addr)) == -1) {
 	DEBUG_MISC((dfd,"icmp_input udp sendto tx errno = %d-%s\n",
 		    errno,strerror(errno)));
-	icmp_error(m, ICMP_UNREACH,ICMP_UNREACH_NET, 0,strerror(errno));
+	icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
 	udp_detach(so);
       }
     } /* if ip->ip_dst.s_addr == alias_addr.s_addr */
@@ -243,7 +235,7 @@ end_error:
 
 #define ICMP_MAXDATALEN (IP_MSS-28)
 void
-icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
+icmp_send_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
            const char *message)
 {
   unsigned hlen, shlen, s_ip_len;
@@ -251,8 +243,8 @@ icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
   register struct icmp *icp;
   register struct mbuf *m;
 
-  DEBUG_CALL("icmp_error");
-  DEBUG_ARG("msrc = %lx", (long )msrc);
+  DEBUG_CALL("icmp_send_error");
+  DEBUG_ARG("msrc = %p", msrc);
   DEBUG_ARG("msrc_len = %d", msrc->m_len);
 
   if(type!=ICMP_UNREACH && type!=ICMP_TIMXCEED) goto end_error;
@@ -441,7 +433,7 @@ void icmp_receive(struct socket *so)
         }
         DEBUG_MISC((dfd, " udp icmp rx errno = %d-%s\n", errno,
                     strerror(errno)));
-        icmp_error(so->so_m, ICMP_UNREACH, error_code, 0, strerror(errno));
+        icmp_send_error(so->so_m, ICMP_UNREACH, error_code, 0, strerror(errno));
     } else {
         icmp_reflect(so->so_m);
         so->so_m = NULL; /* Don't m_free() it again! */
diff --git a/qemu/slirp/ip_icmp.h b/qemu/slirp/ip_icmp.h
index be4426b8e..846761d08 100644
--- a/qemu/slirp/ip_icmp.h
+++ b/qemu/slirp/ip_icmp.h
@@ -156,8 +156,8 @@ struct icmp {
 void icmp_init(Slirp *slirp);
 void icmp_cleanup(Slirp *slirp);
 void icmp_input(struct mbuf *, int);
-void icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
-                const char *message);
+void icmp_send_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
+                     const char *message);
 void icmp_reflect(struct mbuf *);
 void icmp_receive(struct socket *so);
 void icmp_detach(struct socket *so);
diff --git a/qemu/slirp/ip_input.c b/qemu/slirp/ip_input.c
index 880bdfd3c..cdd54833a 100644
--- a/qemu/slirp/ip_input.c
+++ b/qemu/slirp/ip_input.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include <qemu/osdep.h>
 #include "ip_icmp.h"
@@ -79,12 +80,16 @@ ip_input(struct mbuf *m)
 	register struct ip *ip;
 	int hlen;
 
+	if (!slirp->in_enabled) {
+		goto bad;
+	}
+
 	DEBUG_CALL("ip_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m_len = %d", m->m_len);
 
 	if (m->m_len < sizeof (struct ip)) {
-		return;
+		goto bad;
 	}
 
 	ip = mtod(m, struct ip *);
@@ -131,9 +136,9 @@ ip_input(struct mbuf *m)
 	   m_adj(m, ip->ip_len - m->m_len);
 
 	/* check ip_ttl for a correct ICMP reply */
-	if(ip->ip_ttl==0) {
-	  icmp_error(m, ICMP_TIMXCEED,ICMP_TIMXCEED_INTRANS, 0,"ttl");
-	  goto bad;
+	if (ip->ip_ttl == 0) {
+	    icmp_send_error(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, 0, "ttl");
+	    goto bad;
 	}
 
 	/*
@@ -199,7 +204,7 @@ ip_input(struct mbuf *m)
 	 */
 	switch (ip->ip_p) {
 	 case IPPROTO_TCP:
-		tcp_input(m, hlen, (struct socket *)NULL);
+		tcp_input(m, hlen, (struct socket *)NULL, AF_INET);
 		break;
 	 case IPPROTO_UDP:
 		udp_input(m, hlen);
@@ -232,9 +237,9 @@ ip_reass(Slirp *slirp, struct ip *ip, struct ipq *fp)
 	int i, next;
 
 	DEBUG_CALL("ip_reass");
-	DEBUG_ARG("ip = %lx", (long)ip);
-	DEBUG_ARG("fp = %lx", (long)fp);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("ip = %p", ip);
+	DEBUG_ARG("fp = %p", fp);
+	DEBUG_ARG("m = %p", m);
 
 	/*
 	 * Presence of header sizes in mbufs
@@ -400,7 +405,7 @@ static void
 ip_enq(register struct ipasfrag *p, register struct ipasfrag *prev)
 {
 	DEBUG_CALL("ip_enq");
-	DEBUG_ARG("prev = %lx", (long)prev);
+	DEBUG_ARG("prev = %p", prev);
 	p->ipf_prev =  prev;
 	p->ipf_next = prev->ipf_next;
 	((struct ipasfrag *)(prev->ipf_next))->ipf_prev = p;
@@ -636,7 +641,7 @@ typedef uint32_t n_time;
 	}
 	return (0);
 bad:
- 	icmp_error(m, type, code, 0, 0);
+	icmp_send_error(m, type, code, 0, 0);
 
 	return (1);
 }
diff --git a/qemu/slirp/ip_output.c b/qemu/slirp/ip_output.c
index c82830fe7..0d6b3b831 100644
--- a/qemu/slirp/ip_output.c
+++ b/qemu/slirp/ip_output.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 /* Number of packets queued before we start sending
@@ -60,8 +61,8 @@ ip_output(struct socket *so, struct mbuf *m0)
 	int len, off, error = 0;
 
 	DEBUG_CALL("ip_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m0 = %lx", (long)m0);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m0 = %p", m0);
 
 	ip = mtod(m, struct ip *);
 	/*
diff --git a/qemu/slirp/libslirp.h b/qemu/slirp/libslirp.h
index 5bdcbd50f..127aa41d4 100644
--- a/qemu/slirp/libslirp.h
+++ b/qemu/slirp/libslirp.h
@@ -8,11 +8,14 @@ typedef struct Slirp Slirp;
 
 int get_dns_addr(struct in_addr *pdns_addr);
 
-Slirp *slirp_init(int restricted, struct in_addr vnetwork,
+Slirp *slirp_init(int restricted, bool in_enabled, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
-                  const char *vhostname, const char *tftp_path,
-                  const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, const char **vdnssearch,
+                  bool in6_enabled,
+                  struct in6_addr vprefix_addr6, uint8_t vprefix_len,
+                  struct in6_addr vhost6, const char *vhostname,
+                  const char *tftp_path, const char *bootfile,
+                  struct in_addr vdhcp_start, struct in_addr vnameserver,
+                  struct in6_addr vnameserver6, const char **vdnssearch,
                   void *opaque);
 void slirp_cleanup(Slirp *slirp);
 
diff --git a/qemu/slirp/mbuf.c b/qemu/slirp/mbuf.c
index 4fefb043b..d13698839 100644
--- a/qemu/slirp/mbuf.c
+++ b/qemu/slirp/mbuf.c
@@ -15,29 +15,30 @@
  * the flags
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 #define MBUF_THRESH 30
 
 /*
  * Find a nice value for msize
- * XXX if_maxlinkhdr already in mtu
  */
-#define SLIRP_MSIZE (IF_MTU + IF_MAXLINKHDR + offsetof(struct mbuf, m_dat) + 6)
+#define SLIRP_MSIZE\
+    (offsetof(struct mbuf, m_dat) + IF_MAXLINKHDR + TCPIPHDR_DELTA + IF_MTU)
 
 void
 m_init(Slirp *slirp)
 {
-    slirp->m_freelist.m_next = slirp->m_freelist.m_prev = &slirp->m_freelist;
-    slirp->m_usedlist.m_next = slirp->m_usedlist.m_prev = &slirp->m_usedlist;
+    slirp->m_freelist.qh_link = slirp->m_freelist.qh_rlink = &slirp->m_freelist;
+    slirp->m_usedlist.qh_link = slirp->m_usedlist.qh_rlink = &slirp->m_usedlist;
 }
 
 void m_cleanup(Slirp *slirp)
 {
     struct mbuf *m, *next;
 
-    m = slirp->m_usedlist.m_next;
-    while (m != &slirp->m_usedlist) {
+    m = (struct mbuf *) slirp->m_usedlist.qh_link;
+    while ((struct quehead *) m != &slirp->m_usedlist) {
         next = m->m_next;
         if (m->m_flags & M_EXT) {
             free(m->m_ext);
@@ -45,8 +46,8 @@ void m_cleanup(Slirp *slirp)
         free(m);
         m = next;
     }
-    m = slirp->m_freelist.m_next;
-    while (m != &slirp->m_freelist) {
+    m = (struct mbuf *) slirp->m_freelist.qh_link;
+    while ((struct quehead *) m != &slirp->m_freelist) {
         next = m->m_next;
         free(m);
         m = next;
@@ -69,7 +70,7 @@ m_get(Slirp *slirp)
 
 	DEBUG_CALL("m_get");
 
-	if (slirp->m_freelist.m_next == &slirp->m_freelist) {
+	if (slirp->m_freelist.qh_link == &slirp->m_freelist) {
 		m = (struct mbuf *)malloc(SLIRP_MSIZE);
 		if (m == NULL) goto end_error;
 		slirp->mbuf_alloced++;
@@ -77,7 +78,7 @@ m_get(Slirp *slirp)
 			flags = M_DOFREE;
 		m->slirp = slirp;
 	} else {
-		m = slirp->m_freelist.m_next;
+		m = (struct mbuf *) slirp->m_freelist.qh_link;
 		remque(m);
 	}
 
@@ -91,10 +92,10 @@ m_get(Slirp *slirp)
 	m->m_len = 0;
         m->m_nextpkt = NULL;
         m->m_prevpkt = NULL;
-        m->arp_requested = false;
+        m->resolution_requested = false;
         m->expiration_date = (uint64_t)-1;
 end_error:
-	DEBUG_ARG("m = %lx", (long )m);
+	DEBUG_ARG("m = %p", m);
 	return m;
 }
 
@@ -103,7 +104,7 @@ m_free(struct mbuf *m)
 {
 
   DEBUG_CALL("m_free");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
 
   if(m) {
 	/* Remove from m_usedlist */
@@ -221,10 +222,11 @@ dtom(Slirp *slirp, void *dat)
 	struct mbuf *m;
 
 	DEBUG_CALL("dtom");
-	DEBUG_ARG("dat = %lx", (long )dat);
+	DEBUG_ARG("dat = %p", dat);
 
 	/* bug corrected for M_EXT buffers */
-	for (m = slirp->m_usedlist.m_next; m != &slirp->m_usedlist;
+	for (m = (struct mbuf *) slirp->m_usedlist.qh_link;
+	     (struct quehead *) m != &slirp->m_usedlist;
 	     m = m->m_next) {
 	  if (m->m_flags & M_EXT) {
 	    if( (char *)dat>=m->m_ext && (char *)dat<(m->m_ext + m->m_size) )
diff --git a/qemu/slirp/mbuf.h b/qemu/slirp/mbuf.h
index b144f1ce3..36fb81409 100644
--- a/qemu/slirp/mbuf.h
+++ b/qemu/slirp/mbuf.h
@@ -79,13 +79,11 @@ struct mbuf {
 	int	m_len;			/* Amount of data in this mbuf */
 
 	Slirp *slirp;
-	bool	arp_requested;
+	bool	resolution_requested;
 	uint64_t expiration_date;
+	char   *m_ext;
 	/* start of dynamic buffer area, must be last element */
-	union {
-		char	m_dat[1]; /* ANSI don't like 0 sized arrays */
-		char	*m_ext;
-	};
+	char    m_dat[];
 };
 
 #define ifq_prev m_prev
diff --git a/qemu/slirp/misc.c b/qemu/slirp/misc.c
index 578e8b2c1..2fbd04856 100644
--- a/qemu/slirp/misc.c
+++ b/qemu/slirp/misc.c
@@ -5,6 +5,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include <libslirp.h>
 
@@ -16,11 +17,6 @@
 int slirp_debug = DBG_CALL|DBG_MISC|DBG_ERROR;
 #endif
 
-struct quehead {
-	struct quehead *qh_link;
-	struct quehead *qh_rlink;
-};
-
 inline void
 insque(void *a, void *b)
 {
@@ -123,9 +119,9 @@ fork_exec(struct socket *so, const char *ex, int do_pty)
 	pid_t pid;
 
 	DEBUG_CALL("fork_exec");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ex = %lx", (long)ex);
-	DEBUG_ARG("do_pty = %lx", (long)do_pty);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ex = %p", ex);
+	DEBUG_ARG("do_pty = %x", do_pty);
 
 	if (do_pty == 2) {
                 return 0;
diff --git a/qemu/slirp/misc.h b/qemu/slirp/misc.h
index 41a32583d..0d0c059e6 100644
--- a/qemu/slirp/misc.h
+++ b/qemu/slirp/misc.h
@@ -45,6 +45,11 @@ struct emu_t {
     struct emu_t *next;
 };
 
+struct slirp_quehead {
+    struct slirp_quehead *qh_link;
+    struct slirp_quehead *qh_rlink;
+};
+
 void slirp_insque(void *, void *);
 void slirp_remque(void *);
 int add_exec(struct ex_list **, int, char *, struct in_addr, int);
diff --git a/qemu/slirp/ndp_table.c b/qemu/slirp/ndp_table.c
new file mode 100644
index 000000000..9d4c39b45
--- /dev/null
+++ b/qemu/slirp/ndp_table.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+
+void ndp_table_add(Slirp *slirp, struct in6_addr ip_addr,
+                    uint8_t ethaddr[ETH_ALEN])
+{
+    NdpTable *ndp_table = &slirp->ndp_table;
+    int i;
+
+    DEBUG_CALL("ndp_table_add");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &(ip_addr), addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("ip = %s", addrstr);
+#endif
+    DEBUG_ARGS((dfd, " hw addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                ethaddr[0], ethaddr[1], ethaddr[2],
+                ethaddr[3], ethaddr[4], ethaddr[5]));
+
+    if (IN6_IS_ADDR_MULTICAST(&ip_addr) || IN6_IS_ADDR_UNSPECIFIED(&ip_addr)) {
+        /* Do not register multicast or unspecified addresses */
+        DEBUG_CALL(" abort: do not register multicast or unspecified address");
+        return;
+    }
+
+    /* Search for an entry */
+    for (i = 0; i < NDP_TABLE_SIZE; i++) {
+        if (in6_equal(&ndp_table->table[i].ip_addr, &ip_addr)) {
+            DEBUG_CALL(" already in table: update the entry");
+            /* Update the entry */
+            memcpy(ndp_table->table[i].eth_addr, ethaddr, ETH_ALEN);
+            return;
+        }
+    }
+
+    /* No entry found, create a new one */
+    DEBUG_CALL(" create new entry");
+    ndp_table->table[ndp_table->next_victim].ip_addr = ip_addr;
+    memcpy(ndp_table->table[ndp_table->next_victim].eth_addr,
+            ethaddr, ETH_ALEN);
+    ndp_table->next_victim = (ndp_table->next_victim + 1) % NDP_TABLE_SIZE;
+}
+
+bool ndp_table_search(Slirp *slirp, struct in6_addr ip_addr,
+                      uint8_t out_ethaddr[ETH_ALEN])
+{
+    NdpTable *ndp_table = &slirp->ndp_table;
+    int i;
+
+    DEBUG_CALL("ndp_table_search");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &(ip_addr), addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("ip = %s", addrstr);
+#endif
+
+    assert(!IN6_IS_ADDR_UNSPECIFIED(&ip_addr));
+
+    /* Multicast address: fec0::abcd:efgh/8 -> 33:33:ab:cd:ef:gh */
+    if (IN6_IS_ADDR_MULTICAST(&ip_addr)) {
+        out_ethaddr[0] = 0x33; out_ethaddr[1] = 0x33;
+        out_ethaddr[2] = ip_addr.s6_addr[12];
+        out_ethaddr[3] = ip_addr.s6_addr[13];
+        out_ethaddr[4] = ip_addr.s6_addr[14];
+        out_ethaddr[5] = ip_addr.s6_addr[15];
+        DEBUG_ARGS((dfd, " multicast addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                    out_ethaddr[0], out_ethaddr[1], out_ethaddr[2],
+                    out_ethaddr[3], out_ethaddr[4], out_ethaddr[5]));
+        return 1;
+    }
+
+    for (i = 0; i < NDP_TABLE_SIZE; i++) {
+        if (in6_equal(&ndp_table->table[i].ip_addr, &ip_addr)) {
+            memcpy(out_ethaddr, ndp_table->table[i].eth_addr,  ETH_ALEN);
+            DEBUG_ARGS((dfd, " found hw addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                        out_ethaddr[0], out_ethaddr[1], out_ethaddr[2],
+                        out_ethaddr[3], out_ethaddr[4], out_ethaddr[5]));
+            return 1;
+        }
+    }
+
+    DEBUG_CALL(" ip not found in table");
+    return 0;
+}
diff --git a/qemu/slirp/sbuf.c b/qemu/slirp/sbuf.c
index 08ec2b4f4..dd4cb8c13 100644
--- a/qemu/slirp/sbuf.c
+++ b/qemu/slirp/sbuf.c
@@ -5,6 +5,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include <qemu/main-loop.h>
 
@@ -72,8 +73,8 @@ sbappend(struct socket *so, struct mbuf *m)
 	int ret = 0;
 
 	DEBUG_CALL("sbappend");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m->m_len = %d", m->m_len);
 
 	/* Shouldn't happen, but...  e.g. foreign host closes connection */
diff --git a/qemu/slirp/slirp.c b/qemu/slirp/slirp.c
index 35f819afb..9f4bea3d3 100644
--- a/qemu/slirp/slirp.c
+++ b/qemu/slirp/slirp.c
@@ -21,11 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 #include "sysemu/char.h"
 #include "slirp.h"
 #include "hw/hw.h"
+#include "qemu/cutils.h"
 
 /* host loopback address */
 struct in_addr loopback_addr;
@@ -197,21 +200,29 @@ static void slirp_init_once(void)
 static void slirp_state_save(QEMUFile *f, void *opaque);
 static int slirp_state_load(QEMUFile *f, void *opaque, int version_id);
 
-Slirp *slirp_init(int restricted, struct in_addr vnetwork,
+Slirp *slirp_init(int restricted, bool in_enabled, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
-                  const char *vhostname, const char *tftp_path,
-                  const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, const char **vdnssearch,
+                  bool in6_enabled,
+                  struct in6_addr vprefix_addr6, uint8_t vprefix_len,
+                  struct in6_addr vhost6, const char *vhostname,
+                  const char *tftp_path, const char *bootfile,
+                  struct in_addr vdhcp_start, struct in_addr vnameserver,
+                  struct in6_addr vnameserver6, const char **vdnssearch,
                   void *opaque)
 {
     Slirp *slirp = g_malloc0(sizeof(Slirp));
 
     slirp_init_once();
 
+    slirp->grand = g_rand_new();
     slirp->restricted = restricted;
 
+    slirp->in_enabled = in_enabled;
+    slirp->in6_enabled = in6_enabled;
+
     if_init(slirp);
     ip_init(slirp);
+    ip6_init(slirp);
 
     /* Initialise mbufs *after* setting the MTU */
     m_init(slirp);
@@ -219,6 +230,9 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
     slirp->vnetwork_addr = vnetwork;
     slirp->vnetwork_mask = vnetmask;
     slirp->vhost_addr = vhost;
+    slirp->vprefix_addr6 = vprefix_addr6;
+    slirp->vprefix_len = vprefix_len;
+    slirp->vhost_addr6 = vhost6;
     if (vhostname) {
         pstrcpy(slirp->client_hostname, sizeof(slirp->client_hostname),
                 vhostname);
@@ -227,6 +241,7 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
     slirp->bootp_filename = g_strdup(bootfile);
     slirp->vdhcp_startaddr = vdhcp_start;
     slirp->vnameserver_addr = vnameserver;
+    slirp->vnameserver_addr6 = vnameserver6;
 
     if (vdnssearch) {
         translate_dnssearch(slirp, vdnssearch);
@@ -234,7 +249,7 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
 
     slirp->opaque = opaque;
 
-    register_savevm(NULL, "slirp", 0, 3,
+    register_savevm(NULL, "slirp", 0, 4,
                     slirp_state_save, slirp_state_load, slirp);
 
     QTAILQ_INSERT_TAIL(&slirp_instances, slirp, entry);
@@ -249,8 +264,11 @@ void slirp_cleanup(Slirp *slirp)
     unregister_savevm(NULL, "slirp", slirp);
 
     ip_cleanup(slirp);
+    ip6_cleanup(slirp);
     m_cleanup(slirp);
 
+    g_rand_free(slirp->grand);
+
     g_free(slirp->vdnssearch);
     g_free(slirp->tftp_prefix);
     g_free(slirp->bootp_filename);
@@ -516,7 +534,12 @@ void slirp_pollfds_poll(GArray *pollfds, int select_error)
                  * test for G_IO_IN below if this succeeds
                  */
                 if (revents & G_IO_PRI) {
-                    sorecvoob(so);
+                    ret = sorecvoob(so);
+                    if (ret < 0) {
+                        /* Socket error might have resulted in the socket being
+                         * removed, do not try to do anything more with it. */
+                        continue;
+                    }
                 }
                 /*
                  * Check sockets for reading
@@ -535,6 +558,11 @@ void slirp_pollfds_poll(GArray *pollfds, int select_error)
                     if (ret > 0) {
                         tcp_output(sototcpcb(so));
                     }
+                    if (ret < 0) {
+                        /* Socket error might have resulted in the socket being
+                         * removed, do not try to do anything more with it. */
+                        continue;
+                    }
                 }
 
                 /*
@@ -566,7 +594,8 @@ void slirp_pollfds_poll(GArray *pollfds, int select_error)
                         /*
                          * Continue tcp_input
                          */
-                        tcp_input((struct mbuf *)NULL, sizeof(struct ip), so);
+                        tcp_input((struct mbuf *)NULL, sizeof(struct ip), so,
+                                  so->so_ffamily);
                         /* continue; */
                     } else {
                         ret = sowrite(so);
@@ -615,7 +644,8 @@ void slirp_pollfds_poll(GArray *pollfds, int select_error)
                         }
 
                     }
-                    tcp_input((struct mbuf *)NULL, sizeof(struct ip), so);
+                    tcp_input((struct mbuf *)NULL, sizeof(struct ip), so,
+                              so->so_ffamily);
                 } /* SS_ISFCONNECTING */
 #endif
             }
@@ -678,6 +708,10 @@ static void arp_input(Slirp *slirp, const uint8_t *pkt, int pkt_len)
     int ar_op;
     struct ex_list *ex_ptr;
 
+    if (!slirp->in_enabled) {
+        return;
+    }
+
     ar_op = ntohs(ah->ar_op);
     switch(ar_op) {
     case ARPOP_REQUEST:
@@ -742,40 +776,42 @@ void slirp_input(Slirp *slirp, const uint8_t *pkt, int pkt_len)
         arp_input(slirp, pkt, pkt_len);
         break;
     case ETH_P_IP:
+    case ETH_P_IPV6:
         m = m_get(slirp);
         if (!m)
             return;
-        /* Note: we add to align the IP header */
-        if (M_FREEROOM(m) < pkt_len + 2) {
-            m_inc(m, pkt_len + 2);
+        /* Note: we add 2 to align the IP header on 4 bytes,
+         * and add the margin for the tcpiphdr overhead  */
+        if (M_FREEROOM(m) < pkt_len + TCPIPHDR_DELTA + 2) {
+            m_inc(m, pkt_len + TCPIPHDR_DELTA + 2);
         }
-        m->m_len = pkt_len + 2;
-        memcpy(m->m_data + 2, pkt, pkt_len);
+        m->m_len = pkt_len + TCPIPHDR_DELTA + 2;
+        memcpy(m->m_data + TCPIPHDR_DELTA + 2, pkt, pkt_len);
 
-        m->m_data += 2 + ETH_HLEN;
-        m->m_len -= 2 + ETH_HLEN;
+        m->m_data += TCPIPHDR_DELTA + 2 + ETH_HLEN;
+        m->m_len -= TCPIPHDR_DELTA + 2 + ETH_HLEN;
 
-        ip_input(m);
+        if (proto == ETH_P_IP) {
+            ip_input(m);
+        } else if (proto == ETH_P_IPV6) {
+            ip6_input(m);
+        }
         break;
+
     default:
         break;
     }
 }
 
-/* Output the IP packet to the ethernet device. Returns 0 if the packet must be
- * re-queued.
+/* Prepare the IPv4 packet to be sent to the ethernet device. Returns 1 if no
+ * packet should be sent, 0 if the packet must be re-queued, 2 if the packet
+ * is ready to go.
  */
-int if_encap(Slirp *slirp, struct mbuf *ifm)
+static int if_encap4(Slirp *slirp, struct mbuf *ifm, struct ethhdr *eh,
+        uint8_t ethaddr[ETH_ALEN])
 {
-    uint8_t buf[1600];
-    struct ethhdr *eh = (struct ethhdr *)buf;
-    uint8_t ethaddr[ETH_ALEN];
     const struct ip *iph = (const struct ip *)ifm->m_data;
 
-    if (ifm->m_len + ETH_HLEN > sizeof(buf)) {
-        return 1;
-    }
-
     if (iph->ip_dst.s_addr == 0) {
         /* 0.0.0.0 can not be a destination address, something went wrong,
          * avoid making it worse */
@@ -786,7 +822,7 @@ int if_encap(Slirp *slirp, struct mbuf *ifm)
         struct ethhdr *reh = (struct ethhdr *)arp_req;
         struct arphdr *rah = (struct arphdr *)(arp_req + ETH_HLEN);
 
-        if (!ifm->arp_requested) {
+        if (!ifm->resolution_requested) {
             /* If the client addr is not known, send an ARP request */
             memset(reh->h_dest, 0xff, ETH_ALEN);
             memcpy(reh->h_source, special_ethaddr, ETH_ALEN - 4);
@@ -812,22 +848,93 @@ int if_encap(Slirp *slirp, struct mbuf *ifm)
             rah->ar_tip = iph->ip_dst.s_addr;
             slirp->client_ipaddr = iph->ip_dst;
             slirp_output(slirp->opaque, arp_req, sizeof(arp_req));
-            ifm->arp_requested = true;
+            ifm->resolution_requested = true;
 
             /* Expire request and drop outgoing packet after 1 second */
             ifm->expiration_date = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + 1000000000ULL;
         }
         return 0;
     } else {
-        memcpy(eh->h_dest, ethaddr, ETH_ALEN);
         memcpy(eh->h_source, special_ethaddr, ETH_ALEN - 4);
         /* XXX: not correct */
         memcpy(&eh->h_source[2], &slirp->vhost_addr, 4);
         eh->h_proto = htons(ETH_P_IP);
-        memcpy(buf + sizeof(struct ethhdr), ifm->m_data, ifm->m_len);
-        slirp_output(slirp->opaque, buf, ifm->m_len + ETH_HLEN);
+
+        /* Send this */
+        return 2;
+    }
+}
+
+/* Prepare the IPv6 packet to be sent to the ethernet device. Returns 1 if no
+ * packet should be sent, 0 if the packet must be re-queued, 2 if the packet
+ * is ready to go.
+ */
+static int if_encap6(Slirp *slirp, struct mbuf *ifm, struct ethhdr *eh,
+        uint8_t ethaddr[ETH_ALEN])
+{
+    const struct ip6 *ip6h = mtod(ifm, const struct ip6 *);
+    if (!ndp_table_search(slirp, ip6h->ip_dst, ethaddr)) {
+        if (!ifm->resolution_requested) {
+            ndp_send_ns(slirp, ip6h->ip_dst);
+            ifm->resolution_requested = true;
+            ifm->expiration_date =
+                qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + 1000000000ULL;
+        }
+        return 0;
+    } else {
+        eh->h_proto = htons(ETH_P_IPV6);
+        in6_compute_ethaddr(ip6h->ip_src, eh->h_source);
+
+        /* Send this */
+        return 2;
+    }
+}
+
+/* Output the IP packet to the ethernet device. Returns 0 if the packet must be
+ * re-queued.
+ */
+int if_encap(Slirp *slirp, struct mbuf *ifm)
+{
+    uint8_t buf[1600];
+    struct ethhdr *eh = (struct ethhdr *)buf;
+    uint8_t ethaddr[ETH_ALEN];
+    const struct ip *iph = (const struct ip *)ifm->m_data;
+    int ret;
+
+    if (ifm->m_len + ETH_HLEN > sizeof(buf)) {
         return 1;
     }
+
+    switch (iph->ip_v) {
+    case IPVERSION:
+        ret = if_encap4(slirp, ifm, eh, ethaddr);
+        if (ret < 2) {
+            return ret;
+        }
+        break;
+
+    case IP6VERSION:
+        ret = if_encap6(slirp, ifm, eh, ethaddr);
+        if (ret < 2) {
+            return ret;
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+        break;
+    }
+
+    memcpy(eh->h_dest, ethaddr, ETH_ALEN);
+    DEBUG_ARGS((dfd, " src = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                eh->h_source[0], eh->h_source[1], eh->h_source[2],
+                eh->h_source[3], eh->h_source[4], eh->h_source[5]));
+    DEBUG_ARGS((dfd, " dst = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                eh->h_dest[0], eh->h_dest[1], eh->h_dest[2],
+                eh->h_dest[3], eh->h_dest[4], eh->h_dest[5]));
+    memcpy(buf + sizeof(struct ethhdr), ifm->m_data, ifm->m_len);
+    slirp_output(slirp->opaque, buf, ifm->m_len + ETH_HLEN);
+    return 1;
 }
 
 /* Drop host forwarding rule, return 0 if found. */
@@ -1011,10 +1118,26 @@ static void slirp_sbuf_save(QEMUFile *f, struct sbuf *sbuf)
 static void slirp_socket_save(QEMUFile *f, struct socket *so)
 {
     qemu_put_be32(f, so->so_urgc);
-    qemu_put_be32(f, so->so_faddr.s_addr);
-    qemu_put_be32(f, so->so_laddr.s_addr);
-    qemu_put_be16(f, so->so_fport);
-    qemu_put_be16(f, so->so_lport);
+    qemu_put_be16(f, so->so_ffamily);
+    switch (so->so_ffamily) {
+    case AF_INET:
+        qemu_put_be32(f, so->so_faddr.s_addr);
+        qemu_put_be16(f, so->so_fport);
+        break;
+    default:
+        error_report(
+                "so_ffamily unknown, unable to save so_faddr and so_fport\n");
+    }
+    qemu_put_be16(f, so->so_lfamily);
+    switch (so->so_lfamily) {
+    case AF_INET:
+        qemu_put_be32(f, so->so_laddr.s_addr);
+        qemu_put_be16(f, so->so_lport);
+        break;
+    default:
+        error_report(
+                "so_ffamily unknown, unable to save so_laddr and so_lport\n");
+    }
     qemu_put_byte(f, so->so_iptos);
     qemu_put_byte(f, so->so_emu);
     qemu_put_byte(f, so->so_type);
@@ -1128,16 +1251,40 @@ static int slirp_sbuf_load(QEMUFile *f, struct sbuf *sbuf)
     return 0;
 }
 
-static int slirp_socket_load(QEMUFile *f, struct socket *so)
+static int slirp_socket_load(QEMUFile *f, struct socket *so, int version_id)
 {
     if (tcp_attach(so) < 0)
         return -ENOMEM;
 
     so->so_urgc = qemu_get_be32(f);
-    so->so_faddr.s_addr = qemu_get_be32(f);
-    so->so_laddr.s_addr = qemu_get_be32(f);
-    so->so_fport = qemu_get_be16(f);
-    so->so_lport = qemu_get_be16(f);
+    if (version_id <= 3) {
+        so->so_ffamily = AF_INET;
+        so->so_faddr.s_addr = qemu_get_be32(f);
+        so->so_laddr.s_addr = qemu_get_be32(f);
+        so->so_fport = qemu_get_be16(f);
+        so->so_lport = qemu_get_be16(f);
+    } else {
+        so->so_ffamily = qemu_get_be16(f);
+        switch (so->so_ffamily) {
+        case AF_INET:
+            so->so_faddr.s_addr = qemu_get_be32(f);
+            so->so_fport = qemu_get_be16(f);
+            break;
+        default:
+            error_report(
+                "so_ffamily unknown, unable to restore so_faddr and so_lport");
+        }
+        so->so_lfamily = qemu_get_be16(f);
+        switch (so->so_lfamily) {
+        case AF_INET:
+            so->so_laddr.s_addr = qemu_get_be32(f);
+            so->so_lport = qemu_get_be16(f);
+            break;
+        default:
+            error_report(
+                "so_ffamily unknown, unable to restore so_laddr and so_lport");
+        }
+    }
     so->so_iptos = qemu_get_byte(f);
     so->so_emu = qemu_get_byte(f);
     so->so_type = qemu_get_byte(f);
@@ -1173,7 +1320,7 @@ static int slirp_state_load(QEMUFile *f, void *opaque, int version_id)
         if (!so)
             return -ENOMEM;
 
-        ret = slirp_socket_load(f, so);
+        ret = slirp_socket_load(f, so, version_id);
 
         if (ret < 0)
             return ret;
diff --git a/qemu/slirp/slirp.h b/qemu/slirp/slirp.h
index 6589d7eef..203deec48 100644
--- a/qemu/slirp/slirp.h
+++ b/qemu/slirp/slirp.h
@@ -1,11 +1,9 @@
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
-#include "config-host.h"
 #include "slirp_config.h"
 
 #ifdef _WIN32
-# include <inttypes.h>
 
 typedef char *caddr_t;
 
@@ -16,52 +14,20 @@ typedef char *caddr_t;
 # include <iphlpapi.h>
 
 #else
-# define ioctlsocket ioctl
-# define closesocket(s) close(s)
 # if !defined(__HAIKU__)
 #  define O_BINARY 0
 # endif
 #endif
 
-#include <sys/types.h>
 #ifdef HAVE_SYS_BITYPES_H
 # include <sys/bitypes.h>
 #endif
 
-#include <sys/time.h>
-
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-
-#ifdef HAVE_STDLIB_H
-# include <stdlib.h>
-#endif
-
-#include <stdio.h>
-#include <errno.h>
 
 #ifndef HAVE_MEMMOVE
 #define memmove(x, y, z) bcopy(y, x, z)
 #endif
 
-#if TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# ifdef HAVE_SYS_TIME_H
-#  include <sys/time.h>
-# else
-#  include <time.h>
-# endif
-#endif
-
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
-
 #ifndef _WIN32
 #include <sys/uio.h>
 #endif
@@ -82,11 +48,9 @@ void *malloc(size_t arg);
 void free(void *ptr);
 #endif
 
-#include <fcntl.h>
 #ifndef NO_UNIX_SOCKETS
 #include <sys/un.h>
 #endif
-#include <signal.h>
 #ifdef HAVE_SYS_SIGNAL_H
 # include <sys/signal.h>
 #endif
@@ -114,36 +78,34 @@ void free(void *ptr);
 #include <ppp/slirppp.h>
 #endif
 
-#ifdef __STDC__
-#include <stdarg.h>
-#else
-#include <varargs.h>
-#endif
-
-#include <sys/stat.h>
-
 /* Avoid conflicting with the libc insque() and remque(), which
    have different prototypes. */
 #define insque slirp_insque
 #define remque slirp_remque
+#define quehead slirp_quehead
 
 #ifdef HAVE_SYS_STROPTS_H
 #include <sys/stropts.h>
 #endif
 
+#include <glib.h>
+
 #include "debug.h"
 
 #include "qemu/queue.h"
 #include "qemu/sockets.h"
+#include "net/eth.h"
 
 #include "libslirp.h"
 #include "ip.h"
+#include "ip6.h"
 #include "tcp.h"
 #include "tcp_timer.h"
 #include "tcp_var.h"
 #include "tcpip.h"
 #include "udp.h"
 #include "ip_icmp.h"
+#include "ip6_icmp.h"
 #include "mbuf.h"
 #include "sbuf.h"
 #include "socket.h"
@@ -158,12 +120,6 @@ void free(void *ptr);
 #include "bootp.h"
 #include "tftp.h"
 
-#define ETH_ALEN 6
-#define ETH_HLEN 14
-
-#define ETH_P_IP  0x0800        /* Internet Protocol packet  */
-#define ETH_P_ARP 0x0806        /* Address Resolution packet */
-
 #define ARPOP_REQUEST 1         /* ARP request */
 #define ARPOP_REPLY   2         /* ARP reply   */
 
@@ -201,18 +157,41 @@ void arp_table_add(Slirp *slirp, uint32_t ip_addr, uint8_t ethaddr[ETH_ALEN]);
 bool arp_table_search(Slirp *slirp, uint32_t ip_addr,
                       uint8_t out_ethaddr[ETH_ALEN]);
 
+struct ndpentry {
+    unsigned char   eth_addr[ETH_ALEN];     /* sender hardware address */
+    struct in6_addr ip_addr;                /* sender IP address       */
+} QEMU_PACKED;
+
+#define NDP_TABLE_SIZE 16
+
+typedef struct NdpTable {
+    struct ndpentry table[NDP_TABLE_SIZE];
+    int next_victim;
+} NdpTable;
+
+void ndp_table_add(Slirp *slirp, struct in6_addr ip_addr,
+                   uint8_t ethaddr[ETH_ALEN]);
+bool ndp_table_search(Slirp *slirp, struct in6_addr ip_addr,
+                      uint8_t out_ethaddr[ETH_ALEN]);
+
 struct Slirp {
     QTAILQ_ENTRY(Slirp) entry;
     u_int time_fasttimo;
     u_int last_slowtimo;
     bool do_slowtimo;
 
+    bool in_enabled, in6_enabled;
+
     /* virtual network configuration */
     struct in_addr vnetwork_addr;
     struct in_addr vnetwork_mask;
     struct in_addr vhost_addr;
+    struct in6_addr vprefix_addr6;
+    uint8_t vprefix_len;
+    struct in6_addr vhost_addr6;
     struct in_addr vdhcp_startaddr;
     struct in_addr vnameserver_addr;
+    struct in6_addr vnameserver_addr6;
 
     struct in_addr client_ipaddr;
     char client_hostname[33];
@@ -221,12 +200,13 @@ struct Slirp {
     struct ex_list *exec_list;
 
     /* mbuf states */
-    struct mbuf m_freelist, m_usedlist;
+    struct quehead m_freelist;
+    struct quehead m_usedlist;
     int mbuf_alloced;
 
     /* if states */
-    struct mbuf if_fastq;   /* fast queue (for interactive data) */
-    struct mbuf if_batchq;  /* queue for non-interactive data */
+    struct quehead if_fastq;   /* fast queue (for interactive data) */
+    struct quehead if_batchq;  /* queue for non-interactive data */
     struct mbuf *next_m;    /* pointer to next mbuf to output */
     bool if_start_busy;     /* avoid if_start recursion */
 
@@ -259,6 +239,10 @@ struct Slirp {
     struct tftp_session tftp_sessions[TFTP_SESSIONS_MAX];
 
     ArpTable arp_table;
+    NdpTable ndp_table;
+
+    GRand *grand;
+    QEMUTimer *ra_timer;
 
     void *opaque;
 };
@@ -301,6 +285,7 @@ int translate_dnssearch(Slirp *s, const char ** names);
 
 /* cksum.c */
 int cksum(struct mbuf *m, int len);
+int ip6_cksum(struct mbuf *m);
 
 /* if.c */
 void if_init(Slirp *);
@@ -316,8 +301,16 @@ void ip_stripoptions(register struct mbuf *, struct mbuf *);
 /* ip_output.c */
 int ip_output(struct socket *, struct mbuf *);
 
+/* ip6_input.c */
+void ip6_init(Slirp *);
+void ip6_cleanup(Slirp *);
+void ip6_input(struct mbuf *);
+
+/* ip6_output */
+int ip6_output(struct socket *, struct mbuf *, int fast);
+
 /* tcp_input.c */
-void tcp_input(register struct mbuf *, int, struct socket *);
+void tcp_input(register struct mbuf *, int, struct socket *, unsigned short af);
 int tcp_mss(register struct tcpcb *, u_int);
 
 /* tcp_output.c */
@@ -328,11 +321,12 @@ void tcp_setpersist(register struct tcpcb *);
 void tcp_init(Slirp *);
 void tcp_cleanup(Slirp *);
 void tcp_template(struct tcpcb *);
-void tcp_respond(struct tcpcb *, register struct tcpiphdr *, register struct mbuf *, tcp_seq, tcp_seq, int);
+void tcp_respond(struct tcpcb *, register struct tcpiphdr *,
+        register struct mbuf *, tcp_seq, tcp_seq, int, unsigned short);
 struct tcpcb * tcp_newtcpcb(struct socket *);
 struct tcpcb * tcp_close(register struct tcpcb *);
 void tcp_sockclosed(struct tcpcb *);
-int tcp_fconnect(struct socket *);
+int tcp_fconnect(struct socket *, unsigned short af);
 void tcp_connect(struct socket *);
 int tcp_attach(struct socket *);
 uint8_t tcp_tos(struct socket *);
@@ -353,9 +347,4 @@ struct tcpcb *tcp_drop(struct tcpcb *tp, int err);
 #define max(x,y) ((x) > (y) ? (x) : (y))
 #endif
 
-#ifdef _WIN32
-#undef errno
-#define errno (WSAGetLastError())
-#endif
-
 #endif
diff --git a/qemu/slirp/socket.c b/qemu/slirp/socket.c
index 37ac5cf2f..a10eff18c 100644
--- a/qemu/slirp/socket.c
+++ b/qemu/slirp/socket.c
@@ -5,6 +5,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include <slirp.h>
 #include "ip_icmp.h"
@@ -15,24 +16,26 @@
 static void sofcantrcvmore(struct socket *so);
 static void sofcantsendmore(struct socket *so);
 
-struct socket *
-solookup(struct socket *head, struct in_addr laddr, u_int lport,
-         struct in_addr faddr, u_int fport)
+struct socket *solookup(struct socket **last, struct socket *head,
+        struct sockaddr_storage *lhost, struct sockaddr_storage *fhost)
 {
-	struct socket *so;
-
-	for (so = head->so_next; so != head; so = so->so_next) {
-		if (so->so_lport == lport &&
-		    so->so_laddr.s_addr == laddr.s_addr &&
-		    so->so_faddr.s_addr == faddr.s_addr &&
-		    so->so_fport == fport)
-		   break;
-	}
-
-	if (so == head)
-	   return (struct socket *)NULL;
-	return so;
+    struct socket *so = *last;
+
+    /* Optimisation */
+    if (so != head && sockaddr_equal(&(so->lhost.ss), lhost)
+            && (!fhost || sockaddr_equal(&so->fhost.ss, fhost))) {
+        return so;
+    }
+
+    for (so = head->so_next; so != head; so = so->so_next) {
+        if (sockaddr_equal(&(so->lhost.ss), lhost)
+                && (!fhost || sockaddr_equal(&so->fhost.ss, fhost))) {
+            *last = so;
+            return so;
+        }
+    }
 
+    return (struct socket *)NULL;
 }
 
 /*
@@ -91,7 +94,7 @@ size_t sopreprbuf(struct socket *so, struct iovec *iov, int *np)
 	int mss = so->so_tcpcb->t_maxseg;
 
 	DEBUG_CALL("sopreprbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	if (len <= 0)
 		return 0;
@@ -155,7 +158,7 @@ soread(struct socket *so)
 	struct iovec iov[2];
 
 	DEBUG_CALL("soread");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -173,9 +176,24 @@ soread(struct socket *so)
 		if (nn < 0 && (errno == EINTR || errno == EAGAIN))
 			return 0;
 		else {
+			int err;
+			socklen_t slen = sizeof err;
+
+			err = errno;
+			if (nn == 0) {
+				getsockopt(so->s, SOL_SOCKET, SO_ERROR,
+					   &err, &slen);
+			}
+
 			DEBUG_MISC((dfd, " --- soread() disconnected, nn = %d, errno = %d-%s\n", nn, errno,strerror(errno)));
 			sofcantrcvmore(so);
-			tcp_sockclosed(sototcpcb(so));
+
+			if (err == ECONNRESET || err == ECONNREFUSED
+			    || err == ENOTCONN || err == EPIPE) {
+				tcp_drop(sototcpcb(so), err);
+			} else {
+				tcp_sockclosed(sototcpcb(so));
+			}
 			return -1;
 		}
 	}
@@ -215,7 +233,7 @@ int soreadbuf(struct socket *so, const char *buf, int size)
 	struct iovec iov[2];
 
 	DEBUG_CALL("soreadbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -257,13 +275,14 @@ err:
  * so when OOB data arrives, we soread() it and everything
  * in the send buffer is sent as urgent data
  */
-void
+int
 sorecvoob(struct socket *so)
 {
 	struct tcpcb *tp = sototcpcb(so);
+	int ret;
 
 	DEBUG_CALL("sorecvoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * We take a guess at how much urgent data has arrived.
@@ -273,11 +292,15 @@ sorecvoob(struct socket *so)
 	 * urgent data, or the read() doesn't return all the
 	 * urgent data.
 	 */
-	soread(so);
-	tp->snd_up = tp->snd_una + so->so_snd.sb_cc;
-	tp->t_force = 1;
-	tcp_output(tp);
-	tp->t_force = 0;
+	ret = soread(so);
+	if (ret > 0) {
+	    tp->snd_up = tp->snd_una + so->so_snd.sb_cc;
+	    tp->t_force = 1;
+	    tcp_output(tp);
+	    tp->t_force = 0;
+	}
+
+	return ret;
 }
 
 /*
@@ -293,7 +316,7 @@ sosendoob(struct socket *so)
 	int n, len;
 
 	DEBUG_CALL("sosendoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 	DEBUG_ARG("sb->sb_cc = %d", sb->sb_cc);
 
 	if (so->so_urgc > 2048)
@@ -351,7 +374,7 @@ sowrite(struct socket *so)
 	struct iovec iov[2];
 
 	DEBUG_CALL("sowrite");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_urgc) {
 		sosendoob(so);
@@ -437,11 +460,12 @@ sowrite(struct socket *so)
 void
 sorecvfrom(struct socket *so)
 {
-	struct sockaddr_in addr;
-	socklen_t addrlen = sizeof(struct sockaddr_in);
+	struct sockaddr_storage addr;
+	struct sockaddr_storage saddr, daddr;
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
 
 	DEBUG_CALL("sorecvfrom");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_type == IPPROTO_ICMP) {   /* This is a "ping" reply */
 	  char buff[256];
@@ -459,7 +483,7 @@ sorecvfrom(struct socket *so)
 
 	    DEBUG_MISC((dfd," udp icmp rx errno = %d-%s\n",
 			errno,strerror(errno)));
-	    icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
+	    icmp_send_error(so->so_m, ICMP_UNREACH, code, 0, strerror(errno));
 	  } else {
 	    icmp_reflect(so->so_m);
             so->so_m = NULL; /* Don't m_free() it again! */
@@ -479,7 +503,18 @@ sorecvfrom(struct socket *so)
 	  if (!m) {
 	      return;
 	  }
-	  m->m_data += IF_MAXLINKHDR;
+	  switch (so->so_ffamily) {
+	  case AF_INET:
+	      m->m_data += IF_MAXLINKHDR + sizeof(struct udpiphdr);
+	      break;
+	  case AF_INET6:
+	      m->m_data += IF_MAXLINKHDR + sizeof(struct ip6)
+	                                 + sizeof(struct udphdr);
+	      break;
+	  default:
+	      g_assert_not_reached();
+	      break;
+	  }
 
 	  /*
 	   * XXX Shouldn't FIONREAD packets destined for port 53,
@@ -501,13 +536,37 @@ sorecvfrom(struct socket *so)
 	  DEBUG_MISC((dfd, " did recvfrom %d, errno = %d-%s\n",
 		      m->m_len, errno,strerror(errno)));
 	  if(m->m_len<0) {
-	    u_char code=ICMP_UNREACH_PORT;
-
-	    if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
-	    else if(errno == ENETUNREACH) code=ICMP_UNREACH_NET;
-
-	    DEBUG_MISC((dfd," rx error, tx icmp ICMP_UNREACH:%i\n", code));
-	    icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
+	    /* Report error as ICMP */
+	    switch (so->so_lfamily) {
+	    uint8_t code;
+	    case AF_INET:
+	      code = ICMP_UNREACH_PORT;
+
+	      if (errno == EHOSTUNREACH) {
+		code = ICMP_UNREACH_HOST;
+	      } else if (errno == ENETUNREACH) {
+		code = ICMP_UNREACH_NET;
+	      }
+
+	      DEBUG_MISC((dfd, " rx error, tx icmp ICMP_UNREACH:%i\n", code));
+	      icmp_send_error(so->so_m, ICMP_UNREACH, code, 0, strerror(errno));
+	      break;
+	    case AF_INET6:
+	      code = ICMP6_UNREACH_PORT;
+
+	      if (errno == EHOSTUNREACH) {
+		code = ICMP6_UNREACH_ADDRESS;
+	      } else if (errno == ENETUNREACH) {
+		code = ICMP6_UNREACH_NO_ROUTE;
+	      }
+
+	      DEBUG_MISC((dfd, " rx error, tx icmp6 ICMP_UNREACH:%i\n", code));
+	      icmp6_send_error(so->so_m, ICMP6_UNREACH, code);
+	      break;
+	    default:
+	      g_assert_not_reached();
+	      break;
+	    }
 	    m_free(m);
 	  } else {
 	  /*
@@ -525,9 +584,26 @@ sorecvfrom(struct socket *so)
 
 	    /*
 	     * If this packet was destined for CTL_ADDR,
-	     * make it look like that's where it came from, done by udp_output
+	     * make it look like that's where it came from
 	     */
-	    udp_output(so, m, &addr);
+	    saddr = addr;
+	    sotranslate_in(so, &saddr);
+	    daddr = so->lhost.ss;
+
+	    switch (so->so_ffamily) {
+	    case AF_INET:
+	        udp_output(so, m, (struct sockaddr_in *) &saddr,
+	                   (struct sockaddr_in *) &daddr,
+	                   so->so_iptos);
+	        break;
+	    case AF_INET6:
+	        udp6_output(so, m, (struct sockaddr_in6 *) &saddr,
+	                    (struct sockaddr_in6 *) &daddr);
+	        break;
+	    default:
+	        g_assert_not_reached();
+	        break;
+	    }
 	  } /* rx error */
 	} /* if ping packet */
 }
@@ -538,33 +614,20 @@ sorecvfrom(struct socket *so)
 int
 sosendto(struct socket *so, struct mbuf *m)
 {
-	Slirp *slirp = so->slirp;
 	int ret;
-	struct sockaddr_in addr;
+	struct sockaddr_storage addr;
 
 	DEBUG_CALL("sosendto");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
-
-        addr.sin_family = AF_INET;
-	if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-	    slirp->vnetwork_addr.s_addr) {
-	  /* It's an alias */
-	  if (so->so_faddr.s_addr == slirp->vnameserver_addr.s_addr) {
-	    if (get_dns_addr(&addr.sin_addr) < 0)
-	      addr.sin_addr = loopback_addr;
-	  } else {
-	    addr.sin_addr = loopback_addr;
-	  }
-	} else
-	  addr.sin_addr = so->so_faddr;
-	addr.sin_port = so->so_fport;
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
-	DEBUG_MISC((dfd, " sendto()ing, addr.sin_port=%d, addr.sin_addr.s_addr=%.16s\n", ntohs(addr.sin_port), inet_ntoa(addr.sin_addr)));
+	addr = so->fhost.ss;
+	DEBUG_CALL(" sendto()ing)");
+	sotranslate_out(so, &addr);
 
 	/* Don't care what port we get */
 	ret = sendto(so->s, m->m_data, m->m_len, 0,
-		     (struct sockaddr *)&addr, sizeof (struct sockaddr));
+		     (struct sockaddr *)&addr, sockaddr_size(&addr));
 	if (ret < 0)
 		return -1;
 
@@ -619,6 +682,7 @@ tcp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 
 	so->so_state &= SS_PERSISTENT_MASK;
 	so->so_state |= (SS_FACCEPTCONN | flags);
+	so->so_lfamily = AF_INET;
 	so->so_lport = lport; /* Kept in network format */
 	so->so_laddr.s_addr = laddr; /* Ditto */
 
@@ -645,6 +709,7 @@ tcp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 	qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
 
 	getsockname(s,(struct sockaddr *)&addr,&addrlen);
+	so->so_ffamily = AF_INET;
 	so->so_fport = addr.sin_port;
 	if (addr.sin_addr.s_addr == 0 || addr.sin_addr.s_addr == loopback_addr.s_addr)
 	   so->so_faddr = slirp->vhost_addr;
@@ -718,3 +783,113 @@ sofwdrain(struct socket *so)
 	else
 		sofcantsendmore(so);
 }
+
+/*
+ * Translate addr in host addr when it is a virtual address
+ */
+void sotranslate_out(struct socket *so, struct sockaddr_storage *addr)
+{
+    Slirp *slirp = so->slirp;
+    struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+
+    switch (addr->ss_family) {
+    case AF_INET:
+        if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+                slirp->vnetwork_addr.s_addr) {
+            /* It's an alias */
+            if (so->so_faddr.s_addr == slirp->vnameserver_addr.s_addr) {
+                if (get_dns_addr(&sin->sin_addr) < 0) {
+                    sin->sin_addr = loopback_addr;
+                }
+            } else {
+                sin->sin_addr = loopback_addr;
+            }
+        }
+
+        DEBUG_MISC((dfd, " addr.sin_port=%d, "
+            "addr.sin_addr.s_addr=%.16s\n",
+            ntohs(sin->sin_port), inet_ntoa(sin->sin_addr)));
+        break;
+
+    case AF_INET6:
+        if (in6_equal_net(&so->so_faddr6, &slirp->vprefix_addr6,
+                    slirp->vprefix_len)) {
+            if (in6_equal(&so->so_faddr6, &slirp->vnameserver_addr6)) {
+                /*if (get_dns_addr(&addr) < 0) {*/ /* TODO */
+                    sin6->sin6_addr = in6addr_loopback;
+                /*}*/
+            } else {
+                sin6->sin6_addr = in6addr_loopback;
+            }
+        }
+        break;
+
+    default:
+        break;
+    }
+}
+
+void sotranslate_in(struct socket *so, struct sockaddr_storage *addr)
+{
+    Slirp *slirp = so->slirp;
+    struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+
+    switch (addr->ss_family) {
+    case AF_INET:
+        if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+            slirp->vnetwork_addr.s_addr) {
+            uint32_t inv_mask = ~slirp->vnetwork_mask.s_addr;
+
+            if ((so->so_faddr.s_addr & inv_mask) == inv_mask) {
+                sin->sin_addr = slirp->vhost_addr;
+            } else if (sin->sin_addr.s_addr == loopback_addr.s_addr ||
+                       so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
+                sin->sin_addr = so->so_faddr;
+            }
+        }
+        break;
+
+    case AF_INET6:
+        if (in6_equal_net(&so->so_faddr6, &slirp->vprefix_addr6,
+                    slirp->vprefix_len)) {
+            if (in6_equal(&sin6->sin6_addr, &in6addr_loopback)
+                    || !in6_equal(&so->so_faddr6, &slirp->vhost_addr6)) {
+                sin6->sin6_addr = so->so_faddr6;
+            }
+        }
+        break;
+
+    default:
+        break;
+    }
+}
+
+/*
+ * Translate connections from localhost to the real hostname
+ */
+void sotranslate_accept(struct socket *so)
+{
+    Slirp *slirp = so->slirp;
+
+    switch (so->so_ffamily) {
+    case AF_INET:
+        if (so->so_faddr.s_addr == INADDR_ANY ||
+            (so->so_faddr.s_addr & loopback_mask) ==
+            (loopback_addr.s_addr & loopback_mask)) {
+           so->so_faddr = slirp->vhost_addr;
+        }
+        break;
+
+   case AF_INET6:
+        if (in6_equal(&so->so_faddr6, &in6addr_any) ||
+                in6_equal(&so->so_faddr6, &in6addr_loopback)) {
+           so->so_faddr6 = slirp->vhost_addr6;
+        }
+        break;
+
+    default:
+        break;
+    }
+}
diff --git a/qemu/slirp/socket.h b/qemu/slirp/socket.h
index 57e0407eb..b602e69b9 100644
--- a/qemu/slirp/socket.h
+++ b/qemu/slirp/socket.h
@@ -31,10 +31,27 @@ struct socket {
   struct tcpiphdr *so_ti;	   /* Pointer to the original ti within
 				    * so_mconn, for non-blocking connections */
   int so_urgc;
-  struct in_addr so_faddr;	   /* foreign host table entry */
-  struct in_addr so_laddr;	   /* local host table entry */
-  uint16_t so_fport;		   /* foreign port */
-  uint16_t so_lport;		   /* local port */
+  union {   /* foreign host */
+      struct sockaddr_storage ss;
+      struct sockaddr_in sin;
+      struct sockaddr_in6 sin6;
+  } fhost;
+#define so_faddr fhost.sin.sin_addr
+#define so_fport fhost.sin.sin_port
+#define so_faddr6 fhost.sin6.sin6_addr
+#define so_fport6 fhost.sin6.sin6_port
+#define so_ffamily fhost.ss.ss_family
+
+  union {   /* local host */
+      struct sockaddr_storage ss;
+      struct sockaddr_in sin;
+      struct sockaddr_in6 sin6;
+  } lhost;
+#define so_laddr lhost.sin.sin_addr
+#define so_lport lhost.sin.sin_port
+#define so_laddr6 lhost.sin6.sin6_addr
+#define so_lport6 lhost.sin6.sin6_port
+#define so_lfamily lhost.ss.ss_family
 
   uint8_t	so_iptos;	/* Type of service */
   uint8_t	so_emu;		/* Is the socket emulated? */
@@ -76,11 +93,53 @@ struct socket {
 #define SS_HOSTFWD		0x1000	/* Socket describes host->guest forwarding */
 #define SS_INCOMING		0x2000	/* Connection was initiated by a host on the internet */
 
-struct socket * solookup(struct socket *, struct in_addr, u_int, struct in_addr, u_int);
-struct socket * socreate(Slirp *);
+static inline int sockaddr_equal(struct sockaddr_storage *a,
+        struct sockaddr_storage *b)
+{
+    if (a->ss_family != b->ss_family) {
+        return 0;
+    }
+
+    switch (a->ss_family) {
+    case AF_INET:
+    {
+        struct sockaddr_in *a4 = (struct sockaddr_in *) a;
+        struct sockaddr_in *b4 = (struct sockaddr_in *) b;
+        return a4->sin_addr.s_addr == b4->sin_addr.s_addr
+               && a4->sin_port == b4->sin_port;
+    }
+    case AF_INET6:
+    {
+        struct sockaddr_in6 *a6 = (struct sockaddr_in6 *) a;
+        struct sockaddr_in6 *b6 = (struct sockaddr_in6 *) b;
+        return (in6_equal(&a6->sin6_addr, &b6->sin6_addr)
+                && a6->sin6_port == b6->sin6_port);
+    }
+    default:
+        g_assert_not_reached();
+    }
+
+    return 0;
+}
+
+static inline socklen_t sockaddr_size(struct sockaddr_storage *a)
+{
+    switch (a->ss_family) {
+    case AF_INET:
+        return sizeof(struct sockaddr_in);
+    case AF_INET6:
+        return sizeof(struct sockaddr_in6);
+    default:
+        g_assert_not_reached();
+    }
+}
+
+struct socket *solookup(struct socket **, struct socket *,
+        struct sockaddr_storage *, struct sockaddr_storage *);
+struct socket *socreate(Slirp *);
 void sofree(struct socket *);
 int soread(struct socket *);
-void sorecvoob(struct socket *);
+int sorecvoob(struct socket *);
 int sosendoob(struct socket *);
 int sowrite(struct socket *);
 void sorecvfrom(struct socket *);
@@ -94,4 +153,9 @@ struct iovec; /* For win32 */
 size_t sopreprbuf(struct socket *so, struct iovec *iov, int *np);
 int soreadbuf(struct socket *so, const char *buf, int size);
 
+void sotranslate_out(struct socket *, struct sockaddr_storage *);
+void sotranslate_in(struct socket *, struct sockaddr_storage *);
+void sotranslate_accept(struct socket *);
+
+
 #endif /* _SOCKET_H_ */
diff --git a/qemu/slirp/tcp.h b/qemu/slirp/tcp.h
index 2e2b4033a..61befcde5 100644
--- a/qemu/slirp/tcp.h
+++ b/qemu/slirp/tcp.h
@@ -106,6 +106,8 @@ struct tcphdr {
  */
 #undef TCP_MSS
 #define	TCP_MSS	1460
+#undef TCP6_MSS
+#define TCP6_MSS 1440
 
 #undef TCP_MAXWIN
 #define	TCP_MAXWIN	65535	/* largest value for (unscaled) window */
diff --git a/qemu/slirp/tcp_input.c b/qemu/slirp/tcp_input.c
index f946db8dc..e2b5d4ebb 100644
--- a/qemu/slirp/tcp_input.c
+++ b/qemu/slirp/tcp_input.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include "ip_icmp.h"
 
@@ -213,9 +214,10 @@ present:
  * protocol specification dated September, 1981 very closely.
  */
 void
-tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
+tcp_input(struct mbuf *m, int iphlen, struct socket *inso, unsigned short af)
 {
-  	struct ip save_ip, *ip;
+	struct ip save_ip, *ip;
+	struct ip6 save_ip6, *ip6;
 	register struct tcpiphdr *ti;
 	caddr_t optp = NULL;
 	int optlen = 0;
@@ -227,12 +229,15 @@ tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
 	int iss = 0;
 	u_long tiwin;
 	int ret;
+	struct sockaddr_storage lhost, fhost;
+	struct sockaddr_in *lhost4, *fhost4;
+	struct sockaddr_in6 *lhost6, *fhost6;
     struct ex_list *ex_ptr;
     Slirp *slirp;
 
 	DEBUG_CALL("tcp_input");
-	DEBUG_ARGS((dfd, " m = %8lx  iphlen = %2d  inso = %lx\n",
-		    (long )m, iphlen, (long )inso ));
+	DEBUG_ARGS((dfd, " m = %p  iphlen = %2d  inso = %p\n",
+		    m, iphlen, inso));
 
 	/*
 	 * If called with m == 0, then we're continuing the connect
@@ -253,37 +258,83 @@ tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
 	}
 	slirp = m->slirp;
 
-	/*
-	 * Get IP and TCP header together in first mbuf.
-	 * Note: IP leaves IP header in first mbuf.
-	 */
-	ti = mtod(m, struct tcpiphdr *);
-	if (iphlen > sizeof(struct ip )) {
-	  ip_stripoptions(m, (struct mbuf *)0);
-	  iphlen=sizeof(struct ip );
-	}
-	/* XXX Check if too short */
+	ip = mtod(m, struct ip *);
+	ip6 = mtod(m, struct ip6 *);
 
+	switch (af) {
+	case AF_INET:
+	    if (iphlen > sizeof(struct ip)) {
+	        ip_stripoptions(m, (struct mbuf *)0);
+	        iphlen = sizeof(struct ip);
+	    }
+	    /* XXX Check if too short */
 
-	/*
-	 * Save a copy of the IP header in case we want restore it
-	 * for sending an ICMP error message in response.
-	 */
-	ip=mtod(m, struct ip *);
-	save_ip = *ip;
-	save_ip.ip_len+= iphlen;
 
-	/*
-	 * Checksum extended TCP header and data.
-	 */
-	tlen = ((struct ip *)ti)->ip_len;
-        tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
-        memset(&ti->ti_i.ih_mbuf, 0 , sizeof(struct mbuf_ptr));
-	ti->ti_x1 = 0;
-	ti->ti_len = htons((uint16_t)tlen);
-	len = sizeof(struct ip ) + tlen;
-	if(cksum(m, len)) {
-	  goto drop;
+	    /*
+	     * Save a copy of the IP header in case we want restore it
+	     * for sending an ICMP error message in response.
+	     */
+	    save_ip = *ip;
+	    save_ip.ip_len += iphlen;
+
+	    /*
+	     * Get IP and TCP header together in first mbuf.
+	     * Note: IP leaves IP header in first mbuf.
+	     */
+	    m->m_data -= sizeof(struct tcpiphdr) - sizeof(struct ip)
+	                                         - sizeof(struct tcphdr);
+	    m->m_len += sizeof(struct tcpiphdr) - sizeof(struct ip)
+	                                        - sizeof(struct tcphdr);
+	    ti = mtod(m, struct tcpiphdr *);
+
+	    /*
+	     * Checksum extended TCP header and data.
+	     */
+	    tlen = ip->ip_len;
+	    tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
+	    memset(&ti->ih_mbuf, 0 , sizeof(struct mbuf_ptr));
+	    memset(&ti->ti, 0, sizeof(ti->ti));
+	    ti->ti_x0 = 0;
+	    ti->ti_src = save_ip.ip_src;
+	    ti->ti_dst = save_ip.ip_dst;
+	    ti->ti_pr = save_ip.ip_p;
+	    ti->ti_len = htons((uint16_t)tlen);
+	    break;
+
+	case AF_INET6:
+	    /*
+	     * Save a copy of the IP header in case we want restore it
+	     * for sending an ICMP error message in response.
+	     */
+	    save_ip6 = *ip6;
+	    /*
+	     * Get IP and TCP header together in first mbuf.
+	     * Note: IP leaves IP header in first mbuf.
+	     */
+	    m->m_data -= sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+	                                         + sizeof(struct tcphdr));
+	    m->m_len  += sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+	                                         + sizeof(struct tcphdr));
+	    ti = mtod(m, struct tcpiphdr *);
+
+	    tlen = ip6->ip_pl;
+	    tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
+	    memset(&ti->ih_mbuf, 0 , sizeof(struct mbuf_ptr));
+	    memset(&ti->ti, 0, sizeof(ti->ti));
+	    ti->ti_x0 = 0;
+	    ti->ti_src6 = save_ip6.ip_src;
+	    ti->ti_dst6 = save_ip6.ip_dst;
+	    ti->ti_nh6 = save_ip6.ip_nh;
+	    ti->ti_len = htons((uint16_t)tlen);
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
+
+	len = ((sizeof(struct tcpiphdr) - sizeof(struct tcphdr)) + tlen);
+	if (cksum(m, len)) {
+	    goto drop;
 	}
 
 	/*
@@ -320,17 +371,31 @@ tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
 	 * Locate pcb for segment.
 	 */
 findso:
-	so = slirp->tcp_last_so;
-	if (so->so_fport != ti->ti_dport ||
-	    so->so_lport != ti->ti_sport ||
-	    so->so_laddr.s_addr != ti->ti_src.s_addr ||
-	    so->so_faddr.s_addr != ti->ti_dst.s_addr) {
-		so = solookup(&slirp->tcb, ti->ti_src, ti->ti_sport,
-			       ti->ti_dst, ti->ti_dport);
-		if (so)
-			slirp->tcp_last_so = so;
+	lhost.ss_family = af;
+	fhost.ss_family = af;
+	switch (af) {
+	case AF_INET:
+	    lhost4 = (struct sockaddr_in *) &lhost;
+	    lhost4->sin_addr = ti->ti_src;
+	    lhost4->sin_port = ti->ti_sport;
+	    fhost4 = (struct sockaddr_in *) &fhost;
+	    fhost4->sin_addr = ti->ti_dst;
+	    fhost4->sin_port = ti->ti_dport;
+	    break;
+	case AF_INET6:
+	    lhost6 = (struct sockaddr_in6 *) &lhost;
+	    lhost6->sin6_addr = ti->ti_src6;
+	    lhost6->sin6_port = ti->ti_sport;
+	    fhost6 = (struct sockaddr_in6 *) &fhost;
+	    fhost6->sin6_addr = ti->ti_dst6;
+	    fhost6->sin6_port = ti->ti_dport;
+	    break;
+	default:
+	    g_assert_not_reached();
 	}
 
+	so = solookup(&slirp->tcp_last_so, &slirp->tcb, &lhost, &fhost);
+
 	/*
 	 * If the state is CLOSED (i.e., TCB does not exist) then
 	 * all data in the incoming segment is discarded.
@@ -374,13 +439,21 @@ findso:
 	  sbreserve(&so->so_snd, TCP_SNDSPACE);
 	  sbreserve(&so->so_rcv, TCP_RCVSPACE);
 
-	  so->so_laddr = ti->ti_src;
-	  so->so_lport = ti->ti_sport;
-	  so->so_faddr = ti->ti_dst;
-	  so->so_fport = ti->ti_dport;
-
-	  if ((so->so_iptos = tcp_tos(so)) == 0)
-	    so->so_iptos = ((struct ip *)ti)->ip_tos;
+	  so->lhost.ss = lhost;
+	  so->fhost.ss = fhost;
+
+	  so->so_iptos = tcp_tos(so);
+	  if (so->so_iptos == 0) {
+	      switch (af) {
+	      case AF_INET:
+	          so->so_iptos = ((struct ip *)ti)->ip_tos;
+	          break;
+	      case AF_INET6:
+	          break;
+	      default:
+	          g_assert_not_reached();
+	      }
+	  }
 
 	  tp = sototcpcb(so);
 	  tp->t_state = TCPS_LISTEN;
@@ -559,8 +632,9 @@ findso:
 	   * If this is destined for the control address, then flag to
 	   * tcp_ctl once connected, otherwise connect
 	   */
-	  if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-	      slirp->vnetwork_addr.s_addr) {
+	  if (af == AF_INET &&
+	         (so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+	         slirp->vnetwork_addr.s_addr) {
 	    if (so->so_faddr.s_addr != slirp->vhost_addr.s_addr &&
 		so->so_faddr.s_addr != slirp->vnameserver_addr.s_addr) {
 		/* May be an add exec */
@@ -584,24 +658,60 @@ findso:
 	    goto cont_input;
 	  }
 
-	  if((tcp_fconnect(so) == -1) && (errno != EINPROGRESS) && (errno != EWOULDBLOCK)) {
-	    u_char code=ICMP_UNREACH_NET;
+	  if ((tcp_fconnect(so, so->so_ffamily) == -1) &&
+              (errno != EAGAIN) &&
+              (errno != EINPROGRESS) && (errno != EWOULDBLOCK)
+          ) {
+	    uint8_t code;
 	    DEBUG_MISC((dfd, " tcp fconnect errno = %d-%s\n",
 			errno,strerror(errno)));
 	    if(errno == ECONNREFUSED) {
 	      /* ACK the SYN, send RST to refuse the connection */
-	      tcp_respond(tp, ti, m, ti->ti_seq+1, (tcp_seq)0,
-			  TH_RST|TH_ACK);
+	      tcp_respond(tp, ti, m, ti->ti_seq + 1, (tcp_seq) 0,
+			  TH_RST | TH_ACK, af);
 	    } else {
-	      if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
+	      switch (af) {
+	      case AF_INET:
+		code = ICMP_UNREACH_NET;
+		if (errno == EHOSTUNREACH) {
+		  code = ICMP_UNREACH_HOST;
+		}
+		break;
+	      case AF_INET6:
+		code = ICMP6_UNREACH_NO_ROUTE;
+		if (errno == EHOSTUNREACH) {
+		  code = ICMP6_UNREACH_ADDRESS;
+		}
+		break;
+	      default:
+		g_assert_not_reached();
+	      }
 	      HTONL(ti->ti_seq);             /* restore tcp header */
 	      HTONL(ti->ti_ack);
 	      HTONS(ti->ti_win);
 	      HTONS(ti->ti_urp);
 	      m->m_data -= sizeof(struct tcpiphdr)+off-sizeof(struct tcphdr);
 	      m->m_len  += sizeof(struct tcpiphdr)+off-sizeof(struct tcphdr);
-	      *ip=save_ip;
-	      icmp_error(m, ICMP_UNREACH,code, 0,strerror(errno));
+	      switch (af) {
+	      case AF_INET:
+		m->m_data += sizeof(struct tcpiphdr) - sizeof(struct ip)
+						     - sizeof(struct tcphdr);
+		m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct ip)
+						     - sizeof(struct tcphdr);
+		*ip = save_ip;
+		icmp_send_error(m, ICMP_UNREACH, code, 0, strerror(errno));
+		break;
+	      case AF_INET6:
+		m->m_data += sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+						     + sizeof(struct tcphdr));
+		m->m_len  -= sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+						     + sizeof(struct tcphdr));
+		*ip6 = save_ip6;
+		icmp6_send_error(m, ICMP6_UNREACH, code);
+		break;
+	      default:
+		g_assert_not_reached();
+	      }
 	    }
             tcp_close(tp);
 	    m_free(m);
@@ -616,6 +726,12 @@ findso:
 	    so->so_ti = ti;
 	    tp->t_timer[TCPT_KEEP] = TCPTV_KEEP_INIT;
 	    tp->t_state = TCPS_SYN_RECEIVED;
+	    /*
+	     * Initialize receive sequence numbers now so that we can send a
+	     * valid RST if the remote end rejects our connection.
+	     */
+	    tp->irs = ti->ti_seq;
+	    tcp_rcvseqinit(tp);
 	    tcp_template(tp);
 	  }
 	  return;
@@ -917,8 +1033,8 @@ trimthenstep6:
 
 		if (SEQ_LEQ(ti->ti_ack, tp->snd_una)) {
 			if (ti->ti_len == 0 && tiwin == tp->snd_wnd) {
-			  DEBUG_MISC((dfd, " dup ack  m = %lx  so = %lx\n",
-				      (long )m, (long )so));
+			  DEBUG_MISC((dfd, " dup ack  m = %p  so = %p\n",
+				      m, so));
 				/*
 				 * If we have outstanding data (other than
 				 * a window probe), this is a completely
@@ -1273,11 +1389,11 @@ dropafterack:
 dropwithreset:
 	/* reuses m if m!=NULL, m_free() unnecessary */
 	if (tiflags & TH_ACK)
-		tcp_respond(tp, ti, m, (tcp_seq)0, ti->ti_ack, TH_RST);
+		tcp_respond(tp, ti, m, (tcp_seq)0, ti->ti_ack, TH_RST, af);
 	else {
 		if (tiflags & TH_SYN) ti->ti_len++;
-		tcp_respond(tp, ti, m, ti->ti_seq+ti->ti_len, (tcp_seq)0,
-		    TH_RST|TH_ACK);
+		tcp_respond(tp, ti, m, ti->ti_seq + ti->ti_len, (tcp_seq) 0,
+		    TH_RST | TH_ACK, af);
 	}
 
 	return;
@@ -1296,7 +1412,7 @@ tcp_dooptions(struct tcpcb *tp, u_char *cp, int cnt, struct tcpiphdr *ti)
 	int opt, optlen;
 
 	DEBUG_CALL("tcp_dooptions");
-	DEBUG_ARGS((dfd, " tp = %lx  cnt=%i\n", (long)tp, cnt));
+	DEBUG_ARGS((dfd, " tp = %p  cnt=%i\n", tp, cnt));
 
 	for (; cnt > 0; cnt -= optlen, cp += optlen) {
 		opt = cp[0];
@@ -1377,7 +1493,7 @@ tcp_xmit_timer(register struct tcpcb *tp, int rtt)
 	register short delta;
 
 	DEBUG_CALL("tcp_xmit_timer");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("rtt = %d", rtt);
 
 	if (tp->t_srtt != 0) {
@@ -1465,10 +1581,22 @@ tcp_mss(struct tcpcb *tp, u_int offer)
 	int mss;
 
 	DEBUG_CALL("tcp_mss");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("offer = %d", offer);
 
-	mss = min(IF_MTU, IF_MRU) - sizeof(struct tcpiphdr);
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    mss = min(IF_MTU, IF_MRU) - sizeof(struct tcphdr)
+	                              + sizeof(struct ip);
+	    break;
+	case AF_INET6:
+	    mss = min(IF_MTU, IF_MRU) - sizeof(struct tcphdr)
+	                              + sizeof(struct ip6);
+	    break;
+	default:
+	    g_assert_not_reached();
+	}
+
 	if (offer)
 		mss = min(mss, offer);
 	mss = max(mss, 32);
diff --git a/qemu/slirp/tcp_output.c b/qemu/slirp/tcp_output.c
index 8aa3d9047..99b0a9b1c 100644
--- a/qemu/slirp/tcp_output.c
+++ b/qemu/slirp/tcp_output.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 static const u_char  tcp_outflags[TCP_NSTATES] = {
@@ -60,13 +61,15 @@ tcp_output(struct tcpcb *tp)
 	register long len, win;
 	int off, flags, error;
 	register struct mbuf *m;
-	register struct tcpiphdr *ti;
+	register struct tcpiphdr *ti, tcpiph_save;
+	struct ip *ip;
+	struct ip6 *ip6;
 	u_char opt[MAX_TCPOPTLEN];
 	unsigned optlen, hdrlen;
 	int idle, sendalot;
 
 	DEBUG_CALL("tcp_output");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/*
 	 * Determine length of data that should be transmitted,
@@ -446,16 +449,45 @@ send:
 	 * the template, but need a way to checksum without them.
 	 */
 	m->m_len = hdrlen + len; /* XXX Needed? m_len should be correct */
+	tcpiph_save = *mtod(m, struct tcpiphdr *);
+
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    ip = mtod(m, struct ip *);
+
+	    ip->ip_len = m->m_len;
+	    ip->ip_dst = tcpiph_save.ti_dst;
+	    ip->ip_src = tcpiph_save.ti_src;
+	    ip->ip_p = tcpiph_save.ti_pr;
+
+	    ip->ip_ttl = IPDEFTTL;
+	    ip->ip_tos = so->so_iptos;
+	    error = ip_output(so, m);
+	    break;
+
+	case AF_INET6:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    ip6 = mtod(m, struct ip6 *);
+
+	    ip6->ip_pl = tcpiph_save.ti_len;
+	    ip6->ip_dst = tcpiph_save.ti_dst6;
+	    ip6->ip_src = tcpiph_save.ti_src6;
+	    ip6->ip_nh = tcpiph_save.ti_nh6;
+
+	    error = ip6_output(so, m, 0);
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
 
-    {
-
-	((struct ip *)ti)->ip_len = m->m_len;
-
-	((struct ip *)ti)->ip_ttl = IPDEFTTL;
-	((struct ip *)ti)->ip_tos = so->so_iptos;
-
-	error = ip_output(so, m);
-    }
 	if (error) {
 out:
 		return (error);
diff --git a/qemu/slirp/tcp_subr.c b/qemu/slirp/tcp_subr.c
index 7571c5a28..6b9fef200 100644
--- a/qemu/slirp/tcp_subr.c
+++ b/qemu/slirp/tcp_subr.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 /* patchable/settable parameters for tcp */
@@ -75,13 +76,30 @@ tcp_template(struct tcpcb *tp)
 	register struct tcpiphdr *n = &tp->t_template;
 
 	n->ti_mbuf = NULL;
-	n->ti_x1 = 0;
-	n->ti_pr = IPPROTO_TCP;
-	n->ti_len = htons(sizeof (struct tcpiphdr) - sizeof (struct ip));
-	n->ti_src = so->so_faddr;
-	n->ti_dst = so->so_laddr;
-	n->ti_sport = so->so_fport;
-	n->ti_dport = so->so_lport;
+	memset(&n->ti, 0, sizeof(n->ti));
+	n->ti_x0 = 0;
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    n->ti_pr = IPPROTO_TCP;
+	    n->ti_len = htons(sizeof(struct tcphdr));
+	    n->ti_src = so->so_faddr;
+	    n->ti_dst = so->so_laddr;
+	    n->ti_sport = so->so_fport;
+	    n->ti_dport = so->so_lport;
+	    break;
+
+	case AF_INET6:
+	    n->ti_nh6 = IPPROTO_TCP;
+	    n->ti_len = htons(sizeof(struct tcphdr));
+	    n->ti_src6 = so->so_faddr6;
+	    n->ti_dst6 = so->so_laddr6;
+	    n->ti_sport = so->so_fport6;
+	    n->ti_dport = so->so_lport6;
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
 
 	n->ti_seq = 0;
 	n->ti_ack = 0;
@@ -108,7 +126,7 @@ tcp_template(struct tcpcb *tp)
  */
 void
 tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
-            tcp_seq ack, tcp_seq seq, int flags)
+            tcp_seq ack, tcp_seq seq, int flags, unsigned short af)
 {
 	register int tlen;
 	int win = 0;
@@ -130,6 +148,7 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 		m->m_data += IF_MAXLINKHDR;
 		*mtod(m, struct tcpiphdr *) = *ti;
 		ti = mtod(m, struct tcpiphdr *);
+		memset(&ti->ti, 0, sizeof(ti->ti));
 		flags = TH_ACK;
 	} else {
 		/*
@@ -141,16 +160,26 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 		m->m_len = sizeof (struct tcpiphdr);
 		tlen = 0;
 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
-		xchg(ti->ti_dst.s_addr, ti->ti_src.s_addr, uint32_t);
-		xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		switch (af) {
+		case AF_INET:
+		    xchg(ti->ti_dst.s_addr, ti->ti_src.s_addr, uint32_t);
+		    xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		    break;
+		case AF_INET6:
+		    xchg(ti->ti_dst6, ti->ti_src6, struct in6_addr);
+		    xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		    break;
+		default:
+		    g_assert_not_reached();
+		}
 #undef xchg
 	}
 	ti->ti_len = htons((u_short)(sizeof (struct tcphdr) + tlen));
 	tlen += sizeof (struct tcpiphdr);
 	m->m_len = tlen;
 
-        ti->ti_mbuf = NULL;
-	ti->ti_x1 = 0;
+	ti->ti_mbuf = NULL;
+	ti->ti_x0 = 0;
 	ti->ti_seq = htonl(seq);
 	ti->ti_ack = htonl(ack);
 	ti->ti_x2 = 0;
@@ -163,14 +192,49 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 	ti->ti_urp = 0;
 	ti->ti_sum = 0;
 	ti->ti_sum = cksum(m, tlen);
-	((struct ip *)ti)->ip_len = tlen;
-
-	if(flags & TH_RST)
-	  ((struct ip *)ti)->ip_ttl = MAXTTL;
-	else
-	  ((struct ip *)ti)->ip_ttl = IPDEFTTL;
 
-	(void) ip_output((struct socket *)0, m);
+	struct tcpiphdr tcpiph_save = *(mtod(m, struct tcpiphdr *));
+	struct ip *ip;
+	struct ip6 *ip6;
+
+	switch (af) {
+	case AF_INET:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    ip = mtod(m, struct ip *);
+	    ip->ip_len = tlen;
+	    ip->ip_dst = tcpiph_save.ti_dst;
+	    ip->ip_src = tcpiph_save.ti_src;
+	    ip->ip_p = tcpiph_save.ti_pr;
+
+	    if (flags & TH_RST) {
+	        ip->ip_ttl = MAXTTL;
+	    } else {
+	        ip->ip_ttl = IPDEFTTL;
+	    }
+
+	    ip_output(NULL, m);
+	    break;
+
+	case AF_INET6:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    ip6 = mtod(m, struct ip6 *);
+	    ip6->ip_pl = tlen;
+	    ip6->ip_dst = tcpiph_save.ti_dst6;
+	    ip6->ip_src = tcpiph_save.ti_src6;
+	    ip6->ip_nh = tcpiph_save.ti_nh6;
+
+	    ip6_output(NULL, m, 0);
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
 }
 
 /*
@@ -189,7 +253,7 @@ tcp_newtcpcb(struct socket *so)
 
 	memset((char *) tp, 0, sizeof(struct tcpcb));
 	tp->seg_next = tp->seg_prev = (struct tcpiphdr*)tp;
-	tp->t_maxseg = TCP_MSS;
+	tp->t_maxseg = (so->so_ffamily == AF_INET) ? TCP_MSS : TCP6_MSS;
 
 	tp->t_flags = TCP_DO_RFC1323 ? (TF_REQ_SCALE|TF_REQ_TSTMP) : 0;
 	tp->t_socket = so;
@@ -224,7 +288,7 @@ tcp_newtcpcb(struct socket *so)
 struct tcpcb *tcp_drop(struct tcpcb *tp, int err)
 {
 	DEBUG_CALL("tcp_drop");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("errno = %d", errno);
 
 	if (TCPS_HAVERCVDSYN(tp->t_state)) {
@@ -249,7 +313,7 @@ tcp_close(struct tcpcb *tp)
 	register struct mbuf *m;
 
 	DEBUG_CALL("tcp_close");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/* free the reassembly queue, if any */
 	t = tcpfrag_list_first(tp);
@@ -290,7 +354,11 @@ tcp_sockclosed(struct tcpcb *tp)
 {
 
 	DEBUG_CALL("tcp_sockclosed");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
+
+	if (!tp) {
+		return;
+	}
 
 	switch (tp->t_state) {
 
@@ -310,8 +378,7 @@ tcp_sockclosed(struct tcpcb *tp)
 		tp->t_state = TCPS_LAST_ACK;
 		break;
 	}
-	if (tp)
-		tcp_output(tp);
+	tcp_output(tp);
 }
 
 /*
@@ -324,42 +391,29 @@ tcp_sockclosed(struct tcpcb *tp)
  * nonblocking.  Connect returns after the SYN is sent, and does
  * not wait for ACK+SYN.
  */
-int tcp_fconnect(struct socket *so)
+int tcp_fconnect(struct socket *so, unsigned short af)
 {
-  Slirp *slirp = so->slirp;
   int ret=0;
 
   DEBUG_CALL("tcp_fconnect");
-  DEBUG_ARG("so = %lx", (long )so);
+  DEBUG_ARG("so = %p", so);
 
-  if( (ret = so->s = qemu_socket(AF_INET,SOCK_STREAM,0)) >= 0) {
+  ret = so->s = qemu_socket(af, SOCK_STREAM, 0);
+  if (ret >= 0) {
     int opt, s=so->s;
-    struct sockaddr_in addr;
+    struct sockaddr_storage addr;
 
     qemu_set_nonblock(s);
     socket_set_fast_reuse(s);
     opt = 1;
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(opt));
 
-    addr.sin_family = AF_INET;
-    if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-        slirp->vnetwork_addr.s_addr) {
-      /* It's an alias */
-      if (so->so_faddr.s_addr == slirp->vnameserver_addr.s_addr) {
-	if (get_dns_addr(&addr.sin_addr) < 0)
-	  addr.sin_addr = loopback_addr;
-      } else {
-	addr.sin_addr = loopback_addr;
-      }
-    } else
-      addr.sin_addr = so->so_faddr;
-    addr.sin_port = so->so_fport;
-
-    DEBUG_MISC((dfd, " connect()ing, addr.sin_port=%d, "
-		"addr.sin_addr.s_addr=%.16s\n",
-		ntohs(addr.sin_port), inet_ntoa(addr.sin_addr)));
+    addr = so->fhost.ss;
+    DEBUG_CALL(" connect()ing")
+    sotranslate_out(so, &addr);
+
     /* We don't care what port we get */
-    ret = connect(s,(struct sockaddr *)&addr,sizeof (addr));
+    ret = connect(s, (struct sockaddr *)&addr, sockaddr_size(&addr));
 
     /*
      * If it's not in progress, it failed, so we just return 0,
@@ -387,13 +441,13 @@ void tcp_connect(struct socket *inso)
 {
     Slirp *slirp = inso->slirp;
     struct socket *so;
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(struct sockaddr_in);
+    struct sockaddr_storage addr;
+    socklen_t addrlen = sizeof(struct sockaddr_storage);
     struct tcpcb *tp;
     int s, opt;
 
     DEBUG_CALL("tcp_connect");
-    DEBUG_ARG("inso = %lx", (long)inso);
+    DEBUG_ARG("inso = %p", inso);
 
     /*
      * If it's an SS_ACCEPTONCE socket, no need to socreate()
@@ -413,8 +467,8 @@ void tcp_connect(struct socket *inso)
             free(so); /* NOT sofree */
             return;
         }
-        so->so_laddr = inso->so_laddr;
-        so->so_lport = inso->so_lport;
+        so->lhost = inso->lhost;
+        so->so_ffamily = inso->so_ffamily;
     }
 
     tcp_mss(sototcpcb(so), 0);
@@ -430,14 +484,8 @@ void tcp_connect(struct socket *inso)
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
     socket_set_nodelay(s);
 
-    so->so_fport = addr.sin_port;
-    so->so_faddr = addr.sin_addr;
-    /* Translate connections from localhost to the real hostname */
-    if (so->so_faddr.s_addr == 0 ||
-        (so->so_faddr.s_addr & loopback_mask) ==
-        (loopback_addr.s_addr & loopback_mask)) {
-        so->so_faddr = slirp->vhost_addr;
-    }
+    so->fhost.ss = addr;
+    sotranslate_accept(so);
 
     /* Close the accept() socket, set right state */
     if (inso->so_state & SS_FACCEPTONCE) {
@@ -564,8 +612,8 @@ tcp_emu(struct socket *so, struct mbuf *m)
 	char *bptr;
 
 	DEBUG_CALL("tcp_emu");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
 	switch(so->so_emu) {
 		int x, i;
@@ -900,7 +948,7 @@ int tcp_ctl(struct socket *so)
     int do_pty;
 
     DEBUG_CALL("tcp_ctl");
-    DEBUG_ARG("so = %lx", (long )so);
+    DEBUG_ARG("so = %p", so);
 
     if (so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
         /* Check if it's pty_exec */
diff --git a/qemu/slirp/tcp_timer.c b/qemu/slirp/tcp_timer.c
index 6c5bb11cc..8f5dd772a 100644
--- a/qemu/slirp/tcp_timer.c
+++ b/qemu/slirp/tcp_timer.c
@@ -30,6 +30,7 @@
  * tcp_timer.c,v 1.2 1994/08/02 07:49:10 davidg Exp
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 
 static struct tcpcb *tcp_timers(register struct tcpcb *tp, int timer);
@@ -277,7 +278,8 @@ tcp_timers(register struct tcpcb *tp, int timer)
 			 * correspondent TCP to respond.
 			 */
 			tcp_respond(tp, &tp->t_template, (struct mbuf *)NULL,
-			    tp->rcv_nxt, tp->snd_una - 1, 0);
+			    tp->rcv_nxt, tp->snd_una - 1, 0,
+			    tp->t_socket->so_ffamily);
 			tp->t_timer[TCPT_KEEP] = TCPTV_KEEPINTVL;
 		} else
 			tp->t_timer[TCPT_KEEP] = TCPTV_KEEP_IDLE;
diff --git a/qemu/slirp/tcpip.h b/qemu/slirp/tcpip.h
index 7974ce3d5..124b4a9f6 100644
--- a/qemu/slirp/tcpip.h
+++ b/qemu/slirp/tcpip.h
@@ -37,15 +37,32 @@
  * Tcp+ip header, after ip options removed.
  */
 struct tcpiphdr {
-	struct 	ipovly ti_i;		/* overlaid ip structure */
-	struct	tcphdr ti_t;		/* tcp header */
+    struct mbuf_ptr ih_mbuf;	/* backpointer to mbuf */
+    union {
+        struct {
+            struct  in_addr ih_src; /* source internet address */
+            struct  in_addr ih_dst; /* destination internet address */
+            uint8_t ih_x1;          /* (unused) */
+            uint8_t ih_pr;          /* protocol */
+        } ti_i4;
+        struct {
+            struct  in6_addr ih_src;
+            struct  in6_addr ih_dst;
+            uint8_t ih_x1;
+            uint8_t ih_nh;
+        } ti_i6;
+    } ti;
+    uint16_t    ti_x0;
+    uint16_t    ti_len;             /* protocol length */
+    struct      tcphdr ti_t;        /* tcp header */
 };
-#define	ti_mbuf		ti_i.ih_mbuf.mptr
-#define	ti_x1		ti_i.ih_x1
-#define	ti_pr		ti_i.ih_pr
-#define	ti_len		ti_i.ih_len
-#define	ti_src		ti_i.ih_src
-#define	ti_dst		ti_i.ih_dst
+#define	ti_mbuf		ih_mbuf.mptr
+#define	ti_pr		ti.ti_i4.ih_pr
+#define	ti_src		ti.ti_i4.ih_src
+#define	ti_dst		ti.ti_i4.ih_dst
+#define	ti_src6		ti.ti_i6.ih_src
+#define	ti_dst6		ti.ti_i6.ih_dst
+#define	ti_nh6		ti.ti_i6.ih_nh
 #define	ti_sport	ti_t.th_sport
 #define	ti_dport	ti_t.th_dport
 #define	ti_seq		ti_t.th_seq
@@ -65,6 +82,13 @@ struct tcpiphdr {
 #define tcpfrag_list_end(F, T) (tcpiphdr2qlink(F) == (struct qlink*)(T))
 #define tcpfrag_list_empty(T) ((T)->seg_next == (struct tcpiphdr*)(T))
 
+/* This is the difference between the size of a tcpiphdr structure, and the
+ * size of actual ip+tcp headers, rounded up since we need to align data.  */
+#define TCPIPHDR_DELTA\
+    (max(0,\
+         (sizeof(struct tcpiphdr)\
+          - sizeof(struct ip) - sizeof(struct tcphdr) + 3) & ~3))
+
 /*
  * Just a clean way to get to the first byte
  * of the packet
diff --git a/qemu/slirp/tftp.c b/qemu/slirp/tftp.c
index a329fb281..12b5ff6e2 100644
--- a/qemu/slirp/tftp.c
+++ b/qemu/slirp/tftp.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 
 static inline int tftp_session_in_use(struct tftp_session *spt)
 {
@@ -45,7 +47,8 @@ static void tftp_session_terminate(struct tftp_session *spt)
     spt->slirp = NULL;
 }
 
-static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
+static int tftp_session_allocate(Slirp *slirp, struct sockaddr_storage *srcsas,
+                                 struct tftp_t *tp)
 {
   struct tftp_session *spt;
   int k;
@@ -67,7 +70,7 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
 
  found:
   memset(spt, 0, sizeof(*spt));
-  memcpy(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip));
+  spt->client_addr = *srcsas;
   spt->fd = -1;
   spt->client_port = tp->udp.uh_sport;
   spt->slirp = slirp;
@@ -77,7 +80,8 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
   return k;
 }
 
-static int tftp_session_find(Slirp *slirp, struct tftp_t *tp)
+static int tftp_session_find(Slirp *slirp, struct sockaddr_storage *srcsas,
+                             struct tftp_t *tp)
 {
   struct tftp_session *spt;
   int k;
@@ -86,7 +90,7 @@ static int tftp_session_find(Slirp *slirp, struct tftp_t *tp)
     spt = &slirp->tftp_sessions[k];
 
     if (tftp_session_in_use(spt)) {
-      if (!memcmp(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip))) {
+      if (sockaddr_equal(&spt->client_addr, srcsas)) {
 	if (spt->client_port == tp->udp.uh_sport) {
 	  return k;
 	}
@@ -119,11 +123,53 @@ static int tftp_read_data(struct tftp_session *spt, uint32_t block_nr,
     return bytes_read;
 }
 
+static struct tftp_t *tftp_prep_mbuf_data(struct tftp_session *spt,
+                                          struct mbuf *m)
+{
+    struct tftp_t *tp;
+
+    memset(m->m_data, 0, m->m_size);
+
+    m->m_data += IF_MAXLINKHDR;
+    if (spt->client_addr.ss_family == AF_INET6) {
+        m->m_data += sizeof(struct ip6);
+    } else {
+        m->m_data += sizeof(struct ip);
+    }
+    tp = (void *)m->m_data;
+    m->m_data += sizeof(struct udphdr);
+
+    return tp;
+}
+
+static void tftp_udp_output(struct tftp_session *spt, struct mbuf *m,
+                            struct tftp_t *recv_tp)
+{
+    if (spt->client_addr.ss_family == AF_INET6) {
+        struct sockaddr_in6 sa6, da6;
+
+        sa6.sin6_addr = spt->slirp->vhost_addr6;
+        sa6.sin6_port = recv_tp->udp.uh_dport;
+        da6.sin6_addr = ((struct sockaddr_in6 *)&spt->client_addr)->sin6_addr;
+        da6.sin6_port = spt->client_port;
+
+        udp6_output(NULL, m, &sa6, &da6);
+    } else {
+        struct sockaddr_in sa4, da4;
+
+        sa4.sin_addr = spt->slirp->vhost_addr;
+        sa4.sin_port = recv_tp->udp.uh_dport;
+        da4.sin_addr = ((struct sockaddr_in *)&spt->client_addr)->sin_addr;
+        da4.sin_port = spt->client_port;
+
+        udp_output(NULL, m, &sa4, &da4, IPTOS_LOWDELAY);
+    }
+}
+
 static int tftp_send_oack(struct tftp_session *spt,
                           const char *keys[], uint32_t values[], int nb,
                           struct tftp_t *recv_tp)
 {
-    struct sockaddr_in saddr, daddr;
     struct mbuf *m;
     struct tftp_t *tp;
     int i, n = 0;
@@ -131,13 +177,9 @@ static int tftp_send_oack(struct tftp_session *spt,
     m = m_get(spt->slirp);
 
     if (!m)
-	return -1;
-
-    memset(m->m_data, 0, m->m_size);
+        return -1;
 
-    m->m_data += IF_MAXLINKHDR;
-    tp = (void *)m->m_data;
-    m->m_data += sizeof(struct udpiphdr);
+    tp = tftp_prep_mbuf_data(spt, m);
 
     tp->tp_op = htons(TFTP_OACK);
     for (i = 0; i < nb; i++) {
@@ -147,15 +189,8 @@ static int tftp_send_oack(struct tftp_session *spt,
                       values[i]) + 1;
     }
 
-    saddr.sin_addr = recv_tp->ip.ip_dst;
-    saddr.sin_port = recv_tp->udp.uh_dport;
-
-    daddr.sin_addr = spt->client_ip;
-    daddr.sin_port = spt->client_port;
-
-    m->m_len = sizeof(struct tftp_t) - 514 + n -
-        sizeof(struct ip) - sizeof(struct udphdr);
-    udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+    m->m_len = sizeof(struct tftp_t) - 514 + n - sizeof(struct udphdr);
+    tftp_udp_output(spt, m, recv_tp);
 
     return 0;
 }
@@ -164,7 +199,6 @@ static void tftp_send_error(struct tftp_session *spt,
                             uint16_t errorcode, const char *msg,
                             struct tftp_t *recv_tp)
 {
-  struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
 
@@ -176,24 +210,15 @@ static void tftp_send_error(struct tftp_session *spt,
 
   memset(m->m_data, 0, m->m_size);
 
-  m->m_data += IF_MAXLINKHDR;
-  tp = (void *)m->m_data;
-  m->m_data += sizeof(struct udpiphdr);
+  tp = tftp_prep_mbuf_data(spt, m);
 
   tp->tp_op = htons(TFTP_ERROR);
   tp->x.tp_error.tp_error_code = htons(errorcode);
   pstrcpy((char *)tp->x.tp_error.tp_msg, sizeof(tp->x.tp_error.tp_msg), msg);
 
-  saddr.sin_addr = recv_tp->ip.ip_dst;
-  saddr.sin_port = recv_tp->udp.uh_dport;
-
-  daddr.sin_addr = spt->client_ip;
-  daddr.sin_port = spt->client_port;
-
-  m->m_len = sizeof(struct tftp_t) - 514 + 3 + strlen(msg) -
-        sizeof(struct ip) - sizeof(struct udphdr);
-
-  udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+  m->m_len = sizeof(struct tftp_t) - 514 + 3 + strlen(msg)
+             - sizeof(struct udphdr);
+  tftp_udp_output(spt, m, recv_tp);
 
 out:
   tftp_session_terminate(spt);
@@ -202,7 +227,6 @@ out:
 static void tftp_send_next_block(struct tftp_session *spt,
                                  struct tftp_t *recv_tp)
 {
-  struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
   int nobytes;
@@ -215,19 +239,11 @@ static void tftp_send_next_block(struct tftp_session *spt,
 
   memset(m->m_data, 0, m->m_size);
 
-  m->m_data += IF_MAXLINKHDR;
-  tp = (void *)m->m_data;
-  m->m_data += sizeof(struct udpiphdr);
+  tp = tftp_prep_mbuf_data(spt, m);
 
   tp->tp_op = htons(TFTP_DATA);
   tp->x.tp_data.tp_block_nr = htons((spt->block_nr + 1) & 0xffff);
 
-  saddr.sin_addr = recv_tp->ip.ip_dst;
-  saddr.sin_port = recv_tp->udp.uh_dport;
-
-  daddr.sin_addr = spt->client_ip;
-  daddr.sin_port = spt->client_port;
-
   nobytes = tftp_read_data(spt, spt->block_nr, tp->x.tp_data.tp_buf, 512);
 
   if (nobytes < 0) {
@@ -240,10 +256,8 @@ static void tftp_send_next_block(struct tftp_session *spt,
     return;
   }
 
-  m->m_len = sizeof(struct tftp_t) - (512 - nobytes) -
-        sizeof(struct ip) - sizeof(struct udphdr);
-
-  udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+  m->m_len = sizeof(struct tftp_t) - (512 - nobytes) - sizeof(struct udphdr);
+  tftp_udp_output(spt, m, recv_tp);
 
   if (nobytes == 512) {
     tftp_session_update(spt);
@@ -255,7 +269,8 @@ static void tftp_send_next_block(struct tftp_session *spt,
   spt->block_nr++;
 }
 
-static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_rrq(Slirp *slirp, struct sockaddr_storage *srcsas,
+                            struct tftp_t *tp, int pktlen)
 {
   struct tftp_session *spt;
   int s, k;
@@ -266,12 +281,12 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
   int nb_options = 0;
 
   /* check if a session already exists and if so terminate it */
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
   if (s >= 0) {
     tftp_session_terminate(&slirp->tftp_sessions[s]);
   }
 
-  s = tftp_session_allocate(slirp, tp);
+  s = tftp_session_allocate(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -396,11 +411,12 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
   tftp_send_next_block(spt, tp);
 }
 
-static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_ack(Slirp *slirp, struct sockaddr_storage *srcsas,
+                            struct tftp_t *tp, int pktlen)
 {
   int s;
 
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -409,11 +425,12 @@ static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
   tftp_send_next_block(&slirp->tftp_sessions[s], tp);
 }
 
-static void tftp_handle_error(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_error(Slirp *slirp, struct sockaddr_storage *srcsas,
+                              struct tftp_t *tp, int pktlen)
 {
   int s;
 
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -422,21 +439,21 @@ static void tftp_handle_error(Slirp *slirp, struct tftp_t *tp, int pktlen)
   tftp_session_terminate(&slirp->tftp_sessions[s]);
 }
 
-void tftp_input(struct mbuf *m)
+void tftp_input(struct sockaddr_storage *srcsas, struct mbuf *m)
 {
   struct tftp_t *tp = (struct tftp_t *)m->m_data;
 
   switch(ntohs(tp->tp_op)) {
   case TFTP_RRQ:
-    tftp_handle_rrq(m->slirp, tp, m->m_len);
+    tftp_handle_rrq(m->slirp, srcsas, tp, m->m_len);
     break;
 
   case TFTP_ACK:
-    tftp_handle_ack(m->slirp, tp, m->m_len);
+    tftp_handle_ack(m->slirp, srcsas, tp, m->m_len);
     break;
 
   case TFTP_ERROR:
-    tftp_handle_error(m->slirp, tp, m->m_len);
+    tftp_handle_error(m->slirp, srcsas, tp, m->m_len);
     break;
   }
 }
diff --git a/qemu/slirp/tftp.h b/qemu/slirp/tftp.h
index e1cc24b9b..1cb1adf59 100644
--- a/qemu/slirp/tftp.h
+++ b/qemu/slirp/tftp.h
@@ -16,7 +16,6 @@
 #define TFTP_FILENAME_MAX 512
 
 struct tftp_t {
-  struct ip ip;
   struct udphdr udp;
   uint16_t tp_op;
   union {
@@ -30,20 +29,20 @@ struct tftp_t {
     } tp_error;
     char tp_buf[512 + 2];
   } x;
-};
+} __attribute__((packed));
 
 struct tftp_session {
     Slirp *slirp;
     char *filename;
     int fd;
 
-    struct in_addr client_ip;
+    struct sockaddr_storage client_addr;
     uint16_t client_port;
     uint32_t block_nr;
 
     int timestamp;
 };
 
-void tftp_input(struct mbuf *m);
+void tftp_input(struct sockaddr_storage *srcsas, struct mbuf *m);
 
 #endif
diff --git a/qemu/slirp/udp.c b/qemu/slirp/udp.c
index f77e00f5a..247024fd8 100644
--- a/qemu/slirp/udp.c
+++ b/qemu/slirp/udp.c
@@ -38,6 +38,7 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include <slirp.h>
 #include "ip_icmp.h"
 
@@ -70,9 +71,11 @@ udp_input(register struct mbuf *m, int iphlen)
 	int len;
 	struct ip save_ip;
 	struct socket *so;
+	struct sockaddr_storage lhost;
+	struct sockaddr_in *lhost4;
 
 	DEBUG_CALL("udp_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("iphlen = %d", iphlen);
 
 	/*
@@ -125,6 +128,11 @@ udp_input(register struct mbuf *m, int iphlen)
 	  }
 	}
 
+	lhost.ss_family = AF_INET;
+	lhost4 = (struct sockaddr_in *) &lhost;
+	lhost4->sin_addr = ip->ip_src;
+	lhost4->sin_port = uh->uh_sport;
+
         /*
          *  handle DHCP/BOOTP
          */
@@ -140,7 +148,11 @@ udp_input(register struct mbuf *m, int iphlen)
          */
         if (ntohs(uh->uh_dport) == TFTP_SERVER &&
             ip->ip_dst.s_addr == slirp->vhost_addr.s_addr) {
-            tftp_input(m);
+            m->m_data += iphlen;
+            m->m_len -= iphlen;
+            tftp_input(&lhost, m);
+            m->m_data -= iphlen;
+            m->m_len += iphlen;
             goto bad;
         }
 
@@ -151,25 +163,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	/*
 	 * Locate pcb for datagram.
 	 */
-	so = slirp->udp_last_so;
-	if (so == &slirp->udb || so->so_lport != uh->uh_sport ||
-	    so->so_laddr.s_addr != ip->ip_src.s_addr) {
-		struct socket *tmp;
-
-		for (tmp = slirp->udb.so_next; tmp != &slirp->udb;
-		     tmp = tmp->so_next) {
-			if (tmp->so_lport == uh->uh_sport &&
-			    tmp->so_laddr.s_addr == ip->ip_src.s_addr) {
-				so = tmp;
-				break;
-			}
-		}
-		if (tmp == &slirp->udb) {
-		  so = NULL;
-		} else {
-		  slirp->udp_last_so = so;
-		}
-	}
+	so = solookup(&slirp->udp_last_so, &slirp->udb, &lhost, NULL);
 
 	if (so == NULL) {
 	  /*
@@ -180,7 +174,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	  if (!so) {
 	      goto bad;
 	  }
-	  if(udp_attach(so) == -1) {
+	  if (udp_attach(so, AF_INET) == -1) {
 	    DEBUG_MISC((dfd," udp_attach errno = %d-%s\n",
 			errno,strerror(errno)));
 	    sofree(so);
@@ -190,6 +184,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	  /*
 	   * Setup fields
 	   */
+	  so->so_lfamily = AF_INET;
 	  so->so_laddr = ip->ip_src;
 	  so->so_lport = uh->uh_sport;
 
@@ -202,6 +197,7 @@ udp_input(register struct mbuf *m, int iphlen)
 	   */
 	}
 
+        so->so_ffamily = AF_INET;
         so->so_faddr = ip->ip_dst; /* XXX */
         so->so_fport = uh->uh_dport; /* XXX */
 
@@ -217,7 +213,9 @@ udp_input(register struct mbuf *m, int iphlen)
 	  m->m_data -= iphlen;
 	  *ip=save_ip;
 	  DEBUG_MISC((dfd,"udp tx errno = %d-%s\n",errno,strerror(errno)));
-	  icmp_error(m, ICMP_UNREACH,ICMP_UNREACH_NET, 0,strerror(errno));
+	  icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0,
+	                  strerror(errno));
+	  goto bad;
 	}
 
 	m_free(so->so_m);   /* used for ICMP if error on sorecvfrom */
@@ -233,7 +231,7 @@ bad:
 	m_free(m);
 }
 
-int udp_output2(struct socket *so, struct mbuf *m,
+int udp_output(struct socket *so, struct mbuf *m,
                 struct sockaddr_in *saddr, struct sockaddr_in *daddr,
                 int iptos)
 {
@@ -241,8 +239,8 @@ int udp_output2(struct socket *so, struct mbuf *m,
 	int error = 0;
 
 	DEBUG_CALL("udp_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("saddr = %lx", (long)saddr->sin_addr.s_addr);
 	DEBUG_ARG("daddr = %lx", (long)daddr->sin_addr.s_addr);
 
@@ -284,35 +282,11 @@ int udp_output2(struct socket *so, struct mbuf *m,
 	return (error);
 }
 
-int udp_output(struct socket *so, struct mbuf *m,
-               struct sockaddr_in *addr)
-
-{
-    Slirp *slirp = so->slirp;
-    struct sockaddr_in saddr, daddr;
-
-    saddr = *addr;
-    if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-        slirp->vnetwork_addr.s_addr) {
-        uint32_t inv_mask = ~slirp->vnetwork_mask.s_addr;
-
-        if ((so->so_faddr.s_addr & inv_mask) == inv_mask) {
-            saddr.sin_addr = slirp->vhost_addr;
-        } else if (addr->sin_addr.s_addr == loopback_addr.s_addr ||
-                   so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
-            saddr.sin_addr = so->so_faddr;
-        }
-    }
-    daddr.sin_addr = so->so_laddr;
-    daddr.sin_port = so->so_lport;
-
-    return udp_output2(so, m, &saddr, &daddr, so->so_iptos);
-}
-
 int
-udp_attach(struct socket *so)
+udp_attach(struct socket *so, unsigned short af)
 {
-  if((so->s = qemu_socket(AF_INET,SOCK_DGRAM,0)) != -1) {
+  so->s = qemu_socket(af, SOCK_DGRAM, 0);
+  if (so->s != -1) {
     so->so_expire = curtime + SO_EXPIRE;
     insque(so, &so->slirp->udb);
   }
@@ -375,13 +349,9 @@ udp_listen(Slirp *slirp, uint32_t haddr, u_int hport, uint32_t laddr,
 	socket_set_fast_reuse(so->s);
 
 	getsockname(so->s,(struct sockaddr *)&addr,&addrlen);
-	so->so_fport = addr.sin_port;
-	if (addr.sin_addr.s_addr == 0 ||
-	    addr.sin_addr.s_addr == loopback_addr.s_addr) {
-	   so->so_faddr = slirp->vhost_addr;
-	} else {
-	   so->so_faddr = addr.sin_addr;
-	}
+	so->fhost.sin = addr;
+	sotranslate_accept(so);
+	so->so_lfamily = AF_INET;
 	so->so_lport = lport;
 	so->so_laddr.s_addr = laddr;
 	if (flags != SS_FACCEPTONCE)
diff --git a/qemu/slirp/udp.h b/qemu/slirp/udp.h
index 9bf31fe7b..10cc7809b 100644
--- a/qemu/slirp/udp.h
+++ b/qemu/slirp/udp.h
@@ -76,12 +76,16 @@ struct mbuf;
 void udp_init(Slirp *);
 void udp_cleanup(Slirp *);
 void udp_input(register struct mbuf *, int);
-int udp_output(struct socket *, struct mbuf *, struct sockaddr_in *);
-int udp_attach(struct socket *);
+int udp_attach(struct socket *, unsigned short af);
 void udp_detach(struct socket *);
 struct socket * udp_listen(Slirp *, uint32_t, u_int, uint32_t, u_int,
                            int);
-int udp_output2(struct socket *so, struct mbuf *m,
+int udp_output(struct socket *so, struct mbuf *m,
                 struct sockaddr_in *saddr, struct sockaddr_in *daddr,
                 int iptos);
+
+void udp6_input(register struct mbuf *);
+int udp6_output(struct socket *so, struct mbuf *m,
+                struct sockaddr_in6 *saddr, struct sockaddr_in6 *daddr);
+
 #endif
diff --git a/qemu/slirp/udp6.c b/qemu/slirp/udp6.c
new file mode 100644
index 000000000..a23026f2e
--- /dev/null
+++ b/qemu/slirp/udp6.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+#include "qemu/osdep.h"
+#include "udp.h"
+
+void udp6_input(struct mbuf *m)
+{
+    Slirp *slirp = m->slirp;
+    struct ip6 *ip, save_ip;
+    struct udphdr *uh;
+    int iphlen = sizeof(struct ip6);
+    int len;
+    struct socket *so;
+    struct sockaddr_in6 lhost;
+
+    DEBUG_CALL("udp6_input");
+    DEBUG_ARG("m = %lx", (long)m);
+
+    if (slirp->restricted) {
+        goto bad;
+    }
+
+    ip = mtod(m, struct ip6 *);
+    m->m_len -= iphlen;
+    m->m_data += iphlen;
+    uh = mtod(m, struct udphdr *);
+    m->m_len += iphlen;
+    m->m_data -= iphlen;
+
+    if (ip6_cksum(m)) {
+        goto bad;
+    }
+
+    len = ntohs((uint16_t)uh->uh_ulen);
+
+    /*
+     * Make mbuf data length reflect UDP length.
+     * If not enough data to reflect UDP length, drop.
+     */
+    if (ntohs(ip->ip_pl) != len) {
+        if (len > ntohs(ip->ip_pl)) {
+            goto bad;
+        }
+        m_adj(m, len - ntohs(ip->ip_pl));
+        ip->ip_pl = htons(len);
+    }
+
+    /*
+     * Save a copy of the IP header in case we want restore it
+     * for sending an ICMP error message in response.
+     */
+    save_ip = *ip;
+
+    /* Locate pcb for datagram. */
+    lhost.sin6_family = AF_INET6;
+    lhost.sin6_addr = ip->ip_src;
+    lhost.sin6_port = uh->uh_sport;
+
+    /* TODO handle DHCP/BOOTP */
+
+    /* handle TFTP */
+    if (ntohs(uh->uh_dport) == TFTP_SERVER &&
+        !memcmp(ip->ip_dst.s6_addr, slirp->vhost_addr6.s6_addr, 16)) {
+        m->m_data += iphlen;
+        m->m_len -= iphlen;
+        tftp_input((struct sockaddr_storage *)&lhost, m);
+        m->m_data -= iphlen;
+        m->m_len += iphlen;
+        goto bad;
+    }
+
+    so = solookup(&slirp->udp_last_so, &slirp->udb,
+                  (struct sockaddr_storage *) &lhost, NULL);
+
+    if (so == NULL) {
+        /* If there's no socket for this packet, create one. */
+        so = socreate(slirp);
+        if (!so) {
+            goto bad;
+        }
+        if (udp_attach(so, AF_INET6) == -1) {
+            DEBUG_MISC((dfd, " udp6_attach errno = %d-%s\n",
+                        errno, strerror(errno)));
+            sofree(so);
+            goto bad;
+        }
+
+        /* Setup fields */
+        so->so_lfamily = AF_INET6;
+        so->so_laddr6 = ip->ip_src;
+        so->so_lport6 = uh->uh_sport;
+    }
+
+    so->so_ffamily = AF_INET6;
+    so->so_faddr6 = ip->ip_dst; /* XXX */
+    so->so_fport6 = uh->uh_dport; /* XXX */
+
+    iphlen += sizeof(struct udphdr);
+    m->m_len -= iphlen;
+    m->m_data += iphlen;
+
+    /*
+     * Now we sendto() the packet.
+     */
+    if (sosendto(so, m) == -1) {
+        m->m_len += iphlen;
+        m->m_data -= iphlen;
+        *ip = save_ip;
+        DEBUG_MISC((dfd, "udp tx errno = %d-%s\n", errno, strerror(errno)));
+        icmp6_send_error(m, ICMP6_UNREACH, ICMP6_UNREACH_NO_ROUTE);
+        goto bad;
+    }
+
+    m_free(so->so_m);   /* used for ICMP if error on sorecvfrom */
+
+    /* restore the orig mbuf packet */
+    m->m_len += iphlen;
+    m->m_data -= iphlen;
+    *ip = save_ip;
+    so->so_m = m;
+
+    return;
+bad:
+    m_free(m);
+}
+
+int udp6_output(struct socket *so, struct mbuf *m,
+        struct sockaddr_in6 *saddr, struct sockaddr_in6 *daddr)
+{
+    struct ip6 *ip;
+    struct udphdr *uh;
+
+    DEBUG_CALL("udp6_output");
+    DEBUG_ARG("so = %lx", (long)so);
+    DEBUG_ARG("m = %lx", (long)m);
+
+    /* adjust for header */
+    m->m_data -= sizeof(struct udphdr);
+    m->m_len += sizeof(struct udphdr);
+    uh = mtod(m, struct udphdr *);
+    m->m_data -= sizeof(struct ip6);
+    m->m_len += sizeof(struct ip6);
+    ip = mtod(m, struct ip6 *);
+
+    /* Build IP header */
+    ip->ip_pl = htons(m->m_len - sizeof(struct ip6));
+    ip->ip_nh = IPPROTO_UDP;
+    ip->ip_src = saddr->sin6_addr;
+    ip->ip_dst = daddr->sin6_addr;
+
+    /* Build UDP header */
+    uh->uh_sport = saddr->sin6_port;
+    uh->uh_dport = daddr->sin6_port;
+    uh->uh_ulen = ip->ip_pl;
+    uh->uh_sum = 0;
+    uh->uh_sum = ip6_cksum(m);
+    if (uh->uh_sum == 0) {
+        uh->uh_sum = 0xffff;
+    }
+
+    return ip6_output(so, m, 0);
+}
diff --git a/qemu/softmmu_template.h b/qemu/softmmu_template.h
index d42d89d54..208f808f3 100644
--- a/qemu/softmmu_template.h
+++ b/qemu/softmmu_template.h
@@ -150,11 +150,11 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
     uint64_t val;
     CPUState *cpu = ENV_GET_CPU(env);
     hwaddr physaddr = iotlbentry->addr;
-    MemoryRegion *mr = iotlb_to_region(cpu, physaddr);
+    MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
     cpu->mem_io_pc = retaddr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
@@ -165,9 +165,6 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
 }
 #endif
 
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -252,9 +249,6 @@ WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
 }
 
 #if DATA_SIZE > 1
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -335,14 +329,6 @@ WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
 }
 #endif /* DATA_SIZE > 1 */
 
-DATA_TYPE
-glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    return helper_te_ld_name (env, addr, oi, GETRA());
-}
-
 #ifndef SOFTMMU_CODE_ACCESS
 
 /* Provide signed versions of the load routines as well.  We can of course
@@ -371,10 +357,10 @@ static inline void glue(io_write, SUFFIX)(CPUArchState *env,
 {
     CPUState *cpu = ENV_GET_CPU(env);
     hwaddr physaddr = iotlbentry->addr;
-    MemoryRegion *mr = iotlb_to_region(cpu, physaddr);
+    MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
@@ -540,14 +526,6 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
 }
 #endif /* DATA_SIZE > 1 */
 
-void
-glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         DATA_TYPE val, int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    helper_te_st_name(env, addr, val, oi, GETRA());
-}
-
 #if DATA_SIZE == 1
 /* Probe for whether the specified guest write access is permitted.
  * If it is not permitted then an exception will be taken in the same
diff --git a/qemu/spice-qemu-char.c b/qemu/spice-qemu-char.c
index d41bb745a..351fcaa03 100644
--- a/qemu/spice-qemu-char.c
+++ b/qemu/spice-qemu-char.c
@@ -1,11 +1,10 @@
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "ui/qemu-spice.h"
 #include "sysemu/char.h"
 #include <spice.h>
 #include <spice/protocol.h>
 
-#include "qemu/osdep.h"
 
 typedef struct SpiceCharDriver {
     CharDriverState*      chr;
@@ -271,13 +270,18 @@ static void spice_chr_accept_input(struct CharDriverState *chr)
 }
 
 static CharDriverState *chr_open(const char *subtype,
-    void (*set_fe_open)(struct CharDriverState *, int))
-
+                                 void (*set_fe_open)(struct CharDriverState *,
+                                                     int),
+                                 ChardevCommon *backend,
+                                 Error **errp)
 {
     CharDriverState *chr;
     SpiceCharDriver *s;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
     s = g_malloc0(sizeof(SpiceCharDriver));
     s->chr = chr;
     s->active = false;
@@ -296,15 +300,16 @@ static CharDriverState *chr_open(const char *subtype,
     return chr;
 }
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
+static CharDriverState *qemu_chr_open_spice_vmc(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
+                                                Error **errp)
 {
+    ChardevSpiceChannel *spicevmc = backend->u.spicevmc.data;
+    const char *type = spicevmc->type;
     const char **psubtype = spice_server_char_device_recognized_subtypes();
+    ChardevCommon *common = qapi_ChardevSpiceChannel_base(spicevmc);
 
-    if (type == NULL) {
-        fprintf(stderr, "spice-qemu-char: missing name parameter\n");
-        print_allowed_subtypes();
-        return NULL;
-    }
     for (; *psubtype != NULL; ++psubtype) {
         if (strcmp(type, *psubtype) == 0) {
             break;
@@ -316,12 +321,18 @@ CharDriverState *qemu_chr_open_spice_vmc(const char *type)
         return NULL;
     }
 
-    return chr_open(type, spice_vmc_set_fe_open);
+    return chr_open(type, spice_vmc_set_fe_open, common, errp);
 }
 
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
+static CharDriverState *qemu_chr_open_spice_port(const char *id,
+                                                 ChardevBackend *backend,
+                                                 ChardevReturn *ret,
+                                                 Error **errp)
 {
+    ChardevSpicePort *spiceport = backend->u.spiceport.data;
+    const char *name = spiceport->fqdn;
+    ChardevCommon *common = qapi_ChardevSpicePort_base(spiceport);
     CharDriverState *chr;
     SpiceCharDriver *s;
 
@@ -330,7 +341,10 @@ CharDriverState *qemu_chr_open_spice_port(const char *name)
         return NULL;
     }
 
-    chr = chr_open("port", spice_port_set_fe_open);
+    chr = chr_open("port", spice_port_set_fe_open, common, errp);
+    if (!chr) {
+        return NULL;
+    }
     s = chr->opaque;
     s->sin.portname = g_strdup(name);
 
@@ -354,34 +368,38 @@ static void qemu_chr_parse_spice_vmc(QemuOpts *opts, ChardevBackend *backend,
                                      Error **errp)
 {
     const char *name = qemu_opt_get(opts, "name");
+    ChardevSpiceChannel *spicevmc;
 
     if (name == NULL) {
         error_setg(errp, "chardev: spice channel: no name given");
         return;
     }
-    backend->spicevmc = g_new0(ChardevSpiceChannel, 1);
-    backend->spicevmc->type = g_strdup(name);
+    spicevmc = backend->u.spicevmc.data = g_new0(ChardevSpiceChannel, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSpiceChannel_base(spicevmc));
+    spicevmc->type = g_strdup(name);
 }
 
 static void qemu_chr_parse_spice_port(QemuOpts *opts, ChardevBackend *backend,
                                       Error **errp)
 {
     const char *name = qemu_opt_get(opts, "name");
+    ChardevSpicePort *spiceport;
 
     if (name == NULL) {
         error_setg(errp, "chardev: spice port: no name given");
         return;
     }
-    backend->spiceport = g_new0(ChardevSpicePort, 1);
-    backend->spiceport->fqdn = g_strdup(name);
+    spiceport = backend->u.spiceport.data = g_new0(ChardevSpicePort, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSpicePort_base(spiceport));
+    spiceport->fqdn = g_strdup(name);
 }
 
 static void register_types(void)
 {
     register_char_driver("spicevmc", CHARDEV_BACKEND_KIND_SPICEVMC,
-                         qemu_chr_parse_spice_vmc);
+                         qemu_chr_parse_spice_vmc, qemu_chr_open_spice_vmc);
     register_char_driver("spiceport", CHARDEV_BACKEND_KIND_SPICEPORT,
-                         qemu_chr_parse_spice_port);
+                         qemu_chr_parse_spice_port, qemu_chr_open_spice_port);
 }
 
 type_init(register_types);
diff --git a/qemu/stubs/Makefile.objs b/qemu/stubs/Makefile.objs
index 9937a1295..4b258a673 100644
--- a/qemu/stubs/Makefile.objs
+++ b/qemu/stubs/Makefile.objs
@@ -1,8 +1,7 @@
 stub-obj-y += arch-query-cpu-def.o
-stub-obj-y += bdrv-commit-all.o
-stub-obj-y += chr-baum-init.o
-stub-obj-y += chr-msmouse.o
-stub-obj-y += chr-testdev.o
+stub-obj-y += bdrv-next-monitor-owned.o
+stub-obj-y += blk-commit-all.o
+stub-obj-y += blockdev-close-all-bdrv-states.o
 stub-obj-y += clock-warp.o
 stub-obj-y += cpu-get-clock.o
 stub-obj-y += cpu-get-icount.o
@@ -23,18 +22,22 @@ stub-obj-y += mon-is-qmp.o
 stub-obj-y += mon-printf.o
 stub-obj-y += monitor-init.o
 stub-obj-y += notify-event.o
-stub-obj-$(CONFIG_SPICE) += qemu-chr-open-spice.o
 stub-obj-y += qtest.o
+stub-obj-y += replay.o
+stub-obj-y += replay-user.o
 stub-obj-y += reset.o
 stub-obj-y += runstate-check.o
 stub-obj-y += set-fd-handler.o
 stub-obj-y += slirp.o
 stub-obj-y += sysbus.o
 stub-obj-y += uuid.o
-stub-obj-y += vc-init.o
 stub-obj-y += vm-stop.o
 stub-obj-y += vmstate.o
 stub-obj-$(CONFIG_WIN32) += fd-register.o
 stub-obj-y += cpus.o
 stub-obj-y += kvm.o
 stub-obj-y += qmp_pc_dimm_device_list.o
+stub-obj-y += target-monitor-defs.o
+stub-obj-y += target-get-monitor-def.o
+stub-obj-y += vhost.o
+stub-obj-y += iohandler.o
diff --git a/qemu/stubs/arch-query-cpu-def.c b/qemu/stubs/arch-query-cpu-def.c
index a975ab453..cefe4beb8 100644
--- a/qemu/stubs/arch-query-cpu-def.c
+++ b/qemu/stubs/arch-query-cpu-def.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/arch_init.h"
 #include "qapi/qmp/qerror.h"
diff --git a/qemu/stubs/bdrv-commit-all.c b/qemu/stubs/bdrv-commit-all.c
deleted file mode 100644
index a8e0a9541..000000000
--- a/qemu/stubs/bdrv-commit-all.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "block/block.h"
-
-int bdrv_commit_all(void)
-{
-    return 0;
-}
diff --git a/qemu/stubs/bdrv-next-monitor-owned.c b/qemu/stubs/bdrv-next-monitor-owned.c
new file mode 100644
index 000000000..2acf6c3ec
--- /dev/null
+++ b/qemu/stubs/bdrv-next-monitor-owned.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/block.h"
+
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs)
+{
+    return NULL;
+}
diff --git a/qemu/stubs/blk-commit-all.c b/qemu/stubs/blk-commit-all.c
new file mode 100644
index 000000000..c82fb7f5f
--- /dev/null
+++ b/qemu/stubs/blk-commit-all.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/block-backend.h"
+
+int blk_commit_all(void)
+{
+    return 0;
+}
diff --git a/qemu/stubs/blockdev-close-all-bdrv-states.c b/qemu/stubs/blockdev-close-all-bdrv-states.c
new file mode 100644
index 000000000..f1f1d9cdc
--- /dev/null
+++ b/qemu/stubs/blockdev-close-all-bdrv-states.c
@@ -0,0 +1,6 @@
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+
+void blockdev_close_all_bdrv_states(void)
+{
+}
diff --git a/qemu/stubs/chr-baum-init.c b/qemu/stubs/chr-baum-init.c
deleted file mode 100644
index f5cc6ce1f..000000000
--- a/qemu/stubs/chr-baum-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_baum_init(void)
-{
-    return NULL;
-}
diff --git a/qemu/stubs/chr-msmouse.c b/qemu/stubs/chr-msmouse.c
deleted file mode 100644
index 812f8b0ab..000000000
--- a/qemu/stubs/chr-msmouse.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *qemu_chr_open_msmouse(void)
-{
-    return 0;
-}
diff --git a/qemu/stubs/chr-testdev.c b/qemu/stubs/chr-testdev.c
deleted file mode 100644
index 23112a2c0..000000000
--- a/qemu/stubs/chr-testdev.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_testdev_init(void)
-{
-    return 0;
-}
diff --git a/qemu/stubs/clock-warp.c b/qemu/stubs/clock-warp.c
index 5565118d1..8acb58a77 100644
--- a/qemu/stubs/clock-warp.c
+++ b/qemu/stubs/clock-warp.c
@@ -1,7 +1,8 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
-void qemu_clock_warp(QEMUClockType type)
+void qemu_start_warp_timer(void)
 {
 }
 
diff --git a/qemu/stubs/cpu-get-clock.c b/qemu/stubs/cpu-get-clock.c
index 5b34c976d..1d07523b8 100644
--- a/qemu/stubs/cpu-get-clock.c
+++ b/qemu/stubs/cpu-get-clock.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
diff --git a/qemu/stubs/cpu-get-icount.c b/qemu/stubs/cpu-get-icount.c
index d68585965..3a6f2ab00 100644
--- a/qemu/stubs/cpu-get-icount.c
+++ b/qemu/stubs/cpu-get-icount.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
diff --git a/qemu/stubs/cpus.c b/qemu/stubs/cpus.c
index 8e6f06b11..e19272297 100644
--- a/qemu/stubs/cpus.c
+++ b/qemu/stubs/cpus.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qom/cpu.h"
 
diff --git a/qemu/stubs/dump.c b/qemu/stubs/dump.c
index 8c24eda84..d9ee23f1e 100644
--- a/qemu/stubs/dump.c
+++ b/qemu/stubs/dump.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/dump-arch.h"
 #include "qmp-commands.h"
diff --git a/qemu/stubs/fd-register.c b/qemu/stubs/fd-register.c
index d0c34fd2a..f91aa3418 100644
--- a/qemu/stubs/fd-register.c
+++ b/qemu/stubs/fd-register.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
diff --git a/qemu/stubs/fdset-add-fd.c b/qemu/stubs/fdset-add-fd.c
index ee1643708..bf9e60aed 100644
--- a/qemu/stubs/fdset-add-fd.c
+++ b/qemu/stubs/fdset-add-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/fdset-find-fd.c b/qemu/stubs/fdset-find-fd.c
index 4f18344ba..1d9caf37e 100644
--- a/qemu/stubs/fdset-find-fd.c
+++ b/qemu/stubs/fdset-find-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/fdset-get-fd.c b/qemu/stubs/fdset-get-fd.c
index 7112c155e..5325044b5 100644
--- a/qemu/stubs/fdset-get-fd.c
+++ b/qemu/stubs/fdset-get-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/fdset-remove-fd.c b/qemu/stubs/fdset-remove-fd.c
index 7f6d61e61..47ea29721 100644
--- a/qemu/stubs/fdset-remove-fd.c
+++ b/qemu/stubs/fdset-remove-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/gdbstub.c b/qemu/stubs/gdbstub.c
index f6a4553a3..2b7aee50d 100644
--- a/qemu/stubs/gdbstub.c
+++ b/qemu/stubs/gdbstub.c
@@ -1,5 +1,4 @@
-#include "stdbool.h"            /* bool (in exec/gdbstub.h) */
-#include "stddef.h"             /* NULL */
+#include "qemu/osdep.h"
 #include "exec/gdbstub.h"       /* xml_builtin */
 
 const char *const xml_builtin[][2] = {
diff --git a/qemu/stubs/get-fd.c b/qemu/stubs/get-fd.c
index 9f2c65cf0..7dfdfb55f 100644
--- a/qemu/stubs/get-fd.c
+++ b/qemu/stubs/get-fd.c
@@ -1,3 +1,5 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/get-next-serial.c b/qemu/stubs/get-next-serial.c
index 40c56d13d..6ff6a6d3b 100644
--- a/qemu/stubs/get-next-serial.c
+++ b/qemu/stubs/get-next-serial.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 CharDriverState *serial_hds[0];
diff --git a/qemu/stubs/get-vm-name.c b/qemu/stubs/get-vm-name.c
index e5f619ffa..fa990136b 100644
--- a/qemu/stubs/get-vm-name.c
+++ b/qemu/stubs/get-vm-name.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 const char *qemu_get_vm_name(void)
diff --git a/qemu/stubs/iohandler.c b/qemu/stubs/iohandler.c
new file mode 100644
index 000000000..22b0ee5b0
--- /dev/null
+++ b/qemu/stubs/iohandler.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+AioContext *iohandler_get_aio_context(void)
+{
+    abort();
+}
diff --git a/qemu/stubs/iothread-lock.c b/qemu/stubs/iothread-lock.c
index dda6f6b58..9b6db2e74 100644
--- a/qemu/stubs/iothread-lock.c
+++ b/qemu/stubs/iothread-lock.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
diff --git a/qemu/stubs/is-daemonized.c b/qemu/stubs/is-daemonized.c
index c0ee9171a..d5cd1dc37 100644
--- a/qemu/stubs/is-daemonized.c
+++ b/qemu/stubs/is-daemonized.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 /* Win32 has its own inline stub */
diff --git a/qemu/stubs/kvm.c b/qemu/stubs/kvm.c
index e7c60b6e0..ddd620499 100644
--- a/qemu/stubs/kvm.c
+++ b/qemu/stubs/kvm.c
@@ -1,7 +1,8 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/kvm.h"
 
-int kvm_arch_irqchip_create(KVMState *s)
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
 {
     return 0;
 }
diff --git a/qemu/stubs/machine-init-done.c b/qemu/stubs/machine-init-done.c
index 28a92555b..9a0d62514 100644
--- a/qemu/stubs/machine-init-done.c
+++ b/qemu/stubs/machine-init-done.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 
diff --git a/qemu/stubs/migr-blocker.c b/qemu/stubs/migr-blocker.c
index 300df6e20..8ab3604df 100644
--- a/qemu/stubs/migr-blocker.c
+++ b/qemu/stubs/migr-blocker.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/migration.h"
 
diff --git a/qemu/stubs/mon-is-qmp.c b/qemu/stubs/mon-is-qmp.c
index dd26f19c8..a8344ced8 100644
--- a/qemu/stubs/mon-is-qmp.c
+++ b/qemu/stubs/mon-is-qmp.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/mon-printf.c b/qemu/stubs/mon-printf.c
index 0ce2ca692..e7c1e0cf7 100644
--- a/qemu/stubs/mon-printf.c
+++ b/qemu/stubs/mon-printf.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/monitor-init.c b/qemu/stubs/monitor-init.c
index 563902b41..de1bc7cd5 100644
--- a/qemu/stubs/monitor-init.c
+++ b/qemu/stubs/monitor-init.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/qemu/stubs/notify-event.c b/qemu/stubs/notify-event.c
index 32f7289d3..14e52268a 100644
--- a/qemu/stubs/notify-event.c
+++ b/qemu/stubs/notify-event.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
diff --git a/qemu/stubs/qemu-chr-open-spice.c b/qemu/stubs/qemu-chr-open-spice.c
deleted file mode 100644
index f1c4849d9..000000000
--- a/qemu/stubs/qemu-chr-open-spice.c
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "qemu-common.h"
-#include "ui/qemu-spice.h"
-
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
-{
-    return NULL;
-}
-
-#if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
-{
-    return NULL;
-}
-#endif
diff --git a/qemu/stubs/qmp_pc_dimm_device_list.c b/qemu/stubs/qmp_pc_dimm_device_list.c
index b584bd8b2..def211564 100644
--- a/qemu/stubs/qmp_pc_dimm_device_list.c
+++ b/qemu/stubs/qmp_pc_dimm_device_list.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "hw/mem/pc-dimm.h"
 
@@ -5,8 +6,3 @@ int qmp_pc_dimm_device_list(Object *obj, void *opaque)
 {
    return 0;
 }
-
-ram_addr_t get_current_ram_size(void)
-{
-    return ram_size;
-}
diff --git a/qemu/stubs/qtest.c b/qemu/stubs/qtest.c
index dc17594bb..891eb954f 100644
--- a/qemu/stubs/qtest.c
+++ b/qemu/stubs/qtest.c
@@ -8,7 +8,13 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/qtest.h"
 
 /* Needed for qtest_allowed() */
 bool qtest_allowed;
+
+bool qtest_driver(void)
+{
+    return false;
+}
diff --git a/qemu/stubs/replay-user.c b/qemu/stubs/replay-user.c
new file mode 100644
index 000000000..b29e7ebba
--- /dev/null
+++ b/qemu/stubs/replay-user.c
@@ -0,0 +1,33 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/replay.h"
+
+bool replay_exception(void)
+{
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    return false;
+}
+
+bool replay_interrupt(void)
+{
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    return false;
+}
diff --git a/qemu/stubs/replay.c b/qemu/stubs/replay.c
new file mode 100644
index 000000000..de9fa1ec9
--- /dev/null
+++ b/qemu/stubs/replay.c
@@ -0,0 +1,69 @@
+#include "qemu/osdep.h"
+#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
+
+ReplayMode replay_mode;
+
+int64_t replay_save_clock(unsigned int kind, int64_t clock)
+{
+    abort();
+    return 0;
+}
+
+int64_t replay_read_clock(unsigned int kind)
+{
+    abort();
+    return 0;
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    return true;
+}
+
+bool replay_events_enabled(void)
+{
+    return false;
+}
+
+void replay_finish(void)
+{
+}
+
+void replay_register_char_driver(CharDriverState *chr)
+{
+}
+
+void replay_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    abort();
+}
+
+void replay_char_write_event_save(int res, int offset)
+{
+    abort();
+}
+
+void replay_char_write_event_load(int *res, int *offset)
+{
+    abort();
+}
+
+int replay_char_read_all_load(uint8_t *buf)
+{
+    abort();
+}
+
+void replay_char_read_all_save_error(int res)
+{
+    abort();
+}
+
+void replay_char_read_all_save_buf(uint8_t *buf, int offset)
+{
+    abort();
+}
+
+void replay_block_event(QEMUBH *bh, uint64_t id)
+{
+}
diff --git a/qemu/stubs/reset.c b/qemu/stubs/reset.c
index ad287251e..5d47711f9 100644
--- a/qemu/stubs/reset.c
+++ b/qemu/stubs/reset.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 
 /* Stub functions for binaries that never call qemu_devices_reset(),
diff --git a/qemu/stubs/runstate-check.c b/qemu/stubs/runstate-check.c
index bd2e3757a..7c5227e84 100644
--- a/qemu/stubs/runstate-check.c
+++ b/qemu/stubs/runstate-check.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 
 bool runstate_check(RunState state)
diff --git a/qemu/stubs/set-fd-handler.c b/qemu/stubs/set-fd-handler.c
index a8481bc3c..06a5da48f 100644
--- a/qemu/stubs/set-fd-handler.c
+++ b/qemu/stubs/set-fd-handler.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
@@ -8,3 +9,13 @@ void qemu_set_fd_handler(int fd,
 {
     abort();
 }
+
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        bool is_external,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        void *opaque)
+{
+    abort();
+}
diff --git a/qemu/stubs/slirp.c b/qemu/stubs/slirp.c
index bd0ac7f27..dcae51f0e 100644
--- a/qemu/stubs/slirp.c
+++ b/qemu/stubs/slirp.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "slirp/slirp.h"
 
diff --git a/qemu/stubs/sysbus.c b/qemu/stubs/sysbus.c
index e13496582..d8da90caa 100644
--- a/qemu/stubs/sysbus.c
+++ b/qemu/stubs/sysbus.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/qdev-core.h"
 
 BusState *sysbus_get_default(void)
diff --git a/qemu/stubs/target-get-monitor-def.c b/qemu/stubs/target-get-monitor-def.c
new file mode 100644
index 000000000..394e0f9a7
--- /dev/null
+++ b/qemu/stubs/target-get-monitor-def.c
@@ -0,0 +1,29 @@
+/*
+ *  Stub for target_get_monitor_def.
+ *
+ *  Copyright IBM Corp., 2015
+ *
+ *  Author: Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    return -1;
+}
diff --git a/qemu/stubs/target-monitor-defs.c b/qemu/stubs/target-monitor-defs.c
new file mode 100644
index 000000000..ac07b1906
--- /dev/null
+++ b/qemu/stubs/target-monitor-defs.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+
+const MonitorDef *target_monitor_defs(void);
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return NULL;
+}
diff --git a/qemu/stubs/uuid.c b/qemu/stubs/uuid.c
index ffc0ed40a..92ad71783 100644
--- a/qemu/stubs/uuid.c
+++ b/qemu/stubs/uuid.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
diff --git a/qemu/stubs/vc-init.c b/qemu/stubs/vc-init.c
deleted file mode 100644
index 308dfa080..000000000
--- a/qemu/stubs/vc-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *vc_init(ChardevVC *vc)
-{
-    return 0;
-}
diff --git a/qemu/stubs/vhost.c b/qemu/stubs/vhost.c
new file mode 100644
index 000000000..2d76cdebd
--- /dev/null
+++ b/qemu/stubs/vhost.c
@@ -0,0 +1,7 @@
+#include "qemu/osdep.h"
+#include "hw/virtio/vhost.h"
+
+bool vhost_has_free_slot(void)
+{
+    return true;
+}
diff --git a/qemu/stubs/vm-stop.c b/qemu/stubs/vm-stop.c
index 69fd86b2e..8271cad65 100644
--- a/qemu/stubs/vm-stop.c
+++ b/qemu/stubs/vm-stop.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 
diff --git a/qemu/stubs/vmstate.c b/qemu/stubs/vmstate.c
index 778bc3fc6..65906271d 100644
--- a/qemu/stubs/vmstate.c
+++ b/qemu/stubs/vmstate.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
 
diff --git a/qemu/target-alpha/cpu.c b/qemu/target-alpha/cpu.c
index 421d7e536..8a155cae9 100644
--- a/qemu/target-alpha/cpu.c
+++ b/qemu/target-alpha/cpu.c
@@ -19,6 +19,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -46,6 +48,12 @@ static bool alpha_cpu_has_work(CPUState *cs)
                                     | CPU_INTERRUPT_MCHK);
 }
 
+static void alpha_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_alpha_ev6;
+    info->print_insn = print_insn_alpha;
+}
+
 static void alpha_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -297,7 +305,16 @@ static void alpha_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = alpha_cpu_get_phys_page_debug;
     dc->vmsd = &vmstate_alpha_cpu;
 #endif
+    cc->disas_set_info = alpha_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 67;
+
+    /*
+     * Reason: alpha_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo alpha_cpu_type_info = {
diff --git a/qemu/target-alpha/cpu.h b/qemu/target-alpha/cpu.h
index 91c56d6bc..420f2a53f 100644
--- a/qemu/target-alpha/cpu.h
+++ b/qemu/target-alpha/cpu.h
@@ -20,7 +20,6 @@
 #if !defined (__CPU_ALPHA_H__)
 #define __CPU_ALPHA_H__
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 64
@@ -32,8 +31,6 @@
 
 #include "fpu/softfloat.h"
 
-#define ELF_MACHINE     EM_ALPHA
-
 #define ICACHE_LINE_SIZE 32
 #define DCACHE_LINE_SIZE 32
 
@@ -289,7 +286,6 @@ struct CPUAlphaState {
 
 #define cpu_list alpha_cpu_list
 #define cpu_exec cpu_alpha_exec
-#define cpu_gen_code cpu_alpha_gen_code
 #define cpu_signal_handler cpu_alpha_signal_handler
 
 #include "exec/cpu-all.h"
@@ -376,7 +372,7 @@ enum {
     PS_USER_MODE = 8
 };
 
-static inline int cpu_mmu_index(CPUAlphaState *env)
+static inline int cpu_mmu_index(CPUAlphaState *env, bool ifetch)
 {
     if (env->pal_mode) {
         return MMU_KERNEL_IDX;
@@ -445,8 +441,9 @@ void QEMU_NORETURN arith_excp(CPUAlphaState *, uintptr_t, int, uint64_t);
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env);
 void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val);
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg);
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val);
 #ifndef CONFIG_USER_ONLY
-void swap_shadow_regs(CPUAlphaState *env);
 QEMU_NORETURN void alpha_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
                                                bool is_write, bool is_exec,
                                                int unused, unsigned size);
diff --git a/qemu/target-alpha/fpu_helper.c b/qemu/target-alpha/fpu_helper.c
index b091aa842..5ab7d5e64 100644
--- a/qemu/target-alpha/fpu_helper.c
+++ b/qemu/target-alpha/fpu_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
@@ -437,7 +438,7 @@ uint64_t helper_cvtqs(CPUAlphaState *env, uint64_t a)
     return float32_to_s(fr);
 }
 
-/* Implement float64 to uint64 conversion without saturation -- we must
+/* Implement float64 to uint64_t conversion without saturation -- we must
    supply the truncated result.  This behaviour is used by the compiler
    to get unsigned conversion for free with the same instruction.  */
 
diff --git a/qemu/target-alpha/gdbstub.c b/qemu/target-alpha/gdbstub.c
index 980f140e7..199f02842 100644
--- a/qemu/target-alpha/gdbstub.c
+++ b/qemu/target-alpha/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
@@ -30,7 +30,7 @@ int alpha_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
 
     switch (n) {
     case 0 ... 30:
-        val = env->ir[n];
+        val = cpu_alpha_load_gr(env, n);
         break;
     case 32 ... 62:
         d.d = env->fir[n - 32];
@@ -66,7 +66,7 @@ int alpha_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
 
     switch (n) {
     case 0 ... 30:
-        env->ir[n] = tmp;
+        cpu_alpha_store_gr(env, n, tmp);
         break;
     case 32 ... 62:
         d.ll = tmp;
diff --git a/qemu/target-alpha/helper.c b/qemu/target-alpha/helper.c
index 46b8ef914..6dec2639b 100644
--- a/qemu/target-alpha/helper.c
+++ b/qemu/target-alpha/helper.c
@@ -17,9 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "fpu/softfloat.h"
@@ -79,6 +77,30 @@ void helper_store_fpcr(CPUAlphaState *env, uint64_t val)
     cpu_alpha_store_fpcr(env, val);
 }
 
+static uint64_t *cpu_alpha_addr_gr(CPUAlphaState *env, unsigned reg)
+{
+#ifndef CONFIG_USER_ONLY
+    if (env->pal_mode) {
+        if (reg >= 8 && reg <= 14) {
+            return &env->shadow[reg - 8];
+        } else if (reg == 25) {
+            return &env->shadow[7];
+        }
+    }
+#endif
+    return &env->ir[reg];
+}
+
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg)
+{
+    return *cpu_alpha_addr_gr(env, reg);
+}
+
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val)
+{
+    *cpu_alpha_addr_gr(env, reg) = val;
+}
+
 #if defined(CONFIG_USER_ONLY)
 int alpha_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
                                int rw, int mmu_idx)
@@ -90,38 +112,6 @@ int alpha_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
     return 1;
 }
 #else
-void swap_shadow_regs(CPUAlphaState *env)
-{
-    uint64_t i0, i1, i2, i3, i4, i5, i6, i7;
-
-    i0 = env->ir[8];
-    i1 = env->ir[9];
-    i2 = env->ir[10];
-    i3 = env->ir[11];
-    i4 = env->ir[12];
-    i5 = env->ir[13];
-    i6 = env->ir[14];
-    i7 = env->ir[25];
-
-    env->ir[8]  = env->shadow[0];
-    env->ir[9]  = env->shadow[1];
-    env->ir[10] = env->shadow[2];
-    env->ir[11] = env->shadow[3];
-    env->ir[12] = env->shadow[4];
-    env->ir[13] = env->shadow[5];
-    env->ir[14] = env->shadow[6];
-    env->ir[25] = env->shadow[7];
-
-    env->shadow[0] = i0;
-    env->shadow[1] = i1;
-    env->shadow[2] = i2;
-    env->shadow[3] = i3;
-    env->shadow[4] = i4;
-    env->shadow[5] = i5;
-    env->shadow[6] = i6;
-    env->shadow[7] = i7;
-}
-
 /* Returns the OSF/1 entMM failure indication, or -1 on success.  */
 static int get_physical_address(CPUAlphaState *env, target_ulong addr,
                                 int prot_need, int mmu_idx,
@@ -375,10 +365,7 @@ void alpha_cpu_do_interrupt(CPUState *cs)
     env->pc = env->palbr + i;
 
     /* Switch to PALmode.  */
-    if (!env->pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
+    env->pal_mode = 1;
 #endif /* !USER_ONLY */
 }
 
@@ -443,7 +430,7 @@ void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                 env->pc, env->ps);
     for (i = 0; i < 31; i++) {
         cpu_fprintf(f, "IR%02d %s " TARGET_FMT_lx " ", i,
-                    linux_reg_names[i], env->ir[i]);
+                    linux_reg_names[i], cpu_alpha_load_gr(env, i));
         if ((i % 3) == 2)
             cpu_fprintf(f, "\n");
     }
diff --git a/qemu/target-alpha/helper.h b/qemu/target-alpha/helper.h
index d221f0d7d..c3d8a3ee4 100644
--- a/qemu/target-alpha/helper.h
+++ b/qemu/target-alpha/helper.h
@@ -10,6 +10,7 @@ DEF_HELPER_FLAGS_1(cttz, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(zap, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
+DEF_HELPER_FLAGS_1(cmpbe0, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
 DEF_HELPER_FLAGS_2(minub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
@@ -91,9 +92,6 @@ DEF_HELPER_FLAGS_2(ieee_input_cmp, TCG_CALL_NO_WG, void, env, i64)
 DEF_HELPER_FLAGS_2(ieee_input_s, TCG_CALL_NO_WG, void, env, i64)
 
 #if !defined (CONFIG_USER_ONLY)
-DEF_HELPER_2(hw_ret, void, env, i64)
-DEF_HELPER_3(call_pal, void, env, i64, i64)
-
 DEF_HELPER_2(ldl_phys, i64, env, i64)
 DEF_HELPER_2(ldq_phys, i64, env, i64)
 DEF_HELPER_2(ldl_l_phys, i64, env, i64)
diff --git a/qemu/target-alpha/int_helper.c b/qemu/target-alpha/int_helper.c
index 74f38cbe7..777e48d08 100644
--- a/qemu/target-alpha/int_helper.c
+++ b/qemu/target-alpha/int_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -58,20 +59,47 @@ uint64_t helper_zap(uint64_t val, uint64_t mask)
     return helper_zapnot(val, ~mask);
 }
 
-uint64_t helper_cmpbge(uint64_t op1, uint64_t op2)
+uint64_t helper_cmpbe0(uint64_t a)
 {
-    uint8_t opa, opb, res;
-    int i;
+    uint64_t m = 0x7f7f7f7f7f7f7f7fULL;
+    uint64_t c = ~(((a & m) + m) | a | m);
+    /* a.......b.......c.......d.......e.......f.......g.......h....... */
+    c |= c << 7;
+    /* ab......bc......cd......de......ef......fg......gh......h....... */
+    c |= c << 14;
+    /* abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... */
+    c |= c << 28;
+    /* abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... */
+    return c >> 56;
+}
 
-    res = 0;
-    for (i = 0; i < 8; i++) {
-        opa = op1 >> (i * 8);
-        opb = op2 >> (i * 8);
-        if (opa >= opb) {
-            res |= 1 << i;
-        }
-    }
-    return res;
+uint64_t helper_cmpbge(uint64_t a, uint64_t b)
+{
+    uint64_t mask = 0x00ff00ff00ff00ffULL;
+    uint64_t test = 0x0100010001000100ULL;
+    uint64_t al, ah, bl, bh, cl, ch;
+
+    /* Separate the bytes to avoid false positives.  */
+    al = a & mask;
+    bl = b & mask;
+    ah = (a >> 8) & mask;
+    bh = (b >> 8) & mask;
+
+    /* "Compare".  If a byte in B is greater than a byte in A,
+       it will clear the test bit.  */
+    cl = ((al | test) - bl) & test;
+    ch = ((ah | test) - bh) & test;
+
+    /* Fold all of the test bits into a contiguous set.  */
+    /* ch=.......a...............c...............e...............g........ */
+    /* cl=.......b...............d...............f...............h........ */
+    cl += ch << 1;
+    /* cl=......ab..............cd..............ef..............gh........ */
+    cl |= cl << 14;
+    /* cl=......abcd............cdef............efgh............gh........ */
+    cl |= cl << 28;
+    /* cl=......abcdefgh........cdefgh..........efgh............gh........ */
+    return cl >> 50;
 }
 
 uint64_t helper_minub8(uint64_t op1, uint64_t op2)
diff --git a/qemu/target-alpha/machine.c b/qemu/target-alpha/machine.c
index e796bbe27..9ab092852 100644
--- a/qemu/target-alpha/machine.c
+++ b/qemu/target-alpha/machine.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 
@@ -70,8 +71,8 @@ static VMStateField vmstate_env_fields[] = {
 
 static const VMStateDescription vmstate_env = {
     .name = "env",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = vmstate_env_fields,
 };
 
diff --git a/qemu/target-alpha/mem_helper.c b/qemu/target-alpha/mem_helper.c
index 7b5e30ddb..7fee9a6e2 100644
--- a/qemu/target-alpha/mem_helper.c
+++ b/qemu/target-alpha/mem_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
@@ -132,7 +133,7 @@ void alpha_cpu_unassigned_access(CPUState *cs, hwaddr addr,
     env->error_code = 0;
 
     /* ??? We should cpu_restore_state to the faulting insn, but this hook
-       does not have access to the retaddr value from the orignal helper.
+       does not have access to the retaddr value from the original helper.
        It's all moot until the QEMU PALcode grows an MCHK handler.  */
 
     cpu_loop_exit(cs);
diff --git a/qemu/target-alpha/sys_helper.c b/qemu/target-alpha/sys_helper.c
index 1c59e108b..e2dec15b6 100644
--- a/qemu/target-alpha/sys_helper.c
+++ b/qemu/target-alpha/sys_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "sysemu/sysemu.h"
@@ -34,34 +35,12 @@ uint64_t helper_load_pcc(CPUAlphaState *env)
 #else
     /* In user-mode, QEMU_CLOCK_VIRTUAL doesn't exist.  Just pass through the host cpu
        clock ticks.  Also, don't bother taking PCC_OFS into account.  */
-    return (uint32_t)cpu_get_real_ticks();
+    return (uint32_t)cpu_get_host_ticks();
 #endif
 }
 
 /* PALcode support special instructions */
 #ifndef CONFIG_USER_ONLY
-void helper_hw_ret(CPUAlphaState *env, uint64_t a)
-{
-    env->pc = a & ~3;
-    env->intr_flag = 0;
-    env->lock_addr = -1;
-    if ((a & 1) == 0) {
-        env->pal_mode = 0;
-        swap_shadow_regs(env);
-    }
-}
-
-void helper_call_pal(CPUAlphaState *env, uint64_t pc, uint64_t entry_ofs)
-{
-    int pal_mode = env->pal_mode;
-    env->exc_addr = pc | pal_mode;
-    env->pc = env->palbr + entry_ofs;
-    if (!pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
-}
-
 void helper_tbia(CPUAlphaState *env)
 {
     tlb_flush(CPU(alpha_env_get_cpu(env)), 1);
diff --git a/qemu/target-alpha/translate.c b/qemu/target-alpha/translate.c
index 81d4ff827..5b86992dd 100644
--- a/qemu/target-alpha/translate.c
+++ b/qemu/target-alpha/translate.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "qemu/host-utils.h"
@@ -27,6 +28,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #undef ALPHA_DEBUG_DISAS
@@ -42,6 +44,9 @@ typedef struct DisasContext DisasContext;
 struct DisasContext {
     struct TranslationBlock *tb;
     uint64_t pc;
+#ifndef CONFIG_USER_ONLY
+    uint64_t palbr;
+#endif
     int mem_idx;
 
     /* Current rounding mode for this TB.  */
@@ -52,6 +57,9 @@ struct DisasContext {
     /* implver value for this CPU.  */
     int implver;
 
+    /* The set of registers active in the current context.  */
+    TCGv *ir;
+
     /* Temporaries for $31 and $f31 as source and destination.  */
     TCGv zero;
     TCGv sink;
@@ -85,14 +93,18 @@ typedef enum {
 } ExitStatus;
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
-static TCGv cpu_ir[31];
+static TCGv_env cpu_env;
+static TCGv cpu_std_ir[31];
 static TCGv cpu_fir[31];
 static TCGv cpu_pc;
 static TCGv cpu_lock_addr;
 static TCGv cpu_lock_st_addr;
 static TCGv cpu_lock_value;
 
+#ifndef CONFIG_USER_ONLY
+static TCGv cpu_pal_ir[31];
+#endif
+
 #include "exec/gen-icount.h"
 
 void alpha_translate_init(void)
@@ -122,6 +134,12 @@ void alpha_translate_init(void)
         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
         "f24", "f25", "f26", "f27", "f28", "f29", "f30"
     };
+#ifndef CONFIG_USER_ONLY
+    static const char shadow_names[8][8] = {
+        "pal_t7", "pal_s0", "pal_s1", "pal_s2",
+        "pal_s3", "pal_s4", "pal_s5", "pal_t11"
+    };
+#endif
 
     static bool done_init = 0;
     int i;
@@ -134,20 +152,31 @@ void alpha_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < 31; i++) {
-        cpu_ir[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                           offsetof(CPUAlphaState, ir[i]),
-                                           greg_names[i]);
+        cpu_std_ir[i] = tcg_global_mem_new_i64(cpu_env,
+                                               offsetof(CPUAlphaState, ir[i]),
+                                               greg_names[i]);
     }
 
     for (i = 0; i < 31; i++) {
-        cpu_fir[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fir[i] = tcg_global_mem_new_i64(cpu_env,
                                             offsetof(CPUAlphaState, fir[i]),
                                             freg_names[i]);
     }
 
+#ifndef CONFIG_USER_ONLY
+    memcpy(cpu_pal_ir, cpu_std_ir, sizeof(cpu_pal_ir));
+    for (i = 0; i < 8; i++) {
+        int r = (i == 7 ? 25 : i + 8);
+        cpu_pal_ir[r] = tcg_global_mem_new_i64(cpu_env,
+                                               offsetof(CPUAlphaState,
+                                                        shadow[i]),
+                                               shadow_names[i]);
+    }
+#endif
+
     for (i = 0; i < ARRAY_SIZE(vars); ++i) {
         const GlobalVar *v = &vars[i];
-        *v->var = tcg_global_mem_new_i64(TCG_AREG0, v->ofs, v->name);
+        *v->var = tcg_global_mem_new_i64(cpu_env, v->ofs, v->name);
     }
 }
 
@@ -170,7 +199,7 @@ static TCGv dest_sink(DisasContext *ctx)
 static TCGv load_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -183,7 +212,7 @@ static TCGv load_gpr_lit(DisasContext *ctx, unsigned reg,
         ctx->lit = tcg_const_i64(lit);
         return ctx->lit;
     } else if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -192,7 +221,7 @@ static TCGv load_gpr_lit(DisasContext *ctx, unsigned reg,
 static TCGv dest_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return dest_sink(ctx);
     }
@@ -304,7 +333,7 @@ static inline void gen_load_mem(DisasContext *ctx,
         addr = tmp;
     }
 
-    va = (fp ? cpu_fir[ra] : cpu_ir[ra]);
+    va = (fp ? cpu_fir[ra] : ctx->ir[ra]);
     tcg_gen_qemu_load(va, addr, ctx->mem_idx);
 
     tcg_temp_free(tmp);
@@ -399,13 +428,13 @@ static ExitStatus gen_store_conditional(DisasContext *ctx, int ra, int rb,
         tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, quad ? MO_LEQ : MO_LESL);
         tcg_gen_brcond_i64(TCG_COND_NE, val, cpu_lock_value, lab_fail);
 
-        tcg_gen_qemu_st_i64(cpu_ir[ra], addr, ctx->mem_idx,
+        tcg_gen_qemu_st_i64(ctx->ir[ra], addr, ctx->mem_idx,
                             quad ? MO_LEQ : MO_LEUL);
-        tcg_gen_movi_i64(cpu_ir[ra], 1);
+        tcg_gen_movi_i64(ctx->ir[ra], 1);
         tcg_gen_br(lab_done);
 
         gen_set_label(lab_fail);
-        tcg_gen_movi_i64(cpu_ir[ra], 0);
+        tcg_gen_movi_i64(ctx->ir[ra], 0);
 
         gen_set_label(lab_done);
         tcg_gen_movi_i64(cpu_lock_addr, -1);
@@ -444,7 +473,7 @@ static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp)
     uint64_t dest = ctx->pc + (disp << 2);
 
     if (ra != 31) {
-        tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+        tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
     }
 
     /* Notice branch-to-next; used to initialize RA with the PC.  */
@@ -1059,12 +1088,13 @@ static void gen_msk_l(DisasContext *ctx, TCGv vc, TCGv va, int rb, bool islit,
     }
 }
 
-static void gen_rx(int ra, int set)
+static void gen_rx(DisasContext *ctx, int ra, int set)
 {
     TCGv_i32 tmp;
 
     if (ra != 31) {
-        tcg_gen_ld8u_i64(cpu_ir[ra], cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_ld8u_i64(ctx->ir[ra], cpu_env,
+                         offsetof(CPUAlphaState, intr_flag));
     }
 
     tmp = tcg_const_i32(set);
@@ -1086,12 +1116,12 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
             break;
         case 0x9E:
             /* RDUNIQUE */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         case 0x9F:
             /* WRUNIQUE */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         default:
@@ -1115,17 +1145,17 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
             break;
         case 0x2D:
             /* WRVPTPTR */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, vptptr));
             break;
         case 0x31:
             /* WRVAL */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
         case 0x32:
             /* RDVAL */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
 
@@ -1135,12 +1165,12 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
 
             /* Note that we already know we're in kernel mode, so we know
                that PS only contains the 3 IPL bits.  */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
 
             /* But make sure and store only the 3 IPL bits from the user.  */
             tmp = tcg_temp_new();
-            tcg_gen_andi_i64(tmp, cpu_ir[IR_A0], PS_INT_MASK);
+            tcg_gen_andi_i64(tmp, ctx->ir[IR_A0], PS_INT_MASK);
             tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, ps));
             tcg_temp_free(tmp);
             break;
@@ -1148,22 +1178,22 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
 
         case 0x36:
             /* RDPS */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
             break;
         case 0x38:
             /* WRUSP */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3A:
             /* RDUSP */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3C:
             /* WHAMI */
-            tcg_gen_ld32s_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld32s_i64(ctx->ir[IR_V0], cpu_env,
                 -offsetof(AlphaCPU, env) + offsetof(CPUState, cpu_index));
             break;
 
@@ -1181,15 +1211,24 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
     return gen_excp(ctx, EXCP_CALL_PAL, palcode);
 #else
     {
-        TCGv pc = tcg_const_i64(ctx->pc);
-        TCGv entry = tcg_const_i64(palcode & 0x80
-                                   ? 0x2000 + (palcode - 0x80) * 64
-                                   : 0x1000 + palcode * 64);
+        TCGv tmp = tcg_temp_new();
+        uint64_t exc_addr = ctx->pc;
+        uint64_t entry = ctx->palbr;
 
-        gen_helper_call_pal(cpu_env, pc, entry);
+        if (ctx->tb->flags & TB_FLAGS_PAL_MODE) {
+            exc_addr |= 1;
+        } else {
+            tcg_gen_movi_i64(tmp, 1);
+            tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        }
+
+        tcg_gen_movi_i64(tmp, exc_addr);
+        tcg_gen_st_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
+        tcg_temp_free(tmp);
 
-        tcg_temp_free(entry);
-        tcg_temp_free(pc);
+        entry += (palcode & 0x80
+                  ? 0x2000 + (palcode - 0x80) * 64
+                  : 0x1000 + palcode * 64);
 
         /* Since the destination is running in PALmode, we don't really
            need the page permissions check.  We'll see the existence of
@@ -1197,11 +1236,13 @@ static ExitStatus gen_call_pal(DisasContext *ctx, int palcode)
            we change the PAL base register.  */
         if (!ctx->singlestep_enabled && !(ctx->tb->cflags & CF_LAST_IO)) {
             tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(cpu_pc, entry);
             tcg_gen_exit_tb((uintptr_t)ctx->tb);
             return EXIT_GOTO_TB;
+        } else {
+            tcg_gen_movi_i64(cpu_pc, entry);
+            return EXIT_PC_UPDATED;
         }
-
-        return EXIT_PC_UPDATED;
     }
 #endif
 }
@@ -1228,8 +1269,6 @@ static int cpu_pr_data(int pr)
     case 11: return offsetof(CPUAlphaState, sysval);
     case 12: return offsetof(CPUAlphaState, usp);
 
-    case 32 ... 39:
-        return offsetof(CPUAlphaState, shadow[pr - 32]);
     case 40 ... 63:
         return offsetof(CPUAlphaState, scratch[pr - 40]);
 
@@ -1241,36 +1280,48 @@ static int cpu_pr_data(int pr)
 
 static ExitStatus gen_mfpr(DisasContext *ctx, TCGv va, int regno)
 {
-    int data = cpu_pr_data(regno);
-
-    /* Special help for VMTIME and WALLTIME.  */
-    if (regno == 250 || regno == 249) {
-	void (*helper)(TCGv) = gen_helper_get_walltime;
-	if (regno == 249) {
-		helper = gen_helper_get_vmtime;
-	}
-        if (ctx->tb->cflags & CF_USE_ICOUNT) {
+    void (*helper)(TCGv);
+    int data;
+
+    switch (regno) {
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(va, cpu_std_ir[regno]);
+        break;
+
+    case 250: /* WALLTIME */
+        helper = gen_helper_get_walltime;
+        goto do_helper;
+    case 249: /* VMTIME */
+        helper = gen_helper_get_vmtime;
+    do_helper:
+        if (use_icount) {
             gen_io_start();
             helper(va);
             gen_io_end();
             return EXIT_PC_STALE;
         } else {
             helper(va);
-            return NO_EXIT;
         }
-    }
+        break;
 
-    /* The basic registers are data only, and unknown registers
-       are read-zero, write-ignore.  */
-    if (data == 0) {
-        tcg_gen_movi_i64(va, 0);
-    } else if (data & PR_BYTE) {
-        tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
-    } else if (data & PR_LONG) {
-        tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
-    } else {
-        tcg_gen_ld_i64(va, cpu_env, data);
+    default:
+        /* The basic registers are data only, and unknown registers
+           are read-zero, write-ignore.  */
+        data = cpu_pr_data(regno);
+        if (data == 0) {
+            tcg_gen_movi_i64(va, 0);
+        } else if (data & PR_BYTE) {
+            tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
+        } else if (data & PR_LONG) {
+            tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
+        } else {
+            tcg_gen_ld_i64(va, cpu_env, data);
+        }
+        break;
     }
+
     return NO_EXIT;
 }
 
@@ -1316,6 +1367,12 @@ static ExitStatus gen_mtpr(DisasContext *ctx, TCGv vb, int regno)
         gen_helper_tb_flush(cpu_env);
         return EXIT_PC_STALE;
 
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(cpu_std_ir[regno], vb);
+        break;
+
     default:
         /* The basic registers are data only, and unknown registers
            are read-zero, write-ignore.  */
@@ -1507,7 +1564,12 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             break;
         case 0x0F:
             /* CMPBGE */
-            gen_helper_cmpbge(vc, va, vb);
+            if (ra == 31) {
+                /* Special case 0 >= X as X == 0.  */
+                gen_helper_cmpbe0(vc, vb);
+            } else {
+                gen_helper_cmpbge(vc, va, vb);
+            }
             break;
         case 0x12:
             /* S8ADDL */
@@ -1952,7 +2014,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_s(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -1972,7 +2034,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_f(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -2295,14 +2357,14 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             break;
         case 0xE000:
             /* RC */
-            gen_rx(ra, 0);
+            gen_rx(ctx, ra, 0);
             break;
         case 0xE800:
             /* ECB */
             break;
         case 0xF000:
             /* RS */
-            gen_rx(ra, 1);
+            gen_rx(ctx, ra, 1);
             break;
         case 0xF800:
             /* WH64 */
@@ -2334,7 +2396,7 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
         vb = load_gpr(ctx, rb);
         tcg_gen_andi_i64(cpu_pc, vb, ~3);
         if (ra != 31) {
-            tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+            tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
         }
         ret = EXIT_PC_UPDATED;
         break;
@@ -2374,10 +2436,10 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
                 goto invalid_opc;
                 break;
             case 0x6:
-                /* Incpu_ir[ra]id */
+                /* Invalid */
                 goto invalid_opc;
             case 0x7:
-                /* Incpu_ir[ra]id */
+                /* Invaliid */
                 goto invalid_opc;
             case 0x8:
                 /* Longword virtual access (hw_ldl) */
@@ -2580,13 +2642,18 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
             /* Pre-EV6 CPUs interpreted this as HW_REI, loading the return
                address from EXC_ADDR.  This turns out to be useful for our
                emulation PALcode, so continue to accept it.  */
-            tmp = tcg_temp_new();
-            tcg_gen_ld_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
-            gen_helper_hw_ret(cpu_env, tmp);
-            tcg_temp_free(tmp);
+            ctx->lit = vb = tcg_temp_new();
+            tcg_gen_ld_i64(vb, cpu_env, offsetof(CPUAlphaState, exc_addr));
         } else {
-            gen_helper_hw_ret(cpu_env, load_gpr(ctx, rb));
+            vb = load_gpr(ctx, rb);
         }
+        tmp = tcg_temp_new();
+        tcg_gen_movi_i64(tmp, 0);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_movi_i64(cpu_lock_addr, -1);
+        tcg_gen_andi_i64(tmp, vb, 1);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        tcg_gen_andi_i64(cpu_pc, vb, ~3);
         ret = EXIT_PC_UPDATED;
         break;
 #else
@@ -2793,18 +2860,14 @@ static ExitStatus translate_one(DisasContext *ctx, uint32_t insn)
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUAlphaState *env, struct TranslationBlock *tb)
 {
+    AlphaCPU *cpu = alpha_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUAlphaState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     target_ulong pc_start;
     target_ulong pc_mask;
     uint32_t insn;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     ExitStatus ret;
     int num_insns;
     int max_insns;
@@ -2813,10 +2876,17 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     ctx.tb = tb;
     ctx.pc = pc_start;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
     ctx.implver = env->implver;
     ctx.singlestep_enabled = cs->singlestep_enabled;
 
+#ifdef CONFIG_USER_ONLY
+    ctx.ir = cpu_std_ir;
+#else
+    ctx.palbr = env->palbr;
+    ctx.ir = (tb->flags & TB_FLAGS_PAL_MODE ? cpu_pal_ir : cpu_std_ir);
+#endif
+
     /* ??? Every TB begins with unset rounding mode, to be initialized on
        the first fp insn of the TB.  Alternately we could define a proper
        default for every TB (e.g. QUAL_RM_N or QUAL_RM_D) and make sure
@@ -2832,6 +2902,9 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     if (in_superpage(&ctx, pc_start)) {
         pc_mask = (1ULL << 41) - 1;
@@ -2841,35 +2914,22 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    gen_excp(&ctx, EXCP_DEBUG, 0);
-                    break;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            ret = gen_excp(&ctx, EXCP_DEBUG, 0);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         insn = cpu_ldl_code(env, ctx.pc);
-        num_insns++;
-
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.pc);
-        }
 
         TCGV_UNUSED_I64(ctx.zero);
         TCGV_UNUSED_I64(ctx.sink);
@@ -2925,16 +2985,8 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -2945,17 +2997,8 @@ static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUAlphaState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUAlphaState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-alpha/vax_helper.c b/qemu/target-alpha/vax_helper.c
index 2e2f49971..e74ac3e04 100644
--- a/qemu/target-alpha/vax_helper.c
+++ b/qemu/target-alpha/vax_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
diff --git a/qemu/target-arm/Makefile.objs b/qemu/target-arm/Makefile.objs
index 9460b409a..82cbe6bba 100644
--- a/qemu/target-arm/Makefile.objs
+++ b/qemu/target-arm/Makefile.objs
@@ -1,5 +1,5 @@
 obj-y += arm-semi.o
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += machine.o psci.o arch_dump.o monitor.o
 obj-$(CONFIG_KVM) += kvm.o
 obj-$(call land,$(CONFIG_KVM),$(call lnot,$(TARGET_AARCH64))) += kvm32.o
 obj-$(call land,$(CONFIG_KVM),$(TARGET_AARCH64)) += kvm64.o
@@ -7,6 +7,5 @@ obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += psci.o
 obj-$(TARGET_AARCH64) += cpu64.o translate-a64.o helper-a64.o gdbstub64.o
 obj-y += crypto_helper.o
diff --git a/qemu/target-arm/arch_dump.c b/qemu/target-arm/arch_dump.c
new file mode 100644
index 000000000..1a9861f69
--- /dev/null
+++ b/qemu/target-arm/arch_dump.c
@@ -0,0 +1,337 @@
+/* Support for writing ELF notes for ARM architectures
+ *
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * Author: Andrew Jones <drjones@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "elf.h"
+#include "sysemu/dump.h"
+
+/* struct user_pt_regs from arch/arm64/include/uapi/asm/ptrace.h */
+struct aarch64_user_regs {
+    uint64_t regs[31];
+    uint64_t sp;
+    uint64_t pc;
+    uint64_t pstate;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_user_regs) != 272);
+
+/* struct elf_prstatus from include/uapi/linux/elfcore.h */
+struct aarch64_elf_prstatus {
+    char pad1[32]; /* 32 == offsetof(struct elf_prstatus, pr_pid) */
+    uint32_t pr_pid;
+    char pad2[76]; /* 76 == offsetof(struct elf_prstatus, pr_reg) -
+                            offsetof(struct elf_prstatus, pr_ppid) */
+    struct aarch64_user_regs pr_reg;
+    uint32_t pr_fpvalid;
+    char pad3[4];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_elf_prstatus) != 392);
+
+/* struct user_fpsimd_state from arch/arm64/include/uapi/asm/ptrace.h
+ *
+ * While the vregs member of user_fpsimd_state is of type __uint128_t,
+ * QEMU uses an array of uint64_t, where the high half of the 128-bit
+ * value is always in the 2n+1'th index. Thus we also break the 128-
+ * bit values into two halves in this reproduction of user_fpsimd_state.
+ */
+struct aarch64_user_vfp_state {
+    uint64_t vregs[64];
+    uint32_t fpsr;
+    uint32_t fpcr;
+    char pad[8];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_user_vfp_state) != 528);
+
+struct aarch64_note {
+    Elf64_Nhdr hdr;
+    char name[8]; /* align_up(sizeof("CORE"), 4) */
+    union {
+        struct aarch64_elf_prstatus prstatus;
+        struct aarch64_user_vfp_state vfp;
+    };
+} QEMU_PACKED;
+
+#define AARCH64_NOTE_HEADER_SIZE offsetof(struct aarch64_note, prstatus)
+#define AARCH64_PRSTATUS_NOTE_SIZE \
+            (AARCH64_NOTE_HEADER_SIZE + sizeof(struct aarch64_elf_prstatus))
+#define AARCH64_PRFPREG_NOTE_SIZE \
+            (AARCH64_NOTE_HEADER_SIZE + sizeof(struct aarch64_user_vfp_state))
+
+static void aarch64_note_init(struct aarch64_note *note, DumpState *s,
+                              const char *name, Elf64_Word namesz,
+                              Elf64_Word type, Elf64_Word descsz)
+{
+    memset(note, 0, sizeof(*note));
+
+    note->hdr.n_namesz = cpu_to_dump32(s, namesz);
+    note->hdr.n_descsz = cpu_to_dump32(s, descsz);
+    note->hdr.n_type = cpu_to_dump32(s, type);
+
+    memcpy(note->name, name, namesz);
+}
+
+static int aarch64_write_elf64_prfpreg(WriteCoreDumpFunction f,
+                                       CPUARMState *env, int cpuid,
+                                       DumpState *s)
+{
+    struct aarch64_note note;
+    int ret, i;
+
+    aarch64_note_init(&note, s, "CORE", 5, NT_PRFPREG, sizeof(note.vfp));
+
+    for (i = 0; i < 64; ++i) {
+        note.vfp.vregs[i] = cpu_to_dump64(s, float64_val(env->vfp.regs[i]));
+    }
+
+    if (s->dump_info.d_endian == ELFDATA2MSB) {
+        /* For AArch64 we must always swap the vfp.regs's 2n and 2n+1
+         * entries when generating BE notes, because even big endian
+         * hosts use 2n+1 for the high half.
+         */
+        for (i = 0; i < 32; ++i) {
+            uint64_t tmp = note.vfp.vregs[2*i];
+            note.vfp.vregs[2*i] = note.vfp.vregs[2*i+1];
+            note.vfp.vregs[2*i+1] = tmp;
+        }
+    }
+
+    note.vfp.fpsr = cpu_to_dump32(s, vfp_get_fpsr(env));
+    note.vfp.fpcr = cpu_to_dump32(s, vfp_get_fpcr(env));
+
+    ret = f(&note, AARCH64_PRFPREG_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+int arm_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque)
+{
+    struct aarch64_note note;
+    CPUARMState *env = &ARM_CPU(cs)->env;
+    DumpState *s = opaque;
+    uint64_t pstate, sp;
+    int ret, i;
+
+    aarch64_note_init(&note, s, "CORE", 5, NT_PRSTATUS, sizeof(note.prstatus));
+
+    note.prstatus.pr_pid = cpu_to_dump32(s, cpuid);
+    note.prstatus.pr_fpvalid = cpu_to_dump32(s, 1);
+
+    if (!is_a64(env)) {
+        aarch64_sync_32_to_64(env);
+        pstate = cpsr_read(env);
+        sp = 0;
+    } else {
+        pstate = pstate_read(env);
+        sp = env->xregs[31];
+    }
+
+    for (i = 0; i < 31; ++i) {
+        note.prstatus.pr_reg.regs[i] = cpu_to_dump64(s, env->xregs[i]);
+    }
+    note.prstatus.pr_reg.sp = cpu_to_dump64(s, sp);
+    note.prstatus.pr_reg.pc = cpu_to_dump64(s, env->pc);
+    note.prstatus.pr_reg.pstate = cpu_to_dump64(s, pstate);
+
+    ret = f(&note, AARCH64_PRSTATUS_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return aarch64_write_elf64_prfpreg(f, env, cpuid, s);
+}
+
+/* struct pt_regs from arch/arm/include/asm/ptrace.h */
+struct arm_user_regs {
+    uint32_t regs[17];
+    char pad[4];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_user_regs) != 72);
+
+/* struct elf_prstatus from include/uapi/linux/elfcore.h */
+struct arm_elf_prstatus {
+    char pad1[24]; /* 24 == offsetof(struct elf_prstatus, pr_pid) */
+    uint32_t pr_pid;
+    char pad2[44]; /* 44 == offsetof(struct elf_prstatus, pr_reg) -
+                            offsetof(struct elf_prstatus, pr_ppid) */
+    struct arm_user_regs pr_reg;
+    uint32_t pr_fpvalid;
+} QEMU_PACKED arm_elf_prstatus;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_elf_prstatus) != 148);
+
+/* struct user_vfp from arch/arm/include/asm/user.h */
+struct arm_user_vfp_state {
+    uint64_t vregs[32];
+    uint32_t fpscr;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_user_vfp_state) != 260);
+
+struct arm_note {
+    Elf32_Nhdr hdr;
+    char name[8]; /* align_up(sizeof("LINUX"), 4) */
+    union {
+        struct arm_elf_prstatus prstatus;
+        struct arm_user_vfp_state vfp;
+    };
+} QEMU_PACKED;
+
+#define ARM_NOTE_HEADER_SIZE offsetof(struct arm_note, prstatus)
+#define ARM_PRSTATUS_NOTE_SIZE \
+            (ARM_NOTE_HEADER_SIZE + sizeof(struct arm_elf_prstatus))
+#define ARM_VFP_NOTE_SIZE \
+            (ARM_NOTE_HEADER_SIZE + sizeof(struct arm_user_vfp_state))
+
+static void arm_note_init(struct arm_note *note, DumpState *s,
+                          const char *name, Elf32_Word namesz,
+                          Elf32_Word type, Elf32_Word descsz)
+{
+    memset(note, 0, sizeof(*note));
+
+    note->hdr.n_namesz = cpu_to_dump32(s, namesz);
+    note->hdr.n_descsz = cpu_to_dump32(s, descsz);
+    note->hdr.n_type = cpu_to_dump32(s, type);
+
+    memcpy(note->name, name, namesz);
+}
+
+static int arm_write_elf32_vfp(WriteCoreDumpFunction f, CPUARMState *env,
+                               int cpuid, DumpState *s)
+{
+    struct arm_note note;
+    int ret, i;
+
+    arm_note_init(&note, s, "LINUX", 6, NT_ARM_VFP, sizeof(note.vfp));
+
+    for (i = 0; i < 32; ++i) {
+        note.vfp.vregs[i] = cpu_to_dump64(s, float64_val(env->vfp.regs[i]));
+    }
+
+    note.vfp.fpscr = cpu_to_dump32(s, vfp_get_fpscr(env));
+
+    ret = f(&note, ARM_VFP_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+int arm_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque)
+{
+    struct arm_note note;
+    CPUARMState *env = &ARM_CPU(cs)->env;
+    DumpState *s = opaque;
+    int ret, i, fpvalid = !!arm_feature(env, ARM_FEATURE_VFP);
+
+    arm_note_init(&note, s, "CORE", 5, NT_PRSTATUS, sizeof(note.prstatus));
+
+    note.prstatus.pr_pid = cpu_to_dump32(s, cpuid);
+    note.prstatus.pr_fpvalid = cpu_to_dump32(s, fpvalid);
+
+    for (i = 0; i < 16; ++i) {
+        note.prstatus.pr_reg.regs[i] = cpu_to_dump32(s, env->regs[i]);
+    }
+    note.prstatus.pr_reg.regs[16] = cpu_to_dump32(s, cpsr_read(env));
+
+    ret = f(&note, ARM_PRSTATUS_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    } else if (fpvalid) {
+        return arm_write_elf32_vfp(f, env, cpuid, s);
+    }
+
+    return 0;
+}
+
+int cpu_get_dump_info(ArchDumpInfo *info,
+                      const GuestPhysBlockList *guest_phys_blocks)
+{
+    ARMCPU *cpu = ARM_CPU(first_cpu);
+    CPUARMState *env = &cpu->env;
+    GuestPhysBlock *block;
+    hwaddr lowest_addr = ULLONG_MAX;
+
+    /* Take a best guess at the phys_base. If we get it wrong then crash
+     * will need '--machdep phys_offset=<phys-offset>' added to its command
+     * line, which isn't any worse than assuming we can use zero, but being
+     * wrong. This is the same algorithm the crash utility uses when
+     * attempting to guess as it loads non-dumpfile formatted files.
+     */
+    QTAILQ_FOREACH(block, &guest_phys_blocks->head, next) {
+        if (block->target_start < lowest_addr) {
+            lowest_addr = block->target_start;
+        }
+    }
+
+    if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+        info->d_machine = EM_AARCH64;
+        info->d_class = ELFCLASS64;
+        info->page_size = (1 << 16); /* aarch64 max pagesize */
+        if (lowest_addr != ULLONG_MAX) {
+            info->phys_base = lowest_addr;
+        }
+    } else {
+        info->d_machine = EM_ARM;
+        info->d_class = ELFCLASS32;
+        info->page_size = (1 << 12);
+        if (lowest_addr < UINT_MAX) {
+            info->phys_base = lowest_addr;
+        }
+    }
+
+    /* We assume the relevant endianness is that of EL1; this is right
+     * for kernels, but might give the wrong answer if you're trying to
+     * dump a hypervisor that happens to be running an opposite-endian
+     * kernel.
+     */
+    info->d_endian = (env->cp15.sctlr_el[1] & SCTLR_EE) != 0
+                     ? ELFDATA2MSB : ELFDATA2LSB;
+
+    return 0;
+}
+
+ssize_t cpu_get_note_size(int class, int machine, int nr_cpus)
+{
+    ARMCPU *cpu = ARM_CPU(first_cpu);
+    CPUARMState *env = &cpu->env;
+    size_t note_size;
+
+    if (class == ELFCLASS64) {
+        note_size = AARCH64_PRSTATUS_NOTE_SIZE;
+        note_size += AARCH64_PRFPREG_NOTE_SIZE;
+    } else {
+        note_size = ARM_PRSTATUS_NOTE_SIZE;
+        if (arm_feature(env, ARM_FEATURE_VFP)) {
+            note_size += ARM_VFP_NOTE_SIZE;
+        }
+    }
+
+    return note_size * nr_cpus;
+}
diff --git a/qemu/target-arm/arm-semi.c b/qemu/target-arm/arm-semi.c
index a2a736956..8be0645eb 100644
--- a/qemu/target-arm/arm-semi.c
+++ b/qemu/target-arm/arm-semi.c
@@ -18,13 +18,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/semihost.h"
@@ -36,6 +30,7 @@
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "hw/arm/arm.h"
+#include "qemu/cutils.h"
 #endif
 
 #define TARGET_SYS_OPEN        0x01
@@ -58,6 +53,7 @@
 #define TARGET_SYS_GET_CMDLINE 0x15
 #define TARGET_SYS_HEAPINFO    0x16
 #define TARGET_SYS_EXIT        0x18
+#define TARGET_SYS_SYNCCACHE   0x19
 
 /* ADP_Stopped_ApplicationExit is used for exit(0),
  * anything else is implemented as exit(1) */
@@ -134,6 +130,7 @@ static void arm_semi_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #ifdef CONFIG_USER_ONLY
     TaskState *ts = cs->opaque;
 #endif
+    target_ulong reg0 = is_a64(env) ? env->xregs[0] : env->regs[0];
 
     if (ret == (target_ulong)-1) {
 #ifdef CONFIG_USER_ONLY
@@ -141,22 +138,46 @@ static void arm_semi_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #else
 	syscall_err = err;
 #endif
-        env->regs[0] = ret;
+        reg0 = ret;
     } else {
         /* Fixup syscalls that use nonstardard return conventions.  */
-        switch (env->regs[0]) {
+        switch (reg0) {
         case TARGET_SYS_WRITE:
         case TARGET_SYS_READ:
-            env->regs[0] = arm_semi_syscall_len - ret;
+            reg0 = arm_semi_syscall_len - ret;
             break;
         case TARGET_SYS_SEEK:
-            env->regs[0] = 0;
+            reg0 = 0;
             break;
         default:
-            env->regs[0] = ret;
+            reg0 = ret;
             break;
         }
     }
+    if (is_a64(env)) {
+        env->xregs[0] = reg0;
+    } else {
+        env->regs[0] = reg0;
+    }
+}
+
+static target_ulong arm_flen_buf(ARMCPU *cpu)
+{
+    /* Return an address in target memory of 64 bytes where the remote
+     * gdb should write its stat struct. (The format of this structure
+     * is defined by GDB's remote protocol and is not target-specific.)
+     * We put this on the guest's stack just below SP.
+     */
+    CPUARMState *env = &cpu->env;
+    target_ulong sp;
+
+    if (is_a64(env)) {
+        sp = env->xregs[31];
+    } else {
+        sp = env->regs[13];
+    }
+
+    return sp - 64;
 }
 
 static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
@@ -166,8 +187,13 @@ static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
     /* The size is always stored in big-endian order, extract
        the value. We assume the size always fit in 32 bits.  */
     uint32_t size;
-    cpu_memory_rw_debug(cs, env->regs[13]-64+32, (uint8_t *)&size, 4, 0);
-    env->regs[0] = be32_to_cpu(size);
+    cpu_memory_rw_debug(cs, arm_flen_buf(cpu) + 32, (uint8_t *)&size, 4, 0);
+    size = be32_to_cpu(size);
+    if (is_a64(env)) {
+        env->xregs[0] = size;
+    } else {
+        env->regs[0] = size;
+    }
 #ifdef CONFIG_USER_ONLY
     ((TaskState *)cs->opaque)->swi_errno = err;
 #else
@@ -175,17 +201,46 @@ static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
 #endif
 }
 
+static target_ulong arm_gdb_syscall(ARMCPU *cpu, gdb_syscall_complete_cb cb,
+                                    const char *fmt, ...)
+{
+    va_list va;
+    CPUARMState *env = &cpu->env;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+
+    /* FIXME: we are implicitly relying on the syscall completing
+     * before this point, which is not guaranteed. We should
+     * put in an explicit synchronization between this and
+     * the callback function.
+     */
+
+    return is_a64(env) ? env->xregs[0] : env->regs[0];
+}
+
 /* Read the input value from the argument block; fail the semihosting
  * call if the memory read fails.
  */
 #define GET_ARG(n) do {                                 \
-    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
-        return (uint32_t)-1;                            \
+    if (is_a64(env)) {                                  \
+        if (get_user_u64(arg ## n, args + (n) * 8)) {   \
+            return -1;                                  \
+        }                                               \
+    } else {                                            \
+        if (get_user_u32(arg ## n, args + (n) * 4)) {   \
+            return -1;                                  \
+        }                                               \
     }                                                   \
 } while (0)
 
-#define SET_ARG(n, val) put_user_ual(val, args + (n) * 4)
-uint32_t do_arm_semihosting(CPUARMState *env)
+#define SET_ARG(n, val)                                 \
+    (is_a64(env) ?                                      \
+     put_user_u64(val, args + (n) * 8) :                \
+     put_user_u32(val, args + (n) * 4))
+
+target_ulong do_arm_semihosting(CPUARMState *env)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
@@ -201,8 +256,15 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     CPUARMState *ts = env;
 #endif
 
-    nr = env->regs[0];
-    args = env->regs[1];
+    if (is_a64(env)) {
+        /* Note that the syscall number is in W0, not X0 */
+        nr = env->xregs[0] & 0xffffffffU;
+        args = env->xregs[1];
+    } else {
+        nr = env->regs[0];
+        args = env->regs[1];
+    }
+
     switch (nr) {
     case TARGET_SYS_OPEN:
         GET_ARG(0);
@@ -223,9 +285,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return result_fileno;
         }
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", arg0,
-                           (int)arg2+1, gdb_open_modeflags[arg1]);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "open,%s,%x,1a4", arg0,
+                                  (int)arg2+1, gdb_open_modeflags[arg1]);
         } else {
             ret = set_swi_errno(ts, open(s, open_modeflags[arg1], 0644));
         }
@@ -234,8 +295,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_CLOSE:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "close,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "close,%x", arg0);
         } else {
             return set_swi_errno(ts, close(arg0));
         }
@@ -248,8 +308,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
               return (uint32_t)-1;
           /* Write to debug console.  stderr is near enough.  */
           if (use_gdb_syscalls()) {
-                gdb_do_syscall(arm_semi_cb, "write,2,%x,1", args);
-                return env->regs[0];
+                return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,1", args);
           } else {
                 return write(STDERR_FILENO, &c, 1);
           }
@@ -260,8 +319,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return (uint32_t)-1;
         len = strlen(s);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "write,2,%x,%x\n", args, len);
-            ret = env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,%x",
+                                   args, len);
         } else {
             ret = write(STDERR_FILENO, s, len);
         }
@@ -274,8 +333,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_READ, arg1, len, 1);
             if (!s) {
@@ -295,8 +354,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "read,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_WRITE, arg1, len, 0);
             if (!s) {
@@ -317,8 +376,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_ISTTY:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "isatty,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "isatty,%x", arg0);
         } else {
             return isatty(arg0);
         }
@@ -326,8 +384,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", arg0, arg1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "lseek,%x,%x,0",
+                                   arg0, arg1);
         } else {
             ret = set_swi_errno(ts, lseek(arg0, arg1, SEEK_SET));
             if (ret == (uint32_t)-1)
@@ -337,9 +395,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_FLEN:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_flen_cb, "fstat,%x,%x",
-                           arg0, env->regs[13]-64);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_flen_cb, "fstat,%x,%x",
+                                   arg0, arm_flen_buf(cpu));
         } else {
             struct stat buf;
             ret = set_swi_errno(ts, fstat(arg0, &buf));
@@ -354,8 +411,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "unlink,%s", arg0, (int)arg1+1);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "unlink,%s",
+                                  arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -372,9 +429,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(2);
         GET_ARG(3);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "rename,%s,%s",
-                           arg0, (int)arg1+1, arg2, (int)arg3+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "rename,%s,%s",
+                                   arg0, (int)arg1+1, arg2, (int)arg3+1);
         } else {
             char *s2;
             s = lock_user_string(arg0);
@@ -398,8 +454,8 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "system,%s", arg0, (int)arg1+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "system,%s",
+                                   arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -558,11 +614,35 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             return 0;
         }
     case TARGET_SYS_EXIT:
-        /* ARM specifies only Stopped_ApplicationExit as normal
-         * exit, everything else is considered an error */
-        ret = (args == ADP_Stopped_ApplicationExit) ? 0 : 1;
+        if (is_a64(env)) {
+            /* The A64 version of this call takes a parameter block,
+             * so the application-exit type can return a subcode which
+             * is the exit status code from the application.
+             */
+            GET_ARG(0);
+            GET_ARG(1);
+
+            if (arg0 == ADP_Stopped_ApplicationExit) {
+                ret = arg1;
+            } else {
+                ret = 1;
+            }
+        } else {
+            /* ARM specifies only Stopped_ApplicationExit as normal
+             * exit, everything else is considered an error */
+            ret = (args == ADP_Stopped_ApplicationExit) ? 0 : 1;
+        }
         gdb_exit(env, ret);
         exit(ret);
+    case TARGET_SYS_SYNCCACHE:
+        /* Clean the D-cache and invalidate the I-cache for the specified
+         * virtual address range. This is a nop for us since we don't
+         * implement caches. This is only present on A64.
+         */
+        if (is_a64(env)) {
+            return 0;
+        }
+        /* fall through -- invalid for A32/T32 */
     default:
         fprintf(stderr, "qemu: Unsupported SemiHosting SWI 0x%02x\n", nr);
         cpu_dump_state(cs, stderr, fprintf, 0);
diff --git a/qemu/target-arm/arm_ldst.h b/qemu/target-arm/arm_ldst.h
index b1ece0173..35c2c4391 100644
--- a/qemu/target-arm/arm_ldst.h
+++ b/qemu/target-arm/arm_ldst.h
@@ -25,10 +25,10 @@
 
 /* Load an instruction and return it in the standard little-endian order */
 static inline uint32_t arm_ldl_code(CPUARMState *env, target_ulong addr,
-                                    bool do_swap)
+                                    bool sctlr_b)
 {
     uint32_t insn = cpu_ldl_code(env, addr);
-    if (do_swap) {
+    if (bswap_code(sctlr_b)) {
         return bswap32(insn);
     }
     return insn;
@@ -36,10 +36,10 @@ static inline uint32_t arm_ldl_code(CPUARMState *env, target_ulong addr,
 
 /* Ditto, for a halfword (Thumb) instruction */
 static inline uint16_t arm_lduw_code(CPUARMState *env, target_ulong addr,
-                                     bool do_swap)
+                                     bool sctlr_b)
 {
     uint16_t insn = cpu_lduw_code(env, addr);
-    if (do_swap) {
+    if (bswap_code(sctlr_b)) {
         return bswap16(insn);
     }
     return insn;
diff --git a/qemu/target-arm/cpu-qom.h b/qemu/target-arm/cpu-qom.h
index 3cbc4a006..1061c08a1 100644
--- a/qemu/target-arm/cpu-qom.h
+++ b/qemu/target-arm/cpu-qom.h
@@ -87,6 +87,9 @@ typedef struct ARMCPU {
     /* GPIO outputs for generic timer */
     qemu_irq gt_timer_outputs[NUM_GTIMERS];
 
+    /* MemoryRegion to use for secure physical accesses */
+    MemoryRegion *secure_memory;
+
     /* 'compatible' string for this CPU for Linux device trees */
     const char *dtb_compatible;
 
@@ -145,11 +148,14 @@ typedef struct ARMCPU {
     uint32_t id_pfr0;
     uint32_t id_pfr1;
     uint32_t id_dfr0;
+    uint32_t pmceid0;
+    uint32_t pmceid1;
     uint32_t id_afr0;
     uint32_t id_mmfr0;
     uint32_t id_mmfr1;
     uint32_t id_mmfr2;
     uint32_t id_mmfr3;
+    uint32_t id_mmfr4;
     uint32_t id_isar0;
     uint32_t id_isar1;
     uint32_t id_isar2;
@@ -216,20 +222,39 @@ bool arm_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                         int flags);
 
-hwaddr arm_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cpu, vaddr addr,
+                                         MemTxAttrs *attrs);
 
 int arm_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int arm_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
+int arm_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque);
+int arm_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque);
+
 /* Callback functions for the generic timer's timers. */
 void arm_gt_ptimer_cb(void *opaque);
 void arm_gt_vtimer_cb(void *opaque);
+void arm_gt_htimer_cb(void *opaque);
+void arm_gt_stimer_cb(void *opaque);
+
+#define ARM_AFF0_SHIFT 0
+#define ARM_AFF0_MASK  (0xFFULL << ARM_AFF0_SHIFT)
+#define ARM_AFF1_SHIFT 8
+#define ARM_AFF1_MASK  (0xFFULL << ARM_AFF1_SHIFT)
+#define ARM_AFF2_SHIFT 16
+#define ARM_AFF2_MASK  (0xFFULL << ARM_AFF2_SHIFT)
+#define ARM_AFF3_SHIFT 32
+#define ARM_AFF3_MASK  (0xFFULL << ARM_AFF3_SHIFT)
+
+#define ARM32_AFFINITY_MASK (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK)
+#define ARM64_AFFINITY_MASK \
+    (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK|ARM_AFF3_MASK)
 
 #ifdef TARGET_AARCH64
 int aarch64_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int aarch64_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-
-void aarch64_cpu_do_interrupt(CPUState *cs);
 #endif
 
 #endif
diff --git a/qemu/target-arm/cpu.c b/qemu/target-arm/cpu.c
index 8b4323dd0..e48e83acb 100644
--- a/qemu/target-arm/cpu.c
+++ b/qemu/target-arm/cpu.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "internals.h"
 #include "qemu-common.h"
@@ -79,6 +81,27 @@ static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
     }
 }
 
+static void cp_reg_check_reset(gpointer key, gpointer value,  gpointer opaque)
+{
+    /* Purely an assertion check: we've already done reset once,
+     * so now check that running the reset for the cpreg doesn't
+     * change its value. This traps bugs where two different cpregs
+     * both try to reset the same state field but to different values.
+     */
+    ARMCPRegInfo *ri = value;
+    ARMCPU *cpu = opaque;
+    uint64_t oldvalue, newvalue;
+
+    if (ri->type & (ARM_CP_SPECIAL | ARM_CP_ALIAS | ARM_CP_NO_RAW)) {
+        return;
+    }
+
+    oldvalue = read_raw_cp_reg(&cpu->env, ri);
+    cp_reg_reset(key, value, opaque);
+    newvalue = read_raw_cp_reg(&cpu->env, ri);
+    assert(oldvalue == newvalue);
+}
+
 /* CPUClass::reset() */
 static void arm_cpu_reset(CPUState *s)
 {
@@ -90,6 +113,8 @@ static void arm_cpu_reset(CPUState *s)
 
     memset(env, 0, offsetof(CPUARMState, features));
     g_hash_table_foreach(cpu->cp_regs, cp_reg_reset, cpu);
+    g_hash_table_foreach(cpu->cp_regs, cp_reg_check_reset, cpu);
+
     env->vfp.xregs[ARM_VFP_FPSID] = cpu->reset_fpsid;
     env->vfp.xregs[ARM_VFP_MVFR0] = cpu->mvfr0;
     env->vfp.xregs[ARM_VFP_MVFR1] = cpu->mvfr1;
@@ -308,10 +333,7 @@ static void arm_cpu_set_irq(void *opaque, int irq, int level)
     switch (irq) {
     case ARM_CPU_VIRQ:
     case ARM_CPU_VFIQ:
-        if (!arm_feature(env, ARM_FEATURE_EL2)) {
-            hw_error("%s: Virtual interrupt line %d with no EL2 support\n",
-                     __func__, irq);
-        }
+        assert(arm_feature(env, ARM_FEATURE_EL2));
         /* fall through */
     case ARM_CPU_IRQ:
     case ARM_CPU_FIQ:
@@ -322,7 +344,7 @@ static void arm_cpu_set_irq(void *opaque, int irq, int level)
         }
         break;
     default:
-        hw_error("arm_cpu_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
 }
 
@@ -341,33 +363,20 @@ static void arm_cpu_kvm_set_irq(void *opaque, int irq, int level)
         kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
         break;
     default:
-        hw_error("arm_cpu_kvm_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
     kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
     kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
 #endif
 }
 
-static bool arm_cpu_is_big_endian(CPUState *cs)
+static bool arm_cpu_virtio_is_big_endian(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
-    int cur_el;
 
     cpu_synchronize_state(cs);
-
-    /* In 32bit guest endianness is determined by looking at CPSR's E bit */
-    if (!is_a64(env)) {
-        return (env->uncached_cpsr & CPSR_E) ? 1 : 0;
-    }
-
-    cur_el = arm_current_el(env);
-
-    if (cur_el == 0) {
-        return (env->cp15.sctlr_el[1] & SCTLR_E0E) != 0;
-    }
-
-    return (env->cp15.sctlr_el[cur_el] & SCTLR_EE) != 0;
+    return arm_cpu_data_is_big_endian(env);
 }
 
 #endif
@@ -406,7 +415,7 @@ static void arm_disas_set_info(CPUState *cpu, disassemble_info *info)
     } else {
         info->print_insn = print_insn_arm;
     }
-    if (env->bswap_code) {
+    if (bswap_code(arm_sctlr_b(env))) {
 #ifdef TARGET_WORDS_BIGENDIAN
         info->endian = BFD_ENDIAN_LITTLE;
 #else
@@ -436,7 +445,7 @@ static void arm_cpu_initfn(Object *obj)
      */
     Aff1 = cs->cpu_index / ARM_CPUS_PER_CLUSTER;
     Aff0 = cs->cpu_index % ARM_CPUS_PER_CLUSTER;
-    cpu->mp_affinity = (Aff1 << 8) | Aff0;
+    cpu->mp_affinity = (Aff1 << ARM_AFF1_SHIFT) | Aff0;
 
 #ifndef CONFIG_USER_ONLY
     /* Our inbound IRQ and FIQ lines */
@@ -453,6 +462,10 @@ static void arm_cpu_initfn(Object *obj)
                                                 arm_gt_ptimer_cb, cpu);
     cpu->gt_timer[GTIMER_VIRT] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
                                                 arm_gt_vtimer_cb, cpu);
+    cpu->gt_timer[GTIMER_HYP] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_htimer_cb, cpu);
+    cpu->gt_timer[GTIMER_SEC] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_stimer_cb, cpu);
     qdev_init_gpio_out(DEVICE(cpu), cpu->gt_timer_outputs,
                        ARRAY_SIZE(cpu->gt_timer_outputs));
 #endif
@@ -518,6 +531,15 @@ static void arm_cpu_post_init(Object *obj)
          */
         qdev_property_add_static(DEVICE(obj), &arm_cpu_has_el3_property,
                                  &error_abort);
+
+#ifndef CONFIG_USER_ONLY
+        object_property_add_link(obj, "secure-memory",
+                                 TYPE_MEMORY_REGION,
+                                 (Object **)&cpu->secure_memory,
+                                 qdev_prop_allow_set_link_before_realize,
+                                 OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                                 &error_abort);
+#endif
     }
 
     if (arm_feature(&cpu->env, ARM_FEATURE_MPU)) {
@@ -616,6 +638,15 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
         cpu->id_aa64pfr0 &= ~0xf000;
     }
 
+    if (!arm_feature(env, ARM_FEATURE_EL2)) {
+        /* Disable the hypervisor feature bits in the processor feature
+         * registers if we don't have EL2. These are id_pfr1[15:12] and
+         * id_aa64pfr0_el1[11:8].
+         */
+        cpu->id_aa64pfr0 &= ~0xf00;
+        cpu->id_pfr1 &= ~0xf000;
+    }
+
     if (!cpu->has_mpu) {
         unset_feature(env, ARM_FEATURE_MPU);
     }
@@ -625,7 +656,7 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
         uint32_t nr = cpu->pmsav7_dregion;
 
         if (nr > 0xff) {
-            error_setg(errp, "PMSAv7 MPU #regions invalid %" PRIu32 "\n", nr);
+            error_setg(errp, "PMSAv7 MPU #regions invalid %" PRIu32, nr);
             return;
         }
 
@@ -641,6 +672,29 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
 
     init_cpreg_list(cpu);
 
+#ifndef CONFIG_USER_ONLY
+    if (cpu->has_el3) {
+        cs->num_ases = 2;
+    } else {
+        cs->num_ases = 1;
+    }
+
+    if (cpu->has_el3) {
+        AddressSpace *as;
+
+        if (!cpu->secure_memory) {
+            cpu->secure_memory = cs->memory;
+        }
+        as = address_space_init_shareable(cpu->secure_memory,
+                                          "cpu-secure-memory");
+        cpu_address_space_init(cs, as, ARMASIdx_S);
+    }
+    cpu_address_space_init(cs,
+                           address_space_init_shareable(cs->memory,
+                                                        "cpu-memory"),
+                           ARMASIdx_NS);
+#endif
+
     qemu_init_vcpu(cs);
     cpu_reset(cs);
 
@@ -1090,6 +1144,8 @@ static void cortex_a15_initfn(Object *obj)
     cpu->id_pfr0 = 0x00001131;
     cpu->id_pfr1 = 0x00011011;
     cpu->id_dfr0 = 0x02010555;
+    cpu->pmceid0 = 0x0000000;
+    cpu->pmceid1 = 0x00000000;
     cpu->id_afr0 = 0x00000000;
     cpu->id_mmfr0 = 0x10201105;
     cpu->id_mmfr1 = 0x20000000;
@@ -1369,6 +1425,17 @@ static int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
 }
 #endif
 
+static gchar *arm_gdb_arch_name(CPUState *cs)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    if (arm_feature(env, ARM_FEATURE_IWMMXT)) {
+        return g_strdup("iwmmxt");
+    }
+    return g_strdup("arm");
+}
+
 static void arm_cpu_class_init(ObjectClass *oc, void *data)
 {
     ARMCPUClass *acc = ARM_CPU_CLASS(oc);
@@ -1393,16 +1460,33 @@ static void arm_cpu_class_init(ObjectClass *oc, void *data)
     cc->handle_mmu_fault = arm_cpu_handle_mmu_fault;
 #else
     cc->do_interrupt = arm_cpu_do_interrupt;
-    cc->get_phys_page_debug = arm_cpu_get_phys_page_debug;
+    cc->do_unaligned_access = arm_cpu_do_unaligned_access;
+    cc->get_phys_page_attrs_debug = arm_cpu_get_phys_page_attrs_debug;
+    cc->asidx_from_attrs = arm_asidx_from_attrs;
     cc->vmsd = &vmstate_arm_cpu;
-    cc->virtio_is_big_endian = arm_cpu_is_big_endian;
+    cc->virtio_is_big_endian = arm_cpu_virtio_is_big_endian;
+    cc->write_elf64_note = arm_cpu_write_elf64_note;
+    cc->write_elf32_note = arm_cpu_write_elf32_note;
 #endif
     cc->gdb_num_core_regs = 26;
     cc->gdb_core_xml_file = "arm-core.xml";
+    cc->gdb_arch_name = arm_gdb_arch_name;
     cc->gdb_stop_before_watchpoint = true;
     cc->debug_excp_handler = arm_debug_excp_handler;
+    cc->debug_check_watchpoint = arm_debug_check_watchpoint;
 
     cc->disas_set_info = arm_disas_set_info;
+
+    /*
+     * Reason: arm_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     *
+     * Once this is fixed, the devices that create ARM CPUs should be
+     * updated not to set cannot_destroy_with_object_finalize_yet,
+     * unless they still screw up something else.
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const ARMCPUInfo *info)
diff --git a/qemu/target-arm/cpu.h b/qemu/target-arm/cpu.h
index 7e89152bd..066ff678d 100644
--- a/qemu/target-arm/cpu.h
+++ b/qemu/target-arm/cpu.h
@@ -19,17 +19,14 @@
 #ifndef CPU_ARM_H
 #define CPU_ARM_H
 
-#include "config.h"
 
 #include "kvm-consts.h"
 
 #if defined(TARGET_AARCH64)
   /* AArch64 definitions */
 #  define TARGET_LONG_BITS 64
-#  define ELF_MACHINE EM_AARCH64
 #else
 #  define TARGET_LONG_BITS 32
-#  define ELF_MACHINE EM_ARM
 #endif
 
 #define TARGET_IS_BIENDIAN 1
@@ -56,6 +53,7 @@
 #define EXCP_SMC            13   /* Secure Monitor Call */
 #define EXCP_VIRQ           14
 #define EXCP_VFIQ           15
+#define EXCP_SEMIHOST       16   /* semihosting call (A64 only) */
 
 #define ARMV7M_EXCP_RESET   1
 #define ARMV7M_EXCP_NMI     2
@@ -96,6 +94,7 @@
 struct arm_boot_info;
 
 #define NB_MMU_MODES 7
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 /* We currently assume float and double are IEEE single and double
    precision respectively.
@@ -113,7 +112,9 @@ typedef struct ARMGenericTimer {
 
 #define GTIMER_PHYS 0
 #define GTIMER_VIRT 1
-#define NUM_GTIMERS 2
+#define GTIMER_HYP  2
+#define GTIMER_SEC  3
+#define NUM_GTIMERS 4
 
 typedef struct {
     uint64_t raw_tcr;
@@ -170,7 +171,7 @@ typedef struct CPUARMState {
     uint32_t GE; /* cpsr[19:16] */
     uint32_t thumb; /* cpsr[5]. 0 = arm mode, 1 = thumb mode. */
     uint32_t condexec_bits; /* IT bits.  cpsr[15:10,26:25].  */
-    uint64_t daif; /* exception masks, in the bits they are in in PSTATE */
+    uint64_t daif; /* exception masks, in the bits they are in PSTATE */
 
     uint64_t elr_el[4]; /* AArch64 exception link regs  */
     uint64_t sp_el[4]; /* AArch64 banked stack pointers */
@@ -219,10 +220,12 @@ typedef struct CPUARMState {
             };
             uint64_t ttbr1_el[4];
         };
+        uint64_t vttbr_el2; /* Virtualization Translation Table Base.  */
         /* MMU translation table base control. */
         TCR tcr_el[4];
-        uint32_t c2_data; /* MPU data cachable bits.  */
-        uint32_t c2_insn; /* MPU instruction cachable bits.  */
+        TCR vtcr_el2; /* Virtualization Translation Control.  */
+        uint32_t c2_data; /* MPU data cacheable bits.  */
+        uint32_t c2_insn; /* MPU instruction cacheable bits.  */
         union { /* MMU domain access control register
                  * MPU write buffer control.
                  */
@@ -275,6 +278,7 @@ typedef struct CPUARMState {
             };
             uint64_t far_el[4];
         };
+        uint64_t hpfar_el2;
         union { /* Translation result. */
             struct {
                 uint64_t _unused_par_0;
@@ -358,6 +362,8 @@ typedef struct CPUARMState {
         };
         uint64_t c14_cntfrq; /* Counter Frequency register */
         uint64_t c14_cntkctl; /* Timer Control register */
+        uint32_t cnthctl_el2; /* Counter/Timer Hyp Control register */
+        uint64_t cntvoff_el2; /* Counter Virtual Offset register */
         ARMGenericTimer c14_timer[NUM_GTIMERS];
         uint32_t c15_cpar; /* XScale Coprocessor Access Register */
         uint32_t c15_ticonfig; /* TI925T configuration byte.  */
@@ -373,11 +379,16 @@ typedef struct CPUARMState {
         uint64_t dbgwvr[16]; /* watchpoint value registers */
         uint64_t dbgwcr[16]; /* watchpoint control registers */
         uint64_t mdscr_el1;
+        uint64_t oslsr_el1; /* OS Lock Status */
+        uint64_t mdcr_el2;
+        uint64_t mdcr_el3;
         /* If the counter is enabled, this stores the last time the counter
          * was reset. Otherwise it stores the counter value
          */
         uint64_t c15_ccnt;
         uint64_t pmccfiltr_el0; /* Performance Monitor Filter Register */
+        uint64_t vpidr_el2; /* Virtualization Processor ID Register */
+        uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
     } cp15;
 
     struct {
@@ -467,9 +478,6 @@ typedef struct CPUARMState {
         uint32_t cregs[16];
     } iwmmxt;
 
-    /* For mixed endian mode.  */
-    bool bswap_code;
-
 #if defined(CONFIG_USER_ONLY)
     /* For usermode syscall translation.  */
     int eabi;
@@ -500,7 +508,7 @@ typedef struct CPUARMState {
 
 ARMCPU *cpu_arm_init(const char *cpu_model);
 int cpu_arm_exec(CPUState *cpu);
-uint32_t do_arm_semihosting(CPUARMState *env);
+target_ulong do_arm_semihosting(CPUARMState *env);
 void aarch64_sync_32_to_64(CPUARMState *env);
 void aarch64_sync_64_to_32(CPUARMState *env);
 
@@ -583,6 +591,22 @@ void pmccntr_sync(CPUARMState *env);
 #define CPTR_TTA      (1U << 20)
 #define CPTR_TFP      (1U << 10)
 
+#define MDCR_EPMAD    (1U << 21)
+#define MDCR_EDAD     (1U << 20)
+#define MDCR_SPME     (1U << 17)
+#define MDCR_SDD      (1U << 16)
+#define MDCR_SPD      (3U << 14)
+#define MDCR_TDRA     (1U << 11)
+#define MDCR_TDOSA    (1U << 10)
+#define MDCR_TDA      (1U << 9)
+#define MDCR_TDE      (1U << 8)
+#define MDCR_HPME     (1U << 7)
+#define MDCR_TPM      (1U << 6)
+#define MDCR_TPMCR    (1U << 5)
+
+/* Not all of the MDCR_EL3 bits are present in the 32-bit SDCR */
+#define SDCR_VALID_MASK (MDCR_EPMAD | MDCR_EDAD | MDCR_SPME | MDCR_SPD)
+
 #define CPSR_M (0x1fU)
 #define CPSR_T (1U << 5)
 #define CPSR_F (1U << 6)
@@ -695,8 +719,17 @@ static inline void pstate_write(CPUARMState *env, uint32_t val)
 
 /* Return the current CPSR value.  */
 uint32_t cpsr_read(CPUARMState *env);
-/* Set the CPSR.  Note that some bits of mask must be all-set or all-clear.  */
-void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask);
+
+typedef enum CPSRWriteType {
+    CPSRWriteByInstr = 0,         /* from guest MSR or CPS */
+    CPSRWriteExceptionReturn = 1, /* from guest exception return insn */
+    CPSRWriteRaw = 2,             /* trust values, do not switch reg banks */
+    CPSRWriteByGDBStub = 3,       /* from the GDB stub */
+} CPSRWriteType;
+
+/* Set the CPSR.  Note that some bits of mask must be all-set or all-clear.*/
+void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
+                CPSRWriteType write_type);
 
 /* Return the current xPSR value.  */
 static inline uint32_t xpsr_read(CPUARMState *env)
@@ -920,7 +953,7 @@ static inline bool arm_is_secure_below_el3(CPUARMState *env)
     if (arm_feature(env, ARM_FEATURE_EL3)) {
         return !(env->cp15.scr_el3 & SCR_NS);
     } else {
-        /* If EL2 is not supported then the secure state is implementation
+        /* If EL3 is not supported then the secure state is implementation
          * defined, in which case QEMU defaults to non-secure.
          */
         return false;
@@ -958,18 +991,33 @@ static inline bool arm_is_secure(CPUARMState *env)
 /* Return true if the specified exception level is running in AArch64 state. */
 static inline bool arm_el_is_aa64(CPUARMState *env, int el)
 {
-    /* We don't currently support EL2, and this isn't valid for EL0
-     * (if we're in EL0, is_a64() is what you want, and if we're not in EL0
-     * then the state of EL0 isn't well defined.)
+    /* This isn't valid for EL0 (if we're in EL0, is_a64() is what you want,
+     * and if we're not in EL0 then the state of EL0 isn't well defined.)
      */
-    assert(el == 1 || el == 3);
+    assert(el >= 1 && el <= 3);
+    bool aa64 = arm_feature(env, ARM_FEATURE_AARCH64);
 
-    /* AArch64-capable CPUs always run with EL1 in AArch64 mode. This
-     * is a QEMU-imposed simplification which we may wish to change later.
-     * If we in future support EL2 and/or EL3, then the state of lower
-     * exception levels is controlled by the HCR.RW and SCR.RW bits.
+    /* The highest exception level is always at the maximum supported
+     * register width, and then lower levels have a register width controlled
+     * by bits in the SCR or HCR registers.
      */
-    return arm_feature(env, ARM_FEATURE_AARCH64);
+    if (el == 3) {
+        return aa64;
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        aa64 = aa64 && (env->cp15.scr_el3 & SCR_RW);
+    }
+
+    if (el == 2) {
+        return aa64;
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !arm_is_secure_below_el3(env)) {
+        aa64 = aa64 && (env->cp15.hcr_el2 & HCR_RW);
+    }
+
+    return aa64;
 }
 
 /* Function for determing whether guest cp register reads and writes should
@@ -1008,11 +1056,11 @@ static inline bool access_secure_reg(CPUARMState *env)
  */
 #define A32_BANKED_CURRENT_REG_GET(_env, _regname)        \
     A32_BANKED_REG_GET((_env), _regname,                \
-                       ((!arm_el_is_aa64((_env), 3) && arm_is_secure(_env))))
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)))
 
 #define A32_BANKED_CURRENT_REG_SET(_env, _regname, _val)                       \
     A32_BANKED_REG_SET((_env), _regname,                                    \
-                       ((!arm_el_is_aa64((_env), 3) && arm_is_secure(_env))),  \
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)), \
                        (_val))
 
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf);
@@ -1228,6 +1276,18 @@ static inline bool cptype_valid(int cptype)
 #define PL1_RW (PL1_R | PL1_W)
 #define PL0_RW (PL0_R | PL0_W)
 
+/* Return the highest implemented Exception Level */
+static inline int arm_highest_el(CPUARMState *env)
+{
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        return 3;
+    }
+    if (arm_feature(env, ARM_FEATURE_EL2)) {
+        return 2;
+    }
+    return 1;
+}
+
 /* Return the current Exception Level (as per ARMv8; note that this differs
  * from the ARMv7 Privilege Level).
  */
@@ -1280,6 +1340,14 @@ typedef enum CPAccessResult {
     /* As CP_ACCESS_TRAP, but for traps directly to EL2 or EL3 */
     CP_ACCESS_TRAP_EL2 = 3,
     CP_ACCESS_TRAP_EL3 = 4,
+    /* As CP_ACCESS_UNCATEGORIZED, but for traps directly to EL2 or EL3 */
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL2 = 5,
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL3 = 6,
+    /* Access fails and results in an exception syndrome for an FP access,
+     * trapped directly to EL2 or EL3
+     */
+    CP_ACCESS_TRAP_FP_EL2 = 7,
+    CP_ACCESS_TRAP_FP_EL3 = 8,
 } CPAccessResult;
 
 /* Access functions for coprocessor registers. These cannot fail and
@@ -1289,7 +1357,9 @@ typedef uint64_t CPReadFn(CPUARMState *env, const ARMCPRegInfo *opaque);
 typedef void CPWriteFn(CPUARMState *env, const ARMCPRegInfo *opaque,
                        uint64_t value);
 /* Access permission check functions for coprocessor registers. */
-typedef CPAccessResult CPAccessFn(CPUARMState *env, const ARMCPRegInfo *opaque);
+typedef CPAccessResult CPAccessFn(CPUARMState *env,
+                                  const ARMCPRegInfo *opaque,
+                                  bool isread);
 /* Hook function for register reset */
 typedef void CPResetFn(CPUARMState *env, const ARMCPRegInfo *opaque);
 
@@ -1445,6 +1515,9 @@ static inline bool cp_access_ok(int current_el,
     return (ri->access >> ((current_el * 2) + isread)) & 1;
 }
 
+/* Raw read of a coprocessor register (as needed for migration, etc) */
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri);
+
 /**
  * write_list_to_cpustate
  * @cpu: ARMCPU
@@ -1477,7 +1550,7 @@ bool write_list_to_cpustate(ARMCPU *cpu);
  */
 bool write_cpustate_to_list(ARMCPU *cpu);
 
-/* Does the core conform to the the "MicroController" profile. e.g. Cortex-M3.
+/* Does the core conform to the "MicroController" profile. e.g. Cortex-M3.
    Note the M in older cores (eg. ARM7TDMI) stands for Multiply. These are
    conventional cores (ie. Application or Realtime profile).  */
 
@@ -1509,8 +1582,6 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
     CPUARMState *env = cs->env_ptr;
     unsigned int cur_el = arm_current_el(env);
     bool secure = arm_is_secure(env);
-    uint32_t scr;
-    uint32_t hcr;
     bool pstate_unmasked;
     int8_t unmasked = 0;
 
@@ -1524,31 +1595,10 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
 
     switch (excp_idx) {
     case EXCP_FIQ:
-        /* If FIQs are routed to EL3 or EL2 then there are cases where we
-         * override the CPSR.F in determining if the exception is masked or
-         * not.  If neither of these are set then we fall back to the CPSR.F
-         * setting otherwise we further assess the state below.
-         */
-        hcr = (env->cp15.hcr_el2 & HCR_FMO);
-        scr = (env->cp15.scr_el3 & SCR_FIQ);
-
-        /* When EL3 is 32-bit, the SCR.FW bit controls whether the CPSR.F bit
-         * masks FIQ interrupts when taken in non-secure state.  If SCR.FW is
-         * set then FIQs can be masked by CPSR.F when non-secure but only
-         * when FIQs are only routed to EL3.
-         */
-        scr &= !((env->cp15.scr_el3 & SCR_FW) && !hcr);
         pstate_unmasked = !(env->daif & PSTATE_F);
         break;
 
     case EXCP_IRQ:
-        /* When EL3 execution state is 32-bit, if HCR.IMO is set then we may
-         * override the CPSR.I masking when in non-secure state.  The SCR.IRQ
-         * setting has already been taken into consideration when setting the
-         * target EL, so it does not have a further affect here.
-         */
-        hcr = (env->cp15.hcr_el2 & HCR_IMO);
-        scr = false;
         pstate_unmasked = !(env->daif & PSTATE_I);
         break;
 
@@ -1573,8 +1623,58 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
      * interrupt.
      */
     if ((target_el > cur_el) && (target_el != 1)) {
-        if (arm_el_is_aa64(env, 3) || ((scr || hcr) && (!secure))) {
-            unmasked = 1;
+        /* Exceptions targeting a higher EL may not be maskable */
+        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+            /* 64-bit masking rules are simple: exceptions to EL3
+             * can't be masked, and exceptions to EL2 can only be
+             * masked from Secure state. The HCR and SCR settings
+             * don't affect the masking logic, only the interrupt routing.
+             */
+            if (target_el == 3 || !secure) {
+                unmasked = 1;
+            }
+        } else {
+            /* The old 32-bit-only environment has a more complicated
+             * masking setup. HCR and SCR bits not only affect interrupt
+             * routing but also change the behaviour of masking.
+             */
+            bool hcr, scr;
+
+            switch (excp_idx) {
+            case EXCP_FIQ:
+                /* If FIQs are routed to EL3 or EL2 then there are cases where
+                 * we override the CPSR.F in determining if the exception is
+                 * masked or not. If neither of these are set then we fall back
+                 * to the CPSR.F setting otherwise we further assess the state
+                 * below.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_FMO);
+                scr = (env->cp15.scr_el3 & SCR_FIQ);
+
+                /* When EL3 is 32-bit, the SCR.FW bit controls whether the
+                 * CPSR.F bit masks FIQ interrupts when taken in non-secure
+                 * state. If SCR.FW is set then FIQs can be masked by CPSR.F
+                 * when non-secure but only when FIQs are only routed to EL3.
+                 */
+                scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr);
+                break;
+            case EXCP_IRQ:
+                /* When EL3 execution state is 32-bit, if HCR.IMO is set then
+                 * we may override the CPSR.I masking when in non-secure state.
+                 * The SCR.IRQ setting has already been taken into consideration
+                 * when setting the target EL, so it does not have a further
+                 * affect here.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_IMO);
+                scr = false;
+                break;
+            default:
+                g_assert_not_reached();
+            }
+
+            if ((scr || hcr) && !secure) {
+                unmasked = 1;
+            }
         }
     }
 
@@ -1587,7 +1687,6 @@ static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
 #define cpu_init(cpu_model) CPU(cpu_arm_init(cpu_model))
 
 #define cpu_exec cpu_arm_exec
-#define cpu_gen_code cpu_arm_gen_code
 #define cpu_signal_handler cpu_arm_signal_handler
 #define cpu_list arm_cpu_list
 
@@ -1667,7 +1766,7 @@ static inline int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx)
 }
 
 /* Determine the current mmu_idx to use for normal loads/stores */
-static inline int cpu_mmu_index(CPUARMState *env)
+static inline int cpu_mmu_index(CPUARMState *env, bool ifetch)
 {
     int el = arm_current_el(env);
 
@@ -1677,16 +1776,43 @@ static inline int cpu_mmu_index(CPUARMState *env)
     return el;
 }
 
-/* Return the Exception Level targeted by debug exceptions;
- * currently always EL1 since we don't implement EL2 or EL3.
- */
+/* Indexes used when registering address spaces with cpu_address_space_init */
+typedef enum ARMASIdx {
+    ARMASIdx_NS = 0,
+    ARMASIdx_S = 1,
+} ARMASIdx;
+
+/* Return the Exception Level targeted by debug exceptions. */
 static inline int arm_debug_target_el(CPUARMState *env)
 {
-    return 1;
+    bool secure = arm_is_secure(env);
+    bool route_to_el2 = false;
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure) {
+        route_to_el2 = env->cp15.hcr_el2 & HCR_TGE ||
+                       env->cp15.mdcr_el2 & (1 << 8);
+    }
+
+    if (route_to_el2) {
+        return 2;
+    } else if (arm_feature(env, ARM_FEATURE_EL3) &&
+               !arm_el_is_aa64(env, 3) && secure) {
+        return 3;
+    } else {
+        return 1;
+    }
 }
 
 static inline bool aa64_generate_debug_exceptions(CPUARMState *env)
 {
+    if (arm_is_secure(env)) {
+        /* MDCR_EL3.SDD disables debug events from Secure state */
+        if (extract32(env->cp15.mdcr_el3, 16, 1) != 0
+            || arm_current_el(env) == 3) {
+            return false;
+        }
+    }
+
     if (arm_current_el(env) == arm_debug_target_el(env)) {
         if ((extract32(env->cp15.mdscr_el1, 13, 1) == 0)
             || (env->daif & PSTATE_D)) {
@@ -1698,10 +1824,42 @@ static inline bool aa64_generate_debug_exceptions(CPUARMState *env)
 
 static inline bool aa32_generate_debug_exceptions(CPUARMState *env)
 {
-    if (arm_current_el(env) == 0 && arm_el_is_aa64(env, 1)) {
+    int el = arm_current_el(env);
+
+    if (el == 0 && arm_el_is_aa64(env, 1)) {
         return aa64_generate_debug_exceptions(env);
     }
-    return arm_current_el(env) != 2;
+
+    if (arm_is_secure(env)) {
+        int spd;
+
+        if (el == 0 && (env->cp15.sder & 1)) {
+            /* SDER.SUIDEN means debug exceptions from Secure EL0
+             * are always enabled. Otherwise they are controlled by
+             * SDCR.SPD like those from other Secure ELs.
+             */
+            return true;
+        }
+
+        spd = extract32(env->cp15.mdcr_el3, 14, 2);
+        switch (spd) {
+        case 1:
+            /* SPD == 0b01 is reserved, but behaves as 0b00. */
+        case 0:
+            /* For 0b00 we return true if external secure invasive debug
+             * is enabled. On real hardware this is controlled by external
+             * signals to the core. QEMU always permits debug, and behaves
+             * as if DBGEN, SPIDEN, NIDEN and SPNIDEN are all tied high.
+             */
+            return true;
+        case 2:
+            return false;
+        case 3:
+            return true;
+        }
+    }
+
+    return el != 2;
 }
 
 /* Return true if debugging exceptions are currently enabled.
@@ -1737,6 +1895,53 @@ static inline bool arm_singlestep_active(CPUARMState *env)
         && arm_generate_debug_exceptions(env);
 }
 
+static inline bool arm_sctlr_b(CPUARMState *env)
+{
+    return
+        /* We need not implement SCTLR.ITD in user-mode emulation, so
+         * let linux-user ignore the fact that it conflicts with SCTLR_B.
+         * This lets people run BE32 binaries with "-cpu any".
+         */
+#ifndef CONFIG_USER_ONLY
+        !arm_feature(env, ARM_FEATURE_V7) &&
+#endif
+        (env->cp15.sctlr_el[1] & SCTLR_B) != 0;
+}
+
+/* Return true if the processor is in big-endian mode. */
+static inline bool arm_cpu_data_is_big_endian(CPUARMState *env)
+{
+    int cur_el;
+
+    /* In 32bit endianness is determined by looking at CPSR's E bit */
+    if (!is_a64(env)) {
+        return
+#ifdef CONFIG_USER_ONLY
+            /* In system mode, BE32 is modelled in line with the
+             * architecture (as word-invariant big-endianness), where loads
+             * and stores are done little endian but from addresses which
+             * are adjusted by XORing with the appropriate constant. So the
+             * endianness to use for the raw data access is not affected by
+             * SCTLR.B.
+             * In user mode, however, we model BE32 as byte-invariant
+             * big-endianness (because user-only code cannot tell the
+             * difference), and so we need to use a data access endianness
+             * that depends on SCTLR.B.
+             */
+            arm_sctlr_b(env) ||
+#endif
+                ((env->uncached_cpsr & CPSR_E) ? 1 : 0);
+    }
+
+    cur_el = arm_current_el(env);
+
+    if (cur_el == 0) {
+        return (env->cp15.sctlr_el[1] & SCTLR_E0E) != 0;
+    }
+
+    return (env->cp15.sctlr_el[cur_el] & SCTLR_EE) != 0;
+}
+
 #include "exec/cpu-all.h"
 
 /* Bit usage in the TB flags field: bit 31 indicates whether we are
@@ -1767,8 +1972,8 @@ static inline bool arm_singlestep_active(CPUARMState *env)
 #define ARM_TBFLAG_VFPEN_MASK       (1 << ARM_TBFLAG_VFPEN_SHIFT)
 #define ARM_TBFLAG_CONDEXEC_SHIFT   8
 #define ARM_TBFLAG_CONDEXEC_MASK    (0xff << ARM_TBFLAG_CONDEXEC_SHIFT)
-#define ARM_TBFLAG_BSWAP_CODE_SHIFT 16
-#define ARM_TBFLAG_BSWAP_CODE_MASK  (1 << ARM_TBFLAG_BSWAP_CODE_SHIFT)
+#define ARM_TBFLAG_SCTLR_B_SHIFT    16
+#define ARM_TBFLAG_SCTLR_B_MASK     (1 << ARM_TBFLAG_SCTLR_B_SHIFT)
 /* We store the bottom two bits of the CPAR as TB flags and handle
  * checks on the other bits at runtime
  */
@@ -1780,6 +1985,8 @@ static inline bool arm_singlestep_active(CPUARMState *env)
  */
 #define ARM_TBFLAG_NS_SHIFT         19
 #define ARM_TBFLAG_NS_MASK          (1 << ARM_TBFLAG_NS_SHIFT)
+#define ARM_TBFLAG_BE_DATA_SHIFT    20
+#define ARM_TBFLAG_BE_DATA_MASK     (1 << ARM_TBFLAG_BE_DATA_SHIFT)
 
 /* Bit usage when in AArch64 state: currently we have no A64 specific bits */
 
@@ -1804,12 +2011,34 @@ static inline bool arm_singlestep_active(CPUARMState *env)
     (((F) & ARM_TBFLAG_VFPEN_MASK) >> ARM_TBFLAG_VFPEN_SHIFT)
 #define ARM_TBFLAG_CONDEXEC(F) \
     (((F) & ARM_TBFLAG_CONDEXEC_MASK) >> ARM_TBFLAG_CONDEXEC_SHIFT)
-#define ARM_TBFLAG_BSWAP_CODE(F) \
-    (((F) & ARM_TBFLAG_BSWAP_CODE_MASK) >> ARM_TBFLAG_BSWAP_CODE_SHIFT)
+#define ARM_TBFLAG_SCTLR_B(F) \
+    (((F) & ARM_TBFLAG_SCTLR_B_MASK) >> ARM_TBFLAG_SCTLR_B_SHIFT)
 #define ARM_TBFLAG_XSCALE_CPAR(F) \
     (((F) & ARM_TBFLAG_XSCALE_CPAR_MASK) >> ARM_TBFLAG_XSCALE_CPAR_SHIFT)
 #define ARM_TBFLAG_NS(F) \
     (((F) & ARM_TBFLAG_NS_MASK) >> ARM_TBFLAG_NS_SHIFT)
+#define ARM_TBFLAG_BE_DATA(F) \
+    (((F) & ARM_TBFLAG_BE_DATA_MASK) >> ARM_TBFLAG_BE_DATA_SHIFT)
+
+static inline bool bswap_code(bool sctlr_b)
+{
+#ifdef CONFIG_USER_ONLY
+    /* BE8 (SCTLR.B = 0, TARGET_WORDS_BIGENDIAN = 1) is mixed endian.
+     * The invalid combination SCTLR.B=1/CPSR.E=1/TARGET_WORDS_BIGENDIAN=0
+     * would also end up as a mixed-endian mode with BE code, LE data.
+     */
+    return
+#ifdef TARGET_WORDS_BIGENDIAN
+        1 ^
+#endif
+        sctlr_b;
+#else
+    /* All code access in ARM is little endian, and there are no loaders
+     * doing swaps that need to be reversed
+     */
+    return 0;
+#endif
+}
 
 /* Return the exception level to which FP-disabled exceptions should
  * be taken, or 0 if FP is enabled.
@@ -1876,6 +2105,17 @@ static inline int fp_exception_el(CPUARMState *env)
     return 0;
 }
 
+#ifdef CONFIG_USER_ONLY
+static inline bool arm_cpu_bswap_data(CPUARMState *env)
+{
+    return
+#ifdef TARGET_WORDS_BIGENDIAN
+       1 ^
+#endif
+       arm_cpu_data_is_big_endian(env);
+}
+#endif
+
 static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
                                         target_ulong *cs_base, int *flags)
 {
@@ -1888,7 +2128,7 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
             | (env->vfp.vec_len << ARM_TBFLAG_VECLEN_SHIFT)
             | (env->vfp.vec_stride << ARM_TBFLAG_VECSTRIDE_SHIFT)
             | (env->condexec_bits << ARM_TBFLAG_CONDEXEC_SHIFT)
-            | (env->bswap_code << ARM_TBFLAG_BSWAP_CODE_SHIFT);
+            | (arm_sctlr_b(env) << ARM_TBFLAG_SCTLR_B_SHIFT);
         if (!(access_secure_reg(env))) {
             *flags |= ARM_TBFLAG_NS_MASK;
         }
@@ -1900,7 +2140,7 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
                    << ARM_TBFLAG_XSCALE_CPAR_SHIFT);
     }
 
-    *flags |= (cpu_mmu_index(env) << ARM_TBFLAG_MMUIDX_SHIFT);
+    *flags |= (cpu_mmu_index(env, false) << ARM_TBFLAG_MMUIDX_SHIFT);
     /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
      * states defined in the ARM ARM for software singlestep:
      *  SS_ACTIVE   PSTATE.SS   State
@@ -1920,6 +2160,9 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
             }
         }
     }
+    if (arm_cpu_data_is_big_endian(env)) {
+        *flags |= ARM_TBFLAG_BE_DATA_MASK;
+    }
     *flags |= fp_exception_el(env) << ARM_TBFLAG_FPEXC_EL_SHIFT;
 
     *cs_base = 0;
@@ -1933,4 +2176,21 @@ enum {
     QEMU_PSCI_CONDUIT_HVC = 2,
 };
 
+#ifndef CONFIG_USER_ONLY
+/* Return the address space index to use for a memory access */
+static inline int arm_asidx_from_attrs(CPUState *cs, MemTxAttrs attrs)
+{
+    return attrs.secure ? ARMASIdx_S : ARMASIdx_NS;
+}
+
+/* Return the AddressSpace to use for a memory access
+ * (which depends on whether the access is S or NS, and whether
+ * the board gave us a separate AddressSpace for S accesses).
+ */
+static inline AddressSpace *arm_addressspace(CPUState *cs, MemTxAttrs attrs)
+{
+    return cpu_get_address_space(cs, arm_asidx_from_attrs(cs, attrs));
+}
+#endif
+
 #endif
diff --git a/qemu/target-arm/cpu64.c b/qemu/target-arm/cpu64.c
index 63c8b1cfa..1635debc1 100644
--- a/qemu/target-arm/cpu64.c
+++ b/qemu/target-arm/cpu64.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #if !defined(CONFIG_USER_ONLY)
@@ -108,6 +110,7 @@ static void aarch64_a57_initfn(Object *obj)
     set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
     set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
     set_feature(&cpu->env, ARM_FEATURE_CRC);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A57;
     cpu->midr = 0x411fd070;
     cpu->revidr = 0x00000000;
@@ -133,6 +136,8 @@ static void aarch64_a57_initfn(Object *obj)
     cpu->id_isar5 = 0x00011121;
     cpu->id_aa64pfr0 = 0x00002222;
     cpu->id_aa64dfr0 = 0x10305106;
+    cpu->pmceid0 = 0x00000000;
+    cpu->pmceid1 = 0x00000000;
     cpu->id_aa64isar0 = 0x00011120;
     cpu->id_aa64mmfr0 = 0x00001124;
     cpu->dbgdidr = 0x3516d000;
@@ -160,6 +165,7 @@ static void aarch64_a53_initfn(Object *obj)
     set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
     set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
     set_feature(&cpu->env, ARM_FEATURE_CRC);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A53;
     cpu->midr = 0x410fd034;
     cpu->revidr = 0x00000000;
@@ -286,19 +292,22 @@ static void aarch64_cpu_set_pc(CPUState *cs, vaddr value)
     }
 }
 
+static gchar *aarch64_gdb_arch_name(CPUState *cs)
+{
+    return g_strdup("aarch64");
+}
+
 static void aarch64_cpu_class_init(ObjectClass *oc, void *data)
 {
     CPUClass *cc = CPU_CLASS(oc);
 
-#if !defined(CONFIG_USER_ONLY)
-    cc->do_interrupt = aarch64_cpu_do_interrupt;
-#endif
     cc->cpu_exec_interrupt = arm_cpu_exec_interrupt;
     cc->set_pc = aarch64_cpu_set_pc;
     cc->gdb_read_register = aarch64_cpu_gdb_read_register;
     cc->gdb_write_register = aarch64_cpu_gdb_write_register;
     cc->gdb_num_core_regs = 34;
     cc->gdb_core_xml_file = "aarch64-core.xml";
+    cc->gdb_arch_name = aarch64_gdb_arch_name;
 }
 
 static void aarch64_cpu_register(const ARMCPUInfo *info)
diff --git a/qemu/target-arm/crypto_helper.c b/qemu/target-arm/crypto_helper.c
index 5d2283806..3b6df3f41 100644
--- a/qemu/target-arm/crypto_helper.c
+++ b/qemu/target-arm/crypto_helper.c
@@ -9,7 +9,7 @@
  * version 2 of the License, or (at your option) any later version.
  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
diff --git a/qemu/target-arm/gdbstub.c b/qemu/target-arm/gdbstub.c
index 1c3439654..3ba9aadd4 100644
--- a/qemu/target-arm/gdbstub.c
+++ b/qemu/target-arm/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
@@ -94,7 +94,7 @@ int arm_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
         return 4;
     case 25:
         /* CPSR */
-        cpsr_write(env, tmp, 0xffffffff);
+        cpsr_write(env, tmp, 0xffffffff, CPSRWriteByGDBStub);
         return 4;
     }
     /* Unknown register.  */
diff --git a/qemu/target-arm/gdbstub64.c b/qemu/target-arm/gdbstub64.c
index 8f3b8d177..634c6bc6f 100644
--- a/qemu/target-arm/gdbstub64.c
+++ b/qemu/target-arm/gdbstub64.c
@@ -16,7 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-arm/helper-a64.c b/qemu/target-arm/helper-a64.c
index 08c95a3f5..c7bfb4d8f 100644
--- a/qemu/target-arm/helper-a64.c
+++ b/qemu/target-arm/helper-a64.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "exec/helper-proto.h"
@@ -70,20 +71,7 @@ uint32_t HELPER(clz32)(uint32_t x)
 
 uint64_t HELPER(rbit64)(uint64_t x)
 {
-    /* assign the correct byte position */
-    x = bswap64(x);
-
-    /* assign the correct nibble position */
-    x = ((x & 0xf0f0f0f0f0f0f0f0ULL) >> 4)
-        | ((x & 0x0f0f0f0f0f0f0f0fULL) << 4);
-
-    /* assign the correct bit position */
-    x = ((x & 0x8888888888888888ULL) >> 3)
-        | ((x & 0x4444444444444444ULL) >> 1)
-        | ((x & 0x2222222222222222ULL) << 1)
-        | ((x & 0x1111111111111111ULL) << 3);
-
-    return x;
+    return revbit64(x);
 }
 
 /* Convert a softfloat float_relation_ (as returned by
@@ -455,92 +443,3 @@ uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes)
     /* Linux crc32c converts the output to one's complement.  */
     return crc32c(acc, buf, bytes) ^ 0xffffffff;
 }
-
-#if !defined(CONFIG_USER_ONLY)
-
-/* Handle a CPU exception.  */
-void aarch64_cpu_do_interrupt(CPUState *cs)
-{
-    ARMCPU *cpu = ARM_CPU(cs);
-    CPUARMState *env = &cpu->env;
-    unsigned int new_el = env->exception.target_el;
-    target_ulong addr = env->cp15.vbar_el[new_el];
-    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
-
-    if (arm_current_el(env) < new_el) {
-        if (env->aarch64) {
-            addr += 0x400;
-        } else {
-            addr += 0x600;
-        }
-    } else if (pstate_read(env) & PSTATE_SP) {
-        addr += 0x200;
-    }
-
-    arm_log_exception(cs->exception_index);
-    qemu_log_mask(CPU_LOG_INT, "...from EL%d\n", arm_current_el(env));
-    if (qemu_loglevel_mask(CPU_LOG_INT)
-        && !excp_is_internal(cs->exception_index)) {
-        qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%" PRIx32 "\n",
-                      env->exception.syndrome);
-    }
-
-    if (arm_is_psci_call(cpu, cs->exception_index)) {
-        arm_handle_psci_call(cpu);
-        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
-        return;
-    }
-
-    switch (cs->exception_index) {
-    case EXCP_PREFETCH_ABORT:
-    case EXCP_DATA_ABORT:
-        env->cp15.far_el[new_el] = env->exception.vaddress;
-        qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
-                      env->cp15.far_el[new_el]);
-        /* fall through */
-    case EXCP_BKPT:
-    case EXCP_UDEF:
-    case EXCP_SWI:
-    case EXCP_HVC:
-    case EXCP_HYP_TRAP:
-    case EXCP_SMC:
-        env->cp15.esr_el[new_el] = env->exception.syndrome;
-        break;
-    case EXCP_IRQ:
-    case EXCP_VIRQ:
-        addr += 0x80;
-        break;
-    case EXCP_FIQ:
-    case EXCP_VFIQ:
-        addr += 0x100;
-        break;
-    default:
-        cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
-    }
-
-    if (is_a64(env)) {
-        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
-        aarch64_save_sp(env, arm_current_el(env));
-        env->elr_el[new_el] = env->pc;
-    } else {
-        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = cpsr_read(env);
-        if (!env->thumb) {
-            env->cp15.esr_el[new_el] |= 1 << 25;
-        }
-        env->elr_el[new_el] = env->regs[15];
-
-        aarch64_sync_32_to_64(env);
-
-        env->condexec_bits = 0;
-    }
-    qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
-                  env->elr_el[new_el]);
-
-    pstate_write(env, PSTATE_DAIF | new_mode);
-    env->aarch64 = 1;
-    aarch64_restore_sp(env, new_el);
-
-    env->pc = addr;
-    cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
-}
-#endif
diff --git a/qemu/target-arm/helper.c b/qemu/target-arm/helper.c
index 01f0d0dac..09638b2e7 100644
--- a/qemu/target-arm/helper.c
+++ b/qemu/target-arm/helper.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "internals.h"
 #include "exec/gdbstub.h"
@@ -11,12 +12,22 @@
 #include "arm_ldst.h"
 #include <zlib.h> /* For crc32 */
 #include "exec/semihost.h"
+#include "sysemu/kvm.h"
+
+#define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
 
 #ifndef CONFIG_USER_ONLY
-static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
-                                 int access_type, ARMMMUIdx mmu_idx,
-                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                                 target_ulong *page_size, uint32_t *fsr);
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi);
+
+static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
+                               int access_type, ARMMMUIdx mmu_idx,
+                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi);
 
 /* Definitions for the PMCCNTR and PMCR registers */
 #define PMCRD   0x8
@@ -144,7 +155,7 @@ static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
     return (char *)env + ri->fieldoffset;
 }
 
-static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* Raw read of a coprocessor register (as needed for migration, etc). */
     if (ri->type & ARM_CP_CONST) {
@@ -325,6 +336,127 @@ void init_cpreg_list(ARMCPU *cpu)
     g_list_free(keys);
 }
 
+/*
+ * Some registers are not accessible if EL3.NS=0 and EL3 is using AArch32 but
+ * they are accessible when EL3 is using AArch64 regardless of EL3.NS.
+ *
+ * access_el3_aa32ns: Used to check AArch32 register views.
+ * access_el3_aa32ns_aa64any: Used to check both AArch32/64 register views.
+ */
+static CPAccessResult access_el3_aa32ns(CPUARMState *env,
+                                        const ARMCPRegInfo *ri,
+                                        bool isread)
+{
+    bool secure = arm_is_secure_below_el3(env);
+
+    assert(!arm_el_is_aa64(env, 3));
+    if (secure) {
+        return CP_ACCESS_TRAP_UNCATEGORIZED;
+    }
+    return CP_ACCESS_OK;
+}
+
+static CPAccessResult access_el3_aa32ns_aa64any(CPUARMState *env,
+                                                const ARMCPRegInfo *ri,
+                                                bool isread)
+{
+    if (!arm_el_is_aa64(env, 3)) {
+        return access_el3_aa32ns(env, ri, isread);
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Some secure-only AArch32 registers trap to EL3 if used from
+ * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
+ * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
+ * We assume that the .access field is set to PL1_RW.
+ */
+static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
+                                            const ARMCPRegInfo *ri,
+                                            bool isread)
+{
+    if (arm_current_el(env) == 3) {
+        return CP_ACCESS_OK;
+    }
+    if (arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    /* This will be EL1 NS and EL2 NS, which just UNDEF */
+    return CP_ACCESS_TRAP_UNCATEGORIZED;
+}
+
+/* Check for traps to "powerdown debug" registers, which are controlled
+ * by MDCR.TDOSA
+ */
+static CPAccessResult access_tdosa(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDOSA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDOSA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to "debug ROM" registers, which are controlled
+ * by MDCR_EL2.TDRA for EL2 but by the more general MDCR_EL3.TDA for EL3.
+ */
+static CPAccessResult access_tdra(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDRA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to general debug registers, which are controlled
+ * by MDCR_EL2.TDA for EL2 and MDCR_EL3.TDA for EL3.
+ */
+static CPAccessResult access_tda(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to performance monitor registers, which are controlled
+ * by MDCR_EL2.TPM for EL2 and MDCR_EL3.TPM for EL3.
+ */
+static CPAccessResult access_tpm(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -595,7 +727,8 @@ static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     env->cp15.cpacr_el1 = value;
 }
 
-static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
 {
     if (arm_feature(env, ARM_FEATURE_V8)) {
         /* Check if CPACR accesses are to be trapped to EL2 */
@@ -612,7 +745,8 @@ static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri)
     return CP_ACCESS_OK;
 }
 
-static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
 {
     /* Check if CPTR accesses are set to trap to EL3 */
     if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
@@ -627,8 +761,12 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
     { .name = "MVA_prefetch",
       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NOP },
+    /* We need to break the TB after ISB to execute self-modifying code
+     * correctly and also to take any pending interrupts immediately.
+     * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
+     */
     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
-      .access = PL0_W, .type = ARM_CP_NOP },
+      .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
       .access = PL0_W, .type = ARM_CP_NOP },
     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
@@ -650,14 +788,26 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
-static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
 {
     /* Performance monitor registers user accessibility is controlled
-     * by PMUSERENR.
+     * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable
+     * trapping to EL2 or EL3 for other accesses.
      */
-    if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
+    int el = arm_current_el(env);
+
+    if (el == 0 && !env->cp15.c9_pmuserenr) {
         return CP_ACCESS_TRAP;
     }
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+
     return CP_ACCESS_OK;
 }
 
@@ -678,8 +828,8 @@ void pmccntr_sync(CPUARMState *env)
 {
     uint64_t temp_ticks;
 
-    temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                          get_ticks_per_sec(), 1000000);
+    temp_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                          ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -717,8 +867,8 @@ static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
         return env->cp15.c15_ccnt;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -738,8 +888,8 @@ static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
         return;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -936,6 +1086,13 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .accessfn = pmreg_access,
       .writefn = pmovsr_write,
       .raw_writefn = raw_write },
+    { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
+      .writefn = pmovsr_write,
+      .raw_writefn = raw_write },
     /* Unimplemented so WI. */
     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
@@ -973,19 +1130,30 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
       .accessfn = pmreg_access },
     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R | PL1_RW,
+      .access = PL0_R | PL1_RW, .accessfn = access_tpm,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
+      .resetvalue = 0,
+      .writefn = pmuserenr_write, .raw_writefn = raw_write },
+    { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
+      .access = PL0_R | PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
       .resetvalue = 0,
       .writefn = pmuserenr_write, .raw_writefn = raw_write },
     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW,
+      .access = PL1_RW, .accessfn = access_tpm,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
       .resetvalue = 0,
       .writefn = pmintenset_write, .raw_writefn = raw_write },
     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
-      .access = PL1_RW, .type = ARM_CP_ALIAS,
+      .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
       .writefn = pmintenclr_write, },
+    { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
+      .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
+      .writefn = pmintenclr_write },
     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .writefn = vbar_write,
@@ -1022,6 +1190,10 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
       .resetvalue = 0 },
+    { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
+      .resetvalue = 0 },
     /* For non-long-descriptor page tables these are PRRR and NMRR;
      * regardless they still act as reads-as-written for QEMU.
      */
@@ -1090,7 +1262,8 @@ static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     env->teecr = value;
 }
 
-static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    bool isread)
 {
     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
         return CP_ACCESS_TRAP;
@@ -1143,57 +1316,132 @@ static const ARMCPRegInfo v6k_cp_reginfo[] = {
 
 #ifndef CONFIG_USER_ONLY
 
-static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
-    if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
-        return CP_ACCESS_TRAP;
+    /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
+     * Writable only at the highest implemented exception level.
+     */
+    int el = arm_current_el(env);
+
+    switch (el) {
+    case 0:
+        if (!extract32(env->cp15.c14_cntkctl, 0, 2)) {
+            return CP_ACCESS_TRAP;
+        }
+        break;
+    case 1:
+        if (!isread && ri->state == ARM_CP_STATE_AA32 &&
+            arm_is_secure_below_el3(env)) {
+            /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
+            return CP_ACCESS_TRAP_UNCATEGORIZED;
+        }
+        break;
+    case 2:
+    case 3:
+        break;
     }
+
+    if (!isread && el < arm_highest_el(env)) {
+        return CP_ACCESS_TRAP_UNCATEGORIZED;
+    }
+
     return CP_ACCESS_OK;
 }
 
-static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
+static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
+                                        bool isread)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 0, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
-static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
+static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
+                                      bool isread)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
      * EL0[PV]TEN is zero.
      */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 1, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
 static CPAccessResult gt_pct_access(CPUARMState *env,
-                                         const ARMCPRegInfo *ri)
+                                    const ARMCPRegInfo *ri,
+                                    bool isread)
 {
-    return gt_counter_access(env, GTIMER_PHYS);
+    return gt_counter_access(env, GTIMER_PHYS, isread);
 }
 
 static CPAccessResult gt_vct_access(CPUARMState *env,
-                                         const ARMCPRegInfo *ri)
+                                    const ARMCPRegInfo *ri,
+                                    bool isread)
 {
-    return gt_counter_access(env, GTIMER_VIRT);
+    return gt_counter_access(env, GTIMER_VIRT, isread);
 }
 
-static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    return gt_timer_access(env, GTIMER_PHYS);
+    return gt_timer_access(env, GTIMER_PHYS, isread);
 }
 
-static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    return gt_timer_access(env, GTIMER_VIRT);
+    return gt_timer_access(env, GTIMER_VIRT, isread);
+}
+
+static CPAccessResult gt_stimer_access(CPUARMState *env,
+                                       const ARMCPRegInfo *ri,
+                                       bool isread)
+{
+    /* The AArch64 register view of the secure physical timer is
+     * always accessible from EL3, and configurably accessible from
+     * Secure EL1.
+     */
+    switch (arm_current_el(env)) {
+    case 1:
+        if (!arm_is_secure(env)) {
+            return CP_ACCESS_TRAP;
+        }
+        if (!(env->cp15.scr_el3 & SCR_ST)) {
+            return CP_ACCESS_TRAP_EL3;
+        }
+        return CP_ACCESS_OK;
+    case 0:
+    case 2:
+        return CP_ACCESS_TRAP;
+    case 3:
+        return CP_ACCESS_OK;
+    default:
+        g_assert_not_reached();
+    }
 }
 
 static uint64_t gt_get_countervalue(CPUARMState *env)
@@ -1209,9 +1457,11 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
         /* Timer enabled: calculate and set current ISTATUS, irq, and
          * reset timer to when ISTATUS next has to change
          */
+        uint64_t offset = timeridx == GTIMER_VIRT ?
+                                      cpu->env.cp15.cntvoff_el2 : 0;
         uint64_t count = gt_get_countervalue(&cpu->env);
         /* Note that this must be unsigned 64 bit arithmetic: */
-        int istatus = count >= gt->cval;
+        int istatus = count - offset >= gt->cval;
         uint64_t nexttick;
 
         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
@@ -1222,7 +1472,7 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
             nexttick = UINT64_MAX;
         } else {
             /* Next transition is when we hit cval */
-            nexttick = gt->cval;
+            nexttick = gt->cval + offset;
         }
         /* Note that the desired next expiry time might be beyond the
          * signed-64-bit range of a QEMUTimer -- in this case we just
@@ -1241,10 +1491,10 @@ static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
     }
 }
 
-static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
+                           int timeridx)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->opc1 & 1;
 
     timer_del(cpu->gt_timer[timeridx]);
 }
@@ -1254,38 +1504,44 @@ static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
     return gt_get_countervalue(env);
 }
 
+static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_get_countervalue(env) - env->cp15.cntvoff_el2;
+}
+
 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->opc1 & 1;
-
     env->cp15.c14_timer[timeridx].cval = value;
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
-static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
+                             int timeridx)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
-                      gt_get_countervalue(env));
+                      (gt_get_countervalue(env) - offset));
 }
 
 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
-    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
+    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
                                          sextract64(value, 0, 32);
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         int timeridx,
                          uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->crm & 1;
     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
 
     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
@@ -1301,6 +1557,127 @@ static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
     }
 }
 
+static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    raw_write(env, ri, value);
+    gt_recalc_timer(cpu, GTIMER_VIRT);
+}
+
+static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_HYP, value);
+}
+
+static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_SEC, value);
+}
+
+static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_SEC, value);
+}
+
+static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_SEC, value);
+}
+
 void arm_gt_ptimer_cb(void *opaque)
 {
     ARMCPU *cpu = opaque;
@@ -1315,6 +1692,20 @@ void arm_gt_vtimer_cb(void *opaque)
     gt_recalc_timer(cpu, GTIMER_VIRT);
 }
 
+void arm_gt_htimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_HYP);
+}
+
+void arm_gt_stimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_SEC);
+}
+
 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     /* Note that CNTFRQ is purely reads-as-written for the benefit
      * of software; writing it doesn't actually change the timer frequency.
@@ -1340,11 +1731,21 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     },
     /* per-timer control */
     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
+      .secure = ARM_CP_SECSTATE_NS,
       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_PHYS].ctl),
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CTL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
+      .secure = ARM_CP_SECSTATE_S,
+      .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .fieldoffset = offsetoflow32(CPUARMState,
+                                   cp15.c14_timer[GTIMER_SEC].ctl),
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
@@ -1352,14 +1753,14 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_VIRT].ctl),
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
@@ -1367,30 +1768,38 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     /* TimerValue views: a 32 bit downcounting view of the underlying state */
     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
+      .secure = ARM_CP_SECSTATE_NS,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
+    },
+    { .name = "CNTP_TVAL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
+      .secure = ARM_CP_SECSTATE_S,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
     },
     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .accessfn = gt_ptimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
     },
     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .accessfn = gt_vtimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     /* The counter itself */
     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
@@ -1401,27 +1810,34 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
-      .accessfn = gt_pct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .accessfn = gt_pct_access, .readfn = gt_cnt_read,
     },
     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
       .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
+      .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
     },
     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
-      .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
     },
     /* Comparison value, indicating when the timer goes off */
     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_NS,
       .access = PL1_RW | PL0_R,
       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
       .accessfn = gt_ptimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CVAL(S)", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_S,
+      .access = PL1_RW | PL0_R,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .accessfn = gt_ptimer_access,
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
@@ -1429,14 +1845,14 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
       .resetvalue = 0, .accessfn = gt_ptimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
       .access = PL1_RW | PL0_R,
       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
       .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
@@ -1444,7 +1860,33 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
       .resetvalue = 0, .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
+    },
+    /* Secure timer -- this is actually restricted to only EL3
+     * and configurably Secure-EL1 via the accessfn.
+     */
+    { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .readfn = gt_sec_tval_read,
+      .writefn = gt_sec_tval_write,
+      .resetfn = gt_sec_timer_reset,
+    },
+    { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
+      .resetvalue = 0,
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     REGINFO_SENTINEL
 };
@@ -1474,15 +1916,21 @@ static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 #ifndef CONFIG_USER_ONLY
 /* get_phys_addr() isn't present for user-mode-only targets */
 
-static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 bool isread)
 {
     if (ri->opc2 & 4) {
-        /* Other states are only available with TrustZone; in
-         * a non-TZ implementation these registers don't exist
-         * at all, which is an Uncategorized trap. This underdecoding
-         * is safe because the reginfo is NO_RAW.
+        /* The ATS12NSO* operations must trap to EL3 if executed in
+         * Secure EL1 (which can only happen if EL3 is AArch64).
+         * They are simply UNDEF if executed from NS EL1.
+         * They function normally from EL2 or EL3.
          */
-        return CP_ACCESS_TRAP_UNCATEGORIZED;
+        if (arm_current_el(env) == 1) {
+            if (arm_is_secure_below_el3(env)) {
+                return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
+            }
+            return CP_ACCESS_TRAP_UNCATEGORIZED;
+        }
     }
     return CP_ACCESS_OK;
 }
@@ -1497,9 +1945,10 @@ static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
     bool ret;
     uint64_t par64;
     MemTxAttrs attrs = {};
+    ARMMMUFaultInfo fi = {};
 
     ret = get_phys_addr(env, value, access_type, mmu_idx,
-                        &phys_addr, &attrs, &prot, &page_size, &fsr);
+                        &phys_addr, &attrs, &prot, &page_size, &fsr, &fi);
     if (extended_addresses_enabled(env)) {
         /* fsr is a DFSR/IFSR value for the long descriptor
          * translation table format, but with WnR always clear.
@@ -1602,6 +2051,26 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
     A32_BANKED_CURRENT_REG_SET(env, par, par64);
 }
 
+static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    int access_type = ri->opc2 & 1;
+    uint64_t par64;
+
+    par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S2NS);
+
+    A32_BANKED_CURRENT_REG_SET(env, par, par64);
+}
+
+static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
+{
+    if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
+        return CP_ACCESS_TRAP;
+    }
+    return CP_ACCESS_OK;
+}
+
 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
@@ -1629,10 +2098,10 @@ static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
         break;
     case 4: /* AT S12E1R, AT S12E1W */
-        mmu_idx = ARMMMUIdx_S12NSE1;
+        mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1;
         break;
     case 6: /* AT S12E0R, AT S12E0W */
-        mmu_idx = ARMMMUIdx_S12NSE0;
+        mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0;
         break;
     default:
         g_assert_not_reached();
@@ -1649,6 +2118,7 @@ static const ARMCPRegInfo vapa_cp_reginfo[] = {
                              offsetoflow32(CPUARMState, cp15.par_ns) },
       .writefn = par_write },
 #ifndef CONFIG_USER_ONLY
+    /* This underdecoding is safe because the reginfo is NO_RAW. */
     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
       .access = PL1_W, .accessfn = ats_access,
       .writefn = ats_write, .type = ARM_CP_NO_RAW },
@@ -1856,7 +2326,7 @@ static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
         }
     }
 
-    /* Update the masks corresponding to the the TCR bank being written
+    /* Update the masks corresponding to the TCR bank being written
      * Note that we always calculate mask and base_mask, but
      * they are only used for short-descriptor tables (ie if EAE is 0);
      * for long-descriptor tables the TCR fields are used differently
@@ -1918,6 +2388,20 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     raw_write(env, ri, value);
 }
 
+static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    /* Accesses to VTTBR may change the VMID so we must flush the TLB.  */
+    if (raw_read(env, ri) != value) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                            ARMMMUIdx_S2NS, -1);
+        raw_write(env, ri, value);
+    }
+}
+
 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_ALIAS,
@@ -2136,7 +2620,19 @@ static const ARMCPRegInfo strongarm_cp_reginfo[] = {
     REGINFO_SENTINEL
 };
 
-static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vpidr_el2;
+    }
+    return raw_read(env, ri);
+}
+
+static uint64_t mpidr_read_val(CPUARMState *env)
 {
     ARMCPU *cpu = ARM_CPU(arm_env_get_cpu(env));
     uint64_t mpidr = cpu->mp_affinity;
@@ -2154,6 +2650,17 @@ static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
     return mpidr;
 }
 
+static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vmpidr_el2;
+    }
+    return mpidr_read_val(env);
+}
+
 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
@@ -2210,7 +2717,8 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     vfp_set_fpsr(env, value);
 }
 
-static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
         return CP_ACCESS_TRAP;
@@ -2225,7 +2733,8 @@ static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
 }
 
 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
-                                          const ARMCPRegInfo *ri)
+                                          const ARMCPRegInfo *ri,
+                                          bool isread)
 {
     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
      * SCTLR_EL1.UCI is set.
@@ -2240,69 +2749,249 @@ static CPAccessResult aa64_cacheop_access(CPUARMState *env,
  * Page D4-1736 (DDI0487A.b)
  */
 
-static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                               uint64_t value)
+static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+    }
+}
+
+static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
+{
+    bool sec = arm_is_secure_below_el3(env);
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                                ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
 {
-    /* Invalidate by VA (AArch64 version) */
     ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E3, -1);
+}
+
+static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    bool sec = arm_is_secure_below_el3(env);
+    bool has_el2 = arm_feature(env, ARM_FEATURE_EL2);
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else if (has_el2) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
+    }
+}
+
+static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E3, -1);
+    }
+}
+
+static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
+{
+    /* Invalidate by VA, EL1&0 (AArch64 version).
+     * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
+     * since we don't support flush-for-specific-ASID-only or
+     * flush-last-level-only.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1SE1,
+                                 ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                 ARMMMUIdx_S12NSE0, -1);
+    }
 }
 
-static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                uint64_t value)
+static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
 {
-    /* Invalidate by VA, all ASIDs (AArch64 version) */
+    /* Invalidate by VA, EL2
+     * Currently handles both VAE2 and VALE2, since we don't support
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
 }
 
-static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
+static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
-    /* Invalidate by ASID (AArch64 version) */
+    /* Invalidate by VA, EL3
+     * Currently handles both VAE3 and VALE3, since we don't support
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int asid = extract64(value, 48, 16);
-    tlb_flush(CPU(cpu), asid == 0);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E3, -1);
 }
 
-static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
+    bool sec = arm_is_secure_below_el3(env);
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        if (sec) {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1SE1,
+                                     ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                     ARMMMUIdx_S12NSE0, -1);
+        }
     }
 }
 
-static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
     }
 }
 
-static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E3, -1);
+    }
+}
+
+static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Invalidate by IPA. This has to invalidate any structures that
+     * contain only stage 2 translation information, but does not need
+     * to apply to structures that contain combined stage 1 and stage 2
+     * translation information.
+     * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
+}
+
+static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
 {
     CPUState *other_cs;
-    int asid = extract64(value, 48, 16);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush(other_cs, asid == 0);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
     }
 }
 
-static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      bool isread)
 {
     /* We don't implement EL2, so the only control on DC ZVA is the
      * bit in the SCTLR which can prohibit access for EL0.
@@ -2319,13 +3008,14 @@ static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
     int dzp_bit = 1 << 4;
 
     /* DZP indicates whether DC ZVA access is allowed */
-    if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
+    if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
         dzp_bit = 0;
     }
     return cpu->dcz_blocksize | dzp_bit;
 }
 
-static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    bool isread)
 {
     if (!(env->pstate & PSTATE_SP)) {
         /* Access to SP_EL0 is undefined if it's being used as
@@ -2364,6 +3054,24 @@ static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
     tlb_flush(CPU(cpu), 1);
 }
 
+static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
+{
+    if ((env->cp15.cptr_el[2] & CPTR_TFP) && arm_current_el(env) == 2) {
+        return CP_ACCESS_TRAP_FP_EL2;
+    }
+    if (env->cp15.cptr_el[3] & CPTR_TFP) {
+        return CP_ACCESS_TRAP_FP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                       uint64_t value)
+{
+    env->cp15.mdcr_el3 = value & SDCR_VALID_MASK;
+}
+
 static const ARMCPRegInfo v8_cp_reginfo[] = {
     /* Minimal set of EL0-visible registers. This will need to be expanded
      * significantly for system emulation of AArch64 CPUs.
@@ -2434,62 +3142,86 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NOP },
     /* TLBI operations */
-    { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
-      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
-      .access = PL2_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_write },
-    { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
-      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
-      .access = PL2_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_is_write },
     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_is_write },
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_asid_is_write },
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbiall_write },
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_asid_write },
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_va_write },
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
       .access = PL1_W, .type = ARM_CP_NO_RAW,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae1_write },
+    { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1_write },
+    { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
 #ifndef CONFIG_USER_ONLY
     /* 64 bit address translation operations */
     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
@@ -2504,6 +3236,31 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
+    { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
+      .access = PL1_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
+      .writefn = par_write },
 #endif
     /* TLB invalidate last level of translation table walk */
     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
@@ -2556,7 +3313,8 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[1]) },
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
     /* We rely on the access checks not allowing the guest to write to the
      * state field when SPSel indicates that it's being used as the stack
      * pointer.
@@ -2574,6 +3332,49 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
       .type = ARM_CP_NO_RAW,
       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
+    { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
+      .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]),
+      .access = PL2_RW, .accessfn = fpexc32_access },
+    { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .resetvalue = 0,
+      .writefn = dacr_write, .raw_writefn = raw_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
+    { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
+    { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
+    { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
+    { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
+    { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
+    { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
+      .resetvalue = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
+    { .name = "SDCR", .type = ARM_CP_ALIAS,
+      .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .writefn = sdcr_write,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
     REGINFO_SENTINEL
 };
 
@@ -2598,9 +3399,36 @@ static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -2613,6 +3441,35 @@ static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
       .resetvalue = 0 },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .accessfn = access_tda,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
@@ -2646,31 +3503,22 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
       .writefn = hcr_write },
-    { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
-      .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
-      .access = PL2_RW, .resetvalue = 0,
-      .writefn = dacr_write, .raw_writefn = raw_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL2_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
-    { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
-      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
-      .access = PL2_RW, .resetvalue = 0,
-      .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .writefn = vbar_write,
@@ -2692,11 +3540,50 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
       .access = PL2_RW, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
       .access = PL2_RW, .writefn = vmsa_tcr_el1_write,
       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
+    { .name = "VTCR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .type = ARM_CP_ALIAS,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW,
+      /* no .writefn needed as this can't cause an ASID change;
+       * no .raw_writefn or .resetfn needed as we never use mask/base_mask
+       */
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
+      .writefn = vttbr_write },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .writefn = vttbr_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
@@ -2715,18 +3602,129 @@ static const ARMCPRegInfo el2_cp_reginfo[] = {
     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbiall_write },
+      .writefn = tlbi_aa64_alle2_write },
     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle2is_write },
     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
       .type = ARM_CP_NO_RAW, .access = PL2_W,
-      .writefn = tlbi_aa64_vaa_write },
+      .writefn = tlbi_aa64_vae2is_write },
+    { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2is_write },
+#ifndef CONFIG_USER_ONLY
+    /* Unlike the other EL2-related AT operations, these must
+     * UNDEF from EL3 if EL2 is not implemented, which is why we
+     * define them here rather than with the rest of the AT ops.
+     */
+    { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
+     * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
+     * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
+     * to behave as if SCR.NS was 1.
+     */
+    { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
+       * reset values as IMPDEF. We choose to reset to 3 to comply with
+       * both ARMv7 and ARMv8.
+       */
+      .access = PL2_RW, .resetvalue = 3,
+      .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .type = ARM_CP_IO, .access = PL2_RW,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
+      .resetfn = gt_hyp_timer_reset,
+      .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .type = ARM_CP_IO,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
+      .resetvalue = 0,
+      .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
+#endif
+    /* The only field of MDCR_EL2 that has a defined architectural reset value
+     * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
+     * don't impelment any PMU event counters, so using zero as a reset
+     * value for MDCR_EL2 is okay
+     */
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
+    { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
     REGINFO_SENTINEL
 };
 
+static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
+{
+    /* The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
+     * At Secure EL1 it traps to EL3.
+     */
+    if (arm_current_el(env) == 3) {
+        return CP_ACCESS_OK;
+    }
+    if (arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
+    if (isread) {
+        return CP_ACCESS_OK;
+    }
+    return CP_ACCESS_TRAP_UNCATEGORIZED;
+}
+
 static const ARMCPRegInfo el3_cp_reginfo[] = {
     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
@@ -2734,7 +3732,8 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .resetvalue = 0, .writefn = scr_write },
     { .name = "SCR",  .type = ARM_CP_ALIAS,
       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
-      .access = PL3_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
       .writefn = scr_write },
     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
@@ -2744,18 +3743,10 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
       .access = PL3_RW, .resetvalue = 0,
       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
-      /* TODO: Implement NSACR trapping of secure EL1 accesses to EL3 */
-    { .name = "NSACR", .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
-      .access = PL3_W | PL1_R, .resetvalue = 0,
-      .fieldoffset = offsetof(CPUARMState, cp15.nsacr) },
     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
-      .access = PL3_RW, .writefn = vbar_write, .resetvalue = 0,
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .writefn = vbar_write, .resetvalue = 0,
       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
-    { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_ALIAS, /* reset handled by AArch32 view */
-      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
-      .access = PL3_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]) },
     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
@@ -2771,7 +3762,6 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .access = PL3_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
@@ -2780,7 +3770,8 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL3_RW, .writefn = vbar_write,
@@ -2790,10 +3781,51 @@ static const ARMCPRegInfo el3_cp_reginfo[] = {
       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
+    { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
+      .access = PL3_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
+    { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3is_write },
+    { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3_write },
+    { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
+    { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
     REGINFO_SENTINEL
 };
 
-static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
 {
     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
      * but the AArch32 CTR has its own reginfo struct)
@@ -2804,6 +3836,23 @@ static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
     return CP_ACCESS_OK;
 }
 
+static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    /* Writes to OSLAR_EL1 may update the OS lock status, which can be
+     * read via a bit in OSLSR_EL1.
+     */
+    int oslock;
+
+    if (ri->state == ARM_CP_STATE_AA32) {
+        oslock = (value == 0xC5ACCE55);
+    } else {
+        oslock = value & 1;
+    }
+
+    env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
+}
+
 static const ARMCPRegInfo debug_cp_reginfo[] = {
     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
@@ -2812,16 +3861,19 @@ static const ARMCPRegInfo debug_cp_reginfo[] = {
      * accessor.
      */
     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL0_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
-      .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL0_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
-      .access = PL1_RW,
+      .access = PL1_RW, .accessfn = access_tda,
       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
       .resetvalue = 0 },
     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
@@ -2830,22 +3882,30 @@ static const ARMCPRegInfo debug_cp_reginfo[] = {
     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
       .type = ARM_CP_ALIAS,
-      .access = PL1_R,
+      .access = PL1_R, .accessfn = access_tda,
       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
-    /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
-      .access = PL1_W, .type = ARM_CP_NOP },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .accessfn = access_tdosa,
+      .writefn = oslar_write },
+    { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
+      .access = PL1_R, .resetvalue = 10,
+      .accessfn = access_tdosa,
+      .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
-      .access = PL1_RW, .type = ARM_CP_NOP },
+      .access = PL1_RW, .accessfn = access_tdosa,
+      .type = ARM_CP_NOP },
     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
      * implement vector catch debug events yet.
      */
     { .name = "DBGVCR",
       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
-      .access = PL1_RW, .type = ARM_CP_NOP },
+      .access = PL1_RW, .accessfn = access_tda,
+      .type = ARM_CP_NOP },
     REGINFO_SENTINEL
 };
 
@@ -3110,7 +4170,8 @@ static void define_debug_regs(ARMCPU *cpu)
     int wrps, brps, ctx_cmps;
     ARMCPRegInfo dbgdidr = {
         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
-        .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
+        .access = PL0_R, .accessfn = access_tda,
+        .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
     };
 
     /* Note that all these register fields hold "number of Xs minus 1". */
@@ -3141,13 +4202,13 @@ static void define_debug_regs(ARMCPU *cpu)
         ARMCPRegInfo dbgregs[] = {
             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
               .writefn = dbgbvr_write, .raw_writefn = raw_write
             },
             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
               .writefn = dbgbcr_write, .raw_writefn = raw_write
             },
@@ -3160,13 +4221,13 @@ static void define_debug_regs(ARMCPU *cpu)
         ARMCPRegInfo dbgregs[] = {
             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
               .writefn = dbgwvr_write, .raw_writefn = raw_write
             },
             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
               .writefn = dbgwcr_write, .raw_writefn = raw_write
             },
@@ -3252,12 +4313,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_isar5 },
-            /* 6..7 are as yet unallocated and must RAZ */
-            { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
-              .resetvalue = 0 },
-            { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
+            { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = cpu->id_mmfr4 },
+            /* 7 is as yet unallocated and must RAZ */
+            { .name = "ID_ISAR7_RESERVED", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = 0 },
             REGINFO_SENTINEL
         };
@@ -3311,7 +4374,11 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_V8)) {
-        /* AArch64 ID registers, which all have impdef reset values */
+        /* AArch64 ID registers, which all have impdef reset values.
+         * Note that within the ID register ranges the unused slots
+         * must all RAZ, not UNDEF; future architecture versions may
+         * define new registers here.
+         */
         ARMCPRegInfo v8_idregs[] = {
             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
@@ -3321,6 +4388,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64pfr1},
+            { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3334,6 +4425,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64dfr1 },
+            { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3342,6 +4441,14 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64afr1 },
+            { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3350,6 +4457,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64isar1 },
+            { .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3358,6 +4489,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64mmfr1 },
+            { .name = "ID_AA64MMFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -3370,6 +4525,42 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->mvfr2 },
+            { .name = "MVFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid0 },
+            { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid0 },
+            { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid1 },
+            { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid1 },
             REGINFO_SENTINEL
         };
         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
@@ -3386,6 +4577,30 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         define_arm_cp_regs(cpu, v8_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_EL2)) {
+        uint64_t vmpidr_def = mpidr_read_val(env);
+        ARMCPRegInfo vpidr_regs[] = {
+            { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            REGINFO_SENTINEL
+        };
+        define_arm_cp_regs(cpu, vpidr_regs);
         define_arm_cp_regs(cpu, el2_cp_reginfo);
         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
         if (!arm_feature(env, ARM_FEATURE_EL3)) {
@@ -3401,18 +4616,82 @@ void register_cp_regs_for_features(ARMCPU *cpu)
          * register the no_el2 reginfos.
          */
         if (arm_feature(env, ARM_FEATURE_EL3)) {
+            /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
+             * of MIDR_EL1 and MPIDR_EL1.
+             */
+            ARMCPRegInfo vpidr_regs[] = {
+                { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_CONST, .resetvalue = cpu->midr,
+                  .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+                { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_NO_RAW,
+                  .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
+                REGINFO_SENTINEL
+            };
+            define_arm_cp_regs(cpu, vpidr_regs);
             define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
         }
     }
     if (arm_feature(env, ARM_FEATURE_EL3)) {
         define_arm_cp_regs(cpu, el3_cp_reginfo);
-        ARMCPRegInfo rvbar = {
-            .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
-            .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
-            .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar
+        ARMCPRegInfo el3_regs[] = {
+            { .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
+              .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar },
+            { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
+              .access = PL3_RW,
+              .raw_writefn = raw_write, .writefn = sctlr_write,
+              .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]),
+              .resetvalue = cpu->reset_sctlr },
+            REGINFO_SENTINEL
         };
-        define_one_arm_cp_reg(cpu, &rvbar);
+
+        define_arm_cp_regs(cpu, el3_regs);
     }
+    /* The behaviour of NSACR is sufficiently various that we don't
+     * try to describe it in a single reginfo:
+     *  if EL3 is 64 bit, then trap to EL3 from S EL1,
+     *     reads as constant 0xc00 from NS EL1 and NS EL2
+     *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
+     *  if v7 without EL3, register doesn't exist
+     *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
+     */
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR", .type = ARM_CP_CONST,
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL1_RW, .accessfn = nsacr_access,
+                .resetvalue = 0xc00
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        } else {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR",
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL3_RW | PL1_R,
+                .resetvalue = 0,
+                .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        }
+    } else {
+        if (arm_feature(env, ARM_FEATURE_V8)) {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR", .type = ARM_CP_CONST,
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL1_R,
+                .resetvalue = 0xc00
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        }
+    }
+
     if (arm_feature(env, ARM_FEATURE_MPU)) {
         if (arm_feature(env, ARM_FEATURE_V6)) {
             /* PMSAv6 not implemented */
@@ -3478,6 +4757,7 @@ void register_cp_regs_for_features(ARMCPU *cpu)
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
               .access = PL1_R, .resetvalue = cpu->midr,
               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
+              .readfn = midr_read,
               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
               .type = ARM_CP_OVERRIDE },
             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
@@ -3501,7 +4781,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
-              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+              .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
+              .readfn = midr_read },
             /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
@@ -3584,13 +4866,22 @@ void register_cp_regs_for_features(ARMCPU *cpu)
     }
 
     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
-        ARMCPRegInfo auxcr = {
-            .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
-            .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
-            .access = PL1_RW, .type = ARM_CP_CONST,
-            .resetvalue = cpu->reset_auxcr
+        ARMCPRegInfo auxcr_reginfo[] = {
+            { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL1_RW, .type = ARM_CP_CONST,
+              .resetvalue = cpu->reset_auxcr },
+            { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL2_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL3_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            REGINFO_SENTINEL
         };
-        define_one_arm_cp_reg(cpu, &auxcr);
+        define_arm_cp_regs(cpu, auxcr_reginfo);
     }
 
     if (arm_feature(env, ARM_FEATURE_CBAR)) {
@@ -4064,23 +5355,47 @@ void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
     /* Helper coprocessor reset function for do-nothing-on-reset registers */
 }
 
-static int bad_mode_switch(CPUARMState *env, int mode)
+static int bad_mode_switch(CPUARMState *env, int mode, CPSRWriteType write_type)
 {
     /* Return true if it is not valid for us to switch to
      * this CPU mode (ie all the UNPREDICTABLE cases in
      * the ARM ARM CPSRWriteByInstr pseudocode).
      */
+
+    /* Changes to or from Hyp via MSR and CPS are illegal. */
+    if (write_type == CPSRWriteByInstr &&
+        ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_HYP ||
+         mode == ARM_CPU_MODE_HYP)) {
+        return 1;
+    }
+
     switch (mode) {
     case ARM_CPU_MODE_USR:
+        return 0;
     case ARM_CPU_MODE_SYS:
     case ARM_CPU_MODE_SVC:
     case ARM_CPU_MODE_ABT:
     case ARM_CPU_MODE_UND:
     case ARM_CPU_MODE_IRQ:
     case ARM_CPU_MODE_FIQ:
+        /* Note that we don't implement the IMPDEF NSACR.RFR which in v7
+         * allows FIQ mode to be Secure-only. (In v8 this doesn't exist.)
+         */
+        /* If HCR.TGE is set then changes from Monitor to NS PL1 via MSR
+         * and CPS are treated as illegal mode changes.
+         */
+        if (write_type == CPSRWriteByInstr &&
+            (env->cp15.hcr_el2 & HCR_TGE) &&
+            (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON &&
+            !arm_is_secure_below_el3(env)) {
+            return 1;
+        }
         return 0;
+    case ARM_CPU_MODE_HYP:
+        return !arm_feature(env, ARM_FEATURE_EL2)
+            || arm_current_el(env) < 2 || arm_is_secure(env);
     case ARM_CPU_MODE_MON:
-        return !arm_is_secure(env);
+        return arm_current_el(env) < 3;
     default:
         return 1;
     }
@@ -4097,7 +5412,8 @@ uint32_t cpsr_read(CPUARMState *env)
         | (env->GE << 16) | (env->daif & CPSR_AIF);
 }
 
-void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
+void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
+                CPSRWriteType write_type)
 {
     uint32_t changed_daif;
 
@@ -4131,7 +5447,7 @@ void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
      * In a V8 implementation, it is permitted for privileged software to
      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
      */
-    if (!arm_feature(env, ARM_FEATURE_V8) &&
+    if (write_type != CPSRWriteRaw && !arm_feature(env, ARM_FEATURE_V8) &&
         arm_feature(env, ARM_FEATURE_EL3) &&
         !arm_feature(env, ARM_FEATURE_EL2) &&
         !arm_is_secure(env)) {
@@ -4178,13 +5494,31 @@ void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
     env->daif &= ~(CPSR_AIF & mask);
     env->daif |= val & CPSR_AIF & mask;
 
-    if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
-        if (bad_mode_switch(env, val & CPSR_M)) {
-            /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
-             * We choose to ignore the attempt and leave the CPSR M field
-             * untouched.
+    if (write_type != CPSRWriteRaw &&
+        ((env->uncached_cpsr ^ val) & mask & CPSR_M)) {
+        if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR) {
+            /* Note that we can only get here in USR mode if this is a
+             * gdb stub write; for this case we follow the architectural
+             * behaviour for guest writes in USR mode of ignoring an attempt
+             * to switch mode. (Those are caught by translate.c for writes
+             * triggered by guest instructions.)
              */
             mask &= ~CPSR_M;
+        } else if (bad_mode_switch(env, val & CPSR_M, write_type)) {
+            /* Attempt to switch to an invalid mode: this is UNPREDICTABLE in
+             * v7, and has defined behaviour in v8:
+             *  + leave CPSR.M untouched
+             *  + allow changes to the other CPSR fields
+             *  + set PSTATE.IL
+             * For user changes via the GDB stub, we don't set PSTATE.IL,
+             * as this would be unnecessarily harsh for a user error.
+             */
+            mask &= ~CPSR_M;
+            if (write_type != CPSRWriteByGDBStub &&
+                arm_feature(env, ARM_FEATURE_V8)) {
+                mask |= CPSR_IL;
+                val |= CPSR_IL;
+            }
         } else {
             switch_mode(env, val & CPSR_M);
         }
@@ -4233,17 +5567,7 @@ uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
 
 uint32_t HELPER(rbit)(uint32_t x)
 {
-    x =  ((x & 0xff000000) >> 24)
-       | ((x & 0x00ff0000) >> 8)
-       | ((x & 0x0000ff00) << 8)
-       | ((x & 0x000000ff) << 24);
-    x =  ((x & 0xf0f0f0f0) >> 4)
-       | ((x & 0x0f0f0f0f) << 4);
-    x =  ((x & 0x88888888) >> 3)
-       | ((x & 0x44444444) >> 1)
-       | ((x & 0x22222222) << 1)
-       | ((x & 0x11111111) << 3);
-    return x;
+    return revbit32(x);
 }
 
 #if defined(CONFIG_USER_ONLY)
@@ -4273,21 +5597,6 @@ void switch_mode(CPUARMState *env, int mode)
     }
 }
 
-void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
-{
-    ARMCPU *cpu = arm_env_get_cpu(env);
-
-    cpu_abort(CPU(cpu), "banked r13 write\n");
-}
-
-uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
-{
-    ARMCPU *cpu = arm_env_get_cpu(env);
-
-    cpu_abort(CPU(cpu), "banked r13 read\n");
-    return 0;
-}
-
 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
                                  uint32_t cur_el, bool secure)
 {
@@ -4301,31 +5610,6 @@ void aarch64_sync_64_to_32(CPUARMState *env)
 
 #else
 
-/* Map CPU modes onto saved register banks.  */
-int bank_number(int mode)
-{
-    switch (mode) {
-    case ARM_CPU_MODE_USR:
-    case ARM_CPU_MODE_SYS:
-        return 0;
-    case ARM_CPU_MODE_SVC:
-        return 1;
-    case ARM_CPU_MODE_ABT:
-        return 2;
-    case ARM_CPU_MODE_UND:
-        return 3;
-    case ARM_CPU_MODE_IRQ:
-        return 4;
-    case ARM_CPU_MODE_FIQ:
-        return 5;
-    case ARM_CPU_MODE_HYP:
-        return 6;
-    case ARM_CPU_MODE_MON:
-        return 7;
-    }
-    hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
-}
-
 void switch_mode(CPUARMState *env, int mode)
 {
     int old_mode;
@@ -4391,7 +5675,7 @@ void switch_mode(CPUARMState *env, int mode)
  *        BIT IRQ     IMO      Non-secure         Secure
  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
  */
-const int8_t target_el_table[2][2][2][2][2][4] = {
+static const int8_t target_el_table[2][2][2][2][2][4] = {
     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
@@ -4417,11 +5701,22 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
                                  uint32_t cur_el, bool secure)
 {
     CPUARMState *env = cs->env_ptr;
-    int rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
+    int rw;
     int scr;
     int hcr;
     int target_el;
-    int is64 = arm_el_is_aa64(env, 3);
+    /* Is the highest EL AArch64? */
+    int is64 = arm_feature(env, ARM_FEATURE_AARCH64);
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
+    } else {
+        /* Either EL2 is the highest EL (and so the EL2 register width
+         * is given by is64); or there is no EL2 or EL3, in which case
+         * the value of 'rw' does not affect the table lookup anyway.
+         */
+        rw = is64;
+    }
 
     switch (excp_idx) {
     case EXCP_IRQ:
@@ -4558,11 +5853,13 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs)
     case EXCP_BKPT:
         if (semihosting_enabled()) {
             int nr;
-            nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
+            nr = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env)) & 0xff;
             if (nr == 0xab) {
                 env->regs[15] += 2;
+                qemu_log_mask(CPU_LOG_INT,
+                              "...handling as semihosting call 0x%x\n",
+                              env->regs[0]);
                 env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
                 return;
             }
         }
@@ -4655,35 +5952,35 @@ void aarch64_sync_32_to_64(CPUARMState *env)
     }
 
     if (mode == ARM_CPU_MODE_IRQ) {
-        env->xregs[16] = env->regs[13];
-        env->xregs[17] = env->regs[14];
+        env->xregs[16] = env->regs[14];
+        env->xregs[17] = env->regs[13];
     } else {
-        env->xregs[16] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
-        env->xregs[17] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
+        env->xregs[16] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
+        env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
     }
 
     if (mode == ARM_CPU_MODE_SVC) {
-        env->xregs[18] = env->regs[13];
-        env->xregs[19] = env->regs[14];
+        env->xregs[18] = env->regs[14];
+        env->xregs[19] = env->regs[13];
     } else {
-        env->xregs[18] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
-        env->xregs[19] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
+        env->xregs[18] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
+        env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
     }
 
     if (mode == ARM_CPU_MODE_ABT) {
-        env->xregs[20] = env->regs[13];
-        env->xregs[21] = env->regs[14];
+        env->xregs[20] = env->regs[14];
+        env->xregs[21] = env->regs[13];
     } else {
-        env->xregs[20] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
-        env->xregs[21] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
+        env->xregs[20] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
+        env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
     }
 
     if (mode == ARM_CPU_MODE_UND) {
-        env->xregs[22] = env->regs[13];
-        env->xregs[23] = env->regs[14];
+        env->xregs[22] = env->regs[14];
+        env->xregs[23] = env->regs[13];
     } else {
-        env->xregs[22] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
-        env->xregs[23] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
+        env->xregs[22] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
+        env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
     }
 
     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
@@ -4760,35 +6057,35 @@ void aarch64_sync_64_to_32(CPUARMState *env)
     }
 
     if (mode == ARM_CPU_MODE_IRQ) {
-        env->regs[13] = env->xregs[16];
-        env->regs[14] = env->xregs[17];
+        env->regs[14] = env->xregs[16];
+        env->regs[13] = env->xregs[17];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
-        env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
+        env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
+        env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
     }
 
     if (mode == ARM_CPU_MODE_SVC) {
-        env->regs[13] = env->xregs[18];
-        env->regs[14] = env->xregs[19];
+        env->regs[14] = env->xregs[18];
+        env->regs[13] = env->xregs[19];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
-        env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
+        env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
+        env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
     }
 
     if (mode == ARM_CPU_MODE_ABT) {
-        env->regs[13] = env->xregs[20];
-        env->regs[14] = env->xregs[21];
+        env->regs[14] = env->xregs[20];
+        env->regs[13] = env->xregs[21];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
-        env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
+        env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
+        env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
     }
 
     if (mode == ARM_CPU_MODE_UND) {
-        env->regs[13] = env->xregs[22];
-        env->regs[14] = env->xregs[23];
+        env->regs[14] = env->xregs[22];
+        env->regs[13] = env->xregs[23];
     } else {
-        env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
-        env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
+        env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
+        env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
     }
 
     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
@@ -4810,8 +6107,7 @@ void aarch64_sync_64_to_32(CPUARMState *env)
     env->regs[15] = env->pc;
 }
 
-/* Handle a CPU exception.  */
-void arm_cpu_do_interrupt(CPUState *cs)
+static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
@@ -4821,16 +6117,6 @@ void arm_cpu_do_interrupt(CPUState *cs)
     uint32_t offset;
     uint32_t moe;
 
-    assert(!IS_M(env));
-
-    arm_log_exception(cs->exception_index);
-
-    if (arm_is_psci_call(cpu, cs->exception_index)) {
-        arm_handle_psci_call(cpu);
-        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
-        return;
-    }
-
     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
     case EC_BREAKPOINT:
@@ -4868,25 +6154,6 @@ void arm_cpu_do_interrupt(CPUState *cs)
             offset = 4;
         break;
     case EXCP_SWI:
-        if (semihosting_enabled()) {
-            /* Check for semihosting interrupt.  */
-            if (env->thumb) {
-                mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
-                    & 0xff;
-            } else {
-                mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
-                    & 0xffffff;
-            }
-            /* Only intercept calls from privileged modes, to provide some
-               semblance of security.  */
-            if (((mask == 0x123456 && !env->thumb)
-                    || (mask == 0xab && env->thumb))
-                  && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
-                env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
-                return;
-            }
-        }
         new_mode = ARM_CPU_MODE_SVC;
         addr = 0x08;
         mask = CPSR_I;
@@ -4894,17 +6161,6 @@ void arm_cpu_do_interrupt(CPUState *cs)
         offset = 0;
         break;
     case EXCP_BKPT:
-        /* See if this is a semihosting syscall.  */
-        if (env->thumb && semihosting_enabled()) {
-            mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
-            if (mask == 0xab
-                  && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
-                env->regs[15] += 2;
-                env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
-                return;
-            }
-        }
         env->exception.fsr = 2;
         /* Fall through to prefetch abort.  */
     case EXCP_PREFETCH_ABORT:
@@ -4990,6 +6246,11 @@ void arm_cpu_do_interrupt(CPUState *cs)
     env->condexec_bits = 0;
     /* Switch to the new mode, and to the correct instruction set.  */
     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
+    /* Set new mode endianness */
+    env->uncached_cpsr &= ~CPSR_E;
+    if (env->cp15.sctlr_el[arm_current_el(env)] & SCTLR_EE) {
+        env->uncached_cpsr |= ~CPSR_E;
+    }
     env->daif |= mask;
     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
      * and we should just guard the thumb mode on V4 */
@@ -4998,9 +6259,227 @@ void arm_cpu_do_interrupt(CPUState *cs)
     }
     env->regs[14] = env->regs[15] + offset;
     env->regs[15] = addr;
-    cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
 }
 
+/* Handle exception entry to a target EL which is using AArch64 */
+static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    unsigned int new_el = env->exception.target_el;
+    target_ulong addr = env->cp15.vbar_el[new_el];
+    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
+
+    if (arm_current_el(env) < new_el) {
+        /* Entry vector offset depends on whether the implemented EL
+         * immediately lower than the target level is using AArch32 or AArch64
+         */
+        bool is_aa64;
+
+        switch (new_el) {
+        case 3:
+            is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
+            break;
+        case 2:
+            is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0;
+            break;
+        case 1:
+            is_aa64 = is_a64(env);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        if (is_aa64) {
+            addr += 0x400;
+        } else {
+            addr += 0x600;
+        }
+    } else if (pstate_read(env) & PSTATE_SP) {
+        addr += 0x200;
+    }
+
+    switch (cs->exception_index) {
+    case EXCP_PREFETCH_ABORT:
+    case EXCP_DATA_ABORT:
+        env->cp15.far_el[new_el] = env->exception.vaddress;
+        qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
+                      env->cp15.far_el[new_el]);
+        /* fall through */
+    case EXCP_BKPT:
+    case EXCP_UDEF:
+    case EXCP_SWI:
+    case EXCP_HVC:
+    case EXCP_HYP_TRAP:
+    case EXCP_SMC:
+        env->cp15.esr_el[new_el] = env->exception.syndrome;
+        break;
+    case EXCP_IRQ:
+    case EXCP_VIRQ:
+        addr += 0x80;
+        break;
+    case EXCP_FIQ:
+    case EXCP_VFIQ:
+        addr += 0x100;
+        break;
+    case EXCP_SEMIHOST:
+        qemu_log_mask(CPU_LOG_INT,
+                      "...handling as semihosting call 0x%" PRIx64 "\n",
+                      env->xregs[0]);
+        env->xregs[0] = do_arm_semihosting(env);
+        return;
+    default:
+        cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
+    }
+
+    if (is_a64(env)) {
+        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
+        aarch64_save_sp(env, arm_current_el(env));
+        env->elr_el[new_el] = env->pc;
+    } else {
+        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = cpsr_read(env);
+        if (!env->thumb) {
+            env->cp15.esr_el[new_el] |= 1 << 25;
+        }
+        env->elr_el[new_el] = env->regs[15];
+
+        aarch64_sync_32_to_64(env);
+
+        env->condexec_bits = 0;
+    }
+    qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
+                  env->elr_el[new_el]);
+
+    pstate_write(env, PSTATE_DAIF | new_mode);
+    env->aarch64 = 1;
+    aarch64_restore_sp(env, new_el);
+
+    env->pc = addr;
+
+    qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
+                  new_el, env->pc, pstate_read(env));
+}
+
+static inline bool check_for_semihosting(CPUState *cs)
+{
+    /* Check whether this exception is a semihosting call; if so
+     * then handle it and return true; otherwise return false.
+     */
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    if (is_a64(env)) {
+        if (cs->exception_index == EXCP_SEMIHOST) {
+            /* This is always the 64-bit semihosting exception.
+             * The "is this usermode" and "is semihosting enabled"
+             * checks have been done at translate time.
+             */
+            qemu_log_mask(CPU_LOG_INT,
+                          "...handling as semihosting call 0x%" PRIx64 "\n",
+                          env->xregs[0]);
+            env->xregs[0] = do_arm_semihosting(env);
+            return true;
+        }
+        return false;
+    } else {
+        uint32_t imm;
+
+        /* Only intercept calls from privileged modes, to provide some
+         * semblance of security.
+         */
+        if (!semihosting_enabled() ||
+            ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)) {
+            return false;
+        }
+
+        switch (cs->exception_index) {
+        case EXCP_SWI:
+            /* Check for semihosting interrupt.  */
+            if (env->thumb) {
+                imm = arm_lduw_code(env, env->regs[15] - 2, arm_sctlr_b(env))
+                    & 0xff;
+                if (imm == 0xab) {
+                    break;
+                }
+            } else {
+                imm = arm_ldl_code(env, env->regs[15] - 4, arm_sctlr_b(env))
+                    & 0xffffff;
+                if (imm == 0x123456) {
+                    break;
+                }
+            }
+            return false;
+        case EXCP_BKPT:
+            /* See if this is a semihosting syscall.  */
+            if (env->thumb) {
+                imm = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env))
+                    & 0xff;
+                if (imm == 0xab) {
+                    env->regs[15] += 2;
+                    break;
+                }
+            }
+            return false;
+        default:
+            return false;
+        }
+
+        qemu_log_mask(CPU_LOG_INT,
+                      "...handling as semihosting call 0x%x\n",
+                      env->regs[0]);
+        env->regs[0] = do_arm_semihosting(env);
+        return true;
+    }
+}
+
+/* Handle a CPU exception for A and R profile CPUs.
+ * Do any appropriate logging, handle PSCI calls, and then hand off
+ * to the AArch64-entry or AArch32-entry function depending on the
+ * target exception level's register width.
+ */
+void arm_cpu_do_interrupt(CPUState *cs)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    unsigned int new_el = env->exception.target_el;
+
+    assert(!IS_M(env));
+
+    arm_log_exception(cs->exception_index);
+    qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
+                  new_el);
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && !excp_is_internal(cs->exception_index)) {
+        qemu_log_mask(CPU_LOG_INT, "...with ESR %x/0x%" PRIx32 "\n",
+                      env->exception.syndrome >> ARM_EL_EC_SHIFT,
+                      env->exception.syndrome);
+    }
+
+    if (arm_is_psci_call(cpu, cs->exception_index)) {
+        arm_handle_psci_call(cpu);
+        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
+        return;
+    }
+
+    /* Semihosting semantics depend on the register width of the
+     * code that caused the exception, not the target exception level,
+     * so must be handled here.
+     */
+    if (check_for_semihosting(cs)) {
+        return;
+    }
+
+    assert(!excp_is_internal(cs->exception_index));
+    if (arm_el_is_aa64(env, new_el)) {
+        arm_cpu_do_interrupt_aarch64(cs);
+    } else {
+        arm_cpu_do_interrupt_aarch32(cs);
+    }
+
+    if (!kvm_enabled()) {
+        cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
+    }
+}
 
 /* Return the exception level which controls this address translation regime */
 static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
@@ -5058,12 +6537,17 @@ static inline bool regime_translation_disabled(CPUARMState *env,
     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
 }
 
+static inline bool regime_translation_big_endian(CPUARMState *env,
+                                                 ARMMMUIdx mmu_idx)
+{
+    return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
+}
+
 /* Return the TCR controlling this translation regime */
 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
 {
     if (mmu_idx == ARMMMUIdx_S2NS) {
-        /* TODO: return VTCR_EL2 */
-        g_assert_not_reached();
+        return &env->cp15.vtcr_el2;
     }
     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
 }
@@ -5073,8 +6557,7 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
                                    int ttbrn)
 {
     if (mmu_idx == ARMMMUIdx_S2NS) {
-        /* TODO: return VTTBR_EL2 */
-        g_assert_not_reached();
+        return env->cp15.vttbr_el2;
     }
     if (ttbrn == 0) {
         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
@@ -5098,6 +6581,18 @@ static inline bool regime_using_lpae_format(CPUARMState *env,
     return false;
 }
 
+/* Returns true if the stage 1 translation regime is using LPAE format page
+ * tables. Used when raising alignment exceptions, whose FSR changes depending
+ * on whether the long or short descriptor format is in use. */
+bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
+        mmu_idx += ARMMMUIdx_S1NSE0;
+    }
+
+    return regime_using_lpae_format(env, mmu_idx);
+}
+
 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
 {
     switch (mmu_idx) {
@@ -5196,6 +6691,28 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
 }
 
+/* Translate S2 section/page access permissions to protection flags
+ *
+ * @env:     CPUARMState
+ * @s2ap:    The 2-bit stage2 access permissions (S2AP)
+ * @xn:      XN (execute-never) bit
+ */
+static int get_S2prot(CPUARMState *env, int s2ap, int xn)
+{
+    int prot = 0;
+
+    if (s2ap & 1) {
+        prot |= PAGE_READ;
+    }
+    if (s2ap & 2) {
+        prot |= PAGE_WRITE;
+    }
+    if (!xn) {
+        prot |= PAGE_EXEC;
+    }
+    return prot;
+}
+
 /* Translate section/page access permissions to protection flags
  *
  * @env:     CPUARMState
@@ -5300,6 +6817,32 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
     return true;
 }
 
+/* Translate a S1 pagetable walk through S2 if needed.  */
+static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
+                               hwaddr addr, MemTxAttrs txattrs,
+                               uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
+{
+    if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) &&
+        !regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+        target_ulong s2size;
+        hwaddr s2pa;
+        int s2prot;
+        int ret;
+
+        ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa,
+                                 &txattrs, &s2prot, &s2size, fsr, fi);
+        if (ret) {
+            fi->s2addr = addr;
+            fi->stage2 = true;
+            fi->s1ptw = true;
+            return ~0;
+        }
+        addr = s2pa;
+    }
+    return addr;
+}
+
 /* All loads done in the course of a page table walk go through here.
  * TODO: rather than ignoring errors from physical memory reads (which
  * are external aborts in ARM terminology) we should propagate this
@@ -5307,26 +6850,55 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
  * was being done for a CPU load/store or an address translation instruction
  * (but not if it was for a debug access).
  */
-static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure)
+static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     MemTxAttrs attrs = {};
+    AddressSpace *as;
 
     attrs.secure = is_secure;
-    return address_space_ldl(cs->as, addr, attrs, NULL);
+    as = arm_addressspace(cs, attrs);
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
+    if (regime_translation_big_endian(env, mmu_idx)) {
+        return address_space_ldl_be(as, addr, attrs, NULL);
+    } else {
+        return address_space_ldl_le(as, addr, attrs, NULL);
+    }
 }
 
-static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure)
+static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
 {
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     MemTxAttrs attrs = {};
+    AddressSpace *as;
 
     attrs.secure = is_secure;
-    return address_space_ldq(cs->as, addr, attrs, NULL);
+    as = arm_addressspace(cs, attrs);
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
+    if (regime_translation_big_endian(env, mmu_idx)) {
+        return address_space_ldq_be(as, addr, attrs, NULL);
+    } else {
+        return address_space_ldq_le(as, addr, attrs, NULL);
+    }
 }
 
 static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
                              int access_type, ARMMMUIdx mmu_idx,
                              hwaddr *phys_ptr, int *prot,
-                             target_ulong *page_size, uint32_t *fsr)
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -5346,7 +6918,8 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
         code = 5;
         goto do_fault;
     }
-    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     domain = (desc >> 5) & 0x0f;
     if (regime_el(env, mmu_idx) == 1) {
@@ -5382,7 +6955,8 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
             /* Fine pagetable.  */
             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
         }
-        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
             code = 7;
@@ -5439,7 +7013,8 @@ do_fault:
 static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
                              int access_type, ARMMMUIdx mmu_idx,
                              hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                             target_ulong *page_size, uint32_t *fsr)
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -5462,7 +7037,8 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
         code = 5;
         goto do_fault;
     }
-    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
         /* Section translation fault, or attempt to use the encoding
@@ -5513,7 +7089,8 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
         ns = extract32(desc, 3, 1);
         /* Lookup l2 entry.  */
         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
-        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
@@ -5587,17 +7164,87 @@ typedef enum {
     permission_fault = 3,
 } MMUFaultType;
 
+/*
+ * check_s2_mmu_setup
+ * @cpu:        ARMCPU
+ * @is_aa64:    True if the translation regime is in AArch64 state
+ * @startlevel: Suggested starting level
+ * @inputsize:  Bitsize of IPAs
+ * @stride:     Page-table stride (See the ARM ARM)
+ *
+ * Returns true if the suggested S2 translation parameters are OK and
+ * false otherwise.
+ */
+static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level,
+                               int inputsize, int stride)
+{
+    const int grainsize = stride + 3;
+    int startsizecheck;
+
+    /* Negative levels are never allowed.  */
+    if (level < 0) {
+        return false;
+    }
+
+    startsizecheck = inputsize - ((3 - level) * stride + grainsize);
+    if (startsizecheck < 1 || startsizecheck > stride + 4) {
+        return false;
+    }
+
+    if (is_aa64) {
+        CPUARMState *env = &cpu->env;
+        unsigned int pamax = arm_pamax(cpu);
+
+        switch (stride) {
+        case 13: /* 64KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 42)) {
+                return false;
+            }
+            break;
+        case 11: /* 16KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 40)) {
+                return false;
+            }
+            break;
+        case 9: /* 4KB Pages.  */
+            if (level == 0 && pamax <= 42) {
+                return false;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        /* Inputsize checks.  */
+        if (inputsize > pamax &&
+            (arm_el_is_aa64(env, 1) || inputsize > 40)) {
+            /* This is CONSTRAINED UNPREDICTABLE and we choose to fault.  */
+            return false;
+        }
+    } else {
+        /* AArch32 only supports 4KB pages. Assert on that.  */
+        assert(stride == 9);
+
+        if (level == 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
                                int access_type, ARMMMUIdx mmu_idx,
                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
-                               target_ulong *page_size_ptr, uint32_t *fsr)
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     /* Read an LPAE long-descriptor translation table. */
     MMUFaultType fault_type = translation_fault;
-    uint32_t level = 1;
-    uint32_t epd;
-    int32_t tsz;
+    uint32_t level;
+    uint32_t epd = 0;
+    int32_t t0sz, t1sz;
     uint32_t tg;
     uint64_t ttbr;
     int ttbr_select;
@@ -5605,13 +7252,15 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
     uint32_t tableattrs;
     target_ulong page_size;
     uint32_t attrs;
-    int32_t granule_sz = 9;
-    int32_t va_size = 32;
+    int32_t stride = 9;
+    int32_t va_size;
+    int inputsize;
     int32_t tbi = 0;
     TCR *tcr = regime_tcr(env, mmu_idx);
     int ap, ns, xn, pxn;
     uint32_t el = regime_el(env, mmu_idx);
     bool ttbr1_valid = true;
+    uint64_t descaddrmask;
 
     /* TODO:
      * This code does not handle the different format TCR for VTCR_EL2.
@@ -5620,9 +7269,12 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
      * support for those page table walks.
      */
     if (arm_el_is_aa64(env, el)) {
+        level = 0;
         va_size = 64;
         if (el > 1) {
-            tbi = extract64(tcr->raw_tcr, 20, 1);
+            if (mmu_idx != ARMMMUIdx_S2NS) {
+                tbi = extract64(tcr->raw_tcr, 20, 1);
+            }
         } else {
             if (extract64(address, 55, 1)) {
                 tbi = extract64(tcr->raw_tcr, 38, 1);
@@ -5638,6 +7290,13 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         if (el > 1) {
             ttbr1_valid = false;
         }
+    } else {
+        level = 1;
+        va_size = 32;
+        /* There is no TTBR1 for EL2 */
+        if (el == 2) {
+            ttbr1_valid = false;
+        }
     }
 
     /* Determine whether this address is in the region controlled by
@@ -5645,12 +7304,28 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
      */
-    uint32_t t0sz = extract32(tcr->raw_tcr, 0, 6);
     if (va_size == 64) {
+        /* AArch64 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 6);
         t0sz = MIN(t0sz, 39);
         t0sz = MAX(t0sz, 16);
+    } else if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* AArch32 stage 1 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 3);
+    } else {
+        /* AArch32 stage 2 translation.  */
+        bool sext = extract32(tcr->raw_tcr, 4, 1);
+        bool sign = extract32(tcr->raw_tcr, 3, 1);
+        t0sz = sextract32(tcr->raw_tcr, 0, 4);
+
+        /* If the sign-extend bit is not the same as t0sz[3], the result
+         * is unpredictable. Flag this as a guest error.  */
+        if (sign != sext) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "AArch32: VTCR.S / VTCR.T0SZ[3] missmatch\n");
+        }
     }
-    uint32_t t1sz = extract32(tcr->raw_tcr, 16, 6);
+    t1sz = extract32(tcr->raw_tcr, 16, 6);
     if (va_size == 64) {
         t1sz = MIN(t1sz, 39);
         t1sz = MAX(t1sz, 16);
@@ -5683,15 +7358,17 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
      */
     if (ttbr_select == 0) {
         ttbr = regime_ttbr(env, mmu_idx, 0);
-        epd = extract32(tcr->raw_tcr, 7, 1);
-        tsz = t0sz;
+        if (el < 2) {
+            epd = extract32(tcr->raw_tcr, 7, 1);
+        }
+        inputsize = va_size - t0sz;
 
         tg = extract32(tcr->raw_tcr, 14, 2);
         if (tg == 1) { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 2) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     } else {
         /* We should only be here if TTBR1 is valid */
@@ -5699,19 +7376,19 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
 
         ttbr = regime_ttbr(env, mmu_idx, 1);
         epd = extract32(tcr->raw_tcr, 23, 1);
-        tsz = t1sz;
+        inputsize = va_size - t1sz;
 
         tg = extract32(tcr->raw_tcr, 30, 2);
         if (tg == 3)  { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 1) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     }
 
     /* Here we should have set up all the parameters for the translation:
-     * va_size, ttbr, epd, tsz, granule_sz, tbi
+     * va_size, inputsize, ttbr, epd, stride, tbi
      */
 
     if (epd) {
@@ -5721,32 +7398,67 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         goto do_fault;
     }
 
-    /* The starting level depends on the virtual address size (which can be
-     * up to 48 bits) and the translation granule size. It indicates the number
-     * of strides (granule_sz bits at a time) needed to consume the bits
-     * of the input address. In the pseudocode this is:
-     *  level = 4 - RoundUp((inputsize - grainsize) / stride)
-     * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
-     * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
-     * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
-     *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
-     *     = 4 - (va_size - tsz - 4) / granule_sz;
-     */
-    level = 4 - (va_size - tsz - 4) / granule_sz;
+    if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* The starting level depends on the virtual address size (which can
+         * be up to 48 bits) and the translation granule size. It indicates
+         * the number of strides (stride bits at a time) needed to
+         * consume the bits of the input address. In the pseudocode this is:
+         *  level = 4 - RoundUp((inputsize - grainsize) / stride)
+         * where their 'inputsize' is our 'inputsize', 'grainsize' is
+         * our 'stride + 3' and 'stride' is our 'stride'.
+         * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
+         * = 4 - (inputsize - stride - 3 + stride - 1) / stride
+         * = 4 - (inputsize - 4) / stride;
+         */
+        level = 4 - (inputsize - 4) / stride;
+    } else {
+        /* For stage 2 translations the starting level is specified by the
+         * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
+         */
+        uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2);
+        uint32_t startlevel;
+        bool ok;
+
+        if (va_size == 32 || stride == 9) {
+            /* AArch32 or 4KB pages */
+            startlevel = 2 - sl0;
+        } else {
+            /* 16KB or 64KB pages */
+            startlevel = 3 - sl0;
+        }
+
+        /* Check that the starting level is valid. */
+        ok = check_s2_mmu_setup(cpu, va_size == 64, startlevel,
+                                inputsize, stride);
+        if (!ok) {
+            fault_type = translation_fault;
+            goto do_fault;
+        }
+        level = startlevel;
+    }
 
     /* Clear the vaddr bits which aren't part of the within-region address,
      * so that we don't have to special case things when calculating the
      * first descriptor address.
      */
-    if (tsz) {
-        address &= (1ULL << (va_size - tsz)) - 1;
+    if (va_size != inputsize) {
+        address &= (1ULL << inputsize) - 1;
     }
 
-    descmask = (1ULL << (granule_sz + 3)) - 1;
+    descmask = (1ULL << (stride + 3)) - 1;
 
     /* Now we can extract the actual base address from the TTBR */
     descaddr = extract64(ttbr, 0, 48);
-    descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
+    descaddr &= ~((1ULL << (inputsize - (stride * (4 - level)))) - 1);
+
+    /* The address field in the descriptor goes up to bit 39 for ARMv7
+     * but up to bit 47 for ARMv8.
+     */
+    if (arm_feature(env, ARM_FEATURE_V8)) {
+        descaddrmask = 0xfffffffff000ULL;
+    } else {
+        descaddrmask = 0xfffffff000ULL;
+    }
 
     /* Secure accesses start with the page table in secure memory and
      * can be downgraded to non-secure at any step. Non-secure accesses
@@ -5758,16 +7470,20 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
         uint64_t descriptor;
         bool nstable;
 
-        descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
+        descaddr |= (address >> (stride * (4 - level))) & descmask;
         descaddr &= ~7ULL;
         nstable = extract32(tableattrs, 4, 1);
-        descriptor = arm_ldq_ptw(cs, descaddr, !nstable);
+        descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fsr, fi);
+        if (fi->s1ptw) {
+            goto do_fault;
+        }
+
         if (!(descriptor & 1) ||
             (!(descriptor & 2) && (level == 3))) {
             /* Invalid, or the Reserved level 3 encoding */
             goto do_fault;
         }
-        descaddr = descriptor & 0xfffffff000ULL;
+        descaddr = descriptor & descaddrmask;
 
         if ((descriptor & 2) && (level < 3)) {
             /* Table entry. The top five bits are attributes which  may
@@ -5783,11 +7499,17 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
          * These are basically the same thing, although the number
          * of bits we pull in from the vaddr varies.
          */
-        page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
+        page_size = (1ULL << ((stride * (4 - level)) + 3));
         descaddr |= (address & (page_size - 1));
-        /* Extract attributes from the descriptor and merge with table attrs */
+        /* Extract attributes from the descriptor */
         attrs = extract64(descriptor, 2, 10)
             | (extract64(descriptor, 52, 12) << 10);
+
+        if (mmu_idx == ARMMMUIdx_S2NS) {
+            /* Stage 2 table descriptors do not include any attribute fields */
+            break;
+        }
+        /* Merge in attributes from table descriptors */
         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
@@ -5809,11 +7531,16 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
     }
 
     ap = extract32(attrs, 4, 2);
-    ns = extract32(attrs, 3, 1);
     xn = extract32(attrs, 12, 1);
-    pxn = extract32(attrs, 11, 1);
 
-    *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        ns = true;
+        *prot = get_S2prot(env, ap, xn);
+    } else {
+        ns = extract32(attrs, 3, 1);
+        pxn = extract32(attrs, 11, 1);
+        *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
+    }
 
     fault_type = permission_fault;
     if (!(*prot & (1 << access_type))) {
@@ -5834,6 +7561,8 @@ static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
 do_fault:
     /* Long-descriptor format IFSR/DFSR value */
     *fsr = (1 << 9) | (fault_type << 2) | level;
+    /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
+    fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS);
     return true;
 }
 
@@ -6096,20 +7825,45 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
  * @page_size: set to the size of the page containing phys_ptr
  * @fsr: set to the DFSR/IFSR value on failure
  */
-static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
-                                 int access_type, ARMMMUIdx mmu_idx,
-                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
-                                 target_ulong *page_size, uint32_t *fsr)
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi)
 {
     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
-        /* TODO: when we support EL2 we should here call ourselves recursively
-         * to do the stage 1 and then stage 2 translations. The arm_ld*_ptw
-         * functions will also need changing to perform ARMMMUIdx_S2NS loads
-         * rather than direct physical memory loads when appropriate.
-         * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
+        /* Call ourselves recursively to do the stage 1 and then stage 2
+         * translations.
          */
-        assert(!arm_feature(env, ARM_FEATURE_EL2));
-        mmu_idx += ARMMMUIdx_S1NSE0;
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            hwaddr ipa;
+            int s2_prot;
+            int ret;
+
+            ret = get_phys_addr(env, address, access_type,
+                                mmu_idx + ARMMMUIdx_S1NSE0, &ipa, attrs,
+                                prot, page_size, fsr, fi);
+
+            /* If S1 fails or S2 is disabled, return early.  */
+            if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+                *phys_ptr = ipa;
+                return ret;
+            }
+
+            /* S1 is done. Now do S2 translation.  */
+            ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS,
+                                     phys_ptr, attrs, &s2_prot,
+                                     page_size, fsr, fi);
+            fi->s2addr = ipa;
+            /* Combine the S1 and S2 perms.  */
+            *prot &= s2_prot;
+            return ret;
+        } else {
+            /*
+             * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
+             */
+            mmu_idx += ARMMMUIdx_S1NSE0;
+        }
     }
 
     /* The page table entries may downgrade secure to non-secure, but
@@ -6158,13 +7912,13 @@ static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
 
     if (regime_using_lpae_format(env, mmu_idx)) {
         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
-                                  attrs, prot, page_size, fsr);
+                                  attrs, prot, page_size, fsr, fi);
     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
-                                attrs, prot, page_size, fsr);
+                                attrs, prot, page_size, fsr, fi);
     } else {
         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
-                                prot, page_size, fsr);
+                                prot, page_size, fsr, fi);
     }
 }
 
@@ -6173,7 +7927,8 @@ static inline bool get_phys_addr(CPUARMState *env, target_ulong address,
  * fsr with ARM DFSR/IFSR fault register format value on failure.
  */
 bool arm_tlb_fill(CPUState *cs, vaddr address,
-                  int access_type, int mmu_idx, uint32_t *fsr)
+                  int access_type, int mmu_idx, uint32_t *fsr,
+                  ARMMMUFaultInfo *fi)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
@@ -6184,7 +7939,7 @@ bool arm_tlb_fill(CPUState *cs, vaddr address,
     MemTxAttrs attrs = {};
 
     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
-                        &attrs, &prot, &page_size, fsr);
+                        &attrs, &prot, &page_size, fsr, fi);
     if (!ret) {
         /* Map a single [sub]page.  */
         phys_addr &= TARGET_PAGE_MASK;
@@ -6197,7 +7952,8 @@ bool arm_tlb_fill(CPUState *cs, vaddr address,
     return ret;
 }
 
-hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
+hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
+                                         MemTxAttrs *attrs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
@@ -6206,36 +7962,19 @@ hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     int prot;
     bool ret;
     uint32_t fsr;
-    MemTxAttrs attrs = {};
+    ARMMMUFaultInfo fi = {};
 
-    ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env), &phys_addr,
-                        &attrs, &prot, &page_size, &fsr);
+    *attrs = (MemTxAttrs) {};
+
+    ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env, false), &phys_addr,
+                        attrs, &prot, &page_size, &fsr, &fi);
 
     if (ret) {
         return -1;
     }
-
     return phys_addr;
 }
 
-void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
-{
-    if ((env->uncached_cpsr & CPSR_M) == mode) {
-        env->regs[13] = val;
-    } else {
-        env->banked_r13[bank_number(mode)] = val;
-    }
-}
-
-uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
-{
-    if ((env->uncached_cpsr & CPSR_M) == mode) {
-        return env->regs[13];
-    } else {
-        return env->banked_r13[bank_number(mode)];
-    }
-}
-
 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -6373,7 +8112,7 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
         void *hostaddr[maxidx];
         int try, i;
-        unsigned mmu_idx = cpu_mmu_index(env);
+        unsigned mmu_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
 
         for (try = 0; try < 2; try++) {
diff --git a/qemu/target-arm/helper.h b/qemu/target-arm/helper.h
index 827b33dfe..84aa63762 100644
--- a/qemu/target-arm/helper.h
+++ b/qemu/target-arm/helper.h
@@ -48,19 +48,23 @@ DEF_HELPER_FLAGS_3(sel_flags, TCG_CALL_NO_RWG_SE,
                    i32, i32, i32, i32)
 DEF_HELPER_2(exception_internal, void, env, i32)
 DEF_HELPER_4(exception_with_syndrome, void, env, i32, i32, i32)
+DEF_HELPER_1(setend, void, env)
 DEF_HELPER_1(wfi, void, env)
 DEF_HELPER_1(wfe, void, env)
 DEF_HELPER_1(yield, void, env)
 DEF_HELPER_1(pre_hvc, void, env)
 DEF_HELPER_2(pre_smc, void, env, i32)
 
+DEF_HELPER_1(check_breakpoints, void, env)
+
 DEF_HELPER_3(cpsr_write, void, env, i32, i32)
+DEF_HELPER_2(cpsr_write_eret, void, env, i32)
 DEF_HELPER_1(cpsr_read, i32, env)
 
 DEF_HELPER_3(v7m_msr, void, env, i32, i32)
 DEF_HELPER_2(v7m_mrs, i32, env, i32)
 
-DEF_HELPER_3(access_check_cp_reg, void, env, ptr, i32)
+DEF_HELPER_4(access_check_cp_reg, void, env, ptr, i32, i32)
 DEF_HELPER_3(set_cp_reg, void, env, ptr, i32)
 DEF_HELPER_2(get_cp_reg, i32, env, ptr)
 DEF_HELPER_3(set_cp_reg64, void, env, ptr, i64)
@@ -73,6 +77,9 @@ DEF_HELPER_1(exception_return, void, env)
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
 DEF_HELPER_3(set_r13_banked, void, env, i32, i32)
 
+DEF_HELPER_3(mrs_banked, i32, env, i32, i32)
+DEF_HELPER_4(msr_banked, void, env, i32, i32, i32)
+
 DEF_HELPER_2(get_user_reg, i32, env, i32)
 DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
diff --git a/qemu/target-arm/internals.h b/qemu/target-arm/internals.h
index 924aff9d0..2e70272be 100644
--- a/qemu/target-arm/internals.h
+++ b/qemu/target-arm/internals.h
@@ -25,6 +25,16 @@
 #ifndef TARGET_ARM_INTERNALS_H
 #define TARGET_ARM_INTERNALS_H
 
+/* register banks for CPU modes */
+#define BANK_USRSYS 0
+#define BANK_SVC    1
+#define BANK_ABT    2
+#define BANK_UND    3
+#define BANK_IRQ    4
+#define BANK_FIQ    5
+#define BANK_HYP    6
+#define BANK_MON    7
+
 static inline bool excp_is_internal(int excp)
 {
     /* Return true if this exception number represents a QEMU-internal
@@ -36,6 +46,7 @@ static inline bool excp_is_internal(int excp)
         || excp == EXCP_HALTED
         || excp == EXCP_EXCEPTION_EXIT
         || excp == EXCP_KERNEL_TRAP
+        || excp == EXCP_SEMIHOST
         || excp == EXCP_STREX;
 }
 
@@ -58,6 +69,7 @@ static const char * const excnames[] = {
     [EXCP_SMC] = "Secure Monitor Call",
     [EXCP_VIRQ] = "Virtual IRQ",
     [EXCP_VFIQ] = "Virtual FIQ",
+    [EXCP_SEMIHOST] = "Semihosting call",
 };
 
 static inline void arm_log_exception(int idx)
@@ -89,15 +101,39 @@ static inline void arm_log_exception(int idx)
 static inline unsigned int aarch64_banked_spsr_index(unsigned int el)
 {
     static const unsigned int map[4] = {
-        [1] = 1, /* EL1.  */
-        [2] = 6, /* EL2.  */
-        [3] = 7, /* EL3.  */
+        [1] = BANK_SVC, /* EL1.  */
+        [2] = BANK_HYP, /* EL2.  */
+        [3] = BANK_MON, /* EL3.  */
     };
     assert(el >= 1 && el <= 3);
     return map[el];
 }
 
-int bank_number(int mode);
+/* Map CPU modes onto saved register banks.  */
+static inline int bank_number(int mode)
+{
+    switch (mode) {
+    case ARM_CPU_MODE_USR:
+    case ARM_CPU_MODE_SYS:
+        return BANK_USRSYS;
+    case ARM_CPU_MODE_SVC:
+        return BANK_SVC;
+    case ARM_CPU_MODE_ABT:
+        return BANK_ABT;
+    case ARM_CPU_MODE_UND:
+        return BANK_UND;
+    case ARM_CPU_MODE_IRQ:
+        return BANK_IRQ;
+    case ARM_CPU_MODE_FIQ:
+        return BANK_FIQ;
+    case ARM_CPU_MODE_HYP:
+        return BANK_HYP;
+    case ARM_CPU_MODE_MON:
+        return BANK_MON;
+    }
+    g_assert_not_reached();
+}
+
 void switch_mode(CPUARMState *, int);
 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu);
 void arm_translate_init(void);
@@ -150,6 +186,31 @@ static inline void update_spsel(CPUARMState *env, uint32_t imm)
     aarch64_restore_sp(env, cur_el);
 }
 
+/*
+ * arm_pamax
+ * @cpu: ARMCPU
+ *
+ * Returns the implementation defined bit-width of physical addresses.
+ * The ARMv8 reference manuals refer to this as PAMax().
+ */
+static inline unsigned int arm_pamax(ARMCPU *cpu)
+{
+    static const unsigned int pamax_map[] = {
+        [0] = 32,
+        [1] = 36,
+        [2] = 40,
+        [3] = 42,
+        [4] = 44,
+        [5] = 48,
+    };
+    unsigned int parange = extract32(cpu->id_aa64mmfr0, 0, 4);
+
+    /* id_aa64mmfr0 is a read-only register so values outside of the
+     * supported mappings can be considered an implementation error.  */
+    assert(parange < ARRAY_SIZE(pamax_map));
+    return pamax_map[parange];
+}
+
 /* Return true if extended addresses are enabled.
  * This is always the case if our translation regime is 64 bit,
  * but depends on TTBCR.EAE for 32 bit.
@@ -233,10 +294,10 @@ static inline uint32_t syn_aa64_smc(uint32_t imm16)
     return (EC_AA64_SMC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
 }
 
-static inline uint32_t syn_aa32_svc(uint32_t imm16, bool is_thumb)
+static inline uint32_t syn_aa32_svc(uint32_t imm16, bool is_16bit)
 {
     return (EC_AA32_SVC << ARM_EL_EC_SHIFT) | (imm16 & 0xffff)
-        | (is_thumb ? 0 : ARM_EL_IL);
+        | (is_16bit ? 0 : ARM_EL_IL);
 }
 
 static inline uint32_t syn_aa32_hvc(uint32_t imm16)
@@ -254,10 +315,10 @@ static inline uint32_t syn_aa64_bkpt(uint32_t imm16)
     return (EC_AA64_BKPT << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
 }
 
-static inline uint32_t syn_aa32_bkpt(uint32_t imm16, bool is_thumb)
+static inline uint32_t syn_aa32_bkpt(uint32_t imm16, bool is_16bit)
 {
     return (EC_AA32_BKPT << ARM_EL_EC_SHIFT) | (imm16 & 0xffff)
-        | (is_thumb ? 0 : ARM_EL_IL);
+        | (is_16bit ? 0 : ARM_EL_IL);
 }
 
 static inline uint32_t syn_aa64_sysregtrap(int op0, int op1, int op2,
@@ -271,48 +332,48 @@ static inline uint32_t syn_aa64_sysregtrap(int op0, int op1, int op2,
 
 static inline uint32_t syn_cp14_rt_trap(int cv, int cond, int opc1, int opc2,
                                         int crn, int crm, int rt, int isread,
-                                        bool is_thumb)
+                                        bool is_16bit)
 {
     return (EC_CP14RTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc2 << 17) | (opc1 << 14)
         | (crn << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp15_rt_trap(int cv, int cond, int opc1, int opc2,
                                         int crn, int crm, int rt, int isread,
-                                        bool is_thumb)
+                                        bool is_16bit)
 {
     return (EC_CP15RTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc2 << 17) | (opc1 << 14)
         | (crn << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp14_rrt_trap(int cv, int cond, int opc1, int crm,
                                          int rt, int rt2, int isread,
-                                         bool is_thumb)
+                                         bool is_16bit)
 {
     return (EC_CP14RRTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc1 << 16)
         | (rt2 << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp15_rrt_trap(int cv, int cond, int opc1, int crm,
                                          int rt, int rt2, int isread,
-                                         bool is_thumb)
+                                         bool is_16bit)
 {
     return (EC_CP15RRTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc1 << 16)
         | (rt2 << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
-static inline uint32_t syn_fp_access_trap(int cv, int cond, bool is_thumb)
+static inline uint32_t syn_fp_access_trap(int cv, int cond, bool is_16bit)
 {
     return (EC_ADVSIMDFPACCESSTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20);
 }
 
@@ -372,6 +433,9 @@ void hw_breakpoint_update(ARMCPU *cpu, int n);
  */
 void hw_breakpoint_update_all(ARMCPU *cpu);
 
+/* Callback function for checking if a watchpoint should trigger. */
+bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp);
+
 /* Callback function for when a watchpoint or breakpoint triggers. */
 void arm_debug_excp_handler(CPUState *cs);
 
@@ -387,8 +451,29 @@ bool arm_is_psci_call(ARMCPU *cpu, int excp_type);
 void arm_handle_psci_call(ARMCPU *cpu);
 #endif
 
+/**
+ * ARMMMUFaultInfo: Information describing an ARM MMU Fault
+ * @s2addr: Address that caused a fault at stage 2
+ * @stage2: True if we faulted at stage 2
+ * @s1ptw: True if we faulted at stage 2 while doing a stage 1 page-table walk
+ */
+typedef struct ARMMMUFaultInfo ARMMMUFaultInfo;
+struct ARMMMUFaultInfo {
+    target_ulong s2addr;
+    bool stage2;
+    bool s1ptw;
+};
+
 /* Do a page table walk and add page to TLB if possible */
 bool arm_tlb_fill(CPUState *cpu, vaddr address, int rw, int mmu_idx,
-                  uint32_t *fsr);
+                  uint32_t *fsr, ARMMMUFaultInfo *fi);
+
+/* Return true if the stage 1 translation regime is using LPAE format page
+ * tables */
+bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx);
+
+/* Raise a data fault alignment exception for the specified virtual address */
+void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, int is_write,
+                                 int is_user, uintptr_t retaddr);
 
 #endif
diff --git a/qemu/target-arm/iwmmxt_helper.c b/qemu/target-arm/iwmmxt_helper.c
index a5069144d..7d87e1a0a 100644
--- a/qemu/target-arm/iwmmxt_helper.c
+++ b/qemu/target-arm/iwmmxt_helper.c
@@ -19,8 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
diff --git a/qemu/target-arm/kvm-consts.h b/qemu/target-arm/kvm-consts.h
index 943bf8980..a2c951859 100644
--- a/qemu/target-arm/kvm-consts.h
+++ b/qemu/target-arm/kvm-consts.h
@@ -15,7 +15,6 @@
 #define ARM_KVM_CONSTS_H
 
 #ifdef CONFIG_KVM
-#include "qemu/compiler.h"
 #include <linux/kvm.h>
 #include <linux/psci.h>
 
diff --git a/qemu/target-arm/kvm-stub.c b/qemu/target-arm/kvm-stub.c
index db2edc2c4..38bf43387 100644
--- a/qemu/target-arm/kvm-stub.c
+++ b/qemu/target-arm/kvm-stub.c
@@ -9,6 +9,7 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "kvm_arm.h"
 
diff --git a/qemu/target-arm/kvm.c b/qemu/target-arm/kvm.c
index b27854208..36710320f 100644
--- a/qemu/target-arm/kvm.c
+++ b/qemu/target-arm/kvm.c
@@ -8,8 +8,7 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 
@@ -17,6 +16,7 @@
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
@@ -24,6 +24,7 @@
 #include "internals.h"
 #include "hw/arm/arm.h"
 #include "exec/memattrs.h"
+#include "hw/boards.h"
 
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
@@ -61,13 +62,18 @@ bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
         goto err;
     }
 
+    if (!init) {
+        /* Caller doesn't want the VCPU to be initialized, so skip it */
+        goto finish;
+    }
+
     ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
     if (ret >= 0) {
         ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
         if (ret < 0) {
             goto err;
         }
-    } else {
+    } else if (cpus_to_try) {
         /* Old kernel which doesn't know about the
          * PREFERRED_TARGET ioctl: we know it will only support
          * creating one kind of guest CPU which is its preferred
@@ -84,8 +90,15 @@ bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
         if (ret < 0) {
             goto err;
         }
+    } else {
+        /* Treat a NULL cpus_to_try argument the same as an empty
+         * list, which means we will fail the call since this must
+         * be an old kernel which doesn't support PREFERRED_TARGET.
+         */
+        goto err;
     }
 
+finish:
     fdarray[0] = kvmfd;
     fdarray[1] = vmfd;
     fdarray[2] = cpufd;
@@ -516,9 +529,23 @@ MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
     return MEMTXATTRS_UNSPECIFIED;
 }
 
+
 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
-    return 0;
+    int ret = 0;
+
+    switch (run->exit_reason) {
+    case KVM_EXIT_DEBUG:
+        if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
+            ret = EXCP_DEBUG;
+        } /* otherwise return to guest */
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
+                      __func__, run->exit_reason);
+        break;
+    }
+    return ret;
 }
 
 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
@@ -541,66 +568,56 @@ int kvm_arch_on_sigbus(int code, void *addr)
     return 1;
 }
 
+/* The #ifdef protections are until 32bit headers are imported and can
+ * be removed once both 32 and 64 bit reach feature parity.
+ */
 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
 {
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-}
-
-int kvm_arch_insert_sw_breakpoint(CPUState *cs,
-                                  struct kvm_sw_breakpoint *bp)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_insert_hw_breakpoint(target_ulong addr,
-                                  target_ulong len, int type)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_remove_hw_breakpoint(target_ulong addr,
-                                  target_ulong len, int type)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_remove_sw_breakpoint(CPUState *cs,
-                                  struct kvm_sw_breakpoint *bp)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-void kvm_arch_remove_all_hw_breakpoints(void)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+#ifdef KVM_GUESTDBG_USE_SW_BP
+    if (kvm_sw_breakpoints_active(cs)) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
+    }
+#endif
+#ifdef KVM_GUESTDBG_USE_HW
+    if (kvm_arm_hw_debug_active(cs)) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
+        kvm_arm_copy_hw_debug_data(&dbg->arch);
+    }
+#endif
 }
 
 void kvm_arch_init_irq_routing(KVMState *s)
 {
 }
 
-int kvm_arch_irqchip_create(KVMState *s)
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
 {
-    int ret;
+     if (machine_kernel_irqchip_split(ms)) {
+         perror("-machine kernel_irqchip=split is not supported on ARM.");
+         exit(1);
+    }
 
     /* If we can create the VGIC using the newer device control API, we
      * let the device do this when it initializes itself, otherwise we
      * fall back to the old API */
+    return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
+}
 
-    ret = kvm_create_device(s, KVM_DEV_TYPE_ARM_VGIC_V2, true);
-    if (ret == 0) {
-        return 1;
+int kvm_arm_vgic_probe(void)
+{
+    if (kvm_create_device(kvm_state,
+                          KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
+        return 3;
+    } else if (kvm_create_device(kvm_state,
+                                 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
+        return 2;
+    } else {
+        return 0;
     }
-
-    return 0;
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/qemu/target-arm/kvm32.c b/qemu/target-arm/kvm32.c
index 421ce0ea0..d44a7f92b 100644
--- a/qemu/target-arm/kvm32.c
+++ b/qemu/target-arm/kvm32.c
@@ -8,8 +8,7 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 
@@ -181,7 +180,6 @@ int kvm_arm_cpreg_level(uint64_t regidx)
     return KVM_PUT_RUNTIME_STATE;
 }
 
-#define ARM_MPIDR_HWID_BITMASK 0xFFFFFF
 #define ARM_CPU_ID_MPIDR       0, 0, 0, 5
 
 int kvm_arch_init_vcpu(CPUState *cs)
@@ -234,7 +232,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (ret) {
         return ret;
     }
-    cpu->mp_affinity = mpidr & ARM_MPIDR_HWID_BITMASK;
+    cpu->mp_affinity = mpidr & ARM32_AFFINITY_MASK;
 
     return kvm_arm_init_cpreg_list(cpu);
 }
@@ -281,30 +279,30 @@ static const Reg regs[] = {
     COREREG(usr_regs.uregs[10], usr_regs[2]),
     COREREG(usr_regs.uregs[11], usr_regs[3]),
     COREREG(usr_regs.uregs[12], usr_regs[4]),
-    COREREG(usr_regs.uregs[13], banked_r13[0]),
-    COREREG(usr_regs.uregs[14], banked_r14[0]),
+    COREREG(usr_regs.uregs[13], banked_r13[BANK_USRSYS]),
+    COREREG(usr_regs.uregs[14], banked_r14[BANK_USRSYS]),
     /* R13, R14, SPSR for SVC, ABT, UND, IRQ banks */
-    COREREG(svc_regs[0], banked_r13[1]),
-    COREREG(svc_regs[1], banked_r14[1]),
-    COREREG64(svc_regs[2], banked_spsr[1]),
-    COREREG(abt_regs[0], banked_r13[2]),
-    COREREG(abt_regs[1], banked_r14[2]),
-    COREREG64(abt_regs[2], banked_spsr[2]),
-    COREREG(und_regs[0], banked_r13[3]),
-    COREREG(und_regs[1], banked_r14[3]),
-    COREREG64(und_regs[2], banked_spsr[3]),
-    COREREG(irq_regs[0], banked_r13[4]),
-    COREREG(irq_regs[1], banked_r14[4]),
-    COREREG64(irq_regs[2], banked_spsr[4]),
+    COREREG(svc_regs[0], banked_r13[BANK_SVC]),
+    COREREG(svc_regs[1], banked_r14[BANK_SVC]),
+    COREREG64(svc_regs[2], banked_spsr[BANK_SVC]),
+    COREREG(abt_regs[0], banked_r13[BANK_ABT]),
+    COREREG(abt_regs[1], banked_r14[BANK_ABT]),
+    COREREG64(abt_regs[2], banked_spsr[BANK_ABT]),
+    COREREG(und_regs[0], banked_r13[BANK_UND]),
+    COREREG(und_regs[1], banked_r14[BANK_UND]),
+    COREREG64(und_regs[2], banked_spsr[BANK_UND]),
+    COREREG(irq_regs[0], banked_r13[BANK_IRQ]),
+    COREREG(irq_regs[1], banked_r14[BANK_IRQ]),
+    COREREG64(irq_regs[2], banked_spsr[BANK_IRQ]),
     /* R8_fiq .. R14_fiq and SPSR_fiq */
     COREREG(fiq_regs[0], fiq_regs[0]),
     COREREG(fiq_regs[1], fiq_regs[1]),
     COREREG(fiq_regs[2], fiq_regs[2]),
     COREREG(fiq_regs[3], fiq_regs[3]),
     COREREG(fiq_regs[4], fiq_regs[4]),
-    COREREG(fiq_regs[5], banked_r13[5]),
-    COREREG(fiq_regs[6], banked_r14[5]),
-    COREREG64(fiq_regs[7], banked_spsr[5]),
+    COREREG(fiq_regs[5], banked_r13[BANK_FIQ]),
+    COREREG(fiq_regs[6], banked_r14[BANK_FIQ]),
+    COREREG64(fiq_regs[7], banked_spsr[BANK_FIQ]),
     /* R15 */
     COREREG(usr_regs.uregs[15], regs[15]),
     /* VFP system registers */
@@ -430,7 +428,7 @@ int kvm_arch_get_registers(CPUState *cs)
     if (ret) {
         return ret;
     }
-    cpsr_write(env, cpsr, 0xffffffff);
+    cpsr_write(env, cpsr, 0xffffffff, CPSRWriteRaw);
 
     /* Make sure the current mode regs are properly set */
     mode = env->uncached_cpsr & CPSR_M;
@@ -476,3 +474,50 @@ int kvm_arch_get_registers(CPUState *cs)
 
     return 0;
 }
+
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return -EINVAL;
+}
+
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return -EINVAL;
+}
+
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return false;
+}
+
+int kvm_arch_insert_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+    return -EINVAL;
+}
+
+int kvm_arch_remove_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+    return -EINVAL;
+}
+
+void kvm_arch_remove_all_hw_breakpoints(void)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+}
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+}
+
+bool kvm_arm_hw_debug_active(CPUState *cs)
+{
+    return false;
+}
diff --git a/qemu/target-arm/kvm64.c b/qemu/target-arm/kvm64.c
index bd60889d1..e8527bf0c 100644
--- a/qemu/target-arm/kvm64.c
+++ b/qemu/target-arm/kvm64.c
@@ -2,22 +2,26 @@
  * ARM implementation of KVM hooks, 64 bit specific code
  *
  * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
+ * Copyright Alex Bennée 2014, Linaro
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
+#include <sys/ptrace.h>
 
+#include <linux/elf.h>
 #include <linux/kvm.h>
 
-#include "config-host.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
+#include "qemu/host-utils.h"
+#include "exec/gdbstub.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
@@ -25,6 +29,360 @@
 #include "internals.h"
 #include "hw/arm/arm.h"
 
+static bool have_guest_debug;
+
+/*
+ * Although the ARM implementation of hardware assisted debugging
+ * allows for different breakpoints per-core, the current GDB
+ * interface treats them as a global pool of registers (which seems to
+ * be the case for x86, ppc and s390). As a result we store one copy
+ * of registers which is used for all active cores.
+ *
+ * Write access is serialised by virtue of the GDB protocol which
+ * updates things. Read access (i.e. when the values are copied to the
+ * vCPU) is also gated by GDB's run control.
+ *
+ * This is not unreasonable as most of the time debugging kernels you
+ * never know which core will eventually execute your function.
+ */
+
+typedef struct {
+    uint64_t bcr;
+    uint64_t bvr;
+} HWBreakpoint;
+
+/* The watchpoint registers can cover more area than the requested
+ * watchpoint so we need to store the additional information
+ * somewhere. We also need to supply a CPUWatchpoint to the GDB stub
+ * when the watchpoint is hit.
+ */
+typedef struct {
+    uint64_t wcr;
+    uint64_t wvr;
+    CPUWatchpoint details;
+} HWWatchpoint;
+
+/* Maximum and current break/watch point counts */
+int max_hw_bps, max_hw_wps;
+GArray *hw_breakpoints, *hw_watchpoints;
+
+#define cur_hw_wps      (hw_watchpoints->len)
+#define cur_hw_bps      (hw_breakpoints->len)
+#define get_hw_bp(i)    (&g_array_index(hw_breakpoints, HWBreakpoint, i))
+#define get_hw_wp(i)    (&g_array_index(hw_watchpoints, HWWatchpoint, i))
+
+/**
+ * kvm_arm_init_debug() - check for guest debug capabilities
+ * @cs: CPUState
+ *
+ * kvm_check_extension returns the number of debug registers we have
+ * or 0 if we have none.
+ *
+ */
+static void kvm_arm_init_debug(CPUState *cs)
+{
+    have_guest_debug = kvm_check_extension(cs->kvm_state,
+                                           KVM_CAP_SET_GUEST_DEBUG);
+
+    max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS);
+    hw_watchpoints = g_array_sized_new(true, true,
+                                       sizeof(HWWatchpoint), max_hw_wps);
+
+    max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS);
+    hw_breakpoints = g_array_sized_new(true, true,
+                                       sizeof(HWBreakpoint), max_hw_bps);
+    return;
+}
+
+/**
+ * insert_hw_breakpoint()
+ * @addr: address of breakpoint
+ *
+ * See ARM ARM D2.9.1 for details but here we are only going to create
+ * simple un-linked breakpoints (i.e. we don't chain breakpoints
+ * together to match address and context or vmid). The hardware is
+ * capable of fancier matching but that will require exposing that
+ * fanciness to GDB's interface
+ *
+ * D7.3.2 DBGBCR<n>_EL1, Debug Breakpoint Control Registers
+ *
+ *  31  24 23  20 19   16 15 14  13  12   9 8   5 4    3 2   1  0
+ * +------+------+-------+-----+----+------+-----+------+-----+---+
+ * | RES0 |  BT  |  LBN  | SSC | HMC| RES0 | BAS | RES0 | PMC | E |
+ * +------+------+-------+-----+----+------+-----+------+-----+---+
+ *
+ * BT: Breakpoint type (0 = unlinked address match)
+ * LBN: Linked BP number (0 = unused)
+ * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12)
+ * BAS: Byte Address Select (RES1 for AArch64)
+ * E: Enable bit
+ */
+static int insert_hw_breakpoint(target_ulong addr)
+{
+    HWBreakpoint brk = {
+        .bcr = 0x1,                             /* BCR E=1, enable */
+        .bvr = addr
+    };
+
+    if (cur_hw_bps >= max_hw_bps) {
+        return -ENOBUFS;
+    }
+
+    brk.bcr = deposit32(brk.bcr, 1, 2, 0x3);   /* PMC = 11 */
+    brk.bcr = deposit32(brk.bcr, 5, 4, 0xf);   /* BAS = RES1 */
+
+    g_array_append_val(hw_breakpoints, brk);
+
+    return 0;
+}
+
+/**
+ * delete_hw_breakpoint()
+ * @pc: address of breakpoint
+ *
+ * Delete a breakpoint and shuffle any above down
+ */
+
+static int delete_hw_breakpoint(target_ulong pc)
+{
+    int i;
+    for (i = 0; i < hw_breakpoints->len; i++) {
+        HWBreakpoint *brk = get_hw_bp(i);
+        if (brk->bvr == pc) {
+            g_array_remove_index(hw_breakpoints, i);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+/**
+ * insert_hw_watchpoint()
+ * @addr: address of watch point
+ * @len: size of area
+ * @type: type of watch point
+ *
+ * See ARM ARM D2.10. As with the breakpoints we can do some advanced
+ * stuff if we want to. The watch points can be linked with the break
+ * points above to make them context aware. However for simplicity
+ * currently we only deal with simple read/write watch points.
+ *
+ * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers
+ *
+ *  31  29 28   24 23  21  20  19 16 15 14  13   12  5 4   3 2   1  0
+ * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
+ * | RES0 |  MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E |
+ * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
+ *
+ * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes))
+ * WT: 0 - unlinked, 1 - linked (not currently used)
+ * LBN: Linked BP number (not currently used)
+ * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11)
+ * BAS: Byte Address Select
+ * LSC: Load/Store control (01: load, 10: store, 11: both)
+ * E: Enable
+ *
+ * The bottom 2 bits of the value register are masked. Therefore to
+ * break on any sizes smaller than an unaligned word you need to set
+ * MASK=0, BAS=bit per byte in question. For larger regions (^2) you
+ * need to ensure you mask the address as required and set BAS=0xff
+ */
+
+static int insert_hw_watchpoint(target_ulong addr,
+                                target_ulong len, int type)
+{
+    HWWatchpoint wp = {
+        .wcr = 1, /* E=1, enable */
+        .wvr = addr & (~0x7ULL),
+        .details = { .vaddr = addr, .len = len }
+    };
+
+    if (cur_hw_wps >= max_hw_wps) {
+        return -ENOBUFS;
+    }
+
+    /*
+     * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state,
+     * valid whether EL3 is implemented or not
+     */
+    wp.wcr = deposit32(wp.wcr, 1, 2, 3);
+
+    switch (type) {
+    case GDB_WATCHPOINT_READ:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 1);
+        wp.details.flags = BP_MEM_READ;
+        break;
+    case GDB_WATCHPOINT_WRITE:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 2);
+        wp.details.flags = BP_MEM_WRITE;
+        break;
+    case GDB_WATCHPOINT_ACCESS:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 3);
+        wp.details.flags = BP_MEM_ACCESS;
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+    if (len <= 8) {
+        /* we align the address and set the bits in BAS */
+        int off = addr & 0x7;
+        int bas = (1 << len) - 1;
+
+        wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas);
+    } else {
+        /* For ranges above 8 bytes we need to be a power of 2 */
+        if (is_power_of_2(len)) {
+            int bits = ctz64(len);
+
+            wp.wvr &= ~((1 << bits) - 1);
+            wp.wcr = deposit32(wp.wcr, 24, 4, bits);
+            wp.wcr = deposit32(wp.wcr, 5, 8, 0xff);
+        } else {
+            return -ENOBUFS;
+        }
+    }
+
+    g_array_append_val(hw_watchpoints, wp);
+    return 0;
+}
+
+
+static bool check_watchpoint_in_range(int i, target_ulong addr)
+{
+    HWWatchpoint *wp = get_hw_wp(i);
+    uint64_t addr_top, addr_bottom = wp->wvr;
+    int bas = extract32(wp->wcr, 5, 8);
+    int mask = extract32(wp->wcr, 24, 4);
+
+    if (mask) {
+        addr_top = addr_bottom + (1 << mask);
+    } else {
+        /* BAS must be contiguous but can offset against the base
+         * address in DBGWVR */
+        addr_bottom = addr_bottom + ctz32(bas);
+        addr_top = addr_bottom + clo32(bas);
+    }
+
+    if (addr >= addr_bottom && addr <= addr_top) {
+        return true;
+    }
+
+    return false;
+}
+
+/**
+ * delete_hw_watchpoint()
+ * @addr: address of breakpoint
+ *
+ * Delete a breakpoint and shuffle any above down
+ */
+
+static int delete_hw_watchpoint(target_ulong addr,
+                                target_ulong len, int type)
+{
+    int i;
+    for (i = 0; i < cur_hw_wps; i++) {
+        if (check_watchpoint_in_range(i, addr)) {
+            g_array_remove_index(hw_watchpoints, i);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+
+int kvm_arch_insert_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        return insert_hw_breakpoint(addr);
+        break;
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_ACCESS:
+        return insert_hw_watchpoint(addr, len, type);
+    default:
+        return -ENOSYS;
+    }
+}
+
+int kvm_arch_remove_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        return delete_hw_breakpoint(addr);
+        break;
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_ACCESS:
+        return delete_hw_watchpoint(addr, len, type);
+    default:
+        return -ENOSYS;
+    }
+}
+
+
+void kvm_arch_remove_all_hw_breakpoints(void)
+{
+    if (cur_hw_wps > 0) {
+        g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
+    }
+    if (cur_hw_bps > 0) {
+        g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
+    }
+}
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
+{
+    int i;
+    memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
+
+    for (i = 0; i < max_hw_wps; i++) {
+        HWWatchpoint *wp = get_hw_wp(i);
+        ptr->dbg_wcr[i] = wp->wcr;
+        ptr->dbg_wvr[i] = wp->wvr;
+    }
+    for (i = 0; i < max_hw_bps; i++) {
+        HWBreakpoint *bp = get_hw_bp(i);
+        ptr->dbg_bcr[i] = bp->bcr;
+        ptr->dbg_bvr[i] = bp->bvr;
+    }
+}
+
+bool kvm_arm_hw_debug_active(CPUState *cs)
+{
+    return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
+}
+
+static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc)
+{
+    int i;
+
+    for (i = 0; i < cur_hw_bps; i++) {
+        HWBreakpoint *bp = get_hw_bp(i);
+        if (bp->bvr == pc) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr)
+{
+    int i;
+
+    for (i = 0; i < cur_hw_wps; i++) {
+        if (check_watchpoint_in_range(i, addr)) {
+            return &get_hw_wp(i)->details;
+        }
+    }
+    return NULL;
+}
+
+
 static inline void set_feature(uint64_t *features, int feature)
 {
     *features |= 1ULL << feature;
@@ -77,7 +435,6 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUClass *ahcc)
     return true;
 }
 
-#define ARM_MPIDR_HWID_BITMASK 0xFF00FFFFFFULL
 #define ARM_CPU_ID_MPIDR       3, 0, 0, 0, 5
 
 int kvm_arch_init_vcpu(CPUState *cs)
@@ -120,7 +477,9 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (ret) {
         return ret;
     }
-    cpu->mp_affinity = mpidr & ARM_MPIDR_HWID_BITMASK;
+    cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
+
+    kvm_arm_init_debug(cs);
 
     return kvm_arm_init_cpreg_list(cpu);
 }
@@ -363,8 +722,7 @@ int kvm_arch_get_registers(CPUState *cs)
     if (is_a64(env)) {
         pstate_write(env, val);
     } else {
-        env->uncached_cpsr = val & CPSR_M;
-        cpsr_write(env, val, 0xffffffff);
+        cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
     }
 
     /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
@@ -464,3 +822,105 @@ int kvm_arch_get_registers(CPUState *cs)
     /* TODO: other registers */
     return ret;
 }
+
+/* C6.6.29 BRK instruction */
+static const uint32_t brk_insn = 0xd4200000;
+
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    if (have_guest_debug) {
+        if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
+            cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
+            return -EINVAL;
+        }
+        return 0;
+    } else {
+        error_report("guest debug not supported on this kernel");
+        return -EINVAL;
+    }
+}
+
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    static uint32_t brk;
+
+    if (have_guest_debug) {
+        if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
+            brk != brk_insn ||
+            cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
+            return -EINVAL;
+        }
+        return 0;
+    } else {
+        error_report("guest debug not supported on this kernel");
+        return -EINVAL;
+    }
+}
+
+/* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
+ *
+ * To minimise translating between kernel and user-space the kernel
+ * ABI just provides user-space with the full exception syndrome
+ * register value to be decoded in QEMU.
+ */
+
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
+{
+    int hsr_ec = debug_exit->hsr >> ARM_EL_EC_SHIFT;
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUClass *cc = CPU_GET_CLASS(cs);
+    CPUARMState *env = &cpu->env;
+
+    /* Ensure PC is synchronised */
+    kvm_cpu_synchronize_state(cs);
+
+    switch (hsr_ec) {
+    case EC_SOFTWARESTEP:
+        if (cs->singlestep_enabled) {
+            return true;
+        } else {
+            /*
+             * The kernel should have suppressed the guest's ability to
+             * single step at this point so something has gone wrong.
+             */
+            error_report("%s: guest single-step while debugging unsupported"
+                         " (%"PRIx64", %"PRIx32")\n",
+                         __func__, env->pc, debug_exit->hsr);
+            return false;
+        }
+        break;
+    case EC_AA64_BKPT:
+        if (kvm_find_sw_breakpoint(cs, env->pc)) {
+            return true;
+        }
+        break;
+    case EC_BREAKPOINT:
+        if (find_hw_breakpoint(cs, env->pc)) {
+            return true;
+        }
+        break;
+    case EC_WATCHPOINT:
+    {
+        CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
+        if (wp) {
+            cs->watchpoint_hit = wp;
+            return true;
+        }
+        break;
+    }
+    default:
+        error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")\n",
+                     __func__, debug_exit->hsr, env->pc);
+    }
+
+    /* If we are not handling the debug exception it must belong to
+     * the guest. Let's re-use the existing TCG interrupt code to set
+     * everything up properly.
+     */
+    cs->exception_index = EXCP_BKPT;
+    env->exception.syndrome = debug_exit->hsr;
+    env->exception.vaddress = debug_exit->far;
+    cc->do_interrupt(cs);
+
+    return false;
+}
diff --git a/qemu/target-arm/kvm_arm.h b/qemu/target-arm/kvm_arm.h
index 7912d7433..345233c18 100644
--- a/qemu/target-arm/kvm_arm.h
+++ b/qemu/target-arm/kvm_arm.h
@@ -124,9 +124,12 @@ void kvm_arm_reset_vcpu(ARMCPU *cpu);
  * kvm_arm_create_scratch_host_vcpu:
  * @cpus_to_try: array of QEMU_KVM_ARM_TARGET_* values (terminated with
  * QEMU_KVM_ARM_TARGET_NONE) to try as fallback if the kernel does not
- * know the PREFERRED_TARGET ioctl
+ * know the PREFERRED_TARGET ioctl. Passing NULL is the same as passing
+ * an empty array.
  * @fdarray: filled in with kvmfd, vmfd, cpufd file descriptors in that order
- * @init: filled in with the necessary values for creating a host vcpu
+ * @init: filled in with the necessary values for creating a host
+ * vcpu. If NULL is provided, will not init the vCPU (though the cpufd
+ * will still be set up).
  *
  * Create a scratch vcpu in its own VM of the type preferred by the host
  * kernel (as would be used for '-cpu host'), for purposes of probing it
@@ -189,6 +192,60 @@ int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu);
  */
 int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu);
 
+int kvm_arm_vgic_probe(void);
+
+#else
+
+static inline int kvm_arm_vgic_probe(void)
+{
+    return 0;
+}
+
 #endif
 
+static inline const char *gic_class_name(void)
+{
+    return kvm_irqchip_in_kernel() ? "kvm-arm-gic" : "arm_gic";
+}
+
+/**
+ * gicv3_class_name
+ *
+ * Return name of GICv3 class to use depending on whether KVM acceleration is
+ * in use. May throw an error if the chosen implementation is not available.
+ *
+ * Returns: class name to use
+ */
+const char *gicv3_class_name(void);
+
+/**
+ * kvm_arm_handle_debug:
+ * @cs: CPUState
+ * @debug_exit: debug part of the KVM exit structure
+ *
+ * Returns: TRUE if the debug exception was handled.
+ */
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit);
+
+/**
+ * kvm_arm_hw_debug_active:
+ * @cs: CPU State
+ *
+ * Return: TRUE if any hardware breakpoints in use.
+ */
+
+bool kvm_arm_hw_debug_active(CPUState *cs);
+
+/**
+ * kvm_arm_copy_hw_debug_data:
+ *
+ * @ptr: kvm_guest_debug_arch structure
+ *
+ * Copy the architecture specific debug registers into the
+ * kvm_guest_debug ioctl structure.
+ */
+struct kvm_guest_debug_arch;
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr);
+
 #endif
diff --git a/qemu/target-arm/machine.c b/qemu/target-arm/machine.c
index 32adfe792..03a73d950 100644
--- a/qemu/target-arm/machine.c
+++ b/qemu/target-arm/machine.c
@@ -1,5 +1,7 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "internals.h"
@@ -171,9 +173,7 @@ static int get_cpsr(QEMUFile *f, void *opaque, size_t size)
         return 0;
     }
 
-    /* Avoid mode switch when restoring CPSR */
-    env->uncached_cpsr = val & CPSR_M;
-    cpsr_write(env, val, 0xffffffff);
+    cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
     return 0;
 }
 
@@ -328,3 +328,20 @@ const VMStateDescription vmstate_arm_cpu = {
         NULL
     }
 };
+
+const char *gicv3_class_name(void)
+{
+    if (kvm_irqchip_in_kernel()) {
+#ifdef TARGET_AARCH64
+        return "kvm-arm-gicv3";
+#else
+        error_report("KVM GICv3 acceleration is not supported on this "
+                     "platform");
+#endif
+    } else {
+        /* TODO: Software emulation is not implemented yet */
+        error_report("KVM is currently required for GICv3 emulation");
+    }
+
+    exit(1);
+}
diff --git a/qemu/target-arm/monitor.c b/qemu/target-arm/monitor.c
new file mode 100644
index 000000000..1ee59a2e4
--- /dev/null
+++ b/qemu/target-arm/monitor.c
@@ -0,0 +1,84 @@
+/*
+ * QEMU monitor.c for ARM.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qmp-commands.h"
+#include "hw/boards.h"
+#include "kvm_arm.h"
+
+static GICCapability *gic_cap_new(int version)
+{
+    GICCapability *cap = g_new0(GICCapability, 1);
+    cap->version = version;
+    /* by default, support none */
+    cap->emulated = false;
+    cap->kernel = false;
+    return cap;
+}
+
+static GICCapabilityList *gic_cap_list_add(GICCapabilityList *head,
+                                           GICCapability *cap)
+{
+    GICCapabilityList *item = g_new0(GICCapabilityList, 1);
+    item->value = cap;
+    item->next = head;
+    return item;
+}
+
+static inline void gic_cap_kvm_probe(GICCapability *v2, GICCapability *v3)
+{
+#ifdef CONFIG_KVM
+    int fdarray[3];
+
+    if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, NULL)) {
+        return;
+    }
+
+    /* Test KVM GICv2 */
+    if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V2)) {
+        v2->kernel = true;
+    }
+
+    /* Test KVM GICv3 */
+    if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V3)) {
+        v3->kernel = true;
+    }
+
+    kvm_arm_destroy_scratch_host_vcpu(fdarray);
+#endif
+}
+
+GICCapabilityList *qmp_query_gic_capabilities(Error **errp)
+{
+    GICCapabilityList *head = NULL;
+    GICCapability *v2 = gic_cap_new(2), *v3 = gic_cap_new(3);
+
+    v2->emulated = true;
+    /* TODO: we'd change to true after we get emulated GICv3. */
+    v3->emulated = false;
+
+    gic_cap_kvm_probe(v2, v3);
+
+    head = gic_cap_list_add(head, v2);
+    head = gic_cap_list_add(head, v3);
+
+    return head;
+}
diff --git a/qemu/target-arm/neon_helper.c b/qemu/target-arm/neon_helper.c
index 47d13e908..1f1844f5b 100644
--- a/qemu/target-arm/neon_helper.c
+++ b/qemu/target-arm/neon_helper.c
@@ -6,8 +6,7 @@
  *
  * This code is licensed under the GNU GPL v2.
  */
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
diff --git a/qemu/target-arm/op_helper.c b/qemu/target-arm/op_helper.c
index 663c05d1d..d626ff1a2 100644
--- a/qemu/target-arm/op_helper.c
+++ b/qemu/target-arm/op_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "internals.h"
@@ -83,19 +84,27 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 {
     bool ret;
     uint32_t fsr = 0;
+    ARMMMUFaultInfo fi = {};
 
-    ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr);
+    ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr, &fi);
     if (unlikely(ret)) {
         ARMCPU *cpu = ARM_CPU(cs);
         CPUARMState *env = &cpu->env;
         uint32_t syn, exc;
-        bool same_el = (arm_current_el(env) != 0);
+        unsigned int target_el;
+        bool same_el;
 
         if (retaddr) {
             /* now we have a real cpu fault */
             cpu_restore_state(cs, retaddr);
         }
 
+        target_el = exception_target_el(env);
+        if (fi.stage2) {
+            target_el = 2;
+            env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
+        }
+        same_el = arm_current_el(env) == target_el;
         /* AArch64 syndrome does not have an LPAE bit */
         syn = fsr & ~(1 << 9);
 
@@ -103,10 +112,10 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
          * information; this is always true for exceptions reported to EL1.
          */
         if (is_write == 2) {
-            syn = syn_insn_abort(same_el, 0, 0, syn);
+            syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
             exc = EXCP_PREFETCH_ABORT;
         } else {
-            syn = syn_data_abort(same_el, 0, 0, 0, is_write == 1, syn);
+            syn = syn_data_abort(same_el, 0, 0, fi.s1ptw, is_write == 1, syn);
             if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
                 fsr |= (1 << 11);
             }
@@ -115,10 +124,48 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 
         env->exception.vaddress = addr;
         env->exception.fsr = fsr;
-        raise_exception(env, exc, syn, exception_target_el(env));
+        raise_exception(env, exc, syn, target_el);
+    }
+}
+
+/* Raise a data fault alignment exception for the specified virtual address */
+void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, int is_write,
+                                 int is_user, uintptr_t retaddr)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    int target_el;
+    bool same_el;
+
+    if (retaddr) {
+        /* now we have a real cpu fault */
+        cpu_restore_state(cs, retaddr);
     }
+
+    target_el = exception_target_el(env);
+    same_el = (arm_current_el(env) == target_el);
+
+    env->exception.vaddress = vaddr;
+
+    /* the DFSR for an alignment fault depends on whether we're using
+     * the LPAE long descriptor format, or the short descriptor format
+     */
+    if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
+        env->exception.fsr = 0x21;
+    } else {
+        env->exception.fsr = 0x1;
+    }
+
+    if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
+        env->exception.fsr |= (1 << 11);
+    }
+
+    raise_exception(env, EXCP_DATA_ABORT,
+                    syn_data_abort(same_el, 0, 0, 0, is_write == 1, 0x21),
+                    target_el);
 }
-#endif
+
+#endif /* !defined(CONFIG_USER_ONLY) */
 
 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 {
@@ -249,6 +296,11 @@ uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
     return res;
 }
 
+void HELPER(setend)(CPUARMState *env)
+{
+    env->uncached_cpsr ^= CPSR_E;
+}
+
 /* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
  * The function returns the target EL (1-3) if the instruction is to be trapped;
  * otherwise it returns 0 indicating it is not trapped.
@@ -375,7 +427,13 @@ uint32_t HELPER(cpsr_read)(CPUARMState *env)
 
 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 {
-    cpsr_write(env, val, mask);
+    cpsr_write(env, val, mask, CPSRWriteByInstr);
+}
+
+/* Write the CPSR for a 32-bit exception return */
+void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
+{
+    cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
 }
 
 /* Access to user mode registers from privileged modes.  */
@@ -384,9 +442,9 @@ uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
     uint32_t val;
 
     if (regno == 13) {
-        val = env->banked_r13[0];
+        val = env->banked_r13[BANK_USRSYS];
     } else if (regno == 14) {
-        val = env->banked_r14[0];
+        val = env->banked_r14[BANK_USRSYS];
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         val = env->usr_regs[regno - 8];
@@ -399,9 +457,9 @@ uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 {
     if (regno == 13) {
-        env->banked_r13[0] = val;
+        env->banked_r13[BANK_USRSYS] = val;
     } else if (regno == 14) {
-        env->banked_r14[0] = val;
+        env->banked_r14[BANK_USRSYS] = val;
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         env->usr_regs[regno - 8] = val;
@@ -410,7 +468,154 @@ void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
     }
 }
 
-void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
+void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
+{
+    if ((env->uncached_cpsr & CPSR_M) == mode) {
+        env->regs[13] = val;
+    } else {
+        env->banked_r13[bank_number(mode)] = val;
+    }
+}
+
+uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
+{
+    if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
+        /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
+         * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
+         */
+        raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                        exception_target_el(env));
+    }
+
+    if ((env->uncached_cpsr & CPSR_M) == mode) {
+        return env->regs[13];
+    } else {
+        return env->banked_r13[bank_number(mode)];
+    }
+}
+
+static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
+                                      uint32_t regno)
+{
+    /* Raise an exception if the requested access is one of the UNPREDICTABLE
+     * cases; otherwise return. This broadly corresponds to the pseudocode
+     * BankedRegisterAccessValid() and SPSRAccessValid(),
+     * except that we have already handled some cases at translate time.
+     */
+    int curmode = env->uncached_cpsr & CPSR_M;
+
+    if (curmode == tgtmode) {
+        goto undef;
+    }
+
+    if (tgtmode == ARM_CPU_MODE_USR) {
+        switch (regno) {
+        case 8 ... 12:
+            if (curmode != ARM_CPU_MODE_FIQ) {
+                goto undef;
+            }
+            break;
+        case 13:
+            if (curmode == ARM_CPU_MODE_SYS) {
+                goto undef;
+            }
+            break;
+        case 14:
+            if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
+                goto undef;
+            }
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (tgtmode == ARM_CPU_MODE_HYP) {
+        switch (regno) {
+        case 17: /* ELR_Hyp */
+            if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
+                goto undef;
+            }
+            break;
+        default:
+            if (curmode != ARM_CPU_MODE_MON) {
+                goto undef;
+            }
+            break;
+        }
+    }
+
+    return;
+
+undef:
+    raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                    exception_target_el(env));
+}
+
+void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
+                        uint32_t regno)
+{
+    msr_mrs_banked_exc_checks(env, tgtmode, regno);
+
+    switch (regno) {
+    case 16: /* SPSRs */
+        env->banked_spsr[bank_number(tgtmode)] = value;
+        break;
+    case 17: /* ELR_Hyp */
+        env->elr_el[2] = value;
+        break;
+    case 13:
+        env->banked_r13[bank_number(tgtmode)] = value;
+        break;
+    case 14:
+        env->banked_r14[bank_number(tgtmode)] = value;
+        break;
+    case 8 ... 12:
+        switch (tgtmode) {
+        case ARM_CPU_MODE_USR:
+            env->usr_regs[regno - 8] = value;
+            break;
+        case ARM_CPU_MODE_FIQ:
+            env->fiq_regs[regno - 8] = value;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
+{
+    msr_mrs_banked_exc_checks(env, tgtmode, regno);
+
+    switch (regno) {
+    case 16: /* SPSRs */
+        return env->banked_spsr[bank_number(tgtmode)];
+    case 17: /* ELR_Hyp */
+        return env->elr_el[2];
+    case 13:
+        return env->banked_r13[bank_number(tgtmode)];
+    case 14:
+        return env->banked_r14[bank_number(tgtmode)];
+    case 8 ... 12:
+        switch (tgtmode) {
+        case ARM_CPU_MODE_USR:
+            return env->usr_regs[regno - 8];
+        case ARM_CPU_MODE_FIQ:
+            return env->fiq_regs[regno - 8];
+        default:
+            g_assert_not_reached();
+        }
+    default:
+        g_assert_not_reached();
+    }
+}
+
+void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
+                                 uint32_t isread)
 {
     const ARMCPRegInfo *ri = rip;
     int target_el;
@@ -424,7 +629,7 @@ void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
         return;
     }
 
-    switch (ri->accessfn(env, ri)) {
+    switch (ri->accessfn(env, ri, isread)) {
     case CP_ACCESS_OK:
         return;
     case CP_ACCESS_TRAP:
@@ -444,6 +649,27 @@ void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
         target_el = exception_target_el(env);
         syndrome = syn_uncategorized();
         break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
+        target_el = 2;
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
+        target_el = 3;
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_FP_EL2:
+        target_el = 2;
+        /* Since we are an implementation that takes exceptions on a trapped
+         * conditional insn only if the insn has passed its condition code
+         * check, we take the IMPDEF choice to always report CV=1 COND=0xe
+         * (which is also the required value for AArch64 traps).
+         */
+        syndrome = syn_fp_access_trap(1, 0xe, false);
+        break;
+    case CP_ACCESS_TRAP_FP_EL3:
+        target_el = 3;
+        syndrome = syn_fp_access_trap(1, 0xe, false);
+        break;
     default:
         g_assert_not_reached();
     }
@@ -558,12 +784,14 @@ void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
     int cur_el = arm_current_el(env);
     bool secure = arm_is_secure(env);
     bool smd = env->cp15.scr_el3 & SCR_SMD;
-    /* On ARMv8 AArch32, SMD only applies to NS state.
-     * On ARMv7 SMD only applies to NS state and only if EL2 is available.
-     * For ARMv7 non EL2, we force SMD to zero so we don't need to re-check
-     * the EL2 condition here.
+    /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
+     * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
+     *  extensions, SMD only applies to NS state.
+     * On ARMv7 without the Virtualization extensions, the SMD bit
+     * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
+     * so we need not special case this here.
      */
-    bool undef = is_a64(env) ? smd : (!secure && smd);
+    bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 
     if (arm_is_psci_call(cpu, EXCP_SMC)) {
         /* If PSCI is enabled and this looks like a valid PSCI call then
@@ -586,12 +814,51 @@ void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
     }
 }
 
+static int el_from_spsr(uint32_t spsr)
+{
+    /* Return the exception level that this SPSR is requesting a return to,
+     * or -1 if it is invalid (an illegal return)
+     */
+    if (spsr & PSTATE_nRW) {
+        switch (spsr & CPSR_M) {
+        case ARM_CPU_MODE_USR:
+            return 0;
+        case ARM_CPU_MODE_HYP:
+            return 2;
+        case ARM_CPU_MODE_FIQ:
+        case ARM_CPU_MODE_IRQ:
+        case ARM_CPU_MODE_SVC:
+        case ARM_CPU_MODE_ABT:
+        case ARM_CPU_MODE_UND:
+        case ARM_CPU_MODE_SYS:
+            return 1;
+        case ARM_CPU_MODE_MON:
+            /* Returning to Mon from AArch64 is never possible,
+             * so this is an illegal return.
+             */
+        default:
+            return -1;
+        }
+    } else {
+        if (extract32(spsr, 1, 1)) {
+            /* Return with reserved M[1] bit set */
+            return -1;
+        }
+        if (extract32(spsr, 0, 4) == 1) {
+            /* return to EL0 with M[0] bit set */
+            return -1;
+        }
+        return extract32(spsr, 2, 2);
+    }
+}
+
 void HELPER(exception_return)(CPUARMState *env)
 {
     int cur_el = arm_current_el(env);
     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
     uint32_t spsr = env->banked_spsr[spsr_idx];
     int new_el;
+    bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 
     aarch64_save_sp(env, cur_el);
 
@@ -608,35 +875,51 @@ void HELPER(exception_return)(CPUARMState *env)
         spsr &= ~PSTATE_SS;
     }
 
-    if (spsr & PSTATE_nRW) {
-        /* TODO: We currently assume EL1/2/3 are running in AArch64.  */
+    new_el = el_from_spsr(spsr);
+    if (new_el == -1) {
+        goto illegal_return;
+    }
+    if (new_el > cur_el
+        || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
+        /* Disallow return to an EL which is unimplemented or higher
+         * than the current one.
+         */
+        goto illegal_return;
+    }
+
+    if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
+        /* Return to an EL which is configured for a different register width */
+        goto illegal_return;
+    }
+
+    if (new_el == 2 && arm_is_secure_below_el3(env)) {
+        /* Return to the non-existent secure-EL2 */
+        goto illegal_return;
+    }
+
+    if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
+        && !arm_is_secure_below_el3(env)) {
+        goto illegal_return;
+    }
+
+    if (!return_to_aa64) {
         env->aarch64 = 0;
-        new_el = 0;
-        env->uncached_cpsr = 0x10;
-        cpsr_write(env, spsr, ~0);
+        /* We do a raw CPSR write because aarch64_sync_64_to_32()
+         * will sort the register banks out for us, and we've already
+         * caught all the bad-mode cases in el_from_spsr().
+         */
+        cpsr_write(env, spsr, ~0, CPSRWriteRaw);
         if (!arm_singlestep_active(env)) {
             env->uncached_cpsr &= ~PSTATE_SS;
         }
         aarch64_sync_64_to_32(env);
 
-        env->regs[15] = env->elr_el[1] & ~0x1;
-    } else {
-        new_el = extract32(spsr, 2, 2);
-        if (new_el > cur_el
-            || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
-            /* Disallow return to an EL which is unimplemented or higher
-             * than the current one.
-             */
-            goto illegal_return;
-        }
-        if (extract32(spsr, 1, 1)) {
-            /* Return with reserved M[1] bit set */
-            goto illegal_return;
-        }
-        if (new_el == 0 && (spsr & PSTATE_SP)) {
-            /* Return to EL0 with M[0] bit set */
-            goto illegal_return;
+        if (spsr & CPSR_T) {
+            env->regs[15] = env->elr_el[cur_el] & ~0x1;
+        } else {
+            env->regs[15] = env->elr_el[cur_el] & ~0x3;
         }
+    } else {
         env->aarch64 = 1;
         pstate_write(env, spsr);
         if (!arm_singlestep_active(env)) {
@@ -859,6 +1142,25 @@ static bool check_breakpoints(ARMCPU *cpu)
     return false;
 }
 
+void HELPER(check_breakpoints)(CPUARMState *env)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    if (check_breakpoints(cpu)) {
+        HELPER(exception_internal(env, EXCP_DEBUG));
+    }
+}
+
+bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
+{
+    /* Called by core code when a CPU watchpoint fires; need to check if this
+     * is also an architectural watchpoint match.
+     */
+    ARMCPU *cpu = ARM_CPU(cs);
+
+    return check_watchpoints(cpu);
+}
+
 void arm_debug_excp_handler(CPUState *cs)
 {
     /* Called by core code when a watchpoint or breakpoint fires;
@@ -870,37 +1172,44 @@ void arm_debug_excp_handler(CPUState *cs)
 
     if (wp_hit) {
         if (wp_hit->flags & BP_CPU) {
+            bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
+            bool same_el = arm_debug_target_el(env) == arm_current_el(env);
+
             cs->watchpoint_hit = NULL;
-            if (check_watchpoints(cpu)) {
-                bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
-                bool same_el = arm_debug_target_el(env) == arm_current_el(env);
-
-                if (extended_addresses_enabled(env)) {
-                    env->exception.fsr = (1 << 9) | 0x22;
-                } else {
-                    env->exception.fsr = 0x2;
-                }
-                env->exception.vaddress = wp_hit->hitaddr;
-                raise_exception(env, EXCP_DATA_ABORT,
-                                syn_watchpoint(same_el, 0, wnr),
-                                arm_debug_target_el(env));
-            } else {
-                cpu_resume_from_signal(cs, NULL);
-            }
-        }
-    } else {
-        if (check_breakpoints(cpu)) {
-            bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
+
             if (extended_addresses_enabled(env)) {
                 env->exception.fsr = (1 << 9) | 0x22;
             } else {
                 env->exception.fsr = 0x2;
             }
-            /* FAR is UNKNOWN, so doesn't need setting */
-            raise_exception(env, EXCP_PREFETCH_ABORT,
-                            syn_breakpoint(same_el),
-                            arm_debug_target_el(env));
+            env->exception.vaddress = wp_hit->hitaddr;
+            raise_exception(env, EXCP_DATA_ABORT,
+                    syn_watchpoint(same_el, 0, wnr),
+                    arm_debug_target_el(env));
+        }
+    } else {
+        uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
+        bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
+
+        /* (1) GDB breakpoints should be handled first.
+         * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
+         * since singlestep is also done by generating a debug internal
+         * exception.
+         */
+        if (cpu_breakpoint_test(cs, pc, BP_GDB)
+            || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
+            return;
+        }
+
+        if (extended_addresses_enabled(env)) {
+            env->exception.fsr = (1 << 9) | 0x22;
+        } else {
+            env->exception.fsr = 0x2;
         }
+        /* FAR is UNKNOWN, so doesn't need setting */
+        raise_exception(env, EXCP_PREFETCH_ABORT,
+                        syn_breakpoint(same_el),
+                        arm_debug_target_el(env));
     }
 }
 
diff --git a/qemu/target-arm/psci.c b/qemu/target-arm/psci.c
index 20e4cb6f9..c55487f87 100644
--- a/qemu/target-arm/psci.c
+++ b/qemu/target-arm/psci.c
@@ -15,6 +15,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include <cpu.h>
 #include <cpu-qom.h>
 #include <exec/helper-proto.h>
diff --git a/qemu/target-arm/translate-a64.c b/qemu/target-arm/translate-a64.c
index 689f2be89..b13cff756 100644
--- a/qemu/target-arm/translate-a64.c
+++ b/qemu/target-arm/translate-a64.c
@@ -16,11 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "tcg-op.h"
@@ -30,25 +26,20 @@
 #include "internals.h"
 #include "qemu/host-utils.h"
 
+#include "exec/semihost.h"
 #include "exec/gen-icount.h"
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
+#include "exec/log.h"
 
 #include "trace-tcg.h"
 
 static TCGv_i64 cpu_X[32];
 static TCGv_i64 cpu_pc;
-static TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
 
 /* Load/store exclusive handling */
-static TCGv_i64 cpu_exclusive_addr;
-static TCGv_i64 cpu_exclusive_val;
 static TCGv_i64 cpu_exclusive_high;
-#ifdef CONFIG_USER_ONLY
-static TCGv_i64 cpu_exclusive_test;
-static TCGv_i32 cpu_exclusive_info;
-#endif
 
 static const char *regnames[] = {
     "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
@@ -95,32 +86,17 @@ void a64_translate_init(void)
 {
     int i;
 
-    cpu_pc = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i64(cpu_env,
                                     offsetof(CPUARMState, pc),
                                     "pc");
     for (i = 0; i < 32; i++) {
-        cpu_X[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_X[i] = tcg_global_mem_new_i64(cpu_env,
                                           offsetof(CPUARMState, xregs[i]),
                                           regnames[i]);
     }
 
-    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
-    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
-    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
-    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
-
-    cpu_exclusive_addr = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
-    cpu_exclusive_val = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_val), "exclusive_val");
-    cpu_exclusive_high = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_high = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_high), "exclusive_high");
-#ifdef CONFIG_USER_ONLY
-    cpu_exclusive_test = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_test), "exclusive_test");
-    cpu_exclusive_info = tcg_global_mem_new_i32(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_info), "exclusive_info");
-#endif
 }
 
 static inline ARMMMUIdx get_a64_user_mem_index(DisasContext *s)
@@ -147,6 +123,8 @@ void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
     CPUARMState *env = &cpu->env;
     uint32_t psr = pstate_read(env);
     int i;
+    int el = arm_current_el(env);
+    const char *ns_status;
 
     cpu_fprintf(f, "PC=%016"PRIx64"  SP=%016"PRIx64"\n",
             env->pc, env->xregs[31]);
@@ -158,13 +136,22 @@ void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
             cpu_fprintf(f, " ");
         }
     }
-    cpu_fprintf(f, "PSTATE=%08x (flags %c%c%c%c)\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) && el != 3) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "\nPSTATE=%08x %c%c%c%c %sEL%d%c\n",
                 psr,
                 psr & PSTATE_N ? 'N' : '-',
                 psr & PSTATE_Z ? 'Z' : '-',
                 psr & PSTATE_C ? 'C' : '-',
-                psr & PSTATE_V ? 'V' : '-');
-    cpu_fprintf(f, "\n");
+                psr & PSTATE_V ? 'V' : '-',
+                ns_status,
+                el,
+                psr & PSTATE_SP ? 'h' : 't');
 
     if (flags & CPU_DUMP_FPU) {
         int numvfpregs = 32;
@@ -188,6 +175,31 @@ void gen_a64_set_pc_im(uint64_t val)
     tcg_gen_movi_i64(cpu_pc, val);
 }
 
+typedef struct DisasCompare64 {
+    TCGCond cond;
+    TCGv_i64 value;
+} DisasCompare64;
+
+static void a64_test_cc(DisasCompare64 *c64, int cc)
+{
+    DisasCompare c32;
+
+    arm_test_cc(&c32, cc);
+
+    /* Sign-extend the 32-bit value so that the GE/LT comparisons work
+       * properly.  The NE/EQ comparisons are also fine with this choice.  */
+    c64->cond = c32.cond;
+    c64->value = tcg_temp_new_i64();
+    tcg_gen_ext_i32_i64(c64->value, c32.value);
+
+    arm_free_cc(&c32);
+}
+
+static void a64_free_cc(DisasCompare64 *c64)
+{
+    tcg_temp_free_i64(c64->value);
+}
+
 static void gen_exception_internal(int excp)
 {
     TCGv_i32 tcg_excp = tcg_const_i32(excp);
@@ -525,13 +537,8 @@ static TCGv_ptr get_fpstatus_ptr(void)
  */
 static inline void gen_set_NZ64(TCGv_i64 result)
 {
-    TCGv_i64 flag = tcg_temp_new_i64();
-
-    tcg_gen_setcondi_i64(TCG_COND_NE, flag, result, 0);
-    tcg_gen_trunc_i64_i32(cpu_ZF, flag);
-    tcg_gen_shri_i64(flag, result, 32);
-    tcg_gen_trunc_i64_i32(cpu_NF, flag);
-    tcg_temp_free_i64(flag);
+    tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
+    tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
 }
 
 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
@@ -540,8 +547,8 @@ static inline void gen_logic_CC(int sf, TCGv_i64 result)
     if (sf) {
         gen_set_NZ64(result);
     } else {
-        tcg_gen_trunc_i64_i32(cpu_ZF, result);
-        tcg_gen_trunc_i64_i32(cpu_NF, result);
+        tcg_gen_extrl_i64_i32(cpu_ZF, result);
+        tcg_gen_mov_i32(cpu_NF, cpu_ZF);
     }
     tcg_gen_movi_i32(cpu_CF, 0);
     tcg_gen_movi_i32(cpu_VF, 0);
@@ -559,7 +566,7 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_movi_i64(tmp, 0);
         tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
 
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         gen_set_NZ64(result);
 
@@ -567,8 +574,7 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
 
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(result);
@@ -580,8 +586,8 @@ static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         TCGv_i32 tmp = tcg_temp_new_i32();
 
         tcg_gen_movi_i32(tmp, 0);
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
@@ -609,15 +615,14 @@ static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         gen_set_NZ64(result);
 
         tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         tcg_gen_xor_i64(flag, result, t0);
         tmp = tcg_temp_new_i64();
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_and_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(flag);
         tcg_temp_free_i64(result);
@@ -627,8 +632,8 @@ static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         TCGv_i32 t1_32 = tcg_temp_new_i32();
         TCGv_i32 tmp;
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
@@ -670,14 +675,13 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         tcg_gen_extu_i32_i64(cf_64, cpu_CF);
         tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);
         tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);
-        tcg_gen_trunc_i64_i32(cpu_CF, cf_64);
+        tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
         gen_set_NZ64(result);
 
         tcg_gen_xor_i64(vf_64, result, t0);
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(vf_64, vf_64, tmp);
-        tcg_gen_shri_i64(vf_64, vf_64, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, vf_64);
+        tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
 
         tcg_gen_mov_i64(dest, result);
 
@@ -691,8 +695,8 @@ static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
         t1_32 = tcg_temp_new_i32();
         tmp = tcg_const_i32(0);
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);
 
@@ -719,7 +723,7 @@ static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
                              TCGv_i64 tcg_addr, int size, int memidx)
 {
     g_assert(size <= 3);
-    tcg_gen_qemu_st_i64(source, tcg_addr, memidx, MO_TE + size);
+    tcg_gen_qemu_st_i64(source, tcg_addr, memidx, s->be_data + size);
 }
 
 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
@@ -734,7 +738,7 @@ static void do_gpr_st(DisasContext *s, TCGv_i64 source,
 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
                              int size, bool is_signed, bool extend, int memidx)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
 
     g_assert(size <= 3);
 
@@ -766,13 +770,18 @@ static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, int size)
     TCGv_i64 tmp = tcg_temp_new_i64();
     tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64));
     if (size < 4) {
-        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TE + size);
+        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s),
+                            s->be_data + size);
     } else {
+        bool be = s->be_data == MO_BE;
         TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();
-        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TEQ);
-        tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
+
         tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
-        tcg_gen_qemu_st_i64(tmp, tcg_hiaddr, get_mem_index(s), MO_TEQ);
+        tcg_gen_qemu_st_i64(tmp, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
+                            s->be_data | MO_Q);
+        tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
+        tcg_gen_qemu_st_i64(tmp, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
+                            s->be_data | MO_Q);
         tcg_temp_free_i64(tcg_hiaddr);
     }
 
@@ -789,17 +798,21 @@ static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, int size)
     TCGv_i64 tmphi;
 
     if (size < 4) {
-        TCGMemOp memop = MO_TE + size;
+        TCGMemOp memop = s->be_data + size;
         tmphi = tcg_const_i64(0);
         tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop);
     } else {
+        bool be = s->be_data == MO_BE;
         TCGv_i64 tcg_hiaddr;
+
         tmphi = tcg_temp_new_i64();
         tcg_hiaddr = tcg_temp_new_i64();
 
-        tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), MO_TEQ);
         tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
-        tcg_gen_qemu_ld_i64(tmphi, tcg_hiaddr, get_mem_index(s), MO_TEQ);
+        tcg_gen_qemu_ld_i64(tmplo, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
+                            s->be_data | MO_Q);
+        tcg_gen_qemu_ld_i64(tmphi, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
+                            s->be_data | MO_Q);
         tcg_temp_free_i64(tcg_hiaddr);
     }
 
@@ -938,7 +951,7 @@ static void clear_vec_high(DisasContext *s, int rd)
 static void do_vec_st(DisasContext *s, int srcidx, int element,
                       TCGv_i64 tcg_addr, int size)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
     TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
     read_vec_element(s, tcg_tmp, srcidx, element, size);
@@ -951,7 +964,7 @@ static void do_vec_st(DisasContext *s, int srcidx, int element,
 static void do_vec_ld(DisasContext *s, int destidx, int element,
                       TCGv_i64 tcg_addr, int size)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
     TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
     tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), memop);
@@ -1234,9 +1247,15 @@ static void handle_sync(DisasContext *s, uint32_t insn,
         return;
     case 4: /* DSB */
     case 5: /* DMB */
-    case 6: /* ISB */
         /* We don't emulate caches so barriers are no-ops */
         return;
+    case 6: /* ISB */
+        /* We need to break the TB after this insn to execute
+         * a self-modified code correctly and also to take
+         * any pending interrupts immediately.
+         */
+        s->is_jmp = DISAS_UPDATE;
+        return;
     default:
         unallocated_encoding(s);
         return;
@@ -1301,7 +1320,7 @@ static void gen_set_nzcv(TCGv_i64 tcg_rt)
     TCGv_i32 nzcv = tcg_temp_new_i32();
 
     /* take NZCV from R[t] */
-    tcg_gen_trunc_i64_i32(nzcv, tcg_rt);
+    tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
 
     /* bit 31, N */
     tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
@@ -1357,16 +1376,18 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
          * runtime; this may result in an exception.
          */
         TCGv_ptr tmpptr;
-        TCGv_i32 tcg_syn;
+        TCGv_i32 tcg_syn, tcg_isread;
         uint32_t syndrome;
 
         gen_a64_set_pc_im(s->pc - 4);
         tmpptr = tcg_const_ptr(ri);
         syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
         tcg_syn = tcg_const_i32(syndrome);
-        gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn);
+        tcg_isread = tcg_const_i32(isread);
+        gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn, tcg_isread);
         tcg_temp_free_ptr(tmpptr);
         tcg_temp_free_i32(tcg_syn);
+        tcg_temp_free_i32(tcg_isread);
     }
 
     /* Handle special cases first */
@@ -1553,8 +1574,27 @@ static void disas_exc(DisasContext *s, uint32_t insn)
             unallocated_encoding(s);
             break;
         }
-        /* HLT */
-        unsupported_encoding(s, insn);
+        /* HLT. This has two purposes.
+         * Architecturally, it is an external halting debug instruction.
+         * Since QEMU doesn't implement external debug, we treat this as
+         * it is required for halting debug disabled: it will UNDEF.
+         * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
+         */
+        if (semihosting_enabled() && imm16 == 0xf000) {
+#ifndef CONFIG_USER_ONLY
+            /* In system mode, don't allow userspace access to semihosting,
+             * to provide some semblance of security (and for consistency
+             * with our 32-bit semihosting).
+             */
+            if (s->current_el == 0) {
+                unsupported_encoding(s, insn);
+                break;
+            }
+#endif
+            gen_exception_internal_insn(s, 0, EXCP_SEMIHOST);
+        } else {
+            unsupported_encoding(s, insn);
+        }
         break;
     case 5:
         if (op2_ll < 1 || op2_ll > 3) {
@@ -1671,7 +1711,7 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
                                TCGv_i64 addr, int size, bool is_pair)
 {
     TCGv_i64 tmp = tcg_temp_new_i64();
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
 
     g_assert(size <= 3);
     tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), memop);
@@ -1733,7 +1773,7 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
     tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label);
 
     tmp = tcg_temp_new_i64();
-    tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), MO_TE + size);
+    tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), s->be_data + size);
     tcg_gen_brcond_i64(TCG_COND_NE, tmp, cpu_exclusive_val, fail_label);
     tcg_temp_free_i64(tmp);
 
@@ -1742,7 +1782,8 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
         TCGv_i64 tmphi = tcg_temp_new_i64();
 
         tcg_gen_addi_i64(addrhi, addr, 1 << size);
-        tcg_gen_qemu_ld_i64(tmphi, addrhi, get_mem_index(s), MO_TE + size);
+        tcg_gen_qemu_ld_i64(tmphi, addrhi, get_mem_index(s),
+                            s->be_data + size);
         tcg_gen_brcond_i64(TCG_COND_NE, tmphi, cpu_exclusive_high, fail_label);
 
         tcg_temp_free_i64(tmphi);
@@ -1750,13 +1791,14 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
     }
 
     /* We seem to still have the exclusive monitor, so do the store */
-    tcg_gen_qemu_st_i64(cpu_reg(s, rt), addr, get_mem_index(s), MO_TE + size);
+    tcg_gen_qemu_st_i64(cpu_reg(s, rt), addr, get_mem_index(s),
+                        s->be_data + size);
     if (is_pair) {
         TCGv_i64 addrhi = tcg_temp_new_i64();
 
         tcg_gen_addi_i64(addrhi, addr, 1 << size);
         tcg_gen_qemu_st_i64(cpu_reg(s, rt2), addrhi,
-                            get_mem_index(s), MO_TE + size);
+                            get_mem_index(s), s->be_data + size);
         tcg_temp_free_i64(addrhi);
     }
 
@@ -1784,9 +1826,6 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
  *  o2: 0 -> exclusive, 1 -> not
  *  o1: 0 -> single register, 1 -> register pair
  *  o0: 1 -> load-acquire/store-release, 0 -> not
- *
- *  o0 == 0 AND o2 == 1 is un-allocated
- *  o1 == 1 is un-allocated except for 32 and 64 bit sizes
  */
 static void disas_ldst_excl(DisasContext *s, uint32_t insn)
 {
@@ -1801,7 +1840,8 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
     int size = extract32(insn, 30, 2);
     TCGv_i64 tcg_addr;
 
-    if ((!is_excl && !is_lasr) ||
+    if ((!is_excl && !is_pair && !is_lasr) ||
+        (!is_excl && is_pair) ||
         (is_pair && size < 2)) {
         unallocated_encoding(s);
         return;
@@ -1830,15 +1870,6 @@ static void disas_ldst_excl(DisasContext *s, uint32_t insn)
         } else {
             do_gpr_ld(s, tcg_rt, tcg_addr, size, false, false);
         }
-        if (is_pair) {
-            TCGv_i64 tcg_rt2 = cpu_reg(s, rt);
-            tcg_gen_addi_i64(tcg_addr, tcg_addr, 1 << size);
-            if (is_store) {
-                do_gpr_st(s, tcg_rt2, tcg_addr, size);
-            } else {
-                do_gpr_ld(s, tcg_rt2, tcg_addr, size, false, false);
-            }
-        }
     }
 }
 
@@ -2582,7 +2613,7 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
             TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
             tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr,
-                                get_mem_index(s), MO_TE + scale);
+                                get_mem_index(s), s->be_data + scale);
             switch (scale) {
             case 0:
                 mulconst = 0x0101010101010101ULL;
@@ -2612,9 +2643,9 @@ static void disas_ldst_single_struct(DisasContext *s, uint32_t insn)
         } else {
             /* Load/store one element per register */
             if (is_load) {
-                do_vec_ld(s, rt, index, tcg_addr, MO_TE + scale);
+                do_vec_ld(s, rt, index, tcg_addr, s->be_data + scale);
             } else {
-                do_vec_st(s, rt, index, tcg_addr, MO_TE + scale);
+                do_vec_st(s, rt, index, tcg_addr, s->be_data + scale);
             }
         }
         tcg_gen_addi_i64(tcg_addr, tcg_addr, ebytes);
@@ -2992,9 +3023,51 @@ static void disas_bitfield(DisasContext *s, uint32_t insn)
     }
 
     tcg_rd = cpu_reg(s, rd);
-    tcg_tmp = read_cpu_reg(s, rn, sf);
 
-    /* OPTME: probably worth recognizing common cases of ext{8,16,32}{u,s} */
+    /* Suppress the zero-extend for !sf.  Since RI and SI are constrained
+       to be smaller than bitsize, we'll never reference data outside the
+       low 32-bits anyway.  */
+    tcg_tmp = read_cpu_reg(s, rn, 1);
+
+    /* Recognize the common aliases.  */
+    if (opc == 0) { /* SBFM */
+        if (ri == 0) {
+            if (si == 7) { /* SXTB */
+                tcg_gen_ext8s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 15) { /* SXTH */
+                tcg_gen_ext16s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 31) { /* SXTW */
+                tcg_gen_ext32s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            }
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* ASR */
+            if (si == 31) {
+                tcg_gen_ext32s_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_sari_i64(tcg_rd, tcg_tmp, ri);
+            goto done;
+        }
+    } else if (opc == 2) { /* UBFM */
+        if (ri == 0) { /* UXTB, UXTH, plus non-canonical AND */
+            tcg_gen_andi_i64(tcg_rd, tcg_tmp, bitmask64(si + 1));
+            return;
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* LSR */
+            if (si == 31) {
+                tcg_gen_ext32u_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_shri_i64(tcg_rd, tcg_tmp, ri);
+            return;
+        }
+        if (si + 1 == ri && si != bitsize - 1) { /* LSL */
+            int shift = bitsize - 1 - si;
+            tcg_gen_shli_i64(tcg_rd, tcg_tmp, shift);
+            goto done;
+        }
+    }
 
     if (opc != 1) { /* SBFM or UBFM */
         tcg_gen_movi_i64(tcg_rd, 0);
@@ -3019,6 +3092,7 @@ static void disas_bitfield(DisasContext *s, uint32_t insn)
         tcg_gen_sari_i64(tcg_rd, tcg_rd, 64 - (pos + len));
     }
 
+ done:
     if (!sf) { /* zero extend final result */
         tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
@@ -3051,17 +3125,7 @@ static void disas_extract(DisasContext *s, uint32_t insn)
 
         tcg_rd = cpu_reg(s, rd);
 
-        if (imm) {
-            /* OPTME: we can special case rm==rn as a rotate */
-            tcg_rm = read_cpu_reg(s, rm, sf);
-            tcg_rn = read_cpu_reg(s, rn, sf);
-            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
-            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
-            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
-            if (!sf) {
-                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
-            }
-        } else {
+        if (unlikely(imm == 0)) {
             /* tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
              * so an extract from bit 0 is a special case.
              */
@@ -3070,8 +3134,27 @@ static void disas_extract(DisasContext *s, uint32_t insn)
             } else {
                 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, rm));
             }
+        } else if (rm == rn) { /* ROR */
+            tcg_rm = cpu_reg(s, rm);
+            if (sf) {
+                tcg_gen_rotri_i64(tcg_rd, tcg_rm, imm);
+            } else {
+                TCGv_i32 tmp = tcg_temp_new_i32();
+                tcg_gen_extrl_i64_i32(tmp, tcg_rm);
+                tcg_gen_rotri_i32(tmp, tmp, imm);
+                tcg_gen_extu_i32_i64(tcg_rd, tmp);
+                tcg_temp_free_i32(tmp);
+            }
+        } else {
+            tcg_rm = read_cpu_reg(s, rm, sf);
+            tcg_rn = read_cpu_reg(s, rn, sf);
+            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
+            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
+            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
+            if (!sf) {
+                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
+            }
         }
-
     }
 }
 
@@ -3131,8 +3214,8 @@ static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
             TCGv_i32 t0, t1;
             t0 = tcg_temp_new_i32();
             t1 = tcg_temp_new_i32();
-            tcg_gen_trunc_i64_i32(t0, src);
-            tcg_gen_trunc_i64_i32(t1, shift_amount);
+            tcg_gen_extrl_i64_i32(t0, src);
+            tcg_gen_extrl_i64_i32(t1, shift_amount);
             tcg_gen_rotr_i32(t0, t0, t1);
             tcg_gen_extu_i32_i64(dst, t0);
             tcg_temp_free_i32(t0);
@@ -3547,8 +3630,9 @@ static void disas_adc_sbc(DisasContext *s, uint32_t insn)
 static void disas_cc(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, op, y, cond, rn, nzcv, is_imm;
-    TCGLabel *label_continue = NULL;
+    TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
     TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
+    DisasCompare c;
 
     if (!extract32(insn, 29, 1)) {
         unallocated_encoding(s);
@@ -3566,19 +3650,13 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     rn = extract32(insn, 5, 5);
     nzcv = extract32(insn, 0, 4);
 
-    if (cond < 0x0e) { /* not always */
-        TCGLabel *label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_tmp = tcg_temp_new_i64();
-        tcg_gen_movi_i64(tcg_tmp, nzcv << 28);
-        gen_set_nzcv(tcg_tmp);
-        tcg_temp_free_i64(tcg_tmp);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
-    /* match, or condition is always */
+    /* Set T0 = !COND.  */
+    tcg_t0 = tcg_temp_new_i32();
+    arm_test_cc(&c, cond);
+    tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
+    arm_free_cc(&c);
+
+    /* Load the arguments for the new comparison.  */
     if (is_imm) {
         tcg_y = new_tmp_a64(s);
         tcg_gen_movi_i64(tcg_y, y);
@@ -3587,6 +3665,7 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     }
     tcg_rn = cpu_reg(s, rn);
 
+    /* Set the flags for the new comparison.  */
     tcg_tmp = tcg_temp_new_i64();
     if (op) {
         gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
@@ -3595,9 +3674,55 @@ static void disas_cc(DisasContext *s, uint32_t insn)
     }
     tcg_temp_free_i64(tcg_tmp);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
+    /* If COND was false, force the flags to #nzcv.  Compute two masks
+     * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
+     * For tcg hosts that support ANDC, we can make do with just T1.
+     * In either case, allow the tcg optimizer to delete any unused mask.
+     */
+    tcg_t1 = tcg_temp_new_i32();
+    tcg_t2 = tcg_temp_new_i32();
+    tcg_gen_neg_i32(tcg_t1, tcg_t0);
+    tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
+
+    if (nzcv & 8) { /* N */
+        tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
+        }
     }
+    if (nzcv & 4) { /* Z */
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
+        }
+    } else {
+        tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
+    }
+    if (nzcv & 2) { /* C */
+        tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
+        }
+    }
+    if (nzcv & 1) { /* V */
+        tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
+        }
+    }
+    tcg_temp_free_i32(tcg_t0);
+    tcg_temp_free_i32(tcg_t1);
+    tcg_temp_free_i32(tcg_t2);
 }
 
 /* C3.5.6 Conditional select
@@ -3609,7 +3734,8 @@ static void disas_cc(DisasContext *s, uint32_t insn)
 static void disas_cond_select(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
-    TCGv_i64 tcg_rd, tcg_src;
+    TCGv_i64 tcg_rd, zero;
+    DisasCompare64 c;
 
     if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
         /* S == 1 or op2<1> == 1 */
@@ -3624,48 +3750,35 @@ static void disas_cond_select(DisasContext *s, uint32_t insn)
     rn = extract32(insn, 5, 5);
     rd = extract32(insn, 0, 5);
 
-    if (rd == 31) {
-        /* silly no-op write; until we use movcond we must special-case
-         * this to avoid a dead temporary across basic blocks.
-         */
-        return;
-    }
-
     tcg_rd = cpu_reg(s, rd);
 
-    if (cond >= 0x0e) { /* condition "always" */
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
-    } else {
-        /* OPTME: we could use movcond here, at the cost of duplicating
-         * a lot of the arm_gen_test_cc() logic.
-         */
-        TCGLabel *label_match = gen_new_label();
-        TCGLabel *label_continue = gen_new_label();
-
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_src = cpu_reg(s, rm);
+    a64_test_cc(&c, cond);
+    zero = tcg_const_i64(0);
 
+    if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
+        /* CSET & CSETM.  */
+        tcg_gen_setcond_i64(tcg_invert_cond(c.cond), tcg_rd, c.value, zero);
+        if (else_inv) {
+            tcg_gen_neg_i64(tcg_rd, tcg_rd);
+        }
+    } else {
+        TCGv_i64 t_true = cpu_reg(s, rn);
+        TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
         if (else_inv && else_inc) {
-            tcg_gen_neg_i64(tcg_rd, tcg_src);
+            tcg_gen_neg_i64(t_false, t_false);
         } else if (else_inv) {
-            tcg_gen_not_i64(tcg_rd, tcg_src);
+            tcg_gen_not_i64(t_false, t_false);
         } else if (else_inc) {
-            tcg_gen_addi_i64(tcg_rd, tcg_src, 1);
-        } else {
-            tcg_gen_mov_i64(tcg_rd, tcg_src);
-        }
-        if (!sf) {
-            tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
+            tcg_gen_addi_i64(t_false, t_false, 1);
         }
-        tcg_gen_br(label_continue);
-        /* match: */
-        gen_set_label(label_match);
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
-        /* continue: */
-        gen_set_label(label_continue);
+        tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
+    }
+
+    tcg_temp_free_i64(zero);
+    a64_free_cc(&c);
+
+    if (!sf) {
+        tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
 }
 
@@ -3680,7 +3793,7 @@ static void handle_clz(DisasContext *s, unsigned int sf,
         gen_helper_clz64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_clz(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3698,7 +3811,7 @@ static void handle_cls(DisasContext *s, unsigned int sf,
         gen_helper_cls64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_cls32(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3716,7 +3829,7 @@ static void handle_rbit(DisasContext *s, unsigned int sf,
         gen_helper_rbit64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_rbit(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -4152,20 +4265,6 @@ static void disas_fp_ccomp(DisasContext *s, uint32_t insn)
     }
 }
 
-/* copy src FP register to dst FP register; type specifies single or double */
-static void gen_mov_fp2fp(DisasContext *s, int type, int dst, int src)
-{
-    if (type) {
-        TCGv_i64 v = read_fp_dreg(s, src);
-        write_fp_dreg(s, dst, v);
-        tcg_temp_free_i64(v);
-    } else {
-        TCGv_i32 v = read_fp_sreg(s, src);
-        write_fp_sreg(s, dst, v);
-        tcg_temp_free_i32(v);
-    }
-}
-
 /* C3.6.24 Floating point conditional select
  *   31  30  29 28       24 23  22  21 20  16 15  12 11 10 9    5 4    0
  * +---+---+---+-----------+------+---+------+------+-----+------+------+
@@ -4175,7 +4274,8 @@ static void gen_mov_fp2fp(DisasContext *s, int type, int dst, int src)
 static void disas_fp_csel(DisasContext *s, uint32_t insn)
 {
     unsigned int mos, type, rm, cond, rn, rd;
-    TCGLabel *label_continue = NULL;
+    TCGv_i64 t_true, t_false, t_zero;
+    DisasCompare64 c;
 
     mos = extract32(insn, 29, 3);
     type = extract32(insn, 22, 2); /* 0 = single, 1 = double */
@@ -4193,21 +4293,23 @@ static void disas_fp_csel(DisasContext *s, uint32_t insn)
         return;
     }
 
-    if (cond < 0x0e) { /* not always */
-        TCGLabel *label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        gen_mov_fp2fp(s, type, rd, rm);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
+    /* Zero extend sreg inputs to 64 bits now.  */
+    t_true = tcg_temp_new_i64();
+    t_false = tcg_temp_new_i64();
+    read_vec_element(s, t_true, rn, 0, type ? MO_64 : MO_32);
+    read_vec_element(s, t_false, rm, 0, type ? MO_64 : MO_32);
 
-    gen_mov_fp2fp(s, type, rd, rn);
+    a64_test_cc(&c, cond);
+    t_zero = tcg_const_i64(0);
+    tcg_gen_movcond_i64(c.cond, t_true, c.value, t_zero, t_true, t_false);
+    tcg_temp_free_i64(t_zero);
+    tcg_temp_free_i64(t_false);
+    a64_free_cc(&c);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
-    }
+    /* Note that sregs write back zeros to the high bits,
+       and we've already done the zero-extension.  */
+    write_fp_dreg(s, rd, t_true);
+    tcg_temp_free_i64(t_true);
 }
 
 /* C3.6.25 Floating-point data-processing (1 source) - single precision */
@@ -5475,16 +5577,16 @@ static void disas_simd_across_lanes(DisasContext *s, uint32_t insn)
         assert(elements == 4);
 
         read_vec_element(s, tcg_elt, rn, 0, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt1, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt1, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 1, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
 
         do_minmaxop(s, tcg_elt1, tcg_elt2, opcode, is_min, fpst);
 
         read_vec_element(s, tcg_elt, rn, 2, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 3, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt3, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt3, tcg_elt);
 
         do_minmaxop(s, tcg_elt2, tcg_elt3, opcode, is_min, fpst);
 
@@ -7647,7 +7749,7 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar,
             static NeonGenNarrowFn * const xtnfns[3] = {
                 gen_helper_neon_narrow_u8,
                 gen_helper_neon_narrow_u16,
-                tcg_gen_trunc_i64_i32,
+                tcg_gen_extrl_i64_i32,
             };
             static NeonGenNarrowEnvFn * const sqxtunfns[3] = {
                 gen_helper_neon_unarrow_sat8,
@@ -7681,10 +7783,8 @@ static void handle_2misc_narrow(DisasContext *s, bool scalar,
             } else {
                 TCGv_i32 tcg_lo = tcg_temp_new_i32();
                 TCGv_i32 tcg_hi = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tcg_lo, tcg_op);
+                tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, cpu_env);
-                tcg_gen_shri_i64(tcg_op, tcg_op, 32);
-                tcg_gen_trunc_i64_i32(tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, cpu_env);
                 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16);
                 tcg_temp_free_i32(tcg_lo);
@@ -8590,16 +8690,10 @@ static void handle_3rd_wide(DisasContext *s, int is_q, int is_u, int size,
     }
 }
 
-static void do_narrow_high_u32(TCGv_i32 res, TCGv_i64 in)
-{
-    tcg_gen_shri_i64(in, in, 32);
-    tcg_gen_trunc_i64_i32(res, in);
-}
-
 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in)
 {
     tcg_gen_addi_i64(in, in, 1U << 31);
-    do_narrow_high_u32(res, in);
+    tcg_gen_extrh_i64_i32(res, in);
 }
 
 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
@@ -8618,7 +8712,7 @@ static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
               gen_helper_neon_narrow_round_high_u8 },
             { gen_helper_neon_narrow_high_u16,
               gen_helper_neon_narrow_round_high_u16 },
-            { do_narrow_high_u32, do_narrow_round_high_u32 },
+            { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 },
         };
         NeonGenNarrowFn *gennarrow = narrowfns[size][is_u];
 
@@ -10883,7 +10977,7 @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s)
 {
     uint32_t insn;
 
-    insn = arm_ldl_code(env, s->pc, s->bswap_code);
+    insn = arm_ldl_code(env, s->pc, s->sctlr_b);
     s->insn = insn;
     s->pc += 4;
 
@@ -10922,15 +11016,11 @@ static void disas_a64_insn(CPUARMState *env, DisasContext *s)
     free_tmp_a64(s);
 }
 
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc)
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
     CPUState *cs = CPU(cpu);
     CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
@@ -10946,9 +11036,14 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     dc->condjmp = 0;
 
     dc->aarch64 = 1;
-    dc->el3_is_aa64 = arm_el_is_aa64(env, 3);
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = 0;
-    dc->bswap_code = 0;
+    dc->sctlr_b = 0;
+    dc->be_data = ARM_TBFLAG_BE_DATA(tb->flags) ? MO_BE : MO_LE;
     dc->condexec_mask = 0;
     dc->condexec_cond = 0;
     dc->mmu_idx = ARM_TBFLAG_MMUIDX(tb->flags);
@@ -10985,51 +11080,51 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
     init_tmp_a64_array(dc);
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     tcg_clear_temp_count();
 
     do {
+        tcg_gen_insn_start(dc->pc, 0);
+        num_insns++;
+
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
-                }
-            }
-        }
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
+                    if (bp->flags & BP_CPU) {
+                        gen_a64_set_pc_im(dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it likely won't be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        dc->pc += 4;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
-        }
-
         if (dc->ss_active && !dc->pstate_ss) {
             /* Singlestep state is Active-pending.
              * If we're in this state at the start of a TB then either
@@ -11041,7 +11136,7 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
+            assert(num_insns == 1);
             gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
                           default_exception_el(dc));
             dc->is_jmp = DISAS_EXC;
@@ -11060,7 +11155,6 @@ void gen_intermediate_code_internal_a64(ARMCPU *cpu,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.
          */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -11131,22 +11225,15 @@ done_generating:
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) &&
+        qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
         log_target_disas(cs, pc_start, dc->pc - pc_start,
-                         4 | (dc->bswap_code << 1));
+                         4 | (bswap_code(dc->sctlr_b) ? 2 : 0));
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
diff --git a/qemu/target-arm/translate.c b/qemu/target-arm/translate.c
index 69ac18c10..940ec8d98 100644
--- a/qemu/target-arm/translate.c
+++ b/qemu/target-arm/translate.c
@@ -18,11 +18,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "internals.h"
@@ -36,6 +32,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define ENABLE_ARCH_4T    arm_dc_feature(s, ARM_FEATURE_V4T)
@@ -52,7 +49,6 @@
 #define ARCH(x) do { if (!ENABLE_ARCH_##x) goto illegal_op; } while(0)
 
 #include "translate.h"
-static uint32_t gen_opc_condexec_bits[OPC_BUF_SIZE];
 
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(s) 1
@@ -60,16 +56,16 @@ static uint32_t gen_opc_condexec_bits[OPC_BUF_SIZE];
 #define IS_USER(s) (s->user)
 #endif
 
-TCGv_ptr cpu_env;
+TCGv_env cpu_env;
 /* We reuse the same 64-bit temporaries for efficiency.  */
 static TCGv_i64 cpu_V0, cpu_V1, cpu_M0;
 static TCGv_i32 cpu_R[16];
-static TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
-static TCGv_i64 cpu_exclusive_addr;
-static TCGv_i64 cpu_exclusive_val;
+TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
+TCGv_i64 cpu_exclusive_addr;
+TCGv_i64 cpu_exclusive_val;
 #ifdef CONFIG_USER_ONLY
-static TCGv_i64 cpu_exclusive_test;
-static TCGv_i32 cpu_exclusive_info;
+TCGv_i64 cpu_exclusive_test;
+TCGv_i32 cpu_exclusive_info;
 #endif
 
 /* FIXME:  These should be removed.  */
@@ -90,23 +86,23 @@ void arm_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                                           offsetof(CPUARMState, regs[i]),
                                           regnames[i]);
     }
-    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
-    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
-    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
-    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
+    cpu_CF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, CF), "CF");
+    cpu_NF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, NF), "NF");
+    cpu_VF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, VF), "VF");
+    cpu_ZF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, ZF), "ZF");
 
-    cpu_exclusive_addr = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_addr = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
-    cpu_exclusive_val = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_val = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_val), "exclusive_val");
 #ifdef CONFIG_USER_ONLY
-    cpu_exclusive_test = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_test = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_test), "exclusive_test");
-    cpu_exclusive_info = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_exclusive_info = tcg_global_mem_new_i32(cpu_env,
         offsetof(CPUARMState, exclusive_info), "exclusive_info");
 #endif
 
@@ -738,81 +734,113 @@ static void gen_thumb2_parallel_addsub(int op1, int op2, TCGv_i32 a, TCGv_i32 b)
 #undef PAS_OP
 
 /*
- * generate a conditional branch based on ARM condition code cc.
+ * Generate a conditional based on ARM condition code cc.
  * This is common between ARM and Aarch64 targets.
  */
-void arm_gen_test_cc(int cc, TCGLabel *label)
+void arm_test_cc(DisasCompare *cmp, int cc)
 {
-    TCGv_i32 tmp;
-    TCGLabel *inv;
+    TCGv_i32 value;
+    TCGCond cond;
+    bool global = true;
 
     switch (cc) {
     case 0: /* eq: Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        break;
     case 1: /* ne: !Z */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
+        cond = TCG_COND_EQ;
+        value = cpu_ZF;
         break;
+
     case 2: /* cs: C */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_CF, 0, label);
-        break;
     case 3: /* cc: !C */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
+        cond = TCG_COND_NE;
+        value = cpu_CF;
         break;
+
     case 4: /* mi: N */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_NF, 0, label);
-        break;
     case 5: /* pl: !N */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_NF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_NF;
         break;
+
     case 6: /* vs: V */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_VF, 0, label);
-        break;
     case 7: /* vc: !V */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_VF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_VF;
         break;
+
     case 8: /* hi: C && !Z */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, inv);
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
-        gen_set_label(inv);
-        break;
-    case 9: /* ls: !C || Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
+    case 9: /* ls: !C || Z -> !(C && !Z) */
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* CF is 1 for C, so -CF is an all-bits-set mask for C;
+           ZF is non-zero for !Z; so AND the two subexpressions.  */
+        tcg_gen_neg_i32(value, cpu_CF);
+        tcg_gen_and_i32(value, value, cpu_ZF);
         break;
+
     case 10: /* ge: N == V -> N ^ V == 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        break;
     case 11: /* lt: N != V -> N ^ V != 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        /* Since we're only interested in the sign bit, == 0 is >= 0.  */
+        cond = TCG_COND_GE;
+        value = tcg_temp_new_i32();
+        global = false;
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
         break;
+
     case 12: /* gt: !Z && N == V */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, inv);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        gen_set_label(inv);
-        break;
     case 13: /* le: Z || N != V */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* (N == V) is equal to the sign bit of ~(NF ^ VF).  Propagate
+         * the sign bit then AND with ZF to yield the result.  */
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
+        tcg_gen_sari_i32(value, value, 31);
+        tcg_gen_andc_i32(value, cpu_ZF, value);
         break;
+
+    case 14: /* always */
+    case 15: /* always */
+        /* Use the ALWAYS condition, which will fold early.
+         * It doesn't matter what we use for the value.  */
+        cond = TCG_COND_ALWAYS;
+        value = cpu_ZF;
+        goto no_invert;
+
     default:
         fprintf(stderr, "Bad condition code 0x%x\n", cc);
         abort();
     }
+
+    if (cc & 1) {
+        cond = tcg_invert_cond(cond);
+    }
+
+ no_invert:
+    cmp->cond = cond;
+    cmp->value = value;
+    cmp->value_global = global;
+}
+
+void arm_free_cc(DisasCompare *cmp)
+{
+    if (!cmp->value_global) {
+        tcg_temp_free_i32(cmp->value);
+    }
+}
+
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label)
+{
+    tcg_gen_brcondi_i32(cmp->cond, cmp->value, 0, label);
+}
+
+void arm_gen_test_cc(int cc, TCGLabel *label)
+{
+    DisasCompare cmp;
+    arm_test_cc(&cmp, cc);
+    arm_jump_cc(&cmp, label);
+    arm_free_cc(&cmp);
 }
 
 static const uint8_t table_logic_cc[16] = {
@@ -839,7 +867,7 @@ static inline void gen_bx_im(DisasContext *s, uint32_t addr)
 {
     TCGv_i32 tmp;
 
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     if (s->thumb != (addr & 1)) {
         tmp = tcg_temp_new_i32();
         tcg_gen_movi_i32(tmp, addr & 1);
@@ -852,7 +880,7 @@ static inline void gen_bx_im(DisasContext *s, uint32_t addr)
 /* Set PC and Thumb state from var.  var is marked as dead.  */
 static inline void gen_bx(DisasContext *s, TCGv_i32 var)
 {
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     tcg_gen_andi_i32(cpu_R[15], var, ~1);
     tcg_gen_andi_i32(var, var, 1);
     store_cpu_field(var, thumb);
@@ -883,6 +911,12 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var)
     }
 }
 
+#ifdef CONFIG_USER_ONLY
+#define IS_USER_ONLY 1
+#else
+#define IS_USER_ONLY 0
+#endif
+
 /* Abstractions of "generate code to do a guest load/store for
  * AArch32", where a vaddr is always 32 bits (and is zero
  * extended if we're a 64 bit core) and  data is also
@@ -892,74 +926,143 @@ static inline void store_reg_from_load(DisasContext *s, int reg, TCGv_i32 var)
  */
 #if TARGET_LONG_BITS == 32
 
-#define DO_GEN_LD(SUFF, OPC)                                             \
-static inline void gen_aa32_ld##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_LD(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
-    tcg_gen_qemu_ld_i32(val, addr, index, OPC);                          \
-}
-
-#define DO_GEN_ST(SUFF, OPC)                                             \
-static inline void gen_aa32_st##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        TCGv addr_be = tcg_temp_new();                                   \
+        tcg_gen_xori_i32(addr_be, addr, BE32_XOR);                       \
+        tcg_gen_qemu_ld_i32(val, addr_be, index, opc);                   \
+        tcg_temp_free(addr_be);                                          \
+        return;                                                          \
+    }                                                                    \
+    tcg_gen_qemu_ld_i32(val, addr, index, opc);                          \
+}
+
+#define DO_GEN_ST(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
-    tcg_gen_qemu_st_i32(val, addr, index, OPC);                          \
-}
-
-static inline void gen_aa32_ld64(TCGv_i64 val, TCGv_i32 addr, int index)
-{
-    tcg_gen_qemu_ld_i64(val, addr, index, MO_TEQ);
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        TCGv addr_be = tcg_temp_new();                                   \
+        tcg_gen_xori_i32(addr_be, addr, BE32_XOR);                       \
+        tcg_gen_qemu_st_i32(val, addr_be, index, opc);                   \
+        tcg_temp_free(addr_be);                                          \
+        return;                                                          \
+    }                                                                    \
+    tcg_gen_qemu_st_i32(val, addr, index, opc);                          \
+}
+
+static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
+{
+    TCGMemOp opc = MO_Q | s->be_data;
+    tcg_gen_qemu_ld_i64(val, addr, index, opc);
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        tcg_gen_rotri_i64(val, val, 32);
+    }
 }
 
-static inline void gen_aa32_st64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
-    tcg_gen_qemu_st_i64(val, addr, index, MO_TEQ);
+    TCGMemOp opc = MO_Q | s->be_data;
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        TCGv_i64 tmp = tcg_temp_new_i64();
+        tcg_gen_rotri_i64(tmp, val, 32);
+        tcg_gen_qemu_st_i64(tmp, addr, index, opc);
+        tcg_temp_free_i64(tmp);
+        return;
+    }
+    tcg_gen_qemu_st_i64(val, addr, index, opc);
 }
 
 #else
 
-#define DO_GEN_LD(SUFF, OPC)                                             \
-static inline void gen_aa32_ld##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_LD(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
     TCGv addr64 = tcg_temp_new();                                        \
     tcg_gen_extu_i32_i64(addr64, addr);                                  \
-    tcg_gen_qemu_ld_i32(val, addr64, index, OPC);                        \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        tcg_gen_xori_i64(addr64, addr64, BE32_XOR);                      \
+    }                                                                    \
+    tcg_gen_qemu_ld_i32(val, addr64, index, opc);                        \
     tcg_temp_free(addr64);                                               \
 }
 
-#define DO_GEN_ST(SUFF, OPC)                                             \
-static inline void gen_aa32_st##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_ST(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
     TCGv addr64 = tcg_temp_new();                                        \
     tcg_gen_extu_i32_i64(addr64, addr);                                  \
-    tcg_gen_qemu_st_i32(val, addr64, index, OPC);                        \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        tcg_gen_xori_i64(addr64, addr64, BE32_XOR);                      \
+    }                                                                    \
+    tcg_gen_qemu_st_i32(val, addr64, index, opc);                        \
     tcg_temp_free(addr64);                                               \
 }
 
-static inline void gen_aa32_ld64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
+    TCGMemOp opc = MO_Q | s->be_data;
     TCGv addr64 = tcg_temp_new();
     tcg_gen_extu_i32_i64(addr64, addr);
-    tcg_gen_qemu_ld_i64(val, addr64, index, MO_TEQ);
+    tcg_gen_qemu_ld_i64(val, addr64, index, opc);
+
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        tcg_gen_rotri_i64(val, val, 32);
+    }
     tcg_temp_free(addr64);
 }
 
-static inline void gen_aa32_st64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
+    TCGMemOp opc = MO_Q | s->be_data;
     TCGv addr64 = tcg_temp_new();
     tcg_gen_extu_i32_i64(addr64, addr);
-    tcg_gen_qemu_st_i64(val, addr64, index, MO_TEQ);
+
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        TCGv tmp = tcg_temp_new();
+        tcg_gen_rotri_i64(tmp, val, 32);
+        tcg_gen_qemu_st_i64(tmp, addr64, index, opc);
+        tcg_temp_free(tmp);
+    } else {
+        tcg_gen_qemu_st_i64(val, addr64, index, opc);
+    }
     tcg_temp_free(addr64);
 }
 
 #endif
 
-DO_GEN_LD(8s, MO_SB)
-DO_GEN_LD(8u, MO_UB)
-DO_GEN_LD(16s, MO_TESW)
-DO_GEN_LD(16u, MO_TEUW)
-DO_GEN_LD(32u, MO_TEUL)
-DO_GEN_ST(8, MO_UB)
-DO_GEN_ST(16, MO_TEUW)
-DO_GEN_ST(32, MO_TEUL)
+DO_GEN_LD(8s, MO_SB, 3)
+DO_GEN_LD(8u, MO_UB, 3)
+DO_GEN_LD(16s, MO_SW, 2)
+DO_GEN_LD(16u, MO_UW, 2)
+DO_GEN_LD(32u, MO_UL, 0)
+/* 'a' variants include an alignment check */
+DO_GEN_LD(16ua, MO_UW | MO_ALIGN, 2)
+DO_GEN_LD(32ua, MO_UL | MO_ALIGN, 0)
+DO_GEN_ST(8, MO_UB, 3)
+DO_GEN_ST(16, MO_UW, 2)
+DO_GEN_ST(32, MO_UL, 0)
 
 static inline void gen_set_pc_im(DisasContext *s, target_ulong val)
 {
@@ -1031,7 +1134,7 @@ static void gen_exception_insn(DisasContext *s, int offset, int excp,
 static inline void gen_lookup_tb(DisasContext *s)
 {
     tcg_gen_movi_i32(cpu_R[15], s->pc & ~1);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static inline void gen_add_data_offset(DisasContext *s, unsigned int insn,
@@ -1254,18 +1357,18 @@ VFP_GEN_FIX(ulto, )
 static inline void gen_vfp_ld(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_ld64(cpu_F0d, addr, get_mem_index(s));
+        gen_aa32_ld64(s, cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_ld32u(cpu_F0s, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, cpu_F0s, addr, get_mem_index(s));
     }
 }
 
 static inline void gen_vfp_st(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_st64(cpu_F0d, addr, get_mem_index(s));
+        gen_aa32_st64(s, cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_st32(cpu_F0s, addr, get_mem_index(s));
+        gen_aa32_st32(s, cpu_F0s, addr, get_mem_index(s));
     }
 }
 
@@ -1557,7 +1660,7 @@ static inline int gen_iwmmxt_shift(uint32_t insn, uint32_t mask, TCGv_i32 dest)
     } else {
         tmp = tcg_temp_new_i32();
         iwmmxt_load_reg(cpu_V0, rd);
-        tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+        tcg_gen_extrl_i64_i32(tmp, cpu_V0);
     }
     tcg_gen_andi_i32(tmp, tmp, mask);
     tcg_gen_mov_i32(dest, tmp);
@@ -1581,9 +1684,9 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             rdhi = (insn >> 16) & 0xf;
             if (insn & ARM_CP_RW_BIT) {			/* TMRRC */
                 iwmmxt_load_reg(cpu_V0, wrd);
-                tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             } else {					/* TMCRR */
                 tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
                 iwmmxt_store_reg(cpu_V0, wrd);
@@ -1601,24 +1704,24 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
         if (insn & ARM_CP_RW_BIT) {
             if ((insn >> 28) == 0xf) {			/* WLDRW wCx */
                 tmp = tcg_temp_new_i32();
-                gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                 iwmmxt_store_creg(wrd, tmp);
             } else {
                 i = 1;
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WLDRD */
-                        gen_aa32_ld64(cpu_M0, addr, get_mem_index(s));
+                        gen_aa32_ld64(s, cpu_M0, addr, get_mem_index(s));
                         i = 0;
                     } else {				/* WLDRW wRd */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     }
                 } else {
                     tmp = tcg_temp_new_i32();
                     if (insn & (1 << 22)) {		/* WLDRH */
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                     } else {				/* WLDRB */
-                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                     }
                 }
                 if (i) {
@@ -1630,24 +1733,24 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
         } else {
             if ((insn >> 28) == 0xf) {			/* WSTRW wCx */
                 tmp = iwmmxt_load_creg(wrd);
-                gen_aa32_st32(tmp, addr, get_mem_index(s));
+                gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             } else {
                 gen_op_iwmmxt_movq_M0_wRn(wrd);
                 tmp = tcg_temp_new_i32();
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WSTRD */
-                        gen_aa32_st64(cpu_M0, addr, get_mem_index(s));
+                        gen_aa32_st64(s, cpu_M0, addr, get_mem_index(s));
                     } else {				/* WSTRW wRd */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     }
                 } else {
                     if (insn & (1 << 22)) {		/* WSTRH */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st16(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                     } else {				/* WSTRB */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st8(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                     }
                 }
             }
@@ -1946,7 +2049,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
         switch ((insn >> 22) & 3) {
         case 0:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 7) << 3);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext8s_i32(tmp, tmp);
             } else {
@@ -1955,7 +2058,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             break;
         case 1:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 3) << 4);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext16s_i32(tmp, tmp);
             } else {
@@ -1964,7 +2067,7 @@ static int disas_iwmmxt_insn(DisasContext *s, uint32_t insn)
             break;
         case 2:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 1) << 5);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             break;
         }
         store_reg(s, rd, tmp);
@@ -2627,9 +2730,9 @@ static int disas_dsp_insn(DisasContext *s, uint32_t insn)
 
         if (insn & ARM_CP_RW_BIT) {			/* MRA */
             iwmmxt_load_reg(cpu_V0, acc);
-            tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
             tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-            tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             tcg_gen_andi_i32(cpu_R[rdhi], cpu_R[rdhi], (1 << (40 - 32)) - 1);
         } else {					/* MAR */
             tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
@@ -2712,15 +2815,15 @@ static TCGv_i32 gen_load_and_replicate(DisasContext *s, TCGv_i32 addr, int size)
     TCGv_i32 tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         gen_neon_dup_u8(tmp, 0);
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
         gen_neon_dup_low16(tmp);
         break;
     case 2:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
         break;
     default: /* Avoid compiler warnings.  */
         abort();
@@ -2951,7 +3054,7 @@ static int handle_vcvt(uint32_t insn, uint32_t rd, uint32_t rm, uint32_t dp,
         } else {
             gen_helper_vfp_tould(tcg_res, tcg_double, tcg_shift, fpst);
         }
-        tcg_gen_trunc_i64_i32(tcg_tmp, tcg_res);
+        tcg_gen_extrl_i64_i32(tcg_tmp, tcg_res);
         tcg_gen_st_f32(tcg_tmp, cpu_env, vfp_reg_offset(0, rd));
         tcg_temp_free_i32(tcg_tmp);
         tcg_temp_free_i64(tcg_res);
@@ -3046,7 +3149,7 @@ static int disas_vfp_insn(DisasContext *s, uint32_t insn)
      */
     if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb), s->fp_excp_el);
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -4057,24 +4160,213 @@ static int gen_set_psr_im(DisasContext *s, uint32_t mask, int spsr, uint32_t val
     return gen_set_psr(s, mask, spsr, tmp);
 }
 
+static bool msr_banked_access_decode(DisasContext *s, int r, int sysm, int rn,
+                                     int *tgtmode, int *regno)
+{
+    /* Decode the r and sysm fields of MSR/MRS banked accesses into
+     * the target mode and register number, and identify the various
+     * unpredictable cases.
+     * MSR (banked) and MRS (banked) are CONSTRAINED UNPREDICTABLE if:
+     *  + executed in user mode
+     *  + using R15 as the src/dest register
+     *  + accessing an unimplemented register
+     *  + accessing a register that's inaccessible at current PL/security state*
+     *  + accessing a register that you could access with a different insn
+     * We choose to UNDEF in all these cases.
+     * Since we don't know which of the various AArch32 modes we are in
+     * we have to defer some checks to runtime.
+     * Accesses to Monitor mode registers from Secure EL1 (which implies
+     * that EL3 is AArch64) must trap to EL3.
+     *
+     * If the access checks fail this function will emit code to take
+     * an exception and return false. Otherwise it will return true,
+     * and set *tgtmode and *regno appropriately.
+     */
+    int exc_target = default_exception_el(s);
+
+    /* These instructions are present only in ARMv8, or in ARMv7 with the
+     * Virtualization Extensions.
+     */
+    if (!arm_dc_feature(s, ARM_FEATURE_V8) &&
+        !arm_dc_feature(s, ARM_FEATURE_EL2)) {
+        goto undef;
+    }
+
+    if (IS_USER(s) || rn == 15) {
+        goto undef;
+    }
+
+    /* The table in the v8 ARM ARM section F5.2.3 describes the encoding
+     * of registers into (r, sysm).
+     */
+    if (r) {
+        /* SPSRs for other modes */
+        switch (sysm) {
+        case 0xe: /* SPSR_fiq */
+            *tgtmode = ARM_CPU_MODE_FIQ;
+            break;
+        case 0x10: /* SPSR_irq */
+            *tgtmode = ARM_CPU_MODE_IRQ;
+            break;
+        case 0x12: /* SPSR_svc */
+            *tgtmode = ARM_CPU_MODE_SVC;
+            break;
+        case 0x14: /* SPSR_abt */
+            *tgtmode = ARM_CPU_MODE_ABT;
+            break;
+        case 0x16: /* SPSR_und */
+            *tgtmode = ARM_CPU_MODE_UND;
+            break;
+        case 0x1c: /* SPSR_mon */
+            *tgtmode = ARM_CPU_MODE_MON;
+            break;
+        case 0x1e: /* SPSR_hyp */
+            *tgtmode = ARM_CPU_MODE_HYP;
+            break;
+        default: /* unallocated */
+            goto undef;
+        }
+        /* We arbitrarily assign SPSR a register number of 16. */
+        *regno = 16;
+    } else {
+        /* general purpose registers for other modes */
+        switch (sysm) {
+        case 0x0 ... 0x6:   /* 0b00xxx : r8_usr ... r14_usr */
+            *tgtmode = ARM_CPU_MODE_USR;
+            *regno = sysm + 8;
+            break;
+        case 0x8 ... 0xe:   /* 0b01xxx : r8_fiq ... r14_fiq */
+            *tgtmode = ARM_CPU_MODE_FIQ;
+            *regno = sysm;
+            break;
+        case 0x10 ... 0x11: /* 0b1000x : r14_irq, r13_irq */
+            *tgtmode = ARM_CPU_MODE_IRQ;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x12 ... 0x13: /* 0b1001x : r14_svc, r13_svc */
+            *tgtmode = ARM_CPU_MODE_SVC;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x14 ... 0x15: /* 0b1010x : r14_abt, r13_abt */
+            *tgtmode = ARM_CPU_MODE_ABT;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x16 ... 0x17: /* 0b1011x : r14_und, r13_und */
+            *tgtmode = ARM_CPU_MODE_UND;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x1c ... 0x1d: /* 0b1110x : r14_mon, r13_mon */
+            *tgtmode = ARM_CPU_MODE_MON;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x1e ... 0x1f: /* 0b1111x : elr_hyp, r13_hyp */
+            *tgtmode = ARM_CPU_MODE_HYP;
+            /* Arbitrarily pick 17 for ELR_Hyp (which is not a banked LR!) */
+            *regno = sysm & 1 ? 13 : 17;
+            break;
+        default: /* unallocated */
+            goto undef;
+        }
+    }
+
+    /* Catch the 'accessing inaccessible register' cases we can detect
+     * at translate time.
+     */
+    switch (*tgtmode) {
+    case ARM_CPU_MODE_MON:
+        if (!arm_dc_feature(s, ARM_FEATURE_EL3) || s->ns) {
+            goto undef;
+        }
+        if (s->current_el == 1) {
+            /* If we're in Secure EL1 (which implies that EL3 is AArch64)
+             * then accesses to Mon registers trap to EL3
+             */
+            exc_target = 3;
+            goto undef;
+        }
+        break;
+    case ARM_CPU_MODE_HYP:
+        /* Note that we can forbid accesses from EL2 here because they
+         * must be from Hyp mode itself
+         */
+        if (!arm_dc_feature(s, ARM_FEATURE_EL2) || s->current_el < 3) {
+            goto undef;
+        }
+        break;
+    default:
+        break;
+    }
+
+    return true;
+
+undef:
+    /* If we get here then some access check did not pass */
+    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(), exc_target);
+    return false;
+}
+
+static void gen_msr_banked(DisasContext *s, int r, int sysm, int rn)
+{
+    TCGv_i32 tcg_reg, tcg_tgtmode, tcg_regno;
+    int tgtmode = 0, regno = 0;
+
+    if (!msr_banked_access_decode(s, r, sysm, rn, &tgtmode, &regno)) {
+        return;
+    }
+
+    /* Sync state because msr_banked() can raise exceptions */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
+    tcg_reg = load_reg(s, rn);
+    tcg_tgtmode = tcg_const_i32(tgtmode);
+    tcg_regno = tcg_const_i32(regno);
+    gen_helper_msr_banked(cpu_env, tcg_reg, tcg_tgtmode, tcg_regno);
+    tcg_temp_free_i32(tcg_tgtmode);
+    tcg_temp_free_i32(tcg_regno);
+    tcg_temp_free_i32(tcg_reg);
+    s->is_jmp = DISAS_UPDATE;
+}
+
+static void gen_mrs_banked(DisasContext *s, int r, int sysm, int rn)
+{
+    TCGv_i32 tcg_reg, tcg_tgtmode, tcg_regno;
+    int tgtmode = 0, regno = 0;
+
+    if (!msr_banked_access_decode(s, r, sysm, rn, &tgtmode, &regno)) {
+        return;
+    }
+
+    /* Sync state because mrs_banked() can raise exceptions */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
+    tcg_reg = tcg_temp_new_i32();
+    tcg_tgtmode = tcg_const_i32(tgtmode);
+    tcg_regno = tcg_const_i32(regno);
+    gen_helper_mrs_banked(tcg_reg, cpu_env, tcg_tgtmode, tcg_regno);
+    tcg_temp_free_i32(tcg_tgtmode);
+    tcg_temp_free_i32(tcg_regno);
+    store_reg(s, rn, tcg_reg);
+    s->is_jmp = DISAS_UPDATE;
+}
+
 /* Generate an old-style exception return. Marks pc as dead. */
 static void gen_exception_return(DisasContext *s, TCGv_i32 pc)
 {
     TCGv_i32 tmp;
     store_reg(s, 15, pc);
     tmp = load_cpu_field(spsr);
-    gen_set_cpsr(tmp, CPSR_ERET_MASK);
+    gen_helper_cpsr_write_eret(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 /* Generate a v6 exception return.  Marks both values as dead.  */
 static void gen_rfe(DisasContext *s, TCGv_i32 pc, TCGv_i32 cpsr)
 {
-    gen_set_cpsr(cpsr, CPSR_ERET_MASK);
+    gen_helper_cpsr_write_eret(cpu_env, cpsr);
     tcg_temp_free_i32(cpsr);
     store_reg(s, 15, pc);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static void gen_nop_hint(DisasContext *s, int val)
@@ -4368,7 +4660,7 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
      */
     if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb), s->fp_excp_el);
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -4418,11 +4710,11 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
             if (size == 3) {
                 tmp64 = tcg_temp_new_i64();
                 if (load) {
-                    gen_aa32_ld64(tmp64, addr, get_mem_index(s));
+                    gen_aa32_ld64(s, tmp64, addr, get_mem_index(s));
                     neon_store_reg64(tmp64, rd);
                 } else {
                     neon_load_reg64(tmp64, rd);
-                    gen_aa32_st64(tmp64, addr, get_mem_index(s));
+                    gen_aa32_st64(s, tmp64, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i64(tmp64);
                 tcg_gen_addi_i32(addr, addr, stride);
@@ -4431,21 +4723,21 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                     if (size == 2) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             neon_store_reg(rd, pass, tmp);
                         } else {
                             tmp = neon_load_reg(rd, pass);
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         tcg_gen_addi_i32(addr, addr, stride);
                     } else if (size == 1) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tmp2 = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp2, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp2, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tcg_gen_shli_i32(tmp2, tmp2, 16);
                             tcg_gen_or_i32(tmp, tmp, tmp2);
@@ -4455,10 +4747,10 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                             tmp = neon_load_reg(rd, pass);
                             tmp2 = tcg_temp_new_i32();
                             tcg_gen_shri_i32(tmp2, tmp, 16);
-                            gen_aa32_st16(tmp, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                             tcg_gen_addi_i32(addr, addr, stride);
-                            gen_aa32_st16(tmp2, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp2, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp2);
                             tcg_gen_addi_i32(addr, addr, stride);
                         }
@@ -4467,7 +4759,7 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                             TCGV_UNUSED_I32(tmp2);
                             for (n = 0; n < 4; n++) {
                                 tmp = tcg_temp_new_i32();
-                                gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                                gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                                 tcg_gen_addi_i32(addr, addr, stride);
                                 if (n == 0) {
                                     tmp2 = tmp;
@@ -4487,7 +4779,7 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                                 } else {
                                     tcg_gen_shri_i32(tmp, tmp2, n * 8);
                                 }
-                                gen_aa32_st8(tmp, addr, get_mem_index(s));
+                                gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                                 tcg_temp_free_i32(tmp);
                                 tcg_gen_addi_i32(addr, addr, stride);
                             }
@@ -4611,13 +4903,13 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                     tmp = tcg_temp_new_i32();
                     switch (size) {
                     case 0:
-                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         break;
                     default: /* Avoid compiler warnings.  */
                         abort();
@@ -4635,13 +4927,13 @@ static int disas_neon_ls_insn(DisasContext *s, uint32_t insn)
                         tcg_gen_shri_i32(tmp, tmp, shift);
                     switch (size) {
                     case 0:
-                        gen_aa32_st8(tmp, addr, get_mem_index(s));
+                        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_st16(tmp, addr, get_mem_index(s));
+                        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         break;
                     }
                     tcg_temp_free_i32(tmp);
@@ -4683,7 +4975,7 @@ static inline void gen_neon_narrow(int size, TCGv_i32 dest, TCGv_i64 src)
     switch (size) {
     case 0: gen_helper_neon_narrow_u8(dest, src); break;
     case 1: gen_helper_neon_narrow_u16(dest, src); break;
-    case 2: tcg_gen_trunc_i64_i32(dest, src); break;
+    case 2: tcg_gen_extrl_i64_i32(dest, src); break;
     default: abort();
     }
 }
@@ -5106,7 +5398,7 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
      */
     if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb), s->fp_excp_el);
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -6254,7 +6546,7 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                                 break;
                             case 2:
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -6269,7 +6561,7 @@ static int disas_neon_data_insn(DisasContext *s, uint32_t insn)
                             case 2:
                                 tcg_gen_addi_i64(cpu_V0, cpu_V0, 1u << 31);
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -7138,7 +7430,7 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
              * call in order to handle c15_cpar.
              */
             TCGv_ptr tmpptr;
-            TCGv_i32 tcg_syn;
+            TCGv_i32 tcg_syn, tcg_isread;
             uint32_t syndrome;
 
             /* Note that since we are an implementation which takes an
@@ -7153,19 +7445,19 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
             case 14:
                 if (is64) {
                     syndrome = syn_cp14_rrt_trap(1, 0xe, opc1, crm, rt, rt2,
-                                                 isread, s->thumb);
+                                                 isread, false);
                 } else {
                     syndrome = syn_cp14_rt_trap(1, 0xe, opc1, opc2, crn, crm,
-                                                rt, isread, s->thumb);
+                                                rt, isread, false);
                 }
                 break;
             case 15:
                 if (is64) {
                     syndrome = syn_cp15_rrt_trap(1, 0xe, opc1, crm, rt, rt2,
-                                                 isread, s->thumb);
+                                                 isread, false);
                 } else {
                     syndrome = syn_cp15_rt_trap(1, 0xe, opc1, opc2, crn, crm,
-                                                rt, isread, s->thumb);
+                                                rt, isread, false);
                 }
                 break;
             default:
@@ -7179,12 +7471,16 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
                 break;
             }
 
+            gen_set_condexec(s);
             gen_set_pc_im(s, s->pc - 4);
             tmpptr = tcg_const_ptr(ri);
             tcg_syn = tcg_const_i32(syndrome);
-            gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn);
+            tcg_isread = tcg_const_i32(isread);
+            gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn,
+                                           tcg_isread);
             tcg_temp_free_ptr(tmpptr);
             tcg_temp_free_i32(tcg_syn);
+            tcg_temp_free_i32(tcg_isread);
         }
 
         /* Handle special cases first */
@@ -7224,11 +7520,11 @@ static int disas_coproc_insn(DisasContext *s, uint32_t insn)
                     tcg_gen_ld_i64(tmp64, cpu_env, ri->fieldoffset);
                 }
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 store_reg(s, rt, tmp);
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 store_reg(s, rt2, tmp);
             } else {
@@ -7334,11 +7630,11 @@ static void gen_storeq_reg(DisasContext *s, int rlow, int rhigh, TCGv_i64 val)
 {
     TCGv_i32 tmp;
     tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rlow, tmp);
     tmp = tcg_temp_new_i32();
     tcg_gen_shri_i64(val, val, 32);
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rhigh, tmp);
 }
 
@@ -7400,14 +7696,14 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
 
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16ua(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32ua(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7418,7 +7714,7 @@ static void gen_load_exclusive(DisasContext *s, int rt, int rt2,
         TCGv_i32 tmp3 = tcg_temp_new_i32();
 
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(cpu_exclusive_val, tmp, tmp3);
         store_reg(s, rt2, tmp3);
@@ -7469,14 +7765,14 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
     tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7487,7 +7783,7 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
         TCGv_i32 tmp2 = tcg_temp_new_i32();
         TCGv_i32 tmp3 = tcg_temp_new_i32();
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(val64, tmp, tmp3);
         tcg_temp_free_i32(tmp3);
@@ -7502,14 +7798,14 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
     tmp = load_reg(s, rt);
     switch (size) {
     case 0:
-        gen_aa32_st8(tmp, addr, get_mem_index(s));
+        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_st16(tmp, addr, get_mem_index(s));
+        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_st32(tmp, addr, get_mem_index(s));
+        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7518,7 +7814,7 @@ static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
     if (size == 3) {
         tcg_gen_addi_i32(addr, addr, 4);
         tmp = load_reg(s, rt2);
-        gen_aa32_st32(tmp, addr, get_mem_index(s));
+        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
         tcg_temp_free_i32(tmp);
     }
     tcg_gen_movi_i32(cpu_R[rd], 0);
@@ -7543,8 +7839,68 @@ static void gen_srs(DisasContext *s,
                     uint32_t mode, uint32_t amode, bool writeback)
 {
     int32_t offset;
-    TCGv_i32 addr = tcg_temp_new_i32();
-    TCGv_i32 tmp = tcg_const_i32(mode);
+    TCGv_i32 addr, tmp;
+    bool undef = false;
+
+    /* SRS is:
+     * - trapped to EL3 if EL3 is AArch64 and we are at Secure EL1
+     *   and specified mode is monitor mode
+     * - UNDEFINED in Hyp mode
+     * - UNPREDICTABLE in User or System mode
+     * - UNPREDICTABLE if the specified mode is:
+     * -- not implemented
+     * -- not a valid mode number
+     * -- a mode that's at a higher exception level
+     * -- Monitor, if we are Non-secure
+     * For the UNPREDICTABLE cases we choose to UNDEF.
+     */
+    if (s->current_el == 1 && !s->ns && mode == ARM_CPU_MODE_MON) {
+        gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(), 3);
+        return;
+    }
+
+    if (s->current_el == 0 || s->current_el == 2) {
+        undef = true;
+    }
+
+    switch (mode) {
+    case ARM_CPU_MODE_USR:
+    case ARM_CPU_MODE_FIQ:
+    case ARM_CPU_MODE_IRQ:
+    case ARM_CPU_MODE_SVC:
+    case ARM_CPU_MODE_ABT:
+    case ARM_CPU_MODE_UND:
+    case ARM_CPU_MODE_SYS:
+        break;
+    case ARM_CPU_MODE_HYP:
+        if (s->current_el == 1 || !arm_dc_feature(s, ARM_FEATURE_EL2)) {
+            undef = true;
+        }
+        break;
+    case ARM_CPU_MODE_MON:
+        /* No need to check specifically for "are we non-secure" because
+         * we've already made EL0 UNDEF and handled the trap for S-EL1;
+         * so if this isn't EL3 then we must be non-secure.
+         */
+        if (s->current_el != 3) {
+            undef = true;
+        }
+        break;
+    default:
+        undef = true;
+    }
+
+    if (undef) {
+        gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(),
+                           default_exception_el(s));
+        return;
+    }
+
+    addr = tcg_temp_new_i32();
+    tmp = tcg_const_i32(mode);
+    /* get_r13_banked() will raise an exception if called from System mode */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
     gen_helper_get_r13_banked(addr, cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     switch (amode) {
@@ -7565,11 +7921,11 @@ static void gen_srs(DisasContext *s,
     }
     tcg_gen_addi_i32(addr, addr, offset);
     tmp = load_reg(s, 14);
-    gen_aa32_st32(tmp, addr, get_mem_index(s));
+    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     tmp = load_cpu_field(spsr);
     tcg_gen_addi_i32(addr, addr, 4);
-    gen_aa32_st32(tmp, addr, get_mem_index(s));
+    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     if (writeback) {
         switch (amode) {
@@ -7594,6 +7950,7 @@ static void gen_srs(DisasContext *s,
         tcg_temp_free_i32(tmp);
     }
     tcg_temp_free_i32(addr);
+    s->is_jmp = DISAS_UPDATE;
 }
 
 static void disas_arm_insn(DisasContext *s, unsigned int insn)
@@ -7675,10 +8032,9 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
         if ((insn & 0x0ffffdff) == 0x01010000) {
             ARCH(6);
             /* setend */
-            if (((insn >> 9) & 1) != s->bswap_code) {
-                /* Dynamic endianness switching not implemented. */
-                qemu_log_mask(LOG_UNIMP, "arm: unimplemented setend\n");
-                goto illegal_op;
+            if (((insn >> 9) & 1) != !!(s->be_data == MO_BE)) {
+                gen_helper_setend(cpu_env);
+                s->is_jmp = DISAS_UPDATE;
             }
             return;
         } else if ((insn & 0x0fffff00) == 0x057ff000) {
@@ -7689,18 +8045,21 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 return;
             case 4: /* dsb */
             case 5: /* dmb */
-            case 6: /* isb */
                 ARCH(7);
                 /* We don't emulate caches so these are a no-op.  */
                 return;
+            case 6: /* isb */
+                /* We need to break the TB after this insn to execute
+                 * self-modifying code correctly and also to take
+                 * any pending interrupts immediately.
+                 */
+                gen_lookup_tb(s);
+                return;
             default:
                 goto illegal_op;
             }
         } else if ((insn & 0x0e5fffe0) == 0x084d0500) {
             /* srs */
-            if (IS_USER(s)) {
-                goto illegal_op;
-            }
             ARCH(6);
             gen_srs(s, (insn & 0x1f), (insn >> 23) & 3, insn & (1 << 21));
             return;
@@ -7724,10 +8083,10 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 tcg_gen_addi_i32(addr, addr, offset);
             /* Load PC into tmp and CPSR into tmp2.  */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             tcg_gen_addi_i32(addr, addr, 4);
             tmp2 = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
             if (insn & (1 << 21)) {
                 /* Base writeback.  */
                 switch (i) {
@@ -7852,7 +8211,26 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
         sh = (insn >> 4) & 0xf;
         rm = insn & 0xf;
         switch (sh) {
-        case 0x0: /* move program status register */
+        case 0x0: /* MSR, MRS */
+            if (insn & (1 << 9)) {
+                /* MSR (banked) and MRS (banked) */
+                int sysm = extract32(insn, 16, 4) |
+                    (extract32(insn, 8, 1) << 4);
+                int r = extract32(insn, 22, 1);
+
+                if (op1 & 1) {
+                    /* MSR (banked) */
+                    gen_msr_banked(s, r, sysm, rm);
+                } else {
+                    /* MRS (banked) */
+                    int rd = extract32(insn, 12, 4);
+
+                    gen_mrs_banked(s, r, sysm, rd);
+                }
+                break;
+            }
+
+            /* MSR, MRS (for PSRs) */
             if (op1 & 1) {
                 /* PSR = reg */
                 tmp = load_reg(s, rm);
@@ -8013,7 +8391,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if ((sh & 2) == 0) {
                     tmp2 = load_reg(s, rn);
@@ -8343,13 +8721,16 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                                 tmp = tcg_temp_new_i32();
                                 switch (op1) {
                                 case 0: /* lda */
-                                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld32u(s, tmp, addr,
+                                                   get_mem_index(s));
                                     break;
                                 case 2: /* ldab */
-                                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld8u(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 case 3: /* ldah */
-                                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld16u(s, tmp, addr,
+                                                   get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8360,13 +8741,16 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                                 tmp = load_reg(s, rm);
                                 switch (op1) {
                                 case 0: /* stl */
-                                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st32(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 case 2: /* stlb */
-                                    gen_aa32_st8(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st8(s, tmp, addr,
+                                                 get_mem_index(s));
                                     break;
                                 case 3: /* stlh */
-                                    gen_aa32_st16(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st16(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8421,11 +8805,11 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                         tmp = load_reg(s, rm);
                         tmp2 = tcg_temp_new_i32();
                         if (insn & (1 << 22)) {
-                            gen_aa32_ld8u(tmp2, addr, get_mem_index(s));
-                            gen_aa32_st8(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld8u(s, tmp2, addr, get_mem_index(s));
+                            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                         } else {
-                            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         }
                         tcg_temp_free_i32(tmp);
                         tcg_temp_free_i32(addr);
@@ -8460,20 +8844,20 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                     if (!load) {
                         /* store */
                         tmp = load_reg(s, rd);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = load_reg(s, rd + 1);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     } else {
                         /* load */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         store_reg(s, rd, tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         rd++;
                     }
                     address_offset = -4;
@@ -8482,25 +8866,25 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                     tmp = tcg_temp_new_i32();
                     switch (sh) {
                     case 1:
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_ld8s(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld8s(s, tmp, addr, get_mem_index(s));
                         break;
                     default:
                     case 3:
-                        gen_aa32_ld16s(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16s(s, tmp, addr, get_mem_index(s));
                         break;
                     }
                 } else {
                     /* store */
                     tmp = load_reg(s, rd);
-                    gen_aa32_st16(tmp, addr, get_mem_index(s));
+                    gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 /* Perform base writeback before the loaded value to
                    ensure correct behavior with overlapping index registers.
-                   ldrd with base writeback is is undefined if the
+                   ldrd with base writeback is undefined if the
                    destination and index registers overlap.  */
                 if (!(insn & (1 << 24))) {
                     gen_add_datah_offset(s, insn, address_offset, addr);
@@ -8679,7 +9063,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                         }
                         tcg_gen_shri_i64(tmp64, tmp64, 32);
                         tmp = tcg_temp_new_i32();
-                        tcg_gen_trunc_i64_i32(tmp, tmp64);
+                        tcg_gen_extrl_i64_i32(tmp, tmp64);
                         tcg_temp_free_i64(tmp64);
                         store_reg(s, rn, tmp);
                         break;
@@ -8848,17 +9232,17 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 /* load */
                 tmp = tcg_temp_new_i32();
                 if (insn & (1 << 22)) {
-                    gen_aa32_ld8u(tmp, tmp2, i);
+                    gen_aa32_ld8u(s, tmp, tmp2, i);
                 } else {
-                    gen_aa32_ld32u(tmp, tmp2, i);
+                    gen_aa32_ld32u(s, tmp, tmp2, i);
                 }
             } else {
                 /* store */
                 tmp = load_reg(s, rd);
                 if (insn & (1 << 22)) {
-                    gen_aa32_st8(tmp, tmp2, i);
+                    gen_aa32_st8(s, tmp, tmp2, i);
                 } else {
-                    gen_aa32_st32(tmp, tmp2, i);
+                    gen_aa32_st32(s, tmp, tmp2, i);
                 }
                 tcg_temp_free_i32(tmp);
             }
@@ -8931,7 +9315,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                         if (is_load) {
                             /* load */
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             if (user) {
                                 tmp2 = tcg_const_i32(i);
                                 gen_helper_set_user_reg(cpu_env, tmp2, tmp);
@@ -8958,7 +9342,7 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                             } else {
                                 tmp = load_reg(s, i);
                             }
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         j++;
@@ -8996,9 +9380,9 @@ static void disas_arm_insn(DisasContext *s, unsigned int insn)
                 if (exc_return) {
                     /* Restore CPSR from SPSR.  */
                     tmp = load_cpu_field(spsr);
-                    gen_set_cpsr(tmp, CPSR_ERET_MASK);
+                    gen_helper_cpsr_write_eret(cpu_env, tmp);
                     tcg_temp_free_i32(tmp);
-                    s->is_jmp = DISAS_UPDATE;
+                    s->is_jmp = DISAS_JUMP;
                 }
             }
             break;
@@ -9188,7 +9572,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
         /* Fall through to 32-bit decode.  */
     }
 
-    insn = arm_lduw_code(env, s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
     s->pc += 2;
     insn |= (uint32_t)insn_hw1 << 16;
 
@@ -9225,20 +9609,20 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 if (insn & (1 << 20)) {
                     /* ldrd */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     store_reg(s, rs, tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     store_reg(s, rd, tmp);
                 } else {
                     /* strd */
                     tmp = load_reg(s, rs);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = load_reg(s, rd);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 if (insn & (1 << 21)) {
@@ -9276,11 +9660,11 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     tcg_gen_add_i32(addr, addr, tmp);
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                 } else { /* tbb */
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i32(addr);
                 tcg_gen_shli_i32(tmp, tmp, 1);
@@ -9317,13 +9701,13 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         tmp = tcg_temp_new_i32();
                         switch (op) {
                         case 0: /* ldab */
-                            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* ldah */
-                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* lda */
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9333,13 +9717,13 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         tmp = load_reg(s, rs);
                         switch (op) {
                         case 0: /* stlb */
-                            gen_aa32_st8(tmp, addr, get_mem_index(s));
+                            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* stlh */
-                            gen_aa32_st16(tmp, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* stl */
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9367,10 +9751,10 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         tcg_gen_addi_i32(addr, addr, -8);
                     /* Load PC into tmp and CPSR into tmp2.  */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp2 = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
                     if (insn & (1 << 21)) {
                         /* Base writeback.  */
                         if (insn & (1 << 24)) {
@@ -9409,7 +9793,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     if (insn & (1 << 20)) {
                         /* Load.  */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         if (i == 15) {
                             gen_bx(s, tmp);
                         } else if (i == rn) {
@@ -9421,7 +9805,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     } else {
                         /* Store.  */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     tcg_gen_addi_i32(addr, addr, 4);
@@ -9749,7 +10133,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if (rs != 15)
                   {
@@ -9773,7 +10157,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 }
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 break;
             case 7: /* Unsigned sum of absolute differences.  */
@@ -9957,6 +10341,18 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
                         }
+
+                        if (extract32(insn, 5, 1)) {
+                            /* MSR (banked) */
+                            int sysm = extract32(insn, 8, 4) |
+                                (extract32(insn, 4, 1) << 4);
+                            int r = op & 1;
+
+                            gen_msr_banked(s, r, sysm, rm);
+                            break;
+                        }
+
+                        /* MSR (for PSRs) */
                         tmp = load_reg(s, rn);
                         if (gen_set_psr(s,
                               msr_mask(s, (insn >> 8) & 0xf, op == 1),
@@ -9999,9 +10395,16 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                             break;
                         case 4: /* dsb */
                         case 5: /* dmb */
-                        case 6: /* isb */
                             /* These execute as NOPs.  */
                             break;
+                        case 6: /* isb */
+                            /* We need to break the TB after this insn
+                             * to execute self-modifying code correctly
+                             * and also to take any pending interrupts
+                             * immediately.
+                             */
+                            gen_lookup_tb(s);
+                            break;
                         default:
                             goto illegal_op;
                         }
@@ -10022,7 +10425,17 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         tcg_gen_subi_i32(tmp, tmp, insn & 0xff);
                         gen_exception_return(s, tmp);
                         break;
-                    case 6: /* mrs cpsr.  */
+                    case 6: /* MRS */
+                        if (extract32(insn, 5, 1)) {
+                            /* MRS (banked) */
+                            int sysm = extract32(insn, 16, 4) |
+                                (extract32(insn, 4, 1) << 4);
+
+                            gen_mrs_banked(s, 0, sysm, rd);
+                            break;
+                        }
+
+                        /* mrs cpsr */
                         tmp = tcg_temp_new_i32();
                         if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             addr = tcg_const_i32(insn & 0xff);
@@ -10033,7 +10446,17 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         }
                         store_reg(s, rd, tmp);
                         break;
-                    case 7: /* mrs spsr.  */
+                    case 7: /* MRS */
+                        if (extract32(insn, 5, 1)) {
+                            /* MRS (banked) */
+                            int sysm = extract32(insn, 16, 4) |
+                                (extract32(insn, 4, 1) << 4);
+
+                            gen_mrs_banked(s, 1, sysm, rd);
+                            break;
+                        }
+
+                        /* mrs spsr.  */
                         /* Not accessible in user mode.  */
                         if (IS_USER(s) || arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
@@ -10344,19 +10767,19 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
             tmp = tcg_temp_new_i32();
             switch (op) {
             case 0:
-                gen_aa32_ld8u(tmp, addr, memidx);
+                gen_aa32_ld8u(s, tmp, addr, memidx);
                 break;
             case 4:
-                gen_aa32_ld8s(tmp, addr, memidx);
+                gen_aa32_ld8s(s, tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_ld16u(tmp, addr, memidx);
+                gen_aa32_ld16u(s, tmp, addr, memidx);
                 break;
             case 5:
-                gen_aa32_ld16s(tmp, addr, memidx);
+                gen_aa32_ld16s(s, tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_ld32u(tmp, addr, memidx);
+                gen_aa32_ld32u(s, tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -10373,13 +10796,13 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
             tmp = load_reg(s, rs);
             switch (op) {
             case 0:
-                gen_aa32_st8(tmp, addr, memidx);
+                gen_aa32_st8(s, tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_st16(tmp, addr, memidx);
+                gen_aa32_st16(s, tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_st32(tmp, addr, memidx);
+                gen_aa32_st32(s, tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -10423,7 +10846,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         }
     }
 
-    insn = arm_lduw_code(env, s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
     s->pc += 2;
 
     switch (insn >> 12) {
@@ -10516,7 +10939,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             addr = tcg_temp_new_i32();
             tcg_gen_movi_i32(addr, val);
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(addr);
             store_reg(s, rd, tmp);
             break;
@@ -10719,28 +11142,28 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
 
         switch (op) {
         case 0: /* str */
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             break;
         case 1: /* strh */
-            gen_aa32_st16(tmp, addr, get_mem_index(s));
+            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
             break;
         case 2: /* strb */
-            gen_aa32_st8(tmp, addr, get_mem_index(s));
+            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
             break;
         case 3: /* ldrsb */
-            gen_aa32_ld8s(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8s(s, tmp, addr, get_mem_index(s));
             break;
         case 4: /* ldr */
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             break;
         case 5: /* ldrh */
-            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
             break;
         case 6: /* ldrb */
-            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
             break;
         case 7: /* ldrsh */
-            gen_aa32_ld16s(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16s(s, tmp, addr, get_mem_index(s));
             break;
         }
         if (op >= 3) { /* load */
@@ -10762,12 +11185,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10784,12 +11207,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st8(tmp, addr, get_mem_index(s));
+            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10806,12 +11229,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st16(tmp, addr, get_mem_index(s));
+            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10827,12 +11250,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10900,12 +11323,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                     if (insn & (1 << 11)) {
                         /* pop */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         store_reg(s, i, tmp);
                     } else {
                         /* push */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     /* advance to the next address.  */
@@ -10917,13 +11340,13 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (insn & (1 << 11)) {
                     /* pop pc */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     /* don't set the pc until the rest of the instruction
                        has completed */
                 } else {
                     /* push lr */
                     tmp = load_reg(s, 14);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 tcg_gen_addi_i32(addr, addr, 4);
@@ -10994,10 +11417,9 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             case 2:
                 /* setend */
                 ARCH(6);
-                if (((insn >> 3) & 1) != s->bswap_code) {
-                    /* Dynamic endianness switching not implemented. */
-                    qemu_log_mask(LOG_UNIMP, "arm: unimplemented setend\n");
-                    goto illegal_op;
+                if (((insn >> 3) & 1) != !!(s->be_data == MO_BE)) {
+                    gen_helper_setend(cpu_env);
+                    s->is_jmp = DISAS_UPDATE;
                 }
                 break;
             case 3:
@@ -11053,7 +11475,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (insn & (1 << 11)) {
                     /* load */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     if (i == rn) {
                         loaded_var = tmp;
                     } else {
@@ -11062,7 +11484,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 } else {
                     /* store */
                     tmp = load_reg(s, i);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 /* advance to the next address */
@@ -11135,22 +11557,46 @@ undef:
                        default_exception_el(s));
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(ARMCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+static bool insn_crosses_page(CPUARMState *env, DisasContext *s)
+{
+    /* Return true if the insn at dc->pc might cross a page boundary.
+     * (False positives are OK, false negatives are not.)
+     */
+    uint16_t insn;
+
+    if ((s->pc & 3) == 0) {
+        /* At a 4-aligned address we can't be crossing a page */
+        return false;
+    }
+
+    /* This must be a Thumb insn */
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
+
+    if ((insn >> 11) >= 0x1d) {
+        /* Top five bits 0b11101 / 0b11110 / 0b11111 : this is the
+         * First half of a 32-bit Thumb insn. Thumb-1 cores might
+         * end up actually treating this as two 16-bit insns (see the
+         * code at the start of disas_thumb2_insn()) but we don't bother
+         * to check for that as it is unlikely, and false positives here
+         * are harmless.
+         */
+        return true;
+    }
+    /* Definitely a 16-bit insn, can't be crossing a page. */
+    return false;
+}
+
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
 {
+    ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
     int max_insns;
+    bool end_of_page;
 
     /* generate intermediate code */
 
@@ -11158,7 +11604,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
      * the A32/T32 complexity to do with conditional execution/IT blocks/etc.
      */
     if (ARM_TBFLAG_AARCH64_STATE(tb->flags)) {
-        gen_intermediate_code_internal_a64(cpu, tb, search_pc);
+        gen_intermediate_code_a64(cpu, tb);
         return;
     }
 
@@ -11172,9 +11618,14 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     dc->condjmp = 0;
 
     dc->aarch64 = 0;
-    dc->el3_is_aa64 = arm_el_is_aa64(env, 3);
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = ARM_TBFLAG_THUMB(tb->flags);
-    dc->bswap_code = ARM_TBFLAG_BSWAP_CODE(tb->flags);
+    dc->sctlr_b = ARM_TBFLAG_SCTLR_B(tb->flags);
+    dc->be_data = ARM_TBFLAG_BE_DATA(tb->flags) ? MO_BE : MO_LE;
     dc->condexec_mask = (ARM_TBFLAG_CONDEXEC(tb->flags) & 0xf) << 1;
     dc->condexec_cond = ARM_TBFLAG_CONDEXEC(tb->flags) >> 4;
     dc->mmu_idx = ARM_TBFLAG_MMUIDX(tb->flags);
@@ -11220,11 +11671,14 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
     /* FIXME: cpu_M0 can probably be the same as cpu_V0.  */
     cpu_M0 = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
@@ -11250,10 +11704,9 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
      * (3) if we leave the TB unexpectedly (eg a data abort on a load)
      * then the CPUARMState will be wrong and we need to reset it.
      * This is handled in the same way as restoration of the
-     * PC in these situations: we will be called again with search_pc=1
-     * and generate a mapping of the condexec bits for each PC in
-     * gen_opc_condexec_bits[]. restore_state_to_opc() then uses
-     * this to restore the condexec bits.
+     * PC in these situations; we save the value of the condexec bits
+     * for each PC via tcg_gen_insn_start(), and restore_state_to_opc()
+     * then uses this to restore them after an exception.
      *
      * Note that there are no instructions which can read the condexec
      * bits, and none which can write non-static values to them, so
@@ -11270,13 +11723,17 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         store_cpu_field(tmp, condexec_bits);
       }
     do {
+        tcg_gen_insn_start(dc->pc,
+                           (dc->condexec_cond << 4) | (dc->condexec_mask >> 1));
+        num_insns++;
+
 #ifdef CONFIG_USER_ONLY
         /* Intercept jump to the magic kernel page.  */
         if (dc->pc >= 0xffff0000) {
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_KERNEL_TRAP);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #else
@@ -11284,40 +11741,40 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_EXCEPTION_EXIT);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #endif
 
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
+                    if (bp->flags & BP_CPU) {
+                        gen_set_condexec(dc);
+                        gen_set_pc_im(dc, dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it's likely not going to be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        /* TODO: Advance PC by correct instruction length to
+                         * avoid disassembler error messages */
+                        dc->pc += 2;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            gen_opc_condexec_bits[lj] = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
         }
 
         if (dc->ss_active && !dc->pstate_ss) {
@@ -11331,7 +11788,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
+            assert(num_insns == 1);
             gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
                           default_exception_el(dc));
             goto done_generating;
@@ -11348,7 +11805,7 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
                 }
             }
         } else {
-            unsigned int insn = arm_ldl_code(env, dc->pc, dc->bswap_code);
+            unsigned int insn = arm_ldl_code(env, dc->pc, dc->sctlr_b);
             dc->pc += 4;
             disas_arm_insn(dc, insn);
         }
@@ -11367,12 +11824,24 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns ++;
+
+        /* We want to stop the TB if the next insn starts in a new page,
+         * or if it spans between this page and the next. This means that
+         * if we're looking at the last halfword in the page we need to
+         * see if it's a 16-bit Thumb insn (which will fit in this TB)
+         * or a 32-bit Thumb insn (which won't).
+         * This is to avoid generating a silly TB with a single 16-bit insn
+         * in it at the end of this page (which would execute correctly
+         * but isn't very efficient).
+         */
+        end_of_page = (dc->pc >= next_page_start) ||
+            ((dc->pc >= next_page_start - 3) && insn_crosses_page(env, dc));
+
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
              !dc->ss_active &&
-             dc->pc < next_page_start &&
+             !end_of_page &&
              num_insns < max_insns);
 
     if (tb->cflags & CF_LAST_IO) {
@@ -11388,47 +11857,45 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
        instruction was a conditional branch or trap, and the PC has
        already been written.  */
     if (unlikely(cs->singlestep_enabled || dc->ss_active)) {
-        /* Make sure the pc is updated, and raise a debug exception.  */
-        if (dc->condjmp) {
-            gen_set_condexec(dc);
-            if (dc->is_jmp == DISAS_SWI) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb),
-                              default_exception_el(dc));
-            } else if (dc->is_jmp == DISAS_HVC) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm), 2);
-            } else if (dc->is_jmp == DISAS_SMC) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_SMC, syn_aa32_smc(), 3);
-            } else if (dc->ss_active) {
-                gen_step_complete_exception(dc);
-            } else {
-                gen_exception_internal(EXCP_DEBUG);
-            }
-            gen_set_label(dc->condlabel);
-        }
-        if (dc->condjmp || !dc->is_jmp) {
-            gen_set_pc_im(dc, dc->pc);
-            dc->condjmp = 0;
-        }
+        /* Unconditional and "condition passed" instruction codepath. */
         gen_set_condexec(dc);
-        if (dc->is_jmp == DISAS_SWI && !dc->condjmp) {
+        switch (dc->is_jmp) {
+        case DISAS_SWI:
             gen_ss_advance(dc);
             gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb),
                           default_exception_el(dc));
-        } else if (dc->is_jmp == DISAS_HVC && !dc->condjmp) {
+            break;
+        case DISAS_HVC:
             gen_ss_advance(dc);
             gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm), 2);
-        } else if (dc->is_jmp == DISAS_SMC && !dc->condjmp) {
+            break;
+        case DISAS_SMC:
             gen_ss_advance(dc);
             gen_exception(EXCP_SMC, syn_aa32_smc(), 3);
-        } else if (dc->ss_active) {
-            gen_step_complete_exception(dc);
-        } else {
-            /* FIXME: Single stepping a WFI insn will not halt
-               the CPU.  */
-            gen_exception_internal(EXCP_DEBUG);
+            break;
+        case DISAS_NEXT:
+        case DISAS_UPDATE:
+            gen_set_pc_im(dc, dc->pc);
+            /* fall through */
+        default:
+            if (dc->ss_active) {
+                gen_step_complete_exception(dc);
+            } else {
+                /* FIXME: Single stepping a WFI insn will not halt
+                   the CPU.  */
+                gen_exception_internal(EXCP_DEBUG);
+            }
+        }
+        if (dc->condjmp) {
+            /* "Condition failed" instruction codepath. */
+            gen_set_label(dc->condlabel);
+            gen_set_condexec(dc);
+            gen_set_pc_im(dc, dc->pc);
+            if (dc->ss_active) {
+                gen_step_complete_exception(dc);
+            } else {
+                gen_exception_internal(EXCP_DEBUG);
+            }
         }
     } else {
         /* While branches must always occur at the end of an IT block,
@@ -11444,9 +11911,11 @@ static inline void gen_intermediate_code_internal(ARMCPU *cpu,
         case DISAS_NEXT:
             gen_goto_tb(dc, 1, dc->pc);
             break;
-        default:
-        case DISAS_JUMP:
         case DISAS_UPDATE:
+            gen_set_pc_im(dc, dc->pc);
+            /* fall through */
+        case DISAS_JUMP:
+        default:
             /* indicate that the hash table must be used to find the next TB */
             tcg_gen_exit_tb(0);
             break;
@@ -11489,33 +11958,17 @@ done_generating:
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) &&
+        qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
         log_target_disas(cs, pc_start, dc->pc - pc_start,
-                         dc->thumb | (dc->bswap_code << 1));
+                         dc->thumb | (dc->sctlr_b << 1));
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -11530,6 +11983,7 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     CPUARMState *env = &cpu->env;
     int i;
     uint32_t psr;
+    const char *ns_status;
 
     if (is_a64(env)) {
         aarch64_cpu_dump_state(cs, f, cpu_fprintf, flags);
@@ -11544,13 +11998,22 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
             cpu_fprintf(f, " ");
     }
     psr = cpsr_read(env);
-    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%d\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        (psr & CPSR_M) != ARM_CPU_MODE_MON) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%s%d\n",
                 psr,
                 psr & (1 << 31) ? 'N' : '-',
                 psr & (1 << 30) ? 'Z' : '-',
                 psr & (1 << 29) ? 'C' : '-',
                 psr & (1 << 28) ? 'V' : '-',
                 psr & CPSR_T ? 'T' : 'A',
+                ns_status,
                 cpu_mode_names[psr & 0xf], (psr & 0x10) ? 32 : 26);
 
     if (flags & CPU_DUMP_FPU) {
@@ -11572,13 +12035,14 @@ void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     }
 }
 
-void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
     if (is_a64(env)) {
-        env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+        env->pc = data[0];
         env->condexec_bits = 0;
     } else {
-        env->regs[15] = tcg_ctx.gen_opc_pc[pc_pos];
-        env->condexec_bits = gen_opc_condexec_bits[pc_pos];
+        env->regs[15] = data[0];
+        env->condexec_bits = data[1];
     }
 }
diff --git a/qemu/target-arm/translate.h b/qemu/target-arm/translate.h
index 9ab978fb7..6a18d7bad 100644
--- a/qemu/target-arm/translate.h
+++ b/qemu/target-arm/translate.h
@@ -16,14 +16,16 @@ typedef struct DisasContext {
     struct TranslationBlock *tb;
     int singlestep_enabled;
     int thumb;
-    int bswap_code;
+    int sctlr_b;
+    TCGMemOp be_data;
 #if !defined(CONFIG_USER_ONLY)
     int user;
 #endif
     ARMMMUIdx mmu_idx; /* MMU index to use for normal loads/stores */
     bool ns;        /* Use non-secure CPREG bank on access */
     int fp_excp_el; /* FP exception EL or 0 if enabled */
-    bool el3_is_aa64;  /* Flag indicating whether EL3 is AArch64 or not */
+    /* Flag indicating that exceptions from secure mode are routed to EL3. */
+    bool secure_routed_to_el3;
     bool vfp_enabled; /* FP enabled via FPSCR.EN */
     int vec_len;
     int vec_stride;
@@ -62,7 +64,21 @@ typedef struct DisasContext {
     TCGv_i64 tmp_a64[TMP_A64_MAX];
 } DisasContext;
 
-extern TCGv_ptr cpu_env;
+typedef struct DisasCompare {
+    TCGCond cond;
+    TCGv_i32 value;
+    bool value_global;
+} DisasCompare;
+
+/* Share the TCG temporaries common between 32 and 64 bit modes.  */
+extern TCGv_env cpu_env;
+extern TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
+extern TCGv_i64 cpu_exclusive_addr;
+extern TCGv_i64 cpu_exclusive_val;
+#ifdef CONFIG_USER_ONLY
+extern TCGv_i64 cpu_exclusive_test;
+extern TCGv_i32 cpu_exclusive_info;
+#endif
 
 static inline int arm_dc_feature(DisasContext *dc, int feature)
 {
@@ -84,7 +100,7 @@ static inline int default_exception_el(DisasContext *s)
      * exceptions can only be routed to ELs above 1, so we target the higher of
      * 1 or the current EL.
      */
-    return (s->mmu_idx == ARMMMUIdx_S1SE0 && !s->el3_is_aa64)
+    return (s->mmu_idx == ARMMMUIdx_S1SE0 && s->secure_routed_to_el3)
             ? 3 : MAX(1, s->current_el);
 }
 
@@ -107,9 +123,7 @@ static inline int default_exception_el(DisasContext *s)
 
 #ifdef TARGET_AARCH64
 void a64_translate_init(void);
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc);
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb);
 void gen_a64_set_pc_im(uint64_t val);
 void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
                             fprintf_function cpu_fprintf, int flags);
@@ -118,9 +132,7 @@ static inline void a64_translate_init(void)
 {
 }
 
-static inline void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                                      TranslationBlock *tb,
-                                                      bool search_pc)
+static inline void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
 }
 
@@ -135,6 +147,9 @@ static inline void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
 }
 #endif
 
+void arm_test_cc(DisasCompare *cmp, int cc);
+void arm_free_cc(DisasCompare *cmp);
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label);
 void arm_gen_test_cc(int cc, TCGLabel *label);
 
 #endif /* TARGET_ARM_TRANSLATE_H */
diff --git a/qemu/target-cris/cpu-qom.h b/qemu/target-cris/cpu-qom.h
index 6fc30c208..df4c0b50a 100644
--- a/qemu/target-cris/cpu-qom.h
+++ b/qemu/target-cris/cpu-qom.h
@@ -73,6 +73,10 @@ static inline CRISCPU *cris_env_get_cpu(CPUCRISState *env)
 
 #define ENV_OFFSET offsetof(CRISCPU, env)
 
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_cris_cpu;
+#endif
+
 void cris_cpu_do_interrupt(CPUState *cpu);
 void crisv10_cpu_do_interrupt(CPUState *cpu);
 bool cris_cpu_exec_interrupt(CPUState *cpu, int int_req);
diff --git a/qemu/target-cris/cpu.c b/qemu/target-cris/cpu.c
index b17e849e2..1cb79dd97 100644
--- a/qemu/target-cris/cpu.c
+++ b/qemu/target-cris/cpu.c
@@ -21,6 +21,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "mmu.h"
@@ -302,12 +304,20 @@ static void cris_cpu_class_init(ObjectClass *oc, void *data)
     cc->handle_mmu_fault = cris_cpu_handle_mmu_fault;
 #else
     cc->get_phys_page_debug = cris_cpu_get_phys_page_debug;
+    dc->vmsd = &vmstate_cris_cpu;
 #endif
 
     cc->gdb_num_core_regs = 49;
     cc->gdb_stop_before_watchpoint = true;
 
     cc->disas_set_info = cris_disas_set_info;
+
+    /*
+     * Reason: cris_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo cris_cpu_type_info = {
diff --git a/qemu/target-cris/cpu.h b/qemu/target-cris/cpu.h
index d422e3571..415cf9143 100644
--- a/qemu/target-cris/cpu.h
+++ b/qemu/target-cris/cpu.h
@@ -20,7 +20,6 @@
 #ifndef CPU_CRIS_H
 #define CPU_CRIS_H
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 32
@@ -29,8 +28,6 @@
 
 #include "exec/cpu-defs.h"
 
-#define ELF_MACHINE	EM_CRIS
-
 #define EXCP_NMI        1
 #define EXCP_GURU       2
 #define EXCP_BUSFAULT   3
@@ -108,6 +105,11 @@
 
 #define NB_MMU_MODES 2
 
+typedef struct {
+    uint32_t hi;
+    uint32_t lo;
+} TLBSet;
+
 typedef struct CPUCRISState {
 	uint32_t regs[16];
 	/* P0 - P15 are referred to as special registers in the docs.  */
@@ -151,7 +153,7 @@ typedef struct CPUCRISState {
 	uint32_t sregs[4][16];
 
 	/* Linear feedback shift reg in the mmu. Used to provide pseudo
-	   randomness for the 'hint' the mmu gives to sw for chosing valid
+	   randomness for the 'hint' the mmu gives to sw for choosing valid
 	   sets on TLB refills.  */
 	uint32_t mmu_rand_lfsr;
 
@@ -161,11 +163,7 @@ typedef struct CPUCRISState {
 	 *
 	 * One for I and another for D.
 	 */
-	struct
-	{
-		uint32_t hi;
-		uint32_t lo;
-	} tlbsets[2][4][16];
+        TLBSet tlbsets[2][4][16];
 
 	CPU_COMMON
 
@@ -224,16 +222,13 @@ enum {
 #define cpu_init(cpu_model) CPU(cpu_cris_init(cpu_model))
 
 #define cpu_exec cpu_cris_exec
-#define cpu_gen_code cpu_cris_gen_code
 #define cpu_signal_handler cpu_cris_signal_handler
 
-#define CPU_SAVE_VERSION 1
-
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUCRISState *env)
+static inline int cpu_mmu_index (CPUCRISState *env, bool ifetch)
 {
 	return !!(env->pregs[PR_CCS] & U_FLAG);
 }
diff --git a/qemu/target-cris/gdbstub.c b/qemu/target-cris/gdbstub.c
index 5db3683ab..1bbf17b04 100644
--- a/qemu/target-cris/gdbstub.c
+++ b/qemu/target-cris/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-cris/helper.c b/qemu/target-cris/helper.c
index df6c9fdcb..1eb9fd918 100644
--- a/qemu/target-cris/helper.c
+++ b/qemu/target-cris/helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "qemu/host-utils.h"
diff --git a/qemu/target-cris/helper.h b/qemu/target-cris/helper.h
index 0b383b25a..ff3595641 100644
--- a/qemu/target-cris/helper.h
+++ b/qemu/target-cris/helper.h
@@ -1,7 +1,6 @@
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_2(tlb_flush_pid, void, env, i32)
 DEF_HELPER_2(spc_write, void, env, i32)
-DEF_HELPER_3(dump, void, i32, i32, i32)
 DEF_HELPER_1(rfe, void, env)
 DEF_HELPER_1(rfn, void, env)
 
diff --git a/qemu/target-cris/machine.c b/qemu/target-cris/machine.c
index 8f9c0dd59..9cc2820e8 100644
--- a/qemu/target-cris/machine.c
+++ b/qemu/target-cris/machine.c
@@ -1,90 +1,92 @@
-#include "hw/hw.h"
-#include "hw/boards.h"
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->regs[i]);
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->pregs[i]);
-
-    qemu_put_be32(f, env->pc);
-    qemu_put_be32(f, env->ksp);
+/*
+ *  CRIS virtual CPU state save/load support
+ *
+ *  Copyright (c) 2012 Red Hat, Inc.
+ *  Written by Juan Quintela <quintela@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
 
-    qemu_put_be32(f, env->dslot);
-    qemu_put_be32(f, env->btaken);
-    qemu_put_be32(f, env->btarget);
-
-    qemu_put_be32(f, env->cc_op);
-    qemu_put_be32(f, env->cc_mask);
-    qemu_put_be32(f, env->cc_dest);
-    qemu_put_be32(f, env->cc_src);
-    qemu_put_be32(f, env->cc_result);
-    qemu_put_be32(f, env->cc_size);
-    qemu_put_be32(f, env->cc_x);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            qemu_put_be32(f, env->sregs[s][i]);
-    }
+#include "qemu/osdep.h"
+#include "hw/hw.h"
 
-    qemu_put_be32(f, env->mmu_rand_lfsr);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].lo);
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].hi);
-            }
-        }
+static const VMStateDescription vmstate_tlbset = {
+    .name = "cpu/tlbset",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(lo, TLBSet),
+        VMSTATE_UINT32(hi, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-	CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        env->regs[i] = qemu_get_be32(f);
-    for (i = 0; i < 16; i++)
-        env->pregs[i] = qemu_get_be32(f);
-
-    env->pc = qemu_get_be32(f);
-    env->ksp = qemu_get_be32(f);
+};
 
-    env->dslot = qemu_get_be32(f);
-    env->btaken = qemu_get_be32(f);
-    env->btarget = qemu_get_be32(f);
-
-    env->cc_op = qemu_get_be32(f);
-    env->cc_mask = qemu_get_be32(f);
-    env->cc_dest = qemu_get_be32(f);
-    env->cc_src = qemu_get_be32(f);
-    env->cc_result = qemu_get_be32(f);
-    env->cc_size = qemu_get_be32(f);
-    env->cc_x = qemu_get_be32(f);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            env->sregs[s][i] = qemu_get_be32(f);
+static const VMStateDescription vmstate_cris_env = {
+    .name = "env",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(pregs, CPUCRISState, 16),
+        VMSTATE_UINT32(pc, CPUCRISState),
+        VMSTATE_UINT32(ksp, CPUCRISState),
+        VMSTATE_INT32(dslot, CPUCRISState),
+        VMSTATE_INT32(btaken, CPUCRISState),
+        VMSTATE_UINT32(btarget, CPUCRISState),
+        VMSTATE_UINT32(cc_op, CPUCRISState),
+        VMSTATE_UINT32(cc_mask, CPUCRISState),
+        VMSTATE_UINT32(cc_dest, CPUCRISState),
+        VMSTATE_UINT32(cc_src, CPUCRISState),
+        VMSTATE_UINT32(cc_result, CPUCRISState),
+        VMSTATE_INT32(cc_size, CPUCRISState),
+        VMSTATE_INT32(cc_x, CPUCRISState),
+        VMSTATE_INT32(locked_irq, CPUCRISState),
+        VMSTATE_INT32(interrupt_vector, CPUCRISState),
+        VMSTATE_INT32(fault_vector, CPUCRISState),
+        VMSTATE_INT32(trap_vector, CPUCRISState),
+        VMSTATE_UINT32_ARRAY(sregs[0], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[1], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[2], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[3], CPUCRISState, 16),
+        VMSTATE_UINT32(mmu_rand_lfsr, CPUCRISState),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
+};
 
-    env->mmu_rand_lfsr = qemu_get_be32(f);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                env->tlbsets[mmu][s][i].lo = qemu_get_be32(f);
-                env->tlbsets[mmu][s][i].hi = qemu_get_be32(f);
-            }
-        }
+const VMStateDescription vmstate_cris_cpu = {
+    .name = "cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_CPU(),
+        VMSTATE_STRUCT(env, CRISCPU, 1, vmstate_cris_env, CPUCRISState),
+        VMSTATE_END_OF_LIST()
     }
-
-    return 0;
-}
+};
diff --git a/qemu/target-cris/mmu.c b/qemu/target-cris/mmu.c
index 1c95a415f..4278d2dce 100644
--- a/qemu/target-cris/mmu.c
+++ b/qemu/target-cris/mmu.c
@@ -18,8 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef CONFIG_USER_ONLY
-
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "mmu.h"
 
@@ -360,4 +359,3 @@ int cris_mmu_translate(struct cris_mmu_result *res,
 	env->pregs[PR_SRS] = old_srs;
 	return miss;
 }
-#endif
diff --git a/qemu/target-cris/op_helper.c b/qemu/target-cris/op_helper.c
index 5c0c14d99..320f2b80d 100644
--- a/qemu/target-cris/op_helper.c
+++ b/qemu/target-cris/op_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/helper-proto.h"
@@ -91,11 +92,6 @@ void helper_spc_write(CPUCRISState *env, uint32_t new_spc)
 #endif
 }
 
-void helper_dump(uint32_t a0, uint32_t a1, uint32_t a2)
-{
-	qemu_log("%s: a0=%x a1=%x\n", __func__, a0, a1);
-}
-
 /* Used by the tlb decoder.  */
 #define EXTRACT_FIELD(src, start, end) \
 	    (((src) >> start) & ((1 << (end - start + 1)) - 1))
diff --git a/qemu/target-cris/translate.c b/qemu/target-cris/translate.c
index 3e59601eb..a73176c11 100644
--- a/qemu/target-cris/translate.c
+++ b/qemu/target-cris/translate.c
@@ -23,6 +23,7 @@
  * The condition code translation is in need of attention.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -34,6 +35,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define DISAS_CRIS 0
@@ -58,7 +60,7 @@
 #define CC_MASK_NZVC 0xf
 #define CC_MASK_RNZV 0x10e
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[16];
 static TCGv cpu_PR[16];
 static TCGv cc_x;
@@ -102,9 +104,9 @@ typedef struct DisasContext {
 
     int cc_size_uptodate; /* -1 invalid or last written value.  */
 
-    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
-    int flags_uptodate; /* Wether or not $ccs is uptodate.  */
-    int flagx_known; /* Wether or not flags_x has the x flag known at
+    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not up-to-date.  */
+    int flags_uptodate; /* Whether or not $ccs is up-to-date.  */
+    int flagx_known; /* Whether or not flags_x has the x flag known at
                 translation time.  */
     int flags_x;
 
@@ -130,8 +132,10 @@ typedef struct DisasContext {
 
 static void gen_BUG(DisasContext *dc, const char *file, int line)
 {
-    printf("BUG: pc=%x %s %d\n", dc->pc, file, line);
-    qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    fprintf(stderr, "BUG: pc=%x %s %d\n", dc->pc, file, line);
+    if (qemu_log_separate()) {
+        qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    }
     cpu_abort(CPU(dc->cpu), "%s:%d\n", file, line);
 }
 
@@ -311,7 +315,7 @@ static void t_gen_asr(TCGv d, TCGv a, TCGv b)
 
 static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
 {
-    TCGLabel *l1 = gen_new_label();
+    TCGv t = tcg_temp_new();
 
     /*
      * d <<= 1
@@ -319,9 +323,9 @@ static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
      *    d -= s;
      */
     tcg_gen_shli_tl(d, a, 1);
-    tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
-    tcg_gen_sub_tl(d, d, b);
-    gen_set_label(l1);
+    tcg_gen_sub_tl(t, d, b);
+    tcg_gen_movcond_tl(TCG_COND_GEU, d, d, b, t, d);
+    tcg_temp_free(t);
 }
 
 static void t_gen_cris_mstep(TCGv d, TCGv a, TCGv b, TCGv ccs)
@@ -769,13 +773,7 @@ static void cris_alu_op_exec(DisasContext *dc, int op,
         t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
         break;
     case CC_OP_BOUND:
-    {
-        TCGLabel *l1 = gen_new_label();
-        tcg_gen_mov_tl(dst, a);
-        tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
-        tcg_gen_mov_tl(dst, b);
-        gen_set_label(l1);
-    }
+        tcg_gen_movcond_tl(TCG_COND_LEU, dst, a, b, a, b);
         break;
     case CC_OP_CMP:
         tcg_gen_sub_tl(dst, a, b);
@@ -783,7 +781,7 @@ static void cris_alu_op_exec(DisasContext *dc, int op,
         t_gen_subx_carry(dc, dst);
         break;
     default:
-        qemu_log("illegal ALU op.\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "illegal ALU op.\n");
         BUG();
         break;
     }
@@ -1089,7 +1087,7 @@ static inline void cris_prepare_jmp (DisasContext *dc, unsigned int type)
 
 static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1103,7 +1101,7 @@ static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 static void gen_load(DisasContext *dc, TCGv dst, TCGv addr, 
              unsigned int size, int sign)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1118,7 +1116,7 @@ static void gen_load(DisasContext *dc, TCGv dst, TCGv addr,
 static void gen_store (DisasContext *dc, TCGv addr, TCGv val,
                unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1482,15 +1480,8 @@ static int dec_scc_r(CPUCRISState *env, DisasContext *dc)
     LOG_DIS("s%s $r%u\n",
             cc_name(cond), dc->op1);
 
-    if (cond != CC_A) {
-        TCGLabel *l1 = gen_new_label();
-        gen_tst_cc(dc, cpu_R[dc->op1], cond);
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->op1], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->op1], cpu_R[dc->op1], 0);
 
     cris_cc_mask(dc, 0);
     return 2;
@@ -2604,9 +2595,9 @@ static int dec_movem_mr(CPUCRISState *env, DisasContext *dc)
     tcg_temp_free(addr);
 
     for (i = 0; i < (nr >> 1); i++) {
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2], tmp[i]);
         tcg_gen_shri_i64(tmp[i], tmp[i], 32);
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
         tcg_temp_free_i64(tmp[i]);
     }
     if (nr & 1) {
@@ -3007,10 +2998,6 @@ static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
     int insn_len = 2;
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-        }
-
     /* Load a halfword onto the instruction register.  */
         dc->ir = cris_fetch(env, dc, dc->pc, 2, 0);
 
@@ -3047,23 +3034,6 @@ static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
     return insn_len;
 }
 
-static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(cris_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                cris_evaluate_flags(dc);
-                tcg_gen_movi_tl(env_pc, dc->pc);
-                t_gen_raise_exception(EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
 #include "translate_v10.c"
 
 /*
@@ -3101,15 +3071,12 @@ static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
  */
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUCRISState *env, struct TranslationBlock *tb)
 {
+    CRISCPU *cpu = cris_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUCRISState *env = &cpu->env;
     uint32_t pc_start;
     unsigned int insn_len;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start;
@@ -3161,13 +3128,13 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
 
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log(
-                "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
+                "pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
                 "pid=%x usp=%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n",
-                search_pc, dc->pc, dc->ppc,
+                dc->pc, dc->ppc,
                 (uint64_t)tb->flags,
                 env->btarget, (unsigned)tb->flags & 7,
                 env->pregs[PR_CCS],
@@ -3183,38 +3150,38 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            if (dc->delayed_branch == 1) {
-                tcg_ctx.gen_opc_pc[lj] = dc->ppc | 1;
-            } else {
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            }
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->delayed_branch == 1
+                           ? dc->ppc | 1 : dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            cris_evaluate_flags(dc);
+            tcg_gen_movi_tl(env_pc, dc->pc);
+            t_gen_raise_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->clear_x = 1;
@@ -3226,7 +3193,6 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
             cris_clear_x_flag(dc);
         }
 
-        num_insns++;
         /* Check for delayed branches here. If we do it before
            actually generating any host code, the simulator will just
            loop doing nothing for on this program location.  */
@@ -3331,16 +3297,8 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !DISAS_CRIS
@@ -3354,16 +3312,6 @@ gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
 #endif
 }
 
-void gen_intermediate_code (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, true);
-}
-
 void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -3416,47 +3364,48 @@ void cris_initialize_tcg(void)
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cc_x = tcg_global_mem_new(TCG_AREG0,
+    cc_x = tcg_global_mem_new(cpu_env,
                               offsetof(CPUCRISState, cc_x), "cc_x");
-    cc_src = tcg_global_mem_new(TCG_AREG0,
+    cc_src = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUCRISState, cc_src), "cc_src");
-    cc_dest = tcg_global_mem_new(TCG_AREG0,
+    cc_dest = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_dest),
                                  "cc_dest");
-    cc_result = tcg_global_mem_new(TCG_AREG0,
+    cc_result = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUCRISState, cc_result),
                                    "cc_result");
-    cc_op = tcg_global_mem_new(TCG_AREG0,
+    cc_op = tcg_global_mem_new(cpu_env,
                                offsetof(CPUCRISState, cc_op), "cc_op");
-    cc_size = tcg_global_mem_new(TCG_AREG0,
+    cc_size = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_size),
                                  "cc_size");
-    cc_mask = tcg_global_mem_new(TCG_AREG0,
+    cc_mask = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_mask),
                                  "cc_mask");
 
-    env_pc = tcg_global_mem_new(TCG_AREG0,
+    env_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUCRISState, pc),
                                 "pc");
-    env_btarget = tcg_global_mem_new(TCG_AREG0,
+    env_btarget = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUCRISState, btarget),
                                      "btarget");
-    env_btaken = tcg_global_mem_new(TCG_AREG0,
+    env_btaken = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUCRISState, btaken),
                                     "btaken");
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                                       offsetof(CPUCRISState, regs[i]),
                                       regnames[i]);
     }
     for (i = 0; i < 16; i++) {
-        cpu_PR[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_PR[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUCRISState, pregs[i]),
                                        pregnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-cris/translate_v10.c b/qemu/target-cris/translate_v10.c
index b742c4cd0..7607eadfb 100644
--- a/qemu/target-cris/translate_v10.c
+++ b/qemu/target-cris/translate_v10.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "crisv10-decode.h"
 
 static const char *regnames_v10[] =
@@ -58,7 +59,7 @@ static inline int dec10_size(unsigned int size)
 
 static inline void cris_illegal_insn(DisasContext *dc)
 {
-    qemu_log("illegal insn at pc=%x\n", dc->pc);
+    qemu_log_mask(LOG_GUEST_ERROR, "illegal insn at pc=%x\n", dc->pc);
     t_gen_raise_exception(EXCP_BREAK);
 }
 
@@ -96,7 +97,7 @@ static void gen_store_v10_conditional(DisasContext *dc, TCGv addr, TCGv val,
 static void gen_store_v10(DisasContext *dc, TCGv addr, TCGv val,
                        unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -535,16 +536,8 @@ static void dec10_reg_scc(DisasContext *dc)
 
     LOG_DIS("s%s $r%u\n", cc_name(cond), dc->src);
 
-    if (cond != CC_A)
-    {
-        TCGLabel *l1 = gen_new_label();
-        gen_tst_cc (dc, cpu_R[dc->src], cond);
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->src], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->src], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->src], cpu_R[dc->src], 0);
 
     cris_cc_mask(dc, 0);
 }
@@ -1207,9 +1200,6 @@ static unsigned int crisv10_decoder(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
-        tcg_gen_debug_insn_start(dc->pc);
-
     /* Load a halfword onto the instruction register.  */
     dc->ir = cpu_lduw_code(env, dc->pc);
 
@@ -1257,45 +1247,45 @@ static unsigned int crisv10_decoder(CPUCRISState *env, DisasContext *dc)
 
 void cris_initialize_crisv10_tcg(void)
 {
-	int i;
-
-	cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-	cc_x = tcg_global_mem_new(TCG_AREG0,
-				  offsetof(CPUCRISState, cc_x), "cc_x");
-	cc_src = tcg_global_mem_new(TCG_AREG0,
-				    offsetof(CPUCRISState, cc_src), "cc_src");
-	cc_dest = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_dest),
-				     "cc_dest");
-	cc_result = tcg_global_mem_new(TCG_AREG0,
-				       offsetof(CPUCRISState, cc_result),
-				       "cc_result");
-	cc_op = tcg_global_mem_new(TCG_AREG0,
-				   offsetof(CPUCRISState, cc_op), "cc_op");
-	cc_size = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_size),
-				     "cc_size");
-	cc_mask = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_mask),
-				     "cc_mask");
-
-	env_pc = tcg_global_mem_new(TCG_AREG0, 
-				    offsetof(CPUCRISState, pc),
-				    "pc");
-	env_btarget = tcg_global_mem_new(TCG_AREG0,
-					 offsetof(CPUCRISState, btarget),
-					 "btarget");
-	env_btaken = tcg_global_mem_new(TCG_AREG0,
-					 offsetof(CPUCRISState, btaken),
-					 "btaken");
-	for (i = 0; i < 16; i++) {
-		cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
-					      offsetof(CPUCRISState, regs[i]),
-					      regnames_v10[i]);
-	}
-	for (i = 0; i < 16; i++) {
-		cpu_PR[i] = tcg_global_mem_new(TCG_AREG0,
-					       offsetof(CPUCRISState, pregs[i]),
-					       pregnames_v10[i]);
-	}
+    int i;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    cc_x = tcg_global_mem_new(cpu_env,
+                              offsetof(CPUCRISState, cc_x), "cc_x");
+    cc_src = tcg_global_mem_new(cpu_env,
+                                offsetof(CPUCRISState, cc_src), "cc_src");
+    cc_dest = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_dest),
+                                 "cc_dest");
+    cc_result = tcg_global_mem_new(cpu_env,
+                                   offsetof(CPUCRISState, cc_result),
+                                   "cc_result");
+    cc_op = tcg_global_mem_new(cpu_env,
+                               offsetof(CPUCRISState, cc_op), "cc_op");
+    cc_size = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_size),
+                                 "cc_size");
+    cc_mask = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_mask),
+                                 "cc_mask");
+
+    env_pc = tcg_global_mem_new(cpu_env,
+                                offsetof(CPUCRISState, pc),
+                                "pc");
+    env_btarget = tcg_global_mem_new(cpu_env,
+                                     offsetof(CPUCRISState, btarget),
+                                     "btarget");
+    env_btaken = tcg_global_mem_new(cpu_env,
+                                    offsetof(CPUCRISState, btaken),
+                                    "btaken");
+    for (i = 0; i < 16; i++) {
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
+                                      offsetof(CPUCRISState, regs[i]),
+                                      regnames_v10[i]);
+    }
+    for (i = 0; i < 16; i++) {
+        cpu_PR[i] = tcg_global_mem_new(cpu_env,
+                                       offsetof(CPUCRISState, pregs[i]),
+                                       pregnames_v10[i]);
+    }
 }
diff --git a/qemu/target-i386/Makefile.objs b/qemu/target-i386/Makefile.objs
index 7a1df2c98..b223d7932 100644
--- a/qemu/target-i386/Makefile.objs
+++ b/qemu/target-i386/Makefile.objs
@@ -1,7 +1,7 @@
-obj-y += translate.o helper.o cpu.o
+obj-y += translate.o helper.o cpu.o bpt_helper.o
 obj-y += excp_helper.o fpu_helper.o cc_helper.o int_helper.o svm_helper.o
-obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o
+obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o mpx_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o
-obj-$(CONFIG_KVM) += kvm.o
+obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o monitor.o
+obj-$(CONFIG_KVM) += kvm.o hyperv.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
diff --git a/qemu/target-i386/arch_dump.c b/qemu/target-i386/arch_dump.c
index eccd8031a..5a2e4be5d 100644
--- a/qemu/target-i386/arch_dump.c
+++ b/qemu/target-i386/arch_dump.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "sysemu/dump.h"
diff --git a/qemu/target-i386/arch_memory_mapping.c b/qemu/target-i386/arch_memory_mapping.c
index 01563fecc..88f341e1b 100644
--- a/qemu/target-i386/arch_memory_mapping.c
+++ b/qemu/target-i386/arch_memory_mapping.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "sysemu/memory_mapping.h"
diff --git a/qemu/target-i386/bpt_helper.c b/qemu/target-i386/bpt_helper.c
new file mode 100644
index 000000000..f47df1998
--- /dev/null
+++ b/qemu/target-i386/bpt_helper.c
@@ -0,0 +1,327 @@
+/*
+ *  i386 breakpoint helpers
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+
+#ifndef CONFIG_USER_ONLY
+static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 1;
+}
+
+static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 2;
+
+}
+static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return hw_global_breakpoint_enabled(dr7, index) ||
+           hw_local_breakpoint_enabled(dr7, index);
+}
+
+static inline int hw_breakpoint_type(unsigned long dr7, int index)
+{
+    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
+}
+
+static inline int hw_breakpoint_len(unsigned long dr7, int index)
+{
+    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
+    return (len == 2) ? 8 : len + 1;
+}
+
+static int hw_breakpoint_insert(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+    target_ulong dr7 = env->dr[7];
+    target_ulong drN = env->dr[index];
+    int err = 0;
+
+    switch (hw_breakpoint_type(dr7, index)) {
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_breakpoint_insert(cs, drN, BP_CPU,
+                                        &env->cpu_breakpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* Notice when we should enable calls to bpt_io.  */
+        return hw_breakpoint_enabled(env->dr[7], index)
+               ? HF_IOBPT_MASK : 0;
+
+    case DR7_TYPE_DATA_WR:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_WRITE,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_DATA_RW:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_ACCESS,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+    }
+    if (err) {
+        env->cpu_breakpoint[index] = NULL;
+    }
+    return 0;
+}
+
+static void hw_breakpoint_remove(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+
+    switch (hw_breakpoint_type(env->dr[7], index)) {
+    case DR7_TYPE_BP_INST:
+        if (env->cpu_breakpoint[index]) {
+            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_DATA_WR:
+    case DR7_TYPE_DATA_RW:
+        if (env->cpu_breakpoint[index]) {
+            cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* HF_IOBPT_MASK cleared elsewhere.  */
+        break;
+    }
+}
+
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7)
+{
+    target_ulong old_dr7 = env->dr[7];
+    int iobpt = 0;
+    int i;
+
+    new_dr7 |= DR7_FIXED_1;
+
+    /* If nothing is changing except the global/local enable bits,
+       then we can make the change more efficient.  */
+    if (((old_dr7 ^ new_dr7) & ~0xff) == 0) {
+        /* Fold the global and local enable bits together into the
+           global fields, then xor to show which registers have
+           changed collective enable state.  */
+        int mod = ((old_dr7 | old_dr7 * 2) ^ (new_dr7 | new_dr7 * 2)) & 0xff;
+
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if ((mod & (2 << i * 2)) && !hw_breakpoint_enabled(new_dr7, i)) {
+                hw_breakpoint_remove(env, i);
+            }
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if (mod & (2 << i * 2) && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= hw_breakpoint_insert(env, i);
+            } else if (hw_breakpoint_type(new_dr7, i) == DR7_TYPE_IO_RW
+                       && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= HF_IOBPT_MASK;
+            }
+        }
+    } else {
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            hw_breakpoint_remove(env, i);
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            iobpt |= hw_breakpoint_insert(env, i);
+        }
+    }
+
+    env->hflags = (env->hflags & ~HF_IOBPT_MASK) | iobpt;
+}
+
+static bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
+{
+    target_ulong dr6;
+    int reg;
+    bool hit_enabled = false;
+
+    dr6 = env->dr[6] & ~0xf;
+    for (reg = 0; reg < DR7_MAX_BP; reg++) {
+        bool bp_match = false;
+        bool wp_match = false;
+
+        switch (hw_breakpoint_type(env->dr[7], reg)) {
+        case DR7_TYPE_BP_INST:
+            if (env->dr[reg] == env->eip) {
+                bp_match = true;
+            }
+            break;
+        case DR7_TYPE_DATA_WR:
+        case DR7_TYPE_DATA_RW:
+            if (env->cpu_watchpoint[reg] &&
+                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
+                wp_match = true;
+            }
+            break;
+        case DR7_TYPE_IO_RW:
+            break;
+        }
+        if (bp_match || wp_match) {
+            dr6 |= 1 << reg;
+            if (hw_breakpoint_enabled(env->dr[7], reg)) {
+                hit_enabled = true;
+            }
+        }
+    }
+
+    if (hit_enabled || force_dr6_update) {
+        env->dr[6] = dr6;
+    }
+
+    return hit_enabled;
+}
+
+void breakpoint_handler(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    CPUBreakpoint *bp;
+
+    if (cs->watchpoint_hit) {
+        if (cs->watchpoint_hit->flags & BP_CPU) {
+            cs->watchpoint_hit = NULL;
+            if (check_hw_breakpoints(env, false)) {
+                raise_exception(env, EXCP01_DB);
+            } else {
+                cpu_resume_from_signal(cs, NULL);
+            }
+        }
+    } else {
+        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
+            if (bp->pc == env->eip) {
+                if (bp->flags & BP_CPU) {
+                    check_hw_breakpoints(env, true);
+                    raise_exception(env, EXCP01_DB);
+                }
+                break;
+            }
+        }
+    }
+}
+#endif
+
+void helper_single_step(CPUX86State *env)
+{
+#ifndef CONFIG_USER_ONLY
+    check_hw_breakpoints(env, true);
+    env->dr[6] |= DR6_BS;
+#endif
+    raise_exception(env, EXCP01_DB);
+}
+
+void helper_set_dr(CPUX86State *env, int reg, target_ulong t0)
+{
+#ifndef CONFIG_USER_ONLY
+    switch (reg) {
+    case 0: case 1: case 2: case 3:
+        if (hw_breakpoint_enabled(env->dr[7], reg)
+            && hw_breakpoint_type(env->dr[7], reg) != DR7_TYPE_IO_RW) {
+            hw_breakpoint_remove(env, reg);
+            env->dr[reg] = t0;
+            hw_breakpoint_insert(env, reg);
+        } else {
+            env->dr[reg] = t0;
+        }
+        return;
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 6:
+        env->dr[6] = t0 | DR6_FIXED_1;
+        return;
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 7:
+        cpu_x86_update_dr7(env, t0);
+        return;
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+#endif
+}
+
+target_ulong helper_get_dr(CPUX86State *env, int reg)
+{
+    switch (reg) {
+    case 0: case 1: case 2: case 3: case 6: case 7:
+        return env->dr[reg];
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[6];
+        }
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[7];
+        }
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+}
+
+/* Check if Port I/O is trapped by a breakpoint.  */
+void helper_bpt_io(CPUX86State *env, uint32_t port,
+                   uint32_t size, target_ulong next_eip)
+{
+#ifndef CONFIG_USER_ONLY
+    target_ulong dr7 = env->dr[7];
+    int i, hit = 0;
+
+    for (i = 0; i < DR7_MAX_BP; ++i) {
+        if (hw_breakpoint_type(dr7, i) == DR7_TYPE_IO_RW
+            && hw_breakpoint_enabled(dr7, i)) {
+            int bpt_len = hw_breakpoint_len(dr7, i);
+            if (port + size - 1 >= env->dr[i]
+                && port <= env->dr[i] + bpt_len - 1) {
+                hit |= 1 << i;
+            }
+        }
+    }
+
+    if (hit) {
+        env->dr[6] = (env->dr[6] & ~0xf) | hit;
+        env->eip = next_eip;
+        raise_exception(env, EXCP01_DB);
+    }
+#endif
+}
diff --git a/qemu/target-i386/cc_helper.c b/qemu/target-i386/cc_helper.c
index ecbf0ec09..83af223c9 100644
--- a/qemu/target-i386/cc_helper.c
+++ b/qemu/target-i386/cc_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -378,17 +379,7 @@ void helper_sti_vm(CPUX86State *env)
 {
     env->eflags |= VIF_MASK;
     if (env->eflags & VIP_MASK) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
 }
 #endif
-
-void helper_set_inhibit_irq(CPUX86State *env)
-{
-    env->hflags |= HF_INHIBIT_IRQ_MASK;
-}
-
-void helper_reset_inhibit_irq(CPUX86State *env)
-{
-    env->hflags &= ~HF_INHIBIT_IRQ_MASK;
-}
diff --git a/qemu/target-i386/cpu-qom.h b/qemu/target-i386/cpu-qom.h
index 7a4fddd85..cb750176c 100644
--- a/qemu/target-i386/cpu-qom.h
+++ b/qemu/target-i386/cpu-qom.h
@@ -22,7 +22,6 @@
 
 #include "qom/cpu.h"
 #include "cpu.h"
-#include "qapi/error.h"
 #include "qemu/notify.h"
 
 #ifdef TARGET_X86_64
@@ -88,7 +87,14 @@ typedef struct X86CPU {
     bool hyperv_vapic;
     bool hyperv_relaxed_timing;
     int hyperv_spinlock_attempts;
+    char *hyperv_vendor_id;
     bool hyperv_time;
+    bool hyperv_crash;
+    bool hyperv_reset;
+    bool hyperv_vpindex;
+    bool hyperv_runtime;
+    bool hyperv_synic;
+    bool hyperv_stimer;
     bool check_cpuid;
     bool enforce_cpuid;
     bool expose_kvm;
diff --git a/qemu/target-i386/cpu.c b/qemu/target-i386/cpu.c
index 7a779b165..d0b5b6915 100644
--- a/qemu/target-i386/cpu.c
+++ b/qemu/target-i386/cpu.c
@@ -16,10 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "cpu.h"
 #include "sysemu/kvm.h"
@@ -43,7 +41,6 @@
 
 #include "sysemu/sysemu.h"
 #include "hw/qdev-properties.h"
-#include "hw/cpu/icc_bus.h"
 #ifndef CONFIG_USER_ONLY
 #include "exec/address-spaces.h"
 #include "hw/xen/xen.h"
@@ -260,8 +257,19 @@ static const char *svm_feature_name[] = {
 static const char *cpuid_7_0_ebx_feature_name[] = {
     "fsgsbase", "tsc_adjust", NULL, "bmi1", "hle", "avx2", NULL, "smep",
     "bmi2", "erms", "invpcid", "rtm", NULL, NULL, "mpx", NULL,
-    "avx512f", NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
-    NULL, NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
+    "avx512f", NULL, "rdseed", "adx", "smap", NULL, "pcommit", "clflushopt",
+    "clwb", NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
+};
+
+static const char *cpuid_7_0_ecx_feature_name[] = {
+    NULL, NULL, NULL, "pku",
+    "ospke", NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
 };
 
 static const char *cpuid_apm_edx_feature_name[] = {
@@ -313,7 +321,7 @@ static const char *cpuid_6_feature_name[] = {
           CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | CPUID_SEP | \
           CPUID_MTRR | CPUID_PGE | CPUID_MCA | CPUID_CMOV | CPUID_PAT | \
           CPUID_PSE36 | CPUID_CLFLUSH | CPUID_ACPI | CPUID_MMX | \
-          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS)
+          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS | CPUID_DE)
           /* partly implemented:
           CPUID_MTRR, CPUID_MCA, CPUID_CLFLUSH (needed for Win64) */
           /* missing:
@@ -321,14 +329,14 @@ static const char *cpuid_6_feature_name[] = {
 #define TCG_EXT_FEATURES (CPUID_EXT_SSE3 | CPUID_EXT_PCLMULQDQ | \
           CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 | CPUID_EXT_CX16 | \
           CPUID_EXT_SSE41 | CPUID_EXT_SSE42 | CPUID_EXT_POPCNT | \
+          CPUID_EXT_XSAVE | /* CPUID_EXT_OSXSAVE is dynamic */   \
           CPUID_EXT_MOVBE | CPUID_EXT_AES | CPUID_EXT_HYPERVISOR)
           /* missing:
           CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_SMX,
           CPUID_EXT_EST, CPUID_EXT_TM2, CPUID_EXT_CID, CPUID_EXT_FMA,
           CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_PCID, CPUID_EXT_DCA,
-          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER, CPUID_EXT_XSAVE,
-          CPUID_EXT_OSXSAVE, CPUID_EXT_AVX, CPUID_EXT_F16C,
-          CPUID_EXT_RDRAND */
+          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER, CPUID_EXT_AVX,
+          CPUID_EXT_F16C, CPUID_EXT_RDRAND */
 
 #ifdef TARGET_X86_64
 #define TCG_EXT2_X86_64_FEATURES (CPUID_EXT2_SYSCALL | CPUID_EXT2_LM)
@@ -346,14 +354,19 @@ static const char *cpuid_6_feature_name[] = {
 #define TCG_SVM_FEATURES 0
 #define TCG_KVM_FEATURES 0
 #define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP | \
-          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX)
+          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX | \
+          CPUID_7_0_EBX_PCOMMIT | CPUID_7_0_EBX_CLFLUSHOPT |            \
+          CPUID_7_0_EBX_CLWB | CPUID_7_0_EBX_MPX | CPUID_7_0_EBX_FSGSBASE)
           /* missing:
-          CPUID_7_0_EBX_FSGSBASE, CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
+          CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
           CPUID_7_0_EBX_ERMS, CPUID_7_0_EBX_INVPCID, CPUID_7_0_EBX_RTM,
           CPUID_7_0_EBX_RDSEED */
+#define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | CPUID_7_0_ECX_OSPKE)
 #define TCG_APM_FEATURES 0
 #define TCG_6_EAX_FEATURES CPUID_6_EAX_ARAT
-
+#define TCG_XSAVE_FEATURES (CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1)
+          /* missing:
+          CPUID_XSAVE_XSAVEC, CPUID_XSAVE_XSAVES */
 
 typedef struct FeatureWordInfo {
     const char **feat_names;
@@ -408,6 +421,13 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .cpuid_reg = R_EBX,
         .tcg_features = TCG_7_0_EBX_FEATURES,
     },
+    [FEAT_7_0_ECX] = {
+        .feat_names = cpuid_7_0_ecx_feature_name,
+        .cpuid_eax = 7,
+        .cpuid_needs_ecx = true, .cpuid_ecx = 0,
+        .cpuid_reg = R_ECX,
+        .tcg_features = TCG_7_0_ECX_FEATURES,
+    },
     [FEAT_8000_0007_EDX] = {
         .feat_names = cpuid_apm_edx_feature_name,
         .cpuid_eax = 0x80000007,
@@ -420,7 +440,7 @@ static FeatureWordInfo feature_word_info[FEATURE_WORDS] = {
         .cpuid_eax = 0xd,
         .cpuid_needs_ecx = true, .cpuid_ecx = 1,
         .cpuid_reg = R_EAX,
-        .tcg_features = 0,
+        .tcg_features = TCG_XSAVE_FEATURES,
     },
     [FEAT_6_EAX] = {
         .feat_names = cpuid_6_feature_name,
@@ -450,24 +470,28 @@ static const X86RegisterInfo32 x86_reg_info_32[CPU_NB_REGS32] = {
 };
 #undef REGISTER
 
-typedef struct ExtSaveArea {
-    uint32_t feature, bits;
-    uint32_t offset, size;
-} ExtSaveArea;
-
-static const ExtSaveArea ext_save_areas[] = {
-    [2] = { .feature = FEAT_1_ECX, .bits = CPUID_EXT_AVX,
+const ExtSaveArea x86_ext_save_areas[] = {
+    [XSTATE_YMM_BIT] =
+          { .feature = FEAT_1_ECX, .bits = CPUID_EXT_AVX,
             .offset = 0x240, .size = 0x100 },
-    [3] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
+    [XSTATE_BNDREGS_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
             .offset = 0x3c0, .size = 0x40  },
-    [4] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
+    [XSTATE_BNDCSR_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
             .offset = 0x400, .size = 0x40  },
-    [5] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+    [XSTATE_OPMASK_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
             .offset = 0x440, .size = 0x40 },
-    [6] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+    [XSTATE_ZMM_Hi256_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
             .offset = 0x480, .size = 0x200 },
-    [7] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+    [XSTATE_Hi16_ZMM_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
             .offset = 0x680, .size = 0x400 },
+    [XSTATE_PKRU_BIT] =
+          { .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_PKU,
+            .offset = 0xA80, .size = 0x8 },
 };
 
 const char *get_register_name_32(unsigned int reg)
@@ -478,38 +502,6 @@ const char *get_register_name_32(unsigned int reg)
     return x86_reg_info_32[reg].name;
 }
 
-/* KVM-specific features that are automatically added to all CPU models
- * when KVM is enabled.
- */
-static uint32_t kvm_default_features[FEATURE_WORDS] = {
-    [FEAT_KVM] = (1 << KVM_FEATURE_CLOCKSOURCE) |
-        (1 << KVM_FEATURE_NOP_IO_DELAY) |
-        (1 << KVM_FEATURE_CLOCKSOURCE2) |
-        (1 << KVM_FEATURE_ASYNC_PF) |
-        (1 << KVM_FEATURE_STEAL_TIME) |
-        (1 << KVM_FEATURE_PV_EOI) |
-        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT),
-    [FEAT_1_ECX] = CPUID_EXT_X2APIC,
-};
-
-/* Features that are not added by default to any CPU model when KVM is enabled.
- */
-static uint32_t kvm_default_unset_features[FEATURE_WORDS] = {
-    [FEAT_1_EDX] = CPUID_ACPI,
-    [FEAT_1_ECX] = CPUID_EXT_MONITOR,
-    [FEAT_8000_0001_ECX] = CPUID_EXT3_SVM,
-};
-
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features)
-{
-    kvm_default_features[w] &= ~features;
-}
-
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features)
-{
-    kvm_default_unset_features[w] &= ~features;
-}
-
 /*
  * Returns the set of feature flags that are supported and migratable by
  * QEMU, for a given FeatureWord.
@@ -689,7 +681,6 @@ struct X86CPUDefinition {
     int stepping;
     FeatureWordArray features;
     char model_id[48];
-    bool cache_info_passthrough;
 };
 
 static X86CPUDefinition builtin_x86_defs[] = {
@@ -705,12 +696,11 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
-            CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
+            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM,
         .xlevel = 0x8000000A,
     },
     {
@@ -806,7 +796,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .features[FEAT_1_EDX] =
             PPRO_FEATURES,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3,
         .xlevel = 0x80000004,
     },
     {
@@ -1113,7 +1103,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1148,7 +1138,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1185,7 +1175,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1223,7 +1213,7 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
@@ -1277,8 +1267,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_DE | CPUID_FP87,
         .features[FEAT_1_ECX] =
             CPUID_EXT_CX16 | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1306,8 +1297,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
         .features[FEAT_1_ECX] =
             CPUID_EXT_POPCNT | CPUID_EXT_CX16 | CPUID_EXT_MONITOR |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1338,8 +1330,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1373,8 +1366,9 @@ static X86CPUDefinition builtin_x86_defs[] = {
             CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_FMA |
             CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1392,30 +1386,41 @@ static X86CPUDefinition builtin_x86_defs[] = {
     },
 };
 
-/**
- * x86_cpu_compat_set_features:
- * @cpu_model: CPU model name to be changed. If NULL, all CPU models are changed
- * @w: Identifies the feature word to be changed.
- * @feat_add: Feature bits to be added to feature word
- * @feat_remove: Feature bits to be removed from feature word
- *
- * Change CPU model feature bits for compatibility.
- *
- * This function may be used by machine-type compatibility functions
- * to enable or disable feature bits on specific CPU models.
+typedef struct PropValue {
+    const char *prop, *value;
+} PropValue;
+
+/* KVM-specific features that are automatically added/removed
+ * from all CPU models when KVM is enabled.
  */
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove)
+static PropValue kvm_default_props[] = {
+    { "kvmclock", "on" },
+    { "kvm-nopiodelay", "on" },
+    { "kvm-asyncpf", "on" },
+    { "kvm-steal-time", "on" },
+    { "kvm-pv-eoi", "on" },
+    { "kvmclock-stable-bit", "on" },
+    { "x2apic", "on" },
+    { "acpi", "off" },
+    { "monitor", "off" },
+    { "svm", "off" },
+    { NULL, NULL },
+};
+
+void x86_cpu_change_kvm_default(const char *prop, const char *value)
 {
-    X86CPUDefinition *def;
-    int i;
-    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
-        def = &builtin_x86_defs[i];
-        if (!cpu_model || !strcmp(cpu_model, def->name)) {
-            def->features[w] |= feat_add;
-            def->features[w] &= ~feat_remove;
+    PropValue *pv;
+    for (pv = kvm_default_props; pv->prop; pv++) {
+        if (!strcmp(pv->prop, prop)) {
+            pv->value = value;
+            break;
         }
     }
+
+    /* It is valid to call this function only for properties that
+     * are already present in the kvm_default_props table.
+     */
+    assert(pv->prop);
 }
 
 static uint32_t x86_cpu_get_supported_feature_word(FeatureWord w,
@@ -1442,6 +1447,7 @@ static X86CPUDefinition host_cpudef;
 
 static Property host_x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("migratable", X86CPU, migratable, true),
+    DEFINE_PROP_BOOL("host-cache-info", X86CPU, cache_info_passthrough, false),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -1468,13 +1474,14 @@ static void host_x86_cpu_class_init(ObjectClass *oc, void *data)
     cpu_x86_fill_model_id(host_cpudef.model_id);
 
     xcc->cpu_def = &host_cpudef;
-    host_cpudef.cache_info_passthrough = true;
 
     /* level, xlevel, xlevel2, and the feature words are initialized on
      * instance_init, because they require KVM to be initialized.
      */
 
     dc->props = host_x86_cpu_properties;
+    /* Reason: host_x86_cpu_initfn() dies when !kvm_enabled() */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void host_x86_cpu_initfn(Object *obj)
@@ -1512,7 +1519,7 @@ static void report_unavailable_features(FeatureWord w, uint32_t mask)
     int i;
 
     for (i = 0; i < 32; ++i) {
-        if (1 << i & mask) {
+        if ((1UL << i) & mask) {
             const char *reg = get_register_name_32(f->cpuid_reg);
             assert(reg);
             fprintf(stderr, "warning: %s doesn't support requested feature: "
@@ -1525,8 +1532,9 @@ static void report_unavailable_features(FeatureWord w, uint32_t mask)
     }
 }
 
-static void x86_cpuid_version_get_family(Object *obj, Visitor *v, void *opaque,
-                                         const char *name, Error **errp)
+static void x86_cpuid_version_get_family(Object *obj, Visitor *v,
+                                         const char *name, void *opaque,
+                                         Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1536,11 +1544,12 @@ static void x86_cpuid_version_get_family(Object *obj, Visitor *v, void *opaque,
     if (value == 0xf) {
         value += (env->cpuid_version >> 20) & 0xff;
     }
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_version_set_family(Object *obj, Visitor *v, void *opaque,
-                                         const char *name, Error **errp)
+static void x86_cpuid_version_set_family(Object *obj, Visitor *v,
+                                         const char *name, void *opaque,
+                                         Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1549,7 +1558,7 @@ static void x86_cpuid_version_set_family(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1568,8 +1577,9 @@ static void x86_cpuid_version_set_family(Object *obj, Visitor *v, void *opaque,
     }
 }
 
-static void x86_cpuid_version_get_model(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void x86_cpuid_version_get_model(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1577,11 +1587,12 @@ static void x86_cpuid_version_get_model(Object *obj, Visitor *v, void *opaque,
 
     value = (env->cpuid_version >> 4) & 0xf;
     value |= ((env->cpuid_version >> 16) & 0xf) << 4;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_version_set_model(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void x86_cpuid_version_set_model(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1590,7 +1601,7 @@ static void x86_cpuid_version_set_model(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1606,7 +1617,7 @@ static void x86_cpuid_version_set_model(Object *obj, Visitor *v, void *opaque,
 }
 
 static void x86_cpuid_version_get_stepping(Object *obj, Visitor *v,
-                                           void *opaque, const char *name,
+                                           const char *name, void *opaque,
                                            Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
@@ -1614,11 +1625,11 @@ static void x86_cpuid_version_get_stepping(Object *obj, Visitor *v,
     int64_t value;
 
     value = env->cpuid_version & 0xf;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void x86_cpuid_version_set_stepping(Object *obj, Visitor *v,
-                                           void *opaque, const char *name,
+                                           const char *name, void *opaque,
                                            Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
@@ -1628,7 +1639,7 @@ static void x86_cpuid_version_set_stepping(Object *obj, Visitor *v,
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1714,18 +1725,18 @@ static void x86_cpuid_set_model_id(Object *obj, const char *model_id,
     }
 }
 
-static void x86_cpuid_get_tsc_freq(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void x86_cpuid_get_tsc_freq(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     int64_t value;
 
     value = cpu->env.tsc_khz * 1000;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     const int64_t min = 0;
@@ -1733,7 +1744,7 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1744,20 +1755,20 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    cpu->env.tsc_khz = value / 1000;
+    cpu->env.tsc_khz = cpu->env.user_tsc_khz = value / 1000;
 }
 
-static void x86_cpuid_get_apic_id(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void x86_cpuid_get_apic_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     int64_t value = cpu->apic_id;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_set_apic_id(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void x86_cpuid_set_apic_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     DeviceState *dev = DEVICE(obj);
@@ -1772,7 +1783,7 @@ static void x86_cpuid_set_apic_id(Object *obj, Visitor *v, void *opaque,
         return;
     }
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -1792,8 +1803,9 @@ static void x86_cpuid_set_apic_id(Object *obj, Visitor *v, void *opaque,
 }
 
 /* Generic getter for "feature-words" and "filtered-features" properties */
-static void x86_cpu_get_feature_words(Object *obj, Visitor *v, void *opaque,
-                                      const char *name, Error **errp)
+static void x86_cpu_get_feature_words(Object *obj, Visitor *v,
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     uint32_t *array = (uint32_t *)opaque;
     FeatureWord w;
@@ -1817,21 +1829,21 @@ static void x86_cpu_get_feature_words(Object *obj, Visitor *v, void *opaque,
         list = &list_entries[w];
     }
 
-    visit_type_X86CPUFeatureWordInfoList(v, &list, "feature-words", &err);
+    visit_type_X86CPUFeatureWordInfoList(v, "feature-words", &list, &err);
     error_propagate(errp, err);
 }
 
-static void x86_get_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void x86_get_hv_spinlocks(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     int64_t value = cpu->hyperv_spinlock_attempts;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_set_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void x86_set_hv_spinlocks(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     const int64_t min = 0xFFF;
     const int64_t max = UINT_MAX;
@@ -1839,7 +1851,7 @@ static void x86_set_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
     Error *err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &err);
+    visit_type_int(v, name, &value, &err);
     if (err) {
         error_propagate(errp, err);
         return;
@@ -1919,8 +1931,8 @@ static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
                 char *err;
                 char num[32];
 
-                tsc_freq = strtosz_suffix_unit(val, &err,
-                                               STRTOSZ_DEFSUFFIX_B, 1000);
+                tsc_freq = qemu_strtosz_suffix_unit(val, &err,
+                                               QEMU_STRTOSZ_DEFSUFFIX_B, 1000);
                 if (tsc_freq < 0 || *err) {
                     error_setg(errp, "bad numerical value %s", val);
                     return;
@@ -2087,6 +2099,18 @@ static int x86_cpu_filter_features(X86CPU *cpu)
     return rv;
 }
 
+static void x86_cpu_apply_props(X86CPU *cpu, PropValue *props)
+{
+    PropValue *pv;
+    for (pv = props; pv->prop; pv++) {
+        if (!pv->value) {
+            continue;
+        }
+        object_property_parse(OBJECT(cpu), pv->value, pv->prop,
+                              &error_abort);
+    }
+}
+
 /* Load data from X86CPUDefinition
  */
 static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
@@ -2102,7 +2126,6 @@ static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
     object_property_set_int(OBJECT(cpu), def->stepping, "stepping", errp);
     object_property_set_int(OBJECT(cpu), def->xlevel, "xlevel", errp);
     object_property_set_int(OBJECT(cpu), def->xlevel2, "xlevel2", errp);
-    cpu->cache_info_passthrough = def->cache_info_passthrough;
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", errp);
     for (w = 0; w < FEATURE_WORDS; w++) {
         env->features[w] = def->features[w];
@@ -2110,11 +2133,11 @@ static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
 
     /* Special cases not set in the X86CPUDefinition structs: */
     if (kvm_enabled()) {
-        FeatureWord w;
-        for (w = 0; w < FEATURE_WORDS; w++) {
-            env->features[w] |= kvm_default_features[w];
-            env->features[w] &= ~kvm_default_unset_features[w];
+        if (!kvm_irqchip_in_kernel()) {
+            x86_cpu_change_kvm_default("x2apic", "off");
         }
+
+        x86_cpu_apply_props(cpu, kvm_default_props);
     }
 
     env->features[FEAT_1_ECX] |= CPUID_EXT_HYPERVISOR;
@@ -2257,7 +2280,7 @@ void x86_cpudef_setup(void)
                 pstrcpy(def->model_id, sizeof(def->model_id),
                         "QEMU Virtual CPU version ");
                 pstrcat(def->model_id, sizeof(def->model_id),
-                        qemu_get_version());
+                        qemu_hw_version());
                 break;
             }
         }
@@ -2306,10 +2329,13 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = (cpu->apic_id << 24) |
                8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
         *ecx = env->features[FEAT_1_ECX];
+        if ((*ecx & CPUID_EXT_XSAVE) && (env->cr[4] & CR4_OSXSAVE_MASK)) {
+            *ecx |= CPUID_EXT_OSXSAVE;
+        }
         *edx = env->features[FEAT_1_EDX];
         if (cs->nr_cores * cs->nr_threads > 1) {
             *ebx |= (cs->nr_cores * cs->nr_threads) << 16;
-            *edx |= 1 << 28;    /* HTT bit */
+            *edx |= CPUID_HT;
         }
         break;
     case 2:
@@ -2399,7 +2425,10 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         if (count == 0) {
             *eax = 0; /* Maximum ECX value for sub-leaves */
             *ebx = env->features[FEAT_7_0_EBX]; /* Feature flags */
-            *ecx = 0; /* Reserved */
+            *ecx = env->features[FEAT_7_0_ECX]; /* Feature flags */
+            if ((*ecx & CPUID_7_0_ECX_PKU) && env->cr[4] & CR4_PKE_MASK) {
+                *ecx |= CPUID_7_0_ECX_OSPKE;
+            }
             *edx = 0; /* Reserved */
         } else {
             *eax = 0;
@@ -2433,7 +2462,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         break;
     case 0xD: {
         KVMState *s = cs->kvm_state;
-        uint64_t kvm_mask;
+        uint64_t ena_mask;
         int i;
 
         /* Processor Extended State */
@@ -2441,35 +2470,39 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         *ebx = 0;
         *ecx = 0;
         *edx = 0;
-        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) || !kvm_enabled()) {
+        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) {
             break;
         }
-        kvm_mask =
-            kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX) |
-            ((uint64_t)kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX) << 32);
+        if (kvm_enabled()) {
+            ena_mask = kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX);
+            ena_mask <<= 32;
+            ena_mask |= kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX);
+        } else {
+            ena_mask = -1;
+        }
 
         if (count == 0) {
             *ecx = 0x240;
-            for (i = 2; i < ARRAY_SIZE(ext_save_areas); i++) {
-                const ExtSaveArea *esa = &ext_save_areas[i];
-                if ((env->features[esa->feature] & esa->bits) == esa->bits &&
-                    (kvm_mask & (1 << i)) != 0) {
+            for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+                const ExtSaveArea *esa = &x86_ext_save_areas[i];
+                if ((env->features[esa->feature] & esa->bits) == esa->bits
+                    && ((ena_mask >> i) & 1) != 0) {
                     if (i < 32) {
-                        *eax |= 1 << i;
+                        *eax |= 1u << i;
                     } else {
-                        *edx |= 1 << (i - 32);
+                        *edx |= 1u << (i - 32);
                     }
                     *ecx = MAX(*ecx, esa->offset + esa->size);
                 }
             }
-            *eax |= kvm_mask & (XSTATE_FP | XSTATE_SSE);
+            *eax |= ena_mask & (XSTATE_FP_MASK | XSTATE_SSE_MASK);
             *ebx = *ecx;
         } else if (count == 1) {
             *eax = env->features[FEAT_XSAVE];
-        } else if (count < ARRAY_SIZE(ext_save_areas)) {
-            const ExtSaveArea *esa = &ext_save_areas[count];
-            if ((env->features[esa->feature] & esa->bits) == esa->bits &&
-                (kvm_mask & (1 << count)) != 0) {
+        } else if (count < ARRAY_SIZE(x86_ext_save_areas)) {
+            const ExtSaveArea *esa = &x86_ext_save_areas[count];
+            if ((env->features[esa->feature] & esa->bits) == esa->bits
+                && ((ena_mask >> count) & 1) != 0) {
                 *eax = esa->size;
                 *ebx = esa->offset;
             }
@@ -2622,6 +2655,8 @@ static void x86_cpu_reset(CPUState *s)
     X86CPU *cpu = X86_CPU(s);
     X86CPUClass *xcc = X86_CPU_GET_CLASS(cpu);
     CPUX86State *env = &cpu->env;
+    target_ulong cr4;
+    uint64_t xcr0;
     int i;
 
     xcc->parent_reset(s);
@@ -2681,7 +2716,8 @@ static void x86_cpu_reset(CPUState *s)
     cpu_set_fpuc(env, 0x37f);
 
     env->mxcsr = 0x1f80;
-    env->xstate_bv = XSTATE_FP | XSTATE_SSE;
+    /* All units are in INIT state.  */
+    env->xstate_bv = 0;
 
     env->pat = 0x0007040600070406ULL;
     env->msr_ia32_misc_enable = MSR_IA32_MISC_ENABLE_DEFAULT;
@@ -2692,7 +2728,31 @@ static void x86_cpu_reset(CPUState *s)
     cpu_breakpoint_remove_all(s, BP_CPU);
     cpu_watchpoint_remove_all(s, BP_CPU);
 
-    env->xcr0 = 1;
+    cr4 = 0;
+    xcr0 = XSTATE_FP_MASK;
+
+#ifdef CONFIG_USER_ONLY
+    /* Enable all the features for user-mode.  */
+    if (env->features[FEAT_1_EDX] & CPUID_SSE) {
+        xcr0 |= XSTATE_SSE_MASK;
+    }
+    for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+        const ExtSaveArea *esa = &x86_ext_save_areas[i];
+        if ((env->features[esa->feature] & esa->bits) == esa->bits) {
+            xcr0 |= 1ull << i;
+        }
+    }
+
+    if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) {
+        cr4 |= CR4_OSFXSR_MASK | CR4_OSXSAVE_MASK;
+    }
+    if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_FSGSBASE) {
+        cr4 |= CR4_FSGSBASE_MASK;
+    }
+#endif
+
+    env->xcr0 = xcr0;
+    cpu_x86_update_cr4(env, cr4);
 
     /*
      * SDM 11.11.5 requires:
@@ -2749,21 +2809,16 @@ static void mce_init(X86CPU *cpu)
 #ifndef CONFIG_USER_ONLY
 static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
 {
-    DeviceState *dev = DEVICE(cpu);
     APICCommonState *apic;
     const char *apic_type = "apic";
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_apic_in_kernel()) {
         apic_type = "kvm-apic";
     } else if (xen_enabled()) {
         apic_type = "xen-apic";
     }
 
-    cpu->apic_state = qdev_try_create(qdev_get_parent_bus(dev), apic_type);
-    if (cpu->apic_state == NULL) {
-        error_setg(errp, "APIC device '%s' could not be created", apic_type);
-        return;
-    }
+    cpu->apic_state = DEVICE(object_new(apic_type));
 
     object_property_add_child(OBJECT(cpu), "apic",
                               OBJECT(cpu->apic_state), NULL);
@@ -2771,15 +2826,30 @@ static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
     /* TODO: convert to link<> */
     apic = APIC_COMMON(cpu->apic_state);
     apic->cpu = cpu;
+    apic->apicbase = APIC_DEFAULT_ADDRESS | MSR_IA32_APICBASE_ENABLE;
 }
 
 static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
 {
+    APICCommonState *apic;
+    static bool apic_mmio_map_once;
+
     if (cpu->apic_state == NULL) {
         return;
     }
     object_property_set_bool(OBJECT(cpu->apic_state), true, "realized",
                              errp);
+
+    /* Map APIC MMIO area */
+    apic = APIC_COMMON(cpu->apic_state);
+    if (!apic_mmio_map_once) {
+        memory_region_add_subregion_overlap(get_system_memory(),
+                                            apic->apicbase &
+                                            MSR_IA32_APICBASE_BASE,
+                                            &apic->io_memory,
+                                            0x1000);
+        apic_mmio_map_once = true;
+     }
 }
 
 static void x86_cpu_machine_done(Notifier *n, void *unused)
@@ -2827,6 +2897,14 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
         env->cpuid_level = 7;
     }
 
+    if (x86_cpu_filter_features(cpu) && cpu->enforce_cpuid) {
+        error_setg(&local_err,
+                   kvm_enabled() ?
+                       "Host doesn't support requested features" :
+                       "TCG doesn't support requested features");
+        goto out;
+    }
+
     /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
      * CPUID[1].EDX.
      */
@@ -2837,14 +2915,6 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
     }
 
 
-    if (x86_cpu_filter_features(cpu) && cpu->enforce_cpuid) {
-        error_setg(&local_err,
-                   kvm_enabled() ?
-                       "Host doesn't support requested features" :
-                       "TCG doesn't support requested features");
-        goto out;
-    }
-
 #ifndef CONFIG_USER_ONLY
     qemu_register_reset(x86_cpu_machine_reset_cb, cpu);
 
@@ -2860,9 +2930,10 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
 
 #ifndef CONFIG_USER_ONLY
     if (tcg_enabled()) {
+        AddressSpace *newas = g_new(AddressSpace, 1);
+
         cpu->cpu_as_mem = g_new(MemoryRegion, 1);
         cpu->cpu_as_root = g_new(MemoryRegion, 1);
-        cs->as = g_new(AddressSpace, 1);
 
         /* Outer container... */
         memory_region_init(cpu->cpu_as_root, OBJECT(cpu), "memory", ~0ull);
@@ -2875,7 +2946,9 @@ static void x86_cpu_realizefn(DeviceState *dev, Error **errp)
                                  get_system_memory(), 0, ~0ull);
         memory_region_add_subregion_overlap(cpu->cpu_as_root, 0, cpu->cpu_as_mem, 0);
         memory_region_set_enabled(cpu->cpu_as_mem, true);
-        address_space_init(cs->as, cpu->cpu_as_root, "CPU");
+        address_space_init(newas, cpu->cpu_as_root, "CPU");
+        cs->num_ases = 1;
+        cpu_address_space_init(cs, newas, 0);
 
         /* ... SMRAM with higher priority, linked from /machine/smram.  */
         cpu->machine_done.notify = x86_cpu_machine_done;
@@ -2919,22 +2992,16 @@ typedef struct BitProperty {
     uint32_t mask;
 } BitProperty;
 
-static void x86_cpu_get_bit_prop(Object *obj,
-                                 struct Visitor *v,
-                                 void *opaque,
-                                 const char *name,
-                                 Error **errp)
+static void x86_cpu_get_bit_prop(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     BitProperty *fp = opaque;
     bool value = (*fp->ptr & fp->mask) == fp->mask;
-    visit_type_bool(v, &value, name, errp);
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void x86_cpu_set_bit_prop(Object *obj,
-                                 struct Visitor *v,
-                                 void *opaque,
-                                 const char *name,
-                                 Error **errp)
+static void x86_cpu_set_bit_prop(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     BitProperty *fp = opaque;
@@ -2946,7 +3013,7 @@ static void x86_cpu_set_bit_prop(Object *obj,
         return;
     }
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -3087,7 +3154,7 @@ static void x86_cpu_initfn(Object *obj)
     /* init various static tables used in TCG mode */
     if (tcg_enabled() && !inited) {
         inited = 1;
-        optimize_flags_init();
+        tcg_x86_init();
     }
 }
 
@@ -3124,14 +3191,8 @@ static bool x86_cpu_has_work(CPUState *cs)
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
 
-#if !defined(CONFIG_USER_ONLY)
-    if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
-        apic_poll_irq(cpu->apic_state);
-        cpu_reset_interrupt(cs, CPU_INTERRUPT_POLL);
-    }
-#endif
-
-    return ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
+    return ((cs->interrupt_request & (CPU_INTERRUPT_HARD |
+                                      CPU_INTERRUPT_POLL)) &&
             (env->eflags & IF_MASK)) ||
            (cs->interrupt_request & (CPU_INTERRUPT_NMI |
                                      CPU_INTERRUPT_INIT |
@@ -3147,12 +3208,19 @@ static Property x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("hv-relaxed", X86CPU, hyperv_relaxed_timing, false),
     DEFINE_PROP_BOOL("hv-vapic", X86CPU, hyperv_vapic, false),
     DEFINE_PROP_BOOL("hv-time", X86CPU, hyperv_time, false),
-    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, false),
+    DEFINE_PROP_BOOL("hv-crash", X86CPU, hyperv_crash, false),
+    DEFINE_PROP_BOOL("hv-reset", X86CPU, hyperv_reset, false),
+    DEFINE_PROP_BOOL("hv-vpindex", X86CPU, hyperv_vpindex, false),
+    DEFINE_PROP_BOOL("hv-runtime", X86CPU, hyperv_runtime, false),
+    DEFINE_PROP_BOOL("hv-synic", X86CPU, hyperv_synic, false),
+    DEFINE_PROP_BOOL("hv-stimer", X86CPU, hyperv_stimer, false),
+    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, true),
     DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
     DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
     DEFINE_PROP_UINT32("level", X86CPU, env.cpuid_level, 0),
     DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, 0),
     DEFINE_PROP_UINT32("xlevel2", X86CPU, env.cpuid_xlevel2, 0),
+    DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor_id),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -3164,7 +3232,6 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
 
     xcc->parent_realize = dc->realize;
     dc->realize = x86_cpu_realizefn;
-    dc->bus_type = TYPE_ICC_BUS;
     dc->props = x86_cpu_properties;
 
     xcc->parent_reset = cc->reset;
@@ -3200,6 +3267,12 @@ static void x86_cpu_common_class_init(ObjectClass *oc, void *data)
 #endif
     cc->cpu_exec_enter = x86_cpu_exec_enter;
     cc->cpu_exec_exit = x86_cpu_exec_exit;
+
+    /*
+     * Reason: x86_cpu_initfn() calls cpu_exec_init(), which saves the
+     * object in cpus -> dangling pointer after final object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo x86_cpu_type_info = {
diff --git a/qemu/target-i386/cpu.h b/qemu/target-i386/cpu.h
index ead28325b..732eb6d7e 100644
--- a/qemu/target-i386/cpu.h
+++ b/qemu/target-i386/cpu.h
@@ -19,8 +19,8 @@
 #ifndef CPU_I386_H
 #define CPU_I386_H
 
-#include "config.h"
 #include "qemu-common.h"
+#include "standard-headers/asm-x86/hyperv.h"
 
 #ifdef TARGET_X86_64
 #define TARGET_LONG_BITS 64
@@ -36,10 +36,10 @@
 #define TARGET_HAS_PRECISE_SMC
 
 #ifdef TARGET_X86_64
-#define ELF_MACHINE     EM_X86_64
+#define I386_ELF_MACHINE  EM_X86_64
 #define ELF_MACHINE_UNAME "x86_64"
 #else
-#define ELF_MACHINE     EM_386
+#define I386_ELF_MACHINE  EM_386
 #define ELF_MACHINE_UNAME "i686"
 #endif
 
@@ -154,6 +154,9 @@
 #define HF_SVMI_SHIFT       21 /* SVM intercepts are active */
 #define HF_OSFXSR_SHIFT     22 /* CR4.OSFXSR */
 #define HF_SMAP_SHIFT       23 /* CR4.SMAP */
+#define HF_IOBPT_SHIFT      24 /* an io breakpoint enabled */
+#define HF_MPX_EN_SHIFT     25 /* MPX Enabled (CR4+XCR0+BNDCFGx) */
+#define HF_MPX_IU_SHIFT     26 /* BND registers in-use */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
@@ -177,6 +180,9 @@
 #define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
 #define HF_SMAP_MASK         (1 << HF_SMAP_SHIFT)
+#define HF_IOBPT_MASK        (1 << HF_IOBPT_SHIFT)
+#define HF_MPX_EN_MASK       (1 << HF_MPX_EN_SHIFT)
+#define HF_MPX_IU_MASK       (1 << HF_MPX_IU_SHIFT)
 
 /* hflags2 */
 
@@ -185,12 +191,14 @@
 #define HF2_NMI_SHIFT            2 /* CPU serving NMI */
 #define HF2_VINTR_SHIFT          3 /* value of V_INTR_MASKING bit */
 #define HF2_SMM_INSIDE_NMI_SHIFT 4 /* CPU serving SMI nested inside NMI */
+#define HF2_MPX_PR_SHIFT         5 /* BNDCFGx.BNDPRESERVE */
 
 #define HF2_GIF_MASK            (1 << HF2_GIF_SHIFT)
 #define HF2_HIF_MASK            (1 << HF2_HIF_SHIFT)
 #define HF2_NMI_MASK            (1 << HF2_NMI_SHIFT)
 #define HF2_VINTR_MASK          (1 << HF2_VINTR_SHIFT)
 #define HF2_SMM_INSIDE_NMI_MASK (1 << HF2_SMM_INSIDE_NMI_SHIFT)
+#define HF2_MPX_PR_MASK         (1 << HF2_MPX_PR_SHIFT)
 
 #define CR0_PE_SHIFT 0
 #define CR0_MP_SHIFT 1
@@ -224,6 +232,7 @@
 #define CR4_OSXSAVE_MASK (1U << 18)
 #define CR4_SMEP_MASK   (1U << 20)
 #define CR4_SMAP_MASK   (1U << 21)
+#define CR4_PKE_MASK   (1U << 22)
 
 #define DR6_BD          (1 << 13)
 #define DR6_BS          (1 << 14)
@@ -234,6 +243,7 @@
 #define DR7_TYPE_SHIFT  16
 #define DR7_LEN_SHIFT   18
 #define DR7_FIXED_1     0x00000400
+#define DR7_GLOBAL_BP_MASK   0xaa
 #define DR7_LOCAL_BP_MASK    0x55
 #define DR7_MAX_BP           4
 #define DR7_TYPE_BP_INST     0x0
@@ -251,6 +261,7 @@
 #define PG_PSE_BIT      7
 #define PG_GLOBAL_BIT   8
 #define PG_PSE_PAT_BIT  12
+#define PG_PKRU_BIT     59
 #define PG_NX_BIT       63
 
 #define PG_PRESENT_MASK  (1 << PG_PRESENT_BIT)
@@ -266,7 +277,8 @@
 #define PG_ADDRESS_MASK  0x000ffffffffff000LL
 #define PG_HI_RSVD_MASK  (PG_ADDRESS_MASK & ~PHYS_ADDR_MASK)
 #define PG_HI_USER_MASK  0x7ff0000000000000LL
-#define PG_NX_MASK       (1LL << PG_NX_BIT)
+#define PG_PKRU_MASK     (15ULL << PG_PKRU_BIT)
+#define PG_NX_MASK       (1ULL << PG_NX_BIT)
 
 #define PG_ERROR_W_BIT     1
 
@@ -275,6 +287,7 @@
 #define PG_ERROR_U_MASK    0x04
 #define PG_ERROR_RSVD_MASK 0x08
 #define PG_ERROR_I_D_MASK  0x10
+#define PG_ERROR_PK_MASK   0x20
 
 #define MCG_CTL_P       (1ULL<<8)   /* MCG_CAP register available */
 #define MCG_SER_P       (1ULL<<24) /* MCA recovery/new status bits */
@@ -282,6 +295,8 @@
 #define MCE_CAP_DEF     (MCG_CTL_P|MCG_SER_P)
 #define MCE_BANKS_DEF   10
 
+#define MCG_CAP_BANKS_MASK 0xff
+
 #define MCG_STATUS_RIPV (1ULL<<0)   /* restart ip valid */
 #define MCG_STATUS_EIPV (1ULL<<1)   /* ip points to correct instruction */
 #define MCG_STATUS_MCIP (1ULL<<2)   /* machine check in progress */
@@ -393,21 +408,32 @@
 #define MSR_IA32_BNDCFGS                0x00000d90
 #define MSR_IA32_XSS                    0x00000da0
 
-#define XSTATE_FP                       (1ULL << 0)
-#define XSTATE_SSE                      (1ULL << 1)
-#define XSTATE_YMM                      (1ULL << 2)
-#define XSTATE_BNDREGS                  (1ULL << 3)
-#define XSTATE_BNDCSR                   (1ULL << 4)
-#define XSTATE_OPMASK                   (1ULL << 5)
-#define XSTATE_ZMM_Hi256                (1ULL << 6)
-#define XSTATE_Hi16_ZMM                 (1ULL << 7)
-
+#define XSTATE_FP_BIT                   0
+#define XSTATE_SSE_BIT                  1
+#define XSTATE_YMM_BIT                  2
+#define XSTATE_BNDREGS_BIT              3
+#define XSTATE_BNDCSR_BIT               4
+#define XSTATE_OPMASK_BIT               5
+#define XSTATE_ZMM_Hi256_BIT            6
+#define XSTATE_Hi16_ZMM_BIT             7
+#define XSTATE_PKRU_BIT                 9
+
+#define XSTATE_FP_MASK                  (1ULL << XSTATE_FP_BIT)
+#define XSTATE_SSE_MASK                 (1ULL << XSTATE_SSE_BIT)
+#define XSTATE_YMM_MASK                 (1ULL << XSTATE_YMM_BIT)
+#define XSTATE_BNDREGS_MASK             (1ULL << XSTATE_BNDREGS_BIT)
+#define XSTATE_BNDCSR_MASK              (1ULL << XSTATE_BNDCSR_BIT)
+#define XSTATE_OPMASK_MASK              (1ULL << XSTATE_OPMASK_BIT)
+#define XSTATE_ZMM_Hi256_MASK           (1ULL << XSTATE_ZMM_Hi256_BIT)
+#define XSTATE_Hi16_ZMM_MASK            (1ULL << XSTATE_Hi16_ZMM_BIT)
+#define XSTATE_PKRU_MASK                (1ULL << XSTATE_PKRU_BIT)
 
 /* CPUID feature words */
 typedef enum FeatureWord {
     FEAT_1_EDX,         /* CPUID[1].EDX */
     FEAT_1_ECX,         /* CPUID[1].ECX */
     FEAT_7_0_EBX,       /* CPUID[EAX=7,ECX=0].EBX */
+    FEAT_7_0_ECX,       /* CPUID[EAX=7,ECX=0].ECX */
     FEAT_8000_0001_EDX, /* CPUID[8000_0001].EDX */
     FEAT_8000_0001_ECX, /* CPUID[8000_0001].ECX */
     FEAT_8000_0007_EDX, /* CPUID[8000_0007].EDX */
@@ -572,10 +598,16 @@ typedef uint32_t FeatureWordArray[FEATURE_WORDS];
 #define CPUID_7_0_EBX_RDSEED   (1U << 18)
 #define CPUID_7_0_EBX_ADX      (1U << 19)
 #define CPUID_7_0_EBX_SMAP     (1U << 20)
+#define CPUID_7_0_EBX_PCOMMIT  (1U << 22) /* Persistent Commit */
+#define CPUID_7_0_EBX_CLFLUSHOPT (1U << 23) /* Flush a Cache Line Optimized */
+#define CPUID_7_0_EBX_CLWB     (1U << 24) /* Cache Line Write Back */
 #define CPUID_7_0_EBX_AVX512PF (1U << 26) /* AVX-512 Prefetch */
 #define CPUID_7_0_EBX_AVX512ER (1U << 27) /* AVX-512 Exponential and Reciprocal */
 #define CPUID_7_0_EBX_AVX512CD (1U << 28) /* AVX-512 Conflict Detection */
 
+#define CPUID_7_0_ECX_PKU      (1U << 3)
+#define CPUID_7_0_ECX_OSPKE    (1U << 4)
+
 #define CPUID_XSAVE_XSAVEOPT   (1U << 0)
 #define CPUID_XSAVE_XSAVEC     (1U << 1)
 #define CPUID_XSAVE_XGETBV1    (1U << 2)
@@ -716,22 +748,18 @@ typedef struct SegmentCache {
     uint32_t flags;
 } SegmentCache;
 
-typedef union {
-    uint8_t _b[64];
-    uint16_t _w[32];
-    uint32_t _l[16];
-    uint64_t _q[8];
-    float32 _s[16];
-    float64 _d[8];
-} XMMReg; /* really zmm */
+#define MMREG_UNION(n, bits)        \
+    union n {                       \
+        uint8_t  _b_##n[(bits)/8];  \
+        uint16_t _w_##n[(bits)/16]; \
+        uint32_t _l_##n[(bits)/32]; \
+        uint64_t _q_##n[(bits)/64]; \
+        float32  _s_##n[(bits)/32]; \
+        float64  _d_##n[(bits)/64]; \
+    }
 
-typedef union {
-    uint8_t _b[8];
-    uint16_t _w[4];
-    uint32_t _l[2];
-    float32 _s[2];
-    uint64_t q;
-} MMXReg;
+typedef MMREG_UNION(ZMMReg, 512) ZMMReg;
+typedef MMREG_UNION(MMXReg, 64)  MMXReg;
 
 typedef struct BNDReg {
     uint64_t lb;
@@ -743,32 +771,36 @@ typedef struct BNDCSReg {
     uint64_t sts;
 } BNDCSReg;
 
+#define BNDCFG_ENABLE       1ULL
+#define BNDCFG_BNDPRESERVE  2ULL
+#define BNDCFG_BDIR_MASK    TARGET_PAGE_MASK
+
 #ifdef HOST_WORDS_BIGENDIAN
-#define XMM_B(n) _b[63 - (n)]
-#define XMM_W(n) _w[31 - (n)]
-#define XMM_L(n) _l[15 - (n)]
-#define XMM_S(n) _s[15 - (n)]
-#define XMM_Q(n) _q[7 - (n)]
-#define XMM_D(n) _d[7 - (n)]
-
-#define MMX_B(n) _b[7 - (n)]
-#define MMX_W(n) _w[3 - (n)]
-#define MMX_L(n) _l[1 - (n)]
-#define MMX_S(n) _s[1 - (n)]
+#define ZMM_B(n) _b_ZMMReg[63 - (n)]
+#define ZMM_W(n) _w_ZMMReg[31 - (n)]
+#define ZMM_L(n) _l_ZMMReg[15 - (n)]
+#define ZMM_S(n) _s_ZMMReg[15 - (n)]
+#define ZMM_Q(n) _q_ZMMReg[7 - (n)]
+#define ZMM_D(n) _d_ZMMReg[7 - (n)]
+
+#define MMX_B(n) _b_MMXReg[7 - (n)]
+#define MMX_W(n) _w_MMXReg[3 - (n)]
+#define MMX_L(n) _l_MMXReg[1 - (n)]
+#define MMX_S(n) _s_MMXReg[1 - (n)]
 #else
-#define XMM_B(n) _b[n]
-#define XMM_W(n) _w[n]
-#define XMM_L(n) _l[n]
-#define XMM_S(n) _s[n]
-#define XMM_Q(n) _q[n]
-#define XMM_D(n) _d[n]
-
-#define MMX_B(n) _b[n]
-#define MMX_W(n) _w[n]
-#define MMX_L(n) _l[n]
-#define MMX_S(n) _s[n]
+#define ZMM_B(n) _b_ZMMReg[n]
+#define ZMM_W(n) _w_ZMMReg[n]
+#define ZMM_L(n) _l_ZMMReg[n]
+#define ZMM_S(n) _s_ZMMReg[n]
+#define ZMM_Q(n) _q_ZMMReg[n]
+#define ZMM_D(n) _d_ZMMReg[n]
+
+#define MMX_B(n) _b_MMXReg[n]
+#define MMX_W(n) _w_MMXReg[n]
+#define MMX_L(n) _l_MMXReg[n]
+#define MMX_S(n) _s_MMXReg[n]
 #endif
-#define MMX_Q(n) q
+#define MMX_Q(n) _q_MMXReg[n]
 
 typedef union {
     floatx80 d __attribute__((aligned(16)));
@@ -793,6 +825,7 @@ typedef struct {
 #define MAX_GP_COUNTERS    (MSR_IA32_PERF_STATUS - MSR_P6_EVNTSEL0)
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define NB_OPMASK_REGS 8
 
@@ -832,6 +865,7 @@ typedef struct CPUX86State {
     BNDReg bnd_regs[4];
     BNDCSReg bndcs_regs;
     uint64_t msr_bndcfgs;
+    uint64_t efer;
 
     /* Beginning of state preserved by INIT (dummy marker).  */
     struct {} start_init_save;
@@ -854,8 +888,8 @@ typedef struct CPUX86State {
     float_status mmx_status; /* for 3DNow! float ops */
     float_status sse_status;
     uint32_t mxcsr;
-    XMMReg xmm_regs[CPU_NB_REGS == 8 ? 8 : 32];
-    XMMReg xmm_t0;
+    ZMMReg xmm_regs[CPU_NB_REGS == 8 ? 8 : 32];
+    ZMMReg xmm_t0;
     MMXReg mmx_t0;
 
     uint64_t opmask_regs[NB_OPMASK_REGS];
@@ -864,7 +898,6 @@ typedef struct CPUX86State {
     uint32_t sysenter_cs;
     target_ulong sysenter_esp;
     target_ulong sysenter_eip;
-    uint64_t efer;
     uint64_t star;
 
     uint64_t vm_hsave;
@@ -908,12 +941,21 @@ typedef struct CPUX86State {
     uint64_t msr_hv_guest_os_id;
     uint64_t msr_hv_vapic;
     uint64_t msr_hv_tsc;
+    uint64_t msr_hv_crash_params[HV_X64_MSR_CRASH_PARAMS];
+    uint64_t msr_hv_runtime;
+    uint64_t msr_hv_synic_control;
+    uint64_t msr_hv_synic_version;
+    uint64_t msr_hv_synic_evt_page;
+    uint64_t msr_hv_synic_msg_page;
+    uint64_t msr_hv_synic_sint[HV_SYNIC_SINT_COUNT];
+    uint64_t msr_hv_stimer_config[HV_SYNIC_STIMER_COUNT];
+    uint64_t msr_hv_stimer_count[HV_SYNIC_STIMER_COUNT];
 
     /* exception/interrupt handling */
     int error_code;
     int exception_is_int;
     target_ulong exception_next_eip;
-    target_ulong dr[8]; /* debug registers */
+    target_ulong dr[8]; /* debug registers; note dr4 and dr5 are unused */
     union {
         struct CPUBreakpoint *cpu_breakpoint[4];
         struct CPUWatchpoint *cpu_watchpoint[4];
@@ -963,6 +1005,7 @@ typedef struct CPUX86State {
     uint32_t sipi_vector;
     bool tsc_valid;
     int64_t tsc_khz;
+    int64_t user_tsc_khz; /* for sanity check only */
     void *kvm_xsave_buf;
 
     uint64_t mcg_cap;
@@ -980,6 +1023,8 @@ typedef struct CPUX86State {
     uint64_t xcr0;
     uint64_t xss;
 
+    uint32_t pkru;
+
     TPRAccess tpr_access_type;
 } CPUX86State;
 
@@ -1098,7 +1143,14 @@ void cpu_x86_frstor(CPUX86State *s, target_ulong ptr, int data32);
 int cpu_x86_signal_handler(int host_signum, void *pinfo,
                            void *puc);
 
-/* cpuid.c */
+/* cpu.c */
+typedef struct ExtSaveArea {
+    uint32_t feature, bits;
+    uint32_t offset, size;
+} ExtSaveArea;
+
+extern const ExtSaveArea x86_ext_save_areas[];
+
 void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                    uint32_t *eax, uint32_t *ebx,
                    uint32_t *ecx, uint32_t *edx);
@@ -1123,42 +1175,13 @@ void x86_stl_phys(CPUState *cs, hwaddr addr, uint32_t val);
 void x86_stq_phys(CPUState *cs, hwaddr addr, uint64_t val);
 #endif
 
-static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 1;
-}
-
-static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 2;
-
-}
-static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return hw_global_breakpoint_enabled(dr7, index) ||
-           hw_local_breakpoint_enabled(dr7, index);
-}
-
-static inline int hw_breakpoint_type(unsigned long dr7, int index)
-{
-    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
-}
-
-static inline int hw_breakpoint_len(unsigned long dr7, int index)
-{
-    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
-    return (len == 2) ? 8 : len + 1;
-}
-
-void hw_breakpoint_insert(CPUX86State *env, int index);
-void hw_breakpoint_remove(CPUX86State *env, int index);
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
 void breakpoint_handler(CPUState *cs);
 
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
 void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7);
 
 /* hw/pc.c */
 uint64_t cpu_get_tsc(CPUX86State *env);
@@ -1187,7 +1210,6 @@ uint64_t cpu_get_tsc(CPUX86State *env);
 #define cpu_init(cpu_model) CPU(cpu_x86_init(cpu_model))
 
 #define cpu_exec cpu_x86_exec
-#define cpu_gen_code cpu_x86_gen_code
 #define cpu_signal_handler cpu_x86_signal_handler
 #define cpu_list x86_cpu_list
 #define cpudef_setup x86_cpudef_setup
@@ -1199,7 +1221,7 @@ uint64_t cpu_get_tsc(CPUX86State *env);
 #define MMU_KSMAP_IDX   0
 #define MMU_USER_IDX    1
 #define MMU_KNOSMAP_IDX 2
-static inline int cpu_mmu_index(CPUX86State *env)
+static inline int cpu_mmu_index(CPUX86State *env, bool ifetch)
 {
     return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
         (!(env->hflags & HF_SMAP_MASK) || (env->eflags & AC_MASK))
@@ -1235,7 +1257,7 @@ static inline target_long lshift(target_long x, int n)
 #define ST1    ST(1)
 
 /* translate.c */
-void optimize_flags_init(void);
+void tcg_x86_init(void);
 
 #include "exec/cpu-all.h"
 #include "svm.h"
@@ -1267,8 +1289,12 @@ void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
 
 /* excp_helper.c */
 void QEMU_NORETURN raise_exception(CPUX86State *env, int exception_index);
+void QEMU_NORETURN raise_exception_ra(CPUX86State *env, int exception_index,
+                                      uintptr_t retaddr);
 void QEMU_NORETURN raise_exception_err(CPUX86State *env, int exception_index,
                                        int error_code);
+void QEMU_NORETURN raise_exception_err_ra(CPUX86State *env, int exception_index,
+                                          int error_code, uintptr_t retaddr);
 void QEMU_NORETURN raise_interrupt(CPUX86State *nenv, int intno, int is_int,
                                    int error_code, int next_eip_addend);
 
@@ -1318,6 +1344,9 @@ static inline MemTxAttrs cpu_get_mem_attrs(CPUX86State *env)
 void cpu_set_mxcsr(CPUX86State *env, uint32_t val);
 void cpu_set_fpuc(CPUX86State *env, uint16_t val);
 
+/* mem_helper.c */
+void helper_lock_init(void);
+
 /* svm_helper.c */
 void cpu_svm_check_intercept_param(CPUX86State *env1, uint32_t type,
                                    uint64_t param);
@@ -1332,12 +1361,18 @@ void cpu_smm_update(X86CPU *cpu);
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove);
-
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features);
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features);
+/* Change the value of a KVM-specific default
+ *
+ * If value is NULL, no default will be set and the original
+ * value from the CPU model table will be kept.
+ *
+ * It is valid to call this funciton only for properties that
+ * are already present in the kvm_default_props table.
+ */
+void x86_cpu_change_kvm_default(const char *prop, const char *value);
 
+/* mpx_helper.c */
+void cpu_sync_bndcs_hflags(CPUX86State *env);
 
 /* Return name of 32-bit register, from a R_* constant */
 const char *get_register_name_32(unsigned int reg);
@@ -1347,4 +1382,7 @@ void enable_compat_apic_id_mode(void);
 #define APIC_DEFAULT_ADDRESS 0xfee00000
 #define APIC_SPACE_SIZE      0x100000
 
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags);
+
 #endif /* CPU_I386_H */
diff --git a/qemu/target-i386/excp_helper.c b/qemu/target-i386/excp_helper.c
index 99fca847d..ef37f4240 100644
--- a/qemu/target-i386/excp_helper.c
+++ b/qemu/target-i386/excp_helper.c
@@ -17,19 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
 #include "exec/helper-proto.h"
 
-#if 0
-#define raise_exception_err(env, a, b)                                  \
-    do {                                                                \
-        qemu_log("raise_exception line=%d\n", __LINE__);                \
-        (raise_exception_err)(env, a, b);                               \
-    } while (0)
-#endif
-
 void helper_raise_interrupt(CPUX86State *env, int intno, int next_eip_addend)
 {
     raise_interrupt(env, intno, 1, 0, next_eip_addend);
@@ -92,7 +85,8 @@ static int check_exception(CPUX86State *env, int intno, int *error_code)
  */
 static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
                                            int is_int, int error_code,
-                                           int next_eip_addend)
+                                           int next_eip_addend,
+                                           uintptr_t retaddr)
 {
     CPUState *cs = CPU(x86_env_get_cpu(env));
 
@@ -108,7 +102,7 @@ static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
     env->error_code = error_code;
     env->exception_is_int = is_int;
     env->exception_next_eip = env->eip + next_eip_addend;
-    cpu_loop_exit(cs);
+    cpu_loop_exit_restore(cs, retaddr);
 }
 
 /* shortcuts to generate exceptions */
@@ -116,16 +110,27 @@ static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
 void QEMU_NORETURN raise_interrupt(CPUX86State *env, int intno, int is_int,
                                    int error_code, int next_eip_addend)
 {
-    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend);
+    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend, 0);
 }
 
 void raise_exception_err(CPUX86State *env, int exception_index,
                          int error_code)
 {
-    raise_interrupt2(env, exception_index, 0, error_code, 0);
+    raise_interrupt2(env, exception_index, 0, error_code, 0, 0);
+}
+
+void raise_exception_err_ra(CPUX86State *env, int exception_index,
+                            int error_code, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, error_code, 0, retaddr);
 }
 
 void raise_exception(CPUX86State *env, int exception_index)
 {
-    raise_interrupt2(env, exception_index, 0, 0, 0);
+    raise_interrupt2(env, exception_index, 0, 0, 0, 0);
+}
+
+void raise_exception_ra(CPUX86State *env, int exception_index, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, 0, 0, retaddr);
 }
diff --git a/qemu/target-i386/fpu_helper.c b/qemu/target-i386/fpu_helper.c
index 1f954e0c5..fee5573a1 100644
--- a/qemu/target-i386/fpu_helper.c
+++ b/qemu/target-i386/fpu_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <math.h>
 #include "cpu.h"
 #include "exec/helper-proto.h"
@@ -67,22 +68,24 @@ static inline void fpop(CPUX86State *env)
     env->fpstt = (env->fpstt + 1) & 7;
 }
 
-static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr)
+static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
+                                   uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
-    temp.l.lower = cpu_ldq_data(env, ptr);
-    temp.l.upper = cpu_lduw_data(env, ptr + 8);
+    temp.l.lower = cpu_ldq_data_ra(env, ptr, retaddr);
+    temp.l.upper = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     return temp.d;
 }
 
-static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr)
+static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr,
+                               uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
     temp.d = f;
-    cpu_stq_data(env, ptr, temp.l.lower);
-    cpu_stw_data(env, ptr + 8, temp.l.upper);
+    cpu_stq_data_ra(env, ptr, temp.l.lower, retaddr);
+    cpu_stw_data_ra(env, ptr + 8, temp.l.upper, retaddr);
 }
 
 /* x87 FPU helpers */
@@ -125,10 +128,10 @@ static inline floatx80 helper_fdiv(CPUX86State *env, floatx80 a, floatx80 b)
     return floatx80_div(a, b, &env->fp_status);
 }
 
-static void fpu_raise_exception(CPUX86State *env)
+static void fpu_raise_exception(CPUX86State *env, uintptr_t retaddr)
 {
     if (env->cr[0] & CR0_NE_MASK) {
-        raise_exception(env, EXCP10_COPR);
+        raise_exception_ra(env, EXCP10_COPR, retaddr);
     }
 #if !defined(CONFIG_USER_ONLY)
     else {
@@ -313,14 +316,14 @@ void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
     int new_fpstt;
 
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = helper_fldt(env, ptr);
+    env->fpregs[new_fpstt].d = helper_fldt(env, ptr, GETPC());
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
 
 void helper_fstt_ST0(CPUX86State *env, target_ulong ptr)
 {
-    helper_fstt(env, ST0, ptr);
+    helper_fstt(env, ST0, ptr, GETPC());
 }
 
 void helper_fpush(CPUX86State *env)
@@ -603,7 +606,7 @@ void helper_fclex(CPUX86State *env)
 void helper_fwait(CPUX86State *env)
 {
     if (env->fpus & FPUS_SE) {
-        fpu_raise_exception(env);
+        fpu_raise_exception(env, GETPC());
     }
 }
 
@@ -633,11 +636,11 @@ void helper_fbld_ST0(CPUX86State *env, target_ulong ptr)
 
     val = 0;
     for (i = 8; i >= 0; i--) {
-        v = cpu_ldub_data(env, ptr + i);
+        v = cpu_ldub_data_ra(env, ptr + i, GETPC());
         val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
     }
     tmp = int64_to_floatx80(val, &env->fp_status);
-    if (cpu_ldub_data(env, ptr + 9) & 0x80) {
+    if (cpu_ldub_data_ra(env, ptr + 9, GETPC()) & 0x80) {
         tmp = floatx80_chs(tmp);
     }
     fpush(env);
@@ -654,10 +657,10 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
     mem_ref = ptr;
     mem_end = mem_ref + 9;
     if (val < 0) {
-        cpu_stb_data(env, mem_end, 0x80);
+        cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
         val = -val;
     } else {
-        cpu_stb_data(env, mem_end, 0x00);
+        cpu_stb_data_ra(env, mem_end, 0x00, GETPC());
     }
     while (mem_ref < mem_end) {
         if (val == 0) {
@@ -666,10 +669,10 @@ void helper_fbst_ST0(CPUX86State *env, target_ulong ptr)
         v = val % 100;
         val = val / 100;
         v = ((v / 10) << 4) | (v % 10);
-        cpu_stb_data(env, mem_ref++, v);
+        cpu_stb_data_ra(env, mem_ref++, v, GETPC());
     }
     while (mem_ref < mem_end) {
-        cpu_stb_data(env, mem_ref++, 0);
+        cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
     }
 }
 
@@ -977,7 +980,8 @@ void helper_fxam_ST0(CPUX86State *env)
     }
 }
 
-void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+static void do_fstenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int fpus, fptag, exp, i;
     uint64_t mant;
@@ -1005,37 +1009,43 @@ void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
     }
     if (data32) {
         /* 32 bit */
-        cpu_stl_data(env, ptr, env->fpuc);
-        cpu_stl_data(env, ptr + 4, fpus);
-        cpu_stl_data(env, ptr + 8, fptag);
-        cpu_stl_data(env, ptr + 12, 0); /* fpip */
-        cpu_stl_data(env, ptr + 16, 0); /* fpcs */
-        cpu_stl_data(env, ptr + 20, 0); /* fpoo */
-        cpu_stl_data(env, ptr + 24, 0); /* fpos */
+        cpu_stl_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stl_data_ra(env, ptr + 4, fpus, retaddr);
+        cpu_stl_data_ra(env, ptr + 8, fptag, retaddr);
+        cpu_stl_data_ra(env, ptr + 12, 0, retaddr); /* fpip */
+        cpu_stl_data_ra(env, ptr + 16, 0, retaddr); /* fpcs */
+        cpu_stl_data_ra(env, ptr + 20, 0, retaddr); /* fpoo */
+        cpu_stl_data_ra(env, ptr + 24, 0, retaddr); /* fpos */
     } else {
         /* 16 bit */
-        cpu_stw_data(env, ptr, env->fpuc);
-        cpu_stw_data(env, ptr + 2, fpus);
-        cpu_stw_data(env, ptr + 4, fptag);
-        cpu_stw_data(env, ptr + 6, 0);
-        cpu_stw_data(env, ptr + 8, 0);
-        cpu_stw_data(env, ptr + 10, 0);
-        cpu_stw_data(env, ptr + 12, 0);
+        cpu_stw_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stw_data_ra(env, ptr + 2, fpus, retaddr);
+        cpu_stw_data_ra(env, ptr + 4, fptag, retaddr);
+        cpu_stw_data_ra(env, ptr + 6, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 8, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 10, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 12, 0, retaddr);
     }
 }
 
-void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fstenv(env, ptr, data32, GETPC());
+}
+
+static void do_fldenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int i, fpus, fptag;
 
     if (data32) {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 4);
-        fptag = cpu_lduw_data(env, ptr + 8);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 4, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     } else {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 2);
-        fptag = cpu_lduw_data(env, ptr + 4);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 2, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 4, retaddr);
     }
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
@@ -1045,17 +1055,22 @@ void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
     }
 }
 
+void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fldenv(env, ptr, data32, GETPC());
+}
+
 void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
 {
     floatx80 tmp;
     int i;
 
-    helper_fstenv(env, ptr, data32);
+    do_fstenv(env, ptr, data32, GETPC());
 
     ptr += (14 << data32);
     for (i = 0; i < 8; i++) {
         tmp = ST(i);
-        helper_fstt(env, tmp, ptr);
+        helper_fstt(env, tmp, ptr, GETPC());
         ptr += 10;
     }
 
@@ -1078,11 +1093,11 @@ void helper_frstor(CPUX86State *env, target_ulong ptr, int data32)
     floatx80 tmp;
     int i;
 
-    helper_fldenv(env, ptr, data32);
+    do_fldenv(env, ptr, data32, GETPC());
     ptr += (14 << data32);
 
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, ptr);
+        tmp = helper_fldt(env, ptr, GETPC());
         ST(i) = tmp;
         ptr += 10;
     }
@@ -1100,82 +1115,183 @@ void cpu_x86_frstor(CPUX86State *env, target_ulong ptr, int data32)
 }
 #endif
 
-void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
+static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
-    int fpus, fptag, i, nb_xmm_regs;
-    floatx80 tmp;
+    int fpus, fptag, i;
     target_ulong addr;
 
-    /* The operand must be 16 byte aligned */
-    if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
-    }
-
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
     fptag = 0;
     for (i = 0; i < 8; i++) {
         fptag |= (env->fptags[i] << i);
     }
-    cpu_stw_data(env, ptr, env->fpuc);
-    cpu_stw_data(env, ptr + 2, fpus);
-    cpu_stw_data(env, ptr + 4, fptag ^ 0xff);
-#ifdef TARGET_X86_64
-    if (data64) {
-        cpu_stq_data(env, ptr + 0x08, 0); /* rip */
-        cpu_stq_data(env, ptr + 0x10, 0); /* rdp */
-    } else
-#endif
-    {
-        cpu_stl_data(env, ptr + 0x08, 0); /* eip */
-        cpu_stl_data(env, ptr + 0x0c, 0); /* sel  */
-        cpu_stl_data(env, ptr + 0x10, 0); /* dp */
-        cpu_stl_data(env, ptr + 0x14, 0); /* sel  */
-    }
+    cpu_stw_data_ra(env, ptr, env->fpuc, ra);
+    cpu_stw_data_ra(env, ptr + 2, fpus, ra);
+    cpu_stw_data_ra(env, ptr + 4, fptag ^ 0xff, ra);
+
+    /* In 32-bit mode this is eip, sel, dp, sel.
+       In 64-bit mode this is rip, rdp.
+       But in either case we don't write actual data, just zeros.  */
+    cpu_stq_data_ra(env, ptr + 0x08, 0, ra); /* eip+sel; rip */
+    cpu_stq_data_ra(env, ptr + 0x10, 0, ra); /* edp+sel; rdp */
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
-        tmp = ST(i);
-        helper_fstt(env, tmp, addr);
+        floatx80 tmp = ST(i);
+        helper_fstt(env, tmp, addr, ra);
         addr += 16;
     }
+}
+
+static void do_xsave_mxcsr(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    cpu_stl_data_ra(env, ptr + 0x18, env->mxcsr, ra); /* mxcsr */
+    cpu_stl_data_ra(env, ptr + 0x1c, 0x0000ffff, ra); /* mxcsr_mask */
+}
+
+static void do_xsave_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+    target_ulong addr;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    addr = ptr + 0xa0;
+    for (i = 0; i < nb_xmm_regs; i++) {
+        cpu_stq_data_ra(env, addr, env->xmm_regs[i].ZMM_Q(0), ra);
+        cpu_stq_data_ra(env, addr + 8, env->xmm_regs[i].ZMM_Q(1), ra);
+        addr += 16;
+    }
+}
+
+static void do_xsave_bndregs(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    int i;
+
+    for (i = 0; i < 4; i++, addr += 16) {
+        cpu_stq_data_ra(env, addr, env->bnd_regs[i].lb, ra);
+        cpu_stq_data_ra(env, addr + 8, env->bnd_regs[i].ub, ra);
+    }
+}
+
+static void do_xsave_bndcsr(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    cpu_stq_data_ra(env, addr, env->bndcs_regs.cfgu, ra);
+    cpu_stq_data_ra(env, addr + 8, env->bndcs_regs.sts, ra);
+}
+
+static void do_xsave_pkru(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    cpu_stq_data_ra(env, addr, env->pkru, ra);
+}
+
+void helper_fxsave(CPUX86State *env, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+
+    /* The operand must be 16 byte aligned */
+    if (ptr & 0xf) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    do_xsave_fpu(env, ptr, ra);
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
-        /* XXX: finish it */
-        cpu_stl_data(env, ptr + 0x18, env->mxcsr); /* mxcsr */
-        cpu_stl_data(env, ptr + 0x1c, 0x0000ffff); /* mxcsr_mask */
-        if (env->hflags & HF_CS64_MASK) {
-            nb_xmm_regs = 16;
-        } else {
-            nb_xmm_regs = 8;
-        }
-        addr = ptr + 0xa0;
+        do_xsave_mxcsr(env, ptr, ra);
         /* Fast FXSAVE leaves out the XMM registers */
         if (!(env->efer & MSR_EFER_FFXSR)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
-            for (i = 0; i < nb_xmm_regs; i++) {
-                cpu_stq_data(env, addr, env->xmm_regs[i].XMM_Q(0));
-                cpu_stq_data(env, addr + 8, env->xmm_regs[i].XMM_Q(1));
-                addr += 16;
-            }
+            do_xsave_sse(env, ptr, ra);
         }
     }
 }
 
-void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
+static uint64_t get_xinuse(CPUX86State *env)
 {
-    int i, fpus, fptag, nb_xmm_regs;
-    floatx80 tmp;
-    target_ulong addr;
+    uint64_t inuse = -1;
 
-    /* The operand must be 16 byte aligned */
-    if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
+    /* For the most part, we don't track XINUSE.  We could calculate it
+       here for all components, but it's probably less work to simply
+       indicate in use.  That said, the state of BNDREGS is important
+       enough to track in HFLAGS, so we might as well use that here.  */
+    if ((env->hflags & HF_MPX_IU_MASK) == 0) {
+       inuse &= ~XSTATE_BNDREGS_MASK;
+    }
+    return inuse;
+}
+
+static void do_xsave(CPUX86State *env, target_ulong ptr, uint64_t rfbm,
+                     uint64_t inuse, uint64_t opt, uintptr_t ra)
+{
+    uint64_t old_bv, new_bv;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, ra);
+    }
+
+    /* The operand must be 64 byte aligned.  */
+    if (ptr & 63) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* Never save anything not enabled by XCR0.  */
+    rfbm &= env->xcr0;
+    opt &= rfbm;
+
+    if (opt & XSTATE_FP_MASK) {
+        do_xsave_fpu(env, ptr, ra);
+    }
+    if (rfbm & XSTATE_SSE_MASK) {
+        /* Note that saving MXCSR is not suppressed by XSAVEOPT.  */
+        do_xsave_mxcsr(env, ptr, ra);
+    }
+    if (opt & XSTATE_SSE_MASK) {
+        do_xsave_sse(env, ptr, ra);
+    }
+    if (opt & XSTATE_BNDREGS_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_BNDREGS_BIT].offset;
+        do_xsave_bndregs(env, ptr + off, ra);
+    }
+    if (opt & XSTATE_BNDCSR_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_BNDCSR_BIT].offset;
+        do_xsave_bndcsr(env, ptr + off, ra);
     }
+    if (opt & XSTATE_PKRU_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_PKRU_BIT].offset;
+        do_xsave_pkru(env, ptr + off, ra);
+    }
+
+    /* Update the XSTATE_BV field.  */
+    old_bv = cpu_ldq_data_ra(env, ptr + 512, ra);
+    new_bv = (old_bv & ~rfbm) | (inuse & rfbm);
+    cpu_stq_data_ra(env, ptr + 512, new_bv, ra);
+}
+
+void helper_xsave(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    do_xsave(env, ptr, rfbm, get_xinuse(env), -1, GETPC());
+}
+
+void helper_xsaveopt(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    uint64_t inuse = get_xinuse(env);
+    do_xsave(env, ptr, rfbm, inuse, inuse, GETPC());
+}
 
-    cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-    fpus = cpu_lduw_data(env, ptr + 2);
-    fptag = cpu_lduw_data(env, ptr + 4);
+static void do_xrstor_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, fpus, fptag;
+    target_ulong addr;
+
+    cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, ra));
+    fpus = cpu_lduw_data_ra(env, ptr + 2, ra);
+    fptag = cpu_lduw_data_ra(env, ptr + 4, ra);
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
     fptag ^= 0xff;
@@ -1185,34 +1301,227 @@ void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, addr);
+        floatx80 tmp = helper_fldt(env, addr, ra);
         ST(i) = tmp;
         addr += 16;
     }
+}
+
+static void do_xrstor_mxcsr(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    cpu_set_mxcsr(env, cpu_ldl_data_ra(env, ptr + 0x18, ra));
+}
+
+static void do_xrstor_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+    target_ulong addr;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    addr = ptr + 0xa0;
+    for (i = 0; i < nb_xmm_regs; i++) {
+        env->xmm_regs[i].ZMM_Q(0) = cpu_ldq_data_ra(env, addr, ra);
+        env->xmm_regs[i].ZMM_Q(1) = cpu_ldq_data_ra(env, addr + 8, ra);
+        addr += 16;
+    }
+}
+
+static void do_xrstor_bndregs(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    int i;
+
+    for (i = 0; i < 4; i++, addr += 16) {
+        env->bnd_regs[i].lb = cpu_ldq_data_ra(env, addr, ra);
+        env->bnd_regs[i].ub = cpu_ldq_data_ra(env, addr + 8, ra);
+    }
+}
+
+static void do_xrstor_bndcsr(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    /* FIXME: Extend highest implemented bit of linear address.  */
+    env->bndcs_regs.cfgu = cpu_ldq_data_ra(env, addr, ra);
+    env->bndcs_regs.sts = cpu_ldq_data_ra(env, addr + 8, ra);
+}
+
+static void do_xrstor_pkru(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    env->pkru = cpu_ldq_data_ra(env, addr, ra);
+}
+
+void helper_fxrstor(CPUX86State *env, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+
+    /* The operand must be 16 byte aligned */
+    if (ptr & 0xf) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    do_xrstor_fpu(env, ptr, ra);
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
-        /* XXX: finish it */
-        cpu_set_mxcsr(env, cpu_ldl_data(env, ptr + 0x18));
-        /* cpu_ldl_data(env, ptr + 0x1c); */
-        if (env->hflags & HF_CS64_MASK) {
-            nb_xmm_regs = 16;
-        } else {
-            nb_xmm_regs = 8;
-        }
-        addr = ptr + 0xa0;
-        /* Fast FXRESTORE leaves out the XMM registers */
+        do_xrstor_mxcsr(env, ptr, ra);
+        /* Fast FXRSTOR leaves out the XMM registers */
         if (!(env->efer & MSR_EFER_FFXSR)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
-            for (i = 0; i < nb_xmm_regs; i++) {
-                env->xmm_regs[i].XMM_Q(0) = cpu_ldq_data(env, addr);
-                env->xmm_regs[i].XMM_Q(1) = cpu_ldq_data(env, addr + 8);
-                addr += 16;
-            }
+            do_xrstor_sse(env, ptr, ra);
+        }
+    }
+}
+
+void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    uintptr_t ra = GETPC();
+    uint64_t xstate_bv, xcomp_bv0, xcomp_bv1;
+
+    rfbm &= env->xcr0;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, ra);
+    }
+
+    /* The operand must be 64 byte aligned.  */
+    if (ptr & 63) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    xstate_bv = cpu_ldq_data_ra(env, ptr + 512, ra);
+
+    if ((int64_t)xstate_bv < 0) {
+        /* FIXME: Compact form.  */
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* Standard form.  */
+
+    /* The XSTATE field must not set bits not present in XCR0.  */
+    if (xstate_bv & ~env->xcr0) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* The XCOMP field must be zero.  */
+    xcomp_bv0 = cpu_ldq_data_ra(env, ptr + 520, ra);
+    xcomp_bv1 = cpu_ldq_data_ra(env, ptr + 528, ra);
+    if (xcomp_bv0 || xcomp_bv1) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    if (rfbm & XSTATE_FP_MASK) {
+        if (xstate_bv & XSTATE_FP_MASK) {
+            do_xrstor_fpu(env, ptr, ra);
+        } else {
+            helper_fninit(env);
+            memset(env->fpregs, 0, sizeof(env->fpregs));
+        }
+    }
+    if (rfbm & XSTATE_SSE_MASK) {
+        /* Note that the standard form of XRSTOR loads MXCSR from memory
+           whether or not the XSTATE_BV bit is set.  */
+        do_xrstor_mxcsr(env, ptr, ra);
+        if (xstate_bv & XSTATE_SSE_MASK) {
+            do_xrstor_sse(env, ptr, ra);
+        } else {
+            /* ??? When AVX is implemented, we may have to be more
+               selective in the clearing.  */
+            memset(env->xmm_regs, 0, sizeof(env->xmm_regs));
+        }
+    }
+    if (rfbm & XSTATE_BNDREGS_MASK) {
+        if (xstate_bv & XSTATE_BNDREGS_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_BNDREGS_BIT].offset;
+            do_xrstor_bndregs(env, ptr + off, ra);
+            env->hflags |= HF_MPX_IU_MASK;
+        } else {
+            memset(env->bnd_regs, 0, sizeof(env->bnd_regs));
+            env->hflags &= ~HF_MPX_IU_MASK;
+        }
+    }
+    if (rfbm & XSTATE_BNDCSR_MASK) {
+        if (xstate_bv & XSTATE_BNDCSR_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_BNDCSR_BIT].offset;
+            do_xrstor_bndcsr(env, ptr + off, ra);
+        } else {
+            memset(&env->bndcs_regs, 0, sizeof(env->bndcs_regs));
+        }
+        cpu_sync_bndcs_hflags(env);
+    }
+    if (rfbm & XSTATE_PKRU_MASK) {
+        uint64_t old_pkru = env->pkru;
+        if (xstate_bv & XSTATE_PKRU_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_PKRU_BIT].offset;
+            do_xrstor_pkru(env, ptr + off, ra);
+        } else {
+            env->pkru = 0;
+        }
+        if (env->pkru != old_pkru) {
+            CPUState *cs = CPU(x86_env_get_cpu(env));
+            tlb_flush(cs, 1);
         }
     }
 }
 
+uint64_t helper_xgetbv(CPUX86State *env, uint32_t ecx)
+{
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+
+    switch (ecx) {
+    case 0:
+        return env->xcr0;
+    case 1:
+        if (env->features[FEAT_XSAVE] & CPUID_XSAVE_XGETBV1) {
+            return env->xcr0 & get_xinuse(env);
+        }
+        break;
+    }
+    raise_exception_ra(env, EXCP0D_GPF, GETPC());
+}
+
+void helper_xsetbv(CPUX86State *env, uint32_t ecx, uint64_t mask)
+{
+    uint32_t dummy, ena_lo, ena_hi;
+    uint64_t ena;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+
+    /* Only XCR0 is defined at present; the FPU may not be disabled.  */
+    if (ecx != 0 || (mask & XSTATE_FP_MASK) == 0) {
+        goto do_gpf;
+    }
+
+    /* Disallow enabling unimplemented features.  */
+    cpu_x86_cpuid(env, 0x0d, 0, &ena_lo, &dummy, &dummy, &ena_hi);
+    ena = ((uint64_t)ena_hi << 32) | ena_lo;
+    if (mask & ~ena) {
+        goto do_gpf;
+    }
+
+    /* Disallow enabling only half of MPX.  */
+    if ((mask ^ (mask * (XSTATE_BNDCSR_MASK / XSTATE_BNDREGS_MASK)))
+        & XSTATE_BNDCSR_MASK) {
+        goto do_gpf;
+    }
+
+    env->xcr0 = mask;
+    cpu_sync_bndcs_hflags(env);
+    return;
+
+ do_gpf:
+    raise_exception_ra(env, EXCP0D_GPF, GETPC());
+}
+
 void cpu_get_fp80(uint64_t *pmant, uint16_t *pexp, floatx80 f)
 {
     CPU_LDoubleU temp;
diff --git a/qemu/target-i386/gdbstub.c b/qemu/target-i386/gdbstub.c
index ff99cfb00..4b5071398 100644
--- a/qemu/target-i386/gdbstub.c
+++ b/qemu/target-i386/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
@@ -61,8 +61,8 @@ int x86_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
         n -= IDX_XMM_REGS;
         if (n < CPU_NB_REGS32 ||
             (TARGET_LONG_BITS == 64 && env->hflags & HF_CS64_MASK)) {
-            stq_p(mem_buf, env->xmm_regs[n].XMM_Q(0));
-            stq_p(mem_buf + 8, env->xmm_regs[n].XMM_Q(1));
+            stq_p(mem_buf, env->xmm_regs[n].ZMM_Q(0));
+            stq_p(mem_buf + 8, env->xmm_regs[n].ZMM_Q(1));
             return 16;
         }
     } else {
@@ -170,8 +170,8 @@ int x86_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
         n -= IDX_XMM_REGS;
         if (n < CPU_NB_REGS32 ||
             (TARGET_LONG_BITS == 64 && env->hflags & HF_CS64_MASK)) {
-            env->xmm_regs[n].XMM_Q(0) = ldq_p(mem_buf);
-            env->xmm_regs[n].XMM_Q(1) = ldq_p(mem_buf + 8);
+            env->xmm_regs[n].ZMM_Q(0) = ldq_p(mem_buf);
+            env->xmm_regs[n].ZMM_Q(1) = ldq_p(mem_buf + 8);
             return 16;
         }
     } else {
diff --git a/qemu/target-i386/helper.c b/qemu/target-i386/helper.c
index 5480a96a0..bf3e76207 100644
--- a/qemu/target-i386/helper.c
+++ b/qemu/target-i386/helper.c
@@ -17,12 +17,14 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "hw/i386/apic_internal.h"
 #endif
 
 static void cpu_x86_version(CPUX86State *env, int *family, int *model)
@@ -177,6 +179,196 @@ done:
     cpu_fprintf(f, "\n");
 }
 
+#ifndef CONFIG_USER_ONLY
+
+/* ARRAY_SIZE check is not required because
+ * DeliveryMode(dm) has a size of 3 bit.
+ */
+static inline const char *dm2str(uint32_t dm)
+{
+    static const char *str[] = {
+        "Fixed",
+        "...",
+        "SMI",
+        "...",
+        "NMI",
+        "INIT",
+        "...",
+        "ExtINT"
+    };
+    return str[dm];
+}
+
+static void dump_apic_lvt(FILE *f, fprintf_function cpu_fprintf,
+                          const char *name, uint32_t lvt, bool is_timer)
+{
+    uint32_t dm = (lvt & APIC_LVT_DELIV_MOD) >> APIC_LVT_DELIV_MOD_SHIFT;
+    cpu_fprintf(f,
+                "%s\t 0x%08x %s %-5s %-6s %-7s %-12s %-6s",
+                name, lvt,
+                lvt & APIC_LVT_INT_POLARITY ? "active-lo" : "active-hi",
+                lvt & APIC_LVT_LEVEL_TRIGGER ? "level" : "edge",
+                lvt & APIC_LVT_MASKED ? "masked" : "",
+                lvt & APIC_LVT_DELIV_STS ? "pending" : "",
+                !is_timer ?
+                    "" : lvt & APIC_LVT_TIMER_PERIODIC ?
+                            "periodic" : lvt & APIC_LVT_TIMER_TSCDEADLINE ?
+                                            "tsc-deadline" : "one-shot",
+                dm2str(dm));
+    if (dm != APIC_DM_NMI) {
+        cpu_fprintf(f, " (vec %u)\n", lvt & APIC_VECTOR_MASK);
+    } else {
+        cpu_fprintf(f, "\n");
+    }
+}
+
+/* ARRAY_SIZE check is not required because
+ * destination shorthand has a size of 2 bit.
+ */
+static inline const char *shorthand2str(uint32_t shorthand)
+{
+    const char *str[] = {
+        "no-shorthand", "self", "all-self", "all"
+    };
+    return str[shorthand];
+}
+
+static inline uint8_t divider_conf(uint32_t divide_conf)
+{
+    uint8_t divide_val = ((divide_conf & 0x8) >> 1) | (divide_conf & 0x3);
+
+    return divide_val == 7 ? 1 : 2 << divide_val;
+}
+
+static inline void mask2str(char *str, uint32_t val, uint8_t size)
+{
+    while (size--) {
+        *str++ = (val >> size) & 1 ? '1' : '0';
+    }
+    *str = 0;
+}
+
+#define MAX_LOGICAL_APIC_ID_MASK_SIZE 16
+
+static void dump_apic_icr(FILE *f, fprintf_function cpu_fprintf,
+                          APICCommonState *s, CPUX86State *env)
+{
+    uint32_t icr = s->icr[0], icr2 = s->icr[1];
+    uint8_t dest_shorthand = \
+        (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
+    bool logical_mod = icr & APIC_ICR_DEST_MOD;
+    char apic_id_str[MAX_LOGICAL_APIC_ID_MASK_SIZE + 1];
+    uint32_t dest_field;
+    bool x2apic;
+
+    cpu_fprintf(f, "ICR\t 0x%08x %s %s %s %s\n",
+                icr,
+                logical_mod ? "logical" : "physical",
+                icr & APIC_ICR_TRIGGER_MOD ? "level" : "edge",
+                icr & APIC_ICR_LEVEL ? "assert" : "de-assert",
+                shorthand2str(dest_shorthand));
+
+    cpu_fprintf(f, "ICR2\t 0x%08x", icr2);
+    if (dest_shorthand != 0) {
+        cpu_fprintf(f, "\n");
+        return;
+    }
+    x2apic = env->features[FEAT_1_ECX] & CPUID_EXT_X2APIC;
+    dest_field = x2apic ? icr2 : icr2 >> APIC_ICR_DEST_SHIFT;
+
+    if (!logical_mod) {
+        if (x2apic) {
+            cpu_fprintf(f, " cpu %u (X2APIC ID)\n", dest_field);
+        } else {
+            cpu_fprintf(f, " cpu %u (APIC ID)\n",
+                        dest_field & APIC_LOGDEST_XAPIC_ID);
+        }
+        return;
+    }
+
+    if (s->dest_mode == 0xf) { /* flat mode */
+        mask2str(apic_id_str, icr2 >> APIC_ICR_DEST_SHIFT, 8);
+        cpu_fprintf(f, " mask %s (APIC ID)\n", apic_id_str);
+    } else if (s->dest_mode == 0) { /* cluster mode */
+        if (x2apic) {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_X2APIC_ID, 16);
+            cpu_fprintf(f, " cluster %u mask %s (X2APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_X2APIC_SHIFT, apic_id_str);
+        } else {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_XAPIC_ID, 4);
+            cpu_fprintf(f, " cluster %u mask %s (APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_XAPIC_SHIFT, apic_id_str);
+        }
+    }
+}
+
+static void dump_apic_interrupt(FILE *f, fprintf_function cpu_fprintf,
+                                const char *name, uint32_t *ireg_tab,
+                                uint32_t *tmr_tab)
+{
+    int i, empty = true;
+
+    cpu_fprintf(f, "%s\t ", name);
+    for (i = 0; i < 256; i++) {
+        if (apic_get_bit(ireg_tab, i)) {
+            cpu_fprintf(f, "%u%s ", i,
+                        apic_get_bit(tmr_tab, i) ? "(level)" : "");
+            empty = false;
+        }
+    }
+    cpu_fprintf(f, "%s\n", empty ? "(none)" : "");
+}
+
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    APICCommonState *s = APIC_COMMON(cpu->apic_state);
+    uint32_t *lvt = s->lvt;
+
+    cpu_fprintf(f, "dumping local APIC state for CPU %-2u\n\n",
+                CPU(cpu)->cpu_index);
+    dump_apic_lvt(f, cpu_fprintf, "LVT0", lvt[APIC_LVT_LINT0], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVT1", lvt[APIC_LVT_LINT1], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTPC", lvt[APIC_LVT_PERFORM], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTERR", lvt[APIC_LVT_ERROR], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTTHMR", lvt[APIC_LVT_THERMAL], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTT", lvt[APIC_LVT_TIMER], true);
+
+    cpu_fprintf(f, "Timer\t DCR=0x%x (divide by %u) initial_count = %u\n",
+                s->divide_conf & APIC_DCR_MASK,
+                divider_conf(s->divide_conf),
+                s->initial_count);
+
+    cpu_fprintf(f, "SPIV\t 0x%08x APIC %s, focus=%s, spurious vec %u\n",
+                s->spurious_vec,
+                s->spurious_vec & APIC_SPURIO_ENABLED ? "enabled" : "disabled",
+                s->spurious_vec & APIC_SPURIO_FOCUS ? "on" : "off",
+                s->spurious_vec & APIC_VECTOR_MASK);
+
+    dump_apic_icr(f, cpu_fprintf, s, &cpu->env);
+
+    cpu_fprintf(f, "ESR\t 0x%08x\n", s->esr);
+
+    dump_apic_interrupt(f, cpu_fprintf, "ISR", s->isr, s->tmr);
+    dump_apic_interrupt(f, cpu_fprintf, "IRR", s->irr, s->tmr);
+
+    cpu_fprintf(f, "\nAPR 0x%02x TPR 0x%02x DFR 0x%02x LDR 0x%02x",
+                s->arb_id, s->tpr, s->dest_mode, s->log_dest);
+    if (s->dest_mode == 0) {
+        cpu_fprintf(f, "(cluster %u: id %u)",
+                    s->log_dest >> APIC_LOGDEST_XAPIC_SHIFT,
+                    s->log_dest & APIC_LOGDEST_XAPIC_ID);
+    }
+    cpu_fprintf(f, " PPR 0x%02x\n", apic_get_ppr(s));
+}
+#else
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+}
+#endif /* !CONFIG_USER_ONLY */
+
 #define DUMP_CODE_BYTES_TOTAL    50
 #define DUMP_CODE_BYTES_BACKWARD 20
 
@@ -344,10 +536,10 @@ void x86_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
         for(i=0;i<nb;i++) {
             cpu_fprintf(f, "XMM%02d=%08x%08x%08x%08x",
                         i,
-                        env->xmm_regs[i].XMM_L(3),
-                        env->xmm_regs[i].XMM_L(2),
-                        env->xmm_regs[i].XMM_L(1),
-                        env->xmm_regs[i].XMM_L(0));
+                        env->xmm_regs[i].ZMM_L(3),
+                        env->xmm_regs[i].ZMM_L(2),
+                        env->xmm_regs[i].ZMM_L(1),
+                        env->xmm_regs[i].ZMM_L(0));
             if ((i & 1) == 1)
                 cpu_fprintf(f, "\n");
             else
@@ -455,6 +647,7 @@ void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3)
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
+    uint32_t hflags;
 
 #if defined(DEBUG_MMU)
     printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
@@ -464,24 +657,33 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
          CR4_SMEP_MASK | CR4_SMAP_MASK)) {
         tlb_flush(CPU(cpu), 1);
     }
+
+    /* Clear bits we're going to recompute.  */
+    hflags = env->hflags & ~(HF_OSFXSR_MASK | HF_SMAP_MASK);
+
     /* SSE handling */
     if (!(env->features[FEAT_1_EDX] & CPUID_SSE)) {
         new_cr4 &= ~CR4_OSFXSR_MASK;
     }
-    env->hflags &= ~HF_OSFXSR_MASK;
     if (new_cr4 & CR4_OSFXSR_MASK) {
-        env->hflags |= HF_OSFXSR_MASK;
+        hflags |= HF_OSFXSR_MASK;
     }
 
     if (!(env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_SMAP)) {
         new_cr4 &= ~CR4_SMAP_MASK;
     }
-    env->hflags &= ~HF_SMAP_MASK;
     if (new_cr4 & CR4_SMAP_MASK) {
-        env->hflags |= HF_SMAP_MASK;
+        hflags |= HF_SMAP_MASK;
+    }
+
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKU)) {
+        new_cr4 &= ~CR4_PKE_MASK;
     }
 
     env->cr[4] = new_cr4;
+    env->hflags = hflags;
+
+    cpu_sync_bndcs_hflags(env);
 }
 
 #if defined(CONFIG_USER_ONLY)
@@ -669,7 +871,7 @@ int x86_cpu_handle_mmu_fault(CPUState *cs, vaddr addr,
             /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
              * Leave bits 20-13 in place for setting accessed/dirty bits below.
              */
-            pte = pde | ((pde & 0x1fe000) << (32 - 13));
+            pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
             rsvd_mask = 0x200000;
             goto do_check_protect_pse36;
         }
@@ -699,38 +901,50 @@ do_check_protect_pse36:
         goto do_fault_rsvd;
     }
     ptep ^= PG_NX_MASK;
-    if ((ptep & PG_NX_MASK) && is_write1 == 2) {
+
+    /* can the page can be put in the TLB?  prot will tell us */
+    if (is_user && !(ptep & PG_USER_MASK)) {
         goto do_fault_protect;
     }
-    switch (mmu_idx) {
-    case MMU_USER_IDX:
-        if (!(ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
-        }
-        if (is_write && !(ptep & PG_RW_MASK)) {
-            goto do_fault_protect;
-        }
-        break;
 
-    case MMU_KSMAP_IDX:
-        if (is_write1 != 2 && (ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
+    prot = 0;
+    if (mmu_idx != MMU_KSMAP_IDX || !(ptep & PG_USER_MASK)) {
+        prot |= PAGE_READ;
+        if ((ptep & PG_RW_MASK) || (!is_user && !(env->cr[0] & CR0_WP_MASK))) {
+            prot |= PAGE_WRITE;
         }
-        /* fall through */
-    case MMU_KNOSMAP_IDX:
-        if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-            (ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
+    }
+    if (!(ptep & PG_NX_MASK) &&
+        (mmu_idx == MMU_USER_IDX ||
+         !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK)))) {
+        prot |= PAGE_EXEC;
+    }
+    if ((env->cr[4] & CR4_PKE_MASK) && (env->hflags & HF_LMA_MASK) &&
+        (ptep & PG_USER_MASK) && env->pkru) {
+        uint32_t pk = (pte & PG_PKRU_MASK) >> PG_PKRU_BIT;
+        uint32_t pkru_ad = (env->pkru >> pk * 2) & 1;
+        uint32_t pkru_wd = (env->pkru >> pk * 2) & 2;
+        uint32_t pkru_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+
+        if (pkru_ad) {
+            pkru_prot &= ~(PAGE_READ | PAGE_WRITE);
+        } else if (pkru_wd && (is_user || env->cr[0] & CR0_WP_MASK)) {
+            pkru_prot &= ~PAGE_WRITE;
         }
-        if ((env->cr[0] & CR0_WP_MASK) &&
-            is_write && !(ptep & PG_RW_MASK)) {
+
+        prot &= pkru_prot;
+        if ((pkru_prot & (1 << is_write1)) == 0) {
+            assert(is_write1 != 2);
+            error_code |= PG_ERROR_PK_MASK;
             goto do_fault_protect;
         }
-        break;
+    }
 
-    default: /* cannot happen */
-        break;
+    if ((prot & (1 << is_write1)) == 0) {
+        goto do_fault_protect;
     }
+
+    /* yes, it can! */
     is_dirty = is_write && !(pte & PG_DIRTY_MASK);
     if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
         pte |= PG_ACCESSED_MASK;
@@ -740,25 +954,13 @@ do_check_protect_pse36:
         x86_stl_phys_notdirty(cs, pte_addr, pte);
     }
 
-    /* the page can be put in the TLB */
-    prot = PAGE_READ;
-    if (!(ptep & PG_NX_MASK) &&
-        (mmu_idx == MMU_USER_IDX ||
-         !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK)))) {
-        prot |= PAGE_EXEC;
-    }
-    if (pte & PG_DIRTY_MASK) {
+    if (!(pte & PG_DIRTY_MASK)) {
         /* only set write access if already dirty... otherwise wait
            for dirty access */
-        if (is_user) {
-            if (ptep & PG_RW_MASK)
-                prot |= PAGE_WRITE;
-        } else {
-            if (!(env->cr[0] & CR0_WP_MASK) ||
-                (ptep & PG_RW_MASK))
-                prot |= PAGE_WRITE;
-        }
+        assert(!is_write);
+        prot &= ~PAGE_WRITE;
     }
+
  do_mapping:
     pte = pte & env->a20_mask;
 
@@ -771,6 +973,7 @@ do_check_protect_pse36:
     page_offset = vaddr & (page_size - 1);
     paddr = pte + page_offset;
 
+    assert(prot & (1 << is_write1));
     tlb_set_page_with_attrs(cs, vaddr, paddr, cpu_get_mem_attrs(env),
                             prot, mmu_idx, page_size);
     return 0;
@@ -883,7 +1086,7 @@ hwaddr x86_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
         if (!(pde & PG_PRESENT_MASK))
             return -1;
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-            pte = pde | ((pde & 0x1fe000) << (32 - 13));
+            pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
             page_size = 4096 * 1024;
         } else {
             /* page directory entry */
@@ -905,134 +1108,6 @@ out:
     return pte | page_offset;
 }
 
-void hw_breakpoint_insert(CPUX86State *env, int index)
-{
-    CPUState *cs = CPU(x86_env_get_cpu(env));
-    int type = 0, err = 0;
-
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            err = cpu_breakpoint_insert(cs, env->dr[index], BP_CPU,
-                                        &env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-        type = BP_CPU | BP_MEM_WRITE;
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    case DR7_TYPE_DATA_RW:
-        type = BP_CPU | BP_MEM_ACCESS;
-        break;
-    }
-
-    if (type != 0) {
-        err = cpu_watchpoint_insert(cs, env->dr[index],
-                                    hw_breakpoint_len(env->dr[7], index),
-                                    type, &env->cpu_watchpoint[index]);
-    }
-
-    if (err) {
-        env->cpu_breakpoint[index] = NULL;
-    }
-}
-
-void hw_breakpoint_remove(CPUX86State *env, int index)
-{
-    CPUState *cs;
-
-    if (!env->cpu_breakpoint[index]) {
-        return;
-    }
-    cs = CPU(x86_env_get_cpu(env));
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-    case DR7_TYPE_DATA_RW:
-        cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    }
-}
-
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
-{
-    target_ulong dr6;
-    int reg;
-    bool hit_enabled = false;
-
-    dr6 = env->dr[6] & ~0xf;
-    for (reg = 0; reg < DR7_MAX_BP; reg++) {
-        bool bp_match = false;
-        bool wp_match = false;
-
-        switch (hw_breakpoint_type(env->dr[7], reg)) {
-        case DR7_TYPE_BP_INST:
-            if (env->dr[reg] == env->eip) {
-                bp_match = true;
-            }
-            break;
-        case DR7_TYPE_DATA_WR:
-        case DR7_TYPE_DATA_RW:
-            if (env->cpu_watchpoint[reg] &&
-                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
-                wp_match = true;
-            }
-            break;
-        case DR7_TYPE_IO_RW:
-            break;
-        }
-        if (bp_match || wp_match) {
-            dr6 |= 1 << reg;
-            if (hw_breakpoint_enabled(env->dr[7], reg)) {
-                hit_enabled = true;
-            }
-        }
-    }
-
-    if (hit_enabled || force_dr6_update) {
-        env->dr[6] = dr6;
-    }
-
-    return hit_enabled;
-}
-
-void breakpoint_handler(CPUState *cs)
-{
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    CPUBreakpoint *bp;
-
-    if (cs->watchpoint_hit) {
-        if (cs->watchpoint_hit->flags & BP_CPU) {
-            cs->watchpoint_hit = NULL;
-            if (check_hw_breakpoints(env, false)) {
-                raise_exception(env, EXCP01_DB);
-            } else {
-                cpu_resume_from_signal(cs, NULL);
-            }
-        }
-    } else {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == env->eip) {
-                if (bp->flags & BP_CPU) {
-                    check_hw_breakpoints(env, true);
-                    raise_exception(env, EXCP01_DB);
-                }
-                break;
-            }
-        }
-    }
-}
-
 typedef struct MCEInjectionParams {
     Monitor *mon;
     X86CPU *cpu;
diff --git a/qemu/target-i386/helper.h b/qemu/target-i386/helper.h
index 74308f442..1320edc01 100644
--- a/qemu/target-i386/helper.h
+++ b/qemu/target-i386/helper.h
@@ -15,6 +15,14 @@ DEF_HELPER_2(idivl_EAX, void, env, tl)
 DEF_HELPER_2(divq_EAX, void, env, tl)
 DEF_HELPER_2(idivq_EAX, void, env, tl)
 #endif
+DEF_HELPER_FLAGS_2(cr4_testbit, TCG_CALL_NO_WG, void, env, i32)
+
+DEF_HELPER_FLAGS_2(bndck, TCG_CALL_NO_WG, void, env, i32)
+DEF_HELPER_FLAGS_3(bndldx32, TCG_CALL_NO_WG, i64, env, tl, tl)
+DEF_HELPER_FLAGS_3(bndldx64, TCG_CALL_NO_WG, i64, env, tl, tl)
+DEF_HELPER_FLAGS_5(bndstx32, TCG_CALL_NO_WG, void, env, tl, tl, i64, i64)
+DEF_HELPER_FLAGS_5(bndstx64, TCG_CALL_NO_WG, void, env, tl, tl, i64, i64)
+DEF_HELPER_1(bnd_jmp, void, env)
 
 DEF_HELPER_2(aam, void, env, int)
 DEF_HELPER_2(aad, void, env, int)
@@ -30,9 +38,9 @@ DEF_HELPER_2(verw, void, env, tl)
 DEF_HELPER_2(lldt, void, env, int)
 DEF_HELPER_2(ltr, void, env, int)
 DEF_HELPER_3(load_seg, void, env, int, int)
-DEF_HELPER_4(ljmp_protected, void, env, int, tl, int)
+DEF_HELPER_4(ljmp_protected, void, env, int, tl, tl)
 DEF_HELPER_5(lcall_real, void, env, int, tl, int, int)
-DEF_HELPER_5(lcall_protected, void, env, int, tl, int, int)
+DEF_HELPER_5(lcall_protected, void, env, int, tl, int, tl)
 DEF_HELPER_2(iret_real, void, env, int)
 DEF_HELPER_3(iret_protected, void, env, int, int)
 DEF_HELPER_3(lret_protected, void, env, int, int)
@@ -40,13 +48,10 @@ DEF_HELPER_2(read_crN, tl, env, int)
 DEF_HELPER_3(write_crN, void, env, int, tl)
 DEF_HELPER_2(lmsw, void, env, tl)
 DEF_HELPER_1(clts, void, env)
-DEF_HELPER_3(movl_drN_T0, void, env, int, tl)
+DEF_HELPER_FLAGS_3(set_dr, TCG_CALL_NO_WG, void, env, int, tl)
+DEF_HELPER_FLAGS_2(get_dr, TCG_CALL_NO_WG, tl, env, int)
 DEF_HELPER_2(invlpg, void, env, tl)
 
-DEF_HELPER_4(enter_level, void, env, int, int, tl)
-#ifdef TARGET_X86_64
-DEF_HELPER_4(enter64_level, void, env, int, int, tl)
-#endif
 DEF_HELPER_1(sysenter, void, env)
 DEF_HELPER_2(sysexit, void, env, int)
 #ifdef TARGET_X86_64
@@ -65,8 +70,6 @@ DEF_HELPER_1(cli, void, env)
 DEF_HELPER_1(sti, void, env)
 DEF_HELPER_1(clac, void, env)
 DEF_HELPER_1(stac, void, env)
-DEF_HELPER_1(set_inhibit_irq, void, env)
-DEF_HELPER_1(reset_inhibit_irq, void, env)
 DEF_HELPER_3(boundw, void, env, tl, int)
 DEF_HELPER_3(boundl, void, env, tl, int)
 DEF_HELPER_1(rsm, void, env)
@@ -92,6 +95,7 @@ DEF_HELPER_3(outw, void, env, i32, i32)
 DEF_HELPER_2(inw, tl, env, i32)
 DEF_HELPER_3(outl, void, env, i32, i32)
 DEF_HELPER_2(inl, tl, env, i32)
+DEF_HELPER_FLAGS_4(bpt_io, TCG_CALL_NO_WG, void, env, i32, i32, tl)
 
 DEF_HELPER_3(svm_check_intercept_param, void, env, i32, i64)
 DEF_HELPER_3(vmexit, void, env, i32, i64)
@@ -187,8 +191,15 @@ DEF_HELPER_3(fstenv, void, env, tl, int)
 DEF_HELPER_3(fldenv, void, env, tl, int)
 DEF_HELPER_3(fsave, void, env, tl, int)
 DEF_HELPER_3(frstor, void, env, tl, int)
-DEF_HELPER_3(fxsave, void, env, tl, int)
-DEF_HELPER_3(fxrstor, void, env, tl, int)
+DEF_HELPER_FLAGS_2(fxsave, TCG_CALL_NO_WG, void, env, tl)
+DEF_HELPER_FLAGS_2(fxrstor, TCG_CALL_NO_WG, void, env, tl)
+DEF_HELPER_FLAGS_3(xsave, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_3(xsaveopt, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_3(xrstor, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_2(xgetbv, TCG_CALL_NO_WG, i64, env, i32)
+DEF_HELPER_FLAGS_3(xsetbv, TCG_CALL_NO_WG, void, env, i32, i64)
+DEF_HELPER_FLAGS_2(rdpkru, TCG_CALL_NO_WG, i64, env, i32)
+DEF_HELPER_FLAGS_3(wrpkru, TCG_CALL_NO_WG, void, env, i32, i64)
 
 DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_FLAGS_1(ctz, TCG_CALL_NO_RWG_SE, tl, tl)
diff --git a/qemu/target-i386/hyperv.c b/qemu/target-i386/hyperv.c
new file mode 100644
index 000000000..39a230f11
--- /dev/null
+++ b/qemu/target-i386/hyperv.c
@@ -0,0 +1,140 @@
+/*
+ * QEMU KVM Hyper-V support
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hyperv.h"
+#include "standard-headers/asm-x86/hyperv.h"
+
+int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit)
+{
+    CPUX86State *env = &cpu->env;
+
+    switch (exit->type) {
+    case KVM_EXIT_HYPERV_SYNIC:
+        if (!cpu->hyperv_synic) {
+            return -1;
+        }
+
+        /*
+         * For now just track changes in SynIC control and msg/evt pages msr's.
+         * When SynIC messaging/events processing will be added in future
+         * here we will do messages queues flushing and pages remapping.
+         */
+        switch (exit->u.synic.msr) {
+        case HV_X64_MSR_SCONTROL:
+            env->msr_hv_synic_control = exit->u.synic.control;
+            break;
+        case HV_X64_MSR_SIMP:
+            env->msr_hv_synic_msg_page = exit->u.synic.msg_page;
+            break;
+        case HV_X64_MSR_SIEFP:
+            env->msr_hv_synic_evt_page = exit->u.synic.evt_page;
+            break;
+        default:
+            return -1;
+        }
+        return 0;
+    case KVM_EXIT_HYPERV_HCALL: {
+        uint16_t code;
+
+        code  = exit->u.hcall.input & 0xffff;
+        switch (code) {
+        case HVCALL_POST_MESSAGE:
+        case HVCALL_SIGNAL_EVENT:
+        default:
+            exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE;
+            return 0;
+        }
+    }
+    default:
+        return -1;
+    }
+}
+
+static void kvm_hv_sint_ack_handler(EventNotifier *notifier)
+{
+    HvSintRoute *sint_route = container_of(notifier, HvSintRoute,
+                                           sint_ack_notifier);
+    event_notifier_test_and_clear(notifier);
+    if (sint_route->sint_ack_clb) {
+        sint_route->sint_ack_clb(sint_route);
+    }
+}
+
+HvSintRoute *kvm_hv_sint_route_create(uint32_t vcpu_id, uint32_t sint,
+                                      HvSintAckClb sint_ack_clb)
+{
+    HvSintRoute *sint_route;
+    int r, gsi;
+
+    sint_route = g_malloc0(sizeof(*sint_route));
+    r = event_notifier_init(&sint_route->sint_set_notifier, false);
+    if (r) {
+        goto err;
+    }
+
+    r = event_notifier_init(&sint_route->sint_ack_notifier, false);
+    if (r) {
+        goto err_sint_set_notifier;
+    }
+
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false,
+                               kvm_hv_sint_ack_handler);
+
+    gsi = kvm_irqchip_add_hv_sint_route(kvm_state, vcpu_id, sint);
+    if (gsi < 0) {
+        goto err_gsi;
+    }
+
+    r = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
+                                           &sint_route->sint_set_notifier,
+                                           &sint_route->sint_ack_notifier, gsi);
+    if (r) {
+        goto err_irqfd;
+    }
+    sint_route->gsi = gsi;
+    sint_route->sint_ack_clb = sint_ack_clb;
+    sint_route->vcpu_id = vcpu_id;
+    sint_route->sint = sint;
+
+    return sint_route;
+
+err_irqfd:
+    kvm_irqchip_release_virq(kvm_state, gsi);
+err_gsi:
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false, NULL);
+    event_notifier_cleanup(&sint_route->sint_ack_notifier);
+err_sint_set_notifier:
+    event_notifier_cleanup(&sint_route->sint_set_notifier);
+err:
+    g_free(sint_route);
+
+    return NULL;
+}
+
+void kvm_hv_sint_route_destroy(HvSintRoute *sint_route)
+{
+    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state,
+                                          &sint_route->sint_set_notifier,
+                                          sint_route->gsi);
+    kvm_irqchip_release_virq(kvm_state, sint_route->gsi);
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false, NULL);
+    event_notifier_cleanup(&sint_route->sint_ack_notifier);
+    event_notifier_cleanup(&sint_route->sint_set_notifier);
+    g_free(sint_route);
+}
+
+int kvm_hv_sint_route_set_sint(HvSintRoute *sint_route)
+{
+    return event_notifier_set(&sint_route->sint_set_notifier);
+}
diff --git a/qemu/target-i386/hyperv.h b/qemu/target-i386/hyperv.h
new file mode 100644
index 000000000..b26201f8b
--- /dev/null
+++ b/qemu/target-i386/hyperv.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU KVM Hyper-V support
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HYPERV_I386_H
+#define HYPERV_I386_H
+
+#include "cpu.h"
+#include "sysemu/kvm.h"
+#include "qemu/event_notifier.h"
+
+typedef struct HvSintRoute HvSintRoute;
+typedef void (*HvSintAckClb)(HvSintRoute *sint_route);
+
+struct HvSintRoute {
+    uint32_t sint;
+    uint32_t vcpu_id;
+    int gsi;
+    EventNotifier sint_set_notifier;
+    EventNotifier sint_ack_notifier;
+    HvSintAckClb sint_ack_clb;
+};
+
+int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit);
+
+HvSintRoute *kvm_hv_sint_route_create(uint32_t vcpu_id, uint32_t sint,
+                                      HvSintAckClb sint_ack_clb);
+
+void kvm_hv_sint_route_destroy(HvSintRoute *sint_route);
+
+int kvm_hv_sint_route_set_sint(HvSintRoute *sint_route);
+
+#endif
diff --git a/qemu/target-i386/int_helper.c b/qemu/target-i386/int_helper.c
index b0d78e6ee..cf5bbb048 100644
--- a/qemu/target-i386/int_helper.c
+++ b/qemu/target-i386/int_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
@@ -48,11 +49,11 @@ void helper_divb_AL(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff);
     den = (t0 & 0xff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -66,11 +67,11 @@ void helper_idivb_AL(CPUX86State *env, target_ulong t0)
     num = (int16_t)env->regs[R_EAX];
     den = (int8_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int8_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -84,11 +85,11 @@ void helper_divw_AX(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (t0 & 0xffff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -103,11 +104,11 @@ void helper_idivw_AX(CPUX86State *env, target_ulong t0)
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (int16_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int16_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -123,12 +124,12 @@ void helper_divl_EAX(CPUX86State *env, target_ulong t0)
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q > 0xffffffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -142,12 +143,12 @@ void helper_idivl_EAX(CPUX86State *env, target_ulong t0)
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q != (int32_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -379,12 +380,12 @@ void helper_divq_EAX(CPUX86State *env, target_ulong t0)
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (div64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
@@ -395,12 +396,12 @@ void helper_idivq_EAX(CPUX86State *env, target_ulong t0)
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (idiv64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
@@ -469,3 +470,13 @@ target_ulong helper_pext(target_ulong src, target_ulong mask)
 #include "shift_helper_template.h"
 #undef SHIFT
 #endif
+
+/* Test that BIT is enabled in CR4.  If not, raise an illegal opcode
+   exception.  This reduces the requirements for rare CR4 bits being
+   mapped into HFLAGS.  */
+void helper_cr4_testbit(CPUX86State *env, uint32_t bit)
+{
+    if (unlikely((env->cr[4] & bit) == 0)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+}
diff --git a/qemu/target-i386/kvm-stub.c b/qemu/target-i386/kvm-stub.c
index 6fefd65c2..8df9c5953 100644
--- a/qemu/target-i386/kvm-stub.c
+++ b/qemu/target-i386/kvm-stub.c
@@ -9,6 +9,7 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "kvm_i386.h"
 
diff --git a/qemu/target-i386/kvm.c b/qemu/target-i386/kvm.c
index 721c580ed..799fdfa68 100644
--- a/qemu/target-i386/kvm.c
+++ b/qemu/target-i386/kvm.c
@@ -12,7 +12,8 @@
  *
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/utsname.h>
@@ -25,16 +26,21 @@
 #include "sysemu/kvm_int.h"
 #include "kvm_i386.h"
 #include "cpu.h"
+#include "hyperv.h"
+
 #include "exec/gdbstub.h"
 #include "qemu/host-utils.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/i386/apic-msidef.h"
+
 #include "exec/ioport.h"
-#include <asm/hyperv.h>
+#include "standard-headers/asm-x86/hyperv.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "migration/migration.h"
 #include "exec/memattrs.h"
 
@@ -67,6 +73,7 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 
 static bool has_msr_star;
 static bool has_msr_hsave_pa;
+static bool has_msr_tsc_aux;
 static bool has_msr_tsc_adjust;
 static bool has_msr_tsc_deadline;
 static bool has_msr_feature_control;
@@ -80,12 +87,27 @@ static int lm_capable_kernel;
 static bool has_msr_hv_hypercall;
 static bool has_msr_hv_vapic;
 static bool has_msr_hv_tsc;
+static bool has_msr_hv_crash;
+static bool has_msr_hv_reset;
+static bool has_msr_hv_vpindex;
+static bool has_msr_hv_runtime;
+static bool has_msr_hv_synic;
+static bool has_msr_hv_stimer;
 static bool has_msr_mtrr;
 static bool has_msr_xss;
 
 static bool has_msr_architectural_pmu;
 static uint32_t num_architectural_pmu_counters;
 
+static int has_xsave;
+static int has_xcrs;
+static int has_pit_state2;
+
+int kvm_has_pit_state2(void)
+{
+    return has_pit_state2;
+}
+
 bool kvm_has_smm(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
@@ -119,6 +141,7 @@ static int kvm_get_tsc(CPUState *cs)
         return ret;
     }
 
+    assert(ret == 1);
     env->tsc = msr_data.entries[0].data;
     return 0;
 }
@@ -502,7 +525,43 @@ static bool hyperv_enabled(X86CPU *cpu)
     return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 &&
            (hyperv_hypercall_available(cpu) ||
             cpu->hyperv_time  ||
-            cpu->hyperv_relaxed_timing);
+            cpu->hyperv_relaxed_timing ||
+            cpu->hyperv_crash ||
+            cpu->hyperv_reset ||
+            cpu->hyperv_vpindex ||
+            cpu->hyperv_runtime ||
+            cpu->hyperv_synic ||
+            cpu->hyperv_stimer);
+}
+
+static int kvm_arch_set_tsc_khz(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    int r;
+
+    if (!env->tsc_khz) {
+        return 0;
+    }
+
+    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL) ?
+        kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz) :
+        -ENOTSUP;
+    if (r < 0) {
+        /* When KVM_SET_TSC_KHZ fails, it's an error only if the current
+         * TSC frequency doesn't match the one we want.
+         */
+        int cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
+                       kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
+                       -ENOTSUP;
+        if (cur_freq <= 0 || cur_freq != env->tsc_khz) {
+            error_report("warning: TSC frequency mismatch between "
+                         "VM and host, and TSC scaling unavailable");
+            return r;
+        }
+    }
+
+    return 0;
 }
 
 static Error *invtsc_mig_blocker;
@@ -532,7 +591,18 @@ int kvm_arch_init_vcpu(CPUState *cs)
     if (hyperv_enabled(cpu)) {
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
-        memcpy(signature, "Microsoft Hv", 12);
+        if (!cpu->hyperv_vendor_id) {
+            memcpy(signature, "Microsoft Hv", 12);
+        } else {
+            size_t len = strlen(cpu->hyperv_vendor_id);
+
+            if (len > 12) {
+                error_report("hv-vendor-id truncated to 12 characters");
+                len = 12;
+            }
+            memset(signature, 0, 12);
+            memcpy(signature, cpu->hyperv_vendor_id, len);
+        }
         c->eax = HYPERV_CPUID_MIN;
         c->ebx = signature[0];
         c->ecx = signature[1];
@@ -568,6 +638,41 @@ int kvm_arch_init_vcpu(CPUState *cs)
             c->eax |= 0x200;
             has_msr_hv_tsc = true;
         }
+        if (cpu->hyperv_crash && has_msr_hv_crash) {
+            c->edx |= HV_X64_GUEST_CRASH_MSR_AVAILABLE;
+        }
+        c->edx |= HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
+        if (cpu->hyperv_reset && has_msr_hv_reset) {
+            c->eax |= HV_X64_MSR_RESET_AVAILABLE;
+        }
+        if (cpu->hyperv_vpindex && has_msr_hv_vpindex) {
+            c->eax |= HV_X64_MSR_VP_INDEX_AVAILABLE;
+        }
+        if (cpu->hyperv_runtime && has_msr_hv_runtime) {
+            c->eax |= HV_X64_MSR_VP_RUNTIME_AVAILABLE;
+        }
+        if (cpu->hyperv_synic) {
+            int sint;
+
+            if (!has_msr_hv_synic ||
+                kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_SYNIC, 0)) {
+                fprintf(stderr, "Hyper-V SynIC is not supported by kernel\n");
+                return -ENOSYS;
+            }
+
+            c->eax |= HV_X64_MSR_SYNIC_AVAILABLE;
+            env->msr_hv_synic_version = HV_SYNIC_VERSION_1;
+            for (sint = 0; sint < ARRAY_SIZE(env->msr_hv_synic_sint); sint++) {
+                env->msr_hv_synic_sint[sint] = HV_SYNIC_SINT_MASKED;
+            }
+        }
+        if (cpu->hyperv_stimer) {
+            if (!has_msr_hv_stimer) {
+                fprintf(stderr, "Hyper-V timers aren't supported by kernel\n");
+                return -ENOSYS;
+            }
+            c->eax |= HV_X64_MSR_SYNTIMER_AVAILABLE;
+        }
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
         if (cpu->hyperv_relaxed_timing) {
@@ -732,7 +837,7 @@ int kvm_arch_init_vcpu(CPUState *cs)
         && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
            (CPUID_MCE | CPUID_MCA)
         && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
-        uint64_t mcg_cap;
+        uint64_t mcg_cap, unsupported_caps;
         int banks;
         int ret;
 
@@ -742,18 +847,24 @@ int kvm_arch_init_vcpu(CPUState *cs)
             return ret;
         }
 
-        if (banks > MCE_BANKS_DEF) {
-            banks = MCE_BANKS_DEF;
+        if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) {
+            error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)",
+                         (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks);
+            return -ENOTSUP;
+        }
+
+        unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK);
+        if (unsupported_caps) {
+            error_report("warning: Unsupported MCG_CAP bits: 0x%" PRIx64,
+                         unsupported_caps);
         }
-        mcg_cap &= MCE_CAP_DEF;
-        mcg_cap |= banks;
-        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &mcg_cap);
+
+        env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK;
+        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap);
         if (ret < 0) {
             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
             return ret;
         }
-
-        env->mcg_cap = mcg_cap;
     }
 
     qemu_add_vm_change_state_handler(cpu_update_state, env);
@@ -781,22 +892,35 @@ int kvm_arch_init_vcpu(CPUState *cs)
         return r;
     }
 
-    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL);
-    if (r && env->tsc_khz) {
-        r = kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz);
-        if (r < 0) {
-            fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
-            return r;
+    r = kvm_arch_set_tsc_khz(cs);
+    if (r < 0) {
+        return r;
+    }
+
+    /* vcpu's TSC frequency is either specified by user, or following
+     * the value used by KVM if the former is not present. In the
+     * latter case, we query it from KVM and record in env->tsc_khz,
+     * so that vcpu's TSC frequency can be migrated later via this field.
+     */
+    if (!env->tsc_khz) {
+        r = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
+            kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
+            -ENOTSUP;
+        if (r > 0) {
+            env->tsc_khz = r;
         }
     }
 
-    if (kvm_has_xsave()) {
+    if (has_xsave) {
         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
     }
 
     if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
         has_msr_mtrr = true;
     }
+    if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP)) {
+        has_msr_tsc_aux = false;
+    }
 
     return 0;
 }
@@ -864,6 +988,10 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_hsave_pa = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_TSC_AUX) {
+                    has_msr_tsc_aux = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_TSC_ADJUST) {
                     has_msr_tsc_adjust = true;
                     continue;
@@ -888,6 +1016,30 @@ static int kvm_get_supported_msrs(KVMState *s)
                     has_msr_xss = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_CRASH_CTL) {
+                    has_msr_hv_crash = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_RESET) {
+                    has_msr_hv_reset = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_INDEX) {
+                    has_msr_hv_vpindex = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_RUNTIME) {
+                    has_msr_hv_runtime = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_SCONTROL) {
+                    has_msr_hv_synic = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_STIMER0_CONFIG) {
+                    has_msr_hv_stimer = true;
+                    continue;
+                }
             }
         }
 
@@ -938,6 +1090,18 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     int ret;
     struct utsname utsname;
 
+#ifdef KVM_CAP_XSAVE
+    has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
+#endif
+
+#ifdef KVM_CAP_XCRS
+    has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
+#endif
+
+#ifdef KVM_CAP_PIT_STATE2
+    has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
+#endif
+
     ret = kvm_get_supported_msrs(s);
     if (ret < 0) {
         return ret;
@@ -1027,7 +1191,7 @@ static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
     lhs->l = (flags >> DESC_L_SHIFT) & 1;
     lhs->g = (flags & DESC_G_MASK) != 0;
     lhs->avl = (flags & DESC_AVL_MASK) != 0;
-    lhs->unusable = 0;
+    lhs->unusable = !lhs->present;
     lhs->padding = 0;
 }
 
@@ -1036,14 +1200,18 @@ static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
     lhs->selector = rhs->selector;
     lhs->base = rhs->base;
     lhs->limit = rhs->limit;
-    lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
-                 (rhs->present * DESC_P_MASK) |
-                 (rhs->dpl << DESC_DPL_SHIFT) |
-                 (rhs->db << DESC_B_SHIFT) |
-                 (rhs->s * DESC_S_MASK) |
-                 (rhs->l << DESC_L_SHIFT) |
-                 (rhs->g * DESC_G_MASK) |
-                 (rhs->avl * DESC_AVL_MASK);
+    if (rhs->unusable) {
+        lhs->flags = 0;
+    } else {
+        lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
+                     (rhs->present * DESC_P_MASK) |
+                     (rhs->dpl << DESC_DPL_SHIFT) |
+                     (rhs->db << DESC_B_SHIFT) |
+                     (rhs->s * DESC_S_MASK) |
+                     (rhs->l << DESC_L_SHIFT) |
+                     (rhs->g * DESC_G_MASK) |
+                     (rhs->avl * DESC_AVL_MASK);
+    }
 }
 
 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
@@ -1115,8 +1283,8 @@ static int kvm_put_fpu(X86CPU *cpu)
     }
     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
     for (i = 0; i < CPU_NB_REGS; i++) {
-        stq_p(&fpu.xmm[i][0], env->xmm_regs[i].XMM_Q(0));
-        stq_p(&fpu.xmm[i][8], env->xmm_regs[i].XMM_Q(1));
+        stq_p(&fpu.xmm[i][0], env->xmm_regs[i].ZMM_Q(0));
+        stq_p(&fpu.xmm[i][8], env->xmm_regs[i].ZMM_Q(1));
     }
     fpu.mxcsr = env->mxcsr;
 
@@ -1137,6 +1305,7 @@ static int kvm_put_fpu(X86CPU *cpu)
 #define XSAVE_OPMASK      272
 #define XSAVE_ZMM_Hi256   288
 #define XSAVE_Hi16_ZMM    416
+#define XSAVE_PKRU        672
 
 static int kvm_put_xsave(X86CPU *cpu)
 {
@@ -1146,7 +1315,7 @@ static int kvm_put_xsave(X86CPU *cpu)
     uint8_t *xmm, *ymmh, *zmmh;
     int i, r;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_put_fpu(cpu);
     }
 
@@ -1177,19 +1346,20 @@ static int kvm_put_xsave(X86CPU *cpu)
     ymmh = (uint8_t *)&xsave->region[XSAVE_YMMH_SPACE];
     zmmh = (uint8_t *)&xsave->region[XSAVE_ZMM_Hi256];
     for (i = 0; i < CPU_NB_REGS; i++, xmm += 16, ymmh += 16, zmmh += 32) {
-        stq_p(xmm,     env->xmm_regs[i].XMM_Q(0));
-        stq_p(xmm+8,   env->xmm_regs[i].XMM_Q(1));
-        stq_p(ymmh,    env->xmm_regs[i].XMM_Q(2));
-        stq_p(ymmh+8,  env->xmm_regs[i].XMM_Q(3));
-        stq_p(zmmh,    env->xmm_regs[i].XMM_Q(4));
-        stq_p(zmmh+8,  env->xmm_regs[i].XMM_Q(5));
-        stq_p(zmmh+16, env->xmm_regs[i].XMM_Q(6));
-        stq_p(zmmh+24, env->xmm_regs[i].XMM_Q(7));
+        stq_p(xmm,     env->xmm_regs[i].ZMM_Q(0));
+        stq_p(xmm+8,   env->xmm_regs[i].ZMM_Q(1));
+        stq_p(ymmh,    env->xmm_regs[i].ZMM_Q(2));
+        stq_p(ymmh+8,  env->xmm_regs[i].ZMM_Q(3));
+        stq_p(zmmh,    env->xmm_regs[i].ZMM_Q(4));
+        stq_p(zmmh+8,  env->xmm_regs[i].ZMM_Q(5));
+        stq_p(zmmh+16, env->xmm_regs[i].ZMM_Q(6));
+        stq_p(zmmh+24, env->xmm_regs[i].ZMM_Q(7));
     }
 
 #ifdef TARGET_X86_64
     memcpy(&xsave->region[XSAVE_Hi16_ZMM], &env->xmm_regs[16],
             16 * sizeof env->xmm_regs[16]);
+    memcpy(&xsave->region[XSAVE_PKRU], &env->pkru, sizeof env->pkru);
 #endif
     r = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
     return r;
@@ -1200,7 +1370,7 @@ static int kvm_put_xcrs(X86CPU *cpu)
     CPUX86State *env = &cpu->env;
     struct kvm_xcrs xcrs = {};
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1277,6 +1447,7 @@ static int kvm_put_tscdeadline_msr(X86CPU *cpu)
         struct kvm_msr_entry entries[1];
     } msr_data;
     struct kvm_msr_entry *msrs = msr_data.entries;
+    int ret;
 
     if (!has_msr_tsc_deadline) {
         return 0;
@@ -1288,7 +1459,13 @@ static int kvm_put_tscdeadline_msr(X86CPU *cpu)
         .nmsrs = 1,
     };
 
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    assert(ret == 1);
+    return 0;
 }
 
 /*
@@ -1303,6 +1480,11 @@ static int kvm_put_msr_feature_control(X86CPU *cpu)
         struct kvm_msrs info;
         struct kvm_msr_entry entry;
     } msr_data;
+    int ret;
+
+    if (!has_msr_feature_control) {
+        return 0;
+    }
 
     kvm_msr_entry_set(&msr_data.entry, MSR_IA32_FEATURE_CONTROL,
                       cpu->env.msr_ia32_feature_control);
@@ -1311,7 +1493,13 @@ static int kvm_put_msr_feature_control(X86CPU *cpu)
         .nmsrs = 1,
     };
 
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    assert(ret == 1);
+    return 0;
 }
 
 static int kvm_put_msrs(X86CPU *cpu, int level)
@@ -1323,6 +1511,7 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
     } msr_data;
     struct kvm_msr_entry *msrs = msr_data.entries;
     int n = 0, i;
+    int ret;
 
     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
@@ -1334,6 +1523,9 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
     if (has_msr_hsave_pa) {
         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
     }
+    if (has_msr_tsc_aux) {
+        kvm_msr_entry_set(&msrs[n++], MSR_TSC_AUX, env->tsc_aux);
+    }
     if (has_msr_tsc_adjust) {
         kvm_msr_entry_set(&msrs[n++], MSR_TSC_ADJUST, env->tsc_adjust);
     }
@@ -1420,6 +1612,50 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
             kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_REFERENCE_TSC,
                               env->msr_hv_tsc);
         }
+        if (has_msr_hv_crash) {
+            int j;
+
+            for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++)
+                kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_CRASH_P0 + j,
+                                  env->msr_hv_crash_params[j]);
+
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_CRASH_CTL,
+                              HV_X64_MSR_CRASH_CTL_NOTIFY);
+        }
+        if (has_msr_hv_runtime) {
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_VP_RUNTIME,
+                              env->msr_hv_runtime);
+        }
+        if (cpu->hyperv_synic) {
+            int j;
+
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_SCONTROL,
+                              env->msr_hv_synic_control);
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_SVERSION,
+                              env->msr_hv_synic_version);
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_SIEFP,
+                              env->msr_hv_synic_evt_page);
+            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_SIMP,
+                              env->msr_hv_synic_msg_page);
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_synic_sint); j++) {
+                kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_SINT0 + j,
+                                  env->msr_hv_synic_sint[j]);
+            }
+        }
+        if (has_msr_hv_stimer) {
+            int j;
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_config); j++) {
+                kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_STIMER0_CONFIG + j*2,
+                                env->msr_hv_stimer_config[j]);
+            }
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_count); j++) {
+                kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_STIMER0_COUNT + j*2,
+                                env->msr_hv_stimer_count[j]);
+            }
+        }
         if (has_msr_mtrr) {
             kvm_msr_entry_set(&msrs[n++], MSR_MTRRdefType, env->mtrr_deftype);
             kvm_msr_entry_set(&msrs[n++],
@@ -1469,8 +1705,13 @@ static int kvm_put_msrs(X86CPU *cpu, int level)
         .nmsrs = n,
     };
 
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
 
+    assert(ret == n);
+    return 0;
 }
 
 
@@ -1496,8 +1737,8 @@ static int kvm_get_fpu(X86CPU *cpu)
     }
     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
     for (i = 0; i < CPU_NB_REGS; i++) {
-        env->xmm_regs[i].XMM_Q(0) = ldq_p(&fpu.xmm[i][0]);
-        env->xmm_regs[i].XMM_Q(1) = ldq_p(&fpu.xmm[i][8]);
+        env->xmm_regs[i].ZMM_Q(0) = ldq_p(&fpu.xmm[i][0]);
+        env->xmm_regs[i].ZMM_Q(1) = ldq_p(&fpu.xmm[i][8]);
     }
     env->mxcsr = fpu.mxcsr;
 
@@ -1512,7 +1753,7 @@ static int kvm_get_xsave(X86CPU *cpu)
     const uint8_t *xmm, *ymmh, *zmmh;
     uint16_t cwd, swd, twd;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_get_fpu(cpu);
     }
 
@@ -1548,19 +1789,20 @@ static int kvm_get_xsave(X86CPU *cpu)
     ymmh = (const uint8_t *)&xsave->region[XSAVE_YMMH_SPACE];
     zmmh = (const uint8_t *)&xsave->region[XSAVE_ZMM_Hi256];
     for (i = 0; i < CPU_NB_REGS; i++, xmm += 16, ymmh += 16, zmmh += 32) {
-        env->xmm_regs[i].XMM_Q(0) = ldq_p(xmm);
-        env->xmm_regs[i].XMM_Q(1) = ldq_p(xmm+8);
-        env->xmm_regs[i].XMM_Q(2) = ldq_p(ymmh);
-        env->xmm_regs[i].XMM_Q(3) = ldq_p(ymmh+8);
-        env->xmm_regs[i].XMM_Q(4) = ldq_p(zmmh);
-        env->xmm_regs[i].XMM_Q(5) = ldq_p(zmmh+8);
-        env->xmm_regs[i].XMM_Q(6) = ldq_p(zmmh+16);
-        env->xmm_regs[i].XMM_Q(7) = ldq_p(zmmh+24);
+        env->xmm_regs[i].ZMM_Q(0) = ldq_p(xmm);
+        env->xmm_regs[i].ZMM_Q(1) = ldq_p(xmm+8);
+        env->xmm_regs[i].ZMM_Q(2) = ldq_p(ymmh);
+        env->xmm_regs[i].ZMM_Q(3) = ldq_p(ymmh+8);
+        env->xmm_regs[i].ZMM_Q(4) = ldq_p(zmmh);
+        env->xmm_regs[i].ZMM_Q(5) = ldq_p(zmmh+8);
+        env->xmm_regs[i].ZMM_Q(6) = ldq_p(zmmh+16);
+        env->xmm_regs[i].ZMM_Q(7) = ldq_p(zmmh+24);
     }
 
 #ifdef TARGET_X86_64
     memcpy(&env->xmm_regs[16], &xsave->region[XSAVE_Hi16_ZMM],
            16 * sizeof env->xmm_regs[16]);
+    memcpy(&env->pkru, &xsave->region[XSAVE_PKRU], sizeof env->pkru);
 #endif
     return 0;
 }
@@ -1571,7 +1813,7 @@ static int kvm_get_xcrs(X86CPU *cpu)
     int i, ret;
     struct kvm_xcrs xcrs;
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1643,13 +1885,16 @@ static int kvm_get_sregs(X86CPU *cpu)
        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
 
-    hflags = (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
+    hflags = env->hflags & HFLAG_COPY_MASK;
+    hflags |= (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
-    hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
-                (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
+
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        hflags |= HF_OSFXSR_MASK;
+    }
 
     if (env->efer & MSR_EFER_LMA) {
         hflags |= HF_LMA_MASK;
@@ -1670,7 +1915,7 @@ static int kvm_get_sregs(X86CPU *cpu)
                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
         }
     }
-    env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
+    env->hflags = hflags;
 
     return 0;
 }
@@ -1696,6 +1941,9 @@ static int kvm_get_msrs(X86CPU *cpu)
     if (has_msr_hsave_pa) {
         msrs[n++].index = MSR_VM_HSAVE_PA;
     }
+    if (has_msr_tsc_aux) {
+        msrs[n++].index = MSR_TSC_AUX;
+    }
     if (has_msr_tsc_adjust) {
         msrs[n++].index = MSR_TSC_ADJUST;
     }
@@ -1775,6 +2023,35 @@ static int kvm_get_msrs(X86CPU *cpu)
     if (has_msr_hv_tsc) {
         msrs[n++].index = HV_X64_MSR_REFERENCE_TSC;
     }
+    if (has_msr_hv_crash) {
+        int j;
+
+        for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++) {
+            msrs[n++].index = HV_X64_MSR_CRASH_P0 + j;
+        }
+    }
+    if (has_msr_hv_runtime) {
+        msrs[n++].index = HV_X64_MSR_VP_RUNTIME;
+    }
+    if (cpu->hyperv_synic) {
+        uint32_t msr;
+
+        msrs[n++].index = HV_X64_MSR_SCONTROL;
+        msrs[n++].index = HV_X64_MSR_SVERSION;
+        msrs[n++].index = HV_X64_MSR_SIEFP;
+        msrs[n++].index = HV_X64_MSR_SIMP;
+        for (msr = HV_X64_MSR_SINT0; msr <= HV_X64_MSR_SINT15; msr++) {
+            msrs[n++].index = msr;
+        }
+    }
+    if (has_msr_hv_stimer) {
+        uint32_t msr;
+
+        for (msr = HV_X64_MSR_STIMER0_CONFIG; msr <= HV_X64_MSR_STIMER3_COUNT;
+             msr++) {
+            msrs[n++].index = msr;
+        }
+    }
     if (has_msr_mtrr) {
         msrs[n++].index = MSR_MTRRdefType;
         msrs[n++].index = MSR_MTRRfix64K_00000;
@@ -1803,6 +2080,7 @@ static int kvm_get_msrs(X86CPU *cpu)
         return ret;
     }
 
+    assert(ret == n);
     for (i = 0; i < ret; i++) {
         uint32_t index = msrs[i].index;
         switch (index) {
@@ -1838,6 +2116,9 @@ static int kvm_get_msrs(X86CPU *cpu)
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
             break;
+        case MSR_TSC_AUX:
+            env->tsc_aux = msrs[i].data;
+            break;
         case MSR_TSC_ADJUST:
             env->tsc_adjust = msrs[i].data;
             break;
@@ -1922,6 +2203,41 @@ static int kvm_get_msrs(X86CPU *cpu)
         case HV_X64_MSR_REFERENCE_TSC:
             env->msr_hv_tsc = msrs[i].data;
             break;
+        case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
+            env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data;
+            break;
+        case HV_X64_MSR_VP_RUNTIME:
+            env->msr_hv_runtime = msrs[i].data;
+            break;
+        case HV_X64_MSR_SCONTROL:
+            env->msr_hv_synic_control = msrs[i].data;
+            break;
+        case HV_X64_MSR_SVERSION:
+            env->msr_hv_synic_version = msrs[i].data;
+            break;
+        case HV_X64_MSR_SIEFP:
+            env->msr_hv_synic_evt_page = msrs[i].data;
+            break;
+        case HV_X64_MSR_SIMP:
+            env->msr_hv_synic_msg_page = msrs[i].data;
+            break;
+        case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
+            env->msr_hv_synic_sint[index - HV_X64_MSR_SINT0] = msrs[i].data;
+            break;
+        case HV_X64_MSR_STIMER0_CONFIG:
+        case HV_X64_MSR_STIMER1_CONFIG:
+        case HV_X64_MSR_STIMER2_CONFIG:
+        case HV_X64_MSR_STIMER3_CONFIG:
+            env->msr_hv_stimer_config[(index - HV_X64_MSR_STIMER0_CONFIG)/2] =
+                                msrs[i].data;
+            break;
+        case HV_X64_MSR_STIMER0_COUNT:
+        case HV_X64_MSR_STIMER1_COUNT:
+        case HV_X64_MSR_STIMER2_COUNT:
+        case HV_X64_MSR_STIMER3_COUNT:
+            env->msr_hv_stimer_count[(index - HV_X64_MSR_STIMER0_COUNT)/2] =
+                                msrs[i].data;
+            break;
         case MSR_MTRRdefType:
             env->mtrr_deftype = msrs[i].data;
             break;
@@ -2221,13 +2537,22 @@ int kvm_arch_put_registers(CPUState *cpu, int level)
 
     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
-    if (level >= KVM_PUT_RESET_STATE && has_msr_feature_control) {
+    if (level >= KVM_PUT_RESET_STATE) {
         ret = kvm_put_msr_feature_control(x86_cpu);
         if (ret < 0) {
             return ret;
         }
     }
 
+    if (level == KVM_PUT_FULL_STATE) {
+        /* We don't check for kvm_arch_set_tsc_khz() errors here,
+         * because TSC frequency mismatch shouldn't abort migration,
+         * unless the user explicitly asked for a more strict TSC
+         * setting (e.g. using an explicit "tsc-freq" option).
+         */
+        kvm_arch_set_tsc_khz(cpu);
+    }
+
     ret = kvm_getput_regs(x86_cpu, 1);
     if (ret < 0) {
         return ret;
@@ -2294,41 +2619,44 @@ int kvm_arch_get_registers(CPUState *cs)
 
     ret = kvm_getput_regs(cpu, 0);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_xsave(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_xcrs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_sregs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_msrs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_mp_state(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_apic(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_vcpu_events(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_debugregs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
-    return 0;
+    ret = 0;
+ out:
+    cpu_sync_bndcs_hflags(&cpu->env);
+    return ret;
 }
 
 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
@@ -2363,7 +2691,7 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
         }
     }
 
-    if (!kvm_irqchip_in_kernel()) {
+    if (!kvm_pic_in_kernel()) {
         qemu_mutex_lock_iothread();
     }
 
@@ -2381,7 +2709,7 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
         }
     }
 
-    if (!kvm_irqchip_in_kernel()) {
+    if (!kvm_pic_in_kernel()) {
         /* Try to inject an interrupt if the guest can accept it */
         if (run->ready_for_interrupt_injection &&
             (cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
@@ -2780,6 +3108,13 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
         ret = kvm_handle_debug(cpu, &run->debug.arch);
         qemu_mutex_unlock_iothread();
         break;
+    case KVM_EXIT_HYPERV:
+        ret = kvm_hv_handle_exit(cpu, &run->hyperv);
+        break;
+    case KVM_EXIT_IOAPIC_EOI:
+        ioapic_eoi_broadcast(run->eoi.vector);
+        ret = 0;
+        break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
         ret = -1;
@@ -2814,6 +3149,39 @@ void kvm_arch_init_irq_routing(KVMState *s)
      */
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_routing_allowed = true;
+
+    if (kvm_irqchip_is_split()) {
+        int i;
+
+        /* If the ioapic is in QEMU and the lapics are in KVM, reserve
+           MSI routes for signaling interrupts to the local apics. */
+        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+            struct MSIMessage msg = { 0x0, 0x0 };
+            if (kvm_irqchip_add_msi_route(s, msg, NULL) < 0) {
+                error_report("Could not enable split IRQ mode.");
+                exit(1);
+            }
+        }
+    }
+}
+
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
+{
+    int ret;
+    if (machine_kernel_irqchip_split(ms)) {
+        ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24);
+        if (ret) {
+            error_report("Could not enable split irqchip mode: %s\n",
+                         strerror(-ret));
+            exit(1);
+        } else {
+            DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n");
+            kvm_split_irqchip = true;
+            return 1;
+        }
+    } else {
+        return 0;
+    }
 }
 
 /* Classic KVM device assignment interface. Will remain x86 only. */
@@ -2957,7 +3325,7 @@ int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/qemu/target-i386/kvm_i386.h b/qemu/target-i386/kvm_i386.h
index c1b312ba2..42b00af1b 100644
--- a/qemu/target-i386/kvm_i386.h
+++ b/qemu/target-i386/kvm_i386.h
@@ -13,6 +13,8 @@
 
 #include "sysemu/kvm.h"
 
+#define kvm_apic_in_kernel() (kvm_irqchip_in_kernel())
+
 bool kvm_allows_irq0_override(void);
 bool kvm_has_smm(void);
 void kvm_synchronize_all_tsc(void);
diff --git a/qemu/target-i386/machine.c b/qemu/target-i386/machine.c
index a0df64b57..ee5b94922 100644
--- a/qemu/target-i386/machine.c
+++ b/qemu/target-i386/machine.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/i386/pc.h"
@@ -6,6 +7,8 @@
 #include "cpu.h"
 #include "sysemu/kvm.h"
 
+#include "qemu/error-report.h"
+
 static const VMStateDescription vmstate_segment = {
     .name = "segment",
     .version_id = 1,
@@ -36,15 +39,15 @@ static const VMStateDescription vmstate_xmm_reg = {
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(0), XMMReg),
-        VMSTATE_UINT64(XMM_Q(1), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(0), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(1), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
 #define VMSTATE_XMM_REGS(_field, _state, _start)                         \
     VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
-                             vmstate_xmm_reg, XMMReg)
+                             vmstate_xmm_reg, ZMMReg)
 
 /* YMMH format is the same as XMM, but for bits 128-255 */
 static const VMStateDescription vmstate_ymmh_reg = {
@@ -52,32 +55,32 @@ static const VMStateDescription vmstate_ymmh_reg = {
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(2), XMMReg),
-        VMSTATE_UINT64(XMM_Q(3), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(2), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(3), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
 #define VMSTATE_YMMH_REGS_VARS(_field, _state, _start, _v)               \
     VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, _v,    \
-                             vmstate_ymmh_reg, XMMReg)
+                             vmstate_ymmh_reg, ZMMReg)
 
 static const VMStateDescription vmstate_zmmh_reg = {
     .name = "zmmh_reg",
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(4), XMMReg),
-        VMSTATE_UINT64(XMM_Q(5), XMMReg),
-        VMSTATE_UINT64(XMM_Q(6), XMMReg),
-        VMSTATE_UINT64(XMM_Q(7), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(4), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(5), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(6), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(7), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
 #define VMSTATE_ZMMH_REGS_VARS(_field, _state, _start)                   \
     VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
-                             vmstate_zmmh_reg, XMMReg)
+                             vmstate_zmmh_reg, ZMMReg)
 
 #ifdef TARGET_X86_64
 static const VMStateDescription vmstate_hi16_zmm_reg = {
@@ -85,21 +88,21 @@ static const VMStateDescription vmstate_hi16_zmm_reg = {
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(0), XMMReg),
-        VMSTATE_UINT64(XMM_Q(1), XMMReg),
-        VMSTATE_UINT64(XMM_Q(2), XMMReg),
-        VMSTATE_UINT64(XMM_Q(3), XMMReg),
-        VMSTATE_UINT64(XMM_Q(4), XMMReg),
-        VMSTATE_UINT64(XMM_Q(5), XMMReg),
-        VMSTATE_UINT64(XMM_Q(6), XMMReg),
-        VMSTATE_UINT64(XMM_Q(7), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(0), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(1), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(2), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(3), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(4), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(5), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(6), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(7), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
 #define VMSTATE_Hi16_ZMM_REGS_VARS(_field, _state, _start)               \
     VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
-                             vmstate_hi16_zmm_reg, XMMReg)
+                             vmstate_hi16_zmm_reg, ZMMReg)
 #endif
 
 static const VMStateDescription vmstate_bnd_regs = {
@@ -331,6 +334,13 @@ static int cpu_post_load(void *opaque, int version_id)
     CPUX86State *env = &cpu->env;
     int i;
 
+    if (env->tsc_khz && env->user_tsc_khz &&
+        env->tsc_khz != env->user_tsc_khz) {
+        error_report("Mismatch between user-specified TSC frequency and "
+                     "migrated TSC frequency");
+        return -EINVAL;
+    }
+
     /*
      * Real mode guest segments register DPL should be zero.
      * Older KVM version were setting it wrongly.
@@ -367,8 +377,12 @@ static int cpu_post_load(void *opaque, int version_id)
 
     cpu_breakpoint_remove_all(cs, BP_CPU);
     cpu_watchpoint_remove_all(cs, BP_CPU);
-    for (i = 0; i < DR7_MAX_BP; i++) {
-        hw_breakpoint_insert(env, i);
+    {
+        /* Indicate all breakpoints disabled, as they are, then
+           let the helper re-enable them.  */
+        target_ulong dr7 = env->dr[7];
+        env->dr[7] = dr7 & ~(DR7_GLOBAL_BP_MASK | DR7_LOCAL_BP_MASK);
+        cpu_x86_update_dr7(env, dr7);
     }
     tlb_flush(cs, 1);
 
@@ -661,6 +675,115 @@ static const VMStateDescription vmstate_msr_hyperv_time = {
     }
 };
 
+static bool hyperv_crash_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < HV_X64_MSR_CRASH_PARAMS; i++) {
+        if (env->msr_hv_crash_params[i]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_crash = {
+    .name = "cpu/msr_hyperv_crash",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_crash_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.msr_hv_crash_params,
+                             X86CPU, HV_X64_MSR_CRASH_PARAMS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_runtime_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->msr_hv_runtime != 0;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_runtime = {
+    .name = "cpu/msr_hyperv_runtime",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_runtime_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.msr_hv_runtime, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_synic_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    if (env->msr_hv_synic_control != 0 ||
+        env->msr_hv_synic_evt_page != 0 ||
+        env->msr_hv_synic_msg_page != 0) {
+        return true;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(env->msr_hv_synic_sint); i++) {
+        if (env->msr_hv_synic_sint[i] != 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_synic = {
+    .name = "cpu/msr_hyperv_synic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_synic_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.msr_hv_synic_control, X86CPU),
+        VMSTATE_UINT64(env.msr_hv_synic_evt_page, X86CPU),
+        VMSTATE_UINT64(env.msr_hv_synic_msg_page, X86CPU),
+        VMSTATE_UINT64_ARRAY(env.msr_hv_synic_sint, X86CPU,
+                             HV_SYNIC_SINT_COUNT),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_stimer_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(env->msr_hv_stimer_config); i++) {
+        if (env->msr_hv_stimer_config[i] || env->msr_hv_stimer_count[i]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_stimer = {
+    .name = "cpu/msr_hyperv_stimer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_stimer_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.msr_hv_stimer_config,
+                             X86CPU, HV_SYNIC_STIMER_COUNT),
+        VMSTATE_UINT64_ARRAY(env.msr_hv_stimer_count,
+                             X86CPU, HV_SYNIC_STIMER_COUNT),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool avx512_needed(void *opaque)
 {
     X86CPU *cpu = opaque;
@@ -674,7 +797,7 @@ static bool avx512_needed(void *opaque)
     }
 
     for (i = 0; i < CPU_NB_REGS; i++) {
-#define ENV_XMM(reg, field) (env->xmm_regs[reg].XMM_Q(field))
+#define ENV_XMM(reg, field) (env->xmm_regs[reg].ZMM_Q(field))
         if (ENV_XMM(i, 4) || ENV_XMM(i, 6) ||
             ENV_XMM(i, 5) || ENV_XMM(i, 7)) {
             return true;
@@ -726,6 +849,47 @@ static const VMStateDescription vmstate_xss = {
     }
 };
 
+#ifdef TARGET_X86_64
+static bool pkru_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pkru != 0;
+}
+
+static const VMStateDescription vmstate_pkru = {
+    .name = "cpu/pkru",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pkru_needed,
+    .fields = (VMStateField[]){
+        VMSTATE_UINT32(env.pkru, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+#endif
+
+static bool tsc_khz_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(mc);
+    return env->tsc_khz && pcmc->save_tsc_khz;
+}
+
+static const VMStateDescription vmstate_tsc_khz = {
+    .name = "cpu/tsc_khz",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = tsc_khz_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(env.tsc_khz, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 VMStateDescription vmstate_x86_cpu = {
     .name = "cpu",
     .version_id = 12,
@@ -842,8 +1006,16 @@ VMStateDescription vmstate_x86_cpu = {
         &vmstate_msr_hypercall_hypercall,
         &vmstate_msr_hyperv_vapic,
         &vmstate_msr_hyperv_time,
+        &vmstate_msr_hyperv_crash,
+        &vmstate_msr_hyperv_runtime,
+        &vmstate_msr_hyperv_synic,
+        &vmstate_msr_hyperv_stimer,
         &vmstate_avx512,
         &vmstate_xss,
+        &vmstate_tsc_khz,
+#ifdef TARGET_X86_64
+        &vmstate_pkru,
+#endif
         NULL
     }
 };
diff --git a/qemu/target-i386/mem_helper.c b/qemu/target-i386/mem_helper.c
index 1aec8a5f1..85e75161b 100644
--- a/qemu/target-i386/mem_helper.c
+++ b/qemu/target-i386/mem_helper.c
@@ -17,37 +17,58 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
 
 /* broken thread support */
 
-static spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
+#if defined(CONFIG_USER_ONLY)
+QemuMutex global_cpu_lock;
 
 void helper_lock(void)
 {
-    spin_lock(&global_cpu_lock);
+    qemu_mutex_lock(&global_cpu_lock);
 }
 
 void helper_unlock(void)
 {
-    spin_unlock(&global_cpu_lock);
+    qemu_mutex_unlock(&global_cpu_lock);
 }
 
+void helper_lock_init(void)
+{
+    qemu_mutex_init(&global_cpu_lock);
+}
+#else
+void helper_lock(void)
+{
+}
+
+void helper_unlock(void)
+{
+}
+
+void helper_lock_init(void)
+{
+}
+#endif
+
 void helper_cmpxchg8b(CPUX86State *env, target_ulong a0)
 {
     uint64_t d;
     int eflags;
 
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d = cpu_ldq_data(env, a0);
+    d = cpu_ldq_data_ra(env, a0, GETPC());
     if (d == (((uint64_t)env->regs[R_EDX] << 32) | (uint32_t)env->regs[R_EAX])) {
-        cpu_stq_data(env, a0, ((uint64_t)env->regs[R_ECX] << 32) | (uint32_t)env->regs[R_EBX]);
+        cpu_stq_data_ra(env, a0, ((uint64_t)env->regs[R_ECX] << 32)
+                                  | (uint32_t)env->regs[R_EBX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d);
+        cpu_stq_data_ra(env, a0, d, GETPC());
         env->regs[R_EDX] = (uint32_t)(d >> 32);
         env->regs[R_EAX] = (uint32_t)d;
         eflags &= ~CC_Z;
@@ -62,19 +83,19 @@ void helper_cmpxchg16b(CPUX86State *env, target_ulong a0)
     int eflags;
 
     if ((a0 & 0xf) != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d0 = cpu_ldq_data(env, a0);
-    d1 = cpu_ldq_data(env, a0 + 8);
+    d0 = cpu_ldq_data_ra(env, a0, GETPC());
+    d1 = cpu_ldq_data_ra(env, a0 + 8, GETPC());
     if (d0 == env->regs[R_EAX] && d1 == env->regs[R_EDX]) {
-        cpu_stq_data(env, a0, env->regs[R_EBX]);
-        cpu_stq_data(env, a0 + 8, env->regs[R_ECX]);
+        cpu_stq_data_ra(env, a0, env->regs[R_EBX], GETPC());
+        cpu_stq_data_ra(env, a0 + 8, env->regs[R_ECX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d0);
-        cpu_stq_data(env, a0 + 8, d1);
+        cpu_stq_data_ra(env, a0, d0, GETPC());
+        cpu_stq_data_ra(env, a0 + 8, d1, GETPC());
         env->regs[R_EDX] = d1;
         env->regs[R_EAX] = d0;
         eflags &= ~CC_Z;
@@ -87,11 +108,14 @@ void helper_boundw(CPUX86State *env, target_ulong a0, int v)
 {
     int low, high;
 
-    low = cpu_ldsw_data(env, a0);
-    high = cpu_ldsw_data(env, a0 + 2);
+    low = cpu_ldsw_data_ra(env, a0, GETPC());
+    high = cpu_ldsw_data_ra(env, a0 + 2, GETPC());
     v = (int16_t)v;
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        if (env->hflags & HF_MPX_EN_MASK) {
+            env->bndcs_regs.sts = 0;
+        }
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -99,10 +123,13 @@ void helper_boundl(CPUX86State *env, target_ulong a0, int v)
 {
     int low, high;
 
-    low = cpu_ldl_data(env, a0);
-    high = cpu_ldl_data(env, a0 + 4);
+    low = cpu_ldl_data_ra(env, a0, GETPC());
+    high = cpu_ldl_data_ra(env, a0 + 4, GETPC());
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        if (env->hflags & HF_MPX_EN_MASK) {
+            env->bndcs_regs.sts = 0;
+        }
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -122,11 +149,7 @@ void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
         X86CPU *cpu = X86_CPU(cs);
         CPUX86State *env = &cpu->env;
 
-        if (retaddr) {
-            /* now we have a real cpu fault */
-            cpu_restore_state(cs, retaddr);
-        }
-        raise_exception_err(env, cs->exception_index, env->error_code);
+        raise_exception_err_ra(env, cs->exception_index, env->error_code, retaddr);
     }
 }
 #endif
diff --git a/qemu/target-i386/misc_helper.c b/qemu/target-i386/misc_helper.c
index 52c5d65e9..e31ec976a 100644
--- a/qemu/target-i386/misc_helper.c
+++ b/qemu/target-i386/misc_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
@@ -95,15 +96,6 @@ void helper_into(CPUX86State *env, int next_eip_addend)
     }
 }
 
-void helper_single_step(CPUX86State *env)
-{
-#ifndef CONFIG_USER_ONLY
-    check_hw_breakpoints(env, true);
-    env->dr[6] |= DR6_BS;
-#endif
-    raise_exception(env, EXCP01_DB);
-}
-
 void helper_cpuid(CPUX86State *env)
 {
     uint32_t eax, ebx, ecx, edx;
@@ -127,10 +119,6 @@ target_ulong helper_read_crN(CPUX86State *env, int reg)
 void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
 {
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-}
 #else
 target_ulong helper_read_crN(CPUX86State *env, int reg)
 {
@@ -176,27 +164,6 @@ void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
         break;
     }
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-    int i;
-
-    if (reg < 4) {
-        hw_breakpoint_remove(env, reg);
-        env->dr[reg] = t0;
-        hw_breakpoint_insert(env, reg);
-    } else if (reg == 7) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_remove(env, i);
-        }
-        env->dr[7] = t0;
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_insert(env, i);
-        }
-    } else {
-        env->dr[reg] = t0;
-    }
-}
 #endif
 
 void helper_lmsw(CPUX86State *env, target_ulong t0)
@@ -220,7 +187,7 @@ void helper_rdtsc(CPUX86State *env)
     uint64_t val;
 
     if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDTSC, 0);
 
@@ -238,7 +205,7 @@ void helper_rdtscp(CPUX86State *env)
 void helper_rdpmc(CPUX86State *env)
 {
     if ((env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDPMC, 0);
 
@@ -394,6 +361,12 @@ void helper_wrmsr(CPUX86State *env)
     case MSR_IA32_MISC_ENABLE:
         env->msr_ia32_misc_enable = val;
         break;
+    case MSR_IA32_BNDCFGS:
+        /* FIXME: #GP if reserved bits are set.  */
+        /* FIXME: Extend highest implemented bit of linear address.  */
+        env->msr_bndcfgs = val;
+        cpu_sync_bndcs_hflags(env);
+        break;
     default:
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
@@ -539,6 +512,9 @@ void helper_rdmsr(CPUX86State *env)
     case MSR_IA32_MISC_ENABLE:
         val = env->msr_ia32_misc_enable;
         break;
+    case MSR_IA32_BNDCFGS:
+        val = env->msr_bndcfgs;
+        break;
     default:
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
@@ -589,7 +565,7 @@ void helper_hlt(CPUX86State *env, int next_eip_addend)
 void helper_monitor(CPUX86State *env, target_ulong ptr)
 {
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     /* XXX: store address? */
     cpu_svm_check_intercept_param(env, SVM_EXIT_MONITOR, 0);
@@ -601,7 +577,7 @@ void helper_mwait(CPUX86State *env, int next_eip_addend)
     X86CPU *cpu;
 
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_MWAIT, 0);
     env->eip += next_eip_addend;
@@ -633,3 +609,30 @@ void helper_debug(CPUX86State *env)
     cs->exception_index = EXCP_DEBUG;
     cpu_loop_exit(cs);
 }
+
+uint64_t helper_rdpkru(CPUX86State *env, uint32_t ecx)
+{
+    if ((env->cr[4] & CR4_PKE_MASK) == 0) {
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+    }
+    if (ecx != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
+    }
+
+    return env->pkru;
+}
+
+void helper_wrpkru(CPUX86State *env, uint32_t ecx, uint64_t val)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+
+    if ((env->cr[4] & CR4_PKE_MASK) == 0) {
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+    }
+    if (ecx != 0 || (val & 0xFFFFFFFF00000000ull)) {
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
+    }
+
+    env->pkru = val;
+    tlb_flush(cs, 1);
+}
diff --git a/qemu/target-i386/monitor.c b/qemu/target-i386/monitor.c
new file mode 100644
index 000000000..fccfe40ab
--- /dev/null
+++ b/qemu/target-i386/monitor.c
@@ -0,0 +1,512 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hw/i386/pc.h"
+#include "sysemu/kvm.h"
+#include "hmp.h"
+
+
+static void print_pte(Monitor *mon, hwaddr addr,
+                      hwaddr pte,
+                      hwaddr mask)
+{
+#ifdef TARGET_X86_64
+    if (addr & (1ULL << 47)) {
+        addr |= -1LL << 48;
+    }
+#endif
+    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
+                   " %c%c%c%c%c%c%c%c%c\n",
+                   addr,
+                   pte & mask,
+                   pte & PG_NX_MASK ? 'X' : '-',
+                   pte & PG_GLOBAL_MASK ? 'G' : '-',
+                   pte & PG_PSE_MASK ? 'P' : '-',
+                   pte & PG_DIRTY_MASK ? 'D' : '-',
+                   pte & PG_ACCESSED_MASK ? 'A' : '-',
+                   pte & PG_PCD_MASK ? 'C' : '-',
+                   pte & PG_PWT_MASK ? 'T' : '-',
+                   pte & PG_USER_MASK ? 'U' : '-',
+                   pte & PG_RW_MASK ? 'W' : '-');
+}
+
+static void tlb_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    uint32_t pgd, pde, pte;
+
+    pgd = env->cr[3] & ~0xfff;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                /* 4M pages */
+                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    if (pte & PG_PRESENT_MASK) {
+                        print_pte(mon, (l1 << 22) + (l2 << 12),
+                                  pte & ~PG_PSE_MASK,
+                                  ~0xfff);
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        /* 2M pages with PAE, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
+                                  ~((hwaddr)(1 << 20) - 1));
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            if (pte & PG_PRESENT_MASK) {
+                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
+                                          + (l3 << 12),
+                                          pte & ~PG_PSE_MASK,
+                                          ~(hwaddr)0xfff);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#ifdef TARGET_X86_64
+static void tlb_info_64(Monitor *mon, CPUArchState *env)
+{
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        /* 1G pages, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
+                                  0x3ffffc0000000ULL);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    /* 2M pages, CR4.PSE is ignored */
+                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
+                                              (l3 << 21), pde,
+                                              0x3ffffffe00000ULL);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            print_pte(mon, (l1 << 39) +
+                                                      (l2 << 30) +
+                                                      (l3 << 21) + (l4 << 12),
+                                                      pte & ~PG_PSE_MASK,
+                                                      0x3fffffffff000ULL);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            tlb_info_64(mon, env);
+        } else
+#endif
+        {
+            tlb_info_pae32(mon, env);
+        }
+    } else {
+        tlb_info_32(mon, env);
+    }
+}
+
+static void mem_print(Monitor *mon, hwaddr *pstart,
+                      int *plast_prot,
+                      hwaddr end, int prot)
+{
+    int prot1;
+    prot1 = *plast_prot;
+    if (prot != prot1) {
+        if (*pstart != -1) {
+            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
+                           TARGET_FMT_plx " %c%c%c\n",
+                           *pstart, end, end - *pstart,
+                           prot1 & PG_USER_MASK ? 'u' : '-',
+                           'r',
+                           prot1 & PG_RW_MASK ? 'w' : '-');
+        }
+        if (prot != 0)
+            *pstart = end;
+        else
+            *pstart = -1;
+        *plast_prot = prot;
+    }
+}
+
+static void mem_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    int prot, last_prot;
+    uint32_t pgd, pde, pte;
+    hwaddr start, end;
+
+    pgd = env->cr[3] & ~0xfff;
+    last_prot = 0;
+    start = -1;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        end = l1 << 22;
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                mem_print(mon, &start, &last_prot, end, prot);
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    end = (l1 << 22) + (l2 << 12);
+                    if (pte & PG_PRESENT_MASK) {
+                        prot = pte & pde &
+                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                    } else {
+                        prot = 0;
+                    }
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+static void mem_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    int prot, last_prot;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+    hwaddr start, end;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        end = l1 << 30;
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                end = (l1 << 30) + (l2 << 21);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                      PG_PRESENT_MASK);
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
+                            if (pte & PG_PRESENT_MASK) {
+                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
+                                                    PG_PRESENT_MASK);
+                            } else {
+                                prot = 0;
+                            }
+                            mem_print(mon, &start, &last_prot, end, prot);
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+
+#ifdef TARGET_X86_64
+static void mem_info_64(Monitor *mon, CPUArchState *env)
+{
+    int prot, last_prot;
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        end = l1 << 39;
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                end = (l1 << 39) + (l2 << 30);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
+                                       PG_PRESENT_MASK);
+                        prot &= pml4e;
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                                  PG_PRESENT_MASK);
+                                    prot &= pml4e & pdpe;
+                                    mem_print(mon, &start, &last_prot, end, prot);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        end = (l1 << 39) + (l2 << 30) +
+                                            (l3 << 21) + (l4 << 12);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
+                                                          PG_PRESENT_MASK);
+                                            prot &= pml4e & pdpe & pde;
+                                        } else {
+                                            prot = 0;
+                                        }
+                                        mem_print(mon, &start, &last_prot, end, prot);
+                                    }
+                                }
+                            } else {
+                                prot = 0;
+                                mem_print(mon, &start, &last_prot, end, prot);
+                            }
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_mem(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            mem_info_64(mon, env);
+        } else
+#endif
+        {
+            mem_info_pae32(mon, env);
+        }
+    } else {
+        mem_info_32(mon, env);
+    }
+}
+
+void hmp_mce(Monitor *mon, const QDict *qdict)
+{
+    X86CPU *cpu;
+    CPUState *cs;
+    int cpu_index = qdict_get_int(qdict, "cpu_index");
+    int bank = qdict_get_int(qdict, "bank");
+    uint64_t status = qdict_get_int(qdict, "status");
+    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint64_t misc = qdict_get_int(qdict, "misc");
+    int flags = MCE_INJECT_UNCOND_AO;
+
+    if (qdict_get_try_bool(qdict, "broadcast", false)) {
+        flags |= MCE_INJECT_BROADCAST;
+    }
+    cs = qemu_get_cpu(cpu_index);
+    if (cs != NULL) {
+        cpu = X86_CPU(cs);
+        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
+                           flags);
+    }
+}
+
+static target_long monitor_get_pc(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->eip + env->segs[R_CS].base;
+}
+
+const MonitorDef monitor_defs[] = {
+#define SEG(name, seg) \
+    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
+    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
+    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
+
+    { "eax", offsetof(CPUX86State, regs[0]) },
+    { "ecx", offsetof(CPUX86State, regs[1]) },
+    { "edx", offsetof(CPUX86State, regs[2]) },
+    { "ebx", offsetof(CPUX86State, regs[3]) },
+    { "esp|sp", offsetof(CPUX86State, regs[4]) },
+    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
+    { "esi", offsetof(CPUX86State, regs[6]) },
+    { "edi", offsetof(CPUX86State, regs[7]) },
+#ifdef TARGET_X86_64
+    { "r8", offsetof(CPUX86State, regs[8]) },
+    { "r9", offsetof(CPUX86State, regs[9]) },
+    { "r10", offsetof(CPUX86State, regs[10]) },
+    { "r11", offsetof(CPUX86State, regs[11]) },
+    { "r12", offsetof(CPUX86State, regs[12]) },
+    { "r13", offsetof(CPUX86State, regs[13]) },
+    { "r14", offsetof(CPUX86State, regs[14]) },
+    { "r15", offsetof(CPUX86State, regs[15]) },
+#endif
+    { "eflags", offsetof(CPUX86State, eflags) },
+    { "eip", offsetof(CPUX86State, eip) },
+    SEG("cs", R_CS)
+    SEG("ds", R_DS)
+    SEG("es", R_ES)
+    SEG("ss", R_SS)
+    SEG("fs", R_FS)
+    SEG("gs", R_GS)
+    { "pc", 0, monitor_get_pc, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict)
+{
+    x86_cpu_dump_local_apic_state(mon_get_cpu(), (FILE *)mon, monitor_fprintf,
+                                  CPU_DUMP_FPU);
+}
+
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict)
+{
+    if (kvm_irqchip_in_kernel()) {
+        kvm_ioapic_dump_state(mon, qdict);
+    } else {
+        ioapic_dump_state(mon, qdict);
+    }
+}
diff --git a/qemu/target-i386/mpx_helper.c b/qemu/target-i386/mpx_helper.c
new file mode 100644
index 000000000..4d1785ece
--- /dev/null
+++ b/qemu/target-i386/mpx_helper.c
@@ -0,0 +1,167 @@
+/*
+ *  x86 MPX helpers
+ *
+ *  Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
+
+
+void cpu_sync_bndcs_hflags(CPUX86State *env)
+{
+    uint32_t hflags = env->hflags;
+    uint32_t hflags2 = env->hflags2;
+    uint32_t bndcsr;
+
+    if ((hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    if ((env->cr[4] & CR4_OSXSAVE_MASK)
+        && (env->xcr0 & XSTATE_BNDCSR_MASK)
+        && (bndcsr & BNDCFG_ENABLE)) {
+        hflags |= HF_MPX_EN_MASK;
+    } else {
+        hflags &= ~HF_MPX_EN_MASK;
+    }
+
+    if (bndcsr & BNDCFG_BNDPRESERVE) {
+        hflags2 |= HF2_MPX_PR_MASK;
+    } else {
+        hflags2 &= ~HF2_MPX_PR_MASK;
+    }
+
+    env->hflags = hflags;
+    env->hflags2 = hflags2;
+}
+
+void helper_bndck(CPUX86State *env, uint32_t fail)
+{
+    if (unlikely(fail)) {
+        env->bndcs_regs.sts = 1;
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
+    }
+}
+
+static uint64_t lookup_bte64(CPUX86State *env, uint64_t base, uintptr_t ra)
+{
+    uint64_t bndcsr, bde, bt;
+
+    if ((env->hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    bde = (extract64(base, 20, 28) << 3) + (extract64(bndcsr, 20, 44) << 12);
+    bt = cpu_ldq_data_ra(env, bde, ra);
+    if ((bt & 1) == 0) {
+        env->bndcs_regs.sts = bde | 2;
+        raise_exception_ra(env, EXCP05_BOUND, ra);
+    }
+
+    return (extract64(base, 3, 17) << 5) + (bt & ~7);
+}
+
+static uint32_t lookup_bte32(CPUX86State *env, uint32_t base, uintptr_t ra)
+{
+    uint32_t bndcsr, bde, bt;
+
+    if ((env->hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    bde = (extract32(base, 12, 20) << 2) + (bndcsr & TARGET_PAGE_MASK);
+    bt = cpu_ldl_data_ra(env, bde, ra);
+    if ((bt & 1) == 0) {
+        env->bndcs_regs.sts = bde | 2;
+        raise_exception_ra(env, EXCP05_BOUND, ra);
+    }
+
+    return (extract32(base, 2, 10) << 4) + (bt & ~3);
+}
+
+uint64_t helper_bndldx64(CPUX86State *env, target_ulong base, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+    uint64_t bte, lb, ub, pt;
+
+    bte = lookup_bte64(env, base, ra);
+    lb = cpu_ldq_data_ra(env, bte, ra);
+    ub = cpu_ldq_data_ra(env, bte + 8, ra);
+    pt = cpu_ldq_data_ra(env, bte + 16, ra);
+
+    if (pt != ptr) {
+        lb = ub = 0;
+    }
+    env->mmx_t0.MMX_Q(0) = ub;
+    return lb;
+}
+
+uint64_t helper_bndldx32(CPUX86State *env, target_ulong base, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+    uint32_t bte, lb, ub, pt;
+
+    bte = lookup_bte32(env, base, ra);
+    lb = cpu_ldl_data_ra(env, bte, ra);
+    ub = cpu_ldl_data_ra(env, bte + 4, ra);
+    pt = cpu_ldl_data_ra(env, bte + 8, ra);
+
+    if (pt != ptr) {
+        lb = ub = 0;
+    }
+    return ((uint64_t)ub << 32) | lb;
+}
+
+void helper_bndstx64(CPUX86State *env, target_ulong base, target_ulong ptr,
+                     uint64_t lb, uint64_t ub)
+{
+    uintptr_t ra = GETPC();
+    uint64_t bte;
+
+    bte = lookup_bte64(env, base, ra);
+    cpu_stq_data_ra(env, bte, lb, ra);
+    cpu_stq_data_ra(env, bte + 8, ub, ra);
+    cpu_stq_data_ra(env, bte + 16, ptr, ra);
+}
+
+void helper_bndstx32(CPUX86State *env, target_ulong base, target_ulong ptr,
+                     uint64_t lb, uint64_t ub)
+{
+    uintptr_t ra = GETPC();
+    uint32_t bte;
+
+    bte = lookup_bte32(env, base, ra);
+    cpu_stl_data_ra(env, bte, lb, ra);
+    cpu_stl_data_ra(env, bte + 4, ub, ra);
+    cpu_stl_data_ra(env, bte + 8, ptr, ra);
+}
+
+void helper_bnd_jmp(CPUX86State *env)
+{
+    if (!(env->hflags2 & HF2_MPX_PR_MASK)) {
+        memset(env->bnd_regs, 0, sizeof(env->bnd_regs));
+        env->hflags &= ~HF_MPX_IU_MASK;
+    }
+}
diff --git a/qemu/target-i386/ops_sse.h b/qemu/target-i386/ops_sse.h
index bee134bae..7a98f5386 100644
--- a/qemu/target-i386/ops_sse.h
+++ b/qemu/target-i386/ops_sse.h
@@ -26,15 +26,15 @@
 #define B(n) MMX_B(n)
 #define W(n) MMX_W(n)
 #define L(n) MMX_L(n)
-#define Q(n) q
+#define Q(n) MMX_Q(n)
 #define SUFFIX _mmx
 #else
-#define Reg XMMReg
+#define Reg ZMMReg
 #define XMM_ONLY(...) __VA_ARGS__
-#define B(n) XMM_B(n)
-#define W(n) XMM_W(n)
-#define L(n) XMM_L(n)
-#define Q(n) XMM_Q(n)
+#define B(n) ZMM_B(n)
+#define W(n) ZMM_W(n)
+#define L(n) ZMM_L(n)
+#define Q(n) ZMM_Q(n)
 #define SUFFIX _xmm
 #endif
 
@@ -483,7 +483,7 @@ void glue(helper_maskmov, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
 
     for (i = 0; i < (8 << SHIFT); i++) {
         if (s->B(i) & 0x80) {
-            cpu_stb_data(env, a0 + i, d->B(i));
+            cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC());
         }
     }
 }
@@ -582,26 +582,26 @@ void glue(helper_pshufhw, SUFFIX)(Reg *d, Reg *s, int order)
 #define SSE_HELPER_S(name, F)                                           \
     void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
-        d->XMM_S(1) = F(32, d->XMM_S(1), s->XMM_S(1));                  \
-        d->XMM_S(2) = F(32, d->XMM_S(2), s->XMM_S(2));                  \
-        d->XMM_S(3) = F(32, d->XMM_S(3), s->XMM_S(3));                  \
+        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
+        d->ZMM_S(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
+        d->ZMM_S(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
+        d->ZMM_S(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
+        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
-        d->XMM_D(1) = F(64, d->XMM_D(1), s->XMM_D(1));                  \
+        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
+        d->ZMM_D(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
+        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
     }
 
 #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status)
@@ -633,216 +633,216 @@ void helper_cvtps2pd(CPUX86State *env, Reg *d, Reg *s)
 {
     float32 s0, s1;
 
-    s0 = s->XMM_S(0);
-    s1 = s->XMM_S(1);
-    d->XMM_D(0) = float32_to_float64(s0, &env->sse_status);
-    d->XMM_D(1) = float32_to_float64(s1, &env->sse_status);
+    s0 = s->ZMM_S(0);
+    s1 = s->ZMM_S(1);
+    d->ZMM_D(0) = float32_to_float64(s0, &env->sse_status);
+    d->ZMM_D(1) = float32_to_float64(s1, &env->sse_status);
 }
 
 void helper_cvtpd2ps(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
-    d->XMM_S(1) = float64_to_float32(s->XMM_D(1), &env->sse_status);
+    d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_S(1) = float64_to_float32(s->ZMM_D(1), &env->sse_status);
     d->Q(1) = 0;
 }
 
 void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_D(0) = float32_to_float64(s->XMM_S(0), &env->sse_status);
+    d->ZMM_D(0) = float32_to_float64(s->ZMM_S(0), &env->sse_status);
 }
 
 void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
+    d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
 }
 
 /* integer to float */
 void helper_cvtdq2ps(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = int32_to_float32(s->XMM_L(0), &env->sse_status);
-    d->XMM_S(1) = int32_to_float32(s->XMM_L(1), &env->sse_status);
-    d->XMM_S(2) = int32_to_float32(s->XMM_L(2), &env->sse_status);
-    d->XMM_S(3) = int32_to_float32(s->XMM_L(3), &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(s->ZMM_L(0), &env->sse_status);
+    d->ZMM_S(1) = int32_to_float32(s->ZMM_L(1), &env->sse_status);
+    d->ZMM_S(2) = int32_to_float32(s->ZMM_L(2), &env->sse_status);
+    d->ZMM_S(3) = int32_to_float32(s->ZMM_L(3), &env->sse_status);
 }
 
 void helper_cvtdq2pd(CPUX86State *env, Reg *d, Reg *s)
 {
     int32_t l0, l1;
 
-    l0 = (int32_t)s->XMM_L(0);
-    l1 = (int32_t)s->XMM_L(1);
-    d->XMM_D(0) = int32_to_float64(l0, &env->sse_status);
-    d->XMM_D(1) = int32_to_float64(l1, &env->sse_status);
+    l0 = (int32_t)s->ZMM_L(0);
+    l1 = (int32_t)s->ZMM_L(1);
+    d->ZMM_D(0) = int32_to_float64(l0, &env->sse_status);
+    d->ZMM_D(1) = int32_to_float64(l1, &env->sse_status);
 }
 
-void helper_cvtpi2ps(CPUX86State *env, XMMReg *d, MMXReg *s)
+void helper_cvtpi2ps(CPUX86State *env, ZMMReg *d, MMXReg *s)
 {
-    d->XMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
-    d->XMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
+    d->ZMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
 }
 
-void helper_cvtpi2pd(CPUX86State *env, XMMReg *d, MMXReg *s)
+void helper_cvtpi2pd(CPUX86State *env, ZMMReg *d, MMXReg *s)
 {
-    d->XMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
-    d->XMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
+    d->ZMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
+    d->ZMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
 }
 
-void helper_cvtsi2ss(CPUX86State *env, XMMReg *d, uint32_t val)
+void helper_cvtsi2ss(CPUX86State *env, ZMMReg *d, uint32_t val)
 {
-    d->XMM_S(0) = int32_to_float32(val, &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(val, &env->sse_status);
 }
 
-void helper_cvtsi2sd(CPUX86State *env, XMMReg *d, uint32_t val)
+void helper_cvtsi2sd(CPUX86State *env, ZMMReg *d, uint32_t val)
 {
-    d->XMM_D(0) = int32_to_float64(val, &env->sse_status);
+    d->ZMM_D(0) = int32_to_float64(val, &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-void helper_cvtsq2ss(CPUX86State *env, XMMReg *d, uint64_t val)
+void helper_cvtsq2ss(CPUX86State *env, ZMMReg *d, uint64_t val)
 {
-    d->XMM_S(0) = int64_to_float32(val, &env->sse_status);
+    d->ZMM_S(0) = int64_to_float32(val, &env->sse_status);
 }
 
-void helper_cvtsq2sd(CPUX86State *env, XMMReg *d, uint64_t val)
+void helper_cvtsq2sd(CPUX86State *env, ZMMReg *d, uint64_t val)
 {
-    d->XMM_D(0) = int64_to_float64(val, &env->sse_status);
+    d->ZMM_D(0) = int64_to_float64(val, &env->sse_status);
 }
 #endif
 
 /* float to integer */
-void helper_cvtps2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvtps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
-    d->XMM_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
-    d->XMM_L(2) = float32_to_int32(s->XMM_S(2), &env->sse_status);
-    d->XMM_L(3) = float32_to_int32(s->XMM_S(3), &env->sse_status);
+    d->ZMM_L(0) = float32_to_int32(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_L(1) = float32_to_int32(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_L(2) = float32_to_int32(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_L(3) = float32_to_int32(s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_cvtpd2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvtpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
-    d->XMM_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
-    d->XMM_Q(1) = 0;
+    d->ZMM_L(0) = float64_to_int32(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_L(1) = float64_to_int32(s->ZMM_D(1), &env->sse_status);
+    d->ZMM_Q(1) = 0;
 }
 
-void helper_cvtps2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvtps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
-    d->MMX_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
+    d->MMX_L(0) = float32_to_int32(s->ZMM_S(0), &env->sse_status);
+    d->MMX_L(1) = float32_to_int32(s->ZMM_S(1), &env->sse_status);
 }
 
-void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
-    d->MMX_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
+    d->MMX_L(0) = float64_to_int32(s->ZMM_D(0), &env->sse_status);
+    d->MMX_L(1) = float64_to_int32(s->ZMM_D(1), &env->sse_status);
 }
 
-int32_t helper_cvtss2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvtss2si(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int32(s->XMM_S(0), &env->sse_status);
+    return float32_to_int32(s->ZMM_S(0), &env->sse_status);
 }
 
-int32_t helper_cvtsd2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvtsd2si(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int32(s->XMM_D(0), &env->sse_status);
+    return float64_to_int32(s->ZMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-int64_t helper_cvtss2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvtss2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int64(s->XMM_S(0), &env->sse_status);
+    return float32_to_int64(s->ZMM_S(0), &env->sse_status);
 }
 
-int64_t helper_cvtsd2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvtsd2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int64(s->XMM_D(0), &env->sse_status);
+    return float64_to_int64(s->ZMM_D(0), &env->sse_status);
 }
 #endif
 
 /* float to integer truncated */
-void helper_cvttps2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvttps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
-    d->XMM_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
-    d->XMM_L(2) = float32_to_int32_round_to_zero(s->XMM_S(2), &env->sse_status);
-    d->XMM_L(3) = float32_to_int32_round_to_zero(s->XMM_S(3), &env->sse_status);
+    d->ZMM_L(0) = float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_L(1) = float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_L(2) = float32_to_int32_round_to_zero(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_L(3) = float32_to_int32_round_to_zero(s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_cvttpd2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvttpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
-    d->XMM_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
-    d->XMM_Q(1) = 0;
+    d->ZMM_L(0) = float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_L(1) = float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status);
+    d->ZMM_Q(1) = 0;
 }
 
-void helper_cvttps2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvttps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
-    d->MMX_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
+    d->MMX_L(0) = float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
+    d->MMX_L(1) = float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status);
 }
 
-void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
-    d->MMX_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
+    d->MMX_L(0) = float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
+    d->MMX_L(1) = float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status);
 }
 
-int32_t helper_cvttss2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvttss2si(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
+    return float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
 }
 
-int32_t helper_cvttsd2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvttsd2si(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
+    return float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-int64_t helper_cvttss2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvttss2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
+    return float32_to_int64_round_to_zero(s->ZMM_S(0), &env->sse_status);
 }
 
-int64_t helper_cvttsd2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvttsd2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
+    return float64_to_int64_round_to_zero(s->ZMM_D(0), &env->sse_status);
 }
 #endif
 
-void helper_rsqrtps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rsqrtps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(0), &env->sse_status),
+    d->ZMM_S(0) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(0), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(1) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(1), &env->sse_status),
+    d->ZMM_S(1) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(1), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(2) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(2), &env->sse_status),
+    d->ZMM_S(2) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(2), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(3) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(3), &env->sse_status),
+    d->ZMM_S(3) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(3), &env->sse_status),
                               &env->sse_status);
 }
 
-void helper_rsqrtss(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(0), &env->sse_status),
+    d->ZMM_S(0) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(0), &env->sse_status),
                               &env->sse_status);
 }
 
-void helper_rcpps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rcpps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_div(float32_one, s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_div(float32_one, s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_div(float32_one, s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_div(float32_one, s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_div(float32_one, s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_div(float32_one, s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_rcpss(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
+    d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
 }
 
 static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
@@ -857,14 +857,14 @@ static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
     return (src >> shift) & mask;
 }
 
-void helper_extrq_r(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_extrq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), s->XMM_B(1), s->XMM_B(0));
+    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1), s->ZMM_B(0));
 }
 
-void helper_extrq_i(CPUX86State *env, XMMReg *d, int index, int length)
+void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length)
 {
-    d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), index, length);
+    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), index, length);
 }
 
 static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
@@ -879,94 +879,94 @@ static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
     return (src & ~(mask << shift)) | ((src & mask) << shift);
 }
 
-void helper_insertq_r(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_insertq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_Q(0) = helper_insertq(s->XMM_Q(0), s->XMM_B(9), s->XMM_B(8));
+    d->ZMM_Q(0) = helper_insertq(s->ZMM_Q(0), s->ZMM_B(9), s->ZMM_B(8));
 }
 
-void helper_insertq_i(CPUX86State *env, XMMReg *d, int index, int length)
+void helper_insertq_i(CPUX86State *env, ZMMReg *d, int index, int length)
 {
-    d->XMM_Q(0) = helper_insertq(d->XMM_Q(0), index, length);
+    d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), index, length);
 }
 
-void helper_haddps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_haddps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_S(0) = float32_add(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
-    r.XMM_S(1) = float32_add(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
-    r.XMM_S(2) = float32_add(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
-    r.XMM_S(3) = float32_add(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
+    r.ZMM_S(0) = float32_add(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(1) = float32_add(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
+    r.ZMM_S(2) = float32_add(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(3) = float32_add(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
     *d = r;
 }
 
-void helper_haddpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_haddpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_D(0) = float64_add(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
-    r.XMM_D(1) = float64_add(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
+    r.ZMM_D(0) = float64_add(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
+    r.ZMM_D(1) = float64_add(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
     *d = r;
 }
 
-void helper_hsubps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_hsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_S(0) = float32_sub(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
-    r.XMM_S(1) = float32_sub(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
-    r.XMM_S(2) = float32_sub(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
-    r.XMM_S(3) = float32_sub(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
+    r.ZMM_S(0) = float32_sub(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(1) = float32_sub(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
+    r.ZMM_S(2) = float32_sub(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(3) = float32_sub(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
     *d = r;
 }
 
-void helper_hsubpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_hsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_D(0) = float64_sub(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
-    r.XMM_D(1) = float64_sub(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
+    r.ZMM_D(0) = float64_sub(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
+    r.ZMM_D(1) = float64_sub(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
     *d = r;
 }
 
-void helper_addsubps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_addsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_sub(d->XMM_S(0), s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_add(d->XMM_S(1), s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_sub(d->XMM_S(2), s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_add(d->XMM_S(3), s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_sub(d->ZMM_S(0), s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_add(d->ZMM_S(1), s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_sub(d->ZMM_S(2), s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_add(d->ZMM_S(3), s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_addsubpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_addsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_D(0) = float64_sub(d->XMM_D(0), s->XMM_D(0), &env->sse_status);
-    d->XMM_D(1) = float64_add(d->XMM_D(1), s->XMM_D(1), &env->sse_status);
+    d->ZMM_D(0) = float64_sub(d->ZMM_D(0), s->ZMM_D(0), &env->sse_status);
+    d->ZMM_D(1) = float64_add(d->ZMM_D(1), s->ZMM_D(1), &env->sse_status);
 }
 
 /* XXX: unordered */
 #define SSE_HELPER_CMP(name, F)                                         \
     void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
-        d->XMM_L(1) = F(32, d->XMM_S(1), s->XMM_S(1));                  \
-        d->XMM_L(2) = F(32, d->XMM_S(2), s->XMM_S(2));                  \
-        d->XMM_L(3) = F(32, d->XMM_S(3), s->XMM_S(3));                  \
+        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
+        d->ZMM_L(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
+        d->ZMM_L(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
+        d->ZMM_L(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
+        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
-        d->XMM_Q(1) = F(64, d->XMM_D(1), s->XMM_D(1));                  \
+        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
+        d->ZMM_Q(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
+        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
     }
 
 #define FPU_CMPEQ(size, a, b)                                           \
@@ -1002,8 +1002,8 @@ void helper_ucomiss(CPUX86State *env, Reg *d, Reg *s)
     int ret;
     float32 s0, s1;
 
-    s0 = d->XMM_S(0);
-    s1 = s->XMM_S(0);
+    s0 = d->ZMM_S(0);
+    s1 = s->ZMM_S(0);
     ret = float32_compare_quiet(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1013,8 +1013,8 @@ void helper_comiss(CPUX86State *env, Reg *d, Reg *s)
     int ret;
     float32 s0, s1;
 
-    s0 = d->XMM_S(0);
-    s1 = s->XMM_S(0);
+    s0 = d->ZMM_S(0);
+    s1 = s->ZMM_S(0);
     ret = float32_compare(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1024,8 +1024,8 @@ void helper_ucomisd(CPUX86State *env, Reg *d, Reg *s)
     int ret;
     float64 d0, d1;
 
-    d0 = d->XMM_D(0);
-    d1 = s->XMM_D(0);
+    d0 = d->ZMM_D(0);
+    d1 = s->ZMM_D(0);
     ret = float64_compare_quiet(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1035,8 +1035,8 @@ void helper_comisd(CPUX86State *env, Reg *d, Reg *s)
     int ret;
     float64 d0, d1;
 
-    d0 = d->XMM_D(0);
-    d1 = s->XMM_D(0);
+    d0 = d->ZMM_D(0);
+    d1 = s->ZMM_D(0);
     ret = float64_compare(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1045,10 +1045,10 @@ uint32_t helper_movmskps(CPUX86State *env, Reg *s)
 {
     int b0, b1, b2, b3;
 
-    b0 = s->XMM_L(0) >> 31;
-    b1 = s->XMM_L(1) >> 31;
-    b2 = s->XMM_L(2) >> 31;
-    b3 = s->XMM_L(3) >> 31;
+    b0 = s->ZMM_L(0) >> 31;
+    b1 = s->ZMM_L(1) >> 31;
+    b2 = s->ZMM_L(2) >> 31;
+    b3 = s->ZMM_L(3) >> 31;
     return b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
 }
 
@@ -1056,8 +1056,8 @@ uint32_t helper_movmskpd(CPUX86State *env, Reg *s)
 {
     int b0, b1;
 
-    b0 = s->XMM_L(1) >> 31;
-    b1 = s->XMM_L(3) >> 31;
+    b0 = s->ZMM_L(1) >> 31;
+    b1 = s->ZMM_L(3) >> 31;
     return b0 | (b1 << 1);
 }
 
@@ -1736,10 +1736,10 @@ void glue(helper_roundps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         }
     }
 
-    d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_round_to_int(s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_round_to_int(s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_round_to_int(s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_round_to_int(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_round_to_int(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_round_to_int(s->ZMM_S(3), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1774,8 +1774,8 @@ void glue(helper_roundpd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         }
     }
 
-    d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
-    d->XMM_D(1) = float64_round_to_int(s->XMM_D(1), &env->sse_status);
+    d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_D(1) = float64_round_to_int(s->ZMM_D(1), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1810,7 +1810,7 @@ void glue(helper_roundss, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         }
     }
 
-    d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
+    d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1845,7 +1845,7 @@ void glue(helper_roundsd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
         }
     }
 
-    d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
+    d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1868,32 +1868,32 @@ void glue(helper_dpps, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
 
     if (mask & (1 << 4)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(0), s->XMM_S(0),
+                              float32_mul(d->ZMM_S(0), s->ZMM_S(0),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 5)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(1), s->XMM_S(1),
+                              float32_mul(d->ZMM_S(1), s->ZMM_S(1),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 6)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(2), s->XMM_S(2),
+                              float32_mul(d->ZMM_S(2), s->ZMM_S(2),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 7)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(3), s->XMM_S(3),
+                              float32_mul(d->ZMM_S(3), s->ZMM_S(3),
                                           &env->sse_status),
                               &env->sse_status);
     }
-    d->XMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
-    d->XMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
-    d->XMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
-    d->XMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
+    d->ZMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
+    d->ZMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
+    d->ZMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
+    d->ZMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
 }
 
 void glue(helper_dppd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
@@ -1902,18 +1902,18 @@ void glue(helper_dppd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
 
     if (mask & (1 << 4)) {
         iresult = float64_add(iresult,
-                              float64_mul(d->XMM_D(0), s->XMM_D(0),
+                              float64_mul(d->ZMM_D(0), s->ZMM_D(0),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 5)) {
         iresult = float64_add(iresult,
-                              float64_mul(d->XMM_D(1), s->XMM_D(1),
+                              float64_mul(d->ZMM_D(1), s->ZMM_D(1),
                                           &env->sse_status),
                               &env->sse_status);
     }
-    d->XMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
-    d->XMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
+    d->ZMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
+    d->ZMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
 }
 
 void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
@@ -2037,10 +2037,10 @@ static inline unsigned pcmpxstrx(CPUX86State *env, Reg *d, Reg *s,
         }
         break;
     case 3:
-        for (j = valids - validd; j >= 0; j--) {
+        for (j = valids; j >= 0; j--) {
             res <<= 1;
             v = 1;
-            for (i = MIN(upper - j, validd); i >= 0; i--) {
+            for (i = MIN(valids - j, validd); i >= 0; i--) {
                 v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
             }
             res |= v;
diff --git a/qemu/target-i386/ops_sse_header.h b/qemu/target-i386/ops_sse_header.h
index a68c7cc0c..64c5857cf 100644
--- a/qemu/target-i386/ops_sse_header.h
+++ b/qemu/target-i386/ops_sse_header.h
@@ -20,18 +20,18 @@
 #define Reg MMXReg
 #define SUFFIX _mmx
 #else
-#define Reg XMMReg
+#define Reg ZMMReg
 #define SUFFIX _xmm
 #endif
 
 #define dh_alias_Reg ptr
-#define dh_alias_XMMReg ptr
+#define dh_alias_ZMMReg ptr
 #define dh_alias_MMXReg ptr
 #define dh_ctype_Reg Reg *
-#define dh_ctype_XMMReg XMMReg *
+#define dh_ctype_ZMMReg ZMMReg *
 #define dh_ctype_MMXReg MMXReg *
 #define dh_is_signed_Reg dh_is_signed_ptr
-#define dh_is_signed_XMMReg dh_is_signed_ptr
+#define dh_is_signed_ZMMReg dh_is_signed_ptr
 #define dh_is_signed_MMXReg dh_is_signed_ptr
 
 DEF_HELPER_3(glue(psrlw, SUFFIX), void, env, Reg, Reg)
@@ -154,52 +154,52 @@ DEF_HELPER_3(cvtss2sd, void, env, Reg, Reg)
 DEF_HELPER_3(cvtsd2ss, void, env, Reg, Reg)
 DEF_HELPER_3(cvtdq2ps, void, env, Reg, Reg)
 DEF_HELPER_3(cvtdq2pd, void, env, Reg, Reg)
-DEF_HELPER_3(cvtpi2ps, void, env, XMMReg, MMXReg)
-DEF_HELPER_3(cvtpi2pd, void, env, XMMReg, MMXReg)
-DEF_HELPER_3(cvtsi2ss, void, env, XMMReg, i32)
-DEF_HELPER_3(cvtsi2sd, void, env, XMMReg, i32)
+DEF_HELPER_3(cvtpi2ps, void, env, ZMMReg, MMXReg)
+DEF_HELPER_3(cvtpi2pd, void, env, ZMMReg, MMXReg)
+DEF_HELPER_3(cvtsi2ss, void, env, ZMMReg, i32)
+DEF_HELPER_3(cvtsi2sd, void, env, ZMMReg, i32)
 
 #ifdef TARGET_X86_64
-DEF_HELPER_3(cvtsq2ss, void, env, XMMReg, i64)
-DEF_HELPER_3(cvtsq2sd, void, env, XMMReg, i64)
+DEF_HELPER_3(cvtsq2ss, void, env, ZMMReg, i64)
+DEF_HELPER_3(cvtsq2sd, void, env, ZMMReg, i64)
 #endif
 
-DEF_HELPER_3(cvtps2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvtpd2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvtps2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_2(cvtss2si, s32, env, XMMReg)
-DEF_HELPER_2(cvtsd2si, s32, env, XMMReg)
+DEF_HELPER_3(cvtps2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvtpd2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvtps2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_2(cvtss2si, s32, env, ZMMReg)
+DEF_HELPER_2(cvtsd2si, s32, env, ZMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER_2(cvtss2sq, s64, env, XMMReg)
-DEF_HELPER_2(cvtsd2sq, s64, env, XMMReg)
+DEF_HELPER_2(cvtss2sq, s64, env, ZMMReg)
+DEF_HELPER_2(cvtsd2sq, s64, env, ZMMReg)
 #endif
 
-DEF_HELPER_3(cvttps2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvttpd2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvttps2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_2(cvttss2si, s32, env, XMMReg)
-DEF_HELPER_2(cvttsd2si, s32, env, XMMReg)
+DEF_HELPER_3(cvttps2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvttpd2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvttps2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_2(cvttss2si, s32, env, ZMMReg)
+DEF_HELPER_2(cvttsd2si, s32, env, ZMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER_2(cvttss2sq, s64, env, XMMReg)
-DEF_HELPER_2(cvttsd2sq, s64, env, XMMReg)
+DEF_HELPER_2(cvttss2sq, s64, env, ZMMReg)
+DEF_HELPER_2(cvttsd2sq, s64, env, ZMMReg)
 #endif
 
-DEF_HELPER_3(rsqrtps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rsqrtss, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rcpps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rcpss, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(extrq_r, void, env, XMMReg, XMMReg)
-DEF_HELPER_4(extrq_i, void, env, XMMReg, int, int)
-DEF_HELPER_3(insertq_r, void, env, XMMReg, XMMReg)
-DEF_HELPER_4(insertq_i, void, env, XMMReg, int, int)
-DEF_HELPER_3(haddps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(haddpd, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(hsubps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(hsubpd, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(addsubps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(addsubpd, void, env, XMMReg, XMMReg)
+DEF_HELPER_3(rsqrtps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rsqrtss, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rcpps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rcpss, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(extrq_r, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_4(extrq_i, void, env, ZMMReg, int, int)
+DEF_HELPER_3(insertq_r, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_4(insertq_i, void, env, ZMMReg, int, int)
+DEF_HELPER_3(haddps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(haddpd, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(hsubps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(hsubpd, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(addsubps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(addsubpd, void, env, ZMMReg, ZMMReg)
 
 #define SSE_HELPER_CMP(name, F)                           \
     DEF_HELPER_3(name ## ps, void, env, Reg, Reg)         \
diff --git a/qemu/target-i386/seg_helper.c b/qemu/target-i386/seg_helper.c
index 8a4271ebe..b5f3d72fe 100644
--- a/qemu/target-i386/seg_helper.c
+++ b/qemu/target-i386/seg_helper.c
@@ -18,10 +18,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/log.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
 
 //#define DEBUG_PCALL
 
@@ -67,8 +69,9 @@
 #endif
 
 /* return non zero if error */
-static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
-                               uint32_t *e2_ptr, int selector)
+static inline int load_segment_ra(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector,
+                               uintptr_t retaddr)
 {
     SegmentCache *dt;
     int index;
@@ -84,11 +87,17 @@ static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
         return -1;
     }
     ptr = dt->base + index;
-    *e1_ptr = cpu_ldl_kernel(env, ptr);
-    *e2_ptr = cpu_ldl_kernel(env, ptr + 4);
+    *e1_ptr = cpu_ldl_kernel_ra(env, ptr, retaddr);
+    *e2_ptr = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
     return 0;
 }
 
+static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector)
+{
+    return load_segment_ra(env, e1_ptr, e2_ptr, selector, 0);
+}
+
 static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
 {
     unsigned int limit;
@@ -124,7 +133,8 @@ static inline void load_seg_vm(CPUX86State *env, int seg, int selector)
 }
 
 static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
-                                       uint32_t *esp_ptr, int dpl)
+                                       uint32_t *esp_ptr, int dpl,
+                                       uintptr_t retaddr)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
     int type, index, shift;
@@ -153,60 +163,61 @@ static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
     shift = type >> 3;
     index = (dpl * 4 + 2) << shift;
     if (index + (4 << shift) - 1 > env->tr.limit) {
-        raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, env->tr.selector & 0xfffc, retaddr);
     }
     if (shift == 0) {
-        *esp_ptr = cpu_lduw_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 2);
+        *esp_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 2, retaddr);
     } else {
-        *esp_ptr = cpu_ldl_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 4);
+        *esp_ptr = cpu_ldl_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 4, retaddr);
     }
 }
 
-static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
+static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl,
+                         uintptr_t retaddr)
 {
     uint32_t e1, e2;
     int rpl, dpl;
 
     if ((selector & 0xfffc) != 0) {
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_CS) {
             if (!(e2 & DESC_CS_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else if (seg_reg == R_SS) {
             /* SS must be writable data */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != cpl || dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else {
             /* not readable code */
             if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             /* if data or non conforming code, checks the rights */
             if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
                 }
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, retaddr);
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector,
                                get_seg_base(e1, e2),
@@ -214,7 +225,7 @@ static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
                                e2);
     } else {
         if (seg_reg == R_SS || seg_reg == R_CS) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
     }
 }
@@ -224,9 +235,9 @@ static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
 #define SWITCH_TSS_CALL 2
 
 /* XXX: restore CPU state in registers (PowerPC case) */
-static void switch_tss(CPUX86State *env, int tss_selector,
-                       uint32_t e1, uint32_t e2, int source,
-                       uint32_t next_eip)
+static void switch_tss_ra(CPUX86State *env, int tss_selector,
+                          uint32_t e1, uint32_t e2, int source,
+                          uint32_t next_eip, uintptr_t retaddr)
 {
     int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
     target_ulong tss_base;
@@ -244,26 +255,26 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* if task gate, we read the TSS segment and we load it */
     if (type == 5) {
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
         }
         tss_selector = e1 >> 16;
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         if (e2 & DESC_S_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((type & 7) != 1) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
     }
 
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
     }
 
     if (type & 8) {
@@ -275,7 +286,7 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     tss_base = get_seg_base(e1, e2);
     if ((tss_selector & 4) != 0 ||
         tss_limit < tss_limit_max) {
-        raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
     }
     old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
     if (old_type & 8) {
@@ -287,30 +298,33 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* read all the registers from the new TSS */
     if (type & 8) {
         /* 32 bit */
-        new_cr3 = cpu_ldl_kernel(env, tss_base + 0x1c);
-        new_eip = cpu_ldl_kernel(env, tss_base + 0x20);
-        new_eflags = cpu_ldl_kernel(env, tss_base + 0x24);
+        new_cr3 = cpu_ldl_kernel_ra(env, tss_base + 0x1c, retaddr);
+        new_eip = cpu_ldl_kernel_ra(env, tss_base + 0x20, retaddr);
+        new_eflags = cpu_ldl_kernel_ra(env, tss_base + 0x24, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_ldl_kernel(env, tss_base + (0x28 + i * 4));
+            new_regs[i] = cpu_ldl_kernel_ra(env, tss_base + (0x28 + i * 4),
+                                            retaddr);
         }
         for (i = 0; i < 6; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x48 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x48 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x60);
-        new_trap = cpu_ldl_kernel(env, tss_base + 0x64);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x60, retaddr);
+        new_trap = cpu_ldl_kernel_ra(env, tss_base + 0x64, retaddr);
     } else {
         /* 16 bit */
         new_cr3 = 0;
-        new_eip = cpu_lduw_kernel(env, tss_base + 0x0e);
-        new_eflags = cpu_lduw_kernel(env, tss_base + 0x10);
+        new_eip = cpu_lduw_kernel_ra(env, tss_base + 0x0e, retaddr);
+        new_eflags = cpu_lduw_kernel_ra(env, tss_base + 0x10, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_lduw_kernel(env, tss_base + (0x12 + i * 2)) |
-                0xffff0000;
+            new_regs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x12 + i * 2),
+                                             retaddr) | 0xffff0000;
         }
         for (i = 0; i < 4; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x22 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x22 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x2a);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x2a, retaddr);
         new_segs[R_FS] = 0;
         new_segs[R_GS] = 0;
         new_trap = 0;
@@ -325,10 +339,10 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* XXX: it can still fail in some cases, so a bigger hack is
        necessary to valid the TLB after having done the accesses */
 
-    v1 = cpu_ldub_kernel(env, env->tr.base);
-    v2 = cpu_ldub_kernel(env, env->tr.base + old_tss_limit_max);
-    cpu_stb_kernel(env, env->tr.base, v1);
-    cpu_stb_kernel(env, env->tr.base + old_tss_limit_max, v2);
+    v1 = cpu_ldub_kernel_ra(env, env->tr.base, retaddr);
+    v2 = cpu_ldub_kernel_ra(env, env->tr.base + old_tss_limit_max, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base, v1, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base + old_tss_limit_max, v2, retaddr);
 
     /* clear busy bit (it is restartable) */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {
@@ -336,9 +350,9 @@ static void switch_tss(CPUX86State *env, int tss_selector,
         uint32_t e2;
 
         ptr = env->gdt.base + (env->tr.selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 &= ~DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
     old_eflags = cpu_compute_eflags(env);
     if (source == SWITCH_TSS_IRET) {
@@ -348,35 +362,35 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* save the current state in the old TSS */
     if (type & 8) {
         /* 32 bit */
-        cpu_stl_kernel(env, env->tr.base + 0x20, next_eip);
-        cpu_stl_kernel(env, env->tr.base + 0x24, old_eflags);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI]);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x20, next_eip, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x24, old_eflags, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI], retaddr);
         for (i = 0; i < 6; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x48 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x48 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     } else {
         /* 16 bit */
-        cpu_stw_kernel(env, env->tr.base + 0x0e, next_eip);
-        cpu_stw_kernel(env, env->tr.base + 0x10, old_eflags);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI]);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x0e, next_eip, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x10, old_eflags, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI], retaddr);
         for (i = 0; i < 4; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x22 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x22 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     }
 
@@ -384,7 +398,7 @@ static void switch_tss(CPUX86State *env, int tss_selector,
        context */
 
     if (source == SWITCH_TSS_CALL) {
-        cpu_stw_kernel(env, tss_base, env->tr.selector);
+        cpu_stw_kernel_ra(env, tss_base, env->tr.selector, retaddr);
         new_eflags |= NT_MASK;
     }
 
@@ -394,9 +408,9 @@ static void switch_tss(CPUX86State *env, int tss_selector,
         uint32_t e2;
 
         ptr = env->gdt.base + (tss_selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
 
     /* set the new CPU state */
@@ -448,23 +462,23 @@ static void switch_tss(CPUX86State *env, int tss_selector,
 
     /* load the LDT */
     if (new_ldt & 4) {
-        raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
     }
 
     if ((new_ldt & 0xfffc) != 0) {
         dt = &env->gdt;
         index = new_ldt & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, retaddr);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         load_seg_cache_raw_dt(&env->ldt, e1, e2);
     }
@@ -472,34 +486,35 @@ static void switch_tss(CPUX86State *env, int tss_selector,
     /* load the segments */
     if (!(new_eflags & VM_MASK)) {
         int cpl = new_segs[R_CS] & 3;
-        tss_load_seg(env, R_CS, new_segs[R_CS], cpl);
-        tss_load_seg(env, R_SS, new_segs[R_SS], cpl);
-        tss_load_seg(env, R_ES, new_segs[R_ES], cpl);
-        tss_load_seg(env, R_DS, new_segs[R_DS], cpl);
-        tss_load_seg(env, R_FS, new_segs[R_FS], cpl);
-        tss_load_seg(env, R_GS, new_segs[R_GS], cpl);
+        tss_load_seg(env, R_CS, new_segs[R_CS], cpl, retaddr);
+        tss_load_seg(env, R_SS, new_segs[R_SS], cpl, retaddr);
+        tss_load_seg(env, R_ES, new_segs[R_ES], cpl, retaddr);
+        tss_load_seg(env, R_DS, new_segs[R_DS], cpl, retaddr);
+        tss_load_seg(env, R_FS, new_segs[R_FS], cpl, retaddr);
+        tss_load_seg(env, R_GS, new_segs[R_GS], cpl, retaddr);
     }
 
     /* check that env->eip is in the CS segment limits */
     if (new_eip > env->segs[R_CS].limit) {
         /* XXX: different exception if CALL? */
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 
 #ifndef CONFIG_USER_ONLY
     /* reset local breakpoints */
     if (env->dr[7] & DR7_LOCAL_BP_MASK) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            if (hw_local_breakpoint_enabled(env->dr[7], i) &&
-                !hw_global_breakpoint_enabled(env->dr[7], i)) {
-                hw_breakpoint_remove(env, i);
-            }
-        }
-        env->dr[7] &= ~DR7_LOCAL_BP_MASK;
+        cpu_x86_update_dr7(env, env->dr[7] & ~DR7_LOCAL_BP_MASK);
     }
 #endif
 }
 
+static void switch_tss(CPUX86State *env, int tss_selector,
+                       uint32_t e1, uint32_t e2, int source,
+                        uint32_t next_eip)
+{
+    switch_tss_ra(env, tss_selector, e1, e2, source, next_eip, 0);
+}
+
 static inline unsigned int get_sp_mask(unsigned int e2)
 {
     if (e2 & DESC_B_MASK) {
@@ -549,30 +564,35 @@ static int exception_has_error_code(int intno)
 #define SEG_ADDL(ssp, sp, sp_mask) ((uint32_t)((ssp) + (sp & (sp_mask))))
 
 /* XXX: add a is_user flag to have proper security support */
-#define PUSHW(ssp, sp, sp_mask, val)                             \
+#define PUSHW_RA(ssp, sp, sp_mask, val, ra)                      \
     {                                                            \
         sp -= 2;                                                 \
-        cpu_stw_kernel(env, (ssp) + (sp & (sp_mask)), (val));    \
+        cpu_stw_kernel_ra(env, (ssp) + (sp & (sp_mask)), (val), ra); \
     }
 
-#define PUSHL(ssp, sp, sp_mask, val)                                    \
+#define PUSHL_RA(ssp, sp, sp_mask, val, ra)                             \
     {                                                                   \
         sp -= 4;                                                        \
-        cpu_stl_kernel(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val)); \
+        cpu_stl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val), ra); \
     }
 
-#define POPW(ssp, sp, sp_mask, val)                              \
+#define POPW_RA(ssp, sp, sp_mask, val, ra)                       \
     {                                                            \
-        val = cpu_lduw_kernel(env, (ssp) + (sp & (sp_mask)));    \
+        val = cpu_lduw_kernel_ra(env, (ssp) + (sp & (sp_mask)), ra); \
         sp += 2;                                                 \
     }
 
-#define POPL(ssp, sp, sp_mask, val)                                     \
+#define POPL_RA(ssp, sp, sp_mask, val, ra)                              \
     {                                                                   \
-        val = (uint32_t)cpu_ldl_kernel(env, SEG_ADDL(ssp, sp, sp_mask)); \
+        val = (uint32_t)cpu_ldl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), ra); \
         sp += 4;                                                        \
     }
 
+#define PUSHW(ssp, sp, sp_mask, val) PUSHW_RA(ssp, sp, sp_mask, val, 0)
+#define PUSHL(ssp, sp, sp_mask, val) PUSHL_RA(ssp, sp, sp_mask, val, 0)
+#define POPW(ssp, sp, sp_mask, val) POPW_RA(ssp, sp, sp_mask, val, 0)
+#define POPL(ssp, sp, sp_mask, val) POPL_RA(ssp, sp, sp_mask, val, 0)
+
 /* protected mode interrupt */
 static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
                                    int error_code, unsigned int next_eip,
@@ -673,7 +693,7 @@ static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
     }
     if (!(e2 & DESC_C_MASK) && dpl < cpl) {
         /* to inner privilege */
-        get_ss_esp_from_tss(env, &ss, &esp, dpl);
+        get_ss_esp_from_tss(env, &ss, &esp, dpl, 0);
         if ((ss & 0xfffc) == 0) {
             raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
         }
@@ -791,18 +811,21 @@ static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
 
 #ifdef TARGET_X86_64
 
-#define PUSHQ(sp, val)                          \
+#define PUSHQ_RA(sp, val, ra)                   \
     {                                           \
         sp -= 8;                                \
-        cpu_stq_kernel(env, sp, (val));         \
+        cpu_stq_kernel_ra(env, sp, (val), ra);  \
     }
 
-#define POPQ(sp, val)                           \
+#define POPQ_RA(sp, val, ra)                    \
     {                                           \
-        val = cpu_ldq_kernel(env, sp);          \
+        val = cpu_ldq_kernel_ra(env, sp, ra);   \
         sp += 8;                                \
     }
 
+#define PUSHQ(sp, val) PUSHQ_RA(sp, val, 0)
+#define POPQ(sp, val) POPQ_RA(sp, val, 0)
+
 static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
@@ -961,7 +984,7 @@ void helper_syscall(CPUX86State *env, int next_eip_addend)
     int selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     selector = (env->star >> 32) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1016,11 +1039,11 @@ void helper_sysret(CPUX86State *env, int dflag)
     int cpl, selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (!(env->cr[0] & CR0_PE_MASK) || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     selector = (env->star >> 48) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1294,6 +1317,9 @@ bool x86_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
     if (interrupt_request & CPU_INTERRUPT_POLL) {
         cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
         apic_poll_irq(cpu->apic_state);
+        /* Don't process multiple interrupt requests in a single call.
+           This is required to make icount-driven execution deterministic. */
+        return true;
     }
 #endif
     if (interrupt_request & CPU_INTERRUPT_SIPI) {
@@ -1353,74 +1379,6 @@ bool x86_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
     return ret;
 }
 
-void helper_enter_level(CPUX86State *env, int level, int data32,
-                        target_ulong t1)
-{
-    target_ulong ssp;
-    uint32_t esp_mask, esp, ebp;
-
-    esp_mask = get_sp_mask(env->segs[R_SS].flags);
-    ssp = env->segs[R_SS].base;
-    ebp = env->regs[R_EBP];
-    esp = env->regs[R_ESP];
-    if (data32) {
-        /* 32 bit */
-        esp -= 4;
-        while (--level) {
-            esp -= 4;
-            ebp -= 4;
-            cpu_stl_data(env, ssp + (esp & esp_mask),
-                         cpu_ldl_data(env, ssp + (ebp & esp_mask)));
-        }
-        esp -= 4;
-        cpu_stl_data(env, ssp + (esp & esp_mask), t1);
-    } else {
-        /* 16 bit */
-        esp -= 2;
-        while (--level) {
-            esp -= 2;
-            ebp -= 2;
-            cpu_stw_data(env, ssp + (esp & esp_mask),
-                         cpu_lduw_data(env, ssp + (ebp & esp_mask)));
-        }
-        esp -= 2;
-        cpu_stw_data(env, ssp + (esp & esp_mask), t1);
-    }
-}
-
-#ifdef TARGET_X86_64
-void helper_enter64_level(CPUX86State *env, int level, int data64,
-                          target_ulong t1)
-{
-    target_ulong esp, ebp;
-
-    ebp = env->regs[R_EBP];
-    esp = env->regs[R_ESP];
-
-    if (data64) {
-        /* 64 bit */
-        esp -= 8;
-        while (--level) {
-            esp -= 8;
-            ebp -= 8;
-            cpu_stq_data(env, esp, cpu_ldq_data(env, ebp));
-        }
-        esp -= 8;
-        cpu_stq_data(env, esp, t1);
-    } else {
-        /* 16 bit */
-        esp -= 2;
-        while (--level) {
-            esp -= 2;
-            ebp -= 2;
-            cpu_stw_data(env, esp, cpu_lduw_data(env, ebp));
-        }
-        esp -= 2;
-        cpu_stw_data(env, esp, t1);
-    }
-}
-#endif
-
 void helper_lldt(CPUX86State *env, int selector)
 {
     SegmentCache *dt;
@@ -1435,7 +1393,7 @@ void helper_lldt(CPUX86State *env, int selector)
         env->ldt.limit = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1448,22 +1406,22 @@ void helper_lldt(CPUX86State *env, int selector)
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
             load_seg_cache_raw_dt(&env->ldt, e1, e2);
             env->ldt.base |= (target_ulong)e3 << 32;
         } else
@@ -1490,7 +1448,7 @@ void helper_ltr(CPUX86State *env, int selector)
         env->tr.flags = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1503,27 +1461,27 @@ void helper_ltr(CPUX86State *env, int selector)
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((e2 & DESC_S_MASK) ||
             (type != 1 && type != 9)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3, e4;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
-            e4 = cpu_ldl_kernel(env, ptr + 12);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
+            e4 = cpu_ldl_kernel_ra(env, ptr + 12, GETPC());
             if ((e4 >> DESC_TYPE_SHIFT) & 0xf) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             load_seg_cache_raw_dt(&env->tr, e1, e2);
             env->tr.base |= (target_ulong)e3 << 32;
@@ -1533,7 +1491,7 @@ void helper_ltr(CPUX86State *env, int selector)
             load_seg_cache_raw_dt(&env->tr, e1, e2);
         }
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
     }
     env->tr.selector = selector;
 }
@@ -1556,7 +1514,7 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
             && (!(env->hflags & HF_CS64_MASK) || cpl == 3)
 #endif
             ) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
     } else {
@@ -1568,51 +1526,51 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
         }
         index = selector & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
 
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_SS) {
             /* must be writable segment */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             if (rpl != cpl || dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
         } else {
             /* must be readable segment */
             if ((e2 & (DESC_CS_MASK | DESC_R_MASK)) == DESC_CS_MASK) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
 
             if (!(e2 & DESC_CS_MASK) || !(e2 & DESC_C_MASK)) {
                 /* if not conforming code, test rights */
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
                 }
             }
         }
 
         if (!(e2 & DESC_P_MASK)) {
             if (seg_reg == R_SS) {
-                raise_exception_err(env, EXCP0C_STACK, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0C_STACK, selector & 0xfffc, GETPC());
             } else {
-                raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
             }
         }
 
         /* set the access bit if not already set */
         if (!(e2 & DESC_A_MASK)) {
             e2 |= DESC_A_MASK;
-            cpu_stl_kernel(env, ptr + 4, e2);
+            cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
         }
 
         cpu_x86_load_seg_cache(env, seg_reg, selector,
@@ -1628,46 +1586,45 @@ void helper_load_seg(CPUX86State *env, int seg_reg, int selector)
 
 /* protected mode jump */
 void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                           int next_eip_addend)
+                           target_ulong next_eip)
 {
     int gate_cs, type;
     uint32_t e1, e2, cpl, dpl, rpl, limit;
-    target_ulong next_eip;
 
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
         limit = get_seg_limit(e1, e2);
         if (new_eip > limit &&
             !(env->hflags & HF_LMA_MASK) && !(e2 & DESC_L_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
                        get_seg_base(e1, e2), limit, e2);
@@ -1683,50 +1640,49 @@ void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         case 9: /* 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            next_eip = env->eip + next_eip_addend;
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip, GETPC());
             break;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             if ((dpl < cpl) || (dpl < rpl)) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
             }
             gate_cs = e1 >> 16;
             new_eip = (e1 & 0xffff);
             if (type == 12) {
                 new_eip |= (e2 & 0xffff0000);
             }
-            if (load_segment(env, &e1, &e2, gate_cs) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+            if (load_segment_ra(env, &e1, &e2, gate_cs, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             dpl = (e2 >> DESC_DPL_SHIFT) & 3;
             /* must be code segment */
             if (((e2 & (DESC_S_MASK | DESC_CS_MASK)) !=
                  (DESC_S_MASK | DESC_CS_MASK))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (((e2 & DESC_C_MASK) && (dpl > cpl)) ||
                 (!(e2 & DESC_C_MASK) && (dpl != cpl))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
             }
             cpu_x86_load_seg_cache(env, R_CS, (gate_cs & 0xfffc) | cpl,
                                    get_seg_base(e1, e2), limit, e2);
             env->eip = new_eip;
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
     }
@@ -1745,11 +1701,11 @@ void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1,
     esp_mask = get_sp_mask(env->segs[R_SS].flags);
     ssp = env->segs[R_SS].base;
     if (shift) {
-        PUSHL(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHL(ssp, esp, esp_mask, next_eip);
+        PUSHL_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHL_RA(ssp, esp, esp_mask, next_eip, GETPC());
     } else {
-        PUSHW(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHW(ssp, esp, esp_mask, next_eip);
+        PUSHW_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHW_RA(ssp, esp, esp_mask, next_eip, GETPC());
     }
 
     SET_ESP(esp, esp_mask);
@@ -1760,47 +1716,46 @@ void helper_lcall_real(CPUX86State *env, int new_cs, target_ulong new_eip1,
 
 /* protected mode call */
 void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                            int shift, int next_eip_addend)
+                            int shift, target_ulong next_eip)
 {
     int new_stack, i;
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
     uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
-    target_ulong ssp, old_ssp, next_eip;
+    target_ulong ssp, old_ssp;
 
-    next_eip = env->eip + next_eip_addend;
     LOG_PCALL("lcall %04x:%08x s=%d\n", new_cs, (uint32_t)new_eip, shift);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     LOG_PCALL("desc=%08x:%08x\n", e1, e2);
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
 
 #ifdef TARGET_X86_64
@@ -1810,8 +1765,8 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
 
             /* 64 bit case */
             rsp = env->regs[R_ESP];
-            PUSHQ(rsp, env->segs[R_CS].selector);
-            PUSHQ(rsp, next_eip);
+            PUSHQ_RA(rsp, env->segs[R_CS].selector, GETPC());
+            PUSHQ_RA(rsp, next_eip, GETPC());
             /* from this point, not restartable */
             env->regs[R_ESP] = rsp;
             cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
@@ -1825,16 +1780,16 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
             sp_mask = get_sp_mask(env->segs[R_SS].flags);
             ssp = env->segs[R_SS].base;
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHL(ssp, sp, sp_mask, next_eip);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHW(ssp, sp, sp_mask, next_eip);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
             }
 
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             /* from this point, not restartable */
             SET_ESP(sp, sp_mask);
@@ -1852,73 +1807,73 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         case 9: /* available 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip, GETPC());
             return;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
         shift = type >> 3;
 
         if (dpl < cpl || dpl < rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         /* check valid bit */
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG,  new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG,  new_cs & 0xfffc, GETPC());
         }
         selector = e1 >> 16;
         offset = (e2 & 0xffff0000) | (e1 & 0x0000ffff);
         param_count = e2 & 0x1f;
         if ((selector & 0xfffc) == 0) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK))) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (dpl > cpl) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 
         if (!(e2 & DESC_C_MASK) && dpl < cpl) {
             /* to inner privilege */
-            get_ss_esp_from_tss(env, &ss, &sp, dpl);
+            get_ss_esp_from_tss(env, &ss, &sp, dpl, GETPC());
             LOG_PCALL("new ss:esp=%04x:%08x param_count=%d env->regs[R_ESP]="
                       TARGET_FMT_lx "\n", ss, sp, param_count,
                       env->regs[R_ESP]);
             if ((ss & 0xfffc) == 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if ((ss & 3) != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
-            if (load_segment(env, &ss_e1, &ss_e2, ss) != 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, ss, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (ss_dpl != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
 
             /* push_size = ((param_count * 2) + 8) << shift; */
@@ -1929,22 +1884,22 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
             sp_mask = get_sp_mask(ss_e2);
             ssp = get_seg_base(ss_e1, ss_e2);
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHL(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_ldl_kernel(env, old_ssp +
-                                         ((env->regs[R_ESP] + i * 4) &
-                                          old_sp_mask));
-                    PUSHL(ssp, sp, sp_mask, val);
+                    val = cpu_ldl_kernel_ra(env, old_ssp +
+                                            ((env->regs[R_ESP] + i * 4) &
+                                             old_sp_mask), GETPC());
+                    PUSHL_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHW(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_lduw_kernel(env, old_ssp +
-                                          ((env->regs[R_ESP] + i * 2) &
-                                           old_sp_mask));
-                    PUSHW(ssp, sp, sp_mask, val);
+                    val = cpu_lduw_kernel_ra(env, old_ssp +
+                                             ((env->regs[R_ESP] + i * 2) &
+                                              old_sp_mask), GETPC());
+                    PUSHW_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             }
             new_stack = 1;
@@ -1958,11 +1913,11 @@ void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
         }
 
         if (shift) {
-            PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHL(ssp, sp, sp_mask, next_eip);
+            PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
         } else {
-            PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHW(ssp, sp, sp_mask, next_eip);
+            PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
         }
 
         /* from this point, not restartable */
@@ -1997,15 +1952,15 @@ void helper_iret_real(CPUX86State *env, int shift)
     ssp = env->segs[R_SS].base;
     if (shift == 1) {
         /* 32 bits */
-        POPL(ssp, sp, sp_mask, new_eip);
-        POPL(ssp, sp, sp_mask, new_cs);
+        POPL_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPL_RA(ssp, sp, sp_mask, new_cs, GETPC());
         new_cs &= 0xffff;
-        POPL(ssp, sp, sp_mask, new_eflags);
+        POPL_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     } else {
         /* 16 bits */
-        POPW(ssp, sp, sp_mask, new_eip);
-        POPW(ssp, sp, sp_mask, new_cs);
-        POPW(ssp, sp, sp_mask, new_eflags);
+        POPW_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_cs, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     }
     env->regs[R_ESP] = (env->regs[R_ESP] & ~sp_mask) | (sp & sp_mask);
     env->segs[R_CS].selector = new_cs;
@@ -2050,7 +2005,8 @@ static inline void validate_seg(CPUX86State *env, int seg_reg, int cpl)
 
 /* protected mode iret */
 static inline void helper_ret_protected(CPUX86State *env, int shift,
-                                        int is_iret, int addend)
+                                        int is_iret, int addend,
+                                        uintptr_t retaddr)
 {
     uint32_t new_cs, new_eflags, new_ss;
     uint32_t new_es, new_ds, new_fs, new_gs;
@@ -2071,32 +2027,32 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
     new_eflags = 0; /* avoid warning */
 #ifdef TARGET_X86_64
     if (shift == 2) {
-        POPQ(sp, new_eip);
-        POPQ(sp, new_cs);
+        POPQ_RA(sp, new_eip, retaddr);
+        POPQ_RA(sp, new_cs, retaddr);
         new_cs &= 0xffff;
         if (is_iret) {
-            POPQ(sp, new_eflags);
+            POPQ_RA(sp, new_eflags, retaddr);
         }
     } else
 #endif
     {
         if (shift == 1) {
             /* 32 bits */
-            POPL(ssp, sp, sp_mask, new_eip);
-            POPL(ssp, sp, sp_mask, new_cs);
+            POPL_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPL_RA(ssp, sp, sp_mask, new_cs, retaddr);
             new_cs &= 0xffff;
             if (is_iret) {
-                POPL(ssp, sp, sp_mask, new_eflags);
+                POPL_RA(ssp, sp, sp_mask, new_eflags, retaddr);
                 if (new_eflags & VM_MASK) {
                     goto return_to_vm86;
                 }
             }
         } else {
             /* 16 bits */
-            POPW(ssp, sp, sp_mask, new_eip);
-            POPW(ssp, sp, sp_mask, new_cs);
+            POPW_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPW_RA(ssp, sp, sp_mask, new_cs, retaddr);
             if (is_iret) {
-                POPW(ssp, sp, sp_mask, new_eflags);
+                POPW_RA(ssp, sp, sp_mask, new_eflags, retaddr);
             }
         }
     }
@@ -2104,32 +2060,32 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
               new_cs, new_eip, shift, addend);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, retaddr) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     if (!(e2 & DESC_S_MASK) ||
         !(e2 & DESC_CS_MASK)) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     cpl = env->hflags & HF_CPL_MASK;
     rpl = new_cs & 3;
     if (rpl < cpl) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     dpl = (e2 >> DESC_DPL_SHIFT) & 3;
     if (e2 & DESC_C_MASK) {
         if (dpl > rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     } else {
         if (dpl != rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     }
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, retaddr);
     }
 
     sp += addend;
@@ -2144,21 +2100,21 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
         /* return to different privilege level */
 #ifdef TARGET_X86_64
         if (shift == 2) {
-            POPQ(sp, new_esp);
-            POPQ(sp, new_ss);
+            POPQ_RA(sp, new_esp, retaddr);
+            POPQ_RA(sp, new_ss, retaddr);
             new_ss &= 0xffff;
         } else
 #endif
         {
             if (shift == 1) {
                 /* 32 bits */
-                POPL(ssp, sp, sp_mask, new_esp);
-                POPL(ssp, sp, sp_mask, new_ss);
+                POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
                 new_ss &= 0xffff;
             } else {
                 /* 16 bits */
-                POPW(ssp, sp, sp_mask, new_esp);
-                POPW(ssp, sp, sp_mask, new_ss);
+                POPW_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPW_RA(ssp, sp, sp_mask, new_ss, retaddr);
             }
         }
         LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
@@ -2177,26 +2133,26 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
             } else
 #endif
             {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
             }
         } else {
             if ((new_ss & 3) != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
-            if (load_segment(env, &ss_e1, &ss_e2, new_ss) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, new_ss, retaddr) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_ss & 0xfffc, retaddr);
             }
             cpu_x86_load_seg_cache(env, R_SS, new_ss,
                                    get_seg_base(ss_e1, ss_e2),
@@ -2246,12 +2202,12 @@ static inline void helper_ret_protected(CPUX86State *env, int shift,
     return;
 
  return_to_vm86:
-    POPL(ssp, sp, sp_mask, new_esp);
-    POPL(ssp, sp, sp_mask, new_ss);
-    POPL(ssp, sp, sp_mask, new_es);
-    POPL(ssp, sp, sp_mask, new_ds);
-    POPL(ssp, sp, sp_mask, new_fs);
-    POPL(ssp, sp, sp_mask, new_gs);
+    POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_es, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ds, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_fs, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_gs, retaddr);
 
     /* modify processor state */
     cpu_load_eflags(env, new_eflags, TF_MASK | AC_MASK | ID_MASK |
@@ -2277,37 +2233,37 @@ void helper_iret_protected(CPUX86State *env, int shift, int next_eip)
     if (env->eflags & NT_MASK) {
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 #endif
-        tss_selector = cpu_lduw_kernel(env, env->tr.base + 0);
+        tss_selector = cpu_lduw_kernel_ra(env, env->tr.base + 0, GETPC());
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0x17;
         /* NOTE: we check both segment and busy TSS */
         if (type != 3) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        switch_tss(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip);
+        switch_tss_ra(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip, GETPC());
     } else {
-        helper_ret_protected(env, shift, 1, 0);
+        helper_ret_protected(env, shift, 1, 0, GETPC());
     }
     env->hflags2 &= ~HF2_NMI_MASK;
 }
 
 void helper_lret_protected(CPUX86State *env, int shift, int addend)
 {
-    helper_ret_protected(env, shift, 0, addend);
+    helper_ret_protected(env, shift, 0, addend, GETPC());
 }
 
 void helper_sysenter(CPUX86State *env)
 {
     if (env->sysenter_cs == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     env->eflags &= ~(VM_MASK | IF_MASK | RF_MASK);
 
@@ -2343,7 +2299,7 @@ void helper_sysexit(CPUX86State *env, int dflag)
 
     cpl = env->hflags & HF_CPL_MASK;
     if (env->sysenter_cs == 0 || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
 #ifdef TARGET_X86_64
     if (dflag == 2) {
@@ -2387,7 +2343,7 @@ target_ulong helper_lsl(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2434,7 +2390,7 @@ target_ulong helper_lar(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2483,7 +2439,7 @@ void helper_verr(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2521,7 +2477,7 @@ void helper_verw(CPUX86State *env, target_ulong selector1)
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2562,7 +2518,8 @@ void cpu_x86_load_seg(CPUX86State *env, int seg_reg, int selector)
 #endif
 
 /* check if Port I/O is allowed in TSS */
-static inline void check_io(CPUX86State *env, int addr, int size)
+static inline void check_io(CPUX86State *env, int addr, int size,
+                            uintptr_t retaddr)
 {
     int io_offset, val, mask;
 
@@ -2572,33 +2529,33 @@ static inline void check_io(CPUX86State *env, int addr, int size)
         env->tr.limit < 103) {
         goto fail;
     }
-    io_offset = cpu_lduw_kernel(env, env->tr.base + 0x66);
+    io_offset = cpu_lduw_kernel_ra(env, env->tr.base + 0x66, retaddr);
     io_offset += (addr >> 3);
     /* Note: the check needs two bytes */
     if ((io_offset + 1) > env->tr.limit) {
         goto fail;
     }
-    val = cpu_lduw_kernel(env, env->tr.base + io_offset);
+    val = cpu_lduw_kernel_ra(env, env->tr.base + io_offset, retaddr);
     val >>= (addr & 7);
     mask = (1 << size) - 1;
     /* all bits must be zero to allow the I/O */
     if ((val & mask) != 0) {
     fail:
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 }
 
 void helper_check_iob(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 1);
+    check_io(env, t0, 1, GETPC());
 }
 
 void helper_check_iow(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 2);
+    check_io(env, t0, 2, GETPC());
 }
 
 void helper_check_iol(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 4);
+    check_io(env, t0, 4, GETPC());
 }
diff --git a/qemu/target-i386/smm_helper.c b/qemu/target-i386/smm_helper.c
index 02e24b923..4dd6a2c54 100644
--- a/qemu/target-i386/smm_helper.c
+++ b/qemu/target-i386/smm_helper.c
@@ -17,8 +17,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/log.h"
 
 /* SMM support */
 
@@ -97,6 +99,10 @@ void do_smm_enter(X86CPU *cpu)
     x86_stl_phys(cs, sm_state + 0x7e94, env->tr.limit);
     x86_stw_phys(cs, sm_state + 0x7e92, (env->tr.flags >> 8) & 0xf0ff);
 
+    /* ??? Vol 1, 16.5.6 Intel MPX and SMM says that IA32_BNDCFGS
+       is saved at offset 7ED0.  Vol 3, 34.4.1.1, Table 32-2, has
+       7EA0-7ED7 as "reserved".  What's this, and what's really
+       supposed to happen?  */
     x86_stq_phys(cs, sm_state + 0x7ed0, env->efer);
 
     x86_stq_phys(cs, sm_state + 0x7ff8, env->regs[R_EAX]);
@@ -266,7 +272,7 @@ void helper_rsm(CPUX86State *env)
 
     val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = x86_ldl_phys(cs, sm_state + 0x7f00) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7f00);
     }
 #else
     cpu_x86_update_cr0(env, x86_ldl_phys(cs, sm_state + 0x7ffc));
@@ -319,7 +325,7 @@ void helper_rsm(CPUX86State *env)
 
     val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = x86_ldl_phys(cs, sm_state + 0x7ef8) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7ef8);
     }
 #endif
     if ((env->hflags2 & HF2_SMM_INSIDE_NMI_MASK) == 0) {
diff --git a/qemu/target-i386/svm_helper.c b/qemu/target-i386/svm_helper.c
index f1fabf54e..ab472f6ee 100644
--- a/qemu/target-i386/svm_helper.c
+++ b/qemu/target-i386/svm_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "exec/helper-proto.h"
diff --git a/qemu/target-i386/translate.c b/qemu/target-i386/translate.c
index 82e2245bf..1a1214dcb 100644
--- a/qemu/target-i386/translate.c
+++ b/qemu/target-i386/translate.c
@@ -16,12 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "qemu/host-utils.h"
 #include "cpu.h"
@@ -33,6 +28,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define PREFIX_REPZ   0x01
@@ -60,24 +56,37 @@
 # define clztl  clz32
 #endif
 
+/* For a switch indexed by MODRM, match all memory operands for a given OP.  */
+#define CASE_MODRM_MEM_OP(OP) \
+    case (0 << 6) | (OP << 3) | 0 ... (0 << 6) | (OP << 3) | 7: \
+    case (1 << 6) | (OP << 3) | 0 ... (1 << 6) | (OP << 3) | 7: \
+    case (2 << 6) | (OP << 3) | 0 ... (2 << 6) | (OP << 3) | 7
+
+#define CASE_MODRM_OP(OP) \
+    case (0 << 6) | (OP << 3) | 0 ... (0 << 6) | (OP << 3) | 7: \
+    case (1 << 6) | (OP << 3) | 0 ... (1 << 6) | (OP << 3) | 7: \
+    case (2 << 6) | (OP << 3) | 0 ... (2 << 6) | (OP << 3) | 7: \
+    case (3 << 6) | (OP << 3) | 0 ... (3 << 6) | (OP << 3) | 7
+
 //#define MACRO_TEST   1
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_A0;
 static TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2, cpu_cc_srcT;
 static TCGv_i32 cpu_cc_op;
 static TCGv cpu_regs[CPU_NB_REGS];
+static TCGv cpu_seg_base[6];
+static TCGv_i64 cpu_bndl[4];
+static TCGv_i64 cpu_bndu[4];
 /* local temps */
-static TCGv cpu_T[2];
+static TCGv cpu_T0, cpu_T1;
 /* local register indexes (only used inside old micro ops) */
 static TCGv cpu_tmp0, cpu_tmp4;
 static TCGv_ptr cpu_ptr0, cpu_ptr1;
 static TCGv_i32 cpu_tmp2_i32, cpu_tmp3_i32;
 static TCGv_i64 cpu_tmp1_i64;
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #ifdef TARGET_X86_64
@@ -90,6 +99,7 @@ typedef struct DisasContext {
     int prefix;
     TCGMemOp aflag;
     TCGMemOp dflag;
+    target_ulong pc_start;
     target_ulong pc; /* pc = eip + cs_base */
     int is_jmp; /* 1 = means jump (stop translation), 2 means CPU
                    static state change (stop translation) */
@@ -126,6 +136,7 @@ typedef struct DisasContext {
     int cpuid_ext2_features;
     int cpuid_ext3_features;
     int cpuid_7_0_ebx_features;
+    int cpuid_xsave_features;
 } DisasContext;
 
 static void gen_eob(DisasContext *s);
@@ -310,6 +321,12 @@ static inline TCGMemOp mo_pushpop(DisasContext *s, TCGMemOp ot)
     }
 }
 
+/* Select the size of the stack pointer.  */
+static inline TCGMemOp mo_stacksize(DisasContext *s)
+{
+    return CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+}
+
 /* Select only size 64 else 32.  Used for SSE operand sizes.  */
 static inline TCGMemOp mo_64_32(TCGMemOp ot)
 {
@@ -372,34 +389,12 @@ static inline void gen_op_mov_v_reg(TCGMemOp ot, TCGv t0, int reg)
     }
 }
 
-static inline void gen_op_movl_A0_reg(int reg)
-{
-    tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
-}
-
-static inline void gen_op_addl_A0_im(int32_t val)
-{
-    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
-#ifdef TARGET_X86_64
-    tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-#endif
-}
-
-#ifdef TARGET_X86_64
-static inline void gen_op_addq_A0_im(int64_t val)
-{
-    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
-}
-#endif
-    
 static void gen_add_A0_im(DisasContext *s, int val)
 {
-#ifdef TARGET_X86_64
-    if (CODE64(s))
-        gen_op_addq_A0_im(val);
-    else
-#endif
-        gen_op_addl_A0_im(val);
+    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
+    if (!CODE64(s)) {
+        tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+    }
 }
 
 static inline void gen_op_jmp_v(TCGv dest)
@@ -415,68 +410,10 @@ static inline void gen_op_add_reg_im(TCGMemOp size, int reg, int32_t val)
 
 static inline void gen_op_add_reg_T0(TCGMemOp size, int reg)
 {
-    tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T[0]);
+    tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T0);
     gen_op_mov_reg_v(size, reg, cpu_tmp0);
 }
 
-static inline void gen_op_addl_A0_reg_sN(int shift, int reg)
-{
-    tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
-    if (shift != 0)
-        tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-    /* For x86_64, this sets the higher half of register to zero.
-       For i386, this is equivalent to a nop. */
-    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
-}
-
-static inline void gen_op_movl_A0_seg(int reg)
-{
-    tcg_gen_ld32u_tl(cpu_A0, cpu_env, offsetof(CPUX86State, segs[reg].base) + REG_L_OFFSET);
-}
-
-static inline void gen_op_addl_A0_seg(DisasContext *s, int reg)
-{
-    tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-    } else {
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-        tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-    }
-#else
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-#endif
-}
-
-#ifdef TARGET_X86_64
-static inline void gen_op_movq_A0_seg(int reg)
-{
-    tcg_gen_ld_tl(cpu_A0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-}
-
-static inline void gen_op_addq_A0_seg(int reg)
-{
-    tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-}
-
-static inline void gen_op_movq_A0_reg(int reg)
-{
-    tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
-}
-
-static inline void gen_op_addq_A0_reg_sN(int shift, int reg)
-{
-    tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
-    if (shift != 0)
-        tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-}
-#endif
-
 static inline void gen_op_ld_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
 {
     tcg_gen_qemu_ld_tl(t0, a0, s->mem_index, idx | MO_LE);
@@ -490,9 +427,9 @@ static inline void gen_op_st_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
 static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d)
 {
     if (d == OR_TMP0) {
-        gen_op_st_v(s, idx, cpu_T[0], cpu_A0);
+        gen_op_st_v(s, idx, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_reg_v(idx, d, cpu_T[0]);
+        gen_op_mov_reg_v(idx, d, cpu_T0);
     }
 }
 
@@ -502,74 +439,77 @@ static inline void gen_jmp_im(target_ulong pc)
     gen_op_jmp_v(cpu_tmp0);
 }
 
-static inline void gen_string_movl_A0_ESI(DisasContext *s)
+/* Compute SEG:REG into A0.  SEG is selected from the override segment
+   (OVR_SEG) and the default segment (DEF_SEG).  OVR_SEG may be -1 to
+   indicate no override.  */
+static void gen_lea_v_seg(DisasContext *s, TCGMemOp aflag, TCGv a0,
+                          int def_seg, int ovr_seg)
 {
-    int override;
-
-    override = s->override;
-    switch (s->aflag) {
+    switch (aflag) {
 #ifdef TARGET_X86_64
     case MO_64:
-        if (override >= 0) {
-            gen_op_movq_A0_seg(override);
-            gen_op_addq_A0_reg_sN(0, R_ESI);
-        } else {
-            gen_op_movq_A0_reg(R_ESI);
+        if (ovr_seg < 0) {
+            tcg_gen_mov_tl(cpu_A0, a0);
+            return;
         }
         break;
 #endif
     case MO_32:
         /* 32 bit address */
-        if (s->addseg && override < 0)
-            override = R_DS;
-        if (override >= 0) {
-            gen_op_movl_A0_seg(override);
-            gen_op_addl_A0_reg_sN(0, R_ESI);
-        } else {
-            gen_op_movl_A0_reg(R_ESI);
+        if (ovr_seg < 0) {
+            if (s->addseg) {
+                ovr_seg = def_seg;
+            } else {
+                tcg_gen_ext32u_tl(cpu_A0, a0);
+                return;
+            }
         }
         break;
     case MO_16:
-        /* 16 address, always override */
-        if (override < 0)
-            override = R_DS;
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_ESI]);
-        gen_op_addl_A0_seg(s, override);
+        /* 16 bit address */
+        tcg_gen_ext16u_tl(cpu_A0, a0);
+        a0 = cpu_A0;
+        if (ovr_seg < 0) {
+            if (s->addseg) {
+                ovr_seg = def_seg;
+            } else {
+                return;
+            }
+        }
         break;
     default:
         tcg_abort();
     }
-}
 
-static inline void gen_string_movl_A0_EDI(DisasContext *s)
-{
-    switch (s->aflag) {
-#ifdef TARGET_X86_64
-    case MO_64:
-        gen_op_movq_A0_reg(R_EDI);
-        break;
-#endif
-    case MO_32:
-        if (s->addseg) {
-            gen_op_movl_A0_seg(R_ES);
-            gen_op_addl_A0_reg_sN(0, R_EDI);
+    if (ovr_seg >= 0) {
+        TCGv seg = cpu_seg_base[ovr_seg];
+
+        if (aflag == MO_64) {
+            tcg_gen_add_tl(cpu_A0, a0, seg);
+        } else if (CODE64(s)) {
+            tcg_gen_ext32u_tl(cpu_A0, a0);
+            tcg_gen_add_tl(cpu_A0, cpu_A0, seg);
         } else {
-            gen_op_movl_A0_reg(R_EDI);
+            tcg_gen_add_tl(cpu_A0, a0, seg);
+            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
         }
-        break;
-    case MO_16:
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_EDI]);
-        gen_op_addl_A0_seg(s, R_ES);
-        break;
-    default:
-        tcg_abort();
     }
 }
 
+static inline void gen_string_movl_A0_ESI(DisasContext *s)
+{
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_ESI], R_DS, s->override);
+}
+
+static inline void gen_string_movl_A0_EDI(DisasContext *s)
+{
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_EDI], R_ES, -1);
+}
+
 static inline void gen_op_movl_T0_Dshift(TCGMemOp ot)
 {
-    tcg_gen_ld32s_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, df));
-    tcg_gen_shli_tl(cpu_T[0], cpu_T[0], ot);
+    tcg_gen_ld32s_tl(cpu_T0, cpu_env, offsetof(CPUX86State, df));
+    tcg_gen_shli_tl(cpu_T0, cpu_T0, ot);
 };
 
 static TCGv gen_ext_tl(TCGv dst, TCGv src, TCGMemOp size, bool sign)
@@ -664,15 +604,10 @@ static void gen_helper_out_func(TCGMemOp ot, TCGv_i32 v, TCGv_i32 n)
 static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
                          uint32_t svm_flags)
 {
-    int state_saved;
     target_ulong next_eip;
 
-    state_saved = 0;
     if (s->pe && (s->cpl > s->iopl || s->vm86)) {
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
-        state_saved = 1;
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         switch (ot) {
         case MO_8:
             gen_helper_check_iob(cpu_env, cpu_tmp2_i32);
@@ -688,13 +623,11 @@ static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
         }
     }
     if(s->flags & HF_SVMI_MASK) {
-        if (!state_saved) {
-            gen_update_cc_op(s);
-            gen_jmp_im(cur_eip);
-        }
+        gen_update_cc_op(s);
+        gen_jmp_im(cur_eip);
         svm_flags |= (1 << (4 + ot));
         next_eip = s->pc - s->cs_base;
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         gen_helper_svm_check_io(cpu_env, cpu_tmp2_i32,
                                 tcg_const_i32(svm_flags),
                                 tcg_const_i32(next_eip - cur_eip));
@@ -704,9 +637,9 @@ static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
 static inline void gen_movs(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     gen_string_movl_A0_EDI(s);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
     gen_op_add_reg_T0(s->aflag, R_EDI);
@@ -714,31 +647,31 @@ static inline void gen_movs(DisasContext *s, TCGMemOp ot)
 
 static void gen_op_update1_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_op_update2_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_op_update3_cc(TCGv reg)
 {
     tcg_gen_mov_tl(cpu_cc_src2, reg);
-    tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static inline void gen_op_testl_T0_T1_cc(void)
 {
-    tcg_gen_and_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
+    tcg_gen_and_tl(cpu_cc_dst, cpu_T0, cpu_T1);
 }
 
 static void gen_op_update_neg_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-    tcg_gen_neg_tl(cpu_cc_src, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+    tcg_gen_neg_tl(cpu_cc_src, cpu_T0);
     tcg_gen_movi_tl(cpu_cc_srcT, 0);
 }
 
@@ -1080,11 +1013,11 @@ static inline void gen_compute_eflags_c(DisasContext *s, TCGv reg)
    value 'b'. In the fast case, T0 is guaranted not to be used. */
 static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1)
 {
-    CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]);
+    CCPrepare cc = gen_prepare_cc(s, b, cpu_T0);
 
     if (cc.mask != -1) {
-        tcg_gen_andi_tl(cpu_T[0], cc.reg, cc.mask);
-        cc.reg = cpu_T[0];
+        tcg_gen_andi_tl(cpu_T0, cc.reg, cc.mask);
+        cc.reg = cpu_T0;
     }
     if (cc.use_reg2) {
         tcg_gen_brcond_tl(cc.cond, cc.reg, cc.reg2, l1);
@@ -1098,12 +1031,12 @@ static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1)
    A translation block must end soon.  */
 static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1)
 {
-    CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]);
+    CCPrepare cc = gen_prepare_cc(s, b, cpu_T0);
 
     gen_update_cc_op(s);
     if (cc.mask != -1) {
-        tcg_gen_andi_tl(cpu_T[0], cc.reg, cc.mask);
-        cc.reg = cpu_T[0];
+        tcg_gen_andi_tl(cpu_T0, cc.reg, cc.mask);
+        cc.reg = cpu_T0;
     }
     set_cc_op(s, CC_OP_DYNAMIC);
     if (cc.use_reg2) {
@@ -1128,9 +1061,9 @@ static TCGLabel *gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
 
 static inline void gen_stos(DisasContext *s, TCGMemOp ot)
 {
-    gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
+    gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
     gen_string_movl_A0_EDI(s);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
 }
@@ -1138,8 +1071,8 @@ static inline void gen_stos(DisasContext *s, TCGMemOp ot)
 static inline void gen_lods(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
-    gen_op_mov_reg_v(ot, R_EAX, cpu_T[0]);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
+    gen_op_mov_reg_v(ot, R_EAX, cpu_T0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
 }
@@ -1147,7 +1080,7 @@ static inline void gen_lods(DisasContext *s, TCGMemOp ot)
 static inline void gen_scas(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_EDI(s);
-    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
     gen_op(s, OP_CMPL, ot, R_EAX);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
@@ -1156,7 +1089,7 @@ static inline void gen_scas(DisasContext *s, TCGMemOp ot)
 static inline void gen_cmps(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_EDI(s);
-    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
     gen_string_movl_A0_ESI(s);
     gen_op(s, OP_CMPL, ot, OR_TMP0);
     gen_op_movl_T0_Dshift(ot);
@@ -1164,6 +1097,19 @@ static inline void gen_cmps(DisasContext *s, TCGMemOp ot)
     gen_op_add_reg_T0(s->aflag, R_EDI);
 }
 
+static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot)
+{
+    if (s->flags & HF_IOBPT_MASK) {
+        TCGv_i32 t_size = tcg_const_i32(1 << ot);
+        TCGv t_next = tcg_const_tl(s->pc - s->cs_base);
+
+        gen_helper_bpt_io(cpu_env, t_port, t_size, t_next);
+        tcg_temp_free_i32(t_size);
+        tcg_temp_free(t_next);
+    }
+}
+
+
 static inline void gen_ins(DisasContext *s, TCGMemOp ot)
 {
     if (s->tb->cflags & CF_USE_ICOUNT) {
@@ -1172,14 +1118,15 @@ static inline void gen_ins(DisasContext *s, TCGMemOp ot)
     gen_string_movl_A0_EDI(s);
     /* Note: we must do this dummy write first to be restartable in
        case of page fault. */
-    tcg_gen_movi_tl(cpu_T[0], 0);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    tcg_gen_movi_tl(cpu_T0, 0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
-    gen_helper_in_func(ot, cpu_T[0], cpu_tmp2_i32);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_helper_in_func(ot, cpu_T0, cpu_tmp2_i32);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
     if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
     }
@@ -1191,15 +1138,15 @@ static inline void gen_outs(DisasContext *s, TCGMemOp ot)
         gen_io_start();
     }
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
 
     tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
-    tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[0]);
+    tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T0);
     gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
-
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
     if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
     }
@@ -1309,63 +1256,63 @@ static void gen_helper_fp_arith_STN_ST0(int op, int opreg)
 static void gen_op(DisasContext *s1, int op, TCGMemOp ot, int d)
 {
     if (d != OR_TMP0) {
-        gen_op_mov_v_reg(ot, cpu_T[0], d);
+        gen_op_mov_v_reg(ot, cpu_T0, d);
     } else {
-        gen_op_ld_v(s1, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s1, ot, cpu_T0, cpu_A0);
     }
     switch(op) {
     case OP_ADCL:
         gen_compute_eflags_c(s1, cpu_tmp4);
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_tmp4);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update3_cc(cpu_tmp4);
         set_cc_op(s1, CC_OP_ADCB + ot);
         break;
     case OP_SBBL:
         gen_compute_eflags_c(s1, cpu_tmp4);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_tmp4);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update3_cc(cpu_tmp4);
         set_cc_op(s1, CC_OP_SBBB + ot);
         break;
     case OP_ADDL:
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update2_cc();
         set_cc_op(s1, CC_OP_ADDB + ot);
         break;
     case OP_SUBL:
-        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update2_cc();
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
     default:
     case OP_ANDL:
-        tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_ORL:
-        tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_XORL:
-        tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_CMPL:
-        tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]);
-        tcg_gen_sub_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
+        tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0);
+        tcg_gen_sub_tl(cpu_cc_dst, cpu_T0, cpu_T1);
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
     }
@@ -1375,20 +1322,20 @@ static void gen_op(DisasContext *s1, int op, TCGMemOp ot, int d)
 static void gen_inc(DisasContext *s1, TCGMemOp ot, int d, int c)
 {
     if (d != OR_TMP0) {
-        gen_op_mov_v_reg(ot, cpu_T[0], d);
+        gen_op_mov_v_reg(ot, cpu_T0, d);
     } else {
-        gen_op_ld_v(s1, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s1, ot, cpu_T0, cpu_A0);
     }
     gen_compute_eflags_c(s1, cpu_cc_src);
     if (c > 0) {
-        tcg_gen_addi_tl(cpu_T[0], cpu_T[0], 1);
+        tcg_gen_addi_tl(cpu_T0, cpu_T0, 1);
         set_cc_op(s1, CC_OP_INCB + ot);
     } else {
-        tcg_gen_addi_tl(cpu_T[0], cpu_T[0], -1);
+        tcg_gen_addi_tl(cpu_T0, cpu_T0, -1);
         set_cc_op(s1, CC_OP_DECB + ot);
     }
     gen_op_st_rm_T0_A0(s1, ot, d);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_shift_flags(DisasContext *s, TCGMemOp ot, TCGv result,
@@ -1443,33 +1390,33 @@ static void gen_shift_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
-    tcg_gen_andi_tl(cpu_T[1], cpu_T[1], mask);
-    tcg_gen_subi_tl(cpu_tmp0, cpu_T[1], 1);
+    tcg_gen_andi_tl(cpu_T1, cpu_T1, mask);
+    tcg_gen_subi_tl(cpu_tmp0, cpu_T1, 1);
 
     if (is_right) {
         if (is_arith) {
-            gen_exts(ot, cpu_T[0]);
-            tcg_gen_sar_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_sar_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            gen_exts(ot, cpu_T0);
+            tcg_gen_sar_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_sar_tl(cpu_T0, cpu_T0, cpu_T1);
         } else {
-            gen_extu(ot, cpu_T[0]);
-            tcg_gen_shr_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            gen_extu(ot, cpu_T0);
+            tcg_gen_shr_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_T1);
         }
     } else {
-        tcg_gen_shl_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-        tcg_gen_shl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_shl_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+        tcg_gen_shl_tl(cpu_T0, cpu_T0, cpu_T1);
     }
 
     /* store */
     gen_op_st_rm_T0_A0(s, ot, op1);
 
-    gen_shift_flags(s, ot, cpu_T[0], cpu_tmp0, cpu_T[1], is_right);
+    gen_shift_flags(s, ot, cpu_T0, cpu_tmp0, cpu_T1, is_right);
 }
 
 static void gen_shift_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
@@ -1479,25 +1426,25 @@ static void gen_shift_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
 
     /* load */
     if (op1 == OR_TMP0)
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     else
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
 
     op2 &= mask;
     if (op2 != 0) {
         if (is_right) {
             if (is_arith) {
-                gen_exts(ot, cpu_T[0]);
-                tcg_gen_sari_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-                tcg_gen_sari_tl(cpu_T[0], cpu_T[0], op2);
+                gen_exts(ot, cpu_T0);
+                tcg_gen_sari_tl(cpu_tmp4, cpu_T0, op2 - 1);
+                tcg_gen_sari_tl(cpu_T0, cpu_T0, op2);
             } else {
-                gen_extu(ot, cpu_T[0]);
-                tcg_gen_shri_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], op2);
+                gen_extu(ot, cpu_T0);
+                tcg_gen_shri_tl(cpu_tmp4, cpu_T0, op2 - 1);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, op2);
             }
         } else {
-            tcg_gen_shli_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-            tcg_gen_shli_tl(cpu_T[0], cpu_T[0], op2);
+            tcg_gen_shli_tl(cpu_tmp4, cpu_T0, op2 - 1);
+            tcg_gen_shli_tl(cpu_T0, cpu_T0, op2);
         }
     }
 
@@ -1507,7 +1454,7 @@ static void gen_shift_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
     /* update eflags if non zero shift */
     if (op2 != 0) {
         tcg_gen_mov_tl(cpu_cc_src, cpu_tmp4);
-        tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+        tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
         set_cc_op(s, (is_right ? CC_OP_SARB : CC_OP_SHLB) + ot);
     }
 }
@@ -1519,41 +1466,41 @@ static void gen_rot_rm_T1(DisasContext *s, TCGMemOp ot, int op1, int is_right)
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
-    tcg_gen_andi_tl(cpu_T[1], cpu_T[1], mask);
+    tcg_gen_andi_tl(cpu_T1, cpu_T1, mask);
 
     switch (ot) {
     case MO_8:
         /* Replicate the 8-bit input so that a 32-bit rotate works.  */
-        tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
-        tcg_gen_muli_tl(cpu_T[0], cpu_T[0], 0x01010101);
+        tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
+        tcg_gen_muli_tl(cpu_T0, cpu_T0, 0x01010101);
         goto do_long;
     case MO_16:
         /* Replicate the 16-bit input so that a 32-bit rotate works.  */
-        tcg_gen_deposit_tl(cpu_T[0], cpu_T[0], cpu_T[0], 16, 16);
+        tcg_gen_deposit_tl(cpu_T0, cpu_T0, cpu_T0, 16, 16);
         goto do_long;
     do_long:
 #ifdef TARGET_X86_64
     case MO_32:
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         if (is_right) {
             tcg_gen_rotr_i32(cpu_tmp2_i32, cpu_tmp2_i32, cpu_tmp3_i32);
         } else {
             tcg_gen_rotl_i32(cpu_tmp2_i32, cpu_tmp2_i32, cpu_tmp3_i32);
         }
-        tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+        tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
         break;
 #endif
     default:
         if (is_right) {
-            tcg_gen_rotr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            tcg_gen_rotr_tl(cpu_T0, cpu_T0, cpu_T1);
         } else {
-            tcg_gen_rotl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            tcg_gen_rotl_tl(cpu_T0, cpu_T0, cpu_T1);
         }
         break;
     }
@@ -1569,12 +1516,12 @@ static void gen_rot_rm_T1(DisasContext *s, TCGMemOp ot, int op1, int is_right)
        since we've computed the flags into CC_SRC, these variables are
        currently dead.  */
     if (is_right) {
-        tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask - 1);
-        tcg_gen_shri_tl(cpu_cc_dst, cpu_T[0], mask);
+        tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask - 1);
+        tcg_gen_shri_tl(cpu_cc_dst, cpu_T0, mask);
         tcg_gen_andi_tl(cpu_cc_dst, cpu_cc_dst, 1);
     } else {
-        tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask);
-        tcg_gen_andi_tl(cpu_cc_dst, cpu_T[0], 1);
+        tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask);
+        tcg_gen_andi_tl(cpu_cc_dst, cpu_T0, 1);
     }
     tcg_gen_andi_tl(cpu_cc_src2, cpu_cc_src2, 1);
     tcg_gen_xor_tl(cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
@@ -1585,7 +1532,7 @@ static void gen_rot_rm_T1(DisasContext *s, TCGMemOp ot, int op1, int is_right)
        exactly as we computed above.  */
     t0 = tcg_const_i32(0);
     t1 = tcg_temp_new_i32();
-    tcg_gen_trunc_tl_i32(t1, cpu_T[1]);
+    tcg_gen_trunc_tl_i32(t1, cpu_T1);
     tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); 
     tcg_gen_movi_i32(cpu_tmp3_i32, CC_OP_EFLAGS);
     tcg_gen_movcond_i32(TCG_COND_NE, cpu_cc_op, t1, t0,
@@ -1605,9 +1552,9 @@ static void gen_rot_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
     op2 &= mask;
@@ -1615,20 +1562,20 @@ static void gen_rot_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
         switch (ot) {
 #ifdef TARGET_X86_64
         case MO_32:
-            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
             if (is_right) {
                 tcg_gen_rotri_i32(cpu_tmp2_i32, cpu_tmp2_i32, op2);
             } else {
                 tcg_gen_rotli_i32(cpu_tmp2_i32, cpu_tmp2_i32, op2);
             }
-            tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+            tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
             break;
 #endif
         default:
             if (is_right) {
-                tcg_gen_rotri_tl(cpu_T[0], cpu_T[0], op2);
+                tcg_gen_rotri_tl(cpu_T0, cpu_T0, op2);
             } else {
-                tcg_gen_rotli_tl(cpu_T[0], cpu_T[0], op2);
+                tcg_gen_rotli_tl(cpu_T0, cpu_T0, op2);
             }
             break;
         case MO_8:
@@ -1641,10 +1588,10 @@ static void gen_rot_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
             if (is_right) {
                 shift = mask + 1 - shift;
             }
-            gen_extu(ot, cpu_T[0]);
-            tcg_gen_shli_tl(cpu_tmp0, cpu_T[0], shift);
-            tcg_gen_shri_tl(cpu_T[0], cpu_T[0], mask + 1 - shift);
-            tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            gen_extu(ot, cpu_T0);
+            tcg_gen_shli_tl(cpu_tmp0, cpu_T0, shift);
+            tcg_gen_shri_tl(cpu_T0, cpu_T0, mask + 1 - shift);
+            tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         }
     }
@@ -1661,12 +1608,12 @@ static void gen_rot_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
            since we've computed the flags into CC_SRC, these variables are
            currently dead.  */
         if (is_right) {
-            tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask - 1);
-            tcg_gen_shri_tl(cpu_cc_dst, cpu_T[0], mask);
+            tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask - 1);
+            tcg_gen_shri_tl(cpu_cc_dst, cpu_T0, mask);
             tcg_gen_andi_tl(cpu_cc_dst, cpu_cc_dst, 1);
         } else {
-            tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask);
-            tcg_gen_andi_tl(cpu_cc_dst, cpu_T[0], 1);
+            tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask);
+            tcg_gen_andi_tl(cpu_cc_dst, cpu_T0, 1);
         }
         tcg_gen_andi_tl(cpu_cc_src2, cpu_cc_src2, 1);
         tcg_gen_xor_tl(cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
@@ -1683,24 +1630,24 @@ static void gen_rotc_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
 
     /* load */
     if (op1 == OR_TMP0)
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     else
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     
     if (is_right) {
         switch (ot) {
         case MO_8:
-            gen_helper_rcrb(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrb(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_16:
-            gen_helper_rcrw(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrw(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_32:
-            gen_helper_rcrl(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrl(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rcrq(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrq(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #endif
         default:
@@ -1709,17 +1656,17 @@ static void gen_rotc_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
     } else {
         switch (ot) {
         case MO_8:
-            gen_helper_rclb(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclb(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_16:
-            gen_helper_rclw(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclw(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_32:
-            gen_helper_rcll(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcll(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rclq(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclq(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #endif
         default:
@@ -1739,9 +1686,9 @@ static void gen_shiftd_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
     count = tcg_temp_new();
@@ -1753,11 +1700,11 @@ static void gen_shiftd_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
            This means "shrdw C, B, A" shifts A:B:A >> C.  Build the B:A
            portion by constructing it as a 32-bit value.  */
         if (is_right) {
-            tcg_gen_deposit_tl(cpu_tmp0, cpu_T[0], cpu_T[1], 16, 16);
-            tcg_gen_mov_tl(cpu_T[1], cpu_T[0]);
-            tcg_gen_mov_tl(cpu_T[0], cpu_tmp0);
+            tcg_gen_deposit_tl(cpu_tmp0, cpu_T0, cpu_T1, 16, 16);
+            tcg_gen_mov_tl(cpu_T1, cpu_T0);
+            tcg_gen_mov_tl(cpu_T0, cpu_tmp0);
         } else {
-            tcg_gen_deposit_tl(cpu_T[1], cpu_T[0], cpu_T[1], 16, 16);
+            tcg_gen_deposit_tl(cpu_T1, cpu_T0, cpu_T1, 16, 16);
         }
         /* FALLTHRU */
 #ifdef TARGET_X86_64
@@ -1765,57 +1712,57 @@ static void gen_shiftd_rm_T1(DisasContext *s, TCGMemOp ot, int op1,
         /* Concatenate the two 32-bit values and use a 64-bit shift.  */
         tcg_gen_subi_tl(cpu_tmp0, count, 1);
         if (is_right) {
-            tcg_gen_concat_tl_i64(cpu_T[0], cpu_T[0], cpu_T[1]);
-            tcg_gen_shr_i64(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shr_i64(cpu_T[0], cpu_T[0], count);
+            tcg_gen_concat_tl_i64(cpu_T0, cpu_T0, cpu_T1);
+            tcg_gen_shr_i64(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shr_i64(cpu_T0, cpu_T0, count);
         } else {
-            tcg_gen_concat_tl_i64(cpu_T[0], cpu_T[1], cpu_T[0]);
-            tcg_gen_shl_i64(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shl_i64(cpu_T[0], cpu_T[0], count);
+            tcg_gen_concat_tl_i64(cpu_T0, cpu_T1, cpu_T0);
+            tcg_gen_shl_i64(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shl_i64(cpu_T0, cpu_T0, count);
             tcg_gen_shri_i64(cpu_tmp0, cpu_tmp0, 32);
-            tcg_gen_shri_i64(cpu_T[0], cpu_T[0], 32);
+            tcg_gen_shri_i64(cpu_T0, cpu_T0, 32);
         }
         break;
 #endif
     default:
         tcg_gen_subi_tl(cpu_tmp0, count, 1);
         if (is_right) {
-            tcg_gen_shr_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
+            tcg_gen_shr_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
 
             tcg_gen_subfi_tl(cpu_tmp4, mask + 1, count);
-            tcg_gen_shr_tl(cpu_T[0], cpu_T[0], count);
-            tcg_gen_shl_tl(cpu_T[1], cpu_T[1], cpu_tmp4);
+            tcg_gen_shr_tl(cpu_T0, cpu_T0, count);
+            tcg_gen_shl_tl(cpu_T1, cpu_T1, cpu_tmp4);
         } else {
-            tcg_gen_shl_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
             if (ot == MO_16) {
                 /* Only needed if count > 16, for Intel behaviour.  */
                 tcg_gen_subfi_tl(cpu_tmp4, 33, count);
-                tcg_gen_shr_tl(cpu_tmp4, cpu_T[1], cpu_tmp4);
+                tcg_gen_shr_tl(cpu_tmp4, cpu_T1, cpu_tmp4);
                 tcg_gen_or_tl(cpu_tmp0, cpu_tmp0, cpu_tmp4);
             }
 
             tcg_gen_subfi_tl(cpu_tmp4, mask + 1, count);
-            tcg_gen_shl_tl(cpu_T[0], cpu_T[0], count);
-            tcg_gen_shr_tl(cpu_T[1], cpu_T[1], cpu_tmp4);
+            tcg_gen_shl_tl(cpu_T0, cpu_T0, count);
+            tcg_gen_shr_tl(cpu_T1, cpu_T1, cpu_tmp4);
         }
         tcg_gen_movi_tl(cpu_tmp4, 0);
-        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T[1], count, cpu_tmp4,
-                           cpu_tmp4, cpu_T[1]);
-        tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T1, count, cpu_tmp4,
+                           cpu_tmp4, cpu_T1);
+        tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_T1);
         break;
     }
 
     /* store */
     gen_op_st_rm_T0_A0(s, ot, op1);
 
-    gen_shift_flags(s, ot, cpu_T[0], cpu_tmp0, count, is_right);
+    gen_shift_flags(s, ot, cpu_T0, cpu_tmp0, count, is_right);
     tcg_temp_free(count);
 }
 
 static void gen_shift(DisasContext *s1, int op, TCGMemOp ot, int d, int s)
 {
     if (s != OR_TMP1)
-        gen_op_mov_v_reg(ot, cpu_T[1], s);
+        gen_op_mov_v_reg(ot, cpu_T1, s);
     switch(op) {
     case OP_ROL:
         gen_rot_rm_T1(s1, ot, d, 0);
@@ -1863,48 +1810,58 @@ static void gen_shifti(DisasContext *s1, int op, TCGMemOp ot, int d, int c)
         break;
     default:
         /* currently not optimized */
-        tcg_gen_movi_tl(cpu_T[1], c);
+        tcg_gen_movi_tl(cpu_T1, c);
         gen_shift(s1, op, ot, d, OR_TMP1);
         break;
     }
 }
 
-static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
-{
-    target_long disp;
-    int havesib;
+/* Decompose an address.  */
+
+typedef struct AddressParts {
+    int def_seg;
     int base;
     int index;
     int scale;
-    int mod, rm, code, override, must_add_seg;
-    TCGv sum;
+    target_long disp;
+} AddressParts;
+
+static AddressParts gen_lea_modrm_0(CPUX86State *env, DisasContext *s,
+                                    int modrm)
+{
+    int def_seg, base, index, scale, mod, rm;
+    target_long disp;
+    bool havesib;
+
+    def_seg = R_DS;
+    index = -1;
+    scale = 0;
+    disp = 0;
 
-    override = s->override;
-    must_add_seg = s->addseg;
-    if (override >= 0)
-        must_add_seg = 1;
     mod = (modrm >> 6) & 3;
     rm = modrm & 7;
+    base = rm | REX_B(s);
+
+    if (mod == 3) {
+        /* Normally filtered out earlier, but including this path
+           simplifies multi-byte nop, as well as bndcl, bndcu, bndcn.  */
+        goto done;
+    }
 
     switch (s->aflag) {
     case MO_64:
     case MO_32:
         havesib = 0;
-        base = rm;
-        index = -1;
-        scale = 0;
-
-        if (base == 4) {
-            havesib = 1;
-            code = cpu_ldub_code(env, s->pc++);
+        if (rm == 4) {
+            int code = cpu_ldub_code(env, s->pc++);
             scale = (code >> 6) & 3;
             index = ((code >> 3) & 7) | REX_X(s);
             if (index == 4) {
                 index = -1;  /* no index */
             }
-            base = (code & 7);
+            base = (code & 7) | REX_B(s);
+            havesib = 1;
         }
-        base |= REX_B(s);
 
         switch (mod) {
         case 0:
@@ -1913,10 +1870,9 @@ static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
                 disp = (int32_t)cpu_ldl_code(env, s->pc);
                 s->pc += 4;
                 if (CODE64(s) && !havesib) {
+                    base = -2;
                     disp += s->pc + s->rip_offset;
                 }
-            } else {
-                disp = 0;
             }
             break;
         case 1:
@@ -1933,204 +1889,132 @@ static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
         if (base == R_ESP && s->popl_esp_hack) {
             disp += s->popl_esp_hack;
         }
-
-        /* Compute the address, with a minimum number of TCG ops.  */
-        TCGV_UNUSED(sum);
-        if (index >= 0) {
-            if (scale == 0) {
-                sum = cpu_regs[index];
-            } else {
-                tcg_gen_shli_tl(cpu_A0, cpu_regs[index], scale);
-                sum = cpu_A0;
-            }
-            if (base >= 0) {
-                tcg_gen_add_tl(cpu_A0, sum, cpu_regs[base]);
-                sum = cpu_A0;
-            }
-        } else if (base >= 0) {
-            sum = cpu_regs[base];
-        }
-        if (TCGV_IS_UNUSED(sum)) {
-            tcg_gen_movi_tl(cpu_A0, disp);
-        } else {
-            tcg_gen_addi_tl(cpu_A0, sum, disp);
-        }
-
-        if (must_add_seg) {
-            if (override < 0) {
-                if (base == R_EBP || base == R_ESP) {
-                    override = R_SS;
-                } else {
-                    override = R_DS;
-                }
-            }
-
-            tcg_gen_ld_tl(cpu_tmp0, cpu_env,
-                          offsetof(CPUX86State, segs[override].base));
-            if (CODE64(s)) {
-                if (s->aflag == MO_32) {
-                    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
-                }
-                tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-                return;
-            }
-
-            tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-        }
-
-        if (s->aflag == MO_32) {
-            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+        if (base == R_EBP || base == R_ESP) {
+            def_seg = R_SS;
         }
         break;
 
     case MO_16:
-        switch (mod) {
-        case 0:
+        if (mod == 0) {
             if (rm == 6) {
+                base = -1;
                 disp = cpu_lduw_code(env, s->pc);
                 s->pc += 2;
-                tcg_gen_movi_tl(cpu_A0, disp);
-                rm = 0; /* avoid SS override */
-                goto no_rm;
-            } else {
-                disp = 0;
+                break;
             }
-            break;
-        case 1:
+        } else if (mod == 1) {
             disp = (int8_t)cpu_ldub_code(env, s->pc++);
-            break;
-        default:
-        case 2:
+        } else {
             disp = (int16_t)cpu_lduw_code(env, s->pc);
             s->pc += 2;
-            break;
         }
 
-        sum = cpu_A0;
         switch (rm) {
         case 0:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBX], cpu_regs[R_ESI]);
+            base = R_EBX;
+            index = R_ESI;
             break;
         case 1:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBX], cpu_regs[R_EDI]);
+            base = R_EBX;
+            index = R_EDI;
             break;
         case 2:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBP], cpu_regs[R_ESI]);
+            base = R_EBP;
+            index = R_ESI;
+            def_seg = R_SS;
             break;
         case 3:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBP], cpu_regs[R_EDI]);
+            base = R_EBP;
+            index = R_EDI;
+            def_seg = R_SS;
             break;
         case 4:
-            sum = cpu_regs[R_ESI];
+            base = R_ESI;
             break;
         case 5:
-            sum = cpu_regs[R_EDI];
+            base = R_EDI;
             break;
         case 6:
-            sum = cpu_regs[R_EBP];
+            base = R_EBP;
+            def_seg = R_SS;
             break;
         default:
         case 7:
-            sum = cpu_regs[R_EBX];
+            base = R_EBX;
             break;
         }
-        tcg_gen_addi_tl(cpu_A0, sum, disp);
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    no_rm:
-        if (must_add_seg) {
-            if (override < 0) {
-                if (rm == 2 || rm == 3 || rm == 6) {
-                    override = R_SS;
-                } else {
-                    override = R_DS;
-                }
-            }
-            gen_op_addl_A0_seg(s, override);
-        }
         break;
 
     default:
         tcg_abort();
     }
+
+ done:
+    return (AddressParts){ def_seg, base, index, scale, disp };
 }
 
-static void gen_nop_modrm(CPUX86State *env, DisasContext *s, int modrm)
+/* Compute the address, with a minimum number of TCG ops.  */
+static TCGv gen_lea_modrm_1(AddressParts a)
 {
-    int mod, rm, base, code;
+    TCGv ea;
 
-    mod = (modrm >> 6) & 3;
-    if (mod == 3)
-        return;
-    rm = modrm & 7;
+    TCGV_UNUSED(ea);
+    if (a.index >= 0) {
+        if (a.scale == 0) {
+            ea = cpu_regs[a.index];
+        } else {
+            tcg_gen_shli_tl(cpu_A0, cpu_regs[a.index], a.scale);
+            ea = cpu_A0;
+        }
+        if (a.base >= 0) {
+            tcg_gen_add_tl(cpu_A0, ea, cpu_regs[a.base]);
+            ea = cpu_A0;
+        }
+    } else if (a.base >= 0) {
+        ea = cpu_regs[a.base];
+    }
+    if (TCGV_IS_UNUSED(ea)) {
+        tcg_gen_movi_tl(cpu_A0, a.disp);
+        ea = cpu_A0;
+    } else if (a.disp != 0) {
+        tcg_gen_addi_tl(cpu_A0, ea, a.disp);
+        ea = cpu_A0;
+    }
 
-    switch (s->aflag) {
-    case MO_64:
-    case MO_32:
-        base = rm;
+    return ea;
+}
 
-        if (base == 4) {
-            code = cpu_ldub_code(env, s->pc++);
-            base = (code & 7);
-        }
+static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
+{
+    AddressParts a = gen_lea_modrm_0(env, s, modrm);
+    TCGv ea = gen_lea_modrm_1(a);
+    gen_lea_v_seg(s, s->aflag, ea, a.def_seg, s->override);
+}
 
-        switch (mod) {
-        case 0:
-            if (base == 5) {
-                s->pc += 4;
-            }
-            break;
-        case 1:
-            s->pc++;
-            break;
-        default:
-        case 2:
-            s->pc += 4;
-            break;
-        }
-        break;
+static void gen_nop_modrm(CPUX86State *env, DisasContext *s, int modrm)
+{
+    (void)gen_lea_modrm_0(env, s, modrm);
+}
 
-    case MO_16:
-        switch (mod) {
-        case 0:
-            if (rm == 6) {
-                s->pc += 2;
-            }
-            break;
-        case 1:
-            s->pc++;
-            break;
-        default:
-        case 2:
-            s->pc += 2;
-            break;
-        }
-        break;
+/* Used for BNDCL, BNDCU, BNDCN.  */
+static void gen_bndck(CPUX86State *env, DisasContext *s, int modrm,
+                      TCGCond cond, TCGv_i64 bndv)
+{
+    TCGv ea = gen_lea_modrm_1(gen_lea_modrm_0(env, s, modrm));
 
-    default:
-        tcg_abort();
+    tcg_gen_extu_tl_i64(cpu_tmp1_i64, ea);
+    if (!CODE64(s)) {
+        tcg_gen_ext32u_i64(cpu_tmp1_i64, cpu_tmp1_i64);
     }
+    tcg_gen_setcond_i64(cond, cpu_tmp1_i64, cpu_tmp1_i64, bndv);
+    tcg_gen_extrl_i64_i32(cpu_tmp2_i32, cpu_tmp1_i64);
+    gen_helper_bndck(cpu_env, cpu_tmp2_i32);
 }
 
 /* used for LEA and MOV AX, mem */
 static void gen_add_A0_ds_seg(DisasContext *s)
 {
-    int override, must_add_seg;
-    must_add_seg = s->addseg;
-    override = R_DS;
-    if (s->override >= 0) {
-        override = s->override;
-        must_add_seg = 1;
-    }
-    if (must_add_seg) {
-#ifdef TARGET_X86_64
-        if (CODE64(s)) {
-            gen_op_addq_A0_seg(override);
-        } else
-#endif
-        {
-            gen_op_addl_A0_seg(s, override);
-        }
-    }
+    gen_lea_v_seg(s, s->aflag, cpu_A0, R_DS, s->override);
 }
 
 /* generate modrm memory load or store of 'reg'. TMP0 is used if reg ==
@@ -2145,23 +2029,23 @@ static void gen_ldst_modrm(CPUX86State *env, DisasContext *s, int modrm,
     if (mod == 3) {
         if (is_store) {
             if (reg != OR_TMP0)
-                gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
             if (reg != OR_TMP0)
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
     } else {
         gen_lea_modrm(env, s, modrm);
         if (is_store) {
             if (reg != OR_TMP0)
-                gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
             if (reg != OR_TMP0)
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
     }
 }
@@ -2258,7 +2142,7 @@ static void gen_cmovcc1(CPUX86State *env, DisasContext *s, TCGMemOp ot, int b,
 
     gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
 
-    cc = gen_prepare_cc(s, b, cpu_T[1]);
+    cc = gen_prepare_cc(s, b, cpu_T1);
     if (cc.mask != -1) {
         TCGv t0 = tcg_temp_new();
         tcg_gen_andi_tl(t0, cc.reg, cc.mask);
@@ -2268,9 +2152,9 @@ static void gen_cmovcc1(CPUX86State *env, DisasContext *s, TCGMemOp ot, int b,
         cc.reg2 = tcg_const_tl(cc.imm);
     }
 
-    tcg_gen_movcond_tl(cc.cond, cpu_T[0], cc.reg, cc.reg2,
-                       cpu_T[0], cpu_regs[reg]);
-    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+    tcg_gen_movcond_tl(cc.cond, cpu_T0, cc.reg, cc.reg2,
+                       cpu_T0, cpu_regs[reg]);
+    gen_op_mov_reg_v(ot, reg, cpu_T0);
 
     if (cc.mask != -1) {
         tcg_temp_free(cc.reg);
@@ -2282,29 +2166,24 @@ static void gen_cmovcc1(CPUX86State *env, DisasContext *s, TCGMemOp ot, int b,
 
 static inline void gen_op_movl_T0_seg(int seg_reg)
 {
-    tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
+    tcg_gen_ld32u_tl(cpu_T0, cpu_env,
                      offsetof(CPUX86State,segs[seg_reg].selector));
 }
 
 static inline void gen_op_movl_seg_T0_vm(int seg_reg)
 {
-    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff);
-    tcg_gen_st32_tl(cpu_T[0], cpu_env, 
+    tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
+    tcg_gen_st32_tl(cpu_T0, cpu_env,
                     offsetof(CPUX86State,segs[seg_reg].selector));
-    tcg_gen_shli_tl(cpu_T[0], cpu_T[0], 4);
-    tcg_gen_st_tl(cpu_T[0], cpu_env, 
-                  offsetof(CPUX86State,segs[seg_reg].base));
+    tcg_gen_shli_tl(cpu_seg_base[seg_reg], cpu_T0, 4);
 }
 
 /* move T0 to seg_reg and compute if the CPU state may change. Never
    call this function with seg_reg == R_CS */
-static void gen_movl_seg_T0(DisasContext *s, int seg_reg, target_ulong cur_eip)
+static void gen_movl_seg_T0(DisasContext *s, int seg_reg)
 {
     if (s->pe && !s->vm86) {
-        /* XXX: optimize by finding processor state dynamically */
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         gen_helper_load_seg(cpu_env, tcg_const_i32(seg_reg), cpu_tmp2_i32);
         /* abort translation because the addseg value may change or
            because ss32 may change. For R_SS, translation must always
@@ -2345,44 +2224,25 @@ gen_svm_check_intercept(DisasContext *s, target_ulong pc_start, uint64_t type)
 
 static inline void gen_stack_update(DisasContext *s, int addend)
 {
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        gen_op_add_reg_im(MO_64, R_ESP, addend);
-    } else
-#endif
-    if (s->ss32) {
-        gen_op_add_reg_im(MO_32, R_ESP, addend);
-    } else {
-        gen_op_add_reg_im(MO_16, R_ESP, addend);
-    }
+    gen_op_add_reg_im(mo_stacksize(s), R_ESP, addend);
 }
 
 /* Generate a push. It depends on ss32, addseg and dflag.  */
 static void gen_push_v(DisasContext *s, TCGv val)
 {
-    TCGMemOp a_ot, d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = mo_stacksize(s);
     int size = 1 << d_ot;
     TCGv new_esp = cpu_A0;
 
     tcg_gen_subi_tl(cpu_A0, cpu_regs[R_ESP], size);
 
-    if (CODE64(s)) {
-        a_ot = MO_64;
-    } else if (s->ss32) {
-        a_ot = MO_32;
+    if (!CODE64(s)) {
         if (s->addseg) {
             new_esp = cpu_tmp4;
             tcg_gen_mov_tl(new_esp, cpu_A0);
-            gen_op_addl_A0_seg(s, R_SS);
-        } else {
-            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
         }
-    } else {
-        a_ot = MO_16;
-        new_esp = cpu_tmp4;
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-        tcg_gen_mov_tl(new_esp, cpu_A0);
-        gen_op_addl_A0_seg(s, R_SS);
+        gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
     }
 
     gen_op_st_v(s, d_ot, val, cpu_A0);
@@ -2393,127 +2253,112 @@ static void gen_push_v(DisasContext *s, TCGv val)
 static TCGMemOp gen_pop_T0(DisasContext *s)
 {
     TCGMemOp d_ot = mo_pushpop(s, s->dflag);
-    TCGv addr = cpu_A0;
 
-    if (CODE64(s)) {
-        addr = cpu_regs[R_ESP];
-    } else if (!s->ss32) {
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_ESP]);
-        gen_op_addl_A0_seg(s, R_SS);
-    } else if (s->addseg) {
-        tcg_gen_mov_tl(cpu_A0, cpu_regs[R_ESP]);
-        gen_op_addl_A0_seg(s, R_SS);
-    } else {
-        tcg_gen_ext32u_tl(cpu_A0, cpu_regs[R_ESP]);
-    }
+    gen_lea_v_seg(s, mo_stacksize(s), cpu_regs[R_ESP], R_SS, -1);
+    gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
 
-    gen_op_ld_v(s, d_ot, cpu_T[0], addr);
     return d_ot;
 }
 
-static void gen_pop_update(DisasContext *s, TCGMemOp ot)
+static inline void gen_pop_update(DisasContext *s, TCGMemOp ot)
 {
     gen_stack_update(s, 1 << ot);
 }
 
-static void gen_stack_A0(DisasContext *s)
+static inline void gen_stack_A0(DisasContext *s)
 {
-    gen_op_movl_A0_reg(R_ESP);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
+    gen_lea_v_seg(s, s->ss32 ? MO_32 : MO_16, cpu_regs[R_ESP], R_SS, -1);
 }
 
-/* NOTE: wrap around in 16 bit not fully handled */
 static void gen_pusha(DisasContext *s)
 {
+    TCGMemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    TCGMemOp d_ot = s->dflag;
+    int size = 1 << d_ot;
     int i;
-    gen_op_movl_A0_reg(R_ESP);
-    gen_op_addl_A0_im(-8 << s->dflag);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
-    for(i = 0;i < 8; i++) {
-        gen_op_mov_v_reg(MO_32, cpu_T[0], 7 - i);
-        gen_op_st_v(s, s->dflag, cpu_T[0], cpu_A0);
-        gen_op_addl_A0_im(1 << s->dflag);
+
+    for (i = 0; i < 8; i++) {
+        tcg_gen_addi_tl(cpu_A0, cpu_regs[R_ESP], (i - 8) * size);
+        gen_lea_v_seg(s, s_ot, cpu_A0, R_SS, -1);
+        gen_op_st_v(s, d_ot, cpu_regs[7 - i], cpu_A0);
     }
-    gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+
+    gen_stack_update(s, -8 * size);
 }
 
-/* NOTE: wrap around in 16 bit not fully handled */
 static void gen_popa(DisasContext *s)
 {
+    TCGMemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    TCGMemOp d_ot = s->dflag;
+    int size = 1 << d_ot;
     int i;
-    gen_op_movl_A0_reg(R_ESP);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    tcg_gen_addi_tl(cpu_T[1], cpu_T[1], 8 << s->dflag);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
-    for(i = 0;i < 8; i++) {
+
+    for (i = 0; i < 8; i++) {
         /* ESP is not reloaded */
-        if (i != 3) {
-            gen_op_ld_v(s, s->dflag, cpu_T[0], cpu_A0);
-            gen_op_mov_reg_v(s->dflag, 7 - i, cpu_T[0]);
+        if (7 - i == R_ESP) {
+            continue;
         }
-        gen_op_addl_A0_im(1 << s->dflag);
+        tcg_gen_addi_tl(cpu_A0, cpu_regs[R_ESP], i * size);
+        gen_lea_v_seg(s, s_ot, cpu_A0, R_SS, -1);
+        gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
+        gen_op_mov_reg_v(d_ot, 7 - i, cpu_T0);
     }
-    gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+
+    gen_stack_update(s, 8 * size);
 }
 
 static void gen_enter(DisasContext *s, int esp_addend, int level)
 {
-    TCGMemOp ot = mo_pushpop(s, s->dflag);
-    int opsize = 1 << ot;
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+    int size = 1 << d_ot;
 
-    level &= 0x1f;
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        gen_op_movl_A0_reg(R_ESP);
-        gen_op_addq_A0_im(-opsize);
-        tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-
-        /* push bp */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-        gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-        if (level) {
-            /* XXX: must save state */
-            gen_helper_enter64_level(cpu_env, tcg_const_i32(level),
-                                     tcg_const_i32((ot == MO_64)),
-                                     cpu_T[1]);
-        }
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[1]);
-        tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
-        gen_op_mov_reg_v(MO_64, R_ESP, cpu_T[1]);
-    } else
-#endif
-    {
-        gen_op_movl_A0_reg(R_ESP);
-        gen_op_addl_A0_im(-opsize);
-        if (!s->ss32)
-            tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-        tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-        if (s->addseg)
-            gen_op_addl_A0_seg(s, R_SS);
-        /* push bp */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-        gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-        if (level) {
-            /* XXX: must save state */
-            gen_helper_enter_level(cpu_env, tcg_const_i32(level),
-                                   tcg_const_i32(s->dflag - 1),
-                                   cpu_T[1]);
-        }
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[1]);
-        tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
-        gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+    /* Push BP; compute FrameTemp into T1.  */
+    tcg_gen_subi_tl(cpu_T1, cpu_regs[R_ESP], size);
+    gen_lea_v_seg(s, a_ot, cpu_T1, R_SS, -1);
+    gen_op_st_v(s, d_ot, cpu_regs[R_EBP], cpu_A0);
+
+    level &= 31;
+    if (level != 0) {
+        int i;
+
+        /* Copy level-1 pointers from the previous frame.  */
+        for (i = 1; i < level; ++i) {
+            tcg_gen_subi_tl(cpu_A0, cpu_regs[R_EBP], size * i);
+            gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+            gen_op_ld_v(s, d_ot, cpu_tmp0, cpu_A0);
+
+            tcg_gen_subi_tl(cpu_A0, cpu_T1, size * i);
+            gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+            gen_op_st_v(s, d_ot, cpu_tmp0, cpu_A0);
+        }
+
+        /* Push the current FrameTemp as the last level.  */
+        tcg_gen_subi_tl(cpu_A0, cpu_T1, size * level);
+        gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+        gen_op_st_v(s, d_ot, cpu_T1, cpu_A0);
     }
+
+    /* Copy the FrameTemp value to EBP.  */
+    gen_op_mov_reg_v(a_ot, R_EBP, cpu_T1);
+
+    /* Compute the final value of ESP.  */
+    tcg_gen_subi_tl(cpu_T1, cpu_T1, esp_addend + size * level);
+    gen_op_mov_reg_v(a_ot, R_ESP, cpu_T1);
+}
+
+static void gen_leave(DisasContext *s)
+{
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = mo_stacksize(s);
+
+    gen_lea_v_seg(s, a_ot, cpu_regs[R_EBP], R_SS, -1);
+    gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
+
+    tcg_gen_addi_tl(cpu_T1, cpu_regs[R_EBP], 1 << d_ot);
+
+    gen_op_mov_reg_v(d_ot, R_EBP, cpu_T0);
+    gen_op_mov_reg_v(a_ot, R_ESP, cpu_T1);
 }
 
 static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
@@ -2524,6 +2369,30 @@ static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
     s->is_jmp = DISAS_TB_JUMP;
 }
 
+/* Generate #UD for the current instruction.  The assumption here is that
+   the instruction is known, but it isn't allowed in the current cpu mode.  */
+static void gen_illegal_opcode(DisasContext *s)
+{
+    gen_exception(s, EXCP06_ILLOP, s->pc_start - s->cs_base);
+}
+
+/* Similarly, except that the assumption here is that we don't decode
+   the instruction at all -- either a missing opcode, an unimplemented
+   feature, or just a bogus instruction stream.  */
+static void gen_unknown_opcode(CPUX86State *env, DisasContext *s)
+{
+    gen_illegal_opcode(s);
+
+    if (qemu_loglevel_mask(LOG_UNIMP)) {
+        target_ulong pc = s->pc_start, end = s->pc;
+        qemu_log("ILLOPC: " TARGET_FMT_lx ":", pc);
+        for (; pc < end; ++pc) {
+            qemu_log(" %02x", cpu_ldub_code(env, pc));
+        }
+        qemu_log("\n");
+    }
+}
+
 /* an interrupt is different from an exception because of the
    privilege checks */
 static void gen_interrupt(DisasContext *s, int intno,
@@ -2544,14 +2413,56 @@ static void gen_debug(DisasContext *s, target_ulong cur_eip)
     s->is_jmp = DISAS_TB_JUMP;
 }
 
-/* generate a generic end of block. Trace exception is also generated
-   if needed */
-static void gen_eob(DisasContext *s)
+static void gen_set_hflag(DisasContext *s, uint32_t mask)
+{
+    if ((s->flags & mask) == 0) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_ld_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_ori_i32(t, t, mask);
+        tcg_gen_st_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_temp_free_i32(t);
+        s->flags |= mask;
+    }
+}
+
+static void gen_reset_hflag(DisasContext *s, uint32_t mask)
+{
+    if (s->flags & mask) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_ld_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_andi_i32(t, t, ~mask);
+        tcg_gen_st_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_temp_free_i32(t);
+        s->flags &= ~mask;
+    }
+}
+
+/* Clear BND registers during legacy branches.  */
+static void gen_bnd_jmp(DisasContext *s)
+{
+    /* Clear the registers only if BND prefix is missing, MPX is enabled,
+       and if the BNDREGs are known to be in use (non-zero) already.
+       The helper itself will check BNDPRESERVE at runtime.  */
+    if ((s->prefix & PREFIX_REPNZ) == 0
+        && (s->flags & HF_MPX_EN_MASK) != 0
+        && (s->flags & HF_MPX_IU_MASK) != 0) {
+        gen_helper_bnd_jmp(cpu_env);
+    }
+}
+
+/* Generate an end of block. Trace exception is also generated if needed.
+   If IIM, set HF_INHIBIT_IRQ_MASK if it isn't already set.  */
+static void gen_eob_inhibit_irq(DisasContext *s, bool inhibit)
 {
     gen_update_cc_op(s);
-    if (s->tb->flags & HF_INHIBIT_IRQ_MASK) {
-        gen_helper_reset_inhibit_irq(cpu_env);
+
+    /* If several instructions disable interrupts, only the first does it.  */
+    if (inhibit && !(s->flags & HF_INHIBIT_IRQ_MASK)) {
+        gen_set_hflag(s, HF_INHIBIT_IRQ_MASK);
+    } else {
+        gen_reset_hflag(s, HF_INHIBIT_IRQ_MASK);
     }
+
     if (s->tb->flags & HF_RF_MASK) {
         gen_helper_reset_rf(cpu_env);
     }
@@ -2565,6 +2476,12 @@ static void gen_eob(DisasContext *s)
     s->is_jmp = DISAS_TB_JUMP;
 }
 
+/* End of block, resetting the inhibit irq flag.  */
+static void gen_eob(DisasContext *s)
+{
+    gen_eob_inhibit_irq(s, false);
+}
+
 /* generate a jump to eip. No segment change must happen before as a
    direct call to the next block may occur */
 static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num)
@@ -2601,28 +2518,28 @@ static inline void gen_ldo_env_A0(DisasContext *s, int offset)
 {
     int mem_index = s->mem_index;
     tcg_gen_qemu_ld_i64(cpu_tmp1_i64, cpu_A0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
     tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
     tcg_gen_qemu_ld_i64(cpu_tmp1_i64, cpu_tmp0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
 }
 
 static inline void gen_sto_env_A0(DisasContext *s, int offset)
 {
     int mem_index = s->mem_index;
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
     tcg_gen_qemu_st_i64(cpu_tmp1_i64, cpu_A0, mem_index, MO_LEQ);
     tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
     tcg_gen_qemu_st_i64(cpu_tmp1_i64, cpu_tmp0, mem_index, MO_LEQ);
 }
 
 static inline void gen_op_movo(int d_offset, int s_offset)
 {
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(XMMReg, XMM_Q(0)));
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(XMMReg, XMM_Q(0)));
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(XMMReg, XMM_Q(1)));
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(0)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(0)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(1)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(1)));
 }
 
 static inline void gen_op_movq(int d_offset, int s_offset)
@@ -2995,7 +2912,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         b1 = 0;
     sse_fn_epp = sse_op_table1[b][b1];
     if (!sse_fn_epp) {
-        goto illegal_op;
+        goto unknown_op;
     }
     if ((b <= 0x5f && b >= 0x10) || b == 0xc6 || b == 0xc2) {
         is_xmm = 1;
@@ -3014,15 +2931,19 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
     }
     if (s->flags & HF_EM_MASK) {
     illegal_op:
-        gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
+        gen_illegal_opcode(s);
         return;
     }
-    if (is_xmm && !(s->flags & HF_OSFXSR_MASK))
-        if ((b != 0x38 && b != 0x3a) || (s->prefix & PREFIX_DATA))
-            goto illegal_op;
+    if (is_xmm
+        && !(s->flags & HF_OSFXSR_MASK)
+        && ((b != 0x38 && b != 0x3a) || (s->prefix & PREFIX_DATA))) {
+        goto unknown_op;
+    }
     if (b == 0x0e) {
-        if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
-            goto illegal_op;
+        if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
+            /* If we were fully decoding this we might use illegal_op.  */
+            goto unknown_op;
+        }
         /* femms */
         gen_helper_emms(cpu_env);
         return;
@@ -3047,8 +2968,9 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         b |= (b1 << 8);
         switch(b) {
         case 0x0e7: /* movntq */
-            if (mod == 3)
+            if (mod == 3) {
                 goto illegal_op;
+            }
             gen_lea_modrm(env, s, modrm);
             gen_stq_env_A0(s, offsetof(CPUX86State, fpregs[reg].mmx));
             break;
@@ -3073,25 +2995,25 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             gen_lea_modrm(env, s, modrm);
             if (b1 & 1) {
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                    xmm_regs[reg].XMM_L(0)));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                    xmm_regs[reg].ZMM_L(0)));
+                gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
             }
             break;
         case 0x6e: /* movd mm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
-                tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
+                tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
             } else
 #endif
             {
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx));
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3101,14 +3023,14 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
-                gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
+                gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T0);
             } else
 #endif
             {
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3142,30 +3064,30 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         case 0x210: /* movss xmm, ea */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+                gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
             }
             break;
         case 0x310: /* movsd xmm, ea */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+                                           xmm_regs[reg].ZMM_Q(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
             break;
         case 0x012: /* movlps */
@@ -3173,12 +3095,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 /* movhlps */
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(1)));
             }
             break;
         case 0x212: /* movsldup */
@@ -3187,40 +3109,40 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(2)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(2)));
             }
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
             break;
         case 0x312: /* movddup */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
-            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
             break;
         case 0x016: /* movhps */
         case 0x116: /* movhpd */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(1)));
+                                           xmm_regs[reg].ZMM_Q(1)));
             } else {
                 /* movlhps */
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
             break;
         case 0x216: /* movshdup */
@@ -3229,15 +3151,15 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(1)));
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(3)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(1)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(3)));
             }
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             break;
         case 0x178:
         case 0x378:
@@ -3263,13 +3185,13 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         case 0x7e: /* movd ea, mm */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(cpu_T[0], cpu_env, 
+                tcg_gen_ld_i64(cpu_T0, cpu_env,
                                offsetof(CPUX86State,fpregs[reg].mmx));
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
             } else
 #endif
             {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env,
                                  offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
             }
@@ -3277,14 +3199,14 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         case 0x17e: /* movd ea, xmm */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(cpu_T[0], cpu_env, 
-                               offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+                tcg_gen_ld_i64(cpu_T0, cpu_env,
+                               offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
             } else
 #endif
             {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
-                                 offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                                 offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
             }
             break;
@@ -3292,13 +3214,13 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
-            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
+            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)));
             break;
         case 0x7f: /* movq ea, mm */
             if (mod != 3) {
@@ -3328,23 +3250,23 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         case 0x211: /* movss ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+                gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
             }
             break;
         case 0x311: /* movsd ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
             }
             break;
         case 0x013: /* movlps */
@@ -3352,7 +3274,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 goto illegal_op;
             }
@@ -3362,7 +3284,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(1)));
+                                           xmm_regs[reg].ZMM_Q(1)));
             } else {
                 goto illegal_op;
             }
@@ -3374,26 +3296,26 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         case 0x172:
         case 0x173:
             if (b1 >= 2) {
-	        goto illegal_op;
+	        goto unknown_op;
             }
             val = cpu_ldub_code(env, s->pc++);
             if (is_xmm) {
-                tcg_gen_movi_tl(cpu_T[0], val);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(1)));
+                tcg_gen_movi_tl(cpu_T0, val);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(1)));
                 op1_offset = offsetof(CPUX86State,xmm_t0);
             } else {
-                tcg_gen_movi_tl(cpu_T[0], val);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(1)));
+                tcg_gen_movi_tl(cpu_T0, val);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(1)));
                 op1_offset = offsetof(CPUX86State,mmx_t0);
             }
             sse_fn_epp = sse_op_table2[((b - 1) & 3) * 8 +
                                        (((modrm >> 3)) & 7)][b1];
             if (!sse_fn_epp) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (is_xmm) {
                 rm = (modrm & 7) | REX_B(s);
@@ -3452,12 +3374,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             if (ot == MO_32) {
                 SSEFunc_0_epi sse_fn_epi = sse_op_table3ai[(b >> 8) & 1];
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 sse_fn_epi(cpu_env, cpu_ptr0, cpu_tmp2_i32);
             } else {
 #ifdef TARGET_X86_64
                 SSEFunc_0_epl sse_fn_epl = sse_op_table3aq[(b >> 8) & 1];
-                sse_fn_epl(cpu_env, cpu_ptr0, cpu_T[0]);
+                sse_fn_epl(cpu_env, cpu_ptr0, cpu_T0);
 #else
                 goto illegal_op;
 #endif
@@ -3502,10 +3424,10 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 if ((b >> 8) & 1) {
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.XMM_Q(0)));
+                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_Q(0)));
                 } else {
-                    gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                    tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
+                    gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                    tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
                 }
                 op2_offset = offsetof(CPUX86State,xmm_t0);
             } else {
@@ -3517,17 +3439,17 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 SSEFunc_i_ep sse_fn_i_ep =
                     sse_op_table3bi[((b >> 7) & 2) | (b & 1)];
                 sse_fn_i_ep(cpu_tmp2_i32, cpu_env, cpu_ptr0);
-                tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+                tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
             } else {
 #ifdef TARGET_X86_64
                 SSEFunc_l_ep sse_fn_l_ep =
                     sse_op_table3bq[((b >> 7) & 2) | (b & 1)];
-                sse_fn_l_ep(cpu_T[0], cpu_env, cpu_ptr0);
+                sse_fn_l_ep(cpu_T0, cpu_env, cpu_ptr0);
 #else
                 goto illegal_op;
 #endif
             }
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         case 0xc4: /* pinsrw */
         case 0x1c4:
@@ -3536,11 +3458,11 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
-                tcg_gen_st16_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,xmm_regs[reg].XMM_W(val)));
+                tcg_gen_st16_tl(cpu_T0, cpu_env,
+                                offsetof(CPUX86State,xmm_regs[reg].ZMM_W(val)));
             } else {
                 val &= 3;
-                tcg_gen_st16_tl(cpu_T[0], cpu_env,
+                tcg_gen_st16_tl(cpu_T0, cpu_env,
                                 offsetof(CPUX86State,fpregs[reg].mmx.MMX_W(val)));
             }
             break;
@@ -3553,41 +3475,41 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             if (b1) {
                 val &= 7;
                 rm = (modrm & 7) | REX_B(s);
-                tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[rm].XMM_W(val)));
+                tcg_gen_ld16u_tl(cpu_T0, cpu_env,
+                                 offsetof(CPUX86State,xmm_regs[rm].ZMM_W(val)));
             } else {
                 val &= 3;
                 rm = (modrm & 7);
-                tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
+                tcg_gen_ld16u_tl(cpu_T0, cpu_env,
                                 offsetof(CPUX86State,fpregs[rm].mmx.MMX_W(val)));
             }
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         case 0x1d6: /* movq ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
-                gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
+                gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(1)));
             }
             break;
         case 0x2d6: /* movq2dq */
             gen_helper_enter_mmx(cpu_env);
             rm = (modrm & 7);
-            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
+            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
                         offsetof(CPUX86State,fpregs[rm].mmx));
-            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
+            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)));
             break;
         case 0x3d6: /* movdq2q */
             gen_helper_enter_mmx(cpu_env);
             rm = (modrm & 7) | REX_B(s);
             gen_op_movq(offsetof(CPUX86State,fpregs[reg & 7].mmx),
-                        offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                        offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             break;
         case 0xd7: /* pmovmskb */
         case 0x1d7:
@@ -3617,12 +3539,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             if (b1 >= 2) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             sse_fn_epp = sse_op_table6[b].op[b1];
             if (!sse_fn_epp) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (!(s->cpuid_ext_features & sse_op_table6[b].ext_mask))
                 goto illegal_op;
@@ -3639,20 +3561,20 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
                     case 0x25: case 0x35: /* pmovsxdq, pmovzxdq */
                         gen_ldq_env_A0(s, op2_offset +
-                                        offsetof(XMMReg, XMM_Q(0)));
+                                        offsetof(ZMMReg, ZMM_Q(0)));
                         break;
                     case 0x21: case 0x31: /* pmovsxbd, pmovzxbd */
                     case 0x24: case 0x34: /* pmovsxwq, pmovzxwq */
                         tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
                                             s->mem_index, MO_LEUL);
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, op2_offset +
-                                        offsetof(XMMReg, XMM_L(0)));
+                                        offsetof(ZMMReg, ZMM_L(0)));
                         break;
                     case 0x22: case 0x32: /* pmovsxbq, pmovzxbq */
                         tcg_gen_qemu_ld_tl(cpu_tmp0, cpu_A0,
                                            s->mem_index, MO_LEUW);
                         tcg_gen_st16_tl(cpu_tmp0, cpu_env, op2_offset +
-                                        offsetof(XMMReg, XMM_W(0)));
+                                        offsetof(ZMMReg, ZMM_W(0)));
                         break;
                     case 0x2a:            /* movntqda */
                         gen_ldo_env_A0(s, op1_offset);
@@ -3672,7 +3594,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 }
             }
             if (sse_fn_epp == SSE_SPECIAL) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -3709,11 +3631,11 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
 
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[reg]);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
-                                 cpu_T[0], tcg_const_i32(8 << ot));
+                gen_helper_crc32(cpu_T0, cpu_tmp2_i32,
+                                 cpu_T0, tcg_const_i32(8 << ot));
 
                 ot = mo_64_32(s->dflag);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             case 0x1f0: /* crc32 or movbe */
@@ -3738,9 +3660,9 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
 
                 gen_lea_modrm(env, s, modrm);
                 if ((b & 1) == 0) {
-                    tcg_gen_qemu_ld_tl(cpu_T[0], cpu_A0,
+                    tcg_gen_qemu_ld_tl(cpu_T0, cpu_A0,
                                        s->mem_index, ot | MO_BE);
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                 } else {
                     tcg_gen_qemu_st_tl(cpu_regs[reg], cpu_A0,
                                        s->mem_index, ot | MO_BE);
@@ -3755,8 +3677,8 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 }
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_andc_tl(cpu_T[0], cpu_regs[s->vex_v], cpu_T[0]);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                tcg_gen_andc_tl(cpu_T0, cpu_regs[s->vex_v], cpu_T0);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 gen_op_update1_cc();
                 set_cc_op(s, CC_OP_LOGICB + ot);
                 break;
@@ -3775,12 +3697,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     /* Extract START, and shift the operand.
                        Shifts larger than operand size get zeros.  */
                     tcg_gen_ext8u_tl(cpu_A0, cpu_regs[s->vex_v]);
-                    tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_A0);
+                    tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_A0);
 
                     bound = tcg_const_tl(ot == MO_64 ? 63 : 31);
                     zero = tcg_const_tl(0);
-                    tcg_gen_movcond_tl(TCG_COND_LEU, cpu_T[0], cpu_A0, bound,
-                                       cpu_T[0], zero);
+                    tcg_gen_movcond_tl(TCG_COND_LEU, cpu_T0, cpu_A0, bound,
+                                       cpu_T0, zero);
                     tcg_temp_free(zero);
 
                     /* Extract the LEN into a mask.  Lengths larger than
@@ -3790,12 +3712,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     tcg_gen_movcond_tl(TCG_COND_LEU, cpu_A0, cpu_A0, bound,
                                        cpu_A0, bound);
                     tcg_temp_free(bound);
-                    tcg_gen_movi_tl(cpu_T[1], 1);
-                    tcg_gen_shl_tl(cpu_T[1], cpu_T[1], cpu_A0);
-                    tcg_gen_subi_tl(cpu_T[1], cpu_T[1], 1);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_movi_tl(cpu_T1, 1);
+                    tcg_gen_shl_tl(cpu_T1, cpu_T1, cpu_A0);
+                    tcg_gen_subi_tl(cpu_T1, cpu_T1, 1);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
 
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                     gen_op_update1_cc();
                     set_cc_op(s, CC_OP_LOGICB + ot);
                 }
@@ -3809,21 +3731,21 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 }
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_ext8u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                tcg_gen_ext8u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 {
                     TCGv bound = tcg_const_tl(ot == MO_64 ? 63 : 31);
                     /* Note that since we're using BMILG (in order to get O
                        cleared) we need to store the inverse into C.  */
                     tcg_gen_setcond_tl(TCG_COND_LT, cpu_cc_src,
-                                       cpu_T[1], bound);
-                    tcg_gen_movcond_tl(TCG_COND_GT, cpu_T[1], cpu_T[1],
-                                       bound, bound, cpu_T[1]);
+                                       cpu_T1, bound);
+                    tcg_gen_movcond_tl(TCG_COND_GT, cpu_T1, cpu_T1,
+                                       bound, bound, cpu_T1);
                     tcg_temp_free(bound);
                 }
                 tcg_gen_movi_tl(cpu_A0, -1);
-                tcg_gen_shl_tl(cpu_A0, cpu_A0, cpu_T[1]);
-                tcg_gen_andc_tl(cpu_T[0], cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                tcg_gen_shl_tl(cpu_A0, cpu_A0, cpu_T1);
+                tcg_gen_andc_tl(cpu_T0, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 gen_op_update1_cc();
                 set_cc_op(s, CC_OP_BMILGB + ot);
                 break;
@@ -3838,7 +3760,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 switch (ot) {
                 default:
-                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                     tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EDX]);
                     tcg_gen_mulu2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                       cpu_tmp2_i32, cpu_tmp3_i32);
@@ -3847,8 +3769,10 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     break;
 #ifdef TARGET_X86_64
                 case MO_64:
-                    tcg_gen_mulu2_i64(cpu_regs[s->vex_v], cpu_regs[reg],
-                                      cpu_T[0], cpu_regs[R_EDX]);
+                    tcg_gen_mulu2_i64(cpu_T0, cpu_T1,
+                                      cpu_T0, cpu_regs[R_EDX]);
+                    tcg_gen_mov_i64(cpu_regs[s->vex_v], cpu_T0);
+                    tcg_gen_mov_i64(cpu_regs[reg], cpu_T1);
                     break;
 #endif
                 }
@@ -3865,11 +3789,11 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 /* Note that by zero-extending the mask operand, we
                    automatically handle zero-extending the result.  */
                 if (ot == MO_64) {
-                    tcg_gen_mov_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_mov_tl(cpu_T1, cpu_regs[s->vex_v]);
                 } else {
-                    tcg_gen_ext32u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_ext32u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 }
-                gen_helper_pdep(cpu_regs[reg], cpu_T[0], cpu_T[1]);
+                gen_helper_pdep(cpu_regs[reg], cpu_T0, cpu_T1);
                 break;
 
             case 0x2f5: /* pext Gy, By, Ey */
@@ -3883,11 +3807,11 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 /* Note that by zero-extending the mask operand, we
                    automatically handle zero-extending the result.  */
                 if (ot == MO_64) {
-                    tcg_gen_mov_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_mov_tl(cpu_T1, cpu_regs[s->vex_v]);
                 } else {
-                    tcg_gen_ext32u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_ext32u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 }
-                gen_helper_pext(cpu_regs[reg], cpu_T[0], cpu_T[1]);
+                gen_helper_pext(cpu_regs[reg], cpu_T0, cpu_T1);
                 break;
 
             case 0x1f6: /* adcx Gy, Ey */
@@ -3946,22 +3870,22 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                         /* If we know TL is 64-bit, and we want a 32-bit
                            result, just do everything in 64-bit arithmetic.  */
                         tcg_gen_ext32u_i64(cpu_regs[reg], cpu_regs[reg]);
-                        tcg_gen_ext32u_i64(cpu_T[0], cpu_T[0]);
-                        tcg_gen_add_i64(cpu_T[0], cpu_T[0], cpu_regs[reg]);
-                        tcg_gen_add_i64(cpu_T[0], cpu_T[0], carry_in);
-                        tcg_gen_ext32u_i64(cpu_regs[reg], cpu_T[0]);
-                        tcg_gen_shri_i64(carry_out, cpu_T[0], 32);
+                        tcg_gen_ext32u_i64(cpu_T0, cpu_T0);
+                        tcg_gen_add_i64(cpu_T0, cpu_T0, cpu_regs[reg]);
+                        tcg_gen_add_i64(cpu_T0, cpu_T0, carry_in);
+                        tcg_gen_ext32u_i64(cpu_regs[reg], cpu_T0);
+                        tcg_gen_shri_i64(carry_out, cpu_T0, 32);
                         break;
 #endif
                     default:
                         /* Otherwise compute the carry-out in two steps.  */
                         zero = tcg_const_tl(0);
-                        tcg_gen_add2_tl(cpu_T[0], carry_out,
-                                        cpu_T[0], zero,
+                        tcg_gen_add2_tl(cpu_T0, carry_out,
+                                        cpu_T0, zero,
                                         carry_in, zero);
                         tcg_gen_add2_tl(cpu_regs[reg], carry_out,
                                         cpu_regs[reg], carry_out,
-                                        cpu_T[0], zero);
+                                        cpu_T0, zero);
                         tcg_temp_free(zero);
                         break;
                     }
@@ -3980,24 +3904,24 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 if (ot == MO_64) {
-                    tcg_gen_andi_tl(cpu_T[1], cpu_regs[s->vex_v], 63);
+                    tcg_gen_andi_tl(cpu_T1, cpu_regs[s->vex_v], 63);
                 } else {
-                    tcg_gen_andi_tl(cpu_T[1], cpu_regs[s->vex_v], 31);
+                    tcg_gen_andi_tl(cpu_T1, cpu_regs[s->vex_v], 31);
                 }
                 if (b == 0x1f7) {
-                    tcg_gen_shl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_shl_tl(cpu_T0, cpu_T0, cpu_T1);
                 } else if (b == 0x2f7) {
                     if (ot != MO_64) {
-                        tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
+                        tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
                     }
-                    tcg_gen_sar_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_sar_tl(cpu_T0, cpu_T0, cpu_T1);
                 } else {
                     if (ot != MO_64) {
-                        tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
+                        tcg_gen_ext32u_tl(cpu_T0, cpu_T0);
                     }
-                    tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_T1);
                 }
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             case 0x0f3:
@@ -4014,36 +3938,36 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
 
                 switch (reg & 7) {
                 case 1: /* blsr By,Ey */
-                    tcg_gen_neg_tl(cpu_T[1], cpu_T[0]);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                    gen_op_mov_reg_v(ot, s->vex_v, cpu_T[0]);
+                    tcg_gen_neg_tl(cpu_T1, cpu_T0);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
+                    gen_op_mov_reg_v(ot, s->vex_v, cpu_T0);
                     gen_op_update2_cc();
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 case 2: /* blsmsk By,Ey */
-                    tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
-                    tcg_gen_subi_tl(cpu_T[0], cpu_T[0], 1);
-                    tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_cc_src);
-                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+                    tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
+                    tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
+                    tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_cc_src);
+                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 case 3: /* blsi By, Ey */
-                    tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
-                    tcg_gen_subi_tl(cpu_T[0], cpu_T[0], 1);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_cc_src);
-                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+                    tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
+                    tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_cc_src);
+                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
 
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
             break;
 
@@ -4055,12 +3979,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             if (b1 >= 2) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             sse_fn_eppi = sse_op_table7[b].op[b1];
             if (!sse_fn_eppi) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (!(s->cpuid_ext_features & sse_op_table7[b].ext_mask))
                 goto illegal_op;
@@ -4074,22 +3998,22 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 val = cpu_ldub_code(env, s->pc++);
                 switch (b) {
                 case 0x14: /* pextrb */
-                    tcg_gen_ld8u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_B(val & 15)));
+                    tcg_gen_ld8u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_B(val & 15)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_UB);
                     }
                     break;
                 case 0x15: /* pextrw */
-                    tcg_gen_ld16u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_W(val & 7)));
+                    tcg_gen_ld16u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_W(val & 7)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_LEUW);
                     }
                     break;
@@ -4097,7 +4021,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     if (ot == MO_32) { /* pextrd */
                         tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(val & 3)));
+                                                xmm_regs[reg].ZMM_L(val & 3)));
                         if (mod == 3) {
                             tcg_gen_extu_i32_tl(cpu_regs[rm], cpu_tmp2_i32);
                         } else {
@@ -4108,7 +4032,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
 #ifdef TARGET_X86_64
                         tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_Q(val & 1)));
+                                                xmm_regs[reg].ZMM_Q(val & 1)));
                         if (mod == 3) {
                             tcg_gen_mov_i64(cpu_regs[rm], cpu_tmp1_i64);
                         } else {
@@ -4121,53 +4045,53 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                     }
                     break;
                 case 0x17: /* extractps */
-                    tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_L(val & 3)));
+                    tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_L(val & 3)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_LEUL);
                     }
                     break;
                 case 0x20: /* pinsrb */
                     if (mod == 3) {
-                        gen_op_mov_v_reg(MO_32, cpu_T[0], rm);
+                        gen_op_mov_v_reg(MO_32, cpu_T0, rm);
                     } else {
-                        tcg_gen_qemu_ld_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_ld_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_UB);
                     }
-                    tcg_gen_st8_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_B(val & 15)));
+                    tcg_gen_st8_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_B(val & 15)));
                     break;
                 case 0x21: /* insertps */
                     if (mod == 3) {
                         tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,xmm_regs[rm]
-                                                .XMM_L((val >> 6) & 3)));
+                                                .ZMM_L((val >> 6) & 3)));
                     } else {
                         tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
                                             s->mem_index, MO_LEUL);
                     }
                     tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                     offsetof(CPUX86State,xmm_regs[reg]
-                                            .XMM_L((val >> 4) & 3)));
+                                            .ZMM_L((val >> 4) & 3)));
                     if ((val >> 0) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(0)));
+                                                xmm_regs[reg].ZMM_L(0)));
                     if ((val >> 1) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(1)));
+                                                xmm_regs[reg].ZMM_L(1)));
                     if ((val >> 2) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(2)));
+                                                xmm_regs[reg].ZMM_L(2)));
                     if ((val >> 3) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(3)));
+                                                xmm_regs[reg].ZMM_L(3)));
                     break;
                 case 0x22:
                     if (ot == MO_32) { /* pinsrd */
@@ -4179,7 +4103,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                         }
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(val & 3)));
+                                                xmm_regs[reg].ZMM_L(val & 3)));
                     } else { /* pinsrq */
 #ifdef TARGET_X86_64
                         if (mod == 3) {
@@ -4190,7 +4114,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                         }
                         tcg_gen_st_i64(cpu_tmp1_i64, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_Q(val & 1)));
+                                                xmm_regs[reg].ZMM_Q(val & 1)));
 #else
                         goto illegal_op;
 #endif
@@ -4252,22 +4176,24 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 b = cpu_ldub_code(env, s->pc++);
                 if (ot == MO_64) {
-                    tcg_gen_rotri_tl(cpu_T[0], cpu_T[0], b & 63);
+                    tcg_gen_rotri_tl(cpu_T0, cpu_T0, b & 63);
                 } else {
-                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                     tcg_gen_rotri_i32(cpu_tmp2_i32, cpu_tmp2_i32, b & 31);
-                    tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+                    tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
                 }
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
             break;
 
         default:
-            goto illegal_op;
+        unknown_op:
+            gen_unknown_opcode(env, s);
+            return;
         }
     } else {
         /* generic MMX or SSE operation */
@@ -4313,13 +4239,13 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
                 switch (sz) {
                 case 2:
                     /* 32 bit access */
-                    gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                    tcg_gen_st32_tl(cpu_T[0], cpu_env,
-                                    offsetof(CPUX86State,xmm_t0.XMM_L(0)));
+                    gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                    tcg_gen_st32_tl(cpu_T0, cpu_env,
+                                    offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
                     break;
                 case 3:
                     /* 64 bit access */
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.XMM_D(0)));
+                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_D(0)));
                     break;
                 default:
                     /* 128 bit access */
@@ -4343,11 +4269,12 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
         }
         switch(b) {
         case 0x0f: /* 3DNow! data insns */
-            if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
-                goto illegal_op;
             val = cpu_ldub_code(env, s->pc++);
             sse_fn_epp = sse_op_table5[val];
             if (!sse_fn_epp) {
+                goto unknown_op;
+            }
+            if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
                 goto illegal_op;
             }
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -4367,7 +4294,7 @@ static void gen_sse(CPUX86State *env, DisasContext *s, int b,
             /* compare insns */
             val = cpu_ldub_code(env, s->pc++);
             if (val >= 8)
-                goto illegal_op;
+                goto unknown_op;
             sse_fn_epp = sse_op_table4[val][b1];
 
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -4412,10 +4339,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     target_ulong next_eip, tval;
     int rex_w, rex_r;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(pc_start);
-    }
-    s->pc = pc_start;
+    s->pc_start = s->pc = pc_start;
     prefixes = 0;
     s->override = -1;
     rex_w = -1;
@@ -4528,7 +4452,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                     b = 0x13a;
                     break;
                 default:   /* Reserved for future use.  */
-                    goto illegal_op;
+                    goto unknown_op;
                 }
             }
             s->vex_v = (~vex3 >> 3) & 0xf;
@@ -4608,13 +4532,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 xor_zero:
                     /* xor reg, reg optimisation */
                     set_cc_op(s, CC_OP_CLR);
-                    tcg_gen_movi_tl(cpu_T[0], 0);
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                     break;
                 } else {
                     opreg = rm;
                 }
-                gen_op_mov_v_reg(ot, cpu_T[1], reg);
+                gen_op_mov_v_reg(ot, cpu_T1, reg);
                 gen_op(s, op, ot, opreg);
                 break;
             case 1: /* OP Gv, Ev */
@@ -4624,17 +4548,17 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
                     gen_lea_modrm(env, s, modrm);
-                    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+                    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
                 } else if (op == OP_XORL && rm == reg) {
                     goto xor_zero;
                 } else {
-                    gen_op_mov_v_reg(ot, cpu_T[1], rm);
+                    gen_op_mov_v_reg(ot, cpu_T1, rm);
                 }
                 gen_op(s, op, ot, reg);
                 break;
             case 2: /* OP A, Iv */
                 val = insn_get(env, s, ot);
-                tcg_gen_movi_tl(cpu_T[1], val);
+                tcg_gen_movi_tl(cpu_T1, val);
                 gen_op(s, op, ot, OR_EAX);
                 break;
             }
@@ -4679,7 +4603,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 val = (int8_t)insn_get(env, s, MO_8);
                 break;
             }
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
             gen_op(s, op, ot, opreg);
         }
         break;
@@ -4706,32 +4630,32 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             if (op == 0)
                 s->rip_offset = insn_const_size(ot);
             gen_lea_modrm(env, s, modrm);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
 
         switch(op) {
         case 0: /* test */
             val = insn_get(env, s, ot);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
             gen_op_testl_T0_T1_cc();
             set_cc_op(s, CC_OP_LOGICB + ot);
             break;
         case 2: /* not */
-            tcg_gen_not_tl(cpu_T[0], cpu_T[0]);
+            tcg_gen_not_tl(cpu_T0, cpu_T0);
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
             break;
         case 3: /* neg */
-            tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
+            tcg_gen_neg_tl(cpu_T0, cpu_T0);
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
             gen_op_update_neg_cc();
             set_cc_op(s, CC_OP_SUBB + ot);
@@ -4739,32 +4663,32 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 4: /* mul */
             switch(ot) {
             case MO_8:
-                gen_op_mov_v_reg(MO_8, cpu_T[1], R_EAX);
-                tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext8u_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_8, cpu_T1, R_EAX);
+                tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext8u_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_andi_tl(cpu_cc_src, cpu_T[0], 0xff00);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_andi_tl(cpu_cc_src, cpu_T0, 0xff00);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
-                gen_op_mov_v_reg(MO_16, cpu_T[1], R_EAX);
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext16u_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_16, cpu_T1, R_EAX);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext16u_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
-                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, 16);
+                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
                 set_cc_op(s, CC_OP_MULW);
                 break;
             default:
             case MO_32:
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_mulu2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                   cpu_tmp2_i32, cpu_tmp3_i32);
@@ -4777,7 +4701,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
 #ifdef TARGET_X86_64
             case MO_64:
                 tcg_gen_mulu2_i64(cpu_regs[R_EAX], cpu_regs[R_EDX],
-                                  cpu_T[0], cpu_regs[R_EAX]);
+                                  cpu_T0, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_src, cpu_regs[R_EDX]);
                 set_cc_op(s, CC_OP_MULQ);
@@ -4788,34 +4712,34 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 5: /* imul */
             switch(ot) {
             case MO_8:
-                gen_op_mov_v_reg(MO_8, cpu_T[1], R_EAX);
-                tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext8s_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_8, cpu_T1, R_EAX);
+                tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext8s_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_ext8s_tl(cpu_tmp0, cpu_T[0]);
-                tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_ext8s_tl(cpu_tmp0, cpu_T0);
+                tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
-                gen_op_mov_v_reg(MO_16, cpu_T[1], R_EAX);
-                tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_16, cpu_T1, R_EAX);
+                tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext16s_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
-                tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
-                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_ext16s_tl(cpu_tmp0, cpu_T0);
+                tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, 16);
+                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
                 set_cc_op(s, CC_OP_MULW);
                 break;
             default:
             case MO_32:
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_muls2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                   cpu_tmp2_i32, cpu_tmp3_i32);
@@ -4830,7 +4754,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
 #ifdef TARGET_X86_64
             case MO_64:
                 tcg_gen_muls2_i64(cpu_regs[R_EAX], cpu_regs[R_EDX],
-                                  cpu_T[0], cpu_regs[R_EAX]);
+                                  cpu_T0, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[R_EAX]);
                 tcg_gen_sari_tl(cpu_cc_src, cpu_regs[R_EAX], 63);
                 tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_regs[R_EDX]);
@@ -4842,22 +4766,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 6: /* div */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divb_AL(cpu_env, cpu_T[0]);
+                gen_helper_divb_AL(cpu_env, cpu_T0);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divw_AX(cpu_env, cpu_T[0]);
+                gen_helper_divw_AX(cpu_env, cpu_T0);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divl_EAX(cpu_env, cpu_T[0]);
+                gen_helper_divl_EAX(cpu_env, cpu_T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divq_EAX(cpu_env, cpu_T[0]);
+                gen_helper_divq_EAX(cpu_env, cpu_T0);
                 break;
 #endif
             }
@@ -4865,28 +4785,24 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 7: /* idiv */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivb_AL(cpu_env, cpu_T[0]);
+                gen_helper_idivb_AL(cpu_env, cpu_T0);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivw_AX(cpu_env, cpu_T[0]);
+                gen_helper_idivw_AX(cpu_env, cpu_T0);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivl_EAX(cpu_env, cpu_T[0]);
+                gen_helper_idivl_EAX(cpu_env, cpu_T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivq_EAX(cpu_env, cpu_T[0]);
+                gen_helper_idivq_EAX(cpu_env, cpu_T0);
                 break;
 #endif
             }
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
 
@@ -4899,7 +4815,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (op >= 2 && b == 0xfe) {
-            goto illegal_op;
+            goto unknown_op;
         }
         if (CODE64(s)) {
             if (op == 2 || op == 4) {
@@ -4915,9 +4831,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (mod != 3) {
             gen_lea_modrm(env, s, modrm);
             if (op >= 2 && op != 3 && op != 5)
-                gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
 
         switch(op) {
@@ -4938,29 +4854,28 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 2: /* call Ev */
             /* XXX: optimize if memory (no 'and' is necessary) */
             if (dflag == MO_16) {
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
             }
             next_eip = s->pc - s->cs_base;
-            tcg_gen_movi_tl(cpu_T[1], next_eip);
-            gen_push_v(s, cpu_T[1]);
-            gen_op_jmp_v(cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T1, next_eip);
+            gen_push_v(s, cpu_T1);
+            gen_op_jmp_v(cpu_T0);
+            gen_bnd_jmp(s);
             gen_eob(s);
             break;
         case 3: /* lcall Ev */
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
             gen_add_A0_im(s, 1 << ot);
-            gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
         do_lcall:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_lcall_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_lcall_protected(cpu_env, cpu_tmp2_i32, cpu_T1,
                                            tcg_const_i32(dflag - 1),
-                                           tcg_const_i32(s->pc - pc_start));
+                                           tcg_const_tl(s->pc - s->cs_base));
             } else {
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_lcall_real(cpu_env, cpu_tmp2_i32, cpu_T[1],
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_lcall_real(cpu_env, cpu_tmp2_i32, cpu_T1,
                                       tcg_const_i32(dflag - 1),
                                       tcg_const_i32(s->pc - s->cs_base));
             }
@@ -4968,33 +4883,32 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             break;
         case 4: /* jmp Ev */
             if (dflag == MO_16) {
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
             }
-            gen_op_jmp_v(cpu_T[0]);
+            gen_op_jmp_v(cpu_T0);
+            gen_bnd_jmp(s);
             gen_eob(s);
             break;
         case 5: /* ljmp Ev */
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
             gen_add_A0_im(s, 1 << ot);
-            gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
         do_ljmp:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_ljmp_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
-                                          tcg_const_i32(s->pc - pc_start));
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_ljmp_protected(cpu_env, cpu_tmp2_i32, cpu_T1,
+                                          tcg_const_tl(s->pc - s->cs_base));
             } else {
                 gen_op_movl_seg_T0_vm(R_CS);
-                gen_op_jmp_v(cpu_T[1]);
+                gen_op_jmp_v(cpu_T1);
             }
             gen_eob(s);
             break;
         case 6: /* push Ev */
-            gen_push_v(s, cpu_T[0]);
+            gen_push_v(s, cpu_T0);
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
 
@@ -5006,7 +4920,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         reg = ((modrm >> 3) & 7) | rex_r;
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_op_mov_v_reg(ot, cpu_T[1], reg);
+        gen_op_mov_v_reg(ot, cpu_T1, reg);
         gen_op_testl_T0_T1_cc();
         set_cc_op(s, CC_OP_LOGICB + ot);
         break;
@@ -5016,8 +4930,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         ot = mo_b_d(b, dflag);
         val = insn_get(env, s, ot);
 
-        gen_op_mov_v_reg(ot, cpu_T[0], OR_EAX);
-        tcg_gen_movi_tl(cpu_T[1], val);
+        gen_op_mov_v_reg(ot, cpu_T0, OR_EAX);
+        tcg_gen_movi_tl(cpu_T1, val);
         gen_op_testl_T0_T1_cc();
         set_cc_op(s, CC_OP_LOGICB + ot);
         break;
@@ -5026,20 +4940,20 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         switch (dflag) {
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
-            tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_64, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
+            tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_64, R_EAX, cpu_T0);
             break;
 #endif
         case MO_32:
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EAX);
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_32, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_16, cpu_T0, R_EAX);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_32, R_EAX, cpu_T0);
             break;
         case MO_16:
-            gen_op_mov_v_reg(MO_8, cpu_T[0], R_EAX);
-            tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_8, cpu_T0, R_EAX);
+            tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
             break;
         default:
             tcg_abort();
@@ -5049,22 +4963,22 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         switch (dflag) {
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_op_mov_v_reg(MO_64, cpu_T[0], R_EAX);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 63);
-            gen_op_mov_reg_v(MO_64, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_64, cpu_T0, R_EAX);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 63);
+            gen_op_mov_reg_v(MO_64, R_EDX, cpu_T0);
             break;
 #endif
         case MO_32:
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
-            tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 31);
-            gen_op_mov_reg_v(MO_32, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
+            tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 31);
+            gen_op_mov_reg_v(MO_32, R_EDX, cpu_T0);
             break;
         case MO_16:
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EAX);
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 15);
-            gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_16, cpu_T0, R_EAX);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 15);
+            gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
             break;
         default:
             tcg_abort();
@@ -5083,25 +4997,25 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         if (b == 0x69) {
             val = insn_get(env, s, ot);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
         } else if (b == 0x6b) {
             val = (int8_t)insn_get(env, s, MO_8);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[1], reg);
+            gen_op_mov_v_reg(ot, cpu_T1, reg);
         }
         switch (ot) {
 #ifdef TARGET_X86_64
         case MO_64:
-            tcg_gen_muls2_i64(cpu_regs[reg], cpu_T[1], cpu_T[0], cpu_T[1]);
+            tcg_gen_muls2_i64(cpu_regs[reg], cpu_T1, cpu_T0, cpu_T1);
             tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[reg]);
             tcg_gen_sari_tl(cpu_cc_src, cpu_cc_dst, 63);
-            tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_T[1]);
+            tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_T1);
             break;
 #endif
         case MO_32:
-            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-            tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+            tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
             tcg_gen_muls2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                               cpu_tmp2_i32, cpu_tmp3_i32);
             tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32);
@@ -5111,14 +5025,14 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             tcg_gen_extu_i32_tl(cpu_cc_src, cpu_tmp2_i32);
             break;
         default:
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            tcg_gen_ext16s_tl(cpu_T1, cpu_T1);
             /* XXX: use 32 bit mul which could be faster */
-            tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-            tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
-            tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+            tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+            tcg_gen_ext16s_tl(cpu_tmp0, cpu_T0);
+            tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         }
         set_cc_op(s, CC_OP_MULB + ot);
@@ -5131,18 +5045,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
             rm = (modrm & 7) | REX_B(s);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_v_reg(ot, cpu_T[1], rm);
-            tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_v_reg(ot, cpu_T1, rm);
+            tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
             gen_lea_modrm(env, s, modrm);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
-            tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
+            tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         }
         gen_op_update2_cc();
         set_cc_op(s, CC_OP_ADDB + ot);
@@ -5212,8 +5126,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (dflag == MO_64) {
             if (!(s->cpuid_ext_features & CPUID_EXT_CX16))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg16b(cpu_env, cpu_A0);
         } else
@@ -5221,8 +5133,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         {
             if (!(s->cpuid_features & CPUID_CX8))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg8b(cpu_env, cpu_A0);
         }
@@ -5232,14 +5142,14 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         /**************************/
         /* push/pop */
     case 0x50 ... 0x57: /* push */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], (b & 7) | REX_B(s));
-        gen_push_v(s, cpu_T[0]);
+        gen_op_mov_v_reg(MO_32, cpu_T0, (b & 7) | REX_B(s));
+        gen_push_v(s, cpu_T0);
         break;
     case 0x58 ... 0x5f: /* pop */
         ot = gen_pop_T0(s);
         /* NOTE: order is important for pop %sp */
         gen_pop_update(s, ot);
-        gen_op_mov_reg_v(ot, (b & 7) | REX_B(s), cpu_T[0]);
+        gen_op_mov_reg_v(ot, (b & 7) | REX_B(s), cpu_T0);
         break;
     case 0x60: /* pusha */
         if (CODE64(s))
@@ -5258,8 +5168,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             val = insn_get(env, s, ot);
         else
             val = (int8_t)insn_get(env, s, MO_8);
-        tcg_gen_movi_tl(cpu_T[0], val);
-        gen_push_v(s, cpu_T[0]);
+        tcg_gen_movi_tl(cpu_T0, val);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x8f: /* pop Ev */
         modrm = cpu_ldub_code(env, s->pc++);
@@ -5269,7 +5179,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             /* NOTE: order is important for pop %sp */
             gen_pop_update(s, ot);
             rm = (modrm & 7) | REX_B(s);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
             /* NOTE: order is important too for MMU exceptions */
             s->popl_esp_hack = 1 << ot;
@@ -5288,20 +5198,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         }
         break;
     case 0xc9: /* leave */
-        /* XXX: exception not precise (ESP is updated before potential exception) */
-        if (CODE64(s)) {
-            gen_op_mov_v_reg(MO_64, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_64, R_ESP, cpu_T[0]);
-        } else if (s->ss32) {
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_32, R_ESP, cpu_T[0]);
-        } else {
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_16, R_ESP, cpu_T[0]);
-        }
-        ot = gen_pop_T0(s);
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[0]);
-        gen_pop_update(s, ot);
+        gen_leave(s);
         break;
     case 0x06: /* push es */
     case 0x0e: /* push cs */
@@ -5310,12 +5207,12 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (CODE64(s))
             goto illegal_op;
         gen_op_movl_T0_seg(b >> 3);
-        gen_push_v(s, cpu_T[0]);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x1a0: /* push fs */
     case 0x1a8: /* push gs */
         gen_op_movl_T0_seg((b >> 3) & 7);
-        gen_push_v(s, cpu_T[0]);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x07: /* pop es */
     case 0x17: /* pop ss */
@@ -5324,25 +5221,23 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             goto illegal_op;
         reg = b >> 3;
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, reg);
         gen_pop_update(s, ot);
-        if (reg == R_SS) {
-            /* if reg == SS, inhibit interrupts/trace. */
-            /* If several instructions disable interrupts, only the
-               _first_ does it */
-            if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                gen_helper_set_inhibit_irq(cpu_env);
-            s->tf = 0;
-        }
+        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
-            gen_eob(s);
+            if (reg == R_SS) {
+                s->tf = 0;
+                gen_eob_inhibit_irq(s, true);
+            } else {
+                gen_eob(s);
+            }
         }
         break;
     case 0x1a1: /* pop fs */
     case 0x1a9: /* pop gs */
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, (b >> 3) & 7, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, (b >> 3) & 7);
         gen_pop_update(s, ot);
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
@@ -5371,11 +5266,11 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             gen_lea_modrm(env, s, modrm);
         }
         val = insn_get(env, s, ot);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         if (mod != 3) {
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_reg_v(ot, (modrm & 7) | REX_B(s), cpu_T[0]);
+            gen_op_mov_reg_v(ot, (modrm & 7) | REX_B(s), cpu_T0);
         }
         break;
     case 0x8a:
@@ -5385,7 +5280,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         reg = ((modrm >> 3) & 7) | rex_r;
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
         break;
     case 0x8e: /* mov seg, Gv */
         modrm = cpu_ldub_code(env, s->pc++);
@@ -5393,18 +5288,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
         gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
-        if (reg == R_SS) {
-            /* if reg == SS, inhibit interrupts/trace */
-            /* If several instructions disable interrupts, only the
-               _first_ does it */
-            if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                gen_helper_set_inhibit_irq(cpu_env);
-            s->tf = 0;
-        }
+        gen_movl_seg_T0(s, reg);
+        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
-            gen_eob(s);
+            if (reg == R_SS) {
+                s->tf = 0;
+                gen_eob_inhibit_irq(s, true);
+            } else {
+                gen_eob(s);
+            }
         }
         break;
     case 0x8c: /* mov Gv, seg */
@@ -5439,45 +5332,42 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             rm = (modrm & 7) | REX_B(s);
 
             if (mod == 3) {
-                gen_op_mov_v_reg(ot, cpu_T[0], rm);
+                gen_op_mov_v_reg(ot, cpu_T0, rm);
                 switch (s_ot) {
                 case MO_UB:
-                    tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
                     break;
                 case MO_SB:
-                    tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
                     break;
                 case MO_UW:
-                    tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
                     break;
                 default:
                 case MO_SW:
-                    tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
                     break;
                 }
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             } else {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, s_ot, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_ld_v(s, s_ot, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             }
         }
         break;
 
     case 0x8d: /* lea */
-        ot = dflag;
         modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         reg = ((modrm >> 3) & 7) | rex_r;
-        /* we must ensure that no segment is added */
-        s->override = -1;
-        val = s->addseg;
-        s->addseg = 0;
-        gen_lea_modrm(env, s, modrm);
-        s->addseg = val;
-        gen_op_mov_reg_v(ot, reg, cpu_A0);
+        {
+            AddressParts a = gen_lea_modrm_0(env, s, modrm);
+            TCGv ea = gen_lea_modrm_1(a);
+            gen_op_mov_reg_v(dflag, reg, ea);
+        }
         break;
 
     case 0xa0: /* mov EAX, Ov */
@@ -5502,27 +5392,27 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             tcg_gen_movi_tl(cpu_A0, offset_addr);
             gen_add_A0_ds_seg(s);
             if ((b & 2) == 0) {
-                gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(ot, R_EAX, cpu_T[0]);
+                gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(ot, R_EAX, cpu_T0);
             } else {
-                gen_op_mov_v_reg(ot, cpu_T[0], R_EAX);
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_mov_v_reg(ot, cpu_T0, R_EAX);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             }
         }
         break;
     case 0xd7: /* xlat */
         tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EBX]);
-        tcg_gen_ext8u_tl(cpu_T[0], cpu_regs[R_EAX]);
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T[0]);
+        tcg_gen_ext8u_tl(cpu_T0, cpu_regs[R_EAX]);
+        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T0);
         gen_extu(s->aflag, cpu_A0);
         gen_add_A0_ds_seg(s);
-        gen_op_ld_v(s, MO_8, cpu_T[0], cpu_A0);
-        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T[0]);
+        gen_op_ld_v(s, MO_8, cpu_T0, cpu_A0);
+        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T0);
         break;
     case 0xb0 ... 0xb7: /* mov R, Ib */
         val = insn_get(env, s, MO_8);
-        tcg_gen_movi_tl(cpu_T[0], val);
-        gen_op_mov_reg_v(MO_8, (b & 7) | REX_B(s), cpu_T[0]);
+        tcg_gen_movi_tl(cpu_T0, val);
+        gen_op_mov_reg_v(MO_8, (b & 7) | REX_B(s), cpu_T0);
         break;
     case 0xb8 ... 0xbf: /* mov R, Iv */
 #ifdef TARGET_X86_64
@@ -5532,16 +5422,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             tmp = cpu_ldq_code(env, s->pc);
             s->pc += 8;
             reg = (b & 7) | REX_B(s);
-            tcg_gen_movi_tl(cpu_T[0], tmp);
-            gen_op_mov_reg_v(MO_64, reg, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, tmp);
+            gen_op_mov_reg_v(MO_64, reg, cpu_T0);
         } else
 #endif
         {
             ot = dflag;
             val = insn_get(env, s, ot);
             reg = (b & 7) | REX_B(s);
-            tcg_gen_movi_tl(cpu_T[0], val);
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, val);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
         break;
 
@@ -5560,21 +5450,21 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (mod == 3) {
             rm = (modrm & 7) | REX_B(s);
         do_xchg_reg:
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_v_reg(ot, cpu_T[1], rm);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_v_reg(ot, cpu_T1, rm);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         } else {
             gen_lea_modrm(env, s, modrm);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
             /* for xchg, lock is implicit */
             if (!(prefixes & PREFIX_LOCK))
                 gen_helper_lock();
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             if (!(prefixes & PREFIX_LOCK))
                 gen_helper_unlock();
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         }
         break;
     case 0xc4: /* les Gv */
@@ -5601,13 +5491,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (mod == 3)
             goto illegal_op;
         gen_lea_modrm(env, s, modrm);
-        gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
         gen_add_A0_im(s, 1 << ot);
         /* load the segment first to handle exceptions properly */
-        gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
-        gen_movl_seg_T0(s, op, pc_start - s->cs_base);
+        gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
+        gen_movl_seg_T0(s, op);
         /* then put the data */
-        gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+        gen_op_mov_reg_v(ot, reg, cpu_T1);
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
@@ -5686,7 +5576,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         } else {
             opreg = rm;
         }
-        gen_op_mov_v_reg(ot, cpu_T[1], reg);
+        gen_op_mov_v_reg(ot, cpu_T1, reg);
 
         if (shift) {
             TCGv imm = tcg_const_tl(cpu_ldub_code(env, s->pc++));
@@ -5837,8 +5727,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 }
                 break;
             case 0x0c: /* fldenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0d: /* fldcw mem */
@@ -5847,8 +5735,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 gen_helper_fldcw(cpu_env, cpu_tmp2_i32);
                 break;
             case 0x0e: /* fnstenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0f: /* fnstcw mem */
@@ -5857,24 +5743,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x1d: /* fldt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldt_ST0(cpu_env, cpu_A0);
                 break;
             case 0x1f: /* fstpt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstt_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
             case 0x2c: /* frstor mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_frstor(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2e: /* fnsave mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fsave(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2f: /* fnstsw mem */
@@ -5883,13 +5761,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x3c: /* fbld */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbld_ST0(cpu_env, cpu_A0);
                 break;
             case 0x3e: /* fbstp */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbst_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
@@ -5903,7 +5777,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 gen_helper_fpop(cpu_env);
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         } else {
             /* register float ops */
@@ -5924,12 +5798,10 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 switch(rm) {
                 case 0: /* fnop */
                     /* check exceptions (FreeBSD FPU probe) */
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
                     gen_helper_fwait(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x0c: /* grp d9/4 */
@@ -5948,7 +5820,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                     gen_helper_fxam_ST0(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x0d: /* grp d9/5 */
@@ -5983,7 +5855,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                         gen_helper_fldz_ST0(cpu_env);
                         break;
                     default:
-                        goto illegal_op;
+                        goto unknown_op;
                     }
                 }
                 break;
@@ -6083,7 +5955,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                     gen_helper_fpop(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x1c:
@@ -6101,7 +5973,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 case 4: /* fsetpm (287 only, just do nop here) */
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x1d: /* fucomi */
@@ -6153,7 +6025,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                     gen_helper_fpop(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x38: /* ffreep sti, undocumented op */
@@ -6164,11 +6036,11 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 switch(rm) {
                 case 0:
                     gen_helper_fnstsw(cpu_tmp2_i32, cpu_env);
-                    tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
-                    gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
+                    tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
+                    gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x3d: /* fucomip */
@@ -6214,7 +6086,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 }
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         }
         break;
@@ -6275,7 +6147,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0x6c: /* insS */
     case 0x6d:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base, 
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes) | 4);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
@@ -6290,7 +6162,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0x6e: /* outsS */
     case 0x6f:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes) | 4);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
@@ -6310,15 +6182,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0xe5:
         ot = mo_b_d32(b, dflag);
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
 	}
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
-        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
-        gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
+        gen_helper_in_func(ot, cpu_T1, cpu_tmp2_i32);
+        gen_op_mov_reg_v(ot, R_EAX, cpu_T1);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6328,17 +6201,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0xe7:
         ot = mo_b_d32(b, dflag);
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
-        gen_op_mov_v_reg(ot, cpu_T[1], R_EAX);
+        gen_op_mov_v_reg(ot, cpu_T1, R_EAX);
 
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
 	}
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6347,15 +6221,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0xec:
     case 0xed:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
 	}
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
-        gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        gen_helper_in_func(ot, cpu_T1, cpu_tmp2_i32);
+        gen_op_mov_reg_v(ot, R_EAX, cpu_T1);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6364,17 +6239,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0xee:
     case 0xef:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
-        gen_op_mov_v_reg(ot, cpu_T[1], R_EAX);
+        gen_op_mov_v_reg(ot, cpu_T1, R_EAX);
 
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
 	}
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
         if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
@@ -6389,14 +6265,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         ot = gen_pop_T0(s);
         gen_stack_update(s, val + (1 << ot));
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(cpu_T[0]);
+        gen_op_jmp_v(cpu_T0);
+        gen_bnd_jmp(s);
         gen_eob(s);
         break;
     case 0xc3: /* ret */
         ot = gen_pop_T0(s);
         gen_pop_update(s, ot);
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(cpu_T[0]);
+        gen_op_jmp_v(cpu_T0);
+        gen_bnd_jmp(s);
         gen_eob(s);
         break;
     case 0xca: /* lret im */
@@ -6411,13 +6289,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         } else {
             gen_stack_A0(s);
             /* pop offset */
-            gen_op_ld_v(s, dflag, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, dflag, cpu_T0, cpu_A0);
             /* NOTE: keeping EIP updated is not a problem in case of
                exception */
-            gen_op_jmp_v(cpu_T[0]);
+            gen_op_jmp_v(cpu_T0);
             /* pop selector */
-            gen_op_addl_A0_im(1 << dflag);
-            gen_op_ld_v(s, dflag, cpu_T[0], cpu_A0);
+            gen_add_A0_im(s, 1 << dflag);
+            gen_op_ld_v(s, dflag, cpu_T0, cpu_A0);
             gen_op_movl_seg_T0_vm(R_CS);
             /* add stack offset */
             gen_stack_update(s, val + (2 << dflag));
@@ -6441,8 +6319,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 set_cc_op(s, CC_OP_EFLAGS);
             }
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_iret_protected(cpu_env, tcg_const_i32(dflag - 1),
                                       tcg_const_i32(s->pc - s->cs_base));
             set_cc_op(s, CC_OP_EFLAGS);
@@ -6463,8 +6339,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else if (!CODE64(s)) {
                 tval &= 0xffffffff;
             }
-            tcg_gen_movi_tl(cpu_T[0], next_eip);
-            gen_push_v(s, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, next_eip);
+            gen_push_v(s, cpu_T0);
+            gen_bnd_jmp(s);
             gen_jmp(s, tval);
         }
         break;
@@ -6478,8 +6355,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             offset = insn_get(env, s, ot);
             selector = insn_get(env, s, MO_16);
 
-            tcg_gen_movi_tl(cpu_T[0], selector);
-            tcg_gen_movi_tl(cpu_T[1], offset);
+            tcg_gen_movi_tl(cpu_T0, selector);
+            tcg_gen_movi_tl(cpu_T1, offset);
         }
         goto do_lcall;
     case 0xe9: /* jmp im */
@@ -6494,6 +6371,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         } else if (!CODE64(s)) {
             tval &= 0xffffffff;
         }
+        gen_bnd_jmp(s);
         gen_jmp(s, tval);
         break;
     case 0xea: /* ljmp im */
@@ -6506,8 +6384,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             offset = insn_get(env, s, ot);
             selector = insn_get(env, s, MO_16);
 
-            tcg_gen_movi_tl(cpu_T[0], selector);
-            tcg_gen_movi_tl(cpu_T[1], offset);
+            tcg_gen_movi_tl(cpu_T0, selector);
+            tcg_gen_movi_tl(cpu_T1, offset);
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
@@ -6533,12 +6411,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (dflag == MO_16) {
             tval &= 0xffff;
         }
+        gen_bnd_jmp(s);
         gen_jcc(s, b, tval, next_eip);
         break;
 
     case 0x190 ... 0x19f: /* setcc Gv */
         modrm = cpu_ldub_code(env, s->pc++);
-        gen_setcc1(s, b, cpu_T[0]);
+        gen_setcc1(s, b, cpu_T0);
         gen_ldst_modrm(env, s, modrm, MO_8, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
@@ -6559,8 +6438,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
             gen_update_cc_op(s);
-            gen_helper_read_eflags(cpu_T[0], cpu_env);
-            gen_push_v(s, cpu_T[0]);
+            gen_helper_read_eflags(cpu_T0, cpu_env);
+            gen_push_v(s, cpu_T0);
         }
         break;
     case 0x9d: /* popf */
@@ -6571,13 +6450,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             ot = gen_pop_T0(s);
             if (s->cpl == 0) {
                 if (dflag != MO_16) {
-                    gen_helper_write_eflags(cpu_env, cpu_T[0],
+                    gen_helper_write_eflags(cpu_env, cpu_T0,
                                             tcg_const_i32((TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK |
                                                            IOPL_MASK)));
                 } else {
-                    gen_helper_write_eflags(cpu_env, cpu_T[0],
+                    gen_helper_write_eflags(cpu_env, cpu_T0,
                                             tcg_const_i32((TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK | IOPL_MASK)
@@ -6586,14 +6465,14 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else {
                 if (s->cpl <= s->iopl) {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                                 tcg_const_i32((TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
                                                                NT_MASK |
                                                                IF_MASK)));
                     } else {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                                 tcg_const_i32((TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
@@ -6603,11 +6482,11 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                     }
                 } else {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                            tcg_const_i32((TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)));
                     } else {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                            tcg_const_i32((TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)
                                                          & 0xffff));
@@ -6624,19 +6503,19 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     case 0x9e: /* sahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
-        gen_op_mov_v_reg(MO_8, cpu_T[0], R_AH);
+        gen_op_mov_v_reg(MO_8, cpu_T0, R_AH);
         gen_compute_eflags(s);
         tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O);
-        tcg_gen_andi_tl(cpu_T[0], cpu_T[0], CC_S | CC_Z | CC_A | CC_P | CC_C);
-        tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_T[0]);
+        tcg_gen_andi_tl(cpu_T0, cpu_T0, CC_S | CC_Z | CC_A | CC_P | CC_C);
+        tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_T0);
         break;
     case 0x9f: /* lahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
         gen_compute_eflags(s);
         /* Note: gen_compute_eflags() only gives the condition codes */
-        tcg_gen_ori_tl(cpu_T[0], cpu_cc_src, 0x02);
-        gen_op_mov_reg_v(MO_8, R_AH, cpu_T[0]);
+        tcg_gen_ori_tl(cpu_T0, cpu_cc_src, 0x02);
+        gen_op_mov_reg_v(MO_8, R_AH, cpu_T0);
         break;
     case 0xf5: /* cmc */
         gen_compute_eflags(s);
@@ -6670,15 +6549,15 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (mod != 3) {
             s->rip_offset = 1;
             gen_lea_modrm(env, s, modrm);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
         /* load shift */
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[1], val);
+        tcg_gen_movi_tl(cpu_T1, val);
         if (op < 4)
-            goto illegal_op;
+            goto unknown_op;
         op -= 4;
         goto bt_op;
     case 0x1a3: /* bt Gv, Ev */
@@ -6698,46 +6577,46 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
-        gen_op_mov_v_reg(MO_32, cpu_T[1], reg);
+        gen_op_mov_v_reg(MO_32, cpu_T1, reg);
         if (mod != 3) {
             gen_lea_modrm(env, s, modrm);
             /* specific case: we need to add a displacement */
-            gen_exts(ot, cpu_T[1]);
-            tcg_gen_sari_tl(cpu_tmp0, cpu_T[1], 3 + ot);
+            gen_exts(ot, cpu_T1);
+            tcg_gen_sari_tl(cpu_tmp0, cpu_T1, 3 + ot);
             tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, ot);
             tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
     bt_op:
-        tcg_gen_andi_tl(cpu_T[1], cpu_T[1], (1 << (3 + ot)) - 1);
-        tcg_gen_shr_tl(cpu_tmp4, cpu_T[0], cpu_T[1]);
+        tcg_gen_andi_tl(cpu_T1, cpu_T1, (1 << (3 + ot)) - 1);
+        tcg_gen_shr_tl(cpu_tmp4, cpu_T0, cpu_T1);
         switch(op) {
         case 0:
             break;
         case 1:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         case 2:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_andc_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_andc_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         default:
         case 3:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         }
         if (op != 0) {
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
         }
 
@@ -6777,7 +6656,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_extu(ot, cpu_T[0]);
+        gen_extu(ot, cpu_T0);
 
         /* Note that lzcnt and tzcnt are in different extensions.  */
         if ((prefixes & PREFIX_REPZ)
@@ -6785,18 +6664,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 ? s->cpuid_ext3_features & CPUID_EXT3_ABM
                 : s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)) {
             int size = 8 << ot;
-            tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
+            tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
             if (b & 1) {
                 /* For lzcnt, reduce the target_ulong result by the
                    number of zeros that we expect to find at the top.  */
-                gen_helper_clz(cpu_T[0], cpu_T[0]);
-                tcg_gen_subi_tl(cpu_T[0], cpu_T[0], TARGET_LONG_BITS - size);
+                gen_helper_clz(cpu_T0, cpu_T0);
+                tcg_gen_subi_tl(cpu_T0, cpu_T0, TARGET_LONG_BITS - size);
             } else {
                 /* For tzcnt, a zero input must return the operand size:
                    force all bits outside the operand size to 1.  */
                 target_ulong mask = (target_ulong)-2 << (size - 1);
-                tcg_gen_ori_tl(cpu_T[0], cpu_T[0], mask);
-                gen_helper_ctz(cpu_T[0], cpu_T[0]);
+                tcg_gen_ori_tl(cpu_T0, cpu_T0, mask);
+                gen_helper_ctz(cpu_T0, cpu_T0);
             }
             /* For lzcnt/tzcnt, C and Z bits are defined and are
                related to the result.  */
@@ -6805,24 +6684,24 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         } else {
             /* For bsr/bsf, only the Z bit is defined and it is related
                to the input and not the result.  */
-            tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+            tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
             set_cc_op(s, CC_OP_LOGICB + ot);
             if (b & 1) {
                 /* For bsr, return the bit index of the first 1 bit,
                    not the count of leading zeros.  */
-                gen_helper_clz(cpu_T[0], cpu_T[0]);
-                tcg_gen_xori_tl(cpu_T[0], cpu_T[0], TARGET_LONG_BITS - 1);
+                gen_helper_clz(cpu_T0, cpu_T0);
+                tcg_gen_xori_tl(cpu_T0, cpu_T0, TARGET_LONG_BITS - 1);
             } else {
-                gen_helper_ctz(cpu_T[0], cpu_T[0]);
+                gen_helper_ctz(cpu_T0, cpu_T0);
             }
             /* ??? The manual says that the output is undefined when the
                input is zero, but real hardware leaves it unchanged, and
                real programs appear to depend on that.  */
             tcg_gen_movi_tl(cpu_tmp0, 0);
-            tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T[0], cpu_cc_dst, cpu_tmp0,
-                               cpu_regs[reg], cpu_T[0]);
+            tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T0, cpu_cc_dst, cpu_tmp0,
+                               cpu_regs[reg], cpu_T0);
         }
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
         break;
         /************************/
         /* bcd */
@@ -6895,8 +6774,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             (HF_MP_MASK | HF_TS_MASK)) {
             gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_fwait(cpu_env);
         }
         break;
@@ -6946,27 +6823,13 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         }
         break;
     case 0xfb: /* sti */
-        if (!s->vm86) {
-            if (s->cpl <= s->iopl) {
-            gen_sti:
-                gen_helper_sti(cpu_env);
-                /* interruptions are enabled only the first insn after sti */
-                /* If several instructions disable interrupts, only the
-                   _first_ does it */
-                if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                    gen_helper_set_inhibit_irq(cpu_env);
-                /* give a chance to handle pending irqs */
-                gen_jmp_im(s->pc - s->cs_base);
-                gen_eob(s);
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
+        if (s->vm86 ? s->iopl == 3 : s->cpl <= s->iopl) {
+            gen_helper_sti(cpu_env);
+            /* interruptions are enabled only the first insn after sti */
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob_inhibit_irq(s, true);
         } else {
-            if (s->iopl == 3) {
-                goto gen_sti;
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
+            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         }
         break;
     case 0x62: /* bound */
@@ -6978,10 +6841,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_op_mov_v_reg(ot, cpu_T[0], reg);
+        gen_op_mov_v_reg(ot, cpu_T0, reg);
         gen_lea_modrm(env, s, modrm);
-        gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         if (ot == MO_16) {
             gen_helper_boundw(cpu_env, cpu_A0, cpu_tmp2_i32);
         } else {
@@ -6992,24 +6854,24 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         reg = (b & 7) | REX_B(s);
 #ifdef TARGET_X86_64
         if (dflag == MO_64) {
-            gen_op_mov_v_reg(MO_64, cpu_T[0], reg);
-            tcg_gen_bswap64_i64(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_64, reg, cpu_T[0]);
+            gen_op_mov_v_reg(MO_64, cpu_T0, reg);
+            tcg_gen_bswap64_i64(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_64, reg, cpu_T0);
         } else
 #endif
         {
-            gen_op_mov_v_reg(MO_32, cpu_T[0], reg);
-            tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_bswap32_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_32, reg, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, reg);
+            tcg_gen_ext32u_tl(cpu_T0, cpu_T0);
+            tcg_gen_bswap32_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_32, reg, cpu_T0);
         }
         break;
     case 0xd6: /* salc */
         if (CODE64(s))
             goto illegal_op;
-        gen_compute_eflags_c(s, cpu_T[0]);
-        tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
-        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T[0]);
+        gen_compute_eflags_c(s, cpu_T0);
+        tcg_gen_neg_tl(cpu_T0, cpu_T0);
+        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T0);
         break;
     case 0xe0: /* loopnz */
     case 0xe1: /* loopz */
@@ -7094,8 +6956,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysenter(cpu_env);
             gen_eob(s);
         }
@@ -7107,8 +6967,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysexit(cpu_env, tcg_const_i32(dflag - 1));
             gen_eob(s);
         }
@@ -7125,8 +6983,6 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysret(cpu_env, tcg_const_i32(dflag - 1));
             /* condition codes are modified only in long mode */
             if (s->lma) {
@@ -7160,7 +7016,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             if (!s->pe || s->vm86)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,ldt.selector));
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                             offsetof(CPUX86State, ldt.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
@@ -7172,8 +7029,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_lldt(cpu_env, cpu_tmp2_i32);
             }
             break;
@@ -7181,7 +7037,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             if (!s->pe || s->vm86)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,tr.selector));
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                             offsetof(CPUX86State, tr.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
@@ -7193,8 +7050,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_ltr(cpu_env, cpu_tmp2_i32);
             }
             break;
@@ -7205,279 +7061,347 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
             gen_update_cc_op(s);
             if (op == 4) {
-                gen_helper_verr(cpu_env, cpu_T[0]);
+                gen_helper_verr(cpu_env, cpu_T0);
             } else {
-                gen_helper_verw(cpu_env, cpu_T[0]);
+                gen_helper_verw(cpu_env, cpu_T0);
             }
             set_cc_op(s, CC_OP_EFLAGS);
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x101:
         modrm = cpu_ldub_code(env, s->pc++);
-        mod = (modrm >> 6) & 3;
-        op = (modrm >> 3) & 7;
-        rm = modrm & 7;
-        switch(op) {
-        case 0: /* sgdt */
-            if (mod == 3)
-                goto illegal_op;
+        switch (modrm) {
+        CASE_MODRM_MEM_OP(0): /* sgdt */
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
-            gen_op_st_v(s, MO_16, cpu_T[0], cpu_A0);
+            tcg_gen_ld32u_tl(cpu_T0,
+                             cpu_env, offsetof(CPUX86State, gdt.limit));
+            gen_op_st_v(s, MO_16, cpu_T0, cpu_A0);
             gen_add_A0_im(s, 2);
-            tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, gdt.base));
             if (dflag == MO_16) {
-                tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
             }
-            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
+            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
             break;
-        case 1:
-            if (mod == 3) {
-                switch (rm) {
-                case 0: /* monitor */
-                    if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) ||
-                        s->cpl != 0)
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EAX]);
-                    gen_extu(s->aflag, cpu_A0);
-                    gen_add_A0_ds_seg(s);
-                    gen_helper_monitor(cpu_env, cpu_A0);
-                    break;
-                case 1: /* mwait */
-                    if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) ||
-                        s->cpl != 0)
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
-                    gen_eob(s);
-                    break;
-                case 2: /* clac */
-                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
-                        s->cpl != 0) {
-                        goto illegal_op;
-                    }
-                    gen_helper_clac(cpu_env);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
-                    break;
-                case 3: /* stac */
-                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
-                        s->cpl != 0) {
-                        goto illegal_op;
-                    }
-                    gen_helper_stac(cpu_env);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
-                    break;
-                default:
-                    goto illegal_op;
-                }
-            } else { /* sidt */
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
-                gen_lea_modrm(env, s, modrm);
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
-                gen_op_st_v(s, MO_16, cpu_T[0], cpu_A0);
-                gen_add_A0_im(s, 2);
-                tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.base));
-                if (dflag == MO_16) {
-                    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
-                }
-                gen_op_st_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
+
+        case 0xc8: /* monitor */
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EAX]);
+            gen_extu(s->aflag, cpu_A0);
+            gen_add_A0_ds_seg(s);
+            gen_helper_monitor(cpu_env, cpu_A0);
+            break;
+
+        case 0xc9: /* mwait */
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+                goto illegal_op;
             }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
+            gen_eob(s);
             break;
-        case 2: /* lgdt */
-        case 3: /* lidt */
-            if (mod == 3) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                switch(rm) {
-                case 0: /* VMRUN */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1),
-                                         tcg_const_i32(s->pc - pc_start));
-                        tcg_gen_exit_tb(0);
-                        s->is_jmp = DISAS_TB_JUMP;
-                    }
-                    break;
-                case 1: /* VMMCALL */
-                    if (!(s->flags & HF_SVME_MASK))
-                        goto illegal_op;
-                    gen_helper_vmmcall(cpu_env);
-                    break;
-                case 2: /* VMLOAD */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                case 3: /* VMSAVE */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                case 4: /* STGI */
-                    if ((!(s->flags & HF_SVME_MASK) &&
-                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
-                        !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_stgi(cpu_env);
-                    }
-                    break;
-                case 5: /* CLGI */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_clgi(cpu_env);
-                    }
-                    break;
-                case 6: /* SKINIT */
-                    if ((!(s->flags & HF_SVME_MASK) && 
-                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
-                        !s->pe)
-                        goto illegal_op;
-                    gen_helper_skinit(cpu_env);
-                    break;
-                case 7: /* INVLPGA */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_invlpga(cpu_env,
-                                           tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                default:
-                    goto illegal_op;
-                }
-            } else if (s->cpl != 0) {
+
+        case 0xca: /* clac */
+            if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
+                || s->cpl != 0) {
+                goto illegal_op;
+            }
+            gen_helper_clac(cpu_env);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        case 0xcb: /* stac */
+            if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
+                || s->cpl != 0) {
+                goto illegal_op;
+            }
+            gen_helper_stac(cpu_env);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        CASE_MODRM_MEM_OP(1): /* sidt */
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.limit));
+            gen_op_st_v(s, MO_16, cpu_T0, cpu_A0);
+            gen_add_A0_im(s, 2);
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.base));
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            break;
+
+        case 0xd0: /* xgetbv */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                                 | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xgetbv(cpu_tmp1_i64, cpu_env, cpu_tmp2_i32);
+            tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], cpu_tmp1_i64);
+            break;
+
+        case 0xd1: /* xsetbv */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                                 | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_svm_check_intercept(s, pc_start,
-                                        op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
-                gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_16, cpu_T[1], cpu_A0);
-                gen_add_A0_im(s, 2);
-                gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
-                if (dflag == MO_16) {
-                    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
-                }
-                if (op == 2) {
-                    tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,gdt.base));
-                    tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,gdt.limit));
-                } else {
-                    tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,idt.base));
-                    tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,idt.limit));
-                }
+                break;
             }
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xsetbv(cpu_env, cpu_tmp2_i32, cpu_tmp1_i64);
+            /* End TB because translation flags may change.  */
+            gen_jmp_im(s->pc - pc_start);
+            gen_eob(s);
             break;
-        case 4: /* smsw */
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
-#if defined TARGET_X86_64 && defined HOST_WORDS_BIGENDIAN
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]) + 4);
-#else
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
-#endif
-            gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 1);
+
+        case 0xd8: /* VMRUN */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1),
+                             tcg_const_i32(s->pc - pc_start));
+            tcg_gen_exit_tb(0);
+            s->is_jmp = DISAS_TB_JUMP;
             break;
-        case 6: /* lmsw */
+
+        case 0xd9: /* VMMCALL */
+            if (!(s->flags & HF_SVME_MASK)) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmmcall(cpu_env);
+            break;
+
+        case 0xda: /* VMLOAD */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        case 0xdb: /* VMSAVE */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        case 0xdc: /* STGI */
+            if ((!(s->flags & HF_SVME_MASK)
+                   && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_stgi(cpu_env);
+            break;
+
+        case 0xdd: /* CLGI */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_clgi(cpu_env);
+            break;
+
+        case 0xde: /* SKINIT */
+            if ((!(s->flags & HF_SVME_MASK)
+                 && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !s->pe) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_skinit(cpu_env);
+            break;
+
+        case 0xdf: /* INVLPGA */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_invlpga(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        CASE_MODRM_MEM_OP(2): /* lgdt */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_WRITE);
+            gen_lea_modrm(env, s, modrm);
+            gen_op_ld_v(s, MO_16, cpu_T1, cpu_A0);
+            gen_add_A0_im(s, 2);
+            gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_st32_tl(cpu_T1, cpu_env, offsetof(CPUX86State, gdt.limit));
+            break;
+
+        CASE_MODRM_MEM_OP(3): /* lidt */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_WRITE);
+            gen_lea_modrm(env, s, modrm);
+            gen_op_ld_v(s, MO_16, cpu_T1, cpu_A0);
+            gen_add_A0_im(s, 2);
+            gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.base));
+            tcg_gen_st32_tl(cpu_T1, cpu_env, offsetof(CPUX86State, idt.limit));
+            break;
+
+        CASE_MODRM_OP(4): /* smsw */
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, cr[0]));
+            if (CODE64(s)) {
+                mod = (modrm >> 6) & 3;
+                ot = (mod != 3 ? MO_16 : s->dflag);
             } else {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-                gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_helper_lmsw(cpu_env, cpu_T[0]);
-                gen_jmp_im(s->pc - s->cs_base);
-                gen_eob(s);
+                ot = MO_16;
             }
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
-        case 7:
-            if (mod != 3) { /* invlpg */
+        case 0xee: /* rdpkru */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_rdpkru(cpu_tmp1_i64, cpu_env, cpu_tmp2_i32);
+            tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], cpu_tmp1_i64);
+            break;
+        case 0xef: /* wrpkru */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_wrpkru(cpu_env, cpu_tmp2_i32, cpu_tmp1_i64);
+            break;
+        CASE_MODRM_OP(6): /* lmsw */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
+            gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
+            gen_helper_lmsw(cpu_env, cpu_T0);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        CASE_MODRM_MEM_OP(7): /* invlpg */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_lea_modrm(env, s, modrm);
+            gen_helper_invlpg(cpu_env, cpu_A0);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        case 0xf8: /* swapgs */
+#ifdef TARGET_X86_64
+            if (CODE64(s)) {
                 if (s->cpl != 0) {
                     gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
                 } else {
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    gen_lea_modrm(env, s, modrm);
-                    gen_helper_invlpg(cpu_env, cpu_A0);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
+                    tcg_gen_mov_tl(cpu_T0, cpu_seg_base[R_GS]);
+                    tcg_gen_ld_tl(cpu_seg_base[R_GS], cpu_env,
+                                  offsetof(CPUX86State, kernelgsbase));
+                    tcg_gen_st_tl(cpu_T0, cpu_env,
+                                  offsetof(CPUX86State, kernelgsbase));
                 }
-            } else {
-                switch (rm) {
-                case 0: /* swapgs */
-#ifdef TARGET_X86_64
-                    if (CODE64(s)) {
-                        if (s->cpl != 0) {
-                            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        } else {
-                            tcg_gen_ld_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,segs[R_GS].base));
-                            tcg_gen_ld_tl(cpu_T[1], cpu_env,
-                                offsetof(CPUX86State,kernelgsbase));
-                            tcg_gen_st_tl(cpu_T[1], cpu_env,
-                                offsetof(CPUX86State,segs[R_GS].base));
-                            tcg_gen_st_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,kernelgsbase));
-                        }
-                    } else
+                break;
+            }
 #endif
-                    {
-                        goto illegal_op;
-                    }
-                    break;
-                case 1: /* rdtscp */
-                    if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP))
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    if (s->tb->cflags & CF_USE_ICOUNT) {
-                        gen_io_start();
-		    }
-                    gen_helper_rdtscp(cpu_env);
-                    if (s->tb->cflags & CF_USE_ICOUNT) {
-                        gen_io_end();
-                        gen_jmp(s, s->pc - s->cs_base);
-                    }
-                    break;
-                default:
-                    goto illegal_op;
-                }
+            goto illegal_op;
+
+        case 0xf9: /* rdtscp */
+            if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP)) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            if (s->tb->cflags & CF_USE_ICOUNT) {
+                gen_io_start();
+            }
+            gen_helper_rdtscp(cpu_env);
+            if (s->tb->cflags & CF_USE_ICOUNT) {
+                gen_io_end();
+                gen_jmp(s, s->pc - s->cs_base);
             }
             break;
+
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x108: /* invd */
     case 0x109: /* wbinvd */
         if (s->cpl != 0) {
@@ -7500,16 +7424,16 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             rm = (modrm & 7) | REX_B(s);
 
             if (mod == 3) {
-                gen_op_mov_v_reg(MO_32, cpu_T[0], rm);
+                gen_op_mov_v_reg(MO_32, cpu_T0, rm);
                 /* sign extend */
                 if (d_ot == MO_64) {
-                    tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
                 }
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             } else {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_32 | MO_SIGN, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_ld_v(s, MO_32 | MO_SIGN, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             }
         } else
 #endif
@@ -7574,9 +7498,9 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             t0 = tcg_temp_local_new();
             gen_update_cc_op(s);
             if (b == 0x102) {
-                gen_helper_lar(t0, cpu_env, cpu_T[0]);
+                gen_helper_lar(t0, cpu_env, cpu_T0);
             } else {
-                gen_helper_lsl(t0, cpu_env, cpu_T[0]);
+                gen_helper_lsl(t0, cpu_env, cpu_T0);
             }
             tcg_gen_andi_tl(cpu_tmp0, cpu_cc_src, CC_Z);
             label1 = gen_new_label();
@@ -7598,7 +7522,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         case 3: /* prefetchnt0 */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(env, s, modrm);
+            gen_nop_modrm(env, s, modrm);
             /* nothing more to do */
             break;
         default: /* nop (multi byte) */
@@ -7606,7 +7530,199 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
             break;
         }
         break;
-    case 0x119 ... 0x11f: /* nop (multi byte) */
+    case 0x11a:
+        modrm = cpu_ldub_code(env, s->pc++);
+        if (s->flags & HF_MPX_EN_MASK) {
+            mod = (modrm >> 6) & 3;
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (prefixes & PREFIX_REPZ) {
+                /* bndcl */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                gen_bndck(env, s, modrm, TCG_COND_LTU, cpu_bndl[reg]);
+            } else if (prefixes & PREFIX_REPNZ) {
+                /* bndcu */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                TCGv_i64 notu = tcg_temp_new_i64();
+                tcg_gen_not_i64(notu, cpu_bndu[reg]);
+                gen_bndck(env, s, modrm, TCG_COND_GTU, notu);
+                tcg_temp_free_i64(notu);
+            } else if (prefixes & PREFIX_DATA) {
+                /* bndmov -- from reg/mem */
+                if (reg >= 4 || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                if (mod == 3) {
+                    int reg2 = (modrm & 7) | REX_B(s);
+                    if (reg2 >= 4 || (prefixes & PREFIX_LOCK)) {
+                        goto illegal_op;
+                    }
+                    if (s->flags & HF_MPX_IU_MASK) {
+                        tcg_gen_mov_i64(cpu_bndl[reg], cpu_bndl[reg2]);
+                        tcg_gen_mov_i64(cpu_bndu[reg], cpu_bndu[reg2]);
+                    }
+                } else {
+                    gen_lea_modrm(env, s, modrm);
+                    if (CODE64(s)) {
+                        tcg_gen_qemu_ld_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 8);
+                        tcg_gen_qemu_ld_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                    } else {
+                        tcg_gen_qemu_ld_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 4);
+                        tcg_gen_qemu_ld_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                    }
+                    /* bnd registers are now in-use */
+                    gen_set_hflag(s, HF_MPX_IU_MASK);
+                }
+            } else if (mod != 3) {
+                /* bndldx */
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16
+                    || a.base < -1) {
+                    goto illegal_op;
+                }
+                if (a.base >= 0) {
+                    tcg_gen_addi_tl(cpu_A0, cpu_regs[a.base], a.disp);
+                } else {
+                    tcg_gen_movi_tl(cpu_A0, 0);
+                }
+                gen_lea_v_seg(s, s->aflag, cpu_A0, a.def_seg, s->override);
+                if (a.index >= 0) {
+                    tcg_gen_mov_tl(cpu_T0, cpu_regs[a.index]);
+                } else {
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                }
+                if (CODE64(s)) {
+                    gen_helper_bndldx64(cpu_bndl[reg], cpu_env, cpu_A0, cpu_T0);
+                    tcg_gen_ld_i64(cpu_bndu[reg], cpu_env,
+                                   offsetof(CPUX86State, mmx_t0.MMX_Q(0)));
+                } else {
+                    gen_helper_bndldx32(cpu_bndu[reg], cpu_env, cpu_A0, cpu_T0);
+                    tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndu[reg]);
+                    tcg_gen_shri_i64(cpu_bndu[reg], cpu_bndu[reg], 32);
+                }
+                gen_set_hflag(s, HF_MPX_IU_MASK);
+            }
+        }
+        gen_nop_modrm(env, s, modrm);
+        break;
+    case 0x11b:
+        modrm = cpu_ldub_code(env, s->pc++);
+        if (s->flags & HF_MPX_EN_MASK) {
+            mod = (modrm >> 6) & 3;
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (mod != 3 && (prefixes & PREFIX_REPZ)) {
+                /* bndmk */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (a.base >= 0) {
+                    tcg_gen_extu_tl_i64(cpu_bndl[reg], cpu_regs[a.base]);
+                    if (!CODE64(s)) {
+                        tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndl[reg]);
+                    }
+                } else if (a.base == -1) {
+                    /* no base register has lower bound of 0 */
+                    tcg_gen_movi_i64(cpu_bndl[reg], 0);
+                } else {
+                    /* rip-relative generates #ud */
+                    goto illegal_op;
+                }
+                tcg_gen_not_tl(cpu_A0, gen_lea_modrm_1(a));
+                if (!CODE64(s)) {
+                    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+                }
+                tcg_gen_extu_tl_i64(cpu_bndu[reg], cpu_A0);
+                /* bnd registers are now in-use */
+                gen_set_hflag(s, HF_MPX_IU_MASK);
+                break;
+            } else if (prefixes & PREFIX_REPNZ) {
+                /* bndcn */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                gen_bndck(env, s, modrm, TCG_COND_GTU, cpu_bndu[reg]);
+            } else if (prefixes & PREFIX_DATA) {
+                /* bndmov -- to reg/mem */
+                if (reg >= 4 || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                if (mod == 3) {
+                    int reg2 = (modrm & 7) | REX_B(s);
+                    if (reg2 >= 4 || (prefixes & PREFIX_LOCK)) {
+                        goto illegal_op;
+                    }
+                    if (s->flags & HF_MPX_IU_MASK) {
+                        tcg_gen_mov_i64(cpu_bndl[reg2], cpu_bndl[reg]);
+                        tcg_gen_mov_i64(cpu_bndu[reg2], cpu_bndu[reg]);
+                    }
+                } else {
+                    gen_lea_modrm(env, s, modrm);
+                    if (CODE64(s)) {
+                        tcg_gen_qemu_st_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 8);
+                        tcg_gen_qemu_st_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                    } else {
+                        tcg_gen_qemu_st_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 4);
+                        tcg_gen_qemu_st_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                    }
+                }
+            } else if (mod != 3) {
+                /* bndstx */
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16
+                    || a.base < -1) {
+                    goto illegal_op;
+                }
+                if (a.base >= 0) {
+                    tcg_gen_addi_tl(cpu_A0, cpu_regs[a.base], a.disp);
+                } else {
+                    tcg_gen_movi_tl(cpu_A0, 0);
+                }
+                gen_lea_v_seg(s, s->aflag, cpu_A0, a.def_seg, s->override);
+                if (a.index >= 0) {
+                    tcg_gen_mov_tl(cpu_T0, cpu_regs[a.index]);
+                } else {
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                }
+                if (CODE64(s)) {
+                    gen_helper_bndstx64(cpu_env, cpu_A0, cpu_T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
+                } else {
+                    gen_helper_bndstx32(cpu_env, cpu_A0, cpu_T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
+                }
+            }
+        }
+        gen_nop_modrm(env, s, modrm);
+        break;
+    case 0x119: case 0x11c ... 0x11f: /* nop (multi byte) */
         modrm = cpu_ldub_code(env, s->pc++);
         gen_nop_modrm(env, s, modrm);
         break;
@@ -7640,18 +7756,18 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 gen_update_cc_op(s);
                 gen_jmp_im(pc_start - s->cs_base);
                 if (b & 2) {
-                    gen_op_mov_v_reg(ot, cpu_T[0], rm);
+                    gen_op_mov_v_reg(ot, cpu_T0, rm);
                     gen_helper_write_crN(cpu_env, tcg_const_i32(reg),
-                                         cpu_T[0]);
+                                         cpu_T0);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
                 } else {
-                    gen_helper_read_crN(cpu_T[0], cpu_env, tcg_const_i32(reg));
-                    gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                    gen_helper_read_crN(cpu_T0, cpu_env, tcg_const_i32(reg));
+                    gen_op_mov_reg_v(ot, rm, cpu_T0);
                 }
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         }
         break;
@@ -7672,19 +7788,21 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
                 ot = MO_64;
             else
                 ot = MO_32;
-            /* XXX: do it dynamically with CR4.DE bit */
-            if (reg == 4 || reg == 5 || reg >= 8)
+            if (reg >= 8) {
                 goto illegal_op;
+            }
             if (b & 2) {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
-                gen_op_mov_v_reg(ot, cpu_T[0], rm);
-                gen_helper_movl_drN_T0(cpu_env, tcg_const_i32(reg), cpu_T[0]);
+                gen_op_mov_v_reg(ot, cpu_T0, rm);
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_set_dr(cpu_env, cpu_tmp2_i32, cpu_T0);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_DR0 + reg);
-                tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,dr[reg]));
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_get_dr(cpu_T0, cpu_env, cpu_tmp2_i32);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
         }
         break;
@@ -7714,83 +7832,195 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         break;
     case 0x1ae:
         modrm = cpu_ldub_code(env, s->pc++);
-        mod = (modrm >> 6) & 3;
-        op = (modrm >> 3) & 7;
-        switch(op) {
-        case 0: /* fxsave */
-            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR) ||
-                (s->prefix & PREFIX_LOCK))
+        switch (modrm) {
+        CASE_MODRM_MEM_OP(0): /* fxsave */
+            if (!(s->cpuid_features & CPUID_FXSR)
+                || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
+            }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_helper_fxsave(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
+            gen_helper_fxsave(cpu_env, cpu_A0);
             break;
-        case 1: /* fxrstor */
-            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR) ||
-                (s->prefix & PREFIX_LOCK))
+
+        CASE_MODRM_MEM_OP(1): /* fxrstor */
+            if (!(s->cpuid_features & CPUID_FXSR)
+                || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
+            }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_helper_fxrstor(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
+            gen_helper_fxrstor(cpu_env, cpu_A0);
             break;
-        case 2: /* ldmxcsr */
-        case 3: /* stmxcsr */
+
+        CASE_MODRM_MEM_OP(2): /* ldmxcsr */
+            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK)) {
+                goto illegal_op;
+            }
             if (s->flags & HF_TS_MASK) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK) ||
-                mod == 3)
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0, s->mem_index, MO_LEUL);
+            gen_helper_ldmxcsr(cpu_env, cpu_tmp2_i32);
+            break;
+
+        CASE_MODRM_MEM_OP(3): /* stmxcsr */
+            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK)) {
                 goto illegal_op;
+            }
+            if (s->flags & HF_TS_MASK) {
+                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                break;
+            }
             gen_lea_modrm(env, s, modrm);
-            if (op == 2) {
-                tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
-                                    s->mem_index, MO_LEUL);
-                gen_helper_ldmxcsr(cpu_env, cpu_tmp2_i32);
-            } else {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, mxcsr));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State, mxcsr));
+            gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
+            break;
+
+        CASE_MODRM_MEM_OP(4): /* xsave */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (prefixes & (PREFIX_LOCK | PREFIX_DATA
+                                | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
             }
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xsave(cpu_env, cpu_A0, cpu_tmp1_i64);
             break;
-        case 5: /* lfence */
-        case 6: /* mfence */
-            if ((modrm & 0xc7) != 0xc0 || !(s->cpuid_features & CPUID_SSE2))
+
+        CASE_MODRM_MEM_OP(5): /* xrstor */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (prefixes & (PREFIX_LOCK | PREFIX_DATA
+                                | PREFIX_REPZ | PREFIX_REPNZ))) {
                 goto illegal_op;
+            }
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xrstor(cpu_env, cpu_A0, cpu_tmp1_i64);
+            /* XRSTOR is how MPX is enabled, which changes how
+               we translate.  Thus we need to end the TB.  */
+            gen_update_cc_op(s);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
             break;
-        case 7: /* sfence / clflush */
-            if ((modrm & 0xc7) == 0xc0) {
-                /* sfence */
-                /* XXX: also check for cpuid_ext2_features & CPUID_EXT2_EMMX */
-                if (!(s->cpuid_features & CPUID_SSE))
+
+        CASE_MODRM_MEM_OP(6): /* xsaveopt / clwb */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            if (prefixes & PREFIX_DATA) {
+                /* clwb */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLWB)) {
                     goto illegal_op;
+                }
+                gen_nop_modrm(env, s, modrm);
             } else {
-                /* clflush */
-                if (!(s->cpuid_features & CPUID_CLFLUSH))
+                /* xsaveopt */
+                if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                    || (s->cpuid_xsave_features & CPUID_XSAVE_XSAVEOPT) == 0
+                    || (prefixes & (PREFIX_REPZ | PREFIX_REPNZ))) {
                     goto illegal_op;
+                }
                 gen_lea_modrm(env, s, modrm);
+                tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                      cpu_regs[R_EDX]);
+                gen_helper_xsaveopt(cpu_env, cpu_A0, cpu_tmp1_i64);
+            }
+            break;
+
+        CASE_MODRM_MEM_OP(7): /* clflush / clflushopt */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            if (prefixes & PREFIX_DATA) {
+                /* clflushopt */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLFLUSHOPT)) {
+                    goto illegal_op;
+                }
+            } else {
+                /* clflush */
+                if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ))
+                    || !(s->cpuid_features & CPUID_CLFLUSH)) {
+                    goto illegal_op;
+                }
+            }
+            gen_nop_modrm(env, s, modrm);
+            break;
+
+        case 0xc0 ... 0xc7: /* rdfsbase (f3 0f ae /0) */
+        case 0xc8 ... 0xc8: /* rdgsbase (f3 0f ae /1) */
+        case 0xd0 ... 0xd7: /* wrfsbase (f3 0f ae /2) */
+        case 0xd8 ... 0xd8: /* wrgsbase (f3 0f ae /3) */
+            if (CODE64(s)
+                && (prefixes & PREFIX_REPZ)
+                && !(prefixes & PREFIX_LOCK)
+                && (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_FSGSBASE)) {
+                TCGv base, treg, src, dst;
+
+                /* Preserve hflags bits by testing CR4 at runtime.  */
+                tcg_gen_movi_i32(cpu_tmp2_i32, CR4_FSGSBASE_MASK);
+                gen_helper_cr4_testbit(cpu_env, cpu_tmp2_i32);
+
+                base = cpu_seg_base[modrm & 8 ? R_GS : R_FS];
+                treg = cpu_regs[(modrm & 7) | REX_B(s)];
+
+                if (modrm & 0x10) {
+                    /* wr*base */
+                    dst = base, src = treg;
+                } else {
+                    /* rd*base */
+                    dst = treg, src = base;
+                }
+
+                if (s->dflag == MO_32) {
+                    tcg_gen_ext32u_tl(dst, src);
+                } else {
+                    tcg_gen_mov_tl(dst, src);
+                }
+                break;
+            }
+            goto unknown_op;
+
+        case 0xf8: /* sfence / pcommit */
+            if (prefixes & PREFIX_DATA) {
+                /* pcommit */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_PCOMMIT)
+                    || (prefixes & PREFIX_LOCK)) {
+                    goto illegal_op;
+                }
+                break;
+            }
+            /* fallthru */
+        case 0xf9 ... 0xff: /* sfence */
+        case 0xe8 ... 0xef: /* lfence */
+        case 0xf0 ... 0xf7: /* mfence */
+            if (!(s->cpuid_features & CPUID_SSE2)
+                || (prefixes & PREFIX_LOCK)) {
+                goto illegal_op;
             }
             break;
+
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x10d: /* 3DNow! prefetch(w) */
         modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(env, s, modrm);
-        /* ignore for now */
+        gen_nop_modrm(env, s, modrm);
         break;
     case 0x1aa: /* rsm */
         gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM);
@@ -7818,8 +8048,8 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         }
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_helper_popcnt(cpu_T[0], cpu_env, cpu_T[0], tcg_const_i32(ot));
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_helper_popcnt(cpu_T0, cpu_env, cpu_T0, tcg_const_i32(ot));
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
 
         set_cc_op(s, CC_OP_EFLAGS);
         break;
@@ -7837,7 +8067,7 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         gen_sse(env, s, b, pc_start, rex_r);
         break;
     default:
-        goto illegal_op;
+        goto unknown_op;
     }
     /* lock generation */
     if (s->prefix & PREFIX_LOCK)
@@ -7847,11 +8077,17 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
     if (s->prefix & PREFIX_LOCK)
         gen_helper_unlock();
     /* XXX: ensure that no lock was generated */
-    gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
+    gen_illegal_opcode(s);
+    return s->pc;
+ unknown_op:
+    if (s->prefix & PREFIX_LOCK)
+        gen_helper_unlock();
+    /* XXX: ensure that no lock was generated */
+    gen_unknown_opcode(env, s);
     return s->pc;
 }
 
-void optimize_flags_init(void)
+void tcg_x86_init(void)
 {
     static const char reg_names[CPU_NB_REGS][4] = {
 #ifdef TARGET_X86_64
@@ -7882,38 +8118,66 @@ void optimize_flags_init(void)
         [R_ESP] = "esp",
 #endif
     };
+    static const char seg_base_names[6][8] = {
+        [R_CS] = "cs_base",
+        [R_DS] = "ds_base",
+        [R_ES] = "es_base",
+        [R_FS] = "fs_base",
+        [R_GS] = "gs_base",
+        [R_SS] = "ss_base",
+    };
+    static const char bnd_regl_names[4][8] = {
+        "bnd0_lb", "bnd1_lb", "bnd2_lb", "bnd3_lb"
+    };
+    static const char bnd_regu_names[4][8] = {
+        "bnd0_ub", "bnd1_ub", "bnd2_ub", "bnd3_ub"
+    };
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_cc_op = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUX86State, cc_op), "cc_op");
-    cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_dst),
+    cpu_cc_dst = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_dst),
                                     "cc_dst");
-    cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_src),
+    cpu_cc_src = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src),
                                     "cc_src");
-    cpu_cc_src2 = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_src2),
+    cpu_cc_src2 = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src2),
                                      "cc_src2");
 
     for (i = 0; i < CPU_NB_REGS; ++i) {
-        cpu_regs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_regs[i] = tcg_global_mem_new(cpu_env,
                                          offsetof(CPUX86State, regs[i]),
                                          reg_names[i]);
     }
+
+    for (i = 0; i < 6; ++i) {
+        cpu_seg_base[i]
+            = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUX86State, segs[i].base),
+                                 seg_base_names[i]);
+    }
+
+    for (i = 0; i < 4; ++i) {
+        cpu_bndl[i]
+            = tcg_global_mem_new_i64(cpu_env,
+                                     offsetof(CPUX86State, bnd_regs[i].lb),
+                                     bnd_regl_names[i]);
+        cpu_bndu[i]
+            = tcg_global_mem_new_i64(cpu_env,
+                                     offsetof(CPUX86State, bnd_regs[i].ub),
+                                     bnd_regu_names[i]);
+    }
+
+    helper_lock_init();
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(X86CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUX86State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
     target_ulong pc_ptr;
-    CPUBreakpoint *bp;
-    int j, lj;
     uint64_t flags;
     target_ulong pc_start;
     target_ulong cs_base;
@@ -7943,13 +8207,14 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
     /* select memory access functions */
     dc->mem_index = 0;
     if (flags & HF_SOFTMMU_MASK) {
-        dc->mem_index = cpu_mmu_index(env);
+	dc->mem_index = cpu_mmu_index(env, false);
     }
     dc->cpuid_features = env->features[FEAT_1_EDX];
     dc->cpuid_ext_features = env->features[FEAT_1_ECX];
     dc->cpuid_ext2_features = env->features[FEAT_8000_0001_EDX];
     dc->cpuid_ext3_features = env->features[FEAT_8000_0001_ECX];
     dc->cpuid_7_0_ebx_features = env->features[FEAT_7_0_EBX];
+    dc->cpuid_xsave_features = env->features[FEAT_XSAVE];
 #ifdef TARGET_X86_64
     dc->lma = (flags >> HF_LMA_SHIFT) & 1;
     dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
@@ -7978,8 +8243,8 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
         printf("ERROR addseg\n");
 #endif
 
-    cpu_T[0] = tcg_temp_new();
-    cpu_T[1] = tcg_temp_new();
+    cpu_T0 = tcg_temp_new();
+    cpu_T1 = tcg_temp_new();
     cpu_A0 = tcg_temp_new();
 
     cpu_tmp0 = tcg_temp_new();
@@ -7993,40 +8258,37 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
 
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     for(;;) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == pc_ptr &&
-                    !((bp->flags & BP_CPU) && (tb->flags & HF_RF_MASK))) {
-                    gen_debug(dc, pc_ptr - dc->cs_base);
-                    goto done_generating;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = pc_ptr;
-            gen_opc_cc_op[lj] = dc->cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        tcg_gen_insn_start(pc_ptr, dc->cc_op);
+        num_insns++;
+
+        /* If RF is set, suppress an internally generated breakpoint.  */
+        if (unlikely(cpu_breakpoint_test(cs, pc_ptr,
+                                         tb->flags & HF_RF_MASK
+                                         ? BP_GDB : BP_ANY))) {
+            gen_debug(dc, pc_ptr - dc->cs_base);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            pc_ptr += 1;
+            goto done_generating;
+        }
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         pc_ptr = disas_insn(env, dc, pc_ptr);
-        num_insns++;
         /* stop translation if indicated */
         if (dc->is_jmp)
             break;
@@ -8074,14 +8336,6 @@ static inline void gen_intermediate_code_internal(X86CPU *cpu,
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    /* we don't forget to fill the last values */
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    }
-
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int disas_flags;
@@ -8098,42 +8352,16 @@ done_generating:
     }
 #endif
 
-    if (!search_pc) {
-        tb->size = pc_ptr - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = pc_ptr - pc_start;
+    tb->icount = num_insns;
 }
 
-void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
+void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUX86State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos)
-{
-    int cc_op;
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OP)) {
-        int i;
-        qemu_log("RESTORE:\n");
-        for(i = 0;i <= pc_pos; i++) {
-            if (tcg_ctx.gen_opc_instr_start[i]) {
-                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i,
-                        tcg_ctx.gen_opc_pc[i]);
-            }
-        }
-        qemu_log("pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
-                pc_pos, tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base,
-                (uint32_t)tb->cs_base);
-    }
-#endif
-    env->eip = tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base;
-    cc_op = gen_opc_cc_op[pc_pos];
-    if (cc_op != CC_OP_DYNAMIC)
+    int cc_op = data[1];
+    env->eip = data[0] - tb->cs_base;
+    if (cc_op != CC_OP_DYNAMIC) {
         env->cc_op = cc_op;
+    }
 }
diff --git a/qemu/target-lm32/cpu.c b/qemu/target-lm32/cpu.c
index c2b77c698..6e7e1b8e6 100644
--- a/qemu/target-lm32/cpu.c
+++ b/qemu/target-lm32/cpu.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 
@@ -131,6 +133,12 @@ static void lm32_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void lm32_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_lm32;
+    info->print_insn = print_insn_lm32;
+}
+
 static void lm32_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -275,6 +283,14 @@ static void lm32_cpu_class_init(ObjectClass *oc, void *data)
     cc->gdb_num_core_regs = 32 + 7;
     cc->gdb_stop_before_watchpoint = true;
     cc->debug_excp_handler = lm32_debug_excp_handler;
+    cc->disas_set_info = lm32_cpu_disas_set_info;
+
+    /*
+     * Reason: lm32_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void lm32_register_cpu_type(const LM32CPUInfo *info)
diff --git a/qemu/target-lm32/cpu.h b/qemu/target-lm32/cpu.h
index 944777d05..f220fc0bb 100644
--- a/qemu/target-lm32/cpu.h
+++ b/qemu/target-lm32/cpu.h
@@ -24,17 +24,14 @@
 
 #define CPUArchState struct CPULM32State
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 struct CPULM32State;
 typedef struct CPULM32State CPULM32State;
 
-#define ELF_MACHINE EM_LATTICEMICO32
-
 #define NB_MMU_MODES 1
 #define TARGET_PAGE_BITS 12
-static inline int cpu_mmu_index(CPULM32State *env)
+static inline int cpu_mmu_index(CPULM32State *env, bool ifetch)
 {
     return 0;
 }
@@ -221,7 +218,6 @@ bool lm32_cpu_do_semihosting(CPUState *cs);
 
 #define cpu_list lm32_cpu_list
 #define cpu_exec cpu_lm32_exec
-#define cpu_gen_code cpu_lm32_gen_code
 #define cpu_signal_handler cpu_lm32_signal_handler
 
 int lm32_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
diff --git a/qemu/target-lm32/gdbstub.c b/qemu/target-lm32/gdbstub.c
index 4979a98d7..8ac1288bb 100644
--- a/qemu/target-lm32/gdbstub.c
+++ b/qemu/target-lm32/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "hw/lm32/lm32_pic.h"
diff --git a/qemu/target-lm32/helper.c b/qemu/target-lm32/helper.c
index a88aa5ad1..655248f81 100644
--- a/qemu/target-lm32/helper.c
+++ b/qemu/target-lm32/helper.c
@@ -17,10 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "exec/semihost.h"
+#include "exec/log.h"
 
 int lm32_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
                               int mmu_idx)
@@ -81,7 +83,7 @@ void lm32_watchpoint_insert(CPULM32State *env, int idx, target_ulong address,
 
     switch (wp_type) {
     case LM32_WP_DISABLED:
-        /* nothing to to */
+        /* nothing to do */
         break;
     case LM32_WP_READ:
         flags = BP_CPU | BP_STOP_BEFORE_ACCESS | BP_MEM_READ;
diff --git a/qemu/target-lm32/lm32-semi.c b/qemu/target-lm32/lm32-semi.c
index ec6524f37..20f1a1cd4 100644
--- a/qemu/target-lm32/lm32-semi.c
+++ b/qemu/target-lm32/lm32-semi.c
@@ -10,10 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <stddef.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/log.h"
diff --git a/qemu/target-lm32/machine.c b/qemu/target-lm32/machine.c
index 8327c6d97..91c943d19 100644
--- a/qemu/target-lm32/machine.c
+++ b/qemu/target-lm32/machine.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 
diff --git a/qemu/target-lm32/op_helper.c b/qemu/target-lm32/op_helper.c
index 61209c19b..b6759e022 100644
--- a/qemu/target-lm32/op_helper.c
+++ b/qemu/target-lm32/op_helper.c
@@ -1,4 +1,4 @@
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
diff --git a/qemu/target-lm32/translate.c b/qemu/target-lm32/translate.c
index cf7042e3e..256a51f84 100644
--- a/qemu/target-lm32/translate.c
+++ b/qemu/target-lm32/translate.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "exec/helper-proto.h"
@@ -28,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define DISAS_LM32 1
@@ -42,7 +44,7 @@
 
 #define MEM_INDEX 0
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[32];
 static TCGv cpu_pc;
 static TCGv cpu_ie;
@@ -1005,10 +1007,6 @@ static const DecoderInfo decinfo[] = {
 
 static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1036,32 +1034,13 @@ static inline void decode(DisasContext *dc, uint32_t ir)
     decinfo[dc->opcode](dc);
 }
 
-static void check_breakpoint(CPULM32State *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(lm32_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline
-void gen_intermediate_code_internal(LM32CPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
 {
+    LM32CPU *cpu = lm32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPULM32State *env = &cpu->env;
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, lj;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1083,41 +1062,41 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
         decode(dc, cpu_ldl_code(env, dc->pc));
         dc->pc += 4;
-        num_insns++;
-
     } while (!dc->is_jmp
          && !tcg_op_buf_full()
          && !cs->singlestep_enabled
@@ -1154,16 +1133,8 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1175,16 +1146,6 @@ void gen_intermediate_code_internal(LM32CPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, true);
-}
-
 void lm32_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -1219,9 +1180,10 @@ void lm32_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "\n\n");
 }
 
-void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
 
 void lm32_translate_init(void)
@@ -1231,48 +1193,48 @@ void lm32_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < ARRAY_SIZE(cpu_R); i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, regs[i]),
                           regnames[i]);
     }
 
     for (i = 0; i < ARRAY_SIZE(cpu_bp); i++) {
-        cpu_bp[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_bp[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, bp[i]),
                           regnames[32+i]);
     }
 
     for (i = 0; i < ARRAY_SIZE(cpu_wp); i++) {
-        cpu_wp[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_wp[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, wp[i]),
                           regnames[36+i]);
     }
 
-    cpu_pc = tcg_global_mem_new(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, pc),
                     "pc");
-    cpu_ie = tcg_global_mem_new(TCG_AREG0,
+    cpu_ie = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, ie),
                     "ie");
-    cpu_icc = tcg_global_mem_new(TCG_AREG0,
+    cpu_icc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, icc),
                     "icc");
-    cpu_dcc = tcg_global_mem_new(TCG_AREG0,
+    cpu_dcc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, dcc),
                     "dcc");
-    cpu_cc = tcg_global_mem_new(TCG_AREG0,
+    cpu_cc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, cc),
                     "cc");
-    cpu_cfg = tcg_global_mem_new(TCG_AREG0,
+    cpu_cfg = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, cfg),
                     "cfg");
-    cpu_eba = tcg_global_mem_new(TCG_AREG0,
+    cpu_eba = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, eba),
                     "eba");
-    cpu_dc = tcg_global_mem_new(TCG_AREG0,
+    cpu_dc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, dc),
                     "dc");
-    cpu_deba = tcg_global_mem_new(TCG_AREG0,
+    cpu_deba = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, deba),
                     "deba");
 }
diff --git a/qemu/target-m68k/cpu.c b/qemu/target-m68k/cpu.c
index 4f246da74..0b5f9a581 100644
--- a/qemu/target-m68k/cpu.c
+++ b/qemu/target-m68k/cpu.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -61,6 +63,11 @@ static void m68k_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void m68k_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_m68k;
+}
+
 /* CPU models */
 
 static ObjectClass *m68k_cpu_class_by_name(const char *cpu_model)
@@ -208,10 +215,19 @@ static void m68k_cpu_class_init(ObjectClass *c, void *data)
 #endif
     cc->cpu_exec_enter = m68k_cpu_exec_enter;
     cc->cpu_exec_exit = m68k_cpu_exec_exit;
+    cc->disas_set_info = m68k_cpu_disas_set_info;
 
-    dc->vmsd = &vmstate_m68k_cpu;
     cc->gdb_num_core_regs = 18;
     cc->gdb_core_xml_file = "cf-core.xml";
+
+    dc->vmsd = &vmstate_m68k_cpu;
+
+    /*
+     * Reason: m68k_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void register_cpu_type(const M68kCPUInfo *info)
diff --git a/qemu/target-m68k/cpu.h b/qemu/target-m68k/cpu.h
index 9a62f6cb5..48b4c872f 100644
--- a/qemu/target-m68k/cpu.h
+++ b/qemu/target-m68k/cpu.h
@@ -24,7 +24,6 @@
 
 #define CPUArchState struct CPUM68KState
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 
@@ -32,8 +31,6 @@
 
 #define MAX_QREGS 32
 
-#define ELF_MACHINE	EM_68K
-
 #define EXCP_ACCESS         2   /* Access (MMU) error.  */
 #define EXCP_ADDRESS        3   /* Address error.  */
 #define EXCP_ILLEGAL        4   /* Illegal instruction.  */
@@ -215,7 +212,6 @@ void register_m68k_insns (CPUM68KState *env);
 #define cpu_init(cpu_model) CPU(cpu_m68k_init(cpu_model))
 
 #define cpu_exec cpu_m68k_exec
-#define cpu_gen_code cpu_m68k_gen_code
 #define cpu_signal_handler cpu_m68k_signal_handler
 #define cpu_list m68k_cpu_list
 
@@ -223,7 +219,7 @@ void register_m68k_insns (CPUM68KState *env);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUM68KState *env)
+static inline int cpu_mmu_index (CPUM68KState *env, bool ifetch)
 {
     return (env->sr & SR_S) == 0 ? 1 : 0;
 }
diff --git a/qemu/target-m68k/gdbstub.c b/qemu/target-m68k/gdbstub.c
index ae8179c01..f02bb5caf 100644
--- a/qemu/target-m68k/gdbstub.c
+++ b/qemu/target-m68k/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-m68k/helper.c b/qemu/target-m68k/helper.c
index 77225a200..a8f6d9d6a 100644
--- a/qemu/target-m68k/helper.c
+++ b/qemu/target-m68k/helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-m68k/m68k-semi.c b/qemu/target-m68k/m68k-semi.c
index 9dffe8de6..f360ef3e1 100644
--- a/qemu/target-m68k/m68k-semi.c
+++ b/qemu/target-m68k/m68k-semi.c
@@ -17,15 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #if defined(CONFIG_USER_ONLY)
diff --git a/qemu/target-m68k/op_helper.c b/qemu/target-m68k/op_helper.c
index 1af0ca647..17d0a1191 100644
--- a/qemu/target-m68k/op_helper.c
+++ b/qemu/target-m68k/op_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
diff --git a/qemu/target-m68k/translate.c b/qemu/target-m68k/translate.c
index a57d2415c..7560c3a80 100644
--- a/qemu/target-m68k/translate.c
+++ b/qemu/target-m68k/translate.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -28,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 //#define DEBUG_DISPATCH 1
@@ -48,7 +50,7 @@
 static TCGv_i32 cpu_halted;
 static TCGv_i32 cpu_exception_index;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 static char cpu_reg_names[3*8*3 + 5*4];
 static TCGv cpu_dregs[8];
@@ -74,48 +76,52 @@ void m68k_tcg_init(void)
     char *p;
     int i;
 
-#define DEFO32(name,  offset) QREG_##name = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUM68KState, offset), #name);
-#define DEFO64(name,  offset) QREG_##name = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUM68KState, offset), #name);
-#define DEFF64(name,  offset) DEFO64(name, offset)
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+
+#define DEFO32(name, offset) \
+    QREG_##name = tcg_global_mem_new_i32(cpu_env, \
+        offsetof(CPUM68KState, offset), #name);
+#define DEFO64(name, offset) \
+    QREG_##name = tcg_global_mem_new_i64(cpu_env, \
+        offsetof(CPUM68KState, offset), #name);
+#define DEFF64(name, offset) DEFO64(name, offset)
 #include "qregs.def"
 #undef DEFO32
 #undef DEFO64
 #undef DEFF64
 
-    cpu_halted = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_halted = tcg_global_mem_new_i32(cpu_env,
                                         -offsetof(M68kCPU, env) +
                                         offsetof(CPUState, halted), "HALTED");
-    cpu_exception_index = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_exception_index = tcg_global_mem_new_i32(cpu_env,
                                                  -offsetof(M68kCPU, env) +
                                                  offsetof(CPUState, exception_index),
                                                  "EXCEPTION");
 
-    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-
     p = cpu_reg_names;
     for (i = 0; i < 8; i++) {
         sprintf(p, "D%d", i);
-        cpu_dregs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_dregs[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUM68KState, dregs[i]), p);
         p += 3;
         sprintf(p, "A%d", i);
-        cpu_aregs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_aregs[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUM68KState, aregs[i]), p);
         p += 3;
         sprintf(p, "F%d", i);
-        cpu_fregs[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fregs[i] = tcg_global_mem_new_i64(cpu_env,
                                           offsetof(CPUM68KState, fregs[i]), p);
         p += 3;
     }
     for (i = 0; i < 4; i++) {
         sprintf(p, "ACC%d", i);
-        cpu_macc[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_macc[i] = tcg_global_mem_new_i64(cpu_env,
                                          offsetof(CPUM68KState, macc[i]), p);
         p += 5;
     }
 
-    NULL_QREG = tcg_global_mem_new(TCG_AREG0, -4, "NULL");
-    store_dummy = tcg_global_mem_new(TCG_AREG0, -8, "NULL");
+    NULL_QREG = tcg_global_mem_new(cpu_env, -4, "NULL");
+    store_dummy = tcg_global_mem_new(cpu_env, -8, "NULL");
 }
 
 /* internal defines */
@@ -2680,7 +2686,7 @@ DISAS_INSN(from_mac)
     if (s->env->macsr & MACSR_FI) {
         gen_helper_get_macf(rx, cpu_env, acc);
     } else if ((s->env->macsr & MACSR_OMC) == 0) {
-        tcg_gen_trunc_i64_i32(rx, acc);
+        tcg_gen_extrl_i64_i32(rx, acc);
     } else if (s->env->macsr & MACSR_SU) {
         gen_helper_get_macs(rx, acc);
     } else {
@@ -2955,10 +2961,6 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 {
     uint16_t insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
@@ -2966,15 +2968,11 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
 {
+    M68kCPU *cpu = m68k_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUM68KState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     int pc_offset;
     int num_insns;
@@ -2993,43 +2991,39 @@ gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
     dc->fpcr = env->fpcr;
     dc->user = (env->sr & SR_S) == 0;
     dc->done_mac = 0;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
         pc_offset = dc->pc - pc_start;
         gen_throws_exception = NULL;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_exception(dc, dc->pc, EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    break;
-                }
-            }
-            if (dc->is_jmp)
-                break;
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_exception(dc, dc->pc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
+
         dc->insn_pc = dc->pc;
 	disas_m68k_insn(env, dc);
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -3073,28 +3067,8 @@ gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-
-    //optimize_flags();
-    //expand_target_qops();
-}
-
-void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 void m68k_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
@@ -3120,7 +3094,8 @@ void m68k_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf (f, "FPRESULT = %12g\n", *(double *)&env->fp_result);
 }
 
-void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-microblaze/cpu.c b/qemu/target-microblaze/cpu.c
index 9ac509af3..fdfb01917 100644
--- a/qemu/target-microblaze/cpu.c
+++ b/qemu/target-microblaze/cpu.c
@@ -21,6 +21,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "hw/qdev-properties.h"
@@ -107,6 +109,8 @@ static void mb_cpu_reset(CPUState *s)
     /* Disable stack protector.  */
     env->shr = ~0;
 
+    env->sregs[SR_PC] = cpu->cfg.base_vectors;
+
 #if defined(CONFIG_USER_ONLY)
     /* start in user mode with interrupts enabled.  */
     env->sregs[SR_MSR] = MSR_EE | MSR_IE | MSR_VM | MSR_UM;
@@ -183,8 +187,6 @@ static void mb_cpu_realizefn(DeviceState *dev, Error **errp)
     env->pvr.regs[10] = 0x0c000000; /* Default to spartan 3a dsp family.  */
     env->pvr.regs[11] = PVR11_USE_MMU | (16 << 17);
 
-    env->sregs[SR_PC] = cpu->cfg.base_vectors;
-
     mcc->parent_realize(dev, errp);
 }
 
@@ -264,6 +266,12 @@ static void mb_cpu_class_init(ObjectClass *oc, void *data)
     cc->gdb_num_core_regs = 32 + 5;
 
     cc->disas_set_info = mb_disas_set_info;
+
+    /*
+     * Reason: mb_cpu_initfn() calls cpu_exec_init(), which saves the
+     * object in cpus -> dangling pointer after final object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo mb_cpu_type_info = {
diff --git a/qemu/target-microblaze/cpu.h b/qemu/target-microblaze/cpu.h
index 7e20e59b8..2f7335eaa 100644
--- a/qemu/target-microblaze/cpu.h
+++ b/qemu/target-microblaze/cpu.h
@@ -19,7 +19,6 @@
 #ifndef CPU_MICROBLAZE_H
 #define CPU_MICROBLAZE_H
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 32
@@ -34,8 +33,6 @@ typedef struct CPUMBState CPUMBState;
 #include "mmu.h"
 #endif
 
-#define ELF_MACHINE	EM_MICROBLAZE
-
 #define EXCP_MMU        1
 #define EXCP_IRQ        2
 #define EXCP_BREAK      3
@@ -297,7 +294,6 @@ int cpu_mb_signal_handler(int host_signum, void *pinfo,
 #define cpu_init(cpu_model) CPU(cpu_mb_init(cpu_model))
 
 #define cpu_exec cpu_mb_exec
-#define cpu_gen_code cpu_mb_gen_code
 #define cpu_signal_handler cpu_mb_signal_handler
 
 /* MMU modes definitions */
@@ -309,7 +305,7 @@ int cpu_mb_signal_handler(int host_signum, void *pinfo,
 #define MMU_USER_IDX    2
 /* See NB_MMU_MODES further up the file.  */
 
-static inline int cpu_mmu_index (CPUMBState *env)
+static inline int cpu_mmu_index (CPUMBState *env, bool ifetch)
 {
         /* Are we in nommu mode?.  */
         if (!(env->sregs[SR_MSR] & MSR_VM))
diff --git a/qemu/target-microblaze/gdbstub.c b/qemu/target-microblaze/gdbstub.c
index a70e2ee3c..89d38980b 100644
--- a/qemu/target-microblaze/gdbstub.c
+++ b/qemu/target-microblaze/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-microblaze/helper.c b/qemu/target-microblaze/helper.c
index 8257b0e0f..4de6bdbf8 100644
--- a/qemu/target-microblaze/helper.c
+++ b/qemu/target-microblaze/helper.c
@@ -18,8 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
+#include "exec/log.h"
 
 #define D(x)
 
@@ -128,7 +130,7 @@ void mb_cpu_do_interrupt(CPUState *cs)
     switch (cs->exception_index) {
         case EXCP_HW_EXCP:
             if (!(env->pvr.regs[0] & PVR0_USE_EXC_MASK)) {
-                qemu_log("Exception raised on system without exceptions!\n");
+                qemu_log_mask(LOG_GUEST_ERROR, "Exception raised on system without exceptions!\n");
                 return;
             }
 
diff --git a/qemu/target-microblaze/mmu.c b/qemu/target-microblaze/mmu.c
index 728da133d..4ac304035 100644
--- a/qemu/target-microblaze/mmu.c
+++ b/qemu/target-microblaze/mmu.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 
 #define D(x)
@@ -60,7 +61,7 @@ static void mmu_change_pid(CPUMBState *env, unsigned int newpid)
     uint32_t t;
 
     if (newpid & ~0xff)
-        qemu_log("Illegal rpid=%x\n", newpid);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal rpid=%x\n", newpid);
 
     for (i = 0; i < ARRAY_SIZE(mmu->rams[RAM_TAG]); i++) {
         /* Lookup and decode.  */
@@ -121,7 +122,7 @@ unsigned int mmu_translate(struct microblaze_mmu *mmu,
             t0 &= 0x3;
 
             if (tlb_zsel > mmu->c_mmu_zones) {
-                qemu_log("tlb zone select out of range! %d\n", tlb_zsel);
+                qemu_log_mask(LOG_GUEST_ERROR, "tlb zone select out of range! %d\n", tlb_zsel);
                 t0 = 1; /* Ignore.  */
             }
 
@@ -183,7 +184,7 @@ uint32_t mmu_read(CPUMBState *env, uint32_t rn)
     uint32_t r;
 
     if (env->mmu.c_mmu < 2 || !env->mmu.c_mmu_tlb_access) {
-        qemu_log("MMU access on MMU-less system\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "MMU access on MMU-less system\n");
         return 0;
     }
 
@@ -192,7 +193,7 @@ uint32_t mmu_read(CPUMBState *env, uint32_t rn)
         case MMU_R_TLBLO:
         case MMU_R_TLBHI:
             if (!(env->mmu.c_mmu_tlb_access & 1)) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return 0;
             }
 
@@ -204,7 +205,7 @@ uint32_t mmu_read(CPUMBState *env, uint32_t rn)
         case MMU_R_PID:
         case MMU_R_ZPR:
             if (!(env->mmu.c_mmu_tlb_access & 1)) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return 0;
             }
             r = env->mmu.regs[rn];
@@ -224,7 +225,7 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
     D(qemu_log("%s rn=%d=%x old=%x\n", __func__, rn, v, env->mmu.regs[rn]));
 
     if (env->mmu.c_mmu < 2 || !env->mmu.c_mmu_tlb_access) {
-        qemu_log("MMU access on MMU-less system\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "MMU access on MMU-less system\n");
         return;
     }
 
@@ -235,7 +236,7 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
             i = env->mmu.regs[MMU_R_TLBX] & 0xff;
             if (rn == MMU_R_TLBHI) {
                 if (i < 3 && !(v & TLB_VALID) && qemu_loglevel_mask(~0))
-                    qemu_log("invalidating index %x at pc=%x\n",
+                    qemu_log_mask(LOG_GUEST_ERROR, "invalidating index %x at pc=%x\n",
                              i, env->sregs[SR_PC]);
                 env->mmu.tids[i] = env->mmu.regs[MMU_R_PID] & 0xff;
                 mmu_flush_idx(env, i);
@@ -246,7 +247,7 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
             break;
         case MMU_R_ZPR:
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
@@ -259,7 +260,7 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
             break;
         case MMU_R_PID:
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
@@ -274,12 +275,12 @@ void mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
             int hit;
 
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
             hit = mmu_translate(&env->mmu, &lu,
-                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env));
+                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env, false));
             if (hit) {
                 env->mmu.regs[MMU_R_TLBX] = lu.idx;
             } else
diff --git a/qemu/target-microblaze/op_helper.c b/qemu/target-microblaze/op_helper.c
index d2b362451..97333881f 100644
--- a/qemu/target-microblaze/op_helper.c
+++ b/qemu/target-microblaze/op_helper.c
@@ -18,7 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -56,7 +56,7 @@ void helper_put(uint32_t id, uint32_t ctrl, uint32_t data)
     int nonblock = ctrl & STREAM_NONBLOCK;
     int exception = ctrl & STREAM_EXCEPTION;
 
-    qemu_log("Unhandled stream put to stream-id=%d data=%x %s%s%s%s%s\n",
+    qemu_log_mask(LOG_UNIMP, "Unhandled stream put to stream-id=%d data=%x %s%s%s%s%s\n",
              id, data,
              test ? "t" : "",
              nonblock ? "n" : "",
@@ -73,7 +73,7 @@ uint32_t helper_get(uint32_t id, uint32_t ctrl)
     int nonblock = ctrl & STREAM_NONBLOCK;
     int exception = ctrl & STREAM_EXCEPTION;
 
-    qemu_log("Unhandled stream get from stream-id=%d %s%s%s%s%s\n",
+    qemu_log_mask(LOG_UNIMP, "Unhandled stream get from stream-id=%d %s%s%s%s%s\n",
              id,
              test ? "t" : "",
              nonblock ? "n" : "",
@@ -151,9 +151,7 @@ uint32_t helper_clz(uint32_t t0)
 
 uint32_t helper_carry(uint32_t a, uint32_t b, uint32_t cf)
 {
-    uint32_t ncf;
-    ncf = compute_carry(a, b, cf);
-    return ncf;
+    return compute_carry(a, b, cf);
 }
 
 static inline int div_prepare(CPUMBState *env, uint32_t a, uint32_t b)
@@ -468,8 +466,8 @@ void helper_memalign(CPUMBState *env, uint32_t addr, uint32_t dr, uint32_t wr,
 void helper_stackprot(CPUMBState *env, uint32_t addr)
 {
     if (addr < env->slr || addr > env->shr) {
-        qemu_log("Stack protector violation at %x %x %x\n",
-                 addr, env->slr, env->shr);
+        qemu_log_mask(CPU_LOG_INT, "Stack protector violation at %x %x %x\n",
+                      addr, env->slr, env->shr);
         env->sregs[SR_EAR] = addr;
         env->sregs[SR_ESR] = ESR_EC_STACKPROT;
         helper_raise_exception(env, EXCP_HW_EXCP);
diff --git a/qemu/target-microblaze/translate.c b/qemu/target-microblaze/translate.c
index f4e969b29..f944965a1 100644
--- a/qemu/target-microblaze/translate.c
+++ b/qemu/target-microblaze/translate.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -27,6 +28,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define SIM_COMPAT 0
@@ -44,7 +46,7 @@
             (((src) >> start) & ((1 << (end - start + 1)) - 1))
 
 static TCGv env_debug;
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[32];
 static TCGv cpu_SR[18];
 static TCGv env_imm;
@@ -313,7 +315,6 @@ static void dec_sub(DisasContext *dc)
 static void dec_pattern(DisasContext *dc)
 {
     unsigned int mode;
-    TCGLabel *l1;
 
     if ((dc->tb_flags & MSR_EE_FLAG)
           && (dc->cpu->env.pvr.regs[2] & PVR2_ILL_OPCODE_EXC_MASK)
@@ -333,29 +334,15 @@ static void dec_pattern(DisasContext *dc)
         case 2:
             LOG_DIS("pcmpeq r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                l1 = gen_new_label();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_EQ,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_EQ, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         case 3:
             LOG_DIS("pcmpne r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
-            l1 = gen_new_label();
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_NE,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_NE, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         default:
@@ -433,7 +420,7 @@ static void dec_msr(DisasContext *dc)
     CPUState *cs = CPU(dc->cpu);
     TCGv t0, t1;
     unsigned int sr, to, rn;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     sr = dc->imm & ((1 << 14) - 1);
     to = dc->imm & (1 << 14);
@@ -598,9 +585,9 @@ static void t_gen_muls(TCGv d, TCGv d2, TCGv a, TCGv b)
     tcg_gen_ext_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -618,9 +605,9 @@ static void t_gen_mulu(TCGv d, TCGv d2, TCGv a, TCGv b)
     tcg_gen_extu_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -745,7 +732,7 @@ static void dec_bit(DisasContext *dc)
     CPUState *cs = CPU(dc->cpu);
     TCGv t0;
     unsigned int op;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     op = dc->ir & ((1 << 9) - 1);
     switch (op) {
@@ -1009,7 +996,7 @@ static void dec_load(DisasContext *dc)
      * address and if that succeeds we write into the destination reg.
      */
     v = tcg_temp_new();
-    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
@@ -1027,7 +1014,7 @@ static void dec_load(DisasContext *dc)
     tcg_temp_free(v);
 
     if (ex) { /* lwx */
-        /* no support for for AXI exclusive so always clear C */
+        /* no support for AXI exclusive so always clear C */
         write_carryi(dc, 0);
     }
 
@@ -1087,7 +1074,7 @@ static void dec_store(DisasContext *dc)
            this compare and the following write to be atomic. For user
            emulation we need to add atomicity between threads.  */
         tval = tcg_temp_new();
-        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env),
+        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env, false),
                            MO_TEUL);
         tcg_gen_brcond_tl(TCG_COND_NE, env_res_val, tval, swx_skip);
         write_carryi(dc, 0);
@@ -1138,7 +1125,7 @@ static void dec_store(DisasContext *dc)
                 break;
         }
     }
-    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     /* Verify alignment if needed.  */
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
@@ -1234,7 +1221,7 @@ static void dec_bcc(DisasContext *dc)
 static void dec_br(DisasContext *dc)
 {
     unsigned int dslot, link, abs, mbar;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     dslot = dc->ir & (1 << 20);
     abs = dc->ir & (1 << 19);
@@ -1366,7 +1353,7 @@ static inline void do_rte(DisasContext *dc)
 static void dec_rts(DisasContext *dc)
 {
     unsigned int b_bit, i_bit, e_bit;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     i_bit = dc->ir & (1 << 21);
     b_bit = dc->ir & (1 << 22);
@@ -1531,14 +1518,14 @@ static void dec_null(DisasContext *dc)
         t_gen_raise_exception(dc, EXCP_HW_EXCP);
         return;
     }
-    qemu_log ("unknown insn pc=%x opc=%x\n", dc->pc, dc->opcode);
+    qemu_log_mask(LOG_GUEST_ERROR, "unknown insn pc=%x opc=%x\n", dc->pc, dc->opcode);
     dc->abort_at_next_insn = 1;
 }
 
 /* Insns connected to FSL or AXI stream attached devices.  */
 static void dec_stream(DisasContext *dc)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
     TCGv_i32 t_id, t_ctrl;
     int ctrl;
 
@@ -1603,10 +1590,6 @@ static inline void decode(DisasContext *dc, uint32_t ir)
 {
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1645,30 +1628,12 @@ static inline void decode(DisasContext *dc, uint32_t ir)
     }
 }
 
-static void check_breakpoint(CPUMBState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(mb_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMBState *env, struct TranslationBlock *tb)
 {
+    MicroBlazeCPU *cpu = mb_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMBState *env = &cpu->env;
     uint32_t pc_start;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start, org_flags;
@@ -1705,47 +1670,51 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do
     {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
 #if SIM_COMPAT
         if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
             tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
             gen_helper_debug();
         }
 #endif
-        check_breakpoint(env, dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-                        tcg_ctx.gen_opc_icount[lj] = num_insns;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         dc->clear_imm = 1;
         decode(dc, cpu_ldl_code(env, dc->pc));
         if (dc->clear_imm)
             dc->tb_flags &= ~IMM_FLAG;
         dc->pc += 4;
-        num_insns++;
 
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1836,15 +1805,8 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-                tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !SIM_COMPAT
@@ -1861,16 +1823,6 @@ gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
     assert(!dc->abort_at_next_insn);
 }
 
-void gen_intermediate_code (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, true);
-}
-
 void mb_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                        int flags)
 {
@@ -1918,40 +1870,41 @@ void mb_tcg_init(void)
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
-    env_debug = tcg_global_mem_new(TCG_AREG0, 
+    env_debug = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, debug),
                     "debug0");
-    env_iflags = tcg_global_mem_new(TCG_AREG0, 
+    env_iflags = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, iflags),
                     "iflags");
-    env_imm = tcg_global_mem_new(TCG_AREG0, 
+    env_imm = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, imm),
                     "imm");
-    env_btarget = tcg_global_mem_new(TCG_AREG0,
+    env_btarget = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, btarget),
                      "btarget");
-    env_btaken = tcg_global_mem_new(TCG_AREG0,
+    env_btaken = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, btaken),
                      "btaken");
-    env_res_addr = tcg_global_mem_new(TCG_AREG0,
+    env_res_addr = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, res_addr),
                      "res_addr");
-    env_res_val = tcg_global_mem_new(TCG_AREG0,
+    env_res_val = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, res_val),
                      "res_val");
     for (i = 0; i < ARRAY_SIZE(cpu_R); i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPUMBState, regs[i]),
                           regnames[i]);
     }
     for (i = 0; i < ARRAY_SIZE(cpu_SR); i++) {
-        cpu_SR[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_SR[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPUMBState, sregs[i]),
                           special_regnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->sregs[SR_PC] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->sregs[SR_PC] = data[0];
 }
diff --git a/qemu/target-mips/cpu.c b/qemu/target-mips/cpu.c
index 4027d0f41..0e2ecbebe 100644
--- a/qemu/target-mips/cpu.c
+++ b/qemu/target-mips/cpu.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "kvm_mips.h"
 #include "qemu-common.h"
@@ -53,12 +55,15 @@ static bool mips_cpu_has_work(CPUState *cs)
     CPUMIPSState *env = &cpu->env;
     bool has_work = false;
 
-    /* It is implementation dependent if non-enabled interrupts
-       wake-up the CPU, however most of the implementations only
+    /* Prior to MIPS Release 6 it is implementation dependent if non-enabled
+       interrupts wake-up the CPU, however most of the implementations only
        check for interrupts that can be taken. */
     if ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
         cpu_mips_hw_interrupts_pending(env)) {
-        has_work = true;
+        if (cpu_mips_hw_interrupts_enabled(env) ||
+            (env->insn_flags & ISA_MIPS32R6)) {
+            has_work = true;
+        }
     }
 
     /* MIPS-MT has the ability to halt the CPU.  */
@@ -73,6 +78,15 @@ static bool mips_cpu_has_work(CPUState *cs)
             has_work = false;
         }
     }
+    /* MIPS Release 6 has the ability to halt the CPU.  */
+    if (env->CP0_Config5 & (1 << CP0C5_VP)) {
+        if (cs->interrupt_request & CPU_INTERRUPT_WAKE) {
+            has_work = true;
+        }
+        if (!mips_vp_active(env)) {
+            has_work = false;
+        }
+    }
     return has_work;
 }
 
@@ -97,6 +111,14 @@ static void mips_cpu_reset(CPUState *s)
 #endif
 }
 
+static void mips_cpu_disas_set_info(CPUState *s, disassemble_info *info) {
+#ifdef TARGET_WORDS_BIGENDIAN
+    info->print_insn = print_insn_big_mips;
+#else
+    info->print_insn = print_insn_little_mips;
+#endif
+}
+
 static void mips_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -150,9 +172,17 @@ static void mips_cpu_class_init(ObjectClass *c, void *data)
     cc->get_phys_page_debug = mips_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_mips_cpu;
 #endif
+    cc->disas_set_info = mips_cpu_disas_set_info;
 
     cc->gdb_num_core_regs = 73;
     cc->gdb_stop_before_watchpoint = true;
+
+    /*
+     * Reason: mips_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo mips_cpu_type_info = {
diff --git a/qemu/target-mips/cpu.h b/qemu/target-mips/cpu.h
index 075c561c8..866924d18 100644
--- a/qemu/target-mips/cpu.h
+++ b/qemu/target-mips/cpu.h
@@ -5,11 +5,8 @@
 
 #define ALIGNED_ONLY
 
-#define ELF_MACHINE	EM_MIPS
-
 #define CPUArchState struct CPUMIPSState
 
-#include "config.h"
 #include "qemu-common.h"
 #include "mips-defs.h"
 #include "exec/cpu-defs.h"
@@ -21,19 +18,19 @@ typedef struct r4k_tlb_t r4k_tlb_t;
 struct r4k_tlb_t {
     target_ulong VPN;
     uint32_t PageMask;
-    uint_fast8_t ASID;
-    uint_fast16_t G:1;
-    uint_fast16_t C0:3;
-    uint_fast16_t C1:3;
-    uint_fast16_t V0:1;
-    uint_fast16_t V1:1;
-    uint_fast16_t D0:1;
-    uint_fast16_t D1:1;
-    uint_fast16_t XI0:1;
-    uint_fast16_t XI1:1;
-    uint_fast16_t RI0:1;
-    uint_fast16_t RI1:1;
-    uint_fast16_t EHINV:1;
+    uint8_t ASID;
+    unsigned int G:1;
+    unsigned int C0:3;
+    unsigned int C1:3;
+    unsigned int V0:1;
+    unsigned int V1:1;
+    unsigned int D0:1;
+    unsigned int D1:1;
+    unsigned int XI0:1;
+    unsigned int XI1:1;
+    unsigned int RI0:1;
+    unsigned int RI1:1;
+    unsigned int EHINV:1;
     uint64_t PFN[2];
 };
 
@@ -102,6 +99,7 @@ struct CPUMIPSFPUContext {
     uint32_t fcr0;
 #define FCR0_FREP 29
 #define FCR0_UFRP 28
+#define FCR0_HAS2008 23
 #define FCR0_F64 22
 #define FCR0_L 21
 #define FCR0_W 20
@@ -113,6 +111,8 @@ struct CPUMIPSFPUContext {
 #define FCR0_REV 0
     /* fcsr */
     uint32_t fcr31;
+#define FCR31_ABS2008 19
+#define FCR31_NAN2008 18
 #define SET_FP_COND(num,env)     do { ((env).fcr31) |= ((num) ? (1 << ((num) + 24)) : (1 << 23)); } while(0)
 #define CLEAR_FP_COND(num,env)   do { ((env).fcr31) &= ~((num) ? (1 << ((num) + 24)) : (1 << 23)); } while(0)
 #define GET_FP_COND(env)         ((((env).fcr31 >> 24) & 0xfe) | (((env).fcr31 >> 23) & 0x1))
@@ -132,6 +132,7 @@ struct CPUMIPSFPUContext {
 };
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 2
 
 typedef struct CPUMIPSMVPContext CPUMIPSMVPContext;
 struct CPUMIPSMVPContext {
@@ -164,6 +165,7 @@ typedef struct mips_def_t mips_def_t;
 #define MIPS_FPU_MAX 1
 #define MIPS_DSP_ACC 4
 #define MIPS_KSCRATCH_NUM 6
+#define MIPS_MAAR_MAX 16 /* Must be an even number. */
 
 typedef struct TCState TCState;
 struct TCState {
@@ -239,6 +241,8 @@ struct CPUMIPSState {
 
     int32_t CP0_Index;
     /* CP0_MVP* are per MVP registers. */
+    int32_t CP0_VPControl;
+#define CP0VPCtl_DIS    0
     int32_t CP0_Random;
     int32_t CP0_VPEControl;
 #define CP0VPECo_YSI	21
@@ -288,6 +292,8 @@ struct CPUMIPSState {
 # define CP0EnLo_RI 31
 # define CP0EnLo_XI 30
 #endif
+    int32_t CP0_GlobalNumber;
+#define CP0GN_VPId 0
     target_ulong CP0_Context;
     target_ulong CP0_KScratch[MIPS_KSCRATCH_NUM];
     int32_t CP0_PageMask;
@@ -359,7 +365,7 @@ struct CPUMIPSState {
 #define CP0St_IE    0
     int32_t CP0_IntCtl;
 #define CP0IntCtl_IPTI 29
-#define CP0IntCtl_IPPC1 26
+#define CP0IntCtl_IPPCI 26
 #define CP0IntCtl_VS 5
     int32_t CP0_SRSCtl;
 #define CP0SRSCtl_HSS 26
@@ -390,12 +396,13 @@ struct CPUMIPSState {
     target_ulong CP0_EPC;
     int32_t CP0_PRid;
     int32_t CP0_EBase;
+    target_ulong CP0_CMGCRBase;
     int32_t CP0_Config0;
 #define CP0C0_M    31
 #define CP0C0_K23  28
 #define CP0C0_KU   25
 #define CP0C0_MDU  20
-#define CP0C0_MM   17
+#define CP0C0_MM   18
 #define CP0C0_BM   16
 #define CP0C0_BE   15
 #define CP0C0_AT   13
@@ -432,7 +439,7 @@ struct CPUMIPSState {
     int32_t CP0_Config3;
 #define CP0C3_M    31
 #define CP0C3_BPG  30
-#define CP0C3_CMCGR 29
+#define CP0C3_CMGCR 29
 #define CP0C3_MSAP  28
 #define CP0C3_BP 27
 #define CP0C3_BI 26
@@ -470,15 +477,20 @@ struct CPUMIPSState {
 #define CP0C5_CV         29
 #define CP0C5_EVA        28
 #define CP0C5_MSAEn      27
+#define CP0C5_XNP        13
 #define CP0C5_UFE        9
 #define CP0C5_FRE        8
+#define CP0C5_VP         7
 #define CP0C5_SBRI       6
 #define CP0C5_MVH        5
 #define CP0C5_LLB        4
+#define CP0C5_MRP        3
 #define CP0C5_UFR        2
 #define CP0C5_NFExists   0
     int32_t CP0_Config6;
     int32_t CP0_Config7;
+    uint64_t CP0_MAAR[MIPS_MAAR_MAX];
+    int32_t CP0_MAARI;
     /* XXX: Maybe make LLAddr per-TC? */
     uint64_t lladdr;
     target_ulong llval;
@@ -511,6 +523,10 @@ struct CPUMIPSState {
 #define CP0DB_DSS  0
     target_ulong CP0_DEPC;
     int32_t CP0_Performance0;
+    int32_t CP0_ErrCtl;
+#define CP0EC_WST 29
+#define CP0EC_SPR 28
+#define CP0EC_ITC 26
     uint64_t CP0_TagLo;
     int32_t CP0_DataLo;
     int32_t CP0_TagHi;
@@ -526,7 +542,7 @@ struct CPUMIPSState {
 #define EXCP_INST_NOTAVAIL 0x2 /* No valid instruction word for BadInstr */
     uint32_t hflags;    /* CPU State */
     /* TMASK defines different execution modes */
-#define MIPS_HFLAG_TMASK  0x75807FF
+#define MIPS_HFLAG_TMASK  0xF5807FF
 #define MIPS_HFLAG_MODE   0x00007 /* execution modes                    */
     /* The KSU flags must be the lowest bits in hflags. The flag order
        must be the same as defined for CP0 Status. This allows to use
@@ -575,6 +591,7 @@ struct CPUMIPSState {
 #define MIPS_HFLAG_MSA   0x1000000
 #define MIPS_HFLAG_FRE   0x2000000 /* FRE enabled */
 #define MIPS_HFLAG_ELPA  0x4000000
+#define MIPS_HFLAG_ITC_CACHE  0x8000000 /* CACHE instr. operates on ITC tag */
     target_ulong btarget;        /* Jump / branch target               */
     target_ulong bcond;          /* Branch condition (if needed)       */
 
@@ -595,6 +612,7 @@ struct CPUMIPSState {
     const mips_def_t *cpu_model;
     void *irq[8];
     QEMUTimer *timer; /* Internal timer */
+    MemoryRegion *itc_tag; /* ITC Configuration Tags */
 };
 
 #include "cpu-qom.h"
@@ -621,7 +639,6 @@ void mips_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
 void mips_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 
 #define cpu_exec cpu_mips_exec
-#define cpu_gen_code cpu_mips_gen_code
 #define cpu_signal_handler cpu_mips_signal_handler
 #define cpu_list mips_cpu_list
 
@@ -634,28 +651,29 @@ extern uint32_t cpu_rddsp(uint32_t mask_num, CPUMIPSState *env);
 #define MMU_MODE1_SUFFIX _super
 #define MMU_MODE2_SUFFIX _user
 #define MMU_USER_IDX 2
-static inline int cpu_mmu_index (CPUMIPSState *env)
+static inline int cpu_mmu_index (CPUMIPSState *env, bool ifetch)
 {
     return env->hflags & MIPS_HFLAG_KSU;
 }
 
-static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+static inline bool cpu_mips_hw_interrupts_enabled(CPUMIPSState *env)
 {
-    int32_t pending;
-    int32_t status;
-    int r;
-
-    if (!(env->CP0_Status & (1 << CP0St_IE)) ||
-        (env->CP0_Status & (1 << CP0St_EXL)) ||
-        (env->CP0_Status & (1 << CP0St_ERL)) ||
+    return (env->CP0_Status & (1 << CP0St_IE)) &&
+        !(env->CP0_Status & (1 << CP0St_EXL)) &&
+        !(env->CP0_Status & (1 << CP0St_ERL)) &&
+        !(env->hflags & MIPS_HFLAG_DM) &&
         /* Note that the TCStatus IXMT field is initialized to zero,
            and only MT capable cores can set it to one. So we don't
            need to check for MT capabilities here.  */
-        (env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT)) ||
-        (env->hflags & MIPS_HFLAG_DM)) {
-        /* Interrupts are disabled */
-        return 0;
-    }
+        !(env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT));
+}
+
+/* Check if there is pending and not masked out interrupt */
+static inline bool cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+{
+    int32_t pending;
+    int32_t status;
+    bool r;
 
     pending = env->CP0_Cause & CP0Ca_IP_mask;
     status = env->CP0_Status & CP0Ca_IP_mask;
@@ -669,7 +687,7 @@ static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
         /* A MIPS configured with compatibility or VInt (Vectored Interrupts)
            treats the pending lines as individual interrupt lines, the status
            lines are individual masks.  */
-        r = pending & status;
+        r = (pending & status) != 0;
     }
     return r;
 }
@@ -752,6 +770,7 @@ MIPSCPU *cpu_mips_init(const char *cpu_model);
 int cpu_mips_signal_handler(int host_signum, void *pinfo, void *puc);
 
 #define cpu_init(cpu_model) CPU(cpu_mips_init(cpu_model))
+bool cpu_supports_cps_smp(const char *cpu_model);
 
 /* TODO QOM'ify CPU reset and remove */
 void cpu_state_reset(CPUMIPSState *s);
@@ -859,6 +878,26 @@ static inline int mips_vpe_active(CPUMIPSState *env)
     return active;
 }
 
+static inline int mips_vp_active(CPUMIPSState *env)
+{
+    CPUState *other_cs = first_cpu;
+
+    /* Check if the VP disabled other VPs (which means the VP is enabled) */
+    if ((env->CP0_VPControl >> CP0VPCtl_DIS) & 1) {
+        return 1;
+    }
+
+    /* Check if the virtual processor is disabled due to a DVP */
+    CPU_FOREACH(other_cs) {
+        MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+        if ((&other_cpu->env != env) &&
+            ((other_cpu->env.CP0_VPControl >> CP0VPCtl_DIS) & 1)) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
 #include "exec/exec-all.h"
 
 static inline void compute_hflags(CPUMIPSState *env)
@@ -961,6 +1000,15 @@ static inline void compute_hflags(CPUMIPSState *env)
 }
 
 #ifndef CONFIG_USER_ONLY
+static inline void cpu_mips_tlb_flush(CPUMIPSState *env, int flush_global)
+{
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+
+    /* Flush qemu's TLB and discard all shadowed entries.  */
+    tlb_flush(CPU(cpu), flush_global);
+    env->tlb->tlb_in_use = env->tlb->nb_tlb;
+}
+
 /* Called for updates to CP0_Status.  */
 static inline void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 {
@@ -999,17 +1047,29 @@ static inline void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 static inline void cpu_mips_store_status(CPUMIPSState *env, target_ulong val)
 {
     uint32_t mask = env->CP0_Status_rw_bitmask;
+    target_ulong old = env->CP0_Status;
 
     if (env->insn_flags & ISA_MIPS32R6) {
         bool has_supervisor = extract32(mask, CP0St_KSU, 2) == 0x3;
-
+#if defined(TARGET_MIPS64)
+        uint32_t ksux = (1 << CP0St_KX) & val;
+        ksux |= (ksux >> 1) & val; /* KX = 0 forces SX to be 0 */
+        ksux |= (ksux >> 1) & val; /* SX = 0 forces UX to be 0 */
+        val = (val & ~(7 << CP0St_UX)) | ksux;
+#endif
         if (has_supervisor && extract32(val, CP0St_KSU, 2) == 0x3) {
             mask &= ~(3 << CP0St_KSU);
         }
         mask &= ~(((1 << CP0St_SR) | (1 << CP0St_NMI)) & val);
     }
 
-    env->CP0_Status = (env->CP0_Status & ~mask) | (val & mask);
+    env->CP0_Status = (old & ~mask) | (val & mask);
+#if defined(TARGET_MIPS64)
+    if ((env->CP0_Status ^ old) & (old & (7 << CP0St_UX))) {
+        /* Access to at least one of the 64-bit segments has been disabled */
+        cpu_mips_tlb_flush(env, 1);
+    }
+#endif
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         sync_c0_status(env, env, env->current_tc);
     } else {
@@ -1049,4 +1109,28 @@ static inline void cpu_mips_store_cause(CPUMIPSState *env, target_ulong val)
 }
 #endif
 
+static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
+                                                        uint32_t exception,
+                                                        int error_code,
+                                                        uintptr_t pc)
+{
+    CPUState *cs = CPU(mips_env_get_cpu(env));
+
+    if (exception < EXCP_SC) {
+        qemu_log_mask(CPU_LOG_INT, "%s: %d %d\n",
+                      __func__, exception, error_code);
+    }
+    cs->exception_index = exception;
+    env->error_code = error_code;
+
+    cpu_loop_exit_restore(cs, pc);
+}
+
+static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
+                                                    uint32_t exception,
+                                                    uintptr_t pc)
+{
+    do_raise_exception_err(env, exception, 0, pc);
+}
+
 #endif /* !defined (__MIPS_CPU_H__) */
diff --git a/qemu/target-mips/dsp_helper.c b/qemu/target-mips/dsp_helper.c
index 46528de32..df7d2204b 100644
--- a/qemu/target-mips/dsp_helper.c
+++ b/qemu/target-mips/dsp_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/bitops.h"
diff --git a/qemu/target-mips/gdbstub.c b/qemu/target-mips/gdbstub.c
index 9845d880a..b0b4a32ec 100644
--- a/qemu/target-mips/gdbstub.c
+++ b/qemu/target-mips/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-mips/helper.c b/qemu/target-mips/helper.c
index 04ba19fd4..1004edee0 100644
--- a/qemu/target-mips/helper.c
+++ b/qemu/target-mips/helper.c
@@ -16,16 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "sysemu/kvm.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
 
 enum {
     TLBRET_XI = -6,
@@ -127,10 +123,6 @@ static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
     /* effective address (modified for KVM T&E kernel segments) */
     target_ulong address = real_address;
 
-#if 0
-    qemu_log("user mode %d h %08x\n", user_mode, env->hflags);
-#endif
-
 #define USEG_LIMIT      0x7FFFFFFFUL
 #define KSEG0_BASE      0x80000000UL
 #define KSEG1_BASE      0xA0000000UL
@@ -227,11 +219,6 @@ static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
             ret = TLBRET_BADADDR;
         }
     }
-#if 0
-    qemu_log(TARGET_FMT_lx " %d %d => %" HWADDR_PRIx " %d (%d)\n",
-            address, rw, access_type, *physical, *prot, ret);
-#endif
-
     return ret;
 }
 #endif
@@ -303,9 +290,10 @@ static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
         (env->CP0_EntryHi & 0xFF) | (address & (TARGET_PAGE_MASK << 1));
 #if defined(TARGET_MIPS64)
     env->CP0_EntryHi &= env->SEGMask;
-    env->CP0_XContext = (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
-                        ((address & 0xC00000000000ULL) >> (55 - env->SEGBITS)) |
-                        ((address & ((1ULL << env->SEGBITS) - 1) & 0xFFFFFFFFFFFFE000ULL) >> 9);
+    env->CP0_XContext =
+        /* PTEBase */   (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
+        /* R */         (extract64(address, 62, 2) << (env->SEGBITS - 9)) |
+        /* BadVPN2 */   (extract64(address, 13, env->SEGBITS - 13) << 4);
 #endif
     cs->exception_index = exception;
     env->error_code = error_code;
@@ -487,14 +475,16 @@ void mips_cpu_do_interrupt(CPUState *cs)
     int cause = -1;
     const char *name;
 
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         if (cs->exception_index < 0 || cs->exception_index > EXCP_LAST) {
             name = "unknown";
         } else {
             name = excp_names[cs->exception_index];
         }
 
-        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " %s exception\n",
+        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx
+                 " %s exception\n",
                  __func__, env->active_tc.PC, env->CP0_EPC, name);
     }
     if (cs->exception_index == EXCP_EXT_INTERRUPT &&
@@ -531,6 +521,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
  enter_debug_mode:
         if (env->insn_flags & ISA_MIPS3) {
             env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
         }
         env->hflags |= MIPS_HFLAG_DM | MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -555,6 +549,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
         env->CP0_Status |= (1 << CP0St_ERL) | (1 << CP0St_BEV);
         if (env->insn_flags & ISA_MIPS3) {
             env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
         }
         env->hflags |= MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -700,7 +698,7 @@ void mips_cpu_do_interrupt(CPUState *cs)
         goto set_EPC;
     case EXCP_DWATCH:
         cause = 23;
-        /* XXX: TODO: manage defered watch exceptions */
+        /* XXX: TODO: manage deferred watch exceptions */
         goto set_EPC;
     case EXCP_MCHECK:
         cause = 24;
@@ -732,6 +730,10 @@ void mips_cpu_do_interrupt(CPUState *cs)
             env->CP0_Status |= (1 << CP0St_EXL);
             if (env->insn_flags & ISA_MIPS3) {
                 env->hflags |= MIPS_HFLAG_64;
+                if (!(env->insn_flags & ISA_MIPS64R6) ||
+                    env->CP0_Status & (1 << CP0St_KX)) {
+                    env->hflags &= ~MIPS_HFLAG_AWRAP;
+                }
             }
             env->hflags |= MIPS_HFLAG_CP0;
             env->hflags &= ~(MIPS_HFLAG_KSU);
@@ -747,16 +749,15 @@ void mips_cpu_do_interrupt(CPUState *cs)
         env->CP0_Cause = (env->CP0_Cause & ~(0x1f << CP0Ca_EC)) | (cause << CP0Ca_EC);
         break;
     default:
-        qemu_log("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        printf("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        exit(1);
+        abort();
     }
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         qemu_log("%s: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " cause %d\n"
-                "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
-                __func__, env->active_tc.PC, env->CP0_EPC, cause,
-                env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
-                env->CP0_DEPC);
+                 "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
+                 __func__, env->active_tc.PC, env->CP0_EPC, cause,
+                 env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
+                 env->CP0_DEPC);
     }
 #endif
     cs->exception_index = EXCP_NONE;
@@ -768,7 +769,8 @@ bool mips_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
         MIPSCPU *cpu = MIPS_CPU(cs);
         CPUMIPSState *env = &cpu->env;
 
-        if (cpu_mips_hw_interrupts_pending(env)) {
+        if (cpu_mips_hw_interrupts_enabled(env) &&
+            cpu_mips_hw_interrupts_pending(env)) {
             /* Raise it */
             cs->exception_index = EXCP_EXT_INTERRUPT;
             env->error_code = 0;
diff --git a/qemu/target-mips/helper.h b/qemu/target-mips/helper.h
index 2b28e875c..594341d25 100644
--- a/qemu/target-mips/helper.h
+++ b/qemu/target-mips/helper.h
@@ -1,5 +1,6 @@
 DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
+DEF_HELPER_1(raise_exception_debug, noreturn, env)
 
 DEF_HELPER_1(do_semihosting, void, env)
 
@@ -76,6 +77,8 @@ DEF_HELPER_1(mftc0_epc, tl, env)
 DEF_HELPER_1(mftc0_ebase, tl, env)
 DEF_HELPER_2(mftc0_configx, tl, env, tl)
 DEF_HELPER_1(mfc0_lladdr, tl, env)
+DEF_HELPER_1(mfc0_maar, tl, env)
+DEF_HELPER_1(mfhc0_maar, tl, env)
 DEF_HELPER_2(mfc0_watchlo, tl, env, i32)
 DEF_HELPER_2(mfc0_watchhi, tl, env, i32)
 DEF_HELPER_1(mfc0_debug, tl, env)
@@ -87,6 +90,7 @@ DEF_HELPER_1(dmfc0_tccontext, tl, env)
 DEF_HELPER_1(dmfc0_tcschedule, tl, env)
 DEF_HELPER_1(dmfc0_tcschefback, tl, env)
 DEF_HELPER_1(dmfc0_lladdr, tl, env)
+DEF_HELPER_1(dmfc0_maar, tl, env)
 DEF_HELPER_2(dmfc0_watchlo, tl, env, i32)
 #endif /* TARGET_MIPS64 */
 
@@ -143,6 +147,9 @@ DEF_HELPER_2(mtc0_config3, void, env, tl)
 DEF_HELPER_2(mtc0_config4, void, env, tl)
 DEF_HELPER_2(mtc0_config5, void, env, tl)
 DEF_HELPER_2(mtc0_lladdr, void, env, tl)
+DEF_HELPER_2(mtc0_maar, void, env, tl)
+DEF_HELPER_2(mthc0_maar, void, env, tl)
+DEF_HELPER_2(mtc0_maari, void, env, tl)
 DEF_HELPER_3(mtc0_watchlo, void, env, tl, i32)
 DEF_HELPER_3(mtc0_watchhi, void, env, tl, i32)
 DEF_HELPER_2(mtc0_xcontext, void, env, tl)
@@ -150,6 +157,7 @@ DEF_HELPER_2(mtc0_framemask, void, env, tl)
 DEF_HELPER_2(mtc0_debug, void, env, tl)
 DEF_HELPER_2(mttc0_debug, void, env, tl)
 DEF_HELPER_2(mtc0_performance0, void, env, tl)
+DEF_HELPER_2(mtc0_errctl, void, env, tl)
 DEF_HELPER_2(mtc0_taglo, void, env, tl)
 DEF_HELPER_2(mtc0_datalo, void, env, tl)
 DEF_HELPER_2(mtc0_taghi, void, env, tl)
@@ -175,6 +183,10 @@ DEF_HELPER_0(dmt, tl)
 DEF_HELPER_0(emt, tl)
 DEF_HELPER_1(dvpe, tl, env)
 DEF_HELPER_1(evpe, tl, env)
+
+/* R6 Multi-threading */
+DEF_HELPER_1(dvp, tl, env)
+DEF_HELPER_1(evp, tl, env)
 #endif /* !CONFIG_USER_ONLY */
 
 /* microMIPS functions */
@@ -357,6 +369,8 @@ DEF_HELPER_1(rdhwr_cpunum, tl, env)
 DEF_HELPER_1(rdhwr_synci_step, tl, env)
 DEF_HELPER_1(rdhwr_cc, tl, env)
 DEF_HELPER_1(rdhwr_ccres, tl, env)
+DEF_HELPER_1(rdhwr_performance, tl, env)
+DEF_HELPER_1(rdhwr_xnp, tl, env)
 DEF_HELPER_2(pmon, void, env, int)
 DEF_HELPER_1(wait, void, env)
 
@@ -942,3 +956,5 @@ MSALDST_PROTO(h)
 MSALDST_PROTO(w)
 MSALDST_PROTO(d)
 #undef MSALDST_PROTO
+
+DEF_HELPER_3(cache, void, env, tl, i32)
diff --git a/qemu/target-mips/kvm.c b/qemu/target-mips/kvm.c
index d287d4288..950bc05b7 100644
--- a/qemu/target-mips/kvm.c
+++ b/qemu/target-mips/kvm.c
@@ -9,7 +9,7 @@
  * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 
@@ -30,6 +30,9 @@
 #define DPRINTF(fmt, ...) \
     do { if (DEBUG_KVM) { fprintf(stderr, fmt, ## __VA_ARGS__); } } while (0)
 
+static int kvm_mips_fpu_cap;
+static int kvm_mips_msa_cap;
+
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
@@ -46,16 +49,39 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     /* MIPS has 128 signals */
     kvm_set_sigmask_len(s, 16);
 
+    kvm_mips_fpu_cap = kvm_check_extension(s, KVM_CAP_MIPS_FPU);
+    kvm_mips_msa_cap = kvm_check_extension(s, KVM_CAP_MIPS_MSA);
+
     DPRINTF("%s\n", __func__);
     return 0;
 }
 
 int kvm_arch_init_vcpu(CPUState *cs)
 {
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
     int ret = 0;
 
     qemu_add_vm_change_state_handler(kvm_mips_update_state, cs);
 
+    if (kvm_mips_fpu_cap && env->CP0_Config1 & (1 << CP0C1_FP)) {
+        ret = kvm_vcpu_enable_cap(cs, KVM_CAP_MIPS_FPU, 0, 0);
+        if (ret < 0) {
+            /* mark unsupported so it gets disabled on reset */
+            kvm_mips_fpu_cap = 0;
+            ret = 0;
+        }
+    }
+
+    if (kvm_mips_msa_cap && env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        ret = kvm_vcpu_enable_cap(cs, KVM_CAP_MIPS_MSA, 0, 0);
+        if (ret < 0) {
+            /* mark unsupported so it gets disabled on reset */
+            kvm_mips_msa_cap = 0;
+            ret = 0;
+        }
+    }
+
     DPRINTF("%s\n", __func__);
     return ret;
 }
@@ -64,10 +90,14 @@ void kvm_mips_reset_vcpu(MIPSCPU *cpu)
 {
     CPUMIPSState *env = &cpu->env;
 
-    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
-        fprintf(stderr, "Warning: FPU not supported with KVM, disabling\n");
+    if (!kvm_mips_fpu_cap && env->CP0_Config1 & (1 << CP0C1_FP)) {
+        fprintf(stderr, "Warning: KVM does not support FPU, disabling\n");
         env->CP0_Config1 &= ~(1 << CP0C1_FP);
     }
+    if (!kvm_mips_msa_cap && env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        fprintf(stderr, "Warning: KVM does not support MSA, disabling\n");
+        env->CP0_Config3 &= ~(1 << CP0C3_MSAP);
+    }
 
     DPRINTF("%s\n", __func__);
 }
@@ -88,7 +118,6 @@ static inline int cpu_mips_io_interrupts_pending(MIPSCPU *cpu)
 {
     CPUMIPSState *env = &cpu->env;
 
-    DPRINTF("%s: %#x\n", __func__, env->CP0_Cause & (1 << (2 + CP0Ca_IP)));
     return env->CP0_Cause & (0x1 << (2 + CP0Ca_IP));
 }
 
@@ -117,7 +146,6 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 
 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
-    DPRINTF("%s\n", __func__);
     return MEMTXATTRS_UNSPECIFIED;
 }
 
@@ -230,6 +258,13 @@ int kvm_mips_set_ipi_interrupt(MIPSCPU *cpu, int irq, int level)
 #define KVM_REG_MIPS_CP0_STATUS         MIPS_CP0_32(12, 0)
 #define KVM_REG_MIPS_CP0_CAUSE          MIPS_CP0_32(13, 0)
 #define KVM_REG_MIPS_CP0_EPC            MIPS_CP0_64(14, 0)
+#define KVM_REG_MIPS_CP0_PRID           MIPS_CP0_32(15, 0)
+#define KVM_REG_MIPS_CP0_CONFIG         MIPS_CP0_32(16, 0)
+#define KVM_REG_MIPS_CP0_CONFIG1        MIPS_CP0_32(16, 1)
+#define KVM_REG_MIPS_CP0_CONFIG2        MIPS_CP0_32(16, 2)
+#define KVM_REG_MIPS_CP0_CONFIG3        MIPS_CP0_32(16, 3)
+#define KVM_REG_MIPS_CP0_CONFIG4        MIPS_CP0_32(16, 4)
+#define KVM_REG_MIPS_CP0_CONFIG5        MIPS_CP0_32(16, 5)
 #define KVM_REG_MIPS_CP0_ERROREPC       MIPS_CP0_64(30, 0)
 
 static inline int kvm_mips_put_one_reg(CPUState *cs, uint64_t reg_id,
@@ -243,6 +278,17 @@ static inline int kvm_mips_put_one_reg(CPUState *cs, uint64_t reg_id,
     return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
 }
 
+static inline int kvm_mips_put_one_ureg(CPUState *cs, uint64_t reg_id,
+                                        uint32_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
+}
+
 static inline int kvm_mips_put_one_ulreg(CPUState *cs, uint64_t reg_id,
                                          target_ulong *addr)
 {
@@ -256,7 +302,18 @@ static inline int kvm_mips_put_one_ulreg(CPUState *cs, uint64_t reg_id,
 }
 
 static inline int kvm_mips_put_one_reg64(CPUState *cs, uint64_t reg_id,
-                                         uint64_t *addr)
+                                         int64_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
+}
+
+static inline int kvm_mips_put_one_ureg64(CPUState *cs, uint64_t reg_id,
+                                          uint64_t *addr)
 {
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
@@ -277,7 +334,18 @@ static inline int kvm_mips_get_one_reg(CPUState *cs, uint64_t reg_id,
     return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
 }
 
-static inline int kvm_mips_get_one_ulreg(CPUState *cs, uint64 reg_id,
+static inline int kvm_mips_get_one_ureg(CPUState *cs, uint64_t reg_id,
+                                        uint32_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
+}
+
+static inline int kvm_mips_get_one_ulreg(CPUState *cs, uint64_t reg_id,
                                          target_ulong *addr)
 {
     int ret;
@@ -294,8 +362,8 @@ static inline int kvm_mips_get_one_ulreg(CPUState *cs, uint64 reg_id,
     return ret;
 }
 
-static inline int kvm_mips_get_one_reg64(CPUState *cs, uint64 reg_id,
-                                         uint64_t *addr)
+static inline int kvm_mips_get_one_reg64(CPUState *cs, uint64_t reg_id,
+                                         int64_t *addr)
 {
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
@@ -305,6 +373,50 @@ static inline int kvm_mips_get_one_reg64(CPUState *cs, uint64 reg_id,
     return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
 }
 
+static inline int kvm_mips_get_one_ureg64(CPUState *cs, uint64_t reg_id,
+                                          uint64_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
+}
+
+#define KVM_REG_MIPS_CP0_CONFIG_MASK    (1U << CP0C0_M)
+#define KVM_REG_MIPS_CP0_CONFIG1_MASK   ((1U << CP0C1_M) | \
+                                         (1U << CP0C1_FP))
+#define KVM_REG_MIPS_CP0_CONFIG2_MASK   (1U << CP0C2_M)
+#define KVM_REG_MIPS_CP0_CONFIG3_MASK   ((1U << CP0C3_M) | \
+                                         (1U << CP0C3_MSAP))
+#define KVM_REG_MIPS_CP0_CONFIG4_MASK   (1U << CP0C4_M)
+#define KVM_REG_MIPS_CP0_CONFIG5_MASK   ((1U << CP0C5_MSAEn) | \
+                                         (1U << CP0C5_UFE) | \
+                                         (1U << CP0C5_FRE) | \
+                                         (1U << CP0C5_UFR))
+
+static inline int kvm_mips_change_one_reg(CPUState *cs, uint64_t reg_id,
+                                          int32_t *addr, int32_t mask)
+{
+    int err;
+    int32_t tmp, change;
+
+    err = kvm_mips_get_one_reg(cs, reg_id, &tmp);
+    if (err < 0) {
+        return err;
+    }
+
+    /* only change bits in mask */
+    change = (*addr ^ tmp) & mask;
+    if (!change) {
+        return 0;
+    }
+
+    tmp = tmp ^ change;
+    return kvm_mips_put_one_reg(cs, reg_id, &tmp);
+}
+
 /*
  * We freeze the KVM timer when either the VM clock is stopped or the state is
  * saved (the state is dirty).
@@ -322,13 +434,13 @@ static int kvm_mips_save_count(CPUState *cs)
     int err, ret = 0;
 
     /* freeze KVM timer */
-    err = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+    err = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
     if (err < 0) {
         DPRINTF("%s: Failed to get COUNT_CTL (%d)\n", __func__, err);
         ret = err;
     } else if (!(count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) {
         count_ctl |= KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=1 (%d)\n", __func__, err);
             ret = err;
@@ -364,14 +476,14 @@ static int kvm_mips_restore_count(CPUState *cs)
     int err_dc, err, ret = 0;
 
     /* check the timer is frozen */
-    err_dc = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+    err_dc = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
     if (err_dc < 0) {
         DPRINTF("%s: Failed to get COUNT_CTL (%d)\n", __func__, err_dc);
         ret = err_dc;
     } else if (!(count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) {
         /* freeze timer (sets COUNT_RESUME for us) */
         count_ctl |= KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=1 (%d)\n", __func__, err);
             ret = err;
@@ -395,7 +507,7 @@ static int kvm_mips_restore_count(CPUState *cs)
     /* resume KVM timer */
     if (err_dc >= 0) {
         count_ctl &= ~KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=0 (%d)\n", __func__, err);
             ret = err;
@@ -428,8 +540,8 @@ static void kvm_mips_update_state(void *opaque, int running, RunState state)
     } else {
         /* Set clock restore time to now */
         count_resume = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        ret = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_RESUME,
-                                     &count_resume);
+        ret = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_RESUME,
+                                      &count_resume);
         if (ret < 0) {
             fprintf(stderr, "Failed setting COUNT_RESUME\n");
             return;
@@ -444,6 +556,167 @@ static void kvm_mips_update_state(void *opaque, int running, RunState state)
     }
 }
 
+static int kvm_mips_put_fpu_registers(CPUState *cs, int level)
+{
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
+    int err, ret = 0;
+    unsigned int i;
+
+    /* Only put FPU state if we're emulating a CPU with an FPU */
+    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+        /* FPU Control Registers */
+        if (level == KVM_PUT_FULL_STATE) {
+            err = kvm_mips_put_one_ureg(cs, KVM_REG_MIPS_FCR_IR,
+                                        &env->active_fpu.fcr0);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put FCR_IR (%d)\n", __func__, err);
+                ret = err;
+            }
+        }
+        err = kvm_mips_put_one_ureg(cs, KVM_REG_MIPS_FCR_CSR,
+                                    &env->active_fpu.fcr31);
+        if (err < 0) {
+            DPRINTF("%s: Failed to put FCR_CSR (%d)\n", __func__, err);
+            ret = err;
+        }
+
+        /*
+         * FPU register state is a subset of MSA vector state, so don't put FPU
+         * registers if we're emulating a CPU with MSA.
+         */
+        if (!(env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            /* Floating point registers */
+            for (i = 0; i < 32; ++i) {
+                if (env->CP0_Status & (1 << CP0St_FR)) {
+                    err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_FPR_64(i),
+                                                  &env->active_fpu.fpr[i].d);
+                } else {
+                    err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FPR_32(i),
+                                    &env->active_fpu.fpr[i].w[FP_ENDIAN_IDX]);
+                }
+                if (err < 0) {
+                    DPRINTF("%s: Failed to put FPR%u (%d)\n", __func__, i, err);
+                    ret = err;
+                }
+            }
+        }
+    }
+
+    /* Only put MSA state if we're emulating a CPU with MSA */
+    if (env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        /* MSA Control Registers */
+        if (level == KVM_PUT_FULL_STATE) {
+            err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_MSA_IR,
+                                       &env->msair);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put MSA_IR (%d)\n", __func__, err);
+                ret = err;
+            }
+        }
+        err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_MSA_CSR,
+                                   &env->active_tc.msacsr);
+        if (err < 0) {
+            DPRINTF("%s: Failed to put MSA_CSR (%d)\n", __func__, err);
+            ret = err;
+        }
+
+        /* Vector registers (includes FP registers) */
+        for (i = 0; i < 32; ++i) {
+            /* Big endian MSA not supported by QEMU yet anyway */
+            err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_VEC_128(i),
+                                         env->active_fpu.fpr[i].wr.d);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put VEC%u (%d)\n", __func__, i, err);
+                ret = err;
+            }
+        }
+    }
+
+    return ret;
+}
+
+static int kvm_mips_get_fpu_registers(CPUState *cs)
+{
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
+    int err, ret = 0;
+    unsigned int i;
+
+    /* Only get FPU state if we're emulating a CPU with an FPU */
+    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+        /* FPU Control Registers */
+        err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FCR_IR,
+                                    &env->active_fpu.fcr0);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get FCR_IR (%d)\n", __func__, err);
+            ret = err;
+        }
+        err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FCR_CSR,
+                                    &env->active_fpu.fcr31);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get FCR_CSR (%d)\n", __func__, err);
+            ret = err;
+        } else {
+            restore_fp_status(env);
+        }
+
+        /*
+         * FPU register state is a subset of MSA vector state, so don't save FPU
+         * registers if we're emulating a CPU with MSA.
+         */
+        if (!(env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            /* Floating point registers */
+            for (i = 0; i < 32; ++i) {
+                if (env->CP0_Status & (1 << CP0St_FR)) {
+                    err = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_FPR_64(i),
+                                                  &env->active_fpu.fpr[i].d);
+                } else {
+                    err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FPR_32(i),
+                                    &env->active_fpu.fpr[i].w[FP_ENDIAN_IDX]);
+                }
+                if (err < 0) {
+                    DPRINTF("%s: Failed to get FPR%u (%d)\n", __func__, i, err);
+                    ret = err;
+                }
+            }
+        }
+    }
+
+    /* Only get MSA state if we're emulating a CPU with MSA */
+    if (env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        /* MSA Control Registers */
+        err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_MSA_IR,
+                                   &env->msair);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get MSA_IR (%d)\n", __func__, err);
+            ret = err;
+        }
+        err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_MSA_CSR,
+                                   &env->active_tc.msacsr);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get MSA_CSR (%d)\n", __func__, err);
+            ret = err;
+        } else {
+            restore_msa_fp_status(env);
+        }
+
+        /* Vector registers (includes FP registers) */
+        for (i = 0; i < 32; ++i) {
+            /* Big endian MSA not supported by QEMU yet anyway */
+            err = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_VEC_128(i),
+                                         env->active_fpu.fpr[i].wr.d);
+            if (err < 0) {
+                DPRINTF("%s: Failed to get VEC%u (%d)\n", __func__, i, err);
+                ret = err;
+            }
+        }
+    }
+
+    return ret;
+}
+
+
 static int kvm_mips_put_cp0_registers(CPUState *cs, int level)
 {
     MIPSCPU *cpu = MIPS_CPU(cs);
@@ -522,6 +795,53 @@ static int kvm_mips_put_cp0_registers(CPUState *cs, int level)
         DPRINTF("%s: Failed to put CP0_EPC (%d)\n", __func__, err);
         ret = err;
     }
+    err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_CP0_PRID, &env->CP0_PRid);
+    if (err < 0) {
+        DPRINTF("%s: Failed to put CP0_PRID (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG,
+                                  &env->CP0_Config0,
+                                  KVM_REG_MIPS_CP0_CONFIG_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG1,
+                                  &env->CP0_Config1,
+                                  KVM_REG_MIPS_CP0_CONFIG1_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG1 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG2,
+                                  &env->CP0_Config2,
+                                  KVM_REG_MIPS_CP0_CONFIG2_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG2 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG3,
+                                  &env->CP0_Config3,
+                                  KVM_REG_MIPS_CP0_CONFIG3_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG3 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG4,
+                                  &env->CP0_Config4,
+                                  KVM_REG_MIPS_CP0_CONFIG4_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG4 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG5,
+                                  &env->CP0_Config5,
+                                  KVM_REG_MIPS_CP0_CONFIG5_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG5 (%d)\n", __func__, err);
+        ret = err;
+    }
     err = kvm_mips_put_one_ulreg(cs, KVM_REG_MIPS_CP0_ERROREPC,
                                  &env->CP0_ErrorEPC);
     if (err < 0) {
@@ -608,6 +928,41 @@ static int kvm_mips_get_cp0_registers(CPUState *cs)
         DPRINTF("%s: Failed to get CP0_EPC (%d)\n", __func__, err);
         ret = err;
     }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_PRID, &env->CP0_PRid);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_PRID (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG, &env->CP0_Config0);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG1, &env->CP0_Config1);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG1 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG2, &env->CP0_Config2);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG2 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG3, &env->CP0_Config3);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG3 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG4, &env->CP0_Config4);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG4 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG5, &env->CP0_Config5);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG5 (%d)\n", __func__, err);
+        ret = err;
+    }
     err = kvm_mips_get_one_ulreg(cs, KVM_REG_MIPS_CP0_ERROREPC,
                                  &env->CP0_ErrorEPC);
     if (err < 0) {
@@ -646,6 +1001,11 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         return ret;
     }
 
+    ret = kvm_mips_put_fpu_registers(cs, level);
+    if (ret < 0) {
+        return ret;
+    }
+
     return ret;
 }
 
@@ -673,12 +1033,13 @@ int kvm_arch_get_registers(CPUState *cs)
     env->active_tc.PC = regs.pc;
 
     kvm_mips_get_cp0_registers(cs);
+    kvm_mips_get_fpu_registers(cs);
 
     return ret;
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
diff --git a/qemu/target-mips/lmi_helper.c b/qemu/target-mips/lmi_helper.c
index bbfcd59cd..fb1245b39 100644
--- a/qemu/target-mips/lmi_helper.c
+++ b/qemu/target-mips/lmi_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-mips/machine.c b/qemu/target-mips/machine.c
index b15c43a10..22bca1814 100644
--- a/qemu/target-mips/machine.c
+++ b/qemu/target-mips/machine.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 
 #include "cpu.h"
@@ -203,8 +204,8 @@ const VMStateDescription vmstate_tlb = {
 
 const VMStateDescription vmstate_mips_cpu = {
     .name = "cpu",
-    .version_id = 7,
-    .minimum_version_id = 7,
+    .version_id = 8,
+    .minimum_version_id = 8,
     .post_load = cpu_post_load,
     .fields = (VMStateField[]) {
         /* Active TC */
@@ -271,6 +272,8 @@ const VMStateDescription vmstate_mips_cpu = {
         VMSTATE_INT32(env.CP0_Config3, MIPSCPU),
         VMSTATE_INT32(env.CP0_Config6, MIPSCPU),
         VMSTATE_INT32(env.CP0_Config7, MIPSCPU),
+        VMSTATE_UINT64_ARRAY(env.CP0_MAAR, MIPSCPU, MIPS_MAAR_MAX),
+        VMSTATE_INT32(env.CP0_MAARI, MIPSCPU),
         VMSTATE_UINT64(env.lladdr, MIPSCPU),
         VMSTATE_UINTTL_ARRAY(env.CP0_WatchLo, MIPSCPU, 8),
         VMSTATE_INT32_ARRAY(env.CP0_WatchHi, MIPSCPU, 8),
diff --git a/qemu/target-mips/mips-semi.c b/qemu/target-mips/mips-semi.c
index 5050940c2..ed235de99 100644
--- a/qemu/target-mips/mips-semi.c
+++ b/qemu/target-mips/mips-semi.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/stat.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/softmmu-semi.h"
diff --git a/qemu/target-mips/msa_helper.c b/qemu/target-mips/msa_helper.c
index a1cb48f2a..654a0d272 100644
--- a/qemu/target-mips/msa_helper.c
+++ b/qemu/target-mips/msa_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -1352,7 +1353,7 @@ void helper_msa_ctcmsa(CPUMIPSState *env, target_ulong elm, uint32_t cd)
         /* check exception */
         if ((GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)
             & GET_FP_CAUSE(env->active_tc.msacsr)) {
-            helper_raise_exception(env, EXCP_MSAFPE);
+            do_raise_exception(env, EXCP_MSAFPE, GETPC());
         }
         break;
     }
@@ -1505,14 +1506,14 @@ static inline void clear_msacsr_cause(CPUMIPSState *env)
     SET_FP_CAUSE(env->active_tc.msacsr, 0);
 }
 
-static inline void check_msacsr_cause(CPUMIPSState *env)
+static inline void check_msacsr_cause(CPUMIPSState *env, uintptr_t retaddr)
 {
     if ((GET_FP_CAUSE(env->active_tc.msacsr) &
             (GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)) == 0) {
         UPDATE_FP_FLAGS(env->active_tc.msacsr,
                 GET_FP_CAUSE(env->active_tc.msacsr));
     } else {
-        helper_raise_exception(env, EXCP_MSAFPE);
+        do_raise_exception(env, EXCP_MSAFPE, retaddr);
     }
 }
 
@@ -1604,7 +1605,7 @@ static inline int get_enabled_exceptions(const CPUMIPSState *env, int c)
     return c & enable;
 }
 
-static inline float16 float16_from_float32(int32 a, flag ieee,
+static inline float16 float16_from_float32(int32_t a, flag ieee,
                                            float_status *status)
 {
       float16 f_val;
@@ -1615,7 +1616,7 @@ static inline float16 float16_from_float32(int32 a, flag ieee,
       return a < 0 ? (f_val | (1 << 15)) : f_val;
 }
 
-static inline float32 float32_from_float64(int64 a, float_status *status)
+static inline float32 float32_from_float64(int64_t a, float_status *status)
 {
       float32 f_val;
 
@@ -1636,7 +1637,7 @@ static inline float32 float32_from_float16(int16_t a, flag ieee,
       return a < 0 ? (f_val | (1 << 31)) : f_val;
 }
 
-static inline float64 float64_from_float32(int32 a, float_status *status)
+static inline float64 float64_from_float32(int32_t a, float_status *status)
 {
       float64 f_val;
 
@@ -1657,7 +1658,7 @@ static inline float32 float32_from_q16(int16_t a, float_status *status)
     return f_val;
 }
 
-static inline float64 float64_from_q32(int32 a, float_status *status)
+static inline float64 float64_from_q32(int32_t a, float_status *status)
 {
     float64 f_val;
 
@@ -1670,9 +1671,9 @@ static inline float64 float64_from_q32(int32 a, float_status *status)
 
 static inline int16_t float32_to_q16(float32 a, float_status *status)
 {
-    int32 q_val;
-    int32 q_min = 0xffff8000;
-    int32 q_max = 0x00007fff;
+    int32_t q_val;
+    int32_t q_min = 0xffff8000;
+    int32_t q_max = 0x00007fff;
 
     int ieee_ex;
 
@@ -1690,7 +1691,7 @@ static inline int16_t float32_to_q16(float32 a, float_status *status)
 
     if (ieee_ex & float_flag_overflow) {
         float_raise(float_flag_inexact, status);
-        return (int32)a < 0 ? q_min : q_max;
+        return (int32_t)a < 0 ? q_min : q_max;
     }
 
     /* conversion to int */
@@ -1704,7 +1705,7 @@ static inline int16_t float32_to_q16(float32 a, float_status *status)
         set_float_exception_flags(ieee_ex & (~float_flag_invalid)
                                , status);
         float_raise(float_flag_overflow | float_flag_inexact, status);
-        return (int32)a < 0 ? q_min : q_max;
+        return (int32_t)a < 0 ? q_min : q_max;
     }
 
     if (q_val < q_min) {
@@ -1720,11 +1721,11 @@ static inline int16_t float32_to_q16(float32 a, float_status *status)
     return (int16_t)q_val;
 }
 
-static inline int32 float64_to_q32(float64 a, float_status *status)
+static inline int32_t float64_to_q32(float64 a, float_status *status)
 {
-    int64 q_val;
-    int64 q_min = 0xffffffff80000000LL;
-    int64 q_max = 0x000000007fffffffLL;
+    int64_t q_val;
+    int64_t q_min = 0xffffffff80000000LL;
+    int64_t q_max = 0x000000007fffffffLL;
 
     int ieee_ex;
 
@@ -1742,7 +1743,7 @@ static inline int32 float64_to_q32(float64 a, float_status *status)
 
     if (ieee_ex & float_flag_overflow) {
         float_raise(float_flag_inexact, status);
-        return (int64)a < 0 ? q_min : q_max;
+        return (int64_t)a < 0 ? q_min : q_max;
     }
 
     /* conversion to integer */
@@ -1756,20 +1757,20 @@ static inline int32 float64_to_q32(float64 a, float_status *status)
         set_float_exception_flags(ieee_ex & (~float_flag_invalid)
                , status);
         float_raise(float_flag_overflow | float_flag_inexact, status);
-        return (int64)a < 0 ? q_min : q_max;
+        return (int64_t)a < 0 ? q_min : q_max;
     }
 
     if (q_val < q_min) {
         float_raise(float_flag_overflow | float_flag_inexact, status);
-        return (int32)q_min;
+        return (int32_t)q_min;
     }
 
     if (q_max < q_val) {
         float_raise(float_flag_overflow | float_flag_inexact, status);
-        return (int32)q_max;
+        return (int32_t)q_max;
     }
 
-    return (int32)q_val;
+    return (int32_t)q_val;
 }
 
 #define MSA_FLOAT_COND(DEST, OP, ARG1, ARG2, BITS, QUIET)                   \
@@ -1851,7 +1852,8 @@ static inline int32 float64_to_q32(float64 a, float_status *status)
     } while (0)
 
 static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1873,13 +1875,14 @@ static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1903,13 +1906,14 @@ static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1931,13 +1935,14 @@ static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1959,13 +1964,14 @@ static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1987,13 +1993,14 @@ static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2015,13 +2022,14 @@ static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2043,13 +2051,14 @@ static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2071,13 +2080,14 @@ static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2099,13 +2109,14 @@ static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2127,13 +2138,15 @@ static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet) {
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
+{
     wr_t wx, *pwx = &wx;
     uint32_t i;
 
@@ -2154,7 +2167,7 @@ static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
@@ -2165,7 +2178,7 @@ void helper_msa_fcaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 1);
+    compare_af(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2174,7 +2187,7 @@ void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 1);
+    compare_un(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2183,7 +2196,7 @@ void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 1);
+    compare_eq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2192,7 +2205,7 @@ void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 1);
+    compare_ueq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2201,7 +2214,7 @@ void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 1);
+    compare_lt(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2210,7 +2223,7 @@ void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 1);
+    compare_ult(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2219,7 +2232,7 @@ void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 1);
+    compare_le(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2228,7 +2241,7 @@ void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 1);
+    compare_ule(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2237,7 +2250,7 @@ void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 0);
+    compare_af(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2246,7 +2259,7 @@ void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 0);
+    compare_un(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2255,7 +2268,7 @@ void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 0);
+    compare_eq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2264,7 +2277,7 @@ void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 0);
+    compare_ueq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2273,7 +2286,7 @@ void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 0);
+    compare_lt(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2282,7 +2295,7 @@ void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 0);
+    compare_ult(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2291,7 +2304,7 @@ void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 0);
+    compare_le(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2300,7 +2313,7 @@ void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 0);
+    compare_ule(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2309,7 +2322,7 @@ void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 1);
+    compare_or(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2318,7 +2331,7 @@ void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 1);
+    compare_une(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2327,7 +2340,7 @@ void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 1);
+    compare_ne(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2336,7 +2349,7 @@ void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 0);
+    compare_or(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2345,7 +2358,7 @@ void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 0);
+    compare_une(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2354,7 +2367,7 @@ void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 0);
+    compare_ne(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 #define float16_is_zero(ARG) 0
@@ -2404,7 +2417,7 @@ void helper_msa_fadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2434,7 +2447,7 @@ void helper_msa_fsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2464,7 +2477,7 @@ void helper_msa_fmul_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2495,7 +2508,7 @@ void helper_msa_fdiv_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2542,7 +2555,7 @@ void helper_msa_fmadd_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2577,7 +2590,7 @@ void helper_msa_fmsub_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2614,7 +2627,7 @@ void helper_msa_fexp2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2666,7 +2679,7 @@ void helper_msa_fexdo_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2712,7 +2725,7 @@ void helper_msa_ftq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2791,7 +2804,7 @@ void helper_msa_fmin_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2822,7 +2835,7 @@ void helper_msa_fmin_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2865,7 +2878,7 @@ void helper_msa_fmax_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2896,7 +2909,7 @@ void helper_msa_fmax_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2958,7 +2971,7 @@ void helper_msa_ftrunc_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2988,7 +3001,7 @@ void helper_msa_ftrunc_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3018,7 +3031,7 @@ void helper_msa_fsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3067,7 +3080,7 @@ void helper_msa_frsqrt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3097,7 +3110,7 @@ void helper_msa_frcp_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3127,7 +3140,7 @@ void helper_msa_frint_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3181,7 +3194,7 @@ void helper_msa_flog2_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3216,7 +3229,7 @@ void helper_msa_fexupl_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3250,7 +3263,7 @@ void helper_msa_fexupr_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3331,7 +3344,7 @@ void helper_msa_ftint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3361,7 +3374,7 @@ void helper_msa_ftint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3397,7 +3410,7 @@ void helper_msa_ffint_s_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3427,7 +3440,7 @@ void helper_msa_ffint_u_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
diff --git a/qemu/target-mips/op_helper.c b/qemu/target-mips/op_helper.c
index db4f6b946..4417e6ba2 100644
--- a/qemu/target-mips/op_helper.c
+++ b/qemu/target-mips/op_helper.c
@@ -16,55 +16,33 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
 #include "sysemu/kvm.h"
 
-#ifndef CONFIG_USER_ONLY
-static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
-#endif
-
 /*****************************************************************************/
 /* Exceptions processing helpers */
 
-static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
-                                                        uint32_t exception,
-                                                        int error_code,
-                                                        uintptr_t pc)
+void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
+                                int error_code)
 {
-    CPUState *cs = CPU(mips_env_get_cpu(env));
-
-    if (exception < EXCP_SC) {
-        qemu_log("%s: %d %d\n", __func__, exception, error_code);
-    }
-    cs->exception_index = exception;
-    env->error_code = error_code;
-
-    if (pc) {
-        /* now we have a real cpu fault */
-        cpu_restore_state(cs, pc);
-    }
-
-    cpu_loop_exit(cs);
+    do_raise_exception_err(env, exception, error_code, 0);
 }
 
-static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
-                                                    uint32_t exception,
-                                                    uintptr_t pc)
+void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
 {
-    do_raise_exception_err(env, exception, 0, pc);
+    do_raise_exception(env, exception, GETPC());
 }
 
-void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
-                                int error_code)
+void helper_raise_exception_debug(CPUMIPSState *env)
 {
-    do_raise_exception_err(env, exception, error_code, 0);
+    do_raise_exception(env, EXCP_DEBUG, 0);
 }
 
-void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
+static void raise_exception(CPUMIPSState *env, uint32_t exception)
 {
     do_raise_exception(env, exception, 0);
 }
@@ -72,21 +50,21 @@ void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
-    return (type) cpu_##insn##_data(env, addr);                         \
+    return (type) cpu_##insn##_data_ra(env, addr, retaddr);             \
 }
 #else
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: return (type) cpu_##insn##_kernel(env, addr); break;        \
-    case 1: return (type) cpu_##insn##_super(env, addr); break;         \
+    case 0: return (type) cpu_##insn##_kernel_ra(env, addr, retaddr);   \
+    case 1: return (type) cpu_##insn##_super_ra(env, addr, retaddr);    \
     default:                                                            \
-    case 2: return (type) cpu_##insn##_user(env, addr); break;          \
+    case 2: return (type) cpu_##insn##_user_ra(env, addr, retaddr);     \
     }                                                                   \
 }
 #endif
@@ -99,21 +77,21 @@ HELPER_LD(ld, ldq, int64_t)
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
-    cpu_##insn##_data(env, addr, val);                                  \
+    cpu_##insn##_data_ra(env, addr, val, retaddr);                      \
 }
 #else
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: cpu_##insn##_kernel(env, addr, val); break;                 \
-    case 1: cpu_##insn##_super(env, addr, val); break;                  \
+    case 0: cpu_##insn##_kernel_ra(env, addr, val, retaddr); break;     \
+    case 1: cpu_##insn##_super_ra(env, addr, val, retaddr); break;      \
     default:                                                            \
-    case 2: cpu_##insn##_user(env, addr, val); break;                   \
+    case 2: cpu_##insn##_user_ra(env, addr, val, retaddr); break;       \
     }                                                                   \
 }
 #endif
@@ -292,14 +270,15 @@ target_ulong helper_bitswap(target_ulong rt)
 
 static inline hwaddr do_translate_address(CPUMIPSState *env,
                                                       target_ulong address,
-                                                      int rw)
+                                                      int rw, uintptr_t retaddr)
 {
     hwaddr lladdr;
+    CPUState *cs = CPU(mips_env_get_cpu(env));
 
     lladdr = cpu_mips_translate_address(env, address, rw);
 
     if (lladdr == -1LL) {
-        cpu_loop_exit(CPU(mips_env_get_cpu(env)));
+        cpu_loop_exit_restore(cs, retaddr);
     } else {
         return lladdr;
     }
@@ -310,10 +289,10 @@ target_ulong helper_##name(CPUMIPSState *env, target_ulong arg, int mem_idx)  \
 {                                                                             \
     if (arg & almask) {                                                       \
         env->CP0_BadVAddr = arg;                                              \
-        helper_raise_exception(env, EXCP_AdEL);                               \
+        do_raise_exception(env, EXCP_AdEL, GETPC());                          \
     }                                                                         \
-    env->lladdr = do_translate_address(env, arg, 0);                          \
-    env->llval = do_##insn(env, arg, mem_idx);                                \
+    env->lladdr = do_translate_address(env, arg, 0, GETPC());                 \
+    env->llval = do_##insn(env, arg, mem_idx, GETPC());                       \
     return env->llval;                                                        \
 }
 HELPER_LD_ATOMIC(ll, lw, 0x3)
@@ -330,12 +309,12 @@ target_ulong helper_##name(CPUMIPSState *env, target_ulong arg1,              \
                                                                               \
     if (arg2 & almask) {                                                      \
         env->CP0_BadVAddr = arg2;                                             \
-        helper_raise_exception(env, EXCP_AdES);                               \
+        do_raise_exception(env, EXCP_AdES, GETPC());                          \
     }                                                                         \
-    if (do_translate_address(env, arg2, 1) == env->lladdr) {                  \
-        tmp = do_##ld_insn(env, arg2, mem_idx);                               \
+    if (do_translate_address(env, arg2, 1, GETPC()) == env->lladdr) {         \
+        tmp = do_##ld_insn(env, arg2, mem_idx, GETPC());                      \
         if (tmp == env->llval) {                                              \
-            do_##st_insn(env, arg2, arg1, mem_idx);                           \
+            do_##st_insn(env, arg2, arg1, mem_idx, GETPC());                  \
             return 1;                                                         \
         }                                                                     \
     }                                                                         \
@@ -359,31 +338,43 @@ HELPER_ST_ATOMIC(scd, ld, sd, 0x7)
 void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 0)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK(arg2) == 0) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK(arg2) == 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 }
 
 #if defined(TARGET_MIPS64)
@@ -399,55 +390,83 @@ void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
 void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) <= 6)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) <= 6) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 5)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) <= 5) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 4)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) <= 4) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 3)
-        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) <= 3) {
+        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 0)
-        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK64(arg2) <= 0) {
+        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) >= 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 4)
-        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) >= 4) {
+        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 5)
-        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) >= 5) {
+        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 6)
-        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) >= 6) {
+        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) == 7)
-        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
+    if (GET_LMASK64(arg2) == 7) {
+        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx,
+              GETPC());
+    }
 }
 #endif /* TARGET_MIPS64 */
 
@@ -464,13 +483,14 @@ void helper_lwm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
 
         for (i = 0; i < base_reglist; i++) {
             env->active_tc.gpr[multiple_regs[i]] =
-                (target_long)do_lw(env, addr, mem_idx);
+                (target_long)do_lw(env, addr, mem_idx, GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx);
+        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx,
+                                                    GETPC());
     }
 }
 
@@ -484,13 +504,14 @@ void helper_swm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        do_sw(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sw(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 
@@ -505,13 +526,14 @@ void helper_ldm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx);
+            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx,
+                                                         GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx);
+        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx, GETPC());
     }
 }
 
@@ -525,13 +547,14 @@ void helper_sdm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        do_sd(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sd(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 #endif
@@ -548,6 +571,14 @@ static bool mips_vpe_is_wfi(MIPSCPU *c)
     return cpu->halted && mips_vpe_active(env);
 }
 
+static bool mips_vp_is_wfi(MIPSCPU *c)
+{
+    CPUState *cpu = CPU(c);
+    CPUMIPSState *env = &c->env;
+
+    return cpu->halted && mips_vp_active(env);
+}
+
 static inline void mips_vpe_wake(MIPSCPU *c)
 {
     /* Dont set ->halted = 0 directly, let it be done via cpu_has_work
@@ -858,6 +889,16 @@ target_ulong helper_mfc0_lladdr(CPUMIPSState *env)
     return (int32_t)(env->lladdr >> env->CP0_LLAddr_shift);
 }
 
+target_ulong helper_mfc0_maar(CPUMIPSState *env)
+{
+    return (int32_t) env->CP0_MAAR[env->CP0_MAARI];
+}
+
+target_ulong helper_mfhc0_maar(CPUMIPSState *env)
+{
+    return env->CP0_MAAR[env->CP0_MAARI] >> 32;
+}
+
 target_ulong helper_mfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return (int32_t)env->CP0_WatchLo[sel];
@@ -924,6 +965,11 @@ target_ulong helper_dmfc0_lladdr(CPUMIPSState *env)
     return env->lladdr >> env->CP0_LLAddr_shift;
 }
 
+target_ulong helper_dmfc0_maar(CPUMIPSState *env)
+{
+    return env->CP0_MAAR[env->CP0_MAARI];
+}
+
 target_ulong helper_dmfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchLo[sel];
@@ -1330,6 +1376,13 @@ void helper_mtc0_hwrena(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0x0000000F;
 
+    if ((env->CP0_Config1 & (1 << CP0C1_PC)) &&
+        (env->insn_flags & ISA_MIPS32R6)) {
+        mask |= (1 << 4);
+    }
+    if (env->insn_flags & ISA_MIPS32R6) {
+        mask |= (1 << 5);
+    }
     if (env->CP0_Config3 & (1 << CP0C3_ULRI)) {
         mask |= (1 << 29);
 
@@ -1540,6 +1593,36 @@ void helper_mtc0_lladdr(CPUMIPSState *env, target_ulong arg1)
     env->lladdr = (env->lladdr & ~mask) | (arg1 & mask);
 }
 
+#define MTC0_MAAR_MASK(env) \
+        ((0x1ULL << 63) | ((env->PAMask >> 4) & ~0xFFFull) | 0x3)
+
+void helper_mtc0_maar(CPUMIPSState *env, target_ulong arg1)
+{
+    env->CP0_MAAR[env->CP0_MAARI] = arg1 & MTC0_MAAR_MASK(env);
+}
+
+void helper_mthc0_maar(CPUMIPSState *env, target_ulong arg1)
+{
+    env->CP0_MAAR[env->CP0_MAARI] =
+        (((uint64_t) arg1 << 32) & MTC0_MAAR_MASK(env)) |
+        (env->CP0_MAAR[env->CP0_MAARI] & 0x00000000ffffffffULL);
+}
+
+void helper_mtc0_maari(CPUMIPSState *env, target_ulong arg1)
+{
+    int index = arg1 & 0x3f;
+    if (index == 0x3f) {
+        /* Software may write all ones to INDEX to determine the
+           maximum value supported. */
+        env->CP0_MAARI = MIPS_MAAR_MAX - 1;
+    } else if (index < MIPS_MAAR_MAX) {
+        env->CP0_MAARI = index;
+    }
+    /* Other than the all ones, if the
+       value written is not supported, then INDEX is unchanged
+       from its previous value. */
+}
+
 void helper_mtc0_watchlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     /* Watch exceptions for instructions, data loads, data stores
@@ -1594,9 +1677,31 @@ void helper_mtc0_performance0(CPUMIPSState *env, target_ulong arg1)
     env->CP0_Performance0 = arg1 & 0x000007ff;
 }
 
+void helper_mtc0_errctl(CPUMIPSState *env, target_ulong arg1)
+{
+    int32_t wst = arg1 & (1 << CP0EC_WST);
+    int32_t spr = arg1 & (1 << CP0EC_SPR);
+    int32_t itc = env->itc_tag ? (arg1 & (1 << CP0EC_ITC)) : 0;
+
+    env->CP0_ErrCtl = wst | spr | itc;
+
+    if (itc && !wst && !spr) {
+        env->hflags |= MIPS_HFLAG_ITC_CACHE;
+    } else {
+        env->hflags &= ~MIPS_HFLAG_ITC_CACHE;
+    }
+}
+
 void helper_mtc0_taglo(CPUMIPSState *env, target_ulong arg1)
 {
-    env->CP0_TagLo = arg1 & 0xFFFFFCF6;
+    if (env->hflags & MIPS_HFLAG_ITC_CACHE) {
+        /* If CACHE instruction is configured for ITC tags then make all
+           CP0.TagLo bits writable. The actual write to ITC Configuration
+           Tag will take care of the read-only bits. */
+        env->CP0_TagLo = arg1;
+    } else {
+        env->CP0_TagLo = arg1 & 0xFFFFFCF6;
+    }
 }
 
 void helper_mtc0_datalo(CPUMIPSState *env, target_ulong arg1)
@@ -1791,13 +1896,13 @@ target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
                 env->active_tc.CP0_TCStatus & (1 << CP0TCSt_DT)) {
                 env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
                 env->CP0_VPEControl |= 4 << CP0VPECo_EXCPT;
-                helper_raise_exception(env, EXCP_THREAD);
+                do_raise_exception(env, EXCP_THREAD, GETPC());
             }
         }
     } else if (arg1 == 0) {
         if (0 /* TODO: TC underflow */) {
             env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
-            helper_raise_exception(env, EXCP_THREAD);
+            do_raise_exception(env, EXCP_THREAD, GETPC());
         } else {
             // TODO: Deallocate TC
         }
@@ -1805,22 +1910,53 @@ target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
         /* Yield qualifier inputs not implemented. */
         env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
         env->CP0_VPEControl |= 2 << CP0VPECo_EXCPT;
-        helper_raise_exception(env, EXCP_THREAD);
+        do_raise_exception(env, EXCP_THREAD, GETPC());
     }
     return env->CP0_YQMask;
 }
 
+/* R6 Multi-threading */
 #ifndef CONFIG_USER_ONLY
-/* TLB management */
-static void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global)
+target_ulong helper_dvp(CPUMIPSState *env)
 {
-    MIPSCPU *cpu = mips_env_get_cpu(env);
+    CPUState *other_cs = first_cpu;
+    target_ulong prev = env->CP0_VPControl;
+
+    if (!((env->CP0_VPControl >> CP0VPCtl_DIS) & 1)) {
+        CPU_FOREACH(other_cs) {
+            MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+            /* Turn off all VPs except the one executing the dvp. */
+            if (&other_cpu->env != env) {
+                mips_vpe_sleep(other_cpu);
+            }
+        }
+        env->CP0_VPControl |= (1 << CP0VPCtl_DIS);
+    }
+    return prev;
+}
 
-    /* Flush qemu's TLB and discard all shadowed entries.  */
-    tlb_flush(CPU(cpu), flush_global);
-    env->tlb->tlb_in_use = env->tlb->nb_tlb;
+target_ulong helper_evp(CPUMIPSState *env)
+{
+    CPUState *other_cs = first_cpu;
+    target_ulong prev = env->CP0_VPControl;
+
+    if ((env->CP0_VPControl >> CP0VPCtl_DIS) & 1) {
+        CPU_FOREACH(other_cs) {
+            MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+            if ((&other_cpu->env != env) && !mips_vp_is_wfi(other_cpu)) {
+                /* If the VP is WFI, don't disturb its sleep.
+                 * Otherwise, wake it up. */
+                mips_vpe_wake(other_cpu);
+            }
+        }
+        env->CP0_VPControl &= ~(1 << CP0VPCtl_DIS);
+    }
+    return prev;
 }
+#endif /* !CONFIG_USER_ONLY */
 
+#ifndef CONFIG_USER_ONLY
+/* TLB management */
 static void r4k_mips_tlb_flush_extra (CPUMIPSState *env, int first)
 {
     /* Discard entries from env->tlb[first] onwards.  */
@@ -2158,48 +2294,52 @@ void helper_deret(CPUMIPSState *env)
 }
 #endif /* !CONFIG_USER_ONLY */
 
-target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
+static inline void check_hwrena(CPUMIPSState *env, int reg, uintptr_t pc)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 0)))
-        return env->CP0_EBase & 0x3ff;
-    else
-        helper_raise_exception(env, EXCP_RI);
+    if ((env->hflags & MIPS_HFLAG_CP0) || (env->CP0_HWREna & (1 << reg))) {
+        return;
+    }
+    do_raise_exception(env, EXCP_RI, pc);
+}
 
-    return 0;
+target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
+{
+    check_hwrena(env, 0, GETPC());
+    return env->CP0_EBase & 0x3ff;
 }
 
 target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 1)))
-        return env->SYNCI_Step;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 1, GETPC());
+    return env->SYNCI_Step;
 }
 
 target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 2)))
-        return env->CP0_Count;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 2, GETPC());
+#ifdef CONFIG_USER_ONLY
+    return env->CP0_Count;
+#else
+    return (int32_t)cpu_mips_get_count(env);
+#endif
 }
 
 target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 3)))
-        return env->CCRes;
-    else
-        helper_raise_exception(env, EXCP_RI);
+    check_hwrena(env, 3, GETPC());
+    return env->CCRes;
+}
 
-    return 0;
+target_ulong helper_rdhwr_performance(CPUMIPSState *env)
+{
+    check_hwrena(env, 4, GETPC());
+    return env->CP0_Performance0;
+}
+
+target_ulong helper_rdhwr_xnp(CPUMIPSState *env)
+{
+    check_hwrena(env, 5, GETPC());
+    return (env->CP0_Config5 >> CP0C5_XNP) & 1;
 }
 
 void helper_pmon(CPUMIPSState *env, int function)
@@ -2234,7 +2374,9 @@ void helper_wait(CPUMIPSState *env)
 
     cs->halted = 1;
     cpu_reset_interrupt(cs, CPU_INTERRUPT_WAKE);
-    helper_raise_exception(env, EXCP_HLT);
+    /* Last instruction in the block, PC was updated before
+       - no need to recover PC and icount */
+    raise_exception(env, EXCP_HLT);
 }
 
 #if !defined(CONFIG_USER_ONLY)
@@ -2295,9 +2437,9 @@ void mips_cpu_unassigned_access(CPUState *cs, hwaddr addr,
     }
 
     if (is_exec) {
-        helper_raise_exception(env, EXCP_IBE);
+        raise_exception(env, EXCP_IBE);
     } else {
-        helper_raise_exception(env, EXCP_DBE);
+        raise_exception(env, EXCP_DBE);
     }
 }
 #endif /* !CONFIG_USER_ONLY */
@@ -2332,7 +2474,7 @@ target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
                 arg1 = (int32_t)
                        ((env->CP0_Status & (1  << CP0St_FR)) >> CP0St_FR);
             } else {
-                helper_raise_exception(env, EXCP_RI);
+                do_raise_exception(env, EXCP_RI, GETPC());
             }
         }
         break;
@@ -2375,7 +2517,7 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
             env->CP0_Status &= ~(1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 4:
@@ -2387,7 +2529,7 @@ void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t fs, uint32_t rt)
             env->CP0_Status |= (1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 5:
@@ -2518,6 +2660,7 @@ uint64_t helper_float_cvtd_s(CPUMIPSState *env, uint32_t fst0)
     uint64_t fdt2;
 
     fdt2 = float32_to_float64(fst0, &env->active_fpu.fp_status);
+    fdt2 = float64_maybe_silence_nan(fdt2);
     update_fcr31(env, GETPC());
     return fdt2;
 }
@@ -2607,6 +2750,7 @@ uint32_t helper_float_cvts_d(CPUMIPSState *env, uint64_t fdt0)
     uint32_t fst2;
 
     fst2 = float64_to_float32(fdt0, &env->active_fpu.fp_status);
+    fst2 = float32_maybe_silence_nan(fst2);
     update_fcr31(env, GETPC());
     return fst2;
 }
@@ -3628,7 +3772,7 @@ FOP_CONDN_S(sne,  (float32_lt(fst1, fst0, &env->active_fpu.fp_status)
 #if !defined(CONFIG_USER_ONLY)
 #define MEMOP_IDX(DF)                                           \
         TCGMemOpIdx oi = make_memop_idx(MO_TE | DF | MO_UNALN,  \
-                                        cpu_mmu_index(env));
+                                        cpu_mmu_index(env, false));
 #else
 #define MEMOP_IDX(DF)
 #endif
@@ -3684,7 +3828,7 @@ void helper_msa_st_ ## TYPE(CPUMIPSState *env, uint32_t wd,             \
                             target_ulong addr)                          \
 {                                                                       \
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
-    int mmu_idx = cpu_mmu_index(env);                                   \
+    int mmu_idx = cpu_mmu_index(env, false);				\
     int i;                                                              \
     MEMOP_IDX(DF)                                                       \
     ensure_writable_pages(env, addr, mmu_idx, GETRA());                 \
@@ -3704,3 +3848,19 @@ MSA_ST_DF(DF_HALF,   h, cpu_stw_data)
 MSA_ST_DF(DF_WORD,   w, cpu_stl_data)
 MSA_ST_DF(DF_DOUBLE, d, cpu_stq_data)
 #endif
+
+void helper_cache(CPUMIPSState *env, target_ulong addr, uint32_t op)
+{
+#ifndef CONFIG_USER_ONLY
+    target_ulong index = addr & 0x1fffffff;
+    if (op == 9) {
+        /* Index Store Tag */
+        memory_region_dispatch_write(env->itc_tag, index, env->CP0_TagLo,
+                                     8, MEMTXATTRS_UNSPECIFIED);
+    } else if (op == 5) {
+        /* Index Load Tag */
+        memory_region_dispatch_read(env->itc_tag, index, &env->CP0_TagLo,
+                                    8, MEMTXATTRS_UNSPECIFIED);
+    }
+#endif
+}
diff --git a/qemu/target-mips/translate.c b/qemu/target-mips/translate.c
index 22ef84df9..a3a05ec66 100644
--- a/qemu/target-mips/translate.c
+++ b/qemu/target-mips/translate.c
@@ -21,6 +21,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -32,10 +33,9 @@
 #include "exec/semihost.h"
 
 #include "trace-tcg.h"
-
+#include "exec/log.h"
 
 #define MIPS_DEBUG_DISAS 0
-//#define MIPS_DEBUG_SIGN_EXTENSIONS
 
 /* MIPS major opcodes */
 #define MASK_OP_MAJOR(op)  (op & (0x3F << 26))
@@ -325,6 +325,7 @@ enum {
     OPC_TLTIU    = (0x0B << 16) | OPC_REGIMM,
     OPC_TEQI     = (0x0C << 16) | OPC_REGIMM,
     OPC_TNEI     = (0x0E << 16) | OPC_REGIMM,
+    OPC_SIGRIE   = (0x17 << 16) | OPC_REGIMM,
     OPC_SYNCI    = (0x1F << 16) | OPC_REGIMM,
 
     OPC_DAHI     = (0x06 << 16) | OPC_REGIMM,
@@ -893,6 +894,8 @@ enum {
     OPC_EVPE     = 0x01 | (1 << 5) | OPC_MFMC0,
     OPC_DI       = (0 << 5) | (0x0C << 11) | OPC_MFMC0,
     OPC_EI       = (1 << 5) | (0x0C << 11) | OPC_MFMC0,
+    OPC_DVP      = 0x04 | (0 << 3) | (1 << 5) | (0 << 11) | OPC_MFMC0,
+    OPC_EVP      = 0x04 | (0 << 3) | (0 << 5) | (0 << 11) | OPC_MFMC0,
 };
 
 /* Coprocessor 0 (with rs == C0) */
@@ -1352,7 +1355,7 @@ enum {
 };
 
 /* global register indices */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_gpr[32], cpu_PC;
 static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC];
 static TCGv cpu_dspctrl, btarget, bcond;
@@ -1361,9 +1364,6 @@ static TCGv_i32 fpu_fcr0, fpu_fcr31;
 static TCGv_i64 fpu_f64[32];
 static TCGv_i64 msa_wr_d[64];
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #define gen_helper_0e0i(name, arg) do {                           \
@@ -1431,6 +1431,9 @@ typedef struct DisasContext {
     bool mvh;
     int CP0_LLAddr_shift;
     bool ps;
+    bool vp;
+    bool cmgcr;
+    bool mrp;
 } DisasContext;
 
 enum {
@@ -1482,26 +1485,23 @@ static const char * const msaregnames[] = {
     "w30.d0", "w30.d1", "w31.d0", "w31.d1",
 };
 
-#define MIPS_DEBUG(fmt, ...)                                                  \
+#define LOG_DISAS(...)                                                        \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
-            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
-                          TARGET_FMT_lx ": %08x " fmt "\n",                   \
-                          ctx->pc, ctx->opcode , ## __VA_ARGS__);             \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
         }                                                                     \
     } while (0)
 
-#define LOG_DISAS(...)                                                        \
+#define MIPS_INVAL(op)                                                        \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
-            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
+                          TARGET_FMT_lx ": %08x Invalid %s %03x %03x %03x\n", \
+                          ctx->pc, ctx->opcode, op, ctx->opcode >> 26,        \
+                          ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F));  \
         }                                                                     \
     } while (0)
 
-#define MIPS_INVAL(op)                                                        \
-    MIPS_DEBUG("Invalid %s %03x %03x %03x", op, ctx->opcode >> 26,            \
-               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F))
-
 /* General purpose registers moves. */
 static inline void gen_load_gpr (TCGv t, int reg)
 {
@@ -1615,21 +1615,26 @@ static inline void generate_exception_err(DisasContext *ctx, int excp, int err)
     gen_helper_raise_exception_err(cpu_env, texcp, terr);
     tcg_temp_free_i32(terr);
     tcg_temp_free_i32(texcp);
+    ctx->bstate = BS_EXCP;
 }
 
 static inline void generate_exception(DisasContext *ctx, int excp)
 {
-    save_cpu_state(ctx, 1);
     gen_helper_0e0i(raise_exception, excp);
 }
 
+static inline void generate_exception_end(DisasContext *ctx, int excp)
+{
+    generate_exception_err(ctx, excp, 0);
+}
+
 /* Floating point register moves. */
 static void gen_load_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_FRE) {
         generate_exception(ctx, EXCP_RI);
     }
-    tcg_gen_trunc_i64_i32(t, fpu_f64[reg]);
+    tcg_gen_extrl_i64_i32(t, fpu_f64[reg]);
 }
 
 static void gen_store_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
@@ -1647,10 +1652,7 @@ static void gen_store_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
 static void gen_load_fpr32h(DisasContext *ctx, TCGv_i32 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        TCGv_i64 t64 = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t64, fpu_f64[reg], 32);
-        tcg_gen_trunc_i64_i32(t, t64);
-        tcg_temp_free_i64(t64);
+        tcg_gen_extrh_i64_i32(t, fpu_f64[reg]);
     } else {
         gen_load_fpr32(ctx, t, reg | 1);
     }
@@ -1725,12 +1727,23 @@ static target_long addr_add(DisasContext *ctx, target_long base,
     return sum;
 }
 
+/* Sign-extract the low 32-bits to a target_long.  */
 static inline void gen_move_low32(TCGv ret, TCGv_i64 arg)
 {
 #if defined(TARGET_MIPS64)
-    tcg_gen_ext32s_tl(ret, arg);
+    tcg_gen_ext32s_i64(ret, arg);
+#else
+    tcg_gen_extrl_i64_i32(ret, arg);
+#endif
+}
+
+/* Sign-extract the high 32-bits to a target_long.  */
+static inline void gen_move_high32(TCGv ret, TCGv_i64 arg)
+{
+#if defined(TARGET_MIPS64)
+    tcg_gen_sari_i64(ret, arg, 32);
 #else
-    tcg_gen_trunc_i64_tl(ret, arg);
+    tcg_gen_extrh_i64_i32(ret, arg);
 #endif
 }
 
@@ -1753,7 +1766,7 @@ static inline void check_cp1_enabled(DisasContext *ctx)
 static inline void check_cop1x(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with 64-bit floating-point
@@ -1762,7 +1775,7 @@ static inline void check_cop1x(DisasContext *ctx)
 static inline void check_cp1_64bitmode(DisasContext *ctx)
 {
     if (unlikely(~ctx->hflags & (MIPS_HFLAG_F64 | MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /*
@@ -1779,7 +1792,7 @@ static inline void check_cp1_64bitmode(DisasContext *ctx)
 static inline void check_cp1_registers(DisasContext *ctx, int regs)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_F64) && (regs & 1)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with DSP instructions enabled.
@@ -1790,9 +1803,9 @@ static inline void check_dsp(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -1801,9 +1814,9 @@ static inline void check_dspr2(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -1813,7 +1826,7 @@ static inline void check_dspr2(DisasContext *ctx)
 static inline void check_insn(DisasContext *ctx, int flags)
 {
     if (unlikely(!(ctx->insn_flags & flags))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
 }
 
@@ -1823,7 +1836,7 @@ static inline void check_insn(DisasContext *ctx, int flags)
 static inline void check_insn_opc_removed(DisasContext *ctx, int flags)
 {
     if (unlikely(ctx->insn_flags & flags)) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
 }
 
@@ -1843,7 +1856,7 @@ static inline void check_ps(DisasContext *ctx)
 static inline void check_mips_64(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_64)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 #endif
 
@@ -2052,7 +2065,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
     tcg_gen_movi_tl(t0, rt | ((almask << 3) & 0x20));                        \
     tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, llreg));                   \
     tcg_gen_st_tl(arg1, cpu_env, offsetof(CPUMIPSState, llnewval));              \
-    gen_helper_0e0i(raise_exception, EXCP_SC);                               \
+    generate_exception_end(ctx, EXCP_SC);                                    \
     gen_set_label(l2);                                                       \
     tcg_gen_movi_tl(t0, 0);                                                  \
     gen_store_gpr(t0, rt);                                                   \
@@ -2105,14 +2118,12 @@ static target_ulong pc_relative_pc (DisasContext *ctx)
 static void gen_ld(DisasContext *ctx, uint32_t opc,
                    int rt, int base, int16_t offset)
 {
-    const char *opn = "ld";
     TCGv t0, t1, t2;
 
     if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
            We emulate it as a NOP. On other CPU we must perform the
            actual memory access. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2125,20 +2136,16 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lwu";
         break;
     case OPC_LD:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "ld";
         break;
     case OPC_LLD:
     case R6_OPC_LLD:
-        save_cpu_state(ctx, 1);
         op_ld_lld(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "lld";
         break;
     case OPC_LDL:
         t1 = tcg_temp_new();
@@ -2153,16 +2160,14 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, ~7);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 63);
-        t2 = tcg_const_tl(0x7fffffffffffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldl";
         break;
     case OPC_LDR:
         t1 = tcg_temp_new();
@@ -2186,7 +2191,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldr";
         break;
     case OPC_LDPC:
         t1 = tcg_const_tl(pc_relative_pc(ctx));
@@ -2194,7 +2198,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rt);
-        opn = "ldpc";
         break;
 #endif
     case OPC_LWPC:
@@ -2203,35 +2206,29 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rt);
-        opn = "lwpc";
         break;
     case OPC_LW:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lw";
         break;
     case OPC_LH:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lh";
         break;
     case OPC_LHU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lhu";
         break;
     case OPC_LB:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_SB);
         gen_store_gpr(t0, rt);
-        opn = "lb";
         break;
     case OPC_LBU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rt);
-        opn = "lbu";
         break;
     case OPC_LWL:
         t1 = tcg_temp_new();
@@ -2246,17 +2243,15 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, ~3);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 31);
-        t2 = tcg_const_tl(0x7fffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwl";
         break;
     case OPC_LWR:
         t1 = tcg_temp_new();
@@ -2281,18 +2276,13 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwr";
         break;
     case OPC_LL:
     case R6_OPC_LL:
-        save_cpu_state(ctx, 1);
         op_ld_ll(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "ll";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
 }
 
@@ -2300,7 +2290,6 @@ static void gen_ld(DisasContext *ctx, uint32_t opc,
 static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
                     int base, int16_t offset)
 {
-    const char *opn = "st";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -2311,46 +2300,32 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
     case OPC_SD:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
-        opn = "sd";
         break;
     case OPC_SDL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
-        opn = "sdl";
         break;
     case OPC_SDR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
-        opn = "sdr";
         break;
 #endif
     case OPC_SW:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
-        opn = "sw";
         break;
     case OPC_SH:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
-        opn = "sh";
         break;
     case OPC_SB:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_8);
-        opn = "sb";
         break;
     case OPC_SWL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
-        opn = "swl";
         break;
     case OPC_SWR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
-        opn = "swr";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -2360,7 +2335,6 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
 static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
                          int base, int16_t offset)
 {
-    const char *opn = "st_cond";
     TCGv t0, t1;
 
 #ifdef CONFIG_USER_ONLY
@@ -2376,20 +2350,14 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
 #if defined(TARGET_MIPS64)
     case OPC_SCD:
     case R6_OPC_SCD:
-        save_cpu_state(ctx, 1);
         op_st_scd(t1, t0, rt, ctx);
-        opn = "scd";
         break;
 #endif
     case OPC_SC:
     case R6_OPC_SC:
-        save_cpu_state(ctx, 1);
         op_st_sc(t1, t0, rt, ctx);
-        opn = "sc";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
 }
@@ -2398,7 +2366,6 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
 static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                           int base, int16_t offset)
 {
-    const char *opn = "flt_ldst";
     TCGv t0 = tcg_temp_new();
 
     gen_base_offset_addr(ctx, t0, base, offset);
@@ -2413,7 +2380,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
             gen_store_fpr32(ctx, fp0, ft);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwc1";
         break;
     case OPC_SWC1:
         {
@@ -2423,7 +2389,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swc1";
         break;
     case OPC_LDC1:
         {
@@ -2433,7 +2398,6 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
             gen_store_fpr64(ctx, fp0, ft);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldc1";
         break;
     case OPC_SDC1:
         {
@@ -2443,15 +2407,12 @@ static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("flt_ldst");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, fregnames[ft], offset, regnames[base]);
  out:
     tcg_temp_free(t0);
 }
@@ -2479,12 +2440,10 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
 
     if (rt == 0 && opc != OPC_ADDI && opc != OPC_DADDI) {
         /* If no destination, treat it as a NOP.
            For addi, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
     switch (opc) {
@@ -2512,7 +2471,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "addi";
         break;
     case OPC_ADDIU:
         if (rs != 0) {
@@ -2521,7 +2479,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "addiu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADDI:
@@ -2546,7 +2503,6 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "daddi";
         break;
     case OPC_DADDIU:
         if (rs != 0) {
@@ -2554,12 +2510,9 @@ static void gen_arith_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "daddiu";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
 }
 
 /* Logic with immediate operand */
@@ -2570,7 +2523,6 @@ static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     uimm = (uint16_t)imm;
@@ -2580,39 +2532,30 @@ static void gen_logic_imm(DisasContext *ctx, uint32_t opc,
             tcg_gen_andi_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], 0);
-        MIPS_DEBUG("andi %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_ORI:
         if (rs != 0)
             tcg_gen_ori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("ori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_XORI:
         if (likely(rs != 0))
             tcg_gen_xori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("xori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_LUI:
         if (rs != 0 && (ctx->insn_flags & ISA_MIPS32R6)) {
             /* OPC_AUI */
             tcg_gen_addi_tl(cpu_gpr[rt], cpu_gpr[rs], imm << 16);
             tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
-            MIPS_DEBUG("aui %s, %s, %04x", regnames[rt], regnames[rs], imm);
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-            MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
         }
         break;
 
     default:
-        MIPS_DEBUG("Unknown logical immediate opcode %08x", opc);
         break;
     }
 }
@@ -2622,12 +2565,10 @@ static void gen_slt_imm(DisasContext *ctx, uint32_t opc,
                         int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -2635,15 +2576,11 @@ static void gen_slt_imm(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_SLTI:
         tcg_gen_setcondi_tl(TCG_COND_LT, cpu_gpr[rt], t0, uimm);
-        opn = "slti";
         break;
     case OPC_SLTIU:
         tcg_gen_setcondi_tl(TCG_COND_LTU, cpu_gpr[rt], t0, uimm);
-        opn = "sltiu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -2652,12 +2589,10 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = ((uint16_t)imm) & 0x1f;
-    const char *opn = "imm shift";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2667,11 +2602,9 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
     case OPC_SLL:
         tcg_gen_shli_tl(t0, t0, uimm);
         tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
-        opn = "sll";
         break;
     case OPC_SRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "sra";
         break;
     case OPC_SRL:
         if (uimm != 0) {
@@ -2680,7 +2613,6 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "srl";
         break;
     case OPC_ROTR:
         if (uimm != 0) {
@@ -2693,20 +2625,16 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "rotr";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLL:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsll";
         break;
     case OPC_DSRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsra";
         break;
     case OPC_DSRL:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsrl";
         break;
     case OPC_DROTR:
         if (uimm != 0) {
@@ -2714,28 +2642,21 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_mov_tl(cpu_gpr[rt], t0);
         }
-        opn = "drotr";
         break;
     case OPC_DSLL32:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsll32";
         break;
     case OPC_DSRA32:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsra32";
         break;
     case OPC_DSRL32:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsrl32";
         break;
     case OPC_DROTR32:
         tcg_gen_rotri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "drotr32";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -2743,13 +2664,10 @@ static void gen_shift_imm(DisasContext *ctx, uint32_t opc,
 static void gen_arith(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "arith";
-
     if (rd == 0 && opc != OPC_ADD && opc != OPC_SUB
        && opc != OPC_DADD && opc != OPC_DSUB) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2777,7 +2695,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "add";
         break;
     case OPC_ADDU:
         if (rs != 0 && rt != 0) {
@@ -2790,7 +2707,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "addu";
         break;
     case OPC_SUB:
         {
@@ -2815,7 +2731,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "sub";
         break;
     case OPC_SUBU:
         if (rs != 0 && rt != 0) {
@@ -2829,7 +2744,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "subu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADD:
@@ -2854,7 +2768,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dadd";
         break;
     case OPC_DADDU:
         if (rs != 0 && rt != 0) {
@@ -2866,7 +2779,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "daddu";
         break;
     case OPC_DSUB:
         {
@@ -2890,7 +2802,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dsub";
         break;
     case OPC_DSUBU:
         if (rs != 0 && rt != 0) {
@@ -2902,7 +2813,6 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "dsubu";
         break;
 #endif
     case OPC_MUL:
@@ -2912,23 +2822,18 @@ static void gen_arith(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "mul";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Conditional move */
 static void gen_cond_move(DisasContext *ctx, uint32_t opc,
                           int rd, int rs, int rt)
 {
-    const char *opn = "cond move";
     TCGv t0, t1, t2;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2940,38 +2845,28 @@ static void gen_cond_move(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_MOVN:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movn";
         break;
     case OPC_MOVZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movz";
         break;
     case OPC_SELNEZ:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "selnez";
         break;
     case OPC_SELEQZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "seleqz";
         break;
     }
     tcg_temp_free(t2);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Logic */
 static void gen_logic(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "logic";
-
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2982,7 +2877,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "and";
         break;
     case OPC_NOR:
         if (rs != 0 && rt != 0) {
@@ -2994,7 +2888,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], ~((target_ulong)0));
         }
-        opn = "nor";
         break;
     case OPC_OR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3006,7 +2899,6 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "or";
         break;
     case OPC_XOR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3018,23 +2910,18 @@ static void gen_logic(DisasContext *ctx, uint32_t opc,
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "xor";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Set on lower than */
 static void gen_slt(DisasContext *ctx, uint32_t opc,
                     int rd, int rs, int rt)
 {
-    const char *opn = "slt";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3045,15 +2932,11 @@ static void gen_slt(DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_SLT:
         tcg_gen_setcond_tl(TCG_COND_LT, cpu_gpr[rd], t0, t1);
-        opn = "slt";
         break;
     case OPC_SLTU:
         tcg_gen_setcond_tl(TCG_COND_LTU, cpu_gpr[rd], t0, t1);
-        opn = "sltu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3062,13 +2945,11 @@ static void gen_slt(DisasContext *ctx, uint32_t opc,
 static void gen_shift(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "shifts";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3081,19 +2962,16 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shl_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "sllv";
         break;
     case OPC_SRAV:
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "srav";
         break;
     case OPC_SRLV:
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shr_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "srlv";
         break;
     case OPC_ROTRV:
         {
@@ -3107,34 +2985,27 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext_i32_tl(cpu_gpr[rd], t2);
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
-            opn = "rotrv";
         }
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shl_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsllv";
         break;
     case OPC_DSRAV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrav";
         break;
     case OPC_DSRLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shr_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrlv";
         break;
     case OPC_DROTRV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_rotr_tl(cpu_gpr[rd], t1, t0);
-        opn = "drotrv";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3142,11 +3013,8 @@ static void gen_shift(DisasContext *ctx, uint32_t opc,
 /* Arithmetic on HI/LO registers */
 static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
 {
-    const char *opn = "hilo";
-
     if (reg == 0 && (opc == OPC_MFHI || opc == OPC_MFLO)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3164,7 +3032,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[acc]);
         }
-        opn = "mfhi";
         break;
     case OPC_MFLO:
 #if defined(TARGET_MIPS64)
@@ -3175,7 +3042,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[acc]);
         }
-        opn = "mflo";
         break;
     case OPC_MTHI:
         if (reg != 0) {
@@ -3190,7 +3056,6 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         } else {
             tcg_gen_movi_tl(cpu_HI[acc], 0);
         }
-        opn = "mthi";
         break;
     case OPC_MTLO:
         if (reg != 0) {
@@ -3205,11 +3070,8 @@ static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
         } else {
             tcg_gen_movi_tl(cpu_LO[acc], 0);
         }
-        opn = "mtlo";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s", opn, regnames[reg]);
 }
 
 static inline void gen_r6_ld(target_long addr, int reg, int memidx,
@@ -3277,7 +3139,7 @@ static inline void gen_pcrel(DisasContext *ctx, int opc, target_ulong pc,
 #endif
         default:
             MIPS_INVAL("OPC_PCREL");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -3286,12 +3148,10 @@ static inline void gen_pcrel(DisasContext *ctx, int opc, target_ulong pc,
 
 static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
 {
-    const char *opn = "r6 mul/div";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3320,7 +3180,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case R6_OPC_MOD:
         {
@@ -3340,7 +3199,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "mod";
         break;
     case R6_OPC_DIVU:
         {
@@ -3354,7 +3212,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case R6_OPC_MODU:
         {
@@ -3368,7 +3225,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "modu";
         break;
     case R6_OPC_MUL:
         {
@@ -3381,7 +3237,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mul";
         break;
     case R6_OPC_MUH:
         {
@@ -3394,7 +3249,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muh";
         break;
     case R6_OPC_MULU:
         {
@@ -3407,7 +3261,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mulu";
         break;
     case R6_OPC_MUHU:
         {
@@ -3420,7 +3273,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muhu";
         break;
 #if defined(TARGET_MIPS64)
     case R6_OPC_DDIV:
@@ -3438,7 +3290,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case R6_OPC_DMOD:
         {
@@ -3455,7 +3306,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmod";
         break;
     case R6_OPC_DDIVU:
         {
@@ -3466,7 +3316,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case R6_OPC_DMODU:
         {
@@ -3477,11 +3326,9 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmodu";
         break;
     case R6_OPC_DMUL:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmul";
         break;
     case R6_OPC_DMUH:
         {
@@ -3489,11 +3336,9 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_gen_muls2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuh";
         break;
     case R6_OPC_DMULU:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmulu";
         break;
     case R6_OPC_DMUHU:
         {
@@ -3501,16 +3346,13 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
             tcg_gen_mulu2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuhu";
         break;
 #endif
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("r6 mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -3519,7 +3361,6 @@ static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
 static void gen_muldiv(DisasContext *ctx, uint32_t opc,
                        int acc, int rs, int rt)
 {
-    const char *opn = "mul/div";
     TCGv t0, t1;
 
     t0 = tcg_temp_new();
@@ -3553,7 +3394,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case OPC_DIVU:
         {
@@ -3569,7 +3409,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case OPC_MULT:
         {
@@ -3583,7 +3422,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mult";
         break;
     case OPC_MULTU:
         {
@@ -3597,7 +3435,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "multu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DDIV:
@@ -3616,7 +3453,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case OPC_DDIVU:
         {
@@ -3628,15 +3464,12 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case OPC_DMULT:
         tcg_gen_muls2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmult";
         break;
     case OPC_DMULTU:
         tcg_gen_mulu2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmultu";
         break;
 #endif
     case OPC_MADD:
@@ -3650,14 +3483,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "madd";
         break;
     case OPC_MADDU:
         {
@@ -3672,14 +3501,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "maddu";
         break;
     case OPC_MSUB:
         {
@@ -3692,14 +3517,10 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msub";
         break;
     case OPC_MSUBU:
         {
@@ -3714,22 +3535,16 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msubu";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -3738,7 +3553,6 @@ static void gen_muldiv(DisasContext *ctx, uint32_t opc,
 static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
                             int rd, int rs, int rt)
 {
-    const char *opn = "mul vr54xx";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -3748,68 +3562,52 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
     switch (opc) {
     case OPC_VR54XX_MULS:
         gen_helper_muls(t0, cpu_env, t0, t1);
-        opn = "muls";
         break;
     case OPC_VR54XX_MULSU:
         gen_helper_mulsu(t0, cpu_env, t0, t1);
-        opn = "mulsu";
         break;
     case OPC_VR54XX_MACC:
         gen_helper_macc(t0, cpu_env, t0, t1);
-        opn = "macc";
         break;
     case OPC_VR54XX_MACCU:
         gen_helper_maccu(t0, cpu_env, t0, t1);
-        opn = "maccu";
         break;
     case OPC_VR54XX_MSAC:
         gen_helper_msac(t0, cpu_env, t0, t1);
-        opn = "msac";
         break;
     case OPC_VR54XX_MSACU:
         gen_helper_msacu(t0, cpu_env, t0, t1);
-        opn = "msacu";
         break;
     case OPC_VR54XX_MULHI:
         gen_helper_mulhi(t0, cpu_env, t0, t1);
-        opn = "mulhi";
         break;
     case OPC_VR54XX_MULHIU:
         gen_helper_mulhiu(t0, cpu_env, t0, t1);
-        opn = "mulhiu";
         break;
     case OPC_VR54XX_MULSHI:
         gen_helper_mulshi(t0, cpu_env, t0, t1);
-        opn = "mulshi";
         break;
     case OPC_VR54XX_MULSHIU:
         gen_helper_mulshiu(t0, cpu_env, t0, t1);
-        opn = "mulshiu";
         break;
     case OPC_VR54XX_MACCHI:
         gen_helper_macchi(t0, cpu_env, t0, t1);
-        opn = "macchi";
         break;
     case OPC_VR54XX_MACCHIU:
         gen_helper_macchiu(t0, cpu_env, t0, t1);
-        opn = "macchiu";
         break;
     case OPC_VR54XX_MSACHI:
         gen_helper_msachi(t0, cpu_env, t0, t1);
-        opn = "msachi";
         break;
     case OPC_VR54XX_MSACHIU:
         gen_helper_msachiu(t0, cpu_env, t0, t1);
-        opn = "msachiu";
         break;
     default:
         MIPS_INVAL("mul vr54xx");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     gen_store_gpr(t0, rd);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 
  out:
     tcg_temp_free(t0);
@@ -3819,12 +3617,10 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
 static void gen_cl (DisasContext *ctx, uint32_t opc,
                     int rd, int rs)
 {
-    const char *opn = "CLx";
     TCGv t0;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -3833,28 +3629,22 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
     case OPC_CLO:
     case R6_OPC_CLO:
         gen_helper_clo(cpu_gpr[rd], t0);
-        opn = "clo";
         break;
     case OPC_CLZ:
     case R6_OPC_CLZ:
         gen_helper_clz(cpu_gpr[rd], t0);
-        opn = "clz";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DCLO:
     case R6_OPC_DCLO:
         gen_helper_dclo(cpu_gpr[rd], t0);
-        opn = "dclo";
         break;
     case OPC_DCLZ:
     case R6_OPC_DCLZ:
         gen_helper_dclz(cpu_gpr[rd], t0);
-        opn = "dclz";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
 }
 
@@ -3862,12 +3652,10 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
 static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
                                  int rd, int rs, int rt)
 {
-    const char *opn = "loongson";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3899,7 +3687,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
     case OPC_MULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "mult.g";
         break;
     case OPC_MULTU_G_2E:
     case OPC_MULTU_G_2F:
@@ -3907,7 +3694,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "multu.g";
         break;
     case OPC_DIV_G_2E:
     case OPC_DIV_G_2F:
@@ -3930,7 +3716,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "div.g";
         break;
     case OPC_DIVU_G_2E:
     case OPC_DIVU_G_2F:
@@ -3947,7 +3732,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "divu.g";
         break;
     case OPC_MOD_G_2E:
     case OPC_MOD_G_2F:
@@ -3968,7 +3752,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "mod.g";
         break;
     case OPC_MODU_G_2E:
     case OPC_MODU_G_2F:
@@ -3985,18 +3768,15 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "modu.g";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMULT_G_2E:
     case OPC_DMULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmult.g";
         break;
     case OPC_DMULTU_G_2E:
     case OPC_DMULTU_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmultu.g";
         break;
     case OPC_DDIV_G_2E:
     case OPC_DDIV_G_2F:
@@ -4016,7 +3796,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_div_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "ddiv.g";
         break;
     case OPC_DDIVU_G_2E:
     case OPC_DDIVU_G_2F:
@@ -4030,7 +3809,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_divu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "ddivu.g";
         break;
     case OPC_DMOD_G_2E:
     case OPC_DMOD_G_2F:
@@ -4048,7 +3826,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_rem_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "dmod.g";
         break;
     case OPC_DMODU_G_2E:
     case OPC_DMODU_G_2F:
@@ -4062,13 +3839,10 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
             tcg_gen_remu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "dmodu.g";
         break;
 #endif
     }
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -4076,7 +3850,6 @@ static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
 /* Loongson multimedia instructions */
 static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 {
-    const char *opn = "loongson_cp2";
     uint32_t opc, shift_max;
     TCGv_i64 t0, t1;
 
@@ -4099,11 +3872,11 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
     gen_load_fpr64(ctx, t1, rt);
 
 #define LMI_HELPER(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0, t1); break
 #define LMI_HELPER_1(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0); break
 #define LMI_DIRECT(UP, LO, OP) \
-    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); break
 
     switch (opc) {
     LMI_HELPER(PADDSH, paddsh);
@@ -4174,19 +3947,15 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 
     case OPC_PINSRH_0:
         tcg_gen_deposit_i64(t0, t0, t1, 0, 16);
-        opn = "pinsrh_0";
         break;
     case OPC_PINSRH_1:
         tcg_gen_deposit_i64(t0, t0, t1, 16, 16);
-        opn = "pinsrh_1";
         break;
     case OPC_PINSRH_2:
         tcg_gen_deposit_i64(t0, t0, t1, 32, 16);
-        opn = "pinsrh_2";
         break;
     case OPC_PINSRH_3:
         tcg_gen_deposit_i64(t0, t0, t1, 48, 16);
-        opn = "pinsrh_3";
         break;
 
     case OPC_PEXTRH:
@@ -4194,42 +3963,33 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         tcg_gen_shli_i64(t1, t1, 4);
         tcg_gen_shr_i64(t0, t0, t1);
         tcg_gen_ext16u_i64(t0, t0);
-        opn = "pextrh";
         break;
 
     case OPC_ADDU_CP2:
         tcg_gen_add_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
     case OPC_SUBU_CP2:
         tcg_gen_sub_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
 
     case OPC_SLL_CP2:
-        opn = "sll";
         shift_max = 32;
         goto do_shift;
     case OPC_SRL_CP2:
-        opn = "srl";
         shift_max = 32;
         goto do_shift;
     case OPC_SRA_CP2:
-        opn = "sra";
         shift_max = 32;
         goto do_shift;
     case OPC_DSLL_CP2:
-        opn = "dsll";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRL_CP2:
-        opn = "dsrl";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRA_CP2:
-        opn = "dsra";
         shift_max = 64;
         goto do_shift;
     do_shift:
@@ -4284,8 +4044,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_ADD_CP2 ? "add" : "dadd");
             break;
         }
 
@@ -4307,8 +4065,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_SUB_CP2 ? "sub" : "dsub");
             break;
         }
 
@@ -4316,7 +4072,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         tcg_gen_ext32u_i64(t0, t0);
         tcg_gen_ext32u_i64(t1, t1);
         tcg_gen_mul_i64(t0, t0, t1);
-        opn = "pmuluw";
         break;
 
     case OPC_SEQU_CP2:
@@ -4328,8 +4083,8 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
         /* ??? Document is unclear: Set FCC[CC].  Does that mean the
            FD field is the CC field?  */
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("loongson_cp2");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -4338,9 +4093,6 @@ static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 
     gen_store_fpr64(ctx, t0, rd);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn,
-               fregnames[rd], fregnames[rs], fregnames[rt]);
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
 }
@@ -4392,7 +4144,7 @@ static void gen_trap (DisasContext *ctx, uint32_t opc,
         case OPC_TGEU:  /* rs >= rs unsigned */
         case OPC_TGEIU: /* r0 >= 0  unsigned */
             /* Always trap */
-            generate_exception(ctx, EXCP_TRAP);
+            generate_exception_end(ctx, EXCP_TRAP);
             break;
         case OPC_TLT:   /* rs < rs           */
         case OPC_TLTI:  /* r0 < 0            */
@@ -4452,7 +4204,7 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
             save_cpu_state(ctx, 0);
-            gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+            gen_helper_raise_exception_debug(cpu_env);
         }
         tcg_gen_exit_tb(0);
     }
@@ -4475,7 +4227,7 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x"
                   TARGET_FMT_lx "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -4535,14 +4287,14 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             /* Hint = 0 is JR/JALR, hint 16 is JR.HB/JALR.HB, the
                others are reserved. */
             MIPS_INVAL("jump hint");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
         gen_load_gpr(btarget, rs);
         break;
     default:
         MIPS_INVAL("branch/jump");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     if (bcond_compute == 0) {
@@ -4556,20 +4308,17 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         case OPC_BLEZL:   /* 0 <= 0 likely   */
             /* Always take */
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways");
             break;
         case OPC_BGEZAL:  /* 0 >= 0          */
         case OPC_BGEZALL: /* 0 >= 0 likely   */
             /* Always take and link */
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways and link");
             break;
         case OPC_BNE:     /* rx != rx        */
         case OPC_BGTZ:    /* 0 > 0           */
         case OPC_BLTZ:    /* 0 < 0           */
             /* Treat as NOP. */
-            MIPS_DEBUG("bnever (NOP)");
             goto out;
         case OPC_BLTZAL:  /* 0 < 0           */
             /* Handle as an unconditional branch to get correct delay
@@ -4577,24 +4326,20 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             blink = 31;
             btgt = ctx->pc + insn_bytes + delayslot_size;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("bnever and link");
             break;
         case OPC_BLTZALL: /* 0 < 0 likely */
             tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 8);
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever, link and skip");
             ctx->pc += 4;
             goto out;
         case OPC_BNEL:    /* rx != rx likely */
         case OPC_BGTZL:   /* 0 > 0 likely */
         case OPC_BLTZL:   /* 0 < 0 likely */
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever and skip");
             ctx->pc += 4;
             goto out;
         case OPC_J:
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("j " TARGET_FMT_lx, btgt);
             break;
         case OPC_JALX:
             ctx->hflags |= MIPS_HFLAG_BX;
@@ -4602,118 +4347,91 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         case OPC_JAL:
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("jal " TARGET_FMT_lx, btgt);
             break;
         case OPC_JR:
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jr %s", regnames[rs]);
             break;
         case OPC_JALR:
             blink = rt;
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jalr %s, %s", regnames[rt], regnames[rs]);
             break;
         default:
             MIPS_INVAL("branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     } else {
         switch (opc) {
         case OPC_BEQ:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beq %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BEQL:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beql %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BNE:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bne %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BNEL:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bnel %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BGEZ:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGEZL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGEZAL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezal %s, " TARGET_FMT_lx, regnames[rs], btgt);
             blink = 31;
             goto not_likely;
         case OPC_BGEZALL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bgezall %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGTZ:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGTZL:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLEZ:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLEZL:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLTZ:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLTZL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BPOSGE32:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 32);
-            MIPS_DEBUG("bposge32 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #if defined(TARGET_MIPS64)
         case OPC_BPOSGE64:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 64);
-            MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #endif
         case OPC_BLTZAL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzal %s, " TARGET_FMT_lx, regnames[rs], btgt);
         not_likely:
             ctx->hflags |= MIPS_HFLAG_BC;
             break;
         case OPC_BLTZALL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzall %s, " TARGET_FMT_lx, regnames[rs], btgt);
         likely:
             ctx->hflags |= MIPS_HFLAG_BL;
             break;
         default:
             MIPS_INVAL("conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     }
-    MIPS_DEBUG("enter ds: link %d cond %02x target " TARGET_FMT_lx,
-               blink, ctx->hflags, btgt);
 
     ctx->btarget = btgt;
 
@@ -4804,7 +4522,7 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
     default:
 fail:
         MIPS_INVAL("bitops");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         tcg_temp_free(t1);
         return;
@@ -4820,7 +4538,6 @@ static void gen_bshfl (DisasContext *ctx, uint32_t op2, int rt, int rd)
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -4877,7 +4594,7 @@ static void gen_bshfl (DisasContext *ctx, uint32_t op2, int rt, int rd)
 #endif
     default:
         MIPS_INVAL("bsfhl");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         return;
     }
@@ -4920,7 +4637,16 @@ static void gen_align(DisasContext *ctx, int opc, int rd, int rs, int rt,
     t0 = tcg_temp_new();
     gen_load_gpr(t0, rt);
     if (bp == 0) {
-        tcg_gen_mov_tl(cpu_gpr[rd], t0);
+        switch (opc) {
+        case OPC_ALIGN:
+            tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_DALIGN:
+            tcg_gen_mov_tl(cpu_gpr[rd], t0);
+            break;
+#endif
+        }
     } else {
         TCGv t1 = tcg_temp_new();
         gen_load_gpr(t1, rs);
@@ -5050,19 +4776,18 @@ static inline void gen_mtc0_store32 (TCGv arg, target_ulong off)
     tcg_temp_free_i32(t0);
 }
 
-static inline void gen_mtc0_store64 (TCGv arg, target_ulong off)
-{
-    tcg_gen_ext32s_tl(arg, arg);
-    tcg_gen_st_tl(arg, cpu_env, off);
-}
+#define CP0_CHECK(c)                            \
+    do {                                        \
+        if (!(c)) {                             \
+            goto cp0_unimplemented;             \
+        }                                       \
+    } while (0)
 
 static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
 
-    if (!(ctx->hflags & MIPS_HFLAG_ELPA)) {
-        goto mfhc0_read_zero;
-    }
+    CP0_CHECK(ctx->hflags & MIPS_HFLAG_ELPA);
 
     switch (reg) {
     case 2:
@@ -5072,7 +4797,7 @@ static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo0";
             break;
         default:
-            goto mfhc0_read_zero;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
@@ -5082,7 +4807,7 @@ static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo1";
             break;
         default:
-            goto mfhc0_read_zero;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -5092,8 +4817,13 @@ static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
                              ctx->CP0_LLAddr_shift);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mfhc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
         default:
-            goto mfhc0_read_zero;
+            goto cp0_unimplemented;
         }
         break;
     case 28:
@@ -5106,18 +4836,18 @@ static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TagLo";
             break;
         default:
-            goto mfhc0_read_zero;
+            goto cp0_unimplemented;
         }
         break;
     default:
-        goto mfhc0_read_zero;
+        goto cp0_unimplemented;
     }
 
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("mfhc0 %s (reg %d sel %d)\n", rn, reg, sel);
     return;
 
-mfhc0_read_zero:
+cp0_unimplemented:
     LOG_DISAS("mfhc0 %s (reg %d sel %d)\n", rn, reg, sel);
     tcg_gen_movi_tl(arg, 0);
 }
@@ -5127,9 +4857,7 @@ static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     const char *rn = "invalid";
     uint64_t mask = ctx->PAMask >> 36;
 
-    if (!(ctx->hflags & MIPS_HFLAG_ELPA)) {
-        goto mthc0_nop;
-    }
+    CP0_CHECK(ctx->hflags & MIPS_HFLAG_ELPA);
 
     switch (reg) {
     case 2:
@@ -5140,7 +4868,7 @@ static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo0";
             break;
         default:
-            goto mthc0_nop;
+            goto cp0_unimplemented;
         }
         break;
     case 3:
@@ -5151,7 +4879,7 @@ static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "EntryLo1";
             break;
         default:
-            goto mthc0_nop;
+            goto cp0_unimplemented;
         }
         break;
     case 17:
@@ -5163,8 +4891,13 @@ static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
                treating MTHC0 to LLAddr as NOP. */
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mthc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
         default:
-            goto mthc0_nop;
+            goto cp0_unimplemented;
         }
         break;
     case 28:
@@ -5178,15 +4911,15 @@ static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             rn = "TagLo";
             break;
         default:
-            goto mthc0_nop;
+            goto cp0_unimplemented;
         }
         break;
     default:
-        goto mthc0_nop;
+        goto cp0_unimplemented;
     }
 
     (void)rn; /* avoid a compiler warning */
-mthc0_nop:
+cp0_unimplemented:
     LOG_DISAS("mthc0 %s (reg %d sel %d)\n", rn, reg, sel);
 }
 
@@ -5199,13 +4932,6 @@ static inline void gen_mfc0_unimplemented(DisasContext *ctx, TCGv arg)
     }
 }
 
-#define CP0_CHECK(c)                            \
-    do {                                        \
-        if (!(c)) {                             \
-            goto cp0_unimplemented;             \
-        }                                       \
-    } while (0)
-
 static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
@@ -5235,6 +4961,11 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPControl));
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5362,6 +5093,11 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             }
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_GlobalNumber));
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5570,6 +5306,13 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
+        case 3:
+            check_insn(ctx, ISA_MIPS32R2);
+            CP0_CHECK(ctx->cmgcr);
+            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_CMGCRBase));
+            tcg_gen_ext32s_tl(arg, arg);
+            rn = "CMGCRBase";
+            break;
         default:
             goto cp0_unimplemented;
        }
@@ -5619,6 +5362,16 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mfc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_MAARI));
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5750,8 +5503,14 @@ static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         }
         break;
     case 26:
-        tcg_gen_movi_tl(arg, 0); /* unimplemented */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_ErrCtl));
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -5882,6 +5641,11 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             /* ignored */
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5914,12 +5678,14 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             break;
         case 5:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
@@ -5982,6 +5748,11 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6169,7 +5940,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     case 14:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_EPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EPC));
             rn = "EPC";
             break;
         default:
@@ -6246,6 +6017,16 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maari(cpu_env, arg);
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6342,7 +6123,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         switch (sel) {
         case 0:
             /* EJTAG support */
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_DEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_DEPC));
             rn = "DEPC";
             break;
         default:
@@ -6388,8 +6169,15 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         }
        break;
     case 26:
-        /* ignored */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_helper_mtc0_errctl(cpu_env, arg);
+            ctx->bstate = BS_STOP;
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -6445,7 +6233,7 @@ static void gen_mtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
     case 30:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_ErrorEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_ErrorEPC));
             rn = "ErrorEPC";
             break;
         default:
@@ -6517,6 +6305,11 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPControl));
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6618,6 +6411,11 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo1));
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_GlobalNumber));
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6822,6 +6620,12 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
+        case 3:
+            check_insn(ctx, ISA_MIPS32R2);
+            CP0_CHECK(ctx->cmgcr);
+            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_CMGCRBase));
+            rn = "CMGCRBase";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6871,6 +6675,16 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_dmfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_dmfc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_MAARI));
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6998,8 +6812,14 @@ static void gen_dmfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         }
         break;
     case 26:
-        tcg_gen_movi_tl(arg, 0); /* unimplemented */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_ErrCtl));
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -7124,6 +6944,11 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             /* ignored */
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7224,6 +7049,11 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_dmtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7492,6 +7322,16 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
             gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maari(cpu_env, arg);
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7630,8 +7470,15 @@ static void gen_dmtc0(DisasContext *ctx, TCGv arg, int reg, int sel)
         }
         break;
     case 26:
-        /* ignored */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_helper_mtc0_errctl(cpu_env, arg);
+            ctx->bstate = BS_STOP;
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -7944,7 +7791,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mftr (reg %d u %d sel %d h %d)\n", rt, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
@@ -8125,7 +7972,6 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        save_cpu_state(ctx, 1);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(rd);
 
@@ -8149,7 +7995,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mttr (reg %d u %d sel %d h %d)\n", rd, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt, int rd)
@@ -8278,7 +8124,6 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     case OPC_ERET: /* OPC_ERETNC */
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         } else {
             int bit_shift = (ctx->hflags & MIPS_HFLAG_M16) ? 16 : 6;
@@ -8301,12 +8146,11 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         check_insn(ctx, ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             MIPS_INVAL(opn);
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             gen_helper_deret(cpu_env);
             ctx->bstate = BS_EXCP;
@@ -8317,7 +8161,6 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         /* If we get an exception, we want to restart at next instruction */
@@ -8330,11 +8173,10 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
     default:
  die:
         MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %d", opn, regnames[rt], rd);
 }
 #endif /* !CONFIG_USER_ONLY */
 
@@ -8343,12 +8185,10 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
                                 int32_t cc, int32_t offset)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i32 t0 = tcg_temp_new_i32();
 
     if ((ctx->insn_flags & ISA_MIPS32R6) && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8363,26 +8203,22 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1f";
         goto not_likely;
     case OPC_BC1FL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1fl";
         goto likely;
     case OPC_BC1T:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1t";
         goto not_likely;
     case OPC_BC1TL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1tl";
     likely:
         ctx->hflags |= MIPS_HFLAG_BL;
         break;
@@ -8396,7 +8232,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2f";
         goto not_likely;
     case OPC_BC1TANY2:
         {
@@ -8408,7 +8243,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2t";
         goto not_likely;
     case OPC_BC1FANY4:
         {
@@ -8424,7 +8258,6 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4f";
         goto not_likely;
     case OPC_BC1TANY4:
         {
@@ -8440,18 +8273,14 @@ static void gen_compute_branch1(DisasContext *ctx, uint32_t op,
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4t";
     not_likely:
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
     ctx->hflags |= MIPS_HFLAG_BDS32;
  out:
@@ -8464,7 +8293,6 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
                                    int delayslot_size)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i64 t0 = tcg_temp_new_i64();
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
@@ -8472,7 +8300,7 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
                   "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8484,25 +8312,20 @@ static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
     switch (op) {
     case OPC_BC1EQZ:
         tcg_gen_xori_i64(t0, t0, 1);
-        opn = "bc1eqz";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     case OPC_BC1NEZ:
         /* t0 already set */
-        opn = "bc1nez";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
     tcg_gen_trunc_i64_tl(bcond, t0);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
 
     switch (delayslot_size) {
@@ -8733,7 +8556,6 @@ enum r6_f_cmp_op {
 };
 static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
 {
-    const char *opn = "cp1 move";
     TCGv t0 = tcg_temp_new();
 
     switch (opc) {
@@ -8746,7 +8568,6 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfc1";
         break;
     case OPC_MTC1:
         gen_load_gpr(t0, rt);
@@ -8757,16 +8578,14 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             gen_store_fpr32(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mtc1";
         break;
     case OPC_CFC1:
         gen_helper_1e0i(cfc1, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "cfc1";
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
-        save_cpu_state(ctx, 1);
+        save_cpu_state(ctx, 0);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(fs);
 
@@ -8775,18 +8594,15 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
         }
         /* Stop translation as we may have changed hflags */
         ctx->bstate = BS_STOP;
-        opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMFC1:
         gen_load_fpr64(ctx, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "dmfc1";
         break;
     case OPC_DMTC1:
         gen_load_gpr(t0, rt);
         gen_store_fpr64(ctx, t0, fs);
-        opn = "dmtc1";
         break;
 #endif
     case OPC_MFHC1:
@@ -8798,7 +8614,6 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfhc1";
         break;
     case OPC_MTHC1:
         gen_load_gpr(t0, rt);
@@ -8809,15 +8624,12 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
             gen_store_fpr32h(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mthc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 move");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rt], fregnames[fs]);
 
  out:
     tcg_temp_free(t0);
@@ -8947,7 +8759,7 @@ static void gen_sel_s(DisasContext *ctx, enum fopcode op1, int fd, int ft,
         break;
     default:
         MIPS_INVAL("gen_sel_s");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -8984,7 +8796,7 @@ static void gen_sel_d(DisasContext *ctx, enum fopcode op1, int fd, int ft,
         break;
     default:
         MIPS_INVAL("gen_sel_d");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -8998,44 +8810,6 @@ static void gen_sel_d(DisasContext *ctx, enum fopcode op1, int fd, int ft,
 static void gen_farith (DisasContext *ctx, enum fopcode op1,
                         int ft, int fs, int fd, int cc)
 {
-    const char *opn = "farith";
-    const char *condnames[] = {
-            "c.f",
-            "c.un",
-            "c.eq",
-            "c.ueq",
-            "c.olt",
-            "c.ult",
-            "c.ole",
-            "c.ule",
-            "c.sf",
-            "c.ngle",
-            "c.seq",
-            "c.ngl",
-            "c.lt",
-            "c.nge",
-            "c.le",
-            "c.ngt",
-    };
-    const char *condnames_abs[] = {
-            "cabs.f",
-            "cabs.un",
-            "cabs.eq",
-            "cabs.ueq",
-            "cabs.olt",
-            "cabs.ult",
-            "cabs.ole",
-            "cabs.ule",
-            "cabs.sf",
-            "cabs.ngle",
-            "cabs.seq",
-            "cabs.ngl",
-            "cabs.lt",
-            "cabs.nge",
-            "cabs.le",
-            "cabs.ngt",
-    };
-    enum { BINOP, CMPOP, OTHEROP } optype = OTHEROP;
     uint32_t func = ctx->opcode & 0x3f;
     switch (op1) {
     case OPC_ADD_S:
@@ -9050,8 +8824,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "add.s";
-        optype = BINOP;
         break;
     case OPC_SUB_S:
         {
@@ -9065,8 +8837,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sub.s";
-        optype = BINOP;
         break;
     case OPC_MUL_S:
         {
@@ -9080,8 +8850,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mul.s";
-        optype = BINOP;
         break;
     case OPC_DIV_S:
         {
@@ -9095,8 +8863,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "div.s";
-        optype = BINOP;
         break;
     case OPC_SQRT_S:
         {
@@ -9107,7 +8873,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sqrt.s";
         break;
     case OPC_ABS_S:
         {
@@ -9118,7 +8883,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "abs.s";
         break;
     case OPC_MOV_S:
         {
@@ -9128,7 +8892,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mov.s";
         break;
     case OPC_NEG_S:
         {
@@ -9139,7 +8902,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "neg.s";
         break;
     case OPC_ROUND_L_S:
         check_cp1_64bitmode(ctx);
@@ -9153,7 +8915,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "round.l.s";
         break;
     case OPC_TRUNC_L_S:
         check_cp1_64bitmode(ctx);
@@ -9167,7 +8928,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "trunc.l.s";
         break;
     case OPC_CEIL_L_S:
         check_cp1_64bitmode(ctx);
@@ -9181,7 +8941,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "ceil.l.s";
         break;
     case OPC_FLOOR_L_S:
         check_cp1_64bitmode(ctx);
@@ -9195,7 +8954,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "floor.l.s";
         break;
     case OPC_ROUND_W_S:
         {
@@ -9206,7 +8964,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "round.w.s";
         break;
     case OPC_TRUNC_W_S:
         {
@@ -9217,7 +8974,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "trunc.w.s";
         break;
     case OPC_CEIL_W_S:
         {
@@ -9228,7 +8984,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "ceil.w.s";
         break;
     case OPC_FLOOR_W_S:
         {
@@ -9239,27 +8994,22 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "floor.w.s";
         break;
     case OPC_SEL_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "sel.s";
         break;
     case OPC_SELEQZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "seleqz.s";
         break;
     case OPC_SELNEZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "selnez.s";
         break;
     case OPC_MOVCF_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_s(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.s";
         break;
     case OPC_MOVZ_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9276,7 +9026,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.s";
         break;
     case OPC_MOVN_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9293,10 +9042,8 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.s";
         break;
     case OPC_RECIP_S:
-        check_cop1x(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
@@ -9305,10 +9052,8 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "recip.s";
         break;
     case OPC_RSQRT_S:
-        check_cop1x(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
@@ -9317,7 +9062,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "rsqrt.s";
         break;
     case OPC_MADDF_S:
         check_insn(ctx, ISA_MIPS32R6);
@@ -9333,7 +9077,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maddf.s";
         }
         break;
     case OPC_MSUBF_S:
@@ -9350,7 +9093,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "msubf.s";
         }
         break;
     case OPC_RINT_S:
@@ -9361,7 +9103,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_rint_s(fp0, cpu_env, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "rint.s";
         }
         break;
     case OPC_CLASS_S:
@@ -9372,7 +9113,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_class_s(fp0, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "class.s";
         }
         break;
     case OPC_MIN_S: /* OPC_RECIP2_S */
@@ -9388,7 +9128,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "min.s";
         } else {
             /* OPC_RECIP2_S */
             check_cp1_64bitmode(ctx);
@@ -9403,7 +9142,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip2.s";
         }
         break;
     case OPC_MINA_S: /* OPC_RECIP1_S */
@@ -9419,7 +9157,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "mina.s";
         } else {
             /* OPC_RECIP1_S */
             check_cp1_64bitmode(ctx);
@@ -9431,7 +9168,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip1.s";
         }
         break;
     case OPC_MAX_S: /* OPC_RSQRT1_S */
@@ -9445,7 +9181,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "max.s";
         } else {
             /* OPC_RSQRT1_S */
             check_cp1_64bitmode(ctx);
@@ -9457,7 +9192,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt1.s";
         }
         break;
     case OPC_MAXA_S: /* OPC_RSQRT2_S */
@@ -9471,7 +9205,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maxa.s";
         } else {
             /* OPC_RSQRT2_S */
             check_cp1_64bitmode(ctx);
@@ -9486,7 +9219,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt2.s";
         }
         break;
     case OPC_CVT_D_S:
@@ -9501,7 +9233,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.s";
         break;
     case OPC_CVT_W_S:
         {
@@ -9512,7 +9243,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.w.s";
         break;
     case OPC_CVT_L_S:
         check_cp1_64bitmode(ctx);
@@ -9526,7 +9256,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.l.s";
         break;
     case OPC_CVT_PS_S:
         check_ps(ctx);
@@ -9543,7 +9272,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.ps.s";
         break;
     case OPC_CMP_F_S:
     case OPC_CMP_UN_S:
@@ -9564,12 +9292,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_s(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_s(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     case OPC_ADD_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9584,8 +9309,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.d";
-        optype = BINOP;
         break;
     case OPC_SUB_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9600,8 +9323,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.d";
-        optype = BINOP;
         break;
     case OPC_MUL_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9616,8 +9337,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.d";
-        optype = BINOP;
         break;
     case OPC_DIV_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9632,8 +9351,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "div.d";
-        optype = BINOP;
         break;
     case OPC_SQRT_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9645,7 +9362,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sqrt.d";
         break;
     case OPC_ABS_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9657,7 +9373,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.d";
         break;
     case OPC_MOV_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9668,7 +9383,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.d";
         break;
     case OPC_NEG_D:
         check_cp1_registers(ctx, fs | fd);
@@ -9680,7 +9394,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.d";
         break;
     case OPC_ROUND_L_D:
         check_cp1_64bitmode(ctx);
@@ -9692,7 +9405,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "round.l.d";
         break;
     case OPC_TRUNC_L_D:
         check_cp1_64bitmode(ctx);
@@ -9704,7 +9416,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "trunc.l.d";
         break;
     case OPC_CEIL_L_D:
         check_cp1_64bitmode(ctx);
@@ -9716,7 +9427,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ceil.l.d";
         break;
     case OPC_FLOOR_L_D:
         check_cp1_64bitmode(ctx);
@@ -9728,7 +9438,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "floor.l.d";
         break;
     case OPC_ROUND_W_D:
         check_cp1_registers(ctx, fs);
@@ -9742,7 +9451,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "round.w.d";
         break;
     case OPC_TRUNC_W_D:
         check_cp1_registers(ctx, fs);
@@ -9756,7 +9464,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "trunc.w.d";
         break;
     case OPC_CEIL_W_D:
         check_cp1_registers(ctx, fs);
@@ -9770,7 +9477,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "ceil.w.d";
         break;
     case OPC_FLOOR_W_D:
         check_cp1_registers(ctx, fs);
@@ -9784,27 +9490,22 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "floor.w.d";
         break;
     case OPC_SEL_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "sel.d";
         break;
     case OPC_SELEQZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "seleqz.d";
         break;
     case OPC_SELNEZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "selnez.d";
         break;
     case OPC_MOVCF_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_d(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.d";
         break;
     case OPC_MOVZ_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9821,7 +9522,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.d";
         break;
     case OPC_MOVN_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
@@ -9838,10 +9538,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.d";
         break;
     case OPC_RECIP_D:
-        check_cp1_64bitmode(ctx);
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -9850,10 +9549,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip.d";
         break;
     case OPC_RSQRT_D:
-        check_cp1_64bitmode(ctx);
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -9862,7 +9560,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt.d";
         break;
     case OPC_MADDF_D:
         check_insn(ctx, ISA_MIPS32R6);
@@ -9878,7 +9575,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maddf.d";
         }
         break;
     case OPC_MSUBF_D:
@@ -9895,7 +9591,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "msubf.d";
         }
         break;
     case OPC_RINT_D:
@@ -9906,7 +9601,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_rint_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "rint.d";
         }
         break;
     case OPC_CLASS_D:
@@ -9917,7 +9611,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_helper_float_class_d(fp0, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "class.d";
         }
         break;
     case OPC_MIN_D: /* OPC_RECIP2_D */
@@ -9931,7 +9624,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "min.d";
         } else {
             /* OPC_RECIP2_D */
             check_cp1_64bitmode(ctx);
@@ -9946,7 +9638,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip2.d";
         }
         break;
     case OPC_MINA_D: /* OPC_RECIP1_D */
@@ -9960,7 +9651,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "mina.d";
         } else {
             /* OPC_RECIP1_D */
             check_cp1_64bitmode(ctx);
@@ -9972,7 +9662,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip1.d";
         }
         break;
     case OPC_MAX_D: /*  OPC_RSQRT1_D */
@@ -9986,7 +9675,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "max.d";
         } else {
             /* OPC_RSQRT1_D */
             check_cp1_64bitmode(ctx);
@@ -9998,7 +9686,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt1.d";
         }
         break;
     case OPC_MAXA_D: /* OPC_RSQRT2_D */
@@ -10012,7 +9699,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maxa.d";
         } else {
             /* OPC_RSQRT2_D */
             check_cp1_64bitmode(ctx);
@@ -10027,7 +9713,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt2.d";
         }
         break;
     case OPC_CMP_F_D:
@@ -10049,12 +9734,9 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_d(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_d(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     case OPC_CVT_S_D:
         check_cp1_registers(ctx, fs);
@@ -10068,7 +9750,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.d";
         break;
     case OPC_CVT_W_D:
         check_cp1_registers(ctx, fs);
@@ -10082,7 +9763,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.w.d";
         break;
     case OPC_CVT_L_D:
         check_cp1_64bitmode(ctx);
@@ -10094,7 +9774,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.l.d";
         break;
     case OPC_CVT_S_W:
         {
@@ -10105,7 +9784,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.w";
         break;
     case OPC_CVT_D_W:
         check_cp1_registers(ctx, fd);
@@ -10119,7 +9797,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.w";
         break;
     case OPC_CVT_S_L:
         check_cp1_64bitmode(ctx);
@@ -10133,7 +9810,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.l";
         break;
     case OPC_CVT_D_L:
         check_cp1_64bitmode(ctx);
@@ -10145,7 +9821,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.d.l";
         break;
     case OPC_CVT_PS_PW:
         check_ps(ctx);
@@ -10157,7 +9832,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.ps.pw";
         break;
     case OPC_ADD_PS:
         check_ps(ctx);
@@ -10172,7 +9846,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.ps";
         break;
     case OPC_SUB_PS:
         check_ps(ctx);
@@ -10187,7 +9860,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.ps";
         break;
     case OPC_MUL_PS:
         check_ps(ctx);
@@ -10202,7 +9874,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.ps";
         break;
     case OPC_ABS_PS:
         check_ps(ctx);
@@ -10214,7 +9885,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.ps";
         break;
     case OPC_MOV_PS:
         check_ps(ctx);
@@ -10225,7 +9895,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.ps";
         break;
     case OPC_NEG_PS:
         check_ps(ctx);
@@ -10237,12 +9906,10 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.ps";
         break;
     case OPC_MOVCF_PS:
         check_ps(ctx);
         gen_movcf_ps(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.ps";
         break;
     case OPC_MOVZ_PS:
         check_ps(ctx);
@@ -10258,7 +9925,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.ps";
         break;
     case OPC_MOVN_PS:
         check_ps(ctx);
@@ -10275,7 +9941,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
                 gen_set_label(l1);
             }
         }
-        opn = "movn.ps";
         break;
     case OPC_ADDR_PS:
         check_ps(ctx);
@@ -10290,7 +9955,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "addr.ps";
         break;
     case OPC_MULR_PS:
         check_ps(ctx);
@@ -10305,7 +9969,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mulr.ps";
         break;
     case OPC_RECIP2_PS:
         check_ps(ctx);
@@ -10320,7 +9983,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip2.ps";
         break;
     case OPC_RECIP1_PS:
         check_ps(ctx);
@@ -10332,7 +9994,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip1.ps";
         break;
     case OPC_RSQRT1_PS:
         check_ps(ctx);
@@ -10344,7 +10005,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt1.ps";
         break;
     case OPC_RSQRT2_PS:
         check_ps(ctx);
@@ -10359,7 +10019,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt2.ps";
         break;
     case OPC_CVT_S_PU:
         check_cp1_64bitmode(ctx);
@@ -10371,7 +10030,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pu";
         break;
     case OPC_CVT_PW_PS:
         check_ps(ctx);
@@ -10383,7 +10041,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.pw.ps";
         break;
     case OPC_CVT_S_PL:
         check_cp1_64bitmode(ctx);
@@ -10395,7 +10052,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pl";
         break;
     case OPC_PLL_PS:
         check_ps(ctx);
@@ -10410,7 +10066,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pll.ps";
         break;
     case OPC_PLU_PS:
         check_ps(ctx);
@@ -10425,7 +10080,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "plu.ps";
         break;
     case OPC_PUL_PS:
         check_ps(ctx);
@@ -10440,7 +10094,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pul.ps";
         break;
     case OPC_PUU_PS:
         check_ps(ctx);
@@ -10455,7 +10108,6 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "puu.ps";
         break;
     case OPC_CMP_F_PS:
     case OPC_CMP_UN_PS:
@@ -10475,38 +10127,21 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
     case OPC_CMP_NGT_PS:
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
-        optype = CMPOP;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("farith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    switch (optype) {
-    case BINOP:
-        MIPS_DEBUG("%s %s, %s, %s", opn, fregnames[fd], fregnames[fs], fregnames[ft]);
-        break;
-    case CMPOP:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fs], fregnames[ft]);
-        break;
-    default:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fd], fregnames[fs]);
-        break;
-    }
 }
 
 /* Coprocessor 3 (FPU) */
 static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
                            int fd, int fs, int base, int index)
 {
-    const char *opn = "extended float load/store";
-    int store = 0;
     TCGv t0 = tcg_temp_new();
 
     if (base == 0) {
@@ -10529,7 +10164,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwxc1";
         break;
     case OPC_LDXC1:
         check_cop1x(ctx);
@@ -10540,7 +10174,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldxc1";
         break;
     case OPC_LUXC1:
         check_cp1_64bitmode(ctx);
@@ -10552,7 +10185,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "luxc1";
         break;
     case OPC_SWXC1:
         check_cop1x(ctx);
@@ -10562,8 +10194,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i32(fp0, t0, ctx->mem_idx, MO_TEUL);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swxc1";
-        store = 1;
         break;
     case OPC_SDXC1:
         check_cop1x(ctx);
@@ -10574,8 +10204,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdxc1";
-        store = 1;
         break;
     case OPC_SUXC1:
         check_cp1_64bitmode(ctx);
@@ -10586,21 +10214,14 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "suxc1";
-        store = 1;
         break;
     }
     tcg_temp_free(t0);
-    (void)opn; (void)store; /* avoid compiler warnings */
-    MIPS_DEBUG("%s %s, %s(%s)", opn, fregnames[store ? fs : fd],
-               regnames[index], regnames[base]);
 }
 
 static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
                             int fd, int fr, int fs, int ft)
 {
-    const char *opn = "flt3_arith";
-
     switch (opc) {
     case OPC_ALNV_PS:
         check_ps(ctx);
@@ -10638,7 +10259,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             tcg_temp_free_i32(fp);
             tcg_temp_free_i32(fph);
         }
-        opn = "alnv.ps";
         break;
     case OPC_MADD_S:
         check_cop1x(ctx);
@@ -10656,7 +10276,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "madd.s";
         break;
     case OPC_MADD_D:
         check_cop1x(ctx);
@@ -10675,7 +10294,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.d";
         break;
     case OPC_MADD_PS:
         check_ps(ctx);
@@ -10693,7 +10311,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.ps";
         break;
     case OPC_MSUB_S:
         check_cop1x(ctx);
@@ -10711,7 +10328,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "msub.s";
         break;
     case OPC_MSUB_D:
         check_cop1x(ctx);
@@ -10730,7 +10346,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.d";
         break;
     case OPC_MSUB_PS:
         check_ps(ctx);
@@ -10748,7 +10363,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.ps";
         break;
     case OPC_NMADD_S:
         check_cop1x(ctx);
@@ -10766,7 +10380,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmadd.s";
         break;
     case OPC_NMADD_D:
         check_cop1x(ctx);
@@ -10785,7 +10398,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.d";
         break;
     case OPC_NMADD_PS:
         check_ps(ctx);
@@ -10803,7 +10415,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.ps";
         break;
     case OPC_NMSUB_S:
         check_cop1x(ctx);
@@ -10821,7 +10432,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmsub.s";
         break;
     case OPC_NMSUB_D:
         check_cop1x(ctx);
@@ -10840,7 +10450,6 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.d";
         break;
     case OPC_NMSUB_PS:
         check_ps(ctx);
@@ -10858,19 +10467,15 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.ps";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("flt3_arith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s, %s", opn, fregnames[fd], fregnames[fr],
-               fregnames[fs], fregnames[ft]);
 }
 
-static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
+static void gen_rdhwr(DisasContext *ctx, int rt, int rd, int sel)
 {
     TCGv t0;
 
@@ -10883,25 +10488,37 @@ static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
 
     switch (rd) {
     case 0:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cpunum(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 1:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_synci_step(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 2:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cc(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 3:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_ccres(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
+    case 4:
+        check_insn(ctx, ISA_MIPS32R6);
+        if (sel != 0) {
+            /* Performance counter registers are not implemented other than
+             * control register 0.
+             */
+            generate_exception(ctx, EXCP_RI);
+        }
+        gen_helper_rdhwr_performance(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
+    case 5:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_helper_rdhwr_xnp(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
     case 29:
 #if defined(CONFIG_USER_ONLY)
         tcg_gen_ld_tl(t0, cpu_env,
@@ -10915,13 +10532,13 @@ static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
                           offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
             gen_store_gpr(t0, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("rdhwr");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free(t0);
@@ -10949,12 +10566,10 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
         /* FIXME: Need to clear can_do_io.  */
         switch (proc_hflags & MIPS_HFLAG_BMASK_BASE) {
         case MIPS_HFLAG_FBNSLOT:
-            MIPS_DEBUG("forbidden slot");
             gen_goto_tb(ctx, 0, ctx->pc + insn_bytes);
             break;
         case MIPS_HFLAG_B:
             /* unconditional branch */
-            MIPS_DEBUG("unconditional branch");
             if (proc_hflags & MIPS_HFLAG_BX) {
                 tcg_gen_xori_i32(hflags, hflags, MIPS_HFLAG_M16);
             }
@@ -10962,12 +10577,10 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             break;
         case MIPS_HFLAG_BL:
             /* blikely taken case */
-            MIPS_DEBUG("blikely branch taken");
             gen_goto_tb(ctx, 0, ctx->btarget);
             break;
         case MIPS_HFLAG_BC:
             /* Conditional branch */
-            MIPS_DEBUG("conditional branch");
             {
                 TCGLabel *l1 = gen_new_label();
 
@@ -10979,7 +10592,6 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             break;
         case MIPS_HFLAG_BR:
             /* unconditional branch to register */
-            MIPS_DEBUG("branch to register");
             if (ctx->insn_flags & (ASE_MIPS16 | ASE_MICROMIPS)) {
                 TCGv t0 = tcg_temp_new();
                 TCGv_i32 t1 = tcg_temp_new_i32();
@@ -10998,7 +10610,7 @@ static void gen_branch(DisasContext *ctx, int insn_bytes)
             }
             if (ctx->singlestep_enabled) {
                 save_cpu_state(ctx, 0);
-                gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+                gen_helper_raise_exception_debug(cpu_env);
             }
             tcg_gen_exit_tb(0);
             break;
@@ -11023,7 +10635,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
                   "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -11085,7 +10697,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         break;
     default:
         MIPS_INVAL("Compact branch/jump");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -11106,7 +10718,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
             break;
         default:
             MIPS_INVAL("Compact branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
 
@@ -11229,7 +10841,7 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
             break;
         default:
             MIPS_INVAL("Compact conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
 
@@ -11238,7 +10850,6 @@ static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
         gen_set_label(fs);
 
         ctx->hflags |= MIPS_HFLAG_FBNSLOT;
-        MIPS_DEBUG("Compact conditional branch");
     }
 
 out:
@@ -11404,7 +11015,7 @@ static void gen_mips16_save (DisasContext *ctx,
         args = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11500,7 +11111,7 @@ static void gen_mips16_save (DisasContext *ctx,
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11606,7 +11217,7 @@ static void gen_mips16_restore (DisasContext *ctx,
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11637,7 +11248,7 @@ static void gen_addiupc (DisasContext *ctx, int rx, int imm,
     TCGv t0;
 
     if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11652,6 +11263,15 @@ static void gen_addiupc (DisasContext *ctx, int rx, int imm,
     tcg_temp_free(t0);
 }
 
+static void gen_cache_operation(DisasContext *ctx, uint32_t op, int base,
+                                int16_t offset)
+{
+    TCGv_i32 t0 = tcg_const_i32(op);
+    TCGv t1 = tcg_temp_new();
+    gen_base_offset_addr(ctx, t1, base, offset);
+    gen_helper_cache(cpu_env, t1, t0);
+}
+
 #if defined(TARGET_MIPS64)
 static void decode_i64_mips16 (DisasContext *ctx,
                                int ry, int funct, int16_t offset,
@@ -11686,7 +11306,7 @@ static void decode_i64_mips16 (DisasContext *ctx,
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             offset = extended ? offset : offset << 3;
             gen_ld(ctx, OPC_LDPC, ry, 0, offset);
@@ -11761,7 +11381,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -11789,7 +11409,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
         } else {
             gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -11841,7 +11461,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             }
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -11904,7 +11524,7 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -11985,7 +11605,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -12013,7 +11633,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 check_mips_64(ctx);
                 gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
 #endif
             } else {
                 gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -12097,7 +11717,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 gen_arith(ctx, OPC_ADDU, ry, reg32, 0);
                 break;
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -12187,7 +11807,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 goto done;
             }
 
@@ -12226,7 +11846,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                  *      when in debug mode...
                  */
                 check_insn(ctx, ISA_MIPS32);
-                generate_exception(ctx, EXCP_DBp);
+                generate_exception_end(ctx, EXCP_DBp);
             }
             break;
         case RR_SLT:
@@ -12236,7 +11856,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             gen_slt(ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
-            generate_exception(ctx, EXCP_BREAK);
+            generate_exception_end(ctx, EXCP_BREAK);
             break;
         case RR_SLLV:
             gen_shift(ctx, OPC_SLLV, ry, rx, ry);
@@ -12303,7 +11923,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -12367,7 +11987,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
             break;
 #endif
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -12382,7 +12002,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -12526,6 +12146,7 @@ enum {
     ROTR = 0x3,
     SELEQZ = 0x5,
     SELNEZ = 0x6,
+    R6_RDHWR = 0x7,
 
     SLLV = 0x0,
     SRLV = 0x1,
@@ -12556,11 +12177,13 @@ enum {
     MODU = 0x7,
 
     /* The following can be distinguished by their lower 6 bits. */
+    BREAK32 = 0x07,
     INS = 0x0c,
     LSA = 0x0f,
     ALIGN = 0x1f,
     EXT = 0x2c,
-    POOL32AXF = 0x3c
+    POOL32AXF = 0x3c,
+    SIGRIE = 0x3f
 };
 
 /* POOL32AXF encoding of minor opcode field extension */
@@ -13029,12 +12652,11 @@ static void gen_andi16(DisasContext *ctx)
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
                                int base, int16_t offset)
 {
-    const char *opn = "ldst_multiple";
     TCGv t0, t1;
     TCGv_i32 t2;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13049,25 +12671,19 @@ static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
     switch (opc) {
     case LWM32:
         gen_helper_lwm(cpu_env, t0, t1, t2);
-        opn = "lwm";
         break;
     case SWM32:
         gen_helper_swm(cpu_env, t0, t1, t2);
-        opn = "swm";
         break;
 #ifdef TARGET_MIPS64
     case LDM:
         gen_helper_ldm(cpu_env, t0, t1, t2);
-        opn = "ldm";
         break;
     case SDM:
         gen_helper_sdm(cpu_env, t0, t1, t2);
-        opn = "sdm";
         break;
 #endif
     }
-    (void)opn;
-    MIPS_DEBUG("%s, %x, %d(%s)", opn, reglist, offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
     tcg_temp_free_i32(t2);
@@ -13164,7 +12780,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
         gen_HILO(ctx, OPC_MFLO, 0, uMIPS_RS5(ctx->opcode));
         break;
     case BREAK16:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case SDBBP16:
         if (is_uhi(extract32(ctx->opcode, 0, 4))) {
@@ -13174,7 +12790,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
              *      when in debug mode...
              */
             check_insn(ctx, ISA_MIPS32);
-            generate_exception(ctx, EXCP_DBp);
+            generate_exception_end(ctx, EXCP_DBp);
         }
         break;
     case JRADDIUSP + 0:
@@ -13188,7 +12804,7 @@ static void gen_pool16c_insn(DisasContext *ctx)
         }
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -13324,11 +12940,10 @@ static void gen_ldxs (DisasContext *ctx, int base, int index, int rd)
 static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
                            int base, int16_t offset)
 {
-    const char *opn = "ldst_pair";
     TCGv t0, t1;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK || rd == 31) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13340,7 +12955,7 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
     switch (opc) {
     case LWP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
@@ -13349,7 +12964,6 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t1, rd+1);
-        opn = "lwp";
         break;
     case SWP:
         gen_load_gpr(t1, rd);
@@ -13358,12 +12972,11 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL);
-        opn = "swp";
         break;
 #ifdef TARGET_MIPS64
     case LDP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
@@ -13372,7 +12985,6 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t1, rd+1);
-        opn = "ldp";
         break;
     case SDP:
         gen_load_gpr(t1, rd);
@@ -13381,12 +12993,9 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ);
-        opn = "sdp";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s, %s, %d(%s)", opn, regnames[rd], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -13492,7 +13101,8 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             gen_cl(ctx, mips32_op, rt, rs);
             break;
         case RDHWR:
-            gen_rdhwr(ctx, rt, rs);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            gen_rdhwr(ctx, rt, rs, 0);
             break;
         case WSBH:
             gen_bshfl(ctx, OPC_WSBH, rs, rt);
@@ -13668,8 +13278,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             /* NOP */
             break;
         case SYSCALL:
-            generate_exception(ctx, EXCP_SYSCALL);
-            ctx->bstate = BS_STOP;
+            generate_exception_end(ctx, EXCP_SYSCALL);
             break;
         case SDBBP:
             if (is_uhi(extract32(ctx->opcode, 16, 10))) {
@@ -13677,9 +13286,9 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
             } else {
                 check_insn(ctx, ISA_MIPS32);
                 if (ctx->hflags & MIPS_HFLAG_SBRI) {
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                 } else {
-                    generate_exception(ctx, EXCP_DBp);
+                    generate_exception_end(ctx, EXCP_DBp);
                 }
             }
             break;
@@ -13727,7 +13336,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs)
     default:
     pool32axf_invalid:
         MIPS_INVAL("pool32axf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -13995,7 +13604,7 @@ static void gen_pool32fxf(DisasContext *ctx, int rt, int rs)
         break;
     default:
         MIPS_INVAL("pool32fxf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -14048,6 +13657,10 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
                 check_insn(ctx, ISA_MIPS32R6);
                 gen_cond_move(ctx, OPC_SELNEZ, rd, rs, rt);
                 break;
+            case R6_RDHWR:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_rdhwr(ctx, rt, rs, extract32(ctx->opcode, 11, 3));
+                break;
             default:
                 goto pool32a_invalid;
             }
@@ -14191,13 +13804,17 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
         case POOL32AXF:
             gen_pool32axf(env, ctx, rt, rs);
             break;
-        case 0x07:
-            generate_exception(ctx, EXCP_BREAK);
+        case BREAK32:
+            generate_exception_end(ctx, EXCP_BREAK);
+            break;
+        case SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         default:
         pool32a_invalid:
                 MIPS_INVAL("pool32a");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
         }
         break;
@@ -14206,7 +13823,9 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
         switch (minor) {
         case CACHE:
             check_cp0_enabled(ctx);
-            /* Treat as no-op. */
+            if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+                gen_cache_operation(ctx, rt, rs, imm);
+            }
             break;
         case LWC2:
         case SWC2:
@@ -14237,7 +13856,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("pool32b");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -14717,7 +14336,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             default:
             pool32f_invalid:
                 MIPS_INVAL("pool32f");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -14873,7 +14492,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             /* Fall through */
         default:
             MIPS_INVAL("pool32i");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -14962,7 +14581,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("pool32c");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -15240,7 +14859,7 @@ static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
         gen_st(ctx, mips32_op, rt, rs, imm);
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -15252,8 +14871,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     /* make sure instructions are on a halfword boundary */
     if (ctx->pc & 0x1) {
         env->CP0_BadVAddr = ctx->pc;
-        generate_exception(ctx, EXCP_AdEL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_AdEL);
         return 2;
     }
 
@@ -15272,9 +14890,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
         case 7:
         /* LB32, LH32, LWC132, LDC132, LW32 */
             if (ctx->hflags & MIPS_HFLAG_BDS16) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -15285,9 +14901,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
         case 3:
         /* MOVE16, ANDI16, POOL16D, POOL16E, BEQZ16, BNEZ16, B16, LI16 */
             if (ctx->hflags & MIPS_HFLAG_BDS32) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -15359,7 +14973,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     case POOL16F:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & 1) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             /* MOVEP */
             int enc_dest = uMIPS_RD(ctx->opcode);
@@ -15497,7 +15111,7 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
     case RES_29:
     case RES_31:
     case RES_39:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     default:
         decode_micromips32_opc(env, ctx);
@@ -15519,7 +15133,6 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx)
 static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
                            int rd, int base, int offset)
 {
-    const char *opn = "ldx";
     TCGv t0;
 
     check_dsp(ctx);
@@ -15537,42 +15150,33 @@ static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
     case OPC_LBUX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rd);
-        opn = "lbux";
         break;
     case OPC_LHX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW);
         gen_store_gpr(t0, rd);
-        opn = "lhx";
         break;
     case OPC_LWX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rd);
-        opn = "lwx";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_LDX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rd);
-        opn = "ldx";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s(%s)", opn,
-               regnames[rd], regnames[offset], regnames[base]);
     tcg_temp_free(t0);
 }
 
 static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
                               int ret, int v1, int v2)
 {
-    const char *opn = "mipsdsp arith";
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16012,23 +15616,18 @@ static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
                               int ret, int v1, int v2)
 {
     uint32_t op2;
-    const char *opn = "mipsdsp shift";
     TCGv t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16135,7 +15734,7 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
                 break;
             default:            /* Invalid */
                 MIPS_INVAL("MASK SHLL.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -16250,7 +15849,7 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK SHLL.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16260,21 +15859,17 @@ static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
     tcg_temp_free(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                  int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp multiply";
     TCGv_i32 t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16572,23 +16167,17 @@ static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
     tcg_temp_free_i32(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
-
 }
 
 static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int val)
 {
-    const char *opn = "mipsdsp Bit/ Manipulation";
     int16_t imm;
     TCGv t0;
     TCGv val_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16716,23 +16305,18 @@ static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
     }
     tcg_temp_free(t0);
     tcg_temp_free(val_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
                                      uint32_t op1, uint32_t op2,
                                      int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp add compare pick";
     TCGv t1;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16907,22 +16491,17 @@ static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
                                uint32_t op1, int rt, int rs, int sa)
 {
-    const char *opn = "mipsdsp append/dappend";
     TCGv t0;
 
     check_dspr2(ctx);
 
     if (rt == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16959,7 +16538,7 @@ static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK APPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16993,22 +16572,19 @@ static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DAPPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     }
     tcg_temp_free(t0);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int v1, int v2, int check_ret)
 
 {
-    const char *opn = "mipsdsp accumulator";
     TCGv t0;
     TCGv t1;
     TCGv v1_t;
@@ -17017,7 +16593,6 @@ static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -17229,9 +16804,6 @@ static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 /* End MIPSDSP functions. */
@@ -17266,7 +16838,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17281,7 +16853,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
                We need additionally to check other fields */
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case R6_OPC_SDBBP:
@@ -17289,9 +16861,9 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             gen_helper_do_semihosting(cpu_env);
         } else {
             if (ctx->hflags & MIPS_HFLAG_SBRI) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
             } else {
-                generate_exception(ctx, EXCP_DBp);
+                generate_exception_end(ctx, EXCP_DBp);
             }
         }
         break;
@@ -17308,7 +16880,7 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             check_mips_64(ctx);
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_DMULT ... OPC_DDIVU:
@@ -17327,14 +16899,14 @@ static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17402,16 +16974,16 @@ static void decode_opc_special_legacy(CPUMIPSState *env, DisasContext *ctx)
     case OPC_SPIM:
 #ifdef MIPS_STRICT_STANDARD
         MIPS_INVAL("SPIM");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #else
         /* Implemented as RI exception for now. */
         MIPS_INVAL("spim (unofficial)");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #endif
         break;
     default:            /* Invalid */
         MIPS_INVAL("special_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17433,8 +17005,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             rs == 0 && rt == 0) { /* PAUSE */
             if ((ctx->insn_flags & ISA_MIPS32R6) &&
                 (ctx->hflags & MIPS_HFLAG_BMASK)) {
-                MIPS_DEBUG("CTI in delay / forbidden slot");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -17454,7 +17025,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17477,7 +17048,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17507,18 +17078,17 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             /* Pmon entry point, also R4010 selsl */
 #ifdef MIPS_STRICT_STANDARD
             MIPS_INVAL("PMON / selsl");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #else
             gen_helper_0e0i(pmon, sa);
 #endif
         }
         break;
     case OPC_SYSCALL:
-        generate_exception(ctx, EXCP_SYSCALL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_SYSCALL);
         break;
     case OPC_BREAK:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case OPC_SYNC:
         check_insn(ctx, ISA_MIPS2);
@@ -17549,7 +17119,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17567,7 +17137,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17596,7 +17166,7 @@ static void decode_opc_special(CPUMIPSState *env, DisasContext *ctx)
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17659,7 +17229,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
              *      when in debug mode...
              */
             check_insn(ctx, ISA_MIPS32);
-            generate_exception(ctx, EXCP_DBp);
+            generate_exception_end(ctx, EXCP_DBp);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -17681,7 +17251,7 @@ static void decode_opc_special2_legacy(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special2_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17703,12 +17273,15 @@ static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
     case R6_OPC_PREF:
         if (rt >= 24) {
             /* hint codes 24-31 are reserved and signal RI */
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         /* Treat as NOP. */
         break;
     case R6_OPC_CACHE:
-        /* Treat as NOP. */
+        check_cp0_enabled(ctx);
+        if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+            gen_cache_operation(ctx, rt, rs, imm);
+        }
         break;
     case R6_OPC_SC:
         gen_st_cond(ctx, op1, rt, rs, imm);
@@ -17762,7 +17335,7 @@ static void decode_opc_special3_r6(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17808,13 +17381,13 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
                 break;
             default:
                 MIPS_INVAL("MASK ADDUH.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else if (ctx->insn_flags & INSN_LOONGSON2E) {
             gen_loongson_integer(ctx, op1, rd, rs, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_LX_DSP:
@@ -17830,7 +17403,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK LX");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17861,7 +17434,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("MASK ABSQ_S.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17898,7 +17471,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
 
         }
@@ -17938,7 +17511,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU.EQ.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17974,7 +17547,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAW.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17987,7 +17560,6 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
                 TCGv t0, t1;
 
                 if (rt == 0) {
-                    MIPS_DEBUG("NOP");
                     break;
                 }
 
@@ -18005,7 +17577,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             }
         default:            /* Invalid */
             MIPS_INVAL("MASK INSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18040,7 +17612,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18083,7 +17655,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ABSQ_S.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18122,7 +17694,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18167,7 +17739,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU_EQ.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18204,7 +17776,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18243,7 +17815,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAQ.W.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18255,7 +17827,6 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
             TCGv t0, t1;
 
             if (rt == 0) {
-                MIPS_DEBUG("NOP");
                 break;
             }
             check_dsp(ctx);
@@ -18274,7 +17845,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
         }
         default:            /* Invalid */
             MIPS_INVAL("MASK DINSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -18284,7 +17855,7 @@ static void decode_opc_special3_legacy(CPUMIPSState *env, DisasContext *ctx)
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -18345,7 +17916,7 @@ static void decode_opc_special3(CPUMIPSState *env, DisasContext *ctx)
         break;
 #endif
     case OPC_RDHWR:
-        gen_rdhwr(ctx, rt, rd);
+        gen_rdhwr(ctx, rt, rd, extract32(ctx->opcode, 6, 3));
         break;
     case OPC_FORK:
         check_insn(ctx, ASE_MT);
@@ -18365,7 +17936,6 @@ static void decode_opc_special3(CPUMIPSState *env, DisasContext *ctx)
         {
             TCGv t0 = tcg_temp_new();
 
-            save_cpu_state(ctx, 1);
             gen_load_gpr(t0, rs);
             gen_helper_yield(t0, cpu_env, t0);
             gen_store_gpr(t0, rd);
@@ -18386,16 +17956,16 @@ static inline int check_msa_access(DisasContext *ctx)
 {
     if (unlikely((ctx->hflags & MIPS_HFLAG_FPU) &&
                  !(ctx->hflags & MIPS_HFLAG_F64))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return 0;
     }
 
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_MSA))) {
         if (ctx->insn_flags & ASE_MSA) {
-            generate_exception(ctx, EXCP_MSADIS);
+            generate_exception_end(ctx, EXCP_MSADIS);
             return 0;
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return 0;
         }
     }
@@ -18452,8 +18022,7 @@ static void gen_msa_branch(CPUMIPSState *env, DisasContext *ctx, uint32_t op1)
     check_msa_access(ctx);
 
     if (ctx->insn_flags & ISA_MIPS32R6 && ctx->hflags & MIPS_HFLAG_BMASK) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     switch (op1) {
@@ -18528,7 +18097,7 @@ static void gen_msa_i8(CPUMIPSState *env, DisasContext *ctx)
         {
             uint8_t df = (ctx->opcode >> 24) & 0x3;
             if (df == DF_DOUBLE) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
             } else {
                 TCGv_i32 tdf = tcg_const_i32(df);
                 gen_helper_msa_shf_df(cpu_env, tdf, twd, tws, ti8);
@@ -18538,7 +18107,7 @@ static void gen_msa_i8(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18610,7 +18179,7 @@ static void gen_msa_i5(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18646,7 +18215,7 @@ static void gen_msa_bit(CPUMIPSState *env, DisasContext *ctx)
         m = dfm & 0x7;
         df = DF_BYTE;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -18694,7 +18263,7 @@ static void gen_msa_bit(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18889,7 +18458,8 @@ static void gen_msa_3r(CPUMIPSState *env, DisasContext *ctx)
     case OPC_HSUB_S_df:
     case OPC_HSUB_U_df:
         if (df == DF_BYTE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         }
         switch (MASK_MSA_3R(ctx->opcode)) {
         case OPC_DOTP_S_df:
@@ -18926,7 +18496,7 @@ static void gen_msa_3r(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free_i32(twd);
@@ -18958,7 +18528,7 @@ static void gen_msa_elm_3e(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18995,7 +18565,7 @@ static void gen_msa_elm_df(CPUMIPSState *env, DisasContext *ctx, uint32_t df,
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -19013,7 +18583,7 @@ static void gen_msa_elm_df(CPUMIPSState *env, DisasContext *ctx, uint32_t df,
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
     tcg_temp_free_i32(twd);
     tcg_temp_free_i32(tws);
@@ -19043,7 +18613,7 @@ static void gen_msa_elm(CPUMIPSState *env, DisasContext *ctx)
         gen_msa_elm_3e(env, ctx);
         return;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -19198,7 +18768,7 @@ static void gen_msa_3rf(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19226,7 +18796,7 @@ static void gen_msa_2r(CPUMIPSState *env, DisasContext *ctx)
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -19243,7 +18813,7 @@ static void gen_msa_2r(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19358,7 +18928,7 @@ static void gen_msa_vec_v(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19387,7 +18957,7 @@ static void gen_msa_vec(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -19485,7 +19055,7 @@ static void gen_msa(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19502,7 +19072,6 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     if (ctx->pc & 0x3) {
         env->CP0_BadVAddr = ctx->pc;
         generate_exception_err(ctx, EXCP_AdEL, EXCP_INST_NOTAVAIL);
-        ctx->bstate = BS_STOP;
         return;
     }
 
@@ -19510,17 +19079,12 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     if ((ctx->hflags & MIPS_HFLAG_BMASK_BASE) == MIPS_HFLAG_BL) {
         TCGLabel *l1 = gen_new_label();
 
-        MIPS_DEBUG("blikely condition (" TARGET_FMT_lx ")", ctx->pc + 4);
         tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
         tcg_gen_movi_i32(hflags, ctx->hflags & ~MIPS_HFLAG_BMASK);
         gen_goto_tb(ctx, 1, ctx->pc + 4);
         gen_set_label(l1);
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     op = MASK_OP_MAJOR(ctx->opcode);
     rs = (ctx->opcode >> 21) & 0x1f;
     rt = (ctx->opcode >> 16) & 0x1f;
@@ -19558,7 +19122,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                     /* OPC_NAL, OPC_BAL */
                     gen_compute_branch(ctx, op1, 4, 0, -1, imm << 2, 4);
                 } else {
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                 }
             } else {
                 gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
@@ -19570,6 +19134,10 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_trap(ctx, op1, rs, -1, imm);
             break;
+        case OPC_SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         case OPC_SYNCI:
             check_insn(ctx, ISA_MIPS32R2);
             /* Break the TB to be able to sync copied instructions
@@ -19590,7 +19158,6 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 32);
             }
-            MIPS_DEBUG("dahi %s, %04x", regnames[rs], imm);
             break;
         case OPC_DATI:
             check_insn(ctx, ISA_MIPS32R6);
@@ -19598,12 +19165,11 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 48);
             }
-            MIPS_DEBUG("dati %s, %04x", regnames[rs], imm);
             break;
 #endif
         default:            /* Invalid */
             MIPS_INVAL("regimm");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19658,6 +19224,20 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                     gen_helper_evpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
+                case OPC_DVP:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    if (ctx->vp) {
+                        gen_helper_dvp(t0, cpu_env);
+                        gen_store_gpr(t0, rt);
+                    }
+                    break;
+                case OPC_EVP:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    if (ctx->vp) {
+                        gen_helper_evp(t0, cpu_env);
+                        gen_store_gpr(t0, rt);
+                    }
+                    break;
                 case OPC_DI:
                     check_insn(ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
@@ -19678,7 +19258,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                     break;
                 default:            /* Invalid */
                     MIPS_INVAL("mfmc0");
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                     break;
                 }
                 tcg_temp_free(t0);
@@ -19695,7 +19275,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("cp0");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19730,7 +19310,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC, OPC_BLEZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BLEZC, OPC_BGEZC, OPC_BGEC */
@@ -19743,7 +19323,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
     case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC, OPC_BGTZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BGTZC, OPC_BLTZC, OPC_BLTC */
@@ -19810,6 +19390,9 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_cp0_enabled(ctx);
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
+        if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+            gen_cache_operation(ctx, rt, rs, imm);
+        }
         /* Treat as NOP. */
         break;
     case OPC_PREF:
@@ -19973,7 +19556,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             break;
         default:
             MIPS_INVAL("cp1");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -20055,7 +19638,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
                 break;
             default:
                 MIPS_INVAL("cp3");
-                generate_exception (ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -20111,7 +19694,7 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
             gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
         } else {
             MIPS_INVAL("major opcode");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
@@ -20120,15 +19703,16 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
 #if defined(TARGET_MIPS64)
             /* OPC_DAUI */
             check_mips_64(ctx);
-            if (rt != 0) {
+            if (rs == 0) {
+                generate_exception(ctx, EXCP_RI);
+            } else if (rt != 0) {
                 TCGv t0 = tcg_temp_new();
                 gen_load_gpr(t0, rs);
                 tcg_gen_addi_tl(cpu_gpr[rt], t0, imm << 16);
                 tcg_temp_free(t0);
             }
-            MIPS_DEBUG("daui %s, %s, %04x", regnames[rt], regnames[rs], imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             MIPS_INVAL("major opcode");
 #endif
         } else {
@@ -20148,30 +19732,23 @@ static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
         break;
     default:            /* Invalid */
         MIPS_INVAL("major opcode");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
 
-static inline void
-gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMIPSState *env, struct TranslationBlock *tb)
 {
+    MIPSCPU *cpu = mips_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMIPSState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
     target_ulong next_page_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
     int insn_bytes;
     int is_slot;
 
-    if (search_pc)
-        qemu_log("search pc %d\n", search_pc);
-
     pc_start = tb->pc;
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
     ctx.pc = pc_start;
@@ -20181,6 +19758,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     ctx.CP0_Config1 = env->CP0_Config1;
     ctx.tb = tb;
     ctx.bstate = BS_NONE;
+    ctx.btarget = 0;
     ctx.kscrexist = (env->CP0_Config4 >> CP0C4_KScrExist) & 0xff;
     ctx.rxi = (env->CP0_Config3 >> CP0C3_RXI) & 1;
     ctx.ie = (env->CP0_Config4 >> CP0C4_IE) & 3;
@@ -20189,11 +19767,14 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     ctx.PAMask = env->PAMask;
     ctx.mvh = (env->CP0_Config5 >> CP0C5_MVH) & 1;
     ctx.CP0_LLAddr_shift = env->CP0_LLAddr_shift;
+    ctx.cmgcr = (env->CP0_Config3 >> CP0C3_CMGCR) & 1;
     /* Restore delay slot state from the tb context.  */
     ctx.hflags = (uint32_t)tb->flags; /* FIXME: maybe use 64 bits here? */
     ctx.ulri = (env->CP0_Config3 >> CP0C3_ULRI) & 1;
     ctx.ps = ((env->active_fpu.fcr0 >> FCR0_PS) & 1) ||
              (env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F));
+    ctx.vp = (env->CP0_Config5 >> CP0C5_VP) & 1;
+    ctx.mrp = (env->CP0_Config5 >> CP0C5_MRP) & 1;
     restore_cpu_state(env, &ctx);
 #ifdef CONFIG_USER_ONLY
         ctx.mem_idx = MIPS_HFLAG_UM;
@@ -20204,40 +19785,34 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
                                  MO_UNALN : MO_ALIGN;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     LOG_DISAS("\ntb %p idx %d hflags %04x\n", tb, ctx.mem_idx, ctx.hflags);
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    save_cpu_state(&ctx, 1);
-                    ctx.bstate = BS_BRANCH;
-                    gen_helper_0e0i(raise_exception, EXCP_DEBUG);
-                    /* Include the breakpoint location or the tb won't
-                     * be flushed when it must be.  */
-                    ctx.pc += 4;
-                    goto done_generating;
-                }
-            }
-        }
+        tcg_gen_insn_start(ctx.pc, ctx.hflags & MIPS_HFLAG_BMASK, ctx.btarget);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
-            gen_opc_btarget[lj] = ctx.btarget;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            save_cpu_state(&ctx, 1);
+            ctx.bstate = BS_BRANCH;
+            gen_helper_raise_exception_debug(cpu_env);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            goto done_generating;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         is_slot = ctx.hflags & MIPS_HFLAG_BMASK;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
@@ -20251,8 +19826,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
             ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx);
         } else {
-            generate_exception(&ctx, EXCP_RI);
-            ctx.bstate = BS_STOP;
+            generate_exception_end(&ctx, EXCP_RI);
             break;
         }
 
@@ -20275,8 +19849,6 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
         }
         ctx.pc += insn_bytes;
 
-        num_insns++;
-
         /* Execute a branch and its delay slot as a single instruction.
            This is what GDB expects and is consistent with what the
            hardware does (e.g. if a delay slot instruction faults, the
@@ -20304,7 +19876,7 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
     }
     if (cs->singlestep_enabled && ctx.bstate != BS_BRANCH) {
         save_cpu_state(&ctx, ctx.bstate != BS_EXCP);
-        gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+        gen_helper_raise_exception_debug(cpu_env);
     } else {
         switch (ctx.bstate) {
         case BS_STOP:
@@ -20325,15 +19897,9 @@ gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
     LOG_DISAS("\n");
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -20344,16 +19910,6 @@ done_generating:
 #endif
 }
 
-void gen_intermediate_code (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, true);
-}
-
 static void fpu_dump_state(CPUMIPSState *env, FILE *f, fprintf_function fpu_fprintf,
                            int flags)
 {
@@ -20394,40 +19950,6 @@ static void fpu_dump_state(CPUMIPSState *env, FILE *f, fprintf_function fpu_fpri
 #undef printfpr
 }
 
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-/* Debug help: The architecture requires 32bit code to maintain proper
-   sign-extended values on 64bit machines.  */
-
-#define SIGN_EXT_P(val) ((((val) & ~0x7fffffff) == 0) || (((val) & ~0x7fffffff) == ~0x7fffffff))
-
-static void
-cpu_mips_check_sign_extensions (CPUMIPSState *env, FILE *f,
-                                fprintf_function cpu_fprintf,
-                                int flags)
-{
-    int i;
-
-    if (!SIGN_EXT_P(env->active_tc.PC))
-        cpu_fprintf(f, "BROKEN: pc=0x" TARGET_FMT_lx "\n", env->active_tc.PC);
-    if (!SIGN_EXT_P(env->active_tc.HI[0]))
-        cpu_fprintf(f, "BROKEN: HI=0x" TARGET_FMT_lx "\n", env->active_tc.HI[0]);
-    if (!SIGN_EXT_P(env->active_tc.LO[0]))
-        cpu_fprintf(f, "BROKEN: LO=0x" TARGET_FMT_lx "\n", env->active_tc.LO[0]);
-    if (!SIGN_EXT_P(env->btarget))
-        cpu_fprintf(f, "BROKEN: btarget=0x" TARGET_FMT_lx "\n", env->btarget);
-
-    for (i = 0; i < 32; i++) {
-        if (!SIGN_EXT_P(env->active_tc.gpr[i]))
-            cpu_fprintf(f, "BROKEN: %s=0x" TARGET_FMT_lx "\n", regnames[i], env->active_tc.gpr[i]);
-    }
-
-    if (!SIGN_EXT_P(env->CP0_EPC))
-        cpu_fprintf(f, "BROKEN: EPC=0x" TARGET_FMT_lx "\n", env->CP0_EPC);
-    if (!SIGN_EXT_P(env->lladdr))
-        cpu_fprintf(f, "BROKEN: LLAddr=0x" TARGET_FMT_lx "\n", env->lladdr);
-}
-#endif
-
 void mips_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -20459,9 +19981,6 @@ void mips_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                 env->CP0_Config4, env->CP0_Config5);
     if (env->hflags & MIPS_HFLAG_FPU)
         fpu_dump_state(env, f, cpu_fprintf, flags);
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-    cpu_mips_check_sign_extensions(env, f, cpu_fprintf, flags);
-#endif
 }
 
 void mips_tcg_init(void)
@@ -20474,48 +19993,49 @@ void mips_tcg_init(void)
         return;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+
     TCGV_UNUSED(cpu_gpr[0]);
     for (i = 1; i < 32; i++)
-        cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr[i] = tcg_global_mem_new(cpu_env,
                                         offsetof(CPUMIPSState, active_tc.gpr[i]),
                                         regnames[i]);
 
     for (i = 0; i < 32; i++) {
         int off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[0]);
         msa_wr_d[i * 2] =
-                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2]);
+                tcg_global_mem_new_i64(cpu_env, off, msaregnames[i * 2]);
         /* The scalar floating-point unit (FPU) registers are mapped on
          * the MSA vector registers. */
         fpu_f64[i] = msa_wr_d[i * 2];
         off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[1]);
         msa_wr_d[i * 2 + 1] =
-                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2 + 1]);
+                tcg_global_mem_new_i64(cpu_env, off, msaregnames[i * 2 + 1]);
     }
 
-    cpu_PC = tcg_global_mem_new(TCG_AREG0,
+    cpu_PC = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUMIPSState, active_tc.PC), "PC");
     for (i = 0; i < MIPS_DSP_ACC; i++) {
-        cpu_HI[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_HI[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUMIPSState, active_tc.HI[i]),
                                        regnames_HI[i]);
-        cpu_LO[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_LO[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUMIPSState, active_tc.LO[i]),
                                        regnames_LO[i]);
     }
-    cpu_dspctrl = tcg_global_mem_new(TCG_AREG0,
+    cpu_dspctrl = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUMIPSState, active_tc.DSPControl),
                                      "DSPControl");
-    bcond = tcg_global_mem_new(TCG_AREG0,
+    bcond = tcg_global_mem_new(cpu_env,
                                offsetof(CPUMIPSState, bcond), "bcond");
-    btarget = tcg_global_mem_new(TCG_AREG0,
+    btarget = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUMIPSState, btarget), "btarget");
-    hflags = tcg_global_mem_new_i32(TCG_AREG0,
+    hflags = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUMIPSState, hflags), "hflags");
 
-    fpu_fcr0 = tcg_global_mem_new_i32(TCG_AREG0,
+    fpu_fcr0 = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUMIPSState, active_fpu.fcr0),
                                       "fcr0");
-    fpu_fcr31 = tcg_global_mem_new_i32(TCG_AREG0,
+    fpu_fcr31 = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUMIPSState, active_fpu.fcr31),
                                        "fcr31");
 
@@ -20548,6 +20068,16 @@ MIPSCPU *cpu_mips_init(const char *cpu_model)
     return cpu;
 }
 
+bool cpu_supports_cps_smp(const char *cpu_model)
+{
+    const mips_def_t *def = cpu_mips_find_by_name(cpu_model);
+    if (!def) {
+        return false;
+    }
+
+    return (def->CP0_Config3 & (1 << CP0C3_CMGCR)) != 0;
+}
+
 void cpu_state_reset(CPUMIPSState *env)
 {
     MIPSCPU *cpu = mips_env_get_cpu(env);
@@ -20598,6 +20128,7 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_PageGrain_rw_bitmask = env->cpu_model->CP0_PageGrain_rw_bitmask;
     env->CP0_PageGrain = env->cpu_model->CP0_PageGrain;
     env->active_fpu.fcr0 = env->cpu_model->CP1_fcr0;
+    env->active_fpu.fcr31 = env->cpu_model->CP1_fcr31;
     env->msair = env->cpu_model->MSAIR;
     env->insn_flags = env->cpu_model->insn_flags;
 
@@ -20640,12 +20171,16 @@ void cpu_state_reset(CPUMIPSState *env)
     env->CP0_Random = env->tlb->nb_tlb - 1;
     env->tlb->tlb_in_use = env->tlb->nb_tlb;
     env->CP0_Wired = 0;
+    env->CP0_GlobalNumber = (cs->cpu_index & 0xFF) << CP0GN_VPId;
     env->CP0_EBase = (cs->cpu_index & 0x3FF);
     if (kvm_enabled()) {
         env->CP0_EBase |= 0x40000000;
     } else {
         env->CP0_EBase |= 0x80000000;
     }
+    if (env->CP0_Config3 & (1 << CP0C3_CMGCR)) {
+        env->CP0_CMGCRBase = 0x1fbf8000 >> 4;
+    }
     env->CP0_Status = (1 << CP0St_BEV) | (1 << CP0St_ERL);
     /* vectored interrupts not implemented, timer on int 7,
        no performance counters. */
@@ -20714,18 +20249,19 @@ void cpu_state_reset(CPUMIPSState *env)
     }
 }
 
-void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->active_tc.PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->active_tc.PC = data[0];
     env->hflags &= ~MIPS_HFLAG_BMASK;
-    env->hflags |= gen_opc_hflags[pc_pos];
+    env->hflags |= data[1];
     switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
     case MIPS_HFLAG_BR:
         break;
     case MIPS_HFLAG_BC:
     case MIPS_HFLAG_BL:
     case MIPS_HFLAG_B:
-        env->btarget = gen_opc_btarget[pc_pos];
+        env->btarget = data[2];
         break;
     }
 }
diff --git a/qemu/target-mips/translate_init.c b/qemu/target-mips/translate_init.c
index 9304e746b..5af077d0d 100644
--- a/qemu/target-mips/translate_init.c
+++ b/qemu/target-mips/translate_init.c
@@ -84,6 +84,7 @@ struct mips_def_t {
     int32_t CP0_TCStatus_rw_bitmask;
     int32_t CP0_SRSCtl;
     int32_t CP1_fcr0;
+    int32_t CP1_fcr31;
     int32_t MSAIR;
     int32_t SEGBITS;
     int32_t PABITS;
@@ -389,39 +390,46 @@ static const mips_def_t mips_defs[] =
         .mmu_type = MMU_TYPE_R4000,
     },
     {
-        /* A generic CPU providing MIPS32 Release 5 features.
-           FIXME: Eventually this should be replaced by a real CPU model. */
-        .name = "mips32r5-generic",
-        .CP0_PRid = 0x00019700,
-        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+        /* FIXME:
+         * Config3: CMGCR, SC, PW, VZ, CTXTC, CDMM, TL
+         * Config4: MMUExtDef
+         * Config5: EVA, MRP
+         * FIR(FCR0): Has2008
+         * */
+        .name = "P5600",
+        .CP0_PRid = 0x0001A800,
+        .CP0_Config0 = MIPS_CONFIG0 | (1 << CP0C0_MM) | (1 << CP0C0_AR) |
                     (MMU_TYPE_R4000 << CP0C0_MT),
-        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
-                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
-                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
-                       (1 << CP0C1_CA),
+        .CP0_Config1 = MIPS_CONFIG1 | (0x3F << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_FP),
         .CP0_Config2 = MIPS_CONFIG2,
         .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP) |
-                       (1 << CP0C3_LPA),
-        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M),
+                       (1 << CP0C3_BP) | (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
+                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA) | (1 << CP0C3_VInt),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (2 << CP0C4_IE) |
+                       (0x1c << CP0C4_KScrExist),
         .CP0_Config4_rw_bitmask = 0,
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_UFR) | (1 << CP0C5_LLB) |
-                       (1 << CP0C5_MVH),
-        .CP0_Config5_rw_bitmask = (0 << CP0C5_M) | (1 << CP0C5_K) |
-                                  (1 << CP0C5_CV) | (0 << CP0C5_EVA) |
-                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFR) |
-                                  (0 << CP0C5_NFExists),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_MVH) | (1 << CP0C5_LLB) |
+                       (1 << CP0C5_MRP),
+        .CP0_Config5_rw_bitmask = (1 << CP0C5_K) | (1 << CP0C5_CV) |
+                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFE) |
+                                  (1 << CP0C5_FRE) | (1 << CP0C5_UFR),
         .CP0_LLAddr_rw_bitmask = 0,
-        .CP0_LLAddr_shift = 4,
+        .CP0_LLAddr_shift = 0,
         .SYNCI_Step = 32,
         .CCRes = 2,
-        .CP0_Status_rw_bitmask = 0x3778FF1F,
-        .CP0_PageGrain_rw_bitmask = (1 << CP0PG_ELPA),
-        .CP1_fcr0 = (1 << FCR0_UFRP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x93 << FCR0_PRID),
+        .CP0_Status_rw_bitmask = 0x3C68FF1F,
+        .CP0_PageGrain_rw_bitmask = (1U << CP0PG_RIE) | (1 << CP0PG_XIE) |
+                    (1 << CP0PG_ELPA) | (1 << CP0PG_IEC),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_UFRP) | (1 << FCR0_HAS2008) |
+                    (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) | (0x03 << FCR0_PRID),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
         .SEGBITS = 32,
         .PABITS = 40,
-        .insn_flags = CPU_MIPS32R5 | ASE_MIPS16 | ASE_MSA,
+        .insn_flags = CPU_MIPS32R5 | ASE_MSA,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
@@ -442,7 +450,7 @@ static const mips_def_t mips_defs[] =
                        (1 << CP0C3_RXI) | (1U << CP0C3_M),
         .CP0_Config4 = MIPS_CONFIG4 | (0xfc << CP0C4_KScrExist) |
                        (3 << CP0C4_IE) | (1U << CP0C4_M),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_LLB),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_SBRI) | (1 << CP0C5_FRE) |
                                   (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
@@ -453,9 +461,10 @@ static const mips_def_t mips_defs[] =
         .CP0_PageGrain = (1 << CP0PG_IEC) | (1 << CP0PG_XIE) |
                          (1U << CP0PG_RIE),
         .CP0_PageGrain_rw_bitmask = 0,
-        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_HAS2008) | (1 << FCR0_F64) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
         .SEGBITS = 32,
         .PABITS = 32,
         .insn_flags = CPU_MIPS32R6 | ASE_MICROMIPS,
@@ -655,12 +664,14 @@ static const mips_def_t mips_defs[] =
                        (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
                        (0 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP) |
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) |
+                       (1 << CP0C3_CMGCR) | (1 << CP0C3_MSAP) |
                        (1 << CP0C3_BP) | (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
                        (1 << CP0C3_RXI) | (1 << CP0C3_LPA),
         .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (3 << CP0C4_IE) |
                        (0xfc << CP0C4_KScrExist),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_VP) |
+                       (1 << CP0C5_LLB),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_MSAEn) | (1 << CP0C5_SBRI) |
                                   (1 << CP0C5_FRE) | (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
@@ -671,9 +682,10 @@ static const mips_def_t mips_defs[] =
         .CP0_PageGrain = (1 << CP0PG_IEC) | (1 << CP0PG_XIE) |
                          (1U << CP0PG_RIE),
         .CP0_PageGrain_rw_bitmask = (1 << CP0PG_ELPA),
-        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_HAS2008) | (1 << FCR0_F64) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
         .SEGBITS = 48,
         .PABITS = 48,
         .insn_flags = CPU_MIPS64R6 | ASE_MSA,
diff --git a/qemu/target-moxie/cpu.c b/qemu/target-moxie/cpu.c
index 6b035aaab..b4ee84e90 100644
--- a/qemu/target-moxie/cpu.c
+++ b/qemu/target-moxie/cpu.c
@@ -17,6 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -48,6 +50,12 @@ static void moxie_cpu_reset(CPUState *s)
     tlb_flush(s, 1);
 }
 
+static void moxie_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_arch_moxie;
+    info->print_insn = print_insn_moxie;
+}
+
 static void moxie_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -114,6 +122,14 @@ static void moxie_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = moxie_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_moxie_cpu;
 #endif
+    cc->disas_set_info = moxie_cpu_disas_set_info;
+
+    /*
+     * Reason: moxie_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void moxielite_initfn(Object *obj)
diff --git a/qemu/target-moxie/cpu.h b/qemu/target-moxie/cpu.h
index 29572aaba..4ee207796 100644
--- a/qemu/target-moxie/cpu.h
+++ b/qemu/target-moxie/cpu.h
@@ -19,15 +19,12 @@
 #ifndef _CPU_MOXIE_H
 #define _CPU_MOXIE_H
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 32
 
 #define CPUArchState struct CPUMoxieState
 
-#define ELF_MACHINE     0xFEED /* EM_MOXIE */
-
 #define MOXIE_EX_DIV0        0
 #define MOXIE_EX_BAD         1
 #define MOXIE_EX_IRQ         2
@@ -124,10 +121,9 @@ int cpu_moxie_signal_handler(int host_signum, void *pinfo,
 #define cpu_init(cpu_model) CPU(cpu_moxie_init(cpu_model))
 
 #define cpu_exec cpu_moxie_exec
-#define cpu_gen_code cpu_moxie_gen_code
 #define cpu_signal_handler cpu_moxie_signal_handler
 
-static inline int cpu_mmu_index(CPUMoxieState *env)
+static inline int cpu_mmu_index(CPUMoxieState *env, bool ifetch)
 {
     return 0;
 }
diff --git a/qemu/target-moxie/helper.c b/qemu/target-moxie/helper.c
index 6c98965b9..d51e9b9cc 100644
--- a/qemu/target-moxie/helper.c
+++ b/qemu/target-moxie/helper.c
@@ -17,11 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/exec-all.h"
@@ -56,7 +53,7 @@ void helper_raise_exception(CPUMoxieState *env, int ex)
     /* Stash the address where the exception occurred.  */
     cpu_restore_state(cs, GETPC());
     env->sregs[5] = env->pc;
-    /* Jump the the exception handline routine.  */
+    /* Jump to the exception handline routine.  */
     env->pc = env->sregs[1];
     cpu_loop_exit(cs);
 }
diff --git a/qemu/target-moxie/machine.c b/qemu/target-moxie/machine.c
index b9316f0ec..912b79139 100644
--- a/qemu/target-moxie/machine.c
+++ b/qemu/target-moxie/machine.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "machine.h"
diff --git a/qemu/target-moxie/mmu.c b/qemu/target-moxie/mmu.c
index 5217eedac..9203330b3 100644
--- a/qemu/target-moxie/mmu.c
+++ b/qemu/target-moxie/mmu.c
@@ -17,11 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/exec-all.h"
diff --git a/qemu/target-moxie/translate.c b/qemu/target-moxie/translate.c
index e3e913906..a437e2ab6 100644
--- a/qemu/target-moxie/translate.c
+++ b/qemu/target-moxie/translate.c
@@ -21,12 +21,7 @@
  *    http://moxielogic.org/wiki
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -36,6 +31,7 @@
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
+#include "exec/log.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
@@ -60,7 +56,7 @@ enum {
 
 static TCGv cpu_pc;
 static TCGv cpu_gregs[16];
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cc_a, cc_b;
 
 #include "exec/gen-icount.h"
@@ -110,16 +106,16 @@ void moxie_translate_init(void)
         return;
     }
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUMoxieState, pc), "$pc");
     for (i = 0; i < 16; i++)
-        cpu_gregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_gregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUMoxieState, gregs[i]),
                                               gregnames[i]);
 
-    cc_a = tcg_global_mem_new_i32(TCG_AREG0,
+    cc_a = tcg_global_mem_new_i32(cpu_env,
                                   offsetof(CPUMoxieState, cc_a), "cc_a");
-    cc_b = tcg_global_mem_new_i32(TCG_AREG0,
+    cc_b = tcg_global_mem_new_i32(cpu_env,
                                   offsetof(CPUMoxieState, cc_b), "cc_b");
 
     done_init = 1;
@@ -154,10 +150,6 @@ static int decode_opc(MoxieCPU *cpu, DisasContext *ctx)
     /* Set the default instruction length.  */
     int length = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     /* Examine the 16-bit opcode.  */
     opcode = ctx->opcode;
 
@@ -820,17 +812,13 @@ static int decode_opc(MoxieCPU *cpu, DisasContext *ctx)
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
 {
+    MoxieCPU *cpu = moxie_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
-    CPUMoxieState *env = &cpu->env;
-    int num_insns;
+    int num_insns, max_insns;
 
     pc_start = tb->pc;
     ctx.pc = pc_start;
@@ -840,40 +828,40 @@ gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
     ctx.singlestep_enabled = 0;
     ctx.bstate = BS_NONE;
     num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-                    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_EXCP;
-                    goto done_generating;
-                }
-            }
-        }
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_EXCP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            goto done_generating;
         }
+
         ctx.opcode = cpu_lduw_code(env, ctx.pc);
         ctx.pc += decode_opc(cpu, &ctx);
-        num_insns++;
 
+        if (num_insns >= max_insns) {
+            break;
+        }
         if (cs->singlestep_enabled) {
             break;
         }
-
         if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0) {
             break;
         }
@@ -899,29 +887,12 @@ gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
  done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUMoxieState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, true);
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 }
 
-void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-openrisc/cpu.c b/qemu/target-openrisc/cpu.c
index d97f3c03c..ae6ed9e92 100644
--- a/qemu/target-openrisc/cpu.c
+++ b/qemu/target-openrisc/cpu.c
@@ -17,6 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 
@@ -177,6 +179,13 @@ static void openrisc_cpu_class_init(ObjectClass *oc, void *data)
     dc->vmsd = &vmstate_openrisc_cpu;
 #endif
     cc->gdb_num_core_regs = 32 + 3;
+
+    /*
+     * Reason: openrisc_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const OpenRISCCPUInfo *info)
diff --git a/qemu/target-openrisc/cpu.h b/qemu/target-openrisc/cpu.h
index 36c4f20e2..4b63f2580 100644
--- a/qemu/target-openrisc/cpu.h
+++ b/qemu/target-openrisc/cpu.h
@@ -21,14 +21,12 @@
 #define CPU_OPENRISC_H
 
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE    EM_OPENRISC
 
 #define CPUArchState struct CPUOpenRISCState
 
 /* cpu_openrisc_map_address_* in CPUOpenRISCTLBContext need this decl.  */
 struct OpenRISCCPU;
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
@@ -361,7 +359,6 @@ int cpu_openrisc_signal_handler(int host_signum, void *pinfo, void *puc);
 
 #define cpu_list cpu_openrisc_list
 #define cpu_exec cpu_openrisc_exec
-#define cpu_gen_code cpu_openrisc_gen_code
 #define cpu_signal_handler cpu_openrisc_signal_handler
 
 #ifndef CONFIG_USER_ONLY
@@ -403,7 +400,7 @@ static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
     *flags = (env->flags & D_FLAG);
 }
 
-static inline int cpu_mmu_index(CPUOpenRISCState *env)
+static inline int cpu_mmu_index(CPUOpenRISCState *env, bool ifetch)
 {
     if (!(env->sr & SR_IME)) {
         return MMU_NOMMU_IDX;
diff --git a/qemu/target-openrisc/exception.c b/qemu/target-openrisc/exception.c
index 74652a58f..ace3184d5 100644
--- a/qemu/target-openrisc/exception.c
+++ b/qemu/target-openrisc/exception.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exception.h"
 
diff --git a/qemu/target-openrisc/exception_helper.c b/qemu/target-openrisc/exception_helper.c
index 6093953c9..329a9e400 100644
--- a/qemu/target-openrisc/exception_helper.c
+++ b/qemu/target-openrisc/exception_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/qemu/target-openrisc/fpu_helper.c b/qemu/target-openrisc/fpu_helper.c
index c94ed35af..c54404b80 100644
--- a/qemu/target-openrisc/fpu_helper.c
+++ b/qemu/target-openrisc/fpu_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/qemu/target-openrisc/gdbstub.c b/qemu/target-openrisc/gdbstub.c
index 18bcc4616..edc301a7c 100644
--- a/qemu/target-openrisc/gdbstub.c
+++ b/qemu/target-openrisc/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-openrisc/int_helper.c b/qemu/target-openrisc/int_helper.c
index 6e27aebd9..4d1f95890 100644
--- a/qemu/target-openrisc/int_helper.c
+++ b/qemu/target-openrisc/int_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/qemu/target-openrisc/interrupt.c b/qemu/target-openrisc/interrupt.c
index e480cfd1b..963eb1478 100644
--- a/qemu/target-openrisc/interrupt.c
+++ b/qemu/target-openrisc/interrupt.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
diff --git a/qemu/target-openrisc/interrupt_helper.c b/qemu/target-openrisc/interrupt_helper.c
index 55a780c7b..11b4b2056 100644
--- a/qemu/target-openrisc/interrupt_helper.c
+++ b/qemu/target-openrisc/interrupt_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-openrisc/machine.c b/qemu/target-openrisc/machine.c
index 9f66a9cef..b4dc08dfe 100644
--- a/qemu/target-openrisc/machine.c
+++ b/qemu/target-openrisc/machine.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 
diff --git a/qemu/target-openrisc/mmu.c b/qemu/target-openrisc/mmu.c
index 750a93636..4ab414a68 100644
--- a/qemu/target-openrisc/mmu.c
+++ b/qemu/target-openrisc/mmu.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
diff --git a/qemu/target-openrisc/mmu_helper.c b/qemu/target-openrisc/mmu_helper.c
index ee1c6f611..d7952d449 100644
--- a/qemu/target-openrisc/mmu_helper.c
+++ b/qemu/target-openrisc/mmu_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu_ldst.h"
 
diff --git a/qemu/target-openrisc/sys_helper.c b/qemu/target-openrisc/sys_helper.c
index 53ca6bcef..f917be6be 100644
--- a/qemu/target-openrisc/sys_helper.c
+++ b/qemu/target-openrisc/sys_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-openrisc/translate.c b/qemu/target-openrisc/translate.c
index a62cbf401..5d0ab442a 100644
--- a/qemu/target-openrisc/translate.c
+++ b/qemu/target-openrisc/translate.c
@@ -18,13 +18,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu-common.h"
 #include "qemu/log.h"
-#include "config.h"
 #include "qemu/bitops.h"
 #include "exec/cpu_ldst.h"
 
@@ -32,6 +32,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define OPENRISC_DISAS
@@ -52,7 +53,7 @@ typedef struct DisasContext {
     uint32_t delayed_branch;
 } DisasContext;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_sr;
 static TCGv cpu_R[32];
 static TCGv cpu_pc;
@@ -77,39 +78,39 @@ void openrisc_translate_init(void)
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_sr = tcg_global_mem_new(TCG_AREG0,
+    cpu_sr = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, sr), "sr");
-    env_flags = tcg_global_mem_new_i32(TCG_AREG0,
+    env_flags = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUOpenRISCState, flags),
                                        "flags");
-    cpu_pc = tcg_global_mem_new(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, pc), "pc");
-    cpu_npc = tcg_global_mem_new(TCG_AREG0,
+    cpu_npc = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUOpenRISCState, npc), "npc");
-    cpu_ppc = tcg_global_mem_new(TCG_AREG0,
+    cpu_ppc = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUOpenRISCState, ppc), "ppc");
-    jmp_pc = tcg_global_mem_new(TCG_AREG0,
+    jmp_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, jmp_pc), "jmp_pc");
-    env_btaken = tcg_global_mem_new_i32(TCG_AREG0,
+    env_btaken = tcg_global_mem_new_i32(cpu_env,
                                         offsetof(CPUOpenRISCState, btaken),
                                         "btaken");
-    fpcsr = tcg_global_mem_new_i32(TCG_AREG0,
+    fpcsr = tcg_global_mem_new_i32(cpu_env,
                                    offsetof(CPUOpenRISCState, fpcsr),
                                    "fpcsr");
-    machi = tcg_global_mem_new(TCG_AREG0,
+    machi = tcg_global_mem_new(cpu_env,
                                offsetof(CPUOpenRISCState, machi),
                                "machi");
-    maclo = tcg_global_mem_new(TCG_AREG0,
+    maclo = tcg_global_mem_new(cpu_env,
                                offsetof(CPUOpenRISCState, maclo),
                                "maclo");
-    fpmaddhi = tcg_global_mem_new(TCG_AREG0,
+    fpmaddhi = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUOpenRISCState, fpmaddhi),
                                   "fpmaddhi");
-    fpmaddlo = tcg_global_mem_new(TCG_AREG0,
+    fpmaddlo = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUOpenRISCState, fpmaddlo),
                                   "fpmaddlo");
     for (i = 0; i < 32; i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                                       offsetof(CPUOpenRISCState, gpr[i]),
                                       regnames[i]);
     }
@@ -279,7 +280,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_add_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -324,7 +325,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_shri_i64(tcy, tcy, 10);
                 tcg_gen_add_i64(td, ta, tb);
                 tcg_gen_add_i64(td, td, tcy);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -366,7 +367,7 @@ static void dec_calc(DisasContext *dc, uint32_t insn)
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_sub_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -779,9 +780,9 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, dst);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(dst);
             tcg_temp_free(ttmp);
             tcg_temp_free_i64(t1);
@@ -898,7 +899,7 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -934,7 +935,7 @@ static void dec_misc(DisasContext *dc, uint32_t insn)
             tcg_gen_extu_i32_i64(tcy, sr_cy);
             tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
             tcg_gen_add_i64(td, td, tcy);
-            tcg_gen_trunc_i64_i32(res, td);
+            tcg_gen_extrl_i64_i32(res, td);
             tcg_gen_shri_i64(td, td, 32);
             tcg_gen_andi_i64(td, td, 0x3);
             /* Jump to lab when no overflow.  */
@@ -1073,9 +1074,9 @@ static void dec_mac(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1092,9 +1093,9 @@ static void dec_mac(DisasContext *dc, uint32_t insn)
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_sub_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1618,30 +1619,12 @@ static void disas_openrisc_insn(DisasContext *dc, OpenRISCCPU *cpu)
     }
 }
 
-static void check_breakpoint(OpenRISCCPU *cpu, DisasContext *dc)
-{
-    CPUState *cs = CPU(cpu);
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
-static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  int search_pc)
+void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
 {
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, k;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1653,7 +1636,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     dc->ppc = pc_start;
     dc->pc = pc_start;
     dc->flags = cpu->env.cpucfgr;
-    dc->mem_idx = cpu_mmu_index(&cpu->env);
+    dc->mem_idx = cpu_mmu_index(&cpu->env, false);
     dc->synced_flags = dc->tb_flags = tb->flags;
     dc->delayed_branch = !!(dc->tb_flags & D_FLAG);
     dc->singlestep_enabled = cs->singlestep_enabled;
@@ -1663,36 +1646,35 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    k = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
 
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     do {
-        check_breakpoint(cpu, dc);
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (k < j) {
-                k++;
-                while (k < j) {
-                    tcg_ctx.gen_opc_instr_start[k++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[k] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[k] = 1;
-            tcg_ctx.gen_opc_icount[k] = num_insns;
-        }
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            gen_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->ppc = dc->pc - 4;
@@ -1701,7 +1683,6 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
         tcg_gen_movi_tl(cpu_npc, dc->npc);
         disas_openrisc_insn(dc, cpu);
         dc->pc = dc->npc;
-        num_insns++;
         /* delay slot */
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1756,16 +1737,8 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        k++;
-        while (k <= j) {
-            tcg_ctx.gen_opc_instr_start[k++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1777,17 +1750,6 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 0);
-}
-
-void gen_intermediate_code_pc(CPUOpenRISCState *env,
-                              struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 1);
-}
-
 void openrisc_cpu_dump_state(CPUState *cs, FILE *f,
                              fprintf_function cpu_fprintf,
                              int flags)
@@ -1804,7 +1766,7 @@ void openrisc_cpu_dump_state(CPUState *cs, FILE *f,
 }
 
 void restore_state_to_opc(CPUOpenRISCState *env, TranslationBlock *tb,
-                          int pc_pos)
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-ppc/Makefile.objs b/qemu/target-ppc/Makefile.objs
index a7ae392cc..e667e6970 100644
--- a/qemu/target-ppc/Makefile.objs
+++ b/qemu/target-ppc/Makefile.objs
@@ -1,10 +1,10 @@
 obj-y += cpu-models.o
 obj-y += translate.o
 ifeq ($(CONFIG_SOFTMMU),y)
-obj-y += machine.o mmu_helper.o mmu-hash32.o
+obj-y += machine.o mmu_helper.o mmu-hash32.o monitor.o
 obj-$(TARGET_PPC64) += mmu-hash64.o arch_dump.o
 endif
-obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
+obj-$(CONFIG_KVM) += kvm.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += dfp_helper.o
 obj-y += excp_helper.o
diff --git a/qemu/target-ppc/arch_dump.c b/qemu/target-ppc/arch_dump.c
index 5acafc68a..df1fd8c33 100644
--- a/qemu/target-ppc/arch_dump.c
+++ b/qemu/target-ppc/arch_dump.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "elf.h"
 #include "exec/cpu-all.h"
@@ -278,9 +279,3 @@ int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
     PowerPCCPU *cpu = POWERPC_CPU(cs);
     return ppc64_write_all_elf64_notes("CORE", f, cpu, cpuid, opaque);
 }
-
-int ppc64_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                   CPUState *cpu, void *opaque)
-{
-    return 0;
-}
diff --git a/qemu/target-ppc/cpu-models.c b/qemu/target-ppc/cpu-models.c
index 4d5ab4ba1..5209e63a7 100644
--- a/qemu/target-ppc/cpu-models.c
+++ b/qemu/target-ppc/cpu-models.c
@@ -24,6 +24,7 @@
  * inside "#if defined(TODO) ... #endif" statements to make tests easier.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "cpu-models.h"
 
@@ -1138,10 +1139,12 @@
                 "POWER7 v2.3")
     POWERPC_DEF("POWER7+_v2.1",  CPU_POWERPC_POWER7P_v21,            POWER7,
                 "POWER7+ v2.1")
-    POWERPC_DEF("POWER8E_v1.0",  CPU_POWERPC_POWER8E_v10,            POWER8,
-                "POWER8E v1.0")
-    POWERPC_DEF("POWER8_v1.0",   CPU_POWERPC_POWER8_v10,             POWER8,
-                "POWER8 v1.0")
+    POWERPC_DEF("POWER8E_v2.1",  CPU_POWERPC_POWER8E_v21,            POWER8,
+                "POWER8E v2.1")
+    POWERPC_DEF("POWER8_v2.0",   CPU_POWERPC_POWER8_v20,             POWER8,
+                "POWER8 v2.0")
+    POWERPC_DEF("POWER8NVL_v1.0",CPU_POWERPC_POWER8NVL_v10,          POWER8,
+                "POWER8NVL v1.0")
     POWERPC_DEF("970_v2.2",      CPU_POWERPC_970_v22,                970,
                 "PowerPC 970 v2.2")
     POWERPC_DEF("970fx_v1.0",    CPU_POWERPC_970FX_v10,              970,
@@ -1389,8 +1392,9 @@ PowerPCCPUAlias ppc_cpu_aliases[] = {
     { "POWER5gs", "POWER5+_v2.1" },
     { "POWER7", "POWER7_v2.3" },
     { "POWER7+", "POWER7+_v2.1" },
-    { "POWER8E", "POWER8E_v1.0" },
-    { "POWER8", "POWER8_v1.0" },
+    { "POWER8E", "POWER8E_v2.1" },
+    { "POWER8", "POWER8_v2.0" },
+    { "POWER8NVL", "POWER8NVL_v1.0" },
     { "970", "970_v2.2" },
     { "970fx", "970fx_v3.1" },
     { "970mp", "970mp_v1.1" },
diff --git a/qemu/target-ppc/cpu-models.h b/qemu/target-ppc/cpu-models.h
index 9d80e7227..f21a44c83 100644
--- a/qemu/target-ppc/cpu-models.h
+++ b/qemu/target-ppc/cpu-models.h
@@ -557,9 +557,11 @@ enum {
     CPU_POWERPC_POWER7P_BASE       = 0x004A0000,
     CPU_POWERPC_POWER7P_v21        = 0x004A0201,
     CPU_POWERPC_POWER8E_BASE       = 0x004B0000,
-    CPU_POWERPC_POWER8E_v10        = 0x004B0100,
+    CPU_POWERPC_POWER8E_v21        = 0x004B0201,
     CPU_POWERPC_POWER8_BASE        = 0x004D0000,
-    CPU_POWERPC_POWER8_v10         = 0x004D0100,
+    CPU_POWERPC_POWER8_v20         = 0x004D0200,
+    CPU_POWERPC_POWER8NVL_BASE     = 0x004C0000,
+    CPU_POWERPC_POWER8NVL_v10      = 0x004C0100,
     CPU_POWERPC_970_v22            = 0x00390202,
     CPU_POWERPC_970FX_v10          = 0x00391100,
     CPU_POWERPC_970FX_v20          = 0x003C0200,
diff --git a/qemu/target-ppc/cpu-qom.h b/qemu/target-ppc/cpu-qom.h
index 6967a8028..7d5e2b36a 100644
--- a/qemu/target-ppc/cpu-qom.h
+++ b/qemu/target-ppc/cpu-qom.h
@@ -118,13 +118,13 @@ void ppc_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
                         int flags);
 void ppc_cpu_dump_statistics(CPUState *cpu, FILE *f,
                              fprintf_function cpu_fprintf, int flags);
+int ppc_cpu_get_monitor_def(CPUState *cs, const char *name,
+                            uint64_t *pval);
 hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int ppc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int ppc_cpu_gdb_read_register_apple(CPUState *cpu, uint8_t *buf, int reg);
 int ppc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 int ppc_cpu_gdb_write_register_apple(CPUState *cpu, uint8_t *buf, int reg);
-int ppc64_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                   CPUState *cpu, void *opaque);
 int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
                                int cpuid, void *opaque);
 #ifndef CONFIG_USER_ONLY
diff --git a/qemu/target-ppc/cpu.h b/qemu/target-ppc/cpu.h
index 6f76674a4..5282533b3 100644
--- a/qemu/target-ppc/cpu.h
+++ b/qemu/target-ppc/cpu.h
@@ -19,7 +19,6 @@
 #if !defined (__CPU_PPC_H__)
 #define __CPU_PPC_H__
 
-#include "config.h"
 #include "qemu-common.h"
 
 //#define PPC_EMULATE_32BITS_HYPV
@@ -81,9 +80,9 @@
 #include "fpu/softfloat.h"
 
 #if defined (TARGET_PPC64)
-#define ELF_MACHINE     EM_PPC64
+#define PPC_ELF_MACHINE     EM_PPC64
 #else
-#define ELF_MACHINE     EM_PPC
+#define PPC_ELF_MACHINE     EM_PPC
 #endif
 
 /*****************************************************************************/
@@ -117,14 +116,20 @@ enum powerpc_mmu_t {
 #define POWERPC_MMU_AMR      0x00040000
     /* 64 bits PowerPC MMU                                     */
     POWERPC_MMU_64B        = POWERPC_MMU_64 | 0x00000001,
+    /* Architecture 2.03 and later (has LPCR) */
+    POWERPC_MMU_2_03       = POWERPC_MMU_64 | 0x00000002,
     /* Architecture 2.06 variant                               */
     POWERPC_MMU_2_06       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
                              | POWERPC_MMU_AMR | 0x00000003,
     /* Architecture 2.06 "degraded" (no 1T segments)           */
     POWERPC_MMU_2_06a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
                              | 0x00000003,
-    /* Architecture 2.06 "degraded" (no 1T segments or AMR)    */
-    POWERPC_MMU_2_06d      = POWERPC_MMU_64 | 0x00000003,
+    /* Architecture 2.07 variant                               */
+    POWERPC_MMU_2_07       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
+                             | POWERPC_MMU_AMR | 0x00000004,
+    /* Architecture 2.07 "degraded" (no 1T segments)           */
+    POWERPC_MMU_2_07a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
+                             | 0x00000004,
 #endif /* defined(TARGET_PPC64) */
 };
 
@@ -162,6 +167,8 @@ enum powerpc_excp_t {
     POWERPC_EXCP_970,
     /* POWER7 exception model           */
     POWERPC_EXCP_POWER7,
+    /* POWER8 exception model           */
+    POWERPC_EXCP_POWER8,
 #endif /* defined(TARGET_PPC64) */
 };
 
@@ -413,6 +420,7 @@ typedef struct ppc_slb_t ppc_slb_t;
 struct ppc_slb_t {
     uint64_t esid;
     uint64_t vsid;
+    const struct ppc_one_seg_page_size *sps;
 };
 
 #define MAX_SLB_ENTRIES         64
@@ -468,9 +476,17 @@ struct ppc_slb_t {
 #define MSR_RI   1  /* Recoverable interrupt                        1        */
 #define MSR_LE   0  /* Little-endian mode                           1 hflags */
 
-#define LPCR_ILE (1 << (63-38))
-#define LPCR_AIL_SHIFT (63-40)      /* Alternate interrupt location */
-#define LPCR_AIL (3 << LPCR_AIL_SHIFT)
+/* LPCR bits */
+#define LPCR_VPM0         (1ull << (63 - 0))
+#define LPCR_VPM1         (1ull << (63 - 1))
+#define LPCR_ISL          (1ull << (63 - 2))
+#define LPCR_KBV          (1ull << (63 - 3))
+#define LPCR_ILE          (1ull << (63 - 38))
+#define LPCR_MER          (1ull << (63 - 52))
+#define LPCR_LPES0        (1ull << (63 - 60))
+#define LPCR_LPES1        (1ull << (63 - 61))
+#define LPCR_AIL_SHIFT    (63 - 40)      /* Alternate interrupt location */
+#define LPCR_AIL          (3ull << LPCR_AIL_SHIFT)
 
 #define msr_sf   ((env->msr >> MSR_SF)   & 1)
 #define msr_isf  ((env->msr >> MSR_ISF)  & 1)
@@ -678,6 +694,46 @@ enum {
 #define fpscr_eex (((env->fpscr) >> FPSCR_XX) & ((env->fpscr) >> FPSCR_XE) &  \
                    0x1F)
 
+#define FP_FX		(1ull << FPSCR_FX)
+#define FP_FEX		(1ull << FPSCR_FEX)
+#define FP_VX		(1ull << FPSCR_VX)
+#define FP_OX		(1ull << FPSCR_OX)
+#define FP_UX		(1ull << FPSCR_UX)
+#define FP_ZX		(1ull << FPSCR_ZX)
+#define FP_XX		(1ull << FPSCR_XX)
+#define FP_VXSNAN	(1ull << FPSCR_VXSNAN)
+#define FP_VXISI	(1ull << FPSCR_VXISI)
+#define FP_VXIDI	(1ull << FPSCR_VXIDI)
+#define FP_VXZDZ	(1ull << FPSCR_VXZDZ)
+#define FP_VXIMZ	(1ull << FPSCR_VXIMZ)
+#define FP_VXVC		(1ull << FPSCR_VXVC)
+#define FP_FR		(1ull << FSPCR_FR)
+#define FP_FI		(1ull << FPSCR_FI)
+#define FP_C		(1ull << FPSCR_C)
+#define FP_FL		(1ull << FPSCR_FL)
+#define FP_FG		(1ull << FPSCR_FG)
+#define FP_FE		(1ull << FPSCR_FE)
+#define FP_FU		(1ull << FPSCR_FU)
+#define FP_FPCC		(FP_FL | FP_FG | FP_FE | FP_FU)
+#define FP_FPRF		(FP_C  | FP_FL | FP_FG | FP_FE | FP_FU)
+#define FP_VXSOFT	(1ull << FPSCR_VXSOFT)
+#define FP_VXSQRT	(1ull << FPSCR_VXSQRT)
+#define FP_VXCVI	(1ull << FPSCR_VXCVI)
+#define FP_VE		(1ull << FPSCR_VE)
+#define FP_OE		(1ull << FPSCR_OE)
+#define FP_UE		(1ull << FPSCR_UE)
+#define FP_ZE		(1ull << FPSCR_ZE)
+#define FP_XE		(1ull << FPSCR_XE)
+#define FP_NI		(1ull << FPSCR_NI)
+#define FP_RN1		(1ull << FPSCR_RN1)
+#define FP_RN		(1ull << FPSCR_RN)
+
+/* the exception bits which can be cleared by mcrfs - includes FX */
+#define FP_EX_CLEAR_BITS (FP_FX     | FP_OX     | FP_UX     | FP_ZX     | \
+                          FP_XX     | FP_VXSNAN | FP_VXISI  | FP_VXIDI  | \
+                          FP_VXZDZ  | FP_VXIMZ  | FP_VXVC   | FP_VXSOFT | \
+                          FP_VXSQRT | FP_VXCVI)
+
 /*****************************************************************************/
 /* Vector status and control register */
 #define VSCR_NJ		16 /* Vector non-java */
@@ -1073,6 +1129,7 @@ struct CPUPPCState {
     uint64_t insns_flags2;
 #if defined(TARGET_PPC64)
     struct ppc_segment_page_sizes sps;
+    bool ci_large_pages;
 #endif
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
@@ -1182,7 +1239,7 @@ void ppc_store_msr (CPUPPCState *env, target_ulong value);
 
 void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 int ppc_get_compat_smt_threads(PowerPCCPU *cpu);
-int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version);
+void ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version, Error **errp);
 
 /* Time-base and decrementer management */
 #ifndef NO_CPU_IO_DEFS
@@ -1213,6 +1270,7 @@ void store_booke_tcr (CPUPPCState *env, target_ulong val);
 void store_booke_tsr (CPUPPCState *env, target_ulong val);
 void ppc_tlb_invalidate_all (CPUPPCState *env);
 void ppc_tlb_invalidate_one (CPUPPCState *env, target_ulong addr);
+void cpu_ppc_set_papr(PowerPCCPU *cpu);
 #endif
 #endif
 
@@ -1241,7 +1299,6 @@ int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val);
 #define cpu_init(cpu_model) CPU(cpu_ppc_init(cpu_model))
 
 #define cpu_exec cpu_ppc_exec
-#define cpu_gen_code cpu_ppc_gen_code
 #define cpu_signal_handler cpu_ppc_signal_handler
 #define cpu_list ppc_cpu_list
 
@@ -1250,7 +1307,7 @@ int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val);
 #define MMU_MODE1_SUFFIX _kernel
 #define MMU_MODE2_SUFFIX _hypv
 #define MMU_USER_IDX 0
-static inline int cpu_mmu_index (CPUPPCState *env)
+static inline int cpu_mmu_index (CPUPPCState *env, bool ifetch)
 {
     return env->mmu_idx;
 }
@@ -1300,11 +1357,14 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_SRR1              (0x01B)
 #define SPR_CFAR              (0x01C)
 #define SPR_AMR               (0x01D)
+#define SPR_ACOP              (0x01F)
 #define SPR_BOOKE_PID         (0x030)
+#define SPR_BOOKS_PID         (0x030)
 #define SPR_BOOKE_DECAR       (0x036)
 #define SPR_BOOKE_CSRR0       (0x03A)
 #define SPR_BOOKE_CSRR1       (0x03B)
 #define SPR_BOOKE_DEAR        (0x03D)
+#define SPR_IAMR              (0x03D)
 #define SPR_BOOKE_ESR         (0x03E)
 #define SPR_BOOKE_IVPR        (0x03F)
 #define SPR_MPC_EIE           (0x050)
@@ -1334,6 +1394,12 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_UAMOR             (0x09D)
 #define SPR_MPC_ICTRL         (0x09E)
 #define SPR_MPC_BAR           (0x09F)
+#define SPR_PSPB              (0x09F)
+#define SPR_DAWR              (0x0B4)
+#define SPR_RPR               (0x0BA)
+#define SPR_CIABR             (0x0BB)
+#define SPR_DAWRX             (0x0BC)
+#define SPR_HFSCR             (0x0BE)
 #define SPR_VRSAVE            (0x100)
 #define SPR_USPRG0            (0x100)
 #define SPR_USPRG1            (0x101)
@@ -1388,19 +1454,25 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_HSRR1             (0x13B)
 #define SPR_BOOKE_IAC4        (0x13B)
 #define SPR_BOOKE_DAC1        (0x13C)
-#define SPR_LPIDR             (0x13D)
+#define SPR_MMCRH             (0x13C)
 #define SPR_DABR2             (0x13D)
 #define SPR_BOOKE_DAC2        (0x13D)
+#define SPR_TFMR              (0x13D)
 #define SPR_BOOKE_DVC1        (0x13E)
 #define SPR_LPCR              (0x13E)
 #define SPR_BOOKE_DVC2        (0x13F)
+#define SPR_LPIDR             (0x13F)
 #define SPR_BOOKE_TSR         (0x150)
+#define SPR_HMER              (0x150)
+#define SPR_HMEER             (0x151)
 #define SPR_PCR               (0x152)
+#define SPR_BOOKE_LPIDR       (0x152)
 #define SPR_BOOKE_TCR         (0x154)
 #define SPR_BOOKE_TLB0PS      (0x158)
 #define SPR_BOOKE_TLB1PS      (0x159)
 #define SPR_BOOKE_TLB2PS      (0x15A)
 #define SPR_BOOKE_TLB3PS      (0x15B)
+#define SPR_AMOR              (0x15D)
 #define SPR_BOOKE_MAS7_MAS3   (0x174)
 #define SPR_BOOKE_IVOR0       (0x190)
 #define SPR_BOOKE_IVOR1       (0x191)
@@ -1517,6 +1589,7 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_PERF0             (0x300)
 #define SPR_RCPU_MI_RBA0      (0x300)
 #define SPR_MPC_MI_CTR        (0x300)
+#define SPR_POWER_USIER       (0x300)
 #define SPR_PERF1             (0x301)
 #define SPR_RCPU_MI_RBA1      (0x301)
 #define SPR_POWER_UMMCR2      (0x301)
@@ -1566,6 +1639,7 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_PERFF             (0x30F)
 #define SPR_MPC_MD_TW         (0x30F)
 #define SPR_UPERF0            (0x310)
+#define SPR_POWER_SIER        (0x310)
 #define SPR_UPERF1            (0x311)
 #define SPR_POWER_MMCR2       (0x311)
 #define SPR_UPERF2            (0x312)
@@ -1617,7 +1691,9 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_MPC_MD_DBRAM1     (0x32A)
 #define SPR_RCPU_L2U_RA3      (0x32B)
 #define SPR_TAR               (0x32F)
+#define SPR_IC                (0x350)
 #define SPR_VTB               (0x351)
+#define SPR_MMCRC             (0x353)
 #define SPR_440_INV0          (0x370)
 #define SPR_440_INV1          (0x371)
 #define SPR_440_INV2          (0x372)
@@ -1627,8 +1703,14 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_440_ITV2          (0x376)
 #define SPR_440_ITV3          (0x377)
 #define SPR_440_CCR1          (0x378)
+#define SPR_TACR              (0x378)
+#define SPR_TCSCR             (0x379)
+#define SPR_CSIGR             (0x37a)
 #define SPR_DCRIPR            (0x37B)
+#define SPR_POWER_SPMC1       (0x37C)
+#define SPR_POWER_SPMC2       (0x37D)
 #define SPR_POWER_MMCRS       (0x37E)
+#define SPR_WORT              (0x37F)
 #define SPR_PPR               (0x380)
 #define SPR_750_GQR0          (0x390)
 #define SPR_440_DNV0          (0x390)
@@ -1651,6 +1733,7 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define SPR_440_DVLIM         (0x398)
 #define SPR_750_WPAR          (0x399)
 #define SPR_440_IVLIM         (0x399)
+#define SPR_TSCR              (0x399)
 #define SPR_750_DMAU          (0x39A)
 #define SPR_750_DMAL          (0x39B)
 #define SPR_440_RSTCFG        (0x39B)
@@ -1825,9 +1908,10 @@ static inline int cpu_mmu_index (CPUPPCState *env)
 #define   L1CSR1_ICE		0x00000001	/* Instruction Cache Enable */
 
 /* HID0 bits */
-#define HID0_DEEPNAP        (1 << 24)
-#define HID0_DOZE           (1 << 23)
-#define HID0_NAP            (1 << 22)
+#define HID0_DEEPNAP        (1 << 24)           /* pre-2.06 */
+#define HID0_DOZE           (1 << 23)           /* pre-2.06 */
+#define HID0_NAP            (1 << 22)           /* pre-2.06 */
+#define HID0_HILE           (1ull << (63 - 19)) /* POWER8 */
 
 /*****************************************************************************/
 /* PowerPC Instructions types definitions                                    */
@@ -2176,6 +2260,33 @@ enum {
     PCR_TM_DIS          = 1ull << (63-2), /* Trans. memory disable (POWER8) */
 };
 
+/* HMER/HMEER */
+enum {
+    HMER_MALFUNCTION_ALERT      = 1ull << (63 - 0),
+    HMER_PROC_RECV_DONE         = 1ull << (63 - 2),
+    HMER_PROC_RECV_ERROR_MASKED = 1ull << (63 - 3),
+    HMER_TFAC_ERROR             = 1ull << (63 - 4),
+    HMER_TFMR_PARITY_ERROR      = 1ull << (63 - 5),
+    HMER_XSCOM_FAIL             = 1ull << (63 - 8),
+    HMER_XSCOM_DONE             = 1ull << (63 - 9),
+    HMER_PROC_RECV_AGAIN        = 1ull << (63 - 11),
+    HMER_WARN_RISE              = 1ull << (63 - 14),
+    HMER_WARN_FALL              = 1ull << (63 - 15),
+    HMER_SCOM_FIR_HMI           = 1ull << (63 - 16),
+    HMER_TRIG_FIR_HMI           = 1ull << (63 - 17),
+    HMER_HYP_RESOURCE_ERR       = 1ull << (63 - 20),
+    HMER_XSCOM_STATUS_MASK      = 7ull << (63 - 23),
+    HMER_XSCOM_STATUS_LSH       = (63 - 23),
+};
+
+/* Alternate Interrupt Location (AIL) */
+enum {
+    AIL_NONE                = 0,
+    AIL_RESERVED            = 1,
+    AIL_0001_8000           = 2,
+    AIL_C000_0000_0000_4000 = 3,
+};
+
 /*****************************************************************************/
 
 static inline target_ulong cpu_read_xer(CPUPPCState *env)
@@ -2304,6 +2415,16 @@ static inline bool msr_is_64bit(CPUPPCState *env, target_ulong msr)
     return msr & (1ULL << MSR_SF);
 }
 
+/**
+ * Check whether register rx is in the range between start and
+ * start + nregs (as needed by the LSWX and LSWI instructions)
+ */
+static inline bool lsw_reg_in_range(int start, int nregs, int rx)
+{
+    return (start + nregs <= 32 && rx >= start && rx < start + nregs) ||
+           (start + nregs > 32 && (rx >= start || rx < start + nregs - 32));
+}
+
 extern void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
 #include "exec/exec-all.h"
@@ -2328,4 +2449,5 @@ int ppc_get_vcpu_dt_id(PowerPCCPU *cpu);
  */
 PowerPCCPU *ppc_get_vcpu_by_dt_id(int cpu_dt_id);
 
+void ppc_maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len);
 #endif /* !defined (__CPU_PPC_H__) */
diff --git a/qemu/target-ppc/dfp_helper.c b/qemu/target-ppc/dfp_helper.c
index 49820bf21..db0ede698 100644
--- a/qemu/target-ppc/dfp_helper.c
+++ b/qemu/target-ppc/dfp_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -170,27 +171,6 @@ static void dfp_prepare_decimal128(struct PPC_DFP *dfp, uint64_t *a,
     }
 }
 
-#define FP_FX       (1ull << FPSCR_FX)
-#define FP_FEX      (1ull << FPSCR_FEX)
-#define FP_OX       (1ull << FPSCR_OX)
-#define FP_OE       (1ull << FPSCR_OE)
-#define FP_UX       (1ull << FPSCR_UX)
-#define FP_UE       (1ull << FPSCR_UE)
-#define FP_XX       (1ull << FPSCR_XX)
-#define FP_XE       (1ull << FPSCR_XE)
-#define FP_ZX       (1ull << FPSCR_ZX)
-#define FP_ZE       (1ull << FPSCR_ZE)
-#define FP_VX       (1ull << FPSCR_VX)
-#define FP_VXSNAN   (1ull << FPSCR_VXSNAN)
-#define FP_VXISI    (1ull << FPSCR_VXISI)
-#define FP_VXIMZ    (1ull << FPSCR_VXIMZ)
-#define FP_VXZDZ    (1ull << FPSCR_VXZDZ)
-#define FP_VXIDI    (1ull << FPSCR_VXIDI)
-#define FP_VXVC     (1ull << FPSCR_VXVC)
-#define FP_VXCVI    (1ull << FPSCR_VXCVI)
-#define FP_VE       (1ull << FPSCR_VE)
-#define FP_FI       (1ull << FPSCR_FI)
-
 static void dfp_set_FPSCR_flag(struct PPC_DFP *dfp, uint64_t flag,
                 uint64_t enabled)
 {
diff --git a/qemu/target-ppc/excp_helper.c b/qemu/target-ppc/excp_helper.c
index b80347506..ca4ffe8ad 100644
--- a/qemu/target-ppc/excp_helper.c
+++ b/qemu/target-ppc/excp_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
@@ -23,6 +24,7 @@
 #include "helper_regs.h"
 
 //#define DEBUG_OP
+//#define DEBUG_SOFTWARE_TLB
 //#define DEBUG_EXCEPTIONS
 
 #ifdef DEBUG_EXCEPTIONS
@@ -75,7 +77,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
     CPUPPCState *env = &cpu->env;
     target_ulong msr, new_msr, vector;
     int srr0, srr1, asrr0, asrr1;
-    int lpes0, lpes1, lev;
+    int lpes0, lpes1, lev, ail;
 
     if (0) {
         /* XXX: find a suitable condition to enable the hypervisor mode */
@@ -106,6 +108,25 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
     asrr0 = -1;
     asrr1 = -1;
 
+    /* Exception targetting modifiers
+     *
+     * AIL is initialized here but can be cleared by
+     * selected exceptions
+     */
+#if defined(TARGET_PPC64)
+    if (excp_model == POWERPC_EXCP_POWER7 ||
+        excp_model == POWERPC_EXCP_POWER8) {
+        if (excp_model == POWERPC_EXCP_POWER8) {
+            ail = (env->spr[SPR_LPCR] & LPCR_AIL) >> LPCR_AIL_SHIFT;
+        } else {
+            ail = 0;
+        }
+    } else
+#endif /* defined(TARGET_PPC64) */
+    {
+        ail = 0;
+    }
+
     switch (excp) {
     case POWERPC_EXCP_NONE:
         /* Should never happen */
@@ -131,12 +152,11 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
             /* Machine check exception is not enabled.
              * Enter checkstop state.
              */
-            if (qemu_log_enabled()) {
+            fprintf(stderr, "Machine check while not allowed. "
+                    "Entering checkstop state\n");
+            if (qemu_log_separate()) {
                 qemu_log("Machine check while not allowed. "
                         "Entering checkstop state\n");
-            } else {
-                fprintf(stderr, "Machine check while not allowed. "
-                        "Entering checkstop state\n");
             }
             cs->halted = 1;
             cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
@@ -145,6 +165,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
             /* XXX: find a suitable condition to enable the hypervisor mode */
             new_msr |= (target_ulong)MSR_HVB;
         }
+        ail = 0;
 
         /* machine check exceptions don't have ME set */
         new_msr &= ~((target_ulong)1 << MSR_ME);
@@ -200,7 +221,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
         /* Get rS/rD and rA from faulting opcode */
         env->spr[SPR_DSISR] |= (cpu_ldl_code(env, (env->nip - 4))
                                 & 0x03FF0000) >> 16;
-        goto store_current;
+        goto store_next;
     case POWERPC_EXCP_PROGRAM:   /* Program exception                        */
         switch (env->error_code & ~0xF) {
         case POWERPC_EXCP_FP:
@@ -343,6 +364,7 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
             /* XXX: find a suitable condition to enable the hypervisor mode */
             new_msr |= (target_ulong)MSR_HVB;
         }
+        ail = 0;
         goto store_next;
     case POWERPC_EXCP_DSEG:      /* Data segment exception                   */
         if (lpes1 == 0) {
@@ -629,7 +651,8 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
     }
 
 #ifdef TARGET_PPC64
-    if (excp_model == POWERPC_EXCP_POWER7) {
+    if (excp_model == POWERPC_EXCP_POWER7 ||
+        excp_model == POWERPC_EXCP_POWER8) {
         if (env->spr[SPR_LPCR] & LPCR_ILE) {
             new_msr |= (target_ulong)1 << MSR_LE;
         }
@@ -649,6 +672,29 @@ static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
                   excp);
     }
     vector |= env->excp_prefix;
+
+    /* AIL only works if there is no HV transition and we are running with
+     * translations enabled
+     */
+    if (!((msr >> MSR_IR) & 1) || !((msr >> MSR_DR) & 1)) {
+        ail = 0;
+    }
+    /* Handle AIL */
+    if (ail) {
+        new_msr |= (1 << MSR_IR) | (1 << MSR_DR);
+        switch(ail) {
+        case AIL_0001_8000:
+            vector |= 0x18000;
+            break;
+        case AIL_C000_0000_0000_4000:
+            vector |= 0xc000000000004000ull;
+            break;
+        default:
+            cpu_abort(cs, "Invalid AIL combination %d\n", ail);
+            break;
+        }
+    }
+
 #if defined(TARGET_PPC64)
     if (excp_model == POWERPC_EXCP_BOOKE) {
         if (env->spr[SPR_BOOKE_EPCR] & EPCR_ICM) {
diff --git a/qemu/target-ppc/fpu_helper.c b/qemu/target-ppc/fpu_helper.c
index 6cceffc55..b67ebca12 100644
--- a/qemu/target-ppc/fpu_helper.c
+++ b/qemu/target-ppc/fpu_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -194,7 +195,7 @@ static inline uint64_t fload_invalid_op_excp(CPUPPCState *env, int op,
     /* Update the floating-point invalid operation summary */
     env->fpscr |= 1 << FPSCR_VX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (ve != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -211,7 +212,7 @@ static inline void float_zero_divide_excp(CPUPPCState *env)
     env->fpscr |= 1 << FPSCR_ZX;
     env->fpscr &= ~((1 << FPSCR_FR) | (1 << FPSCR_FI));
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ze != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -228,7 +229,7 @@ static inline void float_overflow_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_OX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_oe != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -248,7 +249,7 @@ static inline void float_underflow_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_UX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ue != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -265,7 +266,7 @@ static inline void float_inexact_excp(CPUPPCState *env)
 
     env->fpscr |= 1 << FPSCR_XX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_xe != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -330,31 +331,31 @@ void helper_fpscr_setbit(CPUPPCState *env, uint32_t bit)
     if (prev == 0) {
         switch (bit) {
         case FPSCR_VX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve) {
                 goto raise_ve;
             }
             break;
         case FPSCR_OX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_oe) {
                 goto raise_oe;
             }
             break;
         case FPSCR_UX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ue) {
                 goto raise_ue;
             }
             break;
         case FPSCR_ZX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ze) {
                 goto raise_ze;
             }
             break;
         case FPSCR_XX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_xe) {
                 goto raise_xe;
             }
@@ -369,7 +370,7 @@ void helper_fpscr_setbit(CPUPPCState *env, uint32_t bit)
         case FPSCR_VXSQRT:
         case FPSCR_VXCVI:
             env->fpscr |= 1 << FPSCR_VX;
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve != 0) {
                 goto raise_ve;
             }
diff --git a/qemu/target-ppc/gdbstub.c b/qemu/target-ppc/gdbstub.c
index 14675f456..569c380cf 100644
--- a/qemu/target-ppc/gdbstub.c
+++ b/qemu/target-ppc/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
@@ -88,7 +88,7 @@ static int ppc_gdb_register_len(int n)
    the proper ordering for the binary, and cannot be changed.
    For system mode, TARGET_WORDS_BIGENDIAN is always set, and we must check
    the current mode of the chip to see if we're running in little-endian.  */
-static void maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len)
+void ppc_maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len)
 {
 #ifndef CONFIG_USER_ONLY
     if (!msr_le) {
@@ -158,7 +158,7 @@ int ppc_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
             break;
         }
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     return r;
 }
 
@@ -214,7 +214,7 @@ int ppc_cpu_gdb_read_register_apple(CPUState *cs, uint8_t *mem_buf, int n)
             break;
         }
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     return r;
 }
 
@@ -227,7 +227,7 @@ int ppc_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
     if (!r) {
         return r;
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     if (n < 32) {
         /* gprs */
         env->gpr[n] = ldtul_p(mem_buf);
@@ -277,7 +277,7 @@ int ppc_cpu_gdb_write_register_apple(CPUState *cs, uint8_t *mem_buf, int n)
     if (!r) {
         return r;
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     if (n < 32) {
         /* gprs */
         env->gpr[n] = ldq_p(mem_buf);
diff --git a/qemu/target-ppc/helper.h b/qemu/target-ppc/helper.h
index 869be1509..e5a8f7b9b 100644
--- a/qemu/target-ppc/helper.h
+++ b/qemu/target-ppc/helper.h
@@ -544,6 +544,7 @@ DEF_HELPER_2(74xx_tlbd, void, env, tl)
 DEF_HELPER_2(74xx_tlbi, void, env, tl)
 DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_NO_RWG, void, env, tl)
+DEF_HELPER_FLAGS_2(tlbiva, TCG_CALL_NO_RWG, void, env, tl)
 #if defined(TARGET_PPC64)
 DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_NO_RWG, void, env, tl, tl)
 DEF_HELPER_2(load_slb_esid, tl, env, tl)
diff --git a/qemu/target-ppc/int_helper.c b/qemu/target-ppc/int_helper.c
index 0a55d5e54..27b0258d3 100644
--- a/qemu/target-ppc/int_helper.c
+++ b/qemu/target-ppc/int_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
@@ -2327,24 +2328,28 @@ void helper_vsbox(ppc_avr_t *r, ppc_avr_t *a)
 
 void helper_vcipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u32) {
-        r->AVRW(i) = b->AVRW(i) ^
+        result.AVRW(i) = b->AVRW(i) ^
             (AES_Te0[a->AVRB(AES_shifts[4*i + 0])] ^
              AES_Te1[a->AVRB(AES_shifts[4*i + 1])] ^
              AES_Te2[a->AVRB(AES_shifts[4*i + 2])] ^
              AES_Te3[a->AVRB(AES_shifts[4*i + 3])]);
     }
+    *r = result;
 }
 
 void helper_vcipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_sbox[a->AVRB(AES_shifts[i])]);
+        result.AVRB(i) = b->AVRB(i) ^ (AES_sbox[a->AVRB(AES_shifts[i])]);
     }
+    *r = result;
 }
 
 void helper_vncipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
@@ -2369,11 +2374,13 @@ void helper_vncipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 
 void helper_vncipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_isbox[a->AVRB(AES_ishifts[i])]);
+        result.AVRB(i) = b->AVRB(i) ^ (AES_isbox[a->AVRB(AES_ishifts[i])]);
     }
+    *r = result;
 }
 
 #define ROTRu32(v, n) (((v) >> (n)) | ((v) << (32-n)))
@@ -2460,16 +2467,19 @@ void helper_vshasigmad(ppc_avr_t *r,  ppc_avr_t *a, uint32_t st_six)
 
 void helper_vpermxor(ppc_avr_t *r,  ppc_avr_t *a, ppc_avr_t *b, ppc_avr_t *c)
 {
+    ppc_avr_t result;
     int i;
+
     VECTOR_FOR_INORDER_I(i, u8) {
         int indexA = c->u8[i] >> 4;
         int indexB = c->u8[i] & 0xF;
 #if defined(HOST_WORDS_BIGENDIAN)
-        r->u8[i] = a->u8[indexA] ^ b->u8[indexB];
+        result.u8[i] = a->u8[indexA] ^ b->u8[indexB];
 #else
-        r->u8[i] = a->u8[15-indexA] ^ b->u8[15-indexB];
+        result.u8[i] = a->u8[15-indexA] ^ b->u8[15-indexB];
 #endif
     }
+    *r = result;
 }
 
 #undef VECTOR_FOR_INORDER_I
diff --git a/qemu/target-ppc/kvm-stub.c b/qemu/target-ppc/kvm-stub.c
index ee3f5d2f7..627bcb432 100644
--- a/qemu/target-ppc/kvm-stub.c
+++ b/qemu/target-ppc/kvm-stub.c
@@ -9,6 +9,7 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/ppc/openpic.h"
 
diff --git a/qemu/target-ppc/kvm.c b/qemu/target-ppc/kvm.c
index 110436d08..c4c81467e 100644
--- a/qemu/target-ppc/kvm.c
+++ b/qemu/target-ppc/kvm.c
@@ -14,8 +14,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <dirent.h>
-#include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/vfs.h>
@@ -23,6 +23,7 @@
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
@@ -41,6 +42,7 @@
 #include "exec/gdbstub.h"
 #include "exec/memattrs.h"
 #include "sysemu/hostmem.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_KVM
 
@@ -259,7 +261,8 @@ static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
             info->flags |= KVM_PPC_1T_SEGMENTS;
         }
 
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->slb_size = 32;
         } else {
             info->slb_size = 64;
@@ -272,8 +275,9 @@ static void kvm_get_fallback_smmu_info(PowerPCCPU *cpu,
         info->sps[i].enc[0].pte_enc = 0;
         i++;
 
-        /* 64K on MMU 2.06 */
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        /* 64K on MMU 2.06 and later */
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->sps[i].page_shift = 16;
             info->sps[i].slb_enc = 0x110;
             info->sps[i].enc[0].page_shift = 16;
@@ -330,6 +334,12 @@ static long gethugepagesize(const char *mem_path)
     return fs.f_bsize;
 }
 
+/*
+ * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
+ * may or may not name the same files / on the same filesystem now as
+ * when we actually open and map them.  Iterate over the file
+ * descriptors instead, and use qemu_fd_getpagesize().
+ */
 static int find_max_supported_pagesize(Object *obj, void *opaque)
 {
     char *mem_path;
@@ -412,6 +422,13 @@ static void kvm_fixup_page_sizes(PowerPCCPU *cpu)
     /* Convert to QEMU form */
     memset(&env->sps, 0, sizeof(env->sps));
 
+    /* If we have HV KVM, we need to forbid CI large pages if our
+     * host page size is smaller than 64K.
+     */
+    if (smmu_info.flags & KVM_PPC_PAGE_SIZES_REAL) {
+        env->ci_large_pages = getpagesize() >= 0x10000;
+    }
+
     /*
      * XXX This loop should be an entry wide AND of the capabilities that
      *     the selected CPU has with the capabilities that KVM supports.
@@ -503,6 +520,10 @@ int kvm_arch_init_vcpu(CPUState *cs)
     /* Synchronize sregs with kvm */
     ret = kvm_arch_sync_sregs(cpu);
     if (ret) {
+        if (ret == -EINVAL) {
+            error_report("Register sync failed... If you're using kvm-hv.ko,"
+                         " only \"-cpu host\" is possible");
+        }
         return ret;
     }
 
@@ -641,8 +662,13 @@ static int kvm_put_fp(CPUState *cs)
         for (i = 0; i < 32; i++) {
             uint64_t vsr[2];
 
+#ifdef HOST_WORDS_BIGENDIAN
             vsr[0] = float64_val(env->fpr[i]);
             vsr[1] = env->vsr[i];
+#else
+            vsr[0] = env->vsr[i];
+            vsr[1] = float64_val(env->fpr[i]);
+#endif
             reg.addr = (uintptr_t) &vsr;
             reg.id = vsx ? KVM_REG_PPC_VSR(i) : KVM_REG_PPC_FPR(i);
 
@@ -712,10 +738,17 @@ static int kvm_get_fp(CPUState *cs)
                         vsx ? "VSR" : "FPR", i, strerror(errno));
                 return ret;
             } else {
+#ifdef HOST_WORDS_BIGENDIAN
                 env->fpr[i] = vsr[0];
                 if (vsx) {
                     env->vsr[i] = vsr[1];
                 }
+#else
+                env->fpr[i] = vsr[1];
+                if (vsx) {
+                    env->vsr[i] = vsr[0];
+                }
+#endif
             }
         }
     }
@@ -841,6 +874,44 @@ static int kvm_put_vpa(CPUState *cs)
 }
 #endif /* TARGET_PPC64 */
 
+int kvmppc_put_books_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int i;
+
+    sregs.pvr = env->spr[SPR_PVR];
+
+    sregs.u.s.sdr1 = env->spr[SPR_SDR1];
+
+    /* Sync SLB */
+#ifdef TARGET_PPC64
+    for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
+        sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
+        if (env->slb[i].esid & SLB_ESID_V) {
+            sregs.u.s.ppc64.slb[i].slbe |= i;
+        }
+        sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
+    }
+#endif
+
+    /* Sync SRs */
+    for (i = 0; i < 16; i++) {
+        sregs.u.s.ppc32.sr[i] = env->sr[i];
+    }
+
+    /* Sync BATs */
+    for (i = 0; i < 8; i++) {
+        /* Beware. We have to swap upper and lower bits here */
+        sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
+            | env->DBAT[1][i];
+        sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
+            | env->IBAT[1][i];
+    }
+
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
+}
+
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
@@ -894,39 +965,8 @@ int kvm_arch_put_registers(CPUState *cs, int level)
     }
 
     if (cap_segstate && (level >= KVM_PUT_RESET_STATE)) {
-        struct kvm_sregs sregs;
-
-        sregs.pvr = env->spr[SPR_PVR];
-
-        sregs.u.s.sdr1 = env->spr[SPR_SDR1];
-
-        /* Sync SLB */
-#ifdef TARGET_PPC64
-        for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
-            sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
-            if (env->slb[i].esid & SLB_ESID_V) {
-                sregs.u.s.ppc64.slb[i].slbe |= i;
-            }
-            sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
-        }
-#endif
-
-        /* Sync SRs */
-        for (i = 0; i < 16; i++) {
-            sregs.u.s.ppc32.sr[i] = env->sr[i];
-        }
-
-        /* Sync BATs */
-        for (i = 0; i < 8; i++) {
-            /* Beware. We have to swap upper and lower bits here */
-            sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
-                | env->DBAT[1][i];
-            sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
-                | env->IBAT[1][i];
-        }
-
-        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
-        if (ret) {
+        ret = kvmppc_put_books_sregs(cpu);
+        if (ret < 0) {
             return ret;
         }
     }
@@ -988,12 +1028,197 @@ static void kvm_sync_excp(CPUPPCState *env, int vector, int ivor)
      env->excp_vectors[vector] = env->spr[ivor] + env->spr[SPR_BOOKE_IVPR];
 }
 
+static int kvmppc_get_booke_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int ret;
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_BASE) {
+        env->spr[SPR_BOOKE_CSRR0] = sregs.u.e.csrr0;
+        env->spr[SPR_BOOKE_CSRR1] = sregs.u.e.csrr1;
+        env->spr[SPR_BOOKE_ESR] = sregs.u.e.esr;
+        env->spr[SPR_BOOKE_DEAR] = sregs.u.e.dear;
+        env->spr[SPR_BOOKE_MCSR] = sregs.u.e.mcsr;
+        env->spr[SPR_BOOKE_TSR] = sregs.u.e.tsr;
+        env->spr[SPR_BOOKE_TCR] = sregs.u.e.tcr;
+        env->spr[SPR_DECR] = sregs.u.e.dec;
+        env->spr[SPR_TBL] = sregs.u.e.tb & 0xffffffff;
+        env->spr[SPR_TBU] = sregs.u.e.tb >> 32;
+        env->spr[SPR_VRSAVE] = sregs.u.e.vrsave;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_ARCH206) {
+        env->spr[SPR_BOOKE_PIR] = sregs.u.e.pir;
+        env->spr[SPR_BOOKE_MCSRR0] = sregs.u.e.mcsrr0;
+        env->spr[SPR_BOOKE_MCSRR1] = sregs.u.e.mcsrr1;
+        env->spr[SPR_BOOKE_DECAR] = sregs.u.e.decar;
+        env->spr[SPR_BOOKE_IVPR] = sregs.u.e.ivpr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_64) {
+        env->spr[SPR_BOOKE_EPCR] = sregs.u.e.epcr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_SPRG8) {
+        env->spr[SPR_BOOKE_SPRG8] = sregs.u.e.sprg8;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_IVOR) {
+        env->spr[SPR_BOOKE_IVOR0] = sregs.u.e.ivor_low[0];
+        kvm_sync_excp(env, POWERPC_EXCP_CRITICAL,  SPR_BOOKE_IVOR0);
+        env->spr[SPR_BOOKE_IVOR1] = sregs.u.e.ivor_low[1];
+        kvm_sync_excp(env, POWERPC_EXCP_MCHECK,  SPR_BOOKE_IVOR1);
+        env->spr[SPR_BOOKE_IVOR2] = sregs.u.e.ivor_low[2];
+        kvm_sync_excp(env, POWERPC_EXCP_DSI,  SPR_BOOKE_IVOR2);
+        env->spr[SPR_BOOKE_IVOR3] = sregs.u.e.ivor_low[3];
+        kvm_sync_excp(env, POWERPC_EXCP_ISI,  SPR_BOOKE_IVOR3);
+        env->spr[SPR_BOOKE_IVOR4] = sregs.u.e.ivor_low[4];
+        kvm_sync_excp(env, POWERPC_EXCP_EXTERNAL,  SPR_BOOKE_IVOR4);
+        env->spr[SPR_BOOKE_IVOR5] = sregs.u.e.ivor_low[5];
+        kvm_sync_excp(env, POWERPC_EXCP_ALIGN,  SPR_BOOKE_IVOR5);
+        env->spr[SPR_BOOKE_IVOR6] = sregs.u.e.ivor_low[6];
+        kvm_sync_excp(env, POWERPC_EXCP_PROGRAM,  SPR_BOOKE_IVOR6);
+        env->spr[SPR_BOOKE_IVOR7] = sregs.u.e.ivor_low[7];
+        kvm_sync_excp(env, POWERPC_EXCP_FPU,  SPR_BOOKE_IVOR7);
+        env->spr[SPR_BOOKE_IVOR8] = sregs.u.e.ivor_low[8];
+        kvm_sync_excp(env, POWERPC_EXCP_SYSCALL,  SPR_BOOKE_IVOR8);
+        env->spr[SPR_BOOKE_IVOR9] = sregs.u.e.ivor_low[9];
+        kvm_sync_excp(env, POWERPC_EXCP_APU,  SPR_BOOKE_IVOR9);
+        env->spr[SPR_BOOKE_IVOR10] = sregs.u.e.ivor_low[10];
+        kvm_sync_excp(env, POWERPC_EXCP_DECR,  SPR_BOOKE_IVOR10);
+        env->spr[SPR_BOOKE_IVOR11] = sregs.u.e.ivor_low[11];
+        kvm_sync_excp(env, POWERPC_EXCP_FIT,  SPR_BOOKE_IVOR11);
+        env->spr[SPR_BOOKE_IVOR12] = sregs.u.e.ivor_low[12];
+        kvm_sync_excp(env, POWERPC_EXCP_WDT,  SPR_BOOKE_IVOR12);
+        env->spr[SPR_BOOKE_IVOR13] = sregs.u.e.ivor_low[13];
+        kvm_sync_excp(env, POWERPC_EXCP_DTLB,  SPR_BOOKE_IVOR13);
+        env->spr[SPR_BOOKE_IVOR14] = sregs.u.e.ivor_low[14];
+        kvm_sync_excp(env, POWERPC_EXCP_ITLB,  SPR_BOOKE_IVOR14);
+        env->spr[SPR_BOOKE_IVOR15] = sregs.u.e.ivor_low[15];
+        kvm_sync_excp(env, POWERPC_EXCP_DEBUG,  SPR_BOOKE_IVOR15);
+
+        if (sregs.u.e.features & KVM_SREGS_E_SPE) {
+            env->spr[SPR_BOOKE_IVOR32] = sregs.u.e.ivor_high[0];
+            kvm_sync_excp(env, POWERPC_EXCP_SPEU,  SPR_BOOKE_IVOR32);
+            env->spr[SPR_BOOKE_IVOR33] = sregs.u.e.ivor_high[1];
+            kvm_sync_excp(env, POWERPC_EXCP_EFPDI,  SPR_BOOKE_IVOR33);
+            env->spr[SPR_BOOKE_IVOR34] = sregs.u.e.ivor_high[2];
+            kvm_sync_excp(env, POWERPC_EXCP_EFPRI,  SPR_BOOKE_IVOR34);
+        }
+
+        if (sregs.u.e.features & KVM_SREGS_E_PM) {
+            env->spr[SPR_BOOKE_IVOR35] = sregs.u.e.ivor_high[3];
+            kvm_sync_excp(env, POWERPC_EXCP_EPERFM,  SPR_BOOKE_IVOR35);
+        }
+
+        if (sregs.u.e.features & KVM_SREGS_E_PC) {
+            env->spr[SPR_BOOKE_IVOR36] = sregs.u.e.ivor_high[4];
+            kvm_sync_excp(env, POWERPC_EXCP_DOORI,  SPR_BOOKE_IVOR36);
+            env->spr[SPR_BOOKE_IVOR37] = sregs.u.e.ivor_high[5];
+            kvm_sync_excp(env, POWERPC_EXCP_DOORCI, SPR_BOOKE_IVOR37);
+        }
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_ARCH206_MMU) {
+        env->spr[SPR_BOOKE_MAS0] = sregs.u.e.mas0;
+        env->spr[SPR_BOOKE_MAS1] = sregs.u.e.mas1;
+        env->spr[SPR_BOOKE_MAS2] = sregs.u.e.mas2;
+        env->spr[SPR_BOOKE_MAS3] = sregs.u.e.mas7_3 & 0xffffffff;
+        env->spr[SPR_BOOKE_MAS4] = sregs.u.e.mas4;
+        env->spr[SPR_BOOKE_MAS6] = sregs.u.e.mas6;
+        env->spr[SPR_BOOKE_MAS7] = sregs.u.e.mas7_3 >> 32;
+        env->spr[SPR_MMUCFG] = sregs.u.e.mmucfg;
+        env->spr[SPR_BOOKE_TLB0CFG] = sregs.u.e.tlbcfg[0];
+        env->spr[SPR_BOOKE_TLB1CFG] = sregs.u.e.tlbcfg[1];
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_EXP) {
+        env->spr[SPR_BOOKE_EPR] = sregs.u.e.epr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_PD) {
+        env->spr[SPR_BOOKE_EPLC] = sregs.u.e.eplc;
+        env->spr[SPR_BOOKE_EPSC] = sregs.u.e.epsc;
+    }
+
+    if (sregs.u.e.impl_id == KVM_SREGS_E_IMPL_FSL) {
+        env->spr[SPR_E500_SVR] = sregs.u.e.impl.fsl.svr;
+        env->spr[SPR_Exxx_MCAR] = sregs.u.e.impl.fsl.mcar;
+        env->spr[SPR_HID0] = sregs.u.e.impl.fsl.hid0;
+
+        if (sregs.u.e.impl.fsl.features & KVM_SREGS_E_FSL_PIDn) {
+            env->spr[SPR_BOOKE_PID1] = sregs.u.e.impl.fsl.pid1;
+            env->spr[SPR_BOOKE_PID2] = sregs.u.e.impl.fsl.pid2;
+        }
+    }
+
+    return 0;
+}
+
+static int kvmppc_get_books_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int ret;
+    int i;
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (!env->external_htab) {
+        ppc_store_sdr1(env, sregs.u.s.sdr1);
+    }
+
+    /* Sync SLB */
+#ifdef TARGET_PPC64
+    /*
+     * The packed SLB array we get from KVM_GET_SREGS only contains
+     * information about valid entries. So we flush our internal copy
+     * to get rid of stale ones, then put all valid SLB entries back
+     * in.
+     */
+    memset(env->slb, 0, sizeof(env->slb));
+    for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
+        target_ulong rb = sregs.u.s.ppc64.slb[i].slbe;
+        target_ulong rs = sregs.u.s.ppc64.slb[i].slbv;
+        /*
+         * Only restore valid entries
+         */
+        if (rb & SLB_ESID_V) {
+            ppc_store_slb(cpu, rb & 0xfff, rb & ~0xfffULL, rs);
+        }
+    }
+#endif
+
+    /* Sync SRs */
+    for (i = 0; i < 16; i++) {
+        env->sr[i] = sregs.u.s.ppc32.sr[i];
+    }
+
+    /* Sync BATs */
+    for (i = 0; i < 8; i++) {
+        env->DBAT[0][i] = sregs.u.s.ppc32.dbat[i] & 0xffffffff;
+        env->DBAT[1][i] = sregs.u.s.ppc32.dbat[i] >> 32;
+        env->IBAT[0][i] = sregs.u.s.ppc32.ibat[i] & 0xffffffff;
+        env->IBAT[1][i] = sregs.u.s.ppc32.ibat[i] >> 32;
+    }
+
+    return 0;
+}
+
 int kvm_arch_get_registers(CPUState *cs)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
     CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
-    struct kvm_sregs sregs;
     uint32_t cr;
     int i, ret;
 
@@ -1033,174 +1258,17 @@ int kvm_arch_get_registers(CPUState *cs)
     kvm_get_fp(cs);
 
     if (cap_booke_sregs) {
-        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
+        ret = kvmppc_get_booke_sregs(cpu);
         if (ret < 0) {
             return ret;
         }
-
-        if (sregs.u.e.features & KVM_SREGS_E_BASE) {
-            env->spr[SPR_BOOKE_CSRR0] = sregs.u.e.csrr0;
-            env->spr[SPR_BOOKE_CSRR1] = sregs.u.e.csrr1;
-            env->spr[SPR_BOOKE_ESR] = sregs.u.e.esr;
-            env->spr[SPR_BOOKE_DEAR] = sregs.u.e.dear;
-            env->spr[SPR_BOOKE_MCSR] = sregs.u.e.mcsr;
-            env->spr[SPR_BOOKE_TSR] = sregs.u.e.tsr;
-            env->spr[SPR_BOOKE_TCR] = sregs.u.e.tcr;
-            env->spr[SPR_DECR] = sregs.u.e.dec;
-            env->spr[SPR_TBL] = sregs.u.e.tb & 0xffffffff;
-            env->spr[SPR_TBU] = sregs.u.e.tb >> 32;
-            env->spr[SPR_VRSAVE] = sregs.u.e.vrsave;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_ARCH206) {
-            env->spr[SPR_BOOKE_PIR] = sregs.u.e.pir;
-            env->spr[SPR_BOOKE_MCSRR0] = sregs.u.e.mcsrr0;
-            env->spr[SPR_BOOKE_MCSRR1] = sregs.u.e.mcsrr1;
-            env->spr[SPR_BOOKE_DECAR] = sregs.u.e.decar;
-            env->spr[SPR_BOOKE_IVPR] = sregs.u.e.ivpr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_64) {
-            env->spr[SPR_BOOKE_EPCR] = sregs.u.e.epcr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_SPRG8) {
-            env->spr[SPR_BOOKE_SPRG8] = sregs.u.e.sprg8;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_IVOR) {
-            env->spr[SPR_BOOKE_IVOR0] = sregs.u.e.ivor_low[0];
-            kvm_sync_excp(env, POWERPC_EXCP_CRITICAL,  SPR_BOOKE_IVOR0);
-            env->spr[SPR_BOOKE_IVOR1] = sregs.u.e.ivor_low[1];
-            kvm_sync_excp(env, POWERPC_EXCP_MCHECK,  SPR_BOOKE_IVOR1);
-            env->spr[SPR_BOOKE_IVOR2] = sregs.u.e.ivor_low[2];
-            kvm_sync_excp(env, POWERPC_EXCP_DSI,  SPR_BOOKE_IVOR2);
-            env->spr[SPR_BOOKE_IVOR3] = sregs.u.e.ivor_low[3];
-            kvm_sync_excp(env, POWERPC_EXCP_ISI,  SPR_BOOKE_IVOR3);
-            env->spr[SPR_BOOKE_IVOR4] = sregs.u.e.ivor_low[4];
-            kvm_sync_excp(env, POWERPC_EXCP_EXTERNAL,  SPR_BOOKE_IVOR4);
-            env->spr[SPR_BOOKE_IVOR5] = sregs.u.e.ivor_low[5];
-            kvm_sync_excp(env, POWERPC_EXCP_ALIGN,  SPR_BOOKE_IVOR5);
-            env->spr[SPR_BOOKE_IVOR6] = sregs.u.e.ivor_low[6];
-            kvm_sync_excp(env, POWERPC_EXCP_PROGRAM,  SPR_BOOKE_IVOR6);
-            env->spr[SPR_BOOKE_IVOR7] = sregs.u.e.ivor_low[7];
-            kvm_sync_excp(env, POWERPC_EXCP_FPU,  SPR_BOOKE_IVOR7);
-            env->spr[SPR_BOOKE_IVOR8] = sregs.u.e.ivor_low[8];
-            kvm_sync_excp(env, POWERPC_EXCP_SYSCALL,  SPR_BOOKE_IVOR8);
-            env->spr[SPR_BOOKE_IVOR9] = sregs.u.e.ivor_low[9];
-            kvm_sync_excp(env, POWERPC_EXCP_APU,  SPR_BOOKE_IVOR9);
-            env->spr[SPR_BOOKE_IVOR10] = sregs.u.e.ivor_low[10];
-            kvm_sync_excp(env, POWERPC_EXCP_DECR,  SPR_BOOKE_IVOR10);
-            env->spr[SPR_BOOKE_IVOR11] = sregs.u.e.ivor_low[11];
-            kvm_sync_excp(env, POWERPC_EXCP_FIT,  SPR_BOOKE_IVOR11);
-            env->spr[SPR_BOOKE_IVOR12] = sregs.u.e.ivor_low[12];
-            kvm_sync_excp(env, POWERPC_EXCP_WDT,  SPR_BOOKE_IVOR12);
-            env->spr[SPR_BOOKE_IVOR13] = sregs.u.e.ivor_low[13];
-            kvm_sync_excp(env, POWERPC_EXCP_DTLB,  SPR_BOOKE_IVOR13);
-            env->spr[SPR_BOOKE_IVOR14] = sregs.u.e.ivor_low[14];
-            kvm_sync_excp(env, POWERPC_EXCP_ITLB,  SPR_BOOKE_IVOR14);
-            env->spr[SPR_BOOKE_IVOR15] = sregs.u.e.ivor_low[15];
-            kvm_sync_excp(env, POWERPC_EXCP_DEBUG,  SPR_BOOKE_IVOR15);
-
-            if (sregs.u.e.features & KVM_SREGS_E_SPE) {
-                env->spr[SPR_BOOKE_IVOR32] = sregs.u.e.ivor_high[0];
-                kvm_sync_excp(env, POWERPC_EXCP_SPEU,  SPR_BOOKE_IVOR32);
-                env->spr[SPR_BOOKE_IVOR33] = sregs.u.e.ivor_high[1];
-                kvm_sync_excp(env, POWERPC_EXCP_EFPDI,  SPR_BOOKE_IVOR33);
-                env->spr[SPR_BOOKE_IVOR34] = sregs.u.e.ivor_high[2];
-                kvm_sync_excp(env, POWERPC_EXCP_EFPRI,  SPR_BOOKE_IVOR34);
-            }
-
-            if (sregs.u.e.features & KVM_SREGS_E_PM) {
-                env->spr[SPR_BOOKE_IVOR35] = sregs.u.e.ivor_high[3];
-                kvm_sync_excp(env, POWERPC_EXCP_EPERFM,  SPR_BOOKE_IVOR35);
-            }
-
-            if (sregs.u.e.features & KVM_SREGS_E_PC) {
-                env->spr[SPR_BOOKE_IVOR36] = sregs.u.e.ivor_high[4];
-                kvm_sync_excp(env, POWERPC_EXCP_DOORI,  SPR_BOOKE_IVOR36);
-                env->spr[SPR_BOOKE_IVOR37] = sregs.u.e.ivor_high[5];
-                kvm_sync_excp(env, POWERPC_EXCP_DOORCI, SPR_BOOKE_IVOR37);
-            }
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_ARCH206_MMU) {
-            env->spr[SPR_BOOKE_MAS0] = sregs.u.e.mas0;
-            env->spr[SPR_BOOKE_MAS1] = sregs.u.e.mas1;
-            env->spr[SPR_BOOKE_MAS2] = sregs.u.e.mas2;
-            env->spr[SPR_BOOKE_MAS3] = sregs.u.e.mas7_3 & 0xffffffff;
-            env->spr[SPR_BOOKE_MAS4] = sregs.u.e.mas4;
-            env->spr[SPR_BOOKE_MAS6] = sregs.u.e.mas6;
-            env->spr[SPR_BOOKE_MAS7] = sregs.u.e.mas7_3 >> 32;
-            env->spr[SPR_MMUCFG] = sregs.u.e.mmucfg;
-            env->spr[SPR_BOOKE_TLB0CFG] = sregs.u.e.tlbcfg[0];
-            env->spr[SPR_BOOKE_TLB1CFG] = sregs.u.e.tlbcfg[1];
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_EXP) {
-            env->spr[SPR_BOOKE_EPR] = sregs.u.e.epr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_PD) {
-            env->spr[SPR_BOOKE_EPLC] = sregs.u.e.eplc;
-            env->spr[SPR_BOOKE_EPSC] = sregs.u.e.epsc;
-        }
-
-        if (sregs.u.e.impl_id == KVM_SREGS_E_IMPL_FSL) {
-            env->spr[SPR_E500_SVR] = sregs.u.e.impl.fsl.svr;
-            env->spr[SPR_Exxx_MCAR] = sregs.u.e.impl.fsl.mcar;
-            env->spr[SPR_HID0] = sregs.u.e.impl.fsl.hid0;
-
-            if (sregs.u.e.impl.fsl.features & KVM_SREGS_E_FSL_PIDn) {
-                env->spr[SPR_BOOKE_PID1] = sregs.u.e.impl.fsl.pid1;
-                env->spr[SPR_BOOKE_PID2] = sregs.u.e.impl.fsl.pid2;
-            }
-        }
     }
 
     if (cap_segstate) {
-        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
+        ret = kvmppc_get_books_sregs(cpu);
         if (ret < 0) {
             return ret;
         }
-
-        if (!env->external_htab) {
-            ppc_store_sdr1(env, sregs.u.s.sdr1);
-        }
-
-        /* Sync SLB */
-#ifdef TARGET_PPC64
-        /*
-         * The packed SLB array we get from KVM_GET_SREGS only contains
-         * information about valid entries. So we flush our internal
-         * copy to get rid of stale ones, then put all valid SLB entries
-         * back in.
-         */
-        memset(env->slb, 0, sizeof(env->slb));
-        for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
-            target_ulong rb = sregs.u.s.ppc64.slb[i].slbe;
-            target_ulong rs = sregs.u.s.ppc64.slb[i].slbv;
-            /*
-             * Only restore valid entries
-             */
-            if (rb & SLB_ESID_V) {
-                ppc_store_slb(env, rb, rs);
-            }
-        }
-#endif
-
-        /* Sync SRs */
-        for (i = 0; i < 16; i++) {
-            env->sr[i] = sregs.u.s.ppc32.sr[i];
-        }
-
-        /* Sync BATs */
-        for (i = 0; i < 8; i++) {
-            env->DBAT[0][i] = sregs.u.s.ppc32.dbat[i] & 0xffffffff;
-            env->DBAT[1][i] = sregs.u.s.ppc32.dbat[i] >> 32;
-            env->IBAT[0][i] = sregs.u.s.ppc32.ibat[i] & 0xffffffff;
-            env->IBAT[1][i] = sregs.u.s.ppc32.ibat[i] >> 32;
-        }
     }
 
     if (cap_hior) {
@@ -1310,7 +1378,7 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 
         /* Always wake up soon in case the interrupt was level based */
         timer_mod(idle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                       (get_ticks_per_sec() / 50));
+                       (NANOSECONDS_PER_SECOND / 50));
     }
 
     /* We don't know if there are more interrupts pending after this. However,
@@ -1770,7 +1838,7 @@ uint32_t kvmppc_get_tbfreq(void)
 {
     char line[512];
     char *ns;
-    uint32_t retval = get_ticks_per_sec();
+    uint32_t retval = NANOSECONDS_PER_SECOND;
 
     if (read_cpuinfo("timebase", line, sizeof(line))) {
         return retval;
@@ -1782,8 +1850,7 @@ uint32_t kvmppc_get_tbfreq(void)
 
     ns++;
 
-    retval = atoi(ns);
-    return retval;
+    return atoi(ns);
 }
 
 bool kvmppc_get_host_serial(char **value)
@@ -1830,13 +1897,8 @@ static int kvmppc_find_cpu_dt(char *buf, int buf_len)
     return 0;
 }
 
-/* Read a CPU node property from the host device tree that's a single
- * integer (32-bit or 64-bit).  Returns 0 if anything goes wrong
- * (can't find or open the property, or doesn't understand the
- * format) */
-static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
+static uint64_t kvmppc_read_int_dt(const char *filename)
 {
-    char buf[PATH_MAX], *tmp;
     union {
         uint32_t v32;
         uint64_t v64;
@@ -1844,14 +1906,7 @@ static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
     FILE *f;
     int len;
 
-    if (kvmppc_find_cpu_dt(buf, sizeof(buf))) {
-        return -1;
-    }
-
-    tmp = g_strdup_printf("%s/%s", buf, propname);
-
-    f = fopen(tmp, "rb");
-    g_free(tmp);
+    f = fopen(filename, "rb");
     if (!f) {
         return -1;
     }
@@ -1869,6 +1924,26 @@ static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
     return 0;
 }
 
+/* Read a CPU node property from the host device tree that's a single
+ * integer (32-bit or 64-bit).  Returns 0 if anything goes wrong
+ * (can't find or open the property, or doesn't understand the
+ * format) */
+static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
+{
+    char buf[PATH_MAX], *tmp;
+    uint64_t val;
+
+    if (kvmppc_find_cpu_dt(buf, sizeof(buf))) {
+        return -1;
+    }
+
+    tmp = g_strdup_printf("%s/%s", buf, propname);
+    val = kvmppc_read_int_dt(tmp);
+    g_free(tmp);
+
+    return val;
+}
+
 uint64_t kvmppc_get_clockfreq(void)
 {
     return kvmppc_read_int_cpu_dt("clock-frequency");
@@ -1933,7 +2008,7 @@ int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
     hc[2] = cpu_to_be32(0x48000008);
     hc[3] = cpu_to_be32(bswap32(0x3860ffff));
 
-    return 0;
+    return 1;
 }
 
 static inline int kvmppc_enable_hcall(KVMState *s, target_ulong hcall)
@@ -1953,6 +2028,11 @@ void kvmppc_enable_logical_ci_hcalls(void)
     kvmppc_enable_hcall(kvm_state, H_LOGICAL_CI_STORE);
 }
 
+void kvmppc_enable_set_mode_hcall(void)
+{
+    kvmppc_enable_hcall(kvm_state, H_SET_MODE);
+}
+
 void kvmppc_set_papr(PowerPCCPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -1960,7 +2040,8 @@ void kvmppc_set_papr(PowerPCCPU *cpu)
 
     ret = kvm_vcpu_enable_cap(cs, KVM_CAP_PPC_PAPR, 0);
     if (ret) {
-        cpu_abort(cs, "This KVM version does not support PAPR\n");
+        error_report("This vCPU type or KVM version does not support PAPR");
+        exit(1);
     }
 
     /* Update the capability flag so we sync the right information
@@ -1980,7 +2061,8 @@ void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy)
 
     ret = kvm_vcpu_enable_cap(cs, KVM_CAP_PPC_EPR, 0, mpic_proxy);
     if (ret && mpic_proxy) {
-        cpu_abort(cs, "This KVM version does not support EPR\n");
+        error_report("This KVM version does not support EPR");
+        exit(1);
     }
 }
 
@@ -2066,7 +2148,7 @@ bool kvmppc_spapr_use_multitce(void)
 }
 
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel)
+                              bool need_vfio)
 {
     struct kvm_create_spapr_tce args = {
         .liobn = liobn,
@@ -2080,7 +2162,7 @@ void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
      * destroying the table, which the upper layers -will- do
      */
     *pfd = -1;
-    if (!cap_spapr_tce || (vfio_accel && !cap_spapr_vfio)) {
+    if (!cap_spapr_tce || (need_vfio && !cap_spapr_vfio)) {
         return NULL;
     }
 
@@ -2188,6 +2270,7 @@ static void kvmppc_host_cpu_initfn(Object *obj)
 
 static void kvmppc_host_cpu_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
     uint32_t vmx = kvmppc_get_vmx();
     uint32_t dfp = kvmppc_get_dfp();
@@ -2214,6 +2297,9 @@ static void kvmppc_host_cpu_class_init(ObjectClass *oc, void *data)
     if (icache_size != -1) {
         pcc->l1_icache_size = icache_size;
     }
+
+    /* Reason: kvmppc_host_cpu_initfn() dies when !kvm_enabled() */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 bool kvmppc_has_cap_epr(void)
@@ -2475,7 +2561,7 @@ error_out:
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                             uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     return 0;
 }
@@ -2484,3 +2570,12 @@ int kvm_arch_msi_data_to_gsi(uint32_t data)
 {
     return data & 0xffff;
 }
+
+int kvmppc_enable_hwrng(void)
+{
+    if (!kvm_enabled() || !kvm_check_extension(kvm_state, KVM_CAP_PPC_HWRNG)) {
+        return -1;
+    }
+
+    return kvmppc_enable_hcall(kvm_state, H_RANDOM);
+}
diff --git a/qemu/target-ppc/kvm_ppc.c b/qemu/target-ppc/kvm_ppc.c
deleted file mode 100644
index f769acd44..000000000
--- a/qemu/target-ppc/kvm_ppc.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * PowerPC KVM support
- *
- * Copyright IBM Corp. 2008
- *
- * Authors:
- *  Hollis Blanchard <hollisb@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu-common.h"
-#include "qemu/timer.h"
-#include "kvm_ppc.h"
-#include "sysemu/device_tree.h"
-#include "qemu/main-loop.h"
-
-#define PROC_DEVTREE_PATH "/proc/device-tree"
-
-static QEMUTimer *kvmppc_timer;
-static unsigned int kvmppc_timer_rate;
-
-static void kvmppc_timer_hack(void *opaque)
-{
-    qemu_notify_event();
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
-void kvmppc_init(void)
-{
-    /* XXX The only reason KVM yields control back to qemu is device IO. Since
-     * an idle guest does no IO, qemu's device model will never get a chance to
-     * run. So, until QEMU gains IO threads, we create this timer to ensure
-     * that the device model gets a chance to run. */
-    kvmppc_timer_rate = get_ticks_per_sec() / 10;
-    kvmppc_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, &kvmppc_timer_hack, NULL);
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
diff --git a/qemu/target-ppc/kvm_ppc.h b/qemu/target-ppc/kvm_ppc.h
index 4d30e2795..fc7931227 100644
--- a/qemu/target-ppc/kvm_ppc.h
+++ b/qemu/target-ppc/kvm_ppc.h
@@ -11,8 +11,6 @@
 
 #define TYPE_HOST_POWERPC_CPU "host-" TYPE_POWERPC_CPU
 
-void kvmppc_init(void);
-
 #ifdef CONFIG_KVM
 
 uint32_t kvmppc_get_tbfreq(void);
@@ -25,6 +23,7 @@ int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
 int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
 void kvmppc_enable_logical_ci_hcalls(void);
+void kvmppc_enable_set_mode_hcall(void);
 void kvmppc_set_papr(PowerPCCPU *cpu);
 int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version);
 void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
@@ -37,7 +36,7 @@ int kvmppc_booke_watchdog_enable(PowerPCCPU *cpu);
 off_t kvmppc_alloc_rma(void **rma);
 bool kvmppc_spapr_use_multitce(void);
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel);
+                              bool need_vfio);
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
 int kvmppc_reset_htab(int shift_hint);
 uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
@@ -55,6 +54,8 @@ void kvmppc_hash64_free_pteg(uint64_t token);
 void kvmppc_hash64_write_pte(CPUPPCState *env, target_ulong pte_index,
                              target_ulong pte0, target_ulong pte1);
 bool kvmppc_has_cap_fixup_hcalls(void);
+int kvmppc_enable_hwrng(void);
+int kvmppc_put_books_sregs(PowerPCCPU *cpu);
 
 #else
 
@@ -98,17 +99,16 @@ static inline int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_l
     return -1;
 }
 
-static inline int kvmppc_read_segment_page_sizes(uint32_t *prop, int maxcells)
+static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     return -1;
 }
 
-static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
+static inline void kvmppc_enable_logical_ci_hcalls(void)
 {
-    return -1;
 }
 
-static inline void kvmppc_enable_logical_ci_hcalls(void)
+static inline void kvmppc_enable_set_mode_hcall(void)
 {
 }
 
@@ -176,7 +176,7 @@ static inline int kvmppc_remove_spapr_tce(void *table, int pfd,
 
 static inline int kvmppc_reset_htab(int shift_hint)
 {
-    return -1;
+    return 0;
 }
 
 static inline uint64_t kvmppc_rma_size(uint64_t current_size,
@@ -185,11 +185,6 @@ static inline uint64_t kvmppc_rma_size(uint64_t current_size,
     return ram_size;
 }
 
-static inline int kvmppc_update_sdr1(CPUPPCState *env)
-{
-    return 0;
-}
-
 #endif /* !CONFIG_USER_ONLY */
 
 static inline bool kvmppc_has_cap_epr(void)
@@ -248,18 +243,59 @@ static inline bool kvmppc_has_cap_fixup_hcalls(void)
     abort();
 }
 
+static inline int kvmppc_enable_hwrng(void)
+{
+    return -1;
+}
+
+static inline int kvmppc_put_books_sregs(PowerPCCPU *cpu)
+{
+    abort();
+}
 #endif
 
 #ifndef CONFIG_KVM
+
 #define kvmppc_eieio() do { } while (0)
-#else
+
+static inline void kvmppc_dcbst_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+}
+
+static inline void kvmppc_icbi_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+}
+
+#else   /* CONFIG_KVM */
+
 #define kvmppc_eieio() \
     do {                                          \
         if (kvm_enabled()) {                          \
             asm volatile("eieio" : : : "memory"); \
         } \
     } while (0)
-#endif
+
+/* Store data cache blocks back to memory */
+static inline void kvmppc_dcbst_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+    uint8_t *p;
+
+    for (p = addr; p < addr + len; p += cpu->env.dcache_line_size) {
+        asm volatile("dcbst 0,%0" : : "r"(p) : "memory");
+    }
+}
+
+/* Invalidate instruction cache blocks */
+static inline void kvmppc_icbi_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+    uint8_t *p;
+
+    for (p = addr; p < addr + len; p += cpu->env.icache_line_size) {
+        asm volatile("icbi 0,%0" : : "r"(p));
+    }
+}
+
+#endif  /* CONFIG_KVM */
 
 #ifndef KVM_INTERRUPT_SET
 #define KVM_INTERRUPT_SET -1
diff --git a/qemu/target-ppc/machine.c b/qemu/target-ppc/machine.c
index f4ac7611d..46684fb93 100644
--- a/qemu/target-ppc/machine.c
+++ b/qemu/target-ppc/machine.c
@@ -1,7 +1,9 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "sysemu/kvm.h"
 #include "helper_regs.h"
+#include "mmu-hash64.h"
 
 static int cpu_load_old(QEMUFile *f, void *opaque, int version_id)
 {
@@ -134,7 +136,7 @@ static void cpu_pre_save(void *opaque)
 
     env->spr[SPR_LR] = env->lr;
     env->spr[SPR_CTR] = env->ctr;
-    env->spr[SPR_XER] = env->xer;
+    env->spr[SPR_XER] = cpu_read_xer(env);
 #if defined(TARGET_PPC64)
     env->spr[SPR_CFAR] = env->cfar;
 #endif
@@ -168,7 +170,7 @@ static int cpu_post_load(void *opaque, int version_id)
     env->spr[SPR_PVR] = env->spr_cb[SPR_PVR].default_value;
     env->lr = env->spr[SPR_LR];
     env->ctr = env->spr[SPR_CTR];
-    env->xer = env->spr[SPR_XER];
+    cpu_write_xer(env, env->spr[SPR_XER]);
 #if defined(TARGET_PPC64)
     env->cfar = env->spr[SPR_CFAR];
 #endif
@@ -352,11 +354,30 @@ static bool slb_needed(void *opaque)
     return (cpu->env.mmu_model & POWERPC_MMU_64);
 }
 
+static int slb_post_load(void *opaque, int version_id)
+{
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
+    int i;
+
+    /* We've pulled in the raw esid and vsid values from the migration
+     * stream, but we need to recompute the page size pointers */
+    for (i = 0; i < env->slb_nr; i++) {
+        if (ppc_store_slb(cpu, i, env->slb[i].esid, env->slb[i].vsid) < 0) {
+            /* Migration source had bad values in its SLB */
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 static const VMStateDescription vmstate_slb = {
     .name = "cpu/slb",
     .version_id = 1,
     .minimum_version_id = 1,
     .needed = slb_needed,
+    .post_load = slb_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(env.slb_nr, PowerPCCPU),
         VMSTATE_SLB_ARRAY(env.slb, PowerPCCPU, MAX_SLB_ENTRIES),
diff --git a/qemu/target-ppc/mem_helper.c b/qemu/target-ppc/mem_helper.c
index 6d37dae7b..6d584c912 100644
--- a/qemu/target-ppc/mem_helper.c
+++ b/qemu/target-ppc/mem_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
@@ -100,8 +101,10 @@ void helper_lswx(CPUPPCState *env, target_ulong addr, uint32_t reg,
                  uint32_t ra, uint32_t rb)
 {
     if (likely(xer_bc != 0)) {
-        if (unlikely((ra != 0 && reg < ra && (reg + xer_bc) > ra) ||
-                     (reg < rb && (reg + xer_bc) > rb))) {
+        int num_used_regs = (xer_bc + 3) / 4;
+        if (unlikely((ra != 0 && lsw_reg_in_range(reg, num_used_regs, ra)) ||
+                     lsw_reg_in_range(reg, num_used_regs, rb))) {
+            env->nip += 4;     /* Compensate the "nip - 4" from gen_lswx() */
             helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                        POWERPC_EXCP_INVAL |
                                        POWERPC_EXCP_INVAL_LSWX);
diff --git a/qemu/target-ppc/mfrom_table_gen.c b/qemu/target-ppc/mfrom_table_gen.c
index a140ded47..631791808 100644
--- a/qemu/target-ppc/mfrom_table_gen.c
+++ b/qemu/target-ppc/mfrom_table_gen.c
@@ -1,6 +1,5 @@
 #define _GNU_SOURCE
-#include <stdint.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <math.h>
 
 int main (void)
diff --git a/qemu/target-ppc/misc_helper.c b/qemu/target-ppc/misc_helper.c
index 6b12ca86a..73e3b0583 100644
--- a/qemu/target-ppc/misc_helper.c
+++ b/qemu/target-ppc/misc_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-ppc/mmu-hash32.c b/qemu/target-ppc/mmu-hash32.c
index dfee358d6..39abb2fd3 100644
--- a/qemu/target-ppc/mmu-hash32.c
+++ b/qemu/target-ppc/mmu-hash32.c
@@ -18,23 +18,18 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash32.h"
+#include "exec/log.h"
 
-//#define DEBUG_MMU
 //#define DEBUG_BAT
 
-#ifdef DEBUG_MMU
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
-#else
-#  define LOG_MMU_STATE(cpu) do { } while (0)
-#endif
-
 #ifdef DEBUG_BATS
-#  define LOG_BATS(...) qemu_log(__VA_ARGS__)
+#  define LOG_BATS(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_BATS(...) do { } while (0)
 #endif
@@ -90,9 +85,10 @@ static int ppc_hash32_pp_prot(int key, int pp, int nx)
     return prot;
 }
 
-static int ppc_hash32_pte_prot(CPUPPCState *env,
+static int ppc_hash32_pte_prot(PowerPCCPU *cpu,
                                target_ulong sr, ppc_hash_pte32_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned pp, key;
 
     key = !!(msr_pr ? (sr & SR32_KP) : (sr & SR32_KS));
@@ -101,9 +97,11 @@ static int ppc_hash32_pte_prot(CPUPPCState *env,
     return ppc_hash32_pp_prot(key, pp, !!(sr & SR32_NX));
 }
 
-static target_ulong hash32_bat_size(CPUPPCState *env,
+static target_ulong hash32_bat_size(PowerPCCPU *cpu,
                                     target_ulong batu, target_ulong batl)
 {
+    CPUPPCState *env = &cpu->env;
+
     if ((msr_pr && !(batu & BATU32_VP))
         || (!msr_pr && !(batu & BATU32_VS))) {
         return 0;
@@ -112,7 +110,7 @@ static target_ulong hash32_bat_size(CPUPPCState *env,
     return BATU32_BEPI & ~((batu & BATU32_BL) << 15);
 }
 
-static int hash32_bat_prot(CPUPPCState *env,
+static int hash32_bat_prot(PowerPCCPU *cpu,
                            target_ulong batu, target_ulong batl)
 {
     int pp, prot;
@@ -128,7 +126,7 @@ static int hash32_bat_prot(CPUPPCState *env,
     return prot;
 }
 
-static target_ulong hash32_bat_601_size(CPUPPCState *env,
+static target_ulong hash32_bat_601_size(PowerPCCPU *cpu,
                                 target_ulong batu, target_ulong batl)
 {
     if (!(batl & BATL32_601_V)) {
@@ -138,9 +136,10 @@ static target_ulong hash32_bat_601_size(CPUPPCState *env,
     return BATU32_BEPI & ~((batl & BATL32_601_BL) << 17);
 }
 
-static int hash32_bat_601_prot(CPUPPCState *env,
+static int hash32_bat_601_prot(PowerPCCPU *cpu,
                                target_ulong batu, target_ulong batl)
 {
+    CPUPPCState *env = &cpu->env;
     int key, pp;
 
     pp = batu & BATU32_601_PP;
@@ -152,9 +151,10 @@ static int hash32_bat_601_prot(CPUPPCState *env,
     return ppc_hash32_pp_prot(key, pp, 0);
 }
 
-static hwaddr ppc_hash32_bat_lookup(CPUPPCState *env, target_ulong ea, int rwx,
+static hwaddr ppc_hash32_bat_lookup(PowerPCCPU *cpu, target_ulong ea, int rwx,
                                     int *prot)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong *BATlt, *BATut;
     int i;
 
@@ -173,9 +173,9 @@ static hwaddr ppc_hash32_bat_lookup(CPUPPCState *env, target_ulong ea, int rwx,
         target_ulong mask;
 
         if (unlikely(env->mmu_model == POWERPC_MMU_601)) {
-            mask = hash32_bat_601_size(env, batu, batl);
+            mask = hash32_bat_601_size(cpu, batu, batl);
         } else {
-            mask = hash32_bat_size(env, batu, batl);
+            mask = hash32_bat_size(cpu, batu, batl);
         }
         LOG_BATS("%s: %cBAT%d v " TARGET_FMT_lx " BATu " TARGET_FMT_lx
                  " BATl " TARGET_FMT_lx "\n", __func__,
@@ -185,9 +185,9 @@ static hwaddr ppc_hash32_bat_lookup(CPUPPCState *env, target_ulong ea, int rwx,
             hwaddr raddr = (batl & mask) | (ea & ~mask);
 
             if (unlikely(env->mmu_model == POWERPC_MMU_601)) {
-                *prot = hash32_bat_601_prot(env, batu, batl);
+                *prot = hash32_bat_601_prot(cpu, batu, batl);
             } else {
-                *prot = hash32_bat_prot(env, batu, batl);
+                *prot = hash32_bat_prot(cpu, batu, batl);
             }
 
             return raddr & TARGET_PAGE_MASK;
@@ -216,11 +216,12 @@ static hwaddr ppc_hash32_bat_lookup(CPUPPCState *env, target_ulong ea, int rwx,
     return -1;
 }
 
-static int ppc_hash32_direct_store(CPUPPCState *env, target_ulong sr,
+static int ppc_hash32_direct_store(PowerPCCPU *cpu, target_ulong sr,
                                    target_ulong eaddr, int rwx,
                                    hwaddr *raddr, int *prot)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUState *cs = CPU(cpu);
+    CPUPPCState *env = &cpu->env;
     int key = !!(msr_pr ? (sr & SR32_KP) : (sr & SR32_KS));
 
     qemu_log_mask(CPU_LOG_MMU, "direct store...\n");
@@ -281,9 +282,8 @@ static int ppc_hash32_direct_store(CPUPPCState *env, target_ulong sr,
         }
         return 1;
     default:
-        qemu_log("ERROR: instruction should not need "
+        cpu_abort(cs, "ERROR: instruction should not need "
                  "address translation\n");
-        abort();
     }
     if ((rwx == 1 || key != 1) && (rwx == 0 || key != 0)) {
         *raddr = eaddr;
@@ -301,12 +301,14 @@ static int ppc_hash32_direct_store(CPUPPCState *env, target_ulong sr,
     }
 }
 
-hwaddr get_pteg_offset32(CPUPPCState *env, hwaddr hash)
+hwaddr get_pteg_offset32(PowerPCCPU *cpu, hwaddr hash)
 {
+    CPUPPCState *env = &cpu->env;
+
     return (hash * HASH_PTEG_SIZE_32) & env->htab_mask;
 }
 
-static hwaddr ppc_hash32_pteg_search(CPUPPCState *env, hwaddr pteg_off,
+static hwaddr ppc_hash32_pteg_search(PowerPCCPU *cpu, hwaddr pteg_off,
                                      bool secondary, target_ulong ptem,
                                      ppc_hash_pte32_t *pte)
 {
@@ -315,8 +317,8 @@ static hwaddr ppc_hash32_pteg_search(CPUPPCState *env, hwaddr pteg_off,
     int i;
 
     for (i = 0; i < HPTES_PER_GROUP; i++) {
-        pte0 = ppc_hash32_load_hpte0(env, pte_offset);
-        pte1 = ppc_hash32_load_hpte1(env, pte_offset);
+        pte0 = ppc_hash32_load_hpte0(cpu, pte_offset);
+        pte1 = ppc_hash32_load_hpte1(cpu, pte_offset);
 
         if ((pte0 & HPTE32_V_VALID)
             && (secondary == !!(pte0 & HPTE32_V_SECONDARY))
@@ -332,10 +334,11 @@ static hwaddr ppc_hash32_pteg_search(CPUPPCState *env, hwaddr pteg_off,
     return -1;
 }
 
-static hwaddr ppc_hash32_htab_lookup(CPUPPCState *env,
+static hwaddr ppc_hash32_htab_lookup(PowerPCCPU *cpu,
                                      target_ulong sr, target_ulong eaddr,
                                      ppc_hash_pte32_t *pte)
 {
+    CPUPPCState *env = &cpu->env;
     hwaddr pteg_off, pte_offset;
     hwaddr hash;
     uint32_t vsid, pgidx, ptem;
@@ -356,16 +359,16 @@ static hwaddr ppc_hash32_htab_lookup(CPUPPCState *env,
             " vsid=%" PRIx32 " ptem=%" PRIx32
             " hash=" TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, vsid, ptem, hash);
-    pteg_off = get_pteg_offset32(env, hash);
-    pte_offset = ppc_hash32_pteg_search(env, pteg_off, 0, ptem, pte);
+    pteg_off = get_pteg_offset32(cpu, hash);
+    pte_offset = ppc_hash32_pteg_search(cpu, pteg_off, 0, ptem, pte);
     if (pte_offset == -1) {
         /* Secondary PTEG lookup */
         qemu_log_mask(CPU_LOG_MMU, "1 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
                 " vsid=%" PRIx32 " api=%" PRIx32
                 " hash=" TARGET_FMT_plx "\n", env->htab_base,
                 env->htab_mask, vsid, ptem, ~hash);
-        pteg_off = get_pteg_offset32(env, ~hash);
-        pte_offset = ppc_hash32_pteg_search(env, pteg_off, 1, ptem, pte);
+        pteg_off = get_pteg_offset32(cpu, ~hash);
+        pte_offset = ppc_hash32_pteg_search(cpu, pteg_off, 1, ptem, pte);
     }
 
     return pte_offset;
@@ -407,7 +410,7 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
 
     /* 2. Check Block Address Translation entries (BATs) */
     if (env->nb_BATs != 0) {
-        raddr = ppc_hash32_bat_lookup(env, eaddr, rwx, &prot);
+        raddr = ppc_hash32_bat_lookup(cpu, eaddr, rwx, &prot);
         if (raddr != -1) {
             if (need_prot[rwx] & ~prot) {
                 if (rwx == 2) {
@@ -438,7 +441,7 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
 
     /* 4. Handle direct store segments */
     if (sr & SR32_T) {
-        if (ppc_hash32_direct_store(env, sr, eaddr, rwx,
+        if (ppc_hash32_direct_store(cpu, sr, eaddr, rwx,
                                     &raddr, &prot) == 0) {
             tlb_set_page(cs, eaddr & TARGET_PAGE_MASK,
                          raddr & TARGET_PAGE_MASK, prot, mmu_idx,
@@ -457,7 +460,7 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
     }
 
     /* 6. Locate the PTE in the hash table */
-    pte_offset = ppc_hash32_htab_lookup(env, sr, eaddr, &pte);
+    pte_offset = ppc_hash32_htab_lookup(cpu, sr, eaddr, &pte);
     if (pte_offset == -1) {
         if (rwx == 2) {
             cs->exception_index = POWERPC_EXCP_ISI;
@@ -480,7 +483,7 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
 
     /* 7. Check access permissions */
 
-    prot = ppc_hash32_pte_prot(env, sr, pte);
+    prot = ppc_hash32_pte_prot(cpu, sr, pte);
 
     if (need_prot[rwx] & ~prot) {
         /* Access right violation */
@@ -515,7 +518,7 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
     }
 
     if (new_pte1 != pte.pte1) {
-        ppc_hash32_store_hpte1(env, pte_offset, new_pte1);
+        ppc_hash32_store_hpte1(cpu, pte_offset, new_pte1);
     }
 
     /* 9. Determine the real address from the PTE */
@@ -528,8 +531,9 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
     return 0;
 }
 
-hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong eaddr)
+hwaddr ppc_hash32_get_phys_page_debug(PowerPCCPU *cpu, target_ulong eaddr)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong sr;
     hwaddr pte_offset;
     ppc_hash_pte32_t pte;
@@ -541,7 +545,7 @@ hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong eaddr)
     }
 
     if (env->nb_BATs != 0) {
-        hwaddr raddr = ppc_hash32_bat_lookup(env, eaddr, 0, &prot);
+        hwaddr raddr = ppc_hash32_bat_lookup(cpu, eaddr, 0, &prot);
         if (raddr != -1) {
             return raddr;
         }
@@ -554,7 +558,7 @@ hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong eaddr)
         return -1;
     }
 
-    pte_offset = ppc_hash32_htab_lookup(env, sr, eaddr, &pte);
+    pte_offset = ppc_hash32_htab_lookup(cpu, sr, eaddr, &pte);
     if (pte_offset == -1) {
         return -1;
     }
diff --git a/qemu/target-ppc/mmu-hash32.h b/qemu/target-ppc/mmu-hash32.h
index d515d4ff7..afbb9dd3d 100644
--- a/qemu/target-ppc/mmu-hash32.h
+++ b/qemu/target-ppc/mmu-hash32.h
@@ -3,8 +3,8 @@
 
 #ifndef CONFIG_USER_ONLY
 
-hwaddr get_pteg_offset32(CPUPPCState *env, hwaddr hash);
-hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong addr);
+hwaddr get_pteg_offset32(PowerPCCPU *cpu, hwaddr hash);
+hwaddr ppc_hash32_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr);
 int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong address, int rw,
                                 int mmu_idx);
 
@@ -65,40 +65,42 @@ int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong address, int rw,
 #define HPTE32_R_WIMG           0x00000078
 #define HPTE32_R_PP             0x00000003
 
-static inline target_ulong ppc_hash32_load_hpte0(CPUPPCState *env,
+static inline target_ulong ppc_hash32_load_hpte0(PowerPCCPU *cpu,
                                                  hwaddr pte_offset)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    return ldl_phys(cs->as, env->htab_base + pte_offset);
+    return ldl_phys(CPU(cpu)->as, env->htab_base + pte_offset);
 }
 
-static inline target_ulong ppc_hash32_load_hpte1(CPUPPCState *env,
+static inline target_ulong ppc_hash32_load_hpte1(PowerPCCPU *cpu,
                                                  hwaddr pte_offset)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    return ldl_phys(cs->as, env->htab_base + pte_offset + HASH_PTE_SIZE_32/2);
+    return ldl_phys(CPU(cpu)->as,
+                    env->htab_base + pte_offset + HASH_PTE_SIZE_32 / 2);
 }
 
-static inline void ppc_hash32_store_hpte0(CPUPPCState *env,
+static inline void ppc_hash32_store_hpte0(PowerPCCPU *cpu,
                                           hwaddr pte_offset, target_ulong pte0)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    stl_phys(cs->as, env->htab_base + pte_offset, pte0);
+    stl_phys(CPU(cpu)->as, env->htab_base + pte_offset, pte0);
 }
 
-static inline void ppc_hash32_store_hpte1(CPUPPCState *env,
+static inline void ppc_hash32_store_hpte1(PowerPCCPU *cpu,
                                           hwaddr pte_offset, target_ulong pte1)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    stl_phys(cs->as, env->htab_base + pte_offset + HASH_PTE_SIZE_32/2, pte1);
+    stl_phys(CPU(cpu)->as,
+             env->htab_base + pte_offset + HASH_PTE_SIZE_32 / 2, pte1);
 }
 
 typedef struct {
diff --git a/qemu/target-ppc/mmu-hash64.c b/qemu/target-ppc/mmu-hash64.c
index 7df6edebf..72c4ab5d7 100644
--- a/qemu/target-ppc/mmu-hash64.c
+++ b/qemu/target-ppc/mmu-hash64.c
@@ -17,38 +17,38 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
+#include "qemu/error-report.h"
 #include "kvm_ppc.h"
 #include "mmu-hash64.h"
+#include "exec/log.h"
 
-//#define DEBUG_MMU
 //#define DEBUG_SLB
 
-#ifdef DEBUG_MMU
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
-#else
-#  define LOG_MMU_STATE(cpu) do { } while (0)
-#endif
-
 #ifdef DEBUG_SLB
-#  define LOG_SLB(...) qemu_log(__VA_ARGS__)
+#  define LOG_SLB(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_SLB(...) do { } while (0)
 #endif
 
 /*
- * Used to indicate whether we have allocated htab in the
- * host kernel
+ * Used to indicate that a CPU has its hash page table (HPT) managed
+ * within the host kernel
  */
-bool kvmppc_kern_htab;
+#define MMU_HASH64_KVM_MANAGED_HPT      ((void *)-1)
+
 /*
  * SLB handling
  */
 
-static ppc_slb_t *slb_lookup(CPUPPCState *env, target_ulong eaddr)
+static ppc_slb_t *slb_lookup(PowerPCCPU *cpu, target_ulong eaddr)
 {
+    CPUPPCState *env = &cpu->env;
     uint64_t esid_256M, esid_1T;
     int n;
 
@@ -76,12 +76,13 @@ static ppc_slb_t *slb_lookup(CPUPPCState *env, target_ulong eaddr)
     return NULL;
 }
 
-void dump_slb(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env)
+void dump_slb(FILE *f, fprintf_function cpu_fprintf, PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
     int i;
     uint64_t slbe, slbv;
 
-    cpu_synchronize_state(CPU(ppc_env_get_cpu(env)));
+    cpu_synchronize_state(CPU(cpu));
 
     cpu_fprintf(f, "SLB\tESID\t\t\tVSID\n");
     for (i = 0; i < env->slb_nr; i++) {
@@ -124,7 +125,7 @@ void helper_slbie(CPUPPCState *env, target_ulong addr)
     PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_slb_t *slb;
 
-    slb = slb_lookup(env, addr);
+    slb = slb_lookup(cpu, addr);
     if (!slb) {
         return;
     }
@@ -140,35 +141,62 @@ void helper_slbie(CPUPPCState *env, target_ulong addr)
     }
 }
 
-int ppc_store_slb(CPUPPCState *env, target_ulong rb, target_ulong rs)
+int ppc_store_slb(PowerPCCPU *cpu, target_ulong slot,
+                  target_ulong esid, target_ulong vsid)
 {
-    int slot = rb & 0xfff;
+    CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb = &env->slb[slot];
+    const struct ppc_one_seg_page_size *sps = NULL;
+    int i;
 
-    if (rb & (0x1000 - env->slb_nr)) {
-        return -1; /* Reserved bits set or slot too high */
+    if (slot >= env->slb_nr) {
+        return -1; /* Bad slot number */
+    }
+    if (esid & ~(SLB_ESID_ESID | SLB_ESID_V)) {
+        return -1; /* Reserved bits set */
     }
-    if (rs & (SLB_VSID_B & ~SLB_VSID_B_1T)) {
+    if (vsid & (SLB_VSID_B & ~SLB_VSID_B_1T)) {
         return -1; /* Bad segment size */
     }
-    if ((rs & SLB_VSID_B) && !(env->mmu_model & POWERPC_MMU_1TSEG)) {
+    if ((vsid & SLB_VSID_B) && !(env->mmu_model & POWERPC_MMU_1TSEG)) {
         return -1; /* 1T segment on MMU that doesn't support it */
     }
 
-    /* Mask out the slot number as we store the entry */
-    slb->esid = rb & (SLB_ESID_ESID | SLB_ESID_V);
-    slb->vsid = rs;
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_seg_page_size *sps1 = &env->sps.sps[i];
+
+        if (!sps1->page_shift) {
+            break;
+        }
+
+        if ((vsid & SLB_VSID_LLP_MASK) == sps1->slb_enc) {
+            sps = sps1;
+            break;
+        }
+    }
+
+    if (!sps) {
+        error_report("Bad page size encoding in SLB store: slot "TARGET_FMT_lu
+                     " esid 0x"TARGET_FMT_lx" vsid 0x"TARGET_FMT_lx,
+                     slot, esid, vsid);
+        return -1;
+    }
+
+    slb->esid = esid;
+    slb->vsid = vsid;
+    slb->sps = sps;
 
     LOG_SLB("%s: %d " TARGET_FMT_lx " - " TARGET_FMT_lx " => %016" PRIx64
-            " %016" PRIx64 "\n", __func__, slot, rb, rs,
+            " %016" PRIx64 "\n", __func__, slot, esid, vsid,
             slb->esid, slb->vsid);
 
     return 0;
 }
 
-static int ppc_load_slb_esid(CPUPPCState *env, target_ulong rb,
+static int ppc_load_slb_esid(PowerPCCPU *cpu, target_ulong rb,
                              target_ulong *rt)
 {
+    CPUPPCState *env = &cpu->env;
     int slot = rb & 0xfff;
     ppc_slb_t *slb = &env->slb[slot];
 
@@ -180,9 +208,10 @@ static int ppc_load_slb_esid(CPUPPCState *env, target_ulong rb,
     return 0;
 }
 
-static int ppc_load_slb_vsid(CPUPPCState *env, target_ulong rb,
+static int ppc_load_slb_vsid(PowerPCCPU *cpu, target_ulong rb,
                              target_ulong *rt)
 {
+    CPUPPCState *env = &cpu->env;
     int slot = rb & 0xfff;
     ppc_slb_t *slb = &env->slb[slot];
 
@@ -196,7 +225,9 @@ static int ppc_load_slb_vsid(CPUPPCState *env, target_ulong rb,
 
 void helper_store_slb(CPUPPCState *env, target_ulong rb, target_ulong rs)
 {
-    if (ppc_store_slb(env, rb, rs) < 0) {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    if (ppc_store_slb(cpu, rb & 0xfff, rb & ~0xfffULL, rs) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -204,9 +235,10 @@ void helper_store_slb(CPUPPCState *env, target_ulong rb, target_ulong rs)
 
 target_ulong helper_load_slb_esid(CPUPPCState *env, target_ulong rb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     target_ulong rt = 0;
 
-    if (ppc_load_slb_esid(env, rb, &rt) < 0) {
+    if (ppc_load_slb_esid(cpu, rb, &rt) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -215,9 +247,10 @@ target_ulong helper_load_slb_esid(CPUPPCState *env, target_ulong rb)
 
 target_ulong helper_load_slb_vsid(CPUPPCState *env, target_ulong rb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     target_ulong rt = 0;
 
-    if (ppc_load_slb_vsid(env, rb, &rt) < 0) {
+    if (ppc_load_slb_vsid(cpu, rb, &rt) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -227,10 +260,58 @@ target_ulong helper_load_slb_vsid(CPUPPCState *env, target_ulong rb)
 /*
  * 64-bit hash table MMU handling
  */
+void ppc_hash64_set_sdr1(PowerPCCPU *cpu, target_ulong value,
+                         Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    target_ulong htabsize = value & SDR_64_HTABSIZE;
+
+    env->spr[SPR_SDR1] = value;
+    if (htabsize > 28) {
+        error_setg(errp,
+                   "Invalid HTABSIZE 0x" TARGET_FMT_lx" stored in SDR1",
+                   htabsize);
+        htabsize = 28;
+    }
+    env->htab_mask = (1ULL << (htabsize + 18 - 7)) - 1;
+    env->htab_base = value & SDR_64_HTABORG;
+}
+
+void ppc_hash64_set_external_hpt(PowerPCCPU *cpu, void *hpt, int shift,
+                                 Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    Error *local_err = NULL;
 
-static int ppc_hash64_pte_prot(CPUPPCState *env,
+    cpu_synchronize_state(CPU(cpu));
+
+    if (hpt) {
+        env->external_htab = hpt;
+    } else {
+        env->external_htab = MMU_HASH64_KVM_MANAGED_HPT;
+    }
+    ppc_hash64_set_sdr1(cpu, (target_ulong)(uintptr_t)hpt | (shift - 18),
+                        &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Not strictly necessary, but makes it clearer that an external
+     * htab is in use when debugging */
+    env->htab_base = -1;
+
+    if (kvm_enabled()) {
+        if (kvmppc_put_books_sregs(cpu) < 0) {
+            error_setg(errp, "Unable to update SDR1 in KVM");
+        }
+    }
+}
+
+static int ppc_hash64_pte_prot(PowerPCCPU *cpu,
                                ppc_slb_t *slb, ppc_hash_pte64_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned pp, key;
     /* Some pp bit combinations have undefined behaviour, so default
      * to no access in those cases */
@@ -280,12 +361,12 @@ static int ppc_hash64_pte_prot(CPUPPCState *env,
     return prot;
 }
 
-static int ppc_hash64_amr_prot(CPUPPCState *env, ppc_hash_pte64_t pte)
+static int ppc_hash64_amr_prot(PowerPCCPU *cpu, ppc_hash_pte64_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     int key, amrbits;
     int prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 
-
     /* Only recent MMUs implement Virtual Page Class Key Protection */
     if (!(env->mmu_model & POWERPC_MMU_AMR)) {
         return prot;
@@ -321,25 +402,16 @@ uint64_t ppc_hash64_start_access(PowerPCCPU *cpu, target_ulong pte_index)
     hwaddr pte_offset;
 
     pte_offset = pte_index * HASH_PTE_SIZE_64;
-    if (kvmppc_kern_htab) {
+    if (cpu->env.external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
         /*
          * HTAB is controlled by KVM. Fetch the PTEG into a new buffer.
          */
         token = kvmppc_hash64_read_pteg(cpu, pte_index);
-        if (token) {
-            return token;
-        }
+    } else if (cpu->env.external_htab) {
         /*
-         * pteg read failed, even though we have allocated htab via
-         * kvmppc_reset_htab.
+         * HTAB is controlled by QEMU. Just point to the internally
+         * accessible PTEG.
          */
-        return 0;
-    }
-    /*
-     * HTAB is controlled by QEMU. Just point to the internally
-     * accessible PTEG.
-     */
-    if (cpu->env.external_htab) {
         token = (uint64_t)(uintptr_t) cpu->env.external_htab + pte_offset;
     } else if (cpu->env.htab_base) {
         token = cpu->env.htab_base + pte_offset;
@@ -347,86 +419,73 @@ uint64_t ppc_hash64_start_access(PowerPCCPU *cpu, target_ulong pte_index)
     return token;
 }
 
-void ppc_hash64_stop_access(uint64_t token)
+void ppc_hash64_stop_access(PowerPCCPU *cpu, uint64_t token)
 {
-    if (kvmppc_kern_htab) {
+    if (cpu->env.external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
         kvmppc_hash64_free_pteg(token);
     }
 }
 
-static hwaddr ppc_hash64_pteg_search(CPUPPCState *env, hwaddr hash,
+static hwaddr ppc_hash64_pteg_search(PowerPCCPU *cpu, hwaddr hash,
                                      bool secondary, target_ulong ptem,
                                      ppc_hash_pte64_t *pte)
 {
+    CPUPPCState *env = &cpu->env;
     int i;
     uint64_t token;
     target_ulong pte0, pte1;
     target_ulong pte_index;
 
     pte_index = (hash & env->htab_mask) * HPTES_PER_GROUP;
-    token = ppc_hash64_start_access(ppc_env_get_cpu(env), pte_index);
+    token = ppc_hash64_start_access(cpu, pte_index);
     if (!token) {
         return -1;
     }
     for (i = 0; i < HPTES_PER_GROUP; i++) {
-        pte0 = ppc_hash64_load_hpte0(env, token, i);
-        pte1 = ppc_hash64_load_hpte1(env, token, i);
+        pte0 = ppc_hash64_load_hpte0(cpu, token, i);
+        pte1 = ppc_hash64_load_hpte1(cpu, token, i);
 
         if ((pte0 & HPTE64_V_VALID)
             && (secondary == !!(pte0 & HPTE64_V_SECONDARY))
             && HPTE64_V_COMPARE(pte0, ptem)) {
             pte->pte0 = pte0;
             pte->pte1 = pte1;
-            ppc_hash64_stop_access(token);
+            ppc_hash64_stop_access(cpu, token);
             return (pte_index + i) * HASH_PTE_SIZE_64;
         }
     }
-    ppc_hash64_stop_access(token);
+    ppc_hash64_stop_access(cpu, token);
     /*
      * We didn't find a valid entry.
      */
     return -1;
 }
 
-static uint64_t ppc_hash64_page_shift(ppc_slb_t *slb)
-{
-    uint64_t epnshift;
-
-    /* Page size according to the SLB, which we use to generate the
-     * EPN for hash table lookup..  When we implement more recent MMU
-     * extensions this might be different from the actual page size
-     * encoded in the PTE */
-    if ((slb->vsid & SLB_VSID_LLP_MASK) == SLB_VSID_4K) {
-        epnshift = TARGET_PAGE_BITS;
-    } else if ((slb->vsid & SLB_VSID_LLP_MASK) == SLB_VSID_64K) {
-        epnshift = TARGET_PAGE_BITS_64K;
-    } else {
-        epnshift = TARGET_PAGE_BITS_16M;
-    }
-    return epnshift;
-}
-
-static hwaddr ppc_hash64_htab_lookup(CPUPPCState *env,
+static hwaddr ppc_hash64_htab_lookup(PowerPCCPU *cpu,
                                      ppc_slb_t *slb, target_ulong eaddr,
                                      ppc_hash_pte64_t *pte)
 {
+    CPUPPCState *env = &cpu->env;
     hwaddr pte_offset;
     hwaddr hash;
-    uint64_t vsid, epnshift, epnmask, epn, ptem;
+    uint64_t vsid, epnmask, epn, ptem;
+
+    /* The SLB store path should prevent any bad page size encodings
+     * getting in there, so: */
+    assert(slb->sps);
 
-    epnshift = ppc_hash64_page_shift(slb);
-    epnmask = ~((1ULL << epnshift) - 1);
+    epnmask = ~((1ULL << slb->sps->page_shift) - 1);
 
     if (slb->vsid & SLB_VSID_B) {
         /* 1TB segment */
         vsid = (slb->vsid & SLB_VSID_VSID) >> SLB_VSID_SHIFT_1T;
         epn = (eaddr & ~SEGMENT_MASK_1T) & epnmask;
-        hash = vsid ^ (vsid << 25) ^ (epn >> epnshift);
+        hash = vsid ^ (vsid << 25) ^ (epn >> slb->sps->page_shift);
     } else {
         /* 256M segment */
         vsid = (slb->vsid & SLB_VSID_VSID) >> SLB_VSID_SHIFT;
         epn = (eaddr & ~SEGMENT_MASK_256M) & epnmask;
-        hash = vsid ^ (epn >> epnshift);
+        hash = vsid ^ (epn >> slb->sps->page_shift);
     }
     ptem = (slb->vsid & SLB_VSID_PTEM) | ((epn >> 16) & HPTE64_V_AVPN);
 
@@ -442,7 +501,7 @@ static hwaddr ppc_hash64_htab_lookup(CPUPPCState *env,
             " vsid=" TARGET_FMT_lx " ptem=" TARGET_FMT_lx
             " hash=" TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, vsid, ptem,  hash);
-    pte_offset = ppc_hash64_pteg_search(env, hash, 0, ptem, pte);
+    pte_offset = ppc_hash64_pteg_search(cpu, hash, 0, ptem, pte);
 
     if (pte_offset == -1) {
         /* Secondary PTEG lookup */
@@ -452,24 +511,82 @@ static hwaddr ppc_hash64_htab_lookup(CPUPPCState *env,
                 " hash=" TARGET_FMT_plx "\n", env->htab_base,
                 env->htab_mask, vsid, ptem, ~hash);
 
-        pte_offset = ppc_hash64_pteg_search(env, ~hash, 1, ptem, pte);
+        pte_offset = ppc_hash64_pteg_search(cpu, ~hash, 1, ptem, pte);
     }
 
     return pte_offset;
 }
 
-static hwaddr ppc_hash64_pte_raddr(ppc_slb_t *slb, ppc_hash_pte64_t pte,
-                                   target_ulong eaddr)
+static unsigned hpte_page_shift(const struct ppc_one_seg_page_size *sps,
+    uint64_t pte0, uint64_t pte1)
+{
+    int i;
+
+    if (!(pte0 & HPTE64_V_LARGE)) {
+        if (sps->page_shift != 12) {
+            /* 4kiB page in a non 4kiB segment */
+            return 0;
+        }
+        /* Normal 4kiB page */
+        return 12;
+    }
+
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_page_size *ps = &sps->enc[i];
+        uint64_t mask;
+
+        if (!ps->page_shift) {
+            break;
+        }
+
+        if (ps->page_shift == 12) {
+            /* L bit is set so this can't be a 4kiB page */
+            continue;
+        }
+
+        mask = ((1ULL << ps->page_shift) - 1) & HPTE64_R_RPN;
+
+        if ((pte1 & mask) == (ps->pte_enc << HPTE64_R_RPN_SHIFT)) {
+            return ps->page_shift;
+        }
+    }
+
+    return 0; /* Bad page size encoding */
+}
+
+unsigned ppc_hash64_hpte_page_shift_noslb(PowerPCCPU *cpu,
+                                          uint64_t pte0, uint64_t pte1,
+                                          unsigned *seg_page_shift)
 {
-    hwaddr mask;
-    int target_page_bits;
-    hwaddr rpn = pte.pte1 & HPTE64_R_RPN;
+    CPUPPCState *env = &cpu->env;
+    int i;
+
+    if (!(pte0 & HPTE64_V_LARGE)) {
+        *seg_page_shift = 12;
+        return 12;
+    }
+
     /*
-     * We support 4K, 64K and 16M now
+     * The encodings in env->sps need to be carefully chosen so that
+     * this gives an unambiguous result.
      */
-    target_page_bits = ppc_hash64_page_shift(slb);
-    mask = (1ULL << target_page_bits) - 1;
-    return (rpn & ~mask) | (eaddr & mask);
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_seg_page_size *sps = &env->sps.sps[i];
+        unsigned shift;
+
+        if (!sps->page_shift) {
+            break;
+        }
+
+        shift = hpte_page_shift(sps, pte0, pte1);
+        if (shift) {
+            *seg_page_shift = sps->page_shift;
+            return shift;
+        }
+    }
+
+    *seg_page_shift = 0;
+    return 0;
 }
 
 int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
@@ -478,6 +595,7 @@ int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
     CPUState *cs = CPU(cpu);
     CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb;
+    unsigned apshift;
     hwaddr pte_offset;
     ppc_hash_pte64_t pte;
     int pp_prot, amr_prot, prot;
@@ -499,7 +617,7 @@ int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
     }
 
     /* 2. Translation is on, so look up the SLB */
-    slb = slb_lookup(env, eaddr);
+    slb = slb_lookup(cpu, eaddr);
 
     if (!slb) {
         if (rwx == 2) {
@@ -521,7 +639,7 @@ int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
     }
 
     /* 4. Locate the PTE in the hash table */
-    pte_offset = ppc_hash64_htab_lookup(env, slb, eaddr, &pte);
+    pte_offset = ppc_hash64_htab_lookup(cpu, slb, eaddr, &pte);
     if (pte_offset == -1) {
         if (rwx == 2) {
             cs->exception_index = POWERPC_EXCP_ISI;
@@ -541,10 +659,22 @@ int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
     qemu_log_mask(CPU_LOG_MMU,
                 "found PTE at offset %08" HWADDR_PRIx "\n", pte_offset);
 
+    /* Validate page size encoding */
+    apshift = hpte_page_shift(slb->sps, pte.pte0, pte.pte1);
+    if (!apshift) {
+        error_report("Bad page size encoding in HPTE 0x%"PRIx64" - 0x%"PRIx64
+                     " @ 0x%"HWADDR_PRIx, pte.pte0, pte.pte1, pte_offset);
+        /* Not entirely sure what the right action here, but machine
+         * check seems reasonable */
+        cs->exception_index = POWERPC_EXCP_MCHECK;
+        env->error_code = 0;
+        return 1;
+    }
+
     /* 5. Check access permissions */
 
-    pp_prot = ppc_hash64_pte_prot(env, slb, pte);
-    amr_prot = ppc_hash64_amr_prot(env, pte);
+    pp_prot = ppc_hash64_pte_prot(cpu, slb, pte);
+    amr_prot = ppc_hash64_amr_prot(cpu, pte);
     prot = pp_prot & amr_prot;
 
     if ((need_prot[rwx] & ~prot) != 0) {
@@ -587,51 +717,59 @@ int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
     }
 
     if (new_pte1 != pte.pte1) {
-        ppc_hash64_store_hpte(env, pte_offset / HASH_PTE_SIZE_64,
+        ppc_hash64_store_hpte(cpu, pte_offset / HASH_PTE_SIZE_64,
                               pte.pte0, new_pte1);
     }
 
     /* 7. Determine the real address from the PTE */
 
-    raddr = ppc_hash64_pte_raddr(slb, pte, eaddr);
+    raddr = deposit64(pte.pte1 & HPTE64_R_RPN, 0, apshift, eaddr);
 
     tlb_set_page(cs, eaddr & TARGET_PAGE_MASK, raddr & TARGET_PAGE_MASK,
-                 prot, mmu_idx, TARGET_PAGE_SIZE);
+                 prot, mmu_idx, 1ULL << apshift);
 
     return 0;
 }
 
-hwaddr ppc_hash64_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
+hwaddr ppc_hash64_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr)
 {
+    CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb;
     hwaddr pte_offset;
     ppc_hash_pte64_t pte;
+    unsigned apshift;
 
     if (msr_dr == 0) {
         /* In real mode the top 4 effective address bits are ignored */
         return addr & 0x0FFFFFFFFFFFFFFFULL;
     }
 
-    slb = slb_lookup(env, addr);
+    slb = slb_lookup(cpu, addr);
     if (!slb) {
         return -1;
     }
 
-    pte_offset = ppc_hash64_htab_lookup(env, slb, addr, &pte);
+    pte_offset = ppc_hash64_htab_lookup(cpu, slb, addr, &pte);
     if (pte_offset == -1) {
         return -1;
     }
 
-    return ppc_hash64_pte_raddr(slb, pte, addr) & TARGET_PAGE_MASK;
+    apshift = hpte_page_shift(slb->sps, pte.pte0, pte.pte1);
+    if (!apshift) {
+        return -1;
+    }
+
+    return deposit64(pte.pte1 & HPTE64_R_RPN, 0, apshift, addr)
+        & TARGET_PAGE_MASK;
 }
 
-void ppc_hash64_store_hpte(CPUPPCState *env,
+void ppc_hash64_store_hpte(PowerPCCPU *cpu,
                            target_ulong pte_index,
                            target_ulong pte0, target_ulong pte1)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
-    if (kvmppc_kern_htab) {
+    if (env->external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
         kvmppc_hash64_write_pte(env, pte_index, pte0, pte1);
         return;
     }
@@ -639,9 +777,22 @@ void ppc_hash64_store_hpte(CPUPPCState *env,
     pte_index *= HASH_PTE_SIZE_64;
     if (env->external_htab) {
         stq_p(env->external_htab + pte_index, pte0);
-        stq_p(env->external_htab + pte_index + HASH_PTE_SIZE_64/2, pte1);
+        stq_p(env->external_htab + pte_index + HASH_PTE_SIZE_64 / 2, pte1);
     } else {
-        stq_phys(cs->as, env->htab_base + pte_index, pte0);
-        stq_phys(cs->as, env->htab_base + pte_index + HASH_PTE_SIZE_64/2, pte1);
+        stq_phys(CPU(cpu)->as, env->htab_base + pte_index, pte0);
+        stq_phys(CPU(cpu)->as,
+                 env->htab_base + pte_index + HASH_PTE_SIZE_64 / 2, pte1);
     }
 }
+
+void ppc_hash64_tlb_flush_hpte(PowerPCCPU *cpu,
+                               target_ulong pte_index,
+                               target_ulong pte0, target_ulong pte1)
+{
+    /*
+     * XXX: given the fact that there are too many segments to
+     * invalidate, and we still don't have a tlb_flush_mask(env, n,
+     * mask) in QEMU, we just invalidate all TLBs
+     */
+    tlb_flush(CPU(cpu), 1);
+}
diff --git a/qemu/target-ppc/mmu-hash64.h b/qemu/target-ppc/mmu-hash64.h
index 291750f3e..9bf8b9b26 100644
--- a/qemu/target-ppc/mmu-hash64.h
+++ b/qemu/target-ppc/mmu-hash64.h
@@ -4,13 +4,21 @@
 #ifndef CONFIG_USER_ONLY
 
 #ifdef TARGET_PPC64
-void dump_slb(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env);
-int ppc_store_slb (CPUPPCState *env, target_ulong rb, target_ulong rs);
-hwaddr ppc_hash64_get_phys_page_debug(CPUPPCState *env, target_ulong addr);
+void ppc_hash64_check_page_sizes(PowerPCCPU *cpu, Error **errp);
+void dump_slb(FILE *f, fprintf_function cpu_fprintf, PowerPCCPU *cpu);
+int ppc_store_slb(PowerPCCPU *cpu, target_ulong slot,
+                  target_ulong esid, target_ulong vsid);
+hwaddr ppc_hash64_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr);
 int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong address, int rw,
                                 int mmu_idx);
-void ppc_hash64_store_hpte(CPUPPCState *env, target_ulong index,
+void ppc_hash64_store_hpte(PowerPCCPU *cpu, target_ulong index,
                            target_ulong pte0, target_ulong pte1);
+void ppc_hash64_tlb_flush_hpte(PowerPCCPU *cpu,
+                               target_ulong pte_index,
+                               target_ulong pte0, target_ulong pte1);
+unsigned ppc_hash64_hpte_page_shift_noslb(PowerPCCPU *cpu,
+                                          uint64_t pte0, uint64_t pte1,
+                                          unsigned *seg_page_shift);
 #endif
 
 /*
@@ -40,6 +48,8 @@ void ppc_hash64_store_hpte(CPUPPCState *env, target_ulong index,
 #define SLB_VSID_LLP_MASK       (SLB_VSID_L | SLB_VSID_LP)
 #define SLB_VSID_4K             0x0000000000000000ULL
 #define SLB_VSID_64K            0x0000000000000110ULL
+#define SLB_VSID_16M            0x0000000000000100ULL
+#define SLB_VSID_16G            0x0000000000000120ULL
 
 /*
  * Hash page table definitions
@@ -80,36 +90,39 @@ void ppc_hash64_store_hpte(CPUPPCState *env, target_ulong index,
 #define HPTE64_V_1TB_SEG        0x4000000000000000ULL
 #define HPTE64_V_VRMA_MASK      0x4001ffffff000000ULL
 
+void ppc_hash64_set_sdr1(PowerPCCPU *cpu, target_ulong value,
+                         Error **errp);
+void ppc_hash64_set_external_hpt(PowerPCCPU *cpu, void *hpt, int shift,
+                                 Error **errp);
 
-extern bool kvmppc_kern_htab;
 uint64_t ppc_hash64_start_access(PowerPCCPU *cpu, target_ulong pte_index);
-void ppc_hash64_stop_access(uint64_t token);
+void ppc_hash64_stop_access(PowerPCCPU *cpu, uint64_t token);
 
-static inline target_ulong ppc_hash64_load_hpte0(CPUPPCState *env,
+static inline target_ulong ppc_hash64_load_hpte0(PowerPCCPU *cpu,
                                                  uint64_t token, int index)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
     uint64_t addr;
 
     addr = token + (index * HASH_PTE_SIZE_64);
     if (env->external_htab) {
         return  ldq_p((const void *)(uintptr_t)addr);
     } else {
-        return ldq_phys(cs->as, addr);
+        return ldq_phys(CPU(cpu)->as, addr);
     }
 }
 
-static inline target_ulong ppc_hash64_load_hpte1(CPUPPCState *env,
+static inline target_ulong ppc_hash64_load_hpte1(PowerPCCPU *cpu,
                                                  uint64_t token, int index)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
     uint64_t addr;
 
     addr = token + (index * HASH_PTE_SIZE_64) + HASH_PTE_SIZE_64/2;
     if (env->external_htab) {
         return  ldq_p((const void *)(uintptr_t)addr);
     } else {
-        return ldq_phys(cs->as, addr);
+        return ldq_phys(CPU(cpu)->as, addr);
     }
 }
 
diff --git a/qemu/target-ppc/mmu_helper.c b/qemu/target-ppc/mmu_helper.c
index 527c6adca..ff217941b 100644
--- a/qemu/target-ppc/mmu_helper.c
+++ b/qemu/target-ppc/mmu_helper.c
@@ -16,6 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
@@ -23,28 +25,28 @@
 #include "mmu-hash64.h"
 #include "mmu-hash32.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
 
 //#define DEBUG_MMU
 //#define DEBUG_BATS
 //#define DEBUG_SOFTWARE_TLB
 //#define DUMP_PAGE_TABLES
-//#define DEBUG_SOFTWARE_TLB
 //#define FLUSH_ALL_TLBS
 
 #ifdef DEBUG_MMU
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
+#  define LOG_MMU_STATE(cpu) log_cpu_state_mask(CPU_LOG_MMU, (cpu), 0)
 #else
 #  define LOG_MMU_STATE(cpu) do { } while (0)
 #endif
 
 #ifdef DEBUG_SOFTWARE_TLB
-#  define LOG_SWTLB(...) qemu_log(__VA_ARGS__)
+#  define LOG_SWTLB(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_SWTLB(...) do { } while (0)
 #endif
 
 #ifdef DEBUG_BATS
-#  define LOG_BATS(...) qemu_log(__VA_ARGS__)
+#  define LOG_BATS(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_BATS(...) do { } while (0)
 #endif
@@ -162,7 +164,7 @@ static inline int ppc6xx_tlb_pte_check(mmu_ctx_t *ctx, target_ulong pte0,
             if (ctx->raddr != (hwaddr)-1ULL) {
                 /* all matches should have equal RPN, WIMG & PP */
                 if ((ctx->raddr & mmask) != (pte1 & mmask)) {
-                    qemu_log("Bad RPN/WIMG/PP\n");
+                    qemu_log_mask(CPU_LOG_MMU, "Bad RPN/WIMG/PP\n");
                     return -3;
                 }
             }
@@ -508,7 +510,7 @@ static inline int get_segment_6xx_tlb(CPUPPCState *env, mmu_ctx_t *ctx,
             /* Software TLB search */
             ret = ppc6xx_tlb_check(env, ctx, eaddr, rw, type);
 #if defined(DUMP_PAGE_TABLES)
-            if (qemu_log_enabled()) {
+            if (qemu_log_mask(CPU_LOG_MMU)) {
                 hwaddr curaddr;
                 uint32_t a0, a1, a2, a3;
 
@@ -575,8 +577,8 @@ static inline int get_segment_6xx_tlb(CPUPPCState *env, mmu_ctx_t *ctx,
             /* eciwx or ecowx */
             return -4;
         default:
-            qemu_log("ERROR: instruction should not need "
-                        "address translation\n");
+            qemu_log_mask(CPU_LOG_MMU, "ERROR: instruction should not need "
+                          "address translation\n");
             return -4;
         }
         if ((rw == 1 || ctx->key != 1) && (rw == 0 || ctx->key != 0)) {
@@ -658,32 +660,6 @@ static inline void ppc4xx_tlb_invalidate_all(CPUPPCState *env)
     tlb_flush(CPU(cpu), 1);
 }
 
-static inline void ppc4xx_tlb_invalidate_virt(CPUPPCState *env,
-                                              target_ulong eaddr, uint32_t pid)
-{
-#if !defined(FLUSH_ALL_TLBS)
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
-    ppcemb_tlb_t *tlb;
-    hwaddr raddr;
-    target_ulong page, end;
-    int i;
-
-    for (i = 0; i < env->nb_tlb; i++) {
-        tlb = &env->tlb.tlbe[i];
-        if (ppcemb_tlb_check(env, tlb, &raddr, eaddr, pid, 0, i) == 0) {
-            end = tlb->EPN + tlb->size;
-            for (page = tlb->EPN; page < end; page += TARGET_PAGE_SIZE) {
-                tlb_flush_page(cs, page);
-            }
-            tlb->prot &= ~PAGE_VALID;
-            break;
-        }
-    }
-#else
-    ppc4xx_tlb_invalidate_all(env);
-#endif
-}
-
 static int mmu40x_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                        target_ulong address, int rw,
                                        int access_type)
@@ -1293,10 +1269,12 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env)
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
-        dump_slb(f, cpu_fprintf, env);
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
+        dump_slb(f, cpu_fprintf, ppc_env_get_cpu(env));
         break;
 #endif
     default:
@@ -1433,15 +1411,17 @@ hwaddr ppc_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     switch (env->mmu_model) {
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
-        return ppc_hash64_get_phys_page_debug(env, addr);
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
+        return ppc_hash64_get_phys_page_debug(cpu, addr);
 #endif
 
     case POWERPC_MMU_32B:
     case POWERPC_MMU_601:
-        return ppc_hash32_get_phys_page_debug(env, addr);
+        return ppc_hash32_get_phys_page_debug(cpu, addr);
 
     default:
         ;
@@ -1507,6 +1487,7 @@ static int cpu_ppc_handle_mmu_fault(CPUPPCState *env, target_ulong address,
                                     int rw, int mmu_idx)
 {
     CPUState *cs = CPU(ppc_env_get_cpu(env));
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
     mmu_ctx_t ctx;
     int access_type;
     int ret = 0;
@@ -1608,9 +1589,9 @@ static int cpu_ppc_handle_mmu_fault(CPUPPCState *env, target_ulong address,
                 tlb_miss:
                     env->error_code |= ctx.key << 19;
                     env->spr[SPR_HASH1] = env->htab_base +
-                        get_pteg_offset32(env, ctx.hash[0]);
+                        get_pteg_offset32(cpu, ctx.hash[0]);
                     env->spr[SPR_HASH2] = env->htab_base +
-                        get_pteg_offset32(env, ctx.hash[1]);
+                        get_pteg_offset32(cpu, ctx.hash[1]);
                     break;
                 case POWERPC_MMU_SOFT_74xx:
                     if (rw == 1) {
@@ -1937,9 +1918,11 @@ void ppc_tlb_invalidate_all(CPUPPCState *env)
     case POWERPC_MMU_601:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
 #endif /* defined(TARGET_PPC64) */
         tlb_flush(CPU(cpu), 1);
         break;
@@ -1965,25 +1948,6 @@ void ppc_tlb_invalidate_one(CPUPPCState *env, target_ulong addr)
             ppc6xx_tlb_invalidate_virt(env, addr, 1);
         }
         break;
-    case POWERPC_MMU_SOFT_4xx:
-    case POWERPC_MMU_SOFT_4xx_Z:
-        ppc4xx_tlb_invalidate_virt(env, addr, env->spr[SPR_40x_PID]);
-        break;
-    case POWERPC_MMU_REAL:
-        cpu_abort(CPU(cpu), "No TLB for PowerPC 4xx in real mode\n");
-        break;
-    case POWERPC_MMU_MPC8xx:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "MPC8xx MMU model is not implemented\n");
-        break;
-    case POWERPC_MMU_BOOKE:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "BookE MMU model is not implemented\n");
-        break;
-    case POWERPC_MMU_BOOKE206:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "BookE 2.06 MMU model is not implemented\n");
-        break;
     case POWERPC_MMU_32B:
     case POWERPC_MMU_601:
         /* tlbie invalidate TLBs for all segments */
@@ -2011,9 +1975,11 @@ void ppc_tlb_invalidate_one(CPUPPCState *env, target_ulong addr)
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
         /* tlbie invalidate TLBs for all segments */
         /* XXX: given the fact that there are too many segments to invalidate,
          *      and we still don't have a tlb_flush_mask(env, n, mask) in QEMU,
@@ -2023,9 +1989,8 @@ void ppc_tlb_invalidate_one(CPUPPCState *env, target_ulong addr)
         break;
 #endif /* defined(TARGET_PPC64) */
     default:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "Unknown MMU model\n");
-        break;
+        /* Should never reach here with other MMU models */
+        assert(0);
     }
 #else
     ppc_tlb_invalidate_all(env);
@@ -2041,15 +2006,14 @@ void ppc_store_sdr1(CPUPPCState *env, target_ulong value)
     env->spr[SPR_SDR1] = value;
 #if defined(TARGET_PPC64)
     if (env->mmu_model & POWERPC_MMU_64) {
-        target_ulong htabsize = value & SDR_64_HTABSIZE;
+        PowerPCCPU *cpu = ppc_env_get_cpu(env);
+        Error *local_err = NULL;
 
-        if (htabsize > 28) {
-            fprintf(stderr, "Invalid HTABSIZE 0x" TARGET_FMT_lx
-                    " stored in SDR1\n", htabsize);
-            htabsize = 28;
+        ppc_hash64_set_sdr1(cpu, value, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+            error_free(local_err);
         }
-        env->htab_mask = (1ULL << (htabsize + 18 - 7)) - 1;
-        env->htab_base = value & SDR_64_HTABORG;
     } else
 #endif /* defined(TARGET_PPC64) */
     {
@@ -2080,21 +2044,17 @@ void helper_store_sr(CPUPPCState *env, target_ulong srnum, target_ulong value)
             (int)srnum, value, env->sr[srnum]);
 #if defined(TARGET_PPC64)
     if (env->mmu_model & POWERPC_MMU_64) {
-        uint64_t rb = 0, rs = 0;
+        uint64_t esid, vsid;
 
         /* ESID = srnum */
-        rb |= ((uint32_t)srnum & 0xf) << 28;
-        /* Set the valid bit */
-        rb |= SLB_ESID_V;
-        /* Index = ESID */
-        rb |= (uint32_t)srnum;
+        esid = ((uint64_t)(srnum & 0xf) << 28) | SLB_ESID_V;
 
         /* VSID = VSID */
-        rs |= (value & 0xfffffff) << 12;
+        vsid = (value & 0xfffffff) << 12;
         /* flags = flags */
-        rs |= ((value >> 27) & 0xf) << 8;
+        vsid |= ((value >> 27) & 0xf) << 8;
 
-        ppc_store_slb(env, rb, rs);
+        ppc_store_slb(cpu, srnum, esid, vsid);
     } else
 #endif
     if (env->sr[srnum] != value) {
@@ -2128,6 +2088,16 @@ void helper_tlbie(CPUPPCState *env, target_ulong addr)
     ppc_tlb_invalidate_one(env, addr);
 }
 
+void helper_tlbiva(CPUPPCState *env, target_ulong addr)
+{
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    /* tlbiva instruction only exists on BookE */
+    assert(env->mmu_model == POWERPC_MMU_BOOKE);
+    /* XXX: TODO */
+    cpu_abort(CPU(cpu), "BookE MMU model is not implemented\n");
+}
+
 /* Software driven TLBs management */
 /* PowerPC 602/603 software TLB load instructions helpers */
 static void do_6xx_tlb(CPUPPCState *env, target_ulong new_EPN, int is_code)
diff --git a/qemu/target-ppc/monitor.c b/qemu/target-ppc/monitor.c
new file mode 100644
index 000000000..c2d0806dd
--- /dev/null
+++ b/qemu/target-ppc/monitor.c
@@ -0,0 +1,147 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    unsigned int u;
+    int i;
+
+    u = 0;
+    for (i = 0; i < 8; i++)
+        u |= env->crf[i] << (32 - (4 * (i + 1)));
+
+    return u;
+}
+
+static target_long monitor_get_decr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_decr(env);
+}
+
+static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbu(env);
+}
+
+static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbl(env);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+const MonitorDef monitor_defs[] = {
+    { "fpscr", offsetof(CPUPPCState, fpscr) },
+    /* Next instruction pointer */
+    { "nip|pc", offsetof(CPUPPCState, nip) },
+    { "lr", offsetof(CPUPPCState, lr) },
+    { "ctr", offsetof(CPUPPCState, ctr) },
+    { "decr", 0, &monitor_get_decr, },
+    { "ccr|cr", 0, &monitor_get_ccr, },
+    /* Machine state register */
+    { "xer", offsetof(CPUPPCState, xer) },
+    { "msr", offsetof(CPUPPCState, msr) },
+    { "tbu", 0, &monitor_get_tbu, },
+    { "tbl", 0, &monitor_get_tbl, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+static int ppc_cpu_get_reg_num(const char *numstr, int maxnum, int *pregnum)
+{
+    int regnum;
+    char *endptr = NULL;
+
+    if (!*numstr) {
+        return false;
+    }
+
+    regnum = strtoul(numstr, &endptr, 10);
+    if (*endptr || (regnum >= maxnum)) {
+        return false;
+    }
+    *pregnum = regnum;
+
+    return true;
+}
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    int i, regnum;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    /* General purpose registers */
+    if ((tolower(name[0]) == 'r') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->gpr), &regnum)) {
+        *pval = env->gpr[regnum];
+        return 0;
+    }
+
+    /* Floating point registers */
+    if ((tolower(name[0]) == 'f') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->fpr), &regnum)) {
+        *pval = env->fpr[regnum];
+        return 0;
+    }
+
+    /* Special purpose registers */
+    for (i = 0; i < ARRAY_SIZE(env->spr_cb); ++i) {
+        ppc_spr_t *spr = &env->spr_cb[i];
+
+        if (spr->name && (strcasecmp(name, spr->name) == 0)) {
+            *pval = env->spr[i];
+            return 0;
+        }
+    }
+
+    /* Segment registers */
+#if !defined(CONFIG_USER_ONLY)
+    if ((strncasecmp(name, "sr", 2) == 0) &&
+        ppc_cpu_get_reg_num(name + 2, ARRAY_SIZE(env->sr), &regnum)) {
+        *pval = env->sr[regnum];
+        return 0;
+    }
+#endif
+
+    return -EINVAL;
+}
diff --git a/qemu/target-ppc/timebase_helper.c b/qemu/target-ppc/timebase_helper.c
index 865dcbed2..3b340d70d 100644
--- a/qemu/target-ppc/timebase_helper.c
+++ b/qemu/target-ppc/timebase_helper.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -130,13 +131,14 @@ target_ulong helper_load_dcr(CPUPPCState *env, target_ulong dcrn)
     uint32_t val = 0;
 
     if (unlikely(env->dcr_env == NULL)) {
-        qemu_log("No DCR environment\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "No DCR environment\n");
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL |
                                    POWERPC_EXCP_INVAL_INVAL);
     } else if (unlikely(ppc_dcr_read(env->dcr_env,
                                      (uint32_t)dcrn, &val) != 0)) {
-        qemu_log("DCR read error %d %03x\n", (uint32_t)dcrn, (uint32_t)dcrn);
+        qemu_log_mask(LOG_GUEST_ERROR, "DCR read error %d %03x\n",
+                      (uint32_t)dcrn, (uint32_t)dcrn);
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL | POWERPC_EXCP_PRIV_REG);
     }
@@ -146,13 +148,14 @@ target_ulong helper_load_dcr(CPUPPCState *env, target_ulong dcrn)
 void helper_store_dcr(CPUPPCState *env, target_ulong dcrn, target_ulong val)
 {
     if (unlikely(env->dcr_env == NULL)) {
-        qemu_log("No DCR environment\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "No DCR environment\n");
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL |
                                    POWERPC_EXCP_INVAL_INVAL);
     } else if (unlikely(ppc_dcr_write(env->dcr_env, (uint32_t)dcrn,
                                       (uint32_t)val) != 0)) {
-        qemu_log("DCR write error %d %03x\n", (uint32_t)dcrn, (uint32_t)dcrn);
+        qemu_log_mask(LOG_GUEST_ERROR, "DCR write error %d %03x\n",
+                      (uint32_t)dcrn, (uint32_t)dcrn);
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL | POWERPC_EXCP_PRIV_REG);
     }
diff --git a/qemu/target-ppc/translate.c b/qemu/target-ppc/translate.c
index 84c5cead6..b3860ecde 100644
--- a/qemu/target-ppc/translate.c
+++ b/qemu/target-ppc/translate.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -28,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define CPU_SINGLE_STEP 0x1
@@ -47,7 +49,7 @@
 /* Code translation helpers                                                  */
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static char cpu_reg_names[10*3 + 22*4 /* GPR */
     + 10*4 + 22*5 /* SPE GPRh */
     + 10*4 + 22*5 /* FPR */
@@ -91,7 +93,7 @@ void ppc_translate_init(void)
 
     for (i = 0; i < 8; i++) {
         snprintf(p, cpu_reg_names_size, "crf%d", i);
-        cpu_crf[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_crf[i] = tcg_global_mem_new_i32(cpu_env,
                                             offsetof(CPUPPCState, crf[i]), p);
         p += 5;
         cpu_reg_names_size -= 5;
@@ -99,28 +101,28 @@ void ppc_translate_init(void)
 
     for (i = 0; i < 32; i++) {
         snprintf(p, cpu_reg_names_size, "r%d", i);
-        cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr[i] = tcg_global_mem_new(cpu_env,
                                         offsetof(CPUPPCState, gpr[i]), p);
         p += (i < 10) ? 3 : 4;
         cpu_reg_names_size -= (i < 10) ? 3 : 4;
         snprintf(p, cpu_reg_names_size, "r%dH", i);
-        cpu_gprh[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gprh[i] = tcg_global_mem_new(cpu_env,
                                          offsetof(CPUPPCState, gprh[i]), p);
         p += (i < 10) ? 4 : 5;
         cpu_reg_names_size -= (i < 10) ? 4 : 5;
 
         snprintf(p, cpu_reg_names_size, "fp%d", i);
-        cpu_fpr[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fpr[i] = tcg_global_mem_new_i64(cpu_env,
                                             offsetof(CPUPPCState, fpr[i]), p);
         p += (i < 10) ? 4 : 5;
         cpu_reg_names_size -= (i < 10) ? 4 : 5;
 
         snprintf(p, cpu_reg_names_size, "avr%dH", i);
 #ifdef HOST_WORDS_BIGENDIAN
-        cpu_avrh[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrh[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[0]), p);
 #else
-        cpu_avrh[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrh[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[1]), p);
 #endif
         p += (i < 10) ? 6 : 7;
@@ -128,55 +130,55 @@ void ppc_translate_init(void)
 
         snprintf(p, cpu_reg_names_size, "avr%dL", i);
 #ifdef HOST_WORDS_BIGENDIAN
-        cpu_avrl[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrl[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[1]), p);
 #else
-        cpu_avrl[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrl[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[0]), p);
 #endif
         p += (i < 10) ? 6 : 7;
         cpu_reg_names_size -= (i < 10) ? 6 : 7;
         snprintf(p, cpu_reg_names_size, "vsr%d", i);
-        cpu_vsr[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                             offsetof(CPUPPCState, vsr[i]), p);
+        cpu_vsr[i] = tcg_global_mem_new_i64(cpu_env,
+                                            offsetof(CPUPPCState, vsr[i]), p);
         p += (i < 10) ? 5 : 6;
         cpu_reg_names_size -= (i < 10) ? 5 : 6;
     }
 
-    cpu_nip = tcg_global_mem_new(TCG_AREG0,
+    cpu_nip = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, nip), "nip");
 
-    cpu_msr = tcg_global_mem_new(TCG_AREG0,
+    cpu_msr = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, msr), "msr");
 
-    cpu_ctr = tcg_global_mem_new(TCG_AREG0,
+    cpu_ctr = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, ctr), "ctr");
 
-    cpu_lr = tcg_global_mem_new(TCG_AREG0,
+    cpu_lr = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, lr), "lr");
 
 #if defined(TARGET_PPC64)
-    cpu_cfar = tcg_global_mem_new(TCG_AREG0,
+    cpu_cfar = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUPPCState, cfar), "cfar");
 #endif
 
-    cpu_xer = tcg_global_mem_new(TCG_AREG0,
+    cpu_xer = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, xer), "xer");
-    cpu_so = tcg_global_mem_new(TCG_AREG0,
+    cpu_so = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, so), "SO");
-    cpu_ov = tcg_global_mem_new(TCG_AREG0,
+    cpu_ov = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, ov), "OV");
-    cpu_ca = tcg_global_mem_new(TCG_AREG0,
+    cpu_ca = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, ca), "CA");
 
-    cpu_reserve = tcg_global_mem_new(TCG_AREG0,
+    cpu_reserve = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUPPCState, reserve_addr),
                                      "reserve_addr");
 
-    cpu_fpscr = tcg_global_mem_new(TCG_AREG0,
+    cpu_fpscr = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUPPCState, fpscr), "fpscr");
 
-    cpu_access_type = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_access_type = tcg_global_mem_new_i32(cpu_env,
                                              offsetof(CPUPPCState, access_type), "access_type");
 
     done_init = 1;
@@ -2500,18 +2502,31 @@ static void gen_fmrgow(DisasContext *ctx)
 static void gen_mcrfs(DisasContext *ctx)
 {
     TCGv tmp = tcg_temp_new();
+    TCGv_i32 tmask;
+    TCGv_i64 tnew_fpscr = tcg_temp_new_i64();
     int bfa;
+    int nibble;
+    int shift;
 
     if (unlikely(!ctx->fpu_enabled)) {
         gen_exception(ctx, POWERPC_EXCP_FPU);
         return;
     }
-    bfa = 4 * (7 - crfS(ctx->opcode));
-    tcg_gen_shri_tl(tmp, cpu_fpscr, bfa);
+    bfa = crfS(ctx->opcode);
+    nibble = 7 - bfa;
+    shift = 4 * nibble;
+    tcg_gen_shri_tl(tmp, cpu_fpscr, shift);
     tcg_gen_trunc_tl_i32(cpu_crf[crfD(ctx->opcode)], tmp);
-    tcg_temp_free(tmp);
     tcg_gen_andi_i32(cpu_crf[crfD(ctx->opcode)], cpu_crf[crfD(ctx->opcode)], 0xf);
-    tcg_gen_andi_tl(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
+    tcg_temp_free(tmp);
+    tcg_gen_extu_tl_i64(tnew_fpscr, cpu_fpscr);
+    /* Only the exception bits (including FX) should be cleared if read */
+    tcg_gen_andi_i64(tnew_fpscr, tnew_fpscr, ~((0xF << shift) & FP_EX_CLEAR_BITS));
+    /* FEX and VX need to be updated, so don't set fpscr directly */
+    tmask = tcg_const_i32(1 << nibble);
+    gen_helper_store_fpscr(cpu_env, tnew_fpscr, tmask);
+    tcg_temp_free_i32(tmask);
+    tcg_temp_free_i64(tnew_fpscr);
 }
 
 /* mffs */
@@ -3212,10 +3227,8 @@ static void gen_lswi(DisasContext *ctx)
 
     if (nb == 0)
         nb = 32;
-    nr = nb / 4;
-    if (unlikely(((start + nr) > 32  &&
-                  start <= ra && (start + nr - 32) > ra) ||
-                 ((start + nr) <= 32 && start <= ra && (start + nr) > ra))) {
+    nr = (nb + 3) / 4;
+    if (unlikely(lsw_reg_in_range(start, nr, ra))) {
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_LSWX);
         return;
     }
@@ -4267,14 +4280,17 @@ static inline void gen_op_mfspr(DisasContext *ctx)
     void (*read_cb)(DisasContext *ctx, int gprn, int sprn);
     uint32_t sprn = SPR(ctx->opcode);
 
-#if !defined(CONFIG_USER_ONLY)
-    if (ctx->hv)
+#if defined(CONFIG_USER_ONLY)
+    read_cb = ctx->spr_cb[sprn].uea_read;
+#else
+    if (ctx->pr) {
+        read_cb = ctx->spr_cb[sprn].uea_read;
+    } else if (ctx->hv) {
         read_cb = ctx->spr_cb[sprn].hea_read;
-    else if (!ctx->pr)
+    } else {
         read_cb = ctx->spr_cb[sprn].oea_read;
-    else
+    }
 #endif
-        read_cb = ctx->spr_cb[sprn].uea_read;
     if (likely(read_cb != NULL)) {
         if (likely(read_cb != SPR_NOACCESS)) {
             (*read_cb)(ctx, rD(ctx->opcode), sprn);
@@ -4285,19 +4301,23 @@ static inline void gen_op_mfspr(DisasContext *ctx)
              * allowing userland application to read the PVR
              */
             if (sprn != SPR_PVR) {
-                qemu_log("Trying to read privileged spr %d (0x%03x) at "
-                         TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-                printf("Trying to read privileged spr %d (0x%03x) at "
-                       TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                fprintf(stderr, "Trying to read privileged spr %d (0x%03x) at "
+                        TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                if (qemu_log_separate()) {
+                    qemu_log("Trying to read privileged spr %d (0x%03x) at "
+                             TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                }
             }
             gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         }
     } else {
         /* Not defined */
-        qemu_log("Trying to read invalid spr %d (0x%03x) at "
-                 TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        printf("Trying to read invalid spr %d (0x%03x) at "
-               TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        fprintf(stderr, "Trying to read invalid spr %d (0x%03x) at "
+                TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        if (qemu_log_separate()) {
+            qemu_log("Trying to read invalid spr %d (0x%03x) at "
+                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        }
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_SPR);
     }
 }
@@ -4418,31 +4438,38 @@ static void gen_mtspr(DisasContext *ctx)
     void (*write_cb)(DisasContext *ctx, int sprn, int gprn);
     uint32_t sprn = SPR(ctx->opcode);
 
-#if !defined(CONFIG_USER_ONLY)
-    if (ctx->hv)
+#if defined(CONFIG_USER_ONLY)
+    write_cb = ctx->spr_cb[sprn].uea_write;
+#else
+    if (ctx->pr) {
+        write_cb = ctx->spr_cb[sprn].uea_write;
+    } else if (ctx->hv) {
         write_cb = ctx->spr_cb[sprn].hea_write;
-    else if (!ctx->pr)
+    } else {
         write_cb = ctx->spr_cb[sprn].oea_write;
-    else
+    }
 #endif
-        write_cb = ctx->spr_cb[sprn].uea_write;
     if (likely(write_cb != NULL)) {
         if (likely(write_cb != SPR_NOACCESS)) {
             (*write_cb)(ctx, sprn, rS(ctx->opcode));
         } else {
             /* Privilege exception */
-            qemu_log("Trying to write privileged spr %d (0x%03x) at "
-                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-            printf("Trying to write privileged spr %d (0x%03x) at "
-                   TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+            fprintf(stderr, "Trying to write privileged spr %d (0x%03x) at "
+                    TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+            if (qemu_log_separate()) {
+                qemu_log("Trying to write privileged spr %d (0x%03x) at "
+                         TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+            }
             gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         }
     } else {
         /* Not defined */
-        qemu_log("Trying to write invalid spr %d (0x%03x) at "
-                 TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        printf("Trying to write invalid spr %d (0x%03x) at "
-               TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        if (qemu_log_separate()) {
+            qemu_log("Trying to write invalid spr %d (0x%03x) at "
+                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        }
+        fprintf(stderr, "Trying to write invalid spr %d (0x%03x) at "
+                TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_SPR);
     }
 }
@@ -5896,7 +5923,7 @@ static void gen_tlbiva(DisasContext *ctx)
     }
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
-    gen_helper_tlbie(cpu_env, cpu_gpr[rB(ctx->opcode)]);
+    gen_helper_tlbiva(cpu_env, cpu_gpr[rB(ctx->opcode)]);
     tcg_temp_free(t0);
 #endif
 }
@@ -9884,7 +9911,7 @@ GEN_HANDLER(mtcrf, 0x1F, 0x10, 0x04, 0x00000801, PPC_MISC),
 GEN_HANDLER(mtmsrd, 0x1F, 0x12, 0x05, 0x001EF801, PPC_64B),
 #endif
 GEN_HANDLER(mtmsr, 0x1F, 0x12, 0x04, 0x001FF801, PPC_MISC),
-GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000001, PPC_MISC),
+GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000000, PPC_MISC),
 GEN_HANDLER(dcbf, 0x1F, 0x16, 0x02, 0x03C00001, PPC_CACHE),
 GEN_HANDLER(dcbi, 0x1F, 0x16, 0x0E, 0x03E00001, PPC_CACHE),
 GEN_HANDLER(dcbst, 0x1F, 0x16, 0x01, 0x03E00001, PPC_CACHE),
@@ -10670,6 +10697,13 @@ GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 1, opc3, 0, PPC_NONE, fl2), \
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 2, opc3, 0, PPC_NONE, fl2), \
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 3, opc3, 0, PPC_NONE, fl2)
 
+#undef GEN_XX2IFORM
+#define GEN_XX2IFORM(name, opc2, opc3, fl2)                           \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 0, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 1, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 2, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 3, opc3, 1, PPC_NONE, fl2)
+
 #undef GEN_XX3_RC_FORM
 #define GEN_XX3_RC_FORM(name, opc2, opc3, fl2)                          \
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 0x00, opc3 | 0x00, 0, PPC_NONE, fl2), \
@@ -10731,8 +10765,8 @@ GEN_XX3FORM(xsnmaddadp, 0x04, 0x14, PPC2_VSX),
 GEN_XX3FORM(xsnmaddmdp, 0x04, 0x15, PPC2_VSX),
 GEN_XX3FORM(xsnmsubadp, 0x04, 0x16, PPC2_VSX),
 GEN_XX3FORM(xsnmsubmdp, 0x04, 0x17, PPC2_VSX),
-GEN_XX2FORM(xscmpodp,  0x0C, 0x05, PPC2_VSX),
-GEN_XX2FORM(xscmpudp,  0x0C, 0x04, PPC2_VSX),
+GEN_XX2IFORM(xscmpodp,  0x0C, 0x05, PPC2_VSX),
+GEN_XX2IFORM(xscmpudp,  0x0C, 0x04, PPC2_VSX),
 GEN_XX3FORM(xsmaxdp, 0x00, 0x14, PPC2_VSX),
 GEN_XX3FORM(xsmindp, 0x00, 0x15, PPC2_VSX),
 GEN_XX2FORM(xscvdpsp, 0x12, 0x10, PPC2_VSX),
@@ -11320,9 +11354,11 @@ void ppc_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     case POWERPC_MMU_SOFT_74xx:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
 #endif
         cpu_fprintf(f, " SDR1 " TARGET_FMT_lx "   DAR " TARGET_FMT_lx
                        "  DSISR " TARGET_FMT_lx "\n", env->spr[SPR_SDR1],
@@ -11402,17 +11438,13 @@ void ppc_cpu_dump_statistics(CPUState *cs, FILE*f,
 }
 
 /*****************************************************************************/
-static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUPPCState *env, struct TranslationBlock *tb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUPPCState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     opc_handler_t **table, *handler;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
 
@@ -11469,36 +11501,34 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
 #endif
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
     tcg_clear_temp_count();
     /* Set env in case of segfault during code fetch */
     while (ctx.exception == POWERPC_EXCP_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.nip) {
-                    gen_debug_exception(ctxp);
-                    break;
-                }
-            }
-        }
-        if (unlikely(search_pc)) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.nip;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(ctx.nip);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.nip, BP_ANY))) {
+            gen_debug_exception(ctxp);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.nip += 4;
+            break;
         }
+
         LOG_DISAS("----------------\n");
         LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
                   ctx.nip, ctx.mem_idx, (int)msr_ir);
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         if (unlikely(need_byteswap(&ctx))) {
             ctx.opcode = bswap32(cpu_ldl_code(env, ctx.nip));
@@ -11508,12 +11538,8 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
                     opc3(ctx.opcode), ctx.le_mode ? "little" : "big");
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.nip);
-        }
         ctx.nip += 4;
         table = env->opcodes;
-        num_insns++;
         handler = table[opc1(ctx.opcode)];
         if (is_indirect_opcode(handler)) {
             table = ind_table(handler);
@@ -11525,12 +11551,10 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
         }
         /* Is opcode *REALLY* valid ? */
         if (unlikely(handler->handler == &gen_invalid)) {
-            if (qemu_log_enabled()) {
-                qemu_log("invalid/unsupported opcode: "
-                         "%02x - %02x - %02x (%08x) " TARGET_FMT_lx " %d\n",
-                         opc1(ctx.opcode), opc2(ctx.opcode),
-                         opc3(ctx.opcode), ctx.opcode, ctx.nip - 4, (int)msr_ir);
-            }
+            qemu_log_mask(LOG_GUEST_ERROR, "invalid/unsupported opcode: "
+                          "%02x - %02x - %02x (%08x) " TARGET_FMT_lx " %d\n",
+                          opc1(ctx.opcode), opc2(ctx.opcode),
+                          opc3(ctx.opcode), ctx.opcode, ctx.nip - 4, (int)msr_ir);
         } else {
             uint32_t inval;
 
@@ -11541,13 +11565,11 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
             }
 
             if (unlikely((ctx.opcode & inval) != 0)) {
-                if (qemu_log_enabled()) {
-                    qemu_log("invalid bits: %08x for opcode: "
-                             "%02x - %02x - %02x (%08x) " TARGET_FMT_lx "\n",
-                             ctx.opcode & inval, opc1(ctx.opcode),
-                             opc2(ctx.opcode), opc3(ctx.opcode),
-                             ctx.opcode, ctx.nip - 4);
-                }
+                qemu_log_mask(LOG_GUEST_ERROR, "invalid bits: %08x for opcode: "
+                              "%02x - %02x - %02x (%08x) " TARGET_FMT_lx "\n",
+                              ctx.opcode & inval, opc1(ctx.opcode),
+                              opc2(ctx.opcode), opc3(ctx.opcode),
+                              ctx.opcode, ctx.nip - 4);
                 gen_inval_exception(ctxp, POWERPC_EXCP_INVAL_INVAL);
                 break;
             }
@@ -11592,15 +11614,9 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
     }
     gen_tb_end(tb, num_insns);
 
-    if (unlikely(search_pc)) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.nip - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.nip - pc_start;
+    tb->icount = num_insns;
+
 #if defined(DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int flags;
@@ -11613,17 +11629,8 @@ static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUPPCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUPPCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->nip = tcg_ctx.gen_opc_pc[pc_pos];
+    env->nip = data[0];
 }
diff --git a/qemu/target-ppc/translate_init.c b/qemu/target-ppc/translate_init.c
index 16d7b16ac..f51572552 100644
--- a/qemu/target-ppc/translate_init.c
+++ b/qemu/target-ppc/translate_init.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 #include "exec/gdbstub.h"
 #include <sysemu/kvm.h>
@@ -305,7 +306,7 @@ static void spr_read_ibat (DisasContext *ctx, int gprn, int sprn)
 
 static void spr_read_ibat_h (DisasContext *ctx, int gprn, int sprn)
 {
-    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][(sprn - SPR_IBAT4U) / 2]));
+    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][((sprn - SPR_IBAT4U) / 2) + 4]));
 }
 
 static void spr_write_ibatu (DisasContext *ctx, int sprn, int gprn)
@@ -578,17 +579,33 @@ static inline void vscr_init (CPUPPCState *env, uint32_t val)
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
                          oea_read, oea_write, one_reg_id, initial_value)       \
     _spr_register(env, num, name, uea_read, uea_write, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
+    _spr_register(env, num, name, uea_read, uea_write, initial_value)
 #else
 #if !defined(CONFIG_KVM)
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
-                         oea_read, oea_write, one_reg_id, initial_value) \
+                         oea_read, oea_write, one_reg_id, initial_value)       \
+    _spr_register(env, num, name, uea_read, uea_write,                         \
+                  oea_read, oea_write, oea_read, oea_write, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
     _spr_register(env, num, name, uea_read, uea_write,                         \
-                  oea_read, oea_write, initial_value)
+                  oea_read, oea_write, hea_read, hea_write, initial_value)
 #else
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
-                         oea_read, oea_write, one_reg_id, initial_value) \
+                         oea_read, oea_write, one_reg_id, initial_value)       \
+    _spr_register(env, num, name, uea_read, uea_write,                         \
+                  oea_read, oea_write, oea_read, oea_write,                    \
+                  one_reg_id, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
     _spr_register(env, num, name, uea_read, uea_write,                         \
-                  oea_read, oea_write, one_reg_id, initial_value)
+                  oea_read, oea_write, hea_read, hea_write,                    \
+                  one_reg_id, initial_value)
 #endif
 #endif
 
@@ -597,6 +614,13 @@ static inline void vscr_init (CPUPPCState *env, uint32_t val)
     spr_register_kvm(env, num, name, uea_read, uea_write,                      \
                      oea_read, oea_write, 0, initial_value)
 
+#define spr_register_hv(env, num, name, uea_read, uea_write,                   \
+                        oea_read, oea_write, hea_read, hea_write,              \
+                        initial_value)                                         \
+    spr_register_kvm_hv(env, num, name, uea_read, uea_write,                   \
+                        oea_read, oea_write, hea_read, hea_write,              \
+                        0, initial_value)
+
 static inline void _spr_register(CPUPPCState *env, int num,
                                  const char *name,
                                  void (*uea_read)(DisasContext *ctx, int gprn, int sprn),
@@ -605,6 +629,8 @@ static inline void _spr_register(CPUPPCState *env, int num,
 
                                  void (*oea_read)(DisasContext *ctx, int gprn, int sprn),
                                  void (*oea_write)(DisasContext *ctx, int sprn, int gprn),
+                                 void (*hea_read)(DisasContext *opaque, int gprn, int sprn),
+                                 void (*hea_write)(DisasContext *opaque, int sprn, int gprn),
 #endif
 #if defined(CONFIG_KVM)
                                  uint64_t one_reg_id,
@@ -632,6 +658,8 @@ static inline void _spr_register(CPUPPCState *env, int num,
 #if !defined(CONFIG_USER_ONLY)
     spr->oea_read = oea_read;
     spr->oea_write = oea_write;
+    spr->hea_read = hea_read;
+    spr->hea_write = hea_write;
 #endif
 #if defined(CONFIG_KVM)
     spr->one_reg_id = one_reg_id,
@@ -1035,30 +1063,102 @@ static void gen_spr_7xx (CPUPPCState *env)
 
 #ifdef TARGET_PPC64
 #ifndef CONFIG_USER_ONLY
-static void spr_read_uamr (DisasContext *ctx, int gprn, int sprn)
+static void spr_write_amr(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_load_spr(cpu_gpr[gprn], SPR_AMR);
-    spr_load_dump_spr(SPR_AMR);
-}
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
 
-static void spr_write_uamr (DisasContext *ctx, int sprn, int gprn)
-{
-    gen_store_spr(SPR_AMR, cpu_gpr[gprn]);
+    /* Note, the HV=1 PR=0 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    if (ctx->pr) {
+        gen_load_spr(t1, SPR_UAMOR);
+    } else {
+        gen_load_spr(t1, SPR_AMOR);
+    }
+
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
+    gen_load_spr(t0, SPR_AMR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
+    gen_store_spr(SPR_AMR, t0);
     spr_store_dump_spr(SPR_AMR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
 }
 
-static void spr_write_uamr_pr (DisasContext *ctx, int sprn, int gprn)
+static void spr_write_uamor(DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+
+    /* Note, the HV=1 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    gen_load_spr(t1, SPR_AMOR);
 
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
     gen_load_spr(t0, SPR_UAMOR);
-    tcg_gen_and_tl(t0, t0, cpu_gpr[gprn]);
-    gen_store_spr(SPR_AMR, t0);
-    spr_store_dump_spr(SPR_AMR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
+    gen_store_spr(SPR_UAMOR, t0);
+    spr_store_dump_spr(SPR_UAMOR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
+}
+
+static void spr_write_iamr(DisasContext *ctx, int sprn, int gprn)
+{
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+
+    /* Note, the HV=1 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    gen_load_spr(t1, SPR_AMOR);
+
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
+    gen_load_spr(t0, SPR_IAMR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
+    gen_store_spr(SPR_IAMR, t0);
+    spr_store_dump_spr(SPR_IAMR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
 }
 #endif /* CONFIG_USER_ONLY */
 
-static void gen_spr_amr (CPUPPCState *env)
+static void gen_spr_amr(CPUPPCState *env, bool has_iamr)
 {
 #ifndef CONFIG_USER_ONLY
     /* Virtual Page Class Key protection */
@@ -1066,17 +1166,31 @@ static void gen_spr_amr (CPUPPCState *env)
      * userspace accessible, 29 is privileged.  So we only need to set
      * the kvm ONE_REG id on one of them, we use 29 */
     spr_register(env, SPR_UAMR, "UAMR",
-                 &spr_read_uamr, &spr_write_uamr_pr,
-                 &spr_read_uamr, &spr_write_uamr,
+                 &spr_read_generic, &spr_write_amr,
+                 &spr_read_generic, &spr_write_amr,
                  0);
-    spr_register_kvm(env, SPR_AMR, "AMR",
+    spr_register_kvm_hv(env, SPR_AMR, "AMR",
                      SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_amr,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_AMR, 0);
-    spr_register_kvm(env, SPR_UAMOR, "UAMOR",
+    spr_register_kvm_hv(env, SPR_UAMOR, "UAMOR",
                      SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_uamor,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_UAMOR, 0);
+    spr_register_hv(env, SPR_AMOR, "AMOR",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_generic, &spr_write_generic,
+                    0);
+    if (has_iamr) {
+        spr_register_kvm_hv(env, SPR_IAMR, "IAMR",
+                            SPR_NOACCESS, SPR_NOACCESS,
+                            &spr_read_generic, &spr_write_iamr,
+                            &spr_read_generic, &spr_write_generic,
+                            KVM_REG_PPC_IAMR, 0);
+    }
 #endif /* !CONFIG_USER_ONLY */
 }
 #endif /* TARGET_PPC64 */
@@ -7463,6 +7577,25 @@ static void gen_spr_book3s_dbg(CPUPPCState *env)
                      KVM_REG_PPC_DABRX, 0x00000000);
 }
 
+static void gen_spr_book3s_207_dbg(CPUPPCState *env)
+{
+    spr_register_kvm_hv(env, SPR_DAWR, "DAWR",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_DAWR, 0x00000000);
+    spr_register_kvm_hv(env, SPR_DAWRX, "DAWRX",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_DAWRX, 0x00000000);
+    spr_register_kvm_hv(env, SPR_CIABR, "CIABR",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_CIABR, 0x00000000);
+}
+
 static void gen_spr_970_dbg(CPUPPCState *env)
 {
     /* Breakpoints */
@@ -7602,6 +7735,30 @@ static void gen_spr_power8_pmu_sup(CPUPPCState *env)
                      SPR_NOACCESS, SPR_NOACCESS,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_MMCRS, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SIER, "SIER",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SIER, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SPMC1, "SPMC1",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SPMC1, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SPMC2, "SPMC2",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SPMC2, 0x00000000);
+    spr_register_kvm(env, SPR_TACR, "TACR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TACR, 0x00000000);
+    spr_register_kvm(env, SPR_TCSCR, "TCSCR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TCSCR, 0x00000000);
+    spr_register_kvm(env, SPR_CSIGR, "CSIGR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_CSIGR, 0x00000000);
 }
 
 static void gen_spr_power8_pmu_user(CPUPPCState *env)
@@ -7610,6 +7767,10 @@ static void gen_spr_power8_pmu_user(CPUPPCState *env)
                  &spr_read_ureg, SPR_NOACCESS,
                  &spr_read_ureg, &spr_write_ureg,
                  0x00000000);
+    spr_register(env, SPR_POWER_USIER, "USIER",
+                 &spr_read_generic, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
 }
 
 static void gen_spr_power5p_ear(CPUPPCState *env)
@@ -7713,10 +7874,10 @@ static void spr_write_tar(DisasContext *ctx, int sprn, int gprn)
 
 static void gen_spr_power8_tce_address_control(CPUPPCState *env)
 {
-    spr_register(env, SPR_TAR, "TAR",
-                 &spr_read_tar, &spr_write_tar,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
+    spr_register_kvm(env, SPR_TAR, "TAR",
+                     &spr_read_tar, &spr_write_tar,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TAR, 0x00000000);
 }
 
 static void spr_read_tm(DisasContext *ctx, int gprn, int sprn)
@@ -7841,6 +8002,44 @@ static void gen_spr_power8_fscr(CPUPPCState *env)
                      KVM_REG_PPC_FSCR, initval);
 }
 
+static void gen_spr_power8_pspb(CPUPPCState *env)
+{
+    spr_register_kvm(env, SPR_PSPB, "PSPB",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic32,
+                     KVM_REG_PPC_PSPB, 0);
+}
+
+static void gen_spr_power8_ic(CPUPPCState *env)
+{
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_hv(env, SPR_IC, "IC",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_generic, SPR_NOACCESS,
+                    &spr_read_generic, &spr_write_generic,
+                    0);
+#endif
+}
+
+static void gen_spr_power8_book4(CPUPPCState *env)
+{
+    /* Add a number of P8 book4 registers */
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_kvm(env, SPR_ACOP, "ACOP",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_ACOP, 0);
+    spr_register_kvm(env, SPR_BOOKS_PID, "PID",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_PID, 0);
+    spr_register_kvm(env, SPR_WORT, "WORT",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_WORT, 0);
+#endif
+}
+
 static void init_proc_book3s_64(CPUPPCState *env, int version)
 {
     gen_spr_ne_601(env);
@@ -7862,8 +8061,9 @@ static void init_proc_book3s_64(CPUPPCState *env, int version)
     case BOOK3S_CPU_POWER7:
     case BOOK3S_CPU_POWER8:
         gen_spr_book3s_ids(env);
-        gen_spr_amr(env);
+        gen_spr_amr(env, version >= BOOK3S_CPU_POWER8);
         gen_spr_book3s_purr(env);
+        env->ci_large_pages = true;
         break;
     default:
         g_assert_not_reached();
@@ -7890,10 +8090,15 @@ static void init_proc_book3s_64(CPUPPCState *env, int version)
         gen_spr_power8_pmu_sup(env);
         gen_spr_power8_pmu_user(env);
         gen_spr_power8_tm(env);
+        gen_spr_power8_pspb(env);
         gen_spr_vtb(env);
+        gen_spr_power8_ic(env);
+        gen_spr_power8_book4(env);
     }
     if (version < BOOK3S_CPU_POWER8) {
         gen_spr_book3s_dbg(env);
+    } else {
+        gen_spr_book3s_207_dbg(env);
     }
 #if !defined(CONFIG_USER_ONLY)
     switch (version) {
@@ -8019,7 +8224,7 @@ POWERPC_FAMILY(POWER5P)(ObjectClass *oc, void *data)
                     (1ull << MSR_DR) |
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI);
-    pcc->mmu_model = POWERPC_MMU_64B;
+    pcc->mmu_model = POWERPC_MMU_2_03;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
 #endif
@@ -8033,8 +8238,8 @@ POWERPC_FAMILY(POWER5P)(ObjectClass *oc, void *data)
     pcc->l1_icache_size = 0x10000;
 }
 
-static void powerpc_get_compat(Object *obj, Visitor *v,
-                               void *opaque, const char *name, Error **errp)
+static void powerpc_get_compat(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     char *value = (char *)"";
     Property *prop = opaque;
@@ -8058,18 +8263,18 @@ static void powerpc_get_compat(Object *obj, Visitor *v,
         break;
     }
 
-    visit_type_str(v, &value, name, errp);
+    visit_type_str(v, name, &value, errp);
 }
 
-static void powerpc_set_compat(Object *obj, Visitor *v,
-                               void *opaque, const char *name, Error **errp)
+static void powerpc_set_compat(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     Error *error = NULL;
     char *value = NULL;
     Property *prop = opaque;
     uint32_t *max_compat = qdev_get_prop_ptr(DEVICE(obj), prop);
 
-    visit_type_str(v, &value, name, &error);
+    visit_type_str(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -8103,6 +8308,36 @@ static Property powerpc_servercpu_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+#ifdef CONFIG_SOFTMMU
+static const struct ppc_segment_page_sizes POWER7_POWER8_sps = {
+    .sps = {
+        {
+            .page_shift = 12, /* 4K */
+            .slb_enc = 0,
+            .enc = { { .page_shift = 12, .pte_enc = 0 },
+                     { .page_shift = 16, .pte_enc = 0x7 },
+                     { .page_shift = 24, .pte_enc = 0x38 }, },
+        },
+        {
+            .page_shift = 16, /* 64K */
+            .slb_enc = SLB_VSID_64K,
+            .enc = { { .page_shift = 16, .pte_enc = 0x1 },
+                     { .page_shift = 24, .pte_enc = 0x8 }, },
+        },
+        {
+            .page_shift = 24, /* 16M */
+            .slb_enc = SLB_VSID_16M,
+            .enc = { { .page_shift = 24, .pte_enc = 0 }, },
+        },
+        {
+            .page_shift = 34, /* 16G */
+            .slb_enc = SLB_VSID_16G,
+            .enc = { { .page_shift = 34, .pte_enc = 0x3 }, },
+        },
+    }
+};
+#endif /* CONFIG_SOFTMMU */
+
 static void init_proc_POWER7 (CPUPPCState *env)
 {
     init_proc_book3s_64(env, BOOK3S_CPU_POWER7);
@@ -8166,6 +8401,7 @@ POWERPC_FAMILY(POWER7)(ObjectClass *oc, void *data)
     pcc->mmu_model = POWERPC_MMU_2_06;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
+    pcc->sps = &POWER7_POWER8_sps;
 #endif
     pcc->excp_model = POWERPC_EXCP_POWER7;
     pcc->bus_model = PPC_FLAGS_INPUT_POWER7;
@@ -8186,6 +8422,9 @@ static void init_proc_POWER8(CPUPPCState *env)
 
 static bool ppc_pvr_match_power8(PowerPCCPUClass *pcc, uint32_t pvr)
 {
+    if ((pvr & CPU_POWERPC_POWER_SERVER_MASK) == CPU_POWERPC_POWER8NVL_BASE) {
+        return true;
+    }
     if ((pvr & CPU_POWERPC_POWER_SERVER_MASK) == CPU_POWERPC_POWER8E_BASE) {
         return true;
     }
@@ -8243,11 +8482,12 @@ POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data)
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI) |
                     (1ull << MSR_LE);
-    pcc->mmu_model = POWERPC_MMU_2_06;
+    pcc->mmu_model = POWERPC_MMU_2_07;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
+    pcc->sps = &POWER7_POWER8_sps;
 #endif
-    pcc->excp_model = POWERPC_EXCP_POWER7;
+    pcc->excp_model = POWERPC_EXCP_POWER8;
     pcc->bus_model = PPC_FLAGS_INPUT_POWER7;
     pcc->bfd_mach = bfd_mach_ppc64;
     pcc->flags = POWERPC_FLAG_VRE | POWERPC_FLAG_SE |
@@ -8258,8 +8498,33 @@ POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data)
     pcc->l1_icache_size = 0x8000;
     pcc->interrupts_big_endian = ppc_cpu_interrupts_big_endian_lpcr;
 }
-#endif /* defined (TARGET_PPC64) */
 
+#if !defined(CONFIG_USER_ONLY)
+
+void cpu_ppc_set_papr(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    ppc_spr_t *amor = &env->spr_cb[SPR_AMOR];
+
+    /* PAPR always has exception vectors in RAM not ROM. To ensure this,
+     * MSR[IP] should never be set.
+     *
+     * We also disallow setting of MSR_HV
+     */
+    env->msr_mask &= ~((1ull << MSR_EP) | MSR_HVB);
+
+    /* Set a full AMOR so guest can use the AMR as it sees fit */
+    env->spr[SPR_AMOR] = amor->default_value = 0xffffffffffffffffull;
+
+    /* Tell KVM that we're in PAPR mode */
+    if (kvm_enabled()) {
+        kvmppc_set_papr(cpu);
+    }
+}
+
+#endif /* !defined(CONFIG_USER_ONLY) */
+
+#endif /* defined (TARGET_PPC64) */
 
 /*****************************************************************************/
 /* Generic CPU instantiation routine                                         */
@@ -8470,8 +8735,6 @@ static void dump_ppc_sprs (CPUPPCState *env)
 #endif
 
 /*****************************************************************************/
-#include <stdlib.h>
-#include <string.h>
 
 /* Opcode types */
 enum {
@@ -8577,11 +8840,7 @@ static int register_ind_insn (opc_handler_t **ppc_opcodes,
                               unsigned char idx1, unsigned char idx2,
                               opc_handler_t *handler)
 {
-    int ret;
-
-    ret = register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
-
-    return ret;
+    return register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
 }
 
 static int register_dblind_insn (opc_handler_t **ppc_opcodes,
@@ -8754,14 +9013,25 @@ static void dump_ppc_insns (CPUPPCState *env)
 }
 #endif
 
+static bool avr_need_swap(CPUPPCState *env)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    return msr_le;
+#else
+    return !msr_le;
+#endif
+}
+
 static int gdb_get_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
         stfq_p(mem_buf, env->fpr[n]);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         return 8;
     }
     if (n == 32) {
         stl_p(mem_buf, env->fpscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8770,10 +9040,12 @@ static int gdb_get_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 static int gdb_set_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         env->fpr[n] = ldfq_p(mem_buf);
         return 8;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         helper_store_fpscr(env, ldl_p(mem_buf), 0xffffffff);
         return 4;
     }
@@ -8783,21 +9055,25 @@ static int gdb_set_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 static int gdb_get_avr_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
-#ifdef HOST_WORDS_BIGENDIAN
-        stq_p(mem_buf, env->avr[n].u64[0]);
-        stq_p(mem_buf+8, env->avr[n].u64[1]);
-#else
-        stq_p(mem_buf, env->avr[n].u64[1]);
-        stq_p(mem_buf+8, env->avr[n].u64[0]);
-#endif
+        if (!avr_need_swap(env)) {
+            stq_p(mem_buf, env->avr[n].u64[0]);
+            stq_p(mem_buf+8, env->avr[n].u64[1]);
+        } else {
+            stq_p(mem_buf, env->avr[n].u64[1]);
+            stq_p(mem_buf+8, env->avr[n].u64[0]);
+        }
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        ppc_maybe_bswap_register(env, mem_buf + 8, 8);
         return 16;
     }
     if (n == 32) {
         stl_p(mem_buf, env->vscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     if (n == 33) {
         stl_p(mem_buf, (uint32_t)env->spr[SPR_VRSAVE]);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8806,20 +9082,24 @@ static int gdb_get_avr_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 static int gdb_set_avr_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
-#ifdef HOST_WORDS_BIGENDIAN
-        env->avr[n].u64[0] = ldq_p(mem_buf);
-        env->avr[n].u64[1] = ldq_p(mem_buf+8);
-#else
-        env->avr[n].u64[1] = ldq_p(mem_buf);
-        env->avr[n].u64[0] = ldq_p(mem_buf+8);
-#endif
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        ppc_maybe_bswap_register(env, mem_buf + 8, 8);
+        if (!avr_need_swap(env)) {
+            env->avr[n].u64[0] = ldq_p(mem_buf);
+            env->avr[n].u64[1] = ldq_p(mem_buf+8);
+        } else {
+            env->avr[n].u64[1] = ldq_p(mem_buf);
+            env->avr[n].u64[0] = ldq_p(mem_buf+8);
+        }
         return 16;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->vscr = ldl_p(mem_buf);
         return 4;
     }
     if (n == 33) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->spr[SPR_VRSAVE] = (target_ulong)ldl_p(mem_buf);
         return 4;
     }
@@ -8831,6 +9111,7 @@ static int gdb_get_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
     if (n < 32) {
 #if defined(TARGET_PPC64)
         stl_p(mem_buf, env->gpr[n] >> 32);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
 #else
         stl_p(mem_buf, env->gprh[n]);
 #endif
@@ -8838,10 +9119,12 @@ static int gdb_get_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
     }
     if (n == 32) {
         stq_p(mem_buf, env->spe_acc);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         return 8;
     }
     if (n == 33) {
         stl_p(mem_buf, env->spe_fscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8852,7 +9135,11 @@ static int gdb_set_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
     if (n < 32) {
 #if defined(TARGET_PPC64)
         target_ulong lo = (uint32_t)env->gpr[n];
-        target_ulong hi = (target_ulong)ldl_p(mem_buf) << 32;
+        target_ulong hi;
+
+        ppc_maybe_bswap_register(env, mem_buf, 4);
+
+        hi = (target_ulong)ldl_p(mem_buf) << 32;
         env->gpr[n] = lo | hi;
 #else
         env->gprh[n] = ldl_p(mem_buf);
@@ -8860,16 +9147,38 @@ static int gdb_set_spe_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
         return 4;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         env->spe_acc = ldq_p(mem_buf);
         return 8;
     }
     if (n == 33) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->spe_fscr = ldl_p(mem_buf);
         return 4;
     }
     return 0;
 }
 
+static int gdb_get_vsx_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
+{
+    if (n < 32) {
+        stq_p(mem_buf, env->vsr[n]);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        return 8;
+    }
+    return 0;
+}
+
+static int gdb_set_vsx_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
+{
+    if (n < 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        env->vsr[n] = ldq_p(mem_buf);
+        return 8;
+    }
+    return 0;
+}
+
 static int ppc_fixup_cpu(PowerPCCPU *cpu)
 {
     CPUPPCState *env = &cpu->env;
@@ -8975,6 +9284,10 @@ static void ppc_cpu_realizefn(DeviceState *dev, Error **errp)
         gdb_register_coprocessor(cs, gdb_get_spe_reg, gdb_set_spe_reg,
                                  34, "power-spe.xml", 0);
     }
+    if (pcc->insns_flags2 & PPC2_VSX) {
+        gdb_register_coprocessor(cs, gdb_get_vsx_reg, gdb_set_vsx_reg,
+                                 32, "power-vsx.xml", 0);
+    }
 
     qemu_init_vcpu(cs);
 
@@ -9189,7 +9502,7 @@ int ppc_get_compat_smt_threads(PowerPCCPU *cpu)
     return ret;
 }
 
-int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version)
+void ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version, Error **errp)
 {
     int ret = 0;
     CPUPPCState *env = &cpu->env;
@@ -9211,12 +9524,13 @@ int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version)
         break;
     }
 
-    if (kvm_enabled() && kvmppc_set_compat(cpu, cpu->cpu_version) < 0) {
-        error_report("Unable to set compatibility mode in KVM");
-        ret = -1;
+    if (kvm_enabled()) {
+        ret = kvmppc_set_compat(cpu, cpu->cpu_version);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret,
+                             "Unable to set CPU compatibility mode in KVM");
+        }
     }
-
-    return ret;
 }
 
 static gint ppc_cpu_compare_class_pvr(gconstpointer a, gconstpointer b)
@@ -9306,7 +9620,6 @@ static gint ppc_cpu_compare_class_name(gconstpointer a, gconstpointer b)
     return -1;
 }
 
-#include <ctype.h>
 
 static ObjectClass *ppc_cpu_class_by_name(const char *name);
 
@@ -9350,8 +9663,7 @@ static ObjectClass *ppc_cpu_class_by_name(const char *name)
                 break;
         }
         if (i == 8) {
-            ret = OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
-            return ret;
+            return OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
         }
     }
 
@@ -9582,7 +9894,7 @@ static void ppc_cpu_reset(CPUState *s)
 
 #if defined(TARGET_PPC64)
     if (env->mmu_model & POWERPC_MMU_64) {
-        env->msr |= (1ULL << MSR_SF);
+        msr |= (1ULL << MSR_SF);
     }
 #endif
 
@@ -9685,6 +9997,15 @@ static bool ppc_pvr_match_default(PowerPCCPUClass *pcc, uint32_t pvr)
     return pcc->pvr == pvr;
 }
 
+static gchar *ppc_gdb_arch_name(CPUState *cs)
+{
+#if defined(TARGET_PPC64)
+    return g_strdup("powerpc:common64");
+#else
+    return g_strdup("powerpc:common");
+#endif
+}
+
 static void ppc_cpu_class_init(ObjectClass *oc, void *data)
 {
     PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
@@ -9716,7 +10037,6 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
     cc->vmsd = &vmstate_ppc_cpu;
 #if defined(TARGET_PPC64)
     cc->write_elf64_note = ppc64_cpu_write_elf64_note;
-    cc->write_elf64_qemunote = ppc64_cpu_write_elf64_qemunote;
 #endif
 #endif
     cc->cpu_exec_enter = ppc_cpu_exec_enter;
@@ -9729,6 +10049,7 @@ static void ppc_cpu_class_init(ObjectClass *oc, void *data)
     cc->gdb_num_core_regs = 71 + 32;
 #endif
 
+    cc->gdb_arch_name = ppc_gdb_arch_name;
 #if defined(TARGET_PPC64)
     cc->gdb_core_xml_file = "power64-core.xml";
 #else
diff --git a/qemu/target-ppc/user_only_helper.c b/qemu/target-ppc/user_only_helper.c
index 829f66f50..6aff34713 100644
--- a/qemu/target-ppc/user_only_helper.c
+++ b/qemu/target-ppc/user_only_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 
 int ppc_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
diff --git a/qemu/target-s390x/arch_dump.c b/qemu/target-s390x/arch_dump.c
index dab63eb44..4731869f6 100644
--- a/qemu/target-s390x/arch_dump.c
+++ b/qemu/target-s390x/arch_dump.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "elf.h"
 #include "exec/cpu-all.h"
@@ -246,9 +247,3 @@ ssize_t cpu_get_note_size(int class, int machine, int nr_cpus)
 
     return (elf_note_size) * nr_cpus;
 }
-
-int s390_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                  CPUState *cpu, void *opaque)
-{
-    return 0;
-}
diff --git a/qemu/target-s390x/cc_helper.c b/qemu/target-s390x/cc_helper.c
index bfce3f1e6..0d9411bdf 100644
--- a/qemu/target-s390x/cc_helper.c
+++ b/qemu/target-s390x/cc_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -560,7 +561,7 @@ void HELPER(sacf)(CPUS390XState *env, uint64_t a1)
         env->psw.mask |= PSW_ASC_HOME;
         break;
     default:
-        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
+        HELPER_LOG("unknown sacf mode: %" PRIx64 "\n", a1);
         program_interrupt(env, PGM_SPECIFICATION, 2);
         break;
     }
diff --git a/qemu/target-s390x/cpu-qom.h b/qemu/target-s390x/cpu-qom.h
index 491c1b876..1c9093396 100644
--- a/qemu/target-s390x/cpu-qom.h
+++ b/qemu/target-s390x/cpu-qom.h
@@ -47,6 +47,8 @@ typedef struct S390CPUClass {
     CPUClass parent_class;
     /*< public >*/
 
+    int64_t next_cpu_id;
+
     DeviceRealize parent_realize;
     void (*parent_reset)(CPUState *cpu);
     void (*load_normal)(CPUState *cpu);
@@ -66,6 +68,7 @@ typedef struct S390CPU {
     /*< public >*/
 
     CPUS390XState env;
+    int64_t id;
     /* needed for live migration */
     void *irqstate;
     uint32_t irqstate_saved_size;
@@ -91,8 +94,6 @@ void s390_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
 int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
                               int cpuid, void *opaque);
 
-int s390_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                  CPUState *cpu, void *opaque);
 hwaddr s390_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 hwaddr s390_cpu_get_phys_addr_debug(CPUState *cpu, vaddr addr);
 int s390_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
diff --git a/qemu/target-s390x/cpu.c b/qemu/target-s390x/cpu.c
index c3e21b445..4bfff341d 100644
--- a/qemu/target-s390x/cpu.c
+++ b/qemu/target-s390x/cpu.c
@@ -23,14 +23,20 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "qemu/error-report.h"
 #include "hw/hw.h"
 #include "trace.h"
+#include "qapi/visitor.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/arch_init.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/sclp.h"
 #endif
 
 #define CR0_RESET       0xE0UL
@@ -184,11 +190,49 @@ static void s390_cpu_machine_reset_cb(void *opaque)
 }
 #endif
 
+static void s390_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_s390_64;
+    info->print_insn = print_insn_s390;
+}
+
 static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
     S390CPUClass *scc = S390_CPU_GET_CLASS(dev);
+    S390CPU *cpu = S390_CPU(dev);
+    CPUS390XState *env = &cpu->env;
+    Error *err = NULL;
+
+#if !defined(CONFIG_USER_ONLY)
+    if (cpu->id >= max_cpus) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", max allowed: %d", cpu->id, max_cpus - 1);
+        goto out;
+    }
+#endif
+    if (cpu_exists(cpu->id)) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", it already exists", cpu->id);
+        goto out;
+    }
+    if (cpu->id != scc->next_cpu_id) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", The next available id is %" PRIi64, cpu->id,
+                   scc->next_cpu_id);
+        goto out;
+    }
 
+    cpu_exec_init(cs, &err);
+    if (err != NULL) {
+        goto out;
+    }
+    scc->next_cpu_id++;
+
+#if !defined(CONFIG_USER_ONLY)
+    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
+#endif
+    env->cpu_num = cpu->id;
     s390_cpu_gdb_init(cs);
     qemu_init_vcpu(cs);
 #if !defined(CONFIG_USER_ONLY)
@@ -197,7 +241,55 @@ static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
     cpu_reset(cs);
 #endif
 
-    scc->parent_realize(dev, errp);
+    scc->parent_realize(dev, &err);
+
+#if !defined(CONFIG_USER_ONLY)
+    if (dev->hotplugged) {
+        raise_irq_cpu_hotplug();
+    }
+#endif
+
+out:
+    error_propagate(errp, err);
+}
+
+static void s390x_cpu_get_id(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
+{
+    S390CPU *cpu = S390_CPU(obj);
+    int64_t value = cpu->id;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void s390x_cpu_set_id(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
+{
+    S390CPU *cpu = S390_CPU(obj);
+    DeviceState *dev = DEVICE(obj);
+    const int64_t min = 0;
+    const int64_t max = UINT32_MAX;
+    Error *err = NULL;
+    int64_t value;
+
+    if (dev->realized) {
+        error_setg(errp, "Attempt to set property '%s' on '%s' after "
+                   "it was realized", name, object_get_typename(obj));
+        return;
+    }
+
+    visit_type_int(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    if (value < min || value > max) {
+        error_setg(errp, "Property %s.%s doesn't take value %" PRId64
+                   " (minimum: %" PRId64 ", maximum: %" PRId64 ")" ,
+                   object_get_typename(obj), name, value, min, max);
+        return;
+    }
+    cpu->id = value;
 }
 
 static void s390_cpu_initfn(Object *obj)
@@ -206,15 +298,16 @@ static void s390_cpu_initfn(Object *obj)
     S390CPU *cpu = S390_CPU(obj);
     CPUS390XState *env = &cpu->env;
     static bool inited;
-    static int cpu_num = 0;
 #if !defined(CONFIG_USER_ONLY)
     struct tm tm;
 #endif
 
     cs->env_ptr = env;
-    cpu_exec_init(cs, &error_abort);
+    cs->halted = 1;
+    cs->exception_index = EXCP_HLT;
+    object_property_add(OBJECT(cpu), "id", "int64_t", s390x_cpu_get_id,
+                        s390x_cpu_set_id, NULL, NULL, NULL);
 #if !defined(CONFIG_USER_ONLY)
-    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
     qemu_get_timedate(&tm, 0);
     env->tod_offset = TOD_UNIX_EPOCH +
                       (time2tod(mktimegm(&tm)) * 1000000000ULL);
@@ -223,7 +316,6 @@ static void s390_cpu_initfn(Object *obj)
     env->cpu_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, s390x_cpu_timer, cpu);
     s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
 #endif
-    env->cpu_num = cpu_num++;
 
     if (tcg_enabled() && !inited) {
         inited = true;
@@ -319,12 +411,18 @@ unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu)
 }
 #endif
 
+static gchar *s390_gdb_arch_name(CPUState *cs)
+{
+    return g_strdup("s390:64-bit");
+}
+
 static void s390_cpu_class_init(ObjectClass *oc, void *data)
 {
     S390CPUClass *scc = S390_CPU_CLASS(oc);
     CPUClass *cc = CPU_CLASS(scc);
     DeviceClass *dc = DEVICE_CLASS(oc);
 
+    scc->next_cpu_id = 0;
     scc->parent_realize = dc->realize;
     dc->realize = s390_cpu_realizefn;
 
@@ -347,12 +445,21 @@ static void s390_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = s390_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_s390_cpu;
     cc->write_elf64_note = s390_cpu_write_elf64_note;
-    cc->write_elf64_qemunote = s390_cpu_write_elf64_qemunote;
     cc->cpu_exec_interrupt = s390_cpu_exec_interrupt;
     cc->debug_excp_handler = s390x_cpu_debug_excp_handler;
 #endif
+    cc->disas_set_info = s390_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = S390_NUM_CORE_REGS;
     cc->gdb_core_xml_file = "s390x-core64.xml";
+    cc->gdb_arch_name = s390_gdb_arch_name;
+
+    /*
+     * Reason: s390_cpu_realizefn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo s390_cpu_type_info = {
diff --git a/qemu/target-s390x/cpu.h b/qemu/target-s390x/cpu.h
index 63aebf484..6d97c089a 100644
--- a/qemu/target-s390x/cpu.h
+++ b/qemu/target-s390x/cpu.h
@@ -22,12 +22,10 @@
 #ifndef CPU_S390X_H
 #define CPU_S390X_H
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 64
 
-#define ELF_MACHINE	EM_S390
 #define ELF_MACHINE_UNAME "S390X"
 
 #define CPUArchState struct CPUS390XState
@@ -43,6 +41,7 @@
 #include "fpu/softfloat.h"
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define MMU_MODE0_SUFFIX _primary
 #define MMU_MODE1_SUFFIX _secondary
@@ -143,8 +142,6 @@ typedef struct CPUS390XState {
     uint32_t cpu_num;
     uint32_t machine_type;
 
-    uint8_t *storage_keys;
-
     uint64_t tod_offset;
     uint64_t tod_basetime;
     QEMUTimer *tod_timer;
@@ -310,7 +307,7 @@ static inline CPU_DoubleU *get_freg(CPUS390XState *cs, int nr)
 #define MMU_SECONDARY_IDX       1
 #define MMU_HOME_IDX            2
 
-static inline int cpu_mmu_index (CPUS390XState *env)
+static inline int cpu_mmu_index (CPUS390XState *env, bool ifetch)
 {
     switch (env->psw.mask & PSW_MASK_ASC) {
     case PSW_ASC_PRIMARY:
@@ -416,6 +413,8 @@ void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilen);
 #endif
 
 S390CPU *cpu_s390x_init(const char *cpu_model);
+S390CPU *s390x_new_cpu(const char *cpu_model, int64_t id, Error **errp);
+S390CPU *cpu_s390x_create(const char *cpu_model, Error **errp);
 void s390x_translate_init(void);
 int cpu_s390x_exec(CPUState *cpu);
 
@@ -473,10 +472,8 @@ void s390x_tod_timer(void *opaque);
 void s390x_cpu_timer(void *opaque);
 
 int s390_virtio_hypercall(CPUS390XState *env);
-void s390_virtio_irq(int config_change, uint64_t token);
 
 #ifdef CONFIG_KVM
-void kvm_s390_virtio_irq(int config_change, uint64_t token);
 void kvm_s390_service_interrupt(uint32_t parm);
 void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
@@ -487,9 +484,6 @@ int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
 int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
 int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_clock);
 #else
-static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
-{
-}
 static inline void kvm_s390_service_interrupt(uint32_t parm)
 {
 }
@@ -547,9 +541,6 @@ int gtod_load(QEMUFile *f, void *opaque, int version_id);
 /* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm);
 
-/* from s390-virtio-bus */
-extern const hwaddr virtio_size;
-
 #else
 static inline unsigned int s390_cpu_halt(S390CPU *cpu)
 {
@@ -571,7 +562,7 @@ void cpu_unlock(void);
 typedef struct SubchDev SubchDev;
 
 #ifndef CONFIG_USER_ONLY
-extern void io_subsystem_reset(void);
+extern void subsystem_reset(void);
 SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
                          uint16_t schid);
 bool css_subch_visible(SubchDev *sch);
@@ -600,7 +591,6 @@ bool css_present(uint8_t cssid);
 
 #define cpu_init(model) CPU(cpu_s390x_init(model))
 #define cpu_exec cpu_s390x_exec
-#define cpu_gen_code cpu_s390x_gen_code
 #define cpu_signal_handler cpu_s390x_signal_handler
 
 void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
@@ -1163,12 +1153,13 @@ int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
 int kvm_s390_get_memslot_count(KVMState *s);
-void kvm_s390_clear_cmma_callback(void *opaque);
+void kvm_s390_cmma_reset(void);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_reset_vcpu(S390CPU *cpu);
 int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit);
 void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
 int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
+void kvm_s390_crypto_reset(void);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
@@ -1192,7 +1183,7 @@ static inline int kvm_s390_cpu_restart(S390CPU *cpu)
 {
     return -ENOSYS;
 }
-static inline void kvm_s390_clear_cmma_callback(void *opaque)
+static inline void kvm_s390_cmma_reset(void)
 {
 }
 static inline int kvm_s390_get_memslot_count(KVMState *s)
@@ -1218,6 +1209,9 @@ static inline int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
 {
     return 0;
 }
+static inline void kvm_s390_crypto_reset(void)
+{
+}
 #endif
 
 static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
@@ -1228,11 +1222,10 @@ static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
     return 0;
 }
 
-static inline void cmma_reset(S390CPU *cpu)
+static inline void s390_cmma_reset(void)
 {
     if (kvm_enabled()) {
-        CPUState *cs = CPU(cpu);
-        kvm_s390_clear_cmma_callback(cs->kvm_state);
+        kvm_s390_cmma_reset();
     }
 }
 
@@ -1264,6 +1257,13 @@ static inline int s390_assign_subch_ioeventfd(EventNotifier *notifier,
     return kvm_s390_assign_subch_ioeventfd(notifier, sch_id, vq, assign);
 }
 
+static inline void s390_crypto_reset(void)
+{
+    if (kvm_enabled()) {
+        kvm_s390_crypto_reset();
+    }
+}
+
 #ifdef CONFIG_KVM
 static inline bool vregs_needed(void *opaque)
 {
@@ -1278,4 +1278,49 @@ static inline bool vregs_needed(void *opaque)
     return 0;
 }
 #endif
+
+/* machine check interruption code */
+
+/* subclasses */
+#define MCIC_SC_SD 0x8000000000000000ULL
+#define MCIC_SC_PD 0x4000000000000000ULL
+#define MCIC_SC_SR 0x2000000000000000ULL
+#define MCIC_SC_CD 0x0800000000000000ULL
+#define MCIC_SC_ED 0x0400000000000000ULL
+#define MCIC_SC_DG 0x0100000000000000ULL
+#define MCIC_SC_W  0x0080000000000000ULL
+#define MCIC_SC_CP 0x0040000000000000ULL
+#define MCIC_SC_SP 0x0020000000000000ULL
+#define MCIC_SC_CK 0x0010000000000000ULL
+
+/* subclass modifiers */
+#define MCIC_SCM_B  0x0002000000000000ULL
+#define MCIC_SCM_DA 0x0000000020000000ULL
+#define MCIC_SCM_AP 0x0000000000080000ULL
+
+/* storage errors */
+#define MCIC_SE_SE 0x0000800000000000ULL
+#define MCIC_SE_SC 0x0000400000000000ULL
+#define MCIC_SE_KE 0x0000200000000000ULL
+#define MCIC_SE_DS 0x0000100000000000ULL
+#define MCIC_SE_IE 0x0000000080000000ULL
+
+/* validity bits */
+#define MCIC_VB_WP 0x0000080000000000ULL
+#define MCIC_VB_MS 0x0000040000000000ULL
+#define MCIC_VB_PM 0x0000020000000000ULL
+#define MCIC_VB_IA 0x0000010000000000ULL
+#define MCIC_VB_FA 0x0000008000000000ULL
+#define MCIC_VB_VR 0x0000004000000000ULL
+#define MCIC_VB_EC 0x0000002000000000ULL
+#define MCIC_VB_FP 0x0000001000000000ULL
+#define MCIC_VB_GR 0x0000000800000000ULL
+#define MCIC_VB_CR 0x0000000400000000ULL
+#define MCIC_VB_ST 0x0000000100000000ULL
+#define MCIC_VB_AR 0x0000000040000000ULL
+#define MCIC_VB_PR 0x0000000000200000ULL
+#define MCIC_VB_FC 0x0000000000100000ULL
+#define MCIC_VB_CT 0x0000000000020000ULL
+#define MCIC_VB_CC 0x0000000000010000ULL
+
 #endif
diff --git a/qemu/target-s390x/fpu_helper.c b/qemu/target-s390x/fpu_helper.c
index 45b7ddfbe..1c7f67354 100644
--- a/qemu/target-s390x/fpu_helper.c
+++ b/qemu/target-s390x/fpu_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu_ldst.h"
 #include "exec/helper-proto.h"
diff --git a/qemu/target-s390x/gdbstub.c b/qemu/target-s390x/gdbstub.c
index 31f204964..9fc36cb54 100644
--- a/qemu/target-s390x/gdbstub.c
+++ b/qemu/target-s390x/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "qemu/bitops.h"
@@ -174,6 +174,115 @@ static int cpu_write_vreg(CPUS390XState *env, uint8_t *mem_buf, int n)
     }
 }
 
+/* the values represent the positions in s390-cr.xml */
+#define S390_C0_REGNUM 0
+#define S390_C15_REGNUM 15
+/* total number of registers in s390-cr.xml */
+#define S390_NUM_C_REGS 16
+
+#ifndef CONFIG_USER_ONLY
+static int cpu_read_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        return gdb_get_regl(mem_buf, env->cregs[n]);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        env->cregs[n] = ldtul_p(mem_buf);
+        if (tcg_enabled()) {
+            tlb_flush(ENV_GET_CPU(env), 1);
+        }
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+
+/* the values represent the positions in s390-virt.xml */
+#define S390_VIRT_CKC_REGNUM    0
+#define S390_VIRT_CPUTM_REGNUM  1
+#define S390_VIRT_BEA_REGNUM    2
+#define S390_VIRT_PREFIX_REGNUM 3
+#define S390_VIRT_PP_REGNUM     4
+#define S390_VIRT_PFT_REGNUM    5
+#define S390_VIRT_PFS_REGNUM    6
+#define S390_VIRT_PFC_REGNUM    7
+/* total number of registers in s390-virt.xml */
+#define S390_NUM_VIRT_REGS 8
+
+static int cpu_read_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        return gdb_get_regl(mem_buf, env->ckc);
+    case S390_VIRT_CPUTM_REGNUM:
+        return gdb_get_regl(mem_buf, env->cputm);
+    case S390_VIRT_BEA_REGNUM:
+        return gdb_get_regl(mem_buf, env->gbea);
+    case S390_VIRT_PREFIX_REGNUM:
+        return gdb_get_regl(mem_buf, env->psa);
+    case S390_VIRT_PP_REGNUM:
+        return gdb_get_regl(mem_buf, env->pp);
+    case S390_VIRT_PFT_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_token);
+    case S390_VIRT_PFS_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_select);
+    case S390_VIRT_PFC_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_compare);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        env->ckc = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_CPUTM_REGNUM:
+        env->cputm = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_BEA_REGNUM:
+        env->gbea = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PREFIX_REGNUM:
+        env->psa = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PP_REGNUM:
+        env->pp = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFT_REGNUM:
+        env->pfault_token = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFS_REGNUM:
+        env->pfault_select = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFC_REGNUM:
+        env->pfault_compare = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+#endif
+
 void s390_cpu_gdb_init(CPUState *cs)
 {
     gdb_register_coprocessor(cs, cpu_read_ac_reg,
@@ -187,4 +296,16 @@ void s390_cpu_gdb_init(CPUState *cs)
     gdb_register_coprocessor(cs, cpu_read_vreg,
                              cpu_write_vreg,
                              S390_NUM_VREGS, "s390-vx.xml", 0);
+
+#ifndef CONFIG_USER_ONLY
+    gdb_register_coprocessor(cs, cpu_read_c_reg,
+                             cpu_write_c_reg,
+                             S390_NUM_C_REGS, "s390-cr.xml", 0);
+
+    if (kvm_enabled()) {
+        gdb_register_coprocessor(cs, cpu_read_virt_reg,
+                                 cpu_write_virt_reg,
+                                 S390_NUM_VIRT_REGS, "s390-virt.xml", 0);
+    }
+#endif
 }
diff --git a/qemu/target-s390x/helper.c b/qemu/target-s390x/helper.c
index d88700695..92abe7e67 100644
--- a/qemu/target-s390x/helper.c
+++ b/qemu/target-s390x/helper.c
@@ -18,6 +18,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "qemu/timer.h"
@@ -33,7 +35,7 @@
 #ifdef DEBUG_S390_STDOUT
 #define DPRINTF(fmt, ...) \
     do { fprintf(stderr, fmt, ## __VA_ARGS__); \
-         qemu_log(fmt, ##__VA_ARGS__); } while (0)
+         if (qemu_log_separate()) qemu_log(fmt, ##__VA_ARGS__); } while (0)
 #else
 #define DPRINTF(fmt, ...) \
     do { qemu_log(fmt, ## __VA_ARGS__); } while (0)
@@ -64,14 +66,51 @@ void s390x_cpu_timer(void *opaque)
 }
 #endif
 
-S390CPU *cpu_s390x_init(const char *cpu_model)
+S390CPU *cpu_s390x_create(const char *cpu_model, Error **errp)
 {
     S390CPU *cpu;
 
     cpu = S390_CPU(object_new(TYPE_S390_CPU));
 
-    object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
+    return cpu;
+}
+
+S390CPU *s390x_new_cpu(const char *cpu_model, int64_t id, Error **errp)
+{
+    S390CPU *cpu;
+    Error *err = NULL;
 
+    cpu = cpu_s390x_create(cpu_model, &err);
+    if (err != NULL) {
+        goto out;
+    }
+
+    object_property_set_int(OBJECT(cpu), id, "id", &err);
+    if (err != NULL) {
+        goto out;
+    }
+    object_property_set_bool(OBJECT(cpu), true, "realized", &err);
+
+out:
+    if (err) {
+        error_propagate(errp, err);
+        object_unref(OBJECT(cpu));
+        cpu = NULL;
+    }
+    return cpu;
+}
+
+S390CPU *cpu_s390x_init(const char *cpu_model)
+{
+    Error *err = NULL;
+    S390CPU *cpu;
+    /* Use to track CPU ID for linux-user only */
+    static int64_t next_cpu_id;
+
+    cpu = s390x_new_cpu(cpu_model, next_cpu_id++, &err);
+    if (err) {
+        error_report_err(err);
+    }
     return cpu;
 }
 
@@ -133,7 +172,7 @@ int s390_cpu_handle_mmu_fault(CPUState *cs, vaddr orig_vaddr,
     }
 
     /* check out of RAM access */
-    if (raddr > (ram_size + virtio_size)) {
+    if (raddr > ram_size) {
         DPRINTF("%s: raddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
                 (uint64_t)raddr, (uint64_t)ram_size);
         trigger_pgm_exception(env, PGM_ADDRESSING, ILEN_LATER);
@@ -162,8 +201,9 @@ hwaddr s390_cpu_get_phys_page_debug(CPUState *cs, vaddr vaddr)
         vaddr &= 0x7fffffff;
     }
 
-    mmu_translate(env, vaddr, MMU_INST_FETCH, asc, &raddr, &prot, false);
-
+    if (mmu_translate(env, vaddr, MMU_INST_FETCH, asc, &raddr, &prot, false)) {
+        return -1;
+    }
     return raddr;
 }
 
diff --git a/qemu/target-s390x/int_helper.c b/qemu/target-s390x/int_helper.c
index a46c736d6..cc1071eea 100644
--- a/qemu/target-s390x/int_helper.c
+++ b/qemu/target-s390x/int_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
diff --git a/qemu/target-s390x/interrupt.c b/qemu/target-s390x/interrupt.c
index 1404d0afd..bad60a7e1 100644
--- a/qemu/target-s390x/interrupt.c
+++ b/qemu/target-s390x/interrupt.c
@@ -7,6 +7,7 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
 
@@ -27,17 +28,6 @@ void s390_sclp_extint(uint32_t parm)
     }
 }
 
-void s390_virtio_irq(int config_change, uint64_t token)
-{
-    if (kvm_enabled()) {
-        kvm_s390_virtio_irq(config_change, token);
-    } else {
-        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
-
-        cpu_inject_ext(dummy_cpu, EXT_VIRTIO, config_change, token);
-    }
-}
-
 void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
                        uint32_t io_int_parm, uint32_t io_int_word)
 {
diff --git a/qemu/target-s390x/ioinst.c b/qemu/target-s390x/ioinst.c
index 77f2a1fb9..142ff9384 100644
--- a/qemu/target-s390x/ioinst.c
+++ b/qemu/target-s390x/ioinst.c
@@ -9,7 +9,7 @@
  * directory.
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "ioinst.h"
@@ -614,9 +614,11 @@ static void ioinst_handle_chsc_sei(ChscReq *req, ChscResp *res)
             (*res_flags) |= 0x80;
         } else {
             (*res_flags) &= ~0x80;
+            css_clear_sei_pending();
         }
     } else {
-        res->code = cpu_to_be16(0x0004);
+        res->code = cpu_to_be16(0x0005);
+        res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
     }
 }
 
diff --git a/qemu/target-s390x/kvm.c b/qemu/target-s390x/kvm.c
index ae3a0affe..e1859cae0 100644
--- a/qemu/target-s390x/kvm.c
+++ b/qemu/target-s390x/kvm.c
@@ -21,7 +21,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/mman.h>
 
@@ -173,16 +173,15 @@ int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit)
     return kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
 }
 
-void kvm_s390_clear_cmma_callback(void *opaque)
+void kvm_s390_cmma_reset(void)
 {
     int rc;
-    KVMState *s = opaque;
     struct kvm_device_attr attr = {
         .group = KVM_S390_VM_MEM_CTRL,
         .attr = KVM_S390_VM_MEM_CLR_CMMA,
     };
 
-    rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+    rc = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
     trace_kvm_clear_cmma(rc);
 }
 
@@ -200,9 +199,6 @@ static void kvm_s390_enable_cmma(KVMState *s)
     }
 
     rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
-    if (!rc) {
-        qemu_register_reset(kvm_s390_clear_cmma_callback, s);
-    }
     trace_kvm_enable_cmma(rc);
 }
 
@@ -249,7 +245,7 @@ static void kvm_s390_init_dea_kw(void)
     }
 }
 
-static void kvm_s390_init_crypto(void)
+void kvm_s390_crypto_reset(void)
 {
     kvm_s390_init_aes_kw();
     kvm_s390_init_dea_kw();
@@ -262,7 +258,9 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
     cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
     cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
 
-    kvm_s390_enable_cmma(s);
+    if (!mem_path) {
+        kvm_s390_enable_cmma(s);
+    }
 
     if (!kvm_check_extension(s, KVM_CAP_S390_GMAP)
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
@@ -301,8 +299,6 @@ void kvm_s390_reset_vcpu(S390CPU *cpu)
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
         error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
     }
-
-    kvm_s390_init_crypto();
 }
 
 static int can_sync_regs(CPUState *cs, int regs)
@@ -346,6 +342,12 @@ int kvm_arch_put_registers(CPUState *cs, int level)
         }
         cs->kvm_run->s.regs.fpc = env->fpc;
         cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_VRS;
+    } else if (can_sync_regs(cs, KVM_SYNC_FPRS)) {
+        for (i = 0; i < 16; i++) {
+            cs->kvm_run->s.regs.fprs[i] = get_freg(env, i)->ll;
+        }
+        cs->kvm_run->s.regs.fpc = env->fpc;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_FPRS;
     } else {
         /* Floating point */
         for (i = 0; i < 16; i++) {
@@ -486,6 +488,11 @@ int kvm_arch_get_registers(CPUState *cs)
             env->vregs[i][1].ll = cs->kvm_run->s.regs.vrs[i][1];
         }
         env->fpc = cs->kvm_run->s.regs.fpc;
+    } else if (can_sync_regs(cs, KVM_SYNC_FPRS)) {
+        for (i = 0; i < 16; i++) {
+            get_freg(env, i)->ll = cs->kvm_run->s.regs.fprs[i];
+        }
+        env->fpc = cs->kvm_run->s.regs.fpc;
     } else {
         r = kvm_vcpu_ioctl(cs, KVM_GET_FPU, &fpu);
         if (r < 0) {
@@ -588,9 +595,9 @@ int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
  * @addr:      the logical start address in guest memory
  * @ar:        the access register number
  * @hostbuf:   buffer in host memory. NULL = do only checks w/o copying
- * @len:       length that should be transfered
+ * @len:       length that should be transferred
  * @is_write:  true = write, false = read
- * Returns:    0 on success, non-zero if an exception or error occured
+ * Returns:    0 on success, non-zero if an exception or error occurred
  *
  * Use KVM ioctl to read/write from/to guest memory. An access exception
  * is injected into the vCPU in case of translation errors.
@@ -929,17 +936,6 @@ void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
     __kvm_s390_floating_interrupt(irq);
 }
 
-void kvm_s390_virtio_irq(int config_change, uint64_t token)
-{
-    struct kvm_s390_irq irq = {
-        .type = KVM_S390_INT_VIRTIO,
-        .u.ext.ext_params = config_change,
-        .u.ext.ext_params2 = token,
-    };
-
-    kvm_s390_floating_interrupt(&irq);
-}
-
 void kvm_s390_service_interrupt(uint32_t parm)
 {
     struct kvm_s390_irq irq = {
@@ -1437,7 +1433,7 @@ static int kvm_s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
         cpu_physical_memory_write(offsetof(LowCore, ar_access_id), &ar_id, 1);
     }
     for (i = 0; i < 16; ++i) {
-        *((uint64 *)mem + i) = get_freg(&cpu->env, i)->ll;
+        *((uint64_t *)mem + i) = get_freg(&cpu->env, i)->ll;
     }
     memcpy(mem + 128, &cpu->env.regs, 128);
     memcpy(mem + 256, &cpu->env.psw, 16);
@@ -1796,13 +1792,6 @@ static bool is_special_wait_psw(CPUState *cs)
     return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
-static void guest_panicked(void)
-{
-    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE,
-                                   &error_abort);
-    vm_stop(RUN_STATE_GUEST_PANICKED);
-}
-
 static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
 {
     CPUState *cs = CPU(cpu);
@@ -1811,7 +1800,7 @@ static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
                  str, cs->cpu_index, ldq_phys(cs->as, cpu->env.psa + pswoffset),
                  ldq_phys(cs->as, cpu->env.psa + pswoffset + 8));
     s390_cpu_halt(cpu);
-    guest_panicked();
+    qemu_system_guest_panicked();
 }
 
 static int handle_intercept(S390CPU *cpu)
@@ -1844,7 +1833,7 @@ static int handle_intercept(S390CPU *cpu)
                 if (is_special_wait_psw(cs)) {
                     qemu_system_shutdown_request();
                 } else {
-                    guest_panicked();
+                    qemu_system_guest_panicked();
                 }
             }
             r = EXCP_HALTED;
@@ -2072,12 +2061,30 @@ void kvm_s390_io_interrupt(uint16_t subchannel_id,
     kvm_s390_floating_interrupt(&irq);
 }
 
+static uint64_t build_channel_report_mcic(void)
+{
+    uint64_t mcic;
+
+    /* subclass: indicate channel report pending */
+    mcic = MCIC_SC_CP |
+    /* subclass modifiers: none */
+    /* storage errors: none */
+    /* validity bits: no damage */
+        MCIC_VB_WP | MCIC_VB_MS | MCIC_VB_PM | MCIC_VB_IA | MCIC_VB_FP |
+        MCIC_VB_GR | MCIC_VB_CR | MCIC_VB_ST | MCIC_VB_AR | MCIC_VB_PR |
+        MCIC_VB_FC | MCIC_VB_CT | MCIC_VB_CC;
+    if (kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS)) {
+        mcic |= MCIC_VB_VR;
+    }
+    return mcic;
+}
+
 void kvm_s390_crw_mchk(void)
 {
     struct kvm_s390_irq irq = {
         .type = KVM_S390_MCHK,
         .u.mchk.cr14 = 1 << 28,
-        .u.mchk.mcic = 0x00400f1d40330000ULL,
+        .u.mchk.mcic = build_channel_report_mcic(),
     };
     kvm_s390_floating_interrupt(&irq);
 }
@@ -2215,7 +2222,7 @@ int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
 }
 
 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
-                              uint64_t address, uint32_t data)
+                             uint64_t address, uint32_t data, PCIDevice *dev)
 {
     S390PCIBusDevice *pbdev;
     uint32_t fid = data >> ZPCI_MSI_VEC_BITS;
diff --git a/qemu/target-s390x/machine.c b/qemu/target-s390x/machine.c
index b76fb0831..6b2609054 100644
--- a/qemu/target-s390x/machine.c
+++ b/qemu/target-s390x/machine.c
@@ -14,6 +14,7 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
@@ -48,7 +49,7 @@ static inline bool fpu_needed(void *opaque)
     return true;
 }
 
-const VMStateDescription vmstate_fpu = {
+static const VMStateDescription vmstate_fpu = {
     .name = "cpu/fpu",
     .version_id = 1,
     .minimum_version_id = 1,
@@ -75,7 +76,7 @@ const VMStateDescription vmstate_fpu = {
     }
 };
 
-const VMStateDescription vmstate_vregs = {
+static const VMStateDescription vmstate_vregs = {
     .name = "cpu/vregs",
     .version_id = 1,
     .minimum_version_id = 1,
diff --git a/qemu/target-s390x/mem_helper.c b/qemu/target-s390x/mem_helper.c
index 6f8bd796a..707862203 100644
--- a/qemu/target-s390x/mem_helper.c
+++ b/qemu/target-s390x/mem_helper.c
@@ -18,9 +18,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
+#include "hw/s390x/storage-keys.h"
 
 /*****************************************************************************/
 /* Softmmu support */
@@ -68,7 +70,7 @@ static inline uint64_t adj_len_to_page(uint64_t len, uint64_t addr)
 static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
                         uint32_t l)
 {
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
 
     while (l > 0) {
         void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
@@ -91,7 +93,7 @@ static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
 static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
                          uint32_t l)
 {
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
 
     while (l > 0) {
         void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
@@ -937,40 +939,73 @@ uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
 /* insert storage key extended */
 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return 0;
     }
 
-    return env->storage_keys[addr / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+    return key;
 }
 
 /* set storage key extended */
 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return;
     }
 
-    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    key = (uint8_t) r1;
+    skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
 }
 
 /* reset reference bit extended */
 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
 {
-    uint8_t re;
-    uint8_t key;
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
+    uint8_t re, key;
 
     if (r2 > ram_size) {
         return 0;
     }
 
-    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+
     re = key & (SK_R | SK_C);
-    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
+    key &= ~SK_R;
+
+    if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
 
     /*
      * cc
diff --git a/qemu/target-s390x/misc_helper.c b/qemu/target-s390x/misc_helper.c
index 8eac0e12b..71cbe34e0 100644
--- a/qemu/target-s390x/misc_helper.c
+++ b/qemu/target-s390x/misc_helper.c
@@ -18,11 +18,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/memory.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
-#include <string.h>
 #include "sysemu/kvm.h"
 #include "qemu/timer.h"
 #include "exec/address-spaces.h"
@@ -127,8 +127,9 @@ static int modified_clear_reset(S390CPU *cpu)
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_full_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
+    s390_crypto_reset();
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
     resume_all_vcpus();
@@ -145,8 +146,8 @@ static int load_normal_reset(S390CPU *cpu)
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
     scc->initial_cpu_reset(CPU(cpu));
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
@@ -233,11 +234,8 @@ void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3)
         }
         iplb = g_malloc0(sizeof(struct IplParameterBlock));
         cpu_physical_memory_read(addr, iplb, sizeof(struct IplParameterBlock));
-        if (!s390_ipl_update_diag308(iplb)) {
-            env->regs[r1 + 1] = DIAG_308_RC_OK;
-        } else {
-            env->regs[r1 + 1] = DIAG_308_RC_INVALID;
-        }
+        s390_ipl_update_diag308(iplb);
+        env->regs[r1 + 1] = DIAG_308_RC_OK;
         g_free(iplb);
         return;
     case 6:
@@ -301,7 +299,7 @@ void HELPER(spx)(CPUS390XState *env, uint64_t a1)
     uint32_t prefix = a1 & 0x7fffe000;
 
     env->psa = prefix;
-    qemu_log("prefix: %#x\n", prefix);
+    HELPER_LOG("prefix: %#x\n", prefix);
     tlb_flush_page(cs, 0);
     tlb_flush_page(cs, TARGET_PAGE_SIZE);
 }
diff --git a/qemu/target-s390x/mmu_helper.c b/qemu/target-s390x/mmu_helper.c
index 1ea6d812c..b11a02706 100644
--- a/qemu/target-s390x/mmu_helper.c
+++ b/qemu/target-s390x/mmu_helper.c
@@ -15,10 +15,13 @@
  * GNU General Public License for more details.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "exec/address-spaces.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
+#include "trace.h"
+#include "hw/s390x/storage-keys.h"
 
 /* #define DEBUG_S390 */
 /* #define DEBUG_S390_PTE */
@@ -28,7 +31,7 @@
 #ifdef DEBUG_S390_STDOUT
 #define DPRINTF(fmt, ...) \
     do { fprintf(stderr, fmt, ## __VA_ARGS__); \
-         qemu_log(fmt, ##__VA_ARGS__); } while (0)
+         if (qemu_log_separate()) qemu_log(fmt, ##__VA_ARGS__); } while (0)
 #else
 #define DPRINTF(fmt, ...) \
     do { qemu_log(fmt, ## __VA_ARGS__); } while (0)
@@ -87,7 +90,7 @@ static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
 
     tec = vaddr | (rw == MMU_DATA_STORE ? FS_WRITE : FS_READ) | asc >> 46;
 
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
+    DPRINTF("%s: trans_exc_code=%016" PRIx64 "\n", __func__, tec);
 
     if (!exc) {
         return;
@@ -309,8 +312,15 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
 int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
                   target_ulong *raddr, int *flags, bool exc)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     int r = -1;
-    uint8_t *sk;
+    uint8_t key;
+
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
 
     *flags = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
     vaddr &= TARGET_PAGE_MASK;
@@ -358,14 +368,23 @@ int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
     /* Convert real address -> absolute address */
     *raddr = mmu_real2abs(env, *raddr);
 
-    if (*raddr < ram_size) {
-        sk = &env->storage_keys[*raddr / TARGET_PAGE_SIZE];
+    if (r == 0 && *raddr < ram_size) {
+        if (skeyclass->get_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_get_skeys_nonzero(r);
+            return 0;
+        }
+
         if (*flags & PAGE_READ) {
-            *sk |= SK_R;
+            key |= SK_R;
         }
 
         if (*flags & PAGE_WRITE) {
-            *sk |= SK_C;
+            key |= SK_C;
+        }
+
+        if (skeyclass->set_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_set_skeys_nonzero(r);
+            return 0;
         }
     }
 
diff --git a/qemu/target-s390x/translate.c b/qemu/target-s390x/translate.c
index c748290d5..c871ef2bb 100644
--- a/qemu/target-s390x/translate.c
+++ b/qemu/target-s390x/translate.c
@@ -28,6 +28,7 @@
 #  define LOG_DISAS(...) do { } while (0)
 #endif
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -36,13 +37,14 @@
 #include "exec/cpu_ldst.h"
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 #include "exec/gen-icount.h"
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 /* Information that (most) every instruction needs to manipulate.  */
@@ -124,7 +126,7 @@ void s390_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
     for (i = 0; i < 32; i++) {
         cpu_fprintf(f, "V%02d=%016" PRIx64 "%016" PRIx64, i,
                     env->vregs[i][0].ll, env->vregs[i][1].ll);
-        cpu_fprintf(f, (i % 2) ? " " : "\n");
+        cpu_fprintf(f, (i % 2) ? "\n" : " ");
     }
 
 #ifndef CONFIG_USER_ONLY
@@ -161,42 +163,40 @@ static char cpu_reg_names[32][4];
 static TCGv_i64 regs[16];
 static TCGv_i64 fregs[16];
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 void s390x_translate_init(void)
 {
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    psw_addr = tcg_global_mem_new_i64(TCG_AREG0,
+    psw_addr = tcg_global_mem_new_i64(cpu_env,
                                       offsetof(CPUS390XState, psw.addr),
                                       "psw_addr");
-    psw_mask = tcg_global_mem_new_i64(TCG_AREG0,
+    psw_mask = tcg_global_mem_new_i64(cpu_env,
                                       offsetof(CPUS390XState, psw.mask),
                                       "psw_mask");
-    gbea = tcg_global_mem_new_i64(TCG_AREG0,
+    gbea = tcg_global_mem_new_i64(cpu_env,
                                   offsetof(CPUS390XState, gbea),
                                   "gbea");
 
-    cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUS390XState, cc_op),
+    cc_op = tcg_global_mem_new_i32(cpu_env, offsetof(CPUS390XState, cc_op),
                                    "cc_op");
-    cc_src = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_src),
+    cc_src = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_src),
                                     "cc_src");
-    cc_dst = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_dst),
+    cc_dst = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_dst),
                                     "cc_dst");
-    cc_vr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_vr),
+    cc_vr = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_vr),
                                    "cc_vr");
 
     for (i = 0; i < 16; i++) {
         snprintf(cpu_reg_names[i], sizeof(cpu_reg_names[0]), "r%d", i);
-        regs[i] = tcg_global_mem_new(TCG_AREG0,
+        regs[i] = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUS390XState, regs[i]),
                                      cpu_reg_names[i]);
     }
 
     for (i = 0; i < 16; i++) {
         snprintf(cpu_reg_names[i + 16], sizeof(cpu_reg_names[0]), "f%d", i);
-        fregs[i] = tcg_global_mem_new(TCG_AREG0,
+        fregs[i] = tcg_global_mem_new(cpu_env,
                                       offsetof(CPUS390XState, vregs[i][0].d),
                                       cpu_reg_names[i + 16]);
     }
@@ -811,7 +811,7 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
     case CC_OP_LTGT0_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_dst);
         c->u.s32.b = tcg_const_i32(0);
         break;
     case CC_OP_LTGT_32:
@@ -819,9 +819,9 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
     case CC_OP_SUBU_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_src);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_src);
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.b, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.b, cc_dst);
         break;
 
     case CC_OP_LTGT0_64:
@@ -851,11 +851,11 @@ static void disas_jcc(DisasContext *s, DisasCompare *c, uint32_t mask)
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_vr);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_vr);
         if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
             tcg_gen_movi_i32(c->u.s32.b, 0);
         } else {
-            tcg_gen_trunc_i64_i32(c->u.s32.b, cc_src);
+            tcg_gen_extrl_i64_i32(c->u.s32.b, cc_src);
         }
         break;
 
@@ -1532,7 +1532,7 @@ static ExitStatus op_bct32(DisasContext *s, DisasOps *o)
     store_reg32_i64(r1, t);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_const_i32(0);
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
     tcg_temp_free_i64(t);
 
     return help_branch(s, &c, is_imm, imm, o->in2);
@@ -1556,7 +1556,7 @@ static ExitStatus op_bcth(DisasContext *s, DisasOps *o)
     store_reg32h_i64(r1, t);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_const_i32(0);
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
     tcg_temp_free_i64(t);
 
     return help_branch(s, &c, 1, imm, o->in2);
@@ -1599,8 +1599,8 @@ static ExitStatus op_bx32(DisasContext *s, DisasOps *o)
     tcg_gen_add_i64(t, regs[r1], regs[r3]);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
-    tcg_gen_trunc_i64_i32(c.u.s32.b, regs[r3 | 1]);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.b, regs[r3 | 1]);
     store_reg32_i64(r1, t);
     tcg_temp_free_i64(t);
 
@@ -1905,7 +1905,7 @@ static ExitStatus op_clm(DisasContext *s, DisasOps *o)
 {
     TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
     TCGv_i32 t1 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
     potential_page_fault(s);
     gen_helper_clm(cc_op, cpu_env, t1, m3, o->in2);
     set_cc_static(s);
@@ -1977,7 +1977,7 @@ static ExitStatus op_cs(DisasContext *s, DisasOps *o)
 
     /* Store CC back to cc_op.  Wait until after the store so that any
        exception gets the old cc_op value.  */
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
@@ -2027,7 +2027,7 @@ static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
     /* Save back state now that we've passed all exceptions.  */
     tcg_gen_mov_i64(regs[r1], outh);
     tcg_gen_mov_i64(regs[r1 + 1], outl);
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(outh);
     tcg_temp_free_i64(outl);
     tcg_temp_free_i64(cc);
@@ -2051,7 +2051,7 @@ static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
 {
     TCGv_i64 t1 = tcg_temp_new_i64();
     TCGv_i32 t2 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t2, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in1);
     gen_helper_cvd(t1, t2);
     tcg_temp_free_i32(t2);
     tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s));
@@ -3235,8 +3235,8 @@ static ExitStatus op_rll32(DisasContext *s, DisasOps *o)
     TCGv_i32 t1 = tcg_temp_new_i32();
     TCGv_i32 t2 = tcg_temp_new_i32();
     TCGv_i32 to = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
-    tcg_gen_trunc_i64_i32(t2, o->in2);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in2);
     tcg_gen_rotl_i32(to, t1, t2);
     tcg_gen_extu_i32_i64(o->out, to);
     tcg_temp_free_i32(t1);
@@ -5319,18 +5319,14 @@ static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(S390CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUS390XState *env, struct TranslationBlock *tb)
 {
+    S390CPU *cpu = s390_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUS390XState *env = &cpu->env;
     DisasContext dc;
     target_ulong pc_start;
     uint64_t next_page_start;
-    int j, lj = -1;
     int num_insns, max_insns;
-    CPUBreakpoint *bp;
     ExitStatus status;
     bool do_debug;
 
@@ -5353,41 +5349,32 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     gen_tb_start(tb);
 
     do {
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            gen_opc_cc_op[lj] = dc.cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (++num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
-            gen_io_start();
+        tcg_gen_insn_start(dc.pc, dc.cc_op);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            status = EXIT_PC_STALE;
+            do_debug = true;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
+            gen_io_start();
         }
 
         status = NO_EXIT;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc.pc) {
-                    status = EXIT_PC_STALE;
-                    do_debug = true;
-                    break;
-                }
-            }
-        }
         if (status == NO_EXIT) {
             status = translate_one(env, &dc);
         }
@@ -5432,16 +5419,8 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc.pc - pc_start;
+    tb->icount = num_insns;
 
 #if defined(S390X_DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -5452,21 +5431,11 @@ static inline void gen_intermediate_code_internal(S390CPU *cpu,
 #endif
 }
 
-void gen_intermediate_code (CPUS390XState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUS390XState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    int cc_op;
-    env->psw.addr = tcg_ctx.gen_opc_pc[pc_pos];
-    cc_op = gen_opc_cc_op[pc_pos];
+    int cc_op = data[1];
+    env->psw.addr = data[0];
     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
         env->cc_op = cc_op;
     }
diff --git a/qemu/target-sh4/Makefile.objs b/qemu/target-sh4/Makefile.objs
index a285358ad..2c25d96e6 100644
--- a/qemu/target-sh4/Makefile.objs
+++ b/qemu/target-sh4/Makefile.objs
@@ -1,2 +1,3 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += gdbstub.o
diff --git a/qemu/target-sh4/cpu.c b/qemu/target-sh4/cpu.c
index 5c65ab4df..86ba38808 100644
--- a/qemu/target-sh4/cpu.c
+++ b/qemu/target-sh4/cpu.c
@@ -19,6 +19,8 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -70,6 +72,12 @@ static void superh_cpu_reset(CPUState *s)
     set_default_nan_mode(1, &env->fp_status);
 }
 
+static void superh_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_sh4;
+    info->print_insn = print_insn_sh;
+}
+
 typedef struct SuperHCPUListState {
     fprintf_function cpu_fprintf;
     FILE *file;
@@ -288,8 +296,18 @@ static void superh_cpu_class_init(ObjectClass *oc, void *data)
 #else
     cc->get_phys_page_debug = superh_cpu_get_phys_page_debug;
 #endif
-    dc->vmsd = &vmstate_sh_cpu;
+    cc->disas_set_info = superh_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 59;
+
+    dc->vmsd = &vmstate_sh_cpu;
+
+    /*
+     * Reason: superh_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo superh_cpu_type_info = {
diff --git a/qemu/target-sh4/cpu.h b/qemu/target-sh4/cpu.h
index 34bb3d779..3b23e967b 100644
--- a/qemu/target-sh4/cpu.h
+++ b/qemu/target-sh4/cpu.h
@@ -19,13 +19,10 @@
 #ifndef _CPU_SH4_H
 #define _CPU_SH4_H
 
-#include "config.h"
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 32
 
-#define ELF_MACHINE	EM_SH
-
 /* CPU Subtypes */
 #define SH_CPU_SH7750  (1 << 0)
 #define SH_CPU_SH7750S (1 << 1)
@@ -122,6 +119,7 @@ typedef struct tlb_t {
 #define ITLB_SIZE 4
 
 #define NB_MMU_MODES 2
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 enum sh_features {
     SH_FEATURE_SH4A = 1,
@@ -227,7 +225,6 @@ void cpu_load_tlb(CPUSH4State * env);
 #define cpu_init(cpu_model) CPU(cpu_sh4_init(cpu_model))
 
 #define cpu_exec cpu_sh4_exec
-#define cpu_gen_code cpu_sh4_gen_code
 #define cpu_signal_handler cpu_sh4_signal_handler
 #define cpu_list sh4_cpu_list
 
@@ -235,7 +232,7 @@ void cpu_load_tlb(CPUSH4State * env);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUSH4State *env)
+static inline int cpu_mmu_index (CPUSH4State *env, bool ifetch)
 {
     return (env->sr & (1u << SR_MD)) == 0 ? 1 : 0;
 }
diff --git a/qemu/target-sh4/gdbstub.c b/qemu/target-sh4/gdbstub.c
index a365a27aa..1b59ea8c7 100644
--- a/qemu/target-sh4/gdbstub.c
+++ b/qemu/target-sh4/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-sh4/helper.c b/qemu/target-sh4/helper.c
index a533f08ea..6438338f2 100644
--- a/qemu/target-sh4/helper.c
+++ b/qemu/target-sh4/helper.c
@@ -16,14 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/log.h"
 
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/sh4/sh_intc.h"
@@ -60,7 +56,7 @@ int superh_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
 
 int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 {
-    /* For user mode, only U0 area is cachable. */
+    /* For user mode, only U0 area is cacheable. */
     return !(addr & 0x80000000);
 }
 
@@ -826,11 +822,11 @@ int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 
     /* check area */
     if (env->sr & (1u << SR_MD)) {
-        /* For previledged mode, P2 and P4 area is not cachable. */
+        /* For privileged mode, P2 and P4 area is not cacheable. */
         if ((0xA0000000 <= addr && addr < 0xC0000000) || 0xE0000000 <= addr)
             return 0;
     } else {
-        /* For user mode, only U0 area is cachable. */
+        /* For user mode, only U0 area is cacheable. */
         if (0x80000000 <= addr)
             return 0;
     }
diff --git a/qemu/target-sh4/helper.h b/qemu/target-sh4/helper.h
index c9bc40704..dce859cae 100644
--- a/qemu/target-sh4/helper.h
+++ b/qemu/target-sh4/helper.h
@@ -18,28 +18,28 @@ DEF_HELPER_2(ld_fpscr, void, env, i32)
 
 DEF_HELPER_FLAGS_1(fabs_FT, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_FLAGS_1(fabs_DT, TCG_CALL_NO_RWG_SE, f64, f64)
-DEF_HELPER_3(fadd_FT, f32, env, f32, f32)
-DEF_HELPER_3(fadd_DT, f64, env, f64, f64)
-DEF_HELPER_2(fcnvsd_FT_DT, f64, env, f32)
-DEF_HELPER_2(fcnvds_DT_FT, f32, env, f64)
+DEF_HELPER_FLAGS_3(fadd_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fadd_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fcnvsd_FT_DT, TCG_CALL_NO_WG, f64, env, f32)
+DEF_HELPER_FLAGS_2(fcnvds_DT_FT, TCG_CALL_NO_WG, f32, env, f64)
 
 DEF_HELPER_3(fcmp_eq_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_eq_DT, void, env, f64, f64)
 DEF_HELPER_3(fcmp_gt_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_gt_DT, void, env, f64, f64)
-DEF_HELPER_3(fdiv_FT, f32, env, f32, f32)
-DEF_HELPER_3(fdiv_DT, f64, env, f64, f64)
-DEF_HELPER_2(float_FT, f32, env, i32)
-DEF_HELPER_2(float_DT, f64, env, i32)
-DEF_HELPER_4(fmac_FT, f32, env, f32, f32, f32)
-DEF_HELPER_3(fmul_FT, f32, env, f32, f32)
-DEF_HELPER_3(fmul_DT, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fdiv_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fdiv_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(float_FT, TCG_CALL_NO_WG, f32, env, i32)
+DEF_HELPER_FLAGS_2(float_DT, TCG_CALL_NO_WG, f64, env, i32)
+DEF_HELPER_FLAGS_4(fmac_FT, TCG_CALL_NO_WG, f32, env, f32, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
 DEF_HELPER_FLAGS_1(fneg_T, TCG_CALL_NO_RWG_SE, f32, f32)
-DEF_HELPER_3(fsub_FT, f32, env, f32, f32)
-DEF_HELPER_3(fsub_DT, f64, env, f64, f64)
-DEF_HELPER_2(fsqrt_FT, f32, env, f32)
-DEF_HELPER_2(fsqrt_DT, f64, env, f64)
-DEF_HELPER_2(ftrc_FT, i32, env, f32)
-DEF_HELPER_2(ftrc_DT, i32, env, f64)
+DEF_HELPER_FLAGS_3(fsub_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fsub_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fsqrt_FT, TCG_CALL_NO_WG, f32, env, f32)
+DEF_HELPER_FLAGS_2(fsqrt_DT, TCG_CALL_NO_WG, f64, env, f64)
+DEF_HELPER_FLAGS_2(ftrc_FT, TCG_CALL_NO_WG, i32, env, f32)
+DEF_HELPER_FLAGS_2(ftrc_DT, TCG_CALL_NO_WG, i32, env, f64)
 DEF_HELPER_3(fipr, void, env, i32, i32)
 DEF_HELPER_2(ftrv, void, env, i32)
diff --git a/qemu/target-sh4/monitor.c b/qemu/target-sh4/monitor.c
new file mode 100644
index 000000000..426e5d491
--- /dev/null
+++ b/qemu/target-sh4/monitor.c
@@ -0,0 +1,53 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
+{
+    monitor_printf(mon, " tlb%i:\t"
+                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
+                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
+                   "dirty=%hhu writethrough=%hhu\n",
+                   idx,
+                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
+                   tlb->v, tlb->sh, tlb->c, tlb->pr,
+                   tlb->d, tlb->wt);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    int i;
+
+    monitor_printf (mon, "ITLB:\n");
+    for (i = 0 ; i < ITLB_SIZE ; i++)
+        print_tlb (mon, i, &env->itlb[i]);
+    monitor_printf (mon, "UTLB:\n");
+    for (i = 0 ; i < UTLB_SIZE ; i++)
+        print_tlb (mon, i, &env->utlb[i]);
+}
diff --git a/qemu/target-sh4/op_helper.c b/qemu/target-sh4/op_helper.c
index cbc11aecc..368e687d6 100644
--- a/qemu/target-sh4/op_helper.c
+++ b/qemu/target-sh4/op_helper.c
@@ -16,8 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <assert.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
diff --git a/qemu/target-sh4/translate.c b/qemu/target-sh4/translate.c
index 3b4a1b5ce..7c189680a 100644
--- a/qemu/target-sh4/translate.c
+++ b/qemu/target-sh4/translate.c
@@ -19,6 +19,7 @@
 
 #define DEBUG_DISAS
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -28,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 typedef struct DisasContext {
@@ -59,7 +61,7 @@ enum {
 };
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_gregs[24];
 static TCGv cpu_sr, cpu_sr_m, cpu_sr_q, cpu_sr_t;
 static TCGv cpu_pc, cpu_ssr, cpu_spc, cpu_gbr;
@@ -70,8 +72,6 @@ static TCGv cpu_fregs[32];
 /* internal register indexes */
 static TCGv cpu_flags, cpu_delayed_pc;
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 void sh4_translate_init(void)
@@ -102,53 +102,53 @@ void sh4_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < 24; i++)
-        cpu_gregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_gregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUSH4State, gregs[i]),
                                               gregnames[i]);
 
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, pc), "PC");
-    cpu_sr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sr = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, sr), "SR");
-    cpu_sr_m = tcg_global_mem_new_i32(TCG_AREG0,
-                                    offsetof(CPUSH4State, sr_m), "SR_M");
-    cpu_sr_q = tcg_global_mem_new_i32(TCG_AREG0,
-                                    offsetof(CPUSH4State, sr_q), "SR_Q");
-    cpu_sr_t = tcg_global_mem_new_i32(TCG_AREG0,
-                                    offsetof(CPUSH4State, sr_t), "SR_T");
-    cpu_ssr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sr_m = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_m), "SR_M");
+    cpu_sr_q = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_q), "SR_Q");
+    cpu_sr_t = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_t), "SR_T");
+    cpu_ssr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, ssr), "SSR");
-    cpu_spc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_spc = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, spc), "SPC");
-    cpu_gbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_gbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, gbr), "GBR");
-    cpu_vbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_vbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, vbr), "VBR");
-    cpu_sgr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sgr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, sgr), "SGR");
-    cpu_dbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_dbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, dbr), "DBR");
-    cpu_mach = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_mach = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, mach), "MACH");
-    cpu_macl = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_macl = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, macl), "MACL");
-    cpu_pr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pr = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, pr), "PR");
-    cpu_fpscr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_fpscr = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUSH4State, fpscr), "FPSCR");
-    cpu_fpul = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_fpul = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, fpul), "FPUL");
 
-    cpu_flags = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_flags = tcg_global_mem_new_i32(cpu_env,
 				       offsetof(CPUSH4State, flags), "_flags_");
-    cpu_delayed_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_delayed_pc = tcg_global_mem_new_i32(cpu_env,
 					    offsetof(CPUSH4State, delayed_pc),
 					    "_delayed_pc_");
-    cpu_ldst = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_ldst = tcg_global_mem_new_i32(cpu_env,
 				      offsetof(CPUSH4State, ldst), "_ldst_");
 
     for (i = 0; i < 32; i++)
-        cpu_fregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_fregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUSH4State, fregs[i]),
                                               fregnames[i]);
 
@@ -288,10 +288,10 @@ static inline void gen_load_fpr64(TCGv_i64 t, int reg)
 static inline void gen_store_fpr64 (TCGv_i64 t, int reg)
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg + 1], tmp);
     tcg_gen_shri_i64(t, t, 32);
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg], tmp);
     tcg_temp_free_i32(tmp);
 }
@@ -612,15 +612,11 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0x6008:		/* swap.b Rm,Rn */
 	{
-	    TCGv high, low;
-	    high = tcg_temp_new();
-	    tcg_gen_andi_i32(high, REG(B7_4), 0xffff0000);
-	    low = tcg_temp_new();
+            TCGv low = tcg_temp_new();;
 	    tcg_gen_ext16u_i32(low, REG(B7_4));
 	    tcg_gen_bswap16_i32(low, low);
-	    tcg_gen_or_i32(REG(B11_8), high, low);
+            tcg_gen_deposit_i32(REG(B11_8), REG(B7_4), low, 0, 16);
 	    tcg_temp_free(low);
-	    tcg_temp_free(high);
 	}
 	return;
     case 0x6009:		/* swap.w Rm,Rn */
@@ -692,18 +688,11 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv cmp1 = tcg_temp_new();
 	    TCGv cmp2 = tcg_temp_new();
-	    tcg_gen_xor_i32(cmp1, REG(B7_4), REG(B11_8));
-	    tcg_gen_andi_i32(cmp2, cmp1, 0xff000000);
-            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, cmp2, 0);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x00ff0000);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x0000ff00);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x000000ff);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-            tcg_gen_or_i32(cpu_sr_t, cpu_sr_t, cmp2);
+            tcg_gen_xor_i32(cmp2, REG(B7_4), REG(B11_8));
+            tcg_gen_subi_i32(cmp1, cmp2, 0x01010101);
+            tcg_gen_andc_i32(cmp1, cmp1, cmp2);
+            tcg_gen_andi_i32(cmp1, cmp1, 0x80808080);
+            tcg_gen_setcondi_i32(TCG_COND_NE, cpu_sr_t, cmp1, 0);
 	    tcg_temp_free(cmp2);
 	    tcg_temp_free(cmp1);
 	}
@@ -843,67 +832,54 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0x400c:		/* shad Rm,Rn */
 	{
-            TCGLabel *label1 = gen_new_label();
-            TCGLabel *label2 = gen_new_label();
-            TCGLabel *label3 = gen_new_label();
-            TCGLabel *label4 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_sar_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B11_8), 0, label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    tcg_gen_br(label4);
-	    gen_set_label(label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0xffffffff);
-	    gen_set_label(label4);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_sar_i32(t2, REG(B11_8), t0);
+            tcg_gen_sari_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x400d:		/* shld Rm,Rn */
 	{
-            TCGLabel *label1 = gen_new_label();
-            TCGLabel *label2 = gen_new_label();
-            TCGLabel *label3 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_shr_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    gen_set_label(label3);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_shr_i32(t2, REG(B11_8), t0);
+            tcg_gen_shri_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x3008:		/* sub Rm,Rn */
@@ -1814,10 +1790,6 @@ static void decode_opc(DisasContext * ctx)
 {
     uint32_t old_flags = ctx->flags;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     _decode_opc(ctx);
 
     if (old_flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
@@ -1844,16 +1816,12 @@ static void decode_opc(DisasContext * ctx)
         gen_store_flags(ctx->flags);
 }
 
-static inline void
-gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
 {
+    SuperHCPU *cpu = sh_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSH4State *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int i, ii;
     int num_insns;
     int max_insns;
 
@@ -1870,45 +1838,39 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
     ctx.features = env->features;
     ctx.has_movcal = (ctx.flags & TB_FLAG_PENDING_MOVCA);
 
-    ii = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-		    /* We have hit a breakpoint - make sure PC is up-to-date */
-		    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_BRANCH;
-		    break;
-		}
-	    }
-	}
-        if (search_pc) {
-            i = tcg_op_buf_count();
-            if (ii < i) {
-                ii++;
-                while (ii < i)
-                    tcg_ctx.gen_opc_instr_start[ii++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[ii] = ctx.pc;
-            gen_opc_hflags[ii] = ctx.flags;
-            tcg_ctx.gen_opc_instr_start[ii] = 1;
-            tcg_ctx.gen_opc_icount[ii] = num_insns;
+        tcg_gen_insn_start(ctx.pc, ctx.flags);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            /* We have hit a breakpoint - make sure PC is up-to-date */
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_BRANCH;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-#if 0
-	fprintf(stderr, "Loading opcode at address 0x%08x\n", ctx.pc);
-	fflush(stderr);
-#endif
+        }
+
         ctx.opcode = cpu_lduw_code(env, ctx.pc);
 	decode_opc(&ctx);
-        num_insns++;
 	ctx.pc += 2;
 	if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
 	    break;
@@ -1948,15 +1910,8 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        i = tcg_op_buf_count();
-        ii++;
-        while (ii <= i)
-            tcg_ctx.gen_opc_instr_start[ii++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
@@ -1967,18 +1922,9 @@ gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
 #endif
 }
 
-void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSH4State * env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    env->flags = gen_opc_hflags[pc_pos];
+    env->pc = data[0];
+    env->flags = data[1];
 }
diff --git a/qemu/target-sparc/Makefile.objs b/qemu/target-sparc/Makefile.objs
index 1cd81cccc..ec905698c 100644
--- a/qemu/target-sparc/Makefile.objs
+++ b/qemu/target-sparc/Makefile.objs
@@ -1,4 +1,4 @@
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += machine.o monitor.o
 obj-y += translate.o helper.o cpu.o
 obj-y += fop_helper.o cc_helper.o win_helper.o mmu_helper.o ldst_helper.o
 obj-$(TARGET_SPARC) += int32_helper.o
diff --git a/qemu/target-sparc/cc_helper.c b/qemu/target-sparc/cc_helper.c
index 35dab7321..44c440934 100644
--- a/qemu/target-sparc/cc_helper.c
+++ b/qemu/target-sparc/cc_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-sparc/cpu-qom.h b/qemu/target-sparc/cpu-qom.h
index 477c4d513..5096b1047 100644
--- a/qemu/target-sparc/cpu-qom.h
+++ b/qemu/target-sparc/cpu-qom.h
@@ -75,6 +75,10 @@ static inline SPARCCPU *sparc_env_get_cpu(CPUSPARCState *env)
 
 #define ENV_OFFSET offsetof(SPARCCPU, env)
 
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_sparc_cpu;
+#endif
+
 void sparc_cpu_do_interrupt(CPUState *cpu);
 void sparc_cpu_dump_state(CPUState *cpu, FILE *f,
                           fprintf_function cpu_fprintf, int flags);
diff --git a/qemu/target-sparc/cpu.c b/qemu/target-sparc/cpu.c
index 9528e3afb..fe4119e2b 100644
--- a/qemu/target-sparc/cpu.c
+++ b/qemu/target-sparc/cpu.c
@@ -17,6 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu/error-report.h"
 
@@ -90,6 +92,14 @@ static bool sparc_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
     return false;
 }
 
+static void cpu_sparc_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_sparc;
+#ifdef TARGET_SPARC64
+    info->mach = bfd_mach_sparc_v9b;
+#endif
+}
+
 static int cpu_sparc_register(SPARCCPU *cpu, const char *cpu_model)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
@@ -847,13 +857,22 @@ static void sparc_cpu_class_init(ObjectClass *oc, void *data)
     cc->do_unassigned_access = sparc_cpu_unassigned_access;
     cc->do_unaligned_access = sparc_cpu_do_unaligned_access;
     cc->get_phys_page_debug = sparc_cpu_get_phys_page_debug;
+    cc->vmsd = &vmstate_sparc_cpu;
 #endif
+    cc->disas_set_info = cpu_sparc_disas_set_info;
 
 #if defined(TARGET_SPARC64) && !defined(TARGET_ABI32)
     cc->gdb_num_core_regs = 86;
 #else
     cc->gdb_num_core_regs = 72;
 #endif
+
+    /*
+     * Reason: sparc_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo sparc_cpu_type_info = {
diff --git a/qemu/target-sparc/cpu.h b/qemu/target-sparc/cpu.h
index 0522b65f1..dc4612275 100644
--- a/qemu/target-sparc/cpu.h
+++ b/qemu/target-sparc/cpu.h
@@ -1,7 +1,6 @@
 #ifndef CPU_SPARC_H
 #define CPU_SPARC_H
 
-#include "config.h"
 #include "qemu-common.h"
 #include "qemu/bswap.h"
 
@@ -31,12 +30,6 @@
 
 #include "fpu/softfloat.h"
 
-#if !defined(TARGET_SPARC64)
-#define ELF_MACHINE     EM_SPARC
-#else
-#define ELF_MACHINE     EM_SPARCV9
-#endif
-
 /*#define EXCP_INTERRUPT 0x100*/
 
 /* trap definitions */
@@ -236,6 +229,7 @@ typedef struct trap_state {
     uint32_t tt;
 } trap_state;
 #endif
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 typedef struct sparc_def_t {
     const char *name;
@@ -371,16 +365,14 @@ struct CPUTimer
     uint32_t    frequency;
     uint32_t    disabled;
     uint64_t    disabled_mask;
+    uint32_t    npt;
+    uint64_t    npt_mask;
     int64_t     clock_offset;
     QEMUTimer  *qtimer;
 };
 
 typedef struct CPUTimer CPUTimer;
 
-struct QEMUFile;
-void cpu_put_timer(struct QEMUFile *f, CPUTimer *s);
-void cpu_get_timer(struct QEMUFile *f, CPUTimer *s);
-
 typedef struct CPUSPARCState CPUSPARCState;
 
 struct CPUSPARCState {
@@ -542,6 +534,7 @@ int cpu_sparc_exec(CPUState *cpu);
 /* win_helper.c */
 target_ulong cpu_get_psr(CPUSPARCState *env1);
 void cpu_put_psr(CPUSPARCState *env1, target_ulong val);
+void cpu_put_psr_raw(CPUSPARCState *env1, target_ulong val);
 #ifdef TARGET_SPARC64
 target_ulong cpu_get_ccr(CPUSPARCState *env1);
 void cpu_put_ccr(CPUSPARCState *env1, target_ulong val);
@@ -598,12 +591,9 @@ int cpu_sparc_signal_handler(int host_signum, void *pinfo, void *puc);
 #endif
 
 #define cpu_exec cpu_sparc_exec
-#define cpu_gen_code cpu_sparc_gen_code
 #define cpu_signal_handler cpu_sparc_signal_handler
 #define cpu_list sparc_cpu_list
 
-#define CPU_SAVE_VERSION 7
-
 /* MMU modes definitions */
 #if defined (TARGET_SPARC64)
 #define MMU_USER_IDX   0
@@ -642,7 +632,7 @@ static inline int cpu_supervisor_mode(CPUSPARCState *env1)
 }
 #endif
 
-static inline int cpu_mmu_index(CPUSPARCState *env1)
+static inline int cpu_mmu_index(CPUSPARCState *env1, bool ifetch)
 {
 #if defined(CONFIG_USER_ONLY)
     return MMU_USER_IDX;
diff --git a/qemu/target-sparc/fop_helper.c b/qemu/target-sparc/fop_helper.c
index ee4592ef2..08306436a 100644
--- a/qemu/target-sparc/fop_helper.c
+++ b/qemu/target-sparc/fop_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-sparc/gdbstub.c b/qemu/target-sparc/gdbstub.c
index 3de3242b2..e530dc52f 100644
--- a/qemu/target-sparc/gdbstub.c
+++ b/qemu/target-sparc/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
diff --git a/qemu/target-sparc/helper.c b/qemu/target-sparc/helper.c
index 4850c7cec..8349cbe2c 100644
--- a/qemu/target-sparc/helper.c
+++ b/qemu/target-sparc/helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
@@ -51,10 +52,16 @@ void helper_tick_set_count(void *opaque, uint64_t count)
 #endif
 }
 
-uint64_t helper_tick_get_count(void *opaque)
+uint64_t helper_tick_get_count(CPUSPARCState *env, void *opaque, int mem_idx)
 {
 #if !defined(CONFIG_USER_ONLY)
-    return cpu_tick_get_count(opaque);
+    CPUTimer *timer = opaque;
+
+    if (timer->npt && mem_idx < MMU_KERNEL_IDX) {
+        helper_raise_exception(env, TT_PRIV_INSN);
+    }
+
+    return cpu_tick_get_count(timer);
 #else
     return 0;
 #endif
diff --git a/qemu/target-sparc/helper.h b/qemu/target-sparc/helper.h
index 1ad23e8db..4374f0dd2 100644
--- a/qemu/target-sparc/helper.h
+++ b/qemu/target-sparc/helper.h
@@ -25,7 +25,7 @@ DEF_HELPER_2(set_softint, void, env, i64)
 DEF_HELPER_2(clear_softint, void, env, i64)
 DEF_HELPER_2(write_softint, void, env, i64)
 DEF_HELPER_2(tick_set_count, void, ptr, i64)
-DEF_HELPER_1(tick_get_count, i64, ptr)
+DEF_HELPER_3(tick_get_count, i64, env, ptr, int)
 DEF_HELPER_2(tick_set_limit, void, ptr, i64)
 #endif
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
diff --git a/qemu/target-sparc/int32_helper.c b/qemu/target-sparc/int32_helper.c
index 7c380ba2a..09afe136e 100644
--- a/qemu/target-sparc/int32_helper.c
+++ b/qemu/target-sparc/int32_helper.c
@@ -17,9 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "sysemu/sysemu.h"
+#include "exec/log.h"
 
 #define DEBUG_PCALL
 
diff --git a/qemu/target-sparc/int64_helper.c b/qemu/target-sparc/int64_helper.c
index b02d22b19..29360fa5f 100644
--- a/qemu/target-sparc/int64_helper.c
+++ b/qemu/target-sparc/int64_helper.c
@@ -17,8 +17,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/log.h"
 #include "trace.h"
 
 #define DEBUG_PCALL
@@ -156,9 +158,8 @@ void sparc_cpu_do_interrupt(CPUState *cs)
     } else if ((intno & 0x1c0) == TT_FILL) {
         cpu_set_cwp(env, cpu_cwp_inc(env, env->cwp + 1));
     }
-    env->tbr &= ~0x7fffULL;
-    env->tbr |= ((env->tl > 1) ? 1 << 14 : 0) | (intno << 5);
-    env->pc = env->tbr;
+    env->pc = env->tbr  & ~0x7fffULL;
+    env->pc |= ((env->tl > 1) ? 1 << 14 : 0) | (intno << 5);
     env->npc = env->pc + 4;
     cs->exception_index = -1;
 }
diff --git a/qemu/target-sparc/ldst_helper.c b/qemu/target-sparc/ldst_helper.c
index c7ad47d35..658e7d858 100644
--- a/qemu/target-sparc/ldst_helper.c
+++ b/qemu/target-sparc/ldst_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
@@ -2058,11 +2059,11 @@ void helper_ldda_asi(CPUSPARCState *env, target_ulong addr, int asi, int rd)
                 bswap64s(&env->gregs[rd + 1]);
             }
         } else {
-            env->regwptr[rd] = cpu_ldq_nucleus(env, addr);
-            env->regwptr[rd + 1] = cpu_ldq_nucleus(env, addr + 8);
+            env->regwptr[rd - 8] = cpu_ldq_nucleus(env, addr);
+            env->regwptr[rd + 1 - 8] = cpu_ldq_nucleus(env, addr + 8);
             if (asi == 0x2c) {
-                bswap64s(&env->regwptr[rd]);
-                bswap64s(&env->regwptr[rd + 1]);
+                bswap64s(&env->regwptr[rd - 8]);
+                bswap64s(&env->regwptr[rd + 1 - 8]);
             }
         }
         break;
@@ -2075,8 +2076,8 @@ void helper_ldda_asi(CPUSPARCState *env, target_ulong addr, int asi, int rd)
             env->gregs[rd] = helper_ld_asi(env, addr, asi, 4, 0);
             env->gregs[rd + 1] = helper_ld_asi(env, addr + 4, asi, 4, 0);
         } else {
-            env->regwptr[rd] = helper_ld_asi(env, addr, asi, 4, 0);
-            env->regwptr[rd + 1] = helper_ld_asi(env, addr + 4, asi, 4, 0);
+            env->regwptr[rd - 8] = helper_ld_asi(env, addr, asi, 4, 0);
+            env->regwptr[rd + 1 - 8] = helper_ld_asi(env, addr + 4, asi, 4, 0);
         }
         break;
     }
diff --git a/qemu/target-sparc/machine.c b/qemu/target-sparc/machine.c
index 3f3de4c65..1046016f3 100644
--- a/qemu/target-sparc/machine.c
+++ b/qemu/target-sparc/machine.c
@@ -1,218 +1,191 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "qemu/timer.h"
 
 #include "cpu.h"
 
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUSPARCState *env = opaque;
-    int i;
-    uint32_t tmp;
-
-    // if env->cwp == env->nwindows - 1, this will set the ins of the last
-    // window as the outs of the first window
-    cpu_set_cwp(env, env->cwp);
+#ifdef TARGET_SPARC64
+static const VMStateDescription vmstate_cpu_timer = {
+    .name = "cpu_timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(frequency, CPUTimer),
+        VMSTATE_UINT32(disabled, CPUTimer),
+        VMSTATE_UINT64(disabled_mask, CPUTimer),
+        VMSTATE_UINT32(npt, CPUTimer),
+        VMSTATE_UINT64(npt_mask, CPUTimer),
+        VMSTATE_INT64(clock_offset, CPUTimer),
+        VMSTATE_TIMER_PTR(qtimer, CPUTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
 
-    for(i = 0; i < 8; i++)
-        qemu_put_betls(f, &env->gregs[i]);
-    qemu_put_be32s(f, &env->nwindows);
-    for(i = 0; i < env->nwindows * 16; i++)
-        qemu_put_betls(f, &env->regbase[i]);
+#define VMSTATE_CPU_TIMER(_f, _s)                             \
+    VMSTATE_STRUCT_POINTER(_f, _s, vmstate_cpu_timer, CPUTimer)
 
-    /* FPU */
-    for (i = 0; i < TARGET_DPREGS; i++) {
-        qemu_put_be32(f, env->fpr[i].l.upper);
-        qemu_put_be32(f, env->fpr[i].l.lower);
+static const VMStateDescription vmstate_trap_state = {
+    .name = "trap_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(tpc, trap_state),
+        VMSTATE_UINT64(tnpc, trap_state),
+        VMSTATE_UINT64(tstate, trap_state),
+        VMSTATE_UINT32(tt, trap_state),
+        VMSTATE_END_OF_LIST()
     }
+};
 
-    qemu_put_betls(f, &env->pc);
-    qemu_put_betls(f, &env->npc);
-    qemu_put_betls(f, &env->y);
-    tmp = cpu_get_psr(env);
-    qemu_put_be32(f, tmp);
-    qemu_put_betls(f, &env->fsr);
-    qemu_put_betls(f, &env->tbr);
-    tmp = env->interrupt_index;
-    qemu_put_be32(f, tmp);
-    qemu_put_be32s(f, &env->pil_in);
-#ifndef TARGET_SPARC64
-    qemu_put_be32s(f, &env->wim);
-    /* MMU */
-    for (i = 0; i < 32; i++)
-        qemu_put_be32s(f, &env->mmuregs[i]);
-    for (i = 0; i < 4; i++) {
-        qemu_put_be64s(f, &env->mxccdata[i]);
-    }
-    for (i = 0; i < 8; i++) {
-        qemu_put_be64s(f, &env->mxccregs[i]);
-    }
-    qemu_put_be32s(f, &env->mmubpctrv);
-    qemu_put_be32s(f, &env->mmubpctrc);
-    qemu_put_be32s(f, &env->mmubpctrs);
-    qemu_put_be64s(f, &env->mmubpaction);
-    for (i = 0; i < 4; i++) {
-        qemu_put_be64s(f, &env->mmubpregs[i]);
-    }
-#else
-    qemu_put_be64s(f, &env->lsu);
-    for (i = 0; i < 16; i++) {
-        qemu_put_be64s(f, &env->immuregs[i]);
-        qemu_put_be64s(f, &env->dmmuregs[i]);
-    }
-    for (i = 0; i < 64; i++) {
-        qemu_put_be64s(f, &env->itlb[i].tag);
-        qemu_put_be64s(f, &env->itlb[i].tte);
-        qemu_put_be64s(f, &env->dtlb[i].tag);
-        qemu_put_be64s(f, &env->dtlb[i].tte);
+static const VMStateDescription vmstate_tlb_entry = {
+    .name = "tlb_entry",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(tag, SparcTLBEntry),
+        VMSTATE_UINT64(tte, SparcTLBEntry),
+        VMSTATE_END_OF_LIST()
     }
-    qemu_put_be32s(f, &env->mmu_version);
-    for (i = 0; i < MAXTL_MAX; i++) {
-        qemu_put_be64s(f, &env->ts[i].tpc);
-        qemu_put_be64s(f, &env->ts[i].tnpc);
-        qemu_put_be64s(f, &env->ts[i].tstate);
-        qemu_put_be32s(f, &env->ts[i].tt);
-    }
-    qemu_put_be32s(f, &env->xcc);
-    qemu_put_be32s(f, &env->asi);
-    qemu_put_be32s(f, &env->pstate);
-    qemu_put_be32s(f, &env->tl);
-    qemu_put_be32s(f, &env->cansave);
-    qemu_put_be32s(f, &env->canrestore);
-    qemu_put_be32s(f, &env->otherwin);
-    qemu_put_be32s(f, &env->wstate);
-    qemu_put_be32s(f, &env->cleanwin);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->agregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->bgregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->igregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->mgregs[i]);
-    qemu_put_be64s(f, &env->fprs);
-    qemu_put_be64s(f, &env->tick_cmpr);
-    qemu_put_be64s(f, &env->stick_cmpr);
-    cpu_put_timer(f, env->tick);
-    cpu_put_timer(f, env->stick);
-    qemu_put_be64s(f, &env->gsr);
-    qemu_put_be32s(f, &env->gl);
-    qemu_put_be64s(f, &env->hpstate);
-    for (i = 0; i < MAXTL_MAX; i++)
-        qemu_put_be64s(f, &env->htstate[i]);
-    qemu_put_be64s(f, &env->hintp);
-    qemu_put_be64s(f, &env->htba);
-    qemu_put_be64s(f, &env->hver);
-    qemu_put_be64s(f, &env->hstick_cmpr);
-    qemu_put_be64s(f, &env->ssr);
-    cpu_put_timer(f, env->hstick);
+};
 #endif
+
+static int get_psr(QEMUFile *f, void *opaque, size_t size)
+{
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+    uint32_t val = qemu_get_be32(f);
+
+    /* needed to ensure that the wrapping registers are correctly updated */
+    env->cwp = 0;
+    cpu_put_psr_raw(env, val);
+
+    return 0;
 }
 
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
+static void put_psr(QEMUFile *f, void *opaque, size_t size)
 {
-    CPUSPARCState *env = opaque;
-    SPARCCPU *cpu = sparc_env_get_cpu(env);
-    int i;
-    uint32_t tmp;
-
-    if (version_id < 6)
-        return -EINVAL;
-    for(i = 0; i < 8; i++)
-        qemu_get_betls(f, &env->gregs[i]);
-    qemu_get_be32s(f, &env->nwindows);
-    for(i = 0; i < env->nwindows * 16; i++)
-        qemu_get_betls(f, &env->regbase[i]);
-
-    /* FPU */
-    for (i = 0; i < TARGET_DPREGS; i++) {
-        env->fpr[i].l.upper = qemu_get_be32(f);
-        env->fpr[i].l.lower = qemu_get_be32(f);
-    }
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+    uint32_t val;
+
+    val = cpu_get_psr(env);
+
+    qemu_put_be32(f, val);
+}
 
-    qemu_get_betls(f, &env->pc);
-    qemu_get_betls(f, &env->npc);
-    qemu_get_betls(f, &env->y);
-    tmp = qemu_get_be32(f);
-    env->cwp = 0; /* needed to ensure that the wrapping registers are
-                     correctly updated */
-    cpu_put_psr(env, tmp);
-    qemu_get_betls(f, &env->fsr);
-    qemu_get_betls(f, &env->tbr);
-    tmp = qemu_get_be32(f);
-    env->interrupt_index = tmp;
-    qemu_get_be32s(f, &env->pil_in);
+static const VMStateInfo vmstate_psr = {
+    .name = "psr",
+    .get = get_psr,
+    .put = put_psr,
+};
+
+static void cpu_pre_save(void *opaque)
+{
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+
+    /* if env->cwp == env->nwindows - 1, this will set the ins of the last
+     * window as the outs of the first window
+     */
+    cpu_set_cwp(env, env->cwp);
+}
+
+/* 32-bit SPARC retains migration compatibility with older versions
+ * of QEMU; 64-bit SPARC has had a migration break since then, so the
+ * versions are different.
+ */
 #ifndef TARGET_SPARC64
-    qemu_get_be32s(f, &env->wim);
-    /* MMU */
-    for (i = 0; i < 32; i++)
-        qemu_get_be32s(f, &env->mmuregs[i]);
-    for (i = 0; i < 4; i++) {
-        qemu_get_be64s(f, &env->mxccdata[i]);
-    }
-    for (i = 0; i < 8; i++) {
-        qemu_get_be64s(f, &env->mxccregs[i]);
-    }
-    qemu_get_be32s(f, &env->mmubpctrv);
-    qemu_get_be32s(f, &env->mmubpctrc);
-    qemu_get_be32s(f, &env->mmubpctrs);
-    qemu_get_be64s(f, &env->mmubpaction);
-    for (i = 0; i < 4; i++) {
-        qemu_get_be64s(f, &env->mmubpregs[i]);
-    }
+#define SPARC_VMSTATE_VER 7
 #else
-    qemu_get_be64s(f, &env->lsu);
-    for (i = 0; i < 16; i++) {
-        qemu_get_be64s(f, &env->immuregs[i]);
-        qemu_get_be64s(f, &env->dmmuregs[i]);
-    }
-    for (i = 0; i < 64; i++) {
-        qemu_get_be64s(f, &env->itlb[i].tag);
-        qemu_get_be64s(f, &env->itlb[i].tte);
-        qemu_get_be64s(f, &env->dtlb[i].tag);
-        qemu_get_be64s(f, &env->dtlb[i].tte);
-    }
-    qemu_get_be32s(f, &env->mmu_version);
-    for (i = 0; i < MAXTL_MAX; i++) {
-        qemu_get_be64s(f, &env->ts[i].tpc);
-        qemu_get_be64s(f, &env->ts[i].tnpc);
-        qemu_get_be64s(f, &env->ts[i].tstate);
-        qemu_get_be32s(f, &env->ts[i].tt);
-    }
-    qemu_get_be32s(f, &env->xcc);
-    qemu_get_be32s(f, &env->asi);
-    qemu_get_be32s(f, &env->pstate);
-    qemu_get_be32s(f, &env->tl);
-    qemu_get_be32s(f, &env->cansave);
-    qemu_get_be32s(f, &env->canrestore);
-    qemu_get_be32s(f, &env->otherwin);
-    qemu_get_be32s(f, &env->wstate);
-    qemu_get_be32s(f, &env->cleanwin);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->agregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->bgregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->igregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->mgregs[i]);
-    qemu_get_be64s(f, &env->fprs);
-    qemu_get_be64s(f, &env->tick_cmpr);
-    qemu_get_be64s(f, &env->stick_cmpr);
-    cpu_get_timer(f, env->tick);
-    cpu_get_timer(f, env->stick);
-    qemu_get_be64s(f, &env->gsr);
-    qemu_get_be32s(f, &env->gl);
-    qemu_get_be64s(f, &env->hpstate);
-    for (i = 0; i < MAXTL_MAX; i++)
-        qemu_get_be64s(f, &env->htstate[i]);
-    qemu_get_be64s(f, &env->hintp);
-    qemu_get_be64s(f, &env->htba);
-    qemu_get_be64s(f, &env->hver);
-    qemu_get_be64s(f, &env->hstick_cmpr);
-    qemu_get_be64s(f, &env->ssr);
-    cpu_get_timer(f, env->hstick);
+#define SPARC_VMSTATE_VER 9
 #endif
-    tlb_flush(CPU(cpu), 1);
-    return 0;
-}
+
+const VMStateDescription vmstate_sparc_cpu = {
+    .name = "cpu",
+    .version_id = SPARC_VMSTATE_VER,
+    .minimum_version_id = SPARC_VMSTATE_VER,
+    .minimum_version_id_old = SPARC_VMSTATE_VER,
+    .pre_save = cpu_pre_save,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL_ARRAY(env.gregs, SPARCCPU, 8),
+        VMSTATE_UINT32(env.nwindows, SPARCCPU),
+        VMSTATE_VARRAY_MULTIPLY(env.regbase, SPARCCPU, env.nwindows, 16,
+                                vmstate_info_uinttl, target_ulong),
+        VMSTATE_CPUDOUBLE_ARRAY(env.fpr, SPARCCPU, TARGET_DPREGS),
+        VMSTATE_UINTTL(env.pc, SPARCCPU),
+        VMSTATE_UINTTL(env.npc, SPARCCPU),
+        VMSTATE_UINTTL(env.y, SPARCCPU),
+        {
+
+            .name = "psr",
+            .version_id = 0,
+            .size = sizeof(uint32_t),
+            .info = &vmstate_psr,
+            .flags = VMS_SINGLE,
+            .offset = 0,
+        },
+        VMSTATE_UINTTL(env.fsr, SPARCCPU),
+        VMSTATE_UINTTL(env.tbr, SPARCCPU),
+        VMSTATE_INT32(env.interrupt_index, SPARCCPU),
+        VMSTATE_UINT32(env.pil_in, SPARCCPU),
+#ifndef TARGET_SPARC64
+        /* MMU */
+        VMSTATE_UINT32(env.wim, SPARCCPU),
+        VMSTATE_UINT32_ARRAY(env.mmuregs, SPARCCPU, 32),
+        VMSTATE_UINT64_ARRAY(env.mxccdata, SPARCCPU, 4),
+        VMSTATE_UINT64_ARRAY(env.mxccregs, SPARCCPU, 8),
+        VMSTATE_UINT32(env.mmubpctrv, SPARCCPU),
+        VMSTATE_UINT32(env.mmubpctrc, SPARCCPU),
+        VMSTATE_UINT32(env.mmubpctrs, SPARCCPU),
+        VMSTATE_UINT64(env.mmubpaction, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.mmubpregs, SPARCCPU, 4),
+#else
+        VMSTATE_UINT64(env.lsu, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.immuregs, SPARCCPU, 16),
+        VMSTATE_UINT64_ARRAY(env.dmmuregs, SPARCCPU, 16),
+        VMSTATE_STRUCT_ARRAY(env.itlb, SPARCCPU, 64, 0,
+                             vmstate_tlb_entry, SparcTLBEntry),
+        VMSTATE_STRUCT_ARRAY(env.dtlb, SPARCCPU, 64, 0,
+                             vmstate_tlb_entry, SparcTLBEntry),
+        VMSTATE_UINT32(env.mmu_version, SPARCCPU),
+        VMSTATE_STRUCT_ARRAY(env.ts, SPARCCPU, MAXTL_MAX, 0,
+                             vmstate_trap_state, trap_state),
+        VMSTATE_UINT32(env.xcc, SPARCCPU),
+        VMSTATE_UINT32(env.asi, SPARCCPU),
+        VMSTATE_UINT32(env.pstate, SPARCCPU),
+        VMSTATE_UINT32(env.tl, SPARCCPU),
+        VMSTATE_UINT32(env.cansave, SPARCCPU),
+        VMSTATE_UINT32(env.canrestore, SPARCCPU),
+        VMSTATE_UINT32(env.otherwin, SPARCCPU),
+        VMSTATE_UINT32(env.wstate, SPARCCPU),
+        VMSTATE_UINT32(env.cleanwin, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.agregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.bgregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.igregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.mgregs, SPARCCPU, 8),
+        VMSTATE_UINT64(env.fprs, SPARCCPU),
+        VMSTATE_UINT64(env.tick_cmpr, SPARCCPU),
+        VMSTATE_UINT64(env.stick_cmpr, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.tick, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.stick, SPARCCPU),
+        VMSTATE_UINT64(env.gsr, SPARCCPU),
+        VMSTATE_UINT32(env.gl, SPARCCPU),
+        VMSTATE_UINT64(env.hpstate, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.htstate, SPARCCPU, MAXTL_MAX),
+        VMSTATE_UINT64(env.hintp, SPARCCPU),
+        VMSTATE_UINT64(env.htba, SPARCCPU),
+        VMSTATE_UINT64(env.hver, SPARCCPU),
+        VMSTATE_UINT64(env.hstick_cmpr, SPARCCPU),
+        VMSTATE_UINT64(env.ssr, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.hstick, SPARCCPU),
+        /* On SPARC32 env.psrpil and env.cwp are migrated as part of the PSR */
+        VMSTATE_UINT32(env.psrpil, SPARCCPU),
+        VMSTATE_UINT32(env.cwp, SPARCCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    },
+};
diff --git a/qemu/target-sparc/mmu_helper.c b/qemu/target-sparc/mmu_helper.c
index 2a0c6f0d3..aa80c4829 100644
--- a/qemu/target-sparc/mmu_helper.c
+++ b/qemu/target-sparc/mmu_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "exec/address-spaces.h"
@@ -849,7 +850,7 @@ hwaddr sparc_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
     SPARCCPU *cpu = SPARC_CPU(cs);
     CPUSPARCState *env = &cpu->env;
     hwaddr phys_addr;
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
     MemoryRegionSection section;
 
     if (cpu_sparc_get_phys_page(env, &phys_addr, addr, 2, mmu_idx) != 0) {
diff --git a/qemu/target-sparc/monitor.c b/qemu/target-sparc/monitor.c
new file mode 100644
index 000000000..7cc1b0f87
--- /dev/null
+++ b/qemu/target-sparc/monitor.c
@@ -0,0 +1,159 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+#ifndef TARGET_SPARC64
+static target_long monitor_get_psr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+
+    return cpu_get_psr(env);
+}
+#endif
+
+static target_long monitor_get_reg(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->regwptr[val];
+}
+
+const MonitorDef monitor_defs[] = {
+    { "g0", offsetof(CPUSPARCState, gregs[0]) },
+    { "g1", offsetof(CPUSPARCState, gregs[1]) },
+    { "g2", offsetof(CPUSPARCState, gregs[2]) },
+    { "g3", offsetof(CPUSPARCState, gregs[3]) },
+    { "g4", offsetof(CPUSPARCState, gregs[4]) },
+    { "g5", offsetof(CPUSPARCState, gregs[5]) },
+    { "g6", offsetof(CPUSPARCState, gregs[6]) },
+    { "g7", offsetof(CPUSPARCState, gregs[7]) },
+    { "o0", 0, monitor_get_reg },
+    { "o1", 1, monitor_get_reg },
+    { "o2", 2, monitor_get_reg },
+    { "o3", 3, monitor_get_reg },
+    { "o4", 4, monitor_get_reg },
+    { "o5", 5, monitor_get_reg },
+    { "o6", 6, monitor_get_reg },
+    { "o7", 7, monitor_get_reg },
+    { "l0", 8, monitor_get_reg },
+    { "l1", 9, monitor_get_reg },
+    { "l2", 10, monitor_get_reg },
+    { "l3", 11, monitor_get_reg },
+    { "l4", 12, monitor_get_reg },
+    { "l5", 13, monitor_get_reg },
+    { "l6", 14, monitor_get_reg },
+    { "l7", 15, monitor_get_reg },
+    { "i0", 16, monitor_get_reg },
+    { "i1", 17, monitor_get_reg },
+    { "i2", 18, monitor_get_reg },
+    { "i3", 19, monitor_get_reg },
+    { "i4", 20, monitor_get_reg },
+    { "i5", 21, monitor_get_reg },
+    { "i6", 22, monitor_get_reg },
+    { "i7", 23, monitor_get_reg },
+    { "pc", offsetof(CPUSPARCState, pc) },
+    { "npc", offsetof(CPUSPARCState, npc) },
+    { "y", offsetof(CPUSPARCState, y) },
+#ifndef TARGET_SPARC64
+    { "psr", 0, &monitor_get_psr, },
+    { "wim", offsetof(CPUSPARCState, wim) },
+#endif
+    { "tbr", offsetof(CPUSPARCState, tbr) },
+    { "fsr", offsetof(CPUSPARCState, fsr) },
+    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
+    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
+    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
+    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
+    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
+    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
+    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
+    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
+    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
+    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
+    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
+    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
+    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
+    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
+    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
+    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
+    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
+    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
+    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
+    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
+    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
+    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
+    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
+    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
+    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
+    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
+    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
+    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
+    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
+    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
+    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
+    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
+#ifdef TARGET_SPARC64
+    { "f32", offsetof(CPUSPARCState, fpr[16]) },
+    { "f34", offsetof(CPUSPARCState, fpr[17]) },
+    { "f36", offsetof(CPUSPARCState, fpr[18]) },
+    { "f38", offsetof(CPUSPARCState, fpr[19]) },
+    { "f40", offsetof(CPUSPARCState, fpr[20]) },
+    { "f42", offsetof(CPUSPARCState, fpr[21]) },
+    { "f44", offsetof(CPUSPARCState, fpr[22]) },
+    { "f46", offsetof(CPUSPARCState, fpr[23]) },
+    { "f48", offsetof(CPUSPARCState, fpr[24]) },
+    { "f50", offsetof(CPUSPARCState, fpr[25]) },
+    { "f52", offsetof(CPUSPARCState, fpr[26]) },
+    { "f54", offsetof(CPUSPARCState, fpr[27]) },
+    { "f56", offsetof(CPUSPARCState, fpr[28]) },
+    { "f58", offsetof(CPUSPARCState, fpr[29]) },
+    { "f60", offsetof(CPUSPARCState, fpr[30]) },
+    { "f62", offsetof(CPUSPARCState, fpr[31]) },
+    { "asi", offsetof(CPUSPARCState, asi) },
+    { "pstate", offsetof(CPUSPARCState, pstate) },
+    { "cansave", offsetof(CPUSPARCState, cansave) },
+    { "canrestore", offsetof(CPUSPARCState, canrestore) },
+    { "otherwin", offsetof(CPUSPARCState, otherwin) },
+    { "wstate", offsetof(CPUSPARCState, wstate) },
+    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
+    { "fprs", offsetof(CPUSPARCState, fprs) },
+#endif
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
diff --git a/qemu/target-sparc/translate.c b/qemu/target-sparc/translate.c
index c58dd4e95..7998ff57b 100644
--- a/qemu/target-sparc/translate.c
+++ b/qemu/target-sparc/translate.c
@@ -18,11 +18,7 @@
    License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "disas/disas.h"
@@ -33,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define DEBUG_DISAS
@@ -42,11 +39,13 @@
                          according to jump_pc[T2] */
 
 /* global register indexes */
-static TCGv_ptr cpu_env, cpu_regwptr;
+static TCGv_env cpu_env;
+static TCGv_ptr cpu_regwptr;
 static TCGv cpu_cc_src, cpu_cc_src2, cpu_cc_dst;
 static TCGv_i32 cpu_cc_op;
 static TCGv_i32 cpu_psr;
-static TCGv cpu_fsr, cpu_pc, cpu_npc, cpu_gregs[8];
+static TCGv cpu_fsr, cpu_pc, cpu_npc;
+static TCGv cpu_regs[32];
 static TCGv cpu_y;
 #ifndef CONFIG_USER_ONLY
 static TCGv cpu_tbr;
@@ -64,9 +63,6 @@ static TCGv cpu_wim;
 /* Floating point registers */
 static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 
-static target_ulong gen_opc_npc[OPC_BUF_SIZE];
-static target_ulong gen_opc_jump_pc[2];
-
 #include "exec/gen-icount.h"
 
 typedef struct DisasContext {
@@ -164,7 +160,7 @@ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src)
         TCGv_i64 t = tcg_temp_new_i64();
 
         tcg_gen_shri_i64(t, cpu_fpr[src / 2], 32);
-        tcg_gen_trunc_i64_i32(ret, t);
+        tcg_gen_extrl_i64_i32(ret, t);
         tcg_temp_free_i64(t);
 
         return ret;
@@ -279,36 +275,31 @@ static inline void gen_address_mask(DisasContext *dc, TCGv addr)
 
 static inline TCGv gen_load_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0 || reg >= 8) {
+    if (reg > 0) {
+        assert(reg < 32);
+        return cpu_regs[reg];
+    } else {
         TCGv t = get_temp_tl(dc);
-        if (reg == 0) {
-            tcg_gen_movi_tl(t, 0);
-        } else {
-            tcg_gen_ld_tl(t, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
-        }
+        tcg_gen_movi_tl(t, 0);
         return t;
-    } else {
-        return cpu_gregs[reg];
     }
 }
 
 static inline void gen_store_gpr(DisasContext *dc, int reg, TCGv v)
 {
     if (reg > 0) {
-        if (reg < 8) {
-            tcg_gen_mov_tl(cpu_gregs[reg], v);
-        } else {
-            tcg_gen_st_tl(v, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
-        }
+        assert(reg < 32);
+        tcg_gen_mov_tl(cpu_regs[reg], v);
     }
 }
 
 static inline TCGv gen_dest_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0 || reg >= 8) {
-        return get_temp_tl(dc);
+    if (reg > 0) {
+        assert(reg < 32);
+        return cpu_regs[reg];
     } else {
-        return cpu_gregs[reg];
+        return get_temp_tl(dc);
     }
 }
 
@@ -379,8 +370,8 @@ static TCGv_i32 gen_add32_carry32(void)
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_dst);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_dst);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src);
 #else
     cc_src1_32 = cpu_cc_dst;
     cc_src2_32 = cpu_cc_src;
@@ -405,8 +396,8 @@ static TCGv_i32 gen_sub32_carry32(void)
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_src);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src2);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src2);
 #else
     cc_src1_32 = cpu_cc_src;
     cc_src2_32 = cpu_cc_src2;
@@ -955,17 +946,44 @@ static inline void gen_branch2(DisasContext *dc, target_ulong pc1,
     gen_goto_tb(dc, 1, pc2, pc2 + 4);
 }
 
-static inline void gen_branch_a(DisasContext *dc, target_ulong pc1,
-                                target_ulong pc2, TCGv r_cond)
+static void gen_branch_a(DisasContext *dc, target_ulong pc1)
 {
     TCGLabel *l1 = gen_new_label();
+    target_ulong npc = dc->npc;
 
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_cond, 0, l1);
 
-    gen_goto_tb(dc, 0, pc2, pc1);
+    gen_goto_tb(dc, 0, npc, pc1);
 
     gen_set_label(l1);
-    gen_goto_tb(dc, 1, pc2 + 4, pc2 + 8);
+    gen_goto_tb(dc, 1, npc + 4, npc + 8);
+
+    dc->is_br = 1;
+}
+
+static void gen_branch_n(DisasContext *dc, target_ulong pc1)
+{
+    target_ulong npc = dc->npc;
+
+    if (likely(npc != DYNAMIC_PC)) {
+        dc->pc = npc;
+        dc->jump_pc[0] = pc1;
+        dc->jump_pc[1] = npc + 4;
+        dc->npc = JUMP_PC;
+    } else {
+        TCGv t, z;
+
+        tcg_gen_mov_tl(cpu_pc, cpu_npc);
+
+        tcg_gen_addi_tl(cpu_npc, cpu_npc, 4);
+        t = tcg_const_tl(pc1);
+        z = tcg_const_tl(0);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_npc, cpu_cond, z, t, cpu_npc);
+        tcg_temp_free(t);
+        tcg_temp_free(z);
+
+        dc->pc = DYNAMIC_PC;
+    }
 }
 
 static inline void gen_generic_branch(DisasContext *dc)
@@ -1398,18 +1416,9 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
         flush_cond(dc);
         gen_cond(cpu_cond, cc, cond, dc);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1447,18 +1456,9 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
         flush_cond(dc);
         gen_fcond(cpu_cond, cc, cond);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1476,18 +1476,9 @@ static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
     flush_cond(dc);
     gen_cond_reg(cpu_cond, cond, r_reg);
     if (a) {
-        gen_branch_a(dc, target, dc->npc, cpu_cond);
-        dc->is_br = 1;
+        gen_branch_a(dc, target);
     } else {
-        dc->pc = dc->npc;
-        dc->jump_pc[0] = target;
-        if (unlikely(dc->npc == DYNAMIC_PC)) {
-            dc->jump_pc[1] = DYNAMIC_PC;
-            tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-        } else {
-            dc->jump_pc[1] = dc->npc + 4;
-            dc->npc = JUMP_PC;
-        }
+        gen_branch_n(dc, target);
     }
 }
 
@@ -2164,9 +2155,13 @@ static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
     tcg_temp_free_i32(r_size);
     tcg_temp_free_i32(r_asi);
 
-    t = gen_dest_gpr(dc, rd + 1);
+    /* ??? Work around an apparent bug in Ubuntu gcc 4.8.2-10ubuntu2+12,
+       whereby "rd + 1" elicits "error: array subscript is above array".
+       Since we have already asserted that rd is even, the semantics
+       are unchanged.  */
+    t = gen_dest_gpr(dc, rd | 1);
     tcg_gen_trunc_i64_tl(t, t64);
-    gen_store_gpr(dc, rd + 1, t);
+    gen_store_gpr(dc, rd | 1, t);
 
     tcg_gen_shri_i64(t64, t64, 32);
     tcg_gen_trunc_i64_tl(hi, t64);
@@ -2254,11 +2249,11 @@ static void gen_fmovs(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
        the later.  */
     c32 = tcg_temp_new_i32();
     if (cmp->is_bool) {
-        tcg_gen_trunc_i64_i32(c32, cmp->c1);
+        tcg_gen_extrl_i64_i32(c32, cmp->c1);
     } else {
         TCGv_i64 c64 = tcg_temp_new_i64();
         tcg_gen_setcond_i64(cmp->cond, c64, cmp->c1, cmp->c2);
-        tcg_gen_trunc_i64_i32(c32, c64);
+        tcg_gen_extrl_i64_i32(c32, c64);
         tcg_temp_free_i64(c64);
     }
 
@@ -2297,7 +2292,7 @@ static void gen_fmovq(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
 }
 
 #ifndef CONFIG_USER_ONLY
-static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_ptr cpu_env)
+static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_env cpu_env)
 {
     TCGv_i32 r_tl = tcg_temp_new_i32();
 
@@ -2482,10 +2477,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     TCGv_i64 cpu_src1_64, cpu_src2_64, cpu_dst_64;
     target_long simm;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     opc = GET_FIELD(insn, 0, 1);
     rd = GET_FIELD(insn, 2, 6);
 
@@ -2715,12 +2706,16 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 case 0x4: /* V9 rdtick */
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
-                        gen_helper_tick_get_count(cpu_dst, r_tickptr);
+                        gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr,
+                                                  r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                         gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
@@ -2757,12 +2752,16 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 case 0x18: /* System tick */
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, stick));
-                        gen_helper_tick_get_count(cpu_dst, r_tickptr);
+                        gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr,
+                                                  r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                         gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
@@ -2870,12 +2869,16 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 case 4: // tick
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
-                        gen_helper_tick_get_count(cpu_tmp0, r_tickptr);
+                        gen_helper_tick_get_count(cpu_tmp0, cpu_env,
+                                                  r_tickptr, r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                     }
                     break;
                 case 5: // tba
@@ -4667,7 +4670,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         TCGv r_const;
 
                         gen_address_mask(dc, cpu_addr);
-                        tcg_gen_qemu_ld8s(cpu_val, cpu_addr, dc->mem_idx);
+                        tcg_gen_qemu_ld8u(cpu_val, cpu_addr, dc->mem_idx);
                         r_const = tcg_const_tl(0xff);
                         tcg_gen_qemu_st8(r_const, cpu_addr, dc->mem_idx);
                         tcg_temp_free(r_const);
@@ -5104,7 +5107,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (IS_IMM) {
                         goto illegal_insn;
                     }
-                    if (!supervisor(dc)) {
+                    /* LEON3 allows CASA from user space with ASI 0xa */
+                    if ((GET_FIELD(insn, 19, 26) != 0xa) && !supervisor(dc)) {
                         goto priv_insn;
                     }
 #endif
@@ -5213,16 +5217,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     }
 }
 
-static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool spc)
+void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
 {
+    SPARCCPU *cpu = sparc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSPARCState *env = &cpu->env;
     target_ulong pc_start, last_pc;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
     unsigned int insn;
@@ -5234,7 +5234,7 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
     last_pc = dc->pc;
     dc->npc = (target_ulong) tb->cs_base;
     dc->cc_op = CC_OP_DYNAMIC;
-    dc->mem_idx = cpu_mmu_index(env);
+    dc->mem_idx = cpu_mmu_index(env, false);
     dc->def = env->def;
     dc->fpu_enabled = tb_fpu_enabled(tb->flags);
     dc->address_mask_32bit = tb_am_enabled(tb->flags);
@@ -5242,42 +5242,41 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
 
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    if (dc->pc != pc_start)
-                        save_state(dc);
-                    gen_helper_debug(cpu_env);
-                    tcg_gen_exit_tb(0);
-                    dc->is_br = 1;
-                    goto exit_gen_loop;
-                }
-            }
+        if (dc->npc & JUMP_PC) {
+            assert(dc->jump_pc[1] == dc->pc + 4);
+            tcg_gen_insn_start(dc->pc, dc->jump_pc[0] | JUMP_PC);
+        } else {
+            tcg_gen_insn_start(dc->pc, dc->npc);
         }
-        if (spc) {
-            qemu_log("Search PC...\n");
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-                gen_opc_npc[lj] = dc->npc;
-                tcg_ctx.gen_opc_instr_start[lj] = 1;
-                tcg_ctx.gen_opc_icount[lj] = num_insns;
+        num_insns++;
+        last_pc = dc->pc;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            if (dc->pc != pc_start) {
+                save_state(dc);
             }
+            gen_helper_debug(cpu_env);
+            tcg_gen_exit_tb(0);
+            dc->is_br = 1;
+            goto exit_gen_loop;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-        last_pc = dc->pc;
+        }
+
         insn = cpu_ldl_code(env, dc->pc);
 
         disas_sparc_insn(dc, insn);
-        num_insns++;
 
         if (dc->is_br)
             break;
@@ -5316,20 +5315,9 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
     }
     gen_tb_end(tb, num_insns);
 
-    if (spc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-#if 0
-        log_page_dump();
-#endif
-        gen_opc_jump_pc[0] = dc->jump_pc[0];
-        gen_opc_jump_pc[1] = dc->jump_pc[1];
-    } else {
-        tb->size = last_pc + 4 - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = last_pc + 4 - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("--------------\n");
@@ -5340,130 +5328,118 @@ static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
 #endif
 }
 
-void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, true);
-}
-
 void gen_intermediate_code_init(CPUSPARCState *env)
 {
-    unsigned int i;
     static int inited;
-    static const char * const gregnames[8] = {
-        NULL, // g0 not used
-        "g1",
-        "g2",
-        "g3",
-        "g4",
-        "g5",
-        "g6",
-        "g7",
+    static const char gregnames[32][4] = {
+        "g0", "g1", "g2", "g3", "g4", "g5", "g6", "g7",
+        "o0", "o1", "o2", "o3", "o4", "o5", "o6", "o7",
+        "l0", "l1", "l2", "l3", "l4", "l5", "l6", "l7",
+        "i0", "i1", "i2", "i3", "i4", "i5", "i6", "i7",
     };
-    static const char * const fregnames[32] = {
+    static const char fregnames[32][4] = {
         "f0", "f2", "f4", "f6", "f8", "f10", "f12", "f14",
         "f16", "f18", "f20", "f22", "f24", "f26", "f28", "f30",
         "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",
         "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",
     };
 
-    /* init various static tables */
-    if (!inited) {
-        inited = 1;
-
-        cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-        cpu_regwptr = tcg_global_mem_new_ptr(TCG_AREG0,
-                                             offsetof(CPUSPARCState, regwptr),
-                                             "regwptr");
+    static const struct { TCGv_i32 *ptr; int off; const char *name; } r32[] = {
 #ifdef TARGET_SPARC64
-        cpu_xcc = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, xcc),
-                                         "xcc");
-        cpu_asi = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, asi),
-                                         "asi");
-        cpu_fprs = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, fprs),
-                                          "fprs");
-        cpu_gsr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, gsr),
-                                     "gsr");
-        cpu_tick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                           offsetof(CPUSPARCState, tick_cmpr),
-                                           "tick_cmpr");
-        cpu_stick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                            offsetof(CPUSPARCState, stick_cmpr),
-                                            "stick_cmpr");
-        cpu_hstick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                             offsetof(CPUSPARCState, hstick_cmpr),
-                                             "hstick_cmpr");
-        cpu_hintp = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, hintp),
-                                       "hintp");
-        cpu_htba = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, htba),
-                                      "htba");
-        cpu_hver = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, hver),
-                                      "hver");
-        cpu_ssr = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUSPARCState, ssr), "ssr");
-        cpu_ver = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUSPARCState, version), "ver");
-        cpu_softint = tcg_global_mem_new_i32(TCG_AREG0,
-                                             offsetof(CPUSPARCState, softint),
-                                             "softint");
+        { &cpu_xcc, offsetof(CPUSPARCState, xcc), "xcc" },
+        { &cpu_asi, offsetof(CPUSPARCState, asi), "asi" },
+        { &cpu_fprs, offsetof(CPUSPARCState, fprs), "fprs" },
+        { &cpu_softint, offsetof(CPUSPARCState, softint), "softint" },
 #else
-        cpu_wim = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, wim),
-                                     "wim");
+        { &cpu_wim, offsetof(CPUSPARCState, wim), "wim" },
 #endif
-        cpu_cond = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cond),
-                                      "cond");
-        cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cc_src),
-                                        "cc_src");
-        cpu_cc_src2 = tcg_global_mem_new(TCG_AREG0,
-                                         offsetof(CPUSPARCState, cc_src2),
-                                         "cc_src2");
-        cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cc_dst),
-                                        "cc_dst");
-        cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, cc_op),
-                                           "cc_op");
-        cpu_psr = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, psr),
-                                         "psr");
-        cpu_fsr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, fsr),
-                                     "fsr");
-        cpu_pc = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, pc),
-                                    "pc");
-        cpu_npc = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, npc),
-                                     "npc");
-        cpu_y = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, y), "y");
+        { &cpu_cc_op, offsetof(CPUSPARCState, cc_op), "cc_op" },
+        { &cpu_psr, offsetof(CPUSPARCState, psr), "psr" },
+    };
+
+    static const struct { TCGv *ptr; int off; const char *name; } rtl[] = {
+#ifdef TARGET_SPARC64
+        { &cpu_gsr, offsetof(CPUSPARCState, gsr), "gsr" },
+        { &cpu_tick_cmpr, offsetof(CPUSPARCState, tick_cmpr), "tick_cmpr" },
+        { &cpu_stick_cmpr, offsetof(CPUSPARCState, stick_cmpr), "stick_cmpr" },
+        { &cpu_hstick_cmpr, offsetof(CPUSPARCState, hstick_cmpr),
+          "hstick_cmpr" },
+        { &cpu_hintp, offsetof(CPUSPARCState, hintp), "hintp" },
+        { &cpu_htba, offsetof(CPUSPARCState, htba), "htba" },
+        { &cpu_hver, offsetof(CPUSPARCState, hver), "hver" },
+        { &cpu_ssr, offsetof(CPUSPARCState, ssr), "ssr" },
+        { &cpu_ver, offsetof(CPUSPARCState, version), "ver" },
+#endif
+        { &cpu_cond, offsetof(CPUSPARCState, cond), "cond" },
+        { &cpu_cc_src, offsetof(CPUSPARCState, cc_src), "cc_src" },
+        { &cpu_cc_src2, offsetof(CPUSPARCState, cc_src2), "cc_src2" },
+        { &cpu_cc_dst, offsetof(CPUSPARCState, cc_dst), "cc_dst" },
+        { &cpu_fsr, offsetof(CPUSPARCState, fsr), "fsr" },
+        { &cpu_pc, offsetof(CPUSPARCState, pc), "pc" },
+        { &cpu_npc, offsetof(CPUSPARCState, npc), "npc" },
+        { &cpu_y, offsetof(CPUSPARCState, y), "y" },
 #ifndef CONFIG_USER_ONLY
-        cpu_tbr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, tbr),
-                                     "tbr");
+        { &cpu_tbr, offsetof(CPUSPARCState, tbr), "tbr" },
 #endif
-        for (i = 1; i < 8; i++) {
-            cpu_gregs[i] = tcg_global_mem_new(TCG_AREG0,
-                                              offsetof(CPUSPARCState, gregs[i]),
-                                              gregnames[i]);
-        }
-        for (i = 0; i < TARGET_DPREGS; i++) {
-            cpu_fpr[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                                offsetof(CPUSPARCState, fpr[i]),
-                                                fregnames[i]);
-        }
+    };
+
+    unsigned int i;
+
+    /* init various static tables */
+    if (inited) {
+        return;
+    }
+    inited = 1;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+
+    cpu_regwptr = tcg_global_mem_new_ptr(cpu_env,
+                                         offsetof(CPUSPARCState, regwptr),
+                                         "regwptr");
+
+    for (i = 0; i < ARRAY_SIZE(r32); ++i) {
+        *r32[i].ptr = tcg_global_mem_new_i32(cpu_env, r32[i].off, r32[i].name);
+    }
+
+    for (i = 0; i < ARRAY_SIZE(rtl); ++i) {
+        *rtl[i].ptr = tcg_global_mem_new(cpu_env, rtl[i].off, rtl[i].name);
+    }
+
+    TCGV_UNUSED(cpu_regs[0]);
+    for (i = 1; i < 8; ++i) {
+        cpu_regs[i] = tcg_global_mem_new(cpu_env,
+                                         offsetof(CPUSPARCState, gregs[i]),
+                                         gregnames[i]);
+    }
+
+    for (i = 8; i < 32; ++i) {
+        cpu_regs[i] = tcg_global_mem_new(cpu_regwptr,
+                                         (i - 8) * sizeof(target_ulong),
+                                         gregnames[i]);
+    }
+
+    for (i = 0; i < TARGET_DPREGS; i++) {
+        cpu_fpr[i] = tcg_global_mem_new_i64(cpu_env,
+                                            offsetof(CPUSPARCState, fpr[i]),
+                                            fregnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    target_ulong npc;
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    npc = gen_opc_npc[pc_pos];
-    if (npc == 1) {
+    target_ulong pc = data[0];
+    target_ulong npc = data[1];
+
+    env->pc = pc;
+    if (npc == DYNAMIC_PC) {
         /* dynamic NPC: already stored */
-    } else if (npc == 2) {
+    } else if (npc & JUMP_PC) {
         /* jump PC: use 'cond' and the jump targets of the translation */
         if (env->cond) {
-            env->npc = gen_opc_jump_pc[0];
+            env->npc = npc & ~3;
         } else {
-            env->npc = gen_opc_jump_pc[1];
+            env->npc = pc + 4;
         }
     } else {
         env->npc = npc;
diff --git a/qemu/target-sparc/vis_helper.c b/qemu/target-sparc/vis_helper.c
index 383cc8bdf..8a9b763d0 100644
--- a/qemu/target-sparc/vis_helper.c
+++ b/qemu/target-sparc/vis_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -447,7 +448,7 @@ uint32_t helper_fpackfix(uint64_t gsr, uint64_t rs2)
     for (word = 0; word < 2; word++) {
         uint32_t val;
         int32_t src = rs2 >> (word * 32);
-        int64_t scaled = src << scale;
+        int64_t scaled = (int64_t)src << scale;
         int64_t from_fixed = scaled >> 16;
 
         val = (from_fixed < -32768 ? -32768 :
diff --git a/qemu/target-sparc/win_helper.c b/qemu/target-sparc/win_helper.c
index f01ae08f6..a8a6c0cfc 100644
--- a/qemu/target-sparc/win_helper.c
+++ b/qemu/target-sparc/win_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "trace.h"
@@ -64,23 +65,28 @@ target_ulong cpu_get_psr(CPUSPARCState *env)
 #endif
 }
 
-void cpu_put_psr(CPUSPARCState *env, target_ulong val)
+void cpu_put_psr_raw(CPUSPARCState *env, target_ulong val)
 {
     env->psr = val & PSR_ICC;
 #if !defined(TARGET_SPARC64)
     env->psref = (val & PSR_EF) ? 1 : 0;
     env->psrpil = (val & PSR_PIL) >> 8;
-#endif
-#if ((!defined(TARGET_SPARC64)) && !defined(CONFIG_USER_ONLY))
-    cpu_check_irqs(env);
-#endif
-#if !defined(TARGET_SPARC64)
     env->psrs = (val & PSR_S) ? 1 : 0;
     env->psrps = (val & PSR_PS) ? 1 : 0;
     env->psret = (val & PSR_ET) ? 1 : 0;
-    cpu_set_cwp(env, val & PSR_CWP);
 #endif
     env->cc_op = CC_OP_FLAGS;
+#if !defined(TARGET_SPARC64)
+    cpu_set_cwp(env, val & PSR_CWP);
+#endif
+}
+
+void cpu_put_psr(CPUSPARCState *env, target_ulong val)
+{
+    cpu_put_psr_raw(env, val);
+#if ((!defined(TARGET_SPARC64)) && !defined(CONFIG_USER_ONLY))
+    cpu_check_irqs(env);
+#endif
 }
 
 int cpu_cwp_inc(CPUSPARCState *env, int cwp)
diff --git a/qemu/target-tilegx/Makefile.objs b/qemu/target-tilegx/Makefile.objs
new file mode 100644
index 000000000..0db778f40
--- /dev/null
+++ b/qemu/target-tilegx/Makefile.objs
@@ -0,0 +1 @@
+obj-y += cpu.o translate.o helper.o simd_helper.o
diff --git a/qemu/target-tilegx/cpu.c b/qemu/target-tilegx/cpu.c
new file mode 100644
index 000000000..d2d091203
--- /dev/null
+++ b/qemu/target-tilegx/cpu.c
@@ -0,0 +1,187 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu-common.h"
+#include "hw/qdev-properties.h"
+#include "migration/vmstate.h"
+#include "linux-user/syscall_defs.h"
+
+static void tilegx_cpu_dump_state(CPUState *cs, FILE *f,
+                                  fprintf_function cpu_fprintf, int flags)
+{
+    static const char * const reg_names[TILEGX_R_COUNT] = {
+         "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+         "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+        "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+        "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+        "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+        "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+        "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr"
+    };
+
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+    CPUTLGState *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_fprintf(f, "%-4s" TARGET_FMT_lx "%s",
+                    reg_names[i], env->regs[i],
+                    (i % 4) == 3 ? "\n" : " ");
+    }
+    cpu_fprintf(f, "PC  " TARGET_FMT_lx " CEX " TARGET_FMT_lx "\n\n",
+                env->pc, env->spregs[TILEGX_SPR_CMPEXCH]);
+}
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model)
+{
+    TileGXCPU *cpu;
+
+    cpu = TILEGX_CPU(object_new(TYPE_TILEGX_CPU));
+
+    object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
+
+    return cpu;
+}
+
+static void tilegx_cpu_set_pc(CPUState *cs, vaddr value)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    cpu->env.pc = value;
+}
+
+static bool tilegx_cpu_has_work(CPUState *cs)
+{
+    return true;
+}
+
+static void tilegx_cpu_reset(CPUState *s)
+{
+    TileGXCPU *cpu = TILEGX_CPU(s);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(cpu);
+    CPUTLGState *env = &cpu->env;
+
+    tcc->parent_reset(s);
+
+    memset(env, 0, sizeof(CPUTLGState));
+    tlb_flush(s, 1);
+}
+
+static void tilegx_cpu_realizefn(DeviceState *dev, Error **errp)
+{
+    CPUState *cs = CPU(dev);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(dev);
+
+    cpu_reset(cs);
+    qemu_init_vcpu(cs);
+
+    tcc->parent_realize(dev, errp);
+}
+
+static void tilegx_cpu_initfn(Object *obj)
+{
+    CPUState *cs = CPU(obj);
+    TileGXCPU *cpu = TILEGX_CPU(obj);
+    CPUTLGState *env = &cpu->env;
+    static bool tcg_initialized;
+
+    cs->env_ptr = env;
+    cpu_exec_init(cs, &error_abort);
+
+    if (tcg_enabled() && !tcg_initialized) {
+        tcg_initialized = true;
+        tilegx_tcg_init();
+    }
+}
+
+static void tilegx_cpu_do_interrupt(CPUState *cs)
+{
+    cs->exception_index = -1;
+}
+
+static int tilegx_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
+                                       int mmu_idx)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    /* The sigcode field will be filled in by do_signal in main.c.  */
+    cs->exception_index = TILEGX_EXCP_SIGNAL;
+    cpu->env.excaddr = address;
+    cpu->env.signo = TARGET_SIGSEGV;
+    cpu->env.sigcode = 0;
+
+    return 1;
+}
+
+static bool tilegx_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        tilegx_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
+static void tilegx_cpu_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
+    TileGXCPUClass *tcc = TILEGX_CPU_CLASS(oc);
+
+    tcc->parent_realize = dc->realize;
+    dc->realize = tilegx_cpu_realizefn;
+
+    tcc->parent_reset = cc->reset;
+    cc->reset = tilegx_cpu_reset;
+
+    cc->has_work = tilegx_cpu_has_work;
+    cc->do_interrupt = tilegx_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = tilegx_cpu_exec_interrupt;
+    cc->dump_state = tilegx_cpu_dump_state;
+    cc->set_pc = tilegx_cpu_set_pc;
+    cc->handle_mmu_fault = tilegx_cpu_handle_mmu_fault;
+    cc->gdb_num_core_regs = 0;
+
+    /*
+     * Reason: tilegx_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo tilegx_cpu_type_info = {
+    .name = TYPE_TILEGX_CPU,
+    .parent = TYPE_CPU,
+    .instance_size = sizeof(TileGXCPU),
+    .instance_init = tilegx_cpu_initfn,
+    .class_size = sizeof(TileGXCPUClass),
+    .class_init = tilegx_cpu_class_init,
+};
+
+static void tilegx_cpu_register_types(void)
+{
+    type_register_static(&tilegx_cpu_type_info);
+}
+
+type_init(tilegx_cpu_register_types)
diff --git a/qemu/target-tilegx/cpu.h b/qemu/target-tilegx/cpu.h
new file mode 100644
index 000000000..022cad186
--- /dev/null
+++ b/qemu/target-tilegx/cpu.h
@@ -0,0 +1,181 @@
+/*
+ *  TILE-Gx virtual CPU header
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef CPU_TILEGX_H
+#define CPU_TILEGX_H
+
+#include "qemu-common.h"
+
+#define TARGET_LONG_BITS 64
+
+#define CPUArchState struct CPUTLGState
+
+#include "exec/cpu-defs.h"
+
+
+/* TILE-Gx common register alias */
+#define TILEGX_R_RE    0   /*  0 register, for function/syscall return value */
+#define TILEGX_R_ERR   1   /*  1 register, for syscall errno flag */
+#define TILEGX_R_NR    10  /* 10 register, for syscall number */
+#define TILEGX_R_BP    52  /* 52 register, optional frame pointer */
+#define TILEGX_R_TP    53  /* TP register, thread local storage data */
+#define TILEGX_R_SP    54  /* SP register, stack pointer */
+#define TILEGX_R_LR    55  /* LR register, may save pc, but it is not pc */
+#define TILEGX_R_COUNT 56  /* Only 56 registers are really useful */
+#define TILEGX_R_SN    56  /* SN register, obsoleted, it likes zero register */
+#define TILEGX_R_IDN0  57  /* IDN0 register, cause IDN_ACCESS exception */
+#define TILEGX_R_IDN1  58  /* IDN1 register, cause IDN_ACCESS exception */
+#define TILEGX_R_UDN0  59  /* UDN0 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN1  60  /* UDN1 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN2  61  /* UDN2 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN3  62  /* UDN3 register, cause UDN_ACCESS exception */
+#define TILEGX_R_ZERO  63  /* Zero register, always zero */
+#define TILEGX_R_NOREG 255 /* Invalid register value */
+
+/* TILE-Gx special registers used by outside */
+enum {
+    TILEGX_SPR_CMPEXCH = 0,
+    TILEGX_SPR_CRITICAL_SEC = 1,
+    TILEGX_SPR_SIM_CONTROL = 2,
+    TILEGX_SPR_EX_CONTEXT_0_0 = 3,
+    TILEGX_SPR_EX_CONTEXT_0_1 = 4,
+    TILEGX_SPR_COUNT
+};
+
+/* Exception numbers */
+typedef enum {
+    TILEGX_EXCP_NONE = 0,
+    TILEGX_EXCP_SYSCALL = 1,
+    TILEGX_EXCP_SIGNAL = 2,
+    TILEGX_EXCP_OPCODE_UNKNOWN = 0x101,
+    TILEGX_EXCP_OPCODE_UNIMPLEMENTED = 0x102,
+    TILEGX_EXCP_OPCODE_CMPEXCH = 0x103,
+    TILEGX_EXCP_OPCODE_CMPEXCH4 = 0x104,
+    TILEGX_EXCP_OPCODE_EXCH = 0x105,
+    TILEGX_EXCP_OPCODE_EXCH4 = 0x106,
+    TILEGX_EXCP_OPCODE_FETCHADD = 0x107,
+    TILEGX_EXCP_OPCODE_FETCHADD4 = 0x108,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ = 0x109,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ4 = 0x10a,
+    TILEGX_EXCP_OPCODE_FETCHAND = 0x10b,
+    TILEGX_EXCP_OPCODE_FETCHAND4 = 0x10c,
+    TILEGX_EXCP_OPCODE_FETCHOR = 0x10d,
+    TILEGX_EXCP_OPCODE_FETCHOR4 = 0x10e,
+    TILEGX_EXCP_REG_IDN_ACCESS = 0x181,
+    TILEGX_EXCP_REG_UDN_ACCESS = 0x182,
+    TILEGX_EXCP_UNALIGNMENT = 0x201,
+    TILEGX_EXCP_DBUG_BREAK = 0x301
+} TileExcp;
+
+typedef struct CPUTLGState {
+    uint64_t regs[TILEGX_R_COUNT];     /* Common used registers by outside */
+    uint64_t spregs[TILEGX_SPR_COUNT]; /* Special used registers by outside */
+    uint64_t pc;                       /* Current pc */
+
+#if defined(CONFIG_USER_ONLY)
+    uint64_t excaddr;                  /* exception address */
+    uint64_t atomic_srca;              /* Arguments to atomic "exceptions" */
+    uint64_t atomic_srcb;
+    uint32_t atomic_dstr;
+    uint32_t signo;                    /* Signal number */
+    uint32_t sigcode;                  /* Signal code */
+#endif
+
+    CPU_COMMON
+} CPUTLGState;
+
+#include "qom/cpu.h"
+
+#define TYPE_TILEGX_CPU "tilegx-cpu"
+
+#define TILEGX_CPU_CLASS(klass) \
+    OBJECT_CLASS_CHECK(TileGXCPUClass, (klass), TYPE_TILEGX_CPU)
+#define TILEGX_CPU(obj) \
+    OBJECT_CHECK(TileGXCPU, (obj), TYPE_TILEGX_CPU)
+#define TILEGX_CPU_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(TileGXCPUClass, (obj), TYPE_TILEGX_CPU)
+
+/**
+ * TileGXCPUClass:
+ * @parent_realize: The parent class' realize handler.
+ * @parent_reset: The parent class' reset handler.
+ *
+ * A Tile-Gx CPU model.
+ */
+typedef struct TileGXCPUClass {
+    /*< private >*/
+    CPUClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+    void (*parent_reset)(CPUState *cpu);
+} TileGXCPUClass;
+
+/**
+ * TileGXCPU:
+ * @env: #CPUTLGState
+ *
+ * A Tile-GX CPU.
+ */
+typedef struct TileGXCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUTLGState env;
+} TileGXCPU;
+
+static inline TileGXCPU *tilegx_env_get_cpu(CPUTLGState *env)
+{
+    return container_of(env, TileGXCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(tilegx_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(TileGXCPU, env)
+
+/* TILE-Gx memory attributes */
+#define TARGET_PAGE_BITS 16  /* TILE-Gx uses 64KB page size */
+#define TARGET_PHYS_ADDR_SPACE_BITS 42
+#define TARGET_VIRT_ADDR_SPACE_BITS 64
+#define MMU_USER_IDX    0  /* Current memory operation is in user mode */
+
+#include "exec/cpu-all.h"
+
+void tilegx_tcg_init(void);
+int cpu_tilegx_exec(CPUState *s);
+int cpu_tilegx_signal_handler(int host_signum, void *pinfo, void *puc);
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model);
+
+#define cpu_init(cpu_model) CPU(cpu_tilegx_init(cpu_model))
+
+#define cpu_exec cpu_tilegx_exec
+#define cpu_signal_handler cpu_tilegx_signal_handler
+
+static inline void cpu_get_tb_cpu_state(CPUTLGState *env, target_ulong *pc,
+                                        target_ulong *cs_base, int *flags)
+{
+    *pc = env->pc;
+    *cs_base = 0;
+    *flags = 0;
+}
+
+#include "exec/exec-all.h"
+
+#endif
diff --git a/qemu/target-tilegx/helper.c b/qemu/target-tilegx/helper.c
new file mode 100644
index 000000000..616c5c7cf
--- /dev/null
+++ b/qemu/target-tilegx/helper.c
@@ -0,0 +1,162 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+#include <zlib.h> /* For crc32 */
+#include "syscall_defs.h"
+
+void helper_exception(CPUTLGState *env, uint32_t excp)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+
+    cs->exception_index = excp;
+    cpu_loop_exit(cs);
+}
+
+void helper_ext01_ics(CPUTLGState *env)
+{
+    uint64_t val = env->spregs[TILEGX_SPR_EX_CONTEXT_0_1];
+
+    switch (val) {
+    case 0:
+    case 1:
+        env->spregs[TILEGX_SPR_CRITICAL_SEC] = val;
+        break;
+    default:
+#if defined(CONFIG_USER_ONLY)
+        env->signo = TARGET_SIGILL;
+        env->sigcode = TARGET_ILL_ILLOPC;
+        helper_exception(env, TILEGX_EXCP_SIGNAL);
+#else
+        helper_exception(env, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+        break;
+    }
+}
+
+uint64_t helper_cntlz(uint64_t arg)
+{
+    return clz64(arg);
+}
+
+uint64_t helper_cnttz(uint64_t arg)
+{
+    return ctz64(arg);
+}
+
+uint64_t helper_pcnt(uint64_t arg)
+{
+    return ctpop64(arg);
+}
+
+uint64_t helper_revbits(uint64_t arg)
+{
+    return revbit64(arg);
+}
+
+/*
+ * Functional Description
+ *     uint64_t a = rf[SrcA];
+ *     uint64_t b = rf[SrcB];
+ *     uint64_t d = rf[Dest];
+ *     uint64_t output = 0;
+ *     unsigned int counter;
+ *     for (counter = 0; counter < (WORD_SIZE / BYTE_SIZE); counter++)
+ *     {
+ *         int sel = getByte (b, counter) & 0xf;
+ *         uint8_t byte = (sel < 8) ? getByte (d, sel) : getByte (a, (sel - 8));
+ *         output = setByte (output, counter, byte);
+ *     }
+ *     rf[Dest] = output;
+ */
+uint64_t helper_shufflebytes(uint64_t dest, uint64_t srca, uint64_t srcb)
+{
+    uint64_t vdst = 0;
+    int count;
+
+    for (count = 0; count < 64; count += 8) {
+        uint64_t sel = srcb >> count;
+        uint64_t src = (sel & 8) ? srca : dest;
+        vdst |= extract64(src, (sel & 7) * 8, 8) << count;
+    }
+
+    return vdst;
+}
+
+uint64_t helper_crc32_8(uint64_t accum, uint64_t input)
+{
+    uint8_t buf = input;
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, &buf, 1) ^ 0xffffffff;
+}
+
+uint64_t helper_crc32_32(uint64_t accum, uint64_t input)
+{
+    uint8_t buf[4];
+
+    stl_le_p(buf, input);
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, buf, 4) ^ 0xffffffff;
+}
+
+uint64_t helper_cmula(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = srcd;
+    uint32_t imagd = srcd >> 32;
+    uint32_t realr = reala * realb - imaga * imagb + reald;
+    uint32_t imagr = reala * imagb + imaga * realb + imagd;
+
+    return deposit64(realr, 32, 32, imagr);
+}
+
+uint64_t helper_cmulaf(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = (int16_t)srcd;
+    uint32_t imagd = (int16_t)(srcd >> 16);
+    int32_t realr = reala * realb - imaga * imagb;
+    int32_t imagr = reala * imagb + imaga * realb;
+
+    return deposit32((realr >> 15) + reald, 16, 16, (imagr >> 15) + imagd);
+}
+
+uint64_t helper_cmul2(uint64_t srca, uint64_t srcb, int shift, int round)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    int32_t realr = reala * realb - imaga * imagb + round;
+    int32_t imagr = reala * imagb + imaga * realb + round;
+
+    return deposit32(realr >> shift, 16, 16, imagr >> shift);
+}
diff --git a/qemu/target-tilegx/helper.h b/qemu/target-tilegx/helper.h
new file mode 100644
index 000000000..9281d0f42
--- /dev/null
+++ b/qemu/target-tilegx/helper.h
@@ -0,0 +1,26 @@
+DEF_HELPER_2(exception, noreturn, env, i32)
+DEF_HELPER_1(ext01_ics, void, env)
+DEF_HELPER_FLAGS_1(cntlz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(cnttz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pcnt, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(revbits, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_3(shufflebytes, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_32, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmula, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmulaf, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_4(cmul2, TCG_CALL_NO_RWG_SE, i64, i64, i64, int, int)
+
+DEF_HELPER_FLAGS_2(v1int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(v1multu, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2mults, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
diff --git a/qemu/target-tilegx/opcode_tilegx.h b/qemu/target-tilegx/opcode_tilegx.h
new file mode 100644
index 000000000..989436d2f
--- /dev/null
+++ b/qemu/target-tilegx/opcode_tilegx.h
@@ -0,0 +1,1406 @@
+/* TILE-Gx opcode information.
+ *
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ *
+ *
+ *
+ *
+ *
+ */
+
+#ifndef __ARCH_OPCODE_H__
+#define __ARCH_OPCODE_H__
+
+#ifndef __ASSEMBLER__
+
+typedef uint64_t tilegx_bundle_bits;
+
+/* These are the bits that determine if a bundle is in the X encoding. */
+#define TILEGX_BUNDLE_MODE_MASK ((tilegx_bundle_bits)3 << 62)
+
+enum
+{
+  /* Maximum number of instructions in a bundle (2 for X, 3 for Y). */
+  TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE = 3,
+
+  /* How many different pipeline encodings are there? X0, X1, Y0, Y1, Y2. */
+  TILEGX_NUM_PIPELINE_ENCODINGS = 5,
+
+  /* Log base 2 of TILEGX_BUNDLE_SIZE_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES = 3,
+
+  /* Instructions take this many bytes. */
+  TILEGX_BUNDLE_SIZE_IN_BYTES = 1 << TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES,
+
+  /* Log base 2 of TILEGX_BUNDLE_ALIGNMENT_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES = 3,
+
+  /* Bundles should be aligned modulo this number of bytes. */
+  TILEGX_BUNDLE_ALIGNMENT_IN_BYTES =
+    (1 << TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES),
+
+  /* Number of registers (some are magic, such as network I/O). */
+  TILEGX_NUM_REGISTERS = 64,
+};
+
+/* Make a few "tile_" variables to simplify common code between
+   architectures.  */
+
+typedef tilegx_bundle_bits tile_bundle_bits;
+#define TILE_BUNDLE_SIZE_IN_BYTES TILEGX_BUNDLE_SIZE_IN_BYTES
+#define TILE_BUNDLE_ALIGNMENT_IN_BYTES TILEGX_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_LOG2_BUNDLE_ALIGNMENT_IN_BYTES \
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_BPT_BUNDLE TILEGX_BPT_BUNDLE
+
+/* 64-bit pattern for a { bpt ; nop } bundle. */
+#define TILEGX_BPT_BUNDLE 0x286a44ae51485000ULL
+
+static inline unsigned int
+get_BFEnd_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_BFOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 24)) & 0xf);
+}
+
+static inline unsigned int
+get_BFStart_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3f);
+}
+
+static inline unsigned int
+get_BrOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x0001ffc0);
+}
+
+static inline unsigned int
+get_BrType_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 54)) & 0x1f);
+}
+
+static inline unsigned int
+get_Dest_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 43)) & 0x000000c0);
+}
+
+static inline unsigned int
+get_Dest_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Imm16_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm16_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_JumpOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x7ffffff);
+}
+
+static inline unsigned int
+get_JumpOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0x1);
+}
+
+static inline unsigned int
+get_MF_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3fff);
+}
+
+static inline unsigned int
+get_MT_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x00003fc0);
+}
+
+static inline unsigned int
+get_Mode(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 62)) & 0x3);
+}
+
+static inline unsigned int
+get_Opcode_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 28)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 59)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 27)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y2(tilegx_bundle_bits n)
+{
+  return (((n >> 26)) & 0x00000001) |
+         (((unsigned int)(n >> 56)) & 0x00000002);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_ShAmt_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_SrcA_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y2(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcBDest_Y2(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+
+static inline int
+sign_extend(int n, int num_bits)
+{
+  int shift = (int)(sizeof(int) * 8 - num_bits);
+  return (n << shift) >> shift;
+}
+
+
+
+static inline tilegx_bundle_bits
+create_BFEnd_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_BFOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 24);
+}
+
+static inline tilegx_bundle_bits
+create_BFStart_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_BrOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x0001ffc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_BrType_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1f)) << 54);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x000000c0)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xffff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xffff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7ffffff)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_MF_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3fff)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_MT_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x00003fc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_Mode(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 62);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x7) << 28);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7)) << 59);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 27);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xf)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x00000001) << 26) |
+         (((tilegx_bundle_bits)(n & 0x00000002)) << 56);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_SrcBDest_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+
+enum
+{
+  ADDI_IMM8_OPCODE_X0 = 1,
+  ADDI_IMM8_OPCODE_X1 = 1,
+  ADDI_OPCODE_Y0 = 0,
+  ADDI_OPCODE_Y1 = 1,
+  ADDLI_OPCODE_X0 = 1,
+  ADDLI_OPCODE_X1 = 0,
+  ADDXI_IMM8_OPCODE_X0 = 2,
+  ADDXI_IMM8_OPCODE_X1 = 2,
+  ADDXI_OPCODE_Y0 = 1,
+  ADDXI_OPCODE_Y1 = 2,
+  ADDXLI_OPCODE_X0 = 2,
+  ADDXLI_OPCODE_X1 = 1,
+  ADDXSC_RRR_0_OPCODE_X0 = 1,
+  ADDXSC_RRR_0_OPCODE_X1 = 1,
+  ADDX_RRR_0_OPCODE_X0 = 2,
+  ADDX_RRR_0_OPCODE_X1 = 2,
+  ADDX_RRR_0_OPCODE_Y0 = 0,
+  ADDX_RRR_0_OPCODE_Y1 = 0,
+  ADD_RRR_0_OPCODE_X0 = 3,
+  ADD_RRR_0_OPCODE_X1 = 3,
+  ADD_RRR_0_OPCODE_Y0 = 1,
+  ADD_RRR_0_OPCODE_Y1 = 1,
+  ANDI_IMM8_OPCODE_X0 = 3,
+  ANDI_IMM8_OPCODE_X1 = 3,
+  ANDI_OPCODE_Y0 = 2,
+  ANDI_OPCODE_Y1 = 3,
+  AND_RRR_0_OPCODE_X0 = 4,
+  AND_RRR_0_OPCODE_X1 = 4,
+  AND_RRR_5_OPCODE_Y0 = 0,
+  AND_RRR_5_OPCODE_Y1 = 0,
+  BEQZT_BRANCH_OPCODE_X1 = 16,
+  BEQZ_BRANCH_OPCODE_X1 = 17,
+  BFEXTS_BF_OPCODE_X0 = 4,
+  BFEXTU_BF_OPCODE_X0 = 5,
+  BFINS_BF_OPCODE_X0 = 6,
+  BF_OPCODE_X0 = 3,
+  BGEZT_BRANCH_OPCODE_X1 = 18,
+  BGEZ_BRANCH_OPCODE_X1 = 19,
+  BGTZT_BRANCH_OPCODE_X1 = 20,
+  BGTZ_BRANCH_OPCODE_X1 = 21,
+  BLBCT_BRANCH_OPCODE_X1 = 22,
+  BLBC_BRANCH_OPCODE_X1 = 23,
+  BLBST_BRANCH_OPCODE_X1 = 24,
+  BLBS_BRANCH_OPCODE_X1 = 25,
+  BLEZT_BRANCH_OPCODE_X1 = 26,
+  BLEZ_BRANCH_OPCODE_X1 = 27,
+  BLTZT_BRANCH_OPCODE_X1 = 28,
+  BLTZ_BRANCH_OPCODE_X1 = 29,
+  BNEZT_BRANCH_OPCODE_X1 = 30,
+  BNEZ_BRANCH_OPCODE_X1 = 31,
+  BRANCH_OPCODE_X1 = 2,
+  CMOVEQZ_RRR_0_OPCODE_X0 = 5,
+  CMOVEQZ_RRR_4_OPCODE_Y0 = 0,
+  CMOVNEZ_RRR_0_OPCODE_X0 = 6,
+  CMOVNEZ_RRR_4_OPCODE_Y0 = 1,
+  CMPEQI_IMM8_OPCODE_X0 = 4,
+  CMPEQI_IMM8_OPCODE_X1 = 4,
+  CMPEQI_OPCODE_Y0 = 3,
+  CMPEQI_OPCODE_Y1 = 4,
+  CMPEQ_RRR_0_OPCODE_X0 = 7,
+  CMPEQ_RRR_0_OPCODE_X1 = 5,
+  CMPEQ_RRR_3_OPCODE_Y0 = 0,
+  CMPEQ_RRR_3_OPCODE_Y1 = 2,
+  CMPEXCH4_RRR_0_OPCODE_X1 = 6,
+  CMPEXCH_RRR_0_OPCODE_X1 = 7,
+  CMPLES_RRR_0_OPCODE_X0 = 8,
+  CMPLES_RRR_0_OPCODE_X1 = 8,
+  CMPLES_RRR_2_OPCODE_Y0 = 0,
+  CMPLES_RRR_2_OPCODE_Y1 = 0,
+  CMPLEU_RRR_0_OPCODE_X0 = 9,
+  CMPLEU_RRR_0_OPCODE_X1 = 9,
+  CMPLEU_RRR_2_OPCODE_Y0 = 1,
+  CMPLEU_RRR_2_OPCODE_Y1 = 1,
+  CMPLTSI_IMM8_OPCODE_X0 = 5,
+  CMPLTSI_IMM8_OPCODE_X1 = 5,
+  CMPLTSI_OPCODE_Y0 = 4,
+  CMPLTSI_OPCODE_Y1 = 5,
+  CMPLTS_RRR_0_OPCODE_X0 = 10,
+  CMPLTS_RRR_0_OPCODE_X1 = 10,
+  CMPLTS_RRR_2_OPCODE_Y0 = 2,
+  CMPLTS_RRR_2_OPCODE_Y1 = 2,
+  CMPLTUI_IMM8_OPCODE_X0 = 6,
+  CMPLTUI_IMM8_OPCODE_X1 = 6,
+  CMPLTU_RRR_0_OPCODE_X0 = 11,
+  CMPLTU_RRR_0_OPCODE_X1 = 11,
+  CMPLTU_RRR_2_OPCODE_Y0 = 3,
+  CMPLTU_RRR_2_OPCODE_Y1 = 3,
+  CMPNE_RRR_0_OPCODE_X0 = 12,
+  CMPNE_RRR_0_OPCODE_X1 = 12,
+  CMPNE_RRR_3_OPCODE_Y0 = 1,
+  CMPNE_RRR_3_OPCODE_Y1 = 3,
+  CMULAF_RRR_0_OPCODE_X0 = 13,
+  CMULA_RRR_0_OPCODE_X0 = 14,
+  CMULFR_RRR_0_OPCODE_X0 = 15,
+  CMULF_RRR_0_OPCODE_X0 = 16,
+  CMULHR_RRR_0_OPCODE_X0 = 17,
+  CMULH_RRR_0_OPCODE_X0 = 18,
+  CMUL_RRR_0_OPCODE_X0 = 19,
+  CNTLZ_UNARY_OPCODE_X0 = 1,
+  CNTLZ_UNARY_OPCODE_Y0 = 1,
+  CNTTZ_UNARY_OPCODE_X0 = 2,
+  CNTTZ_UNARY_OPCODE_Y0 = 2,
+  CRC32_32_RRR_0_OPCODE_X0 = 20,
+  CRC32_8_RRR_0_OPCODE_X0 = 21,
+  DBLALIGN2_RRR_0_OPCODE_X0 = 22,
+  DBLALIGN2_RRR_0_OPCODE_X1 = 13,
+  DBLALIGN4_RRR_0_OPCODE_X0 = 23,
+  DBLALIGN4_RRR_0_OPCODE_X1 = 14,
+  DBLALIGN6_RRR_0_OPCODE_X0 = 24,
+  DBLALIGN6_RRR_0_OPCODE_X1 = 15,
+  DBLALIGN_RRR_0_OPCODE_X0 = 25,
+  DRAIN_UNARY_OPCODE_X1 = 1,
+  DTLBPR_UNARY_OPCODE_X1 = 2,
+  EXCH4_RRR_0_OPCODE_X1 = 16,
+  EXCH_RRR_0_OPCODE_X1 = 17,
+  FDOUBLE_ADDSUB_RRR_0_OPCODE_X0 = 26,
+  FDOUBLE_ADD_FLAGS_RRR_0_OPCODE_X0 = 27,
+  FDOUBLE_MUL_FLAGS_RRR_0_OPCODE_X0 = 28,
+  FDOUBLE_PACK1_RRR_0_OPCODE_X0 = 29,
+  FDOUBLE_PACK2_RRR_0_OPCODE_X0 = 30,
+  FDOUBLE_SUB_FLAGS_RRR_0_OPCODE_X0 = 31,
+  FDOUBLE_UNPACK_MAX_RRR_0_OPCODE_X0 = 32,
+  FDOUBLE_UNPACK_MIN_RRR_0_OPCODE_X0 = 33,
+  FETCHADD4_RRR_0_OPCODE_X1 = 18,
+  FETCHADDGEZ4_RRR_0_OPCODE_X1 = 19,
+  FETCHADDGEZ_RRR_0_OPCODE_X1 = 20,
+  FETCHADD_RRR_0_OPCODE_X1 = 21,
+  FETCHAND4_RRR_0_OPCODE_X1 = 22,
+  FETCHAND_RRR_0_OPCODE_X1 = 23,
+  FETCHOR4_RRR_0_OPCODE_X1 = 24,
+  FETCHOR_RRR_0_OPCODE_X1 = 25,
+  FINV_UNARY_OPCODE_X1 = 3,
+  FLUSHWB_UNARY_OPCODE_X1 = 4,
+  FLUSH_UNARY_OPCODE_X1 = 5,
+  FNOP_UNARY_OPCODE_X0 = 3,
+  FNOP_UNARY_OPCODE_X1 = 6,
+  FNOP_UNARY_OPCODE_Y0 = 3,
+  FNOP_UNARY_OPCODE_Y1 = 8,
+  FSINGLE_ADD1_RRR_0_OPCODE_X0 = 34,
+  FSINGLE_ADDSUB2_RRR_0_OPCODE_X0 = 35,
+  FSINGLE_MUL1_RRR_0_OPCODE_X0 = 36,
+  FSINGLE_MUL2_RRR_0_OPCODE_X0 = 37,
+  FSINGLE_PACK1_UNARY_OPCODE_X0 = 4,
+  FSINGLE_PACK1_UNARY_OPCODE_Y0 = 4,
+  FSINGLE_PACK2_RRR_0_OPCODE_X0 = 38,
+  FSINGLE_SUB1_RRR_0_OPCODE_X0 = 39,
+  ICOH_UNARY_OPCODE_X1 = 7,
+  ILL_UNARY_OPCODE_X1 = 8,
+  ILL_UNARY_OPCODE_Y1 = 9,
+  IMM8_OPCODE_X0 = 4,
+  IMM8_OPCODE_X1 = 3,
+  INV_UNARY_OPCODE_X1 = 9,
+  IRET_UNARY_OPCODE_X1 = 10,
+  JALRP_UNARY_OPCODE_X1 = 11,
+  JALRP_UNARY_OPCODE_Y1 = 10,
+  JALR_UNARY_OPCODE_X1 = 12,
+  JALR_UNARY_OPCODE_Y1 = 11,
+  JAL_JUMP_OPCODE_X1 = 0,
+  JRP_UNARY_OPCODE_X1 = 13,
+  JRP_UNARY_OPCODE_Y1 = 12,
+  JR_UNARY_OPCODE_X1 = 14,
+  JR_UNARY_OPCODE_Y1 = 13,
+  JUMP_OPCODE_X1 = 4,
+  J_JUMP_OPCODE_X1 = 1,
+  LD1S_ADD_IMM8_OPCODE_X1 = 7,
+  LD1S_OPCODE_Y2 = 0,
+  LD1S_UNARY_OPCODE_X1 = 15,
+  LD1U_ADD_IMM8_OPCODE_X1 = 8,
+  LD1U_OPCODE_Y2 = 1,
+  LD1U_UNARY_OPCODE_X1 = 16,
+  LD2S_ADD_IMM8_OPCODE_X1 = 9,
+  LD2S_OPCODE_Y2 = 2,
+  LD2S_UNARY_OPCODE_X1 = 17,
+  LD2U_ADD_IMM8_OPCODE_X1 = 10,
+  LD2U_OPCODE_Y2 = 3,
+  LD2U_UNARY_OPCODE_X1 = 18,
+  LD4S_ADD_IMM8_OPCODE_X1 = 11,
+  LD4S_OPCODE_Y2 = 1,
+  LD4S_UNARY_OPCODE_X1 = 19,
+  LD4U_ADD_IMM8_OPCODE_X1 = 12,
+  LD4U_OPCODE_Y2 = 2,
+  LD4U_UNARY_OPCODE_X1 = 20,
+  LDNA_UNARY_OPCODE_X1 = 21,
+  LDNT1S_ADD_IMM8_OPCODE_X1 = 13,
+  LDNT1S_UNARY_OPCODE_X1 = 22,
+  LDNT1U_ADD_IMM8_OPCODE_X1 = 14,
+  LDNT1U_UNARY_OPCODE_X1 = 23,
+  LDNT2S_ADD_IMM8_OPCODE_X1 = 15,
+  LDNT2S_UNARY_OPCODE_X1 = 24,
+  LDNT2U_ADD_IMM8_OPCODE_X1 = 16,
+  LDNT2U_UNARY_OPCODE_X1 = 25,
+  LDNT4S_ADD_IMM8_OPCODE_X1 = 17,
+  LDNT4S_UNARY_OPCODE_X1 = 26,
+  LDNT4U_ADD_IMM8_OPCODE_X1 = 18,
+  LDNT4U_UNARY_OPCODE_X1 = 27,
+  LDNT_ADD_IMM8_OPCODE_X1 = 19,
+  LDNT_UNARY_OPCODE_X1 = 28,
+  LD_ADD_IMM8_OPCODE_X1 = 20,
+  LD_OPCODE_Y2 = 3,
+  LD_UNARY_OPCODE_X1 = 29,
+  LNK_UNARY_OPCODE_X1 = 30,
+  LNK_UNARY_OPCODE_Y1 = 14,
+  LDNA_ADD_IMM8_OPCODE_X1 = 21,
+  MFSPR_IMM8_OPCODE_X1 = 22,
+  MF_UNARY_OPCODE_X1 = 31,
+  MM_BF_OPCODE_X0 = 7,
+  MNZ_RRR_0_OPCODE_X0 = 40,
+  MNZ_RRR_0_OPCODE_X1 = 26,
+  MNZ_RRR_4_OPCODE_Y0 = 2,
+  MNZ_RRR_4_OPCODE_Y1 = 2,
+  MODE_OPCODE_YA2 = 1,
+  MODE_OPCODE_YB2 = 2,
+  MODE_OPCODE_YC2 = 3,
+  MTSPR_IMM8_OPCODE_X1 = 23,
+  MULAX_RRR_0_OPCODE_X0 = 41,
+  MULAX_RRR_3_OPCODE_Y0 = 2,
+  MULA_HS_HS_RRR_0_OPCODE_X0 = 42,
+  MULA_HS_HS_RRR_9_OPCODE_Y0 = 0,
+  MULA_HS_HU_RRR_0_OPCODE_X0 = 43,
+  MULA_HS_LS_RRR_0_OPCODE_X0 = 44,
+  MULA_HS_LU_RRR_0_OPCODE_X0 = 45,
+  MULA_HU_HU_RRR_0_OPCODE_X0 = 46,
+  MULA_HU_HU_RRR_9_OPCODE_Y0 = 1,
+  MULA_HU_LS_RRR_0_OPCODE_X0 = 47,
+  MULA_HU_LU_RRR_0_OPCODE_X0 = 48,
+  MULA_LS_LS_RRR_0_OPCODE_X0 = 49,
+  MULA_LS_LS_RRR_9_OPCODE_Y0 = 2,
+  MULA_LS_LU_RRR_0_OPCODE_X0 = 50,
+  MULA_LU_LU_RRR_0_OPCODE_X0 = 51,
+  MULA_LU_LU_RRR_9_OPCODE_Y0 = 3,
+  MULX_RRR_0_OPCODE_X0 = 52,
+  MULX_RRR_3_OPCODE_Y0 = 3,
+  MUL_HS_HS_RRR_0_OPCODE_X0 = 53,
+  MUL_HS_HS_RRR_8_OPCODE_Y0 = 0,
+  MUL_HS_HU_RRR_0_OPCODE_X0 = 54,
+  MUL_HS_LS_RRR_0_OPCODE_X0 = 55,
+  MUL_HS_LU_RRR_0_OPCODE_X0 = 56,
+  MUL_HU_HU_RRR_0_OPCODE_X0 = 57,
+  MUL_HU_HU_RRR_8_OPCODE_Y0 = 1,
+  MUL_HU_LS_RRR_0_OPCODE_X0 = 58,
+  MUL_HU_LU_RRR_0_OPCODE_X0 = 59,
+  MUL_LS_LS_RRR_0_OPCODE_X0 = 60,
+  MUL_LS_LS_RRR_8_OPCODE_Y0 = 2,
+  MUL_LS_LU_RRR_0_OPCODE_X0 = 61,
+  MUL_LU_LU_RRR_0_OPCODE_X0 = 62,
+  MUL_LU_LU_RRR_8_OPCODE_Y0 = 3,
+  MZ_RRR_0_OPCODE_X0 = 63,
+  MZ_RRR_0_OPCODE_X1 = 27,
+  MZ_RRR_4_OPCODE_Y0 = 3,
+  MZ_RRR_4_OPCODE_Y1 = 3,
+  NAP_UNARY_OPCODE_X1 = 32,
+  NOP_UNARY_OPCODE_X0 = 5,
+  NOP_UNARY_OPCODE_X1 = 33,
+  NOP_UNARY_OPCODE_Y0 = 5,
+  NOP_UNARY_OPCODE_Y1 = 15,
+  NOR_RRR_0_OPCODE_X0 = 64,
+  NOR_RRR_0_OPCODE_X1 = 28,
+  NOR_RRR_5_OPCODE_Y0 = 1,
+  NOR_RRR_5_OPCODE_Y1 = 1,
+  ORI_IMM8_OPCODE_X0 = 7,
+  ORI_IMM8_OPCODE_X1 = 24,
+  OR_RRR_0_OPCODE_X0 = 65,
+  OR_RRR_0_OPCODE_X1 = 29,
+  OR_RRR_5_OPCODE_Y0 = 2,
+  OR_RRR_5_OPCODE_Y1 = 2,
+  PCNT_UNARY_OPCODE_X0 = 6,
+  PCNT_UNARY_OPCODE_Y0 = 6,
+  REVBITS_UNARY_OPCODE_X0 = 7,
+  REVBITS_UNARY_OPCODE_Y0 = 7,
+  REVBYTES_UNARY_OPCODE_X0 = 8,
+  REVBYTES_UNARY_OPCODE_Y0 = 8,
+  ROTLI_SHIFT_OPCODE_X0 = 1,
+  ROTLI_SHIFT_OPCODE_X1 = 1,
+  ROTLI_SHIFT_OPCODE_Y0 = 0,
+  ROTLI_SHIFT_OPCODE_Y1 = 0,
+  ROTL_RRR_0_OPCODE_X0 = 66,
+  ROTL_RRR_0_OPCODE_X1 = 30,
+  ROTL_RRR_6_OPCODE_Y0 = 0,
+  ROTL_RRR_6_OPCODE_Y1 = 0,
+  RRR_0_OPCODE_X0 = 5,
+  RRR_0_OPCODE_X1 = 5,
+  RRR_0_OPCODE_Y0 = 5,
+  RRR_0_OPCODE_Y1 = 6,
+  RRR_1_OPCODE_Y0 = 6,
+  RRR_1_OPCODE_Y1 = 7,
+  RRR_2_OPCODE_Y0 = 7,
+  RRR_2_OPCODE_Y1 = 8,
+  RRR_3_OPCODE_Y0 = 8,
+  RRR_3_OPCODE_Y1 = 9,
+  RRR_4_OPCODE_Y0 = 9,
+  RRR_4_OPCODE_Y1 = 10,
+  RRR_5_OPCODE_Y0 = 10,
+  RRR_5_OPCODE_Y1 = 11,
+  RRR_6_OPCODE_Y0 = 11,
+  RRR_6_OPCODE_Y1 = 12,
+  RRR_7_OPCODE_Y0 = 12,
+  RRR_7_OPCODE_Y1 = 13,
+  RRR_8_OPCODE_Y0 = 13,
+  RRR_9_OPCODE_Y0 = 14,
+  SHIFT_OPCODE_X0 = 6,
+  SHIFT_OPCODE_X1 = 6,
+  SHIFT_OPCODE_Y0 = 15,
+  SHIFT_OPCODE_Y1 = 14,
+  SHL16INSLI_OPCODE_X0 = 7,
+  SHL16INSLI_OPCODE_X1 = 7,
+  SHL1ADDX_RRR_0_OPCODE_X0 = 67,
+  SHL1ADDX_RRR_0_OPCODE_X1 = 31,
+  SHL1ADDX_RRR_7_OPCODE_Y0 = 1,
+  SHL1ADDX_RRR_7_OPCODE_Y1 = 1,
+  SHL1ADD_RRR_0_OPCODE_X0 = 68,
+  SHL1ADD_RRR_0_OPCODE_X1 = 32,
+  SHL1ADD_RRR_1_OPCODE_Y0 = 0,
+  SHL1ADD_RRR_1_OPCODE_Y1 = 0,
+  SHL2ADDX_RRR_0_OPCODE_X0 = 69,
+  SHL2ADDX_RRR_0_OPCODE_X1 = 33,
+  SHL2ADDX_RRR_7_OPCODE_Y0 = 2,
+  SHL2ADDX_RRR_7_OPCODE_Y1 = 2,
+  SHL2ADD_RRR_0_OPCODE_X0 = 70,
+  SHL2ADD_RRR_0_OPCODE_X1 = 34,
+  SHL2ADD_RRR_1_OPCODE_Y0 = 1,
+  SHL2ADD_RRR_1_OPCODE_Y1 = 1,
+  SHL3ADDX_RRR_0_OPCODE_X0 = 71,
+  SHL3ADDX_RRR_0_OPCODE_X1 = 35,
+  SHL3ADDX_RRR_7_OPCODE_Y0 = 3,
+  SHL3ADDX_RRR_7_OPCODE_Y1 = 3,
+  SHL3ADD_RRR_0_OPCODE_X0 = 72,
+  SHL3ADD_RRR_0_OPCODE_X1 = 36,
+  SHL3ADD_RRR_1_OPCODE_Y0 = 2,
+  SHL3ADD_RRR_1_OPCODE_Y1 = 2,
+  SHLI_SHIFT_OPCODE_X0 = 2,
+  SHLI_SHIFT_OPCODE_X1 = 2,
+  SHLI_SHIFT_OPCODE_Y0 = 1,
+  SHLI_SHIFT_OPCODE_Y1 = 1,
+  SHLXI_SHIFT_OPCODE_X0 = 3,
+  SHLXI_SHIFT_OPCODE_X1 = 3,
+  SHLX_RRR_0_OPCODE_X0 = 73,
+  SHLX_RRR_0_OPCODE_X1 = 37,
+  SHL_RRR_0_OPCODE_X0 = 74,
+  SHL_RRR_0_OPCODE_X1 = 38,
+  SHL_RRR_6_OPCODE_Y0 = 1,
+  SHL_RRR_6_OPCODE_Y1 = 1,
+  SHRSI_SHIFT_OPCODE_X0 = 4,
+  SHRSI_SHIFT_OPCODE_X1 = 4,
+  SHRSI_SHIFT_OPCODE_Y0 = 2,
+  SHRSI_SHIFT_OPCODE_Y1 = 2,
+  SHRS_RRR_0_OPCODE_X0 = 75,
+  SHRS_RRR_0_OPCODE_X1 = 39,
+  SHRS_RRR_6_OPCODE_Y0 = 2,
+  SHRS_RRR_6_OPCODE_Y1 = 2,
+  SHRUI_SHIFT_OPCODE_X0 = 5,
+  SHRUI_SHIFT_OPCODE_X1 = 5,
+  SHRUI_SHIFT_OPCODE_Y0 = 3,
+  SHRUI_SHIFT_OPCODE_Y1 = 3,
+  SHRUXI_SHIFT_OPCODE_X0 = 6,
+  SHRUXI_SHIFT_OPCODE_X1 = 6,
+  SHRUX_RRR_0_OPCODE_X0 = 76,
+  SHRUX_RRR_0_OPCODE_X1 = 40,
+  SHRU_RRR_0_OPCODE_X0 = 77,
+  SHRU_RRR_0_OPCODE_X1 = 41,
+  SHRU_RRR_6_OPCODE_Y0 = 3,
+  SHRU_RRR_6_OPCODE_Y1 = 3,
+  SHUFFLEBYTES_RRR_0_OPCODE_X0 = 78,
+  ST1_ADD_IMM8_OPCODE_X1 = 25,
+  ST1_OPCODE_Y2 = 0,
+  ST1_RRR_0_OPCODE_X1 = 42,
+  ST2_ADD_IMM8_OPCODE_X1 = 26,
+  ST2_OPCODE_Y2 = 1,
+  ST2_RRR_0_OPCODE_X1 = 43,
+  ST4_ADD_IMM8_OPCODE_X1 = 27,
+  ST4_OPCODE_Y2 = 2,
+  ST4_RRR_0_OPCODE_X1 = 44,
+  STNT1_ADD_IMM8_OPCODE_X1 = 28,
+  STNT1_RRR_0_OPCODE_X1 = 45,
+  STNT2_ADD_IMM8_OPCODE_X1 = 29,
+  STNT2_RRR_0_OPCODE_X1 = 46,
+  STNT4_ADD_IMM8_OPCODE_X1 = 30,
+  STNT4_RRR_0_OPCODE_X1 = 47,
+  STNT_ADD_IMM8_OPCODE_X1 = 31,
+  STNT_RRR_0_OPCODE_X1 = 48,
+  ST_ADD_IMM8_OPCODE_X1 = 32,
+  ST_OPCODE_Y2 = 3,
+  ST_RRR_0_OPCODE_X1 = 49,
+  SUBXSC_RRR_0_OPCODE_X0 = 79,
+  SUBXSC_RRR_0_OPCODE_X1 = 50,
+  SUBX_RRR_0_OPCODE_X0 = 80,
+  SUBX_RRR_0_OPCODE_X1 = 51,
+  SUBX_RRR_0_OPCODE_Y0 = 2,
+  SUBX_RRR_0_OPCODE_Y1 = 2,
+  SUB_RRR_0_OPCODE_X0 = 81,
+  SUB_RRR_0_OPCODE_X1 = 52,
+  SUB_RRR_0_OPCODE_Y0 = 3,
+  SUB_RRR_0_OPCODE_Y1 = 3,
+  SWINT0_UNARY_OPCODE_X1 = 34,
+  SWINT1_UNARY_OPCODE_X1 = 35,
+  SWINT2_UNARY_OPCODE_X1 = 36,
+  SWINT3_UNARY_OPCODE_X1 = 37,
+  TBLIDXB0_UNARY_OPCODE_X0 = 9,
+  TBLIDXB0_UNARY_OPCODE_Y0 = 9,
+  TBLIDXB1_UNARY_OPCODE_X0 = 10,
+  TBLIDXB1_UNARY_OPCODE_Y0 = 10,
+  TBLIDXB2_UNARY_OPCODE_X0 = 11,
+  TBLIDXB2_UNARY_OPCODE_Y0 = 11,
+  TBLIDXB3_UNARY_OPCODE_X0 = 12,
+  TBLIDXB3_UNARY_OPCODE_Y0 = 12,
+  UNARY_RRR_0_OPCODE_X0 = 82,
+  UNARY_RRR_0_OPCODE_X1 = 53,
+  UNARY_RRR_1_OPCODE_Y0 = 3,
+  UNARY_RRR_1_OPCODE_Y1 = 3,
+  V1ADDI_IMM8_OPCODE_X0 = 8,
+  V1ADDI_IMM8_OPCODE_X1 = 33,
+  V1ADDUC_RRR_0_OPCODE_X0 = 83,
+  V1ADDUC_RRR_0_OPCODE_X1 = 54,
+  V1ADD_RRR_0_OPCODE_X0 = 84,
+  V1ADD_RRR_0_OPCODE_X1 = 55,
+  V1ADIFFU_RRR_0_OPCODE_X0 = 85,
+  V1AVGU_RRR_0_OPCODE_X0 = 86,
+  V1CMPEQI_IMM8_OPCODE_X0 = 9,
+  V1CMPEQI_IMM8_OPCODE_X1 = 34,
+  V1CMPEQ_RRR_0_OPCODE_X0 = 87,
+  V1CMPEQ_RRR_0_OPCODE_X1 = 56,
+  V1CMPLES_RRR_0_OPCODE_X0 = 88,
+  V1CMPLES_RRR_0_OPCODE_X1 = 57,
+  V1CMPLEU_RRR_0_OPCODE_X0 = 89,
+  V1CMPLEU_RRR_0_OPCODE_X1 = 58,
+  V1CMPLTSI_IMM8_OPCODE_X0 = 10,
+  V1CMPLTSI_IMM8_OPCODE_X1 = 35,
+  V1CMPLTS_RRR_0_OPCODE_X0 = 90,
+  V1CMPLTS_RRR_0_OPCODE_X1 = 59,
+  V1CMPLTUI_IMM8_OPCODE_X0 = 11,
+  V1CMPLTUI_IMM8_OPCODE_X1 = 36,
+  V1CMPLTU_RRR_0_OPCODE_X0 = 91,
+  V1CMPLTU_RRR_0_OPCODE_X1 = 60,
+  V1CMPNE_RRR_0_OPCODE_X0 = 92,
+  V1CMPNE_RRR_0_OPCODE_X1 = 61,
+  V1DDOTPUA_RRR_0_OPCODE_X0 = 161,
+  V1DDOTPUSA_RRR_0_OPCODE_X0 = 93,
+  V1DDOTPUS_RRR_0_OPCODE_X0 = 94,
+  V1DDOTPU_RRR_0_OPCODE_X0 = 162,
+  V1DOTPA_RRR_0_OPCODE_X0 = 95,
+  V1DOTPUA_RRR_0_OPCODE_X0 = 163,
+  V1DOTPUSA_RRR_0_OPCODE_X0 = 96,
+  V1DOTPUS_RRR_0_OPCODE_X0 = 97,
+  V1DOTPU_RRR_0_OPCODE_X0 = 164,
+  V1DOTP_RRR_0_OPCODE_X0 = 98,
+  V1INT_H_RRR_0_OPCODE_X0 = 99,
+  V1INT_H_RRR_0_OPCODE_X1 = 62,
+  V1INT_L_RRR_0_OPCODE_X0 = 100,
+  V1INT_L_RRR_0_OPCODE_X1 = 63,
+  V1MAXUI_IMM8_OPCODE_X0 = 12,
+  V1MAXUI_IMM8_OPCODE_X1 = 37,
+  V1MAXU_RRR_0_OPCODE_X0 = 101,
+  V1MAXU_RRR_0_OPCODE_X1 = 64,
+  V1MINUI_IMM8_OPCODE_X0 = 13,
+  V1MINUI_IMM8_OPCODE_X1 = 38,
+  V1MINU_RRR_0_OPCODE_X0 = 102,
+  V1MINU_RRR_0_OPCODE_X1 = 65,
+  V1MNZ_RRR_0_OPCODE_X0 = 103,
+  V1MNZ_RRR_0_OPCODE_X1 = 66,
+  V1MULTU_RRR_0_OPCODE_X0 = 104,
+  V1MULUS_RRR_0_OPCODE_X0 = 105,
+  V1MULU_RRR_0_OPCODE_X0 = 106,
+  V1MZ_RRR_0_OPCODE_X0 = 107,
+  V1MZ_RRR_0_OPCODE_X1 = 67,
+  V1SADAU_RRR_0_OPCODE_X0 = 108,
+  V1SADU_RRR_0_OPCODE_X0 = 109,
+  V1SHLI_SHIFT_OPCODE_X0 = 7,
+  V1SHLI_SHIFT_OPCODE_X1 = 7,
+  V1SHL_RRR_0_OPCODE_X0 = 110,
+  V1SHL_RRR_0_OPCODE_X1 = 68,
+  V1SHRSI_SHIFT_OPCODE_X0 = 8,
+  V1SHRSI_SHIFT_OPCODE_X1 = 8,
+  V1SHRS_RRR_0_OPCODE_X0 = 111,
+  V1SHRS_RRR_0_OPCODE_X1 = 69,
+  V1SHRUI_SHIFT_OPCODE_X0 = 9,
+  V1SHRUI_SHIFT_OPCODE_X1 = 9,
+  V1SHRU_RRR_0_OPCODE_X0 = 112,
+  V1SHRU_RRR_0_OPCODE_X1 = 70,
+  V1SUBUC_RRR_0_OPCODE_X0 = 113,
+  V1SUBUC_RRR_0_OPCODE_X1 = 71,
+  V1SUB_RRR_0_OPCODE_X0 = 114,
+  V1SUB_RRR_0_OPCODE_X1 = 72,
+  V2ADDI_IMM8_OPCODE_X0 = 14,
+  V2ADDI_IMM8_OPCODE_X1 = 39,
+  V2ADDSC_RRR_0_OPCODE_X0 = 115,
+  V2ADDSC_RRR_0_OPCODE_X1 = 73,
+  V2ADD_RRR_0_OPCODE_X0 = 116,
+  V2ADD_RRR_0_OPCODE_X1 = 74,
+  V2ADIFFS_RRR_0_OPCODE_X0 = 117,
+  V2AVGS_RRR_0_OPCODE_X0 = 118,
+  V2CMPEQI_IMM8_OPCODE_X0 = 15,
+  V2CMPEQI_IMM8_OPCODE_X1 = 40,
+  V2CMPEQ_RRR_0_OPCODE_X0 = 119,
+  V2CMPEQ_RRR_0_OPCODE_X1 = 75,
+  V2CMPLES_RRR_0_OPCODE_X0 = 120,
+  V2CMPLES_RRR_0_OPCODE_X1 = 76,
+  V2CMPLEU_RRR_0_OPCODE_X0 = 121,
+  V2CMPLEU_RRR_0_OPCODE_X1 = 77,
+  V2CMPLTSI_IMM8_OPCODE_X0 = 16,
+  V2CMPLTSI_IMM8_OPCODE_X1 = 41,
+  V2CMPLTS_RRR_0_OPCODE_X0 = 122,
+  V2CMPLTS_RRR_0_OPCODE_X1 = 78,
+  V2CMPLTUI_IMM8_OPCODE_X0 = 17,
+  V2CMPLTUI_IMM8_OPCODE_X1 = 42,
+  V2CMPLTU_RRR_0_OPCODE_X0 = 123,
+  V2CMPLTU_RRR_0_OPCODE_X1 = 79,
+  V2CMPNE_RRR_0_OPCODE_X0 = 124,
+  V2CMPNE_RRR_0_OPCODE_X1 = 80,
+  V2DOTPA_RRR_0_OPCODE_X0 = 125,
+  V2DOTP_RRR_0_OPCODE_X0 = 126,
+  V2INT_H_RRR_0_OPCODE_X0 = 127,
+  V2INT_H_RRR_0_OPCODE_X1 = 81,
+  V2INT_L_RRR_0_OPCODE_X0 = 128,
+  V2INT_L_RRR_0_OPCODE_X1 = 82,
+  V2MAXSI_IMM8_OPCODE_X0 = 18,
+  V2MAXSI_IMM8_OPCODE_X1 = 43,
+  V2MAXS_RRR_0_OPCODE_X0 = 129,
+  V2MAXS_RRR_0_OPCODE_X1 = 83,
+  V2MINSI_IMM8_OPCODE_X0 = 19,
+  V2MINSI_IMM8_OPCODE_X1 = 44,
+  V2MINS_RRR_0_OPCODE_X0 = 130,
+  V2MINS_RRR_0_OPCODE_X1 = 84,
+  V2MNZ_RRR_0_OPCODE_X0 = 131,
+  V2MNZ_RRR_0_OPCODE_X1 = 85,
+  V2MULFSC_RRR_0_OPCODE_X0 = 132,
+  V2MULS_RRR_0_OPCODE_X0 = 133,
+  V2MULTS_RRR_0_OPCODE_X0 = 134,
+  V2MZ_RRR_0_OPCODE_X0 = 135,
+  V2MZ_RRR_0_OPCODE_X1 = 86,
+  V2PACKH_RRR_0_OPCODE_X0 = 136,
+  V2PACKH_RRR_0_OPCODE_X1 = 87,
+  V2PACKL_RRR_0_OPCODE_X0 = 137,
+  V2PACKL_RRR_0_OPCODE_X1 = 88,
+  V2PACKUC_RRR_0_OPCODE_X0 = 138,
+  V2PACKUC_RRR_0_OPCODE_X1 = 89,
+  V2SADAS_RRR_0_OPCODE_X0 = 139,
+  V2SADAU_RRR_0_OPCODE_X0 = 140,
+  V2SADS_RRR_0_OPCODE_X0 = 141,
+  V2SADU_RRR_0_OPCODE_X0 = 142,
+  V2SHLI_SHIFT_OPCODE_X0 = 10,
+  V2SHLI_SHIFT_OPCODE_X1 = 10,
+  V2SHLSC_RRR_0_OPCODE_X0 = 143,
+  V2SHLSC_RRR_0_OPCODE_X1 = 90,
+  V2SHL_RRR_0_OPCODE_X0 = 144,
+  V2SHL_RRR_0_OPCODE_X1 = 91,
+  V2SHRSI_SHIFT_OPCODE_X0 = 11,
+  V2SHRSI_SHIFT_OPCODE_X1 = 11,
+  V2SHRS_RRR_0_OPCODE_X0 = 145,
+  V2SHRS_RRR_0_OPCODE_X1 = 92,
+  V2SHRUI_SHIFT_OPCODE_X0 = 12,
+  V2SHRUI_SHIFT_OPCODE_X1 = 12,
+  V2SHRU_RRR_0_OPCODE_X0 = 146,
+  V2SHRU_RRR_0_OPCODE_X1 = 93,
+  V2SUBSC_RRR_0_OPCODE_X0 = 147,
+  V2SUBSC_RRR_0_OPCODE_X1 = 94,
+  V2SUB_RRR_0_OPCODE_X0 = 148,
+  V2SUB_RRR_0_OPCODE_X1 = 95,
+  V4ADDSC_RRR_0_OPCODE_X0 = 149,
+  V4ADDSC_RRR_0_OPCODE_X1 = 96,
+  V4ADD_RRR_0_OPCODE_X0 = 150,
+  V4ADD_RRR_0_OPCODE_X1 = 97,
+  V4INT_H_RRR_0_OPCODE_X0 = 151,
+  V4INT_H_RRR_0_OPCODE_X1 = 98,
+  V4INT_L_RRR_0_OPCODE_X0 = 152,
+  V4INT_L_RRR_0_OPCODE_X1 = 99,
+  V4PACKSC_RRR_0_OPCODE_X0 = 153,
+  V4PACKSC_RRR_0_OPCODE_X1 = 100,
+  V4SHLSC_RRR_0_OPCODE_X0 = 154,
+  V4SHLSC_RRR_0_OPCODE_X1 = 101,
+  V4SHL_RRR_0_OPCODE_X0 = 155,
+  V4SHL_RRR_0_OPCODE_X1 = 102,
+  V4SHRS_RRR_0_OPCODE_X0 = 156,
+  V4SHRS_RRR_0_OPCODE_X1 = 103,
+  V4SHRU_RRR_0_OPCODE_X0 = 157,
+  V4SHRU_RRR_0_OPCODE_X1 = 104,
+  V4SUBSC_RRR_0_OPCODE_X0 = 158,
+  V4SUBSC_RRR_0_OPCODE_X1 = 105,
+  V4SUB_RRR_0_OPCODE_X0 = 159,
+  V4SUB_RRR_0_OPCODE_X1 = 106,
+  WH64_UNARY_OPCODE_X1 = 38,
+  XORI_IMM8_OPCODE_X0 = 20,
+  XORI_IMM8_OPCODE_X1 = 45,
+  XOR_RRR_0_OPCODE_X0 = 160,
+  XOR_RRR_0_OPCODE_X1 = 107,
+  XOR_RRR_5_OPCODE_Y0 = 3,
+  XOR_RRR_5_OPCODE_Y1 = 3
+};
+
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* __ARCH_OPCODE_H__ */
diff --git a/qemu/target-tilegx/simd_helper.c b/qemu/target-tilegx/simd_helper.c
new file mode 100644
index 000000000..2d40ddb63
--- /dev/null
+++ b/qemu/target-tilegx/simd_helper.c
@@ -0,0 +1,166 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+
+
+/* Broadcast a value to all elements of a vector.  */
+#define V1(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+uint64_t helper_v1multu(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 64; i += 8) {
+        unsigned ae = extract64(a, i, 8);
+        unsigned be = extract64(b, i, 8);
+        r = deposit64(r, i, 8, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v2mults(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    /* While the instruction talks about signed inputs, with a
+       truncated result the sign of the inputs doesn't matter.  */
+    for (i = 0; i < 64; i += 16) {
+        unsigned ae = extract64(a, i, 16);
+        unsigned be = extract64(b, i, 16);
+        r = deposit64(r, i, 16, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v1shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v2shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v1shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v2shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v1shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 7;
+    for (i = 0; i < 64; i += 8) {
+        r = deposit64(r, i, 8, sextract64(a, i + b, 8 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v2shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 15;
+    for (i = 0; i < 64; i += 16) {
+        r = deposit64(r, i, 16, sextract64(a, i + b, 16 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i + 32, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i + 32, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i + 32, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i + 32, 16));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i, 16));
+    }
+    return r;
+}
diff --git a/qemu/target-tilegx/spr_def_64.h b/qemu/target-tilegx/spr_def_64.h
new file mode 100644
index 000000000..67a6c1751
--- /dev/null
+++ b/qemu/target-tilegx/spr_def_64.h
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ */
+
+#ifndef __DOXYGEN__
+
+#ifndef __ARCH_SPR_DEF_64_H__
+#define __ARCH_SPR_DEF_64_H__
+
+#define SPR_AUX_PERF_COUNT_0 0x2105
+#define SPR_AUX_PERF_COUNT_1 0x2106
+#define SPR_AUX_PERF_COUNT_CTL 0x2107
+#define SPR_AUX_PERF_COUNT_STS 0x2108
+#define SPR_CMPEXCH_VALUE 0x2780
+#define SPR_CYCLE 0x2781
+#define SPR_DONE 0x2705
+#define SPR_DSTREAM_PF 0x2706
+#define SPR_EVENT_BEGIN 0x2782
+#define SPR_EVENT_END 0x2783
+#define SPR_EX_CONTEXT_0_0 0x2580
+#define SPR_EX_CONTEXT_0_1 0x2581
+#define SPR_EX_CONTEXT_0_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_0_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_0_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_0_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_0_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_0_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_1_0 0x2480
+#define SPR_EX_CONTEXT_1_1 0x2481
+#define SPR_EX_CONTEXT_1_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_1_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_1_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_1_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_1_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_1_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_2_0 0x2380
+#define SPR_EX_CONTEXT_2_1 0x2381
+#define SPR_EX_CONTEXT_2_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_2_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_2_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_2_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_2_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_2_1__ICS_MASK  0x4
+#define SPR_FAIL 0x2707
+#define SPR_IDN_AVAIL_EN 0x1a05
+#define SPR_IDN_DATA_AVAIL 0x0a80
+#define SPR_IDN_DEADLOCK_TIMEOUT 0x1806
+#define SPR_IDN_DEMUX_COUNT_0 0x0a05
+#define SPR_IDN_DEMUX_COUNT_1 0x0a06
+#define SPR_IDN_DIRECTION_PROTECT 0x1405
+#define SPR_IDN_PENDING 0x0a08
+#define SPR_ILL_TRANS_REASON__I_STREAM_VA_RMASK 0x1
+#define SPR_INTCTRL_0_STATUS 0x2505
+#define SPR_INTCTRL_1_STATUS 0x2405
+#define SPR_INTCTRL_2_STATUS 0x2305
+#define SPR_INTERRUPT_CRITICAL_SECTION 0x2708
+#define SPR_INTERRUPT_MASK_0 0x2506
+#define SPR_INTERRUPT_MASK_1 0x2406
+#define SPR_INTERRUPT_MASK_2 0x2306
+#define SPR_INTERRUPT_MASK_RESET_0 0x2507
+#define SPR_INTERRUPT_MASK_RESET_1 0x2407
+#define SPR_INTERRUPT_MASK_RESET_2 0x2307
+#define SPR_INTERRUPT_MASK_SET_0 0x2508
+#define SPR_INTERRUPT_MASK_SET_1 0x2408
+#define SPR_INTERRUPT_MASK_SET_2 0x2308
+#define SPR_INTERRUPT_VECTOR_BASE_0 0x2509
+#define SPR_INTERRUPT_VECTOR_BASE_1 0x2409
+#define SPR_INTERRUPT_VECTOR_BASE_2 0x2309
+#define SPR_INTERRUPT_VECTOR_BASE_3 0x2209
+#define SPR_IPI_EVENT_0 0x1f05
+#define SPR_IPI_EVENT_1 0x1e05
+#define SPR_IPI_EVENT_2 0x1d05
+#define SPR_IPI_EVENT_RESET_0 0x1f06
+#define SPR_IPI_EVENT_RESET_1 0x1e06
+#define SPR_IPI_EVENT_RESET_2 0x1d06
+#define SPR_IPI_EVENT_SET_0 0x1f07
+#define SPR_IPI_EVENT_SET_1 0x1e07
+#define SPR_IPI_EVENT_SET_2 0x1d07
+#define SPR_IPI_MASK_0 0x1f08
+#define SPR_IPI_MASK_1 0x1e08
+#define SPR_IPI_MASK_2 0x1d08
+#define SPR_IPI_MASK_RESET_0 0x1f09
+#define SPR_IPI_MASK_RESET_1 0x1e09
+#define SPR_IPI_MASK_RESET_2 0x1d09
+#define SPR_IPI_MASK_SET_0 0x1f0a
+#define SPR_IPI_MASK_SET_1 0x1e0a
+#define SPR_IPI_MASK_SET_2 0x1d0a
+#define SPR_MPL_AUX_PERF_COUNT_SET_0 0x2100
+#define SPR_MPL_AUX_PERF_COUNT_SET_1 0x2101
+#define SPR_MPL_AUX_PERF_COUNT_SET_2 0x2102
+#define SPR_MPL_AUX_TILE_TIMER_SET_0 0x1700
+#define SPR_MPL_AUX_TILE_TIMER_SET_1 0x1701
+#define SPR_MPL_AUX_TILE_TIMER_SET_2 0x1702
+#define SPR_MPL_IDN_ACCESS_SET_0 0x0a00
+#define SPR_MPL_IDN_ACCESS_SET_1 0x0a01
+#define SPR_MPL_IDN_ACCESS_SET_2 0x0a02
+#define SPR_MPL_IDN_AVAIL_SET_0 0x1a00
+#define SPR_MPL_IDN_AVAIL_SET_1 0x1a01
+#define SPR_MPL_IDN_AVAIL_SET_2 0x1a02
+#define SPR_MPL_IDN_COMPLETE_SET_0 0x0500
+#define SPR_MPL_IDN_COMPLETE_SET_1 0x0501
+#define SPR_MPL_IDN_COMPLETE_SET_2 0x0502
+#define SPR_MPL_IDN_FIREWALL_SET_0 0x1400
+#define SPR_MPL_IDN_FIREWALL_SET_1 0x1401
+#define SPR_MPL_IDN_FIREWALL_SET_2 0x1402
+#define SPR_MPL_IDN_TIMER_SET_0 0x1800
+#define SPR_MPL_IDN_TIMER_SET_1 0x1801
+#define SPR_MPL_IDN_TIMER_SET_2 0x1802
+#define SPR_MPL_INTCTRL_0_SET_0 0x2500
+#define SPR_MPL_INTCTRL_0_SET_1 0x2501
+#define SPR_MPL_INTCTRL_0_SET_2 0x2502
+#define SPR_MPL_INTCTRL_1_SET_0 0x2400
+#define SPR_MPL_INTCTRL_1_SET_1 0x2401
+#define SPR_MPL_INTCTRL_1_SET_2 0x2402
+#define SPR_MPL_INTCTRL_2_SET_0 0x2300
+#define SPR_MPL_INTCTRL_2_SET_1 0x2301
+#define SPR_MPL_INTCTRL_2_SET_2 0x2302
+#define SPR_MPL_IPI_0 0x1f04
+#define SPR_MPL_IPI_0_SET_0 0x1f00
+#define SPR_MPL_IPI_0_SET_1 0x1f01
+#define SPR_MPL_IPI_0_SET_2 0x1f02
+#define SPR_MPL_IPI_1 0x1e04
+#define SPR_MPL_IPI_1_SET_0 0x1e00
+#define SPR_MPL_IPI_1_SET_1 0x1e01
+#define SPR_MPL_IPI_1_SET_2 0x1e02
+#define SPR_MPL_IPI_2 0x1d04
+#define SPR_MPL_IPI_2_SET_0 0x1d00
+#define SPR_MPL_IPI_2_SET_1 0x1d01
+#define SPR_MPL_IPI_2_SET_2 0x1d02
+#define SPR_MPL_PERF_COUNT_SET_0 0x2000
+#define SPR_MPL_PERF_COUNT_SET_1 0x2001
+#define SPR_MPL_PERF_COUNT_SET_2 0x2002
+#define SPR_MPL_UDN_ACCESS_SET_0 0x0b00
+#define SPR_MPL_UDN_ACCESS_SET_1 0x0b01
+#define SPR_MPL_UDN_ACCESS_SET_2 0x0b02
+#define SPR_MPL_UDN_AVAIL_SET_0 0x1b00
+#define SPR_MPL_UDN_AVAIL_SET_1 0x1b01
+#define SPR_MPL_UDN_AVAIL_SET_2 0x1b02
+#define SPR_MPL_UDN_COMPLETE_SET_0 0x0600
+#define SPR_MPL_UDN_COMPLETE_SET_1 0x0601
+#define SPR_MPL_UDN_COMPLETE_SET_2 0x0602
+#define SPR_MPL_UDN_FIREWALL_SET_0 0x1500
+#define SPR_MPL_UDN_FIREWALL_SET_1 0x1501
+#define SPR_MPL_UDN_FIREWALL_SET_2 0x1502
+#define SPR_MPL_UDN_TIMER_SET_0 0x1900
+#define SPR_MPL_UDN_TIMER_SET_1 0x1901
+#define SPR_MPL_UDN_TIMER_SET_2 0x1902
+#define SPR_MPL_WORLD_ACCESS_SET_0 0x2700
+#define SPR_MPL_WORLD_ACCESS_SET_1 0x2701
+#define SPR_MPL_WORLD_ACCESS_SET_2 0x2702
+#define SPR_PASS 0x2709
+#define SPR_PERF_COUNT_0 0x2005
+#define SPR_PERF_COUNT_1 0x2006
+#define SPR_PERF_COUNT_CTL 0x2007
+#define SPR_PERF_COUNT_DN_CTL 0x2008
+#define SPR_PERF_COUNT_STS 0x2009
+#define SPR_PROC_STATUS 0x2784
+#define SPR_SIM_CONTROL 0x2785
+#define SPR_SINGLE_STEP_CONTROL_0 0x0405
+#define SPR_SINGLE_STEP_CONTROL_0__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_0__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_1 0x0305
+#define SPR_SINGLE_STEP_CONTROL_1__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_1__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_2 0x0205
+#define SPR_SINGLE_STEP_CONTROL_2__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_2__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_EN_0_0 0x250a
+#define SPR_SINGLE_STEP_EN_0_1 0x240a
+#define SPR_SINGLE_STEP_EN_0_2 0x230a
+#define SPR_SINGLE_STEP_EN_1_0 0x250b
+#define SPR_SINGLE_STEP_EN_1_1 0x240b
+#define SPR_SINGLE_STEP_EN_1_2 0x230b
+#define SPR_SINGLE_STEP_EN_2_0 0x250c
+#define SPR_SINGLE_STEP_EN_2_1 0x240c
+#define SPR_SINGLE_STEP_EN_2_2 0x230c
+#define SPR_SYSTEM_SAVE_0_0 0x2582
+#define SPR_SYSTEM_SAVE_0_1 0x2583
+#define SPR_SYSTEM_SAVE_0_2 0x2584
+#define SPR_SYSTEM_SAVE_0_3 0x2585
+#define SPR_SYSTEM_SAVE_1_0 0x2482
+#define SPR_SYSTEM_SAVE_1_1 0x2483
+#define SPR_SYSTEM_SAVE_1_2 0x2484
+#define SPR_SYSTEM_SAVE_1_3 0x2485
+#define SPR_SYSTEM_SAVE_2_0 0x2382
+#define SPR_SYSTEM_SAVE_2_1 0x2383
+#define SPR_SYSTEM_SAVE_2_2 0x2384
+#define SPR_SYSTEM_SAVE_2_3 0x2385
+#define SPR_TILE_COORD 0x270b
+#define SPR_TILE_RTF_HWM 0x270c
+#define SPR_TILE_TIMER_CONTROL 0x1605
+#define SPR_UDN_AVAIL_EN 0x1b05
+#define SPR_UDN_DATA_AVAIL 0x0b80
+#define SPR_UDN_DEADLOCK_TIMEOUT 0x1906
+#define SPR_UDN_DEMUX_COUNT_0 0x0b05
+#define SPR_UDN_DEMUX_COUNT_1 0x0b06
+#define SPR_UDN_DEMUX_COUNT_2 0x0b07
+#define SPR_UDN_DEMUX_COUNT_3 0x0b08
+#define SPR_UDN_DIRECTION_PROTECT 0x1505
+#define SPR_UDN_PENDING 0x0b0a
+#define SPR_WATCH_MASK 0x200a
+#define SPR_WATCH_VAL 0x200b
+
+#endif /* !defined(__ARCH_SPR_DEF_64_H__) */
+
+#endif /* !defined(__DOXYGEN__) */
diff --git a/qemu/target-tilegx/translate.c b/qemu/target-tilegx/translate.c
new file mode 100644
index 000000000..03918ebd5
--- /dev/null
+++ b/qemu/target-tilegx/translate.c
@@ -0,0 +1,2451 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/log.h"
+#include "exec/log.h"
+#include "disas/disas.h"
+#include "tcg-op.h"
+#include "exec/cpu_ldst.h"
+#include "linux-user/syscall_defs.h"
+
+#include "opcode_tilegx.h"
+#include "spr_def_64.h"
+
+#define FMT64X                          "%016" PRIx64
+
+static TCGv_env cpu_env;
+static TCGv cpu_pc;
+static TCGv cpu_regs[TILEGX_R_COUNT];
+
+static const char * const reg_names[64] = {
+     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+     "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+    "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+    "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+    "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+    "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+    "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr",
+    "sn", "idn0", "idn1", "udn0", "udn1", "udn2", "udn2", "zero"
+};
+
+/* Modified registers are cached in temporaries until the end of the bundle. */
+typedef struct {
+    unsigned reg;
+    TCGv val;
+} DisasContextTemp;
+
+#define MAX_WRITEBACK 4
+
+/* This is the state at translation time.  */
+typedef struct {
+    uint64_t pc;		/* Current pc */
+
+    TCGv zero;                  /* For zero register */
+
+    DisasContextTemp wb[MAX_WRITEBACK];
+    int num_wb;
+    int mmuidx;
+    bool exit_tb;
+    TileExcp atomic_excp;
+
+    struct {
+        TCGCond cond;    /* branch condition */
+        TCGv dest;       /* branch destination */
+        TCGv val1;       /* value to be compared against zero, for cond */
+    } jmp;               /* Jump object, only once in each TB block */
+} DisasContext;
+
+#include "exec/gen-icount.h"
+
+/* Differentiate the various pipe encodings.  */
+#define TY_X0  0
+#define TY_X1  1
+#define TY_Y0  2
+#define TY_Y1  3
+
+/* Remerge the base opcode and extension fields for switching.
+   The X opcode fields are 3 bits; Y0/Y1 opcode fields are 4 bits;
+   Y2 opcode field is 2 bits.  */
+#define OE(OP, EXT, XY) (TY_##XY + OP * 4 + EXT * 64)
+
+/* Similar, but for Y2 only.  */
+#define OEY2(OP, MODE) (OP + MODE * 4)
+
+/* Similar, but make sure opcode names match up.  */
+#define OE_RR_X0(E)    OE(RRR_0_OPCODE_X0, E##_UNARY_OPCODE_X0, X0)
+#define OE_RR_X1(E)    OE(RRR_0_OPCODE_X1, E##_UNARY_OPCODE_X1, X1)
+#define OE_RR_Y0(E)    OE(RRR_1_OPCODE_Y0, E##_UNARY_OPCODE_Y0, Y0)
+#define OE_RR_Y1(E)    OE(RRR_1_OPCODE_Y1, E##_UNARY_OPCODE_Y1, Y1)
+#define OE_RRR(E,N,XY) OE(RRR_##N##_OPCODE_##XY, E##_RRR_##N##_OPCODE_##XY, XY)
+#define OE_IM(E,XY)    OE(IMM8_OPCODE_##XY, E##_IMM8_OPCODE_##XY, XY)
+#define OE_SH(E,XY)    OE(SHIFT_OPCODE_##XY, E##_SHIFT_OPCODE_##XY, XY)
+
+#define V1_IMM(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2_IMM(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+static void gen_exception(DisasContext *dc, TileExcp num)
+{
+    TCGv_i32 tmp;
+
+    tcg_gen_movi_tl(cpu_pc, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+
+    tmp = tcg_const_i32(num);
+    gen_helper_exception(cpu_env, tmp);
+    tcg_temp_free_i32(tmp);
+    dc->exit_tb = true;
+}
+
+static bool check_gr(DisasContext *dc, uint8_t reg)
+{
+    if (likely(reg < TILEGX_R_COUNT)) {
+        return true;
+    }
+
+    switch (reg) {
+    case TILEGX_R_SN:
+    case TILEGX_R_ZERO:
+        break;
+    case TILEGX_R_IDN0:
+    case TILEGX_R_IDN1:
+        gen_exception(dc, TILEGX_EXCP_REG_IDN_ACCESS);
+        break;
+    case TILEGX_R_UDN0:
+    case TILEGX_R_UDN1:
+    case TILEGX_R_UDN2:
+    case TILEGX_R_UDN3:
+        gen_exception(dc, TILEGX_EXCP_REG_UDN_ACCESS);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return false;
+}
+
+static TCGv load_zero(DisasContext *dc)
+{
+    if (TCGV_IS_UNUSED_I64(dc->zero)) {
+        dc->zero = tcg_const_i64(0);
+    }
+    return dc->zero;
+}
+
+static TCGv load_gr(DisasContext *dc, unsigned reg)
+{
+    if (check_gr(dc, reg)) {
+        return cpu_regs[reg];
+    }
+    return load_zero(dc);
+}
+
+static TCGv dest_gr(DisasContext *dc, unsigned reg)
+{
+    int n;
+
+    /* Skip the result, mark the exception if necessary, and continue */
+    check_gr(dc, reg);
+
+    n = dc->num_wb++;
+    dc->wb[n].reg = reg;
+    return dc->wb[n].val = tcg_temp_new_i64();
+}
+
+static void gen_saturate_op(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                            void (*operate)(TCGv, TCGv, TCGv))
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_ext32s_tl(tdest, tsrca);
+    tcg_gen_ext32s_tl(t0, tsrcb);
+    operate(tdest, tdest, t0);
+
+    tcg_gen_movi_tl(t0, 0x7fffffff);
+    tcg_gen_movcond_tl(TCG_COND_GT, tdest, tdest, t0, t0, tdest);
+    tcg_gen_movi_tl(t0, -0x80000000LL);
+    tcg_gen_movcond_tl(TCG_COND_LT, tdest, tdest, t0, t0, tdest);
+
+    tcg_temp_free(t0);
+}
+
+static void gen_atomic_excp(DisasContext *dc, unsigned dest, TCGv tdest,
+                            TCGv tsrca, TCGv tsrcb, TileExcp excp)
+{
+#ifdef CONFIG_USER_ONLY
+    TCGv_i32 t;
+
+    tcg_gen_st_tl(tsrca, cpu_env, offsetof(CPUTLGState, atomic_srca));
+    tcg_gen_st_tl(tsrcb, cpu_env, offsetof(CPUTLGState, atomic_srcb));
+    t = tcg_const_i32(dest);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUTLGState, atomic_dstr));
+    tcg_temp_free_i32(t);
+
+    /* We're going to write the real result in the exception.  But in
+       the meantime we've already created a writeback register, and
+       we don't want that to remain uninitialized.  */
+    tcg_gen_movi_tl(tdest, 0);
+
+    /* Note that we need to delay issuing the exception that implements
+       the atomic operation until after writing back the results of the
+       instruction occupying the X0 pipe.  */
+    dc->atomic_excp = excp;
+#else
+    gen_exception(dc, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+}
+
+/* Shift the 128-bit value TSRCA:TSRCD right by the number of bytes
+   specified by the bottom 3 bits of TSRCB, and set TDEST to the
+   low 64 bits of the resulting value.  */
+static void gen_dblalign(TCGv tdest, TCGv tsrcd, TCGv tsrca, TCGv tsrcb)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_andi_tl(t0, tsrcb, 7);
+    tcg_gen_shli_tl(t0, t0, 3);
+    tcg_gen_shr_tl(tdest, tsrcd, t0);
+
+    /* We want to do "t0 = tsrca << (64 - t0)".  Two's complement
+       arithmetic on a 6-bit field tells us that 64 - t0 is equal
+       to (t0 ^ 63) + 1.  So we can do the shift in two parts,
+       neither of which will be an invalid shift by 64.  */
+    tcg_gen_xori_tl(t0, t0, 63);
+    tcg_gen_shl_tl(t0, tsrca, t0);
+    tcg_gen_shli_tl(t0, t0, 1);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+/* Similarly, except that the 128-bit value is TSRCA:TSRCB, and the
+   right shift is an immediate.  */
+static void gen_dblaligni(TCGv tdest, TCGv tsrca, TCGv tsrcb, int shr)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_shri_tl(t0, tsrcb, shr);
+    tcg_gen_shli_tl(tdest, tsrca, 64 - shr);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+typedef enum {
+    LU, LS, HU, HS
+} MulHalf;
+
+static void gen_ext_half(TCGv d, TCGv s, MulHalf h)
+{
+    switch (h) {
+    case LU:
+        tcg_gen_ext32u_tl(d, s);
+        break;
+    case LS:
+        tcg_gen_ext32s_tl(d, s);
+        break;
+    case HU:
+        tcg_gen_shri_tl(d, s, 32);
+        break;
+    case HS:
+        tcg_gen_sari_tl(d, s, 32);
+        break;
+    }
+}
+
+static void gen_mul_half(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                         MulHalf ha, MulHalf hb)
+{
+    TCGv t = tcg_temp_new();
+    gen_ext_half(t, tsrca, ha);
+    gen_ext_half(tdest, tsrcb, hb);
+    tcg_gen_mul_tl(tdest, tdest, t);
+    tcg_temp_free(t);
+}
+
+static void gen_cmul2(TCGv tdest, TCGv tsrca, TCGv tsrcb, int sh, int rd)
+{
+    TCGv_i32 tsh = tcg_const_i32(sh);
+    TCGv_i32 trd = tcg_const_i32(rd);
+    gen_helper_cmul2(tdest, tsrca, tsrcb, tsh, trd);
+    tcg_temp_free_i32(tsh);
+    tcg_temp_free_i32(trd);
+}
+
+static TileExcp gen_st_opcode(DisasContext *dc, unsigned dest, unsigned srca,
+                              unsigned srcb, TCGMemOp memop, const char *name)
+{
+    if (dest) {
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    tcg_gen_qemu_st_tl(load_gr(dc, srcb), load_gr(dc, srca),
+		       dc->mmuidx, memop);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", name,
+                  reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_st_add_opcode(DisasContext *dc, unsigned srca, unsigned srcb,
+                                  int imm, TCGMemOp memop, const char *name)
+{
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+
+    tcg_gen_qemu_st_tl(tsrcb, tsrca, dc->mmuidx, memop);
+    tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", name,
+                  reg_names[srca], reg_names[srcb], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+/* Equality comparison with zero can be done quickly and efficiently.  */
+static void gen_v1cmpeq0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* ~(((v & m) + m) | m | v).  Sets the msb for each byte == 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, m);
+    tcg_gen_nor_tl(c, c, v);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+static void gen_v1cmpne0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* (((v & m) + m) | v) & ~m.  Sets the msb for each byte != 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, v);
+    tcg_gen_andc_tl(c, c, m);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+/* Vector addition can be performed via arithmetic plus masking.  It is
+   efficient this way only for 4 or more elements.  */
+static void gen_v12add(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tmask = tcg_const_tl(~sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a & ~sign) + (b & ~sign)) ^ ((a ^ b) & sign).  */
+    tcg_gen_and_tl(t0, tsrca, tmask);
+    tcg_gen_and_tl(t1, tsrcb, tmask);
+    tcg_gen_add_tl(tdest, t0, t1);
+    tcg_gen_xor_tl(t0, tsrca, tsrcb);
+    tcg_gen_andc_tl(t0, t0, tmask);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tmask);
+}
+
+/* Similarly for vector subtraction.  */
+static void gen_v12sub(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tsign = tcg_const_tl(sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a | sign) - (b & ~sign)) ^ ((a ^ ~b) & sign).  */
+    tcg_gen_or_tl(t0, tsrca, tsign);
+    tcg_gen_andc_tl(t1, tsrcb, tsign);
+    tcg_gen_sub_tl(tdest, t0, t1);
+    tcg_gen_eqv_tl(t0, tsrca, tsrcb);
+    tcg_gen_and_tl(t0, t0, tsign);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tsign);
+}
+
+static void gen_v4sh(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extrl_i64_i32(bl, b64);
+    tcg_gen_andi_i32(bl, bl, 31);
+    generate(al, al, bl);
+    generate(ah, ah, bl);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+}
+
+static void gen_v4op(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+    TCGv_i32 bh = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extr_i64_i32(bl, bh, b64);
+    generate(al, al, bl);
+    generate(ah, ah, bh);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+    tcg_temp_free_i32(bh);
+}
+
+static TileExcp gen_signal(DisasContext *dc, int signo, int sigcode,
+                           const char *mnemonic)
+{
+    TCGv_i32 t0 = tcg_const_i32(signo);
+    TCGv_i32 t1 = tcg_const_i32(sigcode);
+
+    tcg_gen_st_i32(t0, cpu_env, offsetof(CPUTLGState, signo));
+    tcg_gen_st_i32(t1, cpu_env, offsetof(CPUTLGState, sigcode));
+
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(t0);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+    return TILEGX_EXCP_SIGNAL;
+}
+
+static bool parse_from_addli(uint64_t bundle, int *signo, int *sigcode)
+{
+    int imm;
+
+    if ((get_Opcode_X0(bundle) != ADDLI_OPCODE_X0)
+        || (get_Dest_X0(bundle) != TILEGX_R_ZERO)
+        || (get_SrcA_X0(bundle) != TILEGX_R_ZERO)) {
+        return false;
+    }
+
+    imm = get_Imm16_X0(bundle);
+    *signo = imm & 0x3f;
+    *sigcode = (imm >> 6) & 0xf;
+
+    /* ??? The linux kernel validates both signo and the sigcode vs the
+       known max for each signal.  Don't bother here.  */
+    return true;
+}
+
+static TileExcp gen_specill(DisasContext *dc, unsigned dest, unsigned srca,
+                            uint64_t bundle)
+{
+    const char *mnemonic;
+    int signo;
+    int sigcode;
+
+    if (dest == 0x1c && srca == 0x25) {
+        signo = TARGET_SIGTRAP;
+        sigcode = TARGET_TRAP_BRKPT;
+        mnemonic = "bpt";
+    } else if (dest == 0x1d && srca == 0x25
+               && parse_from_addli(bundle, &signo, &sigcode)) {
+        mnemonic = "raise";
+    } else {
+        signo = TARGET_SIGILL;
+        sigcode = TARGET_ILL_ILLOPC;
+        mnemonic = "ill";
+    }
+
+    return gen_signal(dc, signo, sigcode, mnemonic);
+}
+
+static TileExcp gen_rr_opcode(DisasContext *dc, unsigned opext,
+                              unsigned dest, unsigned srca, uint64_t bundle)
+{
+    TCGv tdest, tsrca;
+    const char *mnemonic;
+    TCGMemOp memop;
+    TileExcp ret = TILEGX_EXCP_NONE;
+    bool prefetch_nofault = false;
+
+    /* Eliminate instructions with no output before doing anything else.  */
+    switch (opext) {
+    case OE_RR_Y0(NOP):
+    case OE_RR_Y1(NOP):
+    case OE_RR_X0(NOP):
+    case OE_RR_X1(NOP):
+        mnemonic = "nop";
+        goto done0;
+    case OE_RR_Y0(FNOP):
+    case OE_RR_Y1(FNOP):
+    case OE_RR_X0(FNOP):
+    case OE_RR_X1(FNOP):
+        mnemonic = "fnop";
+        goto done0;
+    case OE_RR_X1(DRAIN):
+        mnemonic = "drain";
+        goto done0;
+    case OE_RR_X1(FLUSHWB):
+        mnemonic = "flushwb";
+        goto done0;
+    case OE_RR_X1(ILL):
+        return gen_specill(dc, dest, srca, bundle);
+    case OE_RR_Y1(ILL):
+        return gen_signal(dc, TARGET_SIGILL, TARGET_ILL_ILLOPC, "ill");
+    case OE_RR_X1(MF):
+        mnemonic = "mf";
+        goto done0;
+    case OE_RR_X1(NAP):
+        /* ??? This should yield, especially in system mode.  */
+        mnemonic = "nap";
+        goto done0;
+    case OE_RR_X1(IRET):
+        gen_helper_ext01_ics(cpu_env);
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_ld_tl(dc->jmp.dest, cpu_env,
+                      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]));
+        tcg_gen_andi_tl(dc->jmp.dest, dc->jmp.dest, ~7);
+        mnemonic = "iret";
+        goto done0;
+    case OE_RR_X1(SWINT0):
+    case OE_RR_X1(SWINT2):
+    case OE_RR_X1(SWINT3):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(SWINT1):
+        ret = TILEGX_EXCP_SYSCALL;
+        mnemonic = "swint1";
+    done0:
+        if (srca || dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+        return ret;
+
+    case OE_RR_X1(DTLBPR):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(FINV):
+        mnemonic = "finv";
+        goto done1;
+    case OE_RR_X1(FLUSH):
+        mnemonic = "flush";
+        goto done1;
+    case OE_RR_X1(ICOH):
+        mnemonic = "icoh";
+        goto done1;
+    case OE_RR_X1(INV):
+        mnemonic = "inv";
+        goto done1;
+    case OE_RR_X1(WH64):
+        mnemonic = "wh64";
+        goto done1;
+    case OE_RR_X1(JRP):
+    case OE_RR_Y1(JRP):
+        mnemonic = "jrp";
+        goto do_jr;
+    case OE_RR_X1(JR):
+    case OE_RR_Y1(JR):
+        mnemonic = "jr";
+        goto do_jr;
+    case OE_RR_X1(JALRP):
+    case OE_RR_Y1(JALRP):
+        mnemonic = "jalrp";
+        goto do_jalr;
+    case OE_RR_X1(JALR):
+    case OE_RR_Y1(JALR):
+        mnemonic = "jalr";
+    do_jalr:
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+    do_jr:
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_andi_tl(dc->jmp.dest, load_gr(dc, srca), ~7);
+    done1:
+        if (dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s", mnemonic, reg_names[srca]);
+        return ret;
+    }
+
+    tdest = dest_gr(dc, dest);
+    tsrca = load_gr(dc, srca);
+
+    switch (opext) {
+    case OE_RR_X0(CNTLZ):
+    case OE_RR_Y0(CNTLZ):
+        gen_helper_cntlz(tdest, tsrca);
+        mnemonic = "cntlz";
+        break;
+    case OE_RR_X0(CNTTZ):
+    case OE_RR_Y0(CNTTZ):
+        gen_helper_cnttz(tdest, tsrca);
+        mnemonic = "cnttz";
+        break;
+    case OE_RR_X0(FSINGLE_PACK1):
+    case OE_RR_Y0(FSINGLE_PACK1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(LD1S):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OE_RR_X1(LD1U):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD2S):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OE_RR_X1(LD2U):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD4S):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OE_RR_X1(LD4U):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LDNT1S):
+        memop = MO_SB;
+        mnemonic = "ldnt1s";
+        goto do_load;
+    case OE_RR_X1(LDNT1U):
+        memop = MO_UB;
+        mnemonic = "ldnt1u";
+        goto do_load;
+    case OE_RR_X1(LDNT2S):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s";
+        goto do_load;
+    case OE_RR_X1(LDNT2U):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u";
+        goto do_load;
+    case OE_RR_X1(LDNT4S):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s";
+        goto do_load;
+    case OE_RR_X1(LDNT4U):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u";
+        goto do_load;
+    case OE_RR_X1(LDNT):
+        memop = MO_TEQ;
+        mnemonic = "ldnt";
+        goto do_load;
+    case OE_RR_X1(LD):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        break;
+    case OE_RR_X1(LDNA):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        mnemonic = "ldna";
+        break;
+    case OE_RR_X1(LNK):
+    case OE_RR_Y1(LNK):
+        if (srca) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        tcg_gen_movi_tl(tdest, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "lnk";
+        break;
+    case OE_RR_X0(PCNT):
+    case OE_RR_Y0(PCNT):
+        gen_helper_pcnt(tdest, tsrca);
+        mnemonic = "pcnt";
+        break;
+    case OE_RR_X0(REVBITS):
+    case OE_RR_Y0(REVBITS):
+        gen_helper_revbits(tdest, tsrca);
+        mnemonic = "revbits";
+        break;
+    case OE_RR_X0(REVBYTES):
+    case OE_RR_Y0(REVBYTES):
+        tcg_gen_bswap64_tl(tdest, tsrca);
+        mnemonic = "revbytes";
+        break;
+    case OE_RR_X0(TBLIDXB0):
+    case OE_RR_Y0(TBLIDXB0):
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tsrca, 2, 8);
+        mnemonic = "tblidxb0";
+        break;
+    case OE_RR_X0(TBLIDXB1):
+    case OE_RR_Y0(TBLIDXB1):
+        tcg_gen_shri_tl(tdest, tsrca, 8);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb1";
+        break;
+    case OE_RR_X0(TBLIDXB2):
+    case OE_RR_Y0(TBLIDXB2):
+        tcg_gen_shri_tl(tdest, tsrca, 16);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb2";
+        break;
+    case OE_RR_X0(TBLIDXB3):
+    case OE_RR_Y0(TBLIDXB3):
+        tcg_gen_shri_tl(tdest, tsrca, 24);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb3";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca]);
+    return ret;
+}
+
+static TileExcp gen_rrr_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, unsigned srcb)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+    TCGv t0;
+    const char *mnemonic;
+
+    switch (opext) {
+    case OE_RRR(ADDXSC, 0, X0):
+    case OE_RRR(ADDXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_add_tl);
+        mnemonic = "addxsc";
+        break;
+    case OE_RRR(ADDX, 0, X0):
+    case OE_RRR(ADDX, 0, X1):
+    case OE_RRR(ADDX, 0, Y0):
+    case OE_RRR(ADDX, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addx";
+        break;
+    case OE_RRR(ADD, 0, X0):
+    case OE_RRR(ADD, 0, X1):
+    case OE_RRR(ADD, 0, Y0):
+    case OE_RRR(ADD, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        mnemonic = "add";
+        break;
+    case OE_RRR(AND, 0, X0):
+    case OE_RRR(AND, 0, X1):
+    case OE_RRR(AND, 5, Y0):
+    case OE_RRR(AND, 5, Y1):
+        tcg_gen_and_tl(tdest, tsrca, tsrcb);
+        mnemonic = "and";
+        break;
+    case OE_RRR(CMOVEQZ, 0, X0):
+    case OE_RRR(CMOVEQZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmoveqz";
+        break;
+    case OE_RRR(CMOVNEZ, 0, X0):
+    case OE_RRR(CMOVNEZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmovnez";
+        break;
+    case OE_RRR(CMPEQ, 0, X0):
+    case OE_RRR(CMPEQ, 0, X1):
+    case OE_RRR(CMPEQ, 3, Y0):
+    case OE_RRR(CMPEQ, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_EQ, tdest, tsrca, tsrcb);
+        mnemonic = "cmpeq";
+        break;
+    case OE_RRR(CMPEXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH4);
+        mnemonic = "cmpexch4";
+        break;
+    case OE_RRR(CMPEXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH);
+        mnemonic = "cmpexch";
+        break;
+    case OE_RRR(CMPLES, 0, X0):
+    case OE_RRR(CMPLES, 0, X1):
+    case OE_RRR(CMPLES, 2, Y0):
+    case OE_RRR(CMPLES, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LE, tdest, tsrca, tsrcb);
+        mnemonic = "cmples";
+        break;
+    case OE_RRR(CMPLEU, 0, X0):
+    case OE_RRR(CMPLEU, 0, X1):
+    case OE_RRR(CMPLEU, 2, Y0):
+    case OE_RRR(CMPLEU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LEU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpleu";
+        break;
+    case OE_RRR(CMPLTS, 0, X0):
+    case OE_RRR(CMPLTS, 0, X1):
+    case OE_RRR(CMPLTS, 2, Y0):
+    case OE_RRR(CMPLTS, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LT, tdest, tsrca, tsrcb);
+        mnemonic = "cmplts";
+        break;
+    case OE_RRR(CMPLTU, 0, X0):
+    case OE_RRR(CMPLTU, 0, X1):
+    case OE_RRR(CMPLTU, 2, Y0):
+    case OE_RRR(CMPLTU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LTU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpltu";
+        break;
+    case OE_RRR(CMPNE, 0, X0):
+    case OE_RRR(CMPNE, 0, X1):
+    case OE_RRR(CMPNE, 3, Y0):
+    case OE_RRR(CMPNE, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_NE, tdest, tsrca, tsrcb);
+        mnemonic = "cmpne";
+        break;
+    case OE_RRR(CMULAF, 0, X0):
+        gen_helper_cmulaf(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmulaf";
+        break;
+    case OE_RRR(CMULA, 0, X0):
+        gen_helper_cmula(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmula";
+        break;
+    case OE_RRR(CMULFR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 1 << 14);
+        mnemonic = "cmulfr";
+        break;
+    case OE_RRR(CMULF, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 0);
+        mnemonic = "cmulf";
+        break;
+    case OE_RRR(CMULHR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 1 << 15);
+        mnemonic = "cmulhr";
+        break;
+    case OE_RRR(CMULH, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 0);
+        mnemonic = "cmulh";
+        break;
+    case OE_RRR(CMUL, 0, X0):
+        gen_helper_cmula(tdest, load_zero(dc), tsrca, tsrcb);
+        mnemonic = "cmul";
+        break;
+    case OE_RRR(CRC32_32, 0, X0):
+        gen_helper_crc32_32(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_32";
+        break;
+    case OE_RRR(CRC32_8, 0, X0):
+        gen_helper_crc32_8(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_8";
+        break;
+    case OE_RRR(DBLALIGN2, 0, X0):
+    case OE_RRR(DBLALIGN2, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 16);
+        mnemonic = "dblalign2";
+        break;
+    case OE_RRR(DBLALIGN4, 0, X0):
+    case OE_RRR(DBLALIGN4, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 32);
+        mnemonic = "dblalign4";
+        break;
+    case OE_RRR(DBLALIGN6, 0, X0):
+    case OE_RRR(DBLALIGN6, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 48);
+        mnemonic = "dblalign6";
+        break;
+    case OE_RRR(DBLALIGN, 0, X0):
+        gen_dblalign(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "dblalign";
+        break;
+    case OE_RRR(EXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH4);
+        mnemonic = "exch4";
+        break;
+    case OE_RRR(EXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH);
+        mnemonic = "exch";
+        break;
+    case OE_RRR(FDOUBLE_ADDSUB, 0, X0):
+    case OE_RRR(FDOUBLE_ADD_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_MUL_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_PACK1, 0, X0):
+    case OE_RRR(FDOUBLE_PACK2, 0, X0):
+    case OE_RRR(FDOUBLE_SUB_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MAX, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MIN, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(FETCHADD4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD4);
+        mnemonic = "fetchadd4";
+        break;
+    case OE_RRR(FETCHADDGEZ4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ4);
+        mnemonic = "fetchaddgez4";
+        break;
+    case OE_RRR(FETCHADDGEZ, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ);
+        mnemonic = "fetchaddgez";
+        break;
+    case OE_RRR(FETCHADD, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD);
+        mnemonic = "fetchadd";
+        break;
+    case OE_RRR(FETCHAND4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND4);
+        mnemonic = "fetchand4";
+        break;
+    case OE_RRR(FETCHAND, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND);
+        mnemonic = "fetchand";
+        break;
+    case OE_RRR(FETCHOR4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR4);
+        mnemonic = "fetchor4";
+        break;
+    case OE_RRR(FETCHOR, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR);
+        mnemonic = "fetchor";
+        break;
+    case OE_RRR(FSINGLE_ADD1, 0, X0):
+    case OE_RRR(FSINGLE_ADDSUB2, 0, X0):
+    case OE_RRR(FSINGLE_MUL1, 0, X0):
+    case OE_RRR(FSINGLE_MUL2, 0, X0):
+    case OE_RRR(FSINGLE_PACK2, 0, X0):
+    case OE_RRR(FSINGLE_SUB1, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(MNZ, 0, X0):
+    case OE_RRR(MNZ, 0, X1):
+    case OE_RRR(MNZ, 4, Y0):
+    case OE_RRR(MNZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mnz";
+        break;
+    case OE_RRR(MULAX, 0, X0):
+    case OE_RRR(MULAX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulax";
+        break;
+    case OE_RRR(MULA_HS_HS, 0, X0):
+    case OE_RRR(MULA_HS_HS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hs";
+        break;
+    case OE_RRR(MULA_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hu";
+        break;
+    case OE_RRR(MULA_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_ls";
+        break;
+    case OE_RRR(MULA_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_lu";
+        break;
+    case OE_RRR(MULA_HU_HU, 0, X0):
+    case OE_RRR(MULA_HU_HU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_hu";
+        break;
+    case OE_RRR(MULA_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_ls";
+        break;
+    case OE_RRR(MULA_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_lu";
+        break;
+    case OE_RRR(MULA_LS_LS, 0, X0):
+    case OE_RRR(MULA_LS_LS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_ls";
+        break;
+    case OE_RRR(MULA_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_lu";
+        break;
+    case OE_RRR(MULA_LU_LU, 0, X0):
+    case OE_RRR(MULA_LU_LU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_lu_lu";
+        break;
+    case OE_RRR(MULX, 0, X0):
+    case OE_RRR(MULX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulx";
+        break;
+    case OE_RRR(MUL_HS_HS, 0, X0):
+    case OE_RRR(MUL_HS_HS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        mnemonic = "mul_hs_hs";
+        break;
+    case OE_RRR(MUL_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        mnemonic = "mul_hs_hu";
+        break;
+    case OE_RRR(MUL_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        mnemonic = "mul_hs_ls";
+        break;
+    case OE_RRR(MUL_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        mnemonic = "mul_hs_lu";
+        break;
+    case OE_RRR(MUL_HU_HU, 0, X0):
+    case OE_RRR(MUL_HU_HU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        mnemonic = "mul_hu_hu";
+        break;
+    case OE_RRR(MUL_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        mnemonic = "mul_hu_ls";
+        break;
+    case OE_RRR(MUL_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        mnemonic = "mul_hu_lu";
+        break;
+    case OE_RRR(MUL_LS_LS, 0, X0):
+    case OE_RRR(MUL_LS_LS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        mnemonic = "mul_ls_ls";
+        break;
+    case OE_RRR(MUL_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        mnemonic = "mul_ls_lu";
+        break;
+    case OE_RRR(MUL_LU_LU, 0, X0):
+    case OE_RRR(MUL_LU_LU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        mnemonic = "mul_lu_lu";
+        break;
+    case OE_RRR(MZ, 0, X0):
+    case OE_RRR(MZ, 0, X1):
+    case OE_RRR(MZ, 4, Y0):
+    case OE_RRR(MZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mz";
+        break;
+    case OE_RRR(NOR, 0, X0):
+    case OE_RRR(NOR, 0, X1):
+    case OE_RRR(NOR, 5, Y0):
+    case OE_RRR(NOR, 5, Y1):
+        tcg_gen_nor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "nor";
+        break;
+    case OE_RRR(OR, 0, X0):
+    case OE_RRR(OR, 0, X1):
+    case OE_RRR(OR, 5, Y0):
+    case OE_RRR(OR, 5, Y1):
+        tcg_gen_or_tl(tdest, tsrca, tsrcb);
+        mnemonic = "or";
+        break;
+    case OE_RRR(ROTL, 0, X0):
+    case OE_RRR(ROTL, 0, X1):
+    case OE_RRR(ROTL, 6, Y0):
+    case OE_RRR(ROTL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_rotl_tl(tdest, tsrca, tdest);
+        mnemonic = "rotl";
+        break;
+    case OE_RRR(SHL1ADDX, 0, X0):
+    case OE_RRR(SHL1ADDX, 0, X1):
+    case OE_RRR(SHL1ADDX, 7, Y0):
+    case OE_RRR(SHL1ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl1addx";
+        break;
+    case OE_RRR(SHL1ADD, 0, X0):
+    case OE_RRR(SHL1ADD, 0, X1):
+    case OE_RRR(SHL1ADD, 1, Y0):
+    case OE_RRR(SHL1ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl1add";
+        break;
+    case OE_RRR(SHL2ADDX, 0, X0):
+    case OE_RRR(SHL2ADDX, 0, X1):
+    case OE_RRR(SHL2ADDX, 7, Y0):
+    case OE_RRR(SHL2ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl2addx";
+        break;
+    case OE_RRR(SHL2ADD, 0, X0):
+    case OE_RRR(SHL2ADD, 0, X1):
+    case OE_RRR(SHL2ADD, 1, Y0):
+    case OE_RRR(SHL2ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl2add";
+        break;
+    case OE_RRR(SHL3ADDX, 0, X0):
+    case OE_RRR(SHL3ADDX, 0, X1):
+    case OE_RRR(SHL3ADDX, 7, Y0):
+    case OE_RRR(SHL3ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl3addx";
+        break;
+    case OE_RRR(SHL3ADD, 0, X0):
+    case OE_RRR(SHL3ADD, 0, X1):
+    case OE_RRR(SHL3ADD, 1, Y0):
+    case OE_RRR(SHL3ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl3add";
+        break;
+    case OE_RRR(SHLX, 0, X0):
+    case OE_RRR(SHLX, 0, X1):
+        tcg_gen_andi_tl(tdest, tsrcb, 31);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlx";
+        break;
+    case OE_RRR(SHL, 0, X0):
+    case OE_RRR(SHL, 0, X1):
+    case OE_RRR(SHL, 6, Y0):
+    case OE_RRR(SHL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        mnemonic = "shl";
+        break;
+    case OE_RRR(SHRS, 0, X0):
+    case OE_RRR(SHRS, 0, X1):
+    case OE_RRR(SHRS, 6, Y0):
+    case OE_RRR(SHRS, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_sar_tl(tdest, tsrca, tdest);
+        mnemonic = "shrs";
+        break;
+    case OE_RRR(SHRUX, 0, X0):
+    case OE_RRR(SHRUX, 0, X1):
+        t0 = tcg_temp_new();
+        tcg_gen_andi_tl(t0, tsrcb, 31);
+        tcg_gen_ext32u_tl(tdest, tsrca);
+        tcg_gen_shr_tl(tdest, tdest, t0);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        tcg_temp_free(t0);
+        mnemonic = "shrux";
+        break;
+    case OE_RRR(SHRU, 0, X0):
+    case OE_RRR(SHRU, 0, X1):
+    case OE_RRR(SHRU, 6, Y0):
+    case OE_RRR(SHRU, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shr_tl(tdest, tsrca, tdest);
+        mnemonic = "shru";
+        break;
+    case OE_RRR(SHUFFLEBYTES, 0, X0):
+        gen_helper_shufflebytes(tdest, load_gr(dc, dest), tsrca, tsrca);
+        mnemonic = "shufflebytes";
+        break;
+    case OE_RRR(SUBXSC, 0, X0):
+    case OE_RRR(SUBXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_sub_tl);
+        mnemonic = "subxsc";
+        break;
+    case OE_RRR(SUBX, 0, X0):
+    case OE_RRR(SUBX, 0, X1):
+    case OE_RRR(SUBX, 0, Y0):
+    case OE_RRR(SUBX, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "subx";
+        break;
+    case OE_RRR(SUB, 0, X0):
+    case OE_RRR(SUB, 0, X1):
+    case OE_RRR(SUB, 0, Y0):
+    case OE_RRR(SUB, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        mnemonic = "sub";
+        break;
+    case OE_RRR(V1ADDUC, 0, X0):
+    case OE_RRR(V1ADDUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1ADD, 0, X0):
+    case OE_RRR(V1ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1add";
+        break;
+    case OE_RRR(V1ADIFFU, 0, X0):
+    case OE_RRR(V1AVGU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPEQ, 0, X0):
+    case OE_RRR(V1CMPEQ, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeq";
+        break;
+    case OE_RRR(V1CMPLES, 0, X0):
+    case OE_RRR(V1CMPLES, 0, X1):
+    case OE_RRR(V1CMPLEU, 0, X0):
+    case OE_RRR(V1CMPLEU, 0, X1):
+    case OE_RRR(V1CMPLTS, 0, X0):
+    case OE_RRR(V1CMPLTS, 0, X1):
+    case OE_RRR(V1CMPLTU, 0, X0):
+    case OE_RRR(V1CMPLTU, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPNE, 0, X0):
+    case OE_RRR(V1CMPNE, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpne0(tdest);
+        mnemonic = "v1cmpne";
+        break;
+    case OE_RRR(V1DDOTPUA, 0, X0):
+    case OE_RRR(V1DDOTPUSA, 0, X0):
+    case OE_RRR(V1DDOTPUS, 0, X0):
+    case OE_RRR(V1DDOTPU, 0, X0):
+    case OE_RRR(V1DOTPA, 0, X0):
+    case OE_RRR(V1DOTPUA, 0, X0):
+    case OE_RRR(V1DOTPUSA, 0, X0):
+    case OE_RRR(V1DOTPUS, 0, X0):
+    case OE_RRR(V1DOTPU, 0, X0):
+    case OE_RRR(V1DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1INT_H, 0, X0):
+    case OE_RRR(V1INT_H, 0, X1):
+        gen_helper_v1int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_h";
+        break;
+    case OE_RRR(V1INT_L, 0, X0):
+    case OE_RRR(V1INT_L, 0, X1):
+        gen_helper_v1int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_l";
+        break;
+    case OE_RRR(V1MAXU, 0, X0):
+    case OE_RRR(V1MAXU, 0, X1):
+    case OE_RRR(V1MINU, 0, X0):
+    case OE_RRR(V1MINU, 0, X1):
+    case OE_RRR(V1MNZ, 0, X0):
+    case OE_RRR(V1MNZ, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1MULTU, 0, X0):
+        gen_helper_v1multu(tdest, tsrca, tsrcb);
+        mnemonic = "v1multu";
+        break;
+    case OE_RRR(V1MULUS, 0, X0):
+    case OE_RRR(V1MULU, 0, X0):
+    case OE_RRR(V1MZ, 0, X0):
+    case OE_RRR(V1MZ, 0, X1):
+    case OE_RRR(V1SADAU, 0, X0):
+    case OE_RRR(V1SADU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SHL, 0, X0):
+    case OE_RRR(V1SHL, 0, X1):
+        gen_helper_v1shl(tdest, tsrca, tsrcb);
+        mnemonic = "v1shl";
+        break;
+    case OE_RRR(V1SHRS, 0, X0):
+    case OE_RRR(V1SHRS, 0, X1):
+        gen_helper_v1shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v1shrs";
+        break;
+    case OE_RRR(V1SHRU, 0, X0):
+    case OE_RRR(V1SHRU, 0, X1):
+        gen_helper_v1shru(tdest, tsrca, tsrcb);
+        mnemonic = "v1shru";
+        break;
+    case OE_RRR(V1SUBUC, 0, X0):
+    case OE_RRR(V1SUBUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SUB, 0, X0):
+    case OE_RRR(V1SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1sub";
+        break;
+    case OE_RRR(V2ADDSC, 0, X0):
+    case OE_RRR(V2ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2ADD, 0, X0):
+    case OE_RRR(V2ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2add";
+        break;
+    case OE_RRR(V2ADIFFS, 0, X0):
+    case OE_RRR(V2AVGS, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X1):
+    case OE_RRR(V2CMPLES, 0, X0):
+    case OE_RRR(V2CMPLES, 0, X1):
+    case OE_RRR(V2CMPLEU, 0, X0):
+    case OE_RRR(V2CMPLEU, 0, X1):
+    case OE_RRR(V2CMPLTS, 0, X0):
+    case OE_RRR(V2CMPLTS, 0, X1):
+    case OE_RRR(V2CMPLTU, 0, X0):
+    case OE_RRR(V2CMPLTU, 0, X1):
+    case OE_RRR(V2CMPNE, 0, X0):
+    case OE_RRR(V2CMPNE, 0, X1):
+    case OE_RRR(V2DOTPA, 0, X0):
+    case OE_RRR(V2DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2INT_H, 0, X0):
+    case OE_RRR(V2INT_H, 0, X1):
+        gen_helper_v2int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_h";
+        break;
+    case OE_RRR(V2INT_L, 0, X0):
+    case OE_RRR(V2INT_L, 0, X1):
+        gen_helper_v2int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_l";
+        break;
+    case OE_RRR(V2MAXS, 0, X0):
+    case OE_RRR(V2MAXS, 0, X1):
+    case OE_RRR(V2MINS, 0, X0):
+    case OE_RRR(V2MINS, 0, X1):
+    case OE_RRR(V2MNZ, 0, X0):
+    case OE_RRR(V2MNZ, 0, X1):
+    case OE_RRR(V2MULFSC, 0, X0):
+    case OE_RRR(V2MULS, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2MULTS, 0, X0):
+        gen_helper_v2mults(tdest, tsrca, tsrcb);
+        mnemonic = "v2mults";
+        break;
+    case OE_RRR(V2MZ, 0, X0):
+    case OE_RRR(V2MZ, 0, X1):
+    case OE_RRR(V2PACKH, 0, X0):
+    case OE_RRR(V2PACKH, 0, X1):
+    case OE_RRR(V2PACKL, 0, X0):
+    case OE_RRR(V2PACKL, 0, X1):
+    case OE_RRR(V2PACKUC, 0, X0):
+    case OE_RRR(V2PACKUC, 0, X1):
+    case OE_RRR(V2SADAS, 0, X0):
+    case OE_RRR(V2SADAU, 0, X0):
+    case OE_RRR(V2SADS, 0, X0):
+    case OE_RRR(V2SADU, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SHL, 0, X0):
+    case OE_RRR(V2SHL, 0, X1):
+        gen_helper_v2shl(tdest, tsrca, tsrcb);
+        mnemonic = "v2shl";
+        break;
+    case OE_RRR(V2SHRS, 0, X0):
+    case OE_RRR(V2SHRS, 0, X1):
+        gen_helper_v2shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v2shrs";
+        break;
+    case OE_RRR(V2SHRU, 0, X0):
+    case OE_RRR(V2SHRU, 0, X1):
+        gen_helper_v2shru(tdest, tsrca, tsrcb);
+        mnemonic = "v2shru";
+        break;
+    case OE_RRR(V2SUBSC, 0, X0):
+    case OE_RRR(V2SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SUB, 0, X0):
+    case OE_RRR(V2SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(V4ADDSC, 0, X0):
+    case OE_RRR(V4ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4ADD, 0, X0):
+    case OE_RRR(V4ADD, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_add_i32);
+        mnemonic = "v4add";
+        break;
+    case OE_RRR(V4INT_H, 0, X0):
+    case OE_RRR(V4INT_H, 0, X1):
+        tcg_gen_shri_tl(tdest, tsrcb, 32);
+        tcg_gen_deposit_tl(tdest, tsrca, tdest, 0, 32);
+        mnemonic = "v4int_h";
+        break;
+    case OE_RRR(V4INT_L, 0, X0):
+    case OE_RRR(V4INT_L, 0, X1):
+        tcg_gen_deposit_tl(tdest, tsrcb, tsrca, 32, 32);
+        mnemonic = "v4int_l";
+        break;
+    case OE_RRR(V4PACKSC, 0, X0):
+    case OE_RRR(V4PACKSC, 0, X1):
+    case OE_RRR(V4SHLSC, 0, X0):
+    case OE_RRR(V4SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SHL, 0, X0):
+    case OE_RRR(V4SHL, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shl_i32);
+        mnemonic = "v4shl";
+        break;
+    case OE_RRR(V4SHRS, 0, X0):
+    case OE_RRR(V4SHRS, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_sar_i32);
+        mnemonic = "v4shrs";
+        break;
+    case OE_RRR(V4SHRU, 0, X0):
+    case OE_RRR(V4SHRU, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shr_i32);
+        mnemonic = "v4shru";
+        break;
+    case OE_RRR(V4SUBSC, 0, X0):
+    case OE_RRR(V4SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SUB, 0, X0):
+    case OE_RRR(V4SUB, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_sub_i32);
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(XOR, 0, X0):
+    case OE_RRR(XOR, 0, X1):
+    case OE_RRR(XOR, 5, Y0):
+    case OE_RRR(XOR, 5, Y1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "xor";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_rri_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, int imm)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    bool prefetch_nofault = false;
+    const char *mnemonic;
+    TCGMemOp memop;
+    int i2, i3;
+    TCGv t0;
+
+    switch (opext) {
+    case OE(ADDI_OPCODE_Y0, 0, Y0):
+    case OE(ADDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDI, X0):
+    case OE_IM(ADDI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addi";
+        break;
+    case OE(ADDXI_OPCODE_Y0, 0, Y0):
+    case OE(ADDXI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDXI, X0):
+    case OE_IM(ADDXI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxi";
+        break;
+    case OE(ANDI_OPCODE_Y0, 0, Y0):
+    case OE(ANDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ANDI, X0):
+    case OE_IM(ANDI, X1):
+        tcg_gen_andi_tl(tdest, tsrca, imm);
+        mnemonic = "andi";
+        break;
+    case OE(CMPEQI_OPCODE_Y0, 0, Y0):
+    case OE(CMPEQI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPEQI, X0):
+    case OE_IM(CMPEQI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_EQ, tdest, tsrca, imm);
+        mnemonic = "cmpeqi";
+        break;
+    case OE(CMPLTSI_OPCODE_Y0, 0, Y0):
+    case OE(CMPLTSI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPLTSI, X0):
+    case OE_IM(CMPLTSI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LT, tdest, tsrca, imm);
+        mnemonic = "cmpltsi";
+        break;
+    case OE_IM(CMPLTUI, X0):
+    case OE_IM(CMPLTUI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LTU, tdest, tsrca, imm);
+        mnemonic = "cmpltui";
+        break;
+    case OE_IM(LD1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ld1s_add"; /* prefetch_add_l1_fault */
+        goto do_load_add;
+    case OE_IM(LD1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ld1u_add"; /* prefetch_add_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ld2s_add"; /* prefetch_add_l2_fault */
+        goto do_load_add;
+    case OE_IM(LD2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ld2u_add"; /* prefetch_add_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ld4s_add"; /* prefetch_add_l3_fault */
+        goto do_load_add;
+    case OE_IM(LD4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ld4u_add"; /* prefetch_add_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LDNT1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ldnt1s_add";
+        goto do_load_add;
+    case OE_IM(LDNT1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ldnt1u_add";
+        goto do_load_add;
+    case OE_IM(LDNT2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s_add";
+        goto do_load_add;
+    case OE_IM(LDNT2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u_add";
+        goto do_load_add;
+    case OE_IM(LDNT4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s_add";
+        goto do_load_add;
+    case OE_IM(LDNT4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u_add";
+        goto do_load_add;
+    case OE_IM(LDNT_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ldnt_add";
+        goto do_load_add;
+    case OE_IM(LD_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ld_add";
+    do_load_add:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        break;
+    case OE_IM(LDNA_ADD, X1):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        mnemonic = "ldna_add";
+        break;
+    case OE_IM(ORI, X0):
+    case OE_IM(ORI, X1):
+        tcg_gen_ori_tl(tdest, tsrca, imm);
+        mnemonic = "ori";
+        break;
+    case OE_IM(V1ADDI, X0):
+    case OE_IM(V1ADDI, X1):
+        t0 = tcg_const_tl(V1_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V1_IMM(0x80));
+        tcg_temp_free(t0);
+        mnemonic = "v1addi";
+        break;
+    case OE_IM(V1CMPEQI, X0):
+    case OE_IM(V1CMPEQI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, V1_IMM(imm));
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeqi";
+        break;
+    case OE_IM(V1CMPLTSI, X0):
+    case OE_IM(V1CMPLTSI, X1):
+    case OE_IM(V1CMPLTUI, X0):
+    case OE_IM(V1CMPLTUI, X1):
+    case OE_IM(V1MAXUI, X0):
+    case OE_IM(V1MAXUI, X1):
+    case OE_IM(V1MINUI, X0):
+    case OE_IM(V1MINUI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(V2ADDI, X0):
+    case OE_IM(V2ADDI, X1):
+        t0 = tcg_const_tl(V2_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V2_IMM(0x8000));
+        tcg_temp_free(t0);
+        mnemonic = "v2addi";
+        break;
+    case OE_IM(V2CMPEQI, X0):
+    case OE_IM(V2CMPEQI, X1):
+    case OE_IM(V2CMPLTSI, X0):
+    case OE_IM(V2CMPLTSI, X1):
+    case OE_IM(V2CMPLTUI, X0):
+    case OE_IM(V2CMPLTUI, X1):
+    case OE_IM(V2MAXSI, X0):
+    case OE_IM(V2MAXSI, X1):
+    case OE_IM(V2MINSI, X0):
+    case OE_IM(V2MINSI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(XORI, X0):
+    case OE_IM(XORI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, imm);
+        mnemonic = "xori";
+        break;
+
+    case OE_SH(ROTLI, X0):
+    case OE_SH(ROTLI, X1):
+    case OE_SH(ROTLI, Y0):
+    case OE_SH(ROTLI, Y1):
+        tcg_gen_rotli_tl(tdest, tsrca, imm);
+        mnemonic = "rotli";
+        break;
+    case OE_SH(SHLI, X0):
+    case OE_SH(SHLI, X1):
+    case OE_SH(SHLI, Y0):
+    case OE_SH(SHLI, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, imm);
+        mnemonic = "shli";
+        break;
+    case OE_SH(SHLXI, X0):
+    case OE_SH(SHLXI, X1):
+        tcg_gen_shli_tl(tdest, tsrca, imm & 31);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(SHRSI, X0):
+    case OE_SH(SHRSI, X1):
+    case OE_SH(SHRSI, Y0):
+    case OE_SH(SHRSI, Y1):
+        tcg_gen_sari_tl(tdest, tsrca, imm);
+        mnemonic = "shrsi";
+        break;
+    case OE_SH(SHRUI, X0):
+    case OE_SH(SHRUI, X1):
+    case OE_SH(SHRUI, Y0):
+    case OE_SH(SHRUI, Y1):
+        tcg_gen_shri_tl(tdest, tsrca, imm);
+        mnemonic = "shrui";
+        break;
+    case OE_SH(SHRUXI, X0):
+    case OE_SH(SHRUXI, X1):
+        if ((imm & 31) == 0) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+            tcg_gen_shri_tl(tdest, tdest, imm & 31);
+        }
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(V1SHLI, X0):
+    case OE_SH(V1SHLI, X1):
+        i2 = imm & 7;
+        i3 = 0xff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v1shli";
+        break;
+    case OE_SH(V1SHRSI, X0):
+    case OE_SH(V1SHRSI, X1):
+        t0 = tcg_const_tl(imm & 7);
+        gen_helper_v1shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v1shrsi";
+        break;
+    case OE_SH(V1SHRUI, X0):
+    case OE_SH(V1SHRUI, X1):
+        i2 = imm & 7;
+        i3 = (0xff << i2) & 0xff;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v1shrui";
+        break;
+    case OE_SH(V2SHLI, X0):
+    case OE_SH(V2SHLI, X1):
+        i2 = imm & 15;
+        i3 = 0xffff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v2shli";
+        break;
+    case OE_SH(V2SHRSI, X0):
+    case OE_SH(V2SHRSI, X1):
+        t0 = tcg_const_tl(imm & 15);
+        gen_helper_v2shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v2shrsi";
+        break;
+    case OE_SH(V2SHRUI, X0):
+    case OE_SH(V2SHRUI, X1):
+        i2 = imm & 15;
+        i3 = (0xffff << i2) & 0xffff;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v2shrui";
+        break;
+
+    case OE(ADDLI_OPCODE_X0, 0, X0):
+    case OE(ADDLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addli";
+        break;
+    case OE(ADDXLI_OPCODE_X0, 0, X0):
+    case OE(ADDXLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxli";
+        break;
+    case OE(SHL16INSLI_OPCODE_X0, 0, X0):
+    case OE(SHL16INSLI_OPCODE_X1, 0, X1):
+        tcg_gen_shli_tl(tdest, tsrca, 16);
+        tcg_gen_ori_tl(tdest, tdest, imm & 0xffff);
+        mnemonic = "shl16insli";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", mnemonic,
+                  reg_names[dest], reg_names[srca], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_bf_opcode_x0(DisasContext *dc, unsigned ext,
+                                 unsigned dest, unsigned srca,
+                                 unsigned bfs, unsigned bfe)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcd;
+    int len;
+    const char *mnemonic;
+
+    /* The bitfield is either between E and S inclusive,
+       or up from S and down from E inclusive.  */
+    if (bfs <= bfe) {
+        len = bfe - bfs + 1;
+    } else {
+        len = (64 - bfs) + (bfe + 1);
+    }
+
+    switch (ext) {
+    case BFEXTU_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_shri_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfextu";
+        break;
+
+    case BFEXTS_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_sari_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfexts";
+        break;
+
+    case BFINS_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs <= bfe) {
+            tcg_gen_deposit_tl(tdest, tsrcd, tsrca, bfs, len);
+        } else {
+            tcg_gen_rotri_tl(tdest, tsrcd, bfs);
+            tcg_gen_deposit_tl(tdest, tdest, tsrca, 0, len);
+            tcg_gen_rotli_tl(tdest, tdest, bfs);
+        }
+        mnemonic = "bfins";
+        break;
+
+    case MM_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs == 0) {
+            tcg_gen_deposit_tl(tdest, tsrca, tsrcd, 0, len);
+        } else {
+            uint64_t mask = len == 64 ? -1 : rol64((1ULL << len) - 1, bfs);
+            TCGv tmp = tcg_const_tl(mask);
+
+            tcg_gen_and_tl(tdest, tsrcd, tmp);
+            tcg_gen_andc_tl(tmp, tsrca, tmp);
+            tcg_gen_or_tl(tdest, tdest, tmp);
+            tcg_temp_free(tmp);
+        }
+        mnemonic = "mm";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %u, %u", mnemonic,
+                  reg_names[dest], reg_names[srca], bfs, bfe);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_branch_opcode_x1(DisasContext *dc, unsigned ext,
+                                     unsigned srca, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic;
+
+    dc->jmp.dest = tcg_const_tl(tgt);
+    dc->jmp.val1 = tcg_temp_new();
+    tcg_gen_mov_tl(dc->jmp.val1, load_gr(dc, srca));
+
+    /* Note that the "predict taken" opcodes have bit 0 clear.
+       Therefore, fold the two cases together by setting bit 0.  */
+    switch (ext | 1) {
+    case BEQZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        mnemonic = "beqz";
+        break;
+    case BNEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        mnemonic = "bnez";
+        break;
+    case BGEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GE;
+        mnemonic = "bgez";
+        break;
+    case BGTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GT;
+        mnemonic = "bgtz";
+        break;
+    case BLEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LE;
+        mnemonic = "blez";
+        break;
+    case BLTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LT;
+        mnemonic = "bltz";
+        break;
+    case BLBC_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbc";
+        break;
+    case BLBS_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbs";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s%s %s, " TARGET_FMT_lx " <%s>",
+                 mnemonic, ext & 1 ? "" : "t",
+                 reg_names[srca], tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_jump_opcode_x1(DisasContext *dc, unsigned ext, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic = "j";
+
+    /* The extension field is 1 bit, therefore we only have JAL and J.  */
+    if (ext == JAL_JUMP_OPCODE_X1) {
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "jal";
+    }
+    dc->jmp.cond = TCG_COND_ALWAYS;
+    dc->jmp.dest = tcg_const_tl(tgt);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s " TARGET_FMT_lx " <%s>",
+                 mnemonic, tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+typedef struct {
+    const char *name;
+    intptr_t offset;
+    void (*get)(TCGv, TCGv_ptr);
+    void (*put)(TCGv_ptr, TCGv);
+} TileSPR;
+
+static const TileSPR *find_spr(unsigned spr)
+{
+    /* Allow the compiler to construct the binary search tree.  */
+#define D(N, O, G, P) \
+    case SPR_##N: { static const TileSPR x = { #N, O, G, P }; return &x; }
+
+    switch (spr) {
+    D(CMPEXCH_VALUE,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CMPEXCH]), 0, 0)
+    D(INTERRUPT_CRITICAL_SECTION,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CRITICAL_SEC]), 0, 0)
+    D(SIM_CONTROL,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_SIM_CONTROL]), 0, 0)
+    D(EX_CONTEXT_0_0,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]), 0, 0)
+    D(EX_CONTEXT_0_1,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_1]), 0, 0)
+    }
+
+#undef D
+
+    qemu_log_mask(LOG_UNIMP, "UNIMP SPR %u\n", spr);
+    return NULL;
+}
+
+static TileExcp gen_mtspr_x1(DisasContext *dc, unsigned spr, unsigned srca)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tsrca;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr spr[%u], %s", spr, reg_names[srca]);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tsrca = load_gr(dc, srca);
+    if (def->put) {
+        def->put(cpu_env, tsrca);
+    } else {
+        tcg_gen_st_tl(tsrca, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, %s", def->name, reg_names[srca]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_mfspr_x1(DisasContext *dc, unsigned dest, unsigned spr)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tdest;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, spr[%u]", reg_names[dest], spr);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tdest = dest_gr(dc, dest);
+    if (def->get) {
+        def->get(tdest, cpu_env);
+    } else {
+        tcg_gen_ld_tl(tdest, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mfspr %s, %s", reg_names[dest], def->name);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp decode_y0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y0(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y0(bundle);
+    unsigned dest = get_Dest_Y0(bundle);
+    unsigned srca = get_SrcA_Y0(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_1_OPCODE_Y0:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y0), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y0:
+    case RRR_2_OPCODE_Y0:
+    case RRR_3_OPCODE_Y0:
+    case RRR_4_OPCODE_Y0:
+    case RRR_5_OPCODE_Y0:
+    case RRR_6_OPCODE_Y0:
+    case RRR_7_OPCODE_Y0:
+    case RRR_8_OPCODE_Y0:
+    case RRR_9_OPCODE_Y0:
+        srcb = get_SrcB_Y0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y0:
+        ext = get_ShiftOpcodeExtension_Y0(bundle);
+        imm = get_ShAmt_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y0), dest, srca, imm);
+
+    case ADDI_OPCODE_Y0:
+    case ADDXI_OPCODE_Y0:
+    case ANDI_OPCODE_Y0:
+    case CMPEQI_OPCODE_Y0:
+    case CMPLTSI_OPCODE_Y0:
+        imm = (int8_t)get_Imm8_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y1(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y1(bundle);
+    unsigned dest = get_Dest_Y1(bundle);
+    unsigned srca = get_SrcA_Y1(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (get_Opcode_Y1(bundle)) {
+    case RRR_1_OPCODE_Y1:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y1), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y1:
+    case RRR_2_OPCODE_Y1:
+    case RRR_3_OPCODE_Y1:
+    case RRR_4_OPCODE_Y1:
+    case RRR_5_OPCODE_Y1:
+    case RRR_6_OPCODE_Y1:
+    case RRR_7_OPCODE_Y1:
+        srcb = get_SrcB_Y1(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y1:
+        ext = get_ShiftOpcodeExtension_Y1(bundle);
+        imm = get_ShAmt_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y1), dest, srca, imm);
+
+    case ADDI_OPCODE_Y1:
+    case ADDXI_OPCODE_Y1:
+    case ANDI_OPCODE_Y1:
+    case CMPEQI_OPCODE_Y1:
+    case CMPLTSI_OPCODE_Y1:
+        imm = (int8_t)get_Imm8_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y2(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned mode = get_Mode(bundle);
+    unsigned opc = get_Opcode_Y2(bundle);
+    unsigned srca = get_SrcA_Y2(bundle);
+    unsigned srcbdest = get_SrcBDest_Y2(bundle);
+    const char *mnemonic;
+    TCGMemOp memop;
+    bool prefetch_nofault = false;
+
+    switch (OEY2(opc, mode)) {
+    case OEY2(LD1S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OEY2(LD1U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD2S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OEY2(LD2U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD4S_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OEY2(LD4U_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(dest_gr(dc, srcbdest), load_gr(dc, srca),
+                               dc->mmuidx, memop);
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                      reg_names[srcbdest], reg_names[srca]);
+        return TILEGX_EXCP_NONE;
+
+    case OEY2(ST1_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_UB, "st1");
+    case OEY2(ST2_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUW, "st2");
+    case OEY2(ST4_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUL, "st4");
+    case OEY2(ST_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEQ, "st");
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X0(bundle);
+    unsigned dest = get_Dest_X0(bundle);
+    unsigned srca = get_SrcA_X0(bundle);
+    unsigned ext, srcb, bfs, bfe;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X0:
+        ext = get_RRROpcodeExtension_X0(bundle);
+        if (ext == UNARY_RRR_0_OPCODE_X0) {
+            ext = get_UnaryOpcodeExtension_X0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X0), dest, srca, bundle);
+        }
+        srcb = get_SrcB_X0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, X0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X0:
+        ext = get_ShiftOpcodeExtension_X0(bundle);
+        imm = get_ShAmt_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case IMM8_OPCODE_X0:
+        ext = get_Imm8OpcodeExtension_X0(bundle);
+        imm = (int8_t)get_Imm8_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case BF_OPCODE_X0:
+        ext = get_BFOpcodeExtension_X0(bundle);
+        bfs = get_BFStart_X0(bundle);
+        bfe = get_BFEnd_X0(bundle);
+        return gen_bf_opcode_x0(dc, ext, dest, srca, bfs, bfe);
+
+    case ADDLI_OPCODE_X0:
+    case SHL16INSLI_OPCODE_X0:
+    case ADDXLI_OPCODE_X0:
+        imm = (int16_t)get_Imm16_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X1(bundle);
+    unsigned dest = get_Dest_X1(bundle);
+    unsigned srca = get_SrcA_X1(bundle);
+    unsigned ext, srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X1:
+        ext = get_RRROpcodeExtension_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case UNARY_RRR_0_OPCODE_X1:
+            ext = get_UnaryOpcodeExtension_X1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X1), dest, srca, bundle);
+        case ST1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "st1");
+        case ST2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "st2");
+        case ST4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "st4");
+        case STNT1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "stnt1");
+        case STNT2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "stnt2");
+        case STNT4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "stnt4");
+        case STNT_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "stnt");
+        case ST_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "st");
+        }
+        return gen_rrr_opcode(dc, OE(opc, ext, X1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X1:
+        ext = get_ShiftOpcodeExtension_X1(bundle);
+        imm = get_ShAmt_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case IMM8_OPCODE_X1:
+        ext = get_Imm8OpcodeExtension_X1(bundle);
+        imm = (int8_t)get_Dest_Imm8_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case ST1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "st1_add");
+        case ST2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "st2_add");
+        case ST4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "st4_add");
+        case STNT1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "stnt1_add");
+        case STNT2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "stnt2_add");
+        case STNT4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "stnt4_add");
+        case STNT_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "stnt_add");
+        case ST_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "st_add");
+        case MFSPR_IMM8_OPCODE_X1:
+            return gen_mfspr_x1(dc, dest, get_MF_Imm14_X1(bundle));
+        case MTSPR_IMM8_OPCODE_X1:
+            return gen_mtspr_x1(dc, get_MT_Imm14_X1(bundle), srca);
+        }
+        imm = (int8_t)get_Imm8_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case BRANCH_OPCODE_X1:
+        ext = get_BrType_X1(bundle);
+        imm = sextract32(get_BrOff_X1(bundle), 0, 17);
+        return gen_branch_opcode_x1(dc, ext, srca, imm);
+
+    case JUMP_OPCODE_X1:
+        ext = get_JumpOpcodeExtension_X1(bundle);
+        imm = sextract32(get_JumpOff_X1(bundle), 0, 27);
+        return gen_jump_opcode_x1(dc, ext, imm);
+
+    case ADDLI_OPCODE_X1:
+    case SHL16INSLI_OPCODE_X1:
+    case ADDXLI_OPCODE_X1:
+        imm = (int16_t)get_Imm16_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static void notice_excp(DisasContext *dc, uint64_t bundle,
+                        const char *type, TileExcp excp)
+{
+    if (likely(excp == TILEGX_EXCP_NONE)) {
+        return;
+    }
+    gen_exception(dc, excp);
+    switch (excp) {
+    case TILEGX_EXCP_OPCODE_UNIMPLEMENTED:
+        qemu_log_mask(LOG_UNIMP, "UNIMP %s, [" FMT64X "]\n", type, bundle);
+        break;
+    case TILEGX_EXCP_OPCODE_UNKNOWN:
+        qemu_log_mask(LOG_UNIMP, "UNKNOWN %s, [" FMT64X "]\n", type, bundle);
+        break;
+    default:
+        break;
+    }
+}
+
+static void translate_one_bundle(DisasContext *dc, uint64_t bundle)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dc->wb); i++) {
+        DisasContextTemp *wb = &dc->wb[i];
+        wb->reg = TILEGX_R_NOREG;
+        TCGV_UNUSED_I64(wb->val);
+    }
+    dc->num_wb = 0;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "  %" PRIx64 ":  { ", dc->pc);
+    if (get_Mode(bundle)) {
+        notice_excp(dc, bundle, "y0", decode_y0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y1", decode_y1(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y2", decode_y2(dc, bundle));
+    } else {
+        notice_excp(dc, bundle, "x0", decode_x0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "x1", decode_x1(dc, bundle));
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, " }\n");
+
+    for (i = dc->num_wb - 1; i >= 0; --i) {
+        DisasContextTemp *wb = &dc->wb[i];
+        if (wb->reg < TILEGX_R_COUNT) {
+            tcg_gen_mov_i64(cpu_regs[wb->reg], wb->val);
+        }
+        tcg_temp_free_i64(wb->val);
+    }
+
+    if (dc->jmp.cond != TCG_COND_NEVER) {
+        if (dc->jmp.cond == TCG_COND_ALWAYS) {
+            tcg_gen_mov_i64(cpu_pc, dc->jmp.dest);
+        } else {
+            TCGv next = tcg_const_i64(dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+            tcg_gen_movcond_i64(dc->jmp.cond, cpu_pc,
+                                dc->jmp.val1, load_zero(dc),
+                                dc->jmp.dest, next);
+            tcg_temp_free_i64(dc->jmp.val1);
+            tcg_temp_free_i64(next);
+        }
+        tcg_temp_free_i64(dc->jmp.dest);
+        tcg_gen_exit_tb(0);
+        dc->exit_tb = true;
+    } else if (dc->atomic_excp != TILEGX_EXCP_NONE) {
+        gen_exception(dc, dc->atomic_excp);
+    }
+}
+
+void gen_intermediate_code(CPUTLGState *env, struct TranslationBlock *tb)
+{
+    TileGXCPU *cpu = tilegx_env_get_cpu(env);
+    DisasContext ctx;
+    DisasContext *dc = &ctx;
+    CPUState *cs = CPU(cpu);
+    uint64_t pc_start = tb->pc;
+    uint64_t next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+    int num_insns = 0;
+    int max_insns = tb->cflags & CF_COUNT_MASK;
+
+    dc->pc = pc_start;
+    dc->mmuidx = 0;
+    dc->exit_tb = false;
+    dc->atomic_excp = TILEGX_EXCP_NONE;
+    dc->jmp.cond = TCG_COND_NEVER;
+    TCGV_UNUSED_I64(dc->jmp.dest);
+    TCGV_UNUSED_I64(dc->jmp.val1);
+    TCGV_UNUSED_I64(dc->zero);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+    }
+    if (!max_insns) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (cs->singlestep_enabled || singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+    gen_tb_start(tb);
+
+    while (1) {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        translate_one_bundle(dc, cpu_ldq_data(env, dc->pc));
+
+        if (dc->exit_tb) {
+            /* PC updated and EXIT_TB/GOTO_TB/exception emitted.  */
+            break;
+        }
+        dc->pc += TILEGX_BUNDLE_SIZE_IN_BYTES;
+        if (num_insns >= max_insns
+            || dc->pc >= next_page_start
+            || tcg_op_buf_full()) {
+            /* Ending the TB due to TB size or page boundary.  Set PC.  */
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            tcg_gen_exit_tb(0);
+            break;
+        }
+    }
+
+    gen_tb_end(tb, num_insns);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "\n");
+}
+
+void restore_state_to_opc(CPUTLGState *env, TranslationBlock *tb,
+                          target_ulong *data)
+{
+    env->pc = data[0];
+}
+
+void tilegx_tcg_init(void)
+{
+    int i;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    cpu_pc = tcg_global_mem_new_i64(cpu_env, offsetof(CPUTLGState, pc), "pc");
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_regs[i] = tcg_global_mem_new_i64(cpu_env,
+                                             offsetof(CPUTLGState, regs[i]),
+                                             reg_names[i]);
+    }
+}
diff --git a/qemu/target-tricore/Makefile.objs b/qemu/target-tricore/Makefile.objs
index 21e820d8f..7a0567071 100644
--- a/qemu/target-tricore/Makefile.objs
+++ b/qemu/target-tricore/Makefile.objs
@@ -1 +1 @@
-obj-y += translate.o helper.o cpu.o op_helper.o
+obj-y += translate.o helper.o cpu.o op_helper.o fpu_helper.o
diff --git a/qemu/target-tricore/cpu.c b/qemu/target-tricore/cpu.c
index 2029ef651..69fca8c06 100644
--- a/qemu/target-tricore/cpu.c
+++ b/qemu/target-tricore/cpu.c
@@ -17,6 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 
@@ -170,6 +172,12 @@ static void tricore_cpu_class_init(ObjectClass *c, void *data)
     cc->set_pc = tricore_cpu_set_pc;
     cc->synchronize_from_tb = tricore_cpu_synchronize_from_tb;
 
+    /*
+     * Reason: tricore_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const TriCoreCPUInfo *info)
diff --git a/qemu/target-tricore/cpu.h b/qemu/target-tricore/cpu.h
index 916ee27ad..90045a93d 100644
--- a/qemu/target-tricore/cpu.h
+++ b/qemu/target-tricore/cpu.h
@@ -20,13 +20,10 @@
 #define __TRICORE_CPU_H__
 
 #include "tricore-defs.h"
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
-#define ELF_MACHINE     EM_TRICORE
-
 #define CPUArchState struct CPUTriCoreState
 
 struct CPUTriCoreState;
@@ -186,8 +183,7 @@ struct CPUTriCoreState {
     uint32_t M2CNT;
     uint32_t M3CNT;
     /* Floating Point Registers */
-    /* XXX: */
-
+    float_status fp_status;
     /* QEMU */
     int error_code;
     uint32_t hflags;    /* CPU State */
@@ -220,6 +216,7 @@ struct CPUTriCoreState {
 #define MASK_PSW_GW  0x00000100
 #define MASK_PSW_CDE 0x00000080
 #define MASK_PSW_CDC 0x0000007f
+#define MASK_PSW_FPU_RM 0x3000000
 
 #define MASK_SYSCON_PRO_TEN 0x2
 #define MASK_SYSCON_FCD_SF  0x1
@@ -273,6 +270,7 @@ enum {
     TRAPC_ASSERT   = 5,
     TRAPC_SYSCALL  = 6,
     TRAPC_NMI      = 7,
+    TRAPC_IRQ      = 8
 };
 
 /* Class 0 TIN */
@@ -341,6 +339,8 @@ enum {
 uint32_t psw_read(CPUTriCoreState *env);
 void psw_write(CPUTriCoreState *env, uint32_t val);
 
+void fpu_set_state(CPUTriCoreState *env);
+
 #include "cpu-qom.h"
 #define MMU_USER_IDX 2
 
@@ -350,7 +350,7 @@ void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #define cpu_signal_handler cpu_tricore_signal_handler
 #define cpu_list tricore_cpu_list
 
-static inline int cpu_mmu_index(CPUTriCoreState *env)
+static inline int cpu_mmu_index(CPUTriCoreState *env, bool ifetch)
 {
     return 0;
 }
diff --git a/qemu/target-tricore/fpu_helper.c b/qemu/target-tricore/fpu_helper.c
new file mode 100644
index 000000000..98fe9472b
--- /dev/null
+++ b/qemu/target-tricore/fpu_helper.c
@@ -0,0 +1,217 @@
+/*
+ *  TriCore emulation for qemu: fpu helper.
+ *
+ *  Copyright (c) 2016 Bastian Koppelmann University of Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+#define ADD_NAN   0x7cf00001
+#define DIV_NAN   0x7fc00008
+#define MUL_NAN   0x7fc00002
+#define FPU_FS PSW_USB_C
+#define FPU_FI PSW_USB_V
+#define FPU_FV PSW_USB_SV
+#define FPU_FZ PSW_USB_AV
+#define FPU_FU PSW_USB_SAV
+
+/* we don't care about input_denormal */
+static inline uint8_t f_get_excp_flags(CPUTriCoreState *env)
+{
+    return get_float_exception_flags(&env->fp_status)
+           & (float_flag_invalid
+              | float_flag_overflow
+              | float_flag_underflow
+              | float_flag_output_denormal
+              | float_flag_divbyzero
+              | float_flag_inexact);
+}
+
+static inline bool f_is_denormal(float32 arg)
+{
+    return float32_is_zero_or_denormal(arg) && !float32_is_zero(arg);
+}
+
+static void f_update_psw_flags(CPUTriCoreState *env, uint8_t flags)
+{
+    uint8_t some_excp = 0;
+    set_float_exception_flags(0, &env->fp_status);
+
+    if (flags & float_flag_invalid) {
+        env->FPU_FI = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_overflow) {
+        env->FPU_FV = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_underflow || flags & float_flag_output_denormal) {
+        env->FPU_FU = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_divbyzero) {
+        env->FPU_FZ = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_inexact || flags & float_flag_output_denormal) {
+        env->PSW |= 1 << 26;
+        some_excp = 1;
+    }
+
+    env->FPU_FS = some_excp;
+}
+
+#define FADD_SUB(op)                                                           \
+uint32_t helper_f##op(CPUTriCoreState *env, uint32_t r1, uint32_t r2)          \
+{                                                                              \
+    float32 arg1 = make_float32(r1);                                           \
+    float32 arg2 = make_float32(r2);                                           \
+    uint32_t flags;                                                            \
+    float32 f_result;                                                          \
+                                                                               \
+    f_result = float32_##op(arg2, arg1, &env->fp_status);                      \
+    flags = f_get_excp_flags(env);                                             \
+    if (flags) {                                                               \
+        /* If the output is a NaN, but the inputs aren't,                      \
+           we return a unique value.  */                                       \
+        if ((flags & float_flag_invalid)                                       \
+            && !float32_is_any_nan(arg1)                                       \
+            && !float32_is_any_nan(arg2)) {                                    \
+            f_result = ADD_NAN;                                                \
+        }                                                                      \
+        f_update_psw_flags(env, flags);                                        \
+    } else {                                                                   \
+        env->FPU_FS = 0;                                                       \
+    }                                                                          \
+    return (uint32_t)f_result;                                                 \
+}
+FADD_SUB(add)
+FADD_SUB(sub)
+
+uint32_t helper_fmul(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+    float32 f_result;
+
+    f_result = float32_mul(arg1, arg2, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        /* If the output is a NaN, but the inputs aren't,
+           we return a unique value.  */
+        if ((flags & float_flag_invalid)
+            && !float32_is_any_nan(arg1)
+            && !float32_is_any_nan(arg2)) {
+                f_result = MUL_NAN;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)f_result;
+
+}
+
+uint32_t helper_fdiv(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+    float32 f_result;
+
+    f_result = float32_div(arg1, arg2 , &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        /* If the output is a NaN, but the inputs aren't,
+           we return a unique value.  */
+        if ((flags & float_flag_invalid)
+            && !float32_is_any_nan(arg1)
+            && !float32_is_any_nan(arg2)) {
+                f_result = DIV_NAN;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+
+    return (uint32_t)f_result;
+}
+
+uint32_t helper_fcmp(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t result, flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+
+    set_flush_inputs_to_zero(0, &env->fp_status);
+
+    result = 1 << (float32_compare_quiet(arg1, arg2, &env->fp_status) + 1);
+    result |= f_is_denormal(arg1) << 4;
+    result |= f_is_denormal(arg2) << 5;
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+
+    set_flush_inputs_to_zero(1, &env->fp_status);
+    return result;
+}
+
+uint32_t helper_ftoi(CPUTriCoreState *env, uint32_t arg)
+{
+    float32 f_arg = make_float32(arg);
+    int32_t result, flags;
+
+    result = float32_to_int32(f_arg, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        if (float32_is_any_nan(f_arg)) {
+            result = 0;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)result;
+}
+
+uint32_t helper_itof(CPUTriCoreState *env, uint32_t arg)
+{
+    float32 f_result;
+    uint32_t flags;
+    f_result = int32_to_float32(arg, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)f_result;
+}
diff --git a/qemu/target-tricore/helper.c b/qemu/target-tricore/helper.c
index f52504c9f..71b31cdb9 100644
--- a/qemu/target-tricore/helper.c
+++ b/qemu/target-tricore/helper.c
@@ -15,12 +15,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 
@@ -66,8 +61,8 @@ int cpu_tricore_handle_mmu_fault(CPUState *cs, target_ulong address,
     access_type = ACCESS_INT;
     ret = get_physical_address(env, &physical, &prot,
                                address, rw, access_type);
-    qemu_log("%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx
-             " prot %d\n", __func__, address, ret, physical, prot);
+    qemu_log_mask(CPU_LOG_MMU, "%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx
+                  " prot %d\n", __func__, address, ret, physical, prot);
 
     if (ret == TLBRET_MATCH) {
         tlb_set_page(cs, address & TARGET_PAGE_MASK,
@@ -115,10 +110,18 @@ void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf)
     g_slist_free(list);
 }
 
+void fpu_set_state(CPUTriCoreState *env)
+{
+    set_float_rounding_mode(env->PSW & MASK_PSW_FPU_RM, &env->fp_status);
+    set_flush_inputs_to_zero(1, &env->fp_status);
+    set_flush_to_zero(1, &env->fp_status);
+    set_default_nan_mode(1, &env->fp_status);
+}
+
 uint32_t psw_read(CPUTriCoreState *env)
 {
     /* clear all USB bits */
-    env->PSW &= 0xffffff;
+    env->PSW &= 0x6ffffff;
     /* now set them from the cache */
     env->PSW |= ((env->PSW_USB_C != 0) << 31);
     env->PSW |= ((env->PSW_USB_V   & (1 << 31))  >> 1);
@@ -132,9 +135,11 @@ uint32_t psw_read(CPUTriCoreState *env)
 void psw_write(CPUTriCoreState *env, uint32_t val)
 {
     env->PSW_USB_C = (val & MASK_USB_C);
-    env->PSW_USB_V = (val & MASK_USB_V << 1);
-    env->PSW_USB_SV = (val & MASK_USB_SV << 2);
-    env->PSW_USB_AV = ((val & MASK_USB_AV) << 3);
-    env->PSW_USB_SAV = ((val & MASK_USB_SAV) << 4);
+    env->PSW_USB_V = (val & MASK_USB_V) << 1;
+    env->PSW_USB_SV = (val & MASK_USB_SV) << 2;
+    env->PSW_USB_AV = (val & MASK_USB_AV) << 3;
+    env->PSW_USB_SAV = (val & MASK_USB_SAV) << 4;
     env->PSW = val;
+
+    fpu_set_state(env);
 }
diff --git a/qemu/target-tricore/helper.h b/qemu/target-tricore/helper.h
index cc221f1a9..9333e161a 100644
--- a/qemu/target-tricore/helper.h
+++ b/qemu/target-tricore/helper.h
@@ -105,6 +105,13 @@ DEF_HELPER_FLAGS_1(parity, TCG_CALL_NO_RWG_SE, i32, i32)
 /* float */
 DEF_HELPER_FLAGS_4(pack, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32, i32)
 DEF_HELPER_1(unpack, i64, i32)
+DEF_HELPER_3(fadd, i32, env, i32, i32)
+DEF_HELPER_3(fsub, i32, env, i32, i32)
+DEF_HELPER_3(fmul, i32, env, i32, i32)
+DEF_HELPER_3(fdiv, i32, env, i32, i32)
+DEF_HELPER_3(fcmp, i32, env, i32, i32)
+DEF_HELPER_2(ftoi, i32, env, i32)
+DEF_HELPER_2(itof, i32, env, i32)
 /* dvinit */
 DEF_HELPER_3(dvinit_b_13, i64, env, i32, i32)
 DEF_HELPER_3(dvinit_b_131, i64, env, i32, i32)
@@ -132,6 +139,7 @@ DEF_HELPER_2(lducx, void, env, i32)
 DEF_HELPER_2(stlcx, void, env, i32)
 DEF_HELPER_2(stucx, void, env, i32)
 DEF_HELPER_1(svlcx, void, env)
+DEF_HELPER_1(svucx, void, env)
 DEF_HELPER_1(rslcx, void, env)
 /* Address mode helper */
 DEF_HELPER_1(br_update, i32, i32)
@@ -139,3 +147,5 @@ DEF_HELPER_2(circ_update, i32, i32, i32)
 /* PSW cache helper */
 DEF_HELPER_2(psw_write, void, env, i32)
 DEF_HELPER_1(psw_read, i32, env)
+/* Exceptions */
+DEF_HELPER_3(raise_exception_sync, noreturn, env, i32, i32)
diff --git a/qemu/target-tricore/op_helper.c b/qemu/target-tricore/op_helper.c
index 53edbdae1..40656c357 100644
--- a/qemu/target-tricore/op_helper.c
+++ b/qemu/target-tricore/op_helper.c
@@ -14,13 +14,100 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
 #include <zlib.h> /* for crc32 */
 
+
+/* Exception helpers */
+
+static void QEMU_NORETURN
+raise_exception_sync_internal(CPUTriCoreState *env, uint32_t class, int tin,
+                              uintptr_t pc, uint32_t fcd_pc)
+{
+    CPUState *cs = CPU(tricore_env_get_cpu(env));
+    /* in case we come from a helper-call we need to restore the PC */
+    if (pc) {
+        cpu_restore_state(cs, pc);
+    }
+
+    /* Tin is loaded into d[15] */
+    env->gpr_d[15] = tin;
+
+    if (class == TRAPC_CTX_MNG && tin == TIN3_FCU) {
+        /* upper context cannot be saved, if the context list is empty */
+    } else {
+        helper_svucx(env);
+    }
+
+    /* The return address in a[11] is updated */
+    if (class == TRAPC_CTX_MNG && tin == TIN3_FCD) {
+        env->SYSCON |= MASK_SYSCON_FCD_SF;
+        /* when we run out of CSAs after saving a context a FCD trap is taken
+           and the return address is the start of the trap handler which used
+           the last CSA */
+        env->gpr_a[11] = fcd_pc;
+    } else if (class == TRAPC_SYSCALL) {
+        env->gpr_a[11] = env->PC + 4;
+    } else {
+        env->gpr_a[11] = env->PC;
+    }
+    /* The stack pointer in A[10] is set to the Interrupt Stack Pointer (ISP)
+       when the processor was not previously using the interrupt stack
+       (in case of PSW.IS = 0). The stack pointer bit is set for using the
+       interrupt stack: PSW.IS = 1. */
+    if ((env->PSW & MASK_PSW_IS) == 0) {
+        env->gpr_a[10] = env->ISP;
+    }
+    env->PSW |= MASK_PSW_IS;
+    /* The I/O mode is set to Supervisor mode, which means all permissions
+       are enabled: PSW.IO = 10 B .*/
+    env->PSW |= (2 << 10);
+
+    /*The current Protection Register Set is set to 0: PSW.PRS = 00 B .*/
+    env->PSW &= ~MASK_PSW_PRS;
+
+    /* The Call Depth Counter (CDC) is cleared, and the call depth limit is
+       set for 64: PSW.CDC = 0000000 B .*/
+    env->PSW &= ~MASK_PSW_CDC;
+
+    /* Call Depth Counter is enabled, PSW.CDE = 1. */
+    env->PSW |= MASK_PSW_CDE;
+
+    /* Write permission to global registers A[0], A[1], A[8], A[9] is
+       disabled: PSW.GW = 0. */
+    env->PSW &= ~MASK_PSW_GW;
+
+    /*The interrupt system is globally disabled: ICR.IE = 0. The ‘old’
+      ICR.IE and ICR.CCPN are saved */
+
+    /* PCXI.PIE = ICR.IE */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.PCPN = ICR.CCPN */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* Update PC using the trap vector table */
+    env->PC = env->BTV | (class << 5);
+
+    cpu_loop_exit(cs);
+}
+
+void helper_raise_exception_sync(CPUTriCoreState *env, uint32_t class,
+                                 uint32_t tin)
+{
+    raise_exception_sync_internal(env, class, tin, 0, 0);
+}
+
+static void raise_exception_sync_helper(CPUTriCoreState *env, uint32_t class,
+                                        uint32_t tin, uintptr_t pc)
+{
+    raise_exception_sync_internal(env, class, tin, pc, 0);
+}
+
 /* Addressing mode helper */
 
 static uint16_t reverse16(uint16_t val)
@@ -958,6 +1045,8 @@ uint64_t helper_msub64_q_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2,
             } else {
                result = INT64_MIN;
             }
+        } else {
+            env->PSW_USB_V = 0;
         }
     } else {
         if (ovf < 0) {
@@ -2279,7 +2368,7 @@ static bool cdc_zero(target_ulong *psw)
 static void save_context_upper(CPUTriCoreState *env, int ea)
 {
     cpu_stl_data(env, ea, env->PCXI);
-    cpu_stl_data(env, ea+4, env->PSW);
+    cpu_stl_data(env, ea+4, psw_read(env));
     cpu_stl_data(env, ea+8, env->gpr_a[10]);
     cpu_stl_data(env, ea+12, env->gpr_a[11]);
     cpu_stl_data(env, ea+16, env->gpr_d[8]);
@@ -2369,11 +2458,13 @@ void helper_call(CPUTriCoreState *env, uint32_t next_pc)
     /* if (FCX == 0) trap(FCU); */
     if (env->FCX == 0) {
         /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
     }
     /* if (PSW.CDE) then if (cdc_increment()) then trap(CDO); */
     if (psw & MASK_PSW_CDE) {
         if (cdc_increment(&psw)) {
             /* CDO trap */
+            raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CDO, GETPC());
         }
     }
     /* PSW.CDE = 1;*/
@@ -2409,6 +2500,7 @@ void helper_call(CPUTriCoreState *env, uint32_t next_pc)
     /* if (tmp_FCX == LCX) trap(FCD);*/
     if (tmp_FCX == env->LCX) {
         /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
     }
     psw_write(env, psw);
 }
@@ -2421,18 +2513,25 @@ void helper_ret(CPUTriCoreState *env)
 
     psw = psw_read(env);
      /* if (PSW.CDE) then if (cdc_decrement()) then trap(CDU);*/
-    if (env->PSW & MASK_PSW_CDE) {
-        if (cdc_decrement(&(env->PSW))) {
+    if (psw & MASK_PSW_CDE) {
+        if (cdc_decrement(&psw)) {
             /* CDU trap */
+            psw_write(env, psw);
+            raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CDU, GETPC());
         }
     }
     /*   if (PCXI[19: 0] == 0) then trap(CSU); */
     if ((env->PCXI & 0xfffff) == 0) {
         /* CSU trap */
+        psw_write(env, psw);
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
     }
     /* if (PCXI.UL == 0) then trap(CTYP); */
     if ((env->PCXI & MASK_PCXI_UL) == 0) {
         /* CTYP trap */
+        cdc_increment(&psw); /* restore to the start of helper */
+        psw_write(env, psw);
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
     }
     /* PC = {A11 [31: 1], 1’b0}; */
     env->PC = env->gpr_a[11] & 0xfffffffe;
@@ -2467,6 +2566,7 @@ void helper_bisr(CPUTriCoreState *env, uint32_t const9)
 
     if (env->FCX == 0) {
         /* FCU trap */
+       raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
     }
 
     tmp_FCX = env->FCX;
@@ -2498,6 +2598,7 @@ void helper_bisr(CPUTriCoreState *env, uint32_t const9)
 
     if (tmp_FCX == env->LCX) {
         /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
     }
 }
 
@@ -2509,14 +2610,17 @@ void helper_rfe(CPUTriCoreState *env)
     /* if (PCXI[19: 0] == 0) then trap(CSU); */
     if ((env->PCXI & 0xfffff) == 0) {
         /* raise csu trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
     }
     /* if (PCXI.UL == 0) then trap(CTYP); */
     if ((env->PCXI & MASK_PCXI_UL) == 0) {
         /* raise CTYP trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
     }
     /* if (!cdc_zero() AND PSW.CDE) then trap(NEST); */
     if (!cdc_zero(&(env->PSW)) && (env->PSW & MASK_PSW_CDE)) {
-        /* raise MNG trap */
+        /* raise NEST trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_NEST, GETPC());
     }
     env->PC = env->gpr_a[11] & ~0x1;
     /* ICR.IE = PCXI.PIE; */
@@ -2592,6 +2696,7 @@ void helper_svlcx(CPUTriCoreState *env)
 
     if (env->FCX == 0) {
         /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
     }
     /* tmp_FCX = FCX; */
     tmp_FCX = env->FCX;
@@ -2622,6 +2727,50 @@ void helper_svlcx(CPUTriCoreState *env)
     /* if (tmp_FCX == LCX) trap(FCD);*/
     if (tmp_FCX == env->LCX) {
         /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
+    }
+}
+
+void helper_svucx(CPUTriCoreState *env)
+{
+    target_ulong tmp_FCX;
+    target_ulong ea;
+    target_ulong new_FCX;
+
+    if (env->FCX == 0) {
+        /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
+    }
+    /* tmp_FCX = FCX; */
+    tmp_FCX = env->FCX;
+    /* EA = {FCX.FCXS, 6'b0, FCX.FCXO, 6'b0}; */
+    ea = ((env->FCX & MASK_FCX_FCXS) << 12) +
+         ((env->FCX & MASK_FCX_FCXO) << 6);
+    /* new_FCX = M(EA, word); */
+    new_FCX = cpu_ldl_data(env, ea);
+    /* M(EA, 16 * word) = {PCXI, PSW, A[10], A[11], D[8], D[9], D[10], D[11],
+                           A[12], A[13], A[14], A[15], D[12], D[13], D[14],
+                           D[15]}; */
+    save_context_upper(env, ea);
+
+    /* PCXI.PCPN = ICR.CCPN; */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* PCXI.PIE = ICR.IE; */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.UL = 1; */
+    env->PCXI |= MASK_PCXI_UL;
+
+    /* PCXI[19: 0] = FCX[19: 0]; */
+    env->PCXI = (env->PCXI & 0xfff00000) + (env->FCX & 0xfffff);
+    /* FCX[19: 0] = new_FCX[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (new_FCX & 0xfffff);
+
+    /* if (tmp_FCX == LCX) trap(FCD);*/
+    if (tmp_FCX == env->LCX) {
+        /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
     }
 }
 
@@ -2632,10 +2781,12 @@ void helper_rslcx(CPUTriCoreState *env)
     /*   if (PCXI[19: 0] == 0) then trap(CSU); */
     if ((env->PCXI & 0xfffff) == 0) {
         /* CSU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
     }
     /* if (PCXI.UL == 1) then trap(CTYP); */
     if ((env->PCXI & MASK_PCXI_UL) != 0) {
         /* CTYP trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
     }
     /* EA = {PCXI.PCXS, 6'b0, PCXI.PCXO, 6'b0}; */
     ea = ((env->PCXI & MASK_PCXI_PCXS) << 12) +
diff --git a/qemu/target-tricore/translate.c b/qemu/target-tricore/translate.c
index 7dc7a325b..912bf226b 100644
--- a/qemu/target-tricore/translate.c
+++ b/qemu/target-tricore/translate.c
@@ -18,6 +18,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
@@ -27,6 +28,7 @@
 #include "exec/helper-gen.h"
 
 #include "tricore-opcodes.h"
+#include "exec/log.h"
 
 /*
  * TCG registers
@@ -45,7 +47,7 @@ static TCGv cpu_PSW_SV;
 static TCGv cpu_PSW_AV;
 static TCGv cpu_PSW_SAV;
 /* CPU env */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 #include "exec/gen-icount.h"
 
@@ -214,6 +216,15 @@ void tricore_cpu_dump_state(CPUState *cs, FILE *f,
 #define EA_B_ABSOLUT(con) (((offset & 0xf00000) << 8) | \
                            ((offset & 0x0fffff) << 1))
 
+/* For two 32-bit registers used a 64-bit register, the first
+   registernumber needs to be even. Otherwise we trap. */
+static inline void generate_trap(DisasContext *ctx, int class, int tin);
+#define CHECK_REG_PAIR(reg) do {                      \
+    if (reg & 0x1) {                                  \
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_OPD); \
+    }                                                 \
+} while (0)
+
 /* Functions for load/save to/from memory */
 
 static inline void gen_offset_ld(DisasContext *ctx, TCGv r1, TCGv r2,
@@ -299,6 +310,7 @@ static void gen_ldmst(DisasContext *ctx, int ereg, TCGv ea)
     TCGv temp = tcg_temp_new();
     TCGv temp2 = tcg_temp_new();
 
+    CHECK_REG_PAIR(ereg);
     /* temp = (M(EA, word) */
     tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
     /* temp = temp & ~E[a][63:32]) */
@@ -457,11 +469,11 @@ gen_add64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
     tcg_gen_xor_i64(t1, result, r1);
     tcg_gen_xor_i64(t0, r1, r2);
     tcg_gen_andc_i64(t1, t1, t0);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t1, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
     /* calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
     /* calc AV/SAV bits */
-    tcg_gen_trunc_shr_i64_i32(temp, result, 32);
+    tcg_gen_extrh_i64_i32(temp, result);
     tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
     tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
     /* calc SAV */
@@ -540,14 +552,14 @@ static inline void gen_madd32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
     tcg_gen_mul_i64(t1, t1, t3);
     tcg_gen_add_i64(t1, t2, t1);
 
-    tcg_gen_trunc_i64_i32(ret, t1);
+    tcg_gen_extrl_i64_i32(ret, t1);
     /* calc V
        t1 > 0x7fffffff */
     tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
     /* t1 < -0x80000000 */
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
     tcg_gen_or_i64(t2, t2, t3);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -621,7 +633,7 @@ gen_maddu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
     /* only the add overflows, if t2 < t1
        calc V bit */
     tcg_gen_setcond_i64(TCG_COND_LTU, t2, t2, t1);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -1110,12 +1122,12 @@ gen_madd32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
     tcg_gen_sari_i64(t2, t2, up_shift);
 
     tcg_gen_add_i64(t3, t1, t2);
-    tcg_gen_trunc_i64_i32(temp3, t3);
+    tcg_gen_extrl_i64_i32(temp3, t3);
     /* calc v bit */
     tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
     tcg_gen_or_i64(t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
@@ -1273,7 +1285,7 @@ gen_madd64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
     tcg_gen_xor_i64(t3, t4, t1);
     tcg_gen_xor_i64(t2, t1, t2);
     tcg_gen_andc_i64(t3, t3, t2);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t3, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
        case, we negate the ovf. */
@@ -1356,14 +1368,14 @@ static inline void gen_msub32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
     tcg_gen_mul_i64(t1, t1, t3);
     tcg_gen_sub_i64(t1, t2, t1);
 
-    tcg_gen_trunc_i64_i32(ret, t1);
+    tcg_gen_extrl_i64_i32(ret, t1);
     /* calc V
        t2 > 0x7fffffff */
     tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
     /* result < -0x80000000 */
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
     tcg_gen_or_i64(t2, t2, t3);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
 
     /* Calc SV bit */
@@ -1445,7 +1457,7 @@ gen_msubu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
     tcg_gen_extr_i64_i32(ret_low, ret_high, t3);
     /* calc V bit, only the sub can overflow, if t1 > t2 */
     tcg_gen_setcond_i64(TCG_COND_GTU, t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -1630,11 +1642,11 @@ gen_sub64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
     tcg_gen_xor_i64(t1, result, r1);
     tcg_gen_xor_i64(t0, r1, r2);
     tcg_gen_and_i64(t1, t1, t0);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t1, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
     /* calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
     /* calc AV/SAV bits */
-    tcg_gen_trunc_shr_i64_i32(temp, result, 32);
+    tcg_gen_extrh_i64_i32(temp, result);
     tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
     tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
     /* calc SAV */
@@ -1973,12 +1985,12 @@ gen_msub32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
     tcg_gen_add_i64(t2, t2, t4);
 
     tcg_gen_sub_i64(t3, t1, t2);
-    tcg_gen_trunc_i64_i32(temp3, t3);
+    tcg_gen_extrl_i64_i32(temp3, t3);
     /* calc v bit */
     tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
     tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
     tcg_gen_or_i64(t1, t1, t2);
-    tcg_gen_trunc_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
     tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
     /* Calc SV bit */
     tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
@@ -2126,7 +2138,7 @@ gen_msub64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
     tcg_gen_xor_i64(t3, t4, t1);
     tcg_gen_xor_i64(t2, t1, t2);
     tcg_gen_and_i64(t3, t3, t2);
-    tcg_gen_trunc_shr_i64_i32(cpu_PSW_V, t3, 32);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
     /* We produce an overflow on the host if the mul before was
        (0x80000000 * 0x80000000) << 1). If this is the
        case, we negate the ovf. */
@@ -3242,6 +3254,19 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
     }
 }
 
+static void generate_trap(DisasContext *ctx, int class, int tin)
+{
+    TCGv_i32 classtemp = tcg_const_i32(class);
+    TCGv_i32 tintemp = tcg_const_i32(tin);
+
+    gen_save_pc(ctx->pc);
+    gen_helper_raise_exception_sync(cpu_env, classtemp, tintemp);
+    ctx->bstate = BS_EXCP;
+
+    tcg_temp_free(classtemp);
+    tcg_temp_free(tintemp);
+}
+
 static inline void gen_branch_cond(DisasContext *ctx, TCGCond cond, TCGv r1,
                                    TCGv r2, int16_t address)
 {
@@ -3540,7 +3565,7 @@ static void gen_compute_branch(DisasContext *ctx, uint32_t opc, int r1,
         }
         break;
     default:
-        printf("Branch Error at %x\n", ctx->pc);
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     ctx->bstate = BS_BRANCH;
 }
@@ -3615,7 +3640,9 @@ static void decode_src_opc(CPUTriCoreState *env, DisasContext *ctx, int op1)
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             tcg_gen_movi_tl(cpu_gpr_d[r1], const4);
             tcg_gen_sari_tl(cpu_gpr_d[r1+1], cpu_gpr_d[r1], 31);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC1_16_SRC_SH:
         gen_shi(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
@@ -3623,6 +3650,8 @@ static void decode_src_opc(CPUTriCoreState *env, DisasContext *ctx, int op1)
     case OPC1_16_SRC_SHA:
         gen_shaci(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3706,6 +3735,8 @@ static void decode_srr_opc(DisasContext *ctx, int op1)
     case OPC1_16_SRR_XOR:
         tcg_gen_xor_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3745,6 +3776,8 @@ static void decode_ssr_opc(DisasContext *ctx, int op1)
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3782,6 +3815,8 @@ static void decode_sc_opc(DisasContext *ctx, int op1)
     case OPC1_16_SC_SUB_A:
         tcg_gen_subi_tl(cpu_gpr_a[10], cpu_gpr_a[10], const16);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3822,6 +3857,8 @@ static void decode_slr_opc(DisasContext *ctx, int op1)
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESL);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3859,6 +3896,8 @@ static void decode_sro_opc(DisasContext *ctx, int op1)
     case OPC1_16_SRO_ST_W:
         gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address * 4, MO_LESL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3883,6 +3922,9 @@ static void decode_sr_system(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_16_SR_FRET:
         gen_fret(ctx);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -3925,6 +3967,8 @@ static void decode_sr_accu(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_16_SR_SAT_HU:
         gen_saturate_u(cpu_gpr_d[r1], cpu_gpr_d[r1], 0xffff);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4135,6 +4179,8 @@ static void decode_16Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
         r1 = MASK_OP_SR_S1D(ctx->opcode);
         tcg_gen_not_tl(cpu_gpr_d[r1], cpu_gpr_d[r1]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4161,14 +4207,18 @@ static void decode_abs_ldw(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
         break;
     case OPC2_32_ABS_LD_D:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
         break;
     case OPC2_32_ABS_LD_DA:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
         break;
     case OPC2_32_ABS_LD_W:
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -4200,6 +4250,8 @@ static void decode_abs_ldb(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_ABS_LD_HU:
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -4225,6 +4277,8 @@ static void decode_abs_ldst_swap(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_ABS_SWAP_W:
         gen_swap(ctx, r1, temp);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -4251,6 +4305,8 @@ static void decode_abs_ldst_context(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_ABS_STUCX:
         gen_helper_1arg(stucx, EA_ABS_FORMAT(off18));
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4272,15 +4328,18 @@ static void decode_abs_store(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
         break;
     case OPC2_32_ABS_ST_D:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
         break;
     case OPC2_32_ABS_ST_DA:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
         break;
     case OPC2_32_ABS_ST_W:
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
         break;
-
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -4305,6 +4364,8 @@ static void decode_abs_storeb_h(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_ABS_ST_H:
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -4347,6 +4408,8 @@ static void decode_bit_andacc(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl, &tcg_gen_and_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4379,6 +4442,8 @@ static void decode_bit_logical_t(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4436,6 +4501,8 @@ static void decode_bit_logical_t2(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4475,6 +4542,8 @@ static void decode_bit_orand(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4511,6 +4580,8 @@ static void decode_bit_sh_logic1(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
     tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
@@ -4550,6 +4621,8 @@ static void decode_bit_sh_logic2(CPUTriCoreState *env, DisasContext *ctx)
         gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
     tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
@@ -4594,20 +4667,25 @@ static void decode_bo_addrmode_post_pre_base(CPUTriCoreState *env,
         break;
     case OPC2_32_BO_CACHEI_WI_SHORTOFF:
     case OPC2_32_BO_CACHEI_W_SHORTOFF:
-        /* TODO: Raise illegal opcode trap,
-                 if !tricore_feature(TRICORE_FEATURE_131) */
+        if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_CACHEI_W_POSTINC:
     case OPC2_32_BO_CACHEI_WI_POSTINC:
         if (tricore_feature(env, TRICORE_FEATURE_131)) {
             tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_CACHEI_W_PREINC:
     case OPC2_32_BO_CACHEI_WI_PREINC:
         if (tricore_feature(env, TRICORE_FEATURE_131)) {
             tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_ST_A_SHORTOFF:
         gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LESL);
@@ -4632,14 +4710,17 @@ static void decode_bo_addrmode_post_pre_base(CPUTriCoreState *env,
         gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_UB);
         break;
     case OPC2_32_BO_ST_D_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_st_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_ST_D_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_ST_D_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
@@ -4647,14 +4728,17 @@ static void decode_bo_addrmode_post_pre_base(CPUTriCoreState *env,
         tcg_temp_free(temp);
         break;
     case OPC2_32_BO_ST_DA_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_st_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_ST_DA_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_ST_DA_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
@@ -4703,6 +4787,8 @@ static void decode_bo_addrmode_post_pre_base(CPUTriCoreState *env,
     case OPC2_32_BO_ST_W_PREINC:
         gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4722,7 +4808,7 @@ static void decode_bo_addrmode_bitreverse_circular(CPUTriCoreState *env,
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -4754,10 +4840,12 @@ static void decode_bo_addrmode_bitreverse_circular(CPUTriCoreState *env,
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_ST_D_BR:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_ST_D_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -4767,10 +4855,12 @@ static void decode_bo_addrmode_bitreverse_circular(CPUTriCoreState *env,
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_ST_DA_BR:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_ST_DA_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -4805,6 +4895,8 @@ static void decode_bo_addrmode_bitreverse_circular(CPUTriCoreState *env,
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -4859,14 +4951,17 @@ static void decode_bo_addrmode_ld_post_pre_base(CPUTriCoreState *env,
         gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_SB);
         break;
     case OPC2_32_BO_LD_D_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_ld_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_LD_D_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_LD_D_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
@@ -4874,14 +4969,17 @@ static void decode_bo_addrmode_ld_post_pre_base(CPUTriCoreState *env,
         tcg_temp_free(temp);
         break;
     case OPC2_32_BO_LD_DA_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_ld_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_LD_DA_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_LD_DA_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
@@ -4935,6 +5033,8 @@ static void decode_bo_addrmode_ld_post_pre_base(CPUTriCoreState *env,
     case OPC2_32_BO_LD_W_PREINC:
         gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -4955,7 +5055,7 @@ static void decode_bo_addrmode_ld_bitreverse_circular(CPUTriCoreState *env,
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -4986,10 +5086,12 @@ static void decode_bo_addrmode_ld_bitreverse_circular(CPUTriCoreState *env,
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_LD_D_BR:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_LD_D_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -4999,10 +5101,12 @@ static void decode_bo_addrmode_ld_bitreverse_circular(CPUTriCoreState *env,
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_LD_DA_BR:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_LD_DA_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -5045,6 +5149,8 @@ static void decode_bo_addrmode_ld_bitreverse_circular(CPUTriCoreState *env,
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -5137,6 +5243,8 @@ static void decode_bo_addrmode_stctx_post_pre_base(CPUTriCoreState *env,
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         gen_swapmsk(ctx, r1, cpu_gpr_a[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -5159,7 +5267,7 @@ static void decode_bo_addrmode_ldmst_bitreverse_circular(CPUTriCoreState *env,
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -5196,6 +5304,8 @@ static void decode_bo_addrmode_ldmst_bitreverse_circular(CPUTriCoreState *env,
         gen_swapmsk(ctx, r1, temp2);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -5233,7 +5343,7 @@ static void decode_bol_opc(CPUTriCoreState *env, DisasContext *ctx, int32_t op1)
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], address, MO_LEUL);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_ST_W_LONGOFF:
@@ -5243,44 +5353,46 @@ static void decode_bol_opc(CPUTriCoreState *env, DisasContext *ctx, int32_t op1)
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_SB);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_LD_BU_LONGOFF:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_UB);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_LD_H_LONGOFF:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LESW);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_LD_HU_LONGOFF:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LEUW);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_ST_B_LONGOFF:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_SB);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_BOL_ST_H_LONGOFF:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LESW);
         } else {
-            /* raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5346,6 +5458,8 @@ static void decode_rc_logical_shift(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_RC_XOR:
         tcg_gen_xori_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -5544,6 +5658,8 @@ static void decode_rc_accumulator(CPUTriCoreState *env, DisasContext *ctx)
         gen_accumulating_condi(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1],
                                const9, &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -5563,6 +5679,8 @@ static void decode_rc_serviceroutine(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_RC_SYSCALL:
         /* TODO: Add exception generation */
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5583,6 +5701,7 @@ static void decode_rc_mul(CPUTriCoreState *env, DisasContext *ctx)
         gen_muli_i32s(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
         break;
     case OPC2_32_RC_MUL_64:
+        CHECK_REG_PAIR(r2);
         gen_muli_i64s(cpu_gpr_d[r2], cpu_gpr_d[r2+1], cpu_gpr_d[r1], const9);
         break;
     case OPC2_32_RC_MULS_32:
@@ -5590,12 +5709,15 @@ static void decode_rc_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RC_MUL_U_64:
         const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        CHECK_REG_PAIR(r2);
         gen_muli_i64u(cpu_gpr_d[r2], cpu_gpr_d[r2+1], cpu_gpr_d[r1], const9);
         break;
     case OPC2_32_RC_MULS_U_32:
         const9 = MASK_OP_RC_CONST9(ctx->opcode);
         gen_mulsui_i32(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5617,6 +5739,7 @@ static void decode_rcpw_insert(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RCPW_IMASK:
+        CHECK_REG_PAIR(r2);
         /* if pos + width > 31 undefined result */
         if (pos + width <= 31) {
             tcg_gen_movi_tl(cpu_gpr_d[r2+1], ((1u << width) - 1) << pos);
@@ -5631,6 +5754,8 @@ static void decode_rcpw_insert(CPUTriCoreState *env, DisasContext *ctx)
             tcg_temp_free(temp);
         }
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5672,6 +5797,8 @@ static void decode_rcrw_insert(CPUTriCoreState *env, DisasContext *ctx)
 
         tcg_temp_free(temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -5718,6 +5845,8 @@ static void decode_rcr_cond_select(CPUTriCoreState *env, DisasContext *ctx)
         tcg_temp_free(temp);
         tcg_temp_free(temp2);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5739,6 +5868,8 @@ static void decode_rcr_madd(CPUTriCoreState *env, DisasContext *ctx)
         gen_maddi32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MADD_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddi64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
@@ -5746,10 +5877,14 @@ static void decode_rcr_madd(CPUTriCoreState *env, DisasContext *ctx)
         gen_maddsi_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MADDS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsi_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
     case OPC2_32_RCR_MADD_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         const9 = MASK_OP_RCR_CONST9(ctx->opcode);
         gen_maddui64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                        cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
@@ -5759,10 +5894,14 @@ static void decode_rcr_madd(CPUTriCoreState *env, DisasContext *ctx)
         gen_maddsui_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MADDS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         const9 = MASK_OP_RCR_CONST9(ctx->opcode);
         gen_maddsui_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                        cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5784,6 +5923,8 @@ static void decode_rcr_msub(CPUTriCoreState *env, DisasContext *ctx)
         gen_msubi32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MSUB_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubi64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
@@ -5791,10 +5932,14 @@ static void decode_rcr_msub(CPUTriCoreState *env, DisasContext *ctx)
         gen_msubsi_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MSUBS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubsi_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
     case OPC2_32_RCR_MSUB_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         const9 = MASK_OP_RCR_CONST9(ctx->opcode);
         gen_msubui64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                        cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
@@ -5804,10 +5949,14 @@ static void decode_rcr_msub(CPUTriCoreState *env, DisasContext *ctx)
         gen_msubsui_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
         break;
     case OPC2_32_RCR_MSUBS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         const9 = MASK_OP_RCR_CONST9(ctx->opcode);
         gen_msubsui_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                        cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -5842,13 +5991,11 @@ static void decode_rlc_opc(CPUTriCoreState *env, DisasContext *ctx,
         break;
     case OPC1_32_RLC_MOV_64:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
-            if ((r2 & 0x1) != 0) {
-                /* TODO: raise OPD trap */
-            }
+            CHECK_REG_PAIR(r2);
             tcg_gen_movi_tl(cpu_gpr_d[r2], const16);
             tcg_gen_movi_tl(cpu_gpr_d[r2+1], const16 >> 15);
         } else {
-            /* TODO: raise illegal opcode trap */
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
         }
         break;
     case OPC1_32_RLC_MOV_U:
@@ -5865,6 +6012,8 @@ static void decode_rlc_opc(CPUTriCoreState *env, DisasContext *ctx,
         const16 = MASK_OP_RLC_CONST16(ctx->opcode);
         gen_mtcr(env, ctx, cpu_gpr_d[r1], const16);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6181,6 +6330,8 @@ static void decode_rr_accumulator(CPUTriCoreState *env, DisasContext *ctx)
         gen_accumulating_cond(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
                               cpu_gpr_d[r2], &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6255,6 +6406,8 @@ static void decode_rr_logical_shift(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_RR_XOR:
         tcg_gen_xor_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -6322,6 +6475,8 @@ static void decode_rr_address(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_RR_SUB_A:
         tcg_gen_sub_tl(cpu_gpr_a[r3], cpu_gpr_a[r1], cpu_gpr_a[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6349,6 +6504,8 @@ static void decode_rr_idirect(CPUTriCoreState *env, DisasContext *ctx)
         gen_fcall_save_ctx(ctx);
         tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], ~0x1);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_gen_exit_tb(0);
     ctx->bstate = BS_BRANCH;
@@ -6371,9 +6528,11 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
         gen_helper_bmerge(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RR_BSPLIT:
+        CHECK_REG_PAIR(r3);
         gen_bsplit(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
         break;
     case OPC2_32_RR_DVINIT_B:
+        CHECK_REG_PAIR(r3);
         gen_dvinit_b(env, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r2]);
         break;
@@ -6381,7 +6540,7 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
         temp = tcg_temp_new();
         temp2 = tcg_temp_new();
         temp3 = tcg_temp_new();
-
+        CHECK_REG_PAIR(r3);
         tcg_gen_shri_tl(temp3, cpu_gpr_d[r1], 8);
         /* reset av */
         tcg_gen_movi_tl(cpu_PSW_AV, 0);
@@ -6411,6 +6570,7 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
         tcg_temp_free(temp3);
         break;
     case OPC2_32_RR_DVINIT_H:
+        CHECK_REG_PAIR(r3);
         gen_dvinit_h(env, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r2]);
         break;
@@ -6418,7 +6578,7 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
         temp = tcg_temp_new();
         temp2 = tcg_temp_new();
         temp3 = tcg_temp_new();
-
+        CHECK_REG_PAIR(r3);
         tcg_gen_shri_tl(temp3, cpu_gpr_d[r1], 16);
         /* reset av */
         tcg_gen_movi_tl(cpu_PSW_AV, 0);
@@ -6449,6 +6609,7 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
     case OPC2_32_RR_DVINIT:
         temp = tcg_temp_new();
         temp2 = tcg_temp_new();
+        CHECK_REG_PAIR(r3);
         /* overflow = ((D[b] == 0) ||
                       ((D[b] == 0xFFFFFFFF) && (D[a] == 0x80000000))) */
         tcg_gen_setcondi_tl(TCG_COND_EQ, temp, cpu_gpr_d[r2], 0xffffffff);
@@ -6485,25 +6646,49 @@ static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
         gen_helper_parity(cpu_gpr_d[r3], cpu_gpr_d[r1]);
         break;
     case OPC2_32_RR_UNPACK:
+        CHECK_REG_PAIR(r3);
         gen_unpack(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
         break;
     case OPC2_32_RR_CRC32:
         if (tricore_feature(env, TRICORE_FEATURE_161)) {
             gen_helper_crc32(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_RR_DIV:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             GEN_HELPER_RR(divide, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
                           cpu_gpr_d[r2]);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_RR_DIV_U:
         if (tricore_feature(env, TRICORE_FEATURE_16)) {
             GEN_HELPER_RR(divide_u, cpu_gpr_d[r3], cpu_gpr_d[r3+1],
                           cpu_gpr_d[r1], cpu_gpr_d[r2]);
-        } /* TODO: else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC2_32_RR_MUL_F:
+        gen_helper_fmul(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
+    case OPC2_32_RR_DIV_F:
+        gen_helper_fdiv(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_CMP_F:
+        gen_helper_fcmp(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_FTOI:
+        gen_helper_ftoi(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_ITOF:
+        gen_helper_itof(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6525,6 +6710,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
     switch (op2) {
     case OPC2_32_RR1_MUL_H_32_LL:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_LL(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
@@ -6532,6 +6718,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MUL_H_32_LU:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_LU(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
@@ -6539,6 +6726,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MUL_H_32_UL:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_UL(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
@@ -6546,6 +6734,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MUL_H_32_UU:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_UU(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
@@ -6553,6 +6742,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MULM_H_64_LL:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_LL(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         /* reset V bit */
@@ -6563,6 +6753,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MULM_H_64_LU:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_LU(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         /* reset V bit */
@@ -6573,6 +6764,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MULM_H_64_UL:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_UL(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         /* reset V bit */
@@ -6583,6 +6775,7 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RR1_MULM_H_64_UU:
         temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
         GEN_HELPER_UU(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
         /* reset V bit */
@@ -6608,6 +6801,8 @@ static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
         GEN_HELPER_UU(mulr_h, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], n);
         gen_calc_usb_mulr_h(cpu_gpr_d[r3]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(n);
 }
@@ -6634,6 +6829,7 @@ static void decode_rr1_mulq(CPUTriCoreState *env, DisasContext *ctx)
         gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], cpu_gpr_d[r2], n, 32);
         break;
     case OPC2_32_RR1_MUL_Q_64:
+        CHECK_REG_PAIR(r3);
         gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                   n, 0);
         break;
@@ -6642,6 +6838,7 @@ static void decode_rr1_mulq(CPUTriCoreState *env, DisasContext *ctx)
         gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], temp, n, 16);
         break;
     case OPC2_32_RR1_MUL_Q_64_L:
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
         gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp, n, 0);
         break;
@@ -6650,6 +6847,7 @@ static void decode_rr1_mulq(CPUTriCoreState *env, DisasContext *ctx)
         gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], temp, n, 16);
         break;
     case OPC2_32_RR1_MUL_Q_64_U:
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
         gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp, n, 0);
         break;
@@ -6673,6 +6871,8 @@ static void decode_rr1_mulq(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_mulr_q(cpu_gpr_d[r3], temp, temp2, n);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -6693,6 +6893,7 @@ static void decode_rr2_mul(CPUTriCoreState *env, DisasContext *ctx)
         gen_mul_i32s(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RR2_MUL_64:
+        CHECK_REG_PAIR(r3);
         gen_mul_i64s(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r2]);
         break;
@@ -6701,6 +6902,7 @@ static void decode_rr2_mul(CPUTriCoreState *env, DisasContext *ctx)
                             cpu_gpr_d[r2]);
         break;
     case OPC2_32_RR2_MUL_U_64:
+        CHECK_REG_PAIR(r3);
         gen_mul_i64u(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r2]);
         break;
@@ -6708,6 +6910,8 @@ static void decode_rr2_mul(CPUTriCoreState *env, DisasContext *ctx)
         gen_helper_mul_suov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
                             cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6748,6 +6952,7 @@ static void decode_rrpw_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
         }
         break;
     case OPC2_32_RRPW_IMASK:
+        CHECK_REG_PAIR(r3);
         if (pos + width <= 31) {
             tcg_gen_movi_tl(cpu_gpr_d[r3+1], ((1u << width) - 1) << pos);
             tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r2], pos);
@@ -6759,6 +6964,8 @@ static void decode_rrpw_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
                                width, pos);
         }
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6804,6 +7011,8 @@ static void decode_rrr_cond_select(CPUTriCoreState *env, DisasContext *ctx)
                            cpu_gpr_d[r1], cpu_gpr_d[r2]);
         tcg_temp_free(temp);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6821,37 +7030,60 @@ static void decode_rrr_divide(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RRR_DVADJ:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(dvadj, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_DVSTEP:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(dvstep, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_DVSTEP_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(dvstep_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_IXMAX:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(ixmax, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_IXMAX_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(ixmax_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_IXMIN:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(ixmin, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_IXMIN_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
         GEN_HELPER_RRR(ixmin_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR_PACK:
+        CHECK_REG_PAIR(r3);
         gen_helper_pack(cpu_gpr_d[r4], cpu_PSW_C, cpu_gpr_d[r3],
                         cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
         break;
+    case OPC2_32_RRR_ADD_F:
+        gen_helper_fadd(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_SUB_F:
+        gen_helper_fsub(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r3]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6872,6 +7104,8 @@ static void decode_rrr2_madd(CPUTriCoreState *env, DisasContext *ctx)
                      cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MADD_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
@@ -6880,10 +7114,14 @@ static void decode_rrr2_madd(CPUTriCoreState *env, DisasContext *ctx)
                                cpu_gpr_d[r3], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MADDS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MADD_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddu64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
@@ -6892,9 +7130,13 @@ static void decode_rrr2_madd(CPUTriCoreState *env, DisasContext *ctx)
                                cpu_gpr_d[r3], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MADDS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsu_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6915,6 +7157,8 @@ static void decode_rrr2_msub(CPUTriCoreState *env, DisasContext *ctx)
                       cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MSUB_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
@@ -6923,6 +7167,8 @@ static void decode_rrr2_msub(CPUTriCoreState *env, DisasContext *ctx)
                                cpu_gpr_d[r3], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MSUBS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
@@ -6935,9 +7181,13 @@ static void decode_rrr2_msub(CPUTriCoreState *env, DisasContext *ctx)
                                cpu_gpr_d[r3], cpu_gpr_d[r2]);
         break;
     case OPC2_32_RRR2_MSUBS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubsu_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -6956,66 +7206,98 @@ static void decode_rrr1_madd(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RRR1_MADD_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADD_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADD_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADD_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDM_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDM_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDM_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDM_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDMS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDMS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDMS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDMS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
@@ -7051,6 +7333,8 @@ static void decode_rrr1_madd(CPUTriCoreState *env, DisasContext *ctx)
         gen_maddr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
                        cpu_gpr_d[r2], n, MODE_UU);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -7076,6 +7360,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                      cpu_gpr_d[r2], n, 32, env);
         break;
     case OPC2_32_RRR1_MADD_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                      n, env);
@@ -7086,6 +7372,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                      temp, n, 16, env);
         break;
     case OPC2_32_RRR1_MADD_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
         gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7097,6 +7385,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                      temp, n, 16, env);
         break;
     case OPC2_32_RRR1_MADD_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
         gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7108,6 +7398,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16add32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MADD_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
         tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
         gen_m16add64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7119,6 +7411,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16add32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MADD_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_m16add64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7129,6 +7423,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                       cpu_gpr_d[r2], n, 32);
         break;
     case OPC2_32_RRR1_MADDS_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n);
@@ -7139,6 +7435,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                       temp, n, 16);
         break;
     case OPC2_32_RRR1_MADDS_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
         gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7150,6 +7448,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
                       temp, n, 16);
         break;
     case OPC2_32_RRR1_MADDS_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
         gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7161,6 +7461,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16adds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MADDS_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
         tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
         gen_m16adds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7172,16 +7474,20 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16adds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MADDS_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_m16adds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                         cpu_gpr_d[r3+1], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MADDR_H_64_UL:
+        CHECK_REG_PAIR(r3);
         gen_maddr64_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
                       cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
         break;
     case OPC2_32_RRR1_MADDRS_H_64_UL:
+        CHECK_REG_PAIR(r3);
         gen_maddr64s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
                        cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
         break;
@@ -7205,6 +7511,8 @@ static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_maddrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -7224,77 +7532,109 @@ static void decode_rrr1_maddsu_h(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RRR1_MADDSU_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDSU_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDSU_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDSU_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDSUS_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDSUS_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDSUS_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDSUS_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDSUM_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDSUM_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDSUM_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDSUM_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UU);
         break;
     case OPC2_32_RRR1_MADDSUMS_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_LL);
         break;
     case OPC2_32_RRR1_MADDSUMS_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_LU);
         break;
     case OPC2_32_RRR1_MADDSUMS_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_UL);
         break;
     case OPC2_32_RRR1_MADDSUMS_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_UU);
@@ -7331,6 +7671,8 @@ static void decode_rrr1_maddsu_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_maddsur32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
                          cpu_gpr_d[r2], n, MODE_UU);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -7348,66 +7690,98 @@ static void decode_rrr1_msub(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RRR1_MSUB_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUB_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUB_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUB_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBM_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBM_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBM_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBM_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBMS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBMS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBMS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBMS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
@@ -7443,6 +7817,8 @@ static void decode_rrr1_msub(CPUTriCoreState *env, DisasContext *ctx)
         gen_msubr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
                        cpu_gpr_d[r2], n, MODE_UU);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -7468,6 +7844,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                      cpu_gpr_d[r2], n, 32, env);
         break;
     case OPC2_32_RRR1_MSUB_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                      n, env);
@@ -7478,6 +7856,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                      temp, n, 16, env);
         break;
     case OPC2_32_RRR1_MSUB_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
         gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7489,6 +7869,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                      temp, n, 16, env);
         break;
     case OPC2_32_RRR1_MSUB_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
         gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7500,6 +7882,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16sub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MSUB_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
         tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
         gen_m16sub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7511,6 +7895,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16sub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MSUB_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_m16sub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7521,6 +7907,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                       cpu_gpr_d[r2], n, 32);
         break;
     case OPC2_32_RRR1_MSUBS_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n);
@@ -7531,6 +7919,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                       temp, n, 16);
         break;
     case OPC2_32_RRR1_MSUBS_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
         gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7542,6 +7932,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
                       temp, n, 16);
         break;
     case OPC2_32_RRR1_MSUBS_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
         gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
@@ -7553,6 +7945,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16subs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MSUBS_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
         tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
         gen_m16subs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
@@ -7564,16 +7958,20 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_m16subs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MSUBS_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_m16subs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                         cpu_gpr_d[r3+1], temp, temp2, n);
         break;
     case OPC2_32_RRR1_MSUBR_H_64_UL:
+        CHECK_REG_PAIR(r3);
         gen_msubr64_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
                       cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
         break;
     case OPC2_32_RRR1_MSUBRS_H_64_UL:
+        CHECK_REG_PAIR(r3);
         gen_msubr64s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
                        cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
         break;
@@ -7597,6 +7995,8 @@ static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
         gen_msubrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -7616,77 +8016,109 @@ static void decode_rrr1_msubad_h(CPUTriCoreState *env, DisasContext *ctx)
 
     switch (op2) {
     case OPC2_32_RRR1_MSUBAD_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBAD_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBAD_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBAD_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBADS_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBADS_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBADS_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBADS_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBADM_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBADM_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBADM_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBADM_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                       n, MODE_UU);
         break;
     case OPC2_32_RRR1_MSUBADMS_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_LL);
         break;
     case OPC2_32_RRR1_MSUBADMS_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_LU);
         break;
     case OPC2_32_RRR1_MSUBADMS_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_UL);
         break;
     case OPC2_32_RRR1_MSUBADMS_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
         gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
                        cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
                        n, MODE_UU);
@@ -7723,6 +8155,8 @@ static void decode_rrr1_msubad_h(CPUTriCoreState *env, DisasContext *ctx)
         gen_msubadr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
                          cpu_gpr_d[r2], n, MODE_UU);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -7756,6 +8190,7 @@ static void decode_rrrr_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPC2_32_RRRR_EXTR:
     case OPC2_32_RRRR_EXTR_U:
+        CHECK_REG_PAIR(r3);
         tcg_gen_andi_tl(tmp_width, cpu_gpr_d[r3+1], 0x1f);
         tcg_gen_andi_tl(tmp_pos, cpu_gpr_d[r3], 0x1f);
         tcg_gen_add_tl(tmp_pos, tmp_pos, tmp_width);
@@ -7769,11 +8204,14 @@ static void decode_rrrr_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
         }
         break;
     case OPC2_32_RRRR_INSERT:
+        CHECK_REG_PAIR(r3);
         tcg_gen_andi_tl(tmp_width, cpu_gpr_d[r3+1], 0x1f);
         tcg_gen_andi_tl(tmp_pos, cpu_gpr_d[r3], 0x1f);
         gen_insert(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r2], tmp_width,
                    tmp_pos);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(tmp_pos);
     tcg_temp_free(tmp_width);
@@ -7834,6 +8272,8 @@ static void decode_rrrw_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
 
         tcg_temp_free(temp2);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -7905,14 +8345,24 @@ static void decode_sys_interrupts(CPUTriCoreState *env, DisasContext *ctx)
                 (ctx->hflags & TRICORE_HFLAG_KUU) == TRICORE_HFLAG_UM1) {
                 tcg_gen_deposit_tl(cpu_ICR, cpu_ICR, cpu_gpr_d[r1], 8, 1);
             } /* else raise privilege trap */
-        } /* else raise illegal opcode trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_SYS_TRAPSV:
-        /* TODO: raise sticky overflow trap */
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_GE, cpu_PSW_SV, 0, l1);
+        generate_trap(ctx, TRAPC_ASSERT, TIN5_SOVF);
+        gen_set_label(l1);
         break;
     case OPC2_32_SYS_TRAPV:
-        /* TODO: raise overflow trap */
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_GE, cpu_PSW_V, 0, l1);
+        generate_trap(ctx, TRAPC_ASSERT, TIN5_OVF);
+        gen_set_label(l1);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -8119,6 +8569,8 @@ static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
         temp2 = tcg_temp_new(); /* width*/
         temp3 = tcg_temp_new(); /* pos */
 
+        CHECK_REG_PAIR(r3);
+
         tcg_gen_andi_tl(temp2, cpu_gpr_d[r3+1], 0x1f);
         tcg_gen_andi_tl(temp3, cpu_gpr_d[r3], 0x1f);
 
@@ -8207,6 +8659,7 @@ static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
         break;
     case OPCM_32_RRR_DIVIDE:
         decode_rrr_divide(env, ctx);
+        break;
 /* RRR2 Format */
     case OPCM_32_RRR2_MADD:
         decode_rrr2_madd(env, ctx);
@@ -8236,6 +8689,7 @@ static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
 /* RRRR format */
     case OPCM_32_RRRR_EXTRACT_INSERT:
         decode_rrrr_extract_insert(env, ctx);
+        break;
 /* RRRW format */
     case OPCM_32_RRRW_EXTRACT_INSERT:
         decode_rrrw_extract_insert(env, ctx);
@@ -8250,6 +8704,8 @@ static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
         tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
         tcg_gen_mov_tl(cpu_PSW_SAV, cpu_PSW_V);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -8266,43 +8722,44 @@ static void decode_opc(CPUTriCoreState *env, DisasContext *ctx, int *is_branch)
     }
 }
 
-static inline void
-gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
-                              int search_pc)
+void gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
 {
+    TriCoreCPU *cpu = tricore_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUTriCoreState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    int num_insns;
+    int num_insns, max_insns;
 
-    if (search_pc) {
-        qemu_log("search pc %d\n", search_pc);
+    num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
     }
 
-    num_insns = 0;
     pc_start = tb->pc;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.tb = tb;
     ctx.singlestep_enabled = cs->singlestep_enabled;
     ctx.bstate = BS_NONE;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
 
     tcg_clear_temp_count();
     gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
         ctx.opcode = cpu_ldl_code(env, ctx.pc);
         decode_opc(env, &ctx, 0);
 
-        num_insns++;
-
-        if (tcg_op_buf_full()) {
-            gen_save_pc(ctx.next_pc);
-            tcg_gen_exit_tb(0);
-            break;
-        }
-        if (singlestep) {
+        if (num_insns >= max_insns || tcg_op_buf_full()) {
             gen_save_pc(ctx.next_pc);
             tcg_gen_exit_tb(0);
             break;
@@ -8311,12 +8768,9 @@ gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
     }
 
     gen_tb_end(tb, num_insns);
-    if (search_pc) {
-        printf("done_generating search pc\n");
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
     if (tcg_check_temp_count()) {
         printf("LEAK at %08x\n", env->PC);
     }
@@ -8331,21 +8785,10 @@ gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
 }
 
 void
-gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, false);
-}
-
-void
-gen_intermediate_code_pc(CPUTriCoreState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, true);
-}
-
-void
-restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb, int pc_pos)
+restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb,
+                     target_ulong *data)
 {
-    env->PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->PC = data[0];
 }
 /*
  *
@@ -8357,17 +8800,18 @@ void cpu_state_reset(CPUTriCoreState *env)
 {
     /* Reset Regs to Default Value */
     env->PSW = 0xb80;
+    fpu_set_state(env);
 }
 
 static void tricore_tcg_init_csfr(void)
 {
-    cpu_PCXI = tcg_global_mem_new(TCG_AREG0,
+    cpu_PCXI = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PCXI), "PCXI");
-    cpu_PSW = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PSW), "PSW");
-    cpu_PC = tcg_global_mem_new(TCG_AREG0,
+    cpu_PC = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PC), "PC");
-    cpu_ICR = tcg_global_mem_new(TCG_AREG0,
+    cpu_ICR = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, ICR), "ICR");
 }
 
@@ -8381,30 +8825,30 @@ void tricore_tcg_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
     /* reg init */
     for (i = 0 ; i < 16 ; i++) {
-        cpu_gpr_a[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr_a[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUTriCoreState, gpr_a[i]),
                                           regnames_a[i]);
     }
     for (i = 0 ; i < 16 ; i++) {
-        cpu_gpr_d[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr_d[i] = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUTriCoreState, gpr_d[i]),
                                            regnames_d[i]);
     }
     tricore_tcg_init_csfr();
     /* init PSW flag cache */
-    cpu_PSW_C = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_C = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUTriCoreState, PSW_USB_C),
                                    "PSW_C");
-    cpu_PSW_V = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_V = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUTriCoreState, PSW_USB_V),
                                    "PSW_V");
-    cpu_PSW_SV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_SV = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUTriCoreState, PSW_USB_SV),
                                     "PSW_SV");
-    cpu_PSW_AV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_AV = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUTriCoreState, PSW_USB_AV),
                                     "PSW_AV");
-    cpu_PSW_SAV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_SAV = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUTriCoreState, PSW_USB_SAV),
                                      "PSW_SAV");
 }
diff --git a/qemu/target-tricore/tricore-opcodes.h b/qemu/target-tricore/tricore-opcodes.h
index 1bfed0ce4..df666b081 100644
--- a/qemu/target-tricore/tricore-opcodes.h
+++ b/qemu/target-tricore/tricore-opcodes.h
@@ -1126,6 +1126,20 @@ enum {
     OPC2_32_RR_CRC32                             = 0x03,
     OPC2_32_RR_DIV                               = 0x20,
     OPC2_32_RR_DIV_U                             = 0x21,
+    OPC2_32_RR_MUL_F                             = 0x04,
+    OPC2_32_RR_DIV_F                             = 0x05,
+    OPC2_32_RR_FTOI                              = 0x10,
+    OPC2_32_RR_ITOF                              = 0x14,
+    OPC2_32_RR_CMP_F                             = 0x00,
+    OPC2_32_RR_FTOIZ                             = 0x13,
+    OPC2_32_RR_FTOQ31                            = 0x11,
+    OPC2_32_RR_FTOQ31Z                           = 0x18,
+    OPC2_32_RR_FTOU                              = 0x12,
+    OPC2_32_RR_FTOUZ                             = 0x17,
+    OPC2_32_RR_Q31TOF                            = 0x15,
+    OPC2_32_RR_QSEED_F                           = 0x19,
+    OPC2_32_RR_UPDFL                             = 0x0c,
+    OPC2_32_RR_UTOF                              = 0x16,
 };
 /* OPCM_32_RR_IDIRECT                               */
 enum {
@@ -1209,6 +1223,10 @@ enum {
     OPC2_32_RRR_IXMIN                            = 0x08,
     OPC2_32_RRR_IXMIN_U                          = 0x09,
     OPC2_32_RRR_PACK                             = 0x00,
+    OPC2_32_RRR_ADD_F                            = 0x02,
+    OPC2_32_RRR_SUB_F                            = 0x03,
+    OPC2_32_RRR_MADD_F                           = 0x06,
+    OPC2_32_RRR_MSUB_F                           = 0x07,
 };
 /*
  * RRR1 Format
diff --git a/qemu/target-unicore32/cpu.c b/qemu/target-unicore32/cpu.c
index fc451a1a3..66f43acff 100644
--- a/qemu/target-unicore32/cpu.c
+++ b/qemu/target-unicore32/cpu.c
@@ -12,6 +12,8 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -155,6 +157,13 @@ static void uc32_cpu_class_init(ObjectClass *oc, void *data)
     cc->get_phys_page_debug = uc32_cpu_get_phys_page_debug;
 #endif
     dc->vmsd = &vmstate_uc32_cpu;
+
+    /*
+     * Reason: uc32_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void uc32_register_cpu_type(const UniCore32CPUInfo *info)
diff --git a/qemu/target-unicore32/cpu.h b/qemu/target-unicore32/cpu.h
index 45e31e54b..9c1fbf9b7 100644
--- a/qemu/target-unicore32/cpu.h
+++ b/qemu/target-unicore32/cpu.h
@@ -17,11 +17,8 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS     32
 #define TARGET_VIRT_ADDR_SPACE_BITS     32
 
-#define ELF_MACHINE             EM_UNICORE32
-
 #define CPUArchState                struct CPUUniCore32State
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
@@ -131,7 +128,7 @@ int uc32_cpu_signal_handler(int host_signum, void *pinfo, void *puc);
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index(CPUUniCore32State *env)
+static inline int cpu_mmu_index(CPUUniCore32State *env, bool ifetch)
 {
     return (env->uncached_asr & ASR_M) == ASR_MODE_USER ? 1 : 0;
 }
diff --git a/qemu/target-unicore32/helper.c b/qemu/target-unicore32/helper.c
index ae63277c8..21f5f3574 100644
--- a/qemu/target-unicore32/helper.c
+++ b/qemu/target-unicore32/helper.c
@@ -9,6 +9,7 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "exec/helper-proto.h"
diff --git a/qemu/target-unicore32/op_helper.c b/qemu/target-unicore32/op_helper.c
index 0266dbdf7..f5847307a 100644
--- a/qemu/target-unicore32/op_helper.c
+++ b/qemu/target-unicore32/op_helper.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exec/cpu_ldst.h"
diff --git a/qemu/target-unicore32/softmmu.c b/qemu/target-unicore32/softmmu.c
index 9a3786ddd..d267fed87 100644
--- a/qemu/target-unicore32/softmmu.c
+++ b/qemu/target-unicore32/softmmu.c
@@ -12,6 +12,7 @@
 #error This file only exist under softmmu circumstance
 #endif
 
+#include "qemu/osdep.h"
 #include <cpu.h>
 
 #undef DEBUG_UC32
diff --git a/qemu/target-unicore32/translate.c b/qemu/target-unicore32/translate.c
index 2fc78e6f3..39af3af05 100644
--- a/qemu/target-unicore32/translate.c
+++ b/qemu/target-unicore32/translate.c
@@ -8,11 +8,7 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "disas/disas.h"
@@ -24,6 +20,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 /* internal defines */
@@ -51,7 +48,7 @@ typedef struct DisasContext {
    conditional executions state has been updated.  */
 #define DISAS_SYSCALL 5
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv_i32 cpu_R[32];
 
 /* FIXME:  These should be removed.  */
@@ -74,7 +71,7 @@ void uc32_translate_init(void)
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
 
     for (i = 0; i < 32; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                                 offsetof(CPUUniCore32State, regs[i]), regnames[i]);
     }
 }
@@ -1794,10 +1791,6 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
     UniCore32CPU *cpu = uc32_env_get_cpu(env);
     unsigned int insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_ldl_code(env, s->pc);
     s->pc += 4;
 
@@ -1867,17 +1860,12 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
     }
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
-        TranslationBlock *tb, bool search_pc)
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
 {
+    UniCore32CPU *cpu = uc32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUUniCore32State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     uint32_t next_page_start;
     int num_insns;
@@ -1899,12 +1887,14 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
     cpu_F0d = tcg_temp_new_i64();
     cpu_F1d = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
 #ifndef CONFIG_USER_ONLY
     if ((env->uncached_asr & ASR_M) == ASR_MODE_USER) {
@@ -1916,33 +1906,22 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
 
     gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_set_pc_im(dc->pc);
-                    gen_exception(EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2; /* FIXME */
-                    goto done_generating;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_set_pc_im(dc->pc);
+            gen_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            goto done_generating;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -1961,7 +1940,6 @@ static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -2043,26 +2021,8 @@ done_generating:
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -2133,7 +2093,8 @@ void uc32_cpu_dump_state(CPUState *cs, FILE *f,
     cpu_dump_state_ucf64(env, f, cpu_fprintf, flags);
 }
 
-void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->regs[31] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->regs[31] = data[0];
 }
diff --git a/qemu/target-unicore32/ucf64_helper.c b/qemu/target-unicore32/ucf64_helper.c
index 5af008fc6..6c919010c 100644
--- a/qemu/target-unicore32/ucf64_helper.c
+++ b/qemu/target-unicore32/ucf64_helper.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/qemu/target-xtensa/Makefile.objs b/qemu/target-xtensa/Makefile.objs
index 5c150a870..481de9197 100644
--- a/qemu/target-xtensa/Makefile.objs
+++ b/qemu/target-xtensa/Makefile.objs
@@ -2,5 +2,6 @@ obj-y += xtensa-semi.o
 obj-y += core-dc232b.o
 obj-y += core-dc233c.o
 obj-y += core-fsf.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += gdbstub.o
diff --git a/qemu/target-xtensa/core-dc232b.c b/qemu/target-xtensa/core-dc232b.c
index 06826c042..bb8ed4197 100644
--- a/qemu/target-xtensa/core-dc232b.c
+++ b/qemu/target-xtensa/core-dc232b.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
diff --git a/qemu/target-xtensa/core-dc233c.c b/qemu/target-xtensa/core-dc233c.c
index 8daf7d9f8..40475e520 100644
--- a/qemu/target-xtensa/core-dc233c.c
+++ b/qemu/target-xtensa/core-dc233c.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
diff --git a/qemu/target-xtensa/core-fsf.c b/qemu/target-xtensa/core-fsf.c
index f6ea6b944..15ef470e8 100644
--- a/qemu/target-xtensa/core-fsf.c
+++ b/qemu/target-xtensa/core-fsf.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
diff --git a/qemu/target-xtensa/cpu.c b/qemu/target-xtensa/cpu.c
index da8129db5..01b251fdc 100644
--- a/qemu/target-xtensa/cpu.c
+++ b/qemu/target-xtensa/cpu.c
@@ -28,6 +28,8 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
@@ -155,6 +157,13 @@ static void xtensa_cpu_class_init(ObjectClass *oc, void *data)
 #endif
     cc->debug_excp_handler = xtensa_breakpoint_handler;
     dc->vmsd = &vmstate_xtensa_cpu;
+
+    /*
+     * Reason: xtensa_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo xtensa_cpu_type_info = {
diff --git a/qemu/target-xtensa/cpu.h b/qemu/target-xtensa/cpu.h
index 96bfc82e9..d0bd9dada 100644
--- a/qemu/target-xtensa/cpu.h
+++ b/qemu/target-xtensa/cpu.h
@@ -30,11 +30,9 @@
 
 #define ALIGNED_ONLY
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE EM_XTENSA
 
 #define CPUArchState struct CPUXtensaState
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
@@ -65,6 +63,7 @@ enum {
     XTENSA_OPTION_MP_SYNCHRO,
     XTENSA_OPTION_CONDITIONAL_STORE,
     XTENSA_OPTION_ATOMCTL,
+    XTENSA_OPTION_DEPBITS,
 
     /* Interrupts and exceptions */
     XTENSA_OPTION_EXCEPTION,
@@ -383,7 +382,6 @@ typedef struct CPUXtensaState {
 #include "cpu-qom.h"
 
 #define cpu_exec cpu_xtensa_exec
-#define cpu_gen_code cpu_xtensa_gen_code
 #define cpu_signal_handler cpu_xtensa_signal_handler
 #define cpu_list xtensa_cpu_list
 
@@ -492,7 +490,7 @@ static inline uint32_t xtensa_replicate_windowstart(CPUXtensaState *env)
 #define MMU_MODE2_SUFFIX _ring2
 #define MMU_MODE3_SUFFIX _ring3
 
-static inline int cpu_mmu_index(CPUXtensaState *env)
+static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch)
 {
     return xtensa_get_cring(env);
 }
diff --git a/qemu/target-xtensa/gdbstub.c b/qemu/target-xtensa/gdbstub.c
index bc2e1b55f..51d4db083 100644
--- a/qemu/target-xtensa/gdbstub.c
+++ b/qemu/target-xtensa/gdbstub.c
@@ -17,7 +17,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 
@@ -63,8 +63,8 @@ int xtensa_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
         return gdb_get_reg32(mem_buf, env->regs[reg->targno & 0x0f]);
 
     default:
-        qemu_log("%s from reg %d of unsupported type %d\n",
-                 __func__, n, reg->type);
+        qemu_log_mask(LOG_UNIMP, "%s from reg %d of unsupported type %d\n",
+                      __func__, n, reg->type);
         return 0;
     }
 }
@@ -117,8 +117,8 @@ int xtensa_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
         break;
 
     default:
-        qemu_log("%s to reg %d of unsupported type %d\n",
-                 __func__, n, reg->type);
+        qemu_log_mask(LOG_UNIMP, "%s to reg %d of unsupported type %d\n",
+                      __func__, n, reg->type);
         return 0;
     }
 
diff --git a/qemu/target-xtensa/helper.c b/qemu/target-xtensa/helper.c
index 76be50d09..839f4a74a 100644
--- a/qemu/target-xtensa/helper.c
+++ b/qemu/target-xtensa/helper.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
@@ -254,8 +255,8 @@ void xtensa_cpu_do_interrupt(CPUState *cs)
                     env->config->exception_vector[cs->exception_index]);
             env->exception_taken = 1;
         } else {
-            qemu_log("%s(pc = %08x) bad exception_index: %d\n",
-                    __func__, env->pc, cs->exception_index);
+            qemu_log_mask(CPU_LOG_INT, "%s(pc = %08x) bad exception_index: %d\n",
+                          __func__, env->pc, cs->exception_index);
         }
         break;
 
@@ -541,8 +542,8 @@ static int get_physical_addr_mmu(CPUXtensaState *env, bool update_tlb,
             wi = ++env->autorefill_idx & 0x3;
             xtensa_tlb_set_entry(env, dtlb, wi, ei, vpn, pte);
             env->sregs[EXCVADDR] = vaddr;
-            qemu_log("%s: autorefill(%08x): %08x -> %08x\n",
-                    __func__, vaddr, vpn, pte);
+            qemu_log_mask(CPU_LOG_MMU, "%s: autorefill(%08x): %08x -> %08x\n",
+                          __func__, vaddr, vpn, pte);
         } else {
             xtensa_tlb_set_entry_mmu(env, &tmp_entry, dtlb, wi, ei, vpn, pte);
             entry = &tmp_entry;
@@ -590,8 +591,8 @@ static int get_pte(CPUXtensaState *env, uint32_t vaddr, uint32_t *pte)
     int ret = get_physical_addr_mmu(env, false, pt_vaddr, 0, 0,
             &paddr, &page_size, &access, false);
 
-    qemu_log("%s: trying autorefill(%08x) -> %08x\n", __func__,
-            vaddr, ret ? ~0 : paddr);
+    qemu_log_mask(CPU_LOG_MMU, "%s: trying autorefill(%08x) -> %08x\n",
+                  __func__, vaddr, ret ? ~0 : paddr);
 
     if (ret == 0) {
         *pte = ldl_phys(cs->as, paddr);
diff --git a/qemu/target-xtensa/monitor.c b/qemu/target-xtensa/monitor.c
new file mode 100644
index 000000000..f3fa4cd27
--- /dev/null
+++ b/qemu/target-xtensa/monitor.c
@@ -0,0 +1,35 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
diff --git a/qemu/target-xtensa/op_helper.c b/qemu/target-xtensa/op_helper.c
index be657e615..62fa33d8d 100644
--- a/qemu/target-xtensa/op_helper.c
+++ b/qemu/target-xtensa/op_helper.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -57,8 +58,8 @@ void tlb_fill(CPUState *cs,
     int ret = xtensa_get_physical_addr(env, true, vaddr, is_write, mmu_idx,
             &paddr, &page_size, &access);
 
-    qemu_log("%s(%08x, %d, %d) -> %08x, ret = %d\n", __func__,
-            vaddr, is_write, mmu_idx, paddr, ret);
+    qemu_log_mask(CPU_LOG_MMU, "%s(%08x, %d, %d) -> %08x, ret = %d\n",
+                  __func__, vaddr, is_write, mmu_idx, paddr, ret);
 
     if (ret == 0) {
         tlb_set_page(cs,
@@ -245,8 +246,8 @@ void HELPER(entry)(CPUXtensaState *env, uint32_t pc, uint32_t s, uint32_t imm)
 {
     int callinc = (env->sregs[PS] & PS_CALLINC) >> PS_CALLINC_SHIFT;
     if (s > 3 || ((env->sregs[PS] & (PS_WOE | PS_EXCM)) ^ PS_WOE) != 0) {
-        qemu_log("Illegal entry instruction(pc = %08x), PS = %08x\n",
-                pc, env->sregs[PS]);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal entry instruction(pc = %08x), PS = %08x\n",
+                      pc, env->sregs[PS]);
         HELPER(exception_cause)(env, pc, ILLEGAL_INSTRUCTION_CAUSE);
     } else {
         uint32_t windowstart = xtensa_replicate_windowstart(env) >>
@@ -307,9 +308,9 @@ uint32_t HELPER(retw)(CPUXtensaState *env, uint32_t pc)
 
     if (n == 0 || (m != 0 && m != n) ||
             ((env->sregs[PS] & (PS_WOE | PS_EXCM)) ^ PS_WOE) != 0) {
-        qemu_log("Illegal retw instruction(pc = %08x), "
-                "PS = %08x, m = %d, n = %d\n",
-                pc, env->sregs[PS], m, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal retw instruction(pc = %08x), "
+                      "PS = %08x, m = %d, n = %d\n",
+                      pc, env->sregs[PS], m, n);
         HELPER(exception_cause)(env, pc, ILLEGAL_INSTRUCTION_CAUSE);
     } else {
         int owb = windowbase;
@@ -743,8 +744,8 @@ void xtensa_tlb_set_entry(CPUXtensaState *env, bool dtlb,
             xtensa_tlb_set_entry_mmu(env, entry, dtlb, wi, ei, vpn, pte);
             tlb_flush_page(cs, entry->vaddr);
         } else {
-            qemu_log("%s %d, %d, %d trying to set immutable entry\n",
-                    __func__, dtlb, wi, ei);
+            qemu_log_mask(LOG_GUEST_ERROR, "%s %d, %d, %d trying to set immutable entry\n",
+                          __func__, dtlb, wi, ei);
         }
     } else {
         tlb_flush_page(cs, entry->vaddr);
@@ -806,15 +807,15 @@ static void set_dbreak(CPUXtensaState *env, unsigned i, uint32_t dbreaka,
     }
     /* contiguous mask after inversion is one less than some power of 2 */
     if ((~mask + 1) & ~mask) {
-        qemu_log("DBREAKC mask is not contiguous: 0x%08x\n", dbreakc);
+        qemu_log_mask(LOG_GUEST_ERROR, "DBREAKC mask is not contiguous: 0x%08x\n", dbreakc);
         /* cut mask after the first zero bit */
         mask = 0xffffffff << (32 - clo32(mask));
     }
     if (cpu_watchpoint_insert(cs, dbreaka & mask, ~mask + 1,
             flags, &env->cpu_watchpoint[i])) {
         env->cpu_watchpoint[i] = NULL;
-        qemu_log("Failed to set data breakpoint at 0x%08x/%d\n",
-                dbreaka & mask, ~mask + 1);
+        qemu_log_mask(LOG_GUEST_ERROR, "Failed to set data breakpoint at 0x%08x/%d\n",
+                      dbreaka & mask, ~mask + 1);
     }
 }
 
diff --git a/qemu/target-xtensa/overlay_tool.h b/qemu/target-xtensa/overlay_tool.h
index eda03aaca..e8a7fda3d 100644
--- a/qemu/target-xtensa/overlay_tool.h
+++ b/qemu/target-xtensa/overlay_tool.h
@@ -30,6 +30,10 @@
     { .targno = (no), .type = (typ), .group = (grp), .size = (sz) },
 #define XTREG_END { .targno = -1 },
 
+#ifndef XCHAL_HAVE_DEPBITS
+#define XCHAL_HAVE_DEPBITS 0
+#endif
+
 #ifndef XCHAL_HAVE_DIV32
 #define XCHAL_HAVE_DIV32 0
 #endif
@@ -69,6 +73,7 @@
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I && XCHAL_HW_MIN_VERSION >= 230000, \
         XTENSA_OPTION_ATOMCTL) | \
+    XCHAL_OPTION(XCHAL_HAVE_DEPBITS, XTENSA_OPTION_DEPBITS) | \
     /* Interrupts and exceptions */ \
     XCHAL_OPTION(XCHAL_HAVE_EXCEPTIONS, XTENSA_OPTION_EXCEPTION) | \
     XCHAL_OPTION(XCHAL_HAVE_VECBASE, XTENSA_OPTION_RELOCATABLE_VECTOR) | \
diff --git a/qemu/target-xtensa/translate.c b/qemu/target-xtensa/translate.c
index f2118c24c..989448846 100644
--- a/qemu/target-xtensa/translate.c
+++ b/qemu/target-xtensa/translate.c
@@ -28,7 +28,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -43,6 +43,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 typedef struct DisasContext {
@@ -73,7 +74,7 @@ typedef struct DisasContext {
     unsigned cpenable;
 } DisasContext;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv_i32 cpu_pc;
 static TCGv_i32 cpu_R[16];
 static TCGv_i32 cpu_FR[16];
@@ -217,24 +218,24 @@ void xtensa_translate_init(void)
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
             offsetof(CPUXtensaState, pc), "pc");
 
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                 offsetof(CPUXtensaState, regs[i]),
                 regnames[i]);
     }
 
     for (i = 0; i < 16; i++) {
-        cpu_FR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_FR[i] = tcg_global_mem_new_i32(cpu_env,
                 offsetof(CPUXtensaState, fregs[i].f32[FP_F32_LOW]),
                 fregnames[i]);
     }
 
     for (i = 0; i < 256; ++i) {
         if (sregnames[i].name) {
-            cpu_SR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+            cpu_SR[i] = tcg_global_mem_new_i32(cpu_env,
                     offsetof(CPUXtensaState, sregs[i]),
                     sregnames[i].name);
         }
@@ -242,7 +243,7 @@ void xtensa_translate_init(void)
 
     for (i = 0; i < 256; ++i) {
         if (uregnames[i].name) {
-            cpu_UR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+            cpu_UR[i] = tcg_global_mem_new_i32(cpu_env,
                     offsetof(CPUXtensaState, uregs[i]),
                     uregnames[i].name);
         }
@@ -501,9 +502,9 @@ static bool gen_check_sr(DisasContext *dc, uint32_t sr, unsigned access)
 {
     if (!xtensa_option_bits_enabled(dc->config, sregnames[sr].opt_bits)) {
         if (sregnames[sr].name) {
-            qemu_log("SR %s is not configured\n", sregnames[sr].name);
+            qemu_log_mask(LOG_GUEST_ERROR, "SR %s is not configured\n", sregnames[sr].name);
         } else {
-            qemu_log("SR %d is not implemented\n", sr);
+            qemu_log_mask(LOG_UNIMP, "SR %d is not implemented\n", sr);
         }
         gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
         return false;
@@ -514,8 +515,8 @@ static bool gen_check_sr(DisasContext *dc, uint32_t sr, unsigned access)
             [SR_X] = "xsr",
         };
         assert(access < ARRAY_SIZE(access_text) && access_text[access]);
-        qemu_log("SR %s is not available for %s\n", sregnames[sr].name,
-                access_text[access]);
+        qemu_log_mask(LOG_GUEST_ERROR, "SR %s is not available for %s\n", sregnames[sr].name,
+                      access_text[access]);
         gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
         return false;
     }
@@ -875,18 +876,18 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 {
 #define HAS_OPTION_BITS(opt) do { \
         if (!option_bits_enabled(dc, opt)) { \
-            qemu_log("Option is not enabled %s:%d\n", \
-                    __FILE__, __LINE__); \
+            qemu_log_mask(LOG_GUEST_ERROR, "Option is not enabled %s:%d\n", \
+                          __FILE__, __LINE__); \
             goto invalid_opcode; \
         } \
     } while (0)
 
 #define HAS_OPTION(opt) HAS_OPTION_BITS(XTENSA_OPTION_BIT(opt))
 
-#define TBD() qemu_log("TBD(pc = %08x): %s:%d\n", dc->pc, __FILE__, __LINE__)
+#define TBD() qemu_log_mask(LOG_UNIMP, "TBD(pc = %08x): %s:%d\n", dc->pc, __FILE__, __LINE__)
 #define RESERVED() do { \
-        qemu_log("RESERVED(pc = %08x, %02x%02x%02x): %s:%d\n", \
-                dc->pc, b0, b1, b2, __FILE__, __LINE__); \
+        qemu_log_mask(LOG_GUEST_ERROR, "RESERVED(pc = %08x, %02x%02x%02x): %s:%d\n", \
+                      dc->pc, b0, b1, b2, __FILE__, __LINE__); \
         goto invalid_opcode; \
     } while (0)
 
@@ -1186,7 +1187,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                                 gen_jump(dc, cpu_SR[EPC1 + RRR_S - 1]);
                             }
                         } else {
-                            qemu_log("RFI %d is illegal\n", RRR_S);
+                            qemu_log_mask(LOG_GUEST_ERROR, "RFI %d is illegal\n", RRR_S);
                             gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
                         }
                         break;
@@ -1222,7 +1223,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                                 gen_helper_simcall(cpu_env);
                             }
                         } else {
-                            qemu_log("SIMCALL but semihosting is disabled\n");
+                            qemu_log_mask(LOG_GUEST_ERROR, "SIMCALL but semihosting is disabled\n");
                             gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
                         }
                         break;
@@ -1544,7 +1545,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                     TCGv_i64 tmp = tcg_temp_new_i64(); \
                     tcg_gen_extu_i32_i64(tmp, reg); \
                     tcg_gen_##cmd##_i64(v, v, tmp); \
-                    tcg_gen_trunc_i64_i32(cpu_R[RRR_R], v); \
+                    tcg_gen_extrl_i64_i32(cpu_R[RRR_R], v); \
                     tcg_temp_free_i64(v); \
                     tcg_temp_free_i64(tmp); \
                 } while (0)
@@ -1865,7 +1866,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                     if (uregnames[st].name) {
                         tcg_gen_mov_i32(cpu_R[RRR_R], cpu_UR[st]);
                     } else {
-                        qemu_log("RUR %d not implemented, ", st);
+                        qemu_log_mask(LOG_UNIMP, "RUR %d not implemented, ", st);
                         TBD();
                     }
                 }
@@ -1876,7 +1877,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                     if (uregnames[RSR_SR].name) {
                         gen_wur(RSR_SR, cpu_R[RRR_T]);
                     } else {
-                        qemu_log("WUR %d not implemented, ", RSR_SR);
+                        qemu_log_mask(LOG_UNIMP, "WUR %d not implemented, ", RSR_SR);
                         TBD();
                     }
                 }
@@ -1943,7 +1944,8 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             switch (OP2) {
             case 0: /*L32E*/
                 HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                if (gen_check_privilege(dc)) {
+                if (gen_check_privilege(dc) &&
+                    gen_window_check2(dc, RRR_S, RRR_T)) {
                     TCGv_i32 addr = tcg_temp_new_i32();
                     tcg_gen_addi_i32(addr, cpu_R[RRR_S],
                             (0xffffffc0 | (RRR_R << 2)));
@@ -1954,7 +1956,8 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 
             case 4: /*S32E*/
                 HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                if (gen_check_privilege(dc)) {
+                if (gen_check_privilege(dc) &&
+                    gen_window_check2(dc, RRR_S, RRR_T)) {
                     TCGv_i32 addr = tcg_temp_new_i32();
                     tcg_gen_addi_i32(addr, cpu_R[RRR_S],
                             (0xffffffc0 | (RRR_R << 2)));
@@ -1963,6 +1966,17 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
                 }
                 break;
 
+            case 5: /*S32N*/
+                if (gen_window_check2(dc, RRI4_S, RRI4_T)) {
+                    TCGv_i32 addr = tcg_temp_new_i32();
+
+                    tcg_gen_addi_i32(addr, cpu_R[RRI4_S], RRI4_IMM4 << 2);
+                    gen_load_store_alignment(dc, 2, addr, false);
+                    tcg_gen_qemu_st32(cpu_R[RRI4_T], addr, dc->cring);
+                    tcg_temp_free(addr);
+                }
+                break;
+
             default:
                 RESERVED();
                 break;
@@ -1970,6 +1984,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             break;
 
         case 10: /*FP0*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
             switch (OP2) {
             case 0: /*ADD.Sf*/
@@ -2104,6 +2128,16 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
             break;
 
         case 11: /*FP1*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2 + 16, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
 
 #define gen_compare(rel, br, a, b) \
@@ -2973,7 +3007,7 @@ static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
     return;
 
 invalid_opcode:
-    qemu_log("INVALID(pc = %08x)\n", dc->pc);
+    qemu_log_mask(LOG_GUEST_ERROR, "INVALID(pc = %08x)\n", dc->pc);
     gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
 #undef HAS_OPTION
 }
@@ -2984,22 +3018,6 @@ static inline unsigned xtensa_insn_len(CPUXtensaState *env, DisasContext *dc)
     return xtensa_op0_insn_len(OP0);
 }
 
-static void check_breakpoint(CPUXtensaState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(xtensa_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_i32(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 static void gen_ibreak_check(CPUXtensaState *env, DisasContext *dc)
 {
     unsigned i;
@@ -3013,15 +3031,12 @@ static void gen_ibreak_check(CPUXtensaState *env, DisasContext *dc)
     }
 }
 
-static inline
-void gen_intermediate_code_internal(XtensaCPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
 {
+    XtensaCPU *cpu = xtensa_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUXtensaState *env = &cpu->env;
     DisasContext dc;
     int insn_count = 0;
-    int j, lj = -1;
     int max_insns = tb->cflags & CF_COUNT_MASK;
     uint32_t pc_start = tb->pc;
     uint32_t next_page_start =
@@ -3030,6 +3045,9 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     dc.config = env->config;
     dc.singlestep_enabled = cs->singlestep_enabled;
@@ -3062,28 +3080,24 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
     }
 
     do {
-        check_breakpoint(env, &dc);
+        tcg_gen_insn_start(dc.pc);
+        ++insn_count;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = insn_count;
-        }
+        ++dc.ccount_delta;
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, dc.pc);
+            gen_exception(&dc, EXCP_DEBUG);
+            dc.is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
         }
 
-        ++dc.ccount_delta;
-
-        if (insn_count + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (insn_count == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -3104,7 +3118,6 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
         }
 
         disas_xtensa_insn(env, &dc);
-        ++insn_count;
         if (dc.icount) {
             tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount);
         }
@@ -3142,24 +3155,8 @@ void gen_intermediate_code_internal(XtensaCPU *cpu,
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
-                (j - lj) * sizeof(tcg_ctx.gen_opc_instr_start[0]));
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = insn_count;
-    }
-}
-
-void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, true);
+    tb->size = dc.pc - pc_start;
+    tb->icount = insn_count;
 }
 
 void xtensa_cpu_dump_state(CPUState *cs, FILE *f,
@@ -3213,7 +3210,8 @@ void xtensa_cpu_dump_state(CPUState *cs, FILE *f,
     }
 }
 
-void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/qemu/target-xtensa/xtensa-semi.c b/qemu/target-xtensa/xtensa-semi.c
index 16e9d8c7b..370e365c6 100644
--- a/qemu/target-xtensa/xtensa-semi.c
+++ b/qemu/target-xtensa/xtensa-semi.c
@@ -25,10 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <stddef.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/log.h"
@@ -313,7 +310,7 @@ void HELPER(simcall)(CPUXtensaState *env)
         break;
 
     default:
-        qemu_log("%s(%d): not implemented\n", __func__, regs[2]);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s(%d): not implemented\n", __func__, regs[2]);
         regs[2] = -1;
         regs[3] = TARGET_ENOSYS;
         break;
diff --git a/qemu/tcg-runtime.c b/qemu/tcg-runtime.c
index 9daba6945..ea2ad649c 100644
--- a/qemu/tcg-runtime.c
+++ b/qemu/tcg-runtime.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include "qemu/host-utils.h"
 
 /* This file is compiled once, and thus we can't include the standard
diff --git a/qemu/tcg/README b/qemu/tcg/README
index a550ff176..f4a8ac170 100644
--- a/qemu/tcg/README
+++ b/qemu/tcg/README
@@ -314,11 +314,17 @@ This operation would be equivalent to
 
   dest = (t1 & ~0x0f00) | ((t2 << 8) & 0x0f00)
 
-* trunc_shr_i32 t0, t1, pos
+* extrl_i64_i32 t0, t1
 
-For 64-bit hosts only, right shift the 64-bit input T1 by POS and
-truncate to 32-bit output T0.  Depending on the host, this may be
-a simple mov/shift, or may require additional canonicalization.
+For 64-bit hosts only, extract the low 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple move,
+or may require additional canonicalization.
+
+* extrh_i64_i32 t0, t1
+
+For 64-bit hosts only, extract the high 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple shift,
+or may require additional canonicalization.
 
 ********* Conditional moves
 
@@ -454,8 +460,9 @@ function tcg_gen_xxx(args).
 
 4) Backend
 
-tcg-target.h contains the target specific definitions. tcg-target.c
-contains the target specific code.
+tcg-target.h contains the target specific definitions. tcg-target.inc.c
+contains the target specific code; it is #included by tcg/tcg.c, rather
+than being a standalone C file.
 
 4.1) Assumptions
 
@@ -466,13 +473,25 @@ On a 32 bit target, all 64 bit operations are converted to 32 bits. A
 few specific operations must be implemented to allow it (see add2_i32,
 sub2_i32, brcond2_i32).
 
+On a 64 bit target, the values are transfered between 32 and 64-bit
+registers using the following ops:
+- trunc_shr_i64_i32
+- ext_i32_i64
+- extu_i32_i64
+
+They ensure that the values are correctly truncated or extended when
+moved from a 32-bit to a 64-bit register or vice-versa. Note that the
+trunc_shr_i64_i32 is an optional op. It is not necessary to implement
+it if all the following conditions are met:
+- 64-bit registers can hold 32-bit values
+- 32-bit values in a 64-bit register do not need to stay zero or
+  sign extended
+- all 32-bit TCG ops ignore the high part of 64-bit registers
+
 Floating point operations are not supported in this version. A
 previous incarnation of the code generator had full support of them,
 but it is better to concentrate on integer operations first.
 
-On a 64 bit target, no assumption is made in TCG about the storage of
-the 32 bit values in 64 bit registers.
-
 4.2) Constraints
 
 GCC like constraints are used to define the constraints of every
diff --git a/qemu/tcg/aarch64/tcg-target.h b/qemu/tcg/aarch64/tcg-target.h
index 8aec04d2b..19a04a6e7 100644
--- a/qemu/tcg/aarch64/tcg-target.h
+++ b/qemu/tcg/aarch64/tcg-target.h
@@ -70,7 +70,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          1
diff --git a/qemu/tcg/aarch64/tcg-target.c b/qemu/tcg/aarch64/tcg-target.inc.c
index b7ec4f5ac..a8fb4420d 100644
--- a/qemu/tcg/aarch64/tcg-target.c
+++ b/qemu/tcg/aarch64/tcg-target.inc.c
@@ -18,19 +18,19 @@
    makes things much cleaner.  */
 QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%x0", "%x1", "%x2", "%x3", "%x4", "%x5", "%x6", "%x7",
     "%x8", "%x9", "%x10", "%x11", "%x12", "%x13", "%x14", "%x15",
     "%x16", "%x17", "%x18", "%x19", "%x20", "%x21", "%x22", "%x23",
     "%x24", "%x25", "%x26", "%x27", "%x28", "%fp", "%x30", "%sp",
 };
-#endif /* NDEBUG */
+#endif /* CONFIG_DEBUG_TCG */
 
 static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
     TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
-    TCG_REG_X28, /* we will reserve this for GUEST_BASE if configured */
+    TCG_REG_X28, /* we will reserve this for guest_base if configured */
 
     TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
     TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
@@ -56,17 +56,18 @@ static const int tcg_target_call_oarg_regs[1] = {
 #define TCG_REG_TMP TCG_REG_X30
 
 #ifndef CONFIG_SOFTMMU
-# ifdef CONFIG_USE_GUEST_BASE
-#  define TCG_REG_GUEST_BASE TCG_REG_X28
-# else
-#  define TCG_REG_GUEST_BASE TCG_REG_XZR
-# endif
+/* Note that XZR cannot be encoded in the address base register slot,
+   as that actaully encodes SP.  So if we need to zero-extend the guest
+   address, via the address index register slot, we need to load even
+   a zero guest base into a register.  */
+#define USE_GUEST_BASE     (guest_base != 0 || TARGET_LONG_BITS == 32)
+#define TCG_REG_GUEST_BASE TCG_REG_X28
 #endif
 
 static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
 {
     ptrdiff_t offset = target - code_ptr;
-    assert(offset == sextract64(offset, 0, 26));
+    tcg_debug_assert(offset == sextract64(offset, 0, 26));
     /* read instruction, mask away previous PC_REL26 parameter contents,
        set the proper offset, then write back the instruction. */
     *code_ptr = deposit32(*code_ptr, 0, 26, offset);
@@ -75,14 +76,14 @@ static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
 static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
 {
     ptrdiff_t offset = target - code_ptr;
-    assert(offset == sextract64(offset, 0, 19));
+    tcg_debug_assert(offset == sextract64(offset, 0, 19));
     *code_ptr = deposit32(*code_ptr, 5, 19, offset);
 }
 
 static inline void patch_reloc(tcg_insn_unit *code_ptr, int type,
                                intptr_t value, intptr_t addend)
 {
-    assert(addend == 0);
+    tcg_debug_assert(addend == 0);
     switch (type) {
     case R_AARCH64_JUMP26:
     case R_AARCH64_CALL26:
@@ -401,7 +402,7 @@ static void tcg_out_insn_3314(TCGContext *s, AArch64Insn insn,
     insn |= pre << 24;
     insn |= w << 23;
 
-    assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
+    tcg_debug_assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
     insn |= (ofs & (0x7f << 3)) << (15 - 3);
 
     tcg_out32(s, insn | r2 << 10 | rn << 5 | r1);
@@ -411,9 +412,9 @@ static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
                               TCGReg rd, TCGReg rn, uint64_t aimm)
 {
     if (aimm > 0xfff) {
-        assert((aimm & 0xfff) == 0);
+        tcg_debug_assert((aimm & 0xfff) == 0);
         aimm >>= 12;
-        assert(aimm <= 0xfff);
+        tcg_debug_assert(aimm <= 0xfff);
         aimm |= 1 << 12;  /* apply LSL 12 */
     }
     tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
@@ -443,7 +444,7 @@ static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
 static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
                               TCGReg rd, uint16_t half, unsigned shift)
 {
-    assert((shift & ~0x30) == 0);
+    tcg_debug_assert((shift & ~0x30) == 0);
     tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
 }
 
@@ -537,7 +538,7 @@ static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
 {
     unsigned h, l, r, c;
 
-    assert(is_limm(limm));
+    tcg_debug_assert(is_limm(limm));
 
     h = clz64(limm);
     l = ctz64(limm);
@@ -792,7 +793,7 @@ static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
 static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
 {
     ptrdiff_t offset = target - s->code_ptr;
-    assert(offset == sextract64(offset, 0, 26));
+    tcg_debug_assert(offset == sextract64(offset, 0, 26));
     tcg_out_insn(s, 3206, B, offset);
 }
 
@@ -866,7 +867,7 @@ static void tcg_out_brcond(TCGContext *s, TCGMemOp ext, TCGCond c, TCGArg a,
         offset = tcg_in32(s) >> 5;
     } else {
         offset = l->u.value_ptr - s->code_ptr;
-        assert(offset == sextract64(offset, 0, 19));
+        tcg_debug_assert(offset == sextract64(offset, 0, 19));
     }
 
     if (need_cmp) {
@@ -989,7 +990,7 @@ static void * const qemu_st_helpers[16] = {
 static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
 {
     ptrdiff_t offset = tcg_pcrel_diff(s, target);
-    assert(offset == sextract64(offset, 0, 21));
+    tcg_debug_assert(offset == sextract64(offset, 0, 21));
     tcg_out_insn(s, 3406, ADR, rd, offset);
 }
 
@@ -1051,14 +1052,29 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
    slow path for the failure case, which will be patched later when finalizing
    the slow path. Generated code returns the host addend in X1,
    clobbers X0,X2,X3,TMP. */
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp s_bits,
+static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
                              tcg_insn_unit **label_ptr, int mem_index,
                              bool is_read)
 {
-    TCGReg base = TCG_AREG0;
     int tlb_offset = is_read ?
         offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    TCGReg base = TCG_AREG0, x3;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if ((opc & MO_AMASK) == MO_ALIGN || s_mask == 0) {
+        tlb_mask = TARGET_PAGE_MASK | s_mask;
+        x3 = addr_reg;
+    } else {
+        tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, addr_reg, s_mask);
+        tlb_mask = TARGET_PAGE_MASK;
+        x3 = TCG_REG_X3;
+    }
 
     /* Extract the TLB index from the address into X0.
        X0<CPU_TLB_BITS:0> =
@@ -1066,11 +1082,9 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp s_bits,
     tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
                  TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
 
-    /* Store the page mask part of the address and the low s_bits into X3.
-       Later this allows checking for equality and alignment at the same time.
-       X3 = addr_reg & (PAGE_MASK | ((1 << s_bits) - 1)) */
-    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64, TCG_REG_X3,
-                     addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+    /* Store the page mask part of the address into X3.  */
+    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, x3, tlb_mask);
 
     /* Add any "high bits" from the tlb offset to the env address into X2,
        to take advantage of the LSL12 form of the ADDI instruction.
@@ -1207,18 +1221,21 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
     const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
 #ifdef CONFIG_SOFTMMU
     unsigned mem_index = get_mmuidx(oi);
-    TCGMemOp s_bits = memop & MO_SIZE;
     tcg_insn_unit *label_ptr;
 
-    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
     tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
                            TCG_REG_X1, otype, addr_reg);
     add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
                         s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
-                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR,
-                           otype, addr_reg);
+    if (USE_GUEST_BASE) {
+        tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
+                               TCG_REG_GUEST_BASE, otype, addr_reg);
+    } else {
+        tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
+                               addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
+    }
 #endif /* CONFIG_SOFTMMU */
 }
 
@@ -1229,18 +1246,21 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
     const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
 #ifdef CONFIG_SOFTMMU
     unsigned mem_index = get_mmuidx(oi);
-    TCGMemOp s_bits = memop & MO_SIZE;
     tcg_insn_unit *label_ptr;
 
-    tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
     tcg_out_qemu_st_direct(s, memop, data_reg,
                            TCG_REG_X1, otype, addr_reg);
-    add_qemu_ldst_label(s, false, oi, s_bits == MO_64, data_reg, addr_reg,
-                        s->code_ptr, label_ptr);
+    add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
+                        data_reg, addr_reg, s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    tcg_out_qemu_st_direct(s, memop, data_reg,
-                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR,
-                           otype, addr_reg);
+    if (USE_GUEST_BASE) {
+        tcg_out_qemu_st_direct(s, memop, data_reg,
+                               TCG_REG_GUEST_BASE, otype, addr_reg);
+    } else {
+        tcg_out_qemu_st_direct(s, memop, data_reg,
+                               addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
+    }
 #endif /* CONFIG_SOFTMMU */
 }
 
@@ -1274,7 +1294,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
 #ifndef USE_DIRECT_JUMP
 #error "USE_DIRECT_JUMP required for aarch64"
 #endif
-        assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
+        tcg_debug_assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
         s->tb_jmp_offset[a0] = tcg_current_code_size(s);
         /* actual branch destination will be patched by
            aarch64_tb_set_jmp_target later, beware retranslation. */
@@ -1556,6 +1576,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i32:
         tcg_out_sxt(s, ext, MO_16, a0, a1);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
         break;
@@ -1567,6 +1588,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i32:
         tcg_out_uxt(s, MO_16, a0, a1);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
         break;
@@ -1712,6 +1734,8 @@ static const TCGTargetOpDef aarch64_op_defs[] = {
     { INDEX_op_ext8u_i64, { "r", "r" } },
     { INDEX_op_ext16u_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
 
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
     { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
@@ -1794,9 +1818,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
 
-#if defined(CONFIG_USE_GUEST_BASE)
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, GUEST_BASE);
+#if !defined(CONFIG_SOFTMMU)
+    if (USE_GUEST_BASE) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
     }
 #endif
diff --git a/qemu/tcg/arm/tcg-target.c b/qemu/tcg/arm/tcg-target.inc.c
index ae2ec7a92..2b7fbddbf 100644
--- a/qemu/tcg/arm/tcg-target.c
+++ b/qemu/tcg/arm/tcg-target.inc.c
@@ -67,7 +67,7 @@ bool use_idiv_instructions;
 # define USING_SOFTMMU 0
 #endif
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%r0",
     "%r1",
@@ -124,8 +124,8 @@ static inline void reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
                         intptr_t value, intptr_t addend)
 {
-    assert(type == R_ARM_PC24);
-    assert(addend == 0);
+    tcg_debug_assert(type == R_ARM_PC24);
+    tcg_debug_assert(addend == 0);
     reloc_pc24(code_ptr, (tcg_insn_unit *)value);
 }
 
@@ -492,7 +492,7 @@ static inline void tcg_out_dat_rI(TCGContext *s, int cond, int opc, TCGArg dst,
      */
     if (rhs_is_const) {
         int rot = encode_imm(rhs);
-        assert(rot >= 0);
+        tcg_debug_assert(rot >= 0);
         tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
     } else {
         tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
@@ -511,7 +511,7 @@ static void tcg_out_dat_rIK(TCGContext *s, int cond, int opc, int opinv,
         if (rot < 0) {
             rhs = ~rhs;
             rot = encode_imm(rhs);
-            assert(rot >= 0);
+            tcg_debug_assert(rot >= 0);
             opc = opinv;
         }
         tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
@@ -532,7 +532,7 @@ static void tcg_out_dat_rIN(TCGContext *s, int cond, int opc, int opneg,
         if (rot < 0) {
             rhs = -rhs;
             rot = encode_imm(rhs);
-            assert(rot >= 0);
+            tcg_debug_assert(rot >= 0);
             opc = opneg;
         }
         tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
@@ -1100,7 +1100,7 @@ static TCGReg NAME(TCGContext *s, TCGReg argreg, ARGTYPE arg)              \
     } else {                                                               \
         int ofs = (argreg - 4) * 4;                                        \
         EXT_ARG;                                                           \
-        assert(ofs + 4 <= TCG_STATIC_CALL_ARGS_SIZE);                      \
+        tcg_debug_assert(ofs + 4 <= TCG_STATIC_CALL_ARGS_SIZE);            \
         tcg_out_st32_12(s, COND_AL, arg, TCG_REG_CALL_STACK, ofs);         \
     }                                                                      \
     return argreg + 1;                                                     \
@@ -1493,8 +1493,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
     add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
                         s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
         tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP);
     } else {
         tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo);
@@ -1623,8 +1623,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
     add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
                         s->code_ptr, label_ptr);
 #else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
         tcg_out_qemu_st_index(s, COND_AL, opc, datalo,
                               datahi, addrlo, TCG_REG_TMP);
     } else {
diff --git a/qemu/tcg/i386/tcg-target.h b/qemu/tcg/i386/tcg-target.h
index 25b513354..92be34171 100644
--- a/qemu/tcg/i386/tcg-target.h
+++ b/qemu/tcg/i386/tcg-target.h
@@ -102,7 +102,8 @@ extern bool have_bmi1;
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
 #define TCG_TARGET_HAS_ext8s_i64        1
diff --git a/qemu/tcg/i386/tcg-target.c b/qemu/tcg/i386/tcg-target.inc.c
index 887f22f67..007407c3f 100644
--- a/qemu/tcg/i386/tcg-target.c
+++ b/qemu/tcg/i386/tcg-target.inc.c
@@ -24,7 +24,7 @@
 
 #include "tcg-be-ldst.h"
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #if TCG_TARGET_REG_BITS == 64
     "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
@@ -425,7 +425,7 @@ static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
     }
     if (opc & P_DATA16) {
         /* We should never be asking for both 16 and 64-bit operation.  */
-        assert((opc & P_REXW) == 0);
+        tcg_debug_assert((opc & P_REXW) == 0);
         tcg_out8(s, 0x66);
     }
     if (opc & P_ADDR32) {
@@ -599,7 +599,7 @@ static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
         if (index < 0) {
             index = 4;
         } else {
-            assert(index != TCG_REG_ESP);
+            tcg_debug_assert(index != TCG_REG_ESP);
         }
 
         tcg_out_opc(s, opc, r, rm, index);
@@ -745,14 +745,14 @@ static inline void tcg_out_rolw_8(TCGContext *s, int reg)
 static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
 {
     /* movzbl */
-    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
     tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
 }
 
 static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
 {
     /* movsbl */
-    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
     tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
 }
 
@@ -1172,14 +1172,16 @@ static void * const qemu_st_helpers[16] = {
    First argument register is clobbered.  */
 
 static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
-                                    int mem_index, TCGMemOp s_bits,
+                                    int mem_index, TCGMemOp opc,
                                     tcg_insn_unit **label_ptr, int which)
 {
     const TCGReg r0 = TCG_REG_L0;
     const TCGReg r1 = TCG_REG_L1;
     TCGType ttype = TCG_TYPE_I32;
-    TCGType htype = TCG_TYPE_I32;
-    int trexw = 0, hrexw = 0;
+    TCGType tlbtype = TCG_TYPE_I32;
+    int trexw = 0, hrexw = 0, tlbrexw = 0;
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    bool aligned = (opc & MO_AMASK) == MO_ALIGN || s_mask == 0;
 
     if (TCG_TARGET_REG_BITS == 64) {
         if (TARGET_LONG_BITS == 64) {
@@ -1187,20 +1189,29 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
             trexw = P_REXW;
         }
         if (TCG_TYPE_PTR == TCG_TYPE_I64) {
-            htype = TCG_TYPE_I64;
             hrexw = P_REXW;
+            if (TARGET_PAGE_BITS + CPU_TLB_BITS > 32) {
+                tlbtype = TCG_TYPE_I64;
+                tlbrexw = P_REXW;
+            }
         }
     }
 
-    tcg_out_mov(s, htype, r0, addrlo);
-    tcg_out_mov(s, ttype, r1, addrlo);
+    tcg_out_mov(s, tlbtype, r0, addrlo);
+    if (aligned) {
+        tcg_out_mov(s, ttype, r1, addrlo);
+    } else {
+        /* For unaligned access check that we don't cross pages using
+           the page address of the last byte.  */
+        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask);
+    }
 
-    tcg_out_shifti(s, SHIFT_SHR + hrexw, r0,
+    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
                    TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
 
     tgen_arithi(s, ARITH_AND + trexw, r1,
-                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
-    tgen_arithi(s, ARITH_AND + hrexw, r0,
+                TARGET_PAGE_MASK | (aligned ? s_mask : 0), 0);
+    tgen_arithi(s, ARITH_AND + tlbrexw, r0,
                 (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
 
     tcg_out_modrm_sib_offset(s, OPC_LEA + hrexw, r0, TCG_AREG0, r0, 0,
@@ -1424,7 +1435,7 @@ int arch_prctl(int code, unsigned long addr);
 static int guest_base_flags;
 static inline void setup_guest_base_seg(void)
 {
-    if (arch_prctl(ARCH_SET_GS, GUEST_BASE) == 0) {
+    if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
         guest_base_flags = P_GS;
     }
 }
@@ -1545,7 +1556,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     int mem_index;
-    TCGMemOp s_bits;
     tcg_insn_unit *label_ptr[2];
 #endif
 
@@ -1558,9 +1568,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
 
 #if defined(CONFIG_SOFTMMU)
     mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
                      label_ptr, offsetof(CPUTLBEntry, addr_read));
 
     /* TLB Hit.  */
@@ -1571,7 +1580,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
                         s->code_ptr, label_ptr);
 #else
     {
-        int32_t offset = GUEST_BASE;
+        int32_t offset = guest_base;
         TCGReg base = addrlo;
         int index = -1;
         int seg = 0;
@@ -1580,7 +1589,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
            We can do this with the ADDR32 prefix if we're not using
            a guest base, or when using segmentation.  Otherwise we
            need to zero-extend manually.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
+        if (guest_base == 0 || guest_base_flags) {
             seg = guest_base_flags;
             offset = 0;
             if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
@@ -1591,8 +1600,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
                 tcg_out_ext32u(s, TCG_REG_L0, base);
                 base = TCG_REG_L0;
             }
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            if (offset != guest_base) {
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
                 index = TCG_REG_L1;
                 offset = 0;
             }
@@ -1687,7 +1696,6 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     int mem_index;
-    TCGMemOp s_bits;
     tcg_insn_unit *label_ptr[2];
 #endif
 
@@ -1700,9 +1708,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
 
 #if defined(CONFIG_SOFTMMU)
     mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
 
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, s_bits,
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
                      label_ptr, offsetof(CPUTLBEntry, addr_write));
 
     /* TLB Hit.  */
@@ -1713,12 +1720,12 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
                         s->code_ptr, label_ptr);
 #else
     {
-        int32_t offset = GUEST_BASE;
+        int32_t offset = guest_base;
         TCGReg base = addrlo;
         int seg = 0;
 
         /* See comment in tcg_out_qemu_ld re zero-extension of addrlo.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
+        if (guest_base == 0 || guest_base_flags) {
             seg = guest_base_flags;
             offset = 0;
             if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
@@ -1727,12 +1734,12 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
         } else if (TCG_TARGET_REG_BITS == 64) {
             /* ??? Note that we can't use the same SIB addressing scheme
                as for loads, since we require L0 free for bswap.  */
-            if (offset != GUEST_BASE) {
+            if (offset != guest_base) {
                 if (TARGET_LONG_BITS == 32) {
                     tcg_out_ext32u(s, TCG_REG_L0, base);
                     base = TCG_REG_L0;
                 }
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
                 tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
                 base = TCG_REG_L1;
                 offset = 0;
@@ -2064,9 +2071,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, args[0]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_ext32u(s, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_ext32s(s, args[0], args[1]);
         break;
@@ -2201,6 +2210,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ext16u_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
     { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
     { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
 
@@ -2306,8 +2318,8 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_out_opc(s, OPC_RET, 0, 0, 0);
 
 #if !defined(CONFIG_SOFTMMU)
-    /* Try to set up a segment register to point to GUEST_BASE.  */
-    if (GUEST_BASE) {
+    /* Try to set up a segment register to point to guest_base.  */
+    if (guest_base) {
         setup_guest_base_seg();
     }
 #endif
diff --git a/qemu/tcg/ia64/tcg-target.h b/qemu/tcg/ia64/tcg-target.h
index a04ed8126..ae9b79f02 100644
--- a/qemu/tcg/ia64/tcg-target.h
+++ b/qemu/tcg/ia64/tcg-target.h
@@ -160,7 +160,8 @@ typedef enum {
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_mulsh_i64        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
diff --git a/qemu/tcg/ia64/tcg-target.c b/qemu/tcg/ia64/tcg-target.inc.c
index 81cb9f79f..7557e6a9d 100644
--- a/qemu/tcg/ia64/tcg-target.c
+++ b/qemu/tcg/ia64/tcg-target.inc.c
@@ -27,7 +27,7 @@
  * Register definitions
  */
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
      "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
      "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
@@ -40,13 +40,8 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 };
 #endif
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG TCG_REG_R55
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
 #endif
 
 /* Branch registers */
@@ -715,8 +710,8 @@ static uint64_t get_reloc_pcrel21b_slot2(tcg_insn_unit *pc)
 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
                         intptr_t value, intptr_t addend)
 {
-    assert(addend == 0);
-    assert(type == R_IA64_PCREL21B);
+    tcg_debug_assert(addend == 0);
+    tcg_debug_assert(type == R_IA64_PCREL21B);
     reloc_pcrel21b_slot2(code_ptr, (tcg_insn_unit *)value);
 }
 
@@ -814,7 +809,7 @@ static inline void tcg_out_mov(TCGContext *s, TCGType type,
 
 static inline uint64_t tcg_opc_movi_a(int qp, TCGReg dst, int64_t src)
 {
-    assert(src == sextract64(src, 0, 22));
+    tcg_debug_assert(src == sextract64(src, 0, 22));
     return tcg_opc_a5(qp, OPC_ADDL_A5, dst, src, TCG_REG_R0);
 }
 
@@ -1577,7 +1572,7 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
     be->labels = l;
 }
 
-static void tcg_out_tb_finalize(TCGContext *s)
+static bool tcg_out_tb_finalize(TCGContext *s)
 {
     static const void * const helpers[8] = {
         helper_ret_stb_mmu,
@@ -1599,7 +1594,7 @@ static void tcg_out_tb_finalize(TCGContext *s)
         /* The out-of-line thunks are all the same; load the return address
            from B0, load the GP, and branch to the code.  Note that we are
            always post-call, so the register window has rolled, so we're
-           using incomming parameter register numbers, not outgoing.  */
+           using incoming parameter register numbers, not outgoing.  */
         if (dest == NULL) {
             uintptr_t *desc = (uintptr_t *)helpers[x];
             uintptr_t func = desc[0], gp = desc[1], disp;
@@ -1625,7 +1620,16 @@ static void tcg_out_tb_finalize(TCGContext *s)
         }
 
         reloc_pcrel21b_slot2(l->label_ptr, dest);
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return false;
+        }
     }
+    return true;
 }
 
 static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
@@ -1765,7 +1769,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
     bswap = opc & MO_BSWAP;
 
 #if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mII,
                        INSN_NOP_M,
                        tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
@@ -1829,7 +1833,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
         }
     }
 #else
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, MmI,
                        tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
                                    TCG_GUEST_BASE_REG, addr_reg),
@@ -1889,7 +1893,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
     bswap = opc & MO_BSWAP;
 
 #if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mII,
                        INSN_NOP_M,
                        tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
@@ -1935,7 +1939,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
                    INSN_NOP_M,
                    INSN_NOP_I);
 #else
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         add_guest_base = tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
                                      TCG_GUEST_BASE_REG, addr_reg);
         addr_reg = TCG_REG_R2;
@@ -1944,7 +1948,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
     }
 
     if (!bswap) {
-        tcg_out_bundle(s, (GUEST_BASE ? MmI : mmI),
+        tcg_out_bundle(s, (guest_base ? MmI : mmI),
                        add_guest_base,
                        tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
                                    data_reg, addr_reg),
@@ -2148,9 +2152,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
         tcg_out_ext(s, OPC_ZXT2_I29, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_ext(s, OPC_SXT4_I29, args[0], args[1]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_ext(s, OPC_ZXT4_I29, args[0], args[1]);
         break;
@@ -2301,6 +2307,8 @@ static const TCGTargetOpDef ia64_op_defs[] = {
     { INDEX_op_ext16u_i64, { "r", "rZ"} },
     { INDEX_op_ext32s_i64, { "r", "rZ"} },
     { INDEX_op_ext32u_i64, { "r", "rZ"} },
+    { INDEX_op_ext_i32_i64, { "r", "rZ" } },
+    { INDEX_op_extu_i32_i64, { "r", "rZ" } },
 
     { INDEX_op_bswap16_i64, { "r", "rZ" } },
     { INDEX_op_bswap32_i64, { "r", "rZ" } },
@@ -2349,14 +2357,14 @@ static void tcg_target_qemu_prologue(TCGContext *s)
                    tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
                                TCG_REG_B6, TCG_REG_R33, 0));
 
-    /* ??? If GUEST_BASE < 0x200000, we could load the register via
+    /* ??? If guest_base < 0x200000, we could load the register via
        an ADDL in the M slot of the next bundle.  */
-    if (GUEST_BASE != 0) {
+    if (guest_base != 0) {
         tcg_out_bundle(s, mlx,
                        INSN_NOP_M,
-                       tcg_opc_l2 (GUEST_BASE),
+                       tcg_opc_l2(guest_base),
                        tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
-                                   TCG_GUEST_BASE_REG, GUEST_BASE));
+                                   TCG_GUEST_BASE_REG, guest_base));
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 
diff --git a/qemu/tcg/mips/tcg-target.h b/qemu/tcg/mips/tcg-target.h
index f5ba52cac..b1cda37b6 100644
--- a/qemu/tcg/mips/tcg-target.h
+++ b/qemu/tcg/mips/tcg-target.h
@@ -96,6 +96,13 @@ extern bool use_mips32_instructions;
 extern bool use_mips32r2_instructions;
 #endif
 
+/* MIPS32R6 instruction set detection */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 6)
+#define use_mips32r6_instructions  1
+#else
+#define use_mips32r6_instructions  0
+#endif
+
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          1
@@ -105,8 +112,8 @@ extern bool use_mips32r2_instructions;
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
-#define TCG_TARGET_HAS_mulu2_i32        1
-#define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_mulu2_i32        (!use_mips32r6_instructions)
+#define TCG_TARGET_HAS_muls2_i32        (!use_mips32r6_instructions)
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
 
diff --git a/qemu/tcg/mips/tcg-target.c b/qemu/tcg/mips/tcg-target.inc.c
index e97980df0..aaf881cfd 100644
--- a/qemu/tcg/mips/tcg-target.c
+++ b/qemu/tcg/mips/tcg-target.inc.c
@@ -35,7 +35,7 @@
 #define LO_OFF    (MIPS_BE * 4)
 #define HI_OFF    (4 - LO_OFF)
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "zero",
     "at",
@@ -76,7 +76,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #define TCG_TMP1  TCG_REG_T9
 
 /* check if we really need so many registers :P */
-static const TCGReg tcg_target_reg_alloc_order[] = {
+static const int tcg_target_reg_alloc_order[] = {
     /* Call saved registers.  */
     TCG_REG_S0,
     TCG_REG_S1,
@@ -127,7 +127,7 @@ static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
 {
     /* Let the compiler perform the right-shift as part of the arithmetic.  */
     ptrdiff_t disp = target - (pc + 1);
-    assert(disp == (int16_t)disp);
+    tcg_debug_assert(disp == (int16_t)disp);
     return disp & 0xffff;
 }
 
@@ -138,7 +138,7 @@ static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target)
 
 static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target)
 {
-    assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
+    tcg_debug_assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
     return ((uintptr_t)target >> 2) & 0x3ffffff;
 }
 
@@ -150,8 +150,8 @@ static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target)
 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
                         intptr_t value, intptr_t addend)
 {
-    assert(type == R_MIPS_PC16);
-    assert(addend == 0);
+    tcg_debug_assert(type == R_MIPS_PC16);
+    tcg_debug_assert(addend == 0);
     reloc_pc16(code_ptr, (tcg_insn_unit *)value);
 }
 
@@ -288,16 +288,24 @@ typedef enum {
     OPC_SRLV     = OPC_SPECIAL | 0x06,
     OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
     OPC_SRAV     = OPC_SPECIAL | 0x07,
-    OPC_JR       = OPC_SPECIAL | 0x08,
+    OPC_JR_R5    = OPC_SPECIAL | 0x08,
     OPC_JALR     = OPC_SPECIAL | 0x09,
     OPC_MOVZ     = OPC_SPECIAL | 0x0A,
     OPC_MOVN     = OPC_SPECIAL | 0x0B,
     OPC_MFHI     = OPC_SPECIAL | 0x10,
     OPC_MFLO     = OPC_SPECIAL | 0x12,
     OPC_MULT     = OPC_SPECIAL | 0x18,
+    OPC_MUL_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x18,
+    OPC_MUH      = OPC_SPECIAL | (0x03 <<  6) | 0x18,
     OPC_MULTU    = OPC_SPECIAL | 0x19,
+    OPC_MULU     = OPC_SPECIAL | (0x02 <<  6) | 0x19,
+    OPC_MUHU     = OPC_SPECIAL | (0x03 <<  6) | 0x19,
     OPC_DIV      = OPC_SPECIAL | 0x1A,
+    OPC_DIV_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x1A,
+    OPC_MOD      = OPC_SPECIAL | (0x03 <<  6) | 0x1A,
     OPC_DIVU     = OPC_SPECIAL | 0x1B,
+    OPC_DIVU_R6  = OPC_SPECIAL | (0x02 <<  6) | 0x1B,
+    OPC_MODU     = OPC_SPECIAL | (0x03 <<  6) | 0x1B,
     OPC_ADDU     = OPC_SPECIAL | 0x21,
     OPC_SUBU     = OPC_SPECIAL | 0x23,
     OPC_AND      = OPC_SPECIAL | 0x24,
@@ -306,13 +314,15 @@ typedef enum {
     OPC_NOR      = OPC_SPECIAL | 0x27,
     OPC_SLT      = OPC_SPECIAL | 0x2A,
     OPC_SLTU     = OPC_SPECIAL | 0x2B,
+    OPC_SELEQZ   = OPC_SPECIAL | 0x35,
+    OPC_SELNEZ   = OPC_SPECIAL | 0x37,
 
     OPC_REGIMM   = 0x01 << 26,
     OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
     OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
 
     OPC_SPECIAL2 = 0x1c << 26,
-    OPC_MUL      = OPC_SPECIAL2 | 0x002,
+    OPC_MUL_R5   = OPC_SPECIAL2 | 0x002,
 
     OPC_SPECIAL3 = 0x1f << 26,
     OPC_EXT      = OPC_SPECIAL3 | 0x000,
@@ -320,6 +330,15 @@ typedef enum {
     OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
     OPC_SEB      = OPC_SPECIAL3 | 0x420,
     OPC_SEH      = OPC_SPECIAL3 | 0x620,
+
+    /* MIPS r6 doesn't have JR, JALR should be used instead */
+    OPC_JR       = use_mips32r6_instructions ? OPC_JALR : OPC_JR_R5,
+
+    /*
+     * MIPS r6 replaces MUL with an alternative encoding which is
+     * backwards-compatible at the assembly level.
+     */
+    OPC_MUL      = use_mips32r6_instructions ? OPC_MUL_R6 : OPC_MUL_R5,
 } MIPSInsn;
 
 /*
@@ -413,7 +432,7 @@ static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
     if ((from ^ dest) & -(1 << 28)) {
         return false;
     }
-    assert((dest & 3) == 0);
+    tcg_debug_assert((dest & 3) == 0);
 
     inst = opc;
     inst |= (dest >> 2) & 0x3ffffff;
@@ -567,6 +586,55 @@ static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
     }
 }
 
+static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
+                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
+                            bool cbh, bool is_sub)
+{
+    TCGReg th = TCG_TMP1;
+
+    /* If we have a negative constant such that negating it would
+       make the high part zero, we can (usually) eliminate one insn.  */
+    if (cbl && cbh && bh == -1 && bl != 0) {
+        bl = -bl;
+        bh = 0;
+        is_sub = !is_sub;
+    }
+
+    /* By operating on the high part first, we get to use the final
+       carry operation to move back from the temporary.  */
+    if (!cbh) {
+        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
+    } else if (bh != 0 || ah == rl) {
+        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
+    } else {
+        th = ah;
+    }
+
+    /* Note that tcg optimization should eliminate the bl == 0 case.  */
+    if (is_sub) {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
+            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
+        }
+        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
+    } else {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
+        } else if (rl == al && rl == bl) {
+            tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, 31);
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
+        }
+        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
+    }
+}
+
 /* Bit 0 set if inversion required; bit 1 set if swapping required.  */
 #define MIPS_CMP_INV  1
 #define MIPS_CMP_SWAP 2
@@ -739,9 +807,9 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
     TCGReg tmp0 = TCG_TMP0;
     TCGReg tmp1 = ret;
 
-    assert(ret != TCG_TMP0);
+    tcg_debug_assert(ret != TCG_TMP0);
     if (ret == ah || ret == bh) {
-        assert(ret != TCG_TMP1);
+        tcg_debug_assert(ret != TCG_TMP1);
         tmp1 = TCG_TMP1;
     }
 
@@ -792,13 +860,20 @@ static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
 }
 
 static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGReg c1, TCGReg c2, TCGReg v)
+                            TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2)
 {
-    MIPSInsn m_opc = OPC_MOVN;
+    bool eqz = false;
+
+    /* If one of the values is zero, put it last to match SEL*Z instructions */
+    if (use_mips32r6_instructions && v1 == 0) {
+        v1 = v2;
+        v2 = 0;
+        cond = tcg_invert_cond(cond);
+    }
 
     switch (cond) {
     case TCG_COND_EQ:
-        m_opc = OPC_MOVZ;
+        eqz = true;
         /* FALLTHRU */
     case TCG_COND_NE:
         if (c2 != 0) {
@@ -811,14 +886,32 @@ static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
         /* Minimize code size by preferring a compare not requiring INV.  */
         if (mips_cmp_map[cond] & MIPS_CMP_INV) {
             cond = tcg_invert_cond(cond);
-            m_opc = OPC_MOVZ;
+            eqz = true;
         }
         tcg_out_setcond(s, cond, TCG_TMP0, c1, c2);
         c1 = TCG_TMP0;
         break;
     }
 
-    tcg_out_opc_reg(s, m_opc, ret, v, c1);
+    if (use_mips32r6_instructions) {
+        MIPSInsn m_opc_t = eqz ? OPC_SELEQZ : OPC_SELNEZ;
+        MIPSInsn m_opc_f = eqz ? OPC_SELNEZ : OPC_SELEQZ;
+
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, m_opc_f, TCG_TMP1, v2, c1);
+        }
+        tcg_out_opc_reg(s, m_opc_t, ret, v1, c1);
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP1);
+        }
+    } else {
+        MIPSInsn m_opc = eqz ? OPC_MOVZ : OPC_MOVN;
+
+        tcg_out_opc_reg(s, m_opc, ret, v1, c1);
+
+        /* This should be guaranteed via constraints */
+        tcg_debug_assert(v2 == ret);
+    }
 }
 
 static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
@@ -934,9 +1027,11 @@ static int tcg_out_call_iarg_reg2(TCGContext *s, int i, TCGReg al, TCGReg ah)
 /* Perform the tlb comparison operation.  The complete host address is
    placed in BASE.  Clobbers AT, T0, A0.  */
 static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
-                             TCGReg addrh, int mem_index, TCGMemOp s_bits,
+                             TCGReg addrh, TCGMemOpIdx oi,
                              tcg_insn_unit *label_ptr[2], bool is_load)
 {
+    TCGMemOp s_bits = get_memop(oi) & MO_SIZE;
+    int mem_index = get_mmuidx(oi);
     int cmp_off
         = (is_load
            ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
@@ -962,30 +1057,34 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
         add_off -= 0x7ff0;
     }
 
-    /* Load the tlb comparator.  */
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + LO_OFF);
-        tcg_out_opc_imm(s, OPC_LW, base, TCG_REG_A0, cmp_off + HI_OFF);
-    } else {
-        tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off);
-    }
+    /* Load the (low half) tlb comparator.  */
+    tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0,
+                    cmp_off + (TARGET_LONG_BITS == 64 ? LO_OFF : 0));
 
     /* Mask the page bits, keeping the alignment bits to compare against.
-       In between, load the tlb addend for the fast path.  */
+       In between on 32-bit targets, load the tlb addend for the fast path.  */
     tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1,
                  TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
+    }
     tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl);
 
     label_ptr[0] = s->code_ptr;
     tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0);
 
+    /* Load and test the high half tlb comparator.  */
     if (TARGET_LONG_BITS == 64) {
         /* delay slot */
-        tcg_out_nop(s);
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + HI_OFF);
+
+        /* Load the tlb addend for the fast path. We can't do it earlier with
+           64-bit targets or we'll clobber a0 before reading the high half tlb
+           comparator.  */
+        tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
 
         label_ptr[1] = s->code_ptr;
-        tcg_out_opc_br(s, OPC_BNE, addrh, base);
+        tcg_out_opc_br(s, OPC_BNE, addrh, TCG_TMP0);
     }
 
     /* delay slot */
@@ -1156,8 +1255,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
 #endif
     /* Note that we've eliminated V0 from the output registers,
        so we won't overwrite the base register during loading.  */
@@ -1171,21 +1268,17 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
     opc = get_memop(oi);
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
-
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 1);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1);
     tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
     add_qemu_ldst_label(s, 1, oi, data_regl, data_regh, addr_regl, addr_regh,
                         s->code_ptr, label_ptr);
 #else
-    if (GUEST_BASE == 0 && data_regl != addr_regl) {
+    if (guest_base == 0 && data_regl != addr_regl) {
         base = addr_regl;
-    } else if (GUEST_BASE == (int16_t)GUEST_BASE) {
-        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
+    } else if (guest_base == (int16_t)guest_base) {
+        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
     } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+        tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
         tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
     }
     tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
@@ -1233,55 +1326,6 @@ static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
     }
 }
 
-static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
-                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
-                            bool cbh, bool is_sub)
-{
-    TCGReg th = TCG_TMP1;
-
-    /* If we have a negative constant such that negating it would
-       make the high part zero, we can (usually) eliminate one insn.  */
-    if (cbl && cbh && bh == -1 && bl != 0) {
-        bl = -bl;
-        bh = 0;
-        is_sub = !is_sub;
-    }
-
-    /* By operating on the high part first, we get to use the final
-       carry operation to move back from the temporary.  */
-    if (!cbh) {
-        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
-    } else if (bh != 0 || ah == rl) {
-        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
-    } else {
-        th = ah;
-    }
-
-    /* Note that tcg optimization should eliminate the bl == 0 case.  */
-    if (is_sub) {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
-            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
-        }
-        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
-    } else {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
-        } else if (rl == al && rl == bl) {
-            tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, 31);
-            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
-        }
-        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
-    }
-}
-
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 {
     TCGReg addr_regl, addr_regh __attribute__((unused));
@@ -1290,8 +1334,6 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
     TCGMemOp opc;
 #if defined(CONFIG_SOFTMMU)
     tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
 #endif
 
     data_regl = *args++;
@@ -1302,26 +1344,22 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
     opc = get_memop(oi);
 
 #if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & 3;
-
     /* Note that we eliminated the helper's address argument,
        so we can reuse that for the base.  */
     base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 0);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0);
     tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
     add_qemu_ldst_label(s, 0, oi, data_regl, data_regh, addr_regl, addr_regh,
                         s->code_ptr, label_ptr);
 #else
-    if (GUEST_BASE == 0) {
+    if (guest_base == 0) {
         base = addr_regl;
     } else {
         base = TCG_REG_A0;
-        if (GUEST_BASE == (int16_t)GUEST_BASE) {
-            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
+        if (guest_base == (int16_t)guest_base) {
+            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
         } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+            tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
             tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
         }
     }
@@ -1432,8 +1470,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_and_i32:
         if (c2 && a2 != (uint16_t)a2) {
             int msb = ctz32(~a2) - 1;
-            assert(use_mips32r2_instructions);
-            assert(is_p2m1(a2));
+            tcg_debug_assert(use_mips32r2_instructions);
+            tcg_debug_assert(is_p2m1(a2));
             tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0);
             break;
         }
@@ -1451,21 +1489,45 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         i1 = OPC_MULT, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_mulsh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUH, a0, a1, a2);
+            break;
+        }
         i1 = OPC_MULT, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_muluh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUHU, a0, a1, a2);
+            break;
+        }
         i1 = OPC_MULTU, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_div_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIV_R6, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIV, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_divu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIVU_R6, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIVU, i2 = OPC_MFLO;
         goto do_hilo1;
     case INDEX_op_rem_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MOD, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIV, i2 = OPC_MFHI;
         goto do_hilo1;
     case INDEX_op_remu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MODU, a0, a1, a2);
+            break;
+        }
         i1 = OPC_DIVU, i2 = OPC_MFHI;
     do_hilo1:
         tcg_out_opc_reg(s, i1, 0, a1, a2);
@@ -1542,7 +1604,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], a0, a1, a2, args[3]);
+        tcg_out_movcond(s, args[5], a0, a1, a2, args[3], args[4]);
         break;
 
     case INDEX_op_setcond_i32:
@@ -1598,8 +1660,10 @@ static const TCGTargetOpDef mips_op_defs[] = {
 
     { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
     { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
+#if !use_mips32r6_instructions
     { INDEX_op_muls2_i32, { "r", "r", "rZ", "rZ" } },
     { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
+#endif
     { INDEX_op_mulsh_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_muluh_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
@@ -1629,7 +1693,11 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
 
     { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+#if use_mips32r6_instructions
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
+#else
     { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
+#endif
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
@@ -1667,7 +1735,6 @@ static int tcg_target_callee_save_regs[] = {
 /* The Linux kernel doesn't provide any information about the available
    instruction set. Probe it using a signal handler. */
 
-#include <signal.h>
 
 #ifndef use_movnz_instructions
 bool use_movnz_instructions = false;
diff --git a/qemu/tcg/optimize.c b/qemu/tcg/optimize.c
index 18283cfd7..f01160815 100644
--- a/qemu/tcg/optimize.c
+++ b/qemu/tcg/optimize.c
@@ -23,10 +23,8 @@
  * THE SOFTWARE.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 
-#include <stdlib.h>
-#include <stdio.h>
 
 #include "qemu-common.h"
 #include "tcg-op.h"
@@ -35,14 +33,8 @@
         glue(glue(case INDEX_op_, x), _i32):    \
         glue(glue(case INDEX_op_, x), _i64)
 
-typedef enum {
-    TCG_TEMP_UNDEF = 0,
-    TCG_TEMP_CONST,
-    TCG_TEMP_COPY,
-} tcg_temp_state;
-
 struct tcg_temp_info {
-    tcg_temp_state state;
+    bool is_const;
     uint16_t prev_copy;
     uint16_t next_copy;
     tcg_target_ulong val;
@@ -50,23 +42,47 @@ struct tcg_temp_info {
 };
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
+static TCGTempSet temps_used;
+
+static inline bool temp_is_const(TCGArg arg)
+{
+    return temps[arg].is_const;
+}
+
+static inline bool temp_is_copy(TCGArg arg)
+{
+    return temps[arg].next_copy != arg;
+}
 
-/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
-   the copy flag from the left temp.  */
+/* Reset TEMP's state, possibly removing the temp for the list of copies.  */
 static void reset_temp(TCGArg temp)
 {
-    if (temps[temp].state == TCG_TEMP_COPY) {
-        if (temps[temp].prev_copy == temps[temp].next_copy) {
-            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
-        } else {
-            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        }
-    }
-    temps[temp].state = TCG_TEMP_UNDEF;
+    temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
+    temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
+    temps[temp].next_copy = temp;
+    temps[temp].prev_copy = temp;
+    temps[temp].is_const = false;
     temps[temp].mask = -1;
 }
 
+/* Reset all temporaries, given that there are NB_TEMPS of them.  */
+static void reset_all_temps(int nb_temps)
+{
+    bitmap_zero(temps_used.l, nb_temps);
+}
+
+/* Initialize and activate a temporary.  */
+static void init_temp_info(TCGArg temp)
+{
+    if (!test_bit(temp, temps_used.l)) {
+        temps[temp].next_copy = temp;
+        temps[temp].prev_copy = temp;
+        temps[temp].is_const = false;
+        temps[temp].mask = -1;
+        set_bit(temp, temps_used.l);
+    }
+}
+
 static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
                                 TCGOpcode opc, int nargs)
 {
@@ -98,16 +114,6 @@ static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
     return new_op;
 }
 
-/* Reset all temporaries, given that there are NB_TEMPS of them.  */
-static void reset_all_temps(int nb_temps)
-{
-    int i;
-    for (i = 0; i < nb_temps; i++) {
-        temps[i].state = TCG_TEMP_UNDEF;
-        temps[i].mask = -1;
-    }
-}
-
 static int op_bits(TCGOpcode op)
 {
     const TCGOpDef *def = &tcg_op_defs[op];
@@ -179,8 +185,7 @@ static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
         return true;
     }
 
-    if (temps[arg1].state != TCG_TEMP_COPY
-        || temps[arg2].state != TCG_TEMP_COPY) {
+    if (!temp_is_copy(arg1) || !temp_is_copy(arg2)) {
         return false;
     }
 
@@ -202,7 +207,7 @@ static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg *args,
     op->opc = new_op;
 
     reset_temp(dst);
-    temps[dst].state = TCG_TEMP_CONST;
+    temps[dst].is_const = true;
     temps[dst].val = val;
     mask = val;
     if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) {
@@ -223,11 +228,6 @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
         return;
     }
 
-    if (temps[src].state == TCG_TEMP_CONST) {
-        tcg_opt_gen_movi(s, op, args, dst, temps[src].val);
-        return;
-    }
-
     TCGOpcode new_op = op_to_mov(op->opc);
     tcg_target_ulong mask;
 
@@ -241,19 +241,13 @@ static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
     }
     temps[dst].mask = mask;
 
-    assert(temps[src].state != TCG_TEMP_CONST);
-
     if (s->temps[src].type == s->temps[dst].type) {
-        if (temps[src].state != TCG_TEMP_COPY) {
-            temps[src].state = TCG_TEMP_COPY;
-            temps[src].next_copy = src;
-            temps[src].prev_copy = src;
-        }
-        temps[dst].state = TCG_TEMP_COPY;
         temps[dst].next_copy = temps[src].next_copy;
         temps[dst].prev_copy = src;
         temps[temps[dst].next_copy].prev_copy = dst;
         temps[src].next_copy = dst;
+        temps[dst].is_const = temps[src].is_const;
+        temps[dst].val = temps[src].val;
     }
 
     args[0] = dst;
@@ -292,7 +286,6 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     case INDEX_op_shr_i32:
         return (uint32_t)x >> (y & 31);
 
-    case INDEX_op_trunc_shr_i32:
     case INDEX_op_shr_i64:
         return (uint64_t)x >> (y & 63);
 
@@ -347,12 +340,18 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     CASE_OP_32_64(ext16u):
         return (uint16_t)x;
 
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         return (int32_t)x;
 
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext32u_i64:
         return (uint32_t)x;
 
+    case INDEX_op_extrh_i64_i32:
+        return (uint64_t)x >> 32;
+
     case INDEX_op_muluh_i32:
         return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
     case INDEX_op_mulsh_i32:
@@ -395,7 +394,7 @@ static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y)
 {
     TCGArg res = do_constant_folding_2(op, x, y);
     if (op_bits(op) == 32) {
-        res &= 0xffffffff;
+        res = (int32_t)res;
     }
     return res;
 }
@@ -481,7 +480,7 @@ static bool do_constant_folding_cond_eq(TCGCond c)
 static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
                                        TCGArg y, TCGCond c)
 {
-    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
+    if (temp_is_const(x) && temp_is_const(y)) {
         switch (op_bits(op)) {
         case 32:
             return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
@@ -492,7 +491,7 @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
         }
     } else if (temps_are_copies(x, y)) {
         return do_constant_folding_cond_eq(c);
-    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+    } else if (temp_is_const(y) && temps[y].val == 0) {
         switch (c) {
         case TCG_COND_LTU:
             return 0;
@@ -513,12 +512,10 @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
     TCGArg al = p1[0], ah = p1[1];
     TCGArg bl = p2[0], bh = p2[1];
 
-    if (temps[bl].state == TCG_TEMP_CONST
-        && temps[bh].state == TCG_TEMP_CONST) {
+    if (temp_is_const(bl) && temp_is_const(bh)) {
         uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
 
-        if (temps[al].state == TCG_TEMP_CONST
-            && temps[ah].state == TCG_TEMP_CONST) {
+        if (temp_is_const(al) && temp_is_const(ah)) {
             uint64_t a;
             a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
             return do_constant_folding_cond_64(a, b, c);
@@ -544,8 +541,8 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 {
     TCGArg a1 = *p1, a2 = *p2;
     int sum = 0;
-    sum += temps[a1].state == TCG_TEMP_CONST;
-    sum -= temps[a2].state == TCG_TEMP_CONST;
+    sum += temp_is_const(a1);
+    sum -= temp_is_const(a2);
 
     /* Prefer the constant in second argument, and then the form
        op a, a, b, which is better handled on non-RISC hosts. */
@@ -560,10 +557,10 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
 {
     int sum = 0;
-    sum += temps[p1[0]].state == TCG_TEMP_CONST;
-    sum += temps[p1[1]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    sum += temp_is_const(p1[0]);
+    sum += temp_is_const(p1[1]);
+    sum -= temp_is_const(p2[0]);
+    sum -= temp_is_const(p2[1]);
     if (sum > 0) {
         TCGArg t;
         t = p1[0], p1[0] = p2[0], p2[0] = t;
@@ -598,17 +595,29 @@ void tcg_optimize(TCGContext *s)
         const TCGOpDef *def = &tcg_op_defs[opc];
 
         oi_next = op->next;
+
+        /* Count the arguments, and initialize the temps that are
+           going to be used */
         if (opc == INDEX_op_call) {
             nb_oargs = op->callo;
             nb_iargs = op->calli;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                tmp = args[i];
+                if (tmp != TCG_CALL_DUMMY_ARG) {
+                    init_temp_info(tmp);
+                }
+            }
         } else {
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                init_temp_info(args[i]);
+            }
         }
 
         /* Do copy propagation */
         for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            if (temps[args[i]].state == TCG_TEMP_COPY) {
+            if (temp_is_copy(args[i])) {
                 args[i] = find_better_copy(s, args[i]);
             }
         }
@@ -678,8 +687,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
                 tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
@@ -689,7 +697,7 @@ void tcg_optimize(TCGContext *s)
                 TCGOpcode neg_op;
                 bool have_neg;
 
-                if (temps[args[2]].state == TCG_TEMP_CONST) {
+                if (temp_is_const(args[2])) {
                     /* Proceed with possible constant folding. */
                     break;
                 }
@@ -703,8 +711,7 @@ void tcg_optimize(TCGContext *s)
                 if (!have_neg) {
                     break;
                 }
-                if (temps[args[1]].state == TCG_TEMP_CONST
-                    && temps[args[1]].val == 0) {
+                if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
                     op->opc = neg_op;
                     reset_temp(args[0]);
                     args[1] = args[2];
@@ -714,34 +721,30 @@ void tcg_optimize(TCGContext *s)
             break;
         CASE_OP_32_64(xor):
         CASE_OP_32_64(nand):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(andc):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == -1) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == -1) {
                 i = 2;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == 0) {
                 i = 2;
                 goto try_not;
             }
@@ -782,9 +785,8 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(or):
         CASE_OP_32_64(xor):
         CASE_OP_32_64(andc):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
                 tcg_opt_gen_mov(s, op, args, args[0], args[1]);
                 continue;
             }
@@ -792,9 +794,8 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(and):
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
                 tcg_opt_gen_mov(s, op, args, args[0], args[1]);
                 continue;
             }
@@ -832,17 +833,26 @@ void tcg_optimize(TCGContext *s)
 
         CASE_OP_32_64(and):
             mask = temps[args[2]].mask;
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
         and_const:
                 affected = temps[args[1]].mask & ~mask;
             }
             mask = temps[args[1]].mask & mask;
             break;
 
+        case INDEX_op_ext_i32_i64:
+            if ((temps[args[1]].mask & 0x80000000) != 0) {
+                break;
+            }
+        case INDEX_op_extu_i32_i64:
+            /* We do not compute affected as it is a size changing op.  */
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+
         CASE_OP_32_64(andc):
             /* Known-zeros does not imply known-ones.  Therefore unless
                args[2] is constant, we can't infer anything from it.  */
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 mask = ~temps[args[2]].mask;
                 goto and_const;
             }
@@ -851,37 +861,40 @@ void tcg_optimize(TCGContext *s)
             break;
 
         case INDEX_op_sar_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (int32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_sar_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (int64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
         case INDEX_op_shr_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (uint32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_shr_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (uint64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
-        case INDEX_op_trunc_shr_i32:
-            mask = (uint64_t)temps[args[1]].mask >> args[2];
+        case INDEX_op_extrl_i64_i32:
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+        case INDEX_op_extrh_i64_i32:
+            mask = (uint64_t)temps[args[1]].mask >> 32;
             break;
 
         CASE_OP_32_64(shl):
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & (TCG_TARGET_REG_BITS - 1);
                 mask = temps[args[1]].mask << tmp;
             }
@@ -946,12 +959,12 @@ void tcg_optimize(TCGContext *s)
         }
 
         if (partmask == 0) {
-            assert(nb_oargs == 1);
+            tcg_debug_assert(nb_oargs == 1);
             tcg_opt_gen_movi(s, op, args, args[0], 0);
             continue;
         }
         if (affected == 0) {
-            assert(nb_oargs == 1);
+            tcg_debug_assert(nb_oargs == 1);
             tcg_opt_gen_mov(s, op, args, args[0], args[1]);
             continue;
         }
@@ -962,8 +975,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(mul):
         CASE_OP_32_64(muluh):
         CASE_OP_32_64(mulsh):
-            if ((temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0)) {
+            if ((temp_is_const(args[2]) && temps[args[2]].val == 0)) {
                 tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
@@ -1018,21 +1030,17 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(ext16u):
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
+        case INDEX_op_ext_i32_i64:
+        case INDEX_op_extu_i32_i64:
+        case INDEX_op_extrl_i64_i32:
+        case INDEX_op_extrh_i64_i32:
+            if (temp_is_const(args[1])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val, 0);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
 
-        case INDEX_op_trunc_shr_i32:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val, args[2]);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
-                break;
-            }
-            goto do_default;
-
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
         CASE_OP_32_64(mul):
@@ -1055,8 +1063,7 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(divu):
         CASE_OP_32_64(rem):
         CASE_OP_32_64(remu):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val,
                                           temps[args[2]].val);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
@@ -1065,8 +1072,7 @@ void tcg_optimize(TCGContext *s)
             goto do_default;
 
         CASE_OP_32_64(deposit):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = deposit64(temps[args[1]].val, args[3], args[4],
                                 temps[args[2]].val);
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
@@ -1106,10 +1112,8 @@ void tcg_optimize(TCGContext *s)
 
         case INDEX_op_add2_i32:
         case INDEX_op_sub2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST
-                && temps[args[4]].state == TCG_TEMP_CONST
-                && temps[args[5]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])
+                && temp_is_const(args[4]) && temp_is_const(args[5])) {
                 uint32_t al = temps[args[2]].val;
                 uint32_t ah = temps[args[3]].val;
                 uint32_t bl = temps[args[4]].val;
@@ -1128,8 +1132,8 @@ void tcg_optimize(TCGContext *s)
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (uint32_t)a);
-                tcg_opt_gen_movi(s, op2, args2, rh, (uint32_t)(a >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)a);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(a >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1138,8 +1142,7 @@ void tcg_optimize(TCGContext *s)
             goto do_default;
 
         case INDEX_op_mulu2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])) {
                 uint32_t a = temps[args[2]].val;
                 uint32_t b = temps[args[3]].val;
                 uint64_t r = (uint64_t)a * b;
@@ -1149,8 +1152,8 @@ void tcg_optimize(TCGContext *s)
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (uint32_t)r);
-                tcg_opt_gen_movi(s, op2, args2, rh, (uint32_t)(r >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)r);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(r >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1171,10 +1174,8 @@ void tcg_optimize(TCGContext *s)
                     tcg_op_remove(s, op);
                 }
             } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
-                       && temps[args[2]].state == TCG_TEMP_CONST
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[2]].val == 0
-                       && temps[args[3]].val == 0) {
+                       && temp_is_const(args[2]) && temps[args[2]].val == 0
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_brcond_high:
@@ -1236,10 +1237,8 @@ void tcg_optimize(TCGContext *s)
             do_setcond_const:
                 tcg_opt_gen_movi(s, op, args, args[0], tmp);
             } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[4]].state == TCG_TEMP_CONST
-                       && temps[args[3]].val == 0
-                       && temps[args[4]].val == 0) {
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0
+                       && temp_is_const(args[4]) && temps[args[4]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_setcond_high:
@@ -1299,7 +1298,9 @@ void tcg_optimize(TCGContext *s)
             if (!(args[nb_oargs + nb_iargs + 1]
                   & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i);
+                    if (test_bit(i, temps_used.l)) {
+                        reset_temp(i);
+                    }
                 }
             }
             goto do_reset_output;
diff --git a/qemu/tcg/ppc/tcg-target.h b/qemu/tcg/ppc/tcg-target.h
index 7ce704882..b4f081876 100644
--- a/qemu/tcg/ppc/tcg-target.h
+++ b/qemu/tcg/ppc/tcg-target.h
@@ -77,7 +77,8 @@ typedef enum {
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/qemu/tcg/ppc/tcg-target.c b/qemu/tcg/ppc/tcg-target.inc.c
index 2b6eafa03..00bb90fc2 100644
--- a/qemu/tcg/ppc/tcg-target.c
+++ b/qemu/tcg/ppc/tcg-target.inc.c
@@ -80,22 +80,16 @@
 
 static tcg_insn_unit *tb_ret_addr;
 
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
-#endif
-
 #include "elf.h"
 static bool have_isa_2_06;
 #define HAVE_ISA_2_06  have_isa_2_06
 #define HAVE_ISEL      have_isa_2_06
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG 30
-#else
-#define TCG_GUEST_BASE_REG 0
 #endif
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "r0",
     "r1",
@@ -213,7 +207,7 @@ static inline bool in_range_b(tcg_target_long target)
 static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
 {
     ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
-    assert(in_range_b(disp));
+    tcg_debug_assert(in_range_b(disp));
     return disp & 0x3fffffc;
 }
 
@@ -225,7 +219,7 @@ static void reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
 static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
 {
     ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
-    assert(disp == (int16_t) disp);
+    tcg_debug_assert(disp == (int16_t) disp);
     return disp & 0xfffc;
 }
 
@@ -251,7 +245,7 @@ static void patch_reloc(tcg_insn_unit *code_ptr, int type,
 {
     tcg_insn_unit *target = (tcg_insn_unit *)value;
 
-    assert(addend == 0);
+    tcg_debug_assert(addend == 0);
     switch (type) {
     case R_PPC_REL14:
         reloc_pc14(code_ptr, target);
@@ -571,7 +565,7 @@ static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
                                int sh, int mb)
 {
-    assert(TCG_TARGET_REG_BITS == 64);
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
     sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
     mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
     tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
@@ -706,14 +700,14 @@ static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
 {
     int mb, me;
 
-    if ((c & 0xffff) == c) {
+    if (mask_operand(c, &mb, &me)) {
+        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
+    } else if ((c & 0xffff) == c) {
         tcg_out32(s, ANDI | SAI(src, dst, c));
         return;
     } else if ((c & 0xffff0000) == c) {
         tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
         return;
-    } else if (mask_operand(c, &mb, &me)) {
-        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
     } else {
         tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
         tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
@@ -724,19 +718,19 @@ static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
 {
     int mb, me;
 
-    assert(TCG_TARGET_REG_BITS == 64);
-    if ((c & 0xffff) == c) {
-        tcg_out32(s, ANDI | SAI(src, dst, c));
-        return;
-    } else if ((c & 0xffff0000) == c) {
-        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
-        return;
-    } else if (mask64_operand(c, &mb, &me)) {
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    if (mask64_operand(c, &mb, &me)) {
         if (mb == 0) {
             tcg_out_rld(s, RLDICR, dst, src, 0, me);
         } else {
             tcg_out_rld(s, RLDICL, dst, src, 0, mb);
         }
+    } else if ((c & 0xffff) == c) {
+        tcg_out32(s, ANDI | SAI(src, dst, c));
+        return;
+    } else if ((c & 0xffff0000) == c) {
+        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
+        return;
     } else {
         tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
         tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
@@ -840,7 +834,7 @@ static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
 {
     int opi, opx;
 
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
     if (type == TCG_TYPE_I32) {
         opi = LWZ, opx = LWZX;
     } else {
@@ -854,7 +848,7 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
 {
     int opi, opx;
 
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
     if (type == TCG_TYPE_I32) {
         opi = STW, opx = STWX;
     } else {
@@ -987,7 +981,7 @@ static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
 {
     int crop, sh;
 
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
 
     /* Ignore high bits of a potential constant arg2.  */
     if (type == TCG_TYPE_I32) {
@@ -1245,11 +1239,36 @@ static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
 
 void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
 {
-    TCGContext s;
+    tcg_insn_unit i1, i2;
+    uint64_t pair;
+    intptr_t diff = addr - jmp_addr;
 
-    s.code_buf = s.code_ptr = (tcg_insn_unit *)jmp_addr;
-    tcg_out_b(&s, 0, (tcg_insn_unit *)addr);
-    flush_icache_range(jmp_addr, jmp_addr + tcg_current_code_size(&s));
+    if (in_range_b(diff)) {
+        i1 = B | (diff & 0x3fffffc);
+        i2 = NOP;
+    } else if (USE_REG_RA) {
+        intptr_t lo, hi;
+        diff = addr - (uintptr_t)tb_ret_addr;
+        lo = (int16_t)diff;
+        hi = (int32_t)(diff - lo);
+        tcg_debug_assert(diff == hi + lo);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, TCG_REG_RA, hi >> 16);
+        i2 = ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, lo);
+    } else {
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 32 || addr == (int32_t)addr);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, 0, addr >> 16);
+        i2 = ORI | SAI(TCG_REG_TMP1, TCG_REG_TMP1, addr);
+    }
+#ifdef HOST_WORDS_BIGENDIAN
+    pair = (uint64_t)i1 << 32 | i2;
+#else
+    pair = (uint64_t)i2 << 32 | i1;
+#endif
+
+    /* ??? __atomic_store_8, presuming there's some way to do that
+       for 32-bit, otherwise this is good enough for 64-bit.  */
+    *(uint64_t *)jmp_addr = pair;
+    flush_icache_range(jmp_addr, jmp_addr + 8);
 }
 
 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
@@ -1361,7 +1380,7 @@ static void * const qemu_st_helpers[16] = {
    in CR7, loads the addend of the TLB into R3, and returns the register
    containing the guest address (zero-extended into R4).  Clobbers R0 and R2. */
 
-static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc,
                                TCGReg addrlo, TCGReg addrhi,
                                int mem_index, bool is_read)
 {
@@ -1371,6 +1390,7 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
            : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
     int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
     TCGReg base = TCG_AREG0;
+    TCGMemOp s_bits = opc & MO_SIZE;
 
     /* Extract the page index, shifted into place for tlb index.  */
     if (TCG_TARGET_REG_BITS == 64) {
@@ -1422,17 +1442,37 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp s_bits,
        to minimize any load use delay.  */
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off);
 
-    /* Clear the non-page, non-alignment bits from the address.  */
+    /* Clear the non-page, non-alignment bits from the address */
     if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
+        /* We don't support unaligned accesses on 32-bits, preserve
+         * the bottom bits and thus trigger a comparison failure on
+         * unaligned accesses
+         */
         tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
                     (32 - s_bits) & 31, 31 - TARGET_PAGE_BITS);
-    } else if (!s_bits) {
-        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo,
-                    0, 63 - TARGET_PAGE_BITS);
+    } else if (s_bits) {
+        /* > byte access, we need to handle alignment */
+        if ((opc & MO_AMASK) == MO_ALIGN) {
+            /* Alignment required by the front-end, same as 32-bits */
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
+                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
+       } else {
+           /* We support unaligned accesses, we need to make sure we fail
+            * if we cross a page boundary. The trick is to add the
+            * access_size-1 to the address before masking the low bits.
+            * That will make the address overflow to the next page if we
+            * cross a page boundary which will then force a mismatch of
+            * the TLB compare since the next page cannot possibly be in
+            * the same TLB index.
+            */
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1));
+            tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0,
+                        0, 63 - TARGET_PAGE_BITS);
+        }
     } else {
-        tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
-                    64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - s_bits);
-        tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
+        /* Byte access, just chop off the bits below the page index */
+        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS);
     }
 
     if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
@@ -1592,7 +1632,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
 
 #ifdef CONFIG_SOFTMMU
     mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, s_bits, addrlo, addrhi, mem_index, true);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
 
     /* Load a pointer into the current opcode w/conditional branch-link. */
     label_ptr = s->code_ptr;
@@ -1600,7 +1640,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
 
     rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
     if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
         tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
         addrlo = TCG_REG_TMP1;
@@ -1667,7 +1707,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 
 #ifdef CONFIG_SOFTMMU
     mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, s_bits, addrlo, addrhi, mem_index, false);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
 
     /* Load a pointer into the current opcode w/conditional branch-link. */
     label_ptr = s->code_ptr;
@@ -1675,7 +1715,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
 
     rbase = TCG_REG_R3;
 #else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
     if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
         tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
         addrlo = TCG_REG_TMP1;
@@ -1779,9 +1819,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     }
     tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 #endif
@@ -1817,7 +1857,7 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     }
 
     /* Epilogue */
-    assert(tb_ret_addr == s->code_ptr);
+    tcg_debug_assert(tb_ret_addr == s->code_ptr);
 
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
     for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
@@ -1840,12 +1880,10 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         if (USE_REG_RA) {
             ptrdiff_t disp = tcg_pcrel_diff(s, tb_ret_addr);
 
-            /* If we can use a direct branch, otherwise use the value in RA.
-               Note that the direct branch is always forward.  If it's in
-               range now, it'll still be in range after the movi.  Don't
-               bother about the 20 bytes where the test here fails but it
-               would succeed below.  */
-            if (!in_range_b(disp)) {
+            /* Use a direct branch if we can, otherwise use the value in RA.
+               Note that the direct branch is always backward, thus we need
+               to account for the possibility of 5 insns from the movi.  */
+            if (!in_range_b(disp - 20)) {
                 tcg_out32(s, MTSPR | RS(TCG_REG_RA) | CTR);
                 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
                 tcg_out32(s, BCCTR | BO_ALWAYS);
@@ -1856,14 +1894,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_b(s, 0, tb_ret_addr);
         break;
     case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* Direct jump method.  */
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            s->code_ptr += 7;
-        } else {
-            /* Indirect jump method.  */
-            tcg_abort();
+        tcg_debug_assert(s->tb_jmp_offset);
+        /* Direct jump.  Ensure the next insns are 8-byte aligned. */
+        if ((uintptr_t)s->code_ptr & 7) {
+            tcg_out32(s, NOP);
         }
+        s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
+        /* To be replaced by either a branch+nop or a load into TMP1.  */
+        s->code_ptr += 2;
+        tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
         s->tb_next_offset[args[0]] = tcg_current_code_size(s);
         break;
     case INDEX_op_br:
@@ -2200,12 +2240,16 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_ext16s_i64:
         c = EXTSH;
         goto gen_ext;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         c = EXTSW;
         goto gen_ext;
     gen_ext:
         tcg_out32(s, c | RS(args[1]) | RA(args[0]));
         break;
+    case INDEX_op_extu_i32_i64:
+        tcg_out_ext32u(s, args[0], args[1]);
+        break;
 
     case INDEX_op_setcond_i32:
         tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
@@ -2482,6 +2526,8 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_ext8s_i64, { "r", "r" } },
     { INDEX_op_ext16s_i64, { "r", "r" } },
     { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
     { INDEX_op_bswap16_i64, { "r", "r" } },
     { INDEX_op_bswap32_i64, { "r", "r" } },
     { INDEX_op_bswap64_i64, { "r", "r" } },
@@ -2679,8 +2725,6 @@ static void __attribute__((constructor)) tcg_cache_init(void)
 }
 
 #elif defined __APPLE__
-#include <stdio.h>
-#include <sys/types.h>
 #include <sys/sysctl.h>
 
 static void __attribute__((constructor)) tcg_cache_init(void)
@@ -2699,11 +2743,6 @@ static void __attribute__((constructor)) tcg_cache_init(void)
 }
 
 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
 #include <sys/sysctl.h>
 
 static void __attribute__((constructor)) tcg_cache_init(void)
diff --git a/qemu/tcg/s390/tcg-target.h b/qemu/tcg/s390/tcg-target.h
index 91576d594..d9dc03873 100644
--- a/qemu/tcg/s390/tcg-target.h
+++ b/qemu/tcg/s390/tcg-target.h
@@ -72,7 +72,8 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/qemu/tcg/s390/tcg-target.c b/qemu/tcg/s390/tcg-target.inc.c
index aa718eca0..580553239 100644
--- a/qemu/tcg/s390/tcg-target.c
+++ b/qemu/tcg/s390/tcg-target.inc.c
@@ -51,17 +51,10 @@
 /* A scratch register that may be be used throughout the backend.  */
 #define TCG_TMP0        TCG_REG_R14
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 #define TCG_GUEST_BASE_REG TCG_REG_R13
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
 #endif
 
-
 /* All of the following instructions are prefixed with their instruction
    format, and are defined as 8- or 16-bit quantities, even when the two
    halves of the 16-bit quantity may appear 32 bits apart in the insn.
@@ -228,7 +221,7 @@ typedef enum S390Opcode {
     RX_STH      = 0x40,
 } S390Opcode;
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
     "%r8", "%r9", "%r10" "%r11" "%r12" "%r13" "%r14" "%r15"
@@ -355,15 +348,15 @@ static void patch_reloc(tcg_insn_unit *code_ptr, int type,
                         intptr_t value, intptr_t addend)
 {
     intptr_t pcrel2 = (tcg_insn_unit *)value - (code_ptr - 1);
-    assert(addend == -2);
+    tcg_debug_assert(addend == -2);
 
     switch (type) {
     case R_390_PC16DBL:
-        assert(pcrel2 == (int16_t)pcrel2);
+        tcg_debug_assert(pcrel2 == (int16_t)pcrel2);
         tcg_patch16(code_ptr, pcrel2);
         break;
     case R_390_PC32DBL:
-        assert(pcrel2 == (int32_t)pcrel2);
+        tcg_debug_assert(pcrel2 == (int32_t)pcrel2);
         tcg_patch32(code_ptr, pcrel2);
         break;
     default:
@@ -1504,20 +1497,36 @@ QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
 static TCGReg tcg_out_tlb_read(TCGContext* s, TCGReg addr_reg, TCGMemOp opc,
                                int mem_index, bool is_ld)
 {
-    TCGMemOp s_bits = opc & MO_SIZE;
-    uint64_t tlb_mask = TARGET_PAGE_MASK | ((1 << s_bits) - 1);
-    int ofs;
+    int s_mask = (1 << (opc & MO_SIZE)) - 1;
+    int ofs, a_off;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if ((opc & MO_AMASK) == MO_ALIGN || s_mask == 0) {
+        a_off = 0;
+        tlb_mask = TARGET_PAGE_MASK | s_mask;
+    } else {
+        a_off = s_mask;
+        tlb_mask = TARGET_PAGE_MASK;
+    }
 
     if (facilities & FACILITY_GEN_INST_EXT) {
         tcg_out_risbg(s, TCG_REG_R2, addr_reg,
                       64 - CPU_TLB_BITS - CPU_TLB_ENTRY_BITS,
                       63 - CPU_TLB_ENTRY_BITS,
                       64 + CPU_TLB_ENTRY_BITS - TARGET_PAGE_BITS, 1);
-        tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
+        if (a_off) {
+            tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
+            tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
+        } else {
+            tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
+        }
     } else {
         tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
                      TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-        tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_R3, addr_reg);
+        tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
         tgen_andi(s, TCG_TYPE_I64, TCG_REG_R2,
                   (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
         tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
@@ -1622,9 +1631,9 @@ static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
         tgen_ext32u(s, TCG_TMP0, *addr_reg);
         *addr_reg = TCG_TMP0;
     }
-    if (GUEST_BASE < 0x80000) {
+    if (guest_base < 0x80000) {
         *index_reg = TCG_REG_NONE;
-        *disp = GUEST_BASE;
+        *disp = guest_base;
     } else {
         *index_reg = TCG_GUEST_BASE_REG;
         *disp = 0;
@@ -2090,6 +2099,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16s_i64:
         tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
         break;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tgen_ext32s(s, args[0], args[1]);
         break;
@@ -2099,6 +2109,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_ext16u_i64:
         tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tgen_ext32u(s, args[0], args[1]);
         break;
@@ -2251,6 +2262,9 @@ static const TCGTargetOpDef s390_op_defs[] = {
     { INDEX_op_ext32s_i64, { "r", "r" } },
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
     { INDEX_op_bswap16_i64, { "r", "r" } },
     { INDEX_op_bswap32_i64, { "r", "r" } },
     { INDEX_op_bswap64_i64, { "r", "r" } },
@@ -2328,10 +2342,12 @@ static void tcg_target_qemu_prologue(TCGContext *s)
                   TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
 
-    if (GUEST_BASE >= 0x80000) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base >= 0x80000) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
+#endif
 
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
     /* br %r3 (go to TB) */
diff --git a/qemu/tcg/sparc/tcg-target.h b/qemu/tcg/sparc/tcg-target.h
index f584de476..2cd72d2d4 100644
--- a/qemu/tcg/sparc/tcg-target.h
+++ b/qemu/tcg/sparc/tcg-target.h
@@ -118,7 +118,8 @@ extern bool use_vis3_instructions;
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 
-#define TCG_TARGET_HAS_trunc_shr_i32    1
+#define TCG_TARGET_HAS_extrl_i64_i32    1
+#define TCG_TARGET_HAS_extrh_i64_i32    1
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          0
diff --git a/qemu/tcg/sparc/tcg-target.c b/qemu/tcg/sparc/tcg-target.inc.c
index 1a870a81d..d641cfd8c 100644
--- a/qemu/tcg/sparc/tcg-target.c
+++ b/qemu/tcg/sparc/tcg-target.inc.c
@@ -24,7 +24,7 @@
 
 #include "tcg-be-null.h"
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "%g0",
     "%g1",
@@ -83,10 +83,8 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #define TCG_REG_T1  TCG_REG_G1
 #define TCG_REG_T2  TCG_REG_O7
 
-#ifdef CONFIG_USE_GUEST_BASE
+#ifndef CONFIG_SOFTMMU
 # define TCG_GUEST_BASE_REG TCG_REG_I5
-#else
-# define TCG_GUEST_BASE_REG TCG_REG_G0
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
@@ -291,7 +289,7 @@ static void patch_reloc(tcg_insn_unit *code_ptr, int type,
 {
     uint32_t insn;
 
-    assert(addend == 0);
+    tcg_debug_assert(addend == 0);
     value = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr);
 
     switch (type) {
@@ -955,9 +953,9 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
               INSN_IMM13(-frame_size));
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE != 0) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+#ifndef CONFIG_SOFTMMU
+    if (guest_base != 0) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
         tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
     }
 #endif
@@ -1110,7 +1108,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
     } else {
         func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SSIZE)];
     }
-    assert(func != NULL);
+    tcg_debug_assert(func != NULL);
     tcg_out_call_nodelay(s, func);
     /* delay slot */
     tcg_out_movi(s, TCG_TYPE_I32, param, oi);
@@ -1146,7 +1144,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
         addr = TCG_REG_T1;
     }
     tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
                     qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
 #endif /* CONFIG_SOFTMMU */
 }
@@ -1189,7 +1187,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
     tcg_out_mov(s, TCG_TYPE_REG, param++, data);
 
     func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)];
-    assert(func != NULL);
+    tcg_debug_assert(func != NULL);
     tcg_out_call_nodelay(s, func);
     /* delay slot */
     tcg_out_movi(s, TCG_TYPE_I32, param, oi);
@@ -1201,7 +1199,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
         addr = TCG_REG_T1;
     }
     tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
                     qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
 #endif /* CONFIG_SOFTMMU */
 }
@@ -1407,18 +1405,19 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_divu_i64:
         c = ARITH_UDIVX;
         goto gen_arith;
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
         break;
+    case INDEX_op_extu_i32_i64:
     case INDEX_op_ext32u_i64:
         tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
         break;
-    case INDEX_op_trunc_shr_i32:
-        if (a2 == 0) {
-            tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
-        } else {
-            tcg_out_arithi(s, a0, a1, a2, SHIFT_SRLX);
-        }
+    case INDEX_op_extrl_i64_i32:
+        tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
+        break;
+    case INDEX_op_extrh_i64_i32:
+        tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
         break;
 
     case INDEX_op_brcond_i64:
@@ -1531,9 +1530,12 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_neg_i64, { "R", "RJ" } },
     { INDEX_op_not_i64, { "R", "RJ" } },
 
-    { INDEX_op_ext32s_i64, { "R", "r" } },
-    { INDEX_op_ext32u_i64, { "R", "r" } },
-    { INDEX_op_trunc_shr_i32,  { "r", "R" } },
+    { INDEX_op_ext32s_i64, { "R", "R" } },
+    { INDEX_op_ext32u_i64, { "R", "R" } },
+    { INDEX_op_ext_i32_i64, { "R", "r" } },
+    { INDEX_op_extu_i32_i64, { "R", "r" } },
+    { INDEX_op_extrl_i64_i32,  { "r", "R" } },
+    { INDEX_op_extrh_i64_i32,  { "r", "R" } },
 
     { INDEX_op_brcond_i64, { "RZ", "RJ" } },
     { INDEX_op_setcond_i64, { "R", "RZ", "RJ" } },
@@ -1643,7 +1645,7 @@ void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
 
     /* We can reach the entire address space for 32-bit.  For 64-bit
        the code_gen_buffer can't be larger than 2GB.  */
-    assert(disp == (int32_t)disp);
+    tcg_debug_assert(disp == (int32_t)disp);
 
     *ptr = CALL | (uint32_t)disp >> 2;
     flush_icache_range(jmp_addr, jmp_addr + 4);
diff --git a/qemu/tcg/tcg-be-ldst.h b/qemu/tcg/tcg-be-ldst.h
index 40a2369b7..17777aec5 100644
--- a/qemu/tcg/tcg-be-ldst.h
+++ b/qemu/tcg/tcg-be-ldst.h
@@ -56,7 +56,7 @@ static inline void tcg_out_tb_init(TCGContext *s)
 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
 static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
 
-static void tcg_out_tb_finalize(TCGContext *s)
+static bool tcg_out_tb_finalize(TCGContext *s)
 {
     TCGLabelQemuLdst *lb;
 
@@ -67,7 +67,16 @@ static void tcg_out_tb_finalize(TCGContext *s)
         } else {
             tcg_out_qemu_st_slow_path(s, lb);
         }
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return false;
+        }
     }
+    return true;
 }
 
 /*
diff --git a/qemu/tcg/tcg-be-null.h b/qemu/tcg/tcg-be-null.h
index 74c57d5a6..5222fe29e 100644
--- a/qemu/tcg/tcg-be-null.h
+++ b/qemu/tcg/tcg-be-null.h
@@ -38,6 +38,7 @@ static inline void tcg_out_tb_init(TCGContext *s)
  * Generate TB finalization at the end of block
  */
 
-static inline void tcg_out_tb_finalize(TCGContext *s)
+static inline bool tcg_out_tb_finalize(TCGContext *s)
 {
+    return true;
 }
diff --git a/qemu/tcg/tcg-common.c b/qemu/tcg/tcg-common.c
new file mode 100644
index 000000000..97305a3ef
--- /dev/null
+++ b/qemu/tcg/tcg-common.c
@@ -0,0 +1,38 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "tcg/tcg.h"
+
+#if defined(CONFIG_TCG_INTERPRETER)
+uintptr_t tci_tb_ptr;
+#endif
+
+TCGOpDef tcg_op_defs[] = {
+#define DEF(s, oargs, iargs, cargs, flags) \
+         { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
+#include "tcg-opc.h"
+#undef DEF
+};
+const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
diff --git a/qemu/tcg/tcg-op.c b/qemu/tcg/tcg-op.c
index 45098c310..f554b86d4 100644
--- a/qemu/tcg/tcg-op.c
+++ b/qemu/tcg/tcg-op.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "tcg.h"
 #include "tcg-op.h"
 
@@ -1737,28 +1738,28 @@ void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
 
 /* Size changing operations.  */
 
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned count)
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
 {
-    tcg_debug_assert(count < 64);
     if (TCG_TARGET_REG_BITS == 32) {
-        if (count >= 32) {
-            tcg_gen_shri_i32(ret, TCGV_HIGH(arg), count - 32);
-        } else if (count == 0) {
-            tcg_gen_mov_i32(ret, TCGV_LOW(arg));
-        } else {
-            TCGv_i64 t = tcg_temp_new_i64();
-            tcg_gen_shri_i64(t, arg, count);
-            tcg_gen_mov_i32(ret, TCGV_LOW(t));
-            tcg_temp_free_i64(t);
-        }
-    } else if (TCG_TARGET_HAS_trunc_shr_i32) {
-        tcg_gen_op3i_i32(INDEX_op_trunc_shr_i32, ret,
-                         MAKE_TCGV_I32(GET_TCGV_I64(arg)), count);
-    } else if (count == 0) {
+        tcg_gen_mov_i32(ret, TCGV_LOW(arg));
+    } else if (TCG_TARGET_HAS_extrl_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
+    } else {
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
+    }
+}
+
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
+{
+    if (TCG_TARGET_REG_BITS == 32) {
+        tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
+    } else if (TCG_TARGET_HAS_extrh_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
     } else {
         TCGv_i64 t = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t, arg, count);
+        tcg_gen_shri_i64(t, arg, 32);
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
         tcg_temp_free_i64(t);
     }
@@ -1770,9 +1771,8 @@ void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32u_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1782,9 +1782,8 @@ void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32s_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1820,8 +1819,8 @@ void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
     } else {
-        tcg_gen_trunc_shr_i64_i32(lo, arg, 0);
-        tcg_gen_trunc_shr_i64_i32(hi, arg, 32);
+        tcg_gen_extrl_i64_i32(lo, arg);
+        tcg_gen_extrh_i64_i32(hi, arg);
     }
 }
 
diff --git a/qemu/tcg/tcg-op.h b/qemu/tcg/tcg-op.h
index d1d763f6f..c446d3dc7 100644
--- a/qemu/tcg/tcg-op.h
+++ b/qemu/tcg/tcg-op.h
@@ -684,7 +684,8 @@ static inline void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg)
 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high);
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned int c);
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg);
 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg);
 
@@ -693,28 +694,59 @@ static inline void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi)
     tcg_gen_deposit_i64(ret, lo, hi, 32, 32);
 }
 
-static inline void tcg_gen_trunc_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
-{
-    tcg_gen_trunc_shr_i64_i32(ret, arg, 0);
-}
-
 /* QEMU specific operations.  */
 
 #ifndef TARGET_LONG_BITS
 #error must include QEMU headers
 #endif
 
-/* debug info: write the PC of the corresponding QEMU CPU instruction */
-static inline void tcg_gen_debug_insn_start(uint64_t pc)
+#if TARGET_INSN_START_WORDS == 1
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc)
+{
+    tcg_gen_op1(&tcg_ctx, INDEX_op_insn_start, pc);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc)
+{
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 2
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
 {
-    /* XXX: must really use a 32 bit size for TCGArg in all cases */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-    tcg_gen_op2ii(INDEX_op_debug_insn_start,
-                  (uint32_t)(pc), (uint32_t)(pc >> 32));
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start, pc, a1);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
+{
+    tcg_gen_op4(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 3
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op3(&tcg_ctx, INDEX_op_insn_start, pc, a1, a2);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op6(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32),
+                (uint32_t)a2, (uint32_t)(a2 >> 32));
+}
+# endif
 #else
-    tcg_gen_op1i(INDEX_op_debug_insn_start, pc);
+# error "Unhandled number of operands to insn_start"
 #endif
-}
 
 static inline void tcg_gen_exit_tb(uintptr_t val)
 {
@@ -724,7 +756,6 @@ static inline void tcg_gen_exit_tb(uintptr_t val)
 void tcg_gen_goto_tb(unsigned idx);
 
 #if TARGET_LONG_BITS == 32
-#define TCGv TCGv_i32
 #define tcg_temp_new() tcg_temp_new_i32()
 #define tcg_global_reg_new tcg_global_reg_new_i32
 #define tcg_global_mem_new tcg_global_mem_new_i32
@@ -736,7 +767,6 @@ void tcg_gen_goto_tb(unsigned idx);
 #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32
 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32
 #else
-#define TCGv TCGv_i64
 #define tcg_temp_new() tcg_temp_new_i64()
 #define tcg_global_reg_new tcg_global_reg_new_i64
 #define tcg_global_mem_new tcg_global_mem_new_i64
@@ -853,7 +883,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_divu_tl tcg_gen_divu_i64
 #define tcg_gen_remu_tl tcg_gen_remu_i64
 #define tcg_gen_discard_tl tcg_gen_discard_i64
-#define tcg_gen_trunc_tl_i32 tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_tl_i32 tcg_gen_extrl_i64_i32
 #define tcg_gen_trunc_i64_tl tcg_gen_mov_i64
 #define tcg_gen_extu_i32_tl tcg_gen_extu_i32_i64
 #define tcg_gen_ext_i32_tl tcg_gen_ext_i32_i64
@@ -932,7 +962,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_remu_tl tcg_gen_remu_i32
 #define tcg_gen_discard_tl tcg_gen_discard_i32
 #define tcg_gen_trunc_tl_i32 tcg_gen_mov_i32
-#define tcg_gen_trunc_i64_tl tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_i64_tl tcg_gen_extrl_i64_i32
 #define tcg_gen_extu_i32_tl tcg_gen_mov_i32
 #define tcg_gen_ext_i32_tl tcg_gen_mov_i32
 #define tcg_gen_extu_tl_i64 tcg_gen_extu_i32_i64
diff --git a/qemu/tcg/tcg-opc.h b/qemu/tcg/tcg-opc.h
index 13ccb60a5..6d0410c4b 100644
--- a/qemu/tcg/tcg-opc.h
+++ b/qemu/tcg/tcg-opc.h
@@ -138,8 +138,14 @@ DEF(rotl_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(rotr_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(deposit_i64, 1, 2, 2, IMPL64 | IMPL(TCG_TARGET_HAS_deposit_i64))
 
-DEF(trunc_shr_i32, 1, 1, 1,
-    IMPL(TCG_TARGET_HAS_trunc_shr_i32)
+/* size changing ops */
+DEF(ext_i32_i64, 1, 1, 0, IMPL64)
+DEF(extu_i32_i64, 1, 1, 0, IMPL64)
+DEF(extrl_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrl_i64_i32)
+    | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
+DEF(extrh_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrh_i64_i32)
     | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
 
 DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | IMPL64)
@@ -167,18 +173,15 @@ DEF(muls2_i64, 2, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_muls2_i64))
 DEF(muluh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_muluh_i64))
 DEF(mulsh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_mulsh_i64))
 
+#define TLADDR_ARGS  (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
+#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
+
 /* QEMU specific */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-DEF(debug_insn_start, 0, 0, 2, TCG_OPF_NOT_PRESENT)
-#else
-DEF(debug_insn_start, 0, 0, 1, TCG_OPF_NOT_PRESENT)
-#endif
+DEF(insn_start, 0, 0, TLADDR_ARGS * TARGET_INSN_START_WORDS,
+    TCG_OPF_NOT_PRESENT)
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 
-#define TLADDR_ARGS    (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
-#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
-
 DEF(qemu_ld_i32, 1, TLADDR_ARGS, 1,
     TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
 DEF(qemu_st_i32, 0, TLADDR_ARGS + 1, 1,
diff --git a/qemu/tcg/tcg.c b/qemu/tcg/tcg.c
index 0892a9bbf..796addd1f 100644
--- a/qemu/tcg/tcg.c
+++ b/qemu/tcg/tcg.c
@@ -26,17 +26,12 @@
 #define USE_LIVENESS_ANALYSIS
 #define USE_TCG_OPTIMIZATIONS
 
-#include "config.h"
+#include "qemu/osdep.h"
 
 /* Define to jump the ELF file used to communicate with GDB.  */
 #undef DEBUG_JIT
 
-#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
-/* define it to suppress various consistency checks (faster) */
-#define NDEBUG
-#endif
-
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/host-utils.h"
 #include "qemu/timer.h"
 
@@ -60,8 +55,10 @@
 #endif
 
 #include "elf.h"
+#include "exec/log.h"
 
-/* Forward declarations for functions declared in tcg-target.c and used here. */
+/* Forward declarations for functions declared in tcg-target.inc.c and
+   used here. */
 static void tcg_target_init(TCGContext *s);
 static void tcg_target_qemu_prologue(TCGContext *s);
 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
@@ -95,7 +92,7 @@ static void tcg_register_jit_int(void *buf, size_t size,
                                  size_t debug_frame_size)
     __attribute__((unused));
 
-/* Forward declarations for functions declared and used in tcg-target.c. */
+/* Forward declarations for functions declared and used in tcg-target.inc.c. */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
                        intptr_t arg2);
@@ -110,15 +107,9 @@ static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
 static int tcg_target_const_match(tcg_target_long val, TCGType type,
                                   const TCGArgConstraint *arg_ct);
 static void tcg_out_tb_init(TCGContext *s);
-static void tcg_out_tb_finalize(TCGContext *s);
+static bool tcg_out_tb_finalize(TCGContext *s);
 
 
-TCGOpDef tcg_op_defs[] = {
-#define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
-#include "tcg-opc.h"
-#undef DEF
-};
-const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
 
 static TCGRegSet tcg_target_available_regs[2];
 static TCGRegSet tcg_target_call_clobber_regs;
@@ -233,7 +224,7 @@ static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
     intptr_t value = (intptr_t)ptr;
     TCGRelocation *r;
 
-    assert(!l->has_value);
+    tcg_debug_assert(!l->has_value);
 
     for (r = l->u.first_reloc; r != NULL; r = r->next) {
         patch_reloc(r->ptr, r->type, value, r->addend);
@@ -255,7 +246,7 @@ TCGLabel *gen_new_label(void)
     return l;
 }
 
-#include "tcg-target.c"
+#include "tcg-target.inc.c"
 
 /* pool based memory allocation */
 void *tcg_malloc_internal(TCGContext *s, int size)
@@ -323,6 +314,8 @@ static const TCGHelperInfo all_helpers[] = {
 #include "exec/helper-tcg.h"
 };
 
+static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
+
 void tcg_context_init(TCGContext *s)
 {
     int op, total_args, n, i;
@@ -365,34 +358,64 @@ void tcg_context_init(TCGContext *s)
     }
 
     tcg_target_init(s);
+
+    /* Reverse the order of the saved registers, assuming they're all at
+       the start of tcg_target_reg_alloc_order.  */
+    for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
+        int r = tcg_target_reg_alloc_order[n];
+        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
+            break;
+        }
+    }
+    for (i = 0; i < n; ++i) {
+        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
+    }
+    for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
+        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
+    }
 }
 
 void tcg_prologue_init(TCGContext *s)
 {
-    /* init global prologue and epilogue */
-    s->code_buf = s->code_gen_prologue;
-    s->code_ptr = s->code_buf;
+    size_t prologue_size, total_size;
+    void *buf0, *buf1;
+
+    /* Put the prologue at the beginning of code_gen_buffer.  */
+    buf0 = s->code_gen_buffer;
+    s->code_ptr = buf0;
+    s->code_buf = buf0;
+    s->code_gen_prologue = buf0;
+
+    /* Generate the prologue.  */
     tcg_target_qemu_prologue(s);
-    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+    buf1 = s->code_ptr;
+    flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
+
+    /* Deduct the prologue from the buffer.  */
+    prologue_size = tcg_current_code_size(s);
+    s->code_gen_ptr = buf1;
+    s->code_gen_buffer = buf1;
+    s->code_buf = buf1;
+    total_size = s->code_gen_buffer_size - prologue_size;
+    s->code_gen_buffer_size = total_size;
+
+    /* Compute a high-water mark, at which we voluntarily flush the buffer
+       and start over.  The size here is arbitrary, significantly larger
+       than we expect the code generation for any one opcode to require.  */
+    s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024);
+
+    tcg_register_jit(s->code_gen_buffer, total_size);
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        size_t size = tcg_current_code_size(s);
-        qemu_log("PROLOGUE: [size=%zu]\n", size);
-        log_disas(s->code_buf, size);
+        qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
+        log_disas(buf0, prologue_size);
         qemu_log("\n");
         qemu_log_flush();
     }
 #endif
 }
 
-void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)
-{
-    s->frame_start = start;
-    s->frame_end = start + size;
-    s->frame_reg = reg;
-}
-
 void tcg_func_start(TCGContext *s)
 {
     tcg_pool_reset(s);
@@ -416,128 +439,133 @@ void tcg_func_start(TCGContext *s)
     s->be = tcg_malloc(sizeof(TCGBackendData));
 }
 
-static inline void tcg_temp_alloc(TCGContext *s, int n)
+static inline int temp_idx(TCGContext *s, TCGTemp *ts)
 {
-    if (n > TCG_MAX_TEMPS)
-        tcg_abort();
+    ptrdiff_t n = ts - s->temps;
+    tcg_debug_assert(n >= 0 && n < s->nb_temps);
+    return n;
 }
 
-static inline int tcg_global_reg_new_internal(TCGType type, int reg,
-                                              const char *name)
+static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
+{
+    int n = s->nb_temps++;
+    tcg_debug_assert(n < TCG_MAX_TEMPS);
+    return memset(&s->temps[n], 0, sizeof(TCGTemp));
+}
+
+static inline TCGTemp *tcg_global_alloc(TCGContext *s)
+{
+    tcg_debug_assert(s->nb_globals == s->nb_temps);
+    s->nb_globals++;
+    return tcg_temp_alloc(s);
+}
+
+static int tcg_global_reg_new_internal(TCGContext *s, TCGType type,
+                                       TCGReg reg, const char *name)
 {
-    TCGContext *s = &tcg_ctx;
     TCGTemp *ts;
-    int idx;
 
-#if TCG_TARGET_REG_BITS == 32
-    if (type != TCG_TYPE_I32)
-        tcg_abort();
-#endif
-    if (tcg_regset_test_reg(s->reserved_regs, reg))
+    if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
         tcg_abort();
-    idx = s->nb_globals;
-    tcg_temp_alloc(s, s->nb_globals + 1);
-    ts = &s->temps[s->nb_globals];
+    }
+
+    ts = tcg_global_alloc(s);
     ts->base_type = type;
     ts->type = type;
     ts->fixed_reg = 1;
     ts->reg = reg;
     ts->name = name;
-    s->nb_globals++;
     tcg_regset_set_reg(s->reserved_regs, reg);
-    return idx;
+
+    return temp_idx(s, ts);
 }
 
-TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name)
+void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
 {
     int idx;
+    s->frame_start = start;
+    s->frame_end = start + size;
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
+    s->frame_temp = &s->temps[idx];
+}
 
-    idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name);
+TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name)
+{
+    TCGContext *s = &tcg_ctx;
+    int idx;
+
+    if (tcg_regset_test_reg(s->reserved_regs, reg)) {
+        tcg_abort();
+    }
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name);
     return MAKE_TCGV_I32(idx);
 }
 
-TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)
+TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name)
 {
+    TCGContext *s = &tcg_ctx;
     int idx;
 
-    idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name);
+    if (tcg_regset_test_reg(s->reserved_regs, reg)) {
+        tcg_abort();
+    }
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name);
     return MAKE_TCGV_I64(idx);
 }
 
-static inline int tcg_global_mem_new_internal(TCGType type, int reg,
-                                              intptr_t offset,
-                                              const char *name)
+int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
+                                intptr_t offset, const char *name)
 {
     TCGContext *s = &tcg_ctx;
-    TCGTemp *ts;
-    int idx;
+    TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)];
+    TCGTemp *ts = tcg_global_alloc(s);
+    int indirect_reg = 0, bigendian = 0;
+#ifdef HOST_WORDS_BIGENDIAN
+    bigendian = 1;
+#endif
 
-    idx = s->nb_globals;
-#if TCG_TARGET_REG_BITS == 32
-    if (type == TCG_TYPE_I64) {
+    if (!base_ts->fixed_reg) {
+        indirect_reg = 1;
+        base_ts->indirect_base = 1;
+    }
+
+    if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
+        TCGTemp *ts2 = tcg_global_alloc(s);
         char buf[64];
-        tcg_temp_alloc(s, s->nb_globals + 2);
-        ts = &s->temps[s->nb_globals];
-        ts->base_type = type;
+
+        ts->base_type = TCG_TYPE_I64;
         ts->type = TCG_TYPE_I32;
-        ts->fixed_reg = 0;
+        ts->indirect_reg = indirect_reg;
         ts->mem_allocated = 1;
-        ts->mem_reg = reg;
-#ifdef HOST_WORDS_BIGENDIAN
-        ts->mem_offset = offset + 4;
-#else
-        ts->mem_offset = offset;
-#endif
+        ts->mem_base = base_ts;
+        ts->mem_offset = offset + bigendian * 4;
         pstrcpy(buf, sizeof(buf), name);
         pstrcat(buf, sizeof(buf), "_0");
         ts->name = strdup(buf);
-        ts++;
 
-        ts->base_type = type;
-        ts->type = TCG_TYPE_I32;
-        ts->fixed_reg = 0;
-        ts->mem_allocated = 1;
-        ts->mem_reg = reg;
-#ifdef HOST_WORDS_BIGENDIAN
-        ts->mem_offset = offset;
-#else
-        ts->mem_offset = offset + 4;
-#endif
+        tcg_debug_assert(ts2 == ts + 1);
+        ts2->base_type = TCG_TYPE_I64;
+        ts2->type = TCG_TYPE_I32;
+        ts2->indirect_reg = indirect_reg;
+        ts2->mem_allocated = 1;
+        ts2->mem_base = base_ts;
+        ts2->mem_offset = offset + (1 - bigendian) * 4;
         pstrcpy(buf, sizeof(buf), name);
         pstrcat(buf, sizeof(buf), "_1");
         ts->name = strdup(buf);
-
-        s->nb_globals += 2;
-    } else
-#endif
-    {
-        tcg_temp_alloc(s, s->nb_globals + 1);
-        ts = &s->temps[s->nb_globals];
+    } else {
         ts->base_type = type;
         ts->type = type;
-        ts->fixed_reg = 0;
+        ts->indirect_reg = indirect_reg;
         ts->mem_allocated = 1;
-        ts->mem_reg = reg;
+        ts->mem_base = base_ts;
         ts->mem_offset = offset;
         ts->name = name;
-        s->nb_globals++;
     }
-    return idx;
-}
-
-TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)
-{
-    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
-    return MAKE_TCGV_I32(idx);
+    return temp_idx(s, ts);
 }
 
-TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)
-{
-    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
-    return MAKE_TCGV_I64(idx);
-}
-
-static inline int tcg_temp_new_internal(TCGType type, int temp_local)
+static int tcg_temp_new_internal(TCGType type, int temp_local)
 {
     TCGContext *s = &tcg_ctx;
     TCGTemp *ts;
@@ -551,38 +579,30 @@ static inline int tcg_temp_new_internal(TCGType type, int temp_local)
 
         ts = &s->temps[idx];
         ts->temp_allocated = 1;
-        assert(ts->base_type == type);
-        assert(ts->temp_local == temp_local);
+        tcg_debug_assert(ts->base_type == type);
+        tcg_debug_assert(ts->temp_local == temp_local);
     } else {
-        idx = s->nb_temps;
-#if TCG_TARGET_REG_BITS == 32
-        if (type == TCG_TYPE_I64) {
-            tcg_temp_alloc(s, s->nb_temps + 2);
-            ts = &s->temps[s->nb_temps];
-            ts->base_type = type;
-            ts->type = TCG_TYPE_I32;
-            ts->temp_allocated = 1;
-            ts->temp_local = temp_local;
-            ts->name = NULL;
-            ts++;
+        ts = tcg_temp_alloc(s);
+        if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
+            TCGTemp *ts2 = tcg_temp_alloc(s);
+
             ts->base_type = type;
             ts->type = TCG_TYPE_I32;
             ts->temp_allocated = 1;
             ts->temp_local = temp_local;
-            ts->name = NULL;
-            s->nb_temps += 2;
-        } else
-#endif
-        {
-            tcg_temp_alloc(s, s->nb_temps + 1);
-            ts = &s->temps[s->nb_temps];
+
+            tcg_debug_assert(ts2 == ts + 1);
+            ts2->base_type = TCG_TYPE_I64;
+            ts2->type = TCG_TYPE_I32;
+            ts2->temp_allocated = 1;
+            ts2->temp_local = temp_local;
+        } else {
             ts->base_type = type;
             ts->type = type;
             ts->temp_allocated = 1;
             ts->temp_local = temp_local;
-            ts->name = NULL;
-            s->nb_temps++;
         }
+        idx = temp_idx(s, ts);
     }
 
 #if defined(CONFIG_DEBUG_TCG)
@@ -620,9 +640,9 @@ static void tcg_temp_free_internal(int idx)
     }
 #endif
 
-    assert(idx >= s->nb_globals && idx < s->nb_temps);
+    tcg_debug_assert(idx >= s->nb_globals && idx < s->nb_temps);
     ts = &s->temps[idx];
-    assert(ts->temp_allocated != 0);
+    tcg_debug_assert(ts->temp_allocated != 0);
     ts->temp_allocated = 0;
 
     k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
@@ -901,37 +921,30 @@ static void tcg_reg_alloc_start(TCGContext *s)
         ts->mem_allocated = 0;
         ts->fixed_reg = 0;
     }
-    for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
-        s->reg_to_temp[i] = -1;
-    }
+
+    memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
 }
 
-static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,
-                                 int idx)
+static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
+                                 TCGTemp *ts)
 {
-    TCGTemp *ts;
+    int idx = temp_idx(s, ts);
 
-    assert(idx >= 0 && idx < s->nb_temps);
-    ts = &s->temps[idx];
     if (idx < s->nb_globals) {
         pstrcpy(buf, buf_size, ts->name);
+    } else if (ts->temp_local) {
+        snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
     } else {
-        if (ts->temp_local) 
-            snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
-        else
-            snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
+        snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
     }
     return buf;
 }
 
-char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg)
-{
-    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg));
-}
-
-char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)
+static char *tcg_get_arg_str_idx(TCGContext *s, char *buf,
+                                 int buf_size, int idx)
 {
-    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));
+    tcg_debug_assert(idx >= 0 && idx < s->nb_temps);
+    return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]);
 }
 
 /* Find helper name.  */
@@ -996,17 +1009,18 @@ void tcg_dump_ops(TCGContext *s)
         def = &tcg_op_defs[c];
         args = &s->gen_opparam_buf[op->args];
 
-        if (c == INDEX_op_debug_insn_start) {
-            uint64_t pc;
+        if (c == INDEX_op_insn_start) {
+            qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : "");
+
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-            pc = ((uint64_t)args[1] << 32) | args[0];
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
 #else
-            pc = args[0];
+                a = args[i];
 #endif
-            if (oi != s->gen_first_op_idx) {
-                qemu_log("\n");
+                qemu_log(" " TARGET_FMT_lx, a);
             }
-            qemu_log(" ---- 0x%" PRIx64, pc);
         } else if (c == INDEX_op_call) {
             /* variable number of arguments */
             nb_oargs = op->callo;
@@ -1172,25 +1186,25 @@ void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
         if (tdefs->op == (TCGOpcode)-1)
             break;
         op = tdefs->op;
-        assert((unsigned)op < NB_OPS);
+        tcg_debug_assert((unsigned)op < NB_OPS);
         def = &tcg_op_defs[op];
 #if defined(CONFIG_DEBUG_TCG)
         /* Duplicate entry in op definitions? */
-        assert(!def->used);
+        tcg_debug_assert(!def->used);
         def->used = 1;
 #endif
         nb_args = def->nb_iargs + def->nb_oargs;
         for(i = 0; i < nb_args; i++) {
             ct_str = tdefs->args_ct_str[i];
             /* Incomplete TCGTargetOpDef entry? */
-            assert(ct_str != NULL);
+            tcg_debug_assert(ct_str != NULL);
             tcg_regset_clear(def->args_ct[i].u.regs);
             def->args_ct[i].ct = 0;
             if (ct_str[0] >= '0' && ct_str[0] <= '9') {
                 int oarg;
                 oarg = ct_str[0] - '0';
-                assert(oarg < def->nb_oargs);
-                assert(def->args_ct[oarg].ct & TCG_CT_REG);
+                tcg_debug_assert(oarg < def->nb_oargs);
+                tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG);
                 /* TCG_CT_ALIAS is for the output arguments. The input
                    argument is tagged with TCG_CT_IALIAS. */
                 def->args_ct[i] = def->args_ct[oarg];
@@ -1219,7 +1233,7 @@ void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
         }
 
         /* TCGTargetOpDef entry with too much information? */
-        assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
+        tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
 
         /* sort the constraints (XXX: this is just an heuristic) */
         sort_constraints(def, 0, def->nb_oargs);
@@ -1240,7 +1254,7 @@ void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
 
 #if defined(CONFIG_DEBUG_TCG)
     i = 0;
-    for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
+    for (op = 0; op < tcg_op_defs_max; op++) {
         const TCGOpDef *def = &tcg_op_defs[op];
         if (def->flags & TCG_OPF_NOT_PRESENT) {
             /* Wrong entry in op definitions? */
@@ -1396,7 +1410,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                             }
                         }
                     }
-                    /* input arguments are live for preceeding opcodes */
+                    /* input arguments are live for preceding opcodes */
                     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
                         arg = args[i];
                         dead_temps[arg] = 0;
@@ -1406,7 +1420,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                 }
             }
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
             break;
         case INDEX_op_discard:
             /* mark the temporary as dead */
@@ -1542,7 +1556,7 @@ static void tcg_liveness_analysis(TCGContext *s)
                         dead_args |= (1 << i);
                     }
                 }
-                /* input arguments are live for preceeding opcodes */
+                /* input arguments are live for preceding opcodes */
                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
                     arg = args[i];
                     dead_temps[arg] = 0;
@@ -1558,8 +1572,7 @@ static void tcg_liveness_analysis(TCGContext *s)
 /* dummy liveness analysis */
 static void tcg_liveness_analysis(TCGContext *s)
 {
-    int nb_ops;
-    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
+    int nb_ops = s->gen_next_op_idx;
 
     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
     memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
@@ -1568,7 +1581,7 @@ static void tcg_liveness_analysis(TCGContext *s)
 }
 #endif
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static void dump_regs(TCGContext *s)
 {
     TCGTemp *ts;
@@ -1583,7 +1596,8 @@ static void dump_regs(TCGContext *s)
             printf("%s", tcg_target_reg_names[ts->reg]);
             break;
         case TEMP_VAL_MEM:
-            printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]);
+            printf("%d(%s)", (int)ts->mem_offset,
+                   tcg_target_reg_names[ts->mem_base->reg]);
             break;
         case TEMP_VAL_CONST:
             printf("$0x%" TCG_PRIlx, ts->val);
@@ -1599,43 +1613,41 @@ static void dump_regs(TCGContext *s)
     }
 
     for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
-        if (s->reg_to_temp[i] >= 0) {
+        if (s->reg_to_temp[i] != NULL) {
             printf("%s: %s\n", 
                    tcg_target_reg_names[i], 
-                   tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i]));
+                   tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
         }
     }
 }
 
 static void check_regs(TCGContext *s)
 {
-    int reg, k;
+    int reg;
+    int k;
     TCGTemp *ts;
     char buf[64];
 
-    for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-        k = s->reg_to_temp[reg];
-        if (k >= 0) {
-            ts = &s->temps[k];
-            if (ts->val_type != TEMP_VAL_REG ||
-                ts->reg != reg) {
+    for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
+        ts = s->reg_to_temp[reg];
+        if (ts != NULL) {
+            if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
                 printf("Inconsistency for register %s:\n", 
                        tcg_target_reg_names[reg]);
                 goto fail;
             }
         }
     }
-    for(k = 0; k < s->nb_temps; k++) {
+    for (k = 0; k < s->nb_temps; k++) {
         ts = &s->temps[k];
-        if (ts->val_type == TEMP_VAL_REG &&
-            !ts->fixed_reg &&
-            s->reg_to_temp[ts->reg] != k) {
-                printf("Inconsistency for temp %s:\n", 
-                       tcg_get_arg_str_idx(s, buf, sizeof(buf), k));
+        if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
+            && s->reg_to_temp[ts->reg] != ts) {
+            printf("Inconsistency for temp %s:\n",
+                   tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
         fail:
-                printf("reg state:\n");
-                dump_regs(s);
-                tcg_abort();
+            printf("reg state:\n");
+            dump_regs(s);
+            tcg_abort();
         }
     }
 }
@@ -1656,62 +1668,69 @@ static void temp_allocate_frame(TCGContext *s, int temp)
         tcg_abort();
     }
     ts->mem_offset = s->current_frame_offset;
-    ts->mem_reg = s->frame_reg;
+    ts->mem_base = s->frame_temp;
     ts->mem_allocated = 1;
     s->current_frame_offset += sizeof(tcg_target_long);
 }
 
+static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet);
+
 /* sync register 'reg' by saving it to the corresponding temporary */
-static inline void tcg_reg_sync(TCGContext *s, int reg)
+static void tcg_reg_sync(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
-    int temp;
+    TCGTemp *ts = s->reg_to_temp[reg];
 
-    temp = s->reg_to_temp[reg];
-    ts = &s->temps[temp];
-    assert(ts->val_type == TEMP_VAL_REG);
+    tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
     if (!ts->mem_coherent && !ts->fixed_reg) {
         if (!ts->mem_allocated) {
-            temp_allocate_frame(s, temp);
+            temp_allocate_frame(s, temp_idx(s, ts));
+        } else if (ts->indirect_reg) {
+            tcg_regset_set_reg(allocated_regs, ts->reg);
+            temp_load(s, ts->mem_base,
+                      tcg_target_available_regs[TCG_TYPE_PTR],
+                      allocated_regs);
         }
-        tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
+        tcg_out_st(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
     }
     ts->mem_coherent = 1;
 }
 
 /* free register 'reg' by spilling the corresponding temporary if necessary */
-static void tcg_reg_free(TCGContext *s, int reg)
+static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
 {
-    int temp;
+    TCGTemp *ts = s->reg_to_temp[reg];
 
-    temp = s->reg_to_temp[reg];
-    if (temp != -1) {
-        tcg_reg_sync(s, reg);
-        s->temps[temp].val_type = TEMP_VAL_MEM;
-        s->reg_to_temp[reg] = -1;
+    if (ts != NULL) {
+        tcg_reg_sync(s, reg, allocated_regs);
+        ts->val_type = TEMP_VAL_MEM;
+        s->reg_to_temp[reg] = NULL;
     }
 }
 
 /* Allocate a register belonging to reg1 & ~reg2 */
-static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
+static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs,
+                            TCGRegSet allocated_regs, bool rev)
 {
-    int i, reg;
+    int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
+    const int *order;
+    TCGReg reg;
     TCGRegSet reg_ct;
 
-    tcg_regset_andnot(reg_ct, reg1, reg2);
+    tcg_regset_andnot(reg_ct, desired_regs, allocated_regs);
+    order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
 
     /* first try free registers */
-    for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
-        reg = tcg_target_reg_alloc_order[i];
-        if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1)
+    for(i = 0; i < n; i++) {
+        reg = order[i];
+        if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL)
             return reg;
     }
 
     /* XXX: do better spill choice */
-    for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
-        reg = tcg_target_reg_alloc_order[i];
+    for(i = 0; i < n; i++) {
+        reg = order[i];
         if (tcg_regset_test_reg(reg_ct, reg)) {
-            tcg_reg_free(s, reg);
+            tcg_reg_free(s, reg, allocated_regs);
             return reg;
         }
     }
@@ -1719,65 +1738,92 @@ static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
     tcg_abort();
 }
 
-/* mark a temporary as dead. */
-static inline void temp_dead(TCGContext *s, int temp)
+/* Make sure the temporary is in a register.  If needed, allocate the register
+   from DESIRED while avoiding ALLOCATED.  */
+static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
+                      TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
+    TCGReg reg;
 
-    ts = &s->temps[temp];
-    if (!ts->fixed_reg) {
-        if (ts->val_type == TEMP_VAL_REG) {
-            s->reg_to_temp[ts->reg] = -1;
-        }
-        if (temp < s->nb_globals || ts->temp_local) {
-            ts->val_type = TEMP_VAL_MEM;
-        } else {
-            ts->val_type = TEMP_VAL_DEAD;
+    switch (ts->val_type) {
+    case TEMP_VAL_REG:
+        return;
+    case TEMP_VAL_CONST:
+        reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
+        tcg_out_movi(s, ts->type, reg, ts->val);
+        ts->mem_coherent = 0;
+        break;
+    case TEMP_VAL_MEM:
+        reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
+        if (ts->indirect_reg) {
+            tcg_regset_set_reg(allocated_regs, reg);
+            temp_load(s, ts->mem_base,
+                      tcg_target_available_regs[TCG_TYPE_PTR],
+                      allocated_regs);
         }
+        tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
+        ts->mem_coherent = 1;
+        break;
+    case TEMP_VAL_DEAD:
+    default:
+        tcg_abort();
     }
+    ts->reg = reg;
+    ts->val_type = TEMP_VAL_REG;
+    s->reg_to_temp[reg] = ts;
+}
+
+/* mark a temporary as dead. */
+static inline void temp_dead(TCGContext *s, TCGTemp *ts)
+{
+    if (ts->fixed_reg) {
+        return;
+    }
+    if (ts->val_type == TEMP_VAL_REG) {
+        s->reg_to_temp[ts->reg] = NULL;
+    }
+    ts->val_type = (temp_idx(s, ts) < s->nb_globals || ts->temp_local
+                    ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
 }
 
 /* sync a temporary to memory. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
-static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs)
+static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
-
-    ts = &s->temps[temp];
-    if (!ts->fixed_reg) {
-        switch(ts->val_type) {
-        case TEMP_VAL_CONST:
-            ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
-                                    allocated_regs);
-            ts->val_type = TEMP_VAL_REG;
-            s->reg_to_temp[ts->reg] = temp;
-            ts->mem_coherent = 0;
-            tcg_out_movi(s, ts->type, ts->reg, ts->val);
-            /* fallthrough*/
-        case TEMP_VAL_REG:
-            tcg_reg_sync(s, ts->reg);
-            break;
-        case TEMP_VAL_DEAD:
-        case TEMP_VAL_MEM:
-            break;
-        default:
-            tcg_abort();
-        }
+    if (ts->fixed_reg) {
+        return;
+    }
+    switch (ts->val_type) {
+    case TEMP_VAL_CONST:
+        temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs);
+        /* fallthrough */
+    case TEMP_VAL_REG:
+        tcg_reg_sync(s, ts->reg, allocated_regs);
+        break;
+    case TEMP_VAL_DEAD:
+    case TEMP_VAL_MEM:
+        break;
+    default:
+        tcg_abort();
     }
 }
 
 /* save a temporary to memory. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
-static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
+static inline void temp_save(TCGContext *s, TCGTemp *ts,
+                             TCGRegSet allocated_regs)
 {
 #ifdef USE_LIVENESS_ANALYSIS
-    /* The liveness analysis already ensures that globals are back
-       in memory. Keep an assert for safety. */
-    assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg);
-#else
-    temp_sync(s, temp, allocated_regs);
-    temp_dead(s, temp);
+    /* ??? Liveness does not yet incorporate indirect bases.  */
+    if (!ts->indirect_base) {
+        /* The liveness analysis already ensures that globals are back
+           in memory. Keep an tcg_debug_assert for safety. */
+        tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
+        return;
+    }
 #endif
+    temp_sync(s, ts, allocated_regs);
+    temp_dead(s, ts);
 }
 
 /* save globals to their canonical location and assume they can be
@@ -1787,8 +1833,8 @@ static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
 {
     int i;
 
-    for(i = 0; i < s->nb_globals; i++) {
-        temp_save(s, i, allocated_regs);
+    for (i = 0; i < s->nb_globals; i++) {
+        temp_save(s, &s->temps[i], allocated_regs);
     }
 }
 
@@ -1800,12 +1846,17 @@ static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
     int i;
 
     for (i = 0; i < s->nb_globals; i++) {
+        TCGTemp *ts = &s->temps[i];
 #ifdef USE_LIVENESS_ANALYSIS
-        assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg ||
-               s->temps[i].mem_coherent);
-#else
-        temp_sync(s, i, allocated_regs);
+        /* ??? Liveness does not yet incorporate indirect bases.  */
+        if (!ts->indirect_base) {
+            tcg_debug_assert(ts->val_type != TEMP_VAL_REG
+                             || ts->fixed_reg
+                             || ts->mem_coherent);
+            continue;
+        }
 #endif
+        temp_sync(s, ts, allocated_regs);
     }
 }
 
@@ -1813,21 +1864,23 @@ static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
    all globals are stored at their canonical location. */
 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
     int i;
 
-    for(i = s->nb_globals; i < s->nb_temps; i++) {
-        ts = &s->temps[i];
+    for (i = s->nb_globals; i < s->nb_temps; i++) {
+        TCGTemp *ts = &s->temps[i];
         if (ts->temp_local) {
-            temp_save(s, i, allocated_regs);
+            temp_save(s, ts, allocated_regs);
         } else {
 #ifdef USE_LIVENESS_ANALYSIS
-            /* The liveness analysis already ensures that temps are dead.
-               Keep an assert for safety. */
-            assert(ts->val_type == TEMP_VAL_DEAD);
-#else
-            temp_dead(s, i);
+            /* ??? Liveness does not yet incorporate indirect bases.  */
+            if (!ts->indirect_base) {
+                /* The liveness analysis already ensures that temps are dead.
+                   Keep an tcg_debug_assert for safety. */
+                tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
+                continue;
+            }
 #endif
+            temp_dead(s, ts);
         }
     }
 
@@ -1852,16 +1905,17 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
         tcg_out_movi(s, ots->type, ots->reg, val);
     } else {
         /* The movi is not explicitly generated here */
-        if (ots->val_type == TEMP_VAL_REG)
-            s->reg_to_temp[ots->reg] = -1;
+        if (ots->val_type == TEMP_VAL_REG) {
+            s->reg_to_temp[ots->reg] = NULL;
+        }
         ots->val_type = TEMP_VAL_CONST;
         ots->val = val;
     }
     if (NEED_SYNC_ARG(0)) {
-        temp_sync(s, args[0], s->reserved_regs);
+        temp_sync(s, ots, s->reserved_regs);
     }
     if (IS_DEAD_ARG(0)) {
-        temp_dead(s, args[0]);
+        temp_dead(s, ots);
     }
 }
 
@@ -1887,69 +1941,65 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
        we don't have to reload SOURCE the next time it is used. */
     if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
         || ts->val_type == TEMP_VAL_MEM) {
-        ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[itype],
-                                allocated_regs);
-        if (ts->val_type == TEMP_VAL_MEM) {
-            tcg_out_ld(s, itype, ts->reg, ts->mem_reg, ts->mem_offset);
-            ts->mem_coherent = 1;
-        } else if (ts->val_type == TEMP_VAL_CONST) {
-            tcg_out_movi(s, itype, ts->reg, ts->val);
-            ts->mem_coherent = 0;
-        }
-        s->reg_to_temp[ts->reg] = args[1];
-        ts->val_type = TEMP_VAL_REG;
+        temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs);
     }
 
     if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
         /* mov to a non-saved dead register makes no sense (even with
            liveness analysis disabled). */
-        assert(NEED_SYNC_ARG(0));
+        tcg_debug_assert(NEED_SYNC_ARG(0));
         /* The code above should have moved the temp to a register. */
-        assert(ts->val_type == TEMP_VAL_REG);
+        tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
         if (!ots->mem_allocated) {
             temp_allocate_frame(s, args[0]);
         }
-        tcg_out_st(s, otype, ts->reg, ots->mem_reg, ots->mem_offset);
+        if (ots->indirect_reg) {
+            tcg_regset_set_reg(allocated_regs, ts->reg);
+            temp_load(s, ots->mem_base,
+                      tcg_target_available_regs[TCG_TYPE_PTR],
+                      allocated_regs);
+        }
+        tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
         if (IS_DEAD_ARG(1)) {
-            temp_dead(s, args[1]);
+            temp_dead(s, ts);
         }
-        temp_dead(s, args[0]);
+        temp_dead(s, ots);
     } else if (ts->val_type == TEMP_VAL_CONST) {
         /* propagate constant */
         if (ots->val_type == TEMP_VAL_REG) {
-            s->reg_to_temp[ots->reg] = -1;
+            s->reg_to_temp[ots->reg] = NULL;
         }
         ots->val_type = TEMP_VAL_CONST;
         ots->val = ts->val;
         if (IS_DEAD_ARG(1)) {
-            temp_dead(s, args[1]);
+            temp_dead(s, ts);
         }
     } else {
         /* The code in the first if block should have moved the
            temp to a register. */
-        assert(ts->val_type == TEMP_VAL_REG);
+        tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
         if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
             /* the mov can be suppressed */
             if (ots->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ots->reg] = -1;
+                s->reg_to_temp[ots->reg] = NULL;
             }
             ots->reg = ts->reg;
-            temp_dead(s, args[1]);
+            temp_dead(s, ts);
         } else {
             if (ots->val_type != TEMP_VAL_REG) {
                 /* When allocating a new register, make sure to not spill the
                    input one. */
                 tcg_regset_set_reg(allocated_regs, ts->reg);
                 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
-                                         allocated_regs);
+                                         allocated_regs, ots->indirect_base);
             }
             tcg_out_mov(s, otype, ots->reg, ts->reg);
         }
         ots->val_type = TEMP_VAL_REG;
         ots->mem_coherent = 0;
-        s->reg_to_temp[ots->reg] = args[0];
+        s->reg_to_temp[ots->reg] = ots;
         if (NEED_SYNC_ARG(0)) {
-            tcg_reg_sync(s, ots->reg);
+            tcg_reg_sync(s, ots->reg, allocated_regs);
         }
     }
 }
@@ -1960,7 +2010,8 @@ static void tcg_reg_alloc_op(TCGContext *s,
                              uint8_t sync_args)
 {
     TCGRegSet allocated_regs;
-    int i, k, nb_iargs, nb_oargs, reg;
+    int i, k, nb_iargs, nb_oargs;
+    TCGReg reg;
     TCGArg arg;
     const TCGArgConstraint *arg_ct;
     TCGTemp *ts;
@@ -1982,30 +2033,17 @@ static void tcg_reg_alloc_op(TCGContext *s,
         arg = args[i];
         arg_ct = &def->args_ct[i];
         ts = &s->temps[arg];
-        if (ts->val_type == TEMP_VAL_MEM) {
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-            tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            ts->val_type = TEMP_VAL_REG;
-            ts->reg = reg;
-            ts->mem_coherent = 1;
-            s->reg_to_temp[reg] = arg;
-        } else if (ts->val_type == TEMP_VAL_CONST) {
-            if (tcg_target_const_match(ts->val, ts->type, arg_ct)) {
-                /* constant is OK for instruction */
-                const_args[i] = 1;
-                new_args[i] = ts->val;
-                goto iarg_end;
-            } else {
-                /* need to move to a register */
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-                tcg_out_movi(s, ts->type, reg, ts->val);
-                ts->val_type = TEMP_VAL_REG;
-                ts->reg = reg;
-                ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
-            }
+
+        if (ts->val_type == TEMP_VAL_CONST
+            && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
+            /* constant is OK for instruction */
+            const_args[i] = 1;
+            new_args[i] = ts->val;
+            goto iarg_end;
         }
-        assert(ts->val_type == TEMP_VAL_REG);
+
+        temp_load(s, ts, arg_ct->u.regs, allocated_regs);
+
         if (arg_ct->ct & TCG_CT_IALIAS) {
             if (ts->fixed_reg) {
                 /* if fixed register, we must allocate a new register
@@ -2038,7 +2076,8 @@ static void tcg_reg_alloc_op(TCGContext *s,
         allocate_in_reg:
             /* allocate a new register matching the constraint 
                and move the temporary register into it */
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
+                                ts->indirect_base);
             tcg_out_mov(s, ts->type, reg, ts->reg);
         }
         new_args[i] = reg;
@@ -2050,7 +2089,7 @@ static void tcg_reg_alloc_op(TCGContext *s,
     /* mark dead temporaries and free the associated registers */
     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
         if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_dead(s, &s->temps[args[i]]);
         }
     }
 
@@ -2059,9 +2098,9 @@ static void tcg_reg_alloc_op(TCGContext *s,
     } else {
         if (def->flags & TCG_OPF_CALL_CLOBBER) {
             /* XXX: permit generic clobber register list ? */ 
-            for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-                if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
-                    tcg_reg_free(s, reg);
+            for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
+                if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
+                    tcg_reg_free(s, i, allocated_regs);
                 }
             }
         }
@@ -2087,20 +2126,21 @@ static void tcg_reg_alloc_op(TCGContext *s,
                     tcg_regset_test_reg(arg_ct->u.regs, reg)) {
                     goto oarg_end;
                 }
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
+                                    ts->indirect_base);
             }
             tcg_regset_set_reg(allocated_regs, reg);
             /* if a fixed register is used, then a move will be done afterwards */
             if (!ts->fixed_reg) {
                 if (ts->val_type == TEMP_VAL_REG) {
-                    s->reg_to_temp[ts->reg] = -1;
+                    s->reg_to_temp[ts->reg] = NULL;
                 }
                 ts->val_type = TEMP_VAL_REG;
                 ts->reg = reg;
                 /* temp value is modified, so the value kept in memory is
                    potentially not the same */
                 ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
+                s->reg_to_temp[reg] = ts;
             }
         oarg_end:
             new_args[i] = reg;
@@ -2118,10 +2158,10 @@ static void tcg_reg_alloc_op(TCGContext *s,
             tcg_out_mov(s, ts->type, ts->reg, reg);
         }
         if (NEED_SYNC_ARG(i)) {
-            tcg_reg_sync(s, reg);
+            tcg_reg_sync(s, reg, allocated_regs);
         }
         if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_dead(s, ts);
         }
     }
 }
@@ -2136,7 +2176,8 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
                                const TCGArg * const args, uint16_t dead_args,
                                uint8_t sync_args)
 {
-    int flags, nb_regs, i, reg;
+    int flags, nb_regs, i;
+    TCGReg reg;
     TCGArg arg;
     TCGTemp *ts;
     intptr_t stack_offset;
@@ -2172,23 +2213,9 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
 #endif
         if (arg != TCG_CALL_DUMMY_ARG) {
             ts = &s->temps[arg];
-            if (ts->val_type == TEMP_VAL_REG) {
-                tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
-            } else if (ts->val_type == TEMP_VAL_MEM) {
-                reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                    s->reserved_regs);
-                /* XXX: not correct if reading values from the stack */
-                tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-                tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
-            } else if (ts->val_type == TEMP_VAL_CONST) {
-                reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                    s->reserved_regs);
-                /* XXX: sign extend may be needed on some targets */
-                tcg_out_movi(s, ts->type, reg, ts->val);
-                tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
-            } else {
-                tcg_abort();
-            }
+            temp_load(s, ts, tcg_target_available_regs[ts->type],
+                      s->reserved_regs);
+            tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
         }
 #ifndef TCG_TARGET_STACK_GROWSUP
         stack_offset += sizeof(tcg_target_long);
@@ -2202,19 +2229,20 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
         if (arg != TCG_CALL_DUMMY_ARG) {
             ts = &s->temps[arg];
             reg = tcg_target_call_iarg_regs[i];
-            tcg_reg_free(s, reg);
+            tcg_reg_free(s, reg, allocated_regs);
+
             if (ts->val_type == TEMP_VAL_REG) {
                 if (ts->reg != reg) {
                     tcg_out_mov(s, ts->type, reg, ts->reg);
                 }
-            } else if (ts->val_type == TEMP_VAL_MEM) {
-                tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            } else if (ts->val_type == TEMP_VAL_CONST) {
-                /* XXX: sign extend ? */
-                tcg_out_movi(s, ts->type, reg, ts->val);
             } else {
-                tcg_abort();
+                TCGRegSet arg_set;
+
+                tcg_regset_clear(arg_set);
+                tcg_regset_set_reg(arg_set, reg);
+                temp_load(s, ts, arg_set, allocated_regs);
             }
+
             tcg_regset_set_reg(allocated_regs, reg);
         }
     }
@@ -2222,14 +2250,14 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
     /* mark dead temporaries and free the associated registers */
     for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
         if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_dead(s, &s->temps[args[i]]);
         }
     }
     
     /* clobber call registers */
-    for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
-            tcg_reg_free(s, reg);
+    for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
+        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
+            tcg_reg_free(s, i, allocated_regs);
         }
     }
 
@@ -2250,7 +2278,7 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
         arg = args[i];
         ts = &s->temps[arg];
         reg = tcg_target_call_oarg_regs[i];
-        assert(s->reg_to_temp[reg] == -1);
+        tcg_debug_assert(s->reg_to_temp[reg] == NULL);
 
         if (ts->fixed_reg) {
             if (ts->reg != reg) {
@@ -2258,17 +2286,17 @@ static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
             }
         } else {
             if (ts->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ts->reg] = -1;
+                s->reg_to_temp[ts->reg] = NULL;
             }
             ts->val_type = TEMP_VAL_REG;
             ts->reg = reg;
             ts->mem_coherent = 0;
-            s->reg_to_temp[reg] = arg;
+            s->reg_to_temp[reg] = ts;
             if (NEED_SYNC_ARG(i)) {
-                tcg_reg_sync(s, reg);
+                tcg_reg_sync(s, reg, allocated_regs);
             }
             if (IS_DEAD_ARG(i)) {
-                temp_dead(s, args[i]);
+                temp_dead(s, ts);
             }
         }
     }
@@ -2295,14 +2323,31 @@ void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
 #endif
 
 
-static inline int tcg_gen_code_common(TCGContext *s,
-                                      tcg_insn_unit *gen_code_buf,
-                                      long search_pc)
+int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
 {
-    int oi, oi_next;
+    int i, oi, oi_next, num_insns;
+
+#ifdef CONFIG_PROFILER
+    {
+        int n;
+
+        n = s->gen_last_op_idx + 1;
+        s->op_count += n;
+        if (n > s->op_count_max) {
+            s->op_count_max = n;
+        }
+
+        n = s->nb_temps;
+        s->temp_count += n;
+        if (n > s->temp_count_max) {
+            s->temp_count_max = n;
+        }
+    }
+#endif
 
 #ifdef DEBUG_DISAS
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
+                 && qemu_log_in_addr_range(tb->pc))) {
         qemu_log("OP:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
@@ -2329,7 +2374,8 @@ static inline int tcg_gen_code_common(TCGContext *s,
 #endif
 
 #ifdef DEBUG_DISAS
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
+                 && qemu_log_in_addr_range(tb->pc))) {
         qemu_log("OP after optimization and liveness analysis:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
@@ -2338,11 +2384,12 @@ static inline int tcg_gen_code_common(TCGContext *s,
 
     tcg_reg_alloc_start(s);
 
-    s->code_buf = gen_code_buf;
-    s->code_ptr = gen_code_buf;
+    s->code_buf = tb->tc_ptr;
+    s->code_ptr = tb->tc_ptr;
 
     tcg_out_tb_init(s);
 
+    num_insns = -1;
     for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
         TCGOp * const op = &s->gen_op_buf[oi];
         TCGArg * const args = &s->gen_opparam_buf[op->args];
@@ -2365,10 +2412,23 @@ static inline int tcg_gen_code_common(TCGContext *s,
         case INDEX_op_movi_i64:
             tcg_reg_alloc_movi(s, args, dead_args, sync_args);
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
+            if (num_insns >= 0) {
+                s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
+            }
+            num_insns++;
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
+#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
+#else
+                a = args[i];
+#endif
+                s->gen_insn_data[num_insns][i] = a;
+            }
             break;
         case INDEX_op_discard:
-            temp_dead(s, args[0]);
+            temp_dead(s, &s->temps[args[0]]);
             break;
         case INDEX_op_set_label:
             tcg_reg_alloc_bb_end(s, s->reserved_regs);
@@ -2389,40 +2449,24 @@ static inline int tcg_gen_code_common(TCGContext *s,
             tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args);
             break;
         }
-        if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) {
-            return oi;
-        }
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
         check_regs(s);
 #endif
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return -1;
+        }
     }
+    tcg_debug_assert(num_insns >= 0);
+    s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
 
     /* Generate TB finalization at the end of block */
-    tcg_out_tb_finalize(s);
-    return -1;
-}
-
-int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
-{
-#ifdef CONFIG_PROFILER
-    {
-        int n;
-
-        n = s->gen_last_op_idx + 1;
-        s->op_count += n;
-        if (n > s->op_count_max) {
-            s->op_count_max = n;
-        }
-
-        n = s->nb_temps;
-        s->temp_count += n;
-        if (n > s->temp_count_max) {
-            s->temp_count_max = n;
-        }
+    if (!tcg_out_tb_finalize(s)) {
+        return -1;
     }
-#endif
-
-    tcg_gen_code_common(s, gen_code_buf, -1);
 
     /* flush instruction cache */
     flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
@@ -2430,38 +2474,30 @@ int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
     return tcg_current_code_size(s);
 }
 
-/* Return the index of the micro operation such as the pc after is <
-   offset bytes from the start of the TB.  The contents of gen_code_buf must
-   not be changed, though writing the same values is ok.
-   Return -1 if not found. */
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset)
-{
-    return tcg_gen_code_common(s, gen_code_buf, offset);
-}
-
 #ifdef CONFIG_PROFILER
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
 {
     TCGContext *s = &tcg_ctx;
-    int64_t tot;
+    int64_t tb_count = s->tb_count;
+    int64_t tb_div_count = tb_count ? tb_count : 1;
+    int64_t tot = s->interm_time + s->code_time;
 
-    tot = s->interm_time + s->code_time;
     cpu_fprintf(f, "JIT cycles          %" PRId64 " (%0.3f s at 2.4 GHz)\n",
                 tot, tot / 2.4e9);
     cpu_fprintf(f, "translated TBs      %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 
-                s->tb_count, 
-                s->tb_count1 - s->tb_count,
-                s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0);
+                tb_count, s->tb_count1 - tb_count,
+                (double)(s->tb_count1 - s->tb_count)
+                / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
     cpu_fprintf(f, "avg ops/TB          %0.1f max=%d\n", 
-                s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max);
+                (double)s->op_count / tb_div_count, s->op_count_max);
     cpu_fprintf(f, "deleted ops/TB      %0.2f\n",
-                s->tb_count ? 
-                (double)s->del_op_count / s->tb_count : 0);
+                (double)s->del_op_count / tb_div_count);
     cpu_fprintf(f, "avg temps/TB        %0.2f max=%d\n",
-                s->tb_count ? 
-                (double)s->temp_count / s->tb_count : 0,
-                s->temp_count_max);
+                (double)s->temp_count / tb_div_count, s->temp_count_max);
+    cpu_fprintf(f, "avg host code/TB    %0.1f\n",
+                (double)s->code_out_len / tb_div_count);
+    cpu_fprintf(f, "avg search data/TB  %0.1f\n",
+                (double)s->search_out_len / tb_div_count);
     
     cpu_fprintf(f, "cycles/op           %0.1f\n", 
                 s->op_count ? (double)tot / s->op_count : 0);
@@ -2469,8 +2505,11 @@ void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
                 s->code_in_len ? (double)tot / s->code_in_len : 0);
     cpu_fprintf(f, "cycles/out byte     %0.1f\n", 
                 s->code_out_len ? (double)tot / s->code_out_len : 0);
-    if (tot == 0)
+    cpu_fprintf(f, "cycles/search byte     %0.1f\n",
+                s->search_out_len ? (double)tot / s->search_out_len : 0);
+    if (tot == 0) {
         tot = 1;
+    }
     cpu_fprintf(f, "  gen_interm time   %0.1f%%\n", 
                 (double)s->interm_time / tot * 100.0);
     cpu_fprintf(f, "  gen_code time     %0.1f%%\n", 
diff --git a/qemu/tcg/tcg.h b/qemu/tcg/tcg.h
index 231a78152..40c8fbe2a 100644
--- a/qemu/tcg/tcg.h
+++ b/qemu/tcg/tcg.h
@@ -66,7 +66,8 @@ typedef uint64_t TCGRegSet;
 
 #if TCG_TARGET_REG_BITS == 32
 /* Turn some undef macros into false macros.  */
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_div2_i64         0
@@ -128,6 +129,12 @@ typedef uint64_t TCGRegSet;
 # error "Missing unsigned widening multiply"
 #endif
 
+#ifndef TARGET_INSN_START_EXTRA_WORDS
+# define TARGET_INSN_START_WORDS 1
+#else
+# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
+#endif
+
 typedef enum TCGOpcode {
 #define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
 #include "tcg-opc.h"
@@ -187,6 +194,7 @@ typedef struct TCGPool {
 #define TCG_POOL_CHUNK_SIZE 32768
 
 #define TCG_MAX_TEMPS 512
+#define TCG_MAX_INSNS 512
 
 /* when the size of the arguments of a called function is smaller than
    this value, they are statically allocated in the TB stack frame */
@@ -300,6 +308,14 @@ typedef tcg_target_ulong TCGArg;
 typedef struct TCGv_i32_d *TCGv_i32;
 typedef struct TCGv_i64_d *TCGv_i64;
 typedef struct TCGv_ptr_d *TCGv_ptr;
+typedef TCGv_ptr TCGv_env;
+#if TARGET_LONG_BITS == 32
+#define TCGv TCGv_i32
+#elif TARGET_LONG_BITS == 64
+#define TCGv TCGv_i64
+#else
+#error Unhandled TARGET_LONG_BITS value
+#endif
 
 static inline TCGv_i32 QEMU_ARTIFICIAL MAKE_TCGV_I32(intptr_t i)
 {
@@ -440,12 +456,13 @@ typedef enum TCGTempVal {
 } TCGTempVal;
 
 typedef struct TCGTemp {
-    unsigned int reg:8;
-    unsigned int mem_reg:8;
+    TCGReg reg:8;
     TCGTempVal val_type:8;
     TCGType base_type:8;
     TCGType type:8;
     unsigned int fixed_reg:1;
+    unsigned int indirect_reg:1;
+    unsigned int indirect_base:1;
     unsigned int mem_coherent:1;
     unsigned int mem_allocated:1;
     unsigned int temp_local:1; /* If true, the temp is saved across
@@ -454,6 +471,7 @@ typedef struct TCGTemp {
     unsigned int temp_allocated:1; /* never used for code gen */
 
     tcg_target_long val;
+    struct TCGTemp *mem_base;
     intptr_t mem_offset;
     const char *name;
 } TCGTemp;
@@ -507,7 +525,7 @@ struct TCGContext {
     intptr_t current_frame_offset;
     intptr_t frame_start;
     intptr_t frame_end;
-    int frame_reg;
+    TCGTemp *frame_temp;
 
     tcg_insn_unit *code_ptr;
 
@@ -524,6 +542,7 @@ struct TCGContext {
     int64_t del_op_count;
     int64_t code_in_len;
     int64_t code_out_len;
+    int64_t search_out_len;
     int64_t interm_time;
     int64_t code_time;
     int64_t la_time;
@@ -550,28 +569,28 @@ struct TCGContext {
     void *code_gen_prologue;
     void *code_gen_buffer;
     size_t code_gen_buffer_size;
-    /* threshold to flush the translated code buffer */
-    size_t code_gen_buffer_max_size;
     void *code_gen_ptr;
 
+    /* Threshold to flush the translated code buffer.  */
+    void *code_gen_highwater;
+
     TBContext tb_ctx;
 
-    /* The TCGBackendData structure is private to tcg-target.c.  */
+    /* The TCGBackendData structure is private to tcg-target.inc.c.  */
     struct TCGBackendData *be;
 
     TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
     TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
 
-    /* tells in which temporary a given register is. It does not take
-       into account fixed registers */
-    int reg_to_temp[TCG_TARGET_NB_REGS];
+    /* Tells which temporary holds a given register.
+       It does not take into account fixed registers */
+    TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
 
     TCGOp gen_op_buf[OPC_BUF_SIZE];
     TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
 
-    target_ulong gen_opc_pc[OPC_BUF_SIZE];
-    uint16_t gen_opc_icount[OPC_BUF_SIZE];
-    uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
+    uint16_t gen_insn_end_off[TCG_MAX_INSNS];
+    target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
 };
 
 extern TCGContext tcg_ctx;
@@ -594,6 +613,10 @@ void *tcg_malloc_internal(TCGContext *s, int size);
 void tcg_pool_reset(TCGContext *s);
 void tcg_pool_delete(TCGContext *s);
 
+void tb_lock(void);
+void tb_unlock(void);
+void tb_lock_reset(void);
+
 static inline void *tcg_malloc(int size)
 {
     TCGContext *s = &tcg_ctx;
@@ -613,39 +636,54 @@ void tcg_context_init(TCGContext *s);
 void tcg_prologue_init(TCGContext *s);
 void tcg_func_start(TCGContext *s);
 
-int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf);
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset);
+int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
 
-void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size);
+void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
+
+int tcg_global_mem_new_internal(TCGType, TCGv_ptr, intptr_t, const char *);
+
+TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name);
+TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name);
 
-TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name);
-TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name);
 TCGv_i32 tcg_temp_new_internal_i32(int temp_local);
+TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
+
+void tcg_temp_free_i32(TCGv_i32 arg);
+void tcg_temp_free_i64(TCGv_i64 arg);
+
+static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
+                                              const char *name)
+{
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
+    return MAKE_TCGV_I32(idx);
+}
+
 static inline TCGv_i32 tcg_temp_new_i32(void)
 {
     return tcg_temp_new_internal_i32(0);
 }
+
 static inline TCGv_i32 tcg_temp_local_new_i32(void)
 {
     return tcg_temp_new_internal_i32(1);
 }
-void tcg_temp_free_i32(TCGv_i32 arg);
-char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg);
 
-TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name);
-TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name);
-TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
+static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
+                                              const char *name)
+{
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
+    return MAKE_TCGV_I64(idx);
+}
+
 static inline TCGv_i64 tcg_temp_new_i64(void)
 {
     return tcg_temp_new_internal_i64(0);
 }
+
 static inline TCGv_i64 tcg_temp_local_new_i64(void)
 {
     return tcg_temp_new_internal_i64(1);
 }
-void tcg_temp_free_i64(TCGv_i64 arg);
-char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg);
 
 #if defined(CONFIG_DEBUG_TCG)
 /* If you call tcg_clear_temp_count() at the start of a section of
@@ -985,25 +1023,48 @@ void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
 void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
                        TCGMemOpIdx oi, uintptr_t retaddr);
 
+uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+
 /* Temporary aliases until backends are converted.  */
 #ifdef TARGET_WORDS_BIGENDIAN
 # define helper_ret_ldsw_mmu  helper_be_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_be_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_be_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_be_ldul_mmu
+# define helper_ret_ldl_mmu   helper_be_ldul_mmu
 # define helper_ret_ldq_mmu   helper_be_ldq_mmu
 # define helper_ret_stw_mmu   helper_be_stw_mmu
 # define helper_ret_stl_mmu   helper_be_stl_mmu
 # define helper_ret_stq_mmu   helper_be_stq_mmu
+# define helper_ret_ldw_cmmu  helper_be_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_be_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_be_ldq_cmmu
 #else
 # define helper_ret_ldsw_mmu  helper_le_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_le_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_le_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_le_ldul_mmu
+# define helper_ret_ldl_mmu   helper_le_ldul_mmu
 # define helper_ret_ldq_mmu   helper_le_ldq_mmu
 # define helper_ret_stw_mmu   helper_le_stw_mmu
 # define helper_ret_stl_mmu   helper_le_stl_mmu
 # define helper_ret_stq_mmu   helper_le_stq_mmu
+# define helper_ret_ldw_cmmu  helper_le_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_le_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_le_ldq_cmmu
 #endif
 
 #endif /* CONFIG_SOFTMMU */
diff --git a/qemu/tcg/tci/README b/qemu/tcg/tci/README
index dc57f076b..3786b0915 100644
--- a/qemu/tcg/tci/README
+++ b/qemu/tcg/tci/README
@@ -21,7 +21,7 @@ This is what TCI (Tiny Code Interpreter) does.
 2) Implementation
 
 Like each TCG host frontend, TCI implements the code generator in
-tcg-target.c, tcg-target.h. Both files are in directory tcg/tci.
+tcg-target.inc.c, tcg-target.h. Both files are in directory tcg/tci.
 
 The additional file tcg/tci.c adds the interpreter.
 
@@ -123,7 +123,7 @@ u1 = linux-user-test works
   would also improve speed for hosts which support byte alignment).
 
 * A better disassembler for the pseudo code would be nice (a very primitive
-  disassembler is included in tcg-target.c).
+  disassembler is included in tcg-target.inc.c).
 
 * It might be useful to have a runtime option which selects the native TCG
   or TCI, so QEMU would have to include two TCGs. Today, selecting TCI
diff --git a/qemu/tcg/tci/tcg-target.h b/qemu/tcg/tci/tcg-target.h
index cbf3f9b5a..3942f9ccc 100644
--- a/qemu/tcg/tci/tcg-target.h
+++ b/qemu/tcg/tci/tcg-target.h
@@ -40,7 +40,6 @@
 #if !defined(TCG_TARGET_H)
 #define TCG_TARGET_H
 
-#include "config-host.h"
 
 #define TCG_TARGET_INTERPRETER 1
 #define TCG_TARGET_INSN_UNIT_SIZE 1
@@ -84,7 +83,8 @@
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
diff --git a/qemu/tcg/tci/tcg-target.c b/qemu/tcg/tci/tcg-target.inc.c
index 83472dbcd..e2fc52a16 100644
--- a/qemu/tcg/tci/tcg-target.c
+++ b/qemu/tcg/tci/tcg-target.inc.c
@@ -210,6 +210,8 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
 #if TCG_TARGET_HAS_ext32u_i64
     { INDEX_op_ext32u_i64, { R, R } },
 #endif
+    { INDEX_op_ext_i32_i64, { R, R } },
+    { INDEX_op_extu_i32_i64, { R, R } },
 #if TCG_TARGET_HAS_bswap16_i64
     { INDEX_op_bswap16_i64, { R, R } },
 #endif
@@ -313,7 +315,7 @@ static const int tcg_target_call_oarg_regs[] = {
 #endif
 };
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "r00",
     "r01",
@@ -358,9 +360,9 @@ static void patch_reloc(tcg_insn_unit *code_ptr, int type,
                         intptr_t value, intptr_t addend)
 {
     /* tcg_out_reloc always uses the same type, addend. */
-    assert(type == sizeof(tcg_target_long));
-    assert(addend == 0);
-    assert(value != 0);
+    tcg_debug_assert(type == sizeof(tcg_target_long));
+    tcg_debug_assert(addend == 0);
+    tcg_debug_assert(value != 0);
     if (TCG_TARGET_REG_BITS == 32) {
         tcg_patch32(code_ptr, value);
     } else {
@@ -417,7 +419,7 @@ static void tcg_out_op_t(TCGContext *s, TCGOpcode op)
 /* Write register. */
 static void tcg_out_r(TCGContext *s, TCGArg t0)
 {
-    assert(t0 < TCG_TARGET_NB_REGS);
+    tcg_debug_assert(t0 < TCG_TARGET_NB_REGS);
     tcg_out8(s, t0);
 }
 
@@ -425,7 +427,7 @@ static void tcg_out_r(TCGContext *s, TCGArg t0)
 static void tcg_out_ri(TCGContext *s, int const_arg, TCGArg arg)
 {
     if (const_arg) {
-        assert(const_arg == 1);
+        tcg_debug_assert(const_arg == 1);
         tcg_out8(s, TCG_CONST);
         tcg_out_i(s, arg);
     } else {
@@ -437,7 +439,7 @@ static void tcg_out_ri(TCGContext *s, int const_arg, TCGArg arg)
 static void tcg_out_ri32(TCGContext *s, int const_arg, TCGArg arg)
 {
     if (const_arg) {
-        assert(const_arg == 1);
+        tcg_debug_assert(const_arg == 1);
         tcg_out8(s, TCG_CONST);
         tcg_out32(s, arg);
     } else {
@@ -450,7 +452,7 @@ static void tcg_out_ri32(TCGContext *s, int const_arg, TCGArg arg)
 static void tcg_out_ri64(TCGContext *s, int const_arg, TCGArg arg)
 {
     if (const_arg) {
-        assert(const_arg == 1);
+        tcg_debug_assert(const_arg == 1);
         tcg_out8(s, TCG_CONST);
         tcg_out64(s, arg);
     } else {
@@ -464,7 +466,7 @@ static void tci_out_label(TCGContext *s, TCGLabel *label)
 {
     if (label->has_value) {
         tcg_out_i(s, label->u.value);
-        assert(label->u.value);
+        tcg_debug_assert(label->u.value);
     } else {
         tcg_out_reloc(s, s->code_ptr, sizeof(tcg_target_ulong), label, 0);
         s->code_ptr += sizeof(tcg_target_ulong);
@@ -481,12 +483,12 @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
         tcg_out_r(s, arg1);
         tcg_out32(s, arg2);
     } else {
-        assert(type == TCG_TYPE_I64);
+        tcg_debug_assert(type == TCG_TYPE_I64);
 #if TCG_TARGET_REG_BITS == 64
         tcg_out_op_t(s, INDEX_op_ld_i64);
         tcg_out_r(s, ret);
         tcg_out_r(s, arg1);
-        assert(arg2 == (int32_t)arg2);
+        tcg_debug_assert(arg2 == (int32_t)arg2);
         tcg_out32(s, arg2);
 #else
         TODO();
@@ -498,7 +500,7 @@ static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
 {
     uint8_t *old_code_ptr = s->code_ptr;
-    assert(ret != arg);
+    tcg_debug_assert(ret != arg);
 #if TCG_TARGET_REG_BITS == 32
     tcg_out_op_t(s, INDEX_op_mov_i32);
 #else
@@ -519,7 +521,7 @@ static void tcg_out_movi(TCGContext *s, TCGType type,
         tcg_out_r(s, t0);
         tcg_out32(s, arg32);
     } else {
-        assert(type == TCG_TYPE_I64);
+        tcg_debug_assert(type == TCG_TYPE_I64);
 #if TCG_TARGET_REG_BITS == 64
         tcg_out_op_t(s, INDEX_op_movi_i64);
         tcg_out_r(s, t0);
@@ -553,14 +555,14 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
             /* Direct jump method. */
-            assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset));
+            tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset));
             s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
             tcg_out32(s, 0);
         } else {
             /* Indirect jump method. */
             TODO();
         }
-        assert(args[0] < ARRAY_SIZE(s->tb_next_offset));
+        tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_next_offset));
         s->tb_next_offset[args[0]] = tcg_current_code_size(s);
         break;
     case INDEX_op_br:
@@ -611,7 +613,7 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_st_i64:
         tcg_out_r(s, args[0]);
         tcg_out_r(s, args[1]);
-        assert(args[2] == (int32_t)args[2]);
+        tcg_debug_assert(args[2] == (int32_t)args[2]);
         tcg_out32(s, args[2]);
         break;
     case INDEX_op_add_i32:
@@ -638,9 +640,9 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_r(s, args[0]);
         tcg_out_r(s, args[1]);
         tcg_out_r(s, args[2]);
-        assert(args[3] <= UINT8_MAX);
+        tcg_debug_assert(args[3] <= UINT8_MAX);
         tcg_out8(s, args[3]);
-        assert(args[4] <= UINT8_MAX);
+        tcg_debug_assert(args[4] <= UINT8_MAX);
         tcg_out8(s, args[4]);
         break;
 
@@ -669,9 +671,9 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_r(s, args[0]);
         tcg_out_r(s, args[1]);
         tcg_out_r(s, args[2]);
-        assert(args[3] <= UINT8_MAX);
+        tcg_debug_assert(args[3] <= UINT8_MAX);
         tcg_out8(s, args[3]);
-        assert(args[4] <= UINT8_MAX);
+        tcg_debug_assert(args[4] <= UINT8_MAX);
         tcg_out8(s, args[4]);
         break;
     case INDEX_op_div_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
@@ -701,6 +703,8 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_ext16u_i64:   /* Optional (TCG_TARGET_HAS_ext16u_i64). */
     case INDEX_op_ext32s_i64:   /* Optional (TCG_TARGET_HAS_ext32s_i64). */
     case INDEX_op_ext32u_i64:   /* Optional (TCG_TARGET_HAS_ext32u_i64). */
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
 #endif /* TCG_TARGET_REG_BITS == 64 */
     case INDEX_op_neg_i32:      /* Optional (TCG_TARGET_HAS_neg_i32). */
     case INDEX_op_not_i32:      /* Optional (TCG_TARGET_HAS_not_i32). */
@@ -815,7 +819,7 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
         tcg_out_r(s, arg1);
         tcg_out32(s, arg2);
     } else {
-        assert(type == TCG_TYPE_I64);
+        tcg_debug_assert(type == TCG_TYPE_I64);
 #if TCG_TARGET_REG_BITS == 64
         tcg_out_op_t(s, INDEX_op_st_i64);
         tcg_out_r(s, arg);
@@ -846,7 +850,7 @@ static void tcg_target_init(TCGContext *s)
 #endif
 
     /* The current code uses uint8_t for tcg operations. */
-    assert(ARRAY_SIZE(tcg_op_defs) <= UINT8_MAX);
+    tcg_debug_assert(tcg_op_defs_max <= UINT8_MAX);
 
     /* Registers available for 32 bit operations. */
     tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0,
diff --git a/qemu/tci.c b/qemu/tci.c
index 84449489d..82705fe77 100644
--- a/qemu/tci.c
+++ b/qemu/tci.c
@@ -1,7 +1,7 @@
 /*
  * Tiny Code Interpreter for QEMU
  *
- * Copyright (c) 2009, 2011 Stefan Weil
+ * Copyright (c) 2009, 2011, 2016 Stefan Weil
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -17,11 +17,14 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 
-/* Defining NDEBUG disables assertions (which makes the code faster). */
-#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
-# define NDEBUG
+/* Enable TCI assertions only when debugging TCG (and without NDEBUG defined).
+ * Without assertions, the interpreter runs much faster. */
+#if defined(CONFIG_DEBUG_TCG)
+# define tci_assert(cond) assert(cond)
+#else
+# define tci_assert(cond) ((void)0)
 #endif
 
 #include "qemu-common.h"
@@ -52,17 +55,11 @@ typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong);
 #endif
 
-/* Targets which don't use GETPC also don't need tci_tb_ptr
-   which makes them a little faster. */
-#if defined(GETPC)
-uintptr_t tci_tb_ptr;
-#endif
-
 static tcg_target_ulong tci_reg[TCG_TARGET_NB_REGS];
 
 static tcg_target_ulong tci_read_reg(TCGReg index)
 {
-    assert(index < ARRAY_SIZE(tci_reg));
+    tci_assert(index < ARRAY_SIZE(tci_reg));
     return tci_reg[index];
 }
 
@@ -111,9 +108,9 @@ static uint64_t tci_read_reg64(TCGReg index)
 
 static void tci_write_reg(TCGReg index, tcg_target_ulong value)
 {
-    assert(index < ARRAY_SIZE(tci_reg));
-    assert(index != TCG_AREG0);
-    assert(index != TCG_REG_CALL_STACK);
+    tci_assert(index < ARRAY_SIZE(tci_reg));
+    tci_assert(index != TCG_AREG0);
+    tci_assert(index != TCG_REG_CALL_STACK);
     tci_reg[index] = value;
 }
 
@@ -331,7 +328,7 @@ static uint64_t tci_read_ri64(uint8_t **tb_ptr)
 static tcg_target_ulong tci_read_label(uint8_t **tb_ptr)
 {
     tcg_target_ulong label = tci_read_i(tb_ptr);
-    assert(label != 0);
+    tci_assert(label != 0);
     return label;
 }
 
@@ -474,11 +471,11 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 
     tci_reg[TCG_AREG0] = (tcg_target_ulong)env;
     tci_reg[TCG_REG_CALL_STACK] = sp_value;
-    assert(tb_ptr);
+    tci_assert(tb_ptr);
 
     for (;;) {
         TCGOpcode opc = tb_ptr[0];
-#if !defined(NDEBUG)
+#if defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
         uint8_t op_size = tb_ptr[1];
         uint8_t *old_code_ptr = tb_ptr;
 #endif
@@ -531,7 +528,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             break;
         case INDEX_op_br:
             label = tci_read_label(&tb_ptr);
-            assert(tb_ptr == old_code_ptr + op_size);
+            tci_assert(tb_ptr == old_code_ptr + op_size);
             tb_ptr = (uint8_t *)label;
             continue;
         case INDEX_op_setcond_i32:
@@ -606,7 +603,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             t0 = tci_read_r32(&tb_ptr);
             t1 = tci_read_r(&tb_ptr);
             t2 = tci_read_s32(&tb_ptr);
-            assert(t1 != sp_value || (int32_t)t2 < 0);
+            tci_assert(t1 != sp_value || (int32_t)t2 < 0);
             *(uint32_t *)(t1 + t2) = t0;
             break;
 
@@ -731,7 +728,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare32(t0, t1, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -757,7 +754,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare64(tmp64, v64, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -891,7 +888,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             t0 = tci_read_r64(&tb_ptr);
             t1 = tci_read_r(&tb_ptr);
             t2 = tci_read_s32(&tb_ptr);
-            assert(t1 != sp_value || (int32_t)t2 < 0);
+            tci_assert(t1 != sp_value || (int32_t)t2 < 0);
             *(uint64_t *)(t1 + t2) = t0;
             break;
 
@@ -998,7 +995,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare64(t0, t1, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -1033,18 +1030,20 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 #endif
 #if TCG_TARGET_HAS_ext32s_i64
         case INDEX_op_ext32s_i64:
+#endif
+        case INDEX_op_ext_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32s(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_ext32u_i64
         case INDEX_op_ext32u_i64:
+#endif
+        case INDEX_op_extu_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_bswap16_i64
         case INDEX_op_bswap16_i64:
             TODO();
@@ -1085,22 +1084,13 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 
             /* QEMU specific operations. */
 
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#else
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#endif
         case INDEX_op_exit_tb:
             next_tb = *(uint64_t *)tb_ptr;
             goto exit;
             break;
         case INDEX_op_goto_tb:
             t0 = tci_read_i32(&tb_ptr);
-            assert(tb_ptr == old_code_ptr + op_size);
+            tci_assert(tb_ptr == old_code_ptr + op_size);
             tb_ptr += (int32_t)t0;
             continue;
         case INDEX_op_qemu_ld_i32:
@@ -1247,7 +1237,7 @@ uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
             TODO();
             break;
         }
-        assert(tb_ptr == old_code_ptr + op_size);
+        tci_assert(tb_ptr == old_code_ptr + op_size);
     }
 exit:
     return next_tb;
diff --git a/qemu/tests/.gitignore b/qemu/tests/.gitignore
index ccc92e476..9eed22988 100644
--- a/qemu/tests/.gitignore
+++ b/qemu/tests/.gitignore
@@ -8,14 +8,38 @@ check-qom-interface
 check-qom-proplist
 rcutorture
 test-aio
+test-base64
 test-bitops
+test-blockjob-txn
 test-coroutine
+test-crypto-afsplit
+test-crypto-block
 test-crypto-cipher
 test-crypto-hash
+test-crypto-ivgen
+test-crypto-pbkdf
+test-crypto-secret
+test-crypto-tlscredsx509
+test-crypto-tlscredsx509-work/
+test-crypto-tlscredsx509-certs/
+test-crypto-tlssession
+test-crypto-tlssession-work/
+test-crypto-tlssession-client/
+test-crypto-tlssession-server/
+test-crypto-xts
 test-cutils
 test-hbitmap
 test-int128
 test-iov
+test-io-channel-buffer
+test-io-channel-command
+test-io-channel-command.fifo
+test-io-channel-file
+test-io-channel-file.txt
+test-io-channel-socket
+test-io-channel-tls
+test-io-task
+test-logging
 test-mul64
 test-opts-visitor
 test-qapi-event.[ch]
@@ -23,11 +47,13 @@ test-qapi-types.[ch]
 test-qapi-visit.[ch]
 test-qdev-global-props
 test-qemu-opts
+test-qga
 test-qmp-commands
 test-qmp-commands.h
 test-qmp-event
 test-qmp-input-strict
 test-qmp-input-visitor
+test-qmp-introspect.[ch]
 test-qmp-marshal.c
 test-qmp-output-visitor
 test-rcu-list
@@ -36,10 +62,14 @@ test-string-input-visitor
 test-string-output-visitor
 test-thread-pool
 test-throttle
+test-timed-average
 test-visitor-serialization
 test-vmstate
 test-write-threshold
 test-x86-cpuid
 test-xbzrle
+test-netfilter
+test-filter-mirror
+test-filter-redirector
 *-test
 qapi-schema/*.test.*
diff --git a/qemu/tests/Makefile b/qemu/tests/Makefile
index 749458224..9194f1850 100644
--- a/qemu/tests/Makefile
+++ b/qemu/tests/Makefile
@@ -1,5 +1,7 @@
 export SRC_PATH
 
+qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
+
 # Get the list of all supported sysemu targets
 SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \
    $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak)))
@@ -45,6 +47,8 @@ check-unit-y += tests/test-thread-pool$(EXESUF)
 gcov-files-test-thread-pool-y = thread-pool.c
 gcov-files-test-hbitmap-y = util/hbitmap.c
 check-unit-y += tests/test-hbitmap$(EXESUF)
+gcov-files-test-hbitmap-y = blockjob.c
+check-unit-y += tests/test-blockjob-txn$(EXESUF)
 check-unit-y += tests/test-x86-cpuid$(EXESUF)
 # all code tested by test-x86-cpuid is inside topology.h
 gcov-files-test-x86-cpuid-y =
@@ -76,12 +80,36 @@ check-unit-y += tests/test-write-threshold$(EXESUF)
 gcov-files-test-write-threshold-y = block/write-threshold.c
 check-unit-$(CONFIG_GNUTLS_HASH) += tests/test-crypto-hash$(EXESUF)
 check-unit-y += tests/test-crypto-cipher$(EXESUF)
+check-unit-y += tests/test-crypto-secret$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlscredsx509$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlssession$(EXESUF)
+ifneq (,$(findstring qemu-ga,$(TOOLS)))
+check-unit-$(CONFIG_LINUX) += tests/test-qga$(EXESUF)
+endif
+check-unit-y += tests/test-timed-average$(EXESUF)
+check-unit-y += tests/test-io-task$(EXESUF)
+check-unit-y += tests/test-io-channel-socket$(EXESUF)
+check-unit-y += tests/test-io-channel-file$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-io-channel-tls$(EXESUF)
+check-unit-y += tests/test-io-channel-command$(EXESUF)
+check-unit-y += tests/test-io-channel-buffer$(EXESUF)
+check-unit-y += tests/test-base64$(EXESUF)
+check-unit-$(if $(CONFIG_NETTLE_KDF),y,$(CONFIG_GCRYPT_KDF)) += tests/test-crypto-pbkdf$(EXESUF)
+check-unit-y += tests/test-crypto-ivgen$(EXESUF)
+check-unit-y += tests/test-crypto-afsplit$(EXESUF)
+check-unit-y += tests/test-crypto-xts$(EXESUF)
+check-unit-y += tests/test-crypto-block$(EXESUF)
+gcov-files-test-logging-y = tests/test-logging.c
+check-unit-y += tests/test-logging$(EXESUF)
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
 # All QTests for now are POSIX-only, but the dependencies are
 # really in libqtest, not in the testcases themselves.
 
+check-qtest-generic-y = tests/device-introspect-test$(EXESUF)
+gcov-files-generic-y = qdev-monitor.c qmp.c
+
 gcov-files-ipack-y += hw/ipack/ipack.c
 check-qtest-ipack-y += tests/ipoctal232-test$(EXESUF)
 gcov-files-ipack-y += hw/char/ipoctal232.c
@@ -142,6 +170,8 @@ gcov-files-pci-y += hw/display/virtio-gpu-pci.c
 gcov-files-pci-$(CONFIG_VIRTIO_VGA) += hw/display/virtio-vga.c
 check-qtest-pci-y += tests/intel-hda-test$(EXESUF)
 gcov-files-pci-y += hw/audio/intel-hda.c hw/audio/hda-codec.c
+check-qtest-pci-$(CONFIG_EVENTFD) += tests/ivshmem-test$(EXESUF)
+gcov-files-pci-y += hw/misc/ivshmem.c
 
 check-qtest-i386-y = tests/endianness-test$(EXESUF)
 check-qtest-i386-y += tests/fdc-test$(EXESUF)
@@ -152,7 +182,10 @@ check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
 gcov-files-i386-y += hw/block/hd-geometry.c
 check-qtest-i386-y += tests/boot-order-test$(EXESUF)
 check-qtest-i386-y += tests/bios-tables-test$(EXESUF)
+check-qtest-i386-y += tests/pxe-test$(EXESUF)
 check-qtest-i386-y += tests/rtc-test$(EXESUF)
+check-qtest-i386-y += tests/ipmi-kcs-test$(EXESUF)
+check-qtest-i386-y += tests/ipmi-bt-test$(EXESUF)
 check-qtest-i386-y += tests/i440fx-test$(EXESUF)
 check-qtest-i386-y += tests/fw_cfg-test$(EXESUF)
 check-qtest-i386-y += tests/drive_del-test$(EXESUF)
@@ -184,7 +217,13 @@ gcov-files-i386-y += hw/usb/hcd-xhci.c
 check-qtest-i386-y += tests/pc-cpu-test$(EXESUF)
 check-qtest-i386-y += tests/q35-test$(EXESUF)
 gcov-files-i386-y += hw/pci-host/q35.c
-check-qtest-i386-$(CONFIG_LINUX) += tests/vhost-user-test$(EXESUF)
+check-qtest-i386-$(CONFIG_VHOST_NET_TEST_i386) += tests/vhost-user-test$(EXESUF)
+ifeq ($(CONFIG_VHOST_NET_TEST_i386),)
+check-qtest-x86_64-$(CONFIG_VHOST_NET_TEST_x86_64) += tests/vhost-user-test$(EXESUF)
+endif
+check-qtest-i386-y += tests/test-netfilter$(EXESUF)
+check-qtest-i386-y += tests/test-filter-mirror$(EXESUF)
+check-qtest-i386-y += tests/test-filter-redirector$(EXESUF)
 check-qtest-x86_64-y = $(check-qtest-i386-y)
 gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
 gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
@@ -201,6 +240,7 @@ check-qtest-sparc64-y = tests/endianness-test$(EXESUF)
 gcov-files-sparc-y += hw/timer/m48t59.c
 gcov-files-sparc64-y += hw/timer/m48t59.c
 check-qtest-arm-y = tests/tmp105-test$(EXESUF)
+check-qtest-arm-y = tests/ds1338-test$(EXESUF)
 gcov-files-arm-y += hw/misc/tmp105.c
 check-qtest-arm-y += tests/virtio-blk-test$(EXESUF)
 gcov-files-arm-y += arm-softmmu/hw/block/virtio-blk.c
@@ -211,54 +251,135 @@ gcov-files-ppc64-y += ppc64-softmmu/hw/ppc/spapr_pci.c
 check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
 check-qtest-xtensaeb-y = $(check-qtest-xtensa-y)
 
-# qom-test works for all sysemu architectures:
-$(foreach target,$(SYSEMU_TARGET_LIST), \
-	$(if $(findstring tests/qom-test$(EXESUF), $(check-qtest-$(target)-y)),, \
-		$(eval check-qtest-$(target)-y += tests/qom-test$(EXESUF))))
-
-check-qapi-schema-y := $(addprefix tests/qapi-schema/, \
-	comments.json empty.json enum-empty.json enum-missing-data.json \
-	enum-wrong-data.json enum-int-member.json enum-dict-member.json \
-	enum-clash-member.json enum-max-member.json enum-union-clash.json \
-	enum-bad-name.json funny-char.json indented-expr.json \
-	missing-type.json bad-ident.json ident-with-escape.json \
-	escape-outside-string.json unknown-escape.json \
-	escape-too-short.json escape-too-big.json unicode-str.json \
-	double-type.json bad-base.json bad-type-bool.json bad-type-int.json \
-	bad-type-dict.json double-data.json unknown-expr-key.json \
-	redefined-type.json redefined-command.json redefined-builtin.json \
-	redefined-event.json command-int.json bad-data.json event-max.json \
-	type-bypass.json type-bypass-no-gen.json type-bypass-bad-gen.json \
-	data-array-empty.json data-array-unknown.json data-int.json \
-	data-unknown.json data-member-unknown.json data-member-array.json \
-	data-member-array-bad.json returns-array-bad.json returns-int.json \
-	returns-unknown.json returns-alternate.json returns-whitelist.json \
-	missing-colon.json missing-comma-list.json missing-comma-object.json \
-	nested-struct-data.json nested-struct-returns.json non-objects.json \
-	qapi-schema-test.json quoted-structural-chars.json \
-	trailing-comma-list.json trailing-comma-object.json \
-	unclosed-list.json unclosed-object.json unclosed-string.json \
-	duplicate-key.json union-invalid-base.json union-bad-branch.json \
-	union-optional-branch.json union-unknown.json union-max.json \
-	flat-union-optional-discriminator.json flat-union-no-base.json \
-	flat-union-invalid-discriminator.json flat-union-inline.json \
-	flat-union-invalid-branch-key.json flat-union-reverse-define.json \
-	flat-union-string-discriminator.json union-base-no-discriminator.json \
-	flat-union-bad-discriminator.json flat-union-bad-base.json \
-	flat-union-base-star.json \
-	flat-union-array-branch.json flat-union-int-branch.json \
-	flat-union-base-union.json flat-union-branch-clash.json \
-	alternate-nested.json alternate-unknown.json alternate-clash.json \
-	alternate-good.json alternate-base.json alternate-array.json \
-	alternate-conflict-string.json alternate-conflict-dict.json \
-	include-simple.json include-relpath.json include-format-err.json \
-	include-non-file.json include-no-file.json include-before-err.json \
-	include-nested-err.json include-self-cycle.json include-cycle.json \
-	include-repetition.json event-nest-struct.json event-case.json \
-	struct-base-clash.json struct-base-clash-deep.json )
+check-qtest-generic-y += tests/qom-test$(EXESUF)
+
+qapi-schema += alternate-any.json
+qapi-schema += alternate-array.json
+qapi-schema += alternate-base.json
+qapi-schema += alternate-clash.json
+qapi-schema += alternate-conflict-dict.json
+qapi-schema += alternate-conflict-string.json
+qapi-schema += alternate-empty.json
+qapi-schema += alternate-nested.json
+qapi-schema += alternate-unknown.json
+qapi-schema += args-alternate.json
+qapi-schema += args-any.json
+qapi-schema += args-array-empty.json
+qapi-schema += args-array-unknown.json
+qapi-schema += args-int.json
+qapi-schema += args-invalid.json
+qapi-schema += args-member-array-bad.json
+qapi-schema += args-member-case.json
+qapi-schema += args-member-unknown.json
+qapi-schema += args-name-clash.json
+qapi-schema += args-union.json
+qapi-schema += args-unknown.json
+qapi-schema += bad-base.json
+qapi-schema += bad-data.json
+qapi-schema += bad-ident.json
+qapi-schema += bad-type-bool.json
+qapi-schema += bad-type-dict.json
+qapi-schema += bad-type-int.json
+qapi-schema += base-cycle-direct.json
+qapi-schema += base-cycle-indirect.json
+qapi-schema += command-int.json
+qapi-schema += comments.json
+qapi-schema += double-data.json
+qapi-schema += double-type.json
+qapi-schema += duplicate-key.json
+qapi-schema += empty.json
+qapi-schema += enum-bad-name.json
+qapi-schema += enum-bad-prefix.json
+qapi-schema += enum-clash-member.json
+qapi-schema += enum-dict-member.json
+qapi-schema += enum-int-member.json
+qapi-schema += enum-member-case.json
+qapi-schema += enum-missing-data.json
+qapi-schema += enum-wrong-data.json
+qapi-schema += escape-outside-string.json
+qapi-schema += escape-too-big.json
+qapi-schema += escape-too-short.json
+qapi-schema += event-case.json
+qapi-schema += event-nest-struct.json
+qapi-schema += flat-union-array-branch.json
+qapi-schema += flat-union-bad-base.json
+qapi-schema += flat-union-bad-discriminator.json
+qapi-schema += flat-union-base-any.json
+qapi-schema += flat-union-base-union.json
+qapi-schema += flat-union-clash-member.json
+qapi-schema += flat-union-empty.json
+qapi-schema += flat-union-inline.json
+qapi-schema += flat-union-int-branch.json
+qapi-schema += flat-union-invalid-branch-key.json
+qapi-schema += flat-union-invalid-discriminator.json
+qapi-schema += flat-union-no-base.json
+qapi-schema += flat-union-optional-discriminator.json
+qapi-schema += flat-union-string-discriminator.json
+qapi-schema += funny-char.json
+qapi-schema += ident-with-escape.json
+qapi-schema += include-before-err.json
+qapi-schema += include-cycle.json
+qapi-schema += include-format-err.json
+qapi-schema += include-nested-err.json
+qapi-schema += include-no-file.json
+qapi-schema += include-non-file.json
+qapi-schema += include-relpath.json
+qapi-schema += include-repetition.json
+qapi-schema += include-self-cycle.json
+qapi-schema += include-simple.json
+qapi-schema += indented-expr.json
+qapi-schema += leading-comma-list.json
+qapi-schema += leading-comma-object.json
+qapi-schema += missing-colon.json
+qapi-schema += missing-comma-list.json
+qapi-schema += missing-comma-object.json
+qapi-schema += missing-type.json
+qapi-schema += nested-struct-data.json
+qapi-schema += non-objects.json
+qapi-schema += qapi-schema-test.json
+qapi-schema += quoted-structural-chars.json
+qapi-schema += redefined-builtin.json
+qapi-schema += redefined-command.json
+qapi-schema += redefined-event.json
+qapi-schema += redefined-type.json
+qapi-schema += reserved-command-q.json
+qapi-schema += reserved-enum-q.json
+qapi-schema += reserved-member-has.json
+qapi-schema += reserved-member-q.json
+qapi-schema += reserved-member-u.json
+qapi-schema += reserved-member-underscore.json
+qapi-schema += reserved-type-kind.json
+qapi-schema += reserved-type-list.json
+qapi-schema += returns-alternate.json
+qapi-schema += returns-array-bad.json
+qapi-schema += returns-dict.json
+qapi-schema += returns-unknown.json
+qapi-schema += returns-whitelist.json
+qapi-schema += struct-base-clash-deep.json
+qapi-schema += struct-base-clash.json
+qapi-schema += struct-data-invalid.json
+qapi-schema += struct-member-invalid.json
+qapi-schema += trailing-comma-list.json
+qapi-schema += trailing-comma-object.json
+qapi-schema += type-bypass-bad-gen.json
+qapi-schema += unclosed-list.json
+qapi-schema += unclosed-object.json
+qapi-schema += unclosed-string.json
+qapi-schema += unicode-str.json
+qapi-schema += union-base-no-discriminator.json
+qapi-schema += union-branch-case.json
+qapi-schema += union-clash-branches.json
+qapi-schema += union-empty.json
+qapi-schema += union-invalid-base.json
+qapi-schema += union-optional-branch.json
+qapi-schema += union-unknown.json
+qapi-schema += unknown-escape.json
+qapi-schema += unknown-expr-key.json
+check-qapi-schema-y := $(addprefix tests/qapi-schema/, $(qapi-schema))
 
 GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h \
-		     tests/test-qmp-commands.h tests/test-qapi-event.h
+	tests/test-qmp-commands.h tests/test-qapi-event.h \
+	tests/test-qmp-introspect.h
 
 test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/check-qlist.o tests/check-qfloat.o tests/check-qjson.o \
@@ -270,48 +391,59 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-opts-visitor.o tests/test-qmp-event.o \
 	tests/rcutorture.o tests/test-rcu-list.o
 
-test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
-		  tests/test-qapi-event.o
-
 $(test-obj-y): QEMU_INCLUDES += -Itests
 QEMU_CFLAGS += -I$(SRC_PATH)/tests
-qom-core-obj = qom/object.o qom/qom-qobject.o qom/container.o qom/object_interfaces.o
-
-tests/check-qint$(EXESUF): tests/check-qint.o libqemuutil.a
-tests/check-qstring$(EXESUF): tests/check-qstring.o libqemuutil.a
-tests/check-qdict$(EXESUF): tests/check-qdict.o libqemuutil.a
-tests/check-qlist$(EXESUF): tests/check-qlist.o libqemuutil.a
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o libqemuutil.a
-tests/check-qjson$(EXESUF): tests/check-qjson.o libqemuutil.a libqemustub.a
-tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(qom-core-obj) libqemuutil.a libqemustub.a
-tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(qom-core-obj) libqemuutil.a libqemustub.a
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o libqemuutil.a libqemustub.a
-tests/test-throttle$(EXESUF): tests/test-throttle.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
-tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
+
+
+# Deps that are common to various different sets of tests below
+test-util-obj-y = libqemuutil.a libqemustub.a
+test-qom-obj-y = $(qom-obj-y) $(test-util-obj-y)
+test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
+	tests/test-qapi-event.o tests/test-qmp-introspect.o \
+	$(test-qom-obj-y)
+test-crypto-obj-y = $(crypto-obj-y) $(test-qom-obj-y)
+test-io-obj-y = $(io-obj-y) $(test-crypto-obj-y)
+test-block-obj-y = $(block-obj-y) $(test-io-obj-y)
+
+tests/check-qint$(EXESUF): tests/check-qint.o $(test-util-obj-y)
+tests/check-qstring$(EXESUF): tests/check-qstring.o $(test-util-obj-y)
+tests/check-qdict$(EXESUF): tests/check-qdict.o $(test-util-obj-y)
+tests/check-qlist$(EXESUF): tests/check-qlist.o $(test-util-obj-y)
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o $(test-util-obj-y)
+tests/check-qjson$(EXESUF): tests/check-qjson.o $(test-util-obj-y)
+tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(test-qom-obj-y)
+tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(test-qom-obj-y)
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)
+tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)
+tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o $(test-util-obj-y)
+tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)
+tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(test-block-obj-y)
+tests/test-iov$(EXESUF): tests/test-iov.o $(test-util-obj-y)
+tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o $(test-util-obj-y)
 tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
-tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o migration/xbzrle.o page_cache.o libqemuutil.a
+tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o migration/xbzrle.o page_cache.o $(test-util-obj-y)
 tests/test-cutils$(EXESUF): tests/test-cutils.o util/cutils.o
 tests/test-int128$(EXESUF): tests/test-int128.o
-tests/rcutorture$(EXESUF): tests/rcutorture.o libqemuutil.a libqemustub.a
-tests/test-rcu-list$(EXESUF): tests/test-rcu-list.o libqemuutil.a libqemustub.a
+tests/rcutorture$(EXESUF): tests/rcutorture.o $(test-util-obj-y)
+tests/test-rcu-list$(EXESUF): tests/test-rcu-list.o $(test-util-obj-y)
 
 tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
 	hw/core/qdev.o hw/core/qdev-properties.o hw/core/hotplug.o\
 	hw/core/irq.o \
 	hw/core/fw-path-provider.o \
-	$(qom-core-obj) \
-	$(test-qapi-obj-y) \
-	libqemuutil.a libqemustub.a
+	$(test-qapi-obj-y)
 tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
 	migration/vmstate.o migration/qemu-file.o migration/qemu-file-buf.o \
         migration/qemu-file-unix.o qjson.o \
-	$(qom-core-obj) \
+	$(test-qom-obj-y)
+tests/test-timed-average$(EXESUF): tests/test-timed-average.o qemu-timer.o \
+	$(test-util-obj-y)
+tests/test-base64$(EXESUF): tests/test-base64.o \
 	libqemuutil.a libqemustub.a
 
+tests/test-logging$(EXESUF): tests/test-logging.o $(test-util-obj-y)
+
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
@@ -332,21 +464,56 @@ $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-eve
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-event.py \
 		$(gen-out-type) -o tests -p "test-" $<, \
 		"  GEN   $@")
+tests/test-qmp-introspect.c tests/test-qmp-introspect.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
 
-tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-
-tests/test-mul64$(EXESUF): tests/test-mul64.o libqemuutil.a
-tests/test-bitops$(EXESUF): tests/test-bitops.o libqemuutil.a
-tests/test-crypto-hash$(EXESUF): tests/test-crypto-hash.o libqemuutil.a libqemustub.a
-tests/test-crypto-cipher$(EXESUF): tests/test-crypto-cipher.o libqemuutil.a libqemustub.a
+tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y)
+tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y)
+tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y)
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y)
+tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y)
+tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y)
+
+tests/test-mul64$(EXESUF): tests/test-mul64.o $(test-util-obj-y)
+tests/test-bitops$(EXESUF): tests/test-bitops.o $(test-util-obj-y)
+tests/test-crypto-hash$(EXESUF): tests/test-crypto-hash.o $(test-crypto-obj-y)
+tests/test-crypto-cipher$(EXESUF): tests/test-crypto-cipher.o $(test-crypto-obj-y)
+tests/test-crypto-secret$(EXESUF): tests/test-crypto-secret.o $(test-crypto-obj-y)
+tests/test-crypto-xts$(EXESUF): tests/test-crypto-xts.o $(test-crypto-obj-y)
+
+tests/crypto-tls-x509-helpers.o-cflags := $(TASN1_CFLAGS)
+tests/crypto-tls-x509-helpers.o-libs := $(TASN1_LIBS)
+tests/pkix_asn1_tab.o-cflags := $(TASN1_CFLAGS)
+
+tests/test-crypto-tlscredsx509.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlscredsx509$(EXESUF): tests/test-crypto-tlscredsx509.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
+
+tests/test-crypto-tlssession.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlssession$(EXESUF): tests/test-crypto-tlssession.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
+tests/test-io-task$(EXESUF): tests/test-io-task.o $(test-io-obj-y)
+tests/test-io-channel-socket$(EXESUF): tests/test-io-channel-socket.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-file$(EXESUF): tests/test-io-channel-file.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-tls$(EXESUF): tests/test-io-channel-tls.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o \
+	tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-command$(EXESUF): tests/test-io-channel-command.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-buffer$(EXESUF): tests/test-io-channel-buffer.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-crypto-pbkdf$(EXESUF): tests/test-crypto-pbkdf.o $(test-crypto-obj-y)
+tests/test-crypto-ivgen$(EXESUF): tests/test-crypto-ivgen.o $(test-crypto-obj-y)
+tests/test-crypto-afsplit$(EXESUF): tests/test-crypto-afsplit.o $(test-crypto-obj-y)
+tests/test-crypto-block$(EXESUF): tests/test-crypto-block.o $(test-crypto-obj-y)
 
 libqos-obj-y = tests/libqos/pci.o tests/libqos/fw_cfg.o tests/libqos/malloc.o
 libqos-obj-y += tests/libqos/i2c.o tests/libqos/libqos.o
@@ -354,9 +521,11 @@ libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
 libqos-pc-obj-y += tests/libqos/malloc-pc.o tests/libqos/libqos-pc.o
 libqos-pc-obj-y += tests/libqos/ahci.o
 libqos-omap-obj-y = $(libqos-obj-y) tests/libqos/i2c-omap.o
+libqos-imx-obj-y = $(libqos-obj-y) tests/libqos/i2c-imx.o
 libqos-usb-obj-y = $(libqos-pc-obj-y) tests/libqos/usb.o
 libqos-virtio-obj-y = $(libqos-pc-obj-y) tests/libqos/virtio.o tests/libqos/virtio-pci.o tests/libqos/virtio-mmio.o tests/libqos/malloc-generic.o
 
+tests/device-introspect-test$(EXESUF): tests/device-introspect-test.o
 tests/rtc-test$(EXESUF): tests/rtc-test.o
 tests/m48t59-test$(EXESUF): tests/m48t59-test.o
 tests/endianness-test$(EXESUF): tests/endianness-test.o
@@ -364,10 +533,15 @@ tests/spapr-phb-test$(EXESUF): tests/spapr-phb-test.o $(libqos-obj-y)
 tests/fdc-test$(EXESUF): tests/fdc-test.o
 tests/ide-test$(EXESUF): tests/ide-test.o $(libqos-pc-obj-y)
 tests/ahci-test$(EXESUF): tests/ahci-test.o $(libqos-pc-obj-y)
+tests/ipmi-kcs-test$(EXESUF): tests/ipmi-kcs-test.o
+tests/ipmi-bt-test$(EXESUF): tests/ipmi-bt-test.o
 tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
 tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
-tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o $(libqos-obj-y)
+tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o \
+	tests/boot-sector.o $(libqos-obj-y)
+tests/pxe-test$(EXESUF): tests/pxe-test.o tests/boot-sector.o $(libqos-obj-y)
 tests/tmp105-test$(EXESUF): tests/tmp105-test.o $(libqos-omap-obj-y)
+tests/ds1338-test$(EXESUF): tests/ds1338-test.o $(libqos-imx-obj-y)
 tests/i440fx-test$(EXESUF): tests/i440fx-test.o $(libqos-pc-obj-y)
 tests/q35-test$(EXESUF): tests/q35-test.o $(libqos-pc-obj-y)
 tests/fw_cfg-test$(EXESUF): tests/fw_cfg-test.o $(libqos-pc-obj-y)
@@ -381,7 +555,7 @@ tests/wdt_ib700-test$(EXESUF): tests/wdt_ib700-test.o
 tests/tco-test$(EXESUF): tests/tco-test.o $(libqos-pc-obj-y)
 tests/virtio-balloon-test$(EXESUF): tests/virtio-balloon-test.o
 tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o $(libqos-virtio-obj-y)
-tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y)
+tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y) $(libqos-virtio-obj-y)
 tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o $(libqos-pc-obj-y)
 tests/virtio-scsi-test$(EXESUF): tests/virtio-scsi-test.o $(libqos-virtio-obj-y)
 tests/virtio-9p-test$(EXESUF): tests/virtio-9p-test.o
@@ -405,10 +579,15 @@ tests/usb-hcd-uhci-test$(EXESUF): tests/usb-hcd-uhci-test.o $(libqos-usb-obj-y)
 tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-usb-obj-y)
 tests/usb-hcd-xhci-test$(EXESUF): tests/usb-hcd-xhci-test.o $(libqos-usb-obj-y)
 tests/pc-cpu-test$(EXESUF): tests/pc-cpu-test.o
-tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y)
+tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y) $(test-io-obj-y)
 tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
-tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o libqemuutil.a libqemustub.a
-tests/test-write-threshold$(EXESUF): tests/test-write-threshold.o $(block-obj-y) libqemuutil.a libqemustub.a
+tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o $(test-util-obj-y)
+tests/test-write-threshold$(EXESUF): tests/test-write-threshold.o $(test-block-obj-y)
+tests/test-netfilter$(EXESUF): tests/test-netfilter.o $(qtest-obj-y)
+tests/test-filter-mirror$(EXESUF): tests/test-filter-mirror.o $(qtest-obj-y)
+tests/test-filter-redirector$(EXESUF): tests/test-filter-redirector.o $(qtest-obj-y)
+tests/ivshmem-test$(EXESUF): tests/ivshmem-test.o contrib/ivshmem-server/ivshmem-server.o $(libqos-pc-obj-y)
+tests/vhost-user-bridge$(EXESUF): tests/vhost-user-bridge.o
 
 ifeq ($(CONFIG_POSIX),y)
 LIBS += -lutil
@@ -418,13 +597,18 @@ endif
 
 TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
 ifeq ($(CONFIG_POSIX),y)
-QTEST_TARGETS=$(foreach TARGET,$(TARGETS), $(if $(check-qtest-$(TARGET)-y), $(TARGET),))
+QTEST_TARGETS = $(TARGETS)
 check-qtest-y=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
+check-qtest-y += $(check-qtest-generic-y)
+else
+QTEST_TARGETS =
 endif
 
-qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
+qtest-obj-y = tests/libqtest.o $(test-util-obj-y)
 $(check-qtest-y): $(qtest-obj-y)
 
+tests/test-qga: tests/test-qga.o $(qtest-obj-y)
+
 .PHONY: check-help
 check-help:
 	@echo "Regression testing targets:"
@@ -457,8 +641,8 @@ $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: $(check-qtest-y)
 	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
 		QTEST_QEMU_IMG=qemu-img$(EXESUF) \
 		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
-		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
-	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y); do \
+		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y) $(check-qtest-generic-y),"GTESTER $@")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y) $(gcov-files-generic-y); do \
 	  echo Gcov report for $$f:;\
 	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
 	done,)
@@ -469,7 +653,7 @@ $(patsubst %, check-%, $(check-unit-y)): check-%: %
 	$(call quiet-command, \
 		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
 		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $*,"GTESTER $*")
-	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y); do \
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y) $(gcov-files-generic-y); do \
 	  echo Gcov report for $$f:;\
 	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
 	done,)
@@ -479,7 +663,7 @@ $(patsubst %, check-%, $(check-unit-y)): check-%: %
 $(patsubst %, check-report-qtest-%.xml, $(QTEST_TARGETS)): check-report-qtest-%.xml: $(check-qtest-y)
 	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
 		QTEST_QEMU_IMG=qemu-img$(EXESUF) \
-	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
+	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y) $(check-qtest-generic-y),"GTESTER $@")
 
 check-report-unit.xml: $(check-unit-y)
 	$(call quiet-command,gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $^, "GTESTER $@")
@@ -527,7 +711,7 @@ check: check-qapi-schema check-unit check-qtest
 check-clean:
 	$(MAKE) -C tests/tcg clean
 	rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)
-	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)))
+	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y))
 
 clean: check-clean
 
diff --git a/qemu/tests/ac97-test.c b/qemu/tests/ac97-test.c
index af30ea1dd..75cab8f98 100644
--- a/qemu/tests/ac97-test.c
+++ b/qemu/tests/ac97-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/acpi-test-data/pc/DSDT b/qemu/tests/acpi-test-data/pc/DSDT
index c658203db..9d1274d3c 100644
--- a/qemu/tests/acpi-test-data/pc/DSDT
+++ b/qemu/tests/acpi-test-data/pc/DSDT
diff --git a/qemu/tests/acpi-test-data/pc/SSDT.bridge b/qemu/tests/acpi-test-data/pc/DSDT.bridge
index 6e6660b1f..cf48c62aa 100644
--- a/qemu/tests/acpi-test-data/pc/SSDT.bridge
+++ b/qemu/tests/acpi-test-data/pc/DSDT.bridge
diff --git a/qemu/tests/acpi-test-data/pc/SSDT b/qemu/tests/acpi-test-data/pc/SSDT
deleted file mode 100644
index 210d6a71e..000000000
--- a/qemu/tests/acpi-test-data/pc/SSDT
+++ /dev/null
diff --git a/qemu/tests/acpi-test-data/q35/DSDT b/qemu/tests/acpi-test-data/q35/DSDT
index 4723e5954..1c089c34b 100644
--- a/qemu/tests/acpi-test-data/q35/DSDT
+++ b/qemu/tests/acpi-test-data/q35/DSDT
diff --git a/qemu/pc-bios/q35-acpi-dsdt.aml b/qemu/tests/acpi-test-data/q35/DSDT.bridge
index d71b3a328..b29fcda0b 100644
--- a/qemu/pc-bios/q35-acpi-dsdt.aml
+++ b/qemu/tests/acpi-test-data/q35/DSDT.bridge
diff --git a/qemu/tests/acpi-test-data/q35/SSDT b/qemu/tests/acpi-test-data/q35/SSDT
deleted file mode 100644
index 0970c67dd..000000000
--- a/qemu/tests/acpi-test-data/q35/SSDT
+++ /dev/null
diff --git a/qemu/tests/acpi-test-data/q35/SSDT.bridge b/qemu/tests/acpi-test-data/q35/SSDT.bridge
deleted file mode 100644
index a77868861..000000000
--- a/qemu/tests/acpi-test-data/q35/SSDT.bridge
+++ /dev/null
diff --git a/qemu/tests/ahci-test.c b/qemu/tests/ahci-test.c
index 87d769186..6869f7f46 100644
--- a/qemu/tests/ahci-test.c
+++ b/qemu/tests/ahci-test.c
@@ -22,9 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <getopt.h>
 #include <glib.h>
 
@@ -39,15 +37,17 @@
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/pci_regs.h"
 
-/* Test-specific defines -- in MiB */
-#define TEST_IMAGE_SIZE_MB (200 * 1024)
-#define TEST_IMAGE_SECTORS ((TEST_IMAGE_SIZE_MB / AHCI_SECTOR_SIZE)     \
-                            * 1024 * 1024)
+/* Test images sizes in MB */
+#define TEST_IMAGE_SIZE_MB_LARGE (200 * 1024)
+#define TEST_IMAGE_SIZE_MB_SMALL 64
 
 /*** Globals ***/
 static char tmp_path[] = "/tmp/qtest.XXXXXX";
 static char debug_path[] = "/tmp/qtest-blkdebug.XXXXXX";
+static char mig_socket[] = "/tmp/qtest-migration.XXXXXX";
 static bool ahci_pedantic;
+static const char *imgfmt;
+static unsigned test_image_size_mb;
 
 /*** Function Declarations ***/
 static void ahci_test_port_spec(AHCIQState *ahci, uint8_t port);
@@ -60,6 +60,11 @@ static void ahci_test_pmcap(AHCIQState *ahci, uint8_t offset);
 
 /*** Utilities ***/
 
+static uint64_t mb_to_sectors(uint64_t image_size_mb)
+{
+    return (image_size_mb * 1024 * 1024) / AHCI_SECTOR_SIZE;
+}
+
 static void string_bswap16(uint16_t *s, size_t bytes)
 {
     g_assert_cmphex((bytes & 1), ==, 0);
@@ -71,32 +76,6 @@ static void string_bswap16(uint16_t *s, size_t bytes)
     }
 }
 
-static void generate_pattern(void *buffer, size_t len, size_t cycle_len)
-{
-    int i, j;
-    unsigned char *tx = (unsigned char *)buffer;
-    unsigned char p;
-    size_t *sx;
-
-    /* Write an indicative pattern that varies and is unique per-cycle */
-    p = rand() % 256;
-    for (i = j = 0; i < len; i++, j++) {
-        tx[i] = p;
-        if (j % cycle_len == 0) {
-            p = rand() % 256;
-        }
-    }
-
-    /* force uniqueness by writing an id per-cycle */
-    for (i = 0; i < len / cycle_len; i++) {
-        j = i * cycle_len;
-        if (j + sizeof(*sx) <= len) {
-            sx = (size_t *)&tx[j];
-            *sx = i;
-        }
-    }
-}
-
 /**
  * Verify that the transfer did not corrupt our state at all.
  */
@@ -140,8 +119,11 @@ static void ahci_migrate(AHCIQState *from, AHCIQState *to, const char *uri)
 {
     QOSState *tmp = to->parent;
     QPCIDevice *dev = to->dev;
+    char *uri_local = NULL;
+
     if (uri == NULL) {
-        uri = "tcp:127.0.0.1:1234";
+        uri_local = g_strdup_printf("%s%s", "unix:", mig_socket);
+        uri = uri_local;
     }
 
     /* context will be 'to' after completion. */
@@ -161,6 +143,7 @@ static void ahci_migrate(AHCIQState *from, AHCIQState *to, const char *uri)
     from->dev = dev;
 
     verify_state(to);
+    g_free(uri_local);
 }
 
 /*** Test Setup & Teardown ***/
@@ -196,11 +179,11 @@ static AHCIQState *ahci_boot(const char *cli, ...)
         va_end(ap);
     } else {
         cli = "-drive if=none,id=drive0,file=%s,cache=writeback,serial=%s"
-            ",format=qcow2"
+            ",format=%s"
             " -M q35 "
             "-device ide-hd,drive=drive0 "
             "-global ide-hd.ver=%s";
-        s = ahci_boot(cli, tmp_path, "testdisk", "version");
+        s = ahci_boot(cli, tmp_path, "testdisk", imgfmt, "version");
     }
 
     return s;
@@ -230,6 +213,7 @@ static AHCIQState *ahci_boot_and_enable(const char *cli, ...)
     va_list ap;
     uint16_t buff[256];
     uint8_t port;
+    uint8_t hello;
 
     if (cli) {
         va_start(ap, cli);
@@ -244,7 +228,12 @@ static AHCIQState *ahci_boot_and_enable(const char *cli, ...)
     /* Initialize test device */
     port = ahci_port_select(ahci);
     ahci_port_clear(ahci, port);
-    ahci_io(ahci, port, CMD_IDENTIFY, &buff, sizeof(buff), 0);
+    if (is_atapi(ahci, port)) {
+        hello = CMD_PACKET_ID;
+    } else {
+        hello = CMD_IDENTIFY;
+    }
+    ahci_io(ahci, port, hello, &buff, sizeof(buff), 0);
 
     return ahci;
 }
@@ -899,18 +888,12 @@ static void ahci_test_io_rw_simple(AHCIQState *ahci, unsigned bufsize,
 static uint8_t ahci_test_nondata(AHCIQState *ahci, uint8_t ide_cmd)
 {
     uint8_t port;
-    AHCICommand *cmd;
 
     /* Sanitize */
     port = ahci_port_select(ahci);
     ahci_port_clear(ahci, port);
 
-    /* Issue Command */
-    cmd = ahci_command_create(ide_cmd);
-    ahci_command_commit(ahci, cmd, port);
-    ahci_command_issue(ahci, cmd);
-    ahci_command_verify(ahci, cmd);
-    ahci_command_free(cmd);
+    ahci_io(ahci, port, ide_cmd, NULL, 0, 0);
 
     return port;
 }
@@ -926,7 +909,7 @@ static void ahci_test_max(AHCIQState *ahci)
     uint64_t nsect;
     uint8_t port;
     uint8_t cmd;
-    uint64_t config_sect = TEST_IMAGE_SECTORS - 1;
+    uint64_t config_sect = mb_to_sectors(test_image_size_mb) - 1;
 
     if (config_sect > 0xFFFFFF) {
         cmd = CMD_READ_MAX_EXT;
@@ -1060,14 +1043,14 @@ static void test_dma_fragmented(void)
     ahci_command_commit(ahci, cmd, px);
     ahci_command_issue(ahci, cmd);
     ahci_command_verify(ahci, cmd);
-    g_free(cmd);
+    ahci_command_free(cmd);
 
     cmd = ahci_command_create(CMD_READ_DMA);
     ahci_command_adjust(cmd, 0, ptr, bufsize, 32);
     ahci_command_commit(ahci, cmd, px);
     ahci_command_issue(ahci, cmd);
     ahci_command_verify(ahci, cmd);
-    g_free(cmd);
+    ahci_command_free(cmd);
 
     /* Read back the guest's receive buffer into local memory */
     bufread(ptr, rx, bufsize);
@@ -1095,33 +1078,23 @@ static void test_flush_retry(void)
     AHCIQState *ahci;
     AHCICommand *cmd;
     uint8_t port;
-    const char *s;
 
     prepare_blkdebug_script(debug_path, "flush_to_disk");
     ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                                "format=qcow2,cache=writeback,"
+                                "format=%s,cache=writeback,"
                                 "rerror=stop,werror=stop "
                                 "-M q35 "
                                 "-device ide-hd,drive=drive0 ",
                                 debug_path,
-                                tmp_path);
+                                tmp_path, imgfmt);
 
     /* Issue Flush Command and wait for error */
     port = ahci_port_select(ahci);
     ahci_port_clear(ahci, port);
-    cmd = ahci_command_create(CMD_FLUSH_CACHE);
-    ahci_command_commit(ahci, cmd, port);
-    ahci_command_issue_async(ahci, cmd);
-    qmp_eventwait("STOP");
 
-    /* Complete the command */
-    s = "{'execute':'cont' }";
-    qmp_async(s);
-    qmp_eventwait("RESUME");
-    ahci_command_wait(ahci, cmd);
-    ahci_command_verify(ahci, cmd);
+    cmd = ahci_guest_io_halt(ahci, port, CMD_FLUSH_CACHE, 0, 0, 0);
+    ahci_guest_io_resume(ahci, cmd);
 
-    ahci_command_free(cmd);
     ahci_shutdown(ahci);
 }
 
@@ -1131,18 +1104,19 @@ static void test_flush_retry(void)
 static void test_migrate_sanity(void)
 {
     AHCIQState *src, *dst;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     src = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s ", tmp_path);
+                    "-drive if=ide,file=%s,format=%s ", tmp_path, imgfmt);
     dst = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s "
-                    "-incoming %s", tmp_path, uri);
+                    "-drive if=ide,file=%s,format=%s "
+                    "-incoming %s", tmp_path, imgfmt, uri);
 
     ahci_migrate(src, dst, uri);
 
     ahci_shutdown(src);
     ahci_shutdown(dst);
+    g_free(uri);
 }
 
 /**
@@ -1155,14 +1129,14 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     src = ahci_boot_and_enable("-m 1024 -M q35 "
-                               "-hda %s ", tmp_path);
+                               "-drive if=ide,format=%s,file=%s ",
+                               imgfmt, tmp_path);
     dst = ahci_boot("-m 1024 -M q35 "
-                    "-hda %s "
-                    "-incoming %s", tmp_path, uri);
+                    "-drive if=ide,format=%s,file=%s "
+                    "-incoming %s", imgfmt, tmp_path, uri);
 
     set_context(src->parent);
 
@@ -1171,9 +1145,7 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     ahci_port_clear(src, px);
 
     /* create pattern */
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
 
     /* Write, migrate, then read. */
     ahci_io(src, px, cmd_write, tx, bufsize, 0);
@@ -1187,6 +1159,7 @@ static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
     ahci_shutdown(dst);
     g_free(rx);
     g_free(tx);
+    g_free(uri);
 }
 
 static void test_migrate_dma(void)
@@ -1213,29 +1186,25 @@ static void ahci_halted_io_test(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
     uint64_t ptr;
     AHCICommand *cmd;
 
     prepare_blkdebug_script(debug_path, "write_aio");
 
     ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                                "format=qcow2,cache=writeback,"
+                                "format=%s,cache=writeback,"
                                 "rerror=stop,werror=stop "
                                 "-M q35 "
                                 "-device ide-hd,drive=drive0 ",
                                 debug_path,
-                                tmp_path);
+                                tmp_path, imgfmt);
 
     /* Initialize and prepare */
     port = ahci_port_select(ahci);
     ahci_port_clear(ahci, port);
 
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
-
     /* create DMA source buffer and write pattern */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
     ptr = ahci_alloc(ahci, bufsize);
     g_assert(ptr);
     memwrite(ptr, tx, bufsize);
@@ -1282,38 +1251,34 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, uint8_t cmd_write)
     size_t bufsize = 4096;
     unsigned char *tx = g_malloc(bufsize);
     unsigned char *rx = g_malloc0(bufsize);
-    unsigned i;
     uint64_t ptr;
     AHCICommand *cmd;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     prepare_blkdebug_script(debug_path, "write_aio");
 
     src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                               "format=qcow2,cache=writeback,"
+                               "format=%s,cache=writeback,"
                                "rerror=stop,werror=stop "
                                "-M q35 "
                                "-device ide-hd,drive=drive0 ",
                                debug_path,
-                               tmp_path);
+                               tmp_path, imgfmt);
 
     dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
-                    "format=qcow2,cache=writeback,"
+                    "format=%s,cache=writeback,"
                     "rerror=stop,werror=stop "
                     "-M q35 "
                     "-device ide-hd,drive=drive0 "
                     "-incoming %s",
-                    tmp_path, uri);
+                    tmp_path, imgfmt, uri);
 
     set_context(src->parent);
 
     /* Initialize and prepare */
     port = ahci_port_select(src);
     ahci_port_clear(src, port);
-
-    for (i = 0; i < bufsize; i++) {
-        tx[i] = (bufsize - i);
-    }
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
 
     /* create DMA source buffer and write pattern */
     ptr = ahci_alloc(src, bufsize);
@@ -1338,6 +1303,7 @@ static void ahci_migrate_halted_io(uint8_t cmd_read, uint8_t cmd_write)
     ahci_shutdown(dst);
     g_free(rx);
     g_free(tx);
+    g_free(uri);
 }
 
 static void test_migrate_halted_dma(void)
@@ -1359,20 +1325,22 @@ static void test_flush_migrate(void)
     AHCICommand *cmd;
     uint8_t px;
     const char *s;
-    const char *uri = "tcp:127.0.0.1:1234";
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
 
     prepare_blkdebug_script(debug_path, "flush_to_disk");
 
     src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
-                               "cache=writeback,rerror=stop,werror=stop "
+                               "cache=writeback,rerror=stop,werror=stop,"
+                               "format=%s "
                                "-M q35 "
                                "-device ide-hd,drive=drive0 ",
-                               debug_path, tmp_path);
+                               debug_path, tmp_path, imgfmt);
     dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
-                    "cache=writeback,rerror=stop,werror=stop "
+                    "cache=writeback,rerror=stop,werror=stop,"
+                    "format=%s "
                     "-M q35 "
                     "-device ide-hd,drive=drive0 "
-                    "-incoming %s", tmp_path, uri);
+                    "-incoming %s", tmp_path, imgfmt, uri);
 
     set_context(src->parent);
 
@@ -1397,6 +1365,7 @@ static void test_flush_migrate(void)
     ahci_command_free(cmd);
     ahci_shutdown(src);
     ahci_shutdown(dst);
+    g_free(uri);
 }
 
 static void test_max(void)
@@ -1442,6 +1411,98 @@ static void test_ncq_simple(void)
     ahci_shutdown(ahci);
 }
 
+static int prepare_iso(size_t size, unsigned char **buf, char **name)
+{
+    char cdrom_path[] = "/tmp/qtest.iso.XXXXXX";
+    unsigned char *patt;
+    ssize_t ret;
+    int fd = mkstemp(cdrom_path);
+
+    g_assert(buf);
+    g_assert(name);
+    patt = g_malloc(size);
+
+    /* Generate a pattern and build a CDROM image to read from */
+    generate_pattern(patt, size, ATAPI_SECTOR_SIZE);
+    ret = write(fd, patt, size);
+    g_assert(ret == size);
+
+    *name = g_strdup(cdrom_path);
+    *buf = patt;
+    return fd;
+}
+
+static void remove_iso(int fd, char *name)
+{
+    unlink(name);
+    g_free(name);
+    close(fd);
+}
+
+static int ahci_cb_cmp_buff(AHCIQState *ahci, AHCICommand *cmd,
+                            const AHCIOpts *opts)
+{
+    unsigned char *tx = opts->opaque;
+    unsigned char *rx = g_malloc0(opts->size);
+
+    bufread(opts->buffer, rx, opts->size);
+    g_assert_cmphex(memcmp(tx, rx, opts->size), ==, 0);
+    g_free(rx);
+
+    return 0;
+}
+
+static void ahci_test_cdrom(int nsectors, bool dma)
+{
+    AHCIQState *ahci;
+    unsigned char *tx;
+    char *iso;
+    int fd;
+    AHCIOpts opts = {
+        .size = (ATAPI_SECTOR_SIZE * nsectors),
+        .atapi = true,
+        .atapi_dma = dma,
+        .post_cb = ahci_cb_cmp_buff,
+    };
+
+    /* Prepare ISO and fill 'tx' buffer */
+    fd = prepare_iso(1024 * 1024, &tx, &iso);
+    opts.opaque = tx;
+
+    /* Standard startup wonkery, but use ide-cd and our special iso file */
+    ahci = ahci_boot_and_enable("-drive if=none,id=drive0,file=%s,format=raw "
+                                "-M q35 "
+                                "-device ide-cd,drive=drive0 ", iso);
+
+    /* Build & Send AHCI command */
+    ahci_exec(ahci, ahci_port_select(ahci), CMD_ATAPI_READ_10, &opts);
+
+    /* Cleanup */
+    g_free(tx);
+    ahci_shutdown(ahci);
+    remove_iso(fd, iso);
+}
+
+static void test_cdrom_dma(void)
+{
+    ahci_test_cdrom(1, true);
+}
+
+static void test_cdrom_dma_multi(void)
+{
+    ahci_test_cdrom(3, true);
+}
+
+static void test_cdrom_pio(void)
+{
+    ahci_test_cdrom(1, false);
+}
+
+static void test_cdrom_pio_multi(void)
+{
+    ahci_test_cdrom(3, false);
+}
+
 /******************************************************************************/
 /* AHCI I/O Test Matrix Definitions                                           */
 
@@ -1513,7 +1574,7 @@ static uint64_t offset_sector(enum OffsetType ofst,
         return 1;
     case OFFSET_HIGH:
         ceil = (addr_type == ADDR_MODE_LBA28) ? 0xfffffff : 0xffffffffffff;
-        ceil = MIN(ceil, TEST_IMAGE_SECTORS - 1);
+        ceil = MIN(ceil, mb_to_sectors(test_image_size_mb) - 1);
         nsectors = buffsize / AHCI_SECTOR_SIZE;
         return ceil - nsectors + 1;
     default:
@@ -1595,8 +1656,9 @@ static void create_ahci_io_test(enum IOMode type, enum AddrMode addr,
                                 enum BuffLen len, enum OffsetType offset)
 {
     char *name;
-    AHCIIOTestOptions *opts = g_malloc(sizeof(AHCIIOTestOptions));
+    AHCIIOTestOptions *opts;
 
+    opts = g_malloc(sizeof(AHCIIOTestOptions));
     opts->length = len;
     opts->address_type = addr;
     opts->io_type = type;
@@ -1608,6 +1670,13 @@ static void create_ahci_io_test(enum IOMode type, enum AddrMode addr,
                            buff_len_str[len],
                            offset_str[offset]);
 
+    if ((addr == ADDR_MODE_LBA48) && (offset == OFFSET_HIGH) &&
+        (mb_to_sectors(test_image_size_mb) <= 0xFFFFFFF)) {
+        g_test_message("%s: skipped; test image too small", name);
+        g_free(name);
+        return;
+    }
+
     qtest_add_data_func(name, opts, test_io_interface);
     g_free(name);
 }
@@ -1654,15 +1723,33 @@ int main(int argc, char **argv)
         return 0;
     }
 
-    /* Create a temporary qcow2 image */
-    close(mkstemp(tmp_path));
-    mkqcow2(tmp_path, TEST_IMAGE_SIZE_MB);
+    /* Create a temporary image */
+    fd = mkstemp(tmp_path);
+    g_assert(fd >= 0);
+    if (have_qemu_img()) {
+        imgfmt = "qcow2";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_LARGE;
+        mkqcow2(tmp_path, TEST_IMAGE_SIZE_MB_LARGE);
+    } else {
+        g_test_message("QTEST_QEMU_IMG not set or qemu-img missing; "
+                       "skipping LBA48 high-sector tests");
+        imgfmt = "raw";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_SMALL;
+        ret = ftruncate(fd, test_image_size_mb * 1024 * 1024);
+        g_assert(ret == 0);
+    }
+    close(fd);
 
     /* Create temporary blkdebug instructions */
     fd = mkstemp(debug_path);
     g_assert(fd >= 0);
     close(fd);
 
+    /* Reserve a hollow file to use as a socket for migration tests */
+    fd = mkstemp(mig_socket);
+    g_assert(fd >= 0);
+    close(fd);
+
     /* Run the tests */
     qtest_add_func("/ahci/sanity",     test_sanity);
     qtest_add_func("/ahci/pci_spec",   test_pci_spec);
@@ -1700,11 +1787,17 @@ int main(int argc, char **argv)
     qtest_add_func("/ahci/io/ncq/retry", test_halted_ncq);
     qtest_add_func("/ahci/migrate/ncq/halted", test_migrate_halted_ncq);
 
+    qtest_add_func("/ahci/cdrom/dma/single", test_cdrom_dma);
+    qtest_add_func("/ahci/cdrom/dma/multi", test_cdrom_dma_multi);
+    qtest_add_func("/ahci/cdrom/pio/single", test_cdrom_pio);
+    qtest_add_func("/ahci/cdrom/pio/multi", test_cdrom_pio_multi);
+
     ret = g_test_run();
 
     /* Cleanup */
     unlink(tmp_path);
     unlink(debug_path);
+    unlink(mig_socket);
 
     return ret;
 }
diff --git a/qemu/tests/bios-tables-test.c b/qemu/tests/bios-tables-test.c
index 0de1742d7..03528140c 100644
--- a/qemu/tests/bios-tables-test.c
+++ b/qemu/tests/bios-tables-test.c
@@ -10,16 +10,15 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <glib/gstdio.h>
 #include "qemu-common.h"
 #include "libqtest.h"
-#include "qemu/compiler.h"
 #include "hw/acpi/acpi-defs.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/bitmap.h"
+#include "boot-sector.h"
 
 #define MACHINE_PC "pc"
 #define MACHINE_Q35 "q35"
@@ -50,16 +49,9 @@ typedef struct {
     int rsdt_tables_nr;
     GArray *tables;
     uint32_t smbios_ep_addr;
-    struct smbios_entry_point smbios_ep_table;
+    struct smbios_21_entry_point smbios_ep_table;
 } test_data;
 
-#define LOW(x) ((x) & 0xff)
-#define HIGH(x) ((x) >> 8)
-
-#define SIGNATURE 0xdead
-#define SIGNATURE_OFFSET 0x10
-#define BOOT_SECTOR_ADDRESS 0x7c00
-
 #define ACPI_READ_FIELD(field, addr)           \
     do {                                       \
         switch (sizeof(field)) {               \
@@ -119,35 +111,6 @@ typedef struct {
     g_assert_cmpstr(ACPI_ASSERT_CMP_str, ==, expected); \
 } while (0)
 
-/* Boot sector code: write SIGNATURE into memory,
- * then halt.
- * Q35 machine requires a minimum 0x7e000 bytes disk.
- * (bug or feature?)
- */
-static uint8_t boot_sector[0x7e000] = {
-    /* 7c00: mov $0xdead,%ax */
-    [0x00] = 0xb8,
-    [0x01] = LOW(SIGNATURE),
-    [0x02] = HIGH(SIGNATURE),
-    /* 7c03:  mov %ax,0x7c10 */
-    [0x03] = 0xa3,
-    [0x04] = LOW(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
-    [0x05] = HIGH(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
-    /* 7c06: cli */
-    [0x06] = 0xfa,
-    /* 7c07: hlt */
-    [0x07] = 0xf4,
-    /* 7c08: jmp 0x7c07=0x7c0a-3 */
-    [0x08] = 0xeb,
-    [0x09] = LOW(-3),
-    /* We mov 0xdead here: set value to make debugging easier */
-    [SIGNATURE_OFFSET] = LOW(0xface),
-    [SIGNATURE_OFFSET + 1] = HIGH(0xface),
-    /* End of boot sector marker */
-    [0x1FE] = 0x55,
-    [0x1FF] = 0xAA,
-};
-
 static const char *disk = "tests/acpi-test-disk.raw";
 static const char *data_dir = "tests/acpi-test-data";
 #ifdef CONFIG_IASL
@@ -161,31 +124,23 @@ static void free_test_data(test_data *data)
     AcpiSdtTable *temp;
     int i;
 
-    if (data->rsdt_tables_addr) {
-        g_free(data->rsdt_tables_addr);
-    }
+    g_free(data->rsdt_tables_addr);
 
     for (i = 0; i < data->tables->len; ++i) {
         temp = &g_array_index(data->tables, AcpiSdtTable, i);
-        if (temp->aml) {
-            g_free(temp->aml);
+        g_free(temp->aml);
+        if (temp->aml_file &&
+            !temp->tmp_files_retain &&
+            g_strstr_len(temp->aml_file, -1, "aml-")) {
+            unlink(temp->aml_file);
         }
-        if (temp->aml_file) {
-            if (!temp->tmp_files_retain &&
-                g_strstr_len(temp->aml_file, -1, "aml-")) {
-                unlink(temp->aml_file);
-            }
-            g_free(temp->aml_file);
-        }
-        if (temp->asl) {
-            g_free(temp->asl);
-        }
-        if (temp->asl_file) {
-            if (!temp->tmp_files_retain) {
-                unlink(temp->asl_file);
-            }
-            g_free(temp->asl_file);
+        g_free(temp->aml_file);
+        g_free(temp->asl);
+        if (temp->asl_file &&
+            !temp->tmp_files_retain) {
+            unlink(temp->asl_file);
         }
+        g_free(temp->asl_file);
     }
 
     g_array_free(data->tables, false);
@@ -420,9 +375,7 @@ static void dump_aml_files(test_data *data, bool rebuild)
 
         close(fd);
 
-        if (aml_file) {
-            g_free(aml_file);
-        }
+        g_free(aml_file);
     }
 }
 
@@ -479,7 +432,7 @@ static bool load_asl(GArray *sdts, AcpiSdtTable *sdt)
 
 #define COMMENT_END "*/"
 #define DEF_BLOCK "DefinitionBlock ("
-#define BLOCK_NAME_END ".aml"
+#define BLOCK_NAME_END ","
 
 static GString *normalize_asl(gchar *asl_code)
 {
@@ -590,6 +543,22 @@ static void test_acpi_asl(test_data *data)
                         (gchar *)&signature,
                         sdt->asl_file, sdt->aml_file,
                         exp_sdt->asl_file, exp_sdt->aml_file);
+                if (getenv("V")) {
+                    const char *diff_cmd = getenv("DIFF");
+                    if (diff_cmd) {
+                        int ret G_GNUC_UNUSED;
+                        char *diff = g_strdup_printf("%s %s %s", diff_cmd,
+                            exp_sdt->asl_file, sdt->asl_file);
+                        ret = system(diff) ;
+                        g_free(diff);
+                    } else {
+                        fprintf(stderr, "acpi-test: Warning. not showing "
+                            "difference since no diff utility is specified. "
+                            "Set 'DIFF' environment variable to a preferred "
+                            "diff utility and run 'make V=1 check' again to "
+                            "see ASL difference.");
+                    }
+                }
           }
         }
         g_string_free(asl, true);
@@ -601,7 +570,7 @@ static void test_acpi_asl(test_data *data)
 
 static bool smbios_ep_table_ok(test_data *data)
 {
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
     uint32_t addr = data->smbios_ep_addr;
 
     ACPI_READ_ARRAY(ep_table->anchor_string, addr);
@@ -681,7 +650,7 @@ static inline bool smbios_single_instance(uint8_t type)
 static void test_smbios_structs(test_data *data)
 {
     DECLARE_BITMAP(struct_bitmap, SMBIOS_MAX_TYPE+1) = { 0 };
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
     uint32_t addr = ep_table->structure_table_address;
     int i, len, max_len = 0;
     uint8_t type, prv, crt;
@@ -733,10 +702,6 @@ static void test_smbios_structs(test_data *data)
 static void test_acpi_one(const char *params, test_data *data)
 {
     char *args;
-    uint8_t signature_low;
-    uint8_t signature_high;
-    uint16_t signature;
-    int i;
 
     args = g_strdup_printf("-net none -display none %s "
                            "-drive id=hd0,if=none,file=%s,format=raw "
@@ -745,24 +710,7 @@ static void test_acpi_one(const char *params, test_data *data)
 
     qtest_start(args);
 
-   /* Wait at most 1 minute */
-#define TEST_DELAY (1 * G_USEC_PER_SEC / 10)
-#define TEST_CYCLES MAX((60 * G_USEC_PER_SEC / TEST_DELAY), 1)
-
-    /* Poll until code has run and modified memory.  Once it has we know BIOS
-     * initialization is done.  TODO: check that IP reached the halt
-     * instruction.
-     */
-    for (i = 0; i < TEST_CYCLES; ++i) {
-        signature_low = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET);
-        signature_high = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET + 1);
-        signature = (signature_high << 8) | signature_low;
-        if (signature == SIGNATURE) {
-            break;
-        }
-        g_usleep(TEST_DELAY);
-    }
-    g_assert_cmphex(signature, ==, SIGNATURE);
+    boot_sector_test();
 
     test_acpi_rsdp_address(data);
     test_acpi_rsdp_table(data);
@@ -836,15 +784,11 @@ static void test_acpi_q35_tcg_bridge(void)
 int main(int argc, char *argv[])
 {
     const char *arch = qtest_get_arch();
-    FILE *f = fopen(disk, "w");
     int ret;
 
-    if (!f) {
-        fprintf(stderr, "Couldn't open \"%s\": %s", disk, strerror(errno));
-        return 1;
-    }
-    fwrite(boot_sector, 1, sizeof boot_sector, f);
-    fclose(f);
+    ret = boot_sector_init(disk);
+    if(ret)
+        return ret;
 
     g_test_init(&argc, &argv, NULL);
 
@@ -855,6 +799,6 @@ int main(int argc, char *argv[])
         qtest_add_func("acpi/q35/tcg/bridge", test_acpi_q35_tcg_bridge);
     }
     ret = g_test_run();
-    unlink(disk);
+    boot_sector_cleanup(disk);
     return ret;
 }
diff --git a/qemu/tests/boot-order-test.c b/qemu/tests/boot-order-test.c
index 360a6911e..a6d8bd5cb 100644
--- a/qemu/tests/boot-order-test.c
+++ b/qemu/tests/boot-order-test.c
@@ -10,14 +10,12 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "libqos/fw_cfg.h"
 #include "libqtest.h"
 
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
-#undef NO_QEMU_PROTOS
+#include "hw/nvram/fw_cfg_keys.h"
 
 typedef struct {
     const char *args;
diff --git a/qemu/tests/boot-sector.c b/qemu/tests/boot-sector.c
new file mode 100644
index 000000000..3ffe2987f
--- /dev/null
+++ b/qemu/tests/boot-sector.c
@@ -0,0 +1,118 @@
+/*
+ * QEMU boot sector testing helpers.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *  Victor Kaplansky <victork@redhat.com>    
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "boot-sector.h"
+#include "qemu-common.h"
+#include "libqtest.h"
+
+#define LOW(x) ((x) & 0xff)
+#define HIGH(x) ((x) >> 8)
+
+#define SIGNATURE 0xdead
+#define SIGNATURE_OFFSET 0x10
+#define BOOT_SECTOR_ADDRESS 0x7c00
+
+/* Boot sector code: write SIGNATURE into memory,
+ * then halt.
+ * Q35 machine requires a minimum 0x7e000 bytes disk.
+ * (bug or feature?)
+ */
+static uint8_t boot_sector[0x7e000] = {
+    /* The first sector will be placed at RAM address 00007C00, and
+     * the BIOS transfers control to 00007C00
+     */
+
+    /* Data Segment register should be initialized, since pxe
+     * boot loader can leave it dirty.
+     */
+
+    /* 7c00: move $0000,%ax */
+    [0x00] = 0xb8,
+    [0x01] = 0x00,
+    [0x02] = 0x00,
+    /* 7c03: move %ax,%ds */
+    [0x03] = 0x8e,
+    [0x04] = 0xd8,
+
+    /* 7c05: mov $0xdead,%ax */
+    [0x05] = 0xb8,
+    [0x06] = LOW(SIGNATURE),
+    [0x07] = HIGH(SIGNATURE),
+    /* 7c08:  mov %ax,0x7c10 */
+    [0x08] = 0xa3,
+    [0x09] = LOW(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
+    [0x0a] = HIGH(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
+
+    /* 7c0b cli */
+    [0x0b] = 0xfa,
+    /* 7c0c: hlt */
+    [0x0c] = 0xf4,
+    /* 7c0e: jmp 0x7c07=0x7c0f-3 */
+    [0x0d] = 0xeb,
+    [0x0e] = LOW(-3),
+    /* We mov 0xdead here: set value to make debugging easier */
+    [SIGNATURE_OFFSET] = LOW(0xface),
+    [SIGNATURE_OFFSET + 1] = HIGH(0xface),
+    /* End of boot sector marker */
+    [0x1FE] = 0x55,
+    [0x1FF] = 0xAA,
+};
+
+/* Create boot disk file.  */
+int boot_sector_init(const char *fname)
+{
+    FILE *f = fopen(fname, "w");
+
+    if (!f) {
+        fprintf(stderr, "Couldn't open \"%s\": %s", fname, strerror(errno));
+        return 1;
+    }
+    fwrite(boot_sector, 1, sizeof boot_sector, f);
+    fclose(f);
+    return 0;
+}
+
+/* Loop until signature in memory is OK.  */
+void boot_sector_test(void)
+{
+    uint8_t signature_low;
+    uint8_t signature_high;
+    uint16_t signature;
+    int i;
+
+   /* Wait at most 1 minute */
+#define TEST_DELAY (1 * G_USEC_PER_SEC / 10)
+#define TEST_CYCLES MAX((60 * G_USEC_PER_SEC / TEST_DELAY), 1)
+
+    /* Poll until code has run and modified memory.  Once it has we know BIOS
+     * initialization is done.  TODO: check that IP reached the halt
+     * instruction.
+     */
+    for (i = 0; i < TEST_CYCLES; ++i) {
+        signature_low = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET);
+        signature_high = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET + 1);
+        signature = (signature_high << 8) | signature_low;
+        if (signature == SIGNATURE) {
+            break;
+        }
+        g_usleep(TEST_DELAY);
+    }
+
+    g_assert_cmphex(signature, ==, SIGNATURE);
+}
+
+/* unlink boot disk file.  */
+void boot_sector_cleanup(const char *fname)
+{
+    unlink(fname);
+}
diff --git a/qemu/tests/boot-sector.h b/qemu/tests/boot-sector.h
new file mode 100644
index 000000000..38be0290e
--- /dev/null
+++ b/qemu/tests/boot-sector.h
@@ -0,0 +1,26 @@
+/*
+ * QEMU boot sector testing helpers.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *  Victor Kaplansky <victork@redhat.com>    
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef TEST_BOOT_SECTOR
+#define TEST_BOOT_SECTOR
+
+/* Create boot disk file.  */
+int boot_sector_init(const char *fname);
+
+/* Loop until signature in memory is OK.  */
+void boot_sector_test(void);
+
+/* unlink boot disk file.  */
+void boot_sector_cleanup(const char *fname);
+
+#endif /* TEST_BOOT_SECTOR */
diff --git a/qemu/tests/check-block.sh b/qemu/tests/check-block.sh
index b9d9c6a9f..a37797a49 100755
--- a/qemu/tests/check-block.sh
+++ b/qemu/tests/check-block.sh
@@ -9,7 +9,7 @@ if [ ! -x $QEMU_PROG ]; then
     exit 1
 fi
 
-cd $SRC_PATH/tests/qemu-iotests
+cd tests/qemu-iotests
 
 ret=0
 ./check -T -nocache -raw || ret=1
diff --git a/qemu/tests/check-qdict.c b/qemu/tests/check-qdict.c
index a136f2add..a43056c5d 100644
--- a/qemu/tests/check-qdict.c
+++ b/qemu/tests/check-qdict.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qint.h"
diff --git a/qemu/tests/check-qfloat.c b/qemu/tests/check-qfloat.c
index 6404ac8df..3102608f5 100644
--- a/qemu/tests/check-qfloat.c
+++ b/qemu/tests/check-qfloat.c
@@ -10,6 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qfloat.h"
diff --git a/qemu/tests/check-qint.c b/qemu/tests/check-qint.c
index 86868844a..c86f7dfa3 100644
--- a/qemu/tests/check-qint.c
+++ b/qemu/tests/check-qint.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qint.h"
diff --git a/qemu/tests/check-qjson.c b/qemu/tests/check-qjson.c
index 1cfffa593..99de6f525 100644
--- a/qemu/tests/check-qjson.c
+++ b/qemu/tests/check-qjson.c
@@ -10,6 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qstring.h"
@@ -1484,6 +1485,30 @@ static void unterminated_literal(void)
     g_assert(obj == NULL);
 }
 
+static char *make_nest(char *buf, size_t cnt)
+{
+    memset(buf, '[', cnt - 1);
+    buf[cnt - 1] = '{';
+    buf[cnt] = '}';
+    memset(buf + cnt + 1, ']', cnt - 1);
+    buf[2 * cnt] = 0;
+    return buf;
+}
+
+static void limits_nesting(void)
+{
+    enum { max_nesting = 1024 }; /* see qobject/json-streamer.c */
+    char buf[2 * (max_nesting + 1) + 1];
+    QObject *obj;
+
+    obj = qobject_from_json(make_nest(buf, max_nesting));
+    g_assert(obj != NULL);
+    qobject_decref(obj);
+
+    obj = qobject_from_json(make_nest(buf, max_nesting + 1));
+    g_assert(obj == NULL);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -1519,6 +1544,7 @@ int main(int argc, char **argv)
     g_test_add_func("/errors/invalid_array_comma", invalid_array_comma);
     g_test_add_func("/errors/invalid_dict_comma", invalid_dict_comma);
     g_test_add_func("/errors/unterminated/literal", unterminated_literal);
+    g_test_add_func("/errors/limits/nesting", limits_nesting);
 
     return g_test_run();
 }
diff --git a/qemu/tests/check-qlist.c b/qemu/tests/check-qlist.c
index b9c05d43f..f231d5fa9 100644
--- a/qemu/tests/check-qlist.c
+++ b/qemu/tests/check-qlist.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qint.h"
diff --git a/qemu/tests/check-qom-interface.c b/qemu/tests/check-qom-interface.c
index f06380ef1..09354deb7 100644
--- a/qemu/tests/check-qom-interface.c
+++ b/qemu/tests/check-qom-interface.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qom/object.h"
diff --git a/qemu/tests/check-qom-proplist.c b/qemu/tests/check-qom-proplist.c
index 7400b1fce..ffffd872f 100644
--- a/qemu/tests/check-qom-proplist.c
+++ b/qemu/tests/check-qom-proplist.c
@@ -18,8 +18,10 @@
  * Author: Daniel P. Berrange <berrange@redhat.com>
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
+#include "qapi/error.h"
 #include "qom/object.h"
 #include "qemu/module.h"
 
@@ -123,18 +125,28 @@ static void dummy_init(Object *obj)
                              dummy_get_bv,
                              dummy_set_bv,
                              NULL);
-    object_property_add_str(obj, "sv",
-                            dummy_get_sv,
-                            dummy_set_sv,
-                            NULL);
-    object_property_add_enum(obj, "av",
-                             "DummyAnimal",
-                             dummy_animal_map,
-                             dummy_get_av,
-                             dummy_set_av,
-                             NULL);
 }
 
+
+static void dummy_class_init(ObjectClass *cls, void *data)
+{
+    object_class_property_add_bool(cls, "bv",
+                                   dummy_get_bv,
+                                   dummy_set_bv,
+                                   NULL);
+    object_class_property_add_str(cls, "sv",
+                                  dummy_get_sv,
+                                  dummy_set_sv,
+                                  NULL);
+    object_class_property_add_enum(cls, "av",
+                                   "DummyAnimal",
+                                   dummy_animal_map,
+                                   dummy_get_av,
+                                   dummy_set_av,
+                                   NULL);
+}
+
+
 static void dummy_finalize(Object *obj)
 {
     DummyObject *dobj = DUMMY_OBJECT(obj);
@@ -150,8 +162,151 @@ static const TypeInfo dummy_info = {
     .instance_init = dummy_init,
     .instance_finalize = dummy_finalize,
     .class_size = sizeof(DummyObjectClass),
+    .class_init = dummy_class_init,
+};
+
+
+/*
+ * The following 3 object classes are used to
+ * simulate the kind of relationships seen in
+ * qdev, which result in complex object
+ * property destruction ordering.
+ *
+ * DummyDev has a 'bus' child to a DummyBus
+ * DummyBus has a 'backend' child to a DummyBackend
+ * DummyDev has a 'backend' link to DummyBackend
+ *
+ * When DummyDev is finalized, it unparents the
+ * DummyBackend, which unparents the DummyDev
+ * which deletes the 'backend' link from DummyDev
+ * to DummyBackend. This illustrates that the
+ * object_property_del_all() method needs to
+ * cope with the list of properties being changed
+ * while it iterates over them.
+ */
+typedef struct DummyDev DummyDev;
+typedef struct DummyDevClass DummyDevClass;
+typedef struct DummyBus DummyBus;
+typedef struct DummyBusClass DummyBusClass;
+typedef struct DummyBackend DummyBackend;
+typedef struct DummyBackendClass DummyBackendClass;
+
+#define TYPE_DUMMY_DEV "qemu-dummy-dev"
+#define TYPE_DUMMY_BUS "qemu-dummy-bus"
+#define TYPE_DUMMY_BACKEND "qemu-dummy-backend"
+
+#define DUMMY_DEV(obj)                               \
+    OBJECT_CHECK(DummyDev, (obj), TYPE_DUMMY_DEV)
+#define DUMMY_BUS(obj)                               \
+    OBJECT_CHECK(DummyBus, (obj), TYPE_DUMMY_BUS)
+#define DUMMY_BACKEND(obj)                               \
+    OBJECT_CHECK(DummyBackend, (obj), TYPE_DUMMY_BACKEND)
+
+struct DummyDev {
+    Object parent_obj;
+
+    DummyBus *bus;
+};
+
+struct DummyDevClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBus {
+    Object parent_obj;
+
+    DummyBackend *backend;
+};
+
+struct DummyBusClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBackend {
+    Object parent_obj;
+};
+
+struct DummyBackendClass {
+    ObjectClass parent_class;
+};
+
+
+static void dummy_dev_init(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    DummyBus *bus = DUMMY_BUS(object_new(TYPE_DUMMY_BUS));
+    DummyBackend *backend = DUMMY_BACKEND(object_new(TYPE_DUMMY_BACKEND));
+
+    object_property_add_child(obj, "bus", OBJECT(bus), NULL);
+    dev->bus = bus;
+    object_property_add_child(OBJECT(bus), "backend", OBJECT(backend), NULL);
+    bus->backend = backend;
+
+    object_property_add_link(obj, "backend", TYPE_DUMMY_BACKEND,
+                             (Object **)&bus->backend, NULL, 0, NULL);
+}
+
+static void dummy_dev_unparent(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    object_unparent(OBJECT(dev->bus));
+}
+
+static void dummy_dev_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_dev_unparent;
+}
+
+
+static void dummy_bus_init(Object *obj)
+{
+}
+
+static void dummy_bus_unparent(Object *obj)
+{
+    DummyBus *bus = DUMMY_BUS(obj);
+    object_property_del(obj->parent, "backend", NULL);
+    object_unparent(OBJECT(bus->backend));
+}
+
+static void dummy_bus_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_bus_unparent;
+}
+
+static void dummy_backend_init(Object *obj)
+{
+}
+
+
+static const TypeInfo dummy_dev_info = {
+    .name          = TYPE_DUMMY_DEV,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyDev),
+    .instance_init = dummy_dev_init,
+    .class_size = sizeof(DummyDevClass),
+    .class_init = dummy_dev_class_init,
+};
+
+static const TypeInfo dummy_bus_info = {
+    .name          = TYPE_DUMMY_BUS,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBus),
+    .instance_init = dummy_bus_init,
+    .class_size = sizeof(DummyBusClass),
+    .class_init = dummy_bus_class_init,
+};
+
+static const TypeInfo dummy_backend_info = {
+    .name          = TYPE_DUMMY_BACKEND,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBackend),
+    .instance_init = dummy_backend_init,
+    .class_size = sizeof(DummyBackendClass),
 };
 
+
+
 static void test_dummy_createv(void)
 {
     Error *err = NULL;
@@ -283,20 +438,82 @@ static void test_dummy_getenum(void)
                                    &err);
     g_assert(err != NULL);
     error_free(err);
+
+    object_unparent(OBJECT(dobj));
 }
 
 
+static void test_dummy_iterator(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        object_new_with_props(TYPE_DUMMY,
+                              parent,
+                              "dummy0",
+                              &error_abort,
+                              "bv", "yes",
+                              "sv", "Hiss hiss hiss",
+                              "av", "platypus",
+                              NULL));
+
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+    bool seenbv = false, seensv = false, seenav = false, seentype;
+
+    object_property_iter_init(&iter, OBJECT(dobj));
+    while ((prop = object_property_iter_next(&iter))) {
+        if (g_str_equal(prop->name, "bv")) {
+            seenbv = true;
+        } else if (g_str_equal(prop->name, "sv")) {
+            seensv = true;
+        } else if (g_str_equal(prop->name, "av")) {
+            seenav = true;
+        } else if (g_str_equal(prop->name, "type")) {
+            /* This prop comes from the base Object class */
+            seentype = true;
+        } else {
+            g_printerr("Found prop '%s'\n", prop->name);
+            g_assert_not_reached();
+        }
+    }
+    g_assert(seenbv);
+    g_assert(seenav);
+    g_assert(seensv);
+    g_assert(seentype);
+
+    object_unparent(OBJECT(dobj));
+}
+
+
+static void test_dummy_delchild(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyDev *dev = DUMMY_DEV(
+        object_new_with_props(TYPE_DUMMY_DEV,
+                              parent,
+                              "dev0",
+                              &error_abort,
+                              NULL));
+
+    object_unparent(OBJECT(dev));
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
 
     module_call_init(MODULE_INIT_QOM);
     type_register_static(&dummy_info);
+    type_register_static(&dummy_dev_info);
+    type_register_static(&dummy_bus_info);
+    type_register_static(&dummy_backend_info);
 
     g_test_add_func("/qom/proplist/createlist", test_dummy_createlist);
     g_test_add_func("/qom/proplist/createv", test_dummy_createv);
     g_test_add_func("/qom/proplist/badenum", test_dummy_badenum);
     g_test_add_func("/qom/proplist/getenum", test_dummy_getenum);
+    g_test_add_func("/qom/proplist/iterator", test_dummy_iterator);
+    g_test_add_func("/qom/proplist/delchild", test_dummy_delchild);
 
     return g_test_run();
 }
diff --git a/qemu/tests/check-qstring.c b/qemu/tests/check-qstring.c
index 95dc9e3e7..9877b42c8 100644
--- a/qemu/tests/check-qstring.c
+++ b/qemu/tests/check-qstring.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qapi/qmp/qstring.h"
diff --git a/qemu/tests/crypto-tls-x509-helpers.c b/qemu/tests/crypto-tls-x509-helpers.c
new file mode 100644
index 000000000..64073d3bd
--- /dev/null
+++ b/qemu/tests/crypto-tls-x509-helpers.c
@@ -0,0 +1,485 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "qemu/sockets.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+/*
+ * This stores some static data that is needed when
+ * encoding extensions in the x509 certs
+ */
+ASN1_TYPE pkix_asn1;
+
+/*
+ * To avoid consuming random entropy to generate keys,
+ * here's one we prepared earlier :-)
+ */
+gnutls_x509_privkey_t privkey;
+# define PRIVATE_KEY                                              \
+    "-----BEGIN PRIVATE KEY-----\n"                               \
+    "MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBALVcr\n"     \
+    "BL40Tm6yq88FBhJNw1aaoCjmtg0l4dWQZ/e9Fimx4ARxFpT+ji4FE\n"     \
+    "Cgl9s/SGqC+1nvlkm9ViSo0j7MKDbnDB+VRHDvMAzQhA2X7e8M0n9\n"     \
+    "rPolUY2lIVC83q0BBaOBkCj2RSmT2xTEbbC2xLukSrg2WP/ihVOxc\n"     \
+    "kXRuyFtzAgMBAAECgYB7slBexDwXrtItAMIH6m/U+LUpNe0Xx48OL\n"     \
+    "IOn4a4whNgO/o84uIwygUK27ZGFZT0kAGAk8CdF9hA6ArcbQ62s1H\n"     \
+    "myxrUbF9/mrLsQw1NEqpuUk9Ay2Tx5U/wPx35S3W/X2AvR/ZpTnCn\n"     \
+    "2q/7ym9fyiSoj86drD7BTvmKXlOnOwQJBAPOFMp4mMa9NGpGuEssO\n"     \
+    "m3Uwbp6lhcP0cA9MK+iOmeANpoKWfBdk5O34VbmeXnGYWEkrnX+9J\n"     \
+    "bM4wVhnnBWtgBMCQQC+qAEmvwcfhauERKYznMVUVksyeuhxhCe7EK\n"     \
+    "mPh+U2+g0WwdKvGDgO0PPt1gq0ILEjspMDeMHVdTwkaVBo/uMhAkA\n"     \
+    "Z5SsZyCP2aTOPFDypXRdI4eqRcjaEPOUBq27r3uYb/jeboVb2weLa\n"     \
+    "L1MmVuHiIHoa5clswPdWVI2y0em2IGoDAkBPSp/v9VKJEZabk9Frd\n"     \
+    "a+7u4fanrM9QrEjY3KhduslSilXZZSxrWjjAJPyPiqFb3M8XXA26W\n"     \
+    "nz1KYGnqYKhLcBAkB7dt57n9xfrhDpuyVEv+Uv1D3VVAhZlsaZ5Pp\n"     \
+    "dcrhrkJn2sa/+O8OKvdrPSeeu/N5WwYhJf61+CPoenMp7IFci\n"         \
+    "-----END PRIVATE KEY-----\n"
+
+/*
+ * This loads the private key we defined earlier
+ */
+static gnutls_x509_privkey_t test_tls_load_key(void)
+{
+    gnutls_x509_privkey_t key;
+    const gnutls_datum_t data = { (unsigned char *)PRIVATE_KEY,
+                                  strlen(PRIVATE_KEY) };
+    int err;
+
+    err = gnutls_x509_privkey_init(&key);
+    if (err < 0) {
+        g_critical("Failed to init key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_privkey_import(key, &data,
+                                     GNUTLS_X509_FMT_PEM);
+    if (err < 0) {
+        if (err != GNUTLS_E_BASE64_UNEXPECTED_HEADER_ERROR &&
+            err != GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            g_critical("Failed to import key %s", gnutls_strerror(err));
+            abort();
+        }
+
+        err = gnutls_x509_privkey_import_pkcs8(
+            key, &data, GNUTLS_X509_FMT_PEM, NULL, 0);
+        if (err < 0) {
+            g_critical("Failed to import PKCS8 key %s", gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    return key;
+}
+
+
+void test_tls_init(const char *keyfile)
+{
+    gnutls_global_init();
+
+    if (asn1_array2tree(pkix_asn1_tab, &pkix_asn1, NULL) != ASN1_SUCCESS) {
+        abort();
+    }
+
+    privkey = test_tls_load_key();
+    if (!g_file_set_contents(keyfile, PRIVATE_KEY, -1, NULL)) {
+        abort();
+    }
+}
+
+
+void test_tls_cleanup(const char *keyfile)
+{
+    asn1_delete_structure(&pkix_asn1);
+    unlink(keyfile);
+}
+
+/*
+ * Turns an ASN1 object into a DER encoded byte array
+ */
+static void test_tls_der_encode(ASN1_TYPE src,
+                                const char *src_name,
+                                gnutls_datum_t *res)
+{
+  int size;
+  char *data = NULL;
+
+  size = 0;
+  asn1_der_coding(src, src_name, NULL, &size, NULL);
+
+  data = g_new0(char, size);
+
+  asn1_der_coding(src, src_name, data, &size, NULL);
+
+  res->data = (unsigned char *)data;
+  res->size = size;
+}
+
+
+static void
+test_tls_get_ipaddr(const char *addrstr,
+                    char **data,
+                    int *datalen)
+{
+    struct addrinfo *res;
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_flags = AI_NUMERICHOST;
+    g_assert(getaddrinfo(addrstr, NULL, &hints, &res) == 0);
+
+    *datalen = res->ai_addrlen;
+    *data = g_new(char, *datalen);
+    memcpy(*data, res->ai_addr, *datalen);
+    freeaddrinfo(res);
+}
+
+/*
+ * This is a fairly lame x509 certificate generator.
+ *
+ * Do not copy/use this code for generating real certificates
+ * since it leaves out many things that you would want in
+ * certificates for real world usage.
+ *
+ * This is good enough only for doing tests of the QEMU
+ * TLS certificate code
+ */
+void
+test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                       gnutls_x509_crt_t ca)
+{
+    gnutls_x509_crt_t crt;
+    int err;
+    static char buffer[1024 * 1024];
+    size_t size = sizeof(buffer);
+    char serial[5] = { 1, 2, 3, 4, 0 };
+    gnutls_datum_t der;
+    time_t start = time(NULL) + (60 * 60 * req->start_offset);
+    time_t expire = time(NULL) + (60 * 60 * (req->expire_offset
+                                             ? req->expire_offset : 24));
+
+    /*
+     * Prepare our new certificate object
+     */
+    err = gnutls_x509_crt_init(&crt);
+    if (err < 0) {
+        g_critical("Failed to initialize certificate %s", gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_key(crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to set certificate key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * A v3 certificate is required in order to be able
+     * set any of the basic constraints, key purpose and
+     * key usage data
+     */
+    gnutls_x509_crt_set_version(crt, 3);
+
+    if (req->country) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COUNTRY_NAME, 0,
+            req->country, strlen(req->country));
+        if (err < 0) {
+            g_critical("Failed to set certificate country name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->cn) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COMMON_NAME, 0,
+            req->cn, strlen(req->cn));
+        if (err < 0) {
+            g_critical("Failed to set certificate common name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * Setup the subject altnames, which are used
+     * for hostname checks in live sessions
+     */
+    if (req->altname1) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname1,
+            strlen(req->altname1),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->altname2) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname2,
+            strlen(req->altname2),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate %s alt name",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * IP address need to be put into the cert in their
+     * raw byte form, not strings, hence this is a little
+     * more complicated
+     */
+    if (req->ipaddr1) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr1, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+    if (req->ipaddr2) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr2, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+
+
+    /*
+     * Basic constraints are used to decide if the cert
+     * is for a CA or not. We can't use the convenient
+     * gnutls API for setting this, since it hardcodes
+     * the 'critical' field which we want control over
+     */
+    if (req->basicConstraintsEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.BasicConstraints", &ext);
+        asn1_write_value(ext, "cA",
+                         req->basicConstraintsIsCA ? "TRUE" : "FALSE", 1);
+        asn1_write_value(ext, "pathLenConstraint", NULL, 0);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.19",
+            der.data, der.size,
+            req->basicConstraintsCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate basic constraints %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Next up the key usage extension. Again we can't
+     * use the gnutls API since it hardcodes the extension
+     * to be 'critical'
+     */
+    if (req->keyUsageEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+        char str[2];
+
+        str[0] = req->keyUsageValue & 0xff;
+        str[1] = (req->keyUsageValue >> 8) & 0xff;
+
+        asn1_create_element(pkix_asn1, "PKIX1.KeyUsage", &ext);
+        asn1_write_value(ext, "", str, 9);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.15",
+            der.data, der.size,
+            req->keyUsageCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key usage %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Finally the key purpose extension. This time
+     * gnutls has the opposite problem, always hardcoding
+     * it to be non-critical. So once again we have to
+     * set this the hard way building up ASN1 data ourselves
+     */
+    if (req->keyPurposeEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.ExtKeyUsageSyntax", &ext);
+        if (req->keyPurposeOID1) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID1, 1);
+        }
+        if (req->keyPurposeOID2) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID2, 1);
+        }
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.37",
+            der.data, der.size,
+            req->keyPurposeCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key purpose %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Any old serial number will do, so lets pick 5
+     */
+    err = gnutls_x509_crt_set_serial(crt, serial, 5);
+    if (err < 0) {
+        g_critical("Failed to set certificate serial %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_crt_set_activation_time(crt, start);
+    if (err < 0) {
+        g_critical("Failed to set certificate activation %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_expiration_time(crt, expire);
+    if (err < 0) {
+        g_critical("Failed to set certificate expiration %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+
+    /*
+     * If no 'ca' is set then we are self signing
+     * the cert. This is done for the root CA certs
+     */
+    err = gnutls_x509_crt_sign(crt, ca ? ca : crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to sign certificate %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * Finally write the new cert out to disk
+     */
+    err = gnutls_x509_crt_export(
+        crt, GNUTLS_X509_FMT_PEM, buffer, &size);
+    if (err < 0) {
+        g_critical("Failed to export certificate %s: %d",
+                   gnutls_strerror(err), err);
+        abort();
+    }
+
+    if (!g_file_set_contents(req->filename, buffer, -1, NULL)) {
+        g_critical("Failed to write certificate %s",
+                   req->filename);
+        abort();
+    }
+
+    req->crt = crt;
+}
+
+
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts)
+{
+    size_t i;
+    size_t capacity = 1024, offset = 0;
+    char *buffer = g_new0(char, capacity);
+    int err;
+
+    for (i = 0; i < ncerts; i++) {
+        size_t len = capacity - offset;
+    retry:
+        err = gnutls_x509_crt_export(certs[i], GNUTLS_X509_FMT_PEM,
+                                     buffer + offset, &len);
+        if (err < 0) {
+            if (err == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                buffer = g_renew(char, buffer, offset + len);
+                capacity = offset + len;
+                goto retry;
+            }
+            g_critical("Failed to export certificate chain %s: %d",
+                       gnutls_strerror(err), err);
+            abort();
+        }
+        offset += len;
+    }
+
+    if (!g_file_set_contents(filename, buffer, offset, NULL)) {
+        abort();
+    }
+    g_free(buffer);
+}
+
+
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req)
+{
+    if (!req->crt) {
+        return;
+    }
+
+    gnutls_x509_crt_deinit(req->crt);
+    req->crt = NULL;
+
+    if (getenv("QEMU_TEST_DEBUG_CERTS") == NULL) {
+        unlink(req->filename);
+    }
+}
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/qemu/tests/crypto-tls-x509-helpers.h b/qemu/tests/crypto-tls-x509-helpers.h
new file mode 100644
index 000000000..356b49cd5
--- /dev/null
+++ b/qemu/tests/crypto-tls-x509-helpers.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#if !(defined WIN32) && \
+    defined(CONFIG_TASN1) && \
+    defined(LIBGNUTLS_VERSION_NUMBER) && \
+    (LIBGNUTLS_VERSION_NUMBER >= 0x020600)
+# define QCRYPTO_HAVE_TLS_TEST_SUPPORT
+#endif
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+# include <libtasn1.h>
+
+# include "qemu-common.h"
+
+/*
+ * This contains parameter about how to generate
+ * certificates.
+ */
+typedef struct QCryptoTLSTestCertReq QCryptoTLSTestCertReq;
+struct QCryptoTLSTestCertReq {
+    gnutls_x509_crt_t crt;
+
+    const char *filename;
+
+    /* Identifying information */
+    const char *country;
+    const char *cn;
+    const char *altname1;
+    const char *altname2;
+    const char *ipaddr1;
+    const char *ipaddr2;
+
+    /* Basic constraints */
+    bool basicConstraintsEnable;
+    bool basicConstraintsCritical;
+    bool basicConstraintsIsCA;
+
+    /* Key usage */
+    bool keyUsageEnable;
+    bool keyUsageCritical;
+    int keyUsageValue;
+
+    /* Key purpose (aka Extended key usage) */
+    bool keyPurposeEnable;
+    bool keyPurposeCritical;
+    const char *keyPurposeOID1;
+    const char *keyPurposeOID2;
+
+    /* zero for current time, or non-zero for hours from now */
+    int start_offset;
+    /* zero for 24 hours from now, or non-zero for hours from now */
+    int expire_offset;
+};
+
+void test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                            gnutls_x509_crt_t ca);
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts);
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req);
+
+void test_tls_init(const char *keyfile);
+void test_tls_cleanup(const char *keyfile);
+
+# define TLS_CERT_REQ(varname, cavarname,                               \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, cavarname.crt)
+
+# define TLS_ROOT_REQ(varname,                                          \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, NULL)
+
+extern const ASN1_ARRAY_TYPE pkix_asn1_tab[];
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/qemu/tests/device-introspect-test.c b/qemu/tests/device-introspect-test.c
new file mode 100644
index 000000000..447792601
--- /dev/null
+++ b/qemu/tests/device-introspect-test.c
@@ -0,0 +1,124 @@
+/*
+ * Device introspection test cases
+ *
+ * Copyright (c) 2015 Red Hat Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+/*
+ * Covers QMP device-list-properties and HMP device_add help.  We
+ * currently don't check that their output makes sense, only that QEMU
+ * survives.  Useful since we've had an astounding number of crash
+ * bugs around here.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include "qemu-common.h"
+#include "qapi/qmp/qstring.h"
+#include "libqtest.h"
+
+const char common_args[] = "-nodefaults -machine none";
+
+static QList *device_type_list(bool abstract)
+{
+    QDict *resp;
+    QList *ret;
+
+    resp = qmp("{'execute': 'qom-list-types',"
+               " 'arguments': {'implements': 'device', 'abstract': %i}}",
+               abstract);
+    g_assert(qdict_haskey(resp, "return"));
+    ret = qdict_get_qlist(resp, "return");
+    QINCREF(ret);
+    QDECREF(resp);
+    return ret;
+}
+
+static void test_one_device(const char *type)
+{
+    QDict *resp;
+    char *help, *qom_tree;
+
+    resp = qmp("{'execute': 'device-list-properties',"
+               " 'arguments': {'typename': %s}}",
+               type);
+    QDECREF(resp);
+
+    help = hmp("device_add \"%s,help\"", type);
+    g_free(help);
+
+    /*
+     * Some devices leave dangling pointers in QOM behind.
+     * "info qom-tree" has a good chance at crashing then
+     */
+    qom_tree = hmp("info qom-tree");
+    g_free(qom_tree);
+}
+
+static void test_device_intro_list(void)
+{
+    QList *types;
+    char *help;
+
+    qtest_start(common_args);
+
+    types = device_type_list(true);
+    QDECREF(types);
+
+    help = hmp("device_add help");
+    g_free(help);
+
+    qtest_end();
+}
+
+static void test_device_intro_none(void)
+{
+    qtest_start(common_args);
+    test_one_device("nonexistent");
+    qtest_end();
+}
+
+static void test_device_intro_abstract(void)
+{
+    qtest_start(common_args);
+    test_one_device("device");
+    qtest_end();
+}
+
+static void test_device_intro_concrete(void)
+{
+    QList *types;
+    QListEntry *entry;
+    const char *type;
+
+    qtest_start(common_args);
+    types = device_type_list(false);
+
+    QLIST_FOREACH_ENTRY(types, entry) {
+        type = qdict_get_try_str(qobject_to_qdict(qlist_entry_obj(entry)),
+                                "name");
+        g_assert(type);
+        test_one_device(type);
+    }
+
+    QDECREF(types);
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("device/introspect/list", test_device_intro_list);
+    qtest_add_func("device/introspect/none", test_device_intro_none);
+    qtest_add_func("device/introspect/abstract", test_device_intro_abstract);
+    qtest_add_func("device/introspect/concrete", test_device_intro_concrete);
+
+    return g_test_run();
+}
diff --git a/qemu/tests/display-vga-test.c b/qemu/tests/display-vga-test.c
index 7694344ea..5706d338a 100644
--- a/qemu/tests/display-vga-test.c
+++ b/qemu/tests/display-vga-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 static void pci_cirrus(void)
 {
diff --git a/qemu/tests/drive_del-test.c b/qemu/tests/drive_del-test.c
index 8951f6f61..fe03236f3 100644
--- a/qemu/tests/drive_del-test.c
+++ b/qemu/tests/drive_del-test.c
@@ -10,34 +10,24 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
 
 static void drive_add(void)
 {
-    QDict *response;
+    char *resp = hmp("drive_add 0 if=none,id=drive0");
 
-    response = qmp("{'execute': 'human-monitor-command',"
-                   " 'arguments': {"
-                   "   'command-line': 'drive_add 0 if=none,id=drive0'"
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "OK\r\n");
-    QDECREF(response);
+    g_assert_cmpstr(resp, ==, "OK\r\n");
+    g_free(resp);
 }
 
 static void drive_del(void)
 {
-    QDict *response;
+    char *resp = hmp("drive_del drive0");
 
-    response = qmp("{'execute': 'human-monitor-command',"
-                   " 'arguments': {"
-                   "   'command-line': 'drive_del drive0'"
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "");
-    QDECREF(response);
+    g_assert_cmpstr(resp, ==, "");
+    g_free(resp);
 }
 
 static void device_del(void)
diff --git a/qemu/tests/ds1338-test.c b/qemu/tests/ds1338-test.c
new file mode 100644
index 000000000..279241584
--- /dev/null
+++ b/qemu/tests/ds1338-test.c
@@ -0,0 +1,79 @@
+/*
+ * QTest testcase for the DS1338 RTC
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "libqos/i2c.h"
+
+#include <glib.h>
+
+#define IMX25_I2C_0_BASE 0x43F80000
+
+#define DS1338_ADDR 0x68
+
+static I2CAdapter *i2c;
+static uint8_t addr;
+
+static inline uint8_t bcd2bin(uint8_t x)
+{
+    return ((x) & 0x0f) + ((x) >> 4) * 10;
+}
+
+static void send_and_receive(void)
+{
+    uint8_t cmd[1];
+    uint8_t resp[7];
+    time_t now = time(NULL);
+    struct tm *tm_ptr = gmtime(&now);
+
+    /* reset the index in the RTC memory */
+    cmd[0] = 0;
+    i2c_send(i2c, addr, cmd, 1);
+
+    /* retrieve the date */
+    i2c_recv(i2c, addr, resp, 7);
+
+    /* check retrieved time againt local time */
+    g_assert_cmpuint(bcd2bin(resp[4]), == , tm_ptr->tm_mday);
+    g_assert_cmpuint(bcd2bin(resp[5]), == , 1 + tm_ptr->tm_mon);
+    g_assert_cmpuint(2000 + bcd2bin(resp[6]), == , 1900 + tm_ptr->tm_year);
+}
+
+int main(int argc, char **argv)
+{
+    QTestState *s = NULL;
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    s = qtest_start("-display none -machine imx25-pdk");
+    i2c = imx_i2c_create(IMX25_I2C_0_BASE);
+    addr = DS1338_ADDR;
+
+    qtest_add_func("/ds1338/tx-rx", send_and_receive);
+
+    ret = g_test_run();
+
+    if (s) {
+        qtest_quit(s);
+    }
+    g_free(i2c);
+
+    return ret;
+}
diff --git a/qemu/tests/e1000-test.c b/qemu/tests/e1000-test.c
index 7ca6d7e72..a42b3810c 100644
--- a/qemu/tests/e1000-test.c
+++ b/qemu/tests/e1000-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void test_device(gconstpointer data)
diff --git a/qemu/tests/eepro100-test.c b/qemu/tests/eepro100-test.c
index 8bfaccdcb..e17eed0b7 100644
--- a/qemu/tests/eepro100-test.c
+++ b/qemu/tests/eepro100-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 static void test_device(gconstpointer data)
 {
diff --git a/qemu/tests/endianness-test.c b/qemu/tests/endianness-test.c
index 2054338e1..cc5bccd88 100644
--- a/qemu/tests/endianness-test.c
+++ b/qemu/tests/endianness-test.c
@@ -11,11 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
 
 #include "libqtest.h"
 #include "qemu/bswap.h"
diff --git a/qemu/tests/es1370-test.c b/qemu/tests/es1370-test.c
index cc23fb5c6..824dc31c6 100644
--- a/qemu/tests/es1370-test.c
+++ b/qemu/tests/es1370-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/fdc-test.c b/qemu/tests/fdc-test.c
index 416394fc7..53df1d0d8 100644
--- a/qemu/tests/fdc-test.c
+++ b/qemu/tests/fdc-test.c
@@ -22,9 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include <glib.h>
 
@@ -267,7 +265,7 @@ static void test_cmos(void)
     uint8_t cmos;
 
     cmos = cmos_read(CMOS_FLOPPY);
-    g_assert(cmos == 0x40);
+    g_assert(cmos == 0x40 || cmos == 0x50);
 }
 
 static void test_no_media_on_start(void)
@@ -304,9 +302,6 @@ static void test_media_insert(void)
     qmp_discard_response("{'execute':'change', 'arguments':{"
                          " 'device':'floppy0', 'target': %s, 'arg': 'raw' }}",
                          test_image);
-    qmp_discard_response(""); /* ignore event
-                                 (FIXME open -> open transition?!) */
-    qmp_discard_response(""); /* ignore event */
 
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_set(dir, DSKCHG);
@@ -337,7 +332,6 @@ static void test_media_change(void)
      * reset the bit. */
     qmp_discard_response("{'execute':'eject', 'arguments':{"
                          " 'device':'floppy0' }}");
-    qmp_discard_response(""); /* ignore event */
 
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_set(dir, DSKCHG);
diff --git a/qemu/tests/fw_cfg-test.c b/qemu/tests/fw_cfg-test.c
index 9be78e956..b4392c2d3 100644
--- a/qemu/tests/fw_cfg-test.c
+++ b/qemu/tests/fw_cfg-test.c
@@ -10,12 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "libqtest.h"
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
+#include "hw/nvram/fw_cfg_keys.h"
 #include "libqos/fw_cfg.h"
 
 static uint64_t ram_size = 128 << 20;
@@ -37,7 +36,9 @@ static void test_fw_cfg_signature(void)
 
 static void test_fw_cfg_id(void)
 {
-    g_assert_cmpint(qfw_cfg_get_u32(fw_cfg, FW_CFG_ID), ==, 1);
+    uint32_t id = qfw_cfg_get_u32(fw_cfg, FW_CFG_ID);
+    g_assert((id == 1) ||
+             (id == 3));
 }
 
 static void test_fw_cfg_uuid(void)
diff --git a/qemu/tests/guest-debug/test-gdbstub.py b/qemu/tests/guest-debug/test-gdbstub.py
new file mode 100644
index 000000000..31ba6c943
--- /dev/null
+++ b/qemu/tests/guest-debug/test-gdbstub.py
@@ -0,0 +1,176 @@
+#
+# This script needs to be run on startup
+# qemu -kernel ${KERNEL} -s -S
+# and then:
+# gdb ${KERNEL}.vmlinux -x ${QEMU_SRC}/tests/guest-debug/test-gdbstub.py
+
+import gdb
+
+failcount = 0
+
+
+def report(cond, msg):
+    "Report success/fail of test"
+    if cond:
+        print ("PASS: %s" % (msg))
+    else:
+        print ("FAIL: %s" % (msg))
+        failcount += 1
+
+
+def check_step():
+    "Step an instruction, check it moved."
+    start_pc = gdb.parse_and_eval('$pc')
+    gdb.execute("si")
+    end_pc = gdb.parse_and_eval('$pc')
+
+    return not (start_pc == end_pc)
+
+
+def check_break(sym_name):
+    "Setup breakpoint, continue and check we stopped."
+    sym, ok = gdb.lookup_symbol(sym_name)
+    bp = gdb.Breakpoint(sym_name)
+
+    gdb.execute("c")
+
+    # hopefully we came back
+    end_pc = gdb.parse_and_eval('$pc')
+    print ("%s == %s %d" % (end_pc, sym.value(), bp.hit_count))
+    bp.delete()
+
+    # can we test we hit bp?
+    return end_pc == sym.value()
+
+
+# We need to do hbreak manually as the python interface doesn't export it
+def check_hbreak(sym_name):
+    "Setup hardware breakpoint, continue and check we stopped."
+    sym, ok = gdb.lookup_symbol(sym_name)
+    gdb.execute("hbreak %s" % (sym_name))
+    gdb.execute("c")
+
+    # hopefully we came back
+    end_pc = gdb.parse_and_eval('$pc')
+    print ("%s == %s" % (end_pc, sym.value()))
+
+    if end_pc == sym.value():
+        gdb.execute("d 1")
+        return True
+    else:
+        return False
+
+
+class WatchPoint(gdb.Breakpoint):
+
+    def get_wpstr(self, sym_name):
+        "Setup sym and wp_str for given symbol."
+        self.sym, ok = gdb.lookup_symbol(sym_name)
+        wp_addr = gdb.parse_and_eval(sym_name).address
+        self.wp_str = '*(%(type)s)(&%(address)s)' % dict(
+            type = wp_addr.type, address = sym_name)
+
+        return(self.wp_str)
+
+    def __init__(self, sym_name, type):
+        wp_str = self.get_wpstr(sym_name)
+        super(WatchPoint, self).__init__(wp_str, gdb.BP_WATCHPOINT, type)
+
+    def stop(self):
+        end_pc = gdb.parse_and_eval('$pc')
+        print ("HIT WP @ %s" % (end_pc))
+        return True
+
+
+def do_one_watch(sym, wtype, text):
+
+    wp = WatchPoint(sym, wtype)
+    gdb.execute("c")
+    report_str = "%s for %s (%s)" % (text, sym, wp.sym.value())
+
+    if wp.hit_count > 0:
+        report(True, report_str)
+        wp.delete()
+    else:
+        report(False, report_str)
+
+
+def check_watches(sym_name):
+    "Watch a symbol for any access."
+
+    # Should hit for any read
+    do_one_watch(sym_name, gdb.WP_ACCESS, "awatch")
+
+    # Again should hit for reads
+    do_one_watch(sym_name, gdb.WP_READ, "rwatch")
+
+    # Finally when it is written
+    do_one_watch(sym_name, gdb.WP_WRITE, "watch")
+
+
+class CatchBreakpoint(gdb.Breakpoint):
+    def __init__(self, sym_name):
+        super(CatchBreakpoint, self).__init__(sym_name)
+        self.sym, ok = gdb.lookup_symbol(sym_name)
+
+    def stop(self):
+        end_pc = gdb.parse_and_eval('$pc')
+        print ("CB: %s == %s" % (end_pc, self.sym.value()))
+        if end_pc == self.sym.value():
+            report(False, "Hit final catchpoint")
+
+
+def run_test():
+    "Run throught the tests one by one"
+
+    print ("Checking we can step the first few instructions")
+    step_ok = 0
+    for i in range(3):
+        if check_step():
+            step_ok += 1
+
+    report(step_ok == 3, "single step in boot code")
+
+    print ("Checking HW breakpoint works")
+    break_ok = check_hbreak("kernel_init")
+    report(break_ok, "hbreak @ kernel_init")
+
+    # Can't set this up until we are in the kernel proper
+    # if we make it to run_init_process we've over-run and
+    # one of the tests failed
+    print ("Setup catch-all for run_init_process")
+    cbp = CatchBreakpoint("run_init_process")
+    cpb2 = CatchBreakpoint("try_to_run_init_process")
+
+    print ("Checking Normal breakpoint works")
+    break_ok = check_break("wait_for_completion")
+    report(break_ok, "break @ wait_for_completion")
+
+    print ("Checking watchpoint works")
+    check_watches("system_state")
+
+#
+# This runs as the script it sourced (via -x)
+#
+
+try:
+    print ("Connecting to remote")
+    gdb.execute("target remote localhost:1234")
+
+    # These are not very useful in scripts
+    gdb.execute("set pagination off")
+    gdb.execute("set confirm off")
+
+    # Run the actual tests
+    run_test()
+
+except:
+    print ("GDB Exception: %s" % (sys.exc_info()[0]))
+    failcount += 1
+    import code
+    code.InteractiveConsole(locals=globals()).interact()
+    raise
+
+# Finally kill the inferior and exit gdb with a count of failures
+gdb.execute("kill")
+exit(failcount)
diff --git a/qemu/tests/hd-geo-test.c b/qemu/tests/hd-geo-test.c
index 00afc209e..c8e669ac2 100644
--- a/qemu/tests/hd-geo-test.c
+++ b/qemu/tests/hd-geo-test.c
@@ -15,10 +15,8 @@
  * Improvements welcome.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 #include "qemu-common.h"
 #include "libqtest.h"
 
@@ -206,13 +204,13 @@ static int setup_ide(int argc, char *argv[], int argv_sz,
 {
     char *s1, *s2, *s3;
 
-    s1 = g_strdup_printf("-drive id=drive%d,if=%s,format=raw",
+    s1 = g_strdup_printf("-drive id=drive%d,if=%s",
                          ide_idx, dev ? "none" : "ide");
     s2 = dev ? g_strdup("") : g_strdup_printf(",index=%d", ide_idx);
 
     if (img_secs[img_idx] >= 0) {
         setup_mbr(img_idx, mbr);
-        s3 = g_strdup_printf(",file=%s", img_file_name[img_idx]);
+        s3 = g_strdup_printf(",format=raw,file=%s", img_file_name[img_idx]);
     } else {
         s3 = g_strdup(",media=cdrom");
     }
diff --git a/qemu/tests/i440fx-test.c b/qemu/tests/i440fx-test.c
index d0bc8de25..05029e90b 100644
--- a/qemu/tests/i440fx-test.c
+++ b/qemu/tests/i440fx-test.c
@@ -12,13 +12,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <errno.h>
 #include <sys/mman.h>
-#include <stdlib.h>
 
 #include "libqtest.h"
 #include "libqos/pci.h"
@@ -27,8 +23,6 @@
 
 #define BROKEN 1
 
-#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
-
 typedef struct TestData
 {
     int num_cpus;
@@ -191,7 +185,7 @@ static void write_area(uint32_t start, uint32_t end, uint8_t value)
     uint32_t size = end - start + 1;
     uint8_t *data;
 
-    data = g_malloc0(size);
+    data = g_malloc(size);
     memset(data, value, size);
     memwrite(start, data, size);
 
diff --git a/qemu/tests/i82801b11-test.c b/qemu/tests/i82801b11-test.c
index 78d9ce0e6..c3b5ebbca 100644
--- a/qemu/tests/i82801b11-test.c
+++ b/qemu/tests/i82801b11-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/ide-test.c b/qemu/tests/ide-test.c
index 4a07e3a9d..0d9ab4df9 100644
--- a/qemu/tests/ide-test.c
+++ b/qemu/tests/ide-test.c
@@ -22,9 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include <glib.h>
 
@@ -45,8 +43,15 @@
 #define IDE_BASE 0x1f0
 #define IDE_PRIMARY_IRQ 14
 
+#define ATAPI_BLOCK_SIZE 2048
+
+/* How many bytes to receive via ATAPI PIO at one time.
+ * Must be less than 0xFFFF. */
+#define BYTE_COUNT_LIMIT 5120
+
 enum {
     reg_data        = 0x0,
+    reg_feature     = 0x1,
     reg_nsectors    = 0x2,
     reg_lba_low     = 0x3,
     reg_lba_middle  = 0x4,
@@ -80,6 +85,7 @@ enum {
     CMD_WRITE_DMA   = 0xca,
     CMD_FLUSH_CACHE = 0xe7,
     CMD_IDENTIFY    = 0xec,
+    CMD_PACKET      = 0xa0,
 
     CMDF_ABORT      = 0x100,
     CMDF_NO_BM      = 0x200,
@@ -172,7 +178,8 @@ typedef struct PrdtEntry {
 #define assert_bit_clear(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
 
 static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
-                            PrdtEntry *prdt, int prdt_entries)
+                            PrdtEntry *prdt, int prdt_entries,
+                            void(*post_exec)(uint64_t sector, int nb_sectors))
 {
     QPCIDevice *dev;
     uint16_t bmdma_base;
@@ -189,6 +196,9 @@ static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
 
     switch (cmd) {
     case CMD_READ_DMA:
+    case CMD_PACKET:
+        /* Assuming we only test data reads w/ ATAPI, otherwise we need to know
+         * the SCSI command being sent in the packet, too. */
         from_dev = true;
         break;
     case CMD_WRITE_DMA:
@@ -217,14 +227,22 @@ static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
     outl(bmdma_base + bmreg_prdt, guest_prdt);
 
     /* ATA DMA command */
-    outb(IDE_BASE + reg_nsectors, nb_sectors);
-
-    outb(IDE_BASE + reg_lba_low,    sector & 0xff);
-    outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
-    outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    if (cmd == CMD_PACKET) {
+        /* Enables ATAPI DMA; otherwise PIO is attempted */
+        outb(IDE_BASE + reg_feature, 0x01);
+    } else {
+        outb(IDE_BASE + reg_nsectors, nb_sectors);
+        outb(IDE_BASE + reg_lba_low,    sector & 0xff);
+        outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
+        outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    }
 
     outb(IDE_BASE + reg_command, cmd);
 
+    if (post_exec) {
+        post_exec(sector, nb_sectors);
+    }
+
     /* Start DMA transfer */
     outb(bmdma_base + bmreg_cmd, BM_CMD_START | (from_dev ? BM_CMD_WRITE : 0));
 
@@ -278,7 +296,8 @@ static void test_bmdma_simple_rw(void)
     memset(buf, 0x55, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -286,14 +305,15 @@ static void test_bmdma_simple_rw(void)
     memset(buf, 0xaa, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Read and verify 0x55 pattern in sector 0 */
     memset(cmpbuf, 0x55, len);
 
-    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -303,7 +323,7 @@ static void test_bmdma_simple_rw(void)
     /* Read and verify 0xaa pattern in sector 1 */
     memset(cmpbuf, 0xaa, len);
 
-    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -328,13 +348,13 @@ static void test_bmdma_short_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -353,13 +373,13 @@ static void test_bmdma_one_sector_short_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 2,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 2,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -377,13 +397,13 @@ static void test_bmdma_long_prdt(void)
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_ACTIVE | BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -399,7 +419,7 @@ static void test_bmdma_no_busmaster(void)
     PrdtEntry prdt[4096] = { };
 
     status = send_dma_request(CMD_READ_DMA | CMDF_NO_BM, 0, 512,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
 
     /* Not entirely clear what the expected result is, but this is what we get
      * in practice. At least we want to be aware of any changes. */
@@ -488,9 +508,7 @@ static void test_flush(void)
         tmp_path);
 
     /* Delay the completion of the flush request until we explicitly do it */
-    qmp_discard_response("{'execute':'human-monitor-command', 'arguments': {"
-                         " 'command-line':"
-                         " 'qemu-io ide0-hd0 \"break flush_to_os A\"'} }");
+    g_free(hmp("qemu-io ide0-hd0 \"break flush_to_os A\""));
 
     /* FLUSH CACHE command on device 0*/
     outb(IDE_BASE + reg_device, 0);
@@ -502,9 +520,7 @@ static void test_flush(void)
     assert_bit_clear(data, DF | ERR | DRQ);
 
     /* Complete the command */
-    qmp_discard_response("{'execute':'human-monitor-command', 'arguments': {"
-                         " 'command-line':"
-                         " 'qemu-io ide0-hd0 \"resume A\"'} }");
+    g_free(hmp("qemu-io ide0-hd0 \"resume A\""));
 
     /* Check registers */
     data = inb(IDE_BASE + reg_device);
@@ -575,16 +591,215 @@ static void test_flush_nodev(void)
     ide_test_quit();
 }
 
-static void test_pci_retry_flush(const char *machine)
+static void test_pci_retry_flush(void)
 {
     test_retry_flush("pc");
 }
 
-static void test_isa_retry_flush(const char *machine)
+static void test_isa_retry_flush(void)
 {
     test_retry_flush("isapc");
 }
 
+typedef struct Read10CDB {
+    uint8_t opcode;
+    uint8_t flags;
+    uint32_t lba;
+    uint8_t reserved;
+    uint16_t nblocks;
+    uint8_t control;
+    uint16_t padding;
+} __attribute__((__packed__)) Read10CDB;
+
+static void send_scsi_cdb_read10(uint64_t lba, int nblocks)
+{
+    Read10CDB pkt = { .padding = 0 };
+    int i;
+
+    g_assert_cmpint(lba, <=, UINT32_MAX);
+    g_assert_cmpint(nblocks, <=, UINT16_MAX);
+    g_assert_cmpint(nblocks, >=, 0);
+
+    /* Construct SCSI CDB packet */
+    pkt.opcode = 0x28;
+    pkt.lba = cpu_to_be32(lba);
+    pkt.nblocks = cpu_to_be16(nblocks);
+
+    /* Send Packet */
+    for (i = 0; i < sizeof(Read10CDB)/2; i++) {
+        outw(IDE_BASE + reg_data, cpu_to_le16(((uint16_t *)&pkt)[i]));
+    }
+}
+
+static void nsleep(int64_t nsecs)
+{
+    const struct timespec val = { .tv_nsec = nsecs };
+    nanosleep(&val, NULL);
+    clock_set(nsecs);
+}
+
+static uint8_t ide_wait_clear(uint8_t flag)
+{
+    uint8_t data;
+    time_t st;
+
+    /* Wait with a 5 second timeout */
+    time(&st);
+    while (true) {
+        data = inb(IDE_BASE + reg_status);
+        if (!(data & flag)) {
+            return data;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+    g_assert_not_reached();
+}
+
+static void ide_wait_intr(int irq)
+{
+    time_t st;
+    bool intr;
+
+    time(&st);
+    while (true) {
+        intr = get_irq(irq);
+        if (intr) {
+            return;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+
+    g_assert_not_reached();
+}
+
+static void cdrom_pio_impl(int nblocks)
+{
+    FILE *fh;
+    int patt_blocks = MAX(16, nblocks);
+    size_t patt_len = ATAPI_BLOCK_SIZE * patt_blocks;
+    char *pattern = g_malloc(patt_len);
+    size_t rxsize = ATAPI_BLOCK_SIZE * nblocks;
+    uint16_t *rx = g_malloc0(rxsize);
+    int i, j;
+    uint8_t data;
+    uint16_t limit;
+
+    /* Prepopulate the CDROM with an interesting pattern */
+    generate_pattern(pattern, patt_len, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, patt_blocks, fh);
+    fclose(fh);
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    /* PACKET command on device 0 */
+    outb(IDE_BASE + reg_device, 0);
+    outb(IDE_BASE + reg_lba_middle, BYTE_COUNT_LIMIT & 0xFF);
+    outb(IDE_BASE + reg_lba_high, (BYTE_COUNT_LIMIT >> 8 & 0xFF));
+    outb(IDE_BASE + reg_command, CMD_PACKET);
+    /* HP0: Check_Status_A State */
+    nsleep(400);
+    data = ide_wait_clear(BSY);
+    /* HP1: Send_Packet State */
+    assert_bit_set(data, DRQ | DRDY);
+    assert_bit_clear(data, ERR | DF | BSY);
+
+    /* SCSI CDB (READ10) -- read n*2048 bytes from block 0 */
+    send_scsi_cdb_read10(0, nblocks);
+
+    /* Read data back: occurs in bursts of 'BYTE_COUNT_LIMIT' bytes.
+     * If BYTE_COUNT_LIMIT is odd, we transfer BYTE_COUNT_LIMIT - 1 bytes.
+     * We allow an odd limit only when the remaining transfer size is
+     * less than BYTE_COUNT_LIMIT. However, SCSI's read10 command can only
+     * request n blocks, so our request size is always even.
+     * For this reason, we assume there is never a hanging byte to fetch. */
+    g_assert(!(rxsize & 1));
+    limit = BYTE_COUNT_LIMIT & ~1;
+    for (i = 0; i < DIV_ROUND_UP(rxsize, limit); i++) {
+        size_t offset = i * (limit / 2);
+        size_t rem = (rxsize / 2) - offset;
+
+        /* HP3: INTRQ_Wait */
+        ide_wait_intr(IDE_PRIMARY_IRQ);
+
+        /* HP2: Check_Status_B (and clear IRQ) */
+        data = ide_wait_clear(BSY);
+        assert_bit_set(data, DRQ | DRDY);
+        assert_bit_clear(data, ERR | DF | BSY);
+
+        /* HP4: Transfer_Data */
+        for (j = 0; j < MIN((limit / 2), rem); j++) {
+            rx[offset + j] = le16_to_cpu(inw(IDE_BASE + reg_data));
+        }
+    }
+
+    /* Check for final completion IRQ */
+    ide_wait_intr(IDE_PRIMARY_IRQ);
+
+    /* Sanity check final state */
+    data = ide_wait_clear(DRQ);
+    assert_bit_set(data, DRDY);
+    assert_bit_clear(data, DRQ | ERR | DF | BSY);
+
+    g_assert_cmpint(memcmp(pattern, rx, rxsize), ==, 0);
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
+static void test_cdrom_pio(void)
+{
+    cdrom_pio_impl(1);
+}
+
+static void test_cdrom_pio_large(void)
+{
+    /* Test a few loops of the PIO DRQ mechanism. */
+    cdrom_pio_impl(BYTE_COUNT_LIMIT * 4 / ATAPI_BLOCK_SIZE);
+}
+
+
+static void test_cdrom_dma(void)
+{
+    static const size_t len = ATAPI_BLOCK_SIZE;
+    char *pattern = g_malloc(ATAPI_BLOCK_SIZE * 16);
+    char *rx = g_malloc0(len);
+    uintptr_t guest_buf;
+    PrdtEntry prdt[1];
+    FILE *fh;
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    guest_buf = guest_alloc(guest_malloc, len);
+    prdt[0].addr = cpu_to_le32(guest_buf);
+    prdt[0].size = cpu_to_le32(len | PRDT_EOT);
+
+    generate_pattern(pattern, ATAPI_BLOCK_SIZE * 16, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, 16, fh);
+    fclose(fh);
+
+    send_dma_request(CMD_PACKET, 0, 1, prdt, 1, send_scsi_cdb_read10);
+
+    /* Read back data from guest memory into local qtest memory */
+    memread(guest_buf, rx, len);
+    g_assert_cmpint(memcmp(pattern, rx, len), ==, 0);
+
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
 int main(int argc, char **argv)
 {
     const char *arch = qtest_get_arch();
@@ -628,6 +843,10 @@ int main(int argc, char **argv)
     qtest_add_func("/ide/flush/retry_pci", test_pci_retry_flush);
     qtest_add_func("/ide/flush/retry_isa", test_isa_retry_flush);
 
+    qtest_add_func("/ide/cdrom/pio", test_cdrom_pio);
+    qtest_add_func("/ide/cdrom/pio_large", test_cdrom_pio_large);
+    qtest_add_func("/ide/cdrom/dma", test_cdrom_dma);
+
     ret = g_test_run();
 
     /* Cleanup */
diff --git a/qemu/tests/image-fuzzer/runner.py b/qemu/tests/image-fuzzer/runner.py
index 0a8743ef4..96a1c11b2 100755
--- a/qemu/tests/image-fuzzer/runner.py
+++ b/qemu/tests/image-fuzzer/runner.py
@@ -157,7 +157,7 @@ class TestEnv(object):
 
         try:
             os.makedirs(self.current_dir)
-        except OSError, e:
+        except OSError as e:
             print >>sys.stderr, \
                 "Error: The working directory '%s' cannot be used. Reason: %s"\
                 % (self.work_dir, e[1])
@@ -244,7 +244,7 @@ class TestEnv(object):
             temp_log = StringIO.StringIO()
             try:
                 retcode = run_app(temp_log, current_cmd)
-            except OSError, e:
+            except OSError as e:
                 multilog("%sError: Start of '%s' failed. Reason: %s\n\n"
                          % (test_summary, os.path.basename(current_cmd[0]),
                             e[1]),
@@ -301,7 +301,7 @@ if __name__ == '__main__':
         JSON:
 
         '--command' accepts a JSON array of commands. Each command presents
-        an application under test with all its paramaters as a list of strings,
+        an application under test with all its parameters as a list of strings,
         e.g. ["qemu-io", "$test_img", "-c", "write $off $len"].
 
         Supported application aliases: 'qemu-img' and 'qemu-io'.
@@ -356,7 +356,7 @@ if __name__ == '__main__':
         opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:hs:kvd:',
                                        ['command=', 'help', 'seed=', 'config=',
                                         'keep_passed', 'verbose', 'duration='])
-    except getopt.error, e:
+    except getopt.error as e:
         print >>sys.stderr, \
             "Error: %s\n\nTry 'runner.py --help' for more information" % e
         sys.exit(1)
@@ -374,7 +374,7 @@ if __name__ == '__main__':
         elif opt in ('-c', '--command'):
             try:
                 command = json.loads(arg)
-            except (TypeError, ValueError, NameError), e:
+            except (TypeError, ValueError, NameError) as e:
                 print >>sys.stderr, \
                     "Error: JSON array of test commands cannot be loaded.\n" \
                     "Reason: %s" % e
@@ -390,7 +390,7 @@ if __name__ == '__main__':
         elif opt == '--config':
             try:
                 config = json.loads(arg)
-            except (TypeError, ValueError, NameError), e:
+            except (TypeError, ValueError, NameError) as e:
                 print >>sys.stderr, \
                     "Error: JSON array with the fuzzer configuration cannot" \
                     " be loaded\nReason: %s" % e
@@ -414,7 +414,7 @@ if __name__ == '__main__':
 
     try:
         image_generator = __import__(generator_name)
-    except ImportError, e:
+    except ImportError as e:
         print >>sys.stderr, \
             "Error: The image generator '%s' cannot be imported.\n" \
             "Reason: %s" % (generator_name, e)
diff --git a/qemu/tests/intel-hda-test.c b/qemu/tests/intel-hda-test.c
index d89b407dc..1be6add9b 100644
--- a/qemu/tests/intel-hda-test.c
+++ b/qemu/tests/intel-hda-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 #define HDA_ID "hda0"
 #define CODEC_DEVICES " -device hda-output,bus=" HDA_ID ".0" \
diff --git a/qemu/tests/io-channel-helpers.c b/qemu/tests/io-channel-helpers.c
new file mode 100644
index 000000000..05e5579cf
--- /dev/null
+++ b/qemu/tests/io-channel-helpers.c
@@ -0,0 +1,248 @@
+/*
+ * QEMU I/O channel test helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+struct QIOChannelTest {
+    QIOChannel *src;
+    QIOChannel *dst;
+    bool blocking;
+    size_t len;
+    size_t niov;
+    char *input;
+    struct iovec *inputv;
+    char *output;
+    struct iovec *outputv;
+    Error *writeerr;
+    Error *readerr;
+};
+
+
+static void test_skip_iovec(struct iovec **iov,
+                            size_t *niov,
+                            size_t skip,
+                            struct iovec *old)
+{
+    size_t offset = 0;
+    size_t i;
+
+    for (i = 0; i < *niov; i++) {
+        if (skip < (*iov)[i].iov_len) {
+            old->iov_len = (*iov)[i].iov_len;
+            old->iov_base = (*iov)[i].iov_base;
+
+            (*iov)[i].iov_len -= skip;
+            (*iov)[i].iov_base += skip;
+            break;
+        } else {
+            skip -= (*iov)[i].iov_len;
+
+            if (i == 0 && old->iov_base) {
+                (*iov)[i].iov_len = old->iov_len;
+                (*iov)[i].iov_base = old->iov_base;
+                old->iov_len = 0;
+                old->iov_base = NULL;
+            }
+
+            offset++;
+        }
+    }
+
+    *iov = *iov + offset;
+    *niov -= offset;
+}
+
+
+/* This thread sends all data using iovecs */
+static gpointer test_io_thread_writer(gpointer opaque)
+{
+    QIOChannelTest *data = opaque;
+    struct iovec *iov = data->inputv;
+    size_t niov = data->niov;
+    struct iovec old = { 0 };
+
+    qio_channel_set_blocking(data->src, data->blocking, NULL);
+
+    while (niov) {
+        ssize_t ret;
+        ret = qio_channel_writev(data->src,
+                                 iov,
+                                 niov,
+                                 &data->writeerr);
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (data->blocking) {
+                error_setg(&data->writeerr,
+                           "Unexpected I/O blocking");
+                break;
+            } else {
+                qio_channel_wait(data->src,
+                                 G_IO_OUT);
+                continue;
+            }
+        } else if (ret < 0) {
+            break;
+        } else if (ret == 0) {
+            error_setg(&data->writeerr,
+                       "Unexpected zero length write");
+            break;
+        }
+
+        test_skip_iovec(&iov, &niov, ret, &old);
+    }
+
+    return NULL;
+}
+
+
+/* This thread receives all data using iovecs */
+static gpointer test_io_thread_reader(gpointer opaque)
+{
+    QIOChannelTest *data = opaque;
+    struct iovec *iov = data->outputv;
+    size_t niov = data->niov;
+    struct iovec old = { 0 };
+
+    qio_channel_set_blocking(data->dst, data->blocking, NULL);
+
+    while (niov) {
+        ssize_t ret;
+
+        ret = qio_channel_readv(data->dst,
+                                iov,
+                                niov,
+                                &data->readerr);
+
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (data->blocking) {
+                error_setg(&data->readerr,
+                           "Unexpected I/O blocking");
+                break;
+            } else {
+                qio_channel_wait(data->dst,
+                                 G_IO_IN);
+                continue;
+            }
+        } else if (ret < 0) {
+            break;
+        } else if (ret == 0) {
+            break;
+        }
+
+        test_skip_iovec(&iov, &niov, ret, &old);
+    }
+
+    return NULL;
+}
+
+
+QIOChannelTest *qio_channel_test_new(void)
+{
+    QIOChannelTest *data = g_new0(QIOChannelTest, 1);
+    size_t i;
+    size_t offset;
+
+
+    /* We'll send 1 MB of data */
+#define CHUNK_COUNT 250
+#define CHUNK_LEN 4194
+
+    data->len = CHUNK_COUNT * CHUNK_LEN;
+    data->input = g_new0(char, data->len);
+    data->output = g_new0(gchar, data->len);
+
+    /* Fill input with a pattern */
+    for (i = 0; i < data->len; i += CHUNK_LEN) {
+        memset(data->input + i, (i / CHUNK_LEN), CHUNK_LEN);
+    }
+
+    /* We'll split the data across a bunch of IO vecs */
+    data->niov = CHUNK_COUNT;
+    data->inputv = g_new0(struct iovec, data->niov);
+    data->outputv = g_new0(struct iovec, data->niov);
+
+    for (i = 0, offset = 0; i < data->niov; i++, offset += CHUNK_LEN) {
+        data->inputv[i].iov_base = data->input + offset;
+        data->outputv[i].iov_base = data->output + offset;
+        data->inputv[i].iov_len = CHUNK_LEN;
+        data->outputv[i].iov_len = CHUNK_LEN;
+    }
+
+    return data;
+}
+
+void qio_channel_test_run_threads(QIOChannelTest *test,
+                                  bool blocking,
+                                  QIOChannel *src,
+                                  QIOChannel *dst)
+{
+    GThread *reader, *writer;
+
+    test->src = src;
+    test->dst = dst;
+    test->blocking = blocking;
+
+    reader = g_thread_new("reader",
+                          test_io_thread_reader,
+                          test);
+    writer = g_thread_new("writer",
+                          test_io_thread_writer,
+                          test);
+
+    g_thread_join(reader);
+    g_thread_join(writer);
+
+    test->dst = test->src = NULL;
+}
+
+
+void qio_channel_test_run_writer(QIOChannelTest *test,
+                                 QIOChannel *src)
+{
+    test->src = src;
+    test_io_thread_writer(test);
+    test->src = NULL;
+}
+
+
+void qio_channel_test_run_reader(QIOChannelTest *test,
+                                 QIOChannel *dst)
+{
+    test->dst = dst;
+    test_io_thread_reader(test);
+    test->dst = NULL;
+}
+
+
+void qio_channel_test_validate(QIOChannelTest *test)
+{
+    g_assert(test->readerr == NULL);
+    g_assert(test->writeerr == NULL);
+    g_assert_cmpint(memcmp(test->input,
+                           test->output,
+                           test->len), ==, 0);
+
+    g_free(test->inputv);
+    g_free(test->outputv);
+    g_free(test->input);
+    g_free(test->output);
+    g_free(test);
+}
diff --git a/qemu/tests/io-channel-helpers.h b/qemu/tests/io-channel-helpers.h
new file mode 100644
index 000000000..fedc64fd5
--- /dev/null
+++ b/qemu/tests/io-channel-helpers.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU I/O channel test helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "io/channel.h"
+
+#ifndef TEST_IO_CHANNEL_HELPERS
+#define TEST_IO_CHANNEL_HELPERS
+
+typedef struct QIOChannelTest QIOChannelTest;
+
+QIOChannelTest *qio_channel_test_new(void);
+
+void qio_channel_test_run_threads(QIOChannelTest *test,
+                                  bool blocking,
+                                  QIOChannel *src,
+                                  QIOChannel *dst);
+
+void qio_channel_test_run_writer(QIOChannelTest *test,
+                                 QIOChannel *src);
+void qio_channel_test_run_reader(QIOChannelTest *test,
+                                 QIOChannel *dst);
+
+void qio_channel_test_validate(QIOChannelTest *test);
+
+#endif /* TEST_IO_CHANNEL_HELPERS */
diff --git a/qemu/tests/ioh3420-test.c b/qemu/tests/ioh3420-test.c
index c991a5f87..93eb2f750 100644
--- a/qemu/tests/ioh3420-test.c
+++ b/qemu/tests/ioh3420-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/ipmi-bt-test.c b/qemu/tests/ipmi-bt-test.c
new file mode 100644
index 000000000..812907fb7
--- /dev/null
+++ b/qemu/tests/ipmi-bt-test.c
@@ -0,0 +1,433 @@
+/*
+ * IPMI BT test cases, using the external interface for checking
+ *
+ * Copyright (c) 2012 Corey Minyard <cminyard@mvista.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+#include <glib.h>
+
+#include "libqtest.h"
+#include "qemu-common.h"
+
+#define IPMI_IRQ        5
+
+#define IPMI_BT_BASE    0xe4
+
+#define IPMI_BT_CTLREG_CLR_WR_PTR  0
+#define IPMI_BT_CTLREG_CLR_RD_PTR  1
+#define IPMI_BT_CTLREG_H2B_ATN     2
+#define IPMI_BT_CTLREG_B2H_ATN     3
+#define IPMI_BT_CTLREG_SMS_ATN     4
+#define IPMI_BT_CTLREG_H_BUSY      6
+#define IPMI_BT_CTLREG_B_BUSY      7
+
+#define IPMI_BT_CTLREG_GET(b) ((bt_get_ctrlreg() >> (b)) & 1)
+#define IPMI_BT_CTLREG_GET_H2B_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_H2B_ATN)
+#define IPMI_BT_CTLREG_GET_B2H_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_B2H_ATN)
+#define IPMI_BT_CTLREG_GET_SMS_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_SMS_ATN)
+#define IPMI_BT_CTLREG_GET_H_BUSY()  IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_H_BUSY)
+#define IPMI_BT_CTLREG_GET_B_BUSY()  IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_B_BUSY)
+
+#define IPMI_BT_CTLREG_SET(b) bt_write_ctrlreg(1 << (b))
+#define IPMI_BT_CTLREG_SET_CLR_WR_PTR() IPMI_BT_CTLREG_SET( \
+                                                IPMI_BT_CTLREG_CLR_WR_PTR)
+#define IPMI_BT_CTLREG_SET_CLR_RD_PTR() IPMI_BT_CTLREG_SET( \
+                                                IPMI_BT_CTLREG_CLR_RD_PTR)
+#define IPMI_BT_CTLREG_SET_H2B_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_H2B_ATN)
+#define IPMI_BT_CTLREG_SET_B2H_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_B2H_ATN)
+#define IPMI_BT_CTLREG_SET_SMS_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_SMS_ATN)
+#define IPMI_BT_CTLREG_SET_H_BUSY()   IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_H_BUSY)
+
+static int bt_ints_enabled;
+
+static uint8_t bt_get_ctrlreg(void)
+{
+    return inb(IPMI_BT_BASE);
+}
+
+static void bt_write_ctrlreg(uint8_t val)
+{
+    outb(IPMI_BT_BASE, val);
+}
+
+static uint8_t bt_get_buf(void)
+{
+    return inb(IPMI_BT_BASE + 1);
+}
+
+static void bt_write_buf(uint8_t val)
+{
+    outb(IPMI_BT_BASE + 1, val);
+}
+
+static uint8_t bt_get_irqreg(void)
+{
+    return inb(IPMI_BT_BASE + 2);
+}
+
+static void bt_write_irqreg(uint8_t val)
+{
+    outb(IPMI_BT_BASE + 2, val);
+}
+
+static void bt_wait_b_busy(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_BT_CTLREG_GET_B_BUSY() != 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void bt_wait_b2h_atn(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_BT_CTLREG_GET_B2H_ATN() == 0) {
+        g_assert(--count != 0);
+    }
+}
+
+
+static int emu_lfd;
+static int emu_fd;
+static in_port_t emu_port;
+static uint8_t inbuf[100];
+static unsigned int inbuf_len;
+static unsigned int inbuf_pos;
+static int last_was_aa;
+
+static void read_emu_data(void)
+{
+    fd_set readfds;
+    int rv;
+    struct timeval tv;
+
+    FD_ZERO(&readfds);
+    FD_SET(emu_fd, &readfds);
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    rv = select(emu_fd + 1, &readfds, NULL, NULL, &tv);
+    if (rv == -1) {
+        perror("select");
+    }
+    g_assert(rv == 1);
+    rv = read(emu_fd, inbuf, sizeof(inbuf));
+    if (rv == -1) {
+        perror("read");
+    }
+    g_assert(rv > 0);
+    inbuf_len = rv;
+    inbuf_pos = 0;
+}
+
+static void write_emu_msg(uint8_t *msg, unsigned int len)
+{
+    int rv;
+
+#ifdef DEBUG_TEST
+    {
+        unsigned int i;
+        printf("sending:");
+        for (i = 0; i < len; i++) {
+            printf(" %2.2x", msg[i]);
+        }
+        printf("\n");
+    }
+#endif
+    rv = write(emu_fd, msg, len);
+    g_assert(rv == len);
+}
+
+static void get_emu_msg(uint8_t *msg, unsigned int *len)
+{
+    unsigned int outpos = 0;
+
+    for (;;) {
+        while (inbuf_pos < inbuf_len) {
+            uint8_t ch = inbuf[inbuf_pos++];
+
+            g_assert(outpos < *len);
+            if (last_was_aa) {
+                assert(ch & 0x10);
+                msg[outpos++] = ch & ~0x10;
+                last_was_aa = 0;
+            } else if (ch == 0xaa) {
+                last_was_aa = 1;
+            } else {
+                msg[outpos++] = ch;
+                if ((ch == 0xa0) || (ch == 0xa1)) {
+                    /* Message complete */
+                    *len = outpos;
+                    goto done;
+                }
+            }
+        }
+        read_emu_data();
+    }
+ done:
+#ifdef DEBUG_TEST
+    {
+        unsigned int i;
+        printf("Msg:");
+        for (i = 0; i < outpos; i++) {
+            printf(" %2.2x", msg[i]);
+        }
+        printf("\n");
+    }
+#endif
+    return;
+}
+
+static uint8_t
+ipmb_checksum(const unsigned char *data, int size, unsigned char start)
+{
+        unsigned char csum = start;
+
+        for (; size > 0; size--, data++) {
+                csum += *data;
+        }
+        return csum;
+}
+
+static uint8_t get_dev_id_cmd[] = { 0x18, 0x01 };
+static uint8_t get_dev_id_rsp[] = { 0x1c, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
+                                    0x02, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+static uint8_t set_bmc_globals_cmd[] = { 0x18, 0x2e, 0x0f };
+static uint8_t set_bmc_globals_rsp[] = { 0x1c, 0x2e, 0x00 };
+static uint8_t enable_irq_cmd[] = { 0x05, 0xa1 };
+
+static void emu_msg_handler(void)
+{
+    uint8_t msg[100];
+    unsigned int msg_len = sizeof(msg);
+
+    get_emu_msg(msg, &msg_len);
+    g_assert(msg_len >= 5);
+    g_assert(msg[msg_len - 1] == 0xa0);
+    msg_len--;
+    g_assert(ipmb_checksum(msg, msg_len, 0) == 0);
+    msg_len--;
+    if ((msg[1] == get_dev_id_cmd[0]) && (msg[2] == get_dev_id_cmd[1])) {
+        memcpy(msg + 1, get_dev_id_rsp, sizeof(get_dev_id_rsp));
+        msg_len = sizeof(get_dev_id_rsp) + 1;
+        msg[msg_len] = -ipmb_checksum(msg, msg_len, 0);
+        msg_len++;
+        msg[msg_len++] = 0xa0;
+        write_emu_msg(msg, msg_len);
+    } else if ((msg[1] == set_bmc_globals_cmd[0]) &&
+               (msg[2] == set_bmc_globals_cmd[1])) {
+        memcpy(msg + 1, set_bmc_globals_rsp, sizeof(set_bmc_globals_rsp));
+        msg_len = sizeof(set_bmc_globals_rsp) + 1;
+        msg[msg_len] = -ipmb_checksum(msg, msg_len, 0);
+        msg_len++;
+        msg[msg_len++] = 0xa0;
+        write_emu_msg(msg, msg_len);
+        write_emu_msg(enable_irq_cmd, sizeof(enable_irq_cmd));
+    } else {
+        g_assert(0);
+    }
+}
+
+static void bt_cmd(uint8_t *cmd, unsigned int cmd_len,
+                    uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, len, j = 0;
+    uint8_t seq = 5;
+
+    /* Should be idle */
+    g_assert(bt_get_ctrlreg() == 0);
+
+    bt_wait_b_busy();
+    IPMI_BT_CTLREG_SET_CLR_WR_PTR();
+    bt_write_buf(cmd_len + 1);
+    bt_write_buf(cmd[0]);
+    bt_write_buf(seq);
+    for (i = 1; i < cmd_len; i++) {
+        bt_write_buf(cmd[i]);
+    }
+    IPMI_BT_CTLREG_SET_H2B_ATN();
+
+    emu_msg_handler(); /* We should get a message on the socket here. */
+
+    bt_wait_b2h_atn();
+    if (bt_ints_enabled) {
+        g_assert((bt_get_irqreg() & 0x02) == 0x02);
+        g_assert(get_irq(IPMI_IRQ));
+        bt_write_irqreg(0x03);
+    } else {
+        g_assert(!get_irq(IPMI_IRQ));
+    }
+    IPMI_BT_CTLREG_SET_H_BUSY();
+    IPMI_BT_CTLREG_SET_B2H_ATN();
+    IPMI_BT_CTLREG_SET_CLR_RD_PTR();
+    len = bt_get_buf();
+    g_assert(len >= 4);
+    rsp[0] = bt_get_buf();
+    assert(bt_get_buf() == seq);
+    len--;
+    for (j = 1; j < len; j++) {
+        rsp[j] = bt_get_buf();
+    }
+    IPMI_BT_CTLREG_SET_H_BUSY();
+    *rsp_len = j;
+}
+
+
+/*
+ * We should get a connect request and a short message with capabilities.
+ */
+static void test_connect(void)
+{
+    fd_set readfds;
+    int rv;
+    int val;
+    struct timeval tv;
+    uint8_t msg[100];
+    unsigned int msglen;
+    static uint8_t exp1[] = { 0xff, 0x01, 0xa1 }; /* A protocol version */
+    static uint8_t exp2[] = { 0x08, 0x1f, 0xa1 }; /* A capabilities cmd */
+
+    FD_ZERO(&readfds);
+    FD_SET(emu_lfd, &readfds);
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    rv = select(emu_lfd + 1, &readfds, NULL, NULL, &tv);
+    g_assert(rv == 1);
+    emu_fd = accept(emu_lfd, NULL, 0);
+    if (emu_fd < 0) {
+        perror("accept");
+    }
+    g_assert(emu_fd >= 0);
+
+    val = 1;
+    rv = setsockopt(emu_fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val));
+    g_assert(rv != -1);
+
+    /* Report our version */
+    write_emu_msg(exp1, sizeof(exp1));
+
+    /* Validate that we get the info we expect. */
+    msglen = sizeof(msg);
+    get_emu_msg(msg, &msglen);
+    g_assert(msglen == sizeof(exp1));
+    g_assert(memcmp(msg, exp1, msglen) == 0);
+    msglen = sizeof(msg);
+    get_emu_msg(msg, &msglen);
+    g_assert(msglen == sizeof(exp2));
+    g_assert(memcmp(msg, exp2, msglen) == 0);
+}
+
+/*
+ * Send a get_device_id to do a basic test.
+ */
+static void test_bt_base(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    bt_cmd(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(get_dev_id_rsp));
+    g_assert(memcmp(get_dev_id_rsp, rsp, rsplen) == 0);
+}
+
+/*
+ * Enable IRQs for the interface.
+ */
+static void test_enable_irq(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    bt_cmd(set_bmc_globals_cmd, sizeof(set_bmc_globals_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(set_bmc_globals_rsp));
+    g_assert(memcmp(set_bmc_globals_rsp, rsp, rsplen) == 0);
+    bt_write_irqreg(0x01);
+    bt_ints_enabled = 1;
+}
+
+/*
+ * Create a local TCP socket with any port, then save off the port we got.
+ */
+static void open_socket(void)
+{
+    struct sockaddr_in myaddr;
+    socklen_t addrlen;
+
+    myaddr.sin_family = AF_INET;
+    myaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+    myaddr.sin_port = 0;
+    emu_lfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+    if (emu_lfd == -1) {
+        perror("socket");
+        exit(1);
+    }
+    if (bind(emu_lfd, (struct sockaddr *) &myaddr, sizeof(myaddr)) == -1) {
+        perror("bind");
+        exit(1);
+    }
+    addrlen = sizeof(myaddr);
+    if (getsockname(emu_lfd, (struct sockaddr *) &myaddr , &addrlen) == -1) {
+        perror("getsockname");
+        exit(1);
+    }
+    emu_port = ntohs(myaddr.sin_port);
+    assert(listen(emu_lfd, 1) != -1);
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+    char *cmdline;
+    int ret;
+
+    /* Check architecture */
+    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
+        g_test_message("Skipping test for non-x86\n");
+        return 0;
+    }
+
+    open_socket();
+
+    /* Run the tests */
+    g_test_init(&argc, &argv, NULL);
+
+    cmdline = g_strdup_printf("-vnc none"
+          " -chardev socket,id=ipmi0,host=localhost,port=%d,reconnect=10"
+          " -device ipmi-bmc-extern,chardev=ipmi0,id=bmc0"
+          " -device isa-ipmi-bt,bmc=bmc0", emu_port);
+    qtest_start(cmdline);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    qtest_add_func("/ipmi/extern/connect", test_connect);
+    qtest_add_func("/ipmi/extern/bt_base", test_bt_base);
+    qtest_add_func("/ipmi/extern/bt_enable_irq", test_enable_irq);
+    qtest_add_func("/ipmi/extern/bt_base_irq", test_bt_base);
+    ret = g_test_run();
+    qtest_quit(global_qtest);
+
+    return ret;
+}
diff --git a/qemu/tests/ipmi-kcs-test.c b/qemu/tests/ipmi-kcs-test.c
new file mode 100644
index 000000000..42c4b974c
--- /dev/null
+++ b/qemu/tests/ipmi-kcs-test.c
@@ -0,0 +1,293 @@
+/*
+ * IPMI KCS test cases, using the local interface.
+ *
+ * Copyright (c) 2012 Corey Minyard <cminyard@mvista.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <glib.h>
+
+#include "libqtest.h"
+
+#define IPMI_IRQ        5
+
+#define IPMI_KCS_BASE   0xca2
+
+#define IPMI_KCS_STATUS_ABORT           0x60
+#define IPMI_KCS_CMD_WRITE_START        0x61
+#define IPMI_KCS_CMD_WRITE_END          0x62
+#define IPMI_KCS_CMD_READ               0x68
+
+#define IPMI_KCS_ABORTED_BY_CMD         0x01
+
+#define IPMI_KCS_CMDREG_GET_STATE() ((kcs_get_cmdreg() >> 6) & 3)
+#define IPMI_KCS_STATE_IDLE     0
+#define IPMI_KCS_STATE_READ     1
+#define IPMI_KCS_STATE_WRITE    2
+#define IPMI_KCS_STATE_ERROR    3
+#define IPMI_KCS_CMDREG_GET_CD()    ((kcs_get_cmdreg() >> 3) & 1)
+#define IPMI_KCS_CMDREG_GET_ATN()   ((kcs_get_cmdreg() >> 2) & 1)
+#define IPMI_KCS_CMDREG_GET_IBF()   ((kcs_get_cmdreg() >> 1) & 1)
+#define IPMI_KCS_CMDREG_GET_OBF()   ((kcs_get_cmdreg() >> 0) & 1)
+
+static int kcs_ints_enabled;
+
+static uint8_t kcs_get_cmdreg(void)
+{
+    return inb(IPMI_KCS_BASE + 1);
+}
+
+static void kcs_write_cmdreg(uint8_t val)
+{
+    outb(IPMI_KCS_BASE + 1, val);
+}
+
+static uint8_t kcs_get_datareg(void)
+{
+    return inb(IPMI_KCS_BASE);
+}
+
+static void kcs_write_datareg(uint8_t val)
+{
+    outb(IPMI_KCS_BASE, val);
+}
+
+static void kcs_wait_ibf(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_KCS_CMDREG_GET_IBF() != 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void kcs_wait_obf(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_KCS_CMDREG_GET_OBF() == 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void kcs_clear_obf(void)
+{
+    if (kcs_ints_enabled) {
+        g_assert(get_irq(IPMI_IRQ));
+    } else {
+        g_assert(!get_irq(IPMI_IRQ));
+    }
+    g_assert(IPMI_KCS_CMDREG_GET_OBF() == 1);
+    kcs_get_datareg();
+    g_assert(IPMI_KCS_CMDREG_GET_OBF() == 0);
+    g_assert(!get_irq(IPMI_IRQ));
+}
+
+static void kcs_check_state(uint8_t state)
+{
+    g_assert(IPMI_KCS_CMDREG_GET_STATE() == state);
+}
+
+static void kcs_cmd(uint8_t *cmd, unsigned int cmd_len,
+                    uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, j = 0;
+
+    /* Should be idle */
+    g_assert(kcs_get_cmdreg() == 0);
+
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_START);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    for (i = 0; i < cmd_len; i++) {
+        kcs_write_datareg(cmd[i]);
+        kcs_wait_ibf();
+        kcs_check_state(IPMI_KCS_STATE_WRITE);
+        kcs_clear_obf();
+    }
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_END);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+ next_read_byte:
+    kcs_wait_ibf();
+    switch (IPMI_KCS_CMDREG_GET_STATE()) {
+    case IPMI_KCS_STATE_READ:
+        kcs_wait_obf();
+        g_assert(j < *rsp_len);
+        rsp[j++] = kcs_get_datareg();
+        kcs_write_datareg(IPMI_KCS_CMD_READ);
+        goto next_read_byte;
+        break;
+
+    case IPMI_KCS_STATE_IDLE:
+        kcs_wait_obf();
+        kcs_get_datareg();
+        break;
+
+    default:
+        g_assert(0);
+    }
+    *rsp_len = j;
+}
+
+static void kcs_abort(uint8_t *cmd, unsigned int cmd_len,
+                      uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, j = 0;
+    unsigned int retries = 4;
+
+    /* Should be idle */
+    g_assert(kcs_get_cmdreg() == 0);
+
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_START);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    for (i = 0; i < cmd_len; i++) {
+        kcs_write_datareg(cmd[i]);
+        kcs_wait_ibf();
+        kcs_check_state(IPMI_KCS_STATE_WRITE);
+        kcs_clear_obf();
+    }
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_END);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+    kcs_wait_ibf();
+    switch (IPMI_KCS_CMDREG_GET_STATE()) {
+    case IPMI_KCS_STATE_READ:
+        kcs_wait_obf();
+        g_assert(j < *rsp_len);
+        rsp[j++] = kcs_get_datareg();
+        kcs_write_datareg(IPMI_KCS_CMD_READ);
+        break;
+
+    default:
+        g_assert(0);
+    }
+
+    /* Start the abort here */
+ retry_abort:
+    g_assert(retries > 0);
+
+    kcs_wait_ibf();
+    kcs_write_cmdreg(IPMI_KCS_STATUS_ABORT);
+    kcs_wait_ibf();
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+    kcs_wait_ibf();
+    if (IPMI_KCS_CMDREG_GET_STATE() != IPMI_KCS_STATE_READ) {
+        retries--;
+        goto retry_abort;
+    }
+    kcs_wait_obf();
+    rsp[0] = kcs_get_datareg();
+    kcs_write_datareg(IPMI_KCS_CMD_READ);
+    kcs_wait_ibf();
+    if (IPMI_KCS_CMDREG_GET_STATE() != IPMI_KCS_STATE_IDLE) {
+        retries--;
+        goto retry_abort;
+    }
+    kcs_wait_obf();
+    kcs_clear_obf();
+
+    *rsp_len = j;
+}
+
+
+static uint8_t get_dev_id_cmd[] = { 0x18, 0x01 };
+static uint8_t get_dev_id_rsp[] = { 0x1c, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
+                                    0x02, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+/*
+ * Send a get_device_id to do a basic test.
+ */
+static void test_kcs_base(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_cmd(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(get_dev_id_rsp));
+    g_assert(memcmp(get_dev_id_rsp, rsp, rsplen) == 0);
+}
+
+/*
+ * Abort a kcs operation while reading
+ */
+static void test_kcs_abort(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_abort(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsp[0] == IPMI_KCS_ABORTED_BY_CMD);
+}
+
+static uint8_t set_bmc_globals_cmd[] = { 0x18, 0x2e, 0x0f };
+static uint8_t set_bmc_globals_rsp[] = { 0x1c, 0x2e, 0x00 };
+
+/*
+ * Enable interrupts
+ */
+static void test_enable_irq(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_cmd(set_bmc_globals_cmd, sizeof(set_bmc_globals_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(set_bmc_globals_rsp));
+    g_assert(memcmp(set_bmc_globals_rsp, rsp, rsplen) == 0);
+    kcs_ints_enabled = 1;
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+    char *cmdline;
+    int ret;
+
+    /* Check architecture */
+    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
+        g_test_message("Skipping test for non-x86\n");
+        return 0;
+    }
+
+    /* Run the tests */
+    g_test_init(&argc, &argv, NULL);
+
+    cmdline = g_strdup_printf("-vnc none -device ipmi-bmc-sim,id=bmc0"
+                              " -device isa-ipmi-kcs,bmc=bmc0");
+    qtest_start(cmdline);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    qtest_add_func("/ipmi/local/kcs_base", test_kcs_base);
+    qtest_add_func("/ipmi/local/kcs_abort", test_kcs_abort);
+    qtest_add_func("/ipmi/local/kcs_enable_irq", test_enable_irq);
+    qtest_add_func("/ipmi/local/kcs_base_irq", test_kcs_base);
+    qtest_add_func("/ipmi/local/kcs_abort_irq", test_kcs_abort);
+    ret = g_test_run();
+    qtest_quit(global_qtest);
+
+    return ret;
+}
diff --git a/qemu/tests/ipoctal232-test.c b/qemu/tests/ipoctal232-test.c
index 3ac1714b4..846aaf571 100644
--- a/qemu/tests/ipoctal232-test.c
+++ b/qemu/tests/ipoctal232-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/ivshmem-test.c b/qemu/tests/ivshmem-test.c
new file mode 100644
index 000000000..c027ff1e0
--- /dev/null
+++ b/qemu/tests/ivshmem-test.c
@@ -0,0 +1,518 @@
+/*
+ * QTest testcase for ivshmem
+ *
+ * Copyright (c) 2014 SUSE LINUX Products GmbH
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include <glib/gstdio.h>
+#include <sys/mman.h>
+#include "contrib/ivshmem-server/ivshmem-server.h"
+#include "libqos/pci-pc.h"
+#include "libqtest.h"
+#include "qemu-common.h"
+
+#define TMPSHMSIZE (1 << 20)
+static char *tmpshm;
+static void *tmpshmem;
+static char *tmpdir;
+static char *tmpserver;
+
+static void save_fn(QPCIDevice *dev, int devfn, void *data)
+{
+    QPCIDevice **pdev = (QPCIDevice **) data;
+
+    *pdev = dev;
+}
+
+static QPCIDevice *get_device(QPCIBus *pcibus)
+{
+    QPCIDevice *dev;
+
+    dev = NULL;
+    qpci_device_foreach(pcibus, 0x1af4, 0x1110, save_fn, &dev);
+    g_assert(dev != NULL);
+
+    return dev;
+}
+
+typedef struct _IVState {
+    QTestState *qtest;
+    void *reg_base, *mem_base;
+    QPCIBus *pcibus;
+    QPCIDevice *dev;
+} IVState;
+
+enum Reg {
+    INTRMASK = 0,
+    INTRSTATUS = 4,
+    IVPOSITION = 8,
+    DOORBELL = 12,
+};
+
+static const char* reg2str(enum Reg reg) {
+    switch (reg) {
+    case INTRMASK:
+        return "IntrMask";
+    case INTRSTATUS:
+        return "IntrStatus";
+    case IVPOSITION:
+        return "IVPosition";
+    case DOORBELL:
+        return "DoorBell";
+    default:
+        return NULL;
+    }
+}
+
+static inline unsigned in_reg(IVState *s, enum Reg reg)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+    unsigned res;
+
+    global_qtest = s->qtest;
+    res = qpci_io_readl(s->dev, s->reg_base + reg);
+    g_test_message("*%s -> %x\n", name, res);
+    global_qtest = qtest;
+
+    return res;
+}
+
+static inline void out_reg(IVState *s, enum Reg reg, unsigned v)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+
+    global_qtest = s->qtest;
+    g_test_message("%x -> *%s\n", v, name);
+    qpci_io_writel(s->dev, s->reg_base + reg, v);
+    global_qtest = qtest;
+}
+
+static void cleanup_vm(IVState *s)
+{
+    g_free(s->dev);
+    qpci_free_pc(s->pcibus);
+    qtest_quit(s->qtest);
+}
+
+static void setup_vm_cmd(IVState *s, const char *cmd, bool msix)
+{
+    uint64_t barsize;
+
+    s->qtest = qtest_start(cmd);
+    s->pcibus = qpci_init_pc();
+    s->dev = get_device(s->pcibus);
+
+    s->reg_base = qpci_iomap(s->dev, 0, &barsize);
+    g_assert_nonnull(s->reg_base);
+    g_assert_cmpuint(barsize, ==, 256);
+
+    if (msix) {
+        qpci_msix_enable(s->dev);
+    }
+
+    s->mem_base = qpci_iomap(s->dev, 2, &barsize);
+    g_assert_nonnull(s->mem_base);
+    g_assert_cmpuint(barsize, ==, TMPSHMSIZE);
+
+    qpci_device_enable(s->dev);
+}
+
+static void setup_vm(IVState *s)
+{
+    char *cmd = g_strdup_printf("-object memory-backend-file"
+                                ",id=mb1,size=1M,share,mem-path=/dev/shm%s"
+                                " -device ivshmem-plain,memdev=mb1", tmpshm);
+
+    setup_vm_cmd(s, cmd, false);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_single(void)
+{
+    IVState state, *s;
+    uint32_t data[1024];
+    int i;
+
+    setup_vm(&state);
+    s = &state;
+
+    /* initial state of readable registers */
+    g_assert_cmpuint(in_reg(s, INTRMASK), ==, 0);
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 0);
+    g_assert_cmpuint(in_reg(s, IVPOSITION), ==, 0);
+
+    /* trigger interrupt via registers */
+    out_reg(s, INTRMASK, 0xffffffff);
+    g_assert_cmpuint(in_reg(s, INTRMASK), ==, 0xffffffff);
+    out_reg(s, INTRSTATUS, 1);
+    /* check interrupt status */
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 1);
+    /* reading clears */
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 0);
+    /* TODO intercept actual interrupt (needs qtest work) */
+
+    /* invalid register access */
+    out_reg(s, IVPOSITION, 1);
+    in_reg(s, DOORBELL);
+
+    /* ring the (non-functional) doorbell */
+    out_reg(s, DOORBELL, 8 << 16);
+
+    /* write shared memory */
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        data[i] = i;
+    }
+    qtest_memwrite(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+
+    /* verify write */
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(((uint32_t *)tmpshmem)[i], ==, i);
+    }
+
+    /* read it back and verify read */
+    memset(data, 0, sizeof(data));
+    qtest_memread(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(data[i], ==, i);
+    }
+
+    cleanup_vm(s);
+}
+
+static void test_ivshmem_pair(void)
+{
+    IVState state1, state2, *s1, *s2;
+    char *data;
+    int i;
+
+    setup_vm(&state1);
+    s1 = &state1;
+    setup_vm(&state2);
+    s2 = &state2;
+
+    data = g_malloc0(TMPSHMSIZE);
+
+    /* host write, guest 1 & 2 read */
+    memset(tmpshmem, 0x42, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+
+    /* guest 1 write, guest 2 read */
+    memset(data, 0x43, TMPSHMSIZE);
+    qtest_memwrite(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x43);
+    }
+
+    /* guest 2 write, guest 1 read */
+    memset(data, 0x44, TMPSHMSIZE);
+    qtest_memwrite(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x44);
+    }
+
+    cleanup_vm(s1);
+    cleanup_vm(s2);
+    g_free(data);
+}
+
+typedef struct ServerThread {
+    GThread *thread;
+    IvshmemServer *server;
+    int pipe[2]; /* to handle quit */
+} ServerThread;
+
+static void *server_thread(void *data)
+{
+    ServerThread *t = data;
+    IvshmemServer *server = t->server;
+
+    while (true) {
+        fd_set fds;
+        int maxfd, ret;
+
+        FD_ZERO(&fds);
+        FD_SET(t->pipe[0], &fds);
+        maxfd = t->pipe[0] + 1;
+
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            g_critical("select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(t->pipe[0], &fds)) {
+            break;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            g_critical("ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return NULL;
+}
+
+static void setup_vm_with_server(IVState *s, int nvectors, bool msi)
+{
+    char *cmd = g_strdup_printf("-chardev socket,id=chr0,path=%s,nowait "
+                                "-device ivshmem%s,chardev=chr0,vectors=%d",
+                                tmpserver,
+                                msi ? "-doorbell" : ",size=1M,msi=off",
+                                nvectors);
+
+    setup_vm_cmd(s, cmd, msi);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_server(bool msi)
+{
+    IVState state1, state2, *s1, *s2;
+    ServerThread thread;
+    IvshmemServer server;
+    int ret, vm1, vm2;
+    int nvectors = 2;
+    guint64 end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+
+    ret = ivshmem_server_init(&server, tmpserver, tmpshm, true,
+                              TMPSHMSIZE, nvectors,
+                              g_test_verbose());
+    g_assert_cmpint(ret, ==, 0);
+
+    ret = ivshmem_server_start(&server);
+    g_assert_cmpint(ret, ==, 0);
+
+    thread.server = &server;
+    ret = pipe(thread.pipe);
+    g_assert_cmpint(ret, ==, 0);
+    thread.thread = g_thread_new("ivshmem-server", server_thread, &thread);
+    g_assert(thread.thread != NULL);
+
+    setup_vm_with_server(&state1, nvectors, msi);
+    s1 = &state1;
+    setup_vm_with_server(&state2, nvectors, msi);
+    s2 = &state2;
+
+    /* check got different VM ids */
+    vm1 = in_reg(s1, IVPOSITION);
+    vm2 = in_reg(s2, IVPOSITION);
+    g_assert_cmpint(vm1, >=, 0);
+    g_assert_cmpint(vm2, >=, 0);
+    g_assert_cmpint(vm1, !=, vm2);
+
+    /* check number of MSI-X vectors */
+    global_qtest = s1->qtest;
+    if (msi) {
+        ret = qpci_msix_table_size(s1->dev);
+        g_assert_cmpuint(ret, ==, nvectors);
+    }
+
+    /* TODO test behavior before MSI-X is enabled */
+
+    /* ping vm2 -> vm1 on vector 0 */
+    if (msi) {
+        ret = qpci_msix_pending(s1->dev, 0);
+        g_assert_cmpuint(ret, ==, 0);
+    } else {
+        g_assert_cmpuint(in_reg(s1, INTRSTATUS), ==, 0);
+    }
+    out_reg(s2, DOORBELL, vm1 << 16);
+    do {
+        g_usleep(10000);
+        ret = msi ? qpci_msix_pending(s1->dev, 0) : in_reg(s1, INTRSTATUS);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    /* ping vm1 -> vm2 on vector 1 */
+    global_qtest = s2->qtest;
+    if (msi) {
+        ret = qpci_msix_pending(s2->dev, 1);
+        g_assert_cmpuint(ret, ==, 0);
+    } else {
+        g_assert_cmpuint(in_reg(s2, INTRSTATUS), ==, 0);
+    }
+    out_reg(s1, DOORBELL, vm2 << 16 | 1);
+    do {
+        g_usleep(10000);
+        ret = msi ? qpci_msix_pending(s2->dev, 1) : in_reg(s2, INTRSTATUS);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    cleanup_vm(s2);
+    cleanup_vm(s1);
+
+    if (qemu_write_full(thread.pipe[1], "q", 1) != 1) {
+        g_error("qemu_write_full: %s", g_strerror(errno));
+    }
+
+    g_thread_join(thread.thread);
+
+    ivshmem_server_close(&server);
+    close(thread.pipe[1]);
+    close(thread.pipe[0]);
+}
+
+static void test_ivshmem_server_msi(void)
+{
+    test_ivshmem_server(true);
+}
+
+static void test_ivshmem_server_irq(void)
+{
+    test_ivshmem_server(false);
+}
+
+#define PCI_SLOT_HP             0x06
+
+static void test_ivshmem_hotplug(void)
+{
+    gchar *opts;
+
+    qtest_start("");
+
+    opts = g_strdup_printf("'shm': '%s', 'size': '1M'", tmpshm);
+
+    qpci_plug_device_test("ivshmem", "iv1", PCI_SLOT_HP, opts);
+    qpci_unplug_acpi_device_test("iv1", PCI_SLOT_HP);
+
+    qtest_end();
+    g_free(opts);
+}
+
+static void test_ivshmem_memdev(void)
+{
+    IVState state;
+
+    /* just for the sake of checking memory-backend property */
+    setup_vm_cmd(&state, "-object memory-backend-ram,size=1M,id=mb1"
+                 " -device ivshmem-plain,memdev=mb1", false);
+
+    cleanup_vm(&state);
+}
+
+static void cleanup(void)
+{
+    if (tmpshmem) {
+        munmap(tmpshmem, TMPSHMSIZE);
+        tmpshmem = NULL;
+    }
+
+    if (tmpshm) {
+        shm_unlink(tmpshm);
+        g_free(tmpshm);
+        tmpshm = NULL;
+    }
+
+    if (tmpserver) {
+        g_unlink(tmpserver);
+        g_free(tmpserver);
+        tmpserver = NULL;
+    }
+
+    if (tmpdir) {
+        g_rmdir(tmpdir);
+        tmpdir = NULL;
+    }
+}
+
+static void abrt_handler(void *data)
+{
+    cleanup();
+}
+
+static gchar *mktempshm(int size, int *fd)
+{
+    while (true) {
+        gchar *name;
+
+        name = g_strdup_printf("/qtest-%u-%u", getpid(), g_random_int());
+        *fd = shm_open(name, O_CREAT|O_RDWR|O_EXCL,
+                       S_IRWXU|S_IRWXG|S_IRWXO);
+        if (*fd > 0) {
+            g_assert(ftruncate(*fd, size) == 0);
+            return name;
+        }
+
+        g_free(name);
+
+        if (errno != EEXIST) {
+            perror("shm_open");
+            return NULL;
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret, fd;
+    gchar dir[] = "/tmp/ivshmem-test.XXXXXX";
+
+#if !GLIB_CHECK_VERSION(2, 31, 0)
+    if (!g_thread_supported()) {
+        g_thread_init(NULL);
+    }
+#endif
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_abrt_handler(abrt_handler, NULL);
+    /* shm */
+    tmpshm = mktempshm(TMPSHMSIZE, &fd);
+    if (!tmpshm) {
+        return 0;
+    }
+    tmpshmem = mmap(0, TMPSHMSIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    g_assert(tmpshmem != MAP_FAILED);
+    /* server */
+    if (mkdtemp(dir) == NULL) {
+        g_error("mkdtemp: %s", g_strerror(errno));
+    }
+    tmpdir = dir;
+    tmpserver = g_strconcat(tmpdir, "/server", NULL);
+
+    qtest_add_func("/ivshmem/single", test_ivshmem_single);
+    qtest_add_func("/ivshmem/hotplug", test_ivshmem_hotplug);
+    qtest_add_func("/ivshmem/memdev", test_ivshmem_memdev);
+    if (g_test_slow()) {
+        qtest_add_func("/ivshmem/pair", test_ivshmem_pair);
+        qtest_add_func("/ivshmem/server-msi", test_ivshmem_server_msi);
+        qtest_add_func("/ivshmem/server-irq", test_ivshmem_server_irq);
+    }
+
+    ret = g_test_run();
+
+    cleanup();
+
+    return ret;
+}
diff --git a/qemu/tests/libqos/ahci.c b/qemu/tests/libqos/ahci.c
index cf66b3e32..ac6c155c8 100644
--- a/qemu/tests/libqos/ahci.c
+++ b/qemu/tests/libqos/ahci.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "libqtest.h"
@@ -74,7 +75,11 @@ AHCICommandProp ahci_command_properties[] = {
                                  .lba48 = true, .write = true, .ncq = true },
     { .cmd = CMD_READ_MAX,       .lba28 = true },
     { .cmd = CMD_READ_MAX_EXT,   .lba48 = true },
-    { .cmd = CMD_FLUSH_CACHE,    .data = false }
+    { .cmd = CMD_FLUSH_CACHE,    .data = false },
+    { .cmd = CMD_PACKET,         .data = true,  .size = 16,
+                                 .atapi = true, .pio = true },
+    { .cmd = CMD_PACKET_ID,      .data = true,  .pio = true,
+                                 .size = 512,   .read = true }
 };
 
 struct AHCICommand {
@@ -90,7 +95,7 @@ struct AHCICommand {
     /* Data to be transferred to the guest */
     AHCICommandHeader header;
     RegH2DFIS fis;
-    void *atapi_cmd;
+    unsigned char *atapi_cmd;
 };
 
 /**
@@ -110,6 +115,11 @@ void ahci_free(AHCIQState *ahci, uint64_t addr)
     qfree(ahci->parent, addr);
 }
 
+bool is_atapi(AHCIQState *ahci, uint8_t port)
+{
+    return ahci_px_rreg(ahci, port, AHCI_PX_SIG) == AHCI_SIGNATURE_CDROM;
+}
+
 /**
  * Locate, verify, and return a handle to the AHCI device.
  */
@@ -592,6 +602,82 @@ inline unsigned size_to_prdtl(unsigned bytes, unsigned bytes_per_prd)
     return (bytes + bytes_per_prd - 1) / bytes_per_prd;
 }
 
+const AHCIOpts default_opts = { .size = 0 };
+
+/**
+ * ahci_exec: execute a given command on a specific
+ * AHCI port.
+ *
+ * @ahci: The device to send the command to
+ * @port: The port number of the SATA device we wish
+ *        to have execute this command
+ * @op:   The S/ATA command to execute, or if opts.atapi
+ *        is true, the SCSI command code.
+ * @opts: Optional arguments to modify execution behavior.
+ */
+void ahci_exec(AHCIQState *ahci, uint8_t port,
+               uint8_t op, const AHCIOpts *opts_in)
+{
+    AHCICommand *cmd;
+    int rc;
+    AHCIOpts *opts;
+
+    opts = g_memdup((opts_in == NULL ? &default_opts : opts_in),
+                    sizeof(AHCIOpts));
+
+    /* No guest buffer provided, create one. */
+    if (opts->size && !opts->buffer) {
+        opts->buffer = ahci_alloc(ahci, opts->size);
+        g_assert(opts->buffer);
+        qmemset(opts->buffer, 0x00, opts->size);
+    }
+
+    /* Command creation */
+    if (opts->atapi) {
+        cmd = ahci_atapi_command_create(op);
+        if (opts->atapi_dma) {
+            ahci_command_enable_atapi_dma(cmd);
+        }
+    } else {
+        cmd = ahci_command_create(op);
+    }
+    ahci_command_adjust(cmd, opts->lba, opts->buffer,
+                        opts->size, opts->prd_size);
+
+    if (opts->pre_cb) {
+        rc = opts->pre_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+
+    /* Write command to memory and issue it */
+    ahci_command_commit(ahci, cmd, port);
+    ahci_command_issue_async(ahci, cmd);
+    if (opts->error) {
+        qmp_eventwait("STOP");
+    }
+    if (opts->mid_cb) {
+        rc = opts->mid_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+    if (opts->error) {
+        qmp_async("{'execute':'cont' }");
+        qmp_eventwait("RESUME");
+    }
+
+    /* Wait for command to complete and verify sanity */
+    ahci_command_wait(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    if (opts->post_cb) {
+        rc = opts->post_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+    ahci_command_free(cmd);
+    if (opts->buffer != opts_in->buffer) {
+        ahci_free(ahci, opts->buffer);
+    }
+    g_free(opts);
+}
+
 /* Issue a command, expecting it to fail and STOP the VM */
 AHCICommand *ahci_guest_io_halt(AHCIQState *ahci, uint8_t port,
                                 uint8_t ide_cmd, uint64_t buffer,
@@ -659,16 +745,16 @@ void ahci_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
     props = ahci_command_find(ide_cmd);
     g_assert(props);
     ptr = ahci_alloc(ahci, bufsize);
-    g_assert(ptr);
+    g_assert(!bufsize || ptr);
     qmemset(ptr, 0x00, bufsize);
 
-    if (props->write) {
+    if (bufsize && props->write) {
         bufwrite(ptr, buffer, bufsize);
     }
 
     ahci_guest_io(ahci, port, ide_cmd, ptr, bufsize, sector);
 
-    if (props->read) {
+    if (bufsize && props->read) {
         bufread(ptr, buffer, bufsize);
     }
 
@@ -731,6 +817,18 @@ static void command_table_init(AHCICommand *cmd)
     memset(fis->aux, 0x00, ARRAY_SIZE(fis->aux));
 }
 
+void ahci_command_enable_atapi_dma(AHCICommand *cmd)
+{
+    RegH2DFIS *fis = &(cmd->fis);
+    g_assert(cmd->props->atapi);
+    fis->feature_low |= 0x01;
+    cmd->interrupts &= ~AHCI_PX_IS_PSS;
+    cmd->props->dma = true;
+    cmd->props->pio = false;
+    /* BUG: We expect the DMA Setup interrupt for DMA commands */
+    /* cmd->interrupts |= AHCI_PX_IS_DSS; */
+}
+
 AHCICommand *ahci_command_create(uint8_t command_name)
 {
     AHCICommandProp *props = ahci_command_find(command_name);
@@ -742,10 +840,10 @@ AHCICommand *ahci_command_create(uint8_t command_name)
     g_assert(!(props->lba28 && props->lba48));
     g_assert(!(props->read && props->write));
     g_assert(!props->size || props->data);
-    g_assert(!props->ncq || (props->ncq && props->lba48));
+    g_assert(!props->ncq || props->lba48);
 
     /* Defaults and book-keeping */
-    cmd->props = props;
+    cmd->props = g_memdup(props, sizeof(AHCICommandProp));
     cmd->name = command_name;
     cmd->xbytes = props->size;
     cmd->prd_size = 4096;
@@ -767,8 +865,23 @@ AHCICommand *ahci_command_create(uint8_t command_name)
     return cmd;
 }
 
+AHCICommand *ahci_atapi_command_create(uint8_t scsi_cmd)
+{
+    AHCICommand *cmd = ahci_command_create(CMD_PACKET);
+    cmd->atapi_cmd = g_malloc0(16);
+    cmd->atapi_cmd[0] = scsi_cmd;
+    /* ATAPI needs a PIO transfer chunk size set inside of the LBA registers.
+     * The block/sector size is a natural default. */
+    cmd->fis.lba_lo[1] = ATAPI_SECTOR_SIZE >> 8 & 0xFF;
+    cmd->fis.lba_lo[2] = ATAPI_SECTOR_SIZE & 0xFF;
+
+    return cmd;
+}
+
 void ahci_command_free(AHCICommand *cmd)
 {
+    g_free(cmd->atapi_cmd);
+    g_free(cmd->props);
     g_free(cmd);
 }
 
@@ -782,10 +895,34 @@ void ahci_command_clr_flags(AHCICommand *cmd, uint16_t cmdh_flags)
     cmd->header.flags &= ~cmdh_flags;
 }
 
+static void ahci_atapi_command_set_offset(AHCICommand *cmd, uint64_t lba)
+{
+    unsigned char *cbd = cmd->atapi_cmd;
+    g_assert(cbd);
+
+    switch (cbd[0]) {
+    case CMD_ATAPI_READ_10:
+        g_assert_cmpuint(lba, <=, UINT32_MAX);
+        stl_be_p(&cbd[2], lba);
+        break;
+    default:
+        /* SCSI doesn't have uniform packet formats,
+         * so you have to add support for it manually. Sorry! */
+        g_assert_not_reached();
+    }
+}
+
 void ahci_command_set_offset(AHCICommand *cmd, uint64_t lba_sect)
 {
     RegH2DFIS *fis = &(cmd->fis);
-    if (cmd->props->lba28) {
+
+    if (cmd->props->atapi) {
+        ahci_atapi_command_set_offset(cmd, lba_sect);
+        return;
+    } else if (!cmd->props->data && !lba_sect) {
+        /* Not meaningful, ignore. */
+        return;
+    } else if (cmd->props->lba28) {
         g_assert_cmphex(lba_sect, <=, 0xFFFFFFF);
     } else if (cmd->props->lba48 || cmd->props->ncq) {
         g_assert_cmphex(lba_sect, <=, 0xFFFFFFFFFFFF);
@@ -811,6 +948,24 @@ void ahci_command_set_buffer(AHCICommand *cmd, uint64_t buffer)
     cmd->buffer = buffer;
 }
 
+static void ahci_atapi_set_size(AHCICommand *cmd, uint64_t xbytes)
+{
+    unsigned char *cbd = cmd->atapi_cmd;
+    uint64_t nsectors = xbytes / 2048;
+    g_assert(cbd);
+
+    switch (cbd[0]) {
+    case CMD_ATAPI_READ_10:
+        g_assert_cmpuint(nsectors, <=, UINT16_MAX);
+        stw_be_p(&cbd[7], nsectors);
+        break;
+    default:
+        /* SCSI doesn't have uniform packet formats,
+         * so you have to add support for it manually. Sorry! */
+        g_assert_not_reached();
+    }
+}
+
 void ahci_command_set_sizes(AHCICommand *cmd, uint64_t xbytes,
                             unsigned prd_size)
 {
@@ -829,6 +984,8 @@ void ahci_command_set_sizes(AHCICommand *cmd, uint64_t xbytes,
         NCQFIS *nfis = (NCQFIS *)&(cmd->fis);
         nfis->sector_low = sect_count & 0xFF;
         nfis->sector_hi = (sect_count >> 8) & 0xFF;
+    } else if (cmd->props->atapi) {
+        ahci_atapi_set_size(cmd, xbytes);
     } else {
         cmd->fis.count = sect_count;
     }
@@ -877,9 +1034,14 @@ void ahci_command_commit(AHCIQState *ahci, AHCICommand *cmd, uint8_t port)
     g_assert((table_ptr & 0x7F) == 0x00);
     cmd->header.ctba = table_ptr;
 
-    /* Commit the command header and command FIS */
+    /* Commit the command header (part of the Command List Buffer) */
     ahci_set_command_header(ahci, port, cmd->slot, &(cmd->header));
+    /* Now, write the command table (FIS, ACMD, and PRDT) -- FIS first, */
     ahci_write_fis(ahci, cmd);
+    /* Then ATAPI CMD, if needed */
+    if (cmd->props->atapi) {
+        memwrite(table_ptr + 0x40, cmd->atapi_cmd, 16);
+    }
 
     /* Construct and write the PRDs to the command table */
     g_assert_cmphex(prdtl, ==, cmd->header.prdtl);
diff --git a/qemu/tests/libqos/ahci.h b/qemu/tests/libqos/ahci.h
index cffc2c351..71dd7a6e5 100644
--- a/qemu/tests/libqos/ahci.h
+++ b/qemu/tests/libqos/ahci.h
@@ -25,9 +25,6 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdbool.h>
 #include "libqos/libqos.h"
 #include "libqos/pci.h"
 #include "libqos/malloc-pc.h"
@@ -244,6 +241,10 @@
 #define AHCI_VERSION_1_3         (0x00010300)
 
 #define AHCI_SECTOR_SIZE                (512)
+#define ATAPI_SECTOR_SIZE              (2048)
+
+#define AHCI_SIGNATURE_CDROM     (0xeb140101)
+#define AHCI_SIGNATURE_DISK      (0x00000101)
 
 /* FIS types */
 enum {
@@ -277,11 +278,18 @@ enum {
     CMD_READ_MAX_EXT   = 0x27,
     CMD_FLUSH_CACHE    = 0xE7,
     CMD_IDENTIFY       = 0xEC,
+    CMD_PACKET         = 0xA0,
+    CMD_PACKET_ID      = 0xA1,
     /* NCQ */
     READ_FPDMA_QUEUED  = 0x60,
     WRITE_FPDMA_QUEUED = 0x61,
 };
 
+/* ATAPI Commands */
+enum {
+    CMD_ATAPI_READ_10 = 0x28,
+};
+
 /* AHCI Command Header Flags & Masks*/
 #define CMDH_CFL        (0x1F)
 #define CMDH_ATAPI      (0x20)
@@ -451,6 +459,21 @@ typedef struct PRD {
 /* Opaque, defined within ahci.c */
 typedef struct AHCICommand AHCICommand;
 
+/* Options to ahci_exec */
+typedef struct AHCIOpts {
+    size_t size;
+    unsigned prd_size;
+    uint64_t lba;
+    uint64_t buffer;
+    bool atapi;
+    bool atapi_dma;
+    bool error;
+    int (*pre_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    int (*mid_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    int (*post_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    void *opaque;
+} AHCIOpts;
+
 /*** Macro Utilities ***/
 #define BITANY(data, mask) (((data) & (mask)) != 0)
 #define BITSET(data, mask) (((data) & (mask)) == (mask))
@@ -527,14 +550,28 @@ static inline void ahci_px_clr(AHCIQState *ahci, uint8_t port,
 /*** Prototypes ***/
 uint64_t ahci_alloc(AHCIQState *ahci, size_t bytes);
 void ahci_free(AHCIQState *ahci, uint64_t addr);
+void ahci_clean_mem(AHCIQState *ahci);
+
+/* Device management */
 QPCIDevice *get_ahci_device(uint32_t *fingerprint);
 void free_ahci_device(QPCIDevice *dev);
-void ahci_clean_mem(AHCIQState *ahci);
 void ahci_pci_enable(AHCIQState *ahci);
 void start_ahci_device(AHCIQState *ahci);
 void ahci_hba_enable(AHCIQState *ahci);
+
+/* Port Management */
 unsigned ahci_port_select(AHCIQState *ahci);
 void ahci_port_clear(AHCIQState *ahci, uint8_t port);
+
+/* Command header / table management */
+unsigned ahci_pick_cmd(AHCIQState *ahci, uint8_t port);
+void ahci_get_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd);
+void ahci_set_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd);
+void ahci_destroy_command(AHCIQState *ahci, uint8_t port, uint8_t slot);
+
+/* AHCI sanity check routines */
 void ahci_port_check_error(AHCIQState *ahci, uint8_t port);
 void ahci_port_check_interrupts(AHCIQState *ahci, uint8_t port,
                                 uint32_t intr_mask);
@@ -543,14 +580,12 @@ void ahci_port_check_d2h_sanity(AHCIQState *ahci, uint8_t port, uint8_t slot);
 void ahci_port_check_pio_sanity(AHCIQState *ahci, uint8_t port,
                                 uint8_t slot, size_t buffsize);
 void ahci_port_check_cmd_sanity(AHCIQState *ahci, AHCICommand *cmd);
-void ahci_get_command_header(AHCIQState *ahci, uint8_t port,
-                             uint8_t slot, AHCICommandHeader *cmd);
-void ahci_set_command_header(AHCIQState *ahci, uint8_t port,
-                             uint8_t slot, AHCICommandHeader *cmd);
-void ahci_destroy_command(AHCIQState *ahci, uint8_t port, uint8_t slot);
-void ahci_write_fis(AHCIQState *ahci, AHCICommand *cmd);
-unsigned ahci_pick_cmd(AHCIQState *ahci, uint8_t port);
+
+/* Misc */
+bool is_atapi(AHCIQState *ahci, uint8_t port);
 unsigned size_to_prdtl(unsigned bytes, unsigned bytes_per_prd);
+
+/* Command: Macro level execution */
 void ahci_guest_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
                    uint64_t gbuffer, size_t size, uint64_t sector);
 AHCICommand *ahci_guest_io_halt(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
@@ -558,9 +593,12 @@ AHCICommand *ahci_guest_io_halt(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
 void ahci_guest_io_resume(AHCIQState *ahci, AHCICommand *cmd);
 void ahci_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
              void *buffer, size_t bufsize, uint64_t sector);
+void ahci_exec(AHCIQState *ahci, uint8_t port,
+               uint8_t op, const AHCIOpts *opts);
 
-/* Command Lifecycle */
+/* Command: Fine-grained lifecycle */
 AHCICommand *ahci_command_create(uint8_t command_name);
+AHCICommand *ahci_atapi_command_create(uint8_t scsi_cmd);
 void ahci_command_commit(AHCIQState *ahci, AHCICommand *cmd, uint8_t port);
 void ahci_command_issue(AHCIQState *ahci, AHCICommand *cmd);
 void ahci_command_issue_async(AHCIQState *ahci, AHCICommand *cmd);
@@ -568,7 +606,7 @@ void ahci_command_wait(AHCIQState *ahci, AHCICommand *cmd);
 void ahci_command_verify(AHCIQState *ahci, AHCICommand *cmd);
 void ahci_command_free(AHCICommand *cmd);
 
-/* Command adjustments */
+/* Command: adjustments */
 void ahci_command_set_flags(AHCICommand *cmd, uint16_t cmdh_flags);
 void ahci_command_clr_flags(AHCICommand *cmd, uint16_t cmdh_flags);
 void ahci_command_set_offset(AHCICommand *cmd, uint64_t lba_sect);
@@ -577,10 +615,13 @@ void ahci_command_set_size(AHCICommand *cmd, uint64_t xbytes);
 void ahci_command_set_prd_size(AHCICommand *cmd, unsigned prd_size);
 void ahci_command_set_sizes(AHCICommand *cmd, uint64_t xbytes,
                             unsigned prd_size);
+void ahci_command_set_acmd(AHCICommand *cmd, void *acmd);
+void ahci_command_enable_atapi_dma(AHCICommand *cmd);
 void ahci_command_adjust(AHCICommand *cmd, uint64_t lba_sect, uint64_t gbuffer,
                          uint64_t xbytes, unsigned prd_size);
 
-/* Command Misc */
+/* Command: Misc */
 uint8_t ahci_command_slot(AHCICommand *cmd);
+void ahci_write_fis(AHCIQState *ahci, AHCICommand *cmd);
 
 #endif
diff --git a/qemu/tests/libqos/fw_cfg.c b/qemu/tests/libqos/fw_cfg.c
index ef00fedf1..76894d575 100644
--- a/qemu/tests/libqos/fw_cfg.c
+++ b/qemu/tests/libqos/fw_cfg.c
@@ -12,6 +12,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "libqos/fw_cfg.h"
 #include "libqtest.h"
diff --git a/qemu/tests/libqos/fw_cfg.h b/qemu/tests/libqos/fw_cfg.h
index 61b1548b4..e8371b231 100644
--- a/qemu/tests/libqos/fw_cfg.h
+++ b/qemu/tests/libqos/fw_cfg.h
@@ -13,8 +13,6 @@
 #ifndef LIBQOS_FW_CFG_H
 #define LIBQOS_FW_CFG_H
 
-#include <stdint.h>
-#include <sys/types.h>
 
 typedef struct QFWCFG QFWCFG;
 
diff --git a/qemu/tests/libqos/i2c-imx.c b/qemu/tests/libqos/i2c-imx.c
new file mode 100644
index 000000000..51c3468f9
--- /dev/null
+++ b/qemu/tests/libqos/i2c-imx.c
@@ -0,0 +1,208 @@
+/*
+ * QTest i.MX I2C driver
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "libqos/i2c.h"
+
+#include <glib.h>
+
+#include "libqtest.h"
+
+#include "hw/i2c/imx_i2c.h"
+
+enum IMXI2CDirection {
+    IMX_I2C_READ,
+    IMX_I2C_WRITE,
+};
+
+typedef struct IMXI2C {
+    I2CAdapter parent;
+
+    uint64_t addr;
+} IMXI2C;
+
+
+static void imx_i2c_set_slave_addr(IMXI2C *s, uint8_t addr,
+                                   enum IMXI2CDirection direction)
+{
+    writeb(s->addr + I2DR_ADDR, (addr << 1) |
+           (direction == IMX_I2C_READ ? 1 : 0));
+}
+
+static void imx_i2c_send(I2CAdapter *i2c, uint8_t addr,
+                         const uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_WRITE);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        /* write the data */
+        writeb(s->addr + I2DR_ADDR, buf[size]);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) != 0);
+        g_assert((status & I2SR_RXAK) == 0);
+
+        /* ack the interrupt */
+        writeb(s->addr + I2SR_ADDR, 0);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    /* release the bus */
+    data &= ~(I2CR_MSTA | I2CR_MTX);
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+static void imx_i2c_recv(I2CAdapter *i2c, uint8_t addr,
+                         uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_READ);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    /* set the bus for read */
+    data &= ~I2CR_MTX;
+    /* if only one byte don't ack */
+    if (len != 1) {
+        data &= ~I2CR_TXAK;
+    }
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* dummy read */
+    readb(s->addr + I2DR_ADDR);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        if (size == (len - 1)) {
+            /* stop the read transaction */
+            data &= ~(I2CR_MSTA | I2CR_MTX);
+        } else {
+            /* ack the data read */
+            data |= I2CR_TXAK;
+        }
+        writeb(s->addr + I2CR_ADDR, data);
+
+        /* read the data */
+        buf[size] = readb(s->addr + I2DR_ADDR);
+
+        if (size != (len - 1)) {
+            status = readb(s->addr + I2SR_ADDR);
+            g_assert((status & I2SR_IIF) != 0);
+
+            /* ack the interrupt */
+            writeb(s->addr + I2SR_ADDR, 0);
+        }
+
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+I2CAdapter *imx_i2c_create(uint64_t addr)
+{
+    IMXI2C *s = g_malloc0(sizeof(*s));
+    I2CAdapter *i2c = (I2CAdapter *)s;
+
+    s->addr = addr;
+
+    i2c->send = imx_i2c_send;
+    i2c->recv = imx_i2c_recv;
+
+    return i2c;
+}
diff --git a/qemu/tests/libqos/i2c-omap.c b/qemu/tests/libqos/i2c-omap.c
index 3d4d45d84..2028f2f14 100644
--- a/qemu/tests/libqos/i2c-omap.c
+++ b/qemu/tests/libqos/i2c-omap.c
@@ -6,12 +6,11 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "libqos/i2c.h"
 
 #include <glib.h>
-#include <string.h>
 
-#include "qemu/osdep.h"
 #include "qemu/bswap.h"
 #include "libqtest.h"
 
diff --git a/qemu/tests/libqos/i2c.c b/qemu/tests/libqos/i2c.c
index da7592f71..23bc2a3eb 100644
--- a/qemu/tests/libqos/i2c.c
+++ b/qemu/tests/libqos/i2c.c
@@ -6,6 +6,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "libqos/i2c.h"
 #include "libqtest.h"
 
diff --git a/qemu/tests/libqos/i2c.h b/qemu/tests/libqos/i2c.h
index 1ce9af405..6e648f922 100644
--- a/qemu/tests/libqos/i2c.h
+++ b/qemu/tests/libqos/i2c.h
@@ -9,7 +9,6 @@
 #ifndef LIBQOS_I2C_H
 #define LIBQOS_I2C_H
 
-#include <stdint.h>
 
 typedef struct I2CAdapter I2CAdapter;
 struct I2CAdapter {
@@ -27,4 +26,7 @@ void i2c_recv(I2CAdapter *i2c, uint8_t addr,
 /* libi2c-omap.c */
 I2CAdapter *omap_i2c_create(uint64_t addr);
 
+/* libi2c-imx.c */
+I2CAdapter *imx_i2c_create(uint64_t addr);
+
 #endif
diff --git a/qemu/tests/libqos/libqos-pc.c b/qemu/tests/libqos/libqos-pc.c
index 140369937..72b5e3ba0 100644
--- a/qemu/tests/libqos/libqos-pc.c
+++ b/qemu/tests/libqos/libqos-pc.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "libqos/libqos-pc.h"
 #include "libqos/malloc-pc.h"
 
diff --git a/qemu/tests/libqos/libqos.c b/qemu/tests/libqos/libqos.c
index fce625b18..79b0b29b4 100644
--- a/qemu/tests/libqos/libqos.c
+++ b/qemu/tests/libqos/libqos.c
@@ -1,9 +1,5 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <sys/wait.h>
 
 #include "libqtest.h"
@@ -147,6 +143,23 @@ void migrate(QOSState *from, QOSState *to, const char *uri)
     set_context(to);
 }
 
+bool have_qemu_img(void)
+{
+    char *rpath;
+    const char *path = getenv("QTEST_QEMU_IMG");
+    if (!path) {
+        return false;
+    }
+
+    rpath = realpath(path, NULL);
+    if (!rpath) {
+        return false;
+    } else {
+        free(rpath);
+        return true;
+    }
+}
+
 void mkimg(const char *file, const char *fmt, unsigned size_mb)
 {
     gchar *cli;
@@ -155,13 +168,14 @@ void mkimg(const char *file, const char *fmt, unsigned size_mb)
     GError *err = NULL;
     char *qemu_img_path;
     gchar *out, *out2;
-    char *abs_path;
+    char *qemu_img_abs_path;
 
     qemu_img_path = getenv("QTEST_QEMU_IMG");
-    abs_path = realpath(qemu_img_path, NULL);
-    assert(qemu_img_path);
+    g_assert(qemu_img_path);
+    qemu_img_abs_path = realpath(qemu_img_path, NULL);
+    g_assert(qemu_img_abs_path);
 
-    cli = g_strdup_printf("%s create -f %s %s %uM", abs_path,
+    cli = g_strdup_printf("%s create -f %s %s %uM", qemu_img_abs_path,
                           fmt, file, size_mb);
     ret = g_spawn_command_line_sync(cli, &out, &out2, &rc, &err);
     if (err) {
@@ -183,7 +197,7 @@ void mkimg(const char *file, const char *fmt, unsigned size_mb)
     g_free(out);
     g_free(out2);
     g_free(cli);
-    free(abs_path);
+    free(qemu_img_abs_path);
 }
 
 void mkqcow2(const char *file, unsigned size_mb)
@@ -212,3 +226,29 @@ void prepare_blkdebug_script(const char *debug_fn, const char *event)
     ret = fclose(debug_file);
     g_assert(ret == 0);
 }
+
+void generate_pattern(void *buffer, size_t len, size_t cycle_len)
+{
+    int i, j;
+    unsigned char *tx = (unsigned char *)buffer;
+    unsigned char p;
+    size_t *sx;
+
+    /* Write an indicative pattern that varies and is unique per-cycle */
+    p = rand() % 256;
+    for (i = 0; i < len; i++) {
+        tx[i] = p++ % 256;
+        if (i % cycle_len == 0) {
+            p = rand() % 256;
+        }
+    }
+
+    /* force uniqueness by writing an id per-cycle */
+    for (i = 0; i < len / cycle_len; i++) {
+        j = i * cycle_len;
+        if (j + sizeof(*sx) <= len) {
+            sx = (size_t *)&tx[j];
+            *sx = i;
+        }
+    }
+}
diff --git a/qemu/tests/libqos/libqos.h b/qemu/tests/libqos/libqos.h
index e1f14ea6f..ca14d2e9f 100644
--- a/qemu/tests/libqos/libqos.h
+++ b/qemu/tests/libqos/libqos.h
@@ -19,11 +19,13 @@ typedef struct QOSState {
 QOSState *qtest_vboot(QOSOps *ops, const char *cmdline_fmt, va_list ap);
 QOSState *qtest_boot(QOSOps *ops, const char *cmdline_fmt, ...);
 void qtest_shutdown(QOSState *qs);
+bool have_qemu_img(void);
 void mkimg(const char *file, const char *fmt, unsigned size_mb);
 void mkqcow2(const char *file, unsigned size_mb);
 void set_context(QOSState *s);
 void migrate(QOSState *from, QOSState *to, const char *uri);
 void prepare_blkdebug_script(const char *debug_fn, const char *event);
+void generate_pattern(void *buffer, size_t len, size_t cycle_len);
 
 static inline uint64_t qmalloc(QOSState *q, size_t bytes)
 {
diff --git a/qemu/tests/libqos/malloc-generic.c b/qemu/tests/libqos/malloc-generic.c
index d30a2f424..6000df2b8 100644
--- a/qemu/tests/libqos/malloc-generic.c
+++ b/qemu/tests/libqos/malloc-generic.c
@@ -7,6 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "libqos/malloc-generic.h"
 #include "libqos/malloc.h"
diff --git a/qemu/tests/libqos/malloc-pc.c b/qemu/tests/libqos/malloc-pc.c
index 6e253b687..eee706bd6 100644
--- a/qemu/tests/libqos/malloc-pc.c
+++ b/qemu/tests/libqos/malloc-pc.c
@@ -10,11 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqos/malloc-pc.h"
 #include "libqos/fw_cfg.h"
 
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
+#include "hw/nvram/fw_cfg_keys.h"
 
 #include "qemu-common.h"
 #include <glib.h>
diff --git a/qemu/tests/libqos/malloc.c b/qemu/tests/libqos/malloc.c
index 82b9df537..c0df52f33 100644
--- a/qemu/tests/libqos/malloc.c
+++ b/qemu/tests/libqos/malloc.c
@@ -10,10 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqos/malloc.h"
 #include "qemu-common.h"
-#include <stdio.h>
-#include <inttypes.h>
 #include <glib.h>
 
 typedef QTAILQ_HEAD(MemList, MemBlock) MemList;
@@ -270,6 +269,10 @@ uint64_t guest_alloc(QGuestAllocator *allocator, size_t size)
     uint64_t rsize = size;
     uint64_t naddr;
 
+    if (!size) {
+        return 0;
+    }
+
     rsize += (allocator->page_size - 1);
     rsize &= -allocator->page_size;
     g_assert_cmpint((allocator->start + rsize), <=, allocator->end);
diff --git a/qemu/tests/libqos/malloc.h b/qemu/tests/libqos/malloc.h
index 0c6c9b7f3..ae9dac8f6 100644
--- a/qemu/tests/libqos/malloc.h
+++ b/qemu/tests/libqos/malloc.h
@@ -13,8 +13,6 @@
 #ifndef LIBQOS_MALLOC_H
 #define LIBQOS_MALLOC_H
 
-#include <stdint.h>
-#include <sys/types.h>
 #include "qemu/queue.h"
 
 typedef enum {
diff --git a/qemu/tests/libqos/pci-pc.c b/qemu/tests/libqos/pci-pc.c
index 6dba0db00..77f15e5a0 100644
--- a/qemu/tests/libqos/pci-pc.c
+++ b/qemu/tests/libqos/pci-pc.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/pci-pc.h"
 
@@ -183,7 +184,9 @@ static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno, uint64_t *s
     if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
         uint16_t loc;
 
-        g_assert((s->pci_iohole_alloc + size) <= s->pci_iohole_size);
+        g_assert(QEMU_ALIGN_UP(s->pci_iohole_alloc, size) + size
+                 <= s->pci_iohole_size);
+        s->pci_iohole_alloc = QEMU_ALIGN_UP(s->pci_iohole_alloc, size);
         loc = s->pci_iohole_start + s->pci_iohole_alloc;
         s->pci_iohole_alloc += size;
 
@@ -193,7 +196,9 @@ static void *qpci_pc_iomap(QPCIBus *bus, QPCIDevice *dev, int barno, uint64_t *s
     } else {
         uint64_t loc;
 
-        g_assert((s->pci_hole_alloc + size) <= s->pci_hole_size);
+        g_assert(QEMU_ALIGN_UP(s->pci_hole_alloc, size) + size
+                 <= s->pci_hole_size);
+        s->pci_hole_alloc = QEMU_ALIGN_UP(s->pci_hole_alloc, size);
         loc = s->pci_hole_start + s->pci_hole_alloc;
         s->pci_hole_alloc += size;
 
diff --git a/qemu/tests/libqos/pci.c b/qemu/tests/libqos/pci.c
index 4e630c250..0e104e14e 100644
--- a/qemu/tests/libqos/pci.c
+++ b/qemu/tests/libqos/pci.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqos/pci.h"
 
 #include "hw/pci/pci_regs.h"
@@ -34,11 +35,13 @@ void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
 
             if (vendor_id != -1 &&
                 qpci_config_readw(dev, PCI_VENDOR_ID) != vendor_id) {
+                g_free(dev);
                 continue;
             }
 
             if (device_id != -1 &&
                 qpci_config_readw(dev, PCI_DEVICE_ID) != device_id) {
+                g_free(dev);
                 continue;
             }
 
diff --git a/qemu/tests/libqos/pci.h b/qemu/tests/libqos/pci.h
index dfaee9ec3..c06add8db 100644
--- a/qemu/tests/libqos/pci.h
+++ b/qemu/tests/libqos/pci.h
@@ -13,7 +13,6 @@
 #ifndef LIBQOS_PCI_H
 #define LIBQOS_PCI_H
 
-#include <stdint.h>
 #include "libqtest.h"
 
 #define QPCI_DEVFN(dev, fn) (((dev) << 3) | (fn))
diff --git a/qemu/tests/libqos/usb.c b/qemu/tests/libqos/usb.c
index 41d89b848..87efb9078 100644
--- a/qemu/tests/libqos/usb.c
+++ b/qemu/tests/libqos/usb.c
@@ -11,10 +11,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "hw/usb/uhci-regs.h"
 #include "libqos/usb.h"
 
diff --git a/qemu/tests/libqos/virtio-mmio.c b/qemu/tests/libqos/virtio-mmio.c
index b3e62e77d..a4382f366 100644
--- a/qemu/tests/libqos/virtio-mmio.c
+++ b/qemu/tests/libqos/virtio-mmio.c
@@ -7,8 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
 #include "libqtest.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-mmio.h"
diff --git a/qemu/tests/libqos/virtio-pci.c b/qemu/tests/libqos/virtio-pci.c
index f9fb924b8..fde2ff0bc 100644
--- a/qemu/tests/libqos/virtio-pci.c
+++ b/qemu/tests/libqos/virtio-pci.c
@@ -7,8 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
 #include "libqtest.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
diff --git a/qemu/tests/libqos/virtio.c b/qemu/tests/libqos/virtio.c
index 3205b88d9..613decea5 100644
--- a/qemu/tests/libqos/virtio.c
+++ b/qemu/tests/libqos/virtio.c
@@ -7,6 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "libqtest.h"
 #include "libqos/virtio.h"
diff --git a/qemu/tests/libqtest.c b/qemu/tests/libqtest.c
index e5188e032..b12a9e4ca 100644
--- a/qemu/tests/libqtest.c
+++ b/qemu/tests/libqtest.c
@@ -14,22 +14,14 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "libqtest.h"
 
 #include <glib.h>
-#include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
 #include <sys/un.h>
-#include <inttypes.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-
-#include "qemu/compiler.h"
-#include "qemu/osdep.h"
+
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/qjson.h"
@@ -46,9 +38,9 @@ struct QTestState
     bool irq_level[MAX_IRQ];
     GString *rx;
     pid_t qemu_pid;  /* our child QEMU process */
-    struct sigaction sigact_old; /* restored on exit */
 };
 
+static GHookList abrt_hooks;
 static GList *qtest_instances;
 static struct sigaction sigact_old;
 
@@ -110,12 +102,14 @@ static void kill_qemu(QTestState *s)
     }
 }
 
+static void kill_qemu_hook_func(void *s)
+{
+    kill_qemu(s);
+}
+
 static void sigabrt_handler(int signo)
 {
-    GList *elem;
-    for (elem = qtest_instances; elem; elem = elem->next) {
-        kill_qemu(elem->data);
-    }
+    g_hook_list_invoke(&abrt_hooks, FALSE);
 }
 
 static void setup_sigabrt_handler(void)
@@ -136,6 +130,23 @@ static void cleanup_sigabrt_handler(void)
     sigaction(SIGABRT, &sigact_old, NULL);
 }
 
+void qtest_add_abrt_handler(GHookFunc fn, const void *data)
+{
+    GHook *hook;
+
+    /* Only install SIGABRT handler once */
+    if (!abrt_hooks.is_setup) {
+        g_hook_list_init(&abrt_hooks, sizeof(GHook));
+        setup_sigabrt_handler();
+    }
+
+    hook = g_hook_alloc(&abrt_hooks);
+    hook->func = fn;
+    hook->data = (void *)data;
+
+    g_hook_prepend(&abrt_hooks, hook);
+}
+
 QTestState *qtest_init(const char *extra_args)
 {
     QTestState *s;
@@ -156,12 +167,7 @@ QTestState *qtest_init(const char *extra_args)
     sock = init_socket(socket_path);
     qmpsock = init_socket(qmp_socket_path);
 
-    /* Only install SIGABRT handler once */
-    if (!qtest_instances) {
-        setup_sigabrt_handler();
-    }
-
-    qtest_instances = g_list_prepend(qtest_instances, s);
+    qtest_add_abrt_handler(kill_qemu_hook_func, s);
 
     s->qemu_pid = fork();
     if (s->qemu_pid == 0) {
@@ -209,13 +215,14 @@ QTestState *qtest_init(const char *extra_args)
 
 void qtest_quit(QTestState *s)
 {
+    qtest_instances = g_list_remove(qtest_instances, s);
+    g_hook_destroy_link(&abrt_hooks, g_hook_find_data(&abrt_hooks, TRUE, s));
+
     /* Uninstall SIGABRT handler on last instance */
-    if (qtest_instances && !qtest_instances->next) {
+    if (!qtest_instances) {
         cleanup_sigabrt_handler();
     }
 
-    qtest_instances = g_list_remove(qtest_instances, s);
-
     kill_qemu(s);
     close(s->fd);
     close(s->qmp_fd);
@@ -341,7 +348,7 @@ typedef struct {
     QDict *response;
 } QMPResponseParser;
 
-static void qmp_response(JSONMessageParser *parser, QList *tokens)
+static void qmp_response(JSONMessageParser *parser, GQueue *tokens)
 {
     QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser);
     QObject *obj;
@@ -357,7 +364,7 @@ static void qmp_response(JSONMessageParser *parser, QList *tokens)
     qmp->response = (QDict *)obj;
 }
 
-QDict *qtest_qmp_receive(QTestState *s)
+QDict *qmp_fd_receive(int fd)
 {
     QMPResponseParser qmp;
     bool log = getenv("QTEST_LOG") != NULL;
@@ -368,7 +375,7 @@ QDict *qtest_qmp_receive(QTestState *s)
         ssize_t len;
         char c;
 
-        len = read(s->qmp_fd, &c, 1);
+        len = read(fd, &c, 1);
         if (len == -1 && errno == EINTR) {
             continue;
         }
@@ -388,12 +395,17 @@ QDict *qtest_qmp_receive(QTestState *s)
     return qmp.response;
 }
 
+QDict *qtest_qmp_receive(QTestState *s)
+{
+    return qmp_fd_receive(s->qmp_fd);
+}
+
 /**
  * Allow users to send a message without waiting for the reply,
  * in the case that they choose to discard all replies up until
  * a particular EVENT is received.
  */
-void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap)
 {
     va_list ap_copy;
     QObject *qobj;
@@ -417,13 +429,25 @@ void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
             fprintf(stderr, "%s", str);
         }
         /* Send QMP request */
-        socket_send(s->qmp_fd, str, size);
+        socket_send(fd, str, size);
 
         QDECREF(qstr);
         qobject_decref(qobj);
     }
 }
 
+void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(s->qmp_fd, fmt, ap);
+}
+
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(fd, fmt, ap);
+
+    return qmp_fd_receive(fd);
+}
+
 QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
 {
     qtest_async_qmpv(s, fmt, ap);
@@ -432,6 +456,26 @@ QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
     return qtest_qmp_receive(s);
 }
 
+QDict *qmp_fd(int fd, const char *fmt, ...)
+{
+    va_list ap;
+    QDict *response;
+
+    va_start(ap, fmt);
+    response = qmp_fdv(fd, fmt, ap);
+    va_end(ap);
+    return response;
+}
+
+void qmp_fd_send(int fd, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qmp_fd_sendv(fd, fmt, ap);
+    va_end(ap);
+}
+
 QDict *qtest_qmp(QTestState *s, const char *fmt, ...)
 {
     va_list ap;
@@ -484,6 +528,33 @@ void qtest_qmp_eventwait(QTestState *s, const char *event)
     }
 }
 
+char *qtest_hmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    char *cmd;
+    QDict *resp;
+    char *ret;
+
+    cmd = g_strdup_vprintf(fmt, ap);
+    resp = qtest_qmp(s, "{'execute': 'human-monitor-command',"
+                     " 'arguments': {'command-line': %s}}",
+                     cmd);
+    ret = g_strdup(qdict_get_try_str(resp, "return"));
+    g_assert(ret);
+    QDECREF(resp);
+    g_free(cmd);
+    return ret;
+}
+
+char *qtest_hmp(QTestState *s, const char *fmt, ...)
+{
+    va_list ap;
+    char *ret;
+
+    va_start(ap, fmt);
+    ret = qtest_hmpv(s, fmt, ap);
+    va_end(ap);
+    return ret;
+}
 
 const char *qtest_get_arch(void)
 {
@@ -681,14 +752,15 @@ void qtest_memread(QTestState *s, uint64_t addr, void *data, size_t size)
     g_strfreev(args);
 }
 
-void qtest_add_func(const char *str, void (*fn))
+void qtest_add_func(const char *str, void (*fn)(void))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_func(path, fn);
     g_free(path);
 }
 
-void qtest_add_data_func(const char *str, const void *data, void (*fn))
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_data_func(path, data, fn);
@@ -775,6 +847,16 @@ void qmp_discard_response(const char *fmt, ...)
     qtest_qmpv_discard_response(global_qtest, fmt, ap);
     va_end(ap);
 }
+char *hmp(const char *fmt, ...)
+{
+    va_list ap;
+    char *ret;
+
+    va_start(ap, fmt);
+    ret = qtest_hmpv(global_qtest, fmt, ap);
+    va_end(ap);
+    return ret;
+}
 
 bool qtest_big_endian(void)
 {
diff --git a/qemu/tests/libqtest.h b/qemu/tests/libqtest.h
index ec4203152..37f37adbf 100644
--- a/qemu/tests/libqtest.h
+++ b/qemu/tests/libqtest.h
@@ -17,13 +17,7 @@
 #ifndef LIBQTEST_H
 #define LIBQTEST_H
 
-#include <stddef.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdarg.h>
-#include <sys/types.h>
 #include "qapi/qmp/qdict.h"
-#include "glib-compat.h"
 
 typedef struct QTestState QTestState;
 
@@ -120,6 +114,29 @@ QDict *qtest_qmp_receive(QTestState *s);
 void qtest_qmp_eventwait(QTestState *s, const char *event);
 
 /**
+ * qtest_hmpv:
+ * @s: #QTestState instance to operate on.
+ * @fmt...: HMP command to send to QEMU
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *qtest_hmp(QTestState *s, const char *fmt, ...);
+
+/**
+ * qtest_hmpv:
+ * @s: #QTestState instance to operate on.
+ * @fmt: HMP command to send to QEMU
+ * @ap: HMP command arguments
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *qtest_hmpv(QTestState *s, const char *fmt, va_list ap);
+
+/**
  * qtest_get_irq:
  * @s: #QTestState instance to operate on.
  * @num: Interrupt to observe.
@@ -393,7 +410,7 @@ const char *qtest_get_arch(void);
  * The path is prefixed with the architecture under test, as
  * returned by qtest_get_arch().
  */
-void qtest_add_func(const char *str, void (*fn));
+void qtest_add_func(const char *str, void (*fn)(void));
 
 /**
  * qtest_add_data_func:
@@ -405,7 +422,8 @@ void qtest_add_func(const char *str, void (*fn));
  * The path is prefixed with the architecture under test, as
  * returned by qtest_get_arch().
  */
-void qtest_add_data_func(const char *str, const void *data, void (*fn));
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *));
 
 /**
  * qtest_add:
@@ -427,6 +445,8 @@ void qtest_add_data_func(const char *str, const void *data, void (*fn));
         g_free(path); \
     } while (0)
 
+void qtest_add_abrt_handler(GHookFunc fn, const void *data);
+
 /**
  * qtest_start:
  * @args: other arguments to pass to QEMU
@@ -499,6 +519,16 @@ static inline void qmp_eventwait(const char *event)
 }
 
 /**
+ * hmp:
+ * @fmt...: HMP command to send to QEMU
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *hmp(const char *fmt, ...);
+
+/**
  * get_irq:
  * @num: Interrupt to observe.
  *
@@ -818,4 +848,11 @@ static inline int64_t clock_set(int64_t val)
  */
 bool qtest_big_endian(void);
 
+
+QDict *qmp_fd_receive(int fd);
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap);
+void qmp_fd_send(int fd, const char *fmt, ...);
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap);
+QDict *qmp_fd(int fd, const char *fmt, ...);
+
 #endif
diff --git a/qemu/tests/m48t59-test.c b/qemu/tests/m48t59-test.c
index 71b4f2805..a751fd350 100644
--- a/qemu/tests/m48t59-test.c
+++ b/qemu/tests/m48t59-test.c
@@ -12,11 +12,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
 
 #include "libqtest.h"
 
diff --git a/qemu/tests/ne2000-test.c b/qemu/tests/ne2000-test.c
index 61a678ad3..3727875f2 100644
--- a/qemu/tests/ne2000-test.c
+++ b/qemu/tests/ne2000-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/qemu/tests/nvme-test.c b/qemu/tests/nvme-test.c
index ff38b5e48..ec06893ee 100644
--- a/qemu/tests/nvme-test.c
+++ b/qemu/tests/nvme-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/pc-cpu-test.c b/qemu/tests/pc-cpu-test.c
index 3505c7c43..6b34ca588 100644
--- a/qemu/tests/pc-cpu-test.c
+++ b/qemu/tests/pc-cpu-test.c
@@ -7,12 +7,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 
 #include "qemu-common.h"
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "qapi/qmp/types.h"
 
 struct PCTestData {
diff --git a/qemu/tests/pcnet-test.c b/qemu/tests/pcnet-test.c
index 84af4f327..2ddf4965c 100644
--- a/qemu/tests/pcnet-test.c
+++ b/qemu/tests/pcnet-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/qemu/tests/pkix_asn1_tab.c b/qemu/tests/pkix_asn1_tab.c
new file mode 100644
index 000000000..903bc0251
--- /dev/null
+++ b/qemu/tests/pkix_asn1_tab.c
@@ -0,0 +1,1105 @@
+/*
+ * This file is taken from gnutls 1.6.3 under the GPLv2+
+ * and is under copyright of various GNUTLS contributors.
+ */
+
+#include "qemu/osdep.h"
+#include <libtasn1.h>
+
+const ASN1_ARRAY_TYPE pkix_asn1_tab[] = {
+  {"PKIX1", 536875024, 0},
+  {0, 1073741836, 0},
+  {"id-ce", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "29"},
+  {"id-ce-authorityKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "35"},
+  {"AuthorityKeyIdentifier", 1610612741, 0},
+  {"keyIdentifier", 1610637314, "KeyIdentifier"},
+  {0, 4104, "0"},
+  {"authorityCertIssuer", 1610637314, "GeneralNames"},
+  {0, 4104, "1"},
+  {"authorityCertSerialNumber", 536895490, "CertificateSerialNumber"},
+  {0, 4104, "2"},
+  {"KeyIdentifier", 1073741831, 0},
+  {"id-ce-subjectKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "14"},
+  {"SubjectKeyIdentifier", 1073741826, "KeyIdentifier"},
+  {"id-ce-keyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "15"},
+  {"KeyUsage", 1610874886, 0},
+  {"digitalSignature", 1073741825, "0"},
+  {"nonRepudiation", 1073741825, "1"},
+  {"keyEncipherment", 1073741825, "2"},
+  {"dataEncipherment", 1073741825, "3"},
+  {"keyAgreement", 1073741825, "4"},
+  {"keyCertSign", 1073741825, "5"},
+  {"cRLSign", 1073741825, "6"},
+  {"encipherOnly", 1073741825, "7"},
+  {"decipherOnly", 1, "8"},
+  {"id-ce-privateKeyUsagePeriod", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "16"},
+  {"PrivateKeyUsagePeriod", 1610612741, 0},
+  {"notBefore", 1619025937, 0},
+  {0, 4104, "0"},
+  {"notAfter", 545284113, 0},
+  {0, 4104, "1"},
+  {"id-ce-certificatePolicies", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "32"},
+  {"CertificatePolicies", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyInformation"},
+  {"PolicyInformation", 1610612741, 0},
+  {"policyIdentifier", 1073741826, "CertPolicyId"},
+  {"policyQualifiers", 538984459, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyQualifierInfo"},
+  {"CertPolicyId", 1073741836, 0},
+  {"PolicyQualifierInfo", 1610612741, 0},
+  {"policyQualifierId", 1073741826, "PolicyQualifierId"},
+  {"qualifier", 541065229, 0},
+  {"policyQualifierId", 1, 0},
+  {"PolicyQualifierId", 1073741836, 0},
+  {"CPSuri", 1073741826, "IA5String"},
+  {"UserNotice", 1610612741, 0},
+  {"noticeRef", 1073758210, "NoticeReference"},
+  {"explicitText", 16386, "DisplayText"},
+  {"NoticeReference", 1610612741, 0},
+  {"organization", 1073741826, "DisplayText"},
+  {"noticeNumbers", 536870923, 0},
+  {0, 3, 0},
+  {"DisplayText", 1610612754, 0},
+  {"visibleString", 1612709890, "VisibleString"},
+  {"200", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"200", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"200", 524298, "1"},
+  {"id-ce-policyMappings", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "33"},
+  {"PolicyMappings", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 536870917, 0},
+  {"issuerDomainPolicy", 1073741826, "CertPolicyId"},
+  {"subjectDomainPolicy", 2, "CertPolicyId"},
+  {"DirectoryString", 1610612754, 0},
+  {"teletexString", 1612709890, "TeletexString"},
+  {"MAX", 524298, "1"},
+  {"printableString", 1612709890, "PrintableString"},
+  {"MAX", 524298, "1"},
+  {"universalString", 1612709890, "UniversalString"},
+  {"MAX", 524298, "1"},
+  {"utf8String", 1612709890, "UTF8String"},
+  {"MAX", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"MAX", 524298, "1"},
+  {"ia5String", 538968066, "IA5String"},
+  {"MAX", 524298, "1"},
+  {"id-ce-subjectAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "17"},
+  {"SubjectAltName", 1073741826, "GeneralNames"},
+  {"GeneralNames", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralName"},
+  {"GeneralName", 1610612754, 0},
+  {"otherName", 1610620930, "AnotherName"},
+  {0, 4104, "0"},
+  {"rfc822Name", 1610620930, "IA5String"},
+  {0, 4104, "1"},
+  {"dNSName", 1610620930, "IA5String"},
+  {0, 4104, "2"},
+  {"x400Address", 1610620930, "ORAddress"},
+  {0, 4104, "3"},
+  {"directoryName", 1610620930, "RDNSequence"},
+  {0, 2056, "4"},
+  {"ediPartyName", 1610620930, "EDIPartyName"},
+  {0, 4104, "5"},
+  {"uniformResourceIdentifier", 1610620930, "IA5String"},
+  {0, 4104, "6"},
+  {"iPAddress", 1610620935, 0},
+  {0, 4104, "7"},
+  {"registeredID", 536879116, 0},
+  {0, 4104, "8"},
+  {"AnotherName", 1610612741, 0},
+  {"type-id", 1073741836, 0},
+  {"value", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"type-id", 1, 0},
+  {"EDIPartyName", 1610612741, 0},
+  {"nameAssigner", 1610637314, "DirectoryString"},
+  {0, 4104, "0"},
+  {"partyName", 536879106, "DirectoryString"},
+  {0, 4104, "1"},
+  {"id-ce-issuerAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "18"},
+  {"IssuerAltName", 1073741826, "GeneralNames"},
+  {"id-ce-subjectDirectoryAttributes", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "9"},
+  {"SubjectDirectoryAttributes", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-ce-basicConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "19"},
+  {"BasicConstraints", 1610612741, 0},
+  {"cA", 1610645508, 0},
+  {0, 131081, 0},
+  {"pathLenConstraint", 537411587, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-nameConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "30"},
+  {"NameConstraints", 1610612741, 0},
+  {"permittedSubtrees", 1610637314, "GeneralSubtrees"},
+  {0, 4104, "0"},
+  {"excludedSubtrees", 536895490, "GeneralSubtrees"},
+  {0, 4104, "1"},
+  {"GeneralSubtrees", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralSubtree"},
+  {"GeneralSubtree", 1610612741, 0},
+  {"base", 1073741826, "GeneralName"},
+  {"minimum", 1610653698, "BaseDistance"},
+  {0, 1073741833, "0"},
+  {0, 4104, "0"},
+  {"maximum", 536895490, "BaseDistance"},
+  {0, 4104, "1"},
+  {"BaseDistance", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-policyConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "36"},
+  {"PolicyConstraints", 1610612741, 0},
+  {"requireExplicitPolicy", 1610637314, "SkipCerts"},
+  {0, 4104, "0"},
+  {"inhibitPolicyMapping", 536895490, "SkipCerts"},
+  {0, 4104, "1"},
+  {"SkipCerts", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-cRLDistributionPoints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "31"},
+  {"CRLDistributionPoints", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "DistributionPoint"},
+  {"DistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 2056, "0"},
+  {"reasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "1"},
+  {"cRLIssuer", 536895490, "GeneralNames"},
+  {0, 4104, "2"},
+  {"DistributionPointName", 1610612754, 0},
+  {"fullName", 1610620930, "GeneralNames"},
+  {0, 4104, "0"},
+  {"nameRelativeToCRLIssuer", 536879106, "RelativeDistinguishedName"},
+  {0, 4104, "1"},
+  {"ReasonFlags", 1610874886, 0},
+  {"unused", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"privilegeWithdrawn", 1073741825, "7"},
+  {"aACompromise", 1, "8"},
+  {"id-ce-extKeyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "37"},
+  {"ExtKeyUsageSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "KeyPurposeId"},
+  {"KeyPurposeId", 1073741836, 0},
+  {"id-kp-serverAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "1"},
+  {"id-kp-clientAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "2"},
+  {"id-kp-codeSigning", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "3"},
+  {"id-kp-emailProtection", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "4"},
+  {"id-kp-ipsecEndSystem", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "5"},
+  {"id-kp-ipsecTunnel", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "6"},
+  {"id-kp-ipsecUser", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "7"},
+  {"id-kp-timeStamping", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "8"},
+  {"id-pe-authorityInfoAccess", 1879048204, 0},
+  {0, 1073741825, "id-pe"},
+  {0, 1, "1"},
+  {"AuthorityInfoAccessSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AccessDescription"},
+  {"AccessDescription", 1610612741, 0},
+  {"accessMethod", 1073741836, 0},
+  {"accessLocation", 2, "GeneralName"},
+  {"id-ce-cRLNumber", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "20"},
+  {"CRLNumber", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-issuingDistributionPoint", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "28"},
+  {"IssuingDistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 4104, "0"},
+  {"onlyContainsUserCerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "1"},
+  {"onlyContainsCACerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "2"},
+  {"onlySomeReasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "3"},
+  {"indirectCRL", 536911876, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "4"},
+  {"id-ce-deltaCRLIndicator", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "27"},
+  {"BaseCRLNumber", 1073741826, "CRLNumber"},
+  {"id-ce-cRLReasons", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "21"},
+  {"CRLReason", 1610874901, 0},
+  {"unspecified", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"removeFromCRL", 1, "8"},
+  {"id-ce-certificateIssuer", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "29"},
+  {"CertificateIssuer", 1073741826, "GeneralNames"},
+  {"id-ce-holdInstructionCode", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "23"},
+  {"HoldInstructionCode", 1073741836, 0},
+  {"holdInstruction", 1879048204, 0},
+  {"joint-iso-itu-t", 1073741825, "2"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9cm", 1073741825, "10040"},
+  {0, 1, "2"},
+  {"id-holdinstruction-none", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "1"},
+  {"id-holdinstruction-callissuer", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "2"},
+  {"id-holdinstruction-reject", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "3"},
+  {"id-ce-invalidityDate", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "24"},
+  {"InvalidityDate", 1082130449, 0},
+  {"VisibleString", 1610620935, 0},
+  {0, 4360, "26"},
+  {"NumericString", 1610620935, 0},
+  {0, 4360, "18"},
+  {"IA5String", 1610620935, 0},
+  {0, 4360, "22"},
+  {"TeletexString", 1610620935, 0},
+  {0, 4360, "20"},
+  {"PrintableString", 1610620935, 0},
+  {0, 4360, "19"},
+  {"UniversalString", 1610620935, 0},
+  {0, 4360, "28"},
+  {"BMPString", 1610620935, 0},
+  {0, 4360, "30"},
+  {"UTF8String", 1610620935, 0},
+  {0, 4360, "12"},
+  {"id-pkix", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"identified-organization", 1073741825, "3"},
+  {"dod", 1073741825, "6"},
+  {"internet", 1073741825, "1"},
+  {"security", 1073741825, "5"},
+  {"mechanisms", 1073741825, "5"},
+  {"pkix", 1, "7"},
+  {"id-pe", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "1"},
+  {"id-qt", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "2"},
+  {"id-kp", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "3"},
+  {"id-ad", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "48"},
+  {"id-qt-cps", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "1"},
+  {"id-qt-unotice", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "2"},
+  {"id-ad-ocsp", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "1"},
+  {"id-ad-caIssuers", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "2"},
+  {"Attribute", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"values", 536870927, 0},
+  {0, 2, "AttributeValue"},
+  {"AttributeType", 1073741836, 0},
+  {"AttributeValue", 1614807053, 0},
+  {"type", 1, 0},
+  {"AttributeTypeAndValue", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"value", 2, "AttributeValue"},
+  {"id-at", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "4"},
+  {"id-at-initials", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "43"},
+  {"X520initials", 1073741826, "DirectoryString"},
+  {"id-at-generationQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "44"},
+  {"X520generationQualifier", 1073741826, "DirectoryString"},
+  {"id-at-surname", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "4"},
+  {"X520surName", 1073741826, "DirectoryString"},
+  {"id-at-givenName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "42"},
+  {"X520givenName", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-commonName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "3"},
+  {"X520CommonName", 1073741826, "DirectoryString"},
+  {"id-at-localityName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "7"},
+  {"X520LocalityName", 1073741826, "DirectoryString"},
+  {"id-at-stateOrProvinceName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "8"},
+  {"X520StateOrProvinceName", 1073741826, "DirectoryString"},
+  {"id-at-organizationName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "10"},
+  {"X520OrganizationName", 1073741826, "DirectoryString"},
+  {"id-at-organizationalUnitName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "11"},
+  {"X520OrganizationalUnitName", 1073741826, "DirectoryString"},
+  {"id-at-title", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "12"},
+  {"X520Title", 1073741826, "DirectoryString"},
+  {"id-at-description", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "13"},
+  {"X520Description", 1073741826, "DirectoryString"},
+  {"id-at-dnQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "46"},
+  {"X520dnQualifier", 1073741826, "PrintableString"},
+  {"id-at-countryName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "6"},
+  {"X520countryName", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-at-serialNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "5"},
+  {"X520serialNumber", 1073741826, "PrintableString"},
+  {"id-at-telephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "20"},
+  {"X520telephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-facsimileTelephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "23"},
+  {"X520facsimileTelephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-pseudonym", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "65"},
+  {"X520pseudonym", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-streetAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "9"},
+  {"X520streetAddress", 1073741826, "DirectoryString"},
+  {"id-at-postalAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "16"},
+  {"X520postalAddress", 1073741826, "PostalAddress"},
+  {"PostalAddress", 1610612747, 0},
+  {0, 2, "DirectoryString"},
+  {"pkcs", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1, "1"},
+  {"pkcs-9", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "9"},
+  {"emailAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "1"},
+  {"Pkcs9email", 1612709890, "IA5String"},
+  {"ub-emailaddress-length", 524298, "1"},
+  {"Name", 1610612754, 0},
+  {"rdnSequence", 2, "RDNSequence"},
+  {"RDNSequence", 1610612747, 0},
+  {0, 2, "RelativeDistinguishedName"},
+  {"DistinguishedName", 1073741826, "RDNSequence"},
+  {"RelativeDistinguishedName", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AttributeTypeAndValue"},
+  {"Certificate", 1610612741, 0},
+  {"tbsCertificate", 1073741826, "TBSCertificate"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertificate", 1610612741, 0},
+  {"version", 1610653698, "Version"},
+  {0, 1073741833, "v1"},
+  {0, 2056, "0"},
+  {"serialNumber", 1073741826, "CertificateSerialNumber"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"validity", 1073741826, "Validity"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPublicKeyInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"issuerUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "1"},
+  {"subjectUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "2"},
+  {"extensions", 536895490, "Extensions"},
+  {0, 2056, "3"},
+  {"Version", 1610874883, 0},
+  {"v1", 1073741825, "0"},
+  {"v2", 1073741825, "1"},
+  {"v3", 1, "2"},
+  {"CertificateSerialNumber", 1073741827, 0},
+  {"Validity", 1610612741, 0},
+  {"notBefore", 1073741826, "Time"},
+  {"notAfter", 2, "Time"},
+  {"Time", 1610612754, 0},
+  {"utcTime", 1090519057, 0},
+  {"generalTime", 8388625, 0},
+  {"UniqueIdentifier", 1073741830, 0},
+  {"SubjectPublicKeyInfo", 1610612741, 0},
+  {"algorithm", 1073741826, "AlgorithmIdentifier"},
+  {"subjectPublicKey", 6, 0},
+  {"Extensions", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Extension"},
+  {"Extension", 1610612741, 0},
+  {"extnID", 1073741836, 0},
+  {"critical", 1610645508, 0},
+  {0, 131081, 0},
+  {"extnValue", 7, 0},
+  {"CertificateList", 1610612741, 0},
+  {"tbsCertList", 1073741826, "TBSCertList"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertList", 1610612741, 0},
+  {"version", 1073758210, "Version"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"thisUpdate", 1073741826, "Time"},
+  {"nextUpdate", 1073758210, "Time"},
+  {"revokedCertificates", 1610629131, 0},
+  {0, 536870917, 0},
+  {"userCertificate", 1073741826, "CertificateSerialNumber"},
+  {"revocationDate", 1073741826, "Time"},
+  {"crlEntryExtensions", 16386, "Extensions"},
+  {"crlExtensions", 536895490, "Extensions"},
+  {0, 2056, "0"},
+  {"AlgorithmIdentifier", 1610612741, 0},
+  {"algorithm", 1073741836, 0},
+  {"parameters", 541081613, 0},
+  {"algorithm", 1, 0},
+  {"pkcs-1", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "1"},
+  {"rsaEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "1"},
+  {"md2WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "2"},
+  {"md5WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "4"},
+  {"sha1WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "5"},
+  {"id-dsa-with-sha1", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "3"},
+  {"Dss-Sig-Value", 1610612741, 0},
+  {"r", 1073741827, 0},
+  {"s", 3, 0},
+  {"dhpublicnumber", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"ansi-x942", 1073741825, "10046"},
+  {"number-type", 1073741825, "2"},
+  {0, 1, "1"},
+  {"DomainParameters", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"g", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"j", 1073758211, 0},
+  {"validationParms", 16386, "ValidationParms"},
+  {"ValidationParms", 1610612741, 0},
+  {"seed", 1073741830, 0},
+  {"pgenCounter", 3, 0},
+  {"id-dsa", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "1"},
+  {"Dss-Parms", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"g", 3, 0},
+  {"ORAddress", 1610612741, 0},
+  {"built-in-standard-attributes", 1073741826, "BuiltInStandardAttributes"},
+  {"built-in-domain-defined-attributes", 1073758210,
+   "BuiltInDomainDefinedAttributes"},
+  {"extension-attributes", 16386, "ExtensionAttributes"},
+  {"BuiltInStandardAttributes", 1610612741, 0},
+  {"country-name", 1073758210, "CountryName"},
+  {"administration-domain-name", 1073758210, "AdministrationDomainName"},
+  {"network-address", 1610637314, "NetworkAddress"},
+  {0, 2056, "0"},
+  {"terminal-identifier", 1610637314, "TerminalIdentifier"},
+  {0, 2056, "1"},
+  {"private-domain-name", 1610637314, "PrivateDomainName"},
+  {0, 2056, "2"},
+  {"organization-name", 1610637314, "OrganizationName"},
+  {0, 2056, "3"},
+  {"numeric-user-identifier", 1610637314, "NumericUserIdentifier"},
+  {0, 2056, "4"},
+  {"personal-name", 1610637314, "PersonalName"},
+  {0, 2056, "5"},
+  {"organizational-unit-names", 536895490, "OrganizationalUnitNames"},
+  {0, 2056, "6"},
+  {"CountryName", 1610620946, 0},
+  {0, 1073746952, "1"},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"AdministrationDomainName", 1610620946, 0},
+  {0, 1073744904, "2"},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"NetworkAddress", 1073741826, "X121Address"},
+  {"X121Address", 1612709890, "NumericString"},
+  {"ub-x121-address-length", 524298, "1"},
+  {"TerminalIdentifier", 1612709890, "PrintableString"},
+  {"ub-terminal-id-length", 524298, "1"},
+  {"PrivateDomainName", 1610612754, 0},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"OrganizationName", 1612709890, "PrintableString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"NumericUserIdentifier", 1612709890, "NumericString"},
+  {"ub-numeric-user-id-length", 524298, "1"},
+  {"PersonalName", 1610612750, 0},
+  {"surname", 1814044674, "PrintableString"},
+  {0, 1073745928, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "PrintableString"},
+  {0, 1073745928, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "PrintableString"},
+  {0, 1073745928, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "PrintableString"},
+  {0, 1073745928, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"OrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "OrganizationalUnitName"},
+  {"OrganizationalUnitName", 1612709890, "PrintableString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"BuiltInDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "BuiltInDomainDefinedAttribute"},
+  {"BuiltInDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "PrintableString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "PrintableString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ExtensionAttributes", 1612709903, 0},
+  {"ub-extension-attributes", 1074266122, "1"},
+  {0, 2, "ExtensionAttribute"},
+  {"ExtensionAttribute", 1610612741, 0},
+  {"extension-attribute-type", 1611145219, 0},
+  {0, 1073743880, "0"},
+  {"0", 10, "ub-extension-attributes"},
+  {"extension-attribute-value", 541073421, 0},
+  {0, 1073743880, "1"},
+  {"extension-attribute-type", 1, 0},
+  {"common-name", 1342177283, "1"},
+  {"CommonName", 1612709890, "PrintableString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-common-name", 1342177283, "2"},
+  {"TeletexCommonName", 1612709890, "TeletexString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-organization-name", 1342177283, "3"},
+  {"TeletexOrganizationName", 1612709890, "TeletexString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"teletex-personal-name", 1342177283, "4"},
+  {"TeletexPersonalName", 1610612750, 0},
+  {"surname", 1814044674, "TeletexString"},
+  {0, 1073743880, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "TeletexString"},
+  {0, 1073743880, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "TeletexString"},
+  {0, 1073743880, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "TeletexString"},
+  {0, 1073743880, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"teletex-organizational-unit-names", 1342177283, "5"},
+  {"TeletexOrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "TeletexOrganizationalUnitName"},
+  {"TeletexOrganizationalUnitName", 1612709890, "TeletexString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"pds-name", 1342177283, "7"},
+  {"PDSName", 1612709890, "PrintableString"},
+  {"ub-pds-name-length", 524298, "1"},
+  {"physical-delivery-country-name", 1342177283, "8"},
+  {"PhysicalDeliveryCountryName", 1610612754, 0},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"postal-code", 1342177283, "9"},
+  {"PostalCode", 1610612754, 0},
+  {"numeric-code", 1612709890, "NumericString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"printable-code", 538968066, "PrintableString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"physical-delivery-office-name", 1342177283, "10"},
+  {"PhysicalDeliveryOfficeName", 1073741826, "PDSParameter"},
+  {"physical-delivery-office-number", 1342177283, "11"},
+  {"PhysicalDeliveryOfficeNumber", 1073741826, "PDSParameter"},
+  {"extension-OR-address-components", 1342177283, "12"},
+  {"ExtensionORAddressComponents", 1073741826, "PDSParameter"},
+  {"physical-delivery-personal-name", 1342177283, "13"},
+  {"PhysicalDeliveryPersonalName", 1073741826, "PDSParameter"},
+  {"physical-delivery-organization-name", 1342177283, "14"},
+  {"PhysicalDeliveryOrganizationName", 1073741826, "PDSParameter"},
+  {"extension-physical-delivery-address-components", 1342177283, "15"},
+  {"ExtensionPhysicalDeliveryAddressComponents", 1073741826, "PDSParameter"},
+  {"unformatted-postal-address", 1342177283, "16"},
+  {"UnformattedPostalAddress", 1610612750, 0},
+  {"printable-address", 1814052875, 0},
+  {"ub-pds-physical-address-lines", 1074266122, "1"},
+  {0, 538968066, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-unformatted-address-length", 524298, "1"},
+  {"street-address", 1342177283, "17"},
+  {"StreetAddress", 1073741826, "PDSParameter"},
+  {"post-office-box-address", 1342177283, "18"},
+  {"PostOfficeBoxAddress", 1073741826, "PDSParameter"},
+  {"poste-restante-address", 1342177283, "19"},
+  {"PosteRestanteAddress", 1073741826, "PDSParameter"},
+  {"unique-postal-name", 1342177283, "20"},
+  {"UniquePostalName", 1073741826, "PDSParameter"},
+  {"local-postal-attributes", 1342177283, "21"},
+  {"LocalPostalAttributes", 1073741826, "PDSParameter"},
+  {"PDSParameter", 1610612750, 0},
+  {"printable-string", 1814052866, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"extended-network-address", 1342177283, "22"},
+  {"ExtendedNetworkAddress", 1610612754, 0},
+  {"e163-4-address", 1610612741, 0},
+  {"number", 1612718082, "NumericString"},
+  {0, 1073743880, "0"},
+  {"ub-e163-4-number-length", 524298, "1"},
+  {"sub-address", 538992642, "NumericString"},
+  {0, 1073743880, "1"},
+  {"ub-e163-4-sub-address-length", 524298, "1"},
+  {"psap-address", 536879106, "PresentationAddress"},
+  {0, 2056, "0"},
+  {"PresentationAddress", 1610612741, 0},
+  {"pSelector", 1610637319, 0},
+  {0, 2056, "0"},
+  {"sSelector", 1610637319, 0},
+  {0, 2056, "1"},
+  {"tSelector", 1610637319, 0},
+  {0, 2056, "2"},
+  {"nAddresses", 538976271, 0},
+  {0, 1073743880, "3"},
+  {"MAX", 1074266122, "1"},
+  {0, 7, 0},
+  {"terminal-type", 1342177283, "23"},
+  {"TerminalType", 1610874883, 0},
+  {"telex", 1073741825, "3"},
+  {"teletex", 1073741825, "4"},
+  {"g3-facsimile", 1073741825, "5"},
+  {"g4-facsimile", 1073741825, "6"},
+  {"ia5-terminal", 1073741825, "7"},
+  {"videotex", 1, "8"},
+  {"teletex-domain-defined-attributes", 1342177283, "6"},
+  {"TeletexDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "TeletexDomainDefinedAttribute"},
+  {"TeletexDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "TeletexString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "TeletexString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ub-name", 1342177283, "32768"},
+  {"ub-common-name", 1342177283, "64"},
+  {"ub-locality-name", 1342177283, "128"},
+  {"ub-state-name", 1342177283, "128"},
+  {"ub-organization-name", 1342177283, "64"},
+  {"ub-organizational-unit-name", 1342177283, "64"},
+  {"ub-title", 1342177283, "64"},
+  {"ub-match", 1342177283, "128"},
+  {"ub-emailaddress-length", 1342177283, "128"},
+  {"ub-common-name-length", 1342177283, "64"},
+  {"ub-country-name-alpha-length", 1342177283, "2"},
+  {"ub-country-name-numeric-length", 1342177283, "3"},
+  {"ub-domain-defined-attributes", 1342177283, "4"},
+  {"ub-domain-defined-attribute-type-length", 1342177283, "8"},
+  {"ub-domain-defined-attribute-value-length", 1342177283, "128"},
+  {"ub-domain-name-length", 1342177283, "16"},
+  {"ub-extension-attributes", 1342177283, "256"},
+  {"ub-e163-4-number-length", 1342177283, "15"},
+  {"ub-e163-4-sub-address-length", 1342177283, "40"},
+  {"ub-generation-qualifier-length", 1342177283, "3"},
+  {"ub-given-name-length", 1342177283, "16"},
+  {"ub-initials-length", 1342177283, "5"},
+  {"ub-integer-options", 1342177283, "256"},
+  {"ub-numeric-user-id-length", 1342177283, "32"},
+  {"ub-organization-name-length", 1342177283, "64"},
+  {"ub-organizational-unit-name-length", 1342177283, "32"},
+  {"ub-organizational-units", 1342177283, "4"},
+  {"ub-pds-name-length", 1342177283, "16"},
+  {"ub-pds-parameter-length", 1342177283, "30"},
+  {"ub-pds-physical-address-lines", 1342177283, "6"},
+  {"ub-postal-code-length", 1342177283, "16"},
+  {"ub-surname-length", 1342177283, "40"},
+  {"ub-terminal-id-length", 1342177283, "24"},
+  {"ub-unformatted-address-length", 1342177283, "180"},
+  {"ub-x121-address-length", 1342177283, "16"},
+  {"pkcs-7-ContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"content", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"contentType", 1, 0},
+  {"pkcs-7-DigestInfo", 1610612741, 0},
+  {"digestAlgorithm", 1073741826, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"digest", 2, "pkcs-7-Digest"},
+  {"pkcs-7-Digest", 1073741831, 0},
+  {"pkcs-7-ContentType", 1073741836, 0},
+  {"pkcs-7-SignedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"digestAlgorithms", 1073741826, "pkcs-7-DigestAlgorithmIdentifiers"},
+  {"encapContentInfo", 1073741826, "pkcs-7-EncapsulatedContentInfo"},
+  {"certificates", 1610637314, "pkcs-7-CertificateSet"},
+  {0, 4104, "0"},
+  {"crls", 1610637314, "pkcs-7-CertificateRevocationLists"},
+  {0, 4104, "1"},
+  {"signerInfos", 2, "pkcs-7-SignerInfos"},
+  {"pkcs-7-CMSVersion", 1610874883, 0},
+  {"v0", 1073741825, "0"},
+  {"v1", 1073741825, "1"},
+  {"v2", 1073741825, "2"},
+  {"v3", 1073741825, "3"},
+  {"v4", 1, "4"},
+  {"pkcs-7-DigestAlgorithmIdentifiers", 1610612751, 0},
+  {0, 2, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"pkcs-7-DigestAlgorithmIdentifier", 1073741826, "AlgorithmIdentifier"},
+  {"pkcs-7-EncapsulatedContentInfo", 1610612741, 0},
+  {"eContentType", 1073741826, "pkcs-7-ContentType"},
+  {"eContent", 536895495, 0},
+  {0, 2056, "0"},
+  {"pkcs-7-CertificateRevocationLists", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-7-CertificateChoices", 1610612754, 0},
+  {"certificate", 13, 0},
+  {"pkcs-7-CertificateSet", 1610612751, 0},
+  {0, 2, "pkcs-7-CertificateChoices"},
+  {"pkcs-7-SignerInfos", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-10-CertificationRequestInfo", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPKInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"attributes", 536879106, "Attributes"},
+  {0, 4104, "0"},
+  {"Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-10-CertificationRequest", 1610612741, 0},
+  {"certificationRequestInfo", 1073741826, "pkcs-10-CertificationRequestInfo"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"pkcs-9-ub-challengePassword", 1342177283, "255"},
+  {"pkcs-9-certTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "22"},
+  {"pkcs-9-crlTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "23"},
+  {"pkcs-9-at-challengePassword", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "7"},
+  {"pkcs-9-challengePassword", 1610612754, 0},
+  {"printableString", 1612709890, "PrintableString"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"pkcs-9-at-localKeyId", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "21"},
+  {"pkcs-9-localKeyId", 1073741831, 0},
+  {"pkcs-9-at-friendlyName", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "20"},
+  {"pkcs-9-friendlyName", 1612709890, "BMPString"},
+  {"255", 524298, "1"},
+  {"pkcs-8-PrivateKeyInfo", 1610612741, 0},
+  {"version", 1073741826, "pkcs-8-Version"},
+  {"privateKeyAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"privateKey", 1073741826, "pkcs-8-PrivateKey"},
+  {"attributes", 536895490, "Attributes"},
+  {0, 4104, "0"},
+  {"pkcs-8-Version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"pkcs-8-PrivateKey", 1073741831, 0},
+  {"pkcs-8-Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-8-EncryptedPrivateKeyInfo", 1610612741, 0},
+  {"encryptionAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"encryptedData", 2, "pkcs-8-EncryptedData"},
+  {"pkcs-8-EncryptedData", 1073741831, 0},
+  {"pkcs-5", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "5"},
+  {"pkcs-5-encryptionAlgorithm", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {0, 1, "3"},
+  {"pkcs-5-des-EDE3-CBC", 1879048204, 0},
+  {0, 1073741825, "pkcs-5-encryptionAlgorithm"},
+  {0, 1, "7"},
+  {"pkcs-5-des-EDE3-CBC-params", 1612709895, 0},
+  {0, 1048586, "8"},
+  {"pkcs-5-id-PBES2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "13"},
+  {"pkcs-5-PBES2-params", 1610612741, 0},
+  {"keyDerivationFunc", 1073741826, "AlgorithmIdentifier"},
+  {"encryptionScheme", 2, "AlgorithmIdentifier"},
+  {"pkcs-5-id-PBKDF2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "12"},
+  {"pkcs-5-PBKDF2-params", 1610612741, 0},
+  {"salt", 1610612754, 0},
+  {"specified", 1073741831, 0},
+  {"otherSource", 2, "AlgorithmIdentifier"},
+  {"iterationCount", 1611137027, 0},
+  {"1", 10, "MAX"},
+  {"keyLength", 1611153411, 0},
+  {"1", 10, "MAX"},
+  {"prf", 16386, "AlgorithmIdentifier"},
+  {"pkcs-12", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "12"},
+  {"pkcs-12-PFX", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v3", 1, "3"},
+  {"authSafe", 1073741826, "pkcs-7-ContentInfo"},
+  {"macData", 16386, "pkcs-12-MacData"},
+  {"pkcs-12-PbeParams", 1610612741, 0},
+  {"salt", 1073741831, 0},
+  {"iterations", 3, 0},
+  {"pkcs-12-MacData", 1610612741, 0},
+  {"mac", 1073741826, "pkcs-7-DigestInfo"},
+  {"macSalt", 1073741831, 0},
+  {"iterations", 536903683, 0},
+  {0, 9, "1"},
+  {"pkcs-12-AuthenticatedSafe", 1610612747, 0},
+  {0, 2, "pkcs-7-ContentInfo"},
+  {"pkcs-12-SafeContents", 1610612747, 0},
+  {0, 2, "pkcs-12-SafeBag"},
+  {"pkcs-12-SafeBag", 1610612741, 0},
+  {"bagId", 1073741836, 0},
+  {"bagValue", 1614815245, 0},
+  {0, 1073743880, "0"},
+  {"badId", 1, 0},
+  {"bagAttributes", 536887311, 0},
+  {0, 2, "pkcs-12-PKCS12Attribute"},
+  {"pkcs-12-bagtypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-12"},
+  {0, 1073741825, "10"},
+  {0, 1, "1"},
+  {"pkcs-12-keyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "1"},
+  {"pkcs-12-pkcs8ShroudedKeyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "2"},
+  {"pkcs-12-certBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "3"},
+  {"pkcs-12-crlBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "4"},
+  {"pkcs-12-KeyBag", 1073741826, "pkcs-8-PrivateKeyInfo"},
+  {"pkcs-12-PKCS8ShroudedKeyBag", 1073741826, "pkcs-8-EncryptedPrivateKeyInfo"},
+  {"pkcs-12-CertBag", 1610612741, 0},
+  {"certId", 1073741836, 0},
+  {"certValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"certId", 1, 0},
+  {"pkcs-12-CRLBag", 1610612741, 0},
+  {"crlId", 1073741836, 0},
+  {"crlValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"crlId", 1, 0},
+  {"pkcs-12-PKCS12Attribute", 1073741826, "Attribute"},
+  {"pkcs-7-data", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "1"},
+  {"pkcs-7-encryptedData", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "6"},
+  {"pkcs-7-Data", 1073741831, 0},
+  {"pkcs-7-EncryptedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"encryptedContentInfo", 1073741826, "pkcs-7-EncryptedContentInfo"},
+  {"unprotectedAttrs", 536895490, "pkcs-7-UnprotectedAttributes"},
+  {0, 4104, "1"},
+  {"pkcs-7-EncryptedContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"contentEncryptionAlgorithm", 1073741826,
+   "pkcs-7-ContentEncryptionAlgorithmIdentifier"},
+  {"encryptedContent", 536895490, "pkcs-7-EncryptedContent"},
+  {0, 4104, "0"},
+  {"pkcs-7-ContentEncryptionAlgorithmIdentifier", 1073741826,
+   "AlgorithmIdentifier"},
+  {"pkcs-7-EncryptedContent", 1073741831, 0},
+  {"pkcs-7-UnprotectedAttributes", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-at-ldap-DC", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "25"},
+  {"ldap-DC", 1073741826, "IA5String"},
+  {"id-at-ldap-UID", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "1"},
+  {"ldap-UID", 1073741826, "DirectoryString"},
+  {"id-pda", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "9"},
+  {"id-pda-dateOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "1"},
+  {"DateOfBirth", 1082130449, 0},
+  {"id-pda-placeOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "2"},
+  {"PlaceOfBirth", 1073741826, "DirectoryString"},
+  {"id-pda-gender", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "3"},
+  {"Gender", 1612709890, "PrintableString"},
+  {0, 1048586, "1"},
+  {"id-pda-countryOfCitizenship", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "4"},
+  {"CountryOfCitizenship", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-pda-countryOfResidence", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "5"},
+  {"CountryOfResidence", 538968066, "PrintableString"},
+  {0, 1048586, "2"},
+  {0, 0, 0}
+};
diff --git a/qemu/tests/pvpanic-test.c b/qemu/tests/pvpanic-test.c
index a7ad6b306..d435833f7 100644
--- a/qemu/tests/pvpanic-test.c
+++ b/qemu/tests/pvpanic-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 static void test_panic(void)
 {
diff --git a/qemu/tests/pxe-test.c b/qemu/tests/pxe-test.c
new file mode 100644
index 000000000..875e4c4a2
--- /dev/null
+++ b/qemu/tests/pxe-test.c
@@ -0,0 +1,68 @@
+/*
+ * PXE test cases.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>,
+ *  Victor Kaplansky <victork@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include <glib/gstdio.h>
+#include "qemu-common.h"
+#include "libqtest.h"
+#include "boot-sector.h"
+
+#define NETNAME "net0"
+
+static const char *disk = "tests/pxe-test-disk.raw";
+
+static void test_pxe_one(const char *params)
+{
+    char *args;
+
+    args = g_strdup_printf("-machine accel=tcg "
+                           "-netdev user,id=" NETNAME ",tftp=./,bootfile=%s "
+                           "%s ",
+                           disk, params);
+
+    qtest_start(args);
+    boot_sector_test();
+    qtest_quit(global_qtest);
+    g_free(args);
+}
+
+static void test_pxe_e1000(void)
+{
+    test_pxe_one("-device e1000,netdev=" NETNAME);
+}
+
+static void test_pxe_virtio_pci(void)
+{
+    test_pxe_one("-device virtio-net-pci,netdev=" NETNAME);
+}
+
+int main(int argc, char *argv[])
+{
+    int ret;
+    const char *arch = qtest_get_arch();
+
+    ret = boot_sector_init(disk);
+    if(ret)
+        return ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
+        qtest_add_func("pxe/e1000", test_pxe_e1000);
+        qtest_add_func("pxe/virtio", test_pxe_virtio_pci);
+    }
+    ret = g_test_run();
+    boot_sector_cleanup(disk);
+    return ret;
+}
diff --git a/qemu/tests/q35-test.c b/qemu/tests/q35-test.c
index 812abe548..a105f1078 100644
--- a/qemu/tests/q35-test.c
+++ b/qemu/tests/q35-test.c
@@ -9,12 +9,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
 #include "libqos/pci.h"
 #include "libqos/pci-pc.h"
-#include "qemu/osdep.h"
 #include "hw/pci-host/q35.h"
 
 static void smram_set_bit(QPCIDevice *pcidev, uint8_t mask, bool enabled)
diff --git a/qemu/tests/qapi-schema/alternate-any.err b/qemu/tests/qapi-schema/alternate-any.err
new file mode 100644
index 000000000..aaa015473
--- /dev/null
+++ b/qemu/tests/qapi-schema/alternate-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-any.json:2: Alternate 'Alt' member 'one' cannot use type 'any'
diff --git a/qemu/tests/qapi-schema/data-array-empty.exit b/qemu/tests/qapi-schema/alternate-any.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-array-empty.exit
+++ b/qemu/tests/qapi-schema/alternate-any.exit
diff --git a/qemu/tests/qapi-schema/alternate-any.json b/qemu/tests/qapi-schema/alternate-any.json
new file mode 100644
index 000000000..e47a73a11
--- /dev/null
+++ b/qemu/tests/qapi-schema/alternate-any.json
@@ -0,0 +1,4 @@
+# we do not allow the 'any' type as an alternate branch
+{ 'alternate': 'Alt',
+  'data': { 'one': 'any',
+            'two': 'int' } }
diff --git a/qemu/tests/qapi-schema/enum-empty.err b/qemu/tests/qapi-schema/alternate-any.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/enum-empty.err
+++ b/qemu/tests/qapi-schema/alternate-any.out
diff --git a/qemu/tests/qapi-schema/alternate-clash.err b/qemu/tests/qapi-schema/alternate-clash.err
index 51bea3e27..604d8495e 100644
--- a/qemu/tests/qapi-schema/alternate-clash.err
+++ b/qemu/tests/qapi-schema/alternate-clash.err
@@ -1 +1 @@
-tests/qapi-schema/alternate-clash.json:2: Alternate 'Alt1' member 'ONE' clashes with 'one'
+tests/qapi-schema/alternate-clash.json:7: 'a_b' (branch of Alt1) collides with 'a-b' (branch of Alt1)
diff --git a/qemu/tests/qapi-schema/alternate-clash.json b/qemu/tests/qapi-schema/alternate-clash.json
index 39479353b..6d73bc527 100644
--- a/qemu/tests/qapi-schema/alternate-clash.json
+++ b/qemu/tests/qapi-schema/alternate-clash.json
@@ -1,3 +1,8 @@
-# we detect C enum collisions in an alternate
+# Alternate branch name collision
+# Reject an alternate that would result in a collision in generated C
+# names (this would try to generate two enum values 'ALT1_KIND_A_B').
+# TODO: In the future, if alternates are simplified to not generate
+# the implicit Alt1Kind enum, we would still have a collision with the
+# resulting C union trying to have two members named 'a_b'.
 { 'alternate': 'Alt1',
-  'data': { 'one': 'str', 'ONE': 'int' } }
+  'data': { 'a-b': 'str', 'a_b': 'int' } }
diff --git a/qemu/tests/qapi-schema/alternate-empty.err b/qemu/tests/qapi-schema/alternate-empty.err
new file mode 100644
index 000000000..bb06c5bfe
--- /dev/null
+++ b/qemu/tests/qapi-schema/alternate-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-empty.json:2: Alternate 'Alt' should have at least two branches in 'data'
diff --git a/qemu/tests/qapi-schema/data-array-unknown.exit b/qemu/tests/qapi-schema/alternate-empty.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-array-unknown.exit
+++ b/qemu/tests/qapi-schema/alternate-empty.exit
diff --git a/qemu/tests/qapi-schema/alternate-empty.json b/qemu/tests/qapi-schema/alternate-empty.json
new file mode 100644
index 000000000..fff15baf1
--- /dev/null
+++ b/qemu/tests/qapi-schema/alternate-empty.json
@@ -0,0 +1,2 @@
+# alternates must list at least two types to be useful
+{ 'alternate': 'Alt', 'data': { 'i': 'int' } }
diff --git a/qemu/tests/qapi-schema/enum-max-member.out b/qemu/tests/qapi-schema/alternate-empty.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/enum-max-member.out
+++ b/qemu/tests/qapi-schema/alternate-empty.out
diff --git a/qemu/tests/qapi-schema/alternate-good.exit b/qemu/tests/qapi-schema/alternate-good.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/alternate-good.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/alternate-good.json b/qemu/tests/qapi-schema/alternate-good.json
deleted file mode 100644
index 33717704c..000000000
--- a/qemu/tests/qapi-schema/alternate-good.json
+++ /dev/null
@@ -1,9 +0,0 @@
-# Working example of alternate
-{ 'struct': 'Data',
-  'data': { '*number': 'int', '*name': 'str' } }
-{ 'enum': 'Enum',
-  'data': [ 'hello', 'world' ] }
-{ 'alternate': 'Alt',
-  'data': { 'value': 'int',
-            'string': 'Enum',
-            'struct': 'Data' } }
diff --git a/qemu/tests/qapi-schema/alternate-good.out b/qemu/tests/qapi-schema/alternate-good.out
deleted file mode 100644
index 99848eefb..000000000
--- a/qemu/tests/qapi-schema/alternate-good.out
+++ /dev/null
@@ -1,6 +0,0 @@
-[OrderedDict([('struct', 'Data'), ('data', OrderedDict([('*number', 'int'), ('*name', 'str')]))]),
- OrderedDict([('enum', 'Enum'), ('data', ['hello', 'world'])]),
- OrderedDict([('alternate', 'Alt'), ('data', OrderedDict([('value', 'int'), ('string', 'Enum'), ('struct', 'Data')]))])]
-[{'enum_name': 'Enum', 'enum_values': ['hello', 'world']},
- {'enum_name': 'AltKind', 'enum_values': None}]
-[OrderedDict([('struct', 'Data'), ('data', OrderedDict([('*number', 'int'), ('*name', 'str')]))])]
diff --git a/qemu/tests/qapi-schema/alternate-nested.json b/qemu/tests/qapi-schema/alternate-nested.json
index c4233b9f3..8e2218649 100644
--- a/qemu/tests/qapi-schema/alternate-nested.json
+++ b/qemu/tests/qapi-schema/alternate-nested.json
@@ -2,4 +2,4 @@
 { 'alternate': 'Alt1',
   'data': { 'name': 'str', 'value': 'int' } }
 { 'alternate': 'Alt2',
-  'data': { 'nested': 'Alt1' } }
+  'data': { 'nested': 'Alt1', 'b': 'bool' } }
diff --git a/qemu/tests/qapi-schema/alternate-unknown.json b/qemu/tests/qapi-schema/alternate-unknown.json
index ad5c10302..08c80dced 100644
--- a/qemu/tests/qapi-schema/alternate-unknown.json
+++ b/qemu/tests/qapi-schema/alternate-unknown.json
@@ -1,3 +1,3 @@
 # we reject an alternate with unknown type in branch
 { 'alternate': 'Alt',
-  'data': { 'unknown': 'MissingType' } }
+  'data': { 'unknown': 'MissingType', 'i': 'int' } }
diff --git a/qemu/tests/qapi-schema/args-alternate.err b/qemu/tests/qapi-schema/args-alternate.err
new file mode 100644
index 000000000..3086eae56
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-alternate.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-alternate.json:3: 'data' for command 'oops' cannot use alternate type 'Alt'
diff --git a/qemu/tests/qapi-schema/data-int.exit b/qemu/tests/qapi-schema/args-alternate.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-int.exit
+++ b/qemu/tests/qapi-schema/args-alternate.exit
diff --git a/qemu/tests/qapi-schema/args-alternate.json b/qemu/tests/qapi-schema/args-alternate.json
new file mode 100644
index 000000000..69e94d481
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-alternate.json
@@ -0,0 +1,3 @@
+# we do not allow alternate arguments
+{ 'alternate': 'Alt', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Alt' }
diff --git a/qemu/tests/qapi-schema/enum-union-clash.out b/qemu/tests/qapi-schema/args-alternate.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/enum-union-clash.out
+++ b/qemu/tests/qapi-schema/args-alternate.out
diff --git a/qemu/tests/qapi-schema/args-any.err b/qemu/tests/qapi-schema/args-any.err
new file mode 100644
index 000000000..bf9b5e073
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-any.json:2: 'data' for command 'oops' cannot use built-in type 'any'
diff --git a/qemu/tests/qapi-schema/data-member-array-bad.exit b/qemu/tests/qapi-schema/args-any.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-member-array-bad.exit
+++ b/qemu/tests/qapi-schema/args-any.exit
diff --git a/qemu/tests/qapi-schema/args-any.json b/qemu/tests/qapi-schema/args-any.json
new file mode 100644
index 000000000..58fe5e470
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-any.json
@@ -0,0 +1,2 @@
+# we do not allow an 'any' argument
+{ 'command': 'oops', 'data': 'any' }
diff --git a/qemu/tests/qapi-schema/event-max.out b/qemu/tests/qapi-schema/args-any.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/event-max.out
+++ b/qemu/tests/qapi-schema/args-any.out
diff --git a/qemu/tests/qapi-schema/data-array-empty.err b/qemu/tests/qapi-schema/args-array-empty.err
index f713f1489..cb7ed33b3 100644
--- a/qemu/tests/qapi-schema/data-array-empty.err
+++ b/qemu/tests/qapi-schema/args-array-empty.err
@@ -1 +1 @@
-tests/qapi-schema/data-array-empty.json:2: Member 'empty' of 'data' for command 'oops': array type must contain single type name
+tests/qapi-schema/args-array-empty.json:2: Member 'empty' of 'data' for command 'oops': array type must contain single type name
diff --git a/qemu/tests/qapi-schema/data-member-unknown.exit b/qemu/tests/qapi-schema/args-array-empty.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-member-unknown.exit
+++ b/qemu/tests/qapi-schema/args-array-empty.exit
diff --git a/qemu/tests/qapi-schema/data-array-empty.json b/qemu/tests/qapi-schema/args-array-empty.json
index 652dcfb24..652dcfb24 100644
--- a/qemu/tests/qapi-schema/data-array-empty.json
+++ b/qemu/tests/qapi-schema/args-array-empty.json
diff --git a/qemu/tests/qapi-schema/flat-union-base-star.out b/qemu/tests/qapi-schema/args-array-empty.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/flat-union-base-star.out
+++ b/qemu/tests/qapi-schema/args-array-empty.out
diff --git a/qemu/tests/qapi-schema/args-array-unknown.err b/qemu/tests/qapi-schema/args-array-unknown.err
new file mode 100644
index 000000000..cd7a0f98d
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-array-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-array-unknown.json:2: Member 'array' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/qemu/tests/qapi-schema/data-unknown.exit b/qemu/tests/qapi-schema/args-array-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/data-unknown.exit
+++ b/qemu/tests/qapi-schema/args-array-unknown.exit
diff --git a/qemu/tests/qapi-schema/data-array-unknown.json b/qemu/tests/qapi-schema/args-array-unknown.json
index 6f3e88331..6f3e88331 100644
--- a/qemu/tests/qapi-schema/data-array-unknown.json
+++ b/qemu/tests/qapi-schema/args-array-unknown.json
diff --git a/qemu/tests/qapi-schema/flat-union-branch-clash.out b/qemu/tests/qapi-schema/args-array-unknown.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/flat-union-branch-clash.out
+++ b/qemu/tests/qapi-schema/args-array-unknown.out
diff --git a/qemu/tests/qapi-schema/args-int.err b/qemu/tests/qapi-schema/args-int.err
new file mode 100644
index 000000000..dc1d2504f
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-int.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-int.json:2: 'data' for command 'oops' cannot use built-in type 'int'
diff --git a/qemu/tests/qapi-schema/enum-max-member.exit b/qemu/tests/qapi-schema/args-int.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/enum-max-member.exit
+++ b/qemu/tests/qapi-schema/args-int.exit
diff --git a/qemu/tests/qapi-schema/data-int.json b/qemu/tests/qapi-schema/args-int.json
index a334d92e8..a334d92e8 100644
--- a/qemu/tests/qapi-schema/data-int.json
+++ b/qemu/tests/qapi-schema/args-int.json
diff --git a/qemu/tests/qapi-schema/flat-union-reverse-define.err b/qemu/tests/qapi-schema/args-int.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/flat-union-reverse-define.err
+++ b/qemu/tests/qapi-schema/args-int.out
diff --git a/qemu/tests/qapi-schema/args-invalid.err b/qemu/tests/qapi-schema/args-invalid.err
new file mode 100644
index 000000000..fe1e94975
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-invalid.json:1: 'data' for command 'foo' should be a dictionary or type name
diff --git a/qemu/tests/qapi-schema/enum-union-clash.exit b/qemu/tests/qapi-schema/args-invalid.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/enum-union-clash.exit
+++ b/qemu/tests/qapi-schema/args-invalid.exit
diff --git a/qemu/tests/qapi-schema/args-invalid.json b/qemu/tests/qapi-schema/args-invalid.json
new file mode 100644
index 000000000..db0981341
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-invalid.json
@@ -0,0 +1,2 @@
+{ 'command': 'foo',
+  'data': false }
diff --git a/qemu/tests/qapi-schema/nested-struct-returns.out b/qemu/tests/qapi-schema/args-invalid.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/nested-struct-returns.out
+++ b/qemu/tests/qapi-schema/args-invalid.out
diff --git a/qemu/tests/qapi-schema/data-member-array-bad.err b/qemu/tests/qapi-schema/args-member-array-bad.err
index 2c072d598..881b4d954 100644
--- a/qemu/tests/qapi-schema/data-member-array-bad.err
+++ b/qemu/tests/qapi-schema/args-member-array-bad.err
@@ -1 +1 @@
-tests/qapi-schema/data-member-array-bad.json:2: Member 'member' of 'data' for command 'oops': array type must contain single type name
+tests/qapi-schema/args-member-array-bad.json:2: Member 'member' of 'data' for command 'oops': array type must contain single type name
diff --git a/qemu/tests/qapi-schema/event-max.exit b/qemu/tests/qapi-schema/args-member-array-bad.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/event-max.exit
+++ b/qemu/tests/qapi-schema/args-member-array-bad.exit
diff --git a/qemu/tests/qapi-schema/data-member-array-bad.json b/qemu/tests/qapi-schema/args-member-array-bad.json
index b2ff144ec..b2ff144ec 100644
--- a/qemu/tests/qapi-schema/data-member-array-bad.json
+++ b/qemu/tests/qapi-schema/args-member-array-bad.json
diff --git a/qemu/tests/qapi-schema/returns-int.err b/qemu/tests/qapi-schema/args-member-array-bad.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/returns-int.err
+++ b/qemu/tests/qapi-schema/args-member-array-bad.out
diff --git a/qemu/tests/qapi-schema/args-member-case.err b/qemu/tests/qapi-schema/args-member-case.err
new file mode 100644
index 000000000..19c442660
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-member-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-case.json:2: 'Arg' (parameter of no-way-this-will-get-whitelisted) should not use uppercase
diff --git a/qemu/tests/qapi-schema/flat-union-base-star.exit b/qemu/tests/qapi-schema/args-member-case.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/flat-union-base-star.exit
+++ b/qemu/tests/qapi-schema/args-member-case.exit
diff --git a/qemu/tests/qapi-schema/args-member-case.json b/qemu/tests/qapi-schema/args-member-case.json
new file mode 100644
index 000000000..93439bee8
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-member-case.json
@@ -0,0 +1,2 @@
+# Member names should be 'lower-case' unless the struct/command is whitelisted
+{ 'command': 'no-way-this-will-get-whitelisted', 'data': { 'Arg': 'int' } }
diff --git a/qemu/tests/qapi-schema/type-bypass-no-gen.out b/qemu/tests/qapi-schema/args-member-case.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/type-bypass-no-gen.out
+++ b/qemu/tests/qapi-schema/args-member-case.out
diff --git a/qemu/tests/qapi-schema/args-member-unknown.err b/qemu/tests/qapi-schema/args-member-unknown.err
new file mode 100644
index 000000000..f6f82828c
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-member-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-unknown.json:2: Member 'member' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/qemu/tests/qapi-schema/flat-union-branch-clash.exit b/qemu/tests/qapi-schema/args-member-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/flat-union-branch-clash.exit
+++ b/qemu/tests/qapi-schema/args-member-unknown.exit
diff --git a/qemu/tests/qapi-schema/data-member-unknown.json b/qemu/tests/qapi-schema/args-member-unknown.json
index 342a41ec9..342a41ec9 100644
--- a/qemu/tests/qapi-schema/data-member-unknown.json
+++ b/qemu/tests/qapi-schema/args-member-unknown.json
diff --git a/qemu/tests/qapi-schema/type-bypass.err b/qemu/tests/qapi-schema/args-member-unknown.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/type-bypass.err
+++ b/qemu/tests/qapi-schema/args-member-unknown.out
diff --git a/qemu/tests/qapi-schema/args-name-clash.err b/qemu/tests/qapi-schema/args-name-clash.err
new file mode 100644
index 000000000..d953e8d24
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-name-clash.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-name-clash.json:4: 'a_b' (parameter of oops) collides with 'a-b' (parameter of oops)
diff --git a/qemu/tests/qapi-schema/nested-struct-returns.exit b/qemu/tests/qapi-schema/args-name-clash.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/nested-struct-returns.exit
+++ b/qemu/tests/qapi-schema/args-name-clash.exit
diff --git a/qemu/tests/qapi-schema/args-name-clash.json b/qemu/tests/qapi-schema/args-name-clash.json
new file mode 100644
index 000000000..61423cb89
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-name-clash.json
@@ -0,0 +1,4 @@
+# C member name collision
+# Reject members that clash when mapped to C names (we would have two 'a_b'
+# members).
+{ 'command': 'oops', 'data': { 'a-b': 'str', 'a_b': 'str' } }
diff --git a/qemu/tests/qapi-schema/union-bad-branch.out b/qemu/tests/qapi-schema/args-name-clash.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/union-bad-branch.out
+++ b/qemu/tests/qapi-schema/args-name-clash.out
diff --git a/qemu/tests/qapi-schema/args-union.err b/qemu/tests/qapi-schema/args-union.err
new file mode 100644
index 000000000..1d693d74d
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-union.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-union.json:4: 'data' for command 'oops' cannot use union type 'Uni'
diff --git a/qemu/tests/qapi-schema/type-bypass-no-gen.exit b/qemu/tests/qapi-schema/args-union.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/type-bypass-no-gen.exit
+++ b/qemu/tests/qapi-schema/args-union.exit
diff --git a/qemu/tests/qapi-schema/args-union.json b/qemu/tests/qapi-schema/args-union.json
new file mode 100644
index 000000000..7bdcbb7f0
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-union.json
@@ -0,0 +1,4 @@
+# we do not allow union arguments
+# TODO should we support this?
+{ 'union': 'Uni', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Uni' }
diff --git a/qemu/tests/qapi-schema/union-max.out b/qemu/tests/qapi-schema/args-union.out
index e69de29bb..e69de29bb 100644
--- a/qemu/tests/qapi-schema/union-max.out
+++ b/qemu/tests/qapi-schema/args-union.out
diff --git a/qemu/tests/qapi-schema/args-unknown.err b/qemu/tests/qapi-schema/args-unknown.err
new file mode 100644
index 000000000..4d91ec869
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-unknown.json:2: 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/qemu/tests/qapi-schema/union-bad-branch.exit b/qemu/tests/qapi-schema/args-unknown.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/union-bad-branch.exit
+++ b/qemu/tests/qapi-schema/args-unknown.exit
diff --git a/qemu/tests/qapi-schema/data-unknown.json b/qemu/tests/qapi-schema/args-unknown.json
index 32aba43b3..32aba43b3 100644
--- a/qemu/tests/qapi-schema/data-unknown.json
+++ b/qemu/tests/qapi-schema/args-unknown.json
diff --git a/qemu/tests/qapi-schema/args-unknown.out b/qemu/tests/qapi-schema/args-unknown.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/args-unknown.out
diff --git a/qemu/tests/qapi-schema/base-cycle-direct.err b/qemu/tests/qapi-schema/base-cycle-direct.err
new file mode 100644
index 000000000..9c68f6543
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-direct.err
@@ -0,0 +1 @@
+tests/qapi-schema/base-cycle-direct.json:2: Object Loopy contains itself
diff --git a/qemu/tests/qapi-schema/union-max.exit b/qemu/tests/qapi-schema/base-cycle-direct.exit
index d00491fd7..d00491fd7 100644
--- a/qemu/tests/qapi-schema/union-max.exit
+++ b/qemu/tests/qapi-schema/base-cycle-direct.exit
diff --git a/qemu/tests/qapi-schema/base-cycle-direct.json b/qemu/tests/qapi-schema/base-cycle-direct.json
new file mode 100644
index 000000000..4fc66d051
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-direct.json
@@ -0,0 +1,2 @@
+# we reject a loop in base classes
+{ 'struct': 'Loopy', 'base': 'Loopy', 'data': {} }
diff --git a/qemu/tests/qapi-schema/base-cycle-direct.out b/qemu/tests/qapi-schema/base-cycle-direct.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-direct.out
diff --git a/qemu/tests/qapi-schema/base-cycle-indirect.err b/qemu/tests/qapi-schema/base-cycle-indirect.err
new file mode 100644
index 000000000..fc92fe47f
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-indirect.err
@@ -0,0 +1 @@
+tests/qapi-schema/base-cycle-indirect.json:2: Object Base1 contains itself
diff --git a/qemu/tests/qapi-schema/base-cycle-indirect.exit b/qemu/tests/qapi-schema/base-cycle-indirect.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-indirect.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/base-cycle-indirect.json b/qemu/tests/qapi-schema/base-cycle-indirect.json
new file mode 100644
index 000000000..28667721a
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-indirect.json
@@ -0,0 +1,3 @@
+# we reject a loop in base classes
+{ 'struct': 'Base1', 'base': 'Base2', 'data': {} }
+{ 'struct': 'Base2', 'base': 'Base1', 'data': {} }
diff --git a/qemu/tests/qapi-schema/base-cycle-indirect.out b/qemu/tests/qapi-schema/base-cycle-indirect.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/base-cycle-indirect.out
diff --git a/qemu/tests/qapi-schema/command-int.json b/qemu/tests/qapi-schema/command-int.json
index c90d408ab..9a62554fc 100644
--- a/qemu/tests/qapi-schema/command-int.json
+++ b/qemu/tests/qapi-schema/command-int.json
@@ -1,3 +1,2 @@
 # we reject collisions between commands and types
-{ 'command': 'int', 'data': { 'character': 'str' },
-  'returns': { 'value': 'int' } }
+{ 'command': 'int', 'data': { 'character': 'str' } }
diff --git a/qemu/tests/qapi-schema/comments.out b/qemu/tests/qapi-schema/comments.out
index 4ce3dcf12..5d7c13cad 100644
--- a/qemu/tests/qapi-schema/comments.out
+++ b/qemu/tests/qapi-schema/comments.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/qemu/tests/qapi-schema/data-array-unknown.err b/qemu/tests/qapi-schema/data-array-unknown.err
deleted file mode 100644
index 8b731bbcc..000000000
--- a/qemu/tests/qapi-schema/data-array-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-array-unknown.json:2: Member 'array' of 'data' for command 'oops' uses unknown type 'array of NoSuchType'
diff --git a/qemu/tests/qapi-schema/data-int.err b/qemu/tests/qapi-schema/data-int.err
deleted file mode 100644
index 1a9b077c0..000000000
--- a/qemu/tests/qapi-schema/data-int.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-int.json:2: 'data' for command 'oops' cannot use built-in type 'int'
diff --git a/qemu/tests/qapi-schema/data-member-array.exit b/qemu/tests/qapi-schema/data-member-array.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/data-member-array.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/data-member-array.json b/qemu/tests/qapi-schema/data-member-array.json
deleted file mode 100644
index e6f7f5da1..000000000
--- a/qemu/tests/qapi-schema/data-member-array.json
+++ /dev/null
@@ -1,4 +0,0 @@
-# valid array members
-{ 'enum': 'abc', 'data': [ 'a', 'b', 'c' ] }
-{ 'struct': 'def', 'data': { 'array': [ 'abc' ] } }
-{ 'command': 'okay', 'data': { 'member1': [ 'int' ], 'member2': [ 'def' ] } }
diff --git a/qemu/tests/qapi-schema/data-member-array.out b/qemu/tests/qapi-schema/data-member-array.out
deleted file mode 100644
index c39fa2548..000000000
--- a/qemu/tests/qapi-schema/data-member-array.out
+++ /dev/null
@@ -1,5 +0,0 @@
-[OrderedDict([('enum', 'abc'), ('data', ['a', 'b', 'c'])]),
- OrderedDict([('struct', 'def'), ('data', OrderedDict([('array', ['abc'])]))]),
- OrderedDict([('command', 'okay'), ('data', OrderedDict([('member1', ['int']), ('member2', ['def'])]))])]
-[{'enum_name': 'abc', 'enum_values': ['a', 'b', 'c']}]
-[OrderedDict([('struct', 'def'), ('data', OrderedDict([('array', ['abc'])]))])]
diff --git a/qemu/tests/qapi-schema/data-member-unknown.err b/qemu/tests/qapi-schema/data-member-unknown.err
deleted file mode 100644
index ab905db80..000000000
--- a/qemu/tests/qapi-schema/data-member-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-member-unknown.json:2: Member 'member' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/qemu/tests/qapi-schema/data-unknown.err b/qemu/tests/qapi-schema/data-unknown.err
deleted file mode 100644
index 5b07277a9..000000000
--- a/qemu/tests/qapi-schema/data-unknown.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/data-unknown.json:2: 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/qemu/tests/qapi-schema/duplicate-key.err b/qemu/tests/qapi-schema/duplicate-key.err
index 768b276f8..6d02f8353 100644
--- a/qemu/tests/qapi-schema/duplicate-key.err
+++ b/qemu/tests/qapi-schema/duplicate-key.err
@@ -1 +1 @@
-tests/qapi-schema/duplicate-key.json:2:10: Duplicate key "key"
+tests/qapi-schema/duplicate-key.json:3:10: Duplicate key "key"
diff --git a/qemu/tests/qapi-schema/duplicate-key.json b/qemu/tests/qapi-schema/duplicate-key.json
index 1b55d8810..14ac0e8a4 100644
--- a/qemu/tests/qapi-schema/duplicate-key.json
+++ b/qemu/tests/qapi-schema/duplicate-key.json
@@ -1,2 +1,3 @@
+# QAPI cannot include the same key more than once in any {}
 { 'key': 'value',
   'key': 'value' }
diff --git a/qemu/tests/qapi-schema/empty.out b/qemu/tests/qapi-schema/empty.out
index b7f89a45c..8a5b03442 100644
--- a/qemu/tests/qapi-schema/empty.out
+++ b/qemu/tests/qapi-schema/empty.out
@@ -1,3 +1,3 @@
-[]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+object q_empty
diff --git a/qemu/tests/qapi-schema/enum-bad-prefix.err b/qemu/tests/qapi-schema/enum-bad-prefix.err
new file mode 100644
index 000000000..399f5f7af
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-bad-prefix.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-bad-prefix.json:2: Enum 'MyEnum' requires a string for 'prefix'
diff --git a/qemu/tests/qapi-schema/enum-bad-prefix.exit b/qemu/tests/qapi-schema/enum-bad-prefix.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-bad-prefix.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/enum-bad-prefix.json b/qemu/tests/qapi-schema/enum-bad-prefix.json
new file mode 100644
index 000000000..996f628f6
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-bad-prefix.json
@@ -0,0 +1,2 @@
+# The prefix must be a string type
+{ 'enum': 'MyEnum', 'data': [ 'one' ], 'prefix': [ 'fish' ] }
diff --git a/qemu/tests/qapi-schema/enum-bad-prefix.out b/qemu/tests/qapi-schema/enum-bad-prefix.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-bad-prefix.out
diff --git a/qemu/tests/qapi-schema/enum-clash-member.err b/qemu/tests/qapi-schema/enum-clash-member.err
index 48bd1360e..5403c7850 100644
--- a/qemu/tests/qapi-schema/enum-clash-member.err
+++ b/qemu/tests/qapi-schema/enum-clash-member.err
@@ -1 +1 @@
-tests/qapi-schema/enum-clash-member.json:2: Enum 'MyEnum' member 'ONE' clashes with 'one'
+tests/qapi-schema/enum-clash-member.json:2: 'one_two' (member of MyEnum) collides with 'one-two' (member of MyEnum)
diff --git a/qemu/tests/qapi-schema/enum-clash-member.json b/qemu/tests/qapi-schema/enum-clash-member.json
index b7dc02a28..b6928b8bf 100644
--- a/qemu/tests/qapi-schema/enum-clash-member.json
+++ b/qemu/tests/qapi-schema/enum-clash-member.json
@@ -1,2 +1,2 @@
 # we reject enums where members will clash when mapped to C enum
-{ 'enum': 'MyEnum', 'data': [ 'one', 'ONE' ] }
+{ 'enum': 'MyEnum', 'data': [ 'one-two', 'one_two' ] }
diff --git a/qemu/tests/qapi-schema/enum-empty.exit b/qemu/tests/qapi-schema/enum-empty.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/enum-empty.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/enum-empty.json b/qemu/tests/qapi-schema/enum-empty.json
deleted file mode 100644
index 40d4e85a2..000000000
--- a/qemu/tests/qapi-schema/enum-empty.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# An empty enum, although unusual, is currently acceptable
-{ 'enum': 'MyEnum', 'data': [ ] }
diff --git a/qemu/tests/qapi-schema/enum-empty.out b/qemu/tests/qapi-schema/enum-empty.out
deleted file mode 100644
index 3b75c1613..000000000
--- a/qemu/tests/qapi-schema/enum-empty.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('enum', 'MyEnum'), ('data', [])])]
-[{'enum_name': 'MyEnum', 'enum_values': []}]
-[]
diff --git a/qemu/tests/qapi-schema/enum-max-member.err b/qemu/tests/qapi-schema/enum-max-member.err
deleted file mode 100644
index f77837fb4..000000000
--- a/qemu/tests/qapi-schema/enum-max-member.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/enum-max-member.json:3: Enum 'MyEnum' member 'max' clashes with '(automatic)'
diff --git a/qemu/tests/qapi-schema/enum-max-member.json b/qemu/tests/qapi-schema/enum-max-member.json
deleted file mode 100644
index 4bcda0bf0..000000000
--- a/qemu/tests/qapi-schema/enum-max-member.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# we reject user-supplied 'max' for clashing with implicit enum end
-# TODO: should we instead munge the implicit value to avoid the clash?
-{ 'enum': 'MyEnum', 'data': [ 'max' ] }
diff --git a/qemu/tests/qapi-schema/enum-member-case.err b/qemu/tests/qapi-schema/enum-member-case.err
new file mode 100644
index 000000000..b652e9aac
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-member-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-member-case.json:3: 'Value' (member of NoWayThisWillGetWhitelisted) should not use uppercase
diff --git a/qemu/tests/qapi-schema/enum-member-case.exit b/qemu/tests/qapi-schema/enum-member-case.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-member-case.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/enum-member-case.json b/qemu/tests/qapi-schema/enum-member-case.json
new file mode 100644
index 000000000..2096b350c
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-member-case.json
@@ -0,0 +1,3 @@
+# Member names should be 'lower-case' unless the enum is whitelisted
+{ 'enum': 'UuidInfo', 'data': [ 'Value' ] } # UuidInfo is whitelisted
+{ 'enum': 'NoWayThisWillGetWhitelisted', 'data': [ 'Value' ] }
diff --git a/qemu/tests/qapi-schema/enum-member-case.out b/qemu/tests/qapi-schema/enum-member-case.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/enum-member-case.out
diff --git a/qemu/tests/qapi-schema/enum-union-clash.err b/qemu/tests/qapi-schema/enum-union-clash.err
deleted file mode 100644
index c04e1a806..000000000
--- a/qemu/tests/qapi-schema/enum-union-clash.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/enum-union-clash.json:2: enum 'UnionKind' should not end in 'Kind'
diff --git a/qemu/tests/qapi-schema/event-case.out b/qemu/tests/qapi-schema/event-case.out
index 3764bc781..b6b4134a8 100644
--- a/qemu/tests/qapi-schema/event-case.out
+++ b/qemu/tests/qapi-schema/event-case.out
@@ -1,3 +1,4 @@
-[OrderedDict([('event', 'oops')])]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+event oops None
+object q_empty
diff --git a/qemu/tests/qapi-schema/event-max.err b/qemu/tests/qapi-schema/event-max.err
deleted file mode 100644
index c85653437..000000000
--- a/qemu/tests/qapi-schema/event-max.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/event-max.json:2: Event name 'MAX' cannot be created
diff --git a/qemu/tests/qapi-schema/event-max.json b/qemu/tests/qapi-schema/event-max.json
deleted file mode 100644
index f3d7de2a3..000000000
--- a/qemu/tests/qapi-schema/event-max.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# an event named 'MAX' would conflict with implicit C enum
-{ 'event': 'MAX' }
diff --git a/qemu/tests/qapi-schema/flat-union-bad-base.err b/qemu/tests/qapi-schema/flat-union-bad-base.err
index f9c31b2bf..bee24a217 100644
--- a/qemu/tests/qapi-schema/flat-union-bad-base.err
+++ b/qemu/tests/qapi-schema/flat-union-bad-base.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-bad-base.json:9: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-bad-base.json:8: 'string' (member of TestTypeA) collides with 'string' (base of TestUnion)
diff --git a/qemu/tests/qapi-schema/flat-union-bad-base.json b/qemu/tests/qapi-schema/flat-union-bad-base.json
index e2e622bb6..74dd42170 100644
--- a/qemu/tests/qapi-schema/flat-union-bad-base.json
+++ b/qemu/tests/qapi-schema/flat-union-bad-base.json
@@ -1,5 +1,4 @@
-# we require the base to be an existing struct
-# TODO: should we allow an anonymous inline base type?
+# we allow anonymous base, but enforce no duplicate keys
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'TestTypeA',
@@ -7,7 +6,7 @@
 { 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 { 'union': 'TestUnion',
-  'base': { 'enum1': 'TestEnum', 'kind': 'str' },
+  'base': { 'enum1': 'TestEnum', 'string': 'str' },
   'discriminator': 'enum1',
   'data': { 'value1': 'TestTypeA',
             'value2': 'TestTypeB' } }
diff --git a/qemu/tests/qapi-schema/flat-union-base-any.err b/qemu/tests/qapi-schema/flat-union-base-any.err
new file mode 100644
index 000000000..646f1c9cd
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-base-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-base-any.json:8: 'base' for union 'TestUnion' cannot use built-in type 'any'
diff --git a/qemu/tests/qapi-schema/flat-union-base-any.exit b/qemu/tests/qapi-schema/flat-union-base-any.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-base-any.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/flat-union-base-star.json b/qemu/tests/qapi-schema/flat-union-base-any.json
index 5099439a9..fe66b713e 100644
--- a/qemu/tests/qapi-schema/flat-union-base-star.json
+++ b/qemu/tests/qapi-schema/flat-union-base-any.json
@@ -6,7 +6,7 @@
 { 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 { 'union': 'TestUnion',
-  'base': '**',
+  'base': 'any',
   'discriminator': 'enum1',
   'data': { 'value1': 'TestTypeA',
             'value2': 'TestTypeB' } }
diff --git a/qemu/tests/qapi-schema/flat-union-base-any.out b/qemu/tests/qapi-schema/flat-union-base-any.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-base-any.out
diff --git a/qemu/tests/qapi-schema/flat-union-base-star.err b/qemu/tests/qapi-schema/flat-union-base-star.err
deleted file mode 100644
index b7748f08b..000000000
--- a/qemu/tests/qapi-schema/flat-union-base-star.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/flat-union-base-star.json:8: Base '**' is not a valid struct
diff --git a/qemu/tests/qapi-schema/flat-union-base-union.err b/qemu/tests/qapi-schema/flat-union-base-union.err
index ede9859a3..f138395e4 100644
--- a/qemu/tests/qapi-schema/flat-union-base-union.err
+++ b/qemu/tests/qapi-schema/flat-union-base-union.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-base-union.json:11: Base 'UnionBase' is not a valid struct
+tests/qapi-schema/flat-union-base-union.json:14: 'base' for union 'TestUnion' cannot use union type 'UnionBase'
diff --git a/qemu/tests/qapi-schema/flat-union-base-union.json b/qemu/tests/qapi-schema/flat-union-base-union.json
index 6a8ea687a..98b4eba18 100644
--- a/qemu/tests/qapi-schema/flat-union-base-union.json
+++ b/qemu/tests/qapi-schema/flat-union-base-union.json
@@ -1,4 +1,7 @@
-# we require the base to be a struct
+# For now, we require the base to be a struct without variants
+# TODO: It would be possible to allow a union as a base, as long as all
+# permutations of QMP names exposed by base do not clash with any QMP
+# member names added by local variants.
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'TestTypeA',
diff --git a/qemu/tests/qapi-schema/flat-union-branch-clash.err b/qemu/tests/qapi-schema/flat-union-branch-clash.err
deleted file mode 100644
index f11276688..000000000
--- a/qemu/tests/qapi-schema/flat-union-branch-clash.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/flat-union-branch-clash.json:10: Member name 'name' of branch 'value1' clashes with base 'Base'
diff --git a/qemu/tests/qapi-schema/flat-union-clash-member.err b/qemu/tests/qapi-schema/flat-union-clash-member.err
new file mode 100644
index 000000000..2adf69755
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-clash-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-clash-member.json:11: 'name' (member of Branch1) collides with 'name' (member of Base)
diff --git a/qemu/tests/qapi-schema/flat-union-clash-member.exit b/qemu/tests/qapi-schema/flat-union-clash-member.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-clash-member.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/flat-union-branch-clash.json b/qemu/tests/qapi-schema/flat-union-clash-member.json
index 8fb054f00..9efc7719b 100644
--- a/qemu/tests/qapi-schema/flat-union-branch-clash.json
+++ b/qemu/tests/qapi-schema/flat-union-clash-member.json
@@ -1,4 +1,5 @@
-# we check for no duplicate keys between branches and base
+# We check for no duplicate keys between branch members and base
+# base's member 'name' clashes with Branch1's
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'Base',
diff --git a/qemu/tests/qapi-schema/flat-union-clash-member.out b/qemu/tests/qapi-schema/flat-union-clash-member.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-clash-member.out
diff --git a/qemu/tests/qapi-schema/flat-union-empty.err b/qemu/tests/qapi-schema/flat-union-empty.err
new file mode 100644
index 000000000..15754f54e
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-empty.json:4: Union 'Union' cannot have empty 'data'
diff --git a/qemu/tests/qapi-schema/flat-union-empty.exit b/qemu/tests/qapi-schema/flat-union-empty.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/flat-union-empty.json b/qemu/tests/qapi-schema/flat-union-empty.json
new file mode 100644
index 000000000..77f1d9abf
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-empty.json
@@ -0,0 +1,4 @@
+# flat unions cannot be empty
+{ 'enum': 'Empty', 'data': [ ] }
+{ 'struct': 'Base', 'data': { 'type': 'Empty' } }
+{ 'union': 'Union', 'base': 'Base', 'discriminator': 'type', 'data': { } }
diff --git a/qemu/tests/qapi-schema/flat-union-empty.out b/qemu/tests/qapi-schema/flat-union-empty.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/flat-union-empty.out
diff --git a/qemu/tests/qapi-schema/flat-union-inline.err b/qemu/tests/qapi-schema/flat-union-inline.err
index ec586277b..2333358d2 100644
--- a/qemu/tests/qapi-schema/flat-union-inline.err
+++ b/qemu/tests/qapi-schema/flat-union-inline.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-inline.json:7: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-inline.json:7: Member 'value1' of union 'TestUnion' should be a type name
diff --git a/qemu/tests/qapi-schema/flat-union-inline.json b/qemu/tests/qapi-schema/flat-union-inline.json
index 6bfdd6581..62c7cda61 100644
--- a/qemu/tests/qapi-schema/flat-union-inline.json
+++ b/qemu/tests/qapi-schema/flat-union-inline.json
@@ -1,11 +1,11 @@
 # we require branches to be a struct name
-# TODO: should we allow anonymous inline types?
+# TODO: should we allow anonymous inline branch types?
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 { 'struct': 'Base',
   'data': { 'enum1': 'TestEnum', 'kind': 'str' } }
 { 'union': 'TestUnion',
-  'base': { 'enum1': 'TestEnum', 'kind': 'str' },
+  'base': 'Base',
   'discriminator': 'enum1',
   'data': { 'value1': { 'string': 'str' },
             'value2': { 'integer': 'int' } } }
diff --git a/qemu/tests/qapi-schema/flat-union-no-base.err b/qemu/tests/qapi-schema/flat-union-no-base.err
index bb3f70874..841c93b55 100644
--- a/qemu/tests/qapi-schema/flat-union-no-base.err
+++ b/qemu/tests/qapi-schema/flat-union-no-base.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-no-base.json:9: Flat union 'TestUnion' must have a string base field
+tests/qapi-schema/flat-union-no-base.json:9: Flat union 'TestUnion' must have a base
diff --git a/qemu/tests/qapi-schema/flat-union-reverse-define.exit b/qemu/tests/qapi-schema/flat-union-reverse-define.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/flat-union-reverse-define.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/flat-union-reverse-define.json b/qemu/tests/qapi-schema/flat-union-reverse-define.json
deleted file mode 100644
index 648bbfe2b..000000000
--- a/qemu/tests/qapi-schema/flat-union-reverse-define.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{ 'union': 'TestUnion',
-  'base': 'TestBase',
-  'discriminator': 'enum1',
-  'data': { 'value1': 'TestTypeA',
-            'value2': 'TestTypeB' } }
-
-{ 'struct': 'TestBase',
-  'data': { 'enum1': 'TestEnum' } }
-
-{ 'enum': 'TestEnum',
-  'data': [ 'value1', 'value2' ] }
-
-{ 'struct': 'TestTypeA',
-  'data': { 'string': 'str' } }
-
-{ 'struct': 'TestTypeB',
-  'data': { 'integer': 'int' } }
diff --git a/qemu/tests/qapi-schema/flat-union-reverse-define.out b/qemu/tests/qapi-schema/flat-union-reverse-define.out
deleted file mode 100644
index 1ed7b8a51..000000000
--- a/qemu/tests/qapi-schema/flat-union-reverse-define.out
+++ /dev/null
@@ -1,9 +0,0 @@
-[OrderedDict([('union', 'TestUnion'), ('base', 'TestBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'TestTypeA'), ('value2', 'TestTypeB')]))]),
- OrderedDict([('struct', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('enum', 'TestEnum'), ('data', ['value1', 'value2'])]),
- OrderedDict([('struct', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('struct', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
-[{'enum_name': 'TestEnum', 'enum_values': ['value1', 'value2']}]
-[OrderedDict([('struct', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('struct', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('struct', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
diff --git a/qemu/tests/qapi-schema/ident-with-escape.out b/qemu/tests/qapi-schema/ident-with-escape.out
index 402843081..382ce2fa2 100644
--- a/qemu/tests/qapi-schema/ident-with-escape.out
+++ b/qemu/tests/qapi-schema/ident-with-escape.out
@@ -1,3 +1,7 @@
-[OrderedDict([('command', 'fooA'), ('data', OrderedDict([('bar1', 'str')]))])]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+command fooA q_obj_fooA-arg -> None
+   gen=True success_response=True
+object q_empty
+object q_obj_fooA-arg
+    member bar1: str optional=False
diff --git a/qemu/tests/qapi-schema/include-non-file.err b/qemu/tests/qapi-schema/include-non-file.err
index 9658c7880..faae1eacf 100644
--- a/qemu/tests/qapi-schema/include-non-file.err
+++ b/qemu/tests/qapi-schema/include-non-file.err
@@ -1 +1 @@
-tests/qapi-schema/include-non-file.json:1: Expected a file name (string), got: ['foo', 'bar']
+tests/qapi-schema/include-non-file.json:1: Value of 'include' must be a string
diff --git a/qemu/tests/qapi-schema/include-non-file.json b/qemu/tests/qapi-schema/include-non-file.json
index cd43c3f9d..4711aa42e 100644
--- a/qemu/tests/qapi-schema/include-non-file.json
+++ b/qemu/tests/qapi-schema/include-non-file.json
@@ -1 +1 @@
-{ 'include': [ 'foo', 'bar' ] }
+{ 'include': {} }
diff --git a/qemu/tests/qapi-schema/include-relpath.out b/qemu/tests/qapi-schema/include-relpath.out
index 4ce3dcf12..5d7c13cad 100644
--- a/qemu/tests/qapi-schema/include-relpath.out
+++ b/qemu/tests/qapi-schema/include-relpath.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/qemu/tests/qapi-schema/include-repetition.out b/qemu/tests/qapi-schema/include-repetition.out
index 4ce3dcf12..5d7c13cad 100644
--- a/qemu/tests/qapi-schema/include-repetition.out
+++ b/qemu/tests/qapi-schema/include-repetition.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/qemu/tests/qapi-schema/include-simple.out b/qemu/tests/qapi-schema/include-simple.out
index 4ce3dcf12..5d7c13cad 100644
--- a/qemu/tests/qapi-schema/include-simple.out
+++ b/qemu/tests/qapi-schema/include-simple.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/qemu/tests/qapi-schema/indented-expr.out b/qemu/tests/qapi-schema/indented-expr.out
index b5ce9151b..ae3293a3a 100644
--- a/qemu/tests/qapi-schema/indented-expr.out
+++ b/qemu/tests/qapi-schema/indented-expr.out
@@ -1,3 +1,7 @@
-[OrderedDict([('command', 'eins')]), OrderedDict([('command', 'zwei')])]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+command eins None -> None
+   gen=True success_response=True
+object q_empty
+command zwei None -> None
+   gen=True success_response=True
diff --git a/qemu/tests/qapi-schema/leading-comma-list.err b/qemu/tests/qapi-schema/leading-comma-list.err
new file mode 100644
index 000000000..f5c870bb9
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-list.json:2:13: Expected "{", "[", "]", string, boolean or "null"
diff --git a/qemu/tests/qapi-schema/leading-comma-list.exit b/qemu/tests/qapi-schema/leading-comma-list.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-list.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/leading-comma-list.json b/qemu/tests/qapi-schema/leading-comma-list.json
new file mode 100644
index 000000000..c5ba50159
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-list.json
@@ -0,0 +1,2 @@
+{ 'enum': 'Status',
+  'data': [ , 'good', 'bad', 'ugly' ] }
diff --git a/qemu/tests/qapi-schema/leading-comma-list.out b/qemu/tests/qapi-schema/leading-comma-list.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-list.out
diff --git a/qemu/tests/qapi-schema/leading-comma-object.err b/qemu/tests/qapi-schema/leading-comma-object.err
new file mode 100644
index 000000000..f767b9554
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-object.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-object.json:1:3: Expected string or "}"
diff --git a/qemu/tests/qapi-schema/leading-comma-object.exit b/qemu/tests/qapi-schema/leading-comma-object.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-object.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/leading-comma-object.json b/qemu/tests/qapi-schema/leading-comma-object.json
new file mode 100644
index 000000000..c89023ff3
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-object.json
@@ -0,0 +1,2 @@
+{ , 'enum': 'Status',
+  'data': [ 'good', 'bad', 'ugly' ] }
diff --git a/qemu/tests/qapi-schema/leading-comma-object.out b/qemu/tests/qapi-schema/leading-comma-object.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/leading-comma-object.out
diff --git a/qemu/tests/qapi-schema/nested-struct-data.json b/qemu/tests/qapi-schema/nested-struct-data.json
index 3d52d2b39..efbe773de 100644
--- a/qemu/tests/qapi-schema/nested-struct-data.json
+++ b/qemu/tests/qapi-schema/nested-struct-data.json
@@ -1,4 +1,3 @@
 # inline subtypes collide with our desired future use of defaults
 { 'command': 'foo',
-  'data': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' },
-  'returns': {} }
+  'data': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' } }
diff --git a/qemu/tests/qapi-schema/nested-struct-returns.err b/qemu/tests/qapi-schema/nested-struct-returns.err
deleted file mode 100644
index 5238d075b..000000000
--- a/qemu/tests/qapi-schema/nested-struct-returns.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/nested-struct-returns.json:2: Member 'a' of 'returns' for command 'foo' should be a type name
diff --git a/qemu/tests/qapi-schema/nested-struct-returns.json b/qemu/tests/qapi-schema/nested-struct-returns.json
deleted file mode 100644
index d2cd047f0..000000000
--- a/qemu/tests/qapi-schema/nested-struct-returns.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# inline subtypes collide with our desired future use of defaults
-{ 'command': 'foo',
-  'returns': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' } }
diff --git a/qemu/tests/qapi-schema/qapi-schema-test.json b/qemu/tests/qapi-schema/qapi-schema-test.json
index c7eaa865d..f571e1bb3 100644
--- a/qemu/tests/qapi-schema/qapi-schema-test.json
+++ b/qemu/tests/qapi-schema/qapi-schema-test.json
@@ -1,19 +1,42 @@
 # *-*- Mode: Python -*-*
 
+# This file is a stress test of supported qapi constructs that must
+# parse and compile correctly.
+
+{ 'struct': 'TestStruct',
+  'data': { 'integer': 'int', 'boolean': 'bool', 'string': 'str' } }
+
 # for testing enums
-{ 'enum': 'EnumOne',
-  'data': [ 'value1', 'value2', 'value3' ] }
 { 'struct': 'NestedEnumsOne',
-  'data': { 'enum1': 'EnumOne', '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+  'data': { 'enum1': 'EnumOne',   # Intentional forward reference
+            '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+
+# An empty enum, although unusual, is currently acceptable
+{ 'enum': 'MyEnum', 'data': [ ] }
+
+# Likewise for an empty struct, including an empty base
+{ 'struct': 'Empty1', 'data': { } }
+{ 'struct': 'Empty2', 'base': 'Empty1', 'data': { } }
+
+{ 'command': 'user_def_cmd0', 'data': 'Empty2', 'returns': 'Empty2' }
+
+# for testing override of default naming heuristic
+{ 'enum': 'QEnumTwo',
+  'prefix': 'QENUM_TWO',
+  'data': [ 'value1', 'value2' ] }
 
 # for testing nested structs
+{ 'struct': 'UserDefOne',
+  'base': 'UserDefZero',        # intentional forward reference
+  'data': { 'string': 'str',
+            '*enum1': 'EnumOne' } }   # intentional forward reference
+
+{ 'enum': 'EnumOne',
+  'data': [ 'value1', 'value2', 'value3' ] }
+
 { 'struct': 'UserDefZero',
   'data': { 'integer': 'int' } }
 
-{ 'struct': 'UserDefOne',
-  'base': 'UserDefZero',
-  'data': { 'string': 'str', '*enum1': 'EnumOne' } }
-
 { 'struct': 'UserDefTwoDictDict',
   'data': { 'userdef': 'UserDefOne', 'string': 'str' } }
 
@@ -26,35 +49,55 @@
   'data': { 'string0': 'str',
             'dict1': 'UserDefTwoDict' } }
 
+# dummy struct to force generation of array types not otherwise mentioned
+{ 'struct': 'ForceArrays',
+  'data': { 'unused1':['UserDefOne'], 'unused2':['UserDefTwo'],
+            'unused3':['TestStruct'] } }
+
 # for testing unions
+# Among other things, test that a name collision between branches does
+# not cause any problems (since only one branch can be in use at a time),
+# by intentionally using two branches that both have a C member 'a_b'
 { 'struct': 'UserDefA',
-  'data': { 'boolean': 'bool' } }
+  'data': { 'boolean': 'bool', '*a_b': 'int' } }
 
 { 'struct': 'UserDefB',
-  'data': { 'integer': 'int' } }
+  'data': { 'intb': 'int', '*a-b': 'bool' } }
 
-{ 'struct': 'UserDefC',
-  'data': { 'string1': 'str', 'string2': 'str' } }
+{ 'union': 'UserDefFlatUnion',
+  'base': 'UserDefUnionBase',   # intentional forward reference
+  'discriminator': 'enum1',
+  'data': { 'value1' : 'UserDefA',
+            'value2' : 'UserDefB',
+            'value3' : 'UserDefB' } }
 
 { 'struct': 'UserDefUnionBase',
+  'base': 'UserDefZero',
   'data': { 'string': 'str', 'enum1': 'EnumOne' } }
 
-{ 'union': 'UserDefFlatUnion',
-  'base': 'UserDefUnionBase',
-  'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefA', 'value2' : 'UserDefB', 'value3' : 'UserDefB' } }
-# FIXME generated struct UserDefFlatUnion has members for direct base
-# UserDefOne, but lacks members for indirect base UserDefZero
-
-# this variant of UserDefFlatUnion defaults to a union that uses fields with
+# this variant of UserDefFlatUnion defaults to a union that uses members with
 # allocated types to test corner cases in the cleanup/dealloc visitor
 { 'union': 'UserDefFlatUnion2',
-  'base': 'UserDefUnionBase',
+  'base': { '*integer': 'int', 'string': 'str', 'enum1': 'QEnumTwo' },
   'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefC', 'value2' : 'UserDefB', 'value3' : 'UserDefA' } }
+  'data': { 'value1' : 'UserDefC', # intentional forward reference
+            'value2' : 'UserDefB' } }
 
+{ 'struct': 'WrapAlternate',
+  'data': { 'alt': 'UserDefAlternate' } }
 { 'alternate': 'UserDefAlternate',
-  'data': { 'uda': 'UserDefA', 's': 'str', 'i': 'int' } }
+  'data': { 'udfu': 'UserDefFlatUnion', 's': 'str', 'i': 'int' } }
+
+{ 'struct': 'UserDefC',
+  'data': { 'string1': 'str', 'string2': 'str' } }
+
+# for testing use of 'number' within alternates
+{ 'alternate': 'AltStrBool', 'data': { 's': 'str', 'b': 'bool' } }
+{ 'alternate': 'AltStrNum', 'data': { 's': 'str', 'n': 'number' } }
+{ 'alternate': 'AltNumStr', 'data': { 'n': 'number', 's': 'str' } }
+{ 'alternate': 'AltStrInt', 'data': { 's': 'str', 'i': 'int' } }
+{ 'alternate': 'AltIntNum', 'data': { 'i': 'int', 'n': 'number' } }
+{ 'alternate': 'AltNumInt', 'data': { 'n': 'number', 'i': 'int' } }
 
 # for testing native lists
 { 'union': 'UserDefNativeListUnion',
@@ -70,7 +113,8 @@
             'number': ['number'],
             'boolean': ['bool'],
             'string': ['str'],
-            'sizes': ['size'] } }
+            'sizes': ['size'],
+            'any': ['any'] } }
 
 # testing commands
 { 'command': 'user_def_cmd', 'data': {} }
@@ -78,8 +122,11 @@
 { 'command': 'user_def_cmd2',
   'data': {'ud1a': 'UserDefOne', '*ud1b': 'UserDefOne'},
   'returns': 'UserDefTwo' }
-{ 'command': 'user_def_cmd3', 'data': {'a': 'int', '*b': 'int' },
+
+# Returning a non-dictionary requires a name from the whitelist
+{ 'command': 'guest-get-time', 'data': {'a': 'int', '*b': 'int' },
   'returns': 'int' }
+{ 'command': 'guest-sync', 'data': { 'arg': 'any' }, 'returns': 'any' }
 
 # For testing integer range flattening in opts-visitor. The following schema
 # corresponds to the option format:
@@ -108,12 +155,13 @@
 { 'event': 'EVENT_D',
   'data': { 'a' : 'EventStructOne', 'b' : 'str', '*c': 'str', '*enum3': 'EnumOne' } }
 
-# test that we correctly compile downstream extensions
+# test that we correctly compile downstream extensions, as well as munge
+# ticklish names
 { 'enum': '__org.qemu_x-Enum', 'data': [ '__org.qemu_x-value' ] }
 { 'struct': '__org.qemu_x-Base',
   'data': { '__org.qemu_x-member1': '__org.qemu_x-Enum' } }
 { 'struct': '__org.qemu_x-Struct', 'base': '__org.qemu_x-Base',
-  'data': { '__org.qemu_x-member2': 'str' } }
+  'data': { '__org.qemu_x-member2': 'str', '*wchar-t': 'int' } }
 { 'union': '__org.qemu_x-Union1', 'data': { '__org.qemu_x-branch': 'str' } }
 { 'struct': '__org.qemu_x-Struct2',
   'data': { 'array': ['__org.qemu_x-Union1'] } }
diff --git a/qemu/tests/qapi-schema/qapi-schema-test.out b/qemu/tests/qapi-schema/qapi-schema-test.out
index cf0ccc402..19cd214f6 100644
--- a/qemu/tests/qapi-schema/qapi-schema-test.out
+++ b/qemu/tests/qapi-schema/qapi-schema-test.out
@@ -1,55 +1,211 @@
-[OrderedDict([('enum', 'EnumOne'), ('data', ['value1', 'value2', 'value3'])]),
- OrderedDict([('struct', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefTwoDictDict'), ('data', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]),
- OrderedDict([('struct', 'UserDefTwoDict'), ('data', OrderedDict([('string1', 'str'), ('dict2', 'UserDefTwoDictDict'), ('*dict3', 'UserDefTwoDictDict')]))]),
- OrderedDict([('struct', 'UserDefTwo'), ('data', OrderedDict([('string0', 'str'), ('dict1', 'UserDefTwoDict')]))]),
- OrderedDict([('struct', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('struct', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('struct', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('union', 'UserDefFlatUnion'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefA'), ('value2', 'UserDefB'), ('value3', 'UserDefB')]))]),
- OrderedDict([('union', 'UserDefFlatUnion2'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefC'), ('value2', 'UserDefB'), ('value3', 'UserDefA')]))]),
- OrderedDict([('alternate', 'UserDefAlternate'), ('data', OrderedDict([('uda', 'UserDefA'), ('s', 'str'), ('i', 'int')]))]),
- OrderedDict([('union', 'UserDefNativeListUnion'), ('data', OrderedDict([('integer', ['int']), ('s8', ['int8']), ('s16', ['int16']), ('s32', ['int32']), ('s64', ['int64']), ('u8', ['uint8']), ('u16', ['uint16']), ('u32', ['uint32']), ('u64', ['uint64']), ('number', ['number']), ('boolean', ['bool']), ('string', ['str']), ('sizes', ['size'])]))]),
- OrderedDict([('command', 'user_def_cmd'), ('data', OrderedDict())]),
- OrderedDict([('command', 'user_def_cmd1'), ('data', OrderedDict([('ud1a', 'UserDefOne')]))]),
- OrderedDict([('command', 'user_def_cmd2'), ('data', OrderedDict([('ud1a', 'UserDefOne'), ('*ud1b', 'UserDefOne')])), ('returns', 'UserDefTwo')]),
- OrderedDict([('command', 'user_def_cmd3'), ('data', OrderedDict([('a', 'int'), ('*b', 'int')])), ('returns', 'int')]),
- OrderedDict([('struct', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('struct', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))]),
- OrderedDict([('event', 'EVENT_A')]),
- OrderedDict([('event', 'EVENT_B'), ('data', OrderedDict())]),
- OrderedDict([('event', 'EVENT_C'), ('data', OrderedDict([('*a', 'int'), ('*b', 'UserDefOne'), ('c', 'str')]))]),
- OrderedDict([('event', 'EVENT_D'), ('data', OrderedDict([('a', 'EventStructOne'), ('b', 'str'), ('*c', 'str'), ('*enum3', 'EnumOne')]))]),
- OrderedDict([('enum', '__org.qemu_x-Enum'), ('data', ['__org.qemu_x-value'])]),
- OrderedDict([('struct', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member1', '__org.qemu_x-Enum')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct'), ('base', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member2', 'str')]))]),
- OrderedDict([('union', '__org.qemu_x-Union1'), ('data', OrderedDict([('__org.qemu_x-branch', 'str')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct2'), ('data', OrderedDict([('array', ['__org.qemu_x-Union1'])]))]),
- OrderedDict([('union', '__org.qemu_x-Union2'), ('base', '__org.qemu_x-Base'), ('discriminator', '__org.qemu_x-member1'), ('data', OrderedDict([('__org.qemu_x-value', '__org.qemu_x-Struct2')]))]),
- OrderedDict([('alternate', '__org.qemu_x-Alt'), ('data', OrderedDict([('__org.qemu_x-branch', 'str'), ('b', '__org.qemu_x-Base')]))]),
- OrderedDict([('event', '__ORG.QEMU_X-EVENT'), ('data', '__org.qemu_x-Struct')]),
- OrderedDict([('command', '__org.qemu_x-command'), ('data', OrderedDict([('a', ['__org.qemu_x-Enum']), ('b', ['__org.qemu_x-Struct']), ('c', '__org.qemu_x-Union2'), ('d', '__org.qemu_x-Alt')])), ('returns', '__org.qemu_x-Union1')])]
-[{'enum_name': 'EnumOne', 'enum_values': ['value1', 'value2', 'value3']},
- {'enum_name': '__org.qemu_x-Enum', 'enum_values': ['__org.qemu_x-value']},
- {'enum_name': 'UserDefAlternateKind', 'enum_values': None},
- {'enum_name': 'UserDefNativeListUnionKind', 'enum_values': None},
- {'enum_name': '__org.qemu_x-Union1Kind', 'enum_values': None},
- {'enum_name': '__org.qemu_x-AltKind', 'enum_values': None}]
-[OrderedDict([('struct', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefTwoDictDict'), ('data', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]),
- OrderedDict([('struct', 'UserDefTwoDict'), ('data', OrderedDict([('string1', 'str'), ('dict2', 'UserDefTwoDictDict'), ('*dict3', 'UserDefTwoDictDict')]))]),
- OrderedDict([('struct', 'UserDefTwo'), ('data', OrderedDict([('string0', 'str'), ('dict1', 'UserDefTwoDict')]))]),
- OrderedDict([('struct', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('struct', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('struct', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('struct', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('struct', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('struct', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))]),
- OrderedDict([('struct', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member1', '__org.qemu_x-Enum')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct'), ('base', '__org.qemu_x-Base'), ('data', OrderedDict([('__org.qemu_x-member2', 'str')]))]),
- OrderedDict([('struct', '__org.qemu_x-Struct2'), ('data', OrderedDict([('array', ['__org.qemu_x-Union1'])]))])]
+alternate AltIntNum
+    case i: int
+    case n: number
+alternate AltNumInt
+    case n: number
+    case i: int
+alternate AltNumStr
+    case n: number
+    case s: str
+alternate AltStrBool
+    case s: str
+    case b: bool
+alternate AltStrInt
+    case s: str
+    case i: int
+alternate AltStrNum
+    case s: str
+    case n: number
+event EVENT_A None
+event EVENT_B None
+event EVENT_C q_obj_EVENT_C-arg
+event EVENT_D q_obj_EVENT_D-arg
+object Empty1
+object Empty2
+    base Empty1
+enum EnumOne ['value1', 'value2', 'value3']
+object EventStructOne
+    member struct1: UserDefOne optional=False
+    member string: str optional=False
+    member enum2: EnumOne optional=True
+object ForceArrays
+    member unused1: UserDefOneList optional=False
+    member unused2: UserDefTwoList optional=False
+    member unused3: TestStructList optional=False
+enum MyEnum []
+object NestedEnumsOne
+    member enum1: EnumOne optional=False
+    member enum2: EnumOne optional=True
+    member enum3: EnumOne optional=False
+    member enum4: EnumOne optional=True
+enum QEnumTwo ['value1', 'value2']
+    prefix QENUM_TWO
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+object TestStruct
+    member integer: int optional=False
+    member boolean: bool optional=False
+    member string: str optional=False
+object UserDefA
+    member boolean: bool optional=False
+    member a_b: int optional=True
+alternate UserDefAlternate
+    case udfu: UserDefFlatUnion
+    case s: str
+    case i: int
+object UserDefB
+    member intb: int optional=False
+    member a-b: bool optional=True
+object UserDefC
+    member string1: str optional=False
+    member string2: str optional=False
+object UserDefFlatUnion
+    base UserDefUnionBase
+    tag enum1
+    case value1: UserDefA
+    case value2: UserDefB
+    case value3: UserDefB
+object UserDefFlatUnion2
+    base q_obj_UserDefFlatUnion2-base
+    tag enum1
+    case value1: UserDefC
+    case value2: UserDefB
+object UserDefNativeListUnion
+    member type: UserDefNativeListUnionKind optional=False
+    case integer: q_obj_intList-wrapper
+    case s8: q_obj_int8List-wrapper
+    case s16: q_obj_int16List-wrapper
+    case s32: q_obj_int32List-wrapper
+    case s64: q_obj_int64List-wrapper
+    case u8: q_obj_uint8List-wrapper
+    case u16: q_obj_uint16List-wrapper
+    case u32: q_obj_uint32List-wrapper
+    case u64: q_obj_uint64List-wrapper
+    case number: q_obj_numberList-wrapper
+    case boolean: q_obj_boolList-wrapper
+    case string: q_obj_strList-wrapper
+    case sizes: q_obj_sizeList-wrapper
+    case any: q_obj_anyList-wrapper
+enum UserDefNativeListUnionKind ['integer', 's8', 's16', 's32', 's64', 'u8', 'u16', 'u32', 'u64', 'number', 'boolean', 'string', 'sizes', 'any']
+object UserDefOne
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=True
+object UserDefOptions
+    member i64: intList optional=True
+    member u64: uint64List optional=True
+    member u16: uint16List optional=True
+    member i64x: int optional=True
+    member u64x: uint64 optional=True
+object UserDefTwo
+    member string0: str optional=False
+    member dict1: UserDefTwoDict optional=False
+object UserDefTwoDict
+    member string1: str optional=False
+    member dict2: UserDefTwoDictDict optional=False
+    member dict3: UserDefTwoDictDict optional=True
+object UserDefTwoDictDict
+    member userdef: UserDefOne optional=False
+    member string: str optional=False
+object UserDefUnionBase
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=False
+object UserDefZero
+    member integer: int optional=False
+object WrapAlternate
+    member alt: UserDefAlternate optional=False
+event __ORG.QEMU_X-EVENT __org.qemu_x-Struct
+alternate __org.qemu_x-Alt
+    case __org.qemu_x-branch: str
+    case b: __org.qemu_x-Base
+object __org.qemu_x-Base
+    member __org.qemu_x-member1: __org.qemu_x-Enum optional=False
+enum __org.qemu_x-Enum ['__org.qemu_x-value']
+object __org.qemu_x-Struct
+    base __org.qemu_x-Base
+    member __org.qemu_x-member2: str optional=False
+    member wchar-t: int optional=True
+object __org.qemu_x-Struct2
+    member array: __org.qemu_x-Union1List optional=False
+object __org.qemu_x-Union1
+    member type: __org.qemu_x-Union1Kind optional=False
+    case __org.qemu_x-branch: q_obj_str-wrapper
+enum __org.qemu_x-Union1Kind ['__org.qemu_x-branch']
+object __org.qemu_x-Union2
+    base __org.qemu_x-Base
+    tag __org.qemu_x-member1
+    case __org.qemu_x-value: __org.qemu_x-Struct2
+command __org.qemu_x-command q_obj___org.qemu_x-command-arg -> __org.qemu_x-Union1
+   gen=True success_response=True
+command guest-get-time q_obj_guest-get-time-arg -> int
+   gen=True success_response=True
+command guest-sync q_obj_guest-sync-arg -> any
+   gen=True success_response=True
+object q_empty
+object q_obj_EVENT_C-arg
+    member a: int optional=True
+    member b: UserDefOne optional=True
+    member c: str optional=False
+object q_obj_EVENT_D-arg
+    member a: EventStructOne optional=False
+    member b: str optional=False
+    member c: str optional=True
+    member enum3: EnumOne optional=True
+object q_obj_UserDefFlatUnion2-base
+    member integer: int optional=True
+    member string: str optional=False
+    member enum1: QEnumTwo optional=False
+object q_obj___org.qemu_x-command-arg
+    member a: __org.qemu_x-EnumList optional=False
+    member b: __org.qemu_x-StructList optional=False
+    member c: __org.qemu_x-Union2 optional=False
+    member d: __org.qemu_x-Alt optional=False
+object q_obj_anyList-wrapper
+    member data: anyList optional=False
+object q_obj_boolList-wrapper
+    member data: boolList optional=False
+object q_obj_guest-get-time-arg
+    member a: int optional=False
+    member b: int optional=True
+object q_obj_guest-sync-arg
+    member arg: any optional=False
+object q_obj_int16List-wrapper
+    member data: int16List optional=False
+object q_obj_int32List-wrapper
+    member data: int32List optional=False
+object q_obj_int64List-wrapper
+    member data: int64List optional=False
+object q_obj_int8List-wrapper
+    member data: int8List optional=False
+object q_obj_intList-wrapper
+    member data: intList optional=False
+object q_obj_numberList-wrapper
+    member data: numberList optional=False
+object q_obj_sizeList-wrapper
+    member data: sizeList optional=False
+object q_obj_str-wrapper
+    member data: str optional=False
+object q_obj_strList-wrapper
+    member data: strList optional=False
+object q_obj_uint16List-wrapper
+    member data: uint16List optional=False
+object q_obj_uint32List-wrapper
+    member data: uint32List optional=False
+object q_obj_uint64List-wrapper
+    member data: uint64List optional=False
+object q_obj_uint8List-wrapper
+    member data: uint8List optional=False
+object q_obj_user_def_cmd1-arg
+    member ud1a: UserDefOne optional=False
+object q_obj_user_def_cmd2-arg
+    member ud1a: UserDefOne optional=False
+    member ud1b: UserDefOne optional=True
+command user_def_cmd None -> None
+   gen=True success_response=True
+command user_def_cmd0 Empty2 -> Empty2
+   gen=True success_response=True
+command user_def_cmd1 q_obj_user_def_cmd1-arg -> None
+   gen=True success_response=True
+command user_def_cmd2 q_obj_user_def_cmd2-arg -> UserDefTwo
+   gen=True success_response=True
diff --git a/qemu/tests/qapi-schema/reserved-command-q.err b/qemu/tests/qapi-schema/reserved-command-q.err
new file mode 100644
index 000000000..f939e044e
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-command-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-command-q.json:5: 'command' uses invalid name 'q-unix'
diff --git a/qemu/tests/qapi-schema/reserved-command-q.exit b/qemu/tests/qapi-schema/reserved-command-q.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-command-q.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-command-q.json b/qemu/tests/qapi-schema/reserved-command-q.json
new file mode 100644
index 000000000..99f8aae31
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-command-q.json
@@ -0,0 +1,5 @@
+# C entity name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'command': 'unix' }
+{ 'command': 'q-unix' }
diff --git a/qemu/tests/qapi-schema/reserved-command-q.out b/qemu/tests/qapi-schema/reserved-command-q.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-command-q.out
diff --git a/qemu/tests/qapi-schema/reserved-enum-q.err b/qemu/tests/qapi-schema/reserved-enum-q.err
new file mode 100644
index 000000000..e1c3480ee
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-enum-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-enum-q.json:4: Member of enum 'Foo' uses invalid name 'q-Unix'
diff --git a/qemu/tests/qapi-schema/reserved-enum-q.exit b/qemu/tests/qapi-schema/reserved-enum-q.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-enum-q.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-enum-q.json b/qemu/tests/qapi-schema/reserved-enum-q.json
new file mode 100644
index 000000000..3593a765e
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-enum-q.json
@@ -0,0 +1,4 @@
+# C entity name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'enum': 'Foo', 'data': [ 'unix', 'q-Unix' ] }
diff --git a/qemu/tests/qapi-schema/reserved-enum-q.out b/qemu/tests/qapi-schema/reserved-enum-q.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-enum-q.out
diff --git a/qemu/tests/qapi-schema/reserved-member-has.err b/qemu/tests/qapi-schema/reserved-member-has.err
new file mode 100644
index 000000000..e75577144
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-has.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-has.json:5: Member of 'data' for command 'oops' uses reserved name 'has-a'
diff --git a/qemu/tests/qapi-schema/reserved-member-has.exit b/qemu/tests/qapi-schema/reserved-member-has.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-has.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-member-has.json b/qemu/tests/qapi-schema/reserved-member-has.json
new file mode 100644
index 000000000..45b9109bd
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-has.json
@@ -0,0 +1,5 @@
+# C member name collision
+# We reject names like 'has-a', because they can collide with the flag
+# for an optional 'a' in generated C.
+# TODO we could munge the optional flag name to avoid the collision.
+{ 'command': 'oops', 'data': { '*a': 'str', 'has-a': 'str' } }
diff --git a/qemu/tests/qapi-schema/reserved-member-has.out b/qemu/tests/qapi-schema/reserved-member-has.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-has.out
diff --git a/qemu/tests/qapi-schema/reserved-member-q.err b/qemu/tests/qapi-schema/reserved-member-q.err
new file mode 100644
index 000000000..f3d5dd781
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-q.json:4: Member of 'data' for struct 'Foo' uses invalid name 'q-unix'
diff --git a/qemu/tests/qapi-schema/reserved-member-q.exit b/qemu/tests/qapi-schema/reserved-member-q.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-q.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-member-q.json b/qemu/tests/qapi-schema/reserved-member-q.json
new file mode 100644
index 000000000..62fed8fdd
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-q.json
@@ -0,0 +1,4 @@
+# C member name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'struct': 'Foo', 'data': { 'unix':'int', 'q-unix':'bool' } }
diff --git a/qemu/tests/qapi-schema/reserved-member-q.out b/qemu/tests/qapi-schema/reserved-member-q.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-q.out
diff --git a/qemu/tests/qapi-schema/reserved-member-u.err b/qemu/tests/qapi-schema/reserved-member-u.err
new file mode 100644
index 000000000..87d42296c
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-u.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-u.json:7: Member of 'data' for struct 'Oops' uses reserved name 'u'
diff --git a/qemu/tests/qapi-schema/reserved-member-u.exit b/qemu/tests/qapi-schema/reserved-member-u.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-u.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-member-u.json b/qemu/tests/qapi-schema/reserved-member-u.json
new file mode 100644
index 000000000..1eaf0f301
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-u.json
@@ -0,0 +1,7 @@
+# Potential C member name collision
+# We reject use of 'u' as a member name, to allow it for internal use in
+# putting union branch members in a separate namespace from QMP members.
+# This is true even for non-unions, because it is possible to convert a
+# struct to flat union while remaining backwards compatible in QMP.
+# TODO - we could munge the member name to 'q_u' to avoid the collision
+{ 'struct': 'Oops', 'data': { 'u': 'str' } }
diff --git a/qemu/tests/qapi-schema/reserved-member-u.out b/qemu/tests/qapi-schema/reserved-member-u.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-u.out
diff --git a/qemu/tests/qapi-schema/reserved-member-underscore.err b/qemu/tests/qapi-schema/reserved-member-underscore.err
new file mode 100644
index 000000000..65ff0da8c
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-underscore.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-underscore.json:4: Member of 'data' for struct 'Oops' uses invalid name '_oops'
diff --git a/qemu/tests/qapi-schema/reserved-member-underscore.exit b/qemu/tests/qapi-schema/reserved-member-underscore.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-underscore.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-member-underscore.json b/qemu/tests/qapi-schema/reserved-member-underscore.json
new file mode 100644
index 000000000..4a3a01763
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-underscore.json
@@ -0,0 +1,4 @@
+# C member name collision
+# We reject use of a single leading underscore in all names (names must
+# begin with a letter or a downstream extension double-underscore prefix).
+{ 'struct': 'Oops', 'data': { '_oops': 'str' } }
diff --git a/qemu/tests/qapi-schema/reserved-member-underscore.out b/qemu/tests/qapi-schema/reserved-member-underscore.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-member-underscore.out
diff --git a/qemu/tests/qapi-schema/reserved-type-kind.err b/qemu/tests/qapi-schema/reserved-type-kind.err
new file mode 100644
index 000000000..0a38efaad
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-kind.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-kind.json:2: enum 'UnionKind' should not end in 'Kind'
diff --git a/qemu/tests/qapi-schema/reserved-type-kind.exit b/qemu/tests/qapi-schema/reserved-type-kind.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-kind.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/enum-union-clash.json b/qemu/tests/qapi-schema/reserved-type-kind.json
index 593282b6c..9ecaba12b 100644
--- a/qemu/tests/qapi-schema/enum-union-clash.json
+++ b/qemu/tests/qapi-schema/reserved-type-kind.json
@@ -1,4 +1,2 @@
 # we reject types that would conflict with implicit union enum
 { 'enum': 'UnionKind', 'data': [ 'oops' ] }
-{ 'union': 'Union',
-  'data': { 'a': 'int' } }
diff --git a/qemu/tests/qapi-schema/reserved-type-kind.out b/qemu/tests/qapi-schema/reserved-type-kind.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-kind.out
diff --git a/qemu/tests/qapi-schema/reserved-type-list.err b/qemu/tests/qapi-schema/reserved-type-list.err
new file mode 100644
index 000000000..4510fa6d9
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-list.json:5: struct 'FooList' should not end in 'List'
diff --git a/qemu/tests/qapi-schema/reserved-type-list.exit b/qemu/tests/qapi-schema/reserved-type-list.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-list.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/reserved-type-list.json b/qemu/tests/qapi-schema/reserved-type-list.json
new file mode 100644
index 000000000..98d53bf80
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-list.json
@@ -0,0 +1,5 @@
+# Potential C name collision
+# We reserve names ending in 'List' for use by array types.
+# TODO - we could choose array names to avoid collision with user types,
+# in order to let this compile
+{ 'struct': 'FooList', 'data': { 's': 'str' } }
diff --git a/qemu/tests/qapi-schema/reserved-type-list.out b/qemu/tests/qapi-schema/reserved-type-list.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/reserved-type-list.out
diff --git a/qemu/tests/qapi-schema/returns-dict.err b/qemu/tests/qapi-schema/returns-dict.err
new file mode 100644
index 000000000..eb2d0c466
--- /dev/null
+++ b/qemu/tests/qapi-schema/returns-dict.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-dict.json:2: 'returns' for command 'oops' should be a type name
diff --git a/qemu/tests/qapi-schema/returns-dict.exit b/qemu/tests/qapi-schema/returns-dict.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/returns-dict.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/returns-dict.json b/qemu/tests/qapi-schema/returns-dict.json
new file mode 100644
index 000000000..1cfef3ede
--- /dev/null
+++ b/qemu/tests/qapi-schema/returns-dict.json
@@ -0,0 +1,2 @@
+# we reject inline struct return type
+{ 'command': 'oops', 'returns': { 'a': 'str' } }
diff --git a/qemu/tests/qapi-schema/returns-dict.out b/qemu/tests/qapi-schema/returns-dict.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/returns-dict.out
diff --git a/qemu/tests/qapi-schema/returns-int.exit b/qemu/tests/qapi-schema/returns-int.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/returns-int.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/returns-int.json b/qemu/tests/qapi-schema/returns-int.json
deleted file mode 100644
index 870ec6366..000000000
--- a/qemu/tests/qapi-schema/returns-int.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# It is okay (although not extensible) to return a non-dictionary
-# But to make it work, the name must be in a whitelist
-{ 'command': 'guest-get-time', 'returns': 'int' }
diff --git a/qemu/tests/qapi-schema/returns-int.out b/qemu/tests/qapi-schema/returns-int.out
deleted file mode 100644
index 70b3ac5e6..000000000
--- a/qemu/tests/qapi-schema/returns-int.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('command', 'guest-get-time'), ('returns', 'int')])]
-[]
-[]
diff --git a/qemu/tests/qapi-schema/returns-whitelist.err b/qemu/tests/qapi-schema/returns-whitelist.err
index a41f019a5..f47c1ee7c 100644
--- a/qemu/tests/qapi-schema/returns-whitelist.err
+++ b/qemu/tests/qapi-schema/returns-whitelist.err
@@ -1 +1 @@
-tests/qapi-schema/returns-whitelist.json:10: 'returns' for command 'no-way-this-will-get-whitelisted' cannot use built-in type 'array of int'
+tests/qapi-schema/returns-whitelist.json:10: 'returns' for command 'no-way-this-will-get-whitelisted' cannot use built-in type 'int'
diff --git a/qemu/tests/qapi-schema/struct-base-clash-deep.err b/qemu/tests/qapi-schema/struct-base-clash-deep.err
index e3e9f8d28..e2d7943f2 100644
--- a/qemu/tests/qapi-schema/struct-base-clash-deep.err
+++ b/qemu/tests/qapi-schema/struct-base-clash-deep.err
@@ -1 +1 @@
-tests/qapi-schema/struct-base-clash-deep.json:7: Member name 'name' clashes with base 'Base'
+tests/qapi-schema/struct-base-clash-deep.json:10: 'name' (member of Sub) collides with 'name' (member of Base)
diff --git a/qemu/tests/qapi-schema/struct-base-clash-deep.json b/qemu/tests/qapi-schema/struct-base-clash-deep.json
index 552fe9431..fa873ab5d 100644
--- a/qemu/tests/qapi-schema/struct-base-clash-deep.json
+++ b/qemu/tests/qapi-schema/struct-base-clash-deep.json
@@ -1,4 +1,7 @@
-# we check for no duplicate keys with indirect base
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and
+# indirectly for the grandparent base; the collision doesn't care that
+# one instance is optional.
 { 'struct': 'Base',
   'data': { 'name': 'str' } }
 { 'struct': 'Mid',
diff --git a/qemu/tests/qapi-schema/struct-base-clash.err b/qemu/tests/qapi-schema/struct-base-clash.err
index 3ac37fb26..c52f33d27 100644
--- a/qemu/tests/qapi-schema/struct-base-clash.err
+++ b/qemu/tests/qapi-schema/struct-base-clash.err
@@ -1 +1 @@
-tests/qapi-schema/struct-base-clash.json:4: Member name 'name' clashes with base 'Base'
+tests/qapi-schema/struct-base-clash.json:5: 'name' (member of Sub) collides with 'name' (member of Base)
diff --git a/qemu/tests/qapi-schema/struct-base-clash.json b/qemu/tests/qapi-schema/struct-base-clash.json
index f2afc9b6f..11aec80fe 100644
--- a/qemu/tests/qapi-schema/struct-base-clash.json
+++ b/qemu/tests/qapi-schema/struct-base-clash.json
@@ -1,4 +1,5 @@
-# we check for no duplicate keys with base
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and for base.
 { 'struct': 'Base',
   'data': { 'name': 'str' } }
 { 'struct': 'Sub',
diff --git a/qemu/tests/qapi-schema/struct-data-invalid.err b/qemu/tests/qapi-schema/struct-data-invalid.err
new file mode 100644
index 000000000..6644f4c2a
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-data-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-data-invalid.json:1: 'data' for struct 'foo' should be a dictionary or type name
diff --git a/qemu/tests/qapi-schema/struct-data-invalid.exit b/qemu/tests/qapi-schema/struct-data-invalid.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-data-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/struct-data-invalid.json b/qemu/tests/qapi-schema/struct-data-invalid.json
new file mode 100644
index 000000000..9adbc3bb6
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-data-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': false }
diff --git a/qemu/tests/qapi-schema/struct-data-invalid.out b/qemu/tests/qapi-schema/struct-data-invalid.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-data-invalid.out
diff --git a/qemu/tests/qapi-schema/struct-member-invalid.err b/qemu/tests/qapi-schema/struct-member-invalid.err
new file mode 100644
index 000000000..69a326d45
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-member-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-member-invalid.json:1: Member 'a' of 'data' for struct 'foo' should be a type name
diff --git a/qemu/tests/qapi-schema/struct-member-invalid.exit b/qemu/tests/qapi-schema/struct-member-invalid.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-member-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/struct-member-invalid.json b/qemu/tests/qapi-schema/struct-member-invalid.json
new file mode 100644
index 000000000..8f172f7a8
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-member-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': { 'a': false } }
diff --git a/qemu/tests/qapi-schema/struct-member-invalid.out b/qemu/tests/qapi-schema/struct-member-invalid.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/struct-member-invalid.out
diff --git a/qemu/tests/qapi-schema/test-qapi.py b/qemu/tests/qapi-schema/test-qapi.py
index 634ef2d00..649677e01 100644
--- a/qemu/tests/qapi-schema/test-qapi.py
+++ b/qemu/tests/qapi-schema/test-qapi.py
@@ -15,11 +15,42 @@ from pprint import pprint
 import os
 import sys
 
-try:
-    exprs = parse_schema(sys.argv[1])
-except SystemExit:
-    raise
-
-pprint(exprs)
-pprint(enum_types)
-pprint(struct_types)
+
+class QAPISchemaTestVisitor(QAPISchemaVisitor):
+    def visit_enum_type(self, name, info, values, prefix):
+        print 'enum %s %s' % (name, values)
+        if prefix:
+            print '    prefix %s' % prefix
+
+    def visit_object_type(self, name, info, base, members, variants):
+        print 'object %s' % name
+        if base:
+            print '    base %s' % base.name
+        for m in members:
+            print '    member %s: %s optional=%s' % \
+                (m.name, m.type.name, m.optional)
+        self._print_variants(variants)
+
+    def visit_alternate_type(self, name, info, variants):
+        print 'alternate %s' % name
+        self._print_variants(variants)
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response):
+        print 'command %s %s -> %s' % \
+            (name, arg_type and arg_type.name, ret_type and ret_type.name)
+        print '   gen=%s success_response=%s' % (gen, success_response)
+
+    def visit_event(self, name, info, arg_type):
+        print 'event %s %s' % (name, arg_type and arg_type.name)
+
+    @staticmethod
+    def _print_variants(variants):
+        if variants:
+            if variants.tag_name:
+                print '    tag %s' % variants.tag_name
+            for v in variants.variants:
+                print '    case %s: %s' % (v.name, v.type.name)
+
+schema = QAPISchema(sys.argv[1])
+schema.visit(QAPISchemaTestVisitor())
diff --git a/qemu/tests/qapi-schema/type-bypass-no-gen.err b/qemu/tests/qapi-schema/type-bypass-no-gen.err
deleted file mode 100644
index 20cef0a8a..000000000
--- a/qemu/tests/qapi-schema/type-bypass-no-gen.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/type-bypass-no-gen.json:2: Member 'arg' of 'data' for command 'unsafe' uses '**' but did not request 'gen':false
diff --git a/qemu/tests/qapi-schema/type-bypass-no-gen.json b/qemu/tests/qapi-schema/type-bypass-no-gen.json
deleted file mode 100644
index 4feae3719..000000000
--- a/qemu/tests/qapi-schema/type-bypass-no-gen.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# type bypass only works with 'gen':false
-{ 'command': 'unsafe', 'data': { 'arg': '**' }, 'returns': '**' }
diff --git a/qemu/tests/qapi-schema/type-bypass.exit b/qemu/tests/qapi-schema/type-bypass.exit
deleted file mode 100644
index 573541ac9..000000000
--- a/qemu/tests/qapi-schema/type-bypass.exit
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/qemu/tests/qapi-schema/type-bypass.json b/qemu/tests/qapi-schema/type-bypass.json
deleted file mode 100644
index 48b213783..000000000
--- a/qemu/tests/qapi-schema/type-bypass.json
+++ /dev/null
@@ -1,2 +0,0 @@
-# Use of 'gen':false allows bypassing type system
-{ 'command': 'unsafe', 'data': { 'arg': '**' }, 'returns': '**', 'gen': false }
diff --git a/qemu/tests/qapi-schema/type-bypass.out b/qemu/tests/qapi-schema/type-bypass.out
deleted file mode 100644
index eaf20f834..000000000
--- a/qemu/tests/qapi-schema/type-bypass.out
+++ /dev/null
@@ -1,3 +0,0 @@
-[OrderedDict([('command', 'unsafe'), ('data', OrderedDict([('arg', '**')])), ('returns', '**'), ('gen', False)])]
-[]
-[]
diff --git a/qemu/tests/qapi-schema/union-bad-branch.err b/qemu/tests/qapi-schema/union-bad-branch.err
deleted file mode 100644
index 882273556..000000000
--- a/qemu/tests/qapi-schema/union-bad-branch.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/union-bad-branch.json:6: Union 'MyUnion' member 'ONE' clashes with 'one'
diff --git a/qemu/tests/qapi-schema/union-bad-branch.json b/qemu/tests/qapi-schema/union-bad-branch.json
deleted file mode 100644
index 913aa38bc..000000000
--- a/qemu/tests/qapi-schema/union-bad-branch.json
+++ /dev/null
@@ -1,8 +0,0 @@
-# we reject normal unions where branches would collide in C
-{ 'struct': 'One',
-  'data': { 'string': 'str' } }
-{ 'struct': 'Two',
-  'data': { 'number': 'int' } }
-{ 'union': 'MyUnion',
-  'data': { 'one': 'One',
-            'ONE': 'Two' } }
diff --git a/qemu/tests/qapi-schema/union-base-no-discriminator.err b/qemu/tests/qapi-schema/union-base-no-discriminator.err
index fc8b79c45..8b7a24260 100644
--- a/qemu/tests/qapi-schema/union-base-no-discriminator.err
+++ b/qemu/tests/qapi-schema/union-base-no-discriminator.err
@@ -1 +1 @@
-tests/qapi-schema/union-base-no-discriminator.json:11: Union 'TestUnion' requires a discriminator to go along with base
+tests/qapi-schema/union-base-no-discriminator.json:11: Simple union 'TestUnion' must not have a base
diff --git a/qemu/tests/qapi-schema/union-branch-case.err b/qemu/tests/qapi-schema/union-branch-case.err
new file mode 100644
index 000000000..11521901d
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-branch-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-branch-case.json:2: 'Branch' (branch of NoWayThisWillGetWhitelisted) should not use uppercase
diff --git a/qemu/tests/qapi-schema/union-branch-case.exit b/qemu/tests/qapi-schema/union-branch-case.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-branch-case.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/union-branch-case.json b/qemu/tests/qapi-schema/union-branch-case.json
new file mode 100644
index 000000000..e6565dc3b
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-branch-case.json
@@ -0,0 +1,2 @@
+# Branch names should be 'lower-case' unless the union is whitelisted
+{ 'union': 'NoWayThisWillGetWhitelisted', 'data': { 'Branch': 'int' } }
diff --git a/qemu/tests/qapi-schema/union-branch-case.out b/qemu/tests/qapi-schema/union-branch-case.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-branch-case.out
diff --git a/qemu/tests/qapi-schema/union-clash-branches.err b/qemu/tests/qapi-schema/union-clash-branches.err
new file mode 100644
index 000000000..e5b21135b
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-clash-branches.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-clash-branches.json:4: 'a_b' (branch of TestUnion) collides with 'a-b' (branch of TestUnion)
diff --git a/qemu/tests/qapi-schema/union-clash-branches.exit b/qemu/tests/qapi-schema/union-clash-branches.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-clash-branches.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/union-clash-branches.json b/qemu/tests/qapi-schema/union-clash-branches.json
new file mode 100644
index 000000000..3bece8c94
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-clash-branches.json
@@ -0,0 +1,5 @@
+# Union branch name collision
+# Reject a union that would result in a collision in generated C names (this
+# would try to generate two members 'a_b').
+{ 'union': 'TestUnion',
+  'data': { 'a-b': 'int', 'a_b': 'str' } }
diff --git a/qemu/tests/qapi-schema/union-clash-branches.out b/qemu/tests/qapi-schema/union-clash-branches.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-clash-branches.out
diff --git a/qemu/tests/qapi-schema/union-empty.err b/qemu/tests/qapi-schema/union-empty.err
new file mode 100644
index 000000000..12c20221b
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-empty.json:2: Union 'Union' cannot have empty 'data'
diff --git a/qemu/tests/qapi-schema/union-empty.exit b/qemu/tests/qapi-schema/union-empty.exit
new file mode 100644
index 000000000..d00491fd7
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/qemu/tests/qapi-schema/union-empty.json b/qemu/tests/qapi-schema/union-empty.json
new file mode 100644
index 000000000..1f0b13ca2
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-empty.json
@@ -0,0 +1,2 @@
+# unions cannot be empty
+{ 'union': 'Union', 'data': { } }
diff --git a/qemu/tests/qapi-schema/union-empty.out b/qemu/tests/qapi-schema/union-empty.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/qemu/tests/qapi-schema/union-empty.out
diff --git a/qemu/tests/qapi-schema/union-invalid-base.err b/qemu/tests/qapi-schema/union-invalid-base.err
index 9f637963e..03d7b97a9 100644
--- a/qemu/tests/qapi-schema/union-invalid-base.err
+++ b/qemu/tests/qapi-schema/union-invalid-base.err
@@ -1 +1 @@
-tests/qapi-schema/union-invalid-base.json:8: Base 'int' is not a valid struct
+tests/qapi-schema/union-invalid-base.json:8: 'base' for union 'TestUnion' cannot use built-in type 'int'
diff --git a/qemu/tests/qapi-schema/union-max.err b/qemu/tests/qapi-schema/union-max.err
deleted file mode 100644
index 55ce4399d..000000000
--- a/qemu/tests/qapi-schema/union-max.err
+++ /dev/null
@@ -1 +0,0 @@
-tests/qapi-schema/union-max.json:2: Union 'Union' member 'max' clashes with '(automatic)'
diff --git a/qemu/tests/qapi-schema/union-max.json b/qemu/tests/qapi-schema/union-max.json
deleted file mode 100644
index d6ad98699..000000000
--- a/qemu/tests/qapi-schema/union-max.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# we reject 'max' branch in a union, for collision with C enum
-{ 'union': 'Union',
-  'data': { 'max': 'int' } }
diff --git a/qemu/tests/qemu-iotests/001 b/qemu/tests/qemu-iotests/001
index 4e1646941..ffd14e2ce 100755
--- a/qemu/tests/qemu-iotests/001
+++ b/qemu/tests/qemu-iotests/001
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/002 b/qemu/tests/qemu-iotests/002
index 6a865aac7..d4f8e91b9 100755
--- a/qemu/tests/qemu-iotests/002
+++ b/qemu/tests/qemu-iotests/002
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/003 b/qemu/tests/qemu-iotests/003
index 98638d4ce..19889b9fc 100755
--- a/qemu/tests/qemu-iotests/003
+++ b/qemu/tests/qemu-iotests/003
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/004 b/qemu/tests/qemu-iotests/004
index 2ad77ed51..67e1beb20 100755
--- a/qemu/tests/qemu-iotests/004
+++ b/qemu/tests/qemu-iotests/004
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/005 b/qemu/tests/qemu-iotests/005
index ba1236dfb..444737751 100755
--- a/qemu/tests/qemu-iotests/005
+++ b/qemu/tests/qemu-iotests/005
@@ -28,7 +28,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/007 b/qemu/tests/qemu-iotests/007
index 7b5aff59b..fa543eeb7 100755
--- a/qemu/tests/qemu-iotests/007
+++ b/qemu/tests/qemu-iotests/007
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/008 b/qemu/tests/qemu-iotests/008
index 2d28efd42..8e89d74fe 100755
--- a/qemu/tests/qemu-iotests/008
+++ b/qemu/tests/qemu-iotests/008
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/009 b/qemu/tests/qemu-iotests/009
index 57a43f5a1..16e4475ca 100755
--- a/qemu/tests/qemu-iotests/009
+++ b/qemu/tests/qemu-iotests/009
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/010 b/qemu/tests/qemu-iotests/010
index 896a0058f..151dac238 100755
--- a/qemu/tests/qemu-iotests/010
+++ b/qemu/tests/qemu-iotests/010
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/011 b/qemu/tests/qemu-iotests/011
index 1c5158af4..f8d044ec8 100755
--- a/qemu/tests/qemu-iotests/011
+++ b/qemu/tests/qemu-iotests/011
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/012 b/qemu/tests/qemu-iotests/012
index 7c5b6892d..d1d3f2209 100755
--- a/qemu/tests/qemu-iotests/012
+++ b/qemu/tests/qemu-iotests/012
@@ -27,7 +27,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/013 b/qemu/tests/qemu-iotests/013
index ea3cab91d..d013f87da 100755
--- a/qemu/tests/qemu-iotests/013
+++ b/qemu/tests/qemu-iotests/013
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/014 b/qemu/tests/qemu-iotests/014
index b23c2db9b..2ea79e8c8 100755
--- a/qemu/tests/qemu-iotests/014
+++ b/qemu/tests/qemu-iotests/014
@@ -27,7 +27,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/015 b/qemu/tests/qemu-iotests/015
index 6f2609524..aaf9c3f41 100755
--- a/qemu/tests/qemu-iotests/015
+++ b/qemu/tests/qemu-iotests/015
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/017 b/qemu/tests/qemu-iotests/017
index 3af3cdfbc..e3f9e7596 100755
--- a/qemu/tests/qemu-iotests/017
+++ b/qemu/tests/qemu-iotests/017
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/017.out b/qemu/tests/qemu-iotests/017.out
index 7c409fc5b..8fc924194 100644
--- a/qemu/tests/qemu-iotests/017.out
+++ b/qemu/tests/qemu-iotests/017.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/qemu/tests/qemu-iotests/018 b/qemu/tests/qemu-iotests/018
index d8a7d435a..1d39d35c4 100755
--- a/qemu/tests/qemu-iotests/018
+++ b/qemu/tests/qemu-iotests/018
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -66,8 +65,8 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-TEST_IMG=$TEST_IMG_SAVE
-_make_test_img -b "$TEST_IMG.base" 6G
+TEST_IMG="$TEST_IMG_SAVE.orig"
+_make_test_img -b "$TEST_IMG_SAVE.base" 6G
 
 echo "Filling test image"
 echo
@@ -81,8 +80,8 @@ for offset in $TEST_OFFSETS; do
 done
 _check_test_img
 
-mv "$TEST_IMG" "$TEST_IMG.orig"
-$QEMU_IMG convert -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
+TEST_IMG="$TEST_IMG_SAVE"
+$QEMU_IMG convert -f $IMGFMT -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 
 echo "Reading"
 echo
diff --git a/qemu/tests/qemu-iotests/018.out b/qemu/tests/qemu-iotests/018.out
index 39a6011d2..5df966727 100644
--- a/qemu/tests/qemu-iotests/018.out
+++ b/qemu/tests/qemu-iotests/018.out
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/qemu/tests/qemu-iotests/019 b/qemu/tests/qemu-iotests/019
index f5ecbf545..24a789a25 100755
--- a/qemu/tests/qemu-iotests/019
+++ b/qemu/tests/qemu-iotests/019
@@ -27,7 +27,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -54,6 +53,9 @@ _unsupported_imgopts "subformat=monolithicFlat" \
 TEST_OFFSETS="0 4294967296"
 CLUSTER_SIZE=65536
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img 6G
 
 echo "Filling base image"
@@ -71,8 +73,8 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
-_make_test_img -b "$TEST_IMG.base" 6G
+TEST_IMG="$TEST_IMG_SAVE.orig"
+_make_test_img -b "$TEST_IMG_SAVE.base" 6G
 
 echo "Filling test image"
 echo
@@ -86,9 +88,7 @@ for offset in $TEST_OFFSETS; do
 done
 _check_test_img
 
-mv "$TEST_IMG" "$TEST_IMG.orig"
-
-
+TEST_IMG="$TEST_IMG_SAVE"
 
 # Test the conversion twice: One test with the old-style -B option and another
 # one with -o backing_file
@@ -98,7 +98,7 @@ for backing_option in "-B " "-o backing_file="; do
     echo
     echo Testing conversion with $backing_option"$TEST_IMG.base" | _filter_testdir | _filter_imgfmt
     echo
-    $QEMU_IMG convert -O $IMGFMT $backing_option"$TEST_IMG.base" "$TEST_IMG.orig" "$TEST_IMG"
+    $QEMU_IMG convert -f $IMGFMT -O $IMGFMT $backing_option"$TEST_IMG.base" "$TEST_IMG.orig" "$TEST_IMG"
 
     echo "Checking if backing clusters are allocated when they shouldn't"
     echo
diff --git a/qemu/tests/qemu-iotests/019.out b/qemu/tests/qemu-iotests/019.out
index 4695b972d..012426497 100644
--- a/qemu/tests/qemu-iotests/019.out
+++ b/qemu/tests/qemu-iotests/019.out
@@ -1,5 +1,5 @@
 QA output created by 019
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=6442450944
 Filling base image
 
 === IO: pattern 42
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4296015872
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 43
diff --git a/qemu/tests/qemu-iotests/020 b/qemu/tests/qemu-iotests/020
index 2f258dc6e..9c4a68c97 100755
--- a/qemu/tests/qemu-iotests/020
+++ b/qemu/tests/qemu-iotests/020
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -56,6 +55,9 @@ fi
 
 TEST_OFFSETS="0 4294967296"
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img 6G
 
 echo "Filling base image"
@@ -73,7 +75,7 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 _make_test_img -b "$TEST_IMG.base" 6G
 
 echo "Filling test image"
@@ -89,7 +91,7 @@ done
 _check_test_img
 
 $QEMU_IMG commit "$TEST_IMG"
-mv "$TEST_IMG.base" "$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
 
 echo "Reading from the backing file"
 echo
diff --git a/qemu/tests/qemu-iotests/020.out b/qemu/tests/qemu-iotests/020.out
index 71aab1c74..42f6c1b15 100644
--- a/qemu/tests/qemu-iotests/020.out
+++ b/qemu/tests/qemu-iotests/020.out
@@ -1,5 +1,5 @@
 QA output created by 020
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=6442450944
 Filling base image
 
 === IO: pattern 0
@@ -269,7 +269,7 @@ wrote 65536/65536 bytes at offset 4295032832
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/qemu/tests/qemu-iotests/021 b/qemu/tests/qemu-iotests/021
index 1c69024cc..11e8ed718 100755
--- a/qemu/tests/qemu-iotests/021
+++ b/qemu/tests/qemu-iotests/021
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/022 b/qemu/tests/qemu-iotests/022
index d35b8162b..2452a9f86 100755
--- a/qemu/tests/qemu-iotests/022
+++ b/qemu/tests/qemu-iotests/022
@@ -27,7 +27,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/023 b/qemu/tests/qemu-iotests/023
index 9ad06b990..497ae1ed1 100755
--- a/qemu/tests/qemu-iotests/023
+++ b/qemu/tests/qemu-iotests/023
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/024 b/qemu/tests/qemu-iotests/024
index 9bf99e198..e0d77ce2f 100755
--- a/qemu/tests/qemu-iotests/024
+++ b/qemu/tests/qemu-iotests/024
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -60,16 +59,22 @@ CLUSTER_SIZE=65536
 echo "Creating backing file"
 echo
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base_old"
+
 _make_test_img 1G
 io_pattern writev 0 $CLUSTER_SIZE $((2 * CLUSTER_SIZE)) 8 0x11
-mv "$TEST_IMG" "$TEST_IMG.base_old"
+
+TEST_IMG="$TEST_IMG_SAVE.base_new"
 
 echo "Creating new backing file"
 echo
 
 _make_test_img 1G
 io_pattern writev 0 $((2 * CLUSTER_SIZE)) $((4 * CLUSTER_SIZE)) 4 0x22
-mv "$TEST_IMG" "$TEST_IMG.base_new"
+
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 echo "Creating COW image"
 echo
diff --git a/qemu/tests/qemu-iotests/024.out b/qemu/tests/qemu-iotests/024.out
index 521d46942..33cfaf5cf 100644
--- a/qemu/tests/qemu-iotests/024.out
+++ b/qemu/tests/qemu-iotests/024.out
@@ -1,7 +1,7 @@
 QA output created by 024
 Creating backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
+Formatting 'TEST_DIR/t.IMGFMT.base_old', fmt=IMGFMT size=1073741824
 === IO: pattern 0x11
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -21,7 +21,7 @@ wrote 65536/65536 bytes at offset 917504
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Creating new backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
+Formatting 'TEST_DIR/t.IMGFMT.base_new', fmt=IMGFMT size=1073741824
 === IO: pattern 0x22
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -33,7 +33,7 @@ wrote 131072/131072 bytes at offset 786432
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Creating COW image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file='TEST_DIR/t.IMGFMT.base_old'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file=TEST_DIR/t.IMGFMT.base_old
 === IO: pattern 0x33
 wrote 262144/262144 bytes at offset 0
 256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/qemu/tests/qemu-iotests/025 b/qemu/tests/qemu-iotests/025
index 467a4b709..c41370f3b 100755
--- a/qemu/tests/qemu-iotests/025
+++ b/qemu/tests/qemu-iotests/025
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/026 b/qemu/tests/qemu-iotests/026
index 0fc3244c7..f5a7f02b2 100755
--- a/qemu/tests/qemu-iotests/026
+++ b/qemu/tests/qemu-iotests/026
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -66,7 +65,7 @@ for event in \
     \
     l2_load \
     l2_update \
-    l2_alloc.write \
+    l2_alloc_write \
     \
     write_aio \
     \
@@ -126,11 +125,11 @@ CLUSTER_SIZE=512
 
 
 for event in \
-    refblock_alloc.hookup \
-    refblock_alloc.write \
-    refblock_alloc.write_blocks \
-    refblock_alloc.write_table \
-    refblock_alloc.switch_table \
+    refblock_alloc_hookup \
+    refblock_alloc_write \
+    refblock_alloc_write_blocks \
+    refblock_alloc_write_table \
+    refblock_alloc_switch_table \
 
 do
 
@@ -170,9 +169,9 @@ CLUSTER_SIZE=1024
 
 
 for event in \
-    l1_grow.alloc_table \
-    l1_grow.write_table \
-    l1_grow.activate_table \
+    l1_grow_alloc_table \
+    l1_grow_write_table \
+    l1_grow_activate_table \
 
 do
 
diff --git a/qemu/tests/qemu-iotests/026.out b/qemu/tests/qemu-iotests/026.out
index 5e964fb5a..d84d82c11 100644
--- a/qemu/tests/qemu-iotests/026.out
+++ b/qemu/tests/qemu-iotests/026.out
@@ -195,24 +195,24 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write
 Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write -b
 Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
@@ -221,24 +221,24 @@ write failed: Input/output error
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -490,17 +490,17 @@ No errors were found on the image.
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -509,7 +509,7 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -518,41 +518,41 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -561,7 +561,7 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -570,17 +570,17 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -589,7 +589,7 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -598,17 +598,17 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -617,7 +617,7 @@ write failed: No space left on device
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write -b
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
@@ -629,60 +629,60 @@ This means waste of disk space, but no harm to data.
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
 Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: on
+Event: l1_grow_write_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: off
+Event: l1_grow_write_table; errno: 5; imm: off; once: off
 Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: on
+Event: l1_grow_write_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: off
+Event: l1_grow_write_table; errno: 28; imm: off; once: off
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: on
+Event: l1_grow_activate_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: off
+Event: l1_grow_activate_table; errno: 5; imm: off; once: off
 Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
@@ -691,12 +691,12 @@ write failed: Input/output error
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: on
+Event: l1_grow_activate_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: off
+Event: l1_grow_activate_table; errno: 28; imm: off; once: off
 Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
diff --git a/qemu/tests/qemu-iotests/026.out.nocache b/qemu/tests/qemu-iotests/026.out.nocache
index c9d242e9e..9c2c8a948 100644
--- a/qemu/tests/qemu-iotests/026.out.nocache
+++ b/qemu/tests/qemu-iotests/026.out.nocache
@@ -14,6 +14,8 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
@@ -21,6 +23,8 @@ This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
@@ -38,6 +42,8 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -45,6 +51,8 @@ This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -70,7 +78,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 5; imm: off; once: off; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -78,7 +90,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 5; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -102,7 +118,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 28; imm: off; once: off; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 read failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -110,7 +130,11 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 28; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 read failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -118,20 +142,18 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_update; errno: 5; imm: off; once: on; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -140,6 +162,8 @@ This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -150,20 +174,18 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_update; errno: 28; imm: off; once: on; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -172,6 +194,8 @@ This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -179,44 +203,52 @@ wrote 131072/131072 bytes at offset 0
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -234,11 +266,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -254,11 +290,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -274,11 +314,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -294,11 +338,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -314,11 +362,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -334,11 +386,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -354,11 +410,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -374,11 +434,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -394,11 +458,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -414,11 +482,15 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 
@@ -426,116 +498,136 @@ No errors were found on the image.
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 55 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 251 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
@@ -545,64 +637,76 @@ This means waste of disk space, but no harm to data.
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: on
+Event: l1_grow_write_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: off
+Event: l1_grow_write_table; errno: 5; imm: off; once: off
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: on
+Event: l1_grow_write_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: off
+Event: l1_grow_write_table; errno: 28; imm: off; once: off
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: on
+Event: l1_grow_activate_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: off
+Event: l1_grow_activate_table; errno: 5; imm: off; once: off
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 96 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: on
+Event: l1_grow_activate_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: off
+Event: l1_grow_activate_table; errno: 28; imm: off; once: off
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 96 leaked clusters were found on the image.
diff --git a/qemu/tests/qemu-iotests/027 b/qemu/tests/qemu-iotests/027
index 3fa81b83b..08593da77 100755
--- a/qemu/tests/qemu-iotests/027
+++ b/qemu/tests/qemu-iotests/027
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/028 b/qemu/tests/qemu-iotests/028
index a1f4423d4..7783e57c7 100755
--- a/qemu/tests/qemu-iotests/028
+++ b/qemu/tests/qemu-iotests/028
@@ -28,7 +28,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -60,6 +59,9 @@ base_size=$(( image_size - 1024 * 1024 * 1024 ))
 
 offset=$(( base_size - 32 * 1024 ))
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $base_size
 
 echo "Filling base image"
@@ -73,7 +75,7 @@ _check_test_img
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 _make_test_img -b "$TEST_IMG.base" $image_size
 
 echo "Filling test image"
@@ -111,10 +113,12 @@ h=$QEMU_HANDLE
 QEMU_COMM_TIMEOUT=1
 
 # Silence output since it contains the disk image path and QEMU's readline
-# character echoing makes it very hard to filter the output
+# character echoing makes it very hard to filter the output. Plus, there
+# is no telling how many times the command will repeat before succeeding.
 _send_qemu_cmd $h "drive_backup disk ${TEST_IMG}.copy" "(qemu)" >/dev/null
 _send_qemu_cmd $h "" "Formatting" | _filter_img_create
-qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs"
+qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs" >/dev/null
+_send_qemu_cmd $h "info block-jobs" "No active jobs"
 _send_qemu_cmd $h 'quit' ""
 
 # Base image sectors
diff --git a/qemu/tests/qemu-iotests/028.out b/qemu/tests/qemu-iotests/028.out
index 5db167ce7..acd2870ba 100644
--- a/qemu/tests/qemu-iotests/028.out
+++ b/qemu/tests/qemu-iotests/028.out
@@ -1,5 +1,5 @@
 QA output created by 028
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=3221227008
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=3221227008
 Filling base image
 
 === IO: pattern 195
@@ -70,7 +70,7 @@ wrote 512/512 bytes at offset 3221225984
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 196
@@ -468,11 +468,8 @@ No errors were found on the image.
 
 block-backup
 
-Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base' backing_fmt='IMGFMT'
-(qemu)
+Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT
 (qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block-info block-jinfo block-joinfo block-jobinfo block-jobs
-Type backup, device disk: Completed 0 of 4294968832 bytes, speed limit 0 bytes/s
-iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block-info block-jinfo block-joinfo block-jobinfo block-jobs
 No active jobs
 === IO: pattern 195
 read 512/512 bytes at offset 3221194240
diff --git a/qemu/tests/qemu-iotests/029 b/qemu/tests/qemu-iotests/029
index b9cd826c7..e639ac0dd 100755
--- a/qemu/tests/qemu-iotests/029
+++ b/qemu/tests/qemu-iotests/029
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/030 b/qemu/tests/qemu-iotests/030
index 952a524ec..3ac2443e5 100755
--- a/qemu/tests/qemu-iotests/030
+++ b/qemu/tests/qemu-iotests/030
@@ -35,6 +35,7 @@ class TestSingleDrive(iotests.QMPTestCase):
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
         qemu_io('-f', 'raw', '-c', 'write -P 0x1 0 512', backing_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0x1 524288 512', mid_img)
         self.vm = iotests.VM().add_drive("blkdebug::" + test_img)
         self.vm.launch()
 
@@ -90,9 +91,13 @@ class TestSingleDrive(iotests.QMPTestCase):
                          qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
-    def test_stream_partial(self):
+    def test_stream_no_op(self):
         self.assert_no_active_block_jobs()
 
+        # The image map is empty before the operation
+        empty_map = qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img)
+
+        # This is a no-op: no data should ever be copied from the base image
         result = self.vm.qmp('block-stream', device='drive0', base=mid_img)
         self.assert_qmp(result, 'return', {})
 
@@ -101,6 +106,20 @@ class TestSingleDrive(iotests.QMPTestCase):
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
+        self.assertEqual(qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
+                         empty_map, 'image file map changed after a no-op')
+
+    def test_stream_partial(self):
+        self.assert_no_active_block_jobs()
+
+        result = self.vm.qmp('block-stream', device='drive0', base=backing_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_until_completed()
+
+        self.assert_no_active_block_jobs()
+        self.vm.shutdown()
+
         self.assertEqual(qemu_io('-f', iotests.imgfmt, '-c', 'map', mid_img),
                          qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
@@ -245,6 +264,7 @@ class TestEIO(TestErrors):
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_ERROR':
+                    error = True
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/operation', 'read')
 
@@ -257,9 +277,11 @@ class TestEIO(TestErrors):
                     self.assert_qmp(result, 'return', {})
 
                     result = self.vm.qmp('query-block-jobs')
+                    if result == {'return': []}:
+                        # Race; likely already finished. Check.
+                        continue
                     self.assert_qmp(result, 'return[0]/paused', False)
                     self.assert_qmp(result, 'return[0]/io-status', 'ok')
-                    error = True
                 elif event['event'] == 'BLOCK_JOB_COMPLETED':
                     self.assertTrue(error, 'job completed unexpectedly')
                     self.assert_qmp(event, 'data/type', 'stream')
diff --git a/qemu/tests/qemu-iotests/030.out b/qemu/tests/qemu-iotests/030.out
index fa16b5cce..6323079e0 100644
--- a/qemu/tests/qemu-iotests/030.out
+++ b/qemu/tests/qemu-iotests/030.out
@@ -1,5 +1,5 @@
-.............
+..............
 ----------------------------------------------------------------------
-Ran 13 tests
+Ran 14 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/031 b/qemu/tests/qemu-iotests/031
index 2a77ba8cb..1e08abc5e 100755
--- a/qemu/tests/qemu-iotests/031
+++ b/qemu/tests/qemu-iotests/031
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/031.out b/qemu/tests/qemu-iotests/031.out
index fce3ce098..7f5050b81 100644
--- a/qemu/tests/qemu-iotests/031.out
+++ b/qemu/tests/qemu-iotests/031.out
@@ -53,11 +53,6 @@ refcount_order            4
 header_length             72
 
 Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
-Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
@@ -68,7 +63,7 @@ No errors were found on the image.
 
 magic                     0x514649fb
 version                   2
-backing_file_offset       0x128
+backing_file_offset       0x90
 backing_file_size         0x17
 cluster_bits              16
 size                      67108864
@@ -91,11 +86,6 @@ length                    11
 data                      'host_device'
 
 Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
-Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
@@ -126,6 +116,11 @@ refcount_order            4
 header_length             104
 
 Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
diff --git a/qemu/tests/qemu-iotests/032 b/qemu/tests/qemu-iotests/032
index b1ba5c321..24bcb52fc 100755
--- a/qemu/tests/qemu-iotests/032
+++ b/qemu/tests/qemu-iotests/032
@@ -27,7 +27,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/033 b/qemu/tests/qemu-iotests/033
index a61d8ced1..16edcf2f0 100755
--- a/qemu/tests/qemu-iotests/033
+++ b/qemu/tests/qemu-iotests/033
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -57,12 +56,13 @@ do_test()
 	} | $QEMU_IO
 }
 
+for write_zero_cmd in "write -z" "aio_write -z"; do
 for align in 512 4k; do
 	echo
 	echo "== preparing image =="
 	do_test $align "write -P 0xa 0x200 0x400" "$TEST_IMG" | _filter_qemu_io
 	do_test $align "write -P 0xa 0x20000 0x600" "$TEST_IMG" | _filter_qemu_io
-	do_test $align "write -z 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
 
 	echo
 	echo "== verifying patterns (1) =="
@@ -73,7 +73,7 @@ for align in 512 4k; do
 	echo
 	echo "== rewriting zeroes =="
 	do_test $align "write -P 0xb 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
-	do_test $align "write -z 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
 
 	echo
 	echo "== verifying patterns (2) =="
@@ -82,7 +82,7 @@ for align in 512 4k; do
 	echo
 	echo "== rewriting unaligned zeroes =="
 	do_test $align "write -P 0xb 0x0 0x1000" "$TEST_IMG" | _filter_qemu_io
-	do_test $align "write -z 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
 
 	echo
 	echo "== verifying patterns (3) =="
@@ -92,6 +92,7 @@ for align in 512 4k; do
 
 	echo
 done
+done
 
 # success, all done
 echo "*** done"
diff --git a/qemu/tests/qemu-iotests/033.out b/qemu/tests/qemu-iotests/033.out
index c3d18aa45..95929eff7 100644
--- a/qemu/tests/qemu-iotests/033.out
+++ b/qemu/tests/qemu-iotests/033.out
@@ -82,4 +82,86 @@ read 512/512 bytes at offset 512
 read 3072/3072 bytes at offset 1024
 3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
+
+== preparing image ==
+wrote 1024/1024 bytes at offset 512
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1536/1536 bytes at offset 131072
+1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (1) ==
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting zeroes ==
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (2) ==
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+
+== preparing image ==
+wrote 1024/1024 bytes at offset 512
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1536/1536 bytes at offset 131072
+1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (1) ==
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting zeroes ==
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (2) ==
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
 *** done
diff --git a/qemu/tests/qemu-iotests/034 b/qemu/tests/qemu-iotests/034
index 69c785858..c711cfce9 100755
--- a/qemu/tests/qemu-iotests/034
+++ b/qemu/tests/qemu-iotests/034
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,9 +50,13 @@ size=128M
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 $QEMU_IO -c "write -P 0x55 0 1M" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.base"
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/qemu/tests/qemu-iotests/034.out b/qemu/tests/qemu-iotests/034.out
index d12daf206..0764ead8b 100644
--- a/qemu/tests/qemu-iotests/034.out
+++ b/qemu/tests/qemu-iotests/034.out
@@ -1,10 +1,10 @@
 QA output created by 034
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == zero write with backing file ==
 wrote 196608/196608 bytes at offset 65536
diff --git a/qemu/tests/qemu-iotests/035 b/qemu/tests/qemu-iotests/035
index ebe9b8c92..efc38e4d4 100755
--- a/qemu/tests/qemu-iotests/035
+++ b/qemu/tests/qemu-iotests/035
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/036 b/qemu/tests/qemu-iotests/036
index 392f1ef3e..ce638d607 100755
--- a/qemu/tests/qemu-iotests/036
+++ b/qemu/tests/qemu-iotests/036
@@ -28,7 +28,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -57,6 +56,7 @@ _make_test_img 64M
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 63
 
 # Without feature table
+$PYTHON qcow2.py "$TEST_IMG" del-header-ext 0x6803f857
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 _img_info
 
@@ -73,6 +73,7 @@ $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 62
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 63
 
 # Without feature table
+$PYTHON qcow2.py "$TEST_IMG" del-header-ext 0x6803f857
 _img_info
 
 # With feature table containing bit 63
diff --git a/qemu/tests/qemu-iotests/036.out b/qemu/tests/qemu-iotests/036.out
index 5616e37b3..9b009b8c1 100644
--- a/qemu/tests/qemu-iotests/036.out
+++ b/qemu/tests/qemu-iotests/036.out
@@ -22,18 +22,18 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Unknown incompatible feature: 8000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Unknown incompatible feature: 8000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature
 
 === Image with multiple incompatible feature bits ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Unknown incompatible feature: e000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature, Unknown incompatible feature: 6000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature, Unknown incompatible feature: c000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test1, test2, Unknown incompatible feature: 8000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test1, test2, test3
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test2, Unknown incompatible feature: a000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Unknown incompatible feature: e000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature, Unknown incompatible feature: 6000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature, Unknown incompatible feature: c000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test1, test2, Unknown incompatible feature: 8000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test1, test2, test3
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test2, Unknown incompatible feature: a000000000000000
 === Create image with unknown autoclear feature bit ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
@@ -56,6 +56,11 @@ autoclear_features        0x8000000000000000
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 
 === Repair image ===
 
diff --git a/qemu/tests/qemu-iotests/037 b/qemu/tests/qemu-iotests/037
index 9171d8c8a..c476b823d 100755
--- a/qemu/tests/qemu-iotests/037
+++ b/qemu/tests/qemu-iotests/037
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,6 +50,9 @@ size=128M
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 
 function backing_io()
@@ -71,7 +73,7 @@ function backing_io()
 
 backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/qemu/tests/qemu-iotests/037.out b/qemu/tests/qemu-iotests/037.out
index dc40a021a..cd6710c90 100644
--- a/qemu/tests/qemu-iotests/037.out
+++ b/qemu/tests/qemu-iotests/037.out
@@ -1,7 +1,7 @@
 QA output created by 037
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 512
@@ -514,7 +514,7 @@ wrote 512/512 bytes at offset 130048
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 130560
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == COW in a single cluster ==
 wrote 2048/2048 bytes at offset 0
diff --git a/qemu/tests/qemu-iotests/038 b/qemu/tests/qemu-iotests/038
index cfaf00a78..d99a1501d 100755
--- a/qemu/tests/qemu-iotests/038
+++ b/qemu/tests/qemu-iotests/038
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -48,6 +47,9 @@ size=128M
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 
 function backing_io()
@@ -68,7 +70,7 @@ function backing_io()
 
 backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/qemu/tests/qemu-iotests/038.out b/qemu/tests/qemu-iotests/038.out
index e1a7e9441..0bdfb19fa 100644
--- a/qemu/tests/qemu-iotests/038.out
+++ b/qemu/tests/qemu-iotests/038.out
@@ -1,7 +1,7 @@
 QA output created by 038
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 65536
@@ -514,7 +514,7 @@ wrote 65536/65536 bytes at offset 16646144
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 16711680
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 wrote 65536/65536 bytes at offset XXX
diff --git a/qemu/tests/qemu-iotests/039 b/qemu/tests/qemu-iotests/039
index 859705f84..1f4833969 100755
--- a/qemu/tests/qemu-iotests/039
+++ b/qemu/tests/qemu-iotests/039
@@ -28,7 +28,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -47,13 +46,6 @@ _supported_os Linux
 _default_cache_mode "writethrough"
 _supported_cache_modes "writethrough"
 
-_subshell_exec()
-{
-    # Executing crashing commands in a subshell prevents information like the
-    # "Killed" line from being lost
-    (exec "$@")
-}
-
 size=128M
 
 echo
@@ -74,8 +66,8 @@ echo "== Creating a dirty image file =="
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must be set
@@ -109,8 +101,8 @@ echo "== Opening a dirty image read/write should repair it =="
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must be set
@@ -127,8 +119,8 @@ echo "== Creating an image file with lazy_refcounts=off =="
 IMGOPTS="compat=1.1,lazy_refcounts=off"
 _make_test_img $size
 
-_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \
-                        -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
     | _filter_qemu_io
 
 # The dirty bit must not be set since lazy_refcounts=off
@@ -154,6 +146,33 @@ $PYTHON qcow2.py "$TEST_IMG".base dump-header | grep incompatible_features
 _check_test_img
 TEST_IMG="$TEST_IMG".base _check_test_img
 
+echo
+echo "== Changing lazy_refcounts setting at runtime =="
+
+IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=on" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
+IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=off" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
 
 # success, all done
 echo "*** done"
diff --git a/qemu/tests/qemu-iotests/039.out b/qemu/tests/qemu-iotests/039.out
index d09751f9c..32c884694 100644
--- a/qemu/tests/qemu-iotests/039.out
+++ b/qemu/tests/qemu-iotests/039.out
@@ -11,7 +11,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
@@ -46,7 +50,11 @@ read 512/512 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 Rebuilding refcount structure
@@ -60,13 +68,17 @@ incompatible_features     0x0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Killed                  ( exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 incompatible_features     0x0
 No errors were found on the image.
 
 == Committing to a backing file with lazy_refcounts=on ==
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Image committed.
@@ -74,4 +86,30 @@ incompatible_features     0x0
 incompatible_features     0x0
 No errors were found on the image.
 No errors were found on the image.
+
+== Changing lazy_refcounts setting at runtime ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x1
+ERROR cluster 5 refcount=0 reference=1
+ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
+
+2 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x0
+No errors were found on the image.
 *** done
diff --git a/qemu/tests/qemu-iotests/040 b/qemu/tests/qemu-iotests/040
index ea2f98e51..5bdaf3d48 100755
--- a/qemu/tests/qemu-iotests/040
+++ b/qemu/tests/qemu-iotests/040
@@ -41,6 +41,7 @@ class ImageCommitTestCase(iotests.QMPTestCase):
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp_absent(event, 'data/error')
                     self.assert_qmp(event, 'data/type', 'commit')
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/offset', event['data']['len'])
@@ -251,5 +252,34 @@ class TestSetSpeed(ImageCommitTestCase):
 class TestActiveZeroLengthImage(TestSingleDrive):
     image_len = 0
 
+class TestReopenOverlay(ImageCommitTestCase):
+    image_len = 1024 * 1024
+    img0 = os.path.join(iotests.test_dir, '0.img')
+    img1 = os.path.join(iotests.test_dir, '1.img')
+    img2 = os.path.join(iotests.test_dir, '2.img')
+    img3 = os.path.join(iotests.test_dir, '3.img')
+
+    def setUp(self):
+        iotests.create_image(self.img0, self.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img0, self.img1)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img1, self.img2)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img2, self.img3)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xab 0 128K', self.img1)
+        self.vm = iotests.VM().add_drive(self.img3)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.img0)
+        os.remove(self.img1)
+        os.remove(self.img2)
+        os.remove(self.img3)
+
+    # This tests what happens when the overlay image of the 'top' node
+    # needs to be reopened in read-write mode in order to update the
+    # backing image string.
+    def test_reopen_overlay(self):
+        self.run_commit_test(self.img1, self.img0)
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/qemu/tests/qemu-iotests/040.out b/qemu/tests/qemu-iotests/040.out
index 42314e9c0..4fd1c2dcd 100644
--- a/qemu/tests/qemu-iotests/040.out
+++ b/qemu/tests/qemu-iotests/040.out
@@ -1,5 +1,5 @@
-........................
+.........................
 ----------------------------------------------------------------------
-Ran 24 tests
+Ran 25 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/041 b/qemu/tests/qemu-iotests/041
index 3d46ed705..b1c542f99 100755
--- a/qemu/tests/qemu-iotests/041
+++ b/qemu/tests/qemu-iotests/041
@@ -34,14 +34,18 @@ quorum_img3 = os.path.join(iotests.test_dir, 'quorum3.img')
 quorum_repair_img = os.path.join(iotests.test_dir, 'quorum_repair.img')
 quorum_snapshot_file = os.path.join(iotests.test_dir, 'quorum_snapshot.img')
 
-
 class TestSingleDrive(iotests.QMPTestCase):
     image_len = 1 * 1024 * 1024 # MB
+    qmp_cmd = 'drive-mirror'
+    qmp_target = target_img
+    not_found_error = 'DeviceNotFound'
 
     def setUp(self):
         iotests.create_image(backing_img, self.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
         self.vm = iotests.VM().add_drive(test_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -56,8 +60,8 @@ class TestSingleDrive(iotests.QMPTestCase):
     def test_complete(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -70,8 +74,8 @@ class TestSingleDrive(iotests.QMPTestCase):
     def test_cancel(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.cancel_and_wait(force=True)
@@ -82,8 +86,8 @@ class TestSingleDrive(iotests.QMPTestCase):
     def test_cancel_after_ready(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.wait_ready_and_cancel()
@@ -96,8 +100,8 @@ class TestSingleDrive(iotests.QMPTestCase):
     def test_pause(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('block-job-pause', device='drive0')
@@ -123,8 +127,8 @@ class TestSingleDrive(iotests.QMPTestCase):
         self.assert_no_active_block_jobs()
 
         # A small buffer is rounded up automatically
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             buf_size=4096, target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             buf_size=4096, target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -139,8 +143,8 @@ class TestSingleDrive(iotests.QMPTestCase):
 
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,size=%d'
                         % (self.image_len, self.image_len), target_img)
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             buf_size=65536, mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             buf_size=65536, mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -155,8 +159,8 @@ class TestSingleDrive(iotests.QMPTestCase):
 
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
                         % (self.image_len, backing_img), target_img)
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -167,30 +171,85 @@ class TestSingleDrive(iotests.QMPTestCase):
                         'target image does not match source after mirroring')
 
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
-                             target=target_img)
-        self.assert_qmp(result, 'error/class', 'GenericError')
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp(self.qmp_cmd, device='ide1-cd0', sync='full',
+                             target=self.qmp_target)
+        self.assert_qmp(result, 'error/class', self.not_found_error)
 
     def test_image_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_device_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='nonexistent', sync='full',
-                             target=target_img)
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        result = self.vm.qmp(self.qmp_cmd, device='nonexistent', sync='full',
+                             target=self.qmp_target)
+        self.assert_qmp(result, 'error/class', self.not_found_error)
+
+class TestSingleBlockdev(TestSingleDrive):
+    qmp_cmd = 'blockdev-mirror'
+    qmp_target = 'node1'
+    not_found_error = 'GenericError'
+
+    def setUp(self):
+        TestSingleDrive.setUp(self)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        args = {'options':
+                    {'driver': iotests.imgfmt,
+                     'node-name': self.qmp_target,
+                     'file': { 'filename': target_img, 'driver': 'file' } } }
+        result = self.vm.qmp("blockdev-add", **args)
+        self.assert_qmp(result, 'return', {})
+
+    test_large_cluster = None
+    test_image_not_found = None
+    test_small_buffer2 = None
+
+class TestBlockdevAttached(iotests.QMPTestCase):
+    image_len = 1 * 1024 * 1024 # MB
+
+    def setUp(self):
+        iotests.create_image(backing_img, self.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(target_img)
+
+    def test_blockdev_attached(self):
+        self.assert_no_active_block_jobs()
+        args = {'options':
+                    {'driver': iotests.imgfmt,
+                     'id': 'drive1',
+                     'file': { 'filename': target_img, 'driver': 'file' } } }
+        result = self.vm.qmp("blockdev-add", **args)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('blockdev-mirror', device='drive0', sync='full',
+                             target='drive1')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
 class TestSingleDriveZeroLength(TestSingleDrive):
     image_len = 0
     test_small_buffer2 = None
     test_large_cluster = None
 
+class TestSingleBlockdevZeroLength(TestSingleBlockdev):
+    image_len = 0
+
 class TestSingleDriveUnalignedLength(TestSingleDrive):
     image_len = 1025 * 1024
     test_small_buffer2 = None
     test_large_cluster = None
 
+class TestSingleBlockdevUnalignedLength(TestSingleBlockdev):
+    image_len = 1025 * 1024
+
 class TestMirrorNoBacking(iotests.QMPTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
@@ -707,6 +766,9 @@ class TestRepairQuorum(iotests.QMPTestCase):
     def setUp(self):
         self.vm = iotests.VM()
 
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
+
         # Add each individual quorum images
         for i in self.IMAGES:
             qemu_img('create', '-f', iotests.imgfmt, i,
@@ -748,8 +810,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_repair_img)
+        self.assert_has_block_node("repair0", quorum_repair_img)
         # TODO: a better test requiring some QEMU infrastructure will be added
         #       to check that this file is really driven by quorum
         self.vm.shutdown()
@@ -771,8 +832,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
         self.cancel_and_wait(drive="quorum0", force=True)
         # here we check that the last registered quorum file has not been
         # swapped out and unref
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_has_block_node(None, quorum_img3)
         self.vm.shutdown()
 
     def test_cancel_after_ready(self):
@@ -788,10 +848,9 @@ class TestRepairQuorum(iotests.QMPTestCase):
         self.assert_qmp(result, 'return', {})
 
         self.wait_ready_and_cancel(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
         # here we check that the last registered quorum file has not been
         # swapped out and unref
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_has_block_node(None, quorum_img3)
         self.vm.shutdown()
         self.assertTrue(iotests.compare_images(quorum_img2, quorum_repair_img),
                         'target image does not match source after mirroring')
@@ -831,7 +890,11 @@ class TestRepairQuorum(iotests.QMPTestCase):
         if not self.has_quorum():
             return
 
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-mirror', device='drive0', # CD-ROM
+                             sync='full',
                              node_name='repair0',
                              replaces='img1',
                              target=quorum_repair_img, format=iotests.imgfmt)
@@ -877,7 +940,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
                              target=quorum_repair_img, format=iotests.imgfmt)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
-    def test_unexistant_replaces(self):
+    def test_nonexistent_replaces(self):
         if not self.has_quorum():
             return
 
@@ -908,8 +971,7 @@ class TestRepairQuorum(iotests.QMPTestCase):
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_repair_img)
+        self.assert_has_block_node("repair0", quorum_repair_img)
         # TODO: a better test requiring some QEMU infrastructure will be added
         #       to check that this file is really driven by quorum
         self.vm.shutdown()
diff --git a/qemu/tests/qemu-iotests/041.out b/qemu/tests/qemu-iotests/041.out
index 24093bc63..b67d0504a 100644
--- a/qemu/tests/qemu-iotests/041.out
+++ b/qemu/tests/qemu-iotests/041.out
@@ -1,5 +1,5 @@
-......................................................
+............................................................................
 ----------------------------------------------------------------------
-Ran 54 tests
+Ran 76 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/042 b/qemu/tests/qemu-iotests/042
index 94ce3a9cc..351b2830a 100755
--- a/qemu/tests/qemu-iotests/042
+++ b/qemu/tests/qemu-iotests/042
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/043 b/qemu/tests/qemu-iotests/043
index b316b97c0..1c6c22d92 100755
--- a/qemu/tests/qemu-iotests/043
+++ b/qemu/tests/qemu-iotests/043
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/043.out b/qemu/tests/qemu-iotests/043.out
index 012cc008e..b37d2a380 100644
--- a/qemu/tests/qemu-iotests/043.out
+++ b/qemu/tests/qemu-iotests/043.out
@@ -4,20 +4,20 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 == backing file references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 == parent references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.3.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.3.base
 
 == ancestor references another ancestor ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT.2.base' creates an infinite loop.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
 
 == finite chain of length 3 (human) ==
 image: TEST_DIR/t.IMGFMT
@@ -44,6 +44,7 @@ cluster_size: 65536
         "filename": "TEST_DIR/t.IMGFMT",
         "cluster-size": 65536,
         "format": "IMGFMT",
+        "full-backing-filename": "TEST_DIR/t.IMGFMT.2.base",
         "backing-filename": "TEST_DIR/t.IMGFMT.2.base",
         "dirty-flag": false
     },
@@ -52,6 +53,7 @@ cluster_size: 65536
         "filename": "TEST_DIR/t.IMGFMT.2.base",
         "cluster-size": 65536,
         "format": "IMGFMT",
+        "full-backing-filename": "TEST_DIR/t.IMGFMT.1.base",
         "backing-filename": "TEST_DIR/t.IMGFMT.1.base",
         "dirty-flag": false
     },
diff --git a/qemu/tests/qemu-iotests/046 b/qemu/tests/qemu-iotests/046
index e0be46cf2..e528b67cc 100755
--- a/qemu/tests/qemu-iotests/046
+++ b/qemu/tests/qemu-iotests/046
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/046.out b/qemu/tests/qemu-iotests/046.out
index 9d18af53a..ca2c7404a 100644
--- a/qemu/tests/qemu-iotests/046.out
+++ b/qemu/tests/qemu-iotests/046.out
@@ -66,7 +66,7 @@ wrote 65536/65536 bytes at offset 1966080
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 2031616
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 blkdebug: Suspended request 'A'
diff --git a/qemu/tests/qemu-iotests/047 b/qemu/tests/qemu-iotests/047
index c35cd096b..1b8f3d4a6 100755
--- a/qemu/tests/qemu-iotests/047
+++ b/qemu/tests/qemu-iotests/047
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/049 b/qemu/tests/qemu-iotests/049
index 93aa0ea55..fff07604f 100755
--- a/qemu/tests/qemu-iotests/049
+++ b/qemu/tests/qemu-iotests/049
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/049.out b/qemu/tests/qemu-iotests/049.out
index 9f93666c5..4673b67f3 100644
--- a/qemu/tests/qemu-iotests/049.out
+++ b/qemu/tests/qemu-iotests/049.out
@@ -95,17 +95,15 @@ qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1024
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte
 qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for
@@ -120,6 +118,7 @@ qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes.
 
 qemu-img create -f qcow2 -o size=foobar TEST_DIR/t.qcow2
 qemu-img: Parameter 'size' expects a size
+You may use k, M, G or T suffixes for kilobytes, megabytes, gigabytes and terabytes.
 qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 == Check correct interpretation of suffixes for cluster size ==
@@ -157,30 +156,30 @@ Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_si
 == Check compat level option ==
 
 qemu-img create -f qcow2 -o compat=0.10 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.42 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: '0.42'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.42' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.42 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=foobar TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: 'foobar'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='foobar' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=foobar encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == Check preallocation option ==
 
 qemu-img create -f qcow2 -o preallocation=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='off' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=off lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=metadata TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=metadata lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=1234 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: invalid parameter value: 1234
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='1234' lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=1234 lazy_refcounts=off refcount_bits=16
 
 == Check encryption option ==
 
@@ -188,27 +187,21 @@ qemu-img create -f qcow2 -o encryption=off TEST_DIR/t.qcow2 64M
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o encryption=on TEST_DIR/t.qcow2 64M
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=on cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == Check lazy_refcounts option (only with v3) ==
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=on TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=on TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 *** done
diff --git a/qemu/tests/qemu-iotests/050 b/qemu/tests/qemu-iotests/050
index 07802bc49..03b4a5d62 100755
--- a/qemu/tests/qemu-iotests/050
+++ b/qemu/tests/qemu-iotests/050
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,14 +50,19 @@ fi
 echo
 echo "== Creating images =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.old"
+
 size=10M
 _make_test_img $size
 $QEMU_IO -c "write -P 0x40 0 1048576" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.old"
+
+TEST_IMG="$TEST_IMG_SAVE.new"
 
 _make_test_img $size
 $QEMU_IO -c "write -P 0x5a 0 1048576" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.new"
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.old" $size
 $QEMU_IO -c "write -z 0 1048576" "$TEST_IMG" | _filter_qemu_io
diff --git a/qemu/tests/qemu-iotests/050.out b/qemu/tests/qemu-iotests/050.out
index a6cb2e686..3602d580d 100644
--- a/qemu/tests/qemu-iotests/050.out
+++ b/qemu/tests/qemu-iotests/050.out
@@ -1,13 +1,13 @@
 QA output created by 050
 
 == Creating images ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760
+Formatting 'TEST_DIR/t.IMGFMT.old', fmt=IMGFMT size=10485760
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760
+Formatting 'TEST_DIR/t.IMGFMT.new', fmt=IMGFMT size=10485760
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file='TEST_DIR/t.IMGFMT.old'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file=TEST_DIR/t.IMGFMT.old
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/qemu/tests/qemu-iotests/051 b/qemu/tests/qemu-iotests/051
index 4a8055b67..630cb7a11 100755
--- a/qemu/tests/qemu-iotests/051
+++ b/qemu/tests/qemu-iotests/051
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -61,10 +60,11 @@ function do_run_qemu()
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_generated_node_ids
 }
 
 size=128M
+device_id="drive0"
 
 _make_test_img $size
 cp "$TEST_IMG" "$TEST_IMG.orig"
@@ -75,10 +75,10 @@ echo
 echo === Unknown option ===
 echo
 
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=on
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=1234
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=foo
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=on,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=1234,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=foo,if=none,id=$device_id
 
 echo
 echo === Unknown protocol option ===
@@ -108,7 +108,8 @@ echo
 echo === Overriding backing file ===
 echo
 
-echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig" -nodefaults
+echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig",if=none,id=$device_id -nodefaults\
+                  | _filter_generated_node_ids
 
 # Drivers that don't support backing files
 run_qemu -drive file="$TEST_IMG",driver=raw,backing.file.filename="$TEST_IMG.orig"
@@ -139,41 +140,63 @@ echo
 echo === No medium ===
 echo
 
-run_qemu -drive if=floppy
-run_qemu -drive if=ide,media=cdrom
-run_qemu -drive if=scsi,media=cdrom
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive if=floppy
+        run_qemu -drive if=ide,media=cdrom
+        run_qemu -drive if=scsi,media=cdrom
+        run_qemu -drive if=ide
+        run_qemu -drive if=scsi
+        ;;
+     *)
+        ;;
+esac
 
-run_qemu -drive if=ide
 run_qemu -drive if=virtio
-run_qemu -drive if=scsi
 
-run_qemu -drive if=none,id=disk -device ide-cd,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
-
-run_qemu -drive if=none,id=disk -device ide-drive,drive=disk
-run_qemu -drive if=none,id=disk -device ide-hd,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive if=none,id=disk -device ide-cd,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
+        run_qemu -drive if=none,id=disk -device ide-drive,drive=disk
+        run_qemu -drive if=none,id=disk -device ide-hd,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+        ;;
+     *)
+        ;;
+esac
 
 echo
 echo === Read-only ===
 echo
 
-run_qemu -drive file="$TEST_IMG",if=floppy,readonly=on
-run_qemu -drive file="$TEST_IMG",if=ide,media=cdrom,readonly=on
-run_qemu -drive file="$TEST_IMG",if=scsi,media=cdrom,readonly=on
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive file="$TEST_IMG",if=floppy,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=ide,media=cdrom,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=scsi,media=cdrom,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=ide,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=scsi,readonly=on
+        ;;
+     *)
+        ;;
+esac
 
-run_qemu -drive file="$TEST_IMG",if=ide,readonly=on
 run_qemu -drive file="$TEST_IMG",if=virtio,readonly=on
-run_qemu -drive file="$TEST_IMG",if=scsi,readonly=on
-
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-cd,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
 
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-drive,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-hd,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-cd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-drive,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-hd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+        ;;
+     *)
+        ;;
+esac
 
 echo
 echo === Cache modes ===
@@ -182,12 +205,20 @@ echo
 # Cannot use the test image because cache=none might not work on the host FS
 # Use cdrom so that we won't get errors about missing media
 
-run_qemu -drive media=cdrom,cache=none
-run_qemu -drive media=cdrom,cache=directsync
-run_qemu -drive media=cdrom,cache=writeback
-run_qemu -drive media=cdrom,cache=writethrough
-run_qemu -drive media=cdrom,cache=unsafe
-run_qemu -drive media=cdrom,cache=invalid_value
+run_qemu -drive driver=null-co,cache=none
+run_qemu -drive driver=null-co,cache=directsync
+run_qemu -drive driver=null-co,cache=writeback
+run_qemu -drive driver=null-co,cache=writethrough
+run_qemu -drive driver=null-co,cache=unsafe
+run_qemu -drive driver=null-co,cache=invalid_value
+
+# Can't test direct=on here because O_DIRECT might not be supported on this FS
+# Test 142 checks the direct=on cases
+
+for cache in writeback writethrough unsafe invalid_value; do
+    echo -e "info block\ninfo block file\ninfo block backing\ninfo block backing-file" | \
+    run_qemu -drive file="$TEST_IMG",cache=$cache,backing.file.filename="$TEST_IMG.base",backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=$device_id -nodefaults
+done
 
 echo
 echo === Specifying the protocol layer ===
@@ -231,6 +262,24 @@ run_qemu -drive file="$TEST_IMG",iops_size=1234,throttling.iops-size=5678
 run_qemu -drive file="$TEST_IMG",readonly=on,read-only=off
 
 echo
+echo === Catching negative/large throttling values ===
+echo
+
+run_qemu -drive file="$TEST_IMG",iops=-1
+run_qemu -drive file="$TEST_IMG",bps=-2
+run_qemu -drive file="$TEST_IMG",bps_rd=-3
+run_qemu -drive file="$TEST_IMG",bps_rd_max=-3
+run_qemu -drive file="$TEST_IMG",throttling.iops-total=-4
+run_qemu -drive file="$TEST_IMG",throttling.bps-total=-5
+# These are accepted
+run_qemu -drive file="$TEST_IMG",bps=0
+run_qemu -drive file="$TEST_IMG",bps=1
+run_qemu -drive file="$TEST_IMG",bps=1000000000000000
+# While these are not
+run_qemu -drive file="$TEST_IMG",bps=1000000000000001
+run_qemu -drive file="$TEST_IMG",bps=9999999999999999
+
+echo
 echo === Parsing protocol from file name ===
 echo
 
@@ -252,26 +301,30 @@ echo
 
 $QEMU_IO -c "write -P 0x11 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,snapshot=on | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2 -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="file:$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="file:$TEST_IMG",snapshot=on | _filter_qemu_io
+
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,snapshot=on,if=none,id=$device_id\
+                                                 | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,if=none,id=$device_id -snapshot\
+                                                 | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="file:$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="file:$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
 
 # Opening a read-only file r/w with snapshot=on
 chmod u-w "$TEST_IMG"
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
 chmod u+w "$TEST_IMG"
 
 $QEMU_IO -c "read -P 0x11 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=off | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=off,if=none,id=$device_id | _filter_qemu_io
 
 $QEMU_IO -c "read -P 0x22 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo -e 'qemu-io ide0-hd0 "write -P 0x33 0 4k"\ncommit ide0-hd0' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
+echo -e "qemu-io $device_id \"write -P 0x33 0 4k\"\ncommit $device_id" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id\
+                                                                       | _filter_qemu_io
 
 $QEMU_IO -c "read -P 0x33 0 4k" "$TEST_IMG" | _filter_qemu_io
 
diff --git a/qemu/tests/qemu-iotests/051.out b/qemu/tests/qemu-iotests/051.out
index 23c282357..408d613bc 100644
--- a/qemu/tests/qemu-iotests/051.out
+++ b/qemu/tests/qemu-iotests/051.out
@@ -1,20 +1,20 @@
 QA output created by 051
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 === Unknown option ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
 
 === Unknown protocol option ===
@@ -56,10 +56,11 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 === Overriding backing file ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig -nodefaults
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig,if=none,id=drive0 -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
-ide0-hd0: TEST_DIR/t.qcow2 (qcow2)
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
     Cache mode:       writeback
     Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
 (qemu) qququiquit
@@ -107,136 +108,105 @@ QEMU X.Y.Z monitor - type 'help' for more information
 
 === No medium ===
 
-Testing: -drive if=floppy
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
-
-Testing: -drive if=ide,media=cdrom
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
-
-Testing: -drive if=scsi,media=cdrom
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
-
-Testing: -drive if=ide
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: Device needs media, but drive is empty
-QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
-
 Testing: -drive if=virtio
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) QEMU_PROG: -drive if=virtio: Device needs media, but drive is empty
 
-Testing: -drive if=scsi
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: Initialization of device lsi53c895a failed: Device needs media, but drive is empty
 
-Testing: -drive if=none,id=disk -device ide-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
+=== Read-only ===
 
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
+Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive if=none,id=disk -device ide-drive,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
-
-Testing: -drive if=none,id=disk -device ide-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-hd,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
-
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device scsi-disk,drive=disk: Device needs media, but drive is empty
-
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty
 
+=== Cache modes ===
 
-=== Read-only ===
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=floppy,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=ide,media=cdrom,readonly=on
+Testing: -drive driver=null-co,cache=none
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,media=cdrom,readonly=on
+Testing: -drive driver=null-co,cache=directsync
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: Can't use a read-only drive
-QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
+Testing: -drive driver=null-co,cache=writeback
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,readonly=on
+Testing: -drive driver=null-co,cache=writethrough
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-cd,drive=disk
+Testing: -drive driver=null-co,cache=unsafe
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
+Testing: -drive driver=null-co,cache=invalid_value
+QEMU_PROG: -drive driver=null-co,cache=invalid_value: invalid cache option
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-drive,drive=disk
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Can't use a read-only drive
-QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-hd,drive=disk: Can't use a read-only drive
-QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
-
-
-=== Cache modes ===
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writethrough
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
 
-Testing: -drive media=cdrom,cache=none
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
 
-Testing: -drive media=cdrom,cache=directsync
-QEMU X.Y.Z monitor - type 'help' for more information
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
 (qemu) qququiquit
 
-Testing: -drive media=cdrom,cache=writeback
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback, ignore flushes
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
 
-Testing: -drive media=cdrom,cache=writethrough
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qququiquit
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
 
-Testing: -drive media=cdrom,cache=unsafe
-QEMU X.Y.Z monitor - type 'help' for more information
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
 (qemu) qququiquit
 
-Testing: -drive media=cdrom,cache=invalid_value
-QEMU_PROG: -drive media=cdrom,cache=invalid_value: invalid cache option
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0: invalid cache option
 
 
 === Specifying the protocol layer ===
@@ -315,6 +285,45 @@ Testing: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off
 QEMU_PROG: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off: 'read-only' and its alias 'readonly' can't be used at the same time
 
 
+=== Catching negative/large throttling values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=-1
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=-1: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=-2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=-2: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000000
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999: bps/iops/max values must be within [0, 1000000000000000]
+
+
 === Parsing protocol from file name ===
 
 Testing: -hda foo:bar
@@ -342,79 +351,79 @@ QEMU_PROG: -drive file.filename=file:TEST_DIR/t.qcow2: Could not open 'file:TEST
 
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2 -snapshot
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file=file:TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=file:TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
 read 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x2qemu-io ide0-hd0 "write -P 0x22qemu-io ide0-hd0 "write -P 0x22 qemu-io ide0-hd0 "write -P 0x22 0qemu-io ide0-hd0 "write -P 0x22 0 qemu-io ide0-hd0 "write -P 0x22 0 4qemu-io ide0-hd0 "write -P 0x22 0 4kqemu-io ide0-hd0 "write -P 0x22 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) qququiquit
 
 read 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io iqemu-io idqemu-io ideqemu-io ide0qemu-io ide0-qemu-io ide0-hqemu-io ide0-hdqemu-io ide0-hd0qemu-io ide0-hd0 qemu-io ide0-hd0 "qemu-io ide0-hd0 "wqemu-io ide0-hd0 "wrqemu-io ide0-hd0 "wriqemu-io ide0-hd0 "writqemu-io ide0-hd0 "writeqemu-io ide0-hd0 "write qemu-io ide0-hd0 "write -qemu-io ide0-hd0 "write -Pqemu-io ide0-hd0 "write -P qemu-io ide0-hd0 "write -P 0qemu-io ide0-hd0 "write -P 0xqemu-io ide0-hd0 "write -P 0x3qemu-io ide0-hd0 "write -P 0x33qemu-io ide0-hd0 "write -P 0x33 qemu-io ide0-hd0 "write -P 0x33 0qemu-io ide0-hd0 "write -P 0x33 0 qemu-io ide0-hd0 "write -P 0x33 0 4qemu-io ide0-hd0 "write -P 0x33 0 4kqemu-io ide0-hd0 "write -P 0x33 0 4k"
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x3qemu-io drive0 "write -P 0x33qemu-io drive0 "write -P 0x33 qemu-io drive0 "write -P 0x33 0qemu-io drive0 "write -P 0x33 0 qemu-io drive0 "write -P 0x33 0 4qemu-io drive0 "write -P 0x33 0 4kqemu-io drive0 "write -P 0x33 0 4k"
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-(qemu) ccocomcommcommicommitcommit commit icommit idcommit idecommit ide0commit ide0-commit ide0-hcommit ide0-hdcommit ide0-hd0
+(qemu) ccocomcommcommicommitcommit commit dcommit drcommit dricommit drivcommit drivecommit drive0
 (qemu) qququiquit
 
 read 4096/4096 bytes at offset 0
diff --git a/qemu/tests/qemu-iotests/051.pc.out b/qemu/tests/qemu-iotests/051.pc.out
new file mode 100644
index 000000000..ec6d22229
--- /dev/null
+++ b/qemu/tests/qemu-iotests/051.pc.out
@@ -0,0 +1,525 @@
+QA output created by 051
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Unknown option ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+
+=== Unknown protocol option ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+
+=== Invalid format ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=foo: Unknown driver 'foo'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=foo: Unknown driver 'foo'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2: Cannot specify both 'driver' and 'format'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2: Cannot specify both 'driver' and 'format'
+
+
+=== Device without drive ===
+
+Testing: -device virtio-scsi-pci -device scsi-hd
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-hd: drive property not set
+
+
+=== Overriding backing file ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+
+=== Enable and disable lazy refcounting on the command line, plus some invalid values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+
+=== With version 2 images enabling lazy refcounts must fail ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+
+=== No medium ===
+
+Testing: -drive if=floppy
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive if=ide,media=cdrom
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive if=scsi,media=cdrom
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive if=ide
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Device needs media, but drive is empty
+QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
+
+Testing: -drive if=scsi
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Initialization of device lsi53c895a failed: Device needs media, but drive is empty
+
+Testing: -drive if=virtio
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -drive if=virtio: Device needs media, but drive is empty
+
+Testing: -drive if=none,id=disk -device ide-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive if=none,id=disk -device ide-drive,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
+QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
+
+Testing: -drive if=none,id=disk -device ide-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-hd,drive=disk: Device needs media, but drive is empty
+QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-disk,drive=disk: Device needs media, but drive is empty
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty
+
+
+=== Read-only ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=floppy,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=ide,media=cdrom,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,media=cdrom,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Can't use a read-only drive
+QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-drive,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: Can't use a read-only drive
+QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-hd,drive=disk: Can't use a read-only drive
+QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+
+=== Cache modes ===
+
+Testing: -drive driver=null-co,cache=none
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive driver=null-co,cache=directsync
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive driver=null-co,cache=writeback
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive driver=null-co,cache=writethrough
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive driver=null-co,cache=unsafe
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive driver=null-co,cache=invalid_value
+QEMU_PROG: -drive driver=null-co,cache=invalid_value: invalid cache option
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writethrough
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo block
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback, ignore flushes
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block finfo block fiinfo block filinfo block file
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backing
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) iininfinfoinfo info binfo blinfo bloinfo blocinfo blockinfo block info block binfo block bainfo block bacinfo block backinfo block backiinfo block backininfo block backinginfo block backing-info block backing-finfo block backing-fiinfo block backing-filinfo block backing-file
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0: invalid cache option
+
+
+=== Specifying the protocol layer ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.driver=file
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+
+=== Leaving out required options ===
+
+Testing: -drive driver=file
+QEMU_PROG: -drive driver=file: The 'file' block driver requires a file name
+
+Testing: -drive driver=nbd
+QEMU_PROG: -drive driver=nbd: one of path and host must be specified.
+
+Testing: -drive driver=raw
+QEMU_PROG: -drive driver=raw: Can't use 'raw' as a block driver for the protocol level
+
+Testing: -drive file.driver=file
+QEMU_PROG: -drive file.driver=file: The 'file' block driver requires a file name
+
+Testing: -drive file.driver=nbd
+QEMU_PROG: -drive file.driver=nbd: one of path and host must be specified.
+
+Testing: -drive file.driver=raw
+QEMU_PROG: -drive file.driver=raw: Can't use 'raw' as a block driver for the protocol level
+
+Testing: -drive foo=bar
+QEMU_PROG: -drive foo=bar: Must specify either driver or file
+
+
+=== Specifying both an option and its legacy alias ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678: 'throttling.iops-total' and its alias 'iops' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678: 'throttling.iops-read' and its alias 'iops_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678: 'throttling.iops-write' and its alias 'iops_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678: 'throttling.bps-total' and its alias 'bps' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678: 'throttling.bps-read' and its alias 'bps_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678: 'throttling.bps-write' and its alias 'bps_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678: 'throttling.iops-total-max' and its alias 'iops_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678: 'throttling.iops-read-max' and its alias 'iops_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678: 'throttling.iops-write-max' and its alias 'iops_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678: 'throttling.bps-total-max' and its alias 'bps_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678: 'throttling.bps-read-max' and its alias 'bps_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678: 'throttling.bps-write-max' and its alias 'bps_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678: 'throttling.iops-size' and its alias 'iops_size' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off: 'read-only' and its alias 'readonly' can't be used at the same time
+
+
+=== Catching negative/large throttling values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=-1
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=-1: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=-2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=-2: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000000
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999: bps/iops/max values must be within [0, 1000000000000000]
+
+
+=== Parsing protocol from file name ===
+
+Testing: -hda foo:bar
+QEMU_PROG: -hda foo:bar: Unknown protocol 'foo'
+
+Testing: -drive file=foo:bar
+QEMU_PROG: -drive file=foo:bar: Unknown protocol 'foo'
+
+Testing: -drive file.filename=foo:bar
+QEMU_PROG: -drive file.filename=foo:bar: Could not open 'foo:bar': No such file or directory
+
+Testing: -hda file:TEST_DIR/t.qcow2
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=file:TEST_DIR/t.qcow2
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file.filename=file:TEST_DIR/t.qcow2
+QEMU_PROG: -drive file.filename=file:TEST_DIR/t.qcow2: Could not open 'file:TEST_DIR/t.qcow2': No such file or directory
+
+
+=== Snapshot mode ===
+
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file=file:TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x2qemu-io drive0 "write -P 0x22qemu-io drive0 "write -P 0x22 qemu-io drive0 "write -P 0x22 0qemu-io drive0 "write -P 0x22 0 qemu-io drive0 "write -P 0x22 0 4qemu-io drive0 "write -P 0x22 0 4kqemu-io drive0 "write -P 0x22 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) qququiquit
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qqeqemqemuqemu-qemu-iqemu-ioqemu-io qemu-io dqemu-io drqemu-io driqemu-io drivqemu-io driveqemu-io drive0qemu-io drive0 qemu-io drive0 "qemu-io drive0 "wqemu-io drive0 "wrqemu-io drive0 "wriqemu-io drive0 "writqemu-io drive0 "writeqemu-io drive0 "write qemu-io drive0 "write -qemu-io drive0 "write -Pqemu-io drive0 "write -P qemu-io drive0 "write -P 0qemu-io drive0 "write -P 0xqemu-io drive0 "write -P 0x3qemu-io drive0 "write -P 0x33qemu-io drive0 "write -P 0x33 qemu-io drive0 "write -P 0x33 0qemu-io drive0 "write -P 0x33 0 qemu-io drive0 "write -P 0x33 0 4qemu-io drive0 "write -P 0x33 0 4kqemu-io drive0 "write -P 0x33 0 4k"
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) ccocomcommcommicommitcommit commit dcommit drcommit dricommit drivcommit drivecommit drive0
+(qemu) qququiquit
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/qemu/tests/qemu-iotests/052 b/qemu/tests/qemu-iotests/052
index 61959e286..4b647242d 100755
--- a/qemu/tests/qemu-iotests/052
+++ b/qemu/tests/qemu-iotests/052
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/053 b/qemu/tests/qemu-iotests/053
index e589e5f12..2a04f5f55 100755
--- a/qemu/tests/qemu-iotests/053
+++ b/qemu/tests/qemu-iotests/053
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/054 b/qemu/tests/qemu-iotests/054
index bd94153d6..bf47ef9fa 100755
--- a/qemu/tests/qemu-iotests/054
+++ b/qemu/tests/qemu-iotests/054
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/055 b/qemu/tests/qemu-iotests/055
index 017a609f3..c8e357870 100755
--- a/qemu/tests/qemu-iotests/055
+++ b/qemu/tests/qemu-iotests/055
@@ -42,6 +42,8 @@ class TestSingleDrive(iotests.QMPTestCase):
         qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
         self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -104,12 +106,18 @@ class TestSingleDrive(iotests.QMPTestCase):
         self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
 
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-backup', device='ide1-cd0',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-backup', device='drive2', # CD-ROM
                              target=target_img, sync='full')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_medium_not_found_blockdev_backup(self):
-        result = self.vm.qmp('blockdev-backup', device='ide1-cd0',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('blockdev-backup', device='drive2', # CD-ROM
                              target='drive1', sync='full')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
@@ -249,6 +257,8 @@ class TestSingleTransaction(iotests.QMPTestCase):
         qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
         self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -323,9 +333,12 @@ class TestSingleTransaction(iotests.QMPTestCase):
         self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
 
     def do_test_medium_not_found(self, cmd, target):
+        if iotests.qemu_default_machine != 'pc':
+            return
+
         result = self.vm.qmp('transaction', actions=[{
                 'type': cmd,
-                'data': { 'device': 'ide1-cd0',
+                'data': { 'device': 'drive2', # CD-ROM
                           'target': target,
                           'sync': 'full' },
             }
diff --git a/qemu/tests/qemu-iotests/056 b/qemu/tests/qemu-iotests/056
index 54e4bd069..04f2c3c84 100755
--- a/qemu/tests/qemu-iotests/056
+++ b/qemu/tests/qemu-iotests/056
@@ -82,6 +82,31 @@ class TestSyncModesNoneAndTop(iotests.QMPTestCase):
         time.sleep(1)
         self.assertEqual(-1, qemu_io('-c', 'read -P0x41 0 512', target_img).find("verification failed"))
 
+class TestBeforeWriteNotifier(iotests.QMPTestCase):
+    def setUp(self):
+        self.vm = iotests.VM().add_drive_raw("file=blkdebug::null-co://,id=drive0,align=65536,driver=blkdebug")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(target_img)
+
+    def test_before_write_notifier(self):
+        self.vm.pause_drive("drive0")
+        result = self.vm.qmp('drive-backup', device='drive0',
+                             sync='full', target=target_img,
+                             format="file", speed=1)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('block-job-pause', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        # Speed is low enough that this must be an uncopied range, which will
+        # trigger the before write notifier
+        self.vm.hmp_qemu_io('drive0', 'aio_write -P 1 512512 512')
+        self.vm.resume_drive("drive0")
+        result = self.vm.qmp('block-job-resume', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        event = self.cancel_and_wait()
+        self.assert_qmp(event, 'data/type', 'backup')
 
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/qemu/tests/qemu-iotests/056.out b/qemu/tests/qemu-iotests/056.out
index fbc63e62f..8d7e99670 100644
--- a/qemu/tests/qemu-iotests/056.out
+++ b/qemu/tests/qemu-iotests/056.out
@@ -1,5 +1,5 @@
-..
+...
 ----------------------------------------------------------------------
-Ran 2 tests
+Ran 3 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/058 b/qemu/tests/qemu-iotests/058
index f2bdd0bff..2253c6a6d 100755
--- a/qemu/tests/qemu-iotests/058
+++ b/qemu/tests/qemu-iotests/058
@@ -27,16 +27,21 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 nbd_unix_socket=$TEST_DIR/test_qemu_nbd_socket
 nbd_snapshot_img="nbd:unix:$nbd_unix_socket"
+rm -f "${TEST_DIR}/qemu-nbd.pid"
 
 _cleanup_nbd()
 {
-    if [ -n "$NBD_SNAPSHOT_PID" ]; then
-        kill "$NBD_SNAPSHOT_PID"
+    local NBD_SNAPSHOT_PID
+    if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+        read NBD_SNAPSHOT_PID < "${TEST_DIR}/qemu-nbd.pid"
+        rm -f "${TEST_DIR}/qemu-nbd.pid"
+        if [ -n "$NBD_SNAPSHOT_PID" ]; then
+            kill "$NBD_SNAPSHOT_PID"
+        fi
     fi
     rm -f "$nbd_unix_socket"
 }
@@ -60,7 +65,6 @@ _export_nbd_snapshot()
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l $1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
@@ -68,7 +72,6 @@ _export_nbd_snapshot1()
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l snapshot.name=$1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
diff --git a/qemu/tests/qemu-iotests/059 b/qemu/tests/qemu-iotests/059
index 0ded0c3da..6655aaf38 100755
--- a/qemu/tests/qemu-iotests/059
+++ b/qemu/tests/qemu-iotests/059
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -133,6 +132,16 @@ $QEMU_IO -c "write -P 0xa 900G 512" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -v 900G 1024" "$TEST_IMG" | _filter_qemu_io
 
 echo
+echo "=== Testing qemu-img map on extents ==="
+for fmt in monolithicSparse twoGbMaxExtentSparse; do
+    IMGOPTS="subformat=$fmt" _make_test_img 31G
+    $QEMU_IO -c "write 65024 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IO -c "write 2147483136 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IO -c "write 5G 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IMG map "$TEST_IMG" | _filter_testdir
+done
+
+echo
 echo "=== Testing afl image with a very large capacity ==="
 _use_sample_img afl9.vmdk.bz2
 _img_info
diff --git a/qemu/tests/qemu-iotests/059.out b/qemu/tests/qemu-iotests/059.out
index 67e3cf57e..678adb437 100644
--- a/qemu/tests/qemu-iotests/059.out
+++ b/qemu/tests/qemu-iotests/059.out
@@ -2,31 +2,31 @@ QA output created by 059
 
 === Testing invalid granularity ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: Invalid granularity, image may be corrupt
+can't open device TEST_DIR/t.vmdk: Invalid granularity, image may be corrupt
 no file open, try 'help open'
 
 === Testing too big L2 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: L2 table size too big
+can't open device TEST_DIR/t.vmdk: L2 table size too big
 no file open, try 'help open'
 
 === Testing too big L1 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
+can't open device TEST_DIR/t.vmdk: L1 size too big
 no file open, try 'help open'
 
 === Testing monolithicFlat creation and opening ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 2.0G (2147483648 bytes)
 
 === Testing monolithicFlat with zeroed_grain ===
 qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 
 === Testing big twoGbMaxExtentFlat ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
 image: TEST_DIR/t.vmdk
 file format: vmdk
 virtual size: 1.0T (1073741824000 bytes)
@@ -2038,12 +2038,10 @@ Format specific information:
             format: FLAT
 
 === Testing malformed VMFS extent description line ===
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent lines:
-RW 12582912 VMFS "dummy.IMGFMT" 1
-
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1
 
 === Testing truncated sparse ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
 
 === Converting to streamOptimized from image with small cluster size===
@@ -2054,8 +2052,8 @@ wrote 512/512 bytes at offset 10240
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing monolithicFlat with internally generated JSON file name ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
+can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
 
 === Testing version 3 ===
 image: TEST_DIR/iotest-version3.IMGFMT
@@ -2264,7 +2262,7 @@ read 512/512 bytes at offset 64931328
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
 image: TEST_DIR/iotest-version3.IMGFMT
 file format: IMGFMT
 virtual size: 4.0T (4398046511104 bytes)
@@ -2337,6 +2335,31 @@ e1000003f0:  00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
 read 1024/1024 bytes at offset 966367641600
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
+=== Testing qemu-img map on extents ===
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+wrote 1024/1024 bytes at offset 65024
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 2147483136
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 5368709120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          Mapped to       File
+0               0x20000         0x3f0000        TEST_DIR/iotest-version3.vmdk
+0x7fff0000      0x20000         0x410000        TEST_DIR/iotest-version3.vmdk
+0x140000000     0x10000         0x430000        TEST_DIR/iotest-version3.vmdk
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
+wrote 1024/1024 bytes at offset 65024
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 2147483136
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 5368709120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          Mapped to       File
+0               0x20000         0x50000         TEST_DIR/iotest-version3-s001.vmdk
+0x7fff0000      0x10000         0x70000         TEST_DIR/iotest-version3-s001.vmdk
+0x80000000      0x10000         0x50000         TEST_DIR/iotest-version3-s002.vmdk
+0x140000000     0x10000         0x50000         TEST_DIR/iotest-version3-s003.vmdk
+
 === Testing afl image with a very large capacity ===
 qemu-img: Can't get size of device 'image': File too large
 *** done
diff --git a/qemu/tests/qemu-iotests/060 b/qemu/tests/qemu-iotests/060
index c81319c16..8e95c450e 100755
--- a/qemu/tests/qemu-iotests/060
+++ b/qemu/tests/qemu-iotests/060
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/060.out b/qemu/tests/qemu-iotests/060.out
index 751118951..5d40206ef 100644
--- a/qemu/tests/qemu-iotests/060.out
+++ b/qemu/tests/qemu-iotests/060.out
@@ -20,7 +20,7 @@ Format specific information:
     lazy refcounts: false
     refcount bits: 16
     corrupt: true
-qemu-io: can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
+can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/qemu/tests/qemu-iotests/061 b/qemu/tests/qemu-iotests/061
index 8d37f8a65..f5678b10c 100755
--- a/qemu/tests/qemu-iotests/061
+++ b/qemu/tests/qemu-iotests/061
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -58,7 +57,8 @@ echo
 echo "=== Testing dirty version downgrade ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
@@ -91,7 +91,8 @@ echo
 echo "=== Testing dirty lazy_refcounts=off ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "lazy_refcounts=off" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
diff --git a/qemu/tests/qemu-iotests/061.out b/qemu/tests/qemu-iotests/061.out
index 5ec248f79..a03732e19 100644
--- a/qemu/tests/qemu-iotests/061.out
+++ b/qemu/tests/qemu-iotests/061.out
@@ -24,6 +24,11 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 magic                     0x514649fb
 version                   2
 backing_file_offset       0x0
@@ -43,11 +48,6 @@ autoclear_features        0x0
 refcount_order            4
 header_length             72
 
-Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
 read 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
@@ -57,6 +57,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -76,6 +81,11 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 ERROR cluster 5 refcount=0 reference=1
 ERROR cluster 6 refcount=0 reference=1
 Rebuilding refcount structure
@@ -100,11 +110,6 @@ autoclear_features        0x0
 refcount_order            4
 header_length             72
 
-Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
 read 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
@@ -131,6 +136,11 @@ autoclear_features        0x40000000000
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 magic                     0x514649fb
 version                   2
 backing_file_offset       0x0
@@ -150,11 +160,6 @@ autoclear_features        0x0
 refcount_order            4
 header_length             72
 
-Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
 No errors were found on the image.
 
 === Testing version upgrade and resize ===
@@ -214,6 +219,11 @@ No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -233,6 +243,11 @@ autoclear_features        0x0
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 ERROR cluster 5 refcount=0 reference=1
 ERROR cluster 6 refcount=0 reference=1
 Rebuilding refcount structure
@@ -281,18 +296,18 @@ No errors were found on the image.
 === Testing invalid configurations ===
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
+qemu-img: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
 qemu-img: Error while amending options: Invalid argument
-Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
+qemu-img: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
 qemu-img: Error while amending options: Invalid argument
-Unknown compatibility level 0.42.
+qemu-img: Unknown compatibility level 0.42
 qemu-img: Error while amending options: Invalid argument
 qemu-img: Invalid parameter 'foo'
-Changing the cluster size is not supported.
+qemu-img: Changing the cluster size is not supported
 qemu-img: Error while amending options: Operation not supported
-Changing the encryption flag is not supported.
+qemu-img: Changing the encryption flag is not supported
 qemu-img: Error while amending options: Operation not supported
-Cannot change preallocation mode.
+qemu-img: Cannot change preallocation mode
 qemu-img: Error while amending options: Operation not supported
 
 === Testing correct handling of unset value ===
@@ -300,7 +315,7 @@ qemu-img: Error while amending options: Operation not supported
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 Should work:
 Should not work:
-Changing the cluster size is not supported.
+qemu-img: Changing the cluster size is not supported
 qemu-img: Error while amending options: Operation not supported
 
 === Testing zero expansion on inactive clusters ===
@@ -334,7 +349,7 @@ read 131072/131072 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 read 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 0
@@ -350,7 +365,7 @@ read 65536/65536 bytes at offset 65536
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 0
@@ -369,7 +384,7 @@ read 65536/65536 bytes at offset 65536
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
@@ -393,7 +408,7 @@ read 67108864/67108864 bytes at offset 0
 === Testing progress report without snapshot ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 1073741824
@@ -408,7 +423,7 @@ No errors were found on the image.
 === Testing progress report with snapshot ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=4294967296
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 1073741824
diff --git a/qemu/tests/qemu-iotests/062 b/qemu/tests/qemu-iotests/062
index 0511246de..051fb9f41 100755
--- a/qemu/tests/qemu-iotests/062
+++ b/qemu/tests/qemu-iotests/062
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/063 b/qemu/tests/qemu-iotests/063
index a47493a07..352e78c77 100755
--- a/qemu/tests/qemu-iotests/063
+++ b/qemu/tests/qemu-iotests/063
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/064 b/qemu/tests/qemu-iotests/064
index 7564563ab..5792fbbc9 100755
--- a/qemu/tests/qemu-iotests/064
+++ b/qemu/tests/qemu-iotests/064
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/066 b/qemu/tests/qemu-iotests/066
index 1c2452b0c..364166d3b 100755
--- a/qemu/tests/qemu-iotests/066
+++ b/qemu/tests/qemu-iotests/066
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/067 b/qemu/tests/qemu-iotests/067
index 83eefa394..c1df48ede 100755
--- a/qemu/tests/qemu-iotests/067
+++ b/qemu/tests/qemu-iotests/067
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -45,10 +44,20 @@ function do_run_qemu()
     echo
 }
 
+# Remove QMP events from (pretty-printed) output. Doesn't handle
+# nested dicts correctly, but we don't get any of those in this test.
+_filter_qmp_events()
+{
+    tr '\n' '\t' | sed -e \
+	's/{\s*"timestamp":\s*{[^}]*},\s*"event":[^,}]*\(,\s*"data":\s*{[^}]*}\)\?\s*}\s*//g' \
+	| tr '\t' '\n'
+}
+
 function run_qemu()
 {
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu \
-                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g' \
+                          | _filter_generated_node_ids | _filter_qmp_events
 }
 
 size=128M
@@ -59,7 +68,7 @@ echo
 echo === -drive/-device and device_del ===
 echo
 
-run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0 <<EOF
+run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0 <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
@@ -76,7 +85,7 @@ run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -94,7 +103,7 @@ run_qemu <<EOF
   "arguments": { "command-line": "drive_add 0 file=$TEST_IMG,format=$IMGFMT,if=none,id=disk" } }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -122,7 +131,7 @@ run_qemu <<EOF
   }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
diff --git a/qemu/tests/qemu-iotests/067.out b/qemu/tests/qemu-iotests/067.out
index 6ff41bc7a..7e25a4902 100644
--- a/qemu/tests/qemu-iotests/067.out
+++ b/qemu/tests/qemu-iotests/067.out
@@ -3,7 +3,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === -drive/-device and device_del ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0
 {
     QMP_VERSION
 }
@@ -40,6 +40,7 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -57,28 +58,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
                 "encryption_key_missing": false
             },
             "type": "unknown"
-        },
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
         }
     ]
 }
@@ -91,93 +70,13 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virti
     }
 }
 {
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "path": "/machine/peripheral/virtio0/virtio-backend"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "device": "virtio0",
-        "path": "/machine/peripheral/virtio0"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "RESET"
-}
-{
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
     "return": {
     }
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "SHUTDOWN"
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
-
 
 === -drive/device_add and device_del ===
 
@@ -217,6 +116,7 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -233,29 +133,6 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
                 "file": "TEST_DIR/t.qcow2",
                 "encryption_key_missing": false
             },
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
             "type": "unknown"
         }
     ]
@@ -273,93 +150,13 @@ Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
     }
 }
 {
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "path": "/machine/peripheral/virtio0/virtio-backend"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "device": "virtio0",
-        "path": "/machine/peripheral/virtio0"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "RESET"
-}
-{
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
     "return": {
     }
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "SHUTDOWN"
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
-
 
 === drive_add/device_add and device_del ===
 
@@ -377,28 +174,6 @@ Testing:
 {
     "return": [
         {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -424,6 +199,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -440,7 +216,6 @@ Testing:
                 "file": "TEST_DIR/t.qcow2",
                 "encryption_key_missing": false
             },
-            "tray_open": false,
             "type": "unknown"
         }
     ]
@@ -458,93 +233,13 @@ Testing:
     }
 }
 {
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "path": "/machine/peripheral/virtio0/virtio-backend"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "device": "virtio0",
-        "path": "/machine/peripheral/virtio0"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "RESET"
-}
-{
     "return": [
-        {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        }
     ]
 }
 {
     "return": {
     }
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "SHUTDOWN"
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
-
 
 === blockdev_add/device_add and device_del ===
 
@@ -563,28 +258,6 @@ Testing:
 {
     "return": [
         {
-            "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -610,6 +283,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -626,7 +300,6 @@ Testing:
                 "file": "TEST_DIR/t.qcow2",
                 "encryption_key_missing": false
             },
-            "tray_open": false,
             "type": "unknown"
         }
     ]
@@ -644,59 +317,9 @@ Testing:
     }
 }
 {
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "path": "/machine/peripheral/virtio0/virtio-backend"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_DELETED",
-    "data": {
-        "device": "virtio0",
-        "path": "/machine/peripheral/virtio0"
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "RESET"
-}
-{
     "return": [
         {
             "io-status": "ok",
-            "device": "ide1-cd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "floppy0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "device": "sd0",
-            "locked": false,
-            "removable": true,
-            "tray_open": false,
-            "type": "unknown"
-        },
-        {
-            "io-status": "ok",
             "device": "disk",
             "locked": false,
             "removable": true,
@@ -722,6 +345,7 @@ Testing:
                 },
                 "iops_wr": 0,
                 "ro": false,
+                "node-name": "NODE_NAME",
                 "backing_file_depth": 0,
                 "drv": "qcow2",
                 "iops": 0,
@@ -738,7 +362,6 @@ Testing:
                 "file": "TEST_DIR/t.qcow2",
                 "encryption_key_missing": false
             },
-            "tray_open": false,
             "type": "unknown"
         }
     ]
@@ -747,34 +370,4 @@ Testing:
     "return": {
     }
 }
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "SHUTDOWN"
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "ide1-cd0",
-        "tray-open": true
-    }
-}
-{
-    "timestamp": {
-        "seconds":  TIMESTAMP,
-        "microseconds":  TIMESTAMP
-    },
-    "event": "DEVICE_TRAY_MOVED",
-    "data": {
-        "device": "floppy0",
-        "tray-open": true
-    }
-}
-
 *** done
diff --git a/qemu/tests/qemu-iotests/068 b/qemu/tests/qemu-iotests/068
index b72e55599..68f6e825d 100755
--- a/qemu/tests/qemu-iotests/068
+++ b/qemu/tests/qemu-iotests/068
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -50,13 +49,23 @@ echo
 echo "=== Saving and reloading a VM state to/from a qcow2 image ==="
 echo
 _make_test_img $IMG_SIZE
+
+case "$QEMU_DEFAULT_MACHINE" in
+  s390-ccw-virtio)
+      platform_parm="-no-shutdown"
+      ;;
+  *)
+      platform_parm=""
+      ;;
+esac
+
 # Give qemu some time to boot before saving the VM state
 bash -c 'sleep 1; echo -e "savevm 0\nquit"' |\
-    $QEMU -nographic -monitor stdio -serial none -hda "$TEST_IMG" |\
+    $QEMU $platform_parm -nographic -monitor stdio -serial none -hda "$TEST_IMG" |\
     _filter_qemu
 # Now try to continue from that VM state (this should just work)
 echo quit |\
-    $QEMU -nographic -monitor stdio -serial none -hda "$TEST_IMG" -loadvm 0 |\
+    $QEMU $platform_parm -nographic -monitor stdio -serial none -hda "$TEST_IMG" -loadvm 0 |\
     _filter_qemu
 
 # success, all done
diff --git a/qemu/tests/qemu-iotests/069 b/qemu/tests/qemu-iotests/069
index ce9e0541b..96e55ef21 100755
--- a/qemu/tests/qemu-iotests/069
+++ b/qemu/tests/qemu-iotests/069
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/069.out b/qemu/tests/qemu-iotests/069.out
index 4d7e63cf8..f97585677 100644
--- a/qemu/tests/qemu-iotests/069.out
+++ b/qemu/tests/qemu-iotests/069.out
@@ -3,6 +3,6 @@ QA output created by 069
 === Creating an image with a backing file and deleting that file ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=131072
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file='TEST_DIR/t.IMGFMT.base'
-qemu-io: can't open device TEST_DIR/t.IMGFMT: Could not open backing file: Could not open 'TEST_DIR/t.IMGFMT.base': No such file or directory
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file=TEST_DIR/t.IMGFMT.base
+can't open device TEST_DIR/t.IMGFMT: Could not open backing file: Could not open 'TEST_DIR/t.IMGFMT.base': No such file or directory
 *** done
diff --git a/qemu/tests/qemu-iotests/070 b/qemu/tests/qemu-iotests/070
index d649ddf9b..8d08d74ff 100755
--- a/qemu/tests/qemu-iotests/070
+++ b/qemu/tests/qemu-iotests/070
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/070.out b/qemu/tests/qemu-iotests/070.out
index ca743831c..131a5b17d 100644
--- a/qemu/tests/qemu-iotests/070.out
+++ b/qemu/tests/qemu-iotests/070.out
@@ -1,8 +1,9 @@
 QA output created by 070
 
 === Verify open image read-only fails, due to dirty log ===
-qemu-io: can't open device TEST_DIR/iotest-dirtylog-10G-4M.vhdx: VHDX image file 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx' opened read-only, but contains a log that needs to be replayed.  To replay the log, execute:
- qemu-img check -r all 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx': Operation not permitted
+can't open device TEST_DIR/iotest-dirtylog-10G-4M.vhdx: VHDX image file 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx' opened read-only, but contains a log that needs to be replayed
+To replay the log, run:
+qemu-img check -r all 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx'
  no file open, try 'help open'
 === Verify open image replays log  ===
 read 18874368/18874368 bytes at offset 0
diff --git a/qemu/tests/qemu-iotests/071 b/qemu/tests/qemu-iotests/071
index 9eaa49b41..bdfd91fef 100755
--- a/qemu/tests/qemu-iotests/071
+++ b/qemu/tests/qemu-iotests/071
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -104,11 +103,20 @@ echo
 echo "=== Testing blkdebug on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
@@ -133,20 +141,29 @@ echo
 echo "=== Testing blkverify on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=$IMGFMT,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_IMG"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": "drive0",
             "raw": {
-                "driver": "raw",
-                "file": {
-                    "driver": "file",
-                    "filename": "$TEST_IMG.base"
-                }
+                "driver": "file",
+                "filename": "$TEST_IMG.base"
             }
         }
     }
@@ -163,11 +180,20 @@ echo
 echo "=== Testing blkverify on existing raw block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG.base,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG.base"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": {
@@ -193,11 +219,20 @@ echo
 echo "=== Testing blkdebug's set-state through QMP ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
diff --git a/qemu/tests/qemu-iotests/071.out b/qemu/tests/qemu-iotests/071.out
index 9205ce251..2b40eadae 100644
--- a/qemu/tests/qemu-iotests/071.out
+++ b/qemu/tests/qemu-iotests/071.out
@@ -42,44 +42,46 @@ read failed: Input/output error
 
 === Testing blkdebug on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 QEMU_PROG: Failed to flush the L2 table cache: Input/output error
 QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
 
 === Testing blkverify on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=IMGFMT,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkverify on existing raw block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT.base,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkdebug's set-state through QMP ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
@@ -90,8 +92,6 @@ read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 QEMU_PROG: Failed to flush the L2 table cache: Input/output error
 QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
diff --git a/qemu/tests/qemu-iotests/072 b/qemu/tests/qemu-iotests/072
index e4a723d73..aa027c7d2 100755
--- a/qemu/tests/qemu-iotests/072
+++ b/qemu/tests/qemu-iotests/072
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/073 b/qemu/tests/qemu-iotests/073
index 392db5499..ad37a617b 100755
--- a/qemu/tests/qemu-iotests/073
+++ b/qemu/tests/qemu-iotests/073
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/073.out b/qemu/tests/qemu-iotests/073.out
index 733d79ca9..de5452492 100644
--- a/qemu/tests/qemu-iotests/073.out
+++ b/qemu/tests/qemu-iotests/073.out
@@ -2,7 +2,7 @@ QA output created by 073
 
 == creating backing file ==
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/qemu/tests/qemu-iotests/075 b/qemu/tests/qemu-iotests/075
index 6117660c5..770d51c6c 100755
--- a/qemu/tests/qemu-iotests/075
+++ b/qemu/tests/qemu-iotests/075
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/075.out b/qemu/tests/qemu-iotests/075.out
index 5f1d6c120..87beae4e3 100644
--- a/qemu/tests/qemu-iotests/075.out
+++ b/qemu/tests/qemu-iotests/075.out
@@ -9,30 +9,30 @@ read 512/512 bytes at offset 1048064
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == block_size must be a multiple of 512 ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size 513 must be a multiple of 512
+can't open device TEST_DIR/simple-pattern.cloop: block_size 513 must be a multiple of 512
 no file open, try 'help open'
 
 == block_size cannot be zero ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size cannot be zero
+can't open device TEST_DIR/simple-pattern.cloop: block_size cannot be zero
 no file open, try 'help open'
 
 == huge block_size ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size 4294966784 must be 64 MB or less
+can't open device TEST_DIR/simple-pattern.cloop: block_size 4294966784 must be 64 MB or less
 no file open, try 'help open'
 
 == offsets_size overflow ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: n_blocks 4294967295 must be 536870911 or less
+can't open device TEST_DIR/simple-pattern.cloop: n_blocks 4294967295 must be 536870911 or less
 no file open, try 'help open'
 
 == refuse images that require too many offsets ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: image requires too many offsets, try increasing block size
+can't open device TEST_DIR/simple-pattern.cloop: image requires too many offsets, try increasing block size
 no file open, try 'help open'
 
 == refuse images with non-monotonically increasing offsets ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: offsets not monotonically increasing at index 1, image file is corrupt
+can't open device TEST_DIR/simple-pattern.cloop: offsets not monotonically increasing at index 1, image file is corrupt
 no file open, try 'help open'
 
 == refuse images with invalid compressed block size ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: invalid compressed block size at index 1, image file is corrupt
+can't open device TEST_DIR/simple-pattern.cloop: invalid compressed block size at index 1, image file is corrupt
 no file open, try 'help open'
 *** done
diff --git a/qemu/tests/qemu-iotests/076 b/qemu/tests/qemu-iotests/076
index c9b55a980..ef9e6a4ff 100755
--- a/qemu/tests/qemu-iotests/076
+++ b/qemu/tests/qemu-iotests/076
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/076.out b/qemu/tests/qemu-iotests/076.out
index b0000aeed..72645b252 100644
--- a/qemu/tests/qemu-iotests/076.out
+++ b/qemu/tests/qemu-iotests/076.out
@@ -5,15 +5,15 @@ read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Negative catalog size ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
+can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Overflow in catalog allocation ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
+can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Zero sectors per track ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Invalid image: Zero sectors per track
+can't open device TEST_DIR/parallels-v1: Invalid image: Zero sectors per track
 no file open, try 'help open'
 
 == Read from a valid v2 image ==
diff --git a/qemu/tests/qemu-iotests/077 b/qemu/tests/qemu-iotests/077
index 42a90850c..4dc680b7f 100755
--- a/qemu/tests/qemu-iotests/077
+++ b/qemu/tests/qemu-iotests/077
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -63,7 +62,7 @@ EOF
 off=0x1000
 for ev in "head" "after_head" "tail" "after_tail"; do
 cat  <<EOF
-break pwritev_rmw.$ev A
+break pwritev_rmw_$ev A
 aio_write -P 10 $((off + 0x200)) 0x200
 wait_break A
 aio_write -P 11 $((off + 0x400)) 0x200
@@ -76,7 +75,7 @@ done
 
 # Chained dependencies
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x5000 0x200
 wait_break A
 aio_write -P 11 0x5200 0x200
@@ -93,10 +92,10 @@ EOF
 
 # Overlapping multiple requests
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x6000 0x200
 wait_break A
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0x7e00 0x200
 wait_break B
 aio_write -P 11 0x6800 0x1000
@@ -107,10 +106,10 @@ aio_flush
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x8000 0x200
 wait_break A
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0x9e00 0x200
 wait_break B
 aio_write -P 11 0x8800 0x1000
@@ -121,11 +120,11 @@ aio_flush
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xa000 0x200
 wait_break A
 aio_write -P 11 0xa800 0x1000
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0xbe00 0x200
 wait_break B
 resume A
@@ -135,11 +134,11 @@ aio_flush
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xc000 0x200
 wait_break A
 aio_write -P 11 0xc800 0x1000
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0xde00 0x200
 wait_break B
 resume B
@@ -150,7 +149,7 @@ EOF
 
 # Only RMW for the tail part
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xe000 0x1800
 wait_break A
 aio_write -P 11 0xf000 0xc00
@@ -163,7 +162,7 @@ cat  <<EOF
 break pwritev A
 aio_write -P 10 0x10000 0x800
 wait_break A
-break pwritev_rmw.after_tail B
+break pwritev_rmw_after_tail B
 aio_write -P 11 0x10000 0x400
 break pwritev_done C
 resume A
diff --git a/qemu/tests/qemu-iotests/078 b/qemu/tests/qemu-iotests/078
index 7be2c3f69..f333e9ac8 100755
--- a/qemu/tests/qemu-iotests/078
+++ b/qemu/tests/qemu-iotests/078
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/078.out b/qemu/tests/qemu-iotests/078.out
index ca18d2ea3..42b8a8301 100644
--- a/qemu/tests/qemu-iotests/078.out
+++ b/qemu/tests/qemu-iotests/078.out
@@ -5,24 +5,24 @@ read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Negative catalog size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too large
+can't open device TEST_DIR/empty.bochs: Catalog size is too large
 no file open, try 'help open'
 
 == Overflow for catalog size * sizeof(uint32_t) ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too large
+can't open device TEST_DIR/empty.bochs: Catalog size is too large
 no file open, try 'help open'
 
 == Too small catalog bitmap for image size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
+can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
+can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
 no file open, try 'help open'
 
 == Negative extent size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Extent size 2147483648 is too large
+can't open device TEST_DIR/empty.bochs: Extent size 2147483648 is too large
 no file open, try 'help open'
 
 == Zero extent size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Extent size must be at least 512
+can't open device TEST_DIR/empty.bochs: Extent size must be at least 512
 no file open, try 'help open'
 *** done
diff --git a/qemu/tests/qemu-iotests/079 b/qemu/tests/qemu-iotests/079
index ade6efa0d..b2e3f7426 100755
--- a/qemu/tests/qemu-iotests/079
+++ b/qemu/tests/qemu-iotests/079
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/079.out b/qemu/tests/qemu-iotests/079.out
index 6dc5d5763..aab922fb3 100644
--- a/qemu/tests/qemu-iotests/079.out
+++ b/qemu/tests/qemu-iotests/079.out
@@ -1,14 +1,14 @@
 QA output created by 079
 === Check option preallocation and cluster_size ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
 qemu-img: TEST_DIR/t.IMGFMT: Cluster size must be a power of two between 512 and 2048k
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
 *** done
diff --git a/qemu/tests/qemu-iotests/080 b/qemu/tests/qemu-iotests/080
index a2c58aebd..55044c700 100755
--- a/qemu/tests/qemu-iotests/080
+++ b/qemu/tests/qemu-iotests/080
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/080.out b/qemu/tests/qemu-iotests/080.out
index 6061d8443..0daac48b1 100644
--- a/qemu/tests/qemu-iotests/080.out
+++ b/qemu/tests/qemu-iotests/080.out
@@ -2,46 +2,46 @@ QA output created by 080
 
 == Huge header size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
+can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
+can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
 no file open, try 'help open'
 
 == Huge unknown header extension ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid backing file offset
+can't open device TEST_DIR/t.qcow2: Invalid backing file offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Header extension too large
+can't open device TEST_DIR/t.qcow2: Header extension too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Header extension too large
+can't open device TEST_DIR/t.qcow2: Header extension too large
 no file open, try 'help open'
 
 == Huge refcount table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Reference count table too large
+can't open device TEST_DIR/t.qcow2: Reference count table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Reference count table too large
+can't open device TEST_DIR/t.qcow2: Reference count table too large
 no file open, try 'help open'
 
 == Misaligned refcount table ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
+can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
 no file open, try 'help open'
 
 == Huge refcount offset ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
+can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
 no file open, try 'help open'
 
 == Invalid snapshot table ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Too many snapshots
+can't open device TEST_DIR/t.qcow2: Too many snapshots
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Too many snapshots
+can't open device TEST_DIR/t.qcow2: Too many snapshots
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
+can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
+can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
 no file open, try 'help open'
 
 == Hitting snapshot table size limit ==
@@ -52,13 +52,13 @@ read 512/512 bytes at offset 0
 
 == Invalid L1 table ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Active L1 table too large
+can't open device TEST_DIR/t.qcow2: Active L1 table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Active L1 table too large
+can't open device TEST_DIR/t.qcow2: Active L1 table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
+can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
+can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
 no file open, try 'help open'
 
 == Invalid L1 table (with internal snapshot in the image) ==
@@ -67,7 +67,7 @@ qemu-img: Could not open 'TEST_DIR/t.IMGFMT': L1 table is too small
 
 == Invalid backing file size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow2: Backing file name too long
+can't open device TEST_DIR/t.qcow2: Backing file name too long
 no file open, try 'help open'
 
 == Invalid L2 entry (huge physical offset) ==
diff --git a/qemu/tests/qemu-iotests/081 b/qemu/tests/qemu-iotests/081
index d9b042cfc..d89fabcdb 100755
--- a/qemu/tests/qemu-iotests/081
+++ b/qemu/tests/qemu-iotests/081
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -53,7 +52,8 @@ function do_run_qemu()
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp | _filter_qemu_io
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp\
+                          | _filter_qemu_io | _filter_generated_node_ids
 }
 
 test_quorum=$($QEMU_IMG --help|grep quorum)
@@ -101,17 +101,29 @@ $QEMU_IO -c "open -o $quorum" -c "read -P 0x32 0 $size" | _filter_qemu_io
 echo
 echo "== checking mixed reference/option specification =="
 
-run_qemu -drive "file=$TEST_DIR/2.raw,format=$IMGFMT,if=none,id=drive2" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive2",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_DIR/2.raw"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "quorum",
             "id": "drive0-quorum",
             "vote-threshold": 2,
             "children": [
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/1.raw"
@@ -119,7 +131,7 @@ run_qemu -drive "file=$TEST_DIR/2.raw,format=$IMGFMT,if=none,id=drive2" <<EOF
                 },
                 "drive2",
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/3.raw"
diff --git a/qemu/tests/qemu-iotests/081.out b/qemu/tests/qemu-iotests/081.out
index 9f57d9d3b..97df69d71 100644
--- a/qemu/tests/qemu-iotests/081.out
+++ b/qemu/tests/qemu-iotests/081.out
@@ -26,18 +26,17 @@ read 10485760/10485760 bytes at offset 0
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == checking mixed reference/option specification ==
-Testing: -drive file=TEST_DIR/2.IMGFMT,format=IMGFMT,if=none,id=drive2
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "", "sectors-count": 20480, "sector-num": 0}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "drive2", "sectors-count": 20480, "sector-num": 0, "type": "read"}}
 read 10485760/10485760 bytes at offset 0
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 == using quorum rewrite corrupted mode ==
diff --git a/qemu/tests/qemu-iotests/082 b/qemu/tests/qemu-iotests/082
index c83e01e7c..ad1d9fadc 100755
--- a/qemu/tests/qemu-iotests/082
+++ b/qemu/tests/qemu-iotests/082
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/082.out b/qemu/tests/qemu-iotests/082.out
index 3a749b825..a952330ba 100644
--- a/qemu/tests/qemu-iotests/082.out
+++ b/qemu/tests/qemu-iotests/082.out
@@ -147,10 +147,10 @@ refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,help TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,help' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,help encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,? TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,?' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,? encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2, -o help TEST_DIR/t.qcow2 128M
 qemu-img: Invalid option list: backing_file=TEST_DIR/t.qcow2,
diff --git a/qemu/tests/qemu-iotests/083 b/qemu/tests/qemu-iotests/083
index 1b2d3f115..bc724ae05 100755
--- a/qemu/tests/qemu-iotests/083
+++ b/qemu/tests/qemu-iotests/083
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -49,17 +48,6 @@ wait_for_tcp_port() {
 	done
 }
 
-filter_nbd() {
-	# nbd.c error messages contain function names and line numbers that are prone
-	# to change.  Message ordering depends on timing between send and receive
-	# callbacks sometimes, making them unreliable.
-	#
-	# Filter out the TCP port number since this changes between runs.
-	sed -e 's#^.*nbd\.c:.*##g' \
-	    -e 's#nbd:127\.0\.0\.1:[^:]*:#nbd:127\.0\.0\.1:PORT:#g' \
-            -e 's#\(exportname=foo\|PORT\): Failed to .*$#\1#'
-}
-
 check_disconnect() {
 	event=$1
 	when=$2
@@ -84,7 +72,7 @@ EOF
 
 	$PYTHON nbd-fault-injector.py $extra_args "127.0.0.1:$port" "$TEST_DIR/nbd-fault-injector.conf" 2>&1 >/dev/null &
 	wait_for_tcp_port "127\\.0\\.0\\.1:$port"
-	$QEMU_IO -c "read 0 512" "$nbd_url" 2>&1 | _filter_qemu_io | filter_nbd
+	$QEMU_IO -c "read 0 512" "$nbd_url" 2>&1 | _filter_qemu_io | _filter_nbd
 
 	echo
 }
diff --git a/qemu/tests/qemu-iotests/083.out b/qemu/tests/qemu-iotests/083.out
index 8c1441bf4..ef3d1e32a 100644
--- a/qemu/tests/qemu-iotests/083.out
+++ b/qemu/tests/qemu-iotests/083.out
@@ -1,138 +1,128 @@
 QA output created by 083
 === Check disconnect before neg1 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after neg1 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 8 neg1 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 16 neg1 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before export ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after export ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 4 export ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 12 export ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 16 export ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before neg2 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after neg2 ===
 
-
 read failed: Input/output error
 
 === Check disconnect 8 neg2 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 10 neg2 ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before request ===
 
-
 read failed: Input/output error
 
 === Check disconnect after request ===
 
-
 read failed: Input/output error
 
 === Check disconnect before reply ===
 
-
 read failed: Input/output error
 
 === Check disconnect after reply ===
 
-
 read failed: Input/output error
 
 === Check disconnect 4 reply ===
 
-
 read failed: Input/output error
 
 === Check disconnect 8 reply ===
 
-
 read failed: Input/output error
 
 === Check disconnect before data ===
 
-
 read failed: Input/output error
 
 === Check disconnect after data ===
 
-
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Check disconnect before neg-classic ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 8 neg-classic ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 16 neg-classic ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 24 neg-classic ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 28 neg-classic ===
 
-qemu-io: can't open device nbd:127.0.0.1:PORT
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect after neg-classic ===
 
-
 read failed: Input/output error
 
 *** done
diff --git a/qemu/tests/qemu-iotests/084 b/qemu/tests/qemu-iotests/084
index 733018d4a..04f2aa9d7 100755
--- a/qemu/tests/qemu-iotests/084
+++ b/qemu/tests/qemu-iotests/084
@@ -26,7 +26,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/085 b/qemu/tests/qemu-iotests/085
index 56cd6f89b..aa77eca77 100755
--- a/qemu/tests/qemu-iotests/085
+++ b/qemu/tests/qemu-iotests/085
@@ -7,6 +7,7 @@
 # snapshots are performed.
 #
 # Copyright (C) 2014 Red Hat, Inc.
+# Copyright (C) 2015 Igalia, S.L.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -34,17 +35,17 @@ status=1	# failure is the default!
 snapshot_virt0="snapshot-v0.qcow2"
 snapshot_virt1="snapshot-v1.qcow2"
 
-MAX_SNAPSHOTS=10
+SNAPSHOTS=10
 
 _cleanup()
 {
     _cleanup_qemu
-    for i in $(seq 1 ${MAX_SNAPSHOTS})
+    for i in $(seq 1 ${SNAPSHOTS})
     do
         rm -f "${TEST_DIR}/${i}-${snapshot_virt0}"
         rm -f "${TEST_DIR}/${i}-${snapshot_virt1}"
     done
-	_cleanup_test_img
+    rm -f "${TEST_IMG}.1" "${TEST_IMG}.2"
 
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -64,7 +65,7 @@ function create_single_snapshot()
 {
     cmd="{ 'execute': 'blockdev-snapshot-sync',
                       'arguments': { 'device': 'virtio0',
-                                     'snapshot-file':'"${TEST_DIR}/${1}-${snapshot_virt0}"',
+                                     'snapshot-file':'${TEST_DIR}/${1}-${snapshot_virt0}',
                                      'format': 'qcow2' } }"
     _send_qemu_cmd $h "${cmd}" "return"
 }
@@ -76,27 +77,60 @@ function create_group_snapshot()
            {'actions': [
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio0',
-                      'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt0}"' } },
+                      'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt0}' } },
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio1',
-                       'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt1}"' } } ]
+                       'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt1}' } } ]
              } }"
 
     _send_qemu_cmd $h "${cmd}" "return"
 }
 
+# ${1}: unique identifier for the snapshot filename
+# ${2}: true: open backing images; false: don't open them (default)
+function add_snapshot_image()
+{
+    if [ "${2}" = "true" ]; then
+        extra_params=""
+    else
+        extra_params="'backing': '', "
+    fi
+    base_image="${TEST_DIR}/$((${1}-1))-${snapshot_virt0}"
+    snapshot_file="${TEST_DIR}/${1}-${snapshot_virt0}"
+    _make_test_img -b "${base_image}" "$size"
+    mv "${TEST_IMG}" "${snapshot_file}"
+    cmd="{ 'execute': 'blockdev-add', 'arguments':
+           { 'options':
+             { 'driver': 'qcow2', 'node-name': 'snap_${1}', ${extra_params}
+               'file':
+               { 'driver': 'file', 'filename': '${snapshot_file}',
+                 'node-name': 'file_${1}' } } } }"
+    _send_qemu_cmd $h "${cmd}" "return"
+}
+
+# ${1}: unique identifier for the snapshot filename
+# ${2}: expected response, defaults to 'return'
+function blockdev_snapshot()
+{
+    cmd="{ 'execute': 'blockdev-snapshot',
+                      'arguments': { 'node': 'virtio0',
+                                     'overlay':'snap_${1}' } }"
+    _send_qemu_cmd $h "${cmd}" "${2:-return}"
+}
+
 size=128M
 
 _make_test_img $size
-mv "${TEST_IMG}" "${TEST_IMG}.orig"
+mv "${TEST_IMG}" "${TEST_IMG}.1"
 _make_test_img $size
+mv "${TEST_IMG}" "${TEST_IMG}.2"
 
 echo
 echo === Running QEMU ===
 echo
 
 qemu_comm_method="qmp"
-_launch_qemu -drive file="${TEST_IMG}.orig",if=virtio -drive file="${TEST_IMG}",if=virtio
+_launch_qemu -drive file="${TEST_IMG}.1",if=virtio -drive file="${TEST_IMG}.2",if=virtio
 h=$QEMU_HANDLE
 
 echo
@@ -105,6 +139,8 @@ echo
 
 _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
 
+# Tests for the blockdev-snapshot-sync command
+
 echo
 echo === Create a single snapshot on virtio0 ===
 echo
@@ -117,7 +153,7 @@ echo === Invalid command - missing device and nodename ===
 echo
 
 _send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
-                         'arguments': { 'snapshot-file':'"${TEST_DIR}/1-${snapshot_virt0}"',
+                         'arguments': { 'snapshot-file':'${TEST_DIR}/1-${snapshot_virt0}',
                                      'format': 'qcow2' } }" "error"
 
 echo
@@ -132,11 +168,75 @@ echo
 echo === Create several transactional group snapshots ===
 echo
 
-for i in $(seq 2 ${MAX_SNAPSHOTS})
+for i in $(seq 2 ${SNAPSHOTS})
 do
     create_group_snapshot ${i}
 done
 
+# Tests for the blockdev-snapshot command
+
+echo
+echo === Create a couple of snapshots using blockdev-snapshot ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+echo
+echo === Invalid command - cannot create a snapshot using a file BDS ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'file_${SNAPSHOTS}' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as active layer ===
+echo
+
+blockdev_snapshot ${SNAPSHOTS} error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio0' }
+                   }" "error"
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio1' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as backing hd ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}-1)) error
+
+echo
+echo === Invalid command - snapshot node has a backing image ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS} true
+blockdev_snapshot ${SNAPSHOTS} error
+
+echo
+echo === Invalid command - The node does not exist ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}+1)) error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'nodevice',
+                                    'overlay':'snap_${SNAPSHOTS}' }
+                   }" "error"
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/qemu/tests/qemu-iotests/085.out b/qemu/tests/qemu-iotests/085.out
index 5eb8b947c..01c78d689 100644
--- a/qemu/tests/qemu-iotests/085.out
+++ b/qemu/tests/qemu-iotests/085.out
@@ -11,7 +11,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === Create a single snapshot on virtio0 ===
 
-Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2.orig' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.1 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
 
 === Invalid command - missing device and nodename ===
@@ -25,31 +25,65 @@ Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file
 
 === Create several transactional group snapshots ===
 
-Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/1-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/1-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
-Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
+
+=== Create a couple of snapshots using blockdev-snapshot ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/10-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/11-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+
+=== Invalid command - cannot create a snapshot using a file BDS ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot does not support backing images"}}
+
+=== Invalid command - snapshot node used as active layer ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio1"}}
+
+=== Invalid command - snapshot node used as backing hd ===
+
+{"error": {"class": "GenericError", "desc": "Node 'snap_11' is busy: node is used as backing hd of 'virtio0'"}}
+
+=== Invalid command - snapshot node has a backing image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/12-snapshot-v0.IMGFMT
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "The snapshot already has a backing image"}}
+
+=== Invalid command - The node does not exist ===
+
+{"error": {"class": "GenericError", "desc": "Cannot find device=snap_14 nor node_name=snap_14"}}
+{"error": {"class": "GenericError", "desc": "Cannot find device=nodevice nor node_name=nodevice"}}
 *** done
diff --git a/qemu/tests/qemu-iotests/086 b/qemu/tests/qemu-iotests/086
index 234eb9a91..cd4494a66 100755
--- a/qemu/tests/qemu-iotests/086
+++ b/qemu/tests/qemu-iotests/086
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -38,7 +37,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
 . ./common.rc
 . ./common.filter
 
-_supported_fmt qcow2
+_supported_fmt qcow2 raw
 _supported_proto file nfs
 _supported_os Linux
 
diff --git a/qemu/tests/qemu-iotests/087 b/qemu/tests/qemu-iotests/087
index 869474994..e7bca37ef 100755
--- a/qemu/tests/qemu-iotests/087
+++ b/qemu/tests/qemu-iotests/087
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -45,7 +44,8 @@ function do_run_qemu()
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu \
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
+                          | _filter_qemu | _filter_imgfmt \
                           | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
 }
 
@@ -54,7 +54,7 @@ size=128M
 _make_test_img $size
 
 echo
-echo === Missing ID ===
+echo === Missing ID and node-name ===
 echo
 
 run_qemu <<EOF
diff --git a/qemu/tests/qemu-iotests/087.out b/qemu/tests/qemu-iotests/087.out
index c71bb3aa4..055c553cd 100644
--- a/qemu/tests/qemu-iotests/087.out
+++ b/qemu/tests/qemu-iotests/087.out
@@ -1,16 +1,14 @@
 QA output created by 087
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
-=== Missing ID ===
+=== Missing ID and node-name ===
 
 Testing:
 QMP_VERSION
 {"return": {}}
-{"error": {"class": "GenericError", "desc": "Block device needs an ID"}}
+{"error": {"class": "GenericError", "desc": "'id' and/or 'node-name' need to be specified for the root node"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Duplicate ID ===
@@ -23,11 +21,9 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "Device name 'test-node' conflicts with an existing node name"}}
 {"error": {"class": "GenericError", "desc": "node-name=disk is conflicting with a device id"}}
 {"error": {"class": "GenericError", "desc": "Duplicate node name"}}
-{"error": {"class": "GenericError", "desc": "node-name=disk3 is conflicting with a device id"}}
+{"error": {"class": "GenericError", "desc": "Device name 'disk3' conflicts with an existing node name"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === aio=native without O_DIRECT ===
@@ -38,52 +34,36 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "aio=native requires cache.direct=true"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Encrypted image ===
 
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
 Testing: -S
 QMP_VERSION
 {"return": {}}
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
+IMGFMT built-in AES encryption is deprecated
+Support for it will be removed in a future release.
+You can use 'qemu-img convert' to switch to an
+unencrypted IMGFMT image, or a LUKS raw image.
 {"error": {"class": "GenericError", "desc": "blockdev-add doesn't support encrypted devices"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 Testing:
 QMP_VERSION
 {"return": {}}
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
+IMGFMT built-in AES encryption is deprecated
+Support for it will be removed in a future release.
+You can use 'qemu-img convert' to switch to an
+unencrypted IMGFMT image, or a LUKS raw image.
 {"error": {"class": "GenericError", "desc": "Guest must be stopped for opening of encrypted image"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Missing driver ===
 
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
 Testing: -S
 QMP_VERSION
@@ -91,7 +71,5 @@ QMP_VERSION
 {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'driver', expected: string"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 *** done
diff --git a/qemu/tests/qemu-iotests/088 b/qemu/tests/qemu-iotests/088
index f9c312918..b8076f216 100755
--- a/qemu/tests/qemu-iotests/088
+++ b/qemu/tests/qemu-iotests/088
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/088.out b/qemu/tests/qemu-iotests/088.out
index 6e6bfcaf9..a2a83b8a1 100644
--- a/qemu/tests/qemu-iotests/088.out
+++ b/qemu/tests/qemu-iotests/088.out
@@ -2,16 +2,16 @@ QA output created by 088
 
 == Invalid block size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 0
+can't open device TEST_DIR/t.vpc: Invalid block size 0
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 0
+can't open device TEST_DIR/t.vpc: Invalid block size 0
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 128
+can't open device TEST_DIR/t.vpc: Invalid block size 128
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 128
+can't open device TEST_DIR/t.vpc: Invalid block size 128
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 305419896
+can't open device TEST_DIR/t.vpc: Invalid block size 305419896
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 305419896
+can't open device TEST_DIR/t.vpc: Invalid block size 305419896
 no file open, try 'help open'
 *** done
diff --git a/qemu/tests/qemu-iotests/089 b/qemu/tests/qemu-iotests/089
index 3e0038dde..9bfe2307b 100755
--- a/qemu/tests/qemu-iotests/089
+++ b/qemu/tests/qemu-iotests/089
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/090 b/qemu/tests/qemu-iotests/090
index 70b5a6fd7..7380503d5 100755
--- a/qemu/tests/qemu-iotests/090
+++ b/qemu/tests/qemu-iotests/090
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/092 b/qemu/tests/qemu-iotests/092
index 52c529bae..5bbdd071d 100755
--- a/qemu/tests/qemu-iotests/092
+++ b/qemu/tests/qemu-iotests/092
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/092.out b/qemu/tests/qemu-iotests/092.out
index c5c60f917..e18f54c20 100644
--- a/qemu/tests/qemu-iotests/092.out
+++ b/qemu/tests/qemu-iotests/092.out
@@ -2,37 +2,37 @@ QA output created by 092
 
 == Invalid cluster size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
 
 == Invalid L2 table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
 
 == Invalid size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+can't open device TEST_DIR/t.qcow: Image too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+can't open device TEST_DIR/t.qcow: Image too large
 no file open, try 'help open'
 
 == Invalid backing file length ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+can't open device TEST_DIR/t.qcow: Backing file name too long
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+can't open device TEST_DIR/t.qcow: Backing file name too long
 no file open, try 'help open'
 *** done
diff --git a/qemu/tests/qemu-iotests/093 b/qemu/tests/qemu-iotests/093
index c0e9e2b0b..ce8e13cb4 100755
--- a/qemu/tests/qemu-iotests/093
+++ b/qemu/tests/qemu-iotests/093
@@ -3,7 +3,7 @@
 # Tests for IO throttling
 #
 # Copyright (C) 2015 Red Hat, Inc.
-# Copyright (C) 2015 Igalia, S.L.
+# Copyright (C) 2015-2016 Igalia, S.L.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@
 
 import iotests
 
+nsec_per_sec = 1000000000
+
 class ThrottleTestCase(iotests.QMPTestCase):
     test_img = "null-aio://"
     max_drives = 3
@@ -42,16 +44,7 @@ class ThrottleTestCase(iotests.QMPTestCase):
     def tearDown(self):
         self.vm.shutdown()
 
-    def do_test_throttle(self, ndrives, seconds, params):
-        def check_limit(limit, num):
-            # IO throttling algorithm is discrete, allow 10% error so the test
-            # is more robust
-            return limit == 0 or \
-                   (num < seconds * limit * 1.1 / ndrives
-                   and num > seconds * limit * 0.9 / ndrives)
-
-        nsec_per_sec = 1000000000
-
+    def configure_throttle(self, ndrives, params):
         params['group'] = 'test'
 
         # Set the I/O throttling parameters to all drives
@@ -60,13 +53,21 @@ class ThrottleTestCase(iotests.QMPTestCase):
             result = self.vm.qmp("block_set_io_throttle", conv_keys=False, **params)
             self.assert_qmp(result, 'return', {})
 
+    def do_test_throttle(self, ndrives, seconds, params):
+        def check_limit(limit, num):
+            # IO throttling algorithm is discrete, allow 10% error so the test
+            # is more robust
+            return limit == 0 or \
+                   (num < seconds * limit * 1.1 / ndrives
+                   and num > seconds * limit * 0.9 / ndrives)
+
         # Set vm clock to a known value
         ns = seconds * nsec_per_sec
         self.vm.qtest("clock_step %d" % ns)
 
-        # Submit enough requests. They will drain bps_max and iops_max, but the
-        # rest requests won't get executed until we advance the virtual clock
-        # with qtest interface
+        # Submit enough requests so the throttling mechanism kicks
+        # in. The throttled requests won't be executed until we
+        # advance the virtual clock.
         rq_size = 512
         rd_nr = max(params['bps'] / rq_size / 2,
                     params['bps_rd'] / rq_size,
@@ -142,8 +143,44 @@ class ThrottleTestCase(iotests.QMPTestCase):
             for tk in params:
                 limits = dict([(k, 0) for k in params])
                 limits[tk] = params[tk] * ndrives
+                self.configure_throttle(ndrives, limits)
                 self.do_test_throttle(ndrives, 5, limits)
 
+    def test_burst(self):
+        params = {"bps": 4096,
+                  "bps_rd": 4096,
+                  "bps_wr": 4096,
+                  "iops": 10,
+                  "iops_rd": 10,
+                  "iops_wr": 10,
+                 }
+        ndrives = 1
+        # Pick each out of all possible params and test
+        for tk in params:
+            rate = params[tk] * ndrives
+            burst_rate = rate * 7
+            burst_length = 4
+
+            # Configure the throttling settings
+            settings = dict([(k, 0) for k in params])
+            settings[tk] = rate
+            settings['%s_max' % tk] = burst_rate
+            settings['%s_max_length' % tk] = burst_length
+            self.configure_throttle(ndrives, settings)
+
+            # Wait for the bucket to empty so we can do bursts
+            wait_ns = nsec_per_sec * burst_length * burst_rate / rate
+            self.vm.qtest("clock_step %d" % wait_ns)
+
+            # Test I/O at the max burst rate
+            limits = dict([(k, 0) for k in params])
+            limits[tk] = burst_rate
+            self.do_test_throttle(ndrives, burst_length, limits)
+
+            # Now test I/O at the normal rate
+            limits[tk] = rate
+            self.do_test_throttle(ndrives, 5, limits)
+
 class ThrottleTestCoroutine(ThrottleTestCase):
     test_img = "null-co://"
 
diff --git a/qemu/tests/qemu-iotests/093.out b/qemu/tests/qemu-iotests/093.out
index fbc63e62f..89968f35d 100644
--- a/qemu/tests/qemu-iotests/093.out
+++ b/qemu/tests/qemu-iotests/093.out
@@ -1,5 +1,5 @@
-..
+....
 ----------------------------------------------------------------------
-Ran 2 tests
+Ran 4 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/094 b/qemu/tests/qemu-iotests/094
index 27a2be256..0ba0b0c36 100755
--- a/qemu/tests/qemu-iotests/094
+++ b/qemu/tests/qemu-iotests/094
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Test case for drive-mirror to NBD (especially bdrv_swap() on NBD BDS)
+# Test case for drive-mirror to NBD
 #
 # Copyright (C) 2015 Red Hat, Inc.
 #
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 trap "exit \$status" 0 1 2 3 15
@@ -50,8 +49,10 @@ _send_qemu_cmd $QEMU_HANDLE \
     "{'execute': 'qmp_capabilities'}" \
     'return'
 
-# 'format': 'nbd' is not actually "correct", but this is probably the only way
-# to test bdrv_swap() on an NBD BDS
+# 'format': 'nbd' is not actually "correct", but this was the only way to
+# test the bug fixed in commit f53a829.  Though the bug's related code
+# bdrv_swap() was replaced later, let's make sure we don't fall in the same
+# pit again.
 _send_qemu_cmd $QEMU_HANDLE  \
     "{'execute': 'drive-mirror',
       'arguments': {'device': 'src',
diff --git a/qemu/tests/qemu-iotests/095 b/qemu/tests/qemu-iotests/095
index 6630181a7..dad04b9ac 100755
--- a/qemu/tests/qemu-iotests/095
+++ b/qemu/tests/qemu-iotests/095
@@ -50,17 +50,15 @@ _supported_os Linux
 size_smaller=5M
 size_larger=100M
 
-_make_test_img $size_smaller
-mv "${TEST_IMG}" "${TEST_IMG}.base"
+TEST_IMG="$TEST_IMG.base" _make_test_img $size_smaller
 
-_make_test_img -b "${TEST_IMG}.base" $size_larger
-mv "${TEST_IMG}" "${TEST_IMG}.snp1"
+TEST_IMG="$TEST_IMG.snp1" _make_test_img -b "$TEST_IMG.base" $size_larger
 
 _make_test_img -b "${TEST_IMG}.snp1" $size_larger
 
 echo
 echo "=== Base image info before commit and resize ==="
-TEST_IMG="${TEST_IMG}.base" _img_info
+TEST_IMG="${TEST_IMG}.base" _img_info | _filter_img_info
 
 echo
 echo === Running QEMU Live Commit Test ===
@@ -78,7 +76,7 @@ _send_qemu_cmd $h "{ 'execute': 'block-commit',
 
 echo
 echo "=== Base image info after commit and resize ==="
-TEST_IMG="${TEST_IMG}.base" _img_info
+TEST_IMG="${TEST_IMG}.base" _img_info | _filter_img_info
 
 # success, all done
 echo "*** done"
diff --git a/qemu/tests/qemu-iotests/095.out b/qemu/tests/qemu-iotests/095.out
index 267c4836e..73875cab4 100644
--- a/qemu/tests/qemu-iotests/095.out
+++ b/qemu/tests/qemu-iotests/095.out
@@ -1,13 +1,12 @@
 QA output created by 095
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5242880
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.snp1'
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=5242880
+Formatting 'TEST_DIR/t.IMGFMT.snp1', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.snp1
 
 === Base image info before commit and resize ===
 image: TEST_DIR/t.IMGFMT.base
 file format: IMGFMT
 virtual size: 5.0M (5242880 bytes)
-cluster_size: 65536
 
 === Running QEMU Live Commit Test ===
 
@@ -19,5 +18,4 @@ cluster_size: 65536
 image: TEST_DIR/t.IMGFMT.base
 file format: IMGFMT
 virtual size: 100M (104857600 bytes)
-cluster_size: 65536
 *** done
diff --git a/qemu/tests/qemu-iotests/096 b/qemu/tests/qemu-iotests/096
new file mode 100644
index 000000000..e34204b8f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/096
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Test that snapshots move the throttling configuration to the active
+# layer
+#
+# Copyright (C) 2015 Igalia, S.L.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+class TestLiveSnapshot(iotests.QMPTestCase):
+    base_img = os.path.join(iotests.test_dir, 'base.img')
+    target_img = os.path.join(iotests.test_dir, 'target.img')
+    group = 'mygroup'
+    iops = 6000
+    iops_size = 1024
+
+    def setUp(self):
+        opts = []
+        opts.append('node-name=base')
+        opts.append('throttling.group=%s' % self.group)
+        opts.append('throttling.iops-total=%d' % self.iops)
+        opts.append('throttling.iops-size=%d' % self.iops_size)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, '100M')
+        self.vm = iotests.VM().add_drive(self.base_img, ','.join(opts))
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.base_img)
+        os.remove(self.target_img)
+
+    def checkConfig(self, active_layer):
+        result = self.vm.qmp('query-named-block-nodes')
+        for r in result['return']:
+            if r['node-name'] == active_layer:
+                self.assertEqual(r['group'], self.group)
+                self.assertEqual(r['iops'], self.iops)
+                self.assertEqual(r['iops_size'], self.iops_size)
+            else:
+                self.assertFalse(r.has_key('group'))
+                self.assertEqual(r['iops'], 0)
+                self.assertFalse(r.has_key('iops_size'))
+
+    def testSnapshot(self):
+        self.checkConfig('base')
+        self.vm.qmp('blockdev-snapshot-sync',
+                    node_name = 'base',
+                    snapshot_node_name = 'target',
+                    snapshot_file = self.target_img,
+                    format = iotests.imgfmt)
+        self.checkConfig('target')
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/qemu/tests/qemu-iotests/096.out b/qemu/tests/qemu-iotests/096.out
new file mode 100644
index 000000000..ae1213e6f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/096.out
@@ -0,0 +1,5 @@
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/097 b/qemu/tests/qemu-iotests/097
index c7a613b7e..01d8dd033 100755
--- a/qemu/tests/qemu-iotests/097
+++ b/qemu/tests/qemu-iotests/097
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/097.out b/qemu/tests/qemu-iotests/097.out
index 81651f496..48abd2e64 100644
--- a/qemu/tests/qemu-iotests/097.out
+++ b/qemu/tests/qemu-iotests/097.out
@@ -3,8 +3,8 @@ QA output created by 097
 === Test pass 0 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -32,8 +32,8 @@ Offset          Length          File
 === Test pass 1 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -62,8 +62,8 @@ Offset          Length          File
 === Test pass 2 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -92,8 +92,8 @@ Offset          Length          File
 === Test pass 3 ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd'
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
diff --git a/qemu/tests/qemu-iotests/098 b/qemu/tests/qemu-iotests/098
index e2230ad60..b002e969b 100755
--- a/qemu/tests/qemu-iotests/098
+++ b/qemu/tests/qemu-iotests/098
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/098.out b/qemu/tests/qemu-iotests/098.out
index e08a189b4..7634d0e8b 100644
--- a/qemu/tests/qemu-iotests/098.out
+++ b/qemu/tests/qemu-iotests/098.out
@@ -3,7 +3,7 @@ QA output created by 098
 === l1_update ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -12,7 +12,7 @@ No errors were found on the image.
 === empty_image_prepare ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -25,7 +25,7 @@ No errors were found on the image.
 === reftable_update ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -39,7 +39,7 @@ No errors were found on the image.
 === refblock_alloc ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
diff --git a/qemu/tests/qemu-iotests/099 b/qemu/tests/qemu-iotests/099
index 80f3d9aaf..caaf58eee 100755
--- a/qemu/tests/qemu-iotests/099
+++ b/qemu/tests/qemu-iotests/099
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/100 b/qemu/tests/qemu-iotests/100
index 7c1b235b5..5b2fb3333 100755
--- a/qemu/tests/qemu-iotests/100
+++ b/qemu/tests/qemu-iotests/100
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/101 b/qemu/tests/qemu-iotests/101
index 70fbf25f6..ea53f8b8d 100755
--- a/qemu/tests/qemu-iotests/101
+++ b/qemu/tests/qemu-iotests/101
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/102 b/qemu/tests/qemu-iotests/102
index 161b1974c..64b4af944 100755
--- a/qemu/tests/qemu-iotests/102
+++ b/qemu/tests/qemu-iotests/102
@@ -25,7 +25,6 @@ seq=$(basename $0)
 echo "QA output created by $seq"
 
 here=$PWD
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/103 b/qemu/tests/qemu-iotests/103
index fa9a3c1fc..ecbd8ebd7 100755
--- a/qemu/tests/qemu-iotests/103
+++ b/qemu/tests/qemu-iotests/103
@@ -25,7 +25,6 @@ seq=$(basename $0)
 echo "QA output created by $seq"
 
 here=$PWD
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/103.out b/qemu/tests/qemu-iotests/103.out
index d05f49fdb..b7aaadf89 100644
--- a/qemu/tests/qemu-iotests/103.out
+++ b/qemu/tests/qemu-iotests/103.out
@@ -5,10 +5,10 @@ wrote 65536/65536 bytes at offset 0
 
 === Testing invalid option combinations ===
 
-qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
-qemu-io: can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size
-qemu-io: can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size
-qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size
+can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size
+can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
 
 === Testing valid option combinations ===
 
diff --git a/qemu/tests/qemu-iotests/104 b/qemu/tests/qemu-iotests/104
index 2e35ea80d..726d46705 100755
--- a/qemu/tests/qemu-iotests/104
+++ b/qemu/tests/qemu-iotests/104
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 trap "exit \$status" 0 1 2 3 15
diff --git a/qemu/tests/qemu-iotests/105 b/qemu/tests/qemu-iotests/105
index 9bae49e32..3db4ce3cf 100755
--- a/qemu/tests/qemu-iotests/105
+++ b/qemu/tests/qemu-iotests/105
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/107 b/qemu/tests/qemu-iotests/107
index 986203046..d7222dc1c 100755
--- a/qemu/tests/qemu-iotests/107
+++ b/qemu/tests/qemu-iotests/107
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/108 b/qemu/tests/qemu-iotests/108
index ce447498e..2355d98c1 100755
--- a/qemu/tests/qemu-iotests/108
+++ b/qemu/tests/qemu-iotests/108
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/109 b/qemu/tests/qemu-iotests/109
index 0b668da85..f980b0c9e 100755
--- a/qemu/tests/qemu-iotests/109
+++ b/qemu/tests/qemu-iotests/109
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/109.out b/qemu/tests/qemu-iotests/109.out
index 7db92c9ce..38bc073a3 100644
--- a/qemu/tests/qemu-iotests/109.out
+++ b/qemu/tests/qemu-iotests/109.out
@@ -2,8 +2,8 @@ QA output created by 109
 
 === Writing a qcow header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -23,15 +23,15 @@ Images are identical.
 
 === Writing a qcow2 header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
 Specify the 'raw' format explicitly to remove the restrictions.
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 197120, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 197120, "offset": 512, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
 {"return": []}
 read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -44,15 +44,15 @@ Images are identical.
 
 === Writing a qed header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
 Specify the 'raw' format explicitly to remove the restrictions.
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": 262144, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
 {"return": []}
 read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -65,8 +65,8 @@ Images are identical.
 
 === Writing a vdi header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -86,8 +86,8 @@ Images are identical.
 
 === Writing a vmdk header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -107,8 +107,8 @@ Images are identical.
 
 === Writing a vpc header into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -128,7 +128,7 @@ Images are identical.
 
 === Copying sample image empty.bochs into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -149,7 +149,7 @@ Images are identical.
 
 === Copying sample image iotest-dirtylog-10G-4M.vhdx into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -170,7 +170,7 @@ Images are identical.
 
 === Copying sample image parallels-v1 into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -191,7 +191,7 @@ Images are identical.
 
 === Copying sample image simple-pattern.cloop into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
@@ -212,7 +212,7 @@ Images are identical.
 
 === Write legitimate MBR into raw ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 {"return": {}}
 WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
 Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
diff --git a/qemu/tests/qemu-iotests/110 b/qemu/tests/qemu-iotests/110
index a687f9567..9de7369f3 100755
--- a/qemu/tests/qemu-iotests/110
+++ b/qemu/tests/qemu-iotests/110
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/110.out b/qemu/tests/qemu-iotests/110.out
index 152bacf41..b3584ff87 100644
--- a/qemu/tests/qemu-iotests/110.out
+++ b/qemu/tests/qemu-iotests/110.out
@@ -3,7 +3,7 @@ QA output created by 110
 === Reconstructable filename ===
 
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
@@ -11,9 +11,12 @@ backing file: t.IMGFMT.base (actual path: TEST_DIR/t.IMGFMT.base)
 
 === Non-reconstructable filename ===
 
-qemu-img: Cannot use relative backing file names for 'json:{"driver": "IMGFMT", "file": {"set-state.0.event": "read_aio", "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "set-state.0.new_state": 42}}'
+image: json:{"driver": "IMGFMT", "file": {"set-state.0.event": "read_aio", "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "set-state.0.new_state": 42}}
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+backing file: t.IMGFMT.base (cannot determine actual path)
 
 === Backing name is always relative to the backed image ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
 *** done
diff --git a/qemu/tests/qemu-iotests/111 b/qemu/tests/qemu-iotests/111
index 6011c94b7..a1c152d0c 100755
--- a/qemu/tests/qemu-iotests/111
+++ b/qemu/tests/qemu-iotests/111
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/112 b/qemu/tests/qemu-iotests/112
index 3f054a3fc..28eb9aae9 100755
--- a/qemu/tests/qemu-iotests/112
+++ b/qemu/tests/qemu-iotests/112
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -180,6 +179,115 @@ $QEMU_IMG snapshot -c foo "$TEST_IMG"
 # leaked (refcount=UINT64_MAX reference=1)
 _check_test_img
 
+echo
+echo '=== Amend from refcount_bits=16 to refcount_bits=1 ==='
+echo
+
+_make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 16M 32M' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o refcount_bits=1 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Amend from refcount_bits=1 to refcount_bits=64 ==='
+echo
+
+$QEMU_IMG amend -o refcount_bits=64 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Amend to compat=0.10 ==='
+echo
+
+# Should not work because refcount_bits needs to be 16 for compat=0.10
+$QEMU_IMG amend -o compat=0.10 "$TEST_IMG"
+print_refcount_bits
+# Should work
+$QEMU_IMG amend -o compat=0.10,refcount_bits=16 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+# Get back to compat=1.1 and refcount_bits=16
+$QEMU_IMG amend -o compat=1.1 "$TEST_IMG"
+print_refcount_bits
+# Should not work
+$QEMU_IMG amend -o refcount_bits=32,compat=0.10 "$TEST_IMG"
+print_refcount_bits
+
+echo
+echo '=== Amend with snapshot ==='
+echo
+
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+# Just to have different refcounts across the image
+$QEMU_IO -c 'write 0 16M' "$TEST_IMG" | _filter_qemu_io
+
+# Should not work (may work in the future by first decreasing all refcounts so
+# they fit into the target range by copying them)
+$QEMU_IMG amend -o refcount_bits=1 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+# Should work
+$QEMU_IMG amend -o refcount_bits=2 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Testing too many references for check ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+print_refcount_bits
+
+# This cluster should be created at 0x50000
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+# Now make the second L2 entry (the L2 table should be at 0x40000) point to that
+# cluster, so we have two references
+poke_file "$TEST_IMG" $((0x40008)) "\x80\x00\x00\x00\x00\x05\x00\x00"
+
+# This should say "please use amend"
+_check_test_img -r all
+
+# So we do that
+$QEMU_IMG amend -o refcount_bits=2 "$TEST_IMG"
+print_refcount_bits
+
+# And try again
+_check_test_img -r all
+
+echo
+echo '=== Multiple walks necessary during amend ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1,cluster_size=512" _make_test_img 64k
+
+# Cluster 0 is the image header, clusters 1 to 4 are used by the L1 table, a
+# single L2 table, the reftable and a single refblock. This creates 58 data
+# clusters (actually, the L2 table is created here, too), so in total there are
+# then 63 used clusters in the image. With a refcount width of 64, one refblock
+# describes 64 clusters (512 bytes / 64 bits/entry = 64 entries), so this will
+# make the first refblock in the amended image have exactly one free entry.
+$QEMU_IO -c "write 0 $((58 * 512))" "$TEST_IMG" | _filter_qemu_io
+
+# Now change the refcount width; since the first new refblock will have exactly
+# one free entry, that entry will be used to store its own reference. No other
+# refblocks are needed, so then the new reftable will be allocated; since the
+# first new refblock is completely filled up, this will require a new refblock
+# which is why the refcount width changing function will need to run through
+# everything one more time until the allocations are stable.
+# Having more walks than usual should be visible as regressing progress (from
+# 66.67 % (2/3 walks) to 50.00 % (2/4 walks)).
+$QEMU_IMG amend -o refcount_bits=64 -p "$TEST_IMG" | tr '\r' '\n' \
+                                                   | grep -A 1 '66.67'
+print_refcount_bits
+
+_check_test_img
+
 
 # success, all done
 echo '*** done'
diff --git a/qemu/tests/qemu-iotests/112.out b/qemu/tests/qemu-iotests/112.out
index 9a98633f6..81b04d145 100644
--- a/qemu/tests/qemu-iotests/112.out
+++ b/qemu/tests/qemu-iotests/112.out
@@ -21,9 +21,9 @@ refcount bits: 16
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 refcount bits: 16
-qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 
 === Snapshot limit on refcount_bits=1 ===
@@ -81,4 +81,75 @@ Leaked cluster 6 refcount=1 reference=0
 
 2 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
+
+=== Amend from refcount_bits=16 to refcount_bits=1 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 16
+wrote 33554432/33554432 bytes at offset 16777216
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+refcount bits: 1
+
+=== Amend from refcount_bits=1 to refcount_bits=64 ===
+
+No errors were found on the image.
+refcount bits: 64
+
+=== Amend to compat=0.10 ===
+
+qemu-img: compat=0.10 requires refcount_bits=16
+qemu-img: Error while amending options: Operation not supported
+refcount bits: 64
+No errors were found on the image.
+refcount bits: 16
+refcount bits: 16
+qemu-img: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: Error while amending options: Invalid argument
+refcount bits: 16
+
+=== Amend with snapshot ===
+
+wrote 16777216/16777216 bytes at offset 0
+16 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Cannot decrease refcount entry width to 1 bits: Cluster at offset 0x50000 has a refcount of 2
+qemu-img: Error while amending options: Invalid argument
+No errors were found on the image.
+refcount bits: 16
+No errors were found on the image.
+refcount bits: 2
+
+=== Testing too many references for check ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 1
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+ERROR: overflow cluster offset=0x50000
+Use qemu-img amend to increase the refcount entry width or qemu-img convert to create a clean copy if the image cannot be opened for writing
+
+1 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+refcount bits: 2
+ERROR cluster 5 refcount=1 reference=2
+Repairing cluster 5 refcount=1 reference=2
+Repairing OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=2
+Repairing OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=2
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    3 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+
+=== Multiple walks necessary during amend ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 29696/29696 bytes at offset 0
+29 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+    (66.67/100%)
+    (50.00/100%)
+refcount bits: 64
+No errors were found on the image.
 *** done
diff --git a/qemu/tests/qemu-iotests/113 b/qemu/tests/qemu-iotests/113
index a2cd96b17..19b68b272 100755
--- a/qemu/tests/qemu-iotests/113
+++ b/qemu/tests/qemu-iotests/113
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/114 b/qemu/tests/qemu-iotests/114
index d02e7ffbe..f110d4f65 100755
--- a/qemu/tests/qemu-iotests/114
+++ b/qemu/tests/qemu-iotests/114
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/114.out b/qemu/tests/qemu-iotests/114.out
index 6c6b21085..b6d10e480 100644
--- a/qemu/tests/qemu-iotests/114.out
+++ b/qemu/tests/qemu-iotests/114.out
@@ -1,13 +1,13 @@
 QA output created by 114
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
 cluster_size: 65536
 backing file: TEST_DIR/t.IMGFMT.base
 backing file format: foo
-qemu-io: can't open device TEST_DIR/t.qcow2: Could not open backing file: Unknown driver 'foo'
+can't open device TEST_DIR/t.qcow2: Could not open backing file: Unknown driver 'foo'
 read 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/qemu/tests/qemu-iotests/115 b/qemu/tests/qemu-iotests/115
index a6be1876a..665c2ead4 100755
--- a/qemu/tests/qemu-iotests/115
+++ b/qemu/tests/qemu-iotests/115
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/115.out b/qemu/tests/qemu-iotests/115.out
index 7b2c5e02f..dbdad17b2 100644
--- a/qemu/tests/qemu-iotests/115.out
+++ b/qemu/tests/qemu-iotests/115.out
@@ -2,7 +2,7 @@ QA output created by 115
 
 === Testing large refcount and L1 table ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=268435456 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=268435456 preallocation=metadata
 No errors were found on the image.
 100.00% allocated clusters
 *** done
diff --git a/qemu/tests/qemu-iotests/116 b/qemu/tests/qemu-iotests/116
index 713ed484b..df0172fed 100755
--- a/qemu/tests/qemu-iotests/116
+++ b/qemu/tests/qemu-iotests/116
@@ -28,7 +28,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/116.out b/qemu/tests/qemu-iotests/116.out
index b679ceea6..1f11d4446 100644
--- a/qemu/tests/qemu-iotests/116.out
+++ b/qemu/tests/qemu-iotests/116.out
@@ -2,36 +2,36 @@ QA output created by 116
 
 == truncated header cluster ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 
 == invalid header magic ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Image not in QED format
+can't open device TEST_DIR/t.qed: Image not in QED format
 no file open, try 'help open'
 
 == invalid cluster size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 
 == invalid table size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 
 == invalid header size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 
 == invalid L1 table offset ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 
 == invalid image size ==
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
-qemu-io: can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
 no file open, try 'help open'
 *** done
diff --git a/qemu/tests/qemu-iotests/117 b/qemu/tests/qemu-iotests/117
new file mode 100755
index 000000000..9385b3f8d
--- /dev/null
+++ b/qemu/tests/qemu-iotests/117
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Test case for shared BDS between backend trees
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+_make_test_img 64k
+
+_launch_qemu
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'blockdev-add',
+       'arguments': { 'options': { 'id': 'protocol',
+                                   'driver': 'file',
+                                   'filename': '$TEST_IMG' } } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'blockdev-add',
+       'arguments': { 'options': { 'id': 'format',
+                                   'driver': '$IMGFMT',
+                                   'file': 'protocol' } } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line': 'qemu-io format \"write -P 42 0 64k\"' } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+_check_test_img
+
+$QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/117.out b/qemu/tests/qemu-iotests/117.out
new file mode 100644
index 000000000..f52dc1a35
--- /dev/null
+++ b/qemu/tests/qemu-iotests/117.out
@@ -0,0 +1,14 @@
+QA output created by 117
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+{"return": {}}
+{"return": {}}
+{"return": {}}
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+No errors were found on the image.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/qemu/tests/qemu-iotests/118 b/qemu/tests/qemu-iotests/118
new file mode 100755
index 000000000..9e5951f64
--- /dev/null
+++ b/qemu/tests/qemu-iotests/118
@@ -0,0 +1,677 @@
+#!/usr/bin/env python
+#
+# Test case for the QMP 'change' command and all other associated
+# commands
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import stat
+import time
+import iotests
+from iotests import qemu_img
+
+old_img = os.path.join(iotests.test_dir, 'test0.img')
+new_img = os.path.join(iotests.test_dir, 'test1.img')
+
+class ChangeBaseClass(iotests.QMPTestCase):
+    has_opened = False
+    has_closed = False
+
+    def process_events(self):
+        for event in self.vm.get_qmp_events(wait=False):
+            if (event['event'] == 'DEVICE_TRAY_MOVED' and
+                event['data']['device'] == 'drive0'):
+                if event['data']['tray-open'] == False:
+                    self.has_closed = True
+                else:
+                    self.has_opened = True
+
+    def wait_for_open(self):
+        if not self.has_real_tray:
+            return
+
+        timeout = time.clock() + 3
+        while not self.has_opened and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to open')
+
+    def wait_for_close(self):
+        if not self.has_real_tray:
+            return
+
+        timeout = time.clock() + 3
+        while not self.has_closed and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to close')
+
+class GeneralChangeTestsBaseClass(ChangeBaseClass):
+    def test_change(self):
+        result = self.vm.qmp('change', device='drive0', target=new_img,
+                                       arg=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_blockdev_change_medium(self):
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_eject(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_eject_change(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_tray_open_close(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        if self.has_real_tray or not self.was_empty:
+            self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_tray_eject_close(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_open_change(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_cycle(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_close_on_closed(self):
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+    def test_remove_on_closed(self):
+        if not self.has_real_tray:
+            return
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_insert_on_closed(self):
+        if not self.has_real_tray:
+            return
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyFilled(GeneralChangeTestsBaseClass):
+    was_empty = False
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(old_img, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_insert_on_filled(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyEmpty(GeneralChangeTestsBaseClass):
+    was_empty = True
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(None, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(new_img)
+
+    def test_remove_on_empty(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+
+class TestCDInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'cdrom', 'ide')
+
+class TestCDInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'cdrom', 'ide')
+
+class TestFloppyInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'disk', 'floppy')
+
+class TestFloppyInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'disk', 'floppy')
+        # FDDs not having a real tray and there not being a medium inside the
+        # tray at startup means the tray will be considered open
+        self.has_opened = True
+
+class TestChangeReadOnly(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.chmod(old_img, 0666)
+        os.chmod(new_img, 0666)
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_ro_ro_retain(self):
+        os.chmod(old_img, 0444)
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_ro_rw_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_ro_rw(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_rw_ro(self):
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_make_rw_ro_by_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw_by_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_rw_ro_cycle(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'read-only': True,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+GeneralChangeTestsBaseClass = None
+TestInitiallyFilled = None
+TestInitiallyEmpty = None
+
+
+class TestBlockJobsAfterCycle(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1M')
+
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'id': 'drive0',
+                                      'driver': 'null-co'})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/format', 'null-co')
+
+        # For device-less BBs, calling blockdev-open-tray or blockdev-close-tray
+        # is not necessary
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'node0',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': old_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='node0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        try:
+            os.remove(new_img)
+        except OSError:
+            pass
+
+    def test_snapshot_and_commit(self):
+        # We need backing file support
+        if iotests.imgfmt != 'qcow2' and iotests.imgfmt != 'qed':
+            return
+
+        result = self.vm.qmp('blockdev-snapshot-sync', device='drive0',
+                                                       snapshot_file=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+        self.assert_qmp(result,
+                        'return[0]/inserted/image/backing-image/filename',
+                        old_img)
+
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_READY')
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+
+        result = self.vm.qmp('block-job-complete', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_COMPLETED')
+
+
+if __name__ == '__main__':
+    if iotests.qemu_default_machine != 'pc':
+        # We need floppy and IDE CD-ROM
+        iotests.notrun('not suitable for this machine type: %s' %
+                       iotests.qemu_default_machine)
+    # Need to support image creation
+    iotests.main(supported_fmts=['vpc', 'parallels', 'qcow', 'vdi', 'qcow2',
+                                 'vmdk', 'raw', 'vhdx', 'qed'])
diff --git a/qemu/tests/qemu-iotests/118.out b/qemu/tests/qemu-iotests/118.out
new file mode 100644
index 000000000..6a917130b
--- /dev/null
+++ b/qemu/tests/qemu-iotests/118.out
@@ -0,0 +1,5 @@
+...........................................................
+----------------------------------------------------------------------
+Ran 59 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/119 b/qemu/tests/qemu-iotests/119
index 9a11f1b92..4f34fb434 100755
--- a/qemu/tests/qemu-iotests/119
+++ b/qemu/tests/qemu-iotests/119
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -49,7 +48,7 @@ echo "{'execute': 'qmp_capabilities'}
       {'execute': 'human-monitor-command',
        'arguments': {'command-line': 'qemu-io drv \"read -P 0 0 64k\"'}}
       {'execute': 'quit'}" \
-    | $QEMU -drive id=drv,if=none,file="$TEST_IMG",driver=nbd \
+    | $QEMU -nographic -drive id=drv,if=none,file="$TEST_IMG",driver=nbd \
             -qmp stdio -nodefaults \
     | _filter_qmp | _filter_qemu_io
 
diff --git a/qemu/tests/qemu-iotests/120 b/qemu/tests/qemu-iotests/120
index 9f1307876..4f88a67fe 100755
--- a/qemu/tests/qemu-iotests/120
+++ b/qemu/tests/qemu-iotests/120
@@ -26,7 +26,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -49,7 +48,7 @@ echo "{'execute': 'qmp_capabilities'}
       {'execute': 'human-monitor-command',
        'arguments': {'command-line': 'qemu-io drv \"write -P 42 0 64k\"'}}
       {'execute': 'quit'}" \
-    | $QEMU -qmp stdio -nodefaults \
+    | $QEMU -qmp stdio -nographic -nodefaults \
             -drive id=drv,if=none,file="$TEST_IMG",driver=raw,file.driver=$IMGFMT \
     | _filter_qmp | _filter_qemu_io
 $QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
diff --git a/qemu/tests/qemu-iotests/121 b/qemu/tests/qemu-iotests/121
index 0912c3f0c..1307b4e32 100755
--- a/qemu/tests/qemu-iotests/121
+++ b/qemu/tests/qemu-iotests/121
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/121.out b/qemu/tests/qemu-iotests/121.out
index ff18e2c61..5961a44cd 100644
--- a/qemu/tests/qemu-iotests/121.out
+++ b/qemu/tests/qemu-iotests/121.out
@@ -4,7 +4,7 @@ QA output created by 121
 
 --- Test 1 ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66060288 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66060288 preallocation=metadata
 Image resized.
 wrote 1049600/1049600 bytes at offset 65011712
 1.001 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -14,7 +14,7 @@ No errors were found on the image.
 
 --- Test 2 ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66061312 preallocation='metadata'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66061312 preallocation=metadata
 Image resized.
 wrote 133120/133120 bytes at offset 66060288
 130 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/qemu/tests/qemu-iotests/122 b/qemu/tests/qemu-iotests/122
index 350ca9c46..45b359c2b 100755
--- a/qemu/tests/qemu-iotests/122
+++ b/qemu/tests/qemu-iotests/122
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/122.out b/qemu/tests/qemu-iotests/122.out
index 1f853b9e9..98814de5d 100644
--- a/qemu/tests/qemu-iotests/122.out
+++ b/qemu/tests/qemu-iotests/122.out
@@ -5,7 +5,7 @@ wrote 67108864/67108864 bytes at offset 0
 
 === Check allocation status regression with -B ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Offset          Length          File
@@ -14,7 +14,7 @@ Offset          Length          File
 
 === Check that zero clusters are kept in overlay ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 3145728/3145728 bytes at offset 0
@@ -112,20 +112,18 @@ read 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 63963136/63963136 bytes at offset 3145728
 61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 6291456, "depth": 0, "zero": false, "data": true, "offset": 327680},
-{ "start": 6291456, "length": 60817408, "depth": 0, "zero": true, "data": false}]
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true, "offset": 327680}]
 
 convert -c -S 0:
 read 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 63963136/63963136 bytes at offset 3145728
 61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 6291456, "depth": 0, "zero": false, "data": true},
-{ "start": 6291456, "length": 60817408, "depth": 0, "zero": true, "data": false}]
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true}]
 Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 33554432/33554432 bytes at offset 0
 32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 3145728/3145728 bytes at offset 0
 3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/qemu/tests/qemu-iotests/123 b/qemu/tests/qemu-iotests/123
index ad608035d..b18e3fca9 100755
--- a/qemu/tests/qemu-iotests/123
+++ b/qemu/tests/qemu-iotests/123
@@ -25,7 +25,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/124 b/qemu/tests/qemu-iotests/124
index 9ccd11809..de7cdbe00 100644
--- a/qemu/tests/qemu-iotests/124
+++ b/qemu/tests/qemu-iotests/124
@@ -36,6 +36,23 @@ def try_remove(img):
         pass
 
 
+def transaction_action(action, **kwargs):
+    return {
+        'type': action,
+        'data': dict((k.replace('_', '-'), v) for k, v in kwargs.iteritems())
+    }
+
+
+def transaction_bitmap_clear(node, name, **kwargs):
+    return transaction_action('block-dirty-bitmap-clear',
+                              node=node, name=name, **kwargs)
+
+
+def transaction_drive_backup(device, target, **kwargs):
+    return transaction_action('drive-backup', device=device, target=target,
+                              **kwargs)
+
+
 class Bitmap:
     def __init__(self, name, drive):
         self.name = name
@@ -74,24 +91,31 @@ class Bitmap:
                 try_remove(image)
 
 
-class TestIncrementalBackup(iotests.QMPTestCase):
-    def setUp(self):
+class TestIncrementalBackupBase(iotests.QMPTestCase):
+    def __init__(self, *args):
+        super(TestIncrementalBackupBase, self).__init__(*args)
         self.bitmaps = list()
         self.files = list()
         self.drives = list()
         self.vm = iotests.VM()
         self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
 
+
+    def setUp(self):
         # Create a base image with a distinctive patterning
         drive0 = self.add_node('drive0')
         self.img_create(drive0['file'], drive0['fmt'])
         self.vm.add_drive(drive0['file'])
-        io_write_patterns(drive0['file'], (('0x41', 0, 512),
-                                           ('0xd5', '1M', '32k'),
-                                           ('0xdc', '32M', '124k')))
+        self.write_default_pattern(drive0['file'])
         self.vm.launch()
 
 
+    def write_default_pattern(self, target):
+        io_write_patterns(target, (('0x41', 0, 512),
+                                   ('0xd5', '1M', '32k'),
+                                   ('0xdc', '32M', '124k')))
+
+
     def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
         if path is None:
             path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
@@ -108,23 +132,28 @@ class TestIncrementalBackup(iotests.QMPTestCase):
 
 
     def img_create(self, img, fmt=iotests.imgfmt, size='64M',
-                   parent=None, parentFormat=None):
+                   parent=None, parentFormat=None, **kwargs):
+        optargs = []
+        for k,v in kwargs.iteritems():
+            optargs = optargs + ['-o', '%s=%s' % (k,v)]
+        args = ['create', '-f', fmt] + optargs + [img, size]
         if parent:
             if parentFormat is None:
                 parentFormat = fmt
-            iotests.qemu_img('create', '-f', fmt, img, size,
-                             '-b', parent, '-F', parentFormat)
-        else:
-            iotests.qemu_img('create', '-f', fmt, img, size)
+            args = args + ['-b', parent, '-F', parentFormat]
+        iotests.qemu_img(*args)
         self.files.append(img)
 
 
     def do_qmp_backup(self, error='Input/output error', **kwargs):
         res = self.vm.qmp('drive-backup', **kwargs)
         self.assert_qmp(res, 'return', {})
+        return self.wait_qmp_backup(kwargs['device'], error)
 
+
+    def wait_qmp_backup(self, device, error='Input/output error'):
         event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
-                                   match={'data': {'device': kwargs['device']}})
+                                   match={'data': {'device': device}})
         self.assertNotEqual(event, None)
 
         try:
@@ -139,6 +168,12 @@ class TestIncrementalBackup(iotests.QMPTestCase):
             return False
 
 
+    def wait_qmp_backup_cancelled(self, device):
+        event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+
     def create_anchor_backup(self, drive=None):
         if drive is None:
             drive = self.drives[-1]
@@ -233,6 +268,16 @@ class TestIncrementalBackup(iotests.QMPTestCase):
         self.check_backups()
 
 
+    def tearDown(self):
+        self.vm.shutdown()
+        for bitmap in self.bitmaps:
+            bitmap.cleanup()
+        for filename in self.files:
+            try_remove(filename)
+
+
+
+class TestIncrementalBackup(TestIncrementalBackupBase):
     def test_incremental_simple(self):
         '''
         Test: Create and verify three incremental backups.
@@ -264,19 +309,110 @@ class TestIncrementalBackup(iotests.QMPTestCase):
         return self.do_incremental_simple(granularity=131072)
 
 
-    def test_incremental_failure(self):
-        '''Test: Verify backups made after a failure are correct.
+    def test_larger_cluster_target(self):
+        '''
+        Test: Create and verify backups made to a larger cluster size target.
 
-        Simulate a failure during an incremental backup block job,
-        emulate additional writes, then create another incremental backup
-        afterwards and verify that the backup created is correct.
+        With a default granularity of 64KiB, verify that backups made to a
+        larger cluster size target of 128KiB without a backing file works.
         '''
+        drive0 = self.drives[0]
+
+        # Create a cluster_size=128k full backup / "anchor" backup
+        self.img_create(drive0['backup'], cluster_size='128k')
+        self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
+                                           format=drive0['fmt'],
+                                           target=drive0['backup'],
+                                           mode='existing'))
+
+        # Create bitmap and dirty it with some new writes.
+        # overwrite [32736, 32799] which will dirty bitmap clusters at
+        # 32M-64K and 32M. 32M+64K will be left undirtied.
+        bitmap0 = self.add_bitmap('bitmap0', drive0)
+        self.hmp_io_writes(drive0['id'],
+                           (('0xab', 0, 512),
+                            ('0xfe', '16M', '256k'),
+                            ('0x64', '32736k', '64k')))
+
+
+        # Prepare a cluster_size=128k backup target without a backing file.
+        (target, _) = bitmap0.new_target()
+        self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
+
+        # Perform Incremental Backup
+        self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
+                                           sync='incremental',
+                                           bitmap=bitmap0.name,
+                                           format=bitmap0.drive['fmt'],
+                                           target=target,
+                                           mode='existing'))
+        self.make_reference_backup(bitmap0)
+
+        # Add the backing file, then compare and exit.
+        iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
+                         drive0['backup'], '-F', drive0['fmt'], target)
+        self.vm.shutdown()
+        self.check_backups()
+
+
+    def test_incremental_transaction(self):
+        '''Test: Verify backups made from transactionally created bitmaps.
+
+        Create a bitmap "before" VM execution begins, then create a second
+        bitmap AFTER writes have already occurred. Use transactions to create
+        a full backup and synchronize both bitmaps to this backup.
+        Create an incremental backup through both bitmaps and verify that
+        both backups match the current drive0 image.
+        '''
+
+        drive0 = self.drives[0]
+        bitmap0 = self.add_bitmap('bitmap0', drive0)
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+        bitmap1 = self.add_bitmap('bitmap1', drive0)
+
+        result = self.vm.qmp('transaction', actions=[
+            transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
+            transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
+            transaction_drive_backup(drive0['id'], drive0['backup'],
+                                     sync='full', format=drive0['fmt'])
+        ])
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(drive0['id'])
+        self.files.append(drive0['backup'])
+
+        self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
+                                          ('0x55', '8M', '352k'),
+                                          ('0x78', '15872k', '1M')))
+        # Both bitmaps should be correctly in sync.
+        self.create_incremental(bitmap0)
+        self.create_incremental(bitmap1)
+        self.vm.shutdown()
+        self.check_backups()
 
-        # Create a blkdebug interface to this img as 'drive1',
-        # but don't actually create a new image.
-        drive1 = self.add_node('drive1', self.drives[0]['fmt'],
-                               path=self.drives[0]['file'],
-                               backup=self.drives[0]['backup'])
+
+    def test_transaction_failure(self):
+        '''Test: Verify backups made from a transaction that partially fails.
+
+        Add a second drive with its own unique pattern, and add a bitmap to each
+        drive. Use blkdebug to interfere with the backup on just one drive and
+        attempt to create a coherent incremental backup across both drives.
+
+        verify a failure in one but not both, then delete the failed stubs and
+        re-run the same transaction.
+
+        verify that both incrementals are created successfully.
+        '''
+
+        # Create a second drive, with pattern:
+        drive1 = self.add_node('drive1')
+        self.img_create(drive1['file'], drive1['fmt'])
+        io_write_patterns(drive1['file'], (('0x14', 0, 512),
+                                           ('0x5d', '1M', '32k'),
+                                           ('0xcd', '32M', '124k')))
+
+        # Create a blkdebug interface to this img as 'drive1'
         result = self.vm.qmp('blockdev-add', options={
             'id': drive1['id'],
             'driver': drive1['fmt'],
@@ -302,21 +438,73 @@ class TestIncrementalBackup(iotests.QMPTestCase):
         })
         self.assert_qmp(result, 'return', {})
 
-        self.create_anchor_backup(self.drives[0])
-        self.add_bitmap('bitmap0', drive1)
-        # Note: at this point, during a normal execution,
-        # Assume that the VM resumes and begins issuing IO requests here.
+        # Create bitmaps and full backups for both drives
+        drive0 = self.drives[0]
+        dr0bm0 = self.add_bitmap('bitmap0', drive0)
+        dr1bm0 = self.add_bitmap('bitmap0', drive1)
+        self.create_anchor_backup(drive0)
+        self.create_anchor_backup(drive1)
+        self.assert_no_active_block_jobs()
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
 
-        self.hmp_io_writes(drive1['id'], (('0xab', 0, 512),
+        # Emulate some writes
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
                                           ('0xfe', '16M', '256k'),
                                           ('0x64', '32736k', '64k')))
+        self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
+                                          ('0xef', '16M', '256k'),
+                                          ('0x46', '32736k', '64k')))
+
+        # Create incremental backup targets
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Ask for a new incremental backup per-each drive,
+        # expecting drive1's backup to fail:
+        transaction = [
+            transaction_drive_backup(drive0['id'], target0, sync='incremental',
+                                     format=drive0['fmt'], mode='existing',
+                                     bitmap=dr0bm0.name),
+            transaction_drive_backup(drive1['id'], target1, sync='incremental',
+                                     format=drive1['fmt'], mode='existing',
+                                     bitmap=dr1bm0.name)
+        ]
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode': 'grouped'} )
+        self.assert_qmp(result, 'return', {})
 
-        result = self.create_incremental(validate=False)
-        self.assertFalse(result)
-        self.hmp_io_writes(drive1['id'], (('0x9a', 0, 512),
-                                          ('0x55', '8M', '352k'),
-                                          ('0x78', '15872k', '1M')))
-        self.create_incremental()
+        # Observe that drive0's backup is cancelled and drive1 completes with
+        # an error.
+        self.wait_qmp_backup_cancelled(drive0['id'])
+        self.assertFalse(self.wait_qmp_backup(drive1['id']))
+        error = self.vm.event_wait('BLOCK_JOB_ERROR')
+        self.assert_qmp(error, 'data', {'device': drive1['id'],
+                                        'action': 'report',
+                                        'operation': 'read'})
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # Delete drive0's successful target and eliminate our record of the
+        # unsuccessful drive1 target. Then re-run the same transaction.
+        dr0bm0.del_target()
+        dr1bm0.del_target()
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Re-run the exact same transaction.
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode':'grouped'})
+        self.assert_qmp(result, 'return', {})
+
+        # Both should complete successfully this time.
+        self.assertTrue(self.wait_qmp_backup(drive0['id']))
+        self.assertTrue(self.wait_qmp_backup(drive1['id']))
+        self.make_reference_backup(dr0bm0)
+        self.make_reference_backup(dr1bm0)
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # And the images should of course validate.
         self.vm.shutdown()
         self.check_backups()
 
@@ -351,12 +539,66 @@ class TestIncrementalBackup(iotests.QMPTestCase):
                           granularity=64000)
 
 
-    def tearDown(self):
+class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
+    '''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
+
+    def setUp(self):
+        drive0 = self.add_node('drive0')
+        self.img_create(drive0['file'], drive0['fmt'])
+        self.write_default_pattern(drive0['file'])
+        self.vm.launch()
+
+    def test_incremental_failure(self):
+        '''Test: Verify backups made after a failure are correct.
+
+        Simulate a failure during an incremental backup block job,
+        emulate additional writes, then create another incremental backup
+        afterwards and verify that the backup created is correct.
+        '''
+
+        drive0 = self.drives[0]
+        result = self.vm.qmp('blockdev-add', options={
+            'id': drive0['id'],
+            'driver': drive0['fmt'],
+            'file': {
+                'driver': 'blkdebug',
+                'image': {
+                    'driver': 'file',
+                    'filename': drive0['file']
+                },
+                'set-state': [{
+                    'event': 'flush_to_disk',
+                    'state': 1,
+                    'new_state': 2
+                }],
+                'inject-error': [{
+                    'event': 'read_aio',
+                    'errno': 5,
+                    'state': 2,
+                    'immediately': False,
+                    'once': True
+                }],
+            }
+        })
+        self.assert_qmp(result, 'return', {})
+
+        self.create_anchor_backup(drive0)
+        self.add_bitmap('bitmap0', drive0)
+        # Note: at this point, during a normal execution,
+        # Assume that the VM resumes and begins issuing IO requests here.
+
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+
+        result = self.create_incremental(validate=False)
+        self.assertFalse(result)
+        self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
+                                          ('0x55', '8M', '352k'),
+                                          ('0x78', '15872k', '1M')))
+        self.create_incremental()
         self.vm.shutdown()
-        for bitmap in self.bitmaps:
-            bitmap.cleanup()
-        for filename in self.files:
-            try_remove(filename)
+        self.check_backups()
 
 
 if __name__ == '__main__':
diff --git a/qemu/tests/qemu-iotests/124.out b/qemu/tests/qemu-iotests/124.out
index 2f7d3902f..36376bed8 100644
--- a/qemu/tests/qemu-iotests/124.out
+++ b/qemu/tests/qemu-iotests/124.out
@@ -1,5 +1,5 @@
-.......
+..........
 ----------------------------------------------------------------------
-Ran 7 tests
+Ran 10 tests
 
 OK
diff --git a/qemu/tests/qemu-iotests/128 b/qemu/tests/qemu-iotests/128
index e2a0f2f89..0976a1813 100755
--- a/qemu/tests/qemu-iotests/128
+++ b/qemu/tests/qemu-iotests/128
@@ -25,12 +25,16 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 devname="eiodev$$"
 sudo=""
 
+_sudo_qemu_io_wrapper()
+{
+    (exec $sudo "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@")
+}
+
 _setup_eiodev()
 {
 	# This test should either be run as root or with passwordless sudo
@@ -76,7 +80,9 @@ TEST_IMG="/dev/mapper/$devname"
 echo
 echo "== reading from error device =="
 # Opening image should succeed but the read operation should fail
-$sudo $QEMU_IO --format "$IMGFMT" --nocache -c "read 0 65536" "$TEST_IMG" | _filter_qemu_io
+_sudo_qemu_io_wrapper --format "$IMGFMT" --nocache \
+                      -c "read 0 65536" "$TEST_IMG" \
+    | _filter_qemu_io
 
 # success, all done
 echo "*** done"
diff --git a/qemu/tests/qemu-iotests/130 b/qemu/tests/qemu-iotests/130
index bc26247e3..ecc8a5ba1 100755
--- a/qemu/tests/qemu-iotests/130
+++ b/qemu/tests/qemu-iotests/130
@@ -27,7 +27,6 @@ seq="$(basename $0)"
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -59,8 +58,8 @@ echo
 # bdrv_make_empty() involves a header update for qcow2
 
 # Test that a backing file isn't written
-_launch_qemu -drive file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
-_send_qemu_cmd $QEMU_HANDLE "commit ide0-hd0" "(qemu)"
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
 _send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
@@ -68,8 +67,8 @@ _img_info | _filter_img_info
 # Make sure that if there was a backing file that was just overridden on the
 # command line, that backing file is retained, with the right format
 _make_test_img -F raw -b "$TEST_IMG.orig" 64M
-_launch_qemu -drive file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
-_send_qemu_cmd $QEMU_HANDLE "commit ide0-hd0" "(qemu)"
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
 _send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
 _cleanup_qemu
 _img_info | _filter_img_info
diff --git a/qemu/tests/qemu-iotests/130.out b/qemu/tests/qemu-iotests/130.out
index ea68b5d28..ae95b5027 100644
--- a/qemu/tests/qemu-iotests/130.out
+++ b/qemu/tests/qemu-iotests/130.out
@@ -9,14 +9,14 @@ virtual size: 64M (67108864 bytes)
 === HMP commit ===
 
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) ccocomcommcommicommitcommit commit icommit idcommit idecommit ide0commit ide0-commit ide0-hcommit ide0-hdcommit ide0-hd0
+(qemu) ccocomcommcommicommitcommit commit tcommit tecommit tescommit testcommit testdcommit testdicommit testdiscommit testdisk
 (qemu) 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.orig' backing_fmt='raw'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) ccocomcommcommicommitcommit commit icommit idcommit idecommit ide0commit ide0-commit ide0-hcommit ide0-hdcommit ide0-hd0
+(qemu) ccocomcommcommicommitcommit commit tcommit tecommit tescommit testcommit testdcommit testdicommit testdiscommit testdisk
 (qemu) 
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
@@ -32,7 +32,7 @@ wrote 4096/4096 bytes at offset 0
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.orig' backing_fmt='raw'
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 image: TEST_DIR/t.IMGFMT
diff --git a/qemu/tests/qemu-iotests/131 b/qemu/tests/qemu-iotests/131
index 4873f40e9..94a9ae76a 100755
--- a/qemu/tests/qemu-iotests/131
+++ b/qemu/tests/qemu-iotests/131
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/131.out b/qemu/tests/qemu-iotests/131.out
index 021a04c81..ae2412ebf 100644
--- a/qemu/tests/qemu-iotests/131.out
+++ b/qemu/tests/qemu-iotests/131.out
@@ -22,7 +22,7 @@ read 32768/32768 bytes at offset 163840
 read 32768/32768 bytes at offset 0
 32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 == Corrupt image ==
-qemu-io: can't open device TEST_DIR/t.parallels: parallels: Image was not closed correctly; cannot be opened read/write
+can't open device TEST_DIR/t.parallels: parallels: Image was not closed correctly; cannot be opened read/write
 no file open, try 'help open'
 ERROR image was not closed correctly
 
diff --git a/qemu/tests/qemu-iotests/133 b/qemu/tests/qemu-iotests/133
new file mode 100755
index 000000000..9d35a6a1c
--- /dev/null
+++ b/qemu/tests/qemu-iotests/133
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Test for reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img -b "$TEST_IMG.base"
+
+echo
+echo "=== Check that node-name can't be changed ==="
+echo
+
+$QEMU_IO -c 'reopen -o node-name=foo' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.node-name=foo' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.node-name=foo' $TEST_IMG
+
+echo
+echo "=== Check that unchanged node-name is okay ==="
+echo
+
+# Explicitly repeated
+$QEMU_IO -c "open -o node-name=foo $TEST_IMG" -c 'reopen -o node-name=foo'
+$QEMU_IO -c "open -o file.node-name=foo $TEST_IMG" -c 'reopen -o file.node-name=foo'
+$QEMU_IO -c "open -o backing.node-name=foo $TEST_IMG" -c 'reopen -o backing.node-name=foo'
+
+# Implicitly retained
+$QEMU_IO -c "open -o node-name=foo $TEST_IMG" -c 'reopen'
+$QEMU_IO -c "open -o file.node-name=foo $TEST_IMG" -c 'reopen'
+$QEMU_IO -c "open -o backing.node-name=foo $TEST_IMG" -c 'reopen'
+
+echo
+echo "=== Check that driver can't be changed ==="
+echo
+
+$QEMU_IO -c 'reopen -o driver=raw' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.driver=qcow2' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.driver=file' $TEST_IMG
+
+echo
+echo "=== Check that unchanged driver is okay ==="
+echo
+
+# Explicitly repeated (implicit case is covered in node-name test)
+$QEMU_IO -c 'reopen -o driver=qcow2' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.driver=file' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.driver=qcow2' $TEST_IMG
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/133.out b/qemu/tests/qemu-iotests/133.out
new file mode 100644
index 000000000..cc86b9488
--- /dev/null
+++ b/qemu/tests/qemu-iotests/133.out
@@ -0,0 +1,22 @@
+QA output created by 133
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Check that node-name can't be changed ===
+
+Cannot change the option 'node-name'
+Cannot change the option 'node-name'
+Cannot change the option 'node-name'
+
+=== Check that unchanged node-name is okay ===
+
+
+=== Check that driver can't be changed ===
+
+Cannot change the option 'driver'
+Cannot change the option 'driver'
+Cannot change the option 'driver'
+
+=== Check that unchanged driver is okay ===
+
+*** done
diff --git a/qemu/tests/qemu-iotests/134 b/qemu/tests/qemu-iotests/134
index 1c3820b17..af618b881 100755
--- a/qemu/tests/qemu-iotests/134
+++ b/qemu/tests/qemu-iotests/134
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/134.out b/qemu/tests/qemu-iotests/134.out
index a16acb81c..6493704ec 100644
--- a/qemu/tests/qemu-iotests/134.out
+++ b/qemu/tests/qemu-iotests/134.out
@@ -1,43 +1,25 @@
 QA output created by 134
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
-qemu-img: Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
 
 == reading whole image ==
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Disk image 'TEST_DIR/t.qcow2' is encrypted.
 password:
 read 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == rewriting whole image ==
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Disk image 'TEST_DIR/t.qcow2' is encrypted.
 password:
 wrote 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == verify pattern ==
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Disk image 'TEST_DIR/t.qcow2' is encrypted.
 password:
 read 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == verify pattern failure with wrong password ==
-Encrypted images are deprecated
-Support for them will be removed in a future release.
-You can use 'qemu-img convert' to convert your image to an unencrypted one.
 Disk image 'TEST_DIR/t.qcow2' is encrypted.
 password:
 Pattern verification failed at offset 0, 134217728 bytes
diff --git a/qemu/tests/qemu-iotests/135 b/qemu/tests/qemu-iotests/135
index 16bf73656..ce608312f 100755
--- a/qemu/tests/qemu-iotests/135
+++ b/qemu/tests/qemu-iotests/135
@@ -25,7 +25,6 @@ seq=`basename $0`
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/qemu/tests/qemu-iotests/136 b/qemu/tests/qemu-iotests/136
new file mode 100644
index 000000000..e8c6937fc
--- /dev/null
+++ b/qemu/tests/qemu-iotests/136
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+#
+# Tests for block device statistics
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+interval_length = 10
+nsec_per_sec = 1000000000
+op_latency = nsec_per_sec / 1000 # See qtest_latency_ns in accounting.c
+bad_sector = 8192
+bad_offset = bad_sector * 512
+blkdebug_file = os.path.join(iotests.test_dir, 'blkdebug.conf')
+
+class BlockDeviceStatsTestCase(iotests.QMPTestCase):
+    test_img = "null-aio://"
+    total_rd_bytes = 0
+    total_rd_ops = 0
+    total_wr_bytes = 0
+    total_wr_ops = 0
+    total_wr_merged = 0
+    total_flush_ops = 0
+    failed_rd_ops = 0
+    failed_wr_ops = 0
+    invalid_rd_ops = 0
+    invalid_wr_ops = 0
+    wr_highest_offset = 0
+    account_invalid = False
+    account_failed = False
+
+    def blockstats(self, device):
+        result = self.vm.qmp("query-blockstats")
+        for r in result['return']:
+            if r['device'] == device:
+                return r['stats']
+        raise Exception("Device not found for blockstats: %s" % device)
+
+    def create_blkdebug_file(self):
+        file = open(blkdebug_file, 'w')
+        file.write('''
+[inject-error]
+event = "read_aio"
+errno = "5"
+sector = "%d"
+
+[inject-error]
+event = "write_aio"
+errno = "5"
+sector = "%d"
+''' % (bad_sector, bad_sector))
+        file.close()
+
+    def setUp(self):
+        drive_args = []
+        drive_args.append("stats-intervals.0=%d" % interval_length)
+        drive_args.append("stats-account-invalid=%s" %
+                          (self.account_invalid and "on" or "off"))
+        drive_args.append("stats-account-failed=%s" %
+                          (self.account_failed and "on" or "off"))
+        self.create_blkdebug_file()
+        self.vm = iotests.VM().add_drive('blkdebug:%s:%s ' %
+                                         (blkdebug_file, self.test_img),
+                                         ','.join(drive_args))
+        self.vm.launch()
+        # Set an initial value for the clock
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(blkdebug_file)
+
+    def accounted_ops(self, read = False, write = False, flush = False):
+        ops = 0
+        if write:
+            ops += self.total_wr_ops
+            if self.account_failed:
+                ops += self.failed_wr_ops
+            if self.account_invalid:
+                ops += self.invalid_wr_ops
+        if read:
+            ops += self.total_rd_ops
+            if self.account_failed:
+                ops += self.failed_rd_ops
+            if self.account_invalid:
+                ops += self.invalid_rd_ops
+        if flush:
+            ops += self.total_flush_ops
+        return ops
+
+    def accounted_latency(self, read = False, write = False, flush = False):
+        latency = 0
+        if write:
+            latency += self.total_wr_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_wr_ops * op_latency
+        if read:
+            latency += self.total_rd_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_rd_ops * op_latency
+        if flush:
+            latency += self.total_flush_ops * op_latency
+        return latency
+
+    def check_values(self):
+        stats = self.blockstats('drive0')
+
+        # Check that the totals match with what we have calculated
+        self.assertEqual(self.total_rd_bytes, stats['rd_bytes'])
+        self.assertEqual(self.total_wr_bytes, stats['wr_bytes'])
+        self.assertEqual(self.total_rd_ops, stats['rd_operations'])
+        self.assertEqual(self.total_wr_ops, stats['wr_operations'])
+        self.assertEqual(self.total_flush_ops, stats['flush_operations'])
+        self.assertEqual(self.wr_highest_offset, stats['wr_highest_offset'])
+        self.assertEqual(self.failed_rd_ops, stats['failed_rd_operations'])
+        self.assertEqual(self.failed_wr_ops, stats['failed_wr_operations'])
+        self.assertEqual(self.invalid_rd_ops, stats['invalid_rd_operations'])
+        self.assertEqual(self.invalid_wr_ops, stats['invalid_wr_operations'])
+        self.assertEqual(self.account_invalid, stats['account_invalid'])
+        self.assertEqual(self.account_failed, stats['account_failed'])
+        self.assertEqual(self.total_wr_merged, stats['wr_merged'])
+
+        # Check that there's exactly one interval with the length we defined
+        self.assertEqual(1, len(stats['timed_stats']))
+        timed_stats = stats['timed_stats'][0]
+        self.assertEqual(interval_length, timed_stats['interval_length'])
+
+        total_rd_latency = self.accounted_latency(read = True)
+        if (total_rd_latency != 0):
+            self.assertEqual(total_rd_latency, stats['rd_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_rd_latency_ns'])
+            self.assertLess(0, timed_stats['avg_rd_queue_depth'])
+        else:
+            self.assertEqual(0, stats['rd_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_queue_depth'])
+
+        # min read latency <= avg read latency <= max read latency
+        self.assertLessEqual(timed_stats['min_rd_latency_ns'],
+                             timed_stats['avg_rd_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_rd_latency_ns'],
+                             timed_stats['max_rd_latency_ns'])
+
+        total_wr_latency = self.accounted_latency(write = True)
+        if (total_wr_latency != 0):
+            self.assertEqual(total_wr_latency, stats['wr_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_wr_latency_ns'])
+            self.assertLess(0, timed_stats['avg_wr_queue_depth'])
+        else:
+            self.assertEqual(0, stats['wr_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_queue_depth'])
+
+        # min write latency <= avg write latency <= max write latency
+        self.assertLessEqual(timed_stats['min_wr_latency_ns'],
+                             timed_stats['avg_wr_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_wr_latency_ns'],
+                             timed_stats['max_wr_latency_ns'])
+
+        total_flush_latency = self.accounted_latency(flush = True)
+        if (total_flush_latency != 0):
+            self.assertEqual(total_flush_latency, stats['flush_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_flush_latency_ns'])
+        else:
+            self.assertEqual(0, stats['flush_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_flush_latency_ns'])
+
+        # min flush latency <= avg flush latency <= max flush latency
+        self.assertLessEqual(timed_stats['min_flush_latency_ns'],
+                             timed_stats['avg_flush_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_flush_latency_ns'],
+                             timed_stats['max_flush_latency_ns'])
+
+        # idle_time_ns must be > 0 if we have performed any operation
+        if (self.accounted_ops(read = True, write = True, flush = True) != 0):
+            self.assertLess(0, stats['idle_time_ns'])
+        else:
+            self.assertFalse(stats.has_key('idle_time_ns'))
+
+        # This test does not alter these, so they must be all 0
+        self.assertEqual(0, stats['rd_merged'])
+        self.assertEqual(0, stats['failed_flush_operations'])
+        self.assertEqual(0, stats['invalid_flush_operations'])
+
+    def do_test_stats(self, rd_size = 0, rd_ops = 0, wr_size = 0, wr_ops = 0,
+                      flush_ops = 0, invalid_rd_ops = 0, invalid_wr_ops = 0,
+                      failed_rd_ops = 0, failed_wr_ops = 0, wr_merged = 0):
+        # The 'ops' list will contain all the requested I/O operations
+        ops = []
+        for i in range(rd_ops):
+            ops.append("aio_read %d %d" % (i * rd_size, rd_size))
+
+        for i in range(wr_ops):
+            ops.append("aio_write %d %d" % (i * wr_size, wr_size))
+
+        for i in range(flush_ops):
+            ops.append("aio_flush")
+
+        highest_offset = wr_ops * wr_size
+
+        # Two types of invalid operations: unaligned length and unaligned offset
+        for i in range(invalid_rd_ops / 2):
+            ops.append("aio_read 0 511")
+
+        for i in range(invalid_rd_ops / 2, invalid_rd_ops):
+            ops.append("aio_read 13 512")
+
+        for i in range(invalid_wr_ops / 2):
+            ops.append("aio_write 0 511")
+
+        for i in range(invalid_wr_ops / 2, invalid_wr_ops):
+            ops.append("aio_write 13 512")
+
+        for i in range(failed_rd_ops):
+            ops.append("aio_read %d 512" % bad_offset)
+
+        for i in range(failed_wr_ops):
+            ops.append("aio_write %d 512" % bad_offset)
+
+        if failed_wr_ops > 0:
+            highest_offset = max(highest_offset, bad_offset + 512)
+
+        for i in range(wr_merged):
+            first = i * wr_size * 2
+            second = first + wr_size
+            ops.append("multiwrite %d %d ; %d %d" %
+                       (first, wr_size, second, wr_size))
+
+        highest_offset = max(highest_offset, wr_merged * wr_size * 2)
+
+        # Now perform all operations
+        for op in ops:
+            self.vm.hmp_qemu_io("drive0", op)
+
+        # Update the expected totals
+        self.total_rd_bytes += rd_ops * rd_size
+        self.total_rd_ops += rd_ops
+        self.total_wr_bytes += wr_ops * wr_size
+        self.total_wr_ops += wr_ops
+        self.total_wr_merged += wr_merged
+        self.total_flush_ops += flush_ops
+        self.invalid_rd_ops += invalid_rd_ops
+        self.invalid_wr_ops += invalid_wr_ops
+        self.failed_rd_ops += failed_rd_ops
+        self.failed_wr_ops += failed_wr_ops
+
+        self.wr_highest_offset = max(self.wr_highest_offset, highest_offset)
+
+        # Advance the clock so idle_time_ns has a meaningful value
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+        # And check that the actual statistics match the expected ones
+        self.check_values()
+
+    def test_read_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(rd_size = i[0], rd_ops = i[1])
+
+    def test_write_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(wr_size = i[0], wr_ops = i[1])
+
+    def test_invalid(self):
+        self.do_test_stats(invalid_rd_ops = 7)
+        self.do_test_stats(invalid_wr_ops = 3)
+        self.do_test_stats(invalid_rd_ops = 4, invalid_wr_ops = 5)
+
+    def test_failed(self):
+        self.do_test_stats(failed_rd_ops = 8)
+        self.do_test_stats(failed_wr_ops = 6)
+        self.do_test_stats(failed_rd_ops = 5, failed_wr_ops = 12)
+
+    def test_flush(self):
+        self.do_test_stats(flush_ops = 8)
+
+    def test_merged(self):
+        for i in range(5):
+            self.do_test_stats(wr_merged = i * 3)
+
+    def test_all(self):
+        # rd_size, rd_ops, wr_size, wr_ops, flush_ops
+        # invalid_rd_ops,  invalid_wr_ops,
+        # failed_rd_ops,   failed_wr_ops
+        # wr_merged
+        test_values = [[512,    1, 512,   1, 1, 4, 7, 5, 2, 1],
+                       [65536,  1, 2048, 12, 7, 7, 5, 2, 5, 5],
+                       [32768,  9, 8192,  1, 4, 3, 2, 4, 6, 4],
+                       [16384, 11, 3584, 16, 9, 8, 6, 7, 3, 4]]
+        for i in test_values:
+            self.do_test_stats(*i)
+
+    def test_no_op(self):
+        # All values must be sane before doing any I/O
+        self.check_values()
+
+
+class BlockDeviceStatsTestAccountInvalid(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = False
+
+class BlockDeviceStatsTestAccountFailed(BlockDeviceStatsTestCase):
+    account_invalid = False
+    account_failed = True
+
+class BlockDeviceStatsTestAccountBoth(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = True
+
+class BlockDeviceStatsTestCoroutine(BlockDeviceStatsTestCase):
+    test_img = "null-co://"
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["raw"])
diff --git a/qemu/tests/qemu-iotests/136.out b/qemu/tests/qemu-iotests/136.out
new file mode 100644
index 000000000..0a5e9583a
--- /dev/null
+++ b/qemu/tests/qemu-iotests/136.out
@@ -0,0 +1,5 @@
+........................................
+----------------------------------------------------------------------
+Ran 40 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/137 b/qemu/tests/qemu-iotests/137
new file mode 100755
index 000000000..e5e30de2f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/137
@@ -0,0 +1,144 @@
+#!/bin/bash
+#
+# Test qcow2 reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+_make_test_img 64M
+
+echo === Try setting valid values for all options ===
+echo
+
+# Try all options and then check that all of the basic I/O operations still
+# work on this image.
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,pass-discard-request=on" \
+    -c "reopen -o lazy-refcounts=off,pass-discard-request=off" \
+    -c "reopen -o pass-discard-snapshot=on,pass-discard-other=on" \
+    -c "reopen -o pass-discard-snapshot=off,pass-discard-other=off" \
+    -c "reopen -o overlap-check=all" \
+    -c "reopen -o overlap-check=none" \
+    -c "reopen -o overlap-check=cached" \
+    -c "reopen -o overlap-check=constant" \
+    -c "reopen -o overlap-check.template=all" \
+    -c "reopen -o overlap-check.template=none" \
+    -c "reopen -o overlap-check.template=cached" \
+    -c "reopen -o overlap-check.template=constant" \
+    -c "reopen -o overlap-check.main-header=on" \
+    -c "reopen -o overlap-check.main-header=off" \
+    -c "reopen -o overlap-check.active-l1=on" \
+    -c "reopen -o overlap-check.active-l1=off" \
+    -c "reopen -o overlap-check.active-l2=on" \
+    -c "reopen -o overlap-check.active-l2=off" \
+    -c "reopen -o overlap-check.refcount-table=on" \
+    -c "reopen -o overlap-check.refcount-table=off" \
+    -c "reopen -o overlap-check.refcount-block=on" \
+    -c "reopen -o overlap-check.refcount-block=off" \
+    -c "reopen -o overlap-check.snapshot-table=on" \
+    -c "reopen -o overlap-check.snapshot-table=off" \
+    -c "reopen -o overlap-check.inactive-l1=on" \
+    -c "reopen -o overlap-check.inactive-l1=off" \
+    -c "reopen -o overlap-check.inactive-l2=on" \
+    -c "reopen -o overlap-check.inactive-l2=off" \
+    -c "reopen -o cache-size=1M" \
+    -c "reopen -o l2-cache-size=512k" \
+    -c "reopen -o refcount-cache-size=128k" \
+    -c "reopen -o cache-clean-interval=5" \
+    -c "reopen -o cache-clean-interval=0" \
+    -c "reopen -o cache-clean-interval=10" \
+    \
+    -c "write -P 55 0 32M" \
+    -c "read -P 55 0 32M" \
+    -c "discard 0 32M" \
+    -c "write -z 0 32M" \
+    -c "read -P 0 0 32M" \
+    \
+    "$TEST_IMG" | _filter_qemu_io
+
+
+echo
+echo === Try setting some invalid values ===
+echo
+
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=42" \
+    -c "reopen -o cache-size=1M,l2-cache-size=64k,refcount-cache-size=64k" \
+    -c "reopen -o cache-size=1M,l2-cache-size=2M" \
+    -c "reopen -o cache-size=1M,refcount-cache-size=2M" \
+    -c "reopen -o l2-cache-size=256T" \
+    -c "reopen -o refcount-cache-size=256T" \
+    -c "reopen -o overlap-check=constant,overlap-check.template=all" \
+    -c "reopen -o overlap-check=blubb" \
+    -c "reopen -o overlap-check.template=blubb" \
+    -c "reopen -o cache-clean-interval=-1" \
+    "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo === Test transaction semantics ===
+echo
+
+# Whether lazy-refcounts was actually enabled can easily be tested: Check if
+# the dirty bit is set after a crash
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,overlap-check=blubb" \
+    -c "write -P 0x5a 0 512" \
+    -c "sigraise $(kill -l KILL)" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Similarly we can test whether corruption detection has been enabled:
+# Create L1/L2, overwrite first entry in refcount block, allocate something.
+# Disabling the checks should fail, so the corruption must be detected.
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00"
+$QEMU_IO \
+    -c "reopen -o overlap-check=none,lazy-refcounts=42" \
+    -c "write 64k 64k" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/137.out b/qemu/tests/qemu-iotests/137.out
new file mode 100644
index 000000000..88c702cf7
--- /dev/null
+++ b/qemu/tests/qemu-iotests/137.out
@@ -0,0 +1,46 @@
+QA output created by 137
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+=== Try setting valid values for all options ===
+
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+discard 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Try setting some invalid values ===
+
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+l2-cache-size may not exceed cache-size
+refcount-cache-size may not exceed cache-size
+L2 cache size too big
+L2 cache size too big
+Conflicting values for qcow2 options 'overlap-check' ('constant') and 'overlap-check.template' ('all')
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Cache clean interval too big
+
+=== Test transaction semantics ===
+
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@";
+fi )
+incompatible_features     0x0
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+write failed: Input/output error
+*** done
diff --git a/qemu/tests/qemu-iotests/138 b/qemu/tests/qemu-iotests/138
new file mode 100755
index 000000000..21650d819
--- /dev/null
+++ b/qemu/tests/qemu-iotests/138
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# General test case for qcow2's image check
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== Check on an image with a multiple of 2^32 clusters ==='
+echo
+
+IMGOPTS=$(_optstr_add "$IMGOPTS" "cluster_size=512") \
+    _make_test_img 512
+
+# Allocate L2 table
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Put the data cluster at a multiple of 2 TB, resulting in the image apparently
+# having a multiple of 2^32 clusters
+# (To be more specific: It is at 32 PB)
+poke_file "$TEST_IMG" 2048 "\x80\x80\x00\x00\x00\x00\x00\x00"
+
+# An offset of 32 PB results in qemu-img check having to allocate an in-memory
+# refcount table of 128 TB (16 bit refcounts, 512 byte clusters).
+# This should be generally too much for any system and thus fail.
+# What this test is checking is that the qcow2 driver actually tries to allocate
+# such a large amount of memory (and is consequently aborting) instead of having
+# truncated the cluster count somewhere (which would result in much less memory
+# being allocated and then a segfault occurring).
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/138.out b/qemu/tests/qemu-iotests/138.out
new file mode 100644
index 000000000..3fe911f85
--- /dev/null
+++ b/qemu/tests/qemu-iotests/138.out
@@ -0,0 +1,9 @@
+QA output created by 138
+
+=== Check on an image with a multiple of 2^32 clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Check failed: Cannot allocate memory
+*** done
diff --git a/qemu/tests/qemu-iotests/139 b/qemu/tests/qemu-iotests/139
new file mode 100644
index 000000000..a4b969499
--- /dev/null
+++ b/qemu/tests/qemu-iotests/139
@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+#
+# Test cases for the QMP 'x-blockdev-del' command
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+import time
+
+base_img = os.path.join(iotests.test_dir, 'base.img')
+new_img = os.path.join(iotests.test_dir, 'new.img')
+
+class TestBlockdevDel(iotests.QMPTestCase):
+
+    def setUp(self):
+        iotests.qemu_img('create', '-f', iotests.imgfmt, base_img, '1M')
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(base_img)
+        if os.path.isfile(new_img):
+            os.remove(new_img)
+
+    # Check whether a BlockBackend exists
+    def checkBlockBackend(self, backend, node, must_exist = True):
+        result = self.vm.qmp('query-block')
+        backends = filter(lambda x: x['device'] == backend, result['return'])
+        self.assertLessEqual(len(backends), 1)
+        self.assertEqual(must_exist, len(backends) == 1)
+        if must_exist:
+            if node:
+                self.assertEqual(backends[0]['inserted']['node-name'], node)
+            else:
+                self.assertFalse(backends[0].has_key('inserted'))
+
+    # Check whether a BlockDriverState exists
+    def checkBlockDriverState(self, node, must_exist = True):
+        result = self.vm.qmp('query-named-block-nodes')
+        nodes = filter(lambda x: x['node-name'] == node, result['return'])
+        self.assertLessEqual(len(nodes), 1)
+        self.assertEqual(must_exist, len(nodes) == 1)
+
+    # Add a new BlockBackend (with its attached BlockDriverState)
+    def addBlockBackend(self, backend, node):
+        file_node = '%s_file' % node
+        self.checkBlockBackend(backend, node, False)
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'id': backend,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState without a BlockBackend
+    def addBlockDriverState(self, node):
+        file_node = '%s_file' % node
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState that will be used as overlay for the base_img BDS
+    def addBlockDriverStateOverlay(self, node):
+        self.checkBlockDriverState(node, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt,
+                         '-b', base_img, new_img, '1M')
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'backing': '',
+                'file': {'driver': 'file',
+                         'filename': new_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+
+    # Delete a BlockBackend
+    def delBlockBackend(self, backend, node, expect_error = False,
+                        destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        if node:
+            self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', id = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            if node:
+                self.checkBlockDriverState(node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            if node:
+                self.checkBlockDriverState(node, not destroys_media)
+        self.checkBlockBackend(backend, node, must_exist = expect_error)
+
+    # Delete a BlockDriverState
+    def delBlockDriverState(self, node, expect_error = False):
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', node_name = node)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+        else:
+            self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node, expect_error)
+
+    # Add a device model
+    def addDeviceModel(self, device, backend):
+        result = self.vm.qmp('device_add', id = device,
+                             driver = 'virtio-blk-pci', drive = backend)
+        self.assert_qmp(result, 'return', {})
+
+    # Delete a device model
+    def delDeviceModel(self, device):
+        result = self.vm.qmp('device_del', id = device)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('system_reset')
+        self.assert_qmp(result, 'return', {})
+
+        device_path = '/machine/peripheral/%s/virtio-backend' % device
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'path': device_path}})
+        self.assertNotEqual(event, None)
+
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+    # Remove a BlockDriverState
+    def ejectDrive(self, backend, node, expect_error = False,
+                   destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('eject', device = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            self.checkBlockDriverState(node)
+            self.checkBlockBackend(backend, node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            self.checkBlockDriverState(node, not destroys_media)
+            self.checkBlockBackend(backend, None)
+
+    # Insert a BlockDriverState
+    def insertDrive(self, backend, node):
+        self.checkBlockBackend(backend, None)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-insert-medium',
+                             device = backend, node_name = node)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+
+    # Create a snapshot using 'blockdev-snapshot-sync'
+    def createSnapshotSync(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay, False)
+        opts = {'node-name': node,
+                'snapshot-file': new_img,
+                'snapshot-node-name': overlay,
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('blockdev-snapshot-sync', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a snapshot using 'blockdev-snapshot'
+    def createSnapshot(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+        result = self.vm.qmp('blockdev-snapshot',
+                             node = node, overlay = overlay)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a mirror
+    def createMirror(self, backend, node, new_node):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node, False)
+        opts = {'device': backend,
+                'target': new_img,
+                'node-name': new_node,
+                'sync': 'top',
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('drive-mirror', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node)
+
+    # Complete an existing block job
+    def completeBlockJob(self, backend, node_before, node_after):
+        self.checkBlockBackend(backend, node_before)
+        result = self.vm.qmp('block-job-complete', device=backend)
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(backend)
+        self.checkBlockBackend(backend, node_after)
+
+    # Add a BlkDebug node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkdebug drive
+    def addBlkDebug(self, debug, node):
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(debug, False)
+        image = {'driver': iotests.imgfmt,
+                 'node-name': node,
+                 'file': {'driver': 'file',
+                          'filename': base_img}}
+        opts = {'driver': 'blkdebug',
+                'node-name': debug,
+                'image': image}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(debug)
+
+    # Add a BlkVerify node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkverify drive
+    def addBlkVerify(self, blkverify, test, raw):
+        self.checkBlockDriverState(test, False)
+        self.checkBlockDriverState(raw, False)
+        self.checkBlockDriverState(blkverify, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        node_0 = {'driver': iotests.imgfmt,
+                  'node-name': test,
+                  'file': {'driver': 'file',
+                           'filename': base_img}}
+        node_1 = {'driver': iotests.imgfmt,
+                  'node-name': raw,
+                  'file': {'driver': 'file',
+                           'filename': new_img}}
+        opts = {'driver': 'blkverify',
+                'node-name': blkverify,
+                'test': node_0,
+                'raw': node_1}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(test)
+        self.checkBlockDriverState(raw)
+        self.checkBlockDriverState(blkverify)
+
+    # Add a Quorum node
+    def addQuorum(self, quorum, child0, child1):
+        self.checkBlockDriverState(child0, False)
+        self.checkBlockDriverState(child1, False)
+        self.checkBlockDriverState(quorum, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        child_0 = {'driver': iotests.imgfmt,
+                   'node-name': child0,
+                   'file': {'driver': 'file',
+                            'filename': base_img}}
+        child_1 = {'driver': iotests.imgfmt,
+                   'node-name': child1,
+                   'file': {'driver': 'file',
+                            'filename': new_img}}
+        opts = {'driver': 'quorum',
+                'node-name': quorum,
+                'vote-threshold': 1,
+                'children': [ child_0, child_1 ]}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(child0)
+        self.checkBlockDriverState(child1)
+        self.checkBlockDriverState(quorum)
+
+    ########################
+    # The tests start here #
+    ########################
+
+    def testWrongParameters(self):
+        self.addBlockBackend('drive0', 'node0')
+        result = self.vm.qmp('x-blockdev-del')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        result = self.vm.qmp('x-blockdev-del', id='drive0', node_name='node0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockBackend(self):
+        self.addBlockBackend('drive0', 'node0')
+        # You cannot delete a BDS that is attached to a backend
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockDriverState(self):
+        self.addBlockDriverState('node0')
+        # You cannot delete a file BDS directly
+        self.delBlockDriverState('node0_file', expect_error = True)
+        self.delBlockDriverState('node0')
+
+    def testEject(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        self.delBlockBackend('drive0', None)
+
+    def testDeviceModel(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addDeviceModel('device0', 'drive0')
+        self.ejectDrive('drive0', 'node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delDeviceModel('device0')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testAttachMedia(self):
+        # This creates a BlockBackend and removes its media
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        # This creates a new BlockDriverState and inserts it into the backend
+        self.addBlockDriverState('node1')
+        self.insertDrive('drive0', 'node1')
+        # The backend can't be removed: the new BDS has an extra reference
+        self.delBlockBackend('drive0', 'node1', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # The BDS still exists after being ejected, but now it can be removed
+        self.ejectDrive('drive0', 'node1', destroys_media = False)
+        self.delBlockDriverState('node1')
+        self.delBlockBackend('drive0', None)
+
+    def testSnapshotSync(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createSnapshotSync('node0', 'overlay0')
+        # This fails because node0 is now being used as a backing image
+        self.delBlockDriverState('node0', expect_error = True)
+        # This succeeds because overlay0 only has the backend reference
+        self.delBlockBackend('drive0', 'overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testSnapshot(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addBlockDriverStateOverlay('overlay0')
+        self.createSnapshot('node0', 'overlay0')
+        self.delBlockBackend('drive0', 'overlay0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0', expect_error = True)
+        self.ejectDrive('drive0', 'overlay0', destroys_media = False)
+        self.delBlockBackend('drive0', None)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testMirror(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createMirror('drive0', 'node0', 'mirror0')
+        # The block job prevents removing the device
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('mirror0', expect_error = True)
+        self.wait_ready('drive0')
+        self.completeBlockJob('drive0', 'node0', 'mirror0')
+        self.assert_no_active_block_jobs()
+        self.checkBlockDriverState('node0', False)
+        # This succeeds because the backend now points to mirror0
+        self.delBlockBackend('drive0', 'mirror0')
+
+    def testBlkDebug(self):
+        self.addBlkDebug('debug0', 'node0')
+        # 'node0' is used by the blkdebug node
+        self.delBlockDriverState('node0', expect_error = True)
+        # But we can remove the blkdebug node directly
+        self.delBlockDriverState('debug0')
+        self.checkBlockDriverState('node0', False)
+
+    def testBlkVerify(self):
+        self.addBlkVerify('verify0', 'node0', 'node1')
+        # We cannot remove the children of a blkverify device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the blkverify node directly
+        self.delBlockDriverState('verify0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+    def testQuorum(self):
+        if not 'quorum' in iotests.qemu_img_pipe('--help'):
+            return
+        self.addQuorum('quorum0', 'node0', 'node1')
+        # We cannot remove the children of a Quorum device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the Quorum node directly
+        self.delBlockDriverState('quorum0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["qcow2"])
diff --git a/qemu/tests/qemu-iotests/139.out b/qemu/tests/qemu-iotests/139.out
new file mode 100644
index 000000000..281b69efe
--- /dev/null
+++ b/qemu/tests/qemu-iotests/139.out
@@ -0,0 +1,5 @@
+............
+----------------------------------------------------------------------
+Ran 12 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/140 b/qemu/tests/qemu-iotests/140
new file mode 100755
index 000000000..49f9df4eb
--- /dev/null
+++ b/qemu/tests/qemu-iotests/140
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Test case for ejecting a BlockBackend with an NBD server attached to it
+#
+# Verify that the NBD server stops offering the drive when ejecting a
+# BlockDriverState tree from a BlockBackend (that is, a medium from a
+# drive) exposed via an NBD server.
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_DIR/nbd"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt generic
+_supported_proto file
+_supported_os Linux
+
+_make_test_img 64k
+
+$QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+keep_stderr=y \
+_launch_qemu -drive if=none,media=cdrom,id=drv,file="$TEST_IMG",format=$IMGFMT \
+    2> >(_filter_nbd)
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-start',
+       'arguments': { 'addr': { 'type': 'unix',
+                                'data': { 'path': '$TEST_DIR/nbd' }}}}" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-add',
+       'arguments': { 'device': 'drv' }}" \
+    'return'
+
+$QEMU_IO_PROG -f raw -c 'read -P 42 0 64k' \
+    "nbd+unix:///drv?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'eject',
+       'arguments': { 'device': 'drv' }}" \
+    'return'
+
+$QEMU_IO_PROG -f raw -c close \
+    "nbd+unix:///drv?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/140.out b/qemu/tests/qemu-iotests/140.out
new file mode 100644
index 000000000..0409cd017
--- /dev/null
+++ b/qemu/tests/qemu-iotests/140.out
@@ -0,0 +1,15 @@
+QA output created by 140
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"return": {}}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+can't open device nbd+unix:///drv?socket=TEST_DIR/nbd: No export with name 'drv' available
+no file open, try 'help open'
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+*** done
diff --git a/qemu/tests/qemu-iotests/141 b/qemu/tests/qemu-iotests/141
new file mode 100755
index 000000000..b2617e5e2
--- /dev/null
+++ b/qemu/tests/qemu-iotests/141
@@ -0,0 +1,185 @@
+#!/bin/bash
+#
+# Test case for ejecting BDSs with block jobs still running on them
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_DIR/{b,m,o}.$IMGFMT"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+# Needs backing file and backing format support
+_supported_fmt qcow2 qed
+_supported_proto file
+_supported_os Linux
+
+
+test_blockjob()
+{
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'blockdev-add',
+          'arguments': {
+              'options': {
+                  'id': 'drv0',
+                  'driver': '$IMGFMT',
+                  'file': {
+                      'driver': 'file',
+                      'filename': '$TEST_IMG'
+                  }}}}" \
+        'return'
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "$1" \
+        "$2" \
+        | _filter_img_create
+
+    # We want this to return an error because the block job is still running
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'x-blockdev-remove-medium',
+          'arguments': {'device': 'drv0'}}" \
+        'error'
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'block-job-cancel',
+          'arguments': {'device': 'drv0'}}" \
+        "$3"
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'x-blockdev-del',
+          'arguments': {'id': 'drv0'}}" \
+        'return'
+}
+
+
+TEST_IMG="$TEST_DIR/b.$IMGFMT" _make_test_img 1M
+TEST_IMG="$TEST_DIR/m.$IMGFMT" _make_test_img -b "$TEST_DIR/b.$IMGFMT" 1M
+_make_test_img -b "$TEST_DIR/m.$IMGFMT" 1M
+
+_launch_qemu -nodefaults
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{'execute': 'qmp_capabilities'}" \
+    'return'
+
+echo
+echo '=== Testing drive-backup ==='
+echo
+
+# drive-backup will not send BLOCK_JOB_READY by itself, and cancelling the job
+# will consequently result in BLOCK_JOB_CANCELLED being emitted.
+
+test_blockjob \
+    "{'execute': 'drive-backup',
+      'arguments': {'device': 'drv0',
+                    'target': '$TEST_DIR/o.$IMGFMT',
+                    'format': '$IMGFMT',
+                    'sync': 'none'}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+echo
+echo '=== Testing drive-mirror ==='
+echo
+
+# drive-mirror will send BLOCK_JOB_READY basically immediately, and cancelling
+# the job will consequently result in BLOCK_JOB_COMPLETED being emitted.
+
+test_blockjob \
+    "{'execute': 'drive-mirror',
+      'arguments': {'device': 'drv0',
+                    'target': '$TEST_DIR/o.$IMGFMT',
+                    'format': '$IMGFMT',
+                    'sync': 'none'}}" \
+    'BLOCK_JOB_READY' \
+    'BLOCK_JOB_COMPLETED'
+
+echo
+echo '=== Testing active block-commit ==='
+echo
+
+# An active block-commit will send BLOCK_JOB_READY basically immediately, and
+# cancelling the job will consequently result in BLOCK_JOB_COMPLETED being
+# emitted.
+
+test_blockjob \
+    "{'execute': 'block-commit',
+      'arguments': {'device': 'drv0'}}" \
+    'BLOCK_JOB_READY' \
+    'BLOCK_JOB_COMPLETED'
+
+echo
+echo '=== Testing non-active block-commit ==='
+echo
+
+# Give block-commit something to work on, otherwise it would be done
+# immediately, send a BLOCK_JOB_COMPLETED and ejecting the BDS would work just
+# fine without the block job still running.
+
+$QEMU_IO -c 'write 0 1M' "$TEST_DIR/m.$IMGFMT" | _filter_qemu_io
+
+test_blockjob \
+    "{'execute': 'block-commit',
+      'arguments': {'device': 'drv0',
+                    'top':    '$TEST_DIR/m.$IMGFMT',
+                    'speed':  1}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+echo
+echo '=== Testing block-stream ==='
+echo
+
+# Give block-stream something to work on, otherwise it would be done
+# immediately, send a BLOCK_JOB_COMPLETED and ejecting the BDS would work just
+# fine without the block job still running.
+
+$QEMU_IO -c 'write 0 1M' "$TEST_DIR/b.$IMGFMT" | _filter_qemu_io
+
+# With some data to stream (and @speed set to 1), block-stream will not complete
+# until we send the block-job-cancel command. Therefore, no event other than
+# BLOCK_JOB_CANCELLED will be emitted.
+
+test_blockjob \
+    "{'execute': 'block-stream',
+      'arguments': {'device': 'drv0',
+                    'speed': 1}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+_cleanup_qemu
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/141.out b/qemu/tests/qemu-iotests/141.out
new file mode 100644
index 000000000..adceac181
--- /dev/null
+++ b/qemu/tests/qemu-iotests/141.out
@@ -0,0 +1,59 @@
+QA output created by 141
+Formatting 'TEST_DIR/b.IMGFMT', fmt=IMGFMT size=1048576
+Formatting 'TEST_DIR/m.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/b.IMGFMT
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/m.IMGFMT
+{"return": {}}
+
+=== Testing drive-backup ===
+
+{"return": {}}
+Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: backup"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 0, "speed": 0, "type": "backup"}}
+{"return": {}}
+
+=== Testing drive-mirror ===
+
+{"return": {}}
+Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
+{"return": {}}
+
+=== Testing active block-commit ===
+
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+
+=== Testing non-active block-commit ===
+
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 524288, "speed": 1, "type": "commit"}}
+{"return": {}}
+
+=== Testing block-stream ===
+
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: stream"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 524288, "speed": 1, "type": "stream"}}
+{"return": {}}
+*** done
diff --git a/qemu/tests/qemu-iotests/142 b/qemu/tests/qemu-iotests/142
new file mode 100755
index 000000000..29c0606bd
--- /dev/null
+++ b/qemu/tests/qemu-iotests/142
@@ -0,0 +1,358 @@
+#!/bin/bash
+#
+# Test for configuring cache modes of arbitrary nodes (requires O_DIRECT)
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f $TEST_IMG.snap
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+# We test all cache modes anyway, but O_DIRECT needs to be supported
+_default_cache_mode none
+_supported_cache_modes none directsync
+
+function do_run_qemu()
+{
+    echo Testing: "$@"
+    (
+        if ! test -t 0; then
+            while read cmd; do
+                echo $cmd
+            done
+        fi
+        echo quit
+    ) | $QEMU -nographic -monitor stdio -nodefaults "$@"
+    echo
+}
+
+function run_qemu()
+{
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu
+}
+
+size=128M
+
+TEST_IMG="$TEST_IMG.base" _make_test_img $size
+TEST_IMG="$TEST_IMG.snap" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base" $size
+
+echo
+echo === Simple test for all cache modes ===
+echo
+
+run_qemu -drive file="$TEST_IMG",cache=none
+run_qemu -drive file="$TEST_IMG",cache=directsync
+run_qemu -drive file="$TEST_IMG",cache=writeback
+run_qemu -drive file="$TEST_IMG",cache=writethrough
+run_qemu -drive file="$TEST_IMG",cache=unsafe
+run_qemu -drive file="$TEST_IMG",cache=invalid_value
+
+echo
+echo === Check inheritance of cache modes ===
+echo
+
+files="if=none,file=$TEST_IMG,backing.file.filename=$TEST_IMG.base"
+ids="node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file"
+
+function check_cache_all()
+{
+    # cache.direct is supposed to be inherited by both bs->file and
+    # bs->backing
+
+    echo -e "cache.direct=on on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+
+    # cache.writeback is supposed to be inherited by bs->backing; bs->file
+    # always gets cache.writeback=on
+
+    echo -e "\n\ncache.writeback=off on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.writeback=off | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.writeback=off | grep -e "doesn't" -e "does not"
+    echo -e "\ncache.writeback=off on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.writeback=off | grep -e "doesn't" -e "does not"
+    echo -e "\ncache.writeback=off on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.writeback=off | grep -e "doesn't" -e "does not"
+
+    # cache.no-flush is supposed to be inherited by both bs->file and bs->backing
+
+    echo -e "\n\ncache.no-flush=on on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+}
+
+echo
+echo "--- Configure cache modes on the command line ---"
+echo
+
+# First check the inherited cache mode after opening the image.
+
+hmp_cmds="info block none0
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Cache modes after reopen (live snapshot) ---"
+echo
+
+# Then trigger a reopen and check that the cache modes are still the same.
+
+hmp_cmds="snapshot_blkdev -n none0 $TEST_IMG.snap $IMGFMT
+info block
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, flags) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified in the flags, not as an option.
+
+hmp_cmds='qemu-io none0 "reopen -c none"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, options) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified as an option, not in the flags.
+
+hmp_cmds='qemu-io none0 "reopen -o cache.direct=on"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache modes after snapshot ---"
+echo
+
+# This checks that the original image doesn't inherit from the snapshot
+
+hmp_cmds="snapshot_blkdev -n none0 $TEST_IMG.snap $IMGFMT
+qemu-io none0 \"reopen -c none\"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Change cache mode in parent, child has explicit option in JSON ---"
+echo
+
+# This checks that children with options explicitly set by the json:
+# pseudo-protocol don't inherit these options from their parents.
+#
+# Yes, blkdebug::json:... is criminal, but I can't see another way to have a
+# BDS initialised with the json: pseudo-protocol, but still have it inherit
+# options from its parent node.
+
+hmp_cmds="qemu-io none0 \"reopen -o cache.direct=on,cache.no-flush=on\"
+info block none0
+info block image
+info block blkdebug
+info block file"
+
+echo "$hmp_cmds" | run_qemu -drive if=none,file="blkdebug::json:{\"filename\":\"$TEST_IMG\",,\"cache\":{\"direct\":false}}",node-name=image,file.node-name=blkdebug,file.image.node-name=file | grep "Cache"
+
+echo
+echo "=== Check that referenced BDSes don't inherit ==="
+echo
+
+drv_bkfile="if=none,driver=file,filename=$TEST_IMG.base,node-name=backing-file"
+drv_bk="if=none,file=json:{'driver':'$IMGFMT',,'file':'backing-file',,'node-name':'backing'}"
+drv_file="if=none,driver=file,filename=$TEST_IMG,node-name=file"
+drv_img="if=none,id=blk,file=json:{'driver':'$IMGFMT',,'file':'file',,'backing':'backing',,'node-name':'image'}"
+
+function check_cache_all_separate()
+{
+    # Check cache.direct
+
+    echo -e "cache.direct=on on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.direct=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.direct=on -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.direct=on -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.direct=on -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+
+    # Check cache.writeback
+
+    echo -e "\n\ncache.writeback=off on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.writeback=off | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.writeback=off -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.writeback=off -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.writeback=off -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+
+    # Check cache.no-flush
+
+    echo -e "\n\ncache.no-flush=on on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.no-flush=on -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.no-flush=on -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.no-flush=on -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+}
+
+echo
+echo "--- Configure cache modes on the command line ---"
+echo
+
+# First check the inherited cache mode after opening the image.
+
+hmp_cmds="info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all_separate
+
+echo
+echo "--- Cache modes after reopen (live snapshot) ---"
+echo
+
+# Then trigger a reopen and check that the cache modes are still the same.
+
+hmp_cmds="snapshot_blkdev -n blk $TEST_IMG.snap $IMGFMT
+info block blk
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all_separate
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, flags) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified as flags, not as option.
+
+hmp_cmds='qemu-io blk "reopen -c none"
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all_separate
+
+
+echo
+echo "=== Reopening children instead of the root ==="
+echo
+
+files="if=none,file=$TEST_IMG,backing.file.filename=$TEST_IMG.base"
+ids="node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file"
+
+echo
+echo "--- Basic reopen ---"
+echo
+
+hmp_cmds='qemu-io none0 "reopen -o backing.cache.direct=on"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache mode after reopening child ---"
+echo
+
+# This checks that children with options explicitly set with reopen don't
+# inherit these options from their parents any more
+
+# TODO Implement node-name support for 'qemu-io' HMP command for -c
+# Can use only -o to access child node options for now
+
+hmp_cmds="qemu-io none0 \"reopen -o file.cache.direct=off,file.cache.no-flush=off\"
+qemu-io none0 \"reopen -o backing.file.cache.direct=off,backing.file.cache.no-flush=on\"
+qemu-io none0 \"reopen -c none\"
+info block image
+info block file
+info block backing
+info block backing-file"
+
+echo "$hmp_cmds" | run_qemu -drive "$files","$ids" | grep "Cache"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/142.out b/qemu/tests/qemu-iotests/142.out
new file mode 100644
index 000000000..600beca8f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/142.out
@@ -0,0 +1,750 @@
+QA output created by 142
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.snap', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Simple test for all cache modes ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=none
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=directsync
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value: invalid cache option
+
+
+=== Check inheritance of cache modes ===
+
+
+--- Configure cache modes on the command line ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Cache modes after reopen (live snapshot) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, flags) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, options) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache modes after snapshot ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache mode in parent, child has explicit option in JSON ---
+
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+=== Check that referenced BDSes don't inherit ===
+
+
+--- Configure cache modes on the command line ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Cache modes after reopen (live snapshot) ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, flags) ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+=== Reopening children instead of the root ===
+
+
+--- Basic reopen ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache mode after reopening child ---
+
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+*** done
diff --git a/qemu/tests/qemu-iotests/143 b/qemu/tests/qemu-iotests/143
new file mode 100755
index 000000000..ec4ef2221
--- /dev/null
+++ b/qemu/tests/qemu-iotests/143
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Test case for connecting to a non-existing NBD export name
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f "$TEST_DIR/nbd"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+keep_stderr=y \
+_launch_qemu 2> >(_filter_nbd)
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-start',
+       'arguments': { 'addr': { 'type': 'unix',
+                                'data': { 'path': '$TEST_DIR/nbd' }}}}" \
+    'return'
+
+# This should just result in a client error, not in the server crashing
+$QEMU_IO_PROG -f raw -c quit \
+    "nbd+unix:///no_such_export?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/143.out b/qemu/tests/qemu-iotests/143.out
new file mode 100644
index 000000000..d24ad20db
--- /dev/null
+++ b/qemu/tests/qemu-iotests/143.out
@@ -0,0 +1,7 @@
+QA output created by 143
+{"return": {}}
+{"return": {}}
+can't open device nbd+unix:///no_such_export?socket=TEST_DIR/nbd: No export with name 'no_such_export' available
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+*** done
diff --git a/qemu/tests/qemu-iotests/144 b/qemu/tests/qemu-iotests/144
new file mode 100755
index 000000000..00de3c33c
--- /dev/null
+++ b/qemu/tests/qemu-iotests/144
@@ -0,0 +1,114 @@
+#!/bin/bash
+# Check live snapshot, followed by active commit, and another snapshot.
+#
+# This test is to catch the error case of BZ #1300209:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1300209
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+TMP_SNAP1=${TEST_DIR}/tmp.qcow2
+TMP_SNAP2=${TEST_DIR}/tmp2.qcow2
+
+_cleanup()
+{
+    _cleanup_qemu
+    rm -f "${TEST_IMG}" "${TMP_SNAP1}" "${TMP_SNAP2}"
+}
+
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+size=512M
+
+_make_test_img $size
+
+echo
+echo === Launching QEMU ===
+echo
+
+qemu_comm_method="qmp"
+_launch_qemu -drive file="${TEST_IMG}",if=virtio
+h=$QEMU_HANDLE
+
+
+echo
+echo === Performing Live Snapshot 1 ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
+
+
+# First live snapshot, new overlay as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                             'device': 'virtio0',
+                                             'snapshot-file':'${TMP_SNAP1}',
+                                             'format': 'qcow2'
+                                             }
+                    }" "return"
+
+echo
+echo === Performing block-commit on active layer ===
+echo
+
+# Block commit on active layer, push the new overlay into base
+_send_qemu_cmd $h "{ 'execute': 'block-commit',
+                                'arguments': {
+                                                 'device': 'virtio0'
+                                              }
+                    }" "READY"
+
+_send_qemu_cmd $h "{ 'execute': 'block-job-complete',
+                                'arguments': {
+                                                'device': 'virtio0'
+                                              }
+                   }" "COMPLETED"
+
+echo
+echo === Performing Live Snapshot 2 ===
+echo
+
+# New live snapshot, new overlays as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                                'device': 'virtio0',
+                                                'snapshot-file':'${TMP_SNAP2}',
+                                                'format': 'qcow2'
+                                              }
+                   }" "return"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/144.out b/qemu/tests/qemu-iotests/144.out
new file mode 100644
index 000000000..410d74180
--- /dev/null
+++ b/qemu/tests/qemu-iotests/144.out
@@ -0,0 +1,24 @@
+QA output created by 144
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=536870912
+
+=== Launching QEMU ===
+
+
+=== Performing Live Snapshot 1 ===
+
+{"return": {}}
+Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+
+=== Performing block-commit on active layer ===
+
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+
+=== Performing Live Snapshot 2 ===
+
+Formatting 'TEST_DIR/tmp2.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+*** done
diff --git a/qemu/tests/qemu-iotests/145 b/qemu/tests/qemu-iotests/145
new file mode 100755
index 000000000..1eca0e899
--- /dev/null
+++ b/qemu/tests/qemu-iotests/145
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Test the combination of -incoming and snapshot=on
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+	true
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+_make_test_img 1M
+echo quit | $QEMU -nographic -hda "$TEST_IMG" -incoming 'exec:true' -snapshot -serial none -monitor stdio | _filter_qemu
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/145.out b/qemu/tests/qemu-iotests/145.out
new file mode 100644
index 000000000..75b5c8ac3
--- /dev/null
+++ b/qemu/tests/qemu-iotests/145.out
@@ -0,0 +1,5 @@
+QA output created by 145
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) qququiquit
+*** done
diff --git a/qemu/tests/qemu-iotests/146 b/qemu/tests/qemu-iotests/146
new file mode 100755
index 000000000..043711be6
--- /dev/null
+++ b/qemu/tests/qemu-iotests/146
@@ -0,0 +1,165 @@
+#!/bin/bash
+#
+# Test VHD image format creator detection and override
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1    # failure is the default!
+
+_cleanup()
+{
+    _cleanup_qemu
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt vpc
+_supported_proto file
+_supported_os Linux
+
+
+qemu_comm_method="monitor"
+silent=
+
+echo
+echo === Testing VPC Autodetect ===
+echo
+_use_sample_img virtualpc-dynamic.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing VPC with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing VPC with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing Hyper-V Autodetect ===
+echo
+_use_sample_img hyperv2012r2-dynamic.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Hyper-V with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Hyper-V with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing d2v Autodetect ===
+echo
+_use_sample_img d2v-zerofilled.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing d2v with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing d2v with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing Image create, default ===
+echo
+
+TEST_IMG="${TEST_DIR}/vpc-create-test.vpc"
+
+_make_test_img 4G
+
+echo
+echo === Read created image, default opts ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=chs ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=current_size ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Image create, force_size ===
+echo
+
+_make_test_img -o force_size 4G
+
+echo
+echo === Read created image, default opts ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=chs ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=current_size ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/146.out b/qemu/tests/qemu-iotests/146.out
new file mode 100644
index 000000000..4f334d86b
--- /dev/null
+++ b/qemu/tests/qemu-iotests/146.out
@@ -0,0 +1,70 @@
+QA output created by 146
+
+=== Testing VPC Autodetect ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing VPC with current_size force ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing VPC with chs force ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V Autodetect ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V with current_size force ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V with chs force ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing d2v Autodetect ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing d2v with current_size force ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing d2v with chs force ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing Image create, default ===
+
+Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296
+
+=== Read created image, default opts ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=chs ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=current_size ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Image create, force_size ===
+
+Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296 force_size=on
+
+=== Read created image, default opts ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=chs ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=current_size ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+*** done
diff --git a/qemu/tests/qemu-iotests/148 b/qemu/tests/qemu-iotests/148
new file mode 100644
index 000000000..e01b061fe
--- /dev/null
+++ b/qemu/tests/qemu-iotests/148
@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+#
+# Test the rate limit of QMP events
+#
+# Copyright (C) 2016 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+
+imgs = (os.path.join(iotests.test_dir, 'quorum0.img'),
+        os.path.join(iotests.test_dir, 'quorum1.img'),
+        os.path.join(iotests.test_dir, 'quorum2.img'))
+
+img_conf = (os.path.join(iotests.test_dir, 'quorum0.conf'),
+            os.path.join(iotests.test_dir, 'quorum1.conf'),
+            os.path.join(iotests.test_dir, 'quorum2.conf'))
+
+event_rate = 1000000000
+sector_size = 512
+offset = 10
+
+class TestQuorumEvents(iotests.QMPTestCase):
+    read_pattern = 'quorum'
+
+    def create_blkdebug_file(self, blkdebug_file, bad_sector):
+        file = open(blkdebug_file, 'w')
+        file.write('''
+[inject-error]
+event = "read_aio"
+errno = "5"
+sector = "%d"
+''' % bad_sector)
+        file.close()
+
+    def setUp(self):
+        driveopts = ['driver=quorum', 'vote-threshold=2']
+        driveopts.append('read-pattern=%s' % self.read_pattern)
+        for i in range(len(imgs)):
+            iotests.qemu_img('create', '-f', iotests.imgfmt, imgs[i], '1M')
+            self.create_blkdebug_file(img_conf[i], i + offset)
+            driveopts.append('children.%d.driver=%s' % (i, iotests.imgfmt))
+            driveopts.append('children.%d.file.driver=blkdebug' % i)
+            driveopts.append('children.%d.file.config=%s' % (i, img_conf[i]))
+            driveopts.append('children.%d.file.image.filename=%s' % (i, imgs[i]))
+            driveopts.append('children.%d.node-name=img%d' % (i, i))
+        self.vm = iotests.VM()
+        self.vm.add_drive(None, opts = ','.join(driveopts))
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        for i in range(len(imgs)):
+            os.remove(imgs[i])
+            os.remove(img_conf[i])
+
+    def do_check_event(self, node, sector = 0):
+        if node == None:
+            self.assertEqual(self.vm.get_qmp_event(), None)
+            return
+
+        for event in self.vm.get_qmp_events(wait=True):
+            if event['event'] == 'QUORUM_REPORT_BAD':
+                self.assert_qmp(event, 'data/node-name', node)
+                self.assert_qmp(event, 'data/sector-num', sector)
+
+    def testQuorum(self):
+        # Generate an error and get an event
+        self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                            (offset * sector_size, sector_size))
+        self.vm.qtest("clock_step 10")
+        self.do_check_event('img0', offset)
+
+        # I/O errors in the same child: only one event is emitted
+        delay = 10
+        for i in range(3):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                (offset * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            self.do_check_event(None)
+
+        # Wait enough so the event is finally emitted
+        self.vm.qtest("clock_step %d" % (2 * event_rate))
+        self.do_check_event('img0', offset)
+
+        # I/O errors in the same child: all events are emitted
+        delay = 2 * event_rate
+        for i in range(3):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                (offset * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            self.do_check_event('img0', offset)
+
+        # I/O errors in different children: all events are emitted
+        delay = 10
+        for i in range(len(imgs)):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                ((offset + i) * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            # In fifo mode only errors in the first child are detected
+            if i > 0 and self.read_pattern == 'fifo':
+                self.do_check_event(None)
+            else:
+                self.do_check_event('img%d' % i, offset + i)
+
+        # I/O errors in different children: all events are emitted
+        delay = 2 * event_rate
+        for i in range(len(imgs)):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                ((offset + i) * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            # In fifo mode only errors in the first child are detected
+            if i > 0 and self.read_pattern == 'fifo':
+                self.do_check_event(None)
+            else:
+                self.do_check_event('img%d' % i, offset + i)
+
+        # No more pending events
+        self.do_check_event(None)
+
+class TestFifoQuorumEvents(TestQuorumEvents):
+    read_pattern = 'fifo'
+
+if __name__ == '__main__':
+    iotests.verify_quorum()
+    iotests.main(supported_fmts=["raw"])
diff --git a/qemu/tests/qemu-iotests/148.out b/qemu/tests/qemu-iotests/148.out
new file mode 100644
index 000000000..fbc63e62f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/148.out
@@ -0,0 +1,5 @@
+..
+----------------------------------------------------------------------
+Ran 2 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/149 b/qemu/tests/qemu-iotests/149
new file mode 100755
index 000000000..52e23d294
--- /dev/null
+++ b/qemu/tests/qemu-iotests/149
@@ -0,0 +1,519 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Creator/Owner: Daniel P. Berrange <berrange@redhat.com>
+#
+# Exercise the QEMU 'luks' block driver to validate interoperability
+# with the Linux dm-crypt + cryptsetup implementation
+
+import subprocess
+import os
+import os.path
+
+import base64
+
+import iotests
+
+
+class LUKSConfig(object):
+    """Represent configuration parameters for a single LUKS
+       setup to be tested"""
+
+    def __init__(self, name, cipher, keylen, mode, ivgen,
+                 ivgen_hash, hash, password=None, passwords=None):
+
+        self.name = name
+        self.cipher = cipher
+        self.keylen = keylen
+        self.mode = mode
+        self.ivgen = ivgen
+        self.ivgen_hash = ivgen_hash
+        self.hash = hash
+
+        if passwords is not None:
+            self.passwords = passwords
+        else:
+            self.passwords = {}
+
+            if password is None:
+                self.passwords["0"] = "123456"
+            else:
+                self.passwords["0"] = password
+
+    def __repr__(self):
+        return self.name
+
+    def image_name(self):
+        return "luks-%s.img" % self.name
+
+    def image_path(self):
+        return os.path.join(iotests.test_dir, self.image_name())
+
+    def device_name(self):
+        return "qiotest-145-%s" % self.name
+
+    def device_path(self):
+        return "/dev/mapper/" + self.device_name()
+
+    def first_password(self):
+        for i in range(8):
+            slot = str(i)
+            if slot in self.passwords:
+                return (self.passwords[slot], slot)
+        raise Exception("No password found")
+
+    def first_password_base64(self):
+        (pw, slot) = self.first_password()
+        return base64.b64encode(pw)
+
+    def active_slots(self):
+        slots = []
+        for i in range(8):
+            slot = str(i)
+            if slot in self.passwords:
+                slots.append(slot)
+        return slots
+
+def verify_passwordless_sudo():
+    """Check whether sudo is configured to allow
+       password-less access to commands"""
+
+    args = ["sudo", "-n", "/bin/true"]
+
+    proc = subprocess.Popen(args,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate()[0]
+
+    if proc.returncode != 0:
+        iotests.notrun('requires password-less sudo access: %s' % msg)
+
+
+def cryptsetup(args, password=None):
+    """Run the cryptsetup command in batch mode"""
+
+    fullargs = ["sudo", "cryptsetup", "-q", "-v"]
+    fullargs.extend(args)
+
+    iotests.log(" ".join(fullargs), filters=[iotests.filter_test_dir])
+    proc = subprocess.Popen(fullargs,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate(password)[0]
+
+    if proc.returncode != 0:
+        raise Exception(msg)
+
+
+def cryptsetup_add_password(config, slot):
+    """Add another password to a LUKS key slot"""
+
+    (password, mainslot) = config.first_password()
+
+    pwfile = os.path.join(iotests.test_dir, "passwd.txt")
+    with open(pwfile, "w") as fh:
+        fh.write(config.passwords[slot])
+
+    try:
+        args = ["luksAddKey", config.image_path(),
+                "--key-slot", slot,
+                "--key-file", "-",
+                pwfile]
+
+        cryptsetup(args, password)
+    finally:
+        os.unlink(pwfile)
+
+
+def cryptsetup_format(config):
+    """Format a new LUKS volume with cryptsetup, adding the
+    first key slot only"""
+
+    (password, slot) = config.first_password()
+
+    args = ["luksFormat"]
+    cipher = config.cipher + "-" + config.mode + "-" + config.ivgen
+    if config.ivgen_hash is not None:
+        cipher = cipher + ":" + config.ivgen_hash
+    args.extend(["--cipher", cipher])
+    if config.mode == "xts":
+        args.extend(["--key-size", str(config.keylen * 2)])
+    else:
+        args.extend(["--key-size", str(config.keylen)])
+    if config.hash is not None:
+        args.extend(["--hash", config.hash])
+    args.extend(["--key-slot", slot])
+    args.extend(["--key-file", "-"])
+    args.append(config.image_path())
+
+    cryptsetup(args, password)
+
+
+def chown(config):
+    """Set the ownership of a open LUKS device to this user"""
+
+    path = config.device_path()
+
+    args = ["sudo", "chown", "%d:%d" % (os.getuid(), os.getgid()), path]
+    iotests.log(" ".join(args), filters=[iotests.filter_chown])
+    proc = subprocess.Popen(args,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate()[0]
+
+    if proc.returncode != 0:
+        raise Exception("Cannot change owner on %s" % path)
+
+
+def cryptsetup_open(config):
+    """Open an image as a LUKS device"""
+
+    (password, slot) = config.first_password()
+
+    args = ["luksOpen", config.image_path(), config.device_name()]
+
+    cryptsetup(args, password)
+
+
+def cryptsetup_close(config):
+    """Close an active LUKS device """
+
+    args = ["luksClose", config.device_name()]
+    cryptsetup(args)
+
+
+def delete_image(config):
+    """Delete a disk image"""
+
+    try:
+        os.unlink(config.image_path())
+        iotests.log("unlink %s" % config.image_path(),
+                    filters=[iotests.filter_test_dir])
+    except Exception as e:
+        pass
+
+
+def create_image(config, size_mb):
+    """Create a bare disk image with requested size"""
+
+    delete_image(config)
+    iotests.log("truncate %s --size %dMB" % (config.image_path(), size_mb),
+                filters=[iotests.filter_test_dir])
+    with open(config.image_path(), "w") as fn:
+        fn.truncate(size_mb * 1024 * 1024)
+
+
+def qemu_img_create(config, size_mb):
+    """Create and format a disk image with LUKS using qemu-img"""
+
+    opts = [
+        "key-secret=sec0",
+        "cipher-alg=%s-%d" % (config.cipher, config.keylen),
+        "cipher-mode=%s" % config.mode,
+        "ivgen-alg=%s" % config.ivgen,
+        "hash-alg=%s" % config.hash,
+    ]
+    if config.ivgen_hash is not None:
+        opts.append("ivgen-hash-alg=%s" % config.ivgen_hash)
+
+    args = ["create", "-f", "luks",
+            "--object",
+            ("secret,id=sec0,data=%s,format=base64" %
+             config.first_password_base64()),
+            "-o", ",".join(opts),
+            config.image_path(),
+            "%dM" % size_mb]
+
+    iotests.log("qemu-img " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_img_pipe(*args), filters=[iotests.filter_test_dir])
+
+def qemu_io_image_args(config, dev=False):
+    """Get the args for access an image or device with qemu-io"""
+
+    if dev:
+        return [
+            "--image-opts",
+            "driver=file,filename=%s" % config.device_path()]
+    else:
+        return [
+            "--object",
+            ("secret,id=sec0,data=%s,format=base64" %
+             config.first_password_base64()),
+            "--image-opts",
+            ("driver=luks,key-secret=sec0,file.filename=%s" %
+             config.image_path())]
+
+def qemu_io_write_pattern(config, pattern, offset_mb, size_mb, dev=False):
+    """Write a pattern of data to a LUKS image or device"""
+
+    args = ["-c", "write -P 0x%x %dM %dM" % (pattern, offset_mb, size_mb)]
+    args.extend(qemu_io_image_args(config, dev))
+    iotests.log("qemu-io " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_io(*args), filters=[iotests.filter_test_dir,
+                                                 iotests.filter_qemu_io])
+
+
+def qemu_io_read_pattern(config, pattern, offset_mb, size_mb, dev=False):
+    """Read a pattern of data to a LUKS image or device"""
+
+    args = ["-c", "read -P 0x%x %dM %dM" % (pattern, offset_mb, size_mb)]
+    args.extend(qemu_io_image_args(config, dev))
+    iotests.log("qemu-io " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_io(*args), filters=[iotests.filter_test_dir,
+                                                 iotests.filter_qemu_io])
+
+
+def test_once(config, qemu_img=False):
+    """Run the test with a desired LUKS configuration. Can either
+       use qemu-img for creating the initial volume, or cryptsetup,
+       in order to test interoperability in both directions"""
+
+    iotests.log("# ================= %s %s =================" % (
+        "qemu-img" if qemu_img else "dm-crypt", config))
+
+    oneKB = 1024
+    oneMB = oneKB * 1024
+    oneGB = oneMB * 1024
+    oneTB = oneGB * 1024
+
+    # 4 TB, so that we pass the 32-bit sector number boundary.
+    # Important for testing correctness of some IV generators
+    # The files are sparse, so not actually using this much space
+    image_size = 4 * oneTB
+    if qemu_img:
+        iotests.log("# Create image")
+        qemu_img_create(config, image_size / oneMB)
+    else:
+        iotests.log("# Create image")
+        create_image(config, image_size / oneMB)
+
+    lowOffsetMB = 100
+    highOffsetMB = 3 * oneTB / oneMB
+
+    try:
+        if not qemu_img:
+            iotests.log("# Format image")
+            cryptsetup_format(config)
+
+            for slot in config.active_slots()[1:]:
+                iotests.log("# Add password slot %s" % slot)
+                cryptsetup_add_password(config, slot)
+
+        # First we'll open the image using cryptsetup and write a
+        # known pattern of data that we'll then verify with QEMU
+
+        iotests.log("# Open dev")
+        cryptsetup_open(config)
+
+        try:
+            iotests.log("# Set dev owner")
+            chown(config)
+
+            iotests.log("# Write test pattern 0xa7")
+            qemu_io_write_pattern(config, 0xa7, lowOffsetMB, 10, dev=True)
+            iotests.log("# Write test pattern 0x13")
+            qemu_io_write_pattern(config, 0x13, highOffsetMB, 10, dev=True)
+        finally:
+            iotests.log("# Close dev")
+            cryptsetup_close(config)
+
+        # Ok, now we're using QEMU to verify the pattern just
+        # written via dm-crypt
+
+        iotests.log("# Read test pattern 0xa7")
+        qemu_io_read_pattern(config, 0xa7, lowOffsetMB, 10, dev=False)
+        iotests.log("# Read test pattern 0x13")
+        qemu_io_read_pattern(config, 0x13, highOffsetMB, 10, dev=False)
+
+
+        # Write a new pattern to the image, which we'll later
+        # verify with dm-crypt
+        iotests.log("# Write test pattern 0x91")
+        qemu_io_write_pattern(config, 0x91, lowOffsetMB, 10, dev=False)
+        iotests.log("# Write test pattern 0x5e")
+        qemu_io_write_pattern(config, 0x5e, highOffsetMB, 10, dev=False)
+
+
+        # Now we're opening the image with dm-crypt once more
+        # and verifying what QEMU wrote, completing the circle
+        iotests.log("# Open dev")
+        cryptsetup_open(config)
+
+        try:
+            iotests.log("# Set dev owner")
+            chown(config)
+
+            iotests.log("# Read test pattern 0x91")
+            qemu_io_read_pattern(config, 0x91, lowOffsetMB, 10, dev=True)
+            iotests.log("# Read test pattern 0x5e")
+            qemu_io_read_pattern(config, 0x5e, highOffsetMB, 10, dev=True)
+        finally:
+            iotests.log("# Close dev")
+            cryptsetup_close(config)
+    finally:
+        iotests.log("# Delete image")
+        delete_image(config)
+        print
+
+
+# Obviously we only work with the luks image format
+iotests.verify_image_format(supported_fmts=['luks'])
+iotests.verify_platform()
+
+# We need sudo in order to run cryptsetup to create
+# dm-crypt devices. This is safe to use on any
+# machine, since all dm-crypt devices are backed
+# by newly created plain files, and have a dm-crypt
+# name prefix of 'qiotest' to avoid clashing with
+# user LUKS volumes
+verify_passwordless_sudo()
+
+
+# If we look at all permutations of cipher, key size,
+# mode, ivgen, hash, there are ~1000 possible configs.
+#
+# We certainly don't want/need to test every permutation
+# to get good validation of interoperability between QEMU
+# and dm-crypt/cryptsetup.
+#
+# The configs below are a representative set that aim to
+# exercise each axis of configurability.
+#
+configs = [
+    # A common LUKS default
+    LUKSConfig("aes-256-xts-plain64-sha1",
+               "aes", 256, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff ciphers
+    LUKSConfig("twofish-256-xts-plain64-sha1",
+               "twofish", 256, "xts", "plain64", None, "sha1"),
+    LUKSConfig("serpent-256-xts-plain64-sha1",
+               "serpent", 256, "xts", "plain64", None, "sha1"),
+    # Should really be xts, but kernel doesn't support xts+cast5
+    # nor does it do essiv+cast5
+    LUKSConfig("cast5-128-cbc-plain64-sha1",
+               "cast5", 128, "cbc", "plain64", None, "sha1"),
+    LUKSConfig("cast6-256-xts-plain64-sha1",
+               "cast6", 256, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff modes / ivgens
+    LUKSConfig("aes-256-cbc-plain-sha1",
+               "aes", 256, "cbc", "plain", None, "sha1"),
+    LUKSConfig("aes-256-cbc-plain64-sha1",
+               "aes", 256, "cbc", "plain64", None, "sha1"),
+    LUKSConfig("aes-256-cbc-essiv-sha256-sha1",
+               "aes", 256, "cbc", "essiv", "sha256", "sha1"),
+    LUKSConfig("aes-256-xts-essiv-sha256-sha1",
+               "aes", 256, "xts", "essiv", "sha256", "sha1"),
+
+
+    # LUKS default but smaller key sizes
+    LUKSConfig("aes-128-xts-plain64-sha256-sha1",
+               "aes", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("aes-192-xts-plain64-sha256-sha1",
+               "aes", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("twofish-128-xts-plain64-sha1",
+               "twofish", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("twofish-192-xts-plain64-sha1",
+               "twofish", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("serpent-128-xts-plain64-sha1",
+               "serpent", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("serpent-192-xts-plain64-sha1",
+               "serpent", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("cast6-128-xts-plain64-sha1",
+               "cast6", 128, "xts", "plain", None, "sha1"),
+    LUKSConfig("cast6-192-xts-plain64-sha1",
+               "cast6", 192, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff hash
+    LUKSConfig("aes-256-xts-plain64-sha256",
+               "aes", 256, "xts", "plain64", None, "sha256"),
+    LUKSConfig("aes-256-xts-plain64-sha512",
+               "aes", 256, "xts", "plain64", None, "sha512"),
+    LUKSConfig("aes-256-xts-plain64-ripemd160",
+               "aes", 256, "xts", "plain64", None, "ripemd160"),
+
+    # Password in slot 3
+    LUKSConfig("aes-256-xts-plain-sha1-pwslot3",
+               "aes", 256, "xts", "plain", None, "sha1",
+               passwords={
+                   "3": "slot3",
+               }),
+
+    # Passwords in every slot
+    LUKSConfig("aes-256-xts-plain-sha1-pwallslots",
+               "aes", 256, "xts", "plain", None, "sha1",
+               passwords={
+                   "0": "slot1",
+                   "1": "slot1",
+                   "2": "slot2",
+                   "3": "slot3",
+                   "4": "slot4",
+                   "5": "slot5",
+                   "6": "slot6",
+                   "7": "slot7",
+               }),
+]
+
+blacklist = [
+    # We don't have a cast-6 cipher impl for QEMU yet
+    "cast6-256-xts-plain64-sha1",
+    "cast6-128-xts-plain64-sha1",
+    "cast6-192-xts-plain64-sha1",
+
+    # GCrypt doesn't support Twofish with 192 bit key
+    "twofish-192-xts-plain64-sha1",
+
+    # We don't have sha512 hash wired up yet
+    "aes-256-xts-plain64-sha512",
+
+    # We don't have ripemd160 hash wired up yet
+    "aes-256-xts-plain64-ripemd160",
+]
+
+whitelist = []
+if "LUKS_CONFIG" in os.environ:
+    whitelist = os.environ["LUKS_CONFIG"].split(",")
+
+for config in configs:
+    if config.name in blacklist:
+        iotests.log("Skipping %s in blacklist" % config.name)
+        continue
+
+    if len(whitelist) > 0 and config.name not in whitelist:
+        iotests.log("Skipping %s not in whitelist" % config.name)
+        continue
+
+    test_once(config, qemu_img=False)
+
+    # XXX we should support setting passwords in a non-0
+    # key slot with 'qemu-img create' in future
+    (pw, slot) = config.first_password()
+    if slot == "0":
+        test_once(config, qemu_img=True)
diff --git a/qemu/tests/qemu-iotests/149.out b/qemu/tests/qemu-iotests/149.out
new file mode 100644
index 000000000..287f01301
--- /dev/null
+++ b/qemu/tests/qemu-iotests/149.out
@@ -0,0 +1,1880 @@
+# ================= dm-crypt aes-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+
+# ================= qemu-img aes-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+
+# ================= dm-crypt twofish-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-twofish-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher twofish-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+
+# ================= qemu-img twofish-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=twofish-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-twofish-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-twofish-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=twofish-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+
+# ================= dm-crypt serpent-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+
+# ================= dm-crypt cast5-128-cbc-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher cast5-cbc-plain64 --key-size 128 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+
+# ================= qemu-img cast5-128-cbc-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=cast5-128,cipher-mode=cbc,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=cast5-128 cipher-mode=cbc ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+
+Skipping cast6-256-xts-plain64-sha1 in blacklist
+# ================= dm-crypt aes-256-cbc-plain-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-plain-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-plain --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+
+# ================= qemu-img aes-256-cbc-plain-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=plain,hash-alg=sha1 TEST_DIR/luks-aes-256-cbc-plain-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-plain-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=plain hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+
+# ================= dm-crypt aes-256-cbc-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+
+# ================= qemu-img aes-256-cbc-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-256-cbc-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+
+# ================= dm-crypt aes-256-cbc-essiv-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-essiv:sha256 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+
+# ================= qemu-img aes-256-cbc-essiv-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=essiv,hash-alg=sha1,ivgen-hash-alg=sha256 TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=essiv ivgen-hash-alg=sha256 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+
+# ================= dm-crypt aes-256-xts-essiv-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-essiv:sha256 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+
+# ================= qemu-img aes-256-xts-essiv-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=essiv,hash-alg=sha1,ivgen-hash-alg=sha256 TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=essiv ivgen-hash-alg=sha256 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+
+# ================= dm-crypt aes-128-xts-plain64-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+
+# ================= qemu-img aes-128-xts-plain64-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+
+# ================= dm-crypt aes-192-xts-plain64-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 384 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+
+# ================= qemu-img aes-192-xts-plain64-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-192,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-192 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+
+# ================= dm-crypt twofish-128-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-twofish-128-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher twofish-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+
+# ================= qemu-img twofish-128-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=twofish-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-twofish-128-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-twofish-128-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=twofish-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+
+Skipping twofish-192-xts-plain64-sha1 in blacklist
+# ================= dm-crypt serpent-128-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-128-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-128-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-128-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-128-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+
+# ================= dm-crypt serpent-192-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-192-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 384 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-192-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-192,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-192-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-192-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-192 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+
+Skipping cast6-128-xts-plain64-sha1 in blacklist
+Skipping cast6-192-xts-plain64-sha1 in blacklist
+# ================= dm-crypt aes-256-xts-plain64-sha256 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain64-sha256.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 512 --hash sha256 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+
+# ================= qemu-img aes-256-xts-plain64-sha256 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha256 TEST_DIR/luks-aes-256-xts-plain64-sha256.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain64-sha256.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha256
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+
+Skipping aes-256-xts-plain64-sha512 in blacklist
+Skipping aes-256-xts-plain64-ripemd160 in blacklist
+# ================= dm-crypt aes-256-xts-plain-sha1-pwslot3 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain --key-size 512 --hash sha1 --key-slot 3 --key-file - TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+
+# ================= dm-crypt aes-256-xts-plain-sha1-pwallslots =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+# Add password slot 1
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 1 --key-file - TEST_DIR/passwd.txt
+# Add password slot 2
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 2 --key-file - TEST_DIR/passwd.txt
+# Add password slot 3
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 3 --key-file - TEST_DIR/passwd.txt
+# Add password slot 4
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 4 --key-file - TEST_DIR/passwd.txt
+# Add password slot 5
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 5 --key-file - TEST_DIR/passwd.txt
+# Add password slot 6
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 6 --key-file - TEST_DIR/passwd.txt
+# Add password slot 7
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 7 --key-file - TEST_DIR/passwd.txt
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+
+# ================= qemu-img aes-256-xts-plain-sha1-pwallslots =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=c2xvdDE=,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain,hash-alg=sha1 TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+
diff --git a/qemu/tests/qemu-iotests/150 b/qemu/tests/qemu-iotests/150
new file mode 100755
index 000000000..ee8f6375f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/150
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Test that qemu-img convert -S 0 fully allocates the target image
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw qcow2
+_supported_proto file
+_supported_os Linux
+
+
+img_size=1048576
+
+
+echo
+echo '=== Mapping sparse conversion ==='
+echo
+
+$QEMU_IMG_PROG convert -O "$IMGFMT" -S 512 \
+    "json:{ 'driver': 'null-co', 'size': $img_size, 'read-zeroes': true }" \
+    "$TEST_IMG"
+
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+
+
+echo
+echo '=== Mapping non-sparse conversion ==='
+echo
+
+$QEMU_IMG convert -O "$IMGFMT" -S 0 \
+    "json:{ 'driver': 'null-co', 'size': $img_size, 'read-zeroes': true }" \
+    "$TEST_IMG"
+
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/qemu/tests/qemu-iotests/150.out b/qemu/tests/qemu-iotests/150.out
new file mode 100644
index 000000000..2a54e8dcf
--- /dev/null
+++ b/qemu/tests/qemu-iotests/150.out
@@ -0,0 +1,11 @@
+QA output created by 150
+
+=== Mapping sparse conversion ===
+
+Offset          Length          File
+
+=== Mapping non-sparse conversion ===
+
+Offset          Length          File
+0               0x100000        TEST_DIR/t.IMGFMT
+*** done
diff --git a/qemu/tests/qemu-iotests/152 b/qemu/tests/qemu-iotests/152
new file mode 100644
index 000000000..fec546d03
--- /dev/null
+++ b/qemu/tests/qemu-iotests/152
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Tests for drive-mirror with source size unaligned to granularity
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+from iotests import qemu_img
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+target_img = os.path.join(iotests.test_dir, 'target.img')
+
+class TestUnaligned(iotests.QMPTestCase):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, test_img, '512')
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_unaligned(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             granularity=65536, target=target_img)
+        self.complete_and_wait()
+        self.vm.shutdown()
+        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),
+                         "Target size doesn't match source when granularity when unaligend")
+
+    def test_unaligned_with_update(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             granularity=65536, target=target_img)
+        self.wait_ready()
+        self.vm.hmp_qemu_io('drive0', 'write 0 512')
+        self.complete_and_wait(wait_ready=False)
+        self.vm.shutdown()
+        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),
+                         "Target size doesn't match source when granularity when unaligend")
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['raw', 'qcow2'])
diff --git a/qemu/tests/qemu-iotests/152.out b/qemu/tests/qemu-iotests/152.out
new file mode 100644
index 000000000..fbc63e62f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/152.out
@@ -0,0 +1,5 @@
+..
+----------------------------------------------------------------------
+Ran 2 tests
+
+OK
diff --git a/qemu/tests/qemu-iotests/check b/qemu/tests/qemu-iotests/check
index 1fa63193b..4cba2151e 100755
--- a/qemu/tests/qemu-iotests/check
+++ b/qemu/tests/qemu-iotests/check
@@ -19,7 +19,6 @@
 # Control script for QA
 #
 
-tmp=/tmp/$$
 status=0
 needwrap=true
 try=0
@@ -130,6 +129,8 @@ fi
 #    exit 1
 #fi
 
+tmp="${TEST_DIR}"/$$
+
 _wallclock()
 {
     date "+%H %M %S" | $AWK_PROG '{ print $1*3600 + $2*60 + $3 }'
@@ -146,8 +147,8 @@ _wrapup()
     # for hangcheck ...
     # remove files that were used by hangcheck
     #
-    [ -f /tmp/check.pid ] && rm -rf /tmp/check.pid
-    [ -f /tmp/check.sts ] && rm -rf /tmp/check.sts
+    [ -f "${TEST_DIR}"/check.pid ] && rm -rf "${TEST_DIR}"/check.pid
+    [ -f "${TEST_DIR}"/check.sts ] && rm -rf "${TEST_DIR}"/check.sts
 
     if $showme
     then
@@ -197,8 +198,8 @@ END        { if (NR > 0) {
         needwrap=false
     fi
 
-    rm -f /tmp/*.out /tmp/*.err /tmp/*.time
-    rm -f /tmp/check.pid /tmp/check.sts
+    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
+    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
     rm -f $tmp.*
 }
 
@@ -208,16 +209,16 @@ trap "_wrapup; exit \$status" 0 1 2 3 15
 # Save pid of check in a well known place, so that hangcheck can be sure it
 # has the right pid (getting the pid from ps output is not reliable enough).
 #
-rm -rf /tmp/check.pid
-echo $$ >/tmp/check.pid
+rm -rf "${TEST_DIR}"/check.pid
+echo $$ > "${TEST_DIR}"/check.pid
 
 # for hangcheck ...
 # Save the status of check in a well known place, so that hangcheck can be
 # sure to know where check is up to (getting test number from ps output is
 # not reliable enough since the trace stuff has been introduced).
 #
-rm -rf /tmp/check.sts
-echo "preamble" >/tmp/check.sts
+rm -rf "${TEST_DIR}"/check.sts
+echo "preamble" > "${TEST_DIR}"/check.sts
 
 # don't leave old full output behind on a clean run
 rm -f check.full
@@ -231,10 +232,10 @@ FULL_HOST_DETAILS=`_full_platform_details`
 #FULL_MOUNT_OPTIONS=`_scratch_mount_options`
 
 cat <<EOF
-QEMU          -- $QEMU
-QEMU_IMG      -- $QEMU_IMG
-QEMU_IO       -- $QEMU_IO
-QEMU_NBD      -- $QEMU_NBD
+QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
+QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
+QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
+QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
 IMGFMT        -- $FULL_IMGFMT_DETAILS
 IMGPROTO      -- $FULL_IMGPROTO_DETAILS
 PLATFORM      -- $FULL_HOST_DETAILS
@@ -285,7 +286,7 @@ do
         rm -f core $seq.notrun
 
         # for hangcheck ...
-        echo "$seq" >/tmp/check.sts
+        echo "$seq" > "${TEST_DIR}"/check.sts
 
         start=`_wallclock`
         $timestamp && echo -n "        ["`date "+%T"`"]"
@@ -330,6 +331,11 @@ do
             fi
 
             reference="$source_iotests/$seq.out"
+            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
+            if [ -f "$reference_machine" ]; then
+                reference="$reference_machine"
+            fi
+
             if [ "$CACHEMODE" = "none" ]; then
                 [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
             fi
diff --git a/qemu/tests/qemu-iotests/common b/qemu/tests/qemu-iotests/common
index 1030aaf25..49e193112 100644
--- a/qemu/tests/qemu-iotests/common
+++ b/qemu/tests/qemu-iotests/common
@@ -41,7 +41,6 @@ sortme=false
 expunge=true
 have_test_arg=false
 randomize=false
-valgrind=false
 cachemode=false
 rm -f $tmp.list $tmp.tmp $tmp.sed
 
@@ -52,6 +51,8 @@ export IMGOPTS=""
 export CACHEMODE="writeback"
 export QEMU_IO_OPTIONS=""
 export CACHEMODE_IS_DEFAULT=true
+export QEMU_OPTIONS="-nodefaults"
+export VALGRIND_QEMU=
 
 for r
 do
@@ -154,6 +155,7 @@ check options
     -ssh                test ssh
     -nfs                test nfs
     -archipelago        test archipelago
+    -luks               test luks
     -xdiff              graphical mode diff
     -nocache            use O_DIRECT on backing file
     -misalign           misalign memory allocations
@@ -277,7 +279,7 @@ testlist options
             ;;
 
         -valgrind)
-            valgrind=true
+            VALGRIND_QEMU='y'
             xpand=false
             ;;
 
@@ -435,8 +437,3 @@ fi
 if [ "$IMGPROTO" = "nbd" ] ; then
     [ "$QEMU_NBD" = "" ] && _fatal "qemu-nbd not found"
 fi
-
-if $valgrind; then
-    export REAL_QEMU_IO="$QEMU_IO_PROG"
-    export QEMU_IO_PROG=valgrind_qemu_io
-fi
diff --git a/qemu/tests/qemu-iotests/common.config b/qemu/tests/qemu-iotests/common.config
index a1973ad9d..f824651ba 100644
--- a/qemu/tests/qemu-iotests/common.config
+++ b/qemu/tests/qemu-iotests/common.config
@@ -44,6 +44,8 @@ export HOST_OPTIONS=${HOST_OPTIONS:=local.config}
 export CHECK_OPTIONS=${CHECK_OPTIONS:="-g auto"}
 export PWD=`pwd`
 
+export _QEMU_HANDLE=0
+
 # $1 = prog to look for, $2* = default pathnames if not found in $PATH
 set_prog_path()
 {
@@ -103,10 +105,63 @@ if [ -z "$QEMU_NBD_PROG" ]; then
     export QEMU_NBD_PROG="`set_prog_path qemu-nbd`"
 fi
 
-export QEMU=$QEMU_PROG
-export QEMU_IMG=$QEMU_IMG_PROG
-export QEMU_IO="$QEMU_IO_PROG $QEMU_IO_OPTIONS"
-export QEMU_NBD=$QEMU_NBD_PROG
+_qemu_wrapper()
+{
+    (
+        if [ -n "${QEMU_NEED_PID}" ]; then
+            echo $BASHPID > "${TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
+        fi
+        exec "$QEMU_PROG" $QEMU_OPTIONS "$@"
+    )
+}
+
+_qemu_img_wrapper()
+{
+    (exec "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS "$@")
+}
+
+_qemu_io_wrapper()
+{
+    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
+    local RETVAL
+    (
+        if [ "${VALGRIND_QEMU}" == "y" ]; then
+            exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@"
+        else
+            exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@"
+        fi
+    )
+    RETVAL=$?
+    if [ "${VALGRIND_QEMU}" == "y" ]; then
+        if [ $RETVAL == 99 ]; then
+            cat "${VALGRIND_LOGFILE}"
+        fi
+        rm -f "${VALGRIND_LOGFILE}"
+    fi
+    (exit $RETVAL)
+}
+
+_qemu_nbd_wrapper()
+{
+    (
+        echo $BASHPID > "${TEST_DIR}/qemu-nbd.pid"
+        exec "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS "$@"
+    )
+}
+
+export QEMU=_qemu_wrapper
+export QEMU_IMG=_qemu_img_wrapper
+export QEMU_IO=_qemu_io_wrapper
+export QEMU_NBD=_qemu_nbd_wrapper
+
+default_machine=$($QEMU -machine help | sed -n '/(default)/ s/ .*//p')
+default_alias_machine=$($QEMU -machine help | \
+   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
+if [[ "$default_alias_machine" ]]; then
+    default_machine="$default_alias_machine"
+fi
+
+export QEMU_DEFAULT_MACHINE="$default_machine"
 
 [ -f /etc/qemu-iotest.config ]       && . /etc/qemu-iotest.config
 
diff --git a/qemu/tests/qemu-iotests/common.filter b/qemu/tests/qemu-iotests/common.filter
index 012a8122d..8a6e1b57c 100644
--- a/qemu/tests/qemu-iotests/common.filter
+++ b/qemu/tests/qemu-iotests/common.filter
@@ -19,107 +19,6 @@
 # standard filters
 #
 
-# Checks that given_value is in range of correct_value +/- tolerance.
-# Tolerance can be an absolute value or a percentage of the correct value
-# (see examples with tolerances below).
-# Outputs suitable message to stdout if it's not in range.
-#
-# A verbose option, -v, may be used as the LAST argument
-#
-# e.g.
-# foo: 0.0298 = 0.03 +/- 5%
-# _within_tolerance "foo" 0.0298 0.03 5%
-#
-# foo: 0.0298 = 0.03 +/- 0.01
-# _within_tolerance "foo" 0.0298 0.03 0.01
-#
-# foo: 0.0298 = 0.03 -0.01 +0.002
-# _within_tolerance "foo" 0.0298 0.03 0.01 0.002
-#
-# foo: verbose output of 0.0298 = 0.03 +/- 5%
-# _within_tolerance "foo" 0.0298 0.03 5% -v
-_within_tolerance()
-{
-  _name=$1
-  _given_val=$2
-  _correct_val=$3
-  _mintol=$4
-  _maxtol=$_mintol
-  _verbose=0
-  _debug=false
-
-  # maxtol arg is optional
-  # verbose arg is optional
-  if [ $# -ge 5 ]
-  then
-     if [ "$5" = "-v" ]
-     then
-        _verbose=1
-     else
-        _maxtol=$5
-     fi
-  fi
-  if [ $# -ge 6 ]
-  then
-     [ "$6" = "-v" ] && _verbose=1
-  fi
-
-  # find min with or without %
-  _mintolerance=`echo $_mintol | sed -e 's/%//'`
-  if [ $_mintol = $_mintolerance ]
-  then
-      _min=`echo "scale=5; $_correct_val-$_mintolerance" | bc`
-  else
-      _min=`echo "scale=5; $_correct_val-$_mintolerance*0.01*$_correct_val" | bc`
-  fi
-
-  # find max with or without %
-  _maxtolerance=`echo $_maxtol | sed -e 's/%//'`
-  if [ $_maxtol = $_maxtolerance ]
-  then
-      _max=`echo "scale=5; $_correct_val+$_maxtolerance" | bc`
-  else
-      _max=`echo "scale=5; $_correct_val+$_maxtolerance*0.01*$_correct_val" | bc`
-  fi
-
-  $_debug && echo "min = $_min"
-  $_debug && echo "max = $_max"
-
-  cat <<EOF >$tmp.bc.1
-scale=5;
-if ($_min <= $_given_val) 1;
-if ($_min > $_given_val) 0;
-EOF
-
-  cat <<EOF >$tmp.bc.2
-scale=5;
-if ($_given_val <= $_max) 1;
-if ($_given_val > $_max) 0;
-EOF
-
-  _above_min=`bc <$tmp.bc.1`
-  _below_max=`bc <$tmp.bc.2`
-
-  rm -f $tmp.bc.[12]
-
-  _in_range=`expr $_above_min \& $_below_max`
-
-  # fix up min, max precision for output
-  # can vary for 5.3, 6.2
-  _min=`echo $_min | sed -e 's/0*$//'` # get rid of trailling zeroes
-  _max=`echo $_max | sed -e 's/0*$//'` # get rid of trailling zeroes
-
-  if [ $_in_range -eq 1 ]
-  then
-        [ $_verbose -eq 1 ] && echo $_name is in range
-        return 0
-  else
-        [ $_verbose -eq 1 ] && echo $_name has value of $_given_val
-        [ $_verbose -eq 1 ] && echo $_name is NOT in range $_min .. $_max
-        return 1
-  fi
-}
-
 # ctime(3) dates
 #
 _filter_date()
@@ -128,6 +27,11 @@ _filter_date()
         -e 's/[A-Z][a-z][a-z] [A-z][a-z][a-z]  *[0-9][0-9]* [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [0-9][0-9][0-9][0-9]$/DATE/'
 }
 
+_filter_generated_node_ids()
+{
+     sed -re 's/\#block[0-9]{3,}/NODE_NAME/'
+}
+
 # replace occurrences of the actual TEST_DIR value with TEST_DIR
 _filter_testdir()
 {
@@ -182,7 +86,7 @@ _filter_img_create()
         -e "s# encryption=off##g" \
         -e "s# cluster_size=[0-9]\\+##g" \
         -e "s# table_size=[0-9]\\+##g" \
-        -e "s# compat='[^']*'##g" \
+        -e "s# compat=[^ ]*##g" \
         -e "s# compat6=\\(on\\|off\\)##g" \
         -e "s# static=\\(on\\|off\\)##g" \
         -e "s# zeroed_grain=\\(on\\|off\\)##g" \
@@ -225,5 +129,18 @@ _filter_qemu_img_map()
         -e 's/Mapped to *//' | _filter_testdir | _filter_imgfmt
 }
 
+_filter_nbd()
+{
+    # nbd.c error messages contain function names and line numbers that are
+    # prone to change.  Message ordering depends on timing between send and
+    # receive callbacks sometimes, making them unreliable.
+    #
+    # Filter out the TCP port number since this changes between runs.
+    sed -e '/nbd\/.*\.c:/d' \
+        -e 's#nbd:\(//\)\?127\.0\.0\.1:[0-9]*#nbd:\1127.0.0.1:PORT#g' \
+        -e "s#?socket=$TEST_DIR#?socket=TEST_DIR#g" \
+        -e 's#\(exportname=foo\|PORT\): Failed to .*$#\1#'
+}
+
 # make sure this script returns success
 true
diff --git a/qemu/tests/qemu-iotests/common.qemu b/qemu/tests/qemu-iotests/common.qemu
index 4e1996c3e..2548a8700 100644
--- a/qemu/tests/qemu-iotests/common.qemu
+++ b/qemu/tests/qemu-iotests/common.qemu
@@ -30,8 +30,6 @@ QEMU_COMM_TIMEOUT=10
 QEMU_FIFO_IN="${TEST_DIR}/qmp-in-$$"
 QEMU_FIFO_OUT="${TEST_DIR}/qmp-out-$$"
 
-QEMU_PID=
-_QEMU_HANDLE=0
 QEMU_HANDLE=0
 
 # If bash version is >= 4.1, these will be overwritten and dynamic
@@ -131,6 +129,8 @@ function _send_qemu_cmd()
 # $qemu_comm_method: set this variable to 'monitor' (case insensitive)
 #                    to use the QEMU HMP monitor for communication.
 #                    Otherwise, the default of QMP is used.
+# $keep_stderr: Set this variable to 'y' to keep QEMU's stderr output on stderr.
+#               If this variable is empty, stderr will be redirected to stdout.
 # Returns:
 # $QEMU_HANDLE: set to a handle value to communicate with this QEMU instance.
 #
@@ -153,11 +153,20 @@ function _launch_qemu()
     mkfifo "${fifo_out}"
     mkfifo "${fifo_in}"
 
-    "${QEMU}" -nographic -serial none ${comm} -machine accel=qtest "${@}" \
+    if [ -z "$keep_stderr" ]; then
+        QEMU_NEED_PID='y'\
+        ${QEMU} -nographic -serial none ${comm} -machine accel=qtest "${@}" \
                                                                 >"${fifo_out}" \
                                                                 2>&1 \
                                                                 <"${fifo_in}" &
-    QEMU_PID[${_QEMU_HANDLE}]=$!
+    elif [ "$keep_stderr" = "y" ]; then
+        QEMU_NEED_PID='y'\
+        ${QEMU} -nographic -serial none ${comm} -machine accel=qtest "${@}" \
+                                                                >"${fifo_out}" \
+                                                                <"${fifo_in}" &
+    else
+        exit 1
+    fi
 
     if [[ "${BASH_VERSINFO[0]}" -ge "5" ||
         ("${BASH_VERSINFO[0]}" -ge "4"  &&  "${BASH_VERSINFO[1]}" -ge "1") ]]
@@ -196,10 +205,18 @@ function _cleanup_qemu()
     # QEMU_PID[], QEMU_IN[], QEMU_OUT[] all use same indices
     for i in "${!QEMU_OUT[@]}"
     do
-        if [ -z "${wait}" ]; then
-            kill -KILL ${QEMU_PID[$i]} 2>/dev/null
+        local QEMU_PID
+        if [ -f "${TEST_DIR}/qemu-${i}.pid" ]; then
+            read QEMU_PID < "${TEST_DIR}/qemu-${i}.pid"
+            rm -f "${TEST_DIR}/qemu-${i}.pid"
+            if [ -z "${wait}" ] && [ -n "${QEMU_PID}" ]; then
+                kill -KILL ${QEMU_PID} 2>/dev/null
+            fi
+            if [ -n "${QEMU_PID}" ]; then
+                wait ${QEMU_PID} 2>/dev/null # silent kill
+            fi
         fi
-        wait ${QEMU_PID[$i]} 2>/dev/null # silent kill
+
         if [ -n "${wait}" ]; then
             cat <&${QEMU_OUT[$i]} | _filter_testdir | _filter_qemu \
                                   | _filter_qemu_io | _filter_qmp
diff --git a/qemu/tests/qemu-iotests/common.rc b/qemu/tests/qemu-iotests/common.rc
index 22d351404..5249ec592 100644
--- a/qemu/tests/qemu-iotests/common.rc
+++ b/qemu/tests/qemu-iotests/common.rc
@@ -70,16 +70,6 @@ else
     TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
 fi
 
-function valgrind_qemu_io()
-{
-    valgrind --log-file=/tmp/$$.valgrind --error-exitcode=99 $REAL_QEMU_IO "$@"
-    if [ $? != 0 ]; then
-        cat /tmp/$$.valgrind
-    fi
-    rm -f /tmp/$$.valgrind
-}
-
-
 _optstr_add()
 {
     if [ -n "$1" ]; then
@@ -154,7 +144,6 @@ _make_test_img()
     # Start an NBD server on the image file, which is what we'll be talking to
     if [ $IMGPROTO = "nbd" ]; then
         eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810 -f $IMGFMT  $TEST_IMG_FILE &"
-        QEMU_NBD_PID=$!
         sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
     fi
 }
@@ -175,8 +164,11 @@ _cleanup_test_img()
     case "$IMGPROTO" in
 
         nbd)
-            if [ -n "$QEMU_NBD_PID" ]; then
-                kill $QEMU_NBD_PID
+            if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+                local QEMU_NBD_PID
+                read QEMU_NBD_PID < "${TEST_DIR}/qemu-nbd.pid"
+                kill ${QEMU_NBD_PID}
+                rm -f "${TEST_DIR}/qemu-nbd.pid"
             fi
             rm -f "$TEST_IMG_FILE"
             ;;
@@ -295,52 +287,6 @@ _need_to_be_root()
     fi
 }
 
-
-# Do a command, log it to $seq.full, optionally test return status
-# and die if command fails. If called with one argument _do executes the
-# command, logs it, and returns its exit status. With two arguments _do
-# first prints the message passed in the first argument, and then "done"
-# or "fail" depending on the return status of the command passed in the
-# second argument. If the command fails and the variable _do_die_on_error
-# is set to "always" or the two argument form is used and _do_die_on_error
-# is set to "message_only" _do will print an error message to
-# $seq.out and exit.
-
-_do()
-{
-    if [ $# -eq 1 ]; then
-        _cmd=$1
-    elif [ $# -eq 2 ]; then
-        _note=$1
-        _cmd=$2
-        echo -n "$_note... "
-    else
-        echo "Usage: _do [note] cmd" 1>&2
-        status=1; exit
-    fi
-
-    (eval "echo '---' \"$_cmd\"") >>"$OUTPUT_DIR/$seq.full"
-    (eval "$_cmd") >$tmp._out 2>&1; ret=$?
-    cat $tmp._out >>"$OUTPUT_DIR/$seq.full"
-    if [ $# -eq 2 ]; then
-        if [ $ret -eq 0 ]; then
-            echo "done"
-        else
-            echo "fail"
-        fi
-    fi
-    if [ $ret -ne 0  ] \
-        && [ "$_do_die_on_error" = "always" \
-            -o \( $# -eq 2 -a "$_do_die_on_error" = "message_only" \) ]
-    then
-        [ $# -ne 2 ] && echo
-        eval "echo \"$_cmd\" failed \(returned $ret\): see $seq.full"
-        status=1; exit
-    fi
-
-    return $ret
-}
-
 # bail out, setting up .notrun file
 #
 _notrun()
@@ -439,7 +385,17 @@ _unsupported_imgopts()
 #
 _require_command()
 {
-    eval c=\$$1
+    if [ "$1" = "QEMU" ]; then
+        c=$QEMU_PROG
+    elif [ "$1" = "QEMU_IMG" ]; then
+        c=$QEMU_IMG_PROG
+    elif [ "$1" = "QEMU_IO" ]; then
+        c=$QEMU_IO_PROG
+    elif [ "$1" = "QEMU_NBD" ]; then
+        c=$QEMU_NBD_PROG
+    else
+        eval c=\$$1
+    fi
     [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
 }
 
diff --git a/qemu/tests/qemu-iotests/group b/qemu/tests/qemu-iotests/group
index c430b6c23..822953b6f 100644
--- a/qemu/tests/qemu-iotests/group
+++ b/qemu/tests/qemu-iotests/group
@@ -102,6 +102,7 @@
 093 auto
 094 rw auto quick
 095 rw auto quick
+096 rw auto quick
 097 rw auto backing
 098 rw auto backing quick
 099 rw auto quick
@@ -121,6 +122,8 @@
 114 rw auto quick
 115 rw auto
 116 rw auto quick
+117 rw auto
+118 rw auto
 119 rw auto quick
 120 rw auto quick
 121 rw auto
@@ -132,5 +135,21 @@
 130 rw auto quick
 131 rw auto quick
 132 rw auto quick
+133 auto quick
 134 rw auto quick
 135 rw auto
+136 rw auto
+137 rw auto
+138 rw auto quick
+139 rw auto quick
+140 rw auto quick
+141 rw auto quick
+142 auto
+143 auto quick
+144 rw auto quick
+145 auto quick
+146 auto quick
+148 rw auto quick
+149 rw auto sudo
+150 rw auto quick
+152 rw auto quick
diff --git a/qemu/tests/qemu-iotests/iotests.py b/qemu/tests/qemu-iotests/iotests.py
index 8615b1075..56f988ab3 100644
--- a/qemu/tests/qemu-iotests/iotests.py
+++ b/qemu/tests/qemu-iotests/iotests.py
@@ -16,6 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import errno
 import os
 import re
 import subprocess
@@ -27,41 +28,66 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts', '
 import qmp
 import qtest
 import struct
+import json
 
-__all__ = ['imgfmt', 'imgproto', 'test_dir' 'qemu_img', 'qemu_io',
-           'VM', 'QMPTestCase', 'notrun', 'main']
 
-# This will not work if arguments or path contain spaces but is necessary if we
+# This will not work if arguments contain spaces but is necessary if we
 # want to support the override options that ./check supports.
-qemu_img_args = os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ')
-qemu_io_args = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ')
-qemu_args = os.environ.get('QEMU', 'qemu').strip().split(' ')
+qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')]
+if os.environ.get('QEMU_IMG_OPTIONS'):
+    qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ')
+
+qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')]
+if os.environ.get('QEMU_IO_OPTIONS'):
+    qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ')
+
+qemu_args = [os.environ.get('QEMU_PROG', 'qemu')]
+if os.environ.get('QEMU_OPTIONS'):
+    qemu_args += os.environ['QEMU_OPTIONS'].strip().split(' ')
 
 imgfmt = os.environ.get('IMGFMT', 'raw')
 imgproto = os.environ.get('IMGPROTO', 'file')
 test_dir = os.environ.get('TEST_DIR', '/var/tmp')
 output_dir = os.environ.get('OUTPUT_DIR', '.')
 cachemode = os.environ.get('CACHEMODE')
+qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
 
 socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
 
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
-    return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
-    return subprocess.call(qemu_img_args + list(args))
+    exitcode = subprocess.call(qemu_img_args + list(args))
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_pipe(*args):
     '''Run qemu-img and return its output'''
-    return subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(qemu_img_args + list(args),
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return subp.communicate()[0]
 
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
-    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
+    return subp.communicate()[0]
 
 def compare_images(img1, img2):
     '''Return True if two image files are identical'''
@@ -78,6 +104,33 @@ def create_image(name, size):
         i = i + 512
     file.close()
 
+def image_size(img):
+    '''Return image's virtual size'''
+    r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img)
+    return json.loads(r)['virtual-size']
+
+test_dir_re = re.compile(r"%s" % test_dir)
+def filter_test_dir(msg):
+    return test_dir_re.sub("TEST_DIR", msg)
+
+win32_re = re.compile(r"\r")
+def filter_win32(msg):
+    return win32_re.sub("", msg)
+
+qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)")
+def filter_qemu_io(msg):
+    msg = filter_win32(msg)
+    return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg)
+
+chown_re = re.compile(r"chown [0-9]+:[0-9]+")
+def filter_chown(msg):
+    return chown_re.sub("chown UID:GID", msg)
+
+def log(msg, filters=[]):
+    for flt in filters:
+        msg = flt(msg)
+    print msg
+
 # Test if 'match' is a recursive subset of 'event'
 def event_match(event, match=None):
     if match is None:
@@ -117,13 +170,21 @@ class VM(object):
         self._args.append('-monitor')
         self._args.append(args)
 
-    def add_drive(self, path, opts=''):
+    def add_drive_raw(self, opts):
+        self._args.append('-drive')
+        self._args.append(opts)
+        return self
+
+    def add_drive(self, path, opts='', interface='virtio'):
         '''Add a virtio-blk drive to the VM'''
-        options = ['if=virtio',
-                   'format=%s' % imgfmt,
-                   'cache=%s' % cachemode,
-                   'file=%s' % path,
+        options = ['if=%s' % interface,
                    'id=drive%d' % self._num_drives]
+
+        if path is not None:
+            options.append('file=%s' % path)
+            options.append('format=%s' % imgfmt)
+            options.append('cache=%s' % cachemode)
+
         if opts:
             options.append(opts)
 
@@ -189,14 +250,17 @@ class VM(object):
             self._qmp.accept()
             self._qtest.accept()
         except:
-            os.remove(self._monitor_path)
+            _remove_if_exists(self._monitor_path)
+            _remove_if_exists(self._qtest_path)
             raise
 
     def shutdown(self):
         '''Terminate the VM and clean up'''
         if not self._popen is None:
             self._qmp.cmd('quit')
-            self._popen.wait()
+            exitcode = self._popen.wait()
+            if exitcode < 0:
+                sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, ' '.join(self._args)))
             os.remove(self._monitor_path)
             os.remove(self._qtest_path)
             os.remove(self._qemu_log_path)
@@ -290,6 +354,20 @@ class QMPTestCase(unittest.TestCase):
         result = self.vm.qmp('query-block-jobs')
         self.assert_qmp(result, 'return', [])
 
+    def assert_has_block_node(self, node_name=None, file_name=None):
+        """Issue a query-named-block-nodes and assert node_name and/or
+        file_name is present in the result"""
+        def check_equal_or_none(a, b):
+            return a == None or b == None or a == b
+        assert node_name or file_name
+        result = self.vm.qmp('query-named-block-nodes')
+        for x in result["return"]:
+            if check_equal_or_none(x.get("node-name"), node_name) and \
+                    check_equal_or_none(x.get("file"), file_name):
+                return
+        self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \
+                (node_name, file_name, result))
+
     def cancel_and_wait(self, drive='drive0', force=False, resume=False):
         '''Cancel a block job and wait for it to finish, returning the event'''
         result = self.vm.qmp('block-job-cancel', device=drive, force=force)
@@ -349,6 +427,15 @@ class QMPTestCase(unittest.TestCase):
         event = self.wait_until_completed(drive=drive)
         self.assert_qmp(event, 'data/type', 'mirror')
 
+def _remove_if_exists(path):
+    '''Remove file object at path if it exists'''
+    try:
+        os.remove(path)
+    except OSError as exception:
+        if exception.errno == errno.ENOENT:
+           return
+        raise
+
 def notrun(reason):
     '''Skip this test suite'''
     # Each test in qemu-iotests has a number ("seq")
@@ -358,17 +445,27 @@ def notrun(reason):
     print '%s not run: %s' % (seq, reason)
     sys.exit(0)
 
-def main(supported_fmts=[], supported_oses=['linux']):
-    '''Run tests'''
-
-    debug = '-d' in sys.argv
-    verbosity = 1
+def verify_image_format(supported_fmts=[]):
     if supported_fmts and (imgfmt not in supported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
+def verify_platform(supported_oses=['linux']):
     if True not in [sys.platform.startswith(x) for x in supported_oses]:
         notrun('not suitable for this OS: %s' % sys.platform)
 
+def verify_quorum():
+    '''Skip test suite if quorum support is not available'''
+    if 'quorum' not in qemu_img_pipe('--help'):
+        notrun('quorum support missing')
+
+def main(supported_fmts=[], supported_oses=['linux']):
+    '''Run tests'''
+
+    debug = '-d' in sys.argv
+    verbosity = 1
+    verify_image_format(supported_fmts)
+    verify_platform(supported_oses)
+
     # We need to filter out the time taken from the output so that qemu-iotest
     # can reliably diff the results against master output.
     import StringIO
diff --git a/qemu/tests/qemu-iotests/qed.py b/qemu/tests/qemu-iotests/qed.py
index 52ff84559..748068d7f 100755
--- a/qemu/tests/qemu-iotests/qed.py
+++ b/qemu/tests/qemu-iotests/qed.py
@@ -227,7 +227,7 @@ def main():
     qed = QED(open(filename, 'r+b'))
     try:
         globals()[cmd](qed, *sys.argv[3:])
-    except TypeError, e:
+    except TypeError as e:
         sys.stderr.write(globals()[cmd].__doc__ + '\n')
         sys.exit(1)
 
diff --git a/qemu/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2 b/qemu/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2
new file mode 100644
index 000000000..f12cb9203
--- /dev/null
+++ b/qemu/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2
diff --git a/qemu/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2 b/qemu/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2
new file mode 100644
index 000000000..bfeccf7b9
--- /dev/null
+++ b/qemu/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2
diff --git a/qemu/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2 b/qemu/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2
new file mode 100644
index 000000000..783be3c8f
--- /dev/null
+++ b/qemu/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2
diff --git a/qemu/tests/qemu-iotests/socket_scm_helper.c b/qemu/tests/qemu-iotests/socket_scm_helper.c
index 81959835e..80cadf43b 100644
--- a/qemu/tests/qemu-iotests/socket_scm_helper.c
+++ b/qemu/tests/qemu-iotests/socket_scm_helper.c
@@ -10,15 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <stdio.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/socket.h>
 #include <sys/un.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
 
 /* #define SOCKET_SCM_DEBUG */
 
diff --git a/qemu/tests/qom-test.c b/qemu/tests/qom-test.c
index fde04e7a1..bd5cdde26 100644
--- a/qemu/tests/qom-test.c
+++ b/qemu/tests/qom-test.c
@@ -7,12 +7,12 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "qapi/qmp/types.h"
 
 static const char *blacklist_x86[] = {
@@ -47,7 +47,7 @@ static bool is_blacklisted(const char *arch, const char *mach)
 static void test_properties(const char *path, bool recurse)
 {
     char *child_path;
-    QDict *response, *tuple;
+    QDict *response, *tuple, *tmp;
     QList *list;
     QListEntry *entry;
 
@@ -57,6 +57,7 @@ static void test_properties(const char *path, bool recurse)
     g_assert(response);
 
     if (!recurse) {
+        QDECREF(response);
         return;
     }
 
@@ -75,14 +76,16 @@ static void test_properties(const char *path, bool recurse)
         } else {
             const char *prop = qdict_get_str(tuple, "name");
             g_test_message("Testing property %s.%s", path, prop);
-            response = qmp("{ 'execute': 'qom-get',"
-                           "  'arguments': { 'path': %s,"
-                           "                 'property': %s } }",
-                           path, prop);
+            tmp = qmp("{ 'execute': 'qom-get',"
+                      "  'arguments': { 'path': %s,"
+                      "                 'property': %s } }",
+                      path, prop);
             /* qom-get may fail but should not, e.g., segfault. */
-            g_assert(response);
+            g_assert(tmp);
+            QDECREF(tmp);
         }
     }
+    QDECREF(response);
 }
 
 static void test_machine(gconstpointer data)
@@ -98,9 +101,11 @@ static void test_machine(gconstpointer data)
 
     response = qmp("{ 'execute': 'quit' }");
     g_assert(qdict_haskey(response, "return"));
+    QDECREF(response);
 
     qtest_end();
     g_free(args);
+    g_free((void *)machine);
 }
 
 static void add_machine_test_cases(void)
@@ -129,10 +134,12 @@ static void add_machine_test_cases(void)
         mname = qstring_get_str(qstr);
         if (!is_blacklisted(arch, mname)) {
             path = g_strdup_printf("qom/%s", mname);
-            qtest_add_data_func(path, mname, test_machine);
+            qtest_add_data_func(path, g_strdup(mname), test_machine);
         }
     }
+
     qtest_end();
+    QDECREF(response);
 }
 
 int main(int argc, char **argv)
diff --git a/qemu/tests/rcutorture.c b/qemu/tests/rcutorture.c
index d6b304d00..244f0f28b 100644
--- a/qemu/tests/rcutorture.c
+++ b/qemu/tests/rcutorture.c
@@ -60,13 +60,10 @@
  * Test variables.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
 #include "qemu/atomic.h"
 #include "qemu/rcu.h"
-#include "qemu/compiler.h"
 #include "qemu/thread.h"
 
 long long n_reads = 0LL;
diff --git a/qemu/tests/rtc-test.c b/qemu/tests/rtc-test.c
index 4243624de..fa7029aa8 100644
--- a/qemu/tests/rtc-test.c
+++ b/qemu/tests/rtc-test.c
@@ -11,11 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
 
 #include "libqtest.h"
 #include "hw/timer/mc146818rtc_regs.h"
diff --git a/qemu/tests/rtl8139-test.c b/qemu/tests/rtl8139-test.c
index e749be38e..54e5aa7d0 100644
--- a/qemu/tests/rtl8139-test.c
+++ b/qemu/tests/rtl8139-test.c
@@ -7,11 +7,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
 #include "libqos/pci-pc.h"
-#include "qemu/osdep.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
 
@@ -20,7 +19,7 @@ static void nop(void)
 {
 }
 
-#define CLK 33000000
+#define CLK 33333333
 
 static QPCIBus *pcibus;
 static QPCIDevice *dev;
diff --git a/qemu/tests/spapr-phb-test.c b/qemu/tests/spapr-phb-test.c
index b629de475..f53911d9f 100644
--- a/qemu/tests/spapr-phb-test.c
+++ b/qemu/tests/spapr-phb-test.c
@@ -7,6 +7,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "libqtest.h"
diff --git a/qemu/tests/tcg/linux-test.c b/qemu/tests/tcg/linux-test.c
index 1c6c01318..5070d3144 100644
--- a/qemu/tests/tcg/linux-test.c
+++ b/qemu/tests/tcg/linux-test.c
@@ -39,6 +39,7 @@
 #include <dirent.h>
 #include <setjmp.h>
 #include <sys/shm.h>
+#include "qemu/cutils.h"
 
 #define TESTPATH "/tmp/linux-test.tmp"
 #define TESTPORT 7654
diff --git a/qemu/tests/tcg/test-i386-fprem.c b/qemu/tests/tcg/test-i386-fprem.c
index e91fb1ae9..1a7162320 100644
--- a/qemu/tests/tcg/test-i386-fprem.c
+++ b/qemu/tests/tcg/test-i386-fprem.c
@@ -22,10 +22,8 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu/compiler.h"
+
 #include "qemu/osdep.h"
-#include <stdio.h>
-#include <inttypes.h>
 
 /*
  * Inspired by <ieee754.h>'s union ieee854_long_double, but with single
diff --git a/qemu/tests/tcg/testthread.c b/qemu/tests/tcg/testthread.c
index 2679af119..810ba5de6 100644
--- a/qemu/tests/tcg/testthread.c
+++ b/qemu/tests/tcg/testthread.c
@@ -2,7 +2,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <signal.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <pthread.h>
diff --git a/qemu/tests/tco-test.c b/qemu/tests/tco-test.c
index 419f7cf46..ac11175e9 100644
--- a/qemu/tests/tco-test.c
+++ b/qemu/tests/tco-test.c
@@ -6,10 +6,8 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
-#include <stdio.h>
-#include <stdlib.h>
 
 #include "libqtest.h"
 #include "libqos/pci.h"
diff --git a/qemu/tests/test-aio.c b/qemu/tests/test-aio.c
index 217e33772..687dfa062 100644
--- a/qemu/tests/test-aio.c
+++ b/qemu/tests/test-aio.c
@@ -10,8 +10,10 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "block/aio.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"
 #include "qemu/error-report.h"
@@ -118,6 +120,12 @@ static void *test_acquire_thread(void *opaque)
     return NULL;
 }
 
+static void set_event_notifier(AioContext *ctx, EventNotifier *notifier,
+                               EventNotifierHandler *handler)
+{
+    aio_set_event_notifier(ctx, notifier, false, handler);
+}
+
 static void dummy_notifier_read(EventNotifier *unused)
 {
     g_assert(false); /* should never be invoked */
@@ -131,7 +139,7 @@ static void test_acquire(void)
 
     /* Dummy event notifier ensures aio_poll() will block */
     event_notifier_init(&notifier, false);
-    aio_set_event_notifier(ctx, &notifier, dummy_notifier_read);
+    set_event_notifier(ctx, &notifier, dummy_notifier_read);
     g_assert(!aio_poll(ctx, false)); /* consume aio_notify() */
 
     qemu_mutex_init(&data.start_lock);
@@ -149,7 +157,7 @@ static void test_acquire(void)
     aio_context_release(ctx);
 
     qemu_thread_join(&thread);
-    aio_set_event_notifier(ctx, &notifier, NULL);
+    set_event_notifier(ctx, &notifier, NULL);
     event_notifier_cleanup(&notifier);
 
     g_assert(data.thread_acquired);
@@ -308,11 +316,11 @@ static void test_set_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -322,7 +330,7 @@ static void test_wait_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
@@ -336,7 +344,7 @@ static void test_wait_event_notifier(void)
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -347,7 +355,7 @@ static void test_flush_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
@@ -363,18 +371,42 @@ static void test_flush_event_notifier(void)
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!aio_poll(ctx, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     event_notifier_cleanup(&data.e);
 }
 
+static void test_aio_external_client(void)
+{
+    int i, j;
+
+    for (i = 1; i < 3; i++) {
+        EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+        event_notifier_init(&data.e, false);
+        aio_set_event_notifier(ctx, &data.e, true, event_ready_cb);
+        event_notifier_set(&data.e);
+        for (j = 0; j < i; j++) {
+            aio_disable_external(ctx);
+        }
+        for (j = 0; j < i; j++) {
+            assert(!aio_poll(ctx, false));
+            assert(event_notifier_test_and_clear(&data.e));
+            event_notifier_set(&data.e);
+            aio_enable_external(ctx);
+        }
+        assert(aio_poll(ctx, false));
+        set_event_notifier(ctx, &data.e, NULL);
+        event_notifier_cleanup(&data.e);
+    }
+}
+
 static void test_wait_event_notifier_noflush(void)
 {
     EventNotifierTestData data = { .n = 0 };
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -387,7 +419,7 @@ static void test_wait_event_notifier_noflush(void)
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(aio_poll(ctx, false));
@@ -407,10 +439,10 @@ static void test_wait_event_notifier_noflush(void)
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -428,7 +460,7 @@ static void test_timer_schedule(void)
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     aio_poll(ctx, false);
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -467,7 +499,7 @@ static void test_timer_schedule(void)
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -638,11 +670,11 @@ static void test_source_set_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -652,7 +684,7 @@ static void test_source_wait_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
@@ -666,7 +698,7 @@ static void test_source_wait_event_notifier(void)
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -677,7 +709,7 @@ static void test_source_flush_event_notifier(void)
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
@@ -693,7 +725,7 @@ static void test_source_flush_event_notifier(void)
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!g_main_context_iteration(NULL, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     event_notifier_cleanup(&data.e);
 }
@@ -704,7 +736,7 @@ static void test_source_wait_event_notifier_noflush(void)
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -717,7 +749,7 @@ static void test_source_wait_event_notifier_noflush(void)
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(g_main_context_iteration(NULL, false));
@@ -737,10 +769,10 @@ static void test_source_wait_event_notifier_noflush(void)
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -759,7 +791,7 @@ static void test_source_timer_schedule(void)
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     do {} while (g_main_context_iteration(NULL, false));
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -784,7 +816,7 @@ static void test_source_timer_schedule(void)
     g_assert_cmpint(data.n, ==, 2);
     g_assert(qemu_clock_get_ns(data.clock_type) > expiry);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -802,9 +834,7 @@ int main(int argc, char **argv)
 
     ctx = aio_context_new(&local_error);
     if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error, "Failed to create AIO Context: ");
         exit(1);
     }
     src = aio_get_g_source(ctx);
@@ -826,6 +856,7 @@ int main(int argc, char **argv)
     g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
     g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
     g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
+    g_test_add_func("/aio/external-client",         test_aio_external_client);
     g_test_add_func("/aio/timer/schedule",          test_timer_schedule);
 
     g_test_add_func("/aio-gsource/flush",                   test_source_flush);
diff --git a/qemu/tests/test-base64.c b/qemu/tests/test-base64.c
new file mode 100644
index 000000000..922e839dd
--- /dev/null
+++ b/qemu/tests/test-base64.c
@@ -0,0 +1,110 @@
+/*
+ * QEMU base64 helper test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "qapi/error.h"
+#include "qemu/base64.h"
+
+static void test_base64_good(void)
+{
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\n"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+    const char expect[] = "Because we focused on the snake, "
+        "we missed the scorpion.";
+
+    size_t len;
+    uint8_t *actual = qbase64_decode(input,
+                                     -1,
+                                     &len,
+                                     &error_abort);
+
+    g_assert(actual != NULL);
+    g_assert_cmpint(len, ==, strlen(expect));
+    g_assert_cmpstr((char *)actual, ==, expect);
+    g_free(actual);
+}
+
+
+static void test_base64_bad(const char *input,
+                            size_t input_len)
+{
+    size_t len;
+    Error *err = NULL;
+    uint8_t *actual = qbase64_decode(input,
+                                     input_len,
+                                     &len,
+                                     &err);
+
+    g_assert(err != NULL);
+    g_assert(actual == NULL);
+    g_assert_cmpint(len, ==, 0);
+    error_free(err);
+}
+
+
+static void test_base64_embedded_nul(void)
+{
+    /* We put a NUL character in the middle of the base64
+     * text which is invalid data, given the expected length */
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\0"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    test_base64_bad(input, G_N_ELEMENTS(input) - 1);
+}
+
+
+static void test_base64_not_nul_terminated(void)
+{
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\n"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    /* Using '-2' to make us drop the trailing NUL, thus
+     * creating an invalid base64 sequence for decoding */
+    test_base64_bad(input, G_N_ELEMENTS(input) - 2);
+}
+
+
+static void test_base64_invalid_chars(void)
+{
+    /* We put a single quote character in the middle
+     * of the base64 text which is invalid data */
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW'"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    test_base64_bad(input, strlen(input));
+}
+
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/util/base64/good", test_base64_good);
+    g_test_add_func("/util/base64/embedded-nul", test_base64_embedded_nul);
+    g_test_add_func("/util/base64/not-nul-terminated",
+                    test_base64_not_nul_terminated);
+    g_test_add_func("/util/base64/invalid-chars", test_base64_invalid_chars);
+    return g_test_run();
+}
diff --git a/qemu/tests/test-bitops.c b/qemu/tests/test-bitops.c
index 47b5d3ed9..505095060 100644
--- a/qemu/tests/test-bitops.c
+++ b/qemu/tests/test-bitops.c
@@ -6,9 +6,8 @@
  *
  */
 
-#include <glib.h>
-#include <stdint.h>
 #include "qemu/osdep.h"
+#include <glib.h>
 #include "qemu/bitops.h"
 
 typedef struct {
diff --git a/qemu/tests/test-blockjob-txn.c b/qemu/tests/test-blockjob-txn.c
new file mode 100644
index 000000000..55fad9507
--- /dev/null
+++ b/qemu/tests/test-blockjob-txn.c
@@ -0,0 +1,251 @@
+/*
+ * Blockjob transactions tests
+ *
+ * Copyright Red Hat, Inc. 2015
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#include "block/blockjob.h"
+
+typedef struct {
+    BlockJob common;
+    unsigned int iterations;
+    bool use_timer;
+    int rc;
+    int *result;
+} TestBlockJob;
+
+static const BlockJobDriver test_block_job_driver = {
+    .instance_size = sizeof(TestBlockJob),
+};
+
+static void test_block_job_complete(BlockJob *job, void *opaque)
+{
+    BlockDriverState *bs = job->bs;
+    int rc = (intptr_t)opaque;
+
+    if (block_job_is_cancelled(job)) {
+        rc = -ECANCELED;
+    }
+
+    block_job_completed(job, rc);
+    bdrv_unref(bs);
+}
+
+static void coroutine_fn test_block_job_run(void *opaque)
+{
+    TestBlockJob *s = opaque;
+    BlockJob *job = &s->common;
+
+    while (s->iterations--) {
+        if (s->use_timer) {
+            block_job_sleep_ns(job, QEMU_CLOCK_REALTIME, 0);
+        } else {
+            block_job_yield(job);
+        }
+
+        if (block_job_is_cancelled(job)) {
+            break;
+        }
+    }
+
+    block_job_defer_to_main_loop(job, test_block_job_complete,
+                                 (void *)(intptr_t)s->rc);
+}
+
+typedef struct {
+    TestBlockJob *job;
+    int *result;
+} TestBlockJobCBData;
+
+static void test_block_job_cb(void *opaque, int ret)
+{
+    TestBlockJobCBData *data = opaque;
+    if (!ret && block_job_is_cancelled(&data->job->common)) {
+        ret = -ECANCELED;
+    }
+    *data->result = ret;
+    g_free(data);
+}
+
+/* Create a block job that completes with a given return code after a given
+ * number of event loop iterations.  The return code is stored in the given
+ * result pointer.
+ *
+ * The event loop iterations can either be handled automatically with a 0 delay
+ * timer, or they can be stepped manually by entering the coroutine.
+ */
+static BlockJob *test_block_job_start(unsigned int iterations,
+                                      bool use_timer,
+                                      int rc, int *result)
+{
+    BlockDriverState *bs;
+    TestBlockJob *s;
+    TestBlockJobCBData *data;
+
+    data = g_new0(TestBlockJobCBData, 1);
+    bs = bdrv_new();
+    s = block_job_create(&test_block_job_driver, bs, 0, test_block_job_cb,
+                         data, &error_abort);
+    s->iterations = iterations;
+    s->use_timer = use_timer;
+    s->rc = rc;
+    s->result = result;
+    s->common.co = qemu_coroutine_create(test_block_job_run);
+    data->job = s;
+    data->result = result;
+    qemu_coroutine_enter(s->common.co, s);
+    return &s->common;
+}
+
+static void test_single_job(int expected)
+{
+    BlockJob *job;
+    BlockJobTxn *txn;
+    int result = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job = test_block_job_start(1, true, expected, &result);
+    block_job_txn_add_job(txn, job);
+
+    if (expected == -ECANCELED) {
+        block_job_cancel(job);
+    }
+
+    while (result == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+    g_assert_cmpint(result, ==, expected);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_single_job_success(void)
+{
+    test_single_job(0);
+}
+
+static void test_single_job_failure(void)
+{
+    test_single_job(-EIO);
+}
+
+static void test_single_job_cancel(void)
+{
+    test_single_job(-ECANCELED);
+}
+
+static void test_pair_jobs(int expected1, int expected2)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, expected1, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, true, expected2, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    if (expected1 == -ECANCELED) {
+        block_job_cancel(job1);
+    }
+    if (expected2 == -ECANCELED) {
+        block_job_cancel(job2);
+    }
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    /* Failure or cancellation of one job cancels the other job */
+    if (expected1 != 0) {
+        expected2 = -ECANCELED;
+    } else if (expected2 != 0) {
+        expected1 = -ECANCELED;
+    }
+
+    g_assert_cmpint(result1, ==, expected1);
+    g_assert_cmpint(result2, ==, expected2);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_pair_jobs_success(void)
+{
+    test_pair_jobs(0, 0);
+}
+
+static void test_pair_jobs_failure(void)
+{
+    /* Test both orderings.  The two jobs run for a different number of
+     * iterations so the code path is different depending on which job fails
+     * first.
+     */
+    test_pair_jobs(-EIO, 0);
+    test_pair_jobs(0, -EIO);
+}
+
+static void test_pair_jobs_cancel(void)
+{
+    test_pair_jobs(-ECANCELED, 0);
+    test_pair_jobs(0, -ECANCELED);
+}
+
+static void test_pair_jobs_fail_cancel_race(void)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, -ECANCELED, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, false, 0, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    block_job_cancel(job1);
+
+    /* Now make job2 finish before the main loop kicks jobs.  This simulates
+     * the race between a pending kick and another job completing.
+     */
+    block_job_enter(job2);
+    block_job_enter(job2);
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    g_assert_cmpint(result1, ==, -ECANCELED);
+    g_assert_cmpint(result2, ==, -ECANCELED);
+
+    block_job_txn_unref(txn);
+}
+
+int main(int argc, char **argv)
+{
+    qemu_init_main_loop(&error_abort);
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/single/success", test_single_job_success);
+    g_test_add_func("/single/failure", test_single_job_failure);
+    g_test_add_func("/single/cancel", test_single_job_cancel);
+    g_test_add_func("/pair/success", test_pair_jobs_success);
+    g_test_add_func("/pair/failure", test_pair_jobs_failure);
+    g_test_add_func("/pair/cancel", test_pair_jobs_cancel);
+    g_test_add_func("/pair/fail-cancel-race", test_pair_jobs_fail_cancel_race);
+    return g_test_run();
+}
diff --git a/qemu/tests/test-coroutine.c b/qemu/tests/test-coroutine.c
index b552d9f5e..dd4ced946 100644
--- a/qemu/tests/test-coroutine.c
+++ b/qemu/tests/test-coroutine.c
@@ -11,9 +11,10 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 
 /*
  * Check that qemu_in_coroutine() works
diff --git a/qemu/tests/test-crypto-afsplit.c b/qemu/tests/test-crypto-afsplit.c
new file mode 100644
index 000000000..f9f2fcd41
--- /dev/null
+++ b/qemu/tests/test-crypto-afsplit.c
@@ -0,0 +1,194 @@
+/*
+ * QEMU Crypto anti-forensic splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#include "crypto/afsplit.h"
+
+typedef struct QCryptoAFSplitTestData QCryptoAFSplitTestData;
+struct QCryptoAFSplitTestData {
+    const char *path;
+    QCryptoHashAlgorithm hash;
+    uint32_t stripes;
+    size_t blocklen;
+    const uint8_t *key;
+    const uint8_t *splitkey;
+};
+
+static QCryptoAFSplitTestData test_data[] = {
+    {
+        .path = "/crypto/afsplit/sha256/5",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 5,
+        .blocklen = 32,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+            "\xa8\xa9\xaa\xab\xac\xad\xae\xaf",
+        .splitkey = (const uint8_t *)
+            "\xfd\xd2\x73\xb1\x7d\x99\x93\x34"
+            "\x70\xde\xfa\x07\xc5\xac\x58\xd2"
+            "\x30\x67\x2f\x1a\x35\x43\x60\x7d"
+            "\x77\x02\xdb\x62\x3c\xcb\x2c\x33"
+            "\x48\x08\xb6\xf1\x7c\xa3\x20\xa0"
+            "\xad\x2d\x4c\xf3\xcd\x18\x6f\x53"
+            "\xf9\xe8\xe7\x59\x27\x3c\xa9\x54"
+            "\x61\x87\xb3\xaf\xf6\xf7\x7e\x64"
+            "\x86\xaa\x89\x7f\x1f\x9f\xdb\x86"
+            "\xf4\xa2\x16\xff\xa3\x4f\x8c\xa1"
+            "\x59\xc4\x23\x34\x28\xc4\x77\x71"
+            "\x83\xd4\xcd\x8e\x89\x1b\xc7\xc5"
+            "\xae\x4d\xa9\xcd\xc9\x72\x85\x70"
+            "\x13\x68\x52\x83\xfc\xb8\x11\x72"
+            "\xba\x3d\xc6\x4a\x28\xfa\xe2\x86"
+            "\x7b\x27\xab\x58\xe1\xa4\xca\xf6"
+            "\x9e\xbc\xfe\x0c\x92\x79\xb3\xec"
+            "\x1c\x5f\x79\x3b\x0d\x1e\xaa\x1a"
+            "\x77\x0f\x70\x19\x4b\xc8\x80\xee"
+            "\x27\x7c\x6e\x4a\x91\x96\x5c\xf4"
+    },
+    {
+        .path = "/crypto/afsplit/sha256/5000",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 5000,
+        .blocklen = 16,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+    },
+    {
+        .path = "/crypto/afsplit/sha1/1000",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .stripes = 1000,
+        .blocklen = 32,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+            "\xa8\xa9\xaa\xab\xac\xad\xae\xaf",
+    },
+    {
+        .path = "/crypto/afsplit/sha256/big",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 1000,
+        .blocklen = 64,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+    },
+};
+
+
+static inline char hex(int i)
+{
+    if (i < 10) {
+        return '0' + i;
+    }
+    return 'a' + (i - 10);
+}
+
+static char *hex_string(const uint8_t *bytes,
+                        size_t len)
+{
+    char *hexstr = g_new0(char, len * 2 + 1);
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        hexstr[i * 2] = hex((bytes[i] >> 4) & 0xf);
+        hexstr[i * 2 + 1] = hex(bytes[i] & 0xf);
+    }
+    hexstr[len * 2] = '\0';
+
+    return hexstr;
+}
+
+static void test_afsplit(const void *opaque)
+{
+    const QCryptoAFSplitTestData *data = opaque;
+    size_t splitlen = data->blocklen * data->stripes;
+    uint8_t *splitkey = g_new0(uint8_t, splitlen);
+    uint8_t *key = g_new0(uint8_t, data->blocklen);
+    gchar *expect, *actual;
+
+    /* First time we round-trip the key */
+    qcrypto_afsplit_encode(data->hash,
+                           data->blocklen, data->stripes,
+                           data->key, splitkey,
+                           &error_abort);
+
+    qcrypto_afsplit_decode(data->hash,
+                           data->blocklen, data->stripes,
+                           splitkey, key,
+                           &error_abort);
+
+    expect = hex_string(data->key, data->blocklen);
+    actual = hex_string(key, data->blocklen);
+
+    g_assert_cmpstr(actual, ==, expect);
+
+    g_free(actual);
+    g_free(expect);
+
+    /* Second time we merely try decoding a previous split */
+    if (data->splitkey) {
+        memset(key, 0, data->blocklen);
+
+        qcrypto_afsplit_decode(data->hash,
+                               data->blocklen, data->stripes,
+                               data->splitkey, key,
+                               &error_abort);
+
+        expect = hex_string(data->key, data->blocklen);
+        actual = hex_string(key, data->blocklen);
+
+        g_assert_cmpstr(actual, ==, expect);
+
+        g_free(actual);
+        g_free(expect);
+    }
+
+    g_free(key);
+    g_free(splitkey);
+}
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (!qcrypto_hash_supports(test_data[i].hash)) {
+            continue;
+        }
+        g_test_add_data_func(test_data[i].path, &test_data[i], test_afsplit);
+    }
+    return g_test_run();
+}
diff --git a/qemu/tests/test-crypto-block.c b/qemu/tests/test-crypto-block.c
new file mode 100644
index 000000000..a38110d3f
--- /dev/null
+++ b/qemu/tests/test-crypto-block.c
@@ -0,0 +1,363 @@
+/*
+ * QEMU Crypto block encryption
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#include "crypto/block.h"
+#include "qemu/buffer.h"
+#include "crypto/secret.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#if defined(CONFIG_UUID) && (defined(_WIN32) || defined RUSAGE_THREAD)
+#define TEST_LUKS
+#else
+#undef TEST_LUKS
+#endif
+
+static QCryptoBlockCreateOptions qcow_create_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_QCOW,
+    .u.qcow = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+static QCryptoBlockOpenOptions qcow_open_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_QCOW,
+    .u.qcow = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+#ifdef TEST_LUKS
+static QCryptoBlockOpenOptions luks_open_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+/* Creation with all default values */
+static QCryptoBlockCreateOptions luks_create_opts_default = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+/* ...and with explicit values */
+static QCryptoBlockCreateOptions luks_create_opts_aes256_cbc_plain64 = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+        .has_cipher_alg = true,
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .has_cipher_mode = true,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .has_ivgen_alg = true,
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+    },
+};
+
+
+static QCryptoBlockCreateOptions luks_create_opts_aes256_cbc_essiv = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+        .has_cipher_alg = true,
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .has_cipher_mode = true,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .has_ivgen_alg = true,
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .has_ivgen_hash_alg = true,
+        .ivgen_hash_alg = QCRYPTO_HASH_ALG_SHA256,
+        .has_hash_alg = true,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA1,
+    },
+};
+#endif /* TEST_LUKS */
+
+
+static struct QCryptoBlockTestData {
+    const char *path;
+    QCryptoBlockCreateOptions *create_opts;
+    QCryptoBlockOpenOptions *open_opts;
+
+    bool expect_header;
+
+    QCryptoCipherAlgorithm cipher_alg;
+    QCryptoCipherMode cipher_mode;
+    QCryptoHashAlgorithm hash_alg;
+
+    QCryptoIVGenAlgorithm ivgen_alg;
+    QCryptoHashAlgorithm ivgen_hash;
+
+    bool slow;
+} test_data[] = {
+    {
+        .path = "/crypto/block/qcow",
+        .create_opts = &qcow_create_opts,
+        .open_opts = &qcow_open_opts,
+
+        .expect_header = false,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+    },
+#ifdef TEST_LUKS
+    {
+        .path = "/crypto/block/luks/default",
+        .create_opts = &luks_create_opts_default,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_XTS,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA256,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+
+        .slow = true,
+    },
+    {
+        .path = "/crypto/block/luks/aes-256-cbc-plain64",
+        .create_opts = &luks_create_opts_aes256_cbc_plain64,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA256,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+
+        .slow = true,
+    },
+    {
+        .path = "/crypto/block/luks/aes-256-cbc-essiv",
+        .create_opts = &luks_create_opts_aes256_cbc_essiv,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA1,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .ivgen_hash = QCRYPTO_HASH_ALG_SHA256,
+
+        .slow = true,
+    },
+#endif
+};
+
+
+static ssize_t test_block_read_func(QCryptoBlock *block,
+                                    size_t offset,
+                                    uint8_t *buf,
+                                    size_t buflen,
+                                    Error **errp,
+                                    void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(offset + buflen, <=, header->capacity);
+
+    memcpy(buf, header->buffer + offset, buflen);
+
+    return buflen;
+}
+
+
+static ssize_t test_block_init_func(QCryptoBlock *block,
+                                    size_t headerlen,
+                                    Error **errp,
+                                    void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(header->capacity, ==, 0);
+
+    buffer_reserve(header, headerlen);
+
+    return headerlen;
+}
+
+
+static ssize_t test_block_write_func(QCryptoBlock *block,
+                                     size_t offset,
+                                     const uint8_t *buf,
+                                     size_t buflen,
+                                     Error **errp,
+                                     void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(buflen + offset, <=, header->capacity);
+
+    memcpy(header->buffer + offset, buf, buflen);
+    header->offset = offset + buflen;
+
+    return buflen;
+}
+
+
+static Object *test_block_secret(void)
+{
+    return object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+}
+
+static void test_block_assert_setup(const struct QCryptoBlockTestData *data,
+                                    QCryptoBlock *blk)
+{
+    QCryptoIVGen *ivgen;
+    QCryptoCipher *cipher;
+
+    ivgen = qcrypto_block_get_ivgen(blk);
+    cipher = qcrypto_block_get_cipher(blk);
+
+    g_assert(ivgen);
+    g_assert(cipher);
+
+    g_assert_cmpint(data->cipher_alg, ==, cipher->alg);
+    g_assert_cmpint(data->cipher_mode, ==, cipher->mode);
+    g_assert_cmpint(data->hash_alg, ==,
+                    qcrypto_block_get_kdf_hash(blk));
+
+    g_assert_cmpint(data->ivgen_alg, ==,
+                    qcrypto_ivgen_get_algorithm(ivgen));
+    g_assert_cmpint(data->ivgen_hash, ==,
+                    qcrypto_ivgen_get_hash(ivgen));
+}
+
+
+static void test_block(gconstpointer opaque)
+{
+    const struct QCryptoBlockTestData *data = opaque;
+    QCryptoBlock *blk;
+    Buffer header;
+    Object *sec = test_block_secret();
+
+    memset(&header, 0, sizeof(header));
+    buffer_init(&header, "header");
+
+    blk = qcrypto_block_create(data->create_opts,
+                               test_block_init_func,
+                               test_block_write_func,
+                               &header,
+                               &error_abort);
+    g_assert(blk);
+
+    if (data->expect_header) {
+        g_assert_cmpint(header.capacity, >, 0);
+    } else {
+        g_assert_cmpint(header.capacity, ==, 0);
+    }
+
+    test_block_assert_setup(data, blk);
+
+    qcrypto_block_free(blk);
+    object_unparent(sec);
+
+    /* Ensure we can't open without the secret */
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             0,
+                             NULL);
+    g_assert(blk == NULL);
+
+    /* Ensure we can't open without the secret, unless NO_IO */
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             QCRYPTO_BLOCK_OPEN_NO_IO,
+                             &error_abort);
+
+    g_assert(qcrypto_block_get_cipher(blk) == NULL);
+    g_assert(qcrypto_block_get_ivgen(blk) == NULL);
+
+    qcrypto_block_free(blk);
+
+
+    /* Now open for real with secret */
+    sec = test_block_secret();
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             0,
+                             &error_abort);
+    g_assert(blk);
+
+    test_block_assert_setup(data, blk);
+
+    qcrypto_block_free(blk);
+
+    object_unparent(sec);
+
+    buffer_free(&header);
+}
+
+
+int main(int argc, char **argv)
+{
+    gsize i;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (test_data[i].open_opts->format == Q_CRYPTO_BLOCK_FORMAT_LUKS &&
+            !qcrypto_hash_supports(test_data[i].hash_alg)) {
+            continue;
+        }
+        if (!test_data[i].slow ||
+            g_test_slow()) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_block);
+        }
+    }
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-crypto-cipher.c b/qemu/tests/test-crypto-cipher.c
index 9d38d2640..66d1c63fd 100644
--- a/qemu/tests/test-crypto-cipher.c
+++ b/qemu/tests/test-crypto-cipher.c
@@ -18,10 +18,12 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "crypto/init.h"
 #include "crypto/cipher.h"
+#include "qapi/error.h"
 
 typedef struct QCryptoCipherTestData QCryptoCipherTestData;
 struct QCryptoCipherTestData {
@@ -164,6 +166,211 @@ static QCryptoCipherTestData test_data[] = {
             "ffd29f1bb5596ad94ea2d8e6196b7f09"
             "30d8ed0bf2773af36dd82a6280c20926",
     },
+    {
+        /* RFC 2144, Appendix B.1 */
+        .path = "/crypto/cipher/cast5-128",
+        .alg = QCRYPTO_CIPHER_ALG_CAST5_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "0123456712345678234567893456789A",
+        .plaintext = "0123456789abcdef",
+        .ciphertext = "238b4fe5847e44b2",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-128",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000",
+        .plaintext = "d29d576fcea3a3a7ed9099f29273d78e",
+        .ciphertext = "b2288b968ae8b08648d1ce9606fd992d",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-192",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_192,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "0000000000000000",
+        .plaintext = "d29d576fceaba3a7ed9899f2927bd78e",
+        .ciphertext = "130e353e1037c22405e8faefb2c3c3e9",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-256a",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "00000000000000000000000000000000",
+        .plaintext = "d095576fcea3e3a7ed98d9f29073d78e",
+        .ciphertext = "b90ee5862de69168f2bdd5125b45472b",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-256b",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "00000000000000000000000000000000",
+        .plaintext = "00000000010000000200000003000000",
+        .ciphertext = "2061a42782bd52ec691ec383b03ba77c",
+    },
+    {
+        /* Twofish paper "Known Answer Test" */
+        .path = "/crypto/cipher/twofish-128",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "d491db16e7b1c39e86cb086b789f5419",
+        .plaintext = "019f9809de1711858faac3a3ba20fbc3",
+        .ciphertext = "6363977de839486297e661c6c9d668eb",
+    },
+    {
+        /* Twofish paper "Known Answer Test", I=3 */
+        .path = "/crypto/cipher/twofish-192",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_192,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "88b2b2706b105e36b446bb6d731a1e88"
+               "efa71f788965bd44",
+        .plaintext = "39da69d6ba4997d585b6dc073ca341b2",
+        .ciphertext = "182b02d81497ea45f9daacdc29193a65",
+    },
+    {
+        /* Twofish paper "Known Answer Test", I=4 */
+        .path = "/crypto/cipher/twofish-256",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "d43bb7556ea32e46f2a282b7d45b4e0d"
+               "57ff739d4dc92c1bd7fc01700cc8216f",
+        .plaintext = "90afe91bb288544f2c32dc239b2635e6",
+        .ciphertext = "6cb4561c40bf0a9705931cb6d408e7fa",
+    },
+    {
+        /* #1 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-1",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "00000000000000000000000000000000"
+            "00000000000000000000000000000000",
+        .iv =
+            "00000000000000000000000000000000",
+        .plaintext =
+            "00000000000000000000000000000000"
+            "00000000000000000000000000000000",
+        .ciphertext =
+            "917cf69ebd68b2ec9b9fe9a3eadda692"
+            "cd43d2f59598ed858c02c2652fbf922e",
+    },
+    {
+        /* #2, 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-2",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "11111111111111111111111111111111"
+            "22222222222222222222222222222222",
+        .iv =
+            "33333333330000000000000000000000",
+        .plaintext =
+            "44444444444444444444444444444444"
+            "44444444444444444444444444444444",
+        .ciphertext =
+            "c454185e6a16936e39334038acef838b"
+            "fb186fff7480adc4289382ecd6d394f0",
+    },
+    {
+        /* #5 from xts.7, 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-3",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0"
+            "bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0",
+        .iv =
+            "9a785634120000000000000000000000",
+        .plaintext =
+            "44444444444444444444444444444444"
+            "44444444444444444444444444444444",
+        .ciphertext =
+            "b01f86f8edc1863706fa8a4253e34f28"
+            "af319de38334870f4dd1f94cbe9832f1",
+    },
+    {
+        /* #4, 32 byte key, 512 byte PTX  */
+        .path = "/crypto/cipher/aes-xts-128-4",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "27182818284590452353602874713526"
+            "31415926535897932384626433832795",
+        .iv =
+            "00000000000000000000000000000000",
+        .plaintext =
+            "000102030405060708090a0b0c0d0e0f"
+            "101112131415161718191a1b1c1d1e1f"
+            "202122232425262728292a2b2c2d2e2f"
+            "303132333435363738393a3b3c3d3e3f"
+            "404142434445464748494a4b4c4d4e4f"
+            "505152535455565758595a5b5c5d5e5f"
+            "606162636465666768696a6b6c6d6e6f"
+            "707172737475767778797a7b7c7d7e7f"
+            "808182838485868788898a8b8c8d8e8f"
+            "909192939495969798999a9b9c9d9e9f"
+            "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+            "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+            "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+            "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+            "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+            "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
+            "000102030405060708090a0b0c0d0e0f"
+            "101112131415161718191a1b1c1d1e1f"
+            "202122232425262728292a2b2c2d2e2f"
+            "303132333435363738393a3b3c3d3e3f"
+            "404142434445464748494a4b4c4d4e4f"
+            "505152535455565758595a5b5c5d5e5f"
+            "606162636465666768696a6b6c6d6e6f"
+            "707172737475767778797a7b7c7d7e7f"
+            "808182838485868788898a8b8c8d8e8f"
+            "909192939495969798999a9b9c9d9e9f"
+            "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+            "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+            "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+            "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+            "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+            "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff",
+        .ciphertext =
+            "27a7479befa1d476489f308cd4cfa6e2"
+            "a96e4bbe3208ff25287dd3819616e89c"
+            "c78cf7f5e543445f8333d8fa7f560000"
+            "05279fa5d8b5e4ad40e736ddb4d35412"
+            "328063fd2aab53e5ea1e0a9f332500a5"
+            "df9487d07a5c92cc512c8866c7e860ce"
+            "93fdf166a24912b422976146ae20ce84"
+            "6bb7dc9ba94a767aaef20c0d61ad0265"
+            "5ea92dc4c4e41a8952c651d33174be51"
+            "a10c421110e6d81588ede82103a252d8"
+            "a750e8768defffed9122810aaeb99f91"
+            "72af82b604dc4b8e51bcb08235a6f434"
+            "1332e4ca60482a4ba1a03b3e65008fc5"
+            "da76b70bf1690db4eae29c5f1badd03c"
+            "5ccf2a55d705ddcd86d449511ceb7ec3"
+            "0bf12b1fa35b913f9f747a8afd1b130e"
+            "94bff94effd01a91735ca1726acd0b19"
+            "7c4e5b03393697e126826fb6bbde8ecc"
+            "1e08298516e2c9ed03ff3c1b7860f6de"
+            "76d4cecd94c8119855ef5297ca67e9f3"
+            "e7ff72b1e99785ca0a7e7720c5b36dc6"
+            "d72cac9574c8cbbc2f801e23e56fd344"
+            "b07f22154beba0f08ce8891e643ed995"
+            "c94d9a69c9f1b5f499027a78572aeebd"
+            "74d20cc39881c213ee770b1010e4bea7"
+            "18846977ae119f7a023ab58cca0ad752"
+            "afe656bb3c17256a9f6e9bf19fdd5a38"
+            "fc82bbe872c5539edb609ef4f79c203e"
+            "bb140f2e583cb2ad15b4aa5b655016a8"
+            "449277dbd477ef2c8d6c017db738b18d"
+            "eb4a427d1923ce3ff262735779a418f2"
+            "0a282df920147beabe421ee5319d0568",
+    },
 };
 
 
@@ -229,6 +436,7 @@ static void test_cipher(const void *opaque)
     uint8_t *key, *iv, *ciphertext, *plaintext, *outtext;
     size_t nkey, niv, nciphertext, nplaintext;
     char *outtexthex;
+    size_t ivsize, keysize, blocksize;
 
     nkey = unhex_string(data->key, &key);
     niv = unhex_string(data->iv, &iv);
@@ -245,6 +453,19 @@ static void test_cipher(const void *opaque)
         &error_abort);
     g_assert(cipher != NULL);
 
+    keysize = qcrypto_cipher_get_key_len(data->alg);
+    blocksize = qcrypto_cipher_get_block_len(data->alg);
+    ivsize = qcrypto_cipher_get_iv_len(data->alg, data->mode);
+
+    if (data->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        g_assert_cmpint(keysize * 2, ==, nkey);
+    } else {
+        g_assert_cmpint(keysize, ==, nkey);
+    }
+    g_assert_cmpint(ivsize, ==, niv);
+    if (niv) {
+        g_assert_cmpint(blocksize, ==, niv);
+    }
 
     if (iv) {
         g_assert(qcrypto_cipher_setiv(cipher,
@@ -287,6 +508,79 @@ static void test_cipher(const void *opaque)
     qcrypto_cipher_free(cipher);
 }
 
+
+static void test_cipher_null_iv(void)
+{
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext[32] = { 0 };
+    uint8_t ciphertext[32] = { 0 };
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Don't call qcrypto_cipher_setiv */
+
+    qcrypto_cipher_encrypt(cipher,
+                           plaintext,
+                           ciphertext,
+                           sizeof(plaintext),
+                           &error_abort);
+
+    qcrypto_cipher_free(cipher);
+}
+
+static void test_cipher_short_plaintext(void)
+{
+    Error *err = NULL;
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext1[20] = { 0 };
+    uint8_t ciphertext1[20] = { 0 };
+    uint8_t plaintext2[40] = { 0 };
+    uint8_t ciphertext2[40] = { 0 };
+    int ret;
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Should report an error as plaintext is shorter
+     * than block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext1,
+                                 ciphertext1,
+                                 sizeof(plaintext1),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    err = NULL;
+
+    /* Should report an error as plaintext is larger than
+     * block size, but not a multiple of block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext2,
+                                 ciphertext2,
+                                 sizeof(plaintext2),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    qcrypto_cipher_free(cipher);
+}
+
 int main(int argc, char **argv)
 {
     size_t i;
@@ -296,7 +590,16 @@ int main(int argc, char **argv)
     g_assert(qcrypto_init(NULL) == 0);
 
     for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
-        g_test_add_data_func(test_data[i].path, &test_data[i], test_cipher);
+        if (qcrypto_cipher_supports(test_data[i].alg)) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_cipher);
+        }
     }
+
+    g_test_add_func("/crypto/cipher/null-iv",
+                    test_cipher_null_iv);
+
+    g_test_add_func("/crypto/cipher/short-plaintext",
+                    test_cipher_short_plaintext);
+
     return g_test_run();
 }
diff --git a/qemu/tests/test-crypto-hash.c b/qemu/tests/test-crypto-hash.c
index 911437e60..735d6d7e0 100644
--- a/qemu/tests/test-crypto-hash.c
+++ b/qemu/tests/test-crypto-hash.c
@@ -18,6 +18,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "crypto/init.h"
@@ -163,6 +164,11 @@ static void test_hash_digest(void)
     for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
         int ret;
         char *digest;
+        size_t digestsize;
+
+        digestsize = qcrypto_hash_digest_len(i);
+
+        g_assert_cmpint(digestsize * 2, ==, strlen(expected_outputs[i]));
 
         ret = qcrypto_hash_digest(i,
                                   INPUT_TEXT,
diff --git a/qemu/tests/test-crypto-ivgen.c b/qemu/tests/test-crypto-ivgen.c
new file mode 100644
index 000000000..a5ff5d3da
--- /dev/null
+++ b/qemu/tests/test-crypto-ivgen.c
@@ -0,0 +1,174 @@
+/*
+ * QEMU Crypto IV generator algorithms
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/ivgen.h"
+
+
+struct QCryptoIVGenTestData {
+    const char *path;
+    uint64_t sector;
+    QCryptoIVGenAlgorithm ivalg;
+    QCryptoHashAlgorithm hashalg;
+    QCryptoCipherAlgorithm cipheralg;
+    const uint8_t *key;
+    size_t nkey;
+    const uint8_t *iv;
+    size_t niv;
+} test_data[] = {
+    /* Small */
+    {
+        "/crypto/ivgen/plain/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x01\x00\x00\x00\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/plain/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x4c\x3d\x2e\x1f\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Truncation */
+    {
+        "/crypto/ivgen/plain/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x88\x79\x6a\x5b\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Small */
+    {
+        "/crypto/ivgen/plain64/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x01\x00\x00\x00\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/plain64/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x4c\x3d\x2e\x1f\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* No Truncation */
+    {
+        "/crypto/ivgen/plain64/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x88\x79\x6a\x5b\x4c\x3d\x2e\x1f"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Small */
+    {
+        "/crypto/ivgen/essiv/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\xd4\x83\x71\xb2\xa1\x94\x53\x88"
+                               "\x1c\x7a\x2d\06\x2d\x0b\x65\x46",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/essiv/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\x5d\x36\x09\x5d\xc6\x9e\x5e\xe9"
+                               "\xe3\x02\x8d\xd8\x7a\x3d\xe7\x8f",
+        .niv = 16,
+    },
+    /* No Truncation */
+    {
+        "/crypto/ivgen/essiv/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\x58\xbb\x81\x94\x51\x83\x23\x23"
+                               "\x7a\x08\x93\xa9\xdc\xd2\xd9\xab",
+        .niv = 16,
+    },
+};
+
+
+static void test_ivgen(const void *opaque)
+{
+    const struct QCryptoIVGenTestData *data = opaque;
+    uint8_t *iv = g_new0(uint8_t, data->niv);
+    QCryptoIVGen *ivgen = qcrypto_ivgen_new(
+        data->ivalg,
+        data->cipheralg,
+        data->hashalg,
+        data->key,
+        data->nkey,
+        &error_abort);
+
+    qcrypto_ivgen_calculate(ivgen,
+                            data->sector,
+                            iv,
+                            data->niv,
+                            &error_abort);
+
+    g_assert(memcmp(iv, data->iv, data->niv) == 0);
+
+    qcrypto_ivgen_free(ivgen);
+    g_free(iv);
+}
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    g_test_init(&argc, &argv, NULL);
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (test_data[i].ivalg == QCRYPTO_IVGEN_ALG_ESSIV &&
+            !qcrypto_hash_supports(test_data[i].hashalg)) {
+            continue;
+        }
+        g_test_add_data_func(test_data[i].path,
+                             &(test_data[i]),
+                             test_ivgen);
+    }
+    return g_test_run();
+}
diff --git a/qemu/tests/test-crypto-pbkdf.c b/qemu/tests/test-crypto-pbkdf.c
new file mode 100644
index 000000000..8ceceb182
--- /dev/null
+++ b/qemu/tests/test-crypto-pbkdf.c
@@ -0,0 +1,393 @@
+/*
+ * QEMU Crypto cipher algorithms
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#if ((defined(CONFIG_NETTLE) || defined(CONFIG_GCRYPT)) && \
+     (defined(_WIN32) || defined(RUSAGE_THREAD)))
+#include "crypto/pbkdf.h"
+
+typedef struct QCryptoPbkdfTestData QCryptoPbkdfTestData;
+struct QCryptoPbkdfTestData {
+    const char *path;
+    QCryptoHashAlgorithm hash;
+    unsigned int iterations;
+    const char *key;
+    size_t nkey;
+    const char *salt;
+    size_t nsalt;
+    const char *out;
+    size_t nout;
+    bool slow;
+};
+
+/* This test data comes from cryptsetup package
+ *
+ *  $SRC/lib/crypto_backend/pbkdf2_generic.c
+ *
+ * under LGPLv2.1+ license
+ */
+static QCryptoPbkdfTestData test_data[] = {
+    /* RFC 3962 test data */
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\xcd\xed\xb5\x28\x1b\xb2\xf8\x01"
+               "\x56\x5a\x11\x22\xb2\x56\x35\x15"
+               "\x0a\xd1\xf7\xa0\x4b\xb9\xf3\xa3"
+               "\x33\xec\xc0\xe2\xe1\xf7\x08\x37",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\x01\xdb\xee\x7f\x4a\x9e\x24\x3e"
+               "\x98\x8b\x62\xc7\x3c\xda\x93\x5d"
+               "\xa0\x53\x78\xb9\x32\x44\xec\x8f"
+               "\x48\xa9\x9e\x61\xad\x79\x9d\x86",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200a",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\x5c\x08\xeb\x61\xfd\xf7\x1e\x4e"
+               "\x4e\xc3\xcf\x6b\xa1\xf5\x51\x2b"
+               "\xa7\xe5\x2d\xdb\xc5\xe5\x14\x2f"
+               "\x70\x8a\x31\xe2\xe6\x2b\x1e\x13",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter5",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 5,
+        .key = "password",
+        .nkey = 8,
+        .salt = "\0224VxxV4\022", /* "\x1234567878563412 */
+        .nsalt = 8,
+        .out = "\xd1\xda\xa7\x86\x15\xf2\x87\xe6"
+               "\xa1\xc8\xb1\x20\xd7\x06\x2a\x49"
+               "\x3f\x98\xd2\x03\xe6\xbe\x49\xa6"
+               "\xad\xf4\xfa\x57\x4b\x6e\x64\xee",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200b",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 64,
+        .salt = "pass phrase equals block size",
+        .nsalt = 29,
+        .out = "\x13\x9c\x30\xc0\x96\x6b\xc3\x2b"
+               "\xa5\x5f\xdb\xf2\x12\x53\x0a\xc9"
+               "\xc5\xec\x59\xf1\xa4\x52\xf5\xcc"
+               "\x9a\xd9\x40\xfe\xa0\x59\x8e\xd1",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200c",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x9c\xca\xd6\xd4\x68\x77\x0c\xd5"
+               "\x1b\x10\xe6\xa6\x87\x21\xbe\x61"
+               "\x1a\x8b\x4d\x28\x26\x01\xdb\x3b"
+               "\x36\xbe\x92\x46\x91\x5e\xc8\x2a",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter50",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 50,
+        .key = "\360\235\204\236", /* g-clef ("\xf09d849e) */
+        .nkey = 4,
+        .salt = "EXAMPLE.COMpianist",
+        .nsalt = 18,
+        .out = "\x6b\x9c\xf2\x6d\x45\x45\x5a\x43"
+               "\xa5\xb8\xbb\x27\x6a\x40\x3b\x39"
+               "\xe7\xfe\x37\xa0\xc4\x1e\x02\xc2"
+               "\x81\xff\x30\x69\xe1\xe9\x4f\x52",
+        .nout = 32
+    },
+
+    /* RFC-6070 test data */
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter1",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x0c\x60\xc8\x0f\x96\x1f\x0e\x71\xf3\xa9"
+               "\xb5\x24\xaf\x60\x12\x06\x2f\xe0\x37\xa6",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\xea\x6c\x01\x4d\xc7\x2d\x6f\x8c\xcd\x1e"
+               "\xd9\x2a\xce\x1d\x41\xf0\xd8\xde\x89\x57",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x4b\x00\x79\x01\xb7\x65\x48\x9a\xbe\xad"
+               "\x49\xd9\x26\xf7\x21\xd0\x65\xa4\x29\xc1",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter16777216",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 16777216,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\xee\xfe\x3d\x61\xcd\x4d\xa4\xe4\xe9\x94"
+               "\x5b\x3d\x6b\xa2\x15\x8c\x26\x34\xe9\x84",
+        .nout = 20,
+        .slow = true,
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096a",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "passwordPASSWORDpassword",
+        .nkey = 24,
+        .salt = "saltSALTsaltSALTsaltSALTsaltSALTsalt",
+        .nsalt = 36,
+        .out = "\x3d\x2e\xec\x4f\xe4\x1c\x84\x9b\x80\xc8"
+               "\xd8\x36\x62\xc0\xe4\x4a\x8b\x29\x1a\x96"
+               "\x4c\xf2\xf0\x70\x38",
+        .nout = 25
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096b",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "pass\0word",
+        .nkey = 9,
+        .salt = "sa\0lt",
+        .nsalt = 5,
+        .out = "\x56\xfa\x6a\xa7\x55\x48\x09\x9d\xcc\x37"
+               "\xd7\xf0\x34\x25\xe0\xc3",
+        .nout = 16
+    },
+
+    /* non-RFC misc test data */
+#ifdef CONFIG_NETTLE
+    {
+        /* empty password test.
+         * Broken with libgcrypt <= 1.5.0, hence CONFIG_NETTLE */
+        .path = "/crypto/pbkdf/nonrfc/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "",
+        .nkey = 0,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x13\x3a\x4c\xe8\x37\xb4\xd2\x52\x1e\xe2"
+               "\xbf\x03\xe1\x1c\x71\xca\x79\x4e\x07\x97",
+        .nout = 20
+    },
+#endif
+    {
+        /* Password exceeds block size test */
+        .path = "/crypto/pbkdf/nonrfc/sha256/iter1200",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x22\x34\x4b\xc4\xb6\xe3\x26\x75"
+               "\xa8\x09\x0f\x3e\xa8\x0b\xe0\x1d"
+               "\x5f\x95\x12\x6a\x2c\xdd\xc3\xfa"
+               "\xcc\x4a\x5e\x6d\xca\x04\xec\x58",
+        .nout = 32
+    },
+#if 0
+    {
+        .path = "/crypto/pbkdf/nonrfc/sha512/iter1200",
+        .hash = QCRYPTO_HASH_ALG_SHA512,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 129,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x0f\xb2\xed\x2c\x0e\x6e\xfb\x7d"
+               "\x7d\x8e\xdd\x58\x01\xb4\x59\x72"
+               "\x99\x92\x16\x30\x5e\xa4\x36\x8d"
+               "\x76\x14\x80\xf3\xe3\x7a\x22\xb9",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/nonrfc/whirlpool/iter1200",
+        .hash = QCRYPTO_HASH_ALG_WHIRLPOOL,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x9c\x1c\x74\xf5\x88\x26\xe7\x6a"
+               "\x53\x58\xf4\x0c\x39\xe7\x80\x89"
+               "\x07\xc0\x31\x19\x9a\x50\xa2\x48"
+               "\xf1\xd9\xfe\x78\x64\xe5\x84\x50",
+        .nout = 32
+    }
+#endif
+};
+
+
+static inline char hex(int i)
+{
+    if (i < 10) {
+        return '0' + i;
+    }
+    return 'a' + (i - 10);
+}
+
+static char *hex_string(const uint8_t *bytes,
+                        size_t len)
+{
+    char *hexstr = g_new0(char, len * 2 + 1);
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        hexstr[i * 2] = hex((bytes[i] >> 4) & 0xf);
+        hexstr[i * 2 + 1] = hex(bytes[i] & 0xf);
+    }
+    hexstr[len * 2] = '\0';
+
+    return hexstr;
+}
+
+static void test_pbkdf(const void *opaque)
+{
+    const QCryptoPbkdfTestData *data = opaque;
+    size_t nout = data->nout;
+    uint8_t *out = g_new0(uint8_t, nout);
+    gchar *expect, *actual;
+
+    qcrypto_pbkdf2(data->hash,
+                   (uint8_t *)data->key, data->nkey,
+                   (uint8_t *)data->salt, data->nsalt,
+                   data->iterations,
+                   (uint8_t *)out, nout,
+                   &error_abort);
+
+    expect = hex_string((const uint8_t *)data->out, data->nout);
+    actual = hex_string(out, nout);
+
+    g_assert_cmpstr(actual, ==, expect);
+
+    g_free(actual);
+    g_free(expect);
+    g_free(out);
+}
+
+
+static void test_pbkdf_timing(void)
+{
+    uint8_t key[32];
+    uint8_t salt[32];
+    int iters;
+
+    memset(key, 0x5d, sizeof(key));
+    memset(salt, 0x7c, sizeof(salt));
+
+    iters = qcrypto_pbkdf2_count_iters(QCRYPTO_HASH_ALG_SHA256,
+                                       key, sizeof(key),
+                                       salt, sizeof(salt),
+                                       &error_abort);
+
+    g_assert(iters >= (1 << 15));
+}
+
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (!test_data[i].slow ||
+            g_test_slow()) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_pbkdf);
+        }
+    }
+
+    if (g_test_slow()) {
+        g_test_add_func("/crypt0/pbkdf/timing", test_pbkdf_timing);
+    }
+
+    return g_test_run();
+}
+#else
+int main(int argc, char **argv)
+{
+    return 0;
+}
+#endif
diff --git a/qemu/tests/test-crypto-secret.c b/qemu/tests/test-crypto-secret.c
new file mode 100644
index 000000000..aa26c2049
--- /dev/null
+++ b/qemu/tests/test-crypto-secret.c
@@ -0,0 +1,455 @@
+/*
+ * QEMU Crypto secret handling
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "crypto/init.h"
+#include "crypto/secret.h"
+#include "qapi/error.h"
+#include "qemu/module.h"
+
+static void test_secret_direct(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_indirect_good(void)
+{
+    Object *sec;
+    char *fname = NULL;
+    int fd = g_file_open_tmp("secretXXXXXX",
+                             &fname,
+                             NULL);
+
+    g_assert(fd >= 0);
+    g_assert_nonnull(fname);
+
+    g_assert(write(fd, "123456", 6) == 6);
+
+    sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "file", fname,
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+    close(fd);
+    g_free(fname);
+}
+
+
+static void test_secret_indirect_badfile(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "file", "does-not-exist",
+        NULL);
+
+    g_assert(sec == NULL);
+}
+
+
+static void test_secret_indirect_emptyfile(void)
+{
+    Object *sec;
+    char *fname = NULL;
+    int fd = g_file_open_tmp("secretXXXXXX",
+                             &fname,
+                             NULL);
+
+    g_assert(fd >= 0);
+    g_assert_nonnull(fname);
+
+    sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "file", fname,
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "");
+
+    object_unparent(sec);
+    g_free(pw);
+    close(fd);
+    g_free(fname);
+}
+
+
+static void test_secret_noconv_base64_good(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "MTIzNDU2",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_base64("sec0",
+                                               &error_abort);
+
+    g_assert_cmpstr(pw, ==, "MTIzNDU2");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_noconv_base64_bad(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "MTI$NDU2",
+        "format", "base64",
+        NULL);
+
+    g_assert(sec == NULL);
+}
+
+
+static void test_secret_noconv_utf8(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        "format", "raw",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_conv_base64_utf8valid(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "MTIzNDU2",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_conv_base64_utf8invalid(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "f0VMRgIBAQAAAA==",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             NULL);
+    g_assert(pw == NULL);
+
+    object_unparent(sec);
+}
+
+
+static void test_secret_conv_utf8_base64(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_base64("sec0",
+                                               &error_abort);
+
+    g_assert_cmpstr(pw, ==, "MTIzNDU2");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_raw(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data",
+        "\xCC\xBF\xF7\x09\x46\x19\x0B\x52\x2A\x3A\xB4\x6B\xCD\x7A\xB0\xB0",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    object_unparent(master);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_base64(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "base64",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    object_unparent(master);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_short_key(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVc",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_short_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2a",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_missing_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_bad_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TK$$uA+Old2W2a",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    g_test_add_func("/crypto/secret/direct",
+                    test_secret_direct);
+    g_test_add_func("/crypto/secret/indirect/good",
+                    test_secret_indirect_good);
+    g_test_add_func("/crypto/secret/indirect/badfile",
+                    test_secret_indirect_badfile);
+    g_test_add_func("/crypto/secret/indirect/emptyfile",
+                    test_secret_indirect_emptyfile);
+
+    g_test_add_func("/crypto/secret/noconv/base64/good",
+                    test_secret_noconv_base64_good);
+    g_test_add_func("/crypto/secret/noconv/base64/bad",
+                    test_secret_noconv_base64_bad);
+    g_test_add_func("/crypto/secret/noconv/utf8",
+                    test_secret_noconv_utf8);
+    g_test_add_func("/crypto/secret/conv/base64/utf8valid",
+                    test_secret_conv_base64_utf8valid);
+    g_test_add_func("/crypto/secret/conv/base64/utf8invalid",
+                    test_secret_conv_base64_utf8invalid);
+    g_test_add_func("/crypto/secret/conv/utf8/base64",
+                    test_secret_conv_utf8_base64);
+
+    g_test_add_func("/crypto/secret/crypt/raw",
+                    test_secret_crypt_raw);
+    g_test_add_func("/crypto/secret/crypt/base64",
+                    test_secret_crypt_base64);
+    g_test_add_func("/crypto/secret/crypt/shortkey",
+                    test_secret_crypt_short_key);
+    g_test_add_func("/crypto/secret/crypt/shortiv",
+                    test_secret_crypt_short_iv);
+    g_test_add_func("/crypto/secret/crypt/missingiv",
+                    test_secret_crypt_missing_iv);
+    g_test_add_func("/crypto/secret/crypt/badiv",
+                    test_secret_crypt_bad_iv);
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-crypto-tlscredsx509.c b/qemu/tests/test-crypto-tlscredsx509.c
new file mode 100644
index 000000000..af2f80e89
--- /dev/null
+++ b/qemu/tests/test-crypto-tlscredsx509.c
@@ -0,0 +1,730 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "qapi/error.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlscredsx509-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSCredsTestData {
+    bool isServer;
+    const char *cacrt;
+    const char *crt;
+    bool expectFail;
+};
+
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        "testtlscreds",
+        errp,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        "sanity-check", "yes",
+        NULL);
+
+    if (*errp) {
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+/*
+ * This tests sanity checking of our own certificates
+ *
+ * The code being tested is used when TLS creds are created,
+ * and aim to ensure QMEU has been configured with sane
+ * certificates. This allows us to give much much much
+ * clearer error messages to the admin when they misconfigure
+ * things.
+ */
+static void test_tls_creds(const void *opaque)
+{
+    struct QCryptoTLSCredsTestData *data =
+        (struct QCryptoTLSCredsTestData *)opaque;
+    QCryptoTLSCreds *creds;
+    Error *err = NULL;
+
+#define CERT_DIR "tests/test-crypto-tlscredsx509-certs/"
+    mkdir(CERT_DIR, 0700);
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+
+    if (access(data->cacrt, R_OK) == 0) {
+        g_assert(link(data->cacrt,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    }
+    if (data->isServer) {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+    } else {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+    }
+
+    creds = test_tls_creds_create(
+        (data->isServer ?
+         QCRYPTO_TLS_CREDS_ENDPOINT_SERVER :
+         QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT),
+        CERT_DIR,
+        &err);
+
+    if (data->expectFail) {
+        error_free(err);
+        g_assert(creds == NULL);
+    } else {
+        if (err) {
+            g_printerr("Failed to generate creds: %s\n",
+                       error_get_pretty(err));
+            error_free(err);
+        }
+        g_assert(creds != NULL);
+    }
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+    rmdir(CERT_DIR);
+    if (creds) {
+        object_unparent(OBJECT(creds));
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TLS_TEST_REG(name, isServer, caCrt, crt, expectFail)           \
+    struct QCryptoTLSCredsTestData name = {                             \
+        isServer, caCrt, crt, expectFail                                \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlscredsx509/" # name,               \
+                         &name, test_tls_creds);                        \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(perfectserver, true,
+                 cacertreq.filename, servercertreq.filename, false);
+    TLS_TEST_REG(perfectclient, false,
+                 cacertreq.filename, clientcertreq.filename, false);
+
+
+    /* Some other CAs which are good */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacert1req,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert1req, cacert1req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Basic:CA:not-critical */
+    TLS_ROOT_REQ(cacert2req,
+                 "UK", "qemu CA 2", NULL, NULL, NULL, NULL,
+                 true, false, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert2req, cacert2req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Key usage:cert-sign:critical */
+    TLS_ROOT_REQ(cacert3req,
+                 "UK", "qemu CA 3", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert3req, cacert3req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(goodca1, true,
+                 cacert1req.filename, servercert1req.filename, false);
+    TLS_TEST_REG(goodca2, true,
+                 cacert2req.filename, servercert2req.filename, false);
+    TLS_TEST_REG(goodca3, true,
+                 cacert3req.filename, servercert3req.filename, false);
+
+    /* Now some bad certs */
+
+    /* Key usage:dig-sig:not-critical */
+    TLS_ROOT_REQ(cacert4req,
+                 "UK", "qemu CA 4", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, false, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert4req, cacert4req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* no-basic */
+    TLS_ROOT_REQ(cacert5req,
+                 "UK", "qemu CA 5", NULL, NULL, NULL, NULL,
+                 false, false, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert5req, cacert5req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* Key usage:dig-sig:critical */
+    TLS_ROOT_REQ(cacert6req,
+                 "UK", "qemu CA 6", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert6req, cacert6req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Technically a CA cert with basic constraints
+     * key purpose == key signing + non-critical should
+     * be rejected. GNUTLS < 3.1 does not reject it and
+     * we don't anticipate them changing this behaviour
+     */
+    TLS_TEST_REG(badca1, true, cacert4req.filename, servercert4req.filename,
+                (GNUTLS_VERSION_MAJOR == 3 && GNUTLS_VERSION_MINOR >= 1) ||
+                GNUTLS_VERSION_MAJOR > 3);
+    TLS_TEST_REG(badca2, true,
+                 cacert5req.filename, servercert5req.filename, true);
+    TLS_TEST_REG(badca3, true,
+                 cacert6req.filename, servercert6req.filename, true);
+
+
+    /* Various good servers */
+    /* no usage or purpose */
+    TLS_CERT_REQ(servercert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(servercert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(servercert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:server:critical */
+    TLS_CERT_REQ(servercert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:server:not-critical */
+    TLS_CERT_REQ(servercert11req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:client+server:critical */
+    TLS_CERT_REQ(servercert12req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+server:not-critical */
+    TLS_CERT_REQ(servercert13req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodserver1, true,
+                 cacertreq.filename, servercert7req.filename, false);
+    TLS_TEST_REG(goodserver2, true,
+                 cacertreq.filename, servercert8req.filename, false);
+    TLS_TEST_REG(goodserver3, true,
+                 cacertreq.filename, servercert9req.filename, false);
+    TLS_TEST_REG(goodserver4, true,
+                 cacertreq.filename, servercert10req.filename, false);
+    TLS_TEST_REG(goodserver5, true,
+                 cacertreq.filename, servercert11req.filename, false);
+    TLS_TEST_REG(goodserver6, true,
+                 cacertreq.filename, servercert12req.filename, false);
+    TLS_TEST_REG(goodserver7, true,
+                 cacertreq.filename, servercert13req.filename, false);
+
+    /* Bad servers */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(servercert14req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(servercert15req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(servercert16req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badserver1, true,
+                 cacertreq.filename, servercert14req.filename, true);
+    TLS_TEST_REG(badserver2, true,
+                 cacertreq.filename, servercert15req.filename, true);
+    TLS_TEST_REG(badserver3, true,
+                 cacertreq.filename, servercert16req.filename, true);
+
+
+
+    /* Various good clients */
+    /* no usage or purpose */
+    TLS_CERT_REQ(clientcert1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(clientcert2req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(clientcert3req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert4req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client:not-critical */
+    TLS_CERT_REQ(clientcert5req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client+client:critical */
+    TLS_CERT_REQ(clientcert6req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+client:not-critical */
+    TLS_CERT_REQ(clientcert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodclient1, false,
+                 cacertreq.filename, clientcert1req.filename, false);
+    TLS_TEST_REG(goodclient2, false,
+                 cacertreq.filename, clientcert2req.filename, false);
+    TLS_TEST_REG(goodclient3, false,
+                 cacertreq.filename, clientcert3req.filename, false);
+    TLS_TEST_REG(goodclient4, false,
+                 cacertreq.filename, clientcert4req.filename, false);
+    TLS_TEST_REG(goodclient5, false,
+                 cacertreq.filename, clientcert5req.filename, false);
+    TLS_TEST_REG(goodclient6, false,
+                 cacertreq.filename, clientcert6req.filename, false);
+    TLS_TEST_REG(goodclient7, false,
+                 cacertreq.filename, clientcert7req.filename, false);
+
+    /* Bad clients */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(clientcert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(clientcert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badclient1, false,
+                 cacertreq.filename, clientcert8req.filename, true);
+    TLS_TEST_REG(badclient2, false,
+                 cacertreq.filename, clientcert9req.filename, true);
+    TLS_TEST_REG(badclient3, false,
+                 cacertreq.filename, clientcert10req.filename, true);
+
+
+
+    /* Expired stuff */
+
+    TLS_ROOT_REQ(cacertexpreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, -1);
+    TLS_CERT_REQ(servercertexpreq, cacertexpreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, -1);
+    TLS_CERT_REQ(clientcertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, -1);
+
+    TLS_TEST_REG(expired1, true,
+                 cacertexpreq.filename, servercertexpreq.filename, true);
+    TLS_TEST_REG(expired2, true,
+                 cacertreq.filename, servercertexp1req.filename, true);
+    TLS_TEST_REG(expired3, false,
+                 cacertreq.filename, clientcertexp1req.filename, true);
+
+
+    /* Not activated stuff */
+
+    TLS_ROOT_REQ(cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 1, 2);
+    TLS_CERT_REQ(servercertnewreq, cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 1, 2);
+    TLS_CERT_REQ(clientcertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 1, 2);
+
+    TLS_TEST_REG(inactive1, true,
+                 cacertnewreq.filename, servercertnewreq.filename, true);
+    TLS_TEST_REG(inactive2, true,
+                 cacertreq.filename, servercertnew1req.filename, true);
+    TLS_TEST_REG(inactive3, false,
+                 cacertreq.filename, clientcertnew1req.filename, true);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-ctx.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TLS_TEST_REG(chain1, true,
+                 WORKDIR "cacertchain-ctx.pem",
+                 servercertlevel3areq.filename, false);
+    TLS_TEST_REG(chain2, false,
+                 WORKDIR "cacertchain-ctx.pem",
+                 clientcertlevel2breq.filename, false);
+
+    /* Some missing certs - first two are fatal, the last
+     * is ok
+     */
+    TLS_TEST_REG(missingca, true,
+                 "cacertdoesnotexist.pem",
+                 servercert1req.filename, true);
+    TLS_TEST_REG(missingserver, true,
+                 cacert1req.filename,
+                 "servercertdoesnotexist.pem", true);
+    TLS_TEST_REG(missingclient, false,
+                 cacert1req.filename,
+                 "clientcertdoesnotexist.pem", false);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&cacert1req);
+    test_tls_discard_cert(&cacert2req);
+    test_tls_discard_cert(&cacert3req);
+    test_tls_discard_cert(&cacert4req);
+    test_tls_discard_cert(&cacert5req);
+    test_tls_discard_cert(&cacert6req);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercert1req);
+    test_tls_discard_cert(&servercert2req);
+    test_tls_discard_cert(&servercert3req);
+    test_tls_discard_cert(&servercert4req);
+    test_tls_discard_cert(&servercert5req);
+    test_tls_discard_cert(&servercert6req);
+    test_tls_discard_cert(&servercert7req);
+    test_tls_discard_cert(&servercert8req);
+    test_tls_discard_cert(&servercert9req);
+    test_tls_discard_cert(&servercert10req);
+    test_tls_discard_cert(&servercert11req);
+    test_tls_discard_cert(&servercert12req);
+    test_tls_discard_cert(&servercert13req);
+    test_tls_discard_cert(&servercert14req);
+    test_tls_discard_cert(&servercert15req);
+    test_tls_discard_cert(&servercert16req);
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcert1req);
+    test_tls_discard_cert(&clientcert2req);
+    test_tls_discard_cert(&clientcert3req);
+    test_tls_discard_cert(&clientcert4req);
+    test_tls_discard_cert(&clientcert5req);
+    test_tls_discard_cert(&clientcert6req);
+    test_tls_discard_cert(&clientcert7req);
+    test_tls_discard_cert(&clientcert8req);
+    test_tls_discard_cert(&clientcert9req);
+    test_tls_discard_cert(&clientcert10req);
+
+    test_tls_discard_cert(&cacertexpreq);
+    test_tls_discard_cert(&servercertexpreq);
+    test_tls_discard_cert(&servercertexp1req);
+    test_tls_discard_cert(&clientcertexp1req);
+
+    test_tls_discard_cert(&cacertnewreq);
+    test_tls_discard_cert(&servercertnewreq);
+    test_tls_discard_cert(&servercertnew1req);
+    test_tls_discard_cert(&clientcertnew1req);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-ctx.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/qemu/tests/test-crypto-tlssession.c b/qemu/tests/test-crypto-tlssession.c
new file mode 100644
index 000000000..1a4a066d7
--- /dev/null
+++ b/qemu/tests/test-crypto-tlssession.c
@@ -0,0 +1,534 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlssession.h"
+#include "qom/object_interfaces.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "qemu/acl.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlssession-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSSessionTestData {
+    const char *servercacrt;
+    const char *clientcacrt;
+    const char *servercrt;
+    const char *clientcrt;
+    bool expectServerFail;
+    bool expectClientFail;
+    const char *hostname;
+    const char *const *wildcards;
+};
+
+
+static ssize_t testWrite(const char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return write(*fd, buf, len);
+}
+
+static ssize_t testRead(char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return read(*fd, buf, len);
+}
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+         "testtlscredsserver" : "testtlscredsclient"),
+        &err,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        /* We skip initial sanity checks here because we
+         * want to make sure that problems are being
+         * detected at the TLS session validation stage,
+         * and the test-crypto-tlscreds test already
+         * validate the sanity check code.
+         */
+        "sanity-check", "no",
+        NULL
+        );
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+/*
+ * This tests validation checking of peer certificates
+ *
+ * This is replicating the checks that are done for an
+ * active TLS session after handshake completes. To
+ * simulate that we create our TLS contexts, skipping
+ * sanity checks. We then get a socketpair, and
+ * initiate a TLS session across them. Finally do
+ * do actual cert validation tests
+ */
+static void test_crypto_tls_session(const void *opaque)
+{
+    struct QCryptoTLSSessionTestData *data =
+        (struct QCryptoTLSSessionTestData *)opaque;
+    QCryptoTLSCreds *clientCreds;
+    QCryptoTLSCreds *serverCreds;
+    QCryptoTLSSession *clientSess = NULL;
+    QCryptoTLSSession *serverSess = NULL;
+    qemu_acl *acl;
+    const char * const *wildcards;
+    int channel[2];
+    bool clientShake = false;
+    bool serverShake = false;
+    Error *err = NULL;
+    int ret;
+
+    /* We'll use this for our fake client-server connection */
+    ret = socketpair(AF_UNIX, SOCK_STREAM, 0, channel);
+    g_assert(ret == 0);
+
+    /*
+     * We have an evil loop to do the handshake in a single
+     * thread, so we need these non-blocking to avoid deadlock
+     * of ourselves
+     */
+    qemu_set_nonblock(channel[0]);
+    qemu_set_nonblock(channel[1]);
+
+#define CLIENT_CERT_DIR "tests/test-crypto-tlssession-client/"
+#define SERVER_CERT_DIR "tests/test-crypto-tlssession-server/"
+    mkdir(CLIENT_CERT_DIR, 0700);
+    mkdir(SERVER_CERT_DIR, 0700);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    g_assert(link(data->servercacrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->servercrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+
+    g_assert(link(data->clientcacrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->clientcrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+
+    clientCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        CLIENT_CERT_DIR,
+        &err);
+    g_assert(clientCreds != NULL);
+
+    serverCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        SERVER_CERT_DIR,
+        &err);
+    g_assert(serverCreds != NULL);
+
+    acl = qemu_acl_init("tlssessionacl");
+    qemu_acl_reset(acl);
+    wildcards = data->wildcards;
+    while (wildcards && *wildcards) {
+        qemu_acl_append(acl, 0, *wildcards);
+        wildcards++;
+    }
+
+    /* Now the real part of the test, setup the sessions */
+    clientSess = qcrypto_tls_session_new(
+        clientCreds, data->hostname, NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, &err);
+    serverSess = qcrypto_tls_session_new(
+        serverCreds, NULL,
+        data->wildcards ? "tlssessionacl" : NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, &err);
+
+    g_assert(clientSess != NULL);
+    g_assert(serverSess != NULL);
+
+    /* For handshake to work, we need to set the I/O callbacks
+     * to read/write over the socketpair
+     */
+    qcrypto_tls_session_set_callbacks(serverSess,
+                                      testWrite, testRead,
+                                      &channel[0]);
+    qcrypto_tls_session_set_callbacks(clientSess,
+                                      testWrite, testRead,
+                                      &channel[1]);
+
+    /*
+     * Finally we loop around & around doing handshake on each
+     * session until we get an error, or the handshake completes.
+     * This relies on the socketpair being nonblocking to avoid
+     * deadlocking ourselves upon handshake
+     */
+    do {
+        int rv;
+        if (!serverShake) {
+            rv = qcrypto_tls_session_handshake(serverSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(serverSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                serverShake = true;
+            }
+        }
+        if (!clientShake) {
+            rv = qcrypto_tls_session_handshake(clientSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(clientSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                clientShake = true;
+            }
+        }
+    } while (!clientShake && !serverShake);
+
+
+    /* Finally make sure the server validation does what
+     * we were expecting
+     */
+    if (qcrypto_tls_session_check_credentials(serverSess, &err) < 0) {
+        g_assert(data->expectServerFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectServerFail);
+    }
+
+    /*
+     * And the same for the client validation check
+     */
+    if (qcrypto_tls_session_check_credentials(clientSess, &err) < 0) {
+        g_assert(data->expectClientFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectClientFail);
+    }
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    rmdir(CLIENT_CERT_DIR);
+    rmdir(SERVER_CERT_DIR);
+
+    object_unparent(OBJECT(serverCreds));
+    object_unparent(OBJECT(clientCreds));
+
+    qcrypto_tls_session_free(serverSess);
+    qcrypto_tls_session_free(clientSess);
+
+    close(channel[0]);
+    close(channel[1]);
+}
+
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TEST_SESS_REG(name, caCrt,                                     \
+                       serverCrt, clientCrt,                            \
+                       expectServerFail, expectClientFail,              \
+                       hostname, wildcards)                             \
+    struct QCryptoTLSSessionTestData name = {                           \
+        caCrt, caCrt, serverCrt, clientCrt,                             \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+
+# define TEST_SESS_REG_EXT(name, serverCaCrt, clientCaCrt,              \
+                           serverCrt, clientCrt,                        \
+                           expectServerFail, expectClientFail,          \
+                           hostname, wildcards)                         \
+    struct QCryptoTLSSessionTestData name = {                           \
+        serverCaCrt, clientCaCrt, serverCrt, clientCrt,                 \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_ROOT_REQ(altcacertreq,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(clientcertaltreq, altcacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(basicca, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG_EXT(differentca, cacertreq.filename,
+                      altcacertreq.filename, servercertreq.filename,
+                      clientcertaltreq.filename, true, true, "qemu.org", NULL);
+
+
+    /* When an altname is set, the CN is ignored, so it must be duplicated
+     * as an altname for it to match */
+    TLS_CERT_REQ(servercertalt1req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* This intentionally doesn't replicate */
+    TLS_CERT_REQ(servercertalt2req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "wiki.qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(altname1, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG(altname2, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname3, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, true, "wiki.qemu.org", NULL);
+
+    TEST_SESS_REG(altname4, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, true, "qemu.org", NULL);
+    TEST_SESS_REG(altname5, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname6, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "wiki.qemu.org", NULL);
+
+    const char *const wildcards1[] = {
+        "C=UK,CN=dogfood",
+        NULL,
+    };
+    const char *const wildcards2[] = {
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards3[] = {
+        "C=UK,CN=dogfood",
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards4[] = {
+        "C=UK,CN=qemustuff",
+        NULL,
+    };
+    const char *const wildcards5[] = {
+        "C=UK,CN=qemu*",
+        NULL,
+    };
+    const char *const wildcards6[] = {
+        "C=UK,CN=*emu*",
+        NULL,
+    };
+
+    TEST_SESS_REG(wildcard1, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards1);
+    TEST_SESS_REG(wildcard2, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards2);
+    TEST_SESS_REG(wildcard3, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards3);
+    TEST_SESS_REG(wildcard4, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards4);
+    TEST_SESS_REG(wildcard5, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards5);
+    TEST_SESS_REG(wildcard6, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards6);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-sess.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TEST_SESS_REG(cachain, WORKDIR "cacertchain-sess.pem",
+                  servercertlevel3areq.filename, clientcertlevel2breq.filename,
+                  false, false, "qemu.org", NULL);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcertaltreq);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercertalt1req);
+    test_tls_discard_cert(&servercertalt2req);
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&altcacertreq);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-sess.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/qemu/tests/test-crypto-xts.c b/qemu/tests/test-crypto-xts.c
new file mode 100644
index 000000000..7f68b063c
--- /dev/null
+++ b/qemu/tests/test-crypto-xts.c
@@ -0,0 +1,423 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/init.h"
+#include "crypto/xts.h"
+#include "crypto/aes.h"
+
+typedef struct {
+    const char *path;
+    int keylen;
+    unsigned char key1[32];
+    unsigned char key2[32];
+    uint64_t seqnum;
+    unsigned long PTLEN;
+    unsigned char PTX[512], CTX[512];
+} QCryptoXTSTestData;
+
+static const QCryptoXTSTestData test_data[] = {
+    /* #1 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-1-key-32-ptx-32",
+        32,
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        0,
+        32,
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        { 0x91, 0x7c, 0xf6, 0x9e, 0xbd, 0x68, 0xb2, 0xec,
+          0x9b, 0x9f, 0xe9, 0xa3, 0xea, 0xdd, 0xa6, 0x92,
+          0xcd, 0x43, 0xd2, 0xf5, 0x95, 0x98, 0xed, 0x85,
+          0x8c, 0x02, 0xc2, 0x65, 0x2f, 0xbf, 0x92, 0x2e },
+    },
+
+    /* #2, 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-2-key-32-ptx-32",
+        32,
+        { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11,
+          0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
+        { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,
+          0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22 },
+        0x3333333333LL,
+        32,
+        { 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44 },
+        { 0xc4, 0x54, 0x18, 0x5e, 0x6a, 0x16, 0x93, 0x6e,
+          0x39, 0x33, 0x40, 0x38, 0xac, 0xef, 0x83, 0x8b,
+          0xfb, 0x18, 0x6f, 0xff, 0x74, 0x80, 0xad, 0xc4,
+          0x28, 0x93, 0x82, 0xec, 0xd6, 0xd3, 0x94, 0xf0 },
+    },
+
+    /* #5 from xts.7, 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-5-key-32-ptx-32",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        32,
+        { 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44 },
+        { 0xb0, 0x1f, 0x86, 0xf8, 0xed, 0xc1, 0x86, 0x37,
+          0x06, 0xfa, 0x8a, 0x42, 0x53, 0xe3, 0x4f, 0x28,
+          0xaf, 0x31, 0x9d, 0xe3, 0x83, 0x34, 0x87, 0x0f,
+          0x4d, 0xd1, 0xf9, 0x4c, 0xbe, 0x98, 0x32, 0xf1 },
+    },
+
+    /* #4, 32 byte key, 512 byte PTX  */
+    {
+        "/crypto/xts/t-4-key-32-ptx-512",
+        32,
+        { 0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45,
+          0x23, 0x53, 0x60, 0x28, 0x74, 0x71, 0x35, 0x26 },
+        { 0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93,
+          0x23, 0x84, 0x62, 0x64, 0x33, 0x83, 0x27, 0x95 },
+        0,
+        512,
+        {
+            0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+            0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+            0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+            0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+            0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+            0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+            0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+            0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+            0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+            0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+            0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+            0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+            0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+            0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+            0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+            0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+            0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+            0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+            0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+            0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+            0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+            0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+            0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+            0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+            0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+            0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+            0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+            0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+            0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+            0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+            0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+            0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+            0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+            0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+            0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+            0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+            0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+            0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+            0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+            0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+            0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+            0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+            0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+            0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+            0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+            0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+            0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+            0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+            0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+            0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+            0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+            0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+            0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+            0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+            0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+            0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+            0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+            0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+            0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+            0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+            0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+            0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+        },
+        {
+            0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
+            0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
+            0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
+            0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
+            0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
+            0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
+            0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
+            0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
+            0x32, 0x80, 0x63, 0xfd, 0x2a, 0xab, 0x53, 0xe5,
+            0xea, 0x1e, 0x0a, 0x9f, 0x33, 0x25, 0x00, 0xa5,
+            0xdf, 0x94, 0x87, 0xd0, 0x7a, 0x5c, 0x92, 0xcc,
+            0x51, 0x2c, 0x88, 0x66, 0xc7, 0xe8, 0x60, 0xce,
+            0x93, 0xfd, 0xf1, 0x66, 0xa2, 0x49, 0x12, 0xb4,
+            0x22, 0x97, 0x61, 0x46, 0xae, 0x20, 0xce, 0x84,
+            0x6b, 0xb7, 0xdc, 0x9b, 0xa9, 0x4a, 0x76, 0x7a,
+            0xae, 0xf2, 0x0c, 0x0d, 0x61, 0xad, 0x02, 0x65,
+            0x5e, 0xa9, 0x2d, 0xc4, 0xc4, 0xe4, 0x1a, 0x89,
+            0x52, 0xc6, 0x51, 0xd3, 0x31, 0x74, 0xbe, 0x51,
+            0xa1, 0x0c, 0x42, 0x11, 0x10, 0xe6, 0xd8, 0x15,
+            0x88, 0xed, 0xe8, 0x21, 0x03, 0xa2, 0x52, 0xd8,
+            0xa7, 0x50, 0xe8, 0x76, 0x8d, 0xef, 0xff, 0xed,
+            0x91, 0x22, 0x81, 0x0a, 0xae, 0xb9, 0x9f, 0x91,
+            0x72, 0xaf, 0x82, 0xb6, 0x04, 0xdc, 0x4b, 0x8e,
+            0x51, 0xbc, 0xb0, 0x82, 0x35, 0xa6, 0xf4, 0x34,
+            0x13, 0x32, 0xe4, 0xca, 0x60, 0x48, 0x2a, 0x4b,
+            0xa1, 0xa0, 0x3b, 0x3e, 0x65, 0x00, 0x8f, 0xc5,
+            0xda, 0x76, 0xb7, 0x0b, 0xf1, 0x69, 0x0d, 0xb4,
+            0xea, 0xe2, 0x9c, 0x5f, 0x1b, 0xad, 0xd0, 0x3c,
+            0x5c, 0xcf, 0x2a, 0x55, 0xd7, 0x05, 0xdd, 0xcd,
+            0x86, 0xd4, 0x49, 0x51, 0x1c, 0xeb, 0x7e, 0xc3,
+            0x0b, 0xf1, 0x2b, 0x1f, 0xa3, 0x5b, 0x91, 0x3f,
+            0x9f, 0x74, 0x7a, 0x8a, 0xfd, 0x1b, 0x13, 0x0e,
+            0x94, 0xbf, 0xf9, 0x4e, 0xff, 0xd0, 0x1a, 0x91,
+            0x73, 0x5c, 0xa1, 0x72, 0x6a, 0xcd, 0x0b, 0x19,
+            0x7c, 0x4e, 0x5b, 0x03, 0x39, 0x36, 0x97, 0xe1,
+            0x26, 0x82, 0x6f, 0xb6, 0xbb, 0xde, 0x8e, 0xcc,
+            0x1e, 0x08, 0x29, 0x85, 0x16, 0xe2, 0xc9, 0xed,
+            0x03, 0xff, 0x3c, 0x1b, 0x78, 0x60, 0xf6, 0xde,
+            0x76, 0xd4, 0xce, 0xcd, 0x94, 0xc8, 0x11, 0x98,
+            0x55, 0xef, 0x52, 0x97, 0xca, 0x67, 0xe9, 0xf3,
+            0xe7, 0xff, 0x72, 0xb1, 0xe9, 0x97, 0x85, 0xca,
+            0x0a, 0x7e, 0x77, 0x20, 0xc5, 0xb3, 0x6d, 0xc6,
+            0xd7, 0x2c, 0xac, 0x95, 0x74, 0xc8, 0xcb, 0xbc,
+            0x2f, 0x80, 0x1e, 0x23, 0xe5, 0x6f, 0xd3, 0x44,
+            0xb0, 0x7f, 0x22, 0x15, 0x4b, 0xeb, 0xa0, 0xf0,
+            0x8c, 0xe8, 0x89, 0x1e, 0x64, 0x3e, 0xd9, 0x95,
+            0xc9, 0x4d, 0x9a, 0x69, 0xc9, 0xf1, 0xb5, 0xf4,
+            0x99, 0x02, 0x7a, 0x78, 0x57, 0x2a, 0xee, 0xbd,
+            0x74, 0xd2, 0x0c, 0xc3, 0x98, 0x81, 0xc2, 0x13,
+            0xee, 0x77, 0x0b, 0x10, 0x10, 0xe4, 0xbe, 0xa7,
+            0x18, 0x84, 0x69, 0x77, 0xae, 0x11, 0x9f, 0x7a,
+            0x02, 0x3a, 0xb5, 0x8c, 0xca, 0x0a, 0xd7, 0x52,
+            0xaf, 0xe6, 0x56, 0xbb, 0x3c, 0x17, 0x25, 0x6a,
+            0x9f, 0x6e, 0x9b, 0xf1, 0x9f, 0xdd, 0x5a, 0x38,
+            0xfc, 0x82, 0xbb, 0xe8, 0x72, 0xc5, 0x53, 0x9e,
+            0xdb, 0x60, 0x9e, 0xf4, 0xf7, 0x9c, 0x20, 0x3e,
+            0xbb, 0x14, 0x0f, 0x2e, 0x58, 0x3c, 0xb2, 0xad,
+            0x15, 0xb4, 0xaa, 0x5b, 0x65, 0x50, 0x16, 0xa8,
+            0x44, 0x92, 0x77, 0xdb, 0xd4, 0x77, 0xef, 0x2c,
+            0x8d, 0x6c, 0x01, 0x7d, 0xb7, 0x38, 0xb1, 0x8d,
+            0xeb, 0x4a, 0x42, 0x7d, 0x19, 0x23, 0xce, 0x3f,
+            0xf2, 0x62, 0x73, 0x57, 0x79, 0xa4, 0x18, 0xf2,
+            0x0a, 0x28, 0x2d, 0xf9, 0x20, 0x14, 0x7b, 0xea,
+            0xbe, 0x42, 0x1e, 0xe5, 0x31, 0x9d, 0x05, 0x68,
+        }
+    },
+
+    /* #7, 32 byte key, 17 byte PTX */
+    {
+        "/crypto/xts/t-7-key-32-ptx-17",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        17,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+        { 0x6c, 0x16, 0x25, 0xdb, 0x46, 0x71, 0x52, 0x2d,
+          0x3d, 0x75, 0x99, 0x60, 0x1d, 0xe7, 0xca, 0x09, 0xed },
+    },
+
+    /* #15, 32 byte key, 25 byte PTX */
+    {
+        "/crypto/xts/t-15-key-32-ptx-25",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        25,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+          0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18 },
+        { 0x8f, 0x4d, 0xcb, 0xad, 0x55, 0x55, 0x8d, 0x7b,
+          0x4e, 0x01, 0xd9, 0x37, 0x9c, 0xd4, 0xea, 0x22,
+          0xed, 0xbf, 0x9d, 0xac, 0xe4, 0x5d, 0x6f, 0x6a, 0x73 },
+    },
+
+    /* #21, 32 byte key, 31 byte PTX */
+    {
+        "/crypto/xts/t-21-key-32-ptx-31",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        31,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+          0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+          0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e },
+        { 0xd0, 0x5b, 0xc0, 0x90, 0xa8, 0xe0, 0x4f, 0x1b,
+          0x3d, 0x3e, 0xcd, 0xd5, 0xba, 0xec, 0x0f, 0xd4,
+          0xed, 0xbf, 0x9d, 0xac, 0xe4, 0x5d, 0x6f, 0x6a,
+          0x73, 0x06, 0xe6, 0x4b, 0xe5, 0xdd, 0x82 },
+    },
+};
+
+#define STORE64L(x, y)                                                  \
+    do {                                                                \
+        (y)[7] = (unsigned char)(((x) >> 56) & 255);                    \
+        (y)[6] = (unsigned char)(((x) >> 48) & 255);                    \
+        (y)[5] = (unsigned char)(((x) >> 40) & 255);                    \
+        (y)[4] = (unsigned char)(((x) >> 32) & 255);                    \
+        (y)[3] = (unsigned char)(((x) >> 24) & 255);                    \
+        (y)[2] = (unsigned char)(((x) >> 16) & 255);                    \
+        (y)[1] = (unsigned char)(((x) >> 8) & 255);                     \
+        (y)[0] = (unsigned char)((x) & 255);                            \
+    } while (0)
+
+struct TestAES {
+    AES_KEY enc;
+    AES_KEY dec;
+};
+
+static void test_xts_aes_encrypt(const void *ctx,
+                                 size_t length,
+                                 uint8_t *dst,
+                                 const uint8_t *src)
+{
+    const struct TestAES *aesctx = ctx;
+
+    AES_encrypt(src, dst, &aesctx->enc);
+}
+
+
+static void test_xts_aes_decrypt(const void *ctx,
+                                 size_t length,
+                                 uint8_t *dst,
+                                 const uint8_t *src)
+{
+    const struct TestAES *aesctx = ctx;
+
+    AES_decrypt(src, dst, &aesctx->dec);
+}
+
+
+static void test_xts(const void *opaque)
+{
+    const QCryptoXTSTestData *data = opaque;
+    unsigned char OUT[512], Torg[16], T[16];
+    uint64_t seq;
+    int j;
+    unsigned long len;
+    struct TestAES aesdata;
+    struct TestAES aestweak;
+
+    for (j = 0; j < 2; j++) {
+        /* skip the cases where
+         * the length is smaller than 2*blocklen
+         * or the length is not a multiple of 32
+         */
+        if ((j == 1) && ((data->PTLEN < 32) || (data->PTLEN % 32))) {
+            continue;
+        }
+        len = data->PTLEN / 2;
+
+        AES_set_encrypt_key(data->key1, data->keylen / 2 * 8, &aesdata.enc);
+        AES_set_decrypt_key(data->key1, data->keylen / 2 * 8, &aesdata.dec);
+        AES_set_encrypt_key(data->key2, data->keylen / 2 * 8, &aestweak.enc);
+        AES_set_decrypt_key(data->key2, data->keylen / 2 * 8, &aestweak.dec);
+
+        seq = data->seqnum;
+        STORE64L(seq, Torg);
+        memset(Torg + 8, 0, 8);
+
+        memcpy(T, Torg, sizeof(T));
+        if (j == 0) {
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, data->PTLEN, OUT, data->PTX);
+        } else {
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, OUT, data->PTX);
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, &OUT[len], &data->PTX[len]);
+        }
+
+        g_assert(memcmp(OUT, data->CTX, data->PTLEN) == 0);
+
+        memcpy(T, Torg, sizeof(T));
+        if (j == 0) {
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, data->PTLEN, OUT, data->CTX);
+        } else {
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, OUT, data->CTX);
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, &OUT[len], &data->CTX[len]);
+        }
+
+        g_assert(memcmp(OUT, data->PTX, data->PTLEN) == 0);
+    }
+}
+
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        g_test_add_data_func(test_data[i].path, &test_data[i], test_xts);
+    }
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-cutils.c b/qemu/tests/test-cutils.c
index 2a4556d3a..fb8f5b532 100644
--- a/qemu/tests/test-cutils.c
+++ b/qemu/tests/test-cutils.c
@@ -25,12 +25,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <errno.h>
-#include <string.h>
-
-#include "qemu-common.h"
 
+#include "qemu/cutils.h"
 
 static void test_parse_uint_null(void)
 {
@@ -226,6 +224,1212 @@ static void test_parse_uint_full_correct(void)
     g_assert_cmpint(i, ==, 123);
 }
 
+static void test_qemu_strtol_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtol_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtol_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err  = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_full_correct(void)
+{
+    const char *str = "123";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtol_full_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_full_empty(void)
+{
+    const char *str = "";
+    long res = 999L;
+    int err;
+
+    err =  qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_negative(void)
+{
+    const char *str = " \t -321";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtol_full_trailing(void)
+{
+    const char *str = "123xxx";
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+}
+
+static void test_qemu_strtoul_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoul_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoul_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoul_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err  = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_full_correct(void)
+{
+    const char *str = "123";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoul_full_null(void)
+{
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_empty(void)
+{
+    const char *str = "";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+static void test_qemu_strtoul_full_negative(void)
+{
+    const char *str = " \t -321";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+}
+
+static void test_qemu_strtoul_full_trailing(void)
+{
+    const char *str = "123xxx";
+    unsigned long res;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+}
+
+static void test_qemu_strtoll_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoll_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoll_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoll_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_max(void)
+{
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err  = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_full_correct(void)
+{
+    const char *str = "123";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoll_full_null(void)
+{
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_empty(void)
+{
+    const char *str = "";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_negative(void)
+{
+    const char *str = " \t -321";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtoll_full_trailing(void)
+{
+    const char *str = "123xxx";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_max(void)
+{
+
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    int64_t res;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+}
+
+static void test_qemu_strtoull_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoull_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoull_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoull_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_max(void)
+{
+    const char *str = g_strdup_printf("%llu", ULLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err  = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_full_correct(void)
+{
+    const char *str = "18446744073709551614";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551614LLU);
+}
+
+static void test_qemu_strtoull_full_null(void)
+{
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_empty(void)
+{
+    const char *str = "";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_negative(void)
+{
+    const char *str = " \t -321";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551295LLU);
+}
+
+static void test_qemu_strtoull_full_trailing(void)
+{
+    const char *str = "18446744073709551614xxxxxx";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_max(void)
+{
+    const char *str = g_strdup_printf("%lld", ULLONG_MAX);
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+}
+
+static void test_qemu_strtosz_simple(void)
+{
+    const char *str = "12345M";
+    char *endptr = NULL;
+    int64_t res;
+
+    res = qemu_strtosz(str, &endptr);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+    g_assert(endptr == str + 6);
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_units(void)
+{
+    const char *none = "1";
+    const char *b = "1B";
+    const char *k = "1K";
+    const char *m = "1M";
+    const char *g = "1G";
+    const char *t = "1T";
+    const char *p = "1P";
+    const char *e = "1E";
+    int64_t res;
+
+    /* default is M */
+    res = qemu_strtosz(none, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(b, NULL);
+    g_assert_cmpint(res, ==, 1);
+
+    res = qemu_strtosz(k, NULL);
+    g_assert_cmpint(res, ==, K_BYTE);
+
+    res = qemu_strtosz(m, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(g, NULL);
+    g_assert_cmpint(res, ==, G_BYTE);
+
+    res = qemu_strtosz(t, NULL);
+    g_assert_cmpint(res, ==, T_BYTE);
+
+    res = qemu_strtosz(p, NULL);
+    g_assert_cmpint(res, ==, P_BYTE);
+
+    res = qemu_strtosz(e, NULL);
+    g_assert_cmpint(res, ==, E_BYTE);
+}
+
+static void test_qemu_strtosz_float(void)
+{
+    const char *str = "12.345M";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12.345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_erange(void)
+{
+    const char *str = "10E";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, -ERANGE);
+}
+
+static void test_qemu_strtosz_suffix_unit(void)
+{
+    const char *str = "12345";
+    int64_t res;
+
+    res = qemu_strtosz_suffix_unit(str, NULL,
+                                   QEMU_STRTOSZ_DEFSUFFIX_KB, 1000);
+    g_assert_cmpint(res, ==, 12345000);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -247,5 +1451,145 @@ int main(int argc, char **argv)
     g_test_add_func("/cutils/parse_uint_full/correct",
                     test_parse_uint_full_correct);
 
+    /* qemu_strtol() tests */
+    g_test_add_func("/cutils/qemu_strtol/correct", test_qemu_strtol_correct);
+    g_test_add_func("/cutils/qemu_strtol/null", test_qemu_strtol_null);
+    g_test_add_func("/cutils/qemu_strtol/empty", test_qemu_strtol_empty);
+    g_test_add_func("/cutils/qemu_strtol/whitespace",
+                    test_qemu_strtol_whitespace);
+    g_test_add_func("/cutils/qemu_strtol/invalid", test_qemu_strtol_invalid);
+    g_test_add_func("/cutils/qemu_strtol/trailing", test_qemu_strtol_trailing);
+    g_test_add_func("/cutils/qemu_strtol/octal", test_qemu_strtol_octal);
+    g_test_add_func("/cutils/qemu_strtol/decimal", test_qemu_strtol_decimal);
+    g_test_add_func("/cutils/qemu_strtol/hex", test_qemu_strtol_hex);
+    g_test_add_func("/cutils/qemu_strtol/max", test_qemu_strtol_max);
+    g_test_add_func("/cutils/qemu_strtol/overflow", test_qemu_strtol_overflow);
+    g_test_add_func("/cutils/qemu_strtol/underflow",
+                    test_qemu_strtol_underflow);
+    g_test_add_func("/cutils/qemu_strtol/negative", test_qemu_strtol_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/correct",
+                    test_qemu_strtol_full_correct);
+    g_test_add_func("/cutils/qemu_strtol_full/null",
+                    test_qemu_strtol_full_null);
+    g_test_add_func("/cutils/qemu_strtol_full/empty",
+                    test_qemu_strtol_full_empty);
+    g_test_add_func("/cutils/qemu_strtol_full/negative",
+                    test_qemu_strtol_full_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/trailing",
+                    test_qemu_strtol_full_trailing);
+    g_test_add_func("/cutils/qemu_strtol_full/max",
+                    test_qemu_strtol_full_max);
+
+    /* qemu_strtoul() tests */
+    g_test_add_func("/cutils/qemu_strtoul/correct", test_qemu_strtoul_correct);
+    g_test_add_func("/cutils/qemu_strtoul/null", test_qemu_strtoul_null);
+    g_test_add_func("/cutils/qemu_strtoul/empty", test_qemu_strtoul_empty);
+    g_test_add_func("/cutils/qemu_strtoul/whitespace",
+                    test_qemu_strtoul_whitespace);
+    g_test_add_func("/cutils/qemu_strtoul/invalid", test_qemu_strtoul_invalid);
+    g_test_add_func("/cutils/qemu_strtoul/trailing",
+                    test_qemu_strtoul_trailing);
+    g_test_add_func("/cutils/qemu_strtoul/octal", test_qemu_strtoul_octal);
+    g_test_add_func("/cutils/qemu_strtoul/decimal", test_qemu_strtoul_decimal);
+    g_test_add_func("/cutils/qemu_strtoul/hex", test_qemu_strtoul_hex);
+    g_test_add_func("/cutils/qemu_strtoul/max", test_qemu_strtoul_max);
+    g_test_add_func("/cutils/qemu_strtoul/overflow",
+                    test_qemu_strtoul_overflow);
+    g_test_add_func("/cutils/qemu_strtoul/underflow",
+                    test_qemu_strtoul_underflow);
+    g_test_add_func("/cutils/qemu_strtoul/negative",
+                    test_qemu_strtoul_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/correct",
+                    test_qemu_strtoul_full_correct);
+    g_test_add_func("/cutils/qemu_strtoul_full/null",
+                    test_qemu_strtoul_full_null);
+    g_test_add_func("/cutils/qemu_strtoul_full/empty",
+                    test_qemu_strtoul_full_empty);
+    g_test_add_func("/cutils/qemu_strtoul_full/negative",
+                    test_qemu_strtoul_full_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/trailing",
+                    test_qemu_strtoul_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoul_full/max",
+                    test_qemu_strtoul_full_max);
+
+    /* qemu_strtoll() tests */
+    g_test_add_func("/cutils/qemu_strtoll/correct", test_qemu_strtoll_correct);
+    g_test_add_func("/cutils/qemu_strtoll/null", test_qemu_strtoll_null);
+    g_test_add_func("/cutils/qemu_strtoll/empty", test_qemu_strtoll_empty);
+    g_test_add_func("/cutils/qemu_strtoll/whitespace",
+                    test_qemu_strtoll_whitespace);
+    g_test_add_func("/cutils/qemu_strtoll/invalid", test_qemu_strtoll_invalid);
+    g_test_add_func("/cutils/qemu_strtoll/trailing",
+                    test_qemu_strtoll_trailing);
+    g_test_add_func("/cutils/qemu_strtoll/octal", test_qemu_strtoll_octal);
+    g_test_add_func("/cutils/qemu_strtoll/decimal", test_qemu_strtoll_decimal);
+    g_test_add_func("/cutils/qemu_strtoll/hex", test_qemu_strtoll_hex);
+    g_test_add_func("/cutils/qemu_strtoll/max", test_qemu_strtoll_max);
+    g_test_add_func("/cutils/qemu_strtoll/overflow",
+                    test_qemu_strtoll_overflow);
+    g_test_add_func("/cutils/qemu_strtoll/underflow",
+                    test_qemu_strtoll_underflow);
+    g_test_add_func("/cutils/qemu_strtoll/negative",
+                    test_qemu_strtoll_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/correct",
+                    test_qemu_strtoll_full_correct);
+    g_test_add_func("/cutils/qemu_strtoll_full/null",
+                    test_qemu_strtoll_full_null);
+    g_test_add_func("/cutils/qemu_strtoll_full/empty",
+                    test_qemu_strtoll_full_empty);
+    g_test_add_func("/cutils/qemu_strtoll_full/negative",
+                    test_qemu_strtoll_full_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/trailing",
+                    test_qemu_strtoll_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoll_full/max",
+                    test_qemu_strtoll_full_max);
+
+    /* qemu_strtoull() tests */
+    g_test_add_func("/cutils/qemu_strtoull/correct",
+                    test_qemu_strtoull_correct);
+    g_test_add_func("/cutils/qemu_strtoull/null",
+                    test_qemu_strtoull_null);
+    g_test_add_func("/cutils/qemu_strtoull/empty", test_qemu_strtoull_empty);
+    g_test_add_func("/cutils/qemu_strtoull/whitespace",
+                    test_qemu_strtoull_whitespace);
+    g_test_add_func("/cutils/qemu_strtoull/invalid",
+                    test_qemu_strtoull_invalid);
+    g_test_add_func("/cutils/qemu_strtoull/trailing",
+                    test_qemu_strtoull_trailing);
+    g_test_add_func("/cutils/qemu_strtoull/octal", test_qemu_strtoull_octal);
+    g_test_add_func("/cutils/qemu_strtoull/decimal",
+                    test_qemu_strtoull_decimal);
+    g_test_add_func("/cutils/qemu_strtoull/hex", test_qemu_strtoull_hex);
+    g_test_add_func("/cutils/qemu_strtoull/max", test_qemu_strtoull_max);
+    g_test_add_func("/cutils/qemu_strtoull/overflow",
+                    test_qemu_strtoull_overflow);
+    g_test_add_func("/cutils/qemu_strtoull/underflow",
+                    test_qemu_strtoull_underflow);
+    g_test_add_func("/cutils/qemu_strtoull/negative",
+                    test_qemu_strtoull_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/correct",
+                    test_qemu_strtoull_full_correct);
+    g_test_add_func("/cutils/qemu_strtoull_full/null",
+                    test_qemu_strtoull_full_null);
+    g_test_add_func("/cutils/qemu_strtoull_full/empty",
+                    test_qemu_strtoull_full_empty);
+    g_test_add_func("/cutils/qemu_strtoull_full/negative",
+                    test_qemu_strtoull_full_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/trailing",
+                    test_qemu_strtoull_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoull_full/max",
+                    test_qemu_strtoull_full_max);
+
+    g_test_add_func("/cutils/strtosz/simple",
+                    test_qemu_strtosz_simple);
+    g_test_add_func("/cutils/strtosz/units",
+                    test_qemu_strtosz_units);
+    g_test_add_func("/cutils/strtosz/float",
+                    test_qemu_strtosz_float);
+    g_test_add_func("/cutils/strtosz/erange",
+                    test_qemu_strtosz_erange);
+    g_test_add_func("/cutils/strtosz/suffix-unit",
+                    test_qemu_strtosz_suffix_unit);
+
     return g_test_run();
 }
diff --git a/qemu/tests/test-filter-mirror.c b/qemu/tests/test-filter-mirror.c
new file mode 100644
index 000000000..f60bf2adb
--- /dev/null
+++ b/qemu/tests/test-filter-mirror.c
@@ -0,0 +1,93 @@
+/*
+ * QTest testcase for filter-mirror
+ *
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include "libqtest.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+
+static void test_mirror(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int send_sock[2], recv_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello! filter-mirror~";
+    char sock_path[] = "filter-mirror.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, send_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                 "-device e1000,netdev=qtest-bn0,id=qtest-e0 "
+                 "-chardev socket,id=mirror0,path=%s,server,nowait "
+                 "-object filter-mirror,id=qtest-f0,netdev=qtest-bn0,queue=tx,outdev=mirror0 "
+                 , send_sock[1], sock_path);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    recv_sock = unix_connect(sock_path, NULL);
+    g_assert_cmpint(recv_sock, !=, -1);
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+    ret = iov_send(send_sock[0], iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(send_sock[0]);
+
+    ret = qemu_recv(recv_sock, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(recv_sock, recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    close(recv_sock);
+    unlink(sock_path);
+
+#endif
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/netfilter/mirror", test_mirror);
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/qemu/tests/test-filter-redirector.c b/qemu/tests/test-filter-redirector.c
new file mode 100644
index 000000000..b93012cea
--- /dev/null
+++ b/qemu/tests/test-filter-redirector.c
@@ -0,0 +1,221 @@
+/*
+ * QTest testcase for filter-redirector
+ *
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ *
+ * Case 1, tx traffic flow:
+ *
+ * qemu side              | test side
+ *                        |
+ * +---------+            |  +-------+
+ * | backend <---------------+ sock0 |
+ * +----+----+            |  +-------+
+ *      |                 |
+ * +----v----+  +-------+ |
+ * |  rd0    +->+chardev| |
+ * +---------+  +---+---+ |
+ *                  |     |
+ * +---------+      |     |
+ * |  rd1    <------+     |
+ * +----+----+            |
+ *      |                 |
+ * +----v----+            |  +-------+
+ * |  rd2    +--------------->sock1  |
+ * +---------+            |  +-------+
+ *                        +
+ *
+ * --------------------------------------
+ * Case 2, rx traffic flow
+ * qemu side              | test side
+ *                        |
+ * +---------+            |  +-------+
+ * | backend +---------------> sock1 |
+ * +----^----+            |  +-------+
+ *      |                 |
+ * +----+----+  +-------+ |
+ * |  rd0    +<-+chardev| |
+ * +---------+  +---+---+ |
+ *                  ^     |
+ * +---------+      |     |
+ * |  rd1    +------+     |
+ * +----^----+            |
+ *      |                 |
+ * +----+----+            |  +-------+
+ * |  rd2    <---------------+sock0  |
+ * +---------+            |  +-------+
+ *                        +
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include "libqtest.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+
+static void test_redirector_tx(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int backend_sock[2], recv_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello!!";
+    char sock_path0[] = "filter-redirector0.XXXXXX";
+    char sock_path1[] = "filter-redirector1.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, backend_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path0);
+    g_assert_cmpint(ret, !=, -1);
+    ret = mkstemp(sock_path1);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                "-device rtl8139,netdev=qtest-bn0,id=qtest-e0 "
+                "-chardev socket,id=redirector0,path=%s,server,nowait "
+                "-chardev socket,id=redirector1,path=%s,server,nowait "
+                "-chardev socket,id=redirector2,path=%s,nowait "
+                "-object filter-redirector,id=qtest-f0,netdev=qtest-bn0,"
+                "queue=tx,outdev=redirector0 "
+                "-object filter-redirector,id=qtest-f1,netdev=qtest-bn0,"
+                "queue=tx,indev=redirector2 "
+                "-object filter-redirector,id=qtest-f2,netdev=qtest-bn0,"
+                "queue=tx,outdev=redirector1 "
+                , backend_sock[1], sock_path0, sock_path1, sock_path0);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    recv_sock = unix_connect(sock_path1, NULL);
+    g_assert_cmpint(recv_sock, !=, -1);
+
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    ret = iov_send(backend_sock[0], iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(backend_sock[0]);
+
+    ret = qemu_recv(recv_sock, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(recv_sock, recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    close(recv_sock);
+    unlink(sock_path0);
+    unlink(sock_path1);
+    qtest_end();
+
+#endif
+}
+
+static void test_redirector_rx(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int backend_sock[2], send_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello!!";
+    char sock_path0[] = "filter-redirector0.XXXXXX";
+    char sock_path1[] = "filter-redirector1.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, backend_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path0);
+    g_assert_cmpint(ret, !=, -1);
+    ret = mkstemp(sock_path1);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                "-device rtl8139,netdev=qtest-bn0,id=qtest-e0 "
+                "-chardev socket,id=redirector0,path=%s,server,nowait "
+                "-chardev socket,id=redirector1,path=%s,server,nowait "
+                "-chardev socket,id=redirector2,path=%s,nowait "
+                "-object filter-redirector,id=qtest-f0,netdev=qtest-bn0,"
+                "queue=rx,indev=redirector0 "
+                "-object filter-redirector,id=qtest-f1,netdev=qtest-bn0,"
+                "queue=rx,outdev=redirector2 "
+                "-object filter-redirector,id=qtest-f2,netdev=qtest-bn0,"
+                "queue=rx,indev=redirector1 "
+                , backend_sock[1], sock_path0, sock_path1, sock_path0);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    send_sock = unix_connect(sock_path1, NULL);
+    g_assert_cmpint(send_sock, !=, -1);
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+
+    ret = iov_send(send_sock, iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(send_sock);
+
+    ret = qemu_recv(backend_sock[0], &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(backend_sock[0], recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    unlink(sock_path0);
+    unlink(sock_path1);
+    qtest_end();
+
+#endif
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+    qtest_add_func("/netfilter/redirector_tx", test_redirector_tx);
+    qtest_add_func("/netfilter/redirector_rx", test_redirector_rx);
+    ret = g_test_run();
+
+    return ret;
+}
diff --git a/qemu/tests/test-hbitmap.c b/qemu/tests/test-hbitmap.c
index 161eeb496..abe142791 100644
--- a/qemu/tests/test-hbitmap.c
+++ b/qemu/tests/test-hbitmap.c
@@ -9,10 +9,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <sys/types.h>
 #include "qemu/hbitmap.h"
 
 #define LOG_BITS_PER_LONG          (BITS_PER_LONG == 32 ? 5 : 6)
@@ -139,10 +137,8 @@ static void hbitmap_test_teardown(TestHBitmapData *data,
         hbitmap_free(data->hb);
         data->hb = NULL;
     }
-    if (data->bits) {
-        g_free(data->bits);
-        data->bits = NULL;
-    }
+    g_free(data->bits);
+    data->bits = NULL;
 }
 
 /* Set a range in the HBitmap and in the shadow "simple" bitmap.
diff --git a/qemu/tests/test-int128.c b/qemu/tests/test-int128.c
index 0772ef753..cacf6beac 100644
--- a/qemu/tests/test-int128.c
+++ b/qemu/tests/test-int128.c
@@ -6,10 +6,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdio.h>
 #include "qemu/int128.h"
-#include "qemu/osdep.h"
 
 /* clang doesn't support __noclone__ but it does have a mechanism for
  * telling us this. We assume that if we don't have __has_attribute()
diff --git a/qemu/tests/test-io-channel-buffer.c b/qemu/tests/test-io-channel-buffer.c
new file mode 100644
index 000000000..64722a214
--- /dev/null
+++ b/qemu/tests/test-io-channel-buffer.c
@@ -0,0 +1,51 @@
+/*
+ * QEMU I/O channel buffer test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-buffer.h"
+#include "io-channel-helpers.h"
+
+
+static void test_io_channel_buf(void)
+{
+    QIOChannelBuffer *buf;
+    QIOChannelTest *test;
+
+    buf = qio_channel_buffer_new(0);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_writer(test, QIO_CHANNEL(buf));
+    buf->offset = 0;
+    qio_channel_test_run_reader(test, QIO_CHANNEL(buf));
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(buf));
+}
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/io/channel/buf", test_io_channel_buf);
+    return g_test_run();
+}
diff --git a/qemu/tests/test-io-channel-command.c b/qemu/tests/test-io-channel-command.c
new file mode 100644
index 000000000..1d1f461be
--- /dev/null
+++ b/qemu/tests/test-io-channel-command.c
@@ -0,0 +1,131 @@
+/*
+ * QEMU I/O channel command test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-command.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+#ifndef WIN32
+static void test_io_channel_command_fifo(bool async)
+{
+#define TEST_FIFO "tests/test-io-channel-command.fifo"
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    char *srcfifo = g_strdup_printf("PIPE:%s,wronly", TEST_FIFO);
+    char *dstfifo = g_strdup_printf("PIPE:%s,rdonly", TEST_FIFO);
+    const char *srcargv[] = {
+        "/bin/socat", "-", srcfifo, NULL,
+    };
+    const char *dstargv[] = {
+        "/bin/socat", dstfifo, "-", NULL,
+    };
+
+    unlink(TEST_FIFO);
+    if (access("/bin/socat", X_OK) < 0) {
+        return; /* Pretend success if socat is not present */
+    }
+    if (mkfifo(TEST_FIFO, 0600) < 0) {
+        abort();
+    }
+    src = QIO_CHANNEL(qio_channel_command_new_spawn(srcargv,
+                                                    O_WRONLY,
+                                                    &error_abort));
+    dst = QIO_CHANNEL(qio_channel_command_new_spawn(dstargv,
+                                                    O_RDONLY,
+                                                    &error_abort));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, src, dst);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+
+    g_free(srcfifo);
+    g_free(dstfifo);
+    unlink(TEST_FIFO);
+}
+
+
+static void test_io_channel_command_fifo_async(void)
+{
+    test_io_channel_command_fifo(true);
+}
+
+static void test_io_channel_command_fifo_sync(void)
+{
+    test_io_channel_command_fifo(false);
+}
+
+
+static void test_io_channel_command_echo(bool async)
+{
+    QIOChannel *ioc;
+    QIOChannelTest *test;
+    const char *socatargv[] = {
+        "/bin/socat", "-", "-", NULL,
+    };
+
+    if (access("/bin/socat", X_OK) < 0) {
+        return; /* Pretend success if socat is not present */
+    }
+
+    ioc = QIO_CHANNEL(qio_channel_command_new_spawn(socatargv,
+                                                    O_RDWR,
+                                                    &error_abort));
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, ioc, ioc);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(ioc));
+}
+
+
+static void test_io_channel_command_echo_async(void)
+{
+    test_io_channel_command_echo(true);
+}
+
+static void test_io_channel_command_echo_sync(void)
+{
+    test_io_channel_command_echo(false);
+}
+#endif
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+#ifndef WIN32
+    g_test_add_func("/io/channel/command/fifo/sync",
+                    test_io_channel_command_fifo_sync);
+    g_test_add_func("/io/channel/command/fifo/async",
+                    test_io_channel_command_fifo_async);
+    g_test_add_func("/io/channel/command/echo/sync",
+                    test_io_channel_command_echo_sync);
+    g_test_add_func("/io/channel/command/echo/async",
+                    test_io_channel_command_echo_async);
+#endif
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-io-channel-file.c b/qemu/tests/test-io-channel-file.c
new file mode 100644
index 000000000..6bfede6bb
--- /dev/null
+++ b/qemu/tests/test-io-channel-file.c
@@ -0,0 +1,123 @@
+/*
+ * QEMU I/O channel file test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-file.h"
+#include "io/channel-util.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+static void test_io_channel_file(void)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+
+#define TEST_FILE "tests/test-io-channel-file.txt"
+    unlink(TEST_FILE);
+    src = QIO_CHANNEL(qio_channel_file_new_path(
+                          TEST_FILE,
+                          O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0600,
+                          &error_abort));
+    dst = QIO_CHANNEL(qio_channel_file_new_path(
+                          TEST_FILE,
+                          O_RDONLY | O_BINARY, 0,
+                          &error_abort));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_writer(test, src);
+    qio_channel_test_run_reader(test, dst);
+    qio_channel_test_validate(test);
+
+    unlink(TEST_FILE);
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+}
+
+
+static void test_io_channel_fd(void)
+{
+    QIOChannel *ioc;
+    int fd = -1;
+
+#define TEST_FILE "tests/test-io-channel-file.txt"
+    fd = open(TEST_FILE, O_CREAT | O_TRUNC | O_WRONLY, 0600);
+    g_assert_cmpint(fd, >, -1);
+
+    ioc = qio_channel_new_fd(fd, &error_abort);
+
+    g_assert_cmpstr(object_get_typename(OBJECT(ioc)),
+                    ==,
+                    TYPE_QIO_CHANNEL_FILE);
+
+    unlink(TEST_FILE);
+    object_unref(OBJECT(ioc));
+}
+
+
+#ifndef _WIN32
+static void test_io_channel_pipe(bool async)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    int fd[2];
+
+    if (pipe(fd) < 0) {
+        perror("pipe");
+        abort();
+    }
+
+    src = QIO_CHANNEL(qio_channel_file_new_fd(fd[1]));
+    dst = QIO_CHANNEL(qio_channel_file_new_fd(fd[0]));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, src, dst);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+}
+
+
+static void test_io_channel_pipe_async(void)
+{
+    test_io_channel_pipe(true);
+}
+
+static void test_io_channel_pipe_sync(void)
+{
+    test_io_channel_pipe(false);
+}
+#endif /* ! _WIN32 */
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/io/channel/file", test_io_channel_file);
+    g_test_add_func("/io/channel/file/fd", test_io_channel_fd);
+#ifndef _WIN32
+    g_test_add_func("/io/channel/pipe/sync", test_io_channel_pipe_sync);
+    g_test_add_func("/io/channel/pipe/async", test_io_channel_pipe_async);
+#endif
+    return g_test_run();
+}
diff --git a/qemu/tests/test-io-channel-socket.c b/qemu/tests/test-io-channel-socket.c
new file mode 100644
index 000000000..855306b8d
--- /dev/null
+++ b/qemu/tests/test-io-channel-socket.c
@@ -0,0 +1,568 @@
+/*
+ * QEMU I/O channel sockets test
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-socket.h"
+#include "io/channel-util.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+#ifndef AI_ADDRCONFIG
+# define AI_ADDRCONFIG 0
+#endif
+#ifndef EAI_ADDRFAMILY
+# define EAI_ADDRFAMILY 0
+#endif
+
+static int check_bind(const char *hostname, bool *has_proto)
+{
+    int fd = -1;
+    struct addrinfo ai, *res = NULL;
+    int rc;
+    int ret = -1;
+
+    memset(&ai, 0, sizeof(ai));
+    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
+    ai.ai_family = AF_UNSPEC;
+    ai.ai_socktype = SOCK_STREAM;
+
+    /* lookup */
+    rc = getaddrinfo(hostname, NULL, &ai, &res);
+    if (rc != 0) {
+        if (rc == EAI_ADDRFAMILY ||
+            rc == EAI_FAMILY) {
+            *has_proto = false;
+            goto done;
+        }
+        goto cleanup;
+    }
+
+    fd = qemu_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+    if (fd < 0) {
+        goto cleanup;
+    }
+
+    if (bind(fd, res->ai_addr, res->ai_addrlen) < 0) {
+        if (errno == EADDRNOTAVAIL) {
+            *has_proto = false;
+            goto done;
+        }
+        goto cleanup;
+    }
+
+    *has_proto = true;
+ done:
+    ret = 0;
+
+ cleanup:
+    if (fd != -1) {
+        close(fd);
+    }
+    if (res) {
+        freeaddrinfo(res);
+    }
+    return ret;
+}
+
+static int check_protocol_support(bool *has_ipv4, bool *has_ipv6)
+{
+    if (check_bind("127.0.0.1", has_ipv4) < 0) {
+        return -1;
+    }
+    if (check_bind("::1", has_ipv6) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static void test_io_channel_set_socket_bufs(QIOChannel *src,
+                                            QIOChannel *dst)
+{
+    int buflen = 64 * 1024;
+
+    /*
+     * Make the socket buffers small so that we see
+     * the effects of partial reads/writes
+     */
+    setsockopt(((QIOChannelSocket *)src)->fd,
+               SOL_SOCKET, SO_SNDBUF,
+               (char *)&buflen,
+               sizeof(buflen));
+
+    setsockopt(((QIOChannelSocket *)dst)->fd,
+               SOL_SOCKET, SO_SNDBUF,
+               (char *)&buflen,
+               sizeof(buflen));
+}
+
+
+static void test_io_channel_setup_sync(SocketAddress *listen_addr,
+                                       SocketAddress *connect_addr,
+                                       QIOChannel **src,
+                                       QIOChannel **dst)
+{
+    QIOChannelSocket *lioc;
+
+    lioc = qio_channel_socket_new();
+    qio_channel_socket_listen_sync(lioc, listen_addr, &error_abort);
+
+    if (listen_addr->type == SOCKET_ADDRESS_KIND_INET) {
+        SocketAddress *laddr = qio_channel_socket_get_local_address(
+            lioc, &error_abort);
+
+        g_free(connect_addr->u.inet.data->port);
+        connect_addr->u.inet.data->port = g_strdup(laddr->u.inet.data->port);
+
+        qapi_free_SocketAddress(laddr);
+    }
+
+    *src = QIO_CHANNEL(qio_channel_socket_new());
+    qio_channel_socket_connect_sync(
+        QIO_CHANNEL_SOCKET(*src), connect_addr, &error_abort);
+    qio_channel_set_delay(*src, false);
+
+    qio_channel_wait(QIO_CHANNEL(lioc), G_IO_IN);
+    *dst = QIO_CHANNEL(qio_channel_socket_accept(lioc, &error_abort));
+    g_assert(*dst);
+
+    test_io_channel_set_socket_bufs(*src, *dst);
+
+    object_unref(OBJECT(lioc));
+}
+
+
+struct TestIOChannelData {
+    bool err;
+    GMainLoop *loop;
+};
+
+
+static void test_io_channel_complete(Object *src,
+                                     Error *err,
+                                     gpointer opaque)
+{
+    struct TestIOChannelData *data = opaque;
+    data->err = err != NULL;
+    g_main_loop_quit(data->loop);
+}
+
+
+static void test_io_channel_setup_async(SocketAddress *listen_addr,
+                                        SocketAddress *connect_addr,
+                                        QIOChannel **src,
+                                        QIOChannel **dst)
+{
+    QIOChannelSocket *lioc;
+    struct TestIOChannelData data;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+
+    lioc = qio_channel_socket_new();
+    qio_channel_socket_listen_async(
+        lioc, listen_addr,
+        test_io_channel_complete, &data, NULL);
+
+    g_main_loop_run(data.loop);
+    g_main_context_iteration(g_main_context_default(), FALSE);
+
+    g_assert(!data.err);
+
+    if (listen_addr->type == SOCKET_ADDRESS_KIND_INET) {
+        SocketAddress *laddr = qio_channel_socket_get_local_address(
+            lioc, &error_abort);
+
+        g_free(connect_addr->u.inet.data->port);
+        connect_addr->u.inet.data->port = g_strdup(laddr->u.inet.data->port);
+
+        qapi_free_SocketAddress(laddr);
+    }
+
+    *src = QIO_CHANNEL(qio_channel_socket_new());
+
+    qio_channel_socket_connect_async(
+        QIO_CHANNEL_SOCKET(*src), connect_addr,
+        test_io_channel_complete, &data, NULL);
+
+    g_main_loop_run(data.loop);
+    g_main_context_iteration(g_main_context_default(), FALSE);
+
+    g_assert(!data.err);
+
+    qio_channel_wait(QIO_CHANNEL(lioc), G_IO_IN);
+    *dst = QIO_CHANNEL(qio_channel_socket_accept(lioc, &error_abort));
+    g_assert(*dst);
+
+    qio_channel_set_delay(*src, false);
+    test_io_channel_set_socket_bufs(*src, *dst);
+
+    object_unref(OBJECT(lioc));
+
+    g_main_loop_unref(data.loop);
+}
+
+
+static void test_io_channel(bool async,
+                            SocketAddress *listen_addr,
+                            SocketAddress *connect_addr,
+                            bool passFD)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    if (async) {
+        test_io_channel_setup_async(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, true, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+
+        test_io_channel_setup_async(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, false, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+    } else {
+        test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, true, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+
+        test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, false, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+    }
+}
+
+
+static void test_io_channel_ipv4(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_INET;
+    listen_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *listen_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("127.0.0.1"),
+        .port = NULL, /* Auto-select */
+    };
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_INET;
+    connect_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *connect_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("127.0.0.1"),
+        .port = NULL, /* Filled in later */
+    };
+
+    test_io_channel(async, listen_addr, connect_addr, false);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+}
+
+
+static void test_io_channel_ipv4_sync(void)
+{
+    return test_io_channel_ipv4(false);
+}
+
+
+static void test_io_channel_ipv4_async(void)
+{
+    return test_io_channel_ipv4(true);
+}
+
+
+static void test_io_channel_ipv6(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_INET;
+    listen_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *listen_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("::1"),
+        .port = NULL, /* Auto-select */
+    };
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_INET;
+    connect_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *connect_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("::1"),
+        .port = NULL, /* Filled in later */
+    };
+
+    test_io_channel(async, listen_addr, connect_addr, false);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+}
+
+
+static void test_io_channel_ipv6_sync(void)
+{
+    return test_io_channel_ipv6(false);
+}
+
+
+static void test_io_channel_ipv6_async(void)
+{
+    return test_io_channel_ipv6(true);
+}
+
+
+#ifndef _WIN32
+static void test_io_channel_unix(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+#define TEST_SOCKET "test-io-channel-socket.sock"
+    listen_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    listen_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    listen_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    connect_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    connect_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    test_io_channel(async, listen_addr, connect_addr, true);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+    unlink(TEST_SOCKET);
+}
+
+
+static void test_io_channel_unix_sync(void)
+{
+    return test_io_channel_unix(false);
+}
+
+
+static void test_io_channel_unix_async(void)
+{
+    return test_io_channel_unix(true);
+}
+
+static void test_io_channel_unix_fd_pass(void)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+    QIOChannel *src, *dst;
+    int testfd;
+    int fdsend[3];
+    int *fdrecv = NULL;
+    size_t nfdrecv = 0;
+    size_t i;
+    char bufsend[12], bufrecv[12];
+    struct iovec iosend[1], iorecv[1];
+
+#define TEST_SOCKET "test-io-channel-socket.sock"
+#define TEST_FILE "test-io-channel-socket.txt"
+
+    testfd = open(TEST_FILE, O_RDWR|O_TRUNC|O_CREAT, 0700);
+    g_assert(testfd != -1);
+    fdsend[0] = testfd;
+    fdsend[1] = testfd;
+    fdsend[2] = testfd;
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    listen_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    listen_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    connect_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    connect_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+    memcpy(bufsend, "Hello World", G_N_ELEMENTS(bufsend));
+
+    iosend[0].iov_base = bufsend;
+    iosend[0].iov_len = G_N_ELEMENTS(bufsend);
+
+    iorecv[0].iov_base = bufrecv;
+    iorecv[0].iov_len = G_N_ELEMENTS(bufrecv);
+
+    g_assert(qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+    g_assert(qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+    qio_channel_writev_full(src,
+                            iosend,
+                            G_N_ELEMENTS(iosend),
+                            fdsend,
+                            G_N_ELEMENTS(fdsend),
+                            &error_abort);
+
+    qio_channel_readv_full(dst,
+                           iorecv,
+                           G_N_ELEMENTS(iorecv),
+                           &fdrecv,
+                           &nfdrecv,
+                           &error_abort);
+
+    g_assert(nfdrecv == G_N_ELEMENTS(fdsend));
+    /* Each recvd FD should be different from sent FD */
+    for (i = 0; i < nfdrecv; i++) {
+        g_assert_cmpint(fdrecv[i], !=, testfd);
+    }
+    /* Each recvd FD should be different from each other */
+    g_assert_cmpint(fdrecv[0], !=, fdrecv[1]);
+    g_assert_cmpint(fdrecv[0], !=, fdrecv[2]);
+    g_assert_cmpint(fdrecv[1], !=, fdrecv[2]);
+
+    /* Check the I/O buf we sent at the same time matches */
+    g_assert(memcmp(bufsend, bufrecv, G_N_ELEMENTS(bufsend)) == 0);
+
+    /* Write some data into the FD we received */
+    g_assert(write(fdrecv[0], bufsend, G_N_ELEMENTS(bufsend)) ==
+             G_N_ELEMENTS(bufsend));
+
+    /* Read data from the original FD and make sure it matches */
+    memset(bufrecv, 0, G_N_ELEMENTS(bufrecv));
+    g_assert(lseek(testfd, 0, SEEK_SET) == 0);
+    g_assert(read(testfd, bufrecv, G_N_ELEMENTS(bufrecv)) ==
+             G_N_ELEMENTS(bufrecv));
+    g_assert(memcmp(bufsend, bufrecv, G_N_ELEMENTS(bufsend)) == 0);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+    unlink(TEST_SOCKET);
+    unlink(TEST_FILE);
+    close(testfd);
+    for (i = 0; i < nfdrecv; i++) {
+        close(fdrecv[i]);
+    }
+    g_free(fdrecv);
+}
+#endif /* _WIN32 */
+
+
+static void test_io_channel_ipv4_fd(void)
+{
+    QIOChannel *ioc;
+    int fd = -1;
+    struct sockaddr_in sa = {
+        .sin_family = AF_INET,
+        .sin_addr = {
+            .s_addr =  htonl(INADDR_LOOPBACK),
+        }
+        /* Leave port unset for auto-assign */
+    };
+    socklen_t salen = sizeof(sa);
+
+    fd = socket(AF_INET, SOCK_STREAM, 0);
+    g_assert_cmpint(fd, >, -1);
+
+    g_assert_cmpint(bind(fd, (struct sockaddr *)&sa, salen), ==, 0);
+
+    ioc = qio_channel_new_fd(fd, &error_abort);
+
+    g_assert_cmpstr(object_get_typename(OBJECT(ioc)),
+                    ==,
+                    TYPE_QIO_CHANNEL_SOCKET);
+
+    object_unref(OBJECT(ioc));
+}
+
+
+int main(int argc, char **argv)
+{
+    bool has_ipv4, has_ipv6;
+
+    module_call_init(MODULE_INIT_QOM);
+    socket_init();
+
+    g_test_init(&argc, &argv, NULL);
+
+    /* We're creating actual IPv4/6 sockets, so we should
+     * check if the host running tests actually supports
+     * each protocol to avoid breaking tests on machines
+     * with either IPv4 or IPv6 disabled.
+     */
+    if (check_protocol_support(&has_ipv4, &has_ipv6) < 0) {
+        return 1;
+    }
+
+    if (has_ipv4) {
+        g_test_add_func("/io/channel/socket/ipv4-sync",
+                        test_io_channel_ipv4_sync);
+        g_test_add_func("/io/channel/socket/ipv4-async",
+                        test_io_channel_ipv4_async);
+        g_test_add_func("/io/channel/socket/ipv4-fd",
+                        test_io_channel_ipv4_fd);
+    }
+    if (has_ipv6) {
+        g_test_add_func("/io/channel/socket/ipv6-sync",
+                        test_io_channel_ipv6_sync);
+        g_test_add_func("/io/channel/socket/ipv6-async",
+                        test_io_channel_ipv6_async);
+    }
+
+#ifndef _WIN32
+    g_test_add_func("/io/channel/socket/unix-sync",
+                    test_io_channel_unix_sync);
+    g_test_add_func("/io/channel/socket/unix-async",
+                    test_io_channel_unix_async);
+    g_test_add_func("/io/channel/socket/unix-fd-pass",
+                    test_io_channel_unix_fd_pass);
+#endif /* _WIN32 */
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-io-channel-tls.c b/qemu/tests/test-io-channel-tls.c
new file mode 100644
index 000000000..3c361a7be
--- /dev/null
+++ b/qemu/tests/test-io-channel-tls.c
@@ -0,0 +1,340 @@
+/*
+ * QEMU I/O channel TLS test
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "io/channel-tls.h"
+#include "io/channel-socket.h"
+#include "io-channel-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "qemu/acl.h"
+#include "qom/object_interfaces.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-io-channel-tls-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QIOChannelTLSTestData {
+    const char *servercacrt;
+    const char *clientcacrt;
+    const char *servercrt;
+    const char *clientcrt;
+    bool expectServerFail;
+    bool expectClientFail;
+    const char *hostname;
+    const char *const *wildcards;
+};
+
+struct QIOChannelTLSHandshakeData {
+    bool finished;
+    bool failed;
+};
+
+static void test_tls_handshake_done(Object *source,
+                                    Error *err,
+                                    gpointer opaque)
+{
+    struct QIOChannelTLSHandshakeData *data = opaque;
+
+    data->finished = true;
+    data->failed = err != NULL;
+}
+
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+         "testtlscredsserver" : "testtlscredsclient"),
+        errp,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        /* We skip initial sanity checks here because we
+         * want to make sure that problems are being
+         * detected at the TLS session validation stage,
+         * and the test-crypto-tlscreds test already
+         * validate the sanity check code.
+         */
+        "sanity-check", "no",
+        NULL
+        );
+
+    if (*errp) {
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+/*
+ * This tests validation checking of peer certificates
+ *
+ * This is replicating the checks that are done for an
+ * active TLS session after handshake completes. To
+ * simulate that we create our TLS contexts, skipping
+ * sanity checks. When then get a socketpair, and
+ * initiate a TLS session across them. Finally do
+ * do actual cert validation tests
+ */
+static void test_io_channel_tls(const void *opaque)
+{
+    struct QIOChannelTLSTestData *data =
+        (struct QIOChannelTLSTestData *)opaque;
+    QCryptoTLSCreds *clientCreds;
+    QCryptoTLSCreds *serverCreds;
+    QIOChannelTLS *clientChanTLS;
+    QIOChannelTLS *serverChanTLS;
+    QIOChannelSocket *clientChanSock;
+    QIOChannelSocket *serverChanSock;
+    qemu_acl *acl;
+    const char * const *wildcards;
+    int channel[2];
+    struct QIOChannelTLSHandshakeData clientHandshake = { false, false };
+    struct QIOChannelTLSHandshakeData serverHandshake = { false, false };
+    Error *err = NULL;
+    QIOChannelTest *test;
+    GMainContext *mainloop;
+
+    /* We'll use this for our fake client-server connection */
+    g_assert(socketpair(AF_UNIX, SOCK_STREAM, 0, channel) == 0);
+
+#define CLIENT_CERT_DIR "tests/test-crypto-tlssession-client/"
+#define SERVER_CERT_DIR "tests/test-crypto-tlssession-server/"
+    mkdir(CLIENT_CERT_DIR, 0700);
+    mkdir(SERVER_CERT_DIR, 0700);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    g_assert(link(data->servercacrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->servercrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+
+    g_assert(link(data->clientcacrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->clientcrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+
+    clientCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        CLIENT_CERT_DIR,
+        &err);
+    g_assert(clientCreds != NULL);
+
+    serverCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        SERVER_CERT_DIR,
+        &err);
+    g_assert(serverCreds != NULL);
+
+    acl = qemu_acl_init("channeltlsacl");
+    qemu_acl_reset(acl);
+    wildcards = data->wildcards;
+    while (wildcards && *wildcards) {
+        qemu_acl_append(acl, 0, *wildcards);
+        wildcards++;
+    }
+
+    clientChanSock = qio_channel_socket_new_fd(
+        channel[0], &err);
+    g_assert(clientChanSock != NULL);
+    serverChanSock = qio_channel_socket_new_fd(
+        channel[1], &err);
+    g_assert(serverChanSock != NULL);
+
+    /*
+     * We have an evil loop to do the handshake in a single
+     * thread, so we need these non-blocking to avoid deadlock
+     * of ourselves
+     */
+    qio_channel_set_blocking(QIO_CHANNEL(clientChanSock), false, NULL);
+    qio_channel_set_blocking(QIO_CHANNEL(serverChanSock), false, NULL);
+
+    /* Now the real part of the test, setup the sessions */
+    clientChanTLS = qio_channel_tls_new_client(
+        QIO_CHANNEL(clientChanSock), clientCreds,
+        data->hostname, &err);
+    g_assert(clientChanTLS != NULL);
+
+    serverChanTLS = qio_channel_tls_new_server(
+        QIO_CHANNEL(serverChanSock), serverCreds,
+        "channeltlsacl", &err);
+    g_assert(serverChanTLS != NULL);
+
+    qio_channel_tls_handshake(clientChanTLS,
+                              test_tls_handshake_done,
+                              &clientHandshake,
+                              NULL);
+    qio_channel_tls_handshake(serverChanTLS,
+                              test_tls_handshake_done,
+                              &serverHandshake,
+                              NULL);
+
+    /*
+     * Finally we loop around & around doing handshake on each
+     * session until we get an error, or the handshake completes.
+     * This relies on the socketpair being nonblocking to avoid
+     * deadlocking ourselves upon handshake
+     */
+    mainloop = g_main_context_default();
+    do {
+        g_main_context_iteration(mainloop, TRUE);
+    } while (!clientHandshake.finished &&
+             !serverHandshake.finished);
+
+    g_assert(clientHandshake.failed == data->expectClientFail);
+    g_assert(serverHandshake.failed == data->expectServerFail);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, false,
+                                 QIO_CHANNEL(clientChanTLS),
+                                 QIO_CHANNEL(serverChanTLS));
+    qio_channel_test_validate(test);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, true,
+                                 QIO_CHANNEL(clientChanTLS),
+                                 QIO_CHANNEL(serverChanTLS));
+    qio_channel_test_validate(test);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    rmdir(CLIENT_CERT_DIR);
+    rmdir(SERVER_CERT_DIR);
+
+    object_unparent(OBJECT(serverCreds));
+    object_unparent(OBJECT(clientCreds));
+
+    object_unref(OBJECT(serverChanTLS));
+    object_unref(OBJECT(clientChanTLS));
+
+    object_unref(OBJECT(serverChanSock));
+    object_unref(OBJECT(clientChanSock));
+
+    close(channel[0]);
+    close(channel[1]);
+}
+
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TEST_CHANNEL(name, caCrt,                                      \
+                      serverCrt, clientCrt,                             \
+                      expectServerFail, expectClientFail,               \
+                      hostname, wildcards)                              \
+    struct QIOChannelTLSTestData name = {                               \
+        caCrt, caCrt, serverCrt, clientCrt,                             \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qio/channel/tls/" # name,                    \
+                         &name, test_io_channel_tls);
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    const char *const wildcards[] = {
+        "C=UK,CN=qemu*",
+        NULL,
+    };
+    TEST_CHANNEL(basic, cacertreq.filename, servercertreq.filename,
+                 clientcertreq.filename, false, false,
+                 "qemu.org", wildcards);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&cacertreq);
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/qemu/tests/test-io-task.c b/qemu/tests/test-io-task.c
new file mode 100644
index 000000000..5a9775086
--- /dev/null
+++ b/qemu/tests/test-io-task.c
@@ -0,0 +1,270 @@
+/*
+ * QEMU I/O task tests
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "io/task.h"
+#include "qapi/error.h"
+
+#define TYPE_DUMMY "qemu:dummy"
+
+typedef struct DummyObject DummyObject;
+typedef struct DummyObjectClass DummyObjectClass;
+
+struct DummyObject {
+    Object parent;
+};
+
+struct DummyObjectClass {
+    ObjectClass parent;
+};
+
+static const TypeInfo dummy_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_DUMMY,
+    .instance_size = sizeof(DummyObject),
+    .class_size = sizeof(DummyObjectClass),
+};
+
+struct TestTaskData {
+    Object *source;
+    Error *err;
+    bool freed;
+};
+
+
+static void task_callback(Object *source,
+                          Error *err,
+                          gpointer opaque)
+{
+    struct TestTaskData *data = opaque;
+
+    data->source = source;
+    data->err = err;
+}
+
+
+static void test_task_complete(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    Object *src;
+    struct TestTaskData data = { NULL, NULL, false };
+
+    task = qio_task_new(obj, task_callback, &data, NULL);
+    src = qio_task_get_source(task);
+
+    qio_task_complete(task);
+
+    g_assert(obj == src);
+
+    object_unref(obj);
+    object_unref(src);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+    g_assert(data.freed == false);
+}
+
+
+static void task_data_free(gpointer opaque)
+{
+    struct TestTaskData *data = opaque;
+
+    data->freed = true;
+}
+
+
+static void test_task_data_free(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestTaskData data = { NULL, NULL, false };
+
+    task = qio_task_new(obj, task_callback, &data, task_data_free);
+
+    qio_task_complete(task);
+
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+    g_assert(data.freed == true);
+}
+
+
+static void test_task_error(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestTaskData data = { NULL, NULL, false };
+    Error *err = NULL;
+
+    task = qio_task_new(obj, task_callback, &data, NULL);
+
+    error_setg(&err, "Some error");
+
+    qio_task_abort(task, err);
+
+    error_free(err);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == err);
+    g_assert(data.freed == false);
+
+}
+
+
+struct TestThreadWorkerData {
+    Object *source;
+    Error *err;
+    bool fail;
+    GThread *worker;
+    GThread *complete;
+    GMainLoop *loop;
+};
+
+static int test_task_thread_worker(QIOTask *task,
+                                   Error **errp,
+                                   gpointer opaque)
+{
+    struct TestThreadWorkerData *data = opaque;
+
+    data->worker = g_thread_self();
+
+    if (data->fail) {
+        error_setg(errp, "Testing fail");
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static void test_task_thread_callback(Object *source,
+                                      Error *err,
+                                      gpointer opaque)
+{
+    struct TestThreadWorkerData *data = opaque;
+
+    data->source = source;
+    data->err = err;
+
+    data->complete = g_thread_self();
+
+    g_main_loop_quit(data->loop);
+}
+
+
+static void test_task_thread_complete(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestThreadWorkerData data = { 0 };
+    GThread *self;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+
+    task = qio_task_new(obj,
+                        test_task_thread_callback,
+                        &data,
+                        NULL);
+
+    qio_task_run_in_thread(task,
+                           test_task_thread_worker,
+                           &data,
+                           NULL);
+
+    g_main_loop_run(data.loop);
+
+    g_main_loop_unref(data.loop);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+
+    self = g_thread_self();
+
+    /* Make sure the test_task_thread_worker actually got
+     * run in a different thread */
+    g_assert(data.worker != self);
+
+    /* And that the test_task_thread_callback got rnu in
+     * the main loop thread (ie this one) */
+    g_assert(data.complete == self);
+}
+
+
+static void test_task_thread_error(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestThreadWorkerData data = { 0 };
+    GThread *self;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+    data.fail = true;
+
+    task = qio_task_new(obj,
+                        test_task_thread_callback,
+                        &data,
+                        NULL);
+
+    qio_task_run_in_thread(task,
+                           test_task_thread_worker,
+                           &data,
+                           NULL);
+
+    g_main_loop_run(data.loop);
+
+    g_main_loop_unref(data.loop);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err != NULL);
+
+    self = g_thread_self();
+
+    /* Make sure the test_task_thread_worker actually got
+     * run in a different thread */
+    g_assert(data.worker != self);
+
+    /* And that the test_task_thread_callback got rnu in
+     * the main loop thread (ie this one) */
+    g_assert(data.complete == self);
+}
+
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    module_call_init(MODULE_INIT_QOM);
+    type_register_static(&dummy_info);
+    g_test_add_func("/crypto/task/complete", test_task_complete);
+    g_test_add_func("/crypto/task/datafree", test_task_data_free);
+    g_test_add_func("/crypto/task/error", test_task_error);
+    g_test_add_func("/crypto/task/thread_complete", test_task_thread_complete);
+    g_test_add_func("/crypto/task/thread_error", test_task_thread_error);
+    return g_test_run();
+}
diff --git a/qemu/tests/test-iov.c b/qemu/tests/test-iov.c
index 46e4dddc5..3f25268dd 100644
--- a/qemu/tests/test-iov.c
+++ b/qemu/tests/test-iov.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qemu-common.h"
 #include "qemu/iov.h"
diff --git a/qemu/tests/test-logging.c b/qemu/tests/test-logging.c
new file mode 100644
index 000000000..ac8deedc9
--- /dev/null
+++ b/qemu/tests/test-logging.c
@@ -0,0 +1,141 @@
+/*
+ * logging unit-tests
+ *
+ * Copyright (C) 2016 Linaro Ltd.
+ *
+ *  Author: Alex Bennée <alex.bennee@linaro.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "qemu-common.h"
+#include "include/qemu/log.h"
+
+static void test_parse_range(void)
+{
+    qemu_set_dfilter_ranges("0x1000+0x100");
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x1001));
+    g_assert(qemu_log_in_addr_range(0x10ff));
+    g_assert_false(qemu_log_in_addr_range(0x1100));
+
+    qemu_set_dfilter_ranges("0x1000-0x100");
+
+    g_assert_false(qemu_log_in_addr_range(0x1001));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x0f01));
+    g_assert_false(qemu_log_in_addr_range(0x0f00));
+
+    qemu_set_dfilter_ranges("0x1000..0x1100");
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x1100));
+    g_assert_false(qemu_log_in_addr_range(0x1101));
+
+    qemu_set_dfilter_ranges("0x1000..0x1000");
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert_false(qemu_log_in_addr_range(0x1001));
+
+    qemu_set_dfilter_ranges("0x1000+0x100,0x2100-0x100,0x3000..0x3100");
+    g_assert(qemu_log_in_addr_range(0x1050));
+    g_assert(qemu_log_in_addr_range(0x2050));
+    g_assert(qemu_log_in_addr_range(0x3050));
+}
+
+#ifdef CONFIG_HAS_GLIB_SUBPROCESS_TESTS
+static void test_parse_invalid_range_subprocess(void)
+{
+    qemu_set_dfilter_ranges("0x1000+onehundred");
+}
+static void test_parse_invalid_range(void)
+{
+    g_test_trap_subprocess("/logging/parse_invalid_range/subprocess", 0, 0);
+    g_test_trap_assert_failed();
+    g_test_trap_assert_stdout("");
+    g_test_trap_assert_stderr("*Failed to parse range in: 0x1000+onehundred\n");
+}
+static void test_parse_zero_range_subprocess(void)
+{
+    qemu_set_dfilter_ranges("0x1000+0");
+}
+static void test_parse_zero_range(void)
+{
+    g_test_trap_subprocess("/logging/parse_zero_range/subprocess", 0, 0);
+    g_test_trap_assert_failed();
+    g_test_trap_assert_stdout("");
+    g_test_trap_assert_stderr("*Failed to parse range in: 0x1000+0\n");
+}
+
+/* As the only real failure from a bad log filename path spec is
+ * reporting to the user we have to use the g_test_trap_subprocess
+ * mechanism and check no errors reported on stderr.
+ */
+static void test_parse_path_subprocess(void)
+{
+    /* All these should work without issue */
+    qemu_set_log_filename("/tmp/qemu.log");
+    qemu_set_log_filename("/tmp/qemu-%d.log");
+    qemu_set_log_filename("/tmp/qemu.log.%d");
+}
+static void test_parse_path(void)
+{
+    g_test_trap_subprocess ("/logging/parse_path/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stdout("");
+    g_test_trap_assert_stderr("");
+}
+static void test_parse_invalid_path_subprocess(void)
+{
+    qemu_set_log_filename("/tmp/qemu-%d%d.log");
+}
+static void test_parse_invalid_path(void)
+{
+    g_test_trap_subprocess ("/logging/parse_invalid_path/subprocess", 0, 0);
+    g_test_trap_assert_passed();
+    g_test_trap_assert_stdout("");
+    g_test_trap_assert_stderr("Bad logfile format: /tmp/qemu-%d%d.log\n");
+}
+#endif /* CONFIG_HAS_GLIB_SUBPROCESS_TESTS */
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/logging/parse_range", test_parse_range);
+#ifdef CONFIG_HAS_GLIB_SUBPROCESS_TESTS
+    g_test_add_func("/logging/parse_invalid_range/subprocess", test_parse_invalid_range_subprocess);
+    g_test_add_func("/logging/parse_invalid_range", test_parse_invalid_range);
+    g_test_add_func("/logging/parse_zero_range/subprocess", test_parse_zero_range_subprocess);
+    g_test_add_func("/logging/parse_zero_range", test_parse_zero_range);
+    g_test_add_func("/logging/parse_path", test_parse_path);
+    g_test_add_func("/logging/parse_path/subprocess", test_parse_path_subprocess);
+    g_test_add_func("/logging/parse_invalid_path", test_parse_invalid_path);
+    g_test_add_func("/logging/parse_invalid_path/subprocess", test_parse_invalid_path_subprocess);
+#endif
+
+    return g_test_run();
+}
diff --git a/qemu/tests/test-mul64.c b/qemu/tests/test-mul64.c
index a0a17f777..1282ec5a2 100644
--- a/qemu/tests/test-mul64.c
+++ b/qemu/tests/test-mul64.c
@@ -6,10 +6,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdint.h>
 #include "qemu/host-utils.h"
-#include "qemu/osdep.h"
 
 
 typedef struct {
diff --git a/qemu/tests/test-netfilter.c b/qemu/tests/test-netfilter.c
new file mode 100644
index 000000000..7d105c323
--- /dev/null
+++ b/qemu/tests/test-netfilter.c
@@ -0,0 +1,201 @@
+/*
+ * QTest testcase for netfilter
+ *
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+#include "libqtest.h"
+
+/* add a netfilter to a netdev and then remove it */
+static void add_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add a netfilter to a netdev and then remove the netdev */
+static void remove_netdev_with_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove them */
+static void add_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f1'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove the netdev */
+static void remove_netdev_with_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+    qtest_add_func("/netfilter/addremove_one", add_one_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_one",
+                   remove_netdev_with_one_netfilter);
+    qtest_add_func("/netfilter/addremove_multi", add_multi_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_multi",
+                   remove_netdev_with_multi_netfilter);
+
+    qtest_start("-netdev user,id=qtest-bn0 -device e1000,netdev=qtest-bn0");
+    ret = g_test_run();
+
+    qtest_end();
+
+    return ret;
+}
diff --git a/qemu/tests/test-opts-visitor.c b/qemu/tests/test-opts-visitor.c
index 1c753d982..008e67738 100644
--- a/qemu/tests/test-opts-visitor.c
+++ b/qemu/tests/test-opts-visitor.c
@@ -10,13 +10,14 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qemu/config-file.h"     /* qemu_add_opts() */
 #include "qemu/option.h"          /* qemu_opts_parse() */
+#include "qapi/error.h"
 #include "qapi/opts-visitor.h"    /* opts_visitor_new() */
 #include "test-qapi-visit.h"      /* visit_type_UserDefOptions() */
-#include "qapi/dealloc-visitor.h" /* qapi_dealloc_visitor_new() */
 
 static QemuOptsList userdef_opts = {
     .name = "userdef",
@@ -44,7 +45,7 @@ setup_fixture(OptsVisitorFixture *f, gconstpointer test_data)
     g_assert(opts != NULL);
 
     ov = opts_visitor_new(opts);
-    visit_type_UserDefOptions(opts_get_visitor(ov), &f->userdef, NULL,
+    visit_type_UserDefOptions(opts_get_visitor(ov), NULL, &f->userdef,
                               &f->err);
     opts_visitor_cleanup(ov);
     qemu_opts_del(opts);
@@ -54,14 +55,7 @@ setup_fixture(OptsVisitorFixture *f, gconstpointer test_data)
 static void
 teardown_fixture(OptsVisitorFixture *f, gconstpointer test_data)
 {
-    if (f->userdef != NULL) {
-        QapiDeallocVisitor *dv;
-
-        dv = qapi_dealloc_visitor_new();
-        visit_type_UserDefOptions(qapi_dealloc_get_visitor(dv), &f->userdef,
-                                  NULL, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
-    }
+    qapi_free_UserDefOptions(f->userdef);
     error_free(f->err);
 }
 
diff --git a/qemu/tests/test-qdev-global-props.c b/qemu/tests/test-qdev-global-props.c
index 0be98355c..f0cc31e11 100644
--- a/qemu/tests/test-qdev-global-props.c
+++ b/qemu/tests/test-qdev-global-props.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdint.h>
 
 #include "hw/qdev.h"
 #include "qom/object.h"
@@ -116,26 +116,20 @@ static void test_static_globalprop(void)
 #define TYPE_UNUSED_HOTPLUG   "hotplug-type"
 #define TYPE_UNUSED_NOHOTPLUG "nohotplug-type"
 
-static void prop1_accessor(Object *obj,
-                           Visitor *v,
-                           void *opaque,
-                           const char *name,
-                           Error **errp)
+static void prop1_accessor(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     MyType *mt = DYNAMIC_TYPE(obj);
 
-    visit_type_uint32(v, &mt->prop1, name, errp);
+    visit_type_uint32(v, name, &mt->prop1, errp);
 }
 
-static void prop2_accessor(Object *obj,
-                           Visitor *v,
-                           void *opaque,
-                           const char *name,
-                           Error **errp)
+static void prop2_accessor(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     MyType *mt = DYNAMIC_TYPE(obj);
 
-    visit_type_uint32(v, &mt->prop2, name, errp);
+    visit_type_uint32(v, name, &mt->prop2, errp);
 }
 
 static void dynamic_instance_init(Object *obj)
diff --git a/qemu/tests/test-qemu-opts.c b/qemu/tests/test-qemu-opts.c
index 0c1136d1b..32abed5ea 100644
--- a/qemu/tests/test-qemu-opts.c
+++ b/qemu/tests/test-qemu-opts.c
@@ -7,12 +7,12 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qstring.h"
 #include "qemu/config-file.h"
 
 #include <glib.h>
-#include <string.h>
 
 static QemuOptsList opts_list_01 = {
     .name = "opts_list_01",
diff --git a/qemu/tests/test-qga.c b/qemu/tests/test-qga.c
new file mode 100644
index 000000000..72a89dec2
--- /dev/null
+++ b/qemu/tests/test-qga.c
@@ -0,0 +1,869 @@
+#include "qemu/osdep.h"
+#include <locale.h>
+#include <glib.h>
+#include <glib/gstdio.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "libqtest.h"
+
+typedef struct {
+    char *test_dir;
+    GMainLoop *loop;
+    int fd;
+    GPid pid;
+} TestFixture;
+
+static int connect_qga(char *path)
+{
+    int s, ret, len, i = 0;
+    struct sockaddr_un remote;
+
+    s = socket(AF_UNIX, SOCK_STREAM, 0);
+    g_assert(s != -1);
+
+    remote.sun_family = AF_UNIX;
+    do {
+        strcpy(remote.sun_path, path);
+        len = strlen(remote.sun_path) + sizeof(remote.sun_family);
+        ret = connect(s, (struct sockaddr *)&remote, len);
+        if (ret == -1) {
+            g_usleep(G_USEC_PER_SEC);
+        }
+        if (i++ == 10) {
+            return -1;
+        }
+    } while (ret == -1);
+
+    return s;
+}
+
+static void qga_watch(GPid pid, gint status, gpointer user_data)
+{
+    TestFixture *fixture = user_data;
+
+    g_assert_cmpint(status, ==, 0);
+    g_main_loop_quit(fixture->loop);
+}
+
+static void
+fixture_setup(TestFixture *fixture, gconstpointer data)
+{
+    const gchar *extra_arg = data;
+    GError *error = NULL;
+    gchar *cwd, *path, *cmd, **argv = NULL;
+
+    fixture->loop = g_main_loop_new(NULL, FALSE);
+
+    fixture->test_dir = g_strdup("/tmp/qgatest.XXXXXX");
+    g_assert_nonnull(mkdtemp(fixture->test_dir));
+
+    path = g_build_filename(fixture->test_dir, "sock", NULL);
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -m unix-listen -t %s -p %s %s %s",
+                          cwd, G_DIR_SEPARATOR,
+                          fixture->test_dir, path,
+                          getenv("QTEST_LOG") ? "-v" : "",
+                          extra_arg ?: "");
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    g_spawn_async(fixture->test_dir, argv, NULL,
+                  G_SPAWN_SEARCH_PATH|G_SPAWN_DO_NOT_REAP_CHILD,
+                  NULL, NULL, &fixture->pid, &error);
+    g_assert_no_error(error);
+
+    g_child_watch_add(fixture->pid, qga_watch, fixture);
+
+    fixture->fd = connect_qga(path);
+    g_assert_cmpint(fixture->fd, !=, -1);
+
+    g_strfreev(argv);
+    g_free(cmd);
+    g_free(cwd);
+    g_free(path);
+}
+
+static void
+fixture_tear_down(TestFixture *fixture, gconstpointer data)
+{
+    gchar *tmp;
+
+    kill(fixture->pid, SIGTERM);
+
+    g_main_loop_run(fixture->loop);
+    g_main_loop_unref(fixture->loop);
+
+    g_spawn_close_pid(fixture->pid);
+
+    tmp = g_build_filename(fixture->test_dir, "foo", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "qga.state", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "sock", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    g_rmdir(fixture->test_dir);
+    g_free(fixture->test_dir);
+}
+
+static void qmp_assertion_message_error(const char     *domain,
+                                        const char     *file,
+                                        int             line,
+                                        const char     *func,
+                                        const char     *expr,
+                                        QDict          *dict)
+{
+    const char *class, *desc;
+    char *s;
+    QDict *error;
+
+    error = qdict_get_qdict(dict, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    s = g_strdup_printf("assertion failed %s: %s %s", expr, class, desc);
+    g_assertion_message(domain, file, line, func, s);
+    g_free(s);
+}
+
+#define qmp_assert_no_error(err) do {                                   \
+    if (qdict_haskey(err, "error")) {                                   \
+        qmp_assertion_message_error(G_LOG_DOMAIN, __FILE__, __LINE__,   \
+                                    G_STRFUNC, #err, err);              \
+    }                                                                   \
+} while (0)
+
+static void test_qga_sync_delimited(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    unsigned char c;
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync-delimited',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    qmp_fd_send(fixture->fd, cmd);
+    g_free(cmd);
+
+    v = read(fixture->fd, &c, 1);
+    g_assert_cmpint(v, ==, 1);
+    g_assert_cmpint(c, ==, 0xff);
+
+    ret = qmp_fd_receive(fixture->fd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_sync(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_ping(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    QDECREF(ret);
+}
+
+static void test_qga_invalid_cmd(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-invalid-cmd'}");
+    g_assert_nonnull(ret);
+
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    g_assert_cmpstr(class, ==, "CommandNotFound");
+    g_assert_cmpint(strlen(desc), >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    const gchar *version;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-info'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    version = qdict_get_try_str(val, "version");
+    g_assert_cmpstr(version, ==, QEMU_VERSION);
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_vcpus(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-vcpus'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least a cpu */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "logical-id"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_fsinfo(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-fsinfo'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* sanity-check the response if there are any filesystems */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    if (entry) {
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "mountpoint"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "type"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "disk"));
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_block_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    int64_t size;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-block-info'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        /* check there is at least some memory */
+        val = qdict_get_qdict(ret, "return");
+        size = qdict_get_int(val, "size");
+        g_assert_cmpint(size, >, 0);
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_blocks(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-blocks'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        list = qdict_get_qlist(ret, "return");
+        entry = qlist_first(list);
+        /* newer versions of qga may return empty list without error */
+        if (entry) {
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "phys-index"));
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+        }
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_network_get_interfaces(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-network-get-interfaces'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least an interface */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_file_ops(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *path, *enc;
+    unsigned char *dec;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+    FILE *f;
+    char tmp[100];
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* flush */
+    cmd = g_strdup_printf("{'execute': 'guest-file-flush',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* check content */
+    path = g_build_filename(fixture->test_dir, "foo", NULL);
+    f = fopen(path, "r");
+    g_assert_nonnull(f);
+    count = fread(tmp, 1, sizeof(tmp), f);
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    tmp[count] = 0;
+    g_assert_cmpstr(tmp, ==, (char *)helloworld);
+    fclose(f);
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'r' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* read eof */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': '%s' } }",
+                          id, 6, "set");
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 6);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* partial read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert(eof);
+    dec = g_base64_decode(b64, &count);
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert_cmpmem(dec, count, helloworld + 6, sizeof(helloworld) - 6);
+    g_free(dec);
+
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_file_write_read(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *enc;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read (check implicit flush) */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek to 0 */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': '%s' } }",
+                          id, 0, "set");
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_get_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t time;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time, >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_set_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t current, time;
+    gchar *cmd;
+
+    /* get current time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    current = qdict_get_int(ret, "return");
+    g_assert_cmpint(current, >, 0);
+    QDECREF(ret);
+
+    /* set some old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-set-time',"
+                 " 'arguments': { 'time': 1000 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    /* check old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time / 1000, <, G_USEC_PER_SEC * 10);
+    QDECREF(ret);
+
+    /* set back current time */
+    cmd = g_strdup_printf("{'execute': 'guest-set-time',"
+                          " 'arguments': { 'time': %" PRId64 " } }",
+                          current + time * 1000);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+static void test_qga_fstrim(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fstrim',"
+                 " arguments: { minimum: 4194304 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "paths"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_blacklist(gconstpointer data)
+{
+    TestFixture fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    fixture_setup(&fix, "-b guest-ping,guest-get-time");
+
+    /* check blacklist */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    /* check something work */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-fsinfo'}");
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    fixture_tear_down(&fix, NULL);
+}
+
+static void test_qga_config(gconstpointer data)
+{
+    GError *error = NULL;
+    char *cwd, *cmd, *out, *err, *str, **strv, *conf, **argv = NULL;
+    char *env[2];
+    int status, tmp;
+    gsize n;
+    GKeyFile *kf;
+    const char *qga_config =
+        "[general]\n"
+        "daemon=false\n"
+        "method=virtio-serial\n"
+        "path=/path/to/org.qemu.guest_agent.0\n"
+        "pidfile=/var/foo/qemu-ga.pid\n"
+        "statedir=/var/state\n"
+        "verbose=true\n"
+        "blacklist=guest-ping;guest-get-time\n";
+
+    tmp = g_file_open_tmp(NULL, &conf, &error);
+    g_assert_no_error(error);
+    g_assert_cmpint(tmp, >=, 0);
+    g_assert_cmpstr(conf, !=, "");
+
+    g_file_set_contents(conf, qga_config, -1, &error);
+    g_assert_no_error(error);
+
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -D",
+                          cwd, G_DIR_SEPARATOR);
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    env[0] = g_strdup_printf("QGA_CONF=%s", conf);
+    env[1] = NULL;
+    g_spawn_sync(NULL, argv, env, 0,
+                 NULL, NULL, &out, &err, &status, &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(err, ==, "");
+    g_assert_cmpint(status, ==, 0);
+
+    kf = g_key_file_new();
+    g_key_file_load_from_data(kf, out, -1, G_KEY_FILE_NONE, &error);
+    g_assert_no_error(error);
+
+    str = g_key_file_get_start_group(kf);
+    g_assert_cmpstr(str, ==, "general");
+    g_free(str);
+
+    g_assert_false(g_key_file_get_boolean(kf, "general", "daemon", &error));
+    g_assert_no_error(error);
+
+    str = g_key_file_get_string(kf, "general", "method", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "virtio-serial");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "path", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/path/to/org.qemu.guest_agent.0");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "pidfile", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/foo/qemu-ga.pid");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "statedir", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/state");
+    g_free(str);
+
+    g_assert_true(g_key_file_get_boolean(kf, "general", "verbose", &error));
+    g_assert_no_error(error);
+
+    strv = g_key_file_get_string_list(kf, "general", "blacklist", &n, &error);
+    g_assert_cmpint(n, ==, 2);
+#if GLIB_CHECK_VERSION(2, 44, 0)
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-ping"));
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-get-time"));
+#endif
+    g_assert_no_error(error);
+    g_strfreev(strv);
+
+    g_free(out);
+    g_free(err);
+    g_free(conf);
+    g_free(env[0]);
+    g_key_file_free(kf);
+
+    close(tmp);
+}
+
+static void test_qga_fsfreeze_status(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "thawed");
+
+    QDECREF(ret);
+}
+
+static void test_qga_fsfreeze_and_thaw(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-freeze'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "frozen");
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-thaw'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+int main(int argc, char **argv)
+{
+    TestFixture fix;
+    int ret;
+
+    setlocale (LC_ALL, "");
+    g_test_init(&argc, &argv, NULL);
+    fixture_setup(&fix, NULL);
+
+    g_test_add_data_func("/qga/sync-delimited", &fix, test_qga_sync_delimited);
+    g_test_add_data_func("/qga/sync", &fix, test_qga_sync);
+    g_test_add_data_func("/qga/ping", &fix, test_qga_ping);
+    g_test_add_data_func("/qga/info", &fix, test_qga_info);
+    g_test_add_data_func("/qga/network-get-interfaces", &fix,
+                         test_qga_network_get_interfaces);
+    g_test_add_data_func("/qga/get-vcpus", &fix, test_qga_get_vcpus);
+    g_test_add_data_func("/qga/get-fsinfo", &fix, test_qga_get_fsinfo);
+    g_test_add_data_func("/qga/get-memory-block-info", &fix,
+                         test_qga_get_memory_block_info);
+    g_test_add_data_func("/qga/get-memory-blocks", &fix,
+                         test_qga_get_memory_blocks);
+    g_test_add_data_func("/qga/file-ops", &fix, test_qga_file_ops);
+    g_test_add_data_func("/qga/file-write-read", &fix, test_qga_file_write_read);
+    g_test_add_data_func("/qga/get-time", &fix, test_qga_get_time);
+    g_test_add_data_func("/qga/invalid-cmd", &fix, test_qga_invalid_cmd);
+    g_test_add_data_func("/qga/fsfreeze-status", &fix,
+                         test_qga_fsfreeze_status);
+
+    g_test_add_data_func("/qga/blacklist", NULL, test_qga_blacklist);
+    g_test_add_data_func("/qga/config", NULL, test_qga_config);
+
+    if (g_getenv("QGA_TEST_SIDE_EFFECTING")) {
+        g_test_add_data_func("/qga/fsfreeze-and-thaw", &fix,
+                             test_qga_fsfreeze_and_thaw);
+        g_test_add_data_func("/qga/set-time", &fix, test_qga_set_time);
+        g_test_add_data_func("/qga/fstrim", &fix, test_qga_fstrim);
+    }
+
+    ret = g_test_run();
+
+    fixture_tear_down(&fix, NULL);
+
+    return ret;
+}
diff --git a/qemu/tests/test-qmp-commands.c b/qemu/tests/test-qmp-commands.c
index 9918f2306..14a9ebbd5 100644
--- a/qemu/tests/test-qmp-commands.c
+++ b/qemu/tests/test-qmp-commands.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qemu-common.h"
 #include "qapi/qmp/types.h"
@@ -12,6 +13,11 @@ void qmp_user_def_cmd(Error **errp)
 {
 }
 
+Empty2 *qmp_user_def_cmd0(Error **errp)
+{
+    return g_new0(Empty2, 1);
+}
+
 void qmp_user_def_cmd1(UserDefOne * ud1, Error **errp)
 {
 }
@@ -25,11 +31,9 @@ UserDefTwo *qmp_user_def_cmd2(UserDefOne *ud1a,
     UserDefOne *ud1d = g_malloc0(sizeof(UserDefOne));
 
     ud1c->string = strdup(ud1a->string);
-    ud1c->base = g_new0(UserDefZero, 1);
-    ud1c->base->integer = ud1a->base->integer;
+    ud1c->integer = ud1a->integer;
     ud1d->string = strdup(has_udb1 ? ud1b->string : "blah0");
-    ud1d->base = g_new0(UserDefZero, 1);
-    ud1d->base->integer = has_udb1 ? ud1b->base->integer : 0;
+    ud1d->integer = has_udb1 ? ud1b->integer : 0;
 
     ret = g_new0(UserDefTwo, 1);
     ret->string0 = strdup("blah1");
@@ -46,11 +50,16 @@ UserDefTwo *qmp_user_def_cmd2(UserDefOne *ud1a,
     return ret;
 }
 
-int64_t qmp_user_def_cmd3(int64_t a, bool has_b, int64_t b, Error **errp)
+int64_t qmp_guest_get_time(int64_t a, bool has_b, int64_t b, Error **errp)
 {
     return a + (has_b ? b : 0);
 }
 
+QObject *qmp_guest_sync(QObject *arg, Error **errp)
+{
+    return arg;
+}
+
 __org_qemu_x_Union1 *qmp___org_qemu_x_command(__org_qemu_x_EnumList *a,
                                               __org_qemu_x_StructList *b,
                                               __org_qemu_x_Union2 *c,
@@ -59,9 +68,13 @@ __org_qemu_x_Union1 *qmp___org_qemu_x_command(__org_qemu_x_EnumList *a,
 {
     __org_qemu_x_Union1 *ret = g_new0(__org_qemu_x_Union1, 1);
 
-    ret->kind = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
-    ret->__org_qemu_x_branch = strdup("blah1");
+    ret->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    ret->u.__org_qemu_x_branch.data = strdup("blah1");
 
+    /* Also test that 'wchar-t' was munged to 'q_wchar_t' */
+    if (b && b->value && !b->value->has_q_wchar_t) {
+        b->value->q_wchar_t = 1;
+    }
     return ret;
 }
 
@@ -155,7 +168,7 @@ static void test_dispatch_cmd_io(void)
 
     qdict_put(args3, "a", qint_from_int(66));
     qdict_put(req, "arguments", args3);
-    qdict_put(req, "execute", qstring_from_str("user_def_cmd3"));
+    qdict_put(req, "execute", qstring_from_str("guest-get-time"));
 
     ret3 = qobject_to_qint(test_qmp_dispatch(req));
     assert(qint_get_int(ret3) == 66);
@@ -171,20 +184,17 @@ static void test_dealloc_types(void)
     UserDefOneList *ud1list;
 
     ud1test = g_malloc0(sizeof(UserDefOne));
-    ud1test->base = g_new0(UserDefZero, 1);
-    ud1test->base->integer = 42;
+    ud1test->integer = 42;
     ud1test->string = g_strdup("hi there 42");
 
     qapi_free_UserDefOne(ud1test);
 
     ud1a = g_malloc0(sizeof(UserDefOne));
-    ud1a->base = g_new0(UserDefZero, 1);
-    ud1a->base->integer = 43;
+    ud1a->integer = 43;
     ud1a->string = g_strdup("hi there 43");
 
     ud1b = g_malloc0(sizeof(UserDefOne));
-    ud1b->base = g_new0(UserDefZero, 1);
-    ud1b->base->integer = 44;
+    ud1b->integer = 44;
     ud1b->string = g_strdup("hi there 44");
 
     ud1list = g_malloc0(sizeof(UserDefOneList));
@@ -213,7 +223,7 @@ static void test_dealloc_partial(void)
         qdict_put_obj(ud2_dict, "string0", QOBJECT(qstring_from_str(text)));
 
         qiv = qmp_input_visitor_new(QOBJECT(ud2_dict));
-        visit_type_UserDefTwo(qmp_input_get_visitor(qiv), &ud2, NULL, &err);
+        visit_type_UserDefTwo(qmp_input_get_visitor(qiv), NULL, &ud2, &err);
         qmp_input_visitor_cleanup(qiv);
         QDECREF(ud2_dict);
     }
@@ -225,8 +235,7 @@ static void test_dealloc_partial(void)
     assert(ud2->dict1 == NULL);
 
     /* confirm & release construction error */
-    assert(err != NULL);
-    error_free(err);
+    error_free_or_abort(&err);
 
     /* tear down partial object */
     qapi_free_UserDefTwo(ud2);
diff --git a/qemu/tests/test-qmp-event.c b/qemu/tests/test-qmp-event.c
index 1ee40e148..a296fdbac 100644
--- a/qemu/tests/test-qmp-event.c
+++ b/qemu/tests/test-qmp-event.c
@@ -11,8 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdarg.h>
 
 #include "qemu-common.h"
 #include "test-qapi-types.h"
@@ -94,7 +94,7 @@ static bool qdict_cmp_simple(QDict *a, QDict *b)
 
 /* This function is hooked as final emit function, which can verify the
    correctness. */
-static void event_test_emit(TEST_QAPIEvent event, QDict *d, Error **errp)
+static void event_test_emit(test_QAPIEvent event, QDict *d, Error **errp)
 {
     QObject *obj;
     QDict *t;
@@ -179,9 +179,7 @@ static void test_event_c(TestEventData *data,
     QDict *d, *d_data, *d_b;
 
     UserDefOne b;
-    UserDefZero z;
-    z.integer = 2;
-    b.base = &z;
+    b.integer = 2;
     b.string = g_strdup("test1");
     b.has_enum1 = false;
 
@@ -209,11 +207,9 @@ static void test_event_d(TestEventData *data,
 {
     UserDefOne struct1;
     EventStructOne a;
-    UserDefZero z;
     QDict *d, *d_data, *d_a, *d_struct1;
 
-    z.integer = 2;
-    struct1.base = &z;
+    struct1.integer = 2;
     struct1.string = g_strdup("test1");
     struct1.has_enum1 = true;
     struct1.enum1 = ENUM_ONE_VALUE1;
diff --git a/qemu/tests/test-qmp-input-strict.c b/qemu/tests/test-qmp-input-strict.c
index 68f855bdf..d71727e27 100644
--- a/qemu/tests/test-qmp-input-strict.c
+++ b/qemu/tests/test-qmp-input-strict.c
@@ -11,14 +11,18 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdarg.h>
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
+#include "test-qmp-introspect.h"
+#include "qmp-introspect.h"
+#include "qapi-visit.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
@@ -37,78 +41,65 @@ static void validate_teardown(TestInputVisitorData *data,
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
-static GCC_FMT_ATTR(2, 3)
-Visitor *validate_test_init(TestInputVisitorData *data,
-                             const char *json_string, ...)
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *validate_test_init_internal(TestInputVisitorData *data,
+                                            const char *json_string,
+                                            va_list *ap)
 {
     Visitor *v;
-    va_list ap;
 
-    va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
-    va_end(ap);
+    validate_teardown(data, NULL);
 
-    g_assert(data->obj != NULL);
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
 
     data->qiv = qmp_input_visitor_new_strict(data->obj);
-    g_assert(data->qiv != NULL);
+    g_assert(data->qiv);
 
     v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
+    g_assert(v);
 
     return v;
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
+static GCC_FMT_ATTR(2, 3)
+Visitor *validate_test_init(TestInputVisitorData *data,
+                             const char *json_string, ...)
 {
-    Error *err = NULL;
+    Visitor *v;
+    va_list ap;
 
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
+    va_start(ap, json_string);
+    v = validate_test_init_internal(data, json_string, &ap);
+    va_end(ap);
+    return v;
+}
 
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
+/* similar to validate_test_init(), but does not expect a string
+ * literal/format json_string argument and so can be used for
+ * programatically generated strings (and we can't pass in programatically
+ * generated strings via %s format parameters since qobject_from_jsonv()
+ * will wrap those in double-quotes and treat the entire object as a
+ * string)
+ */
+static Visitor *validate_test_init_raw(TestInputVisitorData *data,
+                                       const char *json_string)
+{
+    return validate_test_init_internal(data, json_string, NULL);
 }
 
+
 static void test_validate_struct(TestInputVisitorData *data,
                                   const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, NULL, &p, &error_abort);
     g_free(p->string);
     g_free(p);
 }
@@ -117,7 +108,6 @@ static void test_validate_struct_nested(TestInputVisitorData *data,
                                          const void *unused)
 {
     UserDefTwo *udp = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'string0': 'string0', "
@@ -125,8 +115,7 @@ static void test_validate_struct_nested(TestInputVisitorData *data,
                            "'dict2': { 'userdef': { 'integer': 42, "
                            "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(v, NULL, &udp, &error_abort);
     qapi_free_UserDefTwo(udp);
 }
 
@@ -134,13 +123,11 @@ static void test_validate_list(TestInputVisitorData *data,
                                 const void *unused)
 {
     UserDefOneList *head = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
     qapi_free_UserDefOneList(head);
 }
 
@@ -149,12 +136,10 @@ static void test_validate_union_native_list(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data, "{ 'type': 'integer', 'data' : [ 1, 2 ] }");
 
-    visit_type_UserDefNativeListUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefNativeListUnion(v, NULL, &tmp, &error_abort);
     qapi_free_UserDefNativeListUnion(tmp);
 }
 
@@ -163,16 +148,14 @@ static void test_validate_union_flat(TestInputVisitorData *data,
 {
     UserDefFlatUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data,
                            "{ 'enum1': 'value1', "
+                           "'integer': 41, "
                            "'string': 'str', "
                            "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &error_abort);
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -181,12 +164,10 @@ static void test_validate_alternate(TestInputVisitorData *data,
 {
     UserDefAlternate *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data, "42");
 
-    visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
     qapi_free_UserDefAlternate(tmp);
 }
 
@@ -199,8 +180,8 @@ static void test_validate_fail_struct(TestInputVisitorData *data,
 
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo', 'extra': 42 }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
     if (p) {
         g_free(p->string);
     }
@@ -216,8 +197,8 @@ static void test_validate_fail_struct_nested(TestInputVisitorData *data,
 
     v = validate_test_init(data, "{ 'string0': 'string0', 'dict1': { 'string1': 'string1', 'dict2': { 'userdef1': { 'integer': 42, 'string': 'string', 'extra': [42, 23, {'foo':'bar'}] }, 'string2': 'string2'}}}");
 
-    visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefTwo(v, NULL, &udp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefTwo(udp);
 }
 
@@ -230,8 +211,8 @@ static void test_validate_fail_list(TestInputVisitorData *data,
 
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44, 'extra': 'ggg' } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefOneList(v, NULL, &head, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefOneList(head);
 }
 
@@ -245,8 +226,8 @@ static void test_validate_fail_union_native_list(TestInputVisitorData *data,
     v = validate_test_init(data,
                            "{ 'type': 'integer', 'data' : [ 'string' ] }");
 
-    visit_type_UserDefNativeListUnion(v, &tmp, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefNativeListUnion(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefNativeListUnion(tmp);
 }
 
@@ -259,8 +240,8 @@ static void test_validate_fail_union_flat(TestInputVisitorData *data,
 
     v = validate_test_init(data, "{ 'string': 'c', 'integer': 41, 'boolean': true }");
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -272,10 +253,10 @@ static void test_validate_fail_union_flat_no_discrim(TestInputVisitorData *data,
     Visitor *v;
 
     /* test situation where discriminator field ('enum1' here) is missing */
-    v = validate_test_init(data, "{ 'string': 'c', 'string1': 'd', 'string2': 'e' }");
+    v = validate_test_init(data, "{ 'integer': 42, 'string': 'c', 'string1': 'd', 'string2': 'e' }");
 
-    visit_type_UserDefFlatUnion2(v, &tmp, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefFlatUnion2(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefFlatUnion2(tmp);
 }
 
@@ -288,11 +269,32 @@ static void test_validate_fail_alternate(TestInputVisitorData *data,
 
     v = validate_test_init(data, "3.14");
 
-    visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(err);
+    visit_type_UserDefAlternate(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefAlternate(tmp);
 }
 
+static void do_test_validate_qmp_introspect(TestInputVisitorData *data,
+                                            const char *schema_json)
+{
+    SchemaInfoList *schema = NULL;
+    Visitor *v;
+
+    v = validate_test_init_raw(data, schema_json);
+
+    visit_type_SchemaInfoList(v, NULL, &schema, &error_abort);
+    g_assert(schema);
+
+    qapi_free_SchemaInfoList(schema);
+}
+
+static void test_validate_qmp_introspect(TestInputVisitorData *data,
+                                           const void *unused)
+{
+    do_test_validate_qmp_introspect(data, test_qmp_schema_json);
+    do_test_validate_qmp_introspect(data, qmp_schema_json);
+}
+
 static void validate_test_add(const char *testpath,
                                TestInputVisitorData *data,
                                void (*test_func)(TestInputVisitorData *data, const void *user_data))
@@ -333,6 +335,8 @@ int main(int argc, char **argv)
                       &testdata, test_validate_fail_alternate);
     validate_test_add("/visitor/input-strict/fail/union-native-list",
                       &testdata, test_validate_fail_union_native_list);
+    validate_test_add("/visitor/input-strict/pass/qmp-introspect",
+                      &testdata, test_validate_qmp_introspect);
 
     g_test_run();
 
diff --git a/qemu/tests/test-qmp-input-visitor.c b/qemu/tests/test-qmp-input-visitor.c
index b96195309..80527eb85 100644
--- a/qemu/tests/test-qmp-input-visitor.c
+++ b/qemu/tests/test-qmp-input-visitor.c
@@ -1,7 +1,7 @@
 /*
  * QMP Input Visitor unit-tests.
  *
- * Copyright (C) 2011, 2015 Red Hat Inc.
+ * Copyright (C) 2011-2016 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -10,10 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdarg.h>
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
@@ -36,31 +37,42 @@ static void visitor_input_teardown(TestInputVisitorData *data,
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
-static GCC_FMT_ATTR(2, 3)
-Visitor *visitor_input_test_init(TestInputVisitorData *data,
-                                 const char *json_string, ...)
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *visitor_input_test_init_internal(TestInputVisitorData *data,
+                                                 const char *json_string,
+                                                 va_list *ap)
 {
     Visitor *v;
-    va_list ap;
 
-    va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
-    va_end(ap);
+    visitor_input_teardown(data, NULL);
 
-    g_assert(data->obj != NULL);
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
 
     data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
+    g_assert(data->qiv);
 
     v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
+    g_assert(v);
 
     return v;
 }
 
+static GCC_FMT_ATTR(2, 3)
+Visitor *visitor_input_test_init(TestInputVisitorData *data,
+                                 const char *json_string, ...)
+{
+    Visitor *v;
+    va_list ap;
+
+    va_start(ap, json_string);
+    v = visitor_input_test_init_internal(data, json_string, &ap);
+    va_end(ap);
+    return v;
+}
+
 /* similar to visitor_input_test_init(), but does not expect a string
  * literal/format json_string argument and so can be used for
  * programatically generated strings (and we can't pass in programatically
@@ -71,32 +83,18 @@ Visitor *visitor_input_test_init(TestInputVisitorData *data,
 static Visitor *visitor_input_test_init_raw(TestInputVisitorData *data,
                                             const char *json_string)
 {
-    Visitor *v;
-
-    data->obj = qobject_from_json(json_string);
-
-    g_assert(data->obj != NULL);
-
-    data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
-
-    v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
-
-    return v;
+    return visitor_input_test_init_internal(data, json_string, NULL);
 }
 
 static void test_visitor_in_int(TestInputVisitorData *data,
                                 const void *unused)
 {
     int64_t res = 0, value = -42;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%" PRId64, value);
 
-    visit_type_int(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_int(v, NULL, &res, &error_abort);
     g_assert_cmpint(res, ==, value);
 }
 
@@ -113,22 +111,19 @@ static void test_visitor_in_int_overflow(TestInputVisitorData *data,
      */
     v = visitor_input_test_init(data, "%f", DBL_MAX);
 
-    visit_type_int(v, &res, NULL, &err);
-    g_assert(err);
-    error_free(err);
+    visit_type_int(v, NULL, &res, &err);
+    error_free_or_abort(&err);
 }
 
 static void test_visitor_in_bool(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     bool res = false;
     Visitor *v;
 
     v = visitor_input_test_init(data, "true");
 
-    visit_type_bool(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(v, NULL, &res, &error_abort);
     g_assert_cmpint(res, ==, true);
 }
 
@@ -136,13 +131,11 @@ static void test_visitor_in_number(TestInputVisitorData *data,
                                    const void *unused)
 {
     double res = 0, value = 3.14;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%f", value);
 
-    visit_type_number(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_number(v, NULL, &res, &error_abort);
     g_assert_cmpfloat(res, ==, value);
 }
 
@@ -150,13 +143,11 @@ static void test_visitor_in_string(TestInputVisitorData *data,
                                    const void *unused)
 {
     char *res = NULL, *value = (char *) "Q E M U";
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%s", value);
 
-    visit_type_str(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_str(v, NULL, &res, &error_abort);
     g_assert_cmpstr(res, ==, value);
 
     g_free(res);
@@ -165,7 +156,6 @@ static void test_visitor_in_string(TestInputVisitorData *data,
 static void test_visitor_in_enum(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     Visitor *v;
     EnumOne i;
 
@@ -174,63 +164,21 @@ static void test_visitor_in_enum(TestInputVisitorData *data,
 
         v = visitor_input_test_init(data, "%s", EnumOne_lookup[i]);
 
-        visit_type_EnumOne(v, &res, NULL, &err);
-        g_assert(!err);
+        visit_type_EnumOne(v, NULL, &res, &error_abort);
         g_assert_cmpint(i, ==, res);
-
-        visitor_input_teardown(data, NULL);
     }
-
-    data->obj = NULL;
-    data->qiv = NULL;
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_in_struct(TestInputVisitorData *data,
                                    const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, NULL, &p, &error_abort);
     g_assert_cmpint(p->integer, ==, -42);
     g_assert(p->boolean == true);
     g_assert_cmpstr(p->string, ==, "foo");
@@ -239,17 +187,10 @@ static void test_visitor_in_struct(TestInputVisitorData *data,
     g_free(p);
 }
 
-static void check_and_free_str(char *str, const char *cmp)
-{
-    g_assert_cmpstr(str, ==, cmp);
-    g_free(str);
-}
-
 static void test_visitor_in_struct_nested(TestInputVisitorData *data,
                                           const void *unused)
 {
     UserDefTwo *udp = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "{ 'string0': 'string0', "
@@ -257,34 +198,28 @@ static void test_visitor_in_struct_nested(TestInputVisitorData *data,
                                 "'dict2': { 'userdef': { 'integer': 42, "
                                 "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefTwo(v, &udp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(v, NULL, &udp, &error_abort);
 
-    check_and_free_str(udp->string0, "string0");
-    check_and_free_str(udp->dict1->string1, "string1");
-    g_assert_cmpint(udp->dict1->dict2->userdef->base->integer, ==, 42);
-    check_and_free_str(udp->dict1->dict2->userdef->string, "string");
-    check_and_free_str(udp->dict1->dict2->string, "string2");
+    g_assert_cmpstr(udp->string0, ==, "string0");
+    g_assert_cmpstr(udp->dict1->string1, ==, "string1");
+    g_assert_cmpint(udp->dict1->dict2->userdef->integer, ==, 42);
+    g_assert_cmpstr(udp->dict1->dict2->userdef->string, ==, "string");
+    g_assert_cmpstr(udp->dict1->dict2->string, ==, "string2");
     g_assert(udp->dict1->has_dict3 == false);
 
-    g_free(udp->dict1->dict2->userdef);
-    g_free(udp->dict1->dict2);
-    g_free(udp->dict1);
-    g_free(udp);
+    qapi_free_UserDefTwo(udp);
 }
 
 static void test_visitor_in_list(TestInputVisitorData *data,
                                  const void *unused)
 {
     UserDefOneList *item, *head = NULL;
-    Error *err = NULL;
     Visitor *v;
     int i;
 
     v = visitor_input_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
     g_assert(head != NULL);
 
     for (i = 0, item = head; item; item = item->next, i++) {
@@ -292,31 +227,80 @@ static void test_visitor_in_list(TestInputVisitorData *data,
 
         snprintf(string, sizeof(string), "string%d", i);
         g_assert_cmpstr(item->value->string, ==, string);
-        g_assert_cmpint(item->value->base->integer, ==, 42 + i);
+        g_assert_cmpint(item->value->integer, ==, 42 + i);
     }
 
     qapi_free_UserDefOneList(head);
+    head = NULL;
+
+    /* An empty list is valid */
+    v = visitor_input_test_init(data, "[]");
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
+    g_assert(!head);
+}
+
+static void test_visitor_in_any(TestInputVisitorData *data,
+                                const void *unused)
+{
+    QObject *res = NULL;
+    Visitor *v;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *qobj;
+
+    v = visitor_input_test_init(data, "-42");
+    visit_type_any(v, NULL, &res, &error_abort);
+    qint = qobject_to_qint(res);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobject_decref(res);
+
+    v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
+    visit_type_any(v, NULL, &res, &error_abort);
+    qdict = qobject_to_qdict(res);
+    g_assert(qdict && qdict_size(qdict) == 3);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
+    qobject_decref(res);
 }
 
 static void test_visitor_in_union_flat(TestInputVisitorData *data,
                                        const void *unused)
 {
     Visitor *v;
-    Error *err = NULL;
     UserDefFlatUnion *tmp;
+    UserDefUnionBase *base;
 
     v = visitor_input_test_init(data,
                                 "{ 'enum1': 'value1', "
+                                "'integer': 41, "
                                 "'string': 'str', "
                                 "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, ENUM_ONE_VALUE1);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->enum1, ==, ENUM_ONE_VALUE1);
     g_assert_cmpstr(tmp->string, ==, "str");
-    /* TODO g_assert_cmpint(tmp->integer, ==, 41); */
-    g_assert_cmpint(tmp->value1->boolean, ==, true);
+    g_assert_cmpint(tmp->integer, ==, 41);
+    g_assert_cmpint(tmp->u.value1.boolean, ==, true);
+
+    base = qapi_UserDefFlatUnion_base(tmp);
+    g_assert(&base->enum1 == &tmp->enum1);
+
     qapi_free_UserDefFlatUnion(tmp);
 }
 
@@ -326,14 +310,144 @@ static void test_visitor_in_alternate(TestInputVisitorData *data,
     Visitor *v;
     Error *err = NULL;
     UserDefAlternate *tmp;
+    WrapAlternate *wrap;
 
     v = visitor_input_test_init(data, "42");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QINT);
+    g_assert_cmpint(tmp->u.i, ==, 42);
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "'string'");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QSTRING);
+    g_assert_cmpstr(tmp->u.s, ==, "string");
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "{'integer':1, 'string':'str', "
+                                "'enum1':'value1', 'boolean':true}");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QDICT);
+    g_assert_cmpint(tmp->u.udfu.integer, ==, 1);
+    g_assert_cmpstr(tmp->u.udfu.string, ==, "str");
+    g_assert_cmpint(tmp->u.udfu.enum1, ==, ENUM_ONE_VALUE1);
+    g_assert_cmpint(tmp->u.udfu.u.value1.boolean, ==, true);
+    g_assert_cmpint(tmp->u.udfu.u.value1.has_a_b, ==, false);
+    qapi_free_UserDefAlternate(tmp);
 
-    visit_type_UserDefAlternate(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, USER_DEF_ALTERNATE_KIND_I);
-    g_assert_cmpint(tmp->i, ==, 42);
+    v = visitor_input_test_init(data, "false");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
     qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "{ 'alt': 42 }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QINT);
+    g_assert_cmpint(wrap->alt->u.i, ==, 42);
+    qapi_free_WrapAlternate(wrap);
+
+    v = visitor_input_test_init(data, "{ 'alt': 'string' }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QSTRING);
+    g_assert_cmpstr(wrap->alt->u.s, ==, "string");
+    qapi_free_WrapAlternate(wrap);
+
+    v = visitor_input_test_init(data, "{ 'alt': {'integer':1, 'string':'str', "
+                                "'enum1':'value1', 'boolean':true} }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QDICT);
+    g_assert_cmpint(wrap->alt->u.udfu.integer, ==, 1);
+    g_assert_cmpstr(wrap->alt->u.udfu.string, ==, "str");
+    g_assert_cmpint(wrap->alt->u.udfu.enum1, ==, ENUM_ONE_VALUE1);
+    g_assert_cmpint(wrap->alt->u.udfu.u.value1.boolean, ==, true);
+    g_assert_cmpint(wrap->alt->u.udfu.u.value1.has_a_b, ==, false);
+    qapi_free_WrapAlternate(wrap);
+}
+
+static void test_visitor_in_alternate_number(TestInputVisitorData *data,
+                                             const void *unused)
+{
+    Visitor *v;
+    Error *err = NULL;
+    AltStrBool *asb;
+    AltStrNum *asn;
+    AltNumStr *ans;
+    AltStrInt *asi;
+    AltIntNum *ain;
+    AltNumInt *ani;
+
+    /* Parsing an int */
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrBool(v, NULL, &asb, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrNum(v, NULL, &asn, &error_abort);
+    g_assert_cmpint(asn->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(asn->u.n, ==, 42);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumStr(v, NULL, &ans, &error_abort);
+    g_assert_cmpint(ans->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ans->u.n, ==, 42);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrInt(v, NULL, &asi, &error_abort);
+    g_assert_cmpint(asi->type, ==, QTYPE_QINT);
+    g_assert_cmpint(asi->u.i, ==, 42);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltIntNum(v, NULL, &ain, &error_abort);
+    g_assert_cmpint(ain->type, ==, QTYPE_QINT);
+    g_assert_cmpint(ain->u.i, ==, 42);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumInt(v, NULL, &ani, &error_abort);
+    g_assert_cmpint(ani->type, ==, QTYPE_QINT);
+    g_assert_cmpint(ani->u.i, ==, 42);
+    qapi_free_AltNumInt(ani);
+
+    /* Parsing a double */
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrBool(v, NULL, &asb, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrNum(v, NULL, &asn, &error_abort);
+    g_assert_cmpint(asn->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(asn->u.n, ==, 42.5);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumStr(v, NULL, &ans, &error_abort);
+    g_assert_cmpint(ans->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ans->u.n, ==, 42.5);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrInt(v, NULL, &asi, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltIntNum(v, NULL, &ain, &error_abort);
+    g_assert_cmpint(ain->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ain->u.n, ==, 42.5);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumInt(v, NULL, &ani, &error_abort);
+    g_assert_cmpint(ani->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ani->u.n, ==, 42.5);
+    qapi_free_AltNumInt(ani);
 }
 
 static void test_native_list_integer_helper(TestInputVisitorData *data,
@@ -341,7 +455,6 @@ static void test_native_list_integer_helper(TestInputVisitorData *data,
                                             UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -358,71 +471,71 @@ static void test_native_list_integer_helper(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, kind);
+    g_assert_cmpint(cvalue->type, ==, kind);
 
     switch (kind) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
         intList *elem = NULL;
-        for (i = 0, elem = cvalue->integer; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.integer.data;
+             elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
         int8List *elem = NULL;
-        for (i = 0, elem = cvalue->s8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s8.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
         int16List *elem = NULL;
-        for (i = 0, elem = cvalue->s16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s16.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
         int32List *elem = NULL;
-        for (i = 0, elem = cvalue->s32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s32.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
         int64List *elem = NULL;
-        for (i = 0, elem = cvalue->s64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s64.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
         uint8List *elem = NULL;
-        for (i = 0, elem = cvalue->u8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u8.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
         uint16List *elem = NULL;
-        for (i = 0, elem = cvalue->u16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u16.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
         uint32List *elem = NULL;
-        for (i = 0, elem = cvalue->u32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u32.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
         uint64List *elem = NULL;
-        for (i = 0, elem = cvalue->u64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u64.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
@@ -504,7 +617,6 @@ static void test_visitor_in_native_list_bool(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     boolList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -521,12 +633,11 @@ static void test_visitor_in_native_list_bool(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
 
-    for (i = 0, elem = cvalue->boolean; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.boolean.data; elem; elem = elem->next, i++) {
         g_assert_cmpint(elem->value, ==, (i % 3 == 0) ? 1 : 0);
     }
 
@@ -540,7 +651,6 @@ static void test_visitor_in_native_list_string(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     strList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -556,12 +666,11 @@ static void test_visitor_in_native_list_string(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
 
-    for (i = 0, elem = cvalue->string; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.string.data; elem; elem = elem->next, i++) {
         gchar str[8];
         sprintf(str, "%d", i);
         g_assert_cmpstr(elem->value, ==, str);
@@ -579,7 +688,6 @@ static void test_visitor_in_native_list_number(TestInputVisitorData *data,
 {
     UserDefNativeListUnion *cvalue = NULL;
     numberList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -595,12 +703,11 @@ static void test_visitor_in_native_list_number(TestInputVisitorData *data,
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
 
-    for (i = 0, elem = cvalue->number; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.number.data; elem; elem = elem->next, i++) {
         GString *double_expected = g_string_new("");
         GString *double_actual = g_string_new("");
 
@@ -631,16 +738,69 @@ static void test_visitor_in_errors(TestInputVisitorData *data,
     TestStruct *p = NULL;
     Error *err = NULL;
     Visitor *v;
+    strList *q = NULL;
 
-    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', 'string': -42 }");
+    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', "
+                                "'string': -42 }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    /* FIXME - a failed parse should not leave a partially-allocated p
+     * for us to clean up; this could cause callers to leak memory. */
     g_assert(p->string == NULL);
 
-    error_free(err);
     g_free(p->string);
     g_free(p);
+
+    v = visitor_input_test_init(data, "[ '1', '2', false, '3' ]");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(q);
+    qapi_free_strList(q);
+}
+
+static void test_visitor_in_wrong_type(TestInputVisitorData *data,
+                                       const void *unused)
+{
+    TestStruct *p = NULL;
+    Visitor *v;
+    strList *q = NULL;
+    int64_t i;
+    Error *err = NULL;
+
+    /* Make sure arrays and structs cannot be confused */
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    /* Make sure primitives and struct cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_int(v, NULL, &i, &err);
+    error_free_or_abort(&err);
+
+    /* Make sure primitives and arrays cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_int(v, NULL, &i, &err);
+    error_free_or_abort(&err);
 }
 
 int main(int argc, char **argv)
@@ -667,12 +827,18 @@ int main(int argc, char **argv)
                            &in_visitor_data, test_visitor_in_struct_nested);
     input_visitor_test_add("/visitor/input/list",
                            &in_visitor_data, test_visitor_in_list);
+    input_visitor_test_add("/visitor/input/any",
+                           &in_visitor_data, test_visitor_in_any);
     input_visitor_test_add("/visitor/input/union-flat",
                            &in_visitor_data, test_visitor_in_union_flat);
     input_visitor_test_add("/visitor/input/alternate",
                            &in_visitor_data, test_visitor_in_alternate);
     input_visitor_test_add("/visitor/input/errors",
                            &in_visitor_data, test_visitor_in_errors);
+    input_visitor_test_add("/visitor/input/wrong-type",
+                           &in_visitor_data, test_visitor_in_wrong_type);
+    input_visitor_test_add("/visitor/input/alternate-number",
+                           &in_visitor_data, test_visitor_in_alternate_number);
     input_visitor_test_add("/visitor/input/native_list/int",
                            &in_visitor_data,
                            test_visitor_in_native_list_int);
diff --git a/qemu/tests/test-qmp-output-visitor.c b/qemu/tests/test-qmp-output-visitor.c
index 87ba350b4..c70926793 100644
--- a/qemu/tests/test-qmp-output-visitor.c
+++ b/qemu/tests/test-qmp-output-visitor.c
@@ -1,7 +1,7 @@
 /*
  * QMP Output Visitor unit-tests.
  *
- * Copyright (C) 2011, 2015 Red Hat Inc.
+ * Copyright (C) 2011-2016 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -10,9 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
@@ -45,11 +47,9 @@ static void test_visitor_out_int(TestOutputVisitorData *data,
                                  const void *unused)
 {
     int64_t value = -42;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_int(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_int(data->ov, NULL, &value, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -62,12 +62,10 @@ static void test_visitor_out_int(TestOutputVisitorData *data,
 static void test_visitor_out_bool(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     bool value = true;
     QObject *obj;
 
-    visit_type_bool(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(data->ov, NULL, &value, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -81,11 +79,9 @@ static void test_visitor_out_number(TestOutputVisitorData *data,
                                     const void *unused)
 {
     double value = 3.14;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_number(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_number(data->ov, NULL, &value, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -99,11 +95,9 @@ static void test_visitor_out_string(TestOutputVisitorData *data,
                                     const void *unused)
 {
     char *string = (char *) "Q E M U";
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, NULL, &string, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -117,12 +111,10 @@ static void test_visitor_out_no_string(TestOutputVisitorData *data,
                                        const void *unused)
 {
     char *string = NULL;
-    Error *err = NULL;
     QObject *obj;
 
     /* A null string should return "" */
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, NULL, &string, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -135,13 +127,11 @@ static void test_visitor_out_no_string(TestOutputVisitorData *data,
 static void test_visitor_out_enum(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     QObject *obj;
     EnumOne i;
 
-    for (i = 0; i < ENUM_ONE_MAX; i++) {
-        visit_type_EnumOne(data->ov, &i, "unused", &err);
-        g_assert(!err);
+    for (i = 0; i < ENUM_ONE__MAX; i++) {
+        visit_type_EnumOne(data->ov, "unused", &i, &error_abort);
 
         obj = qmp_output_get_qobject(data->qov);
         g_assert(obj != NULL);
@@ -155,52 +145,17 @@ static void test_visitor_out_enum(TestOutputVisitorData *data,
 static void test_visitor_out_enum_errors(TestOutputVisitorData *data,
                                          const void *unused)
 {
-    EnumOne i, bad_values[] = { ENUM_ONE_MAX, -1 };
+    EnumOne i, bad_values[] = { ENUM_ONE__MAX, -1 };
     Error *err;
 
     for (i = 0; i < ARRAY_SIZE(bad_values) ; i++) {
         err = NULL;
-        visit_type_EnumOne(data->ov, &bad_values[i], "unused", &err);
+        visit_type_EnumOne(data->ov, "unused", &bad_values[i], &err);
         g_assert(err);
         error_free(err);
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_out_struct(TestOutputVisitorData *data,
                                     const void *unused)
@@ -209,12 +164,10 @@ static void test_visitor_out_struct(TestOutputVisitorData *data,
                                .boolean = false,
                                .string = (char *) "foo"};
     TestStruct *p = &test_struct;
-    Error *err = NULL;
     QObject *obj;
     QDict *qdict;
 
-    visit_type_TestStruct(data->ov, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(data->ov, NULL, &p, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -233,7 +186,6 @@ static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
                                            const void *unused)
 {
     int64_t value = 42;
-    Error *err = NULL;
     UserDefTwo *ud2;
     QObject *obj;
     QDict *qdict, *dict1, *dict2, *dict3, *userdef;
@@ -250,20 +202,17 @@ static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
     ud2->dict1->dict2 = g_malloc0(sizeof(*ud2->dict1->dict2));
     ud2->dict1->dict2->userdef = g_new0(UserDefOne, 1);
     ud2->dict1->dict2->userdef->string = g_strdup(string);
-    ud2->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-    ud2->dict1->dict2->userdef->base->integer = value;
+    ud2->dict1->dict2->userdef->integer = value;
     ud2->dict1->dict2->string = g_strdup(strings[2]);
 
     ud2->dict1->dict3 = g_malloc0(sizeof(*ud2->dict1->dict3));
     ud2->dict1->has_dict3 = true;
     ud2->dict1->dict3->userdef = g_new0(UserDefOne, 1);
     ud2->dict1->dict3->userdef->string = g_strdup(string);
-    ud2->dict1->dict3->userdef->base = g_new0(UserDefZero, 1);
-    ud2->dict1->dict3->userdef->base->integer = value;
+    ud2->dict1->dict3->userdef->integer = value;
     ud2->dict1->dict3->string = g_strdup(strings[3]);
 
-    visit_type_UserDefTwo(data->ov, &ud2, "unused", &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(data->ov, "unused", &ud2, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -300,9 +249,9 @@ static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
 static void test_visitor_out_struct_errors(TestOutputVisitorData *data,
                                            const void *unused)
 {
-    EnumOne bad_values[] = { ENUM_ONE_MAX, -1 };
-    UserDefZero b;
-    UserDefOne u = { .base = &b }, *pu = &u;
+    EnumOne bad_values[] = { ENUM_ONE__MAX, -1 };
+    UserDefOne u = {0};
+    UserDefOne *pu = &u;
     Error *err;
     int i;
 
@@ -310,63 +259,39 @@ static void test_visitor_out_struct_errors(TestOutputVisitorData *data,
         err = NULL;
         u.has_enum1 = true;
         u.enum1 = bad_values[i];
-        visit_type_UserDefOne(data->ov, &pu, "unused", &err);
+        visit_type_UserDefOne(data->ov, "unused", &pu, &err);
         g_assert(err);
         error_free(err);
     }
 }
 
-typedef struct TestStructList
-{
-    union {
-        TestStruct *value;
-        uint64_t padding;
-    };
-    struct TestStructList *next;
-} TestStructList;
-
-static void visit_type_TestStructList(Visitor *v, TestStructList **obj,
-                                      const char *name, Error **errp)
-{
-    GenericList *i, **head = (GenericList **)obj;
-
-    visit_start_list(v, name, errp);
-
-    for (*head = i = visit_next_list(v, head, errp); i; i = visit_next_list(v, &i, errp)) {
-        TestStructList *native_i = (TestStructList *)i;
-        visit_type_TestStruct(v, &native_i->value, NULL, errp);
-    }
-
-    visit_end_list(v, errp);
-}
 
 static void test_visitor_out_list(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    char *value_str = (char *) "list value";
+    const char *value_str = "list value";
     TestStructList *p, *head = NULL;
     const int max_items = 10;
     bool value_bool = true;
     int value_int = 10;
-    Error *err = NULL;
     QListEntry *entry;
     QObject *obj;
     QList *qlist;
     int i;
 
+    /* Build the list in reverse order... */
     for (i = 0; i < max_items; i++) {
         p = g_malloc0(sizeof(*p));
         p->value = g_malloc0(sizeof(*p->value));
-        p->value->integer = value_int;
+        p->value->integer = value_int + (max_items - i - 1);
         p->value->boolean = value_bool;
-        p->value->string = value_str;
+        p->value->string = g_strdup(value_str);
 
         p->next = head;
         head = p;
     }
 
-    visit_type_TestStructList(data->ov, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStructList(data->ov, NULL, &head, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
     g_assert(obj != NULL);
@@ -375,6 +300,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
     qlist = qobject_to_qlist(obj);
     g_assert(!qlist_empty(qlist));
 
+    /* ...and ensure that the visitor sees it in order */
     i = 0;
     QLIST_FOREACH_ENTRY(qlist, entry) {
         QDict *qdict;
@@ -382,7 +308,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
         g_assert(qobject_type(entry->value) == QTYPE_QDICT);
         qdict = qobject_to_qdict(entry->value);
         g_assert_cmpint(qdict_size(qdict), ==, 3);
-        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int);
+        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int + i);
         g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, value_bool);
         g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, value_str);
         i++;
@@ -390,13 +316,7 @@ static void test_visitor_out_list(TestOutputVisitorData *data,
     g_assert_cmpint(i, ==, max_items);
 
     QDECREF(qlist);
-
-    for (p = head; p;) {
-        TestStructList *tmp = p->next;
-        g_free(p->value);
-        g_free(p);
-        p = tmp;
-    }
+    qapi_free_TestStructList(head);
 }
 
 static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
@@ -416,8 +336,7 @@ static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
         p->value->dict1->dict2 = g_new0(UserDefTwoDictDict, 1);
         p->value->dict1->dict2->userdef = g_new0(UserDefOne, 1);
         p->value->dict1->dict2->userdef->string = g_strdup(string);
-        p->value->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-        p->value->dict1->dict2->userdef->base->integer = 42;
+        p->value->dict1->dict2->userdef->integer = 42;
         p->value->dict1->dict2->string = g_strdup(string);
         p->value->dict1->has_dict3 = false;
 
@@ -428,23 +347,67 @@ static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
     qapi_free_UserDefTwoList(head);
 }
 
+static void test_visitor_out_any(TestOutputVisitorData *data,
+                                 const void *unused)
+{
+    QObject *qobj;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *obj;
+
+    qobj = QOBJECT(qint_from_int(-42));
+    visit_type_any(data->ov, NULL, &qobj, &error_abort);
+    obj = qmp_output_get_qobject(data->qov);
+    g_assert(obj != NULL);
+    g_assert(qobject_type(obj) == QTYPE_QINT);
+    g_assert_cmpint(qint_get_int(qobject_to_qint(obj)), ==, -42);
+    qobject_decref(obj);
+    qobject_decref(qobj);
+
+    qdict = qdict_new();
+    qdict_put(qdict, "integer", qint_from_int(-42));
+    qdict_put(qdict, "boolean", qbool_from_bool(true));
+    qdict_put(qdict, "string", qstring_from_str("foo"));
+    qobj = QOBJECT(qdict);
+    visit_type_any(data->ov, NULL, &qobj, &error_abort);
+    qobject_decref(qobj);
+    obj = qmp_output_get_qobject(data->qov);
+    g_assert(obj != NULL);
+    qdict = qobject_to_qdict(obj);
+    g_assert(qdict);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
+    qobject_decref(obj);
+}
+
 static void test_visitor_out_union_flat(TestOutputVisitorData *data,
                                         const void *unused)
 {
     QObject *arg;
     QDict *qdict;
 
-    Error *err = NULL;
-
     UserDefFlatUnion *tmp = g_malloc0(sizeof(UserDefFlatUnion));
-    tmp->kind = ENUM_ONE_VALUE1;
+    tmp->enum1 = ENUM_ONE_VALUE1;
     tmp->string = g_strdup("str");
-    tmp->value1 = g_malloc0(sizeof(UserDefA));
-    /* TODO when generator bug is fixed: tmp->integer = 41; */
-    tmp->value1->boolean = true;
+    tmp->integer = 41;
+    tmp->u.value1.boolean = true;
 
-    visit_type_UserDefFlatUnion(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefFlatUnion(data->ov, NULL, &tmp, &error_abort);
     arg = qmp_output_get_qobject(data->qov);
 
     g_assert(qobject_type(arg) == QTYPE_QDICT);
@@ -452,7 +415,7 @@ static void test_visitor_out_union_flat(TestOutputVisitorData *data,
 
     g_assert_cmpstr(qdict_get_str(qdict, "enum1"), ==, "value1");
     g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "str");
-    /* TODO g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41); */
+    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41);
     g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, true);
 
     qapi_free_UserDefFlatUnion(tmp);
@@ -463,20 +426,55 @@ static void test_visitor_out_alternate(TestOutputVisitorData *data,
                                        const void *unused)
 {
     QObject *arg;
-    Error *err = NULL;
+    UserDefAlternate *tmp;
+    QDict *qdict;
 
-    UserDefAlternate *tmp = g_malloc0(sizeof(UserDefAlternate));
-    tmp->kind = USER_DEF_ALTERNATE_KIND_I;
-    tmp->i = 42;
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QINT;
+    tmp->u.i = 42;
 
-    visit_type_UserDefAlternate(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
     arg = qmp_output_get_qobject(data->qov);
 
     g_assert(qobject_type(arg) == QTYPE_QINT);
     g_assert_cmpint(qint_get_int(qobject_to_qint(arg)), ==, 42);
 
     qapi_free_UserDefAlternate(tmp);
+    qobject_decref(arg);
+
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QSTRING;
+    tmp->u.s = g_strdup("hello");
+
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
+    arg = qmp_output_get_qobject(data->qov);
+
+    g_assert(qobject_type(arg) == QTYPE_QSTRING);
+    g_assert_cmpstr(qstring_get_str(qobject_to_qstring(arg)), ==, "hello");
+
+    qapi_free_UserDefAlternate(tmp);
+    qobject_decref(arg);
+
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QDICT;
+    tmp->u.udfu.integer = 1;
+    tmp->u.udfu.string = g_strdup("str");
+    tmp->u.udfu.enum1 = ENUM_ONE_VALUE1;
+    tmp->u.udfu.u.value1.boolean = true;
+
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
+    arg = qmp_output_get_qobject(data->qov);
+
+    g_assert_cmpint(qobject_type(arg), ==, QTYPE_QDICT);
+    qdict = qobject_to_qdict(arg);
+    g_assert_cmpint(qdict_size(qdict), ==, 4);
+    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 1);
+    g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "str");
+    g_assert_cmpstr(qdict_get_str(qdict, "enum1"), ==, "value1");
+    g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, true);
+
+    qapi_free_UserDefAlternate(tmp);
+    qobject_decref(arg);
 }
 
 static void test_visitor_out_empty(TestOutputVisitorData *data,
@@ -485,15 +483,18 @@ static void test_visitor_out_empty(TestOutputVisitorData *data,
     QObject *arg;
 
     arg = qmp_output_get_qobject(data->qov);
-    g_assert(!arg);
+    g_assert(qobject_type(arg) == QTYPE_QNULL);
+    /* Check that qnull reference counting is sane */
+    g_assert(arg->refcnt == 2);
+    qobject_decref(arg);
 }
 
 static void init_native_list(UserDefNativeListUnion *cvalue)
 {
     int i;
-    switch (cvalue->kind) {
+    switch (cvalue->type) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
-        intList **list = &cvalue->integer;
+        intList **list = &cvalue->u.integer.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(intList, 1);
             (*list)->value = i;
@@ -503,7 +504,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
-        int8List **list = &cvalue->s8;
+        int8List **list = &cvalue->u.s8.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int8List, 1);
             (*list)->value = i;
@@ -513,7 +514,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
-        int16List **list = &cvalue->s16;
+        int16List **list = &cvalue->u.s16.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int16List, 1);
             (*list)->value = i;
@@ -523,7 +524,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
-        int32List **list = &cvalue->s32;
+        int32List **list = &cvalue->u.s32.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int32List, 1);
             (*list)->value = i;
@@ -533,7 +534,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
-        int64List **list = &cvalue->s64;
+        int64List **list = &cvalue->u.s64.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int64List, 1);
             (*list)->value = i;
@@ -543,7 +544,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
-        uint8List **list = &cvalue->u8;
+        uint8List **list = &cvalue->u.u8.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint8List, 1);
             (*list)->value = i;
@@ -553,7 +554,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
-        uint16List **list = &cvalue->u16;
+        uint16List **list = &cvalue->u.u16.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint16List, 1);
             (*list)->value = i;
@@ -563,7 +564,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
-        uint32List **list = &cvalue->u32;
+        uint32List **list = &cvalue->u.u32.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint32List, 1);
             (*list)->value = i;
@@ -573,7 +574,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
-        uint64List **list = &cvalue->u64;
+        uint64List **list = &cvalue->u.u64.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint64List, 1);
             (*list)->value = i;
@@ -583,7 +584,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN: {
-        boolList **list = &cvalue->boolean;
+        boolList **list = &cvalue->u.boolean.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(boolList, 1);
             (*list)->value = (i % 3 == 0);
@@ -593,7 +594,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_STRING: {
-        strList **list = &cvalue->string;
+        strList **list = &cvalue->u.string.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(strList, 1);
             (*list)->value = g_strdup_printf("%d", i);
@@ -603,7 +604,7 @@ static void init_native_list(UserDefNativeListUnion *cvalue)
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER: {
-        numberList **list = &cvalue->number;
+        numberList **list = &cvalue->u.number.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(numberList, 1);
             (*list)->value = (double)i / 3;
@@ -709,17 +710,15 @@ static void test_native_list(TestOutputVisitorData *data,
                              UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = g_new0(UserDefNativeListUnion, 1);
-    Error *err = NULL;
     QObject *obj;
 
-    cvalue->kind = kind;
+    cvalue->type = kind;
     init_native_list(cvalue);
 
-    visit_type_UserDefNativeListUnion(data->ov, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(data->ov, NULL, &cvalue, &error_abort);
 
     obj = qmp_output_get_qobject(data->qov);
-    check_native_list(obj, cvalue->kind);
+    check_native_list(obj, cvalue->type);
     qapi_free_UserDefNativeListUnion(cvalue);
     qobject_decref(obj);
 }
@@ -832,6 +831,8 @@ int main(int argc, char **argv)
                             &out_visitor_data, test_visitor_out_struct_errors);
     output_visitor_test_add("/visitor/output/list",
                             &out_visitor_data, test_visitor_out_list);
+    output_visitor_test_add("/visitor/output/any",
+                            &out_visitor_data, test_visitor_out_any);
     output_visitor_test_add("/visitor/output/list-qapi-free",
                             &out_visitor_data, test_visitor_out_list_qapi_free);
     output_visitor_test_add("/visitor/output/union-flat",
diff --git a/qemu/tests/test-rcu-list.c b/qemu/tests/test-rcu-list.c
index daa8bf41d..79d375014 100644
--- a/qemu/tests/test-rcu-list.c
+++ b/qemu/tests/test-rcu-list.c
@@ -20,14 +20,10 @@
  * Copyright (c) 2013 Mike D. Day, IBM Corporation.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
 #include "qemu/atomic.h"
 #include "qemu/rcu.h"
-#include "qemu/compiler.h"
-#include "qemu/osdep.h"
 #include "qemu/thread.h"
 #include "qemu/rcu_queue.h"
 
diff --git a/qemu/tests/test-rfifolock.c b/qemu/tests/test-rfifolock.c
index 0572ebb42..9a3cb243b 100644
--- a/qemu/tests/test-rfifolock.c
+++ b/qemu/tests/test-rfifolock.c
@@ -10,6 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qemu-common.h"
 #include "qemu/rfifolock.h"
diff --git a/qemu/tests/test-string-input-visitor.c b/qemu/tests/test-string-input-visitor.c
index 8e3433e0c..9e6906a56 100644
--- a/qemu/tests/test-string-input-visitor.c
+++ b/qemu/tests/test-string-input-visitor.c
@@ -10,10 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdarg.h>
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/string-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
@@ -59,7 +60,7 @@ static void test_visitor_in_int(TestInputVisitorData *data,
 
     v = visitor_input_test_init(data, "-42");
 
-    visit_type_int(v, &res, NULL, &err);
+    visit_type_int(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, value);
 }
@@ -74,7 +75,7 @@ static void test_visitor_in_intList(TestInputVisitorData *data,
 
     v = visitor_input_test_init(data, "1,2,0,2-4,20,5-9,1-8");
 
-    visit_type_int16List(v, &res, NULL, &error_abort);
+    visit_type_int16List(v, NULL, &res, &error_abort);
     tmp = res;
     while (i < sizeof(value) / sizeof(value[0])) {
         g_assert(tmp);
@@ -100,42 +101,42 @@ static void test_visitor_in_bool(TestInputVisitorData *data,
 
     v = visitor_input_test_init(data, "true");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "yes");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "on");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "false");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "no");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "off");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
 }
@@ -149,7 +150,7 @@ static void test_visitor_in_number(TestInputVisitorData *data,
 
     v = visitor_input_test_init(data, "3.14");
 
-    visit_type_number(v, &res, NULL, &err);
+    visit_type_number(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpfloat(res, ==, value);
 }
@@ -163,7 +164,7 @@ static void test_visitor_in_string(TestInputVisitorData *data,
 
     v = visitor_input_test_init(data, value);
 
-    visit_type_str(v, &res, NULL, &err);
+    visit_type_str(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpstr(res, ==, value);
 
@@ -182,7 +183,7 @@ static void test_visitor_in_enum(TestInputVisitorData *data,
 
         v = visitor_input_test_init(data, EnumOne_lookup[i]);
 
-        visit_type_EnumOne(v, &res, NULL, &err);
+        visit_type_EnumOne(v, NULL, &res, &err);
         g_assert(!err);
         g_assert_cmpint(i, ==, res);
 
@@ -220,29 +221,29 @@ static void test_visitor_in_fuzz(TestInputVisitorData *data,
         }
 
         v = visitor_input_test_init(data, buf);
-        visit_type_int(v, &ires, NULL, NULL);
+        visit_type_int(v, NULL, &ires, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_intList(v, &ilres, NULL, NULL);
+        visit_type_intList(v, NULL, &ilres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_bool(v, &bres, NULL, NULL);
+        visit_type_bool(v, NULL, &bres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_number(v, &nres, NULL, NULL);
+        visit_type_number(v, NULL, &nres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
         sres = NULL;
-        visit_type_str(v, &sres, NULL, NULL);
+        visit_type_str(v, NULL, &sres, NULL);
         g_free(sres);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_EnumOne(v, &eres, NULL, NULL);
+        visit_type_EnumOne(v, NULL, &eres, NULL);
         visitor_input_teardown(data, NULL);
     }
 }
diff --git a/qemu/tests/test-string-output-visitor.c b/qemu/tests/test-string-output-visitor.c
index 101fb27dd..1ecd75b85 100644
--- a/qemu/tests/test-string-output-visitor.c
+++ b/qemu/tests/test-string-output-visitor.c
@@ -10,9 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/string-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
@@ -61,7 +63,7 @@ static void test_visitor_out_int(TestOutputVisitorData *data,
     Error *err = NULL;
     char *str;
 
-    visit_type_int(data->ov, &value, NULL, &err);
+    visit_type_int(data->ov, NULL, &value, &err);
     g_assert(!err);
 
     str = string_output_get_string(data->sov);
@@ -81,7 +83,7 @@ static void test_visitor_out_intList(TestOutputVisitorData *data,
         3, 4, 5, 6, 11, 12, 13, 21, 22, INT64_MAX - 1, INT64_MAX};
     intList *list = NULL, **tmp = &list;
     int i;
-    Error *errp = NULL;
+    Error *err = NULL;
     char *str;
 
     for (i = 0; i < sizeof(value) / sizeof(value[0]); i++) {
@@ -90,8 +92,8 @@ static void test_visitor_out_intList(TestOutputVisitorData *data,
         tmp = &(*tmp)->next;
     }
 
-    visit_type_intList(data->ov, &list, NULL, &errp);
-    g_assert(errp == NULL);
+    visit_type_intList(data->ov, NULL, &list, &err);
+    g_assert(err == NULL);
 
     str = string_output_get_string(data->sov);
     g_assert(str != NULL);
@@ -120,7 +122,7 @@ static void test_visitor_out_bool(TestOutputVisitorData *data,
     bool value = true;
     char *str;
 
-    visit_type_bool(data->ov, &value, NULL, &err);
+    visit_type_bool(data->ov, NULL, &value, &err);
     g_assert(!err);
 
     str = string_output_get_string(data->sov);
@@ -136,7 +138,7 @@ static void test_visitor_out_number(TestOutputVisitorData *data,
     Error *err = NULL;
     char *str;
 
-    visit_type_number(data->ov, &value, NULL, &err);
+    visit_type_number(data->ov, NULL, &value, &err);
     g_assert(!err);
 
     str = string_output_get_string(data->sov);
@@ -153,7 +155,7 @@ static void test_visitor_out_string(TestOutputVisitorData *data,
     Error *err = NULL;
     char *str;
 
-    visit_type_str(data->ov, &string, NULL, &err);
+    visit_type_str(data->ov, NULL, &string, &err);
     g_assert(!err);
 
     str = string_output_get_string(data->sov);
@@ -174,7 +176,7 @@ static void test_visitor_out_no_string(TestOutputVisitorData *data,
     char *str;
 
     /* A null string should return "" */
-    visit_type_str(data->ov, &string, NULL, &err);
+    visit_type_str(data->ov, NULL, &string, &err);
     g_assert(!err);
 
     str = string_output_get_string(data->sov);
@@ -194,10 +196,10 @@ static void test_visitor_out_enum(TestOutputVisitorData *data,
     char *str;
     EnumOne i;
 
-    for (i = 0; i < ENUM_ONE_MAX; i++) {
+    for (i = 0; i < ENUM_ONE__MAX; i++) {
         char *str_human;
 
-        visit_type_EnumOne(data->ov, &i, "unused", &err);
+        visit_type_EnumOne(data->ov, "unused", &i, &err);
         g_assert(!err);
 
         str_human = g_strdup_printf("\"%s\"", EnumOne_lookup[i]);
@@ -217,12 +219,12 @@ static void test_visitor_out_enum(TestOutputVisitorData *data,
 static void test_visitor_out_enum_errors(TestOutputVisitorData *data,
                                          const void *unused)
 {
-    EnumOne i, bad_values[] = { ENUM_ONE_MAX, -1 };
+    EnumOne i, bad_values[] = { ENUM_ONE__MAX, -1 };
     Error *err;
 
     for (i = 0; i < ARRAY_SIZE(bad_values) ; i++) {
         err = NULL;
-        visit_type_EnumOne(data->ov, &bad_values[i], "unused", &err);
+        visit_type_EnumOne(data->ov, "unused", &bad_values[i], &err);
         g_assert(err);
         error_free(err);
     }
@@ -248,39 +250,39 @@ int main(int argc, char **argv)
 
     output_visitor_test_add("/string-visitor/output/int",
                             &out_visitor_data, test_visitor_out_int, false);
-    output_visitor_test_add("/string-visitor/output/int",
+    output_visitor_test_add("/string-visitor/output/int-human",
                             &out_visitor_data, test_visitor_out_int, true);
     output_visitor_test_add("/string-visitor/output/bool",
                             &out_visitor_data, test_visitor_out_bool, false);
-    output_visitor_test_add("/string-visitor/output/bool",
+    output_visitor_test_add("/string-visitor/output/bool-human",
                             &out_visitor_data, test_visitor_out_bool, true);
     output_visitor_test_add("/string-visitor/output/number",
                             &out_visitor_data, test_visitor_out_number, false);
-    output_visitor_test_add("/string-visitor/output/number",
+    output_visitor_test_add("/string-visitor/output/number-human",
                             &out_visitor_data, test_visitor_out_number, true);
     output_visitor_test_add("/string-visitor/output/string",
                             &out_visitor_data, test_visitor_out_string, false);
-    output_visitor_test_add("/string-visitor/output/string",
+    output_visitor_test_add("/string-visitor/output/string-human",
                             &out_visitor_data, test_visitor_out_string, true);
     output_visitor_test_add("/string-visitor/output/no-string",
                             &out_visitor_data, test_visitor_out_no_string,
                             false);
-    output_visitor_test_add("/string-visitor/output/no-string",
+    output_visitor_test_add("/string-visitor/output/no-string-human",
                             &out_visitor_data, test_visitor_out_no_string,
                             true);
     output_visitor_test_add("/string-visitor/output/enum",
                             &out_visitor_data, test_visitor_out_enum, false);
-    output_visitor_test_add("/string-visitor/output/enum",
+    output_visitor_test_add("/string-visitor/output/enum-human",
                             &out_visitor_data, test_visitor_out_enum, true);
     output_visitor_test_add("/string-visitor/output/enum-errors",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             false);
-    output_visitor_test_add("/string-visitor/output/enum-errors",
+    output_visitor_test_add("/string-visitor/output/enum-errors-human",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             true);
     output_visitor_test_add("/string-visitor/output/intList",
                             &out_visitor_data, test_visitor_out_intList, false);
-    output_visitor_test_add("/string-visitor/output/intList",
+    output_visitor_test_add("/string-visitor/output/intList-human",
                             &out_visitor_data, test_visitor_out_intList, true);
 
     g_test_run();
diff --git a/qemu/tests/test-thread-pool.c b/qemu/tests/test-thread-pool.c
index 6a0b9813f..88dc7316b 100644
--- a/qemu/tests/test-thread-pool.c
+++ b/qemu/tests/test-thread-pool.c
@@ -1,8 +1,10 @@
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "block/thread-pool.h"
 #include "block/block.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/error-report.h"
 
@@ -229,9 +231,7 @@ int main(int argc, char **argv)
 
     ctx = aio_context_new(&local_error);
     if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error, "Failed to create AIO Context: ");
         exit(1);
     }
     pool = aio_get_thread_pool(ctx);
diff --git a/qemu/tests/test-throttle.c b/qemu/tests/test-throttle.c
index 016844546..744a52436 100644
--- a/qemu/tests/test-throttle.c
+++ b/qemu/tests/test-throttle.c
@@ -12,9 +12,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include <math.h>
 #include "block/aio.h"
+#include "qapi/error.h"
 #include "qemu/throttle.h"
 #include "qemu/error-report.h"
 #include "block/throttle-groups.h"
@@ -34,6 +36,9 @@ static bool double_cmp(double x, double y)
 /* tests for single bucket operations */
 static void test_leak_bucket(void)
 {
+    throttle_config_init(&cfg);
+    bkt = cfg.buckets[THROTTLE_BPS_TOTAL];
+
     /* set initial value */
     bkt.avg = 150;
     bkt.max = 15;
@@ -56,13 +61,33 @@ static void test_leak_bucket(void)
     g_assert(bkt.avg == 150);
     g_assert(bkt.max == 15);
     g_assert(double_cmp(bkt.level, 0));
+
+    /* check that burst_level leaks correctly */
+    bkt.burst_level = 6;
+    bkt.max = 250;
+    bkt.burst_length = 2; /* otherwise burst_level will not leak */
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 3.5));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 1));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 0));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 0));
 }
 
 static void test_compute_wait(void)
 {
+    unsigned i;
     int64_t wait;
     int64_t result;
 
+    throttle_config_init(&cfg);
+    bkt = cfg.buckets[THROTTLE_BPS_TOTAL];
+
     /* no operation limit set */
     bkt.avg = 0;
     bkt.max = 15;
@@ -92,6 +117,27 @@ static void test_compute_wait(void)
     /* time required to do half an operation */
     result = (int64_t)  NANOSECONDS_PER_SECOND / 150 / 2;
     g_assert(wait == result);
+
+    /* Perform I/O for 2.2 seconds at a rate of bkt.max */
+    bkt.burst_length = 2;
+    bkt.level = 0;
+    bkt.avg = 10;
+    bkt.max = 200;
+    for (i = 0; i < 22; i++) {
+        double units = bkt.max / 10;
+        bkt.level += units;
+        bkt.burst_level += units;
+        throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 10);
+        wait = throttle_compute_wait(&bkt);
+        g_assert(double_cmp(bkt.burst_level, 0));
+        g_assert(double_cmp(bkt.level, (i + 1) * (bkt.max - bkt.avg) / 10));
+        /* We can do bursts for the 2 seconds we have configured in
+         * burst_length. We have 100 extra miliseconds of burst
+         * because bkt.level has been leaking during this time.
+         * After that, we have to wait. */
+        result = i < 21 ? 0 : 1.8 * NANOSECONDS_PER_SECOND;
+        g_assert(wait == result);
+    }
 }
 
 /* functions to test ThrottleState initialization/destroy methods */
@@ -221,6 +267,8 @@ static void set_cfg_value(bool is_max, int index, int value)
 {
     if (is_max) {
         cfg.buckets[index].max = value;
+        /* If max is set, avg should never be 0 */
+        cfg.buckets[index].avg = MAX(cfg.buckets[index].avg, 1);
     } else {
         cfg.buckets[index].avg = value;
     }
@@ -230,17 +278,17 @@ static void test_enabled(void)
 {
     int i;
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     g_assert(!throttle_enabled(&cfg));
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        memset(&cfg, 0, sizeof(cfg));
+        throttle_config_init(&cfg);
         set_cfg_value(false, i, 150);
         g_assert(throttle_enabled(&cfg));
     }
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        memset(&cfg, 0, sizeof(cfg));
+        throttle_config_init(&cfg);
         set_cfg_value(false, i, -150);
         g_assert(!throttle_enabled(&cfg));
     }
@@ -253,32 +301,32 @@ static void test_conflicts_for_one_set(bool is_max,
                                        int read,
                                        int write)
 {
-    memset(&cfg, 0, sizeof(cfg));
-    g_assert(!throttle_conflicting(&cfg));
+    throttle_config_init(&cfg);
+    g_assert(throttle_is_valid(&cfg, NULL));
 
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, read,  1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, write, 1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, read,  1);
     set_cfg_value(is_max, write, 1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
-    g_assert(!throttle_conflicting(&cfg));
+    g_assert(throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, read,  1);
     set_cfg_value(is_max, write, 1);
-    g_assert(!throttle_conflicting(&cfg));
+    g_assert(throttle_is_valid(&cfg, NULL));
 }
 
 static void test_conflicting_config(void)
@@ -312,9 +360,9 @@ static void test_is_valid_for_value(int value, bool should_be_valid)
     int is_max, index;
     for (is_max = 0; is_max < 2; is_max++) {
         for (index = 0; index < BUCKETS_COUNT; index++) {
-            memset(&cfg, 0, sizeof(cfg));
+            throttle_config_init(&cfg);
             set_cfg_value(is_max, index, value);
-            g_assert(throttle_is_valid(&cfg) == should_be_valid);
+            g_assert(throttle_is_valid(&cfg, NULL) == should_be_valid);
         }
     }
 }
@@ -329,6 +377,26 @@ static void test_is_valid(void)
     test_is_valid_for_value(1, true);
 }
 
+static void test_max_is_missing_limit(void)
+{
+    int i;
+
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        throttle_config_init(&cfg);
+        cfg.buckets[i].max = 100;
+        cfg.buckets[i].avg = 0;
+        g_assert(!throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 0;
+        g_assert(throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 100;
+        g_assert(throttle_is_valid(&cfg, NULL));
+    }
+}
+
 static void test_have_timer(void)
 {
     /* zero structures */
@@ -529,7 +597,7 @@ static void test_groups(void)
     g_assert(bdrv1->throttle_state == bdrv3->throttle_state);
 
     /* Setting the config of a group member affects the whole group */
-    memset(&cfg1, 0, sizeof(cfg1));
+    throttle_config_init(&cfg1);
     cfg1.buckets[THROTTLE_BPS_READ].avg  = 500000;
     cfg1.buckets[THROTTLE_BPS_WRITE].avg = 285000;
     cfg1.buckets[THROTTLE_OPS_READ].avg  = 20000;
@@ -561,21 +629,8 @@ static void test_groups(void)
 
 int main(int argc, char **argv)
 {
-    Error *local_error = NULL;
-
-    qemu_init_main_loop(&local_error);
+    qemu_init_main_loop(&error_fatal);
     ctx = qemu_get_aio_context();
-
-    if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     local_error ? error_get_pretty(local_error) :
-                     "Failed to initialize the QEMU main loop");
-        if (local_error) {
-            error_free(local_error);
-        }
-        exit(1);
-    }
-
     bdrv_init();
 
     do {} while (g_main_context_iteration(NULL, false));
@@ -591,6 +646,7 @@ int main(int argc, char **argv)
     g_test_add_func("/throttle/config/enabled",     test_enabled);
     g_test_add_func("/throttle/config/conflicting", test_conflicting_config);
     g_test_add_func("/throttle/config/is_valid",    test_is_valid);
+    g_test_add_func("/throttle/config/max",         test_max_is_missing_limit);
     g_test_add_func("/throttle/config_functions",   test_config_functions);
     g_test_add_func("/throttle/accounting",         test_accounting);
     g_test_add_func("/throttle/groups",             test_groups);
diff --git a/qemu/tests/test-timed-average.c b/qemu/tests/test-timed-average.c
new file mode 100644
index 000000000..1cc4ab302
--- /dev/null
+++ b/qemu/tests/test-timed-average.c
@@ -0,0 +1,90 @@
+/*
+ * Timed average computation tests
+ *
+ * Copyright Nodalink, EURL. 2014
+ *
+ * Authors:
+ *  Benoît Canet     <benoit.canet@nodalink.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "qemu/timed-average.h"
+
+/* This is the clock for QEMU_CLOCK_VIRTUAL */
+static int64_t my_clock_value;
+
+int64_t cpu_get_clock(void)
+{
+    return my_clock_value;
+}
+
+static void account(TimedAverage *ta)
+{
+    timed_average_account(ta, 1);
+    timed_average_account(ta, 5);
+    timed_average_account(ta, 2);
+    timed_average_account(ta, 4);
+    timed_average_account(ta, 3);
+}
+
+static void test_average(void)
+{
+    TimedAverage ta;
+    uint64_t result;
+    int i;
+
+    /* we will compute some average on a period of 1 second */
+    timed_average_init(&ta, QEMU_CLOCK_VIRTUAL, NANOSECONDS_PER_SECOND);
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+
+    my_clock_value += NANOSECONDS_PER_SECOND * 100;
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    /* tests in the same order as the header function declarations */
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/timed-average/average", test_average);
+    return g_test_run();
+}
+
diff --git a/qemu/tests/test-visitor-serialization.c b/qemu/tests/test-visitor-serialization.c
index fa86cae88..9adbc30a4 100644
--- a/qemu/tests/test-visitor-serialization.c
+++ b/qemu/tests/test-visitor-serialization.c
@@ -11,14 +11,14 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdlib.h>
-#include <stdint.h>
 #include <float.h>
 
 #include "qemu-common.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
+#include "qapi/error.h"
 #include "qapi/qmp/types.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
@@ -101,40 +101,40 @@ static void visit_primitive_type(Visitor *v, void **native, Error **errp)
     PrimitiveType *pt = *native;
     switch(pt->type) {
     case PTYPE_STRING:
-        visit_type_str(v, (char **)&pt->value.string, NULL, errp);
+        visit_type_str(v, NULL, (char **)&pt->value.string, errp);
         break;
     case PTYPE_BOOLEAN:
-        visit_type_bool(v, &pt->value.boolean, NULL, errp);
+        visit_type_bool(v, NULL, &pt->value.boolean, errp);
         break;
     case PTYPE_NUMBER:
-        visit_type_number(v, &pt->value.number, NULL, errp);
+        visit_type_number(v, NULL, &pt->value.number, errp);
         break;
     case PTYPE_INTEGER:
-        visit_type_int(v, &pt->value.integer, NULL, errp);
+        visit_type_int(v, NULL, &pt->value.integer, errp);
         break;
     case PTYPE_U8:
-        visit_type_uint8(v, &pt->value.u8, NULL, errp);
+        visit_type_uint8(v, NULL, &pt->value.u8, errp);
         break;
     case PTYPE_U16:
-        visit_type_uint16(v, &pt->value.u16, NULL, errp);
+        visit_type_uint16(v, NULL, &pt->value.u16, errp);
         break;
     case PTYPE_U32:
-        visit_type_uint32(v, &pt->value.u32, NULL, errp);
+        visit_type_uint32(v, NULL, &pt->value.u32, errp);
         break;
     case PTYPE_U64:
-        visit_type_uint64(v, &pt->value.u64, NULL, errp);
+        visit_type_uint64(v, NULL, &pt->value.u64, errp);
         break;
     case PTYPE_S8:
-        visit_type_int8(v, &pt->value.s8, NULL, errp);
+        visit_type_int8(v, NULL, &pt->value.s8, errp);
         break;
     case PTYPE_S16:
-        visit_type_int16(v, &pt->value.s16, NULL, errp);
+        visit_type_int16(v, NULL, &pt->value.s16, errp);
         break;
     case PTYPE_S32:
-        visit_type_int32(v, &pt->value.s32, NULL, errp);
+        visit_type_int32(v, NULL, &pt->value.s32, errp);
         break;
     case PTYPE_S64:
-        visit_type_int64(v, &pt->value.s64, NULL, errp);
+        visit_type_int64(v, NULL, &pt->value.s64, errp);
         break;
     case PTYPE_EOL:
         g_assert_not_reached();
@@ -146,80 +146,46 @@ static void visit_primitive_list(Visitor *v, void **native, Error **errp)
     PrimitiveList *pl = *native;
     switch (pl->type) {
     case PTYPE_STRING:
-        visit_type_strList(v, &pl->value.strings, NULL, errp);
+        visit_type_strList(v, NULL, &pl->value.strings, errp);
         break;
     case PTYPE_BOOLEAN:
-        visit_type_boolList(v, &pl->value.booleans, NULL, errp);
+        visit_type_boolList(v, NULL, &pl->value.booleans, errp);
         break;
     case PTYPE_NUMBER:
-        visit_type_numberList(v, &pl->value.numbers, NULL, errp);
+        visit_type_numberList(v, NULL, &pl->value.numbers, errp);
         break;
     case PTYPE_INTEGER:
-        visit_type_intList(v, &pl->value.integers, NULL, errp);
+        visit_type_intList(v, NULL, &pl->value.integers, errp);
         break;
     case PTYPE_S8:
-        visit_type_int8List(v, &pl->value.s8_integers, NULL, errp);
+        visit_type_int8List(v, NULL, &pl->value.s8_integers, errp);
         break;
     case PTYPE_S16:
-        visit_type_int16List(v, &pl->value.s16_integers, NULL, errp);
+        visit_type_int16List(v, NULL, &pl->value.s16_integers, errp);
         break;
     case PTYPE_S32:
-        visit_type_int32List(v, &pl->value.s32_integers, NULL, errp);
+        visit_type_int32List(v, NULL, &pl->value.s32_integers, errp);
         break;
     case PTYPE_S64:
-        visit_type_int64List(v, &pl->value.s64_integers, NULL, errp);
+        visit_type_int64List(v, NULL, &pl->value.s64_integers, errp);
         break;
     case PTYPE_U8:
-        visit_type_uint8List(v, &pl->value.u8_integers, NULL, errp);
+        visit_type_uint8List(v, NULL, &pl->value.u8_integers, errp);
         break;
     case PTYPE_U16:
-        visit_type_uint16List(v, &pl->value.u16_integers, NULL, errp);
+        visit_type_uint16List(v, NULL, &pl->value.u16_integers, errp);
         break;
     case PTYPE_U32:
-        visit_type_uint32List(v, &pl->value.u32_integers, NULL, errp);
+        visit_type_uint32List(v, NULL, &pl->value.u32_integers, errp);
         break;
     case PTYPE_U64:
-        visit_type_uint64List(v, &pl->value.u64_integers, NULL, errp);
+        visit_type_uint64List(v, NULL, &pl->value.u64_integers, errp);
         break;
     default:
         g_assert_not_reached();
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, NULL, name, sizeof(TestStruct), &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static TestStruct *struct_create(void)
 {
@@ -247,7 +213,7 @@ static void struct_cleanup(TestStruct *ts)
 
 static void visit_struct(Visitor *v, void **native, Error **errp)
 {
-    visit_type_TestStruct(v, (TestStruct **)native, NULL, errp);
+    visit_type_TestStruct(v, NULL, (TestStruct **)native, errp);
 }
 
 static UserDefTwo *nested_struct_create(void)
@@ -258,15 +224,13 @@ static UserDefTwo *nested_struct_create(void)
     udnp->dict1->string1 = strdup("test_string1");
     udnp->dict1->dict2 = g_malloc0(sizeof(*udnp->dict1->dict2));
     udnp->dict1->dict2->userdef = g_new0(UserDefOne, 1);
-    udnp->dict1->dict2->userdef->base = g_new0(UserDefZero, 1);
-    udnp->dict1->dict2->userdef->base->integer = 42;
+    udnp->dict1->dict2->userdef->integer = 42;
     udnp->dict1->dict2->userdef->string = strdup("test_string");
     udnp->dict1->dict2->string = strdup("test_string2");
     udnp->dict1->dict3 = g_malloc0(sizeof(*udnp->dict1->dict3));
     udnp->dict1->has_dict3 = true;
     udnp->dict1->dict3->userdef = g_new0(UserDefOne, 1);
-    udnp->dict1->dict3->userdef->base = g_new0(UserDefZero, 1);
-    udnp->dict1->dict3->userdef->base->integer = 43;
+    udnp->dict1->dict3->userdef->integer = 43;
     udnp->dict1->dict3->userdef->string = strdup("test_string");
     udnp->dict1->dict3->string = strdup("test_string3");
     return udnp;
@@ -278,15 +242,15 @@ static void nested_struct_compare(UserDefTwo *udnp1, UserDefTwo *udnp2)
     g_assert(udnp2);
     g_assert_cmpstr(udnp1->string0, ==, udnp2->string0);
     g_assert_cmpstr(udnp1->dict1->string1, ==, udnp2->dict1->string1);
-    g_assert_cmpint(udnp1->dict1->dict2->userdef->base->integer, ==,
-                    udnp2->dict1->dict2->userdef->base->integer);
+    g_assert_cmpint(udnp1->dict1->dict2->userdef->integer, ==,
+                    udnp2->dict1->dict2->userdef->integer);
     g_assert_cmpstr(udnp1->dict1->dict2->userdef->string, ==,
                     udnp2->dict1->dict2->userdef->string);
     g_assert_cmpstr(udnp1->dict1->dict2->string, ==,
                     udnp2->dict1->dict2->string);
     g_assert(udnp1->dict1->has_dict3 == udnp2->dict1->has_dict3);
-    g_assert_cmpint(udnp1->dict1->dict3->userdef->base->integer, ==,
-                    udnp2->dict1->dict3->userdef->base->integer);
+    g_assert_cmpint(udnp1->dict1->dict3->userdef->integer, ==,
+                    udnp2->dict1->dict3->userdef->integer);
     g_assert_cmpstr(udnp1->dict1->dict3->userdef->string, ==,
                     udnp2->dict1->dict3->userdef->string);
     g_assert_cmpstr(udnp1->dict1->dict3->string, ==,
@@ -300,12 +264,12 @@ static void nested_struct_cleanup(UserDefTwo *udnp)
 
 static void visit_nested_struct(Visitor *v, void **native, Error **errp)
 {
-    visit_type_UserDefTwo(v, (UserDefTwo **)native, NULL, errp);
+    visit_type_UserDefTwo(v, NULL, (UserDefTwo **)native, errp);
 }
 
 static void visit_nested_struct_list(Visitor *v, void **native, Error **errp)
 {
-    visit_type_UserDefTwoList(v, (UserDefTwoList **)native, NULL, errp);
+    visit_type_UserDefTwoList(v, NULL, (UserDefTwoList **)native, errp);
 }
 
 /* test cases */
@@ -338,14 +302,13 @@ static void test_primitives(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     PrimitiveType *pt = args->test_data;
     PrimitiveType *pt_copy = g_malloc0(sizeof(*pt_copy));
-    Error *err = NULL;
     void *serialize_data;
 
     pt_copy->type = pt->type;
-    ops->serialize(pt, &serialize_data, visit_primitive_type, &err);
-    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type, &err);
+    ops->serialize(pt, &serialize_data, visit_primitive_type, &error_abort);
+    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type,
+                     &error_abort);
 
-    g_assert(err == NULL);
     g_assert(pt_copy != NULL);
     if (pt->type == PTYPE_STRING) {
         g_assert_cmpstr(pt->value.string, ==, pt_copy->value.string);
@@ -381,7 +344,6 @@ static void test_primitive_lists(gconstpointer opaque)
     PrimitiveList pl = { .value = { NULL } };
     PrimitiveList pl_copy = { .value = { NULL } };
     PrimitiveList *pl_copy_ptr = &pl_copy;
-    Error *err = NULL;
     void *serialize_data;
     void *cur_head = NULL;
     int i;
@@ -528,10 +490,11 @@ static void test_primitive_lists(gconstpointer opaque)
         }
     }
 
-    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list, &err);
-    ops->deserialize((void **)&pl_copy_ptr, serialize_data, visit_primitive_list, &err);
+    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list,
+                   &error_abort);
+    ops->deserialize((void **)&pl_copy_ptr, serialize_data,
+                     visit_primitive_list, &error_abort);
 
-    g_assert(err == NULL);
     i = 0;
 
     /* compare our deserialized list of primitives to the original */
@@ -688,10 +651,8 @@ static void test_primitive_lists(gconstpointer opaque)
     g_assert_cmpint(i, ==, 33);
 
     ops->cleanup(serialize_data);
-    dealloc_helper(&pl, visit_primitive_list, &err);
-    g_assert(!err);
-    dealloc_helper(&pl_copy, visit_primitive_list, &err);
-    g_assert(!err);
+    dealloc_helper(&pl, visit_primitive_list, &error_abort);
+    dealloc_helper(&pl_copy, visit_primitive_list, &error_abort);
     g_free(args);
 }
 
@@ -701,13 +662,12 @@ static void test_struct(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     TestStruct *ts = struct_create();
     TestStruct *ts_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
 
-    ops->serialize(ts, &serialize_data, visit_struct, &err);
-    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct, &err); 
+    ops->serialize(ts, &serialize_data, visit_struct, &error_abort);
+    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct,
+                     &error_abort);
 
-    g_assert(err == NULL);
     struct_compare(ts, ts_copy);
 
     struct_cleanup(ts);
@@ -723,14 +683,12 @@ static void test_nested_struct(gconstpointer opaque)
     const SerializeOps *ops = args->ops;
     UserDefTwo *udnp = nested_struct_create();
     UserDefTwo *udnp_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
 
-    ops->serialize(udnp, &serialize_data, visit_nested_struct, &err);
+    ops->serialize(udnp, &serialize_data, visit_nested_struct, &error_abort);
     ops->deserialize((void **)&udnp_copy, serialize_data, visit_nested_struct,
-                     &err);
+                     &error_abort);
 
-    g_assert(err == NULL);
     nested_struct_compare(udnp, udnp_copy);
 
     nested_struct_cleanup(udnp);
@@ -745,7 +703,6 @@ static void test_nested_struct_list(gconstpointer opaque)
     TestArgs *args = (TestArgs *) opaque;
     const SerializeOps *ops = args->ops;
     UserDefTwoList *listp = NULL, *tmp, *tmp_copy, *listp_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
     int i = 0;
 
@@ -756,11 +713,10 @@ static void test_nested_struct_list(gconstpointer opaque)
         listp = tmp;
     }
 
-    ops->serialize(listp, &serialize_data, visit_nested_struct_list, &err);
+    ops->serialize(listp, &serialize_data, visit_nested_struct_list,
+                   &error_abort);
     ops->deserialize((void **)&listp_copy, serialize_data,
-                     visit_nested_struct_list, &err); 
-
-    g_assert(err == NULL);
+                     visit_nested_struct_list, &error_abort);
 
     tmp = listp;
     tmp_copy = listp_copy;
diff --git a/qemu/tests/test-vmstate.c b/qemu/tests/test-vmstate.c
index 1d620e04f..713d4443b 100644
--- a/qemu/tests/test-vmstate.c
+++ b/qemu/tests/test-vmstate.c
@@ -22,12 +22,13 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "qemu-common.h"
 #include "migration/migration.h"
 #include "migration/vmstate.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 
 static char temp_file[] = "/tmp/vmst.test.XXXXXX";
 static int temp_fd;
diff --git a/qemu/tests/test-write-threshold.c b/qemu/tests/test-write-threshold.c
index faffa7b85..fdbc8020f 100644
--- a/qemu/tests/test-write-threshold.c
+++ b/qemu/tests/test-write-threshold.c
@@ -6,8 +6,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <stdint.h>
 #include "block/block_int.h"
 #include "block/write-threshold.h"
 
diff --git a/qemu/tests/test-x86-cpuid.c b/qemu/tests/test-x86-cpuid.c
index 6cd20d4a2..8eb0bc6ad 100644
--- a/qemu/tests/test-x86-cpuid.c
+++ b/qemu/tests/test-x86-cpuid.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "hw/i386/topology.h"
diff --git a/qemu/tests/test-xbzrle.c b/qemu/tests/test-xbzrle.c
index db93b0a3d..49f64195a 100644
--- a/qemu/tests/test-xbzrle.c
+++ b/qemu/tests/test-xbzrle.c
@@ -10,14 +10,9 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <string.h>
-#include <sys/time.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "include/migration/migration.h"
 
 #define PAGE_SIZE 4096
diff --git a/qemu/tests/tmp105-test.c b/qemu/tests/tmp105-test.c
index 99db53819..235cae013 100644
--- a/qemu/tests/tmp105-test.c
+++ b/qemu/tests/tmp105-test.c
@@ -7,6 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "libqtest.h"
diff --git a/qemu/tests/tpci200-test.c b/qemu/tests/tpci200-test.c
index 9ae01277e..cb2b00ca8 100644
--- a/qemu/tests/tpci200-test.c
+++ b/qemu/tests/tpci200-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/usb-hcd-ehci-test.c b/qemu/tests/usb-hcd-ehci-test.c
index 75073bf24..a0f13ef40 100644
--- a/qemu/tests/usb-hcd-ehci-test.c
+++ b/qemu/tests/usb-hcd-ehci-test.c
@@ -7,12 +7,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
-#include <stdio.h>
 #include "libqtest.h"
 #include "libqos/pci-pc.h"
-#include "qemu/osdep.h"
 #include "hw/usb/uhci-regs.h"
 #include "hw/usb/ehci-regs.h"
 #include "libqos/usb.h"
diff --git a/qemu/tests/usb-hcd-ohci-test.c b/qemu/tests/usb-hcd-ohci-test.c
index fa592d487..efd6669c7 100644
--- a/qemu/tests/usb-hcd-ohci-test.c
+++ b/qemu/tests/usb-hcd-ohci-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "libqos/usb.h"
 
 
diff --git a/qemu/tests/usb-hcd-uhci-test.c b/qemu/tests/usb-hcd-uhci-test.c
index a96b71683..71ff2ea18 100644
--- a/qemu/tests/usb-hcd-uhci-test.c
+++ b/qemu/tests/usb-hcd-uhci-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "libqos/usb.h"
 #include "hw/usb/uhci-regs.h"
 
diff --git a/qemu/tests/usb-hcd-xhci-test.c b/qemu/tests/usb-hcd-xhci-test.c
index 56ab36731..7e2e212df 100644
--- a/qemu/tests/usb-hcd-xhci-test.c
+++ b/qemu/tests/usb-hcd-xhci-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "libqos/usb.h"
 
 
diff --git a/qemu/tests/vhost-user-bridge.c b/qemu/tests/vhost-user-bridge.c
new file mode 100644
index 000000000..0779ba260
--- /dev/null
+++ b/qemu/tests/vhost-user-bridge.c
@@ -0,0 +1,1418 @@
+/*
+ * Vhost User Bridge
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Victor Kaplansky <victork@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+/*
+ * TODO:
+ *     - main should get parameters from the command line.
+ *     - implement all request handlers. Still not implemented:
+ *          vubr_get_queue_num_exec()
+ *          vubr_send_rarp_exec()
+ *     - test for broken requests and virtqueue.
+ *     - implement features defined by Virtio 1.0 spec.
+ *     - support mergeable buffers and indirect descriptors.
+ *     - implement clean shutdown.
+ *     - implement non-blocking writes to UDP backend.
+ *     - implement polling strategy.
+ *     - implement clean starting/stopping of vq processing
+ *     - implement clean starting/stopping of used and buffers
+ *       dirty page logging.
+ */
+
+#define _FILE_OFFSET_BITS 64
+
+#include "qemu/osdep.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/unistd.h>
+#include <sys/mman.h>
+#include <sys/eventfd.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <qemu/osdep.h>
+
+#include <linux/vhost.h>
+
+#include "qemu/atomic.h"
+#include "standard-headers/linux/virtio_net.h"
+#include "standard-headers/linux/virtio_ring.h"
+
+#define VHOST_USER_BRIDGE_DEBUG 1
+
+#define DPRINT(...) \
+    do { \
+        if (VHOST_USER_BRIDGE_DEBUG) { \
+            printf(__VA_ARGS__); \
+        } \
+    } while (0)
+
+typedef void (*CallbackFunc)(int sock, void *ctx);
+
+typedef struct Event {
+    void *ctx;
+    CallbackFunc callback;
+} Event;
+
+typedef struct Dispatcher {
+    int max_sock;
+    fd_set fdset;
+    Event events[FD_SETSIZE];
+} Dispatcher;
+
+static void
+vubr_die(const char *s)
+{
+    perror(s);
+    exit(1);
+}
+
+static int
+dispatcher_init(Dispatcher *dispr)
+{
+    FD_ZERO(&dispr->fdset);
+    dispr->max_sock = -1;
+    return 0;
+}
+
+static int
+dispatcher_add(Dispatcher *dispr, int sock, void *ctx, CallbackFunc cb)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to add new event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    dispr->events[sock].ctx = ctx;
+    dispr->events[sock].callback = cb;
+
+    FD_SET(sock, &dispr->fdset);
+    if (sock > dispr->max_sock) {
+        dispr->max_sock = sock;
+    }
+    DPRINT("Added sock %d for watching. max_sock: %d\n",
+           sock, dispr->max_sock);
+    return 0;
+}
+
+/* dispatcher_remove() is not currently in use but may be useful
+ * in the future. */
+static int
+dispatcher_remove(Dispatcher *dispr, int sock)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to remove event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    FD_CLR(sock, &dispr->fdset);
+    DPRINT("Sock %d removed from dispatcher watch.\n", sock);
+    return 0;
+}
+
+/* timeout in us */
+static int
+dispatcher_wait(Dispatcher *dispr, uint32_t timeout)
+{
+    struct timeval tv;
+    tv.tv_sec = timeout / 1000000;
+    tv.tv_usec = timeout % 1000000;
+
+    fd_set fdset = dispr->fdset;
+
+    /* wait until some of sockets become readable. */
+    int rc = select(dispr->max_sock + 1, &fdset, 0, 0, &tv);
+
+    if (rc == -1) {
+        vubr_die("select");
+    }
+
+    /* Timeout */
+    if (rc == 0) {
+        return 0;
+    }
+
+    /* Now call callback for every ready socket. */
+
+    int sock;
+    for (sock = 0; sock < dispr->max_sock + 1; sock++) {
+        /* The callback on a socket can remove other sockets from the
+         * dispatcher, thus we have to check that the socket is
+         * still not removed from dispatcher's list
+         */
+        if (FD_ISSET(sock, &fdset) && FD_ISSET(sock, &dispr->fdset)) {
+            Event *e = &dispr->events[sock];
+            e->callback(sock, e->ctx);
+        }
+    }
+
+    return 0;
+}
+
+typedef struct VubrVirtq {
+    int call_fd;
+    int kick_fd;
+    uint32_t size;
+    uint16_t last_avail_index;
+    uint16_t last_used_index;
+    struct vring_desc *desc;
+    struct vring_avail *avail;
+    struct vring_used *used;
+    uint64_t log_guest_addr;
+    int enable;
+} VubrVirtq;
+
+/* Based on qemu/hw/virtio/vhost-user.c */
+
+#define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+/* v1.0 compliant. */
+#define VIRTIO_F_VERSION_1		32
+
+#define VHOST_LOG_PAGE 4096
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
+
+typedef enum VhostUserRequest {
+    VHOST_USER_NONE = 0,
+    VHOST_USER_GET_FEATURES = 1,
+    VHOST_USER_SET_FEATURES = 2,
+    VHOST_USER_SET_OWNER = 3,
+    VHOST_USER_RESET_OWNER = 4,
+    VHOST_USER_SET_MEM_TABLE = 5,
+    VHOST_USER_SET_LOG_BASE = 6,
+    VHOST_USER_SET_LOG_FD = 7,
+    VHOST_USER_SET_VRING_NUM = 8,
+    VHOST_USER_SET_VRING_ADDR = 9,
+    VHOST_USER_SET_VRING_BASE = 10,
+    VHOST_USER_GET_VRING_BASE = 11,
+    VHOST_USER_SET_VRING_KICK = 12,
+    VHOST_USER_SET_VRING_CALL = 13,
+    VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
+    VHOST_USER_MAX
+} VhostUserRequest;
+
+typedef struct VhostUserMemoryRegion {
+    uint64_t guest_phys_addr;
+    uint64_t memory_size;
+    uint64_t userspace_addr;
+    uint64_t mmap_offset;
+} VhostUserMemoryRegion;
+
+typedef struct VhostUserMemory {
+    uint32_t nregions;
+    uint32_t padding;
+    VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+} VhostUserMemory;
+
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
+typedef struct VhostUserMsg {
+    VhostUserRequest request;
+
+#define VHOST_USER_VERSION_MASK     (0x3)
+#define VHOST_USER_REPLY_MASK       (0x1<<2)
+    uint32_t flags;
+    uint32_t size; /* the following payload size */
+    union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
+        uint64_t u64;
+        struct vhost_vring_state state;
+        struct vhost_vring_addr addr;
+        VhostUserMemory memory;
+        VhostUserLog log;
+    } payload;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int fd_num;
+} QEMU_PACKED VhostUserMsg;
+
+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
+
+/* The version of the protocol we support */
+#define VHOST_USER_VERSION    (0x1)
+
+#define MAX_NR_VIRTQUEUE (8)
+
+typedef struct VubrDevRegion {
+    /* Guest Physical address. */
+    uint64_t gpa;
+    /* Memory region size. */
+    uint64_t size;
+    /* QEMU virtual address (userspace). */
+    uint64_t qva;
+    /* Starting offset in our mmaped space. */
+    uint64_t mmap_offset;
+    /* Start address of mmaped space. */
+    uint64_t mmap_addr;
+} VubrDevRegion;
+
+typedef struct VubrDev {
+    int sock;
+    Dispatcher dispatcher;
+    uint32_t nregions;
+    VubrDevRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+    VubrVirtq vq[MAX_NR_VIRTQUEUE];
+    int log_call_fd;
+    uint64_t log_size;
+    uint8_t *log_table;
+    int backend_udp_sock;
+    struct sockaddr_in backend_udp_dest;
+    int ready;
+    uint64_t features;
+    int hdrlen;
+} VubrDev;
+
+static const char *vubr_request_str[] = {
+    [VHOST_USER_NONE]                   =  "VHOST_USER_NONE",
+    [VHOST_USER_GET_FEATURES]           =  "VHOST_USER_GET_FEATURES",
+    [VHOST_USER_SET_FEATURES]           =  "VHOST_USER_SET_FEATURES",
+    [VHOST_USER_SET_OWNER]              =  "VHOST_USER_SET_OWNER",
+    [VHOST_USER_RESET_OWNER]           =  "VHOST_USER_RESET_OWNER",
+    [VHOST_USER_SET_MEM_TABLE]          =  "VHOST_USER_SET_MEM_TABLE",
+    [VHOST_USER_SET_LOG_BASE]           =  "VHOST_USER_SET_LOG_BASE",
+    [VHOST_USER_SET_LOG_FD]             =  "VHOST_USER_SET_LOG_FD",
+    [VHOST_USER_SET_VRING_NUM]          =  "VHOST_USER_SET_VRING_NUM",
+    [VHOST_USER_SET_VRING_ADDR]         =  "VHOST_USER_SET_VRING_ADDR",
+    [VHOST_USER_SET_VRING_BASE]         =  "VHOST_USER_SET_VRING_BASE",
+    [VHOST_USER_GET_VRING_BASE]         =  "VHOST_USER_GET_VRING_BASE",
+    [VHOST_USER_SET_VRING_KICK]         =  "VHOST_USER_SET_VRING_KICK",
+    [VHOST_USER_SET_VRING_CALL]         =  "VHOST_USER_SET_VRING_CALL",
+    [VHOST_USER_SET_VRING_ERR]          =  "VHOST_USER_SET_VRING_ERR",
+    [VHOST_USER_GET_PROTOCOL_FEATURES]  =  "VHOST_USER_GET_PROTOCOL_FEATURES",
+    [VHOST_USER_SET_PROTOCOL_FEATURES]  =  "VHOST_USER_SET_PROTOCOL_FEATURES",
+    [VHOST_USER_GET_QUEUE_NUM]          =  "VHOST_USER_GET_QUEUE_NUM",
+    [VHOST_USER_SET_VRING_ENABLE]       =  "VHOST_USER_SET_VRING_ENABLE",
+    [VHOST_USER_SEND_RARP]              =  "VHOST_USER_SEND_RARP",
+    [VHOST_USER_MAX]                    =  "VHOST_USER_MAX",
+};
+
+static void
+print_buffer(uint8_t *buf, size_t len)
+{
+    int i;
+    printf("Raw buffer:\n");
+    for (i = 0; i < len; i++) {
+        if (i % 16 == 0) {
+            printf("\n");
+        }
+        if (i % 4 == 0) {
+            printf("   ");
+        }
+        printf("%02x ", buf[i]);
+    }
+    printf("\n............................................................\n");
+}
+
+/* Translate guest physical address to our virtual address.  */
+static uint64_t
+gpa_to_va(VubrDev *dev, uint64_t guest_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
+            return guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+/* Translate qemu virtual address to our virtual address.  */
+static uint64_t
+qva_to_va(VubrDev *dev, uint64_t qemu_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
+            return qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+static void
+vubr_message_read(int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+
+    rc = recvmsg(conn_fd, &msg, 0);
+
+    if (rc == 0) {
+        vubr_die("recvmsg");
+        fprintf(stderr, "Peer disconnected.\n");
+        exit(1);
+    }
+    if (rc < 0) {
+        vubr_die("recvmsg");
+    }
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        fprintf(stderr,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        exit(1);
+    }
+
+    if (vmsg->size) {
+        rc = read(conn_fd, &vmsg->payload, vmsg->size);
+        if (rc == 0) {
+            vubr_die("recvmsg");
+            fprintf(stderr, "Peer disconnected.\n");
+            exit(1);
+        }
+        if (rc < 0) {
+            vubr_die("recvmsg");
+        }
+
+        assert(rc == vmsg->size);
+    }
+}
+
+static void
+vubr_message_write(int conn_fd, VhostUserMsg *vmsg)
+{
+    int rc;
+
+    do {
+        rc = write(conn_fd, vmsg, VHOST_USER_HDR_SIZE + vmsg->size);
+    } while (rc < 0 && errno == EINTR);
+
+    if (rc < 0) {
+        vubr_die("write");
+    }
+}
+
+static void
+vubr_backend_udp_sendbuf(VubrDev *dev, uint8_t *buf, size_t len)
+{
+    int slen = sizeof(struct sockaddr_in);
+
+    if (sendto(dev->backend_udp_sock, buf, len, 0,
+               (struct sockaddr *) &dev->backend_udp_dest, slen) == -1) {
+        vubr_die("sendto()");
+    }
+}
+
+static int
+vubr_backend_udp_recvbuf(VubrDev *dev, uint8_t *buf, size_t buflen)
+{
+    int slen = sizeof(struct sockaddr_in);
+    int rc;
+
+    rc = recvfrom(dev->backend_udp_sock, buf, buflen, 0,
+                  (struct sockaddr *) &dev->backend_udp_dest,
+                  (socklen_t *)&slen);
+    if (rc == -1) {
+        vubr_die("recvfrom()");
+    }
+
+    return rc;
+}
+
+static void
+vubr_consume_raw_packet(VubrDev *dev, uint8_t *buf, uint32_t len)
+{
+    int hdrlen = dev->hdrlen;
+    DPRINT("    hdrlen = %d\n", dev->hdrlen);
+
+    if (VHOST_USER_BRIDGE_DEBUG) {
+        print_buffer(buf, len);
+    }
+    vubr_backend_udp_sendbuf(dev, buf + hdrlen, len - hdrlen);
+}
+
+/* Kick the log_call_fd if required. */
+static void
+vubr_log_kick(VubrDev *dev)
+{
+    if (dev->log_call_fd != -1) {
+        DPRINT("Kicking the QEMU's log...\n");
+        eventfd_write(dev->log_call_fd, 1);
+    }
+}
+
+/* Kick the guest if necessary. */
+static void
+vubr_virtqueue_kick(VubrVirtq *vq)
+{
+    if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
+        DPRINT("Kicking the guest...\n");
+        eventfd_write(vq->call_fd, 1);
+    }
+}
+
+static void
+vubr_log_page(uint8_t *log_table, uint64_t page)
+{
+    DPRINT("Logged dirty guest page: %"PRId64"\n", page);
+    atomic_or(&log_table[page / 8], 1 << (page % 8));
+}
+
+static void
+vubr_log_write(VubrDev *dev, uint64_t address, uint64_t length)
+{
+    uint64_t page;
+
+    if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
+        !dev->log_table || !length) {
+        return;
+    }
+
+    assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
+
+    page = address / VHOST_LOG_PAGE;
+    while (page * VHOST_LOG_PAGE < address + length) {
+        vubr_log_page(dev->log_table, page);
+        page += VHOST_LOG_PAGE;
+    }
+    vubr_log_kick(dev);
+}
+
+static void
+vubr_post_buffer(VubrDev *dev, VubrVirtq *vq, uint8_t *buf, int32_t len)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+    int32_t remaining_len = len;
+
+    unsigned int size = vq->size;
+
+    uint16_t avail_index = atomic_mb_read(&avail->idx);
+
+    /* We check the available descriptors before posting the
+     * buffer, so here we assume that enough available
+     * descriptors. */
+    assert(vq->last_avail_index != avail_index);
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    int i = d_index;
+    uint32_t written_len = 0;
+
+    do {
+        DPRINT("Post packet to guest on vq:\n");
+        DPRINT("    size             = %d\n", vq->size);
+        DPRINT("    last_avail_index = %d\n", vq->last_avail_index);
+        DPRINT("    last_used_index  = %d\n", vq->last_used_index);
+        DPRINT("    a_index = %d\n", a_index);
+        DPRINT("    u_index = %d\n", u_index);
+        DPRINT("    d_index = %d\n", d_index);
+        DPRINT("    desc[%d].addr    = 0x%016"PRIx64"\n", i, desc[i].addr);
+        DPRINT("    desc[%d].len     = %d\n", i, desc[i].len);
+        DPRINT("    desc[%d].flags   = %d\n", i, desc[i].flags);
+        DPRINT("    avail->idx = %d\n", avail_index);
+        DPRINT("    used->idx  = %d\n", used->idx);
+
+        if (!(desc[i].flags & VRING_DESC_F_WRITE)) {
+            /* FIXME: we should find writable descriptor. */
+            fprintf(stderr, "Error: descriptor is not writable. Exiting.\n");
+            exit(1);
+        }
+
+        void *chunk_start = (void *)(uintptr_t)gpa_to_va(dev, desc[i].addr);
+        uint32_t chunk_len = desc[i].len;
+        uint32_t chunk_write_len = MIN(remaining_len, chunk_len);
+
+        memcpy(chunk_start, buf + written_len, chunk_write_len);
+        vubr_log_write(dev, desc[i].addr, chunk_write_len);
+        remaining_len -= chunk_write_len;
+        written_len += chunk_write_len;
+
+        if ((remaining_len == 0) || !(desc[i].flags & VRING_DESC_F_NEXT)) {
+            break;
+        }
+
+        i = desc[i].next;
+    } while (1);
+
+    if (remaining_len > 0) {
+            fprintf(stderr,
+                    "Too long packet for RX, remaining_len = %d, Dropping...\n",
+                    remaining_len);
+            return;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vq->last_avail_index++;
+    vq->last_used_index++;
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+
+    /* Kick the guest if necessary. */
+    vubr_virtqueue_kick(vq);
+}
+
+static int
+vubr_process_desc(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    unsigned int size = vq->size;
+
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    uint32_t i, len = 0;
+    size_t buf_size = 4096;
+    uint8_t buf[4096];
+
+    DPRINT("Chunks: ");
+    i = d_index;
+    do {
+        void *chunk_start = (void *)(uintptr_t)gpa_to_va(dev, desc[i].addr);
+        uint32_t chunk_len = desc[i].len;
+
+        assert(!(desc[i].flags & VRING_DESC_F_WRITE));
+
+        if (len + chunk_len < buf_size) {
+            memcpy(buf + len, chunk_start, chunk_len);
+            DPRINT("%d ", chunk_len);
+        } else {
+            fprintf(stderr, "Error: too long packet. Dropping...\n");
+            break;
+        }
+
+        len += chunk_len;
+
+        if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
+            break;
+        }
+
+        i = desc[i].next;
+    } while (1);
+    DPRINT("\n");
+
+    if (!len) {
+        return -1;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vubr_consume_raw_packet(dev, buf, len);
+
+    return 0;
+}
+
+static void
+vubr_process_avail(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    while (vq->last_avail_index != atomic_mb_read(&avail->idx)) {
+        vubr_process_desc(dev, vq);
+        vq->last_avail_index++;
+        vq->last_used_index++;
+    }
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+}
+
+static void
+vubr_backend_recv_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VubrVirtq *rx_vq = &dev->vq[0];
+    uint8_t buf[4096];
+    struct virtio_net_hdr_v1 *hdr = (struct virtio_net_hdr_v1 *)buf;
+    int hdrlen = dev->hdrlen;
+    int buflen = sizeof(buf);
+    int len;
+
+    if (!dev->ready) {
+        return;
+    }
+
+    DPRINT("\n\n   ***   IN UDP RECEIVE CALLBACK    ***\n\n");
+    DPRINT("    hdrlen = %d\n", hdrlen);
+
+    uint16_t avail_index = atomic_mb_read(&rx_vq->avail->idx);
+
+    /* If there is no available descriptors, just do nothing.
+     * The buffer will be handled by next arrived UDP packet,
+     * or next kick on receive virtq. */
+    if (rx_vq->last_avail_index == avail_index) {
+        DPRINT("Got UDP packet, but no available descriptors on RX virtq.\n");
+        return;
+    }
+
+    memset(buf, 0, hdrlen);
+    /* TODO: support mergeable buffers. */
+    if (hdrlen == 12)
+        hdr->num_buffers = 1;
+    len = vubr_backend_udp_recvbuf(dev, buf + hdrlen, buflen - hdrlen);
+
+    vubr_post_buffer(dev, rx_vq, buf, len + hdrlen);
+}
+
+static void
+vubr_kick_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        vubr_die("eventfd_read()");
+    } else {
+        DPRINT("Got kick_data: %016"PRIx64"\n", kick_data);
+        vubr_process_avail(dev, &dev->vq[1]);
+    }
+}
+
+static int
+vubr_none_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_get_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 =
+            ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
+             (1ULL << VHOST_F_LOG_ALL) |
+             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) |
+             (1ULL << VHOST_USER_F_PROTOCOL_FEATURES));
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    dev->features = vmsg->payload.u64;
+    if ((dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
+        (dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))) {
+        dev->hdrlen = 12;
+    } else {
+        dev->hdrlen = 10;
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_owner_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    return 0;
+}
+
+static void
+vubr_close_log(VubrDev *dev)
+{
+    if (dev->log_table) {
+        if (munmap(dev->log_table, dev->log_size) != 0) {
+            vubr_die("munmap()");
+        }
+
+        dev->log_table = 0;
+    }
+    if (dev->log_call_fd != -1) {
+        close(dev->log_call_fd);
+        dev->log_call_fd = -1;
+    }
+}
+
+static int
+vubr_reset_device_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vubr_close_log(dev);
+    dev->ready = 0;
+    dev->features = 0;
+    return 0;
+}
+
+static int
+vubr_set_mem_table_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int i;
+    VhostUserMemory *memory = &vmsg->payload.memory;
+    dev->nregions = memory->nregions;
+
+    DPRINT("Nregions: %d\n", memory->nregions);
+    for (i = 0; i < dev->nregions; i++) {
+        void *mmap_addr;
+        VhostUserMemoryRegion *msg_region = &memory->regions[i];
+        VubrDevRegion *dev_region = &dev->regions[i];
+
+        DPRINT("Region %d\n", i);
+        DPRINT("    guest_phys_addr: 0x%016"PRIx64"\n",
+               msg_region->guest_phys_addr);
+        DPRINT("    memory_size:     0x%016"PRIx64"\n",
+               msg_region->memory_size);
+        DPRINT("    userspace_addr   0x%016"PRIx64"\n",
+               msg_region->userspace_addr);
+        DPRINT("    mmap_offset      0x%016"PRIx64"\n",
+               msg_region->mmap_offset);
+
+        dev_region->gpa = msg_region->guest_phys_addr;
+        dev_region->size = msg_region->memory_size;
+        dev_region->qva = msg_region->userspace_addr;
+        dev_region->mmap_offset = msg_region->mmap_offset;
+
+        /* We don't use offset argument of mmap() since the
+         * mapped address has to be page aligned, and we use huge
+         * pages.  */
+        mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
+                         PROT_READ | PROT_WRITE, MAP_SHARED,
+                         vmsg->fds[i], 0);
+
+        if (mmap_addr == MAP_FAILED) {
+            vubr_die("mmap");
+        }
+        dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
+        DPRINT("    mmap_addr:       0x%016"PRIx64"\n", dev_region->mmap_addr);
+
+        close(vmsg->fds[i]);
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_log_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int fd;
+    uint64_t log_mmap_size, log_mmap_offset;
+    void *rc;
+
+    assert(vmsg->fd_num == 1);
+    fd = vmsg->fds[0];
+
+    assert(vmsg->size == sizeof(vmsg->payload.log));
+    log_mmap_offset = vmsg->payload.log.mmap_offset;
+    log_mmap_size = vmsg->payload.log.mmap_size;
+    DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
+    DPRINT("Log mmap_size:   %"PRId64"\n", log_mmap_size);
+
+    rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+              log_mmap_offset);
+    if (rc == MAP_FAILED) {
+        vubr_die("mmap");
+    }
+    dev->log_table = rc;
+    dev->log_size = log_mmap_size;
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_log_fd_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    assert(vmsg->fd_num == 1);
+    dev->log_call_fd = vmsg->fds[0];
+    DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
+    return 0;
+}
+
+static int
+vubr_set_vring_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].size = num;
+    return 0;
+}
+
+static int
+vubr_set_vring_addr_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    struct vhost_vring_addr *vra = &vmsg->payload.addr;
+    unsigned int index = vra->index;
+    VubrVirtq *vq = &dev->vq[index];
+
+    DPRINT("vhost_vring_addr:\n");
+    DPRINT("    index:  %d\n", vra->index);
+    DPRINT("    flags:  %d\n", vra->flags);
+    DPRINT("    desc_user_addr:   0x%016llx\n", vra->desc_user_addr);
+    DPRINT("    used_user_addr:   0x%016llx\n", vra->used_user_addr);
+    DPRINT("    avail_user_addr:  0x%016llx\n", vra->avail_user_addr);
+    DPRINT("    log_guest_addr:   0x%016llx\n", vra->log_guest_addr);
+
+    vq->desc = (struct vring_desc *)(uintptr_t)qva_to_va(dev, vra->desc_user_addr);
+    vq->used = (struct vring_used *)(uintptr_t)qva_to_va(dev, vra->used_user_addr);
+    vq->avail = (struct vring_avail *)(uintptr_t)qva_to_va(dev, vra->avail_user_addr);
+    vq->log_guest_addr = vra->log_guest_addr;
+
+    DPRINT("Setting virtq addresses:\n");
+    DPRINT("    vring_desc  at %p\n", vq->desc);
+    DPRINT("    vring_used  at %p\n", vq->used);
+    DPRINT("    vring_avail at %p\n", vq->avail);
+
+    vq->last_used_index = vq->used->idx;
+    return 0;
+}
+
+static int
+vubr_set_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].last_avail_index = num;
+
+    return 0;
+}
+
+static int
+vubr_get_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+
+    DPRINT("State.index: %d\n", index);
+    vmsg->payload.state.num = dev->vq[index].last_avail_index;
+    vmsg->size = sizeof(vmsg->payload.state);
+    /* FIXME: this is a work-around for a bug in QEMU enabling
+     * too early vrings. When protocol features are enabled,
+     * we have to respect * VHOST_USER_SET_VRING_ENABLE request. */
+    dev->ready = 0;
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+        dev->vq[index].call_fd = -1;
+    }
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+        dev->vq[index].kick_fd = -1;
+    }
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_vring_kick_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+    }
+    dev->vq[index].kick_fd = vmsg->fds[0];
+    DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    if (index % 2 == 1) {
+        /* TX queue. */
+        dispatcher_add(&dev->dispatcher, dev->vq[index].kick_fd,
+                       dev, vubr_kick_cb);
+
+        DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
+               dev->vq[index].kick_fd, index);
+    }
+    /* We temporarily use this hack to determine that both TX and RX
+     * queues are set up and ready for processing.
+     * FIXME: we need to rely in VHOST_USER_SET_VRING_ENABLE and
+     * actual kicks. */
+    if (dev->vq[0].kick_fd != -1 &&
+        dev->vq[1].kick_fd != -1) {
+        dev->ready = 1;
+        DPRINT("vhost-user-bridge is ready for processing queues.\n");
+    }
+    return 0;
+
+}
+
+static int
+vubr_set_vring_call_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+    }
+    dev->vq[index].call_fd = vmsg->fds[0];
+    DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    return 0;
+}
+
+static int
+vubr_set_vring_err_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* FIXME: unimplented */
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_queue_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_set_vring_enable_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int enable = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.enable:   %d\n", enable);
+    dev->vq[index].enable = enable;
+    return 0;
+}
+
+static int
+vubr_send_rarp_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_execute_request(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* Print out generic part of the request. */
+    DPRINT(
+           "==================   Vhost user message from QEMU   ==================\n");
+    DPRINT("Request: %s (%d)\n", vubr_request_str[vmsg->request],
+           vmsg->request);
+    DPRINT("Flags:   0x%x\n", vmsg->flags);
+    DPRINT("Size:    %d\n", vmsg->size);
+
+    if (vmsg->fd_num) {
+        int i;
+        DPRINT("Fds:");
+        for (i = 0; i < vmsg->fd_num; i++) {
+            DPRINT(" %d", vmsg->fds[i]);
+        }
+        DPRINT("\n");
+    }
+
+    switch (vmsg->request) {
+    case VHOST_USER_NONE:
+        return vubr_none_exec(dev, vmsg);
+    case VHOST_USER_GET_FEATURES:
+        return vubr_get_features_exec(dev, vmsg);
+    case VHOST_USER_SET_FEATURES:
+        return vubr_set_features_exec(dev, vmsg);
+    case VHOST_USER_SET_OWNER:
+        return vubr_set_owner_exec(dev, vmsg);
+    case VHOST_USER_RESET_OWNER:
+        return vubr_reset_device_exec(dev, vmsg);
+    case VHOST_USER_SET_MEM_TABLE:
+        return vubr_set_mem_table_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_BASE:
+        return vubr_set_log_base_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_FD:
+        return vubr_set_log_fd_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_NUM:
+        return vubr_set_vring_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ADDR:
+        return vubr_set_vring_addr_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_BASE:
+        return vubr_set_vring_base_exec(dev, vmsg);
+    case VHOST_USER_GET_VRING_BASE:
+        return vubr_get_vring_base_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_KICK:
+        return vubr_set_vring_kick_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_CALL:
+        return vubr_set_vring_call_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ERR:
+        return vubr_set_vring_err_exec(dev, vmsg);
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        return vubr_get_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_SET_PROTOCOL_FEATURES:
+        return vubr_set_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_GET_QUEUE_NUM:
+        return vubr_get_queue_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ENABLE:
+        return vubr_set_vring_enable_exec(dev, vmsg);
+    case VHOST_USER_SEND_RARP:
+        return vubr_send_rarp_exec(dev, vmsg);
+
+    case VHOST_USER_MAX:
+        assert(vmsg->request != VHOST_USER_MAX);
+    }
+    return 0;
+}
+
+static void
+vubr_receive_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VhostUserMsg vmsg;
+    int reply_requested;
+
+    vubr_message_read(sock, &vmsg);
+    reply_requested = vubr_execute_request(dev, &vmsg);
+    if (reply_requested) {
+        /* Set the version in the flags when sending the reply */
+        vmsg.flags &= ~VHOST_USER_VERSION_MASK;
+        vmsg.flags |= VHOST_USER_VERSION;
+        vmsg.flags |= VHOST_USER_REPLY_MASK;
+        vubr_message_write(sock, &vmsg);
+    }
+}
+
+static void
+vubr_accept_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *)ctx;
+    int conn_fd;
+    struct sockaddr_un un;
+    socklen_t len = sizeof(un);
+
+    conn_fd = accept(sock, (struct sockaddr *) &un, &len);
+    if (conn_fd == -1) {
+        vubr_die("accept()");
+    }
+    DPRINT("Got connection from remote peer on sock %d\n", conn_fd);
+    dispatcher_add(&dev->dispatcher, conn_fd, ctx, vubr_receive_cb);
+}
+
+static VubrDev *
+vubr_new(const char *path)
+{
+    VubrDev *dev = (VubrDev *) calloc(1, sizeof(VubrDev));
+    dev->nregions = 0;
+    int i;
+    struct sockaddr_un un;
+    size_t len;
+
+    for (i = 0; i < MAX_NR_VIRTQUEUE; i++) {
+        dev->vq[i] = (VubrVirtq) {
+            .call_fd = -1, .kick_fd = -1,
+            .size = 0,
+            .last_avail_index = 0, .last_used_index = 0,
+            .desc = 0, .avail = 0, .used = 0,
+            .enable = 0,
+        };
+    }
+
+    /* Init log */
+    dev->log_call_fd = -1;
+    dev->log_size = 0;
+    dev->log_table = 0;
+    dev->ready = 0;
+    dev->features = 0;
+
+    /* Get a UNIX socket. */
+    dev->sock = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (dev->sock == -1) {
+        vubr_die("socket");
+    }
+
+    un.sun_family = AF_UNIX;
+    strcpy(un.sun_path, path);
+    len = sizeof(un.sun_family) + strlen(path);
+    unlink(path);
+
+    if (bind(dev->sock, (struct sockaddr *) &un, len) == -1) {
+        vubr_die("bind");
+    }
+
+    if (listen(dev->sock, 1) == -1) {
+        vubr_die("listen");
+    }
+
+    dispatcher_init(&dev->dispatcher);
+    dispatcher_add(&dev->dispatcher, dev->sock, (void *)dev,
+                   vubr_accept_cb);
+
+    DPRINT("Waiting for connections on UNIX socket %s ...\n", path);
+    return dev;
+}
+
+static void
+vubr_set_host(struct sockaddr_in *saddr, const char *host)
+{
+    if (isdigit(host[0])) {
+        if (!inet_aton(host, &saddr->sin_addr)) {
+            fprintf(stderr, "inet_aton() failed.\n");
+            exit(1);
+        }
+    } else {
+        struct hostent *he = gethostbyname(host);
+
+        if (!he) {
+            fprintf(stderr, "gethostbyname() failed.\n");
+            exit(1);
+        }
+        saddr->sin_addr = *(struct in_addr *)he->h_addr;
+    }
+}
+
+static void
+vubr_backend_udp_setup(VubrDev *dev,
+                       const char *local_host,
+                       const char *local_port,
+                       const char *remote_host,
+                       const char *remote_port)
+{
+    int sock;
+    const char *r;
+
+    int lport, rport;
+
+    lport = strtol(local_port, (char **)&r, 0);
+    if (r == local_port) {
+        fprintf(stderr, "lport parsing failed.\n");
+        exit(1);
+    }
+
+    rport = strtol(remote_port, (char **)&r, 0);
+    if (r == remote_port) {
+        fprintf(stderr, "rport parsing failed.\n");
+        exit(1);
+    }
+
+    struct sockaddr_in si_local = {
+        .sin_family = AF_INET,
+        .sin_port = htons(lport),
+    };
+
+    vubr_set_host(&si_local, local_host);
+
+    /* setup destination for sends */
+    dev->backend_udp_dest = (struct sockaddr_in) {
+        .sin_family = AF_INET,
+        .sin_port = htons(rport),
+    };
+    vubr_set_host(&dev->backend_udp_dest, remote_host);
+
+    sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    if (sock == -1) {
+        vubr_die("socket");
+    }
+
+    if (bind(sock, (struct sockaddr *)&si_local, sizeof(si_local)) == -1) {
+        vubr_die("bind");
+    }
+
+    dev->backend_udp_sock = sock;
+    dispatcher_add(&dev->dispatcher, sock, dev, vubr_backend_recv_cb);
+    DPRINT("Waiting for data from udp backend on %s:%d...\n",
+           local_host, lport);
+}
+
+static void
+vubr_run(VubrDev *dev)
+{
+    while (1) {
+        /* timeout 200ms */
+        dispatcher_wait(&dev->dispatcher, 200000);
+        /* Here one can try polling strategy. */
+    }
+}
+
+static int
+vubr_parse_host_port(const char **host, const char **port, const char *buf)
+{
+    char *p = strchr(buf, ':');
+
+    if (!p) {
+        return -1;
+    }
+    *p = '\0';
+    *host = strdup(buf);
+    *port = strdup(p + 1);
+    return 0;
+}
+
+#define DEFAULT_UD_SOCKET "/tmp/vubr.sock"
+#define DEFAULT_LHOST "127.0.0.1"
+#define DEFAULT_LPORT "4444"
+#define DEFAULT_RHOST "127.0.0.1"
+#define DEFAULT_RPORT "5555"
+
+static const char *ud_socket_path = DEFAULT_UD_SOCKET;
+static const char *lhost = DEFAULT_LHOST;
+static const char *lport = DEFAULT_LPORT;
+static const char *rhost = DEFAULT_RHOST;
+static const char *rport = DEFAULT_RPORT;
+
+int
+main(int argc, char *argv[])
+{
+    VubrDev *dev;
+    int opt;
+
+    while ((opt = getopt(argc, argv, "l:r:u:")) != -1) {
+
+        switch (opt) {
+        case 'l':
+            if (vubr_parse_host_port(&lhost, &lport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'r':
+            if (vubr_parse_host_port(&rhost, &rport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'u':
+            ud_socket_path = strdup(optarg);
+            break;
+        default:
+            goto out;
+        }
+    }
+
+    DPRINT("ud socket: %s\n", ud_socket_path);
+    DPRINT("local:     %s:%s\n", lhost, lport);
+    DPRINT("remote:    %s:%s\n", rhost, rport);
+
+    dev = vubr_new(ud_socket_path);
+    if (!dev) {
+        return 1;
+    }
+
+    vubr_backend_udp_setup(dev, lhost, lport, rhost, rport);
+    vubr_run(dev);
+    return 0;
+
+out:
+    fprintf(stderr, "Usage: %s ", argv[0]);
+    fprintf(stderr, "[-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
+    fprintf(stderr, "\t-u path to unix doman socket. default: %s\n",
+            DEFAULT_UD_SOCKET);
+    fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
+            DEFAULT_LHOST, DEFAULT_LPORT);
+    fprintf(stderr, "\t-r remote host and port. default: %s:%s\n",
+            DEFAULT_RHOST, DEFAULT_RPORT);
+
+    return 1;
+}
diff --git a/qemu/tests/vhost-user-test.c b/qemu/tests/vhost-user-test.c
index 75fedf097..69615968c 100644
--- a/qemu/tests/vhost-user-test.c
+++ b/qemu/tests/vhost-user-test.c
@@ -8,11 +8,12 @@
  *
  */
 
-#define QEMU_GLIB_COMPAT_H
+#include "qemu/osdep.h"
 #include <glib.h>
 
 #include "libqtest.h"
 #include "qemu/option.h"
+#include "qemu/range.h"
 #include "sysemu/char.h"
 #include "sysemu/sysemu.h"
 
@@ -30,22 +31,15 @@
 #define HAVE_MONOTONIC_TIME
 #endif
 
-#if GLIB_CHECK_VERSION(2, 32, 0)
-#define HAVE_MUTEX_INIT
-#define HAVE_COND_INIT
-#define HAVE_THREAD_NEW
-#endif
-
 #define QEMU_CMD_ACCEL  " -machine accel=tcg"
-#define QEMU_CMD_MEM    " -m 512 -object memory-backend-file,id=mem,size=512M,"\
+#define QEMU_CMD_MEM    " -m %d -object memory-backend-file,id=mem,size=%dM,"\
                         "mem-path=%s,share=on -numa node,memdev=mem"
-#define QEMU_CMD_CHR    " -chardev socket,id=chr0,path=%s"
-#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=chr0,vhostforce"
-#define QEMU_CMD_NET    " -device virtio-net-pci,netdev=net0 "
-#define QEMU_CMD_ROM    " -option-rom ../pc-bios/pxe-virtio.rom"
+#define QEMU_CMD_CHR    " -chardev socket,id=%s,path=%s"
+#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=%s,vhostforce"
+#define QEMU_CMD_NET    " -device virtio-net-pci,netdev=net0,romfile=./pc-bios/pxe-virtio.rom"
 
 #define QEMU_CMD        QEMU_CMD_ACCEL QEMU_CMD_MEM QEMU_CMD_CHR \
-                        QEMU_CMD_NETDEV QEMU_CMD_NET QEMU_CMD_ROM
+                        QEMU_CMD_NETDEV QEMU_CMD_NET
 
 #define HUGETLBFS_MAGIC       0x958458f6
 
@@ -53,6 +47,11 @@
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
 
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
+
+#define VHOST_LOG_PAGE 0x1000
+
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
     VHOST_USER_GET_FEATURES = 1,
@@ -69,6 +68,9 @@ typedef enum VhostUserRequest {
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_SET_VRING_ENABLE = 18,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -85,6 +87,11 @@ typedef struct VhostUserMemory {
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -93,11 +100,14 @@ typedef struct VhostUserMsg {
     uint32_t flags;
     uint32_t size; /* the following payload size */
     union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
         uint64_t u64;
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -111,154 +121,104 @@ static VhostUserMsg m __attribute__ ((unused));
 #define VHOST_USER_VERSION    (0x1)
 /*****************************************************************************/
 
-int fds_num = 0, fds[VHOST_MEMORY_MAX_NREGIONS];
-static VhostUserMemory memory;
-static GMutex *data_mutex;
-static GCond *data_cond;
-
-static gint64 _get_time(void)
-{
-#ifdef HAVE_MONOTONIC_TIME
-    return g_get_monotonic_time();
-#else
-    GTimeVal time;
-    g_get_current_time(&time);
-
-    return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
-#endif
-}
-
-static GMutex *_mutex_new(void)
-{
-    GMutex *mutex;
-
-#ifdef HAVE_MUTEX_INIT
-    mutex = g_new(GMutex, 1);
-    g_mutex_init(mutex);
-#else
-    mutex = g_mutex_new();
-#endif
-
-    return mutex;
-}
-
-static void _mutex_free(GMutex *mutex)
-{
-#ifdef HAVE_MUTEX_INIT
-    g_mutex_clear(mutex);
-    g_free(mutex);
-#else
-    g_mutex_free(mutex);
-#endif
-}
-
-static GCond *_cond_new(void)
-{
-    GCond *cond;
-
-#ifdef HAVE_COND_INIT
-    cond = g_new(GCond, 1);
-    g_cond_init(cond);
-#else
-    cond = g_cond_new();
-#endif
-
-    return cond;
-}
-
-static gboolean _cond_wait_until(GCond *cond, GMutex *mutex, gint64 end_time)
+typedef struct TestServer {
+    gchar *socket_path;
+    gchar *mig_path;
+    gchar *chr_name;
+    CharDriverState *chr;
+    int fds_num;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    VhostUserMemory memory;
+    GMutex data_mutex;
+    GCond data_cond;
+    int log_fd;
+    uint64_t rings;
+} TestServer;
+
+#if !GLIB_CHECK_VERSION(2, 32, 0)
+static gboolean g_cond_wait_until(CompatGCond cond, CompatGMutex mutex,
+                                  gint64 end_time)
 {
     gboolean ret = FALSE;
-#ifdef HAVE_COND_INIT
-    ret = g_cond_wait_until(cond, mutex, end_time);
-#else
+    end_time -= g_get_monotonic_time();
     GTimeVal time = { end_time / G_TIME_SPAN_SECOND,
                       end_time % G_TIME_SPAN_SECOND };
     ret = g_cond_timed_wait(cond, mutex, &time);
-#endif
     return ret;
 }
-
-static void _cond_free(GCond *cond)
-{
-#ifdef HAVE_COND_INIT
-    g_cond_clear(cond);
-    g_free(cond);
-#else
-    g_cond_free(cond);
 #endif
-}
 
-static GThread *_thread_new(const gchar *name, GThreadFunc func, gpointer data)
-{
-    GThread *thread = NULL;
-    GError *error = NULL;
-#ifdef HAVE_THREAD_NEW
-    thread = g_thread_try_new(name, func, data, &error);
-#else
-    thread = g_thread_create(func, data, TRUE, &error);
-#endif
-    return thread;
-}
+static const char *tmpfs;
+static const char *root;
 
-static void read_guest_mem(void)
+static void wait_for_fds(TestServer *s)
 {
-    uint32_t *guest_mem;
     gint64 end_time;
-    int i, j;
-    size_t size;
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
 
-    end_time = _get_time() + 5 * G_TIME_SPAN_SECOND;
-    while (!fds_num) {
-        if (!_cond_wait_until(data_cond, data_mutex, end_time)) {
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (!s->fds_num) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
             /* timeout has passed */
-            g_assert(fds_num);
+            g_assert(s->fds_num);
             break;
         }
     }
 
     /* check for sanity */
-    g_assert_cmpint(fds_num, >, 0);
-    g_assert_cmpint(fds_num, ==, memory.nregions);
+    g_assert_cmpint(s->fds_num, >, 0);
+    g_assert_cmpint(s->fds_num, ==, s->memory.nregions);
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void read_guest_mem(const void *data)
+{
+    TestServer *s = (void *)data;
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    g_mutex_lock(&s->data_mutex);
 
     /* iterate all regions */
-    for (i = 0; i < fds_num; i++) {
+    for (i = 0; i < s->fds_num; i++) {
 
         /* We'll check only the region statring at 0x0*/
-        if (memory.regions[i].guest_phys_addr != 0x0) {
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
             continue;
         }
 
-        g_assert_cmpint(memory.regions[i].memory_size, >, 1024);
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
 
-        size =  memory.regions[i].memory_size + memory.regions[i].mmap_offset;
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
 
         guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
-                         MAP_SHARED, fds[i], 0);
+                         MAP_SHARED, s->fds[i], 0);
 
         g_assert(guest_mem != MAP_FAILED);
-        guest_mem += (memory.regions[i].mmap_offset / sizeof(*guest_mem));
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
 
         for (j = 0; j < 256; j++) {
-            uint32_t a = readl(memory.regions[i].guest_phys_addr + j*4);
+            uint32_t a = readl(s->memory.regions[i].guest_phys_addr + j*4);
             uint32_t b = guest_mem[j];
 
             g_assert_cmpint(a, ==, b);
         }
 
-        munmap(guest_mem, memory.regions[i].memory_size);
+        munmap(guest_mem, s->memory.regions[i].memory_size);
     }
 
-    g_assert_cmpint(1, ==, 1);
-    g_mutex_unlock(data_mutex);
+    g_mutex_unlock(&s->data_mutex);
 }
 
 static void *thread_function(void *data)
 {
-    GMainLoop *loop;
-    loop = g_main_loop_new(NULL, FALSE);
+    GMainLoop *loop = data;
     g_main_loop_run(loop);
     return NULL;
 }
@@ -270,7 +230,8 @@ static int chr_can_read(void *opaque)
 
 static void chr_read(void *opaque, const uint8_t *buf, int size)
 {
-    CharDriverState *chr = opaque;
+    TestServer *s = opaque;
+    CharDriverState *chr = s->chr;
     VhostUserMsg msg;
     uint8_t *p = (uint8_t *) &msg;
     int fd;
@@ -280,20 +241,35 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
         return;
     }
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
     memcpy(p, buf, VHOST_USER_HDR_SIZE);
 
     if (msg.size) {
         p += VHOST_USER_HDR_SIZE;
-        qemu_chr_fe_read_all(chr, p, msg.size);
+        g_assert_cmpint(qemu_chr_fe_read_all(chr, p, msg.size), ==, msg.size);
     }
 
     switch (msg.request) {
     case VHOST_USER_GET_FEATURES:
         /* send back features to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.u64);
-        msg.u64 = 0;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 0x1ULL << VHOST_F_LOG_ALL |
+            0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+        break;
+
+    case VHOST_USER_SET_FEATURES:
+	g_assert_cmpint(msg.payload.u64 & (0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES),
+			!=, 0ULL);
+        break;
+
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        /* send back features to qemu */
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
         break;
@@ -301,19 +277,22 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
     case VHOST_USER_GET_VRING_BASE:
         /* send back vring base to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.state);
-        msg.state.num = 0;
+        msg.size = sizeof(m.payload.state);
+        msg.payload.state.num = 0;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+
+        assert(msg.payload.state.index < 2);
+        s->rings &= ~(0x1ULL << msg.payload.state.index);
         break;
 
     case VHOST_USER_SET_MEM_TABLE:
         /* received the mem table */
-        memcpy(&memory, &msg.memory, sizeof(msg.memory));
-        fds_num = qemu_chr_fe_get_msgfds(chr, fds, sizeof(fds) / sizeof(int));
+        memcpy(&s->memory, &msg.payload.memory, sizeof(msg.payload.memory));
+        s->fds_num = qemu_chr_fe_get_msgfds(chr, s->fds, G_N_ELEMENTS(s->fds));
 
         /* signal the test that it can continue */
-        g_cond_signal(data_cond);
+        g_cond_signal(&s->data_cond);
         break;
 
     case VHOST_USER_SET_VRING_KICK:
@@ -327,23 +306,38 @@ static void chr_read(void *opaque, const uint8_t *buf, int size)
          */
         qemu_set_nonblock(fd);
         break;
+
+    case VHOST_USER_SET_LOG_BASE:
+        if (s->log_fd != -1) {
+            close(s->log_fd);
+            s->log_fd = -1;
+        }
+        qemu_chr_fe_get_msgfds(chr, &s->log_fd, 1);
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = 0;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE);
+
+        g_cond_signal(&s->data_cond);
+        break;
+
+    case VHOST_USER_SET_VRING_BASE:
+        assert(msg.payload.state.index < 2);
+        s->rings |= 0x1ULL << msg.payload.state.index;
+        break;
+
     default:
         break;
     }
-    g_mutex_unlock(data_mutex);
+
+    g_mutex_unlock(&s->data_mutex);
 }
 
-static const char *init_hugepagefs(void)
+static const char *init_hugepagefs(const char *path)
 {
-    const char *path;
     struct statfs fs;
     int ret;
 
-    path = getenv("QTEST_HUGETLBFS_PATH");
-    if (!path) {
-        path = "/hugetlbfs";
-    }
-
     if (access(path, R_OK | W_OK | X_OK)) {
         g_test_message("access on path (%s): %s\n", path, strerror(errno));
         return NULL;
@@ -366,44 +360,294 @@ static const char *init_hugepagefs(void)
     return path;
 }
 
+static TestServer *test_server_new(const gchar *name)
+{
+    TestServer *server = g_new0(TestServer, 1);
+    gchar *chr_path;
+
+    server->socket_path = g_strdup_printf("%s/%s.sock", tmpfs, name);
+    server->mig_path = g_strdup_printf("%s/%s.mig", tmpfs, name);
+
+    chr_path = g_strdup_printf("unix:%s,server,nowait", server->socket_path);
+    server->chr_name = g_strdup_printf("chr-%s", name);
+    server->chr = qemu_chr_new(server->chr_name, chr_path, NULL);
+    g_free(chr_path);
+
+    qemu_chr_add_handlers(server->chr, chr_can_read, chr_read, NULL, server);
+
+    g_mutex_init(&server->data_mutex);
+    g_cond_init(&server->data_cond);
+
+    server->log_fd = -1;
+
+    return server;
+}
+
+#define GET_QEMU_CMD(s)                                                        \
+    g_strdup_printf(QEMU_CMD, 512, 512, (root), (s)->chr_name,                 \
+                    (s)->socket_path, (s)->chr_name)
+
+#define GET_QEMU_CMDE(s, mem, extra, ...)                                      \
+    g_strdup_printf(QEMU_CMD extra, (mem), (mem), (root), (s)->chr_name,       \
+                    (s)->socket_path, (s)->chr_name, ##__VA_ARGS__)
+
+static gboolean _test_server_free(TestServer *server)
+{
+    int i;
+
+    qemu_chr_delete(server->chr);
+
+    for (i = 0; i < server->fds_num; i++) {
+        close(server->fds[i]);
+    }
+
+    if (server->log_fd != -1) {
+        close(server->log_fd);
+    }
+
+    unlink(server->socket_path);
+    g_free(server->socket_path);
+
+    unlink(server->mig_path);
+    g_free(server->mig_path);
+
+    g_free(server->chr_name);
+    g_free(server);
+
+    return FALSE;
+}
+
+static void test_server_free(TestServer *server)
+{
+    g_idle_add((GSourceFunc)_test_server_free, server);
+}
+
+static void wait_for_log_fd(TestServer *s)
+{
+    gint64 end_time;
+
+    g_mutex_lock(&s->data_mutex);
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (s->log_fd == -1) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
+            /* timeout has passed */
+            g_assert(s->log_fd != -1);
+            break;
+        }
+    }
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void write_guest_mem(TestServer *s, uint32_t seed)
+{
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    /* iterate all regions */
+    for (i = 0; i < s->fds_num; i++) {
+
+        /* We'll write only the region statring at 0x0 */
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
+            continue;
+        }
+
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
+
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
+
+        guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
+                         MAP_SHARED, s->fds[i], 0);
+
+        g_assert(guest_mem != MAP_FAILED);
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
+
+        for (j = 0; j < 256; j++) {
+            guest_mem[j] = seed + j;
+        }
+
+        munmap(guest_mem, s->memory.regions[i].memory_size);
+        break;
+    }
+}
+
+static guint64 get_log_size(TestServer *s)
+{
+    guint64 log_size = 0;
+    int i;
+
+    for (i = 0; i < s->memory.nregions; ++i) {
+        VhostUserMemoryRegion *reg = &s->memory.regions[i];
+        guint64 last = range_get_last(reg->guest_phys_addr,
+                                       reg->memory_size);
+        log_size = MAX(log_size, last / (8 * VHOST_LOG_PAGE) + 1);
+    }
+
+    return log_size;
+}
+
+typedef struct TestMigrateSource {
+    GSource source;
+    TestServer *src;
+    TestServer *dest;
+} TestMigrateSource;
+
+static gboolean
+test_migrate_source_check(GSource *source)
+{
+    TestMigrateSource *t = (TestMigrateSource *)source;
+    gboolean overlap = t->src->rings && t->dest->rings;
+
+    g_assert(!overlap);
+
+    return FALSE;
+}
+
+#if !GLIB_CHECK_VERSION(2,36,0)
+/* this callback is unnecessary with glib >2.36, the default
+ * prepare for the source does the same */
+static gboolean
+test_migrate_source_prepare(GSource *source, gint *timeout)
+{
+    *timeout = -1;
+    return FALSE;
+}
+#endif
+
+GSourceFuncs test_migrate_source_funcs = {
+#if !GLIB_CHECK_VERSION(2,36,0)
+    .prepare = test_migrate_source_prepare,
+#endif
+    .check = test_migrate_source_check,
+};
+
+static void test_migrate(void)
+{
+    TestServer *s = test_server_new("src");
+    TestServer *dest = test_server_new("dest");
+    char *uri = g_strdup_printf("%s%s", "unix:", dest->mig_path);
+    QTestState *global = global_qtest, *from, *to;
+    GSource *source;
+    gchar *cmd;
+    QDict *rsp;
+    guint8 *log;
+    guint64 size;
+
+    cmd = GET_QEMU_CMDE(s, 2, "");
+    from = qtest_start(cmd);
+    g_free(cmd);
+
+    wait_for_fds(s);
+    size = get_log_size(s);
+    g_assert_cmpint(size, ==, (2 * 1024 * 1024) / (VHOST_LOG_PAGE * 8));
+
+    cmd = GET_QEMU_CMDE(dest, 2, " -incoming %s", uri);
+    to = qtest_init(cmd);
+    g_free(cmd);
+
+    source = g_source_new(&test_migrate_source_funcs,
+                          sizeof(TestMigrateSource));
+    ((TestMigrateSource *)source)->src = s;
+    ((TestMigrateSource *)source)->dest = dest;
+    g_source_attach(source, NULL);
+
+    /* slow down migration to have time to fiddle with log */
+    /* TODO: qtest could learn to break on some places */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 10 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    cmd = g_strdup_printf("{ 'execute': 'migrate',"
+                          "'arguments': { 'uri': '%s' } }",
+                          uri);
+    rsp = qmp(cmd);
+    g_free(cmd);
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    wait_for_log_fd(s);
+
+    log = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, s->log_fd, 0);
+    g_assert(log != MAP_FAILED);
+
+    /* modify first page */
+    write_guest_mem(s, 0x42);
+    log[0] = 1;
+    munmap(log, size);
+
+    /* speed things up */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 0 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    qmp_eventwait("STOP");
+
+    global_qtest = to;
+    qmp_eventwait("RESUME");
+
+    read_guest_mem(dest);
+
+    g_source_destroy(source);
+    g_source_unref(source);
+
+    qtest_quit(to);
+    test_server_free(dest);
+    qtest_quit(from);
+    test_server_free(s);
+    g_free(uri);
+
+    global_qtest = global;
+}
+
 int main(int argc, char **argv)
 {
     QTestState *s = NULL;
-    CharDriverState *chr = NULL;
-    const char *hugefs = 0;
-    char *socket_path = 0;
-    char *qemu_cmd = 0;
-    char *chr_path = 0;
+    TestServer *server = NULL;
+    const char *hugefs;
+    char *qemu_cmd = NULL;
     int ret;
+    char template[] = "/tmp/vhost-test-XXXXXX";
+    GMainLoop *loop;
+    GThread *thread;
 
     g_test_init(&argc, &argv, NULL);
 
     module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_chardev_opts);
 
-    hugefs = init_hugepagefs();
-    if (!hugefs) {
-        return 0;
+    tmpfs = mkdtemp(template);
+    if (!tmpfs) {
+        g_test_message("mkdtemp on path (%s): %s\n", template, strerror(errno));
+    }
+    g_assert(tmpfs);
+
+    hugefs = getenv("QTEST_HUGETLBFS_PATH");
+    if (hugefs) {
+        root = init_hugepagefs(hugefs);
+        g_assert(root);
+    } else {
+        root = tmpfs;
     }
 
-    socket_path = g_strdup_printf("/tmp/vhost-%d.sock", getpid());
-
-    /* create char dev and add read handlers */
-    qemu_add_opts(&qemu_chardev_opts);
-    chr_path = g_strdup_printf("unix:%s,server,nowait", socket_path);
-    chr = qemu_chr_new("chr0", chr_path, NULL);
-    g_free(chr_path);
-    qemu_chr_add_handlers(chr, chr_can_read, chr_read, NULL, chr);
+    server = test_server_new("test");
 
+    loop = g_main_loop_new(NULL, FALSE);
     /* run the main loop thread so the chardev may operate */
-    data_mutex = _mutex_new();
-    data_cond = _cond_new();
-    _thread_new(NULL, thread_function, NULL);
+    thread = g_thread_new(NULL, thread_function, loop);
+
+    qemu_cmd = GET_QEMU_CMD(server);
 
-    qemu_cmd = g_strdup_printf(QEMU_CMD, hugefs, socket_path);
     s = qtest_start(qemu_cmd);
     g_free(qemu_cmd);
 
-    qtest_add_func("/vhost-user/read-guest-mem", read_guest_mem);
+    qtest_add_data_func("/vhost-user/read-guest-mem", server, read_guest_mem);
+    qtest_add_func("/vhost-user/migrate", test_migrate);
 
     ret = g_test_run();
 
@@ -412,10 +656,22 @@ int main(int argc, char **argv)
     }
 
     /* cleanup */
-    unlink(socket_path);
-    g_free(socket_path);
-    _cond_free(data_cond);
-    _mutex_free(data_mutex);
+    test_server_free(server);
+
+    /* finish the helper thread and dispatch pending sources */
+    g_main_loop_quit(loop);
+    g_thread_join(thread);
+    while (g_main_context_pending(NULL)) {
+        g_main_context_iteration (NULL, TRUE);
+    }
+    g_main_loop_unref(loop);
+
+    ret = rmdir(tmpfs);
+    if (ret != 0) {
+        g_test_message("unable to rmdir: path (%s): %s\n",
+                       tmpfs, strerror(errno));
+    }
+    g_assert_cmpint(ret, ==, 0);
 
     return ret;
 }
diff --git a/qemu/tests/virtio-9p-test.c b/qemu/tests/virtio-9p-test.c
index 1fae47797..59d0f1fa9 100644
--- a/qemu/tests/virtio-9p-test.c
+++ b/qemu/tests/virtio-9p-test.c
@@ -7,12 +7,10 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdlib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "libqtest.h"
 #include "qemu-common.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/qemu/tests/virtio-balloon-test.c b/qemu/tests/virtio-balloon-test.c
index becebb51a..b010ce98e 100644
--- a/qemu/tests/virtio-balloon-test.c
+++ b/qemu/tests/virtio-balloon-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/qemu/tests/virtio-blk-test.c b/qemu/tests/virtio-blk-test.c
index 4078321a2..3a66630d7 100644
--- a/qemu/tests/virtio-blk-test.c
+++ b/qemu/tests/virtio-blk-test.c
@@ -8,11 +8,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdio.h>
 #include "libqtest.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
diff --git a/qemu/tests/virtio-console-test.c b/qemu/tests/virtio-console-test.c
index 6be96e8c6..0b9c2a55e 100644
--- a/qemu/tests/virtio-console-test.c
+++ b/qemu/tests/virtio-console-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void console_pci_nop(void)
diff --git a/qemu/tests/virtio-net-test.c b/qemu/tests/virtio-net-test.c
index ea7478c27..04cfcd594 100644
--- a/qemu/tests/virtio-net-test.c
+++ b/qemu/tests/virtio-net-test.c
@@ -7,23 +7,244 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
-#include "libqos/pci.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
+#include "libqos/pci-pc.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
+#include "qemu/bswap.h"
+#include "hw/virtio/virtio-net.h"
 
 #define PCI_SLOT_HP             0x06
+#define PCI_SLOT                0x04
+#define PCI_FN                  0x00
+
+#define QVIRTIO_NET_TIMEOUT_US (30 * 1000 * 1000)
+#define VNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
+
+static void test_end(void)
+{
+    qtest_end();
+}
+
+#ifndef _WIN32
+
+static QVirtioPCIDevice *virtio_net_pci_init(QPCIBus *bus, int slot)
+{
+    QVirtioPCIDevice *dev;
+
+    dev = qvirtio_pci_device_find(bus, QVIRTIO_NET_DEVICE_ID);
+    g_assert(dev != NULL);
+    g_assert_cmphex(dev->vdev.device_type, ==, QVIRTIO_NET_DEVICE_ID);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_acknowledge(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_driver(&qvirtio_pci, &dev->vdev);
+
+    return dev;
+}
+
+static QPCIBus *pci_test_start(int socket)
+{
+    char *cmdline;
+
+    cmdline = g_strdup_printf("-netdev socket,fd=%d,id=hs0 -device "
+                              "virtio-net-pci,netdev=hs0", socket);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    return qpci_init_pc();
+}
+
+static void driver_init(const QVirtioBus *bus, QVirtioDevice *dev)
+{
+    uint32_t features;
+
+    features = qvirtio_get_features(bus, dev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            QVIRTIO_F_RING_INDIRECT_DESC |
+                            QVIRTIO_F_RING_EVENT_IDX);
+    qvirtio_set_features(bus, dev, features);
+
+    qvirtio_set_driver_ok(bus, dev);
+}
+
+static void rx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
+
+static void tx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    uint32_t len;
+    char buffer[64];
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+    memwrite(req_addr + VNET_HDR_SIZE, "TEST", 4);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, false, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    guest_free(alloc, req_addr);
+
+    ret = qemu_recv(socket, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    ret = qemu_recv(socket, buffer, len, 0);
+    g_assert_cmpstr(buffer, ==, "TEST");
+}
+
+static void rx_stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                              QGuestAllocator *alloc, QVirtQueue *vq,
+                              int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qmp("{ 'execute' : 'stop'}");
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    /* We could check the status, but this command is more importantly to
+     * ensure the packet data gets queued in QEMU, before we do 'cont'.
+     */
+    qmp("{ 'execute' : 'query-status'}");
+    qmp("{ 'execute' : 'cont'}");
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
 
-/* Tests only initialization so far. TODO: Replace with functional tests */
-static void pci_nop(void)
+static void send_recv_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
 {
+    rx_test(bus, dev, alloc, rvq, socket);
+    tx_test(bus, dev, alloc, tvq, socket);
 }
 
+static void stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
+{
+    rx_stop_cont_test(bus, dev, alloc, rvq, socket);
+}
+
+static void pci_basic(gconstpointer data)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *tx, *rx;
+    QGuestAllocator *alloc;
+    void (*func) (const QVirtioBus *bus,
+                  QVirtioDevice *dev,
+                  QGuestAllocator *alloc,
+                  QVirtQueue *rvq,
+                  QVirtQueue *tvq,
+                  int socket) = data;
+    int sv[2], ret;
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, sv);
+    g_assert_cmpint(ret, !=, -1);
+
+    bus = pci_test_start(sv[1]);
+    dev = virtio_net_pci_init(bus, PCI_SLOT);
+
+    alloc = pc_alloc_init();
+    rx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 0);
+    tx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 1);
+
+    driver_init(&qvirtio_pci, &dev->vdev);
+    func(&qvirtio_pci, &dev->vdev, alloc, &rx->vq, &tx->vq, sv[0]);
+
+    /* End test */
+    close(sv[0]);
+    guest_free(alloc, tx->vq.desc);
+    pc_alloc_uninit(alloc);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev);
+    qpci_free_pc(bus);
+    test_end();
+}
+#endif
+
 static void hotplug(void)
 {
+    qtest_start("-device virtio-net-pci");
+
     qpci_plug_device_test("virtio-net-pci", "net1", PCI_SLOT_HP, NULL);
     qpci_unplug_acpi_device_test("net1", PCI_SLOT_HP);
+
+    test_end();
 }
 
 int main(int argc, char **argv)
@@ -31,13 +252,14 @@ int main(int argc, char **argv)
     int ret;
 
     g_test_init(&argc, &argv, NULL);
-    qtest_add_func("/virtio/net/pci/nop", pci_nop);
+#ifndef _WIN32
+    qtest_add_data_func("/virtio/net/pci/basic", send_recv_test, pci_basic);
+    qtest_add_data_func("/virtio/net/pci/rx_stop_cont",
+                        stop_cont_test, pci_basic);
+#endif
     qtest_add_func("/virtio/net/pci/hotplug", hotplug);
 
-    qtest_start("-device virtio-net-pci");
     ret = g_test_run();
 
-    qtest_end();
-
     return ret;
 }
diff --git a/qemu/tests/virtio-rng-test.c b/qemu/tests/virtio-rng-test.c
index 41c1cdb1a..771dbd73a 100644
--- a/qemu/tests/virtio-rng-test.c
+++ b/qemu/tests/virtio-rng-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "libqos/pci.h"
 
 #define PCI_SLOT_HP             0x06
diff --git a/qemu/tests/virtio-scsi-test.c b/qemu/tests/virtio-scsi-test.c
index 11ccdd632..d78747a46 100644
--- a/qemu/tests/virtio-scsi-test.c
+++ b/qemu/tests/virtio-scsi-test.c
@@ -8,11 +8,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
-#include <stdio.h>
+#include "block/scsi.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
 #include "libqos/pci-pc.h"
@@ -71,40 +70,6 @@ static void qvirtio_scsi_stop(void)
     qtest_end();
 }
 
-static QVirtIOSCSI *qvirtio_scsi_pci_init(int slot)
-{
-    QVirtIOSCSI *vs;
-    QVirtioPCIDevice *dev;
-    void *addr;
-    int i;
-
-    vs = g_new0(QVirtIOSCSI, 1);
-    vs->alloc = pc_alloc_init();
-    vs->bus = qpci_init_pc();
-
-    dev = qvirtio_pci_device_find(vs->bus, QVIRTIO_SCSI_DEVICE_ID);
-    vs->dev = (QVirtioDevice *)dev;
-    g_assert(dev != NULL);
-    g_assert_cmphex(vs->dev->device_type, ==, QVIRTIO_SCSI_DEVICE_ID);
-
-    qvirtio_pci_device_enable(dev);
-    qvirtio_reset(&qvirtio_pci, vs->dev);
-    qvirtio_set_acknowledge(&qvirtio_pci, vs->dev);
-    qvirtio_set_driver(&qvirtio_pci, vs->dev);
-
-    addr = dev->addr + QVIRTIO_PCI_DEVICE_SPECIFIC_NO_MSIX;
-    vs->num_queues = qvirtio_config_readl(&qvirtio_pci, vs->dev,
-                                          (uint64_t)(uintptr_t)addr);
-
-    g_assert_cmpint(vs->num_queues, <, MAX_NUM_QUEUES);
-
-    for (i = 0; i < vs->num_queues + 2; i++) {
-        vs->vq[i] = qvirtqueue_setup(&qvirtio_pci, vs->dev, vs->alloc, i);
-    }
-
-    return vs;
-}
-
 static void qvirtio_scsi_pci_free(QVirtIOSCSI *vs)
 {
     int i;
@@ -134,7 +99,8 @@ static uint64_t qvirtio_scsi_alloc(QVirtIOSCSI *vs, size_t alloc_size,
 static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
                                       const uint8_t *data_in,
                                       size_t data_in_len,
-                                      uint8_t *data_out, size_t data_out_len)
+                                      uint8_t *data_out, size_t data_out_len,
+                                      QVirtIOSCSICmdResp *resp_out)
 {
     QVirtQueue *vq;
     QVirtIOSCSICmdReq req = { { 0 } };
@@ -174,6 +140,10 @@ static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
 
     response = readb(resp_addr + offsetof(QVirtIOSCSICmdResp, response));
 
+    if (resp_out) {
+        memread(resp_addr, resp_out, sizeof(*resp_out));
+    }
+
     guest_free(vs->alloc, req_addr);
     guest_free(vs->alloc, resp_addr);
     guest_free(vs->alloc, data_in_addr);
@@ -181,6 +151,52 @@ static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
     return response;
 }
 
+static QVirtIOSCSI *qvirtio_scsi_pci_init(int slot)
+{
+    const uint8_t test_unit_ready_cdb[CDB_SIZE] = {};
+    QVirtIOSCSI *vs;
+    QVirtioPCIDevice *dev;
+    QVirtIOSCSICmdResp resp;
+    void *addr;
+    int i;
+
+    vs = g_new0(QVirtIOSCSI, 1);
+    vs->alloc = pc_alloc_init();
+    vs->bus = qpci_init_pc();
+
+    dev = qvirtio_pci_device_find(vs->bus, QVIRTIO_SCSI_DEVICE_ID);
+    vs->dev = (QVirtioDevice *)dev;
+    g_assert(dev != NULL);
+    g_assert_cmphex(vs->dev->device_type, ==, QVIRTIO_SCSI_DEVICE_ID);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, vs->dev);
+    qvirtio_set_acknowledge(&qvirtio_pci, vs->dev);
+    qvirtio_set_driver(&qvirtio_pci, vs->dev);
+
+    addr = dev->addr + QVIRTIO_PCI_DEVICE_SPECIFIC_NO_MSIX;
+    vs->num_queues = qvirtio_config_readl(&qvirtio_pci, vs->dev,
+                                          (uint64_t)(uintptr_t)addr);
+
+    g_assert_cmpint(vs->num_queues, <, MAX_NUM_QUEUES);
+
+    for (i = 0; i < vs->num_queues + 2; i++) {
+        vs->vq[i] = qvirtqueue_setup(&qvirtio_pci, vs->dev, vs->alloc, i);
+    }
+
+    /* Clear the POWER ON OCCURRED unit attention */
+    g_assert_cmpint(virtio_scsi_do_command(vs, test_unit_ready_cdb,
+                                           NULL, 0, NULL, 0, &resp),
+                    ==, 0);
+    g_assert_cmpint(resp.status, ==, CHECK_CONDITION);
+    g_assert_cmpint(resp.sense[0], ==, 0x70); /* Fixed format sense buffer */
+    g_assert_cmpint(resp.sense[2], ==, UNIT_ATTENTION);
+    g_assert_cmpint(resp.sense[12], ==, 0x29); /* POWER ON */
+    g_assert_cmpint(resp.sense[13], ==, 0x00);
+
+    return vs;
+}
+
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
 {
@@ -221,9 +237,12 @@ static void hotplug(void)
 static void test_unaligned_write_same(void)
 {
     QVirtIOSCSI *vs;
-    uint8_t buf[512] = { 0 };
-    const uint8_t write_same_cdb[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
+    uint8_t buf1[512] = { 0 };
+    uint8_t buf2[512] = { 1 };
+    const uint8_t write_same_cdb_1[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
                                                0x01, 0x00, 0x00, 0x02, 0x00 };
+    const uint8_t write_same_cdb_2[CDB_SIZE] = { 0x41, 0x00, 0x00, 0x00, 0x00,
+                                               0x01, 0x00, 0x33, 0x00, 0x00 };
 
     qvirtio_scsi_start("-drive file=blkdebug::null-co://,if=none,id=dr1"
                        ",format=raw,file.align=4k "
@@ -231,7 +250,10 @@ static void test_unaligned_write_same(void)
     vs = qvirtio_scsi_pci_init(PCI_SLOT);
 
     g_assert_cmphex(0, ==,
-        virtio_scsi_do_command(vs, write_same_cdb, NULL, 0, buf, 512));
+        virtio_scsi_do_command(vs, write_same_cdb_1, NULL, 0, buf1, 512, NULL));
+
+    g_assert_cmphex(0, ==,
+        virtio_scsi_do_command(vs, write_same_cdb_2, NULL, 0, buf2, 512, NULL));
 
     qvirtio_scsi_pci_free(vs);
     qvirtio_scsi_stop();
diff --git a/qemu/tests/virtio-serial-test.c b/qemu/tests/virtio-serial-test.c
index bf030a616..480d4abb2 100644
--- a/qemu/tests/virtio-serial-test.c
+++ b/qemu/tests/virtio-serial-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/qemu/tests/vmxnet3-test.c b/qemu/tests/vmxnet3-test.c
index a2ebed39c..6ef0e2f04 100644
--- a/qemu/tests/vmxnet3-test.c
+++ b/qemu/tests/vmxnet3-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/qemu/tests/wdt_ib700-test.c b/qemu/tests/wdt_ib700-test.c
index 82ca59725..efe337045 100644
--- a/qemu/tests/wdt_ib700-test.c
+++ b/qemu/tests/wdt_ib700-test.c
@@ -7,10 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "qemu/timer.h"
 
 static void qmp_check_no_event(void)
diff --git a/qemu/thread-pool.c b/qemu/thread-pool.c
index ac909f498..03ba0b02a 100644
--- a/qemu/thread-pool.c
+++ b/qemu/thread-pool.c
@@ -14,11 +14,11 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
-#include "qemu/osdep.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "trace.h"
 #include "block/thread-pool.h"
 #include "qemu/main-loop.h"
diff --git a/qemu/thunk.c b/qemu/thunk.c
index f501fd72f..f057d86d9 100644
--- a/qemu/thunk.c
+++ b/qemu/thunk.c
@@ -16,9 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "exec/user/thunk.h"
diff --git a/qemu/tpm.c b/qemu/tpm.c
index 4e9b109fb..9a7c7114d 100644
--- a/qemu/tpm.c
+++ b/qemu/tpm.c
@@ -11,7 +11,7 @@
  *
  * Based on net.c
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qerror.h"
 #include "sysemu/tpm_backend.h"
@@ -32,7 +32,7 @@ static TPMDriverOps const *be_drivers[TPM_MAX_DRIVERS] = {
 };
 
 static enum TpmModel tpm_models[TPM_MAX_MODELS] = {
-    TPM_MODEL_MAX,
+    TPM_MODEL__MAX,
 };
 
 int tpm_register_model(enum TpmModel model)
@@ -40,7 +40,7 @@ int tpm_register_model(enum TpmModel model)
     int i;
 
     for (i = 0; i < TPM_MAX_MODELS; i++) {
-        if (tpm_models[i] == TPM_MODEL_MAX) {
+        if (tpm_models[i] == TPM_MODEL__MAX) {
             tpm_models[i] = model;
             return 0;
         }
@@ -260,9 +260,9 @@ static TPMInfo *qmp_query_tpm_inst(TPMBackend *drv)
 
     switch (drv->ops->type) {
     case TPM_TYPE_PASSTHROUGH:
-        res->options->kind = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
+        res->options->type = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
         tpo = g_new0(TPMPassthroughOptions, 1);
-        res->options->passthrough = tpo;
+        res->options->u.passthrough.data = tpo;
         if (drv->path) {
             tpo->path = g_strdup(drv->path);
             tpo->has_path = true;
@@ -272,7 +272,7 @@ static TPMInfo *qmp_query_tpm_inst(TPMBackend *drv)
             tpo->has_cancel_path = true;
         }
         break;
-    case TPM_TYPE_MAX:
+    case TPM_TYPE__MAX:
         break;
     }
 
@@ -311,7 +311,7 @@ TpmTypeList *qmp_query_tpm_types(Error **errp)
     unsigned int i = 0;
     TpmTypeList *head = NULL, *prev = NULL, *cur_item;
 
-    for (i = 0; i < TPM_TYPE_MAX; i++) {
+    for (i = 0; i < TPM_TYPE__MAX; i++) {
         if (!tpm_driver_find_by_type(i)) {
             continue;
         }
@@ -335,7 +335,7 @@ TpmModelList *qmp_query_tpm_models(Error **errp)
     unsigned int i = 0;
     TpmModelList *head = NULL, *prev = NULL, *cur_item;
 
-    for (i = 0; i < TPM_MODEL_MAX; i++) {
+    for (i = 0; i < TPM_MODEL__MAX; i++) {
         if (!tpm_model_is_registered(i)) {
             continue;
         }
diff --git a/qemu/trace-events b/qemu/trace-events
index 94bf3bb18..835074387 100644
--- a/qemu/trace-events
+++ b/qemu/trace-events
@@ -59,6 +59,9 @@ virtio_console_chr_event(unsigned int port, int event) "port %u, event %d"
 
 # block.c
 bdrv_open_common(void *bs, const char *filename, int flags, const char *format_name) "bs %p filename \"%s\" flags %#x format_name \"%s\""
+bdrv_lock_medium(void *bs, bool locked) "bs %p locked %d"
+
+# block/io.c
 multiwrite_cb(void *mcb, int ret) "mcb %p ret %d"
 bdrv_aio_multiwrite(void *mcb, int num_callbacks, int num_reqs) "mcb %p num_callbacks %d num_reqs %d"
 bdrv_aio_discard(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
@@ -66,9 +69,9 @@ bdrv_aio_flush(void *bs, void *opaque) "bs %p opaque %p"
 bdrv_aio_readv(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
 bdrv_aio_writev(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
 bdrv_aio_write_zeroes(void *bs, int64_t sector_num, int nb_sectors, int flags, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d flags %#x opaque %p"
-bdrv_lock_medium(void *bs, bool locked) "bs %p locked %d"
 bdrv_co_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_copy_on_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
+bdrv_co_readv_no_serialising(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_writev(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
 bdrv_co_write_zeroes(void *bs, int64_t sector_num, int nb_sector, int flags) "bs %p sector_num %"PRId64" nb_sectors %d flags %#x"
 bdrv_co_io_em(void *bs, int64_t sector_num, int nb_sectors, int is_write, void *acb) "bs %p sector_num %"PRId64" nb_sectors %d is_write %d acb %p"
@@ -123,9 +126,6 @@ virtio_blk_data_plane_start(void *s) "dataplane %p"
 virtio_blk_data_plane_stop(void *s) "dataplane %p"
 virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
 
-# hw/virtio/dataplane/vring.c
-vring_setup(uint64_t physical, void *desc, void *avail, void *used) "vring physical %#"PRIx64" desc %p avail %p used %p"
-
 # thread-pool.c
 thread_pool_submit(void *pool, void *req, void *opaque) "pool %p req %p opaque %p"
 thread_pool_complete(void *pool, void *req, void *opaque, int ret) "pool %p req %p opaque %p ret %d"
@@ -137,15 +137,15 @@ paio_submit_co(int64_t sector_num, int nb_sectors, int type) "sector_num %"PRId6
 paio_submit(void *acb, void *opaque, int64_t sector_num, int nb_sectors, int type) "acb %p opaque %p sector_num %"PRId64" nb_sectors %d type %d"
 
 # ioport.c
-cpu_in(unsigned int addr, unsigned int val) "addr %#x value %u"
-cpu_out(unsigned int addr, unsigned int val) "addr %#x value %u"
+cpu_in(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
+cpu_out(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
 
 # balloon.c
 # Since requests are raised via monitor, not many tracepoints are needed.
 balloon_event(void *opaque, unsigned long addr) "opaque %p addr %lu"
-virtio_balloon_handle_output(const char *name, uint64_t gpa) "setion name: %s gpa: %"PRIx64""
-virtio_balloon_get_config(uint32_t num_pages, uint32_t acutal) "num_pages: %d acutal: %d"
-virtio_balloon_set_config(uint32_t acutal, uint32_t oldacutal) "acutal: %d oldacutal: %d"
+virtio_balloon_handle_output(const char *name, uint64_t gpa) "section name: %s gpa: %"PRIx64
+virtio_balloon_get_config(uint32_t num_pages, uint32_t actual) "num_pages: %d actual: %d"
+virtio_balloon_set_config(uint32_t actual, uint32_t oldactual) "actual: %d oldactual: %d"
 virtio_balloon_to_target(uint64_t target, uint32_t num_pages) "balloon target: %"PRIx64" num_pages: %d"
 
 # hw/intc/apic_common.c
@@ -194,7 +194,7 @@ ecc_diag_mem_readb(uint64_t addr, uint32_t ret) "Read diagnostic %"PRId64"= %02x
 
 # hw/nvram/fw_cfg.c
 fw_cfg_select(void *s, uint16_t key, int ret) "%p key %d = %d"
-fw_cfg_read(void *s, uint8_t ret) "%p = %d"
+fw_cfg_read(void *s, uint64_t ret) "%p = %"PRIx64
 fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
 
 # hw/block/hd-geometry.c
@@ -550,6 +550,8 @@ usb_mtp_op_get_partial_object(int dev, uint32_t handle, const char *path, uint32
 usb_mtp_op_unknown(int dev, uint32_t code) "dev %d, command code 0x%x"
 usb_mtp_object_alloc(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
 usb_mtp_object_free(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_add_child(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_inotify_event(int dev, const char *path, uint32_t mask, const char *s) "dev %d, path %s mask 0x%x event %s"
 
 # hw/usb/host-libusb.c
 usb_host_open_started(int bus, int addr) "dev %d:%d"
@@ -601,9 +603,6 @@ scsi_request_sense(int target, int lun, int tag) "target %d lun %d tag %d"
 vm_state_notify(int running, int reason) "running %d reason %d"
 load_file(const char *name, const char *path) "name %s location %s"
 runstate_set(int new_state) "new state %d"
-g_malloc(size_t size, void *ptr) "size %zu ptr %p"
-g_realloc(void *ptr, size_t size, void *newptr) "ptr %p size %zu newptr %p"
-g_free(void *ptr) "ptr %p"
 system_wakeup_request(int reason) "reason=%d"
 qemu_system_shutdown_request(void) ""
 qemu_system_powerdown_request(void) ""
@@ -684,7 +683,7 @@ grlib_irqmp_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" valu
 # hw/char/grlib_apbuart.c
 grlib_apbuart_event(int event) "event:%d"
 grlib_apbuart_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" value 0x%x"
-grlib_apbuart_readl_unknown(uint64_t addr) "addr 0x%"PRIx64""
+grlib_apbuart_readl_unknown(uint64_t addr) "addr 0x%"PRIx64
 
 # hw/sparc/leon3.c
 leon3_set_irq(int intno) "Set CPU IRQ %d"
@@ -723,14 +722,36 @@ lm32_uart_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
 lm32_uart_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
 lm32_uart_irq_state(int level) "irq state %d"
 
+# hw/scsi/mptsas.c
+mptsas_command_complete(void *dev, uint32_t ctx, uint32_t status, uint32_t resid) "dev %p context 0x%08x status %x resid %d"
+mptsas_diag_read(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%08x"
+mptsas_diag_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%08x"
+mptsas_irq_intx(void *dev, int level) "dev %p level %d"
+mptsas_irq_msi(void *dev) "dev %p "
+mptsas_mmio_read(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_mmio_unhandled_read(void *dev, uint32_t addr) "dev %p addr 0x%08x"
+mptsas_mmio_unhandled_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_mmio_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_process_message(void *dev, int msg, uint32_t ctx) "dev %p cmd %d context 0x%08x\n"
+mptsas_process_scsi_io_request(void *dev, int bus, int target, int lun, uint64_t len) "dev %p dev %d:%d:%d length %"PRIu64""
+mptsas_reset(void *dev) "dev %p "
+mptsas_scsi_overflow(void *dev, uint32_t ctx, uint64_t req, uint64_t found) "dev %p context 0x%08x: %"PRIu64"/%"PRIu64""
+mptsas_sgl_overflow(void *dev, uint32_t ctx, uint64_t req, uint64_t found) "dev %p context 0x%08x: %"PRIu64"/%"PRIu64""
+mptsas_unhandled_cmd(void *dev, uint32_t ctx, uint8_t msg_cmd) "dev %p context 0x%08x: Unhandled cmd %x"
+mptsas_unhandled_doorbell_cmd(void *dev, int cmd) "dev %p value 0x%08x"
+
+# hw/scsi/mptconfig.c
+mptsas_config_sas_device(void *dev, int address, int port, int phy_handle, int dev_handle, int page) "dev %p address %d (port %d, handles: phy %d dev %d) page %d"
+mptsas_config_sas_phy(void *dev, int address, int port, int phy_handle, int dev_handle, int page) "dev %p address %d (port %d, handles: phy %d dev %d) page %d"
+
 # hw/scsi/megasas.c
 megasas_init_firmware(uint64_t pa) "pa %" PRIx64 " "
 megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags) "queue at %" PRIx64 " len %d head %" PRIx64 " tail %" PRIx64 " flags %x"
 megasas_initq_map_failed(int frame) "scmd %d: failed to map queue"
-megasas_initq_mapped(uint64_t pa) "queue already mapped at %" PRIx64 ""
+megasas_initq_mapped(uint64_t pa) "queue already mapped at %" PRIx64
 megasas_initq_mismatch(int queue_len, int fw_cmds) "queue size %d max fw cmds %d"
 megasas_qf_mapped(unsigned int index) "skip mapped frame %x"
-megasas_qf_new(unsigned int index, uint64_t frame) "frame %x addr %" PRIx64 ""
+megasas_qf_new(unsigned int index, uint64_t frame) "frame %x addr %" PRIx64
 megasas_qf_busy(unsigned long pa) "all frames busy for frame %lx"
 megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int head, unsigned int tail, int busy) "frame %x count %d context %" PRIx64 " head %x tail %x busy %d"
 megasas_qf_update(unsigned int head, unsigned int tail, unsigned int busy) "head %x tail %x busy %d"
@@ -784,7 +805,7 @@ megasas_dcmd_pd_list_query(int cmd, int flags) "scmd %d: query flags %x"
 megasas_dcmd_reset_ld(int cmd, int target_id) "scmd %d: dev %d"
 megasas_dcmd_unsupported(int cmd, unsigned long size) "scmd %d: set properties len %ld"
 megasas_abort_frame(int cmd, int abort_cmd) "scmd %d: frame %x"
-megasas_abort_no_cmd(int cmd, uint64_t context) "scmd %d: no active command for frame context %" PRIx64 ""
+megasas_abort_no_cmd(int cmd, uint64_t context) "scmd %d: no active command for frame context %" PRIx64
 megasas_abort_invalid_context(int cmd, uint64_t context, int abort_cmd) "scmd %d: invalid frame context %" PRIx64 " for abort frame %x"
 megasas_reset(int fw_state) "firmware state %x"
 megasas_init(int sges, int cmds, const char *mode) "Using %d sges, %d cmds, %s mode"
@@ -877,7 +898,7 @@ milkymist_vgafb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08
 mipsnet_send(uint32_t size) "sending len=%u"
 mipsnet_receive(uint32_t size) "receiving len=%u"
 mipsnet_read(uint64_t addr, uint32_t val) "read addr=0x%" PRIx64 " val=0x%x"
-mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64 ""
+mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64
 mipsnet_irq(uint32_t isr, uint32_t intctl) "set irq to %d (%02x)"
 
 # hw/isa/pc87312.c
@@ -891,8 +912,8 @@ pc87312_info_serial(int n, uint32_t base, uint32_t irq) "id=%d, base 0x%x, irq %
 # hw/scsi/vmw_pvscsi.c
 pvscsi_ring_init_data(uint32_t txr_len_log2, uint32_t rxr_len_log2) "TX/RX rings logarithms set to %d/%d"
 pvscsi_ring_init_msg(uint32_t len_log2) "MSG ring logarithm set to %d"
-pvscsi_ring_flush_cmp(uint64_t filled_cmp_ptr) "new production counter of completion ring is 0x%"PRIx64""
-pvscsi_ring_flush_msg(uint64_t filled_cmp_ptr) "new production counter of message ring is 0x%"PRIx64""
+pvscsi_ring_flush_cmp(uint64_t filled_cmp_ptr) "new production counter of completion ring is 0x%"PRIx64
+pvscsi_ring_flush_msg(uint64_t filled_cmp_ptr) "new production counter of message ring is 0x%"PRIx64
 pvscsi_update_irq_level(bool raise, uint64_t mask, uint64_t status) "interrupt level set to %d (MASK: 0x%"PRIx64", STATUS: 0x%"PRIx64")"
 pvscsi_update_irq_msi(void) "sending MSI notification"
 pvscsi_cmp_ring_put(unsigned long addr) "got completion descriptor 0x%lx"
@@ -904,7 +925,7 @@ pvscsi_command_complete_not_found(uint32_t tag) "can't find request for tag 0x%x
 pvscsi_command_complete_data_run(void) "not all data required for command transferred"
 pvscsi_command_complete_sense_len(int len) "sense information length is %d bytes"
 pvscsi_convert_sglist(uint64_t context, unsigned long addr, uint32_t resid) "element: ctx: 0x%"PRIx64" addr: 0x%lx, len: %ul"
-pvscsi_process_req_descr(uint8_t cmd, uint64_t ctx) "SCSI cmd 0x%x, ctx: 0x%"PRIx64""
+pvscsi_process_req_descr(uint8_t cmd, uint64_t ctx) "SCSI cmd 0x%x, ctx: 0x%"PRIx64
 pvscsi_process_req_descr_unknown_device(void) "command directed to unknown device rejected"
 pvscsi_process_req_descr_invalid_dir(void) "command with invalid transfer direction rejected"
 pvscsi_process_io(unsigned long addr) "got descriptor 0x%lx"
@@ -912,15 +933,15 @@ pvscsi_on_cmd_noimpl(const char* cmd) "unimplemented command %s ignored"
 pvscsi_on_cmd_reset_dev(uint32_t tgt, int lun, void* dev) "PVSCSI_CMD_RESET_DEVICE[target %u lun %d (dev 0x%p)]"
 pvscsi_on_cmd_arrived(const char* cmd) "command %s arrived"
 pvscsi_on_cmd_abort(uint64_t ctx, uint32_t tgt) "command PVSCSI_CMD_ABORT_CMD for ctx 0x%"PRIx64", target %u"
-pvscsi_on_cmd_unknown(uint64_t cmd_id) "unknown command %"PRIx64""
+pvscsi_on_cmd_unknown(uint64_t cmd_id) "unknown command %"PRIx64
 pvscsi_on_cmd_unknown_data(uint32_t data) "data for unknown command 0x:%x"
-pvscsi_io_write(const char* cmd, uint64_t val) "%s write: %"PRIx64""
-pvscsi_io_write_unknown(unsigned long addr, unsigned sz, uint64_t val) "unknown write address: 0x%lx size: %u bytes value: 0x%"PRIx64""
-pvscsi_io_read(const char* cmd, uint64_t status) "%s read: 0x%"PRIx64""
+pvscsi_io_write(const char* cmd, uint64_t val) "%s write: %"PRIx64
+pvscsi_io_write_unknown(unsigned long addr, unsigned sz, uint64_t val) "unknown write address: 0x%lx size: %u bytes value: 0x%"PRIx64
+pvscsi_io_read(const char* cmd, uint64_t status) "%s read: 0x%"PRIx64
 pvscsi_io_read_unknown(unsigned long addr, unsigned sz) "unknown read address: 0x%lx size: %u bytes"
 pvscsi_init_msi_fail(int res) "failed to initialize MSI, error %d"
 pvscsi_state(const char* state) "starting %s ..."
-pvscsi_tx_rings_ppn(const char* label, uint64_t ppn) "%s page: %"PRIx64""
+pvscsi_tx_rings_ppn(const char* label, uint64_t ppn) "%s page: %"PRIx64
 pvscsi_tx_rings_num_pages(const char* label, uint32_t num) "Number of %s pages: %u"
 
 # xen-hvm.c
@@ -935,6 +956,13 @@ xen_map_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %
 xen_unmap_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
 xen_map_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
 xen_unmap_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
+handle_ioreq(void *req, uint32_t type, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p type=%d dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_read(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p read type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_write(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p write type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p pio dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio_read_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio read reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_pio_write_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio write reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_move(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p copy dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
 
 # xen-mapcache.c
 xen_map_cache(uint64_t phys_addr) "want %#"PRIx64
@@ -1025,9 +1053,9 @@ esp_pci_sbac_write(uint32_t reg, uint32_t val) "sbac: 0x%8.8x -> 0x%8.8x"
 # monitor.c
 handle_qmp_command(void *mon, const char *cmd_name) "mon %p cmd_name \"%s\""
 monitor_protocol_emitter(void *mon) "mon %p"
-monitor_protocol_event_handler(uint32_t event, void *data, uint64_t last, uint64_t now) "event=%d data=%p last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_handler(uint32_t event, void *qdict) "event=%d data=%p"
 monitor_protocol_event_emit(uint32_t event, void *data) "event=%d data=%p"
-monitor_protocol_event_queue(uint32_t event, void *data, uint64_t rate, uint64_t last, uint64_t now) "event=%d data=%p rate=%" PRId64 " last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_queue(uint32_t event, void *qdict, uint64_t rate) "event=%d data=%p rate=%" PRId64
 monitor_protocol_event_throttle(uint32_t event, uint64_t rate) "event=%d rate=%" PRId64
 
 # hw/net/opencores_eth.c
@@ -1049,10 +1077,10 @@ v9fs_rerror(uint16_t tag, uint8_t id, int err) "tag %d id %d err %d"
 v9fs_version(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
 v9fs_version_return(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
 v9fs_attach(uint16_t tag, uint8_t id, int32_t fid, int32_t afid, char* uname, char* aname) "tag %u id %u fid %d afid %d uname %s aname %s"
-v9fs_attach_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d type %d version %d path %"PRId64""
+v9fs_attach_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d type %d version %d path %"PRId64
 v9fs_stat(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
 v9fs_stat_return(uint16_t tag, uint8_t id, int32_t mode, int32_t atime, int32_t mtime, int64_t length) "tag %d id %d stat={mode %d atime %d mtime %d length %"PRId64"}"
-v9fs_getattr(uint16_t tag, uint8_t id, int32_t fid, uint64_t request_mask) "tag %d id %d fid %d request_mask %"PRIu64""
+v9fs_getattr(uint16_t tag, uint8_t id, int32_t fid, uint64_t request_mask) "tag %d id %d fid %d request_mask %"PRIu64
 v9fs_getattr_return(uint16_t tag, uint8_t id, uint64_t result_mask, uint32_t mode, uint32_t uid, uint32_t gid) "tag %d id %d getattr={result_mask %"PRId64" mode %u uid %u gid %u}"
 v9fs_walk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, uint16_t nwnames) "tag %d id %d fid %d newfid %d nwnames %d"
 v9fs_walk_return(uint16_t tag, uint8_t id, uint16_t nwnames, void* qids) "tag %d id %d nwnames %d qids %p"
@@ -1078,14 +1106,14 @@ v9fs_remove(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
 v9fs_wstat(uint16_t tag, uint8_t id, int32_t fid, int32_t mode, int32_t atime, int32_t mtime) "tag %u id %u fid %d stat={mode %d atime %d mtime %d}"
 v9fs_mknod(uint16_t tag, uint8_t id, int32_t fid, int mode, int major, int minor) "tag %d id %d fid %d mode %d major %d minor %d"
 v9fs_mknod_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d qid={type %d version %d path %"PRId64"}"
-v9fs_lock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length) "tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64""
+v9fs_lock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length) "tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64
 v9fs_lock_return(uint16_t tag, uint8_t id, int8_t status) "tag %d id %d status %d"
-v9fs_getlock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)"tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64""
+v9fs_getlock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)"tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64
 v9fs_getlock_return(uint16_t tag, uint8_t id, uint8_t type, uint64_t start, uint64_t length, uint32_t proc_id) "tag %d id %d type %d start %"PRIu64" length %"PRIu64" proc_id %u"
 v9fs_mkdir(uint16_t tag, uint8_t id, int32_t fid, char* name, int mode, uint32_t gid) "tag %u id %u fid %d name %s mode %d gid %u"
 v9fs_mkdir_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int err) "tag %u id %u qid={type %d version %d path %"PRId64"} err %d"
 v9fs_xattrwalk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, char* name) "tag %d id %d fid %d newfid %d name %s"
-v9fs_xattrwalk_return(uint16_t tag, uint8_t id, int64_t size) "tag %d id %d size %"PRId64""
+v9fs_xattrwalk_return(uint16_t tag, uint8_t id, int64_t size) "tag %d id %d size %"PRId64
 v9fs_xattrcreate(uint16_t tag, uint8_t id, int32_t fid, char* name, int64_t size, int flags) "tag %d id %d fid %d name %s size %"PRId64" flags %d"
 v9fs_readlink(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
 v9fs_readlink_return(uint16_t tag, uint8_t id, char* target) "tag %d id %d name %s"
@@ -1093,12 +1121,12 @@ v9fs_readlink_return(uint16_t tag, uint8_t id, char* target) "tag %d id %d name
 # target-sparc/mmu_helper.c
 mmu_helper_dfault(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DFAULT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
 mmu_helper_dprot(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DPROT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
-mmu_helper_dmiss(uint64_t address, uint64_t context) "DMISS at %"PRIx64" context %"PRIx64""
-mmu_helper_tfault(uint64_t address, uint64_t context) "TFAULT at %"PRIx64" context %"PRIx64""
-mmu_helper_tmiss(uint64_t address, uint64_t context) "TMISS at %"PRIx64" context %"PRIx64""
-mmu_helper_get_phys_addr_code(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64""
-mmu_helper_get_phys_addr_data(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64""
-mmu_helper_mmu_fault(uint64_t address, uint64_t paddr, int mmu_idx, uint32_t tl, uint64_t prim_context, uint64_t sec_context) "Translate at %"PRIx64" -> %"PRIx64", mmu_idx=%d tl=%d primary context=%"PRIx64" secondary context=%"PRIx64""
+mmu_helper_dmiss(uint64_t address, uint64_t context) "DMISS at %"PRIx64" context %"PRIx64
+mmu_helper_tfault(uint64_t address, uint64_t context) "TFAULT at %"PRIx64" context %"PRIx64
+mmu_helper_tmiss(uint64_t address, uint64_t context) "TMISS at %"PRIx64" context %"PRIx64
+mmu_helper_get_phys_addr_code(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64
+mmu_helper_get_phys_addr_data(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64
+mmu_helper_mmu_fault(uint64_t address, uint64_t paddr, int mmu_idx, uint32_t tl, uint64_t prim_context, uint64_t sec_context) "Translate at %"PRIx64" -> %"PRIx64", mmu_idx=%d tl=%d primary context=%"PRIx64" secondary context=%"PRIx64
 
 # target-sparc/int64_helper.c
 int_helper_set_softint(uint32_t softint) "new %08x"
@@ -1133,6 +1161,7 @@ console_select(int nr) "%d"
 console_refresh(int interval) "interval %d ms"
 displaysurface_create(void *display_surface, int w, int h) "surface=%p, %dx%d"
 displaysurface_create_from(void *display_surface, int w, int h, uint32_t format) "surface=%p, %dx%d, format 0x%x"
+displaysurface_create_pixman(void *display_surface) "surface=%p"
 displaysurface_free(void *display_surface) "surface=%p"
 displaychangelistener_register(void *dcl, const char *name) "%p [ %s ]"
 displaychangelistener_unregister(void *dcl, const char *name) "%p [ %s ]"
@@ -1142,7 +1171,8 @@ ppm_save(const char *filename, void *display_surface) "%s surface=%p"
 gd_switch(const char *tab, int width, int height) "tab=%s, width=%d, height=%d"
 gd_update(const char *tab, int x, int y, int w, int h) "tab=%s, x=%d, y=%d, w=%d, h=%d"
 gd_key_event(const char *tab, int gdk_keycode, int qemu_keycode, const char *action) "tab=%s, translated GDK keycode %d to QEMU keycode %d (%s)"
-gd_grab(const char *tab, const char *device, bool on) "tab=%s, %s %d"
+gd_grab(const char *tab, const char *device, const char *reason) "tab=%s, dev=%s, reason=%s"
+gd_ungrab(const char *tab, const char *device) "tab=%s, dev=%s"
 
 # ui/vnc.c
 vnc_key_guest_leds(bool caps, bool num, bool scroll) "caps %d, num %d, scroll %d"
@@ -1171,6 +1201,7 @@ vmware_scratch_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
 vmware_setmode(uint32_t w, uint32_t h, uint32_t bpp) "%dx%d @ %d bpp"
 
 # hw/display/virtio-gpu.c
+virtio_gpu_features(bool virgl) "virgl %d"
 virtio_gpu_cmd_get_display_info(void) ""
 virtio_gpu_cmd_get_caps(void) ""
 virtio_gpu_cmd_set_scanout(uint32_t id, uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "id %d, res 0x%x, w %d, h %d, x %d, y %d"
@@ -1180,22 +1211,57 @@ virtio_gpu_cmd_res_unref(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_back_attach(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_back_detach(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_xfer_toh_2d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_toh_3d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_fromh_3d(uint32_t res) "res 0x%x"
 virtio_gpu_cmd_res_flush(uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "res 0x%x, w %d, h %d, x %d, y %d"
+virtio_gpu_cmd_ctx_create(uint32_t ctx, const char *name) "ctx 0x%x, name %s"
+virtio_gpu_cmd_ctx_destroy(uint32_t ctx) "ctx 0x%x"
+virtio_gpu_cmd_ctx_res_attach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_res_detach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_submit(uint32_t ctx, uint32_t size) "ctx 0x%x, size %d"
+virtio_gpu_update_cursor(uint32_t scanout, uint32_t x, uint32_t y, const char *type, uint32_t res) "scanout %d, x %d, y %d, %s, res 0x%x"
 virtio_gpu_fence_ctrl(uint64_t fence, uint32_t type) "fence 0x%" PRIx64 ", type 0x%x"
 virtio_gpu_fence_resp(uint64_t fence) "fence 0x%" PRIx64
 
 # migration/savevm.c
 qemu_loadvm_state_section(unsigned int section_type) "%d"
+qemu_loadvm_state_section_command(int ret) "%d"
 qemu_loadvm_state_section_partend(uint32_t section_id) "%u"
+qemu_loadvm_state_main(void) ""
+qemu_loadvm_state_main_quit_parent(void) ""
+qemu_loadvm_state_post_main(int ret) "%d"
 qemu_loadvm_state_section_startfull(uint32_t section_id, const char *idstr, uint32_t instance_id, uint32_t version_id) "%u(%s) %u %u"
+qemu_savevm_send_packaged(void) ""
+loadvm_handle_cmd_packaged(unsigned int length) "%u"
+loadvm_handle_cmd_packaged_main(int ret) "%d"
+loadvm_handle_cmd_packaged_received(int ret) "%d"
+loadvm_postcopy_handle_advise(void) ""
+loadvm_postcopy_handle_listen(void) ""
+loadvm_postcopy_handle_run(void) ""
+loadvm_postcopy_handle_run_cpu_sync(void) ""
+loadvm_postcopy_handle_run_vmstart(void) ""
+loadvm_postcopy_ram_handle_discard(void) ""
+loadvm_postcopy_ram_handle_discard_end(void) ""
+loadvm_postcopy_ram_handle_discard_header(const char *ramid, uint16_t len) "%s: %ud"
+loadvm_process_command(uint16_t com, uint16_t len) "com=0x%x len=%d"
+loadvm_process_command_ping(uint32_t val) "%x"
+postcopy_ram_listen_thread_exit(void) ""
+postcopy_ram_listen_thread_start(void) ""
+qemu_savevm_send_postcopy_advise(void) ""
+qemu_savevm_send_postcopy_ram_discard(const char *id, uint16_t len) "%s: %ud"
+savevm_command_send(uint16_t command, uint16_t len) "com=0x%x len=%d"
 savevm_section_start(const char *id, unsigned int section_id) "%s, section_id %u"
 savevm_section_end(const char *id, unsigned int section_id, int ret) "%s, section_id %u -> %d"
 savevm_section_skip(const char *id, unsigned int section_id) "%s, section_id %u"
+savevm_send_open_return_path(void) ""
+savevm_send_ping(uint32_t val) "%x"
+savevm_send_postcopy_listen(void) ""
+savevm_send_postcopy_run(void) ""
 savevm_state_begin(void) ""
 savevm_state_header(void) ""
 savevm_state_iterate(void) ""
-savevm_state_complete(void) ""
-savevm_state_cancel(void) ""
+savevm_state_cleanup(void) ""
+savevm_state_complete_precopy(void) ""
 vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s"
 vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s"
 qemu_announce_self_iter(const char *mac) "%s"
@@ -1213,9 +1279,14 @@ vmstate_subsection_load_good(const char *parent) "%s"
 qemu_file_fclose(void) ""
 
 # migration/ram.c
+get_queued_page(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr) "%s/%" PRIx64 " ram_addr=%" PRIx64
+get_queued_page_not_dirty(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr, int sent) "%s/%" PRIx64 " ram_addr=%" PRIx64 " (sent=%d)"
 migration_bitmap_sync_start(void) ""
-migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""
+migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64
 migration_throttle(void) ""
+ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
+ram_postcopy_send_discard_bitmap(void) ""
+ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: %zx len: %zx"
 
 # hw/display/qxl.c
 disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
@@ -1225,7 +1296,7 @@ qxl_create_guest_primary_rest(int qid, int32_t stride, uint32_t type, uint32_t f
 qxl_destroy_primary(int qid) "%d"
 qxl_enter_vga_mode(int qid) "%d"
 qxl_exit_vga_mode(int qid) "%d"
-qxl_hard_reset(int qid, int64_t loadvm) "%d loadvm=%"PRId64""
+qxl_hard_reset(int qid, int64_t loadvm) "%d loadvm=%"PRId64
 qxl_interface_async_complete_io(int qid, uint32_t current_async, void *cookie) "%d current=%d cookie=%p"
 qxl_interface_attach_worker(int qid) "%d"
 qxl_interface_get_init_info(int qid) "%d"
@@ -1302,8 +1373,8 @@ spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
 spapr_pci_msi_retry(unsigned config_addr, unsigned req_num, unsigned max_irqs) "Guest device at %x asked %u, have only %u"
 
 # hw/pci/pci.c
-pci_update_mappings_del(void *d, uint32_t bus, uint32_t func, uint32_t slot, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
-pci_update_mappings_add(void *d, uint32_t bus, uint32_t func, uint32_t slot, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+pci_update_mappings_del(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+pci_update_mappings_add(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
 
 # hw/net/pcnet.c
 pcnet_s_reset(void *s) "s=%p"
@@ -1338,8 +1409,6 @@ xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority) "ics_write_
 xics_ics_reject(int nr, int srcno) "reject irq %#x [src %d]"
 xics_ics_eoi(int nr) "ics_eoi: irq %#x"
 xics_alloc(int src, int irq) "source#%d, irq %d"
-xics_alloc_failed_hint(int src, int irq) "source#%d, irq %d is already in use"
-xics_alloc_failed_no_left(int src) "source#%d, no irq left"
 xics_alloc_block(int src, int first, int num, bool lsi, int align) "source#%d, first irq %d, %d irqs, lsi=%d, alignnum %d"
 xics_ics_free(int src, int irq, int num) "Source#%d, first irq %d, %d irqs"
 xics_ics_free_warn(int src, int irq) "Source#%d, irq %d is already free"
@@ -1367,11 +1436,25 @@ spapr_iommu_new_table(uint64_t liobn, void *tcet, void *table, int fd) "liobn=%"
 # hw/ppc/ppc.c
 ppc_tb_adjust(uint64_t offs1, uint64_t offs2, int64_t diff, int64_t seconds) "adjusted from 0x%"PRIx64" to 0x%"PRIx64", diff %"PRId64" (%"PRId64"s)"
 
+# hw/ppc/prep.c
+prep_io_800_writeb(uint32_t addr, uint32_t val) "0x%08" PRIx32 " => 0x%02" PRIx32
+prep_io_800_readb(uint32_t addr, uint32_t retval) "0x%08" PRIx32 " <= 0x%02" PRIx32
+
+# io/buffer.c
+buffer_resize(const char *buf, size_t olen, size_t len) "%s: old %zd, new %zd"
+buffer_move_empty(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_move(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_free(const char *buf, size_t len) "%s: capacity %zd"
+
 # util/hbitmap.c
 hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
 hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
 hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
 
+# target-s390x/mmu_helper.c
+get_skeys_nonzero(int rc) "SKEY: Call to get_skeys unexpectedly returned %d"
+set_skeys_nonzero(int rc) "SKEY: Call to set_skeys unexpectedly returned %d"
+
 # target-s390x/ioinst.c
 ioinst(const char *insn) "IOINST: %s"
 ioinst_sch_id(const char *insn, int cssid, int ssid, int schid) "IOINST: %s (%x.%x.%04x)"
@@ -1397,15 +1480,40 @@ flic_no_device_api(int err) "flic: no Device Contral API support %d"
 flic_reset_failed(int err) "flic: reset failed %d"
 
 # migration.c
+await_return_path_close_on_source_close(void) ""
+await_return_path_close_on_source_joining(void) ""
 migrate_set_state(int new_state) "new state %d"
 migrate_fd_cleanup(void) ""
 migrate_fd_error(void) ""
 migrate_fd_cancel(void) ""
-migrate_pending(uint64_t size, uint64_t max) "pending size %" PRIu64 " max %" PRIu64
-migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
-migrate_state_too_big(void) ""
+migrate_handle_rp_req_pages(const char *rbname, size_t start, size_t len) "in %s at %zx len %zx"
+migrate_pending(uint64_t size, uint64_t max, uint64_t post, uint64_t nonpost) "pending size %" PRIu64 " max %" PRIu64 " (post=%" PRIu64 " nonpost=%" PRIu64 ")"
+migrate_send_rp_message(int msg_type, uint16_t len) "%d: len %d"
+migration_completion_file_err(void) ""
+migration_completion_postcopy_end(void) ""
+migration_completion_postcopy_end_after_complete(void) ""
+migration_completion_postcopy_end_before_rp(void) ""
+migration_completion_postcopy_end_after_rp(int rp_error) "%d"
+migration_thread_after_loop(void) ""
+migration_thread_file_err(void) ""
+migration_thread_setup_complete(void) ""
+open_return_path_on_source(void) ""
+open_return_path_on_source_continue(void) ""
+postcopy_start(void) ""
+postcopy_start_set_run(void) ""
+source_return_path_thread_bad_end(void) ""
+source_return_path_thread_end(void) ""
+source_return_path_thread_entry(void) ""
+source_return_path_thread_loop_top(void) ""
+source_return_path_thread_pong(uint32_t val) "%x"
+source_return_path_thread_shut(uint32_t val) "%x"
 migrate_global_state_post_load(const char *state) "loaded state: %s"
 migrate_global_state_pre_save(const char *state) "saved state: %s"
+migration_thread_low_pending(uint64_t pending) "%" PRIu64
+migrate_state_too_big(void) ""
+migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
+process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d"
+process_incoming_migration_co_postcopy_end_main(void) ""
 
 # migration/rdma.c
 qemu_rdma_accept_incoming_migration(void) ""
@@ -1426,7 +1534,7 @@ qemu_rdma_exchange_get_response_none(const char *desc, int type) "Surprise: got
 qemu_rdma_exchange_send_issue_callback(void) ""
 qemu_rdma_exchange_send_waiting(const char *desc) "Waiting for response %s"
 qemu_rdma_exchange_send_received(const char *desc) "Response %s received."
-qemu_rdma_fill(int64_t control_len, int size) "RDMA %" PRId64 " of %d bytes already in buffer"
+qemu_rdma_fill(size_t control_len, size_t size) "RDMA %zd of %zd bytes already in buffer"
 qemu_rdma_init_ram_blocks(int blocks) "Allocated %d local ram block structures"
 qemu_rdma_poll_recv(const char *compstr, int64_t comp, int64_t id, int sent) "completion %s #%" PRId64 " received (%" PRId64 ") left %d"
 qemu_rdma_poll_write(const char *compstr, int64_t comp, int left, uint64_t block, uint64_t chunk, void *local, void *remote) "completions %s (%" PRId64 ") left %d, block %" PRIu64 ", chunk: %" PRIu64 " %p %p"
@@ -1471,6 +1579,25 @@ rdma_start_incoming_migration_after_rdma_listen(void) ""
 rdma_start_outgoing_migration_after_rdma_connect(void) ""
 rdma_start_outgoing_migration_after_rdma_source_init(void) ""
 
+# migration/postcopy-ram.c
+postcopy_discard_send_finish(const char *ramblock, int nwords, int ncmds) "%s mask words sent=%d in %d commands"
+postcopy_discard_send_range(const char *ramblock, unsigned long start, unsigned long length) "%s:%lx/%lx"
+postcopy_ram_discard_range(void *start, size_t length) "%p,+%zx"
+postcopy_cleanup_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_init_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_nhp_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_place_page(void *host_addr) "host=%p"
+postcopy_place_page_zero(void *host_addr) "host=%p"
+postcopy_ram_enable_notify(void) ""
+postcopy_ram_fault_thread_entry(void) ""
+postcopy_ram_fault_thread_exit(void) ""
+postcopy_ram_fault_thread_quit(void) ""
+postcopy_ram_fault_thread_request(uint64_t hostaddr, const char *ramblock, size_t offset) "Request for HVA=%" PRIx64 " rb=%s offset=%zx"
+postcopy_ram_incoming_cleanup_closeuf(void) ""
+postcopy_ram_incoming_cleanup_entry(void) ""
+postcopy_ram_incoming_cleanup_exit(void) ""
+postcopy_ram_incoming_cleanup_join(void) ""
+
 # kvm-all.c
 kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
 kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
@@ -1494,8 +1621,12 @@ disable exec_tb_exit(void *next_tb, unsigned int flags) "tb:%p flags=%x"
 translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p"
 
 # memory.c
-memory_region_ops_read(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
-memory_region_ops_write(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr %#"PRIx64" value %#"PRIx64" size %u"
 
 # qom/object.c
 object_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
@@ -1509,56 +1640,33 @@ xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (ad
 pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
 pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
 
-# hw/vfio/vfio-pci.c
+# hw/vfio/pci.c
 vfio_intx_interrupt(const char *name, char line) " (%s) Pin %c"
-vfio_eoi(const char *name) " (%s) EOI"
-vfio_enable_intx_kvm(const char *name) " (%s) KVM INTx accel enabled"
-vfio_disable_intx_kvm(const char *name) " (%s) KVM INTx accel disabled"
-vfio_update_irq(const char *name, int new_irq, int target_irq) " (%s) IRQ moved %d -> %d"
-vfio_enable_intx(const char *name) " (%s)"
-vfio_disable_intx(const char *name) " (%s)"
+vfio_intx_eoi(const char *name) " (%s) EOI"
+vfio_intx_enable_kvm(const char *name) " (%s) KVM INTx accel enabled"
+vfio_intx_disable_kvm(const char *name) " (%s) KVM INTx accel disabled"
+vfio_intx_update(const char *name, int new_irq, int target_irq) " (%s) IRQ moved %d -> %d"
+vfio_intx_enable(const char *name) " (%s)"
+vfio_intx_disable(const char *name) " (%s)"
 vfio_msi_interrupt(const char *name, int index, uint64_t addr, int data) " (%s) vector %d 0x%"PRIx64"/0x%x"
 vfio_msix_vector_do_use(const char *name, int index) " (%s) vector %d used"
 vfio_msix_vector_release(const char *name, int index) " (%s) vector %d released"
-vfio_enable_msix(const char *name) " (%s)"
-vfio_enable_msi(const char *name, int nr_vectors) " (%s) Enabled %d MSI vectors"
-vfio_disable_msix(const char *name) " (%s)"
-vfio_disable_msi(const char *name) " (%s)"
+vfio_msix_enable(const char *name) " (%s)"
+vfio_msix_pba_disable(const char *name) " (%s)"
+vfio_msix_pba_enable(const char *name) " (%s)"
+vfio_msix_disable(const char *name) " (%s)"
+vfio_msix_fixup(const char *name, int bar, uint64_t start, uint64_t end) " (%s) MSI-X region %d mmap fixup [0x%"PRIx64" - 0x%"PRIx64"]"
+vfio_msi_enable(const char *name, int nr_vectors) " (%s) Enabled %d MSI vectors"
+vfio_msi_disable(const char *name) " (%s)"
 vfio_pci_load_rom(const char *name, unsigned long size, unsigned long offset, unsigned long flags) "Device %s ROM:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
 vfio_rom_read(const char *name, uint64_t addr, int size, uint64_t data) " (%s, 0x%"PRIx64", 0x%x) = 0x%"PRIx64
 vfio_pci_size_rom(const char *name, int size) "%s ROM size 0x%x"
 vfio_vga_write(uint64_t addr, uint64_t data, int size) " (0x%"PRIx64", 0x%"PRIx64", %d)"
 vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"PRIx64
-# remove ) =
-vfio_generic_window_quirk_read(const char * region_name, const char *name, int index, uint64_t addr, int size, uint64_t data) "%s read(%s:BAR%d+0x%"PRIx64", %d = 0x%"PRIx64
-## remove )
-vfio_generic_window_quirk_write(const char * region_name, const char *name, int index, uint64_t addr, uint64_t data, int size) "%s write(%s:BAR%d+0x%"PRIx64", 0x%"PRIx64", %d"
-# remove ) =
-vfio_generic_quirk_read(const char * region_name, const char *name, int index, uint64_t addr, int size, uint64_t data) "%s read(%s:BAR%d+0x%"PRIx64", %d = 0x%"PRIx64
-# remove )
-vfio_generic_quirk_write(const char * region_name, const char *name, int index, uint64_t addr, uint64_t data, int size) "%s write(%s:BAR%d+0x%"PRIx64", 0x%"PRIx64", %d"
-vfio_ati_3c3_quirk_read(uint64_t data) " (0x3c3, 1) = 0x%"PRIx64
-vfio_vga_probe_ati_3c3_quirk(const char *name) "Enabled ATI/AMD quirk 0x3c3 BAR4for device %s"
-vfio_probe_ati_bar4_window_quirk(const char *name) "Enabled ATI/AMD BAR4 window quirk for device %s"
-#issue with )
-vfio_rtl8168_window_quirk_read_fake(const char *region_name, const char *name) "%s fake read(%s"
-vfio_rtl8168_window_quirk_read_table(const char *region_name, const char *name) "%s MSI-X table read(%s"
-vfio_rtl8168_window_quirk_read_direct(const char *region_name, const char *name) "%s direct read(%s"
-vfio_rtl8168_window_quirk_write_table(const char *region_name, const char *name) "%s MSI-X table write(%s"
-vfio_rtl8168_window_quirk_write_direct(const char *region_name, const char *name) "%s direct write(%s"
-vfio_probe_rtl8168_bar2_window_quirk(const char *name) "Enabled RTL8168 BAR2 window quirk for device %s"
-vfio_probe_ati_bar2_4000_quirk(const char *name) "Enabled ATI/AMD BAR2 0x4000 quirk for device %s"
-vfio_nvidia_3d0_quirk_read(int size, uint64_t data) " (0x3d0, %d) = 0x%"PRIx64
-vfio_nvidia_3d0_quirk_write(uint64_t data, int size) " (0x3d0, 0x%"PRIx64", %d)"
-vfio_vga_probe_nvidia_3d0_quirk(const char *name) "Enabled NVIDIA VGA 0x3d0 quirk for device %s"
-vfio_probe_nvidia_bar5_window_quirk(const char *name) "Enabled NVIDIA BAR5 window quirk for device %s"
-vfio_probe_nvidia_bar0_88000_quirk(const char *name) "Enabled NVIDIA BAR0 0x88000 quirk for device %s"
-vfio_probe_nvidia_bar0_1800_quirk_id(int id) "Nvidia NV%02x"
-vfio_probe_nvidia_bar0_1800_quirk(const char *name) "Enabled NVIDIA BAR0 0x1800 quirk for device %s"
 vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) %x"
 vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
-vfio_setup_msi(const char *name, int pos) "%s PCI MSI CAP @0x%x"
-vfio_early_setup_msix(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
+vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
+vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
 vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
 vfio_check_pm_reset(const char *name) "%s Supports PM reset"
 vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"
@@ -1566,13 +1674,47 @@ vfio_pci_hot_reset(const char *name, const char *type) " (%s) %s"
 vfio_pci_hot_reset_has_dep_devices(const char *name) "%s: hot reset dependent devices:"
 vfio_pci_hot_reset_dep_devices(int domain, int bus, int slot, int function, int group_id) "\t%04x:%02x:%02x.%x group %d"
 vfio_pci_hot_reset_result(const char *name, const char *result) "%s hot reset: %s"
-vfio_populate_device_region(const char *region_name, int index, unsigned long size, unsigned long offset, unsigned long flags) "Device %s region %d:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
 vfio_populate_device_config(const char *name, unsigned long size, unsigned long offset, unsigned long flags) "Device %s config:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
 vfio_populate_device_get_irq_info_failure(void) "VFIO_DEVICE_GET_IRQ_INFO failure: %m"
 vfio_initfn(const char *name, int group_id) " (%s) group %d"
 vfio_pci_reset(const char *name) " (%s)"
 vfio_pci_reset_flr(const char *name) "%s FLR/VFIO_DEVICE_RESET"
 vfio_pci_reset_pm(const char *name) "%s PCI PM Reset"
+vfio_pci_emulated_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_device_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_device_id(const char *name, uint16_t val) "%s %04x"
+
+# hw/vfio/pci-quirks.
+vfio_quirk_rom_blacklisted(const char *name, uint16_t vid, uint16_t did) "%s %04x:%04x"
+vfio_quirk_generic_window_address_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_read(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_mirror_read(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_generic_mirror_write(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_ati_3c3_read(const char *name, uint64_t data) "%s 0x%"PRIx64
+vfio_quirk_ati_3c3_probe(const char *name) "%s"
+vfio_quirk_ati_bar4_probe(const char *name) "%s"
+vfio_quirk_ati_bar2_probe(const char *name) "%s"
+vfio_quirk_nvidia_3d0_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_3d0_read(const char *name, uint8_t offset, unsigned size, uint64_t val) " (%s, @0x%x, len=0x%x) %"PRIx64
+vfio_quirk_nvidia_3d0_write(const char *name, uint8_t offset, uint64_t data, unsigned size) "(%s, @0x%x, 0x%"PRIx64", len=0x%x)"
+vfio_quirk_nvidia_3d0_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar5_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_bar5_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar0_msi_ack(const char *name) "%s"
+vfio_quirk_nvidia_bar0_probe(const char *name) "%s"
+vfio_quirk_rtl8168_fake_latch(const char *name, uint64_t val) "%s 0x%"PRIx64
+vfio_quirk_rtl8168_msix_write(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table write[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_msix_read(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table read[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_probe(const char *name) "%s"
+
+vfio_quirk_ati_bonaire_reset_skipped(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_no_smc(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_timeout(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_done(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset(const char *name) "%s"
+
 
 # hw/vfio/vfio-common.c
 vfio_region_write(const char *name, int index, uint64_t addr, uint64_t data, unsigned size) " (%s:region%d+0x%"PRIx64", 0x%"PRIx64 ", %d)"
@@ -1587,19 +1729,25 @@ vfio_disconnect_container(int fd) "close container->fd=%d"
 vfio_put_group(int fd) "close group->fd=%d"
 vfio_get_device(const char * name, unsigned int flags, unsigned int num_regions, unsigned int num_irqs) "Device %s flags: %u, regions: %u, irqs: %u"
 vfio_put_base_device(int fd) "close vdev->fd=%d"
+vfio_region_setup(const char *dev, int index, const char *name, unsigned long flags, unsigned long offset, unsigned long size) "Device %s, region %d \"%s\", flags: %lx, offset: %lx, size: %lx"
+vfio_region_mmap_fault(const char *name, int index, unsigned long offset, unsigned long size, int fault) "Region %s mmaps[%d], [%lx - %lx], fault: %d"
+vfio_region_mmap(const char *name, unsigned long offset, unsigned long end) "Region %s [%lx - %lx]"
+vfio_region_exit(const char *name, int index) "Device %s, region %d"
+vfio_region_finalize(const char *name, int index) "Device %s, region %d"
+vfio_region_mmaps_set_enabled(const char *name, bool enabled) "Region %s mmaps enabled: %d"
 
 # hw/vfio/platform.c
-vfio_platform_populate_regions(int region_index, unsigned long flag, unsigned long size, int fd, unsigned long offset) "- region %d flags = 0x%lx, size = 0x%lx, fd= %d, offset = 0x%lx"
 vfio_platform_base_device_init(char *name, int groupid) "%s belongs to group #%d"
 vfio_platform_realize(char *name, char *compat) "vfio device %s, compat = %s"
 vfio_platform_eoi(int pin, int fd) "EOI IRQ pin %d (fd=%d)"
-vfio_platform_mmap_set_enabled(bool enabled) "fast path = %d"
 vfio_platform_intp_mmap_enable(int pin) "IRQ #%d still active, stay in slow path"
 vfio_platform_intp_interrupt(int pin, int fd) "Inject IRQ #%d (fd = %d)"
 vfio_platform_intp_inject_pending_lockheld(int pin, int fd) "Inject pending IRQ #%d (fd = %d)"
 vfio_platform_populate_interrupts(int pin, int count, int flags) "- IRQ index %d: count %d, flags=0x%x"
 vfio_intp_interrupt_set_pending(int index) "irq %d is set PENDING"
-vfio_platform_start_irqfd_injection(int index, int fd, int resamplefd) "IRQ index=%d, fd = %d, resamplefd = %d"
+vfio_platform_start_level_irqfd_injection(int index, int fd, int resamplefd) "IRQ index=%d, fd = %d, resamplefd = %d"
+vfio_platform_start_edge_irqfd_injection(int index, int fd) "IRQ index=%d, fd = %d"
+
 
 #hw/acpi/memory_hotplug.c
 mhp_acpi_invalid_slot_selected(uint32_t slot) "0x%"PRIx32
@@ -1639,6 +1787,9 @@ cpu_unhalt(int cpu_index) "unhalting cpu %d"
 # hw/arm/virt-acpi-build.c
 virt_acpi_setup(void) "No fw cfg or ACPI disabled. Bailing out."
 
+# hw/alpha/pci.c
+alpha_pci_iack_write(void) ""
+
 # audio/alsaaudio.c
 alsa_revents(int revents) "revents = %d"
 alsa_pollout(int i, int fd) "i = %d fd = %d"
@@ -1654,3 +1805,107 @@ alsa_no_frames(int state) "No frames available and ALSA state is %d"
 # audio/ossaudio.c
 oss_version(int version) "OSS version = %#x"
 oss_invalid_available_size(int size, int bufsize) "Invalid available size, size=%d bufsize=%d"
+
+# crypto/tlscreds.c
+qcrypto_tls_creds_load_dh(void *creds, const char *filename) "TLS creds load DH creds=%p filename=%s"
+qcrypto_tls_creds_get_path(void *creds, const char *filename, const char *path) "TLS creds path creds=%p filename=%s path=%s"
+
+# crypto/tlscredsanon.c
+qcrypto_tls_creds_anon_load(void *creds, const char *dir) "TLS creds anon load creds=%p dir=%s"
+
+# crypto/tlscredsx509.c
+qcrypto_tls_creds_x509_load(void *creds, const char *dir) "TLS creds x509 load creds=%p dir=%s"
+qcrypto_tls_creds_x509_check_basic_constraints(void *creds, const char *file, int status) "TLS creds x509 check basic constraints creds=%p file=%s status=%d"
+qcrypto_tls_creds_x509_check_key_usage(void *creds, const char *file, int status, int usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%d critical=%d"
+qcrypto_tls_creds_x509_check_key_purpose(void *creds, const char *file, int status, const char *usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%s critical=%d"
+qcrypto_tls_creds_x509_load_cert(void *creds, int isServer, const char *file) "TLS creds x509 load cert creds=%p isServer=%d file=%s"
+qcrypto_tls_creds_x509_load_cert_list(void *creds, const char *file) "TLS creds x509 load cert list creds=%p file=%s"
+
+# crypto/tlssession.c
+qcrypto_tls_session_new(void *session, void *creds, const char *hostname, const char *aclname, int endpoint) "TLS session new session=%p creds=%p hostname=%s aclname=%s endpoint=%d"
+
+# net/vhost-user.c
+vhost_user_event(const char *chr, int event) "chr: %s got event: %d"
+
+# linux-user/signal.c
+user_setup_frame(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_setup_rt_frame(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_do_rt_sigreturn(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_do_sigreturn(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_force_sig(void *env, int target_sig, int host_sig) "env=%p signal %d (host %d)"
+user_handle_signal(void *env, int target_sig) "env=%p signal %d"
+user_host_signal(void *env, int host_sig, int target_sig) "env=%p signal %d (target %d("
+user_queue_signal(void *env, int target_sig) "env=%p signal %d"
+user_s390x_restore_sigregs(void *env, uint64_t sc_psw_addr, uint64_t env_psw_addr) "env=%p frame psw.addr %"PRIx64 " current psw.addr %"PRIx64
+
+# io/task.c
+qio_task_new(void *task, void *source, void *func, void *opaque) "Task new task=%p source=%p func=%p opaque=%p"
+qio_task_complete(void *task) "Task complete task=%p"
+qio_task_abort(void *task) "Task abort task=%p"
+qio_task_thread_start(void *task, void *worker, void *opaque) "Task thread start task=%p worker=%p opaque=%p"
+qio_task_thread_run(void *task) "Task thread run task=%p"
+qio_task_thread_exit(void *task) "Task thread exit task=%p"
+qio_task_thread_result(void *task) "Task thread result task=%p"
+
+# io/channel-socket.c
+qio_channel_socket_new(void *ioc) "Socket new ioc=%p"
+qio_channel_socket_new_fd(void *ioc, int fd) "Socket new ioc=%p fd=%d"
+qio_channel_socket_connect_sync(void *ioc, void *addr) "Socket connect sync ioc=%p addr=%p"
+qio_channel_socket_connect_async(void *ioc, void *addr) "Socket connect async ioc=%p addr=%p"
+qio_channel_socket_connect_fail(void *ioc) "Socket connect fail ioc=%p"
+qio_channel_socket_connect_complete(void *ioc, int fd) "Socket connect complete ioc=%p fd=%d"
+qio_channel_socket_listen_sync(void *ioc, void *addr) "Socket listen sync ioc=%p addr=%p"
+qio_channel_socket_listen_async(void *ioc, void *addr) "Socket listen async ioc=%p addr=%p"
+qio_channel_socket_listen_fail(void *ioc) "Socket listen fail ioc=%p"
+qio_channel_socket_listen_complete(void *ioc, int fd) "Socket listen complete ioc=%p fd=%d"
+qio_channel_socket_dgram_sync(void *ioc, void *localAddr, void *remoteAddr) "Socket dgram sync ioc=%p localAddr=%p remoteAddr=%p"
+qio_channel_socket_dgram_async(void *ioc, void *localAddr, void *remoteAddr) "Socket dgram async ioc=%p localAddr=%p remoteAddr=%p"
+qio_channel_socket_dgram_fail(void *ioc) "Socket dgram fail ioc=%p"
+qio_channel_socket_dgram_complete(void *ioc, int fd) "Socket dgram complete ioc=%p fd=%d"
+qio_channel_socket_accept(void *ioc) "Socket accept start ioc=%p"
+qio_channel_socket_accept_fail(void *ioc) "Socket accept fail ioc=%p"
+qio_channel_socket_accept_complete(void *ioc, void *cioc, int fd) "Socket accept complete ioc=%p cioc=%p fd=%d"
+
+# io/channel-file.c
+qio_channel_file_new_fd(void *ioc, int fd) "File new fd ioc=%p fd=%d"
+qio_channel_file_new_path(void *ioc, const char *path, int flags, int mode, int fd) "File new fd ioc=%p path=%s flags=%d mode=%d fd=%d"
+
+# io/channel-tls.c
+qio_channel_tls_new_client(void *ioc, void *master, void *creds, const char *hostname) "TLS new client ioc=%p master=%p creds=%p hostname=%s"
+qio_channel_tls_new_server(void *ioc, void *master, void *creds, const char *aclname) "TLS new client ioc=%p master=%p creds=%p acltname=%s"
+qio_channel_tls_handshake_start(void *ioc) "TLS handshake start ioc=%p"
+qio_channel_tls_handshake_pending(void *ioc, int status) "TLS handshake pending ioc=%p status=%d"
+qio_channel_tls_handshake_fail(void *ioc) "TLS handshake fail ioc=%p"
+qio_channel_tls_handshake_complete(void *ioc) "TLS handshake complete ioc=%p"
+qio_channel_tls_credentials_allow(void *ioc) "TLS credentials allow ioc=%p"
+qio_channel_tls_credentials_deny(void *ioc) "TLS credentials deny ioc=%p"
+
+# io/channel-websock.c
+qio_channel_websock_new_server(void *ioc, void *master) "Websock new client ioc=%p master=%p"
+qio_channel_websock_handshake_start(void *ioc) "Websock handshake start ioc=%p"
+qio_channel_websock_handshake_pending(void *ioc, int status) "Websock handshake pending ioc=%p status=%d"
+qio_channel_websock_handshake_reply(void *ioc) "Websock handshake reply ioc=%p"
+qio_channel_websock_handshake_fail(void *ioc) "Websock handshake fail ioc=%p"
+qio_channel_websock_handshake_complete(void *ioc) "Websock handshake complete ioc=%p"
+
+# io/channel-command.c
+qio_channel_command_new_pid(void *ioc, int writefd, int readfd, int pid) "Command new pid ioc=%p writefd=%d readfd=%d pid=%d"
+qio_channel_command_new_spawn(void *ioc, const char *binary, int flags) "Command new spawn ioc=%p binary=%s flags=%d"
+qio_channel_command_abort(void *ioc, int pid) "Command abort ioc=%p pid=%d"
+qio_channel_command_wait(void *ioc, int pid, int ret, int status) "Command abort ioc=%p pid=%d ret=%d status=%d"
+
+# hw/timer/aspeed_timer.c
+aspeed_timer_ctrl_enable(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_external_clock(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_overflow_interrupt(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_pulse_enable(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_set_ctrl2(uint32_t value) "Value: 0x%" PRIx32
+aspeed_timer_set_value(int timer, int reg, uint32_t value) "Timer %d register %d: 0x%" PRIx32
+aspeed_timer_read(uint64_t offset, unsigned size, uint64_t value) "From 0x%" PRIx64 ": of size %u: 0x%" PRIx64
+
+# hw/intc/aspeed_vic.c
+aspeed_vic_set_irq(int irq, int level) "Enabling IRQ %d: %d"
+aspeed_vic_update_fiq(int flags) "Raising FIQ: %d"
+aspeed_vic_update_irq(int flags) "Raising IRQ: %d"
+aspeed_vic_read(uint64_t offset, unsigned size, uint32_t value) "From 0x%" PRIx64 " of size %u: 0x%" PRIx32
+aspeed_vic_write(uint64_t offset, unsigned size, uint32_t data) "To 0x%" PRIx64 " of size %u: 0x%" PRIx32
diff --git a/qemu/trace/Makefile.objs b/qemu/trace/Makefile.objs
index 32f7a32ce..5145b34d1 100644
--- a/qemu/trace/Makefile.objs
+++ b/qemu/trace/Makefile.objs
@@ -1,24 +1,32 @@
 # -*- mode: makefile -*-
 
 ######################################################################
+# tracetool source files
+# Every rule that invokes tracetool must depend on this so code is regenerated
+# if tracetool itself changes.
+
+tracetool-y = $(SRC_PATH)/scripts/tracetool.py
+tracetool-y += $(shell find $(SRC_PATH)/scripts/tracetool -name "*.py")
+
+######################################################################
 # Auto-generated event descriptions for LTTng ust code
 
 ifeq ($(findstring ust,$(TRACE_BACKENDS)),ust)
 $(obj)/generated-ust-provider.h: $(obj)/generated-ust-provider.h-timestamp
-$(obj)/generated-ust-provider.h-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-ust-provider.h-timestamp: $(SRC_PATH)/trace-events $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=ust-events-h \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-ust.c: $(obj)/generated-ust.c-timestamp $(BUILD_DIR)/config-host.mak
-$(obj)/generated-ust.c-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-ust.c-timestamp: $(SRC_PATH)/trace-events $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=ust-events-c \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-events.h: $(obj)/generated-ust-provider.h
 $(obj)/generated-events.c: $(obj)/generated-ust.c
@@ -28,20 +36,20 @@ endif
 # Auto-generated event descriptions
 
 $(obj)/generated-events.h: $(obj)/generated-events.h-timestamp
-$(obj)/generated-events.h-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-events.h-timestamp: $(SRC_PATH)/trace-events $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=events-h \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-events.c: $(obj)/generated-events.c-timestamp $(BUILD_DIR)/config-host.mak
-$(obj)/generated-events.c-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-events.c-timestamp: $(SRC_PATH)/trace-events $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=events-c \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 util-obj-y += generated-events.o
 
@@ -54,7 +62,7 @@ util-obj-y += generated-events.o
 
 $(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
 	@cmp -s $< $@ || cp $< $@
-$(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+$(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=h \
 		--backends=$(TRACE_BACKENDS) \
@@ -65,7 +73,7 @@ $(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/conf
 
 $(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
 	@cmp -s $< $@ || cp $< $@
-$(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+$(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=c \
 		--backends=$(TRACE_BACKENDS) \
@@ -81,12 +89,12 @@ $(obj)/generated-tracers.o: $(obj)/generated-tracers.c $(obj)/generated-tracers.
 # rule file. So we use '.dtrace' instead
 ifeq ($(findstring dtrace,$(TRACE_BACKENDS)),dtrace)
 $(obj)/generated-tracers-dtrace.dtrace: $(obj)/generated-tracers-dtrace.dtrace-timestamp
-$(obj)/generated-tracers-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-tracers-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=d \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-tracers-dtrace.h: $(obj)/generated-tracers-dtrace.dtrace
 	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   $@")
@@ -100,28 +108,28 @@ endif
 # Translation level
 
 $(obj)/generated-helpers-wrappers.h: $(obj)/generated-helpers-wrappers.h-timestamp
-$(obj)/generated-helpers-wrappers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers-wrappers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-wrapper-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.h: $(obj)/generated-helpers.h-timestamp
-$(obj)/generated-helpers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.c: $(obj)/generated-helpers.c-timestamp
-$(obj)/generated-helpers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-c \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.o: $(obj)/generated-helpers.c
 
@@ -129,12 +137,12 @@ target-obj-y += generated-helpers.o
 
 
 $(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
-$(obj)/generated-tcg-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-tcg-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 
 ######################################################################
diff --git a/qemu/trace/control-internal.h b/qemu/trace/control-internal.h
index 5a8df28c5..dcf67f505 100644
--- a/qemu/trace/control-internal.h
+++ b/qemu/trace/control-internal.h
@@ -10,10 +10,11 @@
 #ifndef TRACE__CONTROL_INTERNAL_H
 #define TRACE__CONTROL_INTERNAL_H
 
-#include <string.h>
 
 
 extern TraceEvent trace_events[];
+extern bool trace_events_dstate[];
+extern int trace_events_enabled_count;
 
 
 static inline TraceEventID trace_event_count(void)
@@ -51,17 +52,24 @@ static inline bool trace_event_get_state_static(TraceEvent *ev)
     return ev->sstate;
 }
 
+static inline bool trace_event_get_state_dynamic_by_id(int id)
+{
+    return unlikely(trace_events_enabled_count) && trace_events_dstate[id];
+}
+
 static inline bool trace_event_get_state_dynamic(TraceEvent *ev)
 {
-    assert(ev != NULL);
-    return ev->dstate;
+    int id = trace_event_get_id(ev);
+    return trace_event_get_state_dynamic_by_id(id);
 }
 
 static inline void trace_event_set_state_dynamic(TraceEvent *ev, bool state)
 {
+    int id = trace_event_get_id(ev);
     assert(ev != NULL);
     assert(trace_event_get_state_static(ev));
-    ev->dstate = state;
+    trace_events_enabled_count += state - trace_events_dstate[id];
+    trace_events_dstate[id] = state;
 }
 
 #endif  /* TRACE__CONTROL_INTERNAL_H */
diff --git a/qemu/trace/control.c b/qemu/trace/control.c
index 995beb384..d099f735d 100644
--- a/qemu/trace/control.c
+++ b/qemu/trace/control.c
@@ -7,14 +7,23 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "trace/control.h"
+#include "qemu/help_option.h"
 #ifdef CONFIG_TRACE_SIMPLE
 #include "trace/simple.h"
 #endif
 #ifdef CONFIG_TRACE_FTRACE
 #include "trace/ftrace.h"
 #endif
+#ifdef CONFIG_TRACE_LOG
+#include "qemu/log.h"
+#endif
 #include "qemu/error-report.h"
+#include "monitor/monitor.h"
+
+int trace_events_enabled_count;
+bool trace_events_dstate[TRACE_EVENT_COUNT];
 
 TraceEvent *trace_event_name(const char *name)
 {
@@ -85,7 +94,54 @@ TraceEvent *trace_event_pattern(const char *pat, TraceEvent *ev)
     return NULL;
 }
 
-static void trace_init_events(const char *fname)
+void trace_list_events(void)
+{
+    int i;
+    for (i = 0; i < trace_event_count(); i++) {
+        TraceEvent *res = trace_event_id(i);
+        fprintf(stderr, "%s\n", trace_event_get_name(res));
+    }
+}
+
+static void do_trace_enable_events(const char *line_buf)
+{
+    const bool enable = ('-' != line_buf[0]);
+    const char *line_ptr = enable ? line_buf : line_buf + 1;
+
+    if (trace_event_is_pattern(line_ptr)) {
+        TraceEvent *ev = NULL;
+        while ((ev = trace_event_pattern(line_ptr, ev)) != NULL) {
+            if (trace_event_get_state_static(ev)) {
+                trace_event_set_state_dynamic(ev, enable);
+            }
+        }
+    } else {
+        TraceEvent *ev = trace_event_name(line_ptr);
+        if (ev == NULL) {
+            error_report("WARNING: trace event '%s' does not exist",
+                         line_ptr);
+        } else if (!trace_event_get_state_static(ev)) {
+            error_report("WARNING: trace event '%s' is not traceable",
+                         line_ptr);
+        } else {
+            trace_event_set_state_dynamic(ev, enable);
+        }
+    }
+}
+
+void trace_enable_events(const char *line_buf)
+{
+    if (is_help_option(line_buf)) {
+        trace_list_events();
+        if (cur_mon == NULL) {
+            exit(0);
+        }
+    } else {
+        do_trace_enable_events(line_buf);
+    }
+}
+
+void trace_init_events(const char *fname)
 {
     Location loc;
     FILE *fp;
@@ -111,27 +167,7 @@ static void trace_init_events(const char *fname)
             if ('#' == line_buf[0]) { /* skip commented lines */
                 continue;
             }
-            const bool enable = ('-' != line_buf[0]);
-            char *line_ptr = enable ? line_buf : line_buf + 1;
-            if (trace_event_is_pattern(line_ptr)) {
-                TraceEvent *ev = NULL;
-                while ((ev = trace_event_pattern(line_ptr, ev)) != NULL) {
-                    if (trace_event_get_state_static(ev)) {
-                        trace_event_set_state_dynamic(ev, enable);
-                    }
-                }
-            } else {
-                TraceEvent *ev = trace_event_name(line_ptr);
-                if (ev == NULL) {
-                    error_report("WARNING: trace event '%s' does not exist",
-                                 line_ptr);
-                } else if (!trace_event_get_state_static(ev)) {
-                    error_report("WARNING: trace event '%s' is not traceable",
-                                 line_ptr);
-                } else {
-                    trace_event_set_state_dynamic(ev, enable);
-                }
-            }
+            trace_enable_events(line_buf);
         }
     }
     if (fclose(fp) != 0) {
@@ -142,17 +178,31 @@ static void trace_init_events(const char *fname)
     loc_pop(&loc);
 }
 
-bool trace_init_backends(const char *events, const char *file)
+void trace_init_file(const char *file)
 {
 #ifdef CONFIG_TRACE_SIMPLE
-    if (!st_init(file)) {
-        fprintf(stderr, "failed to initialize simple tracing backend.\n");
-        return false;
+    st_set_trace_file(file);
+#elif defined CONFIG_TRACE_LOG
+    /* If both the simple and the log backends are enabled, "-trace file"
+     * only applies to the simple backend; use "-D" for the log backend.
+     */
+    if (file) {
+        qemu_set_log_filename(file);
     }
 #else
     if (file) {
         fprintf(stderr, "error: -trace file=...: "
                 "option not supported by the selected tracing backends\n");
+        exit(1);
+    }
+#endif
+}
+
+bool trace_init_backends(void)
+{
+#ifdef CONFIG_TRACE_SIMPLE
+    if (!st_init()) {
+        fprintf(stderr, "failed to initialize simple tracing backend.\n");
         return false;
     }
 #endif
@@ -164,6 +214,5 @@ bool trace_init_backends(const char *events, const char *file)
     }
 #endif
 
-    trace_init_events(events);
     return true;
 }
diff --git a/qemu/trace/control.h b/qemu/trace/control.h
index da9bb6b77..e2ba6d4de 100644
--- a/qemu/trace/control.h
+++ b/qemu/trace/control.h
@@ -1,7 +1,7 @@
 /*
  * Interface for configuring and controlling the state of tracing events.
  *
- * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2011-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -104,7 +104,7 @@ static const char * trace_event_get_name(TraceEvent *ev);
  * As a down side, you must always use an immediate #TraceEventID value.
  */
 #define trace_event_get_state(id)                       \
-    ((id ##_ENABLED) && trace_event_get_state_dynamic(trace_event_id(id)))
+    ((id ##_ENABLED) && trace_event_get_state_dynamic_by_id(id))
 
 /**
  * trace_event_get_state_static:
@@ -150,8 +150,6 @@ static void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
 
 /**
  * trace_init_backends:
- * @events: Name of file with events to be enabled at startup; may be NULL.
- *          Corresponds to commandline option "-trace events=...".
  * @file:   Name of trace output file; may be NULL.
  *          Corresponds to commandline option "-trace file=...".
  *
@@ -159,7 +157,45 @@ static void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
  *
  * Returns: Whether the backends could be successfully initialized.
  */
-bool trace_init_backends(const char *events, const char *file);
+bool trace_init_backends(void);
+
+/**
+ * trace_init_events:
+ * @events: Name of file with events to be enabled at startup; may be NULL.
+ *          Corresponds to commandline option "-trace events=...".
+ *
+ * Read the list of enabled tracing events.
+ *
+ * Returns: Whether the backends could be successfully initialized.
+ */
+void trace_init_events(const char *file);
+
+/**
+ * trace_init_file:
+ * @file:   Name of trace output file; may be NULL.
+ *          Corresponds to commandline option "-trace file=...".
+ *
+ * Record the name of the output file for the tracing backend.
+ * Exits if no selected backend does not support specifying the
+ * output file, and a non-NULL file was passed.
+ */
+void trace_init_file(const char *file);
+
+/**
+ * trace_list_events:
+ *
+ * List all available events.
+ */
+void trace_list_events(void);
+
+/**
+ * trace_enable_events:
+ * @line_buf: A string with a glob pattern of events to be enabled or,
+ *            if the string starts with '-', disabled.
+ *
+ * Enable or disable matching events.
+ */
+void trace_enable_events(const char *line_buf);
 
 
 #include "trace/control-internal.h"
diff --git a/qemu/trace/event-internal.h b/qemu/trace/event-internal.h
index b2310d9be..86f6a511b 100644
--- a/qemu/trace/event-internal.h
+++ b/qemu/trace/event-internal.h
@@ -18,7 +18,6 @@
  * @id: Unique event identifier.
  * @name: Event name.
  * @sstate: Static tracing state.
- * @dstate: Dynamic tracing state.
  *
  * Opaque generic description of a tracing event.
  */
@@ -26,7 +25,6 @@ typedef struct TraceEvent {
     TraceEventID id;
     const char * name;
     const bool sstate;
-    bool dstate;
 } TraceEvent;
 
 
diff --git a/qemu/trace/ftrace.c b/qemu/trace/ftrace.c
index a7ae371e6..e953922f5 100644
--- a/qemu/trace/ftrace.c
+++ b/qemu/trace/ftrace.c
@@ -9,10 +9,7 @@
  *
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <fcntl.h>
-#include <limits.h>
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "trace/control.h"
 
diff --git a/qemu/trace/ftrace.h b/qemu/trace/ftrace.h
index 863e052e9..92372e3ca 100644
--- a/qemu/trace/ftrace.h
+++ b/qemu/trace/ftrace.h
@@ -1,7 +1,6 @@
 #ifndef TRACE_FTRACE_H
 #define TRACE_FTRACE_H
 
-#include <stdbool.h>
 
 
 #define MAX_TRACE_STRLEN 512
diff --git a/qemu/trace/qmp.c b/qemu/trace/qmp.c
index 0b1948952..8aa2660aa 100644
--- a/qemu/trace/qmp.c
+++ b/qemu/trace/qmp.c
@@ -7,7 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/typedefs.h"
+#include "qemu/osdep.h"
 #include "qmp-commands.h"
 #include "trace/control.h"
 
diff --git a/qemu/trace/simple.c b/qemu/trace/simple.c
index 11ad03093..3fdcc8226 100644
--- a/qemu/trace/simple.c
+++ b/qemu/trace/simple.c
@@ -8,12 +8,8 @@
  *
  */
 
-#include <stdlib.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 #ifndef _WIN32
-#include <signal.h>
 #include <pthread.h>
 #endif
 #include "qemu/timer.h"
@@ -322,20 +318,20 @@ void st_set_trace_file_enabled(bool enable)
  * @file        The trace file name or NULL for the default name-<pid> set at
  *              config time
  */
-bool st_set_trace_file(const char *file)
+void st_set_trace_file(const char *file)
 {
     st_set_trace_file_enabled(false);
 
     g_free(trace_file_name);
 
     if (!file) {
-        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, getpid());
+        /* Type cast needed for Windows where getpid() returns an int. */
+        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, (pid_t)getpid());
     } else {
         trace_file_name = g_strdup_printf("%s", file);
     }
 
     st_set_trace_file_enabled(true);
-    return true;
 }
 
 void st_print_trace_file_status(FILE *stream, int (*stream_printf)(FILE *stream, const char *fmt, ...))
@@ -373,7 +369,7 @@ static GThread *trace_thread_create(GThreadFunc fn)
     return thread;
 }
 
-bool st_init(const char *file)
+bool st_init(void)
 {
     GThread *thread;
 
@@ -386,6 +382,5 @@ bool st_init(const char *file)
     }
 
     atexit(st_flush_trace_buffer);
-    st_set_trace_file(file);
     return true;
 }
diff --git a/qemu/trace/simple.h b/qemu/trace/simple.h
index 699799685..1e7de4557 100644
--- a/qemu/trace/simple.h
+++ b/qemu/trace/simple.h
@@ -11,17 +11,14 @@
 #ifndef TRACE_SIMPLE_H
 #define TRACE_SIMPLE_H
 
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdio.h>
 
 #include "trace/generated-events.h"
 
 
 void st_print_trace_file_status(FILE *stream, fprintf_function stream_printf);
 void st_set_trace_file_enabled(bool enable);
-bool st_set_trace_file(const char *file);
-bool st_init(const char *file);
+void st_set_trace_file(const char *file);
+bool st_init(void);
 void st_flush_trace_buffer(void);
 
 typedef struct {
diff --git a/qemu/translate-all.c b/qemu/translate-all.c
index 60a3d8b2b..8329ea60e 100644
--- a/qemu/translate-all.c
+++ b/qemu/translate-all.c
@@ -19,16 +19,10 @@
 #ifdef _WIN32
 #include <windows.h>
 #else
-#include <sys/types.h>
 #include <sys/mman.h>
 #endif
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 
 #include "qemu-common.h"
 #define NO_CPU_IO_DEFS
@@ -43,7 +37,6 @@
 #if __FreeBSD_version >= 700104
 #define HAVE_KINFO_GETVMMAP
 #define sigqueue sigqueue_freebsd  /* avoid redefinition */
-#include <sys/time.h>
 #include <sys/proc.h>
 #include <machine/profile.h>
 #define _KERNEL
@@ -62,6 +55,7 @@
 #include "translate-all.h"
 #include "qemu/bitmap.h"
 #include "qemu/timer.h"
+#include "exec/log.h"
 
 //#define DEBUG_TB_INVALIDATE
 //#define DEBUG_FLUSH
@@ -117,18 +111,48 @@ typedef struct PageDesc {
 
 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
 
-uintptr_t qemu_real_host_page_size;
-uintptr_t qemu_real_host_page_mask;
 uintptr_t qemu_host_page_size;
-uintptr_t qemu_host_page_mask;
+intptr_t qemu_host_page_mask;
 
-/* This is a multi-level map on the virtual address space.
-   The bottom level has pointers to PageDesc.  */
+/* The bottom level has pointers to PageDesc */
 static void *l1_map[V_L1_SIZE];
 
 /* code generation context */
 TCGContext tcg_ctx;
 
+/* translation block context */
+#ifdef CONFIG_USER_ONLY
+__thread int have_tb_lock;
+#endif
+
+void tb_lock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(!have_tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    have_tb_lock++;
+#endif
+}
+
+void tb_unlock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(have_tb_lock);
+    have_tb_lock--;
+    qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+#endif
+}
+
+void tb_lock_reset(void)
+{
+#ifdef CONFIG_USER_ONLY
+    if (have_tb_lock) {
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        have_tb_lock = 0;
+    }
+#endif
+}
+
 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
                          tb_page_addr_t phys_page2);
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
@@ -138,124 +162,137 @@ void cpu_gen_init(void)
     tcg_context_init(&tcg_ctx); 
 }
 
-/* return non zero if the very first instruction is invalid so that
-   the virtual CPU can trigger an exception.
+/* Encode VAL as a signed leb128 sequence at P.
+   Return P incremented past the encoded value.  */
+static uint8_t *encode_sleb128(uint8_t *p, target_long val)
+{
+    int more, byte;
+
+    do {
+        byte = val & 0x7f;
+        val >>= 7;
+        more = !((val == 0 && (byte & 0x40) == 0)
+                 || (val == -1 && (byte & 0x40) != 0));
+        if (more) {
+            byte |= 0x80;
+        }
+        *p++ = byte;
+    } while (more);
 
-   '*gen_code_size_ptr' contains the size of the generated code (host
-   code).
-*/
-int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
+    return p;
+}
+
+/* Decode a signed leb128 sequence at *PP; increment *PP past the
+   decoded value.  Return the decoded value.  */
+static target_long decode_sleb128(uint8_t **pp)
 {
-    TCGContext *s = &tcg_ctx;
-    tcg_insn_unit *gen_code_buf;
-    int gen_code_size;
-#ifdef CONFIG_PROFILER
-    int64_t ti;
-#endif
+    uint8_t *p = *pp;
+    target_long val = 0;
+    int byte, shift = 0;
 
-#ifdef CONFIG_PROFILER
-    s->tb_count1++; /* includes aborted translations because of
-                       exceptions */
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    do {
+        byte = *p++;
+        val |= (target_ulong)(byte & 0x7f) << shift;
+        shift += 7;
+    } while (byte & 0x80);
+    if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
+        val |= -(target_ulong)1 << shift;
+    }
 
-    gen_intermediate_code(env, tb);
+    *pp = p;
+    return val;
+}
 
-    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+/* Encode the data collected about the instructions while compiling TB.
+   Place the data at BLOCK, and return the number of bytes consumed.
 
-    /* generate machine code */
-    gen_code_buf = tb->tc_ptr;
-    tb->tb_next_offset[0] = 0xffff;
-    tb->tb_next_offset[1] = 0xffff;
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
+   The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
+   which come from the target's insn_start data, followed by a uintptr_t
+   which comes from the host pc of the end of the code implementing the insn.
 
-#ifdef CONFIG_PROFILER
-    s->tb_count++;
-    s->interm_time += profile_getclock() - ti;
-    s->code_time -= profile_getclock();
-#endif
-    gen_code_size = tcg_gen_code(s, gen_code_buf);
-    *gen_code_size_ptr = gen_code_size;
-#ifdef CONFIG_PROFILER
-    s->code_time += profile_getclock();
-    s->code_in_len += tb->size;
-    s->code_out_len += gen_code_size;
-#endif
+   Each line of the table is encoded as sleb128 deltas from the previous
+   line.  The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
+   That is, the first column is seeded with the guest pc, the last column
+   with the host pc, and the middle columns with zeros.  */
 
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        qemu_log("OUT: [size=%d]\n", gen_code_size);
-        log_disas(tb->tc_ptr, gen_code_size);
-        qemu_log("\n");
-        qemu_log_flush();
+static int encode_search(TranslationBlock *tb, uint8_t *block)
+{
+    uint8_t *highwater = tcg_ctx.code_gen_highwater;
+    uint8_t *p = block;
+    int i, j, n;
+
+    tb->tc_search = block;
+
+    for (i = 0, n = tb->icount; i < n; ++i) {
+        target_ulong prev;
+
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            if (i == 0) {
+                prev = (j == 0 ? tb->pc : 0);
+            } else {
+                prev = tcg_ctx.gen_insn_data[i - 1][j];
+            }
+            p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
+        }
+        prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
+        p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one row beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           encoding a row without having to check during encoding.  */
+        if (unlikely(p > highwater)) {
+            return -1;
+        }
     }
-#endif
-    return 0;
+
+    return p - block;
 }
 
-/* The cpu state corresponding to 'searched_pc' is restored.
- */
+/* The cpu state corresponding to 'searched_pc' is restored.  */
 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                                      uintptr_t searched_pc)
 {
+    target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
+    uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
     CPUArchState *env = cpu->env_ptr;
-    TCGContext *s = &tcg_ctx;
-    int j;
-    uintptr_t tc_ptr;
+    uint8_t *p = tb->tc_search;
+    int i, j, num_insns = tb->icount;
 #ifdef CONFIG_PROFILER
-    int64_t ti;
+    int64_t ti = profile_getclock();
 #endif
 
-#ifdef CONFIG_PROFILER
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    if (searched_pc < host_pc) {
+        return -1;
+    }
 
-    gen_intermediate_code_pc(env, tb);
+    /* Reconstruct the stored insn data while looking for the point at
+       which the end of the insn exceeds the searched_pc.  */
+    for (i = 0; i < num_insns; ++i) {
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            data[j] += decode_sleb128(&p);
+        }
+        host_pc += decode_sleb128(&p);
+        if (host_pc > searched_pc) {
+            goto found;
+        }
+    }
+    return -1;
 
+ found:
     if (tb->cflags & CF_USE_ICOUNT) {
+        assert(use_icount);
         /* Reset the cycle counter to the start of the block.  */
-        cpu->icount_decr.u16.low += tb->icount;
+        cpu->icount_decr.u16.low += num_insns;
         /* Clear the IO flag.  */
         cpu->can_do_io = 0;
     }
-
-    /* find opc index corresponding to search_pc */
-    tc_ptr = (uintptr_t)tb->tc_ptr;
-    if (searched_pc < tc_ptr)
-        return -1;
-
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
-    j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
-                               searched_pc - tc_ptr);
-    if (j < 0)
-        return -1;
-    /* now find start of instruction before */
-    while (s->gen_opc_instr_start[j] == 0) {
-        j--;
-    }
-    cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
-
-    restore_state_to_opc(env, tb, j);
+    cpu->icount_decr.u16.low -= i;
+    restore_state_to_opc(env, tb, data);
 
 #ifdef CONFIG_PROFILER
-    s->restore_time += profile_getclock() - ti;
-    s->restore_count++;
+    tcg_ctx.restore_time += profile_getclock() - ti;
+    tcg_ctx.restore_count++;
 #endif
     return 0;
 }
@@ -278,44 +315,19 @@ bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
     return false;
 }
 
-#ifdef _WIN32
-static __attribute__((unused)) void map_exec(void *addr, long size)
-{
-    DWORD old_protect;
-    VirtualProtect(addr, size,
-                   PAGE_EXECUTE_READWRITE, &old_protect);
-}
-#else
-static __attribute__((unused)) void map_exec(void *addr, long size)
-{
-    unsigned long start, end, page_size;
-
-    page_size = getpagesize();
-    start = (unsigned long)addr;
-    start &= ~(page_size - 1);
-
-    end = (unsigned long)addr + size;
-    end += page_size - 1;
-    end &= ~(page_size - 1);
-
-    mprotect((void *)start, end - start,
-             PROT_READ | PROT_WRITE | PROT_EXEC);
-}
-#endif
-
 void page_size_init(void)
 {
     /* NOTE: we can always suppose that qemu_host_page_size >=
        TARGET_PAGE_SIZE */
     qemu_real_host_page_size = getpagesize();
-    qemu_real_host_page_mask = ~(qemu_real_host_page_size - 1);
+    qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
     if (qemu_host_page_size == 0) {
         qemu_host_page_size = qemu_real_host_page_size;
     }
     if (qemu_host_page_size < TARGET_PAGE_SIZE) {
         qemu_host_page_size = TARGET_PAGE_SIZE;
     }
-    qemu_host_page_mask = ~(qemu_host_page_size - 1);
+    qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
 }
 
 static void page_init(void)
@@ -387,6 +399,9 @@ static void page_init(void)
 #endif
 }
 
+/* If alloc=1:
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
 {
     PageDesc *pd;
@@ -398,26 +413,26 @@ static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
 
     /* Level 2..N-1.  */
     for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
+        void **p = atomic_rcu_read(lp);
 
         if (p == NULL) {
             if (!alloc) {
                 return NULL;
             }
             p = g_new0(void *, V_L2_SIZE);
-            *lp = p;
+            atomic_rcu_set(lp, p);
         }
 
         lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
     }
 
-    pd = *lp;
+    pd = atomic_rcu_read(lp);
     if (pd == NULL) {
         if (!alloc) {
             return NULL;
         }
         pd = g_new0(PageDesc, V_L2_SIZE);
-        *lp = pd;
+        atomic_rcu_set(lp, pd);
     }
 
     return pd + (index & (V_L2_SIZE - 1));
@@ -428,11 +443,6 @@ static inline PageDesc *page_find(tb_page_addr_t index)
     return page_find_alloc(index, 0);
 }
 
-#if !defined(CONFIG_USER_ONLY)
-#define mmap_lock() do { } while (0)
-#define mmap_unlock() do { } while (0)
-#endif
-
 #if defined(CONFIG_USER_ONLY)
 /* Currently it is not recommended to allocate big chunks of data in
    user mode. It will change when a dedicated libc will be used.  */
@@ -441,14 +451,6 @@ static inline PageDesc *page_find(tb_page_addr_t index)
 #define USE_STATIC_CODE_GEN_BUFFER
 #endif
 
-/* ??? Should configure for this, not list operating systems here.  */
-#if (defined(__linux__) \
-    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__))
-# define USE_MMAP
-#endif
-
 /* Minimum size of the code gen buffer.  This number is randomly chosen,
    but not so small that we can't have a fair number of TB's live.  */
 #define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
@@ -460,6 +462,8 @@ static inline PageDesc *page_find(tb_page_addr_t index)
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__sparc__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__powerpc64__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__aarch64__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (128ul * 1024 * 1024)
 #elif defined(__arm__)
@@ -536,22 +540,102 @@ static inline void *split_cross_256mb(void *buf1, size_t size1)
 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
     __attribute__((aligned(CODE_GEN_ALIGN)));
 
+# ifdef _WIN32
+static inline void do_protect(void *addr, long size, int prot)
+{
+    DWORD old_protect;
+    VirtualProtect(addr, size, prot, &old_protect);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_EXECUTE_READWRITE);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_NOACCESS);
+}
+# else
+static inline void do_protect(void *addr, long size, int prot)
+{
+    uintptr_t start, end;
+
+    start = (uintptr_t)addr;
+    start &= qemu_real_host_page_mask;
+
+    end = (uintptr_t)addr + size;
+    end = ROUND_UP(end, qemu_real_host_page_size);
+
+    mprotect((void *)start, end - start, prot);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PROT_NONE);
+}
+# endif /* WIN32 */
+
 static inline void *alloc_code_gen_buffer(void)
 {
     void *buf = static_code_gen_buffer;
+    size_t full_size, size;
+
+    /* The size of the buffer, rounded down to end on a page boundary.  */
+    full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
+                 & qemu_real_host_page_mask) - (uintptr_t)buf;
+
+    /* Reserve a guard page.  */
+    size = full_size - qemu_real_host_page_size;
+
+    /* Honor a command-line option limiting the size of the buffer.  */
+    if (size > tcg_ctx.code_gen_buffer_size) {
+        size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
+                & qemu_real_host_page_mask) - (uintptr_t)buf;
+    }
+    tcg_ctx.code_gen_buffer_size = size;
+
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
+    if (cross_256mb(buf, size)) {
+        buf = split_cross_256mb(buf, size);
+        size = tcg_ctx.code_gen_buffer_size;
     }
 #endif
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
+
+    map_exec(buf, size);
+    map_none(buf + size, qemu_real_host_page_size);
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
+
     return buf;
 }
-#elif defined(USE_MMAP)
+#elif defined(_WIN32)
+static inline void *alloc_code_gen_buffer(void)
+{
+    size_t size = tcg_ctx.code_gen_buffer_size;
+    void *buf1, *buf2;
+
+    /* Perform the allocation in two steps, so that the guard page
+       is reserved but uncommitted.  */
+    buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
+                        MEM_RESERVE, PAGE_NOACCESS);
+    if (buf1 != NULL) {
+        buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+        assert(buf1 == buf2);
+    }
+
+    return buf1;
+}
+#else
 static inline void *alloc_code_gen_buffer(void)
 {
     int flags = MAP_PRIVATE | MAP_ANONYMOUS;
     uintptr_t start = 0;
+    size_t size = tcg_ctx.code_gen_buffer_size;
     void *buf;
 
     /* Constrain the position of the buffer based on the host cpu.
@@ -567,86 +651,70 @@ static inline void *alloc_code_gen_buffer(void)
        Leave the choice of exact location with the kernel.  */
     flags |= MAP_32BIT;
     /* Cannot expect to map more than 800MB in low memory.  */
-    if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
-        tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
+    if (size > 800u * 1024 * 1024) {
+        tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
     }
 # elif defined(__sparc__)
     start = 0x40000000ul;
 # elif defined(__s390x__)
     start = 0x90000000ul;
 # elif defined(__mips__)
-    /* ??? We ought to more explicitly manage layout for softmmu too.  */
-#  ifdef CONFIG_USER_ONLY
-    start = 0x68000000ul;
-#  elif _MIPS_SIM == _ABI64
+#  if _MIPS_SIM == _ABI64
     start = 0x128000000ul;
 #  else
     start = 0x08000000ul;
 #  endif
 # endif
 
-    buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
-               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    buf = mmap((void *)start, size + qemu_real_host_page_size,
+               PROT_NONE, flags, -1, 0);
     if (buf == MAP_FAILED) {
         return NULL;
     }
 
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+    if (cross_256mb(buf, size)) {
         /* Try again, with the original still mapped, to avoid re-acquiring
            that 256mb crossing.  This time don't specify an address.  */
-        size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
-        void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
-                          flags, -1, 0);
-        if (buf2 != MAP_FAILED) {
-            if (!cross_256mb(buf2, size1)) {
+        size_t size2;
+        void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
+                          PROT_NONE, flags, -1, 0);
+        switch (buf2 != MAP_FAILED) {
+        case 1:
+            if (!cross_256mb(buf2, size)) {
                 /* Success!  Use the new buffer.  */
-                munmap(buf, size1);
-                return buf2;
+                munmap(buf, size);
+                break;
             }
             /* Failure.  Work with what we had.  */
-            munmap(buf2, size1);
+            munmap(buf2, size);
+            /* fallthru */
+        default:
+            /* Split the original buffer.  Free the smaller half.  */
+            buf2 = split_cross_256mb(buf, size);
+            size2 = tcg_ctx.code_gen_buffer_size;
+            if (buf == buf2) {
+                munmap(buf + size2 + qemu_real_host_page_size, size - size2);
+            } else {
+                munmap(buf, size - size2);
+            }
+            size = size2;
+            break;
         }
-
-        /* Split the original buffer.  Free the smaller half.  */
-        buf2 = split_cross_256mb(buf, size1);
-        size2 = tcg_ctx.code_gen_buffer_size;
-        munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
-        return buf2;
+        buf = buf2;
     }
 #endif
 
-    return buf;
-}
-#else
-static inline void *alloc_code_gen_buffer(void)
-{
-    void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
-
-    if (buf == NULL) {
-        return NULL;
-    }
+    /* Make the final buffer accessible.  The guard page at the end
+       will remain inaccessible with PROT_NONE.  */
+    mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
 
-#ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
-        if (buf2 != NULL && !cross_256mb(buf2, size1)) {
-            /* Success!  Use the new buffer.  */
-            free(buf);
-            buf = buf2;
-        } else {
-            /* Failure.  Work with what we had.  Since this is malloc
-               and not mmap, we can't free the other half.  */
-            free(buf2);
-            buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
-        }
-    }
-#endif
+    /* Request large pages for the buffer.  */
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
 
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
     return buf;
 }
-#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+#endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
 
 static inline void code_gen_alloc(size_t tb_size)
 {
@@ -657,24 +725,14 @@ static inline void code_gen_alloc(size_t tb_size)
         exit(1);
     }
 
-    qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
-            QEMU_MADV_HUGEPAGE);
+    /* Estimate a good size for the number of TBs we can support.  We
+       still haven't deducted the prologue from the buffer size here,
+       but that's minimal and won't affect the estimate much.  */
+    tcg_ctx.code_gen_max_blocks
+        = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
+    tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
 
-    /* Steal room for the prologue at the end of the buffer.  This ensures
-       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
-       from TB's to the prologue are going to be in range.  It also means
-       that we don't need to mark (additional) portions of the data segment
-       as executable.  */
-    tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
-            tcg_ctx.code_gen_buffer_size - 1024;
-    tcg_ctx.code_gen_buffer_size -= 1024;
-
-    tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
-        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
-    tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
-            CODE_GEN_AVG_BLOCK_SIZE;
-    tcg_ctx.tb_ctx.tbs =
-            g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
+    qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
 }
 
 /* Must be called before using the QEMU cpus. 'tb_size' is the size
@@ -683,11 +741,9 @@ static inline void code_gen_alloc(size_t tb_size)
 void tcg_exec_init(unsigned long tb_size)
 {
     cpu_gen_init();
-    code_gen_alloc(tb_size);
-    tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
-    tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
     page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    code_gen_alloc(tb_size);
+#if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and
        initialize the prologue now.  */
     tcg_prologue_init(&tcg_ctx);
@@ -705,9 +761,7 @@ static TranslationBlock *tb_alloc(target_ulong pc)
 {
     TranslationBlock *tb;
 
-    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
-        (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
-         tcg_ctx.code_gen_buffer_max_size) {
+    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
         return NULL;
     }
     tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
@@ -730,10 +784,8 @@ void tb_free(TranslationBlock *tb)
 
 static inline void invalidate_page_bitmap(PageDesc *p)
 {
-    if (p->code_bitmap) {
-        g_free(p->code_bitmap);
-        p->code_bitmap = NULL;
-    }
+    g_free(p->code_bitmap);
+    p->code_bitmap = NULL;
     p->code_write_count = 0;
 }
 
@@ -809,7 +861,8 @@ static void tb_invalidate_check(target_ulong address)
 
     address &= TARGET_PAGE_MASK;
     for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
+        for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
+             tb = tb->phys_hash_next) {
             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
                   address >= tb->pc + tb->size)) {
                 printf("ERROR invalidate: address=" TARGET_FMT_lx
@@ -995,6 +1048,7 @@ static void build_page_bitmap(PageDesc *p)
     }
 }
 
+/* Called with mmap_lock held for user mode emulation.  */
 TranslationBlock *tb_gen_code(CPUState *cpu,
                               target_ulong pc, target_ulong cs_base,
                               int flags, int cflags)
@@ -1003,28 +1057,99 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
     TranslationBlock *tb;
     tb_page_addr_t phys_pc, phys_page2;
     target_ulong virt_page2;
-    int code_gen_size;
+    tcg_insn_unit *gen_code_buf;
+    int gen_code_size, search_size;
+#ifdef CONFIG_PROFILER
+    int64_t ti;
+#endif
 
     phys_pc = get_page_addr_code(env, pc);
-    if (use_icount) {
+    if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
         cflags |= CF_USE_ICOUNT;
     }
+
     tb = tb_alloc(pc);
-    if (!tb) {
+    if (unlikely(!tb)) {
+ buffer_overflow:
         /* flush must be done */
         tb_flush(cpu);
         /* cannot fail at this point */
         tb = tb_alloc(pc);
+        assert(tb != NULL);
         /* Don't forget to invalidate previous TB info.  */
         tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
     }
-    tb->tc_ptr = tcg_ctx.code_gen_ptr;
+
+    gen_code_buf = tcg_ctx.code_gen_ptr;
+    tb->tc_ptr = gen_code_buf;
     tb->cs_base = cs_base;
     tb->flags = flags;
     tb->cflags = cflags;
-    cpu_gen_code(env, tb, &code_gen_size);
-    tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
-            code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count1++; /* includes aborted translations because of
+                       exceptions */
+    ti = profile_getclock();
+#endif
+
+    tcg_func_start(&tcg_ctx);
+
+    gen_intermediate_code(env, tb);
+
+    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+
+    /* generate machine code */
+    tb->tb_next_offset[0] = 0xffff;
+    tb->tb_next_offset[1] = 0xffff;
+    tcg_ctx.tb_next_offset = tb->tb_next_offset;
+#ifdef USE_DIRECT_JUMP
+    tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset;
+    tcg_ctx.tb_next = NULL;
+#else
+    tcg_ctx.tb_jmp_offset = NULL;
+    tcg_ctx.tb_next = tb->tb_next;
+#endif
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count++;
+    tcg_ctx.interm_time += profile_getclock() - ti;
+    tcg_ctx.code_time -= profile_getclock();
+#endif
+
+    /* ??? Overflow could be handled better here.  In particular, we
+       don't need to re-do gen_intermediate_code, nor should we re-do
+       the tcg optimization currently hidden inside tcg_gen_code.  All
+       that should be required is to flush the TBs, allocate a new TB,
+       re-initialize it per above, and re-do the actual code generation.  */
+    gen_code_size = tcg_gen_code(&tcg_ctx, tb);
+    if (unlikely(gen_code_size < 0)) {
+        goto buffer_overflow;
+    }
+    search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
+    if (unlikely(search_size < 0)) {
+        goto buffer_overflow;
+    }
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.code_time += profile_getclock();
+    tcg_ctx.code_in_len += tb->size;
+    tcg_ctx.code_out_len += gen_code_size;
+    tcg_ctx.search_out_len += search_size;
+#endif
+
+#ifdef DEBUG_DISAS
+    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
+        qemu_log_in_addr_range(tb->pc)) {
+        qemu_log("OUT: [size=%d]\n", gen_code_size);
+        log_disas(tb->tc_ptr, gen_code_size);
+        qemu_log("\n");
+        qemu_log_flush();
+    }
+#endif
+
+    tcg_ctx.code_gen_ptr = (void *)
+        ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
+                 CODE_GEN_ALIGN);
 
     /* check next page if needed */
     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
@@ -1042,6 +1167,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 {
@@ -1058,6 +1185,8 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
@@ -1206,6 +1335,7 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 }
 
 #if !defined(CONFIG_SOFTMMU)
+/* Called with mmap_lock held.  */
 static void tb_invalidate_phys_page(tb_page_addr_t addr,
                                     uintptr_t pc, void *puc,
                                     bool locked)
@@ -1275,7 +1405,10 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
 }
 #endif
 
-/* add the tb in the target page and protect it if necessary */
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static inline void tb_alloc_page(TranslationBlock *tb,
                                  unsigned int n, tb_page_addr_t page_addr)
 {
@@ -1331,16 +1464,16 @@ static inline void tb_alloc_page(TranslationBlock *tb,
 }
 
 /* add a new TB and link it to the physical page tables. phys_page2 is
-   (-1) to indicate that only one page contains the TB. */
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
                          tb_page_addr_t phys_page2)
 {
     unsigned int h;
     TranslationBlock **ptb;
 
-    /* Grab the mmap lock to stop another thread invalidating this TB
-       before we are done.  */
-    mmap_lock();
     /* add in the physical hash table */
     h = tb_phys_hash_func(phys_pc);
     ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
@@ -1370,7 +1503,6 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
 #ifdef DEBUG_TB_CHECK
     tb_page_check();
 #endif
-    mmap_unlock();
 }
 
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
@@ -1451,36 +1583,6 @@ void tb_check_watchpoint(CPUState *cpu)
 }
 
 #ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUState *cpu, int mask)
-{
-    int old_mask;
-
-    old_mask = cpu->interrupt_request;
-    cpu->interrupt_request |= mask;
-
-    /*
-     * If called from iothread context, wake the target cpu in
-     * case its halted.
-     */
-    if (!qemu_cpu_is_self(cpu)) {
-        qemu_cpu_kick(cpu);
-        return;
-    }
-
-    if (use_icount) {
-        cpu->icount_decr.u16.high = 0xffff;
-        if (!cpu_can_do_io(cpu)
-            && (mask & ~old_mask) != 0) {
-            cpu_abort(cpu, "Raised interrupt while not in I/O function");
-        }
-    } else {
-        cpu->tcg_exit_req = 1;
-    }
-}
-
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
 /* in deterministic execution mode, instructions doing device I/Os
    must be at the end of the TB */
 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
@@ -1533,6 +1635,14 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
     cs_base = tb->cs_base;
     flags = tb->flags;
     tb_phys_invalidate(tb, -1);
+    if (tb->cflags & CF_NOCACHE) {
+        if (tb->orig_tb) {
+            /* Invalidate original TB if this TB was generated in
+             * cpu_exec_nocache() */
+            tb_phys_invalidate(tb->orig_tb, -1);
+        }
+        tb_free(tb);
+    }
     /* FIXME: In theory this could raise an exception.  In practice
        we have already translated the block once so it's probably ok.  */
     tb_gen_code(cpu, pc, cs_base, flags, cflags);
@@ -1590,7 +1700,7 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
     cpu_fprintf(f, "Translation buffer state:\n");
     cpu_fprintf(f, "gen code size       %td/%zd\n",
                 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
-                tcg_ctx.code_gen_buffer_max_size);
+                tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
     cpu_fprintf(f, "TB count            %d/%d\n",
             tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
diff --git a/qemu/translate-common.c b/qemu/translate-common.c
new file mode 100644
index 000000000..ffbfe856c
--- /dev/null
+++ b/qemu/translate-common.c
@@ -0,0 +1,57 @@
+/*
+ *  Host code generation common components
+ *
+ *  Copyright (c) 2015 Peter Crosthwaite <crosthwaite.peter@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qom/cpu.h"
+
+uintptr_t qemu_real_host_page_size;
+intptr_t qemu_real_host_page_mask;
+
+#ifndef CONFIG_USER_ONLY
+/* mask must never be zero, except for A20 change call */
+static void tcg_handle_interrupt(CPUState *cpu, int mask)
+{
+    int old_mask;
+
+    old_mask = cpu->interrupt_request;
+    cpu->interrupt_request |= mask;
+
+    /*
+     * If called from iothread context, wake the target cpu in
+     * case its halted.
+     */
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
+        return;
+    }
+
+    if (use_icount) {
+        cpu->icount_decr.u16.high = 0xffff;
+        if (!cpu->can_do_io
+            && (mask & ~old_mask) != 0) {
+            cpu_abort(cpu, "Raised interrupt while not in I/O function");
+        }
+    } else {
+        cpu->tcg_exit_req = 1;
+    }
+}
+
+CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
+#endif
diff --git a/qemu/ui/Makefile.objs b/qemu/ui/Makefile.objs
index c62d4d972..dc936f150 100644
--- a/qemu/ui/Makefile.objs
+++ b/qemu/ui/Makefile.objs
@@ -2,13 +2,14 @@ vnc-obj-y += vnc.o
 vnc-obj-y += vnc-enc-zlib.o vnc-enc-hextile.o
 vnc-obj-y += vnc-enc-tight.o vnc-palette.o
 vnc-obj-y += vnc-enc-zrle.o
-vnc-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
+vnc-obj-y += vnc-auth-vencrypt.o
 vnc-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
 vnc-obj-y += vnc-ws.o
 vnc-obj-y += vnc-jobs.o
 
 common-obj-y += keymaps.o console.o cursor.o qemu-pixman.o
 common-obj-y += input.o input-keymap.o input-legacy.o
+common-obj-$(CONFIG_LINUX) += input-linux.o
 common-obj-$(CONFIG_SPICE) += spice-core.o spice-input.o spice-display.o
 common-obj-$(CONFIG_SDL) += sdl.mo x_keymap.o
 common-obj-$(CONFIG_COCOA) += cocoa.o
@@ -31,11 +32,17 @@ ifeq ($(CONFIG_OPENGL),y)
 common-obj-y += shader.o
 common-obj-y += console-gl.o
 common-obj-y += egl-helpers.o
+common-obj-y += egl-context.o
+ifeq ($(CONFIG_GTK_GL),y)
+common-obj-$(CONFIG_GTK) += gtk-gl-area.o
+else
 common-obj-$(CONFIG_GTK) += gtk-egl.o
 endif
+endif
 
 gtk.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS)
 gtk-egl.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
+gtk-gl-area.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
 shader.o-cflags += $(OPENGL_CFLAGS)
 console-gl.o-cflags += $(OPENGL_CFLAGS)
 egl-helpers.o-cflags += $(OPENGL_CFLAGS)
diff --git a/qemu/ui/cocoa.m b/qemu/ui/cocoa.m
index 334e6f666..60a7c07ec 100644
--- a/qemu/ui/cocoa.m
+++ b/qemu/ui/cocoa.m
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+
 #import <Cocoa/Cocoa.h>
 #include <crt_externs.h>
 
@@ -31,6 +33,7 @@
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
 #include "sysemu/blockdev.h"
+#include <Carbon/Carbon.h>
 
 #ifndef MAC_OS_X_VERSION_10_5
 #define MAC_OS_X_VERSION_10_5 1050
@@ -70,178 +73,139 @@ bool stretch_video;
 NSTextField *pauseLabel;
 NSArray * supportedImageFileTypes;
 
-// keymap conversion
-int keymap[] =
-{
-//  SdlI    macI    macH    SdlH    104xtH  104xtC  sdl
-    30, //  0       0x00    0x1e            A       QZ_a
-    31, //  1       0x01    0x1f            S       QZ_s
-    32, //  2       0x02    0x20            D       QZ_d
-    33, //  3       0x03    0x21            F       QZ_f
-    35, //  4       0x04    0x23            H       QZ_h
-    34, //  5       0x05    0x22            G       QZ_g
-    44, //  6       0x06    0x2c            Z       QZ_z
-    45, //  7       0x07    0x2d            X       QZ_x
-    46, //  8       0x08    0x2e            C       QZ_c
-    47, //  9       0x09    0x2f            V       QZ_v
-    0,  //  10      0x0A    Undefined
-    48, //  11      0x0B    0x30            B       QZ_b
-    16, //  12      0x0C    0x10            Q       QZ_q
-    17, //  13      0x0D    0x11            W       QZ_w
-    18, //  14      0x0E    0x12            E       QZ_e
-    19, //  15      0x0F    0x13            R       QZ_r
-    21, //  16      0x10    0x15            Y       QZ_y
-    20, //  17      0x11    0x14            T       QZ_t
-    2,  //  18      0x12    0x02            1       QZ_1
-    3,  //  19      0x13    0x03            2       QZ_2
-    4,  //  20      0x14    0x04            3       QZ_3
-    5,  //  21      0x15    0x05            4       QZ_4
-    7,  //  22      0x16    0x07            6       QZ_6
-    6,  //  23      0x17    0x06            5       QZ_5
-    13, //  24      0x18    0x0d            =       QZ_EQUALS
-    10, //  25      0x19    0x0a            9       QZ_9
-    8,  //  26      0x1A    0x08            7       QZ_7
-    12, //  27      0x1B    0x0c            -       QZ_MINUS
-    9,  //  28      0x1C    0x09            8       QZ_8
-    11, //  29      0x1D    0x0b            0       QZ_0
-    27, //  30      0x1E    0x1b            ]       QZ_RIGHTBRACKET
-    24, //  31      0x1F    0x18            O       QZ_o
-    22, //  32      0x20    0x16            U       QZ_u
-    26, //  33      0x21    0x1a            [       QZ_LEFTBRACKET
-    23, //  34      0x22    0x17            I       QZ_i
-    25, //  35      0x23    0x19            P       QZ_p
-    28, //  36      0x24    0x1c            ENTER   QZ_RETURN
-    38, //  37      0x25    0x26            L       QZ_l
-    36, //  38      0x26    0x24            J       QZ_j
-    40, //  39      0x27    0x28            '       QZ_QUOTE
-    37, //  40      0x28    0x25            K       QZ_k
-    39, //  41      0x29    0x27            ;       QZ_SEMICOLON
-    43, //  42      0x2A    0x2b            \       QZ_BACKSLASH
-    51, //  43      0x2B    0x33            ,       QZ_COMMA
-    53, //  44      0x2C    0x35            /       QZ_SLASH
-    49, //  45      0x2D    0x31            N       QZ_n
-    50, //  46      0x2E    0x32            M       QZ_m
-    52, //  47      0x2F    0x34            .       QZ_PERIOD
-    15, //  48      0x30    0x0f            TAB     QZ_TAB
-    57, //  49      0x31    0x39            SPACE   QZ_SPACE
-    41, //  50      0x32    0x29            `       QZ_BACKQUOTE
-    14, //  51      0x33    0x0e            BKSP    QZ_BACKSPACE
-    0,  //  52      0x34    Undefined
-    1,  //  53      0x35    0x01            ESC     QZ_ESCAPE
-    220, // 54      0x36    0xdc    E0,5C   R GUI   QZ_RMETA
-    219, // 55      0x37    0xdb    E0,5B   L GUI   QZ_LMETA
-    42, //  56      0x38    0x2a            L SHFT  QZ_LSHIFT
-    58, //  57      0x39    0x3a            CAPS    QZ_CAPSLOCK
-    56, //  58      0x3A    0x38            L ALT   QZ_LALT
-    29, //  59      0x3B    0x1d            L CTRL  QZ_LCTRL
-    54, //  60      0x3C    0x36            R SHFT  QZ_RSHIFT
-    184,//  61      0x3D    0xb8    E0,38   R ALT   QZ_RALT
-    157,//  62      0x3E    0x9d    E0,1D   R CTRL  QZ_RCTRL
-    0,  //  63      0x3F    Undefined
-    0,  //  64      0x40    Undefined
-    0,  //  65      0x41    Undefined
-    0,  //  66      0x42    Undefined
-    55, //  67      0x43    0x37            KP *    QZ_KP_MULTIPLY
-    0,  //  68      0x44    Undefined
-    78, //  69      0x45    0x4e            KP +    QZ_KP_PLUS
-    0,  //  70      0x46    Undefined
-    69, //  71      0x47    0x45            NUM     QZ_NUMLOCK
-    0,  //  72      0x48    Undefined
-    0,  //  73      0x49    Undefined
-    0,  //  74      0x4A    Undefined
-    181,//  75      0x4B    0xb5    E0,35   KP /    QZ_KP_DIVIDE
-    152,//  76      0x4C    0x9c    E0,1C   KP EN   QZ_KP_ENTER
-    0,  //  77      0x4D    undefined
-    74, //  78      0x4E    0x4a            KP -    QZ_KP_MINUS
-    0,  //  79      0x4F    Undefined
-    0,  //  80      0x50    Undefined
-    0,  //  81      0x51                            QZ_KP_EQUALS
-    82, //  82      0x52    0x52            KP 0    QZ_KP0
-    79, //  83      0x53    0x4f            KP 1    QZ_KP1
-    80, //  84      0x54    0x50            KP 2    QZ_KP2
-    81, //  85      0x55    0x51            KP 3    QZ_KP3
-    75, //  86      0x56    0x4b            KP 4    QZ_KP4
-    76, //  87      0x57    0x4c            KP 5    QZ_KP5
-    77, //  88      0x58    0x4d            KP 6    QZ_KP6
-    71, //  89      0x59    0x47            KP 7    QZ_KP7
-    0,  //  90      0x5A    Undefined
-    72, //  91      0x5B    0x48            KP 8    QZ_KP8
-    73, //  92      0x5C    0x49            KP 9    QZ_KP9
-    0,  //  93      0x5D    Undefined
-    0,  //  94      0x5E    Undefined
-    0,  //  95      0x5F    Undefined
-    63, //  96      0x60    0x3f            F5      QZ_F5
-    64, //  97      0x61    0x40            F6      QZ_F6
-    65, //  98      0x62    0x41            F7      QZ_F7
-    61, //  99      0x63    0x3d            F3      QZ_F3
-    66, //  100     0x64    0x42            F8      QZ_F8
-    67, //  101     0x65    0x43            F9      QZ_F9
-    0,  //  102     0x66    Undefined
-    87, //  103     0x67    0x57            F11     QZ_F11
-    0,  //  104     0x68    Undefined
-    183,//  105     0x69    0xb7                    QZ_PRINT
-    0,  //  106     0x6A    Undefined
-    70, //  107     0x6B    0x46            SCROLL  QZ_SCROLLOCK
-    0,  //  108     0x6C    Undefined
-    68, //  109     0x6D    0x44            F10     QZ_F10
-    0,  //  110     0x6E    Undefined
-    88, //  111     0x6F    0x58            F12     QZ_F12
-    0,  //  112     0x70    Undefined
-    110,//  113     0x71    0x0                     QZ_PAUSE
-    210,//  114     0x72    0xd2    E0,52   INSERT  QZ_INSERT
-    199,//  115     0x73    0xc7    E0,47   HOME    QZ_HOME
-    201,//  116     0x74    0xc9    E0,49   PG UP   QZ_PAGEUP
-    211,//  117     0x75    0xd3    E0,53   DELETE  QZ_DELETE
-    62, //  118     0x76    0x3e            F4      QZ_F4
-    207,//  119     0x77    0xcf    E0,4f   END     QZ_END
-    60, //  120     0x78    0x3c            F2      QZ_F2
-    209,//  121     0x79    0xd1    E0,51   PG DN   QZ_PAGEDOWN
-    59, //  122     0x7A    0x3b            F1      QZ_F1
-    203,//  123     0x7B    0xcb    e0,4B   L ARROW QZ_LEFT
-    205,//  124     0x7C    0xcd    e0,4D   R ARROW QZ_RIGHT
-    208,//  125     0x7D    0xd0    E0,50   D ARROW QZ_DOWN
-    200,//  126     0x7E    0xc8    E0,48   U ARROW QZ_UP
-/* completed according to http://www.libsdl.org/cgi/cvsweb.cgi/SDL12/src/video/quartz/SDL_QuartzKeys.h?rev=1.6&content-type=text/x-cvsweb-markup */
-
-/* Additional 104 Key XP-Keyboard Scancodes from http://www.computer-engineering.org/ps2keyboard/scancodes1.html */
-/*
-    221 //          0xdd            e0,5d   APPS
-        //              E0,2A,E0,37         PRNT SCRN
-        //              E1,1D,45,E1,9D,C5   PAUSE
-    83  //          0x53    0x53            KP .
-// ACPI Scan Codes
-    222 //          0xde            E0, 5E  Power
-    223 //          0xdf            E0, 5F  Sleep
-    227 //          0xe3            E0, 63  Wake
-// Windows Multimedia Scan Codes
-    153 //          0x99            E0, 19  Next Track
-    144 //          0x90            E0, 10  Previous Track
-    164 //          0xa4            E0, 24  Stop
-    162 //          0xa2            E0, 22  Play/Pause
-    160 //          0xa0            E0, 20  Mute
-    176 //          0xb0            E0, 30  Volume Up
-    174 //          0xae            E0, 2E  Volume Down
-    237 //          0xed            E0, 6D  Media Select
-    236 //          0xec            E0, 6C  E-Mail
-    161 //          0xa1            E0, 21  Calculator
-    235 //          0xeb            E0, 6B  My Computer
-    229 //          0xe5            E0, 65  WWW Search
-    178 //          0xb2            E0, 32  WWW Home
-    234 //          0xea            E0, 6A  WWW Back
-    233 //          0xe9            E0, 69  WWW Forward
-    232 //          0xe8            E0, 68  WWW Stop
-    231 //          0xe7            E0, 67  WWW Refresh
-    230 //          0xe6            E0, 66  WWW Favorites
-*/
+// Mac to QKeyCode conversion
+const int mac_to_qkeycode_map[] = {
+    [kVK_ANSI_A] = Q_KEY_CODE_A,
+    [kVK_ANSI_B] = Q_KEY_CODE_B,
+    [kVK_ANSI_C] = Q_KEY_CODE_C,
+    [kVK_ANSI_D] = Q_KEY_CODE_D,
+    [kVK_ANSI_E] = Q_KEY_CODE_E,
+    [kVK_ANSI_F] = Q_KEY_CODE_F,
+    [kVK_ANSI_G] = Q_KEY_CODE_G,
+    [kVK_ANSI_H] = Q_KEY_CODE_H,
+    [kVK_ANSI_I] = Q_KEY_CODE_I,
+    [kVK_ANSI_J] = Q_KEY_CODE_J,
+    [kVK_ANSI_K] = Q_KEY_CODE_K,
+    [kVK_ANSI_L] = Q_KEY_CODE_L,
+    [kVK_ANSI_M] = Q_KEY_CODE_M,
+    [kVK_ANSI_N] = Q_KEY_CODE_N,
+    [kVK_ANSI_O] = Q_KEY_CODE_O,
+    [kVK_ANSI_P] = Q_KEY_CODE_P,
+    [kVK_ANSI_Q] = Q_KEY_CODE_Q,
+    [kVK_ANSI_R] = Q_KEY_CODE_R,
+    [kVK_ANSI_S] = Q_KEY_CODE_S,
+    [kVK_ANSI_T] = Q_KEY_CODE_T,
+    [kVK_ANSI_U] = Q_KEY_CODE_U,
+    [kVK_ANSI_V] = Q_KEY_CODE_V,
+    [kVK_ANSI_W] = Q_KEY_CODE_W,
+    [kVK_ANSI_X] = Q_KEY_CODE_X,
+    [kVK_ANSI_Y] = Q_KEY_CODE_Y,
+    [kVK_ANSI_Z] = Q_KEY_CODE_Z,
+
+    [kVK_ANSI_0] = Q_KEY_CODE_0,
+    [kVK_ANSI_1] = Q_KEY_CODE_1,
+    [kVK_ANSI_2] = Q_KEY_CODE_2,
+    [kVK_ANSI_3] = Q_KEY_CODE_3,
+    [kVK_ANSI_4] = Q_KEY_CODE_4,
+    [kVK_ANSI_5] = Q_KEY_CODE_5,
+    [kVK_ANSI_6] = Q_KEY_CODE_6,
+    [kVK_ANSI_7] = Q_KEY_CODE_7,
+    [kVK_ANSI_8] = Q_KEY_CODE_8,
+    [kVK_ANSI_9] = Q_KEY_CODE_9,
+
+    [kVK_ANSI_Grave] = Q_KEY_CODE_GRAVE_ACCENT,
+    [kVK_ANSI_Minus] = Q_KEY_CODE_MINUS,
+    [kVK_ANSI_Equal] = Q_KEY_CODE_EQUAL,
+    [kVK_Delete] = Q_KEY_CODE_BACKSPACE,
+    [kVK_CapsLock] = Q_KEY_CODE_CAPS_LOCK,
+    [kVK_Tab] = Q_KEY_CODE_TAB,
+    [kVK_Return] = Q_KEY_CODE_RET,
+    [kVK_ANSI_LeftBracket] = Q_KEY_CODE_BRACKET_LEFT,
+    [kVK_ANSI_RightBracket] = Q_KEY_CODE_BRACKET_RIGHT,
+    [kVK_ANSI_Backslash] = Q_KEY_CODE_BACKSLASH,
+    [kVK_ANSI_Semicolon] = Q_KEY_CODE_SEMICOLON,
+    [kVK_ANSI_Quote] = Q_KEY_CODE_APOSTROPHE,
+    [kVK_ANSI_Comma] = Q_KEY_CODE_COMMA,
+    [kVK_ANSI_Period] = Q_KEY_CODE_DOT,
+    [kVK_ANSI_Slash] = Q_KEY_CODE_SLASH,
+    [kVK_Shift] = Q_KEY_CODE_SHIFT,
+    [kVK_RightShift] = Q_KEY_CODE_SHIFT_R,
+    [kVK_Control] = Q_KEY_CODE_CTRL,
+    [kVK_RightControl] = Q_KEY_CODE_CTRL_R,
+    [kVK_Option] = Q_KEY_CODE_ALT,
+    [kVK_RightOption] = Q_KEY_CODE_ALT_R,
+    [kVK_Command] = Q_KEY_CODE_META_L,
+    [0x36] = Q_KEY_CODE_META_R, /* There is no kVK_RightCommand */
+    [kVK_Space] = Q_KEY_CODE_SPC,
+
+    [kVK_ANSI_Keypad0] = Q_KEY_CODE_KP_0,
+    [kVK_ANSI_Keypad1] = Q_KEY_CODE_KP_1,
+    [kVK_ANSI_Keypad2] = Q_KEY_CODE_KP_2,
+    [kVK_ANSI_Keypad3] = Q_KEY_CODE_KP_3,
+    [kVK_ANSI_Keypad4] = Q_KEY_CODE_KP_4,
+    [kVK_ANSI_Keypad5] = Q_KEY_CODE_KP_5,
+    [kVK_ANSI_Keypad6] = Q_KEY_CODE_KP_6,
+    [kVK_ANSI_Keypad7] = Q_KEY_CODE_KP_7,
+    [kVK_ANSI_Keypad8] = Q_KEY_CODE_KP_8,
+    [kVK_ANSI_Keypad9] = Q_KEY_CODE_KP_9,
+    [kVK_ANSI_KeypadDecimal] = Q_KEY_CODE_KP_DECIMAL,
+    [kVK_ANSI_KeypadEnter] = Q_KEY_CODE_KP_ENTER,
+    [kVK_ANSI_KeypadPlus] = Q_KEY_CODE_KP_ADD,
+    [kVK_ANSI_KeypadMinus] = Q_KEY_CODE_KP_SUBTRACT,
+    [kVK_ANSI_KeypadMultiply] = Q_KEY_CODE_KP_MULTIPLY,
+    [kVK_ANSI_KeypadDivide] = Q_KEY_CODE_KP_DIVIDE,
+    [kVK_ANSI_KeypadEquals] = Q_KEY_CODE_KP_EQUALS,
+    [kVK_ANSI_KeypadClear] = Q_KEY_CODE_NUM_LOCK,
+
+    [kVK_UpArrow] = Q_KEY_CODE_UP,
+    [kVK_DownArrow] = Q_KEY_CODE_DOWN,
+    [kVK_LeftArrow] = Q_KEY_CODE_LEFT,
+    [kVK_RightArrow] = Q_KEY_CODE_RIGHT,
+
+    [kVK_Help] = Q_KEY_CODE_INSERT,
+    [kVK_Home] = Q_KEY_CODE_HOME,
+    [kVK_PageUp] = Q_KEY_CODE_PGUP,
+    [kVK_PageDown] = Q_KEY_CODE_PGDN,
+    [kVK_End] = Q_KEY_CODE_END,
+    [kVK_ForwardDelete] = Q_KEY_CODE_DELETE,
+
+    [kVK_Escape] = Q_KEY_CODE_ESC,
+
+    /* The Power key can't be used directly because the operating system uses
+     * it. This key can be emulated by using it in place of another key such as
+     * F1. Don't forget to disable the real key binding.
+     */
+    /* [kVK_F1] = Q_KEY_CODE_POWER, */
+
+    [kVK_F1] = Q_KEY_CODE_F1,
+    [kVK_F2] = Q_KEY_CODE_F2,
+    [kVK_F3] = Q_KEY_CODE_F3,
+    [kVK_F4] = Q_KEY_CODE_F4,
+    [kVK_F5] = Q_KEY_CODE_F5,
+    [kVK_F6] = Q_KEY_CODE_F6,
+    [kVK_F7] = Q_KEY_CODE_F7,
+    [kVK_F8] = Q_KEY_CODE_F8,
+    [kVK_F9] = Q_KEY_CODE_F9,
+    [kVK_F10] = Q_KEY_CODE_F10,
+    [kVK_F11] = Q_KEY_CODE_F11,
+    [kVK_F12] = Q_KEY_CODE_F12,
+    [kVK_F13] = Q_KEY_CODE_PRINT,
+    [kVK_F14] = Q_KEY_CODE_SCROLL_LOCK,
+    [kVK_F15] = Q_KEY_CODE_PAUSE,
+
+    /*
+     * The eject and volume keys can't be used here because they are handled at
+     * a lower level than what an Application can see.
+     */
 };
 
 static int cocoa_keycode_to_qemu(int keycode)
 {
-    if (ARRAY_SIZE(keymap) <= keycode) {
+    if (ARRAY_SIZE(mac_to_qkeycode_map) <= keycode) {
         fprintf(stderr, "(cocoa) warning unknown keycode 0x%x\n", keycode);
         return 0;
     }
-    return keymap[keycode];
+    return mac_to_qkeycode_map[keycode];
 }
 
 /* Displays an alert dialog box with the specified message */
@@ -304,6 +268,7 @@ static void handleAnyDeviceErrors(Error * err)
 - (float) cdx;
 - (float) cdy;
 - (QEMUScreen) gscreen;
+- (void) raiseAllKeys;
 @end
 
 QemuCocoaView *cocoaView;
@@ -554,21 +519,24 @@ QemuCocoaView *cocoaView;
         case NSFlagsChanged:
             keycode = cocoa_keycode_to_qemu([event keyCode]);
 
-            if ((keycode == 219 || keycode == 220) && !isMouseGrabbed) {
+            if ((keycode == Q_KEY_CODE_META_L || keycode == Q_KEY_CODE_META_R)
+               && !isMouseGrabbed) {
               /* Don't pass command key changes to guest unless mouse is grabbed */
               keycode = 0;
             }
 
             if (keycode) {
-                if (keycode == 58 || keycode == 69) { // emulate caps lock and num lock keydown and keyup
-                    qemu_input_event_send_key_number(dcl->con, keycode, true);
-                    qemu_input_event_send_key_number(dcl->con, keycode, false);
+                // emulate caps lock and num lock keydown and keyup
+                if (keycode == Q_KEY_CODE_CAPS_LOCK ||
+                    keycode == Q_KEY_CODE_NUM_LOCK) {
+                    qemu_input_event_send_key_qcode(dcl->con, keycode, true);
+                    qemu_input_event_send_key_qcode(dcl->con, keycode, false);
                 } else if (qemu_console_is_graphic(NULL)) {
                     if (modifiers_state[keycode] == 0) { // keydown
-                        qemu_input_event_send_key_number(dcl->con, keycode, true);
+                        qemu_input_event_send_key_qcode(dcl->con, keycode, true);
                         modifiers_state[keycode] = 1;
                     } else { // keyup
-                        qemu_input_event_send_key_number(dcl->con, keycode, false);
+                        qemu_input_event_send_key_qcode(dcl->con, keycode, false);
                         modifiers_state[keycode] = 0;
                     }
                 }
@@ -595,14 +563,14 @@ QemuCocoaView *cocoaView;
                 switch (keycode) {
 
                     // enable graphic console
-                    case 0x02 ... 0x0a: // '1' to '9' keys
-                        console_select(keycode - 0x02);
+                    case Q_KEY_CODE_1 ... Q_KEY_CODE_9: // '1' to '9' keys
+                        console_select(keycode - 11);
                         break;
                 }
 
             // handle keys for graphic console
             } else if (qemu_console_is_graphic(NULL)) {
-                qemu_input_event_send_key_number(dcl->con, keycode, true);
+                qemu_input_event_send_key_qcode(dcl->con, keycode, true);
 
             // handlekeys for Monitor
             } else {
@@ -650,7 +618,7 @@ QemuCocoaView *cocoaView;
             }
 
             if (qemu_console_is_graphic(NULL)) {
-                qemu_input_event_send_key_number(dcl->con, keycode, false);
+                qemu_input_event_send_key_qcode(dcl->con, keycode, false);
             }
             break;
         case NSMouseMoved:
@@ -723,8 +691,16 @@ QemuCocoaView *cocoaView;
     }
 
     if (mouse_event) {
-        if (last_buttons != buttons) {
-            static uint32_t bmap[INPUT_BUTTON_MAX] = {
+        /* Don't send button events to the guest unless we've got a
+         * mouse grab or window focus. If we have neither then this event
+         * is the user clicking on the background window to activate and
+         * bring us to the front, which will be done by the sendEvent
+         * call below. We definitely don't want to pass that click through
+         * to the guest.
+         */
+        if ((isMouseGrabbed || [[self window] isKeyWindow]) &&
+            (last_buttons != buttons)) {
+            static uint32_t bmap[INPUT_BUTTON__MAX] = {
                 [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
                 [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
                 [INPUT_BUTTON_RIGHT]      = MOUSE_EVENT_RBUTTON,
@@ -798,6 +774,24 @@ QemuCocoaView *cocoaView;
 - (float) cdx {return cdx;}
 - (float) cdy {return cdy;}
 - (QEMUScreen) gscreen {return screen;}
+
+/*
+ * Makes the target think all down keys are being released.
+ * This prevents a stuck key problem, since we will not see
+ * key up events for those keys after we have lost focus.
+ */
+- (void) raiseAllKeys
+{
+    int index;
+    const int max_index = ARRAY_SIZE(modifiers_state);
+
+   for (index = 0; index < max_index; index++) {
+       if (modifiers_state[index]) {
+           modifiers_state[index] = 0;
+           qemu_input_event_send_key_qcode(dcl->con, index, false);
+       }
+   }
+}
 @end
 
 
@@ -809,12 +803,11 @@ QemuCocoaView *cocoaView;
 */
 @interface QemuCocoaAppController : NSObject
 #if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-                                             <NSApplicationDelegate>
+                                       <NSWindowDelegate, NSApplicationDelegate>
 #endif
 {
 }
 - (void)startEmulationWithArgc:(int)argc argv:(char**)argv;
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(NSInteger)returnCode contextInfo:(void *)contextInfo;
 - (void)doToggleFullScreen:(id)sender;
 - (void)toggleFullScreen:(id)sender;
 - (void)showQEMUDoc:(id)sender;
@@ -829,6 +822,8 @@ QemuCocoaView *cocoaView;
 - (void)powerDownQEMU:(id)sender;
 - (void)ejectDeviceMedia:(id)sender;
 - (void)changeDeviceMedia:(id)sender;
+- (BOOL)verifyQuit;
+- (void)openDocumentation:(NSString *)filename;
 @end
 
 @implementation QemuCocoaAppController
@@ -855,13 +850,14 @@ QemuCocoaView *cocoaView;
             exit(1);
         }
         [normalWindow setAcceptsMouseMovedEvents:YES];
-        [normalWindow setTitle:[NSString stringWithFormat:@"QEMU"]];
+        [normalWindow setTitle:@"QEMU"];
         [normalWindow setContentView:cocoaView];
 #if (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_10)
         [normalWindow useOptimizedDrawing:YES];
 #endif
         [normalWindow makeKeyAndOrderFront:self];
         [normalWindow center];
+        [normalWindow setDelegate: self];
         stretch_video = false;
 
         /* Used for displaying pause on the screen */
@@ -878,7 +874,8 @@ QemuCocoaView *cocoaView;
 
         // set the supported image file types that can be opened
         supportedImageFileTypes = [NSArray arrayWithObjects: @"img", @"iso", @"dmg",
-                                 @"qcow", @"qcow2", @"cloop", @"vmdk", nil];
+                                 @"qcow", @"qcow2", @"cloop", @"vmdk", @"cdr",
+                                  nil];
     }
     return self;
 }
@@ -895,29 +892,8 @@ QemuCocoaView *cocoaView;
 - (void)applicationDidFinishLaunching: (NSNotification *) note
 {
     COCOA_DEBUG("QemuCocoaAppController: applicationDidFinishLaunching\n");
-
-    // Display an open dialog box if no arguments were passed or
-    // if qemu was launched from the finder ( the Finder passes "-psn" )
-    if( gArgc <= 1 || strncmp ((char *)gArgv[1], "-psn", 4) == 0) {
-        NSOpenPanel *op = [[NSOpenPanel alloc] init];
-        [op setPrompt:@"Boot image"];
-        [op setMessage:@"Select the disk image you want to boot.\n\nHit the \"Cancel\" button to quit"];
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-        [op setAllowedFileTypes:supportedImageFileTypes];
-        [op beginSheetModalForWindow:normalWindow
-            completionHandler:^(NSInteger returnCode)
-            { [self openPanelDidEnd:op
-                  returnCode:returnCode contextInfo:NULL ]; } ];
-#else
-        // Compatibility code for pre-10.6, using deprecated method
-        [op beginSheetForDirectory:nil file:nil types:filetypes
-              modalForWindow:normalWindow modalDelegate:self
-              didEndSelector:@selector(openPanelDidEnd:returnCode:contextInfo:) contextInfo:NULL];
-#endif
-    } else {
-        // or launch QEMU, with the global args
-        [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
-    }
+    // launch QEMU, with the global args
+    [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
 }
 
 - (void)applicationWillTerminate:(NSNotification *)aNotification
@@ -933,43 +909,40 @@ QemuCocoaView *cocoaView;
     return YES;
 }
 
-- (void)startEmulationWithArgc:(int)argc argv:(char**)argv
+- (NSApplicationTerminateReply)applicationShouldTerminate:
+                                                         (NSApplication *)sender
 {
-    COCOA_DEBUG("QemuCocoaAppController: startEmulationWithArgc\n");
-
-    int status;
-    status = qemu_main(argc, argv, *_NSGetEnviron());
-    exit(status);
+    COCOA_DEBUG("QemuCocoaAppController: applicationShouldTerminate\n");
+    return [self verifyQuit];
 }
 
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(NSInteger)returnCode contextInfo:(void *)contextInfo
+/* Called when the user clicks on a window's close button */
+- (BOOL)windowShouldClose:(id)sender
 {
-    COCOA_DEBUG("QemuCocoaAppController: openPanelDidEnd\n");
-
-    /* The NSFileHandlingPanelOKButton/NSFileHandlingPanelCancelButton values for
-     * returnCode strictly only apply for the 10.6-and-up beginSheetModalForWindow
-     * API. For the legacy pre-10.6 beginSheetForDirectory API they are NSOKButton
-     * and NSCancelButton. However conveniently the values are the same.
-     * We use the non-legacy names because the others are deprecated in OSX 10.10.
+    COCOA_DEBUG("QemuCocoaAppController: windowShouldClose\n");
+    [NSApp terminate: sender];
+    /* If the user allows the application to quit then the call to
+     * NSApp terminate will never return. If we get here then the user
+     * cancelled the quit, so we should return NO to not permit the
+     * closing of this window.
      */
-    if (returnCode == NSFileHandlingPanelCancelButton) {
-        exit(0);
-    } else if (returnCode == NSFileHandlingPanelOKButton) {
-        char *img = (char*)[ [ [ sheet URL ] path ] cStringUsingEncoding:NSASCIIStringEncoding];
-
-        char **argv = g_new(char *, 4);
-
-        [sheet close];
+    return NO;
+}
 
-        argv[0] = g_strdup(gArgv[0]);
-        argv[1] = g_strdup("-hda");
-        argv[2] = g_strdup(img);
-        argv[3] = NULL;
+/* Called when QEMU goes into the background */
+- (void) applicationWillResignActive: (NSNotification *)aNotification
+{
+    COCOA_DEBUG("QemuCocoaAppController: applicationWillResignActive\n");
+    [cocoaView raiseAllKeys];
+}
 
-        // printf("Using argc %d argv %s -hda %s\n", 3, gArgv[0], img);
+- (void)startEmulationWithArgc:(int)argc argv:(char**)argv
+{
+    COCOA_DEBUG("QemuCocoaAppController: startEmulationWithArgc\n");
 
-        [self startEmulationWithArgc:3 argv:(char**)argv];
-    }
+    int status;
+    status = qemu_main(argc, argv, *_NSGetEnviron());
+    exit(status);
 }
 
 /* We abstract the method called by the Enter Fullscreen menu item
@@ -988,20 +961,42 @@ QemuCocoaView *cocoaView;
     [cocoaView toggleFullScreen:sender];
 }
 
+/* Tries to find then open the specified filename */
+- (void) openDocumentation: (NSString *) filename
+{
+    /* Where to look for local files */
+    NSString *path_array[] = {@"../share/doc/qemu/", @"../doc/qemu/", @"../"};
+    NSString *full_file_path;
+
+    /* iterate thru the possible paths until the file is found */
+    int index;
+    for (index = 0; index < ARRAY_SIZE(path_array); index++) {
+        full_file_path = [[NSBundle mainBundle] executablePath];
+        full_file_path = [full_file_path stringByDeletingLastPathComponent];
+        full_file_path = [NSString stringWithFormat: @"%@/%@%@", full_file_path,
+                          path_array[index], filename];
+        if ([[NSWorkspace sharedWorkspace] openFile: full_file_path] == YES) {
+            return;
+        }
+    }
+
+    /* If none of the paths opened a file */
+    NSBeep();
+    QEMU_Alert(@"Failed to open file");
+}
+
 - (void)showQEMUDoc:(id)sender
 {
     COCOA_DEBUG("QemuCocoaAppController: showQEMUDoc\n");
 
-    [[NSWorkspace sharedWorkspace] openFile:[NSString stringWithFormat:@"%@/../doc/qemu/qemu-doc.html",
-        [[NSBundle mainBundle] resourcePath]] withApplication:@"Help Viewer"];
+    [self openDocumentation: @"qemu-doc.html"];
 }
 
 - (void)showQEMUTec:(id)sender
 {
     COCOA_DEBUG("QemuCocoaAppController: showQEMUTec\n");
 
-    [[NSWorkspace sharedWorkspace] openFile:[NSString stringWithFormat:@"%@/../doc/qemu/qemu-tech.html",
-        [[NSBundle mainBundle] resourcePath]] withApplication:@"Help Viewer"];
+    [self openDocumentation: @"qemu-tech.html"];
 }
 
 /* Stretches video to fit host monitor size */
@@ -1117,14 +1112,32 @@ QemuCocoaView *cocoaView;
         }
 
         Error *err = NULL;
-        qmp_change_blockdev([drive cStringUsingEncoding: NSASCIIStringEncoding],
-                            [file cStringUsingEncoding: NSASCIIStringEncoding],
-                            "raw",
-                            &err);
+        qmp_blockdev_change_medium([drive cStringUsingEncoding:
+                                          NSASCIIStringEncoding],
+                                   [file cStringUsingEncoding:
+                                         NSASCIIStringEncoding],
+                                   true, "raw",
+                                   false, 0,
+                                   &err);
         handleAnyDeviceErrors(err);
     }
 }
 
+/* Verifies if the user really wants to quit */
+- (BOOL)verifyQuit
+{
+    NSAlert *alert = [NSAlert new];
+    [alert autorelease];
+    [alert setMessageText: @"Are you sure you want to quit QEMU?"];
+    [alert addButtonWithTitle: @"Cancel"];
+    [alert addButtonWithTitle: @"Quit"];
+    if([alert runModal] == NSAlertSecondButtonReturn) {
+        return YES;
+    } else {
+        return NO;
+    }
+}
+
 @end
 
 
@@ -1148,6 +1161,7 @@ int main (int argc, const char * argv[]) {
                 !strcmp(opt, "-nographic") ||
                 !strcmp(opt, "-version") ||
                 !strcmp(opt, "-curses") ||
+                !strcmp(opt, "-display") ||
                 !strcmp(opt, "-qtest")) {
                 return qemu_main(gArgc, gArgv, *_NSGetEnviron());
             }
@@ -1275,6 +1289,7 @@ static void cocoa_refresh(DisplayChangeListener *dcl)
     NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
 
     COCOA_DEBUG("qemu_cocoa: cocoa_refresh\n");
+    graphic_hw_update(NULL);
 
     if (qemu_input_is_absolute()) {
         if (![cocoaView isAbsoluteEnabled]) {
@@ -1295,7 +1310,6 @@ static void cocoa_refresh(DisplayChangeListener *dcl)
             [cocoaView handleEvent:event];
         }
     } while(event != nil);
-    graphic_hw_update(NULL);
     [pool release];
 }
 
@@ -1341,7 +1355,7 @@ static void add_console_menu_entries(void)
 /* Make menu items for all removable devices.
  * Each device is given an 'Eject' and 'Change' menu item.
  */
-static void addRemovableDevicesMenuItems()
+static void addRemovableDevicesMenuItems(void)
 {
     NSMenu *menu;
     NSMenuItem *menuItem;
diff --git a/qemu/ui/console-gl.c b/qemu/ui/console-gl.c
index cb45cf8a2..74b1bed6e 100644
--- a/qemu/ui/console-gl.c
+++ b/qemu/ui/console-gl.c
@@ -24,6 +24,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/shader.h"
@@ -33,6 +34,7 @@
 
 struct ConsoleGLState {
     GLint texture_blit_prog;
+    GLint texture_blit_vao;
 };
 
 /* ---------------------------------------------------------------------- */
@@ -47,6 +49,9 @@ ConsoleGLState *console_gl_init_context(void)
         exit(1);
     }
 
+    gls->texture_blit_vao =
+        qemu_gl_init_texture_blit(gls->texture_blit_prog);
+
     return gls;
 }
 
@@ -131,7 +136,8 @@ void surface_gl_render_texture(ConsoleGLState *gls,
     glClearColor(0.1f, 0.1f, 0.1f, 0.0f);
     glClear(GL_COLOR_BUFFER_BIT);
 
-    qemu_gl_run_texture_blit(gls->texture_blit_prog);
+    qemu_gl_run_texture_blit(gls->texture_blit_prog,
+                             gls->texture_blit_vao);
 }
 
 void surface_gl_destroy_texture(ConsoleGLState *gls,
diff --git a/qemu/ui/console.c b/qemu/ui/console.c
index 75fc492f7..bf385790b 100644
--- a/qemu/ui/console.c
+++ b/qemu/ui/console.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "hw/qdev-core.h"
@@ -121,6 +122,7 @@ struct QemuConsole {
     DisplayState *ds;
     DisplaySurface *surface;
     int dcls;
+    DisplayChangeListener *gl;
 
     /* Graphic console state.  */
     Object *device;
@@ -260,6 +262,16 @@ void graphic_hw_update(QemuConsole *con)
     }
 }
 
+void graphic_hw_gl_block(QemuConsole *con, bool block)
+{
+    if (!con) {
+        con = active_console;
+    }
+    if (con && con->hw_ops->gl_block) {
+        con->hw_ops->gl_block(con->hw, block);
+    }
+}
+
 void graphic_hw_invalidate(QemuConsole *con)
 {
     if (!con) {
@@ -375,42 +387,29 @@ static void vga_bitblt(QemuConsole *con,
 
 #include "vgafont.h"
 
-#ifndef CONFIG_CURSES
-enum color_names {
-    COLOR_BLACK   = 0,
-    COLOR_RED     = 1,
-    COLOR_GREEN   = 2,
-    COLOR_YELLOW  = 3,
-    COLOR_BLUE    = 4,
-    COLOR_MAGENTA = 5,
-    COLOR_CYAN    = 6,
-    COLOR_WHITE   = 7
-};
-#endif
-
 #define QEMU_RGB(r, g, b)                                               \
     { .red = r << 8, .green = g << 8, .blue = b << 8, .alpha = 0xffff }
 
 static const pixman_color_t color_table_rgb[2][8] = {
     {   /* dark */
-        QEMU_RGB(0x00, 0x00, 0x00),  /* black */
-        QEMU_RGB(0xaa, 0x00, 0x00),  /* red */
-        QEMU_RGB(0x00, 0xaa, 0x00),  /* green */
-        QEMU_RGB(0xaa, 0xaa, 0x00),  /* yellow */
-        QEMU_RGB(0x00, 0x00, 0xaa),  /* blue */
-        QEMU_RGB(0xaa, 0x00, 0xaa),  /* magenta */
-        QEMU_RGB(0x00, 0xaa, 0xaa),  /* cyan */
-        QEMU_RGB(0xaa, 0xaa, 0xaa),  /* white */
+        [QEMU_COLOR_BLACK]   = QEMU_RGB(0x00, 0x00, 0x00),  /* black */
+        [QEMU_COLOR_BLUE]    = QEMU_RGB(0x00, 0x00, 0xaa),  /* blue */
+        [QEMU_COLOR_GREEN]   = QEMU_RGB(0x00, 0xaa, 0x00),  /* green */
+        [QEMU_COLOR_CYAN]    = QEMU_RGB(0x00, 0xaa, 0xaa),  /* cyan */
+        [QEMU_COLOR_RED]     = QEMU_RGB(0xaa, 0x00, 0x00),  /* red */
+        [QEMU_COLOR_MAGENTA] = QEMU_RGB(0xaa, 0x00, 0xaa),  /* magenta */
+        [QEMU_COLOR_YELLOW]  = QEMU_RGB(0xaa, 0xaa, 0x00),  /* yellow */
+        [QEMU_COLOR_WHITE]   = QEMU_RGB(0xaa, 0xaa, 0xaa),  /* white */
     },
     {   /* bright */
-        QEMU_RGB(0x00, 0x00, 0x00),  /* black */
-        QEMU_RGB(0xff, 0x00, 0x00),  /* red */
-        QEMU_RGB(0x00, 0xff, 0x00),  /* green */
-        QEMU_RGB(0xff, 0xff, 0x00),  /* yellow */
-        QEMU_RGB(0x00, 0x00, 0xff),  /* blue */
-        QEMU_RGB(0xff, 0x00, 0xff),  /* magenta */
-        QEMU_RGB(0x00, 0xff, 0xff),  /* cyan */
-        QEMU_RGB(0xff, 0xff, 0xff),  /* white */
+        [QEMU_COLOR_BLACK]   = QEMU_RGB(0x00, 0x00, 0x00),  /* black */
+        [QEMU_COLOR_BLUE]    = QEMU_RGB(0x00, 0x00, 0xff),  /* blue */
+        [QEMU_COLOR_GREEN]   = QEMU_RGB(0x00, 0xff, 0x00),  /* green */
+        [QEMU_COLOR_CYAN]    = QEMU_RGB(0x00, 0xff, 0xff),  /* cyan */
+        [QEMU_COLOR_RED]     = QEMU_RGB(0xff, 0x00, 0x00),  /* red */
+        [QEMU_COLOR_MAGENTA] = QEMU_RGB(0xff, 0x00, 0xff),  /* magenta */
+        [QEMU_COLOR_YELLOW]  = QEMU_RGB(0xff, 0xff, 0x00),  /* yellow */
+        [QEMU_COLOR_WHITE]   = QEMU_RGB(0xff, 0xff, 0xff),  /* white */
     }
 };
 
@@ -449,7 +448,7 @@ static void text_console_resize(QemuConsole *s)
     if (s->width < w1)
         w1 = s->width;
 
-    cells = g_malloc(s->width * s->total_height * sizeof(TextCell));
+    cells = g_new(TextCell, s->width * s->total_height);
     for(y = 0; y < s->total_height; y++) {
         c = &cells[y * s->width];
         if (w1 > 0) {
@@ -559,7 +558,7 @@ static void console_refresh(QemuConsole *s)
     }
 
     vga_fill_rect(s, 0, 0, surface_width(surface), surface_height(surface),
-                  color_table_rgb[0][COLOR_BLACK]);
+                  color_table_rgb[0][QEMU_COLOR_BLACK]);
     y1 = s->y_displayed;
     for (y = 0; y < s->height; y++) {
         c = s->cells + y1 * s->width;
@@ -697,53 +696,53 @@ static void console_handle_escape(QemuConsole *s)
                 break;
             /* set foreground color */
             case 30:
-                s->t_attrib.fgcol=COLOR_BLACK;
+                s->t_attrib.fgcol = QEMU_COLOR_BLACK;
                 break;
             case 31:
-                s->t_attrib.fgcol=COLOR_RED;
+                s->t_attrib.fgcol = QEMU_COLOR_RED;
                 break;
             case 32:
-                s->t_attrib.fgcol=COLOR_GREEN;
+                s->t_attrib.fgcol = QEMU_COLOR_GREEN;
                 break;
             case 33:
-                s->t_attrib.fgcol=COLOR_YELLOW;
+                s->t_attrib.fgcol = QEMU_COLOR_YELLOW;
                 break;
             case 34:
-                s->t_attrib.fgcol=COLOR_BLUE;
+                s->t_attrib.fgcol = QEMU_COLOR_BLUE;
                 break;
             case 35:
-                s->t_attrib.fgcol=COLOR_MAGENTA;
+                s->t_attrib.fgcol = QEMU_COLOR_MAGENTA;
                 break;
             case 36:
-                s->t_attrib.fgcol=COLOR_CYAN;
+                s->t_attrib.fgcol = QEMU_COLOR_CYAN;
                 break;
             case 37:
-                s->t_attrib.fgcol=COLOR_WHITE;
+                s->t_attrib.fgcol = QEMU_COLOR_WHITE;
                 break;
             /* set background color */
             case 40:
-                s->t_attrib.bgcol=COLOR_BLACK;
+                s->t_attrib.bgcol = QEMU_COLOR_BLACK;
                 break;
             case 41:
-                s->t_attrib.bgcol=COLOR_RED;
+                s->t_attrib.bgcol = QEMU_COLOR_RED;
                 break;
             case 42:
-                s->t_attrib.bgcol=COLOR_GREEN;
+                s->t_attrib.bgcol = QEMU_COLOR_GREEN;
                 break;
             case 43:
-                s->t_attrib.bgcol=COLOR_YELLOW;
+                s->t_attrib.bgcol = QEMU_COLOR_YELLOW;
                 break;
             case 44:
-                s->t_attrib.bgcol=COLOR_BLUE;
+                s->t_attrib.bgcol = QEMU_COLOR_BLUE;
                 break;
             case 45:
-                s->t_attrib.bgcol=COLOR_MAGENTA;
+                s->t_attrib.bgcol = QEMU_COLOR_MAGENTA;
                 break;
             case 46:
-                s->t_attrib.bgcol=COLOR_CYAN;
+                s->t_attrib.bgcol = QEMU_COLOR_CYAN;
                 break;
             case 47:
-                s->t_attrib.bgcol=COLOR_WHITE;
+                s->t_attrib.bgcol = QEMU_COLOR_WHITE;
                 break;
         }
     }
@@ -758,6 +757,31 @@ static void console_clear_xy(QemuConsole *s, int x, int y)
     update_xy(s, x, y);
 }
 
+static void console_put_one(QemuConsole *s, int ch)
+{
+    TextCell *c;
+    int y1;
+    if (s->x >= s->width) {
+        /* line wrap */
+        s->x = 0;
+        console_put_lf(s);
+    }
+    y1 = (s->y_base + s->y) % s->total_height;
+    c = &s->cells[y1 * s->width + s->x];
+    c->ch = ch;
+    c->t_attrib = s->t_attrib;
+    update_xy(s, s->x, s->y);
+    s->x++;
+}
+
+static void console_respond_str(QemuConsole *s, const char *buf)
+{
+    while (*buf) {
+        console_put_one(s, *buf);
+        buf++;
+    }
+}
+
 /* set cursor, checking bounds */
 static void set_cursor(QemuConsole *s, int x, int y)
 {
@@ -780,9 +804,9 @@ static void set_cursor(QemuConsole *s, int x, int y)
 
 static void console_putchar(QemuConsole *s, int ch)
 {
-    TextCell *c;
-    int y1, i;
+    int i;
     int x, y;
+    char response[40];
 
     switch(s->state) {
     case TTY_STATE_NORM:
@@ -818,17 +842,7 @@ static void console_putchar(QemuConsole *s, int ch)
             s->state = TTY_STATE_ESC;
             break;
         default:
-            if (s->x >= s->width) {
-                /* line wrap */
-                s->x = 0;
-                console_put_lf(s);
-            }
-            y1 = (s->y_base + s->y) % s->total_height;
-            c = &s->cells[y1 * s->width + s->x];
-            c->ch = ch;
-            c->t_attrib = s->t_attrib;
-            update_xy(s, s->x, s->y);
-            s->x++;
+            console_put_one(s, ch);
             break;
         }
         break;
@@ -957,8 +971,19 @@ static void console_putchar(QemuConsole *s, int ch)
                 console_handle_escape(s);
                 break;
             case 'n':
-                /* report cursor position */
-                /* TODO: send ESC[row;colR */
+                switch (s->esc_params[0]) {
+                case 5:
+                    /* report console status (always succeed)*/
+                    console_respond_str(s, "\033[0n");
+                    break;
+                case 6:
+                    /* report cursor position */
+                    sprintf(response, "\033[%d;%dR",
+                           (s->y_base + s->y) % s->total_height + 1,
+                            s->x + 1);
+                    console_respond_str(s, response);
+                    break;
+                }
                 break;
             case 's':
                 /* save cursor position */
@@ -1107,7 +1132,7 @@ void kbd_put_keysym_console(QemuConsole *s, int keysym)
     }
 }
 
-static const int qcode_to_keysym[Q_KEY_CODE_MAX] = {
+static const int qcode_to_keysym[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_UP]     = QEMU_KEY_UP,
     [Q_KEY_CODE_DOWN]   = QEMU_KEY_DOWN,
     [Q_KEY_CODE_RIGHT]  = QEMU_KEY_RIGHT,
@@ -1164,11 +1189,13 @@ static void text_console_update(void *opaque, console_ch_t *chardata)
         src = (s->y_base + s->text_y[0]) * s->width;
         chardata += s->text_y[0] * s->width;
         for (i = s->text_y[0]; i <= s->text_y[1]; i ++)
-            for (j = 0; j < s->width; j ++, src ++)
-                console_write_ch(chardata ++, s->cells[src].ch |
-                                (s->cells[src].t_attrib.fgcol << 12) |
-                                (s->cells[src].t_attrib.bgcol << 8) |
-                                (s->cells[src].t_attrib.bold << 21));
+            for (j = 0; j < s->width; j++, src++) {
+                console_write_ch(chardata ++,
+                                 ATTR2CHTYPE(s->cells[src].ch,
+                                             s->cells[src].t_attrib.fgcol,
+                                             s->cells[src].t_attrib.bgcol,
+                                             s->cells[src].t_attrib.bold));
+            }
         dpy_text_update(s, s->text_x[0], s->text_y[0],
                         s->text_x[1] - s->text_x[0], i - s->text_y[0]);
         s->text_x[0] = s->width;
@@ -1265,6 +1292,17 @@ DisplaySurface *qemu_create_displaysurface_from(int width, int height,
     return surface;
 }
 
+DisplaySurface *qemu_create_displaysurface_pixman(pixman_image_t *image)
+{
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
+
+    trace_displaysurface_create_pixman(surface);
+    surface->format = pixman_image_get_format(image);
+    surface->image = pixman_image_ref(image);
+
+    return surface;
+}
+
 static void qemu_unmap_displaysurface_guestmem(pixman_image_t *image,
                                                void *unused)
 {
@@ -1305,8 +1343,8 @@ static DisplaySurface *qemu_create_message_surface(int w, int h,
                                                    const char *msg)
 {
     DisplaySurface *surface = qemu_create_displaysurface(w, h);
-    pixman_color_t bg = color_table_rgb[0][COLOR_BLACK];
-    pixman_color_t fg = color_table_rgb[0][COLOR_WHITE];
+    pixman_color_t bg = color_table_rgb[0][QEMU_COLOR_BLACK];
+    pixman_color_t fg = color_table_rgb[0][QEMU_COLOR_WHITE];
     pixman_image_t *glyph;
     int len, x, y, i;
 
@@ -1332,6 +1370,11 @@ void qemu_free_displaysurface(DisplaySurface *surface)
     g_free(surface);
 }
 
+bool console_has_gl(QemuConsole *con)
+{
+    return con->gl != NULL;
+}
+
 void register_displaychangelistener(DisplayChangeListener *dcl)
 {
     static const char nodev[] =
@@ -1339,6 +1382,17 @@ void register_displaychangelistener(DisplayChangeListener *dcl)
     static DisplaySurface *dummy;
     QemuConsole *con;
 
+    if (dcl->ops->dpy_gl_ctx_create) {
+        /* display has opengl support */
+        assert(dcl->con);
+        if (dcl->con->gl) {
+            fprintf(stderr, "can't register two opengl displays (%s, %s)\n",
+                    dcl->ops->dpy_name, dcl->con->gl->ops->dpy_name);
+            exit(1);
+        }
+        dcl->con->gl = dcl;
+    }
+
     trace_displaychangelistener_register(dcl, dcl->ops->dpy_name);
     dcl->ds = get_alloc_displaystate();
     QLIST_INSERT_HEAD(&dcl->ds->listeners, dcl, next);
@@ -1417,9 +1471,13 @@ void dpy_gfx_update(QemuConsole *con, int x, int y, int w, int h)
 {
     DisplayState *s = con->ds;
     DisplayChangeListener *dcl;
-    int width = surface_width(con->surface);
-    int height = surface_height(con->surface);
+    int width = w;
+    int height = h;
 
+    if (con->surface) {
+        width = surface_width(con->surface);
+        height = surface_height(con->surface);
+    }
     x = MAX(x, 0);
     y = MAX(y, 0);
     x = MIN(x, width);
@@ -1619,6 +1677,48 @@ bool dpy_cursor_define_supported(QemuConsole *con)
     return false;
 }
 
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                struct QEMUGLParams *qparams)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_create(con->gl, qparams);
+}
+
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_ctx_destroy(con->gl, ctx);
+}
+
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_make_current(con->gl, ctx);
+}
+
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_get_current(con->gl);
+}
+
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y, uint32_t width, uint32_t height)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_scanout(con->gl, backing_id,
+                                 backing_y_0_top,
+                                 x, y, width, height);
+}
+
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_update(con->gl, x, y, w, h);
+}
+
 /***********************************************************/
 /* register display */
 
@@ -1727,6 +1827,29 @@ QemuConsole *qemu_console_lookup_by_device(DeviceState *dev, uint32_t head)
     return NULL;
 }
 
+QemuConsole *qemu_console_lookup_by_device_name(const char *device_id,
+                                                uint32_t head, Error **errp)
+{
+    DeviceState *dev;
+    QemuConsole *con;
+
+    dev = qdev_find_recursive(sysbus_get_default(), device_id);
+    if (dev == NULL) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device_id);
+        return NULL;
+    }
+
+    con = qemu_console_lookup_by_device(dev, head);
+    if (con == NULL) {
+        error_setg(errp, "Device %s (head %d) is not bound to a QemuConsole",
+                   device_id, head);
+        return NULL;
+    }
+
+    return con;
+}
+
 bool qemu_console_is_visible(QemuConsole *con)
 {
     return (con == active_console) || (con->dcls > 0);
@@ -1878,8 +2001,8 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
     s->t_attrib_default.blink = 0;
     s->t_attrib_default.invers = 0;
     s->t_attrib_default.unvisible = 0;
-    s->t_attrib_default.fgcol = COLOR_WHITE;
-    s->t_attrib_default.bgcol = COLOR_BLACK;
+    s->t_attrib_default.fgcol = QEMU_COLOR_WHITE;
+    s->t_attrib_default.bgcol = QEMU_COLOR_BLACK;
     /* set current text attributes to default */
     s->t_attrib = s->t_attrib_default;
     text_console_resize(s);
@@ -1888,7 +2011,7 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
         char msg[128];
         int len;
 
-        s->t_attrib.bgcol = COLOR_BLUE;
+        s->t_attrib.bgcol = QEMU_COLOR_BLUE;
         len = snprintf(msg, sizeof(msg), "%s console\r\n", chr->label);
         console_puts(chr, (uint8_t*)msg, len);
         s->t_attrib = s->t_attrib_default;
@@ -1899,14 +2022,18 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
         chr->init(chr);
 }
 
-static CharDriverState *text_console_init(ChardevVC *vc)
+static CharDriverState *text_console_init(ChardevVC *vc, Error **errp)
 {
+    ChardevCommon *common = qapi_ChardevVC_base(vc);
     CharDriverState *chr;
     QemuConsole *s;
     unsigned width = 0;
     unsigned height = 0;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
 
     if (vc->has_width) {
         width = vc->width;
@@ -1930,6 +2057,7 @@ static CharDriverState *text_console_init(ChardevVC *vc)
 
     if (!s) {
         g_free(chr);
+        error_setg(errp, "cannot create text console");
         return NULL;
     }
 
@@ -1949,9 +2077,10 @@ static CharDriverState *text_console_init(ChardevVC *vc)
 
 static VcHandler *vc_handler = text_console_init;
 
-CharDriverState *vc_init(ChardevVC *vc)
+static CharDriverState *vc_init(const char *id, ChardevBackend *backend,
+                                ChardevReturn *ret, Error **errp)
 {
-    return vc_handler(vc);
+    return vc_handler(backend->u.vc.data, errp);
 }
 
 void register_vc_handler(VcHandler *handler)
@@ -1991,31 +2120,33 @@ static void qemu_chr_parse_vc(QemuOpts *opts, ChardevBackend *backend,
                               Error **errp)
 {
     int val;
+    ChardevVC *vc;
 
-    backend->vc = g_new0(ChardevVC, 1);
+    vc = backend->u.vc.data = g_new0(ChardevVC, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevVC_base(vc));
 
     val = qemu_opt_get_number(opts, "width", 0);
     if (val != 0) {
-        backend->vc->has_width = true;
-        backend->vc->width = val;
+        vc->has_width = true;
+        vc->width = val;
     }
 
     val = qemu_opt_get_number(opts, "height", 0);
     if (val != 0) {
-        backend->vc->has_height = true;
-        backend->vc->height = val;
+        vc->has_height = true;
+        vc->height = val;
     }
 
     val = qemu_opt_get_number(opts, "cols", 0);
     if (val != 0) {
-        backend->vc->has_cols = true;
-        backend->vc->cols = val;
+        vc->has_cols = true;
+        vc->cols = val;
     }
 
     val = qemu_opt_get_number(opts, "rows", 0);
     if (val != 0) {
-        backend->vc->has_rows = true;
-        backend->vc->rows = val;
+        vc->has_rows = true;
+        vc->rows = val;
     }
 }
 
@@ -2030,7 +2161,8 @@ static const TypeInfo qemu_console_info = {
 static void register_types(void)
 {
     type_register_static(&qemu_console_info);
-    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc);
+    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc,
+                         vc_init);
 }
 
 type_init(register_types);
diff --git a/qemu/ui/curses.c b/qemu/ui/curses.c
index 8edb038bb..b47558956 100644
--- a/qemu/ui/curses.c
+++ b/qemu/ui/curses.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <curses.h>
 
 #ifndef _WIN32
@@ -42,6 +43,8 @@ static WINDOW *screenpad = NULL;
 static int width, height, gwidth, gheight, invalidate;
 static int px, py, sminx, sminy, smaxx, smaxy;
 
+chtype vga_to_curses[256];
+
 static void curses_update(DisplayChangeListener *dcl,
                           int x, int y, int w, int h)
 {
@@ -331,8 +334,14 @@ static void curses_atexit(void)
 static void curses_setup(void)
 {
     int i, colour_default[8] = {
-        COLOR_BLACK, COLOR_BLUE, COLOR_GREEN, COLOR_CYAN,
-        COLOR_RED, COLOR_MAGENTA, COLOR_YELLOW, COLOR_WHITE,
+        [QEMU_COLOR_BLACK]   = COLOR_BLACK,
+        [QEMU_COLOR_BLUE]    = COLOR_BLUE,
+        [QEMU_COLOR_GREEN]   = COLOR_GREEN,
+        [QEMU_COLOR_CYAN]    = COLOR_CYAN,
+        [QEMU_COLOR_RED]     = COLOR_RED,
+        [QEMU_COLOR_MAGENTA] = COLOR_MAGENTA,
+        [QEMU_COLOR_YELLOW]  = COLOR_YELLOW,
+        [QEMU_COLOR_WHITE]   = COLOR_WHITE,
     };
 
     /* input as raw as possible, let everything be interpreted
@@ -341,8 +350,56 @@ static void curses_setup(void)
     nodelay(stdscr, TRUE); nonl(); keypad(stdscr, TRUE);
     start_color(); raw(); scrollok(stdscr, FALSE);
 
-    for (i = 0; i < 64; i ++)
+    /* Make color pair to match color format (3bits bg:3bits fg) */
+    for (i = 0; i < 64; i++) {
         init_pair(i, colour_default[i & 7], colour_default[i >> 3]);
+    }
+    /* Set default color for more than 64 for safety. */
+    for (i = 64; i < COLOR_PAIRS; i++) {
+        init_pair(i, COLOR_WHITE, COLOR_BLACK);
+    }
+
+    /*
+     * Setup mapping for vga to curses line graphics.
+     * FIXME: for better font, have to use ncursesw and setlocale()
+     */
+#if 0
+    /* FIXME: map from where? */
+    ACS_S1;
+    ACS_S3;
+    ACS_S7;
+    ACS_S9;
+#endif
+    /* ACS_* is not constant. So, we can't initialize statically. */
+    vga_to_curses['\0'] = ' ';
+    vga_to_curses[0x04] = ACS_DIAMOND;
+    vga_to_curses[0x0a] = ACS_RARROW;
+    vga_to_curses[0x0b] = ACS_LARROW;
+    vga_to_curses[0x18] = ACS_UARROW;
+    vga_to_curses[0x19] = ACS_DARROW;
+    vga_to_curses[0x9c] = ACS_STERLING;
+    vga_to_curses[0xb0] = ACS_BOARD;
+    vga_to_curses[0xb1] = ACS_CKBOARD;
+    vga_to_curses[0xb3] = ACS_VLINE;
+    vga_to_curses[0xb4] = ACS_RTEE;
+    vga_to_curses[0xbf] = ACS_URCORNER;
+    vga_to_curses[0xc0] = ACS_LLCORNER;
+    vga_to_curses[0xc1] = ACS_BTEE;
+    vga_to_curses[0xc2] = ACS_TTEE;
+    vga_to_curses[0xc3] = ACS_LTEE;
+    vga_to_curses[0xc4] = ACS_HLINE;
+    vga_to_curses[0xc5] = ACS_PLUS;
+    vga_to_curses[0xce] = ACS_LANTERN;
+    vga_to_curses[0xd8] = ACS_NEQUAL;
+    vga_to_curses[0xd9] = ACS_LRCORNER;
+    vga_to_curses[0xda] = ACS_ULCORNER;
+    vga_to_curses[0xdb] = ACS_BLOCK;
+    vga_to_curses[0xe3] = ACS_PI;
+    vga_to_curses[0xf1] = ACS_PLMINUS;
+    vga_to_curses[0xf2] = ACS_GEQUAL;
+    vga_to_curses[0xf3] = ACS_LEQUAL;
+    vga_to_curses[0xf8] = ACS_DEGREE;
+    vga_to_curses[0xfe] = ACS_BULLET;
 }
 
 static void curses_keyboard_setup(void)
@@ -382,7 +439,7 @@ void curses_display_init(DisplayState *ds, int full_screen)
 
     curses_winch_init();
 
-    dcl = (DisplayChangeListener *) g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
diff --git a/qemu/ui/curses_keys.h b/qemu/ui/curses_keys.h
index 18ce6dcee..f7467449b 100644
--- a/qemu/ui/curses_keys.h
+++ b/qemu/ui/curses_keys.h
@@ -29,8 +29,7 @@
 #include "keymaps.h"
 
 
-#define KEY_RELEASE         0x80
-#define KEY_MASK            0x7f
+#define KEY_MASK            SCANCODE_KEYMASK
 #define GREY_CODE           0xe0
 #define GREY                SCANCODE_GREY
 #define SHIFT_CODE          0x2a
@@ -60,6 +59,8 @@ static const int curses2keysym[CURSES_KEYS] = {
     ['\n'] = KEY_ENTER,
     [27] = 27,
     [KEY_BTAB] = '\t' | KEYSYM_SHIFT,
+    [KEY_SPREVIOUS] = KEY_PPAGE | KEYSYM_SHIFT,
+    [KEY_SNEXT] = KEY_NPAGE | KEYSYM_SHIFT,
 };
 
 static const int curses2keycode[CURSES_KEYS] = {
@@ -149,6 +150,9 @@ static const int curses2keycode[CURSES_KEYS] = {
     [KEY_IC] = 82 | GREY, /* Insert */
     [KEY_DC] = 83 | GREY, /* Delete */
 
+    [KEY_SPREVIOUS] = 73 | GREY | SHIFT, /* Shift + Page Up */
+    [KEY_SNEXT] = 81 | GREY | SHIFT, /* Shift + Page Down */
+
     ['!'] = 2 | SHIFT,
     ['@'] = 3 | SHIFT,
     ['#'] = 4 | SHIFT,
diff --git a/qemu/ui/cursor.c b/qemu/ui/cursor.c
index 2b8dd3fa5..a276e01f1 100644
--- a/qemu/ui/cursor.c
+++ b/qemu/ui/cursor.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 
diff --git a/qemu/ui/egl-context.c b/qemu/ui/egl-context.c
new file mode 100644
index 000000000..3a02b68d1
--- /dev/null
+++ b/qemu/ui/egl-context.c
@@ -0,0 +1,35 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/egl-context.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params)
+{
+   EGLContext ctx;
+   EGLint ctx_att[] = {
+      EGL_CONTEXT_CLIENT_VERSION, params->major_ver,
+      EGL_CONTEXT_MINOR_VERSION_KHR, params->minor_ver,
+      EGL_NONE
+   };
+
+   ctx = eglCreateContext(qemu_egl_display, qemu_egl_config,
+                          eglGetCurrentContext(), ctx_att);
+   return ctx;
+}
+
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    eglDestroyContext(qemu_egl_display, ctx);
+}
+
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx)
+{
+   return eglMakeCurrent(qemu_egl_display,
+                         EGL_NO_SURFACE, EGL_NO_SURFACE, ctx);
+}
+
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl)
+{
+    return eglGetCurrentContext();
+}
diff --git a/qemu/ui/egl-helpers.c b/qemu/ui/egl-helpers.c
index 87d77afaa..558edfdeb 100644
--- a/qemu/ui/egl-helpers.c
+++ b/qemu/ui/egl-helpers.c
@@ -1,12 +1,6 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <unistd.h>
-#include <string.h>
-#include <errno.h>
-#include <fcntl.h>
+#include "qemu/osdep.h"
 #include <glob.h>
+#include <dirent.h>
 
 #include "ui/egl-helpers.h"
 
@@ -27,6 +21,133 @@ static int egl_debug;
 
 /* ---------------------------------------------------------------------- */
 
+#ifdef CONFIG_OPENGL_DMABUF
+
+int qemu_egl_rn_fd;
+struct gbm_device *qemu_egl_rn_gbm_dev;
+EGLContext qemu_egl_rn_ctx;
+
+int qemu_egl_rendernode_open(void)
+{
+    DIR *dir;
+    struct dirent *e;
+    int r, fd;
+    char *p;
+
+    dir = opendir("/dev/dri");
+    if (!dir) {
+        return -1;
+    }
+
+    fd = -1;
+    while ((e = readdir(dir))) {
+        if (e->d_type != DT_CHR) {
+            continue;
+        }
+
+        if (strncmp(e->d_name, "renderD", 7)) {
+            continue;
+        }
+
+        r = asprintf(&p, "/dev/dri/%s", e->d_name);
+        if (r < 0) {
+            return -1;
+        }
+
+        r = open(p, O_RDWR | O_CLOEXEC | O_NOCTTY | O_NONBLOCK);
+        if (r < 0) {
+            free(p);
+            continue;
+        }
+        fd = r;
+        free(p);
+        break;
+    }
+
+    closedir(dir);
+    if (fd < 0) {
+        return -1;
+    }
+    return fd;
+}
+
+int egl_rendernode_init(void)
+{
+    qemu_egl_rn_fd = -1;
+
+    qemu_egl_rn_fd = qemu_egl_rendernode_open();
+    if (qemu_egl_rn_fd == -1) {
+        fprintf(stderr, "egl: no drm render node available\n");
+        goto err;
+    }
+
+    qemu_egl_rn_gbm_dev = gbm_create_device(qemu_egl_rn_fd);
+    if (!qemu_egl_rn_gbm_dev) {
+        fprintf(stderr, "egl: gbm_create_device failed\n");
+        goto err;
+    }
+
+    qemu_egl_init_dpy((EGLNativeDisplayType)qemu_egl_rn_gbm_dev, false, false);
+
+    if (!epoxy_has_egl_extension(qemu_egl_display,
+                                 "EGL_KHR_surfaceless_context")) {
+        fprintf(stderr, "egl: EGL_KHR_surfaceless_context not supported\n");
+        goto err;
+    }
+    if (!epoxy_has_egl_extension(qemu_egl_display,
+                                 "EGL_MESA_image_dma_buf_export")) {
+        fprintf(stderr, "egl: EGL_MESA_image_dma_buf_export not supported\n");
+        goto err;
+    }
+
+    qemu_egl_rn_ctx = qemu_egl_init_ctx();
+    if (!qemu_egl_rn_ctx) {
+        fprintf(stderr, "egl: egl_init_ctx failed\n");
+        goto err;
+    }
+
+    return 0;
+
+err:
+    if (qemu_egl_rn_gbm_dev) {
+        gbm_device_destroy(qemu_egl_rn_gbm_dev);
+    }
+    if (qemu_egl_rn_fd != -1) {
+        close(qemu_egl_rn_fd);
+    }
+
+    return -1;
+}
+
+int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc)
+{
+    EGLImageKHR image;
+    EGLint num_planes, fd;
+
+    image = eglCreateImageKHR(qemu_egl_display, eglGetCurrentContext(),
+                              EGL_GL_TEXTURE_2D_KHR,
+                              (EGLClientBuffer)(unsigned long)tex_id,
+                              NULL);
+    if (!image) {
+        return -1;
+    }
+
+    eglExportDMABUFImageQueryMESA(qemu_egl_display, image, fourcc,
+                                  &num_planes, NULL);
+    if (num_planes != 1) {
+        eglDestroyImageKHR(qemu_egl_display, image);
+        return -1;
+    }
+    eglExportDMABUFImageMESA(qemu_egl_display, image, &fd, stride, NULL);
+    eglDestroyImageKHR(qemu_egl_display, image);
+
+    return fd;
+}
+
+#endif /* CONFIG_OPENGL_DMABUF */
+
+/* ---------------------------------------------------------------------- */
+
 EGLSurface qemu_egl_init_surface_x11(EGLContext ectx, Window win)
 {
     EGLSurface esurface;
diff --git a/qemu/ui/gtk-egl.c b/qemu/ui/gtk-egl.c
index 15b41f2ba..431457c74 100644
--- a/qemu/ui/gtk-egl.c
+++ b/qemu/ui/gtk-egl.c
@@ -11,6 +11,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 #include "trace.h"
@@ -21,6 +22,29 @@
 
 #include "sysemu/sysemu.h"
 
+static void gtk_egl_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
 /** DisplayState Callbacks (opengl version) **/
 
 void gd_egl_init(VirtualConsole *vc)
@@ -50,19 +74,26 @@ void gd_egl_draw(VirtualConsole *vc)
     GdkWindow *window;
     int ww, wh;
 
-    if (!vc->gfx.gls || !vc->gfx.ds) {
+    if (!vc->gfx.gls) {
         return;
     }
 
-    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
-                   vc->gfx.esurface, vc->gfx.ectx);
+    if (vc->gfx.scanout_mode) {
+        gd_egl_scanout_flush(&vc->gfx.dcl, 0, 0, vc->gfx.w, vc->gfx.h);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                       vc->gfx.esurface, vc->gfx.ectx);
 
-    window = gtk_widget_get_window(vc->gfx.drawing_area);
-    gdk_drawable_get_size(window, &ww, &wh);
-    surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
-    surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+        window = gtk_widget_get_window(vc->gfx.drawing_area);
+        gdk_drawable_get_size(window, &ww, &wh);
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
 
-    eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+        eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+    }
 }
 
 void gd_egl_update(DisplayChangeListener *dcl,
@@ -99,6 +130,7 @@ void gd_egl_refresh(DisplayChangeListener *dcl)
 
     if (vc->gfx.glupdates) {
         vc->gfx.glupdates = 0;
+        gtk_egl_set_scanout_mode(vc, false);
         gd_egl_draw(vc);
     }
 }
@@ -128,6 +160,81 @@ void gd_egl_switch(DisplayChangeListener *dcl,
     }
 }
 
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+    return qemu_egl_create_context(dcl, params);
+}
+
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_egl_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_egl_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.scanout_mode) {
+        return;
+    }
+    if (!vc->gfx.fbo_id) {
+        return;
+    }
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    gdk_drawable_get_size(window, &ww, &wh);
+    glViewport(0, 0, ww, wh);
+    y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+    y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+    glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                      0, 0, ww, wh,
+                      GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+
+    eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+}
+
 void gtk_egl_init(void)
 {
     GdkDisplay *gdk_display = gdk_display_get_default();
@@ -139,3 +246,12 @@ void gtk_egl_init(void)
 
     display_opengl = 1;
 }
+
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    return eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                          vc->gfx.esurface, ctx);
+}
diff --git a/qemu/ui/gtk-gl-area.c b/qemu/ui/gtk-gl-area.c
new file mode 100644
index 000000000..b86ff3cbe
--- /dev/null
+++ b/qemu/ui/gtk-gl-area.c
@@ -0,0 +1,224 @@
+/*
+ * GTK UI -- glarea opengl code.
+ *
+ * Requires 3.16+ (GtkGLArea widget).
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "trace.h"
+
+#include "ui/console.h"
+#include "ui/gtk.h"
+#include "ui/egl-helpers.h"
+
+#include "sysemu/sysemu.h"
+
+static void gtk_gl_area_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
+/** DisplayState Callbacks (opengl version) **/
+
+void gd_gl_area_draw(VirtualConsole *vc)
+{
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.gls) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    ww = gtk_widget_get_allocated_width(vc->gfx.drawing_area);
+    wh = gtk_widget_get_allocated_height(vc->gfx.drawing_area);
+
+    if (vc->gfx.scanout_mode) {
+        if (!vc->gfx.fbo_id) {
+            return;
+        }
+
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+        /* GtkGLArea sets GL_DRAW_FRAMEBUFFER for us */
+
+        glViewport(0, 0, ww, wh);
+        y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+        y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+        glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                          0, 0, ww, wh,
+                          GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+    }
+}
+
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                   int x, int y, int w, int h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls || !vc->gfx.ds) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    surface_gl_update_texture(vc->gfx.gls, vc->gfx.ds, x, y, w, h);
+    vc->gfx.glupdates++;
+}
+
+void gd_gl_area_refresh(DisplayChangeListener *dcl)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls) {
+        if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        vc->gfx.gls = console_gl_init_context();
+        if (vc->gfx.ds) {
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+
+    graphic_hw_update(dcl->con);
+
+    if (vc->gfx.glupdates) {
+        vc->gfx.glupdates = 0;
+        gtk_gl_area_set_scanout_mode(vc, false);
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+    }
+}
+
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    bool resized = true;
+
+    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
+
+    if (vc->gfx.ds &&
+        surface_width(vc->gfx.ds) == surface_width(surface) &&
+        surface_height(vc->gfx.ds) == surface_height(surface)) {
+        resized = false;
+    }
+
+    if (vc->gfx.gls) {
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+        surface_gl_create_texture(vc->gfx.gls, surface);
+    }
+    vc->gfx.ds = surface;
+
+    if (resized) {
+        gd_update_windowsize(vc);
+    }
+}
+
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    GdkGLContext *ctx;
+    GError *err = NULL;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    ctx = gdk_window_create_gl_context(window, &err);
+    gdk_gl_context_set_required_version(ctx,
+                                        params->major_ver,
+                                        params->minor_ver);
+    gdk_gl_context_realize(ctx, &err);
+    return ctx;
+}
+
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    /* FIXME */
+}
+
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_gl_area_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_gl_area_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+}
+
+void gtk_gl_area_init(void)
+{
+    display_opengl = 1;
+}
+
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl)
+{
+    return gdk_gl_context_get_current();
+}
+
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx)
+{
+    gdk_gl_context_make_current(ctx);
+    return 0;
+}
diff --git a/qemu/ui/gtk.c b/qemu/ui/gtk.c
index df2a79e7a..f372a6d5a 100644
--- a/qemu/ui/gtk.c
+++ b/qemu/ui/gtk.c
@@ -34,7 +34,9 @@
 #define GETTEXT_PACKAGE "qemu"
 #define LOCALEDIR "po"
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 
 #include "ui/console.h"
 #include "ui/gtk.h"
@@ -104,6 +106,15 @@
 #define GDK_KEY_Pause GDK_Pause
 #endif
 
+/* Some older mingw versions lack this constant or have
+ * it conditionally defined */
+#ifdef _WIN32
+# ifndef MAPVK_VK_TO_VSC
+#  define MAPVK_VK_TO_VSC 0
+# endif
+#endif
+
+
 #define HOTKEY_MODIFIERS        (GDK_CONTROL_MASK | GDK_MOD1_MASK)
 
 static const int modifier_keycode[] = {
@@ -165,8 +176,10 @@ struct GtkDisplayState {
     bool ignore_keys;
 };
 
-static void gd_grab_pointer(VirtualConsole *vc);
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason);
 static void gd_ungrab_pointer(GtkDisplayState *s);
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason);
+static void gd_ungrab_keyboard(GtkDisplayState *s);
 
 /** Utility Functions **/
 
@@ -356,6 +369,12 @@ static void gd_update_full_redraw(VirtualConsole *vc)
     GtkWidget *area = vc->gfx.drawing_area;
     int ww, wh;
     gdk_drawable_get_size(gtk_widget_get_window(area), &ww, &wh);
+#if defined(CONFIG_GTK_GL)
+    if (vc->gfx.gls) {
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+        return;
+    }
+#endif
     gtk_widget_queue_draw_area(area, 0, 0, ww, wh);
 }
 
@@ -596,6 +615,27 @@ static const DisplayChangeListenerOps dcl_ops = {
 
 /** DisplayState Callbacks (opengl version) **/
 
+#if defined(CONFIG_GTK_GL)
+
+static const DisplayChangeListenerOps dcl_gl_area_ops = {
+    .dpy_name             = "gtk-egl",
+    .dpy_gfx_update       = gd_gl_area_update,
+    .dpy_gfx_switch       = gd_gl_area_switch,
+    .dpy_gfx_check_format = console_gl_check_format,
+    .dpy_refresh          = gd_gl_area_refresh,
+    .dpy_mouse_set        = gd_mouse_set,
+    .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_gl_area_create_context,
+    .dpy_gl_ctx_destroy      = gd_gl_area_destroy_context,
+    .dpy_gl_ctx_make_current = gd_gl_area_make_current,
+    .dpy_gl_ctx_get_current  = gd_gl_area_get_current_context,
+    .dpy_gl_scanout          = gd_gl_area_scanout,
+    .dpy_gl_update           = gd_gl_area_scanout_flush,
+};
+
+#else
+
 static const DisplayChangeListenerOps dcl_egl_ops = {
     .dpy_name             = "gtk-egl",
     .dpy_gfx_update       = gd_egl_update,
@@ -604,9 +644,17 @@ static const DisplayChangeListenerOps dcl_egl_ops = {
     .dpy_refresh          = gd_egl_refresh,
     .dpy_mouse_set        = gd_mouse_set,
     .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_egl_create_context,
+    .dpy_gl_ctx_destroy      = qemu_egl_destroy_context,
+    .dpy_gl_ctx_make_current = gd_egl_make_current,
+    .dpy_gl_ctx_get_current  = qemu_egl_get_current_context,
+    .dpy_gl_scanout          = gd_egl_scanout,
+    .dpy_gl_update           = gd_egl_scanout_flush,
 };
 
-#endif
+#endif /* CONFIG_GTK_GL */
+#endif /* CONFIG_OPENGL */
 
 /** QEMU Events **/
 
@@ -656,6 +704,39 @@ static gboolean gd_window_close(GtkWidget *widget, GdkEvent *event,
     return TRUE;
 }
 
+static void gd_set_ui_info(VirtualConsole *vc, gint width, gint height)
+{
+    QemuUIInfo info;
+
+    memset(&info, 0, sizeof(info));
+    info.width = width;
+    info.height = height;
+    dpy_set_ui_info(vc->gfx.dcl.con, &info);
+}
+
+#if defined(CONFIG_GTK_GL)
+
+static gboolean gd_render_event(GtkGLArea *area, GdkGLContext *context,
+                                void *opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    if (vc->gfx.gls) {
+        gd_gl_area_draw(vc);
+    }
+    return TRUE;
+}
+
+static void gd_resize_event(GtkGLArea *area,
+                            gint width, gint height, gpointer *opaque)
+{
+    VirtualConsole *vc = (void *)opaque;
+
+    gd_set_ui_info(vc, width, height);
+}
+
+#endif
+
 static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 {
     VirtualConsole *vc = opaque;
@@ -666,8 +747,13 @@ static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 
 #if defined(CONFIG_OPENGL)
     if (vc->gfx.gls) {
+#if defined(CONFIG_GTK_GL)
+        /* invoke render callback please */
+        return FALSE;
+#else
         gd_egl_draw(vc);
         return TRUE;
+#endif
     }
 #endif
 
@@ -849,13 +935,11 @@ static gboolean gd_button_event(GtkWidget *widget, GdkEventButton *button,
     /* implicitly grab the input at the first click in the relative mode */
     if (button->button == 1 && button->type == GDK_BUTTON_PRESS &&
         !qemu_input_is_absolute() && s->ptr_owner != vc) {
-        gd_ungrab_pointer(s);
         if (!vc->window) {
             gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                            TRUE);
         } else {
-            gd_grab_pointer(vc);
-            gd_update_caption(s);
+            gd_grab_pointer(vc, "relative-mode-click");
         }
         return TRUE;
     }
@@ -1092,9 +1176,8 @@ static gboolean gd_win_grab(void *opaque)
     if (vc->s->ptr_owner) {
         gd_ungrab_pointer(vc->s);
     } else {
-        gd_grab_pointer(vc);
+        gd_grab_pointer(vc, "user-request-detached-tab");
     }
-    gd_update_caption(vc->s);
     return TRUE;
 }
 
@@ -1141,10 +1224,6 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
         gtk_widget_hide(s->menu_bar);
         if (vc->type == GD_VC_GFX) {
             gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
-            if (qemu_console_is_graphic(vc->gfx.dcl.con)) {
-                gtk_check_menu_item_set_active
-                    (GTK_CHECK_MENU_ITEM(s->grab_item), TRUE);
-            }
         }
         gtk_window_fullscreen(GTK_WINDOW(s->window));
         s->full_screen = TRUE;
@@ -1157,8 +1236,6 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
             vc->gfx.scale_x = 1.0;
             vc->gfx.scale_y = 1.0;
             gd_update_windowsize(vc);
-            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
-                                           FALSE);
         }
     }
 
@@ -1257,8 +1334,16 @@ static void gd_grab_devices(VirtualConsole *vc, bool grab,
 }
 #endif
 
-static void gd_grab_keyboard(VirtualConsole *vc)
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason)
 {
+    if (vc->s->kbd_owner) {
+        if (vc->s->kbd_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_keyboard(vc->s);
+        }
+    }
+
 #if GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, true, GDK_SOURCE_KEYBOARD,
                    GDK_KEY_PRESS_MASK | GDK_KEY_RELEASE_MASK,
@@ -1269,7 +1354,8 @@ static void gd_grab_keyboard(VirtualConsole *vc)
                       GDK_CURRENT_TIME);
 #endif
     vc->s->kbd_owner = vc;
-    trace_gd_grab(vc->label, "kbd", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "kbd", reason);
 }
 
 static void gd_ungrab_keyboard(GtkDisplayState *s)
@@ -1286,12 +1372,22 @@ static void gd_ungrab_keyboard(GtkDisplayState *s)
 #else
     gdk_keyboard_ungrab(GDK_CURRENT_TIME);
 #endif
-    trace_gd_grab(vc->label, "kbd", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "kbd");
 }
 
-static void gd_grab_pointer(VirtualConsole *vc)
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason)
 {
     GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
+
+    if (vc->s->ptr_owner) {
+        if (vc->s->ptr_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_pointer(vc->s);
+        }
+    }
+
 #if GTK_CHECK_VERSION(3, 0, 0)
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     gd_grab_devices(vc, true, GDK_SOURCE_MOUSE,
@@ -1318,7 +1414,8 @@ static void gd_grab_pointer(VirtualConsole *vc)
                             &vc->s->grab_x_root, &vc->s->grab_y_root, NULL);
 #endif
     vc->s->ptr_owner = vc;
-    trace_gd_grab(vc->label, "ptr", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "ptr", reason);
 }
 
 static void gd_ungrab_pointer(GtkDisplayState *s)
@@ -1343,7 +1440,8 @@ static void gd_ungrab_pointer(GtkDisplayState *s)
                              gtk_widget_get_screen(vc->gfx.drawing_area),
                              vc->s->grab_x_root, vc->s->grab_y_root);
 #endif
-    trace_gd_grab(vc->label, "ptr", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "ptr");
 }
 
 static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
@@ -1352,16 +1450,13 @@ static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
     VirtualConsole *vc = gd_vc_find_current(s);
 
     if (gd_is_grab_active(s)) {
-        if (!gd_grab_on_hover(s)) {
-            gd_grab_keyboard(vc);
-        }
-        gd_grab_pointer(vc);
+        gd_grab_keyboard(vc, "user-request-main-window");
+        gd_grab_pointer(vc, "user-request-main-window");
     } else {
         gd_ungrab_keyboard(s);
         gd_ungrab_pointer(s);
     }
 
-    gd_update_caption(s);
     gd_update_cursor(vc);
 }
 
@@ -1415,9 +1510,7 @@ static gboolean gd_enter_event(GtkWidget *widget, GdkEventCrossing *crossing,
     GtkDisplayState *s = vc->s;
 
     if (gd_grab_on_hover(s)) {
-        gd_ungrab_keyboard(s);
-        gd_grab_keyboard(vc);
-        gd_update_caption(s);
+        gd_grab_keyboard(vc, "grab-on-hover");
     }
     return TRUE;
 }
@@ -1430,7 +1523,6 @@ static gboolean gd_leave_event(GtkWidget *widget, GdkEventCrossing *crossing,
 
     if (gd_grab_on_hover(s)) {
         gd_ungrab_keyboard(s);
-        gd_update_caption(s);
     }
     return TRUE;
 }
@@ -1449,12 +1541,8 @@ static gboolean gd_configure(GtkWidget *widget,
                              GdkEventConfigure *cfg, gpointer opaque)
 {
     VirtualConsole *vc = opaque;
-    QemuUIInfo info;
 
-    memset(&info, 0, sizeof(info));
-    info.width = cfg->width;
-    info.height = cfg->height;
-    dpy_set_ui_info(vc->gfx.dcl.con, &info);
+    gd_set_ui_info(vc, cfg->width, cfg->height);
     return FALSE;
 }
 
@@ -1502,15 +1590,32 @@ static int gd_vc_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
     return len;
 }
 
+static void gd_vc_chr_set_echo(CharDriverState *chr, bool echo)
+{
+    VirtualConsole *vc = chr->opaque;
+
+    vc->vte.echo = echo;
+}
+
 static int nb_vcs;
 static CharDriverState *vcs[MAX_VCS];
 
-static CharDriverState *gd_vc_handler(ChardevVC *unused)
+static CharDriverState *gd_vc_handler(ChardevVC *vc, Error **errp)
 {
+    ChardevCommon *common = qapi_ChardevVC_base(vc);
     CharDriverState *chr;
 
-    chr = g_malloc0(sizeof(*chr));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+
     chr->chr_write = gd_vc_chr_write;
+    chr->chr_set_echo = gd_vc_chr_set_echo;
+
+    /* Temporary, until gd_vc_vte_init runs.  */
+    chr->opaque = g_new0(VirtualConsole, 1);
+
     /* defer OPENED events until our vc is fully initialized */
     chr->explicit_be_open = true;
 
@@ -1524,6 +1629,24 @@ static gboolean gd_vc_in(VteTerminal *terminal, gchar *text, guint size,
 {
     VirtualConsole *vc = user_data;
 
+    if (vc->vte.echo) {
+        VteTerminal *term = VTE_TERMINAL(vc->vte.terminal);
+        int i;
+        for (i = 0; i < size; i++) {
+            uint8_t c = text[i];
+            if (c >= 128 || isprint(c)) {
+                /* 8-bit characters are considered printable.  */
+                vte_terminal_feed(term, &text[i], 1);
+            } else if (c == '\r' || c == '\n') {
+                vte_terminal_feed(term, "\r\n", 2);
+            } else {
+                char ctrl[2] = { '^', 0};
+                ctrl[1] = text[i] ^ 64;
+                vte_terminal_feed(term, ctrl, 2);
+            }
+        }
+    }
+
     qemu_chr_be_write(vc->vte.chr, (uint8_t  *)text, (unsigned int)size);
     return TRUE;
 }
@@ -1536,9 +1659,14 @@ static GSList *gd_vc_vte_init(GtkDisplayState *s, VirtualConsole *vc,
     GtkWidget *box;
     GtkWidget *scrollbar;
     GtkAdjustment *vadjustment;
+    VirtualConsole *tmp_vc = chr->opaque;
 
     vc->s = s;
+    vc->vte.echo = tmp_vc->vte.echo;
+
     vc->vte.chr = chr;
+    chr->opaque = vc;
+    g_free(tmp_vc);
 
     snprintf(buffer, sizeof(buffer), "vc%d", idx);
     vc->label = g_strdup_printf("%s", vc->vte.chr->label
@@ -1548,6 +1676,15 @@ static GSList *gd_vc_vte_init(GtkDisplayState *s, VirtualConsole *vc,
     vc->vte.terminal = vte_terminal_new();
     g_signal_connect(vc->vte.terminal, "commit", G_CALLBACK(gd_vc_in), vc);
 
+    /* The documentation says that the default is UTF-8, but actually it is
+     * 7-bit ASCII at least in VTE 0.38.
+     */
+#if VTE_CHECK_VERSION(0, 40, 0)
+    vte_terminal_set_encoding(VTE_TERMINAL(vc->vte.terminal), "UTF-8", NULL);
+#else
+    vte_terminal_set_encoding(VTE_TERMINAL(vc->vte.terminal), "UTF-8");
+#endif
+
     vte_terminal_set_scrollback_lines(VTE_TERMINAL(vc->vte.terminal), -1);
     vte_terminal_set_size(VTE_TERMINAL(vc->vte.terminal),
                           VC_TERM_X_MIN, VC_TERM_Y_MIN);
@@ -1570,7 +1707,6 @@ static GSList *gd_vc_vte_init(GtkDisplayState *s, VirtualConsole *vc,
     gtk_box_pack_start(GTK_BOX(box), vc->vte.terminal, TRUE, TRUE, 0);
     gtk_box_pack_start(GTK_BOX(box), scrollbar, FALSE, FALSE, 0);
 
-    vc->vte.chr->opaque = vc;
     vc->vte.box = box;
     vc->vte.scrollbar = scrollbar;
 
@@ -1611,6 +1747,15 @@ static void gd_connect_vc_gfx_signals(VirtualConsole *vc)
 #if GTK_CHECK_VERSION(3, 0, 0)
     g_signal_connect(vc->gfx.drawing_area, "draw",
                      G_CALLBACK(gd_draw_event), vc);
+#if defined(CONFIG_GTK_GL)
+    if (display_opengl) {
+        /* wire up GtkGlArea events */
+        g_signal_connect(vc->gfx.drawing_area, "render",
+                         G_CALLBACK(gd_render_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "resize",
+                         G_CALLBACK(gd_resize_event), vc);
+    }
+#endif
 #else
     g_signal_connect(vc->gfx.drawing_area, "expose-event",
                      G_CALLBACK(gd_expose_event), vc);
@@ -1719,26 +1864,13 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
     vc->gfx.scale_x = 1.0;
     vc->gfx.scale_y = 1.0;
 
-    vc->gfx.drawing_area = gtk_drawing_area_new();
-    gtk_widget_add_events(vc->gfx.drawing_area,
-                          GDK_POINTER_MOTION_MASK |
-                          GDK_BUTTON_PRESS_MASK |
-                          GDK_BUTTON_RELEASE_MASK |
-                          GDK_BUTTON_MOTION_MASK |
-                          GDK_ENTER_NOTIFY_MASK |
-                          GDK_LEAVE_NOTIFY_MASK |
-                          GDK_SCROLL_MASK |
-                          GDK_KEY_PRESS_MASK);
-    gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
-
-    vc->type = GD_VC_GFX;
-    vc->tab_item = vc->gfx.drawing_area;
-    vc->focus = vc->gfx.drawing_area;
-    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
-                             vc->tab_item, gtk_label_new(vc->label));
-
 #if defined(CONFIG_OPENGL)
     if (display_opengl) {
+#if defined(CONFIG_GTK_GL)
+        vc->gfx.drawing_area = gtk_gl_area_new();
+        vc->gfx.dcl.ops = &dcl_gl_area_ops;
+#else
+        vc->gfx.drawing_area = gtk_drawing_area_new();
         /*
          * gtk_widget_set_double_buffered() was deprecated in 3.14.
          * It is required for opengl rendering on X11 though.  A
@@ -1754,12 +1886,32 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
 #pragma GCC diagnostic pop
 #endif
         vc->gfx.dcl.ops = &dcl_egl_ops;
+#endif /* CONFIG_GTK_GL */
     } else
 #endif
     {
+        vc->gfx.drawing_area = gtk_drawing_area_new();
         vc->gfx.dcl.ops = &dcl_ops;
     }
 
+
+    gtk_widget_add_events(vc->gfx.drawing_area,
+                          GDK_POINTER_MOTION_MASK |
+                          GDK_BUTTON_PRESS_MASK |
+                          GDK_BUTTON_RELEASE_MASK |
+                          GDK_BUTTON_MOTION_MASK |
+                          GDK_ENTER_NOTIFY_MASK |
+                          GDK_LEAVE_NOTIFY_MASK |
+                          GDK_SCROLL_MASK |
+                          GDK_KEY_PRESS_MASK);
+    gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
+
+    vc->type = GD_VC_GFX;
+    vc->tab_item = vc->gfx.drawing_area;
+    vc->focus = vc->gfx.drawing_area;
+    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
+                             vc->tab_item, gtk_label_new(vc->label));
+
     vc->gfx.dcl.con = con;
     register_displaychangelistener(&vc->gfx.dcl);
 
@@ -1768,6 +1920,7 @@ static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
 
     if (dpy_ui_info_supported(vc->gfx.dcl.con)) {
         gtk_menu_item_activate(GTK_MENU_ITEM(s->zoom_fit_item));
+        s->free_scale = true;
     }
 
     return group;
@@ -1941,7 +2094,8 @@ void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 
     s->free_scale = FALSE;
 
-    setlocale(LC_ALL, "");
+    /* LC_MESSAGES only. See early_gtk_display_init() for details */
+    setlocale(LC_MESSAGES, "");
     bindtextdomain("qemu", CONFIG_QEMU_LOCALEDIR);
     textdomain("qemu");
 
@@ -2010,6 +2164,24 @@ void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 
 void early_gtk_display_init(int opengl)
 {
+    /* The QEMU code relies on the assumption that it's always run in
+     * the C locale. Therefore it is not prepared to deal with
+     * operations that produce different results depending on the
+     * locale, such as printf's formatting of decimal numbers, and
+     * possibly others.
+     *
+     * Since GTK+ calls setlocale() by default -importing the locale
+     * settings from the environment- we must prevent it from doing so
+     * using gtk_disable_setlocale().
+     *
+     * QEMU's GTK+ UI, however, _does_ have translations for some of
+     * the menu items. As a trade-off between a functionally correct
+     * QEMU and a fully internationalized UI we support importing
+     * LC_MESSAGES from the environment (see the setlocale() call
+     * earlier in this file). This allows us to display translated
+     * messages leaving everything else untouched.
+     */
+    gtk_disable_setlocale();
     gtkinit = gtk_init_check(NULL, NULL);
     if (!gtkinit) {
         /* don't exit yet, that'll break -help */
@@ -2022,8 +2194,12 @@ void early_gtk_display_init(int opengl)
         break;
     case 1: /* on */
 #if defined(CONFIG_OPENGL)
+#if defined(CONFIG_GTK_GL)
+        gtk_gl_area_init();
+#else
         gtk_egl_init();
 #endif
+#endif
         break;
     default:
         g_assert_not_reached();
diff --git a/qemu/ui/input-keymap.c b/qemu/ui/input-keymap.c
index 7635cb0dc..f1e700d72 100644
--- a/qemu/ui/input-keymap.c
+++ b/qemu/ui/input-keymap.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "ui/keymaps.h"
 #include "ui/input.h"
@@ -132,18 +133,18 @@ static const int qcode_to_number[] = {
     [Q_KEY_CODE_RO] = 0x73,
     [Q_KEY_CODE_KP_COMMA] = 0x7e,
 
-    [Q_KEY_CODE_MAX] = 0,
+    [Q_KEY_CODE__MAX] = 0,
 };
 
 static int number_to_qcode[0x100];
 
 int qemu_input_key_value_to_number(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return qcode_to_number[value->qcode];
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return qcode_to_number[value->u.qcode.data];
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return value->number;
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return value->u.number.data;
     }
 }
 
@@ -154,7 +155,7 @@ int qemu_input_key_number_to_qcode(uint8_t nr)
     if (first) {
         int qcode, number;
         first = false;
-        for (qcode = 0; qcode < Q_KEY_CODE_MAX; qcode++) {
+        for (qcode = 0; qcode < Q_KEY_CODE__MAX; qcode++) {
             number = qcode_to_number[qcode];
             assert(number < ARRAY_SIZE(number_to_qcode));
             number_to_qcode[number] = qcode;
@@ -166,11 +167,11 @@ int qemu_input_key_number_to_qcode(uint8_t nr)
 
 int qemu_input_key_value_to_qcode(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return value->qcode;
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return value->u.qcode.data;
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return qemu_input_key_number_to_qcode(value->number);
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return qemu_input_key_number_to_qcode(value->u.number.data);
     }
 }
 
@@ -180,8 +181,8 @@ int qemu_input_key_value_to_scancode(const KeyValue *value, bool down,
     int keycode = qemu_input_key_value_to_number(value);
     int count = 0;
 
-    if (value->kind == KEY_VALUE_KIND_QCODE &&
-        value->qcode == Q_KEY_CODE_PAUSE) {
+    if (value->type == KEY_VALUE_KIND_QCODE &&
+        value->u.qcode.data == Q_KEY_CODE_PAUSE) {
         /* specific case */
         int v = down ? 0 : 0x80;
         codes[count++] = 0xe1;
diff --git a/qemu/ui/input-legacy.c b/qemu/ui/input-legacy.c
index e50f2968e..715974740 100644
--- a/qemu/ui/input-legacy.c
+++ b/qemu/ui/input-legacy.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "ui/console.h"
-#include "qapi/error.h"
 #include "qmp-commands.h"
 #include "qapi-types.h"
 #include "ui/keymaps.h"
@@ -38,7 +38,7 @@ struct QEMUPutMouseEntry {
     /* new input core */
     QemuInputHandler h;
     QemuInputHandlerState *s;
-    int axis[INPUT_AXIS_MAX];
+    int axis[INPUT_AXIS__MAX];
     int buttons;
 };
 
@@ -57,17 +57,18 @@ struct QEMUPutLEDEntry {
 static QTAILQ_HEAD(, QEMUPutLEDEntry) led_handlers =
     QTAILQ_HEAD_INITIALIZER(led_handlers);
 
-int index_from_key(const char *key)
+int index_from_key(const char *key, size_t key_length)
 {
     int i;
 
     for (i = 0; QKeyCode_lookup[i] != NULL; i++) {
-        if (!strcmp(key, QKeyCode_lookup[i])) {
+        if (!strncmp(key, QKeyCode_lookup[i], key_length) &&
+            !QKeyCode_lookup[i][key_length]) {
             break;
         }
     }
 
-    /* Return Q_KEY_CODE_MAX if the key is invalid */
+    /* Return Q_KEY_CODE__MAX if the key is invalid */
     return i;
 }
 
@@ -109,12 +110,13 @@ static void legacy_kbd_event(DeviceState *dev, QemuConsole *src,
 {
     QEMUPutKbdEntry *entry = (QEMUPutKbdEntry *)dev;
     int scancodes[3], i, count;
+    InputKeyEvent *key = evt->u.key.data;
 
     if (!entry || !entry->put_kbd) {
         return;
     }
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         entry->put_kbd(entry->opaque, scancodes[i]);
@@ -143,28 +145,31 @@ QEMUPutKbdEntry *qemu_add_kbd_event_handler(QEMUPutKBDEvent *func, void *opaque)
 static void legacy_mouse_event(DeviceState *dev, QemuConsole *src,
                                InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE] = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]  = MOUSE_EVENT_RBUTTON,
     };
     QEMUPutMouseEntry *s = (QEMUPutMouseEntry *)dev;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->buttons |= bmap[evt->btn->button];
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            s->buttons |= bmap[btn->button];
         } else {
-            s->buttons &= ~bmap[evt->btn->button];
+            s->buttons &= ~bmap[btn->button];
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        if (btn->down && btn->button == INPUT_BUTTON_WHEEL_UP) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
                                     -1,
                                     s->buttons);
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+        if (btn->down && btn->button == INPUT_BUTTON_WHEEL_DOWN) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
@@ -173,10 +178,12 @@ static void legacy_mouse_event(DeviceState *dev, QemuConsole *src,
         }
         break;
     case INPUT_EVENT_KIND_ABS:
-        s->axis[evt->abs->axis] = evt->abs->value;
+        move = evt->u.abs.data;
+        s->axis[move->axis] = move->value;
         break;
     case INPUT_EVENT_KIND_REL:
-        s->axis[evt->rel->axis] += evt->rel->value;
+        move = evt->u.rel.data;
+        s->axis[move->axis] += move->value;
         break;
     default:
         break;
@@ -205,7 +212,7 @@ QEMUPutMouseEntry *qemu_add_mouse_event_handler(QEMUPutMouseEvent *func,
 {
     QEMUPutMouseEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutMouseEntry));
+    s = g_new0(QEMUPutMouseEntry, 1);
 
     s->qemu_put_mouse_event = func;
     s->qemu_put_mouse_event_opaque = opaque;
@@ -239,7 +246,7 @@ QEMUPutLEDEntry *qemu_add_led_event_handler(QEMUPutLEDEvent *func,
 {
     QEMUPutLEDEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutLEDEntry));
+    s = g_new0(QEMUPutLEDEntry, 1);
 
     s->put_led = func;
     s->opaque = opaque;
diff --git a/qemu/ui/input-linux.c b/qemu/ui/input-linux.c
new file mode 100644
index 000000000..1d33b5c12
--- /dev/null
+++ b/qemu/ui/input-linux.c
@@ -0,0 +1,507 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/config-file.h"
+#include "qemu/sockets.h"
+#include "sysemu/sysemu.h"
+#include "ui/input.h"
+#include "qom/object_interfaces.h"
+
+#include <sys/ioctl.h>
+#include "standard-headers/linux/input.h"
+
+static int linux_to_qcode[KEY_CNT] = {
+    [KEY_ESC]            = Q_KEY_CODE_ESC,
+    [KEY_1]              = Q_KEY_CODE_1,
+    [KEY_2]              = Q_KEY_CODE_2,
+    [KEY_3]              = Q_KEY_CODE_3,
+    [KEY_4]              = Q_KEY_CODE_4,
+    [KEY_5]              = Q_KEY_CODE_5,
+    [KEY_6]              = Q_KEY_CODE_6,
+    [KEY_7]              = Q_KEY_CODE_7,
+    [KEY_8]              = Q_KEY_CODE_8,
+    [KEY_9]              = Q_KEY_CODE_9,
+    [KEY_0]              = Q_KEY_CODE_0,
+    [KEY_MINUS]          = Q_KEY_CODE_MINUS,
+    [KEY_EQUAL]          = Q_KEY_CODE_EQUAL,
+    [KEY_BACKSPACE]      = Q_KEY_CODE_BACKSPACE,
+    [KEY_TAB]            = Q_KEY_CODE_TAB,
+    [KEY_Q]              = Q_KEY_CODE_Q,
+    [KEY_W]              = Q_KEY_CODE_W,
+    [KEY_E]              = Q_KEY_CODE_E,
+    [KEY_R]              = Q_KEY_CODE_R,
+    [KEY_T]              = Q_KEY_CODE_T,
+    [KEY_Y]              = Q_KEY_CODE_Y,
+    [KEY_U]              = Q_KEY_CODE_U,
+    [KEY_I]              = Q_KEY_CODE_I,
+    [KEY_O]              = Q_KEY_CODE_O,
+    [KEY_P]              = Q_KEY_CODE_P,
+    [KEY_LEFTBRACE]      = Q_KEY_CODE_BRACKET_LEFT,
+    [KEY_RIGHTBRACE]     = Q_KEY_CODE_BRACKET_RIGHT,
+    [KEY_ENTER]          = Q_KEY_CODE_RET,
+    [KEY_LEFTCTRL]       = Q_KEY_CODE_CTRL,
+    [KEY_A]              = Q_KEY_CODE_A,
+    [KEY_S]              = Q_KEY_CODE_S,
+    [KEY_D]              = Q_KEY_CODE_D,
+    [KEY_F]              = Q_KEY_CODE_F,
+    [KEY_G]              = Q_KEY_CODE_G,
+    [KEY_H]              = Q_KEY_CODE_H,
+    [KEY_J]              = Q_KEY_CODE_J,
+    [KEY_K]              = Q_KEY_CODE_K,
+    [KEY_L]              = Q_KEY_CODE_L,
+    [KEY_SEMICOLON]      = Q_KEY_CODE_SEMICOLON,
+    [KEY_APOSTROPHE]     = Q_KEY_CODE_APOSTROPHE,
+    [KEY_GRAVE]          = Q_KEY_CODE_GRAVE_ACCENT,
+    [KEY_LEFTSHIFT]      = Q_KEY_CODE_SHIFT,
+    [KEY_BACKSLASH]      = Q_KEY_CODE_BACKSLASH,
+    [KEY_102ND]          = Q_KEY_CODE_LESS,
+    [KEY_Z]              = Q_KEY_CODE_Z,
+    [KEY_X]              = Q_KEY_CODE_X,
+    [KEY_C]              = Q_KEY_CODE_C,
+    [KEY_V]              = Q_KEY_CODE_V,
+    [KEY_B]              = Q_KEY_CODE_B,
+    [KEY_N]              = Q_KEY_CODE_N,
+    [KEY_M]              = Q_KEY_CODE_M,
+    [KEY_COMMA]          = Q_KEY_CODE_COMMA,
+    [KEY_DOT]            = Q_KEY_CODE_DOT,
+    [KEY_SLASH]          = Q_KEY_CODE_SLASH,
+    [KEY_RIGHTSHIFT]     = Q_KEY_CODE_SHIFT_R,
+    [KEY_LEFTALT]        = Q_KEY_CODE_ALT,
+    [KEY_SPACE]          = Q_KEY_CODE_SPC,
+    [KEY_CAPSLOCK]       = Q_KEY_CODE_CAPS_LOCK,
+    [KEY_F1]             = Q_KEY_CODE_F1,
+    [KEY_F2]             = Q_KEY_CODE_F2,
+    [KEY_F3]             = Q_KEY_CODE_F3,
+    [KEY_F4]             = Q_KEY_CODE_F4,
+    [KEY_F5]             = Q_KEY_CODE_F5,
+    [KEY_F6]             = Q_KEY_CODE_F6,
+    [KEY_F7]             = Q_KEY_CODE_F7,
+    [KEY_F8]             = Q_KEY_CODE_F8,
+    [KEY_F9]             = Q_KEY_CODE_F9,
+    [KEY_F10]            = Q_KEY_CODE_F10,
+    [KEY_NUMLOCK]        = Q_KEY_CODE_NUM_LOCK,
+    [KEY_SCROLLLOCK]     = Q_KEY_CODE_SCROLL_LOCK,
+    [KEY_KP0]            = Q_KEY_CODE_KP_0,
+    [KEY_KP1]            = Q_KEY_CODE_KP_1,
+    [KEY_KP2]            = Q_KEY_CODE_KP_2,
+    [KEY_KP3]            = Q_KEY_CODE_KP_3,
+    [KEY_KP4]            = Q_KEY_CODE_KP_4,
+    [KEY_KP5]            = Q_KEY_CODE_KP_5,
+    [KEY_KP6]            = Q_KEY_CODE_KP_6,
+    [KEY_KP7]            = Q_KEY_CODE_KP_7,
+    [KEY_KP8]            = Q_KEY_CODE_KP_8,
+    [KEY_KP9]            = Q_KEY_CODE_KP_9,
+    [KEY_KPMINUS]        = Q_KEY_CODE_KP_SUBTRACT,
+    [KEY_KPPLUS]         = Q_KEY_CODE_KP_ADD,
+    [KEY_KPDOT]          = Q_KEY_CODE_KP_DECIMAL,
+    [KEY_KPENTER]        = Q_KEY_CODE_KP_ENTER,
+    [KEY_KPSLASH]        = Q_KEY_CODE_KP_DIVIDE,
+    [KEY_KPASTERISK]     = Q_KEY_CODE_KP_MULTIPLY,
+    [KEY_F11]            = Q_KEY_CODE_F11,
+    [KEY_F12]            = Q_KEY_CODE_F12,
+    [KEY_RIGHTCTRL]      = Q_KEY_CODE_CTRL_R,
+    [KEY_SYSRQ]          = Q_KEY_CODE_SYSRQ,
+    [KEY_RIGHTALT]       = Q_KEY_CODE_ALT_R,
+    [KEY_HOME]           = Q_KEY_CODE_HOME,
+    [KEY_UP]             = Q_KEY_CODE_UP,
+    [KEY_PAGEUP]         = Q_KEY_CODE_PGUP,
+    [KEY_LEFT]           = Q_KEY_CODE_LEFT,
+    [KEY_RIGHT]          = Q_KEY_CODE_RIGHT,
+    [KEY_END]            = Q_KEY_CODE_END,
+    [KEY_DOWN]           = Q_KEY_CODE_DOWN,
+    [KEY_PAGEDOWN]       = Q_KEY_CODE_PGDN,
+    [KEY_INSERT]         = Q_KEY_CODE_INSERT,
+    [KEY_DELETE]         = Q_KEY_CODE_DELETE,
+    [KEY_LEFTMETA]       = Q_KEY_CODE_META_L,
+    [KEY_RIGHTMETA]      = Q_KEY_CODE_META_R,
+    [KEY_MENU]           = Q_KEY_CODE_MENU,
+};
+
+static int qemu_input_linux_to_qcode(unsigned int lnx)
+{
+    assert(lnx < KEY_CNT);
+    return linux_to_qcode[lnx];
+}
+
+#define TYPE_INPUT_LINUX "input-linux"
+#define INPUT_LINUX(obj) \
+    OBJECT_CHECK(InputLinux, (obj), TYPE_INPUT_LINUX)
+#define INPUT_LINUX_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(InputLinuxClass, (obj), TYPE_INPUT_LINUX)
+#define INPUT_LINUX_CLASS(klass) \
+    OBJECT_CLASS_CHECK(InputLinuxClass, (klass), TYPE_INPUT_LINUX)
+
+typedef struct InputLinux InputLinux;
+typedef struct InputLinuxClass InputLinuxClass;
+
+struct InputLinux {
+    Object parent;
+
+    char        *evdev;
+    int         fd;
+    bool        repeat;
+    bool        grab_request;
+    bool        grab_active;
+    bool        grab_all;
+    bool        keydown[KEY_CNT];
+    int         keycount;
+    int         wheel;
+    bool        initialized;
+    QTAILQ_ENTRY(InputLinux) next;
+};
+
+struct InputLinuxClass {
+    ObjectClass parent_class;
+};
+
+static QTAILQ_HEAD(, InputLinux) inputs = QTAILQ_HEAD_INITIALIZER(inputs);
+
+static void input_linux_toggle_grab(InputLinux *il)
+{
+    intptr_t request = !il->grab_active;
+    InputLinux *item;
+    int rc;
+
+    rc = ioctl(il->fd, EVIOCGRAB, request);
+    if (rc < 0) {
+        return;
+    }
+    il->grab_active = !il->grab_active;
+
+    if (!il->grab_all) {
+        return;
+    }
+    QTAILQ_FOREACH(item, &inputs, next) {
+        if (item == il || item->grab_all) {
+            /* avoid endless loops */
+            continue;
+        }
+        if (item->grab_active != il->grab_active) {
+            input_linux_toggle_grab(item);
+        }
+    }
+}
+
+static void input_linux_event_keyboard(void *opaque)
+{
+    InputLinux *il = opaque;
+    struct input_event event;
+    int rc;
+
+    for (;;) {
+        rc = read(il->fd, &event, sizeof(event));
+        if (rc != sizeof(event)) {
+            if (rc < 0 && errno != EAGAIN) {
+                fprintf(stderr, "%s: read: %s\n", __func__, strerror(errno));
+                qemu_set_fd_handler(il->fd, NULL, NULL, NULL);
+                close(il->fd);
+            }
+            break;
+        }
+
+        switch (event.type) {
+        case EV_KEY:
+            if (event.value > 2 || (event.value > 1 && !il->repeat)) {
+                /*
+                 * ignore autorepeat + unknown key events
+                 * 0 == up, 1 == down, 2 == autorepeat, other == undefined
+                 */
+                continue;
+            }
+            if (event.code >= KEY_CNT) {
+                /*
+                 * Should not happen.  But better safe than sorry,
+                 * and we make Coverity happy too.
+                 */
+                continue;
+            }
+            /* keep track of key state */
+            if (!il->keydown[event.code] && event.value) {
+                il->keydown[event.code] = true;
+                il->keycount++;
+            }
+            if (il->keydown[event.code] && !event.value) {
+                il->keydown[event.code] = false;
+                il->keycount--;
+            }
+
+            /* send event to guest when grab is active */
+            if (il->grab_active) {
+                int qcode = qemu_input_linux_to_qcode(event.code);
+                qemu_input_event_send_key_qcode(NULL, qcode, event.value);
+            }
+
+            /* hotkey -> record switch request ... */
+            if (il->keydown[KEY_LEFTCTRL] &&
+                il->keydown[KEY_RIGHTCTRL]) {
+                il->grab_request = true;
+            }
+
+            /*
+             * ... and do the switch when all keys are lifted, so we
+             * confuse neither guest nor host with keys which seem to
+             * be stuck due to missing key-up events.
+             */
+            if (il->grab_request && !il->keycount) {
+                il->grab_request = false;
+                input_linux_toggle_grab(il);
+            }
+            break;
+        }
+    }
+}
+
+static void input_linux_event_mouse_button(int button)
+{
+    qemu_input_queue_btn(NULL, button, true);
+    qemu_input_event_sync();
+    qemu_input_queue_btn(NULL, button, false);
+    qemu_input_event_sync();
+}
+
+static void input_linux_event_mouse(void *opaque)
+{
+    InputLinux *il = opaque;
+    struct input_event event;
+    int rc;
+
+    for (;;) {
+        rc = read(il->fd, &event, sizeof(event));
+        if (rc != sizeof(event)) {
+            if (rc < 0 && errno != EAGAIN) {
+                fprintf(stderr, "%s: read: %s\n", __func__, strerror(errno));
+                qemu_set_fd_handler(il->fd, NULL, NULL, NULL);
+                close(il->fd);
+            }
+            break;
+        }
+
+        /* only send event to guest when grab is active */
+        if (!il->grab_active) {
+            continue;
+        }
+
+        switch (event.type) {
+        case EV_KEY:
+            switch (event.code) {
+            case BTN_LEFT:
+                qemu_input_queue_btn(NULL, INPUT_BUTTON_LEFT, event.value);
+                break;
+            case BTN_RIGHT:
+                qemu_input_queue_btn(NULL, INPUT_BUTTON_RIGHT, event.value);
+                break;
+            case BTN_MIDDLE:
+                qemu_input_queue_btn(NULL, INPUT_BUTTON_MIDDLE, event.value);
+                break;
+            case BTN_GEAR_UP:
+                qemu_input_queue_btn(NULL, INPUT_BUTTON_WHEEL_UP, event.value);
+                break;
+            case BTN_GEAR_DOWN:
+                qemu_input_queue_btn(NULL, INPUT_BUTTON_WHEEL_DOWN,
+                                     event.value);
+                break;
+            };
+            break;
+        case EV_REL:
+            switch (event.code) {
+            case REL_X:
+                qemu_input_queue_rel(NULL, INPUT_AXIS_X, event.value);
+                break;
+            case REL_Y:
+                qemu_input_queue_rel(NULL, INPUT_AXIS_Y, event.value);
+                break;
+            case REL_WHEEL:
+                il->wheel = event.value;
+                break;
+            }
+            break;
+        case EV_SYN:
+            qemu_input_event_sync();
+            if (il->wheel != 0) {
+                input_linux_event_mouse_button((il->wheel > 0)
+                                               ? INPUT_BUTTON_WHEEL_UP
+                                               : INPUT_BUTTON_WHEEL_DOWN);
+                il->wheel = 0;
+            }
+            break;
+        }
+    }
+}
+
+static void input_linux_complete(UserCreatable *uc, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(uc);
+    uint32_t evtmap, relmap, absmap;
+    int rc, ver;
+
+    if (!il->evdev) {
+        error_setg(errp, "no input device specified");
+        return;
+    }
+
+    il->fd = open(il->evdev, O_RDWR);
+    if (il->fd < 0)  {
+        error_setg_file_open(errp, errno, il->evdev);
+        return;
+    }
+    qemu_set_nonblock(il->fd);
+
+    rc = ioctl(il->fd, EVIOCGVERSION, &ver);
+    if (rc < 0) {
+        error_setg(errp, "%s: is not an evdev device", il->evdev);
+        goto err_close;
+    }
+
+    rc = ioctl(il->fd, EVIOCGBIT(0, sizeof(evtmap)), &evtmap);
+    if (rc < 0) {
+        error_setg(errp, "%s: failed to read event bits", il->evdev);
+        goto err_close;
+    }
+
+    if (evtmap & (1 << EV_REL)) {
+        rc = ioctl(il->fd, EVIOCGBIT(EV_REL, sizeof(relmap)), &relmap);
+        if (rc < 0) {
+            relmap = 0;
+        }
+    }
+
+    if (evtmap & (1 << EV_ABS)) {
+        ioctl(il->fd, EVIOCGBIT(EV_ABS, sizeof(absmap)), &absmap);
+        if (rc < 0) {
+            absmap = 0;
+        }
+    }
+
+    if ((evtmap & (1 << EV_REL)) &&
+        (relmap & (1 << REL_X))) {
+        /* has relative x axis -> assume mouse */
+        qemu_set_fd_handler(il->fd, input_linux_event_mouse, NULL, il);
+    } else if ((evtmap & (1 << EV_ABS)) &&
+               (absmap & (1 << ABS_X))) {
+        /* has absolute x axis -> not supported */
+        error_setg(errp, "tablet/touchscreen not supported");
+        goto err_close;
+    } else if (evtmap & (1 << EV_KEY)) {
+        /* has keys/buttons (and no x axis) -> assume keyboard */
+        qemu_set_fd_handler(il->fd, input_linux_event_keyboard, NULL, il);
+    } else {
+        /* Huh? What is this? */
+        error_setg(errp, "unknown kind of input device");
+        goto err_close;
+    }
+    input_linux_toggle_grab(il);
+    QTAILQ_INSERT_TAIL(&inputs, il, next);
+    il->initialized = true;
+    return;
+
+err_close:
+    close(il->fd);
+    return;
+}
+
+static void input_linux_instance_finalize(Object *obj)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    if (il->initialized) {
+        QTAILQ_REMOVE(&inputs, il, next);
+        close(il->fd);
+    }
+    g_free(il->evdev);
+}
+
+static char *input_linux_get_evdev(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return g_strdup(il->evdev);
+}
+
+static void input_linux_set_evdev(Object *obj, const char *value,
+                                  Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    if (il->evdev) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+    il->evdev = g_strdup(value);
+}
+
+static bool input_linux_get_grab_all(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return il->grab_all;
+}
+
+static void input_linux_set_grab_all(Object *obj, bool value,
+                                   Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    il->grab_all = value;
+}
+
+static bool input_linux_get_repeat(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return il->repeat;
+}
+
+static void input_linux_set_repeat(Object *obj, bool value,
+                                   Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    il->repeat = value;
+}
+
+static void input_linux_instance_init(Object *obj)
+{
+    object_property_add_str(obj, "evdev",
+                            input_linux_get_evdev,
+                            input_linux_set_evdev, NULL);
+    object_property_add_bool(obj, "grab_all",
+                             input_linux_get_grab_all,
+                             input_linux_set_grab_all, NULL);
+    object_property_add_bool(obj, "repeat",
+                             input_linux_get_repeat,
+                             input_linux_set_repeat, NULL);
+}
+
+static void input_linux_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = input_linux_complete;
+}
+
+static const TypeInfo input_linux_info = {
+    .name = TYPE_INPUT_LINUX,
+    .parent = TYPE_OBJECT,
+    .class_size = sizeof(InputLinuxClass),
+    .class_init = input_linux_class_init,
+    .instance_size = sizeof(InputLinux),
+    .instance_init = input_linux_instance_init,
+    .instance_finalize = input_linux_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&input_linux_info);
+}
+
+type_init(register_types);
diff --git a/qemu/ui/input.c b/qemu/ui/input.c
index 1a552d1de..ed88cda6d 100644
--- a/qemu/ui/input.c
+++ b/qemu/ui/input.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "sysemu/sysemu.h"
 #include "qapi-types.h"
@@ -6,6 +7,7 @@
 #include "trace.h"
 #include "ui/input.h"
 #include "ui/console.h"
+#include "sysemu/replay.h"
 
 struct QemuInputHandlerState {
     DeviceState       *dev;
@@ -80,19 +82,12 @@ void qemu_input_handler_bind(QemuInputHandlerState *s,
                              const char *device_id, int head,
                              Error **errp)
 {
-    DeviceState *dev;
     QemuConsole *con;
+    Error *err = NULL;
 
-    dev = qdev_find_recursive(sysbus_get_default(), device_id);
-    if (dev == NULL) {
-        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
-                  "Device '%s' not found", device_id);
-        return;
-    }
-
-    con = qemu_console_lookup_by_device(dev, head);
-    if (con == NULL) {
-        error_setg(errp, "Device %s is not bound to a QemuConsole", device_id);
+    con = qemu_console_lookup_by_device_name(device_id, head, &err);
+    if (err) {
+        error_propagate(errp, err);
         return;
     }
 
@@ -124,17 +119,22 @@ qemu_input_find_handler(uint32_t mask, QemuConsole *con)
     return NULL;
 }
 
-void qmp_x_input_send_event(bool has_console, int64_t console,
-                            InputEventList *events, Error **errp)
+void qmp_input_send_event(bool has_device, const char *device,
+                          bool has_head, int64_t head,
+                          InputEventList *events, Error **errp)
 {
     InputEventList *e;
     QemuConsole *con;
+    Error *err = NULL;
 
     con = NULL;
-    if (has_console) {
-        con = qemu_console_lookup_by_index(console);
-        if (!con) {
-            error_setg(errp, "console %" PRId64 " not found", console);
+    if (has_device) {
+        if (!has_head) {
+            head = 0;
+        }
+        con = qemu_console_lookup_by_device_name(device, head, &err);
+        if (err) {
+            error_propagate(errp, err);
             return;
         }
     }
@@ -147,10 +147,10 @@ void qmp_x_input_send_event(bool has_console, int64_t console,
     for (e = events; e != NULL; e = e->next) {
         InputEvent *event = e->value;
 
-        if (!qemu_input_find_handler(1 << event->kind, con)) {
+        if (!qemu_input_find_handler(1 << event->type, con)) {
             error_setg(errp, "Input handler not found for "
                              "event type %s",
-                            InputEventKind_lookup[event->kind]);
+                            InputEventKind_lookup[event->type]);
             return;
         }
     }
@@ -166,24 +166,25 @@ void qmp_x_input_send_event(bool has_console, int64_t console,
 
 static void qemu_input_transform_abs_rotate(InputEvent *evt)
 {
+    InputMoveEvent *move = evt->u.abs.data;
     switch (graphic_rotate) {
     case 90:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        if (move->axis == INPUT_AXIS_X) {
+            move->axis = INPUT_AXIS_Y;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            move->axis = INPUT_AXIS_X;
+            move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
         }
         break;
     case 180:
-        evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
         break;
     case 270:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
+        if (move->axis == INPUT_AXIS_X) {
+            move->axis = INPUT_AXIS_Y;
+            move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            move->axis = INPUT_AXIS_X;
         }
         break;
     }
@@ -193,41 +194,48 @@ static void qemu_input_event_trace(QemuConsole *src, InputEvent *evt)
 {
     const char *name;
     int qcode, idx = -1;
+    InputKeyEvent *key;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
 
     if (src) {
         idx = qemu_console_get_index(src);
     }
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_KEY:
-        switch (evt->key->key->kind) {
+        key = evt->u.key.data;
+        switch (key->key->type) {
         case KEY_VALUE_KIND_NUMBER:
-            qcode = qemu_input_key_number_to_qcode(evt->key->key->number);
+            qcode = qemu_input_key_number_to_qcode(key->key->u.number.data);
             name = QKeyCode_lookup[qcode];
-            trace_input_event_key_number(idx, evt->key->key->number,
-                                         name, evt->key->down);
+            trace_input_event_key_number(idx, key->key->u.number.data,
+                                         name, key->down);
             break;
         case KEY_VALUE_KIND_QCODE:
-            name = QKeyCode_lookup[evt->key->key->qcode];
-            trace_input_event_key_qcode(idx, name, evt->key->down);
+            name = QKeyCode_lookup[key->key->u.qcode.data];
+            trace_input_event_key_qcode(idx, name, key->down);
             break;
-        case KEY_VALUE_KIND_MAX:
+        case KEY_VALUE_KIND__MAX:
             /* keep gcc happy */
             break;
         }
         break;
     case INPUT_EVENT_KIND_BTN:
-        name = InputButton_lookup[evt->btn->button];
-        trace_input_event_btn(idx, name, evt->btn->down);
+        btn = evt->u.btn.data;
+        name = InputButton_lookup[btn->button];
+        trace_input_event_btn(idx, name, btn->down);
         break;
     case INPUT_EVENT_KIND_REL:
-        name = InputAxis_lookup[evt->rel->axis];
-        trace_input_event_rel(idx, name, evt->rel->value);
+        move = evt->u.rel.data;
+        name = InputAxis_lookup[move->axis];
+        trace_input_event_rel(idx, name, move->value);
         break;
     case INPUT_EVENT_KIND_ABS:
-        name = InputAxis_lookup[evt->abs->axis];
-        trace_input_event_abs(idx, name, evt->abs->value);
+        move = evt->u.abs.data;
+        name = InputAxis_lookup[move->axis];
+        trace_input_event_abs(idx, name, move->value);
         break;
-    case INPUT_EVENT_KIND_MAX:
+    case INPUT_EVENT_KIND__MAX:
         /* keep gcc happy */
         break;
     }
@@ -300,23 +308,19 @@ static void qemu_input_queue_sync(struct QemuInputEventQueueHead *queue)
     QTAILQ_INSERT_TAIL(queue, item, node);
 }
 
-void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt)
 {
     QemuInputHandlerState *s;
 
-    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
-        return;
-    }
-
     qemu_input_event_trace(src, evt);
 
     /* pre processing */
-    if (graphic_rotate && (evt->kind == INPUT_EVENT_KIND_ABS)) {
+    if (graphic_rotate && (evt->type == INPUT_EVENT_KIND_ABS)) {
             qemu_input_transform_abs_rotate(evt);
     }
 
     /* send event */
-    s = qemu_input_find_handler(1 << evt->kind, src);
+    s = qemu_input_find_handler(1 << evt->type, src);
     if (!s) {
         return;
     }
@@ -324,14 +328,19 @@ void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
     s->events++;
 }
 
-void qemu_input_event_sync(void)
+void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
 {
-    QemuInputHandlerState *s;
-
     if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
         return;
     }
 
+    replay_input_event(src, evt);
+}
+
+void qemu_input_event_sync_impl(void)
+{
+    QemuInputHandlerState *s;
+
     trace_input_event_sync();
 
     QTAILQ_FOREACH(s, &handlers, node) {
@@ -345,13 +354,22 @@ void qemu_input_event_sync(void)
     }
 }
 
+void qemu_input_event_sync(void)
+{
+    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
+        return;
+    }
+
+    replay_input_sync_event();
+}
+
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->key = g_new0(InputKeyEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_KEY;
-    evt->key->key = key;
-    evt->key->down = down;
+    evt->u.key.data = g_new0(InputKeyEvent, 1);
+    evt->type = INPUT_EVENT_KIND_KEY;
+    evt->u.key.data->key = key;
+    evt->u.key.data->down = down;
     return evt;
 }
 
@@ -372,16 +390,16 @@ void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down)
 void qemu_input_event_send_key_number(QemuConsole *src, int num, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_NUMBER;
-    key->number = num;
+    key->type = KEY_VALUE_KIND_NUMBER;
+    key->u.number.data = num;
     qemu_input_event_send_key(src, key, down);
 }
 
 void qemu_input_event_send_key_qcode(QemuConsole *src, QKeyCode q, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_QCODE;
-    key->qcode = q;
+    key->type = KEY_VALUE_KIND_QCODE;
+    key->u.qcode.data = q;
     qemu_input_event_send_key(src, key, down);
 }
 
@@ -398,10 +416,10 @@ void qemu_input_event_send_key_delay(uint32_t delay_ms)
 InputEvent *qemu_input_event_new_btn(InputButton btn, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->btn = g_new0(InputBtnEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_BTN;
-    evt->btn->button = btn;
-    evt->btn->down = down;
+    evt->u.btn.data = g_new0(InputBtnEvent, 1);
+    evt->type = INPUT_EVENT_KIND_BTN;
+    evt->u.btn.data->button = btn;
+    evt->u.btn.data->down = down;
     return evt;
 }
 
@@ -419,7 +437,7 @@ void qemu_input_update_buttons(QemuConsole *src, uint32_t *button_map,
     InputButton btn;
     uint32_t mask;
 
-    for (btn = 0; btn < INPUT_BUTTON_MAX; btn++) {
+    for (btn = 0; btn < INPUT_BUTTON__MAX; btn++) {
         mask = button_map[btn];
         if ((button_old & mask) == (button_new & mask)) {
             continue;
@@ -451,8 +469,8 @@ InputEvent *qemu_input_event_new_move(InputEventKind kind,
     InputEvent *evt = g_new0(InputEvent, 1);
     InputMoveEvent *move = g_new0(InputMoveEvent, 1);
 
-    evt->kind = kind;
-    evt->data = move;
+    evt->type = kind;
+    evt->u.rel.data = move; /* evt->u.rel is the same as evt->u.abs */
     move->axis = axis;
     move->value = value;
     return evt;
diff --git a/qemu/ui/keymaps.c b/qemu/ui/keymaps.c
index 49410ae9d..8899a0b31 100644
--- a/qemu/ui/keymaps.c
+++ b/qemu/ui/keymaps.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "keymaps.h"
 #include "sysemu/sysemu.h"
 
@@ -109,7 +110,7 @@ static kbd_layout_t *parse_keyboard_layout(const name2keysym_t *table,
     }
 
     if (!k) {
-        k = g_malloc0(sizeof(kbd_layout_t));
+        k = g_new0(kbd_layout_t, 1);
     }
 
     for(;;) {
diff --git a/qemu/ui/qemu-pixman.c b/qemu/ui/qemu-pixman.c
index 4116e1507..c9f8dce7f 100644
--- a/qemu/ui/qemu-pixman.c
+++ b/qemu/ui/qemu-pixman.c
@@ -3,6 +3,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 
diff --git a/qemu/ui/sdl.c b/qemu/ui/sdl.c
index 3be29101e..d8cf5bcf7 100644
--- a/qemu/ui/sdl.c
+++ b/qemu/ui/sdl.c
@@ -25,10 +25,12 @@
 /* Avoid compiler warning because macro is redefined in SDL_syswm.h. */
 #undef WIN32_LEAN_AND_MEAN
 
+#include "qemu/osdep.h"
 #include <SDL.h>
 #include <SDL_syswm.h>
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
@@ -60,6 +62,11 @@ static SDL_Cursor *guest_sprite = NULL;
 static SDL_PixelFormat host_format;
 static int scaling_active = 0;
 static Notifier mouse_mode_notifier;
+static int idle_counter;
+
+#define SDL_REFRESH_INTERVAL_BUSY 10
+#define SDL_MAX_IDLE_COUNT (2 * GUI_REFRESH_INTERVAL_DEFAULT \
+                            / SDL_REFRESH_INTERVAL_BUSY + 1)
 
 #if 0
 #define DEBUG_SDL
@@ -465,7 +472,7 @@ static void sdl_mouse_mode_change(Notifier *notify, void *data)
 
 static void sdl_send_mouse_event(int dx, int dy, int x, int y, int state)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = SDL_BUTTON(SDL_BUTTON_LEFT),
         [INPUT_BUTTON_MIDDLE]     = SDL_BUTTON(SDL_BUTTON_MIDDLE),
         [INPUT_BUTTON_RIGHT]      = SDL_BUTTON(SDL_BUTTON_RIGHT),
@@ -802,6 +809,7 @@ static void handle_activation(SDL_Event *ev)
 static void sdl_refresh(DisplayChangeListener *dcl)
 {
     SDL_Event ev1, *ev = &ev1;
+    int idle = 1;
 
     if (last_vm_running != runstate_is_running()) {
         last_vm_running = runstate_is_running();
@@ -817,9 +825,11 @@ static void sdl_refresh(DisplayChangeListener *dcl)
             sdl_update(dcl, 0, 0, real_screen->w, real_screen->h);
             break;
         case SDL_KEYDOWN:
+            idle = 0;
             handle_keydown(ev);
             break;
         case SDL_KEYUP:
+            idle = 0;
             handle_keyup(ev);
             break;
         case SDL_QUIT:
@@ -829,10 +839,12 @@ static void sdl_refresh(DisplayChangeListener *dcl)
             }
             break;
         case SDL_MOUSEMOTION:
+            idle = 0;
             handle_mousemotion(ev);
             break;
         case SDL_MOUSEBUTTONDOWN:
         case SDL_MOUSEBUTTONUP:
+            idle = 0;
             handle_mousebutton(ev);
             break;
         case SDL_ACTIVEEVENT:
@@ -847,6 +859,18 @@ static void sdl_refresh(DisplayChangeListener *dcl)
             break;
         }
     }
+
+    if (idle) {
+        if (idle_counter < SDL_MAX_IDLE_COUNT) {
+            idle_counter++;
+            if (idle_counter >= SDL_MAX_IDLE_COUNT) {
+                dcl->update_interval = GUI_REFRESH_INTERVAL_DEFAULT;
+            }
+        }
+    } else {
+        idle_counter = 0;
+        dcl->update_interval = SDL_REFRESH_INTERVAL_BUSY;
+    }
 }
 
 static void sdl_mouse_warp(DisplayChangeListener *dcl,
@@ -985,7 +1009,7 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
         sdl_grab_start();
     }
 
-    dcl = g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
diff --git a/qemu/ui/sdl2-2d.c b/qemu/ui/sdl2-2d.c
index d0b340f95..95930061e 100644
--- a/qemu/ui/sdl2-2d.c
+++ b/qemu/ui/sdl2-2d.c
@@ -23,6 +23,7 @@
  */
 /* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
@@ -45,10 +46,23 @@ void sdl2_2d_update(DisplayChangeListener *dcl,
         return;
     }
 
+    /*
+     * SDL2 seems to do some double-buffering, and trying to only
+     * update the changed areas results in only one of the two buffers
+     * being updated.  Which flickers alot.  So lets not try to be
+     * clever do a full update every time ...
+     */
+#if 0
     rect.x = x;
     rect.y = y;
     rect.w = w;
     rect.h = h;
+#else
+    rect.x = 0;
+    rect.y = 0;
+    rect.w = surface_width(surf);
+    rect.h = surface_height(surf);
+#endif
 
     SDL_UpdateTexture(scon->texture, NULL, surface_data(surf),
                       surface_stride(surf));
diff --git a/qemu/ui/sdl2-gl.c b/qemu/ui/sdl2-gl.c
index b604c0671..a324ecaca 100644
--- a/qemu/ui/sdl2-gl.c
+++ b/qemu/ui/sdl2-gl.c
@@ -25,17 +25,44 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
 #include "ui/sdl2.h"
 #include "sysemu/sysemu.h"
 
+#include <epoxy/gl.h>
+
+static void sdl2_set_scanout_mode(struct sdl2_console *scon, bool scanout)
+{
+    if (scon->scanout_mode == scanout) {
+        return;
+    }
+
+    scon->scanout_mode = scanout;
+    if (!scon->scanout_mode) {
+        if (scon->fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glDeleteFramebuffers(1, &scon->fbo_id);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            scon->fbo_id = 0;
+        }
+        if (scon->surface) {
+            surface_gl_destroy_texture(scon->gls, scon->surface);
+            surface_gl_create_texture(scon->gls, scon->surface);
+        }
+    }
+}
+
 static void sdl2_gl_render_surface(struct sdl2_console *scon)
 {
     int ww, wh;
 
     SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+    sdl2_set_scanout_mode(scon, false);
 
     SDL_GetWindowSize(scon->real_window, &ww, &wh);
     surface_gl_setup_viewport(scon->gls, scon->surface, ww, wh);
@@ -110,3 +137,112 @@ void sdl2_gl_redraw(struct sdl2_console *scon)
         sdl2_gl_render_surface(scon);
     }
 }
+
+QEMUGLContext sdl2_gl_create_context(DisplayChangeListener *dcl,
+                                     QEMUGLParams *params)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    SDL_GLContext ctx;
+
+    assert(scon->opengl);
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    SDL_GL_SetAttribute(SDL_GL_SHARE_WITH_CURRENT_CONTEXT, 1);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK,
+                        SDL_GL_CONTEXT_PROFILE_CORE);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, params->major_ver);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, params->minor_ver);
+
+    ctx = SDL_GL_CreateContext(scon->real_window);
+    return (QEMUGLContext)ctx;
+}
+
+void sdl2_gl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    SDL_GLContext sdlctx = (SDL_GLContext)ctx;
+
+    SDL_GL_DeleteContext(sdlctx);
+}
+
+int sdl2_gl_make_context_current(DisplayChangeListener *dcl,
+                                 QEMUGLContext ctx)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    SDL_GLContext sdlctx = (SDL_GLContext)ctx;
+
+    assert(scon->opengl);
+
+    return SDL_GL_MakeCurrent(scon->real_window, sdlctx);
+}
+
+QEMUGLContext sdl2_gl_get_current_context(DisplayChangeListener *dcl)
+{
+    SDL_GLContext sdlctx;
+
+    sdlctx = SDL_GL_GetCurrentContext();
+    return (QEMUGLContext)sdlctx;
+}
+
+void sdl2_gl_scanout(DisplayChangeListener *dcl,
+                     uint32_t backing_id, bool backing_y_0_top,
+                     uint32_t x, uint32_t y,
+                     uint32_t w, uint32_t h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+
+    assert(scon->opengl);
+    scon->x = x;
+    scon->y = y;
+    scon->w = w;
+    scon->h = h;
+    scon->tex_id = backing_id;
+    scon->y0_top = backing_y_0_top;
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    if (scon->tex_id == 0 || scon->w == 0 || scon->h == 0) {
+        sdl2_set_scanout_mode(scon, false);
+        return;
+    }
+
+    sdl2_set_scanout_mode(scon, true);
+    if (!scon->fbo_id) {
+        glGenFramebuffers(1, &scon->fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, scon->fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, scon->tex_id, 0);
+}
+
+void sdl2_gl_scanout_flush(DisplayChangeListener *dcl,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    int ww, wh, y1, y2;
+
+    assert(scon->opengl);
+    if (!scon->scanout_mode) {
+        return;
+    }
+    if (!scon->fbo_id) {
+        return;
+    }
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, scon->fbo_id);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    SDL_GetWindowSize(scon->real_window, &ww, &wh);
+    glViewport(0, 0, ww, wh);
+    y1 = scon->y0_top ? 0 : scon->h;
+    y2 = scon->y0_top ? scon->h : 0;
+    glBlitFramebuffer(0, y1, scon->w, y2,
+                      0, 0, ww, wh,
+                      GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, scon->fbo_id);
+
+    SDL_GL_SwapWindow(scon->real_window);
+}
diff --git a/qemu/ui/sdl2-input.c b/qemu/ui/sdl2-input.c
index ac5dc9476..6e315ae80 100644
--- a/qemu/ui/sdl2-input.c
+++ b/qemu/ui/sdl2-input.c
@@ -23,6 +23,7 @@
  */
 /* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
diff --git a/qemu/ui/sdl2.c b/qemu/ui/sdl2.c
index 5cb75aa36..d0424421e 100644
--- a/qemu/ui/sdl2.c
+++ b/qemu/ui/sdl2.c
@@ -23,6 +23,7 @@
  */
 /* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
@@ -49,6 +50,10 @@ static int guest_x, guest_y;
 static SDL_Cursor *guest_sprite;
 static Notifier mouse_mode_notifier;
 
+#define SDL2_REFRESH_INTERVAL_BUSY 10
+#define SDL2_MAX_IDLE_COUNT (2 * GUI_REFRESH_INTERVAL_DEFAULT \
+                             / SDL2_REFRESH_INTERVAL_BUSY + 1)
+
 static void sdl_update_caption(struct sdl2_console *scon);
 
 static struct sdl2_console *get_scon_from_window(uint32_t window_id)
@@ -256,7 +261,7 @@ static void sdl_mouse_mode_change(Notifier *notify, void *data)
 static void sdl_send_mouse_event(struct sdl2_console *scon, int dx, int dy,
                                  int x, int y, int state)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = SDL_BUTTON(SDL_BUTTON_LEFT),
         [INPUT_BUTTON_MIDDLE]     = SDL_BUTTON(SDL_BUTTON_MIDDLE),
         [INPUT_BUTTON_RIGHT]      = SDL_BUTTON(SDL_BUTTON_RIGHT),
@@ -578,6 +583,7 @@ static void handle_windowevent(SDL_Event *ev)
 void sdl2_poll_events(struct sdl2_console *scon)
 {
     SDL_Event ev1, *ev = &ev1;
+    int idle = 1;
 
     if (scon->last_vm_running != runstate_is_running()) {
         scon->last_vm_running = runstate_is_running();
@@ -587,12 +593,15 @@ void sdl2_poll_events(struct sdl2_console *scon)
     while (SDL_PollEvent(ev)) {
         switch (ev->type) {
         case SDL_KEYDOWN:
+            idle = 0;
             handle_keydown(ev);
             break;
         case SDL_KEYUP:
+            idle = 0;
             handle_keyup(ev);
             break;
         case SDL_TEXTINPUT:
+            idle = 0;
             handle_textinput(ev);
             break;
         case SDL_QUIT:
@@ -602,13 +611,16 @@ void sdl2_poll_events(struct sdl2_console *scon)
             }
             break;
         case SDL_MOUSEMOTION:
+            idle = 0;
             handle_mousemotion(ev);
             break;
         case SDL_MOUSEBUTTONDOWN:
         case SDL_MOUSEBUTTONUP:
+            idle = 0;
             handle_mousebutton(ev);
             break;
         case SDL_MOUSEWHEEL:
+            idle = 0;
             handle_mousewheel(ev);
             break;
         case SDL_WINDOWEVENT:
@@ -618,6 +630,18 @@ void sdl2_poll_events(struct sdl2_console *scon)
             break;
         }
     }
+
+    if (idle) {
+        if (scon->idle_counter < SDL2_MAX_IDLE_COUNT) {
+            scon->idle_counter++;
+            if (scon->idle_counter >= SDL2_MAX_IDLE_COUNT) {
+                scon->dcl.update_interval = GUI_REFRESH_INTERVAL_DEFAULT;
+            }
+        }
+    } else {
+        scon->idle_counter = 0;
+        scon->dcl.update_interval = SDL2_REFRESH_INTERVAL_BUSY;
+    }
 }
 
 static void sdl_mouse_warp(DisplayChangeListener *dcl,
@@ -700,6 +724,13 @@ static const DisplayChangeListenerOps dcl_gl_ops = {
     .dpy_refresh             = sdl2_gl_refresh,
     .dpy_mouse_set           = sdl_mouse_warp,
     .dpy_cursor_define       = sdl_mouse_define,
+
+    .dpy_gl_ctx_create       = sdl2_gl_create_context,
+    .dpy_gl_ctx_destroy      = sdl2_gl_destroy_context,
+    .dpy_gl_ctx_make_current = sdl2_gl_make_context_current,
+    .dpy_gl_ctx_get_current  = sdl2_gl_get_current_context,
+    .dpy_gl_scanout          = sdl2_gl_scanout,
+    .dpy_gl_update           = sdl2_gl_scanout_flush,
 };
 #endif
 
diff --git a/qemu/ui/sdl_zoom.c b/qemu/ui/sdl_zoom.c
index 2625c4557..72622c264 100644
--- a/qemu/ui/sdl_zoom.c
+++ b/qemu/ui/sdl_zoom.c
@@ -11,11 +11,9 @@
  *
  */
 
-#include "sdl_zoom.h"
 #include "qemu/osdep.h"
+#include "sdl_zoom.h"
 #include <glib.h>
-#include <stdint.h>
-#include <stdio.h>
 
 static void sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
                            SDL_Rect *dst_rect);
diff --git a/qemu/ui/shader.c b/qemu/ui/shader.c
index 52a463293..9264009b8 100644
--- a/qemu/ui/shader.c
+++ b/qemu/ui/shader.c
@@ -24,26 +24,48 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/shader.h"
 
 /* ---------------------------------------------------------------------- */
 
-void qemu_gl_run_texture_blit(GLint texture_blit_prog)
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog)
 {
-    GLfloat in_position[] = {
+    static const GLfloat in_position[] = {
         -1, -1,
         1,  -1,
         -1,  1,
         1,   1,
     };
     GLint l_position;
+    GLuint vao, buffer;
+
+    glGenVertexArrays(1, &vao);
+    glBindVertexArray(vao);
+
+    /* this is the VBO that holds the vertex data */
+    glGenBuffers(1, &buffer);
+    glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    glBufferData(GL_ARRAY_BUFFER, sizeof(in_position), in_position,
+                 GL_STATIC_DRAW);
 
-    glUseProgram(texture_blit_prog);
     l_position = glGetAttribLocation(texture_blit_prog, "in_position");
-    glVertexAttribPointer(l_position, 2, GL_FLOAT, GL_FALSE, 0, in_position);
+    glVertexAttribPointer(l_position, 2, GL_FLOAT, GL_FALSE, 0, 0);
     glEnableVertexAttribArray(l_position);
-    glDrawArrays(GL_TRIANGLE_STRIP, l_position, 4);
+
+    glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glBindVertexArray(0);
+
+    return vao;
+}
+
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao)
+{
+    glUseProgram(texture_blit_prog);
+    glBindVertexArray(texture_blit_vao);
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 }
 
 /* ---------------------------------------------------------------------- */
diff --git a/qemu/ui/spice-core.c b/qemu/ui/spice-core.c
index bf4fd0749..61db3c18b 100644
--- a/qemu/ui/spice-core.c
+++ b/qemu/ui/spice-core.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <spice.h>
 
 #include <netdb.h>
@@ -200,8 +201,6 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
 {
     SpiceServerInfo *server = g_malloc0(sizeof(*server));
     SpiceChannel *client = g_malloc0(sizeof(*client));
-    server->base = g_malloc0(sizeof(*server->base));
-    client->base = g_malloc0(sizeof(*client->base));
 
     /*
      * Spice server might have called us from spice worker thread
@@ -218,9 +217,11 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     }
 
     if (info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
-        add_addr_info(client->base, (struct sockaddr *)&info->paddr_ext,
+        add_addr_info(qapi_SpiceChannel_base(client),
+                      (struct sockaddr *)&info->paddr_ext,
                       info->plen_ext);
-        add_addr_info(server->base, (struct sockaddr *)&info->laddr_ext,
+        add_addr_info(qapi_SpiceServerInfo_base(server),
+                      (struct sockaddr *)&info->laddr_ext,
                       info->llen_ext);
     } else {
         error_report("spice: %s, extended address is expected",
@@ -229,7 +230,9 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
 
     switch (event) {
     case SPICE_CHANNEL_EVENT_CONNECTED:
-        qapi_event_send_spice_connected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_connected(qapi_SpiceServerInfo_base(server),
+                                        qapi_SpiceChannel_base(client),
+                                        &error_abort);
         break;
     case SPICE_CHANNEL_EVENT_INITIALIZED:
         if (auth) {
@@ -242,7 +245,9 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
         break;
     case SPICE_CHANNEL_EVENT_DISCONNECTED:
         channel_list_del(info);
-        qapi_event_send_spice_disconnected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_disconnected(qapi_SpiceServerInfo_base(server),
+                                           qapi_SpiceChannel_base(client),
+                                           &error_abort);
         break;
     default:
         break;
@@ -378,16 +383,15 @@ static SpiceChannelList *qmp_query_spice_channels(void)
 
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
-        chan->value->base = g_malloc0(sizeof(*chan->value->base));
 
         paddr = (struct sockaddr *)&item->info->paddr_ext;
         plen = item->info->plen_ext;
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
                     NI_NUMERICHOST | NI_NUMERICSERV);
-        chan->value->base->host = g_strdup(host);
-        chan->value->base->port = g_strdup(port);
-        chan->value->base->family = inet_netfamily(paddr->sa_family);
+        chan->value->host = g_strdup(host);
+        chan->value->port = g_strdup(port);
+        chan->value->family = inet_netfamily(paddr->sa_family);
 
         chan->value->connection_id = item->info->connection_id;
         chan->value->channel_type = item->info->type;
@@ -490,9 +494,14 @@ static QemuOptsList qemu_spice_opts = {
         },{
             .name = "playback-compression",
             .type = QEMU_OPT_BOOL,
-        }, {
+        },{
             .name = "seamless-migration",
             .type = QEMU_OPT_BOOL,
+#ifdef HAVE_SPICE_GL
+        },{
+            .name = "gl",
+            .type = QEMU_OPT_BOOL,
+#endif
         },
         { /* end of list */ }
     },
@@ -564,7 +573,8 @@ static void migration_state_notifier(Notifier *notifier, void *data)
 
     if (migration_in_setup(s)) {
         spice_server_migrate_start(spice_server);
-    } else if (migration_has_finished(s)) {
+    } else if (migration_has_finished(s) ||
+               migration_in_postcopy_after_devices(s)) {
         spice_server_migrate_end(spice_server, true);
         spice_have_target_host = false;
     } else if (migration_has_failed(s)) {
@@ -724,8 +734,7 @@ void qemu_spice_init(void)
         qemu_spice_set_passwd(password, false, false);
     }
     if (qemu_opt_get_bool(opts, "sasl", 0)) {
-        if (spice_server_set_sasl_appname(spice_server, "qemu") == -1 ||
-            spice_server_set_sasl(spice_server, 1) == -1) {
+        if (spice_server_set_sasl(spice_server, 1) == -1) {
             error_report("spice: failed to enable sasl");
             exit(1);
         }
@@ -791,6 +800,7 @@ void qemu_spice_init(void)
 
     seamless_migration = qemu_opt_get_bool(opts, "seamless-migration", 0);
     spice_server_set_seamless_migration(spice_server, seamless_migration);
+    spice_server_set_sasl_appname(spice_server, "qemu");
     if (spice_server_init(spice_server, &core_interface) != 0) {
         error_report("failed to initialize spice server");
         exit(1);
@@ -815,6 +825,19 @@ void qemu_spice_init(void)
 #if SPICE_SERVER_VERSION >= 0x000c02
     qemu_spice_register_ports();
 #endif
+
+#ifdef HAVE_SPICE_GL
+    if (qemu_opt_get_bool(opts, "gl", 0)) {
+        if ((port != 0) || (tls_port != 0)) {
+            error_report("SPICE GL support is local-only for now and "
+                         "incompatible with -spice port/tls-port");
+            exit(1);
+        }
+        if (egl_rendernode_init() == 0) {
+            display_opengl = 1;
+        }
+    }
+#endif
 }
 
 int qemu_spice_add_interface(SpiceBaseInstance *sin)
@@ -927,4 +950,4 @@ static void spice_register_config(void)
 {
     qemu_add_opts(&qemu_spice_opts);
 }
-machine_init(spice_register_config);
+opts_init(spice_register_config);
diff --git a/qemu/ui/spice-display.c b/qemu/ui/spice-display.c
index 0360abfd2..242ab5f46 100644
--- a/qemu/ui/spice-display.c
+++ b/qemu/ui/spice-display.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/qemu-spice.h"
 #include "qemu/timer.h"
@@ -408,7 +409,8 @@ void qemu_spice_display_switch(SimpleSpiceDisplay *ssd,
 
     if (surface && ssd->surface &&
         surface_width(surface) == pixman_image_get_width(ssd->surface) &&
-        surface_height(surface) == pixman_image_get_height(ssd->surface)) {
+        surface_height(surface) == pixman_image_get_height(ssd->surface) &&
+        surface_format(surface) == pixman_image_get_format(ssd->surface)) {
         /* no-resize fast path: just swap backing store */
         dprint(1, "%s/%d: fast (%dx%d)\n", __func__, ssd->qxl.id,
                surface_width(surface), surface_height(surface));
@@ -458,6 +460,13 @@ void qemu_spice_display_switch(SimpleSpiceDisplay *ssd,
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
     ssd->notify++;
+
+    qemu_mutex_lock(&ssd->lock);
+    if (ssd->cursor) {
+        g_free(ssd->ptr_define);
+        ssd->ptr_define = qemu_spice_create_cursor_update(ssd, ssd->cursor, 0);
+    }
+    qemu_mutex_unlock(&ssd->lock);
 }
 
 static void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
@@ -465,8 +474,6 @@ static void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
     if (ssd->cursor) {
         assert(ssd->dcl.con);
         dpy_cursor_define(ssd->dcl.con, ssd->cursor);
-        cursor_put(ssd->cursor);
-        ssd->cursor = NULL;
     }
     if (ssd->mouse_x != -1 && ssd->mouse_y != -1) {
         assert(ssd->dcl.con);
@@ -561,7 +568,7 @@ static int interface_get_command(QXLInstance *sin, QXLCommandExt *ext)
 
 static int interface_req_cmd_notification(QXLInstance *sin)
 {
-    dprint(1, "%s/%d:\n", __func__, sin->id);
+    dprint(2, "%s/%d:\n", __func__, sin->id);
     return 1;
 }
 
@@ -614,7 +621,7 @@ static int interface_get_cursor_command(QXLInstance *sin, QXLCommandExt *ext)
 
 static int interface_req_cursor_notification(QXLInstance *sin)
 {
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(2, "%s:\n", __func__);
     return 1;
 }
 
@@ -643,9 +650,23 @@ static void interface_update_area_complete(QXLInstance *sin,
 /* called from spice server thread context only */
 static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
 {
-    /* should never be called, used in qxl native mode only */
-    fprintf(stderr, "%s: abort()\n", __func__);
-    abort();
+    QXLCookie *cookie = (QXLCookie *)(uintptr_t)cookie_token;
+
+    switch (cookie->type) {
+#ifdef HAVE_SPICE_GL
+    case QXL_COOKIE_TYPE_GL_DRAW_DONE:
+    {
+        SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
+        qemu_bh_schedule(ssd->gl_unblock_bh);
+        break;
+    }
+#endif
+    default:
+        /* should never be called, used in qxl native mode only */
+        fprintf(stderr, "%s: abort()\n", __func__);
+        abort();
+    }
+    g_free(cookie);
 }
 
 static void interface_set_client_capabilities(QXLInstance *sin,
@@ -737,9 +758,7 @@ static void display_mouse_set(DisplayChangeListener *dcl,
     qemu_mutex_lock(&ssd->lock);
     ssd->ptr_x = x;
     ssd->ptr_y = y;
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-    }
+    g_free(ssd->ptr_move);
     ssd->ptr_move = qemu_spice_create_cursor_update(ssd, NULL, on);
     qemu_mutex_unlock(&ssd->lock);
 }
@@ -750,15 +769,16 @@ static void display_mouse_define(DisplayChangeListener *dcl,
     SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
 
     qemu_mutex_lock(&ssd->lock);
+    if (c) {
+        cursor_get(c);
+    }
+    cursor_put(ssd->cursor);
+    ssd->cursor = c;
     ssd->hot_x = c->hot_x;
     ssd->hot_y = c->hot_y;
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-        ssd->ptr_move = NULL;
-    }
-    if (ssd->ptr_define) {
-        g_free(ssd->ptr_define);
-    }
+    g_free(ssd->ptr_move);
+    ssd->ptr_move = NULL;
+    g_free(ssd->ptr_define);
     ssd->ptr_define = qemu_spice_create_cursor_update(ssd, c, 0);
     qemu_mutex_unlock(&ssd->lock);
 }
@@ -773,20 +793,128 @@ static const DisplayChangeListenerOps display_listener_ops = {
     .dpy_cursor_define    = display_mouse_define,
 };
 
+#ifdef HAVE_SPICE_GL
+
+static void qemu_spice_gl_block(SimpleSpiceDisplay *ssd, bool block)
+{
+    uint64_t timeout;
+
+    if (block) {
+        timeout = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        timeout += 1000; /* one sec */
+        timer_mod(ssd->gl_unblock_timer, timeout);
+    } else {
+        timer_del(ssd->gl_unblock_timer);
+    }
+    graphic_hw_gl_block(ssd->dcl.con, block);
+}
+
+static void qemu_spice_gl_unblock_bh(void *opaque)
+{
+    SimpleSpiceDisplay *ssd = opaque;
+
+    qemu_spice_gl_block(ssd, false);
+}
+
+static void qemu_spice_gl_block_timer(void *opaque)
+{
+    fprintf(stderr, "WARNING: spice: no gl-draw-done within one second\n");
+}
+
+static QEMUGLContext qemu_spice_gl_create_context(DisplayChangeListener *dcl,
+                                                  QEMUGLParams *params)
+{
+    eglMakeCurrent(qemu_egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE,
+                   qemu_egl_rn_ctx);
+    return qemu_egl_create_context(dcl, params);
+}
+
+static void qemu_spice_gl_scanout(DisplayChangeListener *dcl,
+                                  uint32_t tex_id,
+                                  bool y_0_top,
+                                  uint32_t x, uint32_t y,
+                                  uint32_t w, uint32_t h)
+{
+    SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
+    EGLint stride = 0, fourcc = 0;
+    int fd = -1;
+
+    if (tex_id) {
+        fd = egl_get_fd_for_texture(tex_id, &stride, &fourcc);
+        if (fd < 0) {
+            fprintf(stderr, "%s: failed to get fd for texture\n", __func__);
+            return;
+        }
+        dprint(1, "%s: %dx%d (stride %d, fourcc 0x%x)\n", __func__,
+               w, h, stride, fourcc);
+    } else {
+        dprint(1, "%s: no texture (no framebuffer)\n", __func__);
+    }
+
+    assert(!tex_id || fd >= 0);
+
+    /* note: spice server will close the fd */
+    spice_qxl_gl_scanout(&ssd->qxl, fd,
+                         surface_width(ssd->ds),
+                         surface_height(ssd->ds),
+                         stride, fourcc, y_0_top);
+}
+
+static void qemu_spice_gl_update(DisplayChangeListener *dcl,
+                                 uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
+    uint64_t cookie;
+
+    dprint(2, "%s: %dx%d+%d+%d\n", __func__, w, h, x, y);
+    qemu_spice_gl_block(ssd, true);
+    cookie = (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_GL_DRAW_DONE, 0);
+    spice_qxl_gl_draw_async(&ssd->qxl, x, y, w, h, cookie);
+}
+
+static const DisplayChangeListenerOps display_listener_gl_ops = {
+    .dpy_name             = "spice-egl",
+    .dpy_gfx_update       = display_update,
+    .dpy_gfx_switch       = display_switch,
+    .dpy_gfx_check_format = qemu_pixman_check_format,
+    .dpy_refresh          = display_refresh,
+    .dpy_mouse_set        = display_mouse_set,
+    .dpy_cursor_define    = display_mouse_define,
+
+    .dpy_gl_ctx_create       = qemu_spice_gl_create_context,
+    .dpy_gl_ctx_destroy      = qemu_egl_destroy_context,
+    .dpy_gl_ctx_make_current = qemu_egl_make_context_current,
+    .dpy_gl_ctx_get_current  = qemu_egl_get_current_context,
+
+    .dpy_gl_scanout          = qemu_spice_gl_scanout,
+    .dpy_gl_update           = qemu_spice_gl_update,
+};
+
+#endif /* HAVE_SPICE_GL */
+
 static void qemu_spice_display_init_one(QemuConsole *con)
 {
     SimpleSpiceDisplay *ssd = g_new0(SimpleSpiceDisplay, 1);
 
     qemu_spice_display_init_common(ssd);
 
+    ssd->dcl.ops = &display_listener_ops;
+#ifdef HAVE_SPICE_GL
+    if (display_opengl) {
+        ssd->dcl.ops = &display_listener_gl_ops;
+        ssd->dmabuf_fd = -1;
+        ssd->gl_unblock_bh = qemu_bh_new(qemu_spice_gl_unblock_bh, ssd);
+        ssd->gl_unblock_timer = timer_new_ms(QEMU_CLOCK_REALTIME,
+                                             qemu_spice_gl_block_timer, ssd);
+    }
+#endif
+    ssd->dcl.con = con;
+
     ssd->qxl.base.sif = &dpy_interface.base;
     qemu_spice_add_display_interface(&ssd->qxl, con);
     assert(ssd->worker);
-
     qemu_spice_create_host_memslot(ssd);
 
-    ssd->dcl.ops = &display_listener_ops;
-    ssd->dcl.con = con;
     register_displaychangelistener(&ssd->dcl);
 }
 
diff --git a/qemu/ui/spice-input.c b/qemu/ui/spice-input.c
index c342e0dcf..8eeebdbb2 100644
--- a/qemu/ui/spice-input.c
+++ b/qemu/ui/spice-input.c
@@ -15,10 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include <spice.h>
 #include <spice/enums.h>
@@ -107,7 +104,7 @@ typedef struct QemuSpicePointer {
 static void spice_update_buttons(QemuSpicePointer *pointer,
                                  int wheel, uint32_t button_mask)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]        = 0x01,
         [INPUT_BUTTON_MIDDLE]      = 0x04,
         [INPUT_BUTTON_RIGHT]       = 0x02,
diff --git a/qemu/ui/vnc-auth-sasl.c b/qemu/ui/vnc-auth-sasl.c
index 62a5fc4bf..5ae29c14c 100644
--- a/qemu/ui/vnc-auth-sasl.c
+++ b/qemu/ui/vnc-auth-sasl.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "vnc.h"
 
 /* Max amount of data we send/recv for SASL steps to prevent DOS */
@@ -62,7 +64,7 @@ long vnc_client_write_sasl(VncState *vs)
                           (const char **)&vs->sasl.encoded,
                           &vs->sasl.encodedLength);
         if (err != SASL_OK)
-            return vnc_client_io_error(vs, -1, EIO);
+            return vnc_client_io_error(vs, -1, NULL);
 
         vs->sasl.encodedOffset = 0;
     }
@@ -86,7 +88,11 @@ long vnc_client_write_sasl(VncState *vs)
      * SASL encoded output
      */
     if (vs->output.offset == 0) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     return ret;
@@ -110,7 +116,7 @@ long vnc_client_read_sasl(VncState *vs)
                       &decoded, &decodedLen);
 
     if (err != SASL_OK)
-        return vnc_client_io_error(vs, -1, -EIO);
+        return vnc_client_io_error(vs, -1, NULL);
     VNC_DEBUG("Read SASL Encoded %p size %ld Decoded %p size %d\n",
               encoded, ret, decoded, decodedLen);
     buffer_reserve(&vs->input, decodedLen);
@@ -255,17 +261,17 @@ static int protocol_client_auth_sasl_step(VncState *vs, uint8_t *data, size_t le
         vnc_read_when(vs, protocol_client_auth_sasl_step_len, 4);
     } else {
         if (!vnc_auth_sasl_check_ssf(vs)) {
-            VNC_DEBUG("Authentication rejected for weak SSF %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for weak SSF %p\n", vs->ioc);
             goto authreject;
         }
 
         /* Check username whitelist ACL */
         if (vnc_auth_sasl_check_access(vs) < 0) {
-            VNC_DEBUG("Authentication rejected for ACL %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for ACL %p\n", vs->ioc);
             goto authreject;
         }
 
-        VNC_DEBUG("Authentication successful %d\n", vs->csock);
+        VNC_DEBUG("Authentication successful %p\n", vs->ioc);
         vnc_write_u32(vs, 0); /* Accept auth */
         /*
          * Delay writing in SSF encoded mode until pending output
@@ -383,17 +389,17 @@ static int protocol_client_auth_sasl_start(VncState *vs, uint8_t *data, size_t l
         vnc_read_when(vs, protocol_client_auth_sasl_step_len, 4);
     } else {
         if (!vnc_auth_sasl_check_ssf(vs)) {
-            VNC_DEBUG("Authentication rejected for weak SSF %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for weak SSF %p\n", vs->ioc);
             goto authreject;
         }
 
         /* Check username whitelist ACL */
         if (vnc_auth_sasl_check_access(vs) < 0) {
-            VNC_DEBUG("Authentication rejected for ACL %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for ACL %p\n", vs->ioc);
             goto authreject;
         }
 
-        VNC_DEBUG("Authentication successful %d\n", vs->csock);
+        VNC_DEBUG("Authentication successful %p\n", vs->ioc);
         vnc_write_u32(vs, 0); /* Accept auth */
         start_client_init(vs);
     }
@@ -487,6 +493,33 @@ static int protocol_client_auth_sasl_mechname_len(VncState *vs, uint8_t *data, s
     return 0;
 }
 
+static char *
+vnc_socket_ip_addr_string(QIOChannelSocket *ioc,
+                          bool local,
+                          Error **errp)
+{
+    SocketAddress *addr;
+    char *ret;
+
+    if (local) {
+        addr = qio_channel_socket_get_local_address(ioc, errp);
+    } else {
+        addr = qio_channel_socket_get_remote_address(ioc, errp);
+    }
+    if (!addr) {
+        return NULL;
+    }
+
+    if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+        error_setg(errp, "Not an inet socket type");
+        return NULL;
+    }
+    ret = g_strdup_printf("%s;%s", addr->u.inet.data->host,
+                          addr->u.inet.data->port);
+    qapi_free_SocketAddress(addr);
+    return ret;
+}
+
 void start_auth_sasl(VncState *vs)
 {
     const char *mechlist = NULL;
@@ -495,13 +528,16 @@ void start_auth_sasl(VncState *vs)
     char *localAddr, *remoteAddr;
     int mechlistlen;
 
-    VNC_DEBUG("Initialize SASL auth %d\n", vs->csock);
+    VNC_DEBUG("Initialize SASL auth %p\n", vs->ioc);
 
     /* Get local & remote client addresses in form  IPADDR;PORT */
-    if (!(localAddr = vnc_socket_local_addr("%s;%s", vs->csock)))
+    localAddr = vnc_socket_ip_addr_string(vs->sioc, true, NULL);
+    if (!localAddr) {
         goto authabort;
+    }
 
-    if (!(remoteAddr = vnc_socket_remote_addr("%s;%s", vs->csock))) {
+    remoteAddr = vnc_socket_ip_addr_string(vs->sioc, false, NULL);
+    if (!remoteAddr) {
         g_free(localAddr);
         goto authabort;
     }
@@ -525,21 +561,24 @@ void start_auth_sasl(VncState *vs)
         goto authabort;
     }
 
-#ifdef CONFIG_VNC_TLS
     /* Inform SASL that we've got an external SSF layer from TLS/x509 */
     if (vs->auth == VNC_AUTH_VENCRYPT &&
         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL) {
-        gnutls_cipher_algorithm_t cipher;
+        Error *local_err = NULL;
+        int keysize;
         sasl_ssf_t ssf;
 
-        cipher = gnutls_cipher_get(vs->tls.session);
-        if (!(ssf = (sasl_ssf_t)gnutls_cipher_get_key_size(cipher))) {
-            VNC_DEBUG("%s", "cannot TLS get cipher size\n");
+        keysize = qcrypto_tls_session_get_key_size(vs->tls,
+                                                   &local_err);
+        if (keysize < 0) {
+            VNC_DEBUG("cannot TLS get cipher size: %s\n",
+                      error_get_pretty(local_err));
+            error_free(local_err);
             sasl_dispose(&vs->sasl.conn);
             vs->sasl.conn = NULL;
             goto authabort;
         }
-        ssf *= 8; /* tls key size is bytes, sasl wants bits */
+        ssf = keysize * CHAR_BIT; /* tls key size is bytes, sasl wants bits */
 
         err = sasl_setprop(vs->sasl.conn, SASL_SSF_EXTERNAL, &ssf);
         if (err != SASL_OK) {
@@ -549,20 +588,19 @@ void start_auth_sasl(VncState *vs)
             vs->sasl.conn = NULL;
             goto authabort;
         }
-    } else
-#endif /* CONFIG_VNC_TLS */
+    } else {
         vs->sasl.wantSSF = 1;
+    }
 
     memset (&secprops, 0, sizeof secprops);
-    /* Inform SASL that we've got an external SSF layer from TLS */
-    if (vs->vd->is_unix
-#ifdef CONFIG_VNC_TLS
-        /* Disable SSF, if using TLS+x509+SASL only. TLS without x509
-           is not sufficiently strong */
-        || (vs->auth == VNC_AUTH_VENCRYPT &&
-            vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-#endif /* CONFIG_VNC_TLS */
-        ) {
+    /* Inform SASL that we've got an external SSF layer from TLS.
+     *
+     * Disable SSF, if using TLS+x509+SASL only. TLS without x509
+     * is not sufficiently strong
+     */
+    if (vs->vd->is_unix ||
+        (vs->auth == VNC_AUTH_VENCRYPT &&
+         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)) {
         /* If we've got TLS or UNIX domain sock, we don't care about SSF */
         secprops.min_ssf = 0;
         secprops.max_ssf = 0;
diff --git a/qemu/ui/vnc-auth-vencrypt.c b/qemu/ui/vnc-auth-vencrypt.c
index 8fc965b4a..11c8c9a81 100644
--- a/qemu/ui/vnc-auth-vencrypt.c
+++ b/qemu/ui/vnc-auth-vencrypt.c
@@ -24,7 +24,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
+#include "qapi/error.h"
 #include "qemu/main-loop.h"
 
 static void start_auth_vencrypt_subauth(VncState *vs)
@@ -63,60 +65,23 @@ static void start_auth_vencrypt_subauth(VncState *vs)
     }
 }
 
-static void vnc_tls_handshake_io(void *opaque);
-
-static int vnc_start_vencrypt_handshake(VncState *vs)
+static void vnc_tls_handshake_done(Object *source,
+                                   Error *err,
+                                   gpointer user_data)
 {
-    int ret;
-
-    if ((ret = gnutls_handshake(vs->tls.session)) < 0) {
-       if (!gnutls_error_is_fatal(ret)) {
-           VNC_DEBUG("Handshake interrupted (blocking)\n");
-           if (!gnutls_record_get_direction(vs->tls.session))
-               qemu_set_fd_handler(vs->csock, vnc_tls_handshake_io, NULL, vs);
-           else
-               qemu_set_fd_handler(vs->csock, NULL, vnc_tls_handshake_io, vs);
-           return 0;
-       }
-       VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
-       vnc_client_error(vs);
-       return -1;
-    }
+    VncState *vs = user_data;
 
-    if (vs->vd->tls.x509verify) {
-        if (vnc_tls_validate_certificate(vs) < 0) {
-            VNC_DEBUG("Client verification failed\n");
-            vnc_client_error(vs);
-            return -1;
-        } else {
-            VNC_DEBUG("Client verification passed\n");
-        }
+    if (err) {
+        VNC_DEBUG("Handshake failed %s\n",
+                  error_get_pretty(err));
+        vnc_client_error(vs);
+    } else {
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
+        start_auth_vencrypt_subauth(vs);
     }
-
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    qemu_set_fd_handler(vs->csock, vnc_client_read, vnc_client_write, vs);
-
-    start_auth_vencrypt_subauth(vs);
-
-    return 0;
 }
 
-static void vnc_tls_handshake_io(void *opaque)
-{
-    VncState *vs = (VncState *)opaque;
-
-    VNC_DEBUG("Handshake IO continue\n");
-    vnc_start_vencrypt_handshake(vs);
-}
-
-
-
-#define NEED_X509_AUTH(vs)                              \
-    ((vs)->subauth == VNC_AUTH_VENCRYPT_X509NONE ||   \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509VNC ||    \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509PLAIN ||  \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-
 
 static int protocol_client_vencrypt_auth(VncState *vs, uint8_t *data, size_t len)
 {
@@ -128,20 +93,38 @@ static int protocol_client_vencrypt_auth(VncState *vs, uint8_t *data, size_t len
         vnc_flush(vs);
         vnc_client_error(vs);
     } else {
+        Error *err = NULL;
+        QIOChannelTLS *tls;
         VNC_DEBUG("Accepting auth %d, setting up TLS for handshake\n", auth);
         vnc_write_u8(vs, 1); /* Accept auth */
         vnc_flush(vs);
 
-        if (vnc_tls_client_setup(vs, NEED_X509_AUTH(vs)) < 0) {
-            VNC_DEBUG("Failed to setup TLS\n");
-            return 0;
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+            vs->ioc_tag = 0;
         }
 
-        VNC_DEBUG("Start TLS VeNCrypt handshake process\n");
-        if (vnc_start_vencrypt_handshake(vs) < 0) {
-            VNC_DEBUG("Failed to start TLS handshake\n");
+        tls = qio_channel_tls_new_server(
+            vs->ioc,
+            vs->vd->tlscreds,
+            vs->vd->tlsaclname,
+            &err);
+        if (!tls) {
+            VNC_DEBUG("Failed to setup TLS %s\n", error_get_pretty(err));
+            error_free(err);
+            vnc_client_error(vs);
             return 0;
         }
+
+        VNC_DEBUG("Start TLS VeNCrypt handshake process\n");
+        object_unref(OBJECT(vs->ioc));
+        vs->ioc = QIO_CHANNEL(tls);
+        vs->tls = qio_channel_tls_get_session(tls);
+
+        qio_channel_tls_handshake(tls,
+                                  vnc_tls_handshake_done,
+                                  vs,
+                                  NULL);
     }
     return 0;
 }
diff --git a/qemu/ui/vnc-enc-hextile.c b/qemu/ui/vnc-enc-hextile.c
index 2e768fd89..4215bd7da 100644
--- a/qemu/ui/vnc-enc-hextile.c
+++ b/qemu/ui/vnc-enc-hextile.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 
 static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
diff --git a/qemu/ui/vnc-enc-tight.c b/qemu/ui/vnc-enc-tight.c
index 9a9ddf2e3..e5cba0e5a 100644
--- a/qemu/ui/vnc-enc-tight.c
+++ b/qemu/ui/vnc-enc-tight.c
@@ -26,7 +26,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 /* This needs to be before jpeglib.h line because of conflict with
    INT32 definitions between jmorecfg.h (included by jpeglib.h) and
@@ -40,7 +40,6 @@
 #include <png.h>
 #endif
 #ifdef CONFIG_VNC_JPEG
-#include <stdio.h>
 #include <jpeglib.h>
 #endif
 
diff --git a/qemu/ui/vnc-enc-zlib.c b/qemu/ui/vnc-enc-zlib.c
index d1b97f251..33e9df2f6 100644
--- a/qemu/ui/vnc-enc-zlib.c
+++ b/qemu/ui/vnc-enc-zlib.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 
 #define ZALLOC_ALIGNMENT 16
diff --git a/qemu/ui/vnc-enc-zrle-template.c b/qemu/ui/vnc-enc-zrle-template.c
index 70ae624ee..abf6b86e4 100644
--- a/qemu/ui/vnc-enc-zrle-template.c
+++ b/qemu/ui/vnc-enc-zrle-template.c
@@ -22,7 +22,7 @@
  */
 
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #undef ZRLE_ENDIAN_SUFFIX
 
diff --git a/qemu/ui/vnc-enc-zrle.c b/qemu/ui/vnc-enc-zrle.c
index ed3b48465..5489870e7 100644
--- a/qemu/ui/vnc-enc-zrle.c
+++ b/qemu/ui/vnc-enc-zrle.c
@@ -26,6 +26,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-enc-zrle.h"
 
diff --git a/qemu/ui/vnc-enc-zywrle-template.c b/qemu/ui/vnc-enc-zywrle-template.c
index 561f7bfab..b446380a7 100644
--- a/qemu/ui/vnc-enc-zywrle-template.c
+++ b/qemu/ui/vnc-enc-zywrle-template.c
@@ -100,6 +100,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #define ZYWRLE_QUANTIZE
+#include "qemu/osdep.h"
 #include "vnc-enc-zywrle.h"
 
 #ifndef ZRLE_COMPACT_PIXEL
diff --git a/qemu/ui/vnc-jobs.c b/qemu/ui/vnc-jobs.c
index 22c9abce5..98ca978b0 100644
--- a/qemu/ui/vnc-jobs.c
+++ b/qemu/ui/vnc-jobs.c
@@ -26,9 +26,11 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-jobs.h"
 #include "qemu/sockets.h"
+#include "qemu/main-loop.h"
 #include "block/aio.h"
 
 /*
@@ -54,7 +56,6 @@ struct VncJobQueue {
     QemuCond cond;
     QemuMutex mutex;
     QemuThread thread;
-    Buffer buffer;
     bool exit;
     QTAILQ_HEAD(, VncJob) jobs;
 };
@@ -79,7 +80,7 @@ static void vnc_unlock_queue(VncJobQueue *queue)
 
 VncJob *vnc_job_new(VncState *vs)
 {
-    VncJob *job = g_malloc0(sizeof(VncJob));
+    VncJob *job = g_new0(VncJob, 1);
 
     job->vs = vs;
     vnc_lock_queue(queue);
@@ -90,7 +91,7 @@ VncJob *vnc_job_new(VncState *vs)
 
 int vnc_job_add_rect(VncJob *job, int x, int y, int w, int h)
 {
-    VncRectEntry *entry = g_malloc0(sizeof(VncRectEntry));
+    VncRectEntry *entry = g_new0(VncRectEntry, 1);
 
     entry->rect.x = x;
     entry->rect.y = y;
@@ -166,10 +167,16 @@ void vnc_jobs_consume_buffer(VncState *vs)
 
     vnc_lock_output(vs);
     if (vs->jobs_buffer.offset) {
-        vnc_write(vs, vs->jobs_buffer.buffer, vs->jobs_buffer.offset);
-        buffer_reset(&vs->jobs_buffer);
+        if (vs->ioc != NULL && buffer_empty(&vs->output)) {
+            if (vs->ioc_tag) {
+                g_source_remove(vs->ioc_tag);
+            }
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
+        }
+        buffer_move(&vs->output, &vs->jobs_buffer);
     }
-    flush = vs->csock != -1 && vs->abort != true;
+    flush = vs->ioc != NULL && vs->abort != true;
     vnc_unlock_output(vs);
 
     if (flush) {
@@ -182,6 +189,10 @@ void vnc_jobs_consume_buffer(VncState *vs)
  */
 static void vnc_async_encoding_start(VncState *orig, VncState *local)
 {
+    buffer_init(&local->output, "vnc-worker-output");
+    local->sioc = NULL; /* Don't do any network work on this thread */
+    local->ioc = NULL; /* Don't do any network work on this thread */
+
     local->vnc_encoding = orig->vnc_encoding;
     local->features = orig->features;
     local->vd = orig->vd;
@@ -193,10 +204,6 @@ static void vnc_async_encoding_start(VncState *orig, VncState *local)
     local->zlib = orig->zlib;
     local->hextile = orig->hextile;
     local->zrle = orig->zrle;
-    local->output =  queue->buffer;
-    local->csock = -1; /* Don't do any network work on this thread */
-
-    buffer_reset(&local->output);
 }
 
 static void vnc_async_encoding_end(VncState *orig, VncState *local)
@@ -206,15 +213,13 @@ static void vnc_async_encoding_end(VncState *orig, VncState *local)
     orig->hextile = local->hextile;
     orig->zrle = local->zrle;
     orig->lossy_rect = local->lossy_rect;
-
-    queue->buffer = local->output;
 }
 
 static int vnc_worker_thread_loop(VncJobQueue *queue)
 {
     VncJob *job;
     VncRectEntry *entry, *tmp;
-    VncState vs;
+    VncState vs = {};
     int n_rectangles;
     int saved_offset;
 
@@ -231,10 +236,18 @@ static int vnc_worker_thread_loop(VncJobQueue *queue)
     }
 
     vnc_lock_output(job->vs);
-    if (job->vs->csock == -1 || job->vs->abort == true) {
+    if (job->vs->ioc == NULL || job->vs->abort == true) {
         vnc_unlock_output(job->vs);
         goto disconnected;
     }
+    if (buffer_empty(&job->vs->output)) {
+        /*
+         * Looks like a NOP as it obviously moves no data.  But it
+         * moves the empty buffer, so we don't have to malloc a new
+         * one for vs.output
+         */
+        buffer_move_empty(&vs.output, &job->vs->output);
+    }
     vnc_unlock_output(job->vs);
 
     /* Make a local copy of vs and switch output buffers */
@@ -251,7 +264,7 @@ static int vnc_worker_thread_loop(VncJobQueue *queue)
     QLIST_FOREACH_SAFE(entry, &job->rectangles, next, tmp) {
         int n;
 
-        if (job->vs->csock == -1) {
+        if (job->vs->ioc == NULL) {
             vnc_unlock_display(job->vs->vd);
             /* Copy persistent encoding data */
             vnc_async_encoding_end(job->vs, &vs);
@@ -273,15 +286,14 @@ static int vnc_worker_thread_loop(VncJobQueue *queue)
     vs.output.buffer[saved_offset + 1] = n_rectangles & 0xFF;
 
     vnc_lock_output(job->vs);
-    if (job->vs->csock != -1) {
-        buffer_reserve(&job->vs->jobs_buffer, vs.output.offset);
-        buffer_append(&job->vs->jobs_buffer, vs.output.buffer,
-                      vs.output.offset);
+    if (job->vs->ioc != NULL) {
+        buffer_move(&job->vs->jobs_buffer, &vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
 
 	qemu_bh_schedule(job->vs->bh);
     }  else {
+        buffer_reset(&vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
     }
@@ -298,7 +310,7 @@ disconnected:
 
 static VncJobQueue *vnc_queue_init(void)
 {
-    VncJobQueue *queue = g_malloc0(sizeof(VncJobQueue));
+    VncJobQueue *queue = g_new0(VncJobQueue, 1);
 
     qemu_cond_init(&queue->cond);
     qemu_mutex_init(&queue->mutex);
@@ -310,7 +322,6 @@ static void vnc_queue_clear(VncJobQueue *q)
 {
     qemu_cond_destroy(&queue->cond);
     qemu_mutex_destroy(&queue->mutex);
-    buffer_free(&queue->buffer);
     g_free(q);
     queue = NULL; /* Unset global queue */
 }
diff --git a/qemu/ui/vnc-palette.c b/qemu/ui/vnc-palette.c
index c130deee9..3b89d1af2 100644
--- a/qemu/ui/vnc-palette.c
+++ b/qemu/ui/vnc-palette.c
@@ -26,9 +26,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc-palette.h"
 #include <glib.h>
-#include <string.h>
 
 static VncPaletteEntry *palette_find(const VncPalette *palette,
                                      uint32_t color, unsigned int hash)
diff --git a/qemu/ui/vnc-palette.h b/qemu/ui/vnc-palette.h
index d02f0236c..1bd4318f5 100644
--- a/qemu/ui/vnc-palette.h
+++ b/qemu/ui/vnc-palette.h
@@ -31,8 +31,6 @@
 
 #include "qapi/qmp/qlist.h"
 #include "qemu/queue.h"
-#include <stdint.h>
-#include <stdbool.h>
 
 #define VNC_PALETTE_HASH_SIZE 256
 #define VNC_PALETTE_MAX_SIZE  256
diff --git a/qemu/ui/vnc-tls.c b/qemu/ui/vnc-tls.c
deleted file mode 100644
index 028fc4db1..000000000
--- a/qemu/ui/vnc-tls.c
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * QEMU VNC display driver: TLS helpers
- *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-x509.h"
-#include "vnc.h"
-#include "qemu/sockets.h"
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-/* Very verbose, so only enabled for _VNC_DEBUG >= 2 */
-static void vnc_debug_gnutls_log(int level, const char* str) {
-    VNC_DEBUG("%d %s", level, str);
-}
-#endif /* defined(_VNC_DEBUG) && _VNC_DEBUG >= 2 */
-
-
-#define DH_BITS 1024
-static gnutls_dh_params_t dh_params;
-
-static int vnc_tls_initialize(void)
-{
-    static int tlsinitialized = 0;
-
-    if (tlsinitialized)
-        return 1;
-
-    if (gnutls_global_init () < 0)
-        return 0;
-
-    /* XXX ought to re-generate diffie-hellman params periodically */
-    if (gnutls_dh_params_init (&dh_params) < 0)
-        return 0;
-    if (gnutls_dh_params_generate2 (dh_params, DH_BITS) < 0)
-        return 0;
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-    gnutls_global_set_log_level(10);
-    gnutls_global_set_log_function(vnc_debug_gnutls_log);
-#endif
-
-    tlsinitialized = 1;
-
-    return 1;
-}
-
-static ssize_t vnc_tls_push(gnutls_transport_ptr_t transport,
-                            const void *data,
-                            size_t len) {
-    VncState *vs = (VncState *)transport;
-    int ret;
-
- retry:
-    ret = send(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static ssize_t vnc_tls_pull(gnutls_transport_ptr_t transport,
-                            void *data,
-                            size_t len) {
-    VncState *vs = (VncState *)transport;
-    int ret;
-
- retry:
-    ret = qemu_recv(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static gnutls_anon_server_credentials_t vnc_tls_initialize_anon_cred(void)
-{
-    gnutls_anon_server_credentials_t anon_cred;
-    int ret;
-
-    if ((ret = gnutls_anon_allocate_server_credentials(&anon_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-
-    gnutls_anon_set_server_dh_params(anon_cred, dh_params);
-
-    return anon_cred;
-}
-
-
-static gnutls_certificate_credentials_t vnc_tls_initialize_x509_cred(VncDisplay *vd)
-{
-    gnutls_certificate_credentials_t x509_cred;
-    int ret;
-
-    if (!vd->tls.x509cacert) {
-        VNC_DEBUG("No CA x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509cert) {
-        VNC_DEBUG("No server x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509key) {
-        VNC_DEBUG("No server private key specified\n");
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_allocate_credentials(&x509_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-    if ((ret = gnutls_certificate_set_x509_trust_file(x509_cred,
-                                                      vd->tls.x509cacert,
-                                                      GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load CA certificate %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_set_x509_key_file (x509_cred,
-                                                     vd->tls.x509cert,
-                                                     vd->tls.x509key,
-                                                     GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load certificate & key %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if (vd->tls.x509cacrl) {
-        if ((ret = gnutls_certificate_set_x509_crl_file(x509_cred,
-                                                        vd->tls.x509cacrl,
-                                                        GNUTLS_X509_FMT_PEM)) < 0) {
-            VNC_DEBUG("Cannot load CRL %s\n", gnutls_strerror(ret));
-            gnutls_certificate_free_credentials(x509_cred);
-            return NULL;
-        }
-    }
-
-    gnutls_certificate_set_dh_params (x509_cred, dh_params);
-
-    return x509_cred;
-}
-
-
-int vnc_tls_validate_certificate(VncState *vs)
-{
-    int ret;
-    unsigned int status;
-    const gnutls_datum_t *certs;
-    unsigned int nCerts, i;
-    time_t now;
-
-    VNC_DEBUG("Validating client certificate\n");
-    if ((ret = gnutls_certificate_verify_peers2 (vs->tls.session, &status)) < 0) {
-        VNC_DEBUG("Verify failed %s\n", gnutls_strerror(ret));
-        return -1;
-    }
-
-    if ((now = time(NULL)) == ((time_t)-1)) {
-        return -1;
-    }
-
-    if (status != 0) {
-        if (status & GNUTLS_CERT_INVALID)
-            VNC_DEBUG("The certificate is not trusted.\n");
-
-        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
-            VNC_DEBUG("The certificate hasn't got a known issuer.\n");
-
-        if (status & GNUTLS_CERT_REVOKED)
-            VNC_DEBUG("The certificate has been revoked.\n");
-
-        if (status & GNUTLS_CERT_INSECURE_ALGORITHM)
-            VNC_DEBUG("The certificate uses an insecure algorithm\n");
-
-        return -1;
-    } else {
-        VNC_DEBUG("Certificate is valid!\n");
-    }
-
-    /* Only support x509 for now */
-    if (gnutls_certificate_type_get(vs->tls.session) != GNUTLS_CRT_X509)
-        return -1;
-
-    if (!(certs = gnutls_certificate_get_peers(vs->tls.session, &nCerts)))
-        return -1;
-
-    for (i = 0 ; i < nCerts ; i++) {
-        gnutls_x509_crt_t cert;
-        VNC_DEBUG ("Checking certificate chain %d\n", i);
-        if (gnutls_x509_crt_init (&cert) < 0)
-            return -1;
-
-        if (gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER) < 0) {
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_expiration_time (cert) < now) {
-            VNC_DEBUG("The certificate has expired\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (i == 0) {
-            size_t dnameSize = 1024;
-            vs->tls.dname = g_malloc(dnameSize);
-        requery:
-            if ((ret = gnutls_x509_crt_get_dn (cert, vs->tls.dname, &dnameSize)) != 0) {
-                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
-                    vs->tls.dname = g_realloc(vs->tls.dname, dnameSize);
-                    goto requery;
-                }
-                gnutls_x509_crt_deinit (cert);
-                VNC_DEBUG("Cannot get client distinguished name: %s",
-                          gnutls_strerror (ret));
-                return -1;
-            }
-
-            if (vs->vd->tls.x509verify) {
-                int allow;
-                if (!vs->vd->tls.acl) {
-                    VNC_DEBUG("no ACL activated, allowing access");
-                    gnutls_x509_crt_deinit (cert);
-                    continue;
-                }
-
-                allow = qemu_acl_party_is_allowed(vs->vd->tls.acl,
-                                                  vs->tls.dname);
-
-                VNC_DEBUG("TLS x509 ACL check for %s is %s\n",
-                          vs->tls.dname, allow ? "allowed" : "denied");
-                if (!allow) {
-                    gnutls_x509_crt_deinit (cert);
-                    return -1;
-                }
-            }
-        }
-
-        gnutls_x509_crt_deinit (cert);
-    }
-
-    return 0;
-}
-
-#if defined(GNUTLS_VERSION_NUMBER) && \
-    GNUTLS_VERSION_NUMBER >= 0x020200 /* 2.2.0 */
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    const char *priority = x509 ? "NORMAL" : "NORMAL:+ANON-DH";
-    int rc;
-
-    rc = gnutls_priority_set_direct(s, priority, NULL);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#else
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    static const int cert_types[] = { GNUTLS_CRT_X509, 0 };
-    static const int protocols[] = {
-        GNUTLS_TLS1_1, GNUTLS_TLS1_0, GNUTLS_SSL3, 0
-    };
-    static const int kx_anon[] = { GNUTLS_KX_ANON_DH, 0 };
-    static const int kx_x509[] = {
-        GNUTLS_KX_DHE_DSS, GNUTLS_KX_RSA,
-        GNUTLS_KX_DHE_RSA, GNUTLS_KX_SRP, 0
-    };
-    int rc;
-
-    rc = gnutls_kx_set_priority(s, x509 ? kx_x509 : kx_anon);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_certificate_type_set_priority(s, cert_types);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_protocol_set_priority(s, protocols);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#endif
-
-int vnc_tls_client_setup(VncState *vs,
-                         int needX509Creds) {
-    VNC_DEBUG("Do TLS setup\n");
-    if (vnc_tls_initialize() < 0) {
-        VNC_DEBUG("Failed to init TLS\n");
-        vnc_client_error(vs);
-        return -1;
-    }
-    if (vs->tls.session == NULL) {
-        if (gnutls_init(&vs->tls.session, GNUTLS_SERVER) < 0) {
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (gnutls_set_default_priority(vs->tls.session) < 0) {
-            gnutls_deinit(vs->tls.session);
-            vs->tls.session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (vnc_set_gnutls_priority(vs->tls.session, needX509Creds) < 0) {
-            gnutls_deinit(vs->tls.session);
-            vs->tls.session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (needX509Creds) {
-            gnutls_certificate_server_credentials x509_cred =
-                vnc_tls_initialize_x509_cred(vs->vd);
-            if (!x509_cred) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(vs->tls.session,
-                                       GNUTLS_CRD_CERTIFICATE, x509_cred) < 0) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                gnutls_certificate_free_credentials(x509_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (vs->vd->tls.x509verify) {
-                VNC_DEBUG("Requesting a client certificate\n");
-                gnutls_certificate_server_set_request(vs->tls.session,
-                                                      GNUTLS_CERT_REQUEST);
-            }
-
-        } else {
-            gnutls_anon_server_credentials_t anon_cred =
-                vnc_tls_initialize_anon_cred();
-            if (!anon_cred) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(vs->tls.session,
-                                       GNUTLS_CRD_ANON, anon_cred) < 0) {
-                gnutls_deinit(vs->tls.session);
-                vs->tls.session = NULL;
-                gnutls_anon_free_server_credentials(anon_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-        }
-
-        gnutls_transport_set_ptr(vs->tls.session, (gnutls_transport_ptr_t)vs);
-        gnutls_transport_set_push_function(vs->tls.session, vnc_tls_push);
-        gnutls_transport_set_pull_function(vs->tls.session, vnc_tls_pull);
-    }
-    return 0;
-}
-
-
-void vnc_tls_client_cleanup(VncState *vs)
-{
-    if (vs->tls.session) {
-        gnutls_deinit(vs->tls.session);
-        vs->tls.session = NULL;
-    }
-    g_free(vs->tls.dname);
-}
-
-
-
-static int vnc_set_x509_credential(VncDisplay *vd,
-                                   const char *certdir,
-                                   const char *filename,
-                                   char **cred,
-                                   int ignoreMissing)
-{
-    struct stat sb;
-
-    g_free(*cred);
-    *cred = g_malloc(strlen(certdir) + strlen(filename) + 2);
-
-    strcpy(*cred, certdir);
-    strcat(*cred, "/");
-    strcat(*cred, filename);
-
-    VNC_DEBUG("Check %s\n", *cred);
-    if (stat(*cred, &sb) < 0) {
-        g_free(*cred);
-        *cred = NULL;
-        if (ignoreMissing && errno == ENOENT)
-            return 0;
-        return -1;
-    }
-
-    return 0;
-}
-
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-                               const char *certdir)
-{
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CERT_FILE, &vd->tls.x509cacert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CRL_FILE, &vd->tls.x509cacrl, 1) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_CERT_FILE, &vd->tls.x509cert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_KEY_FILE, &vd->tls.x509key, 0) < 0)
-        goto cleanup;
-
-    return 0;
-
- cleanup:
-    g_free(vd->tls.x509cacert);
-    g_free(vd->tls.x509cacrl);
-    g_free(vd->tls.x509cert);
-    g_free(vd->tls.x509key);
-    vd->tls.x509cacert = vd->tls.x509cacrl = vd->tls.x509cert = vd->tls.x509key = NULL;
-    return -1;
-}
-
diff --git a/qemu/ui/vnc-ws.c b/qemu/ui/vnc-ws.c
index b4cb6bde7..7c79a4c37 100644
--- a/qemu/ui/vnc-ws.c
+++ b/qemu/ui/vnc-ws.c
@@ -18,364 +18,108 @@
  * along with this software; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "vnc.h"
-#include "qemu/main-loop.h"
-#include "crypto/hash.h"
+#include "io/channel-websock.h"
 
-#ifdef CONFIG_VNC_TLS
-#include "qemu/sockets.h"
-
-static int vncws_start_tls_handshake(VncState *vs)
+static void vncws_tls_handshake_done(Object *source,
+                                     Error *err,
+                                     gpointer user_data)
 {
-    int ret = gnutls_handshake(vs->tls.session);
+    VncState *vs = user_data;
 
-    if (ret < 0) {
-        if (!gnutls_error_is_fatal(ret)) {
-            VNC_DEBUG("Handshake interrupted (blocking)\n");
-            if (!gnutls_record_get_direction(vs->tls.session)) {
-                qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io,
-                                    NULL, vs);
-            } else {
-                qemu_set_fd_handler(vs->csock, NULL, vncws_tls_handshake_io,
-                                    vs);
-            }
-            return 0;
-        }
-        VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
+    if (err) {
+        VNC_DEBUG("Handshake failed %s\n", error_get_pretty(err));
         vnc_client_error(vs);
-        return -1;
-    }
-
-    if (vs->vd->tls.x509verify) {
-        if (vnc_tls_validate_certificate(vs) < 0) {
-            VNC_DEBUG("Client verification failed\n");
-            vnc_client_error(vs);
-            return -1;
-        } else {
-            VNC_DEBUG("Client verification passed\n");
-        }
+    } else {
+        VNC_DEBUG("TLS handshake complete, starting websocket handshake\n");
+        vs->ioc_tag = qio_channel_add_watch(
+            QIO_CHANNEL(vs->ioc), G_IO_IN, vncws_handshake_io, vs, NULL);
     }
-
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    qemu_set_fd_handler(vs->csock, vncws_handshake_read, NULL, vs);
-
-    return 0;
 }
 
-void vncws_tls_handshake_io(void *opaque)
-{
-    VncState *vs = (VncState *)opaque;
 
-    if (!vs->tls.session) {
-        VNC_DEBUG("TLS Websocket setup\n");
-        if (vnc_tls_client_setup(vs, vs->vd->tls.x509cert != NULL) < 0) {
-            return;
-        }
-    }
-    VNC_DEBUG("Handshake IO continue\n");
-    vncws_start_tls_handshake(vs);
-}
-#endif /* CONFIG_VNC_TLS */
-
-void vncws_handshake_read(void *opaque)
+gboolean vncws_tls_handshake_io(QIOChannel *ioc G_GNUC_UNUSED,
+                                GIOCondition condition G_GNUC_UNUSED,
+                                void *opaque)
 {
     VncState *vs = opaque;
-    uint8_t *handshake_end;
-    long ret;
-    /* Typical HTTP headers from novnc are 512 bytes, so limiting
-     * total header size to 4096 is easily enough. */
-    size_t want = 4096 - vs->ws_input.offset;
-    buffer_reserve(&vs->ws_input, want);
-    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), want);
+    QIOChannelTLS *tls;
+    Error *err = NULL;
 
-    if (!ret) {
-        if (vs->csock == -1) {
-            vnc_disconnect_finish(vs);
-        }
-        return;
+    VNC_DEBUG("TLS Websocket connection required\n");
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+        vs->ioc_tag = 0;
     }
-    vs->ws_input.offset += ret;
 
-    handshake_end = (uint8_t *)g_strstr_len((char *)vs->ws_input.buffer,
-            vs->ws_input.offset, WS_HANDSHAKE_END);
-    if (handshake_end) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
-        vncws_process_handshake(vs, vs->ws_input.buffer, vs->ws_input.offset);
-        buffer_advance(&vs->ws_input, handshake_end - vs->ws_input.buffer +
-                strlen(WS_HANDSHAKE_END));
-    } else if (vs->ws_input.offset >= 4096) {
-        VNC_DEBUG("End of headers not found in first 4096 bytes\n");
+    tls = qio_channel_tls_new_server(
+        vs->ioc,
+        vs->vd->tlscreds,
+        vs->vd->tlsaclname,
+        &err);
+    if (!tls) {
+        VNC_DEBUG("Failed to setup TLS %s\n", error_get_pretty(err));
+        error_free(err);
         vnc_client_error(vs);
+        return TRUE;
     }
-}
-
-
-long vnc_client_read_ws(VncState *vs)
-{
-    int ret, err;
-    uint8_t *payload;
-    size_t payload_size, header_size;
-    VNC_DEBUG("Read websocket %p size %zd offset %zd\n", vs->ws_input.buffer,
-            vs->ws_input.capacity, vs->ws_input.offset);
-    buffer_reserve(&vs->ws_input, 4096);
-    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
-    if (!ret) {
-        return 0;
-    }
-    vs->ws_input.offset += ret;
 
-    ret = 0;
-    /* consume as much of ws_input buffer as possible */
-    do {
-        if (vs->ws_payload_remain == 0) {
-            err = vncws_decode_frame_header(&vs->ws_input,
-                                            &header_size,
-                                            &vs->ws_payload_remain,
-                                            &vs->ws_payload_mask);
-            if (err <= 0) {
-                return err;
-            }
+    VNC_DEBUG("Start TLS WS handshake process\n");
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = QIO_CHANNEL(tls);
+    vs->tls = qio_channel_tls_get_session(tls);
 
-            buffer_advance(&vs->ws_input, header_size);
-        }
-        if (vs->ws_payload_remain != 0) {
-            err = vncws_decode_frame_payload(&vs->ws_input,
-                                             &vs->ws_payload_remain,
-                                             &vs->ws_payload_mask,
-                                             &payload,
-                                             &payload_size);
-            if (err < 0) {
-                return err;
-            }
-            if (err == 0) {
-                return ret;
-            }
-            ret += err;
+    qio_channel_tls_handshake(tls,
+                              vncws_tls_handshake_done,
+                              vs,
+                              NULL);
 
-            buffer_reserve(&vs->input, payload_size);
-            buffer_append(&vs->input, payload, payload_size);
-
-            buffer_advance(&vs->ws_input, payload_size);
-        }
-    } while (vs->ws_input.offset > 0);
-
-    return ret;
+    return TRUE;
 }
 
-long vnc_client_write_ws(VncState *vs)
-{
-    long ret;
-    VNC_DEBUG("Write WS: Pending output %p size %zd offset %zd\n",
-              vs->output.buffer, vs->output.capacity, vs->output.offset);
-    vncws_encode_frame(&vs->ws_output, vs->output.buffer, vs->output.offset);
-    buffer_reset(&vs->output);
-    ret = vnc_client_write_buf(vs, vs->ws_output.buffer, vs->ws_output.offset);
-    if (!ret) {
-        return 0;
-    }
-
-    buffer_advance(&vs->ws_output, ret);
-
-    if (vs->ws_output.offset == 0) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
-    }
-
-    return ret;
-}
 
-static char *vncws_extract_handshake_entry(const char *handshake,
-        size_t handshake_len, const char *name)
+static void vncws_handshake_done(Object *source,
+                                 Error *err,
+                                 gpointer user_data)
 {
-    char *begin, *end, *ret = NULL;
-    char *line = g_strdup_printf("%s%s: ", WS_HANDSHAKE_DELIM, name);
-    begin = g_strstr_len(handshake, handshake_len, line);
-    if (begin != NULL) {
-        begin += strlen(line);
-        end = g_strstr_len(begin, handshake_len - (begin - handshake),
-                WS_HANDSHAKE_DELIM);
-        if (end != NULL) {
-            ret = g_strndup(begin, end - begin);
-        }
-    }
-    g_free(line);
-    return ret;
-}
-
-static void vncws_send_handshake_response(VncState *vs, const char* key)
-{
-    char combined_key[WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1];
-    char *accept = NULL, *response = NULL;
-    Error *err = NULL;
-
-    g_strlcpy(combined_key, key, WS_CLIENT_KEY_LEN + 1);
-    g_strlcat(combined_key, WS_GUID, WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1);
+    VncState *vs = user_data;
 
-    /* hash and encode it */
-    if (qcrypto_hash_base64(QCRYPTO_HASH_ALG_SHA1,
-                            combined_key,
-                            WS_CLIENT_KEY_LEN + WS_GUID_LEN,
-                            &accept,
-                            &err) < 0) {
-        VNC_DEBUG("Hashing Websocket combined key failed %s\n",
-                  error_get_pretty(err));
-        error_free(err);
+    if (err) {
+        VNC_DEBUG("Websock handshake failed %s\n", error_get_pretty(err));
         vnc_client_error(vs);
-        return;
-    }
-
-    response = g_strdup_printf(WS_HANDSHAKE, accept);
-    vnc_client_write_buf(vs, (const uint8_t *)response, strlen(response));
-
-    g_free(accept);
-    g_free(response);
-
-    vs->encode_ws = 1;
-    vnc_init_state(vs);
-}
-
-void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size)
-{
-    char *protocols = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Protocol");
-    char *version = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Version");
-    char *key = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Key");
-
-    if (protocols && version && key
-            && g_strrstr(protocols, "binary")
-            && !strcmp(version, WS_SUPPORTED_VERSION)
-            && strlen(key) == WS_CLIENT_KEY_LEN) {
-        vncws_send_handshake_response(vs, key);
     } else {
-        VNC_DEBUG("Defective Websockets header or unsupported protocol\n");
-        vnc_client_error(vs);
+        VNC_DEBUG("Websock handshake complete, starting VNC protocol\n");
+        vnc_init_state(vs);
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
-
-    g_free(protocols);
-    g_free(version);
-    g_free(key);
 }
 
-void vncws_encode_frame(Buffer *output, const void *payload,
-        const size_t payload_size)
-{
-    size_t header_size = 0;
-    unsigned char opcode = WS_OPCODE_BINARY_FRAME;
-    union {
-        char buf[WS_HEAD_MAX_LEN];
-        WsHeader ws;
-    } header;
-
-    if (!payload_size) {
-        return;
-    }
-
-    header.ws.b0 = 0x80 | (opcode & 0x0f);
-    if (payload_size <= 125) {
-        header.ws.b1 = (uint8_t)payload_size;
-        header_size = 2;
-    } else if (payload_size < 65536) {
-        header.ws.b1 = 0x7e;
-        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)payload_size);
-        header_size = 4;
-    } else {
-        header.ws.b1 = 0x7f;
-        header.ws.u.s64.l64 = cpu_to_be64(payload_size);
-        header_size = 10;
-    }
-
-    buffer_reserve(output, header_size + payload_size);
-    buffer_append(output, header.buf, header_size);
-    buffer_append(output, payload, payload_size);
-}
 
-int vncws_decode_frame_header(Buffer *input,
-                              size_t *header_size,
-                              size_t *payload_remain,
-                              WsMask *payload_mask)
+gboolean vncws_handshake_io(QIOChannel *ioc G_GNUC_UNUSED,
+                            GIOCondition condition G_GNUC_UNUSED,
+                            void *opaque)
 {
-    unsigned char opcode = 0, fin = 0, has_mask = 0;
-    size_t payload_len;
-    WsHeader *header = (WsHeader *)input->buffer;
-
-    if (input->offset < WS_HEAD_MIN_LEN + 4) {
-        /* header not complete */
-        return 0;
-    }
-
-    fin = (header->b0 & 0x80) >> 7;
-    opcode = header->b0 & 0x0f;
-    has_mask = (header->b1 & 0x80) >> 7;
-    payload_len = header->b1 & 0x7f;
-
-    if (opcode == WS_OPCODE_CLOSE) {
-        /* disconnect */
-        return -1;
-    }
+    VncState *vs = opaque;
+    QIOChannelWebsock *wioc;
 
-    /* Websocket frame sanity check:
-     * * Websocket fragmentation is not supported.
-     * * All  websockets frames sent by a client have to be masked.
-     * * Only binary encoding is supported.
-     */
-    if (!fin || !has_mask || opcode != WS_OPCODE_BINARY_FRAME) {
-        VNC_DEBUG("Received faulty/unsupported Websocket frame\n");
-        return -2;
+    VNC_DEBUG("Websocket negotiate starting\n");
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+        vs->ioc_tag = 0;
     }
 
-    if (payload_len < 126) {
-        *payload_remain = payload_len;
-        *header_size = 6;
-        *payload_mask = header->u.m;
-    } else if (payload_len == 126 && input->offset >= 8) {
-        *payload_remain = be16_to_cpu(header->u.s16.l16);
-        *header_size = 8;
-        *payload_mask = header->u.s16.m16;
-    } else if (payload_len == 127 && input->offset >= 14) {
-        *payload_remain = be64_to_cpu(header->u.s64.l64);
-        *header_size = 14;
-        *payload_mask = header->u.s64.m64;
-    } else {
-        /* header not complete */
-        return 0;
-    }
+    wioc = qio_channel_websock_new_server(vs->ioc);
 
-    return 1;
-}
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = QIO_CHANNEL(wioc);
 
-int vncws_decode_frame_payload(Buffer *input,
-                               size_t *payload_remain, WsMask *payload_mask,
-                               uint8_t **payload, size_t *payload_size)
-{
-    size_t i;
-    uint32_t *payload32;
-
-    *payload = input->buffer;
-    /* If we aren't at the end of the payload, then drop
-     * off the last bytes, so we're always multiple of 4
-     * for purpose of unmasking, except at end of payload
-     */
-    if (input->offset < *payload_remain) {
-        *payload_size = input->offset - (input->offset % 4);
-    } else {
-        *payload_size = *payload_remain;
-    }
-    if (*payload_size == 0) {
-        return 0;
-    }
-    *payload_remain -= *payload_size;
-
-    /* unmask frame */
-    /* process 1 frame (32 bit op) */
-    payload32 = (uint32_t *)(*payload);
-    for (i = 0; i < *payload_size / 4; i++) {
-        payload32[i] ^= payload_mask->u;
-    }
-    /* process the remaining bytes (if any) */
-    for (i *= 4; i < *payload_size; i++) {
-        (*payload)[i] ^= payload_mask->c[i % 4];
-    }
+    qio_channel_websock_handshake(wioc,
+                                  vncws_handshake_done,
+                                  vs,
+                                  NULL);
 
-    return 1;
+    return TRUE;
 }
diff --git a/qemu/ui/vnc-ws.h b/qemu/ui/vnc-ws.h
index 94942258e..652b6fc39 100644
--- a/qemu/ui/vnc-ws.h
+++ b/qemu/ui/vnc-ws.h
@@ -21,72 +21,11 @@
 #ifndef __QEMU_UI_VNC_WS_H
 #define __QEMU_UI_VNC_WS_H
 
-#define B64LEN(__x) (((__x + 2) / 3) * 12 / 3)
-#define SHA1_DIGEST_LEN 20
-
-#define WS_ACCEPT_LEN (B64LEN(SHA1_DIGEST_LEN) + 1)
-#define WS_CLIENT_KEY_LEN 24
-#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
-#define WS_GUID_LEN strlen(WS_GUID)
-
-#define WS_HANDSHAKE "HTTP/1.1 101 Switching Protocols\r\n\
-Upgrade: websocket\r\n\
-Connection: Upgrade\r\n\
-Sec-WebSocket-Accept: %s\r\n\
-Sec-WebSocket-Protocol: binary\r\n\
-\r\n"
-#define WS_HANDSHAKE_DELIM "\r\n"
-#define WS_HANDSHAKE_END "\r\n\r\n"
-#define WS_SUPPORTED_VERSION "13"
-
-#define WS_HEAD_MIN_LEN sizeof(uint16_t)
-#define WS_HEAD_MAX_LEN (WS_HEAD_MIN_LEN + sizeof(uint64_t) + sizeof(uint32_t))
-
-typedef union WsMask {
-    char c[4];
-    uint32_t u;
-} WsMask;
-
-typedef struct QEMU_PACKED WsHeader {
-    unsigned char b0;
-    unsigned char b1;
-    union {
-        struct QEMU_PACKED {
-            uint16_t l16;
-            WsMask m16;
-        } s16;
-        struct QEMU_PACKED {
-            uint64_t l64;
-            WsMask m64;
-        } s64;
-        WsMask m;
-    } u;
-} WsHeader;
-
-enum {
-    WS_OPCODE_CONTINUATION = 0x0,
-    WS_OPCODE_TEXT_FRAME = 0x1,
-    WS_OPCODE_BINARY_FRAME = 0x2,
-    WS_OPCODE_CLOSE = 0x8,
-    WS_OPCODE_PING = 0x9,
-    WS_OPCODE_PONG = 0xA
-};
-
-#ifdef CONFIG_VNC_TLS
-void vncws_tls_handshake_io(void *opaque);
-#endif /* CONFIG_VNC_TLS */
-void vncws_handshake_read(void *opaque);
-long vnc_client_write_ws(VncState *vs);
-long vnc_client_read_ws(VncState *vs);
-void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size);
-void vncws_encode_frame(Buffer *output, const void *payload,
-            const size_t payload_size);
-int vncws_decode_frame_header(Buffer *input,
-                              size_t *header_size,
-                              size_t *payload_remain,
-                              WsMask *payload_mask);
-int vncws_decode_frame_payload(Buffer *input,
-                               size_t *payload_remain, WsMask *payload_mask,
-                               uint8_t **payload, size_t *payload_size);
+gboolean vncws_tls_handshake_io(QIOChannel *ioc,
+                                GIOCondition condition,
+                                void *opaque);
+gboolean vncws_handshake_io(QIOChannel *ioc,
+                            GIOCondition condition,
+                            void *opaque);
 
 #endif /* __QEMU_UI_VNC_WS_H */
diff --git a/qemu/ui/vnc.c b/qemu/ui/vnc.c
index caf82f56f..d2ebf1fb7 100644
--- a/qemu/ui/vnc.c
+++ b/qemu/ui/vnc.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-jobs.h"
 #include "trace.h"
@@ -37,10 +38,13 @@
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/types.h"
 #include "qmp-commands.h"
-#include "qemu/osdep.h"
 #include "ui/input.h"
 #include "qapi-event.h"
 #include "crypto/hash.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qom/object_interfaces.h"
+#include "qemu/cutils.h"
 
 #define VNC_REFRESH_INTERVAL_BASE GUI_REFRESH_INTERVAL_DEFAULT
 #define VNC_REFRESH_INTERVAL_INC  50
@@ -67,8 +71,8 @@ static void vnc_set_share_mode(VncState *vs, VncShareMode mode)
         [VNC_SHARE_MODE_EXCLUSIVE]    = "exclusive",
         [VNC_SHARE_MODE_DISCONNECTED] = "disconnected",
     };
-    fprintf(stderr, "%s/%d: %s -> %s\n", __func__,
-            vs->csock, mn[vs->share_mode], mn[mode]);
+    fprintf(stderr, "%s/%p: %s -> %s\n", __func__,
+            vs->ioc, mn[vs->share_mode], mn[mode]);
 #endif
 
     switch (vs->share_mode) {
@@ -102,105 +106,65 @@ static void vnc_set_share_mode(VncState *vs, VncShareMode mode)
     }
 }
 
-static char *addr_to_string(const char *format,
-                            struct sockaddr_storage *sa,
-                            socklen_t salen) {
-    char *addr;
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
-    int err;
-    size_t addrlen;
-
-    if ((err = getnameinfo((struct sockaddr *)sa, salen,
-                           host, sizeof(host),
-                           serv, sizeof(serv),
-                           NI_NUMERICHOST | NI_NUMERICSERV)) != 0) {
-        VNC_DEBUG("Cannot resolve address %d: %s\n",
-                  err, gai_strerror(err));
-        return NULL;
-    }
-
-    /* Enough for the existing format + the 2 vars we're
-     * substituting in. */
-    addrlen = strlen(format) + strlen(host) + strlen(serv);
-    addr = g_malloc(addrlen + 1);
-    snprintf(addr, addrlen, format, host, serv);
-    addr[addrlen] = '\0';
 
-    return addr;
-}
-
-
-char *vnc_socket_local_addr(const char *format, int fd) {
-    struct sockaddr_storage sa;
-    socklen_t salen;
-
-    salen = sizeof(sa);
-    if (getsockname(fd, (struct sockaddr*)&sa, &salen) < 0)
-        return NULL;
-
-    return addr_to_string(format, &sa, salen);
-}
-
-char *vnc_socket_remote_addr(const char *format, int fd) {
-    struct sockaddr_storage sa;
-    socklen_t salen;
-
-    salen = sizeof(sa);
-    if (getpeername(fd, (struct sockaddr*)&sa, &salen) < 0)
-        return NULL;
+static void vnc_init_basic_info(SocketAddress *addr,
+                                VncBasicInfo *info,
+                                Error **errp)
+{
+    switch (addr->type) {
+    case SOCKET_ADDRESS_KIND_INET:
+        info->host = g_strdup(addr->u.inet.data->host);
+        info->service = g_strdup(addr->u.inet.data->port);
+        if (addr->u.inet.data->ipv6) {
+            info->family = NETWORK_ADDRESS_FAMILY_IPV6;
+        } else {
+            info->family = NETWORK_ADDRESS_FAMILY_IPV4;
+        }
+        break;
 
-    return addr_to_string(format, &sa, salen);
-}
+    case SOCKET_ADDRESS_KIND_UNIX:
+        info->host = g_strdup("");
+        info->service = g_strdup(addr->u.q_unix.data->path);
+        info->family = NETWORK_ADDRESS_FAMILY_UNIX;
+        break;
 
-static VncBasicInfo *vnc_basic_info_get(struct sockaddr_storage *sa,
-                                        socklen_t salen)
-{
-    VncBasicInfo *info;
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
-    int err;
-
-    if ((err = getnameinfo((struct sockaddr *)sa, salen,
-                           host, sizeof(host),
-                           serv, sizeof(serv),
-                           NI_NUMERICHOST | NI_NUMERICSERV)) != 0) {
-        VNC_DEBUG("Cannot resolve address %d: %s\n",
-                  err, gai_strerror(err));
-        return NULL;
+    default:
+        error_setg(errp, "Unsupported socket kind %d",
+                   addr->type);
+        break;
     }
 
-    info = g_malloc0(sizeof(VncBasicInfo));
-    info->host = g_strdup(host);
-    info->service = g_strdup(serv);
-    info->family = inet_netfamily(sa->ss_family);
-    return info;
+    return;
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_server_addr(int fd)
+static void vnc_init_basic_info_from_server_addr(QIOChannelSocket *ioc,
+                                                 VncBasicInfo *info,
+                                                 Error **errp)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen;
+    SocketAddress *addr = NULL;
 
-    salen = sizeof(sa);
-    if (getsockname(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+    addr = qio_channel_socket_get_local_address(ioc, errp);
+    if (!addr) {
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    vnc_init_basic_info(addr, info, errp);
+    qapi_free_SocketAddress(addr);
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_remote_addr(int fd)
+static void vnc_init_basic_info_from_remote_addr(QIOChannelSocket *ioc,
+                                                 VncBasicInfo *info,
+                                                 Error **errp)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen;
+    SocketAddress *addr = NULL;
 
-    salen = sizeof(sa);
-    if (getpeername(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+    addr = qio_channel_socket_get_remote_address(ioc, errp);
+    if (!addr) {
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    vnc_init_basic_info(addr, info, errp);
+    qapi_free_SocketAddress(addr);
 }
 
 static const char *vnc_auth_name(VncDisplay *vd) {
@@ -222,7 +186,6 @@ static const char *vnc_auth_name(VncDisplay *vd) {
     case VNC_AUTH_TLS:
         return "tls";
     case VNC_AUTH_VENCRYPT:
-#ifdef CONFIG_VNC_TLS
         switch (vd->subauth) {
         case VNC_AUTH_VENCRYPT_PLAIN:
             return "vencrypt+plain";
@@ -245,9 +208,6 @@ static const char *vnc_auth_name(VncDisplay *vd) {
         default:
             return "vencrypt";
         }
-#else
-        return "vencrypt";
-#endif
     case VNC_AUTH_SASL:
         return "sasl";
     }
@@ -257,15 +217,18 @@ static const char *vnc_auth_name(VncDisplay *vd) {
 static VncServerInfo *vnc_server_info_get(VncDisplay *vd)
 {
     VncServerInfo *info;
-    VncBasicInfo *bi = vnc_basic_info_get_from_server_addr(vd->lsock);
-    if (!bi) {
-        return NULL;
-    }
+    Error *err = NULL;
 
     info = g_malloc(sizeof(*info));
-    info->base = bi;
+    vnc_init_basic_info_from_server_addr(vd->lsock,
+                                         qapi_VncServerInfo_base(info), &err);
     info->has_auth = true;
     info->auth = g_strdup(vnc_auth_name(vd));
+    if (err) {
+        qapi_free_VncServerInfo(info);
+        info = NULL;
+        error_free(err);
+    }
     return info;
 }
 
@@ -275,13 +238,12 @@ static void vnc_client_cache_auth(VncState *client)
         return;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session &&
-        client->tls.dname) {
-        client->info->has_x509_dname = true;
-        client->info->x509_dname = g_strdup(client->tls.dname);
+    if (client->tls) {
+        client->info->x509_dname =
+            qcrypto_tls_session_get_peer_name(client->tls);
+        client->info->has_x509_dname =
+            client->info->x509_dname != NULL;
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn &&
         client->sasl.username) {
@@ -293,11 +255,16 @@ static void vnc_client_cache_auth(VncState *client)
 
 static void vnc_client_cache_addr(VncState *client)
 {
-    VncBasicInfo *bi = vnc_basic_info_get_from_remote_addr(client->csock);
+    Error *err = NULL;
 
-    if (bi) {
-        client->info = g_malloc0(sizeof(*client->info));
-        client->info->base = bi;
+    client->info = g_malloc0(sizeof(*client->info));
+    vnc_init_basic_info_from_remote_addr(client->sioc,
+                                         qapi_VncClientInfo_base(client->info),
+                                         &err);
+    if (err) {
+        qapi_free_VncClientInfo(client->info);
+        client->info = NULL;
+        error_free(err);
     }
 }
 
@@ -308,7 +275,6 @@ static void vnc_qmp_event(VncState *vs, QAPIEvent event)
     if (!vs->info) {
         return;
     }
-    g_assert(vs->info->base);
 
     si = vnc_server_info_get(vs->vd);
     if (!si) {
@@ -317,7 +283,8 @@ static void vnc_qmp_event(VncState *vs, QAPIEvent event)
 
     switch (event) {
     case QAPI_EVENT_VNC_CONNECTED:
-        qapi_event_send_vnc_connected(si, vs->info->base, &error_abort);
+        qapi_event_send_vnc_connected(si, qapi_VncClientInfo_base(vs->info),
+                                      &error_abort);
         break;
     case QAPI_EVENT_VNC_INITIALIZED:
         qapi_event_send_vnc_initialized(si, vs->info, &error_abort);
@@ -334,36 +301,26 @@ static void vnc_qmp_event(VncState *vs, QAPIEvent event)
 
 static VncClientInfo *qmp_query_vnc_client(const VncState *client)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen = sizeof(sa);
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
     VncClientInfo *info;
+    Error *err = NULL;
 
-    if (getpeername(client->csock, (struct sockaddr *)&sa, &salen) < 0) {
-        return NULL;
-    }
+    info = g_malloc0(sizeof(*info));
 
-    if (getnameinfo((struct sockaddr *)&sa, salen,
-                    host, sizeof(host),
-                    serv, sizeof(serv),
-                    NI_NUMERICHOST | NI_NUMERICSERV) < 0) {
+    vnc_init_basic_info_from_remote_addr(client->sioc,
+                                         qapi_VncClientInfo_base(info),
+                                         &err);
+    if (err) {
+        error_free(err);
+        qapi_free_VncClientInfo(info);
         return NULL;
     }
 
-    info = g_malloc0(sizeof(*info));
-    info->base = g_malloc0(sizeof(*info->base));
-    info->base->host = g_strdup(host);
-    info->base->service = g_strdup(serv);
-    info->base->family = inet_netfamily(sa.ss_family);
-    info->base->websocket = client->websocket;
+    info->websocket = client->websocket;
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session && client->tls.dname) {
-        info->has_x509_dname = true;
-        info->x509_dname = g_strdup(client->tls.dname);
+    if (client->tls) {
+        info->x509_dname = qcrypto_tls_session_get_peer_name(client->tls);
+        info->has_x509_dname = info->x509_dname != NULL;
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn && client->sasl.username) {
         info->has_sasl_username = true;
@@ -407,81 +364,89 @@ VncInfo *qmp_query_vnc(Error **errp)
 {
     VncInfo *info = g_malloc0(sizeof(*info));
     VncDisplay *vd = vnc_display_find(NULL);
+    SocketAddress *addr = NULL;
 
     if (vd == NULL || !vd->enabled) {
         info->enabled = false;
     } else {
-        struct sockaddr_storage sa;
-        socklen_t salen = sizeof(sa);
-        char host[NI_MAXHOST];
-        char serv[NI_MAXSERV];
-
         info->enabled = true;
 
         /* for compatibility with the original command */
         info->has_clients = true;
         info->clients = qmp_query_client_list(vd);
 
-        if (vd->lsock == -1) {
+        if (vd->lsock == NULL) {
             return info;
         }
 
-        if (getsockname(vd->lsock, (struct sockaddr *)&sa,
-                        &salen) == -1) {
-            error_setg(errp, QERR_UNDEFINED_ERROR);
+        addr = qio_channel_socket_get_local_address(vd->lsock, errp);
+        if (!addr) {
             goto out_error;
         }
 
-        if (getnameinfo((struct sockaddr *)&sa, salen,
-                        host, sizeof(host),
-                        serv, sizeof(serv),
-                        NI_NUMERICHOST | NI_NUMERICSERV) < 0) {
-            error_setg(errp, QERR_UNDEFINED_ERROR);
+        switch (addr->type) {
+        case SOCKET_ADDRESS_KIND_INET:
+            info->host = g_strdup(addr->u.inet.data->host);
+            info->service = g_strdup(addr->u.inet.data->port);
+            if (addr->u.inet.data->ipv6) {
+                info->family = NETWORK_ADDRESS_FAMILY_IPV6;
+            } else {
+                info->family = NETWORK_ADDRESS_FAMILY_IPV4;
+            }
+            break;
+
+        case SOCKET_ADDRESS_KIND_UNIX:
+            info->host = g_strdup("");
+            info->service = g_strdup(addr->u.q_unix.data->path);
+            info->family = NETWORK_ADDRESS_FAMILY_UNIX;
+            break;
+
+        default:
+            error_setg(errp, "Unsupported socket kind %d",
+                       addr->type);
             goto out_error;
         }
 
         info->has_host = true;
-        info->host = g_strdup(host);
-
         info->has_service = true;
-        info->service = g_strdup(serv);
-
         info->has_family = true;
-        info->family = inet_netfamily(sa.ss_family);
 
         info->has_auth = true;
         info->auth = g_strdup(vnc_auth_name(vd));
     }
 
+    qapi_free_SocketAddress(addr);
     return info;
 
 out_error:
+    qapi_free_SocketAddress(addr);
     qapi_free_VncInfo(info);
     return NULL;
 }
 
-static VncBasicInfoList *qmp_query_server_entry(int socket,
+static VncBasicInfoList *qmp_query_server_entry(QIOChannelSocket *ioc,
                                                 bool websocket,
                                                 VncBasicInfoList *prev)
 {
     VncBasicInfoList *list;
     VncBasicInfo *info;
-    struct sockaddr_storage sa;
-    socklen_t salen = sizeof(sa);
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
-
-    if (getsockname(socket, (struct sockaddr *)&sa, &salen) < 0 ||
-        getnameinfo((struct sockaddr *)&sa, salen,
-                    host, sizeof(host), serv, sizeof(serv),
-                    NI_NUMERICHOST | NI_NUMERICSERV) < 0) {
+    Error *err = NULL;
+    SocketAddress *addr;
+
+    addr = qio_channel_socket_get_local_address(ioc, &err);
+    if (!addr) {
+        error_free(err);
         return prev;
     }
 
     info = g_new0(VncBasicInfo, 1);
-    info->host = g_strdup(host);
-    info->service = g_strdup(serv);
-    info->family = inet_netfamily(sa.ss_family);
+    vnc_init_basic_info(addr, info, &err);
+    qapi_free_SocketAddress(addr);
+    if (err) {
+        qapi_free_VncBasicInfo(info);
+        error_free(err);
+        return prev;
+    }
     info->websocket = websocket;
 
     list = g_new0(VncBasicInfoList, 1);
@@ -513,7 +478,6 @@ static void qmp_query_auth(VncDisplay *vd, VncInfo2 *info)
         break;
     case VNC_AUTH_VENCRYPT:
         info->auth = VNC_PRIMARY_AUTH_VENCRYPT;
-#ifdef CONFIG_VNC_TLS
         info->has_vencrypt = true;
         switch (vd->subauth) {
         case VNC_AUTH_VENCRYPT_PLAIN:
@@ -547,7 +511,6 @@ static void qmp_query_auth(VncDisplay *vd, VncInfo2 *info)
             info->has_vencrypt = false;
             break;
         }
-#endif
         break;
     case VNC_AUTH_SASL:
         info->auth = VNC_PRIMARY_AUTH_SASL;
@@ -577,13 +540,13 @@ VncInfo2List *qmp_query_vnc_servers(Error **errp)
             info->has_display = true;
             info->display = g_strdup(dev->id);
         }
-        if (vd->lsock != -1) {
-            info->server = qmp_query_server_entry(vd->lsock, false,
-                                                  info->server);
+        if (vd->lsock != NULL) {
+            info->server = qmp_query_server_entry(
+                vd->lsock, false, info->server);
         }
-        if (vd->lwebsock != -1) {
-            info->server = qmp_query_server_entry(vd->lwebsock, true,
-                                                  info->server);
+        if (vd->lwebsock != NULL) {
+            info->server = qmp_query_server_entry(
+                vd->lwebsock, true, info->server);
         }
 
         item = g_new0(VncInfo2List, 1);
@@ -611,10 +574,25 @@ static void framebuffer_update_request(VncState *vs, int incremental,
 static void vnc_refresh(DisplayChangeListener *dcl);
 static int vnc_refresh_server_surface(VncDisplay *vd);
 
+static int vnc_width(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
+                                       VNC_DIRTY_PIXELS_PER_BIT));
+}
+
+static int vnc_height(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
+}
+
 static void vnc_set_area_dirty(DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT],
                                VNC_MAX_WIDTH / VNC_DIRTY_PIXELS_PER_BIT),
-                               int width, int height,
-                               int x, int y, int w, int h) {
+                               VncDisplay *vd,
+                               int x, int y, int w, int h)
+{
+    int width = vnc_width(vd);
+    int height = vnc_height(vd);
+
     /* this is needed this to ensure we updated all affected
      * blocks if x % VNC_DIRTY_PIXELS_PER_BIT != 0 */
     w += (x % VNC_DIRTY_PIXELS_PER_BIT);
@@ -636,10 +614,8 @@ static void vnc_dpy_update(DisplayChangeListener *dcl,
 {
     VncDisplay *vd = container_of(dcl, VncDisplay, dcl);
     struct VncSurface *s = &vd->guest;
-    int width = pixman_image_get_width(vd->server);
-    int height = pixman_image_get_height(vd->server);
 
-    vnc_set_area_dirty(s->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(s->dirty, vd, x, y, w, h);
 }
 
 void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
@@ -653,53 +629,10 @@ void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
     vnc_write_s32(vs, encoding);
 }
 
-void buffer_reserve(Buffer *buffer, size_t len)
-{
-    if ((buffer->capacity - buffer->offset) < len) {
-        buffer->capacity += (len + 1024);
-        buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
-    }
-}
-
-static int buffer_empty(Buffer *buffer)
-{
-    return buffer->offset == 0;
-}
-
-uint8_t *buffer_end(Buffer *buffer)
-{
-    return buffer->buffer + buffer->offset;
-}
-
-void buffer_reset(Buffer *buffer)
-{
-        buffer->offset = 0;
-}
-
-void buffer_free(Buffer *buffer)
-{
-    g_free(buffer->buffer);
-    buffer->offset = 0;
-    buffer->capacity = 0;
-    buffer->buffer = NULL;
-}
-
-void buffer_append(Buffer *buffer, const void *data, size_t len)
-{
-    memcpy(buffer->buffer + buffer->offset, data, len);
-    buffer->offset += len;
-}
-
-void buffer_advance(Buffer *buf, size_t len)
-{
-    memmove(buf->buffer, buf->buffer + len,
-            (buf->offset - len));
-    buf->offset -= len;
-}
 
 static void vnc_desktop_resize(VncState *vs)
 {
-    if (vs->csock == -1 || !vnc_has_feature(vs, VNC_FEATURE_RESIZE)) {
+    if (vs->ioc == NULL || !vnc_has_feature(vs, VNC_FEATURE_RESIZE)) {
         return;
     }
     if (vs->client_width == pixman_image_get_width(vs->vd->server) &&
@@ -752,6 +685,21 @@ void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y)
     return ptr;
 }
 
+static void vnc_update_server_surface(VncDisplay *vd)
+{
+    qemu_pixman_image_unref(vd->server);
+    vd->server = NULL;
+
+    if (QTAILQ_EMPTY(&vd->clients)) {
+        return;
+    }
+
+    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
+                                          vnc_width(vd),
+                                          vnc_height(vd),
+                                          NULL, 0);
+}
+
 static void vnc_dpy_switch(DisplayChangeListener *dcl,
                            DisplaySurface *surface)
 {
@@ -760,26 +708,19 @@ static void vnc_dpy_switch(DisplayChangeListener *dcl,
     int width, height;
 
     vnc_abort_display_jobs(vd);
+    vd->ds = surface;
 
     /* server surface */
-    qemu_pixman_image_unref(vd->server);
-    vd->ds = surface;
-    width = MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
-                                        VNC_DIRTY_PIXELS_PER_BIT));
-    height = MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
-    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
-                                          width, height, NULL, 0);
+    vnc_update_server_surface(vd);
 
     /* guest surface */
-#if 0 /* FIXME */
-    if (ds_get_bytes_per_pixel(ds) != vd->guest.ds->pf.bytes_per_pixel)
-        console_color_init(ds);
-#endif
     qemu_pixman_image_unref(vd->guest.fb);
     vd->guest.fb = pixman_image_ref(surface->image);
     vd->guest.format = surface->format;
+    width = vnc_width(vd);
+    height = vnc_height(vd);
     memset(vd->guest.dirty, 0x00, sizeof(vd->guest.dirty));
-    vnc_set_area_dirty(vd->guest.dirty, width, height, 0, 0,
+    vnc_set_area_dirty(vd->guest.dirty, vd, 0, 0,
                        width, height);
 
     QTAILQ_FOREACH(vs, &vd->clients, next) {
@@ -789,7 +730,7 @@ static void vnc_dpy_switch(DisplayChangeListener *dcl,
             vnc_cursor_define(vs);
         }
         memset(vs->dirty, 0x00, sizeof(vs->dirty));
-        vnc_set_area_dirty(vs->dirty, width, height, 0, 0,
+        vnc_set_area_dirty(vs->dirty, vd, 0, 0,
                            width, height);
     }
 }
@@ -879,6 +820,8 @@ int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 int vnc_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int n = 0;
+    bool encode_raw = false;
+    size_t saved_offs = vs->output.offset;
 
     switch(vs->vnc_encoding) {
         case VNC_ENCODING_ZLIB:
@@ -901,10 +844,24 @@ int vnc_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
             n = vnc_zywrle_send_framebuffer_update(vs, x, y, w, h);
             break;
         default:
-            vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
-            n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+            encode_raw = true;
             break;
     }
+
+    /* If the client has the same pixel format as our internal buffer and
+     * a RAW encoding would need less space fall back to RAW encoding to
+     * save bandwidth and processing power in the client. */
+    if (!encode_raw && vs->write_pixels == vnc_write_pixels_copy &&
+        12 + h * w * VNC_SERVER_FB_BYTES <= (vs->output.offset - saved_offs)) {
+        vs->output.offset = saved_offs;
+        encode_raw = true;
+    }
+
+    if (encode_raw) {
+        vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
+        n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+    }
+
     return n;
 }
 
@@ -933,6 +890,11 @@ static void vnc_dpy_copy(DisplayChangeListener *dcl,
     int i, x, y, pitch, inc, w_lim, s;
     int cmp_bytes;
 
+    if (!vd->server) {
+        /* no client connected */
+        return;
+    }
+
     vnc_refresh_server_surface(vd);
     QTAILQ_FOREACH_SAFE(vs, &vd->clients, next, vn) {
         if (vnc_has_feature(vs, VNC_FEATURE_COPYRECT)) {
@@ -1063,7 +1025,7 @@ static int find_and_clear_dirty_height(VncState *vs,
 static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
 {
     vs->has_dirty += has_dirty;
-    if (vs->need_update && vs->csock != -1) {
+    if (vs->need_update && vs->ioc != NULL) {
         VncDisplay *vd = vs->vd;
         VncJob *job;
         int y;
@@ -1127,7 +1089,7 @@ static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
         return n;
     }
 
-    if (vs->csock == -1) {
+    if (vs->disconnecting) {
         vnc_disconnect_finish(vs);
     } else if (sync) {
         vnc_jobs_join(vs);
@@ -1209,12 +1171,15 @@ static void audio_del(VncState *vs)
 
 static void vnc_disconnect_start(VncState *vs)
 {
-    if (vs->csock == -1)
+    if (vs->disconnecting) {
         return;
+    }
     vnc_set_share_mode(vs, VNC_SHARE_MODE_DISCONNECTED);
-    qemu_set_fd_handler(vs->csock, NULL, NULL, NULL);
-    closesocket(vs->csock);
-    vs->csock = -1;
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+    }
+    qio_channel_close(vs->ioc, NULL);
+    vs->disconnecting = TRUE;
 }
 
 void vnc_disconnect_finish(VncState *vs)
@@ -1228,8 +1193,6 @@ void vnc_disconnect_finish(VncState *vs)
 
     buffer_free(&vs->input);
     buffer_free(&vs->output);
-    buffer_free(&vs->ws_input);
-    buffer_free(&vs->ws_output);
 
     qapi_free_VncClientInfo(vs->info);
 
@@ -1237,9 +1200,6 @@ void vnc_disconnect_finish(VncState *vs)
     vnc_tight_clear(vs);
     vnc_zrle_clear(vs);
 
-#ifdef CONFIG_VNC_TLS
-    vnc_tls_client_cleanup(vs);
-#endif /* CONFIG_VNC_TLS */
 #ifdef CONFIG_VNC_SASL
     vnc_sasl_client_cleanup(vs);
 #endif /* CONFIG_VNC_SASL */
@@ -1249,6 +1209,10 @@ void vnc_disconnect_finish(VncState *vs)
     if (vs->initialized) {
         QTAILQ_REMOVE(&vs->vd->clients, vs, next);
         qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+        if (QTAILQ_EMPTY(&vs->vd->clients)) {
+            /* last client gone */
+            vnc_update_server_surface(vs->vd);
+        }
     }
 
     if (vs->vd->lock_key_sync)
@@ -1265,29 +1229,29 @@ void vnc_disconnect_finish(VncState *vs)
         g_free(vs->lossy_rect[i]);
     }
     g_free(vs->lossy_rect);
+
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = NULL;
+    object_unref(OBJECT(vs->sioc));
+    vs->sioc = NULL;
     g_free(vs);
 }
 
-int vnc_client_io_error(VncState *vs, int ret, int last_errno)
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, Error **errp)
 {
-    if (ret == 0 || ret == -1) {
-        if (ret == -1) {
-            switch (last_errno) {
-                case EINTR:
-                case EAGAIN:
-#ifdef _WIN32
-                case WSAEWOULDBLOCK:
-#endif
-                    return 0;
-                default:
-                    break;
-            }
+    if (ret <= 0) {
+        if (ret == 0) {
+            VNC_DEBUG("Closing down client sock: EOF\n");
+        } else if (ret != QIO_CHANNEL_ERR_BLOCK) {
+            VNC_DEBUG("Closing down client sock: ret %d (%s)\n",
+                      ret, errp ? error_get_pretty(*errp) : "Unknown");
         }
 
-        VNC_DEBUG("Closing down client sock: ret %d, errno %d\n",
-                  ret, ret < 0 ? last_errno : 0);
         vnc_disconnect_start(vs);
-
+        if (errp) {
+            error_free(*errp);
+            *errp = NULL;
+        }
         return 0;
     }
     return ret;
@@ -1300,23 +1264,6 @@ void vnc_client_error(VncState *vs)
     vnc_disconnect_start(vs);
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_write_tls(gnutls_session_t *session,
-                                 const uint8_t *data,
-                                 size_t datalen)
-{
-    long ret = gnutls_write(*session, data, datalen);
-    if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
-        }
-        ret = -1;
-    }
-    return ret;
-}
-#endif /* CONFIG_VNC_TLS */
 
 /*
  * Called to write a chunk of data to the client socket. The data may
@@ -1333,20 +1280,14 @@ static long vnc_client_write_tls(gnutls_session_t *session,
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_write_tls(&vs->tls.session, data, datalen);
-    } else {
-#endif /* CONFIG_VNC_TLS */
-        ret = send(vs->csock, (const void *)data, datalen, 0);
-#ifdef CONFIG_VNC_TLS
-    }
-#endif /* CONFIG_VNC_TLS */
+    Error *err = NULL;
+    ssize_t ret;
+    ret = qio_channel_write(
+        vs->ioc, (const char *)data, datalen, &err);
     VNC_DEBUG("Wrote wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, &err);
 }
 
 
@@ -1360,9 +1301,9 @@ long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
  * the buffered output data if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-static long vnc_client_write_plain(VncState *vs)
+static ssize_t vnc_client_write_plain(VncState *vs)
 {
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     VNC_DEBUG("Write Plain: Pending output %p size %zd offset %zd. Wait SSF %d\n",
@@ -1384,7 +1325,11 @@ static long vnc_client_write_plain(VncState *vs)
     buffer_advance(&vs->output, ret);
 
     if (vs->output.offset == 0) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     return ret;
@@ -1396,10 +1341,8 @@ static long vnc_client_write_plain(VncState *vs)
  * the client socket. Will delegate actual work according to whether
  * SASL SSF layers are enabled (thus requiring encryption calls)
  */
-static void vnc_client_write_locked(void *opaque)
+static void vnc_client_write_locked(VncState *vs)
 {
-    VncState *vs = opaque;
-
 #ifdef CONFIG_VNC_SASL
     if (vs->sasl.conn &&
         vs->sasl.runSSF &&
@@ -1408,23 +1351,22 @@ static void vnc_client_write_locked(void *opaque)
     } else
 #endif /* CONFIG_VNC_SASL */
     {
-        if (vs->encode_ws) {
-            vnc_client_write_ws(vs);
-        } else {
-            vnc_client_write_plain(vs);
-        }
+        vnc_client_write_plain(vs);
     }
 }
 
-void vnc_client_write(void *opaque)
+static void vnc_client_write(VncState *vs)
 {
-    VncState *vs = opaque;
 
     vnc_lock_output(vs);
-    if (vs->output.offset || vs->ws_output.offset) {
-        vnc_client_write_locked(opaque);
-    } else if (vs->csock != -1) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
+    if (vs->output.offset) {
+        vnc_client_write_locked(vs);
+    } else if (vs->ioc != NULL) {
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
     vnc_unlock_output(vs);
 }
@@ -1435,22 +1377,6 @@ void vnc_read_when(VncState *vs, VncReadEvent *func, size_t expecting)
     vs->read_handler_expect = expecting;
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_read_tls(gnutls_session_t *session, uint8_t *data,
-                                size_t datalen)
-{
-    long ret = gnutls_read(*session, data, datalen);
-    if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
-        }
-        ret = -1;
-    }
-    return ret;
-}
-#endif /* CONFIG_VNC_TLS */
 
 /*
  * Called to read a chunk of data from the client socket. The data may
@@ -1467,20 +1393,14 @@ static long vnc_client_read_tls(gnutls_session_t *session, uint8_t *data,
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_read_tls(&vs->tls.session, data, datalen);
-    } else {
-#endif /* CONFIG_VNC_TLS */
-        ret = qemu_recv(vs->csock, data, datalen, 0);
-#ifdef CONFIG_VNC_TLS
-    }
-#endif /* CONFIG_VNC_TLS */
+    ssize_t ret;
+    Error *err = NULL;
+    ret = qio_channel_read(
+        vs->ioc, (char *)data, datalen, &err);
     VNC_DEBUG("Read wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, &err);
 }
 
 
@@ -1492,9 +1412,9 @@ long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
  * Returns the number of bytes read. Returns -1 on error, and
  * disconnects the client socket.
  */
-static long vnc_client_read_plain(VncState *vs)
+static ssize_t vnc_client_read_plain(VncState *vs)
 {
-    int ret;
+    ssize_t ret;
     VNC_DEBUG("Read plain %p size %zd offset %zd\n",
               vs->input.buffer, vs->input.capacity, vs->input.offset);
     buffer_reserve(&vs->input, 4096);
@@ -1517,31 +1437,20 @@ static void vnc_jobs_bh(void *opaque)
  * the client socket. Will delegate actual work according to whether
  * SASL SSF layers are enabled (thus requiring decryption calls)
  */
-void vnc_client_read(void *opaque)
+static void vnc_client_read(VncState *vs)
 {
-    VncState *vs = opaque;
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     if (vs->sasl.conn && vs->sasl.runSSF)
         ret = vnc_client_read_sasl(vs);
     else
 #endif /* CONFIG_VNC_SASL */
-        if (vs->encode_ws) {
-            ret = vnc_client_read_ws(vs);
-            if (ret == -1) {
-                vnc_disconnect_start(vs);
-                return;
-            } else if (ret == -2) {
-                vnc_client_error(vs);
-                return;
-            }
-        } else {
-            ret = vnc_client_read_plain(vs);
-        }
+        ret = vnc_client_read_plain(vs);
     if (!ret) {
-        if (vs->csock == -1)
+        if (vs->disconnecting) {
             vnc_disconnect_finish(vs);
+        }
         return;
     }
 
@@ -1550,7 +1459,7 @@ void vnc_client_read(void *opaque)
         int ret;
 
         ret = vs->read_handler(vs, vs->input.buffer, len);
-        if (vs->csock == -1) {
+        if (vs->disconnecting) {
             vnc_disconnect_finish(vs);
             return;
         }
@@ -1563,12 +1472,30 @@ void vnc_client_read(void *opaque)
     }
 }
 
+gboolean vnc_client_io(QIOChannel *ioc G_GNUC_UNUSED,
+                       GIOCondition condition, void *opaque)
+{
+    VncState *vs = opaque;
+    if (condition & G_IO_IN) {
+        vnc_client_read(vs);
+    }
+    if (condition & G_IO_OUT) {
+        vnc_client_write(vs);
+    }
+    return TRUE;
+}
+
+
 void vnc_write(VncState *vs, const void *data, size_t len)
 {
     buffer_reserve(&vs->output, len);
 
-    if (vs->csock != -1 && buffer_empty(&vs->output)) {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, vnc_client_write, vs);
+    if (vs->ioc != NULL && buffer_empty(&vs->output)) {
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
     }
 
     buffer_append(&vs->output, data, len);
@@ -1609,8 +1536,7 @@ void vnc_write_u8(VncState *vs, uint8_t value)
 void vnc_flush(VncState *vs)
 {
     vnc_lock_output(vs);
-    if (vs->csock != -1 && (vs->output.offset ||
-                            vs->ws_output.offset)) {
+    if (vs->ioc != NULL && vs->output.offset) {
         vnc_client_write_locked(vs);
     }
     vnc_unlock_output(vs);
@@ -1664,7 +1590,7 @@ static void check_pointer_type_change(Notifier *notifier, void *data)
 
 static void pointer_event(VncState *vs, int button_mask, int x, int y)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = 0x01,
         [INPUT_BUTTON_MIDDLE]     = 0x02,
         [INPUT_BUTTON_RIGHT]      = 0x04,
@@ -2024,9 +1950,6 @@ static void ext_key_event(VncState *vs, int down,
 static void framebuffer_update_request(VncState *vs, int incremental,
                                        int x, int y, int w, int h)
 {
-    int width = pixman_image_get_width(vs->vd->server);
-    int height = pixman_image_get_height(vs->vd->server);
-
     vs->need_update = 1;
 
     if (incremental) {
@@ -2034,7 +1957,7 @@ static void framebuffer_update_request(VncState *vs, int incremental,
     }
 
     vs->force_update = 1;
-    vnc_set_area_dirty(vs->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(vs->dirty, vs->vd, x, y, w, h);
 }
 
 static void send_ext_key_event_ack(VncState *vs)
@@ -2124,6 +2047,9 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
             break;
         case VNC_ENCODING_RICH_CURSOR:
             vs->features |= VNC_FEATURE_RICH_CURSOR_MASK;
+            if (vs->vd->cursor) {
+                vnc_cursor_define(vs);
+            }
             break;
         case VNC_ENCODING_EXT_KEY_EVENT:
             send_ext_key_event_ack(vs);
@@ -2189,15 +2115,15 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
-    vs->client_pf.rmax = red_max;
+    vs->client_pf.rmax = red_max ? red_max : 0xFF;
     vs->client_pf.rbits = hweight_long(red_max);
     vs->client_pf.rshift = red_shift;
     vs->client_pf.rmask = red_max << red_shift;
-    vs->client_pf.gmax = green_max;
+    vs->client_pf.gmax = green_max ? green_max : 0xFF;
     vs->client_pf.gbits = hweight_long(green_max);
     vs->client_pf.gshift = green_shift;
     vs->client_pf.gmask = green_max << green_shift;
-    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bmax = blue_max ? blue_max : 0xFF;
     vs->client_pf.bbits = hweight_long(blue_max);
     vs->client_pf.bshift = blue_shift;
     vs->client_pf.bmask = blue_max << blue_shift;
@@ -2631,12 +2557,10 @@ static int protocol_client_auth(VncState *vs, uint8_t *data, size_t len)
            start_auth_vnc(vs);
            break;
 
-#ifdef CONFIG_VNC_TLS
        case VNC_AUTH_VENCRYPT:
            VNC_DEBUG("Accept VeNCrypt auth\n");
            start_auth_vencrypt(vs);
            break;
-#endif /* CONFIG_VNC_TLS */
 
 #ifdef CONFIG_VNC_SASL
        case VNC_AUTH_SASL:
@@ -2999,15 +2923,36 @@ static void vnc_refresh(DisplayChangeListener *dcl)
     }
 }
 
-static void vnc_connect(VncDisplay *vd, int csock,
+static void vnc_connect(VncDisplay *vd, QIOChannelSocket *sioc,
                         bool skipauth, bool websocket)
 {
-    VncState *vs = g_malloc0(sizeof(VncState));
+    VncState *vs = g_new0(VncState, 1);
     int i;
 
-    vs->csock = csock;
+    vs->sioc = sioc;
+    object_ref(OBJECT(vs->sioc));
+    vs->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(vs->ioc));
     vs->vd = vd;
 
+    buffer_init(&vs->input,          "vnc-input/%p", sioc);
+    buffer_init(&vs->output,         "vnc-output/%p", sioc);
+    buffer_init(&vs->jobs_buffer,    "vnc-jobs_buffer/%p", sioc);
+
+    buffer_init(&vs->tight.tight,    "vnc-tight/%p", sioc);
+    buffer_init(&vs->tight.zlib,     "vnc-tight-zlib/%p", sioc);
+    buffer_init(&vs->tight.gradient, "vnc-tight-gradient/%p", sioc);
+#ifdef CONFIG_VNC_JPEG
+    buffer_init(&vs->tight.jpeg,     "vnc-tight-jpeg/%p", sioc);
+#endif
+#ifdef CONFIG_VNC_PNG
+    buffer_init(&vs->tight.png,      "vnc-tight-png/%p", sioc);
+#endif
+    buffer_init(&vs->zlib.zlib,      "vnc-zlib/%p", sioc);
+    buffer_init(&vs->zrle.zrle,      "vnc-zrle/%p", sioc);
+    buffer_init(&vs->zrle.fb,        "vnc-zrle-fb/%p", sioc);
+    buffer_init(&vs->zrle.zlib,      "vnc-zrle-zlib/%p", sioc);
+
     if (skipauth) {
 	vs->auth = VNC_AUTH_NONE;
 	vs->subauth = VNC_AUTH_INVALID;
@@ -3020,30 +2965,29 @@ static void vnc_connect(VncDisplay *vd, int csock,
             vs->subauth = vd->subauth;
         }
     }
-    VNC_DEBUG("Client sock=%d ws=%d auth=%d subauth=%d\n",
-              csock, websocket, vs->auth, vs->subauth);
+    VNC_DEBUG("Client sioc=%p ws=%d auth=%d subauth=%d\n",
+              sioc, websocket, vs->auth, vs->subauth);
 
     vs->lossy_rect = g_malloc0(VNC_STAT_ROWS * sizeof (*vs->lossy_rect));
     for (i = 0; i < VNC_STAT_ROWS; ++i) {
-        vs->lossy_rect[i] = g_malloc0(VNC_STAT_COLS * sizeof (uint8_t));
+        vs->lossy_rect[i] = g_new0(uint8_t, VNC_STAT_COLS);
     }
 
-    VNC_DEBUG("New client on socket %d\n", csock);
+    VNC_DEBUG("New client on socket %p\n", vs->sioc);
     update_displaychangelistener(&vd->dcl, VNC_REFRESH_INTERVAL_BASE);
-    qemu_set_nonblock(vs->csock);
+    qio_channel_set_blocking(vs->ioc, false, NULL);
     if (websocket) {
         vs->websocket = 1;
-#ifdef CONFIG_VNC_TLS
         if (vd->ws_tls) {
-            qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io, NULL, vs);
-        } else
-#endif /* CONFIG_VNC_TLS */
-        {
-            qemu_set_fd_handler(vs->csock, vncws_handshake_read, NULL, vs);
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN, vncws_tls_handshake_io, vs, NULL);
+        } else {
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN, vncws_handshake_io, vs, NULL);
         }
-    } else
-    {
-        qemu_set_fd_handler(vs->csock, vnc_client_read, NULL, vs);
+    } else {
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     vnc_client_cache_addr(vs);
@@ -3068,6 +3012,7 @@ void vnc_init_state(VncState *vs)
 {
     vs->initialized = true;
     VncDisplay *vd = vs->vd;
+    bool first_client = QTAILQ_EMPTY(&vd->clients);
 
     vs->last_x = -1;
     vs->last_y = -1;
@@ -3081,6 +3026,9 @@ void vnc_init_state(VncState *vs)
     vs->bh = qemu_bh_new(vnc_jobs_bh, vs);
 
     QTAILQ_INSERT_TAIL(&vd->clients, vs, next);
+    if (first_client) {
+        vnc_update_server_surface(vd);
+    }
 
     graphic_hw_update(vd->dcl.con);
 
@@ -3097,35 +3045,28 @@ void vnc_init_state(VncState *vs)
     /* vs might be free()ed here */
 }
 
-static void vnc_listen_read(void *opaque, bool websocket)
+static gboolean vnc_listen_io(QIOChannel *ioc,
+                              GIOCondition condition,
+                              void *opaque)
 {
     VncDisplay *vs = opaque;
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(addr);
-    int csock;
+    QIOChannelSocket *sioc = NULL;
+    Error *err = NULL;
 
     /* Catch-up */
     graphic_hw_update(vs->dcl.con);
-    if (websocket) {
-        csock = qemu_accept(vs->lwebsock, (struct sockaddr *)&addr, &addrlen);
+    sioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc), &err);
+    if (sioc != NULL) {
+        qio_channel_set_delay(QIO_CHANNEL(sioc), false);
+        vnc_connect(vs, sioc, false,
+                    ioc != QIO_CHANNEL(vs->lsock));
+        object_unref(OBJECT(sioc));
     } else {
-        csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
-    }
-
-    if (csock != -1) {
-        socket_set_nodelay(csock);
-        vnc_connect(vs, csock, false, websocket);
+        /* client probably closed connection before we got there */
+        error_free(err);
     }
-}
 
-static void vnc_listen_regular_read(void *opaque)
-{
-    vnc_listen_read(opaque, false);
-}
-
-static void vnc_listen_websocket_read(void *opaque)
-{
-    vnc_listen_read(opaque, true);
+    return TRUE;
 }
 
 static const DisplayChangeListenerOps dcl_ops = {
@@ -3151,9 +3092,6 @@ void vnc_display_init(const char *id)
     vs->id = strdup(id);
     QTAILQ_INSERT_TAIL(&vnc_displays, vs, next);
 
-    vs->lsock = -1;
-    vs->lwebsock = -1;
-
     QTAILQ_INIT(&vs->clients);
     vs->expires = TIME_MAX;
 
@@ -3181,22 +3119,29 @@ static void vnc_display_close(VncDisplay *vs)
         return;
     vs->enabled = false;
     vs->is_unix = false;
-    if (vs->lsock != -1) {
-        qemu_set_fd_handler(vs->lsock, NULL, NULL, NULL);
-        close(vs->lsock);
-        vs->lsock = -1;
+    if (vs->lsock != NULL) {
+        if (vs->lsock_tag) {
+            g_source_remove(vs->lsock_tag);
+        }
+        object_unref(OBJECT(vs->lsock));
+        vs->lsock = NULL;
     }
     vs->ws_enabled = false;
-    if (vs->lwebsock != -1) {
-        qemu_set_fd_handler(vs->lwebsock, NULL, NULL, NULL);
-        close(vs->lwebsock);
-        vs->lwebsock = -1;
+    if (vs->lwebsock != NULL) {
+        if (vs->lwebsock_tag) {
+            g_source_remove(vs->lwebsock_tag);
+        }
+        object_unref(OBJECT(vs->lwebsock));
+        vs->lwebsock = NULL;
     }
     vs->auth = VNC_AUTH_INVALID;
     vs->subauth = VNC_AUTH_INVALID;
-#ifdef CONFIG_VNC_TLS
-    vs->tls.x509verify = 0;
-#endif
+    if (vs->tlscreds) {
+        object_unparent(OBJECT(vs->tlscreds));
+        vs->tlscreds = NULL;
+    }
+    g_free(vs->tlsaclname);
+    vs->tlsaclname = NULL;
 }
 
 int vnc_display_password(const char *id, const char *password)
@@ -3233,9 +3178,26 @@ int vnc_display_pw_expire(const char *id, time_t expires)
 char *vnc_display_local_addr(const char *id)
 {
     VncDisplay *vs = vnc_display_find(id);
+    SocketAddress *addr;
+    char *ret;
+    Error *err = NULL;
 
     assert(vs);
-    return vnc_socket_local_addr("%s:%s", vs->lsock);
+
+    addr = qio_channel_socket_get_local_address(vs->lsock, &err);
+    if (!addr) {
+        return NULL;
+    }
+
+    if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+        qapi_free_SocketAddress(addr);
+        return NULL;
+    }
+    ret = g_strdup_printf("%s;%s", addr->u.inet.data->host,
+                          addr->u.inet.data->port);
+    qapi_free_SocketAddress(addr);
+
+    return ret;
 }
 
 static QemuOptsList qemu_vnc_opts = {
@@ -3250,6 +3212,10 @@ static QemuOptsList qemu_vnc_opts = {
             .name = "websocket",
             .type = QEMU_OPT_STRING,
         },{
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+        },{
+            /* Deprecated in favour of tls-creds */
             .name = "x509",
             .type = QEMU_OPT_STRING,
         },{
@@ -3286,9 +3252,11 @@ static QemuOptsList qemu_vnc_opts = {
             .name = "sasl",
             .type = QEMU_OPT_BOOL,
         },{
+            /* Deprecated in favour of tls-creds */
             .name = "tls",
             .type = QEMU_OPT_BOOL,
         },{
+            /* Deprecated in favour of tls-creds */
             .name = "x509verify",
             .type = QEMU_OPT_STRING,
         },{
@@ -3306,13 +3274,12 @@ static QemuOptsList qemu_vnc_opts = {
 };
 
 
-static void
+static int
 vnc_display_setup_auth(VncDisplay *vs,
                        bool password,
                        bool sasl,
-                       bool tls,
-                       bool x509,
-                       bool websocket)
+                       bool websocket,
+                       Error **errp)
 {
     /*
      * We have a choice of 3 authentication options
@@ -3362,17 +3329,24 @@ vnc_display_setup_auth(VncDisplay *vs,
      * result has the same security characteristics.
      */
     if (password) {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 password auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509VNC;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS password auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSVNC;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with password auth\n");
@@ -3385,17 +3359,24 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     } else if (sasl) {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 SASL auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509SASL;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSSASL;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with SASL auth\n");
@@ -3408,17 +3389,24 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     } else {
-        if (tls) {
+        if (vs->tlscreds) {
             vs->auth = VNC_AUTH_VENCRYPT;
             if (websocket) {
                 vs->ws_tls = true;
             }
-            if (x509) {
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
                 VNC_DEBUG("Initializing VNC server with x509 no auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_X509NONE;
-            } else {
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
                 VNC_DEBUG("Initializing VNC server with TLS no auth\n");
                 vs->subauth = VNC_AUTH_VENCRYPT_TLSNONE;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
             }
         } else {
             VNC_DEBUG("Initializing VNC server with no auth\n");
@@ -3431,34 +3419,72 @@ vnc_display_setup_auth(VncDisplay *vs,
             vs->ws_auth = VNC_AUTH_INVALID;
         }
     }
+    return 0;
 }
 
+
+/*
+ * Handle back compat with old CLI syntax by creating some
+ * suitable QCryptoTLSCreds objects
+ */
+static QCryptoTLSCreds *
+vnc_display_create_creds(bool x509,
+                         bool x509verify,
+                         const char *dir,
+                         const char *id,
+                         Error **errp)
+{
+    gchar *credsid = g_strdup_printf("tlsvnc%s", id);
+    Object *parent = object_get_objects_root();
+    Object *creds;
+    Error *err = NULL;
+
+    if (x509) {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_X509,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      "dir", dir,
+                                      "verify-peer", x509verify ? "yes" : "no",
+                                      NULL);
+    } else {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_ANON,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      NULL);
+    }
+
+    g_free(credsid);
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
 void vnc_display_open(const char *id, Error **errp)
 {
     VncDisplay *vs = vnc_display_find(id);
     QemuOpts *opts = qemu_opts_find(&qemu_vnc_opts, id);
-    QemuOpts *sopts, *wsopts;
+    SocketAddress *saddr = NULL, *wsaddr = NULL;
     const char *share, *device_id;
     QemuConsole *con;
     bool password = false;
     bool reverse = false;
     const char *vnc;
-    const char *has_to;
     char *h;
-    bool has_ipv4 = false;
-    bool has_ipv6 = false;
-    const char *websocket;
-    bool tls = false, x509 = false;
-#ifdef CONFIG_VNC_TLS
-    const char *path;
-#endif
+    const char *credid;
     bool sasl = false;
 #ifdef CONFIG_VNC_SASL
     int saslErr;
 #endif
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
     int acl = 0;
-#endif
     int lock_key_sync = 1;
 
     if (!vs) {
@@ -3475,44 +3501,90 @@ void vnc_display_open(const char *id, Error **errp)
         return;
     }
 
-    sopts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    wsopts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-
     h = strrchr(vnc, ':');
     if (h) {
-        char *host;
         size_t hlen = h - vnc;
 
-        if (vnc[0] == '[' && vnc[hlen - 1] == ']') {
-            host = g_strndup(vnc + 1, hlen - 2);
+        const char *websocket = qemu_opt_get(opts, "websocket");
+        int to = qemu_opt_get_number(opts, "to", 0);
+        bool has_ipv4 = qemu_opt_get(opts, "ipv4");
+        bool has_ipv6 = qemu_opt_get(opts, "ipv6");
+        bool ipv4 = qemu_opt_get_bool(opts, "ipv4", false);
+        bool ipv6 = qemu_opt_get_bool(opts, "ipv6", false);
+
+        saddr = g_new0(SocketAddress, 1);
+        if (websocket) {
+            if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA1)) {
+                error_setg(errp,
+                           "SHA1 hash support is required for websockets");
+                goto fail;
+            }
+
+            wsaddr = g_new0(SocketAddress, 1);
+            vs->ws_enabled = true;
+        }
+
+        if (strncmp(vnc, "unix:", 5) == 0) {
+            saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+            saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+            saddr->u.q_unix.data->path = g_strdup(vnc + 5);
+
+            if (vs->ws_enabled) {
+                error_setg(errp, "UNIX sockets not supported with websock");
+                goto fail;
+            }
         } else {
-            host = g_strndup(vnc, hlen);
+            unsigned long long baseport;
+            InetSocketAddress *inet;
+            saddr->type = SOCKET_ADDRESS_KIND_INET;
+            inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+            if (vnc[0] == '[' && vnc[hlen - 1] == ']') {
+                inet->host = g_strndup(vnc + 1, hlen - 2);
+            } else {
+                inet->host = g_strndup(vnc, hlen);
+            }
+            if (parse_uint_full(h + 1, &baseport, 10) < 0) {
+                error_setg(errp, "can't convert to a number: %s", h + 1);
+                goto fail;
+            }
+            if (baseport > 65535 ||
+                baseport + 5900 > 65535) {
+                error_setg(errp, "port %s out of range", h + 1);
+                goto fail;
+            }
+            inet->port = g_strdup_printf(
+                "%d", (int)baseport + 5900);
+
+            if (to) {
+                inet->has_to = true;
+                inet->to = to + 5900;
+            }
+            inet->ipv4 = ipv4;
+            inet->has_ipv4 = has_ipv4;
+            inet->ipv6 = ipv6;
+            inet->has_ipv6 = has_ipv6;
+
+            if (vs->ws_enabled) {
+                wsaddr->type = SOCKET_ADDRESS_KIND_INET;
+                inet = wsaddr->u.inet.data = g_new0(InetSocketAddress, 1);
+                inet->host = g_strdup(saddr->u.inet.data->host);
+                inet->port = g_strdup(websocket);
+
+                if (to) {
+                    inet->has_to = true;
+                    inet->to = to;
+                }
+                inet->ipv4 = ipv4;
+                inet->has_ipv4 = has_ipv4;
+                inet->ipv6 = ipv6;
+                inet->has_ipv6 = has_ipv6;
+            }
         }
-        qemu_opt_set(sopts, "host", host, &error_abort);
-        qemu_opt_set(wsopts, "host", host, &error_abort);
-        qemu_opt_set(sopts, "port", h+1, &error_abort);
-        g_free(host);
     } else {
         error_setg(errp, "no vnc port specified");
         goto fail;
     }
 
-    has_to = qemu_opt_get(opts, "to");
-    has_ipv4 = qemu_opt_get_bool(opts, "ipv4", false);
-    has_ipv6 = qemu_opt_get_bool(opts, "ipv6", false);
-    if (has_to) {
-        qemu_opt_set(sopts, "to", has_to, &error_abort);
-        qemu_opt_set(wsopts, "to", has_to, &error_abort);
-    }
-    if (has_ipv4) {
-        qemu_opt_set(sopts, "ipv4", "on", &error_abort);
-        qemu_opt_set(wsopts, "ipv4", "on", &error_abort);
-    }
-    if (has_ipv6) {
-        qemu_opt_set(sopts, "ipv6", "on", &error_abort);
-        qemu_opt_set(wsopts, "ipv6", "on", &error_abort);
-    }
-
     password = qemu_opt_get_bool(opts, "password", false);
     if (password) {
         if (fips_get_state()) {
@@ -3539,32 +3611,67 @@ void vnc_display_open(const char *id, Error **errp)
         goto fail;
     }
 #endif /* CONFIG_VNC_SASL */
-    tls  = qemu_opt_get_bool(opts, "tls", false);
-#ifdef CONFIG_VNC_TLS
-    path = qemu_opt_get(opts, "x509");
-    if (!path) {
-        path = qemu_opt_get(opts, "x509verify");
-        if (path) {
-            vs->tls.x509verify = true;
-        }
-    }
-    if (path) {
-        x509 = true;
-        if (vnc_tls_set_x509_creds_dir(vs, path) < 0) {
-            error_setg(errp, "Failed to find x509 certificates/keys in %s",
-                       path);
+    credid = qemu_opt_get(opts, "tls-creds");
+    if (credid) {
+        Object *creds;
+        if (qemu_opt_get(opts, "tls") ||
+            qemu_opt_get(opts, "x509") ||
+            qemu_opt_get(opts, "x509verify")) {
+            error_setg(errp,
+                       "'tls-creds' parameter is mutually exclusive with "
+                       "'tls', 'x509' and 'x509verify' parameters");
             goto fail;
         }
+
+        creds = object_resolve_path_component(
+            object_get_objects_root(), credid);
+        if (!creds) {
+            error_setg(errp, "No TLS credentials with id '%s'",
+                       credid);
+            goto fail;
+        }
+        vs->tlscreds = (QCryptoTLSCreds *)
+            object_dynamic_cast(creds,
+                                TYPE_QCRYPTO_TLS_CREDS);
+        if (!vs->tlscreds) {
+            error_setg(errp, "Object with id '%s' is not TLS credentials",
+                       credid);
+            goto fail;
+        }
+        object_ref(OBJECT(vs->tlscreds));
+
+        if (vs->tlscreds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            error_setg(errp,
+                       "Expecting TLS credentials with a server endpoint");
+            goto fail;
+        }
+    } else {
+        const char *path;
+        bool tls = false, x509 = false, x509verify = false;
+        tls  = qemu_opt_get_bool(opts, "tls", false);
+        if (tls) {
+            path = qemu_opt_get(opts, "x509");
+
+            if (path) {
+                x509 = true;
+            } else {
+                path = qemu_opt_get(opts, "x509verify");
+                if (path) {
+                    x509 = true;
+                    x509verify = true;
+                }
+            }
+            vs->tlscreds = vnc_display_create_creds(x509,
+                                                    x509verify,
+                                                    path,
+                                                    vs->id,
+                                                    errp);
+            if (!vs->tlscreds) {
+                goto fail;
+            }
+        }
     }
-#else /* ! CONFIG_VNC_TLS */
-    if (tls) {
-        error_setg(errp, "VNC TLS auth requires gnutls support");
-        goto fail;
-    }
-#endif /* ! CONFIG_VNC_TLS */
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
     acl = qemu_opt_get_bool(opts, "acl", false);
-#endif
 
     share = qemu_opt_get(opts, "share");
     if (share) {
@@ -3583,16 +3690,6 @@ void vnc_display_open(const char *id, Error **errp)
     }
     vs->connections_limit = qemu_opt_get_number(opts, "connections", 32);
 
-    websocket = qemu_opt_get(opts, "websocket");
-    if (websocket) {
-        vs->ws_enabled = true;
-        qemu_opt_set(wsopts, "port", websocket, &error_abort);
-        if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA1)) {
-            error_setg(errp, "SHA1 hash support is required for websockets");
-            goto fail;
-        }
-    }
-
 #ifdef CONFIG_VNC_JPEG
     vs->lossy = qemu_opt_get_bool(opts, "lossy", false);
 #endif
@@ -3604,19 +3701,14 @@ void vnc_display_open(const char *id, Error **errp)
         vs->non_adaptive = true;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (acl && x509 && vs->tls.x509verify) {
-        char *aclname;
-
+    if (acl) {
         if (strcmp(vs->id, "default") == 0) {
-            aclname = g_strdup("vnc.x509dname");
+            vs->tlsaclname = g_strdup("vnc.x509dname");
         } else {
-            aclname = g_strdup_printf("vnc.%s.x509dname", vs->id);
+            vs->tlsaclname = g_strdup_printf("vnc.%s.x509dname", vs->id);
         }
-        vs->tls.acl = qemu_acl_init(aclname);
-        g_free(aclname);
+        qemu_acl_init(vs->tlsaclname);
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (acl && sasl) {
         char *aclname;
@@ -3631,7 +3723,9 @@ void vnc_display_open(const char *id, Error **errp)
     }
 #endif
 
-    vnc_display_setup_auth(vs, password, sasl, tls, x509, websocket);
+    if (vnc_display_setup_auth(vs, password, sasl, vs->ws_enabled, errp) < 0) {
+        goto fail;
+    }
 
 #ifdef CONFIG_VNC_SASL
     if ((saslErr = sasl_server_init(NULL, "qemu")) != SASL_OK) {
@@ -3644,19 +3738,12 @@ void vnc_display_open(const char *id, Error **errp)
 
     device_id = qemu_opt_get(opts, "display");
     if (device_id) {
-        DeviceState *dev;
         int head = qemu_opt_get_number(opts, "head", 0);
+        Error *err = NULL;
 
-        dev = qdev_find_recursive(sysbus_get_default(), device_id);
-        if (dev == NULL) {
-            error_setg(errp, "Device '%s' not found", device_id);
-            goto fail;
-        }
-
-        con = qemu_console_lookup_by_device(dev, head);
-        if (con == NULL) {
-            error_setg(errp, "Device %s is not bound to a QemuConsole",
-                       device_id);
+        con = qemu_console_lookup_by_device_name(device_id, head, &err);
+        if (err) {
+            error_propagate(errp, err);
             goto fail;
         }
     } else {
@@ -3671,56 +3758,55 @@ void vnc_display_open(const char *id, Error **errp)
 
     if (reverse) {
         /* connect to viewer */
-        int csock;
-        vs->lsock = -1;
-        vs->lwebsock = -1;
-        if (strncmp(vnc, "unix:", 5) == 0) {
-            csock = unix_connect(vnc+5, errp);
-        } else {
-            csock = inet_connect(vnc, errp);
+        QIOChannelSocket *sioc = NULL;
+        vs->lsock = NULL;
+        vs->lwebsock = NULL;
+        if (vs->ws_enabled) {
+            error_setg(errp, "Cannot use websockets in reverse mode");
+            goto fail;
         }
-        if (csock < 0) {
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+        sioc = qio_channel_socket_new();
+        if (qio_channel_socket_connect_sync(sioc, saddr, errp) < 0) {
             goto fail;
         }
-        vnc_connect(vs, csock, false, false);
+        vnc_connect(vs, sioc, false, false);
+        object_unref(OBJECT(sioc));
     } else {
-        /* listen for connects */
-        if (strncmp(vnc, "unix:", 5) == 0) {
-            vs->lsock = unix_listen(vnc+5, NULL, 0, errp);
-            if (vs->lsock < 0) {
-                goto fail;
-            }
-            vs->is_unix = true;
-        } else {
-            vs->lsock = inet_listen_opts(sopts, 5900, errp);
-            if (vs->lsock < 0) {
+        vs->lsock = qio_channel_socket_new();
+        if (qio_channel_socket_listen_sync(vs->lsock, saddr, errp) < 0) {
+            goto fail;
+        }
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+        vs->enabled = true;
+
+        if (vs->ws_enabled) {
+            vs->lwebsock = qio_channel_socket_new();
+            if (qio_channel_socket_listen_sync(vs->lwebsock,
+                                               wsaddr, errp) < 0) {
+                object_unref(OBJECT(vs->lsock));
+                vs->lsock = NULL;
                 goto fail;
             }
-            if (vs->ws_enabled) {
-                vs->lwebsock = inet_listen_opts(wsopts, 0, errp);
-                if (vs->lwebsock < 0) {
-                    if (vs->lsock != -1) {
-                        close(vs->lsock);
-                        vs->lsock = -1;
-                    }
-                    goto fail;
-                }
-            }
         }
-        vs->enabled = true;
-        qemu_set_fd_handler(vs->lsock, vnc_listen_regular_read, NULL, vs);
+
+        vs->lsock_tag = qio_channel_add_watch(
+            QIO_CHANNEL(vs->lsock),
+            G_IO_IN, vnc_listen_io, vs, NULL);
         if (vs->ws_enabled) {
-            qemu_set_fd_handler(vs->lwebsock, vnc_listen_websocket_read,
-                                NULL, vs);
+            vs->lwebsock_tag = qio_channel_add_watch(
+                QIO_CHANNEL(vs->lwebsock),
+                G_IO_IN, vnc_listen_io, vs, NULL);
         }
     }
-    qemu_opts_del(sopts);
-    qemu_opts_del(wsopts);
+
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
     return;
 
 fail:
-    qemu_opts_del(sopts);
-    qemu_opts_del(wsopts);
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
     vs->enabled = false;
     vs->ws_enabled = false;
 }
@@ -3728,11 +3814,17 @@ fail:
 void vnc_display_add_client(const char *id, int csock, bool skipauth)
 {
     VncDisplay *vs = vnc_display_find(id);
+    QIOChannelSocket *sioc;
 
     if (!vs) {
         return;
     }
-    vnc_connect(vs, csock, skipauth, false);
+
+    sioc = qio_channel_socket_new_fd(csock, NULL);
+    if (sioc) {
+        vnc_connect(vs, sioc, skipauth, false);
+        object_unref(OBJECT(sioc));
+    }
 }
 
 static void vnc_auto_assign_id(QemuOptsList *olist, QemuOpts *opts)
@@ -3775,9 +3867,7 @@ int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp)
     vnc_display_init(id);
     vnc_display_open(id, &local_err);
     if (local_err != NULL) {
-        error_report("Failed to start VNC server: %s",
-                     error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "Failed to start VNC server: ");
         exit(1);
     }
     return 0;
@@ -3787,4 +3877,4 @@ static void vnc_register_config(void)
 {
     qemu_add_opts(&qemu_vnc_opts);
 }
-machine_init(vnc_register_config);
+opts_init(vnc_register_config);
diff --git a/qemu/ui/vnc.h b/qemu/ui/vnc.h
index 814d720df..81a326116 100644
--- a/qemu/ui/vnc.h
+++ b/qemu/ui/vnc.h
@@ -33,8 +33,11 @@
 #include "ui/console.h"
 #include "audio/audio.h"
 #include "qemu/bitmap.h"
+#include "crypto/tlssession.h"
+#include "qemu/buffer.h"
+#include "io/channel-socket.h"
+#include "io/channel-tls.h"
 #include <zlib.h>
-#include <stdbool.h>
 
 #include "keymaps.h"
 #include "vnc-palette.h"
@@ -55,13 +58,6 @@
  *
  *****************************************************************************/
 
-typedef struct Buffer
-{
-    size_t capacity;
-    size_t offset;
-    uint8_t *buffer;
-} Buffer;
-
 typedef struct VncState VncState;
 typedef struct VncJob VncJob;
 typedef struct VncRect VncRect;
@@ -101,10 +97,7 @@ typedef void VncSendHextileTile(VncState *vs,
 
 typedef struct VncDisplay VncDisplay;
 
-#ifdef CONFIG_VNC_TLS
-#include "vnc-tls.h"
 #include "vnc-auth-vencrypt.h"
-#endif
 #ifdef CONFIG_VNC_SASL
 #include "vnc-auth-sasl.h"
 #endif
@@ -153,8 +146,10 @@ struct VncDisplay
     int num_exclusive;
     int connections_limit;
     VncSharePolicy share_policy;
-    int lsock;
-    int lwebsock;
+    QIOChannelSocket *lsock;
+    guint lsock_tag;
+    QIOChannelSocket *lwebsock;
+    guint lwebsock_tag;
     bool ws_enabled;
     DisplaySurface *ds;
     DisplayChangeListener dcl;
@@ -181,9 +176,8 @@ struct VncDisplay
     bool ws_tls; /* Used by websockets */
     bool lossy;
     bool non_adaptive;
-#ifdef CONFIG_VNC_TLS
-    VncDisplayTLS tls;
-#endif
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
 #ifdef CONFIG_VNC_SASL
     VncDisplaySASL sasl;
 #endif
@@ -257,7 +251,10 @@ struct VncJob
 
 struct VncState
 {
-    int csock;
+    QIOChannelSocket *sioc; /* The underlying socket */
+    QIOChannel *ioc; /* The channel currently used for I/O */
+    guint ioc_tag;
+    gboolean disconnecting;
 
     DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT], VNC_DIRTY_BITS);
     uint8_t **lossy_rect; /* Not an Array to avoid costly memcpy in
@@ -284,9 +281,7 @@ struct VncState
     int auth;
     int subauth; /* Used by VeNCrypt */
     char challenge[VNC_AUTH_CHALLENGE_SIZE];
-#ifdef CONFIG_VNC_TLS
-    VncStateTLS tls;
-#endif
+    QCryptoTLSSession *tls; /* Borrowed pointer from channel, don't free */
 #ifdef CONFIG_VNC_SASL
     VncStateSASL sasl;
 #endif
@@ -297,10 +292,6 @@ struct VncState
 
     Buffer output;
     Buffer input;
-    Buffer ws_input;
-    Buffer ws_output;
-    size_t ws_payload_remain;
-    WsMask ws_payload_mask;
     /* current output mode information */
     VncWritePixels *write_pixels;
     PixelFormat client_pf;
@@ -510,11 +501,12 @@ enum {
  *****************************************************************************/
 
 /* Event loop functions */
-void vnc_client_read(void *opaque);
-void vnc_client_write(void *opaque);
+gboolean vnc_client_io(QIOChannel *ioc,
+                       GIOCondition condition,
+                       void *opaque);
 
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
 
 /* Protocol I/O functions */
 void vnc_write(VncState *vs, const void *data, size_t len);
@@ -533,25 +525,14 @@ uint32_t read_u32(uint8_t *data, size_t offset);
 
 /* Protocol stage functions */
 void vnc_client_error(VncState *vs);
-int vnc_client_io_error(VncState *vs, int ret, int last_errno);
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, Error **errp);
 
 void start_client_init(VncState *vs);
 void start_auth_vnc(VncState *vs);
 
-/* Buffer management */
-void buffer_reserve(Buffer *buffer, size_t len);
-void buffer_reset(Buffer *buffer);
-void buffer_free(Buffer *buffer);
-void buffer_append(Buffer *buffer, const void *data, size_t len);
-void buffer_advance(Buffer *buf, size_t len);
-uint8_t *buffer_end(Buffer *buffer);
-
 
 /* Misc helpers */
 
-char *vnc_socket_local_addr(const char *format, int fd);
-char *vnc_socket_remote_addr(const char *format, int fd);
-
 static inline uint32_t vnc_has_feature(VncState *vs, int feature) {
     return (vs->features & (1 << feature));
 }
diff --git a/qemu/ui/x_keymap.c b/qemu/ui/x_keymap.c
index 1a773174f..27884851d 100644
--- a/qemu/ui/x_keymap.c
+++ b/qemu/ui/x_keymap.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "x_keymap.h"
 
diff --git a/qemu/user-exec.c b/qemu/user-exec.c
index ed9a07f15..d8d597baf 100644
--- a/qemu/user-exec.c
+++ b/qemu/user-exec.c
@@ -16,7 +16,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg.h"
@@ -33,7 +33,6 @@
 #undef ESI
 #undef EDI
 #undef EIP
-#include <signal.h>
 #ifdef __linux__
 #include <sys/ucontext.h>
 #endif
@@ -92,8 +91,8 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
     int ret;
 
 #if defined(DEBUG_SIGNAL)
-    qemu_printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
-                pc, address, is_write, *(unsigned long *)old_set);
+    printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
+           pc, address, is_write, *(unsigned long *)old_set);
 #endif
     /* XXX: locking issue */
     if (is_write && h2g_valid(address)
diff --git a/qemu/util/Makefile.objs b/qemu/util/Makefile.objs
index 114d6578c..a8a777ec4 100644
--- a/qemu/util/Makefile.objs
+++ b/qemu/util/Makefile.objs
@@ -1,13 +1,20 @@
 util-obj-y = osdep.o cutils.o unicode.o qemu-timer-common.o
-util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
-util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-$(CONFIG_POSIX) += event_notifier-posix.o
+util-obj-$(CONFIG_POSIX) += mmap-alloc.o
+util-obj-$(CONFIG_POSIX) += oslib-posix.o
+util-obj-$(CONFIG_POSIX) += qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += qemu-thread-posix.o
+util-obj-$(CONFIG_WIN32) += event_notifier-win32.o
+util-obj-$(CONFIG_POSIX) += memfd.o
+util-obj-$(CONFIG_WIN32) += oslib-win32.o
+util-obj-$(CONFIG_WIN32) += qemu-thread-win32.o
 util-obj-y += envlist.o path.o module.o
 util-obj-$(call lnot,$(CONFIG_INT128)) += host-utils.o
 util-obj-y += bitmap.o bitops.o hbitmap.o
 util-obj-y += fifo8.o
 util-obj-y += acl.o
 util-obj-y += error.o qemu-error.o
-util-obj-$(CONFIG_POSIX) += compatfd.o
 util-obj-y += id.o
 util-obj-y += iov.o qemu-config.o qemu-sockets.o uri.o notify.o
 util-obj-y += qemu-option.o qemu-progress.o
@@ -18,3 +25,10 @@ util-obj-y += getauxval.o
 util-obj-y += readline.o
 util-obj-y += rfifolock.o
 util-obj-y += rcu.o
+util-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+util-obj-y += qemu-coroutine-sleep.o
+util-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
+util-obj-y += buffer.o
+util-obj-y += timed-average.o
+util-obj-y += base64.o
+util-obj-y += log.o
diff --git a/qemu/util/acl.c b/qemu/util/acl.c
index 571d68615..723b6a89b 100644
--- a/qemu/util/acl.c
+++ b/qemu/util/acl.c
@@ -23,6 +23,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/acl.h"
 
diff --git a/qemu/util/base64.c b/qemu/util/base64.c
new file mode 100644
index 000000000..9d3c46cbc
--- /dev/null
+++ b/qemu/util/base64.c
@@ -0,0 +1,60 @@
+/*
+ * QEMU base64 helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/base64.h"
+
+static const char *base64_valid_chars =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n";
+
+uint8_t *qbase64_decode(const char *input,
+                        size_t in_len,
+                        size_t *out_len,
+                        Error **errp)
+{
+    *out_len = 0;
+
+    if (in_len != -1) {
+        /* Lack of NUL terminator is an error */
+        if (input[in_len] != '\0') {
+            error_setg(errp, "Base64 data is not NUL terminated");
+            return NULL;
+        }
+        /* Check there's no NULs embedded since we expect
+         * this to be valid base64 data */
+        if (memchr(input, '\0', in_len) != NULL) {
+            error_setg(errp, "Base64 data contains embedded NUL characters");
+            return NULL;
+        }
+
+        /* Now we know its a valid nul terminated string
+         * strspn is safe to use... */
+    } else {
+        in_len = strlen(input);
+    }
+
+    if (strspn(input, base64_valid_chars) != in_len) {
+        error_setg(errp, "Base64 data contains invalid characters");
+        return NULL;
+    }
+
+    return g_base64_decode(input, out_len);
+}
diff --git a/qemu/util/bitmap.c b/qemu/util/bitmap.c
index 300a68e38..40aadfb4f 100644
--- a/qemu/util/bitmap.c
+++ b/qemu/util/bitmap.c
@@ -9,12 +9,13 @@
  * Version 2.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 #include "qemu/bitmap.h"
 #include "qemu/atomic.h"
 
 /*
- * bitmaps provide an array of bits, implemented using an an
+ * bitmaps provide an array of bits, implemented using an
  * array of unsigned longs.  The number of valid bits in a
  * given bitmap does _not_ need to be an exact multiple of
  * BITS_PER_LONG.
diff --git a/qemu/util/bitops.c b/qemu/util/bitops.c
index 227c38b88..b0c35dd5f 100644
--- a/qemu/util/bitops.c
+++ b/qemu/util/bitops.c
@@ -11,6 +11,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 #define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
diff --git a/qemu/util/buffer.c b/qemu/util/buffer.c
new file mode 100644
index 000000000..a6118bf5b
--- /dev/null
+++ b/qemu/util/buffer.c
@@ -0,0 +1,172 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/buffer.h"
+#include "trace.h"
+
+#define BUFFER_MIN_INIT_SIZE     4096
+#define BUFFER_MIN_SHRINK_SIZE  65536
+
+/* define the factor alpha for the expentional smoothing
+ * that is used in the average size calculation. a shift
+ * of 7 results in an alpha of 1/2^7. */
+#define BUFFER_AVG_SIZE_SHIFT       7
+
+static size_t buffer_req_size(Buffer *buffer, size_t len)
+{
+    return MAX(BUFFER_MIN_INIT_SIZE,
+               pow2ceil(buffer->offset + len));
+}
+
+static void buffer_adj_size(Buffer *buffer, size_t len)
+{
+    size_t old = buffer->capacity;
+    buffer->capacity = buffer_req_size(buffer, len);
+    buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
+    trace_buffer_resize(buffer->name ?: "unnamed",
+                        old, buffer->capacity);
+
+    /* make it even harder for the buffer to shrink, reset average size
+     * to currenty capacity if it is larger than the average. */
+    buffer->avg_size = MAX(buffer->avg_size,
+                           buffer->capacity << BUFFER_AVG_SIZE_SHIFT);
+}
+
+void buffer_init(Buffer *buffer, const char *name, ...)
+{
+    va_list ap;
+
+    va_start(ap, name);
+    buffer->name = g_strdup_vprintf(name, ap);
+    va_end(ap);
+}
+
+static uint64_t buffer_get_avg_size(Buffer *buffer)
+{
+    return buffer->avg_size >> BUFFER_AVG_SIZE_SHIFT;
+}
+
+void buffer_shrink(Buffer *buffer)
+{
+    size_t new;
+
+    /* Calculate the average size of the buffer as
+     * avg_size = avg_size * ( 1 - a ) + required_size * a
+     * where a is 1 / 2 ^ BUFFER_AVG_SIZE_SHIFT. */
+    buffer->avg_size *= (1 << BUFFER_AVG_SIZE_SHIFT) - 1;
+    buffer->avg_size >>= BUFFER_AVG_SIZE_SHIFT;
+    buffer->avg_size += buffer_req_size(buffer, 0);
+
+    /* And then only shrink if the average size of the buffer is much
+     * too big, to avoid bumping up & down the buffers all the time.
+     * realloc() isn't exactly cheap ...  */
+    new = buffer_req_size(buffer, buffer_get_avg_size(buffer));
+    if (new < buffer->capacity >> 3 &&
+        new >= BUFFER_MIN_SHRINK_SIZE) {
+        buffer_adj_size(buffer, buffer_get_avg_size(buffer));
+    }
+
+    buffer_adj_size(buffer, 0);
+}
+
+void buffer_reserve(Buffer *buffer, size_t len)
+{
+    if ((buffer->capacity - buffer->offset) < len) {
+        buffer_adj_size(buffer, len);
+    }
+}
+
+gboolean buffer_empty(Buffer *buffer)
+{
+    return buffer->offset == 0;
+}
+
+uint8_t *buffer_end(Buffer *buffer)
+{
+    return buffer->buffer + buffer->offset;
+}
+
+void buffer_reset(Buffer *buffer)
+{
+    buffer->offset = 0;
+    buffer_shrink(buffer);
+}
+
+void buffer_free(Buffer *buffer)
+{
+    trace_buffer_free(buffer->name ?: "unnamed", buffer->capacity);
+    g_free(buffer->buffer);
+    g_free(buffer->name);
+    buffer->offset = 0;
+    buffer->capacity = 0;
+    buffer->buffer = NULL;
+    buffer->name = NULL;
+}
+
+void buffer_append(Buffer *buffer, const void *data, size_t len)
+{
+    memcpy(buffer->buffer + buffer->offset, data, len);
+    buffer->offset += len;
+}
+
+void buffer_advance(Buffer *buffer, size_t len)
+{
+    memmove(buffer->buffer, buffer->buffer + len,
+            (buffer->offset - len));
+    buffer->offset -= len;
+    buffer_shrink(buffer);
+}
+
+void buffer_move_empty(Buffer *to, Buffer *from)
+{
+    trace_buffer_move_empty(to->name ?: "unnamed",
+                            from->offset,
+                            from->name ?: "unnamed");
+    assert(to->offset == 0);
+
+    g_free(to->buffer);
+    to->offset = from->offset;
+    to->capacity = from->capacity;
+    to->buffer = from->buffer;
+
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
+
+void buffer_move(Buffer *to, Buffer *from)
+{
+    if (to->offset == 0) {
+        buffer_move_empty(to, from);
+        return;
+    }
+
+    trace_buffer_move(to->name ?: "unnamed",
+                      from->offset,
+                      from->name ?: "unnamed");
+    buffer_reserve(to, from->offset);
+    buffer_append(to, from->buffer, from->offset);
+
+    g_free(from->buffer);
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
diff --git a/qemu/util/compatfd.c b/qemu/util/compatfd.c
index e8571502b..9a43042ae 100644
--- a/qemu/util/compatfd.c
+++ b/qemu/util/compatfd.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/compatfd.h"
 #include "qemu/thread.h"
diff --git a/qemu/coroutine-gthread.c b/qemu/util/coroutine-gthread.c
index 6bd6d6b22..fb697eb0b 100644
--- a/qemu/coroutine-gthread.c
+++ b/qemu/util/coroutine-gthread.c
@@ -18,9 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/qemu/coroutine-sigaltstack.c b/qemu/util/coroutine-sigaltstack.c
index 63519fffc..a7c336655 100644
--- a/qemu/coroutine-sigaltstack.c
+++ b/qemu/util/coroutine-sigaltstack.c
@@ -25,13 +25,10 @@
 #ifdef _FORTIFY_SOURCE
 #undef _FORTIFY_SOURCE
 #endif
-#include <stdlib.h>
-#include <setjmp.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include <pthread.h>
-#include <signal.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct {
     Coroutine base;
diff --git a/qemu/coroutine-ucontext.c b/qemu/util/coroutine-ucontext.c
index 259fcb48a..2bb7e10d4 100644
--- a/qemu/coroutine-ucontext.c
+++ b/qemu/util/coroutine-ucontext.c
@@ -22,12 +22,10 @@
 #ifdef _FORTIFY_SOURCE
 #undef _FORTIFY_SOURCE
 #endif
-#include <stdlib.h>
-#include <setjmp.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include <ucontext.h>
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 #ifdef CONFIG_VALGRIND_H
 #include <valgrind/valgrind.h>
diff --git a/qemu/coroutine-win32.c b/qemu/util/coroutine-win32.c
index 17ace37de..02e28e825 100644
--- a/qemu/coroutine-win32.c
+++ b/qemu/util/coroutine-win32.c
@@ -22,8 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine_int.h"
 
 typedef struct
 {
diff --git a/qemu/util/crc32c.c b/qemu/util/crc32c.c
index 886632780..7e99555c1 100644
--- a/qemu/util/crc32c.c
+++ b/qemu/util/crc32c.c
@@ -25,6 +25,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/crc32c.h"
 
diff --git a/qemu/util/cutils.c b/qemu/util/cutils.c
index 5d1c9ebe0..43d1afbbe 100644
--- a/qemu/util/cutils.c
+++ b/qemu/util/cutils.c
@@ -21,15 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/host-utils.h"
 #include <math.h>
-#include <limits.h>
-#include <errno.h>
 
 #include "qemu/sockets.h"
 #include "qemu/iov.h"
 #include "net/net.h"
+#include "qemu/cutils.h"
 
 void strpadcpy(char *buf, int buf_size, const char *str, char pad)
 {
@@ -145,11 +145,6 @@ time_t mktimegm(struct tm *tm)
     return t;
 }
 
-int qemu_fls(int i)
-{
-    return 32 - clz32(i);
-}
-
 /*
  * Make sure data goes on disk, but if possible do not bother to
  * write out the inode just for timestamp updates.
@@ -166,6 +161,46 @@ int qemu_fdatasync(int fd)
 #endif
 }
 
+/* vector definitions */
+#ifdef __ALTIVEC__
+#include <altivec.h>
+/* The altivec.h header says we're allowed to undef these for
+ * C++ compatibility.  Here we don't care about C++, but we
+ * undef them anyway to avoid namespace pollution.
+ */
+#undef vector
+#undef pixel
+#undef bool
+#define VECTYPE        __vector unsigned char
+#define SPLAT(p)       vec_splat(vec_ld(0, p), 0)
+#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
+#define VEC_OR(v1, v2) ((v1) | (v2))
+/* altivec.h may redefine the bool macro as vector type.
+ * Reset it to POSIX semantics. */
+#define bool _Bool
+#elif defined __SSE2__
+#include <emmintrin.h>
+#define VECTYPE        __m128i
+#define SPLAT(p)       _mm_set1_epi8(*(p))
+#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
+#define VEC_OR(v1, v2) (_mm_or_si128(v1, v2))
+#else
+#define VECTYPE        unsigned long
+#define SPLAT(p)       (*(p) * (~0UL / 255))
+#define ALL_EQ(v1, v2) ((v1) == (v2))
+#define VEC_OR(v1, v2) ((v1) | (v2))
+#endif
+
+#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
+
+static bool
+can_use_buffer_find_nonzero_offset_inner(const void *buf, size_t len)
+{
+    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
+                   * sizeof(VECTYPE)) == 0
+            && ((uintptr_t) buf) % sizeof(VECTYPE) == 0);
+}
+
 /*
  * Searches for an area with non-zero content in a buffer
  *
@@ -174,8 +209,8 @@ int qemu_fdatasync(int fd)
  * and addr must be a multiple of sizeof(VECTYPE) due to
  * restriction of optimizations in this function.
  *
- * can_use_buffer_find_nonzero_offset() can be used to check
- * these requirements.
+ * can_use_buffer_find_nonzero_offset_inner() can be used to
+ * check these requirements.
  *
  * The return value is the offset of the non-zero area rounded
  * down to a multiple of sizeof(VECTYPE) for the first
@@ -186,13 +221,13 @@ int qemu_fdatasync(int fd)
  * If the buffer is all zero the return value is equal to len.
  */
 
-size_t buffer_find_nonzero_offset(const void *buf, size_t len)
+static size_t buffer_find_nonzero_offset_inner(const void *buf, size_t len)
 {
     const VECTYPE *p = buf;
     const VECTYPE zero = (VECTYPE){0};
     size_t i;
 
-    assert(can_use_buffer_find_nonzero_offset(buf, len));
+    assert(can_use_buffer_find_nonzero_offset_inner(buf, len));
 
     if (!len) {
         return 0;
@@ -222,6 +257,114 @@ size_t buffer_find_nonzero_offset(const void *buf, size_t len)
 }
 
 /*
+ * GCC before version 4.9 has a bug which will cause the target
+ * attribute work incorrectly and failed to compile in some case,
+ * restrict the gcc version to 4.9+ to prevent the failure.
+ */
+
+#if defined CONFIG_AVX2_OPT && QEMU_GNUC_PREREQ(4, 9)
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#include <cpuid.h>
+#include <immintrin.h>
+
+#define AVX2_VECTYPE        __m256i
+#define AVX2_SPLAT(p)       _mm256_set1_epi8(*(p))
+#define AVX2_ALL_EQ(v1, v2) \
+    (_mm256_movemask_epi8(_mm256_cmpeq_epi8(v1, v2)) == 0xFFFFFFFF)
+#define AVX2_VEC_OR(v1, v2) (_mm256_or_si256(v1, v2))
+
+static bool
+can_use_buffer_find_nonzero_offset_avx2(const void *buf, size_t len)
+{
+    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
+                   * sizeof(AVX2_VECTYPE)) == 0
+            && ((uintptr_t) buf) % sizeof(AVX2_VECTYPE) == 0);
+}
+
+static size_t buffer_find_nonzero_offset_avx2(const void *buf, size_t len)
+{
+    const AVX2_VECTYPE *p = buf;
+    const AVX2_VECTYPE zero = (AVX2_VECTYPE){0};
+    size_t i;
+
+    assert(can_use_buffer_find_nonzero_offset_avx2(buf, len));
+
+    if (!len) {
+        return 0;
+    }
+
+    for (i = 0; i < BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR; i++) {
+        if (!AVX2_ALL_EQ(p[i], zero)) {
+            return i * sizeof(AVX2_VECTYPE);
+        }
+    }
+
+    for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR;
+         i < len / sizeof(AVX2_VECTYPE);
+         i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) {
+        AVX2_VECTYPE tmp0 = AVX2_VEC_OR(p[i + 0], p[i + 1]);
+        AVX2_VECTYPE tmp1 = AVX2_VEC_OR(p[i + 2], p[i + 3]);
+        AVX2_VECTYPE tmp2 = AVX2_VEC_OR(p[i + 4], p[i + 5]);
+        AVX2_VECTYPE tmp3 = AVX2_VEC_OR(p[i + 6], p[i + 7]);
+        AVX2_VECTYPE tmp01 = AVX2_VEC_OR(tmp0, tmp1);
+        AVX2_VECTYPE tmp23 = AVX2_VEC_OR(tmp2, tmp3);
+        if (!AVX2_ALL_EQ(AVX2_VEC_OR(tmp01, tmp23), zero)) {
+            break;
+        }
+    }
+
+    return i * sizeof(AVX2_VECTYPE);
+}
+
+static bool avx2_support(void)
+{
+    int a, b, c, d;
+
+    if (__get_cpuid_max(0, NULL) < 7) {
+        return false;
+    }
+
+    __cpuid_count(7, 0, a, b, c, d);
+
+    return b & bit_AVX2;
+}
+
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len) \
+         __attribute__ ((ifunc("can_use_buffer_find_nonzero_offset_ifunc")));
+size_t buffer_find_nonzero_offset(const void *buf, size_t len) \
+         __attribute__ ((ifunc("buffer_find_nonzero_offset_ifunc")));
+
+static void *buffer_find_nonzero_offset_ifunc(void)
+{
+    typeof(buffer_find_nonzero_offset) *func = (avx2_support()) ?
+        buffer_find_nonzero_offset_avx2 : buffer_find_nonzero_offset_inner;
+
+    return func;
+}
+
+static void *can_use_buffer_find_nonzero_offset_ifunc(void)
+{
+    typeof(can_use_buffer_find_nonzero_offset) *func = (avx2_support()) ?
+        can_use_buffer_find_nonzero_offset_avx2 :
+        can_use_buffer_find_nonzero_offset_inner;
+
+    return func;
+}
+#pragma GCC pop_options
+#else
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    return can_use_buffer_find_nonzero_offset_inner(buf, len);
+}
+
+size_t buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    return buffer_find_nonzero_offset_inner(buf, len);
+}
+#endif
+
+/*
  * Checks if a buffer is all zeroes
  *
  * Attention! The len must be a multiple of 4 * sizeof(long) due to
@@ -281,19 +424,19 @@ int fcntl_setfl(int fd, int flag)
 static int64_t suffix_mul(char suffix, int64_t unit)
 {
     switch (qemu_toupper(suffix)) {
-    case STRTOSZ_DEFSUFFIX_B:
+    case QEMU_STRTOSZ_DEFSUFFIX_B:
         return 1;
-    case STRTOSZ_DEFSUFFIX_KB:
+    case QEMU_STRTOSZ_DEFSUFFIX_KB:
         return unit;
-    case STRTOSZ_DEFSUFFIX_MB:
+    case QEMU_STRTOSZ_DEFSUFFIX_MB:
         return unit * unit;
-    case STRTOSZ_DEFSUFFIX_GB:
+    case QEMU_STRTOSZ_DEFSUFFIX_GB:
         return unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_TB:
+    case QEMU_STRTOSZ_DEFSUFFIX_TB:
         return unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_PB:
+    case QEMU_STRTOSZ_DEFSUFFIX_PB:
         return unit * unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_EB:
+    case QEMU_STRTOSZ_DEFSUFFIX_EB:
         return unit * unit * unit * unit * unit * unit;
     }
     return -1;
@@ -305,7 +448,7 @@ static int64_t suffix_mul(char suffix, int64_t unit)
  * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
  * other error.
  */
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit)
 {
     int64_t retval = -EINVAL;
@@ -348,14 +491,165 @@ fail:
     return retval;
 }
 
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix)
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix)
 {
-    return strtosz_suffix_unit(nptr, end, default_suffix, 1024);
+    return qemu_strtosz_suffix_unit(nptr, end, default_suffix, 1024);
 }
 
-int64_t strtosz(const char *nptr, char **end)
+int64_t qemu_strtosz(const char *nptr, char **end)
 {
-    return strtosz_suffix(nptr, end, STRTOSZ_DEFSUFFIX_MB);
+    return qemu_strtosz_suffix(nptr, end, QEMU_STRTOSZ_DEFSUFFIX_MB);
+}
+
+/**
+ * Helper function for qemu_strto*l() functions.
+ */
+static int check_strtox_error(const char *p, char *endptr, const char **next,
+                              int err)
+{
+    /* If no conversion was performed, prefer BSD behavior over glibc
+     * behavior.
+     */
+    if (err == 0 && endptr == p) {
+        err = EINVAL;
+    }
+    if (!next && *endptr) {
+        return -EINVAL;
+    }
+    if (next) {
+        *next = endptr;
+    }
+    return -err;
+}
+
+/**
+ * QEMU wrappers for strtol(), strtoll(), strtoul(), strotull() C functions.
+ *
+ * Convert ASCII string @nptr to a long integer value
+ * from the given @base. Parameters @nptr, @endptr, @base
+ * follows same semantics as strtol() C function.
+ *
+ * Unlike from strtol() function, if @endptr is not NULL, this
+ * function will return -EINVAL whenever it cannot fully convert
+ * the string in @nptr with given @base to a long. This function returns
+ * the result of the conversion only through the @result parameter.
+ *
+ * If NULL is passed in @endptr, then the whole string in @ntpr
+ * is a number otherwise it returns -EINVAL.
+ *
+ * RETURN VALUE
+ * Unlike from strtol() function, this wrapper returns either
+ * -EINVAL or the errno set by strtol() function (e.g -ERANGE).
+ * If the conversion overflows, -ERANGE is returned, and @result
+ * is set to the max value of the desired type
+ * (e.g. LONG_MAX, LLONG_MAX, ULONG_MAX, ULLONG_MAX). If the case
+ * of underflow, -ERANGE is returned, and @result is set to the min
+ * value of the desired type. For strtol(), strtoll(), @result is set to
+ * LONG_MIN, LLONG_MIN, respectively, and for strtoul(), strtoull() it
+ * is set to 0.
+ */
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtol(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long integer.
+ *
+ * If string contains a negative number, value will be converted to
+ * the unsigned representation of the signed value, unless the original
+ * (nonnegated) value would overflow, in this case, it will set @result
+ * to ULONG_MAX, and return ERANGE.
+ *
+ * The same behavior holds, for qemu_strtoull() but sets @result to
+ * ULLONG_MAX instead of ULONG_MAX.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoul(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to a long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoll(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoull(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
 }
 
 /**
@@ -474,29 +768,6 @@ int qemu_parse_fd(const char *param)
     return fd;
 }
 
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value)
-{
-    if (!is_power_of_2(value)) {
-        value = 0x8000000000000000ULL >> clz64(value);
-    }
-    return value;
-}
-
-/* round up to the nearest power of 2 (0 if overflow) */
-uint64_t pow2ceil(uint64_t value)
-{
-    uint8_t nlz = clz64(value);
-
-    if (is_power_of_2(value)) {
-        return value;
-    }
-    if (!nlz) {
-        return 0;
-    }
-    return 1ULL << (64 - nlz);
-}
-
 /*
  * Implementation of  ULEB128 (http://en.wikipedia.org/wiki/LEB128)
  * Input is limited to 14-bit numbers
diff --git a/qemu/util/envlist.c b/qemu/util/envlist.c
index 099a544a4..e86857e70 100644
--- a/qemu/util/envlist.c
+++ b/qemu/util/envlist.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/envlist.h"
diff --git a/qemu/util/error.c b/qemu/util/error.c
index 14f435187..cae251173 100644
--- a/qemu/util/error.c
+++ b/qemu/util/error.c
@@ -2,30 +2,53 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>,
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qapi/error.h"
+#include "qemu-common.h"
 #include "qemu/error-report.h"
 
 struct Error
 {
     char *msg;
     ErrorClass err_class;
+    const char *src, *func;
+    int line;
+    GString *hint;
 };
 
 Error *error_abort;
+Error *error_fatal;
+
+static void error_handle_fatal(Error **errp, Error *err)
+{
+    if (errp == &error_abort) {
+        fprintf(stderr, "Unexpected error in %s() at %s:%d:\n",
+                err->func, err->src, err->line);
+        error_report_err(err);
+        abort();
+    }
+    if (errp == &error_fatal) {
+        error_report_err(err);
+        exit(1);
+    }
+}
 
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
+static void error_setv(Error **errp,
+                       const char *src, int line, const char *func,
+                       ErrorClass err_class, const char *fmt, va_list ap,
+                       const char *suffix)
 {
     Error *err;
-    va_list ap;
     int saved_errno = errno;
 
     if (errp == NULL) {
@@ -34,99 +57,140 @@ void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
     assert(*errp == NULL);
 
     err = g_malloc0(sizeof(*err));
-
-    va_start(ap, fmt);
     err->msg = g_strdup_vprintf(fmt, ap);
-    va_end(ap);
-    err->err_class = err_class;
-
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
+    if (suffix) {
+        char *msg = err->msg;
+        err->msg = g_strdup_printf("%s: %s", msg, suffix);
+        g_free(msg);
     }
+    err->err_class = err_class;
+    err->src = src;
+    err->line = line;
+    err->func = func;
 
+    error_handle_fatal(errp, err);
     *errp = err;
 
     errno = saved_errno;
 }
 
-void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, err_class, fmt, ap, NULL);
+    va_end(ap);
+}
+
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap, NULL);
+    va_end(ap);
+}
+
+void error_setg_errno_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int os_errno, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
     va_list ap;
     int saved_errno = errno;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
-
-    err = g_malloc0(sizeof(*err));
 
     va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
-    if (os_errno != 0) {
-        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
-        g_free(msg1);
-    } else {
-        err->msg = msg1;
-    }
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap,
+               os_errno != 0 ? strerror(os_errno) : NULL);
     va_end(ap);
-    err->err_class = err_class;
 
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
+    errno = saved_errno;
+}
+
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename)
+{
+    error_setg_errno_internal(errp, src, line, func, os_errno,
+                              "Could not open '%s'", filename);
+}
+
+void error_vprepend(Error **errp, const char *fmt, va_list ap)
+{
+    GString *newmsg;
+
+    if (!errp) {
+        return;
     }
 
-    *errp = err;
+    newmsg = g_string_new(NULL);
+    g_string_vprintf(newmsg, fmt, ap);
+    g_string_append(newmsg, (*errp)->msg);
+    (*errp)->msg = g_string_free(newmsg, 0);
+}
 
-    errno = saved_errno;
+void error_prepend(Error **errp, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vprepend(errp, fmt, ap);
+    va_end(ap);
 }
 
-void error_setg_file_open(Error **errp, int os_errno, const char *filename)
+void error_append_hint(Error **errp, const char *fmt, ...)
 {
-    error_setg_errno(errp, os_errno, "Could not open '%s'", filename);
+    va_list ap;
+    int saved_errno = errno;
+    Error *err;
+
+    if (!errp) {
+        return;
+    }
+    err = *errp;
+    assert(err && errp != &error_abort && errp != &error_fatal);
+
+    if (!err->hint) {
+        err->hint = g_string_new(NULL);
+    }
+    va_start(ap, fmt);
+    g_string_append_vprintf(err->hint, fmt, ap);
+    va_end(ap);
+
+    errno = saved_errno;
 }
 
 #ifdef _WIN32
 
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
     va_list ap;
+    char *suffix = NULL;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
 
-    err = g_malloc0(sizeof(*err));
-
-    va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
     if (win32_err != 0) {
-        char *msg2 = g_win32_error_message(win32_err);
-        err->msg = g_strdup_printf("%s: %s (error: %x)", msg1, msg2,
-                                   (unsigned)win32_err);
-        g_free(msg2);
-        g_free(msg1);
-    } else {
-        err->msg = msg1;
+        suffix = g_win32_error_message(win32_err);
     }
-    va_end(ap);
-    err->err_class = err_class;
 
-    if (errp == &error_abort) {
-        error_report_err(err);
-        abort();
-    }
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR,
+               fmt, ap, suffix);
+    va_end(ap);
 
-    *errp = err;
+    g_free(suffix);
 }
 
 #endif
@@ -138,6 +202,12 @@ Error *error_copy(const Error *err)
     err_new = g_malloc0(sizeof(*err));
     err_new->msg = g_strdup(err->msg);
     err_new->err_class = err->err_class;
+    err_new->src = err->src;
+    err_new->line = err->line;
+    err_new->func = err->func;
+    if (err->hint) {
+        err_new->hint = g_string_new(err->hint->str);
+    }
 
     return err_new;
 }
@@ -155,25 +225,49 @@ const char *error_get_pretty(Error *err)
 void error_report_err(Error *err)
 {
     error_report("%s", error_get_pretty(err));
+    if (err->hint) {
+        error_printf_unless_qmp("%s", err->hint->str);
+    }
     error_free(err);
 }
 
+void error_reportf_err(Error *err, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vprepend(&err, fmt, ap);
+    va_end(ap);
+    error_report_err(err);
+}
+
 void error_free(Error *err)
 {
     if (err) {
         g_free(err->msg);
+        if (err->hint) {
+            g_string_free(err->hint, true);
+        }
         g_free(err);
     }
 }
 
+void error_free_or_abort(Error **errp)
+{
+    assert(errp && *errp);
+    error_free(*errp);
+    *errp = NULL;
+}
+
 void error_propagate(Error **dst_errp, Error *local_err)
 {
-    if (local_err && dst_errp == &error_abort) {
-        error_report_err(local_err);
-        abort();
-    } else if (dst_errp && !*dst_errp) {
+    if (!local_err) {
+        return;
+    }
+    error_handle_fatal(dst_errp, local_err);
+    if (dst_errp && !*dst_errp) {
         *dst_errp = local_err;
-    } else if (local_err) {
+    } else {
         error_free(local_err);
     }
 }
diff --git a/qemu/util/event_notifier-posix.c b/qemu/util/event_notifier-posix.c
index ed4ca2b01..c1f0d79b3 100644
--- a/qemu/util/event_notifier-posix.c
+++ b/qemu/util/event_notifier-posix.c
@@ -10,7 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/event_notifier.h"
 #include "sysemu/char.h"
 #include "qemu/main-loop.h"
@@ -19,11 +21,17 @@
 #include <sys/eventfd.h>
 #endif
 
+#ifdef CONFIG_EVENTFD
+/*
+ * Initialize @e with existing file descriptor @fd.
+ * @fd must be a genuine eventfd object, emulation with pipe won't do.
+ */
 void event_notifier_init_fd(EventNotifier *e, int fd)
 {
     e->rfd = fd;
     e->wfd = fd;
 }
+#endif
 
 int event_notifier_init(EventNotifier *e, int active)
 {
@@ -77,15 +85,17 @@ void event_notifier_cleanup(EventNotifier *e)
     close(e->wfd);
 }
 
-int event_notifier_get_fd(EventNotifier *e)
+int event_notifier_get_fd(const EventNotifier *e)
 {
     return e->rfd;
 }
 
 int event_notifier_set_handler(EventNotifier *e,
+                               bool is_external,
                                EventNotifierHandler *handler)
 {
-    qemu_set_fd_handler(e->rfd, (IOHandler *)handler, NULL, e);
+    aio_set_fd_handler(iohandler_get_aio_context(), e->rfd, is_external,
+                       (IOHandler *)handler, NULL, e);
     return 0;
 }
 
diff --git a/qemu/util/event_notifier-win32.c b/qemu/util/event_notifier-win32.c
index 6dbb530cf..de87df02d 100644
--- a/qemu/util/event_notifier-win32.c
+++ b/qemu/util/event_notifier-win32.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/event_notifier.h"
 #include "qemu/main-loop.h"
@@ -32,6 +33,7 @@ HANDLE event_notifier_get_handle(EventNotifier *e)
 }
 
 int event_notifier_set_handler(EventNotifier *e,
+                               bool is_external,
                                EventNotifierHandler *handler)
 {
     if (handler) {
diff --git a/qemu/util/fifo8.c b/qemu/util/fifo8.c
index 0ea5ad98e..5c64101b3 100644
--- a/qemu/util/fifo8.c
+++ b/qemu/util/fifo8.c
@@ -12,6 +12,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/fifo8.h"
 
diff --git a/qemu/util/getauxval.c b/qemu/util/getauxval.c
index 1732ace2b..0b3bae2dc 100644
--- a/qemu/util/getauxval.c
+++ b/qemu/util/getauxval.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
 #include "qemu/osdep.h"
+#include "qemu-common.h"
 
 #ifdef CONFIG_GETAUXVAL
 /* Don't inline this in qemu/osdep.h, because pulling in <sys/auxv.h> for
diff --git a/qemu/util/hbitmap.c b/qemu/util/hbitmap.c
index 50b888fd6..b22b87d0a 100644
--- a/qemu/util/hbitmap.c
+++ b/qemu/util/hbitmap.c
@@ -9,10 +9,8 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <glib.h>
-#include <assert.h>
 #include "qemu/osdep.h"
+#include <glib.h>
 #include "qemu/hbitmap.h"
 #include "qemu/host-utils.h"
 #include "trace.h"
diff --git a/qemu/util/hexdump.c b/qemu/util/hexdump.c
index 969b3406c..f879ff0ad 100644
--- a/qemu/util/hexdump.c
+++ b/qemu/util/hexdump.c
@@ -13,25 +13,37 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size)
 {
-    unsigned int b;
+    unsigned int b, len, i, c;
 
-    for (b = 0; b < size; b++) {
-        if ((b % 16) == 0) {
-            fprintf(fp, "%s: %04x:", prefix, b);
+    for (b = 0; b < size; b += 16) {
+        len = size - b;
+        if (len > 16) {
+            len = 16;
         }
-        if ((b % 4) == 0) {
-            fprintf(fp, " ");
+        fprintf(fp, "%s: %04x:", prefix, b);
+        for (i = 0; i < 16; i++) {
+            if ((i % 4) == 0) {
+                fprintf(fp, " ");
+            }
+            if (i < len) {
+                fprintf(fp, " %02x", (unsigned char)buf[b + i]);
+            } else {
+                fprintf(fp, "   ");
+            }
         }
-        fprintf(fp, " %02x", (unsigned char)buf[b]);
-        if ((b % 16) == 15) {
-            fprintf(fp, "\n");
+        fprintf(fp, " ");
+        for (i = 0; i < len; i++) {
+            c = buf[b + i];
+            if (c < ' ' || c > '~') {
+                c = '.';
+            }
+            fprintf(fp, "%c", c);
         }
-    }
-    if ((b % 16) != 0) {
         fprintf(fp, "\n");
     }
 }
diff --git a/qemu/util/host-utils.c b/qemu/util/host-utils.c
index 102e5bf30..b166e5758 100644
--- a/qemu/util/host-utils.c
+++ b/qemu/util/host-utils.c
@@ -23,8 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include "qemu/host-utils.h"
 
 /* Long integer helpers */
diff --git a/qemu/util/id.c b/qemu/util/id.c
index 09b22fb8f..614135295 100644
--- a/qemu/util/id.c
+++ b/qemu/util/id.c
@@ -10,7 +10,9 @@
  * or later.  See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/id.h"
 
 bool id_wellformed(const char *id)
 {
@@ -26,3 +28,40 @@ bool id_wellformed(const char *id)
     }
     return true;
 }
+
+#define ID_SPECIAL_CHAR '#'
+
+static const char *const id_subsys_str[ID_MAX] = {
+    [ID_QDEV]  = "qdev",
+    [ID_BLOCK] = "block",
+};
+
+/*
+ *  Generates an ID of the form PREFIX SUBSYSTEM NUMBER
+ *  where:
+ *
+ *  - PREFIX is the reserved character '#'
+ *  - SUBSYSTEM identifies the subsystem creating the ID
+ *  - NUMBER is a decimal number unique within SUBSYSTEM.
+ *
+ *    Example: "#block146"
+ *
+ * Note that these IDs do not satisfy id_wellformed().
+ *
+ * The caller is responsible for freeing the returned string with g_free()
+ */
+char *id_generate(IdSubSystems id)
+{
+    static uint64_t id_counters[ID_MAX];
+    uint32_t rnd;
+
+    assert(id < ARRAY_SIZE(id_subsys_str));
+    assert(id_subsys_str[id]);
+
+    rnd = g_random_int_range(0, 100);
+
+    return g_strdup_printf("%c%s%" PRIu64 "%02" PRId32, ID_SPECIAL_CHAR,
+                                                        id_subsys_str[id],
+                                                        id_counters[id]++,
+                                                        rnd);
+}
diff --git a/qemu/util/iov.c b/qemu/util/iov.c
index a0d5934e8..003fcce66 100644
--- a/qemu/util/iov.c
+++ b/qemu/util/iov.c
@@ -16,11 +16,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
+#include "qemu/cutils.h"
 
-size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
-                    size_t offset, const void *buf, size_t bytes)
+size_t iov_from_buf_full(const struct iovec *iov, unsigned int iov_cnt,
+                         size_t offset, const void *buf, size_t bytes)
 {
     size_t done;
     unsigned int i;
@@ -38,8 +41,8 @@ size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
     return done;
 }
 
-size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
-                  size_t offset, void *buf, size_t bytes)
+size_t iov_to_buf_full(const struct iovec *iov, const unsigned int iov_cnt,
+                       size_t offset, void *buf, size_t bytes)
 {
     size_t done;
     unsigned int i;
diff --git a/qemu/util/log.c b/qemu/util/log.c
new file mode 100644
index 000000000..1857730dc
--- /dev/null
+++ b/qemu/util/log.c
@@ -0,0 +1,313 @@
+/*
+ * Logging support
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/log.h"
+#include "qemu/range.h"
+#include "qemu/error-report.h"
+#include "qemu/cutils.h"
+#include "trace/control.h"
+
+static char *logfilename;
+FILE *qemu_logfile;
+int qemu_loglevel;
+static int log_append = 0;
+static GArray *debug_regions;
+
+void qemu_log(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    if (qemu_logfile) {
+        vfprintf(qemu_logfile, fmt, ap);
+    }
+    va_end(ap);
+}
+
+/* enable or disable low levels log */
+void do_qemu_set_log(int log_flags, bool use_own_buffers)
+{
+    qemu_loglevel = log_flags;
+#ifdef CONFIG_TRACE_LOG
+    qemu_loglevel |= LOG_TRACE;
+#endif
+    if (!qemu_logfile &&
+        (is_daemonized() ? logfilename != NULL : qemu_loglevel)) {
+        if (logfilename) {
+            qemu_logfile = fopen(logfilename, log_append ? "a" : "w");
+            if (!qemu_logfile) {
+                perror(logfilename);
+                _exit(1);
+            }
+            /* In case we are a daemon redirect stderr to logfile */
+            if (is_daemonized()) {
+                dup2(fileno(qemu_logfile), STDERR_FILENO);
+                fclose(qemu_logfile);
+                /* This will skip closing logfile in qemu_log_close() */
+                qemu_logfile = stderr;
+            }
+        } else {
+            /* Default to stderr if no log file specified */
+            assert(!is_daemonized());
+            qemu_logfile = stderr;
+        }
+        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
+        if (use_own_buffers) {
+            static char logfile_buf[4096];
+
+            setvbuf(qemu_logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
+        } else {
+#if defined(_WIN32)
+            /* Win32 doesn't support line-buffering, so use unbuffered output. */
+            setvbuf(qemu_logfile, NULL, _IONBF, 0);
+#else
+            setvbuf(qemu_logfile, NULL, _IOLBF, 0);
+#endif
+            log_append = 1;
+        }
+    }
+    if (qemu_logfile &&
+        (is_daemonized() ? logfilename == NULL : !qemu_loglevel)) {
+        qemu_log_close();
+    }
+}
+/*
+ * Allow the user to include %d in their logfile which will be
+ * substituted with the current PID. This is useful for debugging many
+ * nested linux-user tasks but will result in lots of logs.
+ */
+void qemu_set_log_filename(const char *filename)
+{
+    char *pidstr;
+    g_free(logfilename);
+
+    pidstr = strstr(filename, "%");
+    if (pidstr) {
+        /* We only accept one %d, no other format strings */
+        if (pidstr[1] != 'd' || strchr(pidstr + 2, '%')) {
+            error_report("Bad logfile format: %s", filename);
+            logfilename = NULL;
+        } else {
+            logfilename = g_strdup_printf(filename, getpid());
+        }
+    } else {
+        logfilename = g_strdup(filename);
+    }
+    qemu_log_close();
+    qemu_set_log(qemu_loglevel);
+}
+
+/* Returns true if addr is in our debug filter or no filter defined
+ */
+bool qemu_log_in_addr_range(uint64_t addr)
+{
+    if (debug_regions) {
+        int i = 0;
+        for (i = 0; i < debug_regions->len; i++) {
+            struct Range *range = &g_array_index(debug_regions, Range, i);
+            if (addr >= range->begin && addr <= range->end) {
+                return true;
+            }
+        }
+        return false;
+    } else {
+        return true;
+    }
+}
+
+
+void qemu_set_dfilter_ranges(const char *filter_spec)
+{
+    gchar **ranges = g_strsplit(filter_spec, ",", 0);
+    if (ranges) {
+        gchar **next = ranges;
+        gchar *r = *next++;
+        debug_regions = g_array_sized_new(FALSE, FALSE,
+                                          sizeof(Range), g_strv_length(ranges));
+        while (r) {
+            char *range_op = strstr(r, "-");
+            char *r2 = range_op ? range_op + 1 : NULL;
+            if (!range_op) {
+                range_op = strstr(r, "+");
+                r2 = range_op ? range_op + 1 : NULL;
+            }
+            if (!range_op) {
+                range_op = strstr(r, "..");
+                r2 = range_op ? range_op + 2 : NULL;
+            }
+            if (range_op) {
+                const char *e = NULL;
+                uint64_t r1val, r2val;
+
+                if ((qemu_strtoull(r, &e, 0, &r1val) == 0) &&
+                    (qemu_strtoull(r2, NULL, 0, &r2val) == 0) &&
+                    r2val > 0) {
+                    struct Range range;
+
+                    g_assert(e == range_op);
+
+                    switch (*range_op) {
+                    case '+':
+                    {
+                        range.begin = r1val;
+                        range.end = r1val + (r2val - 1);
+                        break;
+                    }
+                    case '-':
+                    {
+                        range.end = r1val;
+                        range.begin = r1val - (r2val - 1);
+                        break;
+                    }
+                    case '.':
+                        range.begin = r1val;
+                        range.end = r2val;
+                        break;
+                    default:
+                        g_assert_not_reached();
+                    }
+                    g_array_append_val(debug_regions, range);
+
+                } else {
+                    g_error("Failed to parse range in: %s", r);
+                }
+            } else {
+                g_error("Bad range specifier in: %s", r);
+            }
+            r = *next++;
+        }
+        g_strfreev(ranges);
+    }
+}
+
+/* fflush() the log file */
+void qemu_log_flush(void)
+{
+    fflush(qemu_logfile);
+}
+
+/* Close the log file */
+void qemu_log_close(void)
+{
+    if (qemu_logfile) {
+        if (qemu_logfile != stderr) {
+            fclose(qemu_logfile);
+        }
+        qemu_logfile = NULL;
+    }
+}
+
+const QEMULogItem qemu_log_items[] = {
+    { CPU_LOG_TB_OUT_ASM, "out_asm",
+      "show generated host assembly code for each compiled TB" },
+    { CPU_LOG_TB_IN_ASM, "in_asm",
+      "show target assembly code for each compiled TB" },
+    { CPU_LOG_TB_OP, "op",
+      "show micro ops for each compiled TB" },
+    { CPU_LOG_TB_OP_OPT, "op_opt",
+      "show micro ops (x86 only: before eflags optimization) and\n"
+      "after liveness analysis" },
+    { CPU_LOG_INT, "int",
+      "show interrupts/exceptions in short format" },
+    { CPU_LOG_EXEC, "exec",
+      "show trace before each executed TB (lots of logs)" },
+    { CPU_LOG_TB_CPU, "cpu",
+      "show CPU registers before entering a TB (lots of logs)" },
+    { CPU_LOG_MMU, "mmu",
+      "log MMU-related activities" },
+    { CPU_LOG_PCALL, "pcall",
+      "x86 only: show protected mode far calls/returns/exceptions" },
+    { CPU_LOG_RESET, "cpu_reset",
+      "show CPU state before CPU resets" },
+    { LOG_UNIMP, "unimp",
+      "log unimplemented functionality" },
+    { LOG_GUEST_ERROR, "guest_errors",
+      "log when the guest OS does something invalid (eg accessing a\n"
+      "non-existent register)" },
+    { CPU_LOG_PAGE, "page",
+      "dump pages at beginning of user mode emulation" },
+    { CPU_LOG_TB_NOCHAIN, "nochain",
+      "do not chain compiled TBs so that \"exec\" and \"cpu\" show\n"
+      "complete traces" },
+    { 0, NULL, NULL },
+};
+
+static int cmp1(const char *s1, int n, const char *s2)
+{
+    if (strlen(s2) != n) {
+        return 0;
+    }
+    return memcmp(s1, s2, n) == 0;
+}
+
+/* takes a comma separated list of log masks. Return 0 if error. */
+int qemu_str_to_log_mask(const char *str)
+{
+    const QEMULogItem *item;
+    int mask;
+    const char *p, *p1;
+
+    p = str;
+    mask = 0;
+    for (;;) {
+        p1 = strchr(p, ',');
+        if (!p1) {
+            p1 = p + strlen(p);
+        }
+        if (cmp1(p,p1-p,"all")) {
+            for (item = qemu_log_items; item->mask != 0; item++) {
+                mask |= item->mask;
+            }
+#ifdef CONFIG_TRACE_LOG
+        } else if (strncmp(p, "trace:", 6) == 0 && p + 6 != p1) {
+            trace_enable_events(p + 6);
+            mask |= LOG_TRACE;
+#endif
+        } else {
+            for (item = qemu_log_items; item->mask != 0; item++) {
+                if (cmp1(p, p1 - p, item->name)) {
+                    goto found;
+                }
+            }
+            return 0;
+        found:
+            mask |= item->mask;
+        }
+        if (*p1 != ',') {
+            break;
+        }
+        p = p1 + 1;
+    }
+    return mask;
+}
+
+void qemu_print_log_usage(FILE *f)
+{
+    const QEMULogItem *item;
+    fprintf(f, "Log items (comma separated):\n");
+    for (item = qemu_log_items; item->mask != 0; item++) {
+        fprintf(f, "%-15s %s\n", item->name, item->help);
+    }
+#ifdef CONFIG_TRACE_LOG
+    fprintf(f, "trace:PATTERN   enable trace events\n");
+    fprintf(f, "\nUse \"-d trace:help\" to get a list of trace events.\n\n");
+#endif
+}
diff --git a/qemu/util/memfd.c b/qemu/util/memfd.c
new file mode 100644
index 000000000..7c406914c
--- /dev/null
+++ b/qemu/util/memfd.c
@@ -0,0 +1,162 @@
+/*
+ * memfd.c
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * QEMU library functions on POSIX which are shared between QEMU and
+ * the QEMU tools.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <glib.h>
+#include <glib/gprintf.h>
+
+#include <sys/mman.h>
+
+#include "qemu/memfd.h"
+
+#ifdef CONFIG_MEMFD
+#include <sys/memfd.h>
+#elif defined CONFIG_LINUX
+#include <sys/syscall.h>
+#include <asm/unistd.h>
+
+static int memfd_create(const char *name, unsigned int flags)
+{
+#ifdef __NR_memfd_create
+    return syscall(__NR_memfd_create, name, flags);
+#else
+    return -1;
+#endif
+}
+#endif
+
+#ifndef MFD_CLOEXEC
+#define MFD_CLOEXEC 0x0001U
+#endif
+
+#ifndef MFD_ALLOW_SEALING
+#define MFD_ALLOW_SEALING 0x0002U
+#endif
+
+/*
+ * This is a best-effort helper for shared memory allocation, with
+ * optional sealing. The helper will do his best to allocate using
+ * memfd with sealing, but may fallback on other methods without
+ * sealing.
+ */
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd)
+{
+    void *ptr;
+    int mfd = -1;
+
+    *fd = -1;
+
+#ifdef CONFIG_LINUX
+    if (seals) {
+        mfd = memfd_create(name, MFD_ALLOW_SEALING | MFD_CLOEXEC);
+    }
+
+    if (mfd == -1) {
+        /* some systems have memfd without sealing */
+        mfd = memfd_create(name, MFD_CLOEXEC);
+        seals = 0;
+    }
+#endif
+
+    if (mfd != -1) {
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+
+        if (seals && fcntl(mfd, F_ADD_SEALS, seals) == -1) {
+            perror("fcntl");
+            close(mfd);
+            return NULL;
+        }
+    } else {
+        const char *tmpdir = g_get_tmp_dir();
+        gchar *fname;
+
+        fname = g_strdup_printf("%s/memfd-XXXXXX", tmpdir);
+        mfd = mkstemp(fname);
+        unlink(fname);
+        g_free(fname);
+
+        if (mfd == -1) {
+            perror("mkstemp");
+            return NULL;
+        }
+
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+    }
+
+    ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
+    if (ptr == MAP_FAILED) {
+        perror("mmap");
+        close(mfd);
+        return NULL;
+    }
+
+    *fd = mfd;
+    return ptr;
+}
+
+void qemu_memfd_free(void *ptr, size_t size, int fd)
+{
+    if (ptr) {
+        munmap(ptr, size);
+    }
+
+    if (fd != -1) {
+        close(fd);
+    }
+}
+
+enum {
+    MEMFD_KO,
+    MEMFD_OK,
+    MEMFD_TODO
+};
+
+bool qemu_memfd_check(void)
+{
+    static int memfd_check = MEMFD_TODO;
+
+    if (memfd_check == MEMFD_TODO) {
+        int fd;
+        void *ptr;
+
+        ptr = qemu_memfd_alloc("test", 4096, 0, &fd);
+        memfd_check = ptr ? MEMFD_OK : MEMFD_KO;
+        qemu_memfd_free(ptr, 4096, fd);
+    }
+
+    return memfd_check == MEMFD_OK;
+}
diff --git a/qemu/util/mmap-alloc.c b/qemu/util/mmap-alloc.c
new file mode 100644
index 000000000..0b4cc7f7f
--- /dev/null
+++ b/qemu/util/mmap-alloc.c
@@ -0,0 +1,110 @@
+/*
+ * Support for RAM backed by mmaped host memory.
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include <qemu/mmap-alloc.h>
+#include <sys/mman.h>
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+#ifdef CONFIG_LINUX
+#include <sys/vfs.h>
+#endif
+
+size_t qemu_fd_getpagesize(int fd)
+{
+#ifdef CONFIG_LINUX
+    struct statfs fs;
+    int ret;
+
+    if (fd != -1) {
+        do {
+            ret = fstatfs(fd, &fs);
+        } while (ret != 0 && errno == EINTR);
+
+        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
+            return fs.f_bsize;
+        }
+    }
+#endif
+
+    return getpagesize();
+}
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared)
+{
+    /*
+     * Note: this always allocates at least one extra page of virtual address
+     * space, even if size is already aligned.
+     */
+    size_t total = size + align;
+#if defined(__powerpc64__) && defined(__linux__)
+    /* On ppc64 mappings in the same segment (aka slice) must share the same
+     * page size. Since we will be re-allocating part of this segment
+     * from the supplied fd, we should make sure to use the same page size, to
+     * this end we mmap the supplied fd.  In this case, set MAP_NORESERVE to
+     * avoid allocating backing store memory.
+     * We do this unless we are using the system page size, in which case
+     * anonymous memory is OK.
+     */
+    int anonfd = fd == -1 || qemu_fd_getpagesize(fd) == getpagesize() ? -1 : fd;
+    int flags = anonfd == -1 ? MAP_ANONYMOUS : MAP_NORESERVE;
+    void *ptr = mmap(0, total, PROT_NONE, flags | MAP_PRIVATE, anonfd, 0);
+#else
+    void *ptr = mmap(0, total, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+#endif
+    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr1;
+
+    if (ptr == MAP_FAILED) {
+        return MAP_FAILED;
+    }
+
+    /* Make sure align is a power of 2 */
+    assert(!(align & (align - 1)));
+    /* Always align to host page size */
+    assert(align >= getpagesize());
+
+    ptr1 = mmap(ptr + offset, size, PROT_READ | PROT_WRITE,
+                MAP_FIXED |
+                (fd == -1 ? MAP_ANONYMOUS : 0) |
+                (shared ? MAP_SHARED : MAP_PRIVATE),
+                fd, 0);
+    if (ptr1 == MAP_FAILED) {
+        munmap(ptr, total);
+        return MAP_FAILED;
+    }
+
+    ptr += offset;
+    total -= offset;
+
+    if (offset > 0) {
+        munmap(ptr - offset, offset);
+    }
+
+    /*
+     * Leave a single PROT_NONE page allocated after the RAM block, to serve as
+     * a guard page guarding against potential buffer overflows.
+     */
+    if (total > size + getpagesize()) {
+        munmap(ptr + size + getpagesize(), total - size - getpagesize());
+    }
+
+    return ptr;
+}
+
+void qemu_ram_munmap(void *ptr, size_t size)
+{
+    if (ptr) {
+        /* Unmap both the RAM block and the guard page */
+        munmap(ptr, size + getpagesize());
+    }
+}
diff --git a/qemu/util/module.c b/qemu/util/module.c
index 4bd4a94d8..ce058aef6 100644
--- a/qemu/util/module.c
+++ b/qemu/util/module.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #ifdef CONFIG_MODULES
 #include <gmodule.h>
diff --git a/qemu/util/notify.c b/qemu/util/notify.c
index f215dfc21..06de63a83 100644
--- a/qemu/util/notify.c
+++ b/qemu/util/notify.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
 
diff --git a/qemu/util/osdep.c b/qemu/util/osdep.c
index 0092bb61b..d56d07111 100644
--- a/qemu/util/osdep.c
+++ b/qemu/util/osdep.c
@@ -21,24 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
+#include "qemu/osdep.h"
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
 
 #if defined(CONFIG_MADVISE) || defined(CONFIG_POSIX_MADVISE)
 #include <sys/mman.h>
 #endif
 
 #ifdef CONFIG_SOLARIS
-#include <sys/types.h>
 #include <sys/statvfs.h>
 /* See MySQL bug #7156 (http://bugs.mysql.com/bug.php?id=7156) for
    discussion about Solaris header problems */
@@ -46,13 +37,21 @@ extern int madvise(caddr_t, size_t, int);
 #endif
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/sockets.h"
 #include "qemu/error-report.h"
 #include "monitor/monitor.h"
 
 static bool fips_enabled = false;
 
-static const char *qemu_version = QEMU_VERSION;
+/* Starting on QEMU 2.5, qemu_hw_version() returns "2.5+" by default
+ * instead of QEMU_VERSION, so setting hw_version on MachineClass
+ * is no longer mandatory.
+ *
+ * Do NOT change this string, or it will break compatibility on all
+ * machine classes that don't set hw_version.
+ */
+static const char *hw_version = "2.5+";
 
 int socket_set_cork(int fd, int v)
 {
@@ -311,14 +310,14 @@ int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen)
     return ret;
 }
 
-void qemu_set_version(const char *version)
+void qemu_set_hw_version(const char *version)
 {
-    qemu_version = version;
+    hw_version = version;
 }
 
-const char *qemu_get_version(void)
+const char *qemu_hw_version(void)
 {
-    return qemu_version;
+    return hw_version;
 }
 
 void fips_set_state(bool requested)
diff --git a/qemu/util/oslib-posix.c b/qemu/util/oslib-posix.c
index 3ae4987b6..6cc4b8f00 100644
--- a/qemu/util/oslib-posix.c
+++ b/qemu/util/oslib-posix.c
@@ -26,16 +26,8 @@
  * THE SOFTWARE.
  */
 
-/* The following block of code temporarily renames the daemon() function so the
-   compiler does not see the warning associated with it in stdlib.h on OSX */
-#ifdef __APPLE__
-#define daemon qemu_fake_daemon_function
-#include <stdlib.h>
-#undef daemon
-extern int daemon(int, int);
-#endif
-
-#if defined(__linux__) && (defined(__x86_64__) || defined(__arm__))
+#if defined(__linux__) && \
+    (defined(__x86_64__) || defined(__arm__) || defined(__aarch64__))
    /* Use 2 MiB alignment so transparent hugepages can be used by KVM.
       Valgrind does not support alignments larger than 1 MiB,
       therefore we need special code which handles running on Valgrind. */
@@ -46,32 +38,32 @@ extern int daemon(int, int);
 #else
 #  define QEMU_VMALLOC_ALIGN getpagesize()
 #endif
-#define HUGETLBFS_MAGIC       0x958458f6
 
+#include "qemu/osdep.h"
 #include <termios.h>
-#include <unistd.h>
 #include <termios.h>
 
 #include <glib/gprintf.h>
 
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h"
 #include <sys/mman.h>
 #include <libgen.h>
-#include <setjmp.h>
 #include <sys/signal.h>
+#include "qemu/cutils.h"
 
 #ifdef CONFIG_LINUX
 #include <sys/syscall.h>
-#include <sys/vfs.h>
 #endif
 
 #ifdef __FreeBSD__
 #include <sys/sysctl.h>
 #endif
 
+#include <qemu/mmap-alloc.h>
+
 int qemu_get_thread_id(void)
 {
 #if defined(__linux__)
@@ -128,10 +120,7 @@ void *qemu_memalign(size_t alignment, size_t size)
 void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
 {
     size_t align = QEMU_VMALLOC_ALIGN;
-    size_t total = size + align - getpagesize();
-    void *ptr = mmap(0, total, PROT_READ | PROT_WRITE,
-                     MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr = qemu_ram_mmap(-1, size, align, false);
 
     if (ptr == MAP_FAILED) {
         return NULL;
@@ -140,15 +129,6 @@ void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
     if (alignment) {
         *alignment = align;
     }
-    ptr += offset;
-    total -= offset;
-
-    if (offset > 0) {
-        munmap(ptr - offset, offset);
-    }
-    if (total > size) {
-        munmap(ptr + size, total - size);
-    }
 
     trace_qemu_anon_ram_alloc(size, ptr);
     return ptr;
@@ -163,9 +143,7 @@ void qemu_vfree(void *ptr)
 void qemu_anon_ram_free(void *ptr, size_t size)
 {
     trace_qemu_anon_ram_free(ptr, size);
-    if (ptr) {
-        munmap(ptr, size);
-    }
+    qemu_ram_munmap(ptr, size);
 }
 
 void qemu_set_block(int fd)
@@ -352,26 +330,6 @@ static void sigbus_handler(int signal)
     siglongjmp(sigjump, 1);
 }
 
-static size_t fd_getpagesize(int fd)
-{
-#ifdef CONFIG_LINUX
-    struct statfs fs;
-    int ret;
-
-    if (fd != -1) {
-        do {
-            ret = fstatfs(fd, &fs);
-        } while (ret != 0 && errno == EINTR);
-
-        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
-            return fs.f_bsize;
-        }
-    }
-#endif
-
-    return getpagesize();
-}
-
 void os_mem_prealloc(int fd, char *area, size_t memory)
 {
     int ret;
@@ -399,7 +357,7 @@ void os_mem_prealloc(int fd, char *area, size_t memory)
         exit(1);
     } else {
         int i;
-        size_t hpagesize = fd_getpagesize(fd);
+        size_t hpagesize = qemu_fd_getpagesize(fd);
         size_t numpages = DIV_ROUND_UP(memory, hpagesize);
 
         /* MAP_POPULATE silently ignores failures */
@@ -482,3 +440,74 @@ int qemu_read_password(char *buf, int buf_size)
     printf("\n");
     return ret;
 }
+
+
+pid_t qemu_fork(Error **errp)
+{
+    sigset_t oldmask, newmask;
+    struct sigaction sig_action;
+    int saved_errno;
+    pid_t pid;
+
+    /*
+     * Need to block signals now, so that child process can safely
+     * kill off caller's signal handlers without a race.
+     */
+    sigfillset(&newmask);
+    if (pthread_sigmask(SIG_SETMASK, &newmask, &oldmask) != 0) {
+        error_setg_errno(errp, errno,
+                         "cannot block signals");
+        return -1;
+    }
+
+    pid = fork();
+    saved_errno = errno;
+
+    if (pid < 0) {
+        /* attempt to restore signal mask, but ignore failure, to
+         * avoid obscuring the fork failure */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+        error_setg_errno(errp, saved_errno,
+                         "cannot fork child process");
+        errno = saved_errno;
+        return -1;
+    } else if (pid) {
+        /* parent process */
+
+        /* Restore our original signal mask now that the child is
+         * safely running. Only documented failures are EFAULT (not
+         * possible, since we are using just-grabbed mask) or EINVAL
+         * (not possible, since we are using correct arguments).  */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+    } else {
+        /* child process */
+        size_t i;
+
+        /* Clear out all signal handlers from parent so nothing
+         * unexpected can happen in our child once we unblock
+         * signals */
+        sig_action.sa_handler = SIG_DFL;
+        sig_action.sa_flags = 0;
+        sigemptyset(&sig_action.sa_mask);
+
+        for (i = 1; i < NSIG; i++) {
+            /* Only possible errors are EFAULT or EINVAL The former
+             * won't happen, the latter we expect, so no need to check
+             * return value */
+            (void)sigaction(i, &sig_action, NULL);
+        }
+
+        /* Unmask all signals in child, since we've no idea what the
+         * caller's done with their signal mask and don't want to
+         * propagate that to children */
+        sigemptyset(&newmask);
+        if (pthread_sigmask(SIG_SETMASK, &newmask, NULL) != 0) {
+            Error *local_err = NULL;
+            error_setg_errno(&local_err, errno,
+                             "cannot unblock signals");
+            error_report_err(local_err);
+            _exit(1);
+        }
+    }
+    return pid;
+}
diff --git a/qemu/util/oslib-win32.c b/qemu/util/oslib-win32.c
index 730a6707a..c926db4a5 100644
--- a/qemu/util/oslib-win32.c
+++ b/qemu/util/oslib-win32.c
@@ -2,7 +2,7 @@
  * os-win32.c
  *
  * Copyright (c) 2003-2008 Fabrice Bellard
- * Copyright (c) 2010 Red Hat, Inc.
+ * Copyright (c) 2010-2016 Red Hat, Inc.
  *
  * QEMU library functions for win32 which are shared between QEMU and
  * the QEMU tools.
@@ -29,14 +29,15 @@
  * this file are based on code from GNOME glib-2 and use a different license,
  * see the license comment there.
  */
+#include "qemu/osdep.h"
 #include <windows.h>
 #include <glib.h>
-#include <stdlib.h>
-#include "config-host.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
 #include "qemu/sockets.h"
+#include "qemu/cutils.h"
 
 /* this must come after including "trace.h" */
 #include <shlobj.h>
@@ -95,6 +96,7 @@ void qemu_anon_ram_free(void *ptr, size_t size)
     }
 }
 
+#ifndef CONFIG_LOCALTIME_R
 /* FIXME: add proper locking */
 struct tm *gmtime_r(const time_t *timep, struct tm *result)
 {
@@ -118,6 +120,7 @@ struct tm *localtime_r(const time_t *timep, struct tm *result)
     }
     return p;
 }
+#endif /* CONFIG_LOCALTIME_R */
 
 void qemu_set_block(int fd)
 {
@@ -143,6 +146,83 @@ int socket_set_fast_reuse(int fd)
     return 0;
 }
 
+
+static int socket_error(void)
+{
+    switch (WSAGetLastError()) {
+    case 0:
+        return 0;
+    case WSAEINTR:
+        return EINTR;
+    case WSAEINVAL:
+        return EINVAL;
+    case WSA_INVALID_HANDLE:
+        return EBADF;
+    case WSA_NOT_ENOUGH_MEMORY:
+        return ENOMEM;
+    case WSA_INVALID_PARAMETER:
+        return EINVAL;
+    case WSAENAMETOOLONG:
+        return ENAMETOOLONG;
+    case WSAENOTEMPTY:
+        return ENOTEMPTY;
+    case WSAEWOULDBLOCK:
+         /* not using EWOULDBLOCK as we don't want code to have
+          * to check both EWOULDBLOCK and EAGAIN */
+        return EAGAIN;
+    case WSAEINPROGRESS:
+        return EINPROGRESS;
+    case WSAEALREADY:
+        return EALREADY;
+    case WSAENOTSOCK:
+        return ENOTSOCK;
+    case WSAEDESTADDRREQ:
+        return EDESTADDRREQ;
+    case WSAEMSGSIZE:
+        return EMSGSIZE;
+    case WSAEPROTOTYPE:
+        return EPROTOTYPE;
+    case WSAENOPROTOOPT:
+        return ENOPROTOOPT;
+    case WSAEPROTONOSUPPORT:
+        return EPROTONOSUPPORT;
+    case WSAEOPNOTSUPP:
+        return EOPNOTSUPP;
+    case WSAEAFNOSUPPORT:
+        return EAFNOSUPPORT;
+    case WSAEADDRINUSE:
+        return EADDRINUSE;
+    case WSAEADDRNOTAVAIL:
+        return EADDRNOTAVAIL;
+    case WSAENETDOWN:
+        return ENETDOWN;
+    case WSAENETUNREACH:
+        return ENETUNREACH;
+    case WSAENETRESET:
+        return ENETRESET;
+    case WSAECONNABORTED:
+        return ECONNABORTED;
+    case WSAECONNRESET:
+        return ECONNRESET;
+    case WSAENOBUFS:
+        return ENOBUFS;
+    case WSAEISCONN:
+        return EISCONN;
+    case WSAENOTCONN:
+        return ENOTCONN;
+    case WSAETIMEDOUT:
+        return ETIMEDOUT;
+    case WSAECONNREFUSED:
+        return ECONNREFUSED;
+    case WSAELOOP:
+        return ELOOP;
+    case WSAEHOSTUNREACH:
+        return EHOSTUNREACH;
+    default:
+        return EIO;
+    }
+}
+
 int inet_aton(const char *cp, struct in_addr *ia)
 {
     uint32_t addr = inet_addr(cp);
@@ -452,7 +532,7 @@ gint g_poll(GPollFD *fds, guint nfds, gint timeout)
     return retval;
 }
 
-size_t getpagesize(void)
+int getpagesize(void)
 {
     SYSTEM_INFO system_info;
 
@@ -494,3 +574,213 @@ int qemu_read_password(char *buf, int buf_size)
     buf[i] = '\0';
     return 0;
 }
+
+
+pid_t qemu_fork(Error **errp)
+{
+    errno = ENOSYS;
+    error_setg_errno(errp, errno,
+                     "cannot fork child process");
+    return -1;
+}
+
+
+#undef connect
+int qemu_connect_wrap(int sockfd, const struct sockaddr *addr,
+                      socklen_t addrlen)
+{
+    int ret;
+    ret = connect(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef listen
+int qemu_listen_wrap(int sockfd, int backlog)
+{
+    int ret;
+    ret = listen(sockfd, backlog);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef bind
+int qemu_bind_wrap(int sockfd, const struct sockaddr *addr,
+                   socklen_t addrlen)
+{
+    int ret;
+    ret = bind(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef socket
+int qemu_socket_wrap(int domain, int type, int protocol)
+{
+    int ret;
+    ret = socket(domain, type, protocol);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef accept
+int qemu_accept_wrap(int sockfd, struct sockaddr *addr,
+                     socklen_t *addrlen)
+{
+    int ret;
+    ret = accept(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef shutdown
+int qemu_shutdown_wrap(int sockfd, int how)
+{
+    int ret;
+    ret = shutdown(sockfd, how);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef ioctlsocket
+int qemu_ioctlsocket_wrap(int fd, int req, void *val)
+{
+    int ret;
+    ret = ioctlsocket(fd, req, val);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef closesocket
+int qemu_closesocket_wrap(int fd)
+{
+    int ret;
+    ret = closesocket(fd);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getsockopt
+int qemu_getsockopt_wrap(int sockfd, int level, int optname,
+                         void *optval, socklen_t *optlen)
+{
+    int ret;
+    ret = getsockopt(sockfd, level, optname, optval, optlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef setsockopt
+int qemu_setsockopt_wrap(int sockfd, int level, int optname,
+                         const void *optval, socklen_t optlen)
+{
+    int ret;
+    ret = setsockopt(sockfd, level, optname, optval, optlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getpeername
+int qemu_getpeername_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen)
+{
+    int ret;
+    ret = getpeername(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getsockname
+int qemu_getsockname_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen)
+{
+    int ret;
+    ret = getsockname(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef send
+ssize_t qemu_send_wrap(int sockfd, const void *buf, size_t len, int flags)
+{
+    int ret;
+    ret = send(sockfd, buf, len, flags);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef sendto
+ssize_t qemu_sendto_wrap(int sockfd, const void *buf, size_t len, int flags,
+                         const struct sockaddr *addr, socklen_t addrlen)
+{
+    int ret;
+    ret = sendto(sockfd, buf, len, flags, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef recv
+ssize_t qemu_recv_wrap(int sockfd, void *buf, size_t len, int flags)
+{
+    int ret;
+    ret = recv(sockfd, buf, len, flags);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef recvfrom
+ssize_t qemu_recvfrom_wrap(int sockfd, void *buf, size_t len, int flags,
+                           struct sockaddr *addr, socklen_t *addrlen)
+{
+    int ret;
+    ret = recvfrom(sockfd, buf, len, flags, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
diff --git a/qemu/util/path.c b/qemu/util/path.c
index 4e4877e82..5479f76c6 100644
--- a/qemu/util/path.c
+++ b/qemu/util/path.c
@@ -3,15 +3,12 @@
 
    The assumption is that this area does not change.
 */
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/param.h>
 #include <dirent.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdio.h>
 #include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 
 struct pathelem
 {
diff --git a/qemu/util/qemu-config.c b/qemu/util/qemu-config.c
index 5fcfd0e6a..fb973074d 100644
--- a/qemu/util/qemu-config.c
+++ b/qemu/util/qemu-config.c
@@ -1,8 +1,8 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
-#include "qapi/error.h"
 #include "qmp-commands.h"
 
 static QemuOptsList *vm_config_groups[48];
@@ -219,6 +219,14 @@ static QemuOptsList machine_opts = {
             .name = "suppress-vmdesc",
             .type = QEMU_OPT_BOOL,
             .help = "Set on to disable self-describing migration",
+        },{
+            .name = "aes-key-wrap",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable AES key wrapping using the CPACF wrapping key",
+        },{
+            .name = "dea-key-wrap",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable DEA key wrapping using the CPACF wrapping key",
         },
         { /* End of list */ }
     }
diff --git a/qemu/qemu-coroutine-io.c b/qemu/util/qemu-coroutine-io.c
index 28dc7351a..91b9357d4 100644
--- a/qemu/qemu-coroutine-io.c
+++ b/qemu/util/qemu-coroutine-io.c
@@ -22,9 +22,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/sockets.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/iov.h"
 #include "qemu/main-loop.h"
 
@@ -34,18 +35,16 @@ qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
 {
     size_t done = 0;
     ssize_t ret;
-    int err;
     while (done < bytes) {
         ret = iov_send_recv(sockfd, iov, iov_cnt,
                             offset + done, bytes - done, do_send);
         if (ret > 0) {
             done += ret;
         } else if (ret < 0) {
-            err = socket_error();
-            if (err == EAGAIN || err == EWOULDBLOCK) {
+            if (errno == EAGAIN || errno == EWOULDBLOCK) {
                 qemu_coroutine_yield();
             } else if (done == 0) {
-                return -err;
+                return -errno;
             } else {
                 break;
             }
diff --git a/qemu/qemu-coroutine-lock.c b/qemu/util/qemu-coroutine-lock.c
index 6b4903334..da37ca7f9 100644
--- a/qemu/qemu-coroutine-lock.c
+++ b/qemu/util/qemu-coroutine-lock.c
@@ -22,9 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 #include "qemu/queue.h"
 #include "trace.h"
 
diff --git a/qemu/qemu-coroutine-sleep.c b/qemu/util/qemu-coroutine-sleep.c
index 9abb7fdf3..6966831d3 100644
--- a/qemu/qemu-coroutine-sleep.c
+++ b/qemu/util/qemu-coroutine-sleep.c
@@ -11,7 +11,8 @@
  *
  */
 
-#include "block/coroutine.h"
+#include "qemu/osdep.h"
+#include "qemu/coroutine.h"
 #include "qemu/timer.h"
 #include "block/aio.h"
 
diff --git a/qemu/qemu-coroutine.c b/qemu/util/qemu-coroutine.c
index c17a92b10..5816702cc 100644
--- a/qemu/qemu-coroutine.c
+++ b/qemu/util/qemu-coroutine.c
@@ -12,12 +12,13 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "qemu-common.h"
 #include "qemu/thread.h"
 #include "qemu/atomic.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 
 enum {
     POOL_BATCH_SIZE = 64,
diff --git a/qemu/util/qemu-error.c b/qemu/util/qemu-error.c
index 77ea6c614..1ef35664a 100644
--- a/qemu/util/qemu-error.c
+++ b/qemu/util/qemu-error.c
@@ -10,7 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "monitor/monitor.h"
 #include "qemu/error-report.h"
 
@@ -200,8 +200,8 @@ static void error_print_loc(void)
 bool enable_timestamp_msg;
 /*
  * Print an error message to current monitor if we have one, else to stderr.
- * Format arguments like vsprintf().  The result should not contain
- * newlines.
+ * Format arguments like vsprintf().  The resulting message should be
+ * a single phrase, with no newline or trailing punctuation.
  * Prepend the current location and append a newline.
  * It's wrong to call this in a QMP monitor.  Use error_setg() there.
  */
@@ -210,7 +210,7 @@ void error_vreport(const char *fmt, va_list ap)
     GTimeVal tv;
     gchar *timestr;
 
-    if (enable_timestamp_msg) {
+    if (enable_timestamp_msg && !cur_mon) {
         g_get_current_time(&tv);
         timestr = g_time_val_to_iso8601(&tv);
         error_printf("%s ", timestr);
@@ -224,8 +224,8 @@ void error_vreport(const char *fmt, va_list ap)
 
 /*
  * Print an error message to current monitor if we have one, else to stderr.
- * Format arguments like sprintf().  The result should not contain
- * newlines.
+ * Format arguments like sprintf().  The resulting message should be a
+ * single phrase, with no newline or trailing punctuation.
  * Prepend the current location and append a newline.
  * It's wrong to call this in a QMP monitor.  Use error_setg() there.
  */
diff --git a/qemu/util/qemu-openpty.c b/qemu/util/qemu-openpty.c
index 4c5321116..2e8b43bdf 100644
--- a/qemu/util/qemu-openpty.c
+++ b/qemu/util/qemu-openpty.c
@@ -32,7 +32,7 @@
  * linked with -lutil.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 #if defined(__GLIBC__)
diff --git a/qemu/util/qemu-option.c b/qemu/util/qemu-option.c
index efe9d279c..3467dc239 100644
--- a/qemu/util/qemu-option.c
+++ b/qemu/util/qemu-option.c
@@ -23,15 +23,17 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qapi/qmp/types.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/option_int.h"
+#include "qemu/cutils.h"
+#include "qemu/id.h"
+#include "qemu/help_option.h"
 
 /*
  * Extracts the name of an option from the parameter string (p points at the
@@ -180,6 +182,11 @@ void parse_option_size(const char *name, const char *value,
 
     if (value != NULL) {
         sizef = strtod(value, &postfix);
+        if (sizef < 0 || sizef > UINT64_MAX) {
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name,
+                             "a non-negative number below 2^64");
+            return;
+        }
         switch (*postfix) {
         case 'T':
             sizef *= 1024;
@@ -200,10 +207,8 @@ void parse_option_size(const char *name, const char *value,
             break;
         default:
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("You may use k, M, G or T suffixes for "
+            error_append_hint(errp, "You may use k, M, G or T suffixes for "
                     "kilobytes, megabytes, gigabytes and terabytes.\n");
-#endif
             return;
         }
     } else {
@@ -643,9 +648,8 @@ QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
         if (!id_wellformed(id)) {
             error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "id",
                        "an identifier");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("Identifiers consist of letters, digits, '-', '.', '_', starting with a letter.\n");
-#endif
+            error_append_hint(errp, "Identifiers consist of letters, digits, "
+                              "'-', '.', '_', starting with a letter.\n");
             return NULL;
         }
         opts = qemu_opts_find(list, id);
@@ -730,14 +734,35 @@ void qemu_opts_del(QemuOpts *opts)
     g_free(opts);
 }
 
-void qemu_opts_print(QemuOpts *opts, const char *sep)
+/* print value, escaping any commas in value */
+static void escaped_print(const char *value)
+{
+    const char *ptr;
+
+    for (ptr = value; *ptr; ++ptr) {
+        if (*ptr == ',') {
+            putchar(',');
+        }
+        putchar(*ptr);
+    }
+}
+
+void qemu_opts_print(QemuOpts *opts, const char *separator)
 {
     QemuOpt *opt;
     QemuOptDesc *desc = opts->list->desc;
+    const char *sep = "";
+
+    if (opts->id) {
+        printf("id=%s", opts->id); /* passed id_wellformed -> no commas */
+        sep = separator;
+    }
 
     if (desc[0].name == NULL) {
         QTAILQ_FOREACH(opt, &opts->head, next) {
-            printf("%s%s=\"%s\"", sep, opt->name, opt->str);
+            printf("%s%s=", sep, opt->name);
+            escaped_print(opt->str);
+            sep = separator;
         }
         return;
     }
@@ -750,13 +775,15 @@ void qemu_opts_print(QemuOpts *opts, const char *sep)
             continue;
         }
         if (desc->type == QEMU_OPT_STRING) {
-            printf("%s%s='%s'", sep, desc->name, value);
+            printf("%s%s=", sep, desc->name);
+            escaped_print(value);
         } else if ((desc->type == QEMU_OPT_SIZE ||
                     desc->type == QEMU_OPT_NUMBER) && opt) {
             printf("%s%s=%" PRId64, sep, desc->name, opt->value.uint);
         } else {
             printf("%s%s=%s", sep, desc->name, value);
         }
+        sep = separator;
     }
 }
 
@@ -1081,19 +1108,19 @@ int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func,
 {
     Location loc;
     QemuOpts *opts;
-    int rc;
+    int rc = 0;
 
     loc_push_none(&loc);
     QTAILQ_FOREACH(opts, &list->head, next) {
         loc_restore(&opts->loc);
         rc = func(opaque, opts, errp);
         if (rc) {
-            return rc;
+            break;
         }
         assert(!errp || !*errp);
     }
     loc_pop(&loc);
-    return 0;
+    return rc;
 }
 
 static size_t count_opts_list(QemuOptsList *list)
diff --git a/qemu/util/qemu-progress.c b/qemu/util/qemu-progress.c
index 4ee5cd07f..f74523376 100644
--- a/qemu/util/qemu-progress.c
+++ b/qemu/util/qemu-progress.c
@@ -22,9 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
 #include "qemu/osdep.h"
-#include <stdio.h>
+#include "qemu-common.h"
 
 struct progress_state {
     float current;
@@ -152,7 +151,8 @@ void qemu_progress_print(float delta, int max)
     state.current = current;
 
     if (current > (state.last_print + state.min_skip) ||
-        (current == 100) || (current == 0)) {
+        current < (state.last_print - state.min_skip) ||
+        current == 100 || current == 0) {
         state.last_print = state.current;
         state.print();
     }
diff --git a/qemu/util/qemu-sockets.c b/qemu/util/qemu-sockets.c
index 2add83a0f..0d536911c 100644
--- a/qemu/util/qemu-sockets.c
+++ b/qemu/util/qemu-sockets.c
@@ -15,57 +15,25 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "monitor/monitor.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h"
 #include "qemu/main-loop.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi-visit.h"
+#include "qemu/cutils.h"
 
 #ifndef AI_ADDRCONFIG
 # define AI_ADDRCONFIG 0
 #endif
+
 #ifndef AI_V4MAPPED
 # define AI_V4MAPPED 0
 #endif
 
-/* used temporarily until all users are converted to QemuOpts */
-QemuOptsList socket_optslist = {
-    .name = "socket",
-    .head = QTAILQ_HEAD_INITIALIZER(socket_optslist.head),
-    .desc = {
-        {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "host",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "port",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localaddr",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "localport",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "to",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end if list */ }
-    },
-};
 
 static int inet_getport(struct addrinfo *e)
 {
@@ -111,37 +79,86 @@ NetworkAddressFamily inet_netfamily(int family)
     return NETWORK_ADDRESS_FAMILY_UNKNOWN;
 }
 
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
+/*
+ * Matrix we're trying to apply
+ *
+ *  ipv4  ipv6   family
+ *   -     -       PF_UNSPEC
+ *   -     f       PF_INET
+ *   -     t       PF_INET6
+ *   f     -       PF_INET6
+ *   f     f       <error>
+ *   f     t       PF_INET6
+ *   t     -       PF_INET
+ *   t     f       PF_INET
+ *   t     t       PF_INET6
+ *
+ * NB, this matrix is only about getting the neccessary results
+ * from getaddrinfo(). Some of the cases require further work
+ * after reading results from getaddrinfo in order to fully
+ * apply the logic the end user wants. eg with the last case
+ * ipv4=t + ipv6=t + PF_INET6, getaddrinfo alone can only
+ * guarantee the ipv6=t part of the request - we need more
+ * checks to provide ipv4=t part of the guarantee. This is
+ * outside scope of this method and not currently handled by
+ * callers at all.
+ */
+static int inet_ai_family_from_address(InetSocketAddress *addr,
+                                       Error **errp)
+{
+    if (addr->has_ipv6 && addr->has_ipv4 &&
+        !addr->ipv6 && !addr->ipv4) {
+        error_setg(errp, "Cannot disable IPv4 and IPv6 at same time");
+        return PF_UNSPEC;
+    }
+    if ((addr->has_ipv6 && addr->ipv6) || (addr->has_ipv4 && !addr->ipv4)) {
+        return PF_INET6;
+    }
+    if ((addr->has_ipv4 && addr->ipv4) || (addr->has_ipv6 && !addr->ipv6)) {
+        return PF_INET;
+    }
+    return PF_UNSPEC;
+}
+
+static int inet_listen_saddr(InetSocketAddress *saddr,
+                             int port_offset,
+                             bool update_addr,
+                             Error **errp)
 {
     struct addrinfo ai,*res,*e;
-    const char *addr;
     char port[33];
     char uaddr[INET6_ADDRSTRLEN+1];
     char uport[33];
-    int slisten, rc, to, port_min, port_max, p;
+    int slisten, rc, port_min, port_max, p;
+    Error *err = NULL;
 
     memset(&ai,0, sizeof(ai));
     ai.ai_flags = AI_PASSIVE;
-    ai.ai_family = PF_UNSPEC;
+    ai.ai_family = inet_ai_family_from_address(saddr, &err);
     ai.ai_socktype = SOCK_STREAM;
 
-    if ((qemu_opt_get(opts, "host") == NULL) ||
-        (qemu_opt_get(opts, "port") == NULL)) {
-        error_setg(errp, "host and/or port not specified");
+    if (err) {
+        error_propagate(errp, err);
         return -1;
     }
-    pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
-    addr = qemu_opt_get(opts, "host");
 
-    to = qemu_opt_get_number(opts, "to", 0);
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
+    if (saddr->host == NULL) {
+        error_setg(errp, "host not specified");
+        return -1;
+    }
+    if (saddr->port != NULL) {
+        pstrcpy(port, sizeof(port), saddr->port);
+    } else {
+        port[0] = '\0';
+    }
 
     /* lookup */
     if (port_offset) {
         unsigned long long baseport;
+        if (strlen(port) == 0) {
+            error_setg(errp, "port not specified");
+            return -1;
+        }
         if (parse_uint_full(port, &baseport, 10) < 0) {
             error_setg(errp, "can't convert to a number: %s", port);
             return -1;
@@ -153,10 +170,11 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
         }
         snprintf(port, sizeof(port), "%d", (int)baseport + port_offset);
     }
-    rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
+    rc = getaddrinfo(strlen(saddr->host) ? saddr->host : NULL,
+                     strlen(port) ? port : NULL, &ai, &res);
     if (rc != 0) {
-        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
-                   gai_strerror(rc));
+        error_setg(errp, "address resolution failed for %s:%s: %s",
+                   saddr->host, port, gai_strerror(rc));
         return -1;
     }
 
@@ -184,7 +202,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
 #endif
 
         port_min = inet_getport(e);
-        port_max = to ? to + port_offset : port_min;
+        port_max = saddr->has_to ? saddr->to + port_offset : port_min;
         for (p = port_min; p <= port_max; p++) {
             inet_setport(e, p);
             if (bind(slisten, e->ai_addr, e->ai_addrlen) == 0) {
@@ -208,13 +226,15 @@ listen:
         freeaddrinfo(res);
         return -1;
     }
-    qemu_opt_set(opts, "host", uaddr, &error_abort);
-    qemu_opt_set_number(opts, "port", inet_getport(e) - port_offset,
-                        &error_abort);
-    qemu_opt_set_bool(opts, "ipv6", e->ai_family == PF_INET6,
-                      &error_abort);
-    qemu_opt_set_bool(opts, "ipv4", e->ai_family != PF_INET6,
-                      &error_abort);
+    if (update_addr) {
+        g_free(saddr->host);
+        saddr->host = g_strdup(uaddr);
+        g_free(saddr->port);
+        saddr->port = g_strdup_printf("%d",
+                                      inet_getport(e) - port_offset);
+        saddr->has_ipv6 = saddr->ipv6 = e->ai_family == PF_INET6;
+        saddr->has_ipv4 = saddr->ipv4 = e->ai_family != PF_INET6;
+    }
     freeaddrinfo(res);
     return slisten;
 }
@@ -251,7 +271,7 @@ static void wait_for_connect(void *opaque)
 
     do {
         rc = qemu_getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &val, &valsize);
-    } while (rc == -1 && socket_error() == EINTR);
+    } while (rc == -1 && errno == EINTR);
 
     /* update rc to contain error */
     if (!rc && val) {
@@ -313,7 +333,7 @@ static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
     do {
         rc = 0;
         if (connect(sock, addr->ai_addr, addr->ai_addrlen) < 0) {
-            rc = -socket_error();
+            rc = -errno;
         }
     } while (rc == -EINTR);
 
@@ -329,38 +349,50 @@ static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
     return sock;
 }
 
-static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
+static struct addrinfo *inet_parse_connect_saddr(InetSocketAddress *saddr,
+                                                 Error **errp)
 {
     struct addrinfo ai, *res;
     int rc;
-    const char *addr;
-    const char *port;
+    Error *err = NULL;
+    static int useV4Mapped = 1;
 
     memset(&ai, 0, sizeof(ai));
 
-    ai.ai_flags = AI_CANONNAME | AI_V4MAPPED | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
+    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
+    if (atomic_read(&useV4Mapped)) {
+        ai.ai_flags |= AI_V4MAPPED;
+    }
+    ai.ai_family = inet_ai_family_from_address(saddr, &err);
     ai.ai_socktype = SOCK_STREAM;
 
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
-    if (addr == NULL || port == NULL) {
-        error_setg(errp, "host and/or port not specified");
+    if (err) {
+        error_propagate(errp, err);
         return NULL;
     }
 
-    if (qemu_opt_get_bool(opts, "ipv4", 0)) {
-        ai.ai_family = PF_INET;
-    }
-    if (qemu_opt_get_bool(opts, "ipv6", 0)) {
-        ai.ai_family = PF_INET6;
+    if (saddr->host == NULL || saddr->port == NULL) {
+        error_setg(errp, "host and/or port not specified");
+        return NULL;
     }
 
     /* lookup */
-    rc = getaddrinfo(addr, port, &ai, &res);
+    rc = getaddrinfo(saddr->host, saddr->port, &ai, &res);
+
+    /* At least FreeBSD and OS-X 10.6 declare AI_V4MAPPED but
+     * then don't implement it in their getaddrinfo(). Detect
+     * this and retry without the flag since that's preferrable
+     * to a fatal error
+     */
+    if (rc == EAI_BADFLAGS &&
+        (ai.ai_flags & AI_V4MAPPED)) {
+        atomic_set(&useV4Mapped, 0);
+        ai.ai_flags &= ~AI_V4MAPPED;
+        rc = getaddrinfo(saddr->host, saddr->port, &ai, &res);
+    }
     if (rc != 0) {
-        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
-                   gai_strerror(rc));
+        error_setg(errp, "address resolution failed for %s:%s: %s",
+                   saddr->host, saddr->port, gai_strerror(rc));
         return NULL;
     }
     return res;
@@ -369,8 +401,7 @@ static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
 /**
  * Create a socket and connect it to an address.
  *
- * @opts: QEMU options, recognized parameters strings "host" and "port",
- *        bools "ipv4" and "ipv6".
+ * @saddr: Inet socket address specification
  * @errp: set on error
  * @callback: callback function for non-blocking connect
  * @opaque: opaque for callback function
@@ -381,8 +412,8 @@ static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
  * function succeeds, callback will be called when the connection
  * completes, with the file descriptor on success, or -1 on error.
  */
-int inet_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int inet_connect_saddr(InetSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     Error *local_err = NULL;
     struct addrinfo *res, *e;
@@ -390,7 +421,7 @@ int inet_connect_opts(QemuOpts *opts, Error **errp,
     bool in_progress;
     ConnectState *connect_state = NULL;
 
-    res = inet_parse_connect_opts(opts, errp);
+    res = inet_parse_connect_saddr(saddr, errp);
     if (!res) {
         return -1;
     }
@@ -429,38 +460,41 @@ int inet_connect_opts(QemuOpts *opts, Error **errp,
     return sock;
 }
 
-int inet_dgram_opts(QemuOpts *opts, Error **errp)
+static int inet_dgram_saddr(InetSocketAddress *sraddr,
+                            InetSocketAddress *sladdr,
+                            Error **errp)
 {
     struct addrinfo ai, *peer = NULL, *local = NULL;
     const char *addr;
     const char *port;
     int sock = -1, rc;
+    Error *err = NULL;
 
     /* lookup peer addr */
     memset(&ai,0, sizeof(ai));
     ai.ai_flags = AI_CANONNAME | AI_V4MAPPED | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
+    ai.ai_family = inet_ai_family_from_address(sraddr, &err);
     ai.ai_socktype = SOCK_DGRAM;
 
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
+    if (err) {
+        error_propagate(errp, err);
+        goto err;
+    }
+
+    addr = sraddr->host;
+    port = sraddr->port;
     if (addr == NULL || strlen(addr) == 0) {
         addr = "localhost";
     }
     if (port == NULL || strlen(port) == 0) {
         error_setg(errp, "remote port not specified");
-        return -1;
+        goto err;
     }
 
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
-
     if (0 != (rc = getaddrinfo(addr, port, &ai, &peer))) {
         error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
                    gai_strerror(rc));
-	return -1;
+	goto err;
     }
 
     /* lookup local addr */
@@ -469,13 +503,19 @@ int inet_dgram_opts(QemuOpts *opts, Error **errp)
     ai.ai_family = peer->ai_family;
     ai.ai_socktype = SOCK_DGRAM;
 
-    addr = qemu_opt_get(opts, "localaddr");
-    port = qemu_opt_get(opts, "localport");
-    if (addr == NULL || strlen(addr) == 0) {
+    if (sladdr) {
+        addr = sladdr->host;
+        port = sladdr->port;
+        if (addr == NULL || strlen(addr) == 0) {
+            addr = NULL;
+        }
+        if (!port || strlen(port) == 0) {
+            port = "0";
+        }
+    } else {
         addr = NULL;
-    }
-    if (!port || strlen(port) == 0)
         port = "0";
+    }
 
     if (0 != (rc = getaddrinfo(addr, port, &ai, &local))) {
         error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
@@ -584,51 +624,31 @@ fail:
     return NULL;
 }
 
-static void inet_addr_to_opts(QemuOpts *opts, const InetSocketAddress *addr)
-{
-    bool ipv4 = addr->ipv4 || !addr->has_ipv4;
-    bool ipv6 = addr->ipv6 || !addr->has_ipv6;
-
-    if (!ipv4 || !ipv6) {
-        qemu_opt_set_bool(opts, "ipv4", ipv4, &error_abort);
-        qemu_opt_set_bool(opts, "ipv6", ipv6, &error_abort);
-    }
-    if (addr->has_to) {
-        qemu_opt_set_number(opts, "to", addr->to, &error_abort);
-    }
-    qemu_opt_set(opts, "host", addr->host, &error_abort);
-    qemu_opt_set(opts, "port", addr->port, &error_abort);
-}
-
 int inet_listen(const char *str, char *ostr, int olen,
                 int socktype, int port_offset, Error **errp)
 {
-    QemuOpts *opts;
     char *optstr;
     int sock = -1;
     InetSocketAddress *addr;
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
-        qapi_free_InetSocketAddress(addr);
-        sock = inet_listen_opts(opts, port_offset, errp);
+        sock = inet_listen_saddr(addr, port_offset, true, errp);
         if (sock != -1 && ostr) {
             optstr = strchr(str, ',');
-            if (qemu_opt_get_bool(opts, "ipv6", 0)) {
+            if (addr->ipv6) {
                 snprintf(ostr, olen, "[%s]:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
+                         addr->host,
+                         addr->port,
                          optstr ? optstr : "");
             } else {
                 snprintf(ostr, olen, "%s:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
+                         addr->host,
+                         addr->port,
                          optstr ? optstr : "");
             }
         }
-        qemu_opts_del(opts);
+        qapi_free_InetSocketAddress(addr);
     }
     return sock;
 }
@@ -643,17 +663,13 @@ int inet_listen(const char *str, char *ostr, int olen,
  **/
 int inet_connect(const char *str, Error **errp)
 {
-    QemuOpts *opts;
     int sock = -1;
     InetSocketAddress *addr;
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
+        sock = inet_connect_saddr(addr, errp, NULL, NULL);
         qapi_free_InetSocketAddress(addr);
-        sock = inet_connect_opts(opts, errp, NULL, NULL);
-        qemu_opts_del(opts);
     }
     return sock;
 }
@@ -675,7 +691,6 @@ int inet_nonblocking_connect(const char *str,
                              NonBlockingConnectHandler *callback,
                              void *opaque, Error **errp)
 {
-    QemuOpts *opts;
     int sock = -1;
     InetSocketAddress *addr;
 
@@ -683,21 +698,19 @@ int inet_nonblocking_connect(const char *str,
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
+        sock = inet_connect_saddr(addr, errp, callback, opaque);
         qapi_free_InetSocketAddress(addr);
-        sock = inet_connect_opts(opts, errp, callback, opaque);
-        qemu_opts_del(opts);
     }
     return sock;
 }
 
 #ifndef _WIN32
 
-int unix_listen_opts(QemuOpts *opts, Error **errp)
+static int unix_listen_saddr(UnixSocketAddress *saddr,
+                             bool update_addr,
+                             Error **errp)
 {
     struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
     int sock, fd;
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
@@ -708,8 +721,8 @@ int unix_listen_opts(QemuOpts *opts, Error **errp)
 
     memset(&un, 0, sizeof(un));
     un.sun_family = AF_UNIX;
-    if (path && strlen(path)) {
-        snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+    if (saddr->path && strlen(saddr->path)) {
+        snprintf(un.sun_path, sizeof(un.sun_path), "%s", saddr->path);
     } else {
         const char *tmpdir = getenv("TMPDIR");
         tmpdir = tmpdir ? tmpdir : "/tmp";
@@ -734,11 +747,13 @@ int unix_listen_opts(QemuOpts *opts, Error **errp)
             goto err;
         }
         close(fd);
-        qemu_opt_set(opts, "path", un.sun_path, &error_abort);
+        if (update_addr) {
+            g_free(saddr->path);
+            saddr->path = g_strdup(un.sun_path);
+        }
     }
 
-    if ((access(un.sun_path, F_OK) == 0) &&
-        unlink(un.sun_path) < 0) {
+    if (unlink(un.sun_path) < 0 && errno != ENOENT) {
         error_setg_errno(errp, errno,
                          "Failed to unlink socket %s", un.sun_path);
         goto err;
@@ -759,15 +774,14 @@ err:
     return -1;
 }
 
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int unix_connect_saddr(UnixSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
     ConnectState *connect_state = NULL;
     int sock, rc;
 
-    if (path == NULL) {
+    if (saddr->path == NULL) {
         error_setg(errp, "unix connect: no path specified");
         return -1;
     }
@@ -786,13 +800,13 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
 
     memset(&un, 0, sizeof(un));
     un.sun_family = AF_UNIX;
-    snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+    snprintf(un.sun_path, sizeof(un.sun_path), "%s", saddr->path);
 
     /* connect to peer */
     do {
         rc = 0;
         if (connect(sock, (struct sockaddr *) &un, sizeof(un)) < 0) {
-            rc = -socket_error();
+            rc = -errno;
         }
     } while (rc == -EINTR);
 
@@ -819,15 +833,17 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
 
 #else
 
-int unix_listen_opts(QemuOpts *opts, Error **errp)
+static int unix_listen_saddr(UnixSocketAddress *saddr,
+                             bool update_addr,
+                             Error **errp)
 {
     error_setg(errp, "unix sockets are not available on windows");
     errno = ENOTSUP;
     return -1;
 }
 
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int unix_connect_saddr(UnixSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     error_setg(errp, "unix sockets are not available on windows");
     errno = ENOTSUP;
@@ -838,11 +854,11 @@ int unix_connect_opts(QemuOpts *opts, Error **errp,
 /* compatibility wrapper */
 int unix_listen(const char *str, char *ostr, int olen, Error **errp)
 {
-    QemuOpts *opts;
     char *path, *optstr;
     int sock, len;
+    UnixSocketAddress *saddr;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
+    saddr = g_new0(UnixSocketAddress, 1);
 
     optstr = strchr(str, ',');
     if (optstr) {
@@ -850,30 +866,29 @@ int unix_listen(const char *str, char *ostr, int olen, Error **errp)
         if (len) {
             path = g_malloc(len+1);
             snprintf(path, len+1, "%.*s", len, str);
-            qemu_opt_set(opts, "path", path, &error_abort);
-            g_free(path);
+            saddr->path = path;
         }
     } else {
-        qemu_opt_set(opts, "path", str, &error_abort);
+        saddr->path = g_strdup(str);
     }
 
-    sock = unix_listen_opts(opts, errp);
+    sock = unix_listen_saddr(saddr, true, errp);
 
     if (sock != -1 && ostr)
-        snprintf(ostr, olen, "%s%s", qemu_opt_get(opts, "path"), optstr ? optstr : "");
-    qemu_opts_del(opts);
+        snprintf(ostr, olen, "%s%s", saddr->path, optstr ? optstr : "");
+    qapi_free_UnixSocketAddress(saddr);
     return sock;
 }
 
 int unix_connect(const char *path, Error **errp)
 {
-    QemuOpts *opts;
+    UnixSocketAddress *saddr;
     int sock;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    qemu_opt_set(opts, "path", path, &error_abort);
-    sock = unix_connect_opts(opts, errp, NULL, NULL);
-    qemu_opts_del(opts);
+    saddr = g_new0(UnixSocketAddress, 1);
+    saddr->path = g_strdup(path);
+    sock = unix_connect_saddr(saddr, errp, NULL, NULL);
+    qapi_free_UnixSocketAddress(saddr);
     return sock;
 }
 
@@ -882,15 +897,15 @@ int unix_nonblocking_connect(const char *path,
                              NonBlockingConnectHandler *callback,
                              void *opaque, Error **errp)
 {
-    QemuOpts *opts;
+    UnixSocketAddress *saddr;
     int sock = -1;
 
     g_assert(callback != NULL);
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    qemu_opt_set(opts, "path", path, &error_abort);
-    sock = unix_connect_opts(opts, errp, callback, opaque);
-    qemu_opts_del(opts);
+    saddr = g_new0(UnixSocketAddress, 1);
+    saddr->path = g_strdup(path);
+    sock = unix_connect_saddr(saddr, errp, callback, opaque);
+    qapi_free_UnixSocketAddress(saddr);
     return sock;
 }
 
@@ -904,23 +919,23 @@ SocketAddress *socket_parse(const char *str, Error **errp)
             error_setg(errp, "invalid Unix socket address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-            addr->q_unix = g_new(UnixSocketAddress, 1);
-            addr->q_unix->path = g_strdup(str + 5);
+            addr->type = SOCKET_ADDRESS_KIND_UNIX;
+            addr->u.q_unix.data = g_new(UnixSocketAddress, 1);
+            addr->u.q_unix.data->path = g_strdup(str + 5);
         }
     } else if (strstart(str, "fd:", NULL)) {
         if (str[3] == '\0') {
             error_setg(errp, "invalid file descriptor address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_FD;
-            addr->fd = g_new(String, 1);
-            addr->fd->str = g_strdup(str + 3);
+            addr->type = SOCKET_ADDRESS_KIND_FD;
+            addr->u.fd.data = g_new(String, 1);
+            addr->u.fd.data->str = g_strdup(str + 3);
         }
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = inet_parse(str, errp);
-        if (addr->inet == NULL) {
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = inet_parse(str, errp);
+        if (addr->u.inet.data == NULL) {
             goto fail;
         }
     }
@@ -934,23 +949,19 @@ fail:
 int socket_connect(SocketAddress *addr, Error **errp,
                    NonBlockingConnectHandler *callback, void *opaque)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
-        fd = inet_connect_opts(opts, errp, callback, opaque);
+        fd = inet_connect_saddr(addr->u.inet.data, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path, &error_abort);
-        fd = unix_connect_opts(opts, errp, callback, opaque);
+        fd = unix_connect_saddr(addr->u.q_unix.data, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd.data->str, errp);
         if (fd >= 0 && callback) {
             qemu_set_nonblock(fd);
             callback(fd, NULL, opaque);
@@ -960,58 +971,183 @@ int socket_connect(SocketAddress *addr, Error **errp,
     default:
         abort();
     }
-    qemu_opts_del(opts);
     return fd;
 }
 
 int socket_listen(SocketAddress *addr, Error **errp)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
-        fd = inet_listen_opts(opts, 0, errp);
+        fd = inet_listen_saddr(addr->u.inet.data, 0, false, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path, &error_abort);
-        fd = unix_listen_opts(opts, errp);
+        fd = unix_listen_saddr(addr->u.q_unix.data, false, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd.data->str, errp);
         break;
 
     default:
         abort();
     }
-    qemu_opts_del(opts);
     return fd;
 }
 
 int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (remote->kind) {
+    switch (remote->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, remote->inet);
-        if (local) {
-            qemu_opt_set(opts, "localaddr", local->inet->host, &error_abort);
-            qemu_opt_set(opts, "localport", local->inet->port, &error_abort);
-        }
-        fd = inet_dgram_opts(opts, errp);
+        fd = inet_dgram_saddr(remote->u.inet.data,
+                              local ? local->u.inet.data : NULL, errp);
         break;
 
     default:
         error_setg(errp, "socket type unsupported for datagram");
         fd = -1;
     }
-    qemu_opts_del(opts);
     return fd;
 }
+
+
+static SocketAddress *
+socket_sockaddr_to_address_inet(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    char host[NI_MAXHOST];
+    char serv[NI_MAXSERV];
+    SocketAddress *addr;
+    InetSocketAddress *inet;
+    int ret;
+
+    ret = getnameinfo((struct sockaddr *)sa, salen,
+                      host, sizeof(host),
+                      serv, sizeof(serv),
+                      NI_NUMERICHOST | NI_NUMERICSERV);
+    if (ret != 0) {
+        error_setg(errp, "Cannot format numeric socket address: %s",
+                   gai_strerror(ret));
+        return NULL;
+    }
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    inet = addr->u.inet.data = g_new0(InetSocketAddress, 1);
+    inet->host = g_strdup(host);
+    inet->port = g_strdup(serv);
+    if (sa->ss_family == AF_INET) {
+        inet->has_ipv4 = inet->ipv4 = true;
+    } else {
+        inet->has_ipv6 = inet->ipv6 = true;
+    }
+
+    return addr;
+}
+
+
+#ifndef WIN32
+static SocketAddress *
+socket_sockaddr_to_address_unix(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    SocketAddress *addr;
+    struct sockaddr_un *su = (struct sockaddr_un *)sa;
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    if (su->sun_path[0]) {
+        addr->u.q_unix.data->path = g_strndup(su->sun_path,
+                                              sizeof(su->sun_path));
+    }
+
+    return addr;
+}
+#endif /* WIN32 */
+
+SocketAddress *
+socket_sockaddr_to_address(struct sockaddr_storage *sa,
+                           socklen_t salen,
+                           Error **errp)
+{
+    switch (sa->ss_family) {
+    case AF_INET:
+    case AF_INET6:
+        return socket_sockaddr_to_address_inet(sa, salen, errp);
+
+#ifndef WIN32
+    case AF_UNIX:
+        return socket_sockaddr_to_address_unix(sa, salen, errp);
+#endif /* WIN32 */
+
+    default:
+        error_setg(errp, "socket family %d unsupported",
+                   sa->ss_family);
+        return NULL;
+    }
+    return 0;
+}
+
+
+SocketAddress *socket_local_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getsockname(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to query local socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+
+SocketAddress *socket_remote_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getpeername(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to query remote socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+
+void qapi_copy_SocketAddress(SocketAddress **p_dest,
+                             SocketAddress *src)
+{
+    QmpOutputVisitor *qov;
+    QmpInputVisitor *qiv;
+    Visitor *ov, *iv;
+    QObject *obj;
+
+    *p_dest = NULL;
+
+    qov = qmp_output_visitor_new();
+    ov = qmp_output_get_visitor(qov);
+    visit_type_SocketAddress(ov, NULL, &src, &error_abort);
+    obj = qmp_output_get_qobject(qov);
+    qmp_output_visitor_cleanup(qov);
+    if (!obj) {
+        return;
+    }
+
+    qiv = qmp_input_visitor_new(obj);
+    iv = qmp_input_get_visitor(qiv);
+    visit_type_SocketAddress(iv, NULL, p_dest, &error_abort);
+    qmp_input_visitor_cleanup(qiv);
+    qobject_decref(obj);
+}
diff --git a/qemu/util/qemu-thread-posix.c b/qemu/util/qemu-thread-posix.c
index ba67cec62..74a3023f3 100644
--- a/qemu/util/qemu-thread-posix.c
+++ b/qemu/util/qemu-thread-posix.c
@@ -10,16 +10,7 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <time.h>
-#include <signal.h>
-#include <stdint.h>
-#include <string.h>
-#include <limits.h>
-#include <unistd.h>
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #ifdef __linux__
 #include <sys/syscall.h>
 #include <linux/futex.h>
@@ -298,7 +289,16 @@ static inline void futex_wake(QemuEvent *ev, int n)
 
 static inline void futex_wait(QemuEvent *ev, unsigned val)
 {
-    futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0);
+    while (futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0)) {
+        switch (errno) {
+        case EWOULDBLOCK:
+            return;
+        case EINTR:
+            break; /* get out of switch and retry */
+        default:
+            abort();
+        }
+    }
 }
 #else
 static inline void futex_wake(QemuEvent *ev, int n)
@@ -389,7 +389,7 @@ void qemu_event_wait(QemuEvent *ev)
             /*
              * Leave the event reset and tell qemu_event_set that there
              * are waiters.  No need to retry, because there cannot be
-             * a concurent busy->free transition.  After the CAS, the
+             * a concurrent busy->free transition.  After the CAS, the
              * event will be either set or busy.
              */
             if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
diff --git a/qemu/util/qemu-thread-win32.c b/qemu/util/qemu-thread-win32.c
index 406b52f91..98a5ddff8 100644
--- a/qemu/util/qemu-thread-win32.c
+++ b/qemu/util/qemu-thread-win32.c
@@ -10,12 +10,11 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/thread.h"
 #include "qemu/notify.h"
 #include <process.h>
-#include <assert.h>
-#include <limits.h>
 
 static bool name_threads;
 
@@ -238,10 +237,34 @@ void qemu_sem_wait(QemuSemaphore *sem)
     }
 }
 
+/* Wrap a Win32 manual-reset event with a fast userspace path.  The idea
+ * is to reset the Win32 event lazily, as part of a test-reset-test-wait
+ * sequence.  Such a sequence is, indeed, how QemuEvents are used by
+ * RCU and other subsystems!
+ *
+ * Valid transitions:
+ * - free->set, when setting the event
+ * - busy->set, when setting the event, followed by futex_wake
+ * - set->free, when resetting the event
+ * - free->busy, when waiting
+ *
+ * set->busy does not happen (it can be observed from the outside but
+ * it really is set->free->busy).
+ *
+ * busy->free provably cannot happen; to enforce it, the set->free transition
+ * is done with an OR, which becomes a no-op if the event has concurrently
+ * transitioned to free or busy (and is faster than cmpxchg).
+ */
+
+#define EV_SET         0
+#define EV_FREE        1
+#define EV_BUSY       -1
+
 void qemu_event_init(QemuEvent *ev, bool init)
 {
     /* Manual reset.  */
-    ev->event = CreateEvent(NULL, TRUE, init, NULL);
+    ev->event = CreateEvent(NULL, TRUE, TRUE, NULL);
+    ev->value = (init ? EV_SET : EV_FREE);
 }
 
 void qemu_event_destroy(QemuEvent *ev)
@@ -251,17 +274,51 @@ void qemu_event_destroy(QemuEvent *ev)
 
 void qemu_event_set(QemuEvent *ev)
 {
-    SetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) != EV_SET) {
+        if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+            /* There were waiters, wake them up.  */
+            SetEvent(ev->event);
+        }
+    }
 }
 
 void qemu_event_reset(QemuEvent *ev)
 {
-    ResetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) == EV_SET) {
+        /* If there was a concurrent reset (or even reset+wait),
+         * do nothing.  Otherwise change EV_SET->EV_FREE.
+         */
+        atomic_or(&ev->value, EV_FREE);
+    }
 }
 
 void qemu_event_wait(QemuEvent *ev)
 {
-    WaitForSingleObject(ev->event, INFINITE);
+    unsigned value;
+
+    value = atomic_mb_read(&ev->value);
+    if (value != EV_SET) {
+        if (value == EV_FREE) {
+            /* qemu_event_set is not yet going to call SetEvent, but we are
+             * going to do another check for EV_SET below when setting EV_BUSY.
+             * At that point it is safe to call WaitForSingleObject.
+             */
+            ResetEvent(ev->event);
+
+            /* Tell qemu_event_set that there are waiters.  No need to retry
+             * because there cannot be a concurent busy->free transition.
+             * After the CAS, the event will be either set or busy.
+             */
+            if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+                value = EV_SET;
+            } else {
+                value = EV_BUSY;
+            }
+        }
+        if (value == EV_BUSY) {
+            WaitForSingleObject(ev->event, INFINITE);
+        }
+    }
 }
 
 struct QemuThreadData {
diff --git a/qemu/util/qemu-timer-common.c b/qemu/util/qemu-timer-common.c
index 95e0847c7..06d084d36 100644
--- a/qemu/util/qemu-timer-common.c
+++ b/qemu/util/qemu-timer-common.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 
 /***********************************************************/
diff --git a/qemu/util/rcu.c b/qemu/util/rcu.c
index cdcad678b..bceb3e472 100644
--- a/qemu/util/rcu.c
+++ b/qemu/util/rcu.c
@@ -26,12 +26,8 @@
  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <errno.h>
 #include "qemu/rcu.h"
 #include "qemu/atomic.h"
 #include "qemu/thread.h"
@@ -47,7 +43,8 @@
 unsigned long rcu_gp_ctr = RCU_GP_LOCKED;
 
 QemuEvent rcu_gp_event;
-static QemuMutex rcu_gp_lock;
+static QemuMutex rcu_registry_lock;
+static QemuMutex rcu_sync_lock;
 
 /*
  * Check whether a quiescent state was crossed between the beginning of
@@ -66,7 +63,7 @@ static inline int rcu_gp_ongoing(unsigned long *ctr)
  */
 __thread struct rcu_reader_data rcu_reader;
 
-/* Protected by rcu_gp_lock.  */
+/* Protected by rcu_registry_lock.  */
 typedef QLIST_HEAD(, rcu_reader_data) ThreadList;
 static ThreadList registry = QLIST_HEAD_INITIALIZER(registry);
 
@@ -114,10 +111,26 @@ static void wait_for_readers(void)
             break;
         }
 
-        /* Wait for one thread to report a quiescent state and
-         * try again.
+        /* Wait for one thread to report a quiescent state and try again.
+         * Release rcu_registry_lock, so rcu_(un)register_thread() doesn't
+         * wait too much time.
+         *
+         * rcu_register_thread() may add nodes to &registry; it will not
+         * wake up synchronize_rcu, but that is okay because at least another
+         * thread must exit its RCU read-side critical section before
+         * synchronize_rcu is done.  The next iteration of the loop will
+         * move the new thread's rcu_reader from &registry to &qsreaders,
+         * because rcu_gp_ongoing() will return false.
+         *
+         * rcu_unregister_thread() may remove nodes from &qsreaders instead
+         * of &registry if it runs during qemu_event_wait.  That's okay;
+         * the node then will not be added back to &registry by QLIST_SWAP
+         * below.  The invariant is that the node is part of one list when
+         * rcu_registry_lock is released.
          */
+        qemu_mutex_unlock(&rcu_registry_lock);
         qemu_event_wait(&rcu_gp_event);
+        qemu_mutex_lock(&rcu_registry_lock);
     }
 
     /* put back the reader list in the registry */
@@ -126,7 +139,8 @@ static void wait_for_readers(void)
 
 void synchronize_rcu(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
 
     if (!QLIST_EMPTY(&registry)) {
         /* In either case, the atomic_mb_set below blocks stores that free
@@ -149,7 +163,8 @@ void synchronize_rcu(void)
         wait_for_readers();
     }
 
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
 }
 
 
@@ -273,23 +288,24 @@ void call_rcu1(struct rcu_head *node, void (*func)(struct rcu_head *node))
 void rcu_register_thread(void)
 {
     assert(rcu_reader.ctr == 0);
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
     QLIST_INSERT_HEAD(&registry, &rcu_reader, node);
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
 }
 
 void rcu_unregister_thread(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
     QLIST_REMOVE(&rcu_reader, node);
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
 }
 
 static void rcu_init_complete(void)
 {
     QemuThread thread;
 
-    qemu_mutex_init(&rcu_gp_lock);
+    qemu_mutex_init(&rcu_registry_lock);
+    qemu_mutex_init(&rcu_sync_lock);
     qemu_event_init(&rcu_gp_event, true);
 
     qemu_event_init(&rcu_call_ready_event, false);
@@ -306,12 +322,14 @@ static void rcu_init_complete(void)
 #ifdef CONFIG_POSIX
 static void rcu_init_lock(void)
 {
-    qemu_mutex_lock(&rcu_gp_lock);
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
 }
 
 static void rcu_init_unlock(void)
 {
-    qemu_mutex_unlock(&rcu_gp_lock);
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
 }
 #endif
 
diff --git a/qemu/util/readline.c b/qemu/util/readline.c
index cc1302ac0..bbdee790b 100644
--- a/qemu/util/readline.c
+++ b/qemu/util/readline.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/readline.h"
+#include "qemu/cutils.h"
 
 #define IS_NORM 0
 #define IS_ESC  1
diff --git a/qemu/util/rfifolock.c b/qemu/util/rfifolock.c
index afbf7488d..c22f5feee 100644
--- a/qemu/util/rfifolock.c
+++ b/qemu/util/rfifolock.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "qemu/rfifolock.h"
 
 void rfifolock_init(RFifoLock *r, void (*cb)(void *), void *opaque)
diff --git a/qemu/util/throttle.c b/qemu/util/throttle.c
index 706c13111..71246b234 100644
--- a/qemu/util/throttle.c
+++ b/qemu/util/throttle.c
@@ -22,6 +22,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/throttle.h"
 #include "qemu/timer.h"
 #include "block/aio.h"
@@ -40,6 +42,14 @@ void throttle_leak_bucket(LeakyBucket *bkt, int64_t delta_ns)
 
     /* make the bucket leak */
     bkt->level = MAX(bkt->level - leak, 0);
+
+    /* if we allow bursts for more than one second we also need to
+     * keep track of bkt->burst_level so the bkt->max goal per second
+     * is attained */
+    if (bkt->burst_length > 1) {
+        leak = (bkt->max * (double) delta_ns) / NANOSECONDS_PER_SECOND;
+        bkt->burst_level = MAX(bkt->burst_level - leak, 0);
+    }
 }
 
 /* Calculate the time delta since last leak and make proportionals leaks
@@ -90,13 +100,24 @@ int64_t throttle_compute_wait(LeakyBucket *bkt)
         return 0;
     }
 
-    extra = bkt->level - bkt->max;
+    /* If the bucket is full then we have to wait */
+    extra = bkt->level - bkt->max * bkt->burst_length;
+    if (extra > 0) {
+        return throttle_do_compute_wait(bkt->avg, extra);
+    }
 
-    if (extra <= 0) {
-        return 0;
+    /* If the bucket is not full yet we have to make sure that we
+     * fulfill the goal of bkt->max units per second. */
+    if (bkt->burst_length > 1) {
+        /* We use 1/10 of the max value to smooth the throttling.
+         * See throttle_fix_bucket() for more details. */
+        extra = bkt->burst_level - bkt->max / 10;
+        if (extra > 0) {
+            return throttle_do_compute_wait(bkt->max, extra);
+        }
     }
 
-    return throttle_do_compute_wait(bkt->avg, extra);
+    return 0;
 }
 
 /* This function compute the time that must be waited while this IO
@@ -136,10 +157,10 @@ static int64_t throttle_compute_wait_for(ThrottleState *ts,
  * @next_timestamp: the resulting timer
  * @ret:        true if a timer must be set
  */
-bool throttle_compute_timer(ThrottleState *ts,
-                            bool is_write,
-                            int64_t now,
-                            int64_t *next_timestamp)
+static bool throttle_compute_timer(ThrottleState *ts,
+                                   bool is_write,
+                                   int64_t now,
+                                   int64_t *next_timestamp)
 {
     int64_t wait;
 
@@ -170,10 +191,24 @@ void throttle_timers_attach_aio_context(ThrottleTimers *tt,
                                   tt->write_timer_cb, tt->timer_opaque);
 }
 
+/*
+ * Initialize the ThrottleConfig structure to a valid state
+ * @cfg: the config to initialize
+ */
+void throttle_config_init(ThrottleConfig *cfg)
+{
+    unsigned i;
+    memset(cfg, 0, sizeof(*cfg));
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        cfg->buckets[i].burst_length = 1;
+    }
+}
+
 /* To be called first on the ThrottleState */
 void throttle_init(ThrottleState *ts)
 {
     memset(ts, 0, sizeof(ThrottleState));
+    throttle_config_init(&ts->cfg);
 }
 
 /* To be called first on the ThrottleTimers */
@@ -247,13 +282,14 @@ bool throttle_enabled(ThrottleConfig *cfg)
     return false;
 }
 
-/* return true if any two throttling parameters conflicts
- *
+/* check if a throttling configuration is valid
  * @cfg: the throttling configuration to inspect
- * @ret: true if any conflict detected else false
+ * @ret: true if valid else false
+ * @errp: error object
  */
-bool throttle_conflicting(ThrottleConfig *cfg)
+bool throttle_is_valid(ThrottleConfig *cfg, Error **errp)
 {
+    int i;
     bool bps_flag, ops_flag;
     bool bps_max_flag, ops_max_flag;
 
@@ -273,31 +309,40 @@ bool throttle_conflicting(ThrottleConfig *cfg)
                    (cfg->buckets[THROTTLE_OPS_READ].max ||
                    cfg->buckets[THROTTLE_OPS_WRITE].max);
 
-    return bps_flag || ops_flag || bps_max_flag || ops_max_flag;
-}
-
-/* check if a throttling configuration is valid
- * @cfg: the throttling configuration to inspect
- * @ret: true if valid else false
- */
-bool throttle_is_valid(ThrottleConfig *cfg)
-{
-    bool invalid = false;
-    int i;
+    if (bps_flag || ops_flag || bps_max_flag || ops_max_flag) {
+        error_setg(errp, "bps/iops/max total values and read/write values"
+                   " cannot be used at the same time");
+        return false;
+    }
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        if (cfg->buckets[i].avg < 0) {
-            invalid = true;
+        if (cfg->buckets[i].avg < 0 ||
+            cfg->buckets[i].max < 0 ||
+            cfg->buckets[i].avg > THROTTLE_VALUE_MAX ||
+            cfg->buckets[i].max > THROTTLE_VALUE_MAX) {
+            error_setg(errp, "bps/iops/max values must be within [0, %lld]",
+                       THROTTLE_VALUE_MAX);
+            return false;
         }
-    }
 
-    for (i = 0; i < BUCKETS_COUNT; i++) {
-        if (cfg->buckets[i].max < 0) {
-            invalid = true;
+        if (!cfg->buckets[i].burst_length) {
+            error_setg(errp, "the burst length cannot be 0");
+            return false;
+        }
+
+        if (cfg->buckets[i].burst_length > 1 && !cfg->buckets[i].max) {
+            error_setg(errp, "burst length set without burst rate");
+            return false;
+        }
+
+        if (cfg->buckets[i].max && !cfg->buckets[i].avg) {
+            error_setg(errp, "bps_max/iops_max require corresponding"
+                       " bps/iops values");
+            return false;
         }
     }
 
-    return !invalid;
+    return true;
 }
 
 /* fix bucket parameters */
@@ -306,7 +351,7 @@ static void throttle_fix_bucket(LeakyBucket *bkt)
     double min;
 
     /* zero bucket level */
-    bkt->level = 0;
+    bkt->level = bkt->burst_level = 0;
 
     /* The following is done to cope with the Linux CFQ block scheduler
      * which regroup reads and writes by block of 100ms in the guest.
@@ -409,22 +454,36 @@ bool throttle_schedule_timer(ThrottleState *ts,
  */
 void throttle_account(ThrottleState *ts, bool is_write, uint64_t size)
 {
+    const BucketType bucket_types_size[2][2] = {
+        { THROTTLE_BPS_TOTAL, THROTTLE_BPS_READ },
+        { THROTTLE_BPS_TOTAL, THROTTLE_BPS_WRITE }
+    };
+    const BucketType bucket_types_units[2][2] = {
+        { THROTTLE_OPS_TOTAL, THROTTLE_OPS_READ },
+        { THROTTLE_OPS_TOTAL, THROTTLE_OPS_WRITE }
+    };
     double units = 1.0;
+    unsigned i;
 
     /* if cfg.op_size is defined and smaller than size we compute unit count */
     if (ts->cfg.op_size && size > ts->cfg.op_size) {
         units = (double) size / ts->cfg.op_size;
     }
 
-    ts->cfg.buckets[THROTTLE_BPS_TOTAL].level += size;
-    ts->cfg.buckets[THROTTLE_OPS_TOTAL].level += units;
+    for (i = 0; i < 2; i++) {
+        LeakyBucket *bkt;
+
+        bkt = &ts->cfg.buckets[bucket_types_size[is_write][i]];
+        bkt->level += size;
+        if (bkt->burst_length > 1) {
+            bkt->burst_level += size;
+        }
 
-    if (is_write) {
-        ts->cfg.buckets[THROTTLE_BPS_WRITE].level += size;
-        ts->cfg.buckets[THROTTLE_OPS_WRITE].level += units;
-    } else {
-        ts->cfg.buckets[THROTTLE_BPS_READ].level += size;
-        ts->cfg.buckets[THROTTLE_OPS_READ].level += units;
+        bkt = &ts->cfg.buckets[bucket_types_units[is_write][i]];
+        bkt->level += units;
+        if (bkt->burst_length > 1) {
+            bkt->burst_level += units;
+        }
     }
 }
 
diff --git a/qemu/util/timed-average.c b/qemu/util/timed-average.c
new file mode 100644
index 000000000..2eef9cbb1
--- /dev/null
+++ b/qemu/util/timed-average.c
@@ -0,0 +1,231 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free sofware: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Sofware Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu/timed-average.h"
+
+/* This module computes an average of a set of values within a time
+ * window.
+ *
+ * Algorithm:
+ *
+ * - Create two windows with a certain expiration period, and
+ *   offsetted by period / 2.
+ * - Each time you want to account a new value, do it in both windows.
+ * - The minimum / maximum / average values are always returned from
+ *   the oldest window.
+ *
+ * Example:
+ *
+ *        t=0          |t=0.5           |t=1          |t=1.5            |t=2
+ *        wnd0: [0,0.5)|wnd0: [0.5,1.5) |             |wnd0: [1.5,2.5)  |
+ *        wnd1: [0,1)  |                |wnd1: [1,2)  |                 |
+ *
+ * Values are returned from:
+ *
+ *        wnd0---------|wnd1------------|wnd0---------|wnd1-------------|
+ */
+
+/* Update the expiration of a time window
+ *
+ * @w:      the window used
+ * @now:    the current time in nanoseconds
+ * @period: the expiration period in nanoseconds
+ */
+static void update_expiration(TimedAverageWindow *w, int64_t now,
+                              int64_t period)
+{
+    /* time elapsed since the last theoretical expiration */
+    int64_t elapsed = (now - w->expiration) % period;
+    /* time remaininging until the next expiration */
+    int64_t remaining = period - elapsed;
+    /* compute expiration */
+    w->expiration = now + remaining;
+}
+
+/* Reset a window
+ *
+ * @w: the window to reset
+ */
+static void window_reset(TimedAverageWindow *w)
+{
+    w->min = UINT64_MAX;
+    w->max = 0;
+    w->sum = 0;
+    w->count = 0;
+}
+
+/* Get the current window (that is, the one with the earliest
+ * expiration time).
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: a pointer to the current window
+ */
+static TimedAverageWindow *current_window(TimedAverage *ta)
+{
+     return &ta->windows[ta->current];
+}
+
+/* Initialize a TimedAverage structure
+ *
+ * @ta:         the TimedAverage structure
+ * @clock_type: the type of clock to use
+ * @period:     the time window period in nanoseconds
+ */
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period)
+{
+    int64_t now = qemu_clock_get_ns(clock_type);
+
+    /* Returned values are from the oldest window, so they belong to
+     * the interval [ta->period/2,ta->period). By adjusting the
+     * requested period by 4/3, we guarantee that they're in the
+     * interval [2/3 period,4/3 period), closer to the requested
+     * period on average */
+    ta->period = (uint64_t) period * 4 / 3;
+    ta->clock_type = clock_type;
+    ta->current = 0;
+
+    window_reset(&ta->windows[0]);
+    window_reset(&ta->windows[1]);
+
+    /* Both windows are offsetted by half a period */
+    ta->windows[0].expiration = now + ta->period / 2;
+    ta->windows[1].expiration = now + ta->period;
+}
+
+/* Check if the time windows have expired, updating their counters and
+ * expiration time if that's the case.
+ *
+ * @ta: the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) within the current
+ *           window will be stored here
+ */
+static void check_expirations(TimedAverage *ta, uint64_t *elapsed)
+{
+    int64_t now = qemu_clock_get_ns(ta->clock_type);
+    int i;
+
+    assert(ta->period != 0);
+
+    /* Check if the windows have expired */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+        if (w->expiration <= now) {
+            window_reset(w);
+            update_expiration(w, now, ta->period);
+        }
+    }
+
+    /* Make ta->current point to the oldest window */
+    if (ta->windows[0].expiration < ta->windows[1].expiration) {
+        ta->current = 0;
+    } else {
+        ta->current = 1;
+    }
+
+    /* Calculate the elapsed time within the current window */
+    if (elapsed) {
+        int64_t remaining = ta->windows[ta->current].expiration - now;
+        *elapsed = ta->period - remaining;
+    }
+}
+
+/* Account a value
+ *
+ * @ta:    the TimedAverage structure
+ * @value: the value to account
+ */
+void timed_average_account(TimedAverage *ta, uint64_t value)
+{
+    int i;
+    check_expirations(ta, NULL);
+
+    /* Do the accounting in both windows at the same time */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+
+        w->sum += value;
+        w->count++;
+
+        if (value < w->min) {
+            w->min = value;
+        }
+
+        if (value > w->max) {
+            w->max = value;
+        }
+    }
+}
+
+/* Get the minimum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the minimum value
+ */
+uint64_t timed_average_min(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->min < UINT64_MAX ? w->min : 0;
+}
+
+/* Get the average value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the average value
+ */
+uint64_t timed_average_avg(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->count > 0 ? w->sum / w->count : 0;
+}
+
+/* Get the maximum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the maximum value
+ */
+uint64_t timed_average_max(TimedAverage *ta)
+{
+    check_expirations(ta, NULL);
+    return current_window(ta)->max;
+}
+
+/* Get the sum of all accounted values
+ * @ta:      the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) will be stored here
+ * @ret:     the sum of all accounted values
+ */
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, elapsed);
+    w = current_window(ta);
+    return w->sum;
+}
diff --git a/qemu/util/unicode.c b/qemu/util/unicode.c
index d1c865885..a812a3517 100644
--- a/qemu/util/unicode.c
+++ b/qemu/util/unicode.c
@@ -10,7 +10,8 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/unicode.h"
 
 /**
  * mod_utf8_codepoint:
diff --git a/qemu/util/uri.c b/qemu/util/uri.c
index 550b98458..d109d6c01 100644
--- a/qemu/util/uri.c
+++ b/qemu/util/uri.c
@@ -51,9 +51,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <glib.h>
-#include <string.h>
-#include <stdio.h>
 
 #include "qemu/uri.h"
 
diff --git a/qemu/vl.c b/qemu/vl.c
index 0adbbd674..5fd22cb2e 100644
--- a/qemu/vl.c
+++ b/qemu/vl.c
@@ -21,14 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-
-#include "config-host.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
 
 #ifdef CONFIG_SECCOMP
 #include "sysemu/seccomp.h"
@@ -68,7 +63,7 @@ int main(int argc, char **argv)
 #include "hw/isa/isa.h"
 #include "hw/bt.h"
 #include "sysemu/watchdog.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/xen/xen.h"
 #include "hw/qdev.h"
 #include "hw/loader.h"
@@ -78,6 +73,7 @@ int main(int argc, char **argv)
 #include "net/slirp.h"
 #include "monitor/monitor.h"
 #include "ui/console.h"
+#include "ui/input.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/numa.h"
 #include "exec/gdbstub.h"
@@ -113,7 +109,6 @@ int main(int argc, char **argv)
 #include "qemu/queue.h"
 #include "sysemu/cpus.h"
 #include "sysemu/arch_init.h"
-#include "qemu/osdep.h"
 
 #include "ui/qemu-spice.h"
 #include "qapi/string-input-visitor.h"
@@ -122,6 +117,8 @@ int main(int argc, char **argv)
 #include "qapi-event.h"
 #include "exec/semihost.h"
 #include "crypto/init.h"
+#include "sysemu/replay.h"
+#include "qapi/qmp/qerror.h"
 
 #define MAX_VIRTIO_CONSOLES 1
 #define MAX_SCLP_CONSOLES 1
@@ -225,7 +222,6 @@ static struct {
     { .driver = "ide-drive",            .flag = &default_cdrom     },
     { .driver = "scsi-cd",              .flag = &default_cdrom     },
     { .driver = "virtio-serial-pci",    .flag = &default_virtcon   },
-    { .driver = "virtio-serial-s390",   .flag = &default_virtcon   },
     { .driver = "virtio-serial",        .flag = &default_virtcon   },
     { .driver = "VGA",                  .flag = &default_vga       },
     { .driver = "isa-vga",              .flag = &default_vga       },
@@ -269,10 +265,14 @@ static QemuOptsList qemu_sandbox_opts = {
 
 static QemuOptsList qemu_trace_opts = {
     .name = "trace",
-    .implied_opt_name = "trace",
+    .implied_opt_name = "enable",
     .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
     .desc = {
         {
+            .name = "enable",
+            .type = QEMU_OPT_STRING,
+        },
+        {
             .name = "events",
             .type = QEMU_OPT_STRING,
         },{
@@ -474,6 +474,12 @@ static QemuOptsList qemu_icount_opts = {
         }, {
             .name = "sleep",
             .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "rr",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "rrfile",
+            .type = QEMU_OPT_STRING,
         },
         { /* end of list */ }
     },
@@ -512,6 +518,10 @@ static QemuOptsList qemu_fw_cfg_opts = {
             .type = QEMU_OPT_STRING,
             .help = "Sets the name of the file from which\n"
                     "the fw_cfg blob will be loaded",
+        }, {
+            .name = "string",
+            .type = QEMU_OPT_STRING,
+            .help = "Sets content of the blob to be inserted from a string",
         },
         { /* end of list */ }
     },
@@ -534,10 +544,8 @@ const char *qemu_get_vm_name(void)
 
 static void res_free(void)
 {
-    if (boot_splash_filedata != NULL) {
-        g_free(boot_splash_filedata);
-        boot_splash_filedata = NULL;
-    }
+    g_free(boot_splash_filedata);
+    boot_splash_filedata = NULL;
 }
 
 static int default_driver_check(void *opaque, QemuOpts *opts, Error **errp)
@@ -560,8 +568,8 @@ static int default_driver_check(void *opaque, QemuOpts *opts, Error **errp)
 
 static RunState current_run_state = RUN_STATE_PRELAUNCH;
 
-/* We use RUN_STATE_MAX but any invalid value will do */
-static RunState vmstop_requested = RUN_STATE_MAX;
+/* We use RUN_STATE__MAX but any invalid value will do */
+static RunState vmstop_requested = RUN_STATE__MAX;
 static QemuMutex vmstop_lock;
 
 typedef struct {
@@ -573,6 +581,7 @@ static const RunStateTransition runstate_transitions_def[] = {
     /*     from      ->     to      */
     { RUN_STATE_DEBUG, RUN_STATE_RUNNING },
     { RUN_STATE_DEBUG, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_DEBUG, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_INMIGRATE, RUN_STATE_INTERNAL_ERROR },
     { RUN_STATE_INMIGRATE, RUN_STATE_IO_ERROR },
@@ -582,18 +591,25 @@ static const RunStateTransition runstate_transitions_def[] = {
     { RUN_STATE_INMIGRATE, RUN_STATE_SUSPENDED },
     { RUN_STATE_INMIGRATE, RUN_STATE_WATCHDOG },
     { RUN_STATE_INMIGRATE, RUN_STATE_GUEST_PANICKED },
+    { RUN_STATE_INMIGRATE, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_INMIGRATE, RUN_STATE_PRELAUNCH },
+    { RUN_STATE_INMIGRATE, RUN_STATE_POSTMIGRATE },
 
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_IO_ERROR, RUN_STATE_RUNNING },
     { RUN_STATE_IO_ERROR, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_IO_ERROR, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_PAUSED, RUN_STATE_RUNNING },
     { RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_PAUSED, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
     { RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_POSTMIGRATE, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_PRELAUNCH, RUN_STATE_RUNNING },
     { RUN_STATE_PRELAUNCH, RUN_STATE_FINISH_MIGRATE },
@@ -601,8 +617,10 @@ static const RunStateTransition runstate_transitions_def[] = {
 
     { RUN_STATE_FINISH_MIGRATE, RUN_STATE_RUNNING },
     { RUN_STATE_FINISH_MIGRATE, RUN_STATE_POSTMIGRATE },
+    { RUN_STATE_FINISH_MIGRATE, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_RESTORE_VM, RUN_STATE_RUNNING },
+    { RUN_STATE_RESTORE_VM, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_RUNNING, RUN_STATE_DEBUG },
     { RUN_STATE_RUNNING, RUN_STATE_INTERNAL_ERROR },
@@ -619,22 +637,26 @@ static const RunStateTransition runstate_transitions_def[] = {
 
     { RUN_STATE_SHUTDOWN, RUN_STATE_PAUSED },
     { RUN_STATE_SHUTDOWN, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SHUTDOWN, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_DEBUG, RUN_STATE_SUSPENDED },
     { RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
     { RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
     { RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SUSPENDED, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
     { RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_WATCHDOG, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_RUNNING },
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_GUEST_PANICKED, RUN_STATE_PRELAUNCH },
 
-    { RUN_STATE_MAX, RUN_STATE_MAX },
+    { RUN_STATE__MAX, RUN_STATE__MAX },
 };
 
-static bool runstate_valid_transitions[RUN_STATE_MAX][RUN_STATE_MAX];
+static bool runstate_valid_transitions[RUN_STATE__MAX][RUN_STATE__MAX];
 
 bool runstate_check(RunState state)
 {
@@ -658,7 +680,7 @@ static void runstate_init(void)
     const RunStateTransition *p;
 
     memset(&runstate_valid_transitions, 0, sizeof(runstate_valid_transitions));
-    for (p = &runstate_transitions_def[0]; p->from != RUN_STATE_MAX; p++) {
+    for (p = &runstate_transitions_def[0]; p->from != RUN_STATE__MAX; p++) {
         runstate_valid_transitions[p->from][p->to] = true;
     }
 
@@ -668,12 +690,12 @@ static void runstate_init(void)
 /* This function will abort() on invalid state transitions */
 void runstate_set(RunState new_state)
 {
-    assert(new_state < RUN_STATE_MAX);
+    assert(new_state < RUN_STATE__MAX);
 
     if (!runstate_valid_transitions[current_run_state][new_state]) {
-        fprintf(stderr, "ERROR: invalid runstate transition: '%s' -> '%s'\n",
-                RunState_lookup[current_run_state],
-                RunState_lookup[new_state]);
+        error_report("invalid runstate transition: '%s' -> '%s'",
+                     RunState_lookup[current_run_state],
+                     RunState_lookup[new_state]);
         abort();
     }
     trace_runstate_set(new_state);
@@ -706,9 +728,9 @@ static bool qemu_vmstop_requested(RunState *r)
 {
     qemu_mutex_lock(&vmstop_lock);
     *r = vmstop_requested;
-    vmstop_requested = RUN_STATE_MAX;
+    vmstop_requested = RUN_STATE__MAX;
     qemu_mutex_unlock(&vmstop_lock);
-    return *r < RUN_STATE_MAX;
+    return *r < RUN_STATE__MAX;
 }
 
 void qemu_system_vmstop_request_prepare(void)
@@ -728,7 +750,7 @@ void vm_start(void)
     RunState requested;
 
     qemu_vmstop_requested(&requested);
-    if (runstate_is_running() && requested == RUN_STATE_MAX) {
+    if (runstate_is_running() && requested == RUN_STATE__MAX) {
         return;
     }
 
@@ -825,8 +847,9 @@ static void configure_rtc_date_offset(const char *startdate, int legacy)
         rtc_start_date = mktimegm(&tm);
         if (rtc_start_date == -1) {
         date_fail:
-            fprintf(stderr, "Invalid date format. Valid formats are:\n"
-                            "'2006-06-17T16:01:21' or '2006-06-17'\n");
+            error_report("invalid date format");
+            error_printf("valid formats: "
+                         "'2006-06-17T16:01:21' or '2006-06-17'\n");
             exit(1);
         }
         rtc_date_offset = qemu_time() - rtc_start_date;
@@ -842,7 +865,11 @@ static void configure_rtc(QemuOpts *opts)
         if (!strcmp(value, "utc")) {
             rtc_utc = 1;
         } else if (!strcmp(value, "localtime")) {
+            Error *blocker = NULL;
             rtc_utc = 0;
+            error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED,
+                      "-rtc base=localtime");
+            replay_add_blocker(blocker);
         } else {
             configure_rtc_date_offset(value, 0);
         }
@@ -856,7 +883,7 @@ static void configure_rtc(QemuOpts *opts)
         } else if (!strcmp(value, "vm")) {
             rtc_clock = QEMU_CLOCK_VIRTUAL;
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             exit(1);
         }
     }
@@ -876,7 +903,7 @@ static void configure_rtc(QemuOpts *opts)
         } else if (!strcmp(value, "none")) {
             /* discard is default */
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             exit(1);
         }
     }
@@ -902,7 +929,7 @@ static int bt_hci_parse(const char *str)
     bdaddr_t bdaddr;
 
     if (nb_hcis >= MAX_NICS) {
-        fprintf(stderr, "qemu: Too many bluetooth HCIs (max %i).\n", MAX_NICS);
+        error_report("too many bluetooth HCIs (max %i)", MAX_NICS);
         return -1;
     }
 
@@ -928,8 +955,8 @@ static void bt_vhci_add(int vlan_id)
     struct bt_scatternet_s *vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a VHCI to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a VHCI to an empty scatternet %i",
+                     vlan_id);
 
     bt_vhci_init(bt_new_hci(vlan));
 }
@@ -947,7 +974,7 @@ static struct bt_device_s *bt_device_add(const char *opt)
     if (endp) {
         vlan_id = strtol(endp + 6, &endp, 0);
         if (*endp) {
-            fprintf(stderr, "qemu: unrecognised bluetooth vlan Id\n");
+            error_report("unrecognised bluetooth vlan Id");
             return 0;
         }
     }
@@ -955,13 +982,13 @@ static struct bt_device_s *bt_device_add(const char *opt)
     vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a slave device to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a slave device to an empty scatternet %i",
+                     vlan_id);
 
     if (!strcmp(devname, "keyboard"))
         return bt_keyboard_init(vlan);
 
-    fprintf(stderr, "qemu: unsupported bluetooth device `%s'\n", devname);
+    error_report("unsupported bluetooth device '%s'", devname);
     return 0;
 }
 
@@ -984,11 +1011,11 @@ static int bt_parse(const char *opt)
                 if (strstart(endp, ",vlan=", &p)) {
                     vlan = strtol(p, (char **) &endp, 0);
                     if (*endp) {
-                        fprintf(stderr, "qemu: bad scatternet '%s'\n", p);
+                        error_report("bad scatternet '%s'", p);
                         return 1;
                     }
                 } else {
-                    fprintf(stderr, "qemu: bad parameter '%s'\n", endp + 1);
+                    error_report("bad parameter '%s'", endp + 1);
                     return 1;
                 }
             } else
@@ -1000,7 +1027,7 @@ static int bt_parse(const char *opt)
     } else if (strstart(opt, "device:", &endp))
         return !bt_device_add(endp);
 
-    fprintf(stderr, "qemu: bad bluetooth parameter '%s'\n", opt);
+    error_report("bad bluetooth parameter '%s'", opt);
     return 1;
 }
 
@@ -1015,8 +1042,7 @@ static int parse_sandbox(void *opaque, QemuOpts *opts, Error **errp)
             return -1;
         }
 #else
-        error_report("sandboxing request but seccomp is not compiled "
-                     "into this build");
+        error_report("seccomp support is disabled");
         return -1;
 #endif
     }
@@ -1097,7 +1123,7 @@ static int parse_add_fd(void *opaque, QemuOpts *opts, Error **errp)
     }
 #endif
     if (dupfd == -1) {
-        error_report("Error duplicating fd: %s", strerror(errno));
+        error_report("error duplicating fd: %s", strerror(errno));
         return -1;
     }
 
@@ -1217,14 +1243,21 @@ static void smp_parse(QemuOpts *opts)
         } else if (threads == 0) {
             threads = cpus / (cores * sockets);
         } else if (sockets * cores * threads < cpus) {
-            fprintf(stderr, "cpu topology: error: "
-                    "sockets (%u) * cores (%u) * threads (%u) < "
-                    "smp_cpus (%u)\n",
-                    sockets, cores, threads, cpus);
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) < "
+                         "smp_cpus (%u)",
+                         sockets, cores, threads, cpus);
             exit(1);
         }
 
-        max_cpus = qemu_opt_get_number(opts, "maxcpus", 0);
+        max_cpus = qemu_opt_get_number(opts, "maxcpus", cpus);
+        if (sockets * cores * threads > max_cpus) {
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) > "
+                         "maxcpus (%u)",
+                         sockets, cores, threads, max_cpus);
+            exit(1);
+        }
 
         smp_cpus = cpus;
         smp_cores = cores > 0 ? cores : 1;
@@ -1237,21 +1270,26 @@ static void smp_parse(QemuOpts *opts)
     }
 
     if (max_cpus > MAX_CPUMASK_BITS) {
-        fprintf(stderr, "Unsupported number of maxcpus\n");
+        error_report("unsupported number of maxcpus");
         exit(1);
     }
     if (max_cpus < smp_cpus) {
-        fprintf(stderr, "maxcpus must be equal to or greater than smp\n");
+        error_report("maxcpus must be equal to or greater than smp");
         exit(1);
     }
 
+    if (smp_cpus > 1 || smp_cores > 1 || smp_threads > 1) {
+        Error *blocker = NULL;
+        error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED, "smp");
+        replay_add_blocker(blocker);
+    }
 }
 
 static void realtime_init(void)
 {
     if (enable_mlock) {
         if (os_mlock() < 0) {
-            fprintf(stderr, "qemu: locking memory failed\n");
+            error_report("locking memory failed");
             exit(1);
         }
     }
@@ -1338,6 +1376,13 @@ static inline void semihosting_arg_fallback(const char *file, const char *cmd)
     }
 }
 
+/* Now we still need this for compatibility with XEN. */
+bool has_igd_gfx_passthru;
+static void igd_gfx_passthru(void)
+{
+    has_igd_gfx_passthru = current_machine->igd_gfx_passthru;
+}
+
 /***********************************************************/
 /* USB devices */
 
@@ -1398,7 +1443,7 @@ static int usb_parse(const char *cmdline)
     int r;
     r = usb_device_add(cmdline);
     if (r < 0) {
-        fprintf(stderr, "qemu: could not add USB device '%s'\n", cmdline);
+        error_report("could not add USB device '%s'", cmdline);
     }
     return r;
 }
@@ -1424,43 +1469,6 @@ void hmp_usb_del(Monitor *mon, const QDict *qdict)
 
 MachineState *current_machine;
 
-/*
- * Transitional class registration/init used for converting from
- * legacy QEMUMachine to MachineClass.
- */
-static void qemu_machine_class_init(ObjectClass *oc, void *data)
-{
-    MachineClass *mc = MACHINE_CLASS(oc);
-    QEMUMachine *qm = data;
-    mc->name = qm->name;
-    mc->desc = qm->desc;
-    mc->init = qm->init;
-    mc->kvm_type = qm->kvm_type;
-    mc->block_default_type = qm->block_default_type;
-    mc->max_cpus = qm->max_cpus;
-    mc->no_sdcard = qm->no_sdcard;
-    mc->has_dynamic_sysbus = qm->has_dynamic_sysbus;
-    mc->is_default = qm->is_default;
-    mc->default_machine_opts = qm->default_machine_opts;
-    mc->default_boot_order = qm->default_boot_order;
-}
-
-int qemu_register_machine(QEMUMachine *m)
-{
-    char *name = g_strconcat(m->name, TYPE_MACHINE_SUFFIX, NULL);
-    TypeInfo ti = {
-        .name       = name,
-        .parent     = TYPE_MACHINE,
-        .class_init = qemu_machine_class_init,
-        .class_data = (void *)m,
-    };
-
-    type_register(&ti);
-    g_free(name);
-
-    return 0;
-}
-
 static MachineClass *find_machine(const char *name)
 {
     GSList *el, *machines = object_class_get_list(TYPE_MACHINE, false);
@@ -1539,12 +1547,14 @@ MachineInfoList *qmp_query_machines(Error **errp)
 static int machine_help_func(QemuOpts *opts, MachineState *machine)
 {
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
 
     if (!qemu_opt_has_help_opt(opts)) {
         return 0;
     }
 
-    QTAILQ_FOREACH(prop, &OBJECT(machine)->properties, node) {
+    object_property_iter_init(&iter, OBJECT(machine));
+    while ((prop = object_property_iter_next(&iter))) {
         if (!prop->set) {
             continue;
         }
@@ -1645,14 +1655,14 @@ static int qemu_shutdown_requested(void)
 static void qemu_kill_report(void)
 {
     if (!qtest_driver() && shutdown_signal != -1) {
-        fprintf(stderr, "qemu: terminating on signal %d", shutdown_signal);
         if (shutdown_pid == 0) {
             /* This happens for eg ^C at the terminal, so it's worth
              * avoiding printing an odd message in that case.
              */
-            fputc('\n', stderr);
+            error_report("terminating on signal %d", shutdown_signal);
         } else {
-            fprintf(stderr, " from pid " FMT_pid "\n", shutdown_pid);
+            error_report("terminating on signal %d from pid " FMT_pid,
+                         shutdown_signal, shutdown_pid);
         }
         shutdown_signal = -1;
     }
@@ -1661,15 +1671,21 @@ static void qemu_kill_report(void)
 static int qemu_reset_requested(void)
 {
     int r = reset_requested;
-    reset_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_RESET_REQUESTED)) {
+        reset_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static int qemu_suspend_requested(void)
 {
     int r = suspend_requested;
-    suspend_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_SUSPEND_REQUESTED)) {
+        suspend_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static WakeupReason qemu_wakeup_requested(void)
@@ -1729,6 +1745,8 @@ void qemu_system_reset(bool report)
 
     mc = current_machine ? MACHINE_GET_CLASS(current_machine) : NULL;
 
+    cpu_synchronize_all_states();
+
     if (mc && mc->reset) {
         mc->reset();
     } else {
@@ -1740,6 +1758,15 @@ void qemu_system_reset(bool report)
     cpu_synchronize_all_post_reset();
 }
 
+void qemu_system_guest_panicked(void)
+{
+    if (current_cpu) {
+        current_cpu->crash_occurred = true;
+    }
+    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
+    vm_stop(RUN_STATE_GUEST_PANICKED);
+}
+
 void qemu_system_reset_request(void)
 {
     if (no_reboot) {
@@ -1808,12 +1835,18 @@ void qemu_system_killed(int signal, pid_t pid)
     shutdown_signal = signal;
     shutdown_pid = pid;
     no_shutdown = 0;
-    qemu_system_shutdown_request();
+
+    /* Cannot call qemu_system_shutdown_request directly because
+     * we are in a signal handler.
+     */
+    shutdown_requested = 1;
+    qemu_notify_event();
 }
 
 void qemu_system_shutdown_request(void)
 {
     trace_qemu_system_shutdown_request();
+    replay_shutdown_request();
     shutdown_requested = 1;
     qemu_notify_event();
 }
@@ -1862,16 +1895,15 @@ static bool main_loop_should_exit(void)
     }
     if (qemu_reset_requested()) {
         pause_all_vcpus();
-        cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_REPORT);
         resume_all_vcpus();
-        if (runstate_needs_reset()) {
-            runstate_set(RUN_STATE_PAUSED);
+        if (!runstate_check(RUN_STATE_RUNNING) &&
+                !runstate_check(RUN_STATE_INMIGRATE)) {
+            runstate_set(RUN_STATE_PRELAUNCH);
         }
     }
     if (qemu_wakeup_requested()) {
         pause_all_vcpus();
-        cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         notifier_list_notify(&wakeup_notifiers, &wakeup_reason);
         wakeup_reason = QEMU_WAKEUP_REASON_NONE;
@@ -1992,28 +2024,28 @@ static void select_vgahw (const char *p)
         if (vga_available()) {
             vga_interface_type = VGA_STD;
         } else {
-            fprintf(stderr, "Error: standard VGA not available\n");
+            error_report("standard VGA not available");
             exit(0);
         }
     } else if (strstart(p, "cirrus", &opts)) {
         if (cirrus_vga_available()) {
             vga_interface_type = VGA_CIRRUS;
         } else {
-            fprintf(stderr, "Error: Cirrus VGA not available\n");
+            error_report("Cirrus VGA not available");
             exit(0);
         }
     } else if (strstart(p, "vmware", &opts)) {
         if (vmware_vga_available()) {
             vga_interface_type = VGA_VMWARE;
         } else {
-            fprintf(stderr, "Error: VMWare SVGA not available\n");
+            error_report("VMWare SVGA not available");
             exit(0);
         }
     } else if (strstart(p, "virtio", &opts)) {
         if (virtio_vga_available()) {
             vga_interface_type = VGA_VIRTIO;
         } else {
-            fprintf(stderr, "Error: Virtio VGA not available\n");
+            error_report("Virtio VGA not available");
             exit(0);
         }
     } else if (strstart(p, "xenfb", &opts)) {
@@ -2022,26 +2054,26 @@ static void select_vgahw (const char *p)
         if (qxl_vga_available()) {
             vga_interface_type = VGA_QXL;
         } else {
-            fprintf(stderr, "Error: QXL VGA not available\n");
+            error_report("QXL VGA not available");
             exit(0);
         }
     } else if (strstart(p, "tcx", &opts)) {
         if (tcx_vga_available()) {
             vga_interface_type = VGA_TCX;
         } else {
-            fprintf(stderr, "Error: TCX framebuffer not available\n");
+            error_report("TCX framebuffer not available");
             exit(0);
         }
     } else if (strstart(p, "cg3", &opts)) {
         if (cg3_vga_available()) {
             vga_interface_type = VGA_CG3;
         } else {
-            fprintf(stderr, "Error: CG3 framebuffer not available\n");
+            error_report("CG3 framebuffer not available");
             exit(0);
         }
     } else if (!strstart(p, "none", &opts)) {
     invalid_vga:
-        fprintf(stderr, "Unknown vga type: %s\n", p);
+        error_report("unknown vga type: %s", p);
         exit(1);
     }
     while (*opts) {
@@ -2117,7 +2149,7 @@ static DisplayType select_display(const char *p)
                 }
             } else {
             invalid_sdl_args:
-                error_report("Invalid SDL option string");
+                error_report("invalid SDL option string");
                 exit(1);
             }
             opts = nextopt;
@@ -2146,7 +2178,7 @@ static DisplayType select_display(const char *p)
 #ifdef CONFIG_CURSES
         display = DT_CURSES;
 #else
-        error_report("Curses support is disabled");
+        error_report("curses support is disabled");
         exit(1);
 #endif
     } else if (strstart(p, "gtk", &opts)) {
@@ -2175,7 +2207,7 @@ static DisplayType select_display(const char *p)
                 }
             } else {
             invalid_gtk_args:
-                error_report("Invalid GTK option string");
+                error_report("invalid GTK option string");
                 exit(1);
             }
             opts = nextopt;
@@ -2187,7 +2219,7 @@ static DisplayType select_display(const char *p)
     } else if (strstart(p, "none", &opts)) {
         display = DT_NONE;
     } else {
-        error_report("Unknown display type");
+        error_report("unknown display type");
         exit(1);
     }
 
@@ -2255,20 +2287,33 @@ char *qemu_find_file(int type, const char *name)
     return NULL;
 }
 
+static inline bool nonempty_str(const char *str)
+{
+    return str && *str;
+}
+
 static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
 {
     gchar *buf;
     size_t size;
-    const char *name, *file;
+    const char *name, *file, *str;
+    FWCfgState *fw_cfg = (FWCfgState *) opaque;
 
-    if (opaque == NULL) {
+    if (fw_cfg == NULL) {
         error_report("fw_cfg device not available");
         return -1;
     }
     name = qemu_opt_get(opts, "name");
     file = qemu_opt_get(opts, "file");
-    if (name == NULL || *name == '\0' || file == NULL || *file == '\0') {
-        error_report("invalid argument value");
+    str = qemu_opt_get(opts, "string");
+
+    /* we need name and either a file or the content string */
+    if (!(nonempty_str(name) && (nonempty_str(file) || nonempty_str(str)))) {
+        error_report("invalid argument(s)");
+        return -1;
+    }
+    if (nonempty_str(file) && nonempty_str(str)) {
+        error_report("file and string are mutually exclusive");
         return -1;
     }
     if (strlen(name) > FW_CFG_MAX_FILE_PATH - 1) {
@@ -2276,14 +2321,22 @@ static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
         return -1;
     }
     if (strncmp(name, "opt/", 4) != 0) {
-        error_report("WARNING: externally provided fw_cfg item names "
-                     "should be prefixed with \"opt/\"!");
+        error_report("warning: externally provided fw_cfg item names "
+                     "should be prefixed with \"opt/\"");
     }
-    if (!g_file_get_contents(file, &buf, &size, NULL)) {
-        error_report("can't load %s", file);
-        return -1;
+    if (nonempty_str(str)) {
+        size = strlen(str); /* NUL terminator NOT included in fw_cfg blob */
+        buf = g_memdup(str, size);
+    } else {
+        if (!g_file_get_contents(file, &buf, &size, NULL)) {
+            error_report("can't load %s", file);
+            return -1;
+        }
     }
-    fw_cfg_add_file((FWCfgState *)opaque, name, buf, size);
+    /* For legacy, keep user files in a specific global order. */
+    fw_cfg_set_order_override(fw_cfg, FW_CFG_ORDER_OVERRIDE_USER);
+    fw_cfg_add_file(fw_cfg, name, buf, size);
+    fw_cfg_reset_order_override(fw_cfg);
     return 0;
 }
 
@@ -2344,7 +2397,7 @@ static int mon_init_func(void *opaque, QemuOpts *opts, Error **errp)
     } else if (strcmp(mode, "control") == 0) {
         flags = MONITOR_USE_CONTROL;
     } else {
-        fprintf(stderr, "unknown monitor mode \"%s\"\n", mode);
+        error_report("unknown monitor mode \"%s\"", mode);
         exit(1);
     }
 
@@ -2357,7 +2410,7 @@ static int mon_init_func(void *opaque, QemuOpts *opts, Error **errp)
     chardev = qemu_opt_get(opts, "chardev");
     chr = qemu_chr_find(chardev);
     if (chr == NULL) {
-        fprintf(stderr, "chardev \"%s\" not found\n", chardev);
+        error_report("chardev \"%s\" not found", chardev);
         exit(1);
     }
 
@@ -2385,7 +2438,7 @@ static void monitor_parse(const char *optarg, const char *mode, bool pretty)
         }
         opts = qemu_chr_parse_compat(label, optarg);
         if (!opts) {
-            fprintf(stderr, "parse error: %s\n", optarg);
+            error_report("parse error: %s", optarg);
             exit(1);
         }
     }
@@ -2459,14 +2512,14 @@ static int serial_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_SERIAL_PORTS) {
-        fprintf(stderr, "qemu: too many serial ports\n");
+        error_report("too many serial ports");
         exit(1);
     }
     snprintf(label, sizeof(label), "serial%d", index);
     serial_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!serial_hds[index]) {
-        fprintf(stderr, "qemu: could not connect serial device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect serial device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2481,14 +2534,14 @@ static int parallel_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_PARALLEL_PORTS) {
-        fprintf(stderr, "qemu: too many parallel ports\n");
+        error_report("too many parallel ports");
         exit(1);
     }
     snprintf(label, sizeof(label), "parallel%d", index);
     parallel_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!parallel_hds[index]) {
-        fprintf(stderr, "qemu: could not connect parallel device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect parallel device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2505,16 +2558,12 @@ static int virtcon_parse(const char *devname)
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_VIRTIO_CONSOLES) {
-        fprintf(stderr, "qemu: too many virtio consoles\n");
+        error_report("too many virtio consoles");
         exit(1);
     }
 
     bus_opts = qemu_opts_create(device, NULL, 0, &error_abort);
-    if (arch_type == QEMU_ARCH_S390X) {
-        qemu_opt_set(bus_opts, "driver", "virtio-serial-s390", &error_abort);
-    } else {
-        qemu_opt_set(bus_opts, "driver", "virtio-serial-pci", &error_abort);
-    }
+    qemu_opt_set(bus_opts, "driver", "virtio-serial", &error_abort);
 
     dev_opts = qemu_opts_create(device, NULL, 0, &error_abort);
     qemu_opt_set(dev_opts, "driver", "virtconsole", &error_abort);
@@ -2522,8 +2571,8 @@ static int virtcon_parse(const char *devname)
     snprintf(label, sizeof(label), "virtcon%d", index);
     virtcon_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!virtcon_hds[index]) {
-        fprintf(stderr, "qemu: could not connect virtio console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect virtio console"
+                     " to character backend '%s'", devname);
         return -1;
     }
     qemu_opt_set(dev_opts, "chardev", label, &error_abort);
@@ -2543,7 +2592,7 @@ static int sclp_parse(const char *devname)
         return 0;
     }
     if (index == MAX_SCLP_CONSOLES) {
-        fprintf(stderr, "qemu: too many sclp consoles\n");
+        error_report("too many sclp consoles");
         exit(1);
     }
 
@@ -2555,8 +2604,8 @@ static int sclp_parse(const char *devname)
     snprintf(label, sizeof(label), "sclpcon%d", index);
     sclp_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!sclp_hds[index]) {
-        fprintf(stderr, "qemu: could not connect sclp console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect sclp console"
+                     " to character backend '%s'", devname);
         return -1;
     }
     qemu_opt_set(dev_opts, "chardev", label, &error_abort);
@@ -2574,7 +2623,7 @@ static int debugcon_parse(const char *devname)
     }
     opts = qemu_opts_create(qemu_find_opts("device"), "debugcon", 1, NULL);
     if (!opts) {
-        fprintf(stderr, "qemu: already have a debugcon device\n");
+        error_report("already have a debugcon device");
         exit(1);
     }
     qemu_opt_set(opts, "driver", "isa-debugcon", &error_abort);
@@ -2629,8 +2678,8 @@ static gint machine_class_cmp(gconstpointer a, gconstpointer b)
         return mc;
     }
     if (name && !is_help_option(name)) {
-        error_report("Unsupported machine type");
-        error_printf("Use -machine help to list supported machines!\n");
+        error_report("unsupported machine type");
+        error_printf("Use -machine help to list supported machines\n");
     } else {
         printf("Supported machines are:\n");
         machines = g_slist_sort(machines, machine_class_cmp);
@@ -2719,24 +2768,31 @@ static const QEMUOption *lookup_opt(int argc, char **argv,
     return popt;
 }
 
-static gpointer malloc_and_trace(gsize n_bytes)
+static MachineClass *select_machine(void)
 {
-    void *ptr = malloc(n_bytes);
-    trace_g_malloc(n_bytes, ptr);
-    return ptr;
-}
+    MachineClass *machine_class = find_default_machine();
+    const char *optarg;
+    QemuOpts *opts;
+    Location loc;
 
-static gpointer realloc_and_trace(gpointer mem, gsize n_bytes)
-{
-    void *ptr = realloc(mem, n_bytes);
-    trace_g_realloc(mem, n_bytes, ptr);
-    return ptr;
-}
+    loc_push_none(&loc);
 
-static void free_and_trace(gpointer mem)
-{
-    trace_g_free(mem);
-    free(mem);
+    opts = qemu_get_machine_opts();
+    qemu_opts_loc_restore(opts);
+
+    optarg = qemu_opt_get(opts, "type");
+    if (optarg) {
+        machine_class = machine_parse(optarg);
+    }
+
+    if (!machine_class) {
+        error_report("No machine specified, and there is no default");
+        error_printf("Use -machine help to list supported machines\n");
+        exit(1);
+    }
+
+    loc_pop(&loc);
+    return machine_class;
 }
 
 static int machine_set_property(void *opaque,
@@ -2783,13 +2839,25 @@ static bool object_create_initial(const char *type)
     if (g_str_equal(type, "rng-egd")) {
         return false;
     }
+
+    /*
+     * return false for concrete netfilters since
+     * they depend on netdevs already existing
+     */
+    if (g_str_equal(type, "filter-buffer") ||
+        g_str_equal(type, "filter-dump") ||
+        g_str_equal(type, "filter-mirror") ||
+        g_str_equal(type, "filter-redirector")) {
+        return false;
+    }
+
     return true;
 }
 
 
 /*
  * The remainder of object creation happens after the
- * creation of chardev, fsdev and device data types.
+ * creation of chardev, fsdev, net clients and device data types.
  */
 static bool object_create_delayed(const char *type)
 {
@@ -2797,62 +2865,6 @@ static bool object_create_delayed(const char *type)
 }
 
 
-static int object_create(void *opaque, QemuOpts *opts, Error **errp)
-{
-    Error *err = NULL;
-    char *type = NULL;
-    char *id = NULL;
-    void *dummy = NULL;
-    OptsVisitor *ov;
-    QDict *pdict;
-    bool (*type_predicate)(const char *) = opaque;
-
-    ov = opts_visitor_new(opts);
-    pdict = qemu_opts_to_qdict(opts, NULL);
-
-    visit_start_struct(opts_get_visitor(ov), &dummy, NULL, NULL, 0, &err);
-    if (err) {
-        goto out;
-    }
-
-    qdict_del(pdict, "qom-type");
-    visit_type_str(opts_get_visitor(ov), &type, "qom-type", &err);
-    if (err) {
-        goto out;
-    }
-    if (!type_predicate(type)) {
-        goto out;
-    }
-
-    qdict_del(pdict, "id");
-    visit_type_str(opts_get_visitor(ov), &id, "id", &err);
-    if (err) {
-        goto out;
-    }
-
-    object_add(type, id, pdict, opts_get_visitor(ov), &err);
-    if (err) {
-        goto out;
-    }
-    visit_end_struct(opts_get_visitor(ov), &err);
-    if (err) {
-        qmp_object_del(id, NULL);
-    }
-
-out:
-    opts_visitor_cleanup(ov);
-
-    QDECREF(pdict);
-    g_free(id);
-    g_free(type);
-    g_free(dummy);
-    if (err) {
-        error_report_err(err);
-        return -1;
-    }
-    return 0;
-}
-
 static void set_memory_options(uint64_t *ram_slots, ram_addr_t *maxram_size,
                                MachineClass *mc)
 {
@@ -2861,6 +2873,10 @@ static void set_memory_options(uint64_t *ram_slots, ram_addr_t *maxram_size,
     const char *maxmem_str, *slots_str;
     const ram_addr_t default_ram_size = mc->default_ram_size;
     QemuOpts *opts = qemu_find_opts_singleton("memory");
+    Location loc;
+
+    loc_push_none(&loc);
+    qemu_opts_loc_restore(opts);
 
     sz = 0;
     mem_str = qemu_opt_get(opts, "size");
@@ -2935,6 +2951,8 @@ static void set_memory_options(uint64_t *ram_slots, ram_addr_t *maxram_size,
                 "'%s' option", slots_str ? "maxmem" : "slots");
         exit(EXIT_FAILURE);
     }
+
+    loc_pop(&loc);
 }
 
 int main(int argc, char **argv, char **envp)
@@ -2966,13 +2984,7 @@ int main(int argc, char **argv, char **envp)
     bool userconfig = true;
     const char *log_mask = NULL;
     const char *log_file = NULL;
-    GMemVTable mem_trace = {
-        .malloc = malloc_and_trace,
-        .realloc = realloc_and_trace,
-        .free = free_and_trace,
-    };
-    const char *trace_events = NULL;
-    const char *trace_file = NULL;
+    char *trace_file = NULL;
     ram_addr_t maxram_size;
     uint64_t ram_slots = 0;
     FILE *vmstate_dump_file = NULL;
@@ -2986,8 +2998,6 @@ int main(int argc, char **argv, char **envp)
     error_set_progname(argv[0]);
     qemu_init_exec_dir(argv[0]);
 
-    g_mem_set_vtable(&mem_trace);
-
     module_call_init(MODULE_INIT_QOM);
 
     qemu_add_opts(&qemu_drive_opts);
@@ -3018,12 +3028,12 @@ int main(int argc, char **argv, char **envp)
     qemu_add_opts(&qemu_icount_opts);
     qemu_add_opts(&qemu_semihosting_config_opts);
     qemu_add_opts(&qemu_fw_cfg_opts);
+    module_call_init(MODULE_INIT_OPTS);
 
     runstate_init();
 
     if (qcrypto_init(&err) < 0) {
-        fprintf(stderr, "Cannot initialize crypto: %s\n",
-                error_get_pretty(err));
+        error_reportf_err(err, "cannot initialize crypto: ");
         exit(1);
     }
     rtc_clock = QEMU_CLOCK_HOST;
@@ -3031,8 +3041,6 @@ int main(int argc, char **argv, char **envp)
     QLIST_INIT (&vm_change_state_head);
     os_setup_early_signal_handling();
 
-    module_call_init(MODULE_INIT_MACHINE);
-    machine_class = find_default_machine();
     cpu_model = NULL;
     snapshot = 0;
     cyls = heads = secs = 0;
@@ -3181,7 +3189,7 @@ int main(int argc, char **argv, char **envp)
                         }
                     } else if (*p != '\0') {
                     chs_fail:
-                        fprintf(stderr, "qemu: invalid physical CHS format\n");
+                        error_report("invalid physical CHS format");
                         exit(1);
                     }
                     if (hda_opts != NULL) {
@@ -3224,7 +3232,7 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_CURSES
                 display_type = DT_CURSES;
 #else
-                fprintf(stderr, "Curses support is disabled\n");
+                error_report("curses support is disabled");
                 exit(1);
 #endif
                 break;
@@ -3235,8 +3243,7 @@ int main(int argc, char **argv, char **envp)
                 graphic_rotate = strtol(optarg, (char **) &optarg, 10);
                 if (graphic_rotate != 0 && graphic_rotate != 90 &&
                     graphic_rotate != 180 && graphic_rotate != 270) {
-                    fprintf(stderr,
-                        "qemu: only 90, 180, 270 deg rotation is available\n");
+                    error_report("only 90, 180, 270 deg rotation is available");
                     exit(1);
                 }
                 break;
@@ -3295,12 +3302,18 @@ int main(int argc, char **argv, char **envp)
 #endif
 #ifdef CONFIG_SLIRP
             case QEMU_OPTION_tftp:
+                error_report("The -tftp option is deprecated. "
+                             "Please use '-netdev user,tftp=...' instead.");
                 legacy_tftp_prefix = optarg;
                 break;
             case QEMU_OPTION_bootp:
+                error_report("The -bootp option is deprecated. "
+                             "Please use '-netdev user,bootfile=...' instead.");
                 legacy_bootp_filename = optarg;
                 break;
             case QEMU_OPTION_redir:
+                error_report("The -redir option is deprecated. "
+                             "Please use '-netdev user,hostfwd=...' instead.");
                 if (net_slirp_redir(optarg) < 0)
                     exit(1);
                 break;
@@ -3348,6 +3361,9 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_D:
                 log_file = optarg;
                 break;
+            case QEMU_OPTION_DFILTER:
+                qemu_set_dfilter_ranges(optarg);
+                break;
             case QEMU_OPTION_s:
                 add_device_config(DEV_GDB, "tcp::" DEFAULT_GDBSTUB_PORT);
                 break;
@@ -3387,7 +3403,7 @@ int main(int argc, char **argv, char **envp)
                     w = strtol(p, (char **)&p, 10);
                     if (w <= 0) {
                     graphic_error:
-                        fprintf(stderr, "qemu: invalid resolution or depth\n");
+                        error_report("invalid resolution or depth");
                         exit(1);
                     }
                     if (*p != 'x')
@@ -3453,7 +3469,7 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_fsdev:
                 olist = qemu_find_opts("fsdev");
                 if (!olist) {
-                    fprintf(stderr, "fsdev is not supported by this qemu build.\n");
+                    error_report("fsdev support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, true);
@@ -3468,7 +3484,7 @@ int main(int argc, char **argv, char **envp)
 
                 olist = qemu_find_opts("virtfs");
                 if (!olist) {
-                    fprintf(stderr, "virtfs is not supported by this qemu build.\n");
+                    error_report("virtfs support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, true);
@@ -3478,15 +3494,15 @@ int main(int argc, char **argv, char **envp)
 
                 if (qemu_opt_get(opts, "fsdriver") == NULL ||
                     qemu_opt_get(opts, "mount_tag") == NULL) {
-                    fprintf(stderr, "Usage: -virtfs fsdriver,mount_tag=tag.\n");
+                    error_report("Usage: -virtfs fsdriver,mount_tag=tag");
                     exit(1);
                 }
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"),
                                          qemu_opt_get(opts, "mount_tag"),
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate fsdev id: %s\n",
-                            qemu_opt_get(opts, "mount_tag"));
+                    error_report("duplicate fsdev id: %s",
+                                 qemu_opt_get(opts, "mount_tag"));
                     exit(1);
                 }
 
@@ -3495,8 +3511,8 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_SYNC_FILE_RANGE
                     qemu_opt_set(fsdev, "writeout", writeout, &error_abort);
 #else
-                    fprintf(stderr, "writeout=immediate not supported on "
-                            "this platform\n");
+                    error_report("writeout=immediate not supported "
+                                 "on this platform");
                     exit(1);
 #endif
                 }
@@ -3535,7 +3551,7 @@ int main(int argc, char **argv, char **envp)
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"), "v_synth",
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate option: %s\n", "virtfs_synth");
+                    error_report("duplicate option: %s", "virtfs_synth");
                     exit(1);
                 }
                 qemu_opt_set(fsdev, "fsdriver", "synth", &error_abort);
@@ -3556,15 +3572,14 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_watchdog:
                 if (watchdog) {
-                    fprintf(stderr,
-                            "qemu: only one watchdog option may be given\n");
+                    error_report("only one watchdog option may be given");
                     return 1;
                 }
                 watchdog = optarg;
                 break;
             case QEMU_OPTION_watchdog_action:
                 if (select_watchdog_action(optarg) == -1) {
-                    fprintf(stderr, "Unknown -watchdog-action parameter\n");
+                    error_report("unknown -watchdog-action parameter");
                     exit(1);
                 }
                 break;
@@ -3608,7 +3623,7 @@ int main(int argc, char **argv, char **envp)
                 display_type = DT_SDL;
                 break;
 #else
-                fprintf(stderr, "SDL support is disabled\n");
+                error_report("SDL support is disabled");
                 exit(1);
 #endif
             case QEMU_OPTION_pidfile:
@@ -3670,8 +3685,7 @@ int main(int argc, char **argv, char **envp)
                 qemu_opts_parse_noisily(olist, "accel=tcg", false);
                 break;
             case QEMU_OPTION_no_kvm_pit: {
-                fprintf(stderr, "Warning: KVM PIT can no longer be disabled "
-                                "separately.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             }
             case QEMU_OPTION_no_kvm_pit_reinjection: {
@@ -3684,8 +3698,8 @@ int main(int argc, char **argv, char **envp)
                     { /* end of list */ }
                 };
 
-                fprintf(stderr, "Warning: option deprecated, use "
-                        "lost_tick_policy property of kvm-pit instead.\n");
+                error_report("warning: deprecated, replaced by "
+                             "-global kvm-pit.lost_tick_policy=discard");
                 qdev_prop_register_global_list(kvm_pit_lost_tick_policy);
                 break;
             }
@@ -3720,7 +3734,7 @@ int main(int argc, char **argv, char **envp)
                     exit(1);
                 }
 #else
-                fprintf(stderr, "VNC support is disabled\n");
+                error_report("VNC support is disabled");
                 exit(1);
 #endif
                 break;
@@ -3733,7 +3747,7 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_balloon:
                 if (balloon_parse(optarg) < 0) {
-                    fprintf(stderr, "Unknown -balloon argument %s\n", optarg);
+                    error_report("unknown -balloon argument %s", optarg);
                     exit(1);
                 }
                 break;
@@ -3748,15 +3762,14 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_uuid:
                 if(qemu_uuid_parse(optarg, qemu_uuid) < 0) {
-                    fprintf(stderr, "Fail to parse UUID string."
-                            " Wrong format.\n");
+                    error_report("failed to parse UUID string: wrong format");
                     exit(1);
                 }
                 qemu_uuid_set = true;
                 break;
             case QEMU_OPTION_option_rom:
                 if (nb_option_roms >= MAX_OPTION_ROMS) {
-                    fprintf(stderr, "Too many option ROMs\n");
+                    error_report("too many option ROMs");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(qemu_find_opts("option-rom"),
@@ -3768,7 +3781,7 @@ int main(int argc, char **argv, char **envp)
                 option_rom[nb_option_roms].bootindex =
                     qemu_opt_get_number(opts, "bootindex", -1);
                 if (!option_rom[nb_option_roms].name) {
-                    fprintf(stderr, "Option ROM file is not specified\n");
+                    error_report("Option ROM file is not specified");
                     exit(1);
                 }
                 nb_option_roms++;
@@ -3793,9 +3806,8 @@ int main(int argc, char **argv, char **envp)
                         } else  if (strcmp("auto", target) == 0) {
                             semihosting.target = SEMIHOSTING_TARGET_AUTO;
                         } else {
-                            fprintf(stderr, "Unsupported semihosting-config"
-                                    " %s\n",
-                                optarg);
+                            error_report("unsupported semihosting-config %s",
+                                         optarg);
                             exit(1);
                         }
                     } else {
@@ -3805,14 +3817,12 @@ int main(int argc, char **argv, char **envp)
                     qemu_opt_foreach(opts, add_semihosting_arg,
                                      &semihosting, NULL);
                 } else {
-                    fprintf(stderr, "Unsupported semihosting-config %s\n",
-                            optarg);
+                    error_report("unsupported semihosting-config %s", optarg);
                     exit(1);
                 }
                 break;
             case QEMU_OPTION_tdf:
-                fprintf(stderr, "Warning: user space PIT time drift fix "
-                                "is no longer supported.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             case QEMU_OPTION_name:
                 opts = qemu_opts_parse_noisily(qemu_find_opts("name"),
@@ -3823,7 +3833,7 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_prom_env:
                 if (nb_prom_envs >= MAX_PROM_ENVS) {
-                    fprintf(stderr, "Too many prom variables\n");
+                    error_report("too many prom variables");
                     exit(1);
                 }
                 prom_envs[nb_prom_envs] = optarg;
@@ -3894,20 +3904,27 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_trace:
             {
                 opts = qemu_opts_parse_noisily(qemu_find_opts("trace"),
-                                               optarg, false);
+                                               optarg, true);
                 if (!opts) {
                     exit(1);
                 }
-                trace_events = qemu_opt_get(opts, "events");
-                trace_file = qemu_opt_get(opts, "file");
+                if (qemu_opt_get(opts, "enable")) {
+                    trace_enable_events(qemu_opt_get(opts, "enable"));
+                }
+                trace_init_events(qemu_opt_get(opts, "events"));
+                if (trace_file) {
+                    g_free(trace_file);
+                }
+                trace_file = g_strdup(qemu_opt_get(opts, "file"));
+                qemu_opts_del(opts);
                 break;
             }
             case QEMU_OPTION_readconfig:
                 {
                     int ret = qemu_read_config_file(optarg);
                     if (ret < 0) {
-                        fprintf(stderr, "read config %s: %s\n", optarg,
-                            strerror(-ret));
+                        error_report("read config %s: %s", optarg,
+                                     strerror(-ret));
                         exit(1);
                     }
                     break;
@@ -3915,7 +3932,7 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_spice:
                 olist = qemu_find_opts("spice");
                 if (!olist) {
-                    fprintf(stderr, "spice is not supported by this qemu build.\n");
+                    error_report("spice support is disabled");
                     exit(1);
                 }
                 opts = qemu_opts_parse_noisily(olist, optarg, false);
@@ -3932,7 +3949,8 @@ int main(int argc, char **argv, char **envp)
                     } else {
                         fp = fopen(optarg, "w");
                         if (fp == NULL) {
-                            fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                            error_report("open %s: %s", optarg,
+                                         strerror(errno));
                             exit(1);
                         }
                     }
@@ -3993,13 +4011,13 @@ int main(int argc, char **argv, char **envp)
                 break;
             case QEMU_OPTION_dump_vmstate:
                 if (vmstate_dump_file) {
-                    fprintf(stderr, "qemu: only one '-dump-vmstate' "
-                            "option may be given\n");
+                    error_report("only one '-dump-vmstate' "
+                                 "option may be given");
                     exit(1);
                 }
                 vmstate_dump_file = fopen(optarg, "w");
                 if (vmstate_dump_file == NULL) {
-                    fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                    error_report("open %s: %s", optarg, strerror(errno));
                     exit(1);
                 }
                 break;
@@ -4008,23 +4026,18 @@ int main(int argc, char **argv, char **envp)
             }
         }
     }
+    /*
+     * Clear error location left behind by the loop.
+     * Best done right after the loop.  Do not insert code here!
+     */
+    loc_set_none();
 
-    opts = qemu_get_machine_opts();
-    optarg = qemu_opt_get(opts, "type");
-    if (optarg) {
-        machine_class = machine_parse(optarg);
-    }
+    replay_configure(icount_opts);
 
-    if (machine_class == NULL) {
-        fprintf(stderr, "No machine specified, and there is no default.\n"
-                "Use -machine help to list supported machines!\n");
-        exit(1);
-    }
+    machine_class = select_machine();
 
     set_memory_options(&ram_slots, &maxram_size, machine_class);
 
-    loc_set_none();
-
     os_daemonize();
 
     if (qemu_init_main_loop(&main_loop_err)) {
@@ -4064,7 +4077,7 @@ int main(int argc, char **argv, char **envp)
     cpu_exec_init_all();
 
     if (machine_class->hw_version) {
-        qemu_set_version(machine_class->hw_version);
+        qemu_set_hw_version(machine_class->hw_version);
     }
 
     /* Init CPU def lists, based on config
@@ -4079,6 +4092,11 @@ int main(int argc, char **argv, char **envp)
         exit(0);
     }
 
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+
     /* Open the logfile at this point and set the log mask if necessary.
      */
     if (log_file) {
@@ -4093,12 +4111,8 @@ int main(int argc, char **argv, char **envp)
             exit(1);
         }
         qemu_set_log(mask);
-    }
-
-    if (!is_daemonized()) {
-        if (!trace_init_backends(trace_events, trace_file)) {
-            exit(1);
-        }
+    } else {
+        qemu_set_log(0);
     }
 
     /* If no data_dir is specified then try to find it relative to the
@@ -4118,9 +4132,9 @@ int main(int argc, char **argv, char **envp)
 
     machine_class->max_cpus = machine_class->max_cpus ?: 1; /* Default to UP */
     if (max_cpus > machine_class->max_cpus) {
-        fprintf(stderr, "Number of SMP cpus requested (%d), exceeds max cpus "
-                "supported by machine `%s' (%d)\n", max_cpus,
-                machine_class->name, machine_class->max_cpus);
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine '%s' (%d)", max_cpus,
+                     machine_class->name, machine_class->max_cpus);
         exit(1);
     }
 
@@ -4181,12 +4195,12 @@ int main(int argc, char **argv, char **envp)
         if (display_type == DT_NOGRAPHIC
             && (default_parallel || default_serial
                 || default_monitor || default_virtcon)) {
-            fprintf(stderr, "-nographic can not be used with -daemonize\n");
+            error_report("-nographic cannot be used with -daemonize");
             exit(1);
         }
 #ifdef CONFIG_CURSES
         if (display_type == DT_CURSES) {
-            fprintf(stderr, "curses display can not be used with -daemonize\n");
+            error_report("curses display cannot be used with -daemonize");
             exit(1);
         }
 #endif
@@ -4245,12 +4259,12 @@ int main(int argc, char **argv, char **envp)
     }
 
     if ((no_frame || alt_grab || ctrl_grab) && display_type != DT_SDL) {
-        fprintf(stderr, "-no-frame, -alt-grab and -ctrl-grab are only valid "
-                        "for SDL, ignoring option\n");
+        error_report("-no-frame, -alt-grab and -ctrl-grab are only valid "
+                     "for SDL, ignoring option");
     }
     if (no_quit && (display_type != DT_GTK && display_type != DT_SDL)) {
-        fprintf(stderr, "-no-quit is only valid for GTK and SDL, "
-                        "ignoring option\n");
+        error_report("-no-quit is only valid for GTK and SDL, "
+                     "ignoring option");
     }
 
 #if defined(CONFIG_GTK)
@@ -4265,17 +4279,18 @@ int main(int argc, char **argv, char **envp)
 #endif
     if (request_opengl == 1 && display_opengl == 0) {
 #if defined(CONFIG_OPENGL)
-        fprintf(stderr, "OpenGL is not supported by the display.\n");
+        error_report("OpenGL is not supported by the display");
 #else
-        fprintf(stderr, "QEMU was built without opengl support.\n");
+        error_report("OpenGL support is disabled");
 #endif
         exit(1);
     }
 
+    page_size_init();
     socket_init();
 
     if (qemu_opts_foreach(qemu_find_opts("object"),
-                          object_create,
+                          user_creatable_add_opts_foreach,
                           object_create_initial, NULL)) {
         exit(1);
     }
@@ -4293,7 +4308,7 @@ int main(int argc, char **argv, char **envp)
 #endif
 
     if (pid_file && qemu_create_pidfile(pid_file) != 0) {
-        fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
+        error_report("could not acquire pid file: %s", strerror(errno));
         exit(1);
     }
 
@@ -4302,12 +4317,6 @@ int main(int argc, char **argv, char **envp)
         exit(0);
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("object"),
-                          object_create,
-                          object_create_delayed, NULL)) {
-        exit(1);
-    }
-
     machine_opts = qemu_get_machine_opts();
     if (qemu_opt_foreach(machine_opts, machine_set_property, current_machine,
                          NULL)) {
@@ -4318,12 +4327,7 @@ int main(int argc, char **argv, char **envp)
     configure_accelerator(current_machine);
 
     if (qtest_chrdev) {
-        Error *local_err = NULL;
-        qtest_init(qtest_chrdev, qtest_log, &local_err);
-        if (local_err) {
-            error_report_err(local_err);
-            exit(1);
-        }
+        qtest_init(qtest_chrdev, qtest_log, &error_fatal);
     }
 
     machine_opts = qemu_get_machine_opts();
@@ -4334,24 +4338,14 @@ int main(int argc, char **argv, char **envp)
 
     opts = qemu_opts_find(qemu_find_opts("boot-opts"), NULL);
     if (opts) {
-        Error *local_err = NULL;
-
         boot_order = qemu_opt_get(opts, "order");
         if (boot_order) {
-            validate_bootdevices(boot_order, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            validate_bootdevices(boot_order, &error_fatal);
         }
 
         boot_once = qemu_opt_get(opts, "once");
         if (boot_once) {
-            validate_bootdevices(boot_once, &local_err);
-            if (local_err) {
-                error_report_err(local_err);
-                exit(1);
-            }
+            validate_bootdevices(boot_once, &error_fatal);
         }
 
         boot_menu = qemu_opt_get_bool(opts, "menu", boot_menu);
@@ -4370,17 +4364,17 @@ int main(int argc, char **argv, char **envp)
     linux_boot = (kernel_filename != NULL);
 
     if (!linux_boot && *kernel_cmdline != '\0') {
-        fprintf(stderr, "-append only allowed with -kernel option\n");
+        error_report("-append only allowed with -kernel option");
         exit(1);
     }
 
     if (!linux_boot && initrd_filename != NULL) {
-        fprintf(stderr, "-initrd only allowed with -kernel option\n");
+        error_report("-initrd only allowed with -kernel option");
         exit(1);
     }
 
     if (!linux_boot && qemu_opt_get(machine_opts, "dtb")) {
-        fprintf(stderr, "-dtb only allowed with -kernel option\n");
+        error_report("-dtb only allowed with -kernel option");
         exit(1);
     }
 
@@ -4399,7 +4393,7 @@ int main(int argc, char **argv, char **envp)
     cpu_ticks_init();
     if (icount_opts) {
         if (kvm_enabled() || xen_enabled()) {
-            fprintf(stderr, "-icount is not allowed with kvm or xen\n");
+            error_report("-icount is not allowed with kvm or xen");
             exit(1);
         }
         configure_icount(icount_opts, &error_abort);
@@ -4413,6 +4407,12 @@ int main(int argc, char **argv, char **envp)
         exit(1);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          user_creatable_add_opts_foreach,
+                          object_create_delayed, NULL)) {
+        exit(1);
+    }
+
 #ifdef CONFIG_TPM
     if (tpm_init() < 0) {
         exit(1);
@@ -4426,7 +4426,7 @@ int main(int argc, char **argv, char **envp)
     if (!xen_enabled()) {
         /* On 32-bit hosts, QEMU is limited by virtual address space */
         if (ram_size > (2047 << 20) && HOST_LONG_BITS == 32) {
-            fprintf(stderr, "qemu: at most 2047 MB RAM can be simulated\n");
+            error_report("at most 2047 MB RAM can be simulated");
             exit(1);
         }
     }
@@ -4442,9 +4442,10 @@ int main(int argc, char **argv, char **envp)
     }
 
     /* open the virtual block devices */
-    if (snapshot)
-        qemu_opts_foreach(qemu_find_opts("drive"),
-                          drive_enable_snapshot, NULL, NULL);
+    if (snapshot || replay_mode != REPLAY_MODE_NONE) {
+        qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot,
+                          NULL, NULL);
+    }
     if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func,
                           &machine_class->block_default_type, NULL)) {
         exit(1);
@@ -4499,6 +4500,10 @@ int main(int argc, char **argv, char **envp)
     }
     qemu_add_globals();
 
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_INIT);
     qdev_machine_init();
 
     current_machine->ram_size = ram_size;
@@ -4528,11 +4533,16 @@ int main(int argc, char **argv, char **envp)
             exit(1);
     }
 
+    /* Check if IGD GFX passthrough. */
+    igd_gfx_passthru();
+
     /* init generic devices */
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_DEVICE);
     if (qemu_opts_foreach(qemu_find_opts("device"),
                           device_init_func, NULL, NULL)) {
         exit(1);
     }
+    rom_reset_order_override();
 
     /* Did we create any drives that we failed to create a device for? */
     drive_check_orphaned();
@@ -4540,12 +4550,7 @@ int main(int argc, char **argv, char **envp)
     net_check_clients();
 
     if (boot_once) {
-        Error *local_err = NULL;
-        qemu_boot_set(boot_once, &local_err);
-        if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            exit(1);
-        }
+        qemu_boot_set(boot_once, &error_fatal);
         qemu_register_reset(restore_boot_order, g_strdup(boot_order));
     }
 
@@ -4588,7 +4593,7 @@ int main(int argc, char **argv, char **envp)
                       vnc_init_func, NULL, NULL);
     if (show_vnc_port) {
         char *ret = vnc_display_local_addr("default");
-        printf("VNC server running on `%s'\n", ret);
+        printf("VNC server running on '%s'\n", ret);
         g_free(ret);
     }
 #endif
@@ -4610,10 +4615,16 @@ int main(int argc, char **argv, char **envp)
     qemu_run_machine_init_done_notifiers();
 
     if (rom_check_and_register_reset() != 0) {
-        fprintf(stderr, "rom check and register reset failed\n");
+        error_report("rom check and register reset failed");
         exit(1);
     }
 
+    replay_start();
+
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_RESET);
     qemu_system_reset(VMRESET_SILENT);
     register_global_state();
     if (loadvm) {
@@ -4633,9 +4644,7 @@ int main(int argc, char **argv, char **envp)
         Error *local_err = NULL;
         qemu_start_incoming_migration(incoming, &local_err);
         if (local_err) {
-            error_report("-incoming %s: %s", incoming,
-                         error_get_pretty(local_err));
-            error_free(local_err);
+            error_reportf_err(local_err, "-incoming %s: ", incoming);
             exit(1);
         }
     } else if (autostart) {
@@ -4644,13 +4653,9 @@ int main(int argc, char **argv, char **envp)
 
     os_setup_post();
 
-    if (is_daemonized()) {
-        if (!trace_init_backends(trace_events, trace_file)) {
-            exit(1);
-        }
-    }
-
     main_loop();
+    replay_disable_events();
+
     bdrv_close_all();
     pause_all_vcpus();
     res_free();
diff --git a/qemu/xen-common-stub.c b/qemu/xen-common-stub.c
index 906f991f1..699c3f1c6 100644
--- a/qemu/xen-common-stub.c
+++ b/qemu/xen-common-stub.c
@@ -5,6 +5,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/xen/xen.h"
 
diff --git a/qemu/xen-common.c b/qemu/xen-common.c
index 0dcdbc39f..e641ad1ae 100644
--- a/qemu/xen-common.c
+++ b/qemu/xen-common.c
@@ -8,6 +8,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "qmp-commands.h"
 #include "sysemu/char.h"
@@ -113,11 +114,17 @@ static void xen_change_state_handler(void *opaque, int running,
 
 static int xen_init(MachineState *ms)
 {
-    xen_xc = xen_xc_interface_open(0, 0, 0);
-    if (xen_xc == XC_HANDLER_INITIAL_VALUE) {
+    xen_xc = xc_interface_open(0, 0, 0);
+    if (xen_xc == NULL) {
         xen_be_printf(NULL, 0, "can't open xen interface\n");
         return -1;
     }
+    xen_fmem = xenforeignmemory_open(0, 0);
+    if (xen_fmem == NULL) {
+        xen_be_printf(NULL, 0, "can't open xen fmem interface\n");
+        xc_interface_close(xen_xc);
+        return -1;
+    }
     qemu_add_vm_change_state_handler(xen_change_state_handler, NULL);
 
     global_state_set_optional();
diff --git a/qemu/xen-hvm-stub.c b/qemu/xen-hvm-stub.c
index 46867d87d..c5003251c 100644
--- a/qemu/xen-hvm-stub.c
+++ b/qemu/xen-hvm-stub.c
@@ -8,6 +8,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/xen/xen.h"
 #include "exec/memory.h"
@@ -30,7 +31,13 @@ void xen_hvm_inject_msi(uint64_t addr, uint32_t data)
 {
 }
 
-void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
+int xen_is_pirq_msi(uint32_t msi_data)
+{
+    return 0;
+}
+
+void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
+                   Error **errp)
 {
 }
 
@@ -47,10 +54,8 @@ void xen_modified_memory(ram_addr_t start, ram_addr_t length)
 {
 }
 
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory)
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory)
 {
-    return 0;
 }
 
 void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
diff --git a/qemu/xen-hvm.c b/qemu/xen-hvm.c
index 040846236..039680a6d 100644
--- a/qemu/xen-hvm.c
+++ b/qemu/xen-hvm.c
@@ -8,15 +8,18 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include <sys/mman.h>
 
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
+#include "hw/i386/apic-msidef.h"
 #include "hw/xen/xen_common.h"
 #include "hw/xen/xen_backend.h"
 #include "qmp-commands.h"
 
 #include "sysemu/char.h"
+#include "qemu/error-report.h"
 #include "qemu/range.h"
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
@@ -64,17 +67,6 @@ struct shared_vmport_iopage {
 typedef struct shared_vmport_iopage shared_vmport_iopage_t;
 #endif
 
-#if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
-static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
-{
-    return shared_page->vcpu_iodata[i].vp_eport;
-}
-static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
-{
-    return &shared_page->vcpu_iodata[vcpu].vp_ioreq;
-}
-#  define FMT_ioreq_size PRIx64
-#else
 static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
 {
     return shared_page->vcpu_ioreq[i].vp_eport;
@@ -83,8 +75,6 @@ static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
 {
     return &shared_page->vcpu_ioreq[vcpu];
 }
-#  define FMT_ioreq_size "u"
-#endif
 
 #define BUFFER_IO_MAX_DELAY  100
 
@@ -109,7 +99,7 @@ typedef struct XenIOState {
     /* evtchn local port for buffered io */
     evtchn_port_t bufioreq_local_port;
     /* the evtchn fd for polling */
-    XenEvtchn xce_handle;
+    xenevtchn_handle *xce_handle;
     /* which vcpu we are serving */
     int send_vcpu;
 
@@ -156,9 +146,17 @@ void xen_piix_pci_write_config_client(uint32_t address, uint32_t val, int len)
     }
 }
 
+int xen_is_pirq_msi(uint32_t msi_data)
+{
+    /* If vector is 0, the msi is remapped into a pirq, passed as
+     * dest_id.
+     */
+    return ((msi_data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT) == 0;
+}
+
 void xen_hvm_inject_msi(uint64_t addr, uint32_t data)
 {
-    xen_xc_hvm_inject_msi(xen_xc, xen_domid, addr, data);
+    xc_hvm_inject_msi(xen_xc, xen_domid, addr, data);
 }
 
 static void xen_suspend_notifier(Notifier *notifier, void *data)
@@ -180,8 +178,7 @@ qemu_irq *xen_interrupt_controller_init(void)
 
 /* Memory Ops */
 
-static void xen_ram_init(ram_addr_t *below_4g_mem_size,
-                         ram_addr_t *above_4g_mem_size,
+static void xen_ram_init(PCMachineState *pcms,
                          ram_addr_t ram_size, MemoryRegion **ram_memory_p)
 {
     MemoryRegion *sysmem = get_system_memory();
@@ -198,23 +195,23 @@ static void xen_ram_init(ram_addr_t *below_4g_mem_size,
     }
 
     if (ram_size >= user_lowmem) {
-        *above_4g_mem_size = ram_size - user_lowmem;
-        *below_4g_mem_size = user_lowmem;
+        pcms->above_4g_mem_size = ram_size - user_lowmem;
+        pcms->below_4g_mem_size = user_lowmem;
     } else {
-        *above_4g_mem_size = 0;
-        *below_4g_mem_size = ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = ram_size;
     }
-    if (!*above_4g_mem_size) {
+    if (!pcms->above_4g_mem_size) {
         block_len = ram_size;
     } else {
         /*
          * Xen does not allocate the memory continuously, it keeps a
          * hole of the size computed above or passed in.
          */
-        block_len = (1ULL << 32) + *above_4g_mem_size;
+        block_len = (1ULL << 32) + pcms->above_4g_mem_size;
     }
     memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len,
-                           &error_abort);
+                           &error_fatal);
     *ram_memory_p = &ram_memory;
     vmstate_register_ram_global(&ram_memory);
 
@@ -229,17 +226,18 @@ static void xen_ram_init(ram_addr_t *below_4g_mem_size,
      */
     memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo",
                              &ram_memory, 0xc0000,
-                             *below_4g_mem_size - 0xc0000);
+                             pcms->below_4g_mem_size - 0xc0000);
     memory_region_add_subregion(sysmem, 0xc0000, &ram_lo);
-    if (*above_4g_mem_size > 0) {
+    if (pcms->above_4g_mem_size > 0) {
         memory_region_init_alias(&ram_hi, NULL, "xen.ram.hi",
                                  &ram_memory, 0x100000000ULL,
-                                 *above_4g_mem_size);
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(sysmem, 0x100000000ULL, &ram_hi);
     }
 }
 
-void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
+void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
+                   Error **errp)
 {
     unsigned long nr_pfn;
     xen_pfn_t *pfn_list;
@@ -267,7 +265,8 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
     }
 
     if (xc_domain_populate_physmap_exact(xen_xc, xen_domid, nr_pfn, 0, 0, pfn_list)) {
-        hw_error("xen: failed to populate ram at " RAM_ADDR_FMT, ram_addr);
+        error_setg(errp, "xen: failed to populate ram at " RAM_ADDR_FMT,
+                   ram_addr);
     }
 
     g_free(pfn_list);
@@ -304,7 +303,6 @@ static hwaddr xen_phys_offset_to_gaddr(hwaddr start_addr,
     return start_addr;
 }
 
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 340
 static int xen_add_to_physmap(XenIOState *state,
                               hwaddr start_addr,
                               ram_addr_t size,
@@ -345,10 +343,10 @@ go_physmap:
         unsigned long idx = pfn + i;
         xen_pfn_t gpfn = start_gpfn + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             DPRINTF("add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -422,10 +420,10 @@ static int xen_remove_from_physmap(XenIOState *state,
         xen_pfn_t idx = start_addr + i;
         xen_pfn_t gpfn = phys_offset + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             fprintf(stderr, "add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -439,24 +437,6 @@ static int xen_remove_from_physmap(XenIOState *state,
     return 0;
 }
 
-#else
-static int xen_add_to_physmap(XenIOState *state,
-                              hwaddr start_addr,
-                              ram_addr_t size,
-                              MemoryRegion *mr,
-                              hwaddr offset_within_region)
-{
-    return -ENOSYS;
-}
-
-static int xen_remove_from_physmap(XenIOState *state,
-                                   hwaddr start_addr,
-                                   ram_addr_t size)
-{
-    return -ENOSYS;
-}
-#endif
-
 static void xen_set_memory(struct MemoryListener *listener,
                            MemoryRegionSection *section,
                            bool add)
@@ -695,7 +675,7 @@ static ioreq_t *cpu_get_ioreq_from_shared_memory(XenIOState *state, int vcpu)
     if (req->state != STATE_IOREQ_READY) {
         DPRINTF("I/O request not ready: "
                 "%x, ptr: %x, port: %"PRIx64", "
-                "data: %"PRIx64", count: %" FMT_ioreq_size ", size: %" FMT_ioreq_size "\n",
+                "data: %"PRIx64", count: %u, size: %u\n",
                 req->state, req->data_is_ptr, req->addr,
                 req->data, req->count, req->size);
         return NULL;
@@ -715,7 +695,7 @@ static ioreq_t *cpu_get_ioreq(XenIOState *state)
     int i;
     evtchn_port_t port;
 
-    port = xc_evtchn_pending(state->xce_handle);
+    port = xenevtchn_pending(state->xce_handle);
     if (port == state->bufioreq_local_port) {
         timer_mod(state->buffered_io_timer,
                 BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
@@ -734,7 +714,7 @@ static ioreq_t *cpu_get_ioreq(XenIOState *state)
         }
 
         /* unmask the wanted port again */
-        xc_evtchn_unmask(state->xce_handle, port);
+        xenevtchn_unmask(state->xce_handle, port);
 
         /* get the io packet from shared memory */
         state->send_vcpu = i;
@@ -814,9 +794,14 @@ static void cpu_ioreq_pio(ioreq_t *req)
 {
     uint32_t i;
 
+    trace_cpu_ioreq_pio(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (req->dir == IOREQ_READ) {
         if (!req->data_is_ptr) {
             req->data = do_inp(req->addr, req->size);
+            trace_cpu_ioreq_pio_read_reg(req, req->data, req->addr,
+                                         req->size);
         } else {
             uint32_t tmp;
 
@@ -827,6 +812,8 @@ static void cpu_ioreq_pio(ioreq_t *req)
         }
     } else if (req->dir == IOREQ_WRITE) {
         if (!req->data_is_ptr) {
+            trace_cpu_ioreq_pio_write_reg(req, req->data, req->addr,
+                                          req->size);
             do_outp(req->addr, req->size, req->data);
         } else {
             for (i = 0; i < req->count; i++) {
@@ -843,6 +830,9 @@ static void cpu_ioreq_move(ioreq_t *req)
 {
     uint32_t i;
 
+    trace_cpu_ioreq_move(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (!req->data_is_ptr) {
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
@@ -915,11 +905,18 @@ static void handle_vmport_ioreq(XenIOState *state, ioreq_t *req)
 
 static void handle_ioreq(XenIOState *state, ioreq_t *req)
 {
+    trace_handle_ioreq(req, req->type, req->dir, req->df, req->data_is_ptr,
+                       req->addr, req->data, req->count, req->size);
+
     if (!req->data_is_ptr && (req->dir == IOREQ_WRITE) &&
             (req->size < sizeof (target_ulong))) {
         req->data &= ((target_ulong) 1 << (8 * req->size)) - 1;
     }
 
+    if (req->dir == IOREQ_WRITE)
+        trace_handle_ioreq_write(req, req->type, req->df, req->data_is_ptr,
+                                 req->addr, req->data, req->count, req->size);
+
     switch (req->type) {
         case IOREQ_TYPE_PIO:
             cpu_ioreq_pio(req);
@@ -959,23 +956,38 @@ static void handle_ioreq(XenIOState *state, ioreq_t *req)
         default:
             hw_error("Invalid ioreq type 0x%x\n", req->type);
     }
+    if (req->dir == IOREQ_READ) {
+        trace_handle_ioreq_read(req, req->type, req->df, req->data_is_ptr,
+                                req->addr, req->data, req->count, req->size);
+    }
 }
 
 static int handle_buffered_iopage(XenIOState *state)
 {
+    buffered_iopage_t *buf_page = state->buffered_io_page;
     buf_ioreq_t *buf_req = NULL;
     ioreq_t req;
     int qw;
 
-    if (!state->buffered_io_page) {
+    if (!buf_page) {
         return 0;
     }
 
     memset(&req, 0x00, sizeof(req));
 
-    while (state->buffered_io_page->read_pointer != state->buffered_io_page->write_pointer) {
-        buf_req = &state->buffered_io_page->buf_ioreq[
-            state->buffered_io_page->read_pointer % IOREQ_BUFFER_SLOT_NUM];
+    for (;;) {
+        uint32_t rdptr = buf_page->read_pointer, wrptr;
+
+        xen_rmb();
+        wrptr = buf_page->write_pointer;
+        xen_rmb();
+        if (rdptr != buf_page->read_pointer) {
+            continue;
+        }
+        if (rdptr == wrptr) {
+            break;
+        }
+        buf_req = &buf_page->buf_ioreq[rdptr % IOREQ_BUFFER_SLOT_NUM];
         req.size = 1UL << buf_req->size;
         req.count = 1;
         req.addr = buf_req->addr;
@@ -987,15 +999,14 @@ static int handle_buffered_iopage(XenIOState *state)
         req.data_is_ptr = 0;
         qw = (req.size == 8);
         if (qw) {
-            buf_req = &state->buffered_io_page->buf_ioreq[
-                (state->buffered_io_page->read_pointer + 1) % IOREQ_BUFFER_SLOT_NUM];
+            buf_req = &buf_page->buf_ioreq[(rdptr + 1) %
+                                           IOREQ_BUFFER_SLOT_NUM];
             req.data |= ((uint64_t)buf_req->data) << 32;
         }
 
         handle_ioreq(state, &req);
 
-        xen_mb();
-        state->buffered_io_page->read_pointer += qw ? 2 : 1;
+        atomic_add(&buf_page->read_pointer, qw + 1);
     }
 
     return req.count;
@@ -1010,7 +1021,7 @@ static void handle_buffered_io(void *opaque)
                 BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
     } else {
         timer_del(state->buffered_io_timer);
-        xc_evtchn_unmask(state->xce_handle, state->bufioreq_local_port);
+        xenevtchn_unmask(state->xce_handle, state->bufioreq_local_port);
     }
 }
 
@@ -1026,9 +1037,7 @@ static void cpu_handle_ioreq(void *opaque)
         if (req->state != STATE_IOREQ_INPROCESS) {
             fprintf(stderr, "Badness in I/O request ... not in service?!: "
                     "%x, ptr: %x, port: %"PRIx64", "
-                    "data: %"PRIx64", count: %" FMT_ioreq_size
-                    ", size: %" FMT_ioreq_size
-                    ", type: %"FMT_ioreq_size"\n",
+                    "data: %"PRIx64", count: %u, size: %u, type: %u\n",
                     req->state, req->data_is_ptr, req->addr,
                     req->data, req->count, req->size, req->type);
             destroy_hvm_domain(false);
@@ -1054,7 +1063,8 @@ static void cpu_handle_ioreq(void *opaque)
         }
 
         req->state = STATE_IORESP_READY;
-        xc_evtchn_notify(state->xce_handle, state->ioreq_local_port[state->send_vcpu]);
+        xenevtchn_notify(state->xce_handle,
+                         state->ioreq_local_port[state->send_vcpu]);
     }
 }
 
@@ -1062,8 +1072,8 @@ static void xen_main_loop_prepare(XenIOState *state)
 {
     int evtchn_fd = -1;
 
-    if (state->xce_handle != XC_HANDLER_INITIAL_VALUE) {
-        evtchn_fd = xc_evtchn_fd(state->xce_handle);
+    if (state->xce_handle != NULL) {
+        evtchn_fd = xenevtchn_fd(state->xce_handle);
     }
 
     state->buffered_io_timer = timer_new_ms(QEMU_CLOCK_REALTIME, handle_buffered_io,
@@ -1101,7 +1111,7 @@ static void xen_exit_notifier(Notifier *n, void *data)
 {
     XenIOState *state = container_of(n, XenIOState, exit);
 
-    xc_evtchn_close(state->xce_handle);
+    xenevtchn_close(state->xce_handle);
     xs_daemon_close(state->xenstore);
 }
 
@@ -1158,9 +1168,7 @@ static void xen_wakeup_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
 }
 
-/* return 0 means OK, or -1 means critical issue -- will exit(1) */
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory)
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory)
 {
     int i, rc;
     xen_pfn_t ioreq_pfn;
@@ -1170,22 +1178,22 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
 
     state = g_malloc0(sizeof (XenIOState));
 
-    state->xce_handle = xen_xc_evtchn_open(NULL, 0);
-    if (state->xce_handle == XC_HANDLER_INITIAL_VALUE) {
+    state->xce_handle = xenevtchn_open(NULL, 0);
+    if (state->xce_handle == NULL) {
         perror("xen: event channel open");
-        return -1;
+        goto err;
     }
 
     state->xenstore = xs_daemon_open();
     if (state->xenstore == NULL) {
         perror("xen: xenstore open");
-        return -1;
+        goto err;
     }
 
     rc = xen_create_ioreq_server(xen_xc, xen_domid, &state->ioservid);
     if (rc < 0) {
         perror("xen: ioreq server create");
-        return -1;
+        goto err;
     }
 
     state->exit.notify = xen_exit_notifier;
@@ -1201,41 +1209,47 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
                                    &ioreq_pfn, &bufioreq_pfn,
                                    &bufioreq_evtchn);
     if (rc < 0) {
-        hw_error("failed to get ioreq server info: error %d handle=" XC_INTERFACE_FMT,
-                 errno, xen_xc);
+        error_report("failed to get ioreq server info: error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
     }
 
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     DPRINTF("buffered io page at pfn %lx\n", bufioreq_pfn);
     DPRINTF("buffered io evtchn is %x\n", bufioreq_evtchn);
 
-    state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-                                              PROT_READ|PROT_WRITE, ioreq_pfn);
+    state->shared_page = xenforeignmemory_map(xen_fmem, xen_domid,
+                                              PROT_READ|PROT_WRITE,
+                                              1, &ioreq_pfn, NULL);
     if (state->shared_page == NULL) {
-        hw_error("map shared IO page returned error %d handle=" XC_INTERFACE_FMT,
-                 errno, xen_xc);
+        error_report("map shared IO page returned error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
     }
 
     rc = xen_get_vmport_regs_pfn(xen_xc, xen_domid, &ioreq_pfn);
     if (!rc) {
         DPRINTF("shared vmport page at pfn %lx\n", ioreq_pfn);
         state->shared_vmport_page =
-            xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-                                 PROT_READ|PROT_WRITE, ioreq_pfn);
+            xenforeignmemory_map(xen_fmem, xen_domid, PROT_READ|PROT_WRITE,
+                                 1, &ioreq_pfn, NULL);
         if (state->shared_vmport_page == NULL) {
-            hw_error("map shared vmport IO page returned error %d handle="
-                     XC_INTERFACE_FMT, errno, xen_xc);
+            error_report("map shared vmport IO page returned error %d handle=%p",
+                         errno, xen_xc);
+            goto err;
         }
     } else if (rc != -ENOSYS) {
-        hw_error("get vmport regs pfn returned error %d, rc=%d", errno, rc);
+        error_report("get vmport regs pfn returned error %d, rc=%d",
+                     errno, rc);
+        goto err;
     }
 
-    state->buffered_io_page = xc_map_foreign_range(xen_xc, xen_domid,
-                                                   XC_PAGE_SIZE,
+    state->buffered_io_page = xenforeignmemory_map(xen_fmem, xen_domid,
                                                    PROT_READ|PROT_WRITE,
-                                                   bufioreq_pfn);
+                                                   1, &bufioreq_pfn, NULL);
     if (state->buffered_io_page == NULL) {
-        hw_error("map buffered IO page returned error %d", errno);
+        error_report("map buffered IO page returned error %d", errno);
+        goto err;
     }
 
     /* Note: cpus is empty at this point in init */
@@ -1243,34 +1257,35 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
 
     rc = xen_set_ioreq_server_state(xen_xc, xen_domid, state->ioservid, true);
     if (rc < 0) {
-        hw_error("failed to enable ioreq server info: error %d handle=" XC_INTERFACE_FMT,
-                 errno, xen_xc);
+        error_report("failed to enable ioreq server info: error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
     }
 
     state->ioreq_local_port = g_malloc0(max_cpus * sizeof (evtchn_port_t));
 
     /* FIXME: how about if we overflow the page here? */
     for (i = 0; i < max_cpus; i++) {
-        rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
+        rc = xenevtchn_bind_interdomain(state->xce_handle, xen_domid,
                                         xen_vcpu_eport(state->shared_page, i));
         if (rc == -1) {
-            fprintf(stderr, "shared evtchn %d bind error %d\n", i, errno);
-            return -1;
+            error_report("shared evtchn %d bind error %d", i, errno);
+            goto err;
         }
         state->ioreq_local_port[i] = rc;
     }
 
-    rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
+    rc = xenevtchn_bind_interdomain(state->xce_handle, xen_domid,
                                     bufioreq_evtchn);
     if (rc == -1) {
-        fprintf(stderr, "buffered evtchn bind error %d\n", errno);
-        return -1;
+        error_report("buffered evtchn bind error %d", errno);
+        goto err;
     }
     state->bufioreq_local_port = rc;
 
     /* Init RAM management */
     xen_map_cache_init(xen_phys_offset_to_gaddr, state);
-    xen_ram_init(below_4g_mem_size, above_4g_mem_size, ram_size, ram_memory);
+    xen_ram_init(pcms, ram_size, ram_memory);
 
     qemu_add_vm_change_state_handler(xen_hvm_change_state_handler, state);
 
@@ -1287,24 +1302,27 @@ int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
 
     /* Initialize backend core & drivers */
     if (xen_be_init() != 0) {
-        fprintf(stderr, "%s: xen backend core setup failed\n", __FUNCTION__);
-        return -1;
+        error_report("xen backend core setup failed");
+        goto err;
     }
     xen_be_register("console", &xen_console_ops);
     xen_be_register("vkbd", &xen_kbdmouse_ops);
     xen_be_register("qdisk", &xen_blkdev_ops);
     xen_read_physmap(state);
+    return;
 
-    return 0;
+err:
+    error_report("xen hardware virtual machine initialisation failed");
+    exit(1);
 }
 
 void destroy_hvm_domain(bool reboot)
 {
-    XenXC xc_handle;
+    xc_interface *xc_handle;
     int sts;
 
-    xc_handle = xen_xc_interface_open(0, 0, 0);
-    if (xc_handle == XC_HANDLER_INITIAL_VALUE) {
+    xc_handle = xc_interface_open(0, 0, 0);
+    if (xc_handle == NULL) {
         fprintf(stderr, "Cannot acquire xenctrl handle\n");
     } else {
         sts = xc_domain_shutdown(xc_handle, xen_domid,
diff --git a/qemu/xen-mapcache.c b/qemu/xen-mapcache.c
index 8cefd0c62..49f394a77 100644
--- a/qemu/xen-mapcache.c
+++ b/qemu/xen-mapcache.c
@@ -8,7 +8,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 
 #include <sys/resource.h>
 
@@ -169,19 +169,17 @@ static void xen_remap_bucket(MapCacheEntry *entry,
             exit(-1);
         }
     }
-    if (entry->valid_mapping != NULL) {
-        g_free(entry->valid_mapping);
-        entry->valid_mapping = NULL;
-    }
+    g_free(entry->valid_mapping);
+    entry->valid_mapping = NULL;
 
     for (i = 0; i < nb_pfn; i++) {
         pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
     }
 
-    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
-                                     pfns, err, nb_pfn);
+    vaddr_base = xenforeignmemory_map(xen_fmem, xen_domid, PROT_READ|PROT_WRITE,
+                                      nb_pfn, pfns, err);
     if (vaddr_base == NULL) {
-        perror("xc_map_foreign_bulk");
+        perror("xenforeignmemory_map");
         exit(-1);
     }